diff --git a/librocksdb-sys/rocksdb/.circleci/config.yml b/librocksdb-sys/rocksdb/.circleci/config.yml new file mode 100644 index 0000000..c614d3f --- /dev/null +++ b/librocksdb-sys/rocksdb/.circleci/config.yml @@ -0,0 +1,892 @@ +version: 2.1 + +orbs: + win: circleci/windows@5.0.0 + +commands: + install-cmake-on-macos: + steps: + - run: + name: Install cmake on macos + command: | + HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake + + install-jdk8-on-macos: + steps: + - run: + name: Install JDK 8 on macos + command: | + brew install --cask adoptopenjdk/openjdk/adoptopenjdk8 + + increase-max-open-files-on-macos: + steps: + - run: + name: Increase max open files + command: | + sudo sysctl -w kern.maxfiles=1048576 + sudo sysctl -w kern.maxfilesperproc=1048576 + sudo launchctl limit maxfiles 1048576 + + pre-steps: + steps: + - checkout + - run: + name: Setup Environment Variables + command: | + echo "export GTEST_THROW_ON_FAILURE=0" >> $BASH_ENV + echo "export GTEST_OUTPUT=\"xml:/tmp/test-results/\"" >> $BASH_ENV + echo "export SKIP_FORMAT_BUCK_CHECKS=1" >> $BASH_ENV + echo "export GTEST_COLOR=1" >> $BASH_ENV + echo "export CTEST_OUTPUT_ON_FAILURE=1" >> $BASH_ENV + echo "export CTEST_TEST_TIMEOUT=300" >> $BASH_ENV + echo "export ZLIB_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib" >> $BASH_ENV + echo "export BZIP2_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2" >> $BASH_ENV + echo "export SNAPPY_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy" >> $BASH_ENV + echo "export LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4" >> $BASH_ENV + echo "export ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd" >> $BASH_ENV + + windows-build-steps: + steps: + - checkout + - run: + name: "Install thirdparty dependencies" + command: | + echo "Installing CMake..." + choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' -y + mkdir $Env:THIRDPARTY_HOME + cd $Env:THIRDPARTY_HOME + echo "Building Snappy dependency..." + curl https://github.com/google/snappy/archive/refs/tags/1.1.8.zip -O snappy-1.1.8.zip + unzip -q snappy-1.1.8.zip + cd snappy-1.1.8 + mkdir build + cd build + & $Env:CMAKE_BIN -G "$Env:CMAKE_GENERATOR" .. + msbuild.exe Snappy.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64 + - run: + name: "Build RocksDB" + command: | + mkdir build + cd build + & $Env:CMAKE_BIN -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 .. + cd .. + echo "Building with VS version: $Env:CMAKE_GENERATOR" + msbuild.exe build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64 + - run: + name: "Test RocksDB" + shell: powershell.exe + command: | + build_tools\run_ci_db_test.ps1 -SuiteRun arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16 + pre-steps-macos: + steps: + - pre-steps + + post-steps: + steps: + - store_test_results: # store test result if there's any + path: /tmp/test-results + - store_artifacts: # store LOG for debugging if there's any + path: LOG + - run: # on fail, compress Test Logs for diagnosing the issue + name: Compress Test Logs + command: tar -cvzf t.tar.gz t + when: on_fail + - store_artifacts: # on fail, store Test Logs for diagnosing the issue + path: t.tar.gz + destination: test_logs + when: on_fail + - run: # store core dumps if there's any + command: | + mkdir -p /tmp/core_dumps + cp core.* /tmp/core_dumps + when: on_fail + - store_artifacts: + path: /tmp/core_dumps + when: on_fail + + upgrade-cmake: + steps: + - run: + name: Upgrade cmake + command: | + sudo apt remove --purge cmake + sudo snap install cmake --classic + + install-gflags: + steps: + - run: + name: Install gflags + command: | + sudo apt-get update -y && sudo apt-get install -y libgflags-dev + + install-gflags-on-macos: + steps: + - run: + name: Install gflags on macos + command: | + HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags + + setup-folly: + steps: + - run: + name: Checkout folly sources + command: | + make checkout_folly + + build-folly: + steps: + - run: + name: Build folly and dependencies + command: | + make build_folly + + build-for-benchmarks: + steps: + - pre-steps + - run: + name: "Linux build for benchmarks" + command: #sized for the resource-class rocksdb-benchmark-sys1 + make V=1 J=8 -j8 release + + perform-benchmarks: + steps: + - run: + name: "Test low-variance benchmarks" + command: ./tools/benchmark_ci.py --db_dir /tmp/rocksdb-benchmark-datadir --output_dir /tmp/benchmark-results --num_keys 20000000 + environment: + LD_LIBRARY_PATH: /usr/local/lib + # How long to run parts of the test(s) + DURATION_RO: 300 + DURATION_RW: 500 + # Keep threads within physical capacity of server (much lower than default) + NUM_THREADS: 1 + MAX_BACKGROUND_JOBS: 4 + # Don't run a couple of "optional" initial tests + CI_TESTS_ONLY: "true" + # Reduce configured size of levels to ensure more levels in the leveled compaction LSM tree + WRITE_BUFFER_SIZE_MB: 16 + TARGET_FILE_SIZE_BASE_MB: 16 + MAX_BYTES_FOR_LEVEL_BASE_MB: 64 + # The benchmark host has 32GB memory + # The following values are tailored to work with that + # Note, tests may not exercise the targeted issues if the memory is increased on new test hosts. + COMPRESSION_TYPE: "none" + CACHE_INDEX_AND_FILTER_BLOCKS: 1 + MIN_LEVEL_TO_COMPRESS: 3 + CACHE_SIZE_MB: 10240 + MB_WRITE_PER_SEC: 2 + + post-benchmarks: + steps: + - store_artifacts: # store the benchmark output + path: /tmp/benchmark-results + destination: test_logs + - run: + name: Send benchmark report to visualisation + command: | + set +e + set +o pipefail + ./build_tools/benchmark_log_tool.py --tsvfile /tmp/benchmark-results/report.tsv --esdocument https://search-rocksdb-bench-k2izhptfeap2hjfxteolsgsynm.us-west-2.es.amazonaws.com/bench_test3_rix/_doc + true + +executors: + linux-docker: + docker: + # The image configuration is build_tools/ubuntu20_image/Dockerfile + # To update and build the image: + # $ cd build_tools/ubuntu20_image + # $ docker build -t zjay437/rocksdb:0.5 . + # $ docker push zjay437/rocksdb:0.5 + # `zjay437` is the account name for zjay@meta.com which readwrite token is shared internally. To login: + # $ docker login --username zjay437 + # Or please feel free to change it to your docker hub account for hosting the image, meta employee should already have the account and able to login with SSO. + # To avoid impacting the existing CI runs, please bump the version every time creating a new image + # to run the CI image environment locally: + # $ docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -it zjay437/rocksdb:0.5 bash + # option `--cap-add=SYS_PTRACE --security-opt seccomp=unconfined` is used to enable gdb to attach an existing process + - image: zjay437/rocksdb:0.6 + +jobs: + build-macos: + macos: + xcode: 12.5.1 + resource_class: large + environment: + ROCKSDB_DISABLE_JEMALLOC: 1 # jemalloc cause env_test hang, disable it for now + steps: + - increase-max-open-files-on-macos + - install-gflags-on-macos + - pre-steps-macos + - run: ulimit -S -n `ulimit -H -n` && OPT=-DCIRCLECI make V=1 J=32 -j32 all + - post-steps + + build-macos-cmake: + macos: + xcode: 12.5.1 + resource_class: large + parameters: + run_even_tests: + description: run even or odd tests, used to split tests to 2 groups + type: boolean + default: true + steps: + - increase-max-open-files-on-macos + - install-cmake-on-macos + - install-gflags-on-macos + - pre-steps-macos + - run: + name: "cmake generate project file" + command: ulimit -S -n `ulimit -H -n` && mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. + - run: + name: "Build tests" + command: cd build && make V=1 -j32 + - when: + condition: << parameters.run_even_tests >> + steps: + - run: + name: "Run even tests" + command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j32 -I 0,,2 + - when: + condition: + not: << parameters.run_even_tests >> + steps: + - run: + name: "Run odd tests" + command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j32 -I 1,,2 + - post-steps + + build-linux: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - run: make V=1 J=32 -j32 check + - post-steps + + build-linux-encrypted_env-no_compression: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - run: ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=32 -j32 check + - run: | + ./sst_dump --help | grep -E -q 'Supported compression types: kNoCompression$' # Verify no compiled in compression + - post-steps + + build-linux-static_lib-alt_namespace-status_checked: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=static OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j24 check + - post-steps + + build-linux-release: + executor: linux-docker + resource_class: 2xlarge + steps: + - checkout # check out the code in the project directory + - run: make V=1 -j32 LIB_MODE=shared release + - run: ls librocksdb.so # ensure shared lib built + - run: ./db_stress --version # ensure with gflags + - run: make clean + - run: make V=1 -j32 release + - run: ls librocksdb.a # ensure static lib built + - run: ./db_stress --version # ensure with gflags + - run: make clean + - run: apt-get remove -y libgflags-dev + - run: make V=1 -j32 LIB_MODE=shared release + - run: ls librocksdb.so # ensure shared lib built + - run: if ./db_stress --version; then false; else true; fi # ensure without gflags + - run: make clean + - run: make V=1 -j32 release + - run: ls librocksdb.a # ensure static lib built + - run: if ./db_stress --version; then false; else true; fi # ensure without gflags + - post-steps + + build-linux-release-rtti: + executor: linux-docker + resource_class: xlarge + steps: + - checkout # check out the code in the project directory + - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench + - run: ./db_stress --version # ensure with gflags + - run: make clean + - run: apt-get remove -y libgflags-dev + - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench + - run: if ./db_stress --version; then false; else true; fi # ensure without gflags + + build-linux-clang-no_test_run: + executor: linux-docker + resource_class: xlarge + steps: + - checkout # check out the code in the project directory + - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all + - post-steps + + build-linux-clang10-asan: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check # aligned new doesn't work for reason we haven't figured out + - post-steps + + build-linux-clang10-mini-tsan: + executor: linux-docker + resource_class: 2xlarge+ + steps: + - pre-steps + - run: COMPILE_WITH_TSAN=1 CC=clang-13 CXX=clang++-13 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check + - post-steps + + build-linux-clang10-ubsan: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check # aligned new doesn't work for reason we haven't figured out + - post-steps + + build-linux-valgrind: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - run: PORTABLE=1 make V=1 -j32 valgrind_test + - post-steps + + build-linux-clang10-clang-analyze: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path. + - post-steps + - run: + name: "compress test report" + command: tar -cvzf scan_build_report.tar.gz scan_build_report + when: on_fail + - store_artifacts: + path: scan_build_report.tar.gz + destination: scan_build_report + when: on_fail + + build-linux-runner: + machine: true + resource_class: facebook/rocksdb-benchmark-sys1 + steps: + - pre-steps + - run: + name: "Checked Linux build (Runner)" + command: make V=1 J=8 -j8 check + environment: + LD_LIBRARY_PATH: /usr/local/lib + - post-steps + + build-linux-cmake-with-folly: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - setup-folly + - build-folly + - run: (mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20) + - post-steps + + build-linux-cmake-with-folly-lite-no-test: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - setup-folly + - run: (mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20) + - post-steps + + build-linux-cmake-with-benchmark: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - run: mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20 + - post-steps + + build-linux-unity-and-headers: + docker: # executor type + - image: gcc:latest + environment: + EXTRA_CXXFLAGS: -mno-avx512f # Warnings-as-error in avx512fintrin.h, would be used on newer hardware + resource_class: large + steps: + - checkout # check out the code in the project directory + - run: apt-get update -y && apt-get install -y libgflags-dev + - run: + name: "Unity build" + command: make V=1 -j8 unity_test + no_output_timeout: 20m + - run: make V=1 -j8 -k check-headers # could be moved to a different build + - post-steps + + build-linux-gcc-7-with-folly: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - setup-folly + - build-folly + - run: USE_FOLLY=1 LIB_MODE=static CC=gcc-7 CXX=g++-7 V=1 make -j32 check # TODO: LIB_MODE only to work around unresolved linker failures + - post-steps + + build-linux-gcc-7-with-folly-lite-no-test: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - setup-folly + - run: USE_FOLLY_LITE=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 all + - post-steps + + build-linux-gcc-8-no_test_run: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - run: CC=gcc-8 CXX=g++-8 V=1 make -j32 all + - post-steps + + build-linux-cmake-with-folly-coroutines: + executor: linux-docker + resource_class: 2xlarge + environment: + CC: gcc-10 + CXX: g++-10 + steps: + - pre-steps + - setup-folly + - build-folly + - run: (mkdir build && cd build && cmake -DUSE_COROUTINES=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20) + - post-steps + + build-linux-gcc-10-cxx20-no_test_run: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - run: CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j32 all + - post-steps + + build-linux-gcc-11-no_test_run: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench # TODO: LIB_MODE only to work around unresolved linker failures + - post-steps + + build-linux-clang-13-no_test_run: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j32 all microbench + - post-steps + + # Ensure ASAN+UBSAN with folly, and full testsuite with clang 13 + build-linux-clang-13-asan-ubsan-with-folly: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - setup-folly + - build-folly + - run: CC=clang-13 CXX=clang++-13 LIB_MODE=static USE_CLANG=1 USE_FOLLY=1 COMPILE_WITH_UBSAN=1 COMPILE_WITH_ASAN=1 make -j32 check # TODO: LIB_MODE only to work around unresolved linker failures + - post-steps + + # This job is only to make sure the microbench tests are able to run, the benchmark result is not meaningful as the CI host is changing. + build-linux-run-microbench: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - run: DEBUG_LEVEL=0 make -j32 run_microbench + - post-steps + + build-linux-mini-crashtest: + executor: linux-docker + resource_class: large + steps: + - pre-steps + - run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=960 --max_key=2500000 --use_io_uring=0' blackbox_crash_test_with_atomic_flush + - post-steps + + build-linux-crashtest-tiered-storage-bb: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - run: + name: "run crashtest" + command: ulimit -S -n `ulimit -H -n` && make V=1 -j32 CRASH_TEST_EXT_ARGS='--duration=10800 --use_io_uring=0' blackbox_crash_test_with_tiered_storage + no_output_timeout: 100m + - post-steps + + build-linux-crashtest-tiered-storage-wb: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - run: + name: "run crashtest" + command: ulimit -S -n `ulimit -H -n` && make V=1 -j32 CRASH_TEST_EXT_ARGS='--duration=10800 --use_io_uring=0' whitebox_crash_test_with_tiered_storage + no_output_timeout: 100m + - post-steps + + build-windows-vs2022: + executor: + name: win/server-2022 + size: 2xlarge + environment: + THIRDPARTY_HOME: C:/Users/circleci/thirdparty + CMAKE_HOME: C:/Program Files/CMake + CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe + SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.8 + SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.8;C:/Users/circleci/thirdparty/snappy-1.1.8/build + SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.8/build/Debug/snappy.lib + CMAKE_GENERATOR: Visual Studio 17 2022 + steps: + - windows-build-steps + + build-windows-vs2019: + executor: + name: win/server-2019 + size: 2xlarge + environment: + THIRDPARTY_HOME: C:/Users/circleci/thirdparty + CMAKE_HOME: C:/Program Files/CMake + CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe + SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.8 + SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.8;C:/Users/circleci/thirdparty/snappy-1.1.8/build + SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.8/build/Debug/snappy.lib + CMAKE_GENERATOR: Visual Studio 16 2019 + steps: + - windows-build-steps + + build-linux-java: + executor: linux-docker + resource_class: large + steps: + - pre-steps + - run: + name: "Set Java Environment" + command: | + echo "JAVA_HOME=${JAVA_HOME}" + echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV + which java && java -version + which javac && javac -version + - run: + name: "Test RocksDBJava" + command: make V=1 J=8 -j8 jtest + - post-steps + + build-linux-java-static: + executor: linux-docker + resource_class: large + steps: + - pre-steps + - run: + name: "Set Java Environment" + command: | + echo "JAVA_HOME=${JAVA_HOME}" + echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV + which java && java -version + which javac && javac -version + - run: + name: "Build RocksDBJava Static Library" + command: make V=1 J=8 -j8 rocksdbjavastatic + - post-steps + + build-macos-java: + macos: + xcode: 12.5.1 + resource_class: large + environment: + JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home + ROCKSDB_DISABLE_JEMALLOC: 1 # jemalloc causes java 8 crash + steps: + - increase-max-open-files-on-macos + - install-gflags-on-macos + - install-jdk8-on-macos + - pre-steps-macos + - run: + name: "Set Java Environment" + command: | + echo "JAVA_HOME=${JAVA_HOME}" + echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV + which java && java -version + which javac && javac -version + - run: + name: "Test RocksDBJava" + command: make V=1 J=16 -j16 jtest + no_output_timeout: 20m + - post-steps + + build-macos-java-static: + macos: + xcode: 12.5.1 + resource_class: large + environment: + JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home + steps: + - increase-max-open-files-on-macos + - install-gflags-on-macos + - install-cmake-on-macos + - install-jdk8-on-macos + - pre-steps-macos + - run: + name: "Set Java Environment" + command: | + echo "JAVA_HOME=${JAVA_HOME}" + echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV + which java && java -version + which javac && javac -version + - run: + name: "Build RocksDBJava x86 and ARM Static Libraries" + command: make V=1 J=16 -j16 rocksdbjavastaticosx + no_output_timeout: 20m + - post-steps + + build-macos-java-static-universal: + macos: + xcode: 12.5.1 + resource_class: large + environment: + JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home + steps: + - increase-max-open-files-on-macos + - install-gflags-on-macos + - install-cmake-on-macos + - install-jdk8-on-macos + - pre-steps-macos + - run: + name: "Set Java Environment" + command: | + echo "JAVA_HOME=${JAVA_HOME}" + echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV + which java && java -version + which javac && javac -version + - run: + name: "Build RocksDBJava Universal Binary Static Library" + command: make V=1 J=16 -j16 rocksdbjavastaticosx_ub + no_output_timeout: 20m + - post-steps + + build-examples: + executor: linux-docker + resource_class: large + steps: + - pre-steps + - run: + name: "Build examples" + command: | + make V=1 -j4 static_lib && cd examples && make V=1 -j4 + - post-steps + + build-cmake-mingw: + executor: linux-docker + resource_class: large + steps: + - pre-steps + - run: update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix + - run: + name: "Build cmake-mingw" + command: | + export PATH=$JAVA_HOME/bin:$PATH + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni + - post-steps + + build-linux-non-shm: + executor: linux-docker + resource_class: 2xlarge + environment: + TEST_TMPDIR: /tmp/rocksdb_test_tmp + steps: + - pre-steps + - run: make V=1 -j32 check + - post-steps + + build-linux-arm-test-full: + machine: + image: ubuntu-2004:202111-02 + resource_class: arm.large + steps: + - pre-steps + - install-gflags + - run: make V=1 J=4 -j4 check + - post-steps + + build-linux-arm: + machine: + image: ubuntu-2004:202111-02 + resource_class: arm.large + steps: + - pre-steps + - install-gflags + - run: ROCKSDBTESTS_PLATFORM_DEPENDENT=only make V=1 J=4 -j4 all_but_some_tests check_some + - post-steps + + build-linux-arm-cmake-no_test_run: + machine: + image: ubuntu-2004:202111-02 + resource_class: arm.large + environment: + JAVA_HOME: /usr/lib/jvm/java-8-openjdk-arm64 + steps: + - pre-steps + - install-gflags + - run: + name: "Set Java Environment" + command: | + echo "JAVA_HOME=${JAVA_HOME}" + echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV + which java && java -version + which javac && javac -version + - run: + name: "Build with cmake" + command: | + mkdir build + cd build + cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=1 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 .. + make -j4 + - run: + name: "Build Java with cmake" + command: | + rm -rf build + mkdir build + cd build + cmake -DJNI=1 -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=1 .. + make -j4 rocksdb rocksdbjni + - post-steps + + build-format-compatible: + executor: linux-docker + resource_class: 2xlarge + steps: + - pre-steps + - run: + name: "test" + command: | + export TEST_TMPDIR=/dev/shm/rocksdb + rm -rf /dev/shm/rocksdb + mkdir /dev/shm/rocksdb + tools/check_format_compatible.sh + - post-steps + + build-fuzzers: + executor: linux-docker + resource_class: large + steps: + - pre-steps + - run: + name: "Build rocksdb lib" + command: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib + - run: + name: "Build fuzzers" + command: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer + - post-steps + + benchmark-linux: #use a private Circle CI runner (resource_class) to run the job + machine: true + resource_class: facebook/rocksdb-benchmark-sys1 + steps: + - build-for-benchmarks + - perform-benchmarks + - post-benchmarks + +workflows: + version: 2 + jobs-linux-run-tests: + jobs: + - build-linux + - build-linux-cmake-with-folly + - build-linux-cmake-with-folly-lite-no-test + - build-linux-gcc-7-with-folly + - build-linux-gcc-7-with-folly-lite-no-test + - build-linux-cmake-with-folly-coroutines + - build-linux-cmake-with-benchmark + - build-linux-encrypted_env-no_compression + jobs-linux-run-tests-san: + jobs: + - build-linux-clang10-asan + - build-linux-clang10-ubsan + - build-linux-clang10-mini-tsan + - build-linux-static_lib-alt_namespace-status_checked + jobs-linux-no-test-run: + jobs: + - build-linux-release + - build-linux-release-rtti + - build-examples + - build-fuzzers + - build-linux-clang-no_test_run + - build-linux-clang-13-no_test_run + - build-linux-gcc-8-no_test_run + - build-linux-gcc-10-cxx20-no_test_run + - build-linux-gcc-11-no_test_run + - build-linux-arm-cmake-no_test_run + jobs-linux-other-checks: + jobs: + - build-linux-clang10-clang-analyze + - build-linux-unity-and-headers + - build-linux-mini-crashtest + jobs-windows: + jobs: + - build-windows-vs2022 + - build-windows-vs2019 + - build-cmake-mingw + jobs-java: + jobs: + - build-linux-java + - build-linux-java-static + - build-macos-java + - build-macos-java-static + - build-macos-java-static-universal + jobs-macos: + jobs: + - build-macos + - build-macos-cmake: + run_even_tests: true + - build-macos-cmake: + run_even_tests: false + jobs-linux-arm: + jobs: + - build-linux-arm + build-fuzzers: + jobs: + - build-fuzzers + benchmark-linux: + triggers: + - schedule: + cron: "0 * * * *" + filters: + branches: + only: + - main + jobs: + - benchmark-linux + nightly: + triggers: + - schedule: + cron: "0 9 * * *" + filters: + branches: + only: + - main + jobs: + - build-format-compatible + - build-linux-arm-test-full + - build-linux-run-microbench + - build-linux-non-shm + - build-linux-clang-13-asan-ubsan-with-folly + - build-linux-valgrind diff --git a/librocksdb-sys/rocksdb/.circleci/ubsan_suppression_list.txt b/librocksdb-sys/rocksdb/.circleci/ubsan_suppression_list.txt new file mode 100644 index 0000000..d7db818 --- /dev/null +++ b/librocksdb-sys/rocksdb/.circleci/ubsan_suppression_list.txt @@ -0,0 +1,6 @@ +# Supress UBSAN warnings related to stl_tree.h, e.g. +# UndefinedBehaviorSanitizer: undefined-behavior /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/bits/stl_tree.h:1505:43 in +# /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/bits/stl_tree.h:1505:43: +# runtime error: upcast of address 0x000001fa8820 with insufficient space for an object of type +# 'std::_Rb_tree_node, rocksdb::(anonymous namespace)::LockHoldingInfo> >' +src:*bits/stl_tree.h diff --git a/librocksdb-sys/rocksdb/.clang-format b/librocksdb-sys/rocksdb/.clang-format new file mode 100644 index 0000000..7c27981 --- /dev/null +++ b/librocksdb-sys/rocksdb/.clang-format @@ -0,0 +1,5 @@ +# Complete list of style options can be found at: +# http://clang.llvm.org/docs/ClangFormatStyleOptions.html +--- +BasedOnStyle: Google +... diff --git a/librocksdb-sys/rocksdb/.github/workflows/sanity_check.yml b/librocksdb-sys/rocksdb/.github/workflows/sanity_check.yml new file mode 100644 index 0000000..efc9d99 --- /dev/null +++ b/librocksdb-sys/rocksdb/.github/workflows/sanity_check.yml @@ -0,0 +1,45 @@ +name: Check buck targets and code format +on: [push, pull_request] +permissions: + contents: read + +jobs: + check: + name: Check TARGETS file and code format + runs-on: ubuntu-latest + steps: + - name: Checkout feature branch + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Fetch from upstream + run: | + git remote add upstream https://github.com/facebook/rocksdb.git && git fetch upstream + + - name: Where am I + run: | + echo git status && git status + echo "git remote -v" && git remote -v + echo git branch && git branch + + - name: Setup Python + uses: actions/setup-python@v1 + + - name: Install Dependencies + run: python -m pip install --upgrade pip + + - name: Install argparse + run: pip install argparse + + - name: Download clang-format-diff.py + run: wget https://raw.githubusercontent.com/llvm/llvm-project/release/12.x/clang/tools/clang-format/clang-format-diff.py + + - name: Check format + run: VERBOSE_CHECK=1 make check-format + + - name: Compare buckify output + run: make check-buck-targets + + - name: Simple source code checks + run: make check-sources diff --git a/librocksdb-sys/rocksdb/.gitignore b/librocksdb-sys/rocksdb/.gitignore new file mode 100644 index 0000000..8dd7e82 --- /dev/null +++ b/librocksdb-sys/rocksdb/.gitignore @@ -0,0 +1,99 @@ +make_config.mk +rocksdb.pc + +*.a +*.arc +*.d +*.dylib* +*.gcda +*.gcno +*.o +*.o.tmp +*.so +*.so.* +*_test +*_bench +*_stress +*.out +*.class +*.jar +*.*jnilib* +*.d-e +*.o-* +*.swp +*~ +*.vcxproj +*.vcxproj.filters +*.sln +*.cmake +.watchmanconfig +CMakeCache.txt +CMakeFiles/ +build/ + +ldb +manifest_dump +sst_dump +blob_dump +block_cache_trace_analyzer +tools/block_cache_analyzer/*.pyc +column_aware_encoding_exp +util/build_version.cc +build_tools/VALGRIND_LOGS/ +coverage/COVERAGE_REPORT +.gdbhistory +.gdb_history +package/ +unity.a +tags +etags +rocksdb_dump +rocksdb_undump +db_test2 +trace_analyzer +block_cache_trace_analyzer +io_tracer_parser +.DS_Store +.vs +.vscode +.clangd + +java/out +java/target +java/test-libs +java/*.log +java/include/org_rocksdb_*.h + +.idea/ +*.iml + +rocksdb.cc +rocksdb.h +unity.cc +java/crossbuild/.vagrant +.vagrant/ +java/**/*.asc +java/javadoc + +scan_build_report/ +t +LOG + +db_logs/ +tp2/ +fbcode/ +fbcode +buckifier/*.pyc +buckifier/__pycache__ + +compile_commands.json +clang-format-diff.py +.py3/ + +fuzz/proto/gen/ +fuzz/crash-* + +cmake-build-* +third-party/folly/ +.cache +*.sublime-* diff --git a/librocksdb-sys/rocksdb/.lgtm.yml b/librocksdb-sys/rocksdb/.lgtm.yml new file mode 100644 index 0000000..12d6f1d --- /dev/null +++ b/librocksdb-sys/rocksdb/.lgtm.yml @@ -0,0 +1,4 @@ +extraction: + cpp: + index: + build_command: make static_lib diff --git a/librocksdb-sys/rocksdb/.watchmanconfig b/librocksdb-sys/rocksdb/.watchmanconfig new file mode 100644 index 0000000..e5b450d --- /dev/null +++ b/librocksdb-sys/rocksdb/.watchmanconfig @@ -0,0 +1,6 @@ +{ + "content_hash_warming": true, + "content_hash_max_items": 333333, + "hint_num_files_per_dir": 8, + "fsevents_latency": 0.05 +} diff --git a/librocksdb-sys/rocksdb/AUTHORS b/librocksdb-sys/rocksdb/AUTHORS new file mode 100644 index 0000000..a451875 --- /dev/null +++ b/librocksdb-sys/rocksdb/AUTHORS @@ -0,0 +1,12 @@ +Facebook Inc. +Facebook Engineering Team + +Google Inc. +# Initial version authors: +Jeffrey Dean +Sanjay Ghemawat + +# Partial list of contributors: +Kevin Regan +Johan Bilien +Matthew Von-Maszewski (Basho Technologies) diff --git a/librocksdb-sys/rocksdb/CMakeLists.txt b/librocksdb-sys/rocksdb/CMakeLists.txt new file mode 100644 index 0000000..4e30f66 --- /dev/null +++ b/librocksdb-sys/rocksdb/CMakeLists.txt @@ -0,0 +1,1597 @@ +# Prerequisites for Windows: +# This cmake build is for Windows 64-bit only. +# +# Prerequisites: +# You must have at least Visual Studio 2019. Start the Developer Command Prompt window that is a part of Visual Studio installation. +# Run the build commands from within the Developer Command Prompt window to have paths to the compiler and runtime libraries set. +# You must have git.exe in your %PATH% environment variable. +# +# To build Rocksdb for Windows is as easy as 1-2-3-4-5: +# +# 1. Update paths to third-party libraries in thirdparty.inc file +# 2. Create a new directory for build artifacts +# mkdir build +# cd build +# 3. Run cmake to generate project files for Windows, add more options to enable required third-party libraries. +# See thirdparty.inc for more information. +# sample command: cmake -G "Visual Studio 16 2019" -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=1 -DWITH_SNAPPY=1 -DWITH_JEMALLOC=1 -DWITH_JNI=1 .. +# 4. Then build the project in debug mode (you may want to add /m[:] flag to run msbuild in parallel threads +# or simply /m to use all avail cores) +# msbuild rocksdb.sln +# +# rocksdb.sln build features exclusions of test only code in Release. If you build ALL_BUILD then everything +# will be attempted but test only code does not build in Release mode. +# +# 5. And release mode (/m[:] is also supported) +# msbuild rocksdb.sln /p:Configuration=Release +# +# Linux: +# +# 1. Install a recent toolchain if you're on a older distro. C++17 required (GCC >= 7, Clang >= 5) +# 2. mkdir build; cd build +# 3. cmake .. +# 4. make -j + +cmake_minimum_required(VERSION 3.10) + +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules/") +include(ReadVersion) +include(GoogleTest) +get_rocksdb_version(rocksdb_VERSION) +project(rocksdb + VERSION ${rocksdb_VERSION} + DESCRIPTION "An embeddable persistent key-value store for fast storage" + HOMEPAGE_URL https://rocksdb.org/ + LANGUAGES CXX C ASM) + +if(POLICY CMP0042) + cmake_policy(SET CMP0042 NEW) +endif() + +if(NOT CMAKE_BUILD_TYPE) + if(EXISTS "${CMAKE_SOURCE_DIR}/.git") + set(default_build_type "Debug") + else() + set(default_build_type "RelWithDebInfo") + endif() + set(CMAKE_BUILD_TYPE "${default_build_type}" CACHE STRING + "Default BUILD_TYPE is ${default_build_type}" FORCE) +endif() + +find_program(CCACHE_FOUND ccache) +if(CCACHE_FOUND) + set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) + set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache) +endif(CCACHE_FOUND) + +option(WITH_JEMALLOC "build with JeMalloc" OFF) +option(WITH_LIBURING "build with liburing" ON) +option(WITH_SNAPPY "build with SNAPPY" OFF) +option(WITH_LZ4 "build with lz4" OFF) +option(WITH_ZLIB "build with zlib" OFF) +option(WITH_ZSTD "build with zstd" OFF) +option(WITH_WINDOWS_UTF8_FILENAMES "use UTF8 as characterset for opening files, regardles of the system code page" OFF) +if (WITH_WINDOWS_UTF8_FILENAMES) + add_definitions(-DROCKSDB_WINDOWS_UTF8_FILENAMES) +endif() +option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" ON) + +if ($ENV{CIRCLECI}) + message(STATUS "Build for CircieCI env, a few tests may be disabled") + add_definitions(-DCIRCLECI) +endif() + +if( NOT DEFINED CMAKE_CXX_STANDARD ) + set(CMAKE_CXX_STANDARD 17) +endif() + +include(CMakeDependentOption) + +if(MSVC) + option(WITH_GFLAGS "build with GFlags" OFF) + option(WITH_XPRESS "build with windows built in compression" OFF) + option(ROCKSDB_SKIP_THIRDPARTY "skip thirdparty.inc" OFF) + + if(NOT ROCKSDB_SKIP_THIRDPARTY) + include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc) + endif() +else() + if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD" AND NOT CMAKE_SYSTEM_NAME MATCHES "kFreeBSD") + # FreeBSD has jemalloc as default malloc + # but it does not have all the jemalloc files in include/... + set(WITH_JEMALLOC ON) + else() + if(WITH_JEMALLOC) + find_package(JeMalloc REQUIRED) + add_definitions(-DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE) + list(APPEND THIRDPARTY_LIBS JeMalloc::JeMalloc) + endif() + endif() + + if(MINGW) + option(WITH_GFLAGS "build with GFlags" OFF) + else() + option(WITH_GFLAGS "build with GFlags" ON) + endif() + set(GFLAGS_LIB) + if(WITH_GFLAGS) + # Config with namespace available since gflags 2.2.2 + option(GFLAGS_USE_TARGET_NAMESPACE "Use gflags import target with namespace." ON) + find_package(gflags CONFIG) + if(gflags_FOUND) + if(TARGET ${GFLAGS_TARGET}) + # Config with GFLAGS_TARGET available since gflags 2.2.0 + set(GFLAGS_LIB ${GFLAGS_TARGET}) + else() + # Config with GFLAGS_LIBRARIES available since gflags 2.1.0 + set(GFLAGS_LIB ${gflags_LIBRARIES}) + endif() + else() + find_package(gflags REQUIRED) + set(GFLAGS_LIB gflags::gflags) + endif() + include_directories(${GFLAGS_INCLUDE_DIR}) + list(APPEND THIRDPARTY_LIBS ${GFLAGS_LIB}) + add_definitions(-DGFLAGS=1) + endif() + + if(WITH_SNAPPY) + find_package(Snappy CONFIG) + if(NOT Snappy_FOUND) + find_package(Snappy REQUIRED) + endif() + add_definitions(-DSNAPPY) + list(APPEND THIRDPARTY_LIBS Snappy::snappy) + endif() + + if(WITH_ZLIB) + find_package(ZLIB REQUIRED) + add_definitions(-DZLIB) + list(APPEND THIRDPARTY_LIBS ZLIB::ZLIB) + endif() + + option(WITH_BZ2 "build with bzip2" OFF) + if(WITH_BZ2) + find_package(BZip2 REQUIRED) + add_definitions(-DBZIP2) + if(BZIP2_INCLUDE_DIRS) + include_directories(${BZIP2_INCLUDE_DIRS}) + else() + include_directories(${BZIP2_INCLUDE_DIR}) + endif() + list(APPEND THIRDPARTY_LIBS ${BZIP2_LIBRARIES}) + endif() + + if(WITH_LZ4) + find_package(lz4 REQUIRED) + add_definitions(-DLZ4) + list(APPEND THIRDPARTY_LIBS lz4::lz4) + endif() + + if(WITH_ZSTD) + find_package(zstd REQUIRED) + add_definitions(-DZSTD) + include_directories(${ZSTD_INCLUDE_DIR}) + list(APPEND THIRDPARTY_LIBS zstd::zstd) + endif() +endif() + +option(WITH_MD_LIBRARY "build with MD" ON) +if(WIN32 AND MSVC) + if(WITH_MD_LIBRARY) + set(RUNTIME_LIBRARY "MD") + else() + set(RUNTIME_LIBRARY "MT") + endif() +endif() + +if(MSVC) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W4 /wd4127 /wd4800 /wd4996 /wd4351 /wd4100 /wd4204 /wd4324") +else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall -pthread") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers -Wno-strict-aliasing -Wno-invalid-offsetof") + if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wstrict-prototypes") + endif() + if(MINGW) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format") + add_definitions(-D_POSIX_C_SOURCE=1) + endif() + if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer") + include(CheckCXXCompilerFlag) + CHECK_CXX_COMPILER_FLAG("-momit-leaf-frame-pointer" HAVE_OMIT_LEAF_FRAME_POINTER) + if(HAVE_OMIT_LEAF_FRAME_POINTER) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -momit-leaf-frame-pointer") + endif() + endif() +endif() + +include(CheckCCompilerFlag) +if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64") + CHECK_C_COMPILER_FLAG("-mcpu=power9" HAS_POWER9) + if(HAS_POWER9) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcpu=power9 -mtune=power9") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=power9 -mtune=power9") + else() + CHECK_C_COMPILER_FLAG("-mcpu=power8" HAS_POWER8) + if(HAS_POWER8) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcpu=power8 -mtune=power8") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=power8 -mtune=power8") + endif(HAS_POWER8) + endif(HAS_POWER9) + CHECK_C_COMPILER_FLAG("-maltivec" HAS_ALTIVEC) + if(HAS_ALTIVEC) + message(STATUS " HAS_ALTIVEC yes") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -maltivec") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -maltivec") + endif(HAS_ALTIVEC) +endif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64") + +if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|AARCH64") + CHECK_C_COMPILER_FLAG("-march=armv8-a+crc+crypto" HAS_ARMV8_CRC) + if(HAS_ARMV8_CRC) + message(STATUS " HAS_ARMV8_CRC yes") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=armv8-a+crc+crypto -Wno-unused-function") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=armv8-a+crc+crypto -Wno-unused-function") + endif(HAS_ARMV8_CRC) +endif(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|AARCH64") + +if(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x") + CHECK_C_COMPILER_FLAG("-march=native" HAS_S390X_MARCH_NATIVE) + if(HAS_S390X_MARCH_NATIVE) + message(STATUS " HAS_S390X_MARCH_NATIVE yes") + endif(HAS_S390X_MARCH_NATIVE) +endif(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x") + +if(CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64") + CHECK_C_COMPILER_FLAG("-march=loongarch64" HAS_LOONGARCH64) + if(HAS_LOONGARCH64) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcpu=loongarch64 -mtune=loongarch64") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=loongarch64 -mtune=loongarch64") + endif(HAS_LOONGARCH64) +endif(CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64") + +set(PORTABLE 0 CACHE STRING "Minimum CPU arch to support, or 0 = current CPU, 1 = baseline CPU") +if(PORTABLE STREQUAL 1) + # Usually nothing to do; compiler default is typically the most general + if(NOT MSVC) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^s390x") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=z196") + endif() + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^loongarch64") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=loongarch64") + endif() + endif() +elseif(PORTABLE MATCHES [^0]+) + # Name of a CPU arch spec or feature set to require + if(MSVC) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:${PORTABLE}") + else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=${PORTABLE}") + endif() +else() + if(MSVC) + # NOTE: No auto-detection of current CPU, but instead assume some useful + # level of optimization is supported + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2") + else() + # Require instruction set from current CPU (with some legacy or opt-out + # exceptions) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^s390x" AND NOT HAS_S390X_MARCH_NATIVE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=z196") + elseif(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64" AND NOT HAS_ARMV8_CRC) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") + endif() + endif() +endif() + +include(CheckCXXSourceCompiles) +set(OLD_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS}) +if(NOT MSVC) + set(CMAKE_REQUIRED_FLAGS "-msse4.2 -mpclmul") +endif() + +# Check if -latomic is required or not +if (NOT MSVC) + set(CMAKE_REQUIRED_FLAGS "--std=c++17") + CHECK_CXX_SOURCE_COMPILES(" +#include +std::atomic x(0); +int main() { + uint64_t i = x.load(std::memory_order_relaxed); + bool b = x.is_lock_free(); + return 0; +} +" BUILTIN_ATOMIC) + if (NOT BUILTIN_ATOMIC) + #TODO: Check if -latomic exists + list(APPEND THIRDPARTY_LIBS atomic) + endif() +endif() + +if (WITH_LIBURING) + find_package(uring) + if (uring_FOUND) + add_definitions(-DROCKSDB_IOURING_PRESENT) + list(APPEND THIRDPARTY_LIBS uring::uring) + endif() +endif() + +# Reset the required flags +set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQUIRED_FLAGS}) + +option(WITH_IOSTATS_CONTEXT "Enable IO stats context" ON) +if (NOT WITH_IOSTATS_CONTEXT) + add_definitions(-DNIOSTATS_CONTEXT) +endif() + +option(WITH_PERF_CONTEXT "Enable perf context" ON) +if (NOT WITH_PERF_CONTEXT) + add_definitions(-DNPERF_CONTEXT) +endif() + +option(FAIL_ON_WARNINGS "Treat compile warnings as errors" ON) +if(FAIL_ON_WARNINGS) + if(MSVC) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /WX") + else() # assume GCC + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror") + endif() +endif() + +option(WITH_ASAN "build with ASAN" OFF) +if(WITH_ASAN) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") + if(WITH_JEMALLOC) + message(FATAL "ASAN does not work well with JeMalloc") + endif() +endif() + +option(WITH_TSAN "build with TSAN" OFF) +if(WITH_TSAN) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread -Wl,-pie") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread -fPIC") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=thread -fPIC") + if(WITH_JEMALLOC) + message(FATAL "TSAN does not work well with JeMalloc") + endif() +endif() + +option(WITH_UBSAN "build with UBSAN" OFF) +if(WITH_UBSAN) + add_definitions(-DROCKSDB_UBSAN_RUN) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=undefined") + if(WITH_JEMALLOC) + message(FATAL "UBSAN does not work well with JeMalloc") + endif() +endif() + +option(WITH_NUMA "build with NUMA policy support" OFF) +if(WITH_NUMA) + find_package(NUMA REQUIRED) + add_definitions(-DNUMA) + include_directories(${NUMA_INCLUDE_DIR}) + list(APPEND THIRDPARTY_LIBS NUMA::NUMA) +endif() + +option(WITH_TBB "build with Threading Building Blocks (TBB)" OFF) +if(WITH_TBB) + find_package(TBB REQUIRED) + add_definitions(-DTBB) + list(APPEND THIRDPARTY_LIBS TBB::TBB) +endif() + +# Stall notifications eat some performance from inserts +option(DISABLE_STALL_NOTIF "Build with stall notifications" OFF) +if(DISABLE_STALL_NOTIF) + add_definitions(-DROCKSDB_DISABLE_STALL_NOTIFICATION) +endif() + +option(WITH_DYNAMIC_EXTENSION "build with dynamic extension support" OFF) +if(NOT WITH_DYNAMIC_EXTENSION) + add_definitions(-DROCKSDB_NO_DYNAMIC_EXTENSION) +endif() + +option(ASSERT_STATUS_CHECKED "build with assert status checked" OFF) +if (ASSERT_STATUS_CHECKED) + message(STATUS "Build with assert status checked") + add_definitions(-DROCKSDB_ASSERT_STATUS_CHECKED) +endif() + + +# RTTI is by default AUTO which enables it in debug and disables it in release. +set(USE_RTTI AUTO CACHE STRING "Enable RTTI in builds") +set_property(CACHE USE_RTTI PROPERTY STRINGS AUTO ON OFF) +if(USE_RTTI STREQUAL "AUTO") + message(STATUS "Enabling RTTI in Debug builds only (default)") + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DROCKSDB_USE_RTTI") + if(MSVC) + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /GR-") + else() + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fno-rtti") + endif() +elseif(USE_RTTI) + message(STATUS "Enabling RTTI in all builds") + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DROCKSDB_USE_RTTI") + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DROCKSDB_USE_RTTI") +else() + if(MSVC) + message(STATUS "Disabling RTTI in Release builds. Always on in Debug.") + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DROCKSDB_USE_RTTI") + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /GR-") + else() + message(STATUS "Disabling RTTI in all builds") + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fno-rtti") + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fno-rtti") + endif() +endif() + +# Used to run CI build and tests so we can run faster +option(OPTDBG "Build optimized debug build with MSVC" OFF) +option(WITH_RUNTIME_DEBUG "build with debug version of runtime library" ON) +if(MSVC) + if(OPTDBG) + message(STATUS "Debug optimization is enabled") + set(CMAKE_CXX_FLAGS_DEBUG "/Oxt") + else() + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1") + + # Minimal Build is deprecated after MSVC 2015 + if( MSVC_VERSION GREATER 1900 ) + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Gm-") + else() + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Gm") + endif() + + endif() + if(WITH_RUNTIME_DEBUG) + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /${RUNTIME_LIBRARY}d") + else() + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /${RUNTIME_LIBRARY}") + endif() + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /${RUNTIME_LIBRARY}") + + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG") +endif() + +if(CMAKE_COMPILER_IS_GNUCXX) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-builtin-memcmp") +endif() + +if(CMAKE_SYSTEM_NAME MATCHES "Cygwin") + add_definitions(-fno-builtin-memcmp -DCYGWIN) +elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin") + add_definitions(-DOS_MACOSX) +elseif(CMAKE_SYSTEM_NAME MATCHES "Linux") + add_definitions(-DOS_LINUX) +elseif(CMAKE_SYSTEM_NAME MATCHES "SunOS") + add_definitions(-DOS_SOLARIS) +elseif(CMAKE_SYSTEM_NAME MATCHES "kFreeBSD") + add_definitions(-DOS_GNU_KFREEBSD) +elseif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") + add_definitions(-DOS_FREEBSD) +elseif(CMAKE_SYSTEM_NAME MATCHES "NetBSD") + add_definitions(-DOS_NETBSD) +elseif(CMAKE_SYSTEM_NAME MATCHES "OpenBSD") + add_definitions(-DOS_OPENBSD) +elseif(CMAKE_SYSTEM_NAME MATCHES "DragonFly") + add_definitions(-DOS_DRAGONFLYBSD) +elseif(CMAKE_SYSTEM_NAME MATCHES "Android") + add_definitions(-DOS_ANDROID) +elseif(CMAKE_SYSTEM_NAME MATCHES "Windows") + add_definitions(-DWIN32 -DOS_WIN -D_MBCS -DWIN64 -DNOMINMAX) + if(MINGW) + add_definitions(-D_WIN32_WINNT=_WIN32_WINNT_VISTA) + endif() +endif() + +if(NOT WIN32) + add_definitions(-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX) +endif() + +option(WITH_FALLOCATE "build with fallocate" ON) +if(WITH_FALLOCATE) + CHECK_CXX_SOURCE_COMPILES(" +#include +#include +int main() { + int fd = open(\"/dev/null\", 0); + fallocate(fd, FALLOC_FL_KEEP_SIZE, 0, 1024); +} +" HAVE_FALLOCATE) + if(HAVE_FALLOCATE) + add_definitions(-DROCKSDB_FALLOCATE_PRESENT) + endif() +endif() + +CHECK_CXX_SOURCE_COMPILES(" +#include +int main() { + int fd = open(\"/dev/null\", 0); + sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE); +} +" HAVE_SYNC_FILE_RANGE_WRITE) +if(HAVE_SYNC_FILE_RANGE_WRITE) + add_definitions(-DROCKSDB_RANGESYNC_PRESENT) +endif() + +CHECK_CXX_SOURCE_COMPILES(" +#include +int main() { + (void) PTHREAD_MUTEX_ADAPTIVE_NP; +} +" HAVE_PTHREAD_MUTEX_ADAPTIVE_NP) +if(HAVE_PTHREAD_MUTEX_ADAPTIVE_NP) + add_definitions(-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX) +endif() + +include(CheckCXXSymbolExists) +if(CMAKE_SYSTEM_NAME MATCHES "^FreeBSD") + check_cxx_symbol_exists(malloc_usable_size malloc_np.h HAVE_MALLOC_USABLE_SIZE) +else() + check_cxx_symbol_exists(malloc_usable_size malloc.h HAVE_MALLOC_USABLE_SIZE) +endif() +if(HAVE_MALLOC_USABLE_SIZE) + add_definitions(-DROCKSDB_MALLOC_USABLE_SIZE) +endif() + +check_cxx_symbol_exists(sched_getcpu sched.h HAVE_SCHED_GETCPU) +if(HAVE_SCHED_GETCPU) + add_definitions(-DROCKSDB_SCHED_GETCPU_PRESENT) +endif() + +check_cxx_symbol_exists(getauxval "sys/auxv.h" HAVE_AUXV_GETAUXVAL) +if(HAVE_AUXV_GETAUXVAL) + add_definitions(-DROCKSDB_AUXV_GETAUXVAL_PRESENT) +endif() + +check_cxx_symbol_exists(F_FULLFSYNC "fcntl.h" HAVE_FULLFSYNC) +if(HAVE_FULLFSYNC) + add_definitions(-DHAVE_FULLFSYNC) +endif() + +include_directories(${PROJECT_SOURCE_DIR}) +include_directories(${PROJECT_SOURCE_DIR}/include) + +if(USE_COROUTINES) + if(USE_FOLLY OR USE_FOLLY_LITE) + message(FATAL_ERROR "Please specify exactly one of USE_COROUTINES," + " USE_FOLLY, and USE_FOLLY_LITE") + endif() + set(CMAKE_CXX_STANDARD 20) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fcoroutines -Wno-maybe-uninitialized") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-redundant-move") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-invalid-memory-model") + add_compile_definitions(USE_COROUTINES) + set(USE_FOLLY 1) +endif() + +if(USE_FOLLY) + if(USE_FOLLY_LITE) + message(FATAL_ERROR "Please specify one of USE_FOLLY or USE_FOLLY_LITE") + endif() + if(ROCKSDB_BUILD_SHARED) + message(FATAL_ERROR "Cannot build RocksDB shared library with folly") + endif() + set(ROCKSDB_BUILD_SHARED OFF) + set(GFLAGS_SHARED FALSE) + find_package(folly) + # If cmake could not find the folly-config.cmake file, fall back + # to looking in third-party/folly for folly and its dependencies + if(NOT FOLLY_LIBRARIES) + exec_program(python3 ${PROJECT_SOURCE_DIR}/third-party/folly ARGS + build/fbcode_builder/getdeps.py show-inst-dir OUTPUT_VARIABLE + FOLLY_INST_PATH) + exec_program(ls ARGS -d ${FOLLY_INST_PATH}/../boost* OUTPUT_VARIABLE + BOOST_INST_PATH) + exec_program(ls ARGS -d ${FOLLY_INST_PATH}/../fmt* OUTPUT_VARIABLE + FMT_INST_PATH) + exec_program(ls ARGS -d ${FOLLY_INST_PATH}/../gflags* OUTPUT_VARIABLE + GFLAGS_INST_PATH) + set(Boost_DIR ${BOOST_INST_PATH}/lib/cmake/Boost-1.78.0) + if(EXISTS ${FMT_INST_PATH}/lib64) + set(fmt_DIR ${FMT_INST_PATH}/lib64/cmake/fmt) + else() + set(fmt_DIR ${FMT_INST_PATH}/lib/cmake/fmt) + endif() + set(gflags_DIR ${GFLAGS_INST_PATH}/lib/cmake/gflags) + + exec_program(sed ARGS -i 's/gflags_shared//g' + ${FOLLY_INST_PATH}/lib/cmake/folly/folly-targets.cmake) + + include(${FOLLY_INST_PATH}/lib/cmake/folly/folly-config.cmake) + endif() + + add_compile_definitions(USE_FOLLY FOLLY_NO_CONFIG HAVE_CXX11_ATOMIC) + list(APPEND THIRDPARTY_LIBS Folly::folly) + set(FOLLY_LIBS Folly::folly) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--copy-dt-needed-entries") +endif() +find_package(Threads REQUIRED) + +# Main library source code + +set(SOURCES + cache/cache.cc + cache/cache_entry_roles.cc + cache/cache_key.cc + cache/cache_helpers.cc + cache/cache_reservation_manager.cc + cache/charged_cache.cc + cache/clock_cache.cc + cache/compressed_secondary_cache.cc + cache/lru_cache.cc + cache/secondary_cache.cc + cache/secondary_cache_adapter.cc + cache/sharded_cache.cc + db/arena_wrapped_db_iter.cc + db/blob/blob_contents.cc + db/blob/blob_fetcher.cc + db/blob/blob_file_addition.cc + db/blob/blob_file_builder.cc + db/blob/blob_file_cache.cc + db/blob/blob_file_garbage.cc + db/blob/blob_file_meta.cc + db/blob/blob_file_reader.cc + db/blob/blob_garbage_meter.cc + db/blob/blob_log_format.cc + db/blob/blob_log_sequential_reader.cc + db/blob/blob_log_writer.cc + db/blob/blob_source.cc + db/blob/prefetch_buffer_collection.cc + db/builder.cc + db/c.cc + db/column_family.cc + db/compaction/compaction.cc + db/compaction/compaction_iterator.cc + db/compaction/compaction_picker.cc + db/compaction/compaction_job.cc + db/compaction/compaction_picker_fifo.cc + db/compaction/compaction_picker_level.cc + db/compaction/compaction_picker_universal.cc + db/compaction/compaction_service_job.cc + db/compaction/compaction_state.cc + db/compaction/compaction_outputs.cc + db/compaction/sst_partitioner.cc + db/compaction/subcompaction_state.cc + db/convenience.cc + db/db_filesnapshot.cc + db/db_impl/compacted_db_impl.cc + db/db_impl/db_impl.cc + db/db_impl/db_impl_write.cc + db/db_impl/db_impl_compaction_flush.cc + db/db_impl/db_impl_files.cc + db/db_impl/db_impl_open.cc + db/db_impl/db_impl_debug.cc + db/db_impl/db_impl_experimental.cc + db/db_impl/db_impl_readonly.cc + db/db_impl/db_impl_secondary.cc + db/db_info_dumper.cc + db/db_iter.cc + db/dbformat.cc + db/error_handler.cc + db/event_helpers.cc + db/experimental.cc + db/external_sst_file_ingestion_job.cc + db/file_indexer.cc + db/flush_job.cc + db/flush_scheduler.cc + db/forward_iterator.cc + db/import_column_family_job.cc + db/internal_stats.cc + db/logs_with_prep_tracker.cc + db/log_reader.cc + db/log_writer.cc + db/malloc_stats.cc + db/memtable.cc + db/memtable_list.cc + db/merge_helper.cc + db/merge_operator.cc + db/output_validator.cc + db/periodic_task_scheduler.cc + db/range_del_aggregator.cc + db/range_tombstone_fragmenter.cc + db/repair.cc + db/seqno_to_time_mapping.cc + db/snapshot_impl.cc + db/table_cache.cc + db/table_properties_collector.cc + db/transaction_log_impl.cc + db/trim_history_scheduler.cc + db/version_builder.cc + db/version_edit.cc + db/version_edit_handler.cc + db/version_set.cc + db/wal_edit.cc + db/wal_manager.cc + db/wide/wide_column_serialization.cc + db/wide/wide_columns.cc + db/write_batch.cc + db/write_batch_base.cc + db/write_controller.cc + db/write_stall_stats.cc + db/write_thread.cc + env/composite_env.cc + env/env.cc + env/env_chroot.cc + env/env_encryption.cc + env/file_system.cc + env/file_system_tracer.cc + env/fs_remap.cc + env/mock_env.cc + env/unique_id_gen.cc + file/delete_scheduler.cc + file/file_prefetch_buffer.cc + file/file_util.cc + file/filename.cc + file/line_file_reader.cc + file/random_access_file_reader.cc + file/read_write_util.cc + file/readahead_raf.cc + file/sequence_file_reader.cc + file/sst_file_manager_impl.cc + file/writable_file_writer.cc + logging/auto_roll_logger.cc + logging/event_logger.cc + logging/log_buffer.cc + memory/arena.cc + memory/concurrent_arena.cc + memory/jemalloc_nodump_allocator.cc + memory/memkind_kmem_allocator.cc + memory/memory_allocator.cc + memtable/alloc_tracker.cc + memtable/hash_linklist_rep.cc + memtable/hash_skiplist_rep.cc + memtable/skiplistrep.cc + memtable/vectorrep.cc + memtable/write_buffer_manager.cc + monitoring/histogram.cc + monitoring/histogram_windowing.cc + monitoring/in_memory_stats_history.cc + monitoring/instrumented_mutex.cc + monitoring/iostats_context.cc + monitoring/perf_context.cc + monitoring/perf_level.cc + monitoring/persistent_stats_history.cc + monitoring/statistics.cc + monitoring/thread_status_impl.cc + monitoring/thread_status_updater.cc + monitoring/thread_status_util.cc + monitoring/thread_status_util_debug.cc + options/cf_options.cc + options/configurable.cc + options/customizable.cc + options/db_options.cc + options/options.cc + options/options_helper.cc + options/options_parser.cc + port/mmap.cc + port/stack_trace.cc + table/adaptive/adaptive_table_factory.cc + table/block_based/binary_search_index_reader.cc + table/block_based/block.cc + table/block_based/block_based_table_builder.cc + table/block_based/block_based_table_factory.cc + table/block_based/block_based_table_iterator.cc + table/block_based/block_based_table_reader.cc + table/block_based/block_builder.cc + table/block_based/block_cache.cc + table/block_based/block_prefetcher.cc + table/block_based/block_prefix_index.cc + table/block_based/data_block_hash_index.cc + table/block_based/data_block_footer.cc + table/block_based/filter_block_reader_common.cc + table/block_based/filter_policy.cc + table/block_based/flush_block_policy.cc + table/block_based/full_filter_block.cc + table/block_based/hash_index_reader.cc + table/block_based/index_builder.cc + table/block_based/index_reader_common.cc + table/block_based/parsed_full_filter_block.cc + table/block_based/partitioned_filter_block.cc + table/block_based/partitioned_index_iterator.cc + table/block_based/partitioned_index_reader.cc + table/block_based/reader_common.cc + table/block_based/uncompression_dict_reader.cc + table/block_fetcher.cc + table/cuckoo/cuckoo_table_builder.cc + table/cuckoo/cuckoo_table_factory.cc + table/cuckoo/cuckoo_table_reader.cc + table/format.cc + table/get_context.cc + table/iterator.cc + table/merging_iterator.cc + table/compaction_merging_iterator.cc + table/meta_blocks.cc + table/persistent_cache_helper.cc + table/plain/plain_table_bloom.cc + table/plain/plain_table_builder.cc + table/plain/plain_table_factory.cc + table/plain/plain_table_index.cc + table/plain/plain_table_key_coding.cc + table/plain/plain_table_reader.cc + table/sst_file_dumper.cc + table/sst_file_reader.cc + table/sst_file_writer.cc + table/table_factory.cc + table/table_properties.cc + table/two_level_iterator.cc + table/unique_id.cc + test_util/sync_point.cc + test_util/sync_point_impl.cc + test_util/testutil.cc + test_util/transaction_test_util.cc + tools/block_cache_analyzer/block_cache_trace_analyzer.cc + tools/dump/db_dump_tool.cc + tools/io_tracer_parser_tool.cc + tools/ldb_cmd.cc + tools/ldb_tool.cc + tools/sst_dump_tool.cc + tools/trace_analyzer_tool.cc + trace_replay/block_cache_tracer.cc + trace_replay/io_tracer.cc + trace_replay/trace_record_handler.cc + trace_replay/trace_record_result.cc + trace_replay/trace_record.cc + trace_replay/trace_replay.cc + util/async_file_reader.cc + util/cleanable.cc + util/coding.cc + util/compaction_job_stats_impl.cc + util/comparator.cc + util/compression.cc + util/compression_context_cache.cc + util/concurrent_task_limiter_impl.cc + util/crc32c.cc + util/data_structure.cc + util/dynamic_bloom.cc + util/hash.cc + util/murmurhash.cc + util/random.cc + util/rate_limiter.cc + util/ribbon_config.cc + util/slice.cc + util/file_checksum_helper.cc + util/status.cc + util/stderr_logger.cc + util/string_util.cc + util/thread_local.cc + util/threadpool_imp.cc + util/udt_util.cc + util/write_batch_util.cc + util/xxhash.cc + utilities/agg_merge/agg_merge.cc + utilities/backup/backup_engine.cc + utilities/blob_db/blob_compaction_filter.cc + utilities/blob_db/blob_db.cc + utilities/blob_db/blob_db_impl.cc + utilities/blob_db/blob_db_impl_filesnapshot.cc + utilities/blob_db/blob_dump_tool.cc + utilities/blob_db/blob_file.cc + utilities/cache_dump_load.cc + utilities/cache_dump_load_impl.cc + utilities/cassandra/cassandra_compaction_filter.cc + utilities/cassandra/format.cc + utilities/cassandra/merge_operator.cc + utilities/checkpoint/checkpoint_impl.cc + utilities/compaction_filters.cc + utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc + utilities/counted_fs.cc + utilities/debug.cc + utilities/env_mirror.cc + utilities/env_timed.cc + utilities/fault_injection_env.cc + utilities/fault_injection_fs.cc + utilities/fault_injection_secondary_cache.cc + utilities/leveldb_options/leveldb_options.cc + utilities/memory/memory_util.cc + utilities/merge_operators.cc + utilities/merge_operators/bytesxor.cc + utilities/merge_operators/max.cc + utilities/merge_operators/put.cc + utilities/merge_operators/sortlist.cc + utilities/merge_operators/string_append/stringappend.cc + utilities/merge_operators/string_append/stringappend2.cc + utilities/merge_operators/uint64add.cc + utilities/object_registry.cc + utilities/option_change_migration/option_change_migration.cc + utilities/options/options_util.cc + utilities/persistent_cache/block_cache_tier.cc + utilities/persistent_cache/block_cache_tier_file.cc + utilities/persistent_cache/block_cache_tier_metadata.cc + utilities/persistent_cache/persistent_cache_tier.cc + utilities/persistent_cache/volatile_tier_impl.cc + utilities/simulator_cache/cache_simulator.cc + utilities/simulator_cache/sim_cache.cc + utilities/table_properties_collectors/compact_on_deletion_collector.cc + utilities/trace/file_trace_reader_writer.cc + utilities/trace/replayer_impl.cc + utilities/transactions/lock/lock_manager.cc + utilities/transactions/lock/point/point_lock_tracker.cc + utilities/transactions/lock/point/point_lock_manager.cc + utilities/transactions/lock/range/range_tree/range_tree_lock_manager.cc + utilities/transactions/lock/range/range_tree/range_tree_lock_tracker.cc + utilities/transactions/optimistic_transaction_db_impl.cc + utilities/transactions/optimistic_transaction.cc + utilities/transactions/pessimistic_transaction.cc + utilities/transactions/pessimistic_transaction_db.cc + utilities/transactions/snapshot_checker.cc + utilities/transactions/transaction_base.cc + utilities/transactions/transaction_db_mutex_impl.cc + utilities/transactions/transaction_util.cc + utilities/transactions/write_prepared_txn.cc + utilities/transactions/write_prepared_txn_db.cc + utilities/transactions/write_unprepared_txn.cc + utilities/transactions/write_unprepared_txn_db.cc + utilities/ttl/db_ttl_impl.cc + utilities/wal_filter.cc + utilities/write_batch_with_index/write_batch_with_index.cc + utilities/write_batch_with_index/write_batch_with_index_internal.cc) + +list(APPEND SOURCES + utilities/transactions/lock/range/range_tree/lib/locktree/concurrent_tree.cc + utilities/transactions/lock/range/range_tree/lib/locktree/keyrange.cc + utilities/transactions/lock/range/range_tree/lib/locktree/lock_request.cc + utilities/transactions/lock/range/range_tree/lib/locktree/locktree.cc + utilities/transactions/lock/range/range_tree/lib/locktree/manager.cc + utilities/transactions/lock/range/range_tree/lib/locktree/range_buffer.cc + utilities/transactions/lock/range/range_tree/lib/locktree/treenode.cc + utilities/transactions/lock/range/range_tree/lib/locktree/txnid_set.cc + utilities/transactions/lock/range/range_tree/lib/locktree/wfg.cc + utilities/transactions/lock/range/range_tree/lib/standalone_port.cc + utilities/transactions/lock/range/range_tree/lib/util/dbt.cc + utilities/transactions/lock/range/range_tree/lib/util/memarena.cc) + +message(STATUS "ROCKSDB_PLUGINS: ${ROCKSDB_PLUGINS}") +if ( ROCKSDB_PLUGINS ) + string(REPLACE " " ";" PLUGINS ${ROCKSDB_PLUGINS}) + foreach (plugin ${PLUGINS}) + add_subdirectory("plugin/${plugin}") + foreach (src ${${plugin}_SOURCES}) + list(APPEND SOURCES plugin/${plugin}/${src}) + set_source_files_properties( + plugin/${plugin}/${src} + PROPERTIES COMPILE_FLAGS "${${plugin}_COMPILE_FLAGS}") + endforeach() + foreach (test ${${plugin}_TESTS}) + list(APPEND PLUGIN_TESTS plugin/${plugin}/${test}) + set_source_files_properties( + plugin/${plugin}/${test} + PROPERTIES COMPILE_FLAGS "${${plugin}_COMPILE_FLAGS}") + endforeach() + foreach (path ${${plugin}_INCLUDE_PATHS}) + include_directories(${path}) + endforeach() + foreach (lib ${${plugin}_LIBS}) + list(APPEND THIRDPARTY_LIBS ${lib}) + endforeach() + foreach (link_path ${${plugin}_LINK_PATHS}) + link_directories(AFTER ${link_path}) + endforeach() + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${${plugin}_CMAKE_SHARED_LINKER_FLAGS}") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${${plugin}_CMAKE_EXE_LINKER_FLAGS}") + endforeach() +endif() + +if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64") + list(APPEND SOURCES + util/crc32c_ppc.c + util/crc32c_ppc_asm.S) +endif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64") + +if(HAS_ARMV8_CRC) + list(APPEND SOURCES + util/crc32c_arm64.cc) +endif(HAS_ARMV8_CRC) + +if(WIN32) + list(APPEND SOURCES + port/win/io_win.cc + port/win/env_win.cc + port/win/env_default.cc + port/win/port_win.cc + port/win/win_logger.cc + port/win/win_thread.cc) +if(WITH_XPRESS) + list(APPEND SOURCES + port/win/xpress_win.cc) +endif() + +if(WITH_JEMALLOC) + list(APPEND SOURCES + port/win/win_jemalloc.cc) +endif() + +else() + list(APPEND SOURCES + port/port_posix.cc + env/env_posix.cc + env/fs_posix.cc + env/io_posix.cc) +endif() + +if(USE_FOLLY_LITE) + list(APPEND SOURCES + third-party/folly/folly/container/detail/F14Table.cpp + third-party/folly/folly/detail/Futex.cpp + third-party/folly/folly/lang/SafeAssert.cpp + third-party/folly/folly/lang/ToAscii.cpp + third-party/folly/folly/ScopeGuard.cpp + third-party/folly/folly/synchronization/AtomicNotification.cpp + third-party/folly/folly/synchronization/DistributedMutex.cpp + third-party/folly/folly/synchronization/ParkingLot.cpp) + include_directories(${PROJECT_SOURCE_DIR}/third-party/folly) + add_definitions(-DUSE_FOLLY -DFOLLY_NO_CONFIG) + list(APPEND THIRDPARTY_LIBS glog) +endif() + +set(ROCKSDB_STATIC_LIB rocksdb${ARTIFACT_SUFFIX}) +set(ROCKSDB_SHARED_LIB rocksdb-shared${ARTIFACT_SUFFIX}) + + +if(WIN32) + set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib) +else() + set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT}) +endif() + +set(ROCKSDB_PLUGIN_EXTERNS "") +set(ROCKSDB_PLUGIN_BUILTINS "") +message(STATUS "ROCKSDB PLUGINS TO BUILD ${ROCKSDB_PLUGINS}") +foreach(PLUGIN IN LISTS PLUGINS) + set(PLUGIN_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/plugin/${PLUGIN}/") + message(STATUS "PLUGIN ${PLUGIN} including rocksb plugin ${PLUGIN_ROOT}") + set(PLUGINMKFILE "${PLUGIN_ROOT}${PLUGIN}.mk") + if (NOT EXISTS ${PLUGINMKFILE}) + message(FATAL_ERROR "PLUGIN ${PLUGIN} Missing plugin makefile: ${PLUGINMKFILE}") + endif() + file(READ ${PLUGINMKFILE} PLUGINMK) + + string(REGEX MATCH "SOURCES = ([^\n]*)" FOO ${PLUGINMK}) + set(MK_SOURCES ${CMAKE_MATCH_1}) + separate_arguments(MK_SOURCES) + foreach(MK_FILE IN LISTS MK_SOURCES) + list(APPEND SOURCES "${PLUGIN_ROOT}${MK_FILE}") + message(STATUS "PLUGIN ${PLUGIN} Appending ${PLUGIN_ROOT}${MK_FILE} to SOURCES") + endforeach() + + string(REGEX MATCH "_FUNC = ([^\n]*)" FOO ${PLUGINMK}) + if (NOT ${CMAKE_MATCH_1} STREQUAL "") + string(APPEND ROCKSDB_PLUGIN_BUILTINS "{\"${PLUGIN}\", " ${CMAKE_MATCH_1} "},") + string(APPEND ROCKSDB_PLUGIN_EXTERNS "int " ${CMAKE_MATCH_1} "(ROCKSDB_NAMESPACE::ObjectLibrary&, const std::string&); ") + endif() + + string(REGEX MATCH "_LIBS = ([^\n]*)" FOO ${PLUGINMK}) + separate_arguments(CMAKE_MATCH_1) + foreach(MK_LIB IN LISTS CMAKE_MATCH_1) + list(APPEND THIRDPARTY_LIBS "${MK_LIB}") + endforeach() + message(STATUS "PLUGIN ${PLUGIN} THIRDPARTY_LIBS=${THIRDPARTY_LIBS}") + + #TODO: We need to set any compile/link-time flags and add any link libraries +endforeach() + +string(TIMESTAMP TS "%Y-%m-%d %H:%M:%S" UTC) +set(BUILD_DATE "${TS}" CACHE STRING "the time we first built rocksdb") + +find_package(Git) + +if(GIT_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git") + execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_SHA COMMAND "${GIT_EXECUTABLE}" rev-parse HEAD ) + execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" RESULT_VARIABLE GIT_MOD COMMAND "${GIT_EXECUTABLE}" diff-index HEAD --quiet) + execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_DATE COMMAND "${GIT_EXECUTABLE}" log -1 --date=format:"%Y-%m-%d %T" --format="%ad") + execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_TAG RESULT_VARIABLE rv COMMAND "${GIT_EXECUTABLE}" symbolic-ref -q --short HEAD OUTPUT_STRIP_TRAILING_WHITESPACE) + if (rv AND NOT rv EQUAL 0) + execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_TAG COMMAND "${GIT_EXECUTABLE}" describe --tags --exact-match OUTPUT_STRIP_TRAILING_WHITESPACE) + endif() +else() + set(GIT_SHA 0) + set(GIT_MOD 1) +endif() +string(REGEX REPLACE "[^0-9a-fA-F]+" "" GIT_SHA "${GIT_SHA}") +string(REGEX REPLACE "[^0-9: /-]+" "" GIT_DATE "${GIT_DATE}") + +set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc) +configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY) + +add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES} ${BUILD_VERSION_CC}) +target_link_libraries(${ROCKSDB_STATIC_LIB} PRIVATE + ${THIRDPARTY_LIBS} ${SYSTEM_LIBS}) + +if(ROCKSDB_BUILD_SHARED) + add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES} ${BUILD_VERSION_CC}) + target_link_libraries(${ROCKSDB_SHARED_LIB} PRIVATE + ${THIRDPARTY_LIBS} ${SYSTEM_LIBS}) + + if(WIN32) + set_target_properties(${ROCKSDB_SHARED_LIB} PROPERTIES + COMPILE_DEFINITIONS "ROCKSDB_DLL;ROCKSDB_LIBRARY_EXPORTS") + if(MSVC) + set_target_properties(${ROCKSDB_STATIC_LIB} PROPERTIES + COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/${ROCKSDB_STATIC_LIB}.pdb") + set_target_properties(${ROCKSDB_SHARED_LIB} PROPERTIES + COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/${ROCKSDB_SHARED_LIB}.pdb") + endif() + else() + set_target_properties(${ROCKSDB_SHARED_LIB} PROPERTIES + LINKER_LANGUAGE CXX + VERSION ${rocksdb_VERSION} + SOVERSION ${rocksdb_VERSION_MAJOR} + OUTPUT_NAME "rocksdb${ARTIFACT_SUFFIX}") + endif() +endif() + +if(ROCKSDB_BUILD_SHARED AND NOT WIN32) + set(ROCKSDB_LIB ${ROCKSDB_SHARED_LIB}) +else() + set(ROCKSDB_LIB ${ROCKSDB_STATIC_LIB}) +endif() + +option(WITH_JNI "build with JNI" OFF) +# Tests are excluded from Release builds +CMAKE_DEPENDENT_OPTION(WITH_TESTS "build with tests" ON + "CMAKE_BUILD_TYPE STREQUAL Debug" OFF) +option(WITH_BENCHMARK_TOOLS "build with benchmarks" ON) +option(WITH_CORE_TOOLS "build with ldb and sst_dump" ON) +option(WITH_TOOLS "build with tools" ON) + +if(WITH_TESTS OR WITH_BENCHMARK_TOOLS OR WITH_TOOLS OR WITH_JNI OR JNI) + include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third-party/gtest-1.8.1/fused-src) +endif() +if(WITH_JNI OR JNI) + message(STATUS "JNI library is enabled") + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/java) +else() + message(STATUS "JNI library is disabled") +endif() + +# Installation and packaging +if(WIN32) + option(ROCKSDB_INSTALL_ON_WINDOWS "Enable install target on Windows" OFF) +endif() +if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS) + if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) + if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux") + # Change default installation prefix on Linux to /usr + set(CMAKE_INSTALL_PREFIX /usr CACHE PATH "Install path prefix, prepended onto install directories." FORCE) + endif() + endif() + + include(GNUInstallDirs) + include(CMakePackageConfigHelpers) + + set(package_config_destination ${CMAKE_INSTALL_LIBDIR}/cmake/rocksdb) + + configure_package_config_file( + ${CMAKE_CURRENT_LIST_DIR}/cmake/RocksDBConfig.cmake.in RocksDBConfig.cmake + INSTALL_DESTINATION ${package_config_destination} + ) + + write_basic_package_version_file( + RocksDBConfigVersion.cmake + VERSION ${rocksdb_VERSION} + COMPATIBILITY SameMajorVersion + ) + + configure_file( + ${PROJECT_NAME}.pc.in + ${PROJECT_NAME}.pc + @ONLY + ) + + install(DIRECTORY include/rocksdb COMPONENT devel DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") + + foreach (plugin ${PLUGINS}) + foreach (header ${${plugin}_HEADERS}) + install(FILES plugin/${plugin}/${header} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rocksdb/plugin/${plugin}) + endforeach() + endforeach() + + install(DIRECTORY "${PROJECT_SOURCE_DIR}/cmake/modules" COMPONENT devel DESTINATION ${package_config_destination}) + + install( + TARGETS ${ROCKSDB_STATIC_LIB} + EXPORT RocksDBTargets + COMPONENT devel + ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" + INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + ) + + if(ROCKSDB_BUILD_SHARED) + install( + TARGETS ${ROCKSDB_SHARED_LIB} + EXPORT RocksDBTargets + COMPONENT runtime + ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" + RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" + LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" + INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + ) + endif() + + install( + EXPORT RocksDBTargets + COMPONENT devel + DESTINATION ${package_config_destination} + NAMESPACE RocksDB:: + ) + + install( + FILES + ${CMAKE_CURRENT_BINARY_DIR}/RocksDBConfig.cmake + ${CMAKE_CURRENT_BINARY_DIR}/RocksDBConfigVersion.cmake + COMPONENT devel + DESTINATION ${package_config_destination} + ) + + install( + FILES + ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}.pc + COMPONENT devel + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig + ) +endif() + +option(WITH_ALL_TESTS "Build all test, rather than a small subset" ON) + +if(WITH_TESTS OR WITH_BENCHMARK_TOOLS) + add_subdirectory(third-party/gtest-1.8.1/fused-src/gtest) + add_library(testharness STATIC + test_util/mock_time_env.cc + test_util/secondary_cache_test_util.cc + test_util/testharness.cc) + target_link_libraries(testharness gtest) +endif() + +if(WITH_TESTS) + set(TESTS + db/db_basic_test.cc + env/env_basic_test.cc + ) + if(WITH_ALL_TESTS) + list(APPEND TESTS + cache/cache_reservation_manager_test.cc + cache/cache_test.cc + cache/compressed_secondary_cache_test.cc + cache/lru_cache_test.cc + db/blob/blob_counting_iterator_test.cc + db/blob/blob_file_addition_test.cc + db/blob/blob_file_builder_test.cc + db/blob/blob_file_cache_test.cc + db/blob/blob_file_garbage_test.cc + db/blob/blob_file_reader_test.cc + db/blob/blob_garbage_meter_test.cc + db/blob/blob_source_test.cc + db/blob/db_blob_basic_test.cc + db/blob/db_blob_compaction_test.cc + db/blob/db_blob_corruption_test.cc + db/blob/db_blob_index_test.cc + db/column_family_test.cc + db/compact_files_test.cc + db/compaction/clipping_iterator_test.cc + db/compaction/compaction_job_stats_test.cc + db/compaction/compaction_job_test.cc + db/compaction/compaction_iterator_test.cc + db/compaction/compaction_picker_test.cc + db/compaction/compaction_service_test.cc + db/compaction/tiered_compaction_test.cc + db/comparator_db_test.cc + db/corruption_test.cc + db/cuckoo_table_db_test.cc + db/db_readonly_with_timestamp_test.cc + db/db_with_timestamp_basic_test.cc + db/db_block_cache_test.cc + db/db_bloom_filter_test.cc + db/db_compaction_filter_test.cc + db/db_compaction_test.cc + db/db_clip_test.cc + db/db_dynamic_level_test.cc + db/db_encryption_test.cc + db/db_flush_test.cc + db/db_inplace_update_test.cc + db/db_io_failure_test.cc + db/db_iter_test.cc + db/db_iter_stress_test.cc + db/db_iterator_test.cc + db/db_kv_checksum_test.cc + db/db_log_iter_test.cc + db/db_memtable_test.cc + db/db_merge_operator_test.cc + db/db_merge_operand_test.cc + db/db_options_test.cc + db/db_properties_test.cc + db/db_range_del_test.cc + db/db_rate_limiter_test.cc + db/db_secondary_test.cc + db/db_sst_test.cc + db/db_statistics_test.cc + db/db_table_properties_test.cc + db/db_tailing_iter_test.cc + db/db_test.cc + db/db_test2.cc + db/db_logical_block_size_cache_test.cc + db/db_universal_compaction_test.cc + db/db_wal_test.cc + db/db_with_timestamp_compaction_test.cc + db/db_write_buffer_manager_test.cc + db/db_write_test.cc + db/dbformat_test.cc + db/deletefile_test.cc + db/error_handler_fs_test.cc + db/obsolete_files_test.cc + db/external_sst_file_basic_test.cc + db/external_sst_file_test.cc + db/fault_injection_test.cc + db/file_indexer_test.cc + db/filename_test.cc + db/flush_job_test.cc + db/import_column_family_test.cc + db/listener_test.cc + db/log_test.cc + db/manual_compaction_test.cc + db/memtable_list_test.cc + db/merge_helper_test.cc + db/merge_test.cc + db/options_file_test.cc + db/perf_context_test.cc + db/periodic_task_scheduler_test.cc + db/plain_table_db_test.cc + db/seqno_time_test.cc + db/prefix_test.cc + db/range_del_aggregator_test.cc + db/range_tombstone_fragmenter_test.cc + db/repair_test.cc + db/table_properties_collector_test.cc + db/version_builder_test.cc + db/version_edit_test.cc + db/version_set_test.cc + db/wal_manager_test.cc + db/wal_edit_test.cc + db/wide/db_wide_basic_test.cc + db/wide/wide_column_serialization_test.cc + db/write_batch_test.cc + db/write_callback_test.cc + db/write_controller_test.cc + env/env_test.cc + env/io_posix_test.cc + env/mock_env_test.cc + file/delete_scheduler_test.cc + file/prefetch_test.cc + file/random_access_file_reader_test.cc + logging/auto_roll_logger_test.cc + logging/env_logger_test.cc + logging/event_logger_test.cc + memory/arena_test.cc + memory/memory_allocator_test.cc + memtable/inlineskiplist_test.cc + memtable/skiplist_test.cc + memtable/write_buffer_manager_test.cc + monitoring/histogram_test.cc + monitoring/iostats_context_test.cc + monitoring/statistics_test.cc + monitoring/stats_history_test.cc + options/configurable_test.cc + options/customizable_test.cc + options/options_settable_test.cc + options/options_test.cc + table/block_based/block_based_table_reader_test.cc + table/block_based/block_test.cc + table/block_based/data_block_hash_index_test.cc + table/block_based/full_filter_block_test.cc + table/block_based/partitioned_filter_block_test.cc + table/cleanable_test.cc + table/cuckoo/cuckoo_table_builder_test.cc + table/cuckoo/cuckoo_table_reader_test.cc + table/merger_test.cc + table/sst_file_reader_test.cc + table/table_test.cc + table/block_fetcher_test.cc + test_util/testutil_test.cc + trace_replay/block_cache_tracer_test.cc + trace_replay/io_tracer_test.cc + tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc + tools/io_tracer_parser_test.cc + tools/ldb_cmd_test.cc + tools/reduce_levels_test.cc + tools/sst_dump_test.cc + tools/trace_analyzer_test.cc + util/autovector_test.cc + util/bloom_test.cc + util/coding_test.cc + util/crc32c_test.cc + util/defer_test.cc + util/dynamic_bloom_test.cc + util/file_reader_writer_test.cc + util/filelock_test.cc + util/hash_test.cc + util/heap_test.cc + util/random_test.cc + util/rate_limiter_test.cc + util/repeatable_thread_test.cc + util/ribbon_test.cc + util/slice_test.cc + util/slice_transform_test.cc + util/timer_queue_test.cc + util/timer_test.cc + util/thread_list_test.cc + util/thread_local_test.cc + util/udt_util_test.cc + util/work_queue_test.cc + utilities/agg_merge/agg_merge_test.cc + utilities/backup/backup_engine_test.cc + utilities/blob_db/blob_db_test.cc + utilities/cassandra/cassandra_functional_test.cc + utilities/cassandra/cassandra_format_test.cc + utilities/cassandra/cassandra_row_merge_test.cc + utilities/cassandra/cassandra_serialize_test.cc + utilities/checkpoint/checkpoint_test.cc + utilities/env_timed_test.cc + utilities/memory/memory_test.cc + utilities/merge_operators/string_append/stringappend_test.cc + utilities/object_registry_test.cc + utilities/option_change_migration/option_change_migration_test.cc + utilities/options/options_util_test.cc + utilities/persistent_cache/hash_table_test.cc + utilities/persistent_cache/persistent_cache_test.cc + utilities/simulator_cache/cache_simulator_test.cc + utilities/simulator_cache/sim_cache_test.cc + utilities/table_properties_collectors/compact_on_deletion_collector_test.cc + utilities/transactions/optimistic_transaction_test.cc + utilities/transactions/transaction_test.cc + utilities/transactions/lock/point/point_lock_manager_test.cc + utilities/transactions/write_committed_transaction_ts_test.cc + utilities/transactions/write_prepared_transaction_test.cc + utilities/transactions/write_unprepared_transaction_test.cc + utilities/transactions/lock/range/range_locking_test.cc + utilities/transactions/timestamped_snapshot_test.cc + utilities/ttl/ttl_test.cc + utilities/util_merge_operators_test.cc + utilities/write_batch_with_index/write_batch_with_index_test.cc + ${PLUGIN_TESTS} + ) + endif() + + set(TESTUTIL_SOURCE + db/db_test_util.cc + db/db_with_timestamp_test_util.cc + monitoring/thread_status_updater_debug.cc + table/mock_table.cc + utilities/agg_merge/test_agg_merge.cc + utilities/cassandra/test_utils.cc + ) + enable_testing() + add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND}) + set(TESTUTILLIB testutillib${ARTIFACT_SUFFIX}) + add_library(${TESTUTILLIB} STATIC ${TESTUTIL_SOURCE}) + target_link_libraries(${TESTUTILLIB} ${ROCKSDB_LIB} ${FOLLY_LIBS}) + if(MSVC) + set_target_properties(${TESTUTILLIB} PROPERTIES COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/testutillib${ARTIFACT_SUFFIX}.pdb") + endif() + set_target_properties(${TESTUTILLIB} + PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1 + EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1 + EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1 + ) + + foreach(sourcefile ${TESTS}) + get_filename_component(exename ${sourcefile} NAME_WE) + add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile}) + set_target_properties(${exename}${ARTIFACT_SUFFIX} + PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1 + EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1 + EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1 + OUTPUT_NAME ${exename}${ARTIFACT_SUFFIX} + ) + target_link_libraries(${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} testharness gtest ${THIRDPARTY_LIBS} ${ROCKSDB_LIB}) + if(NOT "${exename}" MATCHES "db_sanity_test") + gtest_discover_tests(${exename} DISCOVERY_TIMEOUT 120) + add_dependencies(check ${exename}${ARTIFACT_SUFFIX}) + endif() + endforeach(sourcefile ${TESTS}) + + if(WIN32) + # C executables must link to a shared object + if(ROCKSDB_BUILD_SHARED) + set(ROCKSDB_LIB_FOR_C ${ROCKSDB_SHARED_LIB}) + else() + set(ROCKSDB_LIB_FOR_C OFF) + endif() + else() + set(ROCKSDB_LIB_FOR_C ${ROCKSDB_LIB}) + endif() + + if(ROCKSDB_LIB_FOR_C) + set(C_TESTS db/c_test.c) + add_executable(c_test db/c_test.c) + target_link_libraries(c_test ${ROCKSDB_LIB_FOR_C} testharness) + add_test(NAME c_test COMMAND c_test${ARTIFACT_SUFFIX}) + add_dependencies(check c_test) + endif() +endif() + +if(WITH_BENCHMARK_TOOLS) + add_executable(db_bench${ARTIFACT_SUFFIX} + tools/simulated_hybrid_file_system.cc + tools/db_bench.cc + tools/db_bench_tool.cc) + target_link_libraries(db_bench${ARTIFACT_SUFFIX} + ${ROCKSDB_LIB} ${THIRDPARTY_LIBS}) + + add_executable(cache_bench${ARTIFACT_SUFFIX} + cache/cache_bench.cc + cache/cache_bench_tool.cc) + target_link_libraries(cache_bench${ARTIFACT_SUFFIX} + ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS}) + + add_executable(memtablerep_bench${ARTIFACT_SUFFIX} + memtable/memtablerep_bench.cc) + target_link_libraries(memtablerep_bench${ARTIFACT_SUFFIX} + ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS}) + + add_executable(range_del_aggregator_bench${ARTIFACT_SUFFIX} + db/range_del_aggregator_bench.cc) + target_link_libraries(range_del_aggregator_bench${ARTIFACT_SUFFIX} + ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS}) + + add_executable(table_reader_bench${ARTIFACT_SUFFIX} + table/table_reader_bench.cc) + target_link_libraries(table_reader_bench${ARTIFACT_SUFFIX} + ${ROCKSDB_LIB} testharness ${GFLAGS_LIB} ${FOLLY_LIBS}) + + add_executable(filter_bench${ARTIFACT_SUFFIX} + util/filter_bench.cc) + target_link_libraries(filter_bench${ARTIFACT_SUFFIX} + ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS}) + + add_executable(hash_table_bench${ARTIFACT_SUFFIX} + utilities/persistent_cache/hash_table_bench.cc) + target_link_libraries(hash_table_bench${ARTIFACT_SUFFIX} + ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS}) +endif() + +option(WITH_TRACE_TOOLS "build with trace tools" ON) +if(WITH_TRACE_TOOLS) + add_executable(block_cache_trace_analyzer${ARTIFACT_SUFFIX} + tools/block_cache_analyzer/block_cache_trace_analyzer_tool.cc) + target_link_libraries(block_cache_trace_analyzer${ARTIFACT_SUFFIX} + ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS}) + + add_executable(trace_analyzer${ARTIFACT_SUFFIX} + tools/trace_analyzer.cc) + target_link_libraries(trace_analyzer${ARTIFACT_SUFFIX} + ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS}) + +endif() + +if(WITH_CORE_TOOLS OR WITH_TOOLS) + add_subdirectory(tools) + add_custom_target(core_tools + DEPENDS ${core_tool_deps}) +endif() + +if(WITH_TOOLS) + add_subdirectory(db_stress_tool) + add_custom_target(tools + DEPENDS ${tool_deps}) +endif() + +option(WITH_EXAMPLES "build with examples" OFF) +if(WITH_EXAMPLES) + add_subdirectory(examples) +endif() + +option(WITH_BENCHMARK "build benchmark tests" OFF) +if(WITH_BENCHMARK) + add_subdirectory(${PROJECT_SOURCE_DIR}/microbench/) +endif() + +target_include_directories(${PROJECT_NAME} PUBLIC + $) diff --git a/librocksdb-sys/rocksdb/CODE_OF_CONDUCT.md b/librocksdb-sys/rocksdb/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..d1abc70 --- /dev/null +++ b/librocksdb-sys/rocksdb/CODE_OF_CONDUCT.md @@ -0,0 +1,77 @@ +# Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to make participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies within all project spaces, and it also applies when +an individual is representing the project or its community in public spaces. +Examples of representing a project or community include using an official +project e-mail address, posting via an official social media account, or acting +as an appointed representative at an online or offline event. Representation of +a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at . All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq + diff --git a/librocksdb-sys/rocksdb/CONTRIBUTING.md b/librocksdb-sys/rocksdb/CONTRIBUTING.md new file mode 100644 index 0000000..190100b --- /dev/null +++ b/librocksdb-sys/rocksdb/CONTRIBUTING.md @@ -0,0 +1,17 @@ +# Contributing to RocksDB + +## Code of Conduct +The code of conduct is described in [`CODE_OF_CONDUCT.md`](CODE_OF_CONDUCT.md) + +## Contributor License Agreement ("CLA") + +In order to accept your pull request, we need you to submit a CLA. You +only need to do this once, so if you've done this for another Facebook +open source project, you're good to go. If you are submitting a pull +request for the first time, just let us know that you have completed +the CLA and we can cross-check with your GitHub username. + +Complete your CLA here: + +If you prefer to sign a paper copy, we can send you a PDF. Send us an +e-mail or create a new github issue to request the CLA in PDF format. diff --git a/librocksdb-sys/rocksdb/COPYING b/librocksdb-sys/rocksdb/COPYING new file mode 100644 index 0000000..d159169 --- /dev/null +++ b/librocksdb-sys/rocksdb/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/librocksdb-sys/rocksdb/DEFAULT_OPTIONS_HISTORY.md b/librocksdb-sys/rocksdb/DEFAULT_OPTIONS_HISTORY.md new file mode 100644 index 0000000..82c64d5 --- /dev/null +++ b/librocksdb-sys/rocksdb/DEFAULT_OPTIONS_HISTORY.md @@ -0,0 +1,24 @@ +# RocksDB default options change log (NO LONGER MAINTAINED) +## Unreleased +* delayed_write_rate takes the rate given by rate_limiter if not specified. + +## 5.2 +* Change the default of delayed slowdown value to 16MB/s and further increase the L0 stop condition to 36 files. + +## 5.0 (11/17/2016) +* Options::allow_concurrent_memtable_write and Options::enable_write_thread_adaptive_yield are now true by default +* Options.level0_stop_writes_trigger default value changes from 24 to 32. + +## 4.8.0 (5/2/2016) +* options.max_open_files changes from 5000 to -1. It improves performance, but users need to set file descriptor limit to be large enough and watch memory usage for index and bloom filters. +* options.base_background_compactions changes from max_background_compactions to 1. When users set higher max_background_compactions but the write throughput is not high, the writes are less spiky to disks. +* options.wal_recovery_mode changes from kTolerateCorruptedTailRecords to kPointInTimeRecovery. Avoid some false positive when file system or hardware reorder the writes for file data and metadata. + +## 4.7.0 (4/8/2016) +* options.write_buffer_size changes from 4MB to 64MB. +* options.target_file_size_base changes from 2MB to 64MB. +* options.max_bytes_for_level_base changes from 10MB to 256MB. +* options.soft_pending_compaction_bytes_limit changes from 0 (disabled) to 64GB. +* options.hard_pending_compaction_bytes_limit changes from 0 (disabled) to 256GB. +* table_cache_numshardbits changes from 4 to 6. +* max_file_opening_threads changes from 1 to 16. diff --git a/librocksdb-sys/rocksdb/DUMP_FORMAT.md b/librocksdb-sys/rocksdb/DUMP_FORMAT.md new file mode 100644 index 0000000..009daba --- /dev/null +++ b/librocksdb-sys/rocksdb/DUMP_FORMAT.md @@ -0,0 +1,16 @@ +## RocksDB dump format + +The version 1 RocksDB dump format is fairly simple: + +1) The dump starts with the magic 8 byte identifier "ROCKDUMP" + +2) The magic is followed by an 8 byte big-endian version which is 0x00000001. + +3) Next are arbitrarily sized chunks of bytes prepended by 4 byte little endian number indicating how large each chunk is. + +4) The first chunk is special and is a json string indicating some things about the creation of this dump. It contains the following keys: +* database-path: The path of the database this dump was created from. +* hostname: The hostname of the machine where the dump was created. +* creation-time: Unix seconds since epoc when this dump was created. + +5) Following the info dump the slices paired into are key/value pairs. diff --git a/librocksdb-sys/rocksdb/HISTORY.md b/librocksdb-sys/rocksdb/HISTORY.md new file mode 100644 index 0000000..71b91d9 --- /dev/null +++ b/librocksdb-sys/rocksdb/HISTORY.md @@ -0,0 +1,2440 @@ +# Rocksdb Change Log +> NOTE: Entries for next release do not go here. Follow instructions in `unreleased_history/README.txt` + +## 8.5.0 (07/21/2023) +### Public API Changes +* Removed recently added APIs `GeneralCache` and `MakeSharedGeneralCache()` as our plan changed to stop exposing a general-purpose cache interface. The old forms of these APIs, `Cache` and `NewLRUCache()`, are still available, although general-purpose caching support will be dropped eventually. + +### Behavior Changes +* Option `periodic_compaction_seconds` no longer supports FIFO compaction: setting it has no effect on FIFO compactions. FIFO compaction users should only set option `ttl` instead. +* Move prefetching responsibility to page cache for compaction read for non directIO use case + +### Performance Improvements +* In case of direct_io, if buffer passed by callee is already aligned, RandomAccessFileRead::Read will avoid realloacting a new buffer, reducing memcpy and use already passed aligned buffer. +* Small efficiency improvement to HyperClockCache by reducing chance of compiler-generated heap allocations + +### Bug Fixes +* Fix use_after_free bug in async_io MultiReads when underlying FS enabled kFSBuffer. kFSBuffer is when underlying FS pass their own buffer instead of using RocksDB scratch in FSReadRequest. Right now it's an experimental feature. + +## 8.4.0 (06/26/2023) +### New Features +* Add FSReadRequest::fs_scratch which is a data buffer allocated and provided by underlying FileSystem to RocksDB during reads, when FS wants to provide its own buffer with data instead of using RocksDB provided FSReadRequest::scratch. This can help in cpu optimization by avoiding copy from file system's buffer to RocksDB buffer. More details on how to use/enable it in file_system.h. Right now its supported only for MultiReads(async + sync) with non direct io. +* Start logging non-zero user-defined timestamp sizes in WAL to signal user key format in subsequent records and use it during recovery. This change will break recovery from WAL files written by early versions that contain user-defined timestamps. The workaround is to ensure there are no WAL files to recover (i.e. by flushing before close) before upgrade. +* Added new property "rocksdb.obsolete-sst-files-size-property" that reports the size of SST files that have become obsolete but have not yet been deleted or scheduled for deletion +* Start to record the value of the flag `AdvancedColumnFamilyOptions.persist_user_defined_timestamps` in the Manifest and table properties for a SST file when it is created. And use the recorded flag when creating a table reader for the SST file. This flag is only explicitly record if it's false. +* Add a new option OptimisticTransactionDBOptions::shared_lock_buckets that enables sharing mutexes for validating transactions between DB instances, for better balancing memory efficiency and validation contention across DB instances. Different column families and DBs also now use different hash seeds in this validation, so that the same set of key names will not contend across DBs or column families. +* Add a new ticker `rocksdb.files.marked.trash.deleted` to track the number of trash files deleted by background thread from the trash queue. +* Add an API NewTieredVolatileCache() in include/rocksdb/cache.h to allocate an instance of a block cache with a primary block cache tier and a compressed secondary cache tier. A cache of this type distributes memory reservations against the block cache, such as WriteBufferManager, table reader memory etc., proportionally across both the primary and compressed secondary cache. +* Add `WaitForCompact()` to wait for all flush and compactions jobs to finish. Jobs to wait include the unscheduled (queued, but not scheduled yet). +* Add `WriteBatch::Release()` that releases the batch's serialized data to the caller. + +### Public API Changes +* Add C API `rocksdb_options_add_compact_on_deletion_collector_factory_del_ratio`. +* change the FileSystem::use_async_io() API to SupportedOps API in order to extend it to various operations supported by underlying FileSystem. Right now it contains FSSupportedOps::kAsyncIO and FSSupportedOps::kFSBuffer. More details about FSSupportedOps in filesystem.h +* Add new tickers: `rocksdb.error.handler.bg.error.count`, `rocksdb.error.handler.bg.io.error.count`, `rocksdb.error.handler.bg.retryable.io.error.count` to replace the misspelled ones: `rocksdb.error.handler.bg.errro.count`, `rocksdb.error.handler.bg.io.errro.count`, `rocksdb.error.handler.bg.retryable.io.errro.count` ('error' instead of 'errro'). Users should switch to use the new tickers before 9.0 release as the misspelled old tickers will be completely removed then. +* Overload the API CreateColumnFamilyWithImport() to support creating ColumnFamily by importing multiple ColumnFamilies It requires that CFs should not overlap in user key range. + +### Behavior Changes +* Change the default value for option `level_compaction_dynamic_level_bytes` to true. This affects users who use leveled compaction and do not set this option explicitly. These users may see additional background compactions following DB open. These compactions help to shape the LSM according to `level_compaction_dynamic_level_bytes` such that the size of each level Ln is approximately size of Ln-1 * `max_bytes_for_level_multiplier`. Turning on this option has other benefits too: see more detail in wiki: https://github.com/facebook/rocksdb/wiki/Leveled-Compaction#option-level_compaction_dynamic_level_bytes-and-levels-target-size and in option comment in advanced_options.h (#11525). +* For Leveled Compaction users, `CompactRange()` will now always try to compact to the last non-empty level. (#11468) +For Leveled Compaction users, `CompactRange()` with `bottommost_level_compaction = BottommostLevelCompaction::kIfHaveCompactionFilter` will behave similar to `kForceOptimized` in that it will skip files created during this manual compaction when compacting files in the bottommost level. (#11468) +* RocksDB will try to drop range tombstones during non-bottommost compaction when it is safe to do so. (#11459) +* When a DB is openend with `allow_ingest_behind=true` (currently only Universal compaction is supported), files in the last level, i.e. the ingested files, will not be included in any compaction. (#11489) +* Statistics `rocksdb.sst.read.micros` scope is expanded to all SST reads except for file ingestion and column family import (some compaction reads were previously excluded). + +### Bug Fixes +* Reduced cases of illegally using Env::Default() during static destruction by never destroying the internal PosixEnv itself (except for builds checking for memory leaks). (#11538) +* Fix extra prefetching during seek in async_io when BlockBasedTableOptions.num_file_reads_for_auto_readahead is 1 leading to extra reads than required. +* Fix a bug where compactions that are qualified to be run as 2 subcompactions were only run as one subcompaction. +* Fix a use-after-move bug in block.cc. + +## 8.3.0 (05/19/2023) +### New Features +* Introduced a new option `block_protection_bytes_per_key`, which can be used to enable per key-value integrity protection for in-memory blocks in block cache (#11287). +* Added `JemallocAllocatorOptions::num_arenas`. Setting `num_arenas > 1` may mitigate mutex contention in the allocator, particularly in scenarios where block allocations commonly bypass jemalloc tcache. +* Improve the operational safety of publishing a DB or SST files to many hosts by using different block cache hash seeds on different hosts. The exact behavior is controlled by new option `ShardedCacheOptions::hash_seed`, which also documents the solved problem in more detail. +* Introduced a new option `CompactionOptionsFIFO::file_temperature_age_thresholds` that allows FIFO compaction to compact files to different temperatures based on key age (#11428). +* Added a new ticker stat to count how many times RocksDB detected a corruption while verifying a block checksum: `BLOCK_CHECKSUM_MISMATCH_COUNT`. +* New statistics `rocksdb.file.read.db.open.micros` that measures read time of block-based SST tables or blob files during db open. +* New statistics tickers for various iterator seek behaviors and relevant filtering, as \*`_LEVEL_SEEK_`\*. (#11460) + +### Public API Changes +* EXPERIMENTAL: Add new API `DB::ClipColumnFamily` to clip the key in CF to a certain range. It will physically deletes all keys outside the range including tombstones. +* Add `MakeSharedCache()` construction functions to various cache Options objects, and deprecated the `NewWhateverCache()` functions with long parameter lists. +* Changed the meaning of various Bloom filter stats (prefix vs. whole key), with iterator-related filtering only being tracked in the new \*`_LEVEL_SEEK_`\*. stats. (#11460) + +### Behavior changes +* For x86, CPU features are no longer detected at runtime nor in build scripts, but in source code using common preprocessor defines. This will likely unlock some small performance improvements on some newer hardware, but could hurt performance of the kCRC32c checksum, which is no longer the default, on some "portable" builds. See PR #11419 for details. + +### Bug Fixes +* Delete an empty WAL file on DB open if the log number is less than the min log number to keep +* Delete temp OPTIONS file on DB open if there is a failure to write it out or rename it + +### Performance Improvements +* Improved the I/O efficiency of prefetching SST metadata by recording more information in the DB manifest. Opening files written with previous versions will still rely on heuristics for how much to prefetch (#11406). + +## 8.2.0 (04/24/2023) +### Public API Changes +* `SstFileWriter::DeleteRange()` now returns `Status::InvalidArgument` if the range's end key comes before its start key according to the user comparator. Previously the behavior was undefined. +* Add `multi_get_for_update` to C API. +* Remove unnecessary constructor for CompressionOptions. + +### Behavior changes +* Changed default block cache size from an 8MB to 32MB LRUCache, which increases the default number of cache shards from 16 to 64. This change is intended to minimize cache mutex contention under stress conditions. See https://github.com/facebook/rocksdb/wiki/Block-Cache for more information. +* For level compaction with `level_compaction_dynamic_level_bytes=true`, RocksDB now trivially moves levels down to fill LSM starting from bottommost level during DB open. See more in comments for option `level_compaction_dynamic_level_bytes` (#11321). +* User-provided `ReadOptions` take effect for more reads of non-`CacheEntryRole::kDataBlock` blocks. +* For level compaction with `level_compaction_dynamic_level_bytes=true`, RocksDB now drains unnecessary levels through background compaction automatically (#11340). This together with #11321 makes it automatic to migrate other compaction settings to level compaction with `level_compaction_dynamic_level_bytes=true`. In addition, a live DB that becomes smaller will now have unnecessary levels drained which can help to reduce read and space amp. +* If `CompactRange()` is called with `CompactRangeOptions::bottommost_level_compaction=kForce*` to compact from L0 to L1, RocksDB now will try to do trivial move from L0 to L1 and then do an intra L1 compaction, instead of a L0 to L1 compaction with trivial move disabled (#11375)). + +### Bug Fixes +* In the DB::VerifyFileChecksums API, ensure that file system reads of SST files are equal to the readahead_size in ReadOptions, if specified. Previously, each read was 2x the readahead_size. +* In block cache tracing, fixed some cases of bad hit/miss information (and more) with MultiGet. + +### New Features +* Add experimental `PerfContext` counters `iter_{next|prev|seek}_count` for db iterator, each counting the times of corresponding API being called. +* Allow runtime changes to whether `WriteBufferManager` allows stall or not by calling `SetAllowStall()` +* Added statistics tickers BYTES_COMPRESSED_FROM, BYTES_COMPRESSED_TO, BYTES_COMPRESSION_BYPASSED, BYTES_COMPRESSION_REJECTED, NUMBER_BLOCK_COMPRESSION_BYPASSED, and NUMBER_BLOCK_COMPRESSION_REJECTED. Disabled/deprecated histograms BYTES_COMPRESSED and BYTES_DECOMPRESSED, and ticker NUMBER_BLOCK_NOT_COMPRESSED. The new tickers offer more inight into compression ratios, rejected vs. disabled compression, etc. (#11388) +* New statistics `rocksdb.file.read.{flush|compaction}.micros` that measure read time of block-based SST tables or blob files during flush or compaction. + +## 8.1.0 (03/18/2023) +### Behavior changes +* Compaction output file cutting logic now considers range tombstone start keys. For example, SST partitioner now may receive ParitionRequest for range tombstone start keys. +* If the async_io ReadOption is specified for MultiGet or NewIterator on a platform that doesn't support IO uring, the option is ignored and synchronous IO is used. + +### Bug Fixes +* Fixed an issue for backward iteration when user defined timestamp is enabled in combination with BlobDB. +* Fixed a couple of cases where a Merge operand encountered during iteration wasn't reflected in the `internal_merge_count` PerfContext counter. +* Fixed a bug in CreateColumnFamilyWithImport()/ExportColumnFamily() which did not support range tombstones (#11252). +* Fixed a bug where an excluded column family from an atomic flush contains unflushed data that should've been included in this atomic flush (i.e, data of seqno less than the max seqno of this atomic flush), leading to potential data loss in this excluded column family when `WriteOptions::disableWAL == true` (#11148). + +### New Features +* Add statistics rocksdb.secondary.cache.filter.hits, rocksdb.secondary.cache.index.hits, and rocksdb.secondary.cache.filter.hits +* Added a new PerfContext counter `internal_merge_point_lookup_count` which tracks the number of Merge operands applied while serving point lookup queries. +* Add new statistics rocksdb.table.open.prefetch.tail.read.bytes, rocksdb.table.open.prefetch.tail.{miss|hit} +* Add support for SecondaryCache with HyperClockCache (`HyperClockCacheOptions` inherits `secondary_cache` option from `ShardedCacheOptions`) +* Add new db properties `rocksdb.cf-write-stall-stats`, `rocksdb.db-write-stall-stats`and APIs to examine them in a structured way. In particular, users of `GetMapProperty()` with property `kCFWriteStallStats`/`kDBWriteStallStats` can now use the functions in `WriteStallStatsMapKeys` to find stats in the map. + +### Public API Changes +* Changed various functions and features in `Cache` that are mostly relevant to custom implementations or wrappers. Especially, asychronous lookup functionality is moved from `Lookup()` to a new `StartAsyncLookup()` function. + +## 8.0.0 (02/19/2023) +### Behavior changes +* `ReadOptions::verify_checksums=false` disables checksum verification for more reads of non-`CacheEntryRole::kDataBlock` blocks. +* In case of scan with async_io enabled, if posix doesn't support IOUring, Status::NotSupported error will be returned to the users. Initially that error was swallowed and reads were switched to synchronous reads. + +### Bug Fixes +* Fixed a data race on `ColumnFamilyData::flush_reason` caused by concurrent flushes. +* Fixed an issue in `Get` and `MultiGet` when user-defined timestamps is enabled in combination with BlobDB. +* Fixed some atypical behaviors for `LockWAL()` such as allowing concurrent/recursive use and not expecting `UnlockWAL()` after non-OK result. See API comments. +* Fixed a feature interaction bug where for blobs `GetEntity` would expose the blob reference instead of the blob value. +* Fixed `DisableManualCompaction()` and `CompactRangeOptions::canceled` to cancel compactions even when they are waiting on conflicting compactions to finish +* Fixed a bug in which a successful `GetMergeOperands()` could transiently return `Status::MergeInProgress()` +* Return the correct error (Status::NotSupported()) to MultiGet caller when ReadOptions::async_io flag is true and IO uring is not enabled. Previously, Status::Corruption() was being returned when the actual failure was lack of async IO support. +* Fixed a bug in DB open/recovery from a compressed WAL that was caused due to incorrect handling of certain record fragments with the same offset within a WAL block. + +### Feature Removal +* Remove RocksDB Lite. +* The feature block_cache_compressed is removed. Statistics related to it are removed too. +* Remove deprecated Env::LoadEnv(). Use Env::CreateFromString() instead. +* Remove deprecated FileSystem::Load(). Use FileSystem::CreateFromString() instead. +* Removed the deprecated version of these utility functions and the corresponding Java bindings: `LoadOptionsFromFile`, `LoadLatestOptions`, `CheckOptionsCompatibility`. +* Remove the FactoryFunc from the LoadObject method from the Customizable helper methods. + +### Public API Changes +* Moved rarely-needed Cache class definition to new advanced_cache.h, and added a CacheWrapper class to advanced_cache.h. Minor changes to SimCache API definitions. +* Completely removed the following deprecated/obsolete statistics: the tickers `BLOCK_CACHE_INDEX_BYTES_EVICT`, `BLOCK_CACHE_FILTER_BYTES_EVICT`, `BLOOM_FILTER_MICROS`, `NO_FILE_CLOSES`, `STALL_L0_SLOWDOWN_MICROS`, `STALL_MEMTABLE_COMPACTION_MICROS`, `STALL_L0_NUM_FILES_MICROS`, `RATE_LIMIT_DELAY_MILLIS`, `NO_ITERATORS`, `NUMBER_FILTERED_DELETES`, `WRITE_TIMEDOUT`, `BLOB_DB_GC_NUM_KEYS_OVERWRITTEN`, `BLOB_DB_GC_NUM_KEYS_EXPIRED`, `BLOB_DB_GC_BYTES_OVERWRITTEN`, `BLOB_DB_GC_BYTES_EXPIRED`, `BLOCK_CACHE_COMPRESSION_DICT_BYTES_EVICT` as well as the histograms `STALL_L0_SLOWDOWN_COUNT`, `STALL_MEMTABLE_COMPACTION_COUNT`, `STALL_L0_NUM_FILES_COUNT`, `HARD_RATE_LIMIT_DELAY_COUNT`, `SOFT_RATE_LIMIT_DELAY_COUNT`, `BLOB_DB_GC_MICROS`, and `NUM_DATA_BLOCKS_READ_PER_LEVEL`. Note that as a result, the C++ enum values of the still supported statistics have changed. Developers are advised to not rely on the actual numeric values. +* Deprecated IngestExternalFileOptions::write_global_seqno and change default to false. This option only needs to be set to true to generate a DB compatible with RocksDB versions before 5.16.0. +* Remove deprecated APIs `GetColumnFamilyOptionsFrom{Map|String}(const ColumnFamilyOptions&, ..)`, `GetDBOptionsFrom{Map|String}(const DBOptions&, ..)`, `GetBlockBasedTableOptionsFrom{Map|String}(const BlockBasedTableOptions& table_options, ..)` and ` GetPlainTableOptionsFrom{Map|String}(const PlainTableOptions& table_options,..)`. +* Added a subcode of `Status::Corruption`, `Status::SubCode::kMergeOperatorFailed`, for users to identify corruption failures originating in the merge operator, as opposed to RocksDB's internally identified data corruptions + +### Build Changes +* The `make` build now builds a shared library by default instead of a static library. Use `LIB_MODE=static` to override. + +### New Features +* Compaction filters are now supported for wide-column entities by means of the `FilterV3` API. See the comment of the API for more details. +* Added `do_not_compress_roles` to `CompressedSecondaryCacheOptions` to disable compression on certain kinds of block. Filter blocks are now not compressed by CompressedSecondaryCache by default. +* Added a new `MultiGetEntity` API that enables batched wide-column point lookups. See the API comments for more details. + +## 7.10.0 (01/23/2023) +### Behavior changes +* Make best-efforts recovery verify SST unique ID before Version construction (#10962) +* Introduce `epoch_number` and sort L0 files by `epoch_number` instead of `largest_seqno`. `epoch_number` represents the order of a file being flushed or ingested/imported. Compaction output file will be assigned with the minimum `epoch_number` among input files'. For L0, larger `epoch_number` indicates newer L0 file. + +### Bug Fixes +* Fixed a regression in iterator where range tombstones after `iterate_upper_bound` is processed. +* Fixed a memory leak in MultiGet with async_io read option, caused by IO errors during table file open +* Fixed a bug that multi-level FIFO compaction deletes one file in non-L0 even when `CompactionOptionsFIFO::max_table_files_size` is no exceeded since #10348 or 7.8.0. +* Fixed a bug caused by `DB::SyncWAL()` affecting `track_and_verify_wals_in_manifest`. Without the fix, application may see "open error: Corruption: Missing WAL with log number" while trying to open the db. The corruption is a false alarm but prevents DB open (#10892). +* Fixed a BackupEngine bug in which RestoreDBFromLatestBackup would fail if the latest backup was deleted and there is another valid backup available. +* Fix L0 file misorder corruption caused by ingesting files of overlapping seqnos with memtable entries' through introducing `epoch_number`. Before the fix, `force_consistency_checks=true` may catch the corruption before it's exposed to readers, in which case writes returning `Status::Corruption` would be expected. Also replace the previous incomplete fix (#5958) to the same corruption with this new and more complete fix. +* Fixed a bug in LockWAL() leading to re-locking mutex (#11020). +* Fixed a heap use after free bug in async scan prefetching when the scan thread and another thread try to read and load the same seek block into cache. +* Fixed a heap use after free in async scan prefetching if dictionary compression is enabled, in which case sync read of the compression dictionary gets mixed with async prefetching +* Fixed a data race bug of `CompactRange()` under `change_level=true` acts on overlapping range with an ongoing file ingestion for level compaction. This will either result in overlapping file ranges corruption at a certain level caught by `force_consistency_checks=true` or protentially two same keys both with seqno 0 in two different levels (i.e, new data ends up in lower/older level). The latter will be caught by assertion in debug build but go silently and result in read returning wrong result in release build. This fix is general so it also replaced previous fixes to a similar problem for `CompactFiles()` (#4665), general `CompactRange()` and auto compaction (commit 5c64fb6 and 87dfc1d). +* Fixed a bug in compaction output cutting where small output files were produced due to TTL file cutting states were not being updated (#11075). + +### New Features +* When an SstPartitionerFactory is configured, CompactRange() now automatically selects for compaction any files overlapping a partition boundary that is in the compaction range, even if no actual entries are in the requested compaction range. With this feature, manual compaction can be used to (re-)establish SST partition points when SstPartitioner changes, without a full compaction. +* Add BackupEngine feature to exclude files from backup that are known to be backed up elsewhere, using `CreateBackupOptions::exclude_files_callback`. To restore the DB, the excluded files must be provided in alternative backup directories using `RestoreOptions::alternate_dirs`. + +### Public API Changes +* Substantial changes have been made to the Cache class to support internal development goals. Direct use of Cache class members is discouraged and further breaking modifications are expected in the future. SecondaryCache has some related changes and implementations will need to be updated. (Unlike Cache, SecondaryCache is still intended to support user implementations, and disruptive changes will be avoided.) (#10975) +* Add `MergeOperationOutput::op_failure_scope` for merge operator users to control the blast radius of merge operator failures. Existing merge operator users do not need to make any change to preserve the old behavior + +### Performance Improvements +* Updated xxHash source code, which should improve kXXH3 checksum speed, at least on ARM (#11098). +* Improved CPU efficiency of DB reads, from block cache access improvements (#10975). + +## 7.9.0 (11/21/2022) +### Performance Improvements +* Fixed an iterator performance regression for delete range users when scanning through a consecutive sequence of range tombstones (#10877). + +### Bug Fixes +* Fix memory corruption error in scans if async_io is enabled. Memory corruption happened if there is IOError while reading the data leading to empty buffer and other buffer already in progress of async read goes again for reading. +* Fix failed memtable flush retry bug that could cause wrongly ordered updates, which would surface to writers as `Status::Corruption` in case of `force_consistency_checks=true` (default). It affects use cases that enable both parallel flush (`max_background_flushes > 1` or `max_background_jobs >= 8`) and non-default memtable count (`max_write_buffer_number > 2`). +* Fixed an issue where the `READ_NUM_MERGE_OPERANDS` ticker was not updated when the base key-value or tombstone was read from an SST file. +* Fixed a memory safety bug when using a SecondaryCache with `block_cache_compressed`. `block_cache_compressed` no longer attempts to use SecondaryCache features. +* Fixed a regression in scan for async_io. During seek, valid buffers were getting cleared causing a regression. +* Tiered Storage: fixed excessive keys written to penultimate level in non-debug builds. + +### New Features +* Add basic support for user-defined timestamp to Merge (#10819). +* Add stats for ReadAsync time spent and async read errors. +* Basic support for the wide-column data model is now available. Wide-column entities can be stored using the `PutEntity` API, and retrieved using `GetEntity` and the new `columns` API of iterator. For compatibility, the classic APIs `Get` and `MultiGet`, as well as iterator's `value` API return the value of the anonymous default column of wide-column entities; also, `GetEntity` and iterator's `columns` return any plain key-values in the form of an entity which only has the anonymous default column. `Merge` (and `GetMergeOperands`) currently also apply to the default column; any other columns of entities are unaffected by `Merge` operations. Note that some features like compaction filters, transactions, user-defined timestamps, and the SST file writer do not yet support wide-column entities; also, there is currently no `MultiGet`-like API to retrieve multiple entities at once. We plan to gradually close the above gaps and also implement new features like column-level operations (e.g. updating or querying only certain columns of an entity). +* Marked HyperClockCache as a production-ready alternative to LRUCache for the block cache. HyperClockCache greatly improves hot-path CPU efficiency under high parallel load or high contention, with some documented caveats and limitations. As much as 4.5x higher ops/sec vs. LRUCache has been seen in db_bench under high parallel load. +* Add periodic diagnostics to info_log (LOG file) for HyperClockCache block cache if performance is degraded by bad `estimated_entry_charge` option. + +### Public API Changes +* Marked `block_cache_compressed` as a deprecated feature. Use SecondaryCache instead. +* Added a `SecondaryCache::InsertSaved()` API, with default implementation depending on `Insert()`. Some implementations might need to add a custom implementation of `InsertSaved()`. (Details in API comments.) + +## 7.8.0 (10/22/2022) +### New Features +* `DeleteRange()` now supports user-defined timestamp. +* Provide support for async_io with tailing iterators when ReadOptions.tailing is enabled during scans. +* Tiered Storage: allow data moving up from the last level to the penultimate level if the input level is penultimate level or above. +* Added `DB::Properties::kFastBlockCacheEntryStats`, which is similar to `DB::Properties::kBlockCacheEntryStats`, except returns cached (stale) values in more cases to reduce overhead. +* FIFO compaction now supports migrating from a multi-level DB via DB::Open(). During the migration phase, FIFO compaction picker will: +* picks the sst file with the smallest starting key in the bottom-most non-empty level. +* Note that during the migration phase, the file purge order will only be an approximation of "FIFO" as files in lower-level might sometime contain newer keys than files in upper-level. +* Added an option `ignore_max_compaction_bytes_for_input` to ignore max_compaction_bytes limit when adding files to be compacted from input level. This should help reduce write amplification. The option is enabled by default. +* Tiered Storage: allow data moving up from the last level even if it's a last level only compaction, as long as the penultimate level is empty. +* Add a new option IOOptions.do_not_recurse that can be used by underlying file systems to skip recursing through sub directories and list only files in GetChildren API. +* Add option `preserve_internal_time_seconds` to preserve the time information for the latest data. Which can be used to determine the age of data when `preclude_last_level_data_seconds` is enabled. The time information is attached with SST in table property `rocksdb.seqno.time.map` which can be parsed by tool ldb or sst_dump. + +### Bug Fixes +* Fix a bug in io_uring_prep_cancel in AbortIO API for posix which expects sqe->addr to match with read request submitted and wrong paramter was being passed. +* Fixed a regression in iterator performance when the entire DB is a single memtable introduced in #10449. The fix is in #10705 and #10716. +* Fixed an optimistic transaction validation bug caused by DBImpl::GetLatestSequenceForKey() returning non-latest seq for merge (#10724). +* Fixed a bug in iterator refresh which could segfault for DeleteRange users (#10739). +* Fixed a bug causing manual flush with `flush_opts.wait=false` to stall when database has stopped all writes (#10001). +* Fixed a bug in iterator refresh that was not freeing up SuperVersion, which could cause excessive resource pinniung (#10770). +* Fixed a bug where RocksDB could be doing compaction endlessly when allow_ingest_behind is true and the bottommost level is not filled (#10767). +* Fixed a memory safety bug in experimental HyperClockCache (#10768) +* Fixed some cases where `ldb update_manifest` and `ldb unsafe_remove_sst_file` are not usable because they were requiring the DB files to match the existing manifest state (before updating the manifest to match a desired state). + +### Performance Improvements +* Try to align the compaction output file boundaries to the next level ones, which can reduce more than 10% compaction load for the default level compaction. The feature is enabled by default, to disable, set `AdvancedColumnFamilyOptions.level_compaction_dynamic_file_size` to false. As a side effect, it can create SSTs larger than the target_file_size (capped at 2x target_file_size) or smaller files. +* Improve RoundRobin TTL compaction, which is going to be the same as normal RoundRobin compaction to move the compaction cursor. +* Fix a small CPU regression caused by a change that UserComparatorWrapper was made Customizable, because Customizable itself has small CPU overhead for initialization. + +### Behavior Changes +* Sanitize min_write_buffer_number_to_merge to 1 if atomic flush is enabled to prevent unexpected data loss when WAL is disabled in a multi-column-family setting (#10773). +* With periodic stat dumper waits up every options.stats_dump_period_sec seconds, it won't dump stats for a CF if it has no change in the period, unless 7 periods have been skipped. +* Only periodic stats dumper triggered by options.stats_dump_period_sec will update stats interval. Ones triggered by DB::GetProperty() will not update stats interval and will report based on an interval since the last time stats dump period. + +### Public API changes +* Make kXXH3 checksum the new default, because it is faster on common hardware, especially with kCRC32c affected by a performance bug in some versions of clang (https://github.com/facebook/rocksdb/issues/9891). DBs written with this new setting can be read by RocksDB 6.27 and newer. +* Refactor the classes, APIs and data structures for block cache tracing to allow a user provided trace writer to be used. Introduced an abstract BlockCacheTraceWriter class that takes a structured BlockCacheTraceRecord. The BlockCacheTraceWriter implementation can then format and log the record in whatever way it sees fit. The default BlockCacheTraceWriterImpl does file tracing using a user provided TraceWriter. More details in rocksdb/includb/block_cache_trace_writer.h. + +## 7.7.0 (09/18/2022) +### Bug Fixes +* Fixed a hang when an operation such as `GetLiveFiles` or `CreateNewBackup` is asked to trigger and wait for memtable flush on a read-only DB. Such indirect requests for memtable flush are now ignored on a read-only DB. +* Fixed bug where `FlushWAL(true /* sync */)` (used by `GetLiveFilesStorageInfo()`, which is used by checkpoint and backup) could cause parallel writes at the tail of a WAL file to never be synced. +* Fix periodic_task unable to re-register the same task type, which may cause `SetOptions()` fail to update periodical_task time like: `stats_dump_period_sec`, `stats_persist_period_sec`. +* Fixed a bug in the rocksdb.prefetched.bytes.discarded stat. It was counting the prefetch buffer size, rather than the actual number of bytes discarded from the buffer. +* Fix bug where the directory containing CURRENT can left unsynced after CURRENT is updated to point to the latest MANIFEST, which leads to risk of unsync data loss of CURRENT. +* Update rocksdb.multiget.io.batch.size stat in non-async MultiGet as well. +* Fix a bug in key range overlap checking with concurrent compactions when user-defined timestamp is enabled. User-defined timestamps should be EXCLUDED when checking if two ranges overlap. +* Fixed a bug where the blob cache prepopulating logic did not consider the secondary cache (see #10603). +* Fixed the rocksdb.num.sst.read.per.level, rocksdb.num.index.and.filter.blocks.read.per.level and rocksdb.num.level.read.per.multiget stats in the MultiGet coroutines + +### Public API changes +* Add `rocksdb_column_family_handle_get_id`, `rocksdb_column_family_handle_get_name` to get name, id of column family in C API +* Add a new stat rocksdb.async.prefetch.abort.micros to measure time spent waiting for async prefetch reads to abort + +### Java API Changes +* Add CompactionPriority.RoundRobin. +* Revert to using the default metadata charge policy when creating an LRU cache via the Java API. + +### Behavior Change +* DBOptions::verify_sst_unique_id_in_manifest is now an on-by-default feature that verifies SST file identity whenever they are opened by a DB, rather than only at DB::Open time. +* Right now, when the option migration tool (OptionChangeMigration()) migrates to FIFO compaction, it compacts all the data into one single SST file and move to L0. This might create a problem for some users: the giant file may be soon deleted to satisfy max_table_files_size, and might cayse the DB to be almost empty. We change the behavior so that the files are cut to be smaller, but these files might not follow the data insertion order. With the change, after the migration, migrated data might not be dropped by insertion order by FIFO compaction. +* When a block is firstly found from `CompressedSecondaryCache`, we just insert a dummy block into the primary cache and don’t erase the block from `CompressedSecondaryCache`. A standalone handle is returned to the caller. Only if the block is found again from `CompressedSecondaryCache` before the dummy block is evicted, we erase the block from `CompressedSecondaryCache` and insert it into the primary cache. +* When a block is firstly evicted from the primary cache to `CompressedSecondaryCache`, we just insert a dummy block in `CompressedSecondaryCache`. Only if it is evicted again before the dummy block is evicted from the cache, it is treated as a hot block and is inserted into `CompressedSecondaryCache`. +* Improved the estimation of memory used by cached blobs by taking into account the size of the object owning the blob value and also the allocator overhead if `malloc_usable_size` is available (see #10583). +* Blob values now have their own category in the cache occupancy statistics, as opposed to being lumped into the "Misc" bucket (see #10601). +* Change the optimize_multiget_for_io experimental ReadOptions flag to default on. + +### New Features +* RocksDB does internal auto prefetching if it notices 2 sequential reads if readahead_size is not specified. New option `num_file_reads_for_auto_readahead` is added in BlockBasedTableOptions which indicates after how many sequential reads internal auto prefetching should be start (default is 2). +* Added new perf context counters `block_cache_standalone_handle_count`, `block_cache_real_handle_count`,`compressed_sec_cache_insert_real_count`, `compressed_sec_cache_insert_dummy_count`, `compressed_sec_cache_uncompressed_bytes`, and `compressed_sec_cache_compressed_bytes`. +* Memory for blobs which are to be inserted into the blob cache is now allocated using the cache's allocator (see #10628 and #10647). +* HyperClockCache is an experimental, lock-free Cache alternative for block cache that offers much improved CPU efficiency under high parallel load or high contention, with some caveats. As much as 4.5x higher ops/sec vs. LRUCache has been seen in db_bench under high parallel load. +* `CompressedSecondaryCacheOptions::enable_custom_split_merge` is added for enabling the custom split and merge feature, which split the compressed value into chunks so that they may better fit jemalloc bins. + +### Performance Improvements +* Iterator performance is improved for `DeleteRange()` users. Internally, iterator will skip to the end of a range tombstone when possible, instead of looping through each key and check individually if a key is range deleted. +* Eliminated some allocations and copies in the blob read path. Also, `PinnableSlice` now only points to the blob value and pins the backing resource (cache entry or buffer) in all cases, instead of containing a copy of the blob value. See #10625 and #10647. +* In case of scans with async_io enabled, few optimizations have been added to issue more asynchronous requests in parallel in order to avoid synchronous prefetching. +* `DeleteRange()` users should see improvement in get/iterator performance from mutable memtable (see #10547). + +## 7.6.0 (08/19/2022) +### New Features +* Added `prepopulate_blob_cache` to ColumnFamilyOptions. If enabled, prepopulate warm/hot blobs which are already in memory into blob cache at the time of flush. On a flush, the blob that is in memory (in memtables) get flushed to the device. If using Direct IO, additional IO is incurred to read this blob back into memory again, which is avoided by enabling this option. This further helps if the workload exhibits high temporal locality, where most of the reads go to recently written data. This also helps in case of the remote file system since it involves network traffic and higher latencies. +* Support using secondary cache with the blob cache. When creating a blob cache, the user can set a secondary blob cache by configuring `secondary_cache` in LRUCacheOptions. +* Charge memory usage of blob cache when the backing cache of the blob cache and the block cache are different. If an operation reserving memory for blob cache exceeds the avaible space left in the block cache at some point (i.e, causing a cache full under `LRUCacheOptions::strict_capacity_limit` = true), creation will fail with `Status::MemoryLimit()`. To opt in this feature, enable charging `CacheEntryRole::kBlobCache` in `BlockBasedTableOptions::cache_usage_options`. +* Improve subcompaction range partition so that it is likely to be more even. More evenly distribution of subcompaction will improve compaction throughput for some workloads. All input files' index blocks to sample some anchor key points from which we pick positions to partition the input range. This would introduce some CPU overhead in compaction preparation phase, if subcompaction is enabled, but it should be a small fraction of the CPU usage of the whole compaction process. This also brings a behavier change: subcompaction number is much more likely to maxed out than before. +* Add CompactionPri::kRoundRobin, a compaction picking mode that cycles through all the files with a compact cursor in a round-robin manner. This feature is available since 7.5. +* Provide support for subcompactions for user_defined_timestamp. +* Added an option `memtable_protection_bytes_per_key` that turns on memtable per key-value checksum protection. Each memtable entry will be suffixed by a checksum that is computed during writes, and verified in reads/compaction. Detected corruption will be logged and with corruption status returned to user. +* Added a blob-specific cache priority level - bottom level. Blobs are typically lower-value targets for caching than data blocks, since 1) with BlobDB, data blocks containing blob references conceptually form an index structure which has to be consulted before we can read the blob value, and 2) cached blobs represent only a single key-value, while cached data blocks generally contain multiple KVs. The user can specify the new option `low_pri_pool_ratio` in `LRUCacheOptions` to configure the ratio of capacity reserved for low priority cache entries (and therefore the remaining ratio is the space reserved for the bottom level), or configuring the new argument `low_pri_pool_ratio` in `NewLRUCache()` to achieve the same effect. + +### Public API changes +* Removed Customizable support for RateLimiter and removed its CreateFromString() and Type() functions. +* `CompactRangeOptions::exclusive_manual_compaction` is now false by default. This ensures RocksDB does not introduce artificial parallelism limitations by default. +* Tiered Storage: change `bottommost_temperture` to `last_level_temperture`. The old option name is kept only for migration, please use the new option. The behavior is changed to apply temperature for the `last_level` SST files only. +* Added a new experimental ReadOption flag called optimize_multiget_for_io, which when set attempts to reduce MultiGet latency by spawning coroutines for keys in multiple levels. + +### Bug Fixes +* Fix a bug starting in 7.4.0 in which some fsync operations might be skipped in a DB after any DropColumnFamily on that DB, until it is re-opened. This can lead to data loss on power loss. (For custom FileSystem implementations, this could lead to `FSDirectory::Fsync` or `FSDirectory::Close` after the first `FSDirectory::Close`; Also, valgrind could report call to `close()` with `fd=-1`.) +* Fix a bug where `GenericRateLimiter` could revert the bandwidth set dynamically using `SetBytesPerSecond()` when a user configures a structure enclosing it, e.g., using `GetOptionsFromString()` to configure an `Options` that references an existing `RateLimiter` object. +* Fix race conditions in `GenericRateLimiter`. +* Fix a bug in `FIFOCompactionPicker::PickTTLCompaction` where total_size calculating might cause underflow +* Fix data race bug in hash linked list memtable. With this bug, read request might temporarily miss an old record in the memtable in a race condition to the hash bucket. +* Fix a bug that `best_efforts_recovery` may fail to open the db with mmap read. +* Fixed a bug where blobs read during compaction would pollute the cache. +* Fixed a data race in LRUCache when used with a secondary_cache. +* Fixed a bug where blobs read by iterators would be inserted into the cache even with the `fill_cache` read option set to false. +* Fixed the segfault caused by `AllocateData()` in `CompressedSecondaryCache::SplitValueIntoChunks()` and `MergeChunksIntoValueTest`. +* Fixed a bug in BlobDB where a mix of inlined and blob values could result in an incorrect value being passed to the compaction filter (see #10391). +* Fixed a memory leak bug in stress tests caused by `FaultInjectionSecondaryCache`. + +### Behavior Change +* Added checksum handshake during the copying of decompressed WAL fragment. This together with #9875, #10037, #10212, #10114 and #10319 provides end-to-end integrity protection for write batch during recovery. +* To minimize the internal fragmentation caused by the variable size of the compressed blocks in `CompressedSecondaryCache`, the original block is split according to the jemalloc bin size in `Insert()` and then merged back in `Lookup()`. +* PosixLogger is removed and by default EnvLogger will be used for info logging. The behavior of the two loggers should be very similar when using the default Posix Env. +* Remove [min|max]_timestamp from VersionEdit for now since they are not tracked in MANIFEST anyway but consume two empty std::string (up to 64 bytes) for each file. Should they be added back in the future, we should store them more compactly. +* Improve universal tiered storage compaction picker to avoid extra major compaction triggered by size amplification. If `preclude_last_level_data_seconds` is enabled, the size amplification is calculated within non last_level data only which skip the last level and use the penultimate level as the size base. +* If an error is hit when writing to a file (append, sync, etc), RocksDB is more strict with not issuing more operations to it, except closing the file, with exceptions of some WAL file operations in error recovery path. +* A `WriteBufferManager` constructed with `allow_stall == false` will no longer trigger write stall implicitly by thrashing until memtable count limit is reached. Instead, a column family can continue accumulating writes while that CF is flushing, which means memory may increase. Users who prefer stalling writes must now explicitly set `allow_stall == true`. +* Add `CompressedSecondaryCache` into the stress tests. +* Block cache keys have changed, which will cause any persistent caches to miss between versions. + +### Performance Improvements +* Instead of constructing `FragmentedRangeTombstoneList` during every read operation, it is now constructed once and stored in immutable memtables. This improves speed of querying range tombstones from immutable memtables. +* When using iterators with the integrated BlobDB implementation, blob cache handles are now released immediately when the iterator's position changes. +* MultiGet can now do more IO in parallel by reading data blocks from SST files in multiple levels, if the optimize_multiget_for_io ReadOption flag is set. + +## 7.5.0 (07/15/2022) +### New Features +* Mempurge option flag `experimental_mempurge_threshold` is now a ColumnFamilyOptions and can now be dynamically configured using `SetOptions()`. +* Support backward iteration when `ReadOptions::iter_start_ts` is set. +* Provide support for ReadOptions.async_io with direct_io to improve Seek latency by using async IO to parallelize child iterator seek and doing asynchronous prefetching on sequential scans. +* Added support for blob caching in order to cache frequently used blobs for BlobDB. + * User can configure the new ColumnFamilyOptions `blob_cache` to enable/disable blob caching. + * Either sharing the backend cache with the block cache or using a completely separate cache is supported. + * A new abstraction interface called `BlobSource` for blob read logic gives all users access to blobs, whether they are in the blob cache, secondary cache, or (remote) storage. Blobs can be potentially read both while handling user reads (`Get`, `MultiGet`, or iterator) and during compaction (while dealing with compaction filters, Merges, or garbage collection) but eventually all blob reads go through `Version::GetBlob` or, for MultiGet, `Version::MultiGetBlob` (and then get dispatched to the interface -- `BlobSource`). +* Add experimental tiered compaction feature `AdvancedColumnFamilyOptions::preclude_last_level_data_seconds`, which makes sure the new data inserted within preclude_last_level_data_seconds won't be placed on cold tier (the feature is not complete). + +### Public API changes +* Add metadata related structs and functions in C API, including + * `rocksdb_get_column_family_metadata()` and `rocksdb_get_column_family_metadata_cf()` to obtain `rocksdb_column_family_metadata_t`. + * `rocksdb_column_family_metadata_t` and its get functions & destroy function. + * `rocksdb_level_metadata_t` and its and its get functions & destroy function. + * `rocksdb_file_metadata_t` and its and get functions & destroy functions. +* Add suggest_compact_range() and suggest_compact_range_cf() to C API. +* When using block cache strict capacity limit (`LRUCache` with `strict_capacity_limit=true`), DB operations now fail with Status code `kAborted` subcode `kMemoryLimit` (`IsMemoryLimit()`) instead of `kIncomplete` (`IsIncomplete()`) when the capacity limit is reached, because Incomplete can mean other specific things for some operations. In more detail, `Cache::Insert()` now returns the updated Status code and this usually propagates through RocksDB to the user on failure. +* NewClockCache calls temporarily return an LRUCache (with similar characteristics as the desired ClockCache). This is because ClockCache is being replaced by a new version (the old one had unknown bugs) but this is still under development. +* Add two functions `int ReserveThreads(int threads_to_be_reserved)` and `int ReleaseThreads(threads_to_be_released)` into `Env` class. In the default implementation, both return 0. Newly added `xxxEnv` class that inherits `Env` should implement these two functions for thread reservation/releasing features. +* Add `rocksdb_options_get_prepopulate_blob_cache` and `rocksdb_options_set_prepopulate_blob_cache` to C API. +* Add `prepopulateBlobCache` and `setPrepopulateBlobCache` to Java API. + +### Bug Fixes +* Fix a bug in which backup/checkpoint can include a WAL deleted by RocksDB. +* Fix a bug where concurrent compactions might cause unnecessary further write stalling. In some cases, this might cause write rate to drop to minimum. +* Fix a bug in Logger where if dbname and db_log_dir are on different filesystems, dbname creation would fail wrt to db_log_dir path returning an error and fails to open the DB. +* Fix a CPU and memory efficiency issue introduce by https://github.com/facebook/rocksdb/pull/8336 which made InternalKeyComparator configurable as an unintended side effect. + +## Behavior Change +* In leveled compaction with dynamic levelling, level multiplier is not anymore adjusted due to oversized L0. Instead, compaction score is adjusted by increasing size level target by adding incoming bytes from upper levels. This would deprioritize compactions from upper levels if more data from L0 is coming. This is to fix some unnecessary full stalling due to drastic change of level targets, while not wasting write bandwidth for compaction while writes are overloaded. +* For track_and_verify_wals_in_manifest, revert to the original behavior before #10087: syncing of live WAL file is not tracked, and we track only the synced sizes of **closed** WALs. (PR #10330). +* WAL compression now computes/verifies checksum during compression/decompression. + +### Performance Improvements +* Rather than doing total sort against all files in a level, SortFileByOverlappingRatio() to only find the top 50 files based on score. This can improve write throughput for the use cases where data is loaded in increasing key order and there are a lot of files in one LSM-tree, where applying compaction results is the bottleneck. +* In leveled compaction, L0->L1 trivial move will allow more than one file to be moved in one compaction. This would allow L0 files to be moved down faster when data is loaded in sequential order, making slowdown or stop condition harder to hit. Also seek L0->L1 trivial move when only some files qualify. +* In leveled compaction, try to trivial move more than one files if possible, up to 4 files or max_compaction_bytes. This is to allow higher write throughput for some use cases where data is loaded in sequential order, where appying compaction results is the bottleneck. + +## 7.4.0 (06/19/2022) +### Bug Fixes +* Fixed a bug in calculating key-value integrity protection for users of in-place memtable updates. In particular, the affected users would be those who configure `protection_bytes_per_key > 0` on `WriteBatch` or `WriteOptions`, and configure `inplace_callback != nullptr`. +* Fixed a bug where a snapshot taken during SST file ingestion would be unstable. +* Fixed a bug for non-TransactionDB with avoid_flush_during_recovery = true and TransactionDB where in case of crash, min_log_number_to_keep may not change on recovery and persisting a new MANIFEST with advanced log_numbers for some column families, results in "column family inconsistency" error on second recovery. As a solution, RocksDB will persist the new MANIFEST after successfully syncing the new WAL. If a future recovery starts from the new MANIFEST, then it means the new WAL is successfully synced. Due to the sentinel empty write batch at the beginning, kPointInTimeRecovery of WAL is guaranteed to go after this point. If future recovery starts from the old MANIFEST, it means the writing the new MANIFEST failed. We won't have the "SST ahead of WAL" error. +* Fixed a bug where RocksDB DB::Open() may creates and writes to two new MANIFEST files even before recovery succeeds. Now writes to MANIFEST are persisted only after recovery is successful. +* Fix a race condition in WAL size tracking which is caused by an unsafe iterator access after container is changed. +* Fix unprotected concurrent accesses to `WritableFileWriter::filesize_` by `DB::SyncWAL()` and `DB::Put()` in two write queue mode. +* Fix a bug in WAL tracking. Before this PR (#10087), calling `SyncWAL()` on the only WAL file of the db will not log the event in MANIFEST, thus allowing a subsequent `DB::Open` even if the WAL file is missing or corrupted. +* Fix a bug that could return wrong results with `index_type=kHashSearch` and using `SetOptions` to change the `prefix_extractor`. +* Fixed a bug in WAL tracking with wal_compression. WAL compression writes a kSetCompressionType record which is not associated with any sequence number. As result, WalManager::GetSortedWalsOfType() will skip these WALs and not return them to caller, e.g. Checkpoint, Backup, causing the operations to fail. +* Avoid a crash if the IDENTITY file is accidentally truncated to empty. A new DB ID will be written and generated on Open. +* Fixed a possible corruption for users of `manual_wal_flush` and/or `FlushWAL(true /* sync */)`, together with `track_and_verify_wals_in_manifest == true`. For those users, losing unsynced data (e.g., due to power loss) could make future DB opens fail with a `Status::Corruption` complaining about missing WAL data. +* Fixed a bug in `WriteBatchInternal::Append()` where WAL termination point in write batch was not considered and the function appends an incorrect number of checksums. +* Fixed a crash bug introduced in 7.3.0 affecting users of MultiGet with `kDataBlockBinaryAndHash`. + +### Public API changes +* Add new API GetUnixTime in Snapshot class which returns the unix time at which Snapshot is taken. +* Add transaction `get_pinned` and `multi_get` to C API. +* Add two-phase commit support to C API. +* Add `rocksdb_transaction_get_writebatch_wi` and `rocksdb_transaction_rebuild_from_writebatch` to C API. +* Add `rocksdb_options_get_blob_file_starting_level` and `rocksdb_options_set_blob_file_starting_level` to C API. +* Add `blobFileStartingLevel` and `setBlobFileStartingLevel` to Java API. +* Add SingleDelete for DB in C API +* Add User Defined Timestamp in C API. + * `rocksdb_comparator_with_ts_create` to create timestamp aware comparator + * Put, Get, Delete, SingleDelete, MultiGet APIs has corresponding timestamp aware APIs with suffix `with_ts` + * And Add C API's for Transaction, SstFileWriter, Compaction as mentioned [here](https://github.com/facebook/rocksdb/wiki/User-defined-Timestamp-(Experimental)) +* The contract for implementations of Comparator::IsSameLengthImmediateSuccessor has been updated to work around a design bug in `auto_prefix_mode`. +* The API documentation for `auto_prefix_mode` now notes some corner cases in which it returns different results than `total_order_seek`, due to design bugs that are not easily fixed. Users using built-in comparators and keys at least the size of a fixed prefix length are not affected. +* Obsoleted the NUM_DATA_BLOCKS_READ_PER_LEVEL stat and introduced the NUM_LEVEL_READ_PER_MULTIGET and MULTIGET_COROUTINE_COUNT stats +* Introduced `WriteOptions::protection_bytes_per_key`, which can be used to enable key-value integrity protection for live updates. + +### New Features +* Add FileSystem::ReadAsync API in io_tracing +* Add blob garbage collection parameters `blob_garbage_collection_policy` and `blob_garbage_collection_age_cutoff` to both force-enable and force-disable GC, as well as selectively override age cutoff when using CompactRange. +* Add an extra sanity check in `GetSortedWalFiles()` (also used by `GetLiveFilesStorageInfo()`, `BackupEngine`, and `Checkpoint`) to reduce risk of successfully created backup or checkpoint failing to open because of missing WAL file. +* Add a new column family option `blob_file_starting_level` to enable writing blob files during flushes and compactions starting from the specified LSM tree level. +* Add support for timestamped snapshots (#9879) +* Provide support for AbortIO in posix to cancel submitted asynchronous requests using io_uring. +* Add support for rate-limiting batched `MultiGet()` APIs +* Added several new tickers, perf context statistics, and DB properties to BlobDB + * Added new DB properties "rocksdb.blob-cache-capacity", "rocksdb.blob-cache-usage", "rocksdb.blob-cache-pinned-usage" to show blob cache usage. + * Added new perf context statistics `blob_cache_hit_count`, `blob_read_count`, `blob_read_byte`, `blob_read_time`, `blob_checksum_time` and `blob_decompress_time`. + * Added new tickers `BLOB_DB_CACHE_MISS`, `BLOB_DB_CACHE_HIT`, `BLOB_DB_CACHE_ADD`, `BLOB_DB_CACHE_ADD_FAILURES`, `BLOB_DB_CACHE_BYTES_READ` and `BLOB_DB_CACHE_BYTES_WRITE`. + +### Behavior changes +* DB::Open(), DB::OpenAsSecondary() will fail if a Logger cannot be created (#9984) +* DB::Write does not hold global `mutex_` if this db instance does not need to switch wal and mem-table (#7516). +* Removed support for reading Bloom filters using obsolete block-based filter format. (Support for writing such filters was dropped in 7.0.) For good read performance on old DBs using these filters, a full compaction is required. +* Per KV checksum in write batch is verified before a write batch is written to WAL to detect any corruption to the write batch (#10114). + +### Performance Improvements +* When compiled with folly (Meta-internal integration; experimental in open source build), improve the locking performance (CPU efficiency) of LRUCache by using folly DistributedMutex in place of standard mutex. + +## 7.3.0 (05/20/2022) +### Bug Fixes +* Fixed a bug where manual flush would block forever even though flush options had wait=false. +* Fixed a bug where RocksDB could corrupt DBs with `avoid_flush_during_recovery == true` by removing valid WALs, leading to `Status::Corruption` with message like "SST file is ahead of WALs" when attempting to reopen. +* Fixed a bug in async_io path where incorrect length of data is read by FilePrefetchBuffer if data is consumed from two populated buffers and request for more data is sent. +* Fixed a CompactionFilter bug. Compaction filter used to use `Delete` to remove keys, even if the keys should be removed with `SingleDelete`. Mixing `Delete` and `SingleDelete` may cause undefined behavior. +* Fixed a bug in `WritableFileWriter::WriteDirect` and `WritableFileWriter::WriteDirectWithChecksum`. The rate_limiter_priority specified in ReadOptions was not passed to the RateLimiter when requesting a token. +* Fixed a bug which might cause process crash when I/O error happens when reading an index block in MultiGet(). + +### New Features +* DB::GetLiveFilesStorageInfo is ready for production use. +* Add new stats PREFETCHED_BYTES_DISCARDED which records number of prefetched bytes discarded by RocksDB FilePrefetchBuffer on destruction and POLL_WAIT_MICROS records wait time for FS::Poll API completion. +* RemoteCompaction supports table_properties_collector_factories override on compaction worker. +* Start tracking SST unique id in MANIFEST, which will be used to verify with SST properties during DB open to make sure the SST file is not overwritten or misplaced. A db option `verify_sst_unique_id_in_manifest` is introduced to enable/disable the verification, if enabled all SST files will be opened during DB-open to verify the unique id (default is false), so it's recommended to use it with `max_open_files = -1` to pre-open the files. +* Added the ability to concurrently read data blocks from multiple files in a level in batched MultiGet. This can be enabled by setting the async_io option in ReadOptions. Using this feature requires a FileSystem that supports ReadAsync (PosixFileSystem is not supported yet for this), and for RocksDB to be compiled with folly and c++20. +* Charge memory usage of file metadata. RocksDB holds one file metadata structure in-memory per on-disk table file. If an operation reserving memory for file metadata exceeds the avaible space left in the block +cache at some point (i.e, causing a cache full under `LRUCacheOptions::strict_capacity_limit` = true), creation will fail with `Status::MemoryLimit()`. To opt in this feature, enable charging `CacheEntryRole::kFileMetadata` in `BlockBasedTableOptions::cache_usage_options`. + +### Public API changes +* Add rollback_deletion_type_callback to TransactionDBOptions so that write-prepared transactions know whether to issue a Delete or SingleDelete to cancel a previous key written during prior prepare phase. The PR aims to prevent mixing SingleDeletes and Deletes for the same key that can lead to undefined behaviors for write-prepared transactions. +* EXPERIMENTAL: Add new API AbortIO in file_system to abort the read requests submitted asynchronously. +* CompactionFilter::Decision has a new value: kRemoveWithSingleDelete. If CompactionFilter returns this decision, then CompactionIterator will use `SingleDelete` to mark a key as removed. +* Renamed CompactionFilter::Decision::kRemoveWithSingleDelete to kPurge since the latter sounds more general and hides the implementation details of how compaction iterator handles keys. +* Added ability to specify functions for Prepare and Validate to OptionsTypeInfo. Added methods to OptionTypeInfo to set the functions via an API. These methods are intended for RocksDB plugin developers for configuration management. +* Added a new immutable db options, enforce_single_del_contracts. If set to false (default is true), compaction will NOT fail due to a single delete followed by a delete for the same key. The purpose of this temporay option is to help existing use cases migrate. +* Introduce `BlockBasedTableOptions::cache_usage_options` and use that to replace `BlockBasedTableOptions::reserve_table_builder_memory` and `BlockBasedTableOptions::reserve_table_reader_memory`. +* Changed `GetUniqueIdFromTableProperties` to return a 128-bit unique identifier, which will be the standard size now. The old functionality (192-bit) is available from `GetExtendedUniqueIdFromTableProperties`. Both functions are no longer "experimental" and are ready for production use. +* In IOOptions, mark `prio` as deprecated for future removal. +* In `file_system.h`, mark `IOPriority` as deprecated for future removal. +* Add an option, `CompressionOptions::use_zstd_dict_trainer`, to indicate whether zstd dictionary trainer should be used for generating zstd compression dictionaries. The default value of this option is true for backward compatibility. When this option is set to false, zstd API `ZDICT_finalizeDictionary` is used to generate compression dictionaries. +* Seek API which positions itself every LevelIterator on the correct data block in the correct SST file which can be parallelized if ReadOptions.async_io option is enabled. +* Add new stat number_async_seek in PerfContext that indicates number of async calls made by seek to prefetch data. +* Add support for user-defined timestamps to read only DB. + +### Bug Fixes +* RocksDB calls FileSystem::Poll API during FilePrefetchBuffer destruction which impacts performance as it waits for read requets completion which is not needed anymore. Calling FileSystem::AbortIO to abort those requests instead fixes that performance issue. +* Fixed unnecessary block cache contention when queries within a MultiGet batch and across parallel batches access the same data block, which previously could cause severely degraded performance in this unusual case. (In more typical MultiGet cases, this fix is expected to yield a small or negligible performance improvement.) + +### Behavior changes +* Enforce the existing contract of SingleDelete so that SingleDelete cannot be mixed with Delete because it leads to undefined behavior. Fix a number of unit tests that violate the contract but happen to pass. +* ldb `--try_load_options` default to true if `--db` is specified and not creating a new DB, the user can still explicitly disable that by `--try_load_options=false` (or explicitly enable that by `--try_load_options`). +* During Flush write or Compaction write/read, the WriteController is used to determine whether DB writes are stalled or slowed down. The priority (Env::IOPriority) can then be determined accordingly and be passed in IOOptions to the file system. + +### Performance Improvements +* Avoid calling malloc_usable_size() in LRU Cache's mutex. +* Reduce DB mutex holding time when finding obsolete files to delete. When a file is trivial moved to another level, the internal files will be referenced twice internally and sometimes opened twice too. If a deletion candidate file is not the last reference, we need to destroy the reference and close the file but not deleting the file. Right now we determine it by building a set of all live files. With the improvement, we check the file against all live LSM-tree versions instead. + +## 7.2.0 (04/15/2022) +### Bug Fixes +* Fixed bug which caused rocksdb failure in the situation when rocksdb was accessible using UNC path +* Fixed a race condition when 2PC is disabled and WAL tracking in the MANIFEST is enabled. The race condition is between two background flush threads trying to install flush results, causing a WAL deletion not tracked in the MANIFEST. A future DB open may fail. +* Fixed a heap use-after-free race with DropColumnFamily. +* Fixed a bug that `rocksdb.read.block.compaction.micros` cannot track compaction stats (#9722). +* Fixed `file_type`, `relative_filename` and `directory` fields returned by `GetLiveFilesMetaData()`, which were added in inheriting from `FileStorageInfo`. +* Fixed a bug affecting `track_and_verify_wals_in_manifest`. Without the fix, application may see "open error: Corruption: Missing WAL with log number" while trying to open the db. The corruption is a false alarm but prevents DB open (#9766). +* Fix segfault in FilePrefetchBuffer with async_io as it doesn't wait for pending jobs to complete on destruction. +* Fix ERROR_HANDLER_AUTORESUME_RETRY_COUNT stat whose value was set wrong in portal.h +* Fixed a bug for non-TransactionDB with avoid_flush_during_recovery = true and TransactionDB where in case of crash, min_log_number_to_keep may not change on recovery and persisting a new MANIFEST with advanced log_numbers for some column families, results in "column family inconsistency" error on second recovery. As a solution the corrupted WALs whose numbers are larger than the corrupted wal and smaller than the new WAL will be moved to archive folder. +* Fixed a bug in RocksDB DB::Open() which may creates and writes to two new MANIFEST files even before recovery succeeds. Now writes to MANIFEST are persisted only after recovery is successful. + +### New Features +* For db_bench when --seed=0 or --seed is not set then it uses the current time as the seed value. Previously it used the value 1000. +* For db_bench when --benchmark lists multiple tests and each test uses a seed for a RNG then the seeds across tests will no longer be repeated. +* Added an option to dynamically charge an updating estimated memory usage of block-based table reader to block cache if block cache available. To enable this feature, set `BlockBasedTableOptions::reserve_table_reader_memory = true`. +* Add new stat ASYNC_READ_BYTES that calculates number of bytes read during async read call and users can check if async code path is being called by RocksDB internal automatic prefetching for sequential reads. +* Enable async prefetching if ReadOptions.readahead_size is set along with ReadOptions.async_io in FilePrefetchBuffer. +* Add event listener support on remote compaction compactor side. +* Added a dedicated integer DB property `rocksdb.live-blob-file-garbage-size` that exposes the total amount of garbage in the blob files in the current version. +* RocksDB does internal auto prefetching if it notices sequential reads. It starts with readahead size `initial_auto_readahead_size` which now can be configured through BlockBasedTableOptions. +* Add a merge operator that allows users to register specific aggregation function so that they can does aggregation using different aggregation types for different keys. See comments in include/rocksdb/utilities/agg_merge.h for actual usage. The feature is experimental and the format is subject to change and we won't provide a migration tool. +* Meta-internal / Experimental: Improve CPU performance by replacing many uses of std::unordered_map with folly::F14FastMap when RocksDB is compiled together with Folly. +* Experimental: Add CompressedSecondaryCache, a concrete implementation of rocksdb::SecondaryCache, that integrates with compression libraries (e.g. LZ4) to hold compressed blocks. + +### Behavior changes +* Disallow usage of commit-time-write-batch for write-prepared/write-unprepared transactions if TransactionOptions::use_only_the_last_commit_time_batch_for_recovery is false to prevent two (or more) uncommitted versions of the same key in the database. Otherwise, bottommost compaction may violate the internal key uniqueness invariant of SSTs if the sequence numbers of both internal keys are zeroed out (#9794). +* Make DB::GetUpdatesSince() return NotSupported early for write-prepared/write-unprepared transactions, as the API contract indicates. + +### Public API changes +* Exposed APIs to examine results of block cache stats collections in a structured way. In particular, users of `GetMapProperty()` with property `kBlockCacheEntryStats` can now use the functions in `BlockCacheEntryStatsMapKeys` to find stats in the map. +* Add `fail_if_not_bottommost_level` to IngestExternalFileOptions so that ingestion will fail if the file(s) cannot be ingested to the bottommost level. +* Add output parameter `is_in_sec_cache` to `SecondaryCache::Lookup()`. It is to indicate whether the handle is possibly erased from the secondary cache after the Lookup. + +## 7.1.0 (03/23/2022) +### New Features +* Allow WriteBatchWithIndex to index a WriteBatch that includes keys with user-defined timestamps. The index itself does not have timestamp. +* Add support for user-defined timestamps to write-committed transaction without API change. The `TransactionDB` layer APIs do not allow timestamps because we require that all user-defined-timestamps-aware operations go through the `Transaction` APIs. +* Added BlobDB options to `ldb` +* `BlockBasedTableOptions::detect_filter_construct_corruption` can now be dynamically configured using `DB::SetOptions`. +* Automatically recover from retryable read IO errors during backgorund flush/compaction. +* Experimental support for preserving file Temperatures through backup and restore, and for updating DB metadata for outside changes to file Temperature (`UpdateManifestForFilesState` or `ldb update_manifest --update_temperatures`). +* Experimental support for async_io in ReadOptions which is used by FilePrefetchBuffer to prefetch some of the data asynchronously, if reads are sequential and auto readahead is enabled by rocksdb internally. + +### Bug Fixes +* Fixed a major performance bug in which Bloom filters generated by pre-7.0 releases are not read by early 7.0.x releases (and vice-versa) due to changes to FilterPolicy::Name() in #9590. This can severely impact read performance and read I/O on upgrade or downgrade with existing DB, but not data correctness. +* Fixed a data race on `versions_` between `DBImpl::ResumeImpl()` and threads waiting for recovery to complete (#9496) +* Fixed a bug caused by race among flush, incoming writes and taking snapshots. Queries to snapshots created with these race condition can return incorrect result, e.g. resurfacing deleted data. +* Fixed a bug that DB flush uses `options.compression` even `options.compression_per_level` is set. +* Fixed a bug that DisableManualCompaction may assert when disable an unscheduled manual compaction. +* Fix a race condition when cancel manual compaction with `DisableManualCompaction`. Also DB close can cancel the manual compaction thread. +* Fixed a potential timer crash when open close DB concurrently. +* Fixed a race condition for `alive_log_files_` in non-two-write-queues mode. The race is between the write_thread_ in WriteToWAL() and another thread executing `FindObsoleteFiles()`. The race condition will be caught if `__glibcxx_requires_nonempty` is enabled. +* Fixed a bug that `Iterator::Refresh()` reads stale keys after DeleteRange() performed. +* Fixed a race condition when disable and re-enable manual compaction. +* Fixed automatic error recovery failure in atomic flush. +* Fixed a race condition when mmaping a WritableFile on POSIX. + +### Public API changes +* Added pure virtual FilterPolicy::CompatibilityName(), which is needed for fixing major performance bug involving FilterPolicy naming in SST metadata without affecting Customizable aspect of FilterPolicy. This change only affects those with their own custom or wrapper FilterPolicy classes. +* `options.compression_per_level` is dynamically changeable with `SetOptions()`. +* Added `WriteOptions::rate_limiter_priority`. When set to something other than `Env::IO_TOTAL`, the internal rate limiter (`DBOptions::rate_limiter`) will be charged at the specified priority for writes associated with the API to which the `WriteOptions` was provided. Currently the support covers automatic WAL flushes, which happen during live updates (`Put()`, `Write()`, `Delete()`, etc.) when `WriteOptions::disableWAL == false` and `DBOptions::manual_wal_flush == false`. +* Add DB::OpenAndTrimHistory API. This API will open DB and trim data to the timestamp specified by trim_ts (The data with timestamp larger than specified trim bound will be removed). This API should only be used at a timestamp-enabled column families recovery. If the column family doesn't have timestamp enabled, this API won't trim any data on that column family. This API is not compatible with avoid_flush_during_recovery option. +* Remove BlockBasedTableOptions.hash_index_allow_collision which already takes no effect. + +## 7.0.0 (02/20/2022) +### Bug Fixes +* Fixed a major bug in which batched MultiGet could return old values for keys deleted by DeleteRange when memtable Bloom filter is enabled (memtable_prefix_bloom_size_ratio > 0). (The fix includes a substantial MultiGet performance improvement in the unusual case of both memtable_whole_key_filtering and prefix_extractor.) +* Fixed more cases of EventListener::OnTableFileCreated called with OK status, file_size==0, and no SST file kept. Now the status is Aborted. +* Fixed a read-after-free bug in `DB::GetMergeOperands()`. +* Fix a data loss bug for 2PC write-committed transaction caused by concurrent transaction commit and memtable switch (#9571). +* Fixed NUM_INDEX_AND_FILTER_BLOCKS_READ_PER_LEVEL, NUM_DATA_BLOCKS_READ_PER_LEVEL, and NUM_SST_READ_PER_LEVEL stats to be reported once per MultiGet batch per level. + +### Performance Improvements +* Mitigated the overhead of building the file location hash table used by the online LSM tree consistency checks, which can improve performance for certain workloads (see #9351). +* Switched to using a sorted `std::vector` instead of `std::map` for storing the metadata objects for blob files, which can improve performance for certain workloads, especially when the number of blob files is high. +* DisableManualCompaction() doesn't have to wait scheduled manual compaction to be executed in thread-pool to cancel the job. + +### Public API changes +* Require C++17 compatible compiler (GCC >= 7, Clang >= 5, Visual Studio >= 2017) for compiling RocksDB and any code using RocksDB headers. See #9388. +* Added `ReadOptions::rate_limiter_priority`. When set to something other than `Env::IO_TOTAL`, the internal rate limiter (`DBOptions::rate_limiter`) will be charged at the specified priority for file reads associated with the API to which the `ReadOptions` was provided. +* Remove HDFS support from main repo. +* Remove librados support from main repo. +* Remove obsolete backupable_db.h and type alias `BackupableDBOptions`. Use backup_engine.h and `BackupEngineOptions`. Similar renamings are in the C and Java APIs. +* Removed obsolete utility_db.h and `UtilityDB::OpenTtlDB`. Use db_ttl.h and `DBWithTTL::Open`. +* Remove deprecated API DB::AddFile from main repo. +* Remove deprecated API ObjectLibrary::Register() and the (now obsolete) Regex public API. Use ObjectLibrary::AddFactory() with PatternEntry instead. +* Remove deprecated option DBOption::table_cache_remove_scan_count_limit. +* Remove deprecated API AdvancedColumnFamilyOptions::soft_rate_limit. +* Remove deprecated API AdvancedColumnFamilyOptions::hard_rate_limit. +* Remove deprecated API DBOption::base_background_compactions. +* Remove deprecated API DBOptions::purge_redundant_kvs_while_flush. +* Remove deprecated overloads of API DB::CompactRange. +* Remove deprecated option DBOptions::skip_log_error_on_recovery. +* Remove ReadOptions::iter_start_seqnum which has been deprecated. +* Remove DBOptions::preserved_deletes and DB::SetPreserveDeletesSequenceNumber(). +* Remove deprecated API AdvancedColumnFamilyOptions::rate_limit_delay_max_milliseconds. +* Removed timestamp from WriteOptions. Accordingly, added to DB APIs Put, Delete, SingleDelete, etc. accepting an additional argument 'timestamp'. Added Put, Delete, SingleDelete, etc to WriteBatch accepting an additional argument 'timestamp'. Removed WriteBatch::AssignTimestamps(vector) API. Renamed WriteBatch::AssignTimestamp() to WriteBatch::UpdateTimestamps() with clarified comments. +* Changed type of cache buffer passed to `Cache::CreateCallback` from `void*` to `const void*`. +* Significant updates to FilterPolicy-related APIs and configuration: + * Remove public API support for deprecated, inefficient block-based filter (use_block_based_builder=true). + * Old code and configuration strings that would enable it now quietly enable full filters instead, though any built-in FilterPolicy can still read block-based filters. This includes changing the longstanding default behavior of the Java API. + * Remove deprecated FilterPolicy::CreateFilter() and FilterPolicy::KeyMayMatch() + * Remove `rocksdb_filterpolicy_create()` from C API, as the only C API support for custom filter policies is now obsolete. + * If temporary memory usage in full filter creation is a problem, consider using partitioned filters, smaller SST files, or setting reserve_table_builder_memory=true. + * Remove support for "filter_policy=experimental_ribbon" configuration + string. Use something like "filter_policy=ribbonfilter:10" instead. + * Allow configuration string like "filter_policy=bloomfilter:10" without + bool, to minimize acknowledgement of obsolete block-based filter. + * Made FilterPolicy Customizable. Configuration of filter_policy is now accurately saved in OPTIONS file and can be loaded with LoadOptionsFromFile. (Loading an OPTIONS file generated by a previous version only enables reading and using existing filters, not generating new filters. Previously, no filter_policy would be configured from a saved OPTIONS file.) + * Change meaning of nullptr return from GetBuilderWithContext() from "use + block-based filter" to "generate no filter in this case." + * Also, when user specifies bits_per_key < 0.5, we now round this down + to "no filter" because we expect a filter with >= 80% FP rate is + unlikely to be worth the CPU cost of accessing it (esp with + cache_index_and_filter_blocks=1 or partition_filters=1). + * bits_per_key >= 0.5 and < 1.0 is still rounded up to 1.0 (for 62% FP + rate) + * Remove class definitions for FilterBitsBuilder and FilterBitsReader from + public API, so these can evolve more easily as implementation details. + Custom FilterPolicy can still decide what kind of built-in filter to use + under what conditions. + * Also removed deprecated functions + * FilterPolicy::GetFilterBitsBuilder() + * NewExperimentalRibbonFilterPolicy() + * Remove default implementations of + * FilterPolicy::GetBuilderWithContext() +* Remove default implementation of Name() from FileSystemWrapper. +* Rename `SizeApproximationOptions.include_memtabtles` to `SizeApproximationOptions.include_memtables`. +* Remove deprecated option DBOptions::max_mem_compaction_level. +* Return Status::InvalidArgument from ObjectRegistry::NewObject if a factory exists but the object ould not be created (returns NotFound if the factory is missing). +* Remove deprecated overloads of API DB::GetApproximateSizes. +* Remove deprecated option DBOptions::new_table_reader_for_compaction_inputs. +* Add Transaction::SetReadTimestampForValidation() and Transaction::SetCommitTimestamp(). Default impl returns NotSupported(). +* Add support for decimal patterns to ObjectLibrary::PatternEntry +* Remove deprecated remote compaction APIs `CompactionService::Start()` and `CompactionService::WaitForComplete()`. Please use `CompactionService::StartV2()`, `CompactionService::WaitForCompleteV2()` instead, which provides the same information plus extra data like priority, db_id, etc. +* `ColumnFamilyOptions::OldDefaults` and `DBOptions::OldDefaults` are marked deprecated, as they are no longer maintained. +* Add subcompaction callback APIs: `OnSubcompactionBegin()` and `OnSubcompactionCompleted()`. +* Add file Temperature information to `FileOperationInfo` in event listener API. +* Change the type of SizeApproximationFlags from enum to enum class. Also update the signature of DB::GetApproximateSizes API from uint8_t to SizeApproximationFlags. +* Add Temperature hints information from RocksDB in API `NewSequentialFile()`. backup and checkpoint operations need to open the source files with `NewSequentialFile()`, which will have the temperature hints. Other operations are not covered. + +### Behavior Changes +* Disallow the combination of DBOptions.use_direct_io_for_flush_and_compaction == true and DBOptions.writable_file_max_buffer_size == 0. This combination can cause WritableFileWriter::Append() to loop forever, and it does not make much sense in direct IO. +* `ReadOptions::total_order_seek` no longer affects `DB::Get()`. The original motivation for this interaction has been obsolete since RocksDB has been able to detect whether the current prefix extractor is compatible with that used to generate table files, probably RocksDB 5.14.0. + +## New Features +* Introduced an option `BlockBasedTableOptions::detect_filter_construct_corruption` for detecting corruption during Bloom Filter (format_version >= 5) and Ribbon Filter construction. +* Improved the SstDumpTool to read the comparator from table properties and use it to read the SST File. +* Extended the column family statistics in the info log so the total amount of garbage in the blob files and the blob file space amplification factor are also logged. Also exposed the blob file space amp via the `rocksdb.blob-stats` DB property. +* Introduced the API rocksdb_create_dir_if_missing in c.h that calls underlying file system's CreateDirIfMissing API to create the directory. +* Added last level and non-last level read statistics: `LAST_LEVEL_READ_*`, `NON_LAST_LEVEL_READ_*`. +* Experimental: Add support for new APIs ReadAsync in FSRandomAccessFile that reads the data asynchronously and Poll API in FileSystem that checks if requested read request has completed or not. ReadAsync takes a callback function. Poll API checks for completion of read IO requests and should call callback functions to indicate completion of read requests. + +## 6.29.0 (01/21/2022) +Note: The next release will be major release 7.0. See https://github.com/facebook/rocksdb/issues/9390 for more info. +### Public API change +* Added values to `TraceFilterType`: `kTraceFilterIteratorSeek`, `kTraceFilterIteratorSeekForPrev`, and `kTraceFilterMultiGet`. They can be set in `TraceOptions` to filter out the operation types after which they are named. +* Added `TraceOptions::preserve_write_order`. When enabled it guarantees write records are traced in the same order they are logged to WAL and applied to the DB. By default it is disabled (false) to match the legacy behavior and prevent regression. +* Made the Env class extend the Customizable class. Implementations need to be registered with the ObjectRegistry and to implement a Name() method in order to be created via this method. +* `Options::OldDefaults` is marked deprecated, as it is no longer maintained. +* Add ObjectLibrary::AddFactory and ObjectLibrary::PatternEntry classes. This method and associated class are the preferred mechanism for registering factories with the ObjectLibrary going forward. The ObjectLibrary::Register method, which uses regular expressions and may be problematic, is deprecated and will be in a future release. +* Changed `BlockBasedTableOptions::block_size` from `size_t` to `uint64_t`. +* Added API warning against using `Iterator::Refresh()` together with `DB::DeleteRange()`, which are incompatible and have always risked causing the refreshed iterator to return incorrect results. +* Made `AdvancedColumnFamilyOptions.bottommost_temperature` dynamically changeable with `SetOptions()`. + +### Behavior Changes +* `DB::DestroyColumnFamilyHandle()` will return Status::InvalidArgument() if called with `DB::DefaultColumnFamily()`. +* On 32-bit platforms, mmap reads are no longer quietly disabled, just discouraged. + +### New Features +* Added `Options::DisableExtraChecks()` that can be used to improve peak write performance by disabling checks that should not be necessary in the absence of software logic errors or CPU+memory hardware errors. (Default options are slowly moving toward some performance overheads for extra correctness checking.) + +### Performance Improvements +* Improved read performance when a prefix extractor is used (Seek, Get, MultiGet), even compared to version 6.25 baseline (see bug fix below), by optimizing the common case of prefix extractor compatible with table file and unchanging. + +### Bug Fixes +* Fix a bug that FlushMemTable may return ok even flush not succeed. +* Fixed a bug of Sync() and Fsync() not using `fcntl(F_FULLFSYNC)` on OS X and iOS. +* Fixed a significant performance regression in version 6.26 when a prefix extractor is used on the read path (Seek, Get, MultiGet). (Excessive time was spent in SliceTransform::AsString().) +* Fixed a race condition in SstFileManagerImpl error recovery code that can cause a crash during process shutdown. + +### New Features +* Added RocksJava support for MacOS universal binary (ARM+x86) + +## 6.28.0 (2021-12-17) +### New Features +* Introduced 'CommitWithTimestamp' as a new tag. Currently, there is no API for user to trigger a write with this tag to the WAL. This is part of the efforts to support write-commited transactions with user-defined timestamps. +* Introduce SimulatedHybridFileSystem which can help simulating HDD latency in db_bench. Tiered Storage latency simulation can be enabled using -simulate_hybrid_fs_file (note that it doesn't work if db_bench is interrupted in the middle). -simulate_hdd can also be used to simulate all files on HDD. + +### Bug Fixes +* Fixed a bug in rocksdb automatic implicit prefetching which got broken because of new feature adaptive_readahead and internal prefetching got disabled when iterator moves from one file to next. +* Fixed a bug in TableOptions.prepopulate_block_cache which causes segmentation fault when used with TableOptions.partition_filters = true and TableOptions.cache_index_and_filter_blocks = true. +* Fixed a bug affecting custom memtable factories which are not registered with the `ObjectRegistry`. The bug could result in failure to save the OPTIONS file. +* Fixed a bug causing two duplicate entries to be appended to a file opened in non-direct mode and tracked by `FaultInjectionTestFS`. +* Fixed a bug in TableOptions.prepopulate_block_cache to support block-based filters also. +* Block cache keys no longer use `FSRandomAccessFile::GetUniqueId()` (previously used when available), so a filesystem recycling unique ids can no longer lead to incorrect result or crash (#7405). For files generated by RocksDB >= 6.24, the cache keys are stable across DB::Open and DB directory move / copy / import / export / migration, etc. Although collisions are still theoretically possible, they are (a) impossible in many common cases, (b) not dependent on environmental factors, and (c) much less likely than a CPU miscalculation while executing RocksDB. +* Fixed a bug in C bindings causing iterator to return incorrect result (#9343). + +### Behavior Changes +* MemTableList::TrimHistory now use allocated bytes when max_write_buffer_size_to_maintain > 0(default in TrasactionDB, introduced in PR#5022) Fix #8371. + +### Public API change +* Extend WriteBatch::AssignTimestamp and AssignTimestamps API so that both functions can accept an optional `checker` argument that performs additional checking on timestamp sizes. +* Introduce a new EventListener callback that will be called upon the end of automatic error recovery. +* Add IncreaseFullHistoryTsLow API so users can advance each column family's full_history_ts_low seperately. +* Add GetFullHistoryTsLow API so users can query current full_history_low value of specified column family. + +### Performance Improvements +* Replaced map property `TableProperties::properties_offsets` with uint64_t property `external_sst_file_global_seqno_offset` to save table properties's memory. +* Block cache accesses are faster by RocksDB using cache keys of fixed size (16 bytes). + +### Java API Changes +* Removed Java API `TableProperties.getPropertiesOffsets()` as it exposed internal details to external users. + +## 6.27.0 (2021-11-19) +### New Features +* Added new ChecksumType kXXH3 which is faster than kCRC32c on almost all x86\_64 hardware. +* Added a new online consistency check for BlobDB which validates that the number/total size of garbage blobs does not exceed the number/total size of all blobs in any given blob file. +* Provided support for tracking per-sst user-defined timestamp information in MANIFEST. +* Added new option "adaptive_readahead" in ReadOptions. For iterators, RocksDB does auto-readahead on noticing sequential reads and by enabling this option, readahead_size of current file (if reads are sequential) will be carried forward to next file instead of starting from the scratch at each level (except L0 level files). If reads are not sequential it will fall back to 8KB. This option is applicable only for RocksDB internal prefetch buffer and isn't supported with underlying file system prefetching. +* Added the read count and read bytes related stats to Statistics for tiered storage hot, warm, and cold file reads. +* Added an option to dynamically charge an updating estimated memory usage of block-based table building to block cache if block cache available. It currently only includes charging memory usage of constructing (new) Bloom Filter and Ribbon Filter to block cache. To enable this feature, set `BlockBasedTableOptions::reserve_table_builder_memory = true`. +* Add a new API OnIOError in listener.h that notifies listeners when an IO error occurs during FileSystem operation along with filename, status etc. +* Added compaction readahead support for blob files to the integrated BlobDB implementation, which can improve compaction performance when the database resides on higher-latency storage like HDDs or remote filesystems. Readahead can be configured using the column family option `blob_compaction_readahead_size`. + +### Bug Fixes +* Prevent a `CompactRange()` with `CompactRangeOptions::change_level == true` from possibly causing corruption to the LSM state (overlapping files within a level) when run in parallel with another manual compaction. Note that setting `force_consistency_checks == true` (the default) would cause the DB to enter read-only mode in this scenario and return `Status::Corruption`, rather than committing any corruption. +* Fixed a bug in CompactionIterator when write-prepared transaction is used. A released earliest write conflict snapshot may cause assertion failure in dbg mode and unexpected key in opt mode. +* Fix ticker WRITE_WITH_WAL("rocksdb.write.wal"), this bug is caused by a bad extra `RecordTick(stats_, WRITE_WITH_WAL)` (at 2 place), this fix remove the extra `RecordTick`s and fix the corresponding test case. +* EventListener::OnTableFileCreated was previously called with OK status and file_size==0 in cases of no SST file contents written (because there was no content to add) and the empty file deleted before calling the listener. Now the status is Aborted. +* Fixed a bug in CompactionIterator when write-preared transaction is used. Releasing earliest_snapshot during compaction may cause a SingleDelete to be output after a PUT of the same user key whose seq has been zeroed. +* Added input sanitization on negative bytes passed into `GenericRateLimiter::Request`. +* Fixed an assertion failure in CompactionIterator when write-prepared transaction is used. We prove that certain operations can lead to a Delete being followed by a SingleDelete (same user key). We can drop the SingleDelete. +* Fixed a bug of timestamp-based GC which can cause all versions of a key under full_history_ts_low to be dropped. This bug will be triggered when some of the ikeys' timestamps are lower than full_history_ts_low, while others are newer. +* In some cases outside of the DB read and compaction paths, SST block checksums are now checked where they were not before. +* Explicitly check for and disallow the `BlockBasedTableOptions` if insertion into one of {`block_cache`, `block_cache_compressed`, `persistent_cache`} can show up in another of these. (RocksDB expects to be able to use the same key for different physical data among tiers.) +* Users who configured a dedicated thread pool for bottommost compactions by explicitly adding threads to the `Env::Priority::BOTTOM` pool will no longer see RocksDB schedule automatic compactions exceeding the DB's compaction concurrency limit. For details on per-DB compaction concurrency limit, see API docs of `max_background_compactions` and `max_background_jobs`. +* Fixed a bug of background flush thread picking more memtables to flush and prematurely advancing column family's log_number. +* Fixed an assertion failure in ManifestTailer. +* Fixed a bug that could, with WAL enabled, cause backups, checkpoints, and `GetSortedWalFiles()` to fail randomly with an error like `IO error: 001234.log: No such file or directory` + +### Behavior Changes +* `NUM_FILES_IN_SINGLE_COMPACTION` was only counting the first input level files, now it's including all input files. +* `TransactionUtil::CheckKeyForConflicts` can also perform conflict-checking based on user-defined timestamps in addition to sequence numbers. +* Removed `GenericRateLimiter`'s minimum refill bytes per period previously enforced. + +### Public API change +* When options.ttl is used with leveled compaction with compactinon priority kMinOverlappingRatio, files exceeding half of TTL value will be prioritized more, so that by the time TTL is reached, fewer extra compactions will be scheduled to clear them up. At the same time, when compacting files with data older than half of TTL, output files may be cut off based on those files' boundaries, in order for the early TTL compaction to work properly. +* Made FileSystem and RateLimiter extend the Customizable class and added a CreateFromString method. Implementations need to be registered with the ObjectRegistry and to implement a Name() method in order to be created via this method. +* Clarified in API comments that RocksDB is not exception safe for callbacks and custom extensions. An exception propagating into RocksDB can lead to undefined behavior, including data loss, unreported corruption, deadlocks, and more. +* Marked `WriteBufferManager` as `final` because it is not intended for extension. +* Removed unimportant implementation details from table_properties.h +* Add API `FSDirectory::FsyncWithDirOptions()`, which provides extra information like directory fsync reason in `DirFsyncOptions`. File system like btrfs is using that to skip directory fsync for creating a new file, or when renaming a file, fsync the target file instead of the directory, which improves the `DB::Open()` speed by ~20%. +* `DB::Open()` is not going be blocked by obsolete file purge if `DBOptions::avoid_unnecessary_blocking_io` is set to true. +* In builds where glibc provides `gettid()`, info log ("LOG" file) lines now print a system-wide thread ID from `gettid()` instead of the process-local `pthread_self()`. For all users, the thread ID format is changed from hexadecimal to decimal integer. +* In builds where glibc provides `pthread_setname_np()`, the background thread names no longer contain an ID suffix. For example, "rocksdb:bottom7" (and all other threads in the `Env::Priority::BOTTOM` pool) are now named "rocksdb:bottom". Previously large thread pools could breach the name size limit (e.g., naming "rocksdb:bottom10" would fail). +* Deprecating `ReadOptions::iter_start_seqnum` and `DBOptions::preserve_deletes`, please try using user defined timestamp feature instead. The options will be removed in a future release, currently it logs a warning message when using. + +### Performance Improvements +* Released some memory related to filter construction earlier in `BlockBasedTableBuilder` for `FullFilter` and `PartitionedFilter` case (#9070) + +### Behavior Changes +* `NUM_FILES_IN_SINGLE_COMPACTION` was only counting the first input level files, now it's including all input files. + +## 6.26.0 (2021-10-20) +### Bug Fixes +* Fixes a bug in directed IO mode when calling MultiGet() for blobs in the same blob file. The bug is caused by not sorting the blob read requests by file offsets. +* Fix the incorrect disabling of SST rate limited deletion when the WAL and DB are in different directories. Only WAL rate limited deletion should be disabled if its in a different directory. +* Fix `DisableManualCompaction()` to cancel compactions even when they are waiting on automatic compactions to drain due to `CompactRangeOptions::exclusive_manual_compactions == true`. +* Fix contract of `Env::ReopenWritableFile()` and `FileSystem::ReopenWritableFile()` to specify any existing file must not be deleted or truncated. +* Fixed bug in calls to `IngestExternalFiles()` with files for multiple column families. The bug could have introduced a delay in ingested file keys becoming visible after `IngestExternalFiles()` returned. Furthermore, mutations to ingested file keys while they were invisible could have been dropped (not necessarily immediately). +* Fixed a possible race condition impacting users of `WriteBufferManager` who constructed it with `allow_stall == true`. The race condition led to undefined behavior (in our experience, typically a process crash). +* Fixed a bug where stalled writes would remain stalled forever after the user calls `WriteBufferManager::SetBufferSize()` with `new_size == 0` to dynamically disable memory limiting. +* Make `DB::close()` thread-safe. +* Fix a bug in atomic flush where one bg flush thread will wait forever for a preceding bg flush thread to commit its result to MANIFEST but encounters an error which is mapped to a soft error (DB not stopped). +* Fix a bug in `BackupEngine` where some internal callers of `GenericRateLimiter::Request()` do not honor `bytes <= GetSingleBurstBytes()`. + +### New Features +* Print information about blob files when using "ldb list_live_files_metadata" +* Provided support for SingleDelete with user defined timestamp. +* Experimental new function DB::GetLiveFilesStorageInfo offers essentially a unified version of other functions like GetLiveFiles, GetLiveFilesChecksumInfo, and GetSortedWalFiles. Checkpoints and backups could show small behavioral changes and/or improved performance as they now use this new API. +* Add remote compaction read/write bytes statistics: `REMOTE_COMPACT_READ_BYTES`, `REMOTE_COMPACT_WRITE_BYTES`. +* Introduce an experimental feature to dump out the blocks from block cache and insert them to the secondary cache to reduce the cache warmup time (e.g., used while migrating DB instance). More information are in `class CacheDumper` and `CacheDumpedLoader` at `rocksdb/utilities/cache_dump_load.h` Note that, this feature is subject to the potential change in the future, it is still experimental. +* Introduced a new BlobDB configuration option `blob_garbage_collection_force_threshold`, which can be used to trigger compactions targeting the SST files which reference the oldest blob files when the ratio of garbage in those blob files meets or exceeds the specified threshold. This can reduce space amplification with skewed workloads where the affected SST files might not otherwise get picked up for compaction. +* Added EXPERIMENTAL support for table file (SST) unique identifiers that are stable and universally unique, available with new function `GetUniqueIdFromTableProperties`. Only SST files from RocksDB >= 6.24 support unique IDs. +* Added `GetMapProperty()` support for "rocksdb.dbstats" (`DB::Properties::kDBStats`). As a map property, it includes DB-level internal stats accumulated over the DB's lifetime, such as user write related stats and uptime. + +### Public API change +* Made SystemClock extend the Customizable class and added a CreateFromString method. Implementations need to be registered with the ObjectRegistry and to implement a Name() method in order to be created via this method. +* Made SliceTransform extend the Customizable class and added a CreateFromString method. Implementations need to be registered with the ObjectRegistry and to implement a Name() method in order to be created via this method. The Capped and Prefixed transform classes return a short name (no length); use GetId for the fully qualified name. +* Made FileChecksumGenFactory, SstPartitionerFactory, TablePropertiesCollectorFactory, and WalFilter extend the Customizable class and added a CreateFromString method. +* Some fields of SstFileMetaData are deprecated for compatibility with new base class FileStorageInfo. +* Add `file_temperature` to `IngestExternalFileArg` such that when ingesting SST files, we are able to indicate the temperature of the this batch of files. +* If `DB::Close()` failed with a non aborted status, calling `DB::Close()` again will return the original status instead of Status::OK. +* Add CacheTier to advanced_options.h to describe the cache tier we used. Add a `lowest_used_cache_tier` option to `DBOptions` (immutable) and pass it to BlockBasedTableReader. By default it is `CacheTier::kNonVolatileBlockTier`, which means, we always use both block cache (kVolatileTier) and secondary cache (kNonVolatileBlockTier). By set it to `CacheTier::kVolatileTier`, the DB will not use the secondary cache. +* Even when options.max_compaction_bytes is hit, compaction output files are only cut when it aligns with grandparent files' boundaries. options.max_compaction_bytes could be slightly violated with the change, but the violation is no more than one target SST file size, which is usually much smaller. + +### Performance Improvements +* Improved CPU efficiency of building block-based table (SST) files (#9039 and #9040). + +### Java API Changes +* Add Java API bindings for new integrated BlobDB options +* `keyMayExist()` supports ByteBuffer. +* Fix multiget throwing Null Pointer Exception for num of keys > 70k (https://github.com/facebook/rocksdb/issues/8039). + +## 6.25.0 (2021-09-20) +### Bug Fixes +* Allow secondary instance to refresh iterator. Assign read seq after referencing SuperVersion. +* Fixed a bug of secondary instance's last_sequence going backward, and reads on the secondary fail to see recent updates from the primary. +* Fixed a bug that could lead to duplicate DB ID or DB session ID in POSIX environments without /proc/sys/kernel/random/uuid. +* Fix a race in DumpStats() with column family destruction due to not taking a Ref on each entry while iterating the ColumnFamilySet. +* Fix a race in item ref counting in LRUCache when promoting an item from the SecondaryCache. +* Fix a race in BackupEngine if RateLimiter is reconfigured during concurrent Restore operations. +* Fix a bug on POSIX in which failure to create a lock file (e.g. out of space) can prevent future LockFile attempts in the same process on the same file from succeeding. +* Fix a bug that backup_rate_limiter and restore_rate_limiter in BackupEngine could not limit read rates. +* Fix the implementation of `prepopulate_block_cache = kFlushOnly` to only apply to flushes rather than to all generated files. +* Fix WAL log data corruption when using DBOptions.manual_wal_flush(true) and WriteOptions.sync(true) together. The sync WAL should work with locked log_write_mutex_. +* Add checks for validity of the IO uring completion queue entries, and fail the BlockBasedTableReader MultiGet sub-batch if there's an invalid completion +* Add an interface RocksDbIOUringEnable() that, if defined by the user, will allow them to enable/disable the use of IO uring by RocksDB +* Fix the bug that when direct I/O is used and MultiRead() returns a short result, RandomAccessFileReader::MultiRead() still returns full size buffer, with returned short value together with some data in original buffer. This bug is unlikely cause incorrect results, because (1) since FileSystem layer is expected to retry on short result, returning short results is only possible when asking more bytes in the end of the file, which RocksDB doesn't do when using MultiRead(); (2) checksum is unlikely to match. + +### New Features +* RemoteCompaction's interface now includes `db_name`, `db_id`, `session_id`, which could help the user uniquely identify compaction job between db instances and sessions. +* Added a ticker statistic, "rocksdb.verify_checksum.read.bytes", reporting how many bytes were read from file to serve `VerifyChecksum()` and `VerifyFileChecksums()` queries. +* Added ticker statistics, "rocksdb.backup.read.bytes" and "rocksdb.backup.write.bytes", reporting how many bytes were read and written during backup. +* Added properties for BlobDB: `rocksdb.num-blob-files`, `rocksdb.blob-stats`, `rocksdb.total-blob-file-size`, and `rocksdb.live-blob-file-size`. The existing property `rocksdb.estimate_live-data-size` was also extended to include live bytes residing in blob files. +* Added two new RateLimiter IOPriorities: `Env::IO_USER`,`Env::IO_MID`. `Env::IO_USER` will have superior priority over all other RateLimiter IOPriorities without being subject to fair scheduling constraint. +* `SstFileWriter` now supports `Put`s and `Delete`s with user-defined timestamps. Note that the ingestion logic itself is not timestamp-aware yet. +* Allow a single write batch to include keys from multiple column families whose timestamps' formats can differ. For example, some column families may disable timestamp, while others enable timestamp. +* Add compaction priority information in RemoteCompaction, which can be used to schedule high priority job first. +* Added new callback APIs `OnBlobFileCreationStarted`,`OnBlobFileCreated`and `OnBlobFileDeleted` in `EventListener` class of listener.h. It notifies listeners during creation/deletion of individual blob files in Integrated BlobDB. It also log blob file creation finished event and deletion event in LOG file. +* Batch blob read requests for `DB::MultiGet` using `MultiRead`. +* Add support for fallback to local compaction, the user can return `CompactionServiceJobStatus::kUseLocal` to instruct RocksDB to run the compaction locally instead of waiting for the remote compaction result. +* Add built-in rate limiter's implementation of `RateLimiter::GetTotalPendingRequest(int64_t* total_pending_requests, const Env::IOPriority pri)` for the total number of requests that are pending for bytes in the rate limiter. +* Charge memory usage during data buffering, from which training samples are gathered for dictionary compression, to block cache. Unbuffering data can now be triggered if the block cache becomes full and `strict_capacity_limit=true` for the block cache, in addition to existing conditions that can trigger unbuffering. + +### Public API change +* Remove obsolete implementation details FullKey and ParseFullKey from public API +* Change `SstFileMetaData::size` from `size_t` to `uint64_t`. +* Made Statistics extend the Customizable class and added a CreateFromString method. Implementations of Statistics need to be registered with the ObjectRegistry and to implement a Name() method in order to be created via this method. +* Extended `FlushJobInfo` and `CompactionJobInfo` in listener.h to provide information about the blob files generated by a flush/compaction and garbage collected during compaction in Integrated BlobDB. Added struct members `blob_file_addition_infos` and `blob_file_garbage_infos` that contain this information. +* Extended parameter `output_file_names` of `CompactFiles` API to also include paths of the blob files generated by the compaction in Integrated BlobDB. +* Most `BackupEngine` functions now return `IOStatus` instead of `Status`. Most existing code should be compatible with this change but some calls might need to be updated. +* Add a new field `level_at_creation` in `TablePropertiesCollectorFactory::Context` to capture the level at creating the SST file (i.e, table), of which the properties are being collected. + +### Miscellaneous +* Add a paranoid check where in case FileSystem layer doesn't fill the buffer but returns succeed, checksum is unlikely to match even if buffer contains a previous block. The byte modified is not useful anyway, so it isn't expected to change any behavior when FileSystem is satisfying its contract. + +## 6.24.0 (2021-08-20) +### Bug Fixes +* If the primary's CURRENT file is missing or inaccessible, the secondary instance should not hang repeatedly trying to switch to a new MANIFEST. It should instead return the error code encountered while accessing the file. +* Restoring backups with BackupEngine is now a logically atomic operation, so that if a restore operation is interrupted, DB::Open on it will fail. Using BackupEngineOptions::sync (default) ensures atomicity even in case of power loss or OS crash. +* Fixed a race related to the destruction of `ColumnFamilyData` objects. The earlier logic unlocked the DB mutex before destroying the thread-local `SuperVersion` pointers, which could result in a process crash if another thread managed to get a reference to the `ColumnFamilyData` object. +* Removed a call to `RenameFile()` on a non-existent info log file ("LOG") when opening a new DB. Such a call was guaranteed to fail though did not impact applications since we swallowed the error. Now we also stopped swallowing errors in renaming "LOG" file. +* Fixed an issue where `OnFlushCompleted` was not called for atomic flush. +* Fixed a bug affecting the batched `MultiGet` API when used with keys spanning multiple column families and `sorted_input == false`. +* Fixed a potential incorrect result in opt mode and assertion failures caused by releasing snapshot(s) during compaction. +* Fixed passing of BlobFileCompletionCallback to Compaction job and Atomic flush job which was default paramter (nullptr). BlobFileCompletitionCallback is internal callback that manages addition of blob files to SSTFileManager. +* Fixed MultiGet not updating the block_read_count and block_read_byte PerfContext counters. + +### New Features +* Made the EventListener extend the Customizable class. +* EventListeners that have a non-empty Name() and that are registered with the ObjectRegistry can now be serialized to/from the OPTIONS file. +* Insert warm blocks (data blocks, uncompressed dict blocks, index and filter blocks) in Block cache during flush under option BlockBasedTableOptions.prepopulate_block_cache. Previously it was enabled for only data blocks. +* BlockBasedTableOptions.prepopulate_block_cache can be dynamically configured using DB::SetOptions. +* Add CompactionOptionsFIFO.age_for_warm, which allows RocksDB to move old files to warm tier in FIFO compactions. Note that file temperature is still an experimental feature. +* Add a comment to suggest btrfs user to disable file preallocation by setting `options.allow_fallocate=false`. +* Fast forward option in Trace replay changed to double type to allow replaying at a lower speed, by settings the value between 0 and 1. This option can be set via `ReplayOptions` in `Replayer::Replay()`, or via `--trace_replay_fast_forward` in db_bench. +* Add property `LiveSstFilesSizeAtTemperature` to retrieve sst file size at different temperature. +* Added a stat rocksdb.secondary.cache.hits. +* Added a PerfContext counter secondary_cache_hit_count. +* The integrated BlobDB implementation now supports the tickers `BLOB_DB_BLOB_FILE_BYTES_READ`, `BLOB_DB_GC_NUM_KEYS_RELOCATED`, and `BLOB_DB_GC_BYTES_RELOCATED`, as well as the histograms `BLOB_DB_COMPRESSION_MICROS` and `BLOB_DB_DECOMPRESSION_MICROS`. +* Added hybrid configuration of Ribbon filter and Bloom filter where some LSM levels use Ribbon for memory space efficiency and some use Bloom for speed. See NewRibbonFilterPolicy. This also changes the default behavior of NewRibbonFilterPolicy to use Bloom for flushes under Leveled and Universal compaction and Ribbon otherwise. The C API function `rocksdb_filterpolicy_create_ribbon` is unchanged but adds new `rocksdb_filterpolicy_create_ribbon_hybrid`. + +### Public API change +* Added APIs to decode and replay trace file via Replayer class. Added `DB::NewDefaultReplayer()` to create a default Replayer instance. Added `TraceReader::Reset()` to restart reading a trace file. Created trace_record.h, trace_record_result.h and utilities/replayer.h files to access the decoded Trace records, replay them, and query the actual operation results. +* Added Configurable::GetOptionsMap to the public API for use in creating new Customizable classes. +* Generalized bits_per_key parameters in C API from int to double for greater configurability. Although this is a compatible change for existing C source code, anything depending on C API signatures, such as foreign function interfaces, will need to be updated. + +### Performance Improvements +* Try to avoid updating DBOptions if `SetDBOptions()` does not change any option value. + +### Behavior Changes +* `StringAppendOperator` additionally accepts a string as the delimiter. +* BackupEngineOptions::sync (default true) now applies to restoring backups in addition to creating backups. This could slow down restores, but ensures they are fully persisted before returning OK. (Consider increasing max_background_operations to improve performance.) + +## 6.23.0 (2021-07-16) +### Behavior Changes +* Obsolete keys in the bottommost level that were preserved for a snapshot will now be cleaned upon snapshot release in all cases. This form of compaction (snapshot release triggered compaction) previously had an artificial limitation that multiple tombstones needed to be present. +### Bug Fixes +* Blob file checksums are now printed in hexadecimal format when using the `manifest_dump` `ldb` command. +* `GetLiveFilesMetaData()` now populates the `temperature`, `oldest_ancester_time`, and `file_creation_time` fields of its `LiveFileMetaData` results when the information is available. Previously these fields always contained zero indicating unknown. +* Fix mismatches of OnCompaction{Begin,Completed} in case of DisableManualCompaction(). +* Fix continuous logging of an existing background error on every user write +* Fix a bug that `Get()` return Status::OK() and an empty value for non-existent key when `read_options.read_tier = kBlockCacheTier`. +* Fix a bug that stat in `get_context` didn't accumulate to statistics when query is failed. +* Fixed handling of DBOptions::wal_dir with LoadLatestOptions() or ldb --try_load_options on a copied or moved DB. Previously, when the WAL directory is same as DB directory (default), a copied or moved DB would reference the old path of the DB as the WAL directory, potentially corrupting both copies. Under this change, the wal_dir from DB::GetOptions() or LoadLatestOptions() may now be empty, indicating that the current DB directory is used for WALs. This is also a subtle API change. + +### New Features +* ldb has a new feature, `list_live_files_metadata`, that shows the live SST files, as well as their LSM storage level and the column family they belong to. +* The new BlobDB implementation now tracks the amount of garbage in each blob file in the MANIFEST. +* Integrated BlobDB now supports Merge with base values (Put/Delete etc.). +* RemoteCompaction supports sub-compaction, the job_id in the user interface is changed from `int` to `uint64_t` to support sub-compaction id. +* Expose statistics option in RemoteCompaction worker. + +### Public API change +* Added APIs to the Customizable class to allow developers to create their own Customizable classes. Created the utilities/customizable_util.h file to contain helper methods for developing new Customizable classes. +* Change signature of SecondaryCache::Name(). Make SecondaryCache customizable and add SecondaryCache::CreateFromString method. + +## 6.22.0 (2021-06-18) +### Behavior Changes +* Added two additional tickers, MEMTABLE_PAYLOAD_BYTES_AT_FLUSH and MEMTABLE_GARBAGE_BYTES_AT_FLUSH. These stats can be used to estimate the ratio of "garbage" (outdated) bytes in the memtable that are discarded at flush time. +* Added API comments clarifying safe usage of Disable/EnableManualCompaction and EventListener callbacks for compaction. +### Bug Fixes +* fs_posix.cc GetFreeSpace() always report disk space available to root even when running as non-root. Linux defaults often have disk mounts with 5 to 10 percent of total space reserved only for root. Out of space could result for non-root users. +* Subcompactions are now disabled when user-defined timestamps are used, since the subcompaction boundary picking logic is currently not timestamp-aware, which could lead to incorrect results when different subcompactions process keys that only differ by timestamp. +* Fix an issue that `DeleteFilesInRange()` may cause ongoing compaction reports corruption exception, or ASSERT for debug build. There's no actual data loss or corruption that we find. +* Fixed confusingly duplicated output in LOG for periodic stats ("DUMPING STATS"), including "Compaction Stats" and "File Read Latency Histogram By Level". +* Fixed performance bugs in background gathering of block cache entry statistics, that could consume a lot of CPU when there are many column families with a shared block cache. + +### New Features +* Marked the Ribbon filter and optimize_filters_for_memory features as production-ready, each enabling memory savings for Bloom-like filters. Use `NewRibbonFilterPolicy` in place of `NewBloomFilterPolicy` to use Ribbon filters instead of Bloom, or `ribbonfilter` in place of `bloomfilter` in configuration string. +* Allow `DBWithTTL` to use `DeleteRange` api just like other DBs. `DeleteRangeCF()` which executes `WriteBatchInternal::DeleteRange()` has been added to the handler in `DBWithTTLImpl::Write()` to implement it. +* Add BlockBasedTableOptions.prepopulate_block_cache. If enabled, it prepopulate warm/hot data blocks which are already in memory into block cache at the time of flush. On a flush, the data block that is in memory (in memtables) get flushed to the device. If using Direct IO, additional IO is incurred to read this data back into memory again, which is avoided by enabling this option and it also helps with Distributed FileSystem. More details in include/rocksdb/table.h. +* Added a `cancel` field to `CompactRangeOptions`, allowing individual in-process manual range compactions to be cancelled. + +### New Features +* Added BlobMetaData to the ColumnFamilyMetaData to return information about blob files + +### Public API change +* Added GetAllColumnFamilyMetaData API to retrieve the ColumnFamilyMetaData about all column families. + +## 6.21.0 (2021-05-21) +### Bug Fixes +* Fixed a bug in handling file rename error in distributed/network file systems when the server succeeds but client returns error. The bug can cause CURRENT file to point to non-existing MANIFEST file, thus DB cannot be opened. +* Fixed a bug where ingested files were written with incorrect boundary key metadata. In rare cases this could have led to a level's files being wrongly ordered and queries for the boundary keys returning wrong results. +* Fixed a data race between insertion into memtables and the retrieval of the DB properties `rocksdb.cur-size-active-mem-table`, `rocksdb.cur-size-all-mem-tables`, and `rocksdb.size-all-mem-tables`. +* Fixed the false-positive alert when recovering from the WAL file. Avoid reporting "SST file is ahead of WAL" on a newly created empty column family, if the previous WAL file is corrupted. +* Fixed a bug where `GetLiveFiles()` output included a non-existent file called "OPTIONS-000000". Backups and checkpoints, which use `GetLiveFiles()`, failed on DBs impacted by this bug. Read-write DBs were impacted when the latest OPTIONS file failed to write and `fail_if_options_file_error == false`. Read-only DBs were impacted when no OPTIONS files existed. +* Handle return code by io_uring_submit_and_wait() and io_uring_wait_cqe(). +* In the IngestExternalFile() API, only try to sync the ingested file if the file is linked and the FileSystem/Env supports reopening a writable file. +* Fixed a bug that `AdvancedColumnFamilyOptions.max_compaction_bytes` is under-calculated for manual compaction (`CompactRange()`). Manual compaction is split to multiple compactions if the compaction size exceed the `max_compaction_bytes`. The bug creates much larger compaction which size exceed the user setting. On the other hand, larger manual compaction size can increase the subcompaction parallelism, you can tune that by setting `max_compaction_bytes`. + +### Behavior Changes +* Due to the fix of false-postive alert of "SST file is ahead of WAL", all the CFs with no SST file (CF empty) will bypass the consistency check. We fixed a false-positive, but introduced a very rare true-negative which will be triggered in the following conditions: A CF with some delete operations in the last a few queries which will result in an empty CF (those are flushed to SST file and a compaction triggered which combines this file and all other SST files and generates an empty CF, or there is another reason to write a manifest entry for this CF after a flush that generates no SST file from an empty CF). The deletion entries are logged in a WAL and this WAL was corrupted, while the CF's log number points to the next WAL (due to the flush). Therefore, the DB can only recover to the point without these trailing deletions and cause the inconsistent DB status. + +### New Features +* Add new option allow_stall passed during instance creation of WriteBufferManager. When allow_stall is set, WriteBufferManager will stall all writers shared across multiple DBs and columns if memory usage goes beyond specified WriteBufferManager::buffer_size (soft limit). Stall will be cleared when memory is freed after flush and memory usage goes down below buffer_size. +* Allow `CompactionFilter`s to apply in more table file creation scenarios such as flush and recovery. For compatibility, `CompactionFilter`s by default apply during compaction. Users can customize this behavior by overriding `CompactionFilterFactory::ShouldFilterTableFileCreation()`. +* Added more fields to FilterBuildingContext with LSM details, for custom filter policies that vary behavior based on where they are in the LSM-tree. +* Added DB::Properties::kBlockCacheEntryStats for querying statistics on what percentage of block cache is used by various kinds of blocks, etc. using DB::GetProperty and DB::GetMapProperty. The same information is now dumped to info LOG periodically according to `stats_dump_period_sec`. +* Add an experimental Remote Compaction feature, which allows the user to run Compaction on a different host or process. The feature is still under development, currently only works on some basic use cases. The interface will be changed without backward/forward compatibility support. +* RocksDB would validate total entries read in flush, and compare with counter inserted into it. If flush_verify_memtable_count = true (default), flush will fail. Otherwise, only log to info logs. +* Add `TableProperties::num_filter_entries`, which can be used with `TableProperties::filter_size` to calculate the effective bits per filter entry (unique user key or prefix) for a table file. + +### Performance Improvements +* BlockPrefetcher is used by iterators to prefetch data if they anticipate more data to be used in future. It is enabled implicitly by rocksdb. Added change to take in account read pattern if reads are sequential. This would disable prefetching for random reads in MultiGet and iterators as readahead_size is increased exponential doing large prefetches. + +### Public API change +* Removed a parameter from TableFactory::NewTableBuilder, which should not be called by user code because TableBuilder is not a public API. +* Removed unused structure `CompactionFilterContext`. +* The `skip_filters` parameter to SstFileWriter is now considered deprecated. Use `BlockBasedTableOptions::filter_policy` to control generation of filters. +* ClockCache is known to have bugs that could lead to crash or corruption, so should not be used until fixed. Use NewLRUCache instead. +* Added a new pure virtual function `ApplyToAllEntries` to `Cache`, to replace `ApplyToAllCacheEntries`. Custom `Cache` implementations must add an implementation. Because this function is for gathering statistics, an empty implementation could be acceptable for some applications. +* Added the ObjectRegistry to the ConfigOptions class. This registry instance will be used to find any customizable loadable objects during initialization. +* Expanded the ObjectRegistry functionality to allow nested ObjectRegistry instances. Added methods to register a set of functions with the registry/library as a group. +* Deprecated backupable_db.h and BackupableDBOptions in favor of new versions with appropriate names: backup_engine.h and BackupEngineOptions. Old API compatibility is preserved. + +### Default Option Change +* When options.arena_block_size <= 0 (default value 0), still use writer_buffer_size / 8 but cap to 1MB. Too large alloation size might not be friendly to allocator and might cause performance issues in extreme cases. + +### Build +* By default, try to build with liburing. For make, if ROCKSDB_USE_IO_URING is not set, treat as enable, which means RocksDB will try to build with liburing. Users can disable it with ROCKSDB_USE_IO_URING=0. For cmake, add WITH_LIBURING to control it, with default on. + +## 6.20.0 (2021-04-16) +### Behavior Changes +* `ColumnFamilyOptions::sample_for_compression` now takes effect for creation of all block-based tables. Previously it only took effect for block-based tables created by flush. +* `CompactFiles()` can no longer compact files from lower level to up level, which has the risk to corrupt DB (details: #8063). The validation is also added to all compactions. +* Fixed some cases in which DB::OpenForReadOnly() could write to the filesystem. If you want a Logger with a read-only DB, you must now set DBOptions::info_log yourself, such as using CreateLoggerFromOptions(). +* get_iostats_context() will never return nullptr. If thread-local support is not available, and user does not opt-out iostats context, then compilation will fail. The same applies to perf context as well. +* Added support for WriteBatchWithIndex::NewIteratorWithBase when overwrite_key=false. Previously, this combination was not supported and would assert or return nullptr. +* Improve the behavior of WriteBatchWithIndex for Merge operations. Now more operations may be stored in order to return the correct merged result. + +### Bug Fixes +* Use thread-safe `strerror_r()` to get error messages. +* Fixed a potential hang in shutdown for a DB whose `Env` has high-pri thread pool disabled (`Env::GetBackgroundThreads(Env::Priority::HIGH) == 0`) +* Made BackupEngine thread-safe and added documentation comments to clarify what is safe for multiple BackupEngine objects accessing the same backup directory. +* Fixed crash (divide by zero) when compression dictionary is applied to a file containing only range tombstones. +* Fixed a backward iteration bug with partitioned filter enabled: not including the prefix of the last key of the previous filter partition in current filter partition can cause wrong iteration result. +* Fixed a bug that allowed `DBOptions::max_open_files` to be set with a non-negative integer with `ColumnFamilyOptions::compaction_style = kCompactionStyleFIFO`. + +### Performance Improvements +* On ARM platform, use `yield` instead of `wfe` to relax cpu to gain better performance. + +### Public API change +* Added `TableProperties::slow_compression_estimated_data_size` and `TableProperties::fast_compression_estimated_data_size`. When `ColumnFamilyOptions::sample_for_compression > 0`, they estimate what `TableProperties::data_size` would have been if the "fast" or "slow" (see `ColumnFamilyOptions::sample_for_compression` API doc for definitions) compression had been used instead. +* Update DB::StartIOTrace and remove Env object from the arguments as its redundant and DB already has Env object that is passed down to IOTracer::StartIOTrace +* Added `FlushReason::kWalFull`, which is reported when a memtable is flushed due to the WAL reaching its size limit; those flushes were previously reported as `FlushReason::kWriteBufferManager`. Also, changed the reason for flushes triggered by the write buffer manager to `FlushReason::kWriteBufferManager`; they were previously reported as `FlushReason::kWriteBufferFull`. +* Extend file_checksum_dump ldb command and DB::GetLiveFilesChecksumInfo API for IntegratedBlobDB and get checksum of blob files along with SST files. + +### New Features +* Added the ability to open BackupEngine backups as read-only DBs, using BackupInfo::name_for_open and env_for_open provided by BackupEngine::GetBackupInfo() with include_file_details=true. +* Added BackupEngine support for integrated BlobDB, with blob files shared between backups when table files are shared. Because of current limitations, blob files always use the kLegacyCrc32cAndFileSize naming scheme, and incremental backups must read and checksum all blob files in a DB, even for files that are already backed up. +* Added an optional output parameter to BackupEngine::CreateNewBackup(WithMetadata) to return the BackupID of the new backup. +* Added BackupEngine::GetBackupInfo / GetLatestBackupInfo for querying individual backups. +* Made the Ribbon filter a long-term supported feature in terms of the SST schema(compatible with version >= 6.15.0) though the API for enabling it is expected to change. + +## 6.19.0 (2021-03-21) +### Bug Fixes +* Fixed the truncation error found in APIs/tools when dumping block-based SST files in a human-readable format. After fix, the block-based table can be fully dumped as a readable file. +* When hitting a write slowdown condition, no write delay (previously 1 millisecond) is imposed until `delayed_write_rate` is actually exceeded, with an initial burst allowance of 1 millisecond worth of bytes. Also, beyond the initial burst allowance, `delayed_write_rate` is now more strictly enforced, especially with multiple column families. + +### Public API change +* Changed default `BackupableDBOptions::share_files_with_checksum` to `true` and deprecated `false` because of potential for data loss. Note that accepting this change in behavior can temporarily increase backup data usage because files are not shared between backups using the two different settings. Also removed obsolete option kFlagMatchInterimNaming. +* Add a new option BlockBasedTableOptions::max_auto_readahead_size. RocksDB does auto-readahead for iterators on noticing more than two reads for a table file if user doesn't provide readahead_size. The readahead starts at 8KB and doubles on every additional read upto max_auto_readahead_size and now max_auto_readahead_size can be configured dynamically as well. Found that 256 KB readahead size provides the best performance, based on experiments, for auto readahead. Experiment data is in PR #3282. If value is set 0 then no automatic prefetching will be done by rocksdb. Also changing the value will only affect files opened after the change. +* Add suppport to extend DB::VerifyFileChecksums API to also verify blob files checksum. +* When using the new BlobDB, the amount of data written by flushes/compactions is now broken down into table files and blob files in the compaction statistics; namely, Write(GB) denotes the amount of data written to table files, while Wblob(GB) means the amount of data written to blob files. +* New default BlockBasedTableOptions::format_version=5 to enable new Bloom filter implementation by default, compatible with RocksDB versions >= 6.6.0. +* Add new SetBufferSize API to WriteBufferManager to allow dynamic management of memory allotted to all write buffers. This allows user code to adjust memory monitoring provided by WriteBufferManager as process memory needs change datasets grow and shrink. +* Clarified the required semantics of Read() functions in FileSystem and Env APIs. Please ensure any custom implementations are compliant. +* For the new integrated BlobDB implementation, compaction statistics now include the amount of data read from blob files during compaction (due to garbage collection or compaction filters). Write amplification metrics have also been extended to account for data read from blob files. +* Add EqualWithoutTimestamp() to Comparator. +* Extend support to track blob files in SSTFileManager whenever a blob file is created/deleted. Blob files will be scheduled to delete via SSTFileManager and SStFileManager will now take blob files in account while calculating size and space limits along with SST files. +* Add new Append and PositionedAppend API with checksum handoff to legacy Env. + +### New Features +* Support compaction filters for the new implementation of BlobDB. Add `FilterBlobByKey()` to `CompactionFilter`. Subclasses can override this method so that compaction filters can determine whether the actual blob value has to be read during compaction. Use a new `kUndetermined` in `CompactionFilter::Decision` to indicated that further action is necessary for compaction filter to make a decision. +* Add support to extend retrieval of checksums for blob files from the MANIFEST when checkpointing. During backup, rocksdb can detect corruption in blob files during file copies. +* Add new options for db_bench --benchmarks: flush, waitforcompaction, compact0, compact1. +* Add an option to BackupEngine::GetBackupInfo to include the name and size of each backed-up file. Especially in the presence of file sharing among backups, this offers detailed insight into backup space usage. +* Enable backward iteration on keys with user-defined timestamps. +* Add statistics and info log for error handler: counters for bg error, bg io error, bg retryable io error, auto resume count, auto resume total retry number, and auto resume sucess; Histogram for auto resume retry count in each recovery call. Note that, each auto resume attempt will have one or multiple retries. + +### Behavior Changes +* During flush, only WAL sync retryable IO error is mapped to hard error, which will stall the writes. When WAL is used but only SST file write has retryable IO error, it will be mapped to soft error and write will not be affected. + +## 6.18.0 (2021-02-19) +### Behavior Changes +* When retryable IO error occurs during compaction, it is mapped to soft error and set the BG error. However, auto resume is not called to clean the soft error since compaction will reschedule by itself. In this change, When retryable IO error occurs during compaction, BG error is not set. User will be informed the error via EventHelper. +* Introduce a new trace file format for query tracing and replay and trace file version is bump up to 0.2. A payload map is added as the first portion of the payload. We will not have backward compatible issues when adding new entries to trace records. Added the iterator_upper_bound and iterator_lower_bound in Seek and SeekForPrev tracing function. Added them as the new payload member for iterator tracing. + +### New Features +* Add support for key-value integrity protection in live updates from the user buffers provided to `WriteBatch` through the write to RocksDB's in-memory update buffer (memtable). This is intended to detect some cases of in-memory data corruption, due to either software or hardware errors. Users can enable protection by constructing their `WriteBatch` with `protection_bytes_per_key == 8`. +* Add support for updating `full_history_ts_low` option in manual compaction, which is for old timestamp data GC. +* Add a mechanism for using Makefile to build external plugin code into the RocksDB libraries/binaries. This intends to simplify compatibility and distribution for plugins (e.g., special-purpose `FileSystem`s) whose source code resides outside the RocksDB repo. See "plugin/README.md" for developer details, and "PLUGINS.md" for a listing of available plugins. +* Added memory pre-fetching for experimental Ribbon filter, which especially optimizes performance with batched MultiGet. +* A new, experimental version of BlobDB (key-value separation) is now available. The new implementation is integrated into the RocksDB core, i.e. it is accessible via the usual `rocksdb::DB` API, as opposed to the separate `rocksdb::blob_db::BlobDB` interface used by the earlier version, and can be configured on a per-column family basis using the configuration options `enable_blob_files`, `min_blob_size`, `blob_file_size`, `blob_compression_type`, `enable_blob_garbage_collection`, and `blob_garbage_collection_age_cutoff`. It extends RocksDB's consistency guarantees to blobs, and offers more features and better performance. Note that some features, most notably `Merge`, compaction filters, and backup/restore are not yet supported, and there is no support for migrating a database created by the old implementation. + +### Bug Fixes +* Since 6.15.0, `TransactionDB` returns error `Status`es from calls to `DeleteRange()` and calls to `Write()` where the `WriteBatch` contains a range deletion. Previously such operations may have succeeded while not providing the expected transactional guarantees. There are certain cases where range deletion can still be used on such DBs; see the API doc on `TransactionDB::DeleteRange()` for details. +* `OptimisticTransactionDB` now returns error `Status`es from calls to `DeleteRange()` and calls to `Write()` where the `WriteBatch` contains a range deletion. Previously such operations may have succeeded while not providing the expected transactional guarantees. +* Fix `WRITE_PREPARED`, `WRITE_UNPREPARED` TransactionDB `MultiGet()` may return uncommitted data with snapshot. +* In DB::OpenForReadOnly, if any error happens while checking Manifest file path, it was overridden by Status::NotFound. It has been fixed and now actual error is returned. + +### Public API Change +* Added a "only_mutable_options" flag to the ConfigOptions. When this flag is "true", the Configurable functions and convenience methods (such as GetDBOptionsFromString) will only deal with options that are marked as mutable. When this flag is true, only options marked as mutable can be configured (a Status::InvalidArgument will be returned) and options not marked as mutable will not be returned or compared. The default is "false", meaning to compare all options. +* Add new Append and PositionedAppend APIs to FileSystem to bring the data verification information (data checksum information) from upper layer (e.g., WritableFileWriter) to the storage layer. In this way, the customized FileSystem is able to verify the correctness of data being written to the storage on time. Add checksum_handoff_file_types to DBOptions. User can use this option to control which file types (Currently supported file tyes: kWALFile, kTableFile, kDescriptorFile.) should use the new Append and PositionedAppend APIs to handoff the verification information. Currently, RocksDB only use crc32c to calculate the checksum for write handoff. +* Add an option, `CompressionOptions::max_dict_buffer_bytes`, to limit the in-memory buffering for selecting samples for generating/training a dictionary. The limit is currently loosely adhered to. + + +## 6.17.0 (2021-01-15) +### Behavior Changes +* When verifying full file checksum with `DB::VerifyFileChecksums()`, we now fail with `Status::InvalidArgument` if the name of the checksum generator used for verification does not match the name of the checksum generator used for protecting the file when it was created. +* Since RocksDB does not continue write the same file if a file write fails for any reason, the file scope write IO error is treated the same as retryable IO error. More information about error handling of file scope IO error is included in `ErrorHandler::SetBGError`. + +### Bug Fixes +* Version older than 6.15 cannot decode VersionEdits `WalAddition` and `WalDeletion`, fixed this by changing the encoded format of them to be ignorable by older versions. +* Fix a race condition between DB startups and shutdowns in managing the periodic background worker threads. One effect of this race condition could be the process being terminated. + +### Public API Change +* Add a public API WriteBufferManager::dummy_entries_in_cache_usage() which reports the size of dummy entries stored in cache (passed to WriteBufferManager). Dummy entries are used to account for DataBlocks. +* Add a SystemClock class that contains the time-related methods from Env. The original methods in Env may be deprecated in a future release. This class will allow easier testing, development, and expansion of time-related features. +* Add a public API GetRocksBuildProperties and GetRocksBuildInfoAsString to get properties about the current build. These properties may include settings related to the GIT settings (branch, timestamp). This change also sets the "build date" based on the GIT properties, rather than the actual build time, thereby enabling more reproducible builds. + +## 6.16.0 (2020-12-18) +### Behavior Changes +* Attempting to write a merge operand without explicitly configuring `merge_operator` now fails immediately, causing the DB to enter read-only mode. Previously, failure was deferred until the `merge_operator` was needed by a user read or a background operation. + +### Bug Fixes +* Truncated WALs ending in incomplete records can no longer produce gaps in the recovered data when `WALRecoveryMode::kPointInTimeRecovery` is used. Gaps are still possible when WALs are truncated exactly on record boundaries; for complete protection, users should enable `track_and_verify_wals_in_manifest`. +* Fix a bug where compressed blocks read by MultiGet are not inserted into the compressed block cache when use_direct_reads = true. +* Fixed the issue of full scanning on obsolete files when there are too many outstanding compactions with ConcurrentTaskLimiter enabled. +* Fixed the logic of populating native data structure for `read_amp_bytes_per_bit` during OPTIONS file parsing on big-endian architecture. Without this fix, original code introduced in PR7659, when running on big-endian machine, can mistakenly store read_amp_bytes_per_bit (an uint32) in little endian format. Future access to `read_amp_bytes_per_bit` will give wrong values. Little endian architecture is not affected. +* Fixed prefix extractor with timestamp issues. +* Fixed a bug in atomic flush: in two-phase commit mode, the minimum WAL log number to keep is incorrect. +* Fixed a bug related to checkpoint in PR7789: if there are multiple column families, and the checkpoint is not opened as read only, then in rare cases, data loss may happen in the checkpoint. Since backup engine relies on checkpoint, it may also be affected. +* When ldb --try_load_options is used with the --column_family option, the ColumnFamilyOptions for the specified column family was not loaded from the OPTIONS file. Fix it so its loaded from OPTIONS and then overridden with command line overrides. + +### New Features +* User defined timestamp feature supports `CompactRange` and `GetApproximateSizes`. +* Support getting aggregated table properties (kAggregatedTableProperties and kAggregatedTablePropertiesAtLevel) with DB::GetMapProperty, for easier access to the data in a structured format. +* Experimental option BlockBasedTableOptions::optimize_filters_for_memory now works with experimental Ribbon filter (as well as Bloom filter). + +### Public API Change +* Deprecated public but rarely-used FilterBitsBuilder::CalculateNumEntry, which is replaced with ApproximateNumEntries taking a size_t parameter and returning size_t. +* To improve portability the functions `Env::GetChildren` and `Env::GetChildrenFileAttributes` will no longer return entries for the special directories `.` or `..`. +* Added a new option `track_and_verify_wals_in_manifest`. If `true`, the log numbers and sizes of the synced WALs are tracked in MANIFEST, then during DB recovery, if a synced WAL is missing from disk, or the WAL's size does not match the recorded size in MANIFEST, an error will be reported and the recovery will be aborted. Note that this option does not work with secondary instance. +* `rocksdb_approximate_sizes` and `rocksdb_approximate_sizes_cf` in the C API now requires an error pointer (`char** errptr`) for receiving any error. +* All overloads of DB::GetApproximateSizes now return Status, so that any failure to obtain the sizes is indicated to the caller. + +## 6.15.0 (2020-11-13) +### Bug Fixes +* Fixed a bug in the following combination of features: indexes with user keys (`format_version >= 3`), indexes are partitioned (`index_type == kTwoLevelIndexSearch`), and some index partitions are pinned in memory (`BlockBasedTableOptions::pin_l0_filter_and_index_blocks_in_cache`). The bug could cause keys to be truncated when read from the index leading to wrong read results or other unexpected behavior. +* Fixed a bug when indexes are partitioned (`index_type == kTwoLevelIndexSearch`), some index partitions are pinned in memory (`BlockBasedTableOptions::pin_l0_filter_and_index_blocks_in_cache`), and partitions reads could be mixed between block cache and directly from the file (e.g., with `enable_index_compression == 1` and `mmap_read == 1`, partitions that were stored uncompressed due to poor compression ratio would be read directly from the file via mmap, while partitions that were stored compressed would be read from block cache). The bug could cause index partitions to be mistakenly considered empty during reads leading to wrong read results. +* Since 6.12, memtable lookup should report unrecognized value_type as corruption (#7121). +* Since 6.14, fix false positive flush/compaction `Status::Corruption` failure when `paranoid_file_checks == true` and range tombstones were written to the compaction output files. +* Since 6.14, fix a bug that could cause a stalled write to crash with mixed of slowdown and no_slowdown writes (`WriteOptions.no_slowdown=true`). +* Fixed a bug which causes hang in closing DB when refit level is set in opt build. It was because ContinueBackgroundWork() was called in assert statement which is a no op. It was introduced in 6.14. +* Fixed a bug which causes Get() to return incorrect result when a key's merge operand is applied twice. This can occur if the thread performing Get() runs concurrently with a background flush thread and another thread writing to the MANIFEST file (PR6069). +* Reverted a behavior change silently introduced in 6.14.2, in which the effects of the `ignore_unknown_options` flag (used in option parsing/loading functions) changed. +* Reverted a behavior change silently introduced in 6.14, in which options parsing/loading functions began returning `NotFound` instead of `InvalidArgument` for option names not available in the present version. +* Fixed MultiGet bugs it doesn't return valid data with user defined timestamp. +* Fixed a potential bug caused by evaluating `TableBuilder::NeedCompact()` before `TableBuilder::Finish()` in compaction job. For example, the `NeedCompact()` method of `CompactOnDeletionCollector` returned by built-in `CompactOnDeletionCollectorFactory` requires `BlockBasedTable::Finish()` to return the correct result. The bug can cause a compaction-generated file not to be marked for future compaction based on deletion ratio. +* Fixed a seek issue with prefix extractor and timestamp. +* Fixed a bug of encoding and parsing BlockBasedTableOptions::read_amp_bytes_per_bit as a 64-bit integer. +* Fixed a bug of a recovery corner case, details in PR7621. + +### Public API Change +* Deprecate `BlockBasedTableOptions::pin_l0_filter_and_index_blocks_in_cache` and `BlockBasedTableOptions::pin_top_level_index_and_filter`. These options still take effect until users migrate to the replacement APIs in `BlockBasedTableOptions::metadata_cache_options`. Migration guidance can be found in the API comments on the deprecated options. +* Add new API `DB::VerifyFileChecksums` to verify SST file checksum with corresponding entries in the MANIFEST if present. Current implementation requires scanning and recomputing file checksums. + +### Behavior Changes +* The dictionary compression settings specified in `ColumnFamilyOptions::compression_opts` now additionally affect files generated by flush and compaction to non-bottommost level. Previously those settings at most affected files generated by compaction to bottommost level, depending on whether `ColumnFamilyOptions::bottommost_compression_opts` overrode them. Users who relied on dictionary compression settings in `ColumnFamilyOptions::compression_opts` affecting only the bottommost level can keep the behavior by moving their dictionary settings to `ColumnFamilyOptions::bottommost_compression_opts` and setting its `enabled` flag. +* When the `enabled` flag is set in `ColumnFamilyOptions::bottommost_compression_opts`, those compression options now take effect regardless of the value in `ColumnFamilyOptions::bottommost_compression`. Previously, those compression options only took effect when `ColumnFamilyOptions::bottommost_compression != kDisableCompressionOption`. Now, they additionally take effect when `ColumnFamilyOptions::bottommost_compression == kDisableCompressionOption` (such a setting causes bottommost compression type to fall back to `ColumnFamilyOptions::compression_per_level` if configured, and otherwise fall back to `ColumnFamilyOptions::compression`). + +### New Features +* An EXPERIMENTAL new Bloom alternative that saves about 30% space compared to Bloom filters, with about 3-4x construction time and similar query times is available using NewExperimentalRibbonFilterPolicy. + +## 6.14 (2020-10-09) +### Bug fixes +* Fixed a bug after a `CompactRange()` with `CompactRangeOptions::change_level` set fails due to a conflict in the level change step, which caused all subsequent calls to `CompactRange()` with `CompactRangeOptions::change_level` set to incorrectly fail with a `Status::NotSupported("another thread is refitting")` error. +* Fixed a bug that the bottom most level compaction could still be a trivial move even if `BottommostLevelCompaction.kForce` or `kForceOptimized` is set. + +### Public API Change +* The methods to create and manage EncrypedEnv have been changed. The EncryptionProvider is now passed to NewEncryptedEnv as a shared pointer, rather than a raw pointer. Comparably, the CTREncryptedProvider now takes a shared pointer, rather than a reference, to a BlockCipher. CreateFromString methods have been added to BlockCipher and EncryptionProvider to provide a single API by which different ciphers and providers can be created, respectively. +* The internal classes (CTREncryptionProvider, ROT13BlockCipher, CTRCipherStream) associated with the EncryptedEnv have been moved out of the public API. To create a CTREncryptionProvider, one can either use EncryptionProvider::NewCTRProvider, or EncryptionProvider::CreateFromString("CTR"). To create a new ROT13BlockCipher, one can either use BlockCipher::NewROT13Cipher or BlockCipher::CreateFromString("ROT13"). +* The EncryptionProvider::AddCipher method has been added to allow keys to be added to an EncryptionProvider. This API will allow future providers to support multiple cipher keys. +* Add a new option "allow_data_in_errors". When this new option is set by users, it allows users to opt-in to get error messages containing corrupted keys/values. Corrupt keys, values will be logged in the messages, logs, status etc. that will help users with the useful information regarding affected data. By default value of this option is set false to prevent users data to be exposed in the messages so currently, data will be redacted from logs, messages, status by default. +* AdvancedColumnFamilyOptions::force_consistency_checks is now true by default, for more proactive DB corruption detection at virtually no cost (estimated two extra CPU cycles per million on a major production workload). Corruptions reported by these checks now mention "force_consistency_checks" in case a false positive corruption report is suspected and the option needs to be disabled (unlikely). Since existing column families have a saved setting for force_consistency_checks, only new column families will pick up the new default. + +### General Improvements +* The settings of the DBOptions and ColumnFamilyOptions are now managed by Configurable objects (see New Features). The same convenience methods to configure these options still exist but the backend implementation has been unified under a common implementation. + +### New Features + +* Methods to configure serialize, and compare -- such as TableFactory -- are exposed directly through the Configurable base class (from which these objects inherit). This change will allow for better and more thorough configuration management and retrieval in the future. The options for a Configurable object can be set via the ConfigureFromMap, ConfigureFromString, or ConfigureOption method. The serialized version of the options of an object can be retrieved via the GetOptionString, ToString, or GetOption methods. The list of options supported by an object can be obtained via the GetOptionNames method. The "raw" object (such as the BlockBasedTableOption) for an option may be retrieved via the GetOptions method. Configurable options can be compared via the AreEquivalent method. The settings within a Configurable object may be validated via the ValidateOptions method. The object may be intialized (at which point only mutable options may be updated) via the PrepareOptions method. +* Introduce options.check_flush_compaction_key_order with default value to be true. With this option, during flush and compaction, key order will be checked when writing to each SST file. If the order is violated, the flush or compaction will fail. +* Added is_full_compaction to CompactionJobStats, so that the information is available through the EventListener interface. +* Add more stats for MultiGet in Histogram to get number of data blocks, index blocks, filter blocks and sst files read from file system per level. +* SST files have a new table property called db_host_id, which is set to the hostname by default. A new option in DBOptions, db_host_id, allows the property value to be overridden with a user specified string, or disable it completely by making the option string empty. +* Methods to create customizable extensions -- such as TableFactory -- are exposed directly through the Customizable base class (from which these objects inherit). This change will allow these Customizable classes to be loaded and configured in a standard way (via CreateFromString). More information on how to write and use Customizable classes is in the customizable.h header file. + +## 6.13 (2020-09-12) +### Bug fixes +* Fix a performance regression introduced in 6.4 that makes a upper bound check for every Next() even if keys are within a data block that is within the upper bound. +* Fix a possible corruption to the LSM state (overlapping files within a level) when a `CompactRange()` for refitting levels (`CompactRangeOptions::change_level == true`) and another manual compaction are executed in parallel. +* Sanitize `recycle_log_file_num` to zero when the user attempts to enable it in combination with `WALRecoveryMode::kTolerateCorruptedTailRecords`. Previously the two features were allowed together, which compromised the user's configured crash-recovery guarantees. +* Fix a bug where a level refitting in CompactRange() might race with an automatic compaction that puts the data to the target level of the refitting. The bug has been there for years. +* Fixed a bug in version 6.12 in which BackupEngine::CreateNewBackup could fail intermittently with non-OK status when backing up a read-write DB configured with a DBOptions::file_checksum_gen_factory. +* Fix useless no-op compactions scheduled upon snapshot release when options.disable-auto-compactions = true. +* Fix a bug when max_write_buffer_size_to_maintain is set, immutable flushed memtable destruction is delayed until the next super version is installed. A memtable is not added to delete list because of its reference hold by super version and super version doesn't switch because of empt delete list. So memory usage keeps on increasing beyond write_buffer_size + max_write_buffer_size_to_maintain. +* Avoid converting MERGES to PUTS when allow_ingest_behind is true. +* Fix compression dictionary sampling together with `SstFileWriter`. Previously, the dictionary would be trained/finalized immediately with zero samples. Now, the whole `SstFileWriter` file is buffered in memory and then sampled. +* Fix a bug with `avoid_unnecessary_blocking_io=1` and creating backups (BackupEngine::CreateNewBackup) or checkpoints (Checkpoint::Create). With this setting and WAL enabled, these operations could randomly fail with non-OK status. +* Fix a bug in which bottommost compaction continues to advance the underlying InternalIterator to skip tombstones even after shutdown. + +### New Features +* A new field `std::string requested_checksum_func_name` is added to `FileChecksumGenContext`, which enables the checksum factory to create generators for a suite of different functions. +* Added a new subcommand, `ldb unsafe_remove_sst_file`, which removes a lost or corrupt SST file from a DB's metadata. This command involves data loss and must not be used on a live DB. + +### Performance Improvements +* Reduce thread number for multiple DB instances by re-using one global thread for statistics dumping and persisting. +* Reduce write-amp in heavy write bursts in `kCompactionStyleLevel` compaction style with `level_compaction_dynamic_level_bytes` set. +* BackupEngine incremental backups no longer read DB table files that are already saved to a shared part of the backup directory, unless `share_files_with_checksum` is used with `kLegacyCrc32cAndFileSize` naming (discouraged). + * For `share_files_with_checksum`, we are confident there is no regression (vs. pre-6.12) in detecting DB or backup corruption at backup creation time, mostly because the old design did not leverage this extra checksum computation for detecting inconsistencies at backup creation time. + * For `share_table_files` without "checksum" (not recommended), there is a regression in detecting fundamentally unsafe use of the option, greatly mitigated by file size checking (under "Behavior Changes"). Almost no reason to use `share_files_with_checksum=false` should remain. + * `DB::VerifyChecksum` and `BackupEngine::VerifyBackup` with checksum checking are still able to catch corruptions that `CreateNewBackup` does not. + +### Public API Change +* Expose kTypeDeleteWithTimestamp in EntryType and update GetEntryType() accordingly. +* Added file_checksum and file_checksum_func_name to TableFileCreationInfo, which can pass the table file checksum information through the OnTableFileCreated callback during flush and compaction. +* A warning is added to `DB::DeleteFile()` API describing its known problems and deprecation plan. +* Add a new stats level, i.e. StatsLevel::kExceptTickers (PR7329) to exclude tickers even if application passes a non-null Statistics object. +* Added a new status code IOStatus::IOFenced() for the Env/FileSystem to indicate that writes from this instance are fenced off. Like any other background error, this error is returned to the user in Put/Merge/Delete/Flush calls and can be checked using Status::IsIOFenced(). + +### Behavior Changes +* File abstraction `FSRandomAccessFile.Prefetch()` default return status is changed from `OK` to `NotSupported`. If the user inherited file doesn't implement prefetch, RocksDB will create internal prefetch buffer to improve read performance. +* When retryabel IO error happens during Flush (manifest write error is excluded) and WAL is disabled, originally it is mapped to kHardError. Now,it is mapped to soft error. So DB will not stall the writes unless the memtable is full. At the same time, when auto resume is triggered to recover the retryable IO error during Flush, SwitchMemtable is not called to avoid generating to many small immutable memtables. If WAL is enabled, no behavior changes. +* When considering whether a table file is already backed up in a shared part of backup directory, BackupEngine would already query the sizes of source (DB) and pre-existing destination (backup) files. BackupEngine now uses these file sizes to detect corruption, as at least one of (a) old backup, (b) backup in progress, or (c) current DB is corrupt if there's a size mismatch. + +### Others +* Error in prefetching partitioned index blocks will not be swallowed. It will fail the query and return the IOError users. + +## 6.12 (2020-07-28) +### Public API Change +* Encryption file classes now exposed for inheritance in env_encryption.h +* File I/O listener is extended to cover more I/O operations. Now class `EventListener` in listener.h contains new callback functions: `OnFileFlushFinish()`, `OnFileSyncFinish()`, `OnFileRangeSyncFinish()`, `OnFileTruncateFinish()`, and ``OnFileCloseFinish()``. +* `FileOperationInfo` now reports `duration` measured by `std::chrono::steady_clock` and `start_ts` measured by `std::chrono::system_clock` instead of start and finish timestamps measured by `system_clock`. Note that `system_clock` is called before `steady_clock` in program order at operation starts. +* `DB::GetDbSessionId(std::string& session_id)` is added. `session_id` stores a unique identifier that gets reset every time the DB is opened. This DB session ID should be unique among all open DB instances on all hosts, and should be unique among re-openings of the same or other DBs. This identifier is recorded in the LOG file on the line starting with "DB Session ID:". +* `DB::OpenForReadOnly()` now returns `Status::NotFound` when the specified DB directory does not exist. Previously the error returned depended on the underlying `Env`. This change is available in all 6.11 releases as well. +* A parameter `verify_with_checksum` is added to `BackupEngine::VerifyBackup`, which is false by default. If it is ture, `BackupEngine::VerifyBackup` verifies checksums and file sizes of backup files. Pass `false` for `verify_with_checksum` to maintain the previous behavior and performance of `BackupEngine::VerifyBackup`, by only verifying sizes of backup files. + +### Behavior Changes +* Best-efforts recovery ignores CURRENT file completely. If CURRENT file is missing during recovery, best-efforts recovery still proceeds with MANIFEST file(s). +* In best-efforts recovery, an error that is not Corruption or IOError::kNotFound or IOError::kPathNotFound will be overwritten silently. Fix this by checking all non-ok cases and return early. +* When `file_checksum_gen_factory` is set to `GetFileChecksumGenCrc32cFactory()`, BackupEngine will compare the crc32c checksums of table files computed when creating a backup to the expected checksums stored in the DB manifest, and will fail `CreateNewBackup()` on mismatch (corruption). If the `file_checksum_gen_factory` is not set or set to any other customized factory, there is no checksum verification to detect if SST files in a DB are corrupt when read, copied, and independently checksummed by BackupEngine. +* When a DB sets `stats_dump_period_sec > 0`, either as the initial value for DB open or as a dynamic option change, the first stats dump is staggered in the following X seconds, where X is an integer in `[0, stats_dump_period_sec)`. Subsequent stats dumps are still spaced `stats_dump_period_sec` seconds apart. +* When the paranoid_file_checks option is true, a hash is generated of all keys and values are generated when the SST file is written, and then the values are read back in to validate the file. A corruption is signaled if the two hashes do not match. + +### Bug fixes +* Compressed block cache was automatically disabled with read-only DBs by mistake. Now it is fixed: compressed block cache will be in effective with read-only DB too. +* Fix a bug of wrong iterator result if another thread finishes an update and a DB flush between two statement. +* Disable file deletion after MANIFEST write/sync failure until db re-open or Resume() so that subsequent re-open will not see MANIFEST referencing deleted SSTs. +* Fix a bug when index_type == kTwoLevelIndexSearch in PartitionedIndexBuilder to update FlushPolicy to point to internal key partitioner when it changes from user-key mode to internal-key mode in index partition. +* Make compaction report InternalKey corruption while iterating over the input. +* Fix a bug which may cause MultiGet to be slow because it may read more data than requested, but this won't affect correctness. The bug was introduced in 6.10 release. +* Fail recovery and report once hitting a physical log record checksum mismatch, while reading MANIFEST. RocksDB should not continue processing the MANIFEST any further. +* Fixed a bug in size-amp-triggered and periodic-triggered universal compaction, where the compression settings for the first input level were used rather than the compression settings for the output (bottom) level. + +### New Features +* DB identity (`db_id`) and DB session identity (`db_session_id`) are added to table properties and stored in SST files. SST files generated from SstFileWriter and Repairer have DB identity “SST Writer” and “DB Repairer”, respectively. Their DB session IDs are generated in the same way as `DB::GetDbSessionId`. The session ID for SstFileWriter (resp., Repairer) resets every time `SstFileWriter::Open` (resp., `Repairer::Run`) is called. +* Added experimental option BlockBasedTableOptions::optimize_filters_for_memory for reducing allocated memory size of Bloom filters (~10% savings with Jemalloc) while preserving the same general accuracy. To have an effect, the option requires format_version=5 and malloc_usable_size. Enabling this option is forward and backward compatible with existing format_version=5. +* `BackupableDBOptions::share_files_with_checksum_naming` is added with new default behavior for naming backup files with `share_files_with_checksum`, to address performance and backup integrity issues. See API comments for details. +* Added auto resume function to automatically recover the DB from background Retryable IO Error. When retryable IOError happens during flush and WAL write, the error is mapped to Hard Error and DB will be in read mode. When retryable IO Error happens during compaction, the error will be mapped to Soft Error. DB is still in write/read mode. Autoresume function will create a thread for a DB to call DB->ResumeImpl() to try the recover for Retryable IO Error during flush and WAL write. Compaction will be rescheduled by itself if retryable IO Error happens. Auto resume may also cause other Retryable IO Error during the recovery, so the recovery will fail. Retry the auto resume may solve the issue, so we use max_bgerror_resume_count to decide how many resume cycles will be tried in total. If it is <=0, auto resume retryable IO Error is disabled. Default is INT_MAX, which will lead to a infinit auto resume. bgerror_resume_retry_interval decides the time interval between two auto resumes. +* Option `max_subcompactions` can be set dynamically using DB::SetDBOptions(). +* Added experimental ColumnFamilyOptions::sst_partitioner_factory to define determine the partitioning of sst files. This helps compaction to split the files on interesting boundaries (key prefixes) to make propagation of sst files less write amplifying (covering the whole key space). + +### Performance Improvements +* Eliminate key copies for internal comparisons while accessing ingested block-based tables. +* Reduce key comparisons during random access in all block-based tables. +* BackupEngine avoids unnecessary repeated checksum computation for backing up a table file to the `shared_checksum` directory when using `share_files_with_checksum_naming = kUseDbSessionId` (new default), except on SST files generated before this version of RocksDB, which fall back on using `kLegacyCrc32cAndFileSize`. + +## 6.11 (2020-06-12) +### Bug Fixes +* Fix consistency checking error swallowing in some cases when options.force_consistency_checks = true. +* Fix possible false NotFound status from batched MultiGet using index type kHashSearch. +* Fix corruption caused by enabling delete triggered compaction (NewCompactOnDeletionCollectorFactory) in universal compaction mode, along with parallel compactions. The bug can result in two parallel compactions picking the same input files, resulting in the DB resurrecting older and deleted versions of some keys. +* Fix a use-after-free bug in best-efforts recovery. column_family_memtables_ needs to point to valid ColumnFamilySet. +* Let best-efforts recovery ignore corrupted files during table loading. +* Fix corrupt key read from ingested file when iterator direction switches from reverse to forward at a key that is a prefix of another key in the same file. It is only possible in files with a non-zero global seqno. +* Fix abnormally large estimate from GetApproximateSizes when a range starts near the end of one SST file and near the beginning of another. Now GetApproximateSizes consistently and fairly includes the size of SST metadata in addition to data blocks, attributing metadata proportionally among the data blocks based on their size. +* Fix potential file descriptor leakage in PosixEnv's IsDirectory() and NewRandomAccessFile(). +* Fix false negative from the VerifyChecksum() API when there is a checksum mismatch in an index partition block in a BlockBasedTable format table file (index_type is kTwoLevelIndexSearch). +* Fix sst_dump to return non-zero exit code if the specified file is not a recognized SST file or fails requested checks. +* Fix incorrect results from batched MultiGet for duplicate keys, when the duplicate key matches the largest key of an SST file and the value type for the key in the file is a merge value. + +### Public API Change +* Flush(..., column_family) may return Status::ColumnFamilyDropped() instead of Status::InvalidArgument() if column_family is dropped while processing the flush request. +* BlobDB now explicitly disallows using the default column family's storage directories as blob directory. +* DeleteRange now returns `Status::InvalidArgument` if the range's end key comes before its start key according to the user comparator. Previously the behavior was undefined. +* ldb now uses options.force_consistency_checks = true by default and "--disable_consistency_checks" is added to disable it. +* DB::OpenForReadOnly no longer creates files or directories if the named DB does not exist, unless create_if_missing is set to true. +* The consistency checks that validate LSM state changes (table file additions/deletions during flushes and compactions) are now stricter, more efficient, and no longer optional, i.e. they are performed even if `force_consistency_checks` is `false`. +* Disable delete triggered compaction (NewCompactOnDeletionCollectorFactory) in universal compaction mode and num_levels = 1 in order to avoid a corruption bug. +* `pin_l0_filter_and_index_blocks_in_cache` no longer applies to L0 files larger than `1.5 * write_buffer_size` to give more predictable memory usage. Such L0 files may exist due to intra-L0 compaction, external file ingestion, or user dynamically changing `write_buffer_size` (note, however, that files that are already pinned will continue being pinned, even after such a dynamic change). +* In point-in-time wal recovery mode, fail database recovery in case of IOError while reading the WAL to avoid data loss. +* A new method `Env::LowerThreadPoolCPUPriority(Priority, CpuPriority)` is added to `Env` to be able to lower to a specific priority such as `CpuPriority::kIdle`. + +### New Features +* sst_dump to add a new --readahead_size argument. Users can specify read size when scanning the data. Sst_dump also tries to prefetch tail part of the SST files so usually some number of I/Os are saved there too. +* Generate file checksum in SstFileWriter if Options.file_checksum_gen_factory is set. The checksum and checksum function name are stored in ExternalSstFileInfo after the sst file write is finished. +* Add a value_size_soft_limit in read options which limits the cumulative value size of keys read in batches in MultiGet. Once the cumulative value size of found keys exceeds read_options.value_size_soft_limit, all the remaining keys are returned with status Abort without further finding their values. By default the value_size_soft_limit is std::numeric_limits::max(). +* Enable SST file ingestion with file checksum information when calling IngestExternalFiles(const std::vector& args). Added files_checksums and files_checksum_func_names to IngestExternalFileArg such that user can ingest the sst files with their file checksum information. Added verify_file_checksum to IngestExternalFileOptions (default is True). To be backward compatible, if DB does not enable file checksum or user does not provide checksum information (vectors of files_checksums and files_checksum_func_names are both empty), verification of file checksum is always sucessful. If DB enables file checksum, DB will always generate the checksum for each ingested SST file during Prepare stage of ingestion and store the checksum in Manifest, unless verify_file_checksum is False and checksum information is provided by the application. In this case, we only verify the checksum function name and directly store the ingested checksum in Manifest. If verify_file_checksum is set to True, DB will verify the ingested checksum and function name with the genrated ones. Any mismatch will fail the ingestion. Note that, if IngestExternalFileOptions::write_global_seqno is True, the seqno will be changed in the ingested file. Therefore, the checksum of the file will be changed. In this case, a new checksum will be generated after the seqno is updated and be stored in the Manifest. + +### Performance Improvements +* Eliminate redundant key comparisons during random access in block-based tables. + +## 6.10 (2020-05-02) +### Bug Fixes +* Fix wrong result being read from ingested file. May happen when a key in the file happen to be prefix of another key also in the file. The issue can further cause more data corruption. The issue exists with rocksdb >= 5.0.0 since DB::IngestExternalFile() was introduced. +* Finish implementation of BlockBasedTableOptions::IndexType::kBinarySearchWithFirstKey. It's now ready for use. Significantly reduces read amplification in some setups, especially for iterator seeks. +* Fix a bug by updating CURRENT file so that it points to the correct MANIFEST file after best-efforts recovery. +* Fixed a bug where ColumnFamilyHandle objects were not cleaned up in case an error happened during BlobDB's open after the base DB had been opened. +* Fix a potential undefined behavior caused by trying to dereference nullable pointer (timestamp argument) in DB::MultiGet. +* Fix a bug caused by not including user timestamp in MultiGet LookupKey construction. This can lead to wrong query result since the trailing bytes of a user key, if not shorter than timestamp, will be mistaken for user timestamp. +* Fix a bug caused by using wrong compare function when sorting the input keys of MultiGet with timestamps. +* Upgraded version of bzip library (1.0.6 -> 1.0.8) used with RocksJava to address potential vulnerabilities if an attacker can manipulate compressed data saved and loaded by RocksDB (not normal). See issue #6703. + +### Public API Change +* Add a ConfigOptions argument to the APIs dealing with converting options to and from strings and files. The ConfigOptions is meant to replace some of the options (such as input_strings_escaped and ignore_unknown_options) and allow for more parameters to be passed in the future without changing the function signature. +* Add NewFileChecksumGenCrc32cFactory to the file checksum public API, such that the builtin Crc32c based file checksum generator factory can be used by applications. +* Add IsDirectory to Env and FS to indicate if a path is a directory. + +### New Features +* Added support for pipelined & parallel compression optimization for `BlockBasedTableBuilder`. This optimization makes block building, block compression and block appending a pipeline, and uses multiple threads to accelerate block compression. Users can set `CompressionOptions::parallel_threads` greater than 1 to enable compression parallelism. This feature is experimental for now. +* Provide an allocator for memkind to be used with block cache. This is to work with memory technologies (Intel DCPMM is one such technology currently available) that require different libraries for allocation and management (such as PMDK and memkind). The high capacities available make it possible to provision large caches (up to several TBs in size) beyond what is achievable with DRAM. +* Option `max_background_flushes` can be set dynamically using DB::SetDBOptions(). +* Added functionality in sst_dump tool to check the compressed file size for different compression levels and print the time spent on compressing files with each compression type. Added arguments `--compression_level_from` and `--compression_level_to` to report size of all compression levels and one compression_type must be specified with it so that it will report compressed sizes of one compression type with different levels. +* Added statistics for redundant insertions into block cache: rocksdb.block.cache.*add.redundant. (There is currently no coordination to ensure that only one thread loads a table block when many threads are trying to access that same table block.) + +### Bug Fixes +* Fix a bug when making options.bottommost_compression, options.compression_opts and options.bottommost_compression_opts dynamically changeable: the modified values are not written to option files or returned back to users when being queried. +* Fix a bug where index key comparisons were unaccounted in `PerfContext::user_key_comparison_count` for lookups in files written with `format_version >= 3`. +* Fix many bloom.filter statistics not being updated in batch MultiGet. + +### Performance Improvements +* Improve performance of batch MultiGet with partitioned filters, by sharing block cache lookups to applicable filter blocks. +* Reduced memory copies when fetching and uncompressing compressed blocks from sst files. + +## 6.9.0 (2020-03-29) +### Behavior changes +* Since RocksDB 6.8, ttl-based FIFO compaction can drop a file whose oldest key becomes older than options.ttl while others have not. This fix reverts this and makes ttl-based FIFO compaction use the file's flush time as the criterion. This fix also requires that max_open_files = -1 and compaction_options_fifo.allow_compaction = false to function properly. + +### Public API Change +* Fix spelling so that API now has correctly spelled transaction state name `COMMITTED`, while the old misspelled `COMMITED` is still available as an alias. +* Updated default format_version in BlockBasedTableOptions from 2 to 4. SST files generated with the new default can be read by RocksDB versions 5.16 and newer, and use more efficient encoding of keys in index blocks. +* A new parameter `CreateBackupOptions` is added to both `BackupEngine::CreateNewBackup` and `BackupEngine::CreateNewBackupWithMetadata`, you can decrease CPU priority of `BackupEngine`'s background threads by setting `decrease_background_thread_cpu_priority` and `background_thread_cpu_priority` in `CreateBackupOptions`. +* Updated the public API of SST file checksum. Introduce the FileChecksumGenFactory to create the FileChecksumGenerator for each SST file, such that the FileChecksumGenerator is not shared and it can be more general for checksum implementations. Changed the FileChecksumGenerator interface from Value, Extend, and GetChecksum to Update, Finalize, and GetChecksum. Finalize should be only called once after all data is processed to generate the final checksum. Temproal data should be maintained by the FileChecksumGenerator object itself and finally it can return the checksum string. + +### Bug Fixes +* Fix a bug where range tombstone blocks in ingested files were cached incorrectly during ingestion. If range tombstones were read from those incorrectly cached blocks, the keys they covered would be exposed. +* Fix a data race that might cause crash when calling DB::GetCreationTimeOfOldestFile() by a small chance. The bug was introduced in 6.6 Release. +* Fix a bug where a boolean value optimize_filters_for_hits was for max threads when calling load table handles after a flush or compaction. The value is correct to 1. The bug should not cause user visible problems. +* Fix a bug which might crash the service when write buffer manager fails to insert the dummy handle to the block cache. + +### Performance Improvements +* In CompactRange, for levels starting from 0, if the level does not have any file with any key falling in the specified range, the level is skipped. So instead of always compacting from level 0, the compaction starts from the first level with keys in the specified range until the last such level. +* Reduced memory copy when reading sst footer and blobdb in direct IO mode. +* When restarting a database with large numbers of sst files, large amount of CPU time is spent on getting logical block size of the sst files, which slows down the starting progress, this inefficiency is optimized away with an internal cache for the logical block sizes. + +### New Features +* Basic support for user timestamp in iterator. Seek/SeekToFirst/Next and lower/upper bounds are supported. Reverse iteration is not supported. Merge is not considered. +* When file lock failure when the lock is held by the current process, return acquiring time and thread ID in the error message. +* Added a new option, best_efforts_recovery (default: false), to allow database to open in a db dir with missing table files. During best efforts recovery, missing table files are ignored, and database recovers to the most recent state without missing table file. Cross-column-family consistency is not guaranteed even if WAL is enabled. +* options.bottommost_compression, options.compression_opts and options.bottommost_compression_opts are now dynamically changeable. + +## 6.8.0 (2020-02-24) +### Java API Changes +* Major breaking changes to Java comparators, toward standardizing on ByteBuffer for performant, locale-neutral operations on keys (#6252). +* Added overloads of common API methods using direct ByteBuffers for keys and values (#2283). + +### Bug Fixes +* Fix incorrect results while block-based table uses kHashSearch, together with Prev()/SeekForPrev(). +* Fix a bug that prevents opening a DB after two consecutive crash with TransactionDB, where the first crash recovers from a corrupted WAL with kPointInTimeRecovery but the second cannot. +* Fixed issue #6316 that can cause a corruption of the MANIFEST file in the middle when writing to it fails due to no disk space. +* Add DBOptions::skip_checking_sst_file_sizes_on_db_open. It disables potentially expensive checking of all sst file sizes in DB::Open(). +* BlobDB now ignores trivially moved files when updating the mapping between blob files and SSTs. This should mitigate issue #6338 where out of order flush/compaction notifications could trigger an assertion with the earlier code. +* Batched MultiGet() ignores IO errors while reading data blocks, causing it to potentially continue looking for a key and returning stale results. +* `WriteBatchWithIndex::DeleteRange` returns `Status::NotSupported`. Previously it returned success even though reads on the batch did not account for range tombstones. The corresponding language bindings now cannot be used. In C, that includes `rocksdb_writebatch_wi_delete_range`, `rocksdb_writebatch_wi_delete_range_cf`, `rocksdb_writebatch_wi_delete_rangev`, and `rocksdb_writebatch_wi_delete_rangev_cf`. In Java, that includes `WriteBatchWithIndex::deleteRange`. +* Assign new MANIFEST file number when caller tries to create a new MANIFEST by calling LogAndApply(..., new_descriptor_log=true). This bug can cause MANIFEST being overwritten during recovery if options.write_dbid_to_manifest = true and there are WAL file(s). + +### Performance Improvements +* Perfom readahead when reading from option files. Inside DB, options.log_readahead_size will be used as the readahead size. In other cases, a default 512KB is used. + +### Public API Change +* The BlobDB garbage collector now emits the statistics `BLOB_DB_GC_NUM_FILES` (number of blob files obsoleted during GC), `BLOB_DB_GC_NUM_NEW_FILES` (number of new blob files generated during GC), `BLOB_DB_GC_FAILURES` (number of failed GC passes), `BLOB_DB_GC_NUM_KEYS_RELOCATED` (number of blobs relocated during GC), and `BLOB_DB_GC_BYTES_RELOCATED` (total size of blobs relocated during GC). On the other hand, the following statistics, which are not relevant for the new GC implementation, are now deprecated: `BLOB_DB_GC_NUM_KEYS_OVERWRITTEN`, `BLOB_DB_GC_NUM_KEYS_EXPIRED`, `BLOB_DB_GC_BYTES_OVERWRITTEN`, `BLOB_DB_GC_BYTES_EXPIRED`, and `BLOB_DB_GC_MICROS`. +* Disable recycle_log_file_num when an inconsistent recovery modes are requested: kPointInTimeRecovery and kAbsoluteConsistency + +### New Features +* Added the checksum for each SST file generated by Flush or Compaction. Added sst_file_checksum_func to Options such that user can plugin their own SST file checksum function via override the FileChecksumFunc class. If user does not set the sst_file_checksum_func, SST file checksum calculation will not be enabled. The checksum information inlcuding uint32_t checksum value and a checksum function name (string). The checksum information is stored in FileMetadata in version store and also logged to MANIFEST. A new tool is added to LDB such that user can dump out a list of file checksum information from MANIFEST (stored in an unordered_map). +* `db_bench` now supports `value_size_distribution_type`, `value_size_min`, `value_size_max` options for generating random variable sized value. Added `blob_db_compression_type` option for BlobDB to enable blob compression. +* Replace RocksDB namespace "rocksdb" with flag "ROCKSDB_NAMESPACE" which if is not defined, defined as "rocksdb" in header file rocksdb_namespace.h. + +## 6.7.0 (2020-01-21) +### Public API Change +* Added a rocksdb::FileSystem class in include/rocksdb/file_system.h to encapsulate file creation/read/write operations, and an option DBOptions::file_system to allow a user to pass in an instance of rocksdb::FileSystem. If its a non-null value, this will take precendence over DBOptions::env for file operations. A new API rocksdb::FileSystem::Default() returns a platform default object. The DBOptions::env option and Env::Default() API will continue to be used for threading and other OS related functions, and where DBOptions::file_system is not specified, for file operations. For storage developers who are accustomed to rocksdb::Env, the interface in rocksdb::FileSystem is new and will probably undergo some changes as more storage systems are ported to it from rocksdb::Env. As of now, no env other than Posix has been ported to the new interface. +* A new rocksdb::NewSstFileManager() API that allows the caller to pass in separate Env and FileSystem objects. +* Changed Java API for RocksDB.keyMayExist functions to use Holder instead of StringBuilder, so that retrieved values need not decode to Strings. +* A new `OptimisticTransactionDBOptions` Option that allows users to configure occ validation policy. The default policy changes from kValidateSerial to kValidateParallel to reduce mutex contention. + +### Bug Fixes +* Fix a bug that can cause unnecessary bg thread to be scheduled(#6104). +* Fix crash caused by concurrent CF iterations and drops(#6147). +* Fix a race condition for cfd->log_number_ between manifest switch and memtable switch (PR 6249) when number of column families is greater than 1. +* Fix a bug on fractional cascading index when multiple files at the same level contain the same smallest user key, and those user keys are for merge operands. In this case, Get() the exact key may miss some merge operands. +* Delcare kHashSearch index type feature-incompatible with index_block_restart_interval larger than 1. +* Fixed an issue where the thread pools were not resized upon setting `max_background_jobs` dynamically through the `SetDBOptions` interface. +* Fix a bug that can cause write threads to hang when a slowdown/stall happens and there is a mix of writers with WriteOptions::no_slowdown set/unset. +* Fixed an issue where an incorrect "number of input records" value was used to compute the "records dropped" statistics for compactions. +* Fix a regression bug that causes segfault when hash is used, max_open_files != -1 and total order seek is used and switched back. + +### New Features +* It is now possible to enable periodic compactions for the base DB when using BlobDB. +* BlobDB now garbage collects non-TTL blobs when `enable_garbage_collection` is set to `true` in `BlobDBOptions`. Garbage collection is performed during compaction: any valid blobs located in the oldest N files (where N is the number of non-TTL blob files multiplied by the value of `BlobDBOptions::garbage_collection_cutoff`) encountered during compaction get relocated to new blob files, and old blob files are dropped once they are no longer needed. Note: we recommend enabling periodic compactions for the base DB when using this feature to deal with the case when some old blob files are kept alive by SSTs that otherwise do not get picked for compaction. +* `db_bench` now supports the `garbage_collection_cutoff` option for BlobDB. +* Introduce ReadOptions.auto_prefix_mode. When set to true, iterator will return the same result as total order seek, but may choose to use prefix seek internally based on seek key and iterator upper bound. +* MultiGet() can use IO Uring to parallelize read from the same SST file. This featuer is by default disabled. It can be enabled with environment variable ROCKSDB_USE_IO_URING. + +## 6.6.2 (2020-01-13) +### Bug Fixes +* Fixed a bug where non-L0 compaction input files were not considered to compute the `creation_time` of new compaction outputs. + +## 6.6.1 (2020-01-02) +### Bug Fixes +* Fix a bug in WriteBatchWithIndex::MultiGetFromBatchAndDB, which is called by Transaction::MultiGet, that causes due to stale pointer access when the number of keys is > 32 +* Fixed two performance issues related to memtable history trimming. First, a new SuperVersion is now created only if some memtables were actually trimmed. Second, trimming is only scheduled if there is at least one flushed memtable that is kept in memory for the purposes of transaction conflict checking. +* BlobDB no longer updates the SST to blob file mapping upon failed compactions. +* Fix a bug in which a snapshot read through an iterator could be affected by a DeleteRange after the snapshot (#6062). +* Fixed a bug where BlobDB was comparing the `ColumnFamilyHandle` pointers themselves instead of only the column family IDs when checking whether an API call uses the default column family or not. +* Delete superversions in BackgroundCallPurge. +* Fix use-after-free and double-deleting files in BackgroundCallPurge(). + +## 6.6.0 (2019-11-25) +### Bug Fixes +* Fix data corruption caused by output of intra-L0 compaction on ingested file not being placed in correct order in L0. +* Fix a data race between Version::GetColumnFamilyMetaData() and Compaction::MarkFilesBeingCompacted() for access to being_compacted (#6056). The current fix acquires the db mutex during Version::GetColumnFamilyMetaData(), which may cause regression. +* Fix a bug in DBIter that is_blob_ state isn't updated when iterating backward using seek. +* Fix a bug when format_version=3, partitioned filters, and prefix search are used in conjunction. The bug could result into Seek::(prefix) returning NotFound for an existing prefix. +* Revert the feature "Merging iterator to avoid child iterator reseek for some cases (#5286)" since it might cause strong results when reseek happens with a different iterator upper bound. +* Fix a bug causing a crash during ingest external file when background compaction cause severe error (file not found). +* Fix a bug when partitioned filters and prefix search are used in conjunction, ::SeekForPrev could return invalid for an existing prefix. ::SeekForPrev might be called by the user, or internally on ::Prev, or within ::Seek if the return value involves Delete or a Merge operand. +* Fix OnFlushCompleted fired before flush result persisted in MANIFEST when there's concurrent flush job. The bug exists since OnFlushCompleted was introduced in rocksdb 3.8. +* Fixed an sst_dump crash on some plain table SST files. +* Fixed a memory leak in some error cases of opening plain table SST files. +* Fix a bug when a crash happens while calling WriteLevel0TableForRecovery for multiple column families, leading to a column family's log number greater than the first corrutped log number when the DB is being opened in PointInTime recovery mode during next recovery attempt (#5856). + +### New Features +* Universal compaction to support options.periodic_compaction_seconds. A full compaction will be triggered if any file is over the threshold. +* `GetLiveFilesMetaData` and `GetColumnFamilyMetaData` now expose the file number of SST files as well as the oldest blob file referenced by each SST. +* A batched MultiGet API (DB::MultiGet()) that supports retrieving keys from multiple column families. +* Full and partitioned filters in the block-based table use an improved Bloom filter implementation, enabled with format_version 5 (or above) because previous releases cannot read this filter. This replacement is faster and more accurate, especially for high bits per key or millions of keys in a single (full) filter. For example, the new Bloom filter has the same false positive rate at 9.55 bits per key as the old one at 10 bits per key, and a lower false positive rate at 16 bits per key than the old one at 100 bits per key. +* Added AVX2 instructions to USE_SSE builds to accelerate the new Bloom filter and XXH3-based hash function on compatible x86_64 platforms (Haswell and later, ~2014). +* Support options.ttl or options.periodic_compaction_seconds with options.max_open_files = -1. File's oldest ancester time and file creation time will be written to manifest. If it is availalbe, this information will be used instead of creation_time and file_creation_time in table properties. +* Setting options.ttl for universal compaction now has the same meaning as setting periodic_compaction_seconds. +* SstFileMetaData also returns file creation time and oldest ancester time. +* The `sst_dump` command line tool `recompress` command now displays how many blocks were compressed and how many were not, in particular how many were not compressed because the compression ratio was not met (12.5% threshold for GoodCompressionRatio), as seen in the `number.block.not_compressed` counter stat since version 6.0.0. +* The block cache usage is now takes into account the overhead of metadata per each entry. This results into more accurate management of memory. A side-effect of this feature is that less items are fit into the block cache of the same size, which would result to higher cache miss rates. This can be remedied by increasing the block cache size or passing kDontChargeCacheMetadata to its constuctor to restore the old behavior. +* When using BlobDB, a mapping is maintained and persisted in the MANIFEST between each SST file and the oldest non-TTL blob file it references. +* `db_bench` now supports and by default issues non-TTL Puts to BlobDB. TTL Puts can be enabled by specifying a non-zero value for the `blob_db_max_ttl_range` command line parameter explicitly. +* `sst_dump` now supports printing BlobDB blob indexes in a human-readable format. This can be enabled by specifying the `decode_blob_index` flag on the command line. +* A number of new information elements are now exposed through the EventListener interface. For flushes, the file numbers of the new SST file and the oldest blob file referenced by the SST are propagated. For compactions, the level, file number, and the oldest blob file referenced are passed to the client for each compaction input and output file. + +### Public API Change +* RocksDB release 4.1 or older will not be able to open DB generated by the new release. 4.2 was released on Feb 23, 2016. +* TTL Compactions in Level compaction style now initiate successive cascading compactions on a key range so that it reaches the bottom level quickly on TTL expiry. `creation_time` table property for compaction output files is now set to the minimum of the creation times of all compaction inputs. +* With FIFO compaction style, options.periodic_compaction_seconds will have the same meaning as options.ttl. Whichever stricter will be used. With the default options.periodic_compaction_seconds value with options.ttl's default of 0, RocksDB will give a default of 30 days. +* Added an API GetCreationTimeOfOldestFile(uint64_t* creation_time) to get the file_creation_time of the oldest SST file in the DB. +* FilterPolicy now exposes additional API to make it possible to choose filter configurations based on context, such as table level and compaction style. See `LevelAndStyleCustomFilterPolicy` in db_bloom_filter_test.cc. While most existing custom implementations of FilterPolicy should continue to work as before, those wrapping the return of NewBloomFilterPolicy will require overriding new function `GetBuilderWithContext()`, because calling `GetFilterBitsBuilder()` on the FilterPolicy returned by NewBloomFilterPolicy is no longer supported. +* An unlikely usage of FilterPolicy is no longer supported. Calling GetFilterBitsBuilder() on the FilterPolicy returned by NewBloomFilterPolicy will now cause an assertion violation in debug builds, because RocksDB has internally migrated to a more elaborate interface that is expected to evolve further. Custom implementations of FilterPolicy should work as before, except those wrapping the return of NewBloomFilterPolicy, which will require a new override of a protected function in FilterPolicy. +* NewBloomFilterPolicy now takes bits_per_key as a double instead of an int. This permits finer control over the memory vs. accuracy trade-off in the new Bloom filter implementation and should not change source code compatibility. +* The option BackupableDBOptions::max_valid_backups_to_open is now only used when opening BackupEngineReadOnly. When opening a read/write BackupEngine, anything but the default value logs a warning and is treated as the default. This change ensures that backup deletion has proper accounting of shared files to ensure they are deleted when no longer referenced by a backup. +* Deprecate `snap_refresh_nanos` option. +* Added DisableManualCompaction/EnableManualCompaction to stop and resume manual compaction. +* Add TryCatchUpWithPrimary() to StackableDB in non-LITE mode. +* Add a new Env::LoadEnv() overloaded function to return a shared_ptr to Env. +* Flush sets file name to "(nil)" for OnTableFileCreationCompleted() if the flush does not produce any L0. This can happen if the file is empty thus delete by RocksDB. + +### Default Option Changes +* Changed the default value of periodic_compaction_seconds to `UINT64_MAX - 1` which allows RocksDB to auto-tune periodic compaction scheduling. When using the default value, periodic compactions are now auto-enabled if a compaction filter is used. A value of `0` will turn off the feature completely. +* Changed the default value of ttl to `UINT64_MAX - 1` which allows RocksDB to auto-tune ttl value. When using the default value, TTL will be auto-enabled to 30 days, when the feature is supported. To revert the old behavior, you can explicitly set it to 0. + +### Performance Improvements +* For 64-bit hashing, RocksDB is standardizing on a slightly modified preview version of XXH3. This function is now used for many non-persisted hashes, along with fastrange64() in place of the modulus operator, and some benchmarks show a slight improvement. +* Level iterator to invlidate the iterator more often in prefix seek and the level is filtered out by prefix bloom. + +## 6.5.2 (2019-11-15) +### Bug Fixes +* Fix a assertion failure in MultiGet() when BlockBasedTableOptions::no_block_cache is true and there is no compressed block cache +* Fix a buffer overrun problem in BlockBasedTable::MultiGet() when compression is enabled and no compressed block cache is configured. +* If a call to BackupEngine::PurgeOldBackups or BackupEngine::DeleteBackup suffered a crash, power failure, or I/O error, files could be left over from old backups that could only be purged with a call to GarbageCollect. Any call to PurgeOldBackups, DeleteBackup, or GarbageCollect should now suffice to purge such files. + +## 6.5.1 (2019-10-16) +### Bug Fixes +* Revert the feature "Merging iterator to avoid child iterator reseek for some cases (#5286)" since it might cause strange results when reseek happens with a different iterator upper bound. +* Fix a bug in BlockBasedTableIterator that might return incorrect results when reseek happens with a different iterator upper bound. +* Fix a bug when partitioned filters and prefix search are used in conjunction, ::SeekForPrev could return invalid for an existing prefix. ::SeekForPrev might be called by the user, or internally on ::Prev, or within ::Seek if the return value involves Delete or a Merge operand. + +## 6.5.0 (2019-09-13) +### Bug Fixes +* Fixed a number of data races in BlobDB. +* Fix a bug where the compaction snapshot refresh feature is not disabled as advertised when `snap_refresh_nanos` is set to 0.. +* Fix bloom filter lookups by the MultiGet batching API when BlockBasedTableOptions::whole_key_filtering is false, by checking that a key is in the perfix_extractor domain and extracting the prefix before looking up. +* Fix a bug in file ingestion caused by incorrect file number allocation when the number of column families involved in the ingestion exceeds 2. + +### New Features +* Introduced DBOptions::max_write_batch_group_size_bytes to configure maximum limit on number of bytes that are written in a single batch of WAL or memtable write. It is followed when the leader write size is larger than 1/8 of this limit. +* VerifyChecksum() by default will issue readahead. Allow ReadOptions to be passed in to those functions to override the readhead size. For checksum verifying before external SST file ingestion, a new option IngestExternalFileOptions.verify_checksums_readahead_size, is added for this readahead setting. +* When user uses options.force_consistency_check in RocksDb, instead of crashing the process, we now pass the error back to the users without killing the process. +* Add an option `memtable_insert_hint_per_batch` to WriteOptions. If it is true, each WriteBatch will maintain its own insert hints for each memtable in concurrent write. See include/rocksdb/options.h for more details. + +### Public API Change +* Added max_write_buffer_size_to_maintain option to better control memory usage of immutable memtables. +* Added a lightweight API GetCurrentWalFile() to get last live WAL filename and size. Meant to be used as a helper for backup/restore tooling in a larger ecosystem such as MySQL with a MyRocks storage engine. +* The MemTable Bloom filter, when enabled, now always uses cache locality. Options::bloom_locality now only affects the PlainTable SST format. + +### Performance Improvements +* Improve the speed of the MemTable Bloom filter, reducing the write overhead of enabling it by 1/3 to 1/2, with similar benefit to read performance. + +## 6.4.0 (2019-07-30) +### Default Option Change +* LRUCacheOptions.high_pri_pool_ratio is set to 0.5 (previously 0.0) by default, which means that by default midpoint insertion is enabled. The same change is made for the default value of high_pri_pool_ratio argument in NewLRUCache(). When block cache is not explicitly created, the small block cache created by BlockBasedTable will still has this option to be 0.0. +* Change BlockBasedTableOptions.cache_index_and_filter_blocks_with_high_priority's default value from false to true. + +### Public API Change +* Filter and compression dictionary blocks are now handled similarly to data blocks with regards to the block cache: instead of storing objects in the cache, only the blocks themselves are cached. In addition, filter and compression dictionary blocks (as well as filter partitions) no longer get evicted from the cache when a table is closed. +* Due to the above refactoring, block cache eviction statistics for filter and compression dictionary blocks are temporarily broken. We plan to reintroduce them in a later phase. +* The semantics of the per-block-type block read counts in the performance context now match those of the generic block_read_count. +* Errors related to the retrieval of the compression dictionary are now propagated to the user. +* db_bench adds a "benchmark" stats_history, which prints out the whole stats history. +* Overload GetAllKeyVersions() to support non-default column family. +* Added new APIs ExportColumnFamily() and CreateColumnFamilyWithImport() to support export and import of a Column Family. https://github.com/facebook/rocksdb/issues/3469 +* ldb sometimes uses a string-append merge operator if no merge operator is passed in. This is to allow users to print keys from a DB with a merge operator. +* Replaces old Registra with ObjectRegistry to allow user to create custom object from string, also add LoadEnv() to Env. +* Added new overload of GetApproximateSizes which gets SizeApproximationOptions object and returns a Status. The older overloads are redirecting their calls to this new method and no longer assert if the include_flags doesn't have either of INCLUDE_MEMTABLES or INCLUDE_FILES bits set. It's recommended to use the new method only, as it is more type safe and returns a meaningful status in case of errors. +* LDBCommandRunner::RunCommand() to return the status code as an integer, rather than call exit() using the code. + +### New Features +* Add argument `--secondary_path` to ldb to open the database as the secondary instance. This would keep the original DB intact. +* Compression dictionary blocks are now prefetched and pinned in the cache (based on the customer's settings) the same way as index and filter blocks. +* Added DBOptions::log_readahead_size which specifies the number of bytes to prefetch when reading the log. This is mostly useful for reading a remotely located log, as it can save the number of round-trips. If 0 (default), then the prefetching is disabled. +* Added new option in SizeApproximationOptions used with DB::GetApproximateSizes. When approximating the files total size that is used to store a keys range, allow approximation with an error margin of up to total_files_size * files_size_error_margin. This allows to take some shortcuts in files size approximation, resulting in better performance, while guaranteeing the resulting error is within a reasonable margin. +* Support loading custom objects in unit tests. In the affected unit tests, RocksDB will create custom Env objects based on environment variable TEST_ENV_URI. Users need to make sure custom object types are properly registered. For example, a static library should expose a `RegisterCustomObjects` function. By linking the unit test binary with the static library, the unit test can execute this function. + +### Performance Improvements +* Reduce iterator key comparison for upper/lower bound check. +* Improve performance of row_cache: make reads with newer snapshots than data in an SST file share the same cache key, except in some transaction cases. +* The compression dictionary is no longer copied to a new object upon retrieval. + +### Bug Fixes +* Fix ingested file and directory not being fsync. +* Return TryAgain status in place of Corruption when new tail is not visible to TransactionLogIterator. +* Fixed a regression where the fill_cache read option also affected index blocks. +* Fixed an issue where using cache_index_and_filter_blocks==false affected partitions of partitioned indexes/filters as well. + +## 6.3.2 (2019-08-15) +### Public API Change +* The semantics of the per-block-type block read counts in the performance context now match those of the generic block_read_count. + +### Bug Fixes +* Fixed a regression where the fill_cache read option also affected index blocks. +* Fixed an issue where using cache_index_and_filter_blocks==false affected partitions of partitioned indexes as well. + +## 6.3.1 (2019-07-24) +### Bug Fixes +* Fix auto rolling bug introduced in 6.3.0, which causes segfault if log file creation fails. + +## 6.3.0 (2019-06-18) +### Public API Change +* Now DB::Close() will return Aborted() error when there is unreleased snapshot. Users can retry after all snapshots are released. +* Index blocks are now handled similarly to data blocks with regards to the block cache: instead of storing objects in the cache, only the blocks themselves are cached. In addition, index blocks no longer get evicted from the cache when a table is closed, can now use the compressed block cache (if any), and can be shared among multiple table readers. +* Partitions of partitioned indexes no longer affect the read amplification statistics. +* Due to the above refactoring, block cache eviction statistics for indexes are temporarily broken. We plan to reintroduce them in a later phase. +* options.keep_log_file_num will be enforced strictly all the time. File names of all log files will be tracked, which may take significantly amount of memory if options.keep_log_file_num is large and either of options.max_log_file_size or options.log_file_time_to_roll is set. +* Add initial support for Get/Put with user timestamps. Users can specify timestamps via ReadOptions and WriteOptions when calling DB::Get and DB::Put. +* Accessing a partition of a partitioned filter or index through a pinned reference is no longer considered a cache hit. +* Add C bindings for secondary instance, i.e. DBImplSecondary. +* Rate limited deletion of WALs is only enabled if DBOptions::wal_dir is not set, or explicitly set to db_name passed to DB::Open and DBOptions::db_paths is empty, or same as db_paths[0].path + +### New Features +* Add an option `snap_refresh_nanos` (default to 0) to periodically refresh the snapshot list in compaction jobs. Assign to 0 to disable the feature. +* Add an option `unordered_write` which trades snapshot guarantees with higher write throughput. When used with WRITE_PREPARED transactions with two_write_queues=true, it offers higher throughput with however no compromise on guarantees. +* Allow DBImplSecondary to remove memtables with obsolete data after replaying MANIFEST and WAL. +* Add an option `failed_move_fall_back_to_copy` (default is true) for external SST ingestion. When `move_files` is true and hard link fails, ingestion falls back to copy if `failed_move_fall_back_to_copy` is true. Otherwise, ingestion reports an error. +* Add command `list_file_range_deletes` in ldb, which prints out tombstones in SST files. + +### Performance Improvements +* Reduce binary search when iterator reseek into the same data block. +* DBIter::Next() can skip user key checking if previous entry's seqnum is 0. +* Merging iterator to avoid child iterator reseek for some cases +* Log Writer will flush after finishing the whole record, rather than a fragment. +* Lower MultiGet batching API latency by reading data blocks from disk in parallel + +### General Improvements +* Added new status code kColumnFamilyDropped to distinguish between Column Family Dropped and DB Shutdown in progress. +* Improve ColumnFamilyOptions validation when creating a new column family. + +### Bug Fixes +* Fix a bug in WAL replay of secondary instance by skipping write batches with older sequence numbers than the current last sequence number. +* Fix flush's/compaction's merge processing logic which allowed `Put`s covered by range tombstones to reappear. Note `Put`s may exist even if the user only ever called `Merge()` due to an internal conversion during compaction to the bottommost level. +* Fix/improve memtable earliest sequence assignment and WAL replay so that WAL entries of unflushed column families will not be skipped after replaying the MANIFEST and increasing db sequence due to another flushed/compacted column family. +* Fix a bug caused by secondary not skipping the beginning of new MANIFEST. +* On DB open, delete WAL trash files left behind in wal_dir + +## 6.2.0 (2019-04-30) +### New Features +* Add an option `strict_bytes_per_sync` that causes a file-writing thread to block rather than exceed the limit on bytes pending writeback specified by `bytes_per_sync` or `wal_bytes_per_sync`. +* Improve range scan performance by avoiding per-key upper bound check in BlockBasedTableIterator. +* Introduce Periodic Compaction for Level style compaction. Files are re-compacted periodically and put in the same level. +* Block-based table index now contains exact highest key in the file, rather than an upper bound. This may improve Get() and iterator Seek() performance in some situations, especially when direct IO is enabled and block cache is disabled. A setting BlockBasedTableOptions::index_shortening is introduced to control this behavior. Set it to kShortenSeparatorsAndSuccessor to get the old behavior. +* When reading from option file/string/map, customized envs can be filled according to object registry. +* Improve range scan performance when using explicit user readahead by not creating new table readers for every iterator. +* Add index type BlockBasedTableOptions::IndexType::kBinarySearchWithFirstKey. It significantly reduces read amplification in some setups, especially for iterator seeks. It's not fully implemented yet: IO errors are not handled right. + +### Public API Change +* Change the behavior of OptimizeForPointLookup(): move away from hash-based block-based-table index, and use whole key memtable filtering. +* Change the behavior of OptimizeForSmallDb(): use a 16MB block cache, put index and filter blocks into it, and cost the memtable size to it. DBOptions.OptimizeForSmallDb() and ColumnFamilyOptions.OptimizeForSmallDb() start to take an optional cache object. +* Added BottommostLevelCompaction::kForceOptimized to avoid double compacting newly compacted files in the bottommost level compaction of manual compaction. Note this option may prohibit the manual compaction to produce a single file in the bottommost level. + +### Bug Fixes +* Adjust WriteBufferManager's dummy entry size to block cache from 1MB to 256KB. +* Fix a race condition between WritePrepared::Get and ::Put with duplicate keys. +* Fix crash when memtable prefix bloom is enabled and read/write a key out of domain of prefix extractor. +* Close a WAL file before another thread deletes it. +* Fix an assertion failure `IsFlushPending() == true` caused by one bg thread releasing the db mutex in ~ColumnFamilyData and another thread clearing `flush_requested_` flag. + +## 6.1.1 (2019-04-09) +### New Features +* When reading from option file/string/map, customized comparators and/or merge operators can be filled according to object registry. + +### Public API Change + +### Bug Fixes +* Fix a bug in 2PC where a sequence of txn prepare, memtable flush, and crash could result in losing the prepared transaction. +* Fix a bug in Encryption Env which could cause encrypted files to be read beyond file boundaries. + +## 6.1.0 (2019-03-27) +### New Features +* Introduce two more stats levels, kExceptHistogramOrTimers and kExceptTimers. +* Added a feature to perform data-block sampling for compressibility, and report stats to user. +* Add support for trace filtering. +* Add DBOptions.avoid_unnecessary_blocking_io. If true, we avoid file deletion when destroying ColumnFamilyHandle and Iterator. Instead, a job is scheduled to delete the files in background. + +### Public API Change +* Remove bundled fbson library. +* statistics.stats_level_ becomes atomic. It is preferred to use statistics.set_stats_level() and statistics.get_stats_level() to access it. +* Introduce a new IOError subcode, PathNotFound, to indicate trying to open a nonexistent file or directory for read. +* Add initial support for multiple db instances sharing the same data in single-writer, multi-reader mode. +* Removed some "using std::xxx" from public headers. + +### Bug Fixes +* Fix JEMALLOC_CXX_THROW macro missing from older Jemalloc versions, causing build failures on some platforms. +* Fix SstFileReader not able to open file ingested with write_glbal_seqno=true. + +## 6.0.0 (2019-02-19) +### New Features +* Enabled checkpoint on readonly db (DBImplReadOnly). +* Make DB ignore dropped column families while committing results of atomic flush. +* RocksDB may choose to preopen some files even if options.max_open_files != -1. This may make DB open slightly longer. +* For users of dictionary compression with ZSTD v0.7.0+, we now reuse the same digested dictionary when compressing each of an SST file's data blocks for faster compression speeds. +* For all users of dictionary compression who set `cache_index_and_filter_blocks == true`, we now store dictionary data used for decompression in the block cache for better control over memory usage. For users of ZSTD v1.1.4+ who compile with -DZSTD_STATIC_LINKING_ONLY, this includes a digested dictionary, which is used to increase decompression speed. +* Add support for block checksums verification for external SST files before ingestion. +* Introduce stats history which periodically saves Statistics snapshots and added `GetStatsHistory` API to retrieve these snapshots. +* Add a place holder in manifest which indicate a record from future that can be safely ignored. +* Add support for trace sampling. +* Enable properties block checksum verification for block-based tables. +* For all users of dictionary compression, we now generate a separate dictionary for compressing each bottom-level SST file. Previously we reused a single dictionary for a whole compaction to bottom level. The new approach achieves better compression ratios; however, it uses more memory and CPU for buffering/sampling data blocks and training dictionaries. +* Add whole key bloom filter support in memtable. +* Files written by `SstFileWriter` will now use dictionary compression if it is configured in the file writer's `CompressionOptions`. + +### Public API Change +* Disallow CompactionFilter::IgnoreSnapshots() = false, because it is not very useful and the behavior is confusing. The filter will filter everything if there is no snapshot declared by the time the compaction starts. However, users can define a snapshot after the compaction starts and before it finishes and this new snapshot won't be repeatable, because after the compaction finishes, some keys may be dropped. +* CompactionPri = kMinOverlappingRatio also uses compensated file size, which boosts file with lots of tombstones to be compacted first. +* Transaction::GetForUpdate is extended with a do_validate parameter with default value of true. If false it skips validating the snapshot before doing the read. Similarly ::Merge, ::Put, ::Delete, and ::SingleDelete are extended with assume_tracked with default value of false. If true it indicates that call is assumed to be after a ::GetForUpdate. +* `TableProperties::num_entries` and `TableProperties::num_deletions` now also account for number of range tombstones. +* Remove geodb, spatial_db, document_db, json_document, date_tiered_db, and redis_lists. +* With "ldb ----try_load_options", when wal_dir specified by the option file doesn't exist, ignore it. +* Change time resolution in FileOperationInfo. +* Deleting Blob files also go through SStFileManager. +* Remove CuckooHash memtable. +* The counter stat `number.block.not_compressed` now also counts blocks not compressed due to poor compression ratio. +* Remove ttl option from `CompactionOptionsFIFO`. The option has been deprecated and ttl in `ColumnFamilyOptions` is used instead. +* Support SST file ingestion across multiple column families via DB::IngestExternalFiles. See the function's comment about atomicity. +* Remove Lua compaction filter. + +### Bug Fixes +* Fix a deadlock caused by compaction and file ingestion waiting for each other in the event of write stalls. +* Fix a memory leak when files with range tombstones are read in mmap mode and block cache is enabled +* Fix handling of corrupt range tombstone blocks such that corruptions cannot cause deleted keys to reappear +* Lock free MultiGet +* Fix incorrect `NotFound` point lookup result when querying the endpoint of a file that has been extended by a range tombstone. +* Fix with pipelined write, write leaders's callback failure lead to the whole write group fail. + +### Change Default Options +* Change options.compaction_pri's default to kMinOverlappingRatio + +## 5.18.0 (2018-11-30) +### New Features +* Introduced `JemallocNodumpAllocator` memory allocator. When being use, block cache will be excluded from core dump. +* Introduced `PerfContextByLevel` as part of `PerfContext` which allows storing perf context at each level. Also replaced `__thread` with `thread_local` keyword for perf_context. Added per-level perf context for bloom filter and `Get` query. +* With level_compaction_dynamic_level_bytes = true, level multiplier may be adjusted automatically when Level 0 to 1 compaction is lagged behind. +* Introduced DB option `atomic_flush`. If true, RocksDB supports flushing multiple column families and atomically committing the result to MANIFEST. Useful when WAL is disabled. +* Added `num_deletions` and `num_merge_operands` members to `TableProperties`. +* Added "rocksdb.min-obsolete-sst-number-to-keep" DB property that reports the lower bound on SST file numbers that are being kept from deletion, even if the SSTs are obsolete. +* Add xxhash64 checksum support +* Introduced `MemoryAllocator`, which lets the user specify custom memory allocator for block based table. +* Improved `DeleteRange` to prevent read performance degradation. The feature is no longer marked as experimental. + +### Public API Change +* `DBOptions::use_direct_reads` now affects reads issued by `BackupEngine` on the database's SSTs. +* `NO_ITERATORS` is divided into two counters `NO_ITERATOR_CREATED` and `NO_ITERATOR_DELETE`. Both of them are only increasing now, just as other counters. + +### Bug Fixes +* Fix corner case where a write group leader blocked due to write stall blocks other writers in queue with WriteOptions::no_slowdown set. +* Fix in-memory range tombstone truncation to avoid erroneously covering newer keys at a lower level, and include range tombstones in compacted files whose largest key is the range tombstone's start key. +* Properly set the stop key for a truncated manual CompactRange +* Fix slow flush/compaction when DB contains many snapshots. The problem became noticeable to us in DBs with 100,000+ snapshots, though it will affect others at different thresholds. +* Fix the bug that WriteBatchWithIndex's SeekForPrev() doesn't see the entries with the same key. +* Fix the bug where user comparator was sometimes fed with InternalKey instead of the user key. The bug manifests when during GenerateBottommostFiles. +* Fix a bug in WritePrepared txns where if the number of old snapshots goes beyond the snapshot cache size (128 default) the rest will not be checked when evicting a commit entry from the commit cache. +* Fixed Get correctness bug in the presence of range tombstones where merge operands covered by a range tombstone always result in NotFound. +* Start populating `NO_FILE_CLOSES` ticker statistic, which was always zero previously. +* The default value of NewBloomFilterPolicy()'s argument use_block_based_builder is changed to false. Note that this new default may cause large temp memory usage when building very large SST files. + +## 5.17.0 (2018-10-05) +### Public API Change +* `OnTableFileCreated` will now be called for empty files generated during compaction. In that case, `TableFileCreationInfo::file_path` will be "(nil)" and `TableFileCreationInfo::file_size` will be zero. +* Add `FlushOptions::allow_write_stall`, which controls whether Flush calls start working immediately, even if it causes user writes to stall, or will wait until flush can be performed without causing write stall (similar to `CompactRangeOptions::allow_write_stall`). Note that the default value is false, meaning we add delay to Flush calls until stalling can be avoided when possible. This is behavior change compared to previous RocksDB versions, where Flush calls didn't check if they might cause stall or not. +* Application using PessimisticTransactionDB is expected to rollback/commit recovered transactions before starting new ones. This assumption is used to skip concurrency control during recovery. +* Expose column family id to `OnCompactionCompleted`. + +### New Features +* TransactionOptions::skip_concurrency_control allows pessimistic transactions to skip the overhead of concurrency control. Could be used for optimizing certain transactions or during recovery. + +### Bug Fixes +* Avoid creating empty SSTs and subsequently deleting them in certain cases during compaction. +* Sync CURRENT file contents during checkpoint. + +## 5.16.3 (2018-10-01) +### Bug Fixes +* Fix crash caused when `CompactFiles` run with `CompactionOptions::compression == CompressionType::kDisableCompressionOption`. Now that setting causes the compression type to be chosen according to the column family-wide compression options. + +## 5.16.2 (2018-09-21) +### Bug Fixes +* Fix bug in partition filters with format_version=4. + +## 5.16.1 (2018-09-17) +### Bug Fixes +* Remove trace_analyzer_tool from rocksdb_lib target in TARGETS file. +* Fix RocksDB Java build and tests. +* Remove sync point in Block destructor. + +## 5.16.0 (2018-08-21) +### Public API Change +* The merge operands are passed to `MergeOperator::ShouldMerge` in the reversed order relative to how they were merged (passed to FullMerge or FullMergeV2) for performance reasons +* GetAllKeyVersions() to take an extra argument of `max_num_ikeys`. +* Using ZSTD dictionary trainer (i.e., setting `CompressionOptions::zstd_max_train_bytes` to a nonzero value) now requires ZSTD version 1.1.3 or later. + +### New Features +* Changes the format of index blocks by delta encoding the index values, which are the block handles. This saves the encoding of BlockHandle::offset of the non-head index entries in each restart interval. The feature is backward compatible but not forward compatible. It is disabled by default unless format_version 4 or above is used. +* Add a new tool: trace_analyzer. Trace_analyzer analyzes the trace file generated by using trace_replay API. It can convert the binary format trace file to a human readable txt file, output the statistics of the analyzed query types such as access statistics and size statistics, combining the dumped whole key space file to analyze, support query correlation analyzing, and etc. Current supported query types are: Get, Put, Delete, SingleDelete, DeleteRange, Merge, Iterator (Seek, SeekForPrev only). +* Add hash index support to data blocks, which helps reducing the cpu utilization of point-lookup operations. This feature is backward compatible with the data block created without the hash index. It is disabled by default unless BlockBasedTableOptions::data_block_index_type is set to data_block_index_type = kDataBlockBinaryAndHash. + +### Bug Fixes +* Fix a bug in misreporting the estimated partition index size in properties block. + +## 5.15.0 (2018-07-17) +### Public API Change +* Remove managed iterator. ReadOptions.managed is not effective anymore. +* For bottommost_compression, a compatible CompressionOptions is added via `bottommost_compression_opts`. To keep backward compatible, a new boolean `enabled` is added to CompressionOptions. For compression_opts, it will be always used no matter what value of `enabled` is. For bottommost_compression_opts, it will only be used when user set `enabled=true`, otherwise, compression_opts will be used for bottommost_compression as default. +* With LRUCache, when high_pri_pool_ratio > 0, midpoint insertion strategy will be enabled to put low-pri items to the tail of low-pri list (the midpoint) when they first inserted into the cache. This is to make cache entries never get hit age out faster, improving cache efficiency when large background scan presents. +* For users of `Statistics` objects created via `CreateDBStatistics()`, the format of the string returned by its `ToString()` method has changed. +* The "rocksdb.num.entries" table property no longer counts range deletion tombstones as entries. + +### New Features +* Changes the format of index blocks by storing the key in their raw form rather than converting them to InternalKey. This saves 8 bytes per index key. The feature is backward compatible but not forward compatible. It is disabled by default unless format_version 3 or above is used. +* Avoid memcpy when reading mmap files with OpenReadOnly and max_open_files==-1. +* Support dynamically changing `ColumnFamilyOptions::ttl` via `SetOptions()`. +* Add a new table property, "rocksdb.num.range-deletions", which counts the number of range deletion tombstones in the table. +* Improve the performance of iterators doing long range scans by using readahead, when using direct IO. +* pin_top_level_index_and_filter (default true) in BlockBasedTableOptions can be used in combination with cache_index_and_filter_blocks to prefetch and pin the top-level index of partitioned index and filter blocks in cache. It has no impact when cache_index_and_filter_blocks is false. +* Write properties meta-block at the end of block-based table to save read-ahead IO. + +### Bug Fixes +* Fix deadlock with enable_pipelined_write=true and max_successive_merges > 0 +* Check conflict at output level in CompactFiles. +* Fix corruption in non-iterator reads when mmap is used for file reads +* Fix bug with prefix search in partition filters where a shared prefix would be ignored from the later partitions. The bug could report an eixstent key as missing. The bug could be triggered if prefix_extractor is set and partition filters is enabled. +* Change default value of `bytes_max_delete_chunk` to 0 in NewSstFileManager() as it doesn't work well with checkpoints. +* Fix a bug caused by not copying the block trailer with compressed SST file, direct IO, prefetcher and no compressed block cache. +* Fix write can stuck indefinitely if enable_pipelined_write=true. The issue exists since pipelined write was introduced in 5.5.0. + +## 5.14.0 (2018-05-16) +### Public API Change +* Add a BlockBasedTableOption to align uncompressed data blocks on the smaller of block size or page size boundary, to reduce flash reads by avoiding reads spanning 4K pages. +* The background thread naming convention changed (on supporting platforms) to "rocksdb:", e.g., "rocksdb:low0". +* Add a new ticker stat rocksdb.number.multiget.keys.found to count number of keys successfully read in MultiGet calls +* Touch-up to write-related counters in PerfContext. New counters added: write_scheduling_flushes_compactions_time, write_thread_wait_nanos. Counters whose behavior was fixed or modified: write_memtable_time, write_pre_and_post_process_time, write_delay_time. +* Posix Env's NewRandomRWFile() will fail if the file doesn't exist. +* Now, `DBOptions::use_direct_io_for_flush_and_compaction` only applies to background writes, and `DBOptions::use_direct_reads` applies to both user reads and background reads. This conforms with Linux's `open(2)` manpage, which advises against simultaneously reading a file in buffered and direct modes, due to possibly undefined behavior and degraded performance. +* Iterator::Valid() always returns false if !status().ok(). So, now when doing a Seek() followed by some Next()s, there's no need to check status() after every operation. +* Iterator::Seek()/SeekForPrev()/SeekToFirst()/SeekToLast() always resets status(). +* Introduced `CompressionOptions::kDefaultCompressionLevel`, which is a generic way to tell RocksDB to use the compression library's default level. It is now the default value for `CompressionOptions::level`. Previously the level defaulted to -1, which gave poor compression ratios in ZSTD. + +### New Features +* Introduce TTL for level compaction so that all files older than ttl go through the compaction process to get rid of old data. +* TransactionDBOptions::write_policy can be configured to enable WritePrepared 2PC transactions. Read more about them in the wiki. +* Add DB properties "rocksdb.block-cache-capacity", "rocksdb.block-cache-usage", "rocksdb.block-cache-pinned-usage" to show block cache usage. +* Add `Env::LowerThreadPoolCPUPriority(Priority)` method, which lowers the CPU priority of background (esp. compaction) threads to minimize interference with foreground tasks. +* Fsync parent directory after deleting a file in delete scheduler. +* In level-based compaction, if bottom-pri thread pool was setup via `Env::SetBackgroundThreads()`, compactions to the bottom level will be delegated to that thread pool. +* `prefix_extractor` has been moved from ImmutableCFOptions to MutableCFOptions, meaning it can be dynamically changed without a DB restart. + +### Bug Fixes +* Fsync after writing global seq number to the ingestion file in ExternalSstFileIngestionJob. +* Fix WAL corruption caused by race condition between user write thread and FlushWAL when two_write_queue is not set. +* Fix `BackupableDBOptions::max_valid_backups_to_open` to not delete backup files when refcount cannot be accurately determined. +* Fix memory leak when pin_l0_filter_and_index_blocks_in_cache is used with partitioned filters +* Disable rollback of merge operands in WritePrepared transactions to work around an issue in MyRocks. It can be enabled back by setting TransactionDBOptions::rollback_merge_operands to true. +* Fix wrong results by ReverseBytewiseComparator::FindShortSuccessor() + +### Java API Changes +* Add `BlockBasedTableConfig.setBlockCache` to allow sharing a block cache across DB instances. +* Added SstFileManager to the Java API to allow managing SST files across DB instances. + +## 5.13.0 (2018-03-20) +### Public API Change +* RocksDBOptionsParser::Parse()'s `ignore_unknown_options` argument will only be effective if the option file shows it is generated using a higher version of RocksDB than the current version. +* Remove CompactionEventListener. + +### New Features +* SstFileManager now can cancel compactions if they will result in max space errors. SstFileManager users can also use SetCompactionBufferSize to specify how much space must be leftover during a compaction for auxiliary file functions such as logging and flushing. +* Avoid unnecessarily flushing in `CompactRange()` when the range specified by the user does not overlap unflushed memtables. +* If `ColumnFamilyOptions::max_subcompactions` is set greater than one, we now parallelize large manual level-based compactions. +* Add "rocksdb.live-sst-files-size" DB property to return total bytes of all SST files belong to the latest LSM tree. +* NewSstFileManager to add an argument bytes_max_delete_chunk with default 64MB. With this argument, a file larger than 64MB will be ftruncated multiple times based on this size. + +### Bug Fixes +* Fix a leak in prepared_section_completed_ where the zeroed entries would not removed from the map. +* Fix WAL corruption caused by race condition between user write thread and backup/checkpoint thread. + +## 5.12.0 (2018-02-14) +### Public API Change +* Iterator::SeekForPrev is now a pure virtual method. This is to prevent user who implement the Iterator interface fail to implement SeekForPrev by mistake. +* Add `include_end` option to make the range end exclusive when `include_end == false` in `DeleteFilesInRange()`. +* Add `CompactRangeOptions::allow_write_stall`, which makes `CompactRange` start working immediately, even if it causes user writes to stall. The default value is false, meaning we add delay to `CompactRange` calls until stalling can be avoided when possible. Note this delay is not present in previous RocksDB versions. +* Creating checkpoint with empty directory now returns `Status::InvalidArgument`; previously, it returned `Status::IOError`. +* Adds a BlockBasedTableOption to turn off index block compression. +* Close() method now returns a status when closing a db. + +### New Features +* Improve the performance of iterators doing long range scans by using readahead. +* Add new function `DeleteFilesInRanges()` to delete files in multiple ranges at once for better performance. +* FreeBSD build support for RocksDB and RocksJava. +* Improved performance of long range scans with readahead. +* Updated to and now continuously tested in Visual Studio 2017. + +### Bug Fixes +* Fix `DisableFileDeletions()` followed by `GetSortedWalFiles()` to not return obsolete WAL files that `PurgeObsoleteFiles()` is going to delete. +* Fix Handle error return from WriteBuffer() during WAL file close and DB close. +* Fix advance reservation of arena block addresses. +* Fix handling of empty string as checkpoint directory. + +## 5.11.0 (2018-01-08) +### Public API Change +* Add `autoTune` and `getBytesPerSecond()` to RocksJava RateLimiter + +### New Features +* Add a new histogram stat called rocksdb.db.flush.micros for memtable flush. +* Add "--use_txn" option to use transactional API in db_stress. +* Disable onboard cache for compaction output in Windows platform. +* Improve the performance of iterators doing long range scans by using readahead. + +### Bug Fixes +* Fix a stack-use-after-scope bug in ForwardIterator. +* Fix builds on platforms including Linux, Windows, and PowerPC. +* Fix buffer overrun in backup engine for DBs with huge number of files. +* Fix a mislabel bug for bottom-pri compaction threads. +* Fix DB::Flush() keep waiting after flush finish under certain condition. + +## 5.10.0 (2017-12-11) +### Public API Change +* When running `make` with environment variable `USE_SSE` set and `PORTABLE` unset, will use all machine features available locally. Previously this combination only compiled SSE-related features. + +### New Features +* Provide lifetime hints when writing files on Linux. This reduces hardware write-amp on storage devices supporting multiple streams. +* Add a DB stat, `NUMBER_ITER_SKIP`, which returns how many internal keys were skipped during iterations (e.g., due to being tombstones or duplicate versions of a key). +* Add PerfContext counters, `key_lock_wait_count` and `key_lock_wait_time`, which measure the number of times transactions wait on key locks and total amount of time waiting. + +### Bug Fixes +* Fix IOError on WAL write doesn't propagate to write group follower +* Make iterator invalid on merge error. +* Fix performance issue in `IngestExternalFile()` affecting databases with large number of SST files. +* Fix possible corruption to LSM structure when `DeleteFilesInRange()` deletes a subset of files spanned by a `DeleteRange()` marker. + +## 5.9.0 (2017-11-01) +### Public API Change +* `BackupableDBOptions::max_valid_backups_to_open == 0` now means no backups will be opened during BackupEngine initialization. Previously this condition disabled limiting backups opened. +* `DBOptions::preserve_deletes` is a new option that allows one to specify that DB should not drop tombstones for regular deletes if they have sequence number larger than what was set by the new API call `DB::SetPreserveDeletesSequenceNumber(SequenceNumber seqnum)`. Disabled by default. +* API call `DB::SetPreserveDeletesSequenceNumber(SequenceNumber seqnum)` was added, users who wish to preserve deletes are expected to periodically call this function to advance the cutoff seqnum (all deletes made before this seqnum can be dropped by DB). It's user responsibility to figure out how to advance the seqnum in the way so the tombstones are kept for the desired period of time, yet are eventually processed in time and don't eat up too much space. +* `ReadOptions::iter_start_seqnum` was added; +if set to something > 0 user will see 2 changes in iterators behavior 1) only keys written with sequence larger than this parameter would be returned and 2) the `Slice` returned by iter->key() now points to the memory that keep User-oriented representation of the internal key, rather than user key. New struct `FullKey` was added to represent internal keys, along with a new helper function `ParseFullKey(const Slice& internal_key, FullKey* result);`. +* Deprecate trash_dir param in NewSstFileManager, right now we will rename deleted files to .trash instead of moving them to trash directory +* Allow setting a custom trash/DB size ratio limit in the SstFileManager, after which files that are to be scheduled for deletion are deleted immediately, regardless of any delete ratelimit. +* Return an error on write if write_options.sync = true and write_options.disableWAL = true to warn user of inconsistent options. Previously we will not write to WAL and not respecting the sync options in this case. + +### New Features +* CRC32C is now using the 3-way pipelined SSE algorithm `crc32c_3way` on supported platforms to improve performance. The system will choose to use this algorithm on supported platforms automatically whenever possible. If PCLMULQDQ is not supported it will fall back to the old Fast_CRC32 algorithm. +* `DBOptions::writable_file_max_buffer_size` can now be changed dynamically. +* `DBOptions::bytes_per_sync`, `DBOptions::compaction_readahead_size`, and `DBOptions::wal_bytes_per_sync` can now be changed dynamically, `DBOptions::wal_bytes_per_sync` will flush all memtables and switch to a new WAL file. +* Support dynamic adjustment of rate limit according to demand for background I/O. It can be enabled by passing `true` to the `auto_tuned` parameter in `NewGenericRateLimiter()`. The value passed as `rate_bytes_per_sec` will still be respected as an upper-bound. +* Support dynamically changing `ColumnFamilyOptions::compaction_options_fifo`. +* Introduce `EventListener::OnStallConditionsChanged()` callback. Users can implement it to be notified when user writes are stalled, stopped, or resumed. +* Add a new db property "rocksdb.estimate-oldest-key-time" to return oldest data timestamp. The property is available only for FIFO compaction with compaction_options_fifo.allow_compaction = false. +* Upon snapshot release, recompact bottommost files containing deleted/overwritten keys that previously could not be dropped due to the snapshot. This alleviates space-amp caused by long-held snapshots. +* Support lower bound on iterators specified via `ReadOptions::iterate_lower_bound`. +* Support for differential snapshots (via iterator emitting the sequence of key-values representing the difference between DB state at two different sequence numbers). Supports preserving and emitting puts and regular deletes, doesn't support SingleDeletes, MergeOperator, Blobs and Range Deletes. + +### Bug Fixes +* Fix a potential data inconsistency issue during point-in-time recovery. `DB:Open()` will abort if column family inconsistency is found during PIT recovery. +* Fix possible metadata corruption in databases using `DeleteRange()`. + +## 5.8.0 (2017-08-30) +### Public API Change +* Users of `Statistics::getHistogramString()` will see fewer histogram buckets and different bucket endpoints. +* `Slice::compare` and BytewiseComparator `Compare` no longer accept `Slice`s containing nullptr. +* `Transaction::Get` and `Transaction::GetForUpdate` variants with `PinnableSlice` added. + +### New Features +* Add Iterator::Refresh(), which allows users to update the iterator state so that they can avoid some initialization costs of recreating iterators. +* Replace dynamic_cast<> (except unit test) so people can choose to build with RTTI off. With make, release mode is by default built with -fno-rtti and debug mode is built without it. Users can override it by setting USE_RTTI=0 or 1. +* Universal compactions including the bottom level can be executed in a dedicated thread pool. This alleviates head-of-line blocking in the compaction queue, which cause write stalling, particularly in multi-instance use cases. Users can enable this feature via `Env::SetBackgroundThreads(N, Env::Priority::BOTTOM)`, where `N > 0`. +* Allow merge operator to be called even with a single merge operand during compactions, by appropriately overriding `MergeOperator::AllowSingleOperand`. +* Add `DB::VerifyChecksum()`, which verifies the checksums in all SST files in a running DB. +* Block-based table support for disabling checksums by setting `BlockBasedTableOptions::checksum = kNoChecksum`. + +### Bug Fixes +* Fix wrong latencies in `rocksdb.db.get.micros`, `rocksdb.db.write.micros`, and `rocksdb.sst.read.micros`. +* Fix incorrect dropping of deletions during intra-L0 compaction. +* Fix transient reappearance of keys covered by range deletions when memtable prefix bloom filter is enabled. +* Fix potentially wrong file smallest key when range deletions separated by snapshot are written together. + +## 5.7.0 (2017-07-13) +### Public API Change +* DB property "rocksdb.sstables" now prints keys in hex form. + +### New Features +* Measure estimated number of reads per file. The information can be accessed through DB::GetColumnFamilyMetaData or "rocksdb.sstables" DB property. +* RateLimiter support for throttling background reads, or throttling the sum of background reads and writes. This can give more predictable I/O usage when compaction reads more data than it writes, e.g., due to lots of deletions. +* [Experimental] FIFO compaction with TTL support. It can be enabled by setting CompactionOptionsFIFO.ttl > 0. +* Introduce `EventListener::OnBackgroundError()` callback. Users can implement it to be notified of errors causing the DB to enter read-only mode, and optionally override them. +* Partitioned Index/Filters exiting the experimental mode. To enable partitioned indexes set index_type to kTwoLevelIndexSearch and to further enable partitioned filters set partition_filters to true. To configure the partition size set metadata_block_size. + + +### Bug Fixes +* Fix discarding empty compaction output files when `DeleteRange()` is used together with subcompactions. + +## 5.6.0 (2017-06-06) +### Public API Change +* Scheduling flushes and compactions in the same thread pool is no longer supported by setting `max_background_flushes=0`. Instead, users can achieve this by configuring their high-pri thread pool to have zero threads. +* Replace `Options::max_background_flushes`, `Options::max_background_compactions`, and `Options::base_background_compactions` all with `Options::max_background_jobs`, which automatically decides how many threads to allocate towards flush/compaction. +* options.delayed_write_rate by default take the value of options.rate_limiter rate. +* Replace global variable `IOStatsContext iostats_context` with `IOStatsContext* get_iostats_context()`; replace global variable `PerfContext perf_context` with `PerfContext* get_perf_context()`. + +### New Features +* Change ticker/histogram statistics implementations to use core-local storage. This improves aggregation speed compared to our previous thread-local approach, particularly for applications with many threads. +* Users can pass a cache object to write buffer manager, so that they can cap memory usage for memtable and block cache using one single limit. +* Flush will be triggered when 7/8 of the limit introduced by write_buffer_manager or db_write_buffer_size is triggered, so that the hard threshold is hard to hit. +* Introduce WriteOptions.low_pri. If it is true, low priority writes will be throttled if the compaction is behind. +* `DB::IngestExternalFile()` now supports ingesting files into a database containing range deletions. + +### Bug Fixes +* Shouldn't ignore return value of fsync() in flush. + +## 5.5.0 (2017-05-17) +### New Features +* FIFO compaction to support Intra L0 compaction too with CompactionOptionsFIFO.allow_compaction=true. +* DB::ResetStats() to reset internal stats. +* Statistics::Reset() to reset user stats. +* ldb add option --try_load_options, which will open DB with its own option file. +* Introduce WriteBatch::PopSavePoint to pop the most recent save point explicitly. +* Support dynamically change `max_open_files` option via SetDBOptions() +* Added DB::CreateColumnFamilie() and DB::DropColumnFamilies() to bulk create/drop column families. +* Add debugging function `GetAllKeyVersions` to see internal versions of a range of keys. +* Support file ingestion with universal compaction style +* Support file ingestion behind with option `allow_ingest_behind` +* New option enable_pipelined_write which may improve write throughput in case writing from multiple threads and WAL enabled. + +### Bug Fixes +* Fix the bug that Direct I/O uses direct reads for non-SST file + +## 5.4.0 (2017-04-11) +### Public API Change +* random_access_max_buffer_size no longer has any effect +* Removed Env::EnableReadAhead(), Env::ShouldForwardRawRequest() +* Support dynamically change `stats_dump_period_sec` option via SetDBOptions(). +* Added ReadOptions::max_skippable_internal_keys to set a threshold to fail a request as incomplete when too many keys are being skipped when using iterators. +* DB::Get in place of std::string accepts PinnableSlice, which avoids the extra memcpy of value to std::string in most of cases. + * PinnableSlice releases the pinned resources that contain the value when it is destructed or when ::Reset() is called on it. + * The old API that accepts std::string, although discouraged, is still supported. +* Replace Options::use_direct_writes with Options::use_direct_io_for_flush_and_compaction. Read Direct IO wiki for details. +* Added CompactionEventListener and EventListener::OnFlushBegin interfaces. + +### New Features +* Memtable flush can be avoided during checkpoint creation if total log file size is smaller than a threshold specified by the user. +* Introduce level-based L0->L0 compactions to reduce file count, so write delays are incurred less often. +* (Experimental) Partitioning filters which creates an index on the partitions. The feature can be enabled by setting partition_filters when using kFullFilter. Currently the feature also requires two-level indexing to be enabled. Number of partitions is the same as the number of partitions for indexes, which is controlled by metadata_block_size. + +## 5.3.0 (2017-03-08) +### Public API Change +* Remove disableDataSync option. +* Remove timeout_hint_us option from WriteOptions. The option has been deprecated and has no effect since 3.13.0. +* Remove option min_partial_merge_operands. Partial merge operands will always be merged in flush or compaction if there are more than one. +* Remove option verify_checksums_in_compaction. Compaction will always verify checksum. + +### Bug Fixes +* Fix the bug that iterator may skip keys + +## 5.2.0 (2017-02-08) +### Public API Change +* NewLRUCache() will determine number of shard bits automatically based on capacity, if the user doesn't pass one. This also impacts the default block cache when the user doesn't explicit provide one. +* Change the default of delayed slowdown value to 16MB/s and further increase the L0 stop condition to 36 files. +* Options::use_direct_writes and Options::use_direct_reads are now ready to use. +* (Experimental) Two-level indexing that partition the index and creates a 2nd level index on the partitions. The feature can be enabled by setting kTwoLevelIndexSearch as IndexType and configuring index_per_partition. + +### New Features +* Added new overloaded function GetApproximateSizes that allows to specify if memtable stats should be computed only without computing SST files' stats approximations. +* Added new function GetApproximateMemTableStats that approximates both number of records and size of memtables. +* Add Direct I/O mode for SST file I/O + +### Bug Fixes +* RangeSync() should work if ROCKSDB_FALLOCATE_PRESENT is not set +* Fix wrong results in a data race case in Get() +* Some fixes related to 2PC. +* Fix bugs of data corruption in direct I/O + +## 5.1.0 (2017-01-13) +* Support dynamically change `delete_obsolete_files_period_micros` option via SetDBOptions(). +* Added EventListener::OnExternalFileIngested which will be called when IngestExternalFile() add a file successfully. +* BackupEngine::Open and BackupEngineReadOnly::Open now always return error statuses matching those of the backup Env. + +### Bug Fixes +* Fix the bug that if 2PC is enabled, checkpoints may loss some recent transactions. +* When file copying is needed when creating checkpoints or bulk loading files, fsync the file after the file copying. + +## 5.0.0 (2016-11-17) +### Public API Change +* Options::max_bytes_for_level_multiplier is now a double along with all getters and setters. +* Support dynamically change `delayed_write_rate` and `max_total_wal_size` options via SetDBOptions(). +* Introduce DB::DeleteRange for optimized deletion of large ranges of contiguous keys. +* Support dynamically change `delayed_write_rate` option via SetDBOptions(). +* Options::allow_concurrent_memtable_write and Options::enable_write_thread_adaptive_yield are now true by default. +* Remove Tickers::SEQUENCE_NUMBER to avoid confusion if statistics object is shared among RocksDB instance. Alternatively DB::GetLatestSequenceNumber() can be used to get the same value. +* Options.level0_stop_writes_trigger default value changes from 24 to 32. +* New compaction filter API: CompactionFilter::FilterV2(). Allows to drop ranges of keys. +* Removed flashcache support. +* DB::AddFile() is deprecated and is replaced with DB::IngestExternalFile(). DB::IngestExternalFile() remove all the restrictions that existed for DB::AddFile. + +### New Features +* Add avoid_flush_during_shutdown option, which speeds up DB shutdown by not flushing unpersisted data (i.e. with disableWAL = true). Unpersisted data will be lost. The options is dynamically changeable via SetDBOptions(). +* Add memtable_insert_with_hint_prefix_extractor option. The option is mean to reduce CPU usage for inserting keys into memtable, if keys can be group by prefix and insert for each prefix are sequential or almost sequential. See include/rocksdb/options.h for more details. +* Add LuaCompactionFilter in utilities. This allows developers to write compaction filters in Lua. To use this feature, LUA_PATH needs to be set to the root directory of Lua. +* No longer populate "LATEST_BACKUP" file in backup directory, which formerly contained the number of the latest backup. The latest backup can be determined by finding the highest numbered file in the "meta/" subdirectory. + +## 4.13.0 (2016-10-18) +### Public API Change +* DB::GetOptions() reflect dynamic changed options (i.e. through DB::SetOptions()) and return copy of options instead of reference. +* Added Statistics::getAndResetTickerCount(). + +### New Features +* Add DB::SetDBOptions() to dynamic change base_background_compactions and max_background_compactions. +* Added Iterator::SeekForPrev(). This new API will seek to the last key that less than or equal to the target key. + +## 4.12.0 (2016-09-12) +### Public API Change +* CancelAllBackgroundWork() flushes all memtables for databases containing writes that have bypassed the WAL (writes issued with WriteOptions::disableWAL=true) before shutting down background threads. +* Merge options source_compaction_factor, max_grandparent_overlap_bytes and expanded_compaction_factor into max_compaction_bytes. +* Remove ImmutableCFOptions. +* Add a compression type ZSTD, which can work with ZSTD 0.8.0 or up. Still keep ZSTDNotFinal for compatibility reasons. + +### New Features +* Introduce NewClockCache, which is based on CLOCK algorithm with better concurrent performance in some cases. It can be used to replace the default LRU-based block cache and table cache. To use it, RocksDB need to be linked with TBB lib. +* Change ticker/histogram statistics implementations to accumulate data in thread-local storage, which improves CPU performance by reducing cache coherency costs. Callers of CreateDBStatistics do not need to change anything to use this feature. +* Block cache mid-point insertion, where index and filter block are inserted into LRU block cache with higher priority. The feature can be enabled by setting BlockBasedTableOptions::cache_index_and_filter_blocks_with_high_priority to true and high_pri_pool_ratio > 0 when creating NewLRUCache. + +## 4.11.0 (2016-08-01) +### Public API Change +* options.memtable_prefix_bloom_huge_page_tlb_size => memtable_huge_page_size. When it is set, RocksDB will try to allocate memory from huge page for memtable too, rather than just memtable bloom filter. + +### New Features +* A tool to migrate DB after options change. See include/rocksdb/utilities/option_change_migration.h. +* Add ReadOptions.background_purge_on_iterator_cleanup. If true, we avoid file deletion when destroying iterators. + +## 4.10.0 (2016-07-05) +### Public API Change +* options.memtable_prefix_bloom_bits changes to options.memtable_prefix_bloom_bits_ratio and deprecate options.memtable_prefix_bloom_probes +* enum type CompressionType and PerfLevel changes from char to unsigned char. Value of all PerfLevel shift by one. +* Deprecate options.filter_deletes. + +### New Features +* Add avoid_flush_during_recovery option. +* Add a read option background_purge_on_iterator_cleanup to avoid deleting files in foreground when destroying iterators. Instead, a job is scheduled in high priority queue and would be executed in a separate background thread. +* RepairDB support for column families. RepairDB now associates data with non-default column families using information embedded in the SST/WAL files (4.7 or later). For data written by 4.6 or earlier, RepairDB associates it with the default column family. +* Add options.write_buffer_manager which allows users to control total memtable sizes across multiple DB instances. + +## 4.9.0 (2016-06-09) +### Public API changes +* Add bottommost_compression option, This option can be used to set a specific compression algorithm for the bottommost level (Last level containing files in the DB). +* Introduce CompactionJobInfo::compression, This field state the compression algorithm used to generate the output files of the compaction. +* Deprecate BlockBaseTableOptions.hash_index_allow_collision=false +* Deprecate options builder (GetOptions()). + +### New Features +* Introduce NewSimCache() in rocksdb/utilities/sim_cache.h. This function creates a block cache that is able to give simulation results (mainly hit rate) of simulating block behavior with a configurable cache size. + +## 4.8.0 (2016-05-02) +### Public API Change +* Allow preset compression dictionary for improved compression of block-based tables. This is supported for zlib, zstd, and lz4. The compression dictionary's size is configurable via CompressionOptions::max_dict_bytes. +* Delete deprecated classes for creating backups (BackupableDB) and restoring from backups (RestoreBackupableDB). Now, BackupEngine should be used for creating backups, and BackupEngineReadOnly should be used for restorations. For more details, see https://github.com/facebook/rocksdb/wiki/How-to-backup-RocksDB%3F +* Expose estimate of per-level compression ratio via DB property: "rocksdb.compression-ratio-at-levelN". +* Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status. + +### New Features +* Add ReadOptions::readahead_size. If non-zero, NewIterator will create a new table reader which performs reads of the given size. + +## 4.7.0 (2016-04-08) +### Public API Change +* rename options compaction_measure_io_stats to report_bg_io_stats and include flush too. +* Change some default options. Now default options will optimize for server-workloads. Also enable slowdown and full stop triggers for pending compaction bytes. These changes may cause sub-optimal performance or significant increase of resource usage. To avoid these risks, users can open existing RocksDB with options extracted from RocksDB option files. See https://github.com/facebook/rocksdb/wiki/RocksDB-Options-File for how to use RocksDB option files. Or you can call Options.OldDefaults() to recover old defaults. DEFAULT_OPTIONS_HISTORY.md will track change history of default options. + +## 4.6.0 (2016-03-10) +### Public API Changes +* Change default of BlockBasedTableOptions.format_version to 2. It means default DB created by 4.6 or up cannot be opened by RocksDB version 3.9 or earlier. +* Added strict_capacity_limit option to NewLRUCache. If the flag is set to true, insert to cache will fail if no enough capacity can be free. Signature of Cache::Insert() is updated accordingly. +* Tickers [NUMBER_DB_NEXT, NUMBER_DB_PREV, NUMBER_DB_NEXT_FOUND, NUMBER_DB_PREV_FOUND, ITER_BYTES_READ] are not updated immediately. The are updated when the Iterator is deleted. +* Add monotonically increasing counter (DB property "rocksdb.current-super-version-number") that increments upon any change to the LSM tree. + +### New Features +* Add CompactionPri::kMinOverlappingRatio, a compaction picking mode friendly to write amplification. +* Deprecate Iterator::IsKeyPinned() and replace it with Iterator::GetProperty() with prop_name="rocksdb.iterator.is.key.pinned" + +## 4.5.0 (2016-02-05) +### Public API Changes +* Add a new perf context level between kEnableCount and kEnableTime. Level 2 now does not include timers for mutexes. +* Statistics of mutex operation durations will not be measured by default. If you want to have them enabled, you need to set Statistics::stats_level_ to kAll. +* DBOptions::delete_scheduler and NewDeleteScheduler() are removed, please use DBOptions::sst_file_manager and NewSstFileManager() instead + +### New Features +* ldb tool now supports operations to non-default column families. +* Add kPersistedTier to ReadTier. This option allows Get and MultiGet to read only the persited data and skip mem-tables if writes were done with disableWAL = true. +* Add DBOptions::sst_file_manager. Use NewSstFileManager() in include/rocksdb/sst_file_manager.h to create a SstFileManager that can be used to track the total size of SST files and control the SST files deletion rate. + +## 4.4.0 (2016-01-14) +### Public API Changes +* Change names in CompactionPri and add a new one. +* Deprecate options.soft_rate_limit and add options.soft_pending_compaction_bytes_limit. +* If options.max_write_buffer_number > 3, writes will be slowed down when writing to the last write buffer to delay a full stop. +* Introduce CompactionJobInfo::compaction_reason, this field include the reason to trigger the compaction. +* After slow down is triggered, if estimated pending compaction bytes keep increasing, slowdown more. +* Increase default options.delayed_write_rate to 2MB/s. +* Added a new parameter --path to ldb tool. --path accepts the name of either MANIFEST, SST or a WAL file. Either --db or --path can be used when calling ldb. + +## 4.3.0 (2015-12-08) +### New Features +* CompactionFilter has new member function called IgnoreSnapshots which allows CompactionFilter to be called even if there are snapshots later than the key. +* RocksDB will now persist options under the same directory as the RocksDB database on successful DB::Open, CreateColumnFamily, DropColumnFamily, and SetOptions. +* Introduce LoadLatestOptions() in rocksdb/utilities/options_util.h. This function can construct the latest DBOptions / ColumnFamilyOptions used by the specified RocksDB intance. +* Introduce CheckOptionsCompatibility() in rocksdb/utilities/options_util.h. This function checks whether the input set of options is able to open the specified DB successfully. + +### Public API Changes +* When options.db_write_buffer_size triggers, only the column family with the largest column family size will be flushed, not all the column families. + +## 4.2.0 (2015-11-09) +### New Features +* Introduce CreateLoggerFromOptions(), this function create a Logger for provided DBOptions. +* Add GetAggregatedIntProperty(), which returns the sum of the GetIntProperty of all the column families. +* Add MemoryUtil in rocksdb/utilities/memory.h. It currently offers a way to get the memory usage by type from a list rocksdb instances. + +### Public API Changes +* CompactionFilter::Context includes information of Column Family ID +* The need-compaction hint given by TablePropertiesCollector::NeedCompact() will be persistent and recoverable after DB recovery. This introduces a breaking format change. If you use this experimental feature, including NewCompactOnDeletionCollectorFactory() in the new version, you may not be able to directly downgrade the DB back to version 4.0 or lower. +* TablePropertiesCollectorFactory::CreateTablePropertiesCollector() now takes an option Context, containing the information of column family ID for the file being written. +* Remove DefaultCompactionFilterFactory. + + +## 4.1.0 (2015-10-08) +### New Features +* Added single delete operation as a more efficient way to delete keys that have not been overwritten. +* Added experimental AddFile() to DB interface that allow users to add files created by SstFileWriter into an empty Database, see include/rocksdb/sst_file_writer.h and DB::AddFile() for more info. +* Added support for opening SST files with .ldb suffix which enables opening LevelDB databases. +* CompactionFilter now supports filtering of merge operands and merge results. + +### Public API Changes +* Added SingleDelete() to the DB interface. +* Added AddFile() to DB interface. +* Added SstFileWriter class. +* CompactionFilter has a new method FilterMergeOperand() that RocksDB applies to every merge operand during compaction to decide whether to filter the operand. +* We removed CompactionFilterV2 interfaces from include/rocksdb/compaction_filter.h. The functionality was deprecated already in version 3.13. + +## 4.0.0 (2015-09-09) +### New Features +* Added support for transactions. See include/rocksdb/utilities/transaction.h for more info. +* DB::GetProperty() now accepts "rocksdb.aggregated-table-properties" and "rocksdb.aggregated-table-properties-at-levelN", in which case it returns aggregated table properties of the target column family, or the aggregated table properties of the specified level N if the "at-level" version is used. +* Add compression option kZSTDNotFinalCompression for people to experiment ZSTD although its format is not finalized. +* We removed the need for LATEST_BACKUP file in BackupEngine. We still keep writing it when we create new backups (because of backward compatibility), but we don't read it anymore. + +### Public API Changes +* Removed class Env::RandomRWFile and Env::NewRandomRWFile(). +* Renamed DBOptions.num_subcompactions to DBOptions.max_subcompactions to make the name better match the actual functionality of the option. +* Added Equal() method to the Comparator interface that can optionally be overwritten in cases where equality comparisons can be done more efficiently than three-way comparisons. +* Previous 'experimental' OptimisticTransaction class has been replaced by Transaction class. + +## 3.13.0 (2015-08-06) +### New Features +* RollbackToSavePoint() in WriteBatch/WriteBatchWithIndex +* Add NewCompactOnDeletionCollectorFactory() in utilities/table_properties_collectors, which allows rocksdb to mark a SST file as need-compaction when it observes at least D deletion entries in any N consecutive entries in that SST file. Note that this feature depends on an experimental NeedCompact() API --- the result of this API will not persist after DB restart. +* Add DBOptions::delete_scheduler. Use NewDeleteScheduler() in include/rocksdb/delete_scheduler.h to create a DeleteScheduler that can be shared among multiple RocksDB instances to control the file deletion rate of SST files that exist in the first db_path. + +### Public API Changes +* Deprecated WriteOptions::timeout_hint_us. We no longer support write timeout. If you really need this option, talk to us and we might consider returning it. +* Deprecated purge_redundant_kvs_while_flush option. +* Removed BackupEngine::NewBackupEngine() and NewReadOnlyBackupEngine() that were deprecated in RocksDB 3.8. Please use BackupEngine::Open() instead. +* Deprecated Compaction Filter V2. We are not aware of any existing use-cases. If you use this filter, your compile will break with RocksDB 3.13. Please let us know if you use it and we'll put it back in RocksDB 3.14. +* Env::FileExists now returns a Status instead of a boolean +* Add statistics::getHistogramString() to print detailed distribution of a histogram metric. +* Add DBOptions::skip_stats_update_on_db_open. When it is on, DB::Open() will run faster as it skips the random reads required for loading necessary stats from SST files to optimize compaction. + +## 3.12.0 (2015-07-02) +### New Features +* Added experimental support for optimistic transactions. See include/rocksdb/utilities/optimistic_transaction.h for more info. +* Added a new way to report QPS from db_bench (check out --report_file and --report_interval_seconds) +* Added a cache for individual rows. See DBOptions::row_cache for more info. +* Several new features on EventListener (see include/rocksdb/listener.h): + - OnCompactionCompleted() now returns per-compaction job statistics, defined in include/rocksdb/compaction_job_stats.h. + - Added OnTableFileCreated() and OnTableFileDeleted(). +* Add compaction_options_universal.enable_trivial_move to true, to allow trivial move while performing universal compaction. Trivial move will happen only when all the input files are non overlapping. + +### Public API changes +* EventListener::OnFlushCompleted() now passes FlushJobInfo instead of a list of parameters. +* DB::GetDbIdentity() is now a const function. If this function is overridden in your application, be sure to also make GetDbIdentity() const to avoid compile error. +* Move listeners from ColumnFamilyOptions to DBOptions. +* Add max_write_buffer_number_to_maintain option +* DB::CompactRange()'s parameter reduce_level is changed to change_level, to allow users to move levels to lower levels if allowed. It can be used to migrate a DB from options.level_compaction_dynamic_level_bytes=false to options.level_compaction_dynamic_level_bytes.true. +* Change default value for options.compaction_filter_factory and options.compaction_filter_factory_v2 to nullptr instead of DefaultCompactionFilterFactory and DefaultCompactionFilterFactoryV2. +* If CancelAllBackgroundWork is called without doing a flush after doing loads with WAL disabled, the changes which haven't been flushed before the call to CancelAllBackgroundWork will be lost. +* WBWIIterator::Entry() now returns WriteEntry instead of `const WriteEntry&` +* options.hard_rate_limit is deprecated. +* When options.soft_rate_limit or options.level0_slowdown_writes_trigger is triggered, the way to slow down writes is changed to: write rate to DB is limited to to options.delayed_write_rate. +* DB::GetApproximateSizes() adds a parameter to allow the estimation to include data in mem table, with default to be not to include. It is now only supported in skip list mem table. +* DB::CompactRange() now accept CompactRangeOptions instead of multiple parameters. CompactRangeOptions is defined in include/rocksdb/options.h. +* CompactRange() will now skip bottommost level compaction for level based compaction if there is no compaction filter, bottommost_level_compaction is introduced in CompactRangeOptions to control when it's possible to skip bottommost level compaction. This mean that if you want the compaction to produce a single file you need to set bottommost_level_compaction to BottommostLevelCompaction::kForce. +* Add Cache.GetPinnedUsage() to get the size of memory occupied by entries that are in use by the system. +* DB:Open() will fail if the compression specified in Options is not linked with the binary. If you see this failure, recompile RocksDB with compression libraries present on your system. Also, previously our default compression was snappy. This behavior is now changed. Now, the default compression is snappy only if it's available on the system. If it isn't we change the default to kNoCompression. +* We changed how we account for memory used in block cache. Previously, we only counted the sum of block sizes currently present in block cache. Now, we count the actual memory usage of the blocks. For example, a block of size 4.5KB will use 8KB memory with jemalloc. This might decrease your memory usage and possibly decrease performance. Increase block cache size if you see this happening after an upgrade. +* Add BackupEngineImpl.options_.max_background_operations to specify the maximum number of operations that may be performed in parallel. Add support for parallelized backup and restore. +* Add DB::SyncWAL() that does a WAL sync without blocking writers. + +## 3.11.0 (2015-05-19) +### New Features +* Added a new API Cache::SetCapacity(size_t capacity) to dynamically change the maximum configured capacity of the cache. If the new capacity is less than the existing cache usage, the implementation will try to lower the usage by evicting the necessary number of elements following a strict LRU policy. +* Added an experimental API for handling flashcache devices (blacklists background threads from caching their reads) -- NewFlashcacheAwareEnv +* If universal compaction is used and options.num_levels > 1, compact files are tried to be stored in none-L0 with smaller files based on options.target_file_size_base. The limitation of DB size when using universal compaction is greatly mitigated by using more levels. You can set num_levels = 1 to make universal compaction behave as before. If you set num_levels > 1 and want to roll back to a previous version, you need to compact all files to a big file in level 0 (by setting target_file_size_base to be large and CompactRange(, nullptr, nullptr, true, 0) and reopen the DB with the same version to rewrite the manifest, and then you can open it using previous releases. +* More information about rocksdb background threads are available in Env::GetThreadList(), including the number of bytes read / written by a compaction job, mem-table size and current number of bytes written by a flush job and many more. Check include/rocksdb/thread_status.h for more detail. + +### Public API changes +* TablePropertiesCollector::AddUserKey() is added to replace TablePropertiesCollector::Add(). AddUserKey() exposes key type, sequence number and file size up to now to users. +* DBOptions::bytes_per_sync used to apply to both WAL and table files. As of 3.11 it applies only to table files. If you want to use this option to sync WAL in the background, please use wal_bytes_per_sync + +## 3.10.0 (2015-03-24) +### New Features +* GetThreadStatus() is now able to report detailed thread status, including: + - Thread Operation including flush and compaction. + - The stage of the current thread operation. + - The elapsed time in micros since the current thread operation started. + More information can be found in include/rocksdb/thread_status.h. In addition, when running db_bench with --thread_status_per_interval, db_bench will also report thread status periodically. +* Changed the LRU caching algorithm so that referenced blocks (by iterators) are never evicted. This change made parameter removeScanCountLimit obsolete. Because of that NewLRUCache doesn't take three arguments anymore. table_cache_remove_scan_limit option is also removed +* By default we now optimize the compilation for the compilation platform (using -march=native). If you want to build portable binary, use 'PORTABLE=1' before the make command. +* We now allow level-compaction to place files in different paths by + specifying them in db_paths along with the target_size. + Lower numbered levels will be placed earlier in the db_paths and higher + numbered levels will be placed later in the db_paths vector. +* Potentially big performance improvements if you're using RocksDB with lots of column families (100-1000) +* Added BlockBasedTableOptions.format_version option, which allows user to specify which version of block based table he wants. As a general guideline, newer versions have more features, but might not be readable by older versions of RocksDB. +* Added new block based table format (version 2), which you can enable by setting BlockBasedTableOptions.format_version = 2. This format changes how we encode size information in compressed blocks and should help with memory allocations if you're using Zlib or BZip2 compressions. +* MemEnv (env that stores data in memory) is now available in default library build. You can create it by calling NewMemEnv(). +* Add SliceTransform.SameResultWhenAppended() to help users determine it is safe to apply prefix bloom/hash. +* Block based table now makes use of prefix bloom filter if it is a full fulter. +* Block based table remembers whether a whole key or prefix based bloom filter is supported in SST files. Do a sanity check when reading the file with users' configuration. +* Fixed a bug in ReadOnlyBackupEngine that deleted corrupted backups in some cases, even though the engine was ReadOnly +* options.level_compaction_dynamic_level_bytes, a feature to allow RocksDB to pick dynamic base of bytes for levels. With this feature turned on, we will automatically adjust max bytes for each level. The goal of this feature is to have lower bound on size amplification. For more details, see comments in options.h. +* Added an abstract base class WriteBatchBase for write batches +* Fixed a bug where we start deleting files of a dropped column families even if there are still live references to it + +### Public API changes +* Deprecated skip_log_error_on_recovery and table_cache_remove_scan_count_limit options. +* Logger method logv with log level parameter is now virtual + +### RocksJava +* Added compression per level API. +* MemEnv is now available in RocksJava via RocksMemEnv class. +* lz4 compression is now included in rocksjava static library when running `make rocksdbjavastatic`. +* Overflowing a size_t when setting rocksdb options now throws an IllegalArgumentException, which removes the necessity for a developer to catch these Exceptions explicitly. + +## 3.9.0 (2014-12-08) + +### New Features +* Add rocksdb::GetThreadList(), which in the future will return the current status of all + rocksdb-related threads. We will have more code instruments in the following RocksDB + releases. +* Change convert function in rocksdb/utilities/convenience.h to return Status instead of boolean. + Also add support for nested options in convert function + +### Public API changes +* New API to create a checkpoint added. Given a directory name, creates a new + database which is an image of the existing database. +* New API LinkFile added to Env. If you implement your own Env class, an + implementation of the API LinkFile will have to be provided. +* MemTableRep takes MemTableAllocator instead of Arena + +### Improvements +* RocksDBLite library now becomes smaller and will be compiled with -fno-exceptions flag. + +## 3.8.0 (2014-11-14) + +### Public API changes +* BackupEngine::NewBackupEngine() was deprecated; please use BackupEngine::Open() from now on. +* BackupableDB/RestoreBackupableDB have new GarbageCollect() methods, which will clean up files from corrupt and obsolete backups. +* BackupableDB/RestoreBackupableDB have new GetCorruptedBackups() methods which list corrupt backups. + +### Cleanup +* Bunch of code cleanup, some extra warnings turned on (-Wshadow, -Wshorten-64-to-32, -Wnon-virtual-dtor) + +### New features +* CompactFiles and EventListener, although they are still in experimental state +* Full ColumnFamily support in RocksJava. + +## 3.7.0 (2014-11-06) +### Public API changes +* Introduce SetOptions() API to allow adjusting a subset of options dynamically online +* Introduce 4 new convenient functions for converting Options from string: GetColumnFamilyOptionsFromMap(), GetColumnFamilyOptionsFromString(), GetDBOptionsFromMap(), GetDBOptionsFromString() +* Remove WriteBatchWithIndex.Delete() overloads using SliceParts +* When opening a DB, if options.max_background_compactions is larger than the existing low pri pool of options.env, it will enlarge it. Similarly, options.max_background_flushes is larger than the existing high pri pool of options.env, it will enlarge it. + +## 3.6.0 (2014-10-07) +### Disk format changes +* If you're using RocksDB on ARM platforms and you're using default bloom filter, there is a disk format change you need to be aware of. There are three steps you need to do when you convert to new release: 1. turn off filter policy, 2. compact the whole database, 3. turn on filter policy + +### Behavior changes +* We have refactored our system of stalling writes. Any stall-related statistics' meanings are changed. Instead of per-write stall counts, we now count stalls per-epoch, where epochs are periods between flushes and compactions. You'll find more information in our Tuning Perf Guide once we release RocksDB 3.6. +* When disableDataSync=true, we no longer sync the MANIFEST file. +* Add identity_as_first_hash property to CuckooTable. SST file needs to be rebuilt to be opened by reader properly. + +### Public API changes +* Change target_file_size_base type to uint64_t from int. +* Remove allow_thread_local. This feature was proved to be stable, so we are turning it always-on. + +## 3.5.0 (2014-09-03) +### New Features +* Add include/utilities/write_batch_with_index.h, providing a utility class to query data out of WriteBatch when building it. +* Move BlockBasedTable related options to BlockBasedTableOptions from Options. Change corresponding JNI interface. Options affected include: + no_block_cache, block_cache, block_cache_compressed, block_size, block_size_deviation, block_restart_interval, filter_policy, whole_key_filtering. filter_policy is changed to shared_ptr from a raw pointer. +* Remove deprecated options: disable_seek_compaction and db_stats_log_interval +* OptimizeForPointLookup() takes one parameter for block cache size. It now builds hash index, bloom filter, and block cache. + +### Public API changes +* The Prefix Extractor used with V2 compaction filters is now passed user key to SliceTransform::Transform instead of unparsed RocksDB key. + +## 3.4.0 (2014-08-18) +### New Features +* Support Multiple DB paths in universal style compactions +* Add feature of storing plain table index and bloom filter in SST file. +* CompactRange() will never output compacted files to level 0. This used to be the case when all the compaction input files were at level 0. +* Added iterate_upper_bound to define the extent upto which the forward iterator will return entries. This will prevent iterating over delete markers and overwritten entries for edge cases where you want to break out the iterator anyways. This may improve performance in case there are a large number of delete markers or overwritten entries. + +### Public API changes +* DBOptions.db_paths now is a vector of a DBPath structure which indicates both of path and target size +* NewPlainTableFactory instead of bunch of parameters now accepts PlainTableOptions, which is defined in include/rocksdb/table.h +* Moved include/utilities/*.h to include/rocksdb/utilities/*.h +* Statistics APIs now take uint32_t as type instead of Tickers. Also make two access functions getTickerCount and histogramData const +* Add DB property rocksdb.estimate-num-keys, estimated number of live keys in DB. +* Add DB::GetIntProperty(), which returns DB properties that are integer as uint64_t. +* The Prefix Extractor used with V2 compaction filters is now passed user key to SliceTransform::Transform instead of unparsed RocksDB key. + +## 3.3.0 (2014-07-10) +### New Features +* Added JSON API prototype. +* HashLinklist reduces performance outlier caused by skewed bucket by switching data in the bucket from linked list to skip list. Add parameter threshold_use_skiplist in NewHashLinkListRepFactory(). +* RocksDB is now able to reclaim storage space more effectively during the compaction process. This is done by compensating the size of each deletion entry by the 2X average value size, which makes compaction to be triggered by deletion entries more easily. +* Add TimeOut API to write. Now WriteOptions have a variable called timeout_hint_us. With timeout_hint_us set to non-zero, any write associated with this timeout_hint_us may be aborted when it runs longer than the specified timeout_hint_us, and it is guaranteed that any write completes earlier than the specified time-out will not be aborted due to the time-out condition. +* Add a rate_limiter option, which controls total throughput of flush and compaction. The throughput is specified in bytes/sec. Flush always has precedence over compaction when available bandwidth is constrained. + +### Public API changes +* Removed NewTotalOrderPlainTableFactory because it is not used and implemented semantically incorrect. + +## 3.2.0 (2014-06-20) + +### Public API changes +* We removed seek compaction as a concept from RocksDB because: +1) It makes more sense for spinning disk workloads, while RocksDB is primarily designed for flash and memory, +2) It added some complexity to the important code-paths, +3) None of our internal customers were really using it. +Because of that, Options::disable_seek_compaction is now obsolete. It is still a parameter in Options, so it does not break the build, but it does not have any effect. We plan to completely remove it at some point, so we ask users to please remove this option from your code base. +* Add two parameters to NewHashLinkListRepFactory() for logging on too many entries in a hash bucket when flushing. +* Added new option BlockBasedTableOptions::hash_index_allow_collision. When enabled, prefix hash index for block-based table will not store prefix and allow hash collision, reducing memory consumption. + +### New Features +* PlainTable now supports a new key encoding: for keys of the same prefix, the prefix is only written once. It can be enabled through encoding_type parameter of NewPlainTableFactory() +* Add AdaptiveTableFactory, which is used to convert from a DB of PlainTable to BlockBasedTabe, or vise versa. It can be created using NewAdaptiveTableFactory() + +### Performance Improvements +* Tailing Iterator re-implemeted with ForwardIterator + Cascading Search Hint , see ~20% throughput improvement. + +## 3.1.0 (2014-05-21) + +### Public API changes +* Replaced ColumnFamilyOptions::table_properties_collectors with ColumnFamilyOptions::table_properties_collector_factories + +### New Features +* Hash index for block-based table will be materialized and reconstructed more efficiently. Previously hash index is constructed by scanning the whole table during every table open. +* FIFO compaction style + +## 3.0.0 (2014-05-05) + +### Public API changes +* Added _LEVEL to all InfoLogLevel enums +* Deprecated ReadOptions.prefix and ReadOptions.prefix_seek. Seek() defaults to prefix-based seek when Options.prefix_extractor is supplied. More detail is documented in https://github.com/facebook/rocksdb/wiki/Prefix-Seek-API-Changes +* MemTableRepFactory::CreateMemTableRep() takes info logger as an extra parameter. + +### New Features +* Column family support +* Added an option to use different checksum functions in BlockBasedTableOptions +* Added ApplyToAllCacheEntries() function to Cache + +## 2.8.0 (2014-04-04) + +* Removed arena.h from public header files. +* By default, checksums are verified on every read from database +* Change default value of several options, including: paranoid_checks=true, max_open_files=5000, level0_slowdown_writes_trigger=20, level0_stop_writes_trigger=24, disable_seek_compaction=true, max_background_flushes=1 and allow_mmap_writes=false +* Added is_manual_compaction to CompactionFilter::Context +* Added "virtual void WaitForJoin()" in class Env. Default operation is no-op. +* Removed BackupEngine::DeleteBackupsNewerThan() function +* Added new option -- verify_checksums_in_compaction +* Changed Options.prefix_extractor from raw pointer to shared_ptr (take ownership) + Changed HashSkipListRepFactory and HashLinkListRepFactory constructor to not take SliceTransform object (use Options.prefix_extractor implicitly) +* Added Env::GetThreadPoolQueueLen(), which returns the waiting queue length of thread pools +* Added a command "checkconsistency" in ldb tool, which checks + if file system state matches DB state (file existence and file sizes) +* Separate options related to block based table to a new struct BlockBasedTableOptions. +* WriteBatch has a new function Count() to return total size in the batch, and Data() now returns a reference instead of a copy +* Add more counters to perf context. +* Supports several more DB properties: compaction-pending, background-errors and cur-size-active-mem-table. + +### New Features +* If we find one truncated record at the end of the MANIFEST or WAL files, + we will ignore it. We assume that writers of these records were interrupted + and that we can safely ignore it. +* A new SST format "PlainTable" is added, which is optimized for memory-only workloads. It can be created through NewPlainTableFactory() or NewTotalOrderPlainTableFactory(). +* A new mem table implementation hash linked list optimizing for the case that there are only few keys for each prefix, which can be created through NewHashLinkListRepFactory(). +* Merge operator supports a new function PartialMergeMulti() to allow users to do partial merges against multiple operands. +* Now compaction filter has a V2 interface. It buffers the kv-pairs sharing the same key prefix, process them in batches, and return the batched results back to DB. The new interface uses a new structure CompactionFilterContext for the same purpose as CompactionFilter::Context in V1. +* Geo-spatial support for locations and radial-search. + +## 2.7.0 (2014-01-28) + +### Public API changes + +* Renamed `StackableDB::GetRawDB()` to `StackableDB::GetBaseDB()`. +* Renamed `WriteBatch::Data()` `const std::string& Data() const`. +* Renamed class `TableStats` to `TableProperties`. +* Deleted class `PrefixHashRepFactory`. Please use `NewHashSkipListRepFactory()` instead. +* Supported multi-threaded `EnableFileDeletions()` and `DisableFileDeletions()`. +* Added `DB::GetOptions()`. +* Added `DB::GetDbIdentity()`. + +### New Features + +* Added [BackupableDB](https://github.com/facebook/rocksdb/wiki/How-to-backup-RocksDB%3F) +* Implemented [TailingIterator](https://github.com/facebook/rocksdb/wiki/Tailing-Iterator), a special type of iterator that + doesn't create a snapshot (can be used to read newly inserted data) + and is optimized for doing sequential reads. +* Added property block for table, which allows (1) a table to store + its metadata and (2) end user to collect and store properties they + are interested in. +* Enabled caching index and filter block in block cache (turned off by default). +* Supported error report when doing manual compaction. +* Supported additional Linux platform flavors and Mac OS. +* Put with `SliceParts` - Variant of `Put()` that gathers output like `writev(2)` +* Bug fixes and code refactor for compatibility with upcoming Column + Family feature. + +### Performance Improvements + +* Huge benchmark performance improvements by multiple efforts. For example, increase in readonly QPS from about 530k in 2.6 release to 1.1 million in 2.7 [1] +* Speeding up a way RocksDB deleted obsolete files - no longer listing the whole directory under a lock -- decrease in p99 +* Use raw pointer instead of shared pointer for statistics: [5b825d](https://github.com/facebook/rocksdb/commit/5b825d6964e26ec3b4bb6faa708ebb1787f1d7bd) -- huge increase in performance -- shared pointers are slow +* Optimized locking for `Get()` -- [1fdb3f](https://github.com/facebook/rocksdb/commit/1fdb3f7dc60e96394e3e5b69a46ede5d67fb976c) -- 1.5x QPS increase for some workloads +* Cache speedup - [e8d40c3](https://github.com/facebook/rocksdb/commit/e8d40c31b3cca0c3e1ae9abe9b9003b1288026a9) +* Implemented autovector, which allocates first N elements on stack. Most of vectors in RocksDB are small. Also, we never want to allocate heap objects while holding a mutex. -- [c01676e4](https://github.com/facebook/rocksdb/commit/c01676e46d3be08c3c140361ef1f5884f47d3b3c) +* Lots of efforts to move malloc, memcpy and IO outside of locks diff --git a/librocksdb-sys/rocksdb/INSTALL.md b/librocksdb-sys/rocksdb/INSTALL.md new file mode 100644 index 0000000..f4bb7e6 --- /dev/null +++ b/librocksdb-sys/rocksdb/INSTALL.md @@ -0,0 +1,220 @@ +## Compilation + +**Important**: If you plan to run RocksDB in production, don't compile using default +`make` or `make all`. That will compile RocksDB in debug mode, which is much slower +than release mode. + +RocksDB's library should be able to compile without any dependency installed, +although we recommend installing some compression libraries (see below). +We do depend on newer gcc/clang with C++17 support (GCC >= 7, Clang >= 5). + +There are few options when compiling RocksDB: + +* [recommended] `make static_lib` will compile librocksdb.a, RocksDB static library. Compiles static library in release mode. + +* `make shared_lib` will compile librocksdb.so, RocksDB shared library. Compiles shared library in release mode. + +* `make check` will compile and run all the unit tests. `make check` will compile RocksDB in debug mode. + +* `make all` will compile our static library, and all our tools and unit tests. Our tools +depend on gflags. You will need to have gflags installed to run `make all`. This will compile RocksDB in debug mode. Don't +use binaries compiled by `make all` in production. + +* By default the binary we produce is optimized for the CPU you're compiling on +(`-march=native` or the equivalent). To build a binary compatible with the most +general architecture supported by your CPU and compiler, set `PORTABLE=1` for +the build, but performance will suffer as many operations benefit from newer +and wider instructions. In addition to `PORTABLE=0` (default) and `PORTABLE=1`, +it can be set to an architecture name recognized by your compiler. For example, +on 64-bit x86, a reasonable compromise is `PORTABLE=haswell` which supports +many or most of the available optimizations while still being compatible with +most processors made since roughly 2013. + +## Dependencies + +* You can link RocksDB with following compression libraries: + - [zlib](http://www.zlib.net/) - a library for data compression. + - [bzip2](http://www.bzip.org/) - a library for data compression. + - [lz4](https://github.com/lz4/lz4) - a library for extremely fast data compression. + - [snappy](http://google.github.io/snappy/) - a library for fast + data compression. + - [zstandard](http://www.zstd.net) - Fast real-time compression + algorithm. + +* All our tools depend on: + - [gflags](https://gflags.github.io/gflags/) - a library that handles + command line flags processing. You can compile rocksdb library even + if you don't have gflags installed. + +* `make check` will also check code formatting, which requires [clang-format](https://clang.llvm.org/docs/ClangFormat.html) + +* If you wish to build the RocksJava static target, then cmake is required for building Snappy. + +* If you wish to run microbench (e.g, `make microbench`, `make ribbon_bench` or `cmake -DWITH_BENCHMARK=1`), Google benchmark >= 1.6.0 is needed. +* You can do the following to install Google benchmark. These commands are copied from `./build_tools/ubuntu20_image/Dockerfile`: + +`$ git clone --depth 1 --branch v1.7.0 https://github.com/google/benchmark.git ~/benchmark` + +`$ cd ~/benchmark && mkdir build && cd build && cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_GTEST_TESTS=0 && ninja && ninja install` + +## Supported platforms + +* **Linux - Ubuntu** + * Upgrade your gcc to version at least 7 to get C++17 support. + * Install gflags. First, try: `sudo apt-get install libgflags-dev` + If this doesn't work and you're using Ubuntu, here's a nice tutorial: + (http://askubuntu.com/questions/312173/installing-gflags-12-04) + * Install snappy. This is usually as easy as: + `sudo apt-get install libsnappy-dev`. + * Install zlib. Try: `sudo apt-get install zlib1g-dev`. + * Install bzip2: `sudo apt-get install libbz2-dev`. + * Install lz4: `sudo apt-get install liblz4-dev`. + * Install zstandard: `sudo apt-get install libzstd-dev`. + +* **Linux - CentOS / RHEL** + * Upgrade your gcc to version at least 7 to get C++17 support + * Install gflags: + + git clone https://github.com/gflags/gflags.git + cd gflags + git checkout v2.0 + ./configure && make && sudo make install + + **Notice**: Once installed, please add the include path for gflags to your `CPATH` environment variable and the + lib path to `LIBRARY_PATH`. If installed with default settings, the include path will be `/usr/local/include` + and the lib path will be `/usr/local/lib`. + + * Install snappy: + + sudo yum install snappy snappy-devel + + * Install zlib: + + sudo yum install zlib zlib-devel + + * Install bzip2: + + sudo yum install bzip2 bzip2-devel + + * Install lz4: + + sudo yum install lz4-devel + + * Install ASAN (optional for debugging): + + sudo yum install libasan + + * Install zstandard: + * With [EPEL](https://fedoraproject.org/wiki/EPEL): + + sudo yum install libzstd-devel + + * With CentOS 8: + + sudo dnf install libzstd-devel + + * From source: + + wget https://github.com/facebook/zstd/archive/v1.1.3.tar.gz + mv v1.1.3.tar.gz zstd-1.1.3.tar.gz + tar zxvf zstd-1.1.3.tar.gz + cd zstd-1.1.3 + make && sudo make install + +* **OS X**: + * Install latest C++ compiler that supports C++ 17: + * Update XCode: run `xcode-select --install` (or install it from XCode App's settting). + * Install via [homebrew](http://brew.sh/). + * If you're first time developer in MacOS, you still need to run: `xcode-select --install` in your command line. + * run `brew tap homebrew/versions; brew install gcc7 --use-llvm` to install gcc 7 (or higher). + * run `brew install rocksdb` + +* **FreeBSD** (11.01): + + * You can either install RocksDB from the Ports system using `cd /usr/ports/databases/rocksdb && make install`, or you can follow the details below to install dependencies and compile from source code: + + * Install the dependencies for RocksDB: + + export BATCH=YES + cd /usr/ports/devel/gmake && make install + cd /usr/ports/devel/gflags && make install + + cd /usr/ports/archivers/snappy && make install + cd /usr/ports/archivers/bzip2 && make install + cd /usr/ports/archivers/liblz4 && make install + cd /usr/ports/archivesrs/zstd && make install + + cd /usr/ports/devel/git && make install + + + * Install the dependencies for RocksJava (optional): + + export BATCH=yes + cd /usr/ports/java/openjdk7 && make install + + * Build RocksDB from source: + cd ~ + git clone https://github.com/facebook/rocksdb.git + cd rocksdb + gmake static_lib + + * Build RocksJava from source (optional): + cd rocksdb + export JAVA_HOME=/usr/local/openjdk7 + gmake rocksdbjava + +* **OpenBSD** (6.3/-current): + + * As RocksDB is not available in the ports yet you have to build it on your own: + + * Install the dependencies for RocksDB: + + pkg_add gmake gflags snappy bzip2 lz4 zstd git jdk bash findutils gnuwatch + + * Build RocksDB from source: + + cd ~ + git clone https://github.com/facebook/rocksdb.git + cd rocksdb + gmake static_lib + + * Build RocksJava from source (optional): + + cd rocksdb + export JAVA_HOME=/usr/local/jdk-1.8.0 + export PATH=$PATH:/usr/local/jdk-1.8.0/bin + gmake rocksdbjava + +* **iOS**: + * Run: `TARGET_OS=IOS make static_lib`. When building the project which uses rocksdb iOS library, make sure to define an important pre-processing macros: `IOS_CROSS_COMPILE`. + +* **Windows** (Visual Studio 2017 to up): + * Read and follow the instructions at CMakeLists.txt + * Or install via [vcpkg](https://github.com/microsoft/vcpkg) + * run `vcpkg install rocksdb:x64-windows` + +* **AIX 6.1** + * Install AIX Toolbox rpms with gcc + * Use these environment variables: + + export PORTABLE=1 + export CC=gcc + export AR="ar -X64" + export EXTRA_ARFLAGS=-X64 + export EXTRA_CFLAGS=-maix64 + export EXTRA_CXXFLAGS=-maix64 + export PLATFORM_LDFLAGS="-static-libstdc++ -static-libgcc" + export LIBPATH=/opt/freeware/lib + export JAVA_HOME=/usr/java8_64 + export PATH=/opt/freeware/bin:$PATH + +* **Solaris Sparc** + * Install GCC 7 and higher. + * Use these environment variables: + + export CC=gcc + export EXTRA_CFLAGS=-m64 + export EXTRA_CXXFLAGS=-m64 + export EXTRA_LDFLAGS=-m64 + export PORTABLE=1 + export PLATFORM_LDFLAGS="-static-libstdc++ -static-libgcc" diff --git a/librocksdb-sys/rocksdb/LANGUAGE-BINDINGS.md b/librocksdb-sys/rocksdb/LANGUAGE-BINDINGS.md new file mode 100644 index 0000000..f45680e --- /dev/null +++ b/librocksdb-sys/rocksdb/LANGUAGE-BINDINGS.md @@ -0,0 +1,26 @@ +This is the list of all known third-party language bindings for RocksDB. If something is missing, please open a pull request to add it. + +* Java - https://github.com/facebook/rocksdb/tree/main/java +* Python + * http://python-rocksdb.readthedocs.io/en/latest/ + * http://pyrocksdb.readthedocs.org/en/latest/ (unmaintained) +* Perl - https://metacpan.org/pod/RocksDB +* Node.js - https://npmjs.org/package/rocksdb +* Go + * https://github.com/linxGnu/grocksdb + * https://github.com/tecbot/gorocksdb (unmaintained) +* Ruby - http://rubygems.org/gems/rocksdb-ruby +* Haskell - https://hackage.haskell.org/package/rocksdb-haskell +* PHP - https://github.com/Photonios/rocksdb-php +* C# + * https://github.com/warrenfalk/rocksdb-sharp + * https://github.com/curiosity-ai/rocksdb-sharp +* Rust + * https://github.com/pingcap/rust-rocksdb (used in production fork of https://github.com/spacejam/rust-rocksdb) + * https://github.com/spacejam/rust-rocksdb + * https://github.com/bh1xuw/rust-rocks +* D programming language - https://github.com/b1naryth1ef/rocksdb +* Erlang - https://gitlab.com/barrel-db/erlang-rocksdb +* Elixir - https://github.com/urbint/rox +* Nim - https://github.com/status-im/nim-rocksdb +* Swift and Objective-C (iOS/OSX) - https://github.com/iabudiab/ObjectiveRocks diff --git a/librocksdb-sys/rocksdb/LICENSE.Apache b/librocksdb-sys/rocksdb/LICENSE.Apache new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/librocksdb-sys/rocksdb/LICENSE.Apache @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/librocksdb-sys/rocksdb/LICENSE.leveldb b/librocksdb-sys/rocksdb/LICENSE.leveldb new file mode 100644 index 0000000..7108b0b --- /dev/null +++ b/librocksdb-sys/rocksdb/LICENSE.leveldb @@ -0,0 +1,29 @@ +This contains code that is from LevelDB, and that code is under the following license: + +Copyright (c) 2011 The LevelDB Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/librocksdb-sys/rocksdb/Makefile b/librocksdb-sys/rocksdb/Makefile new file mode 100644 index 0000000..41ca0ee --- /dev/null +++ b/librocksdb-sys/rocksdb/Makefile @@ -0,0 +1,2582 @@ +# Copyright (c) 2011 The LevelDB Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. See the AUTHORS file for names of contributors. + +# Inherit some settings from environment variables, if available + +#----------------------------------------------- + +BASH_EXISTS := $(shell which bash) +SHELL := $(shell which bash) +include common.mk + +CLEAN_FILES = # deliberately empty, so we can append below. +CFLAGS += ${EXTRA_CFLAGS} +CXXFLAGS += ${EXTRA_CXXFLAGS} +LDFLAGS += $(EXTRA_LDFLAGS) +MACHINE ?= $(shell uname -m) +ARFLAGS = ${EXTRA_ARFLAGS} rs +STRIPFLAGS = -S -x + +# Transform parallel LOG output into something more readable. +perl_command = perl -n \ + -e '@a=split("\t",$$_,-1); $$t=$$a[8];' \ + -e '$$t =~ /.*if\s\[\[\s"(.*?\.[\w\/]+)/ and $$t=$$1;' \ + -e '$$t =~ s,^\./,,;' \ + -e '$$t =~ s, >.*,,; chomp $$t;' \ + -e '$$t =~ /.*--gtest_filter=(.*?\.[\w\/]+)/ and $$t=$$1;' \ + -e 'printf "%7.3f %s %s\n", $$a[3], $$a[6] == 0 ? "PASS" : "FAIL", $$t' +quoted_perl_command = $(subst ','\'',$(perl_command)) + +# DEBUG_LEVEL can have three values: +# * DEBUG_LEVEL=2; this is the ultimate debug mode. It will compile rocksdb +# without any optimizations. To compile with level 2, issue `make dbg` +# * DEBUG_LEVEL=1; debug level 1 enables all assertions and debug code, but +# compiles rocksdb with -O2 optimizations. this is the default debug level. +# `make all` or `make ` compile RocksDB with debug level 1. +# We use this debug level when developing RocksDB. +# * DEBUG_LEVEL=0; this is the debug level we use for release. If you're +# running rocksdb in production you most definitely want to compile RocksDB +# with debug level 0. To compile with level 0, run `make shared_lib`, +# `make install-shared`, `make static_lib`, `make install-static` or +# `make install` + +# Set the default DEBUG_LEVEL to 1 +DEBUG_LEVEL?=1 + +# OBJ_DIR is where the object files reside. Default to the current directory +OBJ_DIR?=. + +# Check the MAKECMDGOALS to set the DEBUG_LEVEL and LIB_MODE appropriately + +ifneq ($(filter clean release install, $(MAKECMDGOALS)),) + DEBUG_LEVEL=0 +endif +ifneq ($(filter dbg, $(MAKECMDGOALS)),) + DEBUG_LEVEL=2 +else ifneq ($(filter shared_lib install-shared, $(MAKECMDGOALS)),) + DEBUG_LEVEL=0 + LIB_MODE=shared +else ifneq ($(filter static_lib install-static, $(MAKECMDGOALS)),) + DEBUG_LEVEL=0 + LIB_MODE=static +else ifneq ($(filter jtest rocksdbjava%, $(MAKECMDGOALS)),) + OBJ_DIR=jl + LIB_MODE=shared + ifneq ($(findstring rocksdbjavastatic, $(MAKECMDGOALS)),) + OBJ_DIR=jls + ifneq ($(DEBUG_LEVEL),2) + DEBUG_LEVEL=0 + endif + ifeq ($(MAKECMDGOALS),rocksdbjavastaticpublish) + DEBUG_LEVEL=0 + endif + endif +endif + +# LIB_MODE says whether or not to use/build "shared" or "static" libraries. +# Mode "static" means to link against static libraries (.a) +# Mode "shared" means to link against shared libraries (.so, .sl, .dylib, etc) +# +ifeq ($(DEBUG_LEVEL), 0) +# For optimized, set the default LIB_MODE to static for code size/efficiency + LIB_MODE?=static +else +# For debug, set the default LIB_MODE to shared for efficient `make check` etc. + LIB_MODE?=shared +endif + +$(info $$DEBUG_LEVEL is $(DEBUG_LEVEL), $$LIB_MODE is $(LIB_MODE)) + +# Detect what platform we're building on. +# Export some common variables that might have been passed as Make variables +# instead of environment variables. +dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; \ + export CXXFLAGS="$(EXTRA_CXXFLAGS)"; \ + export LDFLAGS="$(EXTRA_LDFLAGS)"; \ + export COMPILE_WITH_ASAN="$(COMPILE_WITH_ASAN)"; \ + export COMPILE_WITH_TSAN="$(COMPILE_WITH_TSAN)"; \ + export COMPILE_WITH_UBSAN="$(COMPILE_WITH_UBSAN)"; \ + export PORTABLE="$(PORTABLE)"; \ + export ROCKSDB_NO_FBCODE="$(ROCKSDB_NO_FBCODE)"; \ + export USE_CLANG="$(USE_CLANG)"; \ + export LIB_MODE="$(LIB_MODE)"; \ + export ROCKSDB_CXX_STANDARD="$(ROCKSDB_CXX_STANDARD)"; \ + export USE_FOLLY="$(USE_FOLLY)"; \ + "$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk")) +# this file is generated by the previous line to set build flags and sources +include make_config.mk + +# Figure out optimize level. +ifneq ($(DEBUG_LEVEL), 2) + OPTIMIZE_LEVEL ?= -O2 +endif +# `OPTIMIZE_LEVEL` is empty when the user does not set it and `DEBUG_LEVEL=2`. +# In that case, the compiler default (`-O0` for gcc and clang) will be used. +OPT += $(OPTIMIZE_LEVEL) + +# compile with -O2 if debug level is not 2 +ifneq ($(DEBUG_LEVEL), 2) +OPT += -fno-omit-frame-pointer +# Skip for archs that don't support -momit-leaf-frame-pointer +ifeq (,$(shell $(CXX) -fsyntax-only -momit-leaf-frame-pointer -xc /dev/null 2>&1)) +OPT += -momit-leaf-frame-pointer +endif +endif + +ifeq (,$(shell $(CXX) -fsyntax-only -maltivec -xc /dev/null 2>&1)) +CXXFLAGS += -DHAS_ALTIVEC +CFLAGS += -DHAS_ALTIVEC +HAS_ALTIVEC=1 +endif + +ifeq (,$(shell $(CXX) -fsyntax-only -mcpu=power8 -xc /dev/null 2>&1)) +CXXFLAGS += -DHAVE_POWER8 +CFLAGS += -DHAVE_POWER8 +HAVE_POWER8=1 +endif + +# if we're compiling for shared libraries, add the shared flags +ifeq ($(LIB_MODE),shared) +CXXFLAGS += $(PLATFORM_SHARED_CFLAGS) -DROCKSDB_DLL +CFLAGS += $(PLATFORM_SHARED_CFLAGS) -DROCKSDB_DLL +endif + +GIT_COMMAND ?= git +ifeq ($(USE_COROUTINES), 1) + USE_FOLLY = 1 + # glog/logging.h requires HAVE_CXX11_ATOMIC + OPT += -DUSE_COROUTINES -DHAVE_CXX11_ATOMIC + ROCKSDB_CXX_STANDARD = c++2a + USE_RTTI = 1 +ifneq ($(USE_CLANG), 1) + ROCKSDB_CXX_STANDARD = c++20 + PLATFORM_CXXFLAGS += -fcoroutines +endif +endif + +# if we're compiling for release, compile without debug code (-DNDEBUG) +ifeq ($(DEBUG_LEVEL),0) +OPT += -DNDEBUG + +ifneq ($(USE_RTTI), 1) + CXXFLAGS += -fno-rtti +else + CXXFLAGS += -DROCKSDB_USE_RTTI +endif +else +ifneq ($(USE_RTTI), 0) + CXXFLAGS += -DROCKSDB_USE_RTTI +else + CXXFLAGS += -fno-rtti +endif + +ifdef ASSERT_STATUS_CHECKED +# For ASC, turn off constructor elision, preventing the case where a constructor returned +# by a method may pass the ASC check if the status is checked in the inner method. Forcing +# the copy constructor to be invoked disables the optimization and will cause the calling method +# to check the status in order to prevent an error from being raised. +PLATFORM_CXXFLAGS += -fno-elide-constructors +ifeq ($(filter -DROCKSDB_ASSERT_STATUS_CHECKED,$(OPT)),) + OPT += -DROCKSDB_ASSERT_STATUS_CHECKED +endif +endif + +$(warning Warning: Compiling in debug mode. Don't use the resulting binary in production) +endif + +# `USE_LTO=1` enables link-time optimizations. Among other things, this enables +# more devirtualization opportunities and inlining across translation units. +# This can save significant overhead introduced by RocksDB's pluggable +# interfaces/internal abstractions, like in the iterator hierarchy. It works +# better when combined with profile-guided optimizations (not currently +# supported natively in Makefile). +ifeq ($(USE_LTO), 1) + CXXFLAGS += -flto + LDFLAGS += -flto -fuse-linker-plugin +endif + +# `COERCE_CONTEXT_SWITCH=1` will inject spurious wakeup and +# random length of sleep or context switch at critical +# points (e.g, before acquring db mutex) in RocksDB. +# In this way, it coerces as many excution orders as possible in the hope of +# exposing the problematic excution order +COERCE_CONTEXT_SWITCH ?= 0 +ifeq ($(COERCE_CONTEXT_SWITCH), 1) +OPT += -DCOERCE_CONTEXT_SWITCH +endif + +#----------------------------------------------- +include src.mk + +AM_DEFAULT_VERBOSITY ?= 0 + +AM_V_GEN = $(am__v_GEN_$(V)) +am__v_GEN_ = $(am__v_GEN_$(AM_DEFAULT_VERBOSITY)) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_$(V)) +am__v_at_ = $(am__v_at_$(AM_DEFAULT_VERBOSITY)) +am__v_at_0 = @ +am__v_at_1 = + +AM_V_CC = $(am__v_CC_$(V)) +am__v_CC_ = $(am__v_CC_$(AM_DEFAULT_VERBOSITY)) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = + +AM_V_CCLD = $(am__v_CCLD_$(V)) +am__v_CCLD_ = $(am__v_CCLD_$(AM_DEFAULT_VERBOSITY)) +ifneq ($(SKIP_LINK), 1) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +else +am__v_CCLD_0 = @echo " !CCLD " $@; true skip +am__v_CCLD_1 = true skip +endif +AM_V_AR = $(am__v_AR_$(V)) +am__v_AR_ = $(am__v_AR_$(AM_DEFAULT_VERBOSITY)) +am__v_AR_0 = @echo " AR " $@; +am__v_AR_1 = + +AM_LINK = $(AM_V_CCLD)$(CXX) -L. $(patsubst lib%.a, -l%, $(patsubst lib%.$(PLATFORM_SHARED_EXT), -l%, $^)) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS) +AM_SHARE = $(AM_V_CCLD) $(CXX) $(PLATFORM_SHARED_LDFLAGS)$@ -L. $(patsubst lib%.$(PLATFORM_SHARED_EXT), -l%, $^) $(EXEC_LDFLAGS) $(LDFLAGS) -o $@ + +ROCKSDB_PLUGIN_MKS = $(foreach plugin, $(ROCKSDB_PLUGINS), plugin/$(plugin)/*.mk) +include $(ROCKSDB_PLUGIN_MKS) +ROCKSDB_PLUGIN_PROTO =ROCKSDB_NAMESPACE::ObjectLibrary\&, const std::string\& +ROCKSDB_PLUGIN_SOURCES = $(foreach p, $(ROCKSDB_PLUGINS), $(foreach source, $($(p)_SOURCES), plugin/$(p)/$(source))) +ROCKSDB_PLUGIN_HEADERS = $(foreach p, $(ROCKSDB_PLUGINS), $(foreach header, $($(p)_HEADERS), plugin/$(p)/$(header))) +ROCKSDB_PLUGIN_LIBS = $(foreach p, $(ROCKSDB_PLUGINS), $(foreach lib, $($(p)_LIBS), -l$(lib))) +ROCKSDB_PLUGIN_W_FUNCS = $(foreach p, $(ROCKSDB_PLUGINS), $(if $($(p)_FUNC), $(p))) +ROCKSDB_PLUGIN_EXTERNS = $(foreach p, $(ROCKSDB_PLUGIN_W_FUNCS), int $($(p)_FUNC)($(ROCKSDB_PLUGIN_PROTO));) +ROCKSDB_PLUGIN_BUILTINS = $(foreach p, $(ROCKSDB_PLUGIN_W_FUNCS), {\"$(p)\"\, $($(p)_FUNC)}\,) +ROCKSDB_PLUGIN_LDFLAGS = $(foreach plugin, $(ROCKSDB_PLUGINS), $($(plugin)_LDFLAGS)) +ROCKSDB_PLUGIN_PKGCONFIG_REQUIRES = $(foreach plugin, $(ROCKSDB_PLUGINS), $($(plugin)_PKGCONFIG_REQUIRES)) +ROCKSDB_PLUGIN_TESTS = $(foreach p, $(ROCKSDB_PLUGINS), $(foreach test, $($(p)_TESTS), plugin/$(p)/$(test))) + +CXXFLAGS += $(foreach plugin, $(ROCKSDB_PLUGINS), $($(plugin)_CXXFLAGS)) +PLATFORM_LDFLAGS += $(ROCKSDB_PLUGIN_LDFLAGS) + +# Patch up the link flags for JNI from the plugins +JAVA_LDFLAGS += $(ROCKSDB_PLUGIN_LDFLAGS) +JAVA_STATIC_LDFLAGS += $(ROCKSDB_PLUGIN_LDFLAGS) + +# Patch up the list of java native sources with files from the plugins +ROCKSDB_PLUGIN_JNI_NATIVE_SOURCES = $(foreach plugin, $(ROCKSDB_PLUGINS), $(foreach source, $($(plugin)_JNI_NATIVE_SOURCES), plugin/$(plugin)/$(source))) +ALL_JNI_NATIVE_SOURCES = $(JNI_NATIVE_SOURCES) $(ROCKSDB_PLUGIN_JNI_NATIVE_SOURCES) +ROCKSDB_PLUGIN_JNI_CXX_INCLUDEFLAGS = $(foreach plugin, $(ROCKSDB_PLUGINS), -I./plugin/$(plugin)) + +ifneq ($(strip $(ROCKSDB_PLUGIN_PKGCONFIG_REQUIRES)),) +LDFLAGS := $(LDFLAGS) $(shell pkg-config --libs $(ROCKSDB_PLUGIN_PKGCONFIG_REQUIRES)) +ifneq ($(.SHELLSTATUS),0) +$(error pkg-config failed) +endif +CXXFLAGS := $(CXXFLAGS) $(shell pkg-config --cflags $(ROCKSDB_PLUGIN_PKGCONFIG_REQUIRES)) +ifneq ($(.SHELLSTATUS),0) +$(error pkg-config failed) +endif +endif + +CXXFLAGS += $(ARCHFLAG) + +ifeq (,$(shell $(CXX) -fsyntax-only -march=armv8-a+crc+crypto -xc /dev/null 2>&1)) +ifneq ($(PLATFORM),OS_MACOSX) +CXXFLAGS += -march=armv8-a+crc+crypto +CFLAGS += -march=armv8-a+crc+crypto +ARMCRC_SOURCE=1 +endif +endif + +export JAVAC_ARGS +CLEAN_FILES += make_config.mk rocksdb.pc + +ifeq ($(V), 1) +$(info $(shell uname -a)) +$(info $(shell $(CC) --version)) +$(info $(shell $(CXX) --version)) +endif + +missing_make_config_paths := $(shell \ + grep "\./\S*\|/\S*" -o $(CURDIR)/make_config.mk | \ + while read path; \ + do [ -e $$path ] || echo $$path; \ + done | sort | uniq | grep -v "/DOES/NOT/EXIST") + +$(foreach path, $(missing_make_config_paths), \ + $(warning Warning: $(path) does not exist)) + +ifeq ($(PLATFORM), OS_AIX) +# no debug info +else ifneq ($(PLATFORM), IOS) +CFLAGS += -g +CXXFLAGS += -g +else +# no debug info for IOS, that will make our library big +OPT += -DNDEBUG +endif + +ifeq ($(PLATFORM), OS_AIX) +ARFLAGS = -X64 rs +STRIPFLAGS = -X64 -x +endif + +ifeq ($(PLATFORM), OS_SOLARIS) + PLATFORM_CXXFLAGS += -D _GLIBCXX_USE_C99 +endif + +ifeq ($(LIB_MODE),shared) +# So that binaries are executable from build location, in addition to install location +EXEC_LDFLAGS += -Wl,-rpath -Wl,'$$ORIGIN' +endif + +ifeq ($(PLATFORM), OS_MACOSX) +ifeq ($(ARCHFLAG), -arch arm64) +ifneq ($(MACHINE), arm64) +# If we're building on a non-arm64 machine but targeting arm64 Mac, we need to disable +# linking with jemalloc (as it won't be arm64-compatible) and remove some other options +# set during platform detection +DISABLE_JEMALLOC=1 +PLATFORM_CCFLAGS := $(filter-out -march=native, $(PLATFORM_CCFLAGS)) +PLATFORM_CXXFLAGS := $(filter-out -march=native, $(PLATFORM_CXXFLAGS)) +endif +endif +endif + +# ASAN doesn't work well with jemalloc. If we're compiling with ASAN, we should use regular malloc. +ifdef COMPILE_WITH_ASAN + DISABLE_JEMALLOC=1 + ASAN_OPTIONS?=detect_stack_use_after_return=1 + export ASAN_OPTIONS + EXEC_LDFLAGS += -fsanitize=address + PLATFORM_CCFLAGS += -fsanitize=address + PLATFORM_CXXFLAGS += -fsanitize=address +ifeq ($(LIB_MODE),shared) +ifdef USE_CLANG +# Fix false ODR violation; see https://github.com/google/sanitizers/issues/1017 + EXEC_LDFLAGS += -mllvm -asan-use-private-alias=1 + PLATFORM_CXXFLAGS += -mllvm -asan-use-private-alias=1 +endif +endif +endif + +# TSAN doesn't work well with jemalloc. If we're compiling with TSAN, we should use regular malloc. +ifdef COMPILE_WITH_TSAN + DISABLE_JEMALLOC=1 + EXEC_LDFLAGS += -fsanitize=thread + PLATFORM_CCFLAGS += -fsanitize=thread -fPIC -DFOLLY_SANITIZE_THREAD + PLATFORM_CXXFLAGS += -fsanitize=thread -fPIC -DFOLLY_SANITIZE_THREAD + # Turn off -pg when enabling TSAN testing, because that induces + # a link failure. TODO: find the root cause + PROFILING_FLAGS = + # LUA is not supported under TSAN + LUA_PATH = + # Limit keys for crash test under TSAN to avoid error: + # "ThreadSanitizer: DenseSlabAllocator overflow. Dying." + CRASH_TEST_EXT_ARGS += --max_key=1000000 +endif + +# AIX doesn't work with -pg +ifeq ($(PLATFORM), OS_AIX) + PROFILING_FLAGS = +endif + +# USAN doesn't work well with jemalloc. If we're compiling with USAN, we should use regular malloc. +ifdef COMPILE_WITH_UBSAN + DISABLE_JEMALLOC=1 + # Suppress alignment warning because murmurhash relies on casting unaligned + # memory to integer. Fixing it may cause performance regression. 3-way crc32 + # relies on it too, although it can be rewritten to eliminate with minimal + # performance regression. + EXEC_LDFLAGS += -fsanitize=undefined -fno-sanitize-recover=all + PLATFORM_CCFLAGS += -fsanitize=undefined -fno-sanitize-recover=all -DROCKSDB_UBSAN_RUN + PLATFORM_CXXFLAGS += -fsanitize=undefined -fno-sanitize-recover=all -DROCKSDB_UBSAN_RUN +endif + +ifdef ROCKSDB_VALGRIND_RUN + PLATFORM_CCFLAGS += -DROCKSDB_VALGRIND_RUN + PLATFORM_CXXFLAGS += -DROCKSDB_VALGRIND_RUN +endif +ifdef ROCKSDB_FULL_VALGRIND_RUN + # Some tests are slow when run under valgrind and are only run when + # explicitly requested via the ROCKSDB_FULL_VALGRIND_RUN compiler flag. + PLATFORM_CCFLAGS += -DROCKSDB_VALGRIND_RUN -DROCKSDB_FULL_VALGRIND_RUN + PLATFORM_CXXFLAGS += -DROCKSDB_VALGRIND_RUN -DROCKSDB_FULL_VALGRIND_RUN +endif + +ifndef DISABLE_JEMALLOC + ifdef JEMALLOC + PLATFORM_CXXFLAGS += -DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE + PLATFORM_CCFLAGS += -DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE + ifeq ($(USE_FOLLY),1) + PLATFORM_CXXFLAGS += -DUSE_JEMALLOC + PLATFORM_CCFLAGS += -DUSE_JEMALLOC + endif + ifeq ($(USE_FOLLY_LITE),1) + PLATFORM_CXXFLAGS += -DUSE_JEMALLOC + PLATFORM_CCFLAGS += -DUSE_JEMALLOC + endif + endif + ifdef WITH_JEMALLOC_FLAG + PLATFORM_LDFLAGS += -ljemalloc + JAVA_LDFLAGS += -ljemalloc + endif + EXEC_LDFLAGS := $(JEMALLOC_LIB) $(EXEC_LDFLAGS) + PLATFORM_CXXFLAGS += $(JEMALLOC_INCLUDE) + PLATFORM_CCFLAGS += $(JEMALLOC_INCLUDE) +endif + +ifndef USE_FOLLY + USE_FOLLY=0 +endif + +ifndef GTEST_THROW_ON_FAILURE + export GTEST_THROW_ON_FAILURE=1 +endif +ifndef GTEST_HAS_EXCEPTIONS + export GTEST_HAS_EXCEPTIONS=1 +endif + +GTEST_DIR = third-party/gtest-1.8.1/fused-src +# AIX: pre-defined system headers are surrounded by an extern "C" block +ifeq ($(PLATFORM), OS_AIX) + PLATFORM_CCFLAGS += -I$(GTEST_DIR) + PLATFORM_CXXFLAGS += -I$(GTEST_DIR) +else + PLATFORM_CCFLAGS += -isystem $(GTEST_DIR) + PLATFORM_CXXFLAGS += -isystem $(GTEST_DIR) +endif + +# This provides a Makefile simulation of a Meta-internal folly integration. +# It is not validated for general use. +# +# USE_FOLLY links the build targets with libfolly.a. The latter could be +# built using 'make build_folly', or built externally and specified in +# the CXXFLAGS and EXTRA_LDFLAGS env variables. The build_detect_platform +# script tries to detect if an external folly dependency has been specified. +# If not, it exports FOLLY_PATH to the path of the installed Folly and +# dependency libraries. +# +# USE_FOLLY_LITE cherry picks source files from Folly to include in the +# RocksDB library. Its faster and has fewer dependencies on 3rd party +# libraries, but with limited functionality. For example, coroutine +# functionality is not available. +ifeq ($(USE_FOLLY),1) +ifeq ($(USE_FOLLY_LITE),1) +$(error Please specify only one of USE_FOLLY and USE_FOLLY_LITE) +endif +ifneq ($(strip $(FOLLY_PATH)),) + BOOST_PATH = $(shell (ls -d $(FOLLY_PATH)/../boost*)) + DBL_CONV_PATH = $(shell (ls -d $(FOLLY_PATH)/../double-conversion*)) + GFLAGS_PATH = $(shell (ls -d $(FOLLY_PATH)/../gflags*)) + GLOG_PATH = $(shell (ls -d $(FOLLY_PATH)/../glog*)) + LIBEVENT_PATH = $(shell (ls -d $(FOLLY_PATH)/../libevent*)) + XZ_PATH = $(shell (ls -d $(FOLLY_PATH)/../xz*)) + LIBSODIUM_PATH = $(shell (ls -d $(FOLLY_PATH)/../libsodium*)) + FMT_PATH = $(shell (ls -d $(FOLLY_PATH)/../fmt*)) + + # For some reason, glog and fmt libraries are under either lib or lib64 + GLOG_LIB_PATH = $(shell (ls -d $(GLOG_PATH)/lib*)) + FMT_LIB_PATH = $(shell (ls -d $(FMT_PATH)/lib*)) + + # AIX: pre-defined system headers are surrounded by an extern "C" block + ifeq ($(PLATFORM), OS_AIX) + PLATFORM_CCFLAGS += -I$(BOOST_PATH)/include -I$(DBL_CONV_PATH)/include -I$(GLOG_PATH)/include -I$(LIBEVENT_PATH)/include -I$(XZ_PATH)/include -I$(LIBSODIUM_PATH)/include -I$(FOLLY_PATH)/include -I$(FMT_PATH)/include + PLATFORM_CXXFLAGS += -I$(BOOST_PATH)/include -I$(DBL_CONV_PATH)/include -I$(GLOG_PATH)/include -I$(LIBEVENT_PATH)/include -I$(XZ_PATH)/include -I$(LIBSODIUM_PATH)/include -I$(FOLLY_PATH)/include -I$(FMT_PATH)/include + else + PLATFORM_CCFLAGS += -isystem $(BOOST_PATH)/include -isystem $(DBL_CONV_PATH)/include -isystem $(GLOG_PATH)/include -isystem $(LIBEVENT_PATH)/include -isystem $(XZ_PATH)/include -isystem $(LIBSODIUM_PATH)/include -isystem $(FOLLY_PATH)/include -isystem $(FMT_PATH)/include + PLATFORM_CXXFLAGS += -isystem $(BOOST_PATH)/include -isystem $(DBL_CONV_PATH)/include -isystem $(GLOG_PATH)/include -isystem $(LIBEVENT_PATH)/include -isystem $(XZ_PATH)/include -isystem $(LIBSODIUM_PATH)/include -isystem $(FOLLY_PATH)/include -isystem $(FMT_PATH)/include + endif + + # Add -ldl at the end as gcc resolves a symbol in a library by searching only in libraries specified later + # in the command line + PLATFORM_LDFLAGS += $(FOLLY_PATH)/lib/libfolly.a $(BOOST_PATH)/lib/libboost_context.a $(BOOST_PATH)/lib/libboost_filesystem.a $(BOOST_PATH)/lib/libboost_atomic.a $(BOOST_PATH)/lib/libboost_program_options.a $(BOOST_PATH)/lib/libboost_regex.a $(BOOST_PATH)/lib/libboost_system.a $(BOOST_PATH)/lib/libboost_thread.a $(DBL_CONV_PATH)/lib/libdouble-conversion.a $(FMT_LIB_PATH)/libfmt.a $(GLOG_LIB_PATH)/libglog.so $(GFLAGS_PATH)/lib/libgflags.so.2.2 $(LIBEVENT_PATH)/lib/libevent-2.1.so -ldl + PLATFORM_LDFLAGS += -Wl,-rpath=$(GFLAGS_PATH)/lib -Wl,-rpath=$(GLOG_LIB_PATH) -Wl,-rpath=$(LIBEVENT_PATH)/lib -Wl,-rpath=$(LIBSODIUM_PATH)/lib -Wl,-rpath=$(LIBEVENT_PATH)/lib +endif + PLATFORM_CCFLAGS += -DUSE_FOLLY -DFOLLY_NO_CONFIG + PLATFORM_CXXFLAGS += -DUSE_FOLLY -DFOLLY_NO_CONFIG +endif + +ifeq ($(USE_FOLLY_LITE),1) + # Path to the Folly source code and include files + FOLLY_DIR = ./third-party/folly + # AIX: pre-defined system headers are surrounded by an extern "C" block + ifeq ($(PLATFORM), OS_AIX) + PLATFORM_CCFLAGS += -I$(FOLLY_DIR) + PLATFORM_CXXFLAGS += -I$(FOLLY_DIR) + else + PLATFORM_CCFLAGS += -isystem $(FOLLY_DIR) + PLATFORM_CXXFLAGS += -isystem $(FOLLY_DIR) + endif + PLATFORM_CCFLAGS += -DUSE_FOLLY -DFOLLY_NO_CONFIG + PLATFORM_CXXFLAGS += -DUSE_FOLLY -DFOLLY_NO_CONFIG +# TODO: fix linking with fbcode compiler config + PLATFORM_LDFLAGS += -lglog +endif + +ifdef TEST_CACHE_LINE_SIZE + PLATFORM_CCFLAGS += -DTEST_CACHE_LINE_SIZE=$(TEST_CACHE_LINE_SIZE) + PLATFORM_CXXFLAGS += -DTEST_CACHE_LINE_SIZE=$(TEST_CACHE_LINE_SIZE) +endif +ifdef TEST_UINT128_COMPAT + PLATFORM_CCFLAGS += -DTEST_UINT128_COMPAT=1 + PLATFORM_CXXFLAGS += -DTEST_UINT128_COMPAT=1 +endif +ifdef ROCKSDB_MODIFY_NPHASH + PLATFORM_CCFLAGS += -DROCKSDB_MODIFY_NPHASH=1 + PLATFORM_CXXFLAGS += -DROCKSDB_MODIFY_NPHASH=1 +endif + +# This (the first rule) must depend on "all". +default: all + +WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare -Wshadow \ + -Wunused-parameter + +ifeq (,$(filter amd64, $(MACHINE))) + C_WARNING_FLAGS = -Wstrict-prototypes +endif + +ifdef USE_CLANG + # Used by some teams in Facebook + WARNING_FLAGS += -Wshift-sign-overflow -Wambiguous-reversed-operator +endif + +ifeq ($(PLATFORM), OS_OPENBSD) + WARNING_FLAGS += -Wno-unused-lambda-capture +endif + +ifndef DISABLE_WARNING_AS_ERROR + WARNING_FLAGS += -Werror +endif + + +ifdef LUA_PATH + +ifndef LUA_INCLUDE +LUA_INCLUDE=$(LUA_PATH)/include +endif + +LUA_INCLUDE_FILE=$(LUA_INCLUDE)/lualib.h + +ifeq ("$(wildcard $(LUA_INCLUDE_FILE))", "") +# LUA_INCLUDE_FILE does not exist +$(error Cannot find lualib.h under $(LUA_INCLUDE). Try to specify both LUA_PATH and LUA_INCLUDE manually) +endif +LUA_FLAGS = -I$(LUA_INCLUDE) -DLUA -DLUA_COMPAT_ALL +CFLAGS += $(LUA_FLAGS) +CXXFLAGS += $(LUA_FLAGS) + +ifndef LUA_LIB +LUA_LIB = $(LUA_PATH)/lib/liblua.a +endif +ifeq ("$(wildcard $(LUA_LIB))", "") # LUA_LIB does not exist +$(error $(LUA_LIB) does not exist. Try to specify both LUA_PATH and LUA_LIB manually) +endif +EXEC_LDFLAGS += $(LUA_LIB) + +endif + +ifeq ($(NO_THREEWAY_CRC32C), 1) + CXXFLAGS += -DNO_THREEWAY_CRC32C +endif + +CFLAGS += $(C_WARNING_FLAGS) $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT) +CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers + +# Allow offsetof to work on non-standard layout types. Some compiler could +# completely reject our usage of offsetof, but we will solve that when it +# happens. +CXXFLAGS += -Wno-invalid-offsetof + +LDFLAGS += $(PLATFORM_LDFLAGS) + +LIB_OBJECTS = $(patsubst %.cc, $(OBJ_DIR)/%.o, $(LIB_SOURCES)) +LIB_OBJECTS += $(patsubst %.cc, $(OBJ_DIR)/%.o, $(ROCKSDB_PLUGIN_SOURCES)) +ifeq ($(HAVE_POWER8),1) +LIB_OBJECTS += $(patsubst %.c, $(OBJ_DIR)/%.o, $(LIB_SOURCES_C)) +LIB_OBJECTS += $(patsubst %.S, $(OBJ_DIR)/%.o, $(LIB_SOURCES_ASM)) +endif + +ifeq ($(USE_FOLLY_LITE),1) + LIB_OBJECTS += $(patsubst %.cpp, $(OBJ_DIR)/%.o, $(FOLLY_SOURCES)) +endif + +# range_tree is not compatible with non GNU libc on ppc64 +# see https://jira.percona.com/browse/PS-7559 +ifneq ($(PPC_LIBC_IS_GNU),0) + LIB_OBJECTS += $(patsubst %.cc, $(OBJ_DIR)/%.o, $(RANGE_TREE_SOURCES)) +endif + +GTEST = $(OBJ_DIR)/$(GTEST_DIR)/gtest/gtest-all.o +TESTUTIL = $(OBJ_DIR)/test_util/testutil.o +TESTHARNESS = $(OBJ_DIR)/test_util/testharness.o $(TESTUTIL) $(GTEST) +VALGRIND_ERROR = 2 +VALGRIND_VER := $(join $(VALGRIND_VER),valgrind) + +VALGRIND_OPTS = --error-exitcode=$(VALGRIND_ERROR) --leak-check=full +# Not yet supported: --show-leak-kinds=definite,possible,reachable --errors-for-leak-kinds=definite,possible,reachable + +TEST_OBJECTS = $(patsubst %.cc, $(OBJ_DIR)/%.o, $(TEST_LIB_SOURCES) $(MOCK_LIB_SOURCES)) $(GTEST) +BENCH_OBJECTS = $(patsubst %.cc, $(OBJ_DIR)/%.o, $(BENCH_LIB_SOURCES)) +CACHE_BENCH_OBJECTS = $(patsubst %.cc, $(OBJ_DIR)/%.o, $(CACHE_BENCH_LIB_SOURCES)) +TOOL_OBJECTS = $(patsubst %.cc, $(OBJ_DIR)/%.o, $(TOOL_LIB_SOURCES)) +ANALYZE_OBJECTS = $(patsubst %.cc, $(OBJ_DIR)/%.o, $(ANALYZER_LIB_SOURCES)) +STRESS_OBJECTS = $(patsubst %.cc, $(OBJ_DIR)/%.o, $(STRESS_LIB_SOURCES)) + +# Exclude build_version.cc -- a generated source file -- from all sources. Not needed for dependencies +ALL_SOURCES = $(filter-out util/build_version.cc, $(LIB_SOURCES)) $(TEST_LIB_SOURCES) $(MOCK_LIB_SOURCES) $(GTEST_DIR)/gtest/gtest-all.cc +ALL_SOURCES += $(TOOL_LIB_SOURCES) $(BENCH_LIB_SOURCES) $(CACHE_BENCH_LIB_SOURCES) $(ANALYZER_LIB_SOURCES) $(STRESS_LIB_SOURCES) +ALL_SOURCES += $(TEST_MAIN_SOURCES) $(TOOL_MAIN_SOURCES) $(BENCH_MAIN_SOURCES) +ALL_SOURCES += $(ROCKSDB_PLUGIN_SOURCES) $(ROCKSDB_PLUGIN_TESTS) + +PLUGIN_TESTS = $(patsubst %.cc, %, $(notdir $(ROCKSDB_PLUGIN_TESTS))) +TESTS = $(patsubst %.cc, %, $(notdir $(TEST_MAIN_SOURCES))) +TESTS += $(patsubst %.c, %, $(notdir $(TEST_MAIN_SOURCES_C))) +TESTS += $(PLUGIN_TESTS) + +# `make check-headers` to very that each header file includes its own +# dependencies +ifneq ($(filter check-headers, $(MAKECMDGOALS)),) +# TODO: add/support JNI headers + DEV_HEADER_DIRS := $(sort include/ $(dir $(ALL_SOURCES))) +# Some headers like in port/ are platform-specific + DEV_HEADERS := $(shell $(FIND) $(DEV_HEADER_DIRS) -type f -name '*.h' | grep -E -v 'port/|plugin/|lua/|range_tree/') +else + DEV_HEADERS := +endif +HEADER_OK_FILES = $(patsubst %.h, %.h.ok, $(DEV_HEADERS)) + +AM_V_CCH = $(am__v_CCH_$(V)) +am__v_CCH_ = $(am__v_CCH_$(AM_DEFAULT_VERBOSITY)) +am__v_CCH_0 = @echo " CC.h " $<; +am__v_CCH_1 = + +%.h.ok: %.h # .h.ok not actually created, so re-checked on each invocation +# -DROCKSDB_NAMESPACE=42 ensures the namespace header is included + $(AM_V_CCH) echo '#include "$<"' | $(CXX) $(CXXFLAGS) -DROCKSDB_NAMESPACE=42 -x c++ -c - -o /dev/null + +check-headers: $(HEADER_OK_FILES) + +# options_settable_test doesn't pass with UBSAN as we use hack in the test +ifdef ASSERT_STATUS_CHECKED +# TODO: finish fixing all tests to pass this check +TESTS_FAILING_ASC = \ + c_test \ + env_test \ + range_locking_test \ + testutil_test \ + +# Since we have very few ASC exclusions left, excluding them from +# the build is the most convenient way to exclude them from testing +TESTS := $(filter-out $(TESTS_FAILING_ASC),$(TESTS)) +endif + +ROCKSDBTESTS_SUBSET ?= $(TESTS) + +# c_test - doesn't use gtest +# env_test - suspicious use of test::TmpDir +# deletefile_test - serial because it generates giant temporary files in +# its various tests. Parallel can fill up your /dev/shm +# db_bloom_filter_test - serial because excessive space usage by instances +# of DBFilterConstructionReserveMemoryTestWithParam can fill up /dev/shm +NON_PARALLEL_TEST = \ + c_test \ + env_test \ + deletefile_test \ + db_bloom_filter_test \ + $(PLUGIN_TESTS) \ + +PARALLEL_TEST = $(filter-out $(NON_PARALLEL_TEST), $(TESTS)) + +# Not necessarily well thought out or up-to-date, but matches old list +TESTS_PLATFORM_DEPENDENT := \ + db_basic_test \ + db_blob_basic_test \ + db_encryption_test \ + external_sst_file_basic_test \ + auto_roll_logger_test \ + bloom_test \ + dynamic_bloom_test \ + c_test \ + checkpoint_test \ + crc32c_test \ + coding_test \ + inlineskiplist_test \ + env_basic_test \ + env_test \ + env_logger_test \ + io_posix_test \ + hash_test \ + random_test \ + ribbon_test \ + thread_local_test \ + work_queue_test \ + rate_limiter_test \ + perf_context_test \ + iostats_context_test \ + +# Sort ROCKSDBTESTS_SUBSET for filtering, except db_test is special (expensive) +# so is placed first (out-of-order) +ROCKSDBTESTS_SUBSET := $(filter db_test, $(ROCKSDBTESTS_SUBSET)) $(sort $(filter-out db_test, $(ROCKSDBTESTS_SUBSET))) + +ifdef ROCKSDBTESTS_START + ROCKSDBTESTS_SUBSET := $(shell echo $(ROCKSDBTESTS_SUBSET) | sed 's/^.*$(ROCKSDBTESTS_START)/$(ROCKSDBTESTS_START)/') +endif + +ifdef ROCKSDBTESTS_END + ROCKSDBTESTS_SUBSET := $(shell echo $(ROCKSDBTESTS_SUBSET) | sed 's/$(ROCKSDBTESTS_END).*//') +endif + +ifeq ($(ROCKSDBTESTS_PLATFORM_DEPENDENT), only) + ROCKSDBTESTS_SUBSET := $(filter $(TESTS_PLATFORM_DEPENDENT), $(ROCKSDBTESTS_SUBSET)) +else ifeq ($(ROCKSDBTESTS_PLATFORM_DEPENDENT), exclude) + ROCKSDBTESTS_SUBSET := $(filter-out $(TESTS_PLATFORM_DEPENDENT), $(ROCKSDBTESTS_SUBSET)) +endif + +# bench_tool_analyer main is in bench_tool_analyzer_tool, or this would be simpler... +TOOLS = $(patsubst %.cc, %, $(notdir $(patsubst %_tool.cc, %.cc, $(TOOLS_MAIN_SOURCES)))) + +TEST_LIBS = \ + librocksdb_env_basic_test.a + +# TODO: add back forward_iterator_bench, after making it build in all environemnts. +BENCHMARKS = $(patsubst %.cc, %, $(notdir $(BENCH_MAIN_SOURCES))) + +MICROBENCHS = $(patsubst %.cc, %, $(notdir $(MICROBENCH_SOURCES))) + +# if user didn't config LIBNAME, set the default +ifeq ($(LIBNAME),) + LIBNAME=librocksdb +# we should only run rocksdb in production with DEBUG_LEVEL 0 +ifneq ($(DEBUG_LEVEL),0) + LIBDEBUG=_debug +endif +endif +STATIC_LIBRARY = ${LIBNAME}$(LIBDEBUG).a +STATIC_TEST_LIBRARY = ${LIBNAME}_test$(LIBDEBUG).a +STATIC_TOOLS_LIBRARY = ${LIBNAME}_tools$(LIBDEBUG).a +STATIC_STRESS_LIBRARY = ${LIBNAME}_stress$(LIBDEBUG).a + +ALL_STATIC_LIBS = $(STATIC_LIBRARY) $(STATIC_TEST_LIBRARY) $(STATIC_TOOLS_LIBRARY) $(STATIC_STRESS_LIBRARY) + +SHARED_TEST_LIBRARY = ${LIBNAME}_test$(LIBDEBUG).$(PLATFORM_SHARED_EXT) +SHARED_TOOLS_LIBRARY = ${LIBNAME}_tools$(LIBDEBUG).$(PLATFORM_SHARED_EXT) +SHARED_STRESS_LIBRARY = ${LIBNAME}_stress$(LIBDEBUG).$(PLATFORM_SHARED_EXT) + +ALL_SHARED_LIBS = $(SHARED1) $(SHARED2) $(SHARED3) $(SHARED4) $(SHARED_TEST_LIBRARY) $(SHARED_TOOLS_LIBRARY) $(SHARED_STRESS_LIBRARY) + +ifeq ($(LIB_MODE),shared) +LIBRARY=$(SHARED1) +TEST_LIBRARY=$(SHARED_TEST_LIBRARY) +TOOLS_LIBRARY=$(SHARED_TOOLS_LIBRARY) +STRESS_LIBRARY=$(SHARED_STRESS_LIBRARY) +CLOUD_LIBRARY=$(SHARED_CLOUD_LIBRARY) +else +LIBRARY=$(STATIC_LIBRARY) +TEST_LIBRARY=$(STATIC_TEST_LIBRARY) +TOOLS_LIBRARY=$(STATIC_TOOLS_LIBRARY) +endif +STRESS_LIBRARY=$(STATIC_STRESS_LIBRARY) + +ROCKSDB_MAJOR = $(shell grep -E "ROCKSDB_MAJOR.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3) +ROCKSDB_MINOR = $(shell grep -E "ROCKSDB_MINOR.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3) +ROCKSDB_PATCH = $(shell grep -E "ROCKSDB_PATCH.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3) + +# If NO_UPDATE_BUILD_VERSION is set we don't update util/build_version.cc, but +# the file needs to already exist or else the build will fail +ifndef NO_UPDATE_BUILD_VERSION + +# By default, use the current date-time as the date. If there are no changes, +# we will use the last commit date instead. +build_date := $(shell date "+%Y-%m-%d %T") + +ifdef FORCE_GIT_SHA + git_sha := $(FORCE_GIT_SHA) + git_mod := 1 + git_date := $(build_date) +else + git_sha := $(shell git rev-parse HEAD 2>/dev/null) + git_tag := $(shell git symbolic-ref -q --short HEAD 2> /dev/null || git describe --tags --exact-match 2>/dev/null) + git_mod := $(shell git diff-index HEAD --quiet 2>/dev/null; echo $$?) + git_date := $(shell git log -1 --date=format:"%Y-%m-%d %T" --format="%ad" 2>/dev/null) +endif +gen_build_version = sed -e s/@GIT_SHA@/$(git_sha)/ -e s:@GIT_TAG@:"$(git_tag)": -e s/@GIT_MOD@/"$(git_mod)"/ -e s/@BUILD_DATE@/"$(build_date)"/ -e s/@GIT_DATE@/"$(git_date)"/ -e s/@ROCKSDB_PLUGIN_BUILTINS@/'$(ROCKSDB_PLUGIN_BUILTINS)'/ -e s/@ROCKSDB_PLUGIN_EXTERNS@/"$(ROCKSDB_PLUGIN_EXTERNS)"/ util/build_version.cc.in + +# Record the version of the source that we are compiling. +# We keep a record of the git revision in this file. It is then built +# as a regular source file as part of the compilation process. +# One can run "strings executable_filename | grep _build_" to find +# the version of the source that we used to build the executable file. +util/build_version.cc: $(filter-out $(OBJ_DIR)/util/build_version.o, $(LIB_OBJECTS)) util/build_version.cc.in + $(AM_V_GEN)rm -f $@-t + $(AM_V_at)$(gen_build_version) > $@ +endif +CLEAN_FILES += util/build_version.cc + +default: all + +#----------------------------------------------- +# Create platform independent shared libraries. +#----------------------------------------------- +ifneq ($(PLATFORM_SHARED_EXT),) + +ifneq ($(PLATFORM_SHARED_VERSIONED),true) +SHARED1 = ${LIBNAME}$(LIBDEBUG).$(PLATFORM_SHARED_EXT) +SHARED2 = $(SHARED1) +SHARED3 = $(SHARED1) +SHARED4 = $(SHARED1) +SHARED = $(SHARED1) +else +SHARED_MAJOR = $(ROCKSDB_MAJOR) +SHARED_MINOR = $(ROCKSDB_MINOR) +SHARED_PATCH = $(ROCKSDB_PATCH) +SHARED1 = ${LIBNAME}.$(PLATFORM_SHARED_EXT) +ifeq ($(PLATFORM), OS_MACOSX) +SHARED_OSX = $(LIBNAME)$(LIBDEBUG).$(SHARED_MAJOR) +SHARED2 = $(SHARED_OSX).$(PLATFORM_SHARED_EXT) +SHARED3 = $(SHARED_OSX).$(SHARED_MINOR).$(PLATFORM_SHARED_EXT) +SHARED4 = $(SHARED_OSX).$(SHARED_MINOR).$(SHARED_PATCH).$(PLATFORM_SHARED_EXT) +else +SHARED2 = $(SHARED1).$(SHARED_MAJOR) +SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR) +SHARED4 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR).$(SHARED_PATCH) +endif # MACOSX +SHARED = $(SHARED1) $(SHARED2) $(SHARED3) $(SHARED4) +$(SHARED1): $(SHARED4) $(SHARED2) + ln -fs $(SHARED4) $(SHARED1) +$(SHARED2): $(SHARED4) $(SHARED3) + ln -fs $(SHARED4) $(SHARED2) +$(SHARED3): $(SHARED4) + ln -fs $(SHARED4) $(SHARED3) + +endif # PLATFORM_SHARED_VERSIONED +$(SHARED4): $(LIB_OBJECTS) + $(AM_V_CCLD) $(CXX) $(PLATFORM_SHARED_LDFLAGS)$(SHARED3) $(LIB_OBJECTS) $(LDFLAGS) -o $@ +endif # PLATFORM_SHARED_EXT + +.PHONY: check clean coverage ldb_tests package dbg gen-pc build_size \ + release tags tags0 valgrind_check format static_lib shared_lib all \ + rocksdbjavastatic rocksdbjava install install-static install-shared \ + uninstall analyze tools tools_lib check-headers checkout_folly + +all: $(LIBRARY) $(BENCHMARKS) tools tools_lib test_libs $(TESTS) + +all_but_some_tests: $(LIBRARY) $(BENCHMARKS) tools tools_lib test_libs $(ROCKSDBTESTS_SUBSET) + +static_lib: $(STATIC_LIBRARY) + +shared_lib: $(SHARED) + +stress_lib: $(STRESS_LIBRARY) + +tools: $(TOOLS) + +tools_lib: $(TOOLS_LIBRARY) + +test_libs: $(TEST_LIBS) + +benchmarks: $(BENCHMARKS) + +microbench: $(MICROBENCHS) + +run_microbench: $(MICROBENCHS) + for t in $(MICROBENCHS); do echo "===== Running benchmark $$t (`date`)"; ./$$t || exit 1; done; + +dbg: $(LIBRARY) $(BENCHMARKS) tools $(TESTS) + +# creates library and programs +release: clean + LIB_MODE=$(LIB_MODE) DEBUG_LEVEL=0 $(MAKE) $(LIBRARY) tools db_bench + +coverage: clean + COVERAGEFLAGS="-fprofile-arcs -ftest-coverage" LDFLAGS+="-lgcov" $(MAKE) J=1 all check + cd coverage && ./coverage_test.sh + # Delete intermediate files + $(FIND) . -type f \( -name "*.gcda" -o -name "*.gcno" \) -exec rm -f {} \; + +# Run all tests in parallel, accumulating per-test logs in t/log-*. +# +# Each t/run-* file is a tiny generated bourne shell script that invokes one of +# sub-tests. Why use a file for this? Because that makes the invocation of +# parallel below simpler, which in turn makes the parsing of parallel's +# LOG simpler (the latter is for live monitoring as parallel +# tests run). +# +# Test names are extracted by running tests with --gtest_list_tests. +# This filter removes the "#"-introduced comments, and expands to +# fully-qualified names by changing input like this: +# +# DBTest. +# Empty +# WriteEmptyBatch +# MultiThreaded/MultiThreadedDBTest. +# MultiThreaded/0 # GetParam() = 0 +# MultiThreaded/1 # GetParam() = 1 +# +# into this: +# +# DBTest.Empty +# DBTest.WriteEmptyBatch +# MultiThreaded/MultiThreadedDBTest.MultiThreaded/0 +# MultiThreaded/MultiThreadedDBTest.MultiThreaded/1 +# + +parallel_tests = $(patsubst %,parallel_%,$(PARALLEL_TEST)) +.PHONY: gen_parallel_tests $(parallel_tests) +$(parallel_tests): + $(AM_V_at)TEST_BINARY=$(patsubst parallel_%,%,$@); \ + TEST_NAMES=` \ + (./$$TEST_BINARY --gtest_list_tests || echo " $${TEST_BINARY}__list_tests_failure") \ + | awk '/^[^ ]/ { prefix = $$1 } /^[ ]/ { print prefix $$1 }'`; \ + echo " Generating parallel test scripts for $$TEST_BINARY"; \ + for TEST_NAME in $$TEST_NAMES; do \ + TEST_SCRIPT=t/run-$$TEST_BINARY-$${TEST_NAME//\//-}; \ + printf '%s\n' \ + '#!/bin/sh' \ + "d=\$(TEST_TMPDIR)$$TEST_SCRIPT" \ + 'mkdir -p $$d' \ + "TEST_TMPDIR=\$$d $(DRIVER) ./$$TEST_BINARY --gtest_filter=$$TEST_NAME" \ + > $$TEST_SCRIPT; \ + chmod a=rx $$TEST_SCRIPT; \ + done + +gen_parallel_tests: + $(AM_V_at)mkdir -p t + $(AM_V_at)$(FIND) t -type f -name 'run-*' -exec rm -f {} \; + $(MAKE) $(parallel_tests) + +# Reorder input lines (which are one per test) so that the +# longest-running tests appear first in the output. +# Do this by prefixing each selected name with its duration, +# sort the resulting names, and remove the leading numbers. +# FIXME: the "100" we prepend is a fake time, for now. +# FIXME: squirrel away timings from each run and use them +# (when present) on subsequent runs to order these tests. +# +# Without this reordering, these two tests would happen to start only +# after almost all other tests had completed, thus adding 100 seconds +# to the duration of parallel "make check". That's the difference +# between 4 minutes (old) and 2m20s (new). +# +# 152.120 PASS t/DBTest.FileCreationRandomFailure +# 107.816 PASS t/DBTest.EncodeDecompressedBlockSizeTest +# +slow_test_regexp = \ + ^.*MySQLStyleTransactionTest.*$$|^.*SnapshotConcurrentAccessTest.*$$|^.*SeqAdvanceConcurrentTest.*$$|^t/run-table_test-HarnessTest.Randomized$$|^t/run-db_test-.*(?:FileCreationRandomFailure|EncodeDecompressedBlockSizeTest)$$|^.*RecoverFromCorruptedWALWithoutFlush$$ +prioritize_long_running_tests = \ + perl -pe 's,($(slow_test_regexp)),100 $$1,' \ + | sort -k1,1gr \ + | sed 's/^[.0-9]* //' + +# "make check" uses +# Run with "make J=1 check" to disable parallelism in "make check". +# Run with "make J=200% check" to run two parallel jobs per core. +# The default is to run one job per core (J=100%). +# See "man parallel" for its "-j ..." option. +J ?= 100% + +# Use this regexp to select the subset of tests whose names match. +tests-regexp = . +EXCLUDE_TESTS_REGEX ?= "^$$" + +ifeq ($(PRINT_PARALLEL_OUTPUTS), 1) + parallel_redir = +else ifeq ($(QUIET_PARALLEL_TESTS), 1) + parallel_redir = >& t/$(test_log_prefix)log-{/} +else +# Default: print failure output only, as it happens +# Note: gnu_parallel --eta is now always used, but has been modified to provide +# only infrequent updates when not connected to a terminal. (CircleCI will +# kill a job if no output for 10min.) + parallel_redir = >& t/$(test_log_prefix)log-{/} || bash -c "cat t/$(test_log_prefix)log-{/}; exit $$?" +endif + +.PHONY: check_0 +check_0: + printf '%s\n' '' \ + 'To monitor subtest ,' \ + ' run "make watch-log" in a separate window' ''; \ + { \ + printf './%s\n' $(filter-out $(PARALLEL_TEST),$(TESTS)); \ + find t -name 'run-*' -print; \ + } \ + | $(prioritize_long_running_tests) \ + | grep -E '$(tests-regexp)' \ + | grep -E -v '$(EXCLUDE_TESTS_REGEX)' \ + | build_tools/gnu_parallel -j$(J) --plain --joblog=LOG --eta --gnu \ + --tmpdir=$(TEST_TMPDIR) '{} $(parallel_redir)' ; \ + parallel_retcode=$$? ; \ + awk '{ if ($$7 != 0 || $$8 != 0) { if ($$7 == "Exitval") { h = $$0; } else { if (!f) print h; print; f = 1 } } } END { if(f) exit 1; }' < LOG ; \ + awk_retcode=$$?; \ + if [ $$parallel_retcode -ne 0 ] || [ $$awk_retcode -ne 0 ] ; then exit 1 ; fi + +valgrind-exclude-regexp = InlineSkipTest.ConcurrentInsert|TransactionStressTest.DeadlockStress|DBCompactionTest.SuggestCompactRangeNoTwoLevel0Compactions|BackupableDBTest.RateLimiting|DBTest.CloseSpeedup|DBTest.ThreadStatusFlush|DBTest.RateLimitingTest|DBTest.EncodeDecompressedBlockSizeTest|FaultInjectionTest.UninstalledCompaction|HarnessTest.Randomized|ExternalSSTFileTest.CompactDuringAddFileRandom|ExternalSSTFileTest.IngestFileWithGlobalSeqnoRandomized|MySQLStyleTransactionTest.TransactionStressTest + +.PHONY: valgrind_check_0 +valgrind_check_0: test_log_prefix := valgrind_ +valgrind_check_0: + printf '%s\n' '' \ + 'To monitor subtest ,' \ + ' run "make watch-log" in a separate window' ''; \ + { \ + printf './%s\n' $(filter-out $(PARALLEL_TEST) %skiplist_test options_settable_test, $(TESTS)); \ + find t -name 'run-*' -print; \ + } \ + | $(prioritize_long_running_tests) \ + | grep -E '$(tests-regexp)' \ + | grep -E -v '$(valgrind-exclude-regexp)' \ + | build_tools/gnu_parallel -j$(J) --plain --joblog=LOG --eta --gnu \ + --tmpdir=$(TEST_TMPDIR) \ + '(if [[ "{}" == "./"* ]] ; then $(DRIVER) {}; else {}; fi) \ + $(parallel_redir)' \ + +CLEAN_FILES += t LOG $(TEST_TMPDIR) + +# When running parallel "make check", you can monitor its progress +# from another window. +# Run "make watch_LOG" to show the duration,PASS/FAIL,name of parallel +# tests as they are being run. We sort them so that longer-running ones +# appear at the top of the list and any failing tests remain at the top +# regardless of their duration. As with any use of "watch", hit ^C to +# interrupt. +watch-log: + $(WATCH) --interval=0 'sort -k7,7nr -k4,4gr LOG|$(quoted_perl_command)' + +dump-log: + bash -c '$(quoted_perl_command)' < LOG + +# If J != 1 and GNU parallel is installed, run the tests in parallel, +# via the check_0 rule above. Otherwise, run them sequentially. +check: all + $(MAKE) gen_parallel_tests + $(AM_V_GEN)if test "$(J)" != 1 \ + && (build_tools/gnu_parallel --gnu --help 2>/dev/null) | \ + grep -q 'GNU Parallel'; \ + then \ + $(MAKE) T="$$t" check_0; \ + else \ + for t in $(TESTS); do \ + echo "===== Running $$t (`date`)"; ./$$t || exit 1; done; \ + fi + rm -rf $(TEST_TMPDIR) +ifneq ($(PLATFORM), OS_AIX) + $(PYTHON) tools/check_all_python.py +ifndef ASSERT_STATUS_CHECKED # not yet working with these tests + $(PYTHON) tools/ldb_test.py + sh tools/rocksdb_dump_test.sh +endif +endif +ifndef SKIP_FORMAT_BUCK_CHECKS + $(MAKE) check-format + $(MAKE) check-buck-targets + $(MAKE) check-sources +endif + +# TODO add ldb_tests +check_some: $(ROCKSDBTESTS_SUBSET) + for t in $(ROCKSDBTESTS_SUBSET); do echo "===== Running $$t (`date`)"; ./$$t || exit 1; done + +.PHONY: ldb_tests +ldb_tests: ldb + $(PYTHON) tools/ldb_test.py + +include crash_test.mk + +asan_check: clean + COMPILE_WITH_ASAN=1 $(MAKE) check -j32 + $(MAKE) clean + +asan_crash_test: clean + COMPILE_WITH_ASAN=1 $(MAKE) crash_test + $(MAKE) clean + +whitebox_asan_crash_test: clean + COMPILE_WITH_ASAN=1 $(MAKE) whitebox_crash_test + $(MAKE) clean + +blackbox_asan_crash_test: clean + COMPILE_WITH_ASAN=1 $(MAKE) blackbox_crash_test + $(MAKE) clean + +asan_crash_test_with_atomic_flush: clean + COMPILE_WITH_ASAN=1 $(MAKE) crash_test_with_atomic_flush + $(MAKE) clean + +asan_crash_test_with_txn: clean + COMPILE_WITH_ASAN=1 $(MAKE) crash_test_with_txn + $(MAKE) clean + +asan_crash_test_with_best_efforts_recovery: clean + COMPILE_WITH_ASAN=1 $(MAKE) crash_test_with_best_efforts_recovery + $(MAKE) clean + +ubsan_check: clean + COMPILE_WITH_UBSAN=1 $(MAKE) check -j32 + $(MAKE) clean + +ubsan_crash_test: clean + COMPILE_WITH_UBSAN=1 $(MAKE) crash_test + $(MAKE) clean + +whitebox_ubsan_crash_test: clean + COMPILE_WITH_UBSAN=1 $(MAKE) whitebox_crash_test + $(MAKE) clean + +blackbox_ubsan_crash_test: clean + COMPILE_WITH_UBSAN=1 $(MAKE) blackbox_crash_test + $(MAKE) clean + +ubsan_crash_test_with_atomic_flush: clean + COMPILE_WITH_UBSAN=1 $(MAKE) crash_test_with_atomic_flush + $(MAKE) clean + +ubsan_crash_test_with_txn: clean + COMPILE_WITH_UBSAN=1 $(MAKE) crash_test_with_txn + $(MAKE) clean + +ubsan_crash_test_with_best_efforts_recovery: clean + COMPILE_WITH_UBSAN=1 $(MAKE) crash_test_with_best_efforts_recovery + $(MAKE) clean + +full_valgrind_test: + ROCKSDB_FULL_VALGRIND_RUN=1 DISABLE_JEMALLOC=1 $(MAKE) valgrind_check + +full_valgrind_test_some: + ROCKSDB_FULL_VALGRIND_RUN=1 DISABLE_JEMALLOC=1 $(MAKE) valgrind_check_some + +valgrind_test: + ROCKSDB_VALGRIND_RUN=1 DISABLE_JEMALLOC=1 $(MAKE) valgrind_check + +valgrind_test_some: + ROCKSDB_VALGRIND_RUN=1 DISABLE_JEMALLOC=1 $(MAKE) valgrind_check_some + +valgrind_check: $(TESTS) + $(MAKE) DRIVER="$(VALGRIND_VER) $(VALGRIND_OPTS)" gen_parallel_tests + $(AM_V_GEN)if test "$(J)" != 1 \ + && (build_tools/gnu_parallel --gnu --help 2>/dev/null) | \ + grep -q 'GNU Parallel'; \ + then \ + $(MAKE) \ + DRIVER="$(VALGRIND_VER) $(VALGRIND_OPTS)" valgrind_check_0; \ + else \ + for t in $(filter-out %skiplist_test options_settable_test,$(TESTS)); do \ + $(VALGRIND_VER) $(VALGRIND_OPTS) ./$$t; \ + ret_code=$$?; \ + if [ $$ret_code -ne 0 ]; then \ + exit $$ret_code; \ + fi; \ + done; \ + fi + +valgrind_check_some: $(ROCKSDBTESTS_SUBSET) + for t in $(ROCKSDBTESTS_SUBSET); do \ + $(VALGRIND_VER) $(VALGRIND_OPTS) ./$$t; \ + ret_code=$$?; \ + if [ $$ret_code -ne 0 ]; then \ + exit $$ret_code; \ + fi; \ + done + +test_names = \ + ./db_test --gtest_list_tests \ + | perl -n \ + -e 's/ *\#.*//;' \ + -e '/^(\s*)(\S+)/; !$$1 and do {$$p=$$2; break};' \ + -e 'print qq! $$p$$2!' + +analyze: clean + USE_CLANG=1 $(MAKE) analyze_incremental + +analyze_incremental: + $(CLANG_SCAN_BUILD) --use-analyzer=$(CLANG_ANALYZER) \ + --use-c++=$(CXX) --use-cc=$(CC) --status-bugs \ + -o $(CURDIR)/scan_build_report \ + $(MAKE) SKIP_LINK=1 dbg + +CLEAN_FILES += unity.cc +unity.cc: Makefile util/build_version.cc.in + rm -f $@ $@-t + $(AM_V_at)$(gen_build_version) > util/build_version.cc + for source_file in $(LIB_SOURCES); do \ + echo "#include \"$$source_file\"" >> $@-t; \ + done + chmod a=r $@-t + mv $@-t $@ + +unity.a: $(OBJ_DIR)/unity.o + $(AM_V_AR)rm -f $@ + $(AM_V_at)$(AR) $(ARFLAGS) $@ $(OBJ_DIR)/unity.o + + +# try compiling db_test with unity +unity_test: $(OBJ_DIR)/db/db_basic_test.o $(OBJ_DIR)/db/db_test_util.o $(TEST_OBJECTS) $(TOOL_OBJECTS) unity.a + $(AM_LINK) + ./unity_test + +rocksdb.h rocksdb.cc: build_tools/amalgamate.py Makefile $(LIB_SOURCES) unity.cc + build_tools/amalgamate.py -I. -i./include unity.cc -x include/rocksdb/c.h -H rocksdb.h -o rocksdb.cc + +clean: clean-ext-libraries-all clean-rocks clean-rocksjava + +clean-not-downloaded: clean-ext-libraries-bin clean-rocks clean-not-downloaded-rocksjava + +clean-rocks: +# Not practical to exactly match all versions/variants in naming (e.g. debug or not) + rm -f ${LIBNAME}*.so* ${LIBNAME}*.a + rm -f $(BENCHMARKS) $(TOOLS) $(TESTS) $(PARALLEL_TEST) $(MICROBENCHS) + rm -rf $(CLEAN_FILES) ios-x86 ios-arm scan_build_report + $(FIND) . -name "*.[oda]" -exec rm -f {} \; + $(FIND) . -type f \( -name "*.gcda" -o -name "*.gcno" \) -exec rm -f {} \; + +clean-rocksjava: clean-rocks + rm -rf jl jls + cd java && $(MAKE) clean + +clean-not-downloaded-rocksjava: + cd java && $(MAKE) clean-not-downloaded + +clean-ext-libraries-all: + rm -rf bzip2* snappy* zlib* lz4* zstd* + +clean-ext-libraries-bin: + find . -maxdepth 1 -type d \( -name bzip2\* -or -name snappy\* -or -name zlib\* -or -name lz4\* -or -name zstd\* \) -prune -exec rm -rf {} \; + +tags: + ctags -R . + cscope -b `$(FIND) . -name '*.cc'` `$(FIND) . -name '*.h'` `$(FIND) . -name '*.c'` + ctags -e -R -o etags * + +tags0: + ctags -R . + cscope -b `$(FIND) . -name '*.cc' -and ! -name '*_test.cc'` \ + `$(FIND) . -name '*.c' -and ! -name '*_test.c'` \ + `$(FIND) . -name '*.h' -and ! -name '*_test.h'` + ctags -e -R -o etags * + +format: + build_tools/format-diff.sh + +check-format: + build_tools/format-diff.sh -c + +check-buck-targets: + buckifier/check_buck_targets.sh + +check-sources: + build_tools/check-sources.sh + +package: + bash build_tools/make_package.sh $(SHARED_MAJOR).$(SHARED_MINOR) + +# --------------------------------------------------------------------------- +# Unit tests and tools +# --------------------------------------------------------------------------- +$(STATIC_LIBRARY): $(LIB_OBJECTS) + $(AM_V_AR)rm -f $@ $(SHARED1) $(SHARED2) $(SHARED3) $(SHARED4) + $(AM_V_at)$(AR) $(ARFLAGS) $@ $(LIB_OBJECTS) + +$(STATIC_TEST_LIBRARY): $(TEST_OBJECTS) + $(AM_V_AR)rm -f $@ $(SHARED_TEST_LIBRARY) + $(AM_V_at)$(AR) $(ARFLAGS) $@ $^ + +$(STATIC_TOOLS_LIBRARY): $(TOOL_OBJECTS) + $(AM_V_AR)rm -f $@ $(SHARED_TOOLS_LIBRARY) + $(AM_V_at)$(AR) $(ARFLAGS) $@ $^ + +$(STATIC_STRESS_LIBRARY): $(ANALYZE_OBJECTS) $(STRESS_OBJECTS) $(TESTUTIL) + $(AM_V_AR)rm -f $@ $(SHARED_STRESS_LIBRARY) + $(AM_V_at)$(AR) $(ARFLAGS) $@ $^ + +$(SHARED_TEST_LIBRARY): $(TEST_OBJECTS) $(SHARED1) + $(AM_V_AR)rm -f $@ $(STATIC_TEST_LIBRARY) + $(AM_SHARE) + +$(SHARED_TOOLS_LIBRARY): $(TOOL_OBJECTS) $(SHARED1) + $(AM_V_AR)rm -f $@ $(STATIC_TOOLS_LIBRARY) + $(AM_SHARE) + +$(SHARED_STRESS_LIBRARY): $(ANALYZE_OBJECTS) $(STRESS_OBJECTS) $(TESTUTIL) $(SHARED_TOOLS_LIBRARY) $(SHARED1) + $(AM_V_AR)rm -f $@ $(STATIC_STRESS_LIBRARY) + $(AM_SHARE) + +librocksdb_env_basic_test.a: $(OBJ_DIR)/env/env_basic_test.o $(LIB_OBJECTS) $(TESTHARNESS) + $(AM_V_AR)rm -f $@ + $(AM_V_at)$(AR) $(ARFLAGS) $@ $^ + +db_bench: $(OBJ_DIR)/tools/db_bench.o $(BENCH_OBJECTS) $(TESTUTIL) $(LIBRARY) + $(AM_LINK) + +trace_analyzer: $(OBJ_DIR)/tools/trace_analyzer.o $(ANALYZE_OBJECTS) $(TOOLS_LIBRARY) $(LIBRARY) + $(AM_LINK) + +block_cache_trace_analyzer: $(OBJ_DIR)/tools/block_cache_analyzer/block_cache_trace_analyzer_tool.o $(ANALYZE_OBJECTS) $(TOOLS_LIBRARY) $(LIBRARY) + $(AM_LINK) + +cache_bench: $(OBJ_DIR)/cache/cache_bench.o $(CACHE_BENCH_OBJECTS) $(LIBRARY) + $(AM_LINK) + +persistent_cache_bench: $(OBJ_DIR)/utilities/persistent_cache/persistent_cache_bench.o $(LIBRARY) + $(AM_LINK) + +memtablerep_bench: $(OBJ_DIR)/memtable/memtablerep_bench.o $(LIBRARY) + $(AM_LINK) + +filter_bench: $(OBJ_DIR)/util/filter_bench.o $(LIBRARY) + $(AM_LINK) + +db_stress: $(OBJ_DIR)/db_stress_tool/db_stress.o $(STRESS_LIBRARY) $(TOOLS_LIBRARY) $(LIBRARY) + $(AM_LINK) + +write_stress: $(OBJ_DIR)/tools/write_stress.o $(LIBRARY) + $(AM_LINK) + +db_sanity_test: $(OBJ_DIR)/tools/db_sanity_test.o $(LIBRARY) + $(AM_LINK) + +db_repl_stress: $(OBJ_DIR)/tools/db_repl_stress.o $(LIBRARY) + $(AM_LINK) + +define MakeTestRule +$(notdir $(1:%.cc=%)): $(1:%.cc=$$(OBJ_DIR)/%.o) $$(TEST_LIBRARY) $$(LIBRARY) + $$(AM_LINK) +endef + +# For each PLUGIN test, create a rule to generate the test executable +$(foreach test, $(ROCKSDB_PLUGIN_TESTS), $(eval $(call MakeTestRule, $(test)))) + +arena_test: $(OBJ_DIR)/memory/arena_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +memory_allocator_test: memory/memory_allocator_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +autovector_test: $(OBJ_DIR)/util/autovector_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +column_family_test: $(OBJ_DIR)/db/column_family_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +table_properties_collector_test: $(OBJ_DIR)/db/table_properties_collector_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +bloom_test: $(OBJ_DIR)/util/bloom_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +dynamic_bloom_test: $(OBJ_DIR)/util/dynamic_bloom_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +c_test: $(OBJ_DIR)/db/c_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +cache_test: $(OBJ_DIR)/cache/cache_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +coding_test: $(OBJ_DIR)/util/coding_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +hash_test: $(OBJ_DIR)/util/hash_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +random_test: $(OBJ_DIR)/util/random_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +ribbon_test: $(OBJ_DIR)/util/ribbon_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +option_change_migration_test: $(OBJ_DIR)/utilities/option_change_migration/option_change_migration_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +agg_merge_test: $(OBJ_DIR)/utilities/agg_merge/agg_merge_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +stringappend_test: $(OBJ_DIR)/utilities/merge_operators/string_append/stringappend_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +cassandra_format_test: $(OBJ_DIR)/utilities/cassandra/cassandra_format_test.o $(OBJ_DIR)/utilities/cassandra/test_utils.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +cassandra_functional_test: $(OBJ_DIR)/utilities/cassandra/cassandra_functional_test.o $(OBJ_DIR)/utilities/cassandra/test_utils.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +cassandra_row_merge_test: $(OBJ_DIR)/utilities/cassandra/cassandra_row_merge_test.o $(OBJ_DIR)/utilities/cassandra/test_utils.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +cassandra_serialize_test: $(OBJ_DIR)/utilities/cassandra/cassandra_serialize_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +hash_table_test: $(OBJ_DIR)/utilities/persistent_cache/hash_table_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +histogram_test: $(OBJ_DIR)/monitoring/histogram_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +thread_local_test: $(OBJ_DIR)/util/thread_local_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +work_queue_test: $(OBJ_DIR)/util/work_queue_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +udt_util_test: $(OBJ_DIR)/util/udt_util_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +corruption_test: $(OBJ_DIR)/db/corruption_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +crc32c_test: $(OBJ_DIR)/util/crc32c_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +slice_test: $(OBJ_DIR)/util/slice_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +slice_transform_test: $(OBJ_DIR)/util/slice_transform_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_basic_test: $(OBJ_DIR)/db/db_basic_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_blob_basic_test: $(OBJ_DIR)/db/blob/db_blob_basic_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_blob_compaction_test: $(OBJ_DIR)/db/blob/db_blob_compaction_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_readonly_with_timestamp_test: $(OBJ_DIR)/db/db_readonly_with_timestamp_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_wide_basic_test: $(OBJ_DIR)/db/wide/db_wide_basic_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_with_timestamp_basic_test: $(OBJ_DIR)/db/db_with_timestamp_basic_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_with_timestamp_compaction_test: db/db_with_timestamp_compaction_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_encryption_test: $(OBJ_DIR)/db/db_encryption_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_test: $(OBJ_DIR)/db/db_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_test2: $(OBJ_DIR)/db/db_test2.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_logical_block_size_cache_test: $(OBJ_DIR)/db/db_logical_block_size_cache_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_blob_index_test: $(OBJ_DIR)/db/blob/db_blob_index_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_block_cache_test: $(OBJ_DIR)/db/db_block_cache_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_bloom_filter_test: $(OBJ_DIR)/db/db_bloom_filter_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_log_iter_test: $(OBJ_DIR)/db/db_log_iter_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_compaction_filter_test: $(OBJ_DIR)/db/db_compaction_filter_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_compaction_test: $(OBJ_DIR)/db/db_compaction_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_clip_test: $(OBJ_DIR)/db/db_clip_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_dynamic_level_test: $(OBJ_DIR)/db/db_dynamic_level_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_flush_test: $(OBJ_DIR)/db/db_flush_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_inplace_update_test: $(OBJ_DIR)/db/db_inplace_update_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_iterator_test: $(OBJ_DIR)/db/db_iterator_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_kv_checksum_test: $(OBJ_DIR)/db/db_kv_checksum_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_memtable_test: $(OBJ_DIR)/db/db_memtable_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_merge_operator_test: $(OBJ_DIR)/db/db_merge_operator_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_merge_operand_test: $(OBJ_DIR)/db/db_merge_operand_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_options_test: $(OBJ_DIR)/db/db_options_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_range_del_test: $(OBJ_DIR)/db/db_range_del_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_rate_limiter_test: $(OBJ_DIR)/db/db_rate_limiter_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_sst_test: $(OBJ_DIR)/db/db_sst_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_statistics_test: $(OBJ_DIR)/db/db_statistics_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_write_test: $(OBJ_DIR)/db/db_write_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +error_handler_fs_test: $(OBJ_DIR)/db/error_handler_fs_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +external_sst_file_basic_test: $(OBJ_DIR)/db/external_sst_file_basic_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +external_sst_file_test: $(OBJ_DIR)/db/external_sst_file_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +import_column_family_test: $(OBJ_DIR)/db/import_column_family_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_tailing_iter_test: $(OBJ_DIR)/db/db_tailing_iter_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_iter_test: $(OBJ_DIR)/db/db_iter_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_iter_stress_test: $(OBJ_DIR)/db/db_iter_stress_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_universal_compaction_test: $(OBJ_DIR)/db/db_universal_compaction_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_wal_test: $(OBJ_DIR)/db/db_wal_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_io_failure_test: $(OBJ_DIR)/db/db_io_failure_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_properties_test: $(OBJ_DIR)/db/db_properties_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_table_properties_test: $(OBJ_DIR)/db/db_table_properties_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +log_write_bench: $(OBJ_DIR)/util/log_write_bench.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) $(PROFILING_FLAGS) + +seqno_time_test: $(OBJ_DIR)/db/seqno_time_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +plain_table_db_test: $(OBJ_DIR)/db/plain_table_db_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +comparator_db_test: $(OBJ_DIR)/db/comparator_db_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +table_reader_bench: $(OBJ_DIR)/table/table_reader_bench.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) $(PROFILING_FLAGS) + +perf_context_test: $(OBJ_DIR)/db/perf_context_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +prefix_test: $(OBJ_DIR)/db/prefix_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +backup_engine_test: $(OBJ_DIR)/utilities/backup/backup_engine_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +checkpoint_test: $(OBJ_DIR)/utilities/checkpoint/checkpoint_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +cache_simulator_test: $(OBJ_DIR)/utilities/simulator_cache/cache_simulator_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +sim_cache_test: $(OBJ_DIR)/utilities/simulator_cache/sim_cache_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +env_mirror_test: $(OBJ_DIR)/utilities/env_mirror_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +env_timed_test: $(OBJ_DIR)/utilities/env_timed_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +object_registry_test: $(OBJ_DIR)/utilities/object_registry_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +ttl_test: $(OBJ_DIR)/utilities/ttl/ttl_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +write_batch_with_index_test: $(OBJ_DIR)/utilities/write_batch_with_index/write_batch_with_index_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +flush_job_test: $(OBJ_DIR)/db/flush_job_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +compaction_iterator_test: $(OBJ_DIR)/db/compaction/compaction_iterator_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +compaction_job_test: $(OBJ_DIR)/db/compaction/compaction_job_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +compaction_job_stats_test: $(OBJ_DIR)/db/compaction/compaction_job_stats_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +compaction_service_test: $(OBJ_DIR)/db/compaction/compaction_service_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +compact_on_deletion_collector_test: $(OBJ_DIR)/utilities/table_properties_collectors/compact_on_deletion_collector_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +wal_manager_test: $(OBJ_DIR)/db/wal_manager_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +wal_edit_test: $(OBJ_DIR)/db/wal_edit_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +dbformat_test: $(OBJ_DIR)/db/dbformat_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +env_basic_test: $(OBJ_DIR)/env/env_basic_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +env_test: $(OBJ_DIR)/env/env_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +io_posix_test: $(OBJ_DIR)/env/io_posix_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +fault_injection_test: $(OBJ_DIR)/db/fault_injection_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +rate_limiter_test: $(OBJ_DIR)/util/rate_limiter_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +delete_scheduler_test: $(OBJ_DIR)/file/delete_scheduler_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +filename_test: $(OBJ_DIR)/db/filename_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +random_access_file_reader_test: $(OBJ_DIR)/file/random_access_file_reader_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +file_reader_writer_test: $(OBJ_DIR)/util/file_reader_writer_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +block_based_table_reader_test: table/block_based/block_based_table_reader_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +full_filter_block_test: $(OBJ_DIR)/table/block_based/full_filter_block_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +partitioned_filter_block_test: $(OBJ_DIR)/table/block_based/partitioned_filter_block_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +log_test: $(OBJ_DIR)/db/log_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +cleanable_test: $(OBJ_DIR)/table/cleanable_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +table_test: $(OBJ_DIR)/table/table_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +block_fetcher_test: table/block_fetcher_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +block_test: $(OBJ_DIR)/table/block_based/block_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +data_block_hash_index_test: $(OBJ_DIR)/table/block_based/data_block_hash_index_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +inlineskiplist_test: $(OBJ_DIR)/memtable/inlineskiplist_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +skiplist_test: $(OBJ_DIR)/memtable/skiplist_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +write_buffer_manager_test: $(OBJ_DIR)/memtable/write_buffer_manager_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +version_edit_test: $(OBJ_DIR)/db/version_edit_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +version_set_test: $(OBJ_DIR)/db/version_set_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +compaction_picker_test: $(OBJ_DIR)/db/compaction/compaction_picker_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +version_builder_test: $(OBJ_DIR)/db/version_builder_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +file_indexer_test: $(OBJ_DIR)/db/file_indexer_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +reduce_levels_test: $(OBJ_DIR)/tools/reduce_levels_test.o $(TOOLS_LIBRARY) $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +write_batch_test: $(OBJ_DIR)/db/write_batch_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +write_controller_test: $(OBJ_DIR)/db/write_controller_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +merge_helper_test: $(OBJ_DIR)/db/merge_helper_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +memory_test: $(OBJ_DIR)/utilities/memory/memory_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +merge_test: $(OBJ_DIR)/db/merge_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +merger_test: $(OBJ_DIR)/table/merger_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +util_merge_operators_test: $(OBJ_DIR)/utilities/util_merge_operators_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +options_file_test: $(OBJ_DIR)/db/options_file_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +deletefile_test: $(OBJ_DIR)/db/deletefile_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +obsolete_files_test: $(OBJ_DIR)/db/obsolete_files_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +rocksdb_dump: $(OBJ_DIR)/tools/dump/rocksdb_dump.o $(LIBRARY) + $(AM_LINK) + +rocksdb_undump: $(OBJ_DIR)/tools/dump/rocksdb_undump.o $(LIBRARY) + $(AM_LINK) + +cuckoo_table_builder_test: $(OBJ_DIR)/table/cuckoo/cuckoo_table_builder_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +cuckoo_table_reader_test: $(OBJ_DIR)/table/cuckoo/cuckoo_table_reader_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +cuckoo_table_db_test: $(OBJ_DIR)/db/cuckoo_table_db_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +listener_test: $(OBJ_DIR)/db/listener_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +thread_list_test: $(OBJ_DIR)/util/thread_list_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +compact_files_test: $(OBJ_DIR)/db/compact_files_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +configurable_test: options/configurable_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +customizable_test: options/customizable_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +options_test: $(OBJ_DIR)/options/options_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +options_settable_test: $(OBJ_DIR)/options/options_settable_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +options_util_test: $(OBJ_DIR)/utilities/options/options_util_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_bench_tool_test: $(OBJ_DIR)/tools/db_bench_tool_test.o $(BENCH_OBJECTS) $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +trace_analyzer_test: $(OBJ_DIR)/tools/trace_analyzer_test.o $(ANALYZE_OBJECTS) $(TOOLS_LIBRARY) $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +event_logger_test: $(OBJ_DIR)/logging/event_logger_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +timer_queue_test: $(OBJ_DIR)/util/timer_queue_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +sst_dump_test: $(OBJ_DIR)/tools/sst_dump_test.o $(TOOLS_LIBRARY) $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +optimistic_transaction_test: $(OBJ_DIR)/utilities/transactions/optimistic_transaction_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +mock_env_test : $(OBJ_DIR)/env/mock_env_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +manual_compaction_test: $(OBJ_DIR)/db/manual_compaction_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +filelock_test: $(OBJ_DIR)/util/filelock_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +auto_roll_logger_test: $(OBJ_DIR)/logging/auto_roll_logger_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +env_logger_test: $(OBJ_DIR)/logging/env_logger_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +memtable_list_test: $(OBJ_DIR)/db/memtable_list_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +write_callback_test: $(OBJ_DIR)/db/write_callback_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +heap_test: $(OBJ_DIR)/util/heap_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +point_lock_manager_test: utilities/transactions/lock/point/point_lock_manager_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +transaction_test: $(OBJ_DIR)/utilities/transactions/transaction_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +write_committed_transaction_ts_test: $(OBJ_DIR)/utilities/transactions/write_committed_transaction_ts_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +write_prepared_transaction_test: $(OBJ_DIR)/utilities/transactions/write_prepared_transaction_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +write_unprepared_transaction_test: $(OBJ_DIR)/utilities/transactions/write_unprepared_transaction_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +timestamped_snapshot_test: $(OBJ_DIR)/utilities/transactions/timestamped_snapshot_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +tiered_compaction_test: $(OBJ_DIR)/db/compaction/tiered_compaction_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +sst_dump: $(OBJ_DIR)/tools/sst_dump.o $(TOOLS_LIBRARY) $(LIBRARY) + $(AM_LINK) + +blob_dump: $(OBJ_DIR)/tools/blob_dump.o $(TOOLS_LIBRARY) $(LIBRARY) + $(AM_LINK) + +repair_test: $(OBJ_DIR)/db/repair_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +ldb_cmd_test: $(OBJ_DIR)/tools/ldb_cmd_test.o $(TOOLS_LIBRARY) $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +ldb: $(OBJ_DIR)/tools/ldb.o $(TOOLS_LIBRARY) $(LIBRARY) + $(AM_LINK) + +iostats_context_test: $(OBJ_DIR)/monitoring/iostats_context_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_V_CCLD)$(CXX) $^ $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) + +persistent_cache_test: $(OBJ_DIR)/utilities/persistent_cache/persistent_cache_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +statistics_test: $(OBJ_DIR)/monitoring/statistics_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +stats_history_test: $(OBJ_DIR)/monitoring/stats_history_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +compressed_secondary_cache_test: $(OBJ_DIR)/cache/compressed_secondary_cache_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +lru_cache_test: $(OBJ_DIR)/cache/lru_cache_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +range_del_aggregator_test: $(OBJ_DIR)/db/range_del_aggregator_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +range_del_aggregator_bench: $(OBJ_DIR)/db/range_del_aggregator_bench.o $(LIBRARY) + $(AM_LINK) + +blob_db_test: $(OBJ_DIR)/utilities/blob_db/blob_db_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +repeatable_thread_test: $(OBJ_DIR)/util/repeatable_thread_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +range_locking_test: utilities/transactions/lock/range/range_locking_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +range_tombstone_fragmenter_test: $(OBJ_DIR)/db/range_tombstone_fragmenter_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +sst_file_reader_test: $(OBJ_DIR)/table/sst_file_reader_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_secondary_test: $(OBJ_DIR)/db/db_secondary_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +block_cache_tracer_test: $(OBJ_DIR)/trace_replay/block_cache_tracer_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +block_cache_trace_analyzer_test: $(OBJ_DIR)/tools/block_cache_analyzer/block_cache_trace_analyzer_test.o $(OBJ_DIR)/tools/block_cache_analyzer/block_cache_trace_analyzer.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +defer_test: $(OBJ_DIR)/util/defer_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +blob_counting_iterator_test: $(OBJ_DIR)/db/blob/blob_counting_iterator_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +blob_file_addition_test: $(OBJ_DIR)/db/blob/blob_file_addition_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +blob_file_builder_test: $(OBJ_DIR)/db/blob/blob_file_builder_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +blob_file_cache_test: $(OBJ_DIR)/db/blob/blob_file_cache_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +blob_file_garbage_test: $(OBJ_DIR)/db/blob/blob_file_garbage_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +blob_file_reader_test: $(OBJ_DIR)/db/blob/blob_file_reader_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +blob_source_test: $(OBJ_DIR)/db/blob/blob_source_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +blob_garbage_meter_test: $(OBJ_DIR)/db/blob/blob_garbage_meter_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +timer_test: $(OBJ_DIR)/util/timer_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +periodic_task_scheduler_test: $(OBJ_DIR)/db/periodic_task_scheduler_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +testutil_test: $(OBJ_DIR)/test_util/testutil_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +io_tracer_test: $(OBJ_DIR)/trace_replay/io_tracer_test.o $(OBJ_DIR)/trace_replay/io_tracer.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +prefetch_test: $(OBJ_DIR)/file/prefetch_test.o $(OBJ_DIR)/tools/io_tracer_parser_tool.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +io_tracer_parser_test: $(OBJ_DIR)/tools/io_tracer_parser_test.o $(OBJ_DIR)/tools/io_tracer_parser_tool.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +io_tracer_parser: $(OBJ_DIR)/tools/io_tracer_parser.o $(TOOLS_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_blob_corruption_test: $(OBJ_DIR)/db/blob/db_blob_corruption_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +db_write_buffer_manager_test: $(OBJ_DIR)/db/db_write_buffer_manager_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +clipping_iterator_test: $(OBJ_DIR)/db/compaction/clipping_iterator_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +ribbon_bench: $(OBJ_DIR)/microbench/ribbon_bench.o $(LIBRARY) + $(AM_LINK) + +db_basic_bench: $(OBJ_DIR)/microbench/db_basic_bench.o $(LIBRARY) + $(AM_LINK) + +cache_reservation_manager_test: $(OBJ_DIR)/cache/cache_reservation_manager_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +wide_column_serialization_test: $(OBJ_DIR)/db/wide/wide_column_serialization_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +#------------------------------------------------- +# make install related stuff +PREFIX ?= /usr/local +LIBDIR ?= $(PREFIX)/lib +INSTALL_LIBDIR = $(DESTDIR)$(LIBDIR) + +uninstall: + rm -rf $(DESTDIR)$(PREFIX)/include/rocksdb \ + $(INSTALL_LIBDIR)/$(LIBRARY) \ + $(INSTALL_LIBDIR)/$(SHARED4) \ + $(INSTALL_LIBDIR)/$(SHARED3) \ + $(INSTALL_LIBDIR)/$(SHARED2) \ + $(INSTALL_LIBDIR)/$(SHARED1) \ + $(INSTALL_LIBDIR)/pkgconfig/rocksdb.pc + +install-headers: gen-pc + install -d $(INSTALL_LIBDIR) + install -d $(INSTALL_LIBDIR)/pkgconfig + for header_dir in `$(FIND) "include/rocksdb" -type d`; do \ + install -d $(DESTDIR)/$(PREFIX)/$$header_dir; \ + done + for header in `$(FIND) "include/rocksdb" -type f -name *.h`; do \ + install -C -m 644 $$header $(DESTDIR)/$(PREFIX)/$$header; \ + done + for header in $(ROCKSDB_PLUGIN_HEADERS); do \ + install -d $(DESTDIR)/$(PREFIX)/include/rocksdb/`dirname $$header`; \ + install -C -m 644 $$header $(DESTDIR)/$(PREFIX)/include/rocksdb/$$header; \ + done + install -C -m 644 rocksdb.pc $(INSTALL_LIBDIR)/pkgconfig/rocksdb.pc + +install-static: install-headers $(LIBRARY) + install -d $(INSTALL_LIBDIR) + install -C -m 755 $(LIBRARY) $(INSTALL_LIBDIR) + +install-shared: install-headers $(SHARED4) + install -d $(INSTALL_LIBDIR) + install -C -m 755 $(SHARED4) $(INSTALL_LIBDIR) + ln -fs $(SHARED4) $(INSTALL_LIBDIR)/$(SHARED3) + ln -fs $(SHARED4) $(INSTALL_LIBDIR)/$(SHARED2) + ln -fs $(SHARED4) $(INSTALL_LIBDIR)/$(SHARED1) + +# install static by default + install shared if it exists +install: install-static + [ -e $(SHARED4) ] && $(MAKE) install-shared || : + +# Generate the pkg-config file +gen-pc: + -echo 'prefix=$(PREFIX)' > rocksdb.pc + -echo 'exec_prefix=$${prefix}' >> rocksdb.pc + -echo 'includedir=$${prefix}/include' >> rocksdb.pc + -echo 'libdir=$(LIBDIR)' >> rocksdb.pc + -echo '' >> rocksdb.pc + -echo 'Name: rocksdb' >> rocksdb.pc + -echo 'Description: An embeddable persistent key-value store for fast storage' >> rocksdb.pc + -echo Version: $(shell ./build_tools/version.sh full) >> rocksdb.pc + -echo 'Libs: -L$${libdir} $(EXEC_LDFLAGS) -lrocksdb' >> rocksdb.pc + -echo 'Libs.private: $(PLATFORM_LDFLAGS)' >> rocksdb.pc + -echo 'Cflags: -I$${includedir} $(PLATFORM_CXXFLAGS)' >> rocksdb.pc + -echo 'Requires: $(subst ",,$(ROCKSDB_PLUGIN_PKGCONFIG_REQUIRES))' >> rocksdb.pc + +#------------------------------------------------- + + +# --------------------------------------------------------------------------- +# Jni stuff +# --------------------------------------------------------------------------- +JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/linux +ifeq ($(PLATFORM), OS_SOLARIS) + ARCH := $(shell isainfo -b) +else ifeq ($(PLATFORM), OS_OPENBSD) + ifneq (,$(filter amd64 ppc64 ppc64le s390x arm64 aarch64 sparc64 loongarch64, $(MACHINE))) + ARCH := 64 + else + ARCH := 32 + endif +else + ARCH := $(shell getconf LONG_BIT) +endif + +ifeq ($(shell ldd /usr/bin/env 2>/dev/null | grep -q musl; echo $$?),0) + JNI_LIBC = musl +# GNU LibC (or glibc) is so pervasive we can assume it is the default +# else +# JNI_LIBC = glibc +endif + +ifneq ($(origin JNI_LIBC), undefined) + JNI_LIBC_POSTFIX = -$(JNI_LIBC) +endif + +ifeq (,$(ROCKSDBJNILIB)) +ifneq (,$(filter ppc% s390x arm64 aarch64 sparc64 loongarch64, $(MACHINE))) + ROCKSDBJNILIB = librocksdbjni-linux-$(MACHINE)$(JNI_LIBC_POSTFIX).so +else + ROCKSDBJNILIB = librocksdbjni-linux$(ARCH)$(JNI_LIBC_POSTFIX).so +endif +endif +ROCKSDB_JAVA_VERSION ?= $(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH) +ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-linux$(ARCH)$(JNI_LIBC_POSTFIX).jar +ROCKSDB_JAR_ALL = rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar +ROCKSDB_JAVADOCS_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-javadoc.jar +ROCKSDB_SOURCES_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-sources.jar +SHA256_CMD = sha256sum + +ZLIB_VER ?= 1.2.13 +ZLIB_SHA256 ?= b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30 +ZLIB_DOWNLOAD_BASE ?= http://zlib.net +BZIP2_VER ?= 1.0.8 +BZIP2_SHA256 ?= ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269 +BZIP2_DOWNLOAD_BASE ?= http://sourceware.org/pub/bzip2 +SNAPPY_VER ?= 1.1.8 +SNAPPY_SHA256 ?= 16b677f07832a612b0836178db7f374e414f94657c138e6993cbfc5dcc58651f +SNAPPY_DOWNLOAD_BASE ?= https://github.com/google/snappy/archive +LZ4_VER ?= 1.9.3 +LZ4_SHA256 ?= 030644df4611007ff7dc962d981f390361e6c97a34e5cbc393ddfbe019ffe2c1 +LZ4_DOWNLOAD_BASE ?= https://github.com/lz4/lz4/archive +ZSTD_VER ?= 1.4.9 +ZSTD_SHA256 ?= acf714d98e3db7b876e5b540cbf6dee298f60eb3c0723104f6d3f065cd60d6a8 +ZSTD_DOWNLOAD_BASE ?= https://github.com/facebook/zstd/archive +CURL_SSL_OPTS ?= --tlsv1 + +ifeq ($(PLATFORM), OS_MACOSX) +ifeq (,$(findstring librocksdbjni-osx,$(ROCKSDBJNILIB))) +ifeq ($(MACHINE),arm64) + ROCKSDBJNILIB = librocksdbjni-osx-arm64.jnilib +else ifeq ($(MACHINE),x86_64) + ROCKSDBJNILIB = librocksdbjni-osx-x86_64.jnilib +else + ROCKSDBJNILIB = librocksdbjni-osx.jnilib +endif +endif + ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-osx.jar + SHA256_CMD = openssl sha256 -r +ifneq ("$(wildcard $(JAVA_HOME)/include/darwin)","") + JAVA_INCLUDE = -I$(JAVA_HOME)/include -I $(JAVA_HOME)/include/darwin +else + JAVA_INCLUDE = -I/System/Library/Frameworks/JavaVM.framework/Headers/ +endif +endif + +ifeq ($(PLATFORM), OS_FREEBSD) + JAVA_INCLUDE = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/freebsd + ROCKSDBJNILIB = librocksdbjni-freebsd$(ARCH).so + ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-freebsd$(ARCH).jar +endif +ifeq ($(PLATFORM), OS_SOLARIS) + ROCKSDBJNILIB = librocksdbjni-solaris$(ARCH).so + ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-solaris$(ARCH).jar + JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/solaris + SHA256_CMD = digest -a sha256 +endif +ifeq ($(PLATFORM), OS_AIX) + JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/aix + ROCKSDBJNILIB = librocksdbjni-aix.so + EXTRACT_SOURCES = gunzip < TAR_GZ | tar xvf - + SNAPPY_MAKE_TARGET = libsnappy.la +endif +ifeq ($(PLATFORM), OS_OPENBSD) + JAVA_INCLUDE = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/openbsd + ROCKSDBJNILIB = librocksdbjni-openbsd$(ARCH).so + ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-openbsd$(ARCH).jar +endif +export SHA256_CMD + +zlib-$(ZLIB_VER).tar.gz: + curl --fail --output zlib-$(ZLIB_VER).tar.gz --location ${ZLIB_DOWNLOAD_BASE}/zlib-$(ZLIB_VER).tar.gz + ZLIB_SHA256_ACTUAL=`$(SHA256_CMD) zlib-$(ZLIB_VER).tar.gz | cut -d ' ' -f 1`; \ + if [ "$(ZLIB_SHA256)" != "$$ZLIB_SHA256_ACTUAL" ]; then \ + echo zlib-$(ZLIB_VER).tar.gz checksum mismatch, expected=\"$(ZLIB_SHA256)\" actual=\"$$ZLIB_SHA256_ACTUAL\"; \ + exit 1; \ + fi + +libz.a: zlib-$(ZLIB_VER).tar.gz + -rm -rf zlib-$(ZLIB_VER) + tar xvzf zlib-$(ZLIB_VER).tar.gz + if [ -n"$(ARCHFLAG)" ]; then \ + cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' ./configure --static --archs="$(ARCHFLAG)" && $(MAKE); \ + else \ + cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' ./configure --static && $(MAKE); \ + fi + cp zlib-$(ZLIB_VER)/libz.a . + +bzip2-$(BZIP2_VER).tar.gz: + curl --fail --output bzip2-$(BZIP2_VER).tar.gz --location ${CURL_SSL_OPTS} ${BZIP2_DOWNLOAD_BASE}/bzip2-$(BZIP2_VER).tar.gz + BZIP2_SHA256_ACTUAL=`$(SHA256_CMD) bzip2-$(BZIP2_VER).tar.gz | cut -d ' ' -f 1`; \ + if [ "$(BZIP2_SHA256)" != "$$BZIP2_SHA256_ACTUAL" ]; then \ + echo bzip2-$(BZIP2_VER).tar.gz checksum mismatch, expected=\"$(BZIP2_SHA256)\" actual=\"$$BZIP2_SHA256_ACTUAL\"; \ + exit 1; \ + fi + +libbz2.a: bzip2-$(BZIP2_VER).tar.gz + -rm -rf bzip2-$(BZIP2_VER) + tar xvzf bzip2-$(BZIP2_VER).tar.gz + cd bzip2-$(BZIP2_VER) && $(MAKE) CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64 $(ARCHFLAG) ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' AR='ar ${EXTRA_ARFLAGS}' libbz2.a + cp bzip2-$(BZIP2_VER)/libbz2.a . + +snappy-$(SNAPPY_VER).tar.gz: + curl --fail --output snappy-$(SNAPPY_VER).tar.gz --location ${CURL_SSL_OPTS} ${SNAPPY_DOWNLOAD_BASE}/$(SNAPPY_VER).tar.gz + SNAPPY_SHA256_ACTUAL=`$(SHA256_CMD) snappy-$(SNAPPY_VER).tar.gz | cut -d ' ' -f 1`; \ + if [ "$(SNAPPY_SHA256)" != "$$SNAPPY_SHA256_ACTUAL" ]; then \ + echo snappy-$(SNAPPY_VER).tar.gz checksum mismatch, expected=\"$(SNAPPY_SHA256)\" actual=\"$$SNAPPY_SHA256_ACTUAL\"; \ + exit 1; \ + fi + +libsnappy.a: snappy-$(SNAPPY_VER).tar.gz + -rm -rf snappy-$(SNAPPY_VER) + tar xvzf snappy-$(SNAPPY_VER).tar.gz + mkdir snappy-$(SNAPPY_VER)/build + cd snappy-$(SNAPPY_VER)/build && CFLAGS='$(ARCHFLAG) ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' CXXFLAGS='$(ARCHFLAG) ${JAVA_STATIC_DEPS_CXXFLAGS} ${EXTRA_CXXFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON ${PLATFORM_CMAKE_FLAGS} .. && $(MAKE) ${SNAPPY_MAKE_TARGET} + cp snappy-$(SNAPPY_VER)/build/libsnappy.a . + +lz4-$(LZ4_VER).tar.gz: + curl --fail --output lz4-$(LZ4_VER).tar.gz --location ${CURL_SSL_OPTS} ${LZ4_DOWNLOAD_BASE}/v$(LZ4_VER).tar.gz + LZ4_SHA256_ACTUAL=`$(SHA256_CMD) lz4-$(LZ4_VER).tar.gz | cut -d ' ' -f 1`; \ + if [ "$(LZ4_SHA256)" != "$$LZ4_SHA256_ACTUAL" ]; then \ + echo lz4-$(LZ4_VER).tar.gz checksum mismatch, expected=\"$(LZ4_SHA256)\" actual=\"$$LZ4_SHA256_ACTUAL\"; \ + exit 1; \ + fi + +liblz4.a: lz4-$(LZ4_VER).tar.gz + -rm -rf lz4-$(LZ4_VER) + tar xvzf lz4-$(LZ4_VER).tar.gz + cd lz4-$(LZ4_VER)/lib && $(MAKE) CFLAGS='-fPIC -O2 $(ARCHFLAG) ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' all + cp lz4-$(LZ4_VER)/lib/liblz4.a . + +zstd-$(ZSTD_VER).tar.gz: + curl --fail --output zstd-$(ZSTD_VER).tar.gz --location ${CURL_SSL_OPTS} ${ZSTD_DOWNLOAD_BASE}/v$(ZSTD_VER).tar.gz + ZSTD_SHA256_ACTUAL=`$(SHA256_CMD) zstd-$(ZSTD_VER).tar.gz | cut -d ' ' -f 1`; \ + if [ "$(ZSTD_SHA256)" != "$$ZSTD_SHA256_ACTUAL" ]; then \ + echo zstd-$(ZSTD_VER).tar.gz checksum mismatch, expected=\"$(ZSTD_SHA256)\" actual=\"$$ZSTD_SHA256_ACTUAL\"; \ + exit 1; \ + fi + +libzstd.a: zstd-$(ZSTD_VER).tar.gz + -rm -rf zstd-$(ZSTD_VER) + tar xvzf zstd-$(ZSTD_VER).tar.gz + cd zstd-$(ZSTD_VER)/lib && DESTDIR=. PREFIX= $(MAKE) CFLAGS='-fPIC -O2 $(ARCHFLAG) ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' libzstd.a + cp zstd-$(ZSTD_VER)/lib/libzstd.a . + +# A version of each $(LIB_OBJECTS) compiled with -fPIC and a fixed set of static compression libraries +ifneq ($(ROCKSDB_JAVA_NO_COMPRESSION), 1) +JAVA_COMPRESSIONS = libz.a libbz2.a libsnappy.a liblz4.a libzstd.a +endif + +JAVA_STATIC_FLAGS = -DZLIB -DBZIP2 -DSNAPPY -DLZ4 -DZSTD +JAVA_STATIC_INCLUDES = -I./zlib-$(ZLIB_VER) -I./bzip2-$(BZIP2_VER) -I./snappy-$(SNAPPY_VER) -I./snappy-$(SNAPPY_VER)/build -I./lz4-$(LZ4_VER)/lib -I./zstd-$(ZSTD_VER)/lib -I./zstd-$(ZSTD_VER)/lib/dictBuilder + +ifneq ($(findstring rocksdbjavastatic, $(filter-out rocksdbjavastatic_deps, $(MAKECMDGOALS))),) +CXXFLAGS += $(JAVA_STATIC_FLAGS) $(JAVA_STATIC_INCLUDES) +CFLAGS += $(JAVA_STATIC_FLAGS) $(JAVA_STATIC_INCLUDES) +endif +rocksdbjavastatic: +ifeq ($(JAVA_HOME),) + $(error JAVA_HOME is not set) +endif + $(MAKE) rocksdbjavastatic_deps + $(MAKE) rocksdbjavastatic_libobjects + $(MAKE) rocksdbjavastatic_javalib + $(MAKE) rocksdbjava_jar + +rocksdbjavastaticosx: rocksdbjavastaticosx_archs + cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR) HISTORY*.md + cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) librocksdbjni-osx-x86_64.jnilib librocksdbjni-osx-arm64.jnilib + cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class + openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1 + +rocksdbjavastaticosx_ub: rocksdbjavastaticosx_archs + cd java/target; lipo -create -output librocksdbjni-osx.jnilib librocksdbjni-osx-x86_64.jnilib librocksdbjni-osx-arm64.jnilib + cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR) HISTORY*.md + cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) librocksdbjni-osx.jnilib + cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class + openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1 + +rocksdbjavastaticosx_archs: + $(MAKE) rocksdbjavastaticosx_arch_x86_64 + $(MAKE) rocksdbjavastaticosx_arch_arm64 + +rocksdbjavastaticosx_arch_%: +ifeq ($(JAVA_HOME),) + $(error JAVA_HOME is not set) +endif + $(MAKE) clean-ext-libraries-bin + $(MAKE) clean-rocks + ARCHFLAG="-arch $*" $(MAKE) rocksdbjavastatic_deps + ARCHFLAG="-arch $*" $(MAKE) rocksdbjavastatic_libobjects + ARCHFLAG="-arch $*" ROCKSDBJNILIB="librocksdbjni-osx-$*.jnilib" $(MAKE) rocksdbjavastatic_javalib + +ifeq ($(JAR_CMD),) +ifneq ($(JAVA_HOME),) +JAR_CMD := $(JAVA_HOME)/bin/jar +else +JAR_CMD := jar +endif +endif +rocksdbjavastatic_javalib: + cd java; $(MAKE) javalib + rm -f java/target/$(ROCKSDBJNILIB) + $(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC \ + -o ./java/target/$(ROCKSDBJNILIB) $(ALL_JNI_NATIVE_SOURCES) \ + $(LIB_OBJECTS) $(COVERAGEFLAGS) \ + $(JAVA_COMPRESSIONS) $(JAVA_STATIC_LDFLAGS) + cd java/target;if [ "$(DEBUG_LEVEL)" == "0" ]; then \ + strip $(STRIPFLAGS) $(ROCKSDBJNILIB); \ + fi + +rocksdbjava_jar: + cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR) HISTORY*.md + cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB) + cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class + openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1 + +rocksdbjava_javadocs_jar: + cd java/target/apidocs; $(JAR_CMD) -cf ../$(ROCKSDB_JAVADOCS_JAR) * + openssl sha1 java/target/$(ROCKSDB_JAVADOCS_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAVADOCS_JAR).sha1 + +rocksdbjava_sources_jar: + cd java/src/main/java; $(JAR_CMD) -cf ../../../target/$(ROCKSDB_SOURCES_JAR) org + openssl sha1 java/target/$(ROCKSDB_SOURCES_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_SOURCES_JAR).sha1 + +rocksdbjavastatic_deps: $(JAVA_COMPRESSIONS) + +rocksdbjavastatic_libobjects: $(LIB_OBJECTS) + +rocksdbjavastaticrelease: rocksdbjavastaticosx rocksdbjava_javadocs_jar rocksdbjava_sources_jar + cd java/crossbuild && (vagrant destroy -f || true) && vagrant up linux32 && vagrant halt linux32 && vagrant up linux64 && vagrant halt linux64 && vagrant up linux64-musl && vagrant halt linux64-musl + cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md + cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib + cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class + openssl sha1 java/target/$(ROCKSDB_JAR_ALL) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR_ALL).sha1 + +rocksdbjavastaticreleasedocker: rocksdbjavastaticosx rocksdbjavastaticdockerx86 rocksdbjavastaticdockerx86_64 rocksdbjavastaticdockerx86musl rocksdbjavastaticdockerx86_64musl rocksdbjava_javadocs_jar rocksdbjava_sources_jar + cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md + cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib + cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class + openssl sha1 java/target/$(ROCKSDB_JAR_ALL) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR_ALL).sha1 + +rocksdbjavastaticdockerx86: + mkdir -p java/target + docker run --rm --name rocksdb_linux_x86-be --platform linux/386 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos6_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh + +rocksdbjavastaticdockerx86_64: + mkdir -p java/target + docker run --rm --name rocksdb_linux_x64-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos6_x64-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh + +rocksdbjavastaticdockerppc64le: + mkdir -p java/target + docker run --rm --name rocksdb_linux_ppc64le-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos7_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh + +rocksdbjavastaticdockerarm64v8: + mkdir -p java/target + docker run --rm --name rocksdb_linux_arm64v8-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos7_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh + +rocksdbjavastaticdockers390x: + mkdir -p java/target + docker run --rm --name rocksdb_linux_s390x-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:ubuntu18_s390x-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh + +rocksdbjavastaticdockerx86musl: + mkdir -p java/target + docker run --rm --name rocksdb_linux_x86-musl-be --platform linux/386 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:alpine3_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-alpine.sh + +rocksdbjavastaticdockerx86_64musl: + mkdir -p java/target + docker run --rm --name rocksdb_linux_x64-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:alpine3_x64-be /rocksdb-host/java/crossbuild/docker-build-linux-alpine.sh + +rocksdbjavastaticdockerppc64lemusl: + mkdir -p java/target + docker run --rm --name rocksdb_linux_ppc64le-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:alpine3_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux-alpine.sh + +rocksdbjavastaticdockerarm64v8musl: + mkdir -p java/target + docker run --rm --name rocksdb_linux_arm64v8-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:alpine3_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux-alpine.sh + +rocksdbjavastaticdockers390xmusl: + mkdir -p java/target + docker run --rm --name rocksdb_linux_s390x-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:alpine3_s390x-be /rocksdb-host/java/crossbuild/docker-build-linux-alpine.sh + +rocksdbjavastaticpublish: rocksdbjavastaticrelease rocksdbjavastaticpublishcentral + +rocksdbjavastaticpublishdocker: rocksdbjavastaticreleasedocker rocksdbjavastaticpublishcentral + +ROCKSDB_JAVA_RELEASE_CLASSIFIERS = javadoc sources linux64 linux32 linux64-musl linux32-musl osx win64 + +rocksdbjavastaticpublishcentral: rocksdbjavageneratepom + mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar + $(foreach classifier, $(ROCKSDB_JAVA_RELEASE_CLASSIFIERS), mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar -Dclassifier=$(classifier);) + +rocksdbjavageneratepom: + cd java;cat pom.xml.template | sed 's/\$${ROCKSDB_JAVA_VERSION}/$(ROCKSDB_JAVA_VERSION)/' > pom.xml + +rocksdbjavastaticnexusbundlejar: rocksdbjavageneratepom + openssl sha1 -r java/pom.xml | awk '{ print $$1 }' > java/target/pom.xml.sha1 + openssl sha1 -r java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar | awk '{ print $$1 }' > java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar.sha1 + $(foreach classifier, $(ROCKSDB_JAVA_RELEASE_CLASSIFIERS), openssl sha1 -r java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar | awk '{ print $$1 }' > java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar.sha1;) + gpg --yes --output java/target/pom.xml.asc -ab java/pom.xml + gpg --yes -ab java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar + $(foreach classifier, $(ROCKSDB_JAVA_RELEASE_CLASSIFIERS), gpg --yes -ab java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar;) + $(JAR_CMD) cvf java/target/nexus-bundle-rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar -C java pom.xml -C java/target pom.xml.sha1 -C java/target pom.xml.asc -C java/target rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar -C java/target rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar.sha1 -C java/target rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar.asc + $(foreach classifier, $(ROCKSDB_JAVA_RELEASE_CLASSIFIERS), $(JAR_CMD) uf java/target/nexus-bundle-rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar -C java/target rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar -C java/target rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar.sha1 -C java/target rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar.asc;) + + +# A version of each $(LIBOBJECTS) compiled with -fPIC + +jl/%.o: %.cc + $(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) -fPIC -c $< -o $@ $(COVERAGEFLAGS) + +rocksdbjava: $(LIB_OBJECTS) +ifeq ($(JAVA_HOME),) + $(error JAVA_HOME is not set) +endif + $(AM_V_GEN)cd java; $(MAKE) javalib; + $(AM_V_at)rm -f ./java/target/$(ROCKSDBJNILIB) + $(AM_V_at)$(CXX) $(CXXFLAGS) -I./java/. -I./java/rocksjni $(JAVA_INCLUDE) $(ROCKSDB_PLUGIN_JNI_CXX_INCLUDEFLAGS) -shared -fPIC -o ./java/target/$(ROCKSDBJNILIB) $(ALL_JNI_NATIVE_SOURCES) $(LIB_OBJECTS) $(JAVA_LDFLAGS) $(COVERAGEFLAGS) + $(AM_V_at)cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR) HISTORY*.md + $(AM_V_at)cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB) + $(AM_V_at)cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class + $(AM_V_at)openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1 + +jclean: + cd java;$(MAKE) clean; + +jtest_compile: rocksdbjava + cd java;$(MAKE) java_test + +jtest_run: + cd java;$(MAKE) run_test + +jtest: rocksdbjava + cd java;$(MAKE) sample test + +jdb_bench: + cd java;$(MAKE) db_bench; + +commit_prereq: + echo "TODO: bring this back using parts of old precommit_checker.py and rocksdb-lego-determinator" + false # J=$(J) build_tools/precommit_checker.py unit clang_unit release clang_release tsan asan ubsan lite unit_non_shm + # $(MAKE) clean && $(MAKE) jclean && $(MAKE) rocksdbjava; + +# For public CI runs, checkout folly in a way that can build with RocksDB. +# This is mostly intended as a test-only simulation of Meta-internal folly +# integration. +checkout_folly: + if [ -e third-party/folly ]; then \ + cd third-party/folly && ${GIT_COMMAND} fetch origin; \ + else \ + cd third-party && ${GIT_COMMAND} clone https://github.com/facebook/folly.git; \ + fi + @# Pin to a particular version for public CI, so that PR authors don't + @# need to worry about folly breaking our integration. Update periodically + cd third-party/folly && git reset --hard beacd86d63cd71c904632262e6c36f60874d78ba + @# A hack to remove boost dependency. + @# NOTE: this hack is only needed if building using USE_FOLLY_LITE + perl -pi -e 's/^(#include .)/__cpp_rtti && $$1/' third-party/folly/folly/memory/MemoryResource.h + +CXX_M_FLAGS = $(filter -m%, $(CXXFLAGS)) + +build_folly: + FOLLY_INST_PATH=`cd third-party/folly; $(PYTHON) build/fbcode_builder/getdeps.py show-inst-dir`; \ + if [ "$$FOLLY_INST_PATH" ]; then \ + rm -rf $${FOLLY_INST_PATH}/../../*; \ + else \ + echo "Please run checkout_folly first"; \ + false; \ + fi + # Restore the original version of Invoke.h with boost dependency + cd third-party/folly && ${GIT_COMMAND} checkout folly/functional/Invoke.h + cd third-party/folly && \ + CXXFLAGS=" $(CXX_M_FLAGS) -DHAVE_CXX11_ATOMIC " $(PYTHON) build/fbcode_builder/getdeps.py build --no-tests + +# --------------------------------------------------------------------------- +# Build size testing +# --------------------------------------------------------------------------- + +REPORT_BUILD_STATISTIC?=echo STATISTIC: + +build_size: + # === normal build, static === + $(MAKE) clean + $(MAKE) static_lib + $(REPORT_BUILD_STATISTIC) rocksdb.build_size.static_lib $$(stat --printf="%s" librocksdb.a) + strip librocksdb.a + $(REPORT_BUILD_STATISTIC) rocksdb.build_size.static_lib_stripped $$(stat --printf="%s" librocksdb.a) + # === normal build, shared === + $(MAKE) clean + $(MAKE) shared_lib + $(REPORT_BUILD_STATISTIC) rocksdb.build_size.shared_lib $$(stat --printf="%s" `readlink -f librocksdb.so`) + strip `readlink -f librocksdb.so` + $(REPORT_BUILD_STATISTIC) rocksdb.build_size.shared_lib_stripped $$(stat --printf="%s" `readlink -f librocksdb.so`) + +# --------------------------------------------------------------------------- +# Platform-specific compilation +# --------------------------------------------------------------------------- + +ifeq ($(PLATFORM), IOS) +# For iOS, create universal object files to be used on both the simulator and +# a device. +XCODEROOT=$(shell xcode-select -print-path) +PLATFORMSROOT=$(XCODEROOT)/Platforms +SIMULATORROOT=$(PLATFORMSROOT)/iPhoneSimulator.platform/Developer +DEVICEROOT=$(PLATFORMSROOT)/iPhoneOS.platform/Developer +IOSVERSION=$(shell defaults read $(PLATFORMSROOT)/iPhoneOS.platform/version CFBundleShortVersionString) + +.cc.o: + mkdir -p ios-x86/$(dir $@) + $(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@ + mkdir -p ios-arm/$(dir $@) + xcrun -sdk iphoneos $(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -arch armv7s -arch arm64 -c $< -o ios-arm/$@ + lipo ios-x86/$@ ios-arm/$@ -create -output $@ + +.c.o: + mkdir -p ios-x86/$(dir $@) + $(CC) $(CFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@ + mkdir -p ios-arm/$(dir $@) + xcrun -sdk iphoneos $(CC) $(CFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -arch armv7s -arch arm64 -c $< -o ios-arm/$@ + lipo ios-x86/$@ ios-arm/$@ -create -output $@ + +else +ifeq ($(HAVE_POWER8),1) +$(OBJ_DIR)/util/crc32c_ppc.o: util/crc32c_ppc.c + $(AM_V_CC)$(CC) $(CFLAGS) -c $< -o $@ + +$(OBJ_DIR)/util/crc32c_ppc_asm.o: util/crc32c_ppc_asm.S + $(AM_V_CC)$(CC) $(CFLAGS) -c $< -o $@ +endif +$(OBJ_DIR)/%.o: %.cc + $(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) -c $< -o $@ $(COVERAGEFLAGS) + +$(OBJ_DIR)/%.o: %.cpp + $(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) -c $< -o $@ $(COVERAGEFLAGS) + +$(OBJ_DIR)/%.o: %.c + $(AM_V_CC)$(CC) $(CFLAGS) -c $< -o $@ +endif + +# --------------------------------------------------------------------------- +# Source files dependencies detection +# --------------------------------------------------------------------------- +# If skip dependencies is ON, skip including the dep files +ifneq ($(SKIP_DEPENDS), 1) +DEPFILES = $(patsubst %.cc, $(OBJ_DIR)/%.cc.d, $(ALL_SOURCES)) +DEPFILES+ = $(patsubst %.c, $(OBJ_DIR)/%.c.d, $(LIB_SOURCES_C) $(TEST_MAIN_SOURCES_C)) +ifeq ($(USE_FOLLY_LITE),1) + DEPFILES +=$(patsubst %.cpp, $(OBJ_DIR)/%.cpp.d, $(FOLLY_SOURCES)) +endif +endif + +# Add proper dependency support so changing a .h file forces a .cc file to +# rebuild. + +# The .d file indicates .cc file's dependencies on .h files. We generate such +# dependency by g++'s -MM option, whose output is a make dependency rule. +$(OBJ_DIR)/%.cc.d: %.cc + @mkdir -p $(@D) && $(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) \ + -MM -MT'$@' -MT'$(<:.cc=.o)' -MT'$(<:%.cc=$(OBJ_DIR)/%.o)' \ + "$<" -o '$@' + +$(OBJ_DIR)/%.cpp.d: %.cpp + @mkdir -p $(@D) && $(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) \ + -MM -MT'$@' -MT'$(<:.cpp=.o)' -MT'$(<:%.cpp=$(OBJ_DIR)/%.o)' \ + "$<" -o '$@' + +ifeq ($(HAVE_POWER8),1) +DEPFILES_C = $(patsubst %.c, $(OBJ_DIR)/%.c.d, $(LIB_SOURCES_C)) +DEPFILES_ASM = $(patsubst %.S, $(OBJ_DIR)/%.S.d, $(LIB_SOURCES_ASM)) + +$(OBJ_DIR)/%.c.d: %.c + @$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) \ + -MM -MT'$@' -MT'$(<:.c=.o)' "$<" -o '$@' + +$(OBJ_DIR)/%.S.d: %.S + @$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) \ + -MM -MT'$@' -MT'$(<:.S=.o)' "$<" -o '$@' + +$(DEPFILES_C): %.c.d + +$(DEPFILES_ASM): %.S.d +depend: $(DEPFILES) $(DEPFILES_C) $(DEPFILES_ASM) +else +depend: $(DEPFILES) +endif + +build_subset_tests: $(ROCKSDBTESTS_SUBSET) + $(AM_V_GEN)if [ -n "$${ROCKSDBTESTS_SUBSET_TESTS_TO_FILE}" ]; then echo "$(ROCKSDBTESTS_SUBSET)" > "$${ROCKSDBTESTS_SUBSET_TESTS_TO_FILE}"; else echo "$(ROCKSDBTESTS_SUBSET)"; fi + +list_all_tests: + echo "$(ROCKSDBTESTS_SUBSET)" + +# Remove the rules for which dependencies should not be generated and see if any are left. +#If so, include the dependencies; if not, do not include the dependency files +ROCKS_DEP_RULES=$(filter-out clean format check-format check-buck-targets check-headers check-sources jclean jtest package analyze tags rocksdbjavastatic% unity.% unity_test checkout_folly, $(MAKECMDGOALS)) +ifneq ("$(ROCKS_DEP_RULES)", "") +-include $(DEPFILES) +endif diff --git a/librocksdb-sys/rocksdb/PLUGINS.md b/librocksdb-sys/rocksdb/PLUGINS.md new file mode 100644 index 0000000..fefacbe --- /dev/null +++ b/librocksdb-sys/rocksdb/PLUGINS.md @@ -0,0 +1,8 @@ +This is the list of all known third-party plugins for RocksDB. If something is missing, please open a pull request to add it. + +* [Dedupfs](https://github.com/ajkr/dedupfs): an example for plugin developers to reference +* [HDFS](https://github.com/riversand963/rocksdb-hdfs-env): an Env used for interacting with HDFS. Migrated from main RocksDB repo +* [ZenFS](https://github.com/westerndigitalcorporation/zenfs): a file system for zoned block devices +* [RADOS](https://github.com/riversand963/rocksdb-rados-env): an Env used for interacting with RADOS. Migrated from RocksDB main repo. +* [PMEM](https://github.com/pmem/pmem-rocksdb-plugin): a collection of plugins to enable Persistent Memory on RocksDB. +* [IPPCP](https://github.com/intel/ippcp-plugin-rocksdb): a plugin to enable encryption on RocksDB based on Intel optimized open source IPP-Crypto library. \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/README.md b/librocksdb-sys/rocksdb/README.md new file mode 100644 index 0000000..8fcc4ab --- /dev/null +++ b/librocksdb-sys/rocksdb/README.md @@ -0,0 +1,29 @@ +## RocksDB: A Persistent Key-Value Store for Flash and RAM Storage + +[![CircleCI Status](https://circleci.com/gh/facebook/rocksdb.svg?style=svg)](https://circleci.com/gh/facebook/rocksdb) + +RocksDB is developed and maintained by Facebook Database Engineering Team. +It is built on earlier work on [LevelDB](https://github.com/google/leveldb) by Sanjay Ghemawat (sanjay@google.com) +and Jeff Dean (jeff@google.com) + +This code is a library that forms the core building block for a fast +key-value server, especially suited for storing data on flash drives. +It has a Log-Structured-Merge-Database (LSM) design with flexible tradeoffs +between Write-Amplification-Factor (WAF), Read-Amplification-Factor (RAF) +and Space-Amplification-Factor (SAF). It has multi-threaded compactions, +making it especially suitable for storing multiple terabytes of data in a +single database. + +Start with example usage here: https://github.com/facebook/rocksdb/tree/main/examples + +See the [github wiki](https://github.com/facebook/rocksdb/wiki) for more explanation. + +The public interface is in `include/`. Callers should not include or +rely on the details of any other header files in this package. Those +internal APIs may be changed without warning. + +Questions and discussions are welcome on the [RocksDB Developers Public](https://www.facebook.com/groups/rocksdb.dev/) Facebook group and [email list](https://groups.google.com/g/rocksdb) on Google Groups. + +## License + +RocksDB is dual-licensed under both the GPLv2 (found in the COPYING file in the root directory) and Apache 2.0 License (found in the LICENSE.Apache file in the root directory). You may select, at your option, one of the above-listed licenses. diff --git a/librocksdb-sys/rocksdb/TARGETS b/librocksdb-sys/rocksdb/TARGETS new file mode 100644 index 0000000..5125fcf --- /dev/null +++ b/librocksdb-sys/rocksdb/TARGETS @@ -0,0 +1,5607 @@ +# This file @generated by: +#$ python3 buckifier/buckify_rocksdb.py +# --> DO NOT EDIT MANUALLY <-- +# This file is a Facebook-specific integration for buck builds, so can +# only be validated by Facebook employees. +# +# @noautodeps @nocodemods +load("//rocks/buckifier:defs.bzl", "cpp_library_wrapper","rocks_cpp_library_wrapper","cpp_binary_wrapper","cpp_unittest_wrapper","fancy_bench_wrapper","add_c_test_wrapper") + + +cpp_library_wrapper(name="rocksdb_lib", srcs=[ + "cache/cache.cc", + "cache/cache_entry_roles.cc", + "cache/cache_helpers.cc", + "cache/cache_key.cc", + "cache/cache_reservation_manager.cc", + "cache/charged_cache.cc", + "cache/clock_cache.cc", + "cache/compressed_secondary_cache.cc", + "cache/lru_cache.cc", + "cache/secondary_cache.cc", + "cache/secondary_cache_adapter.cc", + "cache/sharded_cache.cc", + "db/arena_wrapped_db_iter.cc", + "db/blob/blob_contents.cc", + "db/blob/blob_fetcher.cc", + "db/blob/blob_file_addition.cc", + "db/blob/blob_file_builder.cc", + "db/blob/blob_file_cache.cc", + "db/blob/blob_file_garbage.cc", + "db/blob/blob_file_meta.cc", + "db/blob/blob_file_reader.cc", + "db/blob/blob_garbage_meter.cc", + "db/blob/blob_log_format.cc", + "db/blob/blob_log_sequential_reader.cc", + "db/blob/blob_log_writer.cc", + "db/blob/blob_source.cc", + "db/blob/prefetch_buffer_collection.cc", + "db/builder.cc", + "db/c.cc", + "db/column_family.cc", + "db/compaction/compaction.cc", + "db/compaction/compaction_iterator.cc", + "db/compaction/compaction_job.cc", + "db/compaction/compaction_outputs.cc", + "db/compaction/compaction_picker.cc", + "db/compaction/compaction_picker_fifo.cc", + "db/compaction/compaction_picker_level.cc", + "db/compaction/compaction_picker_universal.cc", + "db/compaction/compaction_service_job.cc", + "db/compaction/compaction_state.cc", + "db/compaction/sst_partitioner.cc", + "db/compaction/subcompaction_state.cc", + "db/convenience.cc", + "db/db_filesnapshot.cc", + "db/db_impl/compacted_db_impl.cc", + "db/db_impl/db_impl.cc", + "db/db_impl/db_impl_compaction_flush.cc", + "db/db_impl/db_impl_debug.cc", + "db/db_impl/db_impl_experimental.cc", + "db/db_impl/db_impl_files.cc", + "db/db_impl/db_impl_open.cc", + "db/db_impl/db_impl_readonly.cc", + "db/db_impl/db_impl_secondary.cc", + "db/db_impl/db_impl_write.cc", + "db/db_info_dumper.cc", + "db/db_iter.cc", + "db/dbformat.cc", + "db/error_handler.cc", + "db/event_helpers.cc", + "db/experimental.cc", + "db/external_sst_file_ingestion_job.cc", + "db/file_indexer.cc", + "db/flush_job.cc", + "db/flush_scheduler.cc", + "db/forward_iterator.cc", + "db/import_column_family_job.cc", + "db/internal_stats.cc", + "db/log_reader.cc", + "db/log_writer.cc", + "db/logs_with_prep_tracker.cc", + "db/malloc_stats.cc", + "db/memtable.cc", + "db/memtable_list.cc", + "db/merge_helper.cc", + "db/merge_operator.cc", + "db/output_validator.cc", + "db/periodic_task_scheduler.cc", + "db/range_del_aggregator.cc", + "db/range_tombstone_fragmenter.cc", + "db/repair.cc", + "db/seqno_to_time_mapping.cc", + "db/snapshot_impl.cc", + "db/table_cache.cc", + "db/table_properties_collector.cc", + "db/transaction_log_impl.cc", + "db/trim_history_scheduler.cc", + "db/version_builder.cc", + "db/version_edit.cc", + "db/version_edit_handler.cc", + "db/version_set.cc", + "db/wal_edit.cc", + "db/wal_manager.cc", + "db/wide/wide_column_serialization.cc", + "db/wide/wide_columns.cc", + "db/write_batch.cc", + "db/write_batch_base.cc", + "db/write_controller.cc", + "db/write_stall_stats.cc", + "db/write_thread.cc", + "env/composite_env.cc", + "env/env.cc", + "env/env_chroot.cc", + "env/env_encryption.cc", + "env/env_posix.cc", + "env/file_system.cc", + "env/file_system_tracer.cc", + "env/fs_posix.cc", + "env/fs_remap.cc", + "env/io_posix.cc", + "env/mock_env.cc", + "env/unique_id_gen.cc", + "file/delete_scheduler.cc", + "file/file_prefetch_buffer.cc", + "file/file_util.cc", + "file/filename.cc", + "file/line_file_reader.cc", + "file/random_access_file_reader.cc", + "file/read_write_util.cc", + "file/readahead_raf.cc", + "file/sequence_file_reader.cc", + "file/sst_file_manager_impl.cc", + "file/writable_file_writer.cc", + "logging/auto_roll_logger.cc", + "logging/event_logger.cc", + "logging/log_buffer.cc", + "memory/arena.cc", + "memory/concurrent_arena.cc", + "memory/jemalloc_nodump_allocator.cc", + "memory/memkind_kmem_allocator.cc", + "memory/memory_allocator.cc", + "memtable/alloc_tracker.cc", + "memtable/hash_linklist_rep.cc", + "memtable/hash_skiplist_rep.cc", + "memtable/skiplistrep.cc", + "memtable/vectorrep.cc", + "memtable/write_buffer_manager.cc", + "monitoring/histogram.cc", + "monitoring/histogram_windowing.cc", + "monitoring/in_memory_stats_history.cc", + "monitoring/instrumented_mutex.cc", + "monitoring/iostats_context.cc", + "monitoring/perf_context.cc", + "monitoring/perf_level.cc", + "monitoring/persistent_stats_history.cc", + "monitoring/statistics.cc", + "monitoring/thread_status_impl.cc", + "monitoring/thread_status_updater.cc", + "monitoring/thread_status_updater_debug.cc", + "monitoring/thread_status_util.cc", + "monitoring/thread_status_util_debug.cc", + "options/cf_options.cc", + "options/configurable.cc", + "options/customizable.cc", + "options/db_options.cc", + "options/options.cc", + "options/options_helper.cc", + "options/options_parser.cc", + "port/mmap.cc", + "port/port_posix.cc", + "port/stack_trace.cc", + "port/win/env_default.cc", + "port/win/env_win.cc", + "port/win/io_win.cc", + "port/win/port_win.cc", + "port/win/win_logger.cc", + "port/win/win_thread.cc", + "table/adaptive/adaptive_table_factory.cc", + "table/block_based/binary_search_index_reader.cc", + "table/block_based/block.cc", + "table/block_based/block_based_table_builder.cc", + "table/block_based/block_based_table_factory.cc", + "table/block_based/block_based_table_iterator.cc", + "table/block_based/block_based_table_reader.cc", + "table/block_based/block_builder.cc", + "table/block_based/block_cache.cc", + "table/block_based/block_prefetcher.cc", + "table/block_based/block_prefix_index.cc", + "table/block_based/data_block_footer.cc", + "table/block_based/data_block_hash_index.cc", + "table/block_based/filter_block_reader_common.cc", + "table/block_based/filter_policy.cc", + "table/block_based/flush_block_policy.cc", + "table/block_based/full_filter_block.cc", + "table/block_based/hash_index_reader.cc", + "table/block_based/index_builder.cc", + "table/block_based/index_reader_common.cc", + "table/block_based/parsed_full_filter_block.cc", + "table/block_based/partitioned_filter_block.cc", + "table/block_based/partitioned_index_iterator.cc", + "table/block_based/partitioned_index_reader.cc", + "table/block_based/reader_common.cc", + "table/block_based/uncompression_dict_reader.cc", + "table/block_fetcher.cc", + "table/compaction_merging_iterator.cc", + "table/cuckoo/cuckoo_table_builder.cc", + "table/cuckoo/cuckoo_table_factory.cc", + "table/cuckoo/cuckoo_table_reader.cc", + "table/format.cc", + "table/get_context.cc", + "table/iterator.cc", + "table/merging_iterator.cc", + "table/meta_blocks.cc", + "table/persistent_cache_helper.cc", + "table/plain/plain_table_bloom.cc", + "table/plain/plain_table_builder.cc", + "table/plain/plain_table_factory.cc", + "table/plain/plain_table_index.cc", + "table/plain/plain_table_key_coding.cc", + "table/plain/plain_table_reader.cc", + "table/sst_file_dumper.cc", + "table/sst_file_reader.cc", + "table/sst_file_writer.cc", + "table/table_factory.cc", + "table/table_properties.cc", + "table/two_level_iterator.cc", + "table/unique_id.cc", + "test_util/sync_point.cc", + "test_util/sync_point_impl.cc", + "test_util/transaction_test_util.cc", + "tools/dump/db_dump_tool.cc", + "tools/io_tracer_parser_tool.cc", + "tools/ldb_cmd.cc", + "tools/ldb_tool.cc", + "tools/sst_dump_tool.cc", + "trace_replay/block_cache_tracer.cc", + "trace_replay/io_tracer.cc", + "trace_replay/trace_record.cc", + "trace_replay/trace_record_handler.cc", + "trace_replay/trace_record_result.cc", + "trace_replay/trace_replay.cc", + "util/async_file_reader.cc", + "util/build_version.cc", + "util/cleanable.cc", + "util/coding.cc", + "util/compaction_job_stats_impl.cc", + "util/comparator.cc", + "util/compression.cc", + "util/compression_context_cache.cc", + "util/concurrent_task_limiter_impl.cc", + "util/crc32c.cc", + "util/crc32c_arm64.cc", + "util/data_structure.cc", + "util/dynamic_bloom.cc", + "util/file_checksum_helper.cc", + "util/hash.cc", + "util/murmurhash.cc", + "util/random.cc", + "util/rate_limiter.cc", + "util/ribbon_config.cc", + "util/slice.cc", + "util/status.cc", + "util/stderr_logger.cc", + "util/string_util.cc", + "util/thread_local.cc", + "util/threadpool_imp.cc", + "util/udt_util.cc", + "util/write_batch_util.cc", + "util/xxhash.cc", + "utilities/agg_merge/agg_merge.cc", + "utilities/backup/backup_engine.cc", + "utilities/blob_db/blob_compaction_filter.cc", + "utilities/blob_db/blob_db.cc", + "utilities/blob_db/blob_db_impl.cc", + "utilities/blob_db/blob_db_impl_filesnapshot.cc", + "utilities/blob_db/blob_dump_tool.cc", + "utilities/blob_db/blob_file.cc", + "utilities/cache_dump_load.cc", + "utilities/cache_dump_load_impl.cc", + "utilities/cassandra/cassandra_compaction_filter.cc", + "utilities/cassandra/format.cc", + "utilities/cassandra/merge_operator.cc", + "utilities/checkpoint/checkpoint_impl.cc", + "utilities/compaction_filters.cc", + "utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc", + "utilities/convenience/info_log_finder.cc", + "utilities/counted_fs.cc", + "utilities/debug.cc", + "utilities/env_mirror.cc", + "utilities/env_timed.cc", + "utilities/fault_injection_env.cc", + "utilities/fault_injection_fs.cc", + "utilities/fault_injection_secondary_cache.cc", + "utilities/leveldb_options/leveldb_options.cc", + "utilities/memory/memory_util.cc", + "utilities/merge_operators.cc", + "utilities/merge_operators/bytesxor.cc", + "utilities/merge_operators/max.cc", + "utilities/merge_operators/put.cc", + "utilities/merge_operators/sortlist.cc", + "utilities/merge_operators/string_append/stringappend.cc", + "utilities/merge_operators/string_append/stringappend2.cc", + "utilities/merge_operators/uint64add.cc", + "utilities/object_registry.cc", + "utilities/option_change_migration/option_change_migration.cc", + "utilities/options/options_util.cc", + "utilities/persistent_cache/block_cache_tier.cc", + "utilities/persistent_cache/block_cache_tier_file.cc", + "utilities/persistent_cache/block_cache_tier_metadata.cc", + "utilities/persistent_cache/persistent_cache_tier.cc", + "utilities/persistent_cache/volatile_tier_impl.cc", + "utilities/simulator_cache/cache_simulator.cc", + "utilities/simulator_cache/sim_cache.cc", + "utilities/table_properties_collectors/compact_on_deletion_collector.cc", + "utilities/trace/file_trace_reader_writer.cc", + "utilities/trace/replayer_impl.cc", + "utilities/transactions/lock/lock_manager.cc", + "utilities/transactions/lock/point/point_lock_manager.cc", + "utilities/transactions/lock/point/point_lock_tracker.cc", + "utilities/transactions/lock/range/range_tree/lib/locktree/concurrent_tree.cc", + "utilities/transactions/lock/range/range_tree/lib/locktree/keyrange.cc", + "utilities/transactions/lock/range/range_tree/lib/locktree/lock_request.cc", + "utilities/transactions/lock/range/range_tree/lib/locktree/locktree.cc", + "utilities/transactions/lock/range/range_tree/lib/locktree/manager.cc", + "utilities/transactions/lock/range/range_tree/lib/locktree/range_buffer.cc", + "utilities/transactions/lock/range/range_tree/lib/locktree/treenode.cc", + "utilities/transactions/lock/range/range_tree/lib/locktree/txnid_set.cc", + "utilities/transactions/lock/range/range_tree/lib/locktree/wfg.cc", + "utilities/transactions/lock/range/range_tree/lib/standalone_port.cc", + "utilities/transactions/lock/range/range_tree/lib/util/dbt.cc", + "utilities/transactions/lock/range/range_tree/lib/util/memarena.cc", + "utilities/transactions/lock/range/range_tree/range_tree_lock_manager.cc", + "utilities/transactions/lock/range/range_tree/range_tree_lock_tracker.cc", + "utilities/transactions/optimistic_transaction.cc", + "utilities/transactions/optimistic_transaction_db_impl.cc", + "utilities/transactions/pessimistic_transaction.cc", + "utilities/transactions/pessimistic_transaction_db.cc", + "utilities/transactions/snapshot_checker.cc", + "utilities/transactions/transaction_base.cc", + "utilities/transactions/transaction_db_mutex_impl.cc", + "utilities/transactions/transaction_util.cc", + "utilities/transactions/write_prepared_txn.cc", + "utilities/transactions/write_prepared_txn_db.cc", + "utilities/transactions/write_unprepared_txn.cc", + "utilities/transactions/write_unprepared_txn_db.cc", + "utilities/ttl/db_ttl_impl.cc", + "utilities/wal_filter.cc", + "utilities/write_batch_with_index/write_batch_with_index.cc", + "utilities/write_batch_with_index/write_batch_with_index_internal.cc", + ], deps=[ + "//folly/container:f14_hash", + "//folly/experimental/coro:blocking_wait", + "//folly/experimental/coro:collect", + "//folly/experimental/coro:coroutine", + "//folly/experimental/coro:task", + "//folly/synchronization:distributed_mutex", + ], headers=None, link_whole=False, extra_test_libs=False) + +cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[], deps=[":rocksdb_lib"], headers=None, link_whole=True, extra_test_libs=False) + +cpp_library_wrapper(name="rocksdb_test_lib", srcs=[ + "db/db_test_util.cc", + "db/db_with_timestamp_test_util.cc", + "table/mock_table.cc", + "test_util/mock_time_env.cc", + "test_util/secondary_cache_test_util.cc", + "test_util/testharness.cc", + "test_util/testutil.cc", + "tools/block_cache_analyzer/block_cache_trace_analyzer.cc", + "tools/trace_analyzer_tool.cc", + "utilities/agg_merge/test_agg_merge.cc", + "utilities/cassandra/test_utils.cc", + ], deps=[":rocksdb_lib"], headers=None, link_whole=False, extra_test_libs=True) + +cpp_library_wrapper(name="rocksdb_tools_lib", srcs=[ + "test_util/testutil.cc", + "tools/block_cache_analyzer/block_cache_trace_analyzer.cc", + "tools/db_bench_tool.cc", + "tools/simulated_hybrid_file_system.cc", + "tools/trace_analyzer_tool.cc", + ], deps=[":rocksdb_lib"], headers=None, link_whole=False, extra_test_libs=False) + +cpp_library_wrapper(name="rocksdb_cache_bench_tools_lib", srcs=["cache/cache_bench_tool.cc"], deps=[":rocksdb_lib"], headers=None, link_whole=False, extra_test_libs=False) + +rocks_cpp_library_wrapper(name="rocksdb_stress_lib", srcs=[ + "db_stress_tool/batched_ops_stress.cc", + "db_stress_tool/cf_consistency_stress.cc", + "db_stress_tool/db_stress_common.cc", + "db_stress_tool/db_stress_driver.cc", + "db_stress_tool/db_stress_gflags.cc", + "db_stress_tool/db_stress_listener.cc", + "db_stress_tool/db_stress_shared_state.cc", + "db_stress_tool/db_stress_stat.cc", + "db_stress_tool/db_stress_test_base.cc", + "db_stress_tool/db_stress_tool.cc", + "db_stress_tool/expected_state.cc", + "db_stress_tool/expected_value.cc", + "db_stress_tool/multi_ops_txns_stress.cc", + "db_stress_tool/no_batched_ops_stress.cc", + "test_util/testutil.cc", + "tools/block_cache_analyzer/block_cache_trace_analyzer.cc", + "tools/trace_analyzer_tool.cc", + ], headers=None) + + +cpp_binary_wrapper(name="db_stress", srcs=["db_stress_tool/db_stress.cc"], deps=[":rocksdb_stress_lib"], extra_preprocessor_flags=[], extra_bench_libs=False) + +cpp_binary_wrapper(name="ribbon_bench", srcs=["microbench/ribbon_bench.cc"], deps=[], extra_preprocessor_flags=[], extra_bench_libs=True) + +cpp_binary_wrapper(name="db_basic_bench", srcs=["microbench/db_basic_bench.cc"], deps=[], extra_preprocessor_flags=[], extra_bench_libs=True) + +add_c_test_wrapper() + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_0", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:51200/threads:8': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:51200/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2438, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_1", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:51200/threads:8': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2437, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_2", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:409600/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:409600/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}}, slow=False, expected_runtime=2446, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_3", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:51200/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'DataBlockSeek/iterations:1000000': ['real_time', + 'cpu_time', + 'seek_ns', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2437, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_4", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:51200/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'RandomAccessFileReaderRead/enable_statistics:1/iterations:1000000': ['real_time', + 'cpu_time', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2437, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_5", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:51200/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2437, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_6", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:409600/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:51200/threads:8': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2437, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_7", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:51200/threads:8': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'RandomAccessFileReaderRead/enable_statistics:0/iterations:1000000': ['real_time', + 'cpu_time', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct']}}, slow=False, expected_runtime=2438, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_8", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:409600/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2437, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_9", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:51200/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:409600/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2437, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_10", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryPositive/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2437, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_11", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:51200/threads:8': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads']}}, slow=False, expected_runtime=2446, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_12", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2437, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_13", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:409600/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2437, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_14", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:51200/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:51200/threads:8': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads']}}, slow=False, expected_runtime=2437, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_0_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88891, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_1_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88804, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_2_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88803, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_3_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88891, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_4_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88809, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_5_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88803, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_6_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88813, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_7_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88813, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_8_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88709, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_9_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88711, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_10_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88819, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_11_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88711, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_12_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88709, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_13_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88709, sl_iterations=3, regression_threshold=10) + + +fancy_bench_wrapper(suite_name="rocksdb_microbench_suite_14_slow", binary_to_bench_to_metric_list_map={'db_basic_bench': {'DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'neg_qu_pct', + 'threads'], + 'DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['db_size', + 'get_mean', + 'threads', + 'real_time', + 'cpu_time', + 'neg_qu_pct'], + 'DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1': ['real_time', + 'put_mean', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8': ['real_time', + 'cpu_time', + 'db_size', + 'threads'], + 'PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240': ['real_time', + 'cpu_time', + 'db_size', + 'threads']}, + 'ribbon_bench': {'FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'size'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads', + 'fp_pct'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads'], + 'FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576': ['real_time', + 'cpu_time', + 'threads']}}, slow=True, expected_runtime=88711, sl_iterations=3, regression_threshold=10) + + + # Generate a test rule for each entry in ROCKS_TESTS + # Do not build the tests in opt mode, since SyncPoint and other test code + # will not be included. + +cpp_unittest_wrapper(name="agg_merge_test", + srcs=["utilities/agg_merge/agg_merge_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="arena_test", + srcs=["memory/arena_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="auto_roll_logger_test", + srcs=["logging/auto_roll_logger_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="autovector_test", + srcs=["util/autovector_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="backup_engine_test", + srcs=["utilities/backup/backup_engine_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="blob_counting_iterator_test", + srcs=["db/blob/blob_counting_iterator_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="blob_db_test", + srcs=["utilities/blob_db/blob_db_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="blob_file_addition_test", + srcs=["db/blob/blob_file_addition_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="blob_file_builder_test", + srcs=["db/blob/blob_file_builder_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="blob_file_cache_test", + srcs=["db/blob/blob_file_cache_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="blob_file_garbage_test", + srcs=["db/blob/blob_file_garbage_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="blob_file_reader_test", + srcs=["db/blob/blob_file_reader_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="blob_garbage_meter_test", + srcs=["db/blob/blob_garbage_meter_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="blob_source_test", + srcs=["db/blob/blob_source_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="block_based_table_reader_test", + srcs=["table/block_based/block_based_table_reader_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="block_cache_trace_analyzer_test", + srcs=["tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="block_cache_tracer_test", + srcs=["trace_replay/block_cache_tracer_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="block_fetcher_test", + srcs=["table/block_fetcher_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="block_test", + srcs=["table/block_based/block_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="bloom_test", + srcs=["util/bloom_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="cache_reservation_manager_test", + srcs=["cache/cache_reservation_manager_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="cache_simulator_test", + srcs=["utilities/simulator_cache/cache_simulator_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="cache_test", + srcs=["cache/cache_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="cassandra_format_test", + srcs=["utilities/cassandra/cassandra_format_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="cassandra_functional_test", + srcs=["utilities/cassandra/cassandra_functional_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="cassandra_row_merge_test", + srcs=["utilities/cassandra/cassandra_row_merge_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="cassandra_serialize_test", + srcs=["utilities/cassandra/cassandra_serialize_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="checkpoint_test", + srcs=["utilities/checkpoint/checkpoint_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="cleanable_test", + srcs=["table/cleanable_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="clipping_iterator_test", + srcs=["db/compaction/clipping_iterator_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="coding_test", + srcs=["util/coding_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="column_family_test", + srcs=["db/column_family_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="compact_files_test", + srcs=["db/compact_files_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="compact_on_deletion_collector_test", + srcs=["utilities/table_properties_collectors/compact_on_deletion_collector_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="compaction_iterator_test", + srcs=["db/compaction/compaction_iterator_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="compaction_job_stats_test", + srcs=["db/compaction/compaction_job_stats_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="compaction_job_test", + srcs=["db/compaction/compaction_job_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="compaction_picker_test", + srcs=["db/compaction/compaction_picker_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="compaction_service_test", + srcs=["db/compaction/compaction_service_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="comparator_db_test", + srcs=["db/comparator_db_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="compressed_secondary_cache_test", + srcs=["cache/compressed_secondary_cache_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="configurable_test", + srcs=["options/configurable_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="corruption_test", + srcs=["db/corruption_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="crc32c_test", + srcs=["util/crc32c_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="cuckoo_table_builder_test", + srcs=["table/cuckoo/cuckoo_table_builder_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="cuckoo_table_db_test", + srcs=["db/cuckoo_table_db_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="cuckoo_table_reader_test", + srcs=["table/cuckoo/cuckoo_table_reader_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="customizable_test", + srcs=["options/customizable_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="data_block_hash_index_test", + srcs=["table/block_based/data_block_hash_index_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_basic_test", + srcs=["db/db_basic_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_blob_basic_test", + srcs=["db/blob/db_blob_basic_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_blob_compaction_test", + srcs=["db/blob/db_blob_compaction_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_blob_corruption_test", + srcs=["db/blob/db_blob_corruption_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_blob_index_test", + srcs=["db/blob/db_blob_index_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_block_cache_test", + srcs=["db/db_block_cache_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_bloom_filter_test", + srcs=["db/db_bloom_filter_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_clip_test", + srcs=["db/db_clip_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_compaction_filter_test", + srcs=["db/db_compaction_filter_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_compaction_test", + srcs=["db/db_compaction_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_dynamic_level_test", + srcs=["db/db_dynamic_level_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_encryption_test", + srcs=["db/db_encryption_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_flush_test", + srcs=["db/db_flush_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_inplace_update_test", + srcs=["db/db_inplace_update_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_io_failure_test", + srcs=["db/db_io_failure_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_iter_stress_test", + srcs=["db/db_iter_stress_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_iter_test", + srcs=["db/db_iter_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_iterator_test", + srcs=["db/db_iterator_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_kv_checksum_test", + srcs=["db/db_kv_checksum_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_log_iter_test", + srcs=["db/db_log_iter_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_logical_block_size_cache_test", + srcs=["db/db_logical_block_size_cache_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_memtable_test", + srcs=["db/db_memtable_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_merge_operand_test", + srcs=["db/db_merge_operand_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_merge_operator_test", + srcs=["db/db_merge_operator_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_options_test", + srcs=["db/db_options_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_properties_test", + srcs=["db/db_properties_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_range_del_test", + srcs=["db/db_range_del_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_rate_limiter_test", + srcs=["db/db_rate_limiter_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_readonly_with_timestamp_test", + srcs=["db/db_readonly_with_timestamp_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_secondary_test", + srcs=["db/db_secondary_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_sst_test", + srcs=["db/db_sst_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_statistics_test", + srcs=["db/db_statistics_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_table_properties_test", + srcs=["db/db_table_properties_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_tailing_iter_test", + srcs=["db/db_tailing_iter_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_test", + srcs=["db/db_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_test2", + srcs=["db/db_test2.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_universal_compaction_test", + srcs=["db/db_universal_compaction_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_wal_test", + srcs=["db/db_wal_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_wide_basic_test", + srcs=["db/wide/db_wide_basic_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_with_timestamp_basic_test", + srcs=["db/db_with_timestamp_basic_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_with_timestamp_compaction_test", + srcs=["db/db_with_timestamp_compaction_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_write_buffer_manager_test", + srcs=["db/db_write_buffer_manager_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="db_write_test", + srcs=["db/db_write_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="dbformat_test", + srcs=["db/dbformat_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="defer_test", + srcs=["util/defer_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="delete_scheduler_test", + srcs=["file/delete_scheduler_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="deletefile_test", + srcs=["db/deletefile_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="dynamic_bloom_test", + srcs=["util/dynamic_bloom_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_library_wrapper(name="env_basic_test_lib", srcs=["env/env_basic_test.cc"], deps=[":rocksdb_test_lib"], headers=None, link_whole=False, extra_test_libs=True) + +cpp_unittest_wrapper(name="env_basic_test", + srcs=["env/env_basic_test.cc"], + deps=[":env_basic_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="env_logger_test", + srcs=["logging/env_logger_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="env_test", + srcs=["env/env_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="env_timed_test", + srcs=["utilities/env_timed_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="error_handler_fs_test", + srcs=["db/error_handler_fs_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="event_logger_test", + srcs=["logging/event_logger_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="external_sst_file_basic_test", + srcs=["db/external_sst_file_basic_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="external_sst_file_test", + srcs=["db/external_sst_file_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="fault_injection_test", + srcs=["db/fault_injection_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="file_indexer_test", + srcs=["db/file_indexer_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="file_reader_writer_test", + srcs=["util/file_reader_writer_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="filelock_test", + srcs=["util/filelock_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="filename_test", + srcs=["db/filename_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="flush_job_test", + srcs=["db/flush_job_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="full_filter_block_test", + srcs=["table/block_based/full_filter_block_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="hash_table_test", + srcs=["utilities/persistent_cache/hash_table_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="hash_test", + srcs=["util/hash_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="heap_test", + srcs=["util/heap_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="histogram_test", + srcs=["monitoring/histogram_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="import_column_family_test", + srcs=["db/import_column_family_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="inlineskiplist_test", + srcs=["memtable/inlineskiplist_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="io_posix_test", + srcs=["env/io_posix_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="io_tracer_parser_test", + srcs=["tools/io_tracer_parser_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="io_tracer_test", + srcs=["trace_replay/io_tracer_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="iostats_context_test", + srcs=["monitoring/iostats_context_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="ldb_cmd_test", + srcs=["tools/ldb_cmd_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="listener_test", + srcs=["db/listener_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="log_test", + srcs=["db/log_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="lru_cache_test", + srcs=["cache/lru_cache_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="manual_compaction_test", + srcs=["db/manual_compaction_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="memory_allocator_test", + srcs=["memory/memory_allocator_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="memory_test", + srcs=["utilities/memory/memory_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="memtable_list_test", + srcs=["db/memtable_list_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="merge_helper_test", + srcs=["db/merge_helper_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="merge_test", + srcs=["db/merge_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="merger_test", + srcs=["table/merger_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="mock_env_test", + srcs=["env/mock_env_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="object_registry_test", + srcs=["utilities/object_registry_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="obsolete_files_test", + srcs=["db/obsolete_files_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="optimistic_transaction_test", + srcs=["utilities/transactions/optimistic_transaction_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="option_change_migration_test", + srcs=["utilities/option_change_migration/option_change_migration_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="options_file_test", + srcs=["db/options_file_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="options_settable_test", + srcs=["options/options_settable_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="options_test", + srcs=["options/options_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="options_util_test", + srcs=["utilities/options/options_util_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="partitioned_filter_block_test", + srcs=["table/block_based/partitioned_filter_block_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="perf_context_test", + srcs=["db/perf_context_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="periodic_task_scheduler_test", + srcs=["db/periodic_task_scheduler_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="persistent_cache_test", + srcs=["utilities/persistent_cache/persistent_cache_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="plain_table_db_test", + srcs=["db/plain_table_db_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="point_lock_manager_test", + srcs=["utilities/transactions/lock/point/point_lock_manager_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="prefetch_test", + srcs=["file/prefetch_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="prefix_test", + srcs=["db/prefix_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="random_access_file_reader_test", + srcs=["file/random_access_file_reader_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="random_test", + srcs=["util/random_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="range_del_aggregator_test", + srcs=["db/range_del_aggregator_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="range_locking_test", + srcs=["utilities/transactions/lock/range/range_locking_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="range_tombstone_fragmenter_test", + srcs=["db/range_tombstone_fragmenter_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="rate_limiter_test", + srcs=["util/rate_limiter_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="reduce_levels_test", + srcs=["tools/reduce_levels_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="repair_test", + srcs=["db/repair_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="repeatable_thread_test", + srcs=["util/repeatable_thread_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="ribbon_test", + srcs=["util/ribbon_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="seqno_time_test", + srcs=["db/seqno_time_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="sim_cache_test", + srcs=["utilities/simulator_cache/sim_cache_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="skiplist_test", + srcs=["memtable/skiplist_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="slice_test", + srcs=["util/slice_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="slice_transform_test", + srcs=["util/slice_transform_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="sst_dump_test", + srcs=["tools/sst_dump_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="sst_file_reader_test", + srcs=["table/sst_file_reader_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="statistics_test", + srcs=["monitoring/statistics_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="stats_history_test", + srcs=["monitoring/stats_history_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="stringappend_test", + srcs=["utilities/merge_operators/string_append/stringappend_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="table_properties_collector_test", + srcs=["db/table_properties_collector_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="table_test", + srcs=["table/table_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="testutil_test", + srcs=["test_util/testutil_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="thread_list_test", + srcs=["util/thread_list_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="thread_local_test", + srcs=["util/thread_local_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="tiered_compaction_test", + srcs=["db/compaction/tiered_compaction_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="timer_queue_test", + srcs=["util/timer_queue_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="timer_test", + srcs=["util/timer_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="timestamped_snapshot_test", + srcs=["utilities/transactions/timestamped_snapshot_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="trace_analyzer_test", + srcs=["tools/trace_analyzer_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="transaction_test", + srcs=["utilities/transactions/transaction_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="ttl_test", + srcs=["utilities/ttl/ttl_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="udt_util_test", + srcs=["util/udt_util_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="util_merge_operators_test", + srcs=["utilities/util_merge_operators_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="version_builder_test", + srcs=["db/version_builder_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="version_edit_test", + srcs=["db/version_edit_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="version_set_test", + srcs=["db/version_set_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="wal_manager_test", + srcs=["db/wal_manager_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="wide_column_serialization_test", + srcs=["db/wide/wide_column_serialization_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="work_queue_test", + srcs=["util/work_queue_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="write_batch_test", + srcs=["db/write_batch_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="write_batch_with_index_test", + srcs=["utilities/write_batch_with_index/write_batch_with_index_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="write_buffer_manager_test", + srcs=["memtable/write_buffer_manager_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="write_callback_test", + srcs=["db/write_callback_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="write_committed_transaction_ts_test", + srcs=["utilities/transactions/write_committed_transaction_ts_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="write_controller_test", + srcs=["db/write_controller_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="write_prepared_transaction_test", + srcs=["utilities/transactions/write_prepared_transaction_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + + +cpp_unittest_wrapper(name="write_unprepared_transaction_test", + srcs=["utilities/transactions/write_unprepared_transaction_test.cc"], + deps=[":rocksdb_test_lib"], + extra_compiler_flags=[]) + diff --git a/librocksdb-sys/rocksdb/USERS.md b/librocksdb-sys/rocksdb/USERS.md new file mode 100644 index 0000000..43c9cac --- /dev/null +++ b/librocksdb-sys/rocksdb/USERS.md @@ -0,0 +1,165 @@ +This document lists users of RocksDB and their use cases. If you are using RocksDB, please open a pull request and add yourself to the list. + +## Facebook +At Facebook, we use RocksDB as storage engines in multiple data management services and a backend for many different stateful services, including: + +1. MyRocks -- https://github.com/MySQLOnRocksDB/mysql-5.6 +2. MongoRocks -- https://github.com/mongodb-partners/mongo-rocks +3. ZippyDB -- Facebook's distributed key-value store with Paxos-style replication, built on top of RocksDB.[1] https://www.youtube.com/watch?v=DfiN7pG0D0khtt +4. Laser -- Laser is a high query throughput, low (millisecond) latency, key-value storage service built on top of RocksDB.[1] +4. Dragon -- a distributed graph query engine. https://code.facebook.com/posts/1737605303120405/dragon-a-distributed-graph-query-engine/ +5. Stylus -- a low-level stream processing framework writtenin C++.[1] +6. LogDevice -- a distributed data store for logs [2] + +[1] https://research.facebook.com/publications/realtime-data-processing-at-facebook/ + +[2] https://code.facebook.com/posts/357056558062811/logdevice-a-distributed-data-store-for-logs/ + +## Bilibili +[Bilibili](bilibili.com) [uses](https://www.alluxio.io/blog/when-ai-meets-alluxio-at-bilibili-building-an-efficient-ai-platform-for-data-preprocessing-and-model-training/) Alluxio to speed up its ML training workloads, and Alluxio uses RocksDB to store its filesystem metadata, so Bilibili uses RocksDB. + +Bilibili's [real-time platform](https://www.alibabacloud.com/blog/architecture-and-practices-of-bilibilis-real-time-platform_596676) uses Flink, and uses RocksDB as Flink's state store. + +## TikTok +TikTok, or its parent company ByteDance, uses RocksDB as the storage engine for some storage systems, such as its distributed graph database [ByteGraph](https://vldb.org/pvldb/vol15/p3306-li.pdf). + +Also, TikTok uses [Alluxio](alluxio.io) to [speed up Presto queries](https://www.alluxio.io/resources/videos/improving-presto-performance-with-alluxio-at-tiktok/), and Alluxio stores the files' metadata in RocksDB. + +## FoundationDB +[FoundationDB](https://www.foundationdb.org/) [uses](https://github.com/apple/foundationdb/blob/377f1f692da6ab2fe5bdac57035651db3e5fb66d/fdbserver/KeyValueStoreRocksDB.actor.cpp) RocksDB to implement a [key-value store interface](https://github.com/apple/foundationdb/blob/377f1f692da6ab2fe5bdac57035651db3e5fb66d/fdbserver/KeyValueStoreRocksDB.actor.cpp#L1127) in its server backend. + +## Apple +Apple [uses](https://opensource.apple.com/projects/foundationdb/) FoundationDB, so it also uses RocksDB. + +## Snowflake +Snowflake [uses](https://www.snowflake.com/blog/how-foundationdb-powers-snowflake-metadata-forward/) FoundationDB, so it also uses RocksDB. + +## Microsoft +The Bing search engine from Microsoft uses RocksDB as the storage engine for its web data platform: https://blogs.bing.com/Engineering-Blog/october-2021/RocksDB-in-Microsoft-Bing + +## LinkedIn +Two different use cases at Linkedin are using RocksDB as a storage engine: + +1. LinkedIn's follow feed for storing user's activities. Check out the blog post: https://engineering.linkedin.com/blog/2016/03/followfeed--linkedin-s-feed-made-faster-and-smarter +2. Apache Samza, open source framework for stream processing + +Learn more about those use cases in a Tech Talk by Ankit Gupta and Naveen Somasundaram: http://www.youtube.com/watch?v=plqVp_OnSzg + +## Yahoo +Yahoo is using RocksDB as a storage engine for their biggest distributed data store Sherpa. Learn more about it here: http://yahooeng.tumblr.com/post/120730204806/sherpa-scales-new-heights + +## Tencent +[PaxosStore](https://github.com/Tencent/paxosstore) is a distributed database supporting WeChat. It uses RocksDB as its storage engine. + +## Baidu +[Apache Doris](http://doris.apache.org/master/en/) is a MPP analytical database engine released by Baidu. It [uses RocksDB](http://doris.apache.org/master/en/administrator-guide/operation/tablet-meta-tool.html) to manage its tablet's metadata. + +## CockroachDB +CockroachDB is an open-source geo-replicated transactional database. They are using RocksDB as their storage engine. Check out their github: https://github.com/cockroachdb/cockroach + +## DNANexus +DNANexus is using RocksDB to speed up processing of genomics data. +You can learn more from this great blog post by Mike Lin: http://devblog.dnanexus.com/faster-bam-sorting-with-samtools-and-rocksdb/ + +## Iron.io +Iron.io is using RocksDB as a storage engine for their distributed queueing system. +Learn more from Tech Talk by Reed Allman: http://www.youtube.com/watch?v=HTjt6oj-RL4 + +## Tango Me +Tango is using RocksDB as a graph storage to store all users' connection data and other social activity data. + +## Turn +Turn is using RocksDB as a storage layer for their key/value store, serving at peak 2.4MM QPS out of different datacenters. +Check out our RocksDB Protobuf merge operator at: https://github.com/vladb38/rocksdb_protobuf + +## Santander UK/Cloudera Profession Services +Check out their blog post: http://blog.cloudera.com/blog/2015/08/inside-santanders-near-real-time-data-ingest-architecture/ + +## Airbnb +Airbnb is using RocksDB as a storage engine for their personalized search service. You can learn more about it here: https://www.youtube.com/watch?v=ASQ6XMtogMs + +## Alluxio +[Alluxio](https://www.alluxio.io) uses RocksDB to serve and scale file system metadata to beyond 1 Billion files. The detailed design and implementation is described in this engineering blog: +https://www.alluxio.io/blog/scalable-metadata-service-in-alluxio-storing-billions-of-files/ + +## Pinterest +Pinterest's Object Retrieval System uses RocksDB for storage: https://www.youtube.com/watch?v=MtFEVEs_2Vo + +## Smyte +[Smyte](https://www.smyte.com/) uses RocksDB as the storage layer for their core key-value storage, high-performance counters and time-windowed HyperLogLog services. + +## Rakuten Marketing +[Rakuten Marketing](https://marketing.rakuten.com/) uses RocksDB as the disk cache layer for the real-time bidding service in their Performance DSP. + +## VWO, Wingify +[VWO's](https://vwo.com/) Smart Code checker and URL helper uses RocksDB to store all the URLs where VWO's Smart Code is installed. + +## quasardb +[quasardb](https://www.quasardb.net) is a high-performance, distributed, transactional key-value database that integrates well with in-memory analytics engines such as Apache Spark. +quasardb uses a heavily tuned RocksDB as its persistence layer. + +## Netflix +[Netflix](http://techblog.netflix.com/2016/05/application-data-caching-using-ssds.html) Netflix uses RocksDB on AWS EC2 instances with local SSD drives to cache application data. + +## TiKV +[TiKV](https://github.com/pingcap/tikv) is a GEO-replicated, high-performance, distributed, transactional key-value database. TiKV is powered by Rust and Raft. TiKV uses RocksDB as its persistence layer. + +## TiDB +[TiDB](https://github.com/pingcap/tidb) uses the TiKV distributed key-value database, so it uses RocksDB. + +## PingCAP +[PingCAP](https://www.pingcap.com/) is the company behind TiDB, its cloud database service uses RocksDB. + +## Apache Spark +[Spark Structured Streaming](https://docs.databricks.com/structured-streaming/rocksdb-state-store.html) uses RocksDB as the local state store. + +## Databricks +[Databricks](https://www.databricks.com/) [replaces AWS RDS with TiDB](https://www.pingcap.com/case-study/how-databricks-tackles-the-scalability-limit-with-a-mysql-alternative/) for scalability, so it uses RocksDB. + +## Apache Flink +[Apache Flink](https://flink.apache.org/news/2016/03/08/release-1.0.0.html) uses RocksDB to store state locally on a machine. + +## Dgraph +[Dgraph](https://github.com/dgraph-io/dgraph) is an open-source, scalable, distributed, low latency, high throughput Graph database .They use RocksDB to store state locally on a machine. + +## Uber +[Uber](http://eng.uber.com/cherami/) uses RocksDB as a durable and scalable task queue. + +## 360 Pika +[360](http://www.360.cn/) [Pika](https://github.com/Qihoo360/pika) is a nosql compatible with redis. With the huge amount of data stored, redis may suffer for a capacity bottleneck, and pika was born for solving it. It has widely been used in many companies. + +## LzLabs +LzLabs is using RocksDB as a storage engine in their multi-database distributed framework to store application configuration and user data. + +## ProfaneDB +[ProfaneDB](https://profanedb.gitlab.io/) is a database for Protocol Buffers, and uses RocksDB for storage. It is accessible via gRPC, and the schema is defined using directly `.proto` files. + +## IOTA Foundation + [IOTA Foundation](https://www.iota.org/) is using RocksDB in the [IOTA Reference Implementation (IRI)](https://github.com/iotaledger/iri) to store the local state of the Tangle. The Tangle is the first open-source distributed ledger powering the future of the Internet of Things. + +## Avrio Project + [Avrio Project](http://avrio-project.github.io/avrio.network/) is using RocksDB in [Avrio ](https://github.com/avrio-project/avrio) to store blocks, account balances and data and other blockchain-releated data. Avrio is a multiblockchain decentralized cryptocurrency empowering monetary transactions. + +## Crux +[Crux](https://github.com/juxt/crux) is a document database that uses RocksDB for local [EAV](https://en.wikipedia.org/wiki/Entity%E2%80%93attribute%E2%80%93value_model) index storage to enable point-in-time bitemporal Datalog queries. The "unbundled" architecture uses Kafka to provide horizontal scalability. + +## Nebula Graph +[Nebula Graph](https://github.com/vesoft-inc/nebula) is a distributed, scalable, lightning-fast, open source graph database capable of hosting super large scale graphs with dozens of billions of vertices (nodes) and trillions of edges, with milliseconds of latency. + +## YugabyteDB +[YugabyteDB](https://www.yugabyte.com/) is an open source, high performance, distributed SQL database that uses RocksDB as its storage layer. For more information, please see https://github.com/yugabyte/yugabyte-db/. + +## ArangoDB +[ArangoDB](https://www.arangodb.com/) is a native multi-model database with flexible data models for documents, graphs, and key-values, for building high performance applications using a convenient SQL-like query language or JavaScript extensions. It uses RocksDB as its storage engine. + +## Milvus +[Milvus](https://milvus.io/) is an open source vector database for unstructured data. It uses RocksDB not only as one of the supported kv storage engines, but also as a message queue. + +## Kafka +[Kafka](https://kafka.apache.org/) is an open-source distributed event streaming platform, it uses RocksDB to store state in Kafka Streams: https://www.confluent.io/blog/how-to-tune-rocksdb-kafka-streams-state-stores-performance/. + +## Solana Labs +[Solana](https://github.com/solana-labs/solana) is a fast, secure, scalable, and decentralized blockchain. It uses RocksDB as the underlying storage for its ledger store. + +## Others +More databases using RocksDB can be found at [dbdb.io](https://dbdb.io/browse?embeds=rocksdb). diff --git a/librocksdb-sys/rocksdb/Vagrantfile b/librocksdb-sys/rocksdb/Vagrantfile new file mode 100644 index 0000000..07f2e99 --- /dev/null +++ b/librocksdb-sys/rocksdb/Vagrantfile @@ -0,0 +1,39 @@ +# Vagrant file +Vagrant.configure("2") do |config| + + config.vm.provider "virtualbox" do |v| + v.memory = 4096 + v.cpus = 2 + end + + config.vm.define "ubuntu14" do |box| + box.vm.box = "ubuntu/trusty64" + end + + config.vm.define "centos65" do |box| + box.vm.box = "chef/centos-6.5" + end + + config.vm.define "centos7" do |box| + box.vm.box = "centos/7" + box.vm.provision "shell", path: "build_tools/setup_centos7.sh" + end + + config.vm.define "FreeBSD10" do |box| + box.vm.guest = :freebsd + box.vm.box = "robin/freebsd-10" + # FreeBSD does not support 'mount_virtualbox_shared_folder', use NFS + box.vm.synced_folder ".", "/vagrant", :nfs => true, id: "vagrant-root" + box.vm.network "private_network", ip: "10.0.1.10" + + # build everything after creating VM, skip using --no-provision + box.vm.provision "shell", inline: <<-SCRIPT + pkg install -y gmake clang35 + export CXX=/usr/local/bin/clang++35 + cd /vagrant + gmake clean + gmake all OPT=-g + SCRIPT + end + +end diff --git a/librocksdb-sys/rocksdb/WINDOWS_PORT.md b/librocksdb-sys/rocksdb/WINDOWS_PORT.md new file mode 100644 index 0000000..a6e4f93 --- /dev/null +++ b/librocksdb-sys/rocksdb/WINDOWS_PORT.md @@ -0,0 +1,228 @@ +# Microsoft Contribution Notes + +## Contributors +* Alexander Zinoviev https://github.com/zinoale +* Dmitri Smirnov https://github.com/yuslepukhin +* Praveen Rao https://github.com/PraveenSinghRao +* Sherlock Huang https://github.com/SherlockNoMad + +## Introduction +RocksDB is a well proven open source key-value persistent store, optimized for fast storage. It provides scalability with number of CPUs and storage IOPS, to support IO-bound, in-memory and write-once workloads, most importantly, to be flexible to allow for innovation. + +As Microsoft Bing team we have been continuously pushing hard to improve the scalability, efficiency of platform and eventually benefit Bing end-user satisfaction. We would like to explore the opportunity to embrace open source, RocksDB here, to use, enhance and customize for our usage, and also contribute back to the RocksDB community. Herein, we are pleased to offer this RocksDB port for Windows platform. + +These notes describe some decisions and changes we had to make with regards to porting RocksDB on Windows. We hope this will help both reviewers and users of the Windows port. +We are open for comments and improvements. + +## OS specifics +All of the porting, testing and benchmarking was done on Windows Server 2012 R2 Datacenter 64-bit but to the best of our knowledge there is not a specific API we used during porting that is unsupported on other Windows OS after Vista. + +## Porting goals +We strive to achieve the following goals: +* make use of the existing porting interface of RocksDB +* make minimum [WY2]modifications within platform independent code. +* make all unit test pass both in debug and release builds. + * Note: latest introduction of SyncPoint seems to disable running db_test in Release. +* make performance on par with published benchmarks accounting for HW differences +* we would like to keep the port code inline with the main branch with no forking + +## Build system +We have chosen CMake as a widely accepted build system to build the Windows port. It is very fast and convenient. + +At the same time it generates Visual Studio projects that are both usable from a command line and IDE. + +The top-level CMakeLists.txt file contains description of all targets and build rules. It also provides brief instructions on how to build the software for Windows. One more build related file is thirdparty.inc that also resides on the top level. This file must be edited to point to actual third party libraries location. +We think that it would be beneficial to merge the existing make-based build system and the new cmake-based build system into a single one to use on all platforms. + +All building and testing was done for 64-bit. We have not conducted any testing for 32-bit and early reports indicate that it will not run on 32-bit. + +## C++ and STL notes +We had to make some minimum changes within the portable files that either account for OS differences or the shortcomings of C++11 support in the current version of the MS compiler. Most or all of them are expected to be fixed in the upcoming compiler releases. + +We plan to use this port for our business purposes here at Bing and this provided business justification for this port. This also means, we do not have at present to choose the compiler version at will. + +* Certain headers that are not present and not necessary on Windows were simply `#ifndef OS_WIN` in a few places (`unistd.h`) +* All posix specific headers were replaced to port/port.h which worked well +* Replaced `dirent.h` for `port/port_dirent.h` (very few places) with the implementation of the relevant interfaces within `rocksdb::port` namespace +* Replaced `sys/time.h` to `port/sys_time.h` (few places) implemented equivalents within `rocksdb::port` +* `printf %z` specification is not supported on Windows. To imitate existing standards we came up with a string macro `ROCKSDB_PRIszt` which expands to `zu` on posix systems and to `Iu` on windows. +* in class member initialization were moved to a __ctors in some cases +* `constexpr` is not supported. We had to replace `std::numeric_limits<>::max/min()` to its C macros for constants. Sometimes we had to make class members `static const` and place a definition within a .cc file. +* `constexpr` for functions was replaced to a template specialization (1 place) +* Union members that have non-trivial constructors were replaced to `char[]` in one place along with bug fixes (spatial experimental feature) +* Zero-sized arrays are deemed a non-standard extension which we converted to 1 size array and that should work well for the purposes of these classes. +* `std::chrono` lacks nanoseconds support (fixed in the upcoming release of the STL) and we had to use `QueryPerfCounter()` within env_win.cc +* Function local statics initialization is still not safe. Used `std::once` to mitigate within WinEnv. + +## Windows Environments notes +We endeavored to make it functionally on par with posix_env. This means we replicated the functionality of the thread pool and other things as precise as possible, including: +* Replicate posix logic using std:thread primitives. +* Implement all posix_env disk access functionality. +* Set `use_os_buffer=false` to disable OS disk buffering for WinWritableFile and WinRandomAccessFile. +* Replace `pread/pwrite` with `WriteFile/ReadFile` with `OVERLAPPED` structure. +* Use `SetFileInformationByHandle` to compensate absence of `fallocate`. + +### In detail +Even though Windows provides its own efficient thread-pool implementation we chose to replicate posix logic using `std::thread` primitives. This allows anyone to quickly detect any changes within the posix source code and replicate them within windows env. This has proven to work very well. At the same time for anyone who wishes to replace the built-in thread-pool can do so using RocksDB stackable environments. + +For disk access we implemented all of the functionality present within the posix_env which includes memory mapped files, random access, rate-limiter support etc. +The `use_os_buffer` flag on Posix platforms currently denotes disabling read-ahead log via `fadvise` mechanism. Windows does not have `fadvise` system call. What is more, it implements disk cache in a way that differs from Linux greatly. It's not an uncommon practice on Windows to perform un-buffered disk access to gain control of the memory consumption. We think that in our use case this may also be a good configuration option at the expense of disk throughput. To compensate one may increase the configured in-memory cache size instead. Thus we have chosen `use_os_buffer=false` to disable OS disk buffering for `WinWritableFile` and `WinRandomAccessFile`. The OS imposes restrictions on the alignment of the disk offsets, buffers used and the amount of data that is read/written when accessing files in un-buffered mode. When the option is true, the classes behave in a standard way. This allows to perform writes and reads in cases when un-buffered access does not make sense such as WAL and MANIFEST. + +We have replaced `pread/pwrite` with `WriteFile/ReadFile` with `OVERLAPPED` structure so we can atomically seek to the position of the disk operation but still perform the operation synchronously. Thus we able to emulate that functionality of `pread/pwrite` reasonably well. The only difference is that the file pointer is not returned to its original position but that hardly matters given the random nature of access. + +We used `SetFileInformationByHandle` both to truncate files after writing a full final page to disk and to pre-allocate disk space for faster I/O thus compensating for the absence of `fallocate` although some differences remain. For example, the pre-allocated space is not filled with zeros like on Linux, however, on a positive note, the end of file position is also not modified after pre-allocation. + +RocksDB renames, copies and deletes files at will even though they may be opened with another handle at the same time. We had to relax and allow nearly all the concurrent access permissions possible. + +## Thread-Local Storage +Thread-Local storage plays a significant role for RocksDB performance. Rather than creating a separate implementation we chose to create inline wrappers that forward `pthread_specific` calls to Windows `Tls` interfaces within `rocksdb::port` namespace. This leaves the existing meat of the logic in tact and unchanged and just as maintainable. + +To mitigate the lack of thread local storage cleanup on thread-exit we added a limited amount of windows specific code within the same thread_local.cc file that injects a cleanup callback into a `"__tls"` structure within `".CRT$XLB"` data segment. This approach guarantees that the callback is invoked regardless of whether RocksDB used within an executable, standalone DLL or within another DLL. + +## Jemalloc usage + +When RocksDB is used with Jemalloc the latter needs to be initialized before any of the C++ globals or statics. To accomplish that we injected an initialization routine into `".CRT$XCT"` that is automatically invoked by the runtime before initializing static objects. je-uninit is queued to `atexit()`. + +The jemalloc redirecting `new/delete` global operators are used by the linker providing certain conditions are met. See build section in these notes. + +## Stack Trace and Unhandled Exception Handler + +We decided not to implement these two features because the hosting program as a rule has these two things in it. +We experienced no inconveniences debugging issues in the debugger or analyzing process dumps if need be and thus we did not +see this as a priority. + +## Performance results +### Setup +All of the benchmarks are run on the same set of machines. Here are the details of the test setup: +* 2 Intel(R) Xeon(R) E5 2450 0 @ 2.10 GHz (total 16 cores) +* 2 XK0480GDQPH SSD Device, total 894GB free disk +* Machine has 128 GB of RAM +* Operating System: Windows Server 2012 R2 Datacenter +* 100 Million keys; each key is of size 10 bytes, each value is of size 800 bytes +* total database size is ~76GB +* The performance result is based on RocksDB 3.11. +* The parameters used, unless specified, were exactly the same as published in the GitHub Wiki page. + +### RocksDB on flash storage + +#### Test 1. Bulk Load of keys in Random Order + +Version 3.11 + +* Total Run Time: 17.6 min +* Fillrandom: 5.480 micros/op 182465 ops/sec; 142.0 MB/s +* Compact: 486056544.000 micros/op 0 ops/sec + +Version 3.10 + +* Total Run Time: 16.2 min +* Fillrandom: 5.018 micros/op 199269 ops/sec; 155.1 MB/s +* Compact: 441313173.000 micros/op 0 ops/sec; + + +#### Test 2. Bulk Load of keys in Sequential Order + +Version 3.11 + +* Fillseq: 4.944 micros/op 202k ops/sec; 157.4 MB/s + +Version 3.10 + +* Fillseq: 4.105 micros/op 243.6k ops/sec; 189.6 MB/s + + +#### Test 3. Random Write + +Version 3.11 + +* Unbuffered I/O enabled +* Overwrite: 52.661 micros/op 18.9k ops/sec; 14.8 MB/s + +Version 3.10 + +* Unbuffered I/O enabled +* Overwrite: 52.661 micros/op 18.9k ops/sec; + + +#### Test 4. Random Read + +Version 3.11 + +* Unbuffered I/O enabled +* Readrandom: 15.716 micros/op 63.6k ops/sec; 49.5 MB/s + +Version 3.10 + +* Unbuffered I/O enabled +* Readrandom: 15.548 micros/op 64.3k ops/sec; + + +#### Test 5. Multi-threaded read and single-threaded write + +Version 3.11 + +* Unbuffered I/O enabled +* Readwhilewriting: 25.128 micros/op 39.7k ops/sec; + +Version 3.10 + +* Unbuffered I/O enabled +* Readwhilewriting: 24.854 micros/op 40.2k ops/sec; + + +### RocksDB In Memory + +#### Test 1. Point Lookup + +Version 3.11 + +80K writes/sec +* Write Rate Achieved: 40.5k write/sec; +* Readwhilewriting: 0.314 micros/op 3187455 ops/sec; 364.8 MB/s (715454999 of 715454999 found) + +Version 3.10 + +* Write Rate Achieved: 50.6k write/sec +* Readwhilewriting: 0.316 micros/op 3162028 ops/sec; (719576999 of 719576999 found) + + +*10K writes/sec* + +Version 3.11 + +* Write Rate Achieved: 5.8k/s write/sec +* Readwhilewriting: 0.246 micros/op 4062669 ops/sec; 464.9 MB/s (915481999 of 915481999 found) + +Version 3.10 + +* Write Rate Achieved: 5.8k/s write/sec +* Readwhilewriting: 0.244 micros/op 4106253 ops/sec; (927986999 of 927986999 found) + + +#### Test 2. Prefix Range Query + +Version 3.11 + +80K writes/sec +* Write Rate Achieved: 46.3k/s write/sec +* Readwhilewriting: 0.362 micros/op 2765052 ops/sec; 316.4 MB/s (611549999 of 611549999 found) + +Version 3.10 + +* Write Rate Achieved: 45.8k/s write/sec +* Readwhilewriting: 0.317 micros/op 3154941 ops/sec; (708158999 of 708158999 found) + +Version 3.11 + +10K writes/sec +* Write Rate Achieved: 5.78k write/sec +* Readwhilewriting: 0.269 micros/op 3716692 ops/sec; 425.3 MB/s (837401999 of 837401999 found) + +Version 3.10 + +* Write Rate Achieved: 5.7k write/sec +* Readwhilewriting: 0.261 micros/op 3830152 ops/sec; (863482999 of 863482999 found) + + +We think that there is still big room to improve the performance, which will be an ongoing effort for us. + diff --git a/librocksdb-sys/rocksdb/buckifier/bench-slow.json b/librocksdb-sys/rocksdb/buckifier/bench-slow.json new file mode 100644 index 0000000..948b3de --- /dev/null +++ b/librocksdb-sys/rocksdb/buckifier/bench-slow.json @@ -0,0 +1,6163 @@ +[ + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 1712.344628 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 479.941992 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 1811.998557 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 549.901612 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4687.250475 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4818.164105 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 623.018994 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 1997.259639 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 3910.179634 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 3217.553693 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4454.911311 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4195.372795 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:409600/threads:1": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 360.304737 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:1024/enable_statistics:0/wal:1/iterations:51200/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2141.057905 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:51200/threads:8": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 379.059546 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:1024/enable_statistics:0/wal:0/iterations:51200/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2249.892332 + } + ], + "IteratorNext/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2421.708898 + } + ], + "IteratorPrev/comp_style:2/max_data:134217728/per_key_size:256/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1210.835611 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1930.985912 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3730.560675 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3805.367942 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1905.101414 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1346.866095 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2911.433188 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3265.867193 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3500.869188 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2764.891509 + } + ], + "ManualFlush/key_num:65536/per_key_size:1024/iterations:1": [ + "real_time", + "db_size", + "flush_write_bytes", + "flush_time", + "threads", + { + "est_runtime": 8213.664958 + } + ], + "PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1174.289332 + } + ], + "PrefixSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1844.656254 + } + ], + "PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1871.728091 + } + ], + "PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1138.131879 + } + ], + "PrefixSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 934.128086 + } + ], + "PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2050.952519 + } + ], + "PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2119.677364 + } + ], + "PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1620.549616 + } + ], + "PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1540.049484 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 88891, + "name": "rocksdb_microbench_suite_0", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 1774.099605 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 1905.399998 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 3808.99857 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4820.729905 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 651.103057 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 2073.571864 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 501.900122 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 2279.957943 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 517.245591 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4470.408695 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4697.266228 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 378.530036 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:1024/enable_statistics:1/wal:1/iterations:51200/threads:8": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2153.984468 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:1024/enable_statistics:0/wal:0/iterations:409600/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2425.675621 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:51200/threads:8": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 271.474125 + } + ], + "IteratorNext/comp_style:0/max_data:536870912/per_key_size:1024/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2680.208144 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1925.787501 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2138.845144 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3240.472721 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3743.777606 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1889.280273 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1987.727061 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1324.811274 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1351.651528 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2898.268666 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3265.897167 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 4230.885188 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1610.559355 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1479.488999 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1833.697116 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3543.222366 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3977.653591 + } + ], + "ManualFlush/key_num:65536/per_key_size:256/iterations:1": [ + "real_time", + "db_size", + "flush_write_bytes", + "flush_time", + "threads", + { + "est_runtime": 8022.977415 + } + ], + "PrefixSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 900.408425 + } + ], + "PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1086.553529 + } + ], + "PrefixSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1151.02499 + } + ], + "PrefixSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1789.665713 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 88804, + "name": "rocksdb_microbench_suite_1", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBClose/iterations:200": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 7875.438371 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 1809.467556 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 694.578514 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 2007.832073 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 3049.786949 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 3155.268413 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 471.459789 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 513.865636 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 3323.251926 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4287.119322 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4701.167388 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4478.402506 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:409600/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 271.500867 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:1024/enable_statistics:0/wal:1/iterations:51200/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2156.637367 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:51200/threads:8": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 378.808772 + } + ], + "IteratorPrev/comp_style:0/max_data:536870912/per_key_size:1024/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2535.71873 + } + ], + "IteratorPrev/comp_style:1/max_data:134217728/per_key_size:1024/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 850.797161 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1870.864429 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1925.978405 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2138.98528 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 4824.26486 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3568.048799 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1308.673423 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1845.907547 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1909.476304 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3816.002101 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1480.235109 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2428.851465 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3978.136634 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3745.274233 + } + ], + "PrefixSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2299.541611 + } + ], + "PrefixSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1152.817144 + } + ], + "PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2087.829967 + } + ], + "PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1138.066264 + } + ], + "PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1619.322144 + } + ], + "PrefixSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1715.83787 + } + ], + "SimpleGetWithPerfContext/iterations:1000000": [ + "block_seek_nanos", + "get_post_process_time", + "db_size", + "get_snapshot_time", + "block_read_time", + "threads", + "real_time", + "cpu_time", + "get_from_output_files_time", + "new_table_block_iter_nanos", + "get_cpu_nanos", + "user_key_comparison_count", + "neg_qu_pct", + "block_checksum_time", + "get_from_table_nanos", + { + "est_runtime": 1387.59016 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 88803, + "name": "rocksdb_microbench_suite_2", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 1812.356957 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 3809.037737 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 514.201628 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 3294.815224 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4002.903452 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 649.806897 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 498.522549 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 1839.233063 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 1936.181117 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4471.890902 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4264.24321 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4701.026835 + } + ], + "DBOpen/iterations:200": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 7875.060419 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:51200/threads:8": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 360.330841 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:1024/enable_statistics:1/wal:0/iterations:409600/threads:1": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2423.955419 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:409600/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 379.060989 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:1024/enable_statistics:0/wal:1/iterations:409600/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2154.010254 + } + ], + "IteratorNext/comp_style:0/max_data:536870912/per_key_size:256/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2682.352114 + } + ], + "IteratorNext/comp_style:2/max_data:134217728/per_key_size:1024/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 868.172285 + } + ], + "IteratorPrev/comp_style:2/max_data:536870912/per_key_size:1024/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2085.936855 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1972.741386 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1869.527457 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2138.868392 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3745.558567 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3240.715255 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 4832.025298 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3572.947368 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1290.200332 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1903.644533 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2910.058307 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1762.345037 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1441.761423 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1609.655022 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2278.376767 + } + ], + "PrefixSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1153.828342 + } + ], + "PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1408.600116 + } + ], + "PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1136.291725 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 88891, + "name": "rocksdb_microbench_suite_3", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 2898.72783 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 1997.299167 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 532.687188 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 2110.493928 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 2772.793109 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4294.689809 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4002.927614 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4482.402368 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4940.110072 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 651.121374 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 1844.537642 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 497.710941 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 1936.622943 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:1024/enable_statistics:0/wal:0/iterations:409600/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2419.132481 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:51200/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 378.529031 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:51200/threads:8": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 271.438709 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:1024/enable_statistics:1/wal:1/iterations:409600/threads:1": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2149.877539 + } + ], + "IteratorNext/comp_style:1/max_data:134217728/per_key_size:1024/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 869.224505 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2052.861678 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 4748.430327 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3240.626869 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3579.521522 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1296.53393 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1809.525026 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1384.672588 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1904.994109 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3817.997968 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1480.208318 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1613.568431 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3356.701738 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2239.977922 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3751.327826 + } + ], + "ManualFlush/key_num:8192/per_key_size:1024/iterations:1": [ + "real_time", + "db_size", + "flush_write_bytes", + "flush_time", + "threads", + { + "est_runtime": 7597.332723 + } + ], + "PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1122.154905 + } + ], + "PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1870.157606 + } + ], + "PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1739.825979 + } + ], + "PrefixSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1151.690848 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 88809, + "name": "rocksdb_microbench_suite_4", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 1802.08561 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 624.132525 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 1911.844737 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 2007.84697 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 2089.31663 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4497.182856 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4942.804943 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 1883.654744 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 498.06214 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 1937.167999 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 532.702176 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4302.580281 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 3341.771318 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4003.101384 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:51200/threads:8": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 378.81272 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:1024/enable_statistics:1/wal:1/iterations:409600/threads:1": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2157.656253 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:409600/threads:1": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 271.523747 + } + ], + "IteratorPrev/comp_style:0/max_data:536870912/per_key_size:256/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2535.78947 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2139.039426 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3754.165204 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 4752.499353 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1298.604072 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2998.907938 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3820.35491 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3169.465918 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1835.11354 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1608.421875 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3624.747853 + } + ], + "ManualFlush/key_num:8192/per_key_size:256/iterations:1": [ + "real_time", + "db_size", + "flush_write_bytes", + "flush_time", + "threads", + { + "est_runtime": 7557.736147 + } + ], + "PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1173.706929 + } + ], + "PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 976.018148 + } + ], + "PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1021.383458 + } + ], + "PrefixSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2300.700002 + } + ], + "PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2447.080986 + } + ], + "PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1773.944316 + } + ], + "PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1408.654342 + } + ], + "PrefixSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1423.89826 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 88803, + "name": "rocksdb_microbench_suite_5", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 658.890773 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 2073.344847 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 532.66627 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4499.103078 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 3823.666372 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 497.077261 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 1937.011319 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4003.152699 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4358.242188 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4944.478659 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:1024/enable_statistics:1/wal:0/iterations:51200/threads:8": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2201.425905 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:1024/enable_statistics:1/wal:1/iterations:51200/threads:8": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2153.250796 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:409600/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 378.432186 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:51200/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 271.397936 + } + ], + "IteratorNext/comp_style:0/max_data:134217728/per_key_size:256/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1156.84164 + } + ], + "IteratorPrev/comp_style:0/max_data:134217728/per_key_size:256/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 871.341464 + } + ], + "IteratorPrev/comp_style:2/max_data:134217728/per_key_size:1024/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1115.478679 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2136.922862 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1975.004341 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3179.966272 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3769.15995 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 4788.410331 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1307.377381 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1909.364449 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1369.449436 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2912.547738 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3633.675713 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2820.855412 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1479.955208 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1614.562603 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3350.108415 + } + ], + "ManualFlush/key_num:1024/per_key_size:1024/iterations:1": [ + "real_time", + "db_size", + "flush_write_bytes", + "flush_time", + "threads", + { + "est_runtime": 7422.165355 + } + ], + "PrefixSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1844.168905 + } + ], + "PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1870.857522 + } + ], + "PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2402.328004 + } + ], + "PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1739.59811 + } + ], + "PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1809.865409 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 88813, + "name": "rocksdb_microbench_suite_6", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 609.749844 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 551.801692 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4512.440338 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 3823.643659 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 1885.071291 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 497.546918 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 1936.92061 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4003.12535 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 3196.497448 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4346.91794 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4946.256596 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:1024/enable_statistics:0/wal:1/iterations:409600/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2141.297863 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:51200/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 378.198303 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:51200/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 271.417422 + } + ], + "IteratorNext/comp_style:2/max_data:134217728/per_key_size:256/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 926.477715 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1974.320666 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2136.585708 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 4788.447975 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1296.875867 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1906.215547 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2911.477908 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1835.00962 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1607.553483 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3352.038124 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3625.351224 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2814.166152 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3769.100313 + } + ], + "ManualFlush/key_num:1024/per_key_size:256/iterations:1": [ + "real_time", + "db_size", + "flush_write_bytes", + "flush_time", + "threads", + { + "est_runtime": 7418.474332 + } + ], + "PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1171.889503 + } + ], + "PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1078.456365 + } + ], + "PrefixSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1801.200304 + } + ], + "PrefixSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2245.828345 + } + ], + "PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2392.943018 + } + ], + "PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2049.704921 + } + ], + "PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1773.705301 + } + ], + "PrefixSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1410.843275 + } + ], + "PrefixSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1424.541953 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 88813, + "name": "rocksdb_microbench_suite_7", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 694.642444 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 583.226959 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4633.862465 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 451.741737 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 510.357826 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 2108.477344 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4443.989929 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 5215.078632 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 1937.731137 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 3402.268174 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4948.420446 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:1024/enable_statistics:0/wal:0/iterations:51200/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2200.5005 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:1024/enable_statistics:0/wal:1/iterations:409600/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2140.350383 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:1024/enable_statistics:1/wal:0/iterations:409600/threads:1": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2473.029062 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:409600/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 377.799421 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1923.154067 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3832.441913 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3138.670605 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3781.656053 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 4788.510789 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3675.134037 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3265.739118 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 4152.531212 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1623.097106 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1338.666598 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2888.963384 + } + ], + "PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1006.703673 + } + ], + "PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1181.657807 + } + ], + "PrefixSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2304.097915 + } + ], + "PrefixSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1412.397492 + } + ], + "PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2015.978912 + } + ], + "PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1859.4561 + } + ], + "PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1824.916416 + } + ], + "PrefixSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1144.514094 + } + ], + "PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1776.109199 + } + ], + "PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1602.162741 + } + ], + "PrefixSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1896.713718 + } + ], + "RandomAccessFileReaderRead/enable_statistics:1/iterations:1000000": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 5.370559 + } + ] + }, + "ribbon_bench": { + "FilterBuild/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 2.171919 + } + ], + "FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.38186 + } + ], + "FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 9.184089 + } + ], + "FilterQueryNegative/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 1.54233 + } + ], + "FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 1.501483 + } + ], + "FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 1.737477 + } + ], + "FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 82.415772 + } + ], + "FilterQueryPositive/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.399281 + } + ], + "FilterQueryPositive/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 7.024791 + } + ], + "FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 39.015621 + } + ], + "FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.325264 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 88709, + "name": "rocksdb_microbench_suite_8", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 454.919549 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 1712.215589 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4633.82088 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 3152.808792 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 3771.29009 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 5211.225659 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 510.387506 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 2139.111714 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4443.929027 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 1858.972054 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 696.590699 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 578.538571 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4949.611852 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4145.973837 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:51200/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 377.905737 + } + ], + "IteratorPrev/comp_style:1/max_data:134217728/per_key_size:256/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 977.874953 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2012.604554 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1942.165369 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3259.631453 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 4795.420211 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1901.309617 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1918.997951 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1324.859234 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3677.093111 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3837.191621 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1572.19154 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3412.779413 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2348.374525 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2186.840634 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2888.917261 + } + ], + "PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1174.452593 + } + ], + "PrefixSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1816.741937 + } + ], + "PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2448.88995 + } + ], + "PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1138.391757 + } + ], + "PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1411.663065 + } + ], + "PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2090.841484 + } + ], + "PrefixSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1779.515791 + } + ] + }, + "ribbon_bench": { + "FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.194392 + } + ], + "FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 2.60738 + } + ], + "FilterBuild/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 6.614245 + } + ], + "FilterBuild/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.348746 + } + ], + "FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 10.971779 + } + ], + "FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 15.272086 + } + ], + "FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 1.855877 + } + ], + "FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 8.032293 + } + ], + "FilterQueryNegative/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 1.621375 + } + ], + "FilterQueryNegative/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 102.227551 + } + ], + "FilterQueryPositive/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.309645 + } + ], + "FilterQueryPositive/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.448994 + } + ], + "FilterQueryPositive/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.453854 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 88711, + "name": "rocksdb_microbench_suite_9", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 1775.789377 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4633.864621 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 3155.239662 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 3790.561434 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 458.336347 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 510.88842 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 2108.496423 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 557.886298 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 3365.458112 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4145.931239 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 5211.138859 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 694.628355 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4451.395377 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4950.489144 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:1024/enable_statistics:0/wal:1/iterations:51200/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2140.891996 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:1024/enable_statistics:1/wal:0/iterations:409600/threads:1": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2473.313812 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:1024/enable_statistics:1/wal:1/iterations:409600/threads:1": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2167.374891 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:1024/enable_statistics:1/wal:0/iterations:51200/threads:8": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2350.198115 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:51200/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 270.221479 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 378.072129 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2013.660732 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1850.931894 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1941.981279 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1917.767466 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3832.592151 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 4788.541415 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1903.058732 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1343.860292 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3673.431012 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3265.79571 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1830.816871 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1623.280031 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2889.021135 + } + ], + "PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1018.424915 + } + ], + "PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1143.235807 + } + ], + "PrefixSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1582.515889 + } + ], + "PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1187.009146 + } + ], + "PrefixSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1421.928748 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 88819, + "name": "rocksdb_microbench_suite_10", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 2007.895851 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 551.935026 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 1911.863937 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4666.475079 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4809.492906 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 3790.625424 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 387.072927 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 2106.161292 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 1961.147307 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4437.328726 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 5173.846426 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 1859.150921 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 504.228523 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4950.729688 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4041.39475 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:1024/enable_statistics:0/wal:0/iterations:51200/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2473.08233 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:409600/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 360.402539 + } + ], + "IteratorNext/comp_style:2/max_data:536870912/per_key_size:1024/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2192.682661 + } + ], + "IteratorPrev/comp_style:0/max_data:134217728/per_key_size:1024/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 849.453776 + } + ], + "IteratorPrev/comp_style:1/max_data:536870912/per_key_size:1024/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2325.086056 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1893.637846 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2139.270344 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3098.564345 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3857.029519 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3481.020105 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1340.699259 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3715.527152 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3246.485924 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1414.497171 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1641.591443 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1573.758377 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2888.996421 + } + ], + "PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1001.540766 + } + ], + "PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1187.719701 + } + ], + "PrefixSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1145.748787 + } + ], + "PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1776.225564 + } + ], + "PrefixSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1823.654928 + } + ], + "RandomAccessFileReaderRead/enable_statistics:0/iterations:1000000": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 2.984478 + } + ] + }, + "ribbon_bench": { + "FilterBuild/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.311687 + } + ], + "FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.439296 + } + ], + "FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.536606 + } + ], + "FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 6.421361 + } + ], + "FilterQueryNegative/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 1.672049 + } + ], + "FilterQueryNegative/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 7.789373 + } + ], + "FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 15.718081 + } + ], + "FilterQueryPositive/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.26261 + } + ], + "FilterQueryPositive/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 11.512964 + } + ], + "FilterQueryPositive/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.49311 + } + ], + "FilterQueryPositive/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 14.576404 + } + ], + "FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.35537 + } + ], + "FilterQueryPositive/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 54.977852 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 88711, + "name": "rocksdb_microbench_suite_11", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 403.095413 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 505.270116 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 3154.721755 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4687.221851 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 3771.266874 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4804.836604 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 1959.030698 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 557.749335 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 2109.882711 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 2007.868277 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 3823.708033 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 5173.766594 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4195.243654 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4396.047348 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4952.523661 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:1024/enable_statistics:1/wal:0/iterations:51200/threads:8": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2312.17547 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:1024/enable_statistics:1/wal:1/iterations:51200/threads:8": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2199.255566 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:409600/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 360.355195 + } + ], + "DataBlockSeek/iterations:1000000": [ + "real_time", + "cpu_time", + "seek_ns", + "threads", + { + "est_runtime": 16.815497 + } + ], + "IteratorNext/comp_style:1/max_data:134217728/per_key_size:256/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 998.011759 + } + ], + "IteratorNextWithPerfContext/iterations:100000": [ + "find_next_user_entry_time", + "db_size", + "threads", + "internal_key_skipped_count", + "real_time", + "cpu_time", + "iter_next_cpu_nanos", + "user_key_comparison_count", + { + "est_runtime": 829.288054 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1896.181275 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1852.19042 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2139.477433 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1912.38273 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1331.360875 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3420.568984 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3657.279035 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3246.623059 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1572.433675 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2460.911144 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2888.95434 + } + ], + "PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1138.435281 + } + ], + "PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1411.742372 + } + ], + "PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1178.105028 + } + ], + "PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1777.084988 + } + ], + "PrefixSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1678.61669 + } + ], + "PrefixSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1826.405941 + } + ] + }, + "ribbon_bench": { + "FilterBuild/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.654238 + } + ], + "FilterBuild/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.453843 + } + ], + "FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.52447 + } + ], + "FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 2.169285 + } + ], + "FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.335641 + } + ], + "FilterBuild/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.288164 + } + ], + "FilterQueryNegative/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 15.548585 + } + ], + "FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 7.097205 + } + ], + "FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.389287 + } + ], + "FilterQueryPositive/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 14.556778 + } + ], + "FilterQueryPositive/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 6.387351 + } + ], + "FilterQueryPositive/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 47.662364 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 88709, + "name": "rocksdb_microbench_suite_12", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 446.597044 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 507.805668 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 607.525698 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 1937.632471 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 3154.693409 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4666.088632 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4803.243479 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 3772.288434 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 2092.642473 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4425.053714 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 5173.355427 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 1889.78671 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 694.715237 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4145.945651 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4967.640229 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:1024/enable_statistics:0/wal:0/iterations:409600/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2352.471126 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:409600/threads:1": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 360.424861 + } + ], + "IteratorNext/comp_style:0/max_data:134217728/per_key_size:1024/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1000.224231 + } + ], + "IteratorNext/comp_style:1/max_data:536870912/per_key_size:1024/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2192.668329 + } + ], + "IteratorNext/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2480.631028 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1866.980315 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2139.269926 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1917.557528 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2031.89305 + } + ], + "IteratorSeek/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1331.768126 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3678.933137 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3833.856318 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3262.461026 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1659.435948 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1572.985396 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3407.503485 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2839.546177 + } + ], + "PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1183.238636 + } + ], + "PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1412.015301 + } + ], + "PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1815.899454 + } + ], + "PrefixSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1150.426956 + } + ], + "PrefixSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1778.155184 + } + ] + }, + "ribbon_bench": { + "FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.478298 + } + ], + "FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.381091 + } + ], + "FilterBuild/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 7.809054 + } + ], + "FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 1.57804 + } + ], + "FilterQueryNegative/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 2.799623 + } + ], + "FilterQueryNegative/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 129.266223 + } + ], + "FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.761922 + } + ], + "FilterQueryPositive/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 6.667228 + } + ], + "FilterQueryPositive/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.320525 + } + ], + "FilterQueryPositive/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.395642 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 88709, + "name": "rocksdb_microbench_suite_13", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 434.616062 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 510.156004 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 3050.425322 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4667.45867 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 5040.173716 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 3790.602375 + } + ], + "DBGet/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 4804.316933 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 605.701828 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 4442.475931 + } + ], + "DBGet/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 5087.274609 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 1899.844152 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 694.658916 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 1938.347133 + } + ], + "DBGet/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 3910.096422 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 377.714354 + } + ], + "IteratorPrev/comp_style:1/max_data:536870912/per_key_size:256/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2370.382861 + } + ], + "IteratorPrev/comp_style:2/max_data:536870912/per_key_size:256/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2484.65835 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1923.233472 + } + ], + "IteratorSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2139.237156 + } + ], + "IteratorSeek/comp_style:0/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3241.70317 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3684.276482 + } + ], + "IteratorSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2896.854937 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1830.886558 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1337.374059 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1658.766047 + } + ], + "IteratorSeek/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1572.454657 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 3479.088621 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2186.948587 + } + ], + "IteratorSeek/comp_style:2/max_data:536870912/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 4024.639105 + } + ], + "PrefixSeek/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1005.603266 + } + ], + "PrefixSeek/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1181.26407 + } + ], + "PrefixSeek/comp_style:0/max_data:536870912/per_key_size:256/enable_statistics:1/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1849.199043 + } + ], + "PrefixSeek/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1412.076623 + } + ], + "PrefixSeek/comp_style:1/max_data:536870912/per_key_size:1024/enable_statistics:1/enable_filter:0/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2016.125995 + } + ], + "PrefixSeek/comp_style:1/max_data:536870912/per_key_size:256/enable_statistics:0/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 2092.152047 + } + ], + "PrefixSeek/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1141.714694 + } + ], + "PrefixSeek/comp_style:2/max_data:536870912/per_key_size:1024/enable_statistics:0/enable_filter:1/iterations:10240": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 1776.709162 + } + ] + }, + "ribbon_bench": { + "FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.397602 + } + ], + "FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.238302 + } + ], + "FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 2.653086 + } + ], + "FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 7.89775 + } + ], + "FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.364915 + } + ], + "FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 1.638573 + } + ], + "FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 1.331795 + } + ], + "FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 1.821543 + } + ], + "FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 1.467467 + } + ], + "FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 67.68153 + } + ], + "FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 6.708367 + } + ], + "FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 10.709789 + } + ], + "FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 44.929381 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 88711, + "name": "rocksdb_microbench_suite_14", + "regression_threshold": 10, + "sl_iterations": 3 + } +] diff --git a/librocksdb-sys/rocksdb/buckifier/bench.json b/librocksdb-sys/rocksdb/buckifier/bench.json new file mode 100644 index 0000000..e1ea99f --- /dev/null +++ b/librocksdb-sys/rocksdb/buckifier/bench.json @@ -0,0 +1,1594 @@ +[ + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 510.387506 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 497.077261 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 696.590699 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:51200/threads:8": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 271.438709 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:51200/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 377.905737 + } + ] + }, + "ribbon_bench": { + "FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.194392 + } + ], + "FilterBuild/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 6.614245 + } + ], + "FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 9.184089 + } + ], + "FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 1.737477 + } + ], + "FilterQueryPositive/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.49311 + } + ], + "FilterQueryPositive/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 14.576404 + } + ], + "FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.35537 + } + ], + "FilterQueryPositive/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 47.662364 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 2438, + "name": "rocksdb_microbench_suite_0", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 510.88842 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 497.546918 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 694.715237 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 378.072129 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:51200/threads:8": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 271.474125 + } + ] + }, + "ribbon_bench": { + "FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.536606 + } + ], + "FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 7.097205 + } + ], + "FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.389287 + } + ], + "FilterQueryPositive/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.26261 + } + ], + "FilterQueryPositive/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 11.512964 + } + ], + "FilterQueryPositive/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 6.387351 + } + ], + "FilterQueryPositive/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 54.977852 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 2437, + "name": "rocksdb_microbench_suite_1", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 479.941992 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 513.865636 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 694.658916 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:409600/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 379.060989 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:409600/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 377.799421 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 2446, + "name": "rocksdb_microbench_suite_2", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 434.616062 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 694.642444 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 471.459789 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 514.201628 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:51200/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 271.397936 + } + ], + "DataBlockSeek/iterations:1000000": [ + "real_time", + "cpu_time", + "seek_ns", + "threads", + { + "est_runtime": 16.815497 + } + ] + }, + "ribbon_bench": { + "FilterBuild/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.311687 + } + ], + "FilterBuild/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.453843 + } + ], + "FilterBuild/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 7.809054 + } + ], + "FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 6.421361 + } + ], + "FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 1.57804 + } + ], + "FilterQueryPositive/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 14.556778 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 2437, + "name": "rocksdb_microbench_suite_3", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 451.741737 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 458.336347 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 694.628355 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 517.245591 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:51200/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 270.221479 + } + ], + "RandomAccessFileReaderRead/enable_statistics:1/iterations:1000000": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 5.370559 + } + ] + }, + "ribbon_bench": { + "FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.397602 + } + ], + "FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 10.971779 + } + ], + "FilterQueryNegative/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 7.789373 + } + ], + "FilterQueryNegative/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 1.54233 + } + ], + "FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 15.718081 + } + ], + "FilterQueryPositive/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.309645 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 2437, + "name": "rocksdb_microbench_suite_4", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 454.919549 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 403.095413 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:1280/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 694.578514 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 532.66627 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:51200/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 271.417422 + } + ] + }, + "ribbon_bench": { + "FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.335641 + } + ], + "FilterQueryNegative/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 1.672049 + } + ], + "FilterQueryNegative/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 15.548585 + } + ], + "FilterQueryNegative/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 2.799623 + } + ], + "FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 10.709789 + } + ], + "FilterQueryPositive/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 7.024791 + } + ], + "FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 39.015621 + } + ], + "FilterQueryPositive/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.453854 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 2437, + "name": "rocksdb_microbench_suite_5", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 658.890773 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 532.687188 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 498.522549 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:409600/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 360.355195 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:51200/threads:8": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 378.808772 + } + ] + }, + "ribbon_bench": { + "FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.439296 + } + ], + "FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 2.60738 + } + ], + "FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 1.638573 + } + ], + "FilterQueryPositive/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.320525 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 2437, + "name": "rocksdb_microbench_suite_6", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 505.270116 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 651.121374 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 532.702176 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:51200/threads:8": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 378.81272 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:409600/threads:1": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 360.424861 + } + ], + "RandomAccessFileReaderRead/enable_statistics:0/iterations:1000000": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 2.984478 + } + ] + }, + "ribbon_bench": { + "FilterBuild/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.238302 + } + ], + "FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.38186 + } + ], + "FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 1.821543 + } + ], + "FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 1.501483 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 2438, + "name": "rocksdb_microbench_suite_7", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 549.901612 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 651.103057 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 498.06214 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:409600/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 378.432186 + } + ], + "DBPut/comp_style:2/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:409600/threads:1": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 271.523747 + } + ] + }, + "ribbon_bench": { + "FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.52447 + } + ], + "FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.381091 + } + ], + "FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 7.89775 + } + ], + "FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 1.855877 + } + ], + "FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 67.68153 + } + ], + "FilterQueryPositive/filter_impl:0/bits_per_key:20/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 6.667228 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 2437, + "name": "rocksdb_microbench_suite_8", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 551.801692 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 649.806897 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 497.710941 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:51200/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 378.198303 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:409600/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 271.500867 + } + ] + }, + "ribbon_bench": { + "FilterBuild/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.364915 + } + ], + "FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 1.467467 + } + ], + "FilterQueryNegative/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 82.415772 + } + ], + "FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.761922 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 2437, + "name": "rocksdb_microbench_suite_9", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 624.132525 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 510.156004 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 551.935026 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 387.072927 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:409600/threads:1": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 360.304737 + } + ] + }, + "ribbon_bench": { + "FilterBuild/filter_impl:3/bits_per_key:10/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.288164 + } + ], + "FilterQueryPositive/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.395642 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 2437, + "name": "rocksdb_microbench_suite_10", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 507.805668 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 557.749335 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 623.018994 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 377.714354 + } + ], + "DBPut/comp_style:1/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:51200/threads:8": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 379.059546 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 2446, + "name": "rocksdb_microbench_suite_11", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 446.597044 + } + ], + "DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 609.749844 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 510.357826 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 557.886298 + } + ] + }, + "ribbon_bench": { + "FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.478298 + } + ], + "FilterBuild/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 2.169285 + } + ], + "FilterBuild/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.348746 + } + ], + "FilterQueryNegative/filter_impl:2/bits_per_key:10/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 15.272086 + } + ], + "FilterQueryNegative/filter_impl:2/bits_per_key:20/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 8.032293 + } + ], + "FilterQueryNegative/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 102.227551 + } + ], + "FilterQueryNegative/filter_impl:3/bits_per_key:20/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 129.266223 + } + ], + "FilterQueryPositive/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 6.708367 + } + ], + "FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 44.929381 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 2437, + "name": "rocksdb_microbench_suite_12", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 607.525698 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 504.228523 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 578.538571 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:1/iterations:409600/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 360.402539 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:0/iterations:409600/threads:1": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 378.530036 + } + ] + }, + "ribbon_bench": { + "FilterBuild/filter_impl:2/bits_per_key:20/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 2.653086 + } + ], + "FilterQueryNegative/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 1.621375 + } + ], + "FilterQueryPositive/filter_impl:3/bits_per_key:10/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.325264 + } + ], + "FilterQueryPositive/filter_impl:3/bits_per_key:20/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.448994 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 2437, + "name": "rocksdb_microbench_suite_13", + "regression_threshold": 10, + "sl_iterations": 3 + }, + { + "benchmarks": { + "db_basic_bench": { + "DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:1/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 583.226959 + } + ], + "DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:0/negative_query:1/enable_filter:0/iterations:10240/threads:1": [ + "real_time", + "cpu_time", + "db_size", + "neg_qu_pct", + "threads", + { + "est_runtime": 605.701828 + } + ], + "DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [ + "db_size", + "get_mean", + "threads", + "real_time", + "cpu_time", + "neg_qu_pct", + { + "est_runtime": 501.900122 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:0/wal:0/iterations:51200/threads:8": [ + "real_time", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 378.529031 + } + ], + "DBPut/comp_style:0/max_data:107374182400/per_key_size:256/enable_statistics:1/wal:1/iterations:51200/threads:8": [ + "real_time", + "put_mean", + "cpu_time", + "db_size", + "threads", + { + "est_runtime": 360.330841 + } + ] + }, + "ribbon_bench": { + "FilterBuild/filter_impl:0/bits_per_key:10/key_len_avg:10/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 1.654238 + } + ], + "FilterBuild/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1048576": [ + "real_time", + "cpu_time", + "threads", + "size", + { + "est_runtime": 2.171919 + } + ], + "FilterQueryNegative/filter_impl:0/bits_per_key:10/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + "fp_pct", + { + "est_runtime": 1.331795 + } + ], + "FilterQueryPositive/filter_impl:0/bits_per_key:20/key_len_avg:100/entry_num:1024": [ + "real_time", + "cpu_time", + "threads", + { + "est_runtime": 1.399281 + } + ] + } + }, + "do_not_reflow": false, + "expected_runtime_one_iter": 2437, + "name": "rocksdb_microbench_suite_14", + "regression_threshold": 10, + "sl_iterations": 3 + } +] diff --git a/librocksdb-sys/rocksdb/buckifier/buckify_rocksdb.py b/librocksdb-sys/rocksdb/buckifier/buckify_rocksdb.py new file mode 100755 index 0000000..a9e7b44 --- /dev/null +++ b/librocksdb-sys/rocksdb/buckifier/buckify_rocksdb.py @@ -0,0 +1,333 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +from __future__ import absolute_import, division, print_function, unicode_literals + +try: + from builtins import str +except ImportError: + from __builtin__ import str +import fnmatch +import json +import os +import sys + +from targets_builder import TARGETSBuilder + +from util import ColorString + +# This script generates TARGETS file for Buck. +# Buck is a build tool specifying dependencies among different build targets. +# User can pass extra dependencies as a JSON object via command line, and this +# script can include these dependencies in the generate TARGETS file. +# Usage: +# $python3 buckifier/buckify_rocksdb.py +# (This generates a TARGET file without user-specified dependency for unit +# tests.) +# $python3 buckifier/buckify_rocksdb.py \ +# '{"fake": { +# "extra_deps": [":test_dep", "//fakes/module:mock1"], +# "extra_compiler_flags": ["-DFOO_BAR", "-Os"] +# } +# }' +# (Generated TARGETS file has test_dep and mock1 as dependencies for RocksDB +# unit tests, and will use the extra_compiler_flags to compile the unit test +# source.) + +# tests to export as libraries for inclusion in other projects +_EXPORTED_TEST_LIBS = ["env_basic_test"] + +# Parse src.mk files as a Dictionary of +# VAR_NAME => list of files +def parse_src_mk(repo_path): + src_mk = repo_path + "/src.mk" + src_files = {} + for line in open(src_mk): + line = line.strip() + if len(line) == 0 or line[0] == "#": + continue + if "=" in line: + current_src = line.split("=")[0].strip() + src_files[current_src] = [] + elif ".c" in line: + src_path = line.split("\\")[0].strip() + src_files[current_src].append(src_path) + return src_files + + +# get all .cc / .c files +def get_cc_files(repo_path): + cc_files = [] + for root, _dirnames, filenames in os.walk( + repo_path + ): # noqa: B007 T25377293 Grandfathered in + root = root[(len(repo_path) + 1) :] + if "java" in root: + # Skip java + continue + for filename in fnmatch.filter(filenames, "*.cc"): + cc_files.append(os.path.join(root, filename)) + for filename in fnmatch.filter(filenames, "*.c"): + cc_files.append(os.path.join(root, filename)) + return cc_files + + +# Get non_parallel tests from Makefile +def get_non_parallel_tests(repo_path): + Makefile = repo_path + "/Makefile" + + s = set({}) + + found_non_parallel_tests = False + for line in open(Makefile): + line = line.strip() + if line.startswith("NON_PARALLEL_TEST ="): + found_non_parallel_tests = True + elif found_non_parallel_tests: + if line.endswith("\\"): + # remove the trailing \ + line = line[:-1] + line = line.strip() + s.add(line) + else: + # we consumed all the non_parallel tests + break + + return s + + +# Parse extra dependencies passed by user from command line +def get_dependencies(): + deps_map = {"": {"extra_deps": [], "extra_compiler_flags": []}} + if len(sys.argv) < 2: + return deps_map + + def encode_dict(data): + rv = {} + for k, v in data.items(): + if isinstance(v, dict): + v = encode_dict(v) + rv[k] = v + return rv + + extra_deps = json.loads(sys.argv[1], object_hook=encode_dict) + for target_alias, deps in extra_deps.items(): + deps_map[target_alias] = deps + return deps_map + + +# Prepare TARGETS file for buck +def generate_targets(repo_path, deps_map): + print(ColorString.info("Generating TARGETS")) + # parsed src.mk file + src_mk = parse_src_mk(repo_path) + # get all .cc files + cc_files = get_cc_files(repo_path) + # get non_parallel tests from Makefile + non_parallel_tests = get_non_parallel_tests(repo_path) + + if src_mk is None or cc_files is None or non_parallel_tests is None: + return False + + extra_argv = "" + if len(sys.argv) >= 2: + # Heuristically quote and canonicalize whitespace for inclusion + # in how the file was generated. + extra_argv = " '{0}'".format(" ".join(sys.argv[1].split())) + + TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path, extra_argv) + + # rocksdb_lib + TARGETS.add_library( + "rocksdb_lib", + src_mk["LIB_SOURCES"] + + # always add range_tree, it's only excluded on ppc64, which we don't use internally + src_mk["RANGE_TREE_SOURCES"] + src_mk["TOOL_LIB_SOURCES"], + deps=[ + "//folly/container:f14_hash", + "//folly/experimental/coro:blocking_wait", + "//folly/experimental/coro:collect", + "//folly/experimental/coro:coroutine", + "//folly/experimental/coro:task", + "//folly/synchronization:distributed_mutex", + ], + ) + # rocksdb_whole_archive_lib + TARGETS.add_library( + "rocksdb_whole_archive_lib", + [], + deps=[ + ":rocksdb_lib", + ], + headers=None, + extra_external_deps="", + link_whole=True, + ) + # rocksdb_test_lib + TARGETS.add_library( + "rocksdb_test_lib", + src_mk.get("MOCK_LIB_SOURCES", []) + + src_mk.get("TEST_LIB_SOURCES", []) + + src_mk.get("EXP_LIB_SOURCES", []) + + src_mk.get("ANALYZER_LIB_SOURCES", []), + [":rocksdb_lib"], + extra_test_libs=True, + ) + # rocksdb_tools_lib + TARGETS.add_library( + "rocksdb_tools_lib", + src_mk.get("BENCH_LIB_SOURCES", []) + + src_mk.get("ANALYZER_LIB_SOURCES", []) + + ["test_util/testutil.cc"], + [":rocksdb_lib"], + ) + # rocksdb_cache_bench_tools_lib + TARGETS.add_library( + "rocksdb_cache_bench_tools_lib", + src_mk.get("CACHE_BENCH_LIB_SOURCES", []), + [":rocksdb_lib"], + ) + # rocksdb_stress_lib + TARGETS.add_rocksdb_library( + "rocksdb_stress_lib", + src_mk.get("ANALYZER_LIB_SOURCES", []) + + src_mk.get("STRESS_LIB_SOURCES", []) + + ["test_util/testutil.cc"], + ) + # db_stress binary + TARGETS.add_binary( + "db_stress", ["db_stress_tool/db_stress.cc"], [":rocksdb_stress_lib"] + ) + # bench binaries + for src in src_mk.get("MICROBENCH_SOURCES", []): + name = src.rsplit("/", 1)[1].split(".")[0] if "/" in src else src.split(".")[0] + TARGETS.add_binary(name, [src], [], extra_bench_libs=True) + print("Extra dependencies:\n{0}".format(json.dumps(deps_map))) + + # Dictionary test executable name -> relative source file path + test_source_map = {} + + # c_test.c is added through TARGETS.add_c_test(). If there + # are more than one .c test file, we need to extend + # TARGETS.add_c_test() to include other C tests too. + for test_src in src_mk.get("TEST_MAIN_SOURCES_C", []): + if test_src != "db/c_test.c": + print("Don't know how to deal with " + test_src) + return False + TARGETS.add_c_test() + + try: + with open(f"{repo_path}/buckifier/bench.json") as json_file: + fast_fancy_bench_config_list = json.load(json_file) + for config_dict in fast_fancy_bench_config_list: + clean_benchmarks = {} + benchmarks = config_dict["benchmarks"] + for binary, benchmark_dict in benchmarks.items(): + clean_benchmarks[binary] = {} + for benchmark, overloaded_metric_list in benchmark_dict.items(): + clean_benchmarks[binary][benchmark] = [] + for metric in overloaded_metric_list: + if not isinstance(metric, dict): + clean_benchmarks[binary][benchmark].append(metric) + TARGETS.add_fancy_bench_config( + config_dict["name"], + clean_benchmarks, + False, + config_dict["expected_runtime_one_iter"], + config_dict["sl_iterations"], + config_dict["regression_threshold"], + ) + + with open(f"{repo_path}/buckifier/bench-slow.json") as json_file: + slow_fancy_bench_config_list = json.load(json_file) + for config_dict in slow_fancy_bench_config_list: + clean_benchmarks = {} + benchmarks = config_dict["benchmarks"] + for binary, benchmark_dict in benchmarks.items(): + clean_benchmarks[binary] = {} + for benchmark, overloaded_metric_list in benchmark_dict.items(): + clean_benchmarks[binary][benchmark] = [] + for metric in overloaded_metric_list: + if not isinstance(metric, dict): + clean_benchmarks[binary][benchmark].append(metric) + for config_dict in slow_fancy_bench_config_list: + TARGETS.add_fancy_bench_config( + config_dict["name"] + "_slow", + clean_benchmarks, + True, + config_dict["expected_runtime_one_iter"], + config_dict["sl_iterations"], + config_dict["regression_threshold"], + ) + # it is better servicelab experiments break + # than rocksdb github ci + except Exception: + pass + + TARGETS.add_test_header() + + for test_src in src_mk.get("TEST_MAIN_SOURCES", []): + test = test_src.split(".c")[0].strip().split("/")[-1].strip() + test_source_map[test] = test_src + print("" + test + " " + test_src) + + for target_alias, deps in deps_map.items(): + for test, test_src in sorted(test_source_map.items()): + if len(test) == 0: + print(ColorString.warning("Failed to get test name for %s" % test_src)) + continue + + test_target_name = test if not target_alias else test + "_" + target_alias + + if test in _EXPORTED_TEST_LIBS: + test_library = "%s_lib" % test_target_name + TARGETS.add_library( + test_library, + [test_src], + deps=[":rocksdb_test_lib"], + extra_test_libs=True, + ) + TARGETS.register_test( + test_target_name, + test_src, + deps=json.dumps(deps["extra_deps"] + [":" + test_library]), + extra_compiler_flags=json.dumps(deps["extra_compiler_flags"]), + ) + else: + TARGETS.register_test( + test_target_name, + test_src, + deps=json.dumps(deps["extra_deps"] + [":rocksdb_test_lib"]), + extra_compiler_flags=json.dumps(deps["extra_compiler_flags"]), + ) + + print(ColorString.info("Generated TARGETS Summary:")) + print(ColorString.info("- %d libs" % TARGETS.total_lib)) + print(ColorString.info("- %d binarys" % TARGETS.total_bin)) + print(ColorString.info("- %d tests" % TARGETS.total_test)) + return True + + +def get_rocksdb_path(): + # rocksdb = {script_dir}/.. + script_dir = os.path.dirname(sys.argv[0]) + script_dir = os.path.abspath(script_dir) + rocksdb_path = os.path.abspath(os.path.join(script_dir, "../")) + + return rocksdb_path + + +def exit_with_error(msg): + print(ColorString.error(msg)) + sys.exit(1) + + +def main(): + deps_map = get_dependencies() + # Generate TARGETS file for buck + ok = generate_targets(get_rocksdb_path(), deps_map) + if not ok: + exit_with_error("Failed to generate TARGETS files") + + +if __name__ == "__main__": + main() diff --git a/librocksdb-sys/rocksdb/buckifier/check_buck_targets.sh b/librocksdb-sys/rocksdb/buckifier/check_buck_targets.sh new file mode 100755 index 0000000..66c83c5 --- /dev/null +++ b/librocksdb-sys/rocksdb/buckifier/check_buck_targets.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# If clang_format_diff.py command is not specfied, we assume we are able to +# access directly without any path. + +TGT_DIFF=`git diff TARGETS | head -n 1` + +if [ ! -z "$TGT_DIFF" ] +then + echo "TARGETS file has uncommitted changes. Skip this check." + exit 0 +fi + +echo Backup original TARGETS file. + +cp TARGETS TARGETS.bkp + +${PYTHON:-python3} buckifier/buckify_rocksdb.py + +TGT_DIFF=`git diff TARGETS | head -n 1` + +if [ -z "$TGT_DIFF" ] +then + mv TARGETS.bkp TARGETS + exit 0 +else + echo "Please run '${PYTHON:-python3} buckifier/buckify_rocksdb.py' to update TARGETS file." + echo "Do not manually update TARGETS file." + ${PYTHON:-python3} --version + mv TARGETS.bkp TARGETS + exit 1 +fi diff --git a/librocksdb-sys/rocksdb/buckifier/rocks_test_runner.sh b/librocksdb-sys/rocksdb/buckifier/rocks_test_runner.sh new file mode 100755 index 0000000..77f8f23 --- /dev/null +++ b/librocksdb-sys/rocksdb/buckifier/rocks_test_runner.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# Create a tmp directory for the test to use +TEST_DIR=$(mktemp -d /dev/shm/fbcode_rocksdb_XXXXXXX) +# shellcheck disable=SC2068 +TEST_TMPDIR="$TEST_DIR" $@ && rm -rf "$TEST_DIR" diff --git a/librocksdb-sys/rocksdb/buckifier/targets_builder.py b/librocksdb-sys/rocksdb/buckifier/targets_builder.py new file mode 100644 index 0000000..343b220 --- /dev/null +++ b/librocksdb-sys/rocksdb/buckifier/targets_builder.py @@ -0,0 +1,150 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +from __future__ import absolute_import, division, print_function, unicode_literals + +try: + from builtins import object, str +except ImportError: + from __builtin__ import object, str +import pprint + +import targets_cfg + + +def pretty_list(lst, indent=8): + if lst is None or len(lst) == 0: + return "" + + if len(lst) == 1: + return '"%s"' % lst[0] + + separator = '",\n%s"' % (" " * indent) + res = separator.join(sorted(lst)) + res = "\n" + (" " * indent) + '"' + res + '",\n' + (" " * (indent - 4)) + return res + + +class TARGETSBuilder(object): + def __init__(self, path, extra_argv): + self.path = path + header = targets_cfg.rocksdb_target_header_template.format( + extra_argv=extra_argv + ) + with open(path, "wb") as targets_file: + targets_file.write(header.encode("utf-8")) + self.total_lib = 0 + self.total_bin = 0 + self.total_test = 0 + self.tests_cfg = "" + + def add_library( + self, + name, + srcs, + deps=None, + headers=None, + extra_external_deps="", + link_whole=False, + external_dependencies=None, + extra_test_libs=False, + ): + if headers is not None: + headers = "[" + pretty_list(headers) + "]" + with open(self.path, "ab") as targets_file: + targets_file.write( + targets_cfg.library_template.format( + name=name, + srcs=pretty_list(srcs), + headers=headers, + deps=pretty_list(deps), + extra_external_deps=extra_external_deps, + link_whole=link_whole, + external_dependencies=pretty_list(external_dependencies), + extra_test_libs=extra_test_libs, + ).encode("utf-8") + ) + self.total_lib = self.total_lib + 1 + + def add_rocksdb_library(self, name, srcs, headers=None, external_dependencies=None): + if headers is not None: + headers = "[" + pretty_list(headers) + "]" + with open(self.path, "ab") as targets_file: + targets_file.write( + targets_cfg.rocksdb_library_template.format( + name=name, + srcs=pretty_list(srcs), + headers=headers, + external_dependencies=pretty_list(external_dependencies), + ).encode("utf-8") + ) + self.total_lib = self.total_lib + 1 + + def add_binary( + self, + name, + srcs, + deps=None, + extra_preprocessor_flags=None, + extra_bench_libs=False, + ): + with open(self.path, "ab") as targets_file: + targets_file.write( + targets_cfg.binary_template.format( + name=name, + srcs=pretty_list(srcs), + deps=pretty_list(deps), + extra_preprocessor_flags=pretty_list(extra_preprocessor_flags), + extra_bench_libs=extra_bench_libs, + ).encode("utf-8") + ) + self.total_bin = self.total_bin + 1 + + def add_c_test(self): + with open(self.path, "ab") as targets_file: + targets_file.write( + b""" +add_c_test_wrapper() +""" + ) + + def add_test_header(self): + with open(self.path, "ab") as targets_file: + targets_file.write( + b""" + # Generate a test rule for each entry in ROCKS_TESTS + # Do not build the tests in opt mode, since SyncPoint and other test code + # will not be included. +""" + ) + + def add_fancy_bench_config( + self, + name, + bench_config, + slow, + expected_runtime, + sl_iterations, + regression_threshold, + ): + with open(self.path, "ab") as targets_file: + targets_file.write( + targets_cfg.fancy_bench_template.format( + name=name, + bench_config=pprint.pformat(bench_config), + slow=slow, + expected_runtime=expected_runtime, + sl_iterations=sl_iterations, + regression_threshold=regression_threshold, + ).encode("utf-8") + ) + + def register_test(self, test_name, src, deps, extra_compiler_flags): + with open(self.path, "ab") as targets_file: + targets_file.write( + targets_cfg.unittests_template.format( + test_name=test_name, + test_cc=str(src), + deps=deps, + extra_compiler_flags=extra_compiler_flags, + ).encode("utf-8") + ) + self.total_test = self.total_test + 1 diff --git a/librocksdb-sys/rocksdb/buckifier/targets_cfg.py b/librocksdb-sys/rocksdb/buckifier/targets_cfg.py new file mode 100644 index 0000000..491c34d --- /dev/null +++ b/librocksdb-sys/rocksdb/buckifier/targets_cfg.py @@ -0,0 +1,41 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +from __future__ import absolute_import, division, print_function, unicode_literals + +rocksdb_target_header_template = """# This file \100generated by: +#$ python3 buckifier/buckify_rocksdb.py{extra_argv} +# --> DO NOT EDIT MANUALLY <-- +# This file is a Facebook-specific integration for buck builds, so can +# only be validated by Facebook employees. +# +# @noautodeps @nocodemods +load("//rocks/buckifier:defs.bzl", "cpp_library_wrapper","rocks_cpp_library_wrapper","cpp_binary_wrapper","cpp_unittest_wrapper","fancy_bench_wrapper","add_c_test_wrapper") + +""" + + +library_template = """ +cpp_library_wrapper(name="{name}", srcs=[{srcs}], deps=[{deps}], headers={headers}, link_whole={link_whole}, extra_test_libs={extra_test_libs}) +""" + +rocksdb_library_template = """ +rocks_cpp_library_wrapper(name="{name}", srcs=[{srcs}], headers={headers}) + +""" + + +binary_template = """ +cpp_binary_wrapper(name="{name}", srcs=[{srcs}], deps=[{deps}], extra_preprocessor_flags=[{extra_preprocessor_flags}], extra_bench_libs={extra_bench_libs}) +""" + +unittests_template = """ +cpp_unittest_wrapper(name="{test_name}", + srcs=["{test_cc}"], + deps={deps}, + extra_compiler_flags={extra_compiler_flags}) + +""" + +fancy_bench_template = """ +fancy_bench_wrapper(suite_name="{name}", binary_to_bench_to_metric_list_map={bench_config}, slow={slow}, expected_runtime={expected_runtime}, sl_iterations={sl_iterations}, regression_threshold={regression_threshold}) + +""" diff --git a/librocksdb-sys/rocksdb/buckifier/util.py b/librocksdb-sys/rocksdb/buckifier/util.py new file mode 100644 index 0000000..8943fed --- /dev/null +++ b/librocksdb-sys/rocksdb/buckifier/util.py @@ -0,0 +1,118 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +""" +This module keeps commonly used components. +""" +from __future__ import absolute_import, division, print_function, unicode_literals + +try: + from builtins import object +except ImportError: + from __builtin__ import object +import os +import subprocess +import sys +import time + + +class ColorString(object): + """Generate colorful strings on terminal""" + + HEADER = "\033[95m" + BLUE = "\033[94m" + GREEN = "\033[92m" + WARNING = "\033[93m" + FAIL = "\033[91m" + ENDC = "\033[0m" + + @staticmethod + def _make_color_str(text, color): + # In Python2, default encoding for unicode string is ASCII + if sys.version_info.major <= 2: + return "".join([color, text.encode("utf-8"), ColorString.ENDC]) + # From Python3, default encoding for unicode string is UTF-8 + return "".join([color, text, ColorString.ENDC]) + + @staticmethod + def ok(text): + if ColorString.is_disabled: + return text + return ColorString._make_color_str(text, ColorString.GREEN) + + @staticmethod + def info(text): + if ColorString.is_disabled: + return text + return ColorString._make_color_str(text, ColorString.BLUE) + + @staticmethod + def header(text): + if ColorString.is_disabled: + return text + return ColorString._make_color_str(text, ColorString.HEADER) + + @staticmethod + def error(text): + if ColorString.is_disabled: + return text + return ColorString._make_color_str(text, ColorString.FAIL) + + @staticmethod + def warning(text): + if ColorString.is_disabled: + return text + return ColorString._make_color_str(text, ColorString.WARNING) + + is_disabled = False + + +def run_shell_command(shell_cmd, cmd_dir=None): + """Run a single shell command. + @returns a tuple of shell command return code, stdout, stderr""" + + if cmd_dir is not None and not os.path.exists(cmd_dir): + run_shell_command("mkdir -p %s" % cmd_dir) + + start = time.time() + print("\t>>> Running: " + shell_cmd) + p = subprocess.Popen( # noqa + shell_cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=cmd_dir, + ) + stdout, stderr = p.communicate() + end = time.time() + + # Report time if we spent more than 5 minutes executing a command + execution_time = end - start + if execution_time > (60 * 5): + mins = execution_time / 60 + secs = execution_time % 60 + print("\t>time spent: %d minutes %d seconds" % (mins, secs)) + + return p.returncode, stdout, stderr + + +def run_shell_commands(shell_cmds, cmd_dir=None, verbose=False): + """Execute a sequence of shell commands, which is equivalent to + running `cmd1 && cmd2 && cmd3` + @returns boolean indication if all commands succeeds. + """ + + if cmd_dir: + print("\t=== Set current working directory => %s" % cmd_dir) + + for shell_cmd in shell_cmds: + ret_code, stdout, stderr = run_shell_command(shell_cmd, cmd_dir) + if stdout: + if verbose or ret_code != 0: + print(ColorString.info("stdout: \n"), stdout) + if stderr: + # contents in stderr is not necessarily to be error messages. + if verbose or ret_code != 0: + print(ColorString.error("stderr: \n"), stderr) + if ret_code != 0: + return False + + return True diff --git a/librocksdb-sys/rocksdb/build_tools/amalgamate.py b/librocksdb-sys/rocksdb/build_tools/amalgamate.py new file mode 100755 index 0000000..f79e907 --- /dev/null +++ b/librocksdb-sys/rocksdb/build_tools/amalgamate.py @@ -0,0 +1,168 @@ +#!/usr/bin/python +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +# amalgamate.py creates an amalgamation from a unity build. +# It can be run with either Python 2 or 3. +# An amalgamation consists of a header that includes the contents of all public +# headers and a source file that includes the contents of all source files and +# private headers. +# +# This script works by starting with the unity build file and recursively expanding +# #include directives. If the #include is found in a public include directory, +# that header is expanded into the amalgamation header. +# +# A particular header is only expanded once, so this script will +# break if there are multiple inclusions of the same header that are expected to +# expand differently. Similarly, this type of code causes issues: +# +# #ifdef FOO +# #include "bar.h" +# // code here +# #else +# #include "bar.h" // oops, doesn't get expanded +# // different code here +# #endif +# +# The solution is to move the include out of the #ifdef. + +from __future__ import print_function + +import argparse +import re +import sys +from os import path + +include_re = re.compile('^[ \t]*#include[ \t]+"(.*)"[ \t]*$') +included = set() +excluded = set() + + +def find_header(name, abs_path, include_paths): + samedir = path.join(path.dirname(abs_path), name) + if path.exists(samedir): + return samedir + for include_path in include_paths: + include_path = path.join(include_path, name) + if path.exists(include_path): + return include_path + return None + + +def expand_include( + include_path, + f, + abs_path, + source_out, + header_out, + include_paths, + public_include_paths, +): + if include_path in included: + return False + + included.add(include_path) + with open(include_path) as f: + print('#line 1 "{}"'.format(include_path), file=source_out) + process_file( + f, include_path, source_out, header_out, include_paths, public_include_paths + ) + return True + + +def process_file( + f, abs_path, source_out, header_out, include_paths, public_include_paths +): + for (line, text) in enumerate(f): + m = include_re.match(text) + if m: + filename = m.groups()[0] + # first check private headers + include_path = find_header(filename, abs_path, include_paths) + if include_path: + if include_path in excluded: + source_out.write(text) + expanded = False + else: + expanded = expand_include( + include_path, + f, + abs_path, + source_out, + header_out, + include_paths, + public_include_paths, + ) + else: + # now try public headers + include_path = find_header(filename, abs_path, public_include_paths) + if include_path: + # found public header + expanded = False + if include_path in excluded: + source_out.write(text) + else: + expand_include( + include_path, + f, + abs_path, + header_out, + None, + public_include_paths, + [], + ) + else: + sys.exit( + "unable to find {}, included in {} on line {}".format( + filename, abs_path, line + ) + ) + + if expanded: + print('#line {} "{}"'.format(line + 1, abs_path), file=source_out) + elif text != "#pragma once\n": + source_out.write(text) + + +def main(): + parser = argparse.ArgumentParser( + description="Transform a unity build into an amalgamation" + ) + parser.add_argument("source", help="source file") + parser.add_argument( + "-I", + action="append", + dest="include_paths", + help="include paths for private headers", + ) + parser.add_argument( + "-i", + action="append", + dest="public_include_paths", + help="include paths for public headers", + ) + parser.add_argument( + "-x", action="append", dest="excluded", help="excluded header files" + ) + parser.add_argument("-o", dest="source_out", help="output C++ file", required=True) + parser.add_argument( + "-H", dest="header_out", help="output C++ header file", required=True + ) + args = parser.parse_args() + + include_paths = list(map(path.abspath, args.include_paths or [])) + public_include_paths = list(map(path.abspath, args.public_include_paths or [])) + excluded.update(map(path.abspath, args.excluded or [])) + filename = args.source + abs_path = path.abspath(filename) + with open(filename) as f, open(args.source_out, "w") as source_out, open( + args.header_out, "w" + ) as header_out: + print('#line 1 "{}"'.format(filename), file=source_out) + print('#include "{}"'.format(header_out.name), file=source_out) + process_file( + f, abs_path, source_out, header_out, include_paths, public_include_paths + ) + + +if __name__ == "__main__": + main() diff --git a/librocksdb-sys/rocksdb/build_tools/benchmark_log_tool.py b/librocksdb-sys/rocksdb/build_tools/benchmark_log_tool.py new file mode 100755 index 0000000..d1ad459 --- /dev/null +++ b/librocksdb-sys/rocksdb/build_tools/benchmark_log_tool.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +# Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +# This source code is licensed under both the GPLv2 (found in the +# COPYING file in the root directory) and Apache 2.0 License +# (found in the LICENSE.Apache file in the root directory). + +"""Access the results of benchmark runs +Send these results on to OpenSearch graphing service +""" + +import argparse +import itertools +import logging +import os +import re +import sys + +import requests +from dateutil import parser + +logging.basicConfig(level=logging.DEBUG) + + +class Configuration: + opensearch_user = os.environ["ES_USER"] + opensearch_pass = os.environ["ES_PASS"] + + +class BenchmarkResultException(Exception): + def __init__(self, message, content): + super().__init__(self, message) + self.content = content + + +class BenchmarkUtils: + + expected_keys = [ + "ops_sec", + "mb_sec", + "lsm_sz", + "blob_sz", + "c_wgb", + "w_amp", + "c_mbps", + "c_wsecs", + "c_csecs", + "b_rgb", + "b_wgb", + "usec_op", + "p50", + "p99", + "p99.9", + "p99.99", + "pmax", + "uptime", + "stall%", + "Nstall", + "u_cpu", + "s_cpu", + "rss", + "test", + "date", + "version", + "job_id", + ] + + def sanity_check(row): + if "test" not in row: + logging.debug(f"not 'test' in row: {row}") + return False + if row["test"] == "": + logging.debug(f"row['test'] == '': {row}") + return False + if "date" not in row: + logging.debug(f"not 'date' in row: {row}") + return False + if "ops_sec" not in row: + logging.debug(f"not 'ops_sec' in row: {row}") + return False + try: + _ = int(row["ops_sec"]) + except (ValueError, TypeError): + logging.debug(f"int(row['ops_sec']): {row}") + return False + try: + (_, _) = parser.parse(row["date"], fuzzy_with_tokens=True) + except (parser.ParserError): + logging.error( + f"parser.parse((row['date']): not a valid format for date in row: {row}" + ) + return False + return True + + def conform_opensearch(row): + (dt, _) = parser.parse(row["date"], fuzzy_with_tokens=True) + # create a test_date field, which was previously what was expected + # repair the date field, which has what can be a WRONG ISO FORMAT, (no leading 0 on single-digit day-of-month) + # e.g. 2022-07-1T00:14:55 should be 2022-07-01T00:14:55 + row["test_date"] = dt.isoformat() + row["date"] = dt.isoformat() + return {key.replace(".", "_"): value for key, value in row.items()} + + +class ResultParser: + def __init__(self, field="(\w|[+-:.%])+", intrafield="(\s)+", separator="\t"): + self.field = re.compile(field) + self.intra = re.compile(intrafield) + self.sep = re.compile(separator) + + def ignore(self, l_in: str): + if len(l_in) == 0: + return True + if l_in[0:1] == "#": + return True + return False + + def line(self, line_in: str): + """Parse a line into items + Being clever about separators + """ + line = line_in + row = [] + while line != "": + match_item = self.field.match(line) + if match_item: + item = match_item.group(0) + row.append(item) + line = line[len(item) :] + else: + match_intra = self.intra.match(line) + if match_intra: + intra = match_intra.group(0) + # Count the separators + # If there are >1 then generate extra blank fields + # White space with no true separators fakes up a single separator + tabbed = self.sep.split(intra) + sep_count = len(tabbed) - 1 + if sep_count == 0: + sep_count = 1 + for _ in range(sep_count - 1): + row.append("") + line = line[len(intra) :] + else: + raise BenchmarkResultException( + "Invalid TSV line", f"{line_in} at {line}" + ) + return row + + def parse(self, lines): + """Parse something that iterates lines""" + rows = [self.line(line) for line in lines if not self.ignore(line)] + header = rows[0] + width = len(header) + records = [ + {k: v for (k, v) in itertools.zip_longest(header, row[:width])} + for row in rows[1:] + ] + return records + + +def load_report_from_tsv(filename: str): + file = open(filename, "r") + contents = file.readlines() + file.close() + parser = ResultParser() + report = parser.parse(contents) + logging.debug(f"Loaded TSV Report: {report}") + return report + + +def push_report_to_opensearch(report, esdocument): + sanitized = [ + BenchmarkUtils.conform_opensearch(row) + for row in report + if BenchmarkUtils.sanity_check(row) + ] + logging.debug( + f"upload {len(sanitized)} sane of {len(report)} benchmarks to opensearch" + ) + for single_benchmark in sanitized: + logging.debug(f"upload benchmark: {single_benchmark}") + response = requests.post( + esdocument, + json=single_benchmark, + auth=(os.environ["ES_USER"], os.environ["ES_PASS"]), + ) + logging.debug( + f"Sent to OpenSearch, status: {response.status_code}, result: {response.text}" + ) + response.raise_for_status() + + +def push_report_to_null(report): + + for row in report: + if BenchmarkUtils.sanity_check(row): + logging.debug(f"row {row}") + conformed = BenchmarkUtils.conform_opensearch(row) + logging.debug(f"conformed row {conformed}") + + +def main(): + """Tool for fetching, parsing and uploading benchmark results to OpenSearch / ElasticSearch + This tool will + + (1) Open a local tsv benchmark report file + (2) Upload to OpenSearch document, via https/JSON + """ + + parser = argparse.ArgumentParser(description="CircleCI benchmark scraper.") + + # --tsvfile is the name of the file to read results from + # --esdocument is the ElasticSearch document to push these results into + # + parser.add_argument( + "--tsvfile", + default="build_tools/circle_api_scraper_input.txt", + help="File from which to read tsv report", + ) + parser.add_argument( + "--esdocument", + help="ElasticSearch/OpenSearch document URL to upload report into", + ) + parser.add_argument( + "--upload", choices=["opensearch", "none"], default="opensearch" + ) + + args = parser.parse_args() + logging.debug(f"Arguments: {args}") + reports = load_report_from_tsv(args.tsvfile) + if args.upload == "opensearch": + push_report_to_opensearch(reports, args.esdocument) + else: + push_report_to_null(reports) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/librocksdb-sys/rocksdb/build_tools/build_detect_platform b/librocksdb-sys/rocksdb/build_tools/build_detect_platform new file mode 100755 index 0000000..a9a49e2 --- /dev/null +++ b/librocksdb-sys/rocksdb/build_tools/build_detect_platform @@ -0,0 +1,811 @@ +#!/usr/bin/env bash +# +# Detects OS we're compiling on and outputs a file specified by the first +# argument, which in turn gets read while processing Makefile. +# +# The output will set the following variables: +# CC C Compiler path +# CXX C++ Compiler path +# PLATFORM_LDFLAGS Linker flags +# JAVA_LDFLAGS Linker flags for RocksDBJava +# JAVA_STATIC_LDFLAGS Linker flags for RocksDBJava static build +# JAVAC_ARGS Arguments for javac +# PLATFORM_SHARED_EXT Extension for shared libraries +# PLATFORM_SHARED_LDFLAGS Flags for building shared library +# PLATFORM_SHARED_CFLAGS Flags for compiling objects for shared library +# PLATFORM_CCFLAGS C compiler flags +# PLATFORM_CXXFLAGS C++ compiler flags. Will contain: +# PLATFORM_SHARED_VERSIONED Set to 'true' if platform supports versioned +# shared libraries, empty otherwise. +# FIND Command for the find utility +# WATCH Command for the watch utility +# +# The PLATFORM_CCFLAGS and PLATFORM_CXXFLAGS might include the following: +# +# -DROCKSDB_PLATFORM_POSIX if posix-platform based +# -DSNAPPY if the Snappy library is present +# -DLZ4 if the LZ4 library is present +# -DZSTD if the ZSTD library is present +# -DNUMA if the NUMA library is present +# -DTBB if the TBB library is present +# -DMEMKIND if the memkind library is present +# +# Using gflags in rocksdb: +# Our project depends on gflags, which requires users to take some extra steps +# before they can compile the whole repository: +# 1. Install gflags. You may download it from here: +# https://gflags.github.io/gflags/ (Mac users can `brew install gflags`) +# 2. Once installed, add the include path for gflags to your CPATH env var and +# the lib path to LIBRARY_PATH. If installed with default settings, the lib +# will be /usr/local/lib and the include path will be /usr/local/include + +OUTPUT=$1 +if test -z "$OUTPUT"; then + echo "usage: $0 " >&2 + exit 1 +fi + +# we depend on C++17, but should be compatible with newer standards +if [ "$ROCKSDB_CXX_STANDARD" ]; then + PLATFORM_CXXFLAGS="-std=$ROCKSDB_CXX_STANDARD" +else + PLATFORM_CXXFLAGS="-std=c++17" +fi + +# we currently depend on POSIX platform +COMMON_FLAGS="-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX" + +# Default to fbcode gcc on internal fb machines +if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then + FBCODE_BUILD="true" + # If we're compiling with TSAN or shared lib, we need pic build + PIC_BUILD=$COMPILE_WITH_TSAN + if [ "$LIB_MODE" == "shared" ]; then + PIC_BUILD=1 + fi + source "$PWD/build_tools/fbcode_config_platform010.sh" +fi + +# Delete existing output, if it exists +rm -f "$OUTPUT" +touch "$OUTPUT" + +if test -z "$CC"; then + if [ -x "$(command -v cc)" ]; then + CC=cc + elif [ -x "$(command -v clang)" ]; then + CC=clang + else + CC=cc + fi +fi + +if test -z "$CXX"; then + if [ -x "$(command -v g++)" ]; then + CXX=g++ + elif [ -x "$(command -v clang++)" ]; then + CXX=clang++ + else + CXX=g++ + fi +fi + +if test -z "$AR"; then + if [ -x "$(command -v gcc-ar)" ]; then + AR=gcc-ar + elif [ -x "$(command -v llvm-ar)" ]; then + AR=llvm-ar + else + AR=ar + fi +fi + +# Detect OS +if test -z "$TARGET_OS"; then + TARGET_OS=`uname -s` +fi + +if test -z "$TARGET_ARCHITECTURE"; then + TARGET_ARCHITECTURE=`uname -m` +fi + +if test -z "$CLANG_SCAN_BUILD"; then + CLANG_SCAN_BUILD=scan-build +fi + +if test -z "$CLANG_ANALYZER"; then + CLANG_ANALYZER=$(command -v clang++ 2> /dev/null) +fi + +if test -z "$FIND"; then + FIND=find +fi + +if test -z "$WATCH"; then + WATCH=watch +fi + +COMMON_FLAGS="$COMMON_FLAGS ${CFLAGS}" +CROSS_COMPILE= +PLATFORM_CCFLAGS= +PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS" +PLATFORM_SHARED_EXT="so" +PLATFORM_SHARED_LDFLAGS="-Wl,--no-as-needed -shared -Wl,-soname -Wl," +PLATFORM_SHARED_CFLAGS="-fPIC" +PLATFORM_SHARED_VERSIONED=true + +# generic port files (working on all platform by #ifdef) go directly in /port +GENERIC_PORT_FILES=`cd "$ROCKSDB_ROOT"; find port -name '*.cc' | tr "\n" " "` + +# On GCC, we pick libc's memcmp over GCC's memcmp via -fno-builtin-memcmp +case "$TARGET_OS" in + Darwin) + PLATFORM=OS_MACOSX + COMMON_FLAGS="$COMMON_FLAGS -DOS_MACOSX" + PLATFORM_SHARED_EXT=dylib + PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name " + # PORT_FILES=port/darwin/darwin_specific.cc + ;; + IOS) + PLATFORM=IOS + COMMON_FLAGS="$COMMON_FLAGS -DOS_MACOSX -DIOS_CROSS_COMPILE " + PLATFORM_SHARED_EXT=dylib + PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name " + CROSS_COMPILE=true + PLATFORM_SHARED_VERSIONED= + ;; + Linux) + PLATFORM=OS_LINUX + COMMON_FLAGS="$COMMON_FLAGS -DOS_LINUX" + if [ -z "$USE_CLANG" ]; then + COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp" + else + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic" + fi + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt -ldl" + if test -z "$ROCKSDB_USE_IO_URING"; then + ROCKSDB_USE_IO_URING=1 + fi + if test "$ROCKSDB_USE_IO_URING" -ne 0; then + # check for liburing + $CXX $PLATFORM_CXXFLAGS -x c++ - -luring -o test.o 2>/dev/null < + int main() { + struct io_uring ring; + io_uring_queue_init(1, &ring, 0); + return 0; + } +EOF + if [ "$?" = 0 ]; then + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -luring" + COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_IOURING_PRESENT" + fi + fi + # PORT_FILES=port/linux/linux_specific.cc + ;; + SunOS) + PLATFORM=OS_SOLARIS + COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS -m64" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt -static-libstdc++ -static-libgcc -m64" + # PORT_FILES=port/sunos/sunos_specific.cc + ;; + AIX) + PLATFORM=OS_AIX + CC=gcc + COMMON_FLAGS="$COMMON_FLAGS -maix64 -pthread -fno-builtin-memcmp -D_REENTRANT -DOS_AIX -D__STDC_FORMAT_MACROS" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread -lpthread -lrt -maix64 -static-libstdc++ -static-libgcc" + # PORT_FILES=port/aix/aix_specific.cc + ;; + FreeBSD) + PLATFORM=OS_FREEBSD + CXX=clang++ + COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread" + # PORT_FILES=port/freebsd/freebsd_specific.cc + ;; + GNU/kFreeBSD) + PLATFORM=OS_GNU_KFREEBSD + COMMON_FLAGS="$COMMON_FLAGS -DOS_GNU_KFREEBSD" + if [ -z "$USE_CLANG" ]; then + COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp" + else + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic" + fi + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt" + # PORT_FILES=port/gnu_kfreebsd/gnu_kfreebsd_specific.cc + ;; + NetBSD) + PLATFORM=OS_NETBSD + COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_NETBSD" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lgcc_s" + # PORT_FILES=port/netbsd/netbsd_specific.cc + ;; + OpenBSD) + PLATFORM=OS_OPENBSD + CXX=clang++ + COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_OPENBSD" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread" + # PORT_FILES=port/openbsd/openbsd_specific.cc + FIND=gfind + WATCH=gnuwatch + ;; + DragonFly) + PLATFORM=OS_DRAGONFLYBSD + COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_DRAGONFLYBSD" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread" + # PORT_FILES=port/dragonfly/dragonfly_specific.cc + ;; + Cygwin) + PLATFORM=CYGWIN + PLATFORM_SHARED_CFLAGS="" + PLATFORM_CXXFLAGS="-std=gnu++11" + COMMON_FLAGS="$COMMON_FLAGS -DCYGWIN" + if [ -z "$USE_CLANG" ]; then + COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp" + else + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic" + fi + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt" + # PORT_FILES=port/linux/linux_specific.cc + ;; + OS_ANDROID_CROSSCOMPILE) + PLATFORM=OS_ANDROID + COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_ANDROID -DROCKSDB_PLATFORM_POSIX" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS " # All pthread features are in the Android C library + # PORT_FILES=port/android/android.cc + CROSS_COMPILE=true + ;; + *) + echo "Unknown platform!" >&2 + exit 1 +esac + +PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS ${CXXFLAGS}" +JAVA_LDFLAGS="$PLATFORM_LDFLAGS" +JAVA_STATIC_LDFLAGS="$PLATFORM_LDFLAGS" +JAVAC_ARGS="-source 8" + +if [ "$CROSS_COMPILE" = "true" -o "$FBCODE_BUILD" = "true" ]; then + # Cross-compiling; do not try any compilation tests. + # Also don't need any compilation tests if compiling on fbcode + if [ "$FBCODE_BUILD" = "true" ]; then + # Enable backtrace on fbcode since the necessary libraries are present + COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE" + FOLLY_DIR="third-party/folly" + fi + true +else + if ! test $ROCKSDB_DISABLE_FALLOCATE; then + # Test whether fallocate is available + $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null < + #include + int main() { + int fd = open("/dev/null", 0); + fallocate(fd, FALLOC_FL_KEEP_SIZE, 0, 1024); + } +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_FALLOCATE_PRESENT" + fi + fi + + if ! test $ROCKSDB_DISABLE_SNAPPY; then + # Test whether Snappy library is installed + # http://code.google.com/p/snappy/ + $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null < + int main() {} +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lsnappy" + JAVA_LDFLAGS="$JAVA_LDFLAGS -lsnappy" + fi + fi + + if ! test $ROCKSDB_DISABLE_GFLAGS; then + # Test whether gflags library is installed + # http://gflags.github.io/gflags/ + # check if the namespace is gflags + if $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null << EOF + #include + using namespace GFLAGS_NAMESPACE; + int main() {} +EOF + then + COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags" + # check if namespace is gflags + elif $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null << EOF + #include + using namespace gflags; + int main() {} +EOF + then + COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1 -DGFLAGS_NAMESPACE=gflags" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags" + # check if namespace is google + elif $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null << EOF + #include + using namespace google; + int main() {} +EOF + then + COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1 -DGFLAGS_NAMESPACE=google" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags" + fi + fi + + if ! test $ROCKSDB_DISABLE_ZLIB; then + # Test whether zlib library is installed + $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o test.o 2>/dev/null < + int main() {} +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DZLIB" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lz" + JAVA_LDFLAGS="$JAVA_LDFLAGS -lz" + fi + fi + + if ! test $ROCKSDB_DISABLE_BZIP; then + # Test whether bzip library is installed + $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o test.o 2>/dev/null < + int main() {} +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DBZIP2" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbz2" + JAVA_LDFLAGS="$JAVA_LDFLAGS -lbz2" + fi + fi + + if ! test $ROCKSDB_DISABLE_LZ4; then + # Test whether lz4 library is installed + $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o test.o 2>/dev/null < + #include + int main() {} +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DLZ4" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -llz4" + JAVA_LDFLAGS="$JAVA_LDFLAGS -llz4" + fi + fi + + if ! test $ROCKSDB_DISABLE_ZSTD; then + # Test whether zstd library is installed + $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null < + int main() {} +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DZSTD" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lzstd" + JAVA_LDFLAGS="$JAVA_LDFLAGS -lzstd" + fi + fi + + if ! test $ROCKSDB_DISABLE_NUMA; then + # Test whether numa is available + $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o -lnuma 2>/dev/null < + #include + int main() {} +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DNUMA" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lnuma" + JAVA_LDFLAGS="$JAVA_LDFLAGS -lnuma" + fi + fi + + if ! test $ROCKSDB_DISABLE_TBB; then + # Test whether tbb is available + $CXX $PLATFORM_CXXFLAGS $LDFLAGS -x c++ - -o test.o -ltbb 2>/dev/null < + int main() {} +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DTBB" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltbb" + JAVA_LDFLAGS="$JAVA_LDFLAGS -ltbb" + fi + fi + + if ! test $ROCKSDB_DISABLE_JEMALLOC; then + # Test whether jemalloc is available + if echo 'int main() {}' | $CXX $PLATFORM_CXXFLAGS $LDFLAGS -x c++ - -o test.o -ljemalloc \ + 2>/dev/null; then + # This will enable some preprocessor identifiers in the Makefile + JEMALLOC=1 + # JEMALLOC can be enabled either using the flag (like here) or by + # providing direct link to the jemalloc library + WITH_JEMALLOC_FLAG=1 + # check for JEMALLOC installed with HomeBrew + if [ "$PLATFORM" == "OS_MACOSX" ]; then + if [ "$TARGET_ARCHITECTURE" = "arm64" ]; then + # on M1 Macs, homebrew installs here instead of /usr/local + JEMALLOC_PREFIX="/opt/homebrew" + else + JEMALLOC_PREFIX="/usr/local" + fi + if hash brew 2>/dev/null && brew ls --versions jemalloc > /dev/null; then + JEMALLOC_VER=$(brew ls --versions jemalloc | tail -n 1 | cut -f 2 -d ' ') + JEMALLOC_INCLUDE="-I${JEMALLOC_PREFIX}/Cellar/jemalloc/${JEMALLOC_VER}/include" + JEMALLOC_LIB="${JEMALLOC_PREFIX}/Cellar/jemalloc/${JEMALLOC_VER}/lib/libjemalloc_pic.a" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -L${JEMALLOC_PREFIX}/lib $JEMALLOC_LIB" + JAVA_LDFLAGS="$JAVA_LDFLAGS -L${JEMALLOC_PREFIX}/lib $JEMALLOC_LIB" + JAVA_STATIC_LDFLAGS="$JAVA_STATIC_LDFLAGS -L${JEMALLOC_PREFIX}/lib $JEMALLOC_LIB" + fi + fi + fi + fi + if ! test $JEMALLOC && ! test $ROCKSDB_DISABLE_TCMALLOC; then + # jemalloc is not available. Let's try tcmalloc + if echo 'int main() {}' | $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o \ + -ltcmalloc 2>/dev/null; then + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc" + JAVA_LDFLAGS="$JAVA_LDFLAGS -ltcmalloc" + fi + fi + + if ! test $ROCKSDB_DISABLE_MALLOC_USABLE_SIZE; then + # Test whether malloc_usable_size is available + $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null < + int main() { + size_t res = malloc_usable_size(0); + (void)res; + return 0; + } +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_MALLOC_USABLE_SIZE" + fi + fi + + if ! test $ROCKSDB_DISABLE_MEMKIND; then + # Test whether memkind library is installed + $CXX $PLATFORM_CXXFLAGS $LDFLAGS -x c++ - -o test.o -lmemkind 2>/dev/null < + int main() { + memkind_malloc(MEMKIND_DAX_KMEM, 1024); + return 0; + } +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DMEMKIND" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lmemkind" + JAVA_LDFLAGS="$JAVA_LDFLAGS -lmemkind" + fi + fi + + if ! test $ROCKSDB_DISABLE_PTHREAD_MUTEX_ADAPTIVE_NP; then + # Test whether PTHREAD_MUTEX_ADAPTIVE_NP mutex type is available + $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null < + int main() { + int x = PTHREAD_MUTEX_ADAPTIVE_NP; + (void)x; + return 0; + } +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_PTHREAD_ADAPTIVE_MUTEX" + fi + fi + + if ! test $ROCKSDB_DISABLE_BACKTRACE; then + # Test whether backtrace is available + $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null < + int main() { + void* frames[1]; + backtrace_symbols(frames, backtrace(frames, 1)); + return 0; + } +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE" + else + # Test whether execinfo library is installed + $CXX $PLATFORM_CXXFLAGS -lexecinfo -x c++ - -o test.o 2>/dev/null < + int main() { + void* frames[1]; + backtrace_symbols(frames, backtrace(frames, 1)); + } +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lexecinfo" + JAVA_LDFLAGS="$JAVA_LDFLAGS -lexecinfo" + fi + fi + fi + + if ! test $ROCKSDB_DISABLE_PG; then + # Test if -pg is supported + $CXX $PLATFORM_CXXFLAGS -pg -x c++ - -o test.o 2>/dev/null </dev/null < + int main() { + int fd = open("/dev/null", 0); + sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE); + } +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_RANGESYNC_PRESENT" + fi + fi + + if ! test $ROCKSDB_DISABLE_SCHED_GETCPU; then + # Test whether sched_getcpu is supported + $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null < + int main() { + int cpuid = sched_getcpu(); + (void)cpuid; + } +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SCHED_GETCPU_PRESENT" + fi + fi + + if ! test $ROCKSDB_DISABLE_AUXV_GETAUXVAL; then + # Test whether getauxval is supported + $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null < + int main() { + uint64_t auxv = getauxval(AT_HWCAP); + (void)auxv; + } +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_AUXV_GETAUXVAL_PRESENT" + fi + fi + + if ! test $ROCKSDB_DISABLE_ALIGNED_NEW; then + # Test whether c++17 aligned-new is supported + $CXX $PLATFORM_CXXFLAGS -faligned-new -x c++ - -o test.o 2>/dev/null </dev/null < + int main() {} +EOF + if [ "$?" = 0 ]; then + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbenchmark" + fi + fi + if test $USE_FOLLY; then + # Test whether libfolly library is installed + $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null < + int main() {} +EOF + if [ "$?" != 0 ]; then + FOLLY_DIR="./third-party/folly" + fi + fi + +fi + +# TODO(tec): Fix -Wshorten-64-to-32 errors on FreeBSD and enable the warning. +# -Wshorten-64-to-32 breaks compilation on FreeBSD aarch64 and i386 +if ! { [ "$TARGET_OS" = FreeBSD ] && [ "$TARGET_ARCHITECTURE" = arm64 -o "$TARGET_ARCHITECTURE" = i386 ]; }; then + # Test whether -Wshorten-64-to-32 is available + $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o -Wshorten-64-to-32 2>/dev/null </dev/null; then + COMMON_FLAGS="$COMMON_FLAGS -march=native " + else + COMMON_FLAGS="$COMMON_FLAGS -march=z196 " + fi + COMMON_FLAGS="$COMMON_FLAGS" + elif test -n "`echo $TARGET_ARCHITECTURE | grep ^riscv64`"; then + RISC_ISA=$(cat /proc/cpuinfo | grep isa | head -1 | cut --delimiter=: -f 2 | cut -b 2-) + COMMON_FLAGS="$COMMON_FLAGS -march=${RISC_ISA}" + elif [ "$TARGET_OS" == "IOS" ]; then + COMMON_FLAGS="$COMMON_FLAGS" + else + COMMON_FLAGS="$COMMON_FLAGS -march=native " + fi +else + # PORTABLE specified + if [ "$PORTABLE" == 1 ]; then + if test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then + COMMON_FLAGS="$COMMON_FLAGS -march=z196 " + elif test -n "`echo $TARGET_ARCHITECTURE | grep ^riscv64`"; then + RISC_ISA=$(cat /proc/cpuinfo | grep isa | head -1 | cut --delimiter=: -f 2 | cut -b 2-) + COMMON_FLAGS="$COMMON_FLAGS -march=${RISC_ISA}" + elif test "$USE_SSE"; then + # USE_SSE is DEPRECATED + # This is a rough approximation of the old USE_SSE behavior + COMMON_FLAGS="$COMMON_FLAGS -march=haswell" + fi + # Other than those cases, not setting -march= here. + else + # Assume PORTABLE is a minimum assumed cpu type, e.g. PORTABLE=haswell + COMMON_FLAGS="$COMMON_FLAGS -march=${PORTABLE}" + fi + + if [[ "${PLATFORM}" == "OS_MACOSX" ]]; then + # For portability compile for macOS 10.14 or newer + COMMON_FLAGS="$COMMON_FLAGS -mmacosx-version-min=10.14" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -mmacosx-version-min=10.14" + # -mmacosx-version-min must come first here. + PLATFORM_SHARED_LDFLAGS="-mmacosx-version-min=10.14 $PLATFORM_SHARED_LDFLAGS" + PLATFORM_CMAKE_FLAGS="-DCMAKE_OSX_DEPLOYMENT_TARGET=10.14" + JAVA_STATIC_DEPS_COMMON_FLAGS="-mmacosx-version-min=10.14" + JAVA_STATIC_DEPS_LDFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS" + JAVA_STATIC_DEPS_CCFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS" + JAVA_STATIC_DEPS_CXXFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS" + fi +fi + +if test -n "`echo $TARGET_ARCHITECTURE | grep ^ppc64`"; then + # check for GNU libc on ppc64 + $CXX -x c++ - -o /dev/null 2>/dev/null < + #include + #include + + int main(int argc, char *argv[]) { + printf("GNU libc version: %s\n", gnu_get_libc_version()); + return 0; + } +EOF + if [ "$?" != 0 ]; then + PPC_LIBC_IS_GNU=0 + fi +fi + +$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o test.o 2>/dev/null < + int main() { + uint64_t a = 0xffffFFFFffffFFFF; + __uint128_t b = __uint128_t(a) * a; + a = static_cast(b >> 64); + (void)a; + } +EOF +if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DHAVE_UINT128_EXTENSION" +fi + +if [ "$FBCODE_BUILD" != "true" -a "$PLATFORM" = OS_LINUX ]; then + $CXX $COMMON_FLAGS $PLATFORM_SHARED_CFLAGS -x c++ -c - -o test_dl.o 2>/dev/null </dev/null + if [ "$?" = 0 ]; then + EXEC_LDFLAGS+="-ldl" + rm -f test_dl.o + fi + fi +fi + +# check for F_FULLFSYNC +$CXX $PLATFORM_CXXFALGS -x c++ - -o test.o 2>/dev/null < + int main() { + fcntl(0, F_FULLFSYNC); + return 0; + } +EOF +if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DHAVE_FULLFSYNC" +fi + +rm -f test.o test_dl.o + +# Get the path for the folly installation dir +if [ "$USE_FOLLY" ]; then + if [ "$FOLLY_DIR" ]; then + FOLLY_PATH=`cd $FOLLY_DIR && $PYTHON build/fbcode_builder/getdeps.py show-inst-dir folly` + fi +fi + +PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS" +PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS" + +VALGRIND_VER="$VALGRIND_VER" + +ROCKSDB_MAJOR=`build_tools/version.sh major` +ROCKSDB_MINOR=`build_tools/version.sh minor` +ROCKSDB_PATCH=`build_tools/version.sh patch` + +echo "CC=$CC" >> "$OUTPUT" +echo "CXX=$CXX" >> "$OUTPUT" +echo "AR=$AR" >> "$OUTPUT" +echo "PLATFORM=$PLATFORM" >> "$OUTPUT" +echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> "$OUTPUT" +echo "PLATFORM_CMAKE_FLAGS=$PLATFORM_CMAKE_FLAGS" >> "$OUTPUT" +echo "JAVA_LDFLAGS=$JAVA_LDFLAGS" >> "$OUTPUT" +echo "JAVA_STATIC_LDFLAGS=$JAVA_STATIC_LDFLAGS" >> "$OUTPUT" +echo "JAVA_STATIC_DEPS_CCFLAGS=$JAVA_STATIC_DEPS_CCFLAGS" >> "$OUTPUT" +echo "JAVA_STATIC_DEPS_CXXFLAGS=$JAVA_STATIC_DEPS_CXXFLAGS" >> "$OUTPUT" +echo "JAVA_STATIC_DEPS_LDFLAGS=$JAVA_STATIC_DEPS_LDFLAGS" >> "$OUTPUT" +echo "JAVAC_ARGS=$JAVAC_ARGS" >> "$OUTPUT" +echo "VALGRIND_VER=$VALGRIND_VER" >> "$OUTPUT" +echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> "$OUTPUT" +echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> "$OUTPUT" +echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> "$OUTPUT" +echo "PLATFORM_SHARED_EXT=$PLATFORM_SHARED_EXT" >> "$OUTPUT" +echo "PLATFORM_SHARED_LDFLAGS=$PLATFORM_SHARED_LDFLAGS" >> "$OUTPUT" +echo "PLATFORM_SHARED_VERSIONED=$PLATFORM_SHARED_VERSIONED" >> "$OUTPUT" +echo "EXEC_LDFLAGS=$EXEC_LDFLAGS" >> "$OUTPUT" +echo "JEMALLOC_INCLUDE=$JEMALLOC_INCLUDE" >> "$OUTPUT" +echo "JEMALLOC_LIB=$JEMALLOC_LIB" >> "$OUTPUT" +echo "ROCKSDB_MAJOR=$ROCKSDB_MAJOR" >> "$OUTPUT" +echo "ROCKSDB_MINOR=$ROCKSDB_MINOR" >> "$OUTPUT" +echo "ROCKSDB_PATCH=$ROCKSDB_PATCH" >> "$OUTPUT" +echo "CLANG_SCAN_BUILD=$CLANG_SCAN_BUILD" >> "$OUTPUT" +echo "CLANG_ANALYZER=$CLANG_ANALYZER" >> "$OUTPUT" +echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT" +echo "FIND=$FIND" >> "$OUTPUT" +echo "WATCH=$WATCH" >> "$OUTPUT" +echo "FOLLY_PATH=$FOLLY_PATH" >> "$OUTPUT" + +# This will enable some related identifiers for the preprocessor +if test -n "$JEMALLOC"; then + echo "JEMALLOC=1" >> "$OUTPUT" +fi +# Indicates that jemalloc should be enabled using -ljemalloc flag +# The alternative is to porvide a direct link to the library via JEMALLOC_LIB +# and JEMALLOC_INCLUDE +if test -n "$WITH_JEMALLOC_FLAG"; then + echo "WITH_JEMALLOC_FLAG=$WITH_JEMALLOC_FLAG" >> "$OUTPUT" +fi +echo "LUA_PATH=$LUA_PATH" >> "$OUTPUT" +if test -n "$USE_FOLLY"; then + echo "USE_FOLLY=$USE_FOLLY" >> "$OUTPUT" +fi +if test -n "$PPC_LIBC_IS_GNU"; then + echo "PPC_LIBC_IS_GNU=$PPC_LIBC_IS_GNU" >> "$OUTPUT" +fi diff --git a/librocksdb-sys/rocksdb/build_tools/check-sources.sh b/librocksdb-sys/rocksdb/build_tools/check-sources.sh new file mode 100755 index 0000000..5672f7b --- /dev/null +++ b/librocksdb-sys/rocksdb/build_tools/check-sources.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# +# Check for some simple mistakes that should prevent commit or push + +BAD="" + +git grep -n 'namespace rocksdb' -- '*.[ch]*' +if [ "$?" != "1" ]; then + echo "^^^^^ Do not hardcode namespace rocksdb. Use ROCKSDB_NAMESPACE" + BAD=1 +fi + +git grep -n -i 'nocommit' -- ':!build_tools/check-sources.sh' +if [ "$?" != "1" ]; then + echo "^^^^^ Code was not intended to be committed" + BAD=1 +fi + +git grep -n 'include :: Failure' + _GTEST_FAIL_PATTERN = re.compile(r"(unknown file|\S+:\d+): Failure$") + + def __init__(self): + self._last_gtest_name = "Unknown test" + + def parse_error(self, line): + gtest_name_match = self._GTEST_NAME_PATTERN.match(line) + if gtest_name_match: + self._last_gtest_name = gtest_name_match.group(1) + return None + gtest_fail_match = self._GTEST_FAIL_PATTERN.match(line) + if gtest_fail_match: + return "%s failed: %s" % (self._last_gtest_name, gtest_fail_match.group(1)) + return None + + +class MatchErrorParser(ErrorParserBase): + """A simple parser that returns the whole line if it matches the pattern.""" + + def __init__(self, pattern): + self._pattern = re.compile(pattern) + + def parse_error(self, line): + if self._pattern.match(line): + return line + return None + + +class CompilerErrorParser(MatchErrorParser): + def __init__(self): + # format (compile error): + # '::: error: ' + # format (link error): + # ':: error: ' + # The below regex catches both + super(CompilerErrorParser, self).__init__(r"\S+:\d+: error:") + + +class ScanBuildErrorParser(MatchErrorParser): + def __init__(self): + super(ScanBuildErrorParser, self).__init__(r"scan-build: \d+ bugs found.$") + + +class DbCrashErrorParser(MatchErrorParser): + def __init__(self): + super(DbCrashErrorParser, self).__init__(r"\*\*\*.*\^$|TEST FAILED.") + + +class WriteStressErrorParser(MatchErrorParser): + def __init__(self): + super(WriteStressErrorParser, self).__init__( + r"ERROR: write_stress died with exitcode=\d+" + ) + + +class AsanErrorParser(MatchErrorParser): + def __init__(self): + super(AsanErrorParser, self).__init__(r"==\d+==ERROR: AddressSanitizer:") + + +class UbsanErrorParser(MatchErrorParser): + def __init__(self): + # format: '::: runtime error: ' + super(UbsanErrorParser, self).__init__(r"\S+:\d+:\d+: runtime error:") + + +class ValgrindErrorParser(MatchErrorParser): + def __init__(self): + # just grab the summary, valgrind doesn't clearly distinguish errors + # from other log messages. + super(ValgrindErrorParser, self).__init__(r"==\d+== ERROR SUMMARY:") + + +class CompatErrorParser(MatchErrorParser): + def __init__(self): + super(CompatErrorParser, self).__init__(r"==== .*[Ee]rror.* ====$") + + +class TsanErrorParser(MatchErrorParser): + def __init__(self): + super(TsanErrorParser, self).__init__(r"WARNING: ThreadSanitizer:") + + +_TEST_NAME_TO_PARSERS = { + "punit": [CompilerErrorParser, GTestErrorParser], + "unit": [CompilerErrorParser, GTestErrorParser], + "release": [CompilerErrorParser, GTestErrorParser], + "unit_481": [CompilerErrorParser, GTestErrorParser], + "release_481": [CompilerErrorParser, GTestErrorParser], + "clang_unit": [CompilerErrorParser, GTestErrorParser], + "clang_release": [CompilerErrorParser, GTestErrorParser], + "clang_analyze": [CompilerErrorParser, ScanBuildErrorParser], + "code_cov": [CompilerErrorParser, GTestErrorParser], + "unity": [CompilerErrorParser, GTestErrorParser], + "lite": [CompilerErrorParser], + "lite_test": [CompilerErrorParser, GTestErrorParser], + "stress_crash": [CompilerErrorParser, DbCrashErrorParser], + "stress_crash_with_atomic_flush": [CompilerErrorParser, DbCrashErrorParser], + "stress_crash_with_txn": [CompilerErrorParser, DbCrashErrorParser], + "write_stress": [CompilerErrorParser, WriteStressErrorParser], + "asan": [CompilerErrorParser, GTestErrorParser, AsanErrorParser], + "asan_crash": [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser], + "asan_crash_with_atomic_flush": [ + CompilerErrorParser, + AsanErrorParser, + DbCrashErrorParser, + ], + "asan_crash_with_txn": [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser], + "ubsan": [CompilerErrorParser, GTestErrorParser, UbsanErrorParser], + "ubsan_crash": [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser], + "ubsan_crash_with_atomic_flush": [ + CompilerErrorParser, + UbsanErrorParser, + DbCrashErrorParser, + ], + "ubsan_crash_with_txn": [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser], + "valgrind": [CompilerErrorParser, GTestErrorParser, ValgrindErrorParser], + "tsan": [CompilerErrorParser, GTestErrorParser, TsanErrorParser], + "format_compatible": [CompilerErrorParser, CompatErrorParser], + "run_format_compatible": [CompilerErrorParser, CompatErrorParser], + "no_compression": [CompilerErrorParser, GTestErrorParser], + "run_no_compression": [CompilerErrorParser, GTestErrorParser], + "regression": [CompilerErrorParser], + "run_regression": [CompilerErrorParser], +} + + +def main(): + if len(sys.argv) != 2: + return "Usage: %s " % sys.argv[0] + test_name = sys.argv[1] + if test_name not in _TEST_NAME_TO_PARSERS: + return "Unknown test name: %s" % test_name + + error_parsers = [] + for parser_cls in _TEST_NAME_TO_PARSERS[test_name]: + error_parsers.append(parser_cls()) + + for line in sys.stdin: + line = line.strip() + for error_parser in error_parsers: + error_msg = error_parser.parse_error(line) + if error_msg is not None: + print(error_msg) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/librocksdb-sys/rocksdb/build_tools/fb_compile_mongo.sh b/librocksdb-sys/rocksdb/build_tools/fb_compile_mongo.sh new file mode 100755 index 0000000..ec733cd --- /dev/null +++ b/librocksdb-sys/rocksdb/build_tools/fb_compile_mongo.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# fail early +set -e + +if test -z $ROCKSDB_PATH; then + ROCKSDB_PATH=~/rocksdb +fi +source $ROCKSDB_PATH/build_tools/fbcode_config4.8.1.sh + +EXTRA_LDFLAGS="" + +if test -z $ALLOC; then + # default + ALLOC=tcmalloc +elif [[ $ALLOC == "jemalloc" ]]; then + ALLOC=system + EXTRA_LDFLAGS+=" -Wl,--whole-archive $JEMALLOC_LIB -Wl,--no-whole-archive" +fi + +# we need to force mongo to use static library, not shared +STATIC_LIB_DEP_DIR='build/static_library_dependencies' +test -d $STATIC_LIB_DEP_DIR || mkdir $STATIC_LIB_DEP_DIR +test -h $STATIC_LIB_DEP_DIR/`basename $SNAPPY_LIBS` || ln -s $SNAPPY_LIBS $STATIC_LIB_DEP_DIR +test -h $STATIC_LIB_DEP_DIR/`basename $LZ4_LIBS` || ln -s $LZ4_LIBS $STATIC_LIB_DEP_DIR + +EXTRA_LDFLAGS+=" -L $STATIC_LIB_DEP_DIR" + +set -x + +EXTRA_CMD="" +if ! test -e version.json; then + # this is Mongo 3.0 + EXTRA_CMD="--rocksdb \ + --variant-dir=linux2/norm + --cxx=${CXX} \ + --cc=${CC} \ + --use-system-zlib" # add this line back to normal code path + # when https://jira.mongodb.org/browse/SERVER-19123 is resolved +fi + +scons \ + LINKFLAGS="$EXTRA_LDFLAGS $EXEC_LDFLAGS $PLATFORM_LDFLAGS" \ + CCFLAGS="$CXXFLAGS -L $STATIC_LIB_DEP_DIR" \ + LIBS="lz4 gcc stdc++" \ + LIBPATH="$ROCKSDB_PATH" \ + CPPPATH="$ROCKSDB_PATH/include" \ + -j32 \ + --allocator=$ALLOC \ + --nostrip \ + --opt=on \ + --disable-minimum-compiler-version-enforcement \ + --use-system-snappy \ + --disable-warnings-as-errors \ + $EXTRA_CMD $* diff --git a/librocksdb-sys/rocksdb/build_tools/fbcode_config.sh b/librocksdb-sys/rocksdb/build_tools/fbcode_config.sh new file mode 100644 index 0000000..fa629af --- /dev/null +++ b/librocksdb-sys/rocksdb/build_tools/fbcode_config.sh @@ -0,0 +1,175 @@ +#!/bin/sh +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# +# Set environment variables so that we can compile rocksdb using +# fbcode settings. It uses the latest g++ and clang compilers and also +# uses jemalloc +# Environment variables that change the behavior of this script: +# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included + + +BASEDIR=`dirname $BASH_SOURCE` +source "$BASEDIR/dependencies.sh" + +CFLAGS="" + +# libgcc +LIBGCC_INCLUDE="$LIBGCC_BASE/include" +LIBGCC_LIBS=" -L $LIBGCC_BASE/lib" + +# glibc +GLIBC_INCLUDE="$GLIBC_BASE/include" +GLIBC_LIBS=" -L $GLIBC_BASE/lib" + +if ! test $ROCKSDB_DISABLE_SNAPPY; then + # snappy + SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/" + if test -z $PIC_BUILD; then + SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a" + else + SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a" + fi + CFLAGS+=" -DSNAPPY" +fi + +if test -z $PIC_BUILD; then + if ! test $ROCKSDB_DISABLE_ZLIB; then + # location of zlib headers and libraries + ZLIB_INCLUDE=" -I $ZLIB_BASE/include/" + ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a" + CFLAGS+=" -DZLIB" + fi + + if ! test $ROCKSDB_DISABLE_BZIP; then + # location of bzip headers and libraries + BZIP_INCLUDE=" -I $BZIP2_BASE/include/" + BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a" + CFLAGS+=" -DBZIP2" + fi + + if ! test $ROCKSDB_DISABLE_LZ4; then + LZ4_INCLUDE=" -I $LZ4_BASE/include/" + LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a" + CFLAGS+=" -DLZ4" + fi +fi + +if ! test $ROCKSDB_DISABLE_ZSTD; then + ZSTD_INCLUDE=" -I $ZSTD_BASE/include/" + if test -z $PIC_BUILD; then + ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a" + else + ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a" + fi + CFLAGS+=" -DZSTD -DZSTD_STATIC_LINKING_ONLY" +fi + +# location of gflags headers and libraries +GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/" +if test -z $PIC_BUILD; then + GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a" +else + GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a" +fi +CFLAGS+=" -DGFLAGS=gflags" + +# location of jemalloc +JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/" +JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a" + +if test -z $PIC_BUILD; then + # location of numa + NUMA_INCLUDE=" -I $NUMA_BASE/include/" + NUMA_LIB=" $NUMA_BASE/lib/libnuma.a" + CFLAGS+=" -DNUMA" + + # location of libunwind + LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a" +fi + +# location of TBB +TBB_INCLUDE=" -isystem $TBB_BASE/include/" +if test -z $PIC_BUILD; then + TBB_LIBS="$TBB_BASE/lib/libtbb.a" +else + TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a" +fi +CFLAGS+=" -DTBB" + +test "$USE_SSE" || USE_SSE=1 +export USE_SSE +test "$PORTABLE" || PORTABLE=1 +export PORTABLE + +BINUTILS="$BINUTILS_BASE/bin" +AR="$BINUTILS/ar" + +DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE" + +STDLIBS="-L $GCC_BASE/lib64" + +CLANG_BIN="$CLANG_BASE/bin" +CLANG_LIB="$CLANG_BASE/lib" +CLANG_SRC="$CLANG_BASE/../../src" + +CLANG_ANALYZER="$CLANG_BIN/clang++" +CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build" + +if [ -z "$USE_CLANG" ]; then + # gcc + CC="$GCC_BASE/bin/gcc" + CXX="$GCC_BASE/bin/g++" + AR="$GCC_BASE/bin/gcc-ar" + + CFLAGS+=" -B$BINUTILS/gold" + CFLAGS+=" -isystem $GLIBC_INCLUDE" + CFLAGS+=" -isystem $LIBGCC_INCLUDE" + JEMALLOC=1 +else + # clang + CLANG_INCLUDE="$CLANG_LIB/clang/stable/include" + CC="$CLANG_BIN/clang" + CXX="$CLANG_BIN/clang++" + AR="$CLANG_BIN/llvm-ar" + + KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include" + + CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib" + CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/5.x " + CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/5.x/x86_64-facebook-linux " + CFLAGS+=" -isystem $GLIBC_INCLUDE" + CFLAGS+=" -isystem $LIBGCC_INCLUDE" + CFLAGS+=" -isystem $CLANG_INCLUDE" + CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux " + CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE " + CFLAGS+=" -Wno-expansion-to-defined " + CXXFLAGS="-nostdinc++" +fi + +CFLAGS+=" $DEPS_INCLUDE" +CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT" +CXXFLAGS+=" $CFLAGS" + +EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS" +EXEC_LDFLAGS+=" -B$BINUTILS/gold" +EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-5-glibc-2.23/lib/ld.so" +EXEC_LDFLAGS+=" $LIBUNWIND" +EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-5-glibc-2.23/lib" +# required by libtbb +EXEC_LDFLAGS+=" -ldl" + +PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++" + +EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS" + +VALGRIND_VER="$VALGRIND_BASE/bin/" + +LUA_PATH="$LUA_BASE" + +if test -z $PIC_BUILD; then + LUA_LIB=" $LUA_PATH/lib/liblua.a" +else + LUA_LIB=" $LUA_PATH/lib/liblua_pic.a" +fi + +export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB diff --git a/librocksdb-sys/rocksdb/build_tools/fbcode_config_platform010.sh b/librocksdb-sys/rocksdb/build_tools/fbcode_config_platform010.sh new file mode 100644 index 0000000..25835d0 --- /dev/null +++ b/librocksdb-sys/rocksdb/build_tools/fbcode_config_platform010.sh @@ -0,0 +1,175 @@ +#!/bin/sh +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# +# Set environment variables so that we can compile rocksdb using +# fbcode settings. It uses the latest g++ and clang compilers and also +# uses jemalloc +# Environment variables that change the behavior of this script: +# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included + + +BASEDIR=`dirname $BASH_SOURCE` +source "$BASEDIR/dependencies_platform010.sh" + +# Disallow using libraries from default locations as they might not be compatible with platform010 libraries. +CFLAGS=" --sysroot=/DOES/NOT/EXIST" + +# libgcc +LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/trunk" +LIBGCC_LIBS=" -L $LIBGCC_BASE/lib -B$LIBGCC_BASE/lib/gcc/x86_64-facebook-linux/trunk/" + +# glibc +GLIBC_INCLUDE="$GLIBC_BASE/include" +GLIBC_LIBS=" -L $GLIBC_BASE/lib" +GLIBC_LIBS+=" -B$GLIBC_BASE/lib" + +if test -z $PIC_BUILD; then + MAYBE_PIC= +else + MAYBE_PIC=_pic +fi + +if ! test $ROCKSDB_DISABLE_SNAPPY; then + # snappy + SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/" + SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy${MAYBE_PIC}.a" + CFLAGS+=" -DSNAPPY" +fi + +if ! test $ROCKSDB_DISABLE_ZLIB; then + # location of zlib headers and libraries + ZLIB_INCLUDE=" -I $ZLIB_BASE/include/" + ZLIB_LIBS=" $ZLIB_BASE/lib/libz${MAYBE_PIC}.a" + CFLAGS+=" -DZLIB" +fi + +if ! test $ROCKSDB_DISABLE_BZIP; then + # location of bzip headers and libraries + BZIP_INCLUDE=" -I $BZIP2_BASE/include/" + BZIP_LIBS=" $BZIP2_BASE/lib/libbz2${MAYBE_PIC}.a" + CFLAGS+=" -DBZIP2" +fi + +if ! test $ROCKSDB_DISABLE_LZ4; then + LZ4_INCLUDE=" -I $LZ4_BASE/include/" + LZ4_LIBS=" $LZ4_BASE/lib/liblz4${MAYBE_PIC}.a" + CFLAGS+=" -DLZ4" +fi + +if ! test $ROCKSDB_DISABLE_ZSTD; then + ZSTD_INCLUDE=" -I $ZSTD_BASE/include/" + ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd${MAYBE_PIC}.a" + CFLAGS+=" -DZSTD" +fi + +# location of gflags headers and libraries +GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/" +GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags${MAYBE_PIC}.a" +CFLAGS+=" -DGFLAGS=gflags" + +BENCHMARK_INCLUDE=" -I $BENCHMARK_BASE/include/" +BENCHMARK_LIBS=" $BENCHMARK_BASE/lib/libbenchmark${MAYBE_PIC}.a" + +# location of jemalloc +JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/" +JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc${MAYBE_PIC}.a" + +# location of numa +NUMA_INCLUDE=" -I $NUMA_BASE/include/" +NUMA_LIB=" $NUMA_BASE/lib/libnuma${MAYBE_PIC}.a" +CFLAGS+=" -DNUMA" + +# location of libunwind +LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind${MAYBE_PIC}.a" + +# location of TBB +TBB_INCLUDE=" -isystem $TBB_BASE/include/" +TBB_LIBS="$TBB_BASE/lib/libtbb${MAYBE_PIC}.a" +CFLAGS+=" -DTBB" + +# location of LIBURING +LIBURING_INCLUDE=" -isystem $LIBURING_BASE/include/" +LIBURING_LIBS="$LIBURING_BASE/lib/liburing${MAYBE_PIC}.a" +CFLAGS+=" -DLIBURING" + +test "$USE_SSE" || USE_SSE=1 +export USE_SSE +test "$PORTABLE" || PORTABLE=1 +export PORTABLE + +BINUTILS="$BINUTILS_BASE/bin" +AR="$BINUTILS/ar" +AS="$BINUTILS/as" + +DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE $LIBURING_INCLUDE $BENCHMARK_INCLUDE" + +STDLIBS="-L $GCC_BASE/lib64" + +CLANG_BIN="$CLANG_BASE/bin" +CLANG_LIB="$CLANG_BASE/lib" +CLANG_SRC="$CLANG_BASE/../../src" + +CLANG_ANALYZER="$CLANG_BIN/clang++" +CLANG_SCAN_BUILD="$CLANG_SRC/llvm/clang/tools/scan-build/bin/scan-build" + +if [ -z "$USE_CLANG" ]; then + # gcc + CC="$GCC_BASE/bin/gcc" + CXX="$GCC_BASE/bin/g++" + AR="$GCC_BASE/bin/gcc-ar" + + CFLAGS+=" -B$BINUTILS -nostdinc -nostdlib" + CFLAGS+=" -I$GCC_BASE/include" + CFLAGS+=" -isystem $GCC_BASE/lib/gcc/x86_64-redhat-linux-gnu/11.2.1/include" + CFLAGS+=" -isystem $GCC_BASE/lib/gcc/x86_64-redhat-linux-gnu/11.2.1/install-tools/include" + CFLAGS+=" -isystem $GCC_BASE/lib/gcc/x86_64-redhat-linux-gnu/11.2.1/include-fixed/" + CFLAGS+=" -isystem $LIBGCC_INCLUDE" + CFLAGS+=" -isystem $GLIBC_INCLUDE" + CFLAGS+=" -I$GLIBC_INCLUDE" + CFLAGS+=" -I$LIBGCC_BASE/include" + CFLAGS+=" -I$LIBGCC_BASE/include/c++/11.x/" + CFLAGS+=" -I$LIBGCC_BASE/include/c++/11.x/x86_64-facebook-linux/" + CFLAGS+=" -I$LIBGCC_BASE/include/c++/11.x/backward" + CFLAGS+=" -isystem $GLIBC_INCLUDE -I$GLIBC_INCLUDE" + JEMALLOC=1 +else + # clang + CLANG_INCLUDE="$CLANG_LIB/clang/stable/include" + CC="$CLANG_BIN/clang" + CXX="$CLANG_BIN/clang++" + AR="$CLANG_BIN/llvm-ar" + + CFLAGS+=" -B$BINUTILS -nostdinc -nostdlib" + CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/trunk " + CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/trunk/x86_64-facebook-linux " + CFLAGS+=" -isystem $GLIBC_INCLUDE" + CFLAGS+=" -isystem $LIBGCC_INCLUDE" + CFLAGS+=" -isystem $CLANG_INCLUDE" + CFLAGS+=" -Wno-expansion-to-defined " + CXXFLAGS="-nostdinc++" +fi + +KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include" +CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux " +CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE " + +CFLAGS+=" $DEPS_INCLUDE" +CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_IOURING_PRESENT" +CXXFLAGS+=" $CFLAGS" + +EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS $LIBURING_LIBS $BENCHMARK_LIBS" +EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform010/lib/ld.so" +EXEC_LDFLAGS+=" $LIBUNWIND" +EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform010/lib" +EXEC_LDFLAGS+=" -Wl,-rpath=$GCC_BASE/lib64" +# required by libtbb +EXEC_LDFLAGS+=" -ldl" + +PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++" +PLATFORM_LDFLAGS+=" -B$BINUTILS" + +EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS $LIBURING_LIBS $BENCHMARK_LIBS" + +VALGRIND_VER="$VALGRIND_BASE/bin/" + +export CC CXX AR AS CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB diff --git a/librocksdb-sys/rocksdb/build_tools/format-diff.sh b/librocksdb-sys/rocksdb/build_tools/format-diff.sh new file mode 100755 index 0000000..62e8834 --- /dev/null +++ b/librocksdb-sys/rocksdb/build_tools/format-diff.sh @@ -0,0 +1,203 @@ +#!/usr/bin/env bash +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# If clang_format_diff.py command is not specfied, we assume we are able to +# access directly without any path. + +print_usage () { + echo "Usage:" + echo "format-diff.sh [OPTIONS]" + echo "-c: check only." + echo "-h: print this message." +} + +while getopts ':ch' OPTION; do + case "$OPTION" in + c) + CHECK_ONLY=1 + ;; + h) + print_usage + exit 1 + ;; + ?) + print_usage + exit 1 + ;; + esac +done + +REPO_ROOT="$(git rev-parse --show-toplevel)" + +if [ "$CLANG_FORMAT_DIFF" ]; then + echo "Note: CLANG_FORMAT_DIFF='$CLANG_FORMAT_DIFF'" + # Dry run to confirm dependencies like argparse + if $CLANG_FORMAT_DIFF --help >/dev/null < /dev/null; then + true #Good + else + exit 128 + fi +else + # First try directly executing the possibilities + if clang-format-diff --help &> /dev/null < /dev/null; then + CLANG_FORMAT_DIFF=clang-format-diff + elif clang-format-diff.py --help &> /dev/null < /dev/null; then + CLANG_FORMAT_DIFF=clang-format-diff.py + elif $REPO_ROOT/clang-format-diff.py --help &> /dev/null < /dev/null; then + CLANG_FORMAT_DIFF=$REPO_ROOT/clang-format-diff.py + else + # This probably means we need to directly invoke the interpreter. + # But first find clang-format-diff.py + if [ -f "$REPO_ROOT/clang-format-diff.py" ]; then + CFD_PATH="$REPO_ROOT/clang-format-diff.py" + elif which clang-format-diff.py &> /dev/null; then + CFD_PATH="$(which clang-format-diff.py)" + else + echo "You didn't have clang-format-diff.py and/or clang-format available in your computer!" + echo "You can download clang-format-diff.py by running: " + echo " curl --location https://raw.githubusercontent.com/llvm/llvm-project/main/clang/tools/clang-format/clang-format-diff.py -o ${REPO_ROOT}/clang-format-diff.py" + echo "You should make sure the downloaded script is not compromised." + echo "You can download clang-format by running:" + echo " brew install clang-format" + echo " Or" + echo " apt install clang-format" + echo " This might work too:" + echo " yum install git-clang-format" + echo "Then make sure clang-format is available and executable from \$PATH:" + echo " clang-format --version" + exit 128 + fi + # Check argparse pre-req on interpreter, or it will fail + if echo import argparse | ${PYTHON:-python3}; then + true # Good + else + echo "To run clang-format-diff.py, we'll need the library "argparse" to be" + echo "installed. You can try either of the follow ways to install it:" + echo " 1. Manually download argparse: https://pypi.python.org/pypi/argparse" + echo " 2. easy_install argparse (if you have easy_install)" + echo " 3. pip install argparse (if you have pip)" + exit 129 + fi + # Unfortunately, some machines have a Python2 clang-format-diff.py + # installed but only a Python3 interpreter installed. Unfortunately, + # automatic 2to3 migration is insufficient, so suggest downloading latest. + if grep -q "print '" "$CFD_PATH" && \ + ${PYTHON:-python3} --version | grep -q 'ython 3'; then + echo "You have clang-format-diff.py for Python 2 but are using a Python 3" + echo "interpreter (${PYTHON:-python3})." + echo "You can download clang-format-diff.py for Python 3 by running: " + echo " curl --location https://raw.githubusercontent.com/llvm/llvm-project/main/clang/tools/clang-format/clang-format-diff.py -o ${REPO_ROOT}/clang-format-diff.py" + echo "You should make sure the downloaded script is not compromised." + exit 130 + fi + CLANG_FORMAT_DIFF="${PYTHON:-python3} $CFD_PATH" + # This had better work after all those checks + if $CLANG_FORMAT_DIFF --help >/dev/null < /dev/null; then + true #Good + else + exit 128 + fi + fi +fi + +# TODO(kailiu) following work is not complete since we still need to figure +# out how to add the modified files done pre-commit hook to git's commit index. +# +# Check if this script has already been added to pre-commit hook. +# Will suggest user to add this script to pre-commit hook if their pre-commit +# is empty. +# PRE_COMMIT_SCRIPT_PATH="`git rev-parse --show-toplevel`/.git/hooks/pre-commit" +# if ! ls $PRE_COMMIT_SCRIPT_PATH &> /dev/null +# then +# echo "Would you like to add this script to pre-commit hook, which will do " +# echo -n "the format check for all the affected lines before you check in (y/n):" +# read add_to_hook +# if [ "$add_to_hook" == "y" ] +# then +# ln -s `git rev-parse --show-toplevel`/build_tools/format-diff.sh $PRE_COMMIT_SCRIPT_PATH +# fi +# fi +set -e + +uncommitted_code=`git diff HEAD` + +# If there's no uncommitted changes, we assume user are doing post-commit +# format check, in which case we'll try to check the modified lines vs. the +# facebook/rocksdb.git main branch. Otherwise, we'll check format of the +# uncommitted code only. +if [ -z "$uncommitted_code" ] +then + # Attempt to get name of facebook/rocksdb.git remote. + [ "$FORMAT_REMOTE" ] || FORMAT_REMOTE="$(LC_ALL=POSIX LANG=POSIX git remote -v | grep 'facebook/rocksdb.git' | head -n 1 | cut -f 1)" + # Fall back on 'origin' if that fails + [ "$FORMAT_REMOTE" ] || FORMAT_REMOTE=origin + # Use main branch from that remote + [ "$FORMAT_UPSTREAM" ] || FORMAT_UPSTREAM="$FORMAT_REMOTE/$(LC_ALL=POSIX LANG=POSIX git remote show $FORMAT_REMOTE | sed -n '/HEAD branch/s/.*: //p')" + # Get the common ancestor with that remote branch. Everything after that + # common ancestor would be considered the contents of a pull request, so + # should be relevant for formatting fixes. + FORMAT_UPSTREAM_MERGE_BASE="$(git merge-base "$FORMAT_UPSTREAM" HEAD)" + # Get the differences + diffs=$(git diff -U0 "$FORMAT_UPSTREAM_MERGE_BASE" | $CLANG_FORMAT_DIFF -p 1) + echo "Checking format of changes not yet in $FORMAT_UPSTREAM..." +else + # Check the format of uncommitted lines, + diffs=$(git diff -U0 HEAD | $CLANG_FORMAT_DIFF -p 1) + echo "Checking format of uncommitted changes..." +fi + +if [ -z "$diffs" ] +then + echo "Nothing needs to be reformatted!" + exit 0 +elif [ $CHECK_ONLY ] +then + echo "Your change has unformatted code. Please run make format!" + if [ $VERBOSE_CHECK ]; then + clang-format --version + echo "$diffs" + fi + exit 1 +fi + +# Highlight the insertion/deletion from the clang-format-diff.py's output +COLOR_END="\033[0m" +COLOR_RED="\033[0;31m" +COLOR_GREEN="\033[0;32m" + +echo -e "Detect lines that doesn't follow the format rules:\r" +# Add the color to the diff. lines added will be green; lines removed will be red. +echo "$diffs" | + sed -e "s/\(^-.*$\)/`echo -e \"$COLOR_RED\1$COLOR_END\"`/" | + sed -e "s/\(^+.*$\)/`echo -e \"$COLOR_GREEN\1$COLOR_END\"`/" + +echo -e "Would you like to fix the format automatically (y/n): \c" + +# Make sure under any mode, we can read user input. +exec < /dev/tty +read to_fix + +if [ "$to_fix" != "y" ] +then + exit 1 +fi + +# Do in-place format adjustment. +if [ -z "$uncommitted_code" ] +then + git diff -U0 "$FORMAT_UPSTREAM_MERGE_BASE" | $CLANG_FORMAT_DIFF -i -p 1 +else + git diff -U0 HEAD | $CLANG_FORMAT_DIFF -i -p 1 +fi +echo "Files reformatted!" + +# Amend to last commit if user do the post-commit format check +if [ -z "$uncommitted_code" ]; then + echo -e "Would you like to amend the changes to last commit (`git log HEAD --oneline | head -1`)? (y/n): \c" + read to_amend + + if [ "$to_amend" == "y" ] + then + git commit -a --amend --reuse-message HEAD + echo "Amended to last commit" + fi +fi diff --git a/librocksdb-sys/rocksdb/build_tools/gnu_parallel b/librocksdb-sys/rocksdb/build_tools/gnu_parallel new file mode 100755 index 0000000..3365f46 --- /dev/null +++ b/librocksdb-sys/rocksdb/build_tools/gnu_parallel @@ -0,0 +1,7971 @@ +#!/usr/bin/env perl + +# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014 Ole Tange and +# Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see +# or write to the Free Software Foundation, Inc., 51 Franklin St, +# Fifth Floor, Boston, MA 02110-1301 USA + +# open3 used in Job::start +use IPC::Open3; +# &WNOHANG used in reaper +use POSIX qw(:sys_wait_h setsid ceil :errno_h); +# gensym used in Job::start +use Symbol qw(gensym); +# tempfile used in Job::start +use File::Temp qw(tempfile tempdir); +# mkpath used in openresultsfile +use File::Path; +# GetOptions used in get_options_from_array +use Getopt::Long; +# Used to ensure code quality +use strict; +use File::Basename; + +if(not $ENV{HOME}) { + # $ENV{HOME} is sometimes not set if called from PHP + ::warning("\$HOME not set. Using /tmp\n"); + $ENV{HOME} = "/tmp"; +} + +save_stdin_stdout_stderr(); +save_original_signal_handler(); +parse_options(); +::debug("init", "Open file descriptors: ", join(" ",keys %Global::fd), "\n"); +my $number_of_args; +if($Global::max_number_of_args) { + $number_of_args=$Global::max_number_of_args; +} elsif ($opt::X or $opt::m or $opt::xargs) { + $number_of_args = undef; +} else { + $number_of_args = 1; +} + +my @command; +@command = @ARGV; + +my @fhlist; +if($opt::pipepart) { + @fhlist = map { open_or_exit($_) } "/dev/null"; +} else { + @fhlist = map { open_or_exit($_) } @opt::a; + if(not @fhlist and not $opt::pipe) { + @fhlist = (*STDIN); + } +} + +if($opt::skip_first_line) { + # Skip the first line for the first file handle + my $fh = $fhlist[0]; + <$fh>; +} +if($opt::header and not $opt::pipe) { + my $fh = $fhlist[0]; + # split with colsep or \t + # $header force $colsep = \t if undef? + my $delimiter = $opt::colsep; + $delimiter ||= "\$"; + my $id = 1; + for my $fh (@fhlist) { + my $line = <$fh>; + chomp($line); + ::debug("init", "Delimiter: '$delimiter'"); + for my $s (split /$delimiter/o, $line) { + ::debug("init", "Colname: '$s'"); + # Replace {colname} with {2} + # TODO accept configurable short hands + # TODO how to deal with headers in {=...=} + for(@command) { + s:\{$s(|/|//|\.|/\.)\}:\{$id$1\}:g; + } + $Global::input_source_header{$id} = $s; + $id++; + } + } +} else { + my $id = 1; + for my $fh (@fhlist) { + $Global::input_source_header{$id} = $id; + $id++; + } +} + +if($opt::filter_hosts and (@opt::sshlogin or @opt::sshloginfile)) { + # Parallel check all hosts are up. Remove hosts that are down + filter_hosts(); +} + +if($opt::nonall or $opt::onall) { + onall(@command); + wait_and_exit(min(undef_as_zero($Global::exitstatus),254)); +} + +# TODO --transfer foo/./bar --cleanup +# multiple --transfer and --basefile with different /./ + +$Global::JobQueue = JobQueue->new( + \@command,\@fhlist,$Global::ContextReplace,$number_of_args,\@Global::ret_files); + +if($opt::eta or $opt::bar) { + # Count the number of jobs before starting any + $Global::JobQueue->total_jobs(); +} +if($opt::pipepart) { + @Global::cat_partials = map { pipe_part_files($_) } @opt::a; + # Unget the command as many times as there are parts + $Global::JobQueue->{'commandlinequeue'}->unget( + map { $Global::JobQueue->{'commandlinequeue'}->get() } @Global::cat_partials + ); +} +for my $sshlogin (values %Global::host) { + $sshlogin->max_jobs_running(); +} + +init_run_jobs(); +my $sem; +if($Global::semaphore) { + $sem = acquire_semaphore(); +} +$SIG{TERM} = \&start_no_new_jobs; + +start_more_jobs(); +if(not $opt::pipepart) { + if($opt::pipe) { + spreadstdin(); + } +} +::debug("init", "Start draining\n"); +drain_job_queue(); +::debug("init", "Done draining\n"); +reaper(); +::debug("init", "Done reaping\n"); +if($opt::pipe and @opt::a) { + for my $job (@Global::tee_jobs) { + unlink $job->fh(2,"name"); + $job->set_fh(2,"name",""); + $job->print(); + unlink $job->fh(1,"name"); + } +} +::debug("init", "Cleaning\n"); +cleanup(); +if($Global::semaphore) { + $sem->release(); +} +for(keys %Global::sshmaster) { + kill "TERM", $_; +} +::debug("init", "Halt\n"); +if($opt::halt_on_error) { + wait_and_exit($Global::halt_on_error_exitstatus); +} else { + wait_and_exit(min(undef_as_zero($Global::exitstatus),254)); +} + +sub __PIPE_MODE__ {} + +sub pipe_part_files { + # Input: + # $file = the file to read + # Returns: + # @commands that will cat_partial each part + my ($file) = @_; + my $buf = ""; + my $header = find_header(\$buf,open_or_exit($file)); + # find positions + my @pos = find_split_positions($file,$opt::blocksize,length $header); + # Make @cat_partials + my @cat_partials = (); + for(my $i=0; $i<$#pos; $i++) { + push @cat_partials, cat_partial($file, 0, length($header), $pos[$i], $pos[$i+1]); + } + # Remote exec should look like: + # ssh -oLogLevel=quiet lo 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;echo\ \$SHELL\ \|\ grep\ \"/t\\\{0,1\\\}csh\"\ \>\ /dev/null\ \&\&\ setenv\ FOO\ /tmp/foo\ \|\|\ export\ FOO=/tmp/foo\; \(wc\ -\ \$FOO\) + # ssh -tt not allowed. Remote will die due to broken pipe anyway. + # TODO test remote with --fifo / --cat + return @cat_partials; +} + +sub find_header { + # Input: + # $buf_ref = reference to read-in buffer + # $fh = filehandle to read from + # Uses: + # $opt::header + # $opt::blocksize + # Returns: + # $header string + my ($buf_ref, $fh) = @_; + my $header = ""; + if($opt::header) { + if($opt::header eq ":") { $opt::header = "(.*\n)"; } + # Number = number of lines + $opt::header =~ s/^(\d+)$/"(.*\n)"x$1/e; + while(read($fh,substr($$buf_ref,length $$buf_ref,0),$opt::blocksize)) { + if($$buf_ref=~s/^($opt::header)//) { + $header = $1; + last; + } + } + } + return $header; +} + +sub find_split_positions { + # Input: + # $file = the file to read + # $block = (minimal) --block-size of each chunk + # $headerlen = length of header to be skipped + # Uses: + # $opt::recstart + # $opt::recend + # Returns: + # @positions of block start/end + my($file, $block, $headerlen) = @_; + my $size = -s $file; + $block = int $block; + # The optimal dd blocksize for mint, redhat, solaris, openbsd = 2^17..2^20 + # The optimal dd blocksize for freebsd = 2^15..2^17 + my $dd_block_size = 131072; # 2^17 + my @pos; + my ($recstart,$recend) = recstartrecend(); + my $recendrecstart = $recend.$recstart; + my $fh = ::open_or_exit($file); + push(@pos,$headerlen); + for(my $pos = $block+$headerlen; $pos < $size; $pos += $block) { + my $buf; + seek($fh, $pos, 0) || die; + while(read($fh,substr($buf,length $buf,0),$dd_block_size)) { + if($opt::regexp) { + # If match /$recend$recstart/ => Record position + if($buf =~ /(.*$recend)$recstart/os) { + my $i = length($1); + push(@pos,$pos+$i); + # Start looking for next record _after_ this match + $pos += $i; + last; + } + } else { + # If match $recend$recstart => Record position + my $i = index($buf,$recendrecstart); + if($i != -1) { + push(@pos,$pos+$i); + # Start looking for next record _after_ this match + $pos += $i; + last; + } + } + } + } + push(@pos,$size); + close $fh; + return @pos; +} + +sub cat_partial { + # Input: + # $file = the file to read + # ($start, $end, [$start2, $end2, ...]) = start byte, end byte + # Returns: + # Efficient perl command to copy $start..$end, $start2..$end2, ... to stdout + my($file, @start_end) = @_; + my($start, $i); + # Convert start_end to start_len + my @start_len = map { if(++$i % 2) { $start = $_; } else { $_-$start } } @start_end; + return "<". shell_quote_scalar($file) . + q{ perl -e 'while(@ARGV) { sysseek(STDIN,shift,0) || die; $left = shift; while($read = sysread(STDIN,$buf, ($left > 32768 ? 32768 : $left))){ $left -= $read; syswrite(STDOUT,$buf); } }' } . + " @start_len"; +} + +sub spreadstdin { + # read a record + # Spawn a job and print the record to it. + # Uses: + # $opt::blocksize + # STDIN + # $opr::r + # $Global::max_lines + # $Global::max_number_of_args + # $opt::regexp + # $Global::start_no_new_jobs + # $opt::roundrobin + # %Global::running + + my $buf = ""; + my ($recstart,$recend) = recstartrecend(); + my $recendrecstart = $recend.$recstart; + my $chunk_number = 1; + my $one_time_through; + my $blocksize = $opt::blocksize; + my $in = *STDIN; + my $header = find_header(\$buf,$in); + while(1) { + my $anything_written = 0; + if(not read($in,substr($buf,length $buf,0),$blocksize)) { + # End-of-file + $chunk_number != 1 and last; + # Force the while-loop once if everything was read by header reading + $one_time_through++ and last; + } + if($opt::r) { + # Remove empty lines + $buf =~ s/^\s*\n//gm; + if(length $buf == 0) { + next; + } + } + if($Global::max_lines and not $Global::max_number_of_args) { + # Read n-line records + my $n_lines = $buf =~ tr/\n/\n/; + my $last_newline_pos = rindex($buf,"\n"); + while($n_lines % $Global::max_lines) { + $n_lines--; + $last_newline_pos = rindex($buf,"\n",$last_newline_pos-1); + } + # Chop at $last_newline_pos as that is where n-line record ends + $anything_written += + write_record_to_pipe($chunk_number++,\$header,\$buf, + $recstart,$recend,$last_newline_pos+1); + substr($buf,0,$last_newline_pos+1) = ""; + } elsif($opt::regexp) { + if($Global::max_number_of_args) { + # -N => (start..*?end){n} + # -L -N => (start..*?end){n*l} + my $read_n_lines = $Global::max_number_of_args * ($Global::max_lines || 1); + while($buf =~ s/((?:$recstart.*?$recend){$read_n_lines})($recstart.*)$/$2/os) { + # Copy to modifiable variable + my $b = $1; + $anything_written += + write_record_to_pipe($chunk_number++,\$header,\$b, + $recstart,$recend,length $1); + } + } else { + # Find the last recend-recstart in $buf + if($buf =~ s/(.*$recend)($recstart.*?)$/$2/os) { + # Copy to modifiable variable + my $b = $1; + $anything_written += + write_record_to_pipe($chunk_number++,\$header,\$b, + $recstart,$recend,length $1); + } + } + } else { + if($Global::max_number_of_args) { + # -N => (start..*?end){n} + my $i = 0; + my $read_n_lines = $Global::max_number_of_args * ($Global::max_lines || 1); + while(($i = nindex(\$buf,$recendrecstart,$read_n_lines)) != -1) { + $i += length $recend; # find the actual splitting location + $anything_written += + write_record_to_pipe($chunk_number++,\$header,\$buf, + $recstart,$recend,$i); + substr($buf,0,$i) = ""; + } + } else { + # Find the last recend-recstart in $buf + my $i = rindex($buf,$recendrecstart); + if($i != -1) { + $i += length $recend; # find the actual splitting location + $anything_written += + write_record_to_pipe($chunk_number++,\$header,\$buf, + $recstart,$recend,$i); + substr($buf,0,$i) = ""; + } + } + } + if(not $anything_written and not eof($in)) { + # Nothing was written - maybe the block size < record size? + # Increase blocksize exponentially + my $old_blocksize = $blocksize; + $blocksize = ceil($blocksize * 1.3 + 1); + ::warning("A record was longer than $old_blocksize. " . + "Increasing to --blocksize $blocksize\n"); + } + } + ::debug("init", "Done reading input\n"); + + # If there is anything left in the buffer write it + substr($buf,0,0) = ""; + write_record_to_pipe($chunk_number++,\$header,\$buf,$recstart,$recend,length $buf); + + $Global::start_no_new_jobs ||= 1; + if($opt::roundrobin) { + for my $job (values %Global::running) { + close $job->fh(0,"w"); + } + my %incomplete_jobs = %Global::running; + my $sleep = 1; + while(keys %incomplete_jobs) { + my $something_written = 0; + for my $pid (keys %incomplete_jobs) { + my $job = $incomplete_jobs{$pid}; + if($job->stdin_buffer_length()) { + $something_written += $job->non_block_write(); + } else { + delete $incomplete_jobs{$pid} + } + } + if($something_written) { + $sleep = $sleep/2+0.001; + } + $sleep = ::reap_usleep($sleep); + } + } +} + +sub recstartrecend { + # Uses: + # $opt::recstart + # $opt::recend + # Returns: + # $recstart,$recend with default values and regexp conversion + my($recstart,$recend); + if(defined($opt::recstart) and defined($opt::recend)) { + # If both --recstart and --recend is given then both must match + $recstart = $opt::recstart; + $recend = $opt::recend; + } elsif(defined($opt::recstart)) { + # If --recstart is given it must match start of record + $recstart = $opt::recstart; + $recend = ""; + } elsif(defined($opt::recend)) { + # If --recend is given then it must match end of record + $recstart = ""; + $recend = $opt::recend; + } + + if($opt::regexp) { + # If $recstart/$recend contains '|' this should only apply to the regexp + $recstart = "(?:".$recstart.")"; + $recend = "(?:".$recend.")"; + } else { + # $recstart/$recend = printf strings (\n) + $recstart =~ s/\\([0rnt\'\"\\])/"qq|\\$1|"/gee; + $recend =~ s/\\([0rnt\'\"\\])/"qq|\\$1|"/gee; + } + return ($recstart,$recend); +} + +sub nindex { + # See if string is in buffer N times + # Returns: + # the position where the Nth copy is found + my ($buf_ref, $str, $n) = @_; + my $i = 0; + for(1..$n) { + $i = index($$buf_ref,$str,$i+1); + if($i == -1) { last } + } + return $i; +} + +{ + my @robin_queue; + + sub round_robin_write { + # Input: + # $header_ref = ref to $header string + # $block_ref = ref to $block to be written + # $recstart = record start string + # $recend = record end string + # $endpos = end position of $block + # Uses: + # %Global::running + my ($header_ref,$block_ref,$recstart,$recend,$endpos) = @_; + my $something_written = 0; + my $block_passed = 0; + my $sleep = 1; + while(not $block_passed) { + # Continue flushing existing buffers + # until one is empty and a new block is passed + # Make a queue to spread the blocks evenly + if(not @robin_queue) { + push @robin_queue, values %Global::running; + } + while(my $job = shift @robin_queue) { + if($job->stdin_buffer_length() > 0) { + $something_written += $job->non_block_write(); + } else { + $job->set_stdin_buffer($header_ref,$block_ref,$endpos,$recstart,$recend); + $block_passed = 1; + $job->set_virgin(0); + $something_written += $job->non_block_write(); + last; + } + } + $sleep = ::reap_usleep($sleep); + } + return $something_written; + } +} + +sub write_record_to_pipe { + # Fork then + # Write record from pos 0 .. $endpos to pipe + # Input: + # $chunk_number = sequence number - to see if already run + # $header_ref = reference to header string to prepend + # $record_ref = reference to record to write + # $recstart = start string of record + # $recend = end string of record + # $endpos = position in $record_ref where record ends + # Uses: + # $Global::job_already_run + # $opt::roundrobin + # @Global::virgin_jobs + # Returns: + # Number of chunks written (0 or 1) + my ($chunk_number,$header_ref,$record_ref,$recstart,$recend,$endpos) = @_; + if($endpos == 0) { return 0; } + if(vec($Global::job_already_run,$chunk_number,1)) { return 1; } + if($opt::roundrobin) { + return round_robin_write($header_ref,$record_ref,$recstart,$recend,$endpos); + } + # If no virgin found, backoff + my $sleep = 0.0001; # 0.01 ms - better performance on highend + while(not @Global::virgin_jobs) { + ::debug("pipe", "No virgin jobs"); + $sleep = ::reap_usleep($sleep); + # Jobs may not be started because of loadavg + # or too little time between each ssh login. + start_more_jobs(); + } + my $job = shift @Global::virgin_jobs; + # Job is no longer virgin + $job->set_virgin(0); + if(fork()) { + # Skip + } else { + # Chop of at $endpos as we do not know how many rec_sep will + # be removed. + substr($$record_ref,$endpos,length $$record_ref) = ""; + # Remove rec_sep + if($opt::remove_rec_sep) { + Job::remove_rec_sep($record_ref,$recstart,$recend); + } + $job->write($header_ref); + $job->write($record_ref); + close $job->fh(0,"w"); + exit(0); + } + close $job->fh(0,"w"); + return 1; +} + +sub __SEM_MODE__ {} + +sub acquire_semaphore { + # Acquires semaphore. If needed: spawns to the background + # Uses: + # @Global::host + # Returns: + # The semaphore to be released when jobs is complete + $Global::host{':'} = SSHLogin->new(":"); + my $sem = Semaphore->new($Semaphore::name,$Global::host{':'}->max_jobs_running()); + $sem->acquire(); + if($Semaphore::fg) { + # skip + } else { + # If run in the background, the PID will change + # therefore release and re-acquire the semaphore + $sem->release(); + if(fork()) { + exit(0); + } else { + # child + # Get a semaphore for this pid + ::die_bug("Can't start a new session: $!") if setsid() == -1; + $sem = Semaphore->new($Semaphore::name,$Global::host{':'}->max_jobs_running()); + $sem->acquire(); + } + } + return $sem; +} + +sub __PARSE_OPTIONS__ {} + +sub options_hash { + # Returns: + # %hash = the GetOptions config + return + ("debug|D=s" => \$opt::D, + "xargs" => \$opt::xargs, + "m" => \$opt::m, + "X" => \$opt::X, + "v" => \@opt::v, + "joblog=s" => \$opt::joblog, + "results|result|res=s" => \$opt::results, + "resume" => \$opt::resume, + "resume-failed|resumefailed" => \$opt::resume_failed, + "silent" => \$opt::silent, + #"silent-error|silenterror" => \$opt::silent_error, + "keep-order|keeporder|k" => \$opt::keeporder, + "group" => \$opt::group, + "g" => \$opt::retired, + "ungroup|u" => \$opt::ungroup, + "linebuffer|linebuffered|line-buffer|line-buffered" => \$opt::linebuffer, + "tmux" => \$opt::tmux, + "null|0" => \$opt::0, + "quote|q" => \$opt::q, + # Replacement strings + "parens=s" => \$opt::parens, + "rpl=s" => \@opt::rpl, + "plus" => \$opt::plus, + "I=s" => \$opt::I, + "extensionreplace|er=s" => \$opt::U, + "U=s" => \$opt::retired, + "basenamereplace|bnr=s" => \$opt::basenamereplace, + "dirnamereplace|dnr=s" => \$opt::dirnamereplace, + "basenameextensionreplace|bner=s" => \$opt::basenameextensionreplace, + "seqreplace=s" => \$opt::seqreplace, + "slotreplace=s" => \$opt::slotreplace, + "jobs|j=s" => \$opt::jobs, + "delay=f" => \$opt::delay, + "sshdelay=f" => \$opt::sshdelay, + "load=s" => \$opt::load, + "noswap" => \$opt::noswap, + "max-line-length-allowed" => \$opt::max_line_length_allowed, + "number-of-cpus" => \$opt::number_of_cpus, + "number-of-cores" => \$opt::number_of_cores, + "use-cpus-instead-of-cores" => \$opt::use_cpus_instead_of_cores, + "shellquote|shell_quote|shell-quote" => \$opt::shellquote, + "nice=i" => \$opt::nice, + "timeout=s" => \$opt::timeout, + "tag" => \$opt::tag, + "tagstring|tag-string=s" => \$opt::tagstring, + "onall" => \$opt::onall, + "nonall" => \$opt::nonall, + "filter-hosts|filterhosts|filter-host" => \$opt::filter_hosts, + "sshlogin|S=s" => \@opt::sshlogin, + "sshloginfile|slf=s" => \@opt::sshloginfile, + "controlmaster|M" => \$opt::controlmaster, + "return=s" => \@opt::return, + "trc=s" => \@opt::trc, + "transfer" => \$opt::transfer, + "cleanup" => \$opt::cleanup, + "basefile|bf=s" => \@opt::basefile, + "B=s" => \$opt::retired, + "ctrlc|ctrl-c" => \$opt::ctrlc, + "noctrlc|no-ctrlc|no-ctrl-c" => \$opt::noctrlc, + "workdir|work-dir|wd=s" => \$opt::workdir, + "W=s" => \$opt::retired, + "tmpdir=s" => \$opt::tmpdir, + "tempdir=s" => \$opt::tmpdir, + "use-compress-program|compress-program=s" => \$opt::compress_program, + "use-decompress-program|decompress-program=s" => \$opt::decompress_program, + "compress" => \$opt::compress, + "tty" => \$opt::tty, + "T" => \$opt::retired, + "halt-on-error|halt=s" => \$opt::halt_on_error, + "H=i" => \$opt::retired, + "retries=i" => \$opt::retries, + "dry-run|dryrun" => \$opt::dryrun, + "progress" => \$opt::progress, + "eta" => \$opt::eta, + "bar" => \$opt::bar, + "arg-sep|argsep=s" => \$opt::arg_sep, + "arg-file-sep|argfilesep=s" => \$opt::arg_file_sep, + "trim=s" => \$opt::trim, + "env=s" => \@opt::env, + "recordenv|record-env" => \$opt::record_env, + "plain" => \$opt::plain, + "profile|J=s" => \@opt::profile, + "pipe|spreadstdin" => \$opt::pipe, + "robin|round-robin|roundrobin" => \$opt::roundrobin, + "recstart=s" => \$opt::recstart, + "recend=s" => \$opt::recend, + "regexp|regex" => \$opt::regexp, + "remove-rec-sep|removerecsep|rrs" => \$opt::remove_rec_sep, + "files|output-as-files|outputasfiles" => \$opt::files, + "block|block-size|blocksize=s" => \$opt::blocksize, + "tollef" => \$opt::retired, + "gnu" => \$opt::gnu, + "xapply" => \$opt::xapply, + "bibtex" => \$opt::bibtex, + "nn|nonotice|no-notice" => \$opt::no_notice, + # xargs-compatibility - implemented, man, testsuite + "max-procs|P=s" => \$opt::jobs, + "delimiter|d=s" => \$opt::d, + "max-chars|s=i" => \$opt::max_chars, + "arg-file|a=s" => \@opt::a, + "no-run-if-empty|r" => \$opt::r, + "replace|i:s" => \$opt::i, + "E=s" => \$opt::eof, + "eof|e:s" => \$opt::eof, + "max-args|n=i" => \$opt::max_args, + "max-replace-args|N=i" => \$opt::max_replace_args, + "colsep|col-sep|C=s" => \$opt::colsep, + "help|h" => \$opt::help, + "L=f" => \$opt::L, + "max-lines|l:f" => \$opt::max_lines, + "interactive|p" => \$opt::p, + "verbose|t" => \$opt::verbose, + "version|V" => \$opt::version, + "minversion|min-version=i" => \$opt::minversion, + "show-limits|showlimits" => \$opt::show_limits, + "exit|x" => \$opt::x, + # Semaphore + "semaphore" => \$opt::semaphore, + "semaphoretimeout=i" => \$opt::semaphoretimeout, + "semaphorename|id=s" => \$opt::semaphorename, + "fg" => \$opt::fg, + "bg" => \$opt::bg, + "wait" => \$opt::wait, + # Shebang #!/usr/bin/parallel --shebang + "shebang|hashbang" => \$opt::shebang, + "internal-pipe-means-argfiles" => \$opt::internal_pipe_means_argfiles, + "Y" => \$opt::retired, + "skip-first-line" => \$opt::skip_first_line, + "header=s" => \$opt::header, + "cat" => \$opt::cat, + "fifo" => \$opt::fifo, + "pipepart|pipe-part" => \$opt::pipepart, + "hgrp|hostgroup|hostgroups" => \$opt::hostgroups, + ); +} + +sub get_options_from_array { + # Run GetOptions on @array + # Input: + # $array_ref = ref to @ARGV to parse + # @keep_only = Keep only these options + # Uses: + # @ARGV + # Returns: + # true if parsing worked + # false if parsing failed + # @$array_ref is changed + my ($array_ref, @keep_only) = @_; + if(not @$array_ref) { + # Empty array: No need to look more at that + return 1; + } + # A bit of shuffling of @ARGV needed as GetOptionsFromArray is not + # supported everywhere + my @save_argv; + my $this_is_ARGV = (\@::ARGV == $array_ref); + if(not $this_is_ARGV) { + @save_argv = @::ARGV; + @::ARGV = @{$array_ref}; + } + # If @keep_only set: Ignore all values except @keep_only + my %options = options_hash(); + if(@keep_only) { + my (%keep,@dummy); + @keep{@keep_only} = @keep_only; + for my $k (grep { not $keep{$_} } keys %options) { + # Store the value of the option in @dummy + $options{$k} = \@dummy; + } + } + my $retval = GetOptions(%options); + if(not $this_is_ARGV) { + @{$array_ref} = @::ARGV; + @::ARGV = @save_argv; + } + return $retval; +} + +sub parse_options { + # Returns: N/A + # Defaults: + $Global::version = 20141122; + $Global::progname = 'parallel'; + $Global::infinity = 2**31; + $Global::debug = 0; + $Global::verbose = 0; + $Global::quoting = 0; + # Read only table with default --rpl values + %Global::replace = + ( + '{}' => '', + '{#}' => '1 $_=$job->seq()', + '{%}' => '1 $_=$job->slot()', + '{/}' => 's:.*/::', + '{//}' => '$Global::use{"File::Basename"} ||= eval "use File::Basename; 1;"; $_ = dirname($_);', + '{/.}' => 's:.*/::; s:\.[^/.]+$::;', + '{.}' => 's:\.[^/.]+$::', + ); + %Global::plus = + ( + # {} = {+/}/{/} + # = {.}.{+.} = {+/}/{/.}.{+.} + # = {..}.{+..} = {+/}/{/..}.{+..} + # = {...}.{+...} = {+/}/{/...}.{+...} + '{+/}' => 's:/[^/]*$::', + '{+.}' => 's:.*\.::', + '{+..}' => 's:.*\.([^.]*\.):$1:', + '{+...}' => 's:.*\.([^.]*\.[^.]*\.):$1:', + '{..}' => 's:\.[^/.]+$::; s:\.[^/.]+$::', + '{...}' => 's:\.[^/.]+$::; s:\.[^/.]+$::; s:\.[^/.]+$::', + '{/..}' => 's:.*/::; s:\.[^/.]+$::; s:\.[^/.]+$::', + '{/...}' => 's:.*/::; s:\.[^/.]+$::; s:\.[^/.]+$::; s:\.[^/.]+$::', + ); + # Modifiable copy of %Global::replace + %Global::rpl = %Global::replace; + $Global::parens = "{==}"; + $/="\n"; + $Global::ignore_empty = 0; + $Global::interactive = 0; + $Global::stderr_verbose = 0; + $Global::default_simultaneous_sshlogins = 9; + $Global::exitstatus = 0; + $Global::halt_on_error_exitstatus = 0; + $Global::arg_sep = ":::"; + $Global::arg_file_sep = "::::"; + $Global::trim = 'n'; + $Global::max_jobs_running = 0; + $Global::job_already_run = ''; + $ENV{'TMPDIR'} ||= "/tmp"; + + @ARGV=read_options(); + + if(@opt::v) { $Global::verbose = $#opt::v+1; } # Convert -v -v to v=2 + $Global::debug = $opt::D; + $Global::shell = $ENV{'PARALLEL_SHELL'} || parent_shell($$) || $ENV{'SHELL'} || "/bin/sh"; + if(defined $opt::X) { $Global::ContextReplace = 1; } + if(defined $opt::silent) { $Global::verbose = 0; } + if(defined $opt::0) { $/ = "\0"; } + if(defined $opt::d) { my $e="sprintf \"$opt::d\""; $/ = eval $e; } + if(defined $opt::p) { $Global::interactive = $opt::p; } + if(defined $opt::q) { $Global::quoting = 1; } + if(defined $opt::r) { $Global::ignore_empty = 1; } + if(defined $opt::verbose) { $Global::stderr_verbose = 1; } + # Deal with --rpl + sub rpl { + # Modify %Global::rpl + # Replace $old with $new + my ($old,$new) = @_; + if($old ne $new) { + $Global::rpl{$new} = $Global::rpl{$old}; + delete $Global::rpl{$old}; + } + } + if(defined $opt::parens) { $Global::parens = $opt::parens; } + my $parenslen = 0.5*length $Global::parens; + $Global::parensleft = substr($Global::parens,0,$parenslen); + $Global::parensright = substr($Global::parens,$parenslen); + if(defined $opt::plus) { %Global::rpl = (%Global::plus,%Global::rpl); } + if(defined $opt::I) { rpl('{}',$opt::I); } + if(defined $opt::U) { rpl('{.}',$opt::U); } + if(defined $opt::i and $opt::i) { rpl('{}',$opt::i); } + if(defined $opt::basenamereplace) { rpl('{/}',$opt::basenamereplace); } + if(defined $opt::dirnamereplace) { rpl('{//}',$opt::dirnamereplace); } + if(defined $opt::seqreplace) { rpl('{#}',$opt::seqreplace); } + if(defined $opt::slotreplace) { rpl('{%}',$opt::slotreplace); } + if(defined $opt::basenameextensionreplace) { + rpl('{/.}',$opt::basenameextensionreplace); + } + for(@opt::rpl) { + # Create $Global::rpl entries for --rpl options + # E.g: "{..} s:\.[^.]+$:;s:\.[^.]+$:;" + my ($shorthand,$long) = split/ /,$_,2; + $Global::rpl{$shorthand} = $long; + } + if(defined $opt::eof) { $Global::end_of_file_string = $opt::eof; } + if(defined $opt::max_args) { $Global::max_number_of_args = $opt::max_args; } + if(defined $opt::timeout) { $Global::timeoutq = TimeoutQueue->new($opt::timeout); } + if(defined $opt::tmpdir) { $ENV{'TMPDIR'} = $opt::tmpdir; } + if(defined $opt::help) { die_usage(); } + if(defined $opt::colsep) { $Global::trim = 'lr'; } + if(defined $opt::header) { $opt::colsep = defined $opt::colsep ? $opt::colsep : "\t"; } + if(defined $opt::trim) { $Global::trim = $opt::trim; } + if(defined $opt::arg_sep) { $Global::arg_sep = $opt::arg_sep; } + if(defined $opt::arg_file_sep) { $Global::arg_file_sep = $opt::arg_file_sep; } + if(defined $opt::number_of_cpus) { print SSHLogin::no_of_cpus(),"\n"; wait_and_exit(0); } + if(defined $opt::number_of_cores) { + print SSHLogin::no_of_cores(),"\n"; wait_and_exit(0); + } + if(defined $opt::max_line_length_allowed) { + print Limits::Command::real_max_length(),"\n"; wait_and_exit(0); + } + if(defined $opt::version) { version(); wait_and_exit(0); } + if(defined $opt::bibtex) { bibtex(); wait_and_exit(0); } + if(defined $opt::record_env) { record_env(); wait_and_exit(0); } + if(defined $opt::show_limits) { show_limits(); } + if(@opt::sshlogin) { @Global::sshlogin = @opt::sshlogin; } + if(@opt::sshloginfile) { read_sshloginfiles(@opt::sshloginfile); } + if(@opt::return) { push @Global::ret_files, @opt::return; } + if(not defined $opt::recstart and + not defined $opt::recend) { $opt::recend = "\n"; } + if(not defined $opt::blocksize) { $opt::blocksize = "1M"; } + $opt::blocksize = multiply_binary_prefix($opt::blocksize); + if(defined $opt::controlmaster) { $opt::noctrlc = 1; } + if(defined $opt::semaphore) { $Global::semaphore = 1; } + if(defined $opt::semaphoretimeout) { $Global::semaphore = 1; } + if(defined $opt::semaphorename) { $Global::semaphore = 1; } + if(defined $opt::fg) { $Global::semaphore = 1; } + if(defined $opt::bg) { $Global::semaphore = 1; } + if(defined $opt::wait) { $Global::semaphore = 1; } + if(defined $opt::halt_on_error and + $opt::halt_on_error=~/%/) { $opt::halt_on_error /= 100; } + if(defined $opt::timeout and $opt::timeout !~ /^\d+(\.\d+)?%?$/) { + ::error("--timeout must be seconds or percentage\n"); + wait_and_exit(255); + } + if(defined $opt::minversion) { + print $Global::version,"\n"; + if($Global::version < $opt::minversion) { + wait_and_exit(255); + } else { + wait_and_exit(0); + } + } + if(not defined $opt::delay) { + # Set --delay to --sshdelay if not set + $opt::delay = $opt::sshdelay; + } + if($opt::compress_program) { + $opt::compress = 1; + $opt::decompress_program ||= $opt::compress_program." -dc"; + } + if($opt::compress) { + my ($compress, $decompress) = find_compression_program(); + $opt::compress_program ||= $compress; + $opt::decompress_program ||= $decompress; + } + if(defined $opt::nonall) { + # Append a dummy empty argument + push @ARGV, $Global::arg_sep, ""; + } + if(defined $opt::tty) { + # Defaults for --tty: -j1 -u + # Can be overridden with -jXXX -g + if(not defined $opt::jobs) { + $opt::jobs = 1; + } + if(not defined $opt::group) { + $opt::ungroup = 0; + } + } + if(@opt::trc) { + push @Global::ret_files, @opt::trc; + $opt::transfer = 1; + $opt::cleanup = 1; + } + if(defined $opt::max_lines) { + if($opt::max_lines eq "-0") { + # -l -0 (swallowed -0) + $opt::max_lines = 1; + $opt::0 = 1; + $/ = "\0"; + } elsif ($opt::max_lines == 0) { + # If not given (or if 0 is given) => 1 + $opt::max_lines = 1; + } + $Global::max_lines = $opt::max_lines; + if(not $opt::pipe) { + # --pipe -L means length of record - not max_number_of_args + $Global::max_number_of_args ||= $Global::max_lines; + } + } + + # Read more than one arg at a time (-L, -N) + if(defined $opt::L) { + $Global::max_lines = $opt::L; + if(not $opt::pipe) { + # --pipe -L means length of record - not max_number_of_args + $Global::max_number_of_args ||= $Global::max_lines; + } + } + if(defined $opt::max_replace_args) { + $Global::max_number_of_args = $opt::max_replace_args; + $Global::ContextReplace = 1; + } + if((defined $opt::L or defined $opt::max_replace_args) + and + not ($opt::xargs or $opt::m)) { + $Global::ContextReplace = 1; + } + if(defined $opt::tag and not defined $opt::tagstring) { + $opt::tagstring = "\257<\257>"; # Default = {} + } + if(defined $opt::pipepart and + (defined $opt::L or defined $opt::max_lines + or defined $opt::max_replace_args)) { + ::error("--pipepart is incompatible with --max-replace-args, ", + "--max-lines, and -L.\n"); + wait_and_exit(255); + } + if(grep /^$Global::arg_sep$|^$Global::arg_file_sep$/o, @ARGV) { + # Deal with ::: and :::: + @ARGV=read_args_from_command_line(); + } + + # Semaphore defaults + # Must be done before computing number of processes and max_line_length + # because when running as a semaphore GNU Parallel does not read args + $Global::semaphore ||= ($0 =~ m:(^|/)sem$:); # called as 'sem' + if($Global::semaphore) { + # A semaphore does not take input from neither stdin nor file + @opt::a = ("/dev/null"); + push(@Global::unget_argv, [Arg->new("")]); + $Semaphore::timeout = $opt::semaphoretimeout || 0; + if(defined $opt::semaphorename) { + $Semaphore::name = $opt::semaphorename; + } else { + $Semaphore::name = `tty`; + chomp $Semaphore::name; + } + $Semaphore::fg = $opt::fg; + $Semaphore::wait = $opt::wait; + $Global::default_simultaneous_sshlogins = 1; + if(not defined $opt::jobs) { + $opt::jobs = 1; + } + if($Global::interactive and $opt::bg) { + ::error("Jobs running in the ". + "background cannot be interactive.\n"); + ::wait_and_exit(255); + } + } + if(defined $opt::eta) { + $opt::progress = $opt::eta; + } + if(defined $opt::bar) { + $opt::progress = $opt::bar; + } + if(defined $opt::retired) { + ::error("-g has been retired. Use --group.\n"); + ::error("-B has been retired. Use --bf.\n"); + ::error("-T has been retired. Use --tty.\n"); + ::error("-U has been retired. Use --er.\n"); + ::error("-W has been retired. Use --wd.\n"); + ::error("-Y has been retired. Use --shebang.\n"); + ::error("-H has been retired. Use --halt.\n"); + ::error("--tollef has been retired. Use -u -q --arg-sep -- and --load for -l.\n"); + ::wait_and_exit(255); + } + citation_notice(); + + parse_sshlogin(); + parse_env_var(); + + if(remote_hosts() and ($opt::X or $opt::m or $opt::xargs)) { + # As we do not know the max line length on the remote machine + # long commands generated by xargs may fail + # If opt_N is set, it is probably safe + ::warning("Using -X or -m with --sshlogin may fail.\n"); + } + + if(not defined $opt::jobs) { + $opt::jobs = "100%"; + } + open_joblog(); +} + +sub env_quote { + # Input: + # $v = value to quote + # Returns: + # $v = value quoted as environment variable + my $v = $_[0]; + $v =~ s/([\\])/\\$1/g; + $v =~ s/([\[\] \#\'\&\<\>\(\)\;\{\}\t\"\$\`\*\174\!\?\~])/\\$1/g; + $v =~ s/\n/"\n"/g; + return $v; +} + +sub record_env { + # Record current %ENV-keys in ~/.parallel/ignored_vars + # Returns: N/A + my $ignore_filename = $ENV{'HOME'} . "/.parallel/ignored_vars"; + if(open(my $vars_fh, ">", $ignore_filename)) { + print $vars_fh map { $_,"\n" } keys %ENV; + } else { + ::error("Cannot write to $ignore_filename\n"); + ::wait_and_exit(255); + } +} + +sub parse_env_var { + # Parse --env and set $Global::envvar, $Global::envwarn and $Global::envvarlen + # + # Bash functions must be parsed to export them remotely + # Pre-shellshock style bash function: + # myfunc=() {... + # Post-shellshock style bash function: + # BASH_FUNC_myfunc()=() {... + # + # Uses: + # $Global::envvar = eval string that will set variables in both bash and csh + # $Global::envwarn = If functions are used: Give warning in csh + # $Global::envvarlen = length of $Global::envvar + # @opt::env + # $Global::shell + # %ENV + # Returns: N/A + $Global::envvar = ""; + $Global::envwarn = ""; + my @vars = ('parallel_bash_environment'); + for my $varstring (@opt::env) { + # Split up --env VAR1,VAR2 + push @vars, split /,/, $varstring; + } + if(grep { /^_$/ } @vars) { + # --env _ + # Include all vars that are not in a clean environment + if(open(my $vars_fh, "<", $ENV{'HOME'} . "/.parallel/ignored_vars")) { + my @ignore = <$vars_fh>; + chomp @ignore; + my %ignore; + @ignore{@ignore} = @ignore; + close $vars_fh; + push @vars, grep { not defined $ignore{$_} } keys %ENV; + @vars = grep { not /^_$/ } @vars; + } else { + ::error("Run '$Global::progname --record-env' in a clean environment first.\n"); + ::wait_and_exit(255); + } + } + # Duplicate vars as BASH functions to include post-shellshock functions. + # So --env myfunc should also look for BASH_FUNC_myfunc() + @vars = map { $_, "BASH_FUNC_$_()" } @vars; + # Keep only defined variables + @vars = grep { defined($ENV{$_}) } @vars; + # Pre-shellshock style bash function: + # myfunc=() { echo myfunc + # } + # Post-shellshock style bash function: + # BASH_FUNC_myfunc()=() { echo myfunc + # } + my @bash_functions = grep { substr($ENV{$_},0,4) eq "() {" } @vars; + my @non_functions = grep { substr($ENV{$_},0,4) ne "() {" } @vars; + if(@bash_functions) { + # Functions are not supported for all shells + if($Global::shell !~ m:/(bash|rbash|zsh|rzsh|dash|ksh):) { + ::warning("Shell functions may not be supported in $Global::shell\n"); + } + } + + # Pre-shellschock names are without () + my @bash_pre_shellshock = grep { not /\(\)/ } @bash_functions; + # Post-shellschock names are with () + my @bash_post_shellshock = grep { /\(\)/ } @bash_functions; + + my @qcsh = (map { my $a=$_; "setenv $a " . env_quote($ENV{$a}) } + grep { not /^parallel_bash_environment$/ } @non_functions); + my @qbash = (map { my $a=$_; "export $a=" . env_quote($ENV{$a}) } + @non_functions, @bash_pre_shellshock); + + push @qbash, map { my $a=$_; "eval $a\"\$$a\"" } @bash_pre_shellshock; + push @qbash, map { /BASH_FUNC_(.*)\(\)/; "$1 $ENV{$_}" } @bash_post_shellshock; + + #ssh -tt -oLogLevel=quiet lo 'eval `echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;echo\ \$SHELL\ \|\ grep\ \"/t\\\{0,1\\\}csh\"\ \>\ /dev/null\ \&\&\ setenv\ BASH_FUNC_myfunc\ \\\(\\\)\\\ \\\{\\\ \\\ echo\\\ a\"' + #'\"\\\}\ \|\|\ myfunc\(\)\ \{\ \ echo\ a' + #'\}\ \;myfunc\ 1; + + # Check if any variables contain \n + if(my @v = map { s/BASH_FUNC_(.*)\(\)/$1/; $_ } grep { $ENV{$_}=~/\n/ } @vars) { + # \n is bad for csh and will cause it to fail. + $Global::envwarn = ::shell_quote_scalar(q{echo $SHELL | grep -E "/t?csh" > /dev/null && echo CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset }."@v".q{ && exec false;}."\n\n") . $Global::envwarn; + } + + if(not @qcsh) { push @qcsh, "true"; } + if(not @qbash) { push @qbash, "true"; } + # Create lines like: + # echo $SHELL | grep "/t\\{0,1\\}csh" >/dev/null && setenv V1 val1 && setenv V2 val2 || export V1=val1 && export V2=val2 ; echo "$V1$V2" + if(@vars) { + $Global::envvar .= + join"", + (q{echo $SHELL | grep "/t\\{0,1\\}csh" > /dev/null && } + . join(" && ", @qcsh) + . q{ || } + . join(" && ", @qbash) + .q{;}); + if($ENV{'parallel_bash_environment'}) { + $Global::envvar .= 'eval "$parallel_bash_environment";'."\n"; + } + } + $Global::envvarlen = length $Global::envvar; +} + +sub open_joblog { + # Open joblog as specified by --joblog + # Uses: + # $opt::resume + # $opt::resume_failed + # $opt::joblog + # $opt::results + # $Global::job_already_run + # %Global::fd + my $append = 0; + if(($opt::resume or $opt::resume_failed) + and + not ($opt::joblog or $opt::results)) { + ::error("--resume and --resume-failed require --joblog or --results.\n"); + ::wait_and_exit(255); + } + if($opt::joblog) { + if($opt::resume || $opt::resume_failed) { + if(open(my $joblog_fh, "<", $opt::joblog)) { + # Read the joblog + $append = <$joblog_fh>; # If there is a header: Open as append later + my $joblog_regexp; + if($opt::resume_failed) { + # Make a regexp that only matches commands with exit+signal=0 + # 4 host 1360490623.067 3.445 1023 1222 0 0 command + $joblog_regexp='^(\d+)(?:\t[^\t]+){5}\t0\t0\t'; + } else { + # Just match the job number + $joblog_regexp='^(\d+)'; + } + while(<$joblog_fh>) { + if(/$joblog_regexp/o) { + # This is 30% faster than set_job_already_run($1); + vec($Global::job_already_run,($1||0),1) = 1; + } elsif(not /\d+\s+[^\s]+\s+([0-9.]+\s+){6}/) { + ::error("Format of '$opt::joblog' is wrong: $_"); + ::wait_and_exit(255); + } + } + close $joblog_fh; + } + } + if($append) { + # Append to joblog + if(not open($Global::joblog, ">>", $opt::joblog)) { + ::error("Cannot append to --joblog $opt::joblog.\n"); + ::wait_and_exit(255); + } + } else { + if($opt::joblog eq "-") { + # Use STDOUT as joblog + $Global::joblog = $Global::fd{1}; + } elsif(not open($Global::joblog, ">", $opt::joblog)) { + # Overwrite the joblog + ::error("Cannot write to --joblog $opt::joblog.\n"); + ::wait_and_exit(255); + } + print $Global::joblog + join("\t", "Seq", "Host", "Starttime", "JobRuntime", + "Send", "Receive", "Exitval", "Signal", "Command" + ). "\n"; + } + } +} + +sub find_compression_program { + # Find a fast compression program + # Returns: + # $compress_program = compress program with options + # $decompress_program = decompress program with options + + # Search for these. Sorted by speed + my @prg = qw(lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2); + for my $p (@prg) { + if(which($p)) { + return ("$p -c -1","$p -dc"); + } + } + # Fall back to cat + return ("cat","cat"); +} + + +sub read_options { + # Read options from command line, profile and $PARALLEL + # Uses: + # $opt::shebang_wrap + # $opt::shebang + # @ARGV + # $opt::plain + # @opt::profile + # $ENV{'HOME'} + # $ENV{'PARALLEL'} + # Returns: + # @ARGV_no_opt = @ARGV without --options + + # This must be done first as this may exec myself + if(defined $ARGV[0] and ($ARGV[0] =~ /^--shebang/ or + $ARGV[0] =~ /^--shebang-?wrap/ or + $ARGV[0] =~ /^--hashbang/)) { + # Program is called from #! line in script + # remove --shebang-wrap if it is set + $opt::shebang_wrap = ($ARGV[0] =~ s/^--shebang-?wrap *//); + # remove --shebang if it is set + $opt::shebang = ($ARGV[0] =~ s/^--shebang *//); + # remove --hashbang if it is set + $opt::shebang .= ($ARGV[0] =~ s/^--hashbang *//); + if($opt::shebang) { + my $argfile = shell_quote_scalar(pop @ARGV); + # exec myself to split $ARGV[0] into separate fields + exec "$0 --skip-first-line -a $argfile @ARGV"; + } + if($opt::shebang_wrap) { + my @options; + my @parser; + if ($^O eq 'freebsd') { + # FreeBSD's #! puts different values in @ARGV than Linux' does. + my @nooptions = @ARGV; + get_options_from_array(\@nooptions); + while($#ARGV > $#nooptions) { + push @options, shift @ARGV; + } + while(@ARGV and $ARGV[0] ne ":::") { + push @parser, shift @ARGV; + } + if(@ARGV and $ARGV[0] eq ":::") { + shift @ARGV; + } + } else { + @options = shift @ARGV; + } + my $script = shell_quote_scalar(shift @ARGV); + # exec myself to split $ARGV[0] into separate fields + exec "$0 --internal-pipe-means-argfiles @options @parser $script ::: @ARGV"; + } + } + + Getopt::Long::Configure("bundling","require_order"); + my @ARGV_copy = @ARGV; + # Check if there is a --profile to set @opt::profile + get_options_from_array(\@ARGV_copy,"profile|J=s","plain") || die_usage(); + my @ARGV_profile = (); + my @ARGV_env = (); + if(not $opt::plain) { + # Add options from .parallel/config and other profiles + my @config_profiles = ( + "/etc/parallel/config", + $ENV{'HOME'}."/.parallel/config", + $ENV{'HOME'}."/.parallelrc"); + my @profiles = @config_profiles; + if(@opt::profile) { + # --profile overrides default profiles + @profiles = (); + for my $profile (@opt::profile) { + if(-r $profile) { + push @profiles, $profile; + } else { + push @profiles, $ENV{'HOME'}."/.parallel/".$profile; + } + } + } + for my $profile (@profiles) { + if(-r $profile) { + open (my $in_fh, "<", $profile) || ::die_bug("read-profile: $profile"); + while(<$in_fh>) { + /^\s*\#/ and next; + chomp; + push @ARGV_profile, shellwords($_); + } + close $in_fh; + } else { + if(grep /^$profile$/, @config_profiles) { + # config file is not required to exist + } else { + ::error("$profile not readable.\n"); + wait_and_exit(255); + } + } + } + # Add options from shell variable $PARALLEL + if($ENV{'PARALLEL'}) { + @ARGV_env = shellwords($ENV{'PARALLEL'}); + } + } + Getopt::Long::Configure("bundling","require_order"); + get_options_from_array(\@ARGV_profile) || die_usage(); + get_options_from_array(\@ARGV_env) || die_usage(); + get_options_from_array(\@ARGV) || die_usage(); + + # Prepend non-options to @ARGV (such as commands like 'nice') + unshift @ARGV, @ARGV_profile, @ARGV_env; + return @ARGV; +} + +sub read_args_from_command_line { + # Arguments given on the command line after: + # ::: ($Global::arg_sep) + # :::: ($Global::arg_file_sep) + # Removes the arguments from @ARGV and: + # - puts filenames into -a + # - puts arguments into files and add the files to -a + # Input: + # @::ARGV = command option ::: arg arg arg :::: argfiles + # Uses: + # $Global::arg_sep + # $Global::arg_file_sep + # $opt::internal_pipe_means_argfiles + # $opt::pipe + # @opt::a + # Returns: + # @argv_no_argsep = @::ARGV without ::: and :::: and following args + my @new_argv = (); + for(my $arg = shift @ARGV; @ARGV; $arg = shift @ARGV) { + if($arg eq $Global::arg_sep + or + $arg eq $Global::arg_file_sep) { + my $group = $arg; # This group of arguments is args or argfiles + my @group; + while(defined ($arg = shift @ARGV)) { + if($arg eq $Global::arg_sep + or + $arg eq $Global::arg_file_sep) { + # exit while loop if finding new separator + last; + } else { + # If not hitting ::: or :::: + # Append it to the group + push @group, $arg; + } + } + + if($group eq $Global::arg_file_sep + or ($opt::internal_pipe_means_argfiles and $opt::pipe) + ) { + # Group of file names on the command line. + # Append args into -a + push @opt::a, @group; + } elsif($group eq $Global::arg_sep) { + # Group of arguments on the command line. + # Put them into a file. + # Create argfile + my ($outfh,$name) = ::tmpfile(SUFFIX => ".arg"); + unlink($name); + # Put args into argfile + print $outfh map { $_,$/ } @group; + seek $outfh, 0, 0; + # Append filehandle to -a + push @opt::a, $outfh; + } else { + ::die_bug("Unknown command line group: $group"); + } + if(defined($arg)) { + # $arg is ::: or :::: + redo; + } else { + # $arg is undef -> @ARGV empty + last; + } + } + push @new_argv, $arg; + } + # Output: @ARGV = command to run with options + return @new_argv; +} + +sub cleanup { + # Returns: N/A + if(@opt::basefile) { cleanup_basefile(); } +} + +sub __QUOTING_ARGUMENTS_FOR_SHELL__ {} + +sub shell_quote { + # Input: + # @strings = strings to be quoted + # Output: + # @shell_quoted_strings = string quoted with \ as needed by the shell + my @strings = (@_); + for my $a (@strings) { + $a =~ s/([\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\*\>\<\~\|\; \"\!\$\&\'\202-\377])/\\$1/g; + $a =~ s/[\n]/'\n'/g; # filenames with '\n' is quoted using \' + } + return wantarray ? @strings : "@strings"; +} + +sub shell_quote_empty { + # Inputs: + # @strings = strings to be quoted + # Returns: + # @quoted_strings = empty strings quoted as ''. + my @strings = shell_quote(@_); + for my $a (@strings) { + if($a eq "") { + $a = "''"; + } + } + return wantarray ? @strings : "@strings"; +} + +sub shell_quote_scalar { + # Quote the string so shell will not expand any special chars + # Inputs: + # $string = string to be quoted + # Returns: + # $shell_quoted = string quoted with \ as needed by the shell + my $a = $_[0]; + if(defined $a) { + # $a =~ s/([\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\*\>\<\~\|\; \"\!\$\&\'\202-\377])/\\$1/g; + # This is 1% faster than the above + $a =~ s/[\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\*\>\<\~\|\; \"\!\$\&\'\202-\377]/\\$&/go; + $a =~ s/[\n]/'\n'/go; # filenames with '\n' is quoted using \' + } + return $a; +} + +sub shell_quote_file { + # Quote the string so shell will not expand any special chars and prepend ./ if needed + # Input: + # $filename = filename to be shell quoted + # Returns: + # $quoted_filename = filename quoted with \ as needed by the shell and ./ if needed + my $a = shell_quote_scalar(shift); + if(defined $a) { + if($a =~ m:^/: or $a =~ m:^\./:) { + # /abs/path or ./rel/path => skip + } else { + # rel/path => ./rel/path + $a = "./".$a; + } + } + return $a; +} + +sub shellwords { + # Input: + # $string = shell line + # Returns: + # @shell_words = $string split into words as shell would do + $Global::use{"Text::ParseWords"} ||= eval "use Text::ParseWords; 1;"; + return Text::ParseWords::shellwords(@_); +} + + +sub __FILEHANDLES__ {} + + +sub save_stdin_stdout_stderr { + # Remember the original STDIN, STDOUT and STDERR + # and file descriptors opened by the shell (e.g. 3>/tmp/foo) + # Uses: + # %Global::fd + # $Global::original_stderr + # $Global::original_stdin + # Returns: N/A + + # Find file descriptors that are already opened (by the shell) + for my $fdno (1..61) { + # /dev/fd/62 and above are used by bash for <(cmd) + my $fh; + # 2-argument-open is used to be compatible with old perl 5.8.0 + # bug #43570: Perl 5.8.0 creates 61 files + if(open($fh,">&=$fdno")) { + $Global::fd{$fdno}=$fh; + } + } + open $Global::original_stderr, ">&", "STDERR" or + ::die_bug("Can't dup STDERR: $!"); + open $Global::original_stdin, "<&", "STDIN" or + ::die_bug("Can't dup STDIN: $!"); + $Global::is_terminal = (-t $Global::original_stderr) && !$ENV{'CIRCLECI'} && !$ENV{'TRAVIS'}; +} + +sub enough_file_handles { + # Check that we have enough filehandles available for starting + # another job + # Uses: + # $opt::ungroup + # %Global::fd + # Returns: + # 1 if ungrouped (thus not needing extra filehandles) + # 0 if too few filehandles + # 1 if enough filehandles + if(not $opt::ungroup) { + my %fh; + my $enough_filehandles = 1; + # perl uses 7 filehandles for something? + # open3 uses 2 extra filehandles temporarily + # We need a filehandle for each redirected file descriptor + # (normally just STDOUT and STDERR) + for my $i (1..(7+2+keys %Global::fd)) { + $enough_filehandles &&= open($fh{$i}, "<", "/dev/null"); + } + for (values %fh) { close $_; } + return $enough_filehandles; + } else { + # Ungrouped does not need extra file handles + return 1; + } +} + +sub open_or_exit { + # Open a file name or exit if the file cannot be opened + # Inputs: + # $file = filehandle or filename to open + # Uses: + # $Global::stdin_in_opt_a + # $Global::original_stdin + # Returns: + # $fh = file handle to read-opened file + my $file = shift; + if($file eq "-") { + $Global::stdin_in_opt_a = 1; + return ($Global::original_stdin || *STDIN); + } + if(ref $file eq "GLOB") { + # This is an open filehandle + return $file; + } + my $fh = gensym; + if(not open($fh, "<", $file)) { + ::error("Cannot open input file `$file': No such file or directory.\n"); + wait_and_exit(255); + } + return $fh; +} + +sub __RUNNING_THE_JOBS_AND_PRINTING_PROGRESS__ {} + +# Variable structure: +# +# $Global::running{$pid} = Pointer to Job-object +# @Global::virgin_jobs = Pointer to Job-object that have received no input +# $Global::host{$sshlogin} = Pointer to SSHLogin-object +# $Global::total_running = total number of running jobs +# $Global::total_started = total jobs started + +sub init_run_jobs { + $Global::total_running = 0; + $Global::total_started = 0; + $Global::tty_taken = 0; + $SIG{USR1} = \&list_running_jobs; + $SIG{USR2} = \&toggle_progress; + if(@opt::basefile) { setup_basefile(); } +} + +{ + my $last_time; + my %last_mtime; + +sub start_more_jobs { + # Run start_another_job() but only if: + # * not $Global::start_no_new_jobs set + # * not JobQueue is empty + # * not load on server is too high + # * not server swapping + # * not too short time since last remote login + # Uses: + # $Global::max_procs_file + # $Global::max_procs_file_last_mod + # %Global::host + # @opt::sshloginfile + # $Global::start_no_new_jobs + # $opt::filter_hosts + # $Global::JobQueue + # $opt::pipe + # $opt::load + # $opt::noswap + # $opt::delay + # $Global::newest_starttime + # Returns: + # $jobs_started = number of jobs started + my $jobs_started = 0; + my $jobs_started_this_round = 0; + if($Global::start_no_new_jobs) { + return $jobs_started; + } + if(time - ($last_time||0) > 1) { + # At most do this every second + $last_time = time; + if($Global::max_procs_file) { + # --jobs filename + my $mtime = (stat($Global::max_procs_file))[9]; + if($mtime > $Global::max_procs_file_last_mod) { + # file changed: Force re-computing max_jobs_running + $Global::max_procs_file_last_mod = $mtime; + for my $sshlogin (values %Global::host) { + $sshlogin->set_max_jobs_running(undef); + } + } + } + if(@opt::sshloginfile) { + # Is --sshloginfile changed? + for my $slf (@opt::sshloginfile) { + my $actual_file = expand_slf_shorthand($slf); + my $mtime = (stat($actual_file))[9]; + $last_mtime{$actual_file} ||= $mtime; + if($mtime - $last_mtime{$actual_file} > 1) { + ::debug("run","--sshloginfile $actual_file changed. reload\n"); + $last_mtime{$actual_file} = $mtime; + # Reload $slf + # Empty sshlogins + @Global::sshlogin = (); + for (values %Global::host) { + # Don't start new jobs on any host + # except the ones added back later + $_->set_max_jobs_running(0); + } + # This will set max_jobs_running on the SSHlogins + read_sshloginfile($actual_file); + parse_sshlogin(); + $opt::filter_hosts and filter_hosts(); + setup_basefile(); + } + } + } + } + do { + $jobs_started_this_round = 0; + # This will start 1 job on each --sshlogin (if possible) + # thus distribute the jobs on the --sshlogins round robin + + for my $sshlogin (values %Global::host) { + if($Global::JobQueue->empty() and not $opt::pipe) { + # No more jobs in the queue + last; + } + debug("run", "Running jobs before on ", $sshlogin->string(), ": ", + $sshlogin->jobs_running(), "\n"); + if ($sshlogin->jobs_running() < $sshlogin->max_jobs_running()) { + if($opt::load and $sshlogin->loadavg_too_high()) { + # The load is too high or unknown + next; + } + if($opt::noswap and $sshlogin->swapping()) { + # The server is swapping + next; + } + if($sshlogin->too_fast_remote_login()) { + # It has been too short since + next; + } + if($opt::delay and $opt::delay > ::now() - $Global::newest_starttime) { + # It has been too short since last start + next; + } + debug("run", $sshlogin->string(), " has ", $sshlogin->jobs_running(), + " out of ", $sshlogin->max_jobs_running(), + " jobs running. Start another.\n"); + if(start_another_job($sshlogin) == 0) { + # No more jobs to start on this $sshlogin + debug("run","No jobs started on ", $sshlogin->string(), "\n"); + next; + } + $sshlogin->inc_jobs_running(); + $sshlogin->set_last_login_at(::now()); + $jobs_started++; + $jobs_started_this_round++; + } + debug("run","Running jobs after on ", $sshlogin->string(), ": ", + $sshlogin->jobs_running(), " of ", + $sshlogin->max_jobs_running(), "\n"); + } + } while($jobs_started_this_round); + + return $jobs_started; +} +} + +{ + my $no_more_file_handles_warned; + +sub start_another_job { + # If there are enough filehandles + # and JobQueue not empty + # and not $job is in joblog + # Then grab a job from Global::JobQueue, + # start it at sshlogin + # mark it as virgin_job + # Inputs: + # $sshlogin = the SSHLogin to start the job on + # Uses: + # $Global::JobQueue + # $opt::pipe + # $opt::results + # $opt::resume + # @Global::virgin_jobs + # Returns: + # 1 if another jobs was started + # 0 otherwise + my $sshlogin = shift; + # Do we have enough file handles to start another job? + if(enough_file_handles()) { + if($Global::JobQueue->empty() and not $opt::pipe) { + # No more commands to run + debug("start", "Not starting: JobQueue empty\n"); + return 0; + } else { + my $job; + # Skip jobs already in job log + # Skip jobs already in results + do { + $job = get_job_with_sshlogin($sshlogin); + if(not defined $job) { + # No command available for that sshlogin + debug("start", "Not starting: no jobs available for ", + $sshlogin->string(), "\n"); + return 0; + } + } while ($job->is_already_in_joblog() + or + ($opt::results and $opt::resume and $job->is_already_in_results())); + debug("start", "Command to run on '", $job->sshlogin()->string(), "': '", + $job->replaced(),"'\n"); + if($job->start()) { + if($opt::pipe) { + push(@Global::virgin_jobs,$job); + } + debug("start", "Started as seq ", $job->seq(), + " pid:", $job->pid(), "\n"); + return 1; + } else { + # Not enough processes to run the job. + # Put it back on the queue. + $Global::JobQueue->unget($job); + # Count down the number of jobs to run for this SSHLogin. + my $max = $sshlogin->max_jobs_running(); + if($max > 1) { $max--; } else { + ::error("No more processes: cannot run a single job. Something is wrong.\n"); + ::wait_and_exit(255); + } + $sshlogin->set_max_jobs_running($max); + # Sleep up to 300 ms to give other processes time to die + ::usleep(rand()*300); + ::warning("No more processes: ", + "Decreasing number of running jobs to $max. ", + "Raising ulimit -u or /etc/security/limits.conf may help.\n"); + return 0; + } + } + } else { + # No more file handles + $no_more_file_handles_warned++ or + ::warning("No more file handles. ", + "Raising ulimit -n or /etc/security/limits.conf may help.\n"); + return 0; + } +} +} + +$opt::min_progress_interval = 0; + +sub init_progress { + # Uses: + # $opt::bar + # Returns: + # list of computers for progress output + $|=1; + if (not $Global::is_terminal) { + $opt::min_progress_interval = 30; + } + if($opt::bar) { + return("",""); + } + my %progress = progress(); + return ("\nComputers / CPU cores / Max jobs to run\n", + $progress{'workerlist'}); +} + +sub drain_job_queue { + # Uses: + # $opt::progress + # $Global::original_stderr + # $Global::total_running + # $Global::max_jobs_running + # %Global::running + # $Global::JobQueue + # %Global::host + # $Global::start_no_new_jobs + # Returns: N/A + if($opt::progress) { + print $Global::original_stderr init_progress(); + } + my $last_header=""; + my $sleep = 0.2; + my $last_left = 1000000000; + my $last_progress_time = 0; + my $ps_reported = 0; + do { + while($Global::total_running > 0) { + debug($Global::total_running, "==", scalar + keys %Global::running," slots: ", $Global::max_jobs_running); + if($opt::pipe) { + # When using --pipe sometimes file handles are not closed properly + for my $job (values %Global::running) { + close $job->fh(0,"w"); + } + } + # When not connected to terminal, assume CI (e.g. CircleCI). In + # that case we want occasional progress output to prevent abort + # due to timeout with no output, but we also need to stop sending + # progress output if there has been no actual progress, so that + # the job can time out appropriately (CirecleCI: 10m) in case of + # a hung test. But without special output, it is extremely + # annoying to diagnose which test is hung, so we add that using + # `ps` below. + if($opt::progress and + ($Global::is_terminal or (time() - $last_progress_time) >= 30)) { + my %progress = progress(); + if($last_header ne $progress{'header'}) { + print $Global::original_stderr "\n", $progress{'header'}, "\n"; + $last_header = $progress{'header'}; + } + if ($Global::is_terminal) { + print $Global::original_stderr "\r",$progress{'status'}; + } + if ($last_left > $Global::left) { + if (not $Global::is_terminal) { + print $Global::original_stderr $progress{'status'},"\n"; + } + $last_progress_time = time(); + $ps_reported = 0; + } elsif (not $ps_reported and (time() - $last_progress_time) >= 60) { + # No progress in at least 60 seconds: run ps + print $Global::original_stderr "\n"; + my $script_dir = ::dirname($0); + system("$script_dir/ps_with_stack || ps -wwf"); + $ps_reported = 1; + } + $last_left = $Global::left; + flush $Global::original_stderr; + } + if($Global::total_running < $Global::max_jobs_running + and not $Global::JobQueue->empty()) { + # These jobs may not be started because of loadavg + # or too little time between each ssh login. + if(start_more_jobs() > 0) { + # Exponential back-on if jobs were started + $sleep = $sleep/2+0.001; + } + } + # Sometimes SIGCHLD is not registered, so force reaper + $sleep = ::reap_usleep($sleep); + } + if(not $Global::JobQueue->empty()) { + # These jobs may not be started: + # * because there the --filter-hosts has removed all + if(not %Global::host) { + ::error("There are no hosts left to run on.\n"); + ::wait_and_exit(255); + } + # * because of loadavg + # * because of too little time between each ssh login. + start_more_jobs(); + $sleep = ::reap_usleep($sleep); + if($Global::max_jobs_running == 0) { + ::warning("There are no job slots available. Increase --jobs.\n"); + } + } + } while ($Global::total_running > 0 + or + not $Global::start_no_new_jobs and not $Global::JobQueue->empty()); + if($opt::progress) { + my %progress = progress(); + print $Global::original_stderr $opt::progress_sep, $progress{'status'}, "\n"; + flush $Global::original_stderr; + } +} + +sub toggle_progress { + # Turn on/off progress view + # Uses: + # $opt::progress + # $Global::original_stderr + # Returns: N/A + $opt::progress = not $opt::progress; + if($opt::progress) { + print $Global::original_stderr init_progress(); + } +} + +sub progress { + # Uses: + # $opt::bar + # $opt::eta + # %Global::host + # $Global::total_started + # Returns: + # $workerlist = list of workers + # $header = that will fit on the screen + # $status = message that will fit on the screen + if($opt::bar) { + return ("workerlist" => "", "header" => "", "status" => bar()); + } + my $eta = ""; + my ($status,$header)=("",""); + if($opt::eta) { + my($total, $completed, $left, $pctcomplete, $avgtime, $this_eta) = + compute_eta(); + $eta = sprintf("ETA: %ds Left: %d AVG: %.2fs ", + $this_eta, $left, $avgtime); + $Global::left = $left; + } + my $termcols = terminal_columns(); + my @workers = sort keys %Global::host; + my %sshlogin = map { $_ eq ":" ? ($_=>"local") : ($_=>$_) } @workers; + my $workerno = 1; + my %workerno = map { ($_=>$workerno++) } @workers; + my $workerlist = ""; + for my $w (@workers) { + $workerlist .= + $workerno{$w}.":".$sshlogin{$w} ." / ". + ($Global::host{$w}->ncpus() || "-")." / ". + $Global::host{$w}->max_jobs_running()."\n"; + } + $status = "x"x($termcols+1); + if(length $status > $termcols) { + # sshlogin1:XX/XX/XX%/XX.Xs sshlogin2:XX/XX/XX%/XX.Xs sshlogin3:XX/XX/XX%/XX.Xs + $header = "Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete"; + $status = $eta . + join(" ",map + { + if($Global::total_started) { + my $completed = ($Global::host{$_}->jobs_completed()||0); + my $running = $Global::host{$_}->jobs_running(); + my $time = $completed ? (time-$^T)/($completed) : "0"; + sprintf("%s:%d/%d/%d%%/%.1fs ", + $sshlogin{$_}, $running, $completed, + ($running+$completed)*100 + / $Global::total_started, $time); + } + } @workers); + } + if(length $status > $termcols) { + # 1:XX/XX/XX%/XX.Xs 2:XX/XX/XX%/XX.Xs 3:XX/XX/XX%/XX.Xs 4:XX/XX/XX%/XX.Xs + $header = "Computer:jobs running/jobs completed/%of started jobs"; + $status = $eta . + join(" ",map + { + my $completed = ($Global::host{$_}->jobs_completed()||0); + my $running = $Global::host{$_}->jobs_running(); + my $time = $completed ? (time-$^T)/($completed) : "0"; + sprintf("%s:%d/%d/%d%%/%.1fs ", + $workerno{$_}, $running, $completed, + ($running+$completed)*100 + / $Global::total_started, $time); + } @workers); + } + if(length $status > $termcols) { + # sshlogin1:XX/XX/XX% sshlogin2:XX/XX/XX% sshlogin3:XX/XX/XX% + $header = "Computer:jobs running/jobs completed/%of started jobs"; + $status = $eta . + join(" ",map + { sprintf("%s:%d/%d/%d%%", + $sshlogin{$_}, + $Global::host{$_}->jobs_running(), + ($Global::host{$_}->jobs_completed()||0), + ($Global::host{$_}->jobs_running()+ + ($Global::host{$_}->jobs_completed()||0))*100 + / $Global::total_started) } + @workers); + } + if(length $status > $termcols) { + # 1:XX/XX/XX% 2:XX/XX/XX% 3:XX/XX/XX% 4:XX/XX/XX% 5:XX/XX/XX% 6:XX/XX/XX% + $header = "Computer:jobs running/jobs completed/%of started jobs"; + $status = $eta . + join(" ",map + { sprintf("%s:%d/%d/%d%%", + $workerno{$_}, + $Global::host{$_}->jobs_running(), + ($Global::host{$_}->jobs_completed()||0), + ($Global::host{$_}->jobs_running()+ + ($Global::host{$_}->jobs_completed()||0))*100 + / $Global::total_started) } + @workers); + } + if(length $status > $termcols) { + # sshlogin1:XX/XX/XX% sshlogin2:XX/XX/XX% sshlogin3:XX/XX sshlogin4:XX/XX + $header = "Computer:jobs running/jobs completed"; + $status = $eta . + join(" ",map + { sprintf("%s:%d/%d", + $sshlogin{$_}, $Global::host{$_}->jobs_running(), + ($Global::host{$_}->jobs_completed()||0)) } + @workers); + } + if(length $status > $termcols) { + # sshlogin1:XX/XX sshlogin2:XX/XX sshlogin3:XX/XX sshlogin4:XX/XX + $header = "Computer:jobs running/jobs completed"; + $status = $eta . + join(" ",map + { sprintf("%s:%d/%d", + $sshlogin{$_}, $Global::host{$_}->jobs_running(), + ($Global::host{$_}->jobs_completed()||0)) } + @workers); + } + if(length $status > $termcols) { + # 1:XX/XX 2:XX/XX 3:XX/XX 4:XX/XX 5:XX/XX 6:XX/XX + $header = "Computer:jobs running/jobs completed"; + $status = $eta . + join(" ",map + { sprintf("%s:%d/%d", + $workerno{$_}, $Global::host{$_}->jobs_running(), + ($Global::host{$_}->jobs_completed()||0)) } + @workers); + } + if(length $status > $termcols) { + # sshlogin1:XX sshlogin2:XX sshlogin3:XX sshlogin4:XX sshlogin5:XX + $header = "Computer:jobs completed"; + $status = $eta . + join(" ",map + { sprintf("%s:%d", + $sshlogin{$_}, + ($Global::host{$_}->jobs_completed()||0)) } + @workers); + } + if(length $status > $termcols) { + # 1:XX 2:XX 3:XX 4:XX 5:XX 6:XX + $header = "Computer:jobs completed"; + $status = $eta . + join(" ",map + { sprintf("%s:%d", + $workerno{$_}, + ($Global::host{$_}->jobs_completed()||0)) } + @workers); + } + return ("workerlist" => $workerlist, "header" => $header, "status" => $status); +} + +{ + my ($total, $first_completed, $smoothed_avg_time); + + sub compute_eta { + # Calculate important numbers for ETA + # Returns: + # $total = number of jobs in total + # $completed = number of jobs completed + # $left = number of jobs left + # $pctcomplete = percent of jobs completed + # $avgtime = averaged time + # $eta = smoothed eta + $total ||= $Global::JobQueue->total_jobs(); + my $completed = 0; + for(values %Global::host) { $completed += $_->jobs_completed() } + my $left = $total - $completed; + if(not $completed) { + return($total, $completed, $left, 0, 0, 0); + } + my $pctcomplete = $completed / $total; + $first_completed ||= time; + my $timepassed = (time - $first_completed); + my $avgtime = $timepassed / $completed; + $smoothed_avg_time ||= $avgtime; + # Smooth the eta so it does not jump wildly + $smoothed_avg_time = (1 - $pctcomplete) * $smoothed_avg_time + + $pctcomplete * $avgtime; + my $eta = int($left * $smoothed_avg_time); + return($total, $completed, $left, $pctcomplete, $avgtime, $eta); + } +} + +{ + my ($rev,$reset); + + sub bar { + # Return: + # $status = bar with eta, completed jobs, arg and pct + $rev ||= "\033[7m"; + $reset ||= "\033[0m"; + my($total, $completed, $left, $pctcomplete, $avgtime, $eta) = + compute_eta(); + my $arg = $Global::newest_job ? + $Global::newest_job->{'commandline'}->replace_placeholders(["\257<\257>"],0,0) : ""; + # These chars mess up display in the terminal + $arg =~ tr/[\011-\016\033\302-\365]//d; + my $bar_text = + sprintf("%d%% %d:%d=%ds %s", + $pctcomplete*100, $completed, $left, $eta, $arg); + my $terminal_width = terminal_columns(); + my $s = sprintf("%-${terminal_width}s", + substr($bar_text." "x$terminal_width, + 0,$terminal_width)); + my $width = int($terminal_width * $pctcomplete); + substr($s,$width,0) = $reset; + my $zenity = sprintf("%-${terminal_width}s", + substr("# $eta sec $arg", + 0,$terminal_width)); + $s = "\r" . $zenity . "\r" . $pctcomplete*100 . # Prefix with zenity header + "\r" . $rev . $s . $reset; + return $s; + } +} + +{ + my ($columns,$last_column_time); + + sub terminal_columns { + # Get the number of columns of the display + # Returns: + # number of columns of the screen + if(not $columns or $last_column_time < time) { + $last_column_time = time; + $columns = $ENV{'COLUMNS'}; + if(not $columns) { + my $resize = qx{ resize 2>/dev/null }; + $resize =~ /COLUMNS=(\d+);/ and do { $columns = $1; }; + } + $columns ||= 80; + } + return $columns; + } +} + +sub get_job_with_sshlogin { + # Returns: + # next job object for $sshlogin if any available + my $sshlogin = shift; + my $job = undef; + + if ($opt::hostgroups) { + my @other_hostgroup_jobs = (); + + while($job = $Global::JobQueue->get()) { + if($sshlogin->in_hostgroups($job->hostgroups())) { + # Found a job for this hostgroup + last; + } else { + # This job was not in the hostgroups of $sshlogin + push @other_hostgroup_jobs, $job; + } + } + $Global::JobQueue->unget(@other_hostgroup_jobs); + if(not defined $job) { + # No more jobs + return undef; + } + } else { + $job = $Global::JobQueue->get(); + if(not defined $job) { + # No more jobs + ::debug("start", "No more jobs: JobQueue empty\n"); + return undef; + } + } + + my $clean_command = $job->replaced(); + if($clean_command =~ /^\s*$/) { + # Do not run empty lines + if(not $Global::JobQueue->empty()) { + return get_job_with_sshlogin($sshlogin); + } else { + return undef; + } + } + $job->set_sshlogin($sshlogin); + if($opt::retries and $clean_command and + $job->failed_here()) { + # This command with these args failed for this sshlogin + my ($no_of_failed_sshlogins,$min_failures) = $job->min_failed(); + # Only look at the Global::host that have > 0 jobslots + if($no_of_failed_sshlogins == grep { $_->max_jobs_running() > 0 } values %Global::host + and $job->failed_here() == $min_failures) { + # It failed the same or more times on another host: + # run it on this host + } else { + # If it failed fewer times on another host: + # Find another job to run + my $nextjob; + if(not $Global::JobQueue->empty()) { + # This can potentially recurse for all args + no warnings 'recursion'; + $nextjob = get_job_with_sshlogin($sshlogin); + } + # Push the command back on the queue + $Global::JobQueue->unget($job); + return $nextjob; + } + } + return $job; +} + +sub __REMOTE_SSH__ {} + +sub read_sshloginfiles { + # Returns: N/A + for my $s (@_) { + read_sshloginfile(expand_slf_shorthand($s)); + } +} + +sub expand_slf_shorthand { + my $file = shift; + if($file eq "-") { + # skip: It is stdin + } elsif($file eq "..") { + $file = $ENV{'HOME'}."/.parallel/sshloginfile"; + } elsif($file eq ".") { + $file = "/etc/parallel/sshloginfile"; + } elsif(not -r $file) { + if(not -r $ENV{'HOME'}."/.parallel/".$file) { + # Try prepending ~/.parallel + ::error("Cannot open $file.\n"); + ::wait_and_exit(255); + } else { + $file = $ENV{'HOME'}."/.parallel/".$file; + } + } + return $file; +} + +sub read_sshloginfile { + # Returns: N/A + my $file = shift; + my $close = 1; + my $in_fh; + ::debug("init","--slf ",$file); + if($file eq "-") { + $in_fh = *STDIN; + $close = 0; + } else { + if(not open($in_fh, "<", $file)) { + # Try the filename + ::error("Cannot open $file.\n"); + ::wait_and_exit(255); + } + } + while(<$in_fh>) { + chomp; + /^\s*#/ and next; + /^\s*$/ and next; + push @Global::sshlogin, $_; + } + if($close) { + close $in_fh; + } +} + +sub parse_sshlogin { + # Returns: N/A + my @login; + if(not @Global::sshlogin) { @Global::sshlogin = (":"); } + for my $sshlogin (@Global::sshlogin) { + # Split up -S sshlogin,sshlogin + for my $s (split /,/, $sshlogin) { + if ($s eq ".." or $s eq "-") { + # This may add to @Global::sshlogin - possibly bug + read_sshloginfile(expand_slf_shorthand($s)); + } else { + push (@login, $s); + } + } + } + $Global::minimal_command_line_length = 8_000_000; + my @allowed_hostgroups; + for my $ncpu_sshlogin_string (::uniq(@login)) { + my $sshlogin = SSHLogin->new($ncpu_sshlogin_string); + my $sshlogin_string = $sshlogin->string(); + if($sshlogin_string eq "") { + # This is an ssh group: -S @webservers + push @allowed_hostgroups, $sshlogin->hostgroups(); + next; + } + if($Global::host{$sshlogin_string}) { + # This sshlogin has already been added: + # It is probably a host that has come back + # Set the max_jobs_running back to the original + debug("run","Already seen $sshlogin_string\n"); + if($sshlogin->{'ncpus'}) { + # If ncpus set by '#/' of the sshlogin, overwrite it: + $Global::host{$sshlogin_string}->set_ncpus($sshlogin->ncpus()); + } + $Global::host{$sshlogin_string}->set_max_jobs_running(undef); + next; + } + if($sshlogin_string eq ":") { + $sshlogin->set_maxlength(Limits::Command::max_length()); + } else { + # If all chars needs to be quoted, every other character will be \ + $sshlogin->set_maxlength(int(Limits::Command::max_length()/2)); + } + $Global::minimal_command_line_length = + ::min($Global::minimal_command_line_length, $sshlogin->maxlength()); + $Global::host{$sshlogin_string} = $sshlogin; + } + if(@allowed_hostgroups) { + # Remove hosts that are not in these groups + while (my ($string, $sshlogin) = each %Global::host) { + if(not $sshlogin->in_hostgroups(@allowed_hostgroups)) { + delete $Global::host{$string}; + } + } + } + + # debug("start", "sshlogin: ", my_dump(%Global::host),"\n"); + if($opt::transfer or @opt::return or $opt::cleanup or @opt::basefile) { + if(not remote_hosts()) { + # There are no remote hosts + if(@opt::trc) { + ::warning("--trc ignored as there are no remote --sshlogin.\n"); + } elsif (defined $opt::transfer) { + ::warning("--transfer ignored as there are no remote --sshlogin.\n"); + } elsif (@opt::return) { + ::warning("--return ignored as there are no remote --sshlogin.\n"); + } elsif (defined $opt::cleanup) { + ::warning("--cleanup ignored as there are no remote --sshlogin.\n"); + } elsif (@opt::basefile) { + ::warning("--basefile ignored as there are no remote --sshlogin.\n"); + } + } + } +} + +sub remote_hosts { + # Return sshlogins that are not ':' + # Returns: + # list of sshlogins with ':' removed + return grep !/^:$/, keys %Global::host; +} + +sub setup_basefile { + # Transfer basefiles to each $sshlogin + # This needs to be done before first jobs on $sshlogin is run + # Returns: N/A + my $cmd = ""; + my $rsync_destdir; + my $workdir; + for my $sshlogin (values %Global::host) { + if($sshlogin->string() eq ":") { next } + for my $file (@opt::basefile) { + if($file !~ m:^/: and $opt::workdir eq "...") { + ::error("Work dir '...' will not work with relative basefiles\n"); + ::wait_and_exit(255); + } + $workdir ||= Job->new("")->workdir(); + $cmd .= $sshlogin->rsync_transfer_cmd($file,$workdir) . "&"; + } + } + $cmd .= "wait;"; + debug("init", "basesetup: $cmd\n"); + print `$cmd`; +} + +sub cleanup_basefile { + # Remove the basefiles transferred + # Returns: N/A + my $cmd=""; + my $workdir = Job->new("")->workdir(); + for my $sshlogin (values %Global::host) { + if($sshlogin->string() eq ":") { next } + for my $file (@opt::basefile) { + $cmd .= $sshlogin->cleanup_cmd($file,$workdir)."&"; + } + } + $cmd .= "wait;"; + debug("init", "basecleanup: $cmd\n"); + print `$cmd`; +} + +sub filter_hosts { + my(@cores, @cpus, @maxline, @echo); + my $envvar = ::shell_quote_scalar($Global::envvar); + while (my ($host, $sshlogin) = each %Global::host) { + if($host eq ":") { next } + # The 'true' is used to get the $host out later + my $sshcmd = "true $host;" . $sshlogin->sshcommand()." ".$sshlogin->serverlogin(); + push(@cores, $host."\t".$sshcmd." ".$envvar." parallel --number-of-cores\n\0"); + push(@cpus, $host."\t".$sshcmd." ".$envvar." parallel --number-of-cpus\n\0"); + push(@maxline, $host."\t".$sshcmd." ".$envvar." parallel --max-line-length-allowed\n\0"); + # 'echo' is used to get the best possible value for an ssh login time + push(@echo, $host."\t".$sshcmd." echo\n\0"); + } + my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".ssh"); + print $fh @cores, @cpus, @maxline, @echo; + close $fh; + # --timeout 5: Setting up an SSH connection and running a simple + # command should never take > 5 sec. + # --delay 0.1: If multiple sshlogins use the same proxy the delay + # will make it less likely to overload the ssh daemon. + # --retries 3: If the ssh daemon it overloaded, try 3 times + # -s 16000: Half of the max line on UnixWare + my $cmd = "cat $tmpfile | $0 -j0 --timeout 5 -s 16000 --joblog - --plain --delay 0.1 --retries 3 --tag --tagstring {1} -0 --colsep '\t' -k eval {2} 2>/dev/null"; + ::debug("init", $cmd, "\n"); + open(my $host_fh, "-|", $cmd) || ::die_bug("parallel host check: $cmd"); + my (%ncores, %ncpus, %time_to_login, %maxlen, %echo, @down_hosts); + my $prepend = ""; + while(<$host_fh>) { + if(/\'$/) { + # if last char = ' then append next line + # This may be due to quoting of $Global::envvar + $prepend .= $_; + next; + } + $_ = $prepend . $_; + $prepend = ""; + chomp; + my @col = split /\t/, $_; + if(defined $col[6]) { + # This is a line from --joblog + # seq host time spent sent received exit signal command + # 2 : 1372607672.654 0.675 0 0 0 0 eval true\ m\;ssh\ m\ parallel\ --number-of-cores + if($col[0] eq "Seq" and $col[1] eq "Host" and + $col[2] eq "Starttime") { + # Header => skip + next; + } + # Get server from: eval true server\; + $col[8] =~ /eval true..([^;]+).;/ or ::die_bug("col8 does not contain host: $col[8]"); + my $host = $1; + $host =~ tr/\\//d; + $Global::host{$host} or next; + if($col[6] eq "255" or $col[7] eq "15") { + # exit == 255 or signal == 15: ssh failed + # Remove sshlogin + ::debug("init", "--filtered $host\n"); + push(@down_hosts, $host); + @down_hosts = uniq(@down_hosts); + } elsif($col[6] eq "127") { + # signal == 127: parallel not installed remote + # Set ncpus and ncores = 1 + ::warning("Could not figure out ", + "number of cpus on $host. Using 1.\n"); + $ncores{$host} = 1; + $ncpus{$host} = 1; + $maxlen{$host} = Limits::Command::max_length(); + } elsif($col[0] =~ /^\d+$/ and $Global::host{$host}) { + # Remember how log it took to log in + # 2 : 1372607672.654 0.675 0 0 0 0 eval true\ m\;ssh\ m\ echo + $time_to_login{$host} = ::min($time_to_login{$host},$col[3]); + } else { + ::die_bug("host check unmatched long jobline: $_"); + } + } elsif($Global::host{$col[0]}) { + # This output from --number-of-cores, --number-of-cpus, + # --max-line-length-allowed + # ncores: server 8 + # ncpus: server 2 + # maxlen: server 131071 + if(not $ncores{$col[0]}) { + $ncores{$col[0]} = $col[1]; + } elsif(not $ncpus{$col[0]}) { + $ncpus{$col[0]} = $col[1]; + } elsif(not $maxlen{$col[0]}) { + $maxlen{$col[0]} = $col[1]; + } elsif(not $echo{$col[0]}) { + $echo{$col[0]} = $col[1]; + } elsif(m/perl: warning:|LANGUAGE =|LC_ALL =|LANG =|are supported and installed/) { + # Skip these: + # perl: warning: Setting locale failed. + # perl: warning: Please check that your locale settings: + # LANGUAGE = (unset), + # LC_ALL = (unset), + # LANG = "en_US.UTF-8" + # are supported and installed on your system. + # perl: warning: Falling back to the standard locale ("C"). + } else { + ::die_bug("host check too many col0: $_"); + } + } else { + ::die_bug("host check unmatched short jobline ($col[0]): $_"); + } + } + close $host_fh; + $Global::debug or unlink $tmpfile; + delete @Global::host{@down_hosts}; + @down_hosts and ::warning("Removed @down_hosts\n"); + $Global::minimal_command_line_length = 8_000_000; + while (my ($sshlogin, $obj) = each %Global::host) { + if($sshlogin eq ":") { next } + $ncpus{$sshlogin} or ::die_bug("ncpus missing: ".$obj->serverlogin()); + $ncores{$sshlogin} or ::die_bug("ncores missing: ".$obj->serverlogin()); + $time_to_login{$sshlogin} or ::die_bug("time_to_login missing: ".$obj->serverlogin()); + $maxlen{$sshlogin} or ::die_bug("maxlen missing: ".$obj->serverlogin()); + if($opt::use_cpus_instead_of_cores) { + $obj->set_ncpus($ncpus{$sshlogin}); + } else { + $obj->set_ncpus($ncores{$sshlogin}); + } + $obj->set_time_to_login($time_to_login{$sshlogin}); + $obj->set_maxlength($maxlen{$sshlogin}); + $Global::minimal_command_line_length = + ::min($Global::minimal_command_line_length, + int($maxlen{$sshlogin}/2)); + ::debug("init", "Timing from -S:$sshlogin ncpus:",$ncpus{$sshlogin}, + " ncores:", $ncores{$sshlogin}, + " time_to_login:", $time_to_login{$sshlogin}, + " maxlen:", $maxlen{$sshlogin}, + " min_max_len:", $Global::minimal_command_line_length,"\n"); + } +} + +sub onall { + sub tmp_joblog { + my $joblog = shift; + if(not defined $joblog) { + return undef; + } + my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".log"); + close $fh; + return $tmpfile; + } + my @command = @_; + if($Global::quoting) { + @command = shell_quote_empty(@command); + } + + # Copy all @fhlist into tempfiles + my @argfiles = (); + for my $fh (@fhlist) { + my ($outfh, $name) = ::tmpfile(SUFFIX => ".all", UNLINK => 1); + print $outfh (<$fh>); + close $outfh; + push @argfiles, $name; + } + if(@opt::basefile) { setup_basefile(); } + # for each sshlogin do: + # parallel -S $sshlogin $command :::: @argfiles + # + # Pass some of the options to the sub-parallels, not all of them as + # -P should only go to the first, and -S should not be copied at all. + my $options = + join(" ", + ((defined $opt::jobs) ? "-P $opt::jobs" : ""), + ((defined $opt::linebuffer) ? "--linebuffer" : ""), + ((defined $opt::ungroup) ? "-u" : ""), + ((defined $opt::group) ? "-g" : ""), + ((defined $opt::keeporder) ? "--keeporder" : ""), + ((defined $opt::D) ? "-D $opt::D" : ""), + ((defined $opt::plain) ? "--plain" : ""), + ((defined $opt::max_chars) ? "--max-chars ".$opt::max_chars : ""), + ); + my $suboptions = + join(" ", + ((defined $opt::ungroup) ? "-u" : ""), + ((defined $opt::linebuffer) ? "--linebuffer" : ""), + ((defined $opt::group) ? "-g" : ""), + ((defined $opt::files) ? "--files" : ""), + ((defined $opt::keeporder) ? "--keeporder" : ""), + ((defined $opt::colsep) ? "--colsep ".shell_quote($opt::colsep) : ""), + ((@opt::v) ? "-vv" : ""), + ((defined $opt::D) ? "-D $opt::D" : ""), + ((defined $opt::timeout) ? "--timeout ".$opt::timeout : ""), + ((defined $opt::plain) ? "--plain" : ""), + ((defined $opt::retries) ? "--retries ".$opt::retries : ""), + ((defined $opt::max_chars) ? "--max-chars ".$opt::max_chars : ""), + ((defined $opt::arg_sep) ? "--arg-sep ".$opt::arg_sep : ""), + ((defined $opt::arg_file_sep) ? "--arg-file-sep ".$opt::arg_file_sep : ""), + (@opt::env ? map { "--env ".::shell_quote_scalar($_) } @opt::env : ""), + ); + ::debug("init", "| $0 $options\n"); + open(my $parallel_fh, "|-", "$0 --no-notice -j0 $options") || + ::die_bug("This does not run GNU Parallel: $0 $options"); + my @joblogs; + for my $host (sort keys %Global::host) { + my $sshlogin = $Global::host{$host}; + my $joblog = tmp_joblog($opt::joblog); + if($joblog) { + push @joblogs, $joblog; + $joblog = "--joblog $joblog"; + } + my $quad = $opt::arg_file_sep || "::::"; + ::debug("init", "$0 $suboptions -j1 $joblog ", + ((defined $opt::tag) ? + "--tagstring ".shell_quote_scalar($sshlogin->string()) : ""), + " -S ", shell_quote_scalar($sshlogin->string())," ", + join(" ",shell_quote(@command))," $quad @argfiles\n"); + print $parallel_fh "$0 $suboptions -j1 $joblog ", + ((defined $opt::tag) ? + "--tagstring ".shell_quote_scalar($sshlogin->string()) : ""), + " -S ", shell_quote_scalar($sshlogin->string())," ", + join(" ",shell_quote(@command))," $quad @argfiles\n"; + } + close $parallel_fh; + $Global::exitstatus = $? >> 8; + debug("init", "--onall exitvalue ", $?); + if(@opt::basefile) { cleanup_basefile(); } + $Global::debug or unlink(@argfiles); + my %seen; + for my $joblog (@joblogs) { + # Append to $joblog + open(my $fh, "<", $joblog) || ::die_bug("Cannot open tmp joblog $joblog"); + # Skip first line (header); + <$fh>; + print $Global::joblog (<$fh>); + close $fh; + unlink($joblog); + } +} + +sub __SIGNAL_HANDLING__ {} + +sub save_original_signal_handler { + # Remember the original signal handler + # Returns: N/A + $SIG{TERM} ||= sub { exit 0; }; # $SIG{TERM} is not set on Mac OS X + $SIG{INT} = sub { if($opt::tmux) { qx { tmux kill-session -t p$$ }; } + unlink keys %Global::unlink; exit -1 }; + $SIG{TERM} = sub { if($opt::tmux) { qx { tmux kill-session -t p$$ }; } + unlink keys %Global::unlink; exit -1 }; + %Global::original_sig = %SIG; + $SIG{TERM} = sub {}; # Dummy until jobs really start +} + +sub list_running_jobs { + # Returns: N/A + for my $v (values %Global::running) { + print $Global::original_stderr "$Global::progname: ",$v->replaced(),"\n"; + } +} + +sub start_no_new_jobs { + # Returns: N/A + $SIG{TERM} = $Global::original_sig{TERM}; + print $Global::original_stderr + ("$Global::progname: SIGTERM received. No new jobs will be started.\n", + "$Global::progname: Waiting for these ", scalar(keys %Global::running), + " jobs to finish. Send SIGTERM again to stop now.\n"); + list_running_jobs(); + $Global::start_no_new_jobs ||= 1; +} + +sub reaper { + # A job finished. + # Print the output. + # Start another job + # Returns: N/A + my $stiff; + my $children_reaped = 0; + debug("run", "Reaper "); + while (($stiff = waitpid(-1, &WNOHANG)) > 0) { + $children_reaped++; + if($Global::sshmaster{$stiff}) { + # This is one of the ssh -M: ignore + next; + } + my $job = $Global::running{$stiff}; + # '-a <(seq 10)' will give us a pid not in %Global::running + $job or next; + $job->set_exitstatus($? >> 8); + $job->set_exitsignal($? & 127); + debug("run", "died (", $job->exitstatus(), "): ", $job->seq()); + $job->set_endtime(::now()); + if($stiff == $Global::tty_taken) { + # The process that died had the tty => release it + $Global::tty_taken = 0; + } + + if(not $job->should_be_retried()) { + # The job is done + # Free the jobslot + push @Global::slots, $job->slot(); + if($opt::timeout) { + # Update average runtime for timeout + $Global::timeoutq->update_delta_time($job->runtime()); + } + # Force printing now if the job failed and we are going to exit + my $print_now = ($opt::halt_on_error and $opt::halt_on_error == 2 + and $job->exitstatus()); + if($opt::keeporder and not $print_now) { + print_earlier_jobs($job); + } else { + $job->print(); + } + if($job->exitstatus()) { + process_failed_job($job); + } + + } + my $sshlogin = $job->sshlogin(); + $sshlogin->dec_jobs_running(); + $sshlogin->inc_jobs_completed(); + $Global::total_running--; + delete $Global::running{$stiff}; + start_more_jobs(); + } + debug("run", "done "); + return $children_reaped; +} + +sub process_failed_job { + # The jobs had a exit status <> 0, so error + # Returns: N/A + my $job = shift; + $Global::exitstatus++; + $Global::total_failed++; + if($opt::halt_on_error) { + if($opt::halt_on_error == 1 + or + ($opt::halt_on_error < 1 and $Global::total_failed > 3 + and + $Global::total_failed / $Global::total_started > $opt::halt_on_error)) { + # If halt on error == 1 or --halt 10% + # we should gracefully exit + print $Global::original_stderr + ("$Global::progname: Starting no more jobs. ", + "Waiting for ", scalar(keys %Global::running), + " jobs to finish. This job failed:\n", + $job->replaced(),"\n"); + $Global::start_no_new_jobs ||= 1; + $Global::halt_on_error_exitstatus = $job->exitstatus(); + } elsif($opt::halt_on_error == 2) { + # If halt on error == 2 we should exit immediately + print $Global::original_stderr + ("$Global::progname: This job failed:\n", + $job->replaced(),"\n"); + exit ($job->exitstatus()); + } + } +} + +{ + my (%print_later,$job_end_sequence); + + sub print_earlier_jobs { + # Print jobs completed earlier + # Returns: N/A + my $job = shift; + $print_later{$job->seq()} = $job; + $job_end_sequence ||= 1; + debug("run", "Looking for: $job_end_sequence ", + "Current: ", $job->seq(), "\n"); + for(my $j = $print_later{$job_end_sequence}; + $j or vec($Global::job_already_run,$job_end_sequence,1); + $job_end_sequence++, + $j = $print_later{$job_end_sequence}) { + debug("run", "Found job end $job_end_sequence"); + if($j) { + $j->print(); + delete $print_later{$job_end_sequence}; + } + } + } +} + +sub __USAGE__ {} + +sub wait_and_exit { + # If we do not wait, we sometimes get segfault + # Returns: N/A + my $error = shift; + if($error) { + # Kill all without printing + for my $job (values %Global::running) { + $job->kill("TERM"); + $job->kill("TERM"); + } + } + for (keys %Global::unkilled_children) { + kill 9, $_; + waitpid($_,0); + delete $Global::unkilled_children{$_}; + } + wait(); + exit($error); +} + +sub die_usage { + # Returns: N/A + usage(); + wait_and_exit(255); +} + +sub usage { + # Returns: N/A + print join + ("\n", + "Usage:", + "", + "$Global::progname [options] [command [arguments]] < list_of_arguments", + "$Global::progname [options] [command [arguments]] (::: arguments|:::: argfile(s))...", + "cat ... | $Global::progname --pipe [options] [command [arguments]]", + "", + "-j n Run n jobs in parallel", + "-k Keep same order", + "-X Multiple arguments with context replace", + "--colsep regexp Split input on regexp for positional replacements", + "{} {.} {/} {/.} {#} {%} {= perl code =} Replacement strings", + "{3} {3.} {3/} {3/.} {=3 perl code =} Positional replacement strings", + "With --plus: {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} =", + " {+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...}", + "", + "-S sshlogin Example: foo\@server.example.com", + "--slf .. Use ~/.parallel/sshloginfile as the list of sshlogins", + "--trc {}.bar Shorthand for --transfer --return {}.bar --cleanup", + "--onall Run the given command with argument on all sshlogins", + "--nonall Run the given command with no arguments on all sshlogins", + "", + "--pipe Split stdin (standard input) to multiple jobs.", + "--recend str Record end separator for --pipe.", + "--recstart str Record start separator for --pipe.", + "", + "See 'man $Global::progname' for details", + "", + "When using programs that use GNU Parallel to process data for publication please cite:", + "", + "O. Tange (2011): GNU Parallel - The Command-Line Power Tool,", + ";login: The USENIX Magazine, February 2011:42-47.", + "", + "Or you can get GNU Parallel without this requirement by paying 10000 EUR.", + ""); +} + + +sub citation_notice { + # if --no-notice or --plain: do nothing + # if stderr redirected: do nothing + # if ~/.parallel/will-cite: do nothing + # else: print citation notice to stderr + if($opt::no_notice + or + $opt::plain + or + not -t $Global::original_stderr + or + -e $ENV{'HOME'}."/.parallel/will-cite") { + # skip + } else { + print $Global::original_stderr + ("When using programs that use GNU Parallel to process data for publication please cite:\n", + "\n", + " O. Tange (2011): GNU Parallel - The Command-Line Power Tool,\n", + " ;login: The USENIX Magazine, February 2011:42-47.\n", + "\n", + "This helps funding further development; and it won't cost you a cent.\n", + "Or you can get GNU Parallel without this requirement by paying 10000 EUR.\n", + "\n", + "To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'.\n\n", + ); + flush $Global::original_stderr; + } +} + + +sub warning { + my @w = @_; + my $fh = $Global::original_stderr || *STDERR; + my $prog = $Global::progname || "parallel"; + print $fh $prog, ": Warning: ", @w; +} + + +sub error { + my @w = @_; + my $fh = $Global::original_stderr || *STDERR; + my $prog = $Global::progname || "parallel"; + print $fh $prog, ": Error: ", @w; +} + + +sub die_bug { + my $bugid = shift; + print STDERR + ("$Global::progname: This should not happen. You have found a bug.\n", + "Please contact and include:\n", + "* The version number: $Global::version\n", + "* The bugid: $bugid\n", + "* The command line being run\n", + "* The files being read (put the files on a webserver if they are big)\n", + "\n", + "If you get the error on smaller/fewer files, please include those instead.\n"); + ::wait_and_exit(255); +} + +sub version { + # Returns: N/A + if($opt::tollef and not $opt::gnu) { + print "WARNING: YOU ARE USING --tollef. IF THINGS ARE ACTING WEIRD USE --gnu.\n"; + } + print join("\n", + "GNU $Global::progname $Global::version", + "Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014 Ole Tange and Free Software Foundation, Inc.", + "License GPLv3+: GNU GPL version 3 or later ", + "This is free software: you are free to change and redistribute it.", + "GNU $Global::progname comes with no warranty.", + "", + "Web site: http://www.gnu.org/software/${Global::progname}\n", + "When using programs that use GNU Parallel to process data for publication please cite:\n", + "O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ", + ";login: The USENIX Magazine, February 2011:42-47.\n", + "Or you can get GNU Parallel without this requirement by paying 10000 EUR.\n", + ); +} + +sub bibtex { + # Returns: N/A + if($opt::tollef and not $opt::gnu) { + print "WARNING: YOU ARE USING --tollef. IF THINGS ARE ACTING WEIRD USE --gnu.\n"; + } + print join("\n", + "When using programs that use GNU Parallel to process data for publication please cite:", + "", + "\@article{Tange2011a,", + " title = {GNU Parallel - The Command-Line Power Tool},", + " author = {O. Tange},", + " address = {Frederiksberg, Denmark},", + " journal = {;login: The USENIX Magazine},", + " month = {Feb},", + " number = {1},", + " volume = {36},", + " url = {http://www.gnu.org/s/parallel},", + " year = {2011},", + " pages = {42-47}", + "}", + "", + "(Feel free to use \\nocite{Tange2011a})", + "", + "This helps funding further development.", + "", + "Or you can get GNU Parallel without this requirement by paying 10000 EUR.", + "" + ); + while(not -e $ENV{'HOME'}."/.parallel/will-cite") { + print "\nType: 'will cite' and press enter.\n> "; + my $input = ; + if($input =~ /will cite/i) { + mkdir $ENV{'HOME'}."/.parallel"; + open (my $fh, ">", $ENV{'HOME'}."/.parallel/will-cite") + || ::die_bug("Cannot write: ".$ENV{'HOME'}."/.parallel/will-cite"); + close $fh; + print "\nThank you for your support. It is much appreciated. The citation\n", + "notice is now silenced.\n"; + } + } +} + +sub show_limits { + # Returns: N/A + print("Maximal size of command: ",Limits::Command::real_max_length(),"\n", + "Maximal used size of command: ",Limits::Command::max_length(),"\n", + "\n", + "Execution of will continue now, and it will try to read its input\n", + "and run commands; if this is not what you wanted to happen, please\n", + "press CTRL-D or CTRL-C\n"); +} + +sub __GENERIC_COMMON_FUNCTION__ {} + +sub uniq { + # Remove duplicates and return unique values + return keys %{{ map { $_ => 1 } @_ }}; +} + +sub min { + # Returns: + # Minimum value of array + my $min; + for (@_) { + # Skip undefs + defined $_ or next; + defined $min or do { $min = $_; next; }; # Set $_ to the first non-undef + $min = ($min < $_) ? $min : $_; + } + return $min; +} + +sub max { + # Returns: + # Maximum value of array + my $max; + for (@_) { + # Skip undefs + defined $_ or next; + defined $max or do { $max = $_; next; }; # Set $_ to the first non-undef + $max = ($max > $_) ? $max : $_; + } + return $max; +} + +sub sum { + # Returns: + # Sum of values of array + my @args = @_; + my $sum = 0; + for (@args) { + # Skip undefs + $_ and do { $sum += $_; } + } + return $sum; +} + +sub undef_as_zero { + my $a = shift; + return $a ? $a : 0; +} + +sub undef_as_empty { + my $a = shift; + return $a ? $a : ""; +} + +{ + my $hostname; + sub hostname { + if(not $hostname) { + $hostname = `hostname`; + chomp($hostname); + $hostname ||= "nohostname"; + } + return $hostname; + } +} + +sub which { + # Input: + # @programs = programs to find the path to + # Returns: + # @full_path = full paths to @programs. Nothing if not found + my @which; + for my $prg (@_) { + push @which, map { $_."/".$prg } grep { -x $_."/".$prg } split(":",$ENV{'PATH'}); + } + return @which; +} + +{ + my ($regexp,%fakename); + + sub parent_shell { + # Input: + # $pid = pid to see if (grand)*parent is a shell + # Returns: + # $shellpath = path to shell - undef if no shell found + my $pid = shift; + if(not $regexp) { + # All shells known to mankind + # + # ash bash csh dash fdsh fish fizsh ksh ksh93 mksh pdksh + # posh rbash rush rzsh sash sh static-sh tcsh yash zsh + my @shells = qw(ash bash csh dash fdsh fish fizsh ksh + ksh93 mksh pdksh posh rbash rush rzsh + sash sh static-sh tcsh yash zsh -sh -csh); + # Can be formatted as: + # [sh] -sh sh busybox sh + # /bin/sh /sbin/sh /opt/csw/sh + # NOT: foo.sh sshd crash flush pdflush scosh fsflush ssh + my $shell = "(?:".join("|",@shells).")"; + $regexp = '^((\[)('. $shell. ')(\])|(|\S+/|busybox )('. $shell. '))($| )'; + %fakename = ( + # csh and tcsh disguise themselves as -sh/-csh + "-sh" => ["csh", "tcsh"], + "-csh" => ["tcsh", "csh"], + ); + } + my ($children_of_ref, $parent_of_ref, $name_of_ref) = pid_table(); + my $shellpath; + my $testpid = $pid; + while($testpid) { + ::debug("init", "shell? ". $name_of_ref->{$testpid}."\n"); + if($name_of_ref->{$testpid} =~ /$regexp/o) { + ::debug("init", "which ".($3||$6)." => "); + $shellpath = (which($3 || $6,@{$fakename{$3 || $6}}))[0]; + ::debug("init", "shell path $shellpath\n"); + $shellpath and last; + } + $testpid = $parent_of_ref->{$testpid}; + } + return $shellpath; + } +} + +{ + my %pid_parentpid_cmd; + + sub pid_table { + # Returns: + # %children_of = { pid -> children of pid } + # %parent_of = { pid -> pid of parent } + # %name_of = { pid -> commandname } + + if(not %pid_parentpid_cmd) { + # Filter for SysV-style `ps` + my $sysv = q( ps -ef | perl -ane '1..1 and /^(.*)CO?MM?A?N?D/ and $s=length $1;). + q(s/^.{$s}//; print "@F[1,2] $_"' ); + # BSD-style `ps` + my $bsd = q(ps -o pid,ppid,command -ax); + %pid_parentpid_cmd = + ( + 'aix' => $sysv, + 'cygwin' => $sysv, + 'msys' => $sysv, + 'dec_osf' => $sysv, + 'darwin' => $bsd, + 'dragonfly' => $bsd, + 'freebsd' => $bsd, + 'gnu' => $sysv, + 'hpux' => $sysv, + 'linux' => $sysv, + 'mirbsd' => $bsd, + 'netbsd' => $bsd, + 'nto' => $sysv, + 'openbsd' => $bsd, + 'solaris' => $sysv, + 'svr5' => $sysv, + ); + } + $pid_parentpid_cmd{$^O} or ::die_bug("pid_parentpid_cmd for $^O missing"); + + my (@pidtable,%parent_of,%children_of,%name_of); + # Table with pid -> children of pid + @pidtable = `$pid_parentpid_cmd{$^O}`; + my $p=$$; + for (@pidtable) { + # must match: 24436 21224 busybox ash + /(\S+)\s+(\S+)\s+(\S+.*)/ or ::die_bug("pidtable format: $_"); + $parent_of{$1} = $2; + push @{$children_of{$2}}, $1; + $name_of{$1} = $3; + } + return(\%children_of, \%parent_of, \%name_of); + } +} + +sub reap_usleep { + # Reap dead children. + # If no dead children: Sleep specified amount with exponential backoff + # Input: + # $ms = milliseconds to sleep + # Returns: + # $ms/2+0.001 if children reaped + # $ms*1.1 if no children reaped + my $ms = shift; + if(reaper()) { + # Sleep exponentially shorter (1/2^n) if a job finished + return $ms/2+0.001; + } else { + if($opt::timeout) { + $Global::timeoutq->process_timeouts(); + } + usleep($ms); + Job::exit_if_disk_full(); + if($opt::linebuffer) { + for my $job (values %Global::running) { + $job->print(); + } + } + # Sleep exponentially longer (1.1^n) if a job did not finish + # though at most 1000 ms. + return (($ms < 1000) ? ($ms * 1.1) : ($ms)); + } +} + +sub usleep { + # Sleep this many milliseconds. + # Input: + # $ms = milliseconds to sleep + my $ms = shift; + ::debug(int($ms),"ms "); + select(undef, undef, undef, $ms/1000); +} + +sub now { + # Returns time since epoch as in seconds with 3 decimals + # Uses: + # @Global::use + # Returns: + # $time = time now with millisecond accuracy + if(not $Global::use{"Time::HiRes"}) { + if(eval "use Time::HiRes qw ( time );") { + eval "sub TimeHiRestime { return Time::HiRes::time };"; + } else { + eval "sub TimeHiRestime { return time() };"; + } + $Global::use{"Time::HiRes"} = 1; + } + + return (int(TimeHiRestime()*1000))/1000; +} + +sub multiply_binary_prefix { + # Evalualte numbers with binary prefix + # Ki=2^10, Mi=2^20, Gi=2^30, Ti=2^40, Pi=2^50, Ei=2^70, Zi=2^80, Yi=2^80 + # ki=2^10, mi=2^20, gi=2^30, ti=2^40, pi=2^50, ei=2^70, zi=2^80, yi=2^80 + # K =2^10, M =2^20, G =2^30, T =2^40, P =2^50, E =2^70, Z =2^80, Y =2^80 + # k =10^3, m =10^6, g =10^9, t=10^12, p=10^15, e=10^18, z=10^21, y=10^24 + # 13G = 13*1024*1024*1024 = 13958643712 + # Input: + # $s = string with prefixes + # Returns: + # $value = int with prefixes multiplied + my $s = shift; + $s =~ s/ki/*1024/gi; + $s =~ s/mi/*1024*1024/gi; + $s =~ s/gi/*1024*1024*1024/gi; + $s =~ s/ti/*1024*1024*1024*1024/gi; + $s =~ s/pi/*1024*1024*1024*1024*1024/gi; + $s =~ s/ei/*1024*1024*1024*1024*1024*1024/gi; + $s =~ s/zi/*1024*1024*1024*1024*1024*1024*1024/gi; + $s =~ s/yi/*1024*1024*1024*1024*1024*1024*1024*1024/gi; + $s =~ s/xi/*1024*1024*1024*1024*1024*1024*1024*1024*1024/gi; + + $s =~ s/K/*1024/g; + $s =~ s/M/*1024*1024/g; + $s =~ s/G/*1024*1024*1024/g; + $s =~ s/T/*1024*1024*1024*1024/g; + $s =~ s/P/*1024*1024*1024*1024*1024/g; + $s =~ s/E/*1024*1024*1024*1024*1024*1024/g; + $s =~ s/Z/*1024*1024*1024*1024*1024*1024*1024/g; + $s =~ s/Y/*1024*1024*1024*1024*1024*1024*1024*1024/g; + $s =~ s/X/*1024*1024*1024*1024*1024*1024*1024*1024*1024/g; + + $s =~ s/k/*1000/g; + $s =~ s/m/*1000*1000/g; + $s =~ s/g/*1000*1000*1000/g; + $s =~ s/t/*1000*1000*1000*1000/g; + $s =~ s/p/*1000*1000*1000*1000*1000/g; + $s =~ s/e/*1000*1000*1000*1000*1000*1000/g; + $s =~ s/z/*1000*1000*1000*1000*1000*1000*1000/g; + $s =~ s/y/*1000*1000*1000*1000*1000*1000*1000*1000/g; + $s =~ s/x/*1000*1000*1000*1000*1000*1000*1000*1000*1000/g; + + $s = eval $s; + ::debug($s); + return $s; +} + +sub tmpfile { + # Create tempfile as $TMPDIR/parXXXXX + # Returns: + # $filename = file name created + return ::tempfile(DIR=>$ENV{'TMPDIR'}, TEMPLATE => 'parXXXXX', @_); +} + +sub __DEBUGGING__ {} + +sub debug { + # Uses: + # $Global::debug + # %Global::fd + # Returns: N/A + $Global::debug or return; + @_ = grep { defined $_ ? $_ : "" } @_; + if($Global::debug eq "all" or $Global::debug eq $_[0]) { + if($Global::fd{1}) { + # Original stdout was saved + my $stdout = $Global::fd{1}; + print $stdout @_[1..$#_]; + } else { + print @_[1..$#_]; + } + } +} + +sub my_memory_usage { + # Returns: + # memory usage if found + # 0 otherwise + use strict; + use FileHandle; + + my $pid = $$; + if(-e "/proc/$pid/stat") { + my $fh = FileHandle->new("; + chomp $data; + $fh->close; + + my @procinfo = split(/\s+/,$data); + + return undef_as_zero($procinfo[22]); + } else { + return 0; + } +} + +sub my_size { + # Returns: + # $size = size of object if Devel::Size is installed + # -1 otherwise + my @size_this = (@_); + eval "use Devel::Size qw(size total_size)"; + if ($@) { + return -1; + } else { + return total_size(@_); + } +} + +sub my_dump { + # Returns: + # ascii expression of object if Data::Dump(er) is installed + # error code otherwise + my @dump_this = (@_); + eval "use Data::Dump qw(dump);"; + if ($@) { + # Data::Dump not installed + eval "use Data::Dumper;"; + if ($@) { + my $err = "Neither Data::Dump nor Data::Dumper is installed\n". + "Not dumping output\n"; + print $Global::original_stderr $err; + return $err; + } else { + return Dumper(@dump_this); + } + } else { + # Create a dummy Data::Dump:dump as Hans Schou sometimes has + # it undefined + eval "sub Data::Dump:dump {}"; + eval "use Data::Dump qw(dump);"; + return (Data::Dump::dump(@dump_this)); + } +} + +sub my_croak { + eval "use Carp; 1"; + $Carp::Verbose = 1; + croak(@_); +} + +sub my_carp { + eval "use Carp; 1"; + $Carp::Verbose = 1; + carp(@_); +} + +sub __OBJECT_ORIENTED_PARTS__ {} + +package SSHLogin; + +sub new { + my $class = shift; + my $sshlogin_string = shift; + my $ncpus; + my %hostgroups; + # SSHLogins can have these formats: + # @grp+grp/ncpu//usr/bin/ssh user@server + # ncpu//usr/bin/ssh user@server + # /usr/bin/ssh user@server + # user@server + # ncpu/user@server + # @grp+grp/user@server + if($sshlogin_string =~ s:^\@([^/]+)/?::) { + # Look for SSHLogin hostgroups + %hostgroups = map { $_ => 1 } split(/\+/, $1); + } + if ($sshlogin_string =~ s:^(\d+)/::) { + # Override default autodetected ncpus unless missing + $ncpus = $1; + } + my $string = $sshlogin_string; + # An SSHLogin is always in the hostgroup of its $string-name + $hostgroups{$string} = 1; + @Global::hostgroups{keys %hostgroups} = values %hostgroups; + my @unget = (); + my $no_slash_string = $string; + $no_slash_string =~ s/[^-a-z0-9:]/_/gi; + return bless { + 'string' => $string, + 'jobs_running' => 0, + 'jobs_completed' => 0, + 'maxlength' => undef, + 'max_jobs_running' => undef, + 'orig_max_jobs_running' => undef, + 'ncpus' => $ncpus, + 'hostgroups' => \%hostgroups, + 'sshcommand' => undef, + 'serverlogin' => undef, + 'control_path_dir' => undef, + 'control_path' => undef, + 'time_to_login' => undef, + 'last_login_at' => undef, + 'loadavg_file' => $ENV{'HOME'} . "/.parallel/tmp/loadavg-" . + $no_slash_string, + 'loadavg' => undef, + 'last_loadavg_update' => 0, + 'swap_activity_file' => $ENV{'HOME'} . "/.parallel/tmp/swap_activity-" . + $no_slash_string, + 'swap_activity' => undef, + }, ref($class) || $class; +} + +sub DESTROY { + my $self = shift; + # Remove temporary files if they are created. + unlink $self->{'loadavg_file'}; + unlink $self->{'swap_activity_file'}; +} + +sub string { + my $self = shift; + return $self->{'string'}; +} + +sub jobs_running { + my $self = shift; + + return ($self->{'jobs_running'} || "0"); +} + +sub inc_jobs_running { + my $self = shift; + $self->{'jobs_running'}++; +} + +sub dec_jobs_running { + my $self = shift; + $self->{'jobs_running'}--; +} + +sub set_maxlength { + my $self = shift; + $self->{'maxlength'} = shift; +} + +sub maxlength { + my $self = shift; + return $self->{'maxlength'}; +} + +sub jobs_completed { + my $self = shift; + return $self->{'jobs_completed'}; +} + +sub in_hostgroups { + # Input: + # @hostgroups = the hostgroups to look for + # Returns: + # true if intersection of @hostgroups and the hostgroups of this + # SSHLogin is non-empty + my $self = shift; + return grep { defined $self->{'hostgroups'}{$_} } @_; +} + +sub hostgroups { + my $self = shift; + return keys %{$self->{'hostgroups'}}; +} + +sub inc_jobs_completed { + my $self = shift; + $self->{'jobs_completed'}++; +} + +sub set_max_jobs_running { + my $self = shift; + if(defined $self->{'max_jobs_running'}) { + $Global::max_jobs_running -= $self->{'max_jobs_running'}; + } + $self->{'max_jobs_running'} = shift; + if(defined $self->{'max_jobs_running'}) { + # max_jobs_running could be resat if -j is a changed file + $Global::max_jobs_running += $self->{'max_jobs_running'}; + } + # Initialize orig to the first non-zero value that comes around + $self->{'orig_max_jobs_running'} ||= $self->{'max_jobs_running'}; +} + +sub swapping { + my $self = shift; + my $swapping = $self->swap_activity(); + return (not defined $swapping or $swapping) +} + +sub swap_activity { + # If the currently known swap activity is too old: + # Recompute a new one in the background + # Returns: + # last swap activity computed + my $self = shift; + # Should we update the swap_activity file? + my $update_swap_activity_file = 0; + if(-r $self->{'swap_activity_file'}) { + open(my $swap_fh, "<", $self->{'swap_activity_file'}) || ::die_bug("swap_activity_file-r"); + my $swap_out = <$swap_fh>; + close $swap_fh; + if($swap_out =~ /^(\d+)$/) { + $self->{'swap_activity'} = $1; + ::debug("swap", "New swap_activity: ", $self->{'swap_activity'}); + } + ::debug("swap", "Last update: ", $self->{'last_swap_activity_update'}); + if(time - $self->{'last_swap_activity_update'} > 10) { + # last swap activity update was started 10 seconds ago + ::debug("swap", "Older than 10 sec: ", $self->{'swap_activity_file'}); + $update_swap_activity_file = 1; + } + } else { + ::debug("swap", "No swap_activity file: ", $self->{'swap_activity_file'}); + $self->{'swap_activity'} = undef; + $update_swap_activity_file = 1; + } + if($update_swap_activity_file) { + ::debug("swap", "Updating swap_activity file ", $self->{'swap_activity_file'}); + $self->{'last_swap_activity_update'} = time; + -e $ENV{'HOME'}."/.parallel" or mkdir $ENV{'HOME'}."/.parallel"; + -e $ENV{'HOME'}."/.parallel/tmp" or mkdir $ENV{'HOME'}."/.parallel/tmp"; + my $swap_activity; + $swap_activity = swapactivityscript(); + if($self->{'string'} ne ":") { + $swap_activity = $self->sshcommand() . " " . $self->serverlogin() . " " . + ::shell_quote_scalar($swap_activity); + } + # Run swap_activity measuring. + # As the command can take long to run if run remote + # save it to a tmp file before moving it to the correct file + my $file = $self->{'swap_activity_file'}; + my ($dummy_fh, $tmpfile) = ::tmpfile(SUFFIX => ".swp"); + ::debug("swap", "\n", $swap_activity, "\n"); + qx{ ($swap_activity > $tmpfile && mv $tmpfile $file || rm $tmpfile) & }; + } + return $self->{'swap_activity'}; +} + +{ + my $script; + + sub swapactivityscript { + # Returns: + # shellscript for detecting swap activity + # + # arguments for vmstat are OS dependant + # swap_in and swap_out are in different columns depending on OS + # + if(not $script) { + my %vmstat = ( + # linux: $7*$8 + # $ vmstat 1 2 + # procs -----------memory---------- ---swap-- -----io---- -system-- ----cpu---- + # r b swpd free buff cache si so bi bo in cs us sy id wa + # 5 0 51208 1701096 198012 18857888 0 0 37 153 28 19 56 11 33 1 + # 3 0 51208 1701288 198012 18857972 0 0 0 0 3638 10412 15 3 82 0 + 'linux' => ['vmstat 1 2 | tail -n1', '$7*$8'], + + # solaris: $6*$7 + # $ vmstat -S 1 2 + # kthr memory page disk faults cpu + # r b w swap free si so pi po fr de sr s3 s4 -- -- in sy cs us sy id + # 0 0 0 4628952 3208408 0 0 3 1 1 0 0 -0 2 0 0 263 613 246 1 2 97 + # 0 0 0 4552504 3166360 0 0 0 0 0 0 0 0 0 0 0 246 213 240 1 1 98 + 'solaris' => ['vmstat -S 1 2 | tail -1', '$6*$7'], + + # darwin (macosx): $21*$22 + # $ vm_stat -c 2 1 + # Mach Virtual Memory Statistics: (page size of 4096 bytes) + # free active specul inactive throttle wired prgable faults copy 0fill reactive purged file-backed anonymous cmprssed cmprssor dcomprs comprs pageins pageout swapins swapouts + # 346306 829050 74871 606027 0 240231 90367 544858K 62343596 270837K 14178 415070 570102 939846 356 370 116 922 4019813 4 0 0 + # 345740 830383 74875 606031 0 239234 90369 2696 359 553 0 0 570110 941179 356 370 0 0 0 0 0 0 + 'darwin' => ['vm_stat -c 2 1 | tail -n1', '$21*$22'], + + # ultrix: $12*$13 + # $ vmstat -S 1 2 + # procs faults cpu memory page disk + # r b w in sy cs us sy id avm fre si so pi po fr de sr s0 + # 1 0 0 4 23 2 3 0 97 7743 217k 0 0 0 0 0 0 0 0 + # 1 0 0 6 40 8 0 1 99 7743 217k 0 0 3 0 0 0 0 0 + 'ultrix' => ['vmstat -S 1 2 | tail -1', '$12*$13'], + + # aix: $6*$7 + # $ vmstat 1 2 + # System configuration: lcpu=1 mem=2048MB + # + # kthr memory page faults cpu + # ----- ----------- ------------------------ ------------ ----------- + # r b avm fre re pi po fr sr cy in sy cs us sy id wa + # 0 0 333933 241803 0 0 0 0 0 0 10 143 90 0 0 99 0 + # 0 0 334125 241569 0 0 0 0 0 0 37 5368 184 0 9 86 5 + 'aix' => ['vmstat 1 2 | tail -n1', '$6*$7'], + + # freebsd: $8*$9 + # $ vmstat -H 1 2 + # procs memory page disks faults cpu + # r b w avm fre flt re pi po fr sr ad0 ad1 in sy cs us sy id + # 1 0 0 596716 19560 32 0 0 0 33 8 0 0 11 220 277 0 0 99 + # 0 0 0 596716 19560 2 0 0 0 0 0 0 0 11 144 263 0 1 99 + 'freebsd' => ['vmstat -H 1 2 | tail -n1', '$8*$9'], + + # mirbsd: $8*$9 + # $ vmstat 1 2 + # procs memory page disks traps cpu + # r b w avm fre flt re pi po fr sr wd0 cd0 int sys cs us sy id + # 0 0 0 25776 164968 34 0 0 0 0 0 0 0 230 259 38 4 0 96 + # 0 0 0 25776 164968 24 0 0 0 0 0 0 0 237 275 37 0 0 100 + 'mirbsd' => ['vmstat 1 2 | tail -n1', '$8*$9'], + + # netbsd: $7*$8 + # $ vmstat 1 2 + # procs memory page disks faults cpu + # r b avm fre flt re pi po fr sr w0 w1 in sy cs us sy id + # 0 0 138452 6012 54 0 0 0 1 2 3 0 4 100 23 0 0 100 + # 0 0 138456 6008 1 0 0 0 0 0 0 0 7 26 19 0 0 100 + 'netbsd' => ['vmstat 1 2 | tail -n1', '$7*$8'], + + # openbsd: $8*$9 + # $ vmstat 1 2 + # procs memory page disks traps cpu + # r b w avm fre flt re pi po fr sr wd0 wd1 int sys cs us sy id + # 0 0 0 76596 109944 73 0 0 0 0 0 0 1 5 259 22 0 1 99 + # 0 0 0 76604 109936 24 0 0 0 0 0 0 0 7 114 20 0 1 99 + 'openbsd' => ['vmstat 1 2 | tail -n1', '$8*$9'], + + # hpux: $8*$9 + # $ vmstat 1 2 + # procs memory page faults cpu + # r b w avm free re at pi po fr de sr in sy cs us sy id + # 1 0 0 247211 216476 4 1 0 0 0 0 0 102 73005 54 6 11 83 + # 1 0 0 247211 216421 43 9 0 0 0 0 0 144 1675 96 25269512791222387000 25269512791222387000 105 + 'hpux' => ['vmstat 1 2 | tail -n1', '$8*$9'], + + # dec_osf (tru64): $11*$12 + # $ vmstat 1 2 + # Virtual Memory Statistics: (pagesize = 8192) + # procs memory pages intr cpu + # r w u act free wire fault cow zero react pin pout in sy cs us sy id + # 3 181 36 51K 1895 8696 348M 59M 122M 259 79M 0 5 218 302 4 1 94 + # 3 181 36 51K 1893 8696 3 15 21 0 28 0 4 81 321 1 1 98 + 'dec_osf' => ['vmstat 1 2 | tail -n1', '$11*$12'], + + # gnu (hurd): $7*$8 + # $ vmstat -k 1 2 + # (pagesize: 4, size: 512288, swap size: 894972) + # free actv inact wired zeroed react pgins pgouts pfaults cowpfs hrat caobj cache swfree + # 371940 30844 89228 20276 298348 0 48192 19016 756105 99808 98% 876 20628 894972 + # 371940 30844 89228 20276 +0 +0 +0 +0 +42 +2 98% 876 20628 894972 + 'gnu' => ['vmstat -k 1 2 | tail -n1', '$7*$8'], + + # -nto (qnx has no swap) + #-irix + #-svr5 (scosysv) + ); + my $perlscript = ""; + for my $os (keys %vmstat) { + #q[ { vmstat 1 2 2> /dev/null || vmstat -c 1 2; } | ]. + # q[ awk 'NR!=4{next} NF==17||NF==16{print $7*$8} NF==22{print $21*$22} {exit}' ]; + $vmstat{$os}[1] =~ s/\$/\\\\\\\$/g; # $ => \\\$ + $perlscript .= 'if($^O eq "'.$os.'") { print `'.$vmstat{$os}[0].' | awk "{print ' . + $vmstat{$os}[1] . '}"` }'; + } + $perlscript = "perl -e " . ::shell_quote_scalar($perlscript); + $script = $Global::envvar. " " .$perlscript; + } + return $script; + } +} + +sub too_fast_remote_login { + my $self = shift; + if($self->{'last_login_at'} and $self->{'time_to_login'}) { + # sshd normally allows 10 simultaneous logins + # A login takes time_to_login + # So time_to_login/5 should be safe + # If now <= last_login + time_to_login/5: Then it is too soon. + my $too_fast = (::now() <= $self->{'last_login_at'} + + $self->{'time_to_login'}/5); + ::debug("run", "Too fast? $too_fast "); + return $too_fast; + } else { + # No logins so far (or time_to_login not computed): it is not too fast + return 0; + } +} + +sub last_login_at { + my $self = shift; + return $self->{'last_login_at'}; +} + +sub set_last_login_at { + my $self = shift; + $self->{'last_login_at'} = shift; +} + +sub loadavg_too_high { + my $self = shift; + my $loadavg = $self->loadavg(); + return (not defined $loadavg or + $loadavg > $self->max_loadavg()); +} + +sub loadavg { + # If the currently know loadavg is too old: + # Recompute a new one in the background + # The load average is computed as the number of processes waiting for disk + # or CPU right now. So it is the server load this instant and not averaged over + # several minutes. This is needed so GNU Parallel will at most start one job + # that will push the load over the limit. + # + # Returns: + # $last_loadavg = last load average computed (undef if none) + my $self = shift; + # Should we update the loadavg file? + my $update_loadavg_file = 0; + if(open(my $load_fh, "<", $self->{'loadavg_file'})) { + local $/ = undef; + my $load_out = <$load_fh>; + close $load_fh; + my $load =()= ($load_out=~/(^[DR]....[^\[])/gm); + if($load > 0) { + # load is overestimated by 1 + $self->{'loadavg'} = $load - 1; + ::debug("load", "New loadavg: ", $self->{'loadavg'}); + } else { + ::die_bug("loadavg_invalid_content: $load_out"); + } + ::debug("load", "Last update: ", $self->{'last_loadavg_update'}); + if(time - $self->{'last_loadavg_update'} > 10) { + # last loadavg was started 10 seconds ago + ::debug("load", time - $self->{'last_loadavg_update'}, " secs old: ", + $self->{'loadavg_file'}); + $update_loadavg_file = 1; + } + } else { + ::debug("load", "No loadavg file: ", $self->{'loadavg_file'}); + $self->{'loadavg'} = undef; + $update_loadavg_file = 1; + } + if($update_loadavg_file) { + ::debug("load", "Updating loadavg file", $self->{'loadavg_file'}, "\n"); + $self->{'last_loadavg_update'} = time; + -e $ENV{'HOME'}."/.parallel" or mkdir $ENV{'HOME'}."/.parallel"; + -e $ENV{'HOME'}."/.parallel/tmp" or mkdir $ENV{'HOME'}."/.parallel/tmp"; + my $cmd = ""; + if($self->{'string'} ne ":") { + $cmd = $self->sshcommand() . " " . $self->serverlogin() . " "; + } + # TODO Is is called 'ps ax -o state,command' on other platforms? + $cmd .= "ps ax -o state,command"; + # As the command can take long to run if run remote + # save it to a tmp file before moving it to the correct file + my $file = $self->{'loadavg_file'}; + my ($dummy_fh, $tmpfile) = ::tmpfile(SUFFIX => ".loa"); + qx{ ($cmd > $tmpfile && mv $tmpfile $file || rm $tmpfile) & }; + } + return $self->{'loadavg'}; +} + +sub max_loadavg { + my $self = shift; + # If --load is a file it might be changed + if($Global::max_load_file) { + my $mtime = (stat($Global::max_load_file))[9]; + if($mtime > $Global::max_load_file_last_mod) { + $Global::max_load_file_last_mod = $mtime; + for my $sshlogin (values %Global::host) { + $sshlogin->set_max_loadavg(undef); + } + } + } + if(not defined $self->{'max_loadavg'}) { + $self->{'max_loadavg'} = + $self->compute_max_loadavg($opt::load); + } + ::debug("load", "max_loadavg: ", $self->string(), " ", $self->{'max_loadavg'}); + return $self->{'max_loadavg'}; +} + +sub set_max_loadavg { + my $self = shift; + $self->{'max_loadavg'} = shift; +} + +sub compute_max_loadavg { + # Parse the max loadaverage that the user asked for using --load + # Returns: + # max loadaverage + my $self = shift; + my $loadspec = shift; + my $load; + if(defined $loadspec) { + if($loadspec =~ /^\+(\d+)$/) { + # E.g. --load +2 + my $j = $1; + $load = + $self->ncpus() + $j; + } elsif ($loadspec =~ /^-(\d+)$/) { + # E.g. --load -2 + my $j = $1; + $load = + $self->ncpus() - $j; + } elsif ($loadspec =~ /^(\d+)\%$/) { + my $j = $1; + $load = + $self->ncpus() * $j / 100; + } elsif ($loadspec =~ /^(\d+(\.\d+)?)$/) { + $load = $1; + } elsif (-f $loadspec) { + $Global::max_load_file = $loadspec; + $Global::max_load_file_last_mod = (stat($Global::max_load_file))[9]; + if(open(my $in_fh, "<", $Global::max_load_file)) { + my $opt_load_file = join("",<$in_fh>); + close $in_fh; + $load = $self->compute_max_loadavg($opt_load_file); + } else { + print $Global::original_stderr "Cannot open $loadspec\n"; + ::wait_and_exit(255); + } + } else { + print $Global::original_stderr "Parsing of --load failed\n"; + ::die_usage(); + } + if($load < 0.01) { + $load = 0.01; + } + } + return $load; +} + +sub time_to_login { + my $self = shift; + return $self->{'time_to_login'}; +} + +sub set_time_to_login { + my $self = shift; + $self->{'time_to_login'} = shift; +} + +sub max_jobs_running { + my $self = shift; + if(not defined $self->{'max_jobs_running'}) { + my $nproc = $self->compute_number_of_processes($opt::jobs); + $self->set_max_jobs_running($nproc); + } + return $self->{'max_jobs_running'}; +} + +sub orig_max_jobs_running { + my $self = shift; + return $self->{'orig_max_jobs_running'}; +} + +sub compute_number_of_processes { + # Number of processes wanted and limited by system resources + # Returns: + # Number of processes + my $self = shift; + my $opt_P = shift; + my $wanted_processes = $self->user_requested_processes($opt_P); + if(not defined $wanted_processes) { + $wanted_processes = $Global::default_simultaneous_sshlogins; + } + ::debug("load", "Wanted procs: $wanted_processes\n"); + my $system_limit = + $self->processes_available_by_system_limit($wanted_processes); + ::debug("load", "Limited to procs: $system_limit\n"); + return $system_limit; +} + +sub processes_available_by_system_limit { + # If the wanted number of processes is bigger than the system limits: + # Limit them to the system limits + # Limits are: File handles, number of input lines, processes, + # and taking > 1 second to spawn 10 extra processes + # Returns: + # Number of processes + my $self = shift; + my $wanted_processes = shift; + + my $system_limit = 0; + my @jobs = (); + my $job; + my @args = (); + my $arg; + my $more_filehandles = 1; + my $max_system_proc_reached = 0; + my $slow_spawining_warning_printed = 0; + my $time = time; + my %fh; + my @children; + + # Reserve filehandles + # perl uses 7 filehandles for something? + # parallel uses 1 for memory_usage + # parallel uses 4 for ? + for my $i (1..12) { + open($fh{"init-$i"}, "<", "/dev/null"); + } + + for(1..2) { + # System process limit + my $child; + if($child = fork()) { + push (@children,$child); + $Global::unkilled_children{$child} = 1; + } elsif(defined $child) { + # The child takes one process slot + # It will be killed later + $SIG{TERM} = $Global::original_sig{TERM}; + sleep 10000000; + exit(0); + } else { + $max_system_proc_reached = 1; + } + } + my $count_jobs_already_read = $Global::JobQueue->next_seq(); + my $wait_time_for_getting_args = 0; + my $start_time = time; + while(1) { + $system_limit >= $wanted_processes and last; + not $more_filehandles and last; + $max_system_proc_reached and last; + my $before_getting_arg = time; + if($Global::semaphore or $opt::pipe) { + # Skip: No need to get args + } elsif(defined $opt::retries and $count_jobs_already_read) { + # For retries we may need to run all jobs on this sshlogin + # so include the already read jobs for this sshlogin + $count_jobs_already_read--; + } else { + if($opt::X or $opt::m) { + # The arguments may have to be re-spread over several jobslots + # So pessimistically only read one arg per jobslot + # instead of a full commandline + if($Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->empty()) { + if($Global::JobQueue->empty()) { + last; + } else { + ($job) = $Global::JobQueue->get(); + push(@jobs, $job); + } + } else { + ($arg) = $Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->get(); + push(@args, $arg); + } + } else { + # If there are no more command lines, then we have a process + # per command line, so no need to go further + $Global::JobQueue->empty() and last; + ($job) = $Global::JobQueue->get(); + push(@jobs, $job); + } + } + $wait_time_for_getting_args += time - $before_getting_arg; + $system_limit++; + + # Every simultaneous process uses 2 filehandles when grouping + # Every simultaneous process uses 2 filehandles when compressing + $more_filehandles = open($fh{$system_limit*10}, "<", "/dev/null") + && open($fh{$system_limit*10+2}, "<", "/dev/null") + && open($fh{$system_limit*10+3}, "<", "/dev/null") + && open($fh{$system_limit*10+4}, "<", "/dev/null"); + + # System process limit + my $child; + if($child = fork()) { + push (@children,$child); + $Global::unkilled_children{$child} = 1; + } elsif(defined $child) { + # The child takes one process slot + # It will be killed later + $SIG{TERM} = $Global::original_sig{TERM}; + sleep 10000000; + exit(0); + } else { + $max_system_proc_reached = 1; + } + my $forktime = time - $time - $wait_time_for_getting_args; + ::debug("run", "Time to fork $system_limit procs: $wait_time_for_getting_args ", + $forktime, + " (processes so far: ", $system_limit,")\n"); + if($system_limit > 10 and + $forktime > 1 and + $forktime > $system_limit * 0.01 + and not $slow_spawining_warning_printed) { + # It took more than 0.01 second to fork a processes on avg. + # Give the user a warning. He can press Ctrl-C if this + # sucks. + print $Global::original_stderr + ("parallel: Warning: Starting $system_limit processes took > $forktime sec.\n", + "Consider adjusting -j. Press CTRL-C to stop.\n"); + $slow_spawining_warning_printed = 1; + } + } + # Cleanup: Close the files + for (values %fh) { close $_ } + # Cleanup: Kill the children + for my $pid (@children) { + kill 9, $pid; + waitpid($pid,0); + delete $Global::unkilled_children{$pid}; + } + # Cleanup: Unget the command_lines or the @args + $Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->unget(@args); + $Global::JobQueue->unget(@jobs); + if($system_limit < $wanted_processes) { + # The system_limit is less than the wanted_processes + if($system_limit < 1 and not $Global::JobQueue->empty()) { + ::warning("Cannot spawn any jobs. Raising ulimit -u or /etc/security/limits.conf\n", + "or /proc/sys/kernel/pid_max may help.\n"); + ::wait_and_exit(255); + } + if(not $more_filehandles) { + ::warning("Only enough file handles to run ", $system_limit, " jobs in parallel.\n", + "Running 'parallel -j0 -N", $system_limit, " --pipe parallel -j0' or ", + "raising ulimit -n or /etc/security/limits.conf may help.\n"); + } + if($max_system_proc_reached) { + ::warning("Only enough available processes to run ", $system_limit, + " jobs in parallel. Raising ulimit -u or /etc/security/limits.conf\n", + "or /proc/sys/kernel/pid_max may help.\n"); + } + } + if($] == 5.008008 and $system_limit > 1000) { + # https://savannah.gnu.org/bugs/?36942 + $system_limit = 1000; + } + if($Global::JobQueue->empty()) { + $system_limit ||= 1; + } + if($self->string() ne ":" and + $system_limit > $Global::default_simultaneous_sshlogins) { + $system_limit = + $self->simultaneous_sshlogin_limit($system_limit); + } + return $system_limit; +} + +sub simultaneous_sshlogin_limit { + # Test by logging in wanted number of times simultaneously + # Returns: + # min($wanted_processes,$working_simultaneous_ssh_logins-1) + my $self = shift; + my $wanted_processes = shift; + if($self->{'time_to_login'}) { + return $wanted_processes; + } + + # Try twice because it guesses wrong sometimes + # Choose the minimal + my $ssh_limit = + ::min($self->simultaneous_sshlogin($wanted_processes), + $self->simultaneous_sshlogin($wanted_processes)); + if($ssh_limit < $wanted_processes) { + my $serverlogin = $self->serverlogin(); + ::warning("ssh to $serverlogin only allows ", + "for $ssh_limit simultaneous logins.\n", + "You may raise this by changing ", + "/etc/ssh/sshd_config:MaxStartups and MaxSessions on $serverlogin.\n", + "Using only ",$ssh_limit-1," connections ", + "to avoid race conditions.\n"); + } + # Race condition can cause problem if using all sshs. + if($ssh_limit > 1) { $ssh_limit -= 1; } + return $ssh_limit; +} + +sub simultaneous_sshlogin { + # Using $sshlogin try to see if we can do $wanted_processes + # simultaneous logins + # (ssh host echo simultaneouslogin & ssh host echo simultaneouslogin & ...)|grep simul|wc -l + # Returns: + # Number of succesful logins + my $self = shift; + my $wanted_processes = shift; + my $sshcmd = $self->sshcommand(); + my $serverlogin = $self->serverlogin(); + my $sshdelay = $opt::sshdelay ? "sleep $opt::sshdelay;" : ""; + my $cmd = "$sshdelay$sshcmd $serverlogin echo simultaneouslogin &1 &"x$wanted_processes; + ::debug("init", "Trying $wanted_processes logins at $serverlogin\n"); + open (my $simul_fh, "-|", "($cmd)|grep simultaneouslogin | wc -l") or + ::die_bug("simultaneouslogin"); + my $ssh_limit = <$simul_fh>; + close $simul_fh; + chomp $ssh_limit; + return $ssh_limit; +} + +sub set_ncpus { + my $self = shift; + $self->{'ncpus'} = shift; +} + +sub user_requested_processes { + # Parse the number of processes that the user asked for using -j + # Returns: + # the number of processes to run on this sshlogin + my $self = shift; + my $opt_P = shift; + my $processes; + if(defined $opt_P) { + if($opt_P =~ /^\+(\d+)$/) { + # E.g. -P +2 + my $j = $1; + $processes = + $self->ncpus() + $j; + } elsif ($opt_P =~ /^-(\d+)$/) { + # E.g. -P -2 + my $j = $1; + $processes = + $self->ncpus() - $j; + } elsif ($opt_P =~ /^(\d+(\.\d+)?)\%$/) { + # E.g. -P 10.5% + my $j = $1; + $processes = + $self->ncpus() * $j / 100; + } elsif ($opt_P =~ /^(\d+)$/) { + $processes = $1; + if($processes == 0) { + # -P 0 = infinity (or at least close) + $processes = $Global::infinity; + } + } elsif (-f $opt_P) { + $Global::max_procs_file = $opt_P; + $Global::max_procs_file_last_mod = (stat($Global::max_procs_file))[9]; + if(open(my $in_fh, "<", $Global::max_procs_file)) { + my $opt_P_file = join("",<$in_fh>); + close $in_fh; + $processes = $self->user_requested_processes($opt_P_file); + } else { + ::error("Cannot open $opt_P.\n"); + ::wait_and_exit(255); + } + } else { + ::error("Parsing of --jobs/-j/--max-procs/-P failed.\n"); + ::die_usage(); + } + $processes = ::ceil($processes); + } + return $processes; +} + +sub ncpus { + my $self = shift; + if(not defined $self->{'ncpus'}) { + my $sshcmd = $self->sshcommand(); + my $serverlogin = $self->serverlogin(); + if($serverlogin eq ":") { + if($opt::use_cpus_instead_of_cores) { + $self->{'ncpus'} = no_of_cpus(); + } else { + $self->{'ncpus'} = no_of_cores(); + } + } else { + my $ncpu; + my $sqe = ::shell_quote_scalar($Global::envvar); + if($opt::use_cpus_instead_of_cores) { + $ncpu = qx(echo|$sshcmd $serverlogin $sqe parallel --number-of-cpus); + } else { + ::debug("init",qq(echo|$sshcmd $serverlogin $sqe parallel --number-of-cores\n)); + $ncpu = qx(echo|$sshcmd $serverlogin $sqe parallel --number-of-cores); + } + chomp $ncpu; + if($ncpu =~ /^\s*[0-9]+\s*$/s) { + $self->{'ncpus'} = $ncpu; + } else { + ::warning("Could not figure out ", + "number of cpus on $serverlogin ($ncpu). Using 1.\n"); + $self->{'ncpus'} = 1; + } + } + } + return $self->{'ncpus'}; +} + +sub no_of_cpus { + # Returns: + # Number of physical CPUs + local $/="\n"; # If delimiter is set, then $/ will be wrong + my $no_of_cpus; + if ($^O eq 'linux') { + $no_of_cpus = no_of_cpus_gnu_linux() || no_of_cores_gnu_linux(); + } elsif ($^O eq 'freebsd') { + $no_of_cpus = no_of_cpus_freebsd(); + } elsif ($^O eq 'netbsd') { + $no_of_cpus = no_of_cpus_netbsd(); + } elsif ($^O eq 'openbsd') { + $no_of_cpus = no_of_cpus_openbsd(); + } elsif ($^O eq 'gnu') { + $no_of_cpus = no_of_cpus_hurd(); + } elsif ($^O eq 'darwin') { + $no_of_cpus = no_of_cpus_darwin(); + } elsif ($^O eq 'solaris') { + $no_of_cpus = no_of_cpus_solaris(); + } elsif ($^O eq 'aix') { + $no_of_cpus = no_of_cpus_aix(); + } elsif ($^O eq 'hpux') { + $no_of_cpus = no_of_cpus_hpux(); + } elsif ($^O eq 'nto') { + $no_of_cpus = no_of_cpus_qnx(); + } elsif ($^O eq 'svr5') { + $no_of_cpus = no_of_cpus_openserver(); + } elsif ($^O eq 'irix') { + $no_of_cpus = no_of_cpus_irix(); + } elsif ($^O eq 'dec_osf') { + $no_of_cpus = no_of_cpus_tru64(); + } else { + $no_of_cpus = (no_of_cpus_gnu_linux() + || no_of_cpus_freebsd() + || no_of_cpus_netbsd() + || no_of_cpus_openbsd() + || no_of_cpus_hurd() + || no_of_cpus_darwin() + || no_of_cpus_solaris() + || no_of_cpus_aix() + || no_of_cpus_hpux() + || no_of_cpus_qnx() + || no_of_cpus_openserver() + || no_of_cpus_irix() + || no_of_cpus_tru64() + # Number of cores is better than no guess for #CPUs + || nproc() + ); + } + if($no_of_cpus) { + chomp $no_of_cpus; + return $no_of_cpus; + } else { + ::warning("Cannot figure out number of cpus. Using 1.\n"); + return 1; + } +} + +sub no_of_cores { + # Returns: + # Number of CPU cores + local $/="\n"; # If delimiter is set, then $/ will be wrong + my $no_of_cores; + if ($^O eq 'linux') { + $no_of_cores = no_of_cores_gnu_linux(); + } elsif ($^O eq 'freebsd') { + $no_of_cores = no_of_cores_freebsd(); + } elsif ($^O eq 'netbsd') { + $no_of_cores = no_of_cores_netbsd(); + } elsif ($^O eq 'openbsd') { + $no_of_cores = no_of_cores_openbsd(); + } elsif ($^O eq 'gnu') { + $no_of_cores = no_of_cores_hurd(); + } elsif ($^O eq 'darwin') { + $no_of_cores = no_of_cores_darwin(); + } elsif ($^O eq 'solaris') { + $no_of_cores = no_of_cores_solaris(); + } elsif ($^O eq 'aix') { + $no_of_cores = no_of_cores_aix(); + } elsif ($^O eq 'hpux') { + $no_of_cores = no_of_cores_hpux(); + } elsif ($^O eq 'nto') { + $no_of_cores = no_of_cores_qnx(); + } elsif ($^O eq 'svr5') { + $no_of_cores = no_of_cores_openserver(); + } elsif ($^O eq 'irix') { + $no_of_cores = no_of_cores_irix(); + } elsif ($^O eq 'dec_osf') { + $no_of_cores = no_of_cores_tru64(); + } else { + $no_of_cores = (no_of_cores_gnu_linux() + || no_of_cores_freebsd() + || no_of_cores_netbsd() + || no_of_cores_openbsd() + || no_of_cores_hurd() + || no_of_cores_darwin() + || no_of_cores_solaris() + || no_of_cores_aix() + || no_of_cores_hpux() + || no_of_cores_qnx() + || no_of_cores_openserver() + || no_of_cores_irix() + || no_of_cores_tru64() + || nproc() + ); + } + if($no_of_cores) { + chomp $no_of_cores; + return $no_of_cores; + } else { + ::warning("Cannot figure out number of CPU cores. Using 1.\n"); + return 1; + } +} + +sub nproc { + # Returns: + # Number of cores using `nproc` + my $no_of_cores = `nproc 2>/dev/null`; + return $no_of_cores; +} + +sub no_of_cpus_gnu_linux { + # Returns: + # Number of physical CPUs on GNU/Linux + # undef if not GNU/Linux + my $no_of_cpus; + my $no_of_cores; + if(-e "/proc/cpuinfo") { + $no_of_cpus = 0; + $no_of_cores = 0; + my %seen; + open(my $in_fh, "<", "/proc/cpuinfo") || return undef; + while(<$in_fh>) { + if(/^physical id.*[:](.*)/ and not $seen{$1}++) { + $no_of_cpus++; + } + /^processor.*[:]/i and $no_of_cores++; + } + close $in_fh; + } + return ($no_of_cpus||$no_of_cores); +} + +sub no_of_cores_gnu_linux { + # Returns: + # Number of CPU cores on GNU/Linux + # undef if not GNU/Linux + my $no_of_cores; + if(-e "/proc/cpuinfo") { + $no_of_cores = 0; + open(my $in_fh, "<", "/proc/cpuinfo") || return undef; + while(<$in_fh>) { + /^processor.*[:]/i and $no_of_cores++; + } + close $in_fh; + } + return $no_of_cores; +} + +sub no_of_cpus_freebsd { + # Returns: + # Number of physical CPUs on FreeBSD + # undef if not FreeBSD + my $no_of_cpus = + (`sysctl -a dev.cpu 2>/dev/null | grep \%parent | awk '{ print \$2 }' | uniq | wc -l | awk '{ print \$1 }'` + or + `sysctl hw.ncpu 2>/dev/null | awk '{ print \$2 }'`); + chomp $no_of_cpus; + return $no_of_cpus; +} + +sub no_of_cores_freebsd { + # Returns: + # Number of CPU cores on FreeBSD + # undef if not FreeBSD + my $no_of_cores = + (`sysctl hw.ncpu 2>/dev/null | awk '{ print \$2 }'` + or + `sysctl -a hw 2>/dev/null | grep [^a-z]logicalcpu[^a-z] | awk '{ print \$2 }'`); + chomp $no_of_cores; + return $no_of_cores; +} + +sub no_of_cpus_netbsd { + # Returns: + # Number of physical CPUs on NetBSD + # undef if not NetBSD + my $no_of_cpus = `sysctl -n hw.ncpu 2>/dev/null`; + chomp $no_of_cpus; + return $no_of_cpus; +} + +sub no_of_cores_netbsd { + # Returns: + # Number of CPU cores on NetBSD + # undef if not NetBSD + my $no_of_cores = `sysctl -n hw.ncpu 2>/dev/null`; + chomp $no_of_cores; + return $no_of_cores; +} + +sub no_of_cpus_openbsd { + # Returns: + # Number of physical CPUs on OpenBSD + # undef if not OpenBSD + my $no_of_cpus = `sysctl -n hw.ncpu 2>/dev/null`; + chomp $no_of_cpus; + return $no_of_cpus; +} + +sub no_of_cores_openbsd { + # Returns: + # Number of CPU cores on OpenBSD + # undef if not OpenBSD + my $no_of_cores = `sysctl -n hw.ncpu 2>/dev/null`; + chomp $no_of_cores; + return $no_of_cores; +} + +sub no_of_cpus_hurd { + # Returns: + # Number of physical CPUs on HURD + # undef if not HURD + my $no_of_cpus = `nproc`; + chomp $no_of_cpus; + return $no_of_cpus; +} + +sub no_of_cores_hurd { + # Returns: + # Number of physical CPUs on HURD + # undef if not HURD + my $no_of_cores = `nproc`; + chomp $no_of_cores; + return $no_of_cores; +} + +sub no_of_cpus_darwin { + # Returns: + # Number of physical CPUs on Mac Darwin + # undef if not Mac Darwin + my $no_of_cpus = + (`sysctl -n hw.physicalcpu 2>/dev/null` + or + `sysctl -a hw 2>/dev/null | grep [^a-z]physicalcpu[^a-z] | awk '{ print \$2 }'`); + return $no_of_cpus; +} + +sub no_of_cores_darwin { + # Returns: + # Number of CPU cores on Mac Darwin + # undef if not Mac Darwin + my $no_of_cores = + (`sysctl -n hw.logicalcpu 2>/dev/null` + or + `sysctl -a hw 2>/dev/null | grep [^a-z]logicalcpu[^a-z] | awk '{ print \$2 }'`); + return $no_of_cores; +} + +sub no_of_cpus_solaris { + # Returns: + # Number of physical CPUs on Solaris + # undef if not Solaris + if(-x "/usr/sbin/psrinfo") { + my @psrinfo = `/usr/sbin/psrinfo`; + if($#psrinfo >= 0) { + return $#psrinfo +1; + } + } + if(-x "/usr/sbin/prtconf") { + my @prtconf = `/usr/sbin/prtconf | grep cpu..instance`; + if($#prtconf >= 0) { + return $#prtconf +1; + } + } + return undef; +} + +sub no_of_cores_solaris { + # Returns: + # Number of CPU cores on Solaris + # undef if not Solaris + if(-x "/usr/sbin/psrinfo") { + my @psrinfo = `/usr/sbin/psrinfo`; + if($#psrinfo >= 0) { + return $#psrinfo +1; + } + } + if(-x "/usr/sbin/prtconf") { + my @prtconf = `/usr/sbin/prtconf | grep cpu..instance`; + if($#prtconf >= 0) { + return $#prtconf +1; + } + } + return undef; +} + +sub no_of_cpus_aix { + # Returns: + # Number of physical CPUs on AIX + # undef if not AIX + my $no_of_cpus = 0; + if(-x "/usr/sbin/lscfg") { + open(my $in_fh, "-|", "/usr/sbin/lscfg -vs |grep proc | wc -l|tr -d ' '") + || return undef; + $no_of_cpus = <$in_fh>; + chomp ($no_of_cpus); + close $in_fh; + } + return $no_of_cpus; +} + +sub no_of_cores_aix { + # Returns: + # Number of CPU cores on AIX + # undef if not AIX + my $no_of_cores; + if(-x "/usr/bin/vmstat") { + open(my $in_fh, "-|", "/usr/bin/vmstat 1 1") || return undef; + while(<$in_fh>) { + /lcpu=([0-9]*) / and $no_of_cores = $1; + } + close $in_fh; + } + return $no_of_cores; +} + +sub no_of_cpus_hpux { + # Returns: + # Number of physical CPUs on HP-UX + # undef if not HP-UX + my $no_of_cpus = + (`/usr/bin/mpsched -s 2>&1 | grep 'Locality Domain Count' | awk '{ print \$4 }'`); + return $no_of_cpus; +} + +sub no_of_cores_hpux { + # Returns: + # Number of CPU cores on HP-UX + # undef if not HP-UX + my $no_of_cores = + (`/usr/bin/mpsched -s 2>&1 | grep 'Processor Count' | awk '{ print \$3 }'`); + return $no_of_cores; +} + +sub no_of_cpus_qnx { + # Returns: + # Number of physical CPUs on QNX + # undef if not QNX + # BUG: It is now known how to calculate this. + my $no_of_cpus = 0; + return $no_of_cpus; +} + +sub no_of_cores_qnx { + # Returns: + # Number of CPU cores on QNX + # undef if not QNX + # BUG: It is now known how to calculate this. + my $no_of_cores = 0; + return $no_of_cores; +} + +sub no_of_cpus_openserver { + # Returns: + # Number of physical CPUs on SCO OpenServer + # undef if not SCO OpenServer + my $no_of_cpus = 0; + if(-x "/usr/sbin/psrinfo") { + my @psrinfo = `/usr/sbin/psrinfo`; + if($#psrinfo >= 0) { + return $#psrinfo +1; + } + } + return $no_of_cpus; +} + +sub no_of_cores_openserver { + # Returns: + # Number of CPU cores on SCO OpenServer + # undef if not SCO OpenServer + my $no_of_cores = 0; + if(-x "/usr/sbin/psrinfo") { + my @psrinfo = `/usr/sbin/psrinfo`; + if($#psrinfo >= 0) { + return $#psrinfo +1; + } + } + return $no_of_cores; +} + +sub no_of_cpus_irix { + # Returns: + # Number of physical CPUs on IRIX + # undef if not IRIX + my $no_of_cpus = `hinv | grep HZ | grep Processor | awk '{print \$1}'`; + return $no_of_cpus; +} + +sub no_of_cores_irix { + # Returns: + # Number of CPU cores on IRIX + # undef if not IRIX + my $no_of_cores = `hinv | grep HZ | grep Processor | awk '{print \$1}'`; + return $no_of_cores; +} + +sub no_of_cpus_tru64 { + # Returns: + # Number of physical CPUs on Tru64 + # undef if not Tru64 + my $no_of_cpus = `sizer -pr`; + return $no_of_cpus; +} + +sub no_of_cores_tru64 { + # Returns: + # Number of CPU cores on Tru64 + # undef if not Tru64 + my $no_of_cores = `sizer -pr`; + return $no_of_cores; +} + +sub sshcommand { + my $self = shift; + if (not defined $self->{'sshcommand'}) { + $self->sshcommand_of_sshlogin(); + } + return $self->{'sshcommand'}; +} + +sub serverlogin { + my $self = shift; + if (not defined $self->{'serverlogin'}) { + $self->sshcommand_of_sshlogin(); + } + return $self->{'serverlogin'}; +} + +sub sshcommand_of_sshlogin { + # 'server' -> ('ssh -S /tmp/parallel-ssh-RANDOM/host-','server') + # 'user@server' -> ('ssh','user@server') + # 'myssh user@server' -> ('myssh','user@server') + # 'myssh -l user server' -> ('myssh -l user','server') + # '/usr/bin/myssh -l user server' -> ('/usr/bin/myssh -l user','server') + # Returns: + # sshcommand - defaults to 'ssh' + # login@host + my $self = shift; + my ($sshcmd, $serverlogin); + if($self->{'string'} =~ /(.+) (\S+)$/) { + # Own ssh command + $sshcmd = $1; $serverlogin = $2; + } else { + # Normal ssh + if($opt::controlmaster) { + # Use control_path to make ssh faster + my $control_path = $self->control_path_dir()."/ssh-%r@%h:%p"; + $sshcmd = "ssh -S ".$control_path; + $serverlogin = $self->{'string'}; + if(not $self->{'control_path'}{$control_path}++) { + # Master is not running for this control_path + # Start it + my $pid = fork(); + if($pid) { + $Global::sshmaster{$pid} ||= 1; + } else { + $SIG{'TERM'} = undef; + # Ignore the 'foo' being printed + open(STDOUT,">","/dev/null"); + # OpenSSH_3.6.1p2 gives 'tcgetattr: Invalid argument' with -tt + # STDERR >/dev/null to ignore "process_mux_new_session: tcgetattr: Invalid argument" + open(STDERR,">","/dev/null"); + open(STDIN,"<","/dev/null"); + # Run a sleep that outputs data, so it will discover if the ssh connection closes. + my $sleep = ::shell_quote_scalar('$|=1;while(1){sleep 1;print "foo\n"}'); + my @master = ("ssh", "-tt", "-MTS", $control_path, $serverlogin, "perl", "-e", $sleep); + exec(@master); + } + } + } else { + $sshcmd = "ssh"; $serverlogin = $self->{'string'}; + } + } + $self->{'sshcommand'} = $sshcmd; + $self->{'serverlogin'} = $serverlogin; +} + +sub control_path_dir { + # Returns: + # path to directory + my $self = shift; + if(not defined $self->{'control_path_dir'}) { + -e $ENV{'HOME'}."/.parallel" or mkdir $ENV{'HOME'}."/.parallel"; + -e $ENV{'HOME'}."/.parallel/tmp" or mkdir $ENV{'HOME'}."/.parallel/tmp"; + $self->{'control_path_dir'} = + File::Temp::tempdir($ENV{'HOME'} + . "/.parallel/tmp/control_path_dir-XXXX", + CLEANUP => 1); + } + return $self->{'control_path_dir'}; +} + +sub rsync_transfer_cmd { + # Command to run to transfer a file + # Input: + # $file = filename of file to transfer + # $workdir = destination dir + # Returns: + # $cmd = rsync command to run to transfer $file ("" if unreadable) + my $self = shift; + my $file = shift; + my $workdir = shift; + if(not -r $file) { + ::warning($file, " is not readable and will not be transferred.\n"); + return "true"; + } + my $rsync_destdir; + if($file =~ m:^/:) { + # rsync /foo/bar / + $rsync_destdir = "/"; + } else { + $rsync_destdir = ::shell_quote_file($workdir); + } + $file = ::shell_quote_file($file); + my $sshcmd = $self->sshcommand(); + my $rsync_opt = "-rlDzR -e" . ::shell_quote_scalar($sshcmd); + my $serverlogin = $self->serverlogin(); + # Make dir if it does not exist + return "( $sshcmd $serverlogin mkdir -p $rsync_destdir;" . + rsync()." $rsync_opt $file $serverlogin:$rsync_destdir )"; +} + +sub cleanup_cmd { + # Command to run to remove the remote file + # Input: + # $file = filename to remove + # $workdir = destination dir + # Returns: + # $cmd = ssh command to run to remove $file and empty parent dirs + my $self = shift; + my $file = shift; + my $workdir = shift; + my $f = $file; + if($f =~ m:/\./:) { + # foo/bar/./baz/quux => workdir/baz/quux + # /foo/bar/./baz/quux => workdir/baz/quux + $f =~ s:.*/\./:$workdir/:; + } elsif($f =~ m:^[^/]:) { + # foo/bar => workdir/foo/bar + $f = $workdir."/".$f; + } + my @subdirs = split m:/:, ::dirname($f); + my @rmdir; + my $dir = ""; + for(@subdirs) { + $dir .= $_."/"; + unshift @rmdir, ::shell_quote_file($dir); + } + my $rmdir = @rmdir ? "rmdir @rmdir 2>/dev/null;" : ""; + if(defined $opt::workdir and $opt::workdir eq "...") { + $rmdir .= "rm -rf " . ::shell_quote_file($workdir).';'; + } + + $f = ::shell_quote_file($f); + my $sshcmd = $self->sshcommand(); + my $serverlogin = $self->serverlogin(); + return "$sshcmd $serverlogin ".::shell_quote_scalar("(rm -f $f; $rmdir)"); +} + +{ + my $rsync; + + sub rsync { + # rsync 3.1.x uses protocol 31 which is unsupported by 2.5.7. + # If the version >= 3.1.0: downgrade to protocol 30 + if(not $rsync) { + my @out = `rsync --version`; + for (@out) { + if(/version (\d+.\d+)(.\d+)?/) { + if($1 >= 3.1) { + # Version 3.1.0 or later: Downgrade to protocol 30 + $rsync = "rsync --protocol 30"; + } else { + $rsync = "rsync"; + } + } + } + $rsync or ::die_bug("Cannot figure out version of rsync: @out"); + } + return $rsync; + } +} + + +package JobQueue; + +sub new { + my $class = shift; + my $commandref = shift; + my $read_from = shift; + my $context_replace = shift; + my $max_number_of_args = shift; + my $return_files = shift; + my $commandlinequeue = CommandLineQueue->new + ($commandref, $read_from, $context_replace, $max_number_of_args, + $return_files); + my @unget = (); + return bless { + 'unget' => \@unget, + 'commandlinequeue' => $commandlinequeue, + 'total_jobs' => undef, + }, ref($class) || $class; +} + +sub get { + my $self = shift; + + if(@{$self->{'unget'}}) { + my $job = shift @{$self->{'unget'}}; + return ($job); + } else { + my $commandline = $self->{'commandlinequeue'}->get(); + if(defined $commandline) { + my $job = Job->new($commandline); + return $job; + } else { + return undef; + } + } +} + +sub unget { + my $self = shift; + unshift @{$self->{'unget'}}, @_; +} + +sub empty { + my $self = shift; + my $empty = (not @{$self->{'unget'}}) + && $self->{'commandlinequeue'}->empty(); + ::debug("run", "JobQueue->empty $empty "); + return $empty; +} + +sub total_jobs { + my $self = shift; + if(not defined $self->{'total_jobs'}) { + my $job; + my @queue; + my $start = time; + while($job = $self->get()) { + if(time - $start > 10) { + ::warning("Reading all arguments takes longer than 10 seconds.\n"); + $opt::eta && ::warning("Consider removing --eta.\n"); + $opt::bar && ::warning("Consider removing --bar.\n"); + last; + } + push @queue, $job; + } + while($job = $self->get()) { + push @queue, $job; + } + + $self->unget(@queue); + $self->{'total_jobs'} = $#queue+1; + } + return $self->{'total_jobs'}; +} + +sub next_seq { + my $self = shift; + + return $self->{'commandlinequeue'}->seq(); +} + +sub quote_args { + my $self = shift; + return $self->{'commandlinequeue'}->quote_args(); +} + + +package Job; + +sub new { + my $class = shift; + my $commandlineref = shift; + return bless { + 'commandline' => $commandlineref, # CommandLine object + 'workdir' => undef, # --workdir + 'stdin' => undef, # filehandle for stdin (used for --pipe) + # filename for writing stdout to (used for --files) + 'remaining' => "", # remaining data not sent to stdin (used for --pipe) + 'datawritten' => 0, # amount of data sent via stdin (used for --pipe) + 'transfersize' => 0, # size of files using --transfer + 'returnsize' => 0, # size of files using --return + 'pid' => undef, + # hash of { SSHLogins => number of times the command failed there } + 'failed' => undef, + 'sshlogin' => undef, + # The commandline wrapped with rsync and ssh + 'sshlogin_wrap' => undef, + 'exitstatus' => undef, + 'exitsignal' => undef, + # Timestamp for timeout if any + 'timeout' => undef, + 'virgin' => 1, + }, ref($class) || $class; +} + +sub replaced { + my $self = shift; + $self->{'commandline'} or ::die_bug("commandline empty"); + return $self->{'commandline'}->replaced(); +} + +sub seq { + my $self = shift; + return $self->{'commandline'}->seq(); +} + +sub slot { + my $self = shift; + return $self->{'commandline'}->slot(); +} + +{ + my($cattail); + + sub cattail { + # Returns: + # $cattail = perl program for: cattail "decompress program" writerpid [file_to_decompress or stdin] [file_to_unlink] + if(not $cattail) { + $cattail = q{ + # cat followed by tail. + # If $writerpid dead: finish after this round + use Fcntl; + + $|=1; + + my ($cmd, $writerpid, $read_file, $unlink_file) = @ARGV; + if($read_file) { + open(IN,"<",$read_file) || die("cattail: Cannot open $read_file"); + } else { + *IN = *STDIN; + } + + my $flags; + fcntl(IN, F_GETFL, $flags) || die $!; # Get the current flags on the filehandle + $flags |= O_NONBLOCK; # Add non-blocking to the flags + fcntl(IN, F_SETFL, $flags) || die $!; # Set the flags on the filehandle + open(OUT,"|-",$cmd) || die("cattail: Cannot run $cmd"); + + while(1) { + # clear EOF + seek(IN,0,1); + my $writer_running = kill 0, $writerpid; + $read = sysread(IN,$buf,32768); + if($read) { + # We can unlink the file now: The writer has written something + -e $unlink_file and unlink $unlink_file; + # Blocking print + while($buf) { + my $bytes_written = syswrite(OUT,$buf); + # syswrite may be interrupted by SIGHUP + substr($buf,0,$bytes_written) = ""; + } + # Something printed: Wait less next time + $sleep /= 2; + } else { + if(eof(IN) and not $writer_running) { + # Writer dead: There will never be more to read => exit + exit; + } + # TODO This could probably be done more efficiently using select(2) + # Nothing read: Wait longer before next read + # Up to 30 milliseconds + $sleep = ($sleep < 30) ? ($sleep * 1.001 + 0.01) : ($sleep); + usleep($sleep); + } + } + + sub usleep { + # Sleep this many milliseconds. + my $secs = shift; + select(undef, undef, undef, $secs/1000); + } + }; + $cattail =~ s/#.*//mg; + $cattail =~ s/\s+/ /g; + } + return $cattail; + } +} + +sub openoutputfiles { + # Open files for STDOUT and STDERR + # Set file handles in $self->fh + my $self = shift; + my ($outfhw, $errfhw, $outname, $errname); + if($opt::results) { + my $args_as_dirname = $self->{'commandline'}->args_as_dirname(); + # Output in: prefix/name1/val1/name2/val2/stdout + my $dir = $opt::results."/".$args_as_dirname; + if(eval{ File::Path::mkpath($dir); }) { + # OK + } else { + # mkpath failed: Argument probably too long. + # Set $Global::max_file_length, which will keep the individual + # dir names shorter than the max length + max_file_name_length($opt::results); + $args_as_dirname = $self->{'commandline'}->args_as_dirname(); + # prefix/name1/val1/name2/val2/ + $dir = $opt::results."/".$args_as_dirname; + File::Path::mkpath($dir); + } + # prefix/name1/val1/name2/val2/stdout + $outname = "$dir/stdout"; + if(not open($outfhw, "+>", $outname)) { + ::error("Cannot write to `$outname'.\n"); + ::wait_and_exit(255); + } + # prefix/name1/val1/name2/val2/stderr + $errname = "$dir/stderr"; + if(not open($errfhw, "+>", $errname)) { + ::error("Cannot write to `$errname'.\n"); + ::wait_and_exit(255); + } + $self->set_fh(1,"unlink",""); + $self->set_fh(2,"unlink",""); + } elsif(not $opt::ungroup) { + # To group we create temporary files for STDOUT and STDERR + # To avoid the cleanup unlink the files immediately (but keep them open) + if(@Global::tee_jobs) { + # files must be removed when the tee is done + } elsif($opt::files) { + ($outfhw, $outname) = ::tmpfile(SUFFIX => ".par"); + ($errfhw, $errname) = ::tmpfile(SUFFIX => ".par"); + # --files => only remove stderr + $self->set_fh(1,"unlink",""); + $self->set_fh(2,"unlink",$errname); + } else { + ($outfhw, $outname) = ::tmpfile(SUFFIX => ".par"); + ($errfhw, $errname) = ::tmpfile(SUFFIX => ".par"); + $self->set_fh(1,"unlink",$outname); + $self->set_fh(2,"unlink",$errname); + } + } else { + # --ungroup + open($outfhw,">&",$Global::fd{1}) || die; + open($errfhw,">&",$Global::fd{2}) || die; + # File name must be empty as it will otherwise be printed + $outname = ""; + $errname = ""; + $self->set_fh(1,"unlink",$outname); + $self->set_fh(2,"unlink",$errname); + } + # Set writing FD + $self->set_fh(1,'w',$outfhw); + $self->set_fh(2,'w',$errfhw); + $self->set_fh(1,'name',$outname); + $self->set_fh(2,'name',$errname); + if($opt::compress) { + # Send stdout to stdin for $opt::compress_program(1) + # Send stderr to stdin for $opt::compress_program(2) + # cattail get pid: $pid = $self->fh($fdno,'rpid'); + my $cattail = cattail(); + for my $fdno (1,2) { + my $wpid = open(my $fdw,"|-","$opt::compress_program >>". + $self->fh($fdno,'name')) || die $?; + $self->set_fh($fdno,'w',$fdw); + $self->set_fh($fdno,'wpid',$wpid); + my $rpid = open(my $fdr, "-|", "perl", "-e", $cattail, + $opt::decompress_program, $wpid, + $self->fh($fdno,'name'),$self->fh($fdno,'unlink')) || die $?; + $self->set_fh($fdno,'r',$fdr); + $self->set_fh($fdno,'rpid',$rpid); + } + } elsif(not $opt::ungroup) { + # Set reading FD if using --group (--ungroup does not need) + for my $fdno (1,2) { + # Re-open the file for reading + # so fdw can be closed separately + # and fdr can be seeked separately (for --line-buffer) + open(my $fdr,"<", $self->fh($fdno,'name')) || + ::die_bug("fdr: Cannot open ".$self->fh($fdno,'name')); + $self->set_fh($fdno,'r',$fdr); + # Unlink if required + $Global::debug or unlink $self->fh($fdno,"unlink"); + } + } + if($opt::linebuffer) { + # Set non-blocking when using --linebuffer + $Global::use{"Fcntl"} ||= eval "use Fcntl qw(:DEFAULT :flock); 1;"; + for my $fdno (1,2) { + my $fdr = $self->fh($fdno,'r'); + my $flags; + fcntl($fdr, &F_GETFL, $flags) || die $!; # Get the current flags on the filehandle + $flags |= &O_NONBLOCK; # Add non-blocking to the flags + fcntl($fdr, &F_SETFL, $flags) || die $!; # Set the flags on the filehandle + } + } +} + +sub max_file_name_length { + # Figure out the max length of a subdir + # TODO and the max total length + # Ext4 = 255,130816 + my $testdir = shift; + + my $upper = 8_000_000; + my $len = 8; + my $dir="x"x$len; + do { + rmdir($testdir."/".$dir); + $len *= 16; + $dir="x"x$len; + } while (mkdir $testdir."/".$dir); + # Then search for the actual max length between $len/16 and $len + my $min = $len/16; + my $max = $len; + while($max-$min > 5) { + # If we are within 5 chars of the exact value: + # it is not worth the extra time to find the exact value + my $test = int(($min+$max)/2); + $dir="x"x$test; + if(mkdir $testdir."/".$dir) { + rmdir($testdir."/".$dir); + $min = $test; + } else { + $max = $test; + } + } + $Global::max_file_length = $min; + return $min; +} + +sub set_fh { + # Set file handle + my ($self, $fd_no, $key, $fh) = @_; + $self->{'fd'}{$fd_no,$key} = $fh; +} + +sub fh { + # Get file handle + my ($self, $fd_no, $key) = @_; + return $self->{'fd'}{$fd_no,$key}; +} + +sub write { + my $self = shift; + my $remaining_ref = shift; + my $stdin_fh = $self->fh(0,"w"); + syswrite($stdin_fh,$$remaining_ref); +} + +sub set_stdin_buffer { + # Copy stdin buffer from $block_ref up to $endpos + # Prepend with $header_ref + # Remove $recstart and $recend if needed + # Input: + # $header_ref = ref to $header to prepend + # $block_ref = ref to $block to pass on + # $endpos = length of $block to pass on + # $recstart = --recstart regexp + # $recend = --recend regexp + # Returns: + # N/A + my $self = shift; + my ($header_ref,$block_ref,$endpos,$recstart,$recend) = @_; + $self->{'stdin_buffer'} = ($self->virgin() ? $$header_ref : "").substr($$block_ref,0,$endpos); + if($opt::remove_rec_sep) { + remove_rec_sep(\$self->{'stdin_buffer'},$recstart,$recend); + } + $self->{'stdin_buffer_length'} = length $self->{'stdin_buffer'}; + $self->{'stdin_buffer_pos'} = 0; +} + +sub stdin_buffer_length { + my $self = shift; + return $self->{'stdin_buffer_length'}; +} + +sub remove_rec_sep { + my ($block_ref,$recstart,$recend) = @_; + # Remove record separator + $$block_ref =~ s/$recend$recstart//gos; + $$block_ref =~ s/^$recstart//os; + $$block_ref =~ s/$recend$//os; +} + +sub non_block_write { + my $self = shift; + my $something_written = 0; + use POSIX qw(:errno_h); +# use Fcntl; +# my $flags = ''; + for my $buf (substr($self->{'stdin_buffer'},$self->{'stdin_buffer_pos'})) { + my $in = $self->fh(0,"w"); +# fcntl($in, F_GETFL, $flags) +# or die "Couldn't get flags for HANDLE : $!\n"; +# $flags |= O_NONBLOCK; +# fcntl($in, F_SETFL, $flags) +# or die "Couldn't set flags for HANDLE: $!\n"; + my $rv = syswrite($in, $buf); + if (!defined($rv) && $! == EAGAIN) { + # would block + $something_written = 0; + } elsif ($self->{'stdin_buffer_pos'}+$rv != $self->{'stdin_buffer_length'}) { + # incomplete write + # Remove the written part + $self->{'stdin_buffer_pos'} += $rv; + $something_written = $rv; + } else { + # successfully wrote everything + my $a=""; + $self->set_stdin_buffer(\$a,\$a,"",""); + $something_written = $rv; + } + } + + ::debug("pipe", "Non-block: ", $something_written); + return $something_written; +} + + +sub virgin { + my $self = shift; + return $self->{'virgin'}; +} + +sub set_virgin { + my $self = shift; + $self->{'virgin'} = shift; +} + +sub pid { + my $self = shift; + return $self->{'pid'}; +} + +sub set_pid { + my $self = shift; + $self->{'pid'} = shift; +} + +sub starttime { + # Returns: + # UNIX-timestamp this job started + my $self = shift; + return sprintf("%.3f",$self->{'starttime'}); +} + +sub set_starttime { + my $self = shift; + my $starttime = shift || ::now(); + $self->{'starttime'} = $starttime; +} + +sub runtime { + # Returns: + # Run time in seconds + my $self = shift; + return sprintf("%.3f",int(($self->endtime() - $self->starttime())*1000)/1000); +} + +sub endtime { + # Returns: + # UNIX-timestamp this job ended + # 0 if not ended yet + my $self = shift; + return ($self->{'endtime'} || 0); +} + +sub set_endtime { + my $self = shift; + my $endtime = shift; + $self->{'endtime'} = $endtime; +} + +sub timedout { + # Is the job timedout? + # Input: + # $delta_time = time that the job may run + # Returns: + # True or false + my $self = shift; + my $delta_time = shift; + return time > $self->{'starttime'} + $delta_time; +} + +sub kill { + # Kill the job. + # Send the signals to (grand)*children and pid. + # If no signals: TERM TERM KILL + # Wait 200 ms after each TERM. + # Input: + # @signals = signals to send + my $self = shift; + my @signals = @_; + my @family_pids = $self->family_pids(); + # Record this jobs as failed + $self->set_exitstatus(-1); + # Send two TERMs to give time to clean up + ::debug("run", "Kill seq ", $self->seq(), "\n"); + my @send_signals = @signals || ("TERM", "TERM", "KILL"); + for my $signal (@send_signals) { + my $alive = 0; + for my $pid (@family_pids) { + if(kill 0, $pid) { + # The job still running + kill $signal, $pid; + $alive = 1; + } + } + # If a signal was given as input, do not do the sleep below + @signals and next; + + if($signal eq "TERM" and $alive) { + # Wait up to 200 ms between TERMs - but only if any pids are alive + my $sleep = 1; + for (my $sleepsum = 0; kill 0, $family_pids[0] and $sleepsum < 200; + $sleepsum += $sleep) { + $sleep = ::reap_usleep($sleep); + } + } + } +} + +sub family_pids { + # Find the pids with this->pid as (grand)*parent + # Returns: + # @pids = pids of (grand)*children + my $self = shift; + my $pid = $self->pid(); + my @pids; + + my ($children_of_ref, $parent_of_ref, $name_of_ref) = ::pid_table(); + + my @more = ($pid); + # While more (grand)*children + while(@more) { + my @m; + push @pids, @more; + for my $parent (@more) { + if($children_of_ref->{$parent}) { + # add the children of this parent + push @m, @{$children_of_ref->{$parent}}; + } + } + @more = @m; + } + return (@pids); +} + +sub failed { + # return number of times failed for this $sshlogin + # Input: + # $sshlogin + # Returns: + # Number of times failed for $sshlogin + my $self = shift; + my $sshlogin = shift; + return $self->{'failed'}{$sshlogin}; +} + +sub failed_here { + # return number of times failed for the current $sshlogin + # Returns: + # Number of times failed for this sshlogin + my $self = shift; + return $self->{'failed'}{$self->sshlogin()}; +} + +sub add_failed { + # increase the number of times failed for this $sshlogin + my $self = shift; + my $sshlogin = shift; + $self->{'failed'}{$sshlogin}++; +} + +sub add_failed_here { + # increase the number of times failed for the current $sshlogin + my $self = shift; + $self->{'failed'}{$self->sshlogin()}++; +} + +sub reset_failed { + # increase the number of times failed for this $sshlogin + my $self = shift; + my $sshlogin = shift; + delete $self->{'failed'}{$sshlogin}; +} + +sub reset_failed_here { + # increase the number of times failed for this $sshlogin + my $self = shift; + delete $self->{'failed'}{$self->sshlogin()}; +} + +sub min_failed { + # Returns: + # the number of sshlogins this command has failed on + # the minimal number of times this command has failed + my $self = shift; + my $min_failures = + ::min(map { $self->{'failed'}{$_} } keys %{$self->{'failed'}}); + my $number_of_sshlogins_failed_on = scalar keys %{$self->{'failed'}}; + return ($number_of_sshlogins_failed_on,$min_failures); +} + +sub total_failed { + # Returns: + # $total_failures = the number of times this command has failed + my $self = shift; + my $total_failures = 0; + for (values %{$self->{'failed'}}) { + $total_failures += $_; + } + return $total_failures; +} + +sub wrapped { + # Wrap command with: + # * --shellquote + # * --nice + # * --cat + # * --fifo + # * --sshlogin + # * --pipepart (@Global::cat_partials) + # * --pipe + # * --tmux + # The ordering of the wrapping is important: + # * --nice/--cat/--fifo should be done on the remote machine + # * --pipepart/--pipe should be done on the local machine inside --tmux + # Uses: + # $Global::envvar + # $opt::shellquote + # $opt::nice + # $Global::shell + # $opt::cat + # $opt::fifo + # @Global::cat_partials + # $opt::pipe + # $opt::tmux + # Returns: + # $self->{'wrapped'} = the command wrapped with the above + my $self = shift; + if(not defined $self->{'wrapped'}) { + my $command = $Global::envvar.$self->replaced(); + if($opt::shellquote) { + # Prepend echo + # and quote twice + $command = "echo " . + ::shell_quote_scalar(::shell_quote_scalar($command)); + } + if($opt::nice) { + # Prepend \nice -n19 $SHELL -c + # and quote. + # The '\' before nice is needed to avoid tcsh's built-in + $command = '\nice'. " -n". $opt::nice. " ". + $Global::shell. " -c ". + ::shell_quote_scalar($command); + } + if($opt::cat) { + # Prepend 'cat > {};' + # Append '_EXIT=$?;(rm {};exit $_EXIT)' + $command = + $self->{'commandline'}->replace_placeholders(["cat > \257<\257>; "], 0, 0). + $command. + $self->{'commandline'}->replace_placeholders( + ["; _EXIT=\$?; rm \257<\257>; exit \$_EXIT"], 0, 0); + } elsif($opt::fifo) { + # Prepend 'mkfifo {}; (' + # Append ') & _PID=$!; cat > {}; wait $_PID; _EXIT=$?;(rm {};exit $_EXIT)' + $command = + $self->{'commandline'}->replace_placeholders(["mkfifo \257<\257>; ("], 0, 0). + $command. + $self->{'commandline'}->replace_placeholders([") & _PID=\$!; cat > \257<\257>; ", + "wait \$_PID; _EXIT=\$?; ", + "rm \257<\257>; exit \$_EXIT"], + 0,0); + } + # Wrap with ssh + tranferring of files + $command = $self->sshlogin_wrap($command); + if(@Global::cat_partials) { + # Prepend: + # < /tmp/foo perl -e 'while(@ARGV) { sysseek(STDIN,shift,0) || die; $left = shift; while($read = sysread(STDIN,$buf, ($left > 32768 ? 32768 : $left))){ $left -= $read; syswrite(STDOUT,$buf); } }' 0 0 0 11 | + $command = (shift @Global::cat_partials). "|". "(". $command. ")"; + } elsif($opt::pipe) { + # Prepend EOF-detector to avoid starting $command if EOF. + # The $tmpfile might exist if run on a remote system - we accept that risk + my ($dummy_fh, $tmpfile) = ::tmpfile(SUFFIX => ".chr"); + # Unlink to avoid leaving files if --dry-run or --sshlogin + unlink $tmpfile; + $command = + # Exit value: + # empty input = true + # some input = exit val from command + qq{ sh -c 'dd bs=1 count=1 of=$tmpfile 2>/dev/null'; }. + qq{ test \! -s "$tmpfile" && rm -f "$tmpfile" && exec true; }. + qq{ (cat $tmpfile; rm $tmpfile; cat - ) | }. + "($command);"; + } + if($opt::tmux) { + # Wrap command with 'tmux' + $command = $self->tmux_wrap($command); + } + $self->{'wrapped'} = $command; + } + return $self->{'wrapped'}; +} + +sub set_sshlogin { + my $self = shift; + my $sshlogin = shift; + $self->{'sshlogin'} = $sshlogin; + delete $self->{'sshlogin_wrap'}; # If sshlogin is changed the wrap is wrong + delete $self->{'wrapped'}; +} + +sub sshlogin { + my $self = shift; + return $self->{'sshlogin'}; +} + +sub sshlogin_wrap { + # Wrap the command with the commands needed to run remotely + # Returns: + # $self->{'sshlogin_wrap'} = command wrapped with ssh+transfer commands + my $self = shift; + my $command = shift; + if(not defined $self->{'sshlogin_wrap'}) { + my $sshlogin = $self->sshlogin(); + my $sshcmd = $sshlogin->sshcommand(); + my $serverlogin = $sshlogin->serverlogin(); + my ($pre,$post,$cleanup)=("","",""); + + if($serverlogin eq ":") { + # No transfer neeeded + $self->{'sshlogin_wrap'} = $command; + } else { + # --transfer + $pre .= $self->sshtransfer(); + # --return + $post .= $self->sshreturn(); + # --cleanup + $post .= $self->sshcleanup(); + if($post) { + # We need to save the exit status of the job + $post = '_EXIT_status=$?; ' . $post . ' exit $_EXIT_status;'; + } + # If the remote login shell is (t)csh then use 'setenv' + # otherwise use 'export' + # We cannot use parse_env_var(), as PARALLEL_SEQ changes + # for each command + my $parallel_env = + ($Global::envwarn + . q{ 'eval `echo $SHELL | grep "/t\\{0,1\\}csh" > /dev/null } + . q{ && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; } + . q{ setenv PARALLEL_PID '$PARALLEL_PID' } + . q{ || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; } + . q{ PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' }); + my $remote_pre = ""; + my $ssh_options = ""; + if(($opt::pipe or $opt::pipepart) and $opt::ctrlc + or + not ($opt::pipe or $opt::pipepart) and not $opt::noctrlc) { + # TODO Determine if this is needed + # Propagating CTRL-C to kill remote jobs requires + # remote jobs to be run with a terminal. + $ssh_options = "-tt -oLogLevel=quiet"; +# $ssh_options = ""; + # tty - check if we have a tty. + # stty: + # -onlcr - make output 8-bit clean + # isig - pass CTRL-C as signal + # -echo - do not echo input + $remote_pre .= ::shell_quote_scalar('tty >/dev/null && stty isig -onlcr -echo;'); + } + if($opt::workdir) { + my $wd = ::shell_quote_file($self->workdir()); + $remote_pre .= ::shell_quote_scalar("mkdir -p ") . $wd . + ::shell_quote_scalar("; cd ") . $wd . + # exit 255 (instead of exec false) would be the correct thing, + # but that fails on tcsh + ::shell_quote_scalar(qq{ || exec false;}); + } + # This script is to solve the problem of + # * not mixing STDERR and STDOUT + # * terminating with ctrl-c + # It works on Linux but not Solaris + # Finishes on Solaris, but wrong exit code: + # $SIG{CHLD} = sub {exit ($?&127 ? 128+($?&127) : 1+$?>>8)}; + # Hangs on Solaris, but correct exit code on Linux: + # $SIG{CHLD} = sub { $done = 1 }; + # $p->poll; + my $signal_script = "perl -e '". + q{ + use IO::Poll; + $SIG{CHLD} = sub { $done = 1 }; + $p = IO::Poll->new; + $p->mask(STDOUT, POLLHUP); + $pid=fork; unless($pid) {setpgrp; exec $ENV{SHELL}, "-c", @ARGV; die "exec: $!\n"} + $p->poll; + kill SIGHUP, -${pid} unless $done; + wait; exit ($?&127 ? 128+($?&127) : 1+$?>>8) + } . "' "; + $signal_script =~ s/\s+/ /g; + + $self->{'sshlogin_wrap'} = + ($pre + . "$sshcmd $ssh_options $serverlogin $parallel_env " + . $remote_pre +# . ::shell_quote_scalar($signal_script . ::shell_quote_scalar($command)) + . ::shell_quote_scalar($command) + . ";" + . $post); + } + } + return $self->{'sshlogin_wrap'}; +} + +sub transfer { + # Files to transfer + # Returns: + # @transfer - File names of files to transfer + my $self = shift; + my @transfer = (); + $self->{'transfersize'} = 0; + if($opt::transfer) { + for my $record (@{$self->{'commandline'}{'arg_list'}}) { + # Merge arguments from records into args + for my $arg (@$record) { + CORE::push @transfer, $arg->orig(); + # filesize + if(-e $arg->orig()) { + $self->{'transfersize'} += (stat($arg->orig()))[7]; + } + } + } + } + return @transfer; +} + +sub transfersize { + my $self = shift; + return $self->{'transfersize'}; +} + +sub sshtransfer { + # Returns for each transfer file: + # rsync $file remote:$workdir + my $self = shift; + my @pre; + my $sshlogin = $self->sshlogin(); + my $workdir = $self->workdir(); + for my $file ($self->transfer()) { + push @pre, $sshlogin->rsync_transfer_cmd($file,$workdir).";"; + } + return join("",@pre); +} + +sub return { + # Files to return + # Non-quoted and with {...} substituted + # Returns: + # @non_quoted_filenames + my $self = shift; + return $self->{'commandline'}-> + replace_placeholders($self->{'commandline'}{'return_files'},0,0); +} + +sub returnsize { + # This is called after the job has finished + # Returns: + # $number_of_bytes transferred in return + my $self = shift; + for my $file ($self->return()) { + if(-e $file) { + $self->{'returnsize'} += (stat($file))[7]; + } + } + return $self->{'returnsize'}; +} + +sub sshreturn { + # Returns for each return-file: + # rsync remote:$workdir/$file . + my $self = shift; + my $sshlogin = $self->sshlogin(); + my $sshcmd = $sshlogin->sshcommand(); + my $serverlogin = $sshlogin->serverlogin(); + my $rsync_opt = "-rlDzR -e".::shell_quote_scalar($sshcmd); + my $pre = ""; + for my $file ($self->return()) { + $file =~ s:^\./::g; # Remove ./ if any + my $relpath = ($file !~ m:^/:); # Is the path relative? + my $cd = ""; + my $wd = ""; + if($relpath) { + # rsync -avR /foo/./bar/baz.c remote:/tmp/ + # == (on old systems) + # rsync -avR --rsync-path="cd /foo; rsync" remote:bar/baz.c /tmp/ + $wd = ::shell_quote_file($self->workdir()."/"); + } + # Only load File::Basename if actually needed + $Global::use{"File::Basename"} ||= eval "use File::Basename; 1;"; + # dir/./file means relative to dir, so remove dir on remote + $file =~ m:(.*)/\./:; + my $basedir = $1 ? ::shell_quote_file($1."/") : ""; + my $nobasedir = $file; + $nobasedir =~ s:.*/\./::; + $cd = ::shell_quote_file(::dirname($nobasedir)); + my $rsync_cd = '--rsync-path='.::shell_quote_scalar("cd $wd$cd; rsync"); + my $basename = ::shell_quote_scalar(::shell_quote_file(basename($file))); + # --return + # mkdir -p /home/tange/dir/subdir/; + # rsync (--protocol 30) -rlDzR --rsync-path="cd /home/tange/dir/subdir/; rsync" + # server:file.gz /home/tange/dir/subdir/ + $pre .= "mkdir -p $basedir$cd; ".$sshlogin->rsync()." $rsync_cd $rsync_opt $serverlogin:". + $basename . " ".$basedir.$cd.";"; + } + return $pre; +} + +sub sshcleanup { + # Return the sshcommand needed to remove the file + # Returns: + # ssh command needed to remove files from sshlogin + my $self = shift; + my $sshlogin = $self->sshlogin(); + my $sshcmd = $sshlogin->sshcommand(); + my $serverlogin = $sshlogin->serverlogin(); + my $workdir = $self->workdir(); + my $cleancmd = ""; + + for my $file ($self->cleanup()) { + my @subworkdirs = parentdirs_of($file); + $cleancmd .= $sshlogin->cleanup_cmd($file,$workdir).";"; + } + if(defined $opt::workdir and $opt::workdir eq "...") { + $cleancmd .= "$sshcmd $serverlogin rm -rf " . ::shell_quote_scalar($workdir).';'; + } + return $cleancmd; +} + +sub cleanup { + # Returns: + # Files to remove at cleanup + my $self = shift; + if($opt::cleanup) { + my @transfer = $self->transfer(); + my @return = $self->return(); + return (@transfer,@return); + } else { + return (); + } +} + +sub workdir { + # Returns: + # the workdir on a remote machine + my $self = shift; + if(not defined $self->{'workdir'}) { + my $workdir; + if(defined $opt::workdir) { + if($opt::workdir eq ".") { + # . means current dir + my $home = $ENV{'HOME'}; + eval 'use Cwd'; + my $cwd = cwd(); + $workdir = $cwd; + if($home) { + # If homedir exists: remove the homedir from + # workdir if cwd starts with homedir + # E.g. /home/foo/my/dir => my/dir + # E.g. /tmp/my/dir => /tmp/my/dir + my ($home_dev, $home_ino) = (stat($home))[0,1]; + my $parent = ""; + my @dir_parts = split(m:/:,$cwd); + my $part; + while(defined ($part = shift @dir_parts)) { + $part eq "" and next; + $parent .= "/".$part; + my ($parent_dev, $parent_ino) = (stat($parent))[0,1]; + if($parent_dev == $home_dev and $parent_ino == $home_ino) { + # dev and ino is the same: We found the homedir. + $workdir = join("/",@dir_parts); + last; + } + } + } + if($workdir eq "") { + $workdir = "."; + } + } elsif($opt::workdir eq "...") { + $workdir = ".parallel/tmp/" . ::hostname() . "-" . $$ + . "-" . $self->seq(); + } else { + $workdir = $opt::workdir; + # Rsync treats /./ special. We don't want that + $workdir =~ s:/\./:/:g; # Remove /./ + $workdir =~ s:/+$::; # Remove ending / if any + $workdir =~ s:^\./::g; # Remove starting ./ if any + } + } else { + $workdir = "."; + } + $self->{'workdir'} = ::shell_quote_scalar($workdir); + } + return $self->{'workdir'}; +} + +sub parentdirs_of { + # Return: + # all parentdirs except . of this dir or file - sorted desc by length + my $d = shift; + my @parents = (); + while($d =~ s:/[^/]+$::) { + if($d ne ".") { + push @parents, $d; + } + } + return @parents; +} + +sub start { + # Setup STDOUT and STDERR for a job and start it. + # Returns: + # job-object or undef if job not to run + my $job = shift; + # Get the shell command to be executed (possibly with ssh infront). + my $command = $job->wrapped(); + + if($Global::interactive or $Global::stderr_verbose) { + if($Global::interactive) { + print $Global::original_stderr "$command ?..."; + open(my $tty_fh, "<", "/dev/tty") || ::die_bug("interactive-tty"); + my $answer = <$tty_fh>; + close $tty_fh; + my $run_yes = ($answer =~ /^\s*y/i); + if (not $run_yes) { + $command = "true"; # Run the command 'true' + } + } else { + print $Global::original_stderr "$command\n"; + } + } + + my $pid; + $job->openoutputfiles(); + my($stdout_fh,$stderr_fh) = ($job->fh(1,"w"),$job->fh(2,"w")); + local (*IN,*OUT,*ERR); + open OUT, '>&', $stdout_fh or ::die_bug("Can't redirect STDOUT: $!"); + open ERR, '>&', $stderr_fh or ::die_bug("Can't dup STDOUT: $!"); + + if(($opt::dryrun or $Global::verbose) and $opt::ungroup) { + if($Global::verbose <= 1) { + print $stdout_fh $job->replaced(),"\n"; + } else { + # Verbose level > 1: Print the rsync and stuff + print $stdout_fh $command,"\n"; + } + } + if($opt::dryrun) { + $command = "true"; + } + $ENV{'PARALLEL_SEQ'} = $job->seq(); + $ENV{'PARALLEL_PID'} = $$; + ::debug("run", $Global::total_running, " processes . Starting (", + $job->seq(), "): $command\n"); + if($opt::pipe) { + my ($stdin_fh); + # The eval is needed to catch exception from open3 + eval { + $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", $Global::shell, "-c", $command) || + ::die_bug("open3-pipe"); + 1; + }; + $job->set_fh(0,"w",$stdin_fh); + } elsif(@opt::a and not $Global::stdin_in_opt_a and $job->seq() == 1 + and $job->sshlogin()->string() eq ":") { + # Give STDIN to the first job if using -a (but only if running + # locally - otherwise CTRL-C does not work for other jobs Bug#36585) + *IN = *STDIN; + # The eval is needed to catch exception from open3 + eval { + $pid = ::open3("<&IN", ">&OUT", ">&ERR", $Global::shell, "-c", $command) || + ::die_bug("open3-a"); + 1; + }; + # Re-open to avoid complaining + open(STDIN, "<&", $Global::original_stdin) + or ::die_bug("dup-\$Global::original_stdin: $!"); + } elsif ($opt::tty and not $Global::tty_taken and -c "/dev/tty" and + open(my $devtty_fh, "<", "/dev/tty")) { + # Give /dev/tty to the command if no one else is using it + *IN = $devtty_fh; + # The eval is needed to catch exception from open3 + eval { + $pid = ::open3("<&IN", ">&OUT", ">&ERR", $Global::shell, "-c", $command) || + ::die_bug("open3-/dev/tty"); + $Global::tty_taken = $pid; + close $devtty_fh; + 1; + }; + } else { + # The eval is needed to catch exception from open3 + eval { + $pid = ::open3(::gensym, ">&OUT", ">&ERR", $Global::shell, "-c", $command) || + ::die_bug("open3-gensym"); + 1; + }; + } + if($pid) { + # A job was started + $Global::total_running++; + $Global::total_started++; + $job->set_pid($pid); + $job->set_starttime(); + $Global::running{$job->pid()} = $job; + if($opt::timeout) { + $Global::timeoutq->insert($job); + } + $Global::newest_job = $job; + $Global::newest_starttime = ::now(); + return $job; + } else { + # No more processes + ::debug("run", "Cannot spawn more jobs.\n"); + return undef; + } +} + +sub tmux_wrap { + # Wrap command with tmux for session pPID + # Input: + # $actual_command = the actual command being run (incl ssh wrap) + my $self = shift; + my $actual_command = shift; + # Temporary file name. Used for fifo to communicate exit val + my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".tmx"); + $Global::unlink{$tmpfile}=1; + close $fh; + unlink $tmpfile; + my $visual_command = $self->replaced(); + my $title = $visual_command; + # ; causes problems + # ascii 194-245 annoys tmux + $title =~ tr/[\011-\016;\302-\365]//d; + + my $tmux; + if($Global::total_running == 0) { + $tmux = "tmux new-session -s p$$ -d -n ". + ::shell_quote_scalar($title); + print $Global::original_stderr "See output with: tmux attach -t p$$\n"; + } else { + $tmux = "tmux new-window -t p$$ -n ".::shell_quote_scalar($title); + } + return "mkfifo $tmpfile; $tmux ". + # Run in tmux + ::shell_quote_scalar( + "(".$actual_command.');(echo $?$status;echo 255) >'.$tmpfile."&". + "echo ".::shell_quote_scalar($visual_command).";". + "echo \007Job finished at: `date`;sleep 10"). + # Run outside tmux + # Read the first line from the fifo and use that as status code + "; exit `perl -ne 'unlink \$ARGV; 1..1 and print' $tmpfile` "; +} + +sub is_already_in_results { + # Do we already have results for this job? + # Returns: + # $job_already_run = bool whether there is output for this or not + my $job = $_[0]; + my $args_as_dirname = $job->{'commandline'}->args_as_dirname(); + # prefix/name1/val1/name2/val2/ + my $dir = $opt::results."/".$args_as_dirname; + ::debug("run", "Test $dir/stdout", -e "$dir/stdout", "\n"); + return -e "$dir/stdout"; +} + +sub is_already_in_joblog { + my $job = shift; + return vec($Global::job_already_run,$job->seq(),1); +} + +sub set_job_in_joblog { + my $job = shift; + vec($Global::job_already_run,$job->seq(),1) = 1; +} + +sub should_be_retried { + # Should this job be retried? + # Returns + # 0 - do not retry + # 1 - job queued for retry + my $self = shift; + if (not $opt::retries) { + return 0; + } + if(not $self->exitstatus()) { + # Completed with success. If there is a recorded failure: forget it + $self->reset_failed_here(); + return 0 + } else { + # The job failed. Should it be retried? + $self->add_failed_here(); + if($self->total_failed() == $opt::retries) { + # This has been retried enough + return 0; + } else { + # This command should be retried + $self->set_endtime(undef); + $Global::JobQueue->unget($self); + ::debug("run", "Retry ", $self->seq(), "\n"); + return 1; + } + } +} + +sub print { + # Print the output of the jobs + # Returns: N/A + + my $self = shift; + ::debug("print", ">>joboutput ", $self->replaced(), "\n"); + if($opt::dryrun) { + # Nothing was printed to this job: + # cleanup tmp files if --files was set + unlink $self->fh(1,"name"); + } + if($opt::pipe and $self->virgin()) { + # Skip --joblog, --dryrun, --verbose + } else { + if($Global::joblog and defined $self->{'exitstatus'}) { + # Add to joblog when finished + $self->print_joblog(); + } + + # Printing is only relevant for grouped/--line-buffer output. + $opt::ungroup and return; + # Check for disk full + exit_if_disk_full(); + + if(($opt::dryrun or $Global::verbose) + and + not $self->{'verbose_printed'}) { + $self->{'verbose_printed'}++; + if($Global::verbose <= 1) { + print STDOUT $self->replaced(),"\n"; + } else { + # Verbose level > 1: Print the rsync and stuff + print STDOUT $self->wrapped(),"\n"; + } + # If STDOUT and STDERR are merged, + # we want the command to be printed first + # so flush to avoid STDOUT being buffered + flush STDOUT; + } + } + for my $fdno (sort { $a <=> $b } keys %Global::fd) { + # Sort by file descriptor numerically: 1,2,3,..,9,10,11 + $fdno == 0 and next; + my $out_fd = $Global::fd{$fdno}; + my $in_fh = $self->fh($fdno,"r"); + if(not $in_fh) { + if(not $Job::file_descriptor_warning_printed{$fdno}++) { + # ::warning("File descriptor $fdno not defined\n"); + } + next; + } + ::debug("print", "File descriptor $fdno (", $self->fh($fdno,"name"), "):"); + if($opt::files) { + # If --compress: $in_fh must be closed first. + close $self->fh($fdno,"w"); + close $in_fh; + if($opt::pipe and $self->virgin()) { + # Nothing was printed to this job: + # cleanup unused tmp files if --files was set + for my $fdno (1,2) { + unlink $self->fh($fdno,"name"); + unlink $self->fh($fdno,"unlink"); + } + } elsif($fdno == 1 and $self->fh($fdno,"name")) { + print $out_fd $self->fh($fdno,"name"),"\n"; + } + } elsif($opt::linebuffer) { + # Line buffered print out + $self->linebuffer_print($fdno,$in_fh,$out_fd); + } else { + my $buf; + close $self->fh($fdno,"w"); + seek $in_fh, 0, 0; + # $in_fh is now ready for reading at position 0 + if($opt::tag or defined $opt::tagstring) { + my $tag = $self->tag(); + if($fdno == 2) { + # OpenSSH_3.6.1p2 gives 'tcgetattr: Invalid argument' with -tt + # This is a crappy way of ignoring it. + while(<$in_fh>) { + if(/^(client_process_control: )?tcgetattr: Invalid argument\n/) { + # Skip + } else { + print $out_fd $tag,$_; + } + # At most run the loop once + last; + } + } + while(<$in_fh>) { + print $out_fd $tag,$_; + } + } else { + my $buf; + if($fdno == 2) { + # OpenSSH_3.6.1p2 gives 'tcgetattr: Invalid argument' with -tt + # This is a crappy way of ignoring it. + sysread($in_fh,$buf,1_000); + $buf =~ s/^(client_process_control: )?tcgetattr: Invalid argument\n//; + print $out_fd $buf; + } + while(sysread($in_fh,$buf,32768)) { + print $out_fd $buf; + } + } + close $in_fh; + } + flush $out_fd; + } + ::debug("print", "<{'partial_line',$fdno}; + + if(defined $self->{'exitstatus'}) { + # If the job is dead: close printing fh. Needed for --compress + close $self->fh($fdno,"w"); + if($opt::compress) { + # Blocked reading in final round + $Global::use{"Fcntl"} ||= eval "use Fcntl qw(:DEFAULT :flock); 1;"; + for my $fdno (1,2) { + my $fdr = $self->fh($fdno,'r'); + my $flags; + fcntl($fdr, &F_GETFL, $flags) || die $!; # Get the current flags on the filehandle + $flags &= ~&O_NONBLOCK; # Remove non-blocking to the flags + fcntl($fdr, &F_SETFL, $flags) || die $!; # Set the flags on the filehandle + } + } + } + # This seek will clear EOF + seek $in_fh, tell($in_fh), 0; + # The read is non-blocking: The $in_fh is set to non-blocking. + # 32768 --tag = 5.1s + # 327680 --tag = 4.4s + # 1024000 --tag = 4.4s + # 3276800 --tag = 4.3s + # 32768000 --tag = 4.7s + # 10240000 --tag = 4.3s + while(read($in_fh,substr($$partial,length $$partial),3276800)) { + # Append to $$partial + # Find the last \n + my $i = rindex($$partial,"\n"); + if($i != -1) { + # One or more complete lines were found + if($fdno == 2 and not $self->{'printed_first_line',$fdno}++) { + # OpenSSH_3.6.1p2 gives 'tcgetattr: Invalid argument' with -tt + # This is a crappy way of ignoring it. + $$partial =~ s/^(client_process_control: )?tcgetattr: Invalid argument\n//; + # Length of partial line has changed: Find the last \n again + $i = rindex($$partial,"\n"); + } + if($opt::tag or defined $opt::tagstring) { + # Replace ^ with $tag within the full line + my $tag = $self->tag(); + substr($$partial,0,$i+1) =~ s/^/$tag/gm; + # Length of partial line has changed: Find the last \n again + $i = rindex($$partial,"\n"); + } + # Print up to and including the last \n + print $out_fd substr($$partial,0,$i+1); + # Remove the printed part + substr($$partial,0,$i+1)=""; + } + } + if(defined $self->{'exitstatus'}) { + # If the job is dead: print the remaining partial line + # read remaining + if($$partial and ($opt::tag or defined $opt::tagstring)) { + my $tag = $self->tag(); + $$partial =~ s/^/$tag/gm; + } + print $out_fd $$partial; + # Release the memory + $$partial = undef; + if($self->fh($fdno,"rpid") and CORE::kill 0, $self->fh($fdno,"rpid")) { + # decompress still running + } else { + # decompress done: close fh + close $in_fh; + } + } +} + +sub print_joblog { + my $self = shift; + my $cmd; + if($Global::verbose <= 1) { + $cmd = $self->replaced(); + } else { + # Verbose level > 1: Print the rsync and stuff + $cmd = "@command"; + } + print $Global::joblog + join("\t", $self->seq(), $self->sshlogin()->string(), + $self->starttime(), sprintf("%10.3f",$self->runtime()), + $self->transfersize(), $self->returnsize(), + $self->exitstatus(), $self->exitsignal(), $cmd + ). "\n"; + flush $Global::joblog; + $self->set_job_in_joblog(); +} + +sub tag { + my $self = shift; + if(not defined $self->{'tag'}) { + $self->{'tag'} = $self->{'commandline'}-> + replace_placeholders([$opt::tagstring],0,0)."\t"; + } + return $self->{'tag'}; +} + +sub hostgroups { + my $self = shift; + if(not defined $self->{'hostgroups'}) { + $self->{'hostgroups'} = $self->{'commandline'}->{'arg_list'}[0][0]->{'hostgroups'}; + } + return @{$self->{'hostgroups'}}; +} + +sub exitstatus { + my $self = shift; + return $self->{'exitstatus'}; +} + +sub set_exitstatus { + my $self = shift; + my $exitstatus = shift; + if($exitstatus) { + # Overwrite status if non-zero + $self->{'exitstatus'} = $exitstatus; + } else { + # Set status but do not overwrite + # Status may have been set by --timeout + $self->{'exitstatus'} ||= $exitstatus; + } +} + +sub exitsignal { + my $self = shift; + return $self->{'exitsignal'}; +} + +sub set_exitsignal { + my $self = shift; + my $exitsignal = shift; + $self->{'exitsignal'} = $exitsignal; +} + +{ + my ($disk_full_fh, $b8193, $name); + sub exit_if_disk_full { + # Checks if $TMPDIR is full by writing 8kb to a tmpfile + # If the disk is full: Exit immediately. + # Returns: + # N/A + if(not $disk_full_fh) { + ($disk_full_fh, $name) = ::tmpfile(SUFFIX => ".df"); + unlink $name; + $b8193 = "x"x8193; + } + # Linux does not discover if a disk is full if writing <= 8192 + # Tested on: + # bfs btrfs cramfs ext2 ext3 ext4 ext4dev jffs2 jfs minix msdos + # ntfs reiserfs tmpfs ubifs vfat xfs + # TODO this should be tested on different OS similar to this: + # + # doit() { + # sudo mount /dev/ram0 /mnt/loop; sudo chmod 1777 /mnt/loop + # seq 100000 | parallel --tmpdir /mnt/loop/ true & + # seq 6900000 > /mnt/loop/i && echo seq OK + # seq 6980868 > /mnt/loop/i + # seq 10000 > /mnt/loop/ii + # sleep 3 + # sudo umount /mnt/loop/ || sudo umount -l /mnt/loop/ + # echo >&2 + # } + print $disk_full_fh $b8193; + if(not $disk_full_fh + or + tell $disk_full_fh == 0) { + ::error("Output is incomplete. Cannot append to buffer file in $ENV{'TMPDIR'}. Is the disk full?\n"); + ::error("Change \$TMPDIR with --tmpdir or use --compress.\n"); + ::wait_and_exit(255); + } + truncate $disk_full_fh, 0; + seek($disk_full_fh, 0, 0) || die; + } +} + + +package CommandLine; + +sub new { + my $class = shift; + my $seq = shift; + my $commandref = shift; + $commandref || die; + my $arg_queue = shift; + my $context_replace = shift; + my $max_number_of_args = shift; # for -N and normal (-n1) + my $return_files = shift; + my $replacecount_ref = shift; + my $len_ref = shift; + my %replacecount = %$replacecount_ref; + my %len = %$len_ref; + for (keys %$replacecount_ref) { + # Total length of this replacement string {} replaced with all args + $len{$_} = 0; + } + return bless { + 'command' => $commandref, + 'seq' => $seq, + 'len' => \%len, + 'arg_list' => [], + 'arg_queue' => $arg_queue, + 'max_number_of_args' => $max_number_of_args, + 'replacecount' => \%replacecount, + 'context_replace' => $context_replace, + 'return_files' => $return_files, + 'replaced' => undef, + }, ref($class) || $class; +} + +sub seq { + my $self = shift; + return $self->{'seq'}; +} + +{ + my $max_slot_number; + + sub slot { + # Find the number of a free job slot and return it + # Uses: + # @Global::slots + # Returns: + # $jobslot = number of jobslot + my $self = shift; + if(not $self->{'slot'}) { + if(not @Global::slots) { + # $Global::max_slot_number will typically be $Global::max_jobs_running + push @Global::slots, ++$max_slot_number; + } + $self->{'slot'} = shift @Global::slots; + } + return $self->{'slot'}; + } +} + +sub populate { + # Add arguments from arg_queue until the number of arguments or + # max line length is reached + # Uses: + # $Global::minimal_command_line_length + # $opt::cat + # $opt::fifo + # $Global::JobQueue + # $opt::m + # $opt::X + # $CommandLine::already_spread + # $Global::max_jobs_running + # Returns: N/A + my $self = shift; + my $next_arg; + my $max_len = $Global::minimal_command_line_length || Limits::Command::max_length(); + + if($opt::cat or $opt::fifo) { + # Generate a tempfile name that will be used as {} + my($outfh,$name) = ::tmpfile(SUFFIX => ".pip"); + close $outfh; + # Unlink is needed if: ssh otheruser@localhost + unlink $name; + $Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->unget([Arg->new($name)]); + } + + while (not $self->{'arg_queue'}->empty()) { + $next_arg = $self->{'arg_queue'}->get(); + if(not defined $next_arg) { + next; + } + $self->push($next_arg); + if($self->len() >= $max_len) { + # Command length is now > max_length + # If there are arguments: remove the last + # If there are no arguments: Error + # TODO stuff about -x opt_x + if($self->number_of_args() > 1) { + # There is something to work on + $self->{'arg_queue'}->unget($self->pop()); + last; + } else { + my $args = join(" ", map { $_->orig() } @$next_arg); + ::error("Command line too long (", + $self->len(), " >= ", + $max_len, + ") at number ", + $self->{'arg_queue'}->arg_number(), + ": ". + (substr($args,0,50))."...\n"); + $self->{'arg_queue'}->unget($self->pop()); + ::wait_and_exit(255); + } + } + + if(defined $self->{'max_number_of_args'}) { + if($self->number_of_args() >= $self->{'max_number_of_args'}) { + last; + } + } + } + if(($opt::m or $opt::X) and not $CommandLine::already_spread + and $self->{'arg_queue'}->empty() and $Global::max_jobs_running) { + # -m or -X and EOF => Spread the arguments over all jobslots + # (unless they are already spread) + $CommandLine::already_spread ||= 1; + if($self->number_of_args() > 1) { + $self->{'max_number_of_args'} = + ::ceil($self->number_of_args()/$Global::max_jobs_running); + $Global::JobQueue->{'commandlinequeue'}->{'max_number_of_args'} = + $self->{'max_number_of_args'}; + $self->{'arg_queue'}->unget($self->pop_all()); + while($self->number_of_args() < $self->{'max_number_of_args'}) { + $self->push($self->{'arg_queue'}->get()); + } + } + } +} + +sub push { + # Add one or more records as arguments + # Returns: N/A + my $self = shift; + my $record = shift; + push @{$self->{'arg_list'}}, $record; + + my $quote_arg = $Global::noquote ? 0 : not $Global::quoting; + my $rep; + for my $arg (@$record) { + if(defined $arg) { + for my $perlexpr (keys %{$self->{'replacecount'}}) { + # 50% faster than below + $self->{'len'}{$perlexpr} += length $arg->replace($perlexpr,$quote_arg,$self); + # $rep = $arg->replace($perlexpr,$quote_arg,$self); + # $self->{'len'}{$perlexpr} += length $rep; + # ::debug("length", "Length: ", length $rep, + # "(", $perlexpr, "=>", $rep, ")\n"); + } + } + } +} + +sub pop { + # Remove last argument + # Returns: + # the last record + my $self = shift; + my $record = pop @{$self->{'arg_list'}}; + my $quote_arg = $Global::noquote ? 0 : not $Global::quoting; + for my $arg (@$record) { + if(defined $arg) { + for my $perlexpr (keys %{$self->{'replacecount'}}) { + $self->{'len'}{$perlexpr} -= + length $arg->replace($perlexpr,$quote_arg,$self); + } + } + } + return $record; +} + +sub pop_all { + # Remove all arguments and zeros the length of replacement strings + # Returns: + # all records + my $self = shift; + my @popped = @{$self->{'arg_list'}}; + for my $replacement_string (keys %{$self->{'replacecount'}}) { + $self->{'len'}{$replacement_string} = 0; + } + $self->{'arg_list'} = []; + return @popped; +} + +sub number_of_args { + # The number of records + # Returns: + # number of records + my $self = shift; + # Ftq rudef oaawuq ime dqxqmeqp az 2011-01-24 mzp ime iaz nk MQhmd + # Mdzrvadp Nvmdymeaz az 2011-04-10. Ftue oaawuq dqxqmeqp az + # 2013-08-18 ue m nuf tmdpqd me kag tmhq fa geq daf14. Bxqmeq + # qymux oaawuq@fmzsq.pw itqz kag dqmp ftue. + # + # U my ftq ymuzfmuzqd ar m buqoq ar rdqq earfimdq omxxqp SZG + # Bmdmxxqx. Rdqq earfimdq sgmdmzfqqe kag mooqee fa ftq eagdoq + # oapq, ngf U tmhq nqqz iazpqduzs tai ymzk mofgmxxk _dqmp_ ftq + # eagdoq oapq. + # + # Fa fqef ftue U bgf uz m oayyqzf fqxxuzs bqabxq fa qymux yq itqz + # ftqk dqmp ftue. Ftq oayyqzf ime bgf uz m eqofuaz ar ftq oapq + # ftmf za azq iagxp xaaw fa ruj ad uybdahq ftq earfimdq - ea ftq + # eagdoq oapq qcguhmxqzf fa m pgefk oadzqd. Fa ymwq egdq ftq + # oayyqzf iagxp zaf etai gb ur eayq azq vgef sdqbbqp ftdagst ftq + # eagdoq oapq U daf13'qp ftq eagdoq oapq + # tffb://qz.iuwubqpum.ads/iuwu/DAF13 + # + # 2.5 yazfte xmfqd U dqoquhqp mz qymux rday eayqazq ita zaf azxk + # ymzmsqp fa ruzp ftq oayyqzf, ngf mxea ymzmsqp fa sgqee ftq oapq + # tmp fa nq daf13'qp. + # + # Ftue nduzse yq fa ftq oazoxgeuaz ftmf ftqdq _mdq_ bqabxq, ita + # mdq zaf mrruxumfqp iuft ftq bdavqof, ftmf iuxx dqmp ftq eagdoq + # oapq - ftagst uf ymk zaf tmbbqz hqdk arfqz. + # + # This is really the number of records + return $#{$self->{'arg_list'}}+1; +} + +sub number_of_recargs { + # The number of args in records + # Returns: + # number of args records + my $self = shift; + my $sum = 0; + my $nrec = scalar @{$self->{'arg_list'}}; + if($nrec) { + $sum = $nrec * (scalar @{$self->{'arg_list'}[0]}); + } + return $sum; +} + +sub args_as_string { + # Returns: + # all unmodified arguments joined with ' ' (similar to {}) + my $self = shift; + return (join " ", map { $_->orig() } + map { @$_ } @{$self->{'arg_list'}}); +} + +sub args_as_dirname { + # Returns: + # all unmodified arguments joined with '/' (similar to {}) + # \t \0 \\ and / are quoted as: \t \0 \\ \_ + # If $Global::max_file_length: Keep subdirs < $Global::max_file_length + my $self = shift; + my @res = (); + + for my $rec_ref (@{$self->{'arg_list'}}) { + # If headers are used, sort by them. + # Otherwise keep the order from the command line. + my @header_indexes_sorted = header_indexes_sorted($#$rec_ref+1); + for my $n (@header_indexes_sorted) { + CORE::push(@res, + $Global::input_source_header{$n}, + map { my $s = $_; + # \t \0 \\ and / are quoted as: \t \0 \\ \_ + $s =~ s/\\/\\\\/g; + $s =~ s/\t/\\t/g; + $s =~ s/\0/\\0/g; + $s =~ s:/:\\_:g; + if($Global::max_file_length) { + # Keep each subdir shorter than the longest + # allowed file name + $s = substr($s,0,$Global::max_file_length); + } + $s; } + $rec_ref->[$n-1]->orig()); + } + } + return join "/", @res; +} + +sub header_indexes_sorted { + # Sort headers first by number then by name. + # E.g.: 1a 1b 11a 11b + # Returns: + # Indexes of %Global::input_source_header sorted + my $max_col = shift; + + no warnings 'numeric'; + for my $col (1 .. $max_col) { + # Make sure the header is defined. If it is not: use column number + if(not defined $Global::input_source_header{$col}) { + $Global::input_source_header{$col} = $col; + } + } + my @header_indexes_sorted = sort { + # Sort headers numerically then asciibetically + $Global::input_source_header{$a} <=> $Global::input_source_header{$b} + or + $Global::input_source_header{$a} cmp $Global::input_source_header{$b} + } 1 .. $max_col; + return @header_indexes_sorted; +} + +sub len { + # Uses: + # $opt::shellquote + # The length of the command line with args substituted + my $self = shift; + my $len = 0; + # Add length of the original command with no args + # Length of command w/ all replacement args removed + $len += $self->{'len'}{'noncontext'} + @{$self->{'command'}} -1; + ::debug("length", "noncontext + command: $len\n"); + my $recargs = $self->number_of_recargs(); + if($self->{'context_replace'}) { + # Context is duplicated for each arg + $len += $recargs * $self->{'len'}{'context'}; + for my $replstring (keys %{$self->{'replacecount'}}) { + # If the replacements string is more than once: mulitply its length + $len += $self->{'len'}{$replstring} * + $self->{'replacecount'}{$replstring}; + ::debug("length", $replstring, " ", $self->{'len'}{$replstring}, "*", + $self->{'replacecount'}{$replstring}, "\n"); + } + # echo 11 22 33 44 55 66 77 88 99 1010 + # echo 1 2 3 4 5 6 7 8 9 10 1 2 3 4 5 6 7 8 9 10 + # 5 + ctxgrp*arg + ::debug("length", "Ctxgrp: ", $self->{'len'}{'contextgroups'}, + " Groups: ", $self->{'len'}{'noncontextgroups'}, "\n"); + # Add space between context groups + $len += ($recargs-1) * ($self->{'len'}{'contextgroups'}); + } else { + # Each replacement string may occur several times + # Add the length for each time + $len += 1*$self->{'len'}{'context'}; + ::debug("length", "context+noncontext + command: $len\n"); + for my $replstring (keys %{$self->{'replacecount'}}) { + # (space between regargs + length of replacement) + # * number this replacement is used + $len += ($recargs -1 + $self->{'len'}{$replstring}) * + $self->{'replacecount'}{$replstring}; + } + } + if($opt::nice) { + # Pessimistic length if --nice is set + # Worse than worst case: every char needs to be quoted with \ + $len *= 2; + } + if($Global::quoting) { + # Pessimistic length if -q is set + # Worse than worst case: every char needs to be quoted with \ + $len *= 2; + } + if($opt::shellquote) { + # Pessimistic length if --shellquote is set + # Worse than worst case: every char needs to be quoted with \ twice + $len *= 4; + } + # If we are using --env, add the prefix for that, too. + $len += $Global::envvarlen; + + return $len; +} + +sub replaced { + # Uses: + # $Global::noquote + # $Global::quoting + # Returns: + # $replaced = command with place holders replaced and prepended + my $self = shift; + if(not defined $self->{'replaced'}) { + # Don't quote arguments if the input is the full command line + my $quote_arg = $Global::noquote ? 0 : not $Global::quoting; + $self->{'replaced'} = $self->replace_placeholders($self->{'command'},$Global::quoting,$quote_arg); + my $len = length $self->{'replaced'}; + if ($len != $self->len()) { + ::debug("length", $len, " != ", $self->len(), " ", $self->{'replaced'}, "\n"); + } else { + ::debug("length", $len, " == ", $self->len(), " ", $self->{'replaced'}, "\n"); + } + } + return $self->{'replaced'}; +} + +sub replace_placeholders { + # Replace foo{}bar with fooargbar + # Input: + # $targetref = command as shell words + # $quote = should everything be quoted? + # $quote_arg = should replaced arguments be quoted? + # Returns: + # @target with placeholders replaced + my $self = shift; + my $targetref = shift; + my $quote = shift; + my $quote_arg = shift; + my $context_replace = $self->{'context_replace'}; + my @target = @$targetref; + ::debug("replace", "Replace @target\n"); + # -X = context replace + # maybe multiple input sources + # maybe --xapply + if(not @target) { + # @target is empty: Return empty array + return @target; + } + # Fish out the words that have replacement strings in them + my %word; + for (@target) { + my $tt = $_; + ::debug("replace", "Target: $tt"); + # a{1}b{}c{}d + # a{=1 $_=$_ =}b{= $_=$_ =}c{= $_=$_ =}d + # a\257<1 $_=$_ \257>b\257< $_=$_ \257>c\257< $_=$_ \257>d + # A B C => aAbA B CcA B Cd + # -X A B C => aAbAcAd aAbBcBd aAbCcCd + + if($context_replace) { + while($tt =~ s/([^\s\257]* # before {= + (?: + \257< # {= + [^\257]*? # The perl expression + \257> # =} + [^\s\257]* # after =} + )+)/ /x) { + # $1 = pre \257 perlexpr \257 post + $word{"$1"} ||= 1; + } + } else { + while($tt =~ s/( (?: \257<([^\257]*?)\257>) )//x) { + # $f = \257 perlexpr \257 + $word{$1} ||= 1; + } + } + } + my @word = keys %word; + + my %replace; + my @arg; + for my $record (@{$self->{'arg_list'}}) { + # $self->{'arg_list'} = [ [Arg11, Arg12], [Arg21, Arg22], [Arg31, Arg32] ] + # Merge arg-objects from records into @arg for easy access + CORE::push @arg, @$record; + } + # Add one arg if empty to allow {#} and {%} to be computed only once + if(not @arg) { @arg = (Arg->new("")); } + # Number of arguments - used for positional arguments + my $n = $#_+1; + + # This is actually a CommandLine-object, + # but it looks nice to be able to say {= $job->slot() =} + my $job = $self; + for my $word (@word) { + # word = AB \257< perlexpr \257> CD \257< perlexpr \257> EF + my $w = $word; + ::debug("replace", "Replacing in $w\n"); + + # Replace positional arguments + $w =~ s< ([^\s\257]*) # before {= + \257< # {= + (-?\d+) # Position (eg. -2 or 3) + ([^\257]*?) # The perl expression + \257> # =} + ([^\s\257]*) # after =} + > + { $1. # Context (pre) + ( + $arg[$2 > 0 ? $2-1 : $n+$2] ? # If defined: replace + $arg[$2 > 0 ? $2-1 : $n+$2]->replace($3,$quote_arg,$self) + : "") + .$4 }egx;# Context (post) + ::debug("replace", "Positional replaced $word with: $w\n"); + + if($w !~ /\257/) { + # No more replacement strings in $w: No need to do more + if($quote) { + CORE::push(@{$replace{::shell_quote($word)}}, $w); + } else { + CORE::push(@{$replace{$word}}, $w); + } + next; + } + # for each arg: + # compute replacement for each string + # replace replacement strings with replacement in the word value + # push to replace word value + ::debug("replace", "Positional done: $w\n"); + for my $arg (@arg) { + my $val = $w; + my $number_of_replacements = 0; + for my $perlexpr (keys %{$self->{'replacecount'}}) { + # Replace {= perl expr =} with value for each arg + $number_of_replacements += + $val =~ s{\257<\Q$perlexpr\E\257>} + {$arg ? $arg->replace($perlexpr,$quote_arg,$self) : ""}eg; + } + my $ww = $word; + if($quote) { + $ww = ::shell_quote_scalar($word); + $val = ::shell_quote_scalar($val); + } + if($number_of_replacements) { + CORE::push(@{$replace{$ww}}, $val); + } + } + } + + if($quote) { + @target = ::shell_quote(@target); + } + # ::debug("replace", "%replace=",::my_dump(%replace),"\n"); + if(%replace) { + # Substitute the replace strings with the replacement values + # Must be sorted by length if a short word is a substring of a long word + my $regexp = join('|', map { my $s = $_; $s =~ s/(\W)/\\$1/g; $s } + sort { length $b <=> length $a } keys %replace); + for(@target) { + s/($regexp)/join(" ",@{$replace{$1}})/ge; + } + } + ::debug("replace", "Return @target\n"); + return wantarray ? @target : "@target"; +} + + +package CommandLineQueue; + +sub new { + my $class = shift; + my $commandref = shift; + my $read_from = shift; + my $context_replace = shift; + my $max_number_of_args = shift; + my $return_files = shift; + my @unget = (); + my ($count,%replacecount,$posrpl,$perlexpr,%len); + my @command = @$commandref; + # If the first command start with '-' it is probably an option + if($command[0] =~ /^\s*(-\S+)/) { + # Is this really a command in $PATH starting with '-'? + my $cmd = $1; + if(not ::which($cmd)) { + ::error("Command ($cmd) starts with '-'. Is this a wrong option?\n"); + ::wait_and_exit(255); + } + } + # Replace replacement strings with {= perl expr =} + # Protect matching inside {= perl expr =} + # by replacing {= and =} with \257< and \257> + for(@command) { + if(/\257/) { + ::error("Command cannot contain the character \257. Use a function for that.\n"); + ::wait_and_exit(255); + } + s/\Q$Global::parensleft\E(.*?)\Q$Global::parensright\E/\257<$1\257>/gx; + } + for my $rpl (keys %Global::rpl) { + # Replace the short hand string with the {= perl expr =} in $command and $opt::tagstring + # Avoid replacing inside existing {= perl expr =} + for(@command,@Global::ret_files) { + while(s/((^|\257>)[^\257]*?) # Don't replace after \257 unless \257> + \Q$rpl\E/$1\257<$Global::rpl{$rpl}\257>/xg) { + } + } + if(defined $opt::tagstring) { + for($opt::tagstring) { + while(s/((^|\257>)[^\257]*?) # Don't replace after \257 unless \257> + \Q$rpl\E/$1\257<$Global::rpl{$rpl}\257>/x) {} + } + } + # Do the same for the positional replacement strings + # A bit harder as we have to put in the position number + $posrpl = $rpl; + if($posrpl =~ s/^\{//) { + # Only do this if the shorthand start with { + for(@command,@Global::ret_files) { + s/\{(-?\d+)\Q$posrpl\E/\257<$1 $Global::rpl{$rpl}\257>/g; + } + if(defined $opt::tagstring) { + $opt::tagstring =~ s/\{(-?\d+)\Q$posrpl\E/\257<$1 $perlexpr\257>/g; + } + } + } + my $sum = 0; + while($sum == 0) { + # Count how many times each replacement string is used + my @cmd = @command; + my $contextlen = 0; + my $noncontextlen = 0; + my $contextgroups = 0; + for my $c (@cmd) { + while($c =~ s/ \257<([^\257]*?)\257> /\000/x) { + # %replacecount = { "perlexpr" => number of times seen } + # e.g { "$_++" => 2 } + $replacecount{$1} ++; + $sum++; + } + # Measure the length of the context around the {= perl expr =} + # Use that {=...=} has been replaced with \000 above + # So there is no need to deal with \257< + while($c =~ s/ (\S*\000\S*) //x) { + my $w = $1; + $w =~ tr/\000//d; # Remove all \000's + $contextlen += length($w); + $contextgroups++; + } + # All {= perl expr =} have been removed: The rest is non-context + $noncontextlen += length $c; + } + if($opt::tagstring) { + my $t = $opt::tagstring; + while($t =~ s/ \257<([^\257]*)\257> //x) { + # %replacecount = { "perlexpr" => number of times seen } + # e.g { "$_++" => 2 } + # But for tagstring we just need to mark it as seen + $replacecount{$1}||=1; + } + } + + $len{'context'} = 0+$contextlen; + $len{'noncontext'} = $noncontextlen; + $len{'contextgroups'} = $contextgroups; + $len{'noncontextgroups'} = @cmd-$contextgroups; + ::debug("length", "@command Context: ", $len{'context'}, + " Non: ", $len{'noncontext'}, " Ctxgrp: ", $len{'contextgroups'}, + " NonCtxGrp: ", $len{'noncontextgroups'}, "\n"); + if($sum == 0) { + # Default command = {} + # If not replacement string: append {} + if(not @command) { + @command = ("\257<\257>"); + $Global::noquote = 1; + } elsif(($opt::pipe or $opt::pipepart) + and not $opt::fifo and not $opt::cat) { + # With --pipe / --pipe-part you can have no replacement + last; + } else { + # Append {} to the command if there are no {...}'s and no {=...=} + push @command, ("\257<\257>"); + } + } + } + + return bless { + 'unget' => \@unget, + 'command' => \@command, + 'replacecount' => \%replacecount, + 'arg_queue' => RecordQueue->new($read_from,$opt::colsep), + 'context_replace' => $context_replace, + 'len' => \%len, + 'max_number_of_args' => $max_number_of_args, + 'size' => undef, + 'return_files' => $return_files, + 'seq' => 1, + }, ref($class) || $class; +} + +sub get { + my $self = shift; + if(@{$self->{'unget'}}) { + my $cmd_line = shift @{$self->{'unget'}}; + return ($cmd_line); + } else { + my $cmd_line; + $cmd_line = CommandLine->new($self->seq(), + $self->{'command'}, + $self->{'arg_queue'}, + $self->{'context_replace'}, + $self->{'max_number_of_args'}, + $self->{'return_files'}, + $self->{'replacecount'}, + $self->{'len'}, + ); + $cmd_line->populate(); + ::debug("init","cmd_line->number_of_args ", + $cmd_line->number_of_args(), "\n"); + if($opt::pipe or $opt::pipepart) { + if($cmd_line->replaced() eq "") { + # Empty command - pipe requires a command + ::error("--pipe must have a command to pipe into (e.g. 'cat').\n"); + ::wait_and_exit(255); + } + } else { + if($cmd_line->number_of_args() == 0) { + # We did not get more args - maybe at EOF string? + return undef; + } elsif($cmd_line->replaced() eq "") { + # Empty command - get the next instead + return $self->get(); + } + } + $self->set_seq($self->seq()+1); + return $cmd_line; + } +} + +sub unget { + my $self = shift; + unshift @{$self->{'unget'}}, @_; +} + +sub empty { + my $self = shift; + my $empty = (not @{$self->{'unget'}}) && $self->{'arg_queue'}->empty(); + ::debug("run", "CommandLineQueue->empty $empty"); + return $empty; +} + +sub seq { + my $self = shift; + return $self->{'seq'}; +} + +sub set_seq { + my $self = shift; + $self->{'seq'} = shift; +} + +sub quote_args { + my $self = shift; + # If there is not command emulate |bash + return $self->{'command'}; +} + +sub size { + my $self = shift; + if(not $self->{'size'}) { + my @all_lines = (); + while(not $self->{'arg_queue'}->empty()) { + push @all_lines, CommandLine->new($self->{'command'}, + $self->{'arg_queue'}, + $self->{'context_replace'}, + $self->{'max_number_of_args'}); + } + $self->{'size'} = @all_lines; + $self->unget(@all_lines); + } + return $self->{'size'}; +} + + +package Limits::Command; + +# Maximal command line length (for -m and -X) +sub max_length { + # Find the max_length of a command line and cache it + # Returns: + # number of chars on the longest command line allowed + if(not $Limits::Command::line_max_len) { + # Disk cache of max command line length + my $len_cache = $ENV{'HOME'} . "/.parallel/tmp/linelen-" . ::hostname(); + my $cached_limit; + if(-e $len_cache) { + open(my $fh, "<", $len_cache) || ::die_bug("Cannot read $len_cache"); + $cached_limit = <$fh>; + close $fh; + } else { + $cached_limit = real_max_length(); + # If $HOME is write protected: Do not fail + mkdir($ENV{'HOME'} . "/.parallel"); + mkdir($ENV{'HOME'} . "/.parallel/tmp"); + open(my $fh, ">", $len_cache); + print $fh $cached_limit; + close $fh; + } + $Limits::Command::line_max_len = $cached_limit; + if($opt::max_chars) { + if($opt::max_chars <= $cached_limit) { + $Limits::Command::line_max_len = $opt::max_chars; + } else { + ::warning("Value for -s option ", + "should be < $cached_limit.\n"); + } + } + } + return $Limits::Command::line_max_len; +} + +sub real_max_length { + # Find the max_length of a command line + # Returns: + # The maximal command line length + # Use an upper bound of 8 MB if the shell allows for for infinite long lengths + my $upper = 8_000_000; + my $len = 8; + do { + if($len > $upper) { return $len }; + $len *= 16; + } while (is_acceptable_command_line_length($len)); + # Then search for the actual max length between 0 and upper bound + return binary_find_max_length(int($len/16),$len); +} + +sub binary_find_max_length { + # Given a lower and upper bound find the max_length of a command line + # Returns: + # number of chars on the longest command line allowed + my ($lower, $upper) = (@_); + if($lower == $upper or $lower == $upper-1) { return $lower; } + my $middle = int (($upper-$lower)/2 + $lower); + ::debug("init", "Maxlen: $lower,$upper,$middle : "); + if (is_acceptable_command_line_length($middle)) { + return binary_find_max_length($middle,$upper); + } else { + return binary_find_max_length($lower,$middle); + } +} + +sub is_acceptable_command_line_length { + # Test if a command line of this length can run + # Returns: + # 0 if the command line length is too long + # 1 otherwise + my $len = shift; + + local *STDERR; + open (STDERR, ">", "/dev/null"); + system "true "."x"x$len; + close STDERR; + ::debug("init", "$len=$? "); + return not $?; +} + + +package RecordQueue; + +sub new { + my $class = shift; + my $fhs = shift; + my $colsep = shift; + my @unget = (); + my $arg_sub_queue; + if($colsep) { + # Open one file with colsep + $arg_sub_queue = RecordColQueue->new($fhs); + } else { + # Open one or more files if multiple -a + $arg_sub_queue = MultifileQueue->new($fhs); + } + return bless { + 'unget' => \@unget, + 'arg_number' => 0, + 'arg_sub_queue' => $arg_sub_queue, + }, ref($class) || $class; +} + +sub get { + # Returns: + # reference to array of Arg-objects + my $self = shift; + if(@{$self->{'unget'}}) { + $self->{'arg_number'}++; + return shift @{$self->{'unget'}}; + } + my $ret = $self->{'arg_sub_queue'}->get(); + if(defined $Global::max_number_of_args + and $Global::max_number_of_args == 0) { + ::debug("run", "Read 1 but return 0 args\n"); + return [Arg->new("")]; + } else { + return $ret; + } +} + +sub unget { + my $self = shift; + ::debug("run", "RecordQueue-unget '@_'\n"); + $self->{'arg_number'} -= @_; + unshift @{$self->{'unget'}}, @_; +} + +sub empty { + my $self = shift; + my $empty = not @{$self->{'unget'}}; + $empty &&= $self->{'arg_sub_queue'}->empty(); + ::debug("run", "RecordQueue->empty $empty"); + return $empty; +} + +sub arg_number { + my $self = shift; + return $self->{'arg_number'}; +} + + +package RecordColQueue; + +sub new { + my $class = shift; + my $fhs = shift; + my @unget = (); + my $arg_sub_queue = MultifileQueue->new($fhs); + return bless { + 'unget' => \@unget, + 'arg_sub_queue' => $arg_sub_queue, + }, ref($class) || $class; +} + +sub get { + # Returns: + # reference to array of Arg-objects + my $self = shift; + if(@{$self->{'unget'}}) { + return shift @{$self->{'unget'}}; + } + my $unget_ref=$self->{'unget'}; + if($self->{'arg_sub_queue'}->empty()) { + return undef; + } + my $in_record = $self->{'arg_sub_queue'}->get(); + if(defined $in_record) { + my @out_record = (); + for my $arg (@$in_record) { + ::debug("run", "RecordColQueue::arg $arg\n"); + my $line = $arg->orig(); + ::debug("run", "line='$line'\n"); + if($line ne "") { + for my $s (split /$opt::colsep/o, $line, -1) { + push @out_record, Arg->new($s); + } + } else { + push @out_record, Arg->new(""); + } + } + return \@out_record; + } else { + return undef; + } +} + +sub unget { + my $self = shift; + ::debug("run", "RecordColQueue-unget '@_'\n"); + unshift @{$self->{'unget'}}, @_; +} + +sub empty { + my $self = shift; + my $empty = (not @{$self->{'unget'}} and $self->{'arg_sub_queue'}->empty()); + ::debug("run", "RecordColQueue->empty $empty"); + return $empty; +} + + +package MultifileQueue; + +@Global::unget_argv=(); + +sub new { + my $class = shift; + my $fhs = shift; + for my $fh (@$fhs) { + if(-t $fh) { + ::warning("Input is read from the terminal. ". + "Only experts do this on purpose. ". + "Press CTRL-D to exit.\n"); + } + } + return bless { + 'unget' => \@Global::unget_argv, + 'fhs' => $fhs, + 'arg_matrix' => undef, + }, ref($class) || $class; +} + +sub get { + my $self = shift; + if($opt::xapply) { + return $self->xapply_get(); + } else { + return $self->nest_get(); + } +} + +sub unget { + my $self = shift; + ::debug("run", "MultifileQueue-unget '@_'\n"); + unshift @{$self->{'unget'}}, @_; +} + +sub empty { + my $self = shift; + my $empty = (not @Global::unget_argv + and not @{$self->{'unget'}}); + for my $fh (@{$self->{'fhs'}}) { + $empty &&= eof($fh); + } + ::debug("run", "MultifileQueue->empty $empty "); + return $empty; +} + +sub xapply_get { + my $self = shift; + if(@{$self->{'unget'}}) { + return shift @{$self->{'unget'}}; + } + my @record = (); + my $prepend = undef; + my $empty = 1; + for my $fh (@{$self->{'fhs'}}) { + my $arg = read_arg_from_fh($fh); + if(defined $arg) { + # Record $arg for recycling at end of file + push @{$self->{'arg_matrix'}{$fh}}, $arg; + push @record, $arg; + $empty = 0; + } else { + ::debug("run", "EOA "); + # End of file: Recycle arguments + push @{$self->{'arg_matrix'}{$fh}}, shift @{$self->{'arg_matrix'}{$fh}}; + # return last @{$args->{'args'}{$fh}}; + push @record, @{$self->{'arg_matrix'}{$fh}}[-1]; + } + } + if($empty) { + return undef; + } else { + return \@record; + } +} + +sub nest_get { + my $self = shift; + if(@{$self->{'unget'}}) { + return shift @{$self->{'unget'}}; + } + my @record = (); + my $prepend = undef; + my $empty = 1; + my $no_of_inputsources = $#{$self->{'fhs'}} + 1; + if(not $self->{'arg_matrix'}) { + # Initialize @arg_matrix with one arg from each file + # read one line from each file + my @first_arg_set; + my $all_empty = 1; + for (my $fhno = 0; $fhno < $no_of_inputsources ; $fhno++) { + my $arg = read_arg_from_fh($self->{'fhs'}[$fhno]); + if(defined $arg) { + $all_empty = 0; + } + $self->{'arg_matrix'}[$fhno][0] = $arg || Arg->new(""); + push @first_arg_set, $self->{'arg_matrix'}[$fhno][0]; + } + if($all_empty) { + # All filehandles were at eof or eof-string + return undef; + } + return [@first_arg_set]; + } + + # Treat the case with one input source special. For multiple + # input sources we need to remember all previously read values to + # generate all combinations. But for one input source we can + # forget the value after first use. + if($no_of_inputsources == 1) { + my $arg = read_arg_from_fh($self->{'fhs'}[0]); + if(defined($arg)) { + return [$arg]; + } + return undef; + } + for (my $fhno = $no_of_inputsources - 1; $fhno >= 0; $fhno--) { + if(eof($self->{'fhs'}[$fhno])) { + next; + } else { + # read one + my $arg = read_arg_from_fh($self->{'fhs'}[$fhno]); + defined($arg) || next; # If we just read an EOF string: Treat this as EOF + my $len = $#{$self->{'arg_matrix'}[$fhno]} + 1; + $self->{'arg_matrix'}[$fhno][$len] = $arg; + # make all new combinations + my @combarg = (); + for (my $fhn = 0; $fhn < $no_of_inputsources; $fhn++) { + push @combarg, [0, $#{$self->{'arg_matrix'}[$fhn]}]; + } + $combarg[$fhno] = [$len,$len]; # Find only combinations with this new entry + # map combinations + # [ 1, 3, 7 ], [ 2, 4, 1 ] + # => + # [ m[0][1], m[1][3], m[3][7] ], [ m[0][2], m[1][4], m[2][1] ] + my @mapped; + for my $c (expand_combinations(@combarg)) { + my @a; + for my $n (0 .. $no_of_inputsources - 1 ) { + push @a, $self->{'arg_matrix'}[$n][$$c[$n]]; + } + push @mapped, \@a; + } + # append the mapped to the ungotten arguments + push @{$self->{'unget'}}, @mapped; + # get the first + return shift @{$self->{'unget'}}; + } + } + # all are eof or at EOF string; return from the unget queue + return shift @{$self->{'unget'}}; +} + +sub read_arg_from_fh { + # Read one Arg from filehandle + # Returns: + # Arg-object with one read line + # undef if end of file + my $fh = shift; + my $prepend = undef; + my $arg; + do {{ + # This makes 10% faster + if(not ($arg = <$fh>)) { + if(defined $prepend) { + return Arg->new($prepend); + } else { + return undef; + } + } +# ::debug("run", "read $arg\n"); + # Remove delimiter + $arg =~ s:$/$::; + if($Global::end_of_file_string and + $arg eq $Global::end_of_file_string) { + # Ignore the rest of input file + close $fh; + ::debug("run", "EOF-string ($arg) met\n"); + if(defined $prepend) { + return Arg->new($prepend); + } else { + return undef; + } + } + if(defined $prepend) { + $arg = $prepend.$arg; # For line continuation + $prepend = undef; #undef; + } + if($Global::ignore_empty) { + if($arg =~ /^\s*$/) { + redo; # Try the next line + } + } + if($Global::max_lines) { + if($arg =~ /\s$/) { + # Trailing space => continued on next line + $prepend = $arg; + redo; + } + } + }} while (1 == 0); # Dummy loop {{}} for redo + if(defined $arg) { + return Arg->new($arg); + } else { + ::die_bug("multiread arg undefined"); + } +} + +sub expand_combinations { + # Input: + # ([xmin,xmax], [ymin,ymax], ...) + # Returns: ([x,y,...],[x,y,...]) + # where xmin <= x <= xmax and ymin <= y <= ymax + my $minmax_ref = shift; + my $xmin = $$minmax_ref[0]; + my $xmax = $$minmax_ref[1]; + my @p; + if(@_) { + # If there are more columns: Compute those recursively + my @rest = expand_combinations(@_); + for(my $x = $xmin; $x <= $xmax; $x++) { + push @p, map { [$x, @$_] } @rest; + } + } else { + for(my $x = $xmin; $x <= $xmax; $x++) { + push @p, [$x]; + } + } + return @p; +} + + +package Arg; + +sub new { + my $class = shift; + my $orig = shift; + my @hostgroups; + if($opt::hostgroups) { + if($orig =~ s:@(.+)::) { + # We found hostgroups on the arg + @hostgroups = split(/\+/, $1); + if(not grep { defined $Global::hostgroups{$_} } @hostgroups) { + ::warning("No such hostgroup (@hostgroups)\n"); + @hostgroups = (keys %Global::hostgroups); + } + } else { + @hostgroups = (keys %Global::hostgroups); + } + } + return bless { + 'orig' => $orig, + 'hostgroups' => \@hostgroups, + }, ref($class) || $class; +} + +sub replace { + # Calculates the corresponding value for a given perl expression + # Returns: + # The calculated string (quoted if asked for) + my $self = shift; + my $perlexpr = shift; # E.g. $_=$_ or s/.gz// + my $quote = (shift) ? 1 : 0; # should the string be quoted? + # This is actually a CommandLine-object, + # but it looks nice to be able to say {= $job->slot() =} + my $job = shift; + $perlexpr =~ s/^-?\d+ //; # Positional replace treated as normal replace + if(not defined $self->{"rpl",0,$perlexpr}) { + local $_; + if($Global::trim eq "n") { + $_ = $self->{'orig'}; + } else { + $_ = trim_of($self->{'orig'}); + } + ::debug("replace", "eval ", $perlexpr, " ", $_, "\n"); + if(not $Global::perleval{$perlexpr}) { + # Make an anonymous function of the $perlexpr + # And more importantly: Compile it only once + if($Global::perleval{$perlexpr} = + eval('sub { no strict; no warnings; my $job = shift; '. + $perlexpr.' }')) { + # All is good + } else { + # The eval failed. Maybe $perlexpr is invalid perl? + ::error("Cannot use $perlexpr: $@\n"); + ::wait_and_exit(255); + } + } + # Execute the function + $Global::perleval{$perlexpr}->($job); + $self->{"rpl",0,$perlexpr} = $_; + } + if(not defined $self->{"rpl",$quote,$perlexpr}) { + $self->{"rpl",1,$perlexpr} = + ::shell_quote_scalar($self->{"rpl",0,$perlexpr}); + } + return $self->{"rpl",$quote,$perlexpr}; +} + +sub orig { + my $self = shift; + return $self->{'orig'}; +} + +sub trim_of { + # Removes white space as specifed by --trim: + # n = nothing + # l = start + # r = end + # lr|rl = both + # Returns: + # string with white space removed as needed + my @strings = map { defined $_ ? $_ : "" } (@_); + my $arg; + if($Global::trim eq "n") { + # skip + } elsif($Global::trim eq "l") { + for my $arg (@strings) { $arg =~ s/^\s+//; } + } elsif($Global::trim eq "r") { + for my $arg (@strings) { $arg =~ s/\s+$//; } + } elsif($Global::trim eq "rl" or $Global::trim eq "lr") { + for my $arg (@strings) { $arg =~ s/^\s+//; $arg =~ s/\s+$//; } + } else { + ::error("--trim must be one of: r l rl lr.\n"); + ::wait_and_exit(255); + } + return wantarray ? @strings : "@strings"; +} + + +package TimeoutQueue; + +sub new { + my $class = shift; + my $delta_time = shift; + my ($pct); + if($delta_time =~ /(\d+(\.\d+)?)%/) { + # Timeout in percent + $pct = $1/100; + $delta_time = 1_000_000; + } + return bless { + 'queue' => [], + 'delta_time' => $delta_time, + 'pct' => $pct, + 'remedian_idx' => 0, + 'remedian_arr' => [], + 'remedian' => undef, + }, ref($class) || $class; +} + +sub delta_time { + my $self = shift; + return $self->{'delta_time'}; +} + +sub set_delta_time { + my $self = shift; + $self->{'delta_time'} = shift; +} + +sub remedian { + my $self = shift; + return $self->{'remedian'}; +} + +sub set_remedian { + # Set median of the last 999^3 (=997002999) values using Remedian + # + # Rousseeuw, Peter J., and Gilbert W. Bassett Jr. "The remedian: A + # robust averaging method for large data sets." Journal of the + # American Statistical Association 85.409 (1990): 97-104. + my $self = shift; + my $val = shift; + my $i = $self->{'remedian_idx'}++; + my $rref = $self->{'remedian_arr'}; + $rref->[0][$i%999] = $val; + $rref->[1][$i/999%999] = (sort @{$rref->[0]})[$#{$rref->[0]}/2]; + $rref->[2][$i/999/999%999] = (sort @{$rref->[1]})[$#{$rref->[1]}/2]; + $self->{'remedian'} = (sort @{$rref->[2]})[$#{$rref->[2]}/2]; +} + +sub update_delta_time { + # Update delta_time based on runtime of finished job if timeout is + # a percentage + my $self = shift; + my $runtime = shift; + if($self->{'pct'}) { + $self->set_remedian($runtime); + $self->{'delta_time'} = $self->{'pct'} * $self->remedian(); + ::debug("run", "Timeout: $self->{'delta_time'}s "); + } +} + +sub process_timeouts { + # Check if there was a timeout + my $self = shift; + # $self->{'queue'} is sorted by start time + while (@{$self->{'queue'}}) { + my $job = $self->{'queue'}[0]; + if($job->endtime()) { + # Job already finished. No need to timeout the job + # This could be because of --keep-order + shift @{$self->{'queue'}}; + } elsif($job->timedout($self->{'delta_time'})) { + # Need to shift off queue before kill + # because kill calls usleep that calls process_timeouts + shift @{$self->{'queue'}}; + $job->kill(); + } else { + # Because they are sorted by start time the rest are later + last; + } + } +} + +sub insert { + my $self = shift; + my $in = shift; + push @{$self->{'queue'}}, $in; +} + + +package Semaphore; + +# This package provides a counting semaphore +# +# If a process dies without releasing the semaphore the next process +# that needs that entry will clean up dead semaphores +# +# The semaphores are stored in ~/.parallel/semaphores/id- Each +# file in ~/.parallel/semaphores/id-/ is the process ID of the +# process holding the entry. If the process dies, the entry can be +# taken by another process. + +sub new { + my $class = shift; + my $id = shift; + my $count = shift; + $id=~s/([^-_a-z0-9])/unpack("H*",$1)/ige; # Convert non-word chars to hex + $id="id-".$id; # To distinguish it from a process id + my $parallel_dir = $ENV{'HOME'}."/.parallel"; + -d $parallel_dir or mkdir_or_die($parallel_dir); + my $parallel_locks = $parallel_dir."/semaphores"; + -d $parallel_locks or mkdir_or_die($parallel_locks); + my $lockdir = "$parallel_locks/$id"; + my $lockfile = $lockdir.".lock"; + if($count < 1) { ::die_bug("semaphore-count: $count"); } + return bless { + 'lockfile' => $lockfile, + 'lockfh' => Symbol::gensym(), + 'lockdir' => $lockdir, + 'id' => $id, + 'idfile' => $lockdir."/".$id, + 'pid' => $$, + 'pidfile' => $lockdir."/".$$.'@'.::hostname(), + 'count' => $count + 1 # nlinks returns a link for the 'id-' as well + }, ref($class) || $class; +} + +sub acquire { + my $self = shift; + my $sleep = 1; # 1 ms + my $start_time = time; + while(1) { + $self->atomic_link_if_count_less_than() and last; + ::debug("sem", "Remove dead locks"); + my $lockdir = $self->{'lockdir'}; + for my $d (glob "$lockdir/*") { + ::debug("sem", "Lock $d $lockdir\n"); + $d =~ m:$lockdir/([0-9]+)\@([-\._a-z0-9]+)$:o or next; + my ($pid, $host) = ($1, $2); + if($host eq ::hostname()) { + if(not kill 0, $1) { + ::debug("sem", "Dead: $d"); + unlink $d; + } else { + ::debug("sem", "Alive: $d"); + } + } + } + # try again + $self->atomic_link_if_count_less_than() and last; + # Retry slower and slower up to 1 second + $sleep = ($sleep < 1000) ? ($sleep * 1.1) : ($sleep); + # Random to avoid every sleeping job waking up at the same time + ::usleep(rand()*$sleep); + if(defined($opt::timeout) and + $start_time + $opt::timeout > time) { + # Acquire the lock anyway + if(not -e $self->{'idfile'}) { + open (my $fh, ">", $self->{'idfile'}) or + ::die_bug("timeout_write_idfile: $self->{'idfile'}"); + close $fh; + } + link $self->{'idfile'}, $self->{'pidfile'}; + last; + } + } + ::debug("sem", "acquired $self->{'pid'}\n"); +} + +sub release { + my $self = shift; + unlink $self->{'pidfile'}; + if($self->nlinks() == 1) { + # This is the last link, so atomic cleanup + $self->lock(); + if($self->nlinks() == 1) { + unlink $self->{'idfile'}; + rmdir $self->{'lockdir'}; + } + $self->unlock(); + } + ::debug("run", "released $self->{'pid'}\n"); +} + +sub _release { + my $self = shift; + + unlink $self->{'pidfile'}; + $self->lock(); + my $nlinks = $self->nlinks(); + ::debug("sem", $nlinks, "<", $self->{'count'}); + if($nlinks-- > 1) { + unlink $self->{'idfile'}; + open (my $fh, ">", $self->{'idfile'}) or + ::die_bug("write_idfile: $self->{'idfile'}"); + print $fh "#"x$nlinks; + close $fh; + } else { + unlink $self->{'idfile'}; + rmdir $self->{'lockdir'}; + } + $self->unlock(); + ::debug("sem", "released $self->{'pid'}\n"); +} + +sub atomic_link_if_count_less_than { + # Link $file1 to $file2 if nlinks to $file1 < $count + my $self = shift; + my $retval = 0; + $self->lock(); + ::debug($self->nlinks(), "<", $self->{'count'}); + if($self->nlinks() < $self->{'count'}) { + -d $self->{'lockdir'} or mkdir_or_die($self->{'lockdir'}); + if(not -e $self->{'idfile'}) { + open (my $fh, ">", $self->{'idfile'}) or + ::die_bug("write_idfile: $self->{'idfile'}"); + close $fh; + } + $retval = link $self->{'idfile'}, $self->{'pidfile'}; + } + $self->unlock(); + ::debug("run", "atomic $retval"); + return $retval; +} + +sub _atomic_link_if_count_less_than { + # Link $file1 to $file2 if nlinks to $file1 < $count + my $self = shift; + my $retval = 0; + $self->lock(); + my $nlinks = $self->nlinks(); + ::debug("sem", $nlinks, "<", $self->{'count'}); + if($nlinks++ < $self->{'count'}) { + -d $self->{'lockdir'} or mkdir_or_die($self->{'lockdir'}); + if(not -e $self->{'idfile'}) { + open (my $fh, ">", $self->{'idfile'}) or + ::die_bug("write_idfile: $self->{'idfile'}"); + close $fh; + } + open (my $fh, ">", $self->{'idfile'}) or + ::die_bug("write_idfile: $self->{'idfile'}"); + print $fh "#"x$nlinks; + close $fh; + $retval = link $self->{'idfile'}, $self->{'pidfile'}; + } + $self->unlock(); + ::debug("sem", "atomic $retval"); + return $retval; +} + +sub nlinks { + my $self = shift; + if(-e $self->{'idfile'}) { + ::debug("sem", "nlinks", (stat(_))[3], "size", (stat(_))[7], "\n"); + return (stat(_))[3]; + } else { + return 0; + } +} + +sub lock { + my $self = shift; + my $sleep = 100; # 100 ms + my $total_sleep = 0; + $Global::use{"Fcntl"} ||= eval "use Fcntl qw(:DEFAULT :flock); 1;"; + my $locked = 0; + while(not $locked) { + if(tell($self->{'lockfh'}) == -1) { + # File not open + open($self->{'lockfh'}, ">", $self->{'lockfile'}) + or ::debug("run", "Cannot open $self->{'lockfile'}"); + } + if($self->{'lockfh'}) { + # File is open + chmod 0666, $self->{'lockfile'}; # assuming you want it a+rw + if(flock($self->{'lockfh'}, LOCK_EX()|LOCK_NB())) { + # The file is locked: No need to retry + $locked = 1; + last; + } else { + if ($! =~ m/Function not implemented/) { + ::warning("flock: $!"); + ::warning("Will wait for a random while\n"); + ::usleep(rand(5000)); + # File cannot be locked: No need to retry + $locked = 2; + last; + } + } + } + # Locking failed in first round + # Sleep and try again + $sleep = ($sleep < 1000) ? ($sleep * 1.1) : ($sleep); + # Random to avoid every sleeping job waking up at the same time + ::usleep(rand()*$sleep); + $total_sleep += $sleep; + if($opt::semaphoretimeout) { + if($total_sleep/1000 > $opt::semaphoretimeout) { + # Timeout: bail out + ::warning("Semaphore timed out. Ignoring timeout."); + $locked = 3; + last; + } + } else { + if($total_sleep/1000 > 30) { + ::warning("Semaphore stuck for 30 seconds. Consider using --semaphoretimeout."); + } + } + } + ::debug("run", "locked $self->{'lockfile'}"); +} + +sub unlock { + my $self = shift; + unlink $self->{'lockfile'}; + close $self->{'lockfh'}; + ::debug("run", "unlocked\n"); +} + +sub mkdir_or_die { + # If dir is not writable: die + my $dir = shift; + my @dir_parts = split(m:/:,$dir); + my ($ddir,$part); + while(defined ($part = shift @dir_parts)) { + $part eq "" and next; + $ddir .= "/".$part; + -d $ddir and next; + mkdir $ddir; + } + if(not -w $dir) { + ::error("Cannot write to $dir: $!\n"); + ::wait_and_exit(255); + } +} + +# Keep perl -w happy +$opt::x = $Semaphore::timeout = $Semaphore::wait = +$Job::file_descriptor_warning_printed = 0; diff --git a/librocksdb-sys/rocksdb/build_tools/make_package.sh b/librocksdb-sys/rocksdb/build_tools/make_package.sh new file mode 100755 index 0000000..68a5d8a --- /dev/null +++ b/librocksdb-sys/rocksdb/build_tools/make_package.sh @@ -0,0 +1,129 @@ +# shellcheck disable=SC1113 +#/usr/bin/env bash +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +set -e + +function log() { + echo "[+] $1" +} + +function fatal() { + echo "[!] $1" + exit 1 +} + +function platform() { + local __resultvar=$1 + if [[ -f "/etc/yum.conf" ]]; then + eval $__resultvar="centos" + elif [[ -f "/etc/dpkg/dpkg.cfg" ]]; then + eval $__resultvar="ubuntu" + else + fatal "Unknwon operating system" + fi +} +platform OS + +function package() { + if [[ $OS = "ubuntu" ]]; then + if dpkg --get-selections | grep --quiet $1; then + log "$1 is already installed. skipping." + else + # shellcheck disable=SC2068 + apt-get install $@ -y + fi + elif [[ $OS = "centos" ]]; then + if rpm -qa | grep --quiet $1; then + log "$1 is already installed. skipping." + else + # shellcheck disable=SC2068 + yum install $@ -y + fi + fi +} + +function detect_fpm_output() { + if [[ $OS = "ubuntu" ]]; then + export FPM_OUTPUT=deb + elif [[ $OS = "centos" ]]; then + export FPM_OUTPUT=rpm + fi +} +detect_fpm_output + +function gem_install() { + if gem list | grep --quiet $1; then + log "$1 is already installed. skipping." + else + # shellcheck disable=SC2068 + gem install $@ + fi +} + +function main() { + if [[ $# -ne 1 ]]; then + fatal "Usage: $0 " + else + log "using rocksdb version: $1" + fi + + if [[ -d /vagrant ]]; then + if [[ $OS = "ubuntu" ]]; then + package g++-4.8 + export CXX=g++-4.8 + + # the deb would depend on libgflags2, but the static lib is the only thing + # installed by make install + package libgflags-dev + + package ruby-all-dev + elif [[ $OS = "centos" ]]; then + pushd /etc/yum.repos.d + if [[ ! -f /etc/yum.repos.d/devtools-1.1.repo ]]; then + wget http://people.centos.org/tru/devtools-1.1/devtools-1.1.repo + fi + package devtoolset-1.1-gcc --enablerepo=testing-1.1-devtools-6 + package devtoolset-1.1-gcc-c++ --enablerepo=testing-1.1-devtools-6 + export CC=/opt/centos/devtoolset-1.1/root/usr/bin/gcc + export CPP=/opt/centos/devtoolset-1.1/root/usr/bin/cpp + export CXX=/opt/centos/devtoolset-1.1/root/usr/bin/c++ + export PATH=$PATH:/opt/centos/devtoolset-1.1/root/usr/bin + popd + if ! rpm -qa | grep --quiet gflags; then + rpm -i https://github.com/schuhschuh/gflags/releases/download/v2.1.0/gflags-devel-2.1.0-1.amd64.rpm + fi + + package ruby + package ruby-devel + package rubygems + package rpm-build + fi + fi + gem_install fpm + + make static_lib + LIBDIR=/usr/lib + if [[ $FPM_OUTPUT = "rpm" ]]; then + LIBDIR=$(rpm --eval '%_libdir') + fi + + rm -rf package + make install DESTDIR=package PREFIX=/usr LIBDIR=$LIBDIR + + fpm \ + -s dir \ + -t $FPM_OUTPUT \ + -C package \ + -n rocksdb \ + -v $1 \ + --url http://rocksdb.org/ \ + -m rocksdb@fb.com \ + --license BSD \ + --vendor Facebook \ + --description "RocksDB is an embeddable persistent key-value store for fast storage." \ + usr +} + +# shellcheck disable=SC2068 +main $@ diff --git a/librocksdb-sys/rocksdb/build_tools/ps_with_stack b/librocksdb-sys/rocksdb/build_tools/ps_with_stack new file mode 100755 index 0000000..ee42569 --- /dev/null +++ b/librocksdb-sys/rocksdb/build_tools/ps_with_stack @@ -0,0 +1,38 @@ +#!/usr/bin/env perl + +use strict; + +open(my $ps, "-|", "ps -wwf"); +my $cols_known = 0; +my $cmd_col = 0; +my $pid_col = 0; +while (<$ps>) { + print; + my @cols = split(/\s+/); + + if (!$cols_known && /CMD/) { + # Parse relevant ps column headers + for (my $i = 0; $i <= $#cols; $i++) { + if ($cols[$i] eq "CMD") { + $cmd_col = $i; + } + if ($cols[$i] eq "PID") { + $pid_col = $i; + } + } + $cols_known = 1; + } else { + my $pid = $cols[$pid_col]; + my $cmd = $cols[$cmd_col]; + # Match numeric PID and relative path command + # -> The intention is only to dump stack traces for hangs in code under + # test, which means we probably just built it and are executing by + # relative path (e.g. ./my_test or foo/bar_test) rather then by absolute + # path (e.g. /usr/bin/time) or PATH search (e.g. grep). + if ($pid =~ /^[0-9]+$/ && $cmd =~ /^[^\/ ]+[\/]/) { + print "Dumping stacks for $pid...\n"; + system("pstack $pid || gdb -batch -p $pid -ex 'thread apply all bt'"); + } + } +} +close $ps; diff --git a/librocksdb-sys/rocksdb/build_tools/regression_build_test.sh b/librocksdb-sys/rocksdb/build_tools/regression_build_test.sh new file mode 100755 index 0000000..5ecdb1d --- /dev/null +++ b/librocksdb-sys/rocksdb/build_tools/regression_build_test.sh @@ -0,0 +1,396 @@ +#!/usr/bin/env bash +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +set -e + +NUM=10000000 + +if [ $# -eq 1 ];then + DATA_DIR=$1 +elif [ $# -eq 2 ];then + DATA_DIR=$1 + STAT_FILE=$2 +fi + +# On the production build servers, set data and stat +# files/directories not in /tmp or else the tempdir cleaning +# scripts will make you very unhappy. +DATA_DIR=${DATA_DIR:-$(mktemp -t -d rocksdb_XXXX)} +STAT_FILE=${STAT_FILE:-$(mktemp -t -u rocksdb_test_stats_XXXX)} + +function cleanup { + rm -rf $DATA_DIR + rm -f $STAT_FILE.* +} + +trap cleanup EXIT + +make release + +# measure fillseq + fill up the DB for overwrite benchmark +./db_bench \ + --benchmarks=fillseq \ + --db=$DATA_DIR \ + --use_existing_db=0 \ + --bloom_bits=10 \ + --num=$NUM \ + --writes=$NUM \ + --cache_size=6442450944 \ + --cache_numshardbits=6 \ + --table_cache_numshardbits=4 \ + --open_files=55000 \ + --statistics=1 \ + --histogram=1 \ + --disable_wal=1 \ + --sync=0 > ${STAT_FILE}.fillseq + +# measure overwrite performance +./db_bench \ + --benchmarks=overwrite \ + --db=$DATA_DIR \ + --use_existing_db=1 \ + --bloom_bits=10 \ + --num=$NUM \ + --writes=$((NUM / 10)) \ + --cache_size=6442450944 \ + --cache_numshardbits=6 \ + --table_cache_numshardbits=4 \ + --open_files=55000 \ + --statistics=1 \ + --histogram=1 \ + --disable_wal=1 \ + --sync=0 \ + --threads=8 > ${STAT_FILE}.overwrite + +# fill up the db for readrandom benchmark (1GB total size) +./db_bench \ + --benchmarks=fillseq \ + --db=$DATA_DIR \ + --use_existing_db=0 \ + --bloom_bits=10 \ + --num=$NUM \ + --writes=$NUM \ + --cache_size=6442450944 \ + --cache_numshardbits=6 \ + --table_cache_numshardbits=4 \ + --open_files=55000 \ + --statistics=1 \ + --histogram=1 \ + --disable_wal=1 \ + --sync=0 \ + --threads=1 > /dev/null + +# measure readrandom with 6GB block cache +./db_bench \ + --benchmarks=readrandom \ + --db=$DATA_DIR \ + --use_existing_db=1 \ + --bloom_bits=10 \ + --num=$NUM \ + --reads=$((NUM / 5)) \ + --cache_size=6442450944 \ + --cache_numshardbits=6 \ + --table_cache_numshardbits=4 \ + --open_files=55000 \ + --statistics=1 \ + --histogram=1 \ + --disable_wal=1 \ + --sync=0 \ + --threads=16 > ${STAT_FILE}.readrandom + +# measure readrandom with 6GB block cache and tailing iterator +./db_bench \ + --benchmarks=readrandom \ + --db=$DATA_DIR \ + --use_existing_db=1 \ + --bloom_bits=10 \ + --num=$NUM \ + --reads=$((NUM / 5)) \ + --cache_size=6442450944 \ + --cache_numshardbits=6 \ + --table_cache_numshardbits=4 \ + --open_files=55000 \ + --use_tailing_iterator=1 \ + --statistics=1 \ + --histogram=1 \ + --disable_wal=1 \ + --sync=0 \ + --threads=16 > ${STAT_FILE}.readrandomtailing + +# measure readrandom with 100MB block cache +./db_bench \ + --benchmarks=readrandom \ + --db=$DATA_DIR \ + --use_existing_db=1 \ + --bloom_bits=10 \ + --num=$NUM \ + --reads=$((NUM / 5)) \ + --cache_size=104857600 \ + --cache_numshardbits=6 \ + --table_cache_numshardbits=4 \ + --open_files=55000 \ + --statistics=1 \ + --histogram=1 \ + --disable_wal=1 \ + --sync=0 \ + --threads=16 > ${STAT_FILE}.readrandomsmallblockcache + +# measure readrandom with 8k data in memtable +./db_bench \ + --benchmarks=overwrite,readrandom \ + --db=$DATA_DIR \ + --use_existing_db=1 \ + --bloom_bits=10 \ + --num=$NUM \ + --reads=$((NUM / 5)) \ + --writes=512 \ + --cache_size=6442450944 \ + --cache_numshardbits=6 \ + --table_cache_numshardbits=4 \ + --write_buffer_size=1000000000 \ + --open_files=55000 \ + --statistics=1 \ + --histogram=1 \ + --disable_wal=1 \ + --sync=0 \ + --threads=16 > ${STAT_FILE}.readrandom_mem_sst + + +# fill up the db for readrandom benchmark with filluniquerandom (1GB total size) +./db_bench \ + --benchmarks=filluniquerandom \ + --db=$DATA_DIR \ + --use_existing_db=0 \ + --bloom_bits=10 \ + --num=$((NUM / 4)) \ + --writes=$((NUM / 4)) \ + --cache_size=6442450944 \ + --cache_numshardbits=6 \ + --table_cache_numshardbits=4 \ + --open_files=55000 \ + --statistics=1 \ + --histogram=1 \ + --disable_wal=1 \ + --sync=0 \ + --threads=1 > /dev/null + +# dummy test just to compact the data +./db_bench \ + --benchmarks=readrandom \ + --db=$DATA_DIR \ + --use_existing_db=1 \ + --bloom_bits=10 \ + --num=$((NUM / 1000)) \ + --reads=$((NUM / 1000)) \ + --cache_size=6442450944 \ + --cache_numshardbits=6 \ + --table_cache_numshardbits=4 \ + --open_files=55000 \ + --statistics=1 \ + --histogram=1 \ + --disable_wal=1 \ + --sync=0 \ + --threads=16 > /dev/null + +# measure readrandom after load with filluniquerandom with 6GB block cache +./db_bench \ + --benchmarks=readrandom \ + --db=$DATA_DIR \ + --use_existing_db=1 \ + --bloom_bits=10 \ + --num=$((NUM / 4)) \ + --reads=$((NUM / 4)) \ + --cache_size=6442450944 \ + --cache_numshardbits=6 \ + --table_cache_numshardbits=4 \ + --open_files=55000 \ + --disable_auto_compactions=1 \ + --statistics=1 \ + --histogram=1 \ + --disable_wal=1 \ + --sync=0 \ + --threads=16 > ${STAT_FILE}.readrandom_filluniquerandom + +# measure readwhilewriting after load with filluniquerandom with 6GB block cache +./db_bench \ + --benchmarks=readwhilewriting \ + --db=$DATA_DIR \ + --use_existing_db=1 \ + --bloom_bits=10 \ + --num=$((NUM / 4)) \ + --reads=$((NUM / 4)) \ + --benchmark_write_rate_limit=$(( 110 * 1024 )) \ + --write_buffer_size=100000000 \ + --cache_size=6442450944 \ + --cache_numshardbits=6 \ + --table_cache_numshardbits=4 \ + --open_files=55000 \ + --statistics=1 \ + --histogram=1 \ + --disable_wal=1 \ + --sync=0 \ + --threads=16 > ${STAT_FILE}.readwhilewriting + +# measure memtable performance -- none of the data gets flushed to disk +./db_bench \ + --benchmarks=fillrandom,readrandom, \ + --db=$DATA_DIR \ + --use_existing_db=0 \ + --num=$((NUM / 10)) \ + --reads=$NUM \ + --cache_size=6442450944 \ + --cache_numshardbits=6 \ + --table_cache_numshardbits=4 \ + --write_buffer_size=1000000000 \ + --open_files=55000 \ + --statistics=1 \ + --histogram=1 \ + --disable_wal=1 \ + --sync=0 \ + --value_size=10 \ + --threads=16 > ${STAT_FILE}.memtablefillreadrandom + +common_in_mem_args="--db=/dev/shm/rocksdb \ + --num_levels=6 \ + --key_size=20 \ + --prefix_size=12 \ + --keys_per_prefix=10 \ + --value_size=100 \ + --compression_type=none \ + --compression_ratio=1 \ + --write_buffer_size=134217728 \ + --max_write_buffer_number=4 \ + --level0_file_num_compaction_trigger=8 \ + --level0_slowdown_writes_trigger=16 \ + --level0_stop_writes_trigger=24 \ + --target_file_size_base=134217728 \ + --max_bytes_for_level_base=1073741824 \ + --disable_wal=0 \ + --wal_dir=/dev/shm/rocksdb \ + --sync=0 \ + --verify_checksum=1 \ + --delete_obsolete_files_period_micros=314572800 \ + --use_plain_table=1 \ + --open_files=-1 \ + --mmap_read=1 \ + --mmap_write=0 \ + --bloom_bits=10 \ + --bloom_locality=1 \ + --perf_level=0" + +# prepare a in-memory DB with 50M keys, total DB size is ~6G +./db_bench \ + $common_in_mem_args \ + --statistics=0 \ + --max_background_compactions=16 \ + --max_background_flushes=16 \ + --benchmarks=filluniquerandom \ + --use_existing_db=0 \ + --num=52428800 \ + --threads=1 > /dev/null + +# Readwhilewriting +./db_bench \ + $common_in_mem_args \ + --statistics=1 \ + --max_background_compactions=4 \ + --max_background_flushes=0 \ + --benchmarks=readwhilewriting\ + --use_existing_db=1 \ + --duration=600 \ + --threads=32 \ + --benchmark_write_rate_limit=9502720 > ${STAT_FILE}.readwhilewriting_in_ram + +# Seekrandomwhilewriting +./db_bench \ + $common_in_mem_args \ + --statistics=1 \ + --max_background_compactions=4 \ + --max_background_flushes=0 \ + --benchmarks=seekrandomwhilewriting \ + --use_existing_db=1 \ + --use_tailing_iterator=1 \ + --duration=600 \ + --threads=32 \ + --benchmark_write_rate_limit=9502720 > ${STAT_FILE}.seekwhilewriting_in_ram + +# measure fillseq with bunch of column families +./db_bench \ + --benchmarks=fillseq \ + --num_column_families=500 \ + --write_buffer_size=1048576 \ + --db=$DATA_DIR \ + --use_existing_db=0 \ + --num=$NUM \ + --writes=$NUM \ + --open_files=55000 \ + --statistics=1 \ + --histogram=1 \ + --disable_wal=1 \ + --sync=0 > ${STAT_FILE}.fillseq_lots_column_families + +# measure overwrite performance with bunch of column families +./db_bench \ + --benchmarks=overwrite \ + --num_column_families=500 \ + --write_buffer_size=1048576 \ + --db=$DATA_DIR \ + --use_existing_db=1 \ + --num=$NUM \ + --writes=$((NUM / 10)) \ + --open_files=55000 \ + --statistics=1 \ + --histogram=1 \ + --disable_wal=1 \ + --sync=0 \ + --threads=8 > ${STAT_FILE}.overwrite_lots_column_families + +# send data to ods +function send_to_ods { + key="$1" + value="$2" + + if [ -z $JENKINS_HOME ]; then + # running on devbox, just print out the values + echo $1 $2 + return + fi + + if [ -z "$value" ];then + echo >&2 "ERROR: Key $key doesn't have a value." + return + fi + curl --silent "https://www.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=$key&value=$value" \ + --connect-timeout 60 +} + +function send_benchmark_to_ods { + bench="$1" + bench_key="$2" + file="$3" + + QPS=$(grep $bench $file | awk '{print $5}') + P50_MICROS=$(grep $bench $file -A 6 | grep "Percentiles" | awk '{print $3}' ) + P75_MICROS=$(grep $bench $file -A 6 | grep "Percentiles" | awk '{print $5}' ) + P99_MICROS=$(grep $bench $file -A 6 | grep "Percentiles" | awk '{print $7}' ) + + send_to_ods rocksdb.build.$bench_key.qps $QPS + send_to_ods rocksdb.build.$bench_key.p50_micros $P50_MICROS + send_to_ods rocksdb.build.$bench_key.p75_micros $P75_MICROS + send_to_ods rocksdb.build.$bench_key.p99_micros $P99_MICROS +} + +send_benchmark_to_ods overwrite overwrite $STAT_FILE.overwrite +send_benchmark_to_ods fillseq fillseq $STAT_FILE.fillseq +send_benchmark_to_ods readrandom readrandom $STAT_FILE.readrandom +send_benchmark_to_ods readrandom readrandom_tailing $STAT_FILE.readrandomtailing +send_benchmark_to_ods readrandom readrandom_smallblockcache $STAT_FILE.readrandomsmallblockcache +send_benchmark_to_ods readrandom readrandom_memtable_sst $STAT_FILE.readrandom_mem_sst +send_benchmark_to_ods readrandom readrandom_fillunique_random $STAT_FILE.readrandom_filluniquerandom +send_benchmark_to_ods fillrandom memtablefillrandom $STAT_FILE.memtablefillreadrandom +send_benchmark_to_ods readrandom memtablereadrandom $STAT_FILE.memtablefillreadrandom +send_benchmark_to_ods readwhilewriting readwhilewriting $STAT_FILE.readwhilewriting +send_benchmark_to_ods readwhilewriting readwhilewriting_in_ram ${STAT_FILE}.readwhilewriting_in_ram +send_benchmark_to_ods seekrandomwhilewriting seekwhilewriting_in_ram ${STAT_FILE}.seekwhilewriting_in_ram +send_benchmark_to_ods fillseq fillseq_lots_column_families ${STAT_FILE}.fillseq_lots_column_families +send_benchmark_to_ods overwrite overwrite_lots_column_families ${STAT_FILE}.overwrite_lots_column_families diff --git a/librocksdb-sys/rocksdb/build_tools/run_ci_db_test.ps1 b/librocksdb-sys/rocksdb/build_tools/run_ci_db_test.ps1 new file mode 100644 index 0000000..f20d321 --- /dev/null +++ b/librocksdb-sys/rocksdb/build_tools/run_ci_db_test.ps1 @@ -0,0 +1,493 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# This script enables you running RocksDB tests by running +# All the tests concurrently and utilizing all the cores +Param( + [switch]$EnableJE = $false, # Look for and use test executable, append _je to listed exclusions + [switch]$RunAll = $false, # Will attempt discover all *_test[_je].exe binaries and run all + # of them as Google suites. I.e. It will run test cases concurrently + # except those mentioned as $Run, those will run as individual test cases + # And any execlued with $ExcludeExes or $ExcludeCases + # It will also not run any individual test cases + # excluded but $ExcludeCasese + [switch]$RunAllExe = $false, # Look for and use test exdcutables, append _je to exclusions automatically + # It will attempt to run them in parallel w/o breaking them up on individual + # test cases. Those listed with $ExcludeExes will be excluded + [string]$SuiteRun = "", # Split test suites in test cases and run in parallel, not compatible with $RunAll + [string]$Run = "", # Run specified executables in parallel but do not split to test cases + [string]$ExcludeCases = "", # Exclude test cases, expects a comma separated list, no spaces + # Takes effect when $RunAll or $SuiteRun is specified. Must have full + # Test cases name including a group and a parameter if any + [string]$ExcludeExes = "", # Exclude exes from consideration, expects a comma separated list, + # no spaces. Takes effect only when $RunAll is specified + [string]$WorkFolder = "", # Direct tests to use that folder. SSD or Ram drive are better options. + # Number of async tasks that would run concurrently. Recommend a number below 64. + # However, CPU utlization really depends on the storage media. Recommend ram based disk. + # a value of 1 will run everything serially + [int]$Concurrency = 8, + [int]$Limit = -1 # -1 means do not limit for test purposes +) + +# Folders and commands must be fullpath to run assuming +# the current folder is at the root of the git enlistment +$StartDate = (Get-Date) +$StartDate + + +$DebugPreference = "Continue" + +# These tests are not google test suites and we should guard +# Against running them as suites +$RunOnly = New-Object System.Collections.Generic.HashSet[string] +$RunOnly.Add("c_test") | Out-Null +$RunOnly.Add("compact_on_deletion_collector_test") | Out-Null +$RunOnly.Add("merge_test") | Out-Null +$RunOnly.Add("stringappend_test") | Out-Null # Apparently incorrectly written +$RunOnly.Add("backup_engine_test") | Out-Null # Disabled +$RunOnly.Add("timer_queue_test") | Out-Null # Not a gtest + +if($RunAll -and $SuiteRun -ne "") { + Write-Error "$RunAll and $SuiteRun are not compatible" + exit 1 +} + +if($RunAllExe -and $Run -ne "") { + Write-Error "$RunAllExe and $Run are not compatible" + exit 1 +} + +# If running under Appveyor assume that root +[string]$Appveyor = $Env:APPVEYOR_BUILD_FOLDER +if($Appveyor -ne "") { + $RootFolder = $Appveyor +} else { + $RootFolder = $PSScriptRoot -replace '\\build_tools', '' +} + +$LogFolder = -Join($RootFolder, "\db_logs\") +$BinariesFolder = -Join($RootFolder, "\build\Debug\") + +if($WorkFolder -eq "") { + + # If TEST_TMPDIR is set use it + [string]$var = $Env:TEST_TMPDIR + if($var -eq "") { + $WorkFolder = -Join($RootFolder, "\db_tests\") + $Env:TEST_TMPDIR = $WorkFolder + } else { + $WorkFolder = $var + } +} else { +# Override from a command line + $Env:TEST_TMPDIR = $WorkFolder +} + +Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder" +Write-Output "BinariesFolder: $BinariesFolder, LogFolder: $LogFolder" + +# Create test directories in the current folder +md -Path $WorkFolder -ErrorAction Ignore | Out-Null +md -Path $LogFolder -ErrorAction Ignore | Out-Null + + +$ExcludeCasesSet = New-Object System.Collections.Generic.HashSet[string] +if($ExcludeCases -ne "") { + Write-Host "ExcludeCases: $ExcludeCases" + $l = $ExcludeCases -split ' ' + ForEach($t in $l) { + $ExcludeCasesSet.Add($t) | Out-Null + } +} + +$ExcludeExesSet = New-Object System.Collections.Generic.HashSet[string] +if($ExcludeExes -ne "") { + Write-Host "ExcludeExe: $ExcludeExes" + $l = $ExcludeExes -split ' ' + ForEach($t in $l) { + $ExcludeExesSet.Add($t) | Out-Null + } +} + + +# Extract the names of its tests by running db_test with --gtest_list_tests. +# This filter removes the "#"-introduced comments, and expands to +# fully-qualified names by changing input like this: +# +# DBTest. +# Empty +# WriteEmptyBatch +# MultiThreaded/MultiThreadedDBTest. +# MultiThreaded/0 # GetParam() = 0 +# MultiThreaded/1 # GetParam() = 1 +# RibbonTypeParamTest/0. # TypeParam = struct DefaultTypesAndSettings +# CompactnessAndBacktrackAndFpRate +# Extremes +# FindOccupancyForSuccessRate +# +# into this: +# +# DBTest.Empty +# DBTest.WriteEmptyBatch +# MultiThreaded/MultiThreadedDBTest.MultiThreaded/0 +# MultiThreaded/MultiThreadedDBTest.MultiThreaded/1 +# RibbonTypeParamTest/0.CompactnessAndBacktrackAndFpRate +# RibbonTypeParamTest/0.Extremes +# RibbonTypeParamTest/0.FindOccupancyForSuccessRate +# +# Output into the parameter in a form TestName -> Log File Name +function ExtractTestCases([string]$GTestExe, $HashTable) { + + $Tests = @() +# Run db_test to get a list of tests and store it into $a array + &$GTestExe --gtest_list_tests | tee -Variable Tests | Out-Null + + # Current group + $Group="" + + ForEach( $l in $Tests) { + + # remove trailing comment if any + $l = $l -replace '\s+\#.*','' + # Leading whitespace is fine + $l = $l -replace '^\s+','' + # Trailing dot is a test group but no whitespace + if ($l -match "\.$" -and $l -notmatch "\s+") { + $Group = $l + } else { + # Otherwise it is a test name, remove leading space + $test = $l + # create a log name + $test = "$Group$test" + + if($ExcludeCasesSet.Contains($test)) { + Write-Warning "$test case is excluded" + continue + } + + $test_log = $test -replace '[\./]','_' + $test_log += ".log" + $log_path = -join ($LogFolder, $test_log) + + # Add to a hashtable + $HashTable.Add($test, $log_path); + } + } +} + +# The function removes trailing .exe siffix if any, +# creates a name for the log file +# Then adds the test name if it was not excluded into +# a HashTable in a form of test_name -> log_path +function MakeAndAdd([string]$token, $HashTable) { + + $test_name = $token -replace '.exe$', '' + $log_name = -join ($test_name, ".log") + $log_path = -join ($LogFolder, $log_name) + $HashTable.Add($test_name, $log_path) +} + +# This function takes a list of Suites to run +# Lists all the test cases in each of the suite +# and populates HashOfHashes +# Ordered by suite(exe) @{ Exe = @{ TestCase = LogName }} +function ProcessSuites($ListOfSuites, $HashOfHashes) { + + $suite_list = $ListOfSuites + # Problem: if you run --gtest_list_tests on + # a non Google Test executable then it will start executing + # and we will get nowhere + ForEach($suite in $suite_list) { + + if($RunOnly.Contains($suite)) { + Write-Warning "$suite is excluded from running as Google test suite" + continue + } + + if($EnableJE) { + $suite += "_je" + } + + $Cases = [ordered]@{} + $Cases.Clear() + $suite_exe = -Join ($BinariesFolder, $suite) + ExtractTestCases -GTestExe $suite_exe -HashTable $Cases + if($Cases.Count -gt 0) { + $HashOfHashes.Add($suite, $Cases); + } + } + + # Make logs and run + if($CasesToRun.Count -lt 1) { + Write-Error "Failed to extract tests from $SuiteRun" + exit 1 + } + +} + +# This will contain all test executables to run + +# Hash table that contains all non suite +# Test executable to run +$TestExes = [ordered]@{} + +# Check for test exe that are not +# Google Test Suites +# Since this is explicitely mentioned it is not subject +# for exclusions +if($Run -ne "") { + + $test_list = $Run -split ' ' + ForEach($t in $test_list) { + + if($EnableJE) { + $t += "_je" + } + MakeAndAdd -token $t -HashTable $TestExes + } + + if($TestExes.Count -lt 1) { + Write-Error "Failed to extract tests from $Run" + exit 1 + } +} elseif($RunAllExe) { + # Discover all the test binaries + if($EnableJE) { + $pattern = "*_test_je.exe" + } else { + $pattern = "*_test.exe" + } + + $search_path = -join ($BinariesFolder, $pattern) + Write-Host "Binaries Search Path: $search_path" + + $DiscoveredExe = @() + dir -Path $search_path | ForEach-Object { + $DiscoveredExe += ($_.Name) + } + + # Remove exclusions + ForEach($e in $DiscoveredExe) { + $e = $e -replace '.exe$', '' + $bare_name = $e -replace '_je$', '' + + if($ExcludeExesSet.Contains($bare_name)) { + Write-Warning "Test $e is excluded" + continue + } + MakeAndAdd -token $e -HashTable $TestExes + } + + if($TestExes.Count -lt 1) { + Write-Error "Failed to discover test executables" + exit 1 + } +} + +# Ordered by exe @{ Exe = @{ TestCase = LogName }} +$CasesToRun = [ordered]@{} + +if($SuiteRun -ne "") { + $suite_list = $SuiteRun -split ' ' + ProcessSuites -ListOfSuites $suite_list -HashOfHashes $CasesToRun +} elseif ($RunAll) { +# Discover all the test binaries + if($EnableJE) { + $pattern = "*_test_je.exe" + } else { + $pattern = "*_test.exe" + } + + $search_path = -join ($BinariesFolder, $pattern) + Write-Host "Binaries Search Path: $search_path" + + $ListOfExe = @() + dir -Path $search_path | ForEach-Object { + $ListOfExe += ($_.Name) + } + + # Exclude those in RunOnly from running as suites + $ListOfSuites = @() + ForEach($e in $ListOfExe) { + + $e = $e -replace '.exe$', '' + $bare_name = $e -replace '_je$', '' + + if($ExcludeExesSet.Contains($bare_name)) { + Write-Warning "Test $e is excluded" + continue + } + + if($RunOnly.Contains($bare_name)) { + MakeAndAdd -token $e -HashTable $TestExes + } else { + $ListOfSuites += $bare_name + } + } + + ProcessSuites -ListOfSuites $ListOfSuites -HashOfHashes $CasesToRun +} + + +# Invoke a test with a filter and redirect all output +$InvokeTestCase = { + param($exe, $test, $log); + &$exe --gtest_filter=$test > $log 2>&1 +} + +# Invoke all tests and redirect output +$InvokeTestAsync = { + param($exe, $log) + &$exe > $log 2>&1 +} + +# Hash that contains tests to rerun if any failed +# Those tests will be rerun sequentially +# $Rerun = [ordered]@{} +# Test limiting factor here +[int]$count = 0 +# Overall status +[bool]$script:success = $true; + +function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal) +{ + # Array to wait for any of the running jobs + $jobs = @() + # Hash JobToLog + $JobToLog = @{} + + # Wait for all to finish and get the results + while(($JobToLog.Count -gt 0) -or + ($TestCmds.Count -gt 0) -or + ($Suites.Count -gt 0)) { + + # Make sure we have maximum concurrent jobs running if anything + # and the $Limit either not set or allows to proceed + while(($JobToLog.Count -lt $ConcurrencyVal) -and + ((($TestCmds.Count -gt 0) -or ($Suites.Count -gt 0)) -and + (($Limit -lt 0) -or ($count -lt $Limit)))) { + + # We always favore suites to run if available + [string]$exe_name = "" + [string]$log_path = "" + $Cases = @{} + + if($Suites.Count -gt 0) { + # Will the first one + ForEach($e in $Suites.Keys) { + $exe_name = $e + $Cases = $Suites[$e] + break + } + [string]$test_case = "" + [string]$log_path = "" + ForEach($c in $Cases.Keys) { + $test_case = $c + $log_path = $Cases[$c] + break + } + + Write-Host "Starting $exe_name::$test_case" + [string]$Exe = -Join ($BinariesFolder, $exe_name) + $job = Start-Job -Name "$exe_name::$test_case" -ArgumentList @($Exe,$test_case,$log_path) -ScriptBlock $InvokeTestCase + $JobToLog.Add($job, $log_path) + + $Cases.Remove($test_case) + if($Cases.Count -lt 1) { + $Suites.Remove($exe_name) + } + + } elseif ($TestCmds.Count -gt 0) { + + ForEach($e in $TestCmds.Keys) { + $exe_name = $e + $log_path = $TestCmds[$e] + break + } + + Write-Host "Starting $exe_name" + [string]$Exe = -Join ($BinariesFolder, $exe_name) + $job = Start-Job -Name $exe_name -ScriptBlock $InvokeTestAsync -ArgumentList @($Exe,$log_path) + $JobToLog.Add($job, $log_path) + + $TestCmds.Remove($exe_name) + + } else { + Write-Error "In the job loop but nothing to run" + exit 1 + } + + ++$count + } # End of Job starting loop + + if($JobToLog.Count -lt 1) { + break + } + + $jobs = @() + foreach($k in $JobToLog.Keys) { $jobs += $k } + + $completed = Wait-Job -Job $jobs -Any + $log = $JobToLog[$completed] + $JobToLog.Remove($completed) + + $message = -join @($completed.Name, " State: ", ($completed.State)) + + $log_content = @(Get-Content $log) + + if($completed.State -ne "Completed") { + $script:success = $false + Write-Warning $message + $log_content | Write-Warning + } else { + # Scan the log. If we find PASSED and no occurrence of FAILED + # then it is a success + [bool]$pass_found = $false + ForEach($l in $log_content) { + + if(($l -match "^\[\s+FAILED") -or + ($l -match "Assertion failed:")) { + $pass_found = $false + break + } + + if(($l -match "^\[\s+PASSED") -or + ($l -match " : PASSED$") -or + ($l -match "^PASS$") -or # Special c_test case + ($l -match "Passed all tests!") ) { + $pass_found = $true + } + } + + if(!$pass_found) { + $script:success = $false; + Write-Warning $message + $log_content | Write-Warning + } else { + Write-Host $message + } + } + + # Remove cached job info from the system + # Should be no output + Receive-Job -Job $completed | Out-Null + } +} + +RunJobs -Suites $CasesToRun -TestCmds $TestExes -ConcurrencyVal $Concurrency + +$EndDate = (Get-Date) + +New-TimeSpan -Start $StartDate -End $EndDate | + ForEach-Object { + "Elapsed time: {0:g}" -f $_ + } + + +if(!$script:success) { +# This does not succeed killing off jobs quick +# So we simply exit +# Remove-Job -Job $jobs -Force +# indicate failure using this exit code + exit 1 + } + + exit 0 diff --git a/librocksdb-sys/rocksdb/build_tools/setup_centos7.sh b/librocksdb-sys/rocksdb/build_tools/setup_centos7.sh new file mode 100755 index 0000000..474d91a --- /dev/null +++ b/librocksdb-sys/rocksdb/build_tools/setup_centos7.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +set -ex + +ROCKSDB_VERSION="6.7.3" +ZSTD_VERSION="1.4.4" + +echo "This script configures CentOS with everything needed to build and run RocksDB" + +yum update -y && yum install epel-release -y + +yum install -y \ + wget \ + gcc-c++ \ + snappy snappy-devel \ + zlib zlib-devel \ + bzip2 bzip2-devel \ + lz4-devel \ + libasan \ + gflags + +mkdir -pv /usr/local/rocksdb-${ROCKSDB_VERSION} +ln -sfT /usr/local/rocksdb-${ROCKSDB_VERSION} /usr/local/rocksdb + +wget -qO /tmp/zstd-${ZSTD_VERSION}.tar.gz https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz +wget -qO /tmp/rocksdb-${ROCKSDB_VERSION}.tar.gz https://github.com/facebook/rocksdb/archive/v${ROCKSDB_VERSION}.tar.gz + +cd /tmp + +tar xzvf zstd-${ZSTD_VERSION}.tar.gz +tar xzvf rocksdb-${ROCKSDB_VERSION}.tar.gz -C /usr/local/ + +echo "Installing ZSTD..." +pushd zstd-${ZSTD_VERSION} +make && make install +popd + +echo "Compiling RocksDB..." +cd /usr/local/rocksdb +chown -R vagrant:vagrant /usr/local/rocksdb/ +sudo -u vagrant make static_lib +cd examples/ +sudo -u vagrant LD_LIBRARY_PATH=/usr/local/lib/ make all +sudo -u vagrant LD_LIBRARY_PATH=/usr/local/lib/ ./c_simple_example + diff --git a/librocksdb-sys/rocksdb/build_tools/ubuntu20_image/Dockerfile b/librocksdb-sys/rocksdb/build_tools/ubuntu20_image/Dockerfile new file mode 100644 index 0000000..d81a5e4 --- /dev/null +++ b/librocksdb-sys/rocksdb/build_tools/ubuntu20_image/Dockerfile @@ -0,0 +1,57 @@ +# from official ubuntu 20.04 +FROM ubuntu:20.04 +# update system +RUN apt-get update && apt-get upgrade -y +# install basic tools +RUN apt-get install -y vim wget curl +# install tzdata noninteractive +RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata +# install git and default compilers +RUN apt-get install -y git gcc g++ clang clang-tools +# install basic package +RUN apt-get install -y lsb-release software-properties-common gnupg +# install gflags, tbb +RUN apt-get install -y libgflags-dev libtbb-dev +# install compression libs +RUN apt-get install -y libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev +# install cmake +RUN apt-get install -y cmake +RUN apt-get install -y libssl-dev +# install clang-13 +WORKDIR /root +RUN wget https://apt.llvm.org/llvm.sh +RUN chmod +x llvm.sh +RUN ./llvm.sh 13 all +# install gcc-7, 8, 10, 11, default is 9 +RUN apt-get install -y gcc-7 g++-7 +RUN apt-get install -y gcc-8 g++-8 +RUN apt-get install -y gcc-10 g++-10 +RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test +RUN apt-get install -y gcc-11 g++-11 +# install apt-get install -y valgrind +RUN apt-get install -y valgrind +# install folly depencencies +RUN apt-get install -y libgoogle-glog-dev +# install openjdk 8 +RUN apt-get install -y openjdk-8-jdk +ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk-amd64 +# install mingw +RUN apt-get install -y mingw-w64 + +# install gtest-parallel package +RUN git clone --single-branch --branch master --depth 1 https://github.com/google/gtest-parallel.git ~/gtest-parallel +ENV PATH $PATH:/root/gtest-parallel + +# install libprotobuf for fuzzers test +RUN apt-get install -y ninja-build binutils liblzma-dev libz-dev pkg-config autoconf libtool +RUN git clone --branch v1.0 https://github.com/google/libprotobuf-mutator.git ~/libprotobuf-mutator && cd ~/libprotobuf-mutator && git checkout ffd86a32874e5c08a143019aad1aaf0907294c9f && mkdir build && cd build && cmake .. -GNinja -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang++-13 -DCMAKE_BUILD_TYPE=Release -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON && ninja && ninja install +ENV PKG_CONFIG_PATH /usr/local/OFF/:/root/libprotobuf-mutator/build/external.protobuf/lib/pkgconfig/ +ENV PROTOC_BIN /root/libprotobuf-mutator/build/external.protobuf/bin/protoc + +# install the latest google benchmark +RUN git clone --depth 1 --branch v1.7.0 https://github.com/google/benchmark.git ~/benchmark +RUN cd ~/benchmark && mkdir build && cd build && cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_GTEST_TESTS=0 && ninja && ninja install + +# clean up +RUN rm -rf /var/lib/apt/lists/* +RUN rm -rf /root/benchmark diff --git a/librocksdb-sys/rocksdb/build_tools/update_dependencies.sh b/librocksdb-sys/rocksdb/build_tools/update_dependencies.sh new file mode 100755 index 0000000..c549e5b --- /dev/null +++ b/librocksdb-sys/rocksdb/build_tools/update_dependencies.sh @@ -0,0 +1,106 @@ +#!/bin/sh +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# +# Update dependencies.sh file with the latest avaliable versions + +BASEDIR=$(dirname $0) +OUTPUT="" + +function log_header() +{ + echo "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved." >> "$OUTPUT" + echo "# The file is generated using update_dependencies.sh." >> "$OUTPUT" +} + + +function log_variable() +{ + echo "$1=${!1}" >> "$OUTPUT" +} + + +TP2_LATEST="/data/users/$USER/fbsource/fbcode/third-party2/" +## $1 => lib name +## $2 => lib version (if not provided, will try to pick latest) +## $3 => platform (if not provided, will try to pick latest gcc) +## +## get_lib_base will set a variable named ${LIB_NAME}_BASE to the lib location +function get_lib_base() +{ + local lib_name=$1 + local lib_version=$2 + local lib_platform=$3 + + local result="$TP2_LATEST/$lib_name/" + + # Lib Version + if [ -z "$lib_version" ] || [ "$lib_version" = "LATEST" ]; then + # version is not provided, use latest + result=`ls -dr1v $result/*/ | head -n1` + else + result="$result/$lib_version/" + fi + + # Lib Platform + if [ -z "$lib_platform" ]; then + # platform is not provided, use latest gcc + result=`ls -dr1v $result/gcc-*[^fb]/ | head -n1` + else + echo $lib_platform + result="$result/$lib_platform/" + fi + + result=`ls -1d $result/*/ | head -n1` + + echo Finding link $result + + # lib_name => LIB_NAME_BASE + local __res_var=${lib_name^^}"_BASE" + __res_var=`echo $__res_var | tr - _` + # LIB_NAME_BASE=$result + eval $__res_var=`readlink -f $result` + + log_variable $__res_var +} + +########################################################### +# platform010 dependencies # +########################################################### + +OUTPUT="$BASEDIR/dependencies_platform010.sh" + +rm -f "$OUTPUT" +touch "$OUTPUT" + +echo "Writing dependencies to $OUTPUT" + +# Compilers locations +GCC_BASE=`readlink -f $TP2_LATEST/gcc/11.x/centos7-native/*/` +CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/12/platform010/*/` + +log_header +log_variable GCC_BASE +log_variable CLANG_BASE + +# Libraries locations +get_lib_base libgcc 11.x platform010 +get_lib_base glibc 2.34 platform010 +get_lib_base snappy LATEST platform010 +get_lib_base zlib LATEST platform010 +get_lib_base bzip2 LATEST platform010 +get_lib_base lz4 LATEST platform010 +get_lib_base zstd LATEST platform010 +get_lib_base gflags LATEST platform010 +get_lib_base jemalloc LATEST platform010 +get_lib_base numa LATEST platform010 +get_lib_base libunwind LATEST platform010 +get_lib_base tbb 2018_U5 platform010 +get_lib_base liburing LATEST platform010 +get_lib_base benchmark LATEST platform010 + +get_lib_base kernel-headers fb platform010 +get_lib_base binutils LATEST centos7-native +get_lib_base valgrind LATEST platform010 +get_lib_base lua 5.3.4 platform010 + +git diff $OUTPUT diff --git a/librocksdb-sys/rocksdb/build_tools/version.sh b/librocksdb-sys/rocksdb/build_tools/version.sh new file mode 100755 index 0000000..dbc1a92 --- /dev/null +++ b/librocksdb-sys/rocksdb/build_tools/version.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +if [ "$#" = "0" ]; then + echo "Usage: $0 major|minor|patch|full" + exit 1 +fi + +if [ "$1" = "major" ]; then + cat include/rocksdb/version.h | grep MAJOR | head -n1 | awk '{print $3}' +fi +if [ "$1" = "minor" ]; then + cat include/rocksdb/version.h | grep MINOR | head -n1 | awk '{print $3}' +fi +if [ "$1" = "patch" ]; then + cat include/rocksdb/version.h | grep PATCH | head -n1 | awk '{print $3}' +fi +if [ "$1" = "full" ]; then + awk '/#define ROCKSDB/ { env[$2] = $3 } + END { printf "%s.%s.%s\n", env["ROCKSDB_MAJOR"], + env["ROCKSDB_MINOR"], + env["ROCKSDB_PATCH"] }' \ + include/rocksdb/version.h +fi diff --git a/librocksdb-sys/rocksdb/cache/cache.cc b/librocksdb-sys/rocksdb/cache/cache.cc new file mode 100644 index 0000000..a65f5ec --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/cache.cc @@ -0,0 +1,158 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "rocksdb/cache.h" + +#include "cache/lru_cache.h" +#include "rocksdb/secondary_cache.h" +#include "rocksdb/utilities/customizable_util.h" +#include "rocksdb/utilities/options_type.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { +const Cache::CacheItemHelper kNoopCacheItemHelper{}; + +static std::unordered_map + lru_cache_options_type_info = { + {"capacity", + {offsetof(struct LRUCacheOptions, capacity), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, + {"num_shard_bits", + {offsetof(struct LRUCacheOptions, num_shard_bits), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, + {"strict_capacity_limit", + {offsetof(struct LRUCacheOptions, strict_capacity_limit), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"high_pri_pool_ratio", + {offsetof(struct LRUCacheOptions, high_pri_pool_ratio), + OptionType::kDouble, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"low_pri_pool_ratio", + {offsetof(struct LRUCacheOptions, low_pri_pool_ratio), + OptionType::kDouble, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, +}; + +static std::unordered_map + comp_sec_cache_options_type_info = { + {"capacity", + {offsetof(struct CompressedSecondaryCacheOptions, capacity), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"num_shard_bits", + {offsetof(struct CompressedSecondaryCacheOptions, num_shard_bits), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"compression_type", + {offsetof(struct CompressedSecondaryCacheOptions, compression_type), + OptionType::kCompressionType, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"compress_format_version", + {offsetof(struct CompressedSecondaryCacheOptions, + compress_format_version), + OptionType::kUInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"enable_custom_split_merge", + {offsetof(struct CompressedSecondaryCacheOptions, + enable_custom_split_merge), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, +}; + +Status SecondaryCache::CreateFromString( + const ConfigOptions& config_options, const std::string& value, + std::shared_ptr* result) { + if (value.find("compressed_secondary_cache://") == 0) { + std::string args = value; + args.erase(0, std::strlen("compressed_secondary_cache://")); + Status status; + std::shared_ptr sec_cache; + + CompressedSecondaryCacheOptions sec_cache_opts; + status = OptionTypeInfo::ParseStruct(config_options, "", + &comp_sec_cache_options_type_info, "", + args, &sec_cache_opts); + if (status.ok()) { + sec_cache = NewCompressedSecondaryCache(sec_cache_opts); + } + + + if (status.ok()) { + result->swap(sec_cache); + } + return status; + } else { + return LoadSharedObject(config_options, value, result); + } +} + +Status Cache::CreateFromString(const ConfigOptions& config_options, + const std::string& value, + std::shared_ptr* result) { + Status status; + std::shared_ptr cache; + if (value.find('=') == std::string::npos) { + cache = NewLRUCache(ParseSizeT(value)); + } else { + LRUCacheOptions cache_opts; + status = OptionTypeInfo::ParseStruct(config_options, "", + &lru_cache_options_type_info, "", + value, &cache_opts); + if (status.ok()) { + cache = NewLRUCache(cache_opts); + } + } + if (status.ok()) { + result->swap(cache); + } + return status; +} + +bool Cache::AsyncLookupHandle::IsReady() { + return pending_handle == nullptr || pending_handle->IsReady(); +} + +bool Cache::AsyncLookupHandle::IsPending() { return pending_handle != nullptr; } + +Cache::Handle* Cache::AsyncLookupHandle::Result() { + assert(!IsPending()); + return result_handle; +} + +void Cache::StartAsyncLookup(AsyncLookupHandle& async_handle) { + async_handle.found_dummy_entry = false; // in case re-used + assert(!async_handle.IsPending()); + async_handle.result_handle = + Lookup(async_handle.key, async_handle.helper, async_handle.create_context, + async_handle.priority, async_handle.stats); +} + +Cache::Handle* Cache::Wait(AsyncLookupHandle& async_handle) { + WaitAll(&async_handle, 1); + return async_handle.Result(); +} + +void Cache::WaitAll(AsyncLookupHandle* async_handles, size_t count) { + for (size_t i = 0; i < count; ++i) { + if (async_handles[i].IsPending()) { + // If a pending handle gets here, it should be marked at "to be handled + // by a caller" by that caller erasing the pending_cache on it. + assert(async_handles[i].pending_cache == nullptr); + } + } +} + +void Cache::SetEvictionCallback(EvictionCallback&& fn) { + // Overwriting non-empty with non-empty could indicate a bug + assert(!eviction_callback_ || !fn); + eviction_callback_ = std::move(fn); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/cache_bench.cc b/librocksdb-sys/rocksdb/cache/cache_bench.cc new file mode 100644 index 0000000..f836939 --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/cache_bench.cc @@ -0,0 +1,20 @@ +// Copyright (c) 2013-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#ifndef GFLAGS +#include +int main() { + fprintf(stderr, "Please install gflags to run rocksdb tools\n"); + return 1; +} +#else +#include "rocksdb/cache_bench_tool.h" +int main(int argc, char** argv) { + return ROCKSDB_NAMESPACE::cache_bench_tool(argc, argv); +} +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/cache/cache_bench_tool.cc b/librocksdb-sys/rocksdb/cache/cache_bench_tool.cc new file mode 100644 index 0000000..fd1b44d --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/cache_bench_tool.cc @@ -0,0 +1,1034 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "cache_key.h" +#ifdef GFLAGS +#include +#include +#include +#include +#include +#include +#include + +#include "db/db_impl/db_impl.h" +#include "monitoring/histogram.h" +#include "port/port.h" +#include "rocksdb/advanced_cache.h" +#include "rocksdb/convenience.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/secondary_cache.h" +#include "rocksdb/system_clock.h" +#include "rocksdb/table_properties.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/block_based/cachable_entry.h" +#include "util/coding.h" +#include "util/distributed_mutex.h" +#include "util/gflags_compat.h" +#include "util/hash.h" +#include "util/mutexlock.h" +#include "util/random.h" +#include "util/stop_watch.h" +#include "util/string_util.h" + +using GFLAGS_NAMESPACE::ParseCommandLineFlags; + +static constexpr uint32_t KiB = uint32_t{1} << 10; +static constexpr uint32_t MiB = KiB << 10; +static constexpr uint64_t GiB = MiB << 10; + +DEFINE_uint32(threads, 16, "Number of concurrent threads to run."); +DEFINE_uint64(cache_size, 1 * GiB, + "Number of bytes to use as a cache of uncompressed data."); +DEFINE_uint32(num_shard_bits, 6, "shard_bits."); + +DEFINE_double(resident_ratio, 0.25, + "Ratio of keys fitting in cache to keyspace."); +DEFINE_uint64(ops_per_thread, 2000000U, "Number of operations per thread."); +DEFINE_uint32(value_bytes, 8 * KiB, "Size of each value added."); + +DEFINE_uint32(skew, 5, "Degree of skew in key selection. 0 = no skew"); +DEFINE_bool(populate_cache, true, "Populate cache before operations"); + +DEFINE_uint32(lookup_insert_percent, 87, + "Ratio of lookup (+ insert on not found) to total workload " + "(expressed as a percentage)"); +DEFINE_uint32(insert_percent, 2, + "Ratio of insert to total workload (expressed as a percentage)"); +DEFINE_uint32(lookup_percent, 10, + "Ratio of lookup to total workload (expressed as a percentage)"); +DEFINE_uint32(erase_percent, 1, + "Ratio of erase to total workload (expressed as a percentage)"); +DEFINE_bool(gather_stats, false, + "Whether to periodically simulate gathering block cache stats, " + "using one more thread."); +DEFINE_uint32( + gather_stats_sleep_ms, 1000, + "How many milliseconds to sleep between each gathering of stats."); + +DEFINE_uint32(gather_stats_entries_per_lock, 256, + "For Cache::ApplyToAllEntries"); + +DEFINE_bool(lean, false, + "If true, no additional computation is performed besides cache " + "operations."); + +DEFINE_bool(early_exit, false, + "Exit before deallocating most memory. Good for malloc stats, e.g." + "MALLOC_CONF=\"stats_print:true\""); + +DEFINE_bool(histograms, true, + "Whether to track and print histogram statistics."); + +DEFINE_uint32(seed, 0, "Hashing/random seed to use. 0 = choose at random"); + +DEFINE_string(secondary_cache_uri, "", + "Full URI for creating a custom secondary cache object"); +static class std::shared_ptr secondary_cache; + +DEFINE_string(cache_type, "lru_cache", "Type of block cache."); + +// ## BEGIN stress_cache_key sub-tool options ## +// See class StressCacheKey below. +DEFINE_bool(stress_cache_key, false, + "If true, run cache key stress test instead"); +DEFINE_uint32( + sck_files_per_day, 2500000, + "(-stress_cache_key) Simulated files generated per simulated day"); +// NOTE: Giving each run a specified lifetime, rather than e.g. "until +// first collision" ensures equal skew from start-up, when collisions are +// less likely. +DEFINE_uint32(sck_days_per_run, 90, + "(-stress_cache_key) Number of days to simulate in each run"); +// NOTE: The number of observed collisions directly affects the relative +// accuracy of the predicted probabilities. 15 observations should be well +// within factor-of-2 accuracy. +DEFINE_uint32( + sck_min_collision, 15, + "(-stress_cache_key) Keep running until this many collisions seen"); +// sck_file_size_mb can be thought of as average file size. The simulation is +// not precise enough to care about the distribution of file sizes; other +// simulations (https://github.com/pdillinger/unique_id/tree/main/monte_carlo) +// indicate the distribution only makes a small difference (e.g. < 2x factor) +DEFINE_uint32( + sck_file_size_mb, 32, + "(-stress_cache_key) Simulated file size in MiB, for accounting purposes"); +DEFINE_uint32(sck_reopen_nfiles, 100, + "(-stress_cache_key) Simulate DB re-open average every n files"); +DEFINE_uint32(sck_newdb_nreopen, 1000, + "(-stress_cache_key) Simulate new DB average every n re-opens"); +DEFINE_uint32(sck_restarts_per_day, 24, + "(-stress_cache_key) Average simulated process restarts per day " + "(across DBs)"); +DEFINE_uint32( + sck_db_count, 100, + "(-stress_cache_key) Parallel DBs in simulation sharing a block cache"); +DEFINE_uint32( + sck_table_bits, 20, + "(-stress_cache_key) Log2 number of tracked (live) files (across DBs)"); +// sck_keep_bits being well below full 128 bits amplifies the collision +// probability so that the true probability can be estimated through observed +// collisions. (More explanation below.) +DEFINE_uint32( + sck_keep_bits, 50, + "(-stress_cache_key) Number of bits to keep from each cache key (<= 64)"); +// sck_randomize is used to validate whether cache key is performing "better +// than random." Even with this setting, file offsets are not randomized. +DEFINE_bool(sck_randomize, false, + "(-stress_cache_key) Randomize (hash) cache key"); +// See https://github.com/facebook/rocksdb/pull/9058 +DEFINE_bool(sck_footer_unique_id, false, + "(-stress_cache_key) Simulate using proposed footer unique id"); +// ## END stress_cache_key sub-tool options ## + +namespace ROCKSDB_NAMESPACE { + +class CacheBench; +namespace { +// State shared by all concurrent executions of the same benchmark. +class SharedState { + public: + explicit SharedState(CacheBench* cache_bench) + : cv_(&mu_), + cache_bench_(cache_bench) {} + + ~SharedState() {} + + port::Mutex* GetMutex() { return &mu_; } + + port::CondVar* GetCondVar() { return &cv_; } + + CacheBench* GetCacheBench() const { return cache_bench_; } + + void IncInitialized() { num_initialized_++; } + + void IncDone() { num_done_++; } + + bool AllInitialized() const { return num_initialized_ >= FLAGS_threads; } + + bool AllDone() const { return num_done_ >= FLAGS_threads; } + + void SetStart() { start_ = true; } + + bool Started() const { return start_; } + + void AddLookupStats(uint64_t hits, uint64_t misses) { + MutexLock l(&mu_); + lookup_count_ += hits + misses; + lookup_hits_ += hits; + } + + double GetLookupHitRatio() const { + return 1.0 * lookup_hits_ / lookup_count_; + } + + private: + port::Mutex mu_; + port::CondVar cv_; + + CacheBench* cache_bench_; + + uint64_t num_initialized_ = 0; + bool start_ = false; + uint64_t num_done_ = 0; + uint64_t lookup_count_ = 0; + uint64_t lookup_hits_ = 0; +}; + +// Per-thread state for concurrent executions of the same benchmark. +struct ThreadState { + uint32_t tid; + Random64 rnd; + SharedState* shared; + HistogramImpl latency_ns_hist; + uint64_t duration_us = 0; + + ThreadState(uint32_t index, SharedState* _shared) + : tid(index), rnd(FLAGS_seed + 1 + index), shared(_shared) {} +}; + +struct KeyGen { + char key_data[27]; + + Slice GetRand(Random64& rnd, uint64_t max_key, uint32_t skew) { + uint64_t raw = rnd.Next(); + // Skew according to setting + for (uint32_t i = 0; i < skew; ++i) { + raw = std::min(raw, rnd.Next()); + } + uint64_t key = FastRange64(raw, max_key); + // Variable size and alignment + size_t off = key % 8; + key_data[0] = char{42}; + EncodeFixed64(key_data + 1, key); + key_data[9] = char{11}; + EncodeFixed64(key_data + 10, key); + key_data[18] = char{4}; + EncodeFixed64(key_data + 19, key); + assert(27 >= kCacheKeySize); + return Slice(&key_data[off], kCacheKeySize); + } +}; + +Cache::ObjectPtr createValue(Random64& rnd) { + char* rv = new char[FLAGS_value_bytes]; + // Fill with some filler data, and take some CPU time + for (uint32_t i = 0; i < FLAGS_value_bytes; i += 8) { + EncodeFixed64(rv + i, rnd.Next()); + } + return rv; +} + +// Callbacks for secondary cache +size_t SizeFn(Cache::ObjectPtr /*obj*/) { return FLAGS_value_bytes; } + +Status SaveToFn(Cache::ObjectPtr from_obj, size_t /*from_offset*/, + size_t length, char* out) { + memcpy(out, from_obj, length); + return Status::OK(); +} + +Status CreateFn(const Slice& data, Cache::CreateContext* /*context*/, + MemoryAllocator* /*allocator*/, Cache::ObjectPtr* out_obj, + size_t* out_charge) { + *out_obj = new char[data.size()]; + memcpy(*out_obj, data.data(), data.size()); + *out_charge = data.size(); + return Status::OK(); +}; + +void DeleteFn(Cache::ObjectPtr value, MemoryAllocator* /*alloc*/) { + delete[] static_cast(value); +} + +Cache::CacheItemHelper helper1_wos(CacheEntryRole::kDataBlock, DeleteFn); +Cache::CacheItemHelper helper1(CacheEntryRole::kDataBlock, DeleteFn, SizeFn, + SaveToFn, CreateFn, &helper1_wos); +Cache::CacheItemHelper helper2_wos(CacheEntryRole::kIndexBlock, DeleteFn); +Cache::CacheItemHelper helper2(CacheEntryRole::kIndexBlock, DeleteFn, SizeFn, + SaveToFn, CreateFn, &helper2_wos); +Cache::CacheItemHelper helper3_wos(CacheEntryRole::kFilterBlock, DeleteFn); +Cache::CacheItemHelper helper3(CacheEntryRole::kFilterBlock, DeleteFn, SizeFn, + SaveToFn, CreateFn, &helper3_wos); +} // namespace + +class CacheBench { + static constexpr uint64_t kHundredthUint64 = + std::numeric_limits::max() / 100U; + + public: + CacheBench() + : max_key_(static_cast(FLAGS_cache_size / FLAGS_resident_ratio / + FLAGS_value_bytes)), + lookup_insert_threshold_(kHundredthUint64 * + FLAGS_lookup_insert_percent), + insert_threshold_(lookup_insert_threshold_ + + kHundredthUint64 * FLAGS_insert_percent), + lookup_threshold_(insert_threshold_ + + kHundredthUint64 * FLAGS_lookup_percent), + erase_threshold_(lookup_threshold_ + + kHundredthUint64 * FLAGS_erase_percent) { + if (erase_threshold_ != 100U * kHundredthUint64) { + fprintf(stderr, "Percentages must add to 100.\n"); + exit(1); + } + + if (FLAGS_cache_type == "clock_cache") { + fprintf(stderr, "Old clock cache implementation has been removed.\n"); + exit(1); + } else if (FLAGS_cache_type == "hyper_clock_cache") { + HyperClockCacheOptions opts(FLAGS_cache_size, FLAGS_value_bytes, + FLAGS_num_shard_bits); + opts.hash_seed = BitwiseAnd(FLAGS_seed, INT32_MAX); + cache_ = opts.MakeSharedCache(); + } else if (FLAGS_cache_type == "lru_cache") { + LRUCacheOptions opts(FLAGS_cache_size, FLAGS_num_shard_bits, + false /* strict_capacity_limit */, + 0.5 /* high_pri_pool_ratio */); + opts.hash_seed = BitwiseAnd(FLAGS_seed, INT32_MAX); + if (!FLAGS_secondary_cache_uri.empty()) { + Status s = SecondaryCache::CreateFromString( + ConfigOptions(), FLAGS_secondary_cache_uri, &secondary_cache); + if (secondary_cache == nullptr) { + fprintf( + stderr, + "No secondary cache registered matching string: %s status=%s\n", + FLAGS_secondary_cache_uri.c_str(), s.ToString().c_str()); + exit(1); + } + opts.secondary_cache = secondary_cache; + } + + cache_ = NewLRUCache(opts); + } else { + fprintf(stderr, "Cache type not supported."); + exit(1); + } + } + + ~CacheBench() {} + + void PopulateCache() { + Random64 rnd(FLAGS_seed); + KeyGen keygen; + size_t max_occ = 0; + size_t inserts_since_max_occ_increase = 0; + size_t keys_since_last_not_found = 0; + + // Avoid redundant insertions by checking Lookup before Insert. + // Loop until insertions consistently fail to increase max occupancy or + // it becomes difficult to find keys not already inserted. + while (inserts_since_max_occ_increase < 100 && + keys_since_last_not_found < 100) { + Slice key = keygen.GetRand(rnd, max_key_, FLAGS_skew); + + Cache::Handle* handle = cache_->Lookup(key); + if (handle != nullptr) { + cache_->Release(handle); + ++keys_since_last_not_found; + continue; + } + keys_since_last_not_found = 0; + + Status s = + cache_->Insert(key, createValue(rnd), &helper1, FLAGS_value_bytes); + assert(s.ok()); + + handle = cache_->Lookup(key); + if (!handle) { + fprintf(stderr, "Failed to lookup key just inserted.\n"); + assert(false); + exit(42); + } else { + cache_->Release(handle); + } + + size_t occ = cache_->GetOccupancyCount(); + if (occ > max_occ) { + max_occ = occ; + inserts_since_max_occ_increase = 0; + } else { + ++inserts_since_max_occ_increase; + } + } + printf("Population complete (%zu entries, %g average charge)\n", max_occ, + 1.0 * FLAGS_cache_size / max_occ); + } + + bool Run() { + const auto clock = SystemClock::Default().get(); + + PrintEnv(); + SharedState shared(this); + std::vector > threads(FLAGS_threads); + for (uint32_t i = 0; i < FLAGS_threads; i++) { + threads[i].reset(new ThreadState(i, &shared)); + std::thread(ThreadBody, threads[i].get()).detach(); + } + + HistogramImpl stats_hist; + std::string stats_report; + std::thread stats_thread(StatsBody, &shared, &stats_hist, &stats_report); + + uint64_t start_time; + { + MutexLock l(shared.GetMutex()); + while (!shared.AllInitialized()) { + shared.GetCondVar()->Wait(); + } + // Record start time + start_time = clock->NowMicros(); + + // Start all threads + shared.SetStart(); + shared.GetCondVar()->SignalAll(); + + // Wait threads to complete + while (!shared.AllDone()) { + shared.GetCondVar()->Wait(); + } + } + + // Stats gathering is considered background work. This time measurement + // is for foreground work, and not really ideal for that. See below. + uint64_t end_time = clock->NowMicros(); + stats_thread.join(); + + // Wall clock time - includes idle time if threads + // finish at different times (not ideal). + double elapsed_secs = static_cast(end_time - start_time) * 1e-6; + uint32_t ops_per_sec = static_cast( + 1.0 * FLAGS_threads * FLAGS_ops_per_thread / elapsed_secs); + printf("Complete in %.3f s; Rough parallel ops/sec = %u\n", elapsed_secs, + ops_per_sec); + + // Total time in each thread (more accurate throughput measure) + elapsed_secs = 0; + for (uint32_t i = 0; i < FLAGS_threads; i++) { + elapsed_secs += threads[i]->duration_us * 1e-6; + } + ops_per_sec = static_cast(1.0 * FLAGS_threads * + FLAGS_ops_per_thread / elapsed_secs); + printf("Thread ops/sec = %u\n", ops_per_sec); + + printf("Lookup hit ratio: %g\n", shared.GetLookupHitRatio()); + + if (FLAGS_histograms) { + printf("\nOperation latency (ns):\n"); + HistogramImpl combined; + for (uint32_t i = 0; i < FLAGS_threads; i++) { + combined.Merge(threads[i]->latency_ns_hist); + } + printf("%s", combined.ToString().c_str()); + + if (FLAGS_gather_stats) { + printf("\nGather stats latency (us):\n"); + printf("%s", stats_hist.ToString().c_str()); + } + } + printf("\n%s", stats_report.c_str()); + + return true; + } + + private: + std::shared_ptr cache_; + const uint64_t max_key_; + // Cumulative thresholds in the space of a random uint64_t + const uint64_t lookup_insert_threshold_; + const uint64_t insert_threshold_; + const uint64_t lookup_threshold_; + const uint64_t erase_threshold_; + + // A benchmark version of gathering stats on an active block cache by + // iterating over it. The primary purpose is to measure the impact of + // gathering stats with ApplyToAllEntries on throughput- and + // latency-sensitive Cache users. Performance of stats gathering is + // also reported. The last set of gathered stats is also reported, for + // manual sanity checking for logical errors or other unexpected + // behavior of cache_bench or the underlying Cache. + static void StatsBody(SharedState* shared, HistogramImpl* stats_hist, + std::string* stats_report) { + if (!FLAGS_gather_stats) { + return; + } + const auto clock = SystemClock::Default().get(); + uint64_t total_key_size = 0; + uint64_t total_charge = 0; + uint64_t total_entry_count = 0; + uint64_t table_occupancy = 0; + uint64_t table_size = 0; + std::set helpers; + StopWatchNano timer(clock); + + for (;;) { + uint64_t time; + time = clock->NowMicros(); + uint64_t deadline = time + uint64_t{FLAGS_gather_stats_sleep_ms} * 1000; + + { + MutexLock l(shared->GetMutex()); + for (;;) { + if (shared->AllDone()) { + std::ostringstream ostr; + ostr << "Most recent cache entry stats:\n" + << "Number of entries: " << total_entry_count << "\n" + << "Table occupancy: " << table_occupancy << " / " + << table_size << " = " + << (100.0 * table_occupancy / table_size) << "%\n" + << "Total charge: " << BytesToHumanString(total_charge) << "\n" + << "Average key size: " + << (1.0 * total_key_size / total_entry_count) << "\n" + << "Average charge: " + << BytesToHumanString(static_cast( + 1.0 * total_charge / total_entry_count)) + << "\n" + << "Unique helpers: " << helpers.size() << "\n"; + *stats_report = ostr.str(); + return; + } + if (clock->NowMicros() >= deadline) { + break; + } + uint64_t diff = deadline - std::min(clock->NowMicros(), deadline); + shared->GetCondVar()->TimedWait(diff + 1); + } + } + + // Now gather stats, outside of mutex + total_key_size = 0; + total_charge = 0; + total_entry_count = 0; + helpers.clear(); + auto fn = [&](const Slice& key, Cache::ObjectPtr /*value*/, size_t charge, + const Cache::CacheItemHelper* helper) { + total_key_size += key.size(); + total_charge += charge; + ++total_entry_count; + // Something slightly more expensive as in stats by category + helpers.insert(helper); + }; + if (FLAGS_histograms) { + timer.Start(); + } + Cache::ApplyToAllEntriesOptions opts; + opts.average_entries_per_lock = FLAGS_gather_stats_entries_per_lock; + shared->GetCacheBench()->cache_->ApplyToAllEntries(fn, opts); + table_occupancy = shared->GetCacheBench()->cache_->GetOccupancyCount(); + table_size = shared->GetCacheBench()->cache_->GetTableAddressCount(); + if (FLAGS_histograms) { + stats_hist->Add(timer.ElapsedNanos() / 1000); + } + } + } + + static void ThreadBody(ThreadState* thread) { + SharedState* shared = thread->shared; + + { + MutexLock l(shared->GetMutex()); + shared->IncInitialized(); + if (shared->AllInitialized()) { + shared->GetCondVar()->SignalAll(); + } + while (!shared->Started()) { + shared->GetCondVar()->Wait(); + } + } + thread->shared->GetCacheBench()->OperateCache(thread); + + { + MutexLock l(shared->GetMutex()); + shared->IncDone(); + if (shared->AllDone()) { + shared->GetCondVar()->SignalAll(); + } + } + } + + void OperateCache(ThreadState* thread) { + // To use looked-up values + uint64_t result = 0; + uint64_t lookup_misses = 0; + uint64_t lookup_hits = 0; + // To hold handles for a non-trivial amount of time + Cache::Handle* handle = nullptr; + KeyGen gen; + const auto clock = SystemClock::Default().get(); + uint64_t start_time = clock->NowMicros(); + StopWatchNano timer(clock); + + for (uint64_t i = 0; i < FLAGS_ops_per_thread; i++) { + Slice key = gen.GetRand(thread->rnd, max_key_, FLAGS_skew); + uint64_t random_op = thread->rnd.Next(); + + if (FLAGS_histograms) { + timer.Start(); + } + + if (random_op < lookup_insert_threshold_) { + if (handle) { + cache_->Release(handle); + handle = nullptr; + } + // do lookup + handle = cache_->Lookup(key, &helper2, /*context*/ nullptr, + Cache::Priority::LOW); + if (handle) { + ++lookup_hits; + if (!FLAGS_lean) { + // do something with the data + result += NPHash64(static_cast(cache_->Value(handle)), + FLAGS_value_bytes); + } + } else { + ++lookup_misses; + // do insert + Status s = cache_->Insert(key, createValue(thread->rnd), &helper2, + FLAGS_value_bytes, &handle); + assert(s.ok()); + } + } else if (random_op < insert_threshold_) { + if (handle) { + cache_->Release(handle); + handle = nullptr; + } + // do insert + Status s = cache_->Insert(key, createValue(thread->rnd), &helper3, + FLAGS_value_bytes, &handle); + assert(s.ok()); + } else if (random_op < lookup_threshold_) { + if (handle) { + cache_->Release(handle); + handle = nullptr; + } + // do lookup + handle = cache_->Lookup(key, &helper2, /*context*/ nullptr, + Cache::Priority::LOW); + if (handle) { + ++lookup_hits; + if (!FLAGS_lean) { + // do something with the data + result += NPHash64(static_cast(cache_->Value(handle)), + FLAGS_value_bytes); + } + } else { + ++lookup_misses; + } + } else if (random_op < erase_threshold_) { + // do erase + cache_->Erase(key); + } else { + // Should be extremely unlikely (noop) + assert(random_op >= kHundredthUint64 * 100U); + } + if (FLAGS_histograms) { + thread->latency_ns_hist.Add(timer.ElapsedNanos()); + } + thread->shared->AddLookupStats(lookup_hits, lookup_misses); + } + if (FLAGS_early_exit) { + MutexLock l(thread->shared->GetMutex()); + exit(0); + } + if (handle) { + cache_->Release(handle); + handle = nullptr; + } + // Ensure computations on `result` are not optimized away. + if (result == 1) { + printf("You are extremely unlucky(2). Try again.\n"); + exit(1); + } + thread->duration_us = clock->NowMicros() - start_time; + } + + void PrintEnv() const { +#if defined(__GNUC__) && !defined(__OPTIMIZE__) + printf( + "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n"); +#endif +#ifndef NDEBUG + printf("WARNING: Assertions are enabled; benchmarks unnecessarily slow\n"); +#endif + printf("----------------------------\n"); + printf("RocksDB version : %d.%d\n", kMajorVersion, kMinorVersion); + printf("DMutex impl name : %s\n", DMutex::kName()); + printf("Number of threads : %u\n", FLAGS_threads); + printf("Ops per thread : %" PRIu64 "\n", FLAGS_ops_per_thread); + printf("Cache size : %s\n", + BytesToHumanString(FLAGS_cache_size).c_str()); + printf("Num shard bits : %u\n", FLAGS_num_shard_bits); + printf("Max key : %" PRIu64 "\n", max_key_); + printf("Resident ratio : %g\n", FLAGS_resident_ratio); + printf("Skew degree : %u\n", FLAGS_skew); + printf("Populate cache : %d\n", int{FLAGS_populate_cache}); + printf("Lookup+Insert pct : %u%%\n", FLAGS_lookup_insert_percent); + printf("Insert percentage : %u%%\n", FLAGS_insert_percent); + printf("Lookup percentage : %u%%\n", FLAGS_lookup_percent); + printf("Erase percentage : %u%%\n", FLAGS_erase_percent); + std::ostringstream stats; + if (FLAGS_gather_stats) { + stats << "enabled (" << FLAGS_gather_stats_sleep_ms << "ms, " + << FLAGS_gather_stats_entries_per_lock << "/lock)"; + } else { + stats << "disabled"; + } + printf("Gather stats : %s\n", stats.str().c_str()); + printf("----------------------------\n"); + } +}; + +// cache_bench -stress_cache_key is an independent embedded tool for +// estimating the probability of CacheKey collisions through simulation. +// At a high level, it simulates generating SST files over many months, +// keeping them in the DB and/or cache for some lifetime while staying +// under resource caps, and checking for any cache key collisions that +// arise among the set of live files. For efficient simulation, we make +// some simplifying "pessimistic" assumptions (that only increase the +// chance of the simulation reporting a collision relative to the chance +// of collision in practice): +// * Every generated file has a cache entry for every byte offset in the +// file (contiguous range of cache keys) +// * All of every file is cached for its entire lifetime. (Here "lifetime" +// is technically the union of DB and Cache lifetime, though we only +// model a generous DB lifetime, where space usage is always maximized. +// In a effective Cache, lifetime in cache can only substantially exceed +// lifetime in DB if there is little cache activity; cache activity is +// required to hit cache key collisions.) +// +// It would be possible to track an exact set of cache key ranges for the +// set of live files, but we would have no hope of observing collisions +// (overlap in live files) in our simulation. We need to employ some way +// of amplifying collision probability that allows us to predict the real +// collision probability by extrapolation from observed collisions. Our +// basic approach is to reduce each cache key range down to some smaller +// number of bits, and limiting to bits that are shared over the whole +// range. Now we can observe collisions using a set of smaller stripped-down +// (reduced) cache keys. Let's do some case analysis to understand why this +// works: +// * No collision in reduced key - because the reduction is a pure function +// this implies no collision in the full keys +// * Collision detected between two reduced keys - either +// * The reduction has dropped some structured uniqueness info (from one of +// session counter or file number; file offsets are never materialized here). +// This can only artificially inflate the observed and extrapolated collision +// probabilities. We only have to worry about this in designing the reduction. +// * The reduction has preserved all the structured uniqueness in the cache +// key, which means either +// * REJECTED: We have a uniqueness bug in generating cache keys, where +// structured uniqueness info should have been different but isn't. In such a +// case, increasing by 1 the number of bits kept after reduction would not +// reduce observed probabilities by half. (In our observations, the +// probabilities are reduced approximately by half.) +// * ACCEPTED: The lost unstructured uniqueness in the key determines the +// probability that an observed collision would imply an overlap in ranges. +// In short, dropping n bits from key would increase collision probability by +// 2**n, assuming those n bits have full entropy in unstructured uniqueness. +// +// But we also have to account for the key ranges based on file size. If file +// sizes are roughly 2**b offsets, using XOR in 128-bit cache keys for +// "ranges", we know from other simulations (see +// https://github.com/pdillinger/unique_id/) that that's roughly equivalent to +// (less than 2x higher collision probability) using a cache key of size +// 128 - b bits for the whole file. (This is the only place we make an +// "optimistic" assumption, which is more than offset by the real +// implementation stripping off 2 lower bits from block byte offsets for cache +// keys. The simulation assumes byte offsets, which is net pessimistic.) +// +// So to accept the extrapolation as valid, we need to be confident that all +// "lost" bits, excluding those covered by file offset, are full entropy. +// Recall that we have assumed (verifiably, safely) that other structured data +// (file number and session counter) are kept, not lost. Based on the +// implementation comments for OffsetableCacheKey, the only potential hole here +// is that we only have ~103 bits of entropy in "all new" session IDs, and in +// extreme cases, there might be only 1 DB ID. However, because the upper ~39 +// bits of session ID are hashed, the combination of file number and file +// offset only has to add to 25 bits (or more) to ensure full entropy in +// unstructured uniqueness lost in the reduction. Typical file size of 32MB +// suffices (at least for simulation purposes where we assume each file offset +// occupies a cache key). +// +// Example results in comments on OffsetableCacheKey. +class StressCacheKey { + public: + void Run() { + if (FLAGS_sck_footer_unique_id) { + // Proposed footer unique IDs are DB-independent and session-independent + // (but process-dependent) which is most easily simulated here by + // assuming 1 DB and (later below) no session resets without process + // reset. + FLAGS_sck_db_count = 1; + } + + // Describe the simulated workload + uint64_t mb_per_day = + uint64_t{FLAGS_sck_files_per_day} * FLAGS_sck_file_size_mb; + printf("Total cache or DBs size: %gTiB Writing %g MiB/s or %gTiB/day\n", + FLAGS_sck_file_size_mb / 1024.0 / 1024.0 * + std::pow(2.0, FLAGS_sck_table_bits), + mb_per_day / 86400.0, mb_per_day / 1024.0 / 1024.0); + // For extrapolating probability of any collisions from a number of + // observed collisions + multiplier_ = std::pow(2.0, 128 - FLAGS_sck_keep_bits) / + (FLAGS_sck_file_size_mb * 1024.0 * 1024.0); + printf( + "Multiply by %g to correct for simulation losses (but still assume " + "whole file cached)\n", + multiplier_); + restart_nfiles_ = FLAGS_sck_files_per_day / FLAGS_sck_restarts_per_day; + double without_ejection = + std::pow(1.414214, FLAGS_sck_keep_bits) / FLAGS_sck_files_per_day; + // This should be a lower bound for -sck_randomize, usually a terribly + // rough lower bound. + // If observation is worse than this, then something has gone wrong. + printf( + "Without ejection, expect random collision after %g days (%g " + "corrected)\n", + without_ejection, without_ejection * multiplier_); + double with_full_table = + std::pow(2.0, FLAGS_sck_keep_bits - FLAGS_sck_table_bits) / + FLAGS_sck_files_per_day; + // This is an alternate lower bound for -sck_randomize, usually pretty + // accurate. Our cache keys should usually perform "better than random" + // but always no worse. (If observation is substantially worse than this, + // then something has gone wrong.) + printf( + "With ejection and full table, expect random collision after %g " + "days (%g corrected)\n", + with_full_table, with_full_table * multiplier_); + collisions_ = 0; + + // Run until sufficient number of observed collisions. + for (int i = 1; collisions_ < FLAGS_sck_min_collision; i++) { + RunOnce(); + if (collisions_ == 0) { + printf( + "No collisions after %d x %u days " + " \n", + i, FLAGS_sck_days_per_run); + } else { + double est = 1.0 * i * FLAGS_sck_days_per_run / collisions_; + printf("%" PRIu64 + " collisions after %d x %u days, est %g days between (%g " + "corrected) \n", + collisions_, i, FLAGS_sck_days_per_run, est, est * multiplier_); + } + } + } + + void RunOnce() { + // Re-initialized simulated state + const size_t db_count = std::max(size_t{FLAGS_sck_db_count}, size_t{1}); + dbs_.reset(new TableProperties[db_count]{}); + const size_t table_mask = (size_t{1} << FLAGS_sck_table_bits) - 1; + table_.reset(new uint64_t[table_mask + 1]{}); + if (FLAGS_sck_keep_bits > 64) { + FLAGS_sck_keep_bits = 64; + } + + // Details of which bits are dropped in reduction + uint32_t shift_away = 64 - FLAGS_sck_keep_bits; + // Shift away fewer potential file number bits (b) than potential + // session counter bits (a). + uint32_t shift_away_b = shift_away / 3; + uint32_t shift_away_a = shift_away - shift_away_b; + + process_count_ = 0; + session_count_ = 0; + newdb_count_ = 0; + ResetProcess(/*newdbs*/ true); + + Random64 r{std::random_device{}()}; + + uint64_t max_file_count = + uint64_t{FLAGS_sck_files_per_day} * FLAGS_sck_days_per_run; + uint32_t report_count = 0; + uint32_t collisions_this_run = 0; + size_t db_i = 0; + + for (uint64_t file_count = 1; file_count <= max_file_count; + ++file_count, ++db_i) { + // Round-robin through DBs (this faster than %) + if (db_i >= db_count) { + db_i = 0; + } + // Any other periodic actions before simulating next file + if (!FLAGS_sck_footer_unique_id && r.OneIn(FLAGS_sck_reopen_nfiles)) { + ResetSession(db_i, /*newdb*/ r.OneIn(FLAGS_sck_newdb_nreopen)); + } else if (r.OneIn(restart_nfiles_)) { + ResetProcess(/*newdbs*/ false); + } + // Simulate next file + OffsetableCacheKey ock; + dbs_[db_i].orig_file_number += 1; + // skip some file numbers for other file kinds, except in footer unique + // ID, orig_file_number here tracks process-wide generated SST file + // count. + if (!FLAGS_sck_footer_unique_id) { + dbs_[db_i].orig_file_number += (r.Next() & 3); + } + bool is_stable; + BlockBasedTable::SetupBaseCacheKey(&dbs_[db_i], /* ignored */ "", + /* ignored */ 42, &ock, &is_stable); + assert(is_stable); + // Get a representative cache key, which later we analytically generalize + // to a range. + CacheKey ck = ock.WithOffset(0); + uint64_t reduced_key; + if (FLAGS_sck_randomize) { + reduced_key = GetSliceHash64(ck.AsSlice()) >> shift_away; + } else if (FLAGS_sck_footer_unique_id) { + // Special case: keep only file number, not session counter + reduced_key = DecodeFixed64(ck.AsSlice().data()) >> shift_away; + } else { + // Try to keep file number and session counter (shift away other bits) + uint32_t a = DecodeFixed32(ck.AsSlice().data()) << shift_away_a; + uint32_t b = DecodeFixed32(ck.AsSlice().data() + 4) >> shift_away_b; + reduced_key = (uint64_t{a} << 32) + b; + } + if (reduced_key == 0) { + // Unlikely, but we need to exclude tracking this value because we + // use it to mean "empty" in table. This case is OK as long as we + // don't hit it often. + printf("Hit Zero! \n"); + file_count--; + continue; + } + uint64_t h = + NPHash64(reinterpret_cast(&reduced_key), sizeof(reduced_key)); + // Skew expected lifetimes, for high variance (super-Poisson) variance + // in actual lifetimes. + size_t pos = + std::min(Lower32of64(h) & table_mask, Upper32of64(h) & table_mask); + if (table_[pos] == reduced_key) { + collisions_this_run++; + // Our goal is to predict probability of no collisions, not expected + // number of collisions. To make the distinction, we have to get rid + // of observing correlated collisions, which this takes care of: + ResetProcess(/*newdbs*/ false); + } else { + // Replace (end of lifetime for file that was in this slot) + table_[pos] = reduced_key; + } + + if (++report_count == FLAGS_sck_files_per_day) { + report_count = 0; + // Estimate fill % + size_t incr = table_mask / 1000; + size_t sampled_count = 0; + for (size_t i = 0; i <= table_mask; i += incr) { + if (table_[i] != 0) { + sampled_count++; + } + } + // Report + printf( + "%" PRIu64 " days, %" PRIu64 " proc, %" PRIu64 " sess, %" PRIu64 + " newdb, %u coll, occ %g%%, ejected %g%% \r", + file_count / FLAGS_sck_files_per_day, process_count_, + session_count_, newdb_count_ - FLAGS_sck_db_count, + collisions_this_run, 100.0 * sampled_count / 1000.0, + 100.0 * (1.0 - sampled_count / 1000.0 * table_mask / file_count)); + fflush(stdout); + } + } + collisions_ += collisions_this_run; + } + + void ResetSession(size_t i, bool newdb) { + dbs_[i].db_session_id = DBImpl::GenerateDbSessionId(nullptr); + if (newdb) { + ++newdb_count_; + if (FLAGS_sck_footer_unique_id) { + // Simulate how footer id would behave + dbs_[i].db_id = "none"; + } else { + // db_id might be ignored, depending on the implementation details + dbs_[i].db_id = std::to_string(newdb_count_); + dbs_[i].orig_file_number = 0; + } + } + session_count_++; + } + + void ResetProcess(bool newdbs) { + process_count_++; + DBImpl::TEST_ResetDbSessionIdGen(); + for (size_t i = 0; i < FLAGS_sck_db_count; ++i) { + ResetSession(i, newdbs); + } + if (FLAGS_sck_footer_unique_id) { + // For footer unique ID, this tracks process-wide generated SST file + // count. + dbs_[0].orig_file_number = 0; + } + } + + private: + // Use db_session_id and orig_file_number from TableProperties + std::unique_ptr dbs_; + std::unique_ptr table_; + uint64_t process_count_ = 0; + uint64_t session_count_ = 0; + uint64_t newdb_count_ = 0; + uint64_t collisions_ = 0; + uint32_t restart_nfiles_ = 0; + double multiplier_ = 0.0; +}; + +int cache_bench_tool(int argc, char** argv) { + ParseCommandLineFlags(&argc, &argv, true); + + if (FLAGS_stress_cache_key) { + // Alternate tool + StressCacheKey().Run(); + return 0; + } + + if (FLAGS_threads <= 0) { + fprintf(stderr, "threads number <= 0\n"); + exit(1); + } + + if (FLAGS_seed == 0) { + FLAGS_seed = static_cast(port::GetProcessID()); + printf("Using seed = %" PRIu32 "\n", FLAGS_seed); + } + + ROCKSDB_NAMESPACE::CacheBench bench; + if (FLAGS_populate_cache) { + bench.PopulateCache(); + } + if (bench.Run()) { + return 0; + } else { + return 1; + } +} // namespace ROCKSDB_NAMESPACE +} // namespace ROCKSDB_NAMESPACE + +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/cache/cache_entry_roles.cc b/librocksdb-sys/rocksdb/cache/cache_entry_roles.cc new file mode 100644 index 0000000..f83ada2 --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/cache_entry_roles.cc @@ -0,0 +1,104 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "cache/cache_entry_roles.h" + +#include + +#include "port/lang.h" + +namespace ROCKSDB_NAMESPACE { + +std::array kCacheEntryRoleToCamelString{{ + "DataBlock", + "FilterBlock", + "FilterMetaBlock", + "DeprecatedFilterBlock", + "IndexBlock", + "OtherBlock", + "WriteBuffer", + "CompressionDictionaryBuildingBuffer", + "FilterConstruction", + "BlockBasedTableReader", + "FileMetadata", + "BlobValue", + "BlobCache", + "Misc", +}}; + +std::array kCacheEntryRoleToHyphenString{{ + "data-block", + "filter-block", + "filter-meta-block", + "deprecated-filter-block", + "index-block", + "other-block", + "write-buffer", + "compression-dictionary-building-buffer", + "filter-construction", + "block-based-table-reader", + "file-metadata", + "blob-value", + "blob-cache", + "misc", +}}; + +const std::string& GetCacheEntryRoleName(CacheEntryRole role) { + return kCacheEntryRoleToHyphenString[static_cast(role)]; +} + +const std::string& BlockCacheEntryStatsMapKeys::CacheId() { + static const std::string kCacheId = "id"; + return kCacheId; +} + +const std::string& BlockCacheEntryStatsMapKeys::CacheCapacityBytes() { + static const std::string kCacheCapacityBytes = "capacity"; + return kCacheCapacityBytes; +} + +const std::string& +BlockCacheEntryStatsMapKeys::LastCollectionDurationSeconds() { + static const std::string kLastCollectionDurationSeconds = + "secs_for_last_collection"; + return kLastCollectionDurationSeconds; +} + +const std::string& BlockCacheEntryStatsMapKeys::LastCollectionAgeSeconds() { + static const std::string kLastCollectionAgeSeconds = + "secs_since_last_collection"; + return kLastCollectionAgeSeconds; +} + +namespace { + +std::string GetPrefixedCacheEntryRoleName(const std::string& prefix, + CacheEntryRole role) { + const std::string& role_name = GetCacheEntryRoleName(role); + std::string prefixed_role_name; + prefixed_role_name.reserve(prefix.size() + role_name.size()); + prefixed_role_name.append(prefix); + prefixed_role_name.append(role_name); + return prefixed_role_name; +} + +} // namespace + +std::string BlockCacheEntryStatsMapKeys::EntryCount(CacheEntryRole role) { + const static std::string kPrefix = "count."; + return GetPrefixedCacheEntryRoleName(kPrefix, role); +} + +std::string BlockCacheEntryStatsMapKeys::UsedBytes(CacheEntryRole role) { + const static std::string kPrefix = "bytes."; + return GetPrefixedCacheEntryRoleName(kPrefix, role); +} + +std::string BlockCacheEntryStatsMapKeys::UsedPercent(CacheEntryRole role) { + const static std::string kPrefix = "percent."; + return GetPrefixedCacheEntryRoleName(kPrefix, role); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/cache_entry_roles.h b/librocksdb-sys/rocksdb/cache/cache_entry_roles.h new file mode 100644 index 0000000..78bec79 --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/cache_entry_roles.h @@ -0,0 +1,20 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include + +#include "rocksdb/cache.h" + +namespace ROCKSDB_NAMESPACE { + +extern std::array + kCacheEntryRoleToCamelString; +extern std::array + kCacheEntryRoleToHyphenString; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/cache_entry_stats.h b/librocksdb-sys/rocksdb/cache/cache_entry_stats.h new file mode 100644 index 0000000..9968995 --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/cache_entry_stats.h @@ -0,0 +1,182 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include +#include + +#include "cache/cache_key.h" +#include "cache/typed_cache.h" +#include "port/lang.h" +#include "rocksdb/cache.h" +#include "rocksdb/status.h" +#include "rocksdb/system_clock.h" +#include "test_util/sync_point.h" +#include "util/coding_lean.h" + +namespace ROCKSDB_NAMESPACE { + +// A generic helper object for gathering stats about cache entries by +// iterating over them with ApplyToAllEntries. This class essentially +// solves the problem of slowing down a Cache with too many stats +// collectors that could be sharing stat results, such as from multiple +// column families or multiple DBs sharing a Cache. We employ a few +// mitigations: +// * Only one collector for a particular kind of Stats is alive +// for each Cache. This is guaranteed using the Cache itself to hold +// the collector. +// * A mutex ensures only one thread is gathering stats for this +// collector. +// * The most recent gathered stats are saved and simply copied to +// satisfy requests within a time window (default: 3 minutes) of +// completion of the most recent stat gathering. +// +// Template parameter Stats must be copyable and trivially constructable, +// as well as... +// concept Stats { +// // Notification before applying callback to all entries +// void BeginCollection(Cache*, SystemClock*, uint64_t start_time_micros); +// // Get the callback to apply to all entries. `callback` +// // type must be compatible with Cache::ApplyToAllEntries +// callback GetEntryCallback(); +// // Notification after applying callback to all entries +// void EndCollection(Cache*, SystemClock*, uint64_t end_time_micros); +// // Notification that a collection was skipped because of +// // sufficiently recent saved results. +// void SkippedCollection(); +// } +template +class CacheEntryStatsCollector { + public: + // Gather and save stats if saved stats are too old. (Use GetStats() to + // read saved stats.) + // + // Maximum allowed age for a "hit" on saved results is determined by the + // two interval parameters. Both set to 0 forces a re-scan. For example + // with min_interval_seconds=300 and min_interval_factor=100, if the last + // scan took 10s, we would only rescan ("miss") if the age in seconds of + // the saved results is > max(300, 100*10). + // Justification: scans can vary wildly in duration, e.g. from 0.02 sec + // to as much as 20 seconds, so we want to be able to cap the absolute + // and relative frequency of scans. + void CollectStats(int min_interval_seconds, int min_interval_factor) { + // Waits for any pending reader or writer (collector) + std::lock_guard lock(working_mutex_); + + uint64_t max_age_micros = + static_cast(std::max(min_interval_seconds, 0)) * 1000000U; + + if (last_end_time_micros_ > last_start_time_micros_ && + min_interval_factor > 0) { + max_age_micros = std::max( + max_age_micros, min_interval_factor * (last_end_time_micros_ - + last_start_time_micros_)); + } + + uint64_t start_time_micros = clock_->NowMicros(); + if ((start_time_micros - last_end_time_micros_) > max_age_micros) { + last_start_time_micros_ = start_time_micros; + working_stats_.BeginCollection(cache_, clock_, start_time_micros); + + cache_->ApplyToAllEntries(working_stats_.GetEntryCallback(), {}); + TEST_SYNC_POINT_CALLBACK( + "CacheEntryStatsCollector::GetStats:AfterApplyToAllEntries", nullptr); + + uint64_t end_time_micros = clock_->NowMicros(); + last_end_time_micros_ = end_time_micros; + working_stats_.EndCollection(cache_, clock_, end_time_micros); + } else { + working_stats_.SkippedCollection(); + } + + // Save so that we don't need to wait for an outstanding collection in + // order to make of copy of the last saved stats + std::lock_guard lock2(saved_mutex_); + saved_stats_ = working_stats_; + } + + // Gets saved stats, regardless of age + void GetStats(Stats *stats) { + std::lock_guard lock(saved_mutex_); + *stats = saved_stats_; + } + + Cache *GetCache() const { return cache_; } + + // Gets or creates a shared instance of CacheEntryStatsCollector in the + // cache itself, and saves into `ptr`. This shared_ptr will hold the + // entry in cache until all refs are destroyed. + static Status GetShared(Cache *raw_cache, SystemClock *clock, + std::shared_ptr *ptr) { + assert(raw_cache); + BasicTypedCacheInterface + cache{raw_cache}; + + const Slice &cache_key = GetCacheKey(); + auto h = cache.Lookup(cache_key); + if (h == nullptr) { + // Not yet in cache, but Cache doesn't provide a built-in way to + // avoid racing insert. So we double-check under a shared mutex, + // inspired by TableCache. + STATIC_AVOID_DESTRUCTION(std::mutex, static_mutex); + std::lock_guard lock(static_mutex); + + h = cache.Lookup(cache_key); + if (h == nullptr) { + auto new_ptr = new CacheEntryStatsCollector(cache.get(), clock); + // TODO: non-zero charge causes some tests that count block cache + // usage to go flaky. Fix the problem somehow so we can use an + // accurate charge. + size_t charge = 0; + Status s = + cache.Insert(cache_key, new_ptr, charge, &h, Cache::Priority::HIGH); + if (!s.ok()) { + assert(h == nullptr); + delete new_ptr; + return s; + } + } + } + // If we reach here, shared entry is in cache with handle `h`. + assert(cache.get()->GetCacheItemHelper(h) == cache.GetBasicHelper()); + + // Build an aliasing shared_ptr that keeps `ptr` in cache while there + // are references. + *ptr = cache.SharedGuard(h); + return Status::OK(); + } + + private: + explicit CacheEntryStatsCollector(Cache *cache, SystemClock *clock) + : saved_stats_(), + working_stats_(), + last_start_time_micros_(0), + last_end_time_micros_(/*pessimistic*/ 10000000), + cache_(cache), + clock_(clock) {} + + static const Slice &GetCacheKey() { + // For each template instantiation + static CacheKey ckey = CacheKey::CreateUniqueForProcessLifetime(); + static Slice ckey_slice = ckey.AsSlice(); + return ckey_slice; + } + + std::mutex saved_mutex_; + Stats saved_stats_; + + std::mutex working_mutex_; + Stats working_stats_; + uint64_t last_start_time_micros_; + uint64_t last_end_time_micros_; + + Cache *const cache_; + SystemClock *const clock_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/cache_helpers.cc b/librocksdb-sys/rocksdb/cache/cache_helpers.cc new file mode 100644 index 0000000..22597bf --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/cache_helpers.cc @@ -0,0 +1,40 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "cache/cache_helpers.h" + +namespace ROCKSDB_NAMESPACE { + +void ReleaseCacheHandleCleanup(void* arg1, void* arg2) { + Cache* const cache = static_cast(arg1); + assert(cache); + + Cache::Handle* const cache_handle = static_cast(arg2); + assert(cache_handle); + + cache->Release(cache_handle); +} + +Status WarmInCache(Cache* cache, const Slice& key, const Slice& saved, + Cache::CreateContext* create_context, + const Cache::CacheItemHelper* helper, + Cache::Priority priority, size_t* out_charge) { + assert(helper); + assert(helper->create_cb); + Cache::ObjectPtr value; + size_t charge; + Status st = helper->create_cb(saved, create_context, + cache->memory_allocator(), &value, &charge); + if (st.ok()) { + st = + cache->Insert(key, value, helper, charge, /*handle*/ nullptr, priority); + if (out_charge) { + *out_charge = charge; + } + } + return st; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/cache_helpers.h b/librocksdb-sys/rocksdb/cache/cache_helpers.h new file mode 100644 index 0000000..2dca6ec --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/cache_helpers.h @@ -0,0 +1,139 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "rocksdb/advanced_cache.h" +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +// Returns the cached value given a cache handle. +template +T* GetFromCacheHandle(Cache* cache, Cache::Handle* handle) { + assert(cache); + assert(handle); + return static_cast(cache->Value(handle)); +} + +// Turns a T* into a Slice so it can be used as a key with Cache. +template +Slice GetSliceForKey(const T* t) { + return Slice(reinterpret_cast(t), sizeof(T)); +} + +void ReleaseCacheHandleCleanup(void* arg1, void* arg2); + +// Generic resource management object for cache handles that releases the handle +// when destroyed. Has unique ownership of the handle, so copying it is not +// allowed, while moving it transfers ownership. +template +class CacheHandleGuard { + public: + CacheHandleGuard() = default; + + CacheHandleGuard(Cache* cache, Cache::Handle* handle) + : cache_(cache), + handle_(handle), + value_(GetFromCacheHandle(cache, handle)) { + assert(cache_ && handle_ && value_); + } + + CacheHandleGuard(const CacheHandleGuard&) = delete; + CacheHandleGuard& operator=(const CacheHandleGuard&) = delete; + + CacheHandleGuard(CacheHandleGuard&& rhs) noexcept + : cache_(rhs.cache_), handle_(rhs.handle_), value_(rhs.value_) { + assert((!cache_ && !handle_ && !value_) || (cache_ && handle_ && value_)); + + rhs.ResetFields(); + } + + CacheHandleGuard& operator=(CacheHandleGuard&& rhs) noexcept { + if (this == &rhs) { + return *this; + } + + ReleaseHandle(); + + cache_ = rhs.cache_; + handle_ = rhs.handle_; + value_ = rhs.value_; + + assert((!cache_ && !handle_ && !value_) || (cache_ && handle_ && value_)); + + rhs.ResetFields(); + + return *this; + } + + ~CacheHandleGuard() { ReleaseHandle(); } + + bool IsEmpty() const { return !handle_; } + + Cache* GetCache() const { return cache_; } + Cache::Handle* GetCacheHandle() const { return handle_; } + T* GetValue() const { return value_; } + + void TransferTo(Cleanable* cleanable) { + if (cleanable) { + if (handle_ != nullptr) { + assert(cache_); + cleanable->RegisterCleanup(&ReleaseCacheHandleCleanup, cache_, handle_); + } + } + ResetFields(); + } + + void Reset() { + ReleaseHandle(); + ResetFields(); + } + + private: + void ReleaseHandle() { + if (IsEmpty()) { + return; + } + + assert(cache_); + cache_->Release(handle_); + } + + void ResetFields() { + cache_ = nullptr; + handle_ = nullptr; + value_ = nullptr; + } + + private: + Cache* cache_ = nullptr; + Cache::Handle* handle_ = nullptr; + T* value_ = nullptr; +}; + +// Build an aliasing shared_ptr that keeps `handle` in cache while there +// are references, but the pointer is to the value for that cache entry, +// which must be of type T. This is copyable, unlike CacheHandleGuard, but +// does not provide access to caching details. +template +std::shared_ptr MakeSharedCacheHandleGuard(Cache* cache, + Cache::Handle* handle) { + auto wrapper = std::make_shared>(cache, handle); + return std::shared_ptr(wrapper, GetFromCacheHandle(cache, handle)); +} + +// Given the persistable data (saved) for a block cache entry, parse that +// into a cache entry object and insert it into the given cache. The charge +// of the new entry can be returned to the caller through `out_charge`. +Status WarmInCache(Cache* cache, const Slice& key, const Slice& saved, + Cache::CreateContext* create_context, + const Cache::CacheItemHelper* helper, + Cache::Priority priority = Cache::Priority::LOW, + size_t* out_charge = nullptr); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/cache_key.cc b/librocksdb-sys/rocksdb/cache/cache_key.cc new file mode 100644 index 0000000..addff61 --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/cache_key.cc @@ -0,0 +1,364 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "cache/cache_key.h" + +#include +#include + +#include "rocksdb/advanced_cache.h" +#include "table/unique_id_impl.h" +#include "util/hash.h" +#include "util/math.h" + +namespace ROCKSDB_NAMESPACE { + +// Value space plan for CacheKey: +// +// file_num_etc64_ | offset_etc64_ | Only generated by +// ---------------+---------------+------------------------------------------ +// 0 | 0 | Reserved for "empty" CacheKey() +// 0 | > 0, < 1<<63 | CreateUniqueForCacheLifetime +// 0 | >= 1<<63 | CreateUniqueForProcessLifetime +// > 0 | any | OffsetableCacheKey.WithOffset + +CacheKey CacheKey::CreateUniqueForCacheLifetime(Cache *cache) { + // +1 so that we can reserve all zeros for "unset" cache key + uint64_t id = cache->NewId() + 1; + // Ensure we don't collide with CreateUniqueForProcessLifetime + assert((id >> 63) == 0U); + return CacheKey(0, id); +} + +CacheKey CacheKey::CreateUniqueForProcessLifetime() { + // To avoid colliding with CreateUniqueForCacheLifetime, assuming + // Cache::NewId counts up from zero, here we count down from UINT64_MAX. + // If this ever becomes a point of contention, we could sub-divide the + // space and use CoreLocalArray. + static std::atomic counter{UINT64_MAX}; + uint64_t id = counter.fetch_sub(1, std::memory_order_relaxed); + // Ensure we don't collide with CreateUniqueForCacheLifetime + assert((id >> 63) == 1U); + return CacheKey(0, id); +} + +// How we generate CacheKeys and base OffsetableCacheKey, assuming that +// db_session_ids are generated from a base_session_id and +// session_id_counter (by SemiStructuredUniqueIdGen+EncodeSessionId +// in DBImpl::GenerateDbSessionId): +// +// Conceptual inputs: +// db_id (unstructured, from GenerateRawUniqueId or equiv) +// * could be shared between cloned DBs but rare +// * could be constant, if session id suffices +// base_session_id (unstructured, from GenerateRawUniqueId) +// session_id_counter (structured) +// * usually much smaller than 2**24 +// orig_file_number (structured) +// * usually smaller than 2**24 +// offset_in_file (structured, might skip lots of values) +// * usually smaller than 2**32 +// +// Overall approach (see https://github.com/pdillinger/unique_id for +// background): +// +// First, we have three "structured" values, up to 64 bits each, that we +// need to fit, without losses, into 128 bits. In practice, the values will +// be small enough that they should fit. For example, applications generating +// large SST files (large offsets) will naturally produce fewer files (small +// file numbers). But we don't know ahead of time what bounds the values will +// have. +// +// Second, we have unstructured inputs that enable distinct RocksDB processes +// to pick a random point in space, likely very different from others. Xoring +// the structured with the unstructured give us a cache key that is +// structurally distinct between related keys (e.g. same file or same RocksDB +// process) and distinct with high probability between unrelated keys. +// +// The problem of packing three structured values into the space for two is +// complicated by the fact that we want to derive cache keys from SST unique +// IDs, which have already combined structured and unstructured inputs in a +// practically inseparable way. And we want a base cache key that works +// with an offset of any size. So basically, we need to encode these three +// structured values, each up to 64 bits, into 128 bits without knowing any +// of their sizes. The DownwardInvolution() function gives us a mechanism to +// accomplish this. (See its properties in math.h.) Specifically, for inputs +// a, b, and c: +// lower64 = DownwardInvolution(a) ^ ReverseBits(b); +// upper64 = c ^ ReverseBits(a); +// The 128-bit output is unique assuming there exist some i, j, and k +// where a < 2**i, b < 2**j, c < 2**k, i <= 64, j <= 64, k <= 64, and +// i + j + k <= 128. In other words, as long as there exist some bounds +// that would allow us to pack the bits of a, b, and c into the output +// if we know the bound, we can generate unique outputs without knowing +// those bounds. To validate this claim, the inversion function (given +// the bounds) has been implemented in CacheKeyDecoder in +// db_block_cache_test.cc. +// +// With that in mind, the outputs in terms of the conceptual inputs look +// like this, using bitwise-xor of the constituent pieces, low bits on left: +// +// |------------------------- file_num_etc64 -------------------------| +// | +++++++++ base_session_id (lower 64 bits, involution) +++++++++ | +// |-----------------------------------------------------------------| +// | session_id_counter (involution) ..... | | +// |-----------------------------------------------------------------| +// | hash of: ++++++++++++++++++++++++++++++++++++++++++++++++++++++ | +// | * base_session_id (upper ~39 bits) | +// | * db_id (~122 bits entropy) | +// |-----------------------------------------------------------------| +// | | ..... orig_file_number (reversed) | +// |-----------------------------------------------------------------| +// +// +// |------------------------- offset_etc64 --------------------------| +// | ++++++++++ base_session_id (lower 64 bits, reversed) ++++++++++ | +// |-----------------------------------------------------------------| +// | | ..... session_id_counter (reversed) | +// |-----------------------------------------------------------------| +// | offset_in_file ............... | | +// |-----------------------------------------------------------------| +// +// Some oddities or inconveniences of this layout are due to deriving +// the "base" cache key (without offset) from the SST unique ID (see +// GetSstInternalUniqueId). Specifically, +// * Lower 64 of base_session_id occurs in both output words (ok but +// weird) +// * The inclusion of db_id is bad for the conditions under which we +// can guarantee uniqueness, but could be useful in some cases with +// few small files per process, to make up for db session id only having +// ~103 bits of entropy. +// +// In fact, if DB ids were not involved, we would be guaranteed unique +// cache keys for files generated in a single process until total bits for +// biggest session_id_counter, orig_file_number, and offset_in_file +// reach 128 bits. +// +// With the DB id limitation, we only have nice guaranteed unique cache +// keys for files generated in a single process until biggest +// session_id_counter and offset_in_file reach combined 64 bits. This +// is quite good in practice because we can have millions of DB Opens +// with terabyte size SST files, or billions of DB Opens with gigabyte +// size SST files. +// +// One of the considerations in the translation between existing SST unique +// IDs and base cache keys is supporting better SST unique IDs in a future +// format_version. If we use a process-wide file counter instead of +// session counter and file numbers, we only need to combine two 64-bit values +// instead of three. But we don't want to track unique ID versions in the +// manifest, so we want to keep the same translation layer between SST unique +// IDs and base cache keys, even with updated SST unique IDs. If the new +// unique IDs put the file counter where the orig_file_number was, and +// use no structured field where session_id_counter was, then our translation +// layer works fine for two structured fields as well as three (for +// compatibility). The small computation for the translation (one +// DownwardInvolution(), two ReverseBits(), both ~log(64) instructions deep) +// is negligible for computing as part of SST file reader open. +// +// More on how https://github.com/pdillinger/unique_id applies here: +// Every bit of output always includes "unstructured" uniqueness bits and +// often combines with "structured" uniqueness bits. The "unstructured" bits +// change infrequently: only when we cannot guarantee our state tracking for +// "structured" uniqueness hasn't been cloned. Using a static +// SemiStructuredUniqueIdGen for db_session_ids, this means we only get an +// "all new" session id when a new process uses RocksDB. (Between processes, +// we don't know if a DB or other persistent storage has been cloned. We +// assume that if VM hot cloning is used, subsequently generated SST files +// do not interact.) Within a process, only the session_lower of the +// db_session_id changes incrementally ("structured" uniqueness). +// +// This basically means that our offsets, counters and file numbers allow us +// to do somewhat "better than random" (birthday paradox) while in the +// degenerate case of completely new session for each tiny file, we still +// have strong uniqueness properties from the birthday paradox, with ~103 +// bit session IDs or up to 128 bits entropy with different DB IDs sharing a +// cache. +// +// More collision probability analysis: +// Suppose a RocksDB host generates (generously) 2 GB/s (10TB data, 17 DWPD) +// with average process/session lifetime of (pessimistically) 4 minutes. +// In 180 days (generous allowable data lifespan), we generate 31 million GB +// of data, or 2^55 bytes, and 2^16 "all new" session IDs. +// +// First, suppose this is in a single DB (lifetime 180 days): +// 128 bits cache key size +// - 55 <- ideal size for byte offsets + file numbers +// - 2 <- bits for offsets and file numbers not exactly powers of two +// + 2 <- bits saved not using byte offsets in BlockBasedTable::GetCacheKey +// ---- +// 73 <- bits remaining for distinguishing session IDs +// The probability of a collision in 73 bits of session ID data is less than +// 1 in 2**(73 - (2 * 16)), or roughly 1 in a trillion. And this assumes all +// data from the last 180 days is in cache for potential collision, and that +// cache keys under each session id exhaustively cover the remaining 57 bits +// while in reality they'll only cover a small fraction of it. +// +// Although data could be transferred between hosts, each host has its own +// cache and we are already assuming a high rate of "all new" session ids. +// So this doesn't really change the collision calculation. Across a fleet +// of 1 million, each with <1 in a trillion collision possibility, +// fleetwide collision probability is <1 in a million. +// +// Now suppose we have many DBs per host, say 2**10, with same host-wide write +// rate and process/session lifetime. File numbers will be ~10 bits smaller +// and we will have 2**10 times as many session IDs because of simultaneous +// lifetimes. So now collision chance is less than 1 in 2**(83 - (2 * 26)), +// or roughly 1 in a billion. +// +// Suppose instead we generated random or hashed cache keys for each +// (compressed) block. For 1KB compressed block size, that is 2^45 cache keys +// in 180 days. Collision probability is more easily estimated at roughly +// 1 in 2**(128 - (2 * 45)) or roughly 1 in a trillion (assuming all +// data from the last 180 days is in cache, but NOT the other assumption +// for the 1 in a trillion estimate above). +// +// +// Collision probability estimation through simulation: +// A tool ./cache_bench -stress_cache_key broadly simulates host-wide cache +// activity over many months, by making some pessimistic simplifying +// assumptions. See class StressCacheKey in cache_bench_tool.cc for details. +// Here is some sample output with +// `./cache_bench -stress_cache_key -sck_keep_bits=43`: +// +// Total cache or DBs size: 32TiB Writing 925.926 MiB/s or 76.2939TiB/day +// Multiply by 1.15292e+18 to correct for simulation losses (but still +// assume whole file cached) +// +// These come from default settings of 2.5M files per day of 32 MB each, and +// `-sck_keep_bits=43` means that to represent a single file, we are only +// keeping 43 bits of the 128-bit (base) cache key. With file size of 2**25 +// contiguous keys (pessimistic), our simulation is about 2\*\*(128-43-25) or +// about 1 billion billion times more prone to collision than reality. +// +// More default assumptions, relatively pessimistic: +// * 100 DBs in same process (doesn't matter much) +// * Re-open DB in same process (new session ID related to old session ID) on +// average every 100 files generated +// * Restart process (all new session IDs unrelated to old) 24 times per day +// +// After enough data, we get a result at the end (-sck_keep_bits=43): +// +// (keep 43 bits) 18 collisions after 2 x 90 days, est 10 days between +// (1.15292e+19 corrected) +// +// If we believe the (pessimistic) simulation and the mathematical +// extrapolation, we would need to run a billion machines all for 11 billion +// days to expect a cache key collision. To help verify that our extrapolation +// ("corrected") is robust, we can make our simulation more precise by +// increasing the "keep" bits, which takes more running time to get enough +// collision data: +// +// (keep 44 bits) 16 collisions after 5 x 90 days, est 28.125 days between +// (1.6213e+19 corrected) +// (keep 45 bits) 15 collisions after 7 x 90 days, est 42 days between +// (1.21057e+19 corrected) +// (keep 46 bits) 15 collisions after 17 x 90 days, est 102 days between +// (1.46997e+19 corrected) +// (keep 47 bits) 15 collisions after 49 x 90 days, est 294 days between +// (2.11849e+19 corrected) +// +// The extrapolated prediction seems to be within noise (sampling error). +// +// With the `-sck_randomize` option, we can see that typical workloads like +// above have lower collision probability than "random" cache keys (note: +// offsets still non-randomized) by a modest amount (roughly 2-3x less +// collision prone than random), which should make us reasonably comfortable +// even in "degenerate" cases (e.g. repeatedly launch a process to generate +// one file with SstFileWriter): +// +// (rand 43 bits) 22 collisions after 1 x 90 days, est 4.09091 days between +// (4.7165e+18 corrected) +// +// We can see that with more frequent process restarts, +// -sck_restarts_per_day=5000, which means more all-new session IDs, we get +// closer to the "random" cache key performance: +// +// 15 collisions after 1 x 90 days, est 6 days between (6.91753e+18 corrected) +// +// And with less frequent process restarts and re-opens, +// -sck_restarts_per_day=1 -sck_reopen_nfiles=1000, we get lower collision +// probability: +// +// 18 collisions after 8 x 90 days, est 40 days between (4.61169e+19 corrected) +// +// Other tests have been run to validate other conditions behave as expected, +// never behaving "worse than random" unless we start chopping off structured +// data. +// +// Conclusion: Even in extreme cases, rapidly burning through "all new" IDs +// that only arise when a new process is started, the chance of any cache key +// collisions in a giant fleet of machines is negligible. Especially when +// processes live for hours or days, the chance of a cache key collision is +// likely more plausibly due to bad hardware than to bad luck in random +// session ID data. Software defects are surely more likely to cause corruption +// than both of those. +// +// TODO: Nevertheless / regardless, an efficient way to detect (and thus +// quantify) block cache corruptions, including collisions, should be added. +OffsetableCacheKey::OffsetableCacheKey(const std::string &db_id, + const std::string &db_session_id, + uint64_t file_number) { + UniqueId64x2 internal_id; + Status s = GetSstInternalUniqueId(db_id, db_session_id, file_number, + &internal_id, /*force=*/true); + assert(s.ok()); + *this = FromInternalUniqueId(&internal_id); +} + +OffsetableCacheKey OffsetableCacheKey::FromInternalUniqueId(UniqueIdPtr id) { + uint64_t session_lower = id.ptr[0]; + uint64_t file_num_etc = id.ptr[1]; + +#ifndef NDEBUG + bool is_empty = session_lower == 0 && file_num_etc == 0; +#endif + + // Although DBImpl guarantees (in recent versions) that session_lower is not + // zero, that's not entirely sufficient to guarantee that file_num_etc64_ is + // not zero (so that the 0 case can be used by CacheKey::CreateUnique*) + // However, if we are given an "empty" id as input, then we should produce + // "empty" as output. + // As a consequence, this function is only bijective assuming + // id[0] == 0 only if id[1] == 0. + if (session_lower == 0U) { + session_lower = file_num_etc; + } + + // See comments above for how DownwardInvolution and ReverseBits + // make this function invertible under various assumptions. + OffsetableCacheKey rv; + rv.file_num_etc64_ = + DownwardInvolution(session_lower) ^ ReverseBits(file_num_etc); + rv.offset_etc64_ = ReverseBits(session_lower); + + // Because of these transformations and needing to allow arbitrary + // offset (thus, second 64 bits of cache key might be 0), we need to + // make some correction to ensure the first 64 bits is not 0. + // Fortunately, the transformation ensures the second 64 bits is not 0 + // for non-empty base key, so we can swap in the case one is 0 without + // breaking bijectivity (assuming condition above). + assert(is_empty || rv.offset_etc64_ > 0); + if (rv.file_num_etc64_ == 0) { + std::swap(rv.file_num_etc64_, rv.offset_etc64_); + } + assert(is_empty || rv.file_num_etc64_ > 0); + return rv; +} + +// Inverse of FromInternalUniqueId (assuming file_num_etc64 == 0 only if +// offset_etc64 == 0) +UniqueId64x2 OffsetableCacheKey::ToInternalUniqueId() { + uint64_t a = file_num_etc64_; + uint64_t b = offset_etc64_; + if (b == 0) { + std::swap(a, b); + } + UniqueId64x2 rv; + rv[0] = ReverseBits(b); + rv[1] = ReverseBits(a ^ DownwardInvolution(rv[0])); + return rv; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/cache_key.h b/librocksdb-sys/rocksdb/cache/cache_key.h new file mode 100644 index 0000000..0b93c6b --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/cache_key.h @@ -0,0 +1,143 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/slice.h" +#include "table/unique_id_impl.h" + +namespace ROCKSDB_NAMESPACE { + +class Cache; + +// A standard holder for fixed-size block cache keys (and for related caches). +// They are created through one of these, each using its own range of values: +// * CacheKey::CreateUniqueForCacheLifetime +// * CacheKey::CreateUniqueForProcessLifetime +// * Default ctor ("empty" cache key) +// * OffsetableCacheKey->WithOffset +// +// The first two use atomic counters to guarantee uniqueness over the given +// lifetime and the last uses a form of universally unique identifier for +// uniqueness with very high probabilty (and guaranteed for files generated +// during a single process lifetime). +// +// CacheKeys are currently used by calling AsSlice() to pass as a key to +// Cache. For performance, the keys are endianness-dependent (though otherwise +// portable). (Persistable cache entries are not intended to cross platforms.) +class CacheKey { + public: + // For convenience, constructs an "empty" cache key that is never returned + // by other means. + inline CacheKey() : file_num_etc64_(), offset_etc64_() {} + + inline bool IsEmpty() const { + return (file_num_etc64_ == 0) & (offset_etc64_ == 0); + } + + // Use this cache key as a Slice (byte order is endianness-dependent) + inline Slice AsSlice() const { + static_assert(sizeof(*this) == 16, "Standardized on 16-byte cache key"); + assert(!IsEmpty()); + return Slice(reinterpret_cast(this), sizeof(*this)); + } + + // Create a CacheKey that is unique among others associated with this Cache + // instance. Depends on Cache::NewId. This is useful for block cache + // "reservations". + static CacheKey CreateUniqueForCacheLifetime(Cache *cache); + + // Create a CacheKey that is unique among others for the lifetime of this + // process. This is useful for saving in a static data member so that + // different DB instances can agree on a cache key for shared entities, + // such as for CacheEntryStatsCollector. + static CacheKey CreateUniqueForProcessLifetime(); + + protected: + friend class OffsetableCacheKey; + CacheKey(uint64_t file_num_etc64, uint64_t offset_etc64) + : file_num_etc64_(file_num_etc64), offset_etc64_(offset_etc64) {} + uint64_t file_num_etc64_; + uint64_t offset_etc64_; +}; + +constexpr uint8_t kCacheKeySize = static_cast(sizeof(CacheKey)); + +// A file-specific generator of cache keys, sometimes referred to as the +// "base" cache key for a file because all the cache keys for various offsets +// within the file are computed using simple arithmetic. The basis for the +// general approach is dicussed here: https://github.com/pdillinger/unique_id +// Heavily related to GetUniqueIdFromTableProperties. +// +// If the db_id, db_session_id, and file_number come from the file's table +// properties, then the keys will be stable across DB::Open/Close, backup/ +// restore, import/export, etc. +// +// This class "is a" CacheKey only privately so that it is not misused as +// a ready-to-use CacheKey. +class OffsetableCacheKey : private CacheKey { + public: + // For convenience, constructs an "empty" cache key that should not be used. + inline OffsetableCacheKey() : CacheKey() {} + + // Constructs an OffsetableCacheKey with the given information about a file. + // This constructor never generates an "empty" base key. + OffsetableCacheKey(const std::string &db_id, const std::string &db_session_id, + uint64_t file_number); + + // Creates an OffsetableCacheKey from an SST unique ID, so that cache keys + // can be derived from DB manifest data before reading the file from + // storage--so that every part of the file can potentially go in a persistent + // cache. + // + // Calling GetSstInternalUniqueId() on a db_id, db_session_id, and + // file_number and passing the result to this function produces the same + // base cache key as feeding those inputs directly to the constructor. + // + // This is a bijective transformation assuming either id is empty or + // lower 64 bits is non-zero: + // * Empty (all zeros) input -> empty (all zeros) output + // * Lower 64 input is non-zero -> lower 64 output (file_num_etc64_) is + // non-zero + static OffsetableCacheKey FromInternalUniqueId(UniqueIdPtr id); + + // This is the inverse transformation to the above, assuming either empty + // or lower 64 bits (file_num_etc64_) is non-zero. Perhaps only useful for + // testing. + UniqueId64x2 ToInternalUniqueId(); + + inline bool IsEmpty() const { + bool result = file_num_etc64_ == 0; + assert(!(offset_etc64_ > 0 && result)); + return result; + } + + // Construct a CacheKey for an offset within a file. An offset is not + // necessarily a byte offset if a smaller unique identifier of keyable + // offsets is used. + // + // This class was designed to make this hot code extremely fast. + inline CacheKey WithOffset(uint64_t offset) const { + assert(!IsEmpty()); + return CacheKey(file_num_etc64_, offset_etc64_ ^ offset); + } + + // The "common prefix" is a shared prefix for all the returned CacheKeys. + // It is specific to the file but the same for all offsets within the file. + static constexpr size_t kCommonPrefixSize = 8; + inline Slice CommonPrefixSlice() const { + static_assert(sizeof(file_num_etc64_) == kCommonPrefixSize, + "8 byte common prefix expected"); + assert(!IsEmpty()); + assert(&this->file_num_etc64_ == static_cast(this)); + + return Slice(reinterpret_cast(this), kCommonPrefixSize); + } +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/cache_reservation_manager.cc b/librocksdb-sys/rocksdb/cache/cache_reservation_manager.cc new file mode 100644 index 0000000..2a4be42 --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/cache_reservation_manager.cc @@ -0,0 +1,184 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "cache/cache_reservation_manager.h" + +#include +#include +#include +#include + +#include "rocksdb/cache.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "table/block_based/reader_common.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { + +template +CacheReservationManagerImpl::CacheReservationHandle::CacheReservationHandle( + std::size_t incremental_memory_used, + std::shared_ptr cache_res_mgr) + : incremental_memory_used_(incremental_memory_used) { + assert(cache_res_mgr); + cache_res_mgr_ = cache_res_mgr; +} + +template +CacheReservationManagerImpl< + R>::CacheReservationHandle::~CacheReservationHandle() { + Status s = cache_res_mgr_->ReleaseCacheReservation(incremental_memory_used_); + s.PermitUncheckedError(); +} + +template +CacheReservationManagerImpl::CacheReservationManagerImpl( + std::shared_ptr cache, bool delayed_decrease) + : cache_(cache), + delayed_decrease_(delayed_decrease), + cache_allocated_size_(0), + memory_used_(0) { + assert(cache != nullptr); +} + +template +CacheReservationManagerImpl::~CacheReservationManagerImpl() { + for (auto* handle : dummy_handles_) { + cache_.ReleaseAndEraseIfLastRef(handle); + } +} + +template +Status CacheReservationManagerImpl::UpdateCacheReservation( + std::size_t new_mem_used) { + memory_used_ = new_mem_used; + std::size_t cur_cache_allocated_size = + cache_allocated_size_.load(std::memory_order_relaxed); + if (new_mem_used == cur_cache_allocated_size) { + return Status::OK(); + } else if (new_mem_used > cur_cache_allocated_size) { + Status s = IncreaseCacheReservation(new_mem_used); + return s; + } else { + // In delayed decrease mode, we don't decrease cache reservation + // untill the memory usage is less than 3/4 of what we reserve + // in the cache. + // We do this because + // (1) Dummy entry insertion is expensive in block cache + // (2) Delayed releasing previously inserted dummy entries can save such + // expensive dummy entry insertion on memory increase in the near future, + // which is likely to happen when the memory usage is greater than or equal + // to 3/4 of what we reserve + if (delayed_decrease_ && new_mem_used >= cur_cache_allocated_size / 4 * 3) { + return Status::OK(); + } else { + Status s = DecreaseCacheReservation(new_mem_used); + return s; + } + } +} + +template +Status CacheReservationManagerImpl::MakeCacheReservation( + std::size_t incremental_memory_used, + std::unique_ptr* handle) { + assert(handle); + Status s = + UpdateCacheReservation(GetTotalMemoryUsed() + incremental_memory_used); + (*handle).reset(new CacheReservationManagerImpl::CacheReservationHandle( + incremental_memory_used, + std::enable_shared_from_this< + CacheReservationManagerImpl>::shared_from_this())); + return s; +} + +template +Status CacheReservationManagerImpl::ReleaseCacheReservation( + std::size_t incremental_memory_used) { + assert(GetTotalMemoryUsed() >= incremental_memory_used); + std::size_t updated_total_mem_used = + GetTotalMemoryUsed() - incremental_memory_used; + Status s = UpdateCacheReservation(updated_total_mem_used); + return s; +} + +template +Status CacheReservationManagerImpl::IncreaseCacheReservation( + std::size_t new_mem_used) { + Status return_status = Status::OK(); + while (new_mem_used > cache_allocated_size_.load(std::memory_order_relaxed)) { + Cache::Handle* handle = nullptr; + return_status = cache_.Insert(GetNextCacheKey(), kSizeDummyEntry, &handle); + + if (return_status != Status::OK()) { + return return_status; + } + + dummy_handles_.push_back(handle); + cache_allocated_size_ += kSizeDummyEntry; + } + return return_status; +} + +template +Status CacheReservationManagerImpl::DecreaseCacheReservation( + std::size_t new_mem_used) { + Status return_status = Status::OK(); + + // Decrease to the smallest multiple of kSizeDummyEntry that is greater than + // or equal to new_mem_used We do addition instead of new_mem_used <= + // cache_allocated_size_.load(std::memory_order_relaxed) - kSizeDummyEntry to + // avoid underflow of size_t when cache_allocated_size_ = 0 + while (new_mem_used + kSizeDummyEntry <= + cache_allocated_size_.load(std::memory_order_relaxed)) { + assert(!dummy_handles_.empty()); + auto* handle = dummy_handles_.back(); + cache_.ReleaseAndEraseIfLastRef(handle); + dummy_handles_.pop_back(); + cache_allocated_size_ -= kSizeDummyEntry; + } + return return_status; +} + +template +std::size_t CacheReservationManagerImpl::GetTotalReservedCacheSize() { + return cache_allocated_size_.load(std::memory_order_relaxed); +} + +template +std::size_t CacheReservationManagerImpl::GetTotalMemoryUsed() { + return memory_used_; +} + +template +Slice CacheReservationManagerImpl::GetNextCacheKey() { + // Calling this function will have the side-effect of changing the + // underlying cache_key_ that is shared among other keys generated from this + // fucntion. Therefore please make sure the previous keys are saved/copied + // before calling this function. + cache_key_ = CacheKey::CreateUniqueForCacheLifetime(cache_.get()); + return cache_key_.AsSlice(); +} + +template +const Cache::CacheItemHelper* +CacheReservationManagerImpl::TEST_GetCacheItemHelperForRole() { + return CacheInterface::GetHelper(); +} + +template class CacheReservationManagerImpl< + CacheEntryRole::kBlockBasedTableReader>; +template class CacheReservationManagerImpl< + CacheEntryRole::kCompressionDictionaryBuildingBuffer>; +template class CacheReservationManagerImpl; +template class CacheReservationManagerImpl; +template class CacheReservationManagerImpl; +template class CacheReservationManagerImpl; +template class CacheReservationManagerImpl; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/cache_reservation_manager.h b/librocksdb-sys/rocksdb/cache/cache_reservation_manager.h new file mode 100644 index 0000000..08bf59b --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/cache_reservation_manager.h @@ -0,0 +1,317 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "cache/cache_entry_roles.h" +#include "cache/cache_key.h" +#include "cache/typed_cache.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { +// CacheReservationManager is an interface for reserving cache space for the +// memory used +class CacheReservationManager { + public: + // CacheReservationHandle is for managing the lifetime of a cache reservation + // for an incremental amount of memory used (i.e, incremental_memory_used) + class CacheReservationHandle { + public: + virtual ~CacheReservationHandle() {} + }; + virtual ~CacheReservationManager() {} + virtual Status UpdateCacheReservation(std::size_t new_memory_used) = 0; + // TODO(hx235): replace the usage of + // `UpdateCacheReservation(memory_used_delta, increase)` with + // `UpdateCacheReservation(new_memory_used)` so that we only have one + // `UpdateCacheReservation` function + virtual Status UpdateCacheReservation(std::size_t memory_used_delta, + bool increase) = 0; + virtual Status MakeCacheReservation( + std::size_t incremental_memory_used, + std::unique_ptr + *handle) = 0; + virtual std::size_t GetTotalReservedCacheSize() = 0; + virtual std::size_t GetTotalMemoryUsed() = 0; +}; + +// CacheReservationManagerImpl implements interface CacheReservationManager +// for reserving cache space for the memory used by inserting/releasing dummy +// entries in the cache. +// +// This class is NOT thread-safe, except that GetTotalReservedCacheSize() +// can be called without external synchronization. +template +class CacheReservationManagerImpl + : public CacheReservationManager, + public std::enable_shared_from_this> { + public: + class CacheReservationHandle + : public CacheReservationManager::CacheReservationHandle { + public: + CacheReservationHandle( + std::size_t incremental_memory_used, + std::shared_ptr cache_res_mgr); + ~CacheReservationHandle() override; + + private: + std::size_t incremental_memory_used_; + std::shared_ptr cache_res_mgr_; + }; + + // Construct a CacheReservationManagerImpl + // @param cache The cache where dummy entries are inserted and released for + // reserving cache space + // @param delayed_decrease If set true, then dummy entries won't be released + // immediately when memory usage decreases. + // Instead, it will be released when the memory usage + // decreases to 3/4 of what we have reserved so far. + // This is for saving some future dummy entry + // insertion when memory usage increases are likely to + // happen in the near future. + // + // REQUIRED: cache is not nullptr + explicit CacheReservationManagerImpl(std::shared_ptr cache, + bool delayed_decrease = false); + + // no copy constructor, copy assignment, move constructor, move assignment + CacheReservationManagerImpl(const CacheReservationManagerImpl &) = delete; + CacheReservationManagerImpl &operator=(const CacheReservationManagerImpl &) = + delete; + CacheReservationManagerImpl(CacheReservationManagerImpl &&) = delete; + CacheReservationManagerImpl &operator=(CacheReservationManagerImpl &&) = + delete; + + ~CacheReservationManagerImpl() override; + + // One of the two ways of reserving/releasing cache space, + // see MakeCacheReservation() for the other. + // + // Use ONLY one of these two ways to prevent unexpected behavior. + // + // Insert and release dummy entries in the cache to + // match the size of total dummy entries with the least multiple of + // kSizeDummyEntry greater than or equal to new_mem_used + // + // Insert dummy entries if new_memory_used > cache_allocated_size_; + // + // Release dummy entries if new_memory_used < cache_allocated_size_ + // (and new_memory_used < cache_allocated_size_ * 3/4 + // when delayed_decrease is set true); + // + // Keey dummy entries the same if (1) new_memory_used == cache_allocated_size_ + // or (2) new_memory_used is in the interval of + // [cache_allocated_size_ * 3/4, cache_allocated_size) when delayed_decrease + // is set true. + // + // @param new_memory_used The number of bytes used by new memory + // The most recent new_memoy_used passed in will be returned + // in GetTotalMemoryUsed() even when the call return non-ok status. + // + // Since the class is NOT thread-safe, external synchronization on the + // order of calling UpdateCacheReservation() is needed if you want + // GetTotalMemoryUsed() indeed returns the latest memory used. + // + // @return On inserting dummy entries, it returns Status::OK() if all dummy + // entry insertions succeed. + // Otherwise, it returns the first non-ok status; + // On releasing dummy entries, it always returns Status::OK(). + // On keeping dummy entries the same, it always returns Status::OK(). + Status UpdateCacheReservation(std::size_t new_memory_used) override; + + Status UpdateCacheReservation(std::size_t /* memory_used_delta */, + bool /* increase */) override { + return Status::NotSupported(); + } + + // One of the two ways of reserving cache space and releasing is done through + // destruction of CacheReservationHandle. + // See UpdateCacheReservation() for the other way. + // + // Use ONLY one of these two ways to prevent unexpected behavior. + // + // Insert dummy entries in the cache for the incremental memory usage + // to match the size of total dummy entries with the least multiple of + // kSizeDummyEntry greater than or equal to the total memory used. + // + // A CacheReservationHandle is returned as an output parameter. + // The reserved dummy entries are automatically released on the destruction of + // this handle, which achieves better RAII per cache reservation. + // + // WARNING: Deallocate all the handles of the CacheReservationManager object + // before deallocating the object to prevent unexpected behavior. + // + // @param incremental_memory_used The number of bytes increased in memory + // usage. + // + // Calling GetTotalMemoryUsed() afterward will return the total memory + // increased by this number, even when calling MakeCacheReservation() + // returns non-ok status. + // + // Since the class is NOT thread-safe, external synchronization in + // calling MakeCacheReservation() is needed if you want + // GetTotalMemoryUsed() indeed returns the latest memory used. + // + // @param handle An pointer to std::unique_ptr that + // manages the lifetime of the cache reservation represented by the + // handle. + // + // @return It returns Status::OK() if all dummy + // entry insertions succeed. + // Otherwise, it returns the first non-ok status; + // + // REQUIRES: handle != nullptr + Status MakeCacheReservation( + std::size_t incremental_memory_used, + std::unique_ptr *handle) + override; + + // Return the size of the cache (which is a multiple of kSizeDummyEntry) + // successfully reserved by calling UpdateCacheReservation(). + // + // When UpdateCacheReservation() returns non-ok status, + // calling GetTotalReservedCacheSize() after that might return a slightly + // smaller number than the actual reserved cache size due to + // the returned number will always be a multiple of kSizeDummyEntry + // and cache full might happen in the middle of inserting a dummy entry. + std::size_t GetTotalReservedCacheSize() override; + + // Return the latest total memory used indicated by the most recent call of + // UpdateCacheReservation(std::size_t new_memory_used); + std::size_t GetTotalMemoryUsed() override; + + static constexpr std::size_t GetDummyEntrySize() { return kSizeDummyEntry; } + + // For testing only - it is to help ensure the CacheItemHelperForRole + // accessed from CacheReservationManagerImpl and the one accessed from the + // test are from the same translation units + static const Cache::CacheItemHelper *TEST_GetCacheItemHelperForRole(); + + private: + static constexpr std::size_t kSizeDummyEntry = 256 * 1024; + + Slice GetNextCacheKey(); + + Status ReleaseCacheReservation(std::size_t incremental_memory_used); + Status IncreaseCacheReservation(std::size_t new_mem_used); + Status DecreaseCacheReservation(std::size_t new_mem_used); + + using CacheInterface = PlaceholderSharedCacheInterface; + CacheInterface cache_; + bool delayed_decrease_; + std::atomic cache_allocated_size_; + std::size_t memory_used_; + std::vector dummy_handles_; + CacheKey cache_key_; +}; + +class ConcurrentCacheReservationManager + : public CacheReservationManager, + public std::enable_shared_from_this { + public: + class CacheReservationHandle + : public CacheReservationManager::CacheReservationHandle { + public: + CacheReservationHandle( + std::shared_ptr cache_res_mgr, + std::unique_ptr + cache_res_handle) { + assert(cache_res_mgr && cache_res_handle); + cache_res_mgr_ = cache_res_mgr; + cache_res_handle_ = std::move(cache_res_handle); + } + + ~CacheReservationHandle() override { + std::lock_guard lock(cache_res_mgr_->cache_res_mgr_mu_); + cache_res_handle_.reset(); + } + + private: + std::shared_ptr cache_res_mgr_; + std::unique_ptr + cache_res_handle_; + }; + + explicit ConcurrentCacheReservationManager( + std::shared_ptr cache_res_mgr) { + cache_res_mgr_ = std::move(cache_res_mgr); + } + ConcurrentCacheReservationManager(const ConcurrentCacheReservationManager &) = + delete; + ConcurrentCacheReservationManager &operator=( + const ConcurrentCacheReservationManager &) = delete; + ConcurrentCacheReservationManager(ConcurrentCacheReservationManager &&) = + delete; + ConcurrentCacheReservationManager &operator=( + ConcurrentCacheReservationManager &&) = delete; + + ~ConcurrentCacheReservationManager() override {} + + inline Status UpdateCacheReservation(std::size_t new_memory_used) override { + std::lock_guard lock(cache_res_mgr_mu_); + return cache_res_mgr_->UpdateCacheReservation(new_memory_used); + } + + inline Status UpdateCacheReservation(std::size_t memory_used_delta, + bool increase) override { + std::lock_guard lock(cache_res_mgr_mu_); + std::size_t total_mem_used = cache_res_mgr_->GetTotalMemoryUsed(); + Status s; + if (!increase) { + assert(total_mem_used >= memory_used_delta); + s = cache_res_mgr_->UpdateCacheReservation(total_mem_used - + memory_used_delta); + } else { + s = cache_res_mgr_->UpdateCacheReservation(total_mem_used + + memory_used_delta); + } + return s; + } + + inline Status MakeCacheReservation( + std::size_t incremental_memory_used, + std::unique_ptr *handle) + override { + std::unique_ptr + wrapped_handle; + Status s; + { + std::lock_guard lock(cache_res_mgr_mu_); + s = cache_res_mgr_->MakeCacheReservation(incremental_memory_used, + &wrapped_handle); + } + (*handle).reset( + new ConcurrentCacheReservationManager::CacheReservationHandle( + std::enable_shared_from_this< + ConcurrentCacheReservationManager>::shared_from_this(), + std::move(wrapped_handle))); + return s; + } + inline std::size_t GetTotalReservedCacheSize() override { + return cache_res_mgr_->GetTotalReservedCacheSize(); + } + inline std::size_t GetTotalMemoryUsed() override { + std::lock_guard lock(cache_res_mgr_mu_); + return cache_res_mgr_->GetTotalMemoryUsed(); + } + + private: + std::mutex cache_res_mgr_mu_; + std::shared_ptr cache_res_mgr_; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/cache_reservation_manager_test.cc b/librocksdb-sys/rocksdb/cache/cache_reservation_manager_test.cc new file mode 100644 index 0000000..2a0c318 --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/cache_reservation_manager_test.cc @@ -0,0 +1,469 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "cache/cache_reservation_manager.h" + +#include +#include +#include + +#include "cache/cache_entry_roles.h" +#include "rocksdb/cache.h" +#include "rocksdb/slice.h" +#include "test_util/testharness.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { +class CacheReservationManagerTest : public ::testing::Test { + protected: + static constexpr std::size_t kSizeDummyEntry = + CacheReservationManagerImpl::GetDummyEntrySize(); + static constexpr std::size_t kCacheCapacity = 4096 * kSizeDummyEntry; + static constexpr int kNumShardBits = 0; // 2^0 shard + static constexpr std::size_t kMetaDataChargeOverhead = 10000; + + std::shared_ptr cache = NewLRUCache(kCacheCapacity, kNumShardBits); + std::shared_ptr test_cache_rev_mng; + + CacheReservationManagerTest() { + test_cache_rev_mng = + std::make_shared>( + cache); + } +}; + +TEST_F(CacheReservationManagerTest, GenerateCacheKey) { + std::size_t new_mem_used = 1 * kSizeDummyEntry; + Status s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used); + ASSERT_EQ(s, Status::OK()); + ASSERT_GE(cache->GetPinnedUsage(), 1 * kSizeDummyEntry); + ASSERT_LT(cache->GetPinnedUsage(), + 1 * kSizeDummyEntry + kMetaDataChargeOverhead); + + // Next unique Cache key + CacheKey ckey = CacheKey::CreateUniqueForCacheLifetime(cache.get()); + // Get to the underlying values + uint64_t* ckey_data = reinterpret_cast(&ckey); + // Back it up to the one used by CRM (using CacheKey implementation details) + ckey_data[1]--; + + // Specific key (subject to implementation details) + EXPECT_EQ(ckey_data[0], 0); + EXPECT_EQ(ckey_data[1], 2); + + Cache::Handle* handle = cache->Lookup(ckey.AsSlice()); + EXPECT_NE(handle, nullptr) + << "Failed to generate the cache key for the dummy entry correctly"; + // Clean up the returned handle from Lookup() to prevent memory leak + cache->Release(handle); +} + +TEST_F(CacheReservationManagerTest, KeepCacheReservationTheSame) { + std::size_t new_mem_used = 1 * kSizeDummyEntry; + Status s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used); + ASSERT_EQ(s, Status::OK()); + ASSERT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(), + 1 * kSizeDummyEntry); + ASSERT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used); + std::size_t initial_pinned_usage = cache->GetPinnedUsage(); + ASSERT_GE(initial_pinned_usage, 1 * kSizeDummyEntry); + ASSERT_LT(initial_pinned_usage, + 1 * kSizeDummyEntry + kMetaDataChargeOverhead); + + s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used); + EXPECT_EQ(s, Status::OK()) + << "Failed to keep cache reservation the same when new_mem_used equals " + "to current cache reservation"; + EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(), + 1 * kSizeDummyEntry) + << "Failed to bookkeep correctly when new_mem_used equals to current " + "cache reservation"; + EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used) + << "Failed to bookkeep the used memory correctly when new_mem_used " + "equals to current cache reservation"; + EXPECT_EQ(cache->GetPinnedUsage(), initial_pinned_usage) + << "Failed to keep underlying dummy entries the same when new_mem_used " + "equals to current cache reservation"; +} + +TEST_F(CacheReservationManagerTest, + IncreaseCacheReservationByMultiplesOfDummyEntrySize) { + std::size_t new_mem_used = 2 * kSizeDummyEntry; + Status s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used); + EXPECT_EQ(s, Status::OK()) + << "Failed to increase cache reservation correctly"; + EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(), + 2 * kSizeDummyEntry) + << "Failed to bookkeep cache reservation increase correctly"; + EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used) + << "Failed to bookkeep the used memory correctly"; + EXPECT_GE(cache->GetPinnedUsage(), 2 * kSizeDummyEntry) + << "Failed to increase underlying dummy entries in cache correctly"; + EXPECT_LT(cache->GetPinnedUsage(), + 2 * kSizeDummyEntry + kMetaDataChargeOverhead) + << "Failed to increase underlying dummy entries in cache correctly"; +} + +TEST_F(CacheReservationManagerTest, + IncreaseCacheReservationNotByMultiplesOfDummyEntrySize) { + std::size_t new_mem_used = 2 * kSizeDummyEntry + kSizeDummyEntry / 2; + Status s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used); + EXPECT_EQ(s, Status::OK()) + << "Failed to increase cache reservation correctly"; + EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(), + 3 * kSizeDummyEntry) + << "Failed to bookkeep cache reservation increase correctly"; + EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used) + << "Failed to bookkeep the used memory correctly"; + EXPECT_GE(cache->GetPinnedUsage(), 3 * kSizeDummyEntry) + << "Failed to increase underlying dummy entries in cache correctly"; + EXPECT_LT(cache->GetPinnedUsage(), + 3 * kSizeDummyEntry + kMetaDataChargeOverhead) + << "Failed to increase underlying dummy entries in cache correctly"; +} + +TEST(CacheReservationManagerIncreaseReservcationOnFullCacheTest, + IncreaseCacheReservationOnFullCache) { + ; + constexpr std::size_t kSizeDummyEntry = + CacheReservationManagerImpl::GetDummyEntrySize(); + constexpr std::size_t kSmallCacheCapacity = 4 * kSizeDummyEntry; + constexpr std::size_t kBigCacheCapacity = 4096 * kSizeDummyEntry; + constexpr std::size_t kMetaDataChargeOverhead = 10000; + + LRUCacheOptions lo; + lo.capacity = kSmallCacheCapacity; + lo.num_shard_bits = 0; // 2^0 shard + lo.strict_capacity_limit = true; + std::shared_ptr cache = NewLRUCache(lo); + std::shared_ptr test_cache_rev_mng = + std::make_shared>( + cache); + + std::size_t new_mem_used = kSmallCacheCapacity + 1; + Status s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used); + EXPECT_EQ(s, Status::MemoryLimit()) + << "Failed to return status to indicate failure of dummy entry insertion " + "during cache reservation on full cache"; + EXPECT_GE(test_cache_rev_mng->GetTotalReservedCacheSize(), + 1 * kSizeDummyEntry) + << "Failed to bookkeep correctly before cache resevation failure happens " + "due to full cache"; + EXPECT_LE(test_cache_rev_mng->GetTotalReservedCacheSize(), + kSmallCacheCapacity) + << "Failed to bookkeep correctly (i.e, bookkeep only successful dummy " + "entry insertions) when encountering cache resevation failure due to " + "full cache"; + EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used) + << "Failed to bookkeep the used memory correctly"; + EXPECT_GE(cache->GetPinnedUsage(), 1 * kSizeDummyEntry) + << "Failed to insert underlying dummy entries correctly when " + "encountering cache resevation failure due to full cache"; + EXPECT_LE(cache->GetPinnedUsage(), kSmallCacheCapacity) + << "Failed to insert underlying dummy entries correctly when " + "encountering cache resevation failure due to full cache"; + + new_mem_used = kSmallCacheCapacity / 2; // 2 dummy entries + s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used); + EXPECT_EQ(s, Status::OK()) + << "Failed to decrease cache reservation after encountering cache " + "reservation failure due to full cache"; + EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(), + 2 * kSizeDummyEntry) + << "Failed to bookkeep cache reservation decrease correctly after " + "encountering cache reservation due to full cache"; + EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used) + << "Failed to bookkeep the used memory correctly"; + EXPECT_GE(cache->GetPinnedUsage(), 2 * kSizeDummyEntry) + << "Failed to release underlying dummy entries correctly on cache " + "reservation decrease after encountering cache resevation failure due " + "to full cache"; + EXPECT_LT(cache->GetPinnedUsage(), + 2 * kSizeDummyEntry + kMetaDataChargeOverhead) + << "Failed to release underlying dummy entries correctly on cache " + "reservation decrease after encountering cache resevation failure due " + "to full cache"; + + // Create cache full again for subsequent tests + new_mem_used = kSmallCacheCapacity + 1; + s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used); + EXPECT_EQ(s, Status::MemoryLimit()) + << "Failed to return status to indicate failure of dummy entry insertion " + "during cache reservation on full cache"; + EXPECT_GE(test_cache_rev_mng->GetTotalReservedCacheSize(), + 1 * kSizeDummyEntry) + << "Failed to bookkeep correctly before cache resevation failure happens " + "due to full cache"; + EXPECT_LE(test_cache_rev_mng->GetTotalReservedCacheSize(), + kSmallCacheCapacity) + << "Failed to bookkeep correctly (i.e, bookkeep only successful dummy " + "entry insertions) when encountering cache resevation failure due to " + "full cache"; + EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used) + << "Failed to bookkeep the used memory correctly"; + EXPECT_GE(cache->GetPinnedUsage(), 1 * kSizeDummyEntry) + << "Failed to insert underlying dummy entries correctly when " + "encountering cache resevation failure due to full cache"; + EXPECT_LE(cache->GetPinnedUsage(), kSmallCacheCapacity) + << "Failed to insert underlying dummy entries correctly when " + "encountering cache resevation failure due to full cache"; + + // Increase cache capacity so the previously failed insertion can fully + // succeed + cache->SetCapacity(kBigCacheCapacity); + new_mem_used = kSmallCacheCapacity + 1; + s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used); + EXPECT_EQ(s, Status::OK()) + << "Failed to increase cache reservation after increasing cache capacity " + "and mitigating cache full error"; + EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(), + 5 * kSizeDummyEntry) + << "Failed to bookkeep cache reservation increase correctly after " + "increasing cache capacity and mitigating cache full error"; + EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used) + << "Failed to bookkeep the used memory correctly"; + EXPECT_GE(cache->GetPinnedUsage(), 5 * kSizeDummyEntry) + << "Failed to insert underlying dummy entries correctly after increasing " + "cache capacity and mitigating cache full error"; + EXPECT_LT(cache->GetPinnedUsage(), + 5 * kSizeDummyEntry + kMetaDataChargeOverhead) + << "Failed to insert underlying dummy entries correctly after increasing " + "cache capacity and mitigating cache full error"; +} + +TEST_F(CacheReservationManagerTest, + DecreaseCacheReservationByMultiplesOfDummyEntrySize) { + std::size_t new_mem_used = 2 * kSizeDummyEntry; + Status s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used); + ASSERT_EQ(s, Status::OK()); + ASSERT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(), + 2 * kSizeDummyEntry); + ASSERT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used); + ASSERT_GE(cache->GetPinnedUsage(), 2 * kSizeDummyEntry); + ASSERT_LT(cache->GetPinnedUsage(), + 2 * kSizeDummyEntry + kMetaDataChargeOverhead); + + new_mem_used = 1 * kSizeDummyEntry; + s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used); + EXPECT_EQ(s, Status::OK()) + << "Failed to decrease cache reservation correctly"; + EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(), + 1 * kSizeDummyEntry) + << "Failed to bookkeep cache reservation decrease correctly"; + EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used) + << "Failed to bookkeep the used memory correctly"; + EXPECT_GE(cache->GetPinnedUsage(), 1 * kSizeDummyEntry) + << "Failed to decrease underlying dummy entries in cache correctly"; + EXPECT_LT(cache->GetPinnedUsage(), + 1 * kSizeDummyEntry + kMetaDataChargeOverhead) + << "Failed to decrease underlying dummy entries in cache correctly"; +} + +TEST_F(CacheReservationManagerTest, + DecreaseCacheReservationNotByMultiplesOfDummyEntrySize) { + std::size_t new_mem_used = 2 * kSizeDummyEntry; + Status s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used); + ASSERT_EQ(s, Status::OK()); + ASSERT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(), + 2 * kSizeDummyEntry); + ASSERT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used); + ASSERT_GE(cache->GetPinnedUsage(), 2 * kSizeDummyEntry); + ASSERT_LT(cache->GetPinnedUsage(), + 2 * kSizeDummyEntry + kMetaDataChargeOverhead); + + new_mem_used = kSizeDummyEntry / 2; + s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used); + EXPECT_EQ(s, Status::OK()) + << "Failed to decrease cache reservation correctly"; + EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(), + 1 * kSizeDummyEntry) + << "Failed to bookkeep cache reservation decrease correctly"; + EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used) + << "Failed to bookkeep the used memory correctly"; + EXPECT_GE(cache->GetPinnedUsage(), 1 * kSizeDummyEntry) + << "Failed to decrease underlying dummy entries in cache correctly"; + EXPECT_LT(cache->GetPinnedUsage(), + 1 * kSizeDummyEntry + kMetaDataChargeOverhead) + << "Failed to decrease underlying dummy entries in cache correctly"; +} + +TEST(CacheReservationManagerWithDelayedDecreaseTest, + DecreaseCacheReservationWithDelayedDecrease) { + constexpr std::size_t kSizeDummyEntry = + CacheReservationManagerImpl::GetDummyEntrySize(); + constexpr std::size_t kCacheCapacity = 4096 * kSizeDummyEntry; + constexpr std::size_t kMetaDataChargeOverhead = 10000; + + LRUCacheOptions lo; + lo.capacity = kCacheCapacity; + lo.num_shard_bits = 0; + std::shared_ptr cache = NewLRUCache(lo); + std::shared_ptr test_cache_rev_mng = + std::make_shared>( + cache, true /* delayed_decrease */); + + std::size_t new_mem_used = 8 * kSizeDummyEntry; + Status s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used); + ASSERT_EQ(s, Status::OK()); + ASSERT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(), + 8 * kSizeDummyEntry); + ASSERT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used); + std::size_t initial_pinned_usage = cache->GetPinnedUsage(); + ASSERT_GE(initial_pinned_usage, 8 * kSizeDummyEntry); + ASSERT_LT(initial_pinned_usage, + 8 * kSizeDummyEntry + kMetaDataChargeOverhead); + + new_mem_used = 6 * kSizeDummyEntry; + s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used); + EXPECT_EQ(s, Status::OK()) << "Failed to delay decreasing cache reservation"; + EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(), + 8 * kSizeDummyEntry) + << "Failed to bookkeep correctly when delaying cache reservation " + "decrease"; + EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used) + << "Failed to bookkeep the used memory correctly"; + EXPECT_EQ(cache->GetPinnedUsage(), initial_pinned_usage) + << "Failed to delay decreasing underlying dummy entries in cache"; + + new_mem_used = 7 * kSizeDummyEntry; + s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used); + EXPECT_EQ(s, Status::OK()) << "Failed to delay decreasing cache reservation"; + EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(), + 8 * kSizeDummyEntry) + << "Failed to bookkeep correctly when delaying cache reservation " + "decrease"; + EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used) + << "Failed to bookkeep the used memory correctly"; + EXPECT_EQ(cache->GetPinnedUsage(), initial_pinned_usage) + << "Failed to delay decreasing underlying dummy entries in cache"; + + new_mem_used = 6 * kSizeDummyEntry - 1; + s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used); + EXPECT_EQ(s, Status::OK()) + << "Failed to decrease cache reservation correctly when new_mem_used < " + "GetTotalReservedCacheSize() * 3 / 4 on delayed decrease mode"; + EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(), + 6 * kSizeDummyEntry) + << "Failed to bookkeep correctly when new_mem_used < " + "GetTotalReservedCacheSize() * 3 / 4 on delayed decrease mode"; + EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used) + << "Failed to bookkeep the used memory correctly"; + EXPECT_GE(cache->GetPinnedUsage(), 6 * kSizeDummyEntry) + << "Failed to decrease underlying dummy entries in cache when " + "new_mem_used < GetTotalReservedCacheSize() * 3 / 4 on delayed " + "decrease mode"; + EXPECT_LT(cache->GetPinnedUsage(), + 6 * kSizeDummyEntry + kMetaDataChargeOverhead) + << "Failed to decrease underlying dummy entries in cache when " + "new_mem_used < GetTotalReservedCacheSize() * 3 / 4 on delayed " + "decrease mode"; +} + +TEST(CacheReservationManagerDestructorTest, + ReleaseRemainingDummyEntriesOnDestruction) { + constexpr std::size_t kSizeDummyEntry = + CacheReservationManagerImpl::GetDummyEntrySize(); + constexpr std::size_t kCacheCapacity = 4096 * kSizeDummyEntry; + constexpr std::size_t kMetaDataChargeOverhead = 10000; + + LRUCacheOptions lo; + lo.capacity = kCacheCapacity; + lo.num_shard_bits = 0; + std::shared_ptr cache = NewLRUCache(lo); + { + std::shared_ptr test_cache_rev_mng = + std::make_shared>( + cache); + std::size_t new_mem_used = 1 * kSizeDummyEntry; + Status s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used); + ASSERT_EQ(s, Status::OK()); + ASSERT_GE(cache->GetPinnedUsage(), 1 * kSizeDummyEntry); + ASSERT_LT(cache->GetPinnedUsage(), + 1 * kSizeDummyEntry + kMetaDataChargeOverhead); + } + EXPECT_EQ(cache->GetPinnedUsage(), 0 * kSizeDummyEntry) + << "Failed to release remaining underlying dummy entries in cache in " + "CacheReservationManager's destructor"; +} + +TEST(CacheReservationHandleTest, HandleTest) { + constexpr std::size_t kOneGigabyte = 1024 * 1024 * 1024; + constexpr std::size_t kSizeDummyEntry = 256 * 1024; + constexpr std::size_t kMetaDataChargeOverhead = 10000; + + LRUCacheOptions lo; + lo.capacity = kOneGigabyte; + lo.num_shard_bits = 0; + std::shared_ptr cache = NewLRUCache(lo); + + std::shared_ptr test_cache_rev_mng( + std::make_shared>( + cache)); + + std::size_t mem_used = 0; + const std::size_t incremental_mem_used_handle_1 = 1 * kSizeDummyEntry; + const std::size_t incremental_mem_used_handle_2 = 2 * kSizeDummyEntry; + std::unique_ptr handle_1, + handle_2; + + // To test consecutive CacheReservationManager::MakeCacheReservation works + // correctly in terms of returning the handle as well as updating cache + // reservation and the latest total memory used + Status s = test_cache_rev_mng->MakeCacheReservation( + incremental_mem_used_handle_1, &handle_1); + mem_used = mem_used + incremental_mem_used_handle_1; + ASSERT_EQ(s, Status::OK()); + EXPECT_TRUE(handle_1 != nullptr); + EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(), mem_used); + EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), mem_used); + EXPECT_GE(cache->GetPinnedUsage(), mem_used); + EXPECT_LT(cache->GetPinnedUsage(), mem_used + kMetaDataChargeOverhead); + + s = test_cache_rev_mng->MakeCacheReservation(incremental_mem_used_handle_2, + &handle_2); + mem_used = mem_used + incremental_mem_used_handle_2; + ASSERT_EQ(s, Status::OK()); + EXPECT_TRUE(handle_2 != nullptr); + EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(), mem_used); + EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), mem_used); + EXPECT_GE(cache->GetPinnedUsage(), mem_used); + EXPECT_LT(cache->GetPinnedUsage(), mem_used + kMetaDataChargeOverhead); + + // To test + // CacheReservationManager::CacheReservationHandle::~CacheReservationHandle() + // works correctly in releasing the cache reserved for the handle + handle_1.reset(); + EXPECT_TRUE(handle_1 == nullptr); + mem_used = mem_used - incremental_mem_used_handle_1; + EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(), mem_used); + EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), mem_used); + EXPECT_GE(cache->GetPinnedUsage(), mem_used); + EXPECT_LT(cache->GetPinnedUsage(), mem_used + kMetaDataChargeOverhead); + + // To test the actual CacheReservationManager object won't be deallocated + // as long as there remain handles pointing to it. + // We strongly recommend deallocating CacheReservationManager object only + // after all its handles are deallocated to keep things easy to reasonate + test_cache_rev_mng.reset(); + EXPECT_GE(cache->GetPinnedUsage(), mem_used); + EXPECT_LT(cache->GetPinnedUsage(), mem_used + kMetaDataChargeOverhead); + + handle_2.reset(); + // The CacheReservationManager object is now deallocated since all the handles + // and its original pointer is gone + mem_used = mem_used - incremental_mem_used_handle_2; + EXPECT_EQ(mem_used, 0); + EXPECT_EQ(cache->GetPinnedUsage(), mem_used); +} +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/cache/cache_test.cc b/librocksdb-sys/rocksdb/cache/cache_test.cc new file mode 100644 index 0000000..4585faa --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/cache_test.cc @@ -0,0 +1,1061 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "rocksdb/cache.h" + +#include +#include +#include +#include +#include + +#include "cache/lru_cache.h" +#include "cache/typed_cache.h" +#include "port/stack_trace.h" +#include "test_util/secondary_cache_test_util.h" +#include "test_util/testharness.h" +#include "util/coding.h" +#include "util/hash_containers.h" +#include "util/string_util.h" + +// HyperClockCache only supports 16-byte keys, so some of the tests +// originally written for LRUCache do not work on the other caches. +// Those tests were adapted to use 16-byte keys. We kept the original ones. +// TODO: Remove the original tests if they ever become unused. + +namespace ROCKSDB_NAMESPACE { + +namespace { + +// Conversions between numeric keys/values and the types expected by Cache. +std::string EncodeKey16Bytes(int k) { + std::string result; + PutFixed32(&result, k); + result.append(std::string(12, 'a')); // Because we need a 16B output, we + // add a 12-byte padding. + return result; +} + +int DecodeKey16Bytes(const Slice& k) { + assert(k.size() == 16); + return DecodeFixed32(k.data()); // Decodes only the first 4 bytes of k. +} + +std::string EncodeKey32Bits(int k) { + std::string result; + PutFixed32(&result, k); + return result; +} + +int DecodeKey32Bits(const Slice& k) { + assert(k.size() == 4); + return DecodeFixed32(k.data()); +} + +Cache::ObjectPtr EncodeValue(uintptr_t v) { + return reinterpret_cast(v); +} + +int DecodeValue(void* v) { + return static_cast(reinterpret_cast(v)); +} + +const Cache::CacheItemHelper kDumbHelper{ + CacheEntryRole::kMisc, + [](Cache::ObjectPtr /*value*/, MemoryAllocator* /*alloc*/) {}}; + +const Cache::CacheItemHelper kEraseOnDeleteHelper1{ + CacheEntryRole::kMisc, + [](Cache::ObjectPtr value, MemoryAllocator* /*alloc*/) { + Cache* cache = static_cast(value); + cache->Erase("foo"); + }}; + +const Cache::CacheItemHelper kEraseOnDeleteHelper2{ + CacheEntryRole::kMisc, + [](Cache::ObjectPtr value, MemoryAllocator* /*alloc*/) { + Cache* cache = static_cast(value); + cache->Erase(EncodeKey16Bytes(1234)); + }}; +} // anonymous namespace + +class CacheTest : public testing::Test, + public secondary_cache_test_util::WithCacheTypeParam { + public: + static CacheTest* current_; + static std::string type_; + + static void Deleter(Cache::ObjectPtr v, MemoryAllocator*) { + current_->deleted_values_.push_back(DecodeValue(v)); + } + static const Cache::CacheItemHelper kHelper; + + static const int kCacheSize = 1000; + static const int kNumShardBits = 4; + + static const int kCacheSize2 = 100; + static const int kNumShardBits2 = 2; + + std::vector deleted_values_; + std::shared_ptr cache_; + std::shared_ptr cache2_; + + CacheTest() + : cache_(NewCache(kCacheSize, kNumShardBits, false)), + cache2_(NewCache(kCacheSize2, kNumShardBits2, false)) { + current_ = this; + type_ = GetParam(); + } + + ~CacheTest() override {} + + // These functions encode/decode keys in tests cases that use + // int keys. + // Currently, HyperClockCache requires keys to be 16B long, whereas + // LRUCache doesn't, so the encoding depends on the cache type. + std::string EncodeKey(int k) { + auto type = GetParam(); + if (type == kHyperClock) { + return EncodeKey16Bytes(k); + } else { + return EncodeKey32Bits(k); + } + } + + int DecodeKey(const Slice& k) { + auto type = GetParam(); + if (type == kHyperClock) { + return DecodeKey16Bytes(k); + } else { + return DecodeKey32Bits(k); + } + } + + int Lookup(std::shared_ptr cache, int key) { + Cache::Handle* handle = cache->Lookup(EncodeKey(key)); + const int r = (handle == nullptr) ? -1 : DecodeValue(cache->Value(handle)); + if (handle != nullptr) { + cache->Release(handle); + } + return r; + } + + void Insert(std::shared_ptr cache, int key, int value, + int charge = 1) { + EXPECT_OK(cache->Insert(EncodeKey(key), EncodeValue(value), &kHelper, + charge, /*handle*/ nullptr, Cache::Priority::HIGH)); + } + + void Erase(std::shared_ptr cache, int key) { + cache->Erase(EncodeKey(key)); + } + + int Lookup(int key) { return Lookup(cache_, key); } + + void Insert(int key, int value, int charge = 1) { + Insert(cache_, key, value, charge); + } + + void Erase(int key) { Erase(cache_, key); } + + int Lookup2(int key) { return Lookup(cache2_, key); } + + void Insert2(int key, int value, int charge = 1) { + Insert(cache2_, key, value, charge); + } + + void Erase2(int key) { Erase(cache2_, key); } +}; + +const Cache::CacheItemHelper CacheTest::kHelper{CacheEntryRole::kMisc, + &CacheTest::Deleter}; + +CacheTest* CacheTest::current_; +std::string CacheTest::type_; + +class LRUCacheTest : public CacheTest {}; + +TEST_P(CacheTest, UsageTest) { + auto type = GetParam(); + + // cache is std::shared_ptr and will be automatically cleaned up. + const size_t kCapacity = 100000; + auto cache = NewCache(kCapacity, 8, false, kDontChargeCacheMetadata); + auto precise_cache = NewCache(kCapacity, 0, false, kFullChargeCacheMetadata); + ASSERT_EQ(0, cache->GetUsage()); + size_t baseline_meta_usage = precise_cache->GetUsage(); + if (type != kHyperClock) { + ASSERT_EQ(0, baseline_meta_usage); + } + + size_t usage = 0; + char value[10] = "abcdef"; + // make sure everything will be cached + for (int i = 1; i < 100; ++i) { + std::string key; + if (type == kLRU) { + key = std::string(i, 'a'); + } else { + key = EncodeKey(i); + } + auto kv_size = key.size() + 5; + ASSERT_OK(cache->Insert(key, value, &kDumbHelper, kv_size)); + ASSERT_OK(precise_cache->Insert(key, value, &kDumbHelper, kv_size)); + usage += kv_size; + ASSERT_EQ(usage, cache->GetUsage()); + if (type == kHyperClock) { + ASSERT_EQ(baseline_meta_usage + usage, precise_cache->GetUsage()); + } else { + ASSERT_LT(usage, precise_cache->GetUsage()); + } + } + + cache->EraseUnRefEntries(); + precise_cache->EraseUnRefEntries(); + ASSERT_EQ(0, cache->GetUsage()); + ASSERT_EQ(baseline_meta_usage, precise_cache->GetUsage()); + + // make sure the cache will be overloaded + for (size_t i = 1; i < kCapacity; ++i) { + std::string key; + if (type == kLRU) { + key = std::to_string(i); + } else { + key = EncodeKey(static_cast(1000 + i)); + } + ASSERT_OK(cache->Insert(key, value, &kDumbHelper, key.size() + 5)); + ASSERT_OK(precise_cache->Insert(key, value, &kDumbHelper, key.size() + 5)); + } + + // the usage should be close to the capacity + ASSERT_GT(kCapacity, cache->GetUsage()); + ASSERT_GT(kCapacity, precise_cache->GetUsage()); + ASSERT_LT(kCapacity * 0.95, cache->GetUsage()); + if (type != kHyperClock) { + ASSERT_LT(kCapacity * 0.95, precise_cache->GetUsage()); + } else { + // estimated value size of 1 is weird for clock cache, because + // almost all of the capacity will be used for metadata, and due to only + // using power of 2 table sizes, we might hit strict occupancy limit + // before hitting capacity limit. + ASSERT_LT(kCapacity * 0.80, precise_cache->GetUsage()); + } +} + +// TODO: This test takes longer than expected on ClockCache. This is +// because the values size estimate at construction is too sloppy. +// Fix this. +// Why is it so slow? The cache is constructed with an estimate of 1, but +// then the charge is claimed to be 21. This will cause the hash table +// to be extremely sparse, which in turn means clock needs to scan too +// many slots to find victims. +TEST_P(CacheTest, PinnedUsageTest) { + auto type = GetParam(); + + // cache is std::shared_ptr and will be automatically cleaned up. + const size_t kCapacity = 200000; + auto cache = NewCache(kCapacity, 8, false, kDontChargeCacheMetadata); + auto precise_cache = NewCache(kCapacity, 8, false, kFullChargeCacheMetadata); + size_t baseline_meta_usage = precise_cache->GetUsage(); + if (type != kHyperClock) { + ASSERT_EQ(0, baseline_meta_usage); + } + + size_t pinned_usage = 0; + char value[10] = "abcdef"; + + std::forward_list unreleased_handles; + std::forward_list unreleased_handles_in_precise_cache; + + // Add entries. Unpin some of them after insertion. Then, pin some of them + // again. Check GetPinnedUsage(). + for (int i = 1; i < 100; ++i) { + std::string key; + if (type == kLRU) { + key = std::string(i, 'a'); + } else { + key = EncodeKey(i); + } + auto kv_size = key.size() + 5; + Cache::Handle* handle; + Cache::Handle* handle_in_precise_cache; + ASSERT_OK(cache->Insert(key, value, &kDumbHelper, kv_size, &handle)); + assert(handle); + ASSERT_OK(precise_cache->Insert(key, value, &kDumbHelper, kv_size, + &handle_in_precise_cache)); + assert(handle_in_precise_cache); + pinned_usage += kv_size; + ASSERT_EQ(pinned_usage, cache->GetPinnedUsage()); + ASSERT_LT(pinned_usage, precise_cache->GetPinnedUsage()); + if (i % 2 == 0) { + cache->Release(handle); + precise_cache->Release(handle_in_precise_cache); + pinned_usage -= kv_size; + ASSERT_EQ(pinned_usage, cache->GetPinnedUsage()); + ASSERT_LT(pinned_usage, precise_cache->GetPinnedUsage()); + } else { + unreleased_handles.push_front(handle); + unreleased_handles_in_precise_cache.push_front(handle_in_precise_cache); + } + if (i % 3 == 0) { + unreleased_handles.push_front(cache->Lookup(key)); + auto x = precise_cache->Lookup(key); + assert(x); + unreleased_handles_in_precise_cache.push_front(x); + // If i % 2 == 0, then the entry was unpinned before Lookup, so pinned + // usage increased + if (i % 2 == 0) { + pinned_usage += kv_size; + } + ASSERT_EQ(pinned_usage, cache->GetPinnedUsage()); + ASSERT_LT(pinned_usage, precise_cache->GetPinnedUsage()); + } + } + auto precise_cache_pinned_usage = precise_cache->GetPinnedUsage(); + ASSERT_LT(pinned_usage, precise_cache_pinned_usage); + + // check that overloading the cache does not change the pinned usage + for (size_t i = 1; i < 2 * kCapacity; ++i) { + std::string key; + if (type == kLRU) { + key = std::to_string(i); + } else { + key = EncodeKey(static_cast(1000 + i)); + } + ASSERT_OK(cache->Insert(key, value, &kDumbHelper, key.size() + 5)); + ASSERT_OK(precise_cache->Insert(key, value, &kDumbHelper, key.size() + 5)); + } + ASSERT_EQ(pinned_usage, cache->GetPinnedUsage()); + ASSERT_EQ(precise_cache_pinned_usage, precise_cache->GetPinnedUsage()); + + cache->EraseUnRefEntries(); + precise_cache->EraseUnRefEntries(); + ASSERT_EQ(pinned_usage, cache->GetPinnedUsage()); + ASSERT_EQ(precise_cache_pinned_usage, precise_cache->GetPinnedUsage()); + + // release handles for pinned entries to prevent memory leaks + for (auto handle : unreleased_handles) { + cache->Release(handle); + } + for (auto handle : unreleased_handles_in_precise_cache) { + precise_cache->Release(handle); + } + ASSERT_EQ(0, cache->GetPinnedUsage()); + ASSERT_EQ(0, precise_cache->GetPinnedUsage()); + cache->EraseUnRefEntries(); + precise_cache->EraseUnRefEntries(); + ASSERT_EQ(0, cache->GetUsage()); + ASSERT_EQ(baseline_meta_usage, precise_cache->GetUsage()); +} + +TEST_P(CacheTest, HitAndMiss) { + ASSERT_EQ(-1, Lookup(100)); + + Insert(100, 101); + ASSERT_EQ(101, Lookup(100)); + ASSERT_EQ(-1, Lookup(200)); + ASSERT_EQ(-1, Lookup(300)); + + Insert(200, 201); + ASSERT_EQ(101, Lookup(100)); + ASSERT_EQ(201, Lookup(200)); + ASSERT_EQ(-1, Lookup(300)); + + Insert(100, 102); + if (GetParam() == kHyperClock) { + // ClockCache usually doesn't overwrite on Insert + ASSERT_EQ(101, Lookup(100)); + } else { + ASSERT_EQ(102, Lookup(100)); + } + ASSERT_EQ(201, Lookup(200)); + ASSERT_EQ(-1, Lookup(300)); + + ASSERT_EQ(1U, deleted_values_.size()); + if (GetParam() == kHyperClock) { + ASSERT_EQ(102, deleted_values_[0]); + } else { + ASSERT_EQ(101, deleted_values_[0]); + } +} + +TEST_P(CacheTest, InsertSameKey) { + if (GetParam() == kHyperClock) { + ROCKSDB_GTEST_BYPASS( + "ClockCache doesn't guarantee Insert overwrite same key."); + return; + } + Insert(1, 1); + Insert(1, 2); + ASSERT_EQ(2, Lookup(1)); +} + +TEST_P(CacheTest, Erase) { + Erase(200); + ASSERT_EQ(0U, deleted_values_.size()); + + Insert(100, 101); + Insert(200, 201); + Erase(100); + ASSERT_EQ(-1, Lookup(100)); + ASSERT_EQ(201, Lookup(200)); + ASSERT_EQ(1U, deleted_values_.size()); + ASSERT_EQ(101, deleted_values_[0]); + + Erase(100); + ASSERT_EQ(-1, Lookup(100)); + ASSERT_EQ(201, Lookup(200)); + ASSERT_EQ(1U, deleted_values_.size()); +} + +TEST_P(CacheTest, EntriesArePinned) { + if (GetParam() == kHyperClock) { + ROCKSDB_GTEST_BYPASS( + "ClockCache doesn't guarantee Insert overwrite same key."); + return; + } + Insert(100, 101); + Cache::Handle* h1 = cache_->Lookup(EncodeKey(100)); + ASSERT_EQ(101, DecodeValue(cache_->Value(h1))); + ASSERT_EQ(1U, cache_->GetUsage()); + + Insert(100, 102); + Cache::Handle* h2 = cache_->Lookup(EncodeKey(100)); + ASSERT_EQ(102, DecodeValue(cache_->Value(h2))); + ASSERT_EQ(0U, deleted_values_.size()); + ASSERT_EQ(2U, cache_->GetUsage()); + + cache_->Release(h1); + ASSERT_EQ(1U, deleted_values_.size()); + ASSERT_EQ(101, deleted_values_[0]); + ASSERT_EQ(1U, cache_->GetUsage()); + + Erase(100); + ASSERT_EQ(-1, Lookup(100)); + ASSERT_EQ(1U, deleted_values_.size()); + ASSERT_EQ(1U, cache_->GetUsage()); + + cache_->Release(h2); + ASSERT_EQ(2U, deleted_values_.size()); + ASSERT_EQ(102, deleted_values_[1]); + ASSERT_EQ(0U, cache_->GetUsage()); +} + +TEST_P(CacheTest, EvictionPolicy) { + Insert(100, 101); + Insert(200, 201); + // Frequently used entry must be kept around + for (int i = 0; i < 2 * kCacheSize; i++) { + Insert(1000 + i, 2000 + i); + ASSERT_EQ(101, Lookup(100)); + } + ASSERT_EQ(101, Lookup(100)); + ASSERT_EQ(-1, Lookup(200)); +} + +TEST_P(CacheTest, ExternalRefPinsEntries) { + Insert(100, 101); + Cache::Handle* h = cache_->Lookup(EncodeKey(100)); + ASSERT_TRUE(cache_->Ref(h)); + ASSERT_EQ(101, DecodeValue(cache_->Value(h))); + ASSERT_EQ(1U, cache_->GetUsage()); + + for (int i = 0; i < 3; ++i) { + if (i > 0) { + // First release (i == 1) corresponds to Ref(), second release (i == 2) + // corresponds to Lookup(). Then, since all external refs are released, + // the below insertions should push out the cache entry. + cache_->Release(h); + } + // double cache size because the usage bit in block cache prevents 100 from + // being evicted in the first kCacheSize iterations + for (int j = 0; j < 2 * kCacheSize + 100; j++) { + Insert(1000 + j, 2000 + j); + } + // Clock cache is even more stateful and needs more churn to evict + if (GetParam() == kHyperClock) { + for (int j = 0; j < kCacheSize; j++) { + Insert(11000 + j, 11000 + j); + } + } + if (i < 2) { + ASSERT_EQ(101, Lookup(100)); + } + } + ASSERT_EQ(-1, Lookup(100)); +} + +TEST_P(CacheTest, EvictionPolicyRef) { + Insert(100, 101); + Insert(101, 102); + Insert(102, 103); + Insert(103, 104); + Insert(200, 101); + Insert(201, 102); + Insert(202, 103); + Insert(203, 104); + Cache::Handle* h201 = cache_->Lookup(EncodeKey(200)); + Cache::Handle* h202 = cache_->Lookup(EncodeKey(201)); + Cache::Handle* h203 = cache_->Lookup(EncodeKey(202)); + Cache::Handle* h204 = cache_->Lookup(EncodeKey(203)); + Insert(300, 101); + Insert(301, 102); + Insert(302, 103); + Insert(303, 104); + + // Insert entries much more than cache capacity. + for (int i = 0; i < 100 * kCacheSize; i++) { + Insert(1000 + i, 2000 + i); + } + + // Check whether the entries inserted in the beginning + // are evicted. Ones without extra ref are evicted and + // those with are not. + ASSERT_EQ(-1, Lookup(100)); + ASSERT_EQ(-1, Lookup(101)); + ASSERT_EQ(-1, Lookup(102)); + ASSERT_EQ(-1, Lookup(103)); + + ASSERT_EQ(-1, Lookup(300)); + ASSERT_EQ(-1, Lookup(301)); + ASSERT_EQ(-1, Lookup(302)); + ASSERT_EQ(-1, Lookup(303)); + + ASSERT_EQ(101, Lookup(200)); + ASSERT_EQ(102, Lookup(201)); + ASSERT_EQ(103, Lookup(202)); + ASSERT_EQ(104, Lookup(203)); + + // Cleaning up all the handles + cache_->Release(h201); + cache_->Release(h202); + cache_->Release(h203); + cache_->Release(h204); +} + +TEST_P(CacheTest, EvictEmptyCache) { + auto type = GetParam(); + + // Insert item large than capacity to trigger eviction on empty cache. + auto cache = NewCache(1, 0, false); + if (type == kLRU) { + ASSERT_OK(cache->Insert("foo", nullptr, &kDumbHelper, 10)); + } else { + ASSERT_OK(cache->Insert(EncodeKey(1000), nullptr, &kDumbHelper, 10)); + } +} + +TEST_P(CacheTest, EraseFromDeleter) { + auto type = GetParam(); + + // Have deleter which will erase item from cache, which will re-enter + // the cache at that point. + std::shared_ptr cache = NewCache(10, 0, false); + std::string foo, bar; + const Cache::CacheItemHelper* erase_helper; + if (type == kLRU) { + foo = "foo"; + bar = "bar"; + erase_helper = &kEraseOnDeleteHelper1; + } else { + foo = EncodeKey(1234); + bar = EncodeKey(5678); + erase_helper = &kEraseOnDeleteHelper2; + } + + ASSERT_OK(cache->Insert(foo, nullptr, &kDumbHelper, 1)); + ASSERT_OK(cache->Insert(bar, cache.get(), erase_helper, 1)); + + cache->Erase(bar); + ASSERT_EQ(nullptr, cache->Lookup(foo)); + ASSERT_EQ(nullptr, cache->Lookup(bar)); +} + +TEST_P(CacheTest, ErasedHandleState) { + // insert a key and get two handles + Insert(100, 1000); + Cache::Handle* h1 = cache_->Lookup(EncodeKey(100)); + Cache::Handle* h2 = cache_->Lookup(EncodeKey(100)); + ASSERT_EQ(h1, h2); + ASSERT_EQ(DecodeValue(cache_->Value(h1)), 1000); + ASSERT_EQ(DecodeValue(cache_->Value(h2)), 1000); + + // delete the key from the cache + Erase(100); + // can no longer find in the cache + ASSERT_EQ(-1, Lookup(100)); + + // release one handle + cache_->Release(h1); + // still can't find in cache + ASSERT_EQ(-1, Lookup(100)); + + cache_->Release(h2); +} + +TEST_P(CacheTest, HeavyEntries) { + // Add a bunch of light and heavy entries and then count the combined + // size of items still in the cache, which must be approximately the + // same as the total capacity. + const int kLight = 1; + const int kHeavy = 10; + int added = 0; + int index = 0; + while (added < 2 * kCacheSize) { + const int weight = (index & 1) ? kLight : kHeavy; + Insert(index, 1000 + index, weight); + added += weight; + index++; + } + + int cached_weight = 0; + for (int i = 0; i < index; i++) { + const int weight = (i & 1 ? kLight : kHeavy); + int r = Lookup(i); + if (r >= 0) { + cached_weight += weight; + ASSERT_EQ(1000 + i, r); + } + } + ASSERT_LE(cached_weight, kCacheSize + kCacheSize / 10); +} + +TEST_P(CacheTest, NewId) { + uint64_t a = cache_->NewId(); + uint64_t b = cache_->NewId(); + ASSERT_NE(a, b); +} + +TEST_P(CacheTest, ReleaseAndErase) { + std::shared_ptr cache = NewCache(5, 0, false); + Cache::Handle* handle; + Status s = + cache->Insert(EncodeKey(100), EncodeValue(100), &kHelper, 1, &handle); + ASSERT_TRUE(s.ok()); + ASSERT_EQ(5U, cache->GetCapacity()); + ASSERT_EQ(1U, cache->GetUsage()); + ASSERT_EQ(0U, deleted_values_.size()); + auto erased = cache->Release(handle, true); + ASSERT_TRUE(erased); + // This tests that deleter has been called + ASSERT_EQ(1U, deleted_values_.size()); +} + +TEST_P(CacheTest, ReleaseWithoutErase) { + std::shared_ptr cache = NewCache(5, 0, false); + Cache::Handle* handle; + Status s = + cache->Insert(EncodeKey(100), EncodeValue(100), &kHelper, 1, &handle); + ASSERT_TRUE(s.ok()); + ASSERT_EQ(5U, cache->GetCapacity()); + ASSERT_EQ(1U, cache->GetUsage()); + ASSERT_EQ(0U, deleted_values_.size()); + auto erased = cache->Release(handle); + ASSERT_FALSE(erased); + // This tests that deleter is not called. When cache has free capacity it is + // not expected to immediately erase the released items. + ASSERT_EQ(0U, deleted_values_.size()); +} + +namespace { +class Value { + public: + explicit Value(int v) : v_(v) {} + + int v_; + + static constexpr auto kCacheEntryRole = CacheEntryRole::kMisc; +}; + +using SharedCache = BasicTypedSharedCacheInterface; +using TypedHandle = SharedCache::TypedHandle; +} // namespace + +TEST_P(CacheTest, SetCapacity) { + auto type = GetParam(); + if (type == kHyperClock) { + ROCKSDB_GTEST_BYPASS( + "FastLRUCache and HyperClockCache don't support arbitrary capacity " + "adjustments."); + return; + } + // test1: increase capacity + // lets create a cache with capacity 5, + // then, insert 5 elements, then increase capacity + // to 10, returned capacity should be 10, usage=5 + SharedCache cache{NewCache(5, 0, false)}; + std::vector handles(10); + // Insert 5 entries, but not releasing. + for (int i = 0; i < 5; i++) { + std::string key = EncodeKey(i + 1); + Status s = cache.Insert(key, new Value(i + 1), 1, &handles[i]); + ASSERT_TRUE(s.ok()); + } + ASSERT_EQ(5U, cache.get()->GetCapacity()); + ASSERT_EQ(5U, cache.get()->GetUsage()); + cache.get()->SetCapacity(10); + ASSERT_EQ(10U, cache.get()->GetCapacity()); + ASSERT_EQ(5U, cache.get()->GetUsage()); + + // test2: decrease capacity + // insert 5 more elements to cache, then release 5, + // then decrease capacity to 7, final capacity should be 7 + // and usage should be 7 + for (int i = 5; i < 10; i++) { + std::string key = EncodeKey(i + 1); + Status s = cache.Insert(key, new Value(i + 1), 1, &handles[i]); + ASSERT_TRUE(s.ok()); + } + ASSERT_EQ(10U, cache.get()->GetCapacity()); + ASSERT_EQ(10U, cache.get()->GetUsage()); + for (int i = 0; i < 5; i++) { + cache.Release(handles[i]); + } + ASSERT_EQ(10U, cache.get()->GetCapacity()); + ASSERT_EQ(10U, cache.get()->GetUsage()); + cache.get()->SetCapacity(7); + ASSERT_EQ(7, cache.get()->GetCapacity()); + ASSERT_EQ(7, cache.get()->GetUsage()); + + // release remaining 5 to keep valgrind happy + for (int i = 5; i < 10; i++) { + cache.Release(handles[i]); + } + + // Make sure this doesn't crash or upset ASAN/valgrind + cache.get()->DisownData(); +} + +TEST_P(LRUCacheTest, SetStrictCapacityLimit) { + // test1: set the flag to false. Insert more keys than capacity. See if they + // all go through. + SharedCache cache{NewCache(5, 0, false)}; + std::vector handles(10); + Status s; + for (int i = 0; i < 10; i++) { + std::string key = EncodeKey(i + 1); + s = cache.Insert(key, new Value(i + 1), 1, &handles[i]); + ASSERT_OK(s); + ASSERT_NE(nullptr, handles[i]); + } + ASSERT_EQ(10, cache.get()->GetUsage()); + + // test2: set the flag to true. Insert and check if it fails. + std::string extra_key = EncodeKey(100); + Value* extra_value = new Value(0); + cache.get()->SetStrictCapacityLimit(true); + TypedHandle* handle; + s = cache.Insert(extra_key, extra_value, 1, &handle); + ASSERT_TRUE(s.IsMemoryLimit()); + ASSERT_EQ(nullptr, handle); + ASSERT_EQ(10, cache.get()->GetUsage()); + + for (int i = 0; i < 10; i++) { + cache.Release(handles[i]); + } + + // test3: init with flag being true. + SharedCache cache2{NewCache(5, 0, true)}; + for (int i = 0; i < 5; i++) { + std::string key = EncodeKey(i + 1); + s = cache2.Insert(key, new Value(i + 1), 1, &handles[i]); + ASSERT_OK(s); + ASSERT_NE(nullptr, handles[i]); + } + s = cache2.Insert(extra_key, extra_value, 1, &handle); + ASSERT_TRUE(s.IsMemoryLimit()); + ASSERT_EQ(nullptr, handle); + // test insert without handle + s = cache2.Insert(extra_key, extra_value, 1); + // AS if the key have been inserted into cache but get evicted immediately. + ASSERT_OK(s); + ASSERT_EQ(5, cache2.get()->GetUsage()); + ASSERT_EQ(nullptr, cache2.Lookup(extra_key)); + + for (int i = 0; i < 5; i++) { + cache2.Release(handles[i]); + } +} + +TEST_P(CacheTest, OverCapacity) { + size_t n = 10; + + // a LRUCache with n entries and one shard only + SharedCache cache{NewCache(n, 0, false)}; + std::vector handles(n + 1); + + // Insert n+1 entries, but not releasing. + for (int i = 0; i < static_cast(n + 1); i++) { + std::string key = EncodeKey(i + 1); + Status s = cache.Insert(key, new Value(i + 1), 1, &handles[i]); + ASSERT_TRUE(s.ok()); + } + + // Guess what's in the cache now? + for (int i = 0; i < static_cast(n + 1); i++) { + std::string key = EncodeKey(i + 1); + auto h = cache.Lookup(key); + ASSERT_TRUE(h != nullptr); + if (h) cache.Release(h); + } + + // the cache is over capacity since nothing could be evicted + ASSERT_EQ(n + 1U, cache.get()->GetUsage()); + for (int i = 0; i < static_cast(n + 1); i++) { + cache.Release(handles[i]); + } + + if (GetParam() == kHyperClock) { + // Make sure eviction is triggered. + ASSERT_OK(cache.Insert(EncodeKey(-1), nullptr, 1, &handles[0])); + + // cache is under capacity now since elements were released + ASSERT_GE(n, cache.get()->GetUsage()); + + // clean up + cache.Release(handles[0]); + } else { + // LRUCache checks for over-capacity in Release. + + // cache is exactly at capacity now with minimal eviction + ASSERT_EQ(n, cache.get()->GetUsage()); + + // element 0 is evicted and the rest is there + // This is consistent with the LRU policy since the element 0 + // was released first + for (int i = 0; i < static_cast(n + 1); i++) { + std::string key = EncodeKey(i + 1); + auto h = cache.Lookup(key); + if (h) { + ASSERT_NE(static_cast(i), 0U); + cache.Release(h); + } else { + ASSERT_EQ(static_cast(i), 0U); + } + } + } +} + +TEST_P(CacheTest, ApplyToAllEntriesTest) { + std::vector callback_state; + const auto callback = [&](const Slice& key, Cache::ObjectPtr value, + size_t charge, + const Cache::CacheItemHelper* helper) { + callback_state.push_back(std::to_string(DecodeKey(key)) + "," + + std::to_string(DecodeValue(value)) + "," + + std::to_string(charge)); + assert(helper == &CacheTest::kHelper); + }; + + std::vector inserted; + callback_state.clear(); + + for (int i = 0; i < 10; ++i) { + Insert(i, i * 2, i + 1); + inserted.push_back(std::to_string(i) + "," + std::to_string(i * 2) + "," + + std::to_string(i + 1)); + } + cache_->ApplyToAllEntries(callback, /*opts*/ {}); + + std::sort(inserted.begin(), inserted.end()); + std::sort(callback_state.begin(), callback_state.end()); + ASSERT_EQ(inserted.size(), callback_state.size()); + for (int i = 0; i < static_cast(inserted.size()); ++i) { + EXPECT_EQ(inserted[i], callback_state[i]); + } +} + +TEST_P(CacheTest, ApplyToAllEntriesDuringResize) { + // This is a mini-stress test of ApplyToAllEntries, to ensure + // items in the cache that are neither added nor removed + // during ApplyToAllEntries are counted exactly once. + + // Insert some entries that we expect to be seen exactly once + // during iteration. + constexpr int kSpecialCharge = 2; + constexpr int kNotSpecialCharge = 1; + constexpr int kSpecialCount = 100; + size_t expected_usage = 0; + for (int i = 0; i < kSpecialCount; ++i) { + Insert(i, i * 2, kSpecialCharge); + expected_usage += kSpecialCharge; + } + + // For callback + int special_count = 0; + const auto callback = [&](const Slice&, Cache::ObjectPtr, size_t charge, + const Cache::CacheItemHelper*) { + if (charge == static_cast(kSpecialCharge)) { + ++special_count; + } + }; + + // Start counting + std::thread apply_thread([&]() { + // Use small average_entries_per_lock to make the problem difficult + Cache::ApplyToAllEntriesOptions opts; + opts.average_entries_per_lock = 2; + cache_->ApplyToAllEntries(callback, opts); + }); + + // In parallel, add more entries, enough to cause resize but not enough + // to cause ejections. (Note: if any cache shard is over capacity, there + // will be ejections) + for (int i = kSpecialCount * 1; i < kSpecialCount * 5; ++i) { + Insert(i, i * 2, kNotSpecialCharge); + expected_usage += kNotSpecialCharge; + } + + apply_thread.join(); + // verify no evictions + ASSERT_EQ(cache_->GetUsage(), expected_usage); + // verify everything seen in ApplyToAllEntries + ASSERT_EQ(special_count, kSpecialCount); +} + +TEST_P(CacheTest, DefaultShardBits) { + // Prevent excessive allocation (to save time & space) + estimated_value_size_ = 100000; + // Implementations use different minimum shard sizes + size_t min_shard_size = + (GetParam() == kHyperClock ? 32U * 1024U : 512U) * 1024U; + + std::shared_ptr cache = NewCache(32U * min_shard_size); + ShardedCacheBase* sc = dynamic_cast(cache.get()); + ASSERT_EQ(5, sc->GetNumShardBits()); + + cache = NewCache(min_shard_size / 1000U * 999U); + sc = dynamic_cast(cache.get()); + ASSERT_EQ(0, sc->GetNumShardBits()); + + cache = NewCache(3U * 1024U * 1024U * 1024U); + sc = dynamic_cast(cache.get()); + // current maximum of 6 + ASSERT_EQ(6, sc->GetNumShardBits()); + + if constexpr (sizeof(size_t) > 4) { + cache = NewCache(128U * min_shard_size); + sc = dynamic_cast(cache.get()); + // current maximum of 6 + ASSERT_EQ(6, sc->GetNumShardBits()); + } +} + +TEST_P(CacheTest, GetChargeAndDeleter) { + Insert(1, 2); + Cache::Handle* h1 = cache_->Lookup(EncodeKey(1)); + ASSERT_EQ(2, DecodeValue(cache_->Value(h1))); + ASSERT_EQ(1, cache_->GetCharge(h1)); + ASSERT_EQ(&CacheTest::kHelper, cache_->GetCacheItemHelper(h1)); + cache_->Release(h1); +} + +namespace { +bool AreTwoCacheKeysOrdered(Cache* cache) { + std::vector keys; + const auto callback = [&](const Slice& key, Cache::ObjectPtr /*value*/, + size_t /*charge*/, + const Cache::CacheItemHelper* /*helper*/) { + keys.push_back(key.ToString()); + }; + cache->ApplyToAllEntries(callback, /*opts*/ {}); + EXPECT_EQ(keys.size(), 2U); + EXPECT_NE(keys[0], keys[1]); + return keys[0] < keys[1]; +} +} // namespace + +TEST_P(CacheTest, CacheUniqueSeeds) { + // kQuasiRandomHashSeed should generate unique seeds (up to 2 billion before + // repeating) + UnorderedSet seeds_seen; + // Roughly sqrt(number of possible values) for a decent chance at detecting + // a random collision if it's possible (shouldn't be) + uint16_t kSamples = 20000; + seeds_seen.reserve(kSamples); + + // Hash seed should affect ordering of entries in the table, so we should + // have extremely high chance of seeing two entries ordered both ways. + bool seen_forward_order = false; + bool seen_reverse_order = false; + + for (int i = 0; i < kSamples; ++i) { + auto cache = NewCache(2, [=](ShardedCacheOptions& opts) { + opts.hash_seed = LRUCacheOptions::kQuasiRandomHashSeed; + opts.num_shard_bits = 0; + opts.metadata_charge_policy = kDontChargeCacheMetadata; + }); + auto val = cache->GetHashSeed(); + ASSERT_TRUE(seeds_seen.insert(val).second); + + ASSERT_OK(cache->Insert(EncodeKey(1), nullptr, &kHelper, /*charge*/ 1)); + ASSERT_OK(cache->Insert(EncodeKey(2), nullptr, &kHelper, /*charge*/ 1)); + + if (AreTwoCacheKeysOrdered(cache.get())) { + seen_forward_order = true; + } else { + seen_reverse_order = true; + } + } + + ASSERT_TRUE(seen_forward_order); + ASSERT_TRUE(seen_reverse_order); +} + +TEST_P(CacheTest, CacheHostSeed) { + // kHostHashSeed should generate a consistent seed within this process + // (and other processes on the same host, but not unit testing that). + // And we should be able to use that chosen seed as an explicit option + // (for debugging). + // And we should verify consistent ordering of entries. + uint32_t expected_seed = 0; + bool expected_order = false; + // 10 iterations -> chance of a random seed falsely appearing consistent + // should be low, just 1 in 2^9. + for (int i = 0; i < 10; ++i) { + auto cache = NewCache(2, [=](ShardedCacheOptions& opts) { + if (i != 5) { + opts.hash_seed = LRUCacheOptions::kHostHashSeed; + } else { + // Can be used as explicit seed + opts.hash_seed = static_cast(expected_seed); + ASSERT_GE(opts.hash_seed, 0); + } + opts.num_shard_bits = 0; + opts.metadata_charge_policy = kDontChargeCacheMetadata; + }); + ASSERT_OK(cache->Insert(EncodeKey(1), nullptr, &kHelper, /*charge*/ 1)); + ASSERT_OK(cache->Insert(EncodeKey(2), nullptr, &kHelper, /*charge*/ 1)); + uint32_t val = cache->GetHashSeed(); + bool order = AreTwoCacheKeysOrdered(cache.get()); + if (i != 0) { + ASSERT_EQ(val, expected_seed); + ASSERT_EQ(order, expected_order); + } else { + expected_seed = val; + expected_order = order; + } + } + // Printed for reference in case it's needed to reproduce other unit test + // failures on another host + fprintf(stderr, "kHostHashSeed -> %u\n", (unsigned)expected_seed); +} + +INSTANTIATE_TEST_CASE_P(CacheTestInstance, CacheTest, + secondary_cache_test_util::GetTestingCacheTypes()); +INSTANTIATE_TEST_CASE_P(CacheTestInstance, LRUCacheTest, + testing::Values(secondary_cache_test_util::kLRU)); + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/cache/charged_cache.cc b/librocksdb-sys/rocksdb/cache/charged_cache.cc new file mode 100644 index 0000000..e44288e --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/charged_cache.cc @@ -0,0 +1,109 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "cache/charged_cache.h" + +#include "cache/cache_reservation_manager.h" + +namespace ROCKSDB_NAMESPACE { + +ChargedCache::ChargedCache(std::shared_ptr cache, + std::shared_ptr block_cache) + : CacheWrapper(cache), + cache_res_mgr_(std::make_shared( + std::make_shared< + CacheReservationManagerImpl>( + block_cache))) {} + +Status ChargedCache::Insert(const Slice& key, ObjectPtr obj, + const CacheItemHelper* helper, size_t charge, + Handle** handle, Priority priority) { + Status s = target_->Insert(key, obj, helper, charge, handle, priority); + if (s.ok()) { + // Insert may cause the cache entry eviction if the cache is full. So we + // directly call the reservation manager to update the total memory used + // in the cache. + assert(cache_res_mgr_); + cache_res_mgr_->UpdateCacheReservation(target_->GetUsage()) + .PermitUncheckedError(); + } + return s; +} + +Cache::Handle* ChargedCache::Lookup(const Slice& key, + const CacheItemHelper* helper, + CreateContext* create_context, + Priority priority, Statistics* stats) { + auto handle = target_->Lookup(key, helper, create_context, priority, stats); + // Lookup may promote the KV pair from the secondary cache to the primary + // cache. So we directly call the reservation manager to update the total + // memory used in the cache. + if (helper && helper->create_cb) { + assert(cache_res_mgr_); + cache_res_mgr_->UpdateCacheReservation(target_->GetUsage()) + .PermitUncheckedError(); + } + return handle; +} + +void ChargedCache::WaitAll(AsyncLookupHandle* async_handles, size_t count) { + target_->WaitAll(async_handles, count); + // In case of any promotions. Although some could finish by return of + // StartAsyncLookup, Wait/WaitAll will generally be used, so simpler to + // update here. + assert(cache_res_mgr_); + cache_res_mgr_->UpdateCacheReservation(target_->GetUsage()) + .PermitUncheckedError(); +} + +bool ChargedCache::Release(Cache::Handle* handle, bool useful, + bool erase_if_last_ref) { + size_t memory_used_delta = target_->GetUsage(handle); + bool erased = target_->Release(handle, useful, erase_if_last_ref); + if (erased) { + assert(cache_res_mgr_); + cache_res_mgr_ + ->UpdateCacheReservation(memory_used_delta, /* increase */ false) + .PermitUncheckedError(); + } + return erased; +} + +bool ChargedCache::Release(Cache::Handle* handle, bool erase_if_last_ref) { + size_t memory_used_delta = target_->GetUsage(handle); + bool erased = target_->Release(handle, erase_if_last_ref); + if (erased) { + assert(cache_res_mgr_); + cache_res_mgr_ + ->UpdateCacheReservation(memory_used_delta, /* increase */ false) + .PermitUncheckedError(); + } + return erased; +} + +void ChargedCache::Erase(const Slice& key) { + target_->Erase(key); + assert(cache_res_mgr_); + cache_res_mgr_->UpdateCacheReservation(target_->GetUsage()) + .PermitUncheckedError(); +} + +void ChargedCache::EraseUnRefEntries() { + target_->EraseUnRefEntries(); + assert(cache_res_mgr_); + cache_res_mgr_->UpdateCacheReservation(target_->GetUsage()) + .PermitUncheckedError(); +} + +void ChargedCache::SetCapacity(size_t capacity) { + target_->SetCapacity(capacity); + // SetCapacity can result in evictions when the cache capacity is decreased, + // so we would want to update the cache reservation here as well. + assert(cache_res_mgr_); + cache_res_mgr_->UpdateCacheReservation(target_->GetUsage()) + .PermitUncheckedError(); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/charged_cache.h b/librocksdb-sys/rocksdb/cache/charged_cache.h new file mode 100644 index 0000000..f2eacb9 --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/charged_cache.h @@ -0,0 +1,59 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "port/port.h" +#include "rocksdb/advanced_cache.h" + +namespace ROCKSDB_NAMESPACE { + +class ConcurrentCacheReservationManager; + +// A cache interface which wraps around another cache and takes care of +// reserving space in block cache towards a single global memory limit, and +// forwards all the calls to the underlying cache. +class ChargedCache : public CacheWrapper { + public: + ChargedCache(std::shared_ptr cache, + std::shared_ptr block_cache); + + Status Insert(const Slice& key, ObjectPtr obj, const CacheItemHelper* helper, + size_t charge, Handle** handle = nullptr, + Priority priority = Priority::LOW) override; + + Cache::Handle* Lookup(const Slice& key, const CacheItemHelper* helper, + CreateContext* create_context, + Priority priority = Priority::LOW, + Statistics* stats = nullptr) override; + + void WaitAll(AsyncLookupHandle* async_handles, size_t count) override; + + bool Release(Cache::Handle* handle, bool useful, + bool erase_if_last_ref = false) override; + bool Release(Cache::Handle* handle, bool erase_if_last_ref = false) override; + + void Erase(const Slice& key) override; + void EraseUnRefEntries() override; + + static const char* kClassName() { return "ChargedCache"; } + const char* Name() const override { return kClassName(); } + + void SetCapacity(size_t capacity) override; + + inline Cache* GetCache() const { return target_.get(); } + + inline ConcurrentCacheReservationManager* TEST_GetCacheReservationManager() + const { + return cache_res_mgr_.get(); + } + + private: + std::shared_ptr cache_res_mgr_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/clock_cache.cc b/librocksdb-sys/rocksdb/cache/clock_cache.cc new file mode 100644 index 0000000..1695500 --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/clock_cache.cc @@ -0,0 +1,1557 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "cache/clock_cache.h" + +#include +#include +#include + +#include "cache/cache_key.h" +#include "cache/secondary_cache_adapter.h" +#include "logging/logging.h" +#include "monitoring/perf_context_imp.h" +#include "monitoring/statistics_impl.h" +#include "port/lang.h" +#include "rocksdb/env.h" +#include "util/hash.h" +#include "util/math.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +namespace clock_cache { + +namespace { +inline uint64_t GetRefcount(uint64_t meta) { + return ((meta >> ClockHandle::kAcquireCounterShift) - + (meta >> ClockHandle::kReleaseCounterShift)) & + ClockHandle::kCounterMask; +} + +inline uint64_t GetInitialCountdown(Cache::Priority priority) { + // Set initial clock data from priority + // TODO: configuration parameters for priority handling and clock cycle + // count? + switch (priority) { + case Cache::Priority::HIGH: + return ClockHandle::kHighCountdown; + default: + assert(false); + FALLTHROUGH_INTENDED; + case Cache::Priority::LOW: + return ClockHandle::kLowCountdown; + case Cache::Priority::BOTTOM: + return ClockHandle::kBottomCountdown; + } +} + +inline void MarkEmpty(ClockHandle& h) { +#ifndef NDEBUG + // Mark slot as empty, with assertion + uint64_t meta = h.meta.exchange(0, std::memory_order_release); + assert(meta >> ClockHandle::kStateShift == ClockHandle::kStateConstruction); +#else + // Mark slot as empty + h.meta.store(0, std::memory_order_release); +#endif +} + +inline void FreeDataMarkEmpty(ClockHandle& h, MemoryAllocator* allocator) { + // NOTE: in theory there's more room for parallelism if we copy the handle + // data and delay actions like this until after marking the entry as empty, + // but performance tests only show a regression by copying the few words + // of data. + h.FreeData(allocator); + + MarkEmpty(h); +} + +// Called to undo the effect of referencing an entry for internal purposes, +// so it should not be marked as having been used. +inline void Unref(const ClockHandle& h, uint64_t count = 1) { + // Pretend we never took the reference + // WART: there's a tiny chance we release last ref to invisible + // entry here. If that happens, we let eviction take care of it. + h.meta.fetch_sub(ClockHandle::kAcquireIncrement * count, + std::memory_order_release); +} + +inline bool ClockUpdate(ClockHandle& h) { + uint64_t meta = h.meta.load(std::memory_order_relaxed); + + uint64_t acquire_count = + (meta >> ClockHandle::kAcquireCounterShift) & ClockHandle::kCounterMask; + uint64_t release_count = + (meta >> ClockHandle::kReleaseCounterShift) & ClockHandle::kCounterMask; + // fprintf(stderr, "ClockUpdate @ %p: %lu %lu %u\n", &h, acquire_count, + // release_count, (unsigned)(meta >> ClockHandle::kStateShift)); + if (acquire_count != release_count) { + // Only clock update entries with no outstanding refs + return false; + } + if (!((meta >> ClockHandle::kStateShift) & ClockHandle::kStateShareableBit)) { + // Only clock update Shareable entries + return false; + } + if ((meta >> ClockHandle::kStateShift == ClockHandle::kStateVisible) && + acquire_count > 0) { + // Decrement clock + uint64_t new_count = + std::min(acquire_count - 1, uint64_t{ClockHandle::kMaxCountdown} - 1); + // Compare-exchange in the decremented clock info, but + // not aggressively + uint64_t new_meta = + (uint64_t{ClockHandle::kStateVisible} << ClockHandle::kStateShift) | + (new_count << ClockHandle::kReleaseCounterShift) | + (new_count << ClockHandle::kAcquireCounterShift); + h.meta.compare_exchange_strong(meta, new_meta, std::memory_order_relaxed); + return false; + } + // Otherwise, remove entry (either unreferenced invisible or + // unreferenced and expired visible). + if (h.meta.compare_exchange_strong( + meta, + uint64_t{ClockHandle::kStateConstruction} << ClockHandle::kStateShift, + std::memory_order_acquire)) { + // Took ownership. + return true; + } else { + // Compare-exchange failing probably + // indicates the entry was used, so skip it in that case. + return false; + } +} + +// If an entry doesn't receive clock updates but is repeatedly referenced & +// released, the acquire and release counters could overflow without some +// intervention. This is that intervention, which should be inexpensive +// because it only incurs a simple, very predictable check. (Applying a bit +// mask in addition to an increment to every Release likely would be +// relatively expensive, because it's an extra atomic update.) +// +// We do have to assume that we never have many millions of simultaneous +// references to a cache handle, because we cannot represent so many +// references with the difference in counters, masked to the number of +// counter bits. Similarly, we assume there aren't millions of threads +// holding transient references (which might be "undone" rather than +// released by the way). +// +// Consider these possible states for each counter: +// low: less than kMaxCountdown +// medium: kMaxCountdown to half way to overflow + kMaxCountdown +// high: half way to overflow + kMaxCountdown, or greater +// +// And these possible states for the combination of counters: +// acquire / release +// ------- ------- +// low low - Normal / common, with caveats (see below) +// medium low - Can happen while holding some refs +// high low - Violates assumptions (too many refs) +// low medium - Violates assumptions (refs underflow, etc.) +// medium medium - Normal (very read heavy cache) +// high medium - Can happen while holding some refs +// low high - This function is supposed to prevent +// medium high - Violates assumptions (refs underflow, etc.) +// high high - Needs CorrectNearOverflow +// +// Basically, this function detects (high, high) state (inferred from +// release alone being high) and bumps it back down to (medium, medium) +// state with the same refcount and the same logical countdown counter +// (everything > kMaxCountdown is logically the same). Note that bumping +// down to (low, low) would modify the countdown counter, so is "reserved" +// in a sense. +// +// If near-overflow correction is triggered here, there's no guarantee +// that another thread hasn't freed the entry and replaced it with another. +// Therefore, it must be the case that the correction does not affect +// entries unless they are very old (many millions of acquire-release cycles). +// (Our bit manipulation is indeed idempotent and only affects entries in +// exceptional cases.) We assume a pre-empted thread will not stall that long. +// If it did, the state could be corrupted in the (unlikely) case that the top +// bit of the acquire counter is set but not the release counter, and thus +// we only clear the top bit of the acquire counter on resumption. It would +// then appear that there are too many refs and the entry would be permanently +// pinned (which is not terrible for an exceptionally rare occurrence), unless +// it is referenced enough (at least kMaxCountdown more times) for the release +// counter to reach "high" state again and bumped back to "medium." (This +// motivates only checking for release counter in high state, not both in high +// state.) +inline void CorrectNearOverflow(uint64_t old_meta, + std::atomic& meta) { + // We clear both top-most counter bits at the same time. + constexpr uint64_t kCounterTopBit = uint64_t{1} + << (ClockHandle::kCounterNumBits - 1); + constexpr uint64_t kClearBits = + (kCounterTopBit << ClockHandle::kAcquireCounterShift) | + (kCounterTopBit << ClockHandle::kReleaseCounterShift); + // A simple check that allows us to initiate clearing the top bits for + // a large portion of the "high" state space on release counter. + constexpr uint64_t kCheckBits = + (kCounterTopBit | (ClockHandle::kMaxCountdown + 1)) + << ClockHandle::kReleaseCounterShift; + + if (UNLIKELY(old_meta & kCheckBits)) { + meta.fetch_and(~kClearBits, std::memory_order_relaxed); + } +} + +inline bool BeginSlotInsert(const ClockHandleBasicData& proto, ClockHandle& h, + uint64_t initial_countdown, bool* already_matches) { + assert(*already_matches == false); + // Optimistically transition the slot from "empty" to + // "under construction" (no effect on other states) + uint64_t old_meta = h.meta.fetch_or( + uint64_t{ClockHandle::kStateOccupiedBit} << ClockHandle::kStateShift, + std::memory_order_acq_rel); + uint64_t old_state = old_meta >> ClockHandle::kStateShift; + + if (old_state == ClockHandle::kStateEmpty) { + // We've started inserting into an available slot, and taken + // ownership. + return true; + } else if (old_state != ClockHandle::kStateVisible) { + // Slot not usable / touchable now + return false; + } + // Existing, visible entry, which might be a match. + // But first, we need to acquire a ref to read it. In fact, number of + // refs for initial countdown, so that we boost the clock state if + // this is a match. + old_meta = + h.meta.fetch_add(ClockHandle::kAcquireIncrement * initial_countdown, + std::memory_order_acq_rel); + // Like Lookup + if ((old_meta >> ClockHandle::kStateShift) == ClockHandle::kStateVisible) { + // Acquired a read reference + if (h.hashed_key == proto.hashed_key) { + // Match. Release in a way that boosts the clock state + old_meta = + h.meta.fetch_add(ClockHandle::kReleaseIncrement * initial_countdown, + std::memory_order_acq_rel); + // Correct for possible (but rare) overflow + CorrectNearOverflow(old_meta, h.meta); + // Insert detached instead (only if return handle needed) + *already_matches = true; + return false; + } else { + // Mismatch. + Unref(h, initial_countdown); + } + } else if (UNLIKELY((old_meta >> ClockHandle::kStateShift) == + ClockHandle::kStateInvisible)) { + // Pretend we never took the reference + Unref(h, initial_countdown); + } else { + // For other states, incrementing the acquire counter has no effect + // so we don't need to undo it. + // Slot not usable / touchable now. + } + return false; +} + +inline void FinishSlotInsert(const ClockHandleBasicData& proto, ClockHandle& h, + uint64_t initial_countdown, bool keep_ref) { + // Save data fields + ClockHandleBasicData* h_alias = &h; + *h_alias = proto; + + // Transition from "under construction" state to "visible" state + uint64_t new_meta = uint64_t{ClockHandle::kStateVisible} + << ClockHandle::kStateShift; + + // Maybe with an outstanding reference + new_meta |= initial_countdown << ClockHandle::kAcquireCounterShift; + new_meta |= (initial_countdown - keep_ref) + << ClockHandle::kReleaseCounterShift; + +#ifndef NDEBUG + // Save the state transition, with assertion + uint64_t old_meta = h.meta.exchange(new_meta, std::memory_order_release); + assert(old_meta >> ClockHandle::kStateShift == + ClockHandle::kStateConstruction); +#else + // Save the state transition + h.meta.store(new_meta, std::memory_order_release); +#endif +} + +bool TryInsert(const ClockHandleBasicData& proto, ClockHandle& h, + uint64_t initial_countdown, bool keep_ref, + bool* already_matches) { + bool b = BeginSlotInsert(proto, h, initial_countdown, already_matches); + if (b) { + FinishSlotInsert(proto, h, initial_countdown, keep_ref); + } + return b; +} + +// Func must be const HandleImpl& -> void callable +template +void ConstApplyToEntriesRange(const Func& func, const HandleImpl* begin, + const HandleImpl* end, + bool apply_if_will_be_deleted) { + uint64_t check_state_mask = ClockHandle::kStateShareableBit; + if (!apply_if_will_be_deleted) { + check_state_mask |= ClockHandle::kStateVisibleBit; + } + + for (const HandleImpl* h = begin; h < end; ++h) { + // Note: to avoid using compare_exchange, we have to be extra careful. + uint64_t old_meta = h->meta.load(std::memory_order_relaxed); + // Check if it's an entry visible to lookups + if ((old_meta >> ClockHandle::kStateShift) & check_state_mask) { + // Increment acquire counter. Note: it's possible that the entry has + // completely changed since we loaded old_meta, but incrementing acquire + // count is always safe. (Similar to optimistic Lookup here.) + old_meta = h->meta.fetch_add(ClockHandle::kAcquireIncrement, + std::memory_order_acquire); + // Check whether we actually acquired a reference. + if ((old_meta >> ClockHandle::kStateShift) & + ClockHandle::kStateShareableBit) { + // Apply func if appropriate + if ((old_meta >> ClockHandle::kStateShift) & check_state_mask) { + func(*h); + } + // Pretend we never took the reference + Unref(*h); + // No net change, so don't need to check for overflow + } else { + // For other states, incrementing the acquire counter has no effect + // so we don't need to undo it. Furthermore, we cannot safely undo + // it because we did not acquire a read reference to lock the + // entry in a Shareable state. + } + } + } +} + +} // namespace + +void ClockHandleBasicData::FreeData(MemoryAllocator* allocator) const { + if (helper->del_cb) { + helper->del_cb(value, allocator); + } +} + +template +HandleImpl* BaseClockTable::StandaloneInsert( + const ClockHandleBasicData& proto) { + // Heap allocated separate from table + HandleImpl* h = new HandleImpl(); + ClockHandleBasicData* h_alias = h; + *h_alias = proto; + h->SetStandalone(); + // Single reference (standalone entries only created if returning a refed + // Handle back to user) + uint64_t meta = uint64_t{ClockHandle::kStateInvisible} + << ClockHandle::kStateShift; + meta |= uint64_t{1} << ClockHandle::kAcquireCounterShift; + h->meta.store(meta, std::memory_order_release); + // Keep track of how much of usage is standalone + standalone_usage_.fetch_add(proto.GetTotalCharge(), + std::memory_order_relaxed); + return h; +} + +template +typename Table::HandleImpl* BaseClockTable::CreateStandalone( + ClockHandleBasicData& proto, size_t capacity, bool strict_capacity_limit, + bool allow_uncharged) { + Table& derived = static_cast(*this); + typename Table::InsertState state; + derived.StartInsert(state); + + const size_t total_charge = proto.GetTotalCharge(); + if (strict_capacity_limit) { + Status s = ChargeUsageMaybeEvictStrict( + total_charge, capacity, + /*need_evict_for_occupancy=*/false, state); + if (!s.ok()) { + if (allow_uncharged) { + proto.total_charge = 0; + } else { + return nullptr; + } + } + } else { + // Case strict_capacity_limit == false + bool success = ChargeUsageMaybeEvictNonStrict
( + total_charge, capacity, + /*need_evict_for_occupancy=*/false, state); + if (!success) { + // Force the issue + usage_.fetch_add(total_charge, std::memory_order_relaxed); + } + } + + return StandaloneInsert(proto); +} + +template +Status BaseClockTable::ChargeUsageMaybeEvictStrict( + size_t total_charge, size_t capacity, bool need_evict_for_occupancy, + typename Table::InsertState& state) { + if (total_charge > capacity) { + return Status::MemoryLimit( + "Cache entry too large for a single cache shard: " + + std::to_string(total_charge) + " > " + std::to_string(capacity)); + } + // Grab any available capacity, and free up any more required. + size_t old_usage = usage_.load(std::memory_order_relaxed); + size_t new_usage; + if (LIKELY(old_usage != capacity)) { + do { + new_usage = std::min(capacity, old_usage + total_charge); + } while (!usage_.compare_exchange_weak(old_usage, new_usage, + std::memory_order_relaxed)); + } else { + new_usage = old_usage; + } + // How much do we need to evict then? + size_t need_evict_charge = old_usage + total_charge - new_usage; + size_t request_evict_charge = need_evict_charge; + if (UNLIKELY(need_evict_for_occupancy) && request_evict_charge == 0) { + // Require at least 1 eviction. + request_evict_charge = 1; + } + if (request_evict_charge > 0) { + EvictionData data; + static_cast(this)->Evict(request_evict_charge, state, &data); + occupancy_.fetch_sub(data.freed_count, std::memory_order_release); + if (LIKELY(data.freed_charge > need_evict_charge)) { + assert(data.freed_count > 0); + // Evicted more than enough + usage_.fetch_sub(data.freed_charge - need_evict_charge, + std::memory_order_relaxed); + } else if (data.freed_charge < need_evict_charge || + (UNLIKELY(need_evict_for_occupancy) && data.freed_count == 0)) { + // Roll back to old usage minus evicted + usage_.fetch_sub(data.freed_charge + (new_usage - old_usage), + std::memory_order_relaxed); + if (data.freed_charge < need_evict_charge) { + return Status::MemoryLimit( + "Insert failed because unable to evict entries to stay within " + "capacity limit."); + } else { + return Status::MemoryLimit( + "Insert failed because unable to evict entries to stay within " + "table occupancy limit."); + } + } + // If we needed to evict something and we are proceeding, we must have + // evicted something. + assert(data.freed_count > 0); + } + return Status::OK(); +} + +template +inline bool BaseClockTable::ChargeUsageMaybeEvictNonStrict( + size_t total_charge, size_t capacity, bool need_evict_for_occupancy, + typename Table::InsertState& state) { + // For simplicity, we consider that either the cache can accept the insert + // with no evictions, or we must evict enough to make (at least) enough + // space. It could lead to unnecessary failures or excessive evictions in + // some extreme cases, but allows a fast, simple protocol. If we allow a + // race to get us over capacity, then we might never get back to capacity + // limit if the sizes of entries allow each insertion to evict the minimum + // charge. Thus, we should evict some extra if it's not a signifcant + // portion of the shard capacity. This can have the side benefit of + // involving fewer threads in eviction. + size_t old_usage = usage_.load(std::memory_order_relaxed); + size_t need_evict_charge; + // NOTE: if total_charge > old_usage, there isn't yet enough to evict + // `total_charge` amount. Even if we only try to evict `old_usage` amount, + // there's likely something referenced and we would eat CPU looking for + // enough to evict. + if (old_usage + total_charge <= capacity || total_charge > old_usage) { + // Good enough for me (might run over with a race) + need_evict_charge = 0; + } else { + // Try to evict enough space, and maybe some extra + need_evict_charge = total_charge; + if (old_usage > capacity) { + // Not too much to avoid thundering herd while avoiding strict + // synchronization, such as the compare_exchange used with strict + // capacity limit. + need_evict_charge += std::min(capacity / 1024, total_charge) + 1; + } + } + if (UNLIKELY(need_evict_for_occupancy) && need_evict_charge == 0) { + // Special case: require at least 1 eviction if we only have to + // deal with occupancy + need_evict_charge = 1; + } + EvictionData data; + if (need_evict_charge > 0) { + static_cast(this)->Evict(need_evict_charge, state, &data); + // Deal with potential occupancy deficit + if (UNLIKELY(need_evict_for_occupancy) && data.freed_count == 0) { + assert(data.freed_charge == 0); + // Can't meet occupancy requirement + return false; + } else { + // Update occupancy for evictions + occupancy_.fetch_sub(data.freed_count, std::memory_order_release); + } + } + // Track new usage even if we weren't able to evict enough + usage_.fetch_add(total_charge - data.freed_charge, std::memory_order_relaxed); + // No underflow + assert(usage_.load(std::memory_order_relaxed) < SIZE_MAX / 2); + // Success + return true; +} + +void BaseClockTable::TrackAndReleaseEvictedEntry( + ClockHandle* h, BaseClockTable::EvictionData* data) { + data->freed_charge += h->GetTotalCharge(); + data->freed_count += 1; + + bool took_value_ownership = false; + if (eviction_callback_) { + // For key reconstructed from hash + UniqueId64x2 unhashed; + took_value_ownership = + eviction_callback_(ClockCacheShard::ReverseHash( + h->GetHash(), &unhashed, hash_seed_), + reinterpret_cast(h)); + } + if (!took_value_ownership) { + h->FreeData(allocator_); + } + MarkEmpty(*h); +} + +template +Status BaseClockTable::Insert(const ClockHandleBasicData& proto, + typename Table::HandleImpl** handle, + Cache::Priority priority, size_t capacity, + bool strict_capacity_limit) { + using HandleImpl = typename Table::HandleImpl; + Table& derived = static_cast(*this); + + typename Table::InsertState state; + derived.StartInsert(state); + + // Do we have the available occupancy? Optimistically assume we do + // and deal with it if we don't. + size_t old_occupancy = occupancy_.fetch_add(1, std::memory_order_acquire); + // Whether we over-committed and need an eviction to make up for it + bool need_evict_for_occupancy = + !derived.GrowIfNeeded(old_occupancy + 1, state); + + // Usage/capacity handling is somewhat different depending on + // strict_capacity_limit, but mostly pessimistic. + bool use_standalone_insert = false; + const size_t total_charge = proto.GetTotalCharge(); + if (strict_capacity_limit) { + Status s = ChargeUsageMaybeEvictStrict
( + total_charge, capacity, need_evict_for_occupancy, state); + if (!s.ok()) { + // Revert occupancy + occupancy_.fetch_sub(1, std::memory_order_relaxed); + return s; + } + } else { + // Case strict_capacity_limit == false + bool success = ChargeUsageMaybeEvictNonStrict
( + total_charge, capacity, need_evict_for_occupancy, state); + if (!success) { + // Revert occupancy + occupancy_.fetch_sub(1, std::memory_order_relaxed); + if (handle == nullptr) { + // Don't insert the entry but still return ok, as if the entry + // inserted into cache and evicted immediately. + proto.FreeData(allocator_); + return Status::OK(); + } else { + // Need to track usage of fallback standalone insert + usage_.fetch_add(total_charge, std::memory_order_relaxed); + use_standalone_insert = true; + } + } + } + + if (!use_standalone_insert) { + // Attempt a table insert, but abort if we find an existing entry for the + // key. If we were to overwrite old entries, we would either + // * Have to gain ownership over an existing entry to overwrite it, which + // would only work if there are no outstanding (read) references and would + // create a small gap in availability of the entry (old or new) to lookups. + // * Have to insert into a suboptimal location (more probes) so that the + // old entry can be kept around as well. + + uint64_t initial_countdown = GetInitialCountdown(priority); + assert(initial_countdown > 0); + + HandleImpl* e = + derived.DoInsert(proto, initial_countdown, handle != nullptr, state); + + if (e) { + // Successfully inserted + if (handle) { + *handle = e; + } + return Status::OK(); + } + // Not inserted + // Revert occupancy + occupancy_.fetch_sub(1, std::memory_order_relaxed); + // Maybe fall back on standalone insert + if (handle == nullptr) { + // Revert usage + usage_.fetch_sub(total_charge, std::memory_order_relaxed); + // No underflow + assert(usage_.load(std::memory_order_relaxed) < SIZE_MAX / 2); + // As if unrefed entry immdiately evicted + proto.FreeData(allocator_); + return Status::OK(); + } + + use_standalone_insert = true; + } + + // Run standalone insert + assert(use_standalone_insert); + + *handle = StandaloneInsert(proto); + + // The OkOverwritten status is used to count "redundant" insertions into + // block cache. This implementation doesn't strictly check for redundant + // insertions, but we instead are probably interested in how many insertions + // didn't go into the table (instead "standalone"), which could be redundant + // Insert or some other reason (use_standalone_insert reasons above). + return Status::OkOverwritten(); +} + +void BaseClockTable::Ref(ClockHandle& h) { + // Increment acquire counter + uint64_t old_meta = h.meta.fetch_add(ClockHandle::kAcquireIncrement, + std::memory_order_acquire); + + assert((old_meta >> ClockHandle::kStateShift) & + ClockHandle::kStateShareableBit); + // Must have already had a reference + assert(GetRefcount(old_meta) > 0); + (void)old_meta; +} + +#ifndef NDEBUG +void BaseClockTable::TEST_RefN(ClockHandle& h, size_t n) { + // Increment acquire counter + uint64_t old_meta = h.meta.fetch_add(n * ClockHandle::kAcquireIncrement, + std::memory_order_acquire); + + assert((old_meta >> ClockHandle::kStateShift) & + ClockHandle::kStateShareableBit); + (void)old_meta; +} + +void BaseClockTable::TEST_ReleaseNMinus1(ClockHandle* h, size_t n) { + assert(n > 0); + + // Like n-1 Releases, but assumes one more will happen in the caller to take + // care of anything like erasing an unreferenced, invisible entry. + uint64_t old_meta = h->meta.fetch_add( + (n - 1) * ClockHandle::kReleaseIncrement, std::memory_order_acquire); + assert((old_meta >> ClockHandle::kStateShift) & + ClockHandle::kStateShareableBit); + (void)old_meta; +} +#endif + +HyperClockTable::HyperClockTable( + size_t capacity, bool /*strict_capacity_limit*/, + CacheMetadataChargePolicy metadata_charge_policy, + MemoryAllocator* allocator, + const Cache::EvictionCallback* eviction_callback, const uint32_t* hash_seed, + const Opts& opts) + : BaseClockTable(metadata_charge_policy, allocator, eviction_callback, + hash_seed), + length_bits_(CalcHashBits(capacity, opts.estimated_value_size, + metadata_charge_policy)), + length_bits_mask_((size_t{1} << length_bits_) - 1), + occupancy_limit_(static_cast((uint64_t{1} << length_bits_) * + kStrictLoadFactor)), + array_(new HandleImpl[size_t{1} << length_bits_]) { + if (metadata_charge_policy == + CacheMetadataChargePolicy::kFullChargeCacheMetadata) { + usage_ += size_t{GetTableSize()} * sizeof(HandleImpl); + } + + static_assert(sizeof(HandleImpl) == 64U, + "Expecting size / alignment with common cache line size"); +} + +HyperClockTable::~HyperClockTable() { + // Assumes there are no references or active operations on any slot/element + // in the table. + for (size_t i = 0; i < GetTableSize(); i++) { + HandleImpl& h = array_[i]; + switch (h.meta >> ClockHandle::kStateShift) { + case ClockHandle::kStateEmpty: + // noop + break; + case ClockHandle::kStateInvisible: // rare but possible + case ClockHandle::kStateVisible: + assert(GetRefcount(h.meta) == 0); + h.FreeData(allocator_); +#ifndef NDEBUG + Rollback(h.hashed_key, &h); + ReclaimEntryUsage(h.GetTotalCharge()); +#endif + break; + // otherwise + default: + assert(false); + break; + } + } + +#ifndef NDEBUG + for (size_t i = 0; i < GetTableSize(); i++) { + assert(array_[i].displacements.load() == 0); + } +#endif + + assert(usage_.load() == 0 || + usage_.load() == size_t{GetTableSize()} * sizeof(HandleImpl)); + assert(occupancy_ == 0); +} + +void HyperClockTable::StartInsert(InsertState&) {} + +bool HyperClockTable::GrowIfNeeded(size_t new_occupancy, InsertState&) { + return new_occupancy <= occupancy_limit_; +} + +HyperClockTable::HandleImpl* HyperClockTable::DoInsert( + const ClockHandleBasicData& proto, uint64_t initial_countdown, + bool keep_ref, InsertState&) { + bool already_matches = false; + HandleImpl* e = FindSlot( + proto.hashed_key, + [&](HandleImpl* h) { + return TryInsert(proto, *h, initial_countdown, keep_ref, + &already_matches); + }, + [&](HandleImpl* h) { + if (already_matches) { + // Stop searching & roll back displacements + Rollback(proto.hashed_key, h); + return true; + } else { + // Keep going + return false; + } + }, + [&](HandleImpl* h, bool is_last) { + if (is_last) { + // Search is ending. Roll back displacements + Rollback(proto.hashed_key, h); + } else { + h->displacements.fetch_add(1, std::memory_order_relaxed); + } + }); + if (already_matches) { + // Insertion skipped + return nullptr; + } + if (e != nullptr) { + // Successfully inserted + return e; + } + // Else, no available slot found. Occupancy check should generally prevent + // this, except it's theoretically possible for other threads to evict and + // replace entries in the right order to hit every slot when it is populated. + // Assuming random hashing, the chance of that should be no higher than + // pow(kStrictLoadFactor, n) for n slots. That should be infeasible for + // roughly n >= 256, so if this assertion fails, that suggests something is + // going wrong. + assert(GetTableSize() < 256); + return nullptr; +} + +HyperClockTable::HandleImpl* HyperClockTable::Lookup( + const UniqueId64x2& hashed_key) { + HandleImpl* e = FindSlot( + hashed_key, + [&](HandleImpl* h) { + // Mostly branch-free version (similar performance) + /* + uint64_t old_meta = h->meta.fetch_add(ClockHandle::kAcquireIncrement, + std::memory_order_acquire); + bool Shareable = (old_meta >> (ClockHandle::kStateShift + 1)) & 1U; + bool visible = (old_meta >> ClockHandle::kStateShift) & 1U; + bool match = (h->key == key) & visible; + h->meta.fetch_sub(static_cast(Shareable & !match) << + ClockHandle::kAcquireCounterShift, std::memory_order_release); return + match; + */ + // Optimistic lookup should pay off when the table is relatively + // sparse. + constexpr bool kOptimisticLookup = true; + uint64_t old_meta; + if (!kOptimisticLookup) { + old_meta = h->meta.load(std::memory_order_acquire); + if ((old_meta >> ClockHandle::kStateShift) != + ClockHandle::kStateVisible) { + return false; + } + } + // (Optimistically) increment acquire counter + old_meta = h->meta.fetch_add(ClockHandle::kAcquireIncrement, + std::memory_order_acquire); + // Check if it's an entry visible to lookups + if ((old_meta >> ClockHandle::kStateShift) == + ClockHandle::kStateVisible) { + // Acquired a read reference + if (h->hashed_key == hashed_key) { + // Match + return true; + } else { + // Mismatch. Pretend we never took the reference + Unref(*h); + } + } else if (UNLIKELY((old_meta >> ClockHandle::kStateShift) == + ClockHandle::kStateInvisible)) { + // Pretend we never took the reference + Unref(*h); + } else { + // For other states, incrementing the acquire counter has no effect + // so we don't need to undo it. Furthermore, we cannot safely undo + // it because we did not acquire a read reference to lock the + // entry in a Shareable state. + } + return false; + }, + [&](HandleImpl* h) { + return h->displacements.load(std::memory_order_relaxed) == 0; + }, + [&](HandleImpl* /*h*/, bool /*is_last*/) {}); + + return e; +} + +bool HyperClockTable::Release(HandleImpl* h, bool useful, + bool erase_if_last_ref) { + // In contrast with LRUCache's Release, this function won't delete the handle + // when the cache is above capacity and the reference is the last one. Space + // is only freed up by EvictFromClock (called by Insert when space is needed) + // and Erase. We do this to avoid an extra atomic read of the variable usage_. + + uint64_t old_meta; + if (useful) { + // Increment release counter to indicate was used + old_meta = h->meta.fetch_add(ClockHandle::kReleaseIncrement, + std::memory_order_release); + } else { + // Decrement acquire counter to pretend it never happened + old_meta = h->meta.fetch_sub(ClockHandle::kAcquireIncrement, + std::memory_order_release); + } + + assert((old_meta >> ClockHandle::kStateShift) & + ClockHandle::kStateShareableBit); + // No underflow + assert(((old_meta >> ClockHandle::kAcquireCounterShift) & + ClockHandle::kCounterMask) != + ((old_meta >> ClockHandle::kReleaseCounterShift) & + ClockHandle::kCounterMask)); + + if (erase_if_last_ref || UNLIKELY(old_meta >> ClockHandle::kStateShift == + ClockHandle::kStateInvisible)) { + // Update for last fetch_add op + if (useful) { + old_meta += ClockHandle::kReleaseIncrement; + } else { + old_meta -= ClockHandle::kAcquireIncrement; + } + // Take ownership if no refs + do { + if (GetRefcount(old_meta) != 0) { + // Not last ref at some point in time during this Release call + // Correct for possible (but rare) overflow + CorrectNearOverflow(old_meta, h->meta); + return false; + } + if ((old_meta & (uint64_t{ClockHandle::kStateShareableBit} + << ClockHandle::kStateShift)) == 0) { + // Someone else took ownership + return false; + } + // Note that there's a small chance that we release, another thread + // replaces this entry with another, reaches zero refs, and then we end + // up erasing that other entry. That's an acceptable risk / imprecision. + } while (!h->meta.compare_exchange_weak( + old_meta, + uint64_t{ClockHandle::kStateConstruction} << ClockHandle::kStateShift, + std::memory_order_acquire)); + // Took ownership + size_t total_charge = h->GetTotalCharge(); + if (UNLIKELY(h->IsStandalone())) { + h->FreeData(allocator_); + // Delete standalone handle + delete h; + standalone_usage_.fetch_sub(total_charge, std::memory_order_relaxed); + usage_.fetch_sub(total_charge, std::memory_order_relaxed); + } else { + Rollback(h->hashed_key, h); + FreeDataMarkEmpty(*h, allocator_); + ReclaimEntryUsage(total_charge); + } + return true; + } else { + // Correct for possible (but rare) overflow + CorrectNearOverflow(old_meta, h->meta); + return false; + } +} + +#ifndef NDEBUG +void HyperClockTable::TEST_ReleaseN(HandleImpl* h, size_t n) { + if (n > 0) { + // Do n-1 simple releases first + TEST_ReleaseNMinus1(h, n); + + // Then the last release might be more involved + Release(h, /*useful*/ true, /*erase_if_last_ref*/ false); + } +} +#endif + +void HyperClockTable::Erase(const UniqueId64x2& hashed_key) { + (void)FindSlot( + hashed_key, + [&](HandleImpl* h) { + // Could be multiple entries in rare cases. Erase them all. + // Optimistically increment acquire counter + uint64_t old_meta = h->meta.fetch_add(ClockHandle::kAcquireIncrement, + std::memory_order_acquire); + // Check if it's an entry visible to lookups + if ((old_meta >> ClockHandle::kStateShift) == + ClockHandle::kStateVisible) { + // Acquired a read reference + if (h->hashed_key == hashed_key) { + // Match. Set invisible. + old_meta = + h->meta.fetch_and(~(uint64_t{ClockHandle::kStateVisibleBit} + << ClockHandle::kStateShift), + std::memory_order_acq_rel); + // Apply update to local copy + old_meta &= ~(uint64_t{ClockHandle::kStateVisibleBit} + << ClockHandle::kStateShift); + for (;;) { + uint64_t refcount = GetRefcount(old_meta); + assert(refcount > 0); + if (refcount > 1) { + // Not last ref at some point in time during this Erase call + // Pretend we never took the reference + Unref(*h); + break; + } else if (h->meta.compare_exchange_weak( + old_meta, + uint64_t{ClockHandle::kStateConstruction} + << ClockHandle::kStateShift, + std::memory_order_acq_rel)) { + // Took ownership + assert(hashed_key == h->hashed_key); + size_t total_charge = h->GetTotalCharge(); + FreeDataMarkEmpty(*h, allocator_); + ReclaimEntryUsage(total_charge); + // We already have a copy of hashed_key in this case, so OK to + // delay Rollback until after releasing the entry + Rollback(hashed_key, h); + break; + } + } + } else { + // Mismatch. Pretend we never took the reference + Unref(*h); + } + } else if (UNLIKELY((old_meta >> ClockHandle::kStateShift) == + ClockHandle::kStateInvisible)) { + // Pretend we never took the reference + Unref(*h); + } else { + // For other states, incrementing the acquire counter has no effect + // so we don't need to undo it. + } + return false; + }, + [&](HandleImpl* h) { + return h->displacements.load(std::memory_order_relaxed) == 0; + }, + [&](HandleImpl* /*h*/, bool /*is_last*/) {}); +} + +void HyperClockTable::EraseUnRefEntries() { + for (size_t i = 0; i <= this->length_bits_mask_; i++) { + HandleImpl& h = array_[i]; + + uint64_t old_meta = h.meta.load(std::memory_order_relaxed); + if (old_meta & (uint64_t{ClockHandle::kStateShareableBit} + << ClockHandle::kStateShift) && + GetRefcount(old_meta) == 0 && + h.meta.compare_exchange_strong(old_meta, + uint64_t{ClockHandle::kStateConstruction} + << ClockHandle::kStateShift, + std::memory_order_acquire)) { + // Took ownership + size_t total_charge = h.GetTotalCharge(); + Rollback(h.hashed_key, &h); + FreeDataMarkEmpty(h, allocator_); + ReclaimEntryUsage(total_charge); + } + } +} + +template +inline HyperClockTable::HandleImpl* HyperClockTable::FindSlot( + const UniqueId64x2& hashed_key, const MatchFn& match_fn, + const AbortFn& abort_fn, const UpdateFn& update_fn) { + // NOTE: upper 32 bits of hashed_key[0] is used for sharding + // + // We use double-hashing probing. Every probe in the sequence is a + // pseudorandom integer, computed as a linear function of two random hashes, + // which we call base and increment. Specifically, the i-th probe is base + i + // * increment modulo the table size. + size_t base = static_cast(hashed_key[1]); + // We use an odd increment, which is relatively prime with the power-of-two + // table size. This implies that we cycle back to the first probe only + // after probing every slot exactly once. + // TODO: we could also reconsider linear probing, though locality benefits + // are limited because each slot is a full cache line + size_t increment = static_cast(hashed_key[0]) | 1U; + size_t first = ModTableSize(base); + size_t current = first; + bool is_last; + do { + HandleImpl* h = &array_[current]; + if (match_fn(h)) { + return h; + } + if (abort_fn(h)) { + return nullptr; + } + current = ModTableSize(current + increment); + is_last = current == first; + update_fn(h, is_last); + } while (!is_last); + // We looped back. + return nullptr; +} + +inline void HyperClockTable::Rollback(const UniqueId64x2& hashed_key, + const HandleImpl* h) { + size_t current = ModTableSize(hashed_key[1]); + size_t increment = static_cast(hashed_key[0]) | 1U; + while (&array_[current] != h) { + array_[current].displacements.fetch_sub(1, std::memory_order_relaxed); + current = ModTableSize(current + increment); + } +} + +inline void HyperClockTable::ReclaimEntryUsage(size_t total_charge) { + auto old_occupancy = occupancy_.fetch_sub(1U, std::memory_order_release); + (void)old_occupancy; + // No underflow + assert(old_occupancy > 0); + auto old_usage = usage_.fetch_sub(total_charge, std::memory_order_relaxed); + (void)old_usage; + // No underflow + assert(old_usage >= total_charge); +} + +inline void HyperClockTable::Evict(size_t requested_charge, InsertState&, + EvictionData* data) { + // precondition + assert(requested_charge > 0); + + // TODO: make a tuning parameter? + constexpr size_t step_size = 4; + + // First (concurrent) increment clock pointer + uint64_t old_clock_pointer = + clock_pointer_.fetch_add(step_size, std::memory_order_relaxed); + + // Cap the eviction effort at this thread (along with those operating in + // parallel) circling through the whole structure kMaxCountdown times. + // In other words, this eviction run must find something/anything that is + // unreferenced at start of and during the eviction run that isn't reclaimed + // by a concurrent eviction run. + uint64_t max_clock_pointer = + old_clock_pointer + (ClockHandle::kMaxCountdown << length_bits_); + + for (;;) { + for (size_t i = 0; i < step_size; i++) { + HandleImpl& h = array_[ModTableSize(Lower32of64(old_clock_pointer + i))]; + bool evicting = ClockUpdate(h); + if (evicting) { + Rollback(h.hashed_key, &h); + TrackAndReleaseEvictedEntry(&h, data); + } + } + + // Loop exit condition + if (data->freed_charge >= requested_charge) { + return; + } + if (old_clock_pointer >= max_clock_pointer) { + return; + } + + // Advance clock pointer (concurrently) + old_clock_pointer = + clock_pointer_.fetch_add(step_size, std::memory_order_relaxed); + } +} + +template +ClockCacheShard
::ClockCacheShard( + size_t capacity, bool strict_capacity_limit, + CacheMetadataChargePolicy metadata_charge_policy, + MemoryAllocator* allocator, + const Cache::EvictionCallback* eviction_callback, const uint32_t* hash_seed, + const typename Table::Opts& opts) + : CacheShardBase(metadata_charge_policy), + table_(capacity, strict_capacity_limit, metadata_charge_policy, allocator, + eviction_callback, hash_seed, opts), + capacity_(capacity), + strict_capacity_limit_(strict_capacity_limit) { + // Initial charge metadata should not exceed capacity + assert(table_.GetUsage() <= capacity_ || capacity_ < sizeof(HandleImpl)); +} + +template +void ClockCacheShard
::EraseUnRefEntries() { + table_.EraseUnRefEntries(); +} + +template +void ClockCacheShard
::ApplyToSomeEntries( + const std::function& callback, + size_t average_entries_per_lock, size_t* state) { + // The state will be a simple index into the table. Even with a dynamic + // hyper clock cache, entries will generally stay in their existing + // slots, so we don't need to be aware of the high-level organization + // that makes lookup efficient. + size_t length = table_.GetTableSize(); + + assert(average_entries_per_lock > 0); + + size_t index_begin = *state; + size_t index_end = index_begin + average_entries_per_lock; + if (index_end >= length) { + // Going to end. + index_end = length; + *state = SIZE_MAX; + } else { + *state = index_end; + } + + auto hash_seed = table_.GetHashSeed(); + ConstApplyToEntriesRange( + [callback, hash_seed](const HandleImpl& h) { + UniqueId64x2 unhashed; + callback(ReverseHash(h.hashed_key, &unhashed, hash_seed), h.value, + h.GetTotalCharge(), h.helper); + }, + table_.HandlePtr(index_begin), table_.HandlePtr(index_end), false); +} + +int HyperClockTable::CalcHashBits( + size_t capacity, size_t estimated_value_size, + CacheMetadataChargePolicy metadata_charge_policy) { + double average_slot_charge = estimated_value_size * kLoadFactor; + if (metadata_charge_policy == kFullChargeCacheMetadata) { + average_slot_charge += sizeof(HandleImpl); + } + assert(average_slot_charge > 0.0); + uint64_t num_slots = + static_cast(capacity / average_slot_charge + 0.999999); + + int hash_bits = FloorLog2((num_slots << 1) - 1); + if (metadata_charge_policy == kFullChargeCacheMetadata) { + // For very small estimated value sizes, it's possible to overshoot + while (hash_bits > 0 && + uint64_t{sizeof(HandleImpl)} << hash_bits > capacity) { + hash_bits--; + } + } + return hash_bits; +} + +template +void ClockCacheShard
::SetCapacity(size_t capacity) { + capacity_.store(capacity, std::memory_order_relaxed); + // next Insert will take care of any necessary evictions +} + +template +void ClockCacheShard
::SetStrictCapacityLimit( + bool strict_capacity_limit) { + strict_capacity_limit_.store(strict_capacity_limit, + std::memory_order_relaxed); + // next Insert will take care of any necessary evictions +} + +template +Status ClockCacheShard
::Insert(const Slice& key, + const UniqueId64x2& hashed_key, + Cache::ObjectPtr value, + const Cache::CacheItemHelper* helper, + size_t charge, HandleImpl** handle, + Cache::Priority priority) { + if (UNLIKELY(key.size() != kCacheKeySize)) { + return Status::NotSupported("ClockCache only supports key size " + + std::to_string(kCacheKeySize) + "B"); + } + ClockHandleBasicData proto; + proto.hashed_key = hashed_key; + proto.value = value; + proto.helper = helper; + proto.total_charge = charge; + return table_.template Insert
( + proto, handle, priority, capacity_.load(std::memory_order_relaxed), + strict_capacity_limit_.load(std::memory_order_relaxed)); +} + +template +typename Table::HandleImpl* ClockCacheShard
::CreateStandalone( + const Slice& key, const UniqueId64x2& hashed_key, Cache::ObjectPtr obj, + const Cache::CacheItemHelper* helper, size_t charge, bool allow_uncharged) { + if (UNLIKELY(key.size() != kCacheKeySize)) { + return nullptr; + } + ClockHandleBasicData proto; + proto.hashed_key = hashed_key; + proto.value = obj; + proto.helper = helper; + proto.total_charge = charge; + return table_.template CreateStandalone
( + proto, capacity_.load(std::memory_order_relaxed), + strict_capacity_limit_.load(std::memory_order_relaxed), allow_uncharged); +} + +template +typename ClockCacheShard
::HandleImpl* ClockCacheShard
::Lookup( + const Slice& key, const UniqueId64x2& hashed_key) { + if (UNLIKELY(key.size() != kCacheKeySize)) { + return nullptr; + } + return table_.Lookup(hashed_key); +} + +template +bool ClockCacheShard
::Ref(HandleImpl* h) { + if (h == nullptr) { + return false; + } + table_.Ref(*h); + return true; +} + +template +bool ClockCacheShard
::Release(HandleImpl* handle, bool useful, + bool erase_if_last_ref) { + if (handle == nullptr) { + return false; + } + return table_.Release(handle, useful, erase_if_last_ref); +} + +#ifndef NDEBUG +template +void ClockCacheShard
::TEST_RefN(HandleImpl* h, size_t n) { + table_.TEST_RefN(*h, n); +} + +template +void ClockCacheShard
::TEST_ReleaseN(HandleImpl* h, size_t n) { + table_.TEST_ReleaseN(h, n); +} +#endif + +template +bool ClockCacheShard
::Release(HandleImpl* handle, + bool erase_if_last_ref) { + return Release(handle, /*useful=*/true, erase_if_last_ref); +} + +template +void ClockCacheShard
::Erase(const Slice& key, + const UniqueId64x2& hashed_key) { + if (UNLIKELY(key.size() != kCacheKeySize)) { + return; + } + table_.Erase(hashed_key); +} + +template +size_t ClockCacheShard
::GetUsage() const { + return table_.GetUsage(); +} + +template +size_t ClockCacheShard
::GetStandaloneUsage() const { + return table_.GetStandaloneUsage(); +} + +template +size_t ClockCacheShard
::GetCapacity() const { + return capacity_; +} + +template +size_t ClockCacheShard
::GetPinnedUsage() const { + // Computes the pinned usage by scanning the whole hash table. This + // is slow, but avoids keeping an exact counter on the clock usage, + // i.e., the number of not externally referenced elements. + // Why avoid this counter? Because Lookup removes elements from the clock + // list, so it would need to update the pinned usage every time, + // which creates additional synchronization costs. + size_t table_pinned_usage = 0; + const bool charge_metadata = + metadata_charge_policy_ == kFullChargeCacheMetadata; + ConstApplyToEntriesRange( + [&table_pinned_usage, charge_metadata](const HandleImpl& h) { + uint64_t meta = h.meta.load(std::memory_order_relaxed); + uint64_t refcount = GetRefcount(meta); + // Holding one ref for ConstApplyToEntriesRange + assert(refcount > 0); + if (refcount > 1) { + table_pinned_usage += h.GetTotalCharge(); + if (charge_metadata) { + table_pinned_usage += sizeof(HandleImpl); + } + } + }, + table_.HandlePtr(0), table_.HandlePtr(table_.GetTableSize()), true); + + return table_pinned_usage + table_.GetStandaloneUsage(); +} + +template +size_t ClockCacheShard
::GetOccupancyCount() const { + return table_.GetOccupancy(); +} + +template +size_t ClockCacheShard
::GetOccupancyLimit() const { + return table_.GetOccupancyLimit(); +} + +template +size_t ClockCacheShard
::GetTableAddressCount() const { + return table_.GetTableSize(); +} + +// Explicit instantiation +template class ClockCacheShard; + +HyperClockCache::HyperClockCache(const HyperClockCacheOptions& opts) + : ShardedCache(opts) { + assert(opts.estimated_entry_charge > 0 || + opts.metadata_charge_policy != kDontChargeCacheMetadata); + // TODO: should not need to go through two levels of pointer indirection to + // get to table entries + size_t per_shard = GetPerShardCapacity(); + MemoryAllocator* alloc = this->memory_allocator(); + InitShards([&](Shard* cs) { + HyperClockTable::Opts table_opts; + table_opts.estimated_value_size = opts.estimated_entry_charge; + new (cs) Shard(per_shard, opts.strict_capacity_limit, + opts.metadata_charge_policy, alloc, &eviction_callback_, + &hash_seed_, table_opts); + }); +} + +Cache::ObjectPtr HyperClockCache::Value(Handle* handle) { + return reinterpret_cast(handle)->value; +} + +size_t HyperClockCache::GetCharge(Handle* handle) const { + return reinterpret_cast(handle)->GetTotalCharge(); +} + +const Cache::CacheItemHelper* HyperClockCache::GetCacheItemHelper( + Handle* handle) const { + auto h = reinterpret_cast(handle); + return h->helper; +} + +namespace { + +// For each cache shard, estimate what the table load factor would be if +// cache filled to capacity with average entries. This is considered +// indicative of a potential problem if the shard is essentially operating +// "at limit", which we define as high actual usage (>80% of capacity) +// or actual occupancy very close to limit (>95% of limit). +// Also, for each shard compute the recommended estimated_entry_charge, +// and keep the minimum one for use as overall recommendation. +void AddShardEvaluation(const HyperClockCache::Shard& shard, + std::vector& predicted_load_factors, + size_t& min_recommendation) { + size_t usage = shard.GetUsage() - shard.GetStandaloneUsage(); + size_t capacity = shard.GetCapacity(); + double usage_ratio = 1.0 * usage / capacity; + + size_t occupancy = shard.GetOccupancyCount(); + size_t occ_limit = shard.GetOccupancyLimit(); + double occ_ratio = 1.0 * occupancy / occ_limit; + if (usage == 0 || occupancy == 0 || (usage_ratio < 0.8 && occ_ratio < 0.95)) { + // Skip as described above + return; + } + + // If filled to capacity, what would the occupancy ratio be? + double ratio = occ_ratio / usage_ratio; + // Given max load factor, what that load factor be? + double lf = ratio * kStrictLoadFactor; + predicted_load_factors.push_back(lf); + + // Update min_recommendation also + size_t recommendation = usage / occupancy; + min_recommendation = std::min(min_recommendation, recommendation); +} + +} // namespace + +void HyperClockCache::ReportProblems( + const std::shared_ptr& info_log) const { + uint32_t shard_count = GetNumShards(); + std::vector predicted_load_factors; + size_t min_recommendation = SIZE_MAX; + const_cast(this)->ForEachShard( + [&](HyperClockCache::Shard* shard) { + AddShardEvaluation(*shard, predicted_load_factors, min_recommendation); + }); + + if (predicted_load_factors.empty()) { + // None operating "at limit" -> nothing to report + return; + } + std::sort(predicted_load_factors.begin(), predicted_load_factors.end()); + + // First, if the average load factor is within spec, we aren't going to + // complain about a few shards being out of spec. + // NOTE: this is only the average among cache shards operating "at limit," + // which should be representative of what we care about. It it normal, even + // desirable, for a cache to operate "at limit" so this should not create + // selection bias. See AddShardEvaluation(). + // TODO: Consider detecting cases where decreasing the number of shards + // would be good, e.g. serious imbalance among shards. + double average_load_factor = + std::accumulate(predicted_load_factors.begin(), + predicted_load_factors.end(), 0.0) / + shard_count; + + constexpr double kLowSpecLoadFactor = kLoadFactor / 2; + constexpr double kMidSpecLoadFactor = kLoadFactor / 1.414; + if (average_load_factor > kLoadFactor) { + // Out of spec => Consider reporting load factor too high + // Estimate effective overall capacity loss due to enforcing occupancy limit + double lost_portion = 0.0; + int over_count = 0; + for (double lf : predicted_load_factors) { + if (lf > kStrictLoadFactor) { + ++over_count; + lost_portion += (lf - kStrictLoadFactor) / lf / shard_count; + } + } + // >= 20% loss -> error + // >= 10% loss -> consistent warning + // >= 1% loss -> intermittent warning + InfoLogLevel level = InfoLogLevel::INFO_LEVEL; + bool report = true; + if (lost_portion > 0.2) { + level = InfoLogLevel::ERROR_LEVEL; + } else if (lost_portion > 0.1) { + level = InfoLogLevel::WARN_LEVEL; + } else if (lost_portion > 0.01) { + int report_percent = static_cast(lost_portion * 100.0); + if (Random::GetTLSInstance()->PercentTrue(report_percent)) { + level = InfoLogLevel::WARN_LEVEL; + } + } else { + // don't report + report = false; + } + if (report) { + ROCKS_LOG_AT_LEVEL( + info_log, level, + "HyperClockCache@%p unable to use estimated %.1f%% capacity because " + "of " + "full occupancy in %d/%u cache shards (estimated_entry_charge too " + "high). Recommend estimated_entry_charge=%zu", + this, lost_portion * 100.0, over_count, (unsigned)shard_count, + min_recommendation); + } + } else if (average_load_factor < kLowSpecLoadFactor) { + // Out of spec => Consider reporting load factor too low + // But cautiously because low is not as big of a problem. + + // Only report if highest occupancy shard is also below + // spec and only if average is substantially out of spec + if (predicted_load_factors.back() < kLowSpecLoadFactor && + average_load_factor < kLowSpecLoadFactor / 1.414) { + InfoLogLevel level = InfoLogLevel::INFO_LEVEL; + if (average_load_factor < kLowSpecLoadFactor / 2) { + level = InfoLogLevel::WARN_LEVEL; + } + ROCKS_LOG_AT_LEVEL( + info_log, level, + "HyperClockCache@%p table has low occupancy at full capacity. Higher " + "estimated_entry_charge (about %.1fx) would likely improve " + "performance. Recommend estimated_entry_charge=%zu", + this, kMidSpecLoadFactor / average_load_factor, min_recommendation); + } + } +} + +} // namespace clock_cache + +// DEPRECATED (see public API) +std::shared_ptr NewClockCache( + size_t capacity, int num_shard_bits, bool strict_capacity_limit, + CacheMetadataChargePolicy metadata_charge_policy) { + return NewLRUCache(capacity, num_shard_bits, strict_capacity_limit, + /* high_pri_pool_ratio */ 0.5, nullptr, + kDefaultToAdaptiveMutex, metadata_charge_policy, + /* low_pri_pool_ratio */ 0.0); +} + +std::shared_ptr HyperClockCacheOptions::MakeSharedCache() const { + // For sanitized options + HyperClockCacheOptions opts = *this; + if (opts.num_shard_bits >= 20) { + return nullptr; // The cache cannot be sharded into too many fine pieces. + } + if (opts.num_shard_bits < 0) { + // Use larger shard size to reduce risk of large entries clustering + // or skewing individual shards. + constexpr size_t min_shard_size = 32U * 1024U * 1024U; + opts.num_shard_bits = + GetDefaultCacheShardBits(opts.capacity, min_shard_size); + } + std::shared_ptr cache = + std::make_shared(opts); + if (opts.secondary_cache) { + cache = std::make_shared(cache, + opts.secondary_cache); + } + return cache; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/clock_cache.h b/librocksdb-sys/rocksdb/cache/clock_cache.h new file mode 100644 index 0000000..7a1caa0 --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/clock_cache.h @@ -0,0 +1,756 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "cache/cache_key.h" +#include "cache/sharded_cache.h" +#include "port/lang.h" +#include "port/malloc.h" +#include "port/port.h" +#include "rocksdb/cache.h" +#include "rocksdb/secondary_cache.h" +#include "util/autovector.h" +#include "util/math.h" + +namespace ROCKSDB_NAMESPACE { + +namespace clock_cache { + +// Forward declaration of friend class. +class ClockCacheTest; + +// HyperClockCache is an alternative to LRUCache specifically tailored for +// use as BlockBasedTableOptions::block_cache +// +// Benefits +// -------- +// * Fully lock free (no waits or spins) for efficiency under high concurrency +// * Optimized for hot path reads. For concurrency control, most Lookup() and +// essentially all Release() are a single atomic add operation. +// * Eviction on insertion is fully parallel and lock-free. +// * Uses a generalized + aging variant of CLOCK eviction that might outperform +// LRU in some cases. (For background, see +// https://en.wikipedia.org/wiki/Page_replacement_algorithm) +// +// Costs +// ----- +// * Hash table is not resizable (for lock-free efficiency) so capacity is not +// dynamically changeable. Rely on an estimated average value (block) size for +// space+time efficiency. (See estimated_entry_charge option details.) +// * Insert usually does not (but might) overwrite a previous entry associated +// with a cache key. This is OK for RocksDB uses of Cache. +// * Only supports keys of exactly 16 bytes, which is what RocksDB uses for +// block cache (not row cache or table cache). +// * SecondaryCache is not supported. +// * Cache priorities are less aggressively enforced. Unlike LRUCache, enough +// transient LOW or BOTTOM priority items can evict HIGH priority entries that +// are not referenced recently (or often) enough. +// * If pinned entries leave little or nothing eligible for eviction, +// performance can degrade substantially, because of clock eviction eating +// CPU looking for evictable entries and because Release does not +// pro-actively delete unreferenced entries when the cache is over-full. +// Specifically, this makes this implementation more susceptible to the +// following combination: +// * num_shard_bits is high (e.g. 6) +// * capacity small (e.g. some MBs) +// * some large individual entries (e.g. non-partitioned filters) +// where individual entries occupy a large portion of their shard capacity. +// This should be mostly mitigated by the implementation picking a lower +// number of cache shards than LRUCache for a given capacity (when +// num_shard_bits is not overridden; see calls to GetDefaultCacheShardBits()). +// * With strict_capacity_limit=false, respecting the capacity limit is not as +// aggressive as LRUCache. The limit might be transiently exceeded by a very +// small number of entries even when not strictly necessary, and slower to +// recover after pinning forces limit to be substantially exceeded. (Even with +// strict_capacity_limit=true, RocksDB will nevertheless transiently allocate +// memory before discovering it is over the block cache capacity, so this +// should not be a detectable regression in respecting memory limits, except +// on exceptionally small caches.) +// * In some cases, erased or duplicated entries might not be freed +// immediately. They will eventually be freed by eviction from further Inserts. +// * Internal metadata can overflow if the number of simultaneous references +// to a cache handle reaches many millions. +// +// High-level eviction algorithm +// ----------------------------- +// A score (or "countdown") is maintained for each entry, initially determined +// by priority. The score is incremented on each Lookup, up to a max of 3, +// though is easily returned to previous state if useful=false with Release. +// During CLOCK-style eviction iteration, entries with score > 0 are +// decremented if currently unreferenced and entries with score == 0 are +// evicted if currently unreferenced. Note that scoring might not be perfect +// because entries can be referenced transiently within the cache even when +// there are no outside references to the entry. +// +// Cache sharding like LRUCache is used to reduce contention on usage+eviction +// state, though here the performance improvement from more shards is small, +// and (as noted above) potentially detrimental if shard capacity is too close +// to largest entry size. Here cache sharding mostly only affects cache update +// (Insert / Erase) performance, not read performance. +// +// Read efficiency (hot path) +// -------------------------- +// Mostly to minimize the cost of accessing metadata blocks with +// cache_index_and_filter_blocks=true, we focus on optimizing Lookup and +// Release. In terms of concurrency, at a minimum, these operations have +// to do reference counting (and Lookup has to compare full keys in a safe +// way). Can we fold in all the other metadata tracking *for free* with +// Lookup and Release doing a simple atomic fetch_add/fetch_sub? (Assume +// for the moment that Lookup succeeds on the first probe.) +// +// We have a clever way of encoding an entry's reference count and countdown +// clock so that Lookup and Release are each usually a single atomic addition. +// In a single metadata word we have both an "acquire" count, incremented by +// Lookup, and a "release" count, incremented by Release. If useful=false, +// Release can instead decrement the acquire count. Thus the current ref +// count is (acquires - releases), and the countdown clock is min(3, acquires). +// Note that only unreferenced entries (acquires == releases) are eligible +// for CLOCK manipulation and eviction. We tolerate use of more expensive +// compare_exchange operations for cache writes (insertions and erasures). +// +// In a cache receiving many reads and little or no writes, it is possible +// for the acquire and release counters to overflow. Assuming the *current* +// refcount never reaches to many millions, we only have to correct for +// overflow in both counters in Release, not in Lookup. The overflow check +// should be only 1-2 CPU cycles per Release because it is a predictable +// branch on a simple condition on data already in registers. +// +// Slot states +// ----------- +// We encode a state indicator into the same metadata word with the +// acquire and release counters. This allows bigger state transitions to +// be atomic. States: +// +// * Empty - slot is not in use and unowned. All other metadata and data is +// in an undefined state. +// * Construction - slot is exclusively owned by one thread, the thread +// successfully entering this state, for populating or freeing data. +// * Shareable (group) - slot holds an entry with counted references for +// pinning and reading, including +// * Visible - slot holds an entry that can be returned by Lookup +// * Invisible - slot holds an entry that is not visible to Lookup +// (erased by user) but can be read by existing references, and ref count +// changed by Ref and Release. +// +// A special case is "standalone" entries, which are heap-allocated handles +// not in the table. They are always Invisible and freed on zero refs. +// +// State transitions: +// Empty -> Construction (in Insert): The encoding of state enables Insert to +// perform an optimistic atomic bitwise-or to take ownership if a slot is +// empty, or otherwise make no state change. +// +// Construction -> Visible (in Insert): This can be a simple assignment to the +// metadata word because the current thread has exclusive ownership and other +// metadata is meaningless. +// +// Visible -> Invisible (in Erase): This can be a bitwise-and while holding +// a shared reference, which is safe because the change is idempotent (in case +// of parallel Erase). By the way, we never go Invisible->Visible. +// +// Shareable -> Construction (in Evict part of Insert, in Erase, and in +// Release if Invisible): This is for starting to freeing/deleting an +// unreferenced entry. We have to use compare_exchange to ensure we only make +// this transition when there are zero refs. +// +// Construction -> Empty (in same places): This is for completing free/delete +// of an entry. A "release" atomic store suffices, as we have exclusive +// ownership of the slot but have to ensure none of the data member reads are +// re-ordered after committing the state transition. +// +// Insert +// ------ +// If Insert were to guarantee replacing an existing entry for a key, there +// would be complications for concurrency and efficiency. First, consider how +// many probes to get to an entry. To ensure Lookup never waits and +// availability of a key is uninterrupted, we would need to use a different +// slot for a new entry for the same key. This means it is most likely in a +// later probing position than the old version, which should soon be removed. +// (Also, an entry is too big to replace atomically, even if no current refs.) +// +// However, overwrite capability is not really needed by RocksDB. Also, we +// know from our "redundant" stats that overwrites are very rare for the block +// cache, so we should not spend much to make them effective. +// +// So instead we Insert as soon as we find an empty slot in the probing +// sequence without seeing an existing (visible) entry for the same key. This +// way we only insert if we can improve the probing performance, and we don't +// need to probe beyond our insert position, assuming we are willing to let +// the previous entry for the same key die of old age (eventual eviction from +// not being used). We can reach a similar state with concurrent insertions, +// where one will pass over the other while it is "under construction." +// This temporary duplication is acceptable for RocksDB block cache because +// we know redundant insertion is rare. +// +// Another problem to solve is what to return to the caller when we find an +// existing entry whose probing position we cannot improve on, or when the +// table occupancy limit has been reached. If strict_capacity_limit=false, +// we must never fail Insert, and if a Handle* is provided, we have to return +// a usable Cache handle on success. The solution to this (typically rare) +// problem is "standalone" handles, which are usable by the caller but not +// actually available for Lookup in the Cache. Standalone handles are allocated +// independently on the heap and specially marked so that they are freed on +// the heap when their last reference is released. +// +// Usage on capacity +// ----------------- +// Insert takes different approaches to usage tracking depending on +// strict_capacity_limit setting. If true, we enforce a kind of strong +// consistency where compare-exchange is used to ensure the usage number never +// exceeds its limit, and provide threads with an authoritative signal on how +// much "usage" they have taken ownership of. With strict_capacity_limit=false, +// we use a kind of "eventual consistency" where all threads Inserting to the +// same cache shard might race on reserving the same space, but the +// over-commitment will be worked out in later insertions. It is kind of a +// dance because we don't want threads racing each other too much on paying +// down the over-commitment (with eviction) either. +// +// Eviction +// -------- +// A key part of Insert is evicting some entries currently unreferenced to +// make room for new entries. The high-level eviction algorithm is described +// above, but the details are also interesting. A key part is parallelizing +// eviction with a single CLOCK pointer. This works by each thread working on +// eviction pre-emptively incrementing the CLOCK pointer, and then CLOCK- +// updating or evicting the incremented-over slot(s). To reduce contention at +// the cost of possibly evicting too much, each thread increments the clock +// pointer by 4, so commits to updating at least 4 slots per batch. As +// described above, a CLOCK update will decrement the "countdown" of +// unreferenced entries, or evict unreferenced entries with zero countdown. +// Referenced entries are not updated, because we (presumably) don't want +// long-referenced entries to age while referenced. Note however that we +// cannot distinguish transiently referenced entries from cache user +// references, so some CLOCK updates might be somewhat arbitrarily skipped. +// This is OK as long as it is rare enough that eviction order is still +// pretty good. +// +// There is no synchronization on the completion of the CLOCK updates, so it +// is theoretically possible for another thread to cycle back around and have +// two threads racing on CLOCK updates to the same slot. Thus, we cannot rely +// on any implied exclusivity to make the updates or eviction more efficient. +// These updates use an opportunistic compare-exchange (no loop), where a +// racing thread might cause the update to be skipped without retry, but in +// such case the update is likely not needed because the most likely update +// to an entry is that it has become referenced. (TODO: test efficiency of +// avoiding compare-exchange loop) +// +// Release +// ------- +// In the common case, Release is a simple atomic increment of the release +// counter. There is a simple overflow check that only does another atomic +// update in extremely rare cases, so costs almost nothing. +// +// If the Release specifies "not useful", we can instead decrement the +// acquire counter, which returns to the same CLOCK state as before Lookup +// or Ref. +// +// Adding a check for over-full cache on every release to zero-refs would +// likely be somewhat expensive, increasing read contention on cache shard +// metadata. Instead we are less aggressive about deleting entries right +// away in those cases. +// +// However Release tries to immediately delete entries reaching zero refs +// if (a) erase_if_last_ref is set by the caller, or (b) the entry is already +// marked invisible. Both of these are checks on values already in CPU +// registers so do not increase cross-CPU contention when not applicable. +// When applicable, they use a compare-exchange loop to take exclusive +// ownership of the slot for freeing the entry. These are rare cases +// that should not usually affect performance. +// +// Erase +// ----- +// Searches for an entry like Lookup but moves it to Invisible state if found. +// This state transition is with bit operations so is idempotent and safely +// done while only holding a shared "read" reference. Like Release, it makes +// a best effort to immediately release an Invisible entry that reaches zero +// refs, but there are some corner cases where it will only be freed by the +// clock eviction process. + +// ----------------------------------------------------------------------- // + +// The load factor p is a real number in (0, 1) such that at all +// times at most a fraction p of all slots, without counting tombstones, +// are occupied by elements. This means that the probability that a random +// probe hits an occupied slot is at most p, and thus at most 1/p probes +// are required on average. For example, p = 70% implies that between 1 and 2 +// probes are needed on average (bear in mind that this reasoning doesn't +// consider the effects of clustering over time, which should be negligible +// with double hashing). +// Because the size of the hash table is always rounded up to the next +// power of 2, p is really an upper bound on the actual load factor---the +// actual load factor is anywhere between p/2 and p. This is a bit wasteful, +// but bear in mind that slots only hold metadata, not actual values. +// Since space cost is dominated by the values (the LSM blocks), +// overprovisioning the table with metadata only increases the total cache space +// usage by a tiny fraction. +constexpr double kLoadFactor = 0.7; + +// The user can exceed kLoadFactor if the sizes of the inserted values don't +// match estimated_value_size, or in some rare cases with +// strict_capacity_limit == false. To avoid degenerate performance, we set a +// strict upper bound on the load factor. +constexpr double kStrictLoadFactor = 0.84; + +struct ClockHandleBasicData { + Cache::ObjectPtr value = nullptr; + const Cache::CacheItemHelper* helper = nullptr; + // A lossless, reversible hash of the fixed-size (16 byte) cache key. This + // eliminates the need to store a hash separately. + UniqueId64x2 hashed_key = kNullUniqueId64x2; + size_t total_charge = 0; + + inline size_t GetTotalCharge() const { return total_charge; } + + // Calls deleter (if non-null) on cache key and value + void FreeData(MemoryAllocator* allocator) const; + + // Required by concept HandleImpl + const UniqueId64x2& GetHash() const { return hashed_key; } +}; + +struct ClockHandle : public ClockHandleBasicData { + // Constants for handling the atomic `meta` word, which tracks most of the + // state of the handle. The meta word looks like this: + // low bits high bits + // ----------------------------------------------------------------------- + // | acquire counter | release counter | state marker | + // ----------------------------------------------------------------------- + + // For reading or updating counters in meta word. + static constexpr uint8_t kCounterNumBits = 30; + static constexpr uint64_t kCounterMask = (uint64_t{1} << kCounterNumBits) - 1; + + static constexpr uint8_t kAcquireCounterShift = 0; + static constexpr uint64_t kAcquireIncrement = uint64_t{1} + << kAcquireCounterShift; + static constexpr uint8_t kReleaseCounterShift = kCounterNumBits; + static constexpr uint64_t kReleaseIncrement = uint64_t{1} + << kReleaseCounterShift; + + // For reading or updating the state marker in meta word + static constexpr uint8_t kStateShift = 2U * kCounterNumBits; + + // Bits contribution to state marker. + // Occupied means any state other than empty + static constexpr uint8_t kStateOccupiedBit = 0b100; + // Shareable means the entry is reference counted (visible or invisible) + // (only set if also occupied) + static constexpr uint8_t kStateShareableBit = 0b010; + // Visible is only set if also shareable + static constexpr uint8_t kStateVisibleBit = 0b001; + + // Complete state markers (not shifted into full word) + static constexpr uint8_t kStateEmpty = 0b000; + static constexpr uint8_t kStateConstruction = kStateOccupiedBit; + static constexpr uint8_t kStateInvisible = + kStateOccupiedBit | kStateShareableBit; + static constexpr uint8_t kStateVisible = + kStateOccupiedBit | kStateShareableBit | kStateVisibleBit; + + // Constants for initializing the countdown clock. (Countdown clock is only + // in effect with zero refs, acquire counter == release counter, and in that + // case the countdown clock == both of those counters.) + static constexpr uint8_t kHighCountdown = 3; + static constexpr uint8_t kLowCountdown = 2; + static constexpr uint8_t kBottomCountdown = 1; + // During clock update, treat any countdown clock value greater than this + // value the same as this value. + static constexpr uint8_t kMaxCountdown = kHighCountdown; + // TODO: make these coundown values tuning parameters for eviction? + + // See above. Mutable for read reference counting. + mutable std::atomic meta{}; + + // Whether this is a "deteched" handle that is independently allocated + // with `new` (so must be deleted with `delete`). + // TODO: ideally this would be packed into some other data field, such + // as upper bits of total_charge, but that incurs a measurable performance + // regression. + bool standalone = false; + + inline bool IsStandalone() const { return standalone; } + + inline void SetStandalone() { standalone = true; } +}; // struct ClockHandle + +class BaseClockTable { + public: + BaseClockTable(CacheMetadataChargePolicy metadata_charge_policy, + MemoryAllocator* allocator, + const Cache::EvictionCallback* eviction_callback, + const uint32_t* hash_seed) + : metadata_charge_policy_(metadata_charge_policy), + allocator_(allocator), + eviction_callback_(*eviction_callback), + hash_seed_(*hash_seed) {} + + template + typename Table::HandleImpl* CreateStandalone(ClockHandleBasicData& proto, + size_t capacity, + bool strict_capacity_limit, + bool allow_uncharged); + + template + Status Insert(const ClockHandleBasicData& proto, + typename Table::HandleImpl** handle, Cache::Priority priority, + size_t capacity, bool strict_capacity_limit); + + void Ref(ClockHandle& handle); + + size_t GetOccupancy() const { + return occupancy_.load(std::memory_order_relaxed); + } + + size_t GetUsage() const { return usage_.load(std::memory_order_relaxed); } + + size_t GetStandaloneUsage() const { + return standalone_usage_.load(std::memory_order_relaxed); + } + + uint32_t GetHashSeed() const { return hash_seed_; } + + struct EvictionData { + size_t freed_charge = 0; + size_t freed_count = 0; + }; + + void TrackAndReleaseEvictedEntry(ClockHandle* h, EvictionData* data); + +#ifndef NDEBUG + // Acquire N references + void TEST_RefN(ClockHandle& handle, size_t n); + // Helper for TEST_ReleaseN + void TEST_ReleaseNMinus1(ClockHandle* handle, size_t n); +#endif + + private: // fns + // Creates a "standalone" handle for returning from an Insert operation that + // cannot be completed by actually inserting into the table. + // Updates `standalone_usage_` but not `usage_` nor `occupancy_`. + template + HandleImpl* StandaloneInsert(const ClockHandleBasicData& proto); + + // Helper for updating `usage_` for new entry with given `total_charge` + // and evicting if needed under strict_capacity_limit=true rules. This + // means the operation might fail with Status::MemoryLimit. If + // `need_evict_for_occupancy`, then eviction of at least one entry is + // required, and the operation should fail if not possible. + // NOTE: Otherwise, occupancy_ is not managed in this function + template + Status ChargeUsageMaybeEvictStrict(size_t total_charge, size_t capacity, + bool need_evict_for_occupancy, + typename Table::InsertState& state); + + // Helper for updating `usage_` for new entry with given `total_charge` + // and evicting if needed under strict_capacity_limit=false rules. This + // means that updating `usage_` always succeeds even if forced to exceed + // capacity. If `need_evict_for_occupancy`, then eviction of at least one + // entry is required, and the operation should return false if such eviction + // is not possible. `usage_` is not updated in that case. Otherwise, returns + // true, indicating success. + // NOTE: occupancy_ is not managed in this function + template + bool ChargeUsageMaybeEvictNonStrict(size_t total_charge, size_t capacity, + bool need_evict_for_occupancy, + typename Table::InsertState& state); + + protected: // data + // We partition the following members into different cache lines + // to avoid false sharing among Lookup, Release, Erase and Insert + // operations in ClockCacheShard. + + // Clock algorithm sweep pointer. + std::atomic clock_pointer_{}; + + ALIGN_AS(CACHE_LINE_SIZE) + // Number of elements in the table. + std::atomic occupancy_{}; + + // Memory usage by entries tracked by the cache (including standalone) + std::atomic usage_{}; + + // Part of usage by standalone entries (not in table) + std::atomic standalone_usage_{}; + + ALIGN_AS(CACHE_LINE_SIZE) + const CacheMetadataChargePolicy metadata_charge_policy_; + + // From Cache, for deleter + MemoryAllocator* const allocator_; + + // A reference to Cache::eviction_callback_ + const Cache::EvictionCallback& eviction_callback_; + + // A reference to ShardedCacheBase::hash_seed_ + const uint32_t& hash_seed_; +}; + +class HyperClockTable : public BaseClockTable { + public: + // Target size to be exactly a common cache line size (see static_assert in + // clock_cache.cc) + struct ALIGN_AS(64U) HandleImpl : public ClockHandle { + // The number of elements that hash to this slot or a lower one, but wind + // up in this slot or a higher one. + std::atomic displacements{}; + + }; // struct HandleImpl + + struct Opts { + size_t estimated_value_size; + }; + + HyperClockTable(size_t capacity, bool strict_capacity_limit, + CacheMetadataChargePolicy metadata_charge_policy, + MemoryAllocator* allocator, + const Cache::EvictionCallback* eviction_callback, + const uint32_t* hash_seed, const Opts& opts); + ~HyperClockTable(); + + // For BaseClockTable::Insert + struct InsertState {}; + + void StartInsert(InsertState& state); + + // Returns true iff there is room for the proposed number of entries. + bool GrowIfNeeded(size_t new_occupancy, InsertState& state); + + HandleImpl* DoInsert(const ClockHandleBasicData& proto, + uint64_t initial_countdown, bool take_ref, + InsertState& state); + + // Runs the clock eviction algorithm trying to reclaim at least + // requested_charge. Returns how much is evicted, which could be less + // if it appears impossible to evict the requested amount without blocking. + void Evict(size_t requested_charge, InsertState& state, EvictionData* data); + + HandleImpl* Lookup(const UniqueId64x2& hashed_key); + + bool Release(HandleImpl* handle, bool useful, bool erase_if_last_ref); + + void Erase(const UniqueId64x2& hashed_key); + + void EraseUnRefEntries(); + + size_t GetTableSize() const { return size_t{1} << length_bits_; } + + size_t GetOccupancyLimit() const { return occupancy_limit_; } + + const HandleImpl* HandlePtr(size_t idx) const { return &array_[idx]; } + +#ifndef NDEBUG + size_t& TEST_MutableOccupancyLimit() const { + return const_cast(occupancy_limit_); + } + + // Release N references + void TEST_ReleaseN(HandleImpl* handle, size_t n); +#endif + + private: // functions + // Returns x mod 2^{length_bits_}. + inline size_t ModTableSize(uint64_t x) { + return BitwiseAnd(x, length_bits_mask_); + } + + // Returns the first slot in the probe sequence with a handle e such that + // match_fn(e) is true. At every step, the function first tests whether + // match_fn(e) holds. If this is false, it evaluates abort_fn(e) to decide + // whether the search should be aborted, and if so, FindSlot immediately + // returns nullptr. For every handle e that is not a match and not aborted, + // FindSlot runs update_fn(e, is_last) where is_last is set to true iff that + // slot will be the last probed because the next would cycle back to the first + // slot probed. This function uses templates instead of std::function to + // minimize the risk of heap-allocated closures being created. + template + inline HandleImpl* FindSlot(const UniqueId64x2& hashed_key, + const MatchFn& match_fn, const AbortFn& abort_fn, + const UpdateFn& update_fn); + + // Re-decrement all displacements in probe path starting from beginning + // until (not including) the given handle + inline void Rollback(const UniqueId64x2& hashed_key, const HandleImpl* h); + + // Subtracts `total_charge` from `usage_` and 1 from `occupancy_`. + // Ideally this comes after releasing the entry itself so that we + // actually have the available occupancy/usage that is claimed. + // However, that means total_charge has to be saved from the handle + // before releasing it so that it can be provided to this function. + inline void ReclaimEntryUsage(size_t total_charge); + + MemoryAllocator* GetAllocator() const { return allocator_; } + + // Returns the number of bits used to hash an element in the hash + // table. + static int CalcHashBits(size_t capacity, size_t estimated_value_size, + CacheMetadataChargePolicy metadata_charge_policy); + + private: // data + // Number of hash bits used for table index. + // The size of the table is 1 << length_bits_. + const int length_bits_; + + // For faster computation of ModTableSize. + const size_t length_bits_mask_; + + // Maximum number of elements the user can store in the table. + const size_t occupancy_limit_; + + // Array of slots comprising the hash table. + const std::unique_ptr array_; +}; // class HyperClockTable + +// A single shard of sharded cache. +template +class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShardBase { + public: + ClockCacheShard(size_t capacity, bool strict_capacity_limit, + CacheMetadataChargePolicy metadata_charge_policy, + MemoryAllocator* allocator, + const Cache::EvictionCallback* eviction_callback, + const uint32_t* hash_seed, const typename Table::Opts& opts); + + // For CacheShard concept + using HandleImpl = typename Table::HandleImpl; + // Hash is lossless hash of 128-bit key + using HashVal = UniqueId64x2; + using HashCref = const HashVal&; + static inline uint32_t HashPieceForSharding(HashCref hash) { + return Upper32of64(hash[0]); + } + static inline HashVal ComputeHash(const Slice& key, uint32_t seed) { + assert(key.size() == kCacheKeySize); + HashVal in; + HashVal out; + // NOTE: endian dependence + // TODO: use GetUnaligned? + std::memcpy(&in, key.data(), kCacheKeySize); + BijectiveHash2x64(in[1], in[0] ^ seed, &out[1], &out[0]); + return out; + } + + // For reconstructing key from hashed_key. Requires the caller to provide + // backing storage for the Slice in `unhashed` + static inline Slice ReverseHash(const UniqueId64x2& hashed, + UniqueId64x2* unhashed, uint32_t seed) { + BijectiveUnhash2x64(hashed[1], hashed[0], &(*unhashed)[1], &(*unhashed)[0]); + (*unhashed)[0] ^= seed; + // NOTE: endian dependence + return Slice(reinterpret_cast(unhashed), kCacheKeySize); + } + + // Although capacity is dynamically changeable, the number of table slots is + // not, so growing capacity substantially could lead to hitting occupancy + // limit. + void SetCapacity(size_t capacity); + + void SetStrictCapacityLimit(bool strict_capacity_limit); + + Status Insert(const Slice& key, const UniqueId64x2& hashed_key, + Cache::ObjectPtr value, const Cache::CacheItemHelper* helper, + size_t charge, HandleImpl** handle, Cache::Priority priority); + + HandleImpl* CreateStandalone(const Slice& key, const UniqueId64x2& hashed_key, + Cache::ObjectPtr obj, + const Cache::CacheItemHelper* helper, + size_t charge, bool allow_uncharged); + + HandleImpl* Lookup(const Slice& key, const UniqueId64x2& hashed_key); + + bool Release(HandleImpl* handle, bool useful, bool erase_if_last_ref); + + bool Release(HandleImpl* handle, bool erase_if_last_ref = false); + + bool Ref(HandleImpl* handle); + + void Erase(const Slice& key, const UniqueId64x2& hashed_key); + + size_t GetCapacity() const; + + size_t GetUsage() const; + + size_t GetStandaloneUsage() const; + + size_t GetPinnedUsage() const; + + size_t GetOccupancyCount() const; + + size_t GetOccupancyLimit() const; + + size_t GetTableAddressCount() const; + + void ApplyToSomeEntries( + const std::function& callback, + size_t average_entries_per_lock, size_t* state); + + void EraseUnRefEntries(); + + std::string GetPrintableOptions() const { return std::string{}; } + + HandleImpl* Lookup(const Slice& key, const UniqueId64x2& hashed_key, + const Cache::CacheItemHelper* /*helper*/, + Cache::CreateContext* /*create_context*/, + Cache::Priority /*priority*/, Statistics* /*stats*/) { + return Lookup(key, hashed_key); + } + +#ifndef NDEBUG + size_t& TEST_MutableOccupancyLimit() const { + return table_.TEST_MutableOccupancyLimit(); + } + // Acquire/release N references + void TEST_RefN(HandleImpl* handle, size_t n); + void TEST_ReleaseN(HandleImpl* handle, size_t n); +#endif + + private: // data + Table table_; + + // Maximum total charge of all elements stored in the table. + std::atomic capacity_; + + // Whether to reject insertion if cache reaches its full capacity. + std::atomic strict_capacity_limit_; +}; // class ClockCacheShard + +class HyperClockCache +#ifdef NDEBUG + final +#endif + : public ShardedCache> { + public: + using Shard = ClockCacheShard; + + explicit HyperClockCache(const HyperClockCacheOptions& opts); + + const char* Name() const override { return "HyperClockCache"; } + + Cache::ObjectPtr Value(Handle* handle) override; + + size_t GetCharge(Handle* handle) const override; + + const CacheItemHelper* GetCacheItemHelper(Handle* handle) const override; + + void ReportProblems( + const std::shared_ptr& /*info_log*/) const override; +}; // class HyperClockCache + +} // namespace clock_cache + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/compressed_secondary_cache.cc b/librocksdb-sys/rocksdb/cache/compressed_secondary_cache.cc new file mode 100644 index 0000000..f5b7af3 --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/compressed_secondary_cache.cc @@ -0,0 +1,318 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "cache/compressed_secondary_cache.h" + +#include +#include +#include + +#include "memory/memory_allocator_impl.h" +#include "monitoring/perf_context_imp.h" +#include "util/compression.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +CompressedSecondaryCache::CompressedSecondaryCache( + const CompressedSecondaryCacheOptions& opts) + : cache_(opts.LRUCacheOptions::MakeSharedCache()), + cache_options_(opts), + cache_res_mgr_(std::make_shared( + std::make_shared>( + cache_))) {} + +CompressedSecondaryCache::~CompressedSecondaryCache() { + assert(cache_res_mgr_->GetTotalReservedCacheSize() == 0); +} + +std::unique_ptr CompressedSecondaryCache::Lookup( + const Slice& key, const Cache::CacheItemHelper* helper, + Cache::CreateContext* create_context, bool /*wait*/, bool advise_erase, + bool& kept_in_sec_cache) { + assert(helper); + std::unique_ptr handle; + kept_in_sec_cache = false; + Cache::Handle* lru_handle = cache_->Lookup(key); + if (lru_handle == nullptr) { + return nullptr; + } + + void* handle_value = cache_->Value(lru_handle); + if (handle_value == nullptr) { + cache_->Release(lru_handle, /*erase_if_last_ref=*/false); + return nullptr; + } + + CacheAllocationPtr* ptr{nullptr}; + CacheAllocationPtr merged_value; + size_t handle_value_charge{0}; + if (cache_options_.enable_custom_split_merge) { + CacheValueChunk* value_chunk_ptr = + reinterpret_cast(handle_value); + merged_value = MergeChunksIntoValue(value_chunk_ptr, handle_value_charge); + ptr = &merged_value; + } else { + ptr = reinterpret_cast(handle_value); + handle_value_charge = cache_->GetCharge(lru_handle); + } + MemoryAllocator* allocator = cache_options_.memory_allocator.get(); + + Status s; + Cache::ObjectPtr value{nullptr}; + size_t charge{0}; + if (cache_options_.compression_type == kNoCompression || + cache_options_.do_not_compress_roles.Contains(helper->role)) { + s = helper->create_cb(Slice(ptr->get(), handle_value_charge), + create_context, allocator, &value, &charge); + } else { + UncompressionContext uncompression_context(cache_options_.compression_type); + UncompressionInfo uncompression_info(uncompression_context, + UncompressionDict::GetEmptyDict(), + cache_options_.compression_type); + + size_t uncompressed_size{0}; + CacheAllocationPtr uncompressed = UncompressData( + uncompression_info, (char*)ptr->get(), handle_value_charge, + &uncompressed_size, cache_options_.compress_format_version, allocator); + + if (!uncompressed) { + cache_->Release(lru_handle, /*erase_if_last_ref=*/true); + return nullptr; + } + s = helper->create_cb(Slice(uncompressed.get(), uncompressed_size), + create_context, allocator, &value, &charge); + } + + if (!s.ok()) { + cache_->Release(lru_handle, /*erase_if_last_ref=*/true); + return nullptr; + } + + if (advise_erase) { + cache_->Release(lru_handle, /*erase_if_last_ref=*/true); + // Insert a dummy handle. + cache_ + ->Insert(key, /*obj=*/nullptr, + GetHelper(cache_options_.enable_custom_split_merge), + /*charge=*/0) + .PermitUncheckedError(); + } else { + kept_in_sec_cache = true; + cache_->Release(lru_handle, /*erase_if_last_ref=*/false); + } + handle.reset(new CompressedSecondaryCacheResultHandle(value, charge)); + return handle; +} + +Status CompressedSecondaryCache::Insert(const Slice& key, + Cache::ObjectPtr value, + const Cache::CacheItemHelper* helper) { + if (value == nullptr) { + return Status::InvalidArgument(); + } + + Cache::Handle* lru_handle = cache_->Lookup(key); + auto internal_helper = GetHelper(cache_options_.enable_custom_split_merge); + if (lru_handle == nullptr) { + PERF_COUNTER_ADD(compressed_sec_cache_insert_dummy_count, 1); + // Insert a dummy handle if the handle is evicted for the first time. + return cache_->Insert(key, /*obj=*/nullptr, internal_helper, + /*charge=*/0); + } else { + cache_->Release(lru_handle, /*erase_if_last_ref=*/false); + } + + size_t size = (*helper->size_cb)(value); + CacheAllocationPtr ptr = + AllocateBlock(size, cache_options_.memory_allocator.get()); + + Status s = (*helper->saveto_cb)(value, 0, size, ptr.get()); + if (!s.ok()) { + return s; + } + Slice val(ptr.get(), size); + + std::string compressed_val; + if (cache_options_.compression_type != kNoCompression && + !cache_options_.do_not_compress_roles.Contains(helper->role)) { + PERF_COUNTER_ADD(compressed_sec_cache_uncompressed_bytes, size); + CompressionOptions compression_opts; + CompressionContext compression_context(cache_options_.compression_type); + uint64_t sample_for_compression{0}; + CompressionInfo compression_info( + compression_opts, compression_context, CompressionDict::GetEmptyDict(), + cache_options_.compression_type, sample_for_compression); + + bool success = + CompressData(val, compression_info, + cache_options_.compress_format_version, &compressed_val); + + if (!success) { + return Status::Corruption("Error compressing value."); + } + + val = Slice(compressed_val); + size = compressed_val.size(); + PERF_COUNTER_ADD(compressed_sec_cache_compressed_bytes, size); + + if (!cache_options_.enable_custom_split_merge) { + ptr = AllocateBlock(size, cache_options_.memory_allocator.get()); + memcpy(ptr.get(), compressed_val.data(), size); + } + } + + PERF_COUNTER_ADD(compressed_sec_cache_insert_real_count, 1); + if (cache_options_.enable_custom_split_merge) { + size_t charge{0}; + CacheValueChunk* value_chunks_head = + SplitValueIntoChunks(val, cache_options_.compression_type, charge); + return cache_->Insert(key, value_chunks_head, internal_helper, charge); + } else { + CacheAllocationPtr* buf = new CacheAllocationPtr(std::move(ptr)); + return cache_->Insert(key, buf, internal_helper, size); + } +} + +void CompressedSecondaryCache::Erase(const Slice& key) { cache_->Erase(key); } + +Status CompressedSecondaryCache::SetCapacity(size_t capacity) { + MutexLock l(&capacity_mutex_); + cache_options_.capacity = capacity; + cache_->SetCapacity(capacity); + return Status::OK(); +} + +Status CompressedSecondaryCache::GetCapacity(size_t& capacity) { + MutexLock l(&capacity_mutex_); + capacity = cache_options_.capacity; + return Status::OK(); +} + +std::string CompressedSecondaryCache::GetPrintableOptions() const { + std::string ret; + ret.reserve(20000); + const int kBufferSize{200}; + char buffer[kBufferSize]; + ret.append(cache_->GetPrintableOptions()); + snprintf(buffer, kBufferSize, " compression_type : %s\n", + CompressionTypeToString(cache_options_.compression_type).c_str()); + ret.append(buffer); + snprintf(buffer, kBufferSize, " compress_format_version : %d\n", + cache_options_.compress_format_version); + ret.append(buffer); + return ret; +} + +CompressedSecondaryCache::CacheValueChunk* +CompressedSecondaryCache::SplitValueIntoChunks(const Slice& value, + CompressionType compression_type, + size_t& charge) { + assert(!value.empty()); + const char* src_ptr = value.data(); + size_t src_size{value.size()}; + + CacheValueChunk dummy_head = CacheValueChunk(); + CacheValueChunk* current_chunk = &dummy_head; + // Do not split when value size is large or there is no compression. + size_t predicted_chunk_size{0}; + size_t actual_chunk_size{0}; + size_t tmp_size{0}; + while (src_size > 0) { + predicted_chunk_size = sizeof(CacheValueChunk) - 1 + src_size; + auto upper = + std::upper_bound(malloc_bin_sizes_.begin(), malloc_bin_sizes_.end(), + predicted_chunk_size); + // Do not split when value size is too small, too large, close to a bin + // size, or there is no compression. + if (upper == malloc_bin_sizes_.begin() || + upper == malloc_bin_sizes_.end() || + *upper - predicted_chunk_size < malloc_bin_sizes_.front() || + compression_type == kNoCompression) { + tmp_size = predicted_chunk_size; + } else { + tmp_size = *(--upper); + } + + CacheValueChunk* new_chunk = + reinterpret_cast(new char[tmp_size]); + current_chunk->next = new_chunk; + current_chunk = current_chunk->next; + actual_chunk_size = tmp_size - sizeof(CacheValueChunk) + 1; + memcpy(current_chunk->data, src_ptr, actual_chunk_size); + current_chunk->size = actual_chunk_size; + src_ptr += actual_chunk_size; + src_size -= actual_chunk_size; + charge += tmp_size; + } + current_chunk->next = nullptr; + + return dummy_head.next; +} + +CacheAllocationPtr CompressedSecondaryCache::MergeChunksIntoValue( + const void* chunks_head, size_t& charge) { + const CacheValueChunk* head = + reinterpret_cast(chunks_head); + const CacheValueChunk* current_chunk = head; + charge = 0; + while (current_chunk != nullptr) { + charge += current_chunk->size; + current_chunk = current_chunk->next; + } + + CacheAllocationPtr ptr = + AllocateBlock(charge, cache_options_.memory_allocator.get()); + current_chunk = head; + size_t pos{0}; + while (current_chunk != nullptr) { + memcpy(ptr.get() + pos, current_chunk->data, current_chunk->size); + pos += current_chunk->size; + current_chunk = current_chunk->next; + } + + return ptr; +} + +const Cache::CacheItemHelper* CompressedSecondaryCache::GetHelper( + bool enable_custom_split_merge) const { + if (enable_custom_split_merge) { + static const Cache::CacheItemHelper kHelper{ + CacheEntryRole::kMisc, + [](Cache::ObjectPtr obj, MemoryAllocator* /*alloc*/) { + CacheValueChunk* chunks_head = static_cast(obj); + while (chunks_head != nullptr) { + CacheValueChunk* tmp_chunk = chunks_head; + chunks_head = chunks_head->next; + tmp_chunk->Free(); + obj = nullptr; + }; + }}; + return &kHelper; + } else { + static const Cache::CacheItemHelper kHelper{ + CacheEntryRole::kMisc, + [](Cache::ObjectPtr obj, MemoryAllocator* /*alloc*/) { + delete static_cast(obj); + obj = nullptr; + }}; + return &kHelper; + } +} + +std::shared_ptr +CompressedSecondaryCacheOptions::MakeSharedSecondaryCache() const { + return std::make_shared(*this); +} + +Status CompressedSecondaryCache::Deflate(size_t decrease) { + return cache_res_mgr_->UpdateCacheReservation(decrease, /*increase=*/true); +} + +Status CompressedSecondaryCache::Inflate(size_t increase) { + return cache_res_mgr_->UpdateCacheReservation(increase, /*increase=*/false); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/compressed_secondary_cache.h b/librocksdb-sys/rocksdb/cache/compressed_secondary_cache.h new file mode 100644 index 0000000..7bee059 --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/compressed_secondary_cache.h @@ -0,0 +1,140 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include + +#include "cache/cache_reservation_manager.h" +#include "cache/lru_cache.h" +#include "memory/memory_allocator_impl.h" +#include "rocksdb/secondary_cache.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "util/compression.h" +#include "util/mutexlock.h" + +namespace ROCKSDB_NAMESPACE { + +class CompressedSecondaryCacheResultHandle : public SecondaryCacheResultHandle { + public: + CompressedSecondaryCacheResultHandle(Cache::ObjectPtr value, size_t size) + : value_(value), size_(size) {} + ~CompressedSecondaryCacheResultHandle() override = default; + + CompressedSecondaryCacheResultHandle( + const CompressedSecondaryCacheResultHandle&) = delete; + CompressedSecondaryCacheResultHandle& operator=( + const CompressedSecondaryCacheResultHandle&) = delete; + + bool IsReady() override { return true; } + + void Wait() override {} + + Cache::ObjectPtr Value() override { return value_; } + + size_t Size() override { return size_; } + + private: + Cache::ObjectPtr value_; + size_t size_; +}; + +// The CompressedSecondaryCache is a concrete implementation of +// rocksdb::SecondaryCache. +// +// When a block is found from CompressedSecondaryCache::Lookup, we check whether +// there is a dummy block with the same key in the primary cache. +// 1. If the dummy block exits, we erase the block from +// CompressedSecondaryCache and insert it into the primary cache. +// 2. If not, we just insert a dummy block into the primary cache +// (charging the actual size of the block) and don not erase the block from +// CompressedSecondaryCache. A standalone handle is returned to the caller. +// +// When a block is evicted from the primary cache, we check whether +// there is a dummy block with the same key in CompressedSecondaryCache. +// 1. If the dummy block exits, the block is inserted into +// CompressedSecondaryCache. +// 2. If not, we just insert a dummy block (size 0) in CompressedSecondaryCache. +// +// Users can also cast a pointer to CompressedSecondaryCache and call methods on +// it directly, especially custom methods that may be added +// in the future. For example - +// std::unique_ptr cache = +// NewCompressedSecondaryCache(opts); +// static_cast(cache.get())->Erase(key); + +class CompressedSecondaryCache : public SecondaryCache { + public: + explicit CompressedSecondaryCache( + const CompressedSecondaryCacheOptions& opts); + ~CompressedSecondaryCache() override; + + const char* Name() const override { return "CompressedSecondaryCache"; } + + Status Insert(const Slice& key, Cache::ObjectPtr value, + const Cache::CacheItemHelper* helper) override; + + std::unique_ptr Lookup( + const Slice& key, const Cache::CacheItemHelper* helper, + Cache::CreateContext* create_context, bool /*wait*/, bool advise_erase, + bool& kept_in_sec_cache) override; + + bool SupportForceErase() const override { return true; } + + void Erase(const Slice& key) override; + + void WaitAll(std::vector /*handles*/) override {} + + Status SetCapacity(size_t capacity) override; + + Status GetCapacity(size_t& capacity) override; + + Status Deflate(size_t decrease) override; + + Status Inflate(size_t increase) override; + + std::string GetPrintableOptions() const override; + + size_t TEST_GetUsage() { return cache_->GetUsage(); } + + private: + friend class CompressedSecondaryCacheTestBase; + static constexpr std::array malloc_bin_sizes_{ + 128, 256, 512, 1024, 2048, 4096, 8192, 16384}; + + struct CacheValueChunk { + // TODO try "CacheAllocationPtr next;". + CacheValueChunk* next; + size_t size; + // Beginning of the chunk data (MUST BE THE LAST FIELD IN THIS STRUCT!) + char data[1]; + + void Free() { delete[] reinterpret_cast(this); } + }; + + // Split value into chunks to better fit into jemalloc bins. The chunks + // are stored in CacheValueChunk and extra charge is needed for each chunk, + // so the cache charge is recalculated here. + CacheValueChunk* SplitValueIntoChunks(const Slice& value, + CompressionType compression_type, + size_t& charge); + + // After merging chunks, the extra charge for each chunk is removed, so + // the charge is recalculated. + CacheAllocationPtr MergeChunksIntoValue(const void* chunks_head, + size_t& charge); + + // TODO: clean up to use cleaner interfaces in typed_cache.h + const Cache::CacheItemHelper* GetHelper(bool enable_custom_split_merge) const; + std::shared_ptr cache_; + CompressedSecondaryCacheOptions cache_options_; + mutable port::Mutex capacity_mutex_; + std::shared_ptr cache_res_mgr_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/compressed_secondary_cache_test.cc b/librocksdb-sys/rocksdb/cache/compressed_secondary_cache_test.cc new file mode 100644 index 0000000..4e6a137 --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/compressed_secondary_cache_test.cc @@ -0,0 +1,1076 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "cache/compressed_secondary_cache.h" + +#include +#include +#include +#include + +#include "cache/secondary_cache_adapter.h" +#include "memory/jemalloc_nodump_allocator.h" +#include "rocksdb/convenience.h" +#include "test_util/secondary_cache_test_util.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/cast_util.h" + +namespace ROCKSDB_NAMESPACE { + +using secondary_cache_test_util::GetTestingCacheTypes; +using secondary_cache_test_util::WithCacheType; + +// 16 bytes for HCC compatibility +const std::string key0 = "____ ____key0"; +const std::string key1 = "____ ____key1"; +const std::string key2 = "____ ____key2"; +const std::string key3 = "____ ____key3"; + +class CompressedSecondaryCacheTestBase : public testing::Test, + public WithCacheType { + public: + CompressedSecondaryCacheTestBase() {} + ~CompressedSecondaryCacheTestBase() override = default; + + protected: + void BasicTestHelper(std::shared_ptr sec_cache, + bool sec_cache_is_compressed) { + get_perf_context()->Reset(); + bool kept_in_sec_cache{true}; + // Lookup an non-existent key. + std::unique_ptr handle0 = + sec_cache->Lookup(key0, GetHelper(), this, true, /*advise_erase=*/true, + kept_in_sec_cache); + ASSERT_EQ(handle0, nullptr); + + Random rnd(301); + // Insert and Lookup the item k1 for the first time. + std::string str1(rnd.RandomString(1000)); + TestItem item1(str1.data(), str1.length()); + // A dummy handle is inserted if the item is inserted for the first time. + ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper())); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 1); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0); + + std::unique_ptr handle1_1 = + sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/false, + kept_in_sec_cache); + ASSERT_EQ(handle1_1, nullptr); + + // Insert and Lookup the item k1 for the second time and advise erasing it. + ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper())); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 1); + + std::unique_ptr handle1_2 = + sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/true, + kept_in_sec_cache); + ASSERT_NE(handle1_2, nullptr); + ASSERT_FALSE(kept_in_sec_cache); + if (sec_cache_is_compressed) { + ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, + 1000); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, + 1007); + } else { + ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0); + } + + std::unique_ptr val1 = + std::unique_ptr(static_cast(handle1_2->Value())); + ASSERT_NE(val1, nullptr); + ASSERT_EQ(memcmp(val1->Buf(), item1.Buf(), item1.Size()), 0); + + // Lookup the item k1 again. + std::unique_ptr handle1_3 = + sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/true, + kept_in_sec_cache); + ASSERT_EQ(handle1_3, nullptr); + + // Insert and Lookup the item k2. + std::string str2(rnd.RandomString(1000)); + TestItem item2(str2.data(), str2.length()); + ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper())); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 2); + std::unique_ptr handle2_1 = + sec_cache->Lookup(key2, GetHelper(), this, true, /*advise_erase=*/false, + kept_in_sec_cache); + ASSERT_EQ(handle2_1, nullptr); + + ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper())); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 2); + if (sec_cache_is_compressed) { + ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, + 2000); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, + 2014); + } else { + ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0); + } + std::unique_ptr handle2_2 = + sec_cache->Lookup(key2, GetHelper(), this, true, /*advise_erase=*/false, + kept_in_sec_cache); + ASSERT_NE(handle2_2, nullptr); + std::unique_ptr val2 = + std::unique_ptr(static_cast(handle2_2->Value())); + ASSERT_NE(val2, nullptr); + ASSERT_EQ(memcmp(val2->Buf(), item2.Buf(), item2.Size()), 0); + + std::vector handles = {handle1_2.get(), + handle2_2.get()}; + sec_cache->WaitAll(handles); + + sec_cache.reset(); + } + + void BasicTest(bool sec_cache_is_compressed, bool use_jemalloc) { + CompressedSecondaryCacheOptions opts; + opts.capacity = 2048; + opts.num_shard_bits = 0; + + if (sec_cache_is_compressed) { + if (!LZ4_Supported()) { + ROCKSDB_GTEST_SKIP("This test requires LZ4 support."); + opts.compression_type = CompressionType::kNoCompression; + sec_cache_is_compressed = false; + } + } else { + opts.compression_type = CompressionType::kNoCompression; + } + + if (use_jemalloc) { + JemallocAllocatorOptions jopts; + std::shared_ptr allocator; + std::string msg; + if (JemallocNodumpAllocator::IsSupported(&msg)) { + Status s = NewJemallocNodumpAllocator(jopts, &allocator); + if (s.ok()) { + opts.memory_allocator = allocator; + } + } else { + ROCKSDB_GTEST_BYPASS("JEMALLOC not supported"); + } + } + std::shared_ptr sec_cache = + NewCompressedSecondaryCache(opts); + + BasicTestHelper(sec_cache, sec_cache_is_compressed); + } + + void FailsTest(bool sec_cache_is_compressed) { + CompressedSecondaryCacheOptions secondary_cache_opts; + if (sec_cache_is_compressed) { + if (!LZ4_Supported()) { + ROCKSDB_GTEST_SKIP("This test requires LZ4 support."); + secondary_cache_opts.compression_type = CompressionType::kNoCompression; + } + } else { + secondary_cache_opts.compression_type = CompressionType::kNoCompression; + } + + secondary_cache_opts.capacity = 1100; + secondary_cache_opts.num_shard_bits = 0; + std::shared_ptr sec_cache = + NewCompressedSecondaryCache(secondary_cache_opts); + + // Insert and Lookup the first item. + Random rnd(301); + std::string str1(rnd.RandomString(1000)); + TestItem item1(str1.data(), str1.length()); + // Insert a dummy handle. + ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper())); + // Insert k1. + ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper())); + + // Insert and Lookup the second item. + std::string str2(rnd.RandomString(200)); + TestItem item2(str2.data(), str2.length()); + // Insert a dummy handle, k1 is not evicted. + ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper())); + bool kept_in_sec_cache{false}; + std::unique_ptr handle1 = + sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/false, + kept_in_sec_cache); + ASSERT_EQ(handle1, nullptr); + + // Insert k2 and k1 is evicted. + ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper())); + std::unique_ptr handle2 = + sec_cache->Lookup(key2, GetHelper(), this, true, /*advise_erase=*/false, + kept_in_sec_cache); + ASSERT_NE(handle2, nullptr); + std::unique_ptr val2 = + std::unique_ptr(static_cast(handle2->Value())); + ASSERT_NE(val2, nullptr); + ASSERT_EQ(memcmp(val2->Buf(), item2.Buf(), item2.Size()), 0); + + // Insert k1 again and a dummy handle is inserted. + ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper())); + + std::unique_ptr handle1_1 = + sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/false, + kept_in_sec_cache); + ASSERT_EQ(handle1_1, nullptr); + + // Create Fails. + SetFailCreate(true); + std::unique_ptr handle2_1 = + sec_cache->Lookup(key2, GetHelper(), this, true, /*advise_erase=*/true, + kept_in_sec_cache); + ASSERT_EQ(handle2_1, nullptr); + + // Save Fails. + std::string str3 = rnd.RandomString(10); + TestItem item3(str3.data(), str3.length()); + // The Status is OK because a dummy handle is inserted. + ASSERT_OK(sec_cache->Insert(key3, &item3, GetHelperFail())); + ASSERT_NOK(sec_cache->Insert(key3, &item3, GetHelperFail())); + + sec_cache.reset(); + } + + void BasicIntegrationTest(bool sec_cache_is_compressed, + bool enable_custom_split_merge) { + CompressedSecondaryCacheOptions secondary_cache_opts; + + if (sec_cache_is_compressed) { + if (!LZ4_Supported()) { + ROCKSDB_GTEST_SKIP("This test requires LZ4 support."); + secondary_cache_opts.compression_type = CompressionType::kNoCompression; + sec_cache_is_compressed = false; + } + } else { + secondary_cache_opts.compression_type = CompressionType::kNoCompression; + } + + secondary_cache_opts.capacity = 6000; + secondary_cache_opts.num_shard_bits = 0; + secondary_cache_opts.enable_custom_split_merge = enable_custom_split_merge; + std::shared_ptr secondary_cache = + NewCompressedSecondaryCache(secondary_cache_opts); + std::shared_ptr cache = NewCache( + /*_capacity =*/1300, /*_num_shard_bits =*/0, + /*_strict_capacity_limit =*/true, secondary_cache); + std::shared_ptr stats = CreateDBStatistics(); + + get_perf_context()->Reset(); + Random rnd(301); + std::string str1 = rnd.RandomString(1001); + auto item1_1 = new TestItem(str1.data(), str1.length()); + ASSERT_OK(cache->Insert(key1, item1_1, GetHelper(), str1.length())); + + std::string str2 = rnd.RandomString(1012); + auto item2_1 = new TestItem(str2.data(), str2.length()); + // After this Insert, primary cache contains k2 and secondary cache contains + // k1's dummy item. + ASSERT_OK(cache->Insert(key2, item2_1, GetHelper(), str2.length())); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 1); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0); + + std::string str3 = rnd.RandomString(1024); + auto item3_1 = new TestItem(str3.data(), str3.length()); + // After this Insert, primary cache contains k3 and secondary cache contains + // k1's dummy item and k2's dummy item. + ASSERT_OK(cache->Insert(key3, item3_1, GetHelper(), str3.length())); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 2); + + // After this Insert, primary cache contains k1 and secondary cache contains + // k1's dummy item, k2's dummy item, and k3's dummy item. + auto item1_2 = new TestItem(str1.data(), str1.length()); + ASSERT_OK(cache->Insert(key1, item1_2, GetHelper(), str1.length())); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 3); + + // After this Insert, primary cache contains k2 and secondary cache contains + // k1's item, k2's dummy item, and k3's dummy item. + auto item2_2 = new TestItem(str2.data(), str2.length()); + ASSERT_OK(cache->Insert(key2, item2_2, GetHelper(), str2.length())); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 1); + if (sec_cache_is_compressed) { + ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, + str1.length()); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, + 1008); + } else { + ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0); + } + + // After this Insert, primary cache contains k3 and secondary cache contains + // k1's item and k2's item. + auto item3_2 = new TestItem(str3.data(), str3.length()); + ASSERT_OK(cache->Insert(key3, item3_2, GetHelper(), str3.length())); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 2); + if (sec_cache_is_compressed) { + ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, + str1.length() + str2.length()); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, + 2027); + } else { + ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0); + } + + Cache::Handle* handle; + handle = cache->Lookup(key3, GetHelper(), this, Cache::Priority::LOW, + stats.get()); + ASSERT_NE(handle, nullptr); + auto val3 = static_cast(cache->Value(handle)); + ASSERT_NE(val3, nullptr); + ASSERT_EQ(memcmp(val3->Buf(), item3_2->Buf(), item3_2->Size()), 0); + cache->Release(handle); + + // Lookup an non-existent key. + handle = cache->Lookup(key0, GetHelper(), this, Cache::Priority::LOW, + stats.get()); + ASSERT_EQ(handle, nullptr); + + // This Lookup should just insert a dummy handle in the primary cache + // and the k1 is still in the secondary cache. + handle = cache->Lookup(key1, GetHelper(), this, Cache::Priority::LOW, + stats.get()); + ASSERT_NE(handle, nullptr); + ASSERT_EQ(get_perf_context()->block_cache_standalone_handle_count, 1); + auto val1_1 = static_cast(cache->Value(handle)); + ASSERT_NE(val1_1, nullptr); + ASSERT_EQ(memcmp(val1_1->Buf(), str1.data(), str1.size()), 0); + cache->Release(handle); + + // This Lookup should erase k1 from the secondary cache and insert + // it into primary cache; then k3 is demoted. + // k2 and k3 are in secondary cache. + handle = cache->Lookup(key1, GetHelper(), this, Cache::Priority::LOW, + stats.get()); + ASSERT_NE(handle, nullptr); + ASSERT_EQ(get_perf_context()->block_cache_standalone_handle_count, 1); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 3); + cache->Release(handle); + + // k2 is still in secondary cache. + handle = cache->Lookup(key2, GetHelper(), this, Cache::Priority::LOW, + stats.get()); + ASSERT_NE(handle, nullptr); + ASSERT_EQ(get_perf_context()->block_cache_standalone_handle_count, 2); + cache->Release(handle); + + // Testing SetCapacity(). + ASSERT_OK(secondary_cache->SetCapacity(0)); + handle = cache->Lookup(key3, GetHelper(), this, Cache::Priority::LOW, + stats.get()); + ASSERT_EQ(handle, nullptr); + + ASSERT_OK(secondary_cache->SetCapacity(7000)); + size_t capacity; + ASSERT_OK(secondary_cache->GetCapacity(capacity)); + ASSERT_EQ(capacity, 7000); + auto item1_3 = new TestItem(str1.data(), str1.length()); + // After this Insert, primary cache contains k1. + ASSERT_OK(cache->Insert(key1, item1_3, GetHelper(), str2.length())); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 3); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 4); + + auto item2_3 = new TestItem(str2.data(), str2.length()); + // After this Insert, primary cache contains k2 and secondary cache contains + // k1's dummy item. + ASSERT_OK(cache->Insert(key2, item2_3, GetHelper(), str1.length())); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 4); + + auto item1_4 = new TestItem(str1.data(), str1.length()); + // After this Insert, primary cache contains k1 and secondary cache contains + // k1's dummy item and k2's dummy item. + ASSERT_OK(cache->Insert(key1, item1_4, GetHelper(), str2.length())); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 5); + + auto item2_4 = new TestItem(str2.data(), str2.length()); + // After this Insert, primary cache contains k2 and secondary cache contains + // k1's real item and k2's dummy item. + ASSERT_OK(cache->Insert(key2, item2_4, GetHelper(), str2.length())); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 5); + // This Lookup should just insert a dummy handle in the primary cache + // and the k1 is still in the secondary cache. + handle = cache->Lookup(key1, GetHelper(), this, Cache::Priority::LOW, + stats.get()); + + ASSERT_NE(handle, nullptr); + cache->Release(handle); + ASSERT_EQ(get_perf_context()->block_cache_standalone_handle_count, 3); + + cache.reset(); + secondary_cache.reset(); + } + + void BasicIntegrationFailTest(bool sec_cache_is_compressed) { + CompressedSecondaryCacheOptions secondary_cache_opts; + + if (sec_cache_is_compressed) { + if (!LZ4_Supported()) { + ROCKSDB_GTEST_SKIP("This test requires LZ4 support."); + secondary_cache_opts.compression_type = CompressionType::kNoCompression; + } + } else { + secondary_cache_opts.compression_type = CompressionType::kNoCompression; + } + + secondary_cache_opts.capacity = 6000; + secondary_cache_opts.num_shard_bits = 0; + std::shared_ptr secondary_cache = + NewCompressedSecondaryCache(secondary_cache_opts); + + std::shared_ptr cache = NewCache( + /*_capacity=*/1300, /*_num_shard_bits=*/0, + /*_strict_capacity_limit=*/false, secondary_cache); + + Random rnd(301); + std::string str1 = rnd.RandomString(1001); + auto item1 = std::make_unique(str1.data(), str1.length()); + ASSERT_OK(cache->Insert(key1, item1.get(), GetHelper(), str1.length())); + item1.release(); // Appease clang-analyze "potential memory leak" + + Cache::Handle* handle; + handle = cache->Lookup(key2, nullptr, this, Cache::Priority::LOW); + ASSERT_EQ(handle, nullptr); + handle = cache->Lookup(key2, GetHelper(), this, Cache::Priority::LOW); + ASSERT_EQ(handle, nullptr); + + Cache::AsyncLookupHandle ah; + ah.key = key2; + ah.helper = GetHelper(); + ah.create_context = this; + ah.priority = Cache::Priority::LOW; + cache->StartAsyncLookup(ah); + cache->Wait(ah); + ASSERT_EQ(ah.Result(), nullptr); + + cache.reset(); + secondary_cache.reset(); + } + + void IntegrationSaveFailTest(bool sec_cache_is_compressed) { + CompressedSecondaryCacheOptions secondary_cache_opts; + + if (sec_cache_is_compressed) { + if (!LZ4_Supported()) { + ROCKSDB_GTEST_SKIP("This test requires LZ4 support."); + secondary_cache_opts.compression_type = CompressionType::kNoCompression; + } + } else { + secondary_cache_opts.compression_type = CompressionType::kNoCompression; + } + + secondary_cache_opts.capacity = 6000; + secondary_cache_opts.num_shard_bits = 0; + + std::shared_ptr secondary_cache = + NewCompressedSecondaryCache(secondary_cache_opts); + + std::shared_ptr cache = NewCache( + /*_capacity=*/1300, /*_num_shard_bits=*/0, + /*_strict_capacity_limit=*/true, secondary_cache); + + Random rnd(301); + std::string str1 = rnd.RandomString(1001); + auto item1 = new TestItem(str1.data(), str1.length()); + ASSERT_OK(cache->Insert(key1, item1, GetHelperFail(), str1.length())); + + std::string str2 = rnd.RandomString(1002); + auto item2 = new TestItem(str2.data(), str2.length()); + // k1 should be demoted to the secondary cache. + ASSERT_OK(cache->Insert(key2, item2, GetHelperFail(), str2.length())); + + Cache::Handle* handle; + handle = cache->Lookup(key2, GetHelperFail(), this, Cache::Priority::LOW); + ASSERT_NE(handle, nullptr); + cache->Release(handle); + // This lookup should fail, since k1 demotion would have failed. + handle = cache->Lookup(key1, GetHelperFail(), this, Cache::Priority::LOW); + ASSERT_EQ(handle, nullptr); + // Since k1 was not promoted, k2 should still be in cache. + handle = cache->Lookup(key2, GetHelperFail(), this, Cache::Priority::LOW); + ASSERT_NE(handle, nullptr); + cache->Release(handle); + + cache.reset(); + secondary_cache.reset(); + } + + void IntegrationCreateFailTest(bool sec_cache_is_compressed) { + CompressedSecondaryCacheOptions secondary_cache_opts; + + if (sec_cache_is_compressed) { + if (!LZ4_Supported()) { + ROCKSDB_GTEST_SKIP("This test requires LZ4 support."); + secondary_cache_opts.compression_type = CompressionType::kNoCompression; + } + } else { + secondary_cache_opts.compression_type = CompressionType::kNoCompression; + } + + secondary_cache_opts.capacity = 6000; + secondary_cache_opts.num_shard_bits = 0; + + std::shared_ptr secondary_cache = + NewCompressedSecondaryCache(secondary_cache_opts); + + std::shared_ptr cache = NewCache( + /*_capacity=*/1300, /*_num_shard_bits=*/0, + /*_strict_capacity_limit=*/true, secondary_cache); + + Random rnd(301); + std::string str1 = rnd.RandomString(1001); + auto item1 = new TestItem(str1.data(), str1.length()); + ASSERT_OK(cache->Insert(key1, item1, GetHelper(), str1.length())); + + std::string str2 = rnd.RandomString(1002); + auto item2 = new TestItem(str2.data(), str2.length()); + // k1 should be demoted to the secondary cache. + ASSERT_OK(cache->Insert(key2, item2, GetHelper(), str2.length())); + + Cache::Handle* handle; + SetFailCreate(true); + handle = cache->Lookup(key2, GetHelper(), this, Cache::Priority::LOW); + ASSERT_NE(handle, nullptr); + cache->Release(handle); + // This lookup should fail, since k1 creation would have failed + handle = cache->Lookup(key1, GetHelper(), this, Cache::Priority::LOW); + ASSERT_EQ(handle, nullptr); + // Since k1 didn't get promoted, k2 should still be in cache + handle = cache->Lookup(key2, GetHelper(), this, Cache::Priority::LOW); + ASSERT_NE(handle, nullptr); + cache->Release(handle); + + cache.reset(); + secondary_cache.reset(); + } + + void IntegrationFullCapacityTest(bool sec_cache_is_compressed) { + CompressedSecondaryCacheOptions secondary_cache_opts; + + if (sec_cache_is_compressed) { + if (!LZ4_Supported()) { + ROCKSDB_GTEST_SKIP("This test requires LZ4 support."); + secondary_cache_opts.compression_type = CompressionType::kNoCompression; + } + } else { + secondary_cache_opts.compression_type = CompressionType::kNoCompression; + } + + secondary_cache_opts.capacity = 6000; + secondary_cache_opts.num_shard_bits = 0; + + std::shared_ptr secondary_cache = + NewCompressedSecondaryCache(secondary_cache_opts); + + std::shared_ptr cache = NewCache( + /*_capacity=*/1300, /*_num_shard_bits=*/0, + /*_strict_capacity_limit=*/false, secondary_cache); + + Random rnd(301); + std::string str1 = rnd.RandomString(1001); + auto item1_1 = new TestItem(str1.data(), str1.length()); + ASSERT_OK(cache->Insert(key1, item1_1, GetHelper(), str1.length())); + + std::string str2 = rnd.RandomString(1002); + std::string str2_clone{str2}; + auto item2 = new TestItem(str2.data(), str2.length()); + // After this Insert, primary cache contains k2 and secondary cache contains + // k1's dummy item. + ASSERT_OK(cache->Insert(key2, item2, GetHelper(), str2.length())); + + // After this Insert, primary cache contains k1 and secondary cache contains + // k1's dummy item and k2's dummy item. + auto item1_2 = new TestItem(str1.data(), str1.length()); + ASSERT_OK(cache->Insert(key1, item1_2, GetHelper(), str1.length())); + + auto item2_2 = new TestItem(str2.data(), str2.length()); + // After this Insert, primary cache contains k2 and secondary cache contains + // k1's item and k2's dummy item. + ASSERT_OK(cache->Insert(key2, item2_2, GetHelper(), str2.length())); + + Cache::Handle* handle2; + handle2 = cache->Lookup(key2, GetHelper(), this, Cache::Priority::LOW); + ASSERT_NE(handle2, nullptr); + cache->Release(handle2); + + // k1 promotion should fail because cache is at capacity and + // strict_capacity_limit is true, but the lookup should still succeed. + // A k1's dummy item is inserted into primary cache. + Cache::Handle* handle1; + handle1 = cache->Lookup(key1, GetHelper(), this, Cache::Priority::LOW); + ASSERT_NE(handle1, nullptr); + cache->Release(handle1); + + // Since k1 didn't get inserted, k2 should still be in cache + handle2 = cache->Lookup(key2, GetHelper(), this, Cache::Priority::LOW); + ASSERT_NE(handle2, nullptr); + cache->Release(handle2); + + cache.reset(); + secondary_cache.reset(); + } + + void SplitValueIntoChunksTest() { + JemallocAllocatorOptions jopts; + std::shared_ptr allocator; + std::string msg; + if (JemallocNodumpAllocator::IsSupported(&msg)) { + Status s = NewJemallocNodumpAllocator(jopts, &allocator); + if (!s.ok()) { + ROCKSDB_GTEST_BYPASS("JEMALLOC not supported"); + } + } else { + ROCKSDB_GTEST_BYPASS("JEMALLOC not supported"); + } + + using CacheValueChunk = CompressedSecondaryCache::CacheValueChunk; + std::unique_ptr sec_cache = + std::make_unique( + CompressedSecondaryCacheOptions(1000, 0, true, 0.5, 0.0, + allocator)); + Random rnd(301); + // 8500 = 8169 + 233 + 98, so there should be 3 chunks after split. + size_t str_size{8500}; + std::string str = rnd.RandomString(static_cast(str_size)); + size_t charge{0}; + CacheValueChunk* chunks_head = + sec_cache->SplitValueIntoChunks(str, kLZ4Compression, charge); + ASSERT_EQ(charge, str_size + 3 * (sizeof(CacheValueChunk) - 1)); + + CacheValueChunk* current_chunk = chunks_head; + ASSERT_EQ(current_chunk->size, 8192 - sizeof(CacheValueChunk) + 1); + current_chunk = current_chunk->next; + ASSERT_EQ(current_chunk->size, 256 - sizeof(CacheValueChunk) + 1); + current_chunk = current_chunk->next; + ASSERT_EQ(current_chunk->size, 98); + + sec_cache->GetHelper(true)->del_cb(chunks_head, /*alloc*/ nullptr); + } + + void MergeChunksIntoValueTest() { + using CacheValueChunk = CompressedSecondaryCache::CacheValueChunk; + Random rnd(301); + size_t size1{2048}; + std::string str1 = rnd.RandomString(static_cast(size1)); + CacheValueChunk* current_chunk = reinterpret_cast( + new char[sizeof(CacheValueChunk) - 1 + size1]); + CacheValueChunk* chunks_head = current_chunk; + memcpy(current_chunk->data, str1.data(), size1); + current_chunk->size = size1; + + size_t size2{256}; + std::string str2 = rnd.RandomString(static_cast(size2)); + current_chunk->next = reinterpret_cast( + new char[sizeof(CacheValueChunk) - 1 + size2]); + current_chunk = current_chunk->next; + memcpy(current_chunk->data, str2.data(), size2); + current_chunk->size = size2; + + size_t size3{31}; + std::string str3 = rnd.RandomString(static_cast(size3)); + current_chunk->next = reinterpret_cast( + new char[sizeof(CacheValueChunk) - 1 + size3]); + current_chunk = current_chunk->next; + memcpy(current_chunk->data, str3.data(), size3); + current_chunk->size = size3; + current_chunk->next = nullptr; + + std::string str = str1 + str2 + str3; + + std::unique_ptr sec_cache = + std::make_unique( + CompressedSecondaryCacheOptions(1000, 0, true, 0.5, 0.0)); + size_t charge{0}; + CacheAllocationPtr value = + sec_cache->MergeChunksIntoValue(chunks_head, charge); + ASSERT_EQ(charge, size1 + size2 + size3); + std::string value_str{value.get(), charge}; + ASSERT_EQ(strcmp(value_str.data(), str.data()), 0); + + while (chunks_head != nullptr) { + CacheValueChunk* tmp_chunk = chunks_head; + chunks_head = chunks_head->next; + tmp_chunk->Free(); + } + } + + void SplictValueAndMergeChunksTest() { + JemallocAllocatorOptions jopts; + std::shared_ptr allocator; + std::string msg; + if (JemallocNodumpAllocator::IsSupported(&msg)) { + Status s = NewJemallocNodumpAllocator(jopts, &allocator); + if (!s.ok()) { + ROCKSDB_GTEST_BYPASS("JEMALLOC not supported"); + } + } else { + ROCKSDB_GTEST_BYPASS("JEMALLOC not supported"); + } + + using CacheValueChunk = CompressedSecondaryCache::CacheValueChunk; + std::unique_ptr sec_cache = + std::make_unique( + CompressedSecondaryCacheOptions(1000, 0, true, 0.5, 0.0, + allocator)); + Random rnd(301); + // 8500 = 8169 + 233 + 98, so there should be 3 chunks after split. + size_t str_size{8500}; + std::string str = rnd.RandomString(static_cast(str_size)); + size_t charge{0}; + CacheValueChunk* chunks_head = + sec_cache->SplitValueIntoChunks(str, kLZ4Compression, charge); + ASSERT_EQ(charge, str_size + 3 * (sizeof(CacheValueChunk) - 1)); + + CacheAllocationPtr value = + sec_cache->MergeChunksIntoValue(chunks_head, charge); + ASSERT_EQ(charge, str_size); + std::string value_str{value.get(), charge}; + ASSERT_EQ(strcmp(value_str.data(), str.data()), 0); + + sec_cache->GetHelper(true)->del_cb(chunks_head, /*alloc*/ nullptr); + } +}; + +class CompressedSecondaryCacheTest + : public CompressedSecondaryCacheTestBase, + public testing::WithParamInterface { + const std::string& Type() override { return GetParam(); } +}; + +INSTANTIATE_TEST_CASE_P(CompressedSecondaryCacheTest, + CompressedSecondaryCacheTest, GetTestingCacheTypes()); + +class CompressedSecCacheTestWithCompressAndAllocatorParam + : public CompressedSecondaryCacheTestBase, + public ::testing::WithParamInterface< + std::tuple> { + public: + CompressedSecCacheTestWithCompressAndAllocatorParam() { + sec_cache_is_compressed_ = std::get<0>(GetParam()); + use_jemalloc_ = std::get<1>(GetParam()); + } + const std::string& Type() override { return std::get<2>(GetParam()); } + bool sec_cache_is_compressed_; + bool use_jemalloc_; +}; + +TEST_P(CompressedSecCacheTestWithCompressAndAllocatorParam, BasicTes) { + BasicTest(sec_cache_is_compressed_, use_jemalloc_); +} + +INSTANTIATE_TEST_CASE_P(CompressedSecCacheTests, + CompressedSecCacheTestWithCompressAndAllocatorParam, + ::testing::Combine(testing::Bool(), testing::Bool(), + GetTestingCacheTypes())); + +class CompressedSecondaryCacheTestWithCompressionParam + : public CompressedSecondaryCacheTestBase, + public ::testing::WithParamInterface> { + public: + CompressedSecondaryCacheTestWithCompressionParam() { + sec_cache_is_compressed_ = std::get<0>(GetParam()); + } + const std::string& Type() override { return std::get<1>(GetParam()); } + bool sec_cache_is_compressed_; +}; + +TEST_P(CompressedSecondaryCacheTestWithCompressionParam, BasicTestFromString) { + std::shared_ptr sec_cache{nullptr}; + std::string sec_cache_uri; + if (sec_cache_is_compressed_) { + if (LZ4_Supported()) { + sec_cache_uri = + "compressed_secondary_cache://" + "capacity=2048;num_shard_bits=0;compression_type=kLZ4Compression;" + "compress_format_version=2"; + } else { + ROCKSDB_GTEST_SKIP("This test requires LZ4 support."); + sec_cache_uri = + "compressed_secondary_cache://" + "capacity=2048;num_shard_bits=0;compression_type=kNoCompression"; + sec_cache_is_compressed_ = false; + } + Status s = SecondaryCache::CreateFromString(ConfigOptions(), sec_cache_uri, + &sec_cache); + EXPECT_OK(s); + } else { + sec_cache_uri = + "compressed_secondary_cache://" + "capacity=2048;num_shard_bits=0;compression_type=kNoCompression"; + Status s = SecondaryCache::CreateFromString(ConfigOptions(), sec_cache_uri, + &sec_cache); + EXPECT_OK(s); + } + BasicTestHelper(sec_cache, sec_cache_is_compressed_); +} + +TEST_P(CompressedSecondaryCacheTestWithCompressionParam, + BasicTestFromStringWithSplit) { + std::shared_ptr sec_cache{nullptr}; + std::string sec_cache_uri; + if (sec_cache_is_compressed_) { + if (LZ4_Supported()) { + sec_cache_uri = + "compressed_secondary_cache://" + "capacity=2048;num_shard_bits=0;compression_type=kLZ4Compression;" + "compress_format_version=2;enable_custom_split_merge=true"; + } else { + ROCKSDB_GTEST_SKIP("This test requires LZ4 support."); + sec_cache_uri = + "compressed_secondary_cache://" + "capacity=2048;num_shard_bits=0;compression_type=kNoCompression;" + "enable_custom_split_merge=true"; + sec_cache_is_compressed_ = false; + } + Status s = SecondaryCache::CreateFromString(ConfigOptions(), sec_cache_uri, + &sec_cache); + EXPECT_OK(s); + } else { + sec_cache_uri = + "compressed_secondary_cache://" + "capacity=2048;num_shard_bits=0;compression_type=kNoCompression;" + "enable_custom_split_merge=true"; + Status s = SecondaryCache::CreateFromString(ConfigOptions(), sec_cache_uri, + &sec_cache); + EXPECT_OK(s); + } + BasicTestHelper(sec_cache, sec_cache_is_compressed_); +} + + +TEST_P(CompressedSecondaryCacheTestWithCompressionParam, FailsTest) { + FailsTest(sec_cache_is_compressed_); +} + +TEST_P(CompressedSecondaryCacheTestWithCompressionParam, + BasicIntegrationFailTest) { + BasicIntegrationFailTest(sec_cache_is_compressed_); +} + +TEST_P(CompressedSecondaryCacheTestWithCompressionParam, + IntegrationSaveFailTest) { + IntegrationSaveFailTest(sec_cache_is_compressed_); +} + +TEST_P(CompressedSecondaryCacheTestWithCompressionParam, + IntegrationCreateFailTest) { + IntegrationCreateFailTest(sec_cache_is_compressed_); +} + +TEST_P(CompressedSecondaryCacheTestWithCompressionParam, + IntegrationFullCapacityTest) { + IntegrationFullCapacityTest(sec_cache_is_compressed_); +} + +TEST_P(CompressedSecondaryCacheTestWithCompressionParam, EntryRoles) { + CompressedSecondaryCacheOptions opts; + opts.capacity = 2048; + opts.num_shard_bits = 0; + + if (sec_cache_is_compressed_) { + if (!LZ4_Supported()) { + ROCKSDB_GTEST_SKIP("This test requires LZ4 support."); + return; + } + } else { + opts.compression_type = CompressionType::kNoCompression; + } + + // Select a random subset to include, for fast test + Random& r = *Random::GetTLSInstance(); + CacheEntryRoleSet do_not_compress; + for (uint32_t i = 0; i < kNumCacheEntryRoles; ++i) { + // A few included on average, but decent chance of zero + if (r.OneIn(5)) { + do_not_compress.Add(static_cast(i)); + } + } + opts.do_not_compress_roles = do_not_compress; + + std::shared_ptr sec_cache = NewCompressedSecondaryCache(opts); + + // Fixed seed to ensure consistent compressibility (doesn't compress) + std::string junk(Random(301).RandomString(1000)); + + for (uint32_t i = 0; i < kNumCacheEntryRoles; ++i) { + CacheEntryRole role = static_cast(i); + + // Uniquify `junk` + junk[0] = static_cast(i); + TestItem item{junk.data(), junk.length()}; + Slice ith_key = Slice(junk.data(), 16); + + get_perf_context()->Reset(); + ASSERT_OK(sec_cache->Insert(ith_key, &item, GetHelper(role))); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 1U); + + ASSERT_OK(sec_cache->Insert(ith_key, &item, GetHelper(role))); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 1U); + + bool kept_in_sec_cache{true}; + std::unique_ptr handle = + sec_cache->Lookup(ith_key, GetHelper(role), this, true, + /*advise_erase=*/true, kept_in_sec_cache); + ASSERT_NE(handle, nullptr); + + // Lookup returns the right data + std::unique_ptr val = + std::unique_ptr(static_cast(handle->Value())); + ASSERT_NE(val, nullptr); + ASSERT_EQ(memcmp(val->Buf(), item.Buf(), item.Size()), 0); + + bool compressed = + sec_cache_is_compressed_ && !do_not_compress.Contains(role); + if (compressed) { + ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, + 1000); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, + 1007); + } else { + ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0); + ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0); + } + } +} + +INSTANTIATE_TEST_CASE_P(CompressedSecCacheTests, + CompressedSecondaryCacheTestWithCompressionParam, + testing::Combine(testing::Bool(), + GetTestingCacheTypes())); + +class CompressedSecCacheTestWithCompressAndSplitParam + : public CompressedSecondaryCacheTestBase, + public ::testing::WithParamInterface< + std::tuple> { + public: + CompressedSecCacheTestWithCompressAndSplitParam() { + sec_cache_is_compressed_ = std::get<0>(GetParam()); + enable_custom_split_merge_ = std::get<1>(GetParam()); + } + const std::string& Type() override { return std::get<2>(GetParam()); } + bool sec_cache_is_compressed_; + bool enable_custom_split_merge_; +}; + +TEST_P(CompressedSecCacheTestWithCompressAndSplitParam, BasicIntegrationTest) { + BasicIntegrationTest(sec_cache_is_compressed_, enable_custom_split_merge_); +} + +INSTANTIATE_TEST_CASE_P(CompressedSecCacheTests, + CompressedSecCacheTestWithCompressAndSplitParam, + ::testing::Combine(testing::Bool(), testing::Bool(), + GetTestingCacheTypes())); + +TEST_P(CompressedSecondaryCacheTest, SplitValueIntoChunksTest) { + SplitValueIntoChunksTest(); +} + +TEST_P(CompressedSecondaryCacheTest, MergeChunksIntoValueTest) { + MergeChunksIntoValueTest(); +} + +TEST_P(CompressedSecondaryCacheTest, SplictValueAndMergeChunksTest) { + SplictValueAndMergeChunksTest(); +} + +class CompressedSecCacheTestWithTiered : public ::testing::Test { + public: + CompressedSecCacheTestWithTiered() { + LRUCacheOptions lru_opts; + TieredVolatileCacheOptions opts; + lru_opts.capacity = 70 << 20; + opts.cache_opts = &lru_opts; + opts.cache_type = PrimaryCacheType::kCacheTypeLRU; + opts.comp_cache_opts.capacity = 30 << 20; + cache_ = NewTieredVolatileCache(opts); + cache_res_mgr_ = + std::make_shared>( + cache_); + } + + protected: + CacheReservationManager* cache_res_mgr() { return cache_res_mgr_.get(); } + + Cache* GetCache() { + return static_cast_with_check( + cache_.get()) + ->TEST_GetCache(); + } + + SecondaryCache* GetSecondaryCache() { + return static_cast_with_check( + cache_.get()) + ->TEST_GetSecondaryCache(); + } + + size_t GetPercent(size_t val, unsigned int percent) { + return static_cast(val * percent / 100); + } + + private: + std::shared_ptr cache_; + std::shared_ptr cache_res_mgr_; +}; + +bool CacheUsageWithinBounds(size_t val1, size_t val2, size_t error) { + return ((val1 < (val2 + error)) && (val1 > (val2 - error))); +} + +TEST_F(CompressedSecCacheTestWithTiered, CacheReservationManager) { + CompressedSecondaryCache* sec_cache = + reinterpret_cast(GetSecondaryCache()); + + // Use EXPECT_PRED3 instead of EXPECT_NEAR to void too many size_t to + // double explicit casts + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (30 << 20), + GetPercent(30 << 20, 1)); + EXPECT_EQ(sec_cache->TEST_GetUsage(), 0); + + ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(10 << 20)); + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (37 << 20), + GetPercent(37 << 20, 1)); + EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (3 << 20), + GetPercent(3 << 20, 1)); + + ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(0)); + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (30 << 20), + GetPercent(30 << 20, 1)); + EXPECT_EQ(sec_cache->TEST_GetUsage(), 0); +} + +TEST_F(CompressedSecCacheTestWithTiered, + CacheReservationManagerMultipleUpdate) { + CompressedSecondaryCache* sec_cache = + reinterpret_cast(GetSecondaryCache()); + + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (30 << 20), + GetPercent(30 << 20, 1)); + EXPECT_EQ(sec_cache->TEST_GetUsage(), 0); + + int i; + for (i = 0; i < 10; ++i) { + ASSERT_OK(cache_res_mgr()->UpdateCacheReservation((1 + i) << 20)); + } + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (37 << 20), + GetPercent(37 << 20, 1)); + EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (3 << 20), + GetPercent(3 << 20, 1)); + + for (i = 10; i > 0; --i) { + ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(((i - 1) << 20))); + } + EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (30 << 20), + GetPercent(30 << 20, 1)); + EXPECT_EQ(sec_cache->TEST_GetUsage(), 0); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/cache/lru_cache.cc b/librocksdb-sys/rocksdb/cache/lru_cache.cc new file mode 100644 index 0000000..b72c824 --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/lru_cache.cc @@ -0,0 +1,723 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "cache/lru_cache.h" + +#include +#include +#include +#include + +#include "cache/secondary_cache_adapter.h" +#include "monitoring/perf_context_imp.h" +#include "monitoring/statistics_impl.h" +#include "port/lang.h" +#include "util/distributed_mutex.h" + +namespace ROCKSDB_NAMESPACE { +namespace lru_cache { + +LRUHandleTable::LRUHandleTable(int max_upper_hash_bits, + MemoryAllocator* allocator) + : length_bits_(/* historical starting size*/ 4), + list_(new LRUHandle* [size_t{1} << length_bits_] {}), + elems_(0), + max_length_bits_(max_upper_hash_bits), + allocator_(allocator) {} + +LRUHandleTable::~LRUHandleTable() { + auto alloc = allocator_; + ApplyToEntriesRange( + [alloc](LRUHandle* h) { + if (!h->HasRefs()) { + h->Free(alloc); + } + }, + 0, size_t{1} << length_bits_); +} + +LRUHandle* LRUHandleTable::Lookup(const Slice& key, uint32_t hash) { + return *FindPointer(key, hash); +} + +LRUHandle* LRUHandleTable::Insert(LRUHandle* h) { + LRUHandle** ptr = FindPointer(h->key(), h->hash); + LRUHandle* old = *ptr; + h->next_hash = (old == nullptr ? nullptr : old->next_hash); + *ptr = h; + if (old == nullptr) { + ++elems_; + if ((elems_ >> length_bits_) > 0) { // elems_ >= length + // Since each cache entry is fairly large, we aim for a small + // average linked list length (<= 1). + Resize(); + } + } + return old; +} + +LRUHandle* LRUHandleTable::Remove(const Slice& key, uint32_t hash) { + LRUHandle** ptr = FindPointer(key, hash); + LRUHandle* result = *ptr; + if (result != nullptr) { + *ptr = result->next_hash; + --elems_; + } + return result; +} + +LRUHandle** LRUHandleTable::FindPointer(const Slice& key, uint32_t hash) { + LRUHandle** ptr = &list_[hash >> (32 - length_bits_)]; + while (*ptr != nullptr && ((*ptr)->hash != hash || key != (*ptr)->key())) { + ptr = &(*ptr)->next_hash; + } + return ptr; +} + +void LRUHandleTable::Resize() { + if (length_bits_ >= max_length_bits_) { + // Due to reaching limit of hash information, if we made the table bigger, + // we would allocate more addresses but only the same number would be used. + return; + } + if (length_bits_ >= 31) { + // Avoid undefined behavior shifting uint32_t by 32. + return; + } + + uint32_t old_length = uint32_t{1} << length_bits_; + int new_length_bits = length_bits_ + 1; + std::unique_ptr new_list { + new LRUHandle* [size_t{1} << new_length_bits] {} + }; + [[maybe_unused]] uint32_t count = 0; + for (uint32_t i = 0; i < old_length; i++) { + LRUHandle* h = list_[i]; + while (h != nullptr) { + LRUHandle* next = h->next_hash; + uint32_t hash = h->hash; + LRUHandle** ptr = &new_list[hash >> (32 - new_length_bits)]; + h->next_hash = *ptr; + *ptr = h; + h = next; + count++; + } + } + assert(elems_ == count); + list_ = std::move(new_list); + length_bits_ = new_length_bits; +} + +LRUCacheShard::LRUCacheShard(size_t capacity, bool strict_capacity_limit, + double high_pri_pool_ratio, + double low_pri_pool_ratio, bool use_adaptive_mutex, + CacheMetadataChargePolicy metadata_charge_policy, + int max_upper_hash_bits, + MemoryAllocator* allocator, + const Cache::EvictionCallback* eviction_callback) + : CacheShardBase(metadata_charge_policy), + capacity_(0), + high_pri_pool_usage_(0), + low_pri_pool_usage_(0), + strict_capacity_limit_(strict_capacity_limit), + high_pri_pool_ratio_(high_pri_pool_ratio), + high_pri_pool_capacity_(0), + low_pri_pool_ratio_(low_pri_pool_ratio), + low_pri_pool_capacity_(0), + table_(max_upper_hash_bits, allocator), + usage_(0), + lru_usage_(0), + mutex_(use_adaptive_mutex), + eviction_callback_(*eviction_callback) { + // Make empty circular linked list. + lru_.next = &lru_; + lru_.prev = &lru_; + lru_low_pri_ = &lru_; + lru_bottom_pri_ = &lru_; + SetCapacity(capacity); +} + +void LRUCacheShard::EraseUnRefEntries() { + autovector last_reference_list; + { + DMutexLock l(mutex_); + while (lru_.next != &lru_) { + LRUHandle* old = lru_.next; + // LRU list contains only elements which can be evicted. + assert(old->InCache() && !old->HasRefs()); + LRU_Remove(old); + table_.Remove(old->key(), old->hash); + old->SetInCache(false); + assert(usage_ >= old->total_charge); + usage_ -= old->total_charge; + last_reference_list.push_back(old); + } + } + + for (auto entry : last_reference_list) { + entry->Free(table_.GetAllocator()); + } +} + +void LRUCacheShard::ApplyToSomeEntries( + const std::function& callback, + size_t average_entries_per_lock, size_t* state) { + // The state is essentially going to be the starting hash, which works + // nicely even if we resize between calls because we use upper-most + // hash bits for table indexes. + DMutexLock l(mutex_); + int length_bits = table_.GetLengthBits(); + size_t length = size_t{1} << length_bits; + + assert(average_entries_per_lock > 0); + // Assuming we are called with same average_entries_per_lock repeatedly, + // this simplifies some logic (index_end will not overflow). + assert(average_entries_per_lock < length || *state == 0); + + size_t index_begin = *state >> (sizeof(size_t) * 8u - length_bits); + size_t index_end = index_begin + average_entries_per_lock; + if (index_end >= length) { + // Going to end + index_end = length; + *state = SIZE_MAX; + } else { + *state = index_end << (sizeof(size_t) * 8u - length_bits); + } + + table_.ApplyToEntriesRange( + [callback, + metadata_charge_policy = metadata_charge_policy_](LRUHandle* h) { + callback(h->key(), h->value, h->GetCharge(metadata_charge_policy), + h->helper); + }, + index_begin, index_end); +} + +void LRUCacheShard::TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri, + LRUHandle** lru_bottom_pri) { + DMutexLock l(mutex_); + *lru = &lru_; + *lru_low_pri = lru_low_pri_; + *lru_bottom_pri = lru_bottom_pri_; +} + +size_t LRUCacheShard::TEST_GetLRUSize() { + DMutexLock l(mutex_); + LRUHandle* lru_handle = lru_.next; + size_t lru_size = 0; + while (lru_handle != &lru_) { + lru_size++; + lru_handle = lru_handle->next; + } + return lru_size; +} + +double LRUCacheShard::GetHighPriPoolRatio() { + DMutexLock l(mutex_); + return high_pri_pool_ratio_; +} + +double LRUCacheShard::GetLowPriPoolRatio() { + DMutexLock l(mutex_); + return low_pri_pool_ratio_; +} + +void LRUCacheShard::LRU_Remove(LRUHandle* e) { + assert(e->next != nullptr); + assert(e->prev != nullptr); + if (lru_low_pri_ == e) { + lru_low_pri_ = e->prev; + } + if (lru_bottom_pri_ == e) { + lru_bottom_pri_ = e->prev; + } + e->next->prev = e->prev; + e->prev->next = e->next; + e->prev = e->next = nullptr; + assert(lru_usage_ >= e->total_charge); + lru_usage_ -= e->total_charge; + assert(!e->InHighPriPool() || !e->InLowPriPool()); + if (e->InHighPriPool()) { + assert(high_pri_pool_usage_ >= e->total_charge); + high_pri_pool_usage_ -= e->total_charge; + } else if (e->InLowPriPool()) { + assert(low_pri_pool_usage_ >= e->total_charge); + low_pri_pool_usage_ -= e->total_charge; + } +} + +void LRUCacheShard::LRU_Insert(LRUHandle* e) { + assert(e->next == nullptr); + assert(e->prev == nullptr); + if (high_pri_pool_ratio_ > 0 && (e->IsHighPri() || e->HasHit())) { + // Inset "e" to head of LRU list. + e->next = &lru_; + e->prev = lru_.prev; + e->prev->next = e; + e->next->prev = e; + e->SetInHighPriPool(true); + e->SetInLowPriPool(false); + high_pri_pool_usage_ += e->total_charge; + MaintainPoolSize(); + } else if (low_pri_pool_ratio_ > 0 && + (e->IsHighPri() || e->IsLowPri() || e->HasHit())) { + // Insert "e" to the head of low-pri pool. + e->next = lru_low_pri_->next; + e->prev = lru_low_pri_; + e->prev->next = e; + e->next->prev = e; + e->SetInHighPriPool(false); + e->SetInLowPriPool(true); + low_pri_pool_usage_ += e->total_charge; + MaintainPoolSize(); + lru_low_pri_ = e; + } else { + // Insert "e" to the head of bottom-pri pool. + e->next = lru_bottom_pri_->next; + e->prev = lru_bottom_pri_; + e->prev->next = e; + e->next->prev = e; + e->SetInHighPriPool(false); + e->SetInLowPriPool(false); + // if the low-pri pool is empty, lru_low_pri_ also needs to be updated. + if (lru_bottom_pri_ == lru_low_pri_) { + lru_low_pri_ = e; + } + lru_bottom_pri_ = e; + } + lru_usage_ += e->total_charge; +} + +void LRUCacheShard::MaintainPoolSize() { + while (high_pri_pool_usage_ > high_pri_pool_capacity_) { + // Overflow last entry in high-pri pool to low-pri pool. + lru_low_pri_ = lru_low_pri_->next; + assert(lru_low_pri_ != &lru_); + lru_low_pri_->SetInHighPriPool(false); + lru_low_pri_->SetInLowPriPool(true); + assert(high_pri_pool_usage_ >= lru_low_pri_->total_charge); + high_pri_pool_usage_ -= lru_low_pri_->total_charge; + low_pri_pool_usage_ += lru_low_pri_->total_charge; + } + + while (low_pri_pool_usage_ > low_pri_pool_capacity_) { + // Overflow last entry in low-pri pool to bottom-pri pool. + lru_bottom_pri_ = lru_bottom_pri_->next; + assert(lru_bottom_pri_ != &lru_); + lru_bottom_pri_->SetInHighPriPool(false); + lru_bottom_pri_->SetInLowPriPool(false); + assert(low_pri_pool_usage_ >= lru_bottom_pri_->total_charge); + low_pri_pool_usage_ -= lru_bottom_pri_->total_charge; + } +} + +void LRUCacheShard::EvictFromLRU(size_t charge, + autovector* deleted) { + while ((usage_ + charge) > capacity_ && lru_.next != &lru_) { + LRUHandle* old = lru_.next; + // LRU list contains only elements which can be evicted. + assert(old->InCache() && !old->HasRefs()); + LRU_Remove(old); + table_.Remove(old->key(), old->hash); + old->SetInCache(false); + assert(usage_ >= old->total_charge); + usage_ -= old->total_charge; + deleted->push_back(old); + } +} + +void LRUCacheShard::NotifyEvicted( + const autovector& evicted_handles) { + MemoryAllocator* alloc = table_.GetAllocator(); + for (LRUHandle* entry : evicted_handles) { + if (eviction_callback_ && + eviction_callback_(entry->key(), + reinterpret_cast(entry))) { + // Callback took ownership of obj; just free handle + free(entry); + } else { + // Free the entries here outside of mutex for performance reasons. + entry->Free(alloc); + } + } +} + +void LRUCacheShard::SetCapacity(size_t capacity) { + autovector last_reference_list; + { + DMutexLock l(mutex_); + capacity_ = capacity; + high_pri_pool_capacity_ = capacity_ * high_pri_pool_ratio_; + low_pri_pool_capacity_ = capacity_ * low_pri_pool_ratio_; + EvictFromLRU(0, &last_reference_list); + } + + NotifyEvicted(last_reference_list); +} + +void LRUCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) { + DMutexLock l(mutex_); + strict_capacity_limit_ = strict_capacity_limit; +} + +Status LRUCacheShard::InsertItem(LRUHandle* e, LRUHandle** handle) { + Status s = Status::OK(); + autovector last_reference_list; + + { + DMutexLock l(mutex_); + + // Free the space following strict LRU policy until enough space + // is freed or the lru list is empty. + EvictFromLRU(e->total_charge, &last_reference_list); + + if ((usage_ + e->total_charge) > capacity_ && + (strict_capacity_limit_ || handle == nullptr)) { + e->SetInCache(false); + if (handle == nullptr) { + // Don't insert the entry but still return ok, as if the entry inserted + // into cache and get evicted immediately. + last_reference_list.push_back(e); + } else { + free(e); + e = nullptr; + *handle = nullptr; + s = Status::MemoryLimit("Insert failed due to LRU cache being full."); + } + } else { + // Insert into the cache. Note that the cache might get larger than its + // capacity if not enough space was freed up. + LRUHandle* old = table_.Insert(e); + usage_ += e->total_charge; + if (old != nullptr) { + s = Status::OkOverwritten(); + assert(old->InCache()); + old->SetInCache(false); + if (!old->HasRefs()) { + // old is on LRU because it's in cache and its reference count is 0. + LRU_Remove(old); + assert(usage_ >= old->total_charge); + usage_ -= old->total_charge; + last_reference_list.push_back(old); + } + } + if (handle == nullptr) { + LRU_Insert(e); + } else { + // If caller already holds a ref, no need to take one here. + if (!e->HasRefs()) { + e->Ref(); + } + *handle = e; + } + } + } + + NotifyEvicted(last_reference_list); + + return s; +} + +LRUHandle* LRUCacheShard::Lookup(const Slice& key, uint32_t hash, + const Cache::CacheItemHelper* /*helper*/, + Cache::CreateContext* /*create_context*/, + Cache::Priority /*priority*/, + Statistics* /*stats*/) { + DMutexLock l(mutex_); + LRUHandle* e = table_.Lookup(key, hash); + if (e != nullptr) { + assert(e->InCache()); + if (!e->HasRefs()) { + // The entry is in LRU since it's in hash and has no external + // references. + LRU_Remove(e); + } + e->Ref(); + e->SetHit(); + } + return e; +} + +bool LRUCacheShard::Ref(LRUHandle* e) { + DMutexLock l(mutex_); + // To create another reference - entry must be already externally referenced. + assert(e->HasRefs()); + e->Ref(); + return true; +} + +void LRUCacheShard::SetHighPriorityPoolRatio(double high_pri_pool_ratio) { + DMutexLock l(mutex_); + high_pri_pool_ratio_ = high_pri_pool_ratio; + high_pri_pool_capacity_ = capacity_ * high_pri_pool_ratio_; + MaintainPoolSize(); +} + +void LRUCacheShard::SetLowPriorityPoolRatio(double low_pri_pool_ratio) { + DMutexLock l(mutex_); + low_pri_pool_ratio_ = low_pri_pool_ratio; + low_pri_pool_capacity_ = capacity_ * low_pri_pool_ratio_; + MaintainPoolSize(); +} + +bool LRUCacheShard::Release(LRUHandle* e, bool /*useful*/, + bool erase_if_last_ref) { + if (e == nullptr) { + return false; + } + bool must_free; + bool was_in_cache; + { + DMutexLock l(mutex_); + must_free = e->Unref(); + was_in_cache = e->InCache(); + if (must_free && was_in_cache) { + // The item is still in cache, and nobody else holds a reference to it. + if (usage_ > capacity_ || erase_if_last_ref) { + // The LRU list must be empty since the cache is full. + assert(lru_.next == &lru_ || erase_if_last_ref); + // Take this opportunity and remove the item. + table_.Remove(e->key(), e->hash); + e->SetInCache(false); + } else { + // Put the item back on the LRU list, and don't free it. + LRU_Insert(e); + must_free = false; + } + } + // If about to be freed, then decrement the cache usage. + if (must_free) { + assert(usage_ >= e->total_charge); + usage_ -= e->total_charge; + } + } + + // Free the entry here outside of mutex for performance reasons. + if (must_free) { + // Only call eviction callback if we're sure no one requested erasure + // FIXME: disabled because of test churn + if (false && was_in_cache && !erase_if_last_ref && eviction_callback_ && + eviction_callback_(e->key(), reinterpret_cast(e))) { + // Callback took ownership of obj; just free handle + free(e); + } else { + e->Free(table_.GetAllocator()); + } + } + return must_free; +} + +LRUHandle* LRUCacheShard::CreateHandle(const Slice& key, uint32_t hash, + Cache::ObjectPtr value, + const Cache::CacheItemHelper* helper, + size_t charge) { + assert(helper); + // value == nullptr is reserved for indicating failure in SecondaryCache + assert(!(helper->IsSecondaryCacheCompatible() && value == nullptr)); + + // Allocate the memory here outside of the mutex. + // If the cache is full, we'll have to release it. + // It shouldn't happen very often though. + LRUHandle* e = + static_cast(malloc(sizeof(LRUHandle) - 1 + key.size())); + + e->value = value; + e->m_flags = 0; + e->im_flags = 0; + e->helper = helper; + e->key_length = key.size(); + e->hash = hash; + e->refs = 0; + e->next = e->prev = nullptr; + memcpy(e->key_data, key.data(), key.size()); + e->CalcTotalCharge(charge, metadata_charge_policy_); + + return e; +} + +Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, + Cache::ObjectPtr value, + const Cache::CacheItemHelper* helper, + size_t charge, LRUHandle** handle, + Cache::Priority priority) { + LRUHandle* e = CreateHandle(key, hash, value, helper, charge); + e->SetPriority(priority); + e->SetInCache(true); + return InsertItem(e, handle); +} + +LRUHandle* LRUCacheShard::CreateStandalone(const Slice& key, uint32_t hash, + Cache::ObjectPtr value, + const Cache::CacheItemHelper* helper, + size_t charge, + bool allow_uncharged) { + LRUHandle* e = CreateHandle(key, hash, value, helper, charge); + e->SetIsStandalone(true); + e->Ref(); + + autovector last_reference_list; + + { + DMutexLock l(mutex_); + + EvictFromLRU(e->total_charge, &last_reference_list); + + if (strict_capacity_limit_ && (usage_ + e->total_charge) > capacity_) { + if (allow_uncharged) { + e->total_charge = 0; + } else { + free(e); + e = nullptr; + } + } else { + usage_ += e->total_charge; + } + } + + NotifyEvicted(last_reference_list); + return e; +} + +void LRUCacheShard::Erase(const Slice& key, uint32_t hash) { + LRUHandle* e; + bool last_reference = false; + { + DMutexLock l(mutex_); + e = table_.Remove(key, hash); + if (e != nullptr) { + assert(e->InCache()); + e->SetInCache(false); + if (!e->HasRefs()) { + // The entry is in LRU since it's in hash and has no external references + LRU_Remove(e); + assert(usage_ >= e->total_charge); + usage_ -= e->total_charge; + last_reference = true; + } + } + } + + // Free the entry here outside of mutex for performance reasons. + // last_reference will only be true if e != nullptr. + if (last_reference) { + e->Free(table_.GetAllocator()); + } +} + +size_t LRUCacheShard::GetUsage() const { + DMutexLock l(mutex_); + return usage_; +} + +size_t LRUCacheShard::GetPinnedUsage() const { + DMutexLock l(mutex_); + assert(usage_ >= lru_usage_); + return usage_ - lru_usage_; +} + +size_t LRUCacheShard::GetOccupancyCount() const { + DMutexLock l(mutex_); + return table_.GetOccupancyCount(); +} + +size_t LRUCacheShard::GetTableAddressCount() const { + DMutexLock l(mutex_); + return size_t{1} << table_.GetLengthBits(); +} + +void LRUCacheShard::AppendPrintableOptions(std::string& str) const { + const int kBufferSize = 200; + char buffer[kBufferSize]; + { + DMutexLock l(mutex_); + snprintf(buffer, kBufferSize, " high_pri_pool_ratio: %.3lf\n", + high_pri_pool_ratio_); + snprintf(buffer + strlen(buffer), kBufferSize - strlen(buffer), + " low_pri_pool_ratio: %.3lf\n", low_pri_pool_ratio_); + } + str.append(buffer); +} + +LRUCache::LRUCache(const LRUCacheOptions& opts) : ShardedCache(opts) { + size_t per_shard = GetPerShardCapacity(); + MemoryAllocator* alloc = memory_allocator(); + InitShards([&](LRUCacheShard* cs) { + new (cs) LRUCacheShard(per_shard, opts.strict_capacity_limit, + opts.high_pri_pool_ratio, opts.low_pri_pool_ratio, + opts.use_adaptive_mutex, opts.metadata_charge_policy, + /* max_upper_hash_bits */ 32 - opts.num_shard_bits, + alloc, &eviction_callback_); + }); +} + +Cache::ObjectPtr LRUCache::Value(Handle* handle) { + auto h = reinterpret_cast(handle); + return h->value; +} + +size_t LRUCache::GetCharge(Handle* handle) const { + return reinterpret_cast(handle)->GetCharge( + GetShard(0).metadata_charge_policy_); +} + +const Cache::CacheItemHelper* LRUCache::GetCacheItemHelper( + Handle* handle) const { + auto h = reinterpret_cast(handle); + return h->helper; +} + +size_t LRUCache::TEST_GetLRUSize() { + return SumOverShards([](LRUCacheShard& cs) { return cs.TEST_GetLRUSize(); }); +} + +double LRUCache::GetHighPriPoolRatio() { + return GetShard(0).GetHighPriPoolRatio(); +} + +} // namespace lru_cache + +std::shared_ptr LRUCacheOptions::MakeSharedCache() const { + if (num_shard_bits >= 20) { + return nullptr; // The cache cannot be sharded into too many fine pieces. + } + if (high_pri_pool_ratio < 0.0 || high_pri_pool_ratio > 1.0) { + // Invalid high_pri_pool_ratio + return nullptr; + } + if (low_pri_pool_ratio < 0.0 || low_pri_pool_ratio > 1.0) { + // Invalid low_pri_pool_ratio + return nullptr; + } + if (low_pri_pool_ratio + high_pri_pool_ratio > 1.0) { + // Invalid high_pri_pool_ratio and low_pri_pool_ratio combination + return nullptr; + } + // For sanitized options + LRUCacheOptions opts = *this; + if (opts.num_shard_bits < 0) { + opts.num_shard_bits = GetDefaultCacheShardBits(capacity); + } + std::shared_ptr cache = std::make_shared(opts); + if (secondary_cache) { + cache = std::make_shared(cache, secondary_cache); + } + return cache; +} + +std::shared_ptr LRUCacheOptions::MakeSharedRowCache() const { + if (secondary_cache) { + // Not allowed for a RowCache + return nullptr; + } + // Works while RowCache is an alias for Cache + return MakeSharedCache(); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/lru_cache.h b/librocksdb-sys/rocksdb/cache/lru_cache.h new file mode 100644 index 0000000..1a9ba04 --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/lru_cache.h @@ -0,0 +1,467 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once + +#include +#include + +#include "cache/sharded_cache.h" +#include "port/lang.h" +#include "port/likely.h" +#include "port/malloc.h" +#include "port/port.h" +#include "util/autovector.h" +#include "util/distributed_mutex.h" + +namespace ROCKSDB_NAMESPACE { +namespace lru_cache { + +// LRU cache implementation. This class is not thread-safe. + +// An entry is a variable length heap-allocated structure. +// Entries are referenced by cache and/or by any external entity. +// The cache keeps all its entries in a hash table. Some elements +// are also stored on LRU list. +// +// LRUHandle can be in these states: +// 1. Referenced externally AND in hash table. +// In that case the entry is *not* in the LRU list +// (refs >= 1 && in_cache == true) +// 2. Not referenced externally AND in hash table. +// In that case the entry is in the LRU list and can be freed. +// (refs == 0 && in_cache == true) +// 3. Referenced externally AND not in hash table. +// In that case the entry is not in the LRU list and not in hash table. +// The entry must be freed if refs becomes 0 in this state. +// (refs >= 1 && in_cache == false) +// If you call LRUCacheShard::Release enough times on an entry in state 1, it +// will go into state 2. To move from state 1 to state 3, either call +// LRUCacheShard::Erase or LRUCacheShard::Insert with the same key (but +// possibly different value). To move from state 2 to state 1, use +// LRUCacheShard::Lookup. +// While refs > 0, public properties like value and deleter must not change. + +struct LRUHandle { + Cache::ObjectPtr value; + const Cache::CacheItemHelper* helper; + LRUHandle* next_hash; + LRUHandle* next; + LRUHandle* prev; + size_t total_charge; // TODO(opt): Only allow uint32_t? + size_t key_length; + // The hash of key(). Used for fast sharding and comparisons. + uint32_t hash; + // The number of external refs to this entry. The cache itself is not counted. + uint32_t refs; + + // Mutable flags - access controlled by mutex + // The m_ and M_ prefixes (and im_ and IM_ later) are to hopefully avoid + // checking an M_ flag on im_flags or an IM_ flag on m_flags. + uint8_t m_flags; + enum MFlags : uint8_t { + // Whether this entry is referenced by the hash table. + M_IN_CACHE = (1 << 0), + // Whether this entry has had any lookups (hits). + M_HAS_HIT = (1 << 1), + // Whether this entry is in high-pri pool. + M_IN_HIGH_PRI_POOL = (1 << 2), + // Whether this entry is in low-pri pool. + M_IN_LOW_PRI_POOL = (1 << 3), + }; + + // "Immutable" flags - only set in single-threaded context and then + // can be accessed without mutex + uint8_t im_flags; + enum ImFlags : uint8_t { + // Whether this entry is high priority entry. + IM_IS_HIGH_PRI = (1 << 0), + // Whether this entry is low priority entry. + IM_IS_LOW_PRI = (1 << 1), + // Marks result handles that should not be inserted into cache + IM_IS_STANDALONE = (1 << 2), + }; + + // Beginning of the key (MUST BE THE LAST FIELD IN THIS STRUCT!) + char key_data[1]; + + Slice key() const { return Slice(key_data, key_length); } + + // For HandleImpl concept + uint32_t GetHash() const { return hash; } + + // Increase the reference count by 1. + void Ref() { refs++; } + + // Just reduce the reference count by 1. Return true if it was last reference. + bool Unref() { + assert(refs > 0); + refs--; + return refs == 0; + } + + // Return true if there are external refs, false otherwise. + bool HasRefs() const { return refs > 0; } + + bool InCache() const { return m_flags & M_IN_CACHE; } + bool IsHighPri() const { return im_flags & IM_IS_HIGH_PRI; } + bool InHighPriPool() const { return m_flags & M_IN_HIGH_PRI_POOL; } + bool IsLowPri() const { return im_flags & IM_IS_LOW_PRI; } + bool InLowPriPool() const { return m_flags & M_IN_LOW_PRI_POOL; } + bool HasHit() const { return m_flags & M_HAS_HIT; } + bool IsStandalone() const { return im_flags & IM_IS_STANDALONE; } + + void SetInCache(bool in_cache) { + if (in_cache) { + m_flags |= M_IN_CACHE; + } else { + m_flags &= ~M_IN_CACHE; + } + } + + void SetPriority(Cache::Priority priority) { + if (priority == Cache::Priority::HIGH) { + im_flags |= IM_IS_HIGH_PRI; + im_flags &= ~IM_IS_LOW_PRI; + } else if (priority == Cache::Priority::LOW) { + im_flags &= ~IM_IS_HIGH_PRI; + im_flags |= IM_IS_LOW_PRI; + } else { + im_flags &= ~IM_IS_HIGH_PRI; + im_flags &= ~IM_IS_LOW_PRI; + } + } + + void SetInHighPriPool(bool in_high_pri_pool) { + if (in_high_pri_pool) { + m_flags |= M_IN_HIGH_PRI_POOL; + } else { + m_flags &= ~M_IN_HIGH_PRI_POOL; + } + } + + void SetInLowPriPool(bool in_low_pri_pool) { + if (in_low_pri_pool) { + m_flags |= M_IN_LOW_PRI_POOL; + } else { + m_flags &= ~M_IN_LOW_PRI_POOL; + } + } + + void SetHit() { m_flags |= M_HAS_HIT; } + + void SetIsStandalone(bool is_standalone) { + if (is_standalone) { + im_flags |= IM_IS_STANDALONE; + } else { + im_flags &= ~IM_IS_STANDALONE; + } + } + + void Free(MemoryAllocator* allocator) { + assert(refs == 0); + assert(helper); + if (helper->del_cb) { + helper->del_cb(value, allocator); + } + + free(this); + } + + inline size_t CalcuMetaCharge( + CacheMetadataChargePolicy metadata_charge_policy) const { + if (metadata_charge_policy != kFullChargeCacheMetadata) { + return 0; + } else { +#ifdef ROCKSDB_MALLOC_USABLE_SIZE + return malloc_usable_size( + const_cast(static_cast(this))); +#else + // This is the size that is used when a new handle is created. + return sizeof(LRUHandle) - 1 + key_length; +#endif + } + } + + // Calculate the memory usage by metadata. + inline void CalcTotalCharge( + size_t charge, CacheMetadataChargePolicy metadata_charge_policy) { + total_charge = charge + CalcuMetaCharge(metadata_charge_policy); + } + + inline size_t GetCharge( + CacheMetadataChargePolicy metadata_charge_policy) const { + size_t meta_charge = CalcuMetaCharge(metadata_charge_policy); + assert(total_charge >= meta_charge); + return total_charge - meta_charge; + } +}; + +// We provide our own simple hash table since it removes a whole bunch +// of porting hacks and is also faster than some of the built-in hash +// table implementations in some of the compiler/runtime combinations +// we have tested. E.g., readrandom speeds up by ~5% over the g++ +// 4.4.3's builtin hashtable. +class LRUHandleTable { + public: + explicit LRUHandleTable(int max_upper_hash_bits, MemoryAllocator* allocator); + ~LRUHandleTable(); + + LRUHandle* Lookup(const Slice& key, uint32_t hash); + LRUHandle* Insert(LRUHandle* h); + LRUHandle* Remove(const Slice& key, uint32_t hash); + + template + void ApplyToEntriesRange(T func, size_t index_begin, size_t index_end) { + for (size_t i = index_begin; i < index_end; i++) { + LRUHandle* h = list_[i]; + while (h != nullptr) { + auto n = h->next_hash; + assert(h->InCache()); + func(h); + h = n; + } + } + } + + int GetLengthBits() const { return length_bits_; } + + size_t GetOccupancyCount() const { return elems_; } + + MemoryAllocator* GetAllocator() const { return allocator_; } + + private: + // Return a pointer to slot that points to a cache entry that + // matches key/hash. If there is no such cache entry, return a + // pointer to the trailing slot in the corresponding linked list. + LRUHandle** FindPointer(const Slice& key, uint32_t hash); + + void Resize(); + + // Number of hash bits (upper because lower bits used for sharding) + // used for table index. Length == 1 << length_bits_ + int length_bits_; + + // The table consists of an array of buckets where each bucket is + // a linked list of cache entries that hash into the bucket. + std::unique_ptr list_; + + // Number of elements currently in the table. + uint32_t elems_; + + // Set from max_upper_hash_bits (see constructor). + const int max_length_bits_; + + // From Cache, needed for delete + MemoryAllocator* const allocator_; +}; + +// A single shard of sharded cache. +class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShardBase { + public: + // NOTE: the eviction_callback ptr is saved, as is it assumed to be kept + // alive in Cache. + LRUCacheShard(size_t capacity, bool strict_capacity_limit, + double high_pri_pool_ratio, double low_pri_pool_ratio, + bool use_adaptive_mutex, + CacheMetadataChargePolicy metadata_charge_policy, + int max_upper_hash_bits, MemoryAllocator* allocator, + const Cache::EvictionCallback* eviction_callback); + + public: // Type definitions expected as parameter to ShardedCache + using HandleImpl = LRUHandle; + using HashVal = uint32_t; + using HashCref = uint32_t; + + public: // Function definitions expected as parameter to ShardedCache + static inline HashVal ComputeHash(const Slice& key, uint32_t seed) { + return Lower32of64(GetSliceNPHash64(key, seed)); + } + + // Separate from constructor so caller can easily make an array of LRUCache + // if current usage is more than new capacity, the function will attempt to + // free the needed space. + void SetCapacity(size_t capacity); + + // Set the flag to reject insertion if cache if full. + void SetStrictCapacityLimit(bool strict_capacity_limit); + + // Set percentage of capacity reserved for high-pri cache entries. + void SetHighPriorityPoolRatio(double high_pri_pool_ratio); + + // Set percentage of capacity reserved for low-pri cache entries. + void SetLowPriorityPoolRatio(double low_pri_pool_ratio); + + // Like Cache methods, but with an extra "hash" parameter. + Status Insert(const Slice& key, uint32_t hash, Cache::ObjectPtr value, + const Cache::CacheItemHelper* helper, size_t charge, + LRUHandle** handle, Cache::Priority priority); + + LRUHandle* CreateStandalone(const Slice& key, uint32_t hash, + Cache::ObjectPtr obj, + const Cache::CacheItemHelper* helper, + size_t charge, bool allow_uncharged); + + LRUHandle* Lookup(const Slice& key, uint32_t hash, + const Cache::CacheItemHelper* helper, + Cache::CreateContext* create_context, + Cache::Priority priority, Statistics* stats); + + bool Release(LRUHandle* handle, bool useful, bool erase_if_last_ref); + bool Ref(LRUHandle* handle); + void Erase(const Slice& key, uint32_t hash); + + // Although in some platforms the update of size_t is atomic, to make sure + // GetUsage() and GetPinnedUsage() work correctly under any platform, we'll + // protect them with mutex_. + + size_t GetUsage() const; + size_t GetPinnedUsage() const; + size_t GetOccupancyCount() const; + size_t GetTableAddressCount() const; + + void ApplyToSomeEntries( + const std::function& callback, + size_t average_entries_per_lock, size_t* state); + + void EraseUnRefEntries(); + + public: // other function definitions + void TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri, + LRUHandle** lru_bottom_pri); + + // Retrieves number of elements in LRU, for unit test purpose only. + // Not threadsafe. + size_t TEST_GetLRUSize(); + + // Retrieves high pri pool ratio + double GetHighPriPoolRatio(); + + // Retrieves low pri pool ratio + double GetLowPriPoolRatio(); + + void AppendPrintableOptions(std::string& /*str*/) const; + + private: + friend class LRUCache; + // Insert an item into the hash table and, if handle is null, insert into + // the LRU list. Older items are evicted as necessary. Frees `item` on + // non-OK status. + Status InsertItem(LRUHandle* item, LRUHandle** handle); + + void LRU_Remove(LRUHandle* e); + void LRU_Insert(LRUHandle* e); + + // Overflow the last entry in high-pri pool to low-pri pool until size of + // high-pri pool is no larger than the size specify by high_pri_pool_pct. + void MaintainPoolSize(); + + // Free some space following strict LRU policy until enough space + // to hold (usage_ + charge) is freed or the lru list is empty + // This function is not thread safe - it needs to be executed while + // holding the mutex_. + void EvictFromLRU(size_t charge, autovector* deleted); + + void NotifyEvicted(const autovector& evicted_handles); + + LRUHandle* CreateHandle(const Slice& key, uint32_t hash, + Cache::ObjectPtr value, + const Cache::CacheItemHelper* helper, size_t charge); + + // Initialized before use. + size_t capacity_; + + // Memory size for entries in high-pri pool. + size_t high_pri_pool_usage_; + + // Memory size for entries in low-pri pool. + size_t low_pri_pool_usage_; + + // Whether to reject insertion if cache reaches its full capacity. + bool strict_capacity_limit_; + + // Ratio of capacity reserved for high priority cache entries. + double high_pri_pool_ratio_; + + // High-pri pool size, equals to capacity * high_pri_pool_ratio. + // Remember the value to avoid recomputing each time. + double high_pri_pool_capacity_; + + // Ratio of capacity reserved for low priority cache entries. + double low_pri_pool_ratio_; + + // Low-pri pool size, equals to capacity * low_pri_pool_ratio. + // Remember the value to avoid recomputing each time. + double low_pri_pool_capacity_; + + // Dummy head of LRU list. + // lru.prev is newest entry, lru.next is oldest entry. + // LRU contains items which can be evicted, ie reference only by cache + LRUHandle lru_; + + // Pointer to head of low-pri pool in LRU list. + LRUHandle* lru_low_pri_; + + // Pointer to head of bottom-pri pool in LRU list. + LRUHandle* lru_bottom_pri_; + + // ------------^^^^^^^^^^^^^----------- + // Not frequently modified data members + // ------------------------------------ + // + // We separate data members that are updated frequently from the ones that + // are not frequently updated so that they don't share the same cache line + // which will lead into false cache sharing + // + // ------------------------------------ + // Frequently modified data members + // ------------vvvvvvvvvvvvv----------- + LRUHandleTable table_; + + // Memory size for entries residing in the cache. + size_t usage_; + + // Memory size for entries residing only in the LRU list. + size_t lru_usage_; + + // mutex_ protects the following state. + // We don't count mutex_ as the cache's internal state so semantically we + // don't mind mutex_ invoking the non-const actions. + mutable DMutex mutex_; + + // A reference to Cache::eviction_callback_ + const Cache::EvictionCallback& eviction_callback_; +}; + +class LRUCache +#ifdef NDEBUG + final +#endif + : public ShardedCache { + public: + explicit LRUCache(const LRUCacheOptions& opts); + const char* Name() const override { return "LRUCache"; } + ObjectPtr Value(Handle* handle) override; + size_t GetCharge(Handle* handle) const override; + const CacheItemHelper* GetCacheItemHelper(Handle* handle) const override; + + // Retrieves number of elements in LRU, for unit test purpose only. + size_t TEST_GetLRUSize(); + // Retrieves high pri pool ratio. + double GetHighPriPoolRatio(); +}; + +} // namespace lru_cache + +using LRUCache = lru_cache::LRUCache; +using LRUHandle = lru_cache::LRUHandle; +using LRUCacheShard = lru_cache::LRUCacheShard; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/lru_cache_test.cc b/librocksdb-sys/rocksdb/cache/lru_cache_test.cc new file mode 100644 index 0000000..cb7beb7 --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/lru_cache_test.cc @@ -0,0 +1,2615 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "cache/lru_cache.h" + +#include +#include +#include + +#include "cache/cache_key.h" +#include "cache/clock_cache.h" +#include "cache_helpers.h" +#include "db/db_test_util.h" +#include "file/sst_file_manager_impl.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/cache.h" +#include "rocksdb/io_status.h" +#include "rocksdb/sst_file_manager.h" +#include "rocksdb/utilities/cache_dump_load.h" +#include "test_util/secondary_cache_test_util.h" +#include "test_util/testharness.h" +#include "typed_cache.h" +#include "util/coding.h" +#include "util/random.h" +#include "utilities/cache_dump_load_impl.h" +#include "utilities/fault_injection_fs.h" + +namespace ROCKSDB_NAMESPACE { + +class LRUCacheTest : public testing::Test { + public: + LRUCacheTest() {} + ~LRUCacheTest() override { DeleteCache(); } + + void DeleteCache() { + if (cache_ != nullptr) { + cache_->~LRUCacheShard(); + port::cacheline_aligned_free(cache_); + cache_ = nullptr; + } + } + + void NewCache(size_t capacity, double high_pri_pool_ratio = 0.0, + double low_pri_pool_ratio = 1.0, + bool use_adaptive_mutex = kDefaultToAdaptiveMutex) { + DeleteCache(); + cache_ = reinterpret_cast( + port::cacheline_aligned_alloc(sizeof(LRUCacheShard))); + new (cache_) LRUCacheShard(capacity, /*strict_capacity_limit=*/false, + high_pri_pool_ratio, low_pri_pool_ratio, + use_adaptive_mutex, kDontChargeCacheMetadata, + /*max_upper_hash_bits=*/24, + /*allocator*/ nullptr, &eviction_callback_); + } + + void Insert(const std::string& key, + Cache::Priority priority = Cache::Priority::LOW) { + EXPECT_OK(cache_->Insert(key, 0 /*hash*/, nullptr /*value*/, + &kNoopCacheItemHelper, 1 /*charge*/, + nullptr /*handle*/, priority)); + } + + void Insert(char key, Cache::Priority priority = Cache::Priority::LOW) { + Insert(std::string(1, key), priority); + } + + bool Lookup(const std::string& key) { + auto handle = cache_->Lookup(key, 0 /*hash*/, nullptr, nullptr, + Cache::Priority::LOW, nullptr); + if (handle) { + cache_->Release(handle, true /*useful*/, false /*erase*/); + return true; + } + return false; + } + + bool Lookup(char key) { return Lookup(std::string(1, key)); } + + void Erase(const std::string& key) { cache_->Erase(key, 0 /*hash*/); } + + void ValidateLRUList(std::vector keys, + size_t num_high_pri_pool_keys = 0, + size_t num_low_pri_pool_keys = 0, + size_t num_bottom_pri_pool_keys = 0) { + LRUHandle* lru; + LRUHandle* lru_low_pri; + LRUHandle* lru_bottom_pri; + cache_->TEST_GetLRUList(&lru, &lru_low_pri, &lru_bottom_pri); + + LRUHandle* iter = lru; + + bool in_low_pri_pool = false; + bool in_high_pri_pool = false; + + size_t high_pri_pool_keys = 0; + size_t low_pri_pool_keys = 0; + size_t bottom_pri_pool_keys = 0; + + if (iter == lru_bottom_pri) { + in_low_pri_pool = true; + in_high_pri_pool = false; + } + if (iter == lru_low_pri) { + in_low_pri_pool = false; + in_high_pri_pool = true; + } + + for (const auto& key : keys) { + iter = iter->next; + ASSERT_NE(lru, iter); + ASSERT_EQ(key, iter->key().ToString()); + ASSERT_EQ(in_high_pri_pool, iter->InHighPriPool()); + ASSERT_EQ(in_low_pri_pool, iter->InLowPriPool()); + if (in_high_pri_pool) { + ASSERT_FALSE(iter->InLowPriPool()); + high_pri_pool_keys++; + } else if (in_low_pri_pool) { + ASSERT_FALSE(iter->InHighPriPool()); + low_pri_pool_keys++; + } else { + bottom_pri_pool_keys++; + } + if (iter == lru_bottom_pri) { + ASSERT_FALSE(in_low_pri_pool); + ASSERT_FALSE(in_high_pri_pool); + in_low_pri_pool = true; + in_high_pri_pool = false; + } + if (iter == lru_low_pri) { + ASSERT_TRUE(in_low_pri_pool); + ASSERT_FALSE(in_high_pri_pool); + in_low_pri_pool = false; + in_high_pri_pool = true; + } + } + ASSERT_EQ(lru, iter->next); + ASSERT_FALSE(in_low_pri_pool); + ASSERT_TRUE(in_high_pri_pool); + ASSERT_EQ(num_high_pri_pool_keys, high_pri_pool_keys); + ASSERT_EQ(num_low_pri_pool_keys, low_pri_pool_keys); + ASSERT_EQ(num_bottom_pri_pool_keys, bottom_pri_pool_keys); + } + + private: + LRUCacheShard* cache_ = nullptr; + Cache::EvictionCallback eviction_callback_; +}; + +TEST_F(LRUCacheTest, BasicLRU) { + NewCache(5); + for (char ch = 'a'; ch <= 'e'; ch++) { + Insert(ch); + } + ValidateLRUList({"a", "b", "c", "d", "e"}, 0, 5); + for (char ch = 'x'; ch <= 'z'; ch++) { + Insert(ch); + } + ValidateLRUList({"d", "e", "x", "y", "z"}, 0, 5); + ASSERT_FALSE(Lookup("b")); + ValidateLRUList({"d", "e", "x", "y", "z"}, 0, 5); + ASSERT_TRUE(Lookup("e")); + ValidateLRUList({"d", "x", "y", "z", "e"}, 0, 5); + ASSERT_TRUE(Lookup("z")); + ValidateLRUList({"d", "x", "y", "e", "z"}, 0, 5); + Erase("x"); + ValidateLRUList({"d", "y", "e", "z"}, 0, 4); + ASSERT_TRUE(Lookup("d")); + ValidateLRUList({"y", "e", "z", "d"}, 0, 4); + Insert("u"); + ValidateLRUList({"y", "e", "z", "d", "u"}, 0, 5); + Insert("v"); + ValidateLRUList({"e", "z", "d", "u", "v"}, 0, 5); +} + +TEST_F(LRUCacheTest, LowPriorityMidpointInsertion) { + // Allocate 2 cache entries to high-pri pool and 3 to low-pri pool. + NewCache(5, /* high_pri_pool_ratio */ 0.40, /* low_pri_pool_ratio */ 0.60); + + Insert("a", Cache::Priority::LOW); + Insert("b", Cache::Priority::LOW); + Insert("c", Cache::Priority::LOW); + Insert("x", Cache::Priority::HIGH); + Insert("y", Cache::Priority::HIGH); + ValidateLRUList({"a", "b", "c", "x", "y"}, 2, 3); + + // Low-pri entries inserted to the tail of low-pri list (the midpoint). + // After lookup, it will move to the tail of the full list. + Insert("d", Cache::Priority::LOW); + ValidateLRUList({"b", "c", "d", "x", "y"}, 2, 3); + ASSERT_TRUE(Lookup("d")); + ValidateLRUList({"b", "c", "x", "y", "d"}, 2, 3); + + // High-pri entries will be inserted to the tail of full list. + Insert("z", Cache::Priority::HIGH); + ValidateLRUList({"c", "x", "y", "d", "z"}, 2, 3); +} + +TEST_F(LRUCacheTest, BottomPriorityMidpointInsertion) { + // Allocate 2 cache entries to high-pri pool and 2 to low-pri pool. + NewCache(6, /* high_pri_pool_ratio */ 0.35, /* low_pri_pool_ratio */ 0.35); + + Insert("a", Cache::Priority::BOTTOM); + Insert("b", Cache::Priority::BOTTOM); + Insert("i", Cache::Priority::LOW); + Insert("j", Cache::Priority::LOW); + Insert("x", Cache::Priority::HIGH); + Insert("y", Cache::Priority::HIGH); + ValidateLRUList({"a", "b", "i", "j", "x", "y"}, 2, 2, 2); + + // Low-pri entries will be inserted to the tail of low-pri list (the + // midpoint). After lookup, 'k' will move to the tail of the full list, and + // 'x' will spill over to the low-pri pool. + Insert("k", Cache::Priority::LOW); + ValidateLRUList({"b", "i", "j", "k", "x", "y"}, 2, 2, 2); + ASSERT_TRUE(Lookup("k")); + ValidateLRUList({"b", "i", "j", "x", "y", "k"}, 2, 2, 2); + + // High-pri entries will be inserted to the tail of full list. Although y was + // inserted with high priority, it got spilled over to the low-pri pool. As + // a result, j also got spilled over to the bottom-pri pool. + Insert("z", Cache::Priority::HIGH); + ValidateLRUList({"i", "j", "x", "y", "k", "z"}, 2, 2, 2); + Erase("x"); + ValidateLRUList({"i", "j", "y", "k", "z"}, 2, 1, 2); + Erase("y"); + ValidateLRUList({"i", "j", "k", "z"}, 2, 0, 2); + + // Bottom-pri entries will be inserted to the tail of bottom-pri list. + Insert("c", Cache::Priority::BOTTOM); + ValidateLRUList({"i", "j", "c", "k", "z"}, 2, 0, 3); + Insert("d", Cache::Priority::BOTTOM); + ValidateLRUList({"i", "j", "c", "d", "k", "z"}, 2, 0, 4); + Insert("e", Cache::Priority::BOTTOM); + ValidateLRUList({"j", "c", "d", "e", "k", "z"}, 2, 0, 4); + + // Low-pri entries will be inserted to the tail of low-pri list (the + // midpoint). + Insert("l", Cache::Priority::LOW); + ValidateLRUList({"c", "d", "e", "l", "k", "z"}, 2, 1, 3); + Insert("m", Cache::Priority::LOW); + ValidateLRUList({"d", "e", "l", "m", "k", "z"}, 2, 2, 2); + + Erase("k"); + ValidateLRUList({"d", "e", "l", "m", "z"}, 1, 2, 2); + Erase("z"); + ValidateLRUList({"d", "e", "l", "m"}, 0, 2, 2); + + // Bottom-pri entries will be inserted to the tail of bottom-pri list. + Insert("f", Cache::Priority::BOTTOM); + ValidateLRUList({"d", "e", "f", "l", "m"}, 0, 2, 3); + Insert("g", Cache::Priority::BOTTOM); + ValidateLRUList({"d", "e", "f", "g", "l", "m"}, 0, 2, 4); + + // High-pri entries will be inserted to the tail of full list. + Insert("o", Cache::Priority::HIGH); + ValidateLRUList({"e", "f", "g", "l", "m", "o"}, 1, 2, 3); + Insert("p", Cache::Priority::HIGH); + ValidateLRUList({"f", "g", "l", "m", "o", "p"}, 2, 2, 2); +} + +TEST_F(LRUCacheTest, EntriesWithPriority) { + // Allocate 2 cache entries to high-pri pool and 2 to low-pri pool. + NewCache(6, /* high_pri_pool_ratio */ 0.35, /* low_pri_pool_ratio */ 0.35); + + Insert("a", Cache::Priority::LOW); + Insert("b", Cache::Priority::LOW); + ValidateLRUList({"a", "b"}, 0, 2, 0); + // Low-pri entries can overflow to bottom-pri pool. + Insert("c", Cache::Priority::LOW); + ValidateLRUList({"a", "b", "c"}, 0, 2, 1); + + // Bottom-pri entries can take high-pri pool capacity if available + Insert("t", Cache::Priority::LOW); + Insert("u", Cache::Priority::LOW); + ValidateLRUList({"a", "b", "c", "t", "u"}, 0, 2, 3); + Insert("v", Cache::Priority::LOW); + ValidateLRUList({"a", "b", "c", "t", "u", "v"}, 0, 2, 4); + Insert("w", Cache::Priority::LOW); + ValidateLRUList({"b", "c", "t", "u", "v", "w"}, 0, 2, 4); + + Insert("X", Cache::Priority::HIGH); + Insert("Y", Cache::Priority::HIGH); + ValidateLRUList({"t", "u", "v", "w", "X", "Y"}, 2, 2, 2); + + // After lookup, the high-pri entry 'X' got spilled over to the low-pri pool. + // The low-pri entry 'v' got spilled over to the bottom-pri pool. + Insert("Z", Cache::Priority::HIGH); + ValidateLRUList({"u", "v", "w", "X", "Y", "Z"}, 2, 2, 2); + + // Low-pri entries will be inserted to head of low-pri pool. + Insert("a", Cache::Priority::LOW); + ValidateLRUList({"v", "w", "X", "a", "Y", "Z"}, 2, 2, 2); + + // After lookup, the high-pri entry 'Y' got spilled over to the low-pri pool. + // The low-pri entry 'X' got spilled over to the bottom-pri pool. + ASSERT_TRUE(Lookup("v")); + ValidateLRUList({"w", "X", "a", "Y", "Z", "v"}, 2, 2, 2); + + // After lookup, the high-pri entry 'Z' got spilled over to the low-pri pool. + // The low-pri entry 'a' got spilled over to the bottom-pri pool. + ASSERT_TRUE(Lookup("X")); + ValidateLRUList({"w", "a", "Y", "Z", "v", "X"}, 2, 2, 2); + + // After lookup, the low pri entry 'Z' got promoted back to high-pri pool. The + // high-pri entry 'v' got spilled over to the low-pri pool. + ASSERT_TRUE(Lookup("Z")); + ValidateLRUList({"w", "a", "Y", "v", "X", "Z"}, 2, 2, 2); + + Erase("Y"); + ValidateLRUList({"w", "a", "v", "X", "Z"}, 2, 1, 2); + Erase("X"); + ValidateLRUList({"w", "a", "v", "Z"}, 1, 1, 2); + + Insert("d", Cache::Priority::LOW); + Insert("e", Cache::Priority::LOW); + ValidateLRUList({"w", "a", "v", "d", "e", "Z"}, 1, 2, 3); + + Insert("f", Cache::Priority::LOW); + Insert("g", Cache::Priority::LOW); + ValidateLRUList({"v", "d", "e", "f", "g", "Z"}, 1, 2, 3); + ASSERT_TRUE(Lookup("d")); + ValidateLRUList({"v", "e", "f", "g", "Z", "d"}, 2, 2, 2); + + // Erase some entries. + Erase("e"); + Erase("f"); + Erase("Z"); + ValidateLRUList({"v", "g", "d"}, 1, 1, 1); + + // Bottom-pri entries can take low- and high-pri pool capacity if available + Insert("o", Cache::Priority::BOTTOM); + ValidateLRUList({"v", "o", "g", "d"}, 1, 1, 2); + Insert("p", Cache::Priority::BOTTOM); + ValidateLRUList({"v", "o", "p", "g", "d"}, 1, 1, 3); + Insert("q", Cache::Priority::BOTTOM); + ValidateLRUList({"v", "o", "p", "q", "g", "d"}, 1, 1, 4); + + // High-pri entries can overflow to low-pri pool, and bottom-pri entries will + // be evicted. + Insert("x", Cache::Priority::HIGH); + ValidateLRUList({"o", "p", "q", "g", "d", "x"}, 2, 1, 3); + Insert("y", Cache::Priority::HIGH); + ValidateLRUList({"p", "q", "g", "d", "x", "y"}, 2, 2, 2); + Insert("z", Cache::Priority::HIGH); + ValidateLRUList({"q", "g", "d", "x", "y", "z"}, 2, 2, 2); + + // 'g' is bottom-pri before this lookup, it will be inserted to head of + // high-pri pool after lookup. + ASSERT_TRUE(Lookup("g")); + ValidateLRUList({"q", "d", "x", "y", "z", "g"}, 2, 2, 2); + + // High-pri entries will be inserted to head of high-pri pool after lookup. + ASSERT_TRUE(Lookup("z")); + ValidateLRUList({"q", "d", "x", "y", "g", "z"}, 2, 2, 2); + + // Bottom-pri entries will be inserted to head of high-pri pool after lookup. + ASSERT_TRUE(Lookup("d")); + ValidateLRUList({"q", "x", "y", "g", "z", "d"}, 2, 2, 2); + + // Bottom-pri entries will be inserted to the tail of bottom-pri list. + Insert("m", Cache::Priority::BOTTOM); + ValidateLRUList({"x", "m", "y", "g", "z", "d"}, 2, 2, 2); + + // Bottom-pri entries will be inserted to head of high-pri pool after lookup. + ASSERT_TRUE(Lookup("m")); + ValidateLRUList({"x", "y", "g", "z", "d", "m"}, 2, 2, 2); +} + +namespace clock_cache { + +class ClockCacheTest : public testing::Test { + public: + using Shard = HyperClockCache::Shard; + using Table = HyperClockTable; + using HandleImpl = Shard::HandleImpl; + + ClockCacheTest() {} + ~ClockCacheTest() override { DeleteShard(); } + + void DeleteShard() { + if (shard_ != nullptr) { + shard_->~ClockCacheShard(); + port::cacheline_aligned_free(shard_); + shard_ = nullptr; + } + } + + void NewShard(size_t capacity, bool strict_capacity_limit = true) { + DeleteShard(); + shard_ = + reinterpret_cast(port::cacheline_aligned_alloc(sizeof(Shard))); + + Table::Opts opts; + opts.estimated_value_size = 1; + new (shard_) + Shard(capacity, strict_capacity_limit, kDontChargeCacheMetadata, + /*allocator*/ nullptr, &eviction_callback_, &hash_seed_, opts); + } + + Status Insert(const UniqueId64x2& hashed_key, + Cache::Priority priority = Cache::Priority::LOW) { + return shard_->Insert(TestKey(hashed_key), hashed_key, nullptr /*value*/, + &kNoopCacheItemHelper, 1 /*charge*/, + nullptr /*handle*/, priority); + } + + Status Insert(char key, Cache::Priority priority = Cache::Priority::LOW) { + return Insert(TestHashedKey(key), priority); + } + + Status InsertWithLen(char key, size_t len) { + std::string skey(len, key); + return shard_->Insert(skey, TestHashedKey(key), nullptr /*value*/, + &kNoopCacheItemHelper, 1 /*charge*/, + nullptr /*handle*/, Cache::Priority::LOW); + } + + bool Lookup(const Slice& key, const UniqueId64x2& hashed_key, + bool useful = true) { + auto handle = shard_->Lookup(key, hashed_key); + if (handle) { + shard_->Release(handle, useful, /*erase_if_last_ref=*/false); + return true; + } + return false; + } + + bool Lookup(const UniqueId64x2& hashed_key, bool useful = true) { + return Lookup(TestKey(hashed_key), hashed_key, useful); + } + + bool Lookup(char key, bool useful = true) { + return Lookup(TestHashedKey(key), useful); + } + + void Erase(char key) { + UniqueId64x2 hashed_key = TestHashedKey(key); + shard_->Erase(TestKey(hashed_key), hashed_key); + } + + static inline Slice TestKey(const UniqueId64x2& hashed_key) { + return Slice(reinterpret_cast(&hashed_key), 16U); + } + + static inline UniqueId64x2 TestHashedKey(char key) { + // For testing hash near-collision behavior, put the variance in + // hashed_key in bits that are unlikely to be used as hash bits. + return {(static_cast(key) << 56) + 1234U, 5678U}; + } + + Shard* shard_ = nullptr; + + private: + Cache::EvictionCallback eviction_callback_; + uint32_t hash_seed_ = 0; +}; + +TEST_F(ClockCacheTest, Misc) { + NewShard(3); + + // Key size stuff + EXPECT_OK(InsertWithLen('a', 16)); + EXPECT_NOK(InsertWithLen('b', 15)); + EXPECT_OK(InsertWithLen('b', 16)); + EXPECT_NOK(InsertWithLen('c', 17)); + EXPECT_NOK(InsertWithLen('d', 1000)); + EXPECT_NOK(InsertWithLen('e', 11)); + EXPECT_NOK(InsertWithLen('f', 0)); + + // Some of this is motivated by code coverage + std::string wrong_size_key(15, 'x'); + EXPECT_FALSE(Lookup(wrong_size_key, TestHashedKey('x'))); + EXPECT_FALSE(shard_->Ref(nullptr)); + EXPECT_FALSE(shard_->Release(nullptr)); + shard_->Erase(wrong_size_key, TestHashedKey('x')); // no-op +} + +TEST_F(ClockCacheTest, Limits) { + constexpr size_t kCapacity = 3; + NewShard(kCapacity, false /*strict_capacity_limit*/); + for (bool strict_capacity_limit : {false, true, false}) { + SCOPED_TRACE("strict_capacity_limit = " + + std::to_string(strict_capacity_limit)); + + // Also tests switching between strict limit and not + shard_->SetStrictCapacityLimit(strict_capacity_limit); + + UniqueId64x2 hkey = TestHashedKey('x'); + + // Single entry charge beyond capacity + { + Status s = shard_->Insert(TestKey(hkey), hkey, nullptr /*value*/, + &kNoopCacheItemHelper, 5 /*charge*/, + nullptr /*handle*/, Cache::Priority::LOW); + if (strict_capacity_limit) { + EXPECT_TRUE(s.IsMemoryLimit()); + } else { + EXPECT_OK(s); + } + } + + // Single entry fills capacity + { + HandleImpl* h; + ASSERT_OK(shard_->Insert(TestKey(hkey), hkey, nullptr /*value*/, + &kNoopCacheItemHelper, 3 /*charge*/, &h, + Cache::Priority::LOW)); + // Try to insert more + Status s = Insert('a'); + if (strict_capacity_limit) { + EXPECT_TRUE(s.IsMemoryLimit()); + } else { + EXPECT_OK(s); + } + // Release entry filling capacity. + // Cover useful = false case. + shard_->Release(h, false /*useful*/, false /*erase_if_last_ref*/); + } + + // Insert more than table size can handle to exceed occupancy limit. + // (Cleverly using mostly zero-charge entries, but some non-zero to + // verify usage tracking on detached entries.) + { + size_t n = shard_->GetTableAddressCount() + 1; + std::unique_ptr ha { new HandleImpl* [n] {} }; + Status s; + for (size_t i = 0; i < n && s.ok(); ++i) { + hkey[1] = i; + s = shard_->Insert(TestKey(hkey), hkey, nullptr /*value*/, + &kNoopCacheItemHelper, + (i + kCapacity < n) ? 0 : 1 /*charge*/, &ha[i], + Cache::Priority::LOW); + if (i == 0) { + EXPECT_OK(s); + } + } + if (strict_capacity_limit) { + EXPECT_TRUE(s.IsMemoryLimit()); + } else { + EXPECT_OK(s); + } + // Same result if not keeping a reference + s = Insert('a'); + if (strict_capacity_limit) { + EXPECT_TRUE(s.IsMemoryLimit()); + } else { + EXPECT_OK(s); + } + + // Regardless, we didn't allow table to actually get full + EXPECT_LT(shard_->GetOccupancyCount(), shard_->GetTableAddressCount()); + + // Release handles + for (size_t i = 0; i < n; ++i) { + if (ha[i]) { + shard_->Release(ha[i]); + } + } + } + } +} + +TEST_F(ClockCacheTest, ClockEvictionTest) { + for (bool strict_capacity_limit : {false, true}) { + SCOPED_TRACE("strict_capacity_limit = " + + std::to_string(strict_capacity_limit)); + + NewShard(6, strict_capacity_limit); + EXPECT_OK(Insert('a', Cache::Priority::BOTTOM)); + EXPECT_OK(Insert('b', Cache::Priority::LOW)); + EXPECT_OK(Insert('c', Cache::Priority::HIGH)); + EXPECT_OK(Insert('d', Cache::Priority::BOTTOM)); + EXPECT_OK(Insert('e', Cache::Priority::LOW)); + EXPECT_OK(Insert('f', Cache::Priority::HIGH)); + + EXPECT_TRUE(Lookup('a', /*use*/ false)); + EXPECT_TRUE(Lookup('b', /*use*/ false)); + EXPECT_TRUE(Lookup('c', /*use*/ false)); + EXPECT_TRUE(Lookup('d', /*use*/ false)); + EXPECT_TRUE(Lookup('e', /*use*/ false)); + EXPECT_TRUE(Lookup('f', /*use*/ false)); + + // Ensure bottom are evicted first, even if new entries are low + EXPECT_OK(Insert('g', Cache::Priority::LOW)); + EXPECT_OK(Insert('h', Cache::Priority::LOW)); + + EXPECT_FALSE(Lookup('a', /*use*/ false)); + EXPECT_TRUE(Lookup('b', /*use*/ false)); + EXPECT_TRUE(Lookup('c', /*use*/ false)); + EXPECT_FALSE(Lookup('d', /*use*/ false)); + EXPECT_TRUE(Lookup('e', /*use*/ false)); + EXPECT_TRUE(Lookup('f', /*use*/ false)); + // Mark g & h useful + EXPECT_TRUE(Lookup('g', /*use*/ true)); + EXPECT_TRUE(Lookup('h', /*use*/ true)); + + // Then old LOW entries + EXPECT_OK(Insert('i', Cache::Priority::LOW)); + EXPECT_OK(Insert('j', Cache::Priority::LOW)); + + EXPECT_FALSE(Lookup('b', /*use*/ false)); + EXPECT_TRUE(Lookup('c', /*use*/ false)); + EXPECT_FALSE(Lookup('e', /*use*/ false)); + EXPECT_TRUE(Lookup('f', /*use*/ false)); + // Mark g & h useful once again + EXPECT_TRUE(Lookup('g', /*use*/ true)); + EXPECT_TRUE(Lookup('h', /*use*/ true)); + EXPECT_TRUE(Lookup('i', /*use*/ false)); + EXPECT_TRUE(Lookup('j', /*use*/ false)); + + // Then old HIGH entries + EXPECT_OK(Insert('k', Cache::Priority::LOW)); + EXPECT_OK(Insert('l', Cache::Priority::LOW)); + + EXPECT_FALSE(Lookup('c', /*use*/ false)); + EXPECT_FALSE(Lookup('f', /*use*/ false)); + EXPECT_TRUE(Lookup('g', /*use*/ false)); + EXPECT_TRUE(Lookup('h', /*use*/ false)); + EXPECT_TRUE(Lookup('i', /*use*/ false)); + EXPECT_TRUE(Lookup('j', /*use*/ false)); + EXPECT_TRUE(Lookup('k', /*use*/ false)); + EXPECT_TRUE(Lookup('l', /*use*/ false)); + + // Then the (roughly) least recently useful + EXPECT_OK(Insert('m', Cache::Priority::HIGH)); + EXPECT_OK(Insert('n', Cache::Priority::HIGH)); + + EXPECT_TRUE(Lookup('g', /*use*/ false)); + EXPECT_TRUE(Lookup('h', /*use*/ false)); + EXPECT_FALSE(Lookup('i', /*use*/ false)); + EXPECT_FALSE(Lookup('j', /*use*/ false)); + EXPECT_TRUE(Lookup('k', /*use*/ false)); + EXPECT_TRUE(Lookup('l', /*use*/ false)); + + // Now try changing capacity down + shard_->SetCapacity(4); + // Insert to ensure evictions happen + EXPECT_OK(Insert('o', Cache::Priority::LOW)); + EXPECT_OK(Insert('p', Cache::Priority::LOW)); + + EXPECT_FALSE(Lookup('g', /*use*/ false)); + EXPECT_FALSE(Lookup('h', /*use*/ false)); + EXPECT_FALSE(Lookup('k', /*use*/ false)); + EXPECT_FALSE(Lookup('l', /*use*/ false)); + EXPECT_TRUE(Lookup('m', /*use*/ false)); + EXPECT_TRUE(Lookup('n', /*use*/ false)); + EXPECT_TRUE(Lookup('o', /*use*/ false)); + EXPECT_TRUE(Lookup('p', /*use*/ false)); + + // Now try changing capacity up + EXPECT_TRUE(Lookup('m', /*use*/ true)); + EXPECT_TRUE(Lookup('n', /*use*/ true)); + shard_->SetCapacity(6); + EXPECT_OK(Insert('q', Cache::Priority::HIGH)); + EXPECT_OK(Insert('r', Cache::Priority::HIGH)); + EXPECT_OK(Insert('s', Cache::Priority::HIGH)); + EXPECT_OK(Insert('t', Cache::Priority::HIGH)); + + EXPECT_FALSE(Lookup('o', /*use*/ false)); + EXPECT_FALSE(Lookup('p', /*use*/ false)); + EXPECT_TRUE(Lookup('m', /*use*/ false)); + EXPECT_TRUE(Lookup('n', /*use*/ false)); + EXPECT_TRUE(Lookup('q', /*use*/ false)); + EXPECT_TRUE(Lookup('r', /*use*/ false)); + EXPECT_TRUE(Lookup('s', /*use*/ false)); + EXPECT_TRUE(Lookup('t', /*use*/ false)); + } +} + +namespace { +struct DeleteCounter { + int deleted = 0; +}; +const Cache::CacheItemHelper kDeleteCounterHelper{ + CacheEntryRole::kMisc, + [](Cache::ObjectPtr value, MemoryAllocator* /*alloc*/) { + static_cast(value)->deleted += 1; + }}; +} // namespace + +// Testing calls to CorrectNearOverflow in Release +TEST_F(ClockCacheTest, ClockCounterOverflowTest) { + NewShard(6, /*strict_capacity_limit*/ false); + HandleImpl* h; + DeleteCounter val; + UniqueId64x2 hkey = TestHashedKey('x'); + ASSERT_OK(shard_->Insert(TestKey(hkey), hkey, &val, &kDeleteCounterHelper, 1, + &h, Cache::Priority::HIGH)); + + // Some large number outstanding + shard_->TEST_RefN(h, 123456789); + // Simulate many lookup/ref + release, plenty to overflow counters + for (int i = 0; i < 10000; ++i) { + shard_->TEST_RefN(h, 1234567); + shard_->TEST_ReleaseN(h, 1234567); + } + // Mark it invisible (to reach a different CorrectNearOverflow() in Release) + shard_->Erase(TestKey(hkey), hkey); + // Simulate many more lookup/ref + release (one-by-one would be too + // expensive for unit test) + for (int i = 0; i < 10000; ++i) { + shard_->TEST_RefN(h, 1234567); + shard_->TEST_ReleaseN(h, 1234567); + } + // Free all but last 1 + shard_->TEST_ReleaseN(h, 123456789); + // Still alive + ASSERT_EQ(val.deleted, 0); + // Free last ref, which will finalize erasure + shard_->Release(h); + // Deleted + ASSERT_EQ(val.deleted, 1); +} + +TEST_F(ClockCacheTest, ClockTableFull) { + // Force clock cache table to fill up (not usually allowed) in order + // to test full probe sequence that is theoretically possible due to + // parallel operations + NewShard(6, /*strict_capacity_limit*/ false); + size_t size = shard_->GetTableAddressCount(); + ASSERT_LE(size + 3, 256); // for using char keys + // Modify occupancy and capacity limits to attempt insert on full + shard_->TEST_MutableOccupancyLimit() = size + 100; + shard_->SetCapacity(size + 100); + + DeleteCounter val; + std::vector handles; + // NOTE: the three extra insertions should create standalone entries + for (size_t i = 0; i < size + 3; ++i) { + UniqueId64x2 hkey = TestHashedKey(static_cast(i)); + ASSERT_OK(shard_->Insert(TestKey(hkey), hkey, &val, &kDeleteCounterHelper, + 1, &handles.emplace_back(), + Cache::Priority::HIGH)); + } + + for (size_t i = 0; i < size + 3; ++i) { + UniqueId64x2 hkey = TestHashedKey(static_cast(i)); + HandleImpl* h = shard_->Lookup(TestKey(hkey), hkey); + if (i < size) { + ASSERT_NE(h, nullptr); + shard_->Release(h); + } else { + // Standalone entries not visible by lookup + ASSERT_EQ(h, nullptr); + } + } + + for (size_t i = 0; i < size + 3; ++i) { + ASSERT_NE(handles[i], nullptr); + shard_->Release(handles[i]); + if (i < size) { + // Everything still in cache + ASSERT_EQ(val.deleted, 0); + } else { + // Standalone entries freed on release + ASSERT_EQ(val.deleted, i + 1 - size); + } + } + + for (size_t i = size + 3; i > 0; --i) { + UniqueId64x2 hkey = TestHashedKey(static_cast(i - 1)); + shard_->Erase(TestKey(hkey), hkey); + if (i - 1 > size) { + ASSERT_EQ(val.deleted, 3); + } else { + ASSERT_EQ(val.deleted, 3 + size - (i - 1)); + } + } +} + +// This test is mostly to exercise some corner case logic, by forcing two +// keys to have the same hash, and more +TEST_F(ClockCacheTest, CollidingInsertEraseTest) { + NewShard(6, /*strict_capacity_limit*/ false); + DeleteCounter val; + UniqueId64x2 hkey1 = TestHashedKey('x'); + Slice key1 = TestKey(hkey1); + UniqueId64x2 hkey2 = TestHashedKey('y'); + Slice key2 = TestKey(hkey2); + UniqueId64x2 hkey3 = TestHashedKey('z'); + Slice key3 = TestKey(hkey3); + HandleImpl* h1; + ASSERT_OK(shard_->Insert(key1, hkey1, &val, &kDeleteCounterHelper, 1, &h1, + Cache::Priority::HIGH)); + HandleImpl* h2; + ASSERT_OK(shard_->Insert(key2, hkey2, &val, &kDeleteCounterHelper, 1, &h2, + Cache::Priority::HIGH)); + HandleImpl* h3; + ASSERT_OK(shard_->Insert(key3, hkey3, &val, &kDeleteCounterHelper, 1, &h3, + Cache::Priority::HIGH)); + + // Can repeatedly lookup+release despite the hash collision + HandleImpl* tmp_h; + for (bool erase_if_last_ref : {true, false}) { // but not last ref + tmp_h = shard_->Lookup(key1, hkey1); + ASSERT_EQ(h1, tmp_h); + ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref)); + + tmp_h = shard_->Lookup(key2, hkey2); + ASSERT_EQ(h2, tmp_h); + ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref)); + + tmp_h = shard_->Lookup(key3, hkey3); + ASSERT_EQ(h3, tmp_h); + ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref)); + } + + // Make h1 invisible + shard_->Erase(key1, hkey1); + // Redundant erase + shard_->Erase(key1, hkey1); + + // All still alive + ASSERT_EQ(val.deleted, 0); + + // Invisible to Lookup + tmp_h = shard_->Lookup(key1, hkey1); + ASSERT_EQ(nullptr, tmp_h); + + // Can still find h2, h3 + for (bool erase_if_last_ref : {true, false}) { // but not last ref + tmp_h = shard_->Lookup(key2, hkey2); + ASSERT_EQ(h2, tmp_h); + ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref)); + + tmp_h = shard_->Lookup(key3, hkey3); + ASSERT_EQ(h3, tmp_h); + ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref)); + } + + // Also Insert with invisible entry there + ASSERT_OK(shard_->Insert(key1, hkey1, &val, &kDeleteCounterHelper, 1, nullptr, + Cache::Priority::HIGH)); + tmp_h = shard_->Lookup(key1, hkey1); + // Found but distinct handle + ASSERT_NE(nullptr, tmp_h); + ASSERT_NE(h1, tmp_h); + ASSERT_TRUE(shard_->Release(tmp_h, /*erase_if_last_ref*/ true)); + + // tmp_h deleted + ASSERT_EQ(val.deleted--, 1); + + // Release last ref on h1 (already invisible) + ASSERT_TRUE(shard_->Release(h1, /*erase_if_last_ref*/ false)); + + // h1 deleted + ASSERT_EQ(val.deleted--, 1); + h1 = nullptr; + + // Can still find h2, h3 + for (bool erase_if_last_ref : {true, false}) { // but not last ref + tmp_h = shard_->Lookup(key2, hkey2); + ASSERT_EQ(h2, tmp_h); + ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref)); + + tmp_h = shard_->Lookup(key3, hkey3); + ASSERT_EQ(h3, tmp_h); + ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref)); + } + + // Release last ref on h2 + ASSERT_FALSE(shard_->Release(h2, /*erase_if_last_ref*/ false)); + + // h2 still not deleted (unreferenced in cache) + ASSERT_EQ(val.deleted, 0); + + // Can still find it + tmp_h = shard_->Lookup(key2, hkey2); + ASSERT_EQ(h2, tmp_h); + + // Release last ref on h2, with erase + ASSERT_TRUE(shard_->Release(h2, /*erase_if_last_ref*/ true)); + + // h2 deleted + ASSERT_EQ(val.deleted--, 1); + tmp_h = shard_->Lookup(key2, hkey2); + ASSERT_EQ(nullptr, tmp_h); + + // Can still find h3 + for (bool erase_if_last_ref : {true, false}) { // but not last ref + tmp_h = shard_->Lookup(key3, hkey3); + ASSERT_EQ(h3, tmp_h); + ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref)); + } + + // Release last ref on h3, without erase + ASSERT_FALSE(shard_->Release(h3, /*erase_if_last_ref*/ false)); + + // h3 still not deleted (unreferenced in cache) + ASSERT_EQ(val.deleted, 0); + + // Explicit erase + shard_->Erase(key3, hkey3); + + // h3 deleted + ASSERT_EQ(val.deleted--, 1); + tmp_h = shard_->Lookup(key3, hkey3); + ASSERT_EQ(nullptr, tmp_h); +} + +// This uses the public API to effectively test CalcHashBits etc. +TEST_F(ClockCacheTest, TableSizesTest) { + for (size_t est_val_size : {1U, 5U, 123U, 2345U, 345678U}) { + SCOPED_TRACE("est_val_size = " + std::to_string(est_val_size)); + for (double est_count : {1.1, 2.2, 511.9, 512.1, 2345.0}) { + SCOPED_TRACE("est_count = " + std::to_string(est_count)); + size_t capacity = static_cast(est_val_size * est_count); + // kDontChargeCacheMetadata + auto cache = HyperClockCacheOptions( + capacity, est_val_size, /*num shard_bits*/ -1, + /*strict_capacity_limit*/ false, + /*memory_allocator*/ nullptr, kDontChargeCacheMetadata) + .MakeSharedCache(); + // Table sizes are currently only powers of two + EXPECT_GE(cache->GetTableAddressCount(), est_count / kLoadFactor); + EXPECT_LE(cache->GetTableAddressCount(), est_count / kLoadFactor * 2.0); + EXPECT_EQ(cache->GetUsage(), 0); + + // kFullChargeMetaData + // Because table sizes are currently only powers of two, sizes get + // really weird when metadata is a huge portion of capacity. For example, + // doubling the table size could cut by 90% the space available to + // values. Therefore, we omit those weird cases for now. + if (est_val_size >= 512) { + cache = HyperClockCacheOptions( + capacity, est_val_size, /*num shard_bits*/ -1, + /*strict_capacity_limit*/ false, + /*memory_allocator*/ nullptr, kFullChargeCacheMetadata) + .MakeSharedCache(); + double est_count_after_meta = + (capacity - cache->GetUsage()) * 1.0 / est_val_size; + EXPECT_GE(cache->GetTableAddressCount(), + est_count_after_meta / kLoadFactor); + EXPECT_LE(cache->GetTableAddressCount(), + est_count_after_meta / kLoadFactor * 2.0); + } + } + } +} + +} // namespace clock_cache + +class TestSecondaryCache : public SecondaryCache { + public: + // Specifies what action to take on a lookup for a particular key + enum ResultType { + SUCCESS, + // Fail lookup immediately + FAIL, + // Defer the result. It will returned after Wait/WaitAll is called + DEFER, + // Defer the result and eventually return failure + DEFER_AND_FAIL + }; + + using ResultMap = std::unordered_map; + + explicit TestSecondaryCache(size_t capacity) + : cache_(NewLRUCache(capacity, 0, false, 0.5 /* high_pri_pool_ratio */, + nullptr, kDefaultToAdaptiveMutex, + kDontChargeCacheMetadata)), + num_inserts_(0), + num_lookups_(0), + inject_failure_(false) {} + + const char* Name() const override { return "TestSecondaryCache"; } + + void InjectFailure() { inject_failure_ = true; } + + void ResetInjectFailure() { inject_failure_ = false; } + + Status Insert(const Slice& key, Cache::ObjectPtr value, + const Cache::CacheItemHelper* helper) override { + if (inject_failure_) { + return Status::Corruption("Insertion Data Corrupted"); + } + CheckCacheKeyCommonPrefix(key); + size_t size; + char* buf; + Status s; + + num_inserts_++; + size = (*helper->size_cb)(value); + buf = new char[size + sizeof(uint64_t)]; + EncodeFixed64(buf, size); + s = (*helper->saveto_cb)(value, 0, size, buf + sizeof(uint64_t)); + if (!s.ok()) { + delete[] buf; + return s; + } + return cache_.Insert(key, buf, size); + } + + std::unique_ptr Lookup( + const Slice& key, const Cache::CacheItemHelper* helper, + Cache::CreateContext* create_context, bool /*wait*/, + bool /*advise_erase*/, bool& kept_in_sec_cache) override { + std::string key_str = key.ToString(); + TEST_SYNC_POINT_CALLBACK("TestSecondaryCache::Lookup", &key_str); + + std::unique_ptr secondary_handle; + kept_in_sec_cache = false; + ResultType type = ResultType::SUCCESS; + auto iter = result_map_.find(key.ToString()); + if (iter != result_map_.end()) { + type = iter->second; + } + if (type == ResultType::FAIL) { + return secondary_handle; + } + + TypedHandle* handle = cache_.Lookup(key); + num_lookups_++; + if (handle) { + Cache::ObjectPtr value = nullptr; + size_t charge = 0; + Status s; + if (type != ResultType::DEFER_AND_FAIL) { + char* ptr = cache_.Value(handle); + size_t size = DecodeFixed64(ptr); + ptr += sizeof(uint64_t); + s = helper->create_cb(Slice(ptr, size), create_context, + /*alloc*/ nullptr, &value, &charge); + } + if (s.ok()) { + secondary_handle.reset(new TestSecondaryCacheResultHandle( + cache_.get(), handle, value, charge, type)); + kept_in_sec_cache = true; + } else { + cache_.Release(handle); + } + } + return secondary_handle; + } + + bool SupportForceErase() const override { return false; } + + void Erase(const Slice& /*key*/) override {} + + void WaitAll(std::vector handles) override { + for (SecondaryCacheResultHandle* handle : handles) { + TestSecondaryCacheResultHandle* sec_handle = + static_cast(handle); + sec_handle->SetReady(); + } + } + + std::string GetPrintableOptions() const override { return ""; } + + void SetResultMap(ResultMap&& map) { result_map_ = std::move(map); } + + uint32_t num_inserts() { return num_inserts_; } + + uint32_t num_lookups() { return num_lookups_; } + + void CheckCacheKeyCommonPrefix(const Slice& key) { + Slice current_prefix(key.data(), OffsetableCacheKey::kCommonPrefixSize); + if (ckey_prefix_.empty()) { + ckey_prefix_ = current_prefix.ToString(); + } else { + EXPECT_EQ(ckey_prefix_, current_prefix.ToString()); + } + } + + private: + class TestSecondaryCacheResultHandle : public SecondaryCacheResultHandle { + public: + TestSecondaryCacheResultHandle(Cache* cache, Cache::Handle* handle, + Cache::ObjectPtr value, size_t size, + ResultType type) + : cache_(cache), + handle_(handle), + value_(value), + size_(size), + is_ready_(true) { + if (type != ResultType::SUCCESS) { + is_ready_ = false; + } + } + + ~TestSecondaryCacheResultHandle() override { cache_->Release(handle_); } + + bool IsReady() override { return is_ready_; } + + void Wait() override {} + + Cache::ObjectPtr Value() override { + assert(is_ready_); + return value_; + } + + size_t Size() override { return Value() ? size_ : 0; } + + void SetReady() { is_ready_ = true; } + + private: + Cache* cache_; + Cache::Handle* handle_; + Cache::ObjectPtr value_; + size_t size_; + bool is_ready_; + }; + + using SharedCache = + BasicTypedSharedCacheInterface; + using TypedHandle = SharedCache::TypedHandle; + SharedCache cache_; + uint32_t num_inserts_; + uint32_t num_lookups_; + bool inject_failure_; + std::string ckey_prefix_; + ResultMap result_map_; +}; + +using secondary_cache_test_util::GetTestingCacheTypes; +using secondary_cache_test_util::WithCacheTypeParam; + +class BasicSecondaryCacheTest : public testing::Test, + public WithCacheTypeParam {}; + +INSTANTIATE_TEST_CASE_P(BasicSecondaryCacheTest, BasicSecondaryCacheTest, + GetTestingCacheTypes()); + +class DBSecondaryCacheTest : public DBTestBase, public WithCacheTypeParam { + public: + DBSecondaryCacheTest() + : DBTestBase("db_secondary_cache_test", /*env_do_fsync=*/true) { + fault_fs_.reset(new FaultInjectionTestFS(env_->GetFileSystem())); + fault_env_.reset(new CompositeEnvWrapper(env_, fault_fs_)); + } + + std::shared_ptr fault_fs_; + std::unique_ptr fault_env_; +}; + +INSTANTIATE_TEST_CASE_P(DBSecondaryCacheTest, DBSecondaryCacheTest, + GetTestingCacheTypes()); + +TEST_P(BasicSecondaryCacheTest, BasicTest) { + std::shared_ptr secondary_cache = + std::make_shared(4096); + std::shared_ptr cache = + NewCache(1024 /* capacity */, 0 /* num_shard_bits */, + false /* strict_capacity_limit */, secondary_cache); + std::shared_ptr stats = CreateDBStatistics(); + CacheKey k1 = CacheKey::CreateUniqueForCacheLifetime(cache.get()); + CacheKey k2 = CacheKey::CreateUniqueForCacheLifetime(cache.get()); + CacheKey k3 = CacheKey::CreateUniqueForCacheLifetime(cache.get()); + + Random rnd(301); + // Start with warming k3 + std::string str3 = rnd.RandomString(1021); + ASSERT_OK(secondary_cache->InsertSaved(k3.AsSlice(), str3)); + + std::string str1 = rnd.RandomString(1021); + TestItem* item1 = new TestItem(str1.data(), str1.length()); + ASSERT_OK(cache->Insert(k1.AsSlice(), item1, GetHelper(), str1.length())); + std::string str2 = rnd.RandomString(1021); + TestItem* item2 = new TestItem(str2.data(), str2.length()); + // k1 should be demoted to NVM + ASSERT_OK(cache->Insert(k2.AsSlice(), item2, GetHelper(), str2.length())); + + get_perf_context()->Reset(); + Cache::Handle* handle; + handle = cache->Lookup(k2.AsSlice(), GetHelper(), + /*context*/ this, Cache::Priority::LOW, stats.get()); + ASSERT_NE(handle, nullptr); + ASSERT_EQ(static_cast(cache->Value(handle))->Size(), str2.size()); + cache->Release(handle); + + // This lookup should promote k1 and demote k2 + handle = cache->Lookup(k1.AsSlice(), GetHelper(), + /*context*/ this, Cache::Priority::LOW, stats.get()); + ASSERT_NE(handle, nullptr); + ASSERT_EQ(static_cast(cache->Value(handle))->Size(), str1.size()); + cache->Release(handle); + + // This lookup should promote k3 and demote k1 + handle = cache->Lookup(k3.AsSlice(), GetHelper(), + /*context*/ this, Cache::Priority::LOW, stats.get()); + ASSERT_NE(handle, nullptr); + ASSERT_EQ(static_cast(cache->Value(handle))->Size(), str3.size()); + cache->Release(handle); + + ASSERT_EQ(secondary_cache->num_inserts(), 3u); + ASSERT_EQ(secondary_cache->num_lookups(), 2u); + ASSERT_EQ(stats->getTickerCount(SECONDARY_CACHE_HITS), + secondary_cache->num_lookups()); + PerfContext perf_ctx = *get_perf_context(); + ASSERT_EQ(perf_ctx.secondary_cache_hit_count, secondary_cache->num_lookups()); + + cache.reset(); + secondary_cache.reset(); +} + +TEST_P(BasicSecondaryCacheTest, StatsTest) { + std::shared_ptr secondary_cache = + std::make_shared(4096); + std::shared_ptr cache = + NewCache(1024 /* capacity */, 0 /* num_shard_bits */, + false /* strict_capacity_limit */, secondary_cache); + std::shared_ptr stats = CreateDBStatistics(); + CacheKey k1 = CacheKey::CreateUniqueForCacheLifetime(cache.get()); + CacheKey k2 = CacheKey::CreateUniqueForCacheLifetime(cache.get()); + CacheKey k3 = CacheKey::CreateUniqueForCacheLifetime(cache.get()); + + Random rnd(301); + // Start with warming secondary cache + std::string str1 = rnd.RandomString(1020); + std::string str2 = rnd.RandomString(1020); + std::string str3 = rnd.RandomString(1020); + ASSERT_OK(secondary_cache->InsertSaved(k1.AsSlice(), str1)); + ASSERT_OK(secondary_cache->InsertSaved(k2.AsSlice(), str2)); + ASSERT_OK(secondary_cache->InsertSaved(k3.AsSlice(), str3)); + + get_perf_context()->Reset(); + Cache::Handle* handle; + handle = cache->Lookup(k1.AsSlice(), GetHelper(CacheEntryRole::kFilterBlock), + /*context*/ this, Cache::Priority::LOW, stats.get()); + ASSERT_NE(handle, nullptr); + ASSERT_EQ(static_cast(cache->Value(handle))->Size(), str1.size()); + cache->Release(handle); + + handle = cache->Lookup(k2.AsSlice(), GetHelper(CacheEntryRole::kIndexBlock), + /*context*/ this, Cache::Priority::LOW, stats.get()); + ASSERT_NE(handle, nullptr); + ASSERT_EQ(static_cast(cache->Value(handle))->Size(), str2.size()); + cache->Release(handle); + + handle = cache->Lookup(k3.AsSlice(), GetHelper(CacheEntryRole::kDataBlock), + /*context*/ this, Cache::Priority::LOW, stats.get()); + ASSERT_NE(handle, nullptr); + ASSERT_EQ(static_cast(cache->Value(handle))->Size(), str3.size()); + cache->Release(handle); + + ASSERT_EQ(secondary_cache->num_inserts(), 3u); + ASSERT_EQ(secondary_cache->num_lookups(), 3u); + ASSERT_EQ(stats->getTickerCount(SECONDARY_CACHE_HITS), + secondary_cache->num_lookups()); + ASSERT_EQ(stats->getTickerCount(SECONDARY_CACHE_FILTER_HITS), 1); + ASSERT_EQ(stats->getTickerCount(SECONDARY_CACHE_INDEX_HITS), 1); + ASSERT_EQ(stats->getTickerCount(SECONDARY_CACHE_DATA_HITS), 1); + PerfContext perf_ctx = *get_perf_context(); + ASSERT_EQ(perf_ctx.secondary_cache_hit_count, secondary_cache->num_lookups()); + + cache.reset(); + secondary_cache.reset(); +} + +TEST_P(BasicSecondaryCacheTest, BasicFailTest) { + std::shared_ptr secondary_cache = + std::make_shared(2048); + std::shared_ptr cache = + NewCache(1024 /* capacity */, 0 /* num_shard_bits */, + false /* strict_capacity_limit */, secondary_cache); + CacheKey k1 = CacheKey::CreateUniqueForCacheLifetime(cache.get()); + CacheKey k2 = CacheKey::CreateUniqueForCacheLifetime(cache.get()); + + Random rnd(301); + std::string str1 = rnd.RandomString(1020); + auto item1 = std::make_unique(str1.data(), str1.length()); + // NOTE: changed to assert helper != nullptr for efficiency / code size + // ASSERT_TRUE(cache->Insert(k1.AsSlice(), item1.get(), nullptr, + // str1.length()).IsInvalidArgument()); + ASSERT_OK( + cache->Insert(k1.AsSlice(), item1.get(), GetHelper(), str1.length())); + item1.release(); // Appease clang-analyze "potential memory leak" + + Cache::Handle* handle; + handle = cache->Lookup(k2.AsSlice(), nullptr, /*context*/ this, + Cache::Priority::LOW); + ASSERT_EQ(handle, nullptr); + + handle = cache->Lookup(k2.AsSlice(), GetHelper(), + /*context*/ this, Cache::Priority::LOW); + ASSERT_EQ(handle, nullptr); + + Cache::AsyncLookupHandle async_handle; + async_handle.key = k2.AsSlice(); + async_handle.helper = GetHelper(); + async_handle.create_context = this; + async_handle.priority = Cache::Priority::LOW; + cache->StartAsyncLookup(async_handle); + cache->Wait(async_handle); + handle = async_handle.Result(); + ASSERT_EQ(handle, nullptr); + + cache.reset(); + secondary_cache.reset(); +} + +TEST_P(BasicSecondaryCacheTest, SaveFailTest) { + std::shared_ptr secondary_cache = + std::make_shared(2048); + std::shared_ptr cache = + NewCache(1024 /* capacity */, 0 /* num_shard_bits */, + false /* strict_capacity_limit */, secondary_cache); + CacheKey k1 = CacheKey::CreateUniqueForCacheLifetime(cache.get()); + CacheKey k2 = CacheKey::CreateUniqueForCacheLifetime(cache.get()); + + Random rnd(301); + std::string str1 = rnd.RandomString(1020); + TestItem* item1 = new TestItem(str1.data(), str1.length()); + ASSERT_OK(cache->Insert(k1.AsSlice(), item1, GetHelperFail(), str1.length())); + std::string str2 = rnd.RandomString(1020); + TestItem* item2 = new TestItem(str2.data(), str2.length()); + // k1 should be demoted to NVM + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_OK(cache->Insert(k2.AsSlice(), item2, GetHelperFail(), str2.length())); + ASSERT_EQ(secondary_cache->num_inserts(), 1u); + + Cache::Handle* handle; + handle = cache->Lookup(k2.AsSlice(), GetHelperFail(), + /*context*/ this, Cache::Priority::LOW); + ASSERT_NE(handle, nullptr); + cache->Release(handle); + // This lookup should fail, since k1 demotion would have failed + handle = cache->Lookup(k1.AsSlice(), GetHelperFail(), + /*context*/ this, Cache::Priority::LOW); + ASSERT_EQ(handle, nullptr); + // Since k1 didn't get promoted, k2 should still be in cache + handle = cache->Lookup(k2.AsSlice(), GetHelperFail(), + /*context*/ this, Cache::Priority::LOW); + ASSERT_NE(handle, nullptr); + cache->Release(handle); + ASSERT_EQ(secondary_cache->num_inserts(), 1u); + ASSERT_EQ(secondary_cache->num_lookups(), 1u); + + cache.reset(); + secondary_cache.reset(); +} + +TEST_P(BasicSecondaryCacheTest, CreateFailTest) { + std::shared_ptr secondary_cache = + std::make_shared(2048); + std::shared_ptr cache = + NewCache(1024 /* capacity */, 0 /* num_shard_bits */, + false /* strict_capacity_limit */, secondary_cache); + CacheKey k1 = CacheKey::CreateUniqueForCacheLifetime(cache.get()); + CacheKey k2 = CacheKey::CreateUniqueForCacheLifetime(cache.get()); + + Random rnd(301); + std::string str1 = rnd.RandomString(1020); + TestItem* item1 = new TestItem(str1.data(), str1.length()); + ASSERT_OK(cache->Insert(k1.AsSlice(), item1, GetHelper(), str1.length())); + std::string str2 = rnd.RandomString(1020); + TestItem* item2 = new TestItem(str2.data(), str2.length()); + // k1 should be demoted to NVM + ASSERT_OK(cache->Insert(k2.AsSlice(), item2, GetHelper(), str2.length())); + + Cache::Handle* handle; + SetFailCreate(true); + handle = cache->Lookup(k2.AsSlice(), GetHelper(), + /*context*/ this, Cache::Priority::LOW); + ASSERT_NE(handle, nullptr); + cache->Release(handle); + // This lookup should fail, since k1 creation would have failed + handle = cache->Lookup(k1.AsSlice(), GetHelper(), + /*context*/ this, Cache::Priority::LOW); + ASSERT_EQ(handle, nullptr); + // Since k1 didn't get promoted, k2 should still be in cache + handle = cache->Lookup(k2.AsSlice(), GetHelper(), + /*context*/ this, Cache::Priority::LOW); + ASSERT_NE(handle, nullptr); + cache->Release(handle); + ASSERT_EQ(secondary_cache->num_inserts(), 1u); + ASSERT_EQ(secondary_cache->num_lookups(), 1u); + + cache.reset(); + secondary_cache.reset(); +} + +TEST_P(BasicSecondaryCacheTest, FullCapacityTest) { + for (bool strict_capacity_limit : {false, true}) { + std::shared_ptr secondary_cache = + std::make_shared(2048); + std::shared_ptr cache = + NewCache(1024 /* capacity */, 0 /* num_shard_bits */, + strict_capacity_limit, secondary_cache); + CacheKey k1 = CacheKey::CreateUniqueForCacheLifetime(cache.get()); + CacheKey k2 = CacheKey::CreateUniqueForCacheLifetime(cache.get()); + + Random rnd(301); + std::string str1 = rnd.RandomString(1020); + TestItem* item1 = new TestItem(str1.data(), str1.length()); + ASSERT_OK(cache->Insert(k1.AsSlice(), item1, GetHelper(), str1.length())); + std::string str2 = rnd.RandomString(1020); + TestItem* item2 = new TestItem(str2.data(), str2.length()); + // k1 should be demoted to NVM + ASSERT_OK(cache->Insert(k2.AsSlice(), item2, GetHelper(), str2.length())); + + Cache::Handle* handle2; + handle2 = cache->Lookup(k2.AsSlice(), GetHelper(), + /*context*/ this, Cache::Priority::LOW); + ASSERT_NE(handle2, nullptr); + // k1 lookup fails without secondary cache support + Cache::Handle* handle1; + handle1 = cache->Lookup( + k1.AsSlice(), + GetHelper(CacheEntryRole::kDataBlock, /*secondary_compatible=*/false), + /*context*/ this, Cache::Priority::LOW); + ASSERT_EQ(handle1, nullptr); + + // k1 promotion can fail with strict_capacit_limit=true, but Lookup still + // succeeds using a standalone handle + handle1 = cache->Lookup(k1.AsSlice(), GetHelper(), + /*context*/ this, Cache::Priority::LOW); + ASSERT_NE(handle1, nullptr); + + ASSERT_EQ(secondary_cache->num_inserts(), 1u); + ASSERT_EQ(secondary_cache->num_lookups(), 1u); + + // Releasing k2's handle first, k2 is evicted from primary iff k1 promotion + // was charged to the cache (except HCC doesn't erase in Release() over + // capacity) + // FIXME: Insert to secondary from Release disabled + cache->Release(handle2); + cache->Release(handle1); + handle2 = cache->Lookup( + k2.AsSlice(), + GetHelper(CacheEntryRole::kDataBlock, /*secondary_compatible=*/false), + /*context*/ this, Cache::Priority::LOW); + if (strict_capacity_limit || GetParam() == kHyperClock) { + ASSERT_NE(handle2, nullptr); + cache->Release(handle2); + ASSERT_EQ(secondary_cache->num_inserts(), 1u); + } else { + ASSERT_EQ(handle2, nullptr); + // FIXME: Insert to secondary from Release disabled + // ASSERT_EQ(secondary_cache->num_inserts(), 2u); + ASSERT_EQ(secondary_cache->num_inserts(), 1u); + } + + cache.reset(); + secondary_cache.reset(); + } +} + +// In this test, the block cache size is set to 4096, after insert 6 KV-pairs +// and flush, there are 5 blocks in this SST file, 2 data blocks and 3 meta +// blocks. block_1 size is 4096 and block_2 size is 2056. The total size +// of the meta blocks are about 900 to 1000. Therefore, in any situation, +// if we try to insert block_1 to the block cache, it will always fails. Only +// block_2 will be successfully inserted into the block cache. +// CORRECTION: this is not quite right. block_1 can be inserted into the block +// cache because strict_capacity_limit=false, but it is removed from the cache +// in Release() because of being over-capacity, without demoting to secondary +// cache. HyperClockCache doesn't check capacity on release (for efficiency) +// so can demote the over-capacity item to secondary cache. Also, we intend to +// add support for demotion in Release, but that currently causes too much +// unit test churn. +TEST_P(DBSecondaryCacheTest, TestSecondaryCacheCorrectness1) { + if (GetParam() == kHyperClock) { + // See CORRECTION above + ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors"); + return; + } + std::shared_ptr secondary_cache( + new TestSecondaryCache(2048 * 1024)); + std::shared_ptr cache = + NewCache(4 * 1024 /* capacity */, 0 /* num_shard_bits */, + false /* strict_capacity_limit */, secondary_cache); + BlockBasedTableOptions table_options; + table_options.block_cache = cache; + table_options.block_size = 4 * 1024; + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.env = fault_env_.get(); + fault_fs_->SetFailGetUniqueId(true); + + // Set the file paranoid check, so after flush, the file will be read + // all the blocks will be accessed. + options.paranoid_file_checks = true; + DestroyAndReopen(options); + Random rnd(301); + const int N = 6; + for (int i = 0; i < N; i++) { + std::string p_v = rnd.RandomString(1007); + ASSERT_OK(Put(Key(i), p_v)); + } + + ASSERT_OK(Flush()); + // After Flush is successful, RocksDB will do the paranoid check for the new + // SST file. Meta blocks are always cached in the block cache and they + // will not be evicted. When block_2 is cache miss and read out, it is + // inserted to the block cache. Note that, block_1 is never successfully + // inserted to the block cache. Here are 2 lookups in the secondary cache + // for block_1 and block_2 + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 2u); + + Compact("a", "z"); + // Compaction will create the iterator to scan the whole file. So all the + // blocks are needed. Meta blocks are always cached. When block_1 is read + // out, block_2 is evicted from block cache and inserted to secondary + // cache. + ASSERT_EQ(secondary_cache->num_inserts(), 1u); + ASSERT_EQ(secondary_cache->num_lookups(), 3u); + + std::string v = Get(Key(0)); + ASSERT_EQ(1007, v.size()); + // The first data block is not in the cache, similarly, trigger the block + // cache Lookup and secondary cache lookup for block_1. But block_1 will not + // be inserted successfully due to the size. Currently, cache only has + // the meta blocks. + ASSERT_EQ(secondary_cache->num_inserts(), 1u); + ASSERT_EQ(secondary_cache->num_lookups(), 4u); + + v = Get(Key(5)); + ASSERT_EQ(1007, v.size()); + // The second data block is not in the cache, similarly, trigger the block + // cache Lookup and secondary cache lookup for block_2 and block_2 is found + // in the secondary cache. Now block cache has block_2 + ASSERT_EQ(secondary_cache->num_inserts(), 1u); + ASSERT_EQ(secondary_cache->num_lookups(), 5u); + + v = Get(Key(5)); + ASSERT_EQ(1007, v.size()); + // block_2 is in the block cache. There is a block cache hit. No need to + // lookup or insert the secondary cache. + ASSERT_EQ(secondary_cache->num_inserts(), 1u); + ASSERT_EQ(secondary_cache->num_lookups(), 5u); + + v = Get(Key(0)); + ASSERT_EQ(1007, v.size()); + // Lookup the first data block, not in the block cache, so lookup the + // secondary cache. Also not in the secondary cache. After Get, still + // block_1 is will not be cached. + ASSERT_EQ(secondary_cache->num_inserts(), 1u); + ASSERT_EQ(secondary_cache->num_lookups(), 6u); + + v = Get(Key(0)); + ASSERT_EQ(1007, v.size()); + // Lookup the first data block, not in the block cache, so lookup the + // secondary cache. Also not in the secondary cache. After Get, still + // block_1 is will not be cached. + ASSERT_EQ(secondary_cache->num_inserts(), 1u); + ASSERT_EQ(secondary_cache->num_lookups(), 7u); + + Destroy(options); +} + +// In this test, the block cache size is set to 6100, after insert 6 KV-pairs +// and flush, there are 5 blocks in this SST file, 2 data blocks and 3 meta +// blocks. block_1 size is 4096 and block_2 size is 2056. The total size +// of the meta blocks are about 900 to 1000. Therefore, we can successfully +// insert and cache block_1 in the block cache (this is the different place +// from TestSecondaryCacheCorrectness1) +TEST_P(DBSecondaryCacheTest, TestSecondaryCacheCorrectness2) { + if (GetParam() == kHyperClock) { + ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors"); + return; + } + std::shared_ptr secondary_cache( + new TestSecondaryCache(2048 * 1024)); + std::shared_ptr cache = + NewCache(6100 /* capacity */, 0 /* num_shard_bits */, + false /* strict_capacity_limit */, secondary_cache); + BlockBasedTableOptions table_options; + table_options.block_cache = cache; + table_options.block_size = 4 * 1024; + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.paranoid_file_checks = true; + options.env = fault_env_.get(); + fault_fs_->SetFailGetUniqueId(true); + DestroyAndReopen(options); + Random rnd(301); + const int N = 6; + for (int i = 0; i < N; i++) { + std::string p_v = rnd.RandomString(1007); + ASSERT_OK(Put(Key(i), p_v)); + } + + ASSERT_OK(Flush()); + // After Flush is successful, RocksDB will do the paranoid check for the new + // SST file. Meta blocks are always cached in the block cache and they + // will not be evicted. When block_2 is cache miss and read out, it is + // inserted to the block cache. Thefore, block_1 is evicted from block + // cache and successfully inserted to the secondary cache. Here are 2 + // lookups in the secondary cache for block_1 and block_2. + ASSERT_EQ(secondary_cache->num_inserts(), 1u); + ASSERT_EQ(secondary_cache->num_lookups(), 2u); + + Compact("a", "z"); + // Compaction will create the iterator to scan the whole file. So all the + // blocks are needed. After Flush, only block_2 is cached in block cache + // and block_1 is in the secondary cache. So when read block_1, it is + // read out from secondary cache and inserted to block cache. At the same + // time, block_2 is inserted to secondary cache. Now, secondary cache has + // both block_1 and block_2. After compaction, block_1 is in the cache. + ASSERT_EQ(secondary_cache->num_inserts(), 2u); + ASSERT_EQ(secondary_cache->num_lookups(), 3u); + + std::string v = Get(Key(0)); + ASSERT_EQ(1007, v.size()); + // This Get needs to access block_1, since block_1 is cached in block cache + // there is no secondary cache lookup. + ASSERT_EQ(secondary_cache->num_inserts(), 2u); + ASSERT_EQ(secondary_cache->num_lookups(), 3u); + + v = Get(Key(5)); + ASSERT_EQ(1007, v.size()); + // This Get needs to access block_2 which is not in the block cache. So + // it will lookup the secondary cache for block_2 and cache it in the + // block_cache. + ASSERT_EQ(secondary_cache->num_inserts(), 2u); + ASSERT_EQ(secondary_cache->num_lookups(), 4u); + + v = Get(Key(5)); + ASSERT_EQ(1007, v.size()); + // This Get needs to access block_2 which is already in the block cache. + // No need to lookup secondary cache. + ASSERT_EQ(secondary_cache->num_inserts(), 2u); + ASSERT_EQ(secondary_cache->num_lookups(), 4u); + + v = Get(Key(0)); + ASSERT_EQ(1007, v.size()); + // This Get needs to access block_1, since block_1 is not in block cache + // there is one econdary cache lookup. Then, block_1 is cached in the + // block cache. + ASSERT_EQ(secondary_cache->num_inserts(), 2u); + ASSERT_EQ(secondary_cache->num_lookups(), 5u); + + v = Get(Key(0)); + ASSERT_EQ(1007, v.size()); + // This Get needs to access block_1, since block_1 is cached in block cache + // there is no secondary cache lookup. + ASSERT_EQ(secondary_cache->num_inserts(), 2u); + ASSERT_EQ(secondary_cache->num_lookups(), 5u); + + Destroy(options); +} + +// The block cache size is set to 1024*1024, after insert 6 KV-pairs +// and flush, there are 5 blocks in this SST file, 2 data blocks and 3 meta +// blocks. block_1 size is 4096 and block_2 size is 2056. The total size +// of the meta blocks are about 900 to 1000. Therefore, we can successfully +// cache all the blocks in the block cache and there is not secondary cache +// insertion. 2 lookup is needed for the blocks. +TEST_P(DBSecondaryCacheTest, NoSecondaryCacheInsertion) { + std::shared_ptr secondary_cache( + new TestSecondaryCache(2048 * 1024)); + std::shared_ptr cache = + NewCache(1024 * 1024 /* capacity */, 0 /* num_shard_bits */, + false /* strict_capacity_limit */, secondary_cache); + BlockBasedTableOptions table_options; + table_options.block_cache = cache; + table_options.block_size = 4 * 1024; + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.paranoid_file_checks = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.env = fault_env_.get(); + fault_fs_->SetFailGetUniqueId(true); + + DestroyAndReopen(options); + Random rnd(301); + const int N = 6; + for (int i = 0; i < N; i++) { + std::string p_v = rnd.RandomString(1000); + ASSERT_OK(Put(Key(i), p_v)); + } + + ASSERT_OK(Flush()); + // After Flush is successful, RocksDB will do the paranoid check for the new + // SST file. Meta blocks are always cached in the block cache and they + // will not be evicted. Now, block cache is large enough, it cache + // both block_1 and block_2. When first time read block_1 and block_2 + // there are cache misses. So 2 secondary cache lookups are needed for + // the 2 blocks + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 2u); + + Compact("a", "z"); + // Compaction will iterate the whole SST file. Since all the data blocks + // are in the block cache. No need to lookup the secondary cache. + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 2u); + + std::string v = Get(Key(0)); + ASSERT_EQ(1000, v.size()); + // Since the block cache is large enough, all the blocks are cached. we + // do not need to lookup the seondary cache. + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 2u); + + Destroy(options); +} + +TEST_P(DBSecondaryCacheTest, SecondaryCacheIntensiveTesting) { + std::shared_ptr secondary_cache( + new TestSecondaryCache(2048 * 1024)); + std::shared_ptr cache = + NewCache(8 * 1024 /* capacity */, 0 /* num_shard_bits */, + false /* strict_capacity_limit */, secondary_cache); + BlockBasedTableOptions table_options; + table_options.block_cache = cache; + table_options.block_size = 4 * 1024; + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.env = fault_env_.get(); + fault_fs_->SetFailGetUniqueId(true); + DestroyAndReopen(options); + Random rnd(301); + const int N = 256; + for (int i = 0; i < N; i++) { + std::string p_v = rnd.RandomString(1000); + ASSERT_OK(Put(Key(i), p_v)); + } + ASSERT_OK(Flush()); + Compact("a", "z"); + + Random r_index(47); + std::string v; + for (int i = 0; i < 1000; i++) { + uint32_t key_i = r_index.Next() % N; + v = Get(Key(key_i)); + } + + // We have over 200 data blocks there will be multiple insertion + // and lookups. + ASSERT_GE(secondary_cache->num_inserts(), 1u); + ASSERT_GE(secondary_cache->num_lookups(), 1u); + + Destroy(options); +} + +// In this test, the block cache size is set to 4096, after insert 6 KV-pairs +// and flush, there are 5 blocks in this SST file, 2 data blocks and 3 meta +// blocks. block_1 size is 4096 and block_2 size is 2056. The total size +// of the meta blocks are about 900 to 1000. Therefore, in any situation, +// if we try to insert block_1 to the block cache, it will always fails. Only +// block_2 will be successfully inserted into the block cache. +TEST_P(DBSecondaryCacheTest, SecondaryCacheFailureTest) { + if (GetParam() == kHyperClock) { + ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors"); + return; + } + std::shared_ptr secondary_cache( + new TestSecondaryCache(2048 * 1024)); + std::shared_ptr cache = + NewCache(4 * 1024 /* capacity */, 0 /* num_shard_bits */, + false /* strict_capacity_limit */, secondary_cache); + BlockBasedTableOptions table_options; + table_options.block_cache = cache; + table_options.block_size = 4 * 1024; + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.paranoid_file_checks = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.env = fault_env_.get(); + fault_fs_->SetFailGetUniqueId(true); + DestroyAndReopen(options); + Random rnd(301); + const int N = 6; + for (int i = 0; i < N; i++) { + std::string p_v = rnd.RandomString(1007); + ASSERT_OK(Put(Key(i), p_v)); + } + + ASSERT_OK(Flush()); + // After Flush is successful, RocksDB will do the paranoid check for the new + // SST file. Meta blocks are always cached in the block cache and they + // will not be evicted. When block_2 is cache miss and read out, it is + // inserted to the block cache. Note that, block_1 is never successfully + // inserted to the block cache. Here are 2 lookups in the secondary cache + // for block_1 and block_2 + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 2u); + + // Fail the insertion, in LRU cache, the secondary insertion returned status + // is not checked, therefore, the DB will not be influenced. + secondary_cache->InjectFailure(); + Compact("a", "z"); + // Compaction will create the iterator to scan the whole file. So all the + // blocks are needed. Meta blocks are always cached. When block_1 is read + // out, block_2 is evicted from block cache and inserted to secondary + // cache. + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 3u); + + std::string v = Get(Key(0)); + ASSERT_EQ(1007, v.size()); + // The first data block is not in the cache, similarly, trigger the block + // cache Lookup and secondary cache lookup for block_1. But block_1 will not + // be inserted successfully due to the size. Currently, cache only has + // the meta blocks. + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 4u); + + v = Get(Key(5)); + ASSERT_EQ(1007, v.size()); + // The second data block is not in the cache, similarly, trigger the block + // cache Lookup and secondary cache lookup for block_2 and block_2 is found + // in the secondary cache. Now block cache has block_2 + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 5u); + + v = Get(Key(5)); + ASSERT_EQ(1007, v.size()); + // block_2 is in the block cache. There is a block cache hit. No need to + // lookup or insert the secondary cache. + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 5u); + + v = Get(Key(0)); + ASSERT_EQ(1007, v.size()); + // Lookup the first data block, not in the block cache, so lookup the + // secondary cache. Also not in the secondary cache. After Get, still + // block_1 is will not be cached. + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 6u); + + v = Get(Key(0)); + ASSERT_EQ(1007, v.size()); + // Lookup the first data block, not in the block cache, so lookup the + // secondary cache. Also not in the secondary cache. After Get, still + // block_1 is will not be cached. + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 7u); + secondary_cache->ResetInjectFailure(); + + Destroy(options); +} + +TEST_P(BasicSecondaryCacheTest, BasicWaitAllTest) { + std::shared_ptr secondary_cache = + std::make_shared(32 * 1024); + std::shared_ptr cache = + NewCache(1024 /* capacity */, 2 /* num_shard_bits */, + false /* strict_capacity_limit */, secondary_cache); + const int num_keys = 32; + OffsetableCacheKey ock{"foo", "bar", 1}; + + Random rnd(301); + std::vector values; + for (int i = 0; i < num_keys; ++i) { + std::string str = rnd.RandomString(1020); + values.emplace_back(str); + TestItem* item = new TestItem(str.data(), str.length()); + ASSERT_OK(cache->Insert(ock.WithOffset(i).AsSlice(), item, GetHelper(), + str.length())); + } + // Force all entries to be evicted to the secondary cache + if (GetParam() == kHyperClock) { + // HCC doesn't respond immediately to SetCapacity + for (int i = 9000; i < 9030; ++i) { + ASSERT_OK(cache->Insert(ock.WithOffset(i).AsSlice(), nullptr, + &kNoopCacheItemHelper, 256)); + } + } else { + cache->SetCapacity(0); + } + ASSERT_EQ(secondary_cache->num_inserts(), 32u); + cache->SetCapacity(32 * 1024); + + secondary_cache->SetResultMap( + {{ock.WithOffset(3).AsSlice().ToString(), + TestSecondaryCache::ResultType::DEFER}, + {ock.WithOffset(4).AsSlice().ToString(), + TestSecondaryCache::ResultType::DEFER_AND_FAIL}, + {ock.WithOffset(5).AsSlice().ToString(), + TestSecondaryCache::ResultType::FAIL}}); + + std::array async_handles; + std::array cache_keys; + for (size_t i = 0; i < async_handles.size(); ++i) { + auto& ah = async_handles[i]; + cache_keys[i] = ock.WithOffset(i); + ah.key = cache_keys[i].AsSlice(); + ah.helper = GetHelper(); + ah.create_context = this; + ah.priority = Cache::Priority::LOW; + cache->StartAsyncLookup(ah); + } + cache->WaitAll(&async_handles[0], async_handles.size()); + for (size_t i = 0; i < async_handles.size(); ++i) { + SCOPED_TRACE("i = " + std::to_string(i)); + Cache::Handle* result = async_handles[i].Result(); + if (i == 4 || i == 5) { + ASSERT_EQ(result, nullptr); + continue; + } else { + ASSERT_NE(result, nullptr); + TestItem* item = static_cast(cache->Value(result)); + ASSERT_EQ(item->ToString(), values[i]); + } + cache->Release(result); + } + + cache.reset(); + secondary_cache.reset(); +} + +// In this test, we have one KV pair per data block. We indirectly determine +// the cache key associated with each data block (and thus each KV) by using +// a sync point callback in TestSecondaryCache::Lookup. We then control the +// lookup result by setting the ResultMap. +TEST_P(DBSecondaryCacheTest, TestSecondaryCacheMultiGet) { + if (GetParam() == kHyperClock) { + ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors"); + return; + } + std::shared_ptr secondary_cache( + new TestSecondaryCache(2048 * 1024)); + std::shared_ptr cache = + NewCache(1 << 20 /* capacity */, 0 /* num_shard_bits */, + false /* strict_capacity_limit */, secondary_cache); + BlockBasedTableOptions table_options; + table_options.block_cache = cache; + table_options.block_size = 4 * 1024; + table_options.cache_index_and_filter_blocks = false; + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.paranoid_file_checks = true; + DestroyAndReopen(options); + Random rnd(301); + const int N = 8; + std::vector keys; + for (int i = 0; i < N; i++) { + std::string p_v = rnd.RandomString(4000); + keys.emplace_back(p_v); + ASSERT_OK(Put(Key(i), p_v)); + } + + ASSERT_OK(Flush()); + // After Flush is successful, RocksDB does the paranoid check for the new + // SST file. This will try to lookup all data blocks in the secondary + // cache. + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 8u); + + cache->SetCapacity(0); + ASSERT_EQ(secondary_cache->num_inserts(), 8u); + cache->SetCapacity(1 << 20); + + std::vector cache_keys; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "TestSecondaryCache::Lookup", [&cache_keys](void* key) -> void { + cache_keys.emplace_back(*(static_cast(key))); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + for (int i = 0; i < N; ++i) { + std::string v = Get(Key(i)); + ASSERT_EQ(4000, v.size()); + ASSERT_EQ(v, keys[i]); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ASSERT_EQ(secondary_cache->num_lookups(), 16u); + cache->SetCapacity(0); + cache->SetCapacity(1 << 20); + + ASSERT_EQ(Get(Key(2)), keys[2]); + ASSERT_EQ(Get(Key(7)), keys[7]); + secondary_cache->SetResultMap( + {{cache_keys[3], TestSecondaryCache::ResultType::DEFER}, + {cache_keys[4], TestSecondaryCache::ResultType::DEFER_AND_FAIL}, + {cache_keys[5], TestSecondaryCache::ResultType::FAIL}}); + + std::vector mget_keys( + {Key(0), Key(1), Key(2), Key(3), Key(4), Key(5), Key(6), Key(7)}); + std::vector values(mget_keys.size()); + std::vector s(keys.size()); + std::vector key_slices; + for (const std::string& key : mget_keys) { + key_slices.emplace_back(key); + } + uint32_t num_lookups = secondary_cache->num_lookups(); + dbfull()->MultiGet(ReadOptions(), dbfull()->DefaultColumnFamily(), + key_slices.size(), key_slices.data(), values.data(), + s.data(), false); + ASSERT_EQ(secondary_cache->num_lookups(), num_lookups + 5); + for (int i = 0; i < N; ++i) { + ASSERT_OK(s[i]); + ASSERT_EQ(values[i].ToString(), keys[i]); + values[i].Reset(); + } + Destroy(options); +} + +class CacheWithStats : public CacheWrapper { + public: + using CacheWrapper::CacheWrapper; + + static const char* kClassName() { return "CacheWithStats"; } + const char* Name() const override { return kClassName(); } + + Status Insert(const Slice& key, Cache::ObjectPtr value, + const CacheItemHelper* helper, size_t charge, + Handle** handle = nullptr, + Priority priority = Priority::LOW) override { + insert_count_++; + return target_->Insert(key, value, helper, charge, handle, priority); + } + Handle* Lookup(const Slice& key, const CacheItemHelper* helper, + CreateContext* create_context, Priority priority, + Statistics* stats = nullptr) override { + lookup_count_++; + return target_->Lookup(key, helper, create_context, priority, stats); + } + + uint32_t GetInsertCount() { return insert_count_; } + uint32_t GetLookupcount() { return lookup_count_; } + void ResetCount() { + insert_count_ = 0; + lookup_count_ = 0; + } + + private: + uint32_t insert_count_ = 0; + uint32_t lookup_count_ = 0; +}; + +TEST_P(DBSecondaryCacheTest, LRUCacheDumpLoadBasic) { + std::shared_ptr base_cache = + NewCache(1024 * 1024 /* capacity */, 0 /* num_shard_bits */, + false /* strict_capacity_limit */); + std::shared_ptr cache = + std::make_shared(base_cache); + BlockBasedTableOptions table_options; + table_options.block_cache = cache; + table_options.block_size = 4 * 1024; + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.env = fault_env_.get(); + DestroyAndReopen(options); + fault_fs_->SetFailGetUniqueId(true); + + Random rnd(301); + const int N = 256; + std::vector value; + char buf[1000]; + memset(buf, 'a', 1000); + value.resize(N); + for (int i = 0; i < N; i++) { + // std::string p_v = rnd.RandomString(1000); + std::string p_v(buf, 1000); + value[i] = p_v; + ASSERT_OK(Put(Key(i), p_v)); + } + ASSERT_OK(Flush()); + Compact("a", "z"); + + // do th eread for all the key value pairs, so all the blocks should be in + // cache + uint32_t start_insert = cache->GetInsertCount(); + uint32_t start_lookup = cache->GetLookupcount(); + std::string v; + for (int i = 0; i < N; i++) { + v = Get(Key(i)); + ASSERT_EQ(v, value[i]); + } + uint32_t dump_insert = cache->GetInsertCount() - start_insert; + uint32_t dump_lookup = cache->GetLookupcount() - start_lookup; + ASSERT_EQ(63, + static_cast(dump_insert)); // the insert in the block cache + ASSERT_EQ(256, + static_cast(dump_lookup)); // the lookup in the block cache + // We have enough blocks in the block cache + + CacheDumpOptions cd_options; + cd_options.clock = fault_env_->GetSystemClock().get(); + std::string dump_path = db_->GetName() + "/cache_dump"; + std::unique_ptr dump_writer; + Status s = NewToFileCacheDumpWriter(fault_fs_, FileOptions(), dump_path, + &dump_writer); + ASSERT_OK(s); + std::unique_ptr cache_dumper; + s = NewDefaultCacheDumper(cd_options, cache, std::move(dump_writer), + &cache_dumper); + ASSERT_OK(s); + std::vector db_list; + db_list.push_back(db_); + s = cache_dumper->SetDumpFilter(db_list); + ASSERT_OK(s); + s = cache_dumper->DumpCacheEntriesToWriter(); + ASSERT_OK(s); + cache_dumper.reset(); + + // we have a new cache it is empty, then, before we do the Get, we do the + // dumpload + std::shared_ptr secondary_cache = + std::make_shared(2048 * 1024); + // This time with secondary cache + base_cache = NewCache(1024 * 1024 /* capacity */, 0 /* num_shard_bits */, + false /* strict_capacity_limit */, secondary_cache); + cache = std::make_shared(base_cache); + table_options.block_cache = cache; + table_options.block_size = 4 * 1024; + options.create_if_missing = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.env = fault_env_.get(); + + // start to load the data to new block cache + start_insert = secondary_cache->num_inserts(); + start_lookup = secondary_cache->num_lookups(); + std::unique_ptr dump_reader; + s = NewFromFileCacheDumpReader(fault_fs_, FileOptions(), dump_path, + &dump_reader); + ASSERT_OK(s); + std::unique_ptr cache_loader; + s = NewDefaultCacheDumpedLoader(cd_options, table_options, secondary_cache, + std::move(dump_reader), &cache_loader); + ASSERT_OK(s); + s = cache_loader->RestoreCacheEntriesToSecondaryCache(); + ASSERT_OK(s); + uint32_t load_insert = secondary_cache->num_inserts() - start_insert; + uint32_t load_lookup = secondary_cache->num_lookups() - start_lookup; + // check the number we inserted + ASSERT_EQ(64, static_cast(load_insert)); + ASSERT_EQ(0, static_cast(load_lookup)); + ASSERT_OK(s); + + Reopen(options); + + // After load, we do the Get again + start_insert = secondary_cache->num_inserts(); + start_lookup = secondary_cache->num_lookups(); + uint32_t cache_insert = cache->GetInsertCount(); + uint32_t cache_lookup = cache->GetLookupcount(); + for (int i = 0; i < N; i++) { + v = Get(Key(i)); + ASSERT_EQ(v, value[i]); + } + uint32_t final_insert = secondary_cache->num_inserts() - start_insert; + uint32_t final_lookup = secondary_cache->num_lookups() - start_lookup; + // no insert to secondary cache + ASSERT_EQ(0, static_cast(final_insert)); + // lookup the secondary to get all blocks + ASSERT_EQ(64, static_cast(final_lookup)); + uint32_t block_insert = cache->GetInsertCount() - cache_insert; + uint32_t block_lookup = cache->GetLookupcount() - cache_lookup; + // Check the new block cache insert and lookup, should be no insert since all + // blocks are from the secondary cache. + ASSERT_EQ(0, static_cast(block_insert)); + ASSERT_EQ(256, static_cast(block_lookup)); + + fault_fs_->SetFailGetUniqueId(false); + Destroy(options); +} + +TEST_P(DBSecondaryCacheTest, LRUCacheDumpLoadWithFilter) { + std::shared_ptr base_cache = + NewCache(1024 * 1024 /* capacity */, 0 /* num_shard_bits */, + false /* strict_capacity_limit */); + std::shared_ptr cache = + std::make_shared(base_cache); + BlockBasedTableOptions table_options; + table_options.block_cache = cache; + table_options.block_size = 4 * 1024; + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.env = fault_env_.get(); + std::string dbname1 = test::PerThreadDBPath("db_1"); + ASSERT_OK(DestroyDB(dbname1, options)); + DB* db1 = nullptr; + ASSERT_OK(DB::Open(options, dbname1, &db1)); + std::string dbname2 = test::PerThreadDBPath("db_2"); + ASSERT_OK(DestroyDB(dbname2, options)); + DB* db2 = nullptr; + ASSERT_OK(DB::Open(options, dbname2, &db2)); + fault_fs_->SetFailGetUniqueId(true); + + // write the KVs to db1 + Random rnd(301); + const int N = 256; + std::vector value1; + WriteOptions wo; + char buf[1000]; + memset(buf, 'a', 1000); + value1.resize(N); + for (int i = 0; i < N; i++) { + std::string p_v(buf, 1000); + value1[i] = p_v; + ASSERT_OK(db1->Put(wo, Key(i), p_v)); + } + ASSERT_OK(db1->Flush(FlushOptions())); + Slice bg("a"); + Slice ed("b"); + ASSERT_OK(db1->CompactRange(CompactRangeOptions(), &bg, &ed)); + + // Write the KVs to DB2 + std::vector value2; + memset(buf, 'b', 1000); + value2.resize(N); + for (int i = 0; i < N; i++) { + std::string p_v(buf, 1000); + value2[i] = p_v; + ASSERT_OK(db2->Put(wo, Key(i), p_v)); + } + ASSERT_OK(db2->Flush(FlushOptions())); + ASSERT_OK(db2->CompactRange(CompactRangeOptions(), &bg, &ed)); + + // do th eread for all the key value pairs, so all the blocks should be in + // cache + uint32_t start_insert = cache->GetInsertCount(); + uint32_t start_lookup = cache->GetLookupcount(); + ReadOptions ro; + std::string v; + for (int i = 0; i < N; i++) { + ASSERT_OK(db1->Get(ro, Key(i), &v)); + ASSERT_EQ(v, value1[i]); + } + for (int i = 0; i < N; i++) { + ASSERT_OK(db2->Get(ro, Key(i), &v)); + ASSERT_EQ(v, value2[i]); + } + uint32_t dump_insert = cache->GetInsertCount() - start_insert; + uint32_t dump_lookup = cache->GetLookupcount() - start_lookup; + ASSERT_EQ(128, + static_cast(dump_insert)); // the insert in the block cache + ASSERT_EQ(512, + static_cast(dump_lookup)); // the lookup in the block cache + // We have enough blocks in the block cache + + CacheDumpOptions cd_options; + cd_options.clock = fault_env_->GetSystemClock().get(); + std::string dump_path = db1->GetName() + "/cache_dump"; + std::unique_ptr dump_writer; + Status s = NewToFileCacheDumpWriter(fault_fs_, FileOptions(), dump_path, + &dump_writer); + ASSERT_OK(s); + std::unique_ptr cache_dumper; + s = NewDefaultCacheDumper(cd_options, cache, std::move(dump_writer), + &cache_dumper); + ASSERT_OK(s); + std::vector db_list; + db_list.push_back(db1); + s = cache_dumper->SetDumpFilter(db_list); + ASSERT_OK(s); + s = cache_dumper->DumpCacheEntriesToWriter(); + ASSERT_OK(s); + cache_dumper.reset(); + + // we have a new cache it is empty, then, before we do the Get, we do the + // dumpload + std::shared_ptr secondary_cache = + std::make_shared(2048 * 1024); + // This time with secondary_cache + base_cache = NewCache(1024 * 1024 /* capacity */, 0 /* num_shard_bits */, + false /* strict_capacity_limit */, secondary_cache); + cache = std::make_shared(base_cache); + table_options.block_cache = cache; + table_options.block_size = 4 * 1024; + options.create_if_missing = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.env = fault_env_.get(); + + // Start the cache loading process + start_insert = secondary_cache->num_inserts(); + start_lookup = secondary_cache->num_lookups(); + std::unique_ptr dump_reader; + s = NewFromFileCacheDumpReader(fault_fs_, FileOptions(), dump_path, + &dump_reader); + ASSERT_OK(s); + std::unique_ptr cache_loader; + s = NewDefaultCacheDumpedLoader(cd_options, table_options, secondary_cache, + std::move(dump_reader), &cache_loader); + ASSERT_OK(s); + s = cache_loader->RestoreCacheEntriesToSecondaryCache(); + ASSERT_OK(s); + uint32_t load_insert = secondary_cache->num_inserts() - start_insert; + uint32_t load_lookup = secondary_cache->num_lookups() - start_lookup; + // check the number we inserted + ASSERT_EQ(64, static_cast(load_insert)); + ASSERT_EQ(0, static_cast(load_lookup)); + ASSERT_OK(s); + + ASSERT_OK(db1->Close()); + delete db1; + ASSERT_OK(DB::Open(options, dbname1, &db1)); + + // After load, we do the Get again. To validate the cache, we do not allow any + // I/O, so we set the file system to false. + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + fault_fs_->SetFilesystemActive(false, error_msg); + start_insert = secondary_cache->num_inserts(); + start_lookup = secondary_cache->num_lookups(); + uint32_t cache_insert = cache->GetInsertCount(); + uint32_t cache_lookup = cache->GetLookupcount(); + for (int i = 0; i < N; i++) { + ASSERT_OK(db1->Get(ro, Key(i), &v)); + ASSERT_EQ(v, value1[i]); + } + uint32_t final_insert = secondary_cache->num_inserts() - start_insert; + uint32_t final_lookup = secondary_cache->num_lookups() - start_lookup; + // no insert to secondary cache + ASSERT_EQ(0, static_cast(final_insert)); + // lookup the secondary to get all blocks + ASSERT_EQ(64, static_cast(final_lookup)); + uint32_t block_insert = cache->GetInsertCount() - cache_insert; + uint32_t block_lookup = cache->GetLookupcount() - cache_lookup; + // Check the new block cache insert and lookup, should be no insert since all + // blocks are from the secondary cache. + ASSERT_EQ(0, static_cast(block_insert)); + ASSERT_EQ(256, static_cast(block_lookup)); + fault_fs_->SetFailGetUniqueId(false); + fault_fs_->SetFilesystemActive(true); + delete db1; + delete db2; + ASSERT_OK(DestroyDB(dbname1, options)); + ASSERT_OK(DestroyDB(dbname2, options)); +} + +// Test the option not to use the secondary cache in a certain DB. +TEST_P(DBSecondaryCacheTest, TestSecondaryCacheOptionBasic) { + std::shared_ptr secondary_cache( + new TestSecondaryCache(2048 * 1024)); + std::shared_ptr cache = + NewCache(4 * 1024 /* capacity */, 0 /* num_shard_bits */, + false /* strict_capacity_limit */, secondary_cache); + BlockBasedTableOptions table_options; + table_options.block_cache = cache; + table_options.block_size = 4 * 1024; + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.env = fault_env_.get(); + fault_fs_->SetFailGetUniqueId(true); + options.lowest_used_cache_tier = CacheTier::kVolatileTier; + + // Set the file paranoid check, so after flush, the file will be read + // all the blocks will be accessed. + options.paranoid_file_checks = true; + DestroyAndReopen(options); + Random rnd(301); + const int N = 6; + for (int i = 0; i < N; i++) { + std::string p_v = rnd.RandomString(1007); + ASSERT_OK(Put(Key(i), p_v)); + } + + ASSERT_OK(Flush()); + + for (int i = 0; i < N; i++) { + std::string p_v = rnd.RandomString(1007); + ASSERT_OK(Put(Key(i + 70), p_v)); + } + + ASSERT_OK(Flush()); + + // Flush will trigger the paranoid check and read blocks. But only block cache + // will be read. No operations for secondary cache. + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 0u); + + Compact("a", "z"); + + // Compaction will also insert and evict blocks, no operations to the block + // cache. No operations for secondary cache. + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 0u); + + std::string v = Get(Key(0)); + ASSERT_EQ(1007, v.size()); + + // Check the data in first block. Cache miss, direclty read from SST file. + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 0u); + + v = Get(Key(5)); + ASSERT_EQ(1007, v.size()); + + // Check the second block. + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 0u); + + v = Get(Key(5)); + ASSERT_EQ(1007, v.size()); + + // block cache hit + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 0u); + + v = Get(Key(70)); + ASSERT_EQ(1007, v.size()); + + // Check the first block in the second SST file. Cache miss and trigger SST + // file read. No operations for secondary cache. + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 0u); + + v = Get(Key(75)); + ASSERT_EQ(1007, v.size()); + + // Check the second block in the second SST file. Cache miss and trigger SST + // file read. No operations for secondary cache. + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 0u); + + Destroy(options); +} + +// We disable the secondary cache in DBOptions at first. Close and reopen the DB +// with new options, which set the lowest_used_cache_tier to +// kNonVolatileBlockTier. So secondary cache will be used. +TEST_P(DBSecondaryCacheTest, TestSecondaryCacheOptionChange) { + if (GetParam() == kHyperClock) { + ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors"); + return; + } + std::shared_ptr secondary_cache( + new TestSecondaryCache(2048 * 1024)); + std::shared_ptr cache = + NewCache(4 * 1024 /* capacity */, 0 /* num_shard_bits */, + false /* strict_capacity_limit */, secondary_cache); + BlockBasedTableOptions table_options; + table_options.block_cache = cache; + table_options.block_size = 4 * 1024; + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.env = fault_env_.get(); + fault_fs_->SetFailGetUniqueId(true); + options.lowest_used_cache_tier = CacheTier::kVolatileTier; + + // Set the file paranoid check, so after flush, the file will be read + // all the blocks will be accessed. + options.paranoid_file_checks = true; + DestroyAndReopen(options); + Random rnd(301); + const int N = 6; + for (int i = 0; i < N; i++) { + std::string p_v = rnd.RandomString(1007); + ASSERT_OK(Put(Key(i), p_v)); + } + + ASSERT_OK(Flush()); + + for (int i = 0; i < N; i++) { + std::string p_v = rnd.RandomString(1007); + ASSERT_OK(Put(Key(i + 70), p_v)); + } + + ASSERT_OK(Flush()); + + // Flush will trigger the paranoid check and read blocks. But only block cache + // will be read. + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 0u); + + Compact("a", "z"); + + // Compaction will also insert and evict blocks, no operations to the block + // cache. + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 0u); + + std::string v = Get(Key(0)); + ASSERT_EQ(1007, v.size()); + + // Check the data in first block. Cache miss, direclty read from SST file. + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 0u); + + v = Get(Key(5)); + ASSERT_EQ(1007, v.size()); + + // Check the second block. + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 0u); + + v = Get(Key(5)); + ASSERT_EQ(1007, v.size()); + + // block cache hit + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 0u); + + // Change the option to enable secondary cache after we Reopen the DB + options.lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier; + Reopen(options); + + v = Get(Key(70)); + ASSERT_EQ(1007, v.size()); + + // Enable the secondary cache, trigger lookup of the first block in second SST + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 1u); + + v = Get(Key(75)); + ASSERT_EQ(1007, v.size()); + + // trigger lookup of the second block in second SST + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 2u); + Destroy(options); +} + +// Two DB test. We create 2 DBs sharing the same block cache and secondary +// cache. We diable the secondary cache option for DB2. +TEST_P(DBSecondaryCacheTest, TestSecondaryCacheOptionTwoDB) { + if (GetParam() == kHyperClock) { + ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors"); + return; + } + std::shared_ptr secondary_cache( + new TestSecondaryCache(2048 * 1024)); + std::shared_ptr cache = + NewCache(4 * 1024 /* capacity */, 0 /* num_shard_bits */, + false /* strict_capacity_limit */, secondary_cache); + BlockBasedTableOptions table_options; + table_options.block_cache = cache; + table_options.block_size = 4 * 1024; + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.env = fault_env_.get(); + options.paranoid_file_checks = true; + std::string dbname1 = test::PerThreadDBPath("db_t_1"); + ASSERT_OK(DestroyDB(dbname1, options)); + DB* db1 = nullptr; + ASSERT_OK(DB::Open(options, dbname1, &db1)); + std::string dbname2 = test::PerThreadDBPath("db_t_2"); + ASSERT_OK(DestroyDB(dbname2, options)); + DB* db2 = nullptr; + Options options2 = options; + options2.lowest_used_cache_tier = CacheTier::kVolatileTier; + ASSERT_OK(DB::Open(options2, dbname2, &db2)); + fault_fs_->SetFailGetUniqueId(true); + + WriteOptions wo; + Random rnd(301); + const int N = 6; + for (int i = 0; i < N; i++) { + std::string p_v = rnd.RandomString(1007); + ASSERT_OK(db1->Put(wo, Key(i), p_v)); + } + + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 0u); + ASSERT_OK(db1->Flush(FlushOptions())); + + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 2u); + + for (int i = 0; i < N; i++) { + std::string p_v = rnd.RandomString(1007); + ASSERT_OK(db2->Put(wo, Key(i), p_v)); + } + + // No change in the secondary cache, since it is disabled in DB2 + ASSERT_EQ(secondary_cache->num_inserts(), 0u); + ASSERT_EQ(secondary_cache->num_lookups(), 2u); + ASSERT_OK(db2->Flush(FlushOptions())); + ASSERT_EQ(secondary_cache->num_inserts(), 1u); + ASSERT_EQ(secondary_cache->num_lookups(), 2u); + + Slice bg("a"); + Slice ed("b"); + ASSERT_OK(db1->CompactRange(CompactRangeOptions(), &bg, &ed)); + ASSERT_OK(db2->CompactRange(CompactRangeOptions(), &bg, &ed)); + + ASSERT_EQ(secondary_cache->num_inserts(), 1u); + ASSERT_EQ(secondary_cache->num_lookups(), 2u); + + ReadOptions ro; + std::string v; + ASSERT_OK(db1->Get(ro, Key(0), &v)); + ASSERT_EQ(1007, v.size()); + + // DB 1 has lookup block 1 and it is miss in block cache, trigger secondary + // cache lookup + ASSERT_EQ(secondary_cache->num_inserts(), 1u); + ASSERT_EQ(secondary_cache->num_lookups(), 3u); + + ASSERT_OK(db1->Get(ro, Key(5), &v)); + ASSERT_EQ(1007, v.size()); + + // DB 1 lookup the second block and it is miss in block cache, trigger + // secondary cache lookup + ASSERT_EQ(secondary_cache->num_inserts(), 1u); + ASSERT_EQ(secondary_cache->num_lookups(), 4u); + + ASSERT_OK(db2->Get(ro, Key(0), &v)); + ASSERT_EQ(1007, v.size()); + + // For db2, it is not enabled with secondary cache, so no search in the + // secondary cache + ASSERT_EQ(secondary_cache->num_inserts(), 1u); + ASSERT_EQ(secondary_cache->num_lookups(), 4u); + + ASSERT_OK(db2->Get(ro, Key(5), &v)); + ASSERT_EQ(1007, v.size()); + + // For db2, it is not enabled with secondary cache, so no search in the + // secondary cache + ASSERT_EQ(secondary_cache->num_inserts(), 1u); + ASSERT_EQ(secondary_cache->num_lookups(), 4u); + + fault_fs_->SetFailGetUniqueId(false); + fault_fs_->SetFilesystemActive(true); + delete db1; + delete db2; + ASSERT_OK(DestroyDB(dbname1, options)); + ASSERT_OK(DestroyDB(dbname2, options)); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/cache/secondary_cache.cc b/librocksdb-sys/rocksdb/cache/secondary_cache.cc new file mode 100644 index 0000000..8eba8b0 --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/secondary_cache.cc @@ -0,0 +1,44 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "rocksdb/secondary_cache.h" + +#include "cache/cache_entry_roles.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { + +void NoopDelete(Cache::ObjectPtr, MemoryAllocator*) {} + +size_t SliceSize(Cache::ObjectPtr obj) { + return static_cast(obj)->size(); +} + +Status SliceSaveTo(Cache::ObjectPtr from_obj, size_t from_offset, size_t length, + char* out) { + const Slice& slice = *static_cast(from_obj); + std::memcpy(out, slice.data() + from_offset, length); + return Status::OK(); +} + +Status FailCreate(const Slice&, Cache::CreateContext*, MemoryAllocator*, + Cache::ObjectPtr*, size_t*) { + return Status::NotSupported("Only for dumping data into SecondaryCache"); +} + +} // namespace + +Status SecondaryCache::InsertSaved(const Slice& key, const Slice& saved) { + static Cache::CacheItemHelper helper_no_secondary{CacheEntryRole::kMisc, + &NoopDelete}; + static Cache::CacheItemHelper helper{ + CacheEntryRole::kMisc, &NoopDelete, &SliceSize, + &SliceSaveTo, &FailCreate, &helper_no_secondary}; + // NOTE: depends on Insert() being synchronous, not keeping pointer `&saved` + return Insert(key, const_cast(&saved), &helper); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/secondary_cache_adapter.cc b/librocksdb-sys/rocksdb/cache/secondary_cache_adapter.cc new file mode 100644 index 0000000..06441a1 --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/secondary_cache_adapter.cc @@ -0,0 +1,433 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "cache/secondary_cache_adapter.h" + +#include "monitoring/perf_context_imp.h" +#include "util/cast_util.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { +// A distinct pointer value for marking "dummy" cache entries +struct Dummy { + char val[7] = "kDummy"; +}; +const Dummy kDummy{}; +Cache::ObjectPtr const kDummyObj = const_cast(&kDummy); +} // namespace + +// When CacheWithSecondaryAdapter is constructed with the distribute_cache_res +// parameter set to true, it manages the entire memory budget across the +// primary and secondary cache. The secondary cache is assumed to be in +// memory, such as the CompressedSecondaryCache. When a placeholder entry +// is inserted by a CacheReservationManager instance to reserve memory, +// the CacheWithSecondaryAdapter ensures that the reservation is distributed +// proportionally across the primary/secondary caches. +// +// The primary block cache is initially sized to the sum of the primary cache +// budget + teh secondary cache budget, as follows - +// |--------- Primary Cache Configured Capacity -----------| +// |---Secondary Cache Budget----|----Primary Cache Budget-----| +// +// A ConcurrentCacheReservationManager member in the CacheWithSecondaryAdapter, +// pri_cache_res_, +// is used to help with tracking the distribution of memory reservations. +// Initially, it accounts for the entire secondary cache budget as a +// reservation against the primary cache. This shrinks the usable capacity of +// the primary cache to the budget that the user originally desired. +// +// |--Reservation for Sec Cache--|-Pri Cache Usable Capacity---| +// +// When a reservation placeholder is inserted into the adapter, it is inserted +// directly into the primary cache. This means the entire charge of the +// placeholder is counted against the primary cache. To compensate and count +// a portion of it against the secondary cache, the secondary cache Deflate() +// method is called to shrink it. Since the Deflate() causes the secondary +// actual usage to shrink, it is refelcted here by releasing an equal amount +// from the pri_cache_res_ reservation. The Deflate() in the secondary cache +// can be, but is not required to be, implemented using its own cache +// reservation manager. +// +// For example, if the pri/sec ratio is 70/30, and the combined capacity is +// 100MB, the intermediate and final state after inserting a reservation +// placeholder for 10MB would be as follows - +// +// |-Reservation for Sec Cache-|-Pri Cache Usable Capacity-|---R---| +// 1. After inserting the placeholder in primary +// |------- 30MB -------------|------- 60MB -------------|-10MB--| +// 2. After deflating the secondary and adjusting the reservation for +// secondary against the primary +// |------- 27MB -------------|------- 63MB -------------|-10MB--| +// +// Likewise, when the user inserted placeholder is released, the secondary +// cache Inflate() method is called to grow it, and the pri_cache_res_ +// reservation is increased by an equal amount. +// +// Another way of implementing this would have been to simply split the user +// reservation into primary and seconary components. However, this would +// require allocating a structure to track the associated secondary cache +// reservation, which adds some complexity and overhead. +// +CacheWithSecondaryAdapter::CacheWithSecondaryAdapter( + std::shared_ptr target, + std::shared_ptr secondary_cache, bool distribute_cache_res) + : CacheWrapper(std::move(target)), + secondary_cache_(std::move(secondary_cache)), + distribute_cache_res_(distribute_cache_res) { + target_->SetEvictionCallback([this](const Slice& key, Handle* handle) { + return EvictionHandler(key, handle); + }); + if (distribute_cache_res_) { + size_t sec_capacity = 0; + pri_cache_res_ = std::make_shared( + std::make_shared>( + target_)); + Status s = secondary_cache_->GetCapacity(sec_capacity); + assert(s.ok()); + // Initially, the primary cache is sized to uncompressed cache budget plsu + // compressed secondary cache budget. The secondary cache budget is then + // taken away from the primary cache through cache reservations. Later, + // when a placeholder entry is inserted by the caller, its inserted + // into the primary cache and the portion that should be assigned to the + // secondary cache is freed from the reservation. + s = pri_cache_res_->UpdateCacheReservation(sec_capacity); + assert(s.ok()); + sec_cache_res_ratio_ = (double)sec_capacity / target_->GetCapacity(); + } +} + +CacheWithSecondaryAdapter::~CacheWithSecondaryAdapter() { + // `*this` will be destroyed before `*target_`, so we have to prevent + // use after free + target_->SetEvictionCallback({}); +#ifndef NDEBUG + if (distribute_cache_res_) { + size_t sec_capacity = 0; + Status s = secondary_cache_->GetCapacity(sec_capacity); + assert(s.ok()); + assert(pri_cache_res_->GetTotalReservedCacheSize() == sec_capacity); + } +#endif // NDEBUG +} + +bool CacheWithSecondaryAdapter::EvictionHandler(const Slice& key, + Handle* handle) { + auto helper = GetCacheItemHelper(handle); + if (helper->IsSecondaryCacheCompatible()) { + auto obj = target_->Value(handle); + // Ignore dummy entry + if (obj != kDummyObj) { + // Spill into secondary cache. + secondary_cache_->Insert(key, obj, helper).PermitUncheckedError(); + } + } + // Never takes ownership of obj + return false; +} + +bool CacheWithSecondaryAdapter::ProcessDummyResult(Cache::Handle** handle, + bool erase) { + if (*handle && target_->Value(*handle) == kDummyObj) { + target_->Release(*handle, erase); + *handle = nullptr; + return true; + } else { + return false; + } +} + +void CacheWithSecondaryAdapter::CleanupCacheObject( + ObjectPtr obj, const CacheItemHelper* helper) { + if (helper->del_cb) { + helper->del_cb(obj, memory_allocator()); + } +} + +Cache::Handle* CacheWithSecondaryAdapter::Promote( + std::unique_ptr&& secondary_handle, + const Slice& key, const CacheItemHelper* helper, Priority priority, + Statistics* stats, bool found_dummy_entry, bool kept_in_sec_cache) { + assert(secondary_handle->IsReady()); + + ObjectPtr obj = secondary_handle->Value(); + if (!obj) { + // Nothing found. + return nullptr; + } + // Found something. + switch (helper->role) { + case CacheEntryRole::kFilterBlock: + RecordTick(stats, SECONDARY_CACHE_FILTER_HITS); + break; + case CacheEntryRole::kIndexBlock: + RecordTick(stats, SECONDARY_CACHE_INDEX_HITS); + break; + case CacheEntryRole::kDataBlock: + RecordTick(stats, SECONDARY_CACHE_DATA_HITS); + break; + default: + break; + } + PERF_COUNTER_ADD(secondary_cache_hit_count, 1); + RecordTick(stats, SECONDARY_CACHE_HITS); + + // Note: SecondaryCache::Size() is really charge (from the CreateCallback) + size_t charge = secondary_handle->Size(); + Handle* result = nullptr; + // Insert into primary cache, possibly as a standalone+dummy entries. + if (secondary_cache_->SupportForceErase() && !found_dummy_entry) { + // Create standalone and insert dummy + // Allow standalone to be created even if cache is full, to avoid + // reading the entry from storage. + result = + CreateStandalone(key, obj, helper, charge, /*allow_uncharged*/ true); + assert(result); + PERF_COUNTER_ADD(block_cache_standalone_handle_count, 1); + + // Insert dummy to record recent use + // TODO: try to avoid case where inserting this dummy could overwrite a + // regular entry + Status s = Insert(key, kDummyObj, &kNoopCacheItemHelper, /*charge=*/0, + /*handle=*/nullptr, priority); + s.PermitUncheckedError(); + // Nothing to do or clean up on dummy insertion failure + } else { + // Insert regular entry into primary cache. + // Don't allow it to spill into secondary cache again if it was kept there. + Status s = Insert( + key, obj, kept_in_sec_cache ? helper->without_secondary_compat : helper, + charge, &result, priority); + if (s.ok()) { + assert(result); + PERF_COUNTER_ADD(block_cache_real_handle_count, 1); + } else { + // Create standalone result instead, even if cache is full, to avoid + // reading the entry from storage. + result = + CreateStandalone(key, obj, helper, charge, /*allow_uncharged*/ true); + assert(result); + PERF_COUNTER_ADD(block_cache_standalone_handle_count, 1); + } + } + return result; +} + +Status CacheWithSecondaryAdapter::Insert(const Slice& key, ObjectPtr value, + const CacheItemHelper* helper, + size_t charge, Handle** handle, + Priority priority) { + Status s = target_->Insert(key, value, helper, charge, handle, priority); + if (s.ok() && value == nullptr && distribute_cache_res_) { + size_t sec_charge = static_cast(charge * (sec_cache_res_ratio_)); + s = secondary_cache_->Deflate(sec_charge); + assert(s.ok()); + s = pri_cache_res_->UpdateCacheReservation(sec_charge, /*increase=*/false); + assert(s.ok()); + } + + return s; +} + +Cache::Handle* CacheWithSecondaryAdapter::Lookup(const Slice& key, + const CacheItemHelper* helper, + CreateContext* create_context, + Priority priority, + Statistics* stats) { + // NOTE: we could just StartAsyncLookup() and Wait(), but this should be a bit + // more efficient + Handle* result = + target_->Lookup(key, helper, create_context, priority, stats); + bool secondary_compatible = helper && helper->IsSecondaryCacheCompatible(); + bool found_dummy_entry = + ProcessDummyResult(&result, /*erase=*/secondary_compatible); + if (!result && secondary_compatible) { + // Try our secondary cache + bool kept_in_sec_cache = false; + std::unique_ptr secondary_handle = + secondary_cache_->Lookup(key, helper, create_context, /*wait*/ true, + found_dummy_entry, /*out*/ kept_in_sec_cache); + if (secondary_handle) { + result = Promote(std::move(secondary_handle), key, helper, priority, + stats, found_dummy_entry, kept_in_sec_cache); + } + } + return result; +} + +bool CacheWithSecondaryAdapter::Release(Handle* handle, + bool erase_if_last_ref) { + if (erase_if_last_ref) { + ObjectPtr v = target_->Value(handle); + if (v == nullptr && distribute_cache_res_) { + size_t charge = target_->GetCharge(handle); + size_t sec_charge = static_cast(charge * (sec_cache_res_ratio_)); + Status s = secondary_cache_->Inflate(sec_charge); + assert(s.ok()); + s = pri_cache_res_->UpdateCacheReservation(sec_charge, /*increase=*/true); + assert(s.ok()); + } + } + return target_->Release(handle, erase_if_last_ref); +} + +Cache::ObjectPtr CacheWithSecondaryAdapter::Value(Handle* handle) { + ObjectPtr v = target_->Value(handle); + // TODO with stacked secondaries: might fail in EvictionHandler + assert(v != kDummyObj); + return v; +} + +void CacheWithSecondaryAdapter::StartAsyncLookupOnMySecondary( + AsyncLookupHandle& async_handle) { + assert(!async_handle.IsPending()); + assert(async_handle.result_handle == nullptr); + + std::unique_ptr secondary_handle = + secondary_cache_->Lookup(async_handle.key, async_handle.helper, + async_handle.create_context, /*wait*/ false, + async_handle.found_dummy_entry, + /*out*/ async_handle.kept_in_sec_cache); + if (secondary_handle) { + // TODO with stacked secondaries: Check & process if already ready? + async_handle.pending_handle = secondary_handle.release(); + async_handle.pending_cache = secondary_cache_.get(); + } +} + +void CacheWithSecondaryAdapter::StartAsyncLookup( + AsyncLookupHandle& async_handle) { + target_->StartAsyncLookup(async_handle); + if (!async_handle.IsPending()) { + bool secondary_compatible = + async_handle.helper && + async_handle.helper->IsSecondaryCacheCompatible(); + async_handle.found_dummy_entry |= ProcessDummyResult( + &async_handle.result_handle, /*erase=*/secondary_compatible); + + if (async_handle.Result() == nullptr && secondary_compatible) { + // Not found and not pending on another secondary cache + StartAsyncLookupOnMySecondary(async_handle); + } + } +} + +void CacheWithSecondaryAdapter::WaitAll(AsyncLookupHandle* async_handles, + size_t count) { + if (count == 0) { + // Nothing to do + return; + } + // Requests that are pending on *my* secondary cache, at the start of this + // function + std::vector my_pending; + // Requests that are pending on an "inner" secondary cache (managed somewhere + // under target_), as of the start of this function + std::vector inner_pending; + + // Initial accounting of pending handles, excluding those already handled + // by "outer" secondary caches. (See cur->pending_cache = nullptr.) + for (size_t i = 0; i < count; ++i) { + AsyncLookupHandle* cur = async_handles + i; + if (cur->pending_cache) { + assert(cur->IsPending()); + assert(cur->helper); + assert(cur->helper->IsSecondaryCacheCompatible()); + if (cur->pending_cache == secondary_cache_.get()) { + my_pending.push_back(cur); + // Mark as "to be handled by this caller" + cur->pending_cache = nullptr; + } else { + // Remember as potentially needing a lookup in my secondary + inner_pending.push_back(cur); + } + } + } + + // Wait on inner-most cache lookups first + // TODO with stacked secondaries: because we are not using proper + // async/await constructs here yet, there is a false synchronization point + // here where all the results at one level are needed before initiating + // any lookups at the next level. Probably not a big deal, but worth noting. + if (!inner_pending.empty()) { + target_->WaitAll(async_handles, count); + } + + // For those that failed to find something, convert to lookup in my + // secondary cache. + for (AsyncLookupHandle* cur : inner_pending) { + if (cur->Result() == nullptr) { + // Not found, try my secondary + StartAsyncLookupOnMySecondary(*cur); + if (cur->IsPending()) { + assert(cur->pending_cache == secondary_cache_.get()); + my_pending.push_back(cur); + // Mark as "to be handled by this caller" + cur->pending_cache = nullptr; + } + } + } + + // Wait on all lookups on my secondary cache + { + std::vector my_secondary_handles; + for (AsyncLookupHandle* cur : my_pending) { + my_secondary_handles.push_back(cur->pending_handle); + } + secondary_cache_->WaitAll(std::move(my_secondary_handles)); + } + + // Process results + for (AsyncLookupHandle* cur : my_pending) { + std::unique_ptr secondary_handle( + cur->pending_handle); + cur->pending_handle = nullptr; + cur->result_handle = Promote( + std::move(secondary_handle), cur->key, cur->helper, cur->priority, + cur->stats, cur->found_dummy_entry, cur->kept_in_sec_cache); + assert(cur->pending_cache == nullptr); + } +} + +std::string CacheWithSecondaryAdapter::GetPrintableOptions() const { + std::string str = target_->GetPrintableOptions(); + str.append(" secondary_cache:\n"); + str.append(secondary_cache_->GetPrintableOptions()); + return str; +} + +const char* CacheWithSecondaryAdapter::Name() const { + // To the user, at least for now, configure the underlying cache with + // a secondary cache. So we pretend to be that cache + return target_->Name(); +} + +std::shared_ptr NewTieredVolatileCache( + TieredVolatileCacheOptions& opts) { + if (!opts.cache_opts) { + return nullptr; + } + + std::shared_ptr cache; + if (opts.cache_type == PrimaryCacheType::kCacheTypeLRU) { + LRUCacheOptions cache_opts = + *(static_cast_with_check( + opts.cache_opts)); + cache_opts.capacity += opts.comp_cache_opts.capacity; + cache = cache_opts.MakeSharedCache(); + } else if (opts.cache_type == PrimaryCacheType::kCacheTypeHCC) { + HyperClockCacheOptions cache_opts = + *(static_cast_with_check( + opts.cache_opts)); + cache = cache_opts.MakeSharedCache(); + } else { + return nullptr; + } + std::shared_ptr sec_cache; + sec_cache = NewCompressedSecondaryCache(opts.comp_cache_opts); + + return std::make_shared(cache, sec_cache, true); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/secondary_cache_adapter.h b/librocksdb-sys/rocksdb/cache/secondary_cache_adapter.h new file mode 100644 index 0000000..4ac9389 --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/secondary_cache_adapter.h @@ -0,0 +1,76 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "cache/cache_reservation_manager.h" +#include "rocksdb/secondary_cache.h" + +namespace ROCKSDB_NAMESPACE { + +class CacheWithSecondaryAdapter : public CacheWrapper { + public: + explicit CacheWithSecondaryAdapter( + std::shared_ptr target, + std::shared_ptr secondary_cache, + bool distribute_cache_res = false); + + ~CacheWithSecondaryAdapter() override; + + Status Insert(const Slice& key, ObjectPtr value, + const CacheItemHelper* helper, size_t charge, + Handle** handle = nullptr, + Priority priority = Priority::LOW) override; + + Handle* Lookup(const Slice& key, const CacheItemHelper* helper, + CreateContext* create_context, + Priority priority = Priority::LOW, + Statistics* stats = nullptr) override; + + using Cache::Release; + bool Release(Handle* handle, bool erase_if_last_ref = false) override; + + ObjectPtr Value(Handle* handle) override; + + void StartAsyncLookup(AsyncLookupHandle& async_handle) override; + + void WaitAll(AsyncLookupHandle* async_handles, size_t count) override; + + std::string GetPrintableOptions() const override; + + const char* Name() const override; + + Cache* TEST_GetCache() { return target_.get(); } + + SecondaryCache* TEST_GetSecondaryCache() { return secondary_cache_.get(); } + + private: + bool EvictionHandler(const Slice& key, Handle* handle); + + void StartAsyncLookupOnMySecondary(AsyncLookupHandle& async_handle); + + Handle* Promote( + std::unique_ptr&& secondary_handle, + const Slice& key, const CacheItemHelper* helper, Priority priority, + Statistics* stats, bool found_dummy_entry, bool kept_in_sec_cache); + + bool ProcessDummyResult(Cache::Handle** handle, bool erase); + + void CleanupCacheObject(ObjectPtr obj, const CacheItemHelper* helper); + + std::shared_ptr secondary_cache_; + // Whether to proportionally distribute cache memory reservations, i.e + // placeholder entries with null value and a non-zero charge, across + // the primary and secondary caches. + bool distribute_cache_res_; + // A cache reservation manager to keep track of secondary cache memory + // usage by reserving equivalent capacity against the primary cache + std::shared_ptr pri_cache_res_; + // Fraction of a cache memory reservation to be assigned to the secondary + // cache + double sec_cache_res_ratio_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/sharded_cache.cc b/librocksdb-sys/rocksdb/cache/sharded_cache.cc new file mode 100644 index 0000000..322b592 --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/sharded_cache.cc @@ -0,0 +1,137 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "cache/sharded_cache.h" + +#include +#include +#include + +#include "env/unique_id_gen.h" +#include "rocksdb/env.h" +#include "util/hash.h" +#include "util/math.h" +#include "util/mutexlock.h" + +namespace ROCKSDB_NAMESPACE { +namespace { +// The generated seeds must fit in 31 bits so that +// ShardedCacheOptions::hash_seed can be set to it explicitly, for +// diagnostic/debugging purposes. +constexpr uint32_t kSeedMask = 0x7fffffff; +uint32_t DetermineSeed(int32_t hash_seed_option) { + if (hash_seed_option >= 0) { + // User-specified exact seed + return static_cast(hash_seed_option); + } + static SemiStructuredUniqueIdGen gen; + if (hash_seed_option == ShardedCacheOptions::kHostHashSeed) { + std::string hostname; + Status s = Env::Default()->GetHostNameString(&hostname); + if (s.ok()) { + return GetSliceHash(hostname) & kSeedMask; + } else { + // Fall back on something stable within the process. + return BitwiseAnd(gen.GetBaseUpper(), kSeedMask); + } + } else { + // for kQuasiRandomHashSeed and fallback + uint32_t val = gen.GenerateNext() & kSeedMask; + // Perform some 31-bit bijective transformations so that we get + // quasirandom, not just incrementing. (An incrementing seed from a + // random starting point would be fine, but hard to describe in a name.) + // See https://en.wikipedia.org/wiki/Quasirandom and using a murmur-like + // transformation here for our bijection in the lower 31 bits. + // See https://en.wikipedia.org/wiki/MurmurHash + val *= /*31-bit prime*/ 1150630961; + val ^= (val & kSeedMask) >> 17; + val *= /*31-bit prime*/ 1320603883; + return val & kSeedMask; + } +} +} // namespace + +ShardedCacheBase::ShardedCacheBase(const ShardedCacheOptions& opts) + : Cache(opts.memory_allocator), + last_id_(1), + shard_mask_((uint32_t{1} << opts.num_shard_bits) - 1), + hash_seed_(DetermineSeed(opts.hash_seed)), + strict_capacity_limit_(opts.strict_capacity_limit), + capacity_(opts.capacity) {} + +size_t ShardedCacheBase::ComputePerShardCapacity(size_t capacity) const { + uint32_t num_shards = GetNumShards(); + return (capacity + (num_shards - 1)) / num_shards; +} + +size_t ShardedCacheBase::GetPerShardCapacity() const { + return ComputePerShardCapacity(GetCapacity()); +} + +uint64_t ShardedCacheBase::NewId() { + return last_id_.fetch_add(1, std::memory_order_relaxed); +} + +size_t ShardedCacheBase::GetCapacity() const { + MutexLock l(&config_mutex_); + return capacity_; +} + +bool ShardedCacheBase::HasStrictCapacityLimit() const { + MutexLock l(&config_mutex_); + return strict_capacity_limit_; +} + +size_t ShardedCacheBase::GetUsage(Handle* handle) const { + return GetCharge(handle); +} + +std::string ShardedCacheBase::GetPrintableOptions() const { + std::string ret; + ret.reserve(20000); + const int kBufferSize = 200; + char buffer[kBufferSize]; + { + MutexLock l(&config_mutex_); + snprintf(buffer, kBufferSize, " capacity : %" ROCKSDB_PRIszt "\n", + capacity_); + ret.append(buffer); + snprintf(buffer, kBufferSize, " num_shard_bits : %d\n", + GetNumShardBits()); + ret.append(buffer); + snprintf(buffer, kBufferSize, " strict_capacity_limit : %d\n", + strict_capacity_limit_); + ret.append(buffer); + } + snprintf(buffer, kBufferSize, " memory_allocator : %s\n", + memory_allocator() ? memory_allocator()->Name() : "None"); + ret.append(buffer); + AppendPrintableOptions(ret); + return ret; +} + +int GetDefaultCacheShardBits(size_t capacity, size_t min_shard_size) { + int num_shard_bits = 0; + size_t num_shards = capacity / min_shard_size; + while (num_shards >>= 1) { + if (++num_shard_bits >= 6) { + // No more than 6. + return num_shard_bits; + } + } + return num_shard_bits; +} + +int ShardedCacheBase::GetNumShardBits() const { + return BitsSetToOne(shard_mask_); +} + +uint32_t ShardedCacheBase::GetNumShards() const { return shard_mask_ + 1; } + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/sharded_cache.h b/librocksdb-sys/rocksdb/cache/sharded_cache.h new file mode 100644 index 0000000..d78cfc2 --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/sharded_cache.h @@ -0,0 +1,309 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include +#include +#include + +#include "port/lang.h" +#include "port/port.h" +#include "rocksdb/advanced_cache.h" +#include "util/hash.h" +#include "util/mutexlock.h" + +namespace ROCKSDB_NAMESPACE { + +// Optional base class for classes implementing the CacheShard concept +class CacheShardBase { + public: + explicit CacheShardBase(CacheMetadataChargePolicy metadata_charge_policy) + : metadata_charge_policy_(metadata_charge_policy) {} + + using DeleterFn = Cache::DeleterFn; + + // Expected by concept CacheShard (TODO with C++20 support) + // Some Defaults + std::string GetPrintableOptions() const { return ""; } + using HashVal = uint64_t; + using HashCref = uint64_t; + static inline HashVal ComputeHash(const Slice& key, uint32_t seed) { + return GetSliceNPHash64(key, seed); + } + static inline uint32_t HashPieceForSharding(HashCref hash) { + return Lower32of64(hash); + } + void AppendPrintableOptions(std::string& /*str*/) const {} + + // Must be provided for concept CacheShard (TODO with C++20 support) + /* + struct HandleImpl { // for concept HandleImpl + HashVal hash; + HashCref GetHash() const; + ... + }; + Status Insert(const Slice& key, HashCref hash, Cache::ObjectPtr value, + const Cache::CacheItemHelper* helper, size_t charge, + HandleImpl** handle, Cache::Priority priority, + bool standalone) = 0; + Handle* CreateStandalone(const Slice& key, HashCref hash, ObjectPtr obj, + const CacheItemHelper* helper, + size_t charge, bool allow_uncharged) = 0; + HandleImpl* Lookup(const Slice& key, HashCref hash, + const Cache::CacheItemHelper* helper, + Cache::CreateContext* create_context, + Cache::Priority priority, + Statistics* stats) = 0; + bool Release(HandleImpl* handle, bool useful, bool erase_if_last_ref) = 0; + bool Ref(HandleImpl* handle) = 0; + void Erase(const Slice& key, HashCref hash) = 0; + void SetCapacity(size_t capacity) = 0; + void SetStrictCapacityLimit(bool strict_capacity_limit) = 0; + size_t GetUsage() const = 0; + size_t GetPinnedUsage() const = 0; + size_t GetOccupancyCount() const = 0; + size_t GetTableAddressCount() const = 0; + // Handles iterating over roughly `average_entries_per_lock` entries, using + // `state` to somehow record where it last ended up. Caller initially uses + // *state == 0 and implementation sets *state = SIZE_MAX to indicate + // completion. + void ApplyToSomeEntries( + const std::function& callback, + size_t average_entries_per_lock, size_t* state) = 0; + void EraseUnRefEntries() = 0; + */ + + protected: + const CacheMetadataChargePolicy metadata_charge_policy_; +}; + +// Portions of ShardedCache that do not depend on the template parameter +class ShardedCacheBase : public Cache { + public: + explicit ShardedCacheBase(const ShardedCacheOptions& opts); + virtual ~ShardedCacheBase() = default; + + int GetNumShardBits() const; + uint32_t GetNumShards() const; + + uint64_t NewId() override; + + bool HasStrictCapacityLimit() const override; + size_t GetCapacity() const override; + + using Cache::GetUsage; + size_t GetUsage(Handle* handle) const override; + std::string GetPrintableOptions() const override; + + uint32_t GetHashSeed() const override { return hash_seed_; } + + protected: // fns + virtual void AppendPrintableOptions(std::string& str) const = 0; + size_t GetPerShardCapacity() const; + size_t ComputePerShardCapacity(size_t capacity) const; + + protected: // data + std::atomic last_id_; // For NewId + const uint32_t shard_mask_; + const uint32_t hash_seed_; + + // Dynamic configuration parameters, guarded by config_mutex_ + bool strict_capacity_limit_; + size_t capacity_; + mutable port::Mutex config_mutex_; +}; + +// Generic cache interface that shards cache by hash of keys. 2^num_shard_bits +// shards will be created, with capacity split evenly to each of the shards. +// Keys are typically sharded by the lowest num_shard_bits bits of hash value +// so that the upper bits of the hash value can keep a stable ordering of +// table entries even as the table grows (using more upper hash bits). +// See CacheShardBase above for what is expected of the CacheShard parameter. +template +class ShardedCache : public ShardedCacheBase { + public: + using HashVal = typename CacheShard::HashVal; + using HashCref = typename CacheShard::HashCref; + using HandleImpl = typename CacheShard::HandleImpl; + + explicit ShardedCache(const ShardedCacheOptions& opts) + : ShardedCacheBase(opts), + shards_(reinterpret_cast(port::cacheline_aligned_alloc( + sizeof(CacheShard) * GetNumShards()))), + destroy_shards_in_dtor_(false) {} + + virtual ~ShardedCache() { + if (destroy_shards_in_dtor_) { + ForEachShard([](CacheShard* cs) { cs->~CacheShard(); }); + } + port::cacheline_aligned_free(shards_); + } + + CacheShard& GetShard(HashCref hash) { + return shards_[CacheShard::HashPieceForSharding(hash) & shard_mask_]; + } + + const CacheShard& GetShard(HashCref hash) const { + return shards_[CacheShard::HashPieceForSharding(hash) & shard_mask_]; + } + + void SetCapacity(size_t capacity) override { + MutexLock l(&config_mutex_); + capacity_ = capacity; + auto per_shard = ComputePerShardCapacity(capacity); + ForEachShard([=](CacheShard* cs) { cs->SetCapacity(per_shard); }); + } + + void SetStrictCapacityLimit(bool s_c_l) override { + MutexLock l(&config_mutex_); + strict_capacity_limit_ = s_c_l; + ForEachShard( + [s_c_l](CacheShard* cs) { cs->SetStrictCapacityLimit(s_c_l); }); + } + + Status Insert(const Slice& key, ObjectPtr obj, const CacheItemHelper* helper, + size_t charge, Handle** handle = nullptr, + Priority priority = Priority::LOW) override { + assert(helper); + HashVal hash = CacheShard::ComputeHash(key, hash_seed_); + auto h_out = reinterpret_cast(handle); + return GetShard(hash).Insert(key, hash, obj, helper, charge, h_out, + priority); + } + + Handle* CreateStandalone(const Slice& key, ObjectPtr obj, + const CacheItemHelper* helper, size_t charge, + bool allow_uncharged) override { + assert(helper); + HashVal hash = CacheShard::ComputeHash(key, hash_seed_); + HandleImpl* result = GetShard(hash).CreateStandalone( + key, hash, obj, helper, charge, allow_uncharged); + return reinterpret_cast(result); + } + + Handle* Lookup(const Slice& key, const CacheItemHelper* helper = nullptr, + CreateContext* create_context = nullptr, + Priority priority = Priority::LOW, + Statistics* stats = nullptr) override { + HashVal hash = CacheShard::ComputeHash(key, hash_seed_); + HandleImpl* result = GetShard(hash).Lookup(key, hash, helper, + create_context, priority, stats); + return reinterpret_cast(result); + } + + void Erase(const Slice& key) override { + HashVal hash = CacheShard::ComputeHash(key, hash_seed_); + GetShard(hash).Erase(key, hash); + } + + bool Release(Handle* handle, bool useful, + bool erase_if_last_ref = false) override { + auto h = reinterpret_cast(handle); + return GetShard(h->GetHash()).Release(h, useful, erase_if_last_ref); + } + bool Ref(Handle* handle) override { + auto h = reinterpret_cast(handle); + return GetShard(h->GetHash()).Ref(h); + } + bool Release(Handle* handle, bool erase_if_last_ref = false) override { + return Release(handle, true /*useful*/, erase_if_last_ref); + } + using ShardedCacheBase::GetUsage; + size_t GetUsage() const override { + return SumOverShards2(&CacheShard::GetUsage); + } + size_t GetPinnedUsage() const override { + return SumOverShards2(&CacheShard::GetPinnedUsage); + } + size_t GetOccupancyCount() const override { + return SumOverShards2(&CacheShard::GetOccupancyCount); + } + size_t GetTableAddressCount() const override { + return SumOverShards2(&CacheShard::GetTableAddressCount); + } + void ApplyToAllEntries( + const std::function& callback, + const ApplyToAllEntriesOptions& opts) override { + uint32_t num_shards = GetNumShards(); + // Iterate over part of each shard, rotating between shards, to + // minimize impact on latency of concurrent operations. + std::unique_ptr states(new size_t[num_shards]{}); + + size_t aepl = opts.average_entries_per_lock; + aepl = std::min(aepl, size_t{1}); + + bool remaining_work; + do { + remaining_work = false; + for (uint32_t i = 0; i < num_shards; i++) { + if (states[i] != SIZE_MAX) { + shards_[i].ApplyToSomeEntries(callback, aepl, &states[i]); + remaining_work |= states[i] != SIZE_MAX; + } + } + } while (remaining_work); + } + + virtual void EraseUnRefEntries() override { + ForEachShard([](CacheShard* cs) { cs->EraseUnRefEntries(); }); + } + + void DisownData() override { + // Leak data only if that won't generate an ASAN/valgrind warning. + if (!kMustFreeHeapAllocations) { + destroy_shards_in_dtor_ = false; + } + } + + protected: + inline void ForEachShard(const std::function& fn) { + uint32_t num_shards = GetNumShards(); + for (uint32_t i = 0; i < num_shards; i++) { + fn(shards_ + i); + } + } + + inline size_t SumOverShards( + const std::function& fn) const { + uint32_t num_shards = GetNumShards(); + size_t result = 0; + for (uint32_t i = 0; i < num_shards; i++) { + result += fn(shards_[i]); + } + return result; + } + + inline size_t SumOverShards2(size_t (CacheShard::*fn)() const) const { + return SumOverShards([fn](CacheShard& cs) { return (cs.*fn)(); }); + } + + // Must be called exactly once by derived class constructor + void InitShards(const std::function& placement_new) { + ForEachShard(placement_new); + destroy_shards_in_dtor_ = true; + } + + void AppendPrintableOptions(std::string& str) const override { + shards_[0].AppendPrintableOptions(str); + } + + private: + CacheShard* const shards_; + bool destroy_shards_in_dtor_; +}; + +// 512KB is traditional minimum shard size. +int GetDefaultCacheShardBits(size_t capacity, + size_t min_shard_size = 512U * 1024U); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cache/typed_cache.h b/librocksdb-sys/rocksdb/cache/typed_cache.h new file mode 100644 index 0000000..e42aa4c --- /dev/null +++ b/librocksdb-sys/rocksdb/cache/typed_cache.h @@ -0,0 +1,375 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +// APIs for accessing Cache in a type-safe and convenient way. Cache is kept +// at a low, thin level of abstraction so that different implementations can +// be plugged in, but these wrappers provide clean, convenient access to the +// most common operations. +// +// A number of template classes are needed for sharing common structure. The +// key classes are these: +// +// * PlaceholderCacheInterface - Used for making cache reservations, with +// entries that have a charge but no value. +// * BasicTypedCacheInterface - Used for primary cache storage of +// objects of type TValue. +// * FullTypedCacheHelper - Used for secondary cache +// compatible storage of objects of type TValue. +// * For each of these, there's a "Shared" version +// (e.g. FullTypedSharedCacheInterface) that holds a shared_ptr to the Cache, +// rather than assuming external ownership by holding only a raw `Cache*`. + +#pragma once + +#include +#include +#include +#include + +#include "cache/cache_helpers.h" +#include "rocksdb/advanced_cache.h" +#include "rocksdb/advanced_options.h" + +namespace ROCKSDB_NAMESPACE { + +// For future consideration: +// * Pass in value to Insert with std::unique_ptr& to simplify ownership +// transfer logic in callers +// * Make key type a template parameter (e.g. useful for table cache) +// * Closer integration with CacheHandleGuard (opt-in, so not always +// paying the extra overhead) + +#define CACHE_TYPE_DEFS() \ + using Priority = Cache::Priority; \ + using Handle = Cache::Handle; \ + using ObjectPtr = Cache::ObjectPtr; \ + using CreateContext = Cache::CreateContext; \ + using CacheItemHelper = Cache::CacheItemHelper /* caller ; */ + +template +class BaseCacheInterface { + public: + CACHE_TYPE_DEFS(); + + /*implicit*/ BaseCacheInterface(CachePtr cache) : cache_(std::move(cache)) {} + + inline void Release(Handle* handle) { cache_->Release(handle); } + + inline void ReleaseAndEraseIfLastRef(Handle* handle) { + cache_->Release(handle, /*erase_if_last_ref*/ true); + } + + inline void RegisterReleaseAsCleanup(Handle* handle, Cleanable& cleanable) { + cleanable.RegisterCleanup(&ReleaseCacheHandleCleanup, get(), handle); + } + + inline Cache* get() const { return &*cache_; } + + explicit inline operator bool() const noexcept { return cache_ != nullptr; } + + protected: + CachePtr cache_; +}; + +// PlaceholderCacheInterface - Used for making cache reservations, with +// entries that have a charge but no value. CacheEntryRole is required as +// a template parameter. +template +class PlaceholderCacheInterface : public BaseCacheInterface { + public: + CACHE_TYPE_DEFS(); + using BaseCacheInterface::BaseCacheInterface; + + inline Status Insert(const Slice& key, size_t charge, Handle** handle) { + return this->cache_->Insert(key, /*value=*/nullptr, GetHelper(), charge, + handle); + } + + static const Cache::CacheItemHelper* GetHelper() { + static const Cache::CacheItemHelper kHelper{kRole}; + return &kHelper; + } +}; + +template +using PlaceholderSharedCacheInterface = + PlaceholderCacheInterface>; + +template +class BasicTypedCacheHelperFns { + public: + CACHE_TYPE_DEFS(); + // E.g. char* for char[] + using TValuePtr = std::remove_extent_t*; + + protected: + inline static ObjectPtr UpCastValue(TValuePtr value) { return value; } + inline static TValuePtr DownCastValue(ObjectPtr value) { + return static_cast(value); + } + + static void Delete(ObjectPtr value, MemoryAllocator* allocator) { + // FIXME: Currently, no callers actually allocate the ObjectPtr objects + // using the custom allocator, just subobjects that keep a reference to + // the allocator themselves (with CacheAllocationPtr). + if (/*DISABLED*/ false && allocator) { + if constexpr (std::is_destructible_v) { + DownCastValue(value)->~TValue(); + } + allocator->Deallocate(value); + } else { + // Like delete but properly handles TValue=char[] etc. + std::default_delete{}(DownCastValue(value)); + } + } +}; + +// In its own class to try to minimize the number of distinct CacheItemHelper +// instances (e.g. don't vary by CachePtr) +template +class BasicTypedCacheHelper : public BasicTypedCacheHelperFns { + public: + static const Cache::CacheItemHelper* GetBasicHelper() { + static const Cache::CacheItemHelper kHelper{kRole, + &BasicTypedCacheHelper::Delete}; + return &kHelper; + } +}; + +// BasicTypedCacheInterface - Used for primary cache storage of objects of +// type TValue, which can be cleaned up with std::default_delete. The +// role is provided by TValue::kCacheEntryRole or given in an optional +// template parameter. +template +class BasicTypedCacheInterface : public BaseCacheInterface, + public BasicTypedCacheHelper { + public: + CACHE_TYPE_DEFS(); + using typename BasicTypedCacheHelperFns::TValuePtr; + struct TypedHandle : public Handle {}; + using BasicTypedCacheHelper::GetBasicHelper; + // ctor + using BaseCacheInterface::BaseCacheInterface; + struct TypedAsyncLookupHandle : public Cache::AsyncLookupHandle { + TypedHandle* Result() { + return reinterpret_cast(Cache::AsyncLookupHandle::Result()); + } + }; + + inline Status Insert(const Slice& key, TValuePtr value, size_t charge, + TypedHandle** handle = nullptr, + Priority priority = Priority::LOW) { + auto untyped_handle = reinterpret_cast(handle); + return this->cache_->Insert( + key, BasicTypedCacheHelperFns::UpCastValue(value), + GetBasicHelper(), charge, untyped_handle, priority); + } + + inline TypedHandle* Lookup(const Slice& key, Statistics* stats = nullptr) { + return reinterpret_cast( + this->cache_->BasicLookup(key, stats)); + } + + inline void StartAsyncLookup(TypedAsyncLookupHandle& async_handle) { + assert(async_handle.helper == nullptr); + this->cache_->StartAsyncLookup(async_handle); + } + + inline CacheHandleGuard Guard(TypedHandle* handle) { + if (handle) { + return CacheHandleGuard(&*this->cache_, handle); + } else { + return {}; + } + } + + inline std::shared_ptr SharedGuard(TypedHandle* handle) { + if (handle) { + return MakeSharedCacheHandleGuard(&*this->cache_, handle); + } else { + return {}; + } + } + + inline TValuePtr Value(TypedHandle* handle) { + return BasicTypedCacheHelperFns::DownCastValue( + this->cache_->Value(handle)); + } +}; + +// BasicTypedSharedCacheInterface - Like BasicTypedCacheInterface but with a +// shared_ptr for keeping Cache alive. +template +using BasicTypedSharedCacheInterface = + BasicTypedCacheInterface>; + +// TValue must implement ContentSlice() and ~TValue +// TCreateContext must implement Create(std::unique_ptr*, ...) +template +class FullTypedCacheHelperFns : public BasicTypedCacheHelperFns { + public: + CACHE_TYPE_DEFS(); + + protected: + using typename BasicTypedCacheHelperFns::TValuePtr; + using BasicTypedCacheHelperFns::DownCastValue; + using BasicTypedCacheHelperFns::UpCastValue; + + static size_t Size(ObjectPtr v) { + TValuePtr value = DownCastValue(v); + auto slice = value->ContentSlice(); + return slice.size(); + } + + static Status SaveTo(ObjectPtr v, size_t from_offset, size_t length, + char* out) { + TValuePtr value = DownCastValue(v); + auto slice = value->ContentSlice(); + assert(from_offset < slice.size()); + assert(from_offset + length <= slice.size()); + std::copy_n(slice.data() + from_offset, length, out); + return Status::OK(); + } + + static Status Create(const Slice& data, CreateContext* context, + MemoryAllocator* allocator, ObjectPtr* out_obj, + size_t* out_charge) { + std::unique_ptr value = nullptr; + if constexpr (sizeof(TCreateContext) > 0) { + TCreateContext* tcontext = static_cast(context); + tcontext->Create(&value, out_charge, data, allocator); + } else { + TCreateContext::Create(&value, out_charge, data, allocator); + } + *out_obj = UpCastValue(value.release()); + return Status::OK(); + } +}; + +// In its own class to try to minimize the number of distinct CacheItemHelper +// instances (e.g. don't vary by CachePtr) +template +class FullTypedCacheHelper + : public FullTypedCacheHelperFns { + public: + static const Cache::CacheItemHelper* GetFullHelper() { + static const Cache::CacheItemHelper kHelper{ + kRole, + &FullTypedCacheHelper::Delete, + &FullTypedCacheHelper::Size, + &FullTypedCacheHelper::SaveTo, + &FullTypedCacheHelper::Create, + BasicTypedCacheHelper::GetBasicHelper()}; + return &kHelper; + } +}; + +// FullTypedCacheHelper - Used for secondary cache compatible storage of +// objects of type TValue. In addition to BasicTypedCacheInterface constraints, +// we require TValue::ContentSlice() to return persistable data. This +// simplifies usage for the normal case of simple secondary cache compatibility +// (can give you a Slice to the data already in memory). In addition to +// TCreateContext performing the role of Cache::CreateContext, it is also +// expected to provide a function Create(std::unique_ptr* value, +// size_t* out_charge, const Slice& data, MemoryAllocator* allocator) for +// creating new TValue. +template +class FullTypedCacheInterface + : public BasicTypedCacheInterface, + public FullTypedCacheHelper { + public: + CACHE_TYPE_DEFS(); + using typename BasicTypedCacheInterface::TypedHandle; + using typename BasicTypedCacheInterface::TypedAsyncLookupHandle; + using typename BasicTypedCacheHelperFns::TValuePtr; + using BasicTypedCacheHelper::GetBasicHelper; + using FullTypedCacheHelper::GetFullHelper; + using BasicTypedCacheHelperFns::UpCastValue; + using BasicTypedCacheHelperFns::DownCastValue; + // ctor + using BasicTypedCacheInterface::BasicTypedCacheInterface; + + // Insert with SecondaryCache compatibility (subject to CacheTier). + // (Basic Insert() also inherited.) + inline Status InsertFull( + const Slice& key, TValuePtr value, size_t charge, + TypedHandle** handle = nullptr, Priority priority = Priority::LOW, + CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier) { + auto untyped_handle = reinterpret_cast(handle); + auto helper = lowest_used_cache_tier == CacheTier::kNonVolatileBlockTier + ? GetFullHelper() + : GetBasicHelper(); + return this->cache_->Insert(key, UpCastValue(value), helper, charge, + untyped_handle, priority); + } + + // Like SecondaryCache::InsertSaved, with SecondaryCache compatibility + // (subject to CacheTier). + inline Status InsertSaved( + const Slice& key, const Slice& data, TCreateContext* create_context, + Priority priority = Priority::LOW, + CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier, + size_t* out_charge = nullptr) { + ObjectPtr value; + size_t charge; + Status st = GetFullHelper()->create_cb(data, create_context, + this->cache_->memory_allocator(), + &value, &charge); + if (out_charge) { + *out_charge = charge; + } + if (st.ok()) { + st = InsertFull(key, DownCastValue(value), charge, nullptr /*handle*/, + priority, lowest_used_cache_tier); + } else { + GetFullHelper()->del_cb(value, this->cache_->memory_allocator()); + } + return st; + } + + // Lookup with SecondaryCache support (subject to CacheTier). + // (Basic Lookup() also inherited.) + inline TypedHandle* LookupFull( + const Slice& key, TCreateContext* create_context = nullptr, + Priority priority = Priority::LOW, Statistics* stats = nullptr, + CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier) { + if (lowest_used_cache_tier == CacheTier::kNonVolatileBlockTier) { + return reinterpret_cast(this->cache_->Lookup( + key, GetFullHelper(), create_context, priority, stats)); + } else { + return BasicTypedCacheInterface::Lookup(key, + stats); + } + } + + inline void StartAsyncLookupFull( + TypedAsyncLookupHandle& async_handle, + CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier) { + if (lowest_used_cache_tier == CacheTier::kNonVolatileBlockTier) { + async_handle.helper = GetFullHelper(); + this->cache_->StartAsyncLookup(async_handle); + } else { + BasicTypedCacheInterface::StartAsyncLookup( + async_handle); + } + } +}; + +// FullTypedSharedCacheInterface - Like FullTypedCacheInterface but with a +// shared_ptr for keeping Cache alive. +template +using FullTypedSharedCacheInterface = + FullTypedCacheInterface>; + +#undef CACHE_TYPE_DEFS + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/cmake/RocksDBConfig.cmake.in b/librocksdb-sys/rocksdb/cmake/RocksDBConfig.cmake.in new file mode 100644 index 0000000..0bd14be --- /dev/null +++ b/librocksdb-sys/rocksdb/cmake/RocksDBConfig.cmake.in @@ -0,0 +1,54 @@ +@PACKAGE_INIT@ + +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/modules") + +include(CMakeFindDependencyMacro) + +set(GFLAGS_USE_TARGET_NAMESPACE @GFLAGS_USE_TARGET_NAMESPACE@) + +if(@WITH_JEMALLOC@) + find_dependency(JeMalloc) +endif() + +if(@WITH_GFLAGS@) + find_dependency(gflags CONFIG) + if(NOT gflags_FOUND) + find_dependency(gflags) + endif() +endif() + +if(@WITH_SNAPPY@) + find_dependency(Snappy CONFIG) + if(NOT Snappy_FOUND) + find_dependency(Snappy) + endif() +endif() + +if(@WITH_ZLIB@) + find_dependency(ZLIB) +endif() + +if(@WITH_BZ2@) + find_dependency(BZip2) +endif() + +if(@WITH_LZ4@) + find_dependency(lz4) +endif() + +if(@WITH_ZSTD@) + find_dependency(zstd) +endif() + +if(@WITH_NUMA@) + find_dependency(NUMA) +endif() + +if(@WITH_TBB@) + find_dependency(TBB) +endif() + +find_dependency(Threads) + +include("${CMAKE_CURRENT_LIST_DIR}/RocksDBTargets.cmake") +check_required_components(RocksDB) diff --git a/librocksdb-sys/rocksdb/cmake/modules/CxxFlags.cmake b/librocksdb-sys/rocksdb/cmake/modules/CxxFlags.cmake new file mode 100644 index 0000000..7980cca --- /dev/null +++ b/librocksdb-sys/rocksdb/cmake/modules/CxxFlags.cmake @@ -0,0 +1,7 @@ +macro(get_cxx_std_flags FLAGS_VARIABLE) + if( CMAKE_CXX_STANDARD_REQUIRED ) + set(${FLAGS_VARIABLE} ${CMAKE_CXX${CMAKE_CXX_STANDARD}_STANDARD_COMPILE_OPTION}) + else() + set(${FLAGS_VARIABLE} ${CMAKE_CXX${CMAKE_CXX_STANDARD}_EXTENSION_COMPILE_OPTION}) + endif() +endmacro() diff --git a/librocksdb-sys/rocksdb/cmake/modules/FindJeMalloc.cmake b/librocksdb-sys/rocksdb/cmake/modules/FindJeMalloc.cmake new file mode 100644 index 0000000..f695b3e --- /dev/null +++ b/librocksdb-sys/rocksdb/cmake/modules/FindJeMalloc.cmake @@ -0,0 +1,29 @@ +# - Find JeMalloc library +# Find the native JeMalloc includes and library +# +# JeMalloc_INCLUDE_DIRS - where to find jemalloc.h, etc. +# JeMalloc_LIBRARIES - List of libraries when using jemalloc. +# JeMalloc_FOUND - True if jemalloc found. + +find_path(JeMalloc_INCLUDE_DIRS + NAMES jemalloc/jemalloc.h + HINTS ${JEMALLOC_ROOT_DIR}/include) + +find_library(JeMalloc_LIBRARIES + NAMES jemalloc + HINTS ${JEMALLOC_ROOT_DIR}/lib) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(JeMalloc DEFAULT_MSG JeMalloc_LIBRARIES JeMalloc_INCLUDE_DIRS) + +mark_as_advanced( + JeMalloc_LIBRARIES + JeMalloc_INCLUDE_DIRS) + +if(JeMalloc_FOUND AND NOT (TARGET JeMalloc::JeMalloc)) + add_library (JeMalloc::JeMalloc UNKNOWN IMPORTED) + set_target_properties(JeMalloc::JeMalloc + PROPERTIES + IMPORTED_LOCATION ${JeMalloc_LIBRARIES} + INTERFACE_INCLUDE_DIRECTORIES ${JeMalloc_INCLUDE_DIRS}) +endif() diff --git a/librocksdb-sys/rocksdb/cmake/modules/FindNUMA.cmake b/librocksdb-sys/rocksdb/cmake/modules/FindNUMA.cmake new file mode 100644 index 0000000..69b95c9 --- /dev/null +++ b/librocksdb-sys/rocksdb/cmake/modules/FindNUMA.cmake @@ -0,0 +1,29 @@ +# - Find NUMA +# Find the NUMA library and includes +# +# NUMA_INCLUDE_DIRS - where to find numa.h, etc. +# NUMA_LIBRARIES - List of libraries when using NUMA. +# NUMA_FOUND - True if NUMA found. + +find_path(NUMA_INCLUDE_DIRS + NAMES numa.h numaif.h + HINTS ${NUMA_ROOT_DIR}/include) + +find_library(NUMA_LIBRARIES + NAMES numa + HINTS ${NUMA_ROOT_DIR}/lib) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(NUMA DEFAULT_MSG NUMA_LIBRARIES NUMA_INCLUDE_DIRS) + +mark_as_advanced( + NUMA_LIBRARIES + NUMA_INCLUDE_DIRS) + +if(NUMA_FOUND AND NOT (TARGET NUMA::NUMA)) + add_library (NUMA::NUMA UNKNOWN IMPORTED) + set_target_properties(NUMA::NUMA + PROPERTIES + IMPORTED_LOCATION ${NUMA_LIBRARIES} + INTERFACE_INCLUDE_DIRECTORIES ${NUMA_INCLUDE_DIRS}) +endif() diff --git a/librocksdb-sys/rocksdb/cmake/modules/FindSnappy.cmake b/librocksdb-sys/rocksdb/cmake/modules/FindSnappy.cmake new file mode 100644 index 0000000..0b0dbf8 --- /dev/null +++ b/librocksdb-sys/rocksdb/cmake/modules/FindSnappy.cmake @@ -0,0 +1,29 @@ +# - Find Snappy +# Find the snappy compression library and includes +# +# Snappy_INCLUDE_DIRS - where to find snappy.h, etc. +# Snappy_LIBRARIES - List of libraries when using snappy. +# Snappy_FOUND - True if snappy found. + +find_path(Snappy_INCLUDE_DIRS + NAMES snappy.h + HINTS ${snappy_ROOT_DIR}/include) + +find_library(Snappy_LIBRARIES + NAMES snappy + HINTS ${snappy_ROOT_DIR}/lib) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Snappy DEFAULT_MSG Snappy_LIBRARIES Snappy_INCLUDE_DIRS) + +mark_as_advanced( + Snappy_LIBRARIES + Snappy_INCLUDE_DIRS) + +if(Snappy_FOUND AND NOT (TARGET Snappy::snappy)) + add_library (Snappy::snappy UNKNOWN IMPORTED) + set_target_properties(Snappy::snappy + PROPERTIES + IMPORTED_LOCATION ${Snappy_LIBRARIES} + INTERFACE_INCLUDE_DIRECTORIES ${Snappy_INCLUDE_DIRS}) +endif() diff --git a/librocksdb-sys/rocksdb/cmake/modules/FindTBB.cmake b/librocksdb-sys/rocksdb/cmake/modules/FindTBB.cmake new file mode 100644 index 0000000..f6861fa --- /dev/null +++ b/librocksdb-sys/rocksdb/cmake/modules/FindTBB.cmake @@ -0,0 +1,33 @@ +# - Find TBB +# Find the Thread Building Blocks library and includes +# +# TBB_INCLUDE_DIRS - where to find tbb.h, etc. +# TBB_LIBRARIES - List of libraries when using TBB. +# TBB_FOUND - True if TBB found. + +if(NOT DEFINED TBB_ROOT_DIR) + set(TBB_ROOT_DIR "$ENV{TBBROOT}") +endif() + +find_path(TBB_INCLUDE_DIRS + NAMES tbb/tbb.h + HINTS ${TBB_ROOT_DIR}/include) + +find_library(TBB_LIBRARIES + NAMES tbb + HINTS ${TBB_ROOT_DIR}/lib ENV LIBRARY_PATH) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(TBB DEFAULT_MSG TBB_LIBRARIES TBB_INCLUDE_DIRS) + +mark_as_advanced( + TBB_LIBRARIES + TBB_INCLUDE_DIRS) + +if(TBB_FOUND AND NOT (TARGET TBB::TBB)) + add_library (TBB::TBB UNKNOWN IMPORTED) + set_target_properties(TBB::TBB + PROPERTIES + IMPORTED_LOCATION ${TBB_LIBRARIES} + INTERFACE_INCLUDE_DIRECTORIES ${TBB_INCLUDE_DIRS}) +endif() diff --git a/librocksdb-sys/rocksdb/cmake/modules/Findgflags.cmake b/librocksdb-sys/rocksdb/cmake/modules/Findgflags.cmake new file mode 100644 index 0000000..786def2 --- /dev/null +++ b/librocksdb-sys/rocksdb/cmake/modules/Findgflags.cmake @@ -0,0 +1,29 @@ +# - Find gflags library +# Find the gflags includes and library +# +# GFLAGS_INCLUDE_DIR - where to find gflags.h. +# GFLAGS_LIBRARIES - List of libraries when using gflags. +# gflags_FOUND - True if gflags found. + +find_path(GFLAGS_INCLUDE_DIR + NAMES gflags/gflags.h) + +find_library(GFLAGS_LIBRARIES + NAMES gflags) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(gflags + DEFAULT_MSG GFLAGS_LIBRARIES GFLAGS_INCLUDE_DIR) + +mark_as_advanced( + GFLAGS_LIBRARIES + GFLAGS_INCLUDE_DIR) + +if(gflags_FOUND AND NOT (TARGET gflags::gflags)) + add_library(gflags::gflags UNKNOWN IMPORTED) + set_target_properties(gflags::gflags + PROPERTIES + IMPORTED_LOCATION ${GFLAGS_LIBRARIES} + INTERFACE_INCLUDE_DIRECTORIES ${GFLAGS_INCLUDE_DIR} + IMPORTED_LINK_INTERFACE_LANGUAGES "CXX") +endif() diff --git a/librocksdb-sys/rocksdb/cmake/modules/Findlz4.cmake b/librocksdb-sys/rocksdb/cmake/modules/Findlz4.cmake new file mode 100644 index 0000000..7cf7d7f --- /dev/null +++ b/librocksdb-sys/rocksdb/cmake/modules/Findlz4.cmake @@ -0,0 +1,29 @@ +# - Find Lz4 +# Find the lz4 compression library and includes +# +# lz4_INCLUDE_DIRS - where to find lz4.h, etc. +# lz4_LIBRARIES - List of libraries when using lz4. +# lz4_FOUND - True if lz4 found. + +find_path(lz4_INCLUDE_DIRS + NAMES lz4.h + HINTS ${lz4_ROOT_DIR}/include) + +find_library(lz4_LIBRARIES + NAMES lz4 + HINTS ${lz4_ROOT_DIR}/lib) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(lz4 DEFAULT_MSG lz4_LIBRARIES lz4_INCLUDE_DIRS) + +mark_as_advanced( + lz4_LIBRARIES + lz4_INCLUDE_DIRS) + +if(lz4_FOUND AND NOT (TARGET lz4::lz4)) + add_library(lz4::lz4 UNKNOWN IMPORTED) + set_target_properties(lz4::lz4 + PROPERTIES + IMPORTED_LOCATION ${lz4_LIBRARIES} + INTERFACE_INCLUDE_DIRECTORIES ${lz4_INCLUDE_DIRS}) +endif() diff --git a/librocksdb-sys/rocksdb/cmake/modules/Finduring.cmake b/librocksdb-sys/rocksdb/cmake/modules/Finduring.cmake new file mode 100644 index 0000000..8cb14cb --- /dev/null +++ b/librocksdb-sys/rocksdb/cmake/modules/Finduring.cmake @@ -0,0 +1,26 @@ +# - Find liburing +# +# uring_INCLUDE_DIR - Where to find liburing.h +# uring_LIBRARIES - List of libraries when using uring. +# uring_FOUND - True if uring found. + +find_path(uring_INCLUDE_DIR + NAMES liburing.h) +find_library(uring_LIBRARIES + NAMES liburing.a liburing) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(uring + DEFAULT_MSG uring_LIBRARIES uring_INCLUDE_DIR) + +mark_as_advanced( + uring_INCLUDE_DIR + uring_LIBRARIES) + +if(uring_FOUND AND NOT TARGET uring::uring) + add_library(uring::uring UNKNOWN IMPORTED) + set_target_properties(uring::uring PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${uring_INCLUDE_DIR}" + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${uring_LIBRARIES}") +endif() diff --git a/librocksdb-sys/rocksdb/cmake/modules/Findzstd.cmake b/librocksdb-sys/rocksdb/cmake/modules/Findzstd.cmake new file mode 100644 index 0000000..9430821 --- /dev/null +++ b/librocksdb-sys/rocksdb/cmake/modules/Findzstd.cmake @@ -0,0 +1,29 @@ +# - Find zstd +# Find the zstd compression library and includes +# +# zstd_INCLUDE_DIRS - where to find zstd.h, etc. +# zstd_LIBRARIES - List of libraries when using zstd. +# zstd_FOUND - True if zstd found. + +find_path(zstd_INCLUDE_DIRS + NAMES zstd.h + HINTS ${zstd_ROOT_DIR}/include) + +find_library(zstd_LIBRARIES + NAMES zstd + HINTS ${zstd_ROOT_DIR}/lib) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(zstd DEFAULT_MSG zstd_LIBRARIES zstd_INCLUDE_DIRS) + +mark_as_advanced( + zstd_LIBRARIES + zstd_INCLUDE_DIRS) + +if(zstd_FOUND AND NOT (TARGET zstd::zstd)) + add_library (zstd::zstd UNKNOWN IMPORTED) + set_target_properties(zstd::zstd + PROPERTIES + IMPORTED_LOCATION ${zstd_LIBRARIES} + INTERFACE_INCLUDE_DIRECTORIES ${zstd_INCLUDE_DIRS}) +endif() diff --git a/librocksdb-sys/rocksdb/cmake/modules/ReadVersion.cmake b/librocksdb-sys/rocksdb/cmake/modules/ReadVersion.cmake new file mode 100644 index 0000000..ebfd7d6 --- /dev/null +++ b/librocksdb-sys/rocksdb/cmake/modules/ReadVersion.cmake @@ -0,0 +1,10 @@ +# Read rocksdb version from version.h header file. + +function(get_rocksdb_version version_var) + file(READ "${CMAKE_CURRENT_SOURCE_DIR}/include/rocksdb/version.h" version_header_file) + foreach(component MAJOR MINOR PATCH) + string(REGEX MATCH "#define ROCKSDB_${component} ([0-9]+)" _ ${version_header_file}) + set(ROCKSDB_VERSION_${component} ${CMAKE_MATCH_1}) + endforeach() + set(${version_var} "${ROCKSDB_VERSION_MAJOR}.${ROCKSDB_VERSION_MINOR}.${ROCKSDB_VERSION_PATCH}" PARENT_SCOPE) +endfunction() diff --git a/librocksdb-sys/rocksdb/common.mk b/librocksdb-sys/rocksdb/common.mk new file mode 100644 index 0000000..85c99fc --- /dev/null +++ b/librocksdb-sys/rocksdb/common.mk @@ -0,0 +1,30 @@ +ifndef PYTHON + +# Default to python3. Some distros like CentOS 8 do not have `python`. +ifeq ($(origin PYTHON), undefined) + PYTHON := $(shell which python3 || which python || echo python3) +endif +export PYTHON + +endif + +# To setup tmp directory, first recognize some old variables for setting +# test tmp directory or base tmp directory. TEST_TMPDIR is usually read +# by RocksDB tools though Env/FileSystem::GetTestDirectory. +ifeq ($(TEST_TMPDIR),) +TEST_TMPDIR := $(TMPD) +endif +ifeq ($(TEST_TMPDIR),) +ifeq ($(BASE_TMPDIR),) +BASE_TMPDIR :=$(TMPDIR) +endif +ifeq ($(BASE_TMPDIR),) +BASE_TMPDIR :=/tmp +endif +# Use /dev/shm if it has the sticky bit set (otherwise, /tmp or other +# base dir), and create a randomly-named rocksdb.XXXX directory therein. +TEST_TMPDIR := $(shell f=/dev/shm; test -k $$f || f=$(BASE_TMPDIR); \ + perl -le 'use File::Temp "tempdir";' \ + -e 'print tempdir("'$$f'/rocksdb.XXXX", CLEANUP => 0)') +endif +export TEST_TMPDIR diff --git a/librocksdb-sys/rocksdb/coverage/coverage_test.sh b/librocksdb-sys/rocksdb/coverage/coverage_test.sh new file mode 100755 index 0000000..d8d750c --- /dev/null +++ b/librocksdb-sys/rocksdb/coverage/coverage_test.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +# Exit on error. +set -e + +if [ -n "$USE_CLANG" ]; then + echo "Error: Coverage test is supported only for gcc." + exit 1 +fi + +ROOT=".." +# Fetch right version of gcov +if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then + source $ROOT/build_tools/fbcode_config_platform010.sh + GCOV=$GCC_BASE/bin/gcov +else + GCOV=$(which gcov) +fi +echo -e "Using $GCOV" + +COVERAGE_DIR="$PWD/COVERAGE_REPORT" +mkdir -p $COVERAGE_DIR + +# Find all gcno files to generate the coverage report + +PYTHON=${1:-`which python3`} +echo -e "Using $PYTHON" +GCNO_FILES=`find $ROOT -name "*.gcno"` +$GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null | + # Parse the raw gcov report to more human readable form. + $PYTHON $ROOT/coverage/parse_gcov_output.py | + # Write the output to both stdout and report file. + tee $COVERAGE_DIR/coverage_report_all.txt && +echo -e "Generated coverage report for all files: $COVERAGE_DIR/coverage_report_all.txt\n" + +# TODO: we also need to get the files of the latest commits. +# Get the most recently committed files. +LATEST_FILES=` + git show --pretty="format:" --name-only HEAD | + grep -v "^$" | + paste -s -d,` +RECENT_REPORT=$COVERAGE_DIR/coverage_report_recent.txt + +echo -e "Recently updated files: $LATEST_FILES\n" > $RECENT_REPORT +$GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null | + $PYTHON $ROOT/coverage/parse_gcov_output.py -interested-files $LATEST_FILES | + tee -a $RECENT_REPORT && +echo -e "Generated coverage report for recently updated files: $RECENT_REPORT\n" + +# Unless otherwise specified, we'll not generate html report by default +if [ -z "$HTML" ]; then + exit 0 +fi + +# Generate the html report. If we cannot find lcov in this machine, we'll simply +# skip this step. +echo "Generating the html coverage report..." + +LCOV=$(which lcov || true 2>/dev/null) +if [ -z $LCOV ] +then + echo "Skip: Cannot find lcov to generate the html report." + exit 0 +fi + +LCOV_VERSION=$(lcov -v | grep 1.1 || true) +if [ $LCOV_VERSION ] +then + echo "Not supported lcov version. Expect lcov 1.1." + exit 0 +fi + +(cd $ROOT; lcov --no-external \ + --capture \ + --directory $PWD \ + --gcov-tool $GCOV \ + --output-file $COVERAGE_DIR/coverage.info) + +genhtml $COVERAGE_DIR/coverage.info -o $COVERAGE_DIR + +echo "HTML Coverage report is generated in $COVERAGE_DIR" diff --git a/librocksdb-sys/rocksdb/coverage/parse_gcov_output.py b/librocksdb-sys/rocksdb/coverage/parse_gcov_output.py new file mode 100644 index 0000000..b9788ec --- /dev/null +++ b/librocksdb-sys/rocksdb/coverage/parse_gcov_output.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +from __future__ import print_function + +import optparse +import re +import sys + +# the gcov report follows certain pattern. Each file will have two lines +# of report, from which we can extract the file name, total lines and coverage +# percentage. +def parse_gcov_report(gcov_input): + per_file_coverage = {} + total_coverage = None + + for line in sys.stdin: + line = line.strip() + + # --First line of the coverage report (with file name in it)? + match_obj = re.match("^File '(.*)'$", line) + if match_obj: + # fetch the file name from the first line of the report. + current_file = match_obj.group(1) + continue + + # -- Second line of the file report (with coverage percentage) + match_obj = re.match("^Lines executed:(.*)% of (.*)", line) + + if match_obj: + coverage = float(match_obj.group(1)) + lines = int(match_obj.group(2)) + + if current_file is not None: + per_file_coverage[current_file] = (coverage, lines) + current_file = None + else: + # If current_file is not set, we reach the last line of report, + # which contains the summarized coverage percentage. + total_coverage = (coverage, lines) + continue + + # If the line's pattern doesn't fall into the above categories. We + # can simply ignore them since they're either empty line or doesn't + # find executable lines of the given file. + current_file = None + + return per_file_coverage, total_coverage + + +def get_option_parser(): + usage = ( + "Parse the gcov output and generate more human-readable code " + + "coverage report." + ) + parser = optparse.OptionParser(usage) + + parser.add_option( + "--interested-files", + "-i", + dest="filenames", + help="Comma separated files names. if specified, we will display " + + "the coverage report only for interested source files. " + + "Otherwise we will display the coverage report for all " + + "source files.", + ) + return parser + + +def display_file_coverage(per_file_coverage, total_coverage): + # To print out auto-adjustable column, we need to know the longest + # length of file names. + max_file_name_length = max(len(fname) for fname in per_file_coverage.keys()) + + # -- Print header + # size of separator is determined by 3 column sizes: + # file name, coverage percentage and lines. + header_template = "%" + str(max_file_name_length) + "s\t%s\t%s" + separator = "-" * (max_file_name_length + 10 + 20) + print( + header_template % ("Filename", "Coverage", "Lines") + ) # noqa: E999 T25377293 Grandfathered in + print(separator) + + # -- Print body + # template for printing coverage report for each file. + record_template = "%" + str(max_file_name_length) + "s\t%5.2f%%\t%10d" + + for fname, coverage_info in per_file_coverage.items(): + coverage, lines = coverage_info + print(record_template % (fname, coverage, lines)) + + # -- Print footer + if total_coverage: + print(separator) + print(record_template % ("Total", total_coverage[0], total_coverage[1])) + + +def report_coverage(): + parser = get_option_parser() + (options, args) = parser.parse_args() + + interested_files = set() + if options.filenames is not None: + interested_files = {f.strip() for f in options.filenames.split(",")} + + # To make things simple, right now we only read gcov report from the input + per_file_coverage, total_coverage = parse_gcov_report(sys.stdin) + + # Check if we need to display coverage info for interested files. + if len(interested_files): + per_file_coverage = dict( + (fname, per_file_coverage[fname]) + for fname in interested_files + if fname in per_file_coverage + ) + # If we only interested in several files, it makes no sense to report + # the total_coverage + total_coverage = None + + if not len(per_file_coverage): + print("Cannot find coverage info for the given files.", file=sys.stderr) + return + display_file_coverage(per_file_coverage, total_coverage) + + +if __name__ == "__main__": + report_coverage() diff --git a/librocksdb-sys/rocksdb/crash_test.mk b/librocksdb-sys/rocksdb/crash_test.mk new file mode 100644 index 0000000..a71a55c --- /dev/null +++ b/librocksdb-sys/rocksdb/crash_test.mk @@ -0,0 +1,121 @@ +# This file is used by Meta-internal infrastructure as well as by Makefile + +# When included from Makefile, there are rules to build DB_STRESS_CMD. When +# used directly with `make -f crashtest.mk ...` there will be no rules to +# build DB_STRESS_CMD so it must exist prior. +DB_STRESS_CMD?=./db_stress + +include common.mk + +CRASHTEST_MAKE=$(MAKE) -f crash_test.mk +CRASHTEST_PY=$(PYTHON) -u tools/db_crashtest.py --stress_cmd=$(DB_STRESS_CMD) --cleanup_cmd='$(DB_CLEANUP_CMD)' + +.PHONY: crash_test crash_test_with_atomic_flush crash_test_with_txn \ + crash_test_with_best_efforts_recovery crash_test_with_ts \ + blackbox_crash_test blackbox_crash_test_with_atomic_flush \ + blackbox_crash_test_with_txn blackbox_crash_test_with_ts \ + blackbox_crash_test_with_best_efforts_recovery \ + whitebox_crash_test whitebox_crash_test_with_atomic_flush \ + whitebox_crash_test_with_txn whitebox_crash_test_with_ts \ + blackbox_crash_test_with_multiops_wc_txn \ + blackbox_crash_test_with_multiops_wp_txn \ + crash_test_with_tiered_storage blackbox_crash_test_with_tiered_storage \ + whitebox_crash_test_with_tiered_storage \ + whitebox_crash_test_with_optimistic_txn \ + blackbox_crash_test_with_optimistic_txn \ + +crash_test: $(DB_STRESS_CMD) +# Do not parallelize + $(CRASHTEST_MAKE) whitebox_crash_test + $(CRASHTEST_MAKE) blackbox_crash_test + +crash_test_with_atomic_flush: $(DB_STRESS_CMD) +# Do not parallelize + $(CRASHTEST_MAKE) whitebox_crash_test_with_atomic_flush + $(CRASHTEST_MAKE) blackbox_crash_test_with_atomic_flush + +crash_test_with_txn: $(DB_STRESS_CMD) +# Do not parallelize + $(CRASHTEST_MAKE) whitebox_crash_test_with_txn + $(CRASHTEST_MAKE) blackbox_crash_test_with_txn + +crash_test_with_optimistic_txn: $(DB_STRESS_CMD) +# Do not parallelize + $(CRASHTEST_MAKE) whitebox_crash_test_with_optimistic_txn + $(CRASHTEST_MAKE) blackbox_crash_test_with_optimistic_txn + +crash_test_with_best_efforts_recovery: blackbox_crash_test_with_best_efforts_recovery + +crash_test_with_ts: $(DB_STRESS_CMD) +# Do not parallelize + $(CRASHTEST_MAKE) whitebox_crash_test_with_ts + $(CRASHTEST_MAKE) blackbox_crash_test_with_ts + +crash_test_with_tiered_storage: $(DB_STRESS_CMD) +# Do not parallelize + $(CRASHTEST_MAKE) whitebox_crash_test_with_tiered_storage + $(CRASHTEST_MAKE) blackbox_crash_test_with_tiered_storage + +crash_test_with_multiops_wc_txn: $(DB_STRESS_CMD) + $(CRASHTEST_MAKE) blackbox_crash_test_with_multiops_wc_txn + +crash_test_with_multiops_wp_txn: $(DB_STRESS_CMD) + $(CRASHTEST_MAKE) blackbox_crash_test_with_multiops_wp_txn + +blackbox_crash_test: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --simple blackbox $(CRASH_TEST_EXT_ARGS) + $(CRASHTEST_PY) blackbox $(CRASH_TEST_EXT_ARGS) + +blackbox_crash_test_with_atomic_flush: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --cf_consistency blackbox $(CRASH_TEST_EXT_ARGS) + +blackbox_crash_test_with_txn: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --txn blackbox $(CRASH_TEST_EXT_ARGS) + +blackbox_crash_test_with_best_efforts_recovery: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --test_best_efforts_recovery blackbox $(CRASH_TEST_EXT_ARGS) + +blackbox_crash_test_with_ts: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --enable_ts blackbox $(CRASH_TEST_EXT_ARGS) + +blackbox_crash_test_with_multiops_wc_txn: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --test_multiops_txn --write_policy write_committed blackbox $(CRASH_TEST_EXT_ARGS) + +blackbox_crash_test_with_multiops_wp_txn: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --test_multiops_txn --write_policy write_prepared blackbox $(CRASH_TEST_EXT_ARGS) + +blackbox_crash_test_with_tiered_storage: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --test_tiered_storage blackbox $(CRASH_TEST_EXT_ARGS) + +blackbox_crash_test_with_optimistic_txn: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --optimistic_txn blackbox $(CRASH_TEST_EXT_ARGS) + +ifeq ($(CRASH_TEST_KILL_ODD),) + CRASH_TEST_KILL_ODD=888887 +endif + +whitebox_crash_test: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --simple whitebox --random_kill_odd \ + $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) + $(CRASHTEST_PY) whitebox --random_kill_odd \ + $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) + +whitebox_crash_test_with_atomic_flush: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --cf_consistency whitebox --random_kill_odd \ + $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) + +whitebox_crash_test_with_txn: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --txn whitebox --random_kill_odd \ + $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) + +whitebox_crash_test_with_ts: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --enable_ts whitebox --random_kill_odd \ + $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) + +whitebox_crash_test_with_tiered_storage: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --test_tiered_storage whitebox --random_kill_odd \ + $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) + +whitebox_crash_test_with_optimistic_txn: $(DB_STRESS_CMD) + $(CRASHTEST_PY) --optimistic_txn whitebox --random_kill_odd \ + $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) diff --git a/librocksdb-sys/rocksdb/db/arena_wrapped_db_iter.cc b/librocksdb-sys/rocksdb/db/arena_wrapped_db_iter.cc new file mode 100644 index 0000000..b101fbb --- /dev/null +++ b/librocksdb-sys/rocksdb/db/arena_wrapped_db_iter.cc @@ -0,0 +1,165 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/arena_wrapped_db_iter.h" + +#include "memory/arena.h" +#include "rocksdb/env.h" +#include "rocksdb/iterator.h" +#include "rocksdb/options.h" +#include "table/internal_iterator.h" +#include "table/iterator_wrapper.h" +#include "util/user_comparator_wrapper.h" + +namespace ROCKSDB_NAMESPACE { + +Status ArenaWrappedDBIter::GetProperty(std::string prop_name, + std::string* prop) { + if (prop_name == "rocksdb.iterator.super-version-number") { + // First try to pass the value returned from inner iterator. + if (!db_iter_->GetProperty(prop_name, prop).ok()) { + *prop = std::to_string(sv_number_); + } + return Status::OK(); + } + return db_iter_->GetProperty(prop_name, prop); +} + +void ArenaWrappedDBIter::Init( + Env* env, const ReadOptions& read_options, const ImmutableOptions& ioptions, + const MutableCFOptions& mutable_cf_options, const Version* version, + const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iteration, + uint64_t version_number, ReadCallback* read_callback, DBImpl* db_impl, + ColumnFamilyData* cfd, bool expose_blob_index, bool allow_refresh) { + auto mem = arena_.AllocateAligned(sizeof(DBIter)); + db_iter_ = + new (mem) DBIter(env, read_options, ioptions, mutable_cf_options, + ioptions.user_comparator, /* iter */ nullptr, version, + sequence, true, max_sequential_skip_in_iteration, + read_callback, db_impl, cfd, expose_blob_index); + sv_number_ = version_number; + read_options_ = read_options; + allow_refresh_ = allow_refresh; + memtable_range_tombstone_iter_ = nullptr; + + if (!CheckFSFeatureSupport(env->GetFileSystem().get(), + FSSupportedOps::kAsyncIO)) { + read_options_.async_io = false; + } +} + +Status ArenaWrappedDBIter::Refresh() { + if (cfd_ == nullptr || db_impl_ == nullptr || !allow_refresh_) { + return Status::NotSupported("Creating renew iterator is not allowed."); + } + assert(db_iter_ != nullptr); + // TODO(yiwu): For last_seq_same_as_publish_seq_==false, this is not the + // correct behavior. Will be corrected automatically when we take a snapshot + // here for the case of WritePreparedTxnDB. + uint64_t cur_sv_number = cfd_->GetSuperVersionNumber(); + TEST_SYNC_POINT("ArenaWrappedDBIter::Refresh:1"); + TEST_SYNC_POINT("ArenaWrappedDBIter::Refresh:2"); + auto reinit_internal_iter = [&]() { + Env* env = db_iter_->env(); + db_iter_->~DBIter(); + arena_.~Arena(); + new (&arena_) Arena(); + + SuperVersion* sv = cfd_->GetReferencedSuperVersion(db_impl_); + SequenceNumber latest_seq = db_impl_->GetLatestSequenceNumber(); + if (read_callback_) { + read_callback_->Refresh(latest_seq); + } + Init(env, read_options_, *(cfd_->ioptions()), sv->mutable_cf_options, + sv->current, latest_seq, + sv->mutable_cf_options.max_sequential_skip_in_iterations, + cur_sv_number, read_callback_, db_impl_, cfd_, expose_blob_index_, + allow_refresh_); + + InternalIterator* internal_iter = db_impl_->NewInternalIterator( + read_options_, cfd_, sv, &arena_, latest_seq, + /* allow_unprepared_value */ true, /* db_iter */ this); + SetIterUnderDBIter(internal_iter); + }; + while (true) { + if (sv_number_ != cur_sv_number) { + reinit_internal_iter(); + break; + } else { + SequenceNumber latest_seq = db_impl_->GetLatestSequenceNumber(); + // Refresh range-tombstones in MemTable + if (!read_options_.ignore_range_deletions) { + SuperVersion* sv = cfd_->GetThreadLocalSuperVersion(db_impl_); + TEST_SYNC_POINT_CALLBACK("ArenaWrappedDBIter::Refresh:SV", nullptr); + auto t = sv->mem->NewRangeTombstoneIterator( + read_options_, latest_seq, false /* immutable_memtable */); + if (!t || t->empty()) { + // If memtable_range_tombstone_iter_ points to a non-empty tombstone + // iterator, then it means sv->mem is not the memtable that + // memtable_range_tombstone_iter_ points to, so SV must have changed + // after the sv_number_ != cur_sv_number check above. We will fall + // back to re-init the InternalIterator, and the tombstone iterator + // will be freed during db_iter destruction there. + if (memtable_range_tombstone_iter_) { + assert(!*memtable_range_tombstone_iter_ || + sv_number_ != cfd_->GetSuperVersionNumber()); + } + delete t; + } else { // current mutable memtable has range tombstones + if (!memtable_range_tombstone_iter_) { + delete t; + db_impl_->ReturnAndCleanupSuperVersion(cfd_, sv); + // The memtable under DBIter did not have range tombstone before + // refresh. + reinit_internal_iter(); + break; + } else { + delete *memtable_range_tombstone_iter_; + *memtable_range_tombstone_iter_ = new TruncatedRangeDelIterator( + std::unique_ptr(t), + &cfd_->internal_comparator(), nullptr, nullptr); + } + } + db_impl_->ReturnAndCleanupSuperVersion(cfd_, sv); + } + // Refresh latest sequence number + db_iter_->set_sequence(latest_seq); + db_iter_->set_valid(false); + // Check again if the latest super version number is changed + uint64_t latest_sv_number = cfd_->GetSuperVersionNumber(); + if (latest_sv_number != cur_sv_number) { + // If the super version number is changed after refreshing, + // fallback to Re-Init the InternalIterator + cur_sv_number = latest_sv_number; + continue; + } + break; + } + } + return Status::OK(); +} + +ArenaWrappedDBIter* NewArenaWrappedDbIterator( + Env* env, const ReadOptions& read_options, const ImmutableOptions& ioptions, + const MutableCFOptions& mutable_cf_options, const Version* version, + const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations, + uint64_t version_number, ReadCallback* read_callback, DBImpl* db_impl, + ColumnFamilyData* cfd, bool expose_blob_index, bool allow_refresh) { + ArenaWrappedDBIter* iter = new ArenaWrappedDBIter(); + iter->Init(env, read_options, ioptions, mutable_cf_options, version, sequence, + max_sequential_skip_in_iterations, version_number, read_callback, + db_impl, cfd, expose_blob_index, allow_refresh); + if (db_impl != nullptr && cfd != nullptr && allow_refresh) { + iter->StoreRefreshInfo(db_impl, cfd, read_callback, expose_blob_index); + } + + return iter; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/arena_wrapped_db_iter.h b/librocksdb-sys/rocksdb/db/arena_wrapped_db_iter.h new file mode 100644 index 0000000..f15be30 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/arena_wrapped_db_iter.h @@ -0,0 +1,127 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include + +#include + +#include "db/db_impl/db_impl.h" +#include "db/db_iter.h" +#include "db/range_del_aggregator.h" +#include "memory/arena.h" +#include "options/cf_options.h" +#include "rocksdb/db.h" +#include "rocksdb/iterator.h" +#include "util/autovector.h" + +namespace ROCKSDB_NAMESPACE { + +class Arena; +class Version; + +// A wrapper iterator which wraps DB Iterator and the arena, with which the DB +// iterator is supposed to be allocated. This class is used as an entry point of +// a iterator hierarchy whose memory can be allocated inline. In that way, +// accessing the iterator tree can be more cache friendly. It is also faster +// to allocate. +// When using the class's Iterator interface, the behavior is exactly +// the same as the inner DBIter. +class ArenaWrappedDBIter : public Iterator { + public: + ~ArenaWrappedDBIter() override { + if (db_iter_ != nullptr) { + db_iter_->~DBIter(); + } else { + assert(false); + } + } + + // Get the arena to be used to allocate memory for DBIter to be wrapped, + // as well as child iterators in it. + virtual Arena* GetArena() { return &arena_; } + + const ReadOptions& GetReadOptions() { return read_options_; } + + // Set the internal iterator wrapped inside the DB Iterator. Usually it is + // a merging iterator. + virtual void SetIterUnderDBIter(InternalIterator* iter) { + db_iter_->SetIter(iter); + } + + void SetMemtableRangetombstoneIter(TruncatedRangeDelIterator** iter) { + memtable_range_tombstone_iter_ = iter; + } + + bool Valid() const override { return db_iter_->Valid(); } + void SeekToFirst() override { db_iter_->SeekToFirst(); } + void SeekToLast() override { db_iter_->SeekToLast(); } + // 'target' does not contain timestamp, even if user timestamp feature is + // enabled. + void Seek(const Slice& target) override { db_iter_->Seek(target); } + void SeekForPrev(const Slice& target) override { + db_iter_->SeekForPrev(target); + } + void Next() override { db_iter_->Next(); } + void Prev() override { db_iter_->Prev(); } + Slice key() const override { return db_iter_->key(); } + Slice value() const override { return db_iter_->value(); } + const WideColumns& columns() const override { return db_iter_->columns(); } + Status status() const override { return db_iter_->status(); } + Slice timestamp() const override { return db_iter_->timestamp(); } + bool IsBlob() const { return db_iter_->IsBlob(); } + + Status GetProperty(std::string prop_name, std::string* prop) override; + + Status Refresh() override; + + void Init(Env* env, const ReadOptions& read_options, + const ImmutableOptions& ioptions, + const MutableCFOptions& mutable_cf_options, const Version* version, + const SequenceNumber& sequence, + uint64_t max_sequential_skip_in_iterations, uint64_t version_number, + ReadCallback* read_callback, DBImpl* db_impl, ColumnFamilyData* cfd, + bool expose_blob_index, bool allow_refresh); + + // Store some parameters so we can refresh the iterator at a later point + // with these same params + void StoreRefreshInfo(DBImpl* db_impl, ColumnFamilyData* cfd, + ReadCallback* read_callback, bool expose_blob_index) { + db_impl_ = db_impl; + cfd_ = cfd; + read_callback_ = read_callback; + expose_blob_index_ = expose_blob_index; + } + + private: + DBIter* db_iter_ = nullptr; + Arena arena_; + uint64_t sv_number_; + ColumnFamilyData* cfd_ = nullptr; + DBImpl* db_impl_ = nullptr; + ReadOptions read_options_; + ReadCallback* read_callback_; + bool expose_blob_index_ = false; + bool allow_refresh_ = true; + // If this is nullptr, it means the mutable memtable does not contain range + // tombstone when added under this DBIter. + TruncatedRangeDelIterator** memtable_range_tombstone_iter_ = nullptr; +}; + +// Generate the arena wrapped iterator class. +// `db_impl` and `cfd` are used for reneweal. If left null, renewal will not +// be supported. +extern ArenaWrappedDBIter* NewArenaWrappedDbIterator( + Env* env, const ReadOptions& read_options, const ImmutableOptions& ioptions, + const MutableCFOptions& mutable_cf_options, const Version* version, + const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations, + uint64_t version_number, ReadCallback* read_callback, + DBImpl* db_impl = nullptr, ColumnFamilyData* cfd = nullptr, + bool expose_blob_index = false, bool allow_refresh = true); +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_constants.h b/librocksdb-sys/rocksdb/db/blob/blob_constants.h new file mode 100644 index 0000000..a5d09ac --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_constants.h @@ -0,0 +1,16 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +constexpr uint64_t kInvalidBlobFileNumber = 0; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_contents.cc b/librocksdb-sys/rocksdb/db/blob/blob_contents.cc new file mode 100644 index 0000000..8b793c5 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_contents.cc @@ -0,0 +1,42 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_contents.h" + +#include + +#include "cache/cache_entry_roles.h" +#include "cache/cache_helpers.h" +#include "port/malloc.h" + +namespace ROCKSDB_NAMESPACE { + +size_t BlobContents::ApproximateMemoryUsage() const { + size_t usage = 0; + + if (allocation_) { + MemoryAllocator* const allocator = allocation_.get_deleter().allocator; + + if (allocator) { + usage += allocator->UsableSize(allocation_.get(), data_.size()); + } else { +#ifdef ROCKSDB_MALLOC_USABLE_SIZE + usage += malloc_usable_size(allocation_.get()); +#else + usage += data_.size(); +#endif + } + } + +#ifdef ROCKSDB_MALLOC_USABLE_SIZE + usage += malloc_usable_size(const_cast(this)); +#else + usage += sizeof(*this); +#endif + + return usage; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_contents.h b/librocksdb-sys/rocksdb/db/blob/blob_contents.h new file mode 100644 index 0000000..15a672a --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_contents.h @@ -0,0 +1,59 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "memory/memory_allocator_impl.h" +#include "rocksdb/advanced_cache.h" +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +// A class representing a single uncompressed value read from a blob file. +class BlobContents { + public: + BlobContents(CacheAllocationPtr&& allocation, size_t size) + : allocation_(std::move(allocation)), data_(allocation_.get(), size) {} + + BlobContents(const BlobContents&) = delete; + BlobContents& operator=(const BlobContents&) = delete; + + BlobContents(BlobContents&&) = default; + BlobContents& operator=(BlobContents&&) = default; + + ~BlobContents() = default; + + const Slice& data() const { return data_; } + size_t size() const { return data_.size(); } + + size_t ApproximateMemoryUsage() const; + + // For TypedCacheInterface + const Slice& ContentSlice() const { return data_; } + static constexpr CacheEntryRole kCacheEntryRole = CacheEntryRole::kBlobValue; + + private: + CacheAllocationPtr allocation_; + Slice data_; +}; + +class BlobContentsCreator : public Cache::CreateContext { + public: + static void Create(std::unique_ptr* out, size_t* out_charge, + const Slice& contents, MemoryAllocator* alloc) { + auto raw = new BlobContents(AllocateAndCopyBlock(contents, alloc), + contents.size()); + out->reset(raw); + if (out_charge) { + *out_charge = raw->ApproximateMemoryUsage(); + } + } +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_counting_iterator.h b/librocksdb-sys/rocksdb/db/blob/blob_counting_iterator.h new file mode 100644 index 0000000..b21651f --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_counting_iterator.h @@ -0,0 +1,150 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "db/blob/blob_garbage_meter.h" +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/status.h" +#include "table/internal_iterator.h" +#include "test_util/sync_point.h" + +namespace ROCKSDB_NAMESPACE { + +// An internal iterator that passes each key-value encountered to +// BlobGarbageMeter as inflow in order to measure the total number and size of +// blobs in the compaction input on a per-blob file basis. +class BlobCountingIterator : public InternalIterator { + public: + BlobCountingIterator(InternalIterator* iter, + BlobGarbageMeter* blob_garbage_meter) + : iter_(iter), blob_garbage_meter_(blob_garbage_meter) { + assert(iter_); + assert(blob_garbage_meter_); + + UpdateAndCountBlobIfNeeded(); + } + + bool Valid() const override { return iter_->Valid() && status_.ok(); } + + void SeekToFirst() override { + iter_->SeekToFirst(); + UpdateAndCountBlobIfNeeded(); + } + + void SeekToLast() override { + iter_->SeekToLast(); + UpdateAndCountBlobIfNeeded(); + } + + void Seek(const Slice& target) override { + iter_->Seek(target); + UpdateAndCountBlobIfNeeded(); + } + + void SeekForPrev(const Slice& target) override { + iter_->SeekForPrev(target); + UpdateAndCountBlobIfNeeded(); + } + + void Next() override { + assert(Valid()); + + iter_->Next(); + UpdateAndCountBlobIfNeeded(); + } + + bool NextAndGetResult(IterateResult* result) override { + assert(Valid()); + + const bool res = iter_->NextAndGetResult(result); + UpdateAndCountBlobIfNeeded(); + return res; + } + + void Prev() override { + assert(Valid()); + + iter_->Prev(); + UpdateAndCountBlobIfNeeded(); + } + + Slice key() const override { + assert(Valid()); + return iter_->key(); + } + + Slice user_key() const override { + assert(Valid()); + return iter_->user_key(); + } + + Slice value() const override { + assert(Valid()); + return iter_->value(); + } + + Status status() const override { return status_; } + + bool PrepareValue() override { + assert(Valid()); + return iter_->PrepareValue(); + } + + bool MayBeOutOfLowerBound() override { + assert(Valid()); + return iter_->MayBeOutOfLowerBound(); + } + + IterBoundCheck UpperBoundCheckResult() override { + assert(Valid()); + return iter_->UpperBoundCheckResult(); + } + + void SetPinnedItersMgr(PinnedIteratorsManager* pinned_iters_mgr) override { + iter_->SetPinnedItersMgr(pinned_iters_mgr); + } + + bool IsKeyPinned() const override { + assert(Valid()); + return iter_->IsKeyPinned(); + } + + bool IsValuePinned() const override { + assert(Valid()); + return iter_->IsValuePinned(); + } + + Status GetProperty(std::string prop_name, std::string* prop) override { + return iter_->GetProperty(prop_name, prop); + } + + bool IsDeleteRangeSentinelKey() const override { + return iter_->IsDeleteRangeSentinelKey(); + } + + private: + void UpdateAndCountBlobIfNeeded() { + assert(!iter_->Valid() || iter_->status().ok()); + + if (!iter_->Valid()) { + status_ = iter_->status(); + return; + } + + TEST_SYNC_POINT( + "BlobCountingIterator::UpdateAndCountBlobIfNeeded:ProcessInFlow"); + + status_ = blob_garbage_meter_->ProcessInFlow(key(), value()); + } + + InternalIterator* iter_; + BlobGarbageMeter* blob_garbage_meter_; + Status status_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_counting_iterator_test.cc b/librocksdb-sys/rocksdb/db/blob/blob_counting_iterator_test.cc new file mode 100644 index 0000000..c7bbc8f --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_counting_iterator_test.cc @@ -0,0 +1,327 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_counting_iterator.h" + +#include +#include + +#include "db/blob/blob_garbage_meter.h" +#include "db/blob/blob_index.h" +#include "db/blob/blob_log_format.h" +#include "db/dbformat.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/vector_iterator.h" + +namespace ROCKSDB_NAMESPACE { + +void CheckInFlow(const BlobGarbageMeter& blob_garbage_meter, + uint64_t blob_file_number, uint64_t count, uint64_t bytes) { + const auto& flows = blob_garbage_meter.flows(); + + const auto it = flows.find(blob_file_number); + if (it == flows.end()) { + ASSERT_EQ(count, 0); + ASSERT_EQ(bytes, 0); + return; + } + + const auto& in = it->second.GetInFlow(); + + ASSERT_EQ(in.GetCount(), count); + ASSERT_EQ(in.GetBytes(), bytes); +} + +TEST(BlobCountingIteratorTest, CountBlobs) { + // Note: the input consists of three key-values: two are blob references to + // different blob files, while the third one is a plain value. + constexpr char user_key0[] = "key0"; + constexpr char user_key1[] = "key1"; + constexpr char user_key2[] = "key2"; + + const std::vector keys{ + test::KeyStr(user_key0, 1, kTypeBlobIndex), + test::KeyStr(user_key1, 2, kTypeBlobIndex), + test::KeyStr(user_key2, 3, kTypeValue)}; + + constexpr uint64_t first_blob_file_number = 4; + constexpr uint64_t first_offset = 1000; + constexpr uint64_t first_size = 2000; + + std::string first_blob_index; + BlobIndex::EncodeBlob(&first_blob_index, first_blob_file_number, first_offset, + first_size, kNoCompression); + + constexpr uint64_t second_blob_file_number = 6; + constexpr uint64_t second_offset = 2000; + constexpr uint64_t second_size = 4000; + + std::string second_blob_index; + BlobIndex::EncodeBlob(&second_blob_index, second_blob_file_number, + second_offset, second_size, kNoCompression); + + const std::vector values{first_blob_index, second_blob_index, + "raw_value"}; + + assert(keys.size() == values.size()); + + VectorIterator input(keys, values); + BlobGarbageMeter blob_garbage_meter; + + BlobCountingIterator blob_counter(&input, &blob_garbage_meter); + + constexpr uint64_t first_expected_bytes = + first_size + + BlobLogRecord::CalculateAdjustmentForRecordHeader(sizeof(user_key0) - 1); + constexpr uint64_t second_expected_bytes = + second_size + + BlobLogRecord::CalculateAdjustmentForRecordHeader(sizeof(user_key1) - 1); + + // Call SeekToFirst and iterate forward + blob_counter.SeekToFirst(); + ASSERT_TRUE(blob_counter.Valid()); + ASSERT_OK(blob_counter.status()); + ASSERT_EQ(blob_counter.key(), keys[0]); + ASSERT_EQ(blob_counter.user_key(), user_key0); + ASSERT_EQ(blob_counter.value(), values[0]); + CheckInFlow(blob_garbage_meter, first_blob_file_number, 1, + first_expected_bytes); + CheckInFlow(blob_garbage_meter, second_blob_file_number, 0, 0); + + blob_counter.Next(); + ASSERT_TRUE(blob_counter.Valid()); + ASSERT_OK(blob_counter.status()); + ASSERT_EQ(blob_counter.key(), keys[1]); + ASSERT_EQ(blob_counter.user_key(), user_key1); + ASSERT_EQ(blob_counter.value(), values[1]); + CheckInFlow(blob_garbage_meter, first_blob_file_number, 1, + first_expected_bytes); + CheckInFlow(blob_garbage_meter, second_blob_file_number, 1, + second_expected_bytes); + + blob_counter.Next(); + ASSERT_TRUE(blob_counter.Valid()); + ASSERT_OK(blob_counter.status()); + ASSERT_EQ(blob_counter.key(), keys[2]); + ASSERT_EQ(blob_counter.user_key(), user_key2); + ASSERT_EQ(blob_counter.value(), values[2]); + CheckInFlow(blob_garbage_meter, first_blob_file_number, 1, + first_expected_bytes); + CheckInFlow(blob_garbage_meter, second_blob_file_number, 1, + second_expected_bytes); + + blob_counter.Next(); + ASSERT_FALSE(blob_counter.Valid()); + ASSERT_OK(blob_counter.status()); + CheckInFlow(blob_garbage_meter, first_blob_file_number, 1, + first_expected_bytes); + CheckInFlow(blob_garbage_meter, second_blob_file_number, 1, + second_expected_bytes); + + // Do it again using NextAndGetResult + blob_counter.SeekToFirst(); + ASSERT_TRUE(blob_counter.Valid()); + ASSERT_OK(blob_counter.status()); + ASSERT_EQ(blob_counter.key(), keys[0]); + ASSERT_EQ(blob_counter.user_key(), user_key0); + ASSERT_EQ(blob_counter.value(), values[0]); + CheckInFlow(blob_garbage_meter, first_blob_file_number, 2, + 2 * first_expected_bytes); + CheckInFlow(blob_garbage_meter, second_blob_file_number, 1, + second_expected_bytes); + + { + IterateResult result; + ASSERT_TRUE(blob_counter.NextAndGetResult(&result)); + ASSERT_EQ(result.key, keys[1]); + ASSERT_EQ(blob_counter.user_key(), user_key1); + ASSERT_TRUE(blob_counter.Valid()); + ASSERT_OK(blob_counter.status()); + ASSERT_EQ(blob_counter.key(), keys[1]); + ASSERT_EQ(blob_counter.value(), values[1]); + CheckInFlow(blob_garbage_meter, first_blob_file_number, 2, + 2 * first_expected_bytes); + CheckInFlow(blob_garbage_meter, second_blob_file_number, 2, + 2 * second_expected_bytes); + } + + { + IterateResult result; + ASSERT_TRUE(blob_counter.NextAndGetResult(&result)); + ASSERT_EQ(result.key, keys[2]); + ASSERT_EQ(blob_counter.user_key(), user_key2); + ASSERT_TRUE(blob_counter.Valid()); + ASSERT_OK(blob_counter.status()); + ASSERT_EQ(blob_counter.key(), keys[2]); + ASSERT_EQ(blob_counter.value(), values[2]); + CheckInFlow(blob_garbage_meter, first_blob_file_number, 2, + 2 * first_expected_bytes); + CheckInFlow(blob_garbage_meter, second_blob_file_number, 2, + 2 * second_expected_bytes); + } + + { + IterateResult result; + ASSERT_FALSE(blob_counter.NextAndGetResult(&result)); + ASSERT_FALSE(blob_counter.Valid()); + ASSERT_OK(blob_counter.status()); + CheckInFlow(blob_garbage_meter, first_blob_file_number, 2, + 2 * first_expected_bytes); + CheckInFlow(blob_garbage_meter, second_blob_file_number, 2, + 2 * second_expected_bytes); + } + + // Call SeekToLast and iterate backward + blob_counter.SeekToLast(); + ASSERT_TRUE(blob_counter.Valid()); + ASSERT_OK(blob_counter.status()); + ASSERT_EQ(blob_counter.key(), keys[2]); + ASSERT_EQ(blob_counter.user_key(), user_key2); + ASSERT_EQ(blob_counter.value(), values[2]); + CheckInFlow(blob_garbage_meter, first_blob_file_number, 2, + 2 * first_expected_bytes); + CheckInFlow(blob_garbage_meter, second_blob_file_number, 2, + 2 * second_expected_bytes); + + blob_counter.Prev(); + ASSERT_TRUE(blob_counter.Valid()); + ASSERT_OK(blob_counter.status()); + ASSERT_EQ(blob_counter.key(), keys[1]); + ASSERT_EQ(blob_counter.user_key(), user_key1); + ASSERT_EQ(blob_counter.value(), values[1]); + CheckInFlow(blob_garbage_meter, first_blob_file_number, 2, + 2 * first_expected_bytes); + CheckInFlow(blob_garbage_meter, second_blob_file_number, 3, + 3 * second_expected_bytes); + + blob_counter.Prev(); + ASSERT_TRUE(blob_counter.Valid()); + ASSERT_OK(blob_counter.status()); + ASSERT_EQ(blob_counter.key(), keys[0]); + ASSERT_EQ(blob_counter.user_key(), user_key0); + ASSERT_EQ(blob_counter.value(), values[0]); + CheckInFlow(blob_garbage_meter, first_blob_file_number, 3, + 3 * first_expected_bytes); + CheckInFlow(blob_garbage_meter, second_blob_file_number, 3, + 3 * second_expected_bytes); + + blob_counter.Prev(); + ASSERT_FALSE(blob_counter.Valid()); + ASSERT_OK(blob_counter.status()); + CheckInFlow(blob_garbage_meter, first_blob_file_number, 3, + 3 * first_expected_bytes); + CheckInFlow(blob_garbage_meter, second_blob_file_number, 3, + 3 * second_expected_bytes); + + // Call Seek for all keys (plus one that's greater than all of them) + blob_counter.Seek(keys[0]); + ASSERT_TRUE(blob_counter.Valid()); + ASSERT_OK(blob_counter.status()); + ASSERT_EQ(blob_counter.key(), keys[0]); + ASSERT_EQ(blob_counter.user_key(), user_key0); + ASSERT_EQ(blob_counter.value(), values[0]); + CheckInFlow(blob_garbage_meter, first_blob_file_number, 4, + 4 * first_expected_bytes); + CheckInFlow(blob_garbage_meter, second_blob_file_number, 3, + 3 * second_expected_bytes); + + blob_counter.Seek(keys[1]); + ASSERT_TRUE(blob_counter.Valid()); + ASSERT_OK(blob_counter.status()); + ASSERT_EQ(blob_counter.key(), keys[1]); + ASSERT_EQ(blob_counter.user_key(), user_key1); + ASSERT_EQ(blob_counter.value(), values[1]); + CheckInFlow(blob_garbage_meter, first_blob_file_number, 4, + 4 * first_expected_bytes); + CheckInFlow(blob_garbage_meter, second_blob_file_number, 4, + 4 * second_expected_bytes); + + blob_counter.Seek(keys[2]); + ASSERT_TRUE(blob_counter.Valid()); + ASSERT_OK(blob_counter.status()); + ASSERT_EQ(blob_counter.key(), keys[2]); + ASSERT_EQ(blob_counter.user_key(), user_key2); + ASSERT_EQ(blob_counter.value(), values[2]); + CheckInFlow(blob_garbage_meter, first_blob_file_number, 4, + 4 * first_expected_bytes); + CheckInFlow(blob_garbage_meter, second_blob_file_number, 4, + 4 * second_expected_bytes); + + blob_counter.Seek("zzz"); + ASSERT_FALSE(blob_counter.Valid()); + ASSERT_OK(blob_counter.status()); + CheckInFlow(blob_garbage_meter, first_blob_file_number, 4, + 4 * first_expected_bytes); + CheckInFlow(blob_garbage_meter, second_blob_file_number, 4, + 4 * second_expected_bytes); + + // Call SeekForPrev for all keys (plus one that's less than all of them) + blob_counter.SeekForPrev("aaa"); + ASSERT_FALSE(blob_counter.Valid()); + ASSERT_OK(blob_counter.status()); + CheckInFlow(blob_garbage_meter, first_blob_file_number, 4, + 4 * first_expected_bytes); + CheckInFlow(blob_garbage_meter, second_blob_file_number, 4, + 4 * second_expected_bytes); + + blob_counter.SeekForPrev(keys[0]); + ASSERT_TRUE(blob_counter.Valid()); + ASSERT_OK(blob_counter.status()); + ASSERT_EQ(blob_counter.key(), keys[0]); + ASSERT_EQ(blob_counter.user_key(), user_key0); + ASSERT_EQ(blob_counter.value(), values[0]); + CheckInFlow(blob_garbage_meter, first_blob_file_number, 5, + 5 * first_expected_bytes); + CheckInFlow(blob_garbage_meter, second_blob_file_number, 4, + 4 * second_expected_bytes); + + blob_counter.SeekForPrev(keys[1]); + ASSERT_TRUE(blob_counter.Valid()); + ASSERT_OK(blob_counter.status()); + ASSERT_EQ(blob_counter.key(), keys[1]); + ASSERT_EQ(blob_counter.user_key(), user_key1); + ASSERT_EQ(blob_counter.value(), values[1]); + CheckInFlow(blob_garbage_meter, first_blob_file_number, 5, + 5 * first_expected_bytes); + CheckInFlow(blob_garbage_meter, second_blob_file_number, 5, + 5 * second_expected_bytes); + + blob_counter.SeekForPrev(keys[2]); + ASSERT_TRUE(blob_counter.Valid()); + ASSERT_OK(blob_counter.status()); + ASSERT_EQ(blob_counter.key(), keys[2]); + ASSERT_EQ(blob_counter.user_key(), user_key2); + ASSERT_EQ(blob_counter.value(), values[2]); + CheckInFlow(blob_garbage_meter, first_blob_file_number, 5, + 5 * first_expected_bytes); + CheckInFlow(blob_garbage_meter, second_blob_file_number, 5, + 5 * second_expected_bytes); +} + +TEST(BlobCountingIteratorTest, CorruptBlobIndex) { + const std::vector keys{ + test::KeyStr("user_key", 1, kTypeBlobIndex)}; + const std::vector values{"i_am_not_a_blob_index"}; + + assert(keys.size() == values.size()); + + VectorIterator input(keys, values); + BlobGarbageMeter blob_garbage_meter; + + BlobCountingIterator blob_counter(&input, &blob_garbage_meter); + + blob_counter.SeekToFirst(); + ASSERT_FALSE(blob_counter.Valid()); + ASSERT_NOK(blob_counter.status()); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/blob/blob_fetcher.cc b/librocksdb-sys/rocksdb/db/blob/blob_fetcher.cc new file mode 100644 index 0000000..124429f --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_fetcher.cc @@ -0,0 +1,34 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_fetcher.h" + +#include "db/version_set.h" + +namespace ROCKSDB_NAMESPACE { + +Status BlobFetcher::FetchBlob(const Slice& user_key, + const Slice& blob_index_slice, + FilePrefetchBuffer* prefetch_buffer, + PinnableSlice* blob_value, + uint64_t* bytes_read) const { + assert(version_); + + return version_->GetBlob(read_options_, user_key, blob_index_slice, + prefetch_buffer, blob_value, bytes_read); +} + +Status BlobFetcher::FetchBlob(const Slice& user_key, + const BlobIndex& blob_index, + FilePrefetchBuffer* prefetch_buffer, + PinnableSlice* blob_value, + uint64_t* bytes_read) const { + assert(version_); + + return version_->GetBlob(read_options_, user_key, blob_index, prefetch_buffer, + blob_value, bytes_read); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_fetcher.h b/librocksdb-sys/rocksdb/db/blob/blob_fetcher.h new file mode 100644 index 0000000..8aeaf96 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_fetcher.h @@ -0,0 +1,37 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "rocksdb/options.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +class Version; +class Slice; +class FilePrefetchBuffer; +class PinnableSlice; +class BlobIndex; + +// A thin wrapper around the blob retrieval functionality of Version. +class BlobFetcher { + public: + BlobFetcher(const Version* version, const ReadOptions& read_options) + : version_(version), read_options_(read_options) {} + + Status FetchBlob(const Slice& user_key, const Slice& blob_index_slice, + FilePrefetchBuffer* prefetch_buffer, + PinnableSlice* blob_value, uint64_t* bytes_read) const; + + Status FetchBlob(const Slice& user_key, const BlobIndex& blob_index, + FilePrefetchBuffer* prefetch_buffer, + PinnableSlice* blob_value, uint64_t* bytes_read) const; + + private: + const Version* version_; + ReadOptions read_options_; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_file_addition.cc b/librocksdb-sys/rocksdb/db/blob/blob_file_addition.cc new file mode 100644 index 0000000..71b1bb7 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_file_addition.cc @@ -0,0 +1,156 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_file_addition.h" + +#include +#include + +#include "logging/event_logger.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "test_util/sync_point.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { + +// Tags for custom fields. Note that these get persisted in the manifest, +// so existing tags should not be modified. +enum BlobFileAddition::CustomFieldTags : uint32_t { + kEndMarker, + + // Add forward compatible fields here + + ///////////////////////////////////////////////////////////////////// + + kForwardIncompatibleMask = 1 << 6, + + // Add forward incompatible fields here +}; + +void BlobFileAddition::EncodeTo(std::string* output) const { + PutVarint64(output, blob_file_number_); + PutVarint64(output, total_blob_count_); + PutVarint64(output, total_blob_bytes_); + PutLengthPrefixedSlice(output, checksum_method_); + PutLengthPrefixedSlice(output, checksum_value_); + + // Encode any custom fields here. The format to use is a Varint32 tag (see + // CustomFieldTags above) followed by a length prefixed slice. Unknown custom + // fields will be ignored during decoding unless they're in the forward + // incompatible range. + + TEST_SYNC_POINT_CALLBACK("BlobFileAddition::EncodeTo::CustomFields", output); + + PutVarint32(output, kEndMarker); +} + +Status BlobFileAddition::DecodeFrom(Slice* input) { + constexpr char class_name[] = "BlobFileAddition"; + + if (!GetVarint64(input, &blob_file_number_)) { + return Status::Corruption(class_name, "Error decoding blob file number"); + } + + if (!GetVarint64(input, &total_blob_count_)) { + return Status::Corruption(class_name, "Error decoding total blob count"); + } + + if (!GetVarint64(input, &total_blob_bytes_)) { + return Status::Corruption(class_name, "Error decoding total blob bytes"); + } + + Slice checksum_method; + if (!GetLengthPrefixedSlice(input, &checksum_method)) { + return Status::Corruption(class_name, "Error decoding checksum method"); + } + checksum_method_ = checksum_method.ToString(); + + Slice checksum_value; + if (!GetLengthPrefixedSlice(input, &checksum_value)) { + return Status::Corruption(class_name, "Error decoding checksum value"); + } + checksum_value_ = checksum_value.ToString(); + + while (true) { + uint32_t custom_field_tag = 0; + if (!GetVarint32(input, &custom_field_tag)) { + return Status::Corruption(class_name, "Error decoding custom field tag"); + } + + if (custom_field_tag == kEndMarker) { + break; + } + + if (custom_field_tag & kForwardIncompatibleMask) { + return Status::Corruption( + class_name, "Forward incompatible custom field encountered"); + } + + Slice custom_field_value; + if (!GetLengthPrefixedSlice(input, &custom_field_value)) { + return Status::Corruption(class_name, + "Error decoding custom field value"); + } + } + + return Status::OK(); +} + +std::string BlobFileAddition::DebugString() const { + std::ostringstream oss; + + oss << *this; + + return oss.str(); +} + +std::string BlobFileAddition::DebugJSON() const { + JSONWriter jw; + + jw << *this; + + jw.EndObject(); + + return jw.Get(); +} + +bool operator==(const BlobFileAddition& lhs, const BlobFileAddition& rhs) { + return lhs.GetBlobFileNumber() == rhs.GetBlobFileNumber() && + lhs.GetTotalBlobCount() == rhs.GetTotalBlobCount() && + lhs.GetTotalBlobBytes() == rhs.GetTotalBlobBytes() && + lhs.GetChecksumMethod() == rhs.GetChecksumMethod() && + lhs.GetChecksumValue() == rhs.GetChecksumValue(); +} + +bool operator!=(const BlobFileAddition& lhs, const BlobFileAddition& rhs) { + return !(lhs == rhs); +} + +std::ostream& operator<<(std::ostream& os, + const BlobFileAddition& blob_file_addition) { + os << "blob_file_number: " << blob_file_addition.GetBlobFileNumber() + << " total_blob_count: " << blob_file_addition.GetTotalBlobCount() + << " total_blob_bytes: " << blob_file_addition.GetTotalBlobBytes() + << " checksum_method: " << blob_file_addition.GetChecksumMethod() + << " checksum_value: " + << Slice(blob_file_addition.GetChecksumValue()).ToString(/* hex */ true); + + return os; +} + +JSONWriter& operator<<(JSONWriter& jw, + const BlobFileAddition& blob_file_addition) { + jw << "BlobFileNumber" << blob_file_addition.GetBlobFileNumber() + << "TotalBlobCount" << blob_file_addition.GetTotalBlobCount() + << "TotalBlobBytes" << blob_file_addition.GetTotalBlobBytes() + << "ChecksumMethod" << blob_file_addition.GetChecksumMethod() + << "ChecksumValue" + << Slice(blob_file_addition.GetChecksumValue()).ToString(/* hex */ true); + + return jw; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_file_addition.h b/librocksdb-sys/rocksdb/db/blob/blob_file_addition.h new file mode 100644 index 0000000..43b1a0b --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_file_addition.h @@ -0,0 +1,67 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include +#include + +#include "db/blob/blob_constants.h" +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +class JSONWriter; +class Slice; +class Status; + +class BlobFileAddition { + public: + BlobFileAddition() = default; + + BlobFileAddition(uint64_t blob_file_number, uint64_t total_blob_count, + uint64_t total_blob_bytes, std::string checksum_method, + std::string checksum_value) + : blob_file_number_(blob_file_number), + total_blob_count_(total_blob_count), + total_blob_bytes_(total_blob_bytes), + checksum_method_(std::move(checksum_method)), + checksum_value_(std::move(checksum_value)) { + assert(checksum_method_.empty() == checksum_value_.empty()); + } + + uint64_t GetBlobFileNumber() const { return blob_file_number_; } + uint64_t GetTotalBlobCount() const { return total_blob_count_; } + uint64_t GetTotalBlobBytes() const { return total_blob_bytes_; } + const std::string& GetChecksumMethod() const { return checksum_method_; } + const std::string& GetChecksumValue() const { return checksum_value_; } + + void EncodeTo(std::string* output) const; + Status DecodeFrom(Slice* input); + + std::string DebugString() const; + std::string DebugJSON() const; + + private: + enum CustomFieldTags : uint32_t; + + uint64_t blob_file_number_ = kInvalidBlobFileNumber; + uint64_t total_blob_count_ = 0; + uint64_t total_blob_bytes_ = 0; + std::string checksum_method_; + std::string checksum_value_; +}; + +bool operator==(const BlobFileAddition& lhs, const BlobFileAddition& rhs); +bool operator!=(const BlobFileAddition& lhs, const BlobFileAddition& rhs); + +std::ostream& operator<<(std::ostream& os, + const BlobFileAddition& blob_file_addition); +JSONWriter& operator<<(JSONWriter& jw, + const BlobFileAddition& blob_file_addition); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_file_addition_test.cc b/librocksdb-sys/rocksdb/db/blob/blob_file_addition_test.cc new file mode 100644 index 0000000..64cb0a9 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_file_addition_test.cc @@ -0,0 +1,211 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_file_addition.h" + +#include +#include +#include + +#include "test_util/sync_point.h" +#include "test_util/testharness.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { + +class BlobFileAdditionTest : public testing::Test { + public: + static void TestEncodeDecode(const BlobFileAddition& blob_file_addition) { + std::string encoded; + blob_file_addition.EncodeTo(&encoded); + + BlobFileAddition decoded; + Slice input(encoded); + ASSERT_OK(decoded.DecodeFrom(&input)); + + ASSERT_EQ(blob_file_addition, decoded); + } +}; + +TEST_F(BlobFileAdditionTest, Empty) { + BlobFileAddition blob_file_addition; + + ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), kInvalidBlobFileNumber); + ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), 0); + ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(), 0); + ASSERT_TRUE(blob_file_addition.GetChecksumMethod().empty()); + ASSERT_TRUE(blob_file_addition.GetChecksumValue().empty()); + + TestEncodeDecode(blob_file_addition); +} + +TEST_F(BlobFileAdditionTest, NonEmpty) { + constexpr uint64_t blob_file_number = 123; + constexpr uint64_t total_blob_count = 2; + constexpr uint64_t total_blob_bytes = 123456; + const std::string checksum_method("SHA1"); + const std::string checksum_value( + "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52" + "\x5c\xbd"); + + BlobFileAddition blob_file_addition(blob_file_number, total_blob_count, + total_blob_bytes, checksum_method, + checksum_value); + + ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number); + ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), total_blob_count); + ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(), total_blob_bytes); + ASSERT_EQ(blob_file_addition.GetChecksumMethod(), checksum_method); + ASSERT_EQ(blob_file_addition.GetChecksumValue(), checksum_value); + + TestEncodeDecode(blob_file_addition); +} + +TEST_F(BlobFileAdditionTest, DecodeErrors) { + std::string str; + Slice slice(str); + + BlobFileAddition blob_file_addition; + + { + const Status s = blob_file_addition.DecodeFrom(&slice); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "blob file number")); + } + + constexpr uint64_t blob_file_number = 123; + PutVarint64(&str, blob_file_number); + slice = str; + + { + const Status s = blob_file_addition.DecodeFrom(&slice); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "total blob count")); + } + + constexpr uint64_t total_blob_count = 4567; + PutVarint64(&str, total_blob_count); + slice = str; + + { + const Status s = blob_file_addition.DecodeFrom(&slice); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "total blob bytes")); + } + + constexpr uint64_t total_blob_bytes = 12345678; + PutVarint64(&str, total_blob_bytes); + slice = str; + + { + const Status s = blob_file_addition.DecodeFrom(&slice); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "checksum method")); + } + + constexpr char checksum_method[] = "SHA1"; + PutLengthPrefixedSlice(&str, checksum_method); + slice = str; + + { + const Status s = blob_file_addition.DecodeFrom(&slice); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "checksum value")); + } + + constexpr char checksum_value[] = + "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52" + "\x5c\xbd"; + PutLengthPrefixedSlice(&str, checksum_value); + slice = str; + + { + const Status s = blob_file_addition.DecodeFrom(&slice); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "custom field tag")); + } + + constexpr uint32_t custom_tag = 2; + PutVarint32(&str, custom_tag); + slice = str; + + { + const Status s = blob_file_addition.DecodeFrom(&slice); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "custom field value")); + } +} + +TEST_F(BlobFileAdditionTest, ForwardCompatibleCustomField) { + SyncPoint::GetInstance()->SetCallBack( + "BlobFileAddition::EncodeTo::CustomFields", [&](void* arg) { + std::string* output = static_cast(arg); + + constexpr uint32_t forward_compatible_tag = 2; + PutVarint32(output, forward_compatible_tag); + + PutLengthPrefixedSlice(output, "deadbeef"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + constexpr uint64_t blob_file_number = 678; + constexpr uint64_t total_blob_count = 9999; + constexpr uint64_t total_blob_bytes = 100000000; + const std::string checksum_method("CRC32"); + const std::string checksum_value("\x3d\x87\xff\x57"); + + BlobFileAddition blob_file_addition(blob_file_number, total_blob_count, + total_blob_bytes, checksum_method, + checksum_value); + + TestEncodeDecode(blob_file_addition); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(BlobFileAdditionTest, ForwardIncompatibleCustomField) { + SyncPoint::GetInstance()->SetCallBack( + "BlobFileAddition::EncodeTo::CustomFields", [&](void* arg) { + std::string* output = static_cast(arg); + + constexpr uint32_t forward_incompatible_tag = (1 << 6) + 1; + PutVarint32(output, forward_incompatible_tag); + + PutLengthPrefixedSlice(output, "foobar"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + constexpr uint64_t blob_file_number = 456; + constexpr uint64_t total_blob_count = 100; + constexpr uint64_t total_blob_bytes = 2000000; + const std::string checksum_method("CRC32B"); + const std::string checksum_value("\x6d\xbd\xf2\x3a"); + + BlobFileAddition blob_file_addition(blob_file_number, total_blob_count, + total_blob_bytes, checksum_method, + checksum_value); + + std::string encoded; + blob_file_addition.EncodeTo(&encoded); + + BlobFileAddition decoded_blob_file_addition; + Slice input(encoded); + const Status s = decoded_blob_file_addition.DecodeFrom(&input); + + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "Forward incompatible")); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/blob/blob_file_builder.cc b/librocksdb-sys/rocksdb/db/blob/blob_file_builder.cc new file mode 100644 index 0000000..21c1e5d --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_file_builder.cc @@ -0,0 +1,426 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_file_builder.h" + +#include + +#include "db/blob/blob_contents.h" +#include "db/blob/blob_file_addition.h" +#include "db/blob/blob_file_completion_callback.h" +#include "db/blob/blob_index.h" +#include "db/blob/blob_log_format.h" +#include "db/blob/blob_log_writer.h" +#include "db/blob/blob_source.h" +#include "db/event_helpers.h" +#include "db/version_set.h" +#include "file/filename.h" +#include "file/read_write_util.h" +#include "file/writable_file_writer.h" +#include "logging/logging.h" +#include "options/cf_options.h" +#include "options/options_helper.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "test_util/sync_point.h" +#include "trace_replay/io_tracer.h" +#include "util/compression.h" + +namespace ROCKSDB_NAMESPACE { + +BlobFileBuilder::BlobFileBuilder( + VersionSet* versions, FileSystem* fs, + const ImmutableOptions* immutable_options, + const MutableCFOptions* mutable_cf_options, const FileOptions* file_options, + std::string db_id, std::string db_session_id, int job_id, + uint32_t column_family_id, const std::string& column_family_name, + Env::IOPriority io_priority, Env::WriteLifeTimeHint write_hint, + const std::shared_ptr& io_tracer, + BlobFileCompletionCallback* blob_callback, + BlobFileCreationReason creation_reason, + std::vector* blob_file_paths, + std::vector* blob_file_additions) + : BlobFileBuilder([versions]() { return versions->NewFileNumber(); }, fs, + immutable_options, mutable_cf_options, file_options, + db_id, db_session_id, job_id, column_family_id, + column_family_name, io_priority, write_hint, io_tracer, + blob_callback, creation_reason, blob_file_paths, + blob_file_additions) {} + +BlobFileBuilder::BlobFileBuilder( + std::function file_number_generator, FileSystem* fs, + const ImmutableOptions* immutable_options, + const MutableCFOptions* mutable_cf_options, const FileOptions* file_options, + std::string db_id, std::string db_session_id, int job_id, + uint32_t column_family_id, const std::string& column_family_name, + Env::IOPriority io_priority, Env::WriteLifeTimeHint write_hint, + const std::shared_ptr& io_tracer, + BlobFileCompletionCallback* blob_callback, + BlobFileCreationReason creation_reason, + std::vector* blob_file_paths, + std::vector* blob_file_additions) + : file_number_generator_(std::move(file_number_generator)), + fs_(fs), + immutable_options_(immutable_options), + min_blob_size_(mutable_cf_options->min_blob_size), + blob_file_size_(mutable_cf_options->blob_file_size), + blob_compression_type_(mutable_cf_options->blob_compression_type), + prepopulate_blob_cache_(mutable_cf_options->prepopulate_blob_cache), + file_options_(file_options), + db_id_(std::move(db_id)), + db_session_id_(std::move(db_session_id)), + job_id_(job_id), + column_family_id_(column_family_id), + column_family_name_(column_family_name), + io_priority_(io_priority), + write_hint_(write_hint), + io_tracer_(io_tracer), + blob_callback_(blob_callback), + creation_reason_(creation_reason), + blob_file_paths_(blob_file_paths), + blob_file_additions_(blob_file_additions), + blob_count_(0), + blob_bytes_(0) { + assert(file_number_generator_); + assert(fs_); + assert(immutable_options_); + assert(file_options_); + assert(blob_file_paths_); + assert(blob_file_paths_->empty()); + assert(blob_file_additions_); + assert(blob_file_additions_->empty()); +} + +BlobFileBuilder::~BlobFileBuilder() = default; + +Status BlobFileBuilder::Add(const Slice& key, const Slice& value, + std::string* blob_index) { + assert(blob_index); + assert(blob_index->empty()); + + if (value.size() < min_blob_size_) { + return Status::OK(); + } + + { + const Status s = OpenBlobFileIfNeeded(); + if (!s.ok()) { + return s; + } + } + + Slice blob = value; + std::string compressed_blob; + + { + const Status s = CompressBlobIfNeeded(&blob, &compressed_blob); + if (!s.ok()) { + return s; + } + } + + uint64_t blob_file_number = 0; + uint64_t blob_offset = 0; + + { + const Status s = + WriteBlobToFile(key, blob, &blob_file_number, &blob_offset); + if (!s.ok()) { + return s; + } + } + + { + const Status s = CloseBlobFileIfNeeded(); + if (!s.ok()) { + return s; + } + } + + { + const Status s = + PutBlobIntoCacheIfNeeded(value, blob_file_number, blob_offset); + if (!s.ok()) { + ROCKS_LOG_WARN(immutable_options_->info_log, + "Failed to pre-populate the blob into blob cache: %s", + s.ToString().c_str()); + } + } + + BlobIndex::EncodeBlob(blob_index, blob_file_number, blob_offset, blob.size(), + blob_compression_type_); + + return Status::OK(); +} + +Status BlobFileBuilder::Finish() { + if (!IsBlobFileOpen()) { + return Status::OK(); + } + + return CloseBlobFile(); +} + +bool BlobFileBuilder::IsBlobFileOpen() const { return !!writer_; } + +Status BlobFileBuilder::OpenBlobFileIfNeeded() { + if (IsBlobFileOpen()) { + return Status::OK(); + } + + assert(!blob_count_); + assert(!blob_bytes_); + + assert(file_number_generator_); + const uint64_t blob_file_number = file_number_generator_(); + + assert(immutable_options_); + assert(!immutable_options_->cf_paths.empty()); + std::string blob_file_path = + BlobFileName(immutable_options_->cf_paths.front().path, blob_file_number); + + if (blob_callback_) { + blob_callback_->OnBlobFileCreationStarted( + blob_file_path, column_family_name_, job_id_, creation_reason_); + } + + std::unique_ptr file; + + { + assert(file_options_); + Status s = NewWritableFile(fs_, blob_file_path, &file, *file_options_); + + TEST_SYNC_POINT_CALLBACK( + "BlobFileBuilder::OpenBlobFileIfNeeded:NewWritableFile", &s); + + if (!s.ok()) { + return s; + } + } + + // Note: files get added to blob_file_paths_ right after the open, so they + // can be cleaned up upon failure. Contrast this with blob_file_additions_, + // which only contains successfully written files. + assert(blob_file_paths_); + blob_file_paths_->emplace_back(std::move(blob_file_path)); + + assert(file); + file->SetIOPriority(io_priority_); + file->SetWriteLifeTimeHint(write_hint_); + FileTypeSet tmp_set = immutable_options_->checksum_handoff_file_types; + Statistics* const statistics = immutable_options_->stats; + std::unique_ptr file_writer(new WritableFileWriter( + std::move(file), blob_file_paths_->back(), *file_options_, + immutable_options_->clock, io_tracer_, statistics, + immutable_options_->listeners, + immutable_options_->file_checksum_gen_factory.get(), + tmp_set.Contains(FileType::kBlobFile), false)); + + constexpr bool do_flush = false; + + std::unique_ptr blob_log_writer(new BlobLogWriter( + std::move(file_writer), immutable_options_->clock, statistics, + blob_file_number, immutable_options_->use_fsync, do_flush)); + + constexpr bool has_ttl = false; + constexpr ExpirationRange expiration_range; + + BlobLogHeader header(column_family_id_, blob_compression_type_, has_ttl, + expiration_range); + + { + Status s = blob_log_writer->WriteHeader(header); + + TEST_SYNC_POINT_CALLBACK( + "BlobFileBuilder::OpenBlobFileIfNeeded:WriteHeader", &s); + + if (!s.ok()) { + return s; + } + } + + writer_ = std::move(blob_log_writer); + + assert(IsBlobFileOpen()); + + return Status::OK(); +} + +Status BlobFileBuilder::CompressBlobIfNeeded( + Slice* blob, std::string* compressed_blob) const { + assert(blob); + assert(compressed_blob); + assert(compressed_blob->empty()); + assert(immutable_options_); + + if (blob_compression_type_ == kNoCompression) { + return Status::OK(); + } + + // TODO: allow user CompressionOptions, including max_compressed_bytes_per_kb + CompressionOptions opts; + CompressionContext context(blob_compression_type_); + constexpr uint64_t sample_for_compression = 0; + + CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(), + blob_compression_type_, sample_for_compression); + + constexpr uint32_t compression_format_version = 2; + + bool success = false; + + { + StopWatch stop_watch(immutable_options_->clock, immutable_options_->stats, + BLOB_DB_COMPRESSION_MICROS); + success = + CompressData(*blob, info, compression_format_version, compressed_blob); + } + + if (!success) { + return Status::Corruption("Error compressing blob"); + } + + *blob = Slice(*compressed_blob); + + return Status::OK(); +} + +Status BlobFileBuilder::WriteBlobToFile(const Slice& key, const Slice& blob, + uint64_t* blob_file_number, + uint64_t* blob_offset) { + assert(IsBlobFileOpen()); + assert(blob_file_number); + assert(blob_offset); + + uint64_t key_offset = 0; + + Status s = writer_->AddRecord(key, blob, &key_offset, blob_offset); + + TEST_SYNC_POINT_CALLBACK("BlobFileBuilder::WriteBlobToFile:AddRecord", &s); + + if (!s.ok()) { + return s; + } + + *blob_file_number = writer_->get_log_number(); + + ++blob_count_; + blob_bytes_ += BlobLogRecord::kHeaderSize + key.size() + blob.size(); + + return Status::OK(); +} + +Status BlobFileBuilder::CloseBlobFile() { + assert(IsBlobFileOpen()); + + BlobLogFooter footer; + footer.blob_count = blob_count_; + + std::string checksum_method; + std::string checksum_value; + + Status s = writer_->AppendFooter(footer, &checksum_method, &checksum_value); + + TEST_SYNC_POINT_CALLBACK("BlobFileBuilder::WriteBlobToFile:AppendFooter", &s); + + if (!s.ok()) { + return s; + } + + const uint64_t blob_file_number = writer_->get_log_number(); + + if (blob_callback_) { + s = blob_callback_->OnBlobFileCompleted( + blob_file_paths_->back(), column_family_name_, job_id_, + blob_file_number, creation_reason_, s, checksum_value, checksum_method, + blob_count_, blob_bytes_); + } + + assert(blob_file_additions_); + blob_file_additions_->emplace_back(blob_file_number, blob_count_, blob_bytes_, + std::move(checksum_method), + std::move(checksum_value)); + + assert(immutable_options_); + ROCKS_LOG_INFO(immutable_options_->logger, + "[%s] [JOB %d] Generated blob file #%" PRIu64 ": %" PRIu64 + " total blobs, %" PRIu64 " total bytes", + column_family_name_.c_str(), job_id_, blob_file_number, + blob_count_, blob_bytes_); + + writer_.reset(); + blob_count_ = 0; + blob_bytes_ = 0; + + return s; +} + +Status BlobFileBuilder::CloseBlobFileIfNeeded() { + assert(IsBlobFileOpen()); + + const WritableFileWriter* const file_writer = writer_->file(); + assert(file_writer); + + if (file_writer->GetFileSize() < blob_file_size_) { + return Status::OK(); + } + + return CloseBlobFile(); +} + +void BlobFileBuilder::Abandon(const Status& s) { + if (!IsBlobFileOpen()) { + return; + } + if (blob_callback_) { + // BlobFileBuilder::Abandon() is called because of error while writing to + // Blob files. So we can ignore the below error. + blob_callback_ + ->OnBlobFileCompleted(blob_file_paths_->back(), column_family_name_, + job_id_, writer_->get_log_number(), + creation_reason_, s, "", "", blob_count_, + blob_bytes_) + .PermitUncheckedError(); + } + + writer_.reset(); + blob_count_ = 0; + blob_bytes_ = 0; +} + +Status BlobFileBuilder::PutBlobIntoCacheIfNeeded(const Slice& blob, + uint64_t blob_file_number, + uint64_t blob_offset) const { + Status s = Status::OK(); + + BlobSource::SharedCacheInterface blob_cache{immutable_options_->blob_cache}; + auto statistics = immutable_options_->statistics.get(); + bool warm_cache = + prepopulate_blob_cache_ == PrepopulateBlobCache::kFlushOnly && + creation_reason_ == BlobFileCreationReason::kFlush; + + if (blob_cache && warm_cache) { + const OffsetableCacheKey base_cache_key(db_id_, db_session_id_, + blob_file_number); + const CacheKey cache_key = base_cache_key.WithOffset(blob_offset); + const Slice key = cache_key.AsSlice(); + + const Cache::Priority priority = Cache::Priority::BOTTOM; + + s = blob_cache.InsertSaved(key, blob, nullptr /*context*/, priority, + immutable_options_->lowest_used_cache_tier); + + if (s.ok()) { + RecordTick(statistics, BLOB_DB_CACHE_ADD); + RecordTick(statistics, BLOB_DB_CACHE_BYTES_WRITE, blob.size()); + } else { + RecordTick(statistics, BLOB_DB_CACHE_ADD_FAILURES); + } + } + + return s; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_file_builder.h b/librocksdb-sys/rocksdb/db/blob/blob_file_builder.h new file mode 100644 index 0000000..8e7aab5 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_file_builder.h @@ -0,0 +1,112 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include +#include +#include +#include +#include + +#include "rocksdb/advanced_options.h" +#include "rocksdb/compression_type.h" +#include "rocksdb/env.h" +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/types.h" + +namespace ROCKSDB_NAMESPACE { + +class VersionSet; +class FileSystem; +class SystemClock; +struct ImmutableOptions; +struct MutableCFOptions; +struct FileOptions; +class BlobFileAddition; +class Status; +class Slice; +class BlobLogWriter; +class IOTracer; +class BlobFileCompletionCallback; + +class BlobFileBuilder { + public: + BlobFileBuilder(VersionSet* versions, FileSystem* fs, + const ImmutableOptions* immutable_options, + const MutableCFOptions* mutable_cf_options, + const FileOptions* file_options, std::string db_id, + std::string db_session_id, int job_id, + uint32_t column_family_id, + const std::string& column_family_name, + Env::IOPriority io_priority, + Env::WriteLifeTimeHint write_hint, + const std::shared_ptr& io_tracer, + BlobFileCompletionCallback* blob_callback, + BlobFileCreationReason creation_reason, + std::vector* blob_file_paths, + std::vector* blob_file_additions); + + BlobFileBuilder(std::function file_number_generator, + FileSystem* fs, const ImmutableOptions* immutable_options, + const MutableCFOptions* mutable_cf_options, + const FileOptions* file_options, std::string db_id, + std::string db_session_id, int job_id, + uint32_t column_family_id, + const std::string& column_family_name, + Env::IOPriority io_priority, + Env::WriteLifeTimeHint write_hint, + const std::shared_ptr& io_tracer, + BlobFileCompletionCallback* blob_callback, + BlobFileCreationReason creation_reason, + std::vector* blob_file_paths, + std::vector* blob_file_additions); + + BlobFileBuilder(const BlobFileBuilder&) = delete; + BlobFileBuilder& operator=(const BlobFileBuilder&) = delete; + + ~BlobFileBuilder(); + + Status Add(const Slice& key, const Slice& value, std::string* blob_index); + Status Finish(); + void Abandon(const Status& s); + + private: + bool IsBlobFileOpen() const; + Status OpenBlobFileIfNeeded(); + Status CompressBlobIfNeeded(Slice* blob, std::string* compressed_blob) const; + Status WriteBlobToFile(const Slice& key, const Slice& blob, + uint64_t* blob_file_number, uint64_t* blob_offset); + Status CloseBlobFile(); + Status CloseBlobFileIfNeeded(); + + Status PutBlobIntoCacheIfNeeded(const Slice& blob, uint64_t blob_file_number, + uint64_t blob_offset) const; + + std::function file_number_generator_; + FileSystem* fs_; + const ImmutableOptions* immutable_options_; + uint64_t min_blob_size_; + uint64_t blob_file_size_; + CompressionType blob_compression_type_; + PrepopulateBlobCache prepopulate_blob_cache_; + const FileOptions* file_options_; + const std::string db_id_; + const std::string db_session_id_; + int job_id_; + uint32_t column_family_id_; + std::string column_family_name_; + Env::IOPriority io_priority_; + Env::WriteLifeTimeHint write_hint_; + std::shared_ptr io_tracer_; + BlobFileCompletionCallback* blob_callback_; + BlobFileCreationReason creation_reason_; + std::vector* blob_file_paths_; + std::vector* blob_file_additions_; + std::unique_ptr writer_; + uint64_t blob_count_; + uint64_t blob_bytes_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_file_builder_test.cc b/librocksdb-sys/rocksdb/db/blob/blob_file_builder_test.cc new file mode 100644 index 0000000..3a0feee --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_file_builder_test.cc @@ -0,0 +1,680 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_file_builder.h" + +#include +#include +#include +#include +#include + +#include "db/blob/blob_file_addition.h" +#include "db/blob/blob_index.h" +#include "db/blob/blob_log_format.h" +#include "db/blob/blob_log_sequential_reader.h" +#include "env/mock_env.h" +#include "file/filename.h" +#include "file/random_access_file_reader.h" +#include "options/cf_options.h" +#include "rocksdb/env.h" +#include "rocksdb/file_checksum.h" +#include "rocksdb/options.h" +#include "test_util/sync_point.h" +#include "test_util/testharness.h" +#include "util/compression.h" +#include "utilities/fault_injection_env.h" + +namespace ROCKSDB_NAMESPACE { + +class TestFileNumberGenerator { + public: + uint64_t operator()() { return ++next_file_number_; } + + private: + uint64_t next_file_number_ = 1; +}; + +class BlobFileBuilderTest : public testing::Test { + protected: + BlobFileBuilderTest() { + mock_env_.reset(MockEnv::Create(Env::Default())); + fs_ = mock_env_->GetFileSystem().get(); + clock_ = mock_env_->GetSystemClock().get(); + } + + void VerifyBlobFile(uint64_t blob_file_number, + const std::string& blob_file_path, + uint32_t column_family_id, + CompressionType blob_compression_type, + const std::vector>& + expected_key_value_pairs, + const std::vector& blob_indexes) { + assert(expected_key_value_pairs.size() == blob_indexes.size()); + + std::unique_ptr file; + constexpr IODebugContext* dbg = nullptr; + ASSERT_OK( + fs_->NewRandomAccessFile(blob_file_path, file_options_, &file, dbg)); + + std::unique_ptr file_reader( + new RandomAccessFileReader(std::move(file), blob_file_path, clock_)); + + constexpr Statistics* statistics = nullptr; + BlobLogSequentialReader blob_log_reader(std::move(file_reader), clock_, + statistics); + + BlobLogHeader header; + ASSERT_OK(blob_log_reader.ReadHeader(&header)); + ASSERT_EQ(header.version, kVersion1); + ASSERT_EQ(header.column_family_id, column_family_id); + ASSERT_EQ(header.compression, blob_compression_type); + ASSERT_FALSE(header.has_ttl); + ASSERT_EQ(header.expiration_range, ExpirationRange()); + + for (size_t i = 0; i < expected_key_value_pairs.size(); ++i) { + BlobLogRecord record; + uint64_t blob_offset = 0; + + ASSERT_OK(blob_log_reader.ReadRecord( + &record, BlobLogSequentialReader::kReadHeaderKeyBlob, &blob_offset)); + + // Check the contents of the blob file + const auto& expected_key_value = expected_key_value_pairs[i]; + const auto& key = expected_key_value.first; + const auto& value = expected_key_value.second; + + ASSERT_EQ(record.key_size, key.size()); + ASSERT_EQ(record.value_size, value.size()); + ASSERT_EQ(record.expiration, 0); + ASSERT_EQ(record.key, key); + ASSERT_EQ(record.value, value); + + // Make sure the blob reference returned by the builder points to the + // right place + BlobIndex blob_index; + ASSERT_OK(blob_index.DecodeFrom(blob_indexes[i])); + ASSERT_FALSE(blob_index.IsInlined()); + ASSERT_FALSE(blob_index.HasTTL()); + ASSERT_EQ(blob_index.file_number(), blob_file_number); + ASSERT_EQ(blob_index.offset(), blob_offset); + ASSERT_EQ(blob_index.size(), value.size()); + } + + BlobLogFooter footer; + ASSERT_OK(blob_log_reader.ReadFooter(&footer)); + ASSERT_EQ(footer.blob_count, expected_key_value_pairs.size()); + ASSERT_EQ(footer.expiration_range, ExpirationRange()); + } + + std::unique_ptr mock_env_; + FileSystem* fs_; + SystemClock* clock_; + FileOptions file_options_; +}; + +TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) { + // Build a single blob file + constexpr size_t number_of_blobs = 10; + constexpr size_t key_size = 1; + constexpr size_t value_size = 4; + constexpr size_t value_offset = 1234; + + Options options; + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileBuilderTest_BuildAndCheckOneFile"), + 0); + options.enable_blob_files = true; + options.env = mock_env_.get(); + + ImmutableOptions immutable_options(options); + MutableCFOptions mutable_cf_options(options); + + constexpr int job_id = 1; + constexpr uint32_t column_family_id = 123; + constexpr char column_family_name[] = "foobar"; + constexpr Env::IOPriority io_priority = Env::IO_HIGH; + constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM; + + std::vector blob_file_paths; + std::vector blob_file_additions; + + BlobFileBuilder builder( + TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options, + &file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id, + column_family_id, column_family_name, io_priority, write_hint, + nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/, + BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions); + + std::vector> expected_key_value_pairs( + number_of_blobs); + std::vector blob_indexes(number_of_blobs); + + for (size_t i = 0; i < number_of_blobs; ++i) { + auto& expected_key_value = expected_key_value_pairs[i]; + + auto& key = expected_key_value.first; + key = std::to_string(i); + assert(key.size() == key_size); + + auto& value = expected_key_value.second; + value = std::to_string(i + value_offset); + assert(value.size() == value_size); + + auto& blob_index = blob_indexes[i]; + + ASSERT_OK(builder.Add(key, value, &blob_index)); + ASSERT_FALSE(blob_index.empty()); + } + + ASSERT_OK(builder.Finish()); + + // Check the metadata generated + constexpr uint64_t blob_file_number = 2; + + ASSERT_EQ(blob_file_paths.size(), 1); + + const std::string& blob_file_path = blob_file_paths[0]; + + ASSERT_EQ( + blob_file_path, + BlobFileName(immutable_options.cf_paths.front().path, blob_file_number)); + + ASSERT_EQ(blob_file_additions.size(), 1); + + const auto& blob_file_addition = blob_file_additions[0]; + + ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number); + ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), number_of_blobs); + ASSERT_EQ( + blob_file_addition.GetTotalBlobBytes(), + number_of_blobs * (BlobLogRecord::kHeaderSize + key_size + value_size)); + + // Verify the contents of the new blob file as well as the blob references + VerifyBlobFile(blob_file_number, blob_file_path, column_family_id, + kNoCompression, expected_key_value_pairs, blob_indexes); +} + +TEST_F(BlobFileBuilderTest, BuildAndCheckMultipleFiles) { + // Build multiple blob files: file size limit is set to the size of a single + // value, so each blob ends up in a file of its own + constexpr size_t number_of_blobs = 10; + constexpr size_t key_size = 1; + constexpr size_t value_size = 10; + constexpr size_t value_offset = 1234567890; + + Options options; + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileBuilderTest_BuildAndCheckMultipleFiles"), + 0); + options.enable_blob_files = true; + options.blob_file_size = value_size; + options.env = mock_env_.get(); + + ImmutableOptions immutable_options(options); + MutableCFOptions mutable_cf_options(options); + + constexpr int job_id = 1; + constexpr uint32_t column_family_id = 123; + constexpr char column_family_name[] = "foobar"; + constexpr Env::IOPriority io_priority = Env::IO_HIGH; + constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM; + + std::vector blob_file_paths; + std::vector blob_file_additions; + + BlobFileBuilder builder( + TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options, + &file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id, + column_family_id, column_family_name, io_priority, write_hint, + nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/, + BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions); + + std::vector> expected_key_value_pairs( + number_of_blobs); + std::vector blob_indexes(number_of_blobs); + + for (size_t i = 0; i < number_of_blobs; ++i) { + auto& expected_key_value = expected_key_value_pairs[i]; + + auto& key = expected_key_value.first; + key = std::to_string(i); + assert(key.size() == key_size); + + auto& value = expected_key_value.second; + value = std::to_string(i + value_offset); + assert(value.size() == value_size); + + auto& blob_index = blob_indexes[i]; + + ASSERT_OK(builder.Add(key, value, &blob_index)); + ASSERT_FALSE(blob_index.empty()); + } + + ASSERT_OK(builder.Finish()); + + // Check the metadata generated + ASSERT_EQ(blob_file_paths.size(), number_of_blobs); + ASSERT_EQ(blob_file_additions.size(), number_of_blobs); + + for (size_t i = 0; i < number_of_blobs; ++i) { + const uint64_t blob_file_number = i + 2; + + ASSERT_EQ(blob_file_paths[i], + BlobFileName(immutable_options.cf_paths.front().path, + blob_file_number)); + + const auto& blob_file_addition = blob_file_additions[i]; + + ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number); + ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), 1); + ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(), + BlobLogRecord::kHeaderSize + key_size + value_size); + } + + // Verify the contents of the new blob files as well as the blob references + for (size_t i = 0; i < number_of_blobs; ++i) { + std::vector> expected_key_value_pair{ + expected_key_value_pairs[i]}; + std::vector blob_index{blob_indexes[i]}; + + VerifyBlobFile(i + 2, blob_file_paths[i], column_family_id, kNoCompression, + expected_key_value_pair, blob_index); + } +} + +TEST_F(BlobFileBuilderTest, InlinedValues) { + // All values are below the min_blob_size threshold; no blob files get written + constexpr size_t number_of_blobs = 10; + constexpr size_t key_size = 1; + constexpr size_t value_size = 10; + constexpr size_t value_offset = 1234567890; + + Options options; + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileBuilderTest_InlinedValues"), + 0); + options.enable_blob_files = true; + options.min_blob_size = 1024; + options.env = mock_env_.get(); + + ImmutableOptions immutable_options(options); + MutableCFOptions mutable_cf_options(options); + + constexpr int job_id = 1; + constexpr uint32_t column_family_id = 123; + constexpr char column_family_name[] = "foobar"; + constexpr Env::IOPriority io_priority = Env::IO_HIGH; + constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM; + + std::vector blob_file_paths; + std::vector blob_file_additions; + + BlobFileBuilder builder( + TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options, + &file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id, + column_family_id, column_family_name, io_priority, write_hint, + nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/, + BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions); + + for (size_t i = 0; i < number_of_blobs; ++i) { + const std::string key = std::to_string(i); + assert(key.size() == key_size); + + const std::string value = std::to_string(i + value_offset); + assert(value.size() == value_size); + + std::string blob_index; + ASSERT_OK(builder.Add(key, value, &blob_index)); + ASSERT_TRUE(blob_index.empty()); + } + + ASSERT_OK(builder.Finish()); + + // Check the metadata generated + ASSERT_TRUE(blob_file_paths.empty()); + ASSERT_TRUE(blob_file_additions.empty()); +} + +TEST_F(BlobFileBuilderTest, Compression) { + // Build a blob file with a compressed blob + if (!Snappy_Supported()) { + return; + } + + constexpr size_t key_size = 1; + constexpr size_t value_size = 100; + + Options options; + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), "BlobFileBuilderTest_Compression"), + 0); + options.enable_blob_files = true; + options.blob_compression_type = kSnappyCompression; + options.env = mock_env_.get(); + + ImmutableOptions immutable_options(options); + MutableCFOptions mutable_cf_options(options); + + constexpr int job_id = 1; + constexpr uint32_t column_family_id = 123; + constexpr char column_family_name[] = "foobar"; + constexpr Env::IOPriority io_priority = Env::IO_HIGH; + constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM; + + std::vector blob_file_paths; + std::vector blob_file_additions; + + BlobFileBuilder builder( + TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options, + &file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id, + column_family_id, column_family_name, io_priority, write_hint, + nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/, + BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions); + + const std::string key("1"); + const std::string uncompressed_value(value_size, 'x'); + + std::string blob_index; + + ASSERT_OK(builder.Add(key, uncompressed_value, &blob_index)); + ASSERT_FALSE(blob_index.empty()); + + ASSERT_OK(builder.Finish()); + + // Check the metadata generated + constexpr uint64_t blob_file_number = 2; + + ASSERT_EQ(blob_file_paths.size(), 1); + + const std::string& blob_file_path = blob_file_paths[0]; + + ASSERT_EQ( + blob_file_path, + BlobFileName(immutable_options.cf_paths.front().path, blob_file_number)); + + ASSERT_EQ(blob_file_additions.size(), 1); + + const auto& blob_file_addition = blob_file_additions[0]; + + ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number); + ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), 1); + + CompressionOptions opts; + CompressionContext context(kSnappyCompression); + constexpr uint64_t sample_for_compression = 0; + + CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(), + kSnappyCompression, sample_for_compression); + + std::string compressed_value; + ASSERT_TRUE(Snappy_Compress(info, uncompressed_value.data(), + uncompressed_value.size(), &compressed_value)); + + ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(), + BlobLogRecord::kHeaderSize + key_size + compressed_value.size()); + + // Verify the contents of the new blob file as well as the blob reference + std::vector> expected_key_value_pairs{ + {key, compressed_value}}; + std::vector blob_indexes{blob_index}; + + VerifyBlobFile(blob_file_number, blob_file_path, column_family_id, + kSnappyCompression, expected_key_value_pairs, blob_indexes); +} + +TEST_F(BlobFileBuilderTest, CompressionError) { + // Simulate an error during compression + if (!Snappy_Supported()) { + return; + } + + Options options; + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileBuilderTest_CompressionError"), + 0); + options.enable_blob_files = true; + options.blob_compression_type = kSnappyCompression; + options.env = mock_env_.get(); + ImmutableOptions immutable_options(options); + MutableCFOptions mutable_cf_options(options); + + constexpr int job_id = 1; + constexpr uint32_t column_family_id = 123; + constexpr char column_family_name[] = "foobar"; + constexpr Env::IOPriority io_priority = Env::IO_HIGH; + constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM; + + std::vector blob_file_paths; + std::vector blob_file_additions; + + BlobFileBuilder builder( + TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options, + &file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id, + column_family_id, column_family_name, io_priority, write_hint, + nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/, + BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions); + + SyncPoint::GetInstance()->SetCallBack("CompressData:TamperWithReturnValue", + [](void* arg) { + bool* ret = static_cast(arg); + *ret = false; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + constexpr char key[] = "1"; + constexpr char value[] = "deadbeef"; + + std::string blob_index; + + ASSERT_TRUE(builder.Add(key, value, &blob_index).IsCorruption()); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + constexpr uint64_t blob_file_number = 2; + + ASSERT_EQ(blob_file_paths.size(), 1); + ASSERT_EQ( + blob_file_paths[0], + BlobFileName(immutable_options.cf_paths.front().path, blob_file_number)); + + ASSERT_TRUE(blob_file_additions.empty()); +} + +TEST_F(BlobFileBuilderTest, Checksum) { + // Build a blob file with checksum + + class DummyFileChecksumGenerator : public FileChecksumGenerator { + public: + void Update(const char* /* data */, size_t /* n */) override {} + + void Finalize() override {} + + std::string GetChecksum() const override { return std::string("dummy"); } + + const char* Name() const override { return "DummyFileChecksum"; } + }; + + class DummyFileChecksumGenFactory : public FileChecksumGenFactory { + public: + std::unique_ptr CreateFileChecksumGenerator( + const FileChecksumGenContext& /* context */) override { + return std::unique_ptr( + new DummyFileChecksumGenerator); + } + + const char* Name() const override { return "DummyFileChecksumGenFactory"; } + }; + + Options options; + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), "BlobFileBuilderTest_Checksum"), + 0); + options.enable_blob_files = true; + options.file_checksum_gen_factory = + std::make_shared(); + options.env = mock_env_.get(); + + ImmutableOptions immutable_options(options); + MutableCFOptions mutable_cf_options(options); + + constexpr int job_id = 1; + constexpr uint32_t column_family_id = 123; + constexpr char column_family_name[] = "foobar"; + constexpr Env::IOPriority io_priority = Env::IO_HIGH; + constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM; + + std::vector blob_file_paths; + std::vector blob_file_additions; + + BlobFileBuilder builder( + TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options, + &file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id, + column_family_id, column_family_name, io_priority, write_hint, + nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/, + BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions); + + const std::string key("1"); + const std::string value("deadbeef"); + + std::string blob_index; + + ASSERT_OK(builder.Add(key, value, &blob_index)); + ASSERT_FALSE(blob_index.empty()); + + ASSERT_OK(builder.Finish()); + + // Check the metadata generated + constexpr uint64_t blob_file_number = 2; + + ASSERT_EQ(blob_file_paths.size(), 1); + + const std::string& blob_file_path = blob_file_paths[0]; + + ASSERT_EQ( + blob_file_path, + BlobFileName(immutable_options.cf_paths.front().path, blob_file_number)); + + ASSERT_EQ(blob_file_additions.size(), 1); + + const auto& blob_file_addition = blob_file_additions[0]; + + ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number); + ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), 1); + ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(), + BlobLogRecord::kHeaderSize + key.size() + value.size()); + ASSERT_EQ(blob_file_addition.GetChecksumMethod(), "DummyFileChecksum"); + ASSERT_EQ(blob_file_addition.GetChecksumValue(), "dummy"); + + // Verify the contents of the new blob file as well as the blob reference + std::vector> expected_key_value_pairs{ + {key, value}}; + std::vector blob_indexes{blob_index}; + + VerifyBlobFile(blob_file_number, blob_file_path, column_family_id, + kNoCompression, expected_key_value_pairs, blob_indexes); +} + +class BlobFileBuilderIOErrorTest + : public testing::Test, + public testing::WithParamInterface { + protected: + BlobFileBuilderIOErrorTest() : sync_point_(GetParam()) { + mock_env_.reset(MockEnv::Create(Env::Default())); + fs_ = mock_env_->GetFileSystem().get(); + } + + std::unique_ptr mock_env_; + FileSystem* fs_; + FileOptions file_options_; + std::string sync_point_; +}; + +INSTANTIATE_TEST_CASE_P( + BlobFileBuilderTest, BlobFileBuilderIOErrorTest, + ::testing::ValuesIn(std::vector{ + "BlobFileBuilder::OpenBlobFileIfNeeded:NewWritableFile", + "BlobFileBuilder::OpenBlobFileIfNeeded:WriteHeader", + "BlobFileBuilder::WriteBlobToFile:AddRecord", + "BlobFileBuilder::WriteBlobToFile:AppendFooter"})); + +TEST_P(BlobFileBuilderIOErrorTest, IOError) { + // Simulate an I/O error during the specified step of Add() + // Note: blob_file_size will be set to value_size in order for the first blob + // to trigger close + constexpr size_t value_size = 8; + + Options options; + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileBuilderIOErrorTest_IOError"), + 0); + options.enable_blob_files = true; + options.blob_file_size = value_size; + options.env = mock_env_.get(); + + ImmutableOptions immutable_options(options); + MutableCFOptions mutable_cf_options(options); + + constexpr int job_id = 1; + constexpr uint32_t column_family_id = 123; + constexpr char column_family_name[] = "foobar"; + constexpr Env::IOPriority io_priority = Env::IO_HIGH; + constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM; + + std::vector blob_file_paths; + std::vector blob_file_additions; + + BlobFileBuilder builder( + TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options, + &file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id, + column_family_id, column_family_name, io_priority, write_hint, + nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/, + BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions); + + SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* arg) { + Status* const s = static_cast(arg); + assert(s); + + (*s) = Status::IOError(sync_point_); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + constexpr char key[] = "1"; + constexpr char value[] = "deadbeef"; + + std::string blob_index; + + ASSERT_TRUE(builder.Add(key, value, &blob_index).IsIOError()); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + if (sync_point_ == "BlobFileBuilder::OpenBlobFileIfNeeded:NewWritableFile") { + ASSERT_TRUE(blob_file_paths.empty()); + } else { + constexpr uint64_t blob_file_number = 2; + + ASSERT_EQ(blob_file_paths.size(), 1); + ASSERT_EQ(blob_file_paths[0], + BlobFileName(immutable_options.cf_paths.front().path, + blob_file_number)); + } + + ASSERT_TRUE(blob_file_additions.empty()); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/blob/blob_file_cache.cc b/librocksdb-sys/rocksdb/db/blob/blob_file_cache.cc new file mode 100644 index 0000000..5f340aa --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_file_cache.cc @@ -0,0 +1,101 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_file_cache.h" + +#include +#include + +#include "db/blob/blob_file_reader.h" +#include "options/cf_options.h" +#include "rocksdb/cache.h" +#include "rocksdb/slice.h" +#include "test_util/sync_point.h" +#include "trace_replay/io_tracer.h" +#include "util/hash.h" + +namespace ROCKSDB_NAMESPACE { + +BlobFileCache::BlobFileCache(Cache* cache, + const ImmutableOptions* immutable_options, + const FileOptions* file_options, + uint32_t column_family_id, + HistogramImpl* blob_file_read_hist, + const std::shared_ptr& io_tracer) + : cache_(cache), + mutex_(kNumberOfMutexStripes), + immutable_options_(immutable_options), + file_options_(file_options), + column_family_id_(column_family_id), + blob_file_read_hist_(blob_file_read_hist), + io_tracer_(io_tracer) { + assert(cache_); + assert(immutable_options_); + assert(file_options_); +} + +Status BlobFileCache::GetBlobFileReader( + const ReadOptions& read_options, uint64_t blob_file_number, + CacheHandleGuard* blob_file_reader) { + assert(blob_file_reader); + assert(blob_file_reader->IsEmpty()); + + const Slice key = GetSliceForKey(&blob_file_number); + + assert(cache_); + + TypedHandle* handle = cache_.Lookup(key); + if (handle) { + *blob_file_reader = cache_.Guard(handle); + return Status::OK(); + } + + TEST_SYNC_POINT("BlobFileCache::GetBlobFileReader:DoubleCheck"); + + // Check again while holding mutex + MutexLock lock(&mutex_.Get(key)); + + handle = cache_.Lookup(key); + if (handle) { + *blob_file_reader = cache_.Guard(handle); + return Status::OK(); + } + + assert(immutable_options_); + Statistics* const statistics = immutable_options_->stats; + + RecordTick(statistics, NO_FILE_OPENS); + + std::unique_ptr reader; + + { + assert(file_options_); + const Status s = BlobFileReader::Create( + *immutable_options_, read_options, *file_options_, column_family_id_, + blob_file_read_hist_, blob_file_number, io_tracer_, &reader); + if (!s.ok()) { + RecordTick(statistics, NO_FILE_ERRORS); + return s; + } + } + + { + constexpr size_t charge = 1; + + const Status s = cache_.Insert(key, reader.get(), charge, &handle); + if (!s.ok()) { + RecordTick(statistics, NO_FILE_ERRORS); + return s; + } + } + + reader.release(); + + *blob_file_reader = cache_.Guard(handle); + + return Status::OK(); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_file_cache.h b/librocksdb-sys/rocksdb/db/blob/blob_file_cache.h new file mode 100644 index 0000000..740e67a --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_file_cache.h @@ -0,0 +1,56 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "cache/typed_cache.h" +#include "db/blob/blob_file_reader.h" +#include "rocksdb/rocksdb_namespace.h" +#include "util/mutexlock.h" + +namespace ROCKSDB_NAMESPACE { + +class Cache; +struct ImmutableOptions; +struct FileOptions; +class HistogramImpl; +class Status; +class Slice; +class IOTracer; + +class BlobFileCache { + public: + BlobFileCache(Cache* cache, const ImmutableOptions* immutable_options, + const FileOptions* file_options, uint32_t column_family_id, + HistogramImpl* blob_file_read_hist, + const std::shared_ptr& io_tracer); + + BlobFileCache(const BlobFileCache&) = delete; + BlobFileCache& operator=(const BlobFileCache&) = delete; + + Status GetBlobFileReader(const ReadOptions& read_options, + uint64_t blob_file_number, + CacheHandleGuard* blob_file_reader); + + private: + using CacheInterface = + BasicTypedCacheInterface; + using TypedHandle = CacheInterface::TypedHandle; + CacheInterface cache_; + // Note: mutex_ below is used to guard against multiple threads racing to open + // the same file. + Striped> mutex_; + const ImmutableOptions* immutable_options_; + const FileOptions* file_options_; + uint32_t column_family_id_; + HistogramImpl* blob_file_read_hist_; + std::shared_ptr io_tracer_; + + static constexpr size_t kNumberOfMutexStripes = 1 << 7; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_file_cache_test.cc b/librocksdb-sys/rocksdb/db/blob/blob_file_cache_test.cc new file mode 100644 index 0000000..8c3c56d --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_file_cache_test.cc @@ -0,0 +1,278 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_file_cache.h" + +#include +#include + +#include "db/blob/blob_log_format.h" +#include "db/blob/blob_log_writer.h" +#include "env/mock_env.h" +#include "file/filename.h" +#include "file/read_write_util.h" +#include "file/writable_file_writer.h" +#include "options/cf_options.h" +#include "rocksdb/cache.h" +#include "rocksdb/env.h" +#include "rocksdb/file_system.h" +#include "rocksdb/options.h" +#include "rocksdb/statistics.h" +#include "test_util/sync_point.h" +#include "test_util/testharness.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { + +// Creates a test blob file with a single blob in it. +void WriteBlobFile(uint32_t column_family_id, + const ImmutableOptions& immutable_options, + uint64_t blob_file_number) { + assert(!immutable_options.cf_paths.empty()); + + const std::string blob_file_path = + BlobFileName(immutable_options.cf_paths.front().path, blob_file_number); + + std::unique_ptr file; + ASSERT_OK(NewWritableFile(immutable_options.fs.get(), blob_file_path, &file, + FileOptions())); + + std::unique_ptr file_writer(new WritableFileWriter( + std::move(file), blob_file_path, FileOptions(), immutable_options.clock)); + + constexpr Statistics* statistics = nullptr; + constexpr bool use_fsync = false; + constexpr bool do_flush = false; + + BlobLogWriter blob_log_writer(std::move(file_writer), immutable_options.clock, + statistics, blob_file_number, use_fsync, + do_flush); + + constexpr bool has_ttl = false; + constexpr ExpirationRange expiration_range; + + BlobLogHeader header(column_family_id, kNoCompression, has_ttl, + expiration_range); + + ASSERT_OK(blob_log_writer.WriteHeader(header)); + + constexpr char key[] = "key"; + constexpr char blob[] = "blob"; + + std::string compressed_blob; + + uint64_t key_offset = 0; + uint64_t blob_offset = 0; + + ASSERT_OK(blob_log_writer.AddRecord(key, blob, &key_offset, &blob_offset)); + + BlobLogFooter footer; + footer.blob_count = 1; + footer.expiration_range = expiration_range; + + std::string checksum_method; + std::string checksum_value; + + ASSERT_OK( + blob_log_writer.AppendFooter(footer, &checksum_method, &checksum_value)); +} + +} // anonymous namespace + +class BlobFileCacheTest : public testing::Test { + protected: + BlobFileCacheTest() { mock_env_.reset(MockEnv::Create(Env::Default())); } + + std::unique_ptr mock_env_; +}; + +TEST_F(BlobFileCacheTest, GetBlobFileReader) { + Options options; + options.env = mock_env_.get(); + options.statistics = CreateDBStatistics(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileCacheTest_GetBlobFileReader"), + 0); + options.enable_blob_files = true; + + constexpr uint32_t column_family_id = 1; + ImmutableOptions immutable_options(options); + constexpr uint64_t blob_file_number = 123; + + WriteBlobFile(column_family_id, immutable_options, blob_file_number); + + constexpr size_t capacity = 10; + std::shared_ptr backing_cache = NewLRUCache(capacity); + + FileOptions file_options; + constexpr HistogramImpl* blob_file_read_hist = nullptr; + + BlobFileCache blob_file_cache(backing_cache.get(), &immutable_options, + &file_options, column_family_id, + blob_file_read_hist, nullptr /*IOTracer*/); + + // First try: reader should be opened and put in cache + CacheHandleGuard first; + + const ReadOptions read_options; + ASSERT_OK(blob_file_cache.GetBlobFileReader(read_options, blob_file_number, + &first)); + ASSERT_NE(first.GetValue(), nullptr); + ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_OPENS), 1); + ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_ERRORS), 0); + + // Second try: reader should be served from cache + CacheHandleGuard second; + + ASSERT_OK(blob_file_cache.GetBlobFileReader(read_options, blob_file_number, + &second)); + ASSERT_NE(second.GetValue(), nullptr); + ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_OPENS), 1); + ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_ERRORS), 0); + + ASSERT_EQ(first.GetValue(), second.GetValue()); +} + +TEST_F(BlobFileCacheTest, GetBlobFileReader_Race) { + Options options; + options.env = mock_env_.get(); + options.statistics = CreateDBStatistics(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileCacheTest_GetBlobFileReader_Race"), + 0); + options.enable_blob_files = true; + + constexpr uint32_t column_family_id = 1; + ImmutableOptions immutable_options(options); + constexpr uint64_t blob_file_number = 123; + + WriteBlobFile(column_family_id, immutable_options, blob_file_number); + + constexpr size_t capacity = 10; + std::shared_ptr backing_cache = NewLRUCache(capacity); + + FileOptions file_options; + constexpr HistogramImpl* blob_file_read_hist = nullptr; + + BlobFileCache blob_file_cache(backing_cache.get(), &immutable_options, + &file_options, column_family_id, + blob_file_read_hist, nullptr /*IOTracer*/); + + CacheHandleGuard first; + CacheHandleGuard second; + + const ReadOptions read_options; + SyncPoint::GetInstance()->SetCallBack( + "BlobFileCache::GetBlobFileReader:DoubleCheck", [&](void* /* arg */) { + // Disabling sync points to prevent infinite recursion + SyncPoint::GetInstance()->DisableProcessing(); + ASSERT_OK(blob_file_cache.GetBlobFileReader(read_options, + blob_file_number, &second)); + ASSERT_NE(second.GetValue(), nullptr); + ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_OPENS), 1); + ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_ERRORS), 0); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(blob_file_cache.GetBlobFileReader(read_options, blob_file_number, + &first)); + ASSERT_NE(first.GetValue(), nullptr); + ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_OPENS), 1); + ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_ERRORS), 0); + + ASSERT_EQ(first.GetValue(), second.GetValue()); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(BlobFileCacheTest, GetBlobFileReader_IOError) { + Options options; + options.env = mock_env_.get(); + options.statistics = CreateDBStatistics(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileCacheTest_GetBlobFileReader_IOError"), + 0); + options.enable_blob_files = true; + + constexpr size_t capacity = 10; + std::shared_ptr backing_cache = NewLRUCache(capacity); + + ImmutableOptions immutable_options(options); + FileOptions file_options; + constexpr uint32_t column_family_id = 1; + constexpr HistogramImpl* blob_file_read_hist = nullptr; + + BlobFileCache blob_file_cache(backing_cache.get(), &immutable_options, + &file_options, column_family_id, + blob_file_read_hist, nullptr /*IOTracer*/); + + // Note: there is no blob file with the below number + constexpr uint64_t blob_file_number = 123; + + CacheHandleGuard reader; + + const ReadOptions read_options; + ASSERT_TRUE( + blob_file_cache.GetBlobFileReader(read_options, blob_file_number, &reader) + .IsIOError()); + ASSERT_EQ(reader.GetValue(), nullptr); + ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_OPENS), 1); + ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_ERRORS), 1); +} + +TEST_F(BlobFileCacheTest, GetBlobFileReader_CacheFull) { + Options options; + options.env = mock_env_.get(); + options.statistics = CreateDBStatistics(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileCacheTest_GetBlobFileReader_CacheFull"), + 0); + options.enable_blob_files = true; + + constexpr uint32_t column_family_id = 1; + ImmutableOptions immutable_options(options); + constexpr uint64_t blob_file_number = 123; + + WriteBlobFile(column_family_id, immutable_options, blob_file_number); + + constexpr size_t capacity = 0; + constexpr int num_shard_bits = -1; // determined automatically + constexpr bool strict_capacity_limit = true; + std::shared_ptr backing_cache = + NewLRUCache(capacity, num_shard_bits, strict_capacity_limit); + + FileOptions file_options; + constexpr HistogramImpl* blob_file_read_hist = nullptr; + + BlobFileCache blob_file_cache(backing_cache.get(), &immutable_options, + &file_options, column_family_id, + blob_file_read_hist, nullptr /*IOTracer*/); + + // Insert into cache should fail since it has zero capacity and + // strict_capacity_limit is set + CacheHandleGuard reader; + + const ReadOptions read_options; + ASSERT_TRUE( + blob_file_cache.GetBlobFileReader(read_options, blob_file_number, &reader) + .IsMemoryLimit()); + ASSERT_EQ(reader.GetValue(), nullptr); + ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_OPENS), 1); + ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_ERRORS), 1); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/blob/blob_file_completion_callback.h b/librocksdb-sys/rocksdb/db/blob/blob_file_completion_callback.h new file mode 100644 index 0000000..9159677 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_file_completion_callback.h @@ -0,0 +1,84 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once + +#include "db/error_handler.h" +#include "db/event_helpers.h" +#include "file/sst_file_manager_impl.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +class BlobFileCompletionCallback { + public: + BlobFileCompletionCallback( + SstFileManager* sst_file_manager, InstrumentedMutex* mutex, + ErrorHandler* error_handler, EventLogger* event_logger, + const std::vector>& listeners, + const std::string& dbname) + : event_logger_(event_logger), listeners_(listeners), dbname_(dbname) { + sst_file_manager_ = sst_file_manager; + mutex_ = mutex; + error_handler_ = error_handler; + } + + void OnBlobFileCreationStarted(const std::string& file_name, + const std::string& column_family_name, + int job_id, + BlobFileCreationReason creation_reason) { + // Notify the listeners. + EventHelpers::NotifyBlobFileCreationStarted(listeners_, dbname_, + column_family_name, file_name, + job_id, creation_reason); + } + + Status OnBlobFileCompleted(const std::string& file_name, + const std::string& column_family_name, int job_id, + uint64_t file_number, + BlobFileCreationReason creation_reason, + const Status& report_status, + const std::string& checksum_value, + const std::string& checksum_method, + uint64_t blob_count, uint64_t blob_bytes) { + Status s; + + auto sfm = static_cast(sst_file_manager_); + if (sfm) { + // Report new blob files to SstFileManagerImpl + s = sfm->OnAddFile(file_name); + if (sfm->IsMaxAllowedSpaceReached()) { + s = Status::SpaceLimit("Max allowed space was reached"); + TEST_SYNC_POINT( + "BlobFileCompletionCallback::CallBack::MaxAllowedSpaceReached"); + InstrumentedMutexLock l(mutex_); + error_handler_->SetBGError(s, BackgroundErrorReason::kFlush); + } + } + + // Notify the listeners. + EventHelpers::LogAndNotifyBlobFileCreationFinished( + event_logger_, listeners_, dbname_, column_family_name, file_name, + job_id, file_number, creation_reason, + (!report_status.ok() ? report_status : s), + (checksum_value.empty() ? kUnknownFileChecksum : checksum_value), + (checksum_method.empty() ? kUnknownFileChecksumFuncName + : checksum_method), + blob_count, blob_bytes); + return s; + } + + private: + SstFileManager* sst_file_manager_; + InstrumentedMutex* mutex_; + ErrorHandler* error_handler_; + EventLogger* event_logger_; + std::vector> listeners_; + std::string dbname_; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_file_garbage.cc b/librocksdb-sys/rocksdb/db/blob/blob_file_garbage.cc new file mode 100644 index 0000000..52c336f --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_file_garbage.cc @@ -0,0 +1,134 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_file_garbage.h" + +#include +#include + +#include "logging/event_logger.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "test_util/sync_point.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { + +// Tags for custom fields. Note that these get persisted in the manifest, +// so existing tags should not be modified. +enum BlobFileGarbage::CustomFieldTags : uint32_t { + kEndMarker, + + // Add forward compatible fields here + + ///////////////////////////////////////////////////////////////////// + + kForwardIncompatibleMask = 1 << 6, + + // Add forward incompatible fields here +}; + +void BlobFileGarbage::EncodeTo(std::string* output) const { + PutVarint64(output, blob_file_number_); + PutVarint64(output, garbage_blob_count_); + PutVarint64(output, garbage_blob_bytes_); + + // Encode any custom fields here. The format to use is a Varint32 tag (see + // CustomFieldTags above) followed by a length prefixed slice. Unknown custom + // fields will be ignored during decoding unless they're in the forward + // incompatible range. + + TEST_SYNC_POINT_CALLBACK("BlobFileGarbage::EncodeTo::CustomFields", output); + + PutVarint32(output, kEndMarker); +} + +Status BlobFileGarbage::DecodeFrom(Slice* input) { + constexpr char class_name[] = "BlobFileGarbage"; + + if (!GetVarint64(input, &blob_file_number_)) { + return Status::Corruption(class_name, "Error decoding blob file number"); + } + + if (!GetVarint64(input, &garbage_blob_count_)) { + return Status::Corruption(class_name, "Error decoding garbage blob count"); + } + + if (!GetVarint64(input, &garbage_blob_bytes_)) { + return Status::Corruption(class_name, "Error decoding garbage blob bytes"); + } + + while (true) { + uint32_t custom_field_tag = 0; + if (!GetVarint32(input, &custom_field_tag)) { + return Status::Corruption(class_name, "Error decoding custom field tag"); + } + + if (custom_field_tag == kEndMarker) { + break; + } + + if (custom_field_tag & kForwardIncompatibleMask) { + return Status::Corruption( + class_name, "Forward incompatible custom field encountered"); + } + + Slice custom_field_value; + if (!GetLengthPrefixedSlice(input, &custom_field_value)) { + return Status::Corruption(class_name, + "Error decoding custom field value"); + } + } + + return Status::OK(); +} + +std::string BlobFileGarbage::DebugString() const { + std::ostringstream oss; + + oss << *this; + + return oss.str(); +} + +std::string BlobFileGarbage::DebugJSON() const { + JSONWriter jw; + + jw << *this; + + jw.EndObject(); + + return jw.Get(); +} + +bool operator==(const BlobFileGarbage& lhs, const BlobFileGarbage& rhs) { + return lhs.GetBlobFileNumber() == rhs.GetBlobFileNumber() && + lhs.GetGarbageBlobCount() == rhs.GetGarbageBlobCount() && + lhs.GetGarbageBlobBytes() == rhs.GetGarbageBlobBytes(); +} + +bool operator!=(const BlobFileGarbage& lhs, const BlobFileGarbage& rhs) { + return !(lhs == rhs); +} + +std::ostream& operator<<(std::ostream& os, + const BlobFileGarbage& blob_file_garbage) { + os << "blob_file_number: " << blob_file_garbage.GetBlobFileNumber() + << " garbage_blob_count: " << blob_file_garbage.GetGarbageBlobCount() + << " garbage_blob_bytes: " << blob_file_garbage.GetGarbageBlobBytes(); + + return os; +} + +JSONWriter& operator<<(JSONWriter& jw, + const BlobFileGarbage& blob_file_garbage) { + jw << "BlobFileNumber" << blob_file_garbage.GetBlobFileNumber() + << "GarbageBlobCount" << blob_file_garbage.GetGarbageBlobCount() + << "GarbageBlobBytes" << blob_file_garbage.GetGarbageBlobBytes(); + + return jw; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_file_garbage.h b/librocksdb-sys/rocksdb/db/blob/blob_file_garbage.h new file mode 100644 index 0000000..6dc14dd --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_file_garbage.h @@ -0,0 +1,57 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include + +#include "db/blob/blob_constants.h" +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +class JSONWriter; +class Slice; +class Status; + +class BlobFileGarbage { + public: + BlobFileGarbage() = default; + + BlobFileGarbage(uint64_t blob_file_number, uint64_t garbage_blob_count, + uint64_t garbage_blob_bytes) + : blob_file_number_(blob_file_number), + garbage_blob_count_(garbage_blob_count), + garbage_blob_bytes_(garbage_blob_bytes) {} + + uint64_t GetBlobFileNumber() const { return blob_file_number_; } + uint64_t GetGarbageBlobCount() const { return garbage_blob_count_; } + uint64_t GetGarbageBlobBytes() const { return garbage_blob_bytes_; } + + void EncodeTo(std::string* output) const; + Status DecodeFrom(Slice* input); + + std::string DebugString() const; + std::string DebugJSON() const; + + private: + enum CustomFieldTags : uint32_t; + + uint64_t blob_file_number_ = kInvalidBlobFileNumber; + uint64_t garbage_blob_count_ = 0; + uint64_t garbage_blob_bytes_ = 0; +}; + +bool operator==(const BlobFileGarbage& lhs, const BlobFileGarbage& rhs); +bool operator!=(const BlobFileGarbage& lhs, const BlobFileGarbage& rhs); + +std::ostream& operator<<(std::ostream& os, + const BlobFileGarbage& blob_file_garbage); +JSONWriter& operator<<(JSONWriter& jw, + const BlobFileGarbage& blob_file_garbage); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_file_garbage_test.cc b/librocksdb-sys/rocksdb/db/blob/blob_file_garbage_test.cc new file mode 100644 index 0000000..292a8b3 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_file_garbage_test.cc @@ -0,0 +1,174 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_file_garbage.h" + +#include +#include +#include + +#include "test_util/sync_point.h" +#include "test_util/testharness.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { + +class BlobFileGarbageTest : public testing::Test { + public: + static void TestEncodeDecode(const BlobFileGarbage& blob_file_garbage) { + std::string encoded; + blob_file_garbage.EncodeTo(&encoded); + + BlobFileGarbage decoded; + Slice input(encoded); + ASSERT_OK(decoded.DecodeFrom(&input)); + + ASSERT_EQ(blob_file_garbage, decoded); + } +}; + +TEST_F(BlobFileGarbageTest, Empty) { + BlobFileGarbage blob_file_garbage; + + ASSERT_EQ(blob_file_garbage.GetBlobFileNumber(), kInvalidBlobFileNumber); + ASSERT_EQ(blob_file_garbage.GetGarbageBlobCount(), 0); + ASSERT_EQ(blob_file_garbage.GetGarbageBlobBytes(), 0); + + TestEncodeDecode(blob_file_garbage); +} + +TEST_F(BlobFileGarbageTest, NonEmpty) { + constexpr uint64_t blob_file_number = 123; + constexpr uint64_t garbage_blob_count = 1; + constexpr uint64_t garbage_blob_bytes = 9876; + + BlobFileGarbage blob_file_garbage(blob_file_number, garbage_blob_count, + garbage_blob_bytes); + + ASSERT_EQ(blob_file_garbage.GetBlobFileNumber(), blob_file_number); + ASSERT_EQ(blob_file_garbage.GetGarbageBlobCount(), garbage_blob_count); + ASSERT_EQ(blob_file_garbage.GetGarbageBlobBytes(), garbage_blob_bytes); + + TestEncodeDecode(blob_file_garbage); +} + +TEST_F(BlobFileGarbageTest, DecodeErrors) { + std::string str; + Slice slice(str); + + BlobFileGarbage blob_file_garbage; + + { + const Status s = blob_file_garbage.DecodeFrom(&slice); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "blob file number")); + } + + constexpr uint64_t blob_file_number = 123; + PutVarint64(&str, blob_file_number); + slice = str; + + { + const Status s = blob_file_garbage.DecodeFrom(&slice); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "garbage blob count")); + } + + constexpr uint64_t garbage_blob_count = 4567; + PutVarint64(&str, garbage_blob_count); + slice = str; + + { + const Status s = blob_file_garbage.DecodeFrom(&slice); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "garbage blob bytes")); + } + + constexpr uint64_t garbage_blob_bytes = 12345678; + PutVarint64(&str, garbage_blob_bytes); + slice = str; + + { + const Status s = blob_file_garbage.DecodeFrom(&slice); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "custom field tag")); + } + + constexpr uint32_t custom_tag = 2; + PutVarint32(&str, custom_tag); + slice = str; + + { + const Status s = blob_file_garbage.DecodeFrom(&slice); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "custom field value")); + } +} + +TEST_F(BlobFileGarbageTest, ForwardCompatibleCustomField) { + SyncPoint::GetInstance()->SetCallBack( + "BlobFileGarbage::EncodeTo::CustomFields", [&](void* arg) { + std::string* output = static_cast(arg); + + constexpr uint32_t forward_compatible_tag = 2; + PutVarint32(output, forward_compatible_tag); + + PutLengthPrefixedSlice(output, "deadbeef"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + constexpr uint64_t blob_file_number = 678; + constexpr uint64_t garbage_blob_count = 9999; + constexpr uint64_t garbage_blob_bytes = 100000000; + + BlobFileGarbage blob_file_garbage(blob_file_number, garbage_blob_count, + garbage_blob_bytes); + + TestEncodeDecode(blob_file_garbage); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(BlobFileGarbageTest, ForwardIncompatibleCustomField) { + SyncPoint::GetInstance()->SetCallBack( + "BlobFileGarbage::EncodeTo::CustomFields", [&](void* arg) { + std::string* output = static_cast(arg); + + constexpr uint32_t forward_incompatible_tag = (1 << 6) + 1; + PutVarint32(output, forward_incompatible_tag); + + PutLengthPrefixedSlice(output, "foobar"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + constexpr uint64_t blob_file_number = 456; + constexpr uint64_t garbage_blob_count = 100; + constexpr uint64_t garbage_blob_bytes = 2000000; + + BlobFileGarbage blob_file_garbage(blob_file_number, garbage_blob_count, + garbage_blob_bytes); + + std::string encoded; + blob_file_garbage.EncodeTo(&encoded); + + BlobFileGarbage decoded_blob_file_addition; + Slice input(encoded); + const Status s = decoded_blob_file_addition.DecodeFrom(&input); + + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "Forward incompatible")); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/blob/blob_file_meta.cc b/librocksdb-sys/rocksdb/db/blob/blob_file_meta.cc new file mode 100644 index 0000000..4913137 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_file_meta.cc @@ -0,0 +1,62 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_file_meta.h" + +#include +#include + +#include "db/blob/blob_log_format.h" +#include "rocksdb/slice.h" + +namespace ROCKSDB_NAMESPACE { +uint64_t SharedBlobFileMetaData::GetBlobFileSize() const { + return BlobLogHeader::kSize + total_blob_bytes_ + BlobLogFooter::kSize; +} + +std::string SharedBlobFileMetaData::DebugString() const { + std::ostringstream oss; + oss << (*this); + + return oss.str(); +} + +std::ostream& operator<<(std::ostream& os, + const SharedBlobFileMetaData& shared_meta) { + os << "blob_file_number: " << shared_meta.GetBlobFileNumber() + << " total_blob_count: " << shared_meta.GetTotalBlobCount() + << " total_blob_bytes: " << shared_meta.GetTotalBlobBytes() + << " checksum_method: " << shared_meta.GetChecksumMethod() + << " checksum_value: " + << Slice(shared_meta.GetChecksumValue()).ToString(/* hex */ true); + + return os; +} + +std::string BlobFileMetaData::DebugString() const { + std::ostringstream oss; + oss << (*this); + + return oss.str(); +} + +std::ostream& operator<<(std::ostream& os, const BlobFileMetaData& meta) { + const auto& shared_meta = meta.GetSharedMeta(); + assert(shared_meta); + os << (*shared_meta); + + os << " linked_ssts: {"; + for (uint64_t file_number : meta.GetLinkedSsts()) { + os << ' ' << file_number; + } + os << " }"; + + os << " garbage_blob_count: " << meta.GetGarbageBlobCount() + << " garbage_blob_bytes: " << meta.GetGarbageBlobBytes(); + + return os; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_file_meta.h b/librocksdb-sys/rocksdb/db/blob/blob_file_meta.h new file mode 100644 index 0000000..d7c8a12 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_file_meta.h @@ -0,0 +1,170 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include +#include +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +// SharedBlobFileMetaData represents the immutable part of blob files' metadata, +// like the blob file number, total number and size of blobs, or checksum +// method and value. There is supposed to be one object of this class per blob +// file (shared across all versions that include the blob file in question); +// hence, the type is neither copyable nor movable. A blob file can be marked +// obsolete when the corresponding SharedBlobFileMetaData object is destroyed. + +class SharedBlobFileMetaData { + public: + static std::shared_ptr Create( + uint64_t blob_file_number, uint64_t total_blob_count, + uint64_t total_blob_bytes, std::string checksum_method, + std::string checksum_value) { + return std::shared_ptr(new SharedBlobFileMetaData( + blob_file_number, total_blob_count, total_blob_bytes, + std::move(checksum_method), std::move(checksum_value))); + } + + template + static std::shared_ptr Create( + uint64_t blob_file_number, uint64_t total_blob_count, + uint64_t total_blob_bytes, std::string checksum_method, + std::string checksum_value, Deleter deleter) { + return std::shared_ptr( + new SharedBlobFileMetaData(blob_file_number, total_blob_count, + total_blob_bytes, std::move(checksum_method), + std::move(checksum_value)), + deleter); + } + + SharedBlobFileMetaData(const SharedBlobFileMetaData&) = delete; + SharedBlobFileMetaData& operator=(const SharedBlobFileMetaData&) = delete; + + SharedBlobFileMetaData(SharedBlobFileMetaData&&) = delete; + SharedBlobFileMetaData& operator=(SharedBlobFileMetaData&&) = delete; + + uint64_t GetBlobFileSize() const; + uint64_t GetBlobFileNumber() const { return blob_file_number_; } + uint64_t GetTotalBlobCount() const { return total_blob_count_; } + uint64_t GetTotalBlobBytes() const { return total_blob_bytes_; } + const std::string& GetChecksumMethod() const { return checksum_method_; } + const std::string& GetChecksumValue() const { return checksum_value_; } + + std::string DebugString() const; + + private: + SharedBlobFileMetaData(uint64_t blob_file_number, uint64_t total_blob_count, + uint64_t total_blob_bytes, std::string checksum_method, + std::string checksum_value) + : blob_file_number_(blob_file_number), + total_blob_count_(total_blob_count), + total_blob_bytes_(total_blob_bytes), + checksum_method_(std::move(checksum_method)), + checksum_value_(std::move(checksum_value)) { + assert(checksum_method_.empty() == checksum_value_.empty()); + } + + uint64_t blob_file_number_; + uint64_t total_blob_count_; + uint64_t total_blob_bytes_; + std::string checksum_method_; + std::string checksum_value_; +}; + +std::ostream& operator<<(std::ostream& os, + const SharedBlobFileMetaData& shared_meta); + +// BlobFileMetaData contains the part of the metadata for blob files that can +// vary across versions, like the amount of garbage in the blob file. In +// addition, BlobFileMetaData objects point to and share the ownership of the +// SharedBlobFileMetaData object for the corresponding blob file. Similarly to +// SharedBlobFileMetaData, BlobFileMetaData are not copyable or movable. They +// are meant to be jointly owned by the versions in which the blob file has the +// same (immutable *and* mutable) state. + +class BlobFileMetaData { + public: + using LinkedSsts = std::unordered_set; + + static std::shared_ptr Create( + std::shared_ptr shared_meta, + LinkedSsts linked_ssts, uint64_t garbage_blob_count, + uint64_t garbage_blob_bytes) { + return std::shared_ptr( + new BlobFileMetaData(std::move(shared_meta), std::move(linked_ssts), + garbage_blob_count, garbage_blob_bytes)); + } + + BlobFileMetaData(const BlobFileMetaData&) = delete; + BlobFileMetaData& operator=(const BlobFileMetaData&) = delete; + + BlobFileMetaData(BlobFileMetaData&&) = delete; + BlobFileMetaData& operator=(BlobFileMetaData&&) = delete; + + const std::shared_ptr& GetSharedMeta() const { + return shared_meta_; + } + + uint64_t GetBlobFileSize() const { + assert(shared_meta_); + return shared_meta_->GetBlobFileSize(); + } + + uint64_t GetBlobFileNumber() const { + assert(shared_meta_); + return shared_meta_->GetBlobFileNumber(); + } + uint64_t GetTotalBlobCount() const { + assert(shared_meta_); + return shared_meta_->GetTotalBlobCount(); + } + uint64_t GetTotalBlobBytes() const { + assert(shared_meta_); + return shared_meta_->GetTotalBlobBytes(); + } + const std::string& GetChecksumMethod() const { + assert(shared_meta_); + return shared_meta_->GetChecksumMethod(); + } + const std::string& GetChecksumValue() const { + assert(shared_meta_); + return shared_meta_->GetChecksumValue(); + } + + const LinkedSsts& GetLinkedSsts() const { return linked_ssts_; } + + uint64_t GetGarbageBlobCount() const { return garbage_blob_count_; } + uint64_t GetGarbageBlobBytes() const { return garbage_blob_bytes_; } + + std::string DebugString() const; + + private: + BlobFileMetaData(std::shared_ptr shared_meta, + LinkedSsts linked_ssts, uint64_t garbage_blob_count, + uint64_t garbage_blob_bytes) + : shared_meta_(std::move(shared_meta)), + linked_ssts_(std::move(linked_ssts)), + garbage_blob_count_(garbage_blob_count), + garbage_blob_bytes_(garbage_blob_bytes) { + assert(shared_meta_); + assert(garbage_blob_count_ <= shared_meta_->GetTotalBlobCount()); + assert(garbage_blob_bytes_ <= shared_meta_->GetTotalBlobBytes()); + } + + std::shared_ptr shared_meta_; + LinkedSsts linked_ssts_; + uint64_t garbage_blob_count_; + uint64_t garbage_blob_bytes_; +}; + +std::ostream& operator<<(std::ostream& os, const BlobFileMetaData& meta); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_file_reader.cc b/librocksdb-sys/rocksdb/db/blob/blob_file_reader.cc new file mode 100644 index 0000000..6df7f3a --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_file_reader.cc @@ -0,0 +1,624 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_file_reader.h" + +#include +#include + +#include "db/blob/blob_contents.h" +#include "db/blob/blob_log_format.h" +#include "file/file_prefetch_buffer.h" +#include "file/filename.h" +#include "monitoring/statistics_impl.h" +#include "options/cf_options.h" +#include "rocksdb/file_system.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "table/multiget_context.h" +#include "test_util/sync_point.h" +#include "util/compression.h" +#include "util/crc32c.h" +#include "util/stop_watch.h" + +namespace ROCKSDB_NAMESPACE { + +Status BlobFileReader::Create( + const ImmutableOptions& immutable_options, const ReadOptions& read_options, + const FileOptions& file_options, uint32_t column_family_id, + HistogramImpl* blob_file_read_hist, uint64_t blob_file_number, + const std::shared_ptr& io_tracer, + std::unique_ptr* blob_file_reader) { + assert(blob_file_reader); + assert(!*blob_file_reader); + + uint64_t file_size = 0; + std::unique_ptr file_reader; + + { + const Status s = + OpenFile(immutable_options, file_options, blob_file_read_hist, + blob_file_number, io_tracer, &file_size, &file_reader); + if (!s.ok()) { + return s; + } + } + + assert(file_reader); + + Statistics* const statistics = immutable_options.stats; + + CompressionType compression_type = kNoCompression; + + { + const Status s = + ReadHeader(file_reader.get(), read_options, column_family_id, + statistics, &compression_type); + if (!s.ok()) { + return s; + } + } + + { + const Status s = + ReadFooter(file_reader.get(), read_options, file_size, statistics); + if (!s.ok()) { + return s; + } + } + + blob_file_reader->reset( + new BlobFileReader(std::move(file_reader), file_size, compression_type, + immutable_options.clock, statistics)); + + return Status::OK(); +} + +Status BlobFileReader::OpenFile( + const ImmutableOptions& immutable_options, const FileOptions& file_opts, + HistogramImpl* blob_file_read_hist, uint64_t blob_file_number, + const std::shared_ptr& io_tracer, uint64_t* file_size, + std::unique_ptr* file_reader) { + assert(file_size); + assert(file_reader); + + const auto& cf_paths = immutable_options.cf_paths; + assert(!cf_paths.empty()); + + const std::string blob_file_path = + BlobFileName(cf_paths.front().path, blob_file_number); + + FileSystem* const fs = immutable_options.fs.get(); + assert(fs); + + constexpr IODebugContext* dbg = nullptr; + + { + TEST_SYNC_POINT("BlobFileReader::OpenFile:GetFileSize"); + + const Status s = + fs->GetFileSize(blob_file_path, IOOptions(), file_size, dbg); + if (!s.ok()) { + return s; + } + } + + if (*file_size < BlobLogHeader::kSize + BlobLogFooter::kSize) { + return Status::Corruption("Malformed blob file"); + } + + std::unique_ptr file; + + { + TEST_SYNC_POINT("BlobFileReader::OpenFile:NewRandomAccessFile"); + + const Status s = + fs->NewRandomAccessFile(blob_file_path, file_opts, &file, dbg); + if (!s.ok()) { + return s; + } + } + + assert(file); + + if (immutable_options.advise_random_on_open) { + file->Hint(FSRandomAccessFile::kRandom); + } + + file_reader->reset(new RandomAccessFileReader( + std::move(file), blob_file_path, immutable_options.clock, io_tracer, + immutable_options.stats, BLOB_DB_BLOB_FILE_READ_MICROS, + blob_file_read_hist, immutable_options.rate_limiter.get(), + immutable_options.listeners)); + + return Status::OK(); +} + +Status BlobFileReader::ReadHeader(const RandomAccessFileReader* file_reader, + const ReadOptions& read_options, + uint32_t column_family_id, + Statistics* statistics, + CompressionType* compression_type) { + assert(file_reader); + assert(compression_type); + + Slice header_slice; + Buffer buf; + AlignedBuf aligned_buf; + + { + TEST_SYNC_POINT("BlobFileReader::ReadHeader:ReadFromFile"); + + constexpr uint64_t read_offset = 0; + constexpr size_t read_size = BlobLogHeader::kSize; + + // TODO: rate limit reading headers from blob files. + const Status s = + ReadFromFile(file_reader, read_options, read_offset, read_size, + statistics, &header_slice, &buf, &aligned_buf, + Env::IO_TOTAL /* rate_limiter_priority */); + if (!s.ok()) { + return s; + } + + TEST_SYNC_POINT_CALLBACK("BlobFileReader::ReadHeader:TamperWithResult", + &header_slice); + } + + BlobLogHeader header; + + { + const Status s = header.DecodeFrom(header_slice); + if (!s.ok()) { + return s; + } + } + + constexpr ExpirationRange no_expiration_range; + + if (header.has_ttl || header.expiration_range != no_expiration_range) { + return Status::Corruption("Unexpected TTL blob file"); + } + + if (header.column_family_id != column_family_id) { + return Status::Corruption("Column family ID mismatch"); + } + + *compression_type = header.compression; + + return Status::OK(); +} + +Status BlobFileReader::ReadFooter(const RandomAccessFileReader* file_reader, + const ReadOptions& read_options, + uint64_t file_size, Statistics* statistics) { + assert(file_size >= BlobLogHeader::kSize + BlobLogFooter::kSize); + assert(file_reader); + + Slice footer_slice; + Buffer buf; + AlignedBuf aligned_buf; + + { + TEST_SYNC_POINT("BlobFileReader::ReadFooter:ReadFromFile"); + + const uint64_t read_offset = file_size - BlobLogFooter::kSize; + constexpr size_t read_size = BlobLogFooter::kSize; + + // TODO: rate limit reading footers from blob files. + const Status s = + ReadFromFile(file_reader, read_options, read_offset, read_size, + statistics, &footer_slice, &buf, &aligned_buf, + Env::IO_TOTAL /* rate_limiter_priority */); + if (!s.ok()) { + return s; + } + + TEST_SYNC_POINT_CALLBACK("BlobFileReader::ReadFooter:TamperWithResult", + &footer_slice); + } + + BlobLogFooter footer; + + { + const Status s = footer.DecodeFrom(footer_slice); + if (!s.ok()) { + return s; + } + } + + constexpr ExpirationRange no_expiration_range; + + if (footer.expiration_range != no_expiration_range) { + return Status::Corruption("Unexpected TTL blob file"); + } + + return Status::OK(); +} + +Status BlobFileReader::ReadFromFile(const RandomAccessFileReader* file_reader, + const ReadOptions& read_options, + uint64_t read_offset, size_t read_size, + Statistics* statistics, Slice* slice, + Buffer* buf, AlignedBuf* aligned_buf, + Env::IOPriority rate_limiter_priority) { + assert(slice); + assert(buf); + assert(aligned_buf); + + assert(file_reader); + + RecordTick(statistics, BLOB_DB_BLOB_FILE_BYTES_READ, read_size); + + Status s; + + IOOptions io_options; + s = file_reader->PrepareIOOptions(read_options, io_options); + if (!s.ok()) { + return s; + } + + if (file_reader->use_direct_io()) { + constexpr char* scratch = nullptr; + + s = file_reader->Read(io_options, read_offset, read_size, slice, scratch, + aligned_buf, rate_limiter_priority); + } else { + buf->reset(new char[read_size]); + constexpr AlignedBuf* aligned_scratch = nullptr; + + s = file_reader->Read(io_options, read_offset, read_size, slice, buf->get(), + aligned_scratch, rate_limiter_priority); + } + + if (!s.ok()) { + return s; + } + + if (slice->size() != read_size) { + return Status::Corruption("Failed to read data from blob file"); + } + + return Status::OK(); +} + +BlobFileReader::BlobFileReader( + std::unique_ptr&& file_reader, uint64_t file_size, + CompressionType compression_type, SystemClock* clock, + Statistics* statistics) + : file_reader_(std::move(file_reader)), + file_size_(file_size), + compression_type_(compression_type), + clock_(clock), + statistics_(statistics) { + assert(file_reader_); +} + +BlobFileReader::~BlobFileReader() = default; + +Status BlobFileReader::GetBlob( + const ReadOptions& read_options, const Slice& user_key, uint64_t offset, + uint64_t value_size, CompressionType compression_type, + FilePrefetchBuffer* prefetch_buffer, MemoryAllocator* allocator, + std::unique_ptr* result, uint64_t* bytes_read) const { + assert(result); + + const uint64_t key_size = user_key.size(); + + if (!IsValidBlobOffset(offset, key_size, value_size, file_size_)) { + return Status::Corruption("Invalid blob offset"); + } + + if (compression_type != compression_type_) { + return Status::Corruption("Compression type mismatch when reading blob"); + } + + // Note: if verify_checksum is set, we read the entire blob record to be able + // to perform the verification; otherwise, we just read the blob itself. Since + // the offset in BlobIndex actually points to the blob value, we need to make + // an adjustment in the former case. + const uint64_t adjustment = + read_options.verify_checksums + ? BlobLogRecord::CalculateAdjustmentForRecordHeader(key_size) + : 0; + assert(offset >= adjustment); + + const uint64_t record_offset = offset - adjustment; + const uint64_t record_size = value_size + adjustment; + + Slice record_slice; + Buffer buf; + AlignedBuf aligned_buf; + + bool prefetched = false; + + if (prefetch_buffer) { + Status s; + constexpr bool for_compaction = true; + + IOOptions io_options; + s = file_reader_->PrepareIOOptions(read_options, io_options); + if (!s.ok()) { + return s; + } + prefetched = prefetch_buffer->TryReadFromCache( + io_options, file_reader_.get(), record_offset, + static_cast(record_size), &record_slice, &s, + read_options.rate_limiter_priority, for_compaction); + if (!s.ok()) { + return s; + } + } + + if (!prefetched) { + TEST_SYNC_POINT("BlobFileReader::GetBlob:ReadFromFile"); + PERF_COUNTER_ADD(blob_read_count, 1); + PERF_COUNTER_ADD(blob_read_byte, record_size); + PERF_TIMER_GUARD(blob_read_time); + const Status s = ReadFromFile( + file_reader_.get(), read_options, record_offset, + static_cast(record_size), statistics_, &record_slice, &buf, + &aligned_buf, read_options.rate_limiter_priority); + if (!s.ok()) { + return s; + } + } + + TEST_SYNC_POINT_CALLBACK("BlobFileReader::GetBlob:TamperWithResult", + &record_slice); + + if (read_options.verify_checksums) { + const Status s = VerifyBlob(record_slice, user_key, value_size); + if (!s.ok()) { + return s; + } + } + + const Slice value_slice(record_slice.data() + adjustment, value_size); + + { + const Status s = UncompressBlobIfNeeded( + value_slice, compression_type, allocator, clock_, statistics_, result); + if (!s.ok()) { + return s; + } + } + + if (bytes_read) { + *bytes_read = record_size; + } + + return Status::OK(); +} + +void BlobFileReader::MultiGetBlob( + const ReadOptions& read_options, MemoryAllocator* allocator, + autovector>>& + blob_reqs, + uint64_t* bytes_read) const { + const size_t num_blobs = blob_reqs.size(); + assert(num_blobs > 0); + assert(num_blobs <= MultiGetContext::MAX_BATCH_SIZE); + +#ifndef NDEBUG + for (size_t i = 0; i < num_blobs - 1; ++i) { + assert(blob_reqs[i].first->offset <= blob_reqs[i + 1].first->offset); + } +#endif // !NDEBUG + + std::vector read_reqs; + autovector adjustments; + uint64_t total_len = 0; + read_reqs.reserve(num_blobs); + for (size_t i = 0; i < num_blobs; ++i) { + BlobReadRequest* const req = blob_reqs[i].first; + assert(req); + assert(req->user_key); + assert(req->status); + + const size_t key_size = req->user_key->size(); + const uint64_t offset = req->offset; + const uint64_t value_size = req->len; + + if (!IsValidBlobOffset(offset, key_size, value_size, file_size_)) { + *req->status = Status::Corruption("Invalid blob offset"); + continue; + } + if (req->compression != compression_type_) { + *req->status = + Status::Corruption("Compression type mismatch when reading a blob"); + continue; + } + + const uint64_t adjustment = + read_options.verify_checksums + ? BlobLogRecord::CalculateAdjustmentForRecordHeader(key_size) + : 0; + assert(req->offset >= adjustment); + adjustments.push_back(adjustment); + + FSReadRequest read_req; + read_req.offset = req->offset - adjustment; + read_req.len = req->len + adjustment; + total_len += read_req.len; + read_reqs.emplace_back(std::move(read_req)); + } + + RecordTick(statistics_, BLOB_DB_BLOB_FILE_BYTES_READ, total_len); + + Buffer buf; + AlignedBuf aligned_buf; + + Status s; + bool direct_io = file_reader_->use_direct_io(); + if (direct_io) { + for (size_t i = 0; i < read_reqs.size(); ++i) { + read_reqs[i].scratch = nullptr; + } + } else { + buf.reset(new char[total_len]); + std::ptrdiff_t pos = 0; + for (size_t i = 0; i < read_reqs.size(); ++i) { + read_reqs[i].scratch = buf.get() + pos; + pos += read_reqs[i].len; + } + } + TEST_SYNC_POINT("BlobFileReader::MultiGetBlob:ReadFromFile"); + PERF_COUNTER_ADD(blob_read_count, num_blobs); + PERF_COUNTER_ADD(blob_read_byte, total_len); + s = file_reader_->MultiRead(IOOptions(), read_reqs.data(), read_reqs.size(), + direct_io ? &aligned_buf : nullptr, + read_options.rate_limiter_priority); + if (!s.ok()) { + for (auto& req : read_reqs) { + req.status.PermitUncheckedError(); + } + for (auto& blob_req : blob_reqs) { + BlobReadRequest* const req = blob_req.first; + assert(req); + assert(req->status); + + if (!req->status->IsCorruption()) { + // Avoid overwriting corruption status. + *req->status = s; + } + } + return; + } + + assert(s.ok()); + + uint64_t total_bytes = 0; + for (size_t i = 0, j = 0; i < num_blobs; ++i) { + BlobReadRequest* const req = blob_reqs[i].first; + assert(req); + assert(req->user_key); + assert(req->status); + + if (!req->status->ok()) { + continue; + } + + assert(j < read_reqs.size()); + auto& read_req = read_reqs[j++]; + const auto& record_slice = read_req.result; + if (read_req.status.ok() && record_slice.size() != read_req.len) { + read_req.status = + IOStatus::Corruption("Failed to read data from blob file"); + } + + *req->status = read_req.status; + if (!req->status->ok()) { + continue; + } + + // Verify checksums if enabled + if (read_options.verify_checksums) { + *req->status = VerifyBlob(record_slice, *req->user_key, req->len); + if (!req->status->ok()) { + continue; + } + } + + // Uncompress blob if needed + Slice value_slice(record_slice.data() + adjustments[i], req->len); + *req->status = + UncompressBlobIfNeeded(value_slice, compression_type_, allocator, + clock_, statistics_, &blob_reqs[i].second); + if (req->status->ok()) { + total_bytes += record_slice.size(); + } + } + + if (bytes_read) { + *bytes_read = total_bytes; + } +} + +Status BlobFileReader::VerifyBlob(const Slice& record_slice, + const Slice& user_key, uint64_t value_size) { + PERF_TIMER_GUARD(blob_checksum_time); + + BlobLogRecord record; + + const Slice header_slice(record_slice.data(), BlobLogRecord::kHeaderSize); + + { + const Status s = record.DecodeHeaderFrom(header_slice); + if (!s.ok()) { + return s; + } + } + + if (record.key_size != user_key.size()) { + return Status::Corruption("Key size mismatch when reading blob"); + } + + if (record.value_size != value_size) { + return Status::Corruption("Value size mismatch when reading blob"); + } + + record.key = + Slice(record_slice.data() + BlobLogRecord::kHeaderSize, record.key_size); + if (record.key != user_key) { + return Status::Corruption("Key mismatch when reading blob"); + } + + record.value = Slice(record.key.data() + record.key_size, value_size); + + { + TEST_SYNC_POINT_CALLBACK("BlobFileReader::VerifyBlob:CheckBlobCRC", + &record); + + const Status s = record.CheckBlobCRC(); + if (!s.ok()) { + return s; + } + } + + return Status::OK(); +} + +Status BlobFileReader::UncompressBlobIfNeeded( + const Slice& value_slice, CompressionType compression_type, + MemoryAllocator* allocator, SystemClock* clock, Statistics* statistics, + std::unique_ptr* result) { + assert(result); + + if (compression_type == kNoCompression) { + BlobContentsCreator::Create(result, nullptr, value_slice, allocator); + return Status::OK(); + } + + UncompressionContext context(compression_type); + UncompressionInfo info(context, UncompressionDict::GetEmptyDict(), + compression_type); + + size_t uncompressed_size = 0; + constexpr uint32_t compression_format_version = 2; + + CacheAllocationPtr output; + + { + PERF_TIMER_GUARD(blob_decompress_time); + StopWatch stop_watch(clock, statistics, BLOB_DB_DECOMPRESSION_MICROS); + output = UncompressData(info, value_slice.data(), value_slice.size(), + &uncompressed_size, compression_format_version, + allocator); + } + + TEST_SYNC_POINT_CALLBACK( + "BlobFileReader::UncompressBlobIfNeeded:TamperWithResult", &output); + + if (!output) { + return Status::Corruption("Unable to uncompress blob"); + } + + result->reset(new BlobContents(std::move(output), uncompressed_size)); + + return Status::OK(); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_file_reader.h b/librocksdb-sys/rocksdb/db/blob/blob_file_reader.h new file mode 100644 index 0000000..990e325 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_file_reader.h @@ -0,0 +1,112 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include + +#include "db/blob/blob_read_request.h" +#include "file/random_access_file_reader.h" +#include "rocksdb/compression_type.h" +#include "rocksdb/rocksdb_namespace.h" +#include "util/autovector.h" + +namespace ROCKSDB_NAMESPACE { + +class Status; +struct ImmutableOptions; +struct FileOptions; +class HistogramImpl; +struct ReadOptions; +class Slice; +class FilePrefetchBuffer; +class BlobContents; +class Statistics; + +class BlobFileReader { + public: + static Status Create(const ImmutableOptions& immutable_options, + const ReadOptions& read_options, + const FileOptions& file_options, + uint32_t column_family_id, + HistogramImpl* blob_file_read_hist, + uint64_t blob_file_number, + const std::shared_ptr& io_tracer, + std::unique_ptr* reader); + + BlobFileReader(const BlobFileReader&) = delete; + BlobFileReader& operator=(const BlobFileReader&) = delete; + + ~BlobFileReader(); + + Status GetBlob(const ReadOptions& read_options, const Slice& user_key, + uint64_t offset, uint64_t value_size, + CompressionType compression_type, + FilePrefetchBuffer* prefetch_buffer, + MemoryAllocator* allocator, + std::unique_ptr* result, + uint64_t* bytes_read) const; + + // offsets must be sorted in ascending order by caller. + void MultiGetBlob( + const ReadOptions& read_options, MemoryAllocator* allocator, + autovector>>& + blob_reqs, + uint64_t* bytes_read) const; + + CompressionType GetCompressionType() const { return compression_type_; } + + uint64_t GetFileSize() const { return file_size_; } + + private: + BlobFileReader(std::unique_ptr&& file_reader, + uint64_t file_size, CompressionType compression_type, + SystemClock* clock, Statistics* statistics); + + static Status OpenFile(const ImmutableOptions& immutable_options, + const FileOptions& file_opts, + HistogramImpl* blob_file_read_hist, + uint64_t blob_file_number, + const std::shared_ptr& io_tracer, + uint64_t* file_size, + std::unique_ptr* file_reader); + + static Status ReadHeader(const RandomAccessFileReader* file_reader, + const ReadOptions& read_options, + uint32_t column_family_id, Statistics* statistics, + CompressionType* compression_type); + + static Status ReadFooter(const RandomAccessFileReader* file_reader, + const ReadOptions& read_options, uint64_t file_size, + Statistics* statistics); + + using Buffer = std::unique_ptr; + + static Status ReadFromFile(const RandomAccessFileReader* file_reader, + const ReadOptions& read_options, + uint64_t read_offset, size_t read_size, + Statistics* statistics, Slice* slice, Buffer* buf, + AlignedBuf* aligned_buf, + Env::IOPriority rate_limiter_priority); + + static Status VerifyBlob(const Slice& record_slice, const Slice& user_key, + uint64_t value_size); + + static Status UncompressBlobIfNeeded(const Slice& value_slice, + CompressionType compression_type, + MemoryAllocator* allocator, + SystemClock* clock, + Statistics* statistics, + std::unique_ptr* result); + + std::unique_ptr file_reader_; + uint64_t file_size_; + CompressionType compression_type_; + SystemClock* clock_; + Statistics* statistics_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_file_reader_test.cc b/librocksdb-sys/rocksdb/db/blob/blob_file_reader_test.cc new file mode 100644 index 0000000..c8e4e59 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_file_reader_test.cc @@ -0,0 +1,1023 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_file_reader.h" + +#include +#include + +#include "db/blob/blob_contents.h" +#include "db/blob/blob_log_format.h" +#include "db/blob/blob_log_writer.h" +#include "env/mock_env.h" +#include "file/filename.h" +#include "file/read_write_util.h" +#include "file/writable_file_writer.h" +#include "options/cf_options.h" +#include "rocksdb/env.h" +#include "rocksdb/file_system.h" +#include "rocksdb/options.h" +#include "test_util/sync_point.h" +#include "test_util/testharness.h" +#include "util/compression.h" +#include "utilities/fault_injection_env.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { + +// Creates a test blob file with `num` blobs in it. +void WriteBlobFile(const ImmutableOptions& immutable_options, + uint32_t column_family_id, bool has_ttl, + const ExpirationRange& expiration_range_header, + const ExpirationRange& expiration_range_footer, + uint64_t blob_file_number, const std::vector& keys, + const std::vector& blobs, CompressionType compression, + std::vector& blob_offsets, + std::vector& blob_sizes) { + assert(!immutable_options.cf_paths.empty()); + size_t num = keys.size(); + assert(num == blobs.size()); + assert(num == blob_offsets.size()); + assert(num == blob_sizes.size()); + + const std::string blob_file_path = + BlobFileName(immutable_options.cf_paths.front().path, blob_file_number); + std::unique_ptr file; + ASSERT_OK(NewWritableFile(immutable_options.fs.get(), blob_file_path, &file, + FileOptions())); + + std::unique_ptr file_writer(new WritableFileWriter( + std::move(file), blob_file_path, FileOptions(), immutable_options.clock)); + + constexpr Statistics* statistics = nullptr; + constexpr bool use_fsync = false; + constexpr bool do_flush = false; + + BlobLogWriter blob_log_writer(std::move(file_writer), immutable_options.clock, + statistics, blob_file_number, use_fsync, + do_flush); + + BlobLogHeader header(column_family_id, compression, has_ttl, + expiration_range_header); + + ASSERT_OK(blob_log_writer.WriteHeader(header)); + + std::vector compressed_blobs(num); + std::vector blobs_to_write(num); + if (kNoCompression == compression) { + for (size_t i = 0; i < num; ++i) { + blobs_to_write[i] = blobs[i]; + blob_sizes[i] = blobs[i].size(); + } + } else { + CompressionOptions opts; + CompressionContext context(compression); + constexpr uint64_t sample_for_compression = 0; + CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(), + compression, sample_for_compression); + + constexpr uint32_t compression_format_version = 2; + + for (size_t i = 0; i < num; ++i) { + ASSERT_TRUE(CompressData(blobs[i], info, compression_format_version, + &compressed_blobs[i])); + blobs_to_write[i] = compressed_blobs[i]; + blob_sizes[i] = compressed_blobs[i].size(); + } + } + + for (size_t i = 0; i < num; ++i) { + uint64_t key_offset = 0; + ASSERT_OK(blob_log_writer.AddRecord(keys[i], blobs_to_write[i], &key_offset, + &blob_offsets[i])); + } + + BlobLogFooter footer; + footer.blob_count = num; + footer.expiration_range = expiration_range_footer; + + std::string checksum_method; + std::string checksum_value; + ASSERT_OK( + blob_log_writer.AppendFooter(footer, &checksum_method, &checksum_value)); +} + +// Creates a test blob file with a single blob in it. Note: this method +// makes it possible to test various corner cases by allowing the caller +// to specify the contents of various blob file header/footer fields. +void WriteBlobFile(const ImmutableOptions& immutable_options, + uint32_t column_family_id, bool has_ttl, + const ExpirationRange& expiration_range_header, + const ExpirationRange& expiration_range_footer, + uint64_t blob_file_number, const Slice& key, + const Slice& blob, CompressionType compression, + uint64_t* blob_offset, uint64_t* blob_size) { + std::vector keys{key}; + std::vector blobs{blob}; + std::vector blob_offsets{0}; + std::vector blob_sizes{0}; + WriteBlobFile(immutable_options, column_family_id, has_ttl, + expiration_range_header, expiration_range_footer, + blob_file_number, keys, blobs, compression, blob_offsets, + blob_sizes); + if (blob_offset) { + *blob_offset = blob_offsets[0]; + } + if (blob_size) { + *blob_size = blob_sizes[0]; + } +} + +} // anonymous namespace + +class BlobFileReaderTest : public testing::Test { + protected: + BlobFileReaderTest() { mock_env_.reset(MockEnv::Create(Env::Default())); } + std::unique_ptr mock_env_; +}; + +TEST_F(BlobFileReaderTest, CreateReaderAndGetBlob) { + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileReaderTest_CreateReaderAndGetBlob"), + 0); + options.enable_blob_files = true; + + ImmutableOptions immutable_options(options); + + constexpr uint32_t column_family_id = 1; + constexpr bool has_ttl = false; + constexpr ExpirationRange expiration_range; + constexpr uint64_t blob_file_number = 1; + constexpr size_t num_blobs = 3; + const std::vector key_strs = {"key1", "key2", "key3"}; + const std::vector blob_strs = {"blob1", "blob2", "blob3"}; + + const std::vector keys = {key_strs[0], key_strs[1], key_strs[2]}; + const std::vector blobs = {blob_strs[0], blob_strs[1], blob_strs[2]}; + + std::vector blob_offsets(keys.size()); + std::vector blob_sizes(keys.size()); + + WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range, + expiration_range, blob_file_number, keys, blobs, kNoCompression, + blob_offsets, blob_sizes); + + constexpr HistogramImpl* blob_file_read_hist = nullptr; + + std::unique_ptr reader; + + ReadOptions read_options; + ASSERT_OK(BlobFileReader::Create( + immutable_options, read_options, FileOptions(), column_family_id, + blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader)); + + // Make sure the blob can be retrieved with and without checksum verification + read_options.verify_checksums = false; + + constexpr FilePrefetchBuffer* prefetch_buffer = nullptr; + constexpr MemoryAllocator* allocator = nullptr; + + { + std::unique_ptr value; + uint64_t bytes_read = 0; + + ASSERT_OK(reader->GetBlob(read_options, keys[0], blob_offsets[0], + blob_sizes[0], kNoCompression, prefetch_buffer, + allocator, &value, &bytes_read)); + ASSERT_NE(value, nullptr); + ASSERT_EQ(value->data(), blobs[0]); + ASSERT_EQ(bytes_read, blob_sizes[0]); + + // MultiGetBlob + bytes_read = 0; + size_t total_size = 0; + + std::array statuses_buf; + std::array requests_buf; + autovector>> + blob_reqs; + + for (size_t i = 0; i < num_blobs; ++i) { + requests_buf[i] = + BlobReadRequest(keys[i], blob_offsets[i], blob_sizes[i], + kNoCompression, nullptr, &statuses_buf[i]); + blob_reqs.emplace_back(&requests_buf[i], std::unique_ptr()); + } + + reader->MultiGetBlob(read_options, allocator, blob_reqs, &bytes_read); + + for (size_t i = 0; i < num_blobs; ++i) { + const auto& result = blob_reqs[i].second; + + ASSERT_OK(statuses_buf[i]); + ASSERT_NE(result, nullptr); + ASSERT_EQ(result->data(), blobs[i]); + total_size += blob_sizes[i]; + } + ASSERT_EQ(bytes_read, total_size); + } + + read_options.verify_checksums = true; + + { + std::unique_ptr value; + uint64_t bytes_read = 0; + + ASSERT_OK(reader->GetBlob(read_options, keys[1], blob_offsets[1], + blob_sizes[1], kNoCompression, prefetch_buffer, + allocator, &value, &bytes_read)); + ASSERT_NE(value, nullptr); + ASSERT_EQ(value->data(), blobs[1]); + + const uint64_t key_size = keys[1].size(); + ASSERT_EQ(bytes_read, + BlobLogRecord::CalculateAdjustmentForRecordHeader(key_size) + + blob_sizes[1]); + } + + // Invalid offset (too close to start of file) + { + std::unique_ptr value; + uint64_t bytes_read = 0; + + ASSERT_TRUE(reader + ->GetBlob(read_options, keys[0], blob_offsets[0] - 1, + blob_sizes[0], kNoCompression, prefetch_buffer, + allocator, &value, &bytes_read) + .IsCorruption()); + ASSERT_EQ(value, nullptr); + ASSERT_EQ(bytes_read, 0); + } + + // Invalid offset (too close to end of file) + { + std::unique_ptr value; + uint64_t bytes_read = 0; + + ASSERT_TRUE(reader + ->GetBlob(read_options, keys[2], blob_offsets[2] + 1, + blob_sizes[2], kNoCompression, prefetch_buffer, + allocator, &value, &bytes_read) + .IsCorruption()); + ASSERT_EQ(value, nullptr); + ASSERT_EQ(bytes_read, 0); + } + + // Incorrect compression type + { + std::unique_ptr value; + uint64_t bytes_read = 0; + + ASSERT_TRUE(reader + ->GetBlob(read_options, keys[0], blob_offsets[0], + blob_sizes[0], kZSTD, prefetch_buffer, allocator, + &value, &bytes_read) + .IsCorruption()); + ASSERT_EQ(value, nullptr); + ASSERT_EQ(bytes_read, 0); + } + + // Incorrect key size + { + constexpr char shorter_key[] = "k"; + std::unique_ptr value; + uint64_t bytes_read = 0; + + ASSERT_TRUE(reader + ->GetBlob(read_options, shorter_key, + blob_offsets[0] - + (keys[0].size() - sizeof(shorter_key) + 1), + blob_sizes[0], kNoCompression, prefetch_buffer, + allocator, &value, &bytes_read) + .IsCorruption()); + ASSERT_EQ(value, nullptr); + ASSERT_EQ(bytes_read, 0); + + // MultiGetBlob + autovector> key_refs; + for (const auto& key_ref : keys) { + key_refs.emplace_back(std::cref(key_ref)); + } + Slice shorter_key_slice(shorter_key, sizeof(shorter_key) - 1); + key_refs[1] = std::cref(shorter_key_slice); + + autovector offsets{ + blob_offsets[0], + blob_offsets[1] - (keys[1].size() - key_refs[1].get().size()), + blob_offsets[2]}; + + std::array statuses_buf; + std::array requests_buf; + autovector>> + blob_reqs; + + for (size_t i = 0; i < num_blobs; ++i) { + requests_buf[i] = + BlobReadRequest(key_refs[i], offsets[i], blob_sizes[i], + kNoCompression, nullptr, &statuses_buf[i]); + blob_reqs.emplace_back(&requests_buf[i], std::unique_ptr()); + } + + reader->MultiGetBlob(read_options, allocator, blob_reqs, &bytes_read); + + for (size_t i = 0; i < num_blobs; ++i) { + if (i == 1) { + ASSERT_TRUE(statuses_buf[i].IsCorruption()); + } else { + ASSERT_OK(statuses_buf[i]); + } + } + } + + // Incorrect key + { + constexpr char incorrect_key[] = "foo1"; + std::unique_ptr value; + uint64_t bytes_read = 0; + + ASSERT_TRUE(reader + ->GetBlob(read_options, incorrect_key, blob_offsets[0], + blob_sizes[0], kNoCompression, prefetch_buffer, + allocator, &value, &bytes_read) + .IsCorruption()); + ASSERT_EQ(value, nullptr); + ASSERT_EQ(bytes_read, 0); + + // MultiGetBlob + autovector> key_refs; + for (const auto& key_ref : keys) { + key_refs.emplace_back(std::cref(key_ref)); + } + Slice wrong_key_slice(incorrect_key, sizeof(incorrect_key) - 1); + key_refs[2] = std::cref(wrong_key_slice); + + std::array statuses_buf; + std::array requests_buf; + autovector>> + blob_reqs; + + for (size_t i = 0; i < num_blobs; ++i) { + requests_buf[i] = + BlobReadRequest(key_refs[i], blob_offsets[i], blob_sizes[i], + kNoCompression, nullptr, &statuses_buf[i]); + blob_reqs.emplace_back(&requests_buf[i], std::unique_ptr()); + } + + reader->MultiGetBlob(read_options, allocator, blob_reqs, &bytes_read); + + for (size_t i = 0; i < num_blobs; ++i) { + if (i == num_blobs - 1) { + ASSERT_TRUE(statuses_buf[i].IsCorruption()); + } else { + ASSERT_OK(statuses_buf[i]); + } + } + } + + // Incorrect value size + { + std::unique_ptr value; + uint64_t bytes_read = 0; + + ASSERT_TRUE(reader + ->GetBlob(read_options, keys[1], blob_offsets[1], + blob_sizes[1] + 1, kNoCompression, + prefetch_buffer, allocator, &value, &bytes_read) + .IsCorruption()); + ASSERT_EQ(value, nullptr); + ASSERT_EQ(bytes_read, 0); + + // MultiGetBlob + autovector> key_refs; + for (const auto& key_ref : keys) { + key_refs.emplace_back(std::cref(key_ref)); + } + + std::array statuses_buf; + std::array requests_buf; + + requests_buf[0] = + BlobReadRequest(key_refs[0], blob_offsets[0], blob_sizes[0], + kNoCompression, nullptr, &statuses_buf[0]); + requests_buf[1] = + BlobReadRequest(key_refs[1], blob_offsets[1], blob_sizes[1] + 1, + kNoCompression, nullptr, &statuses_buf[1]); + requests_buf[2] = + BlobReadRequest(key_refs[2], blob_offsets[2], blob_sizes[2], + kNoCompression, nullptr, &statuses_buf[2]); + + autovector>> + blob_reqs; + + for (size_t i = 0; i < num_blobs; ++i) { + blob_reqs.emplace_back(&requests_buf[i], std::unique_ptr()); + } + + reader->MultiGetBlob(read_options, allocator, blob_reqs, &bytes_read); + + for (size_t i = 0; i < num_blobs; ++i) { + if (i != 1) { + ASSERT_OK(statuses_buf[i]); + } else { + ASSERT_TRUE(statuses_buf[i].IsCorruption()); + } + } + } +} + +TEST_F(BlobFileReaderTest, Malformed) { + // Write a blob file consisting of nothing but a header, and make sure we + // detect the error when we open it for reading + + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), "BlobFileReaderTest_Malformed"), + 0); + options.enable_blob_files = true; + + ImmutableOptions immutable_options(options); + + constexpr uint32_t column_family_id = 1; + constexpr uint64_t blob_file_number = 1; + + { + constexpr bool has_ttl = false; + constexpr ExpirationRange expiration_range; + + const std::string blob_file_path = + BlobFileName(immutable_options.cf_paths.front().path, blob_file_number); + + std::unique_ptr file; + ASSERT_OK(NewWritableFile(immutable_options.fs.get(), blob_file_path, &file, + FileOptions())); + + std::unique_ptr file_writer( + new WritableFileWriter(std::move(file), blob_file_path, FileOptions(), + immutable_options.clock)); + + constexpr Statistics* statistics = nullptr; + constexpr bool use_fsync = false; + constexpr bool do_flush = false; + + BlobLogWriter blob_log_writer(std::move(file_writer), + immutable_options.clock, statistics, + blob_file_number, use_fsync, do_flush); + + BlobLogHeader header(column_family_id, kNoCompression, has_ttl, + expiration_range); + + ASSERT_OK(blob_log_writer.WriteHeader(header)); + } + + constexpr HistogramImpl* blob_file_read_hist = nullptr; + + std::unique_ptr reader; + const ReadOptions read_options; + ASSERT_TRUE(BlobFileReader::Create(immutable_options, read_options, + FileOptions(), column_family_id, + blob_file_read_hist, blob_file_number, + nullptr /*IOTracer*/, &reader) + .IsCorruption()); +} + +TEST_F(BlobFileReaderTest, TTL) { + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), "BlobFileReaderTest_TTL"), 0); + options.enable_blob_files = true; + + ImmutableOptions immutable_options(options); + + constexpr uint32_t column_family_id = 1; + constexpr bool has_ttl = true; + constexpr ExpirationRange expiration_range; + constexpr uint64_t blob_file_number = 1; + constexpr char key[] = "key"; + constexpr char blob[] = "blob"; + + uint64_t blob_offset = 0; + uint64_t blob_size = 0; + + WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range, + expiration_range, blob_file_number, key, blob, kNoCompression, + &blob_offset, &blob_size); + + constexpr HistogramImpl* blob_file_read_hist = nullptr; + + std::unique_ptr reader; + const ReadOptions read_options; + ASSERT_TRUE(BlobFileReader::Create(immutable_options, read_options, + FileOptions(), column_family_id, + blob_file_read_hist, blob_file_number, + nullptr /*IOTracer*/, &reader) + .IsCorruption()); +} + +TEST_F(BlobFileReaderTest, ExpirationRangeInHeader) { + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileReaderTest_ExpirationRangeInHeader"), + 0); + options.enable_blob_files = true; + + ImmutableOptions immutable_options(options); + + constexpr uint32_t column_family_id = 1; + constexpr bool has_ttl = false; + const ExpirationRange expiration_range_header( + 1, 2); // can be made constexpr when we adopt C++14 + constexpr ExpirationRange expiration_range_footer; + constexpr uint64_t blob_file_number = 1; + constexpr char key[] = "key"; + constexpr char blob[] = "blob"; + + uint64_t blob_offset = 0; + uint64_t blob_size = 0; + + WriteBlobFile(immutable_options, column_family_id, has_ttl, + expiration_range_header, expiration_range_footer, + blob_file_number, key, blob, kNoCompression, &blob_offset, + &blob_size); + + constexpr HistogramImpl* blob_file_read_hist = nullptr; + + std::unique_ptr reader; + const ReadOptions read_options; + ASSERT_TRUE(BlobFileReader::Create(immutable_options, read_options, + FileOptions(), column_family_id, + blob_file_read_hist, blob_file_number, + nullptr /*IOTracer*/, &reader) + .IsCorruption()); +} + +TEST_F(BlobFileReaderTest, ExpirationRangeInFooter) { + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileReaderTest_ExpirationRangeInFooter"), + 0); + options.enable_blob_files = true; + + ImmutableOptions immutable_options(options); + + constexpr uint32_t column_family_id = 1; + constexpr bool has_ttl = false; + constexpr ExpirationRange expiration_range_header; + const ExpirationRange expiration_range_footer( + 1, 2); // can be made constexpr when we adopt C++14 + constexpr uint64_t blob_file_number = 1; + constexpr char key[] = "key"; + constexpr char blob[] = "blob"; + + uint64_t blob_offset = 0; + uint64_t blob_size = 0; + + WriteBlobFile(immutable_options, column_family_id, has_ttl, + expiration_range_header, expiration_range_footer, + blob_file_number, key, blob, kNoCompression, &blob_offset, + &blob_size); + + constexpr HistogramImpl* blob_file_read_hist = nullptr; + + std::unique_ptr reader; + const ReadOptions read_options; + ASSERT_TRUE(BlobFileReader::Create(immutable_options, read_options, + FileOptions(), column_family_id, + blob_file_read_hist, blob_file_number, + nullptr /*IOTracer*/, &reader) + .IsCorruption()); +} + +TEST_F(BlobFileReaderTest, IncorrectColumnFamily) { + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileReaderTest_IncorrectColumnFamily"), + 0); + options.enable_blob_files = true; + + ImmutableOptions immutable_options(options); + + constexpr uint32_t column_family_id = 1; + constexpr bool has_ttl = false; + constexpr ExpirationRange expiration_range; + constexpr uint64_t blob_file_number = 1; + constexpr char key[] = "key"; + constexpr char blob[] = "blob"; + + uint64_t blob_offset = 0; + uint64_t blob_size = 0; + + WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range, + expiration_range, blob_file_number, key, blob, kNoCompression, + &blob_offset, &blob_size); + + constexpr HistogramImpl* blob_file_read_hist = nullptr; + + std::unique_ptr reader; + + constexpr uint32_t incorrect_column_family_id = 2; + const ReadOptions read_options; + ASSERT_TRUE(BlobFileReader::Create(immutable_options, read_options, + FileOptions(), incorrect_column_family_id, + blob_file_read_hist, blob_file_number, + nullptr /*IOTracer*/, &reader) + .IsCorruption()); +} + +TEST_F(BlobFileReaderTest, BlobCRCError) { + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), "BlobFileReaderTest_BlobCRCError"), + 0); + options.enable_blob_files = true; + + ImmutableOptions immutable_options(options); + + constexpr uint32_t column_family_id = 1; + constexpr bool has_ttl = false; + constexpr ExpirationRange expiration_range; + constexpr uint64_t blob_file_number = 1; + constexpr char key[] = "key"; + constexpr char blob[] = "blob"; + + uint64_t blob_offset = 0; + uint64_t blob_size = 0; + + WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range, + expiration_range, blob_file_number, key, blob, kNoCompression, + &blob_offset, &blob_size); + + constexpr HistogramImpl* blob_file_read_hist = nullptr; + + std::unique_ptr reader; + const ReadOptions read_options; + ASSERT_OK(BlobFileReader::Create( + immutable_options, read_options, FileOptions(), column_family_id, + blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader)); + + SyncPoint::GetInstance()->SetCallBack( + "BlobFileReader::VerifyBlob:CheckBlobCRC", [](void* arg) { + BlobLogRecord* const record = static_cast(arg); + assert(record); + + record->blob_crc = 0xfaceb00c; + }); + + SyncPoint::GetInstance()->EnableProcessing(); + + constexpr FilePrefetchBuffer* prefetch_buffer = nullptr; + constexpr MemoryAllocator* allocator = nullptr; + + std::unique_ptr value; + uint64_t bytes_read = 0; + + ASSERT_TRUE(reader + ->GetBlob(ReadOptions(), key, blob_offset, blob_size, + kNoCompression, prefetch_buffer, allocator, &value, + &bytes_read) + .IsCorruption()); + ASSERT_EQ(value, nullptr); + ASSERT_EQ(bytes_read, 0); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(BlobFileReaderTest, Compression) { + if (!Snappy_Supported()) { + return; + } + + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), "BlobFileReaderTest_Compression"), + 0); + options.enable_blob_files = true; + + ImmutableOptions immutable_options(options); + + constexpr uint32_t column_family_id = 1; + constexpr bool has_ttl = false; + constexpr ExpirationRange expiration_range; + constexpr uint64_t blob_file_number = 1; + constexpr char key[] = "key"; + constexpr char blob[] = "blob"; + + uint64_t blob_offset = 0; + uint64_t blob_size = 0; + + WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range, + expiration_range, blob_file_number, key, blob, + kSnappyCompression, &blob_offset, &blob_size); + + constexpr HistogramImpl* blob_file_read_hist = nullptr; + + std::unique_ptr reader; + ReadOptions read_options; + ASSERT_OK(BlobFileReader::Create( + immutable_options, read_options, FileOptions(), column_family_id, + blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader)); + + // Make sure the blob can be retrieved with and without checksum verification + read_options.verify_checksums = false; + + constexpr FilePrefetchBuffer* prefetch_buffer = nullptr; + constexpr MemoryAllocator* allocator = nullptr; + + { + std::unique_ptr value; + uint64_t bytes_read = 0; + + ASSERT_OK(reader->GetBlob(read_options, key, blob_offset, blob_size, + kSnappyCompression, prefetch_buffer, allocator, + &value, &bytes_read)); + ASSERT_NE(value, nullptr); + ASSERT_EQ(value->data(), blob); + ASSERT_EQ(bytes_read, blob_size); + } + + read_options.verify_checksums = true; + + { + std::unique_ptr value; + uint64_t bytes_read = 0; + + ASSERT_OK(reader->GetBlob(read_options, key, blob_offset, blob_size, + kSnappyCompression, prefetch_buffer, allocator, + &value, &bytes_read)); + ASSERT_NE(value, nullptr); + ASSERT_EQ(value->data(), blob); + + constexpr uint64_t key_size = sizeof(key) - 1; + ASSERT_EQ(bytes_read, + BlobLogRecord::CalculateAdjustmentForRecordHeader(key_size) + + blob_size); + } +} + +TEST_F(BlobFileReaderTest, UncompressionError) { + if (!Snappy_Supported()) { + return; + } + + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileReaderTest_UncompressionError"), + 0); + options.enable_blob_files = true; + + ImmutableOptions immutable_options(options); + + constexpr uint32_t column_family_id = 1; + constexpr bool has_ttl = false; + constexpr ExpirationRange expiration_range; + constexpr uint64_t blob_file_number = 1; + constexpr char key[] = "key"; + constexpr char blob[] = "blob"; + + uint64_t blob_offset = 0; + uint64_t blob_size = 0; + + WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range, + expiration_range, blob_file_number, key, blob, + kSnappyCompression, &blob_offset, &blob_size); + + constexpr HistogramImpl* blob_file_read_hist = nullptr; + + std::unique_ptr reader; + const ReadOptions read_options; + ASSERT_OK(BlobFileReader::Create( + immutable_options, read_options, FileOptions(), column_family_id, + blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader)); + + SyncPoint::GetInstance()->SetCallBack( + "BlobFileReader::UncompressBlobIfNeeded:TamperWithResult", [](void* arg) { + CacheAllocationPtr* const output = + static_cast(arg); + assert(output); + + output->reset(); + }); + + SyncPoint::GetInstance()->EnableProcessing(); + + constexpr FilePrefetchBuffer* prefetch_buffer = nullptr; + constexpr MemoryAllocator* allocator = nullptr; + + std::unique_ptr value; + uint64_t bytes_read = 0; + + ASSERT_TRUE(reader + ->GetBlob(ReadOptions(), key, blob_offset, blob_size, + kSnappyCompression, prefetch_buffer, allocator, + &value, &bytes_read) + .IsCorruption()); + ASSERT_EQ(value, nullptr); + ASSERT_EQ(bytes_read, 0); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +class BlobFileReaderIOErrorTest + : public testing::Test, + public testing::WithParamInterface { + protected: + BlobFileReaderIOErrorTest() : sync_point_(GetParam()) { + mock_env_.reset(MockEnv::Create(Env::Default())); + fault_injection_env_.reset(new FaultInjectionTestEnv(mock_env_.get())); + } + + std::unique_ptr mock_env_; + std::unique_ptr fault_injection_env_; + std::string sync_point_; +}; + +INSTANTIATE_TEST_CASE_P(BlobFileReaderTest, BlobFileReaderIOErrorTest, + ::testing::ValuesIn(std::vector{ + "BlobFileReader::OpenFile:GetFileSize", + "BlobFileReader::OpenFile:NewRandomAccessFile", + "BlobFileReader::ReadHeader:ReadFromFile", + "BlobFileReader::ReadFooter:ReadFromFile", + "BlobFileReader::GetBlob:ReadFromFile"})); + +TEST_P(BlobFileReaderIOErrorTest, IOError) { + // Simulates an I/O error during the specified step + + Options options; + options.env = fault_injection_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(fault_injection_env_.get(), + "BlobFileReaderIOErrorTest_IOError"), + 0); + options.enable_blob_files = true; + + ImmutableOptions immutable_options(options); + + constexpr uint32_t column_family_id = 1; + constexpr bool has_ttl = false; + constexpr ExpirationRange expiration_range; + constexpr uint64_t blob_file_number = 1; + constexpr char key[] = "key"; + constexpr char blob[] = "blob"; + + uint64_t blob_offset = 0; + uint64_t blob_size = 0; + + WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range, + expiration_range, blob_file_number, key, blob, kNoCompression, + &blob_offset, &blob_size); + + SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* /* arg */) { + fault_injection_env_->SetFilesystemActive(false, + Status::IOError(sync_point_)); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + constexpr HistogramImpl* blob_file_read_hist = nullptr; + + std::unique_ptr reader; + const ReadOptions read_options; + const Status s = BlobFileReader::Create( + immutable_options, read_options, FileOptions(), column_family_id, + blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader); + + const bool fail_during_create = + (sync_point_ != "BlobFileReader::GetBlob:ReadFromFile"); + + if (fail_during_create) { + ASSERT_TRUE(s.IsIOError()); + } else { + ASSERT_OK(s); + + constexpr FilePrefetchBuffer* prefetch_buffer = nullptr; + constexpr MemoryAllocator* allocator = nullptr; + + std::unique_ptr value; + uint64_t bytes_read = 0; + + ASSERT_TRUE(reader + ->GetBlob(ReadOptions(), key, blob_offset, blob_size, + kNoCompression, prefetch_buffer, allocator, + &value, &bytes_read) + .IsIOError()); + ASSERT_EQ(value, nullptr); + ASSERT_EQ(bytes_read, 0); + } + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +class BlobFileReaderDecodingErrorTest + : public testing::Test, + public testing::WithParamInterface { + protected: + BlobFileReaderDecodingErrorTest() : sync_point_(GetParam()) { + mock_env_.reset(MockEnv::Create(Env::Default())); + } + + std::unique_ptr mock_env_; + std::string sync_point_; +}; + +INSTANTIATE_TEST_CASE_P(BlobFileReaderTest, BlobFileReaderDecodingErrorTest, + ::testing::ValuesIn(std::vector{ + "BlobFileReader::ReadHeader:TamperWithResult", + "BlobFileReader::ReadFooter:TamperWithResult", + "BlobFileReader::GetBlob:TamperWithResult"})); + +TEST_P(BlobFileReaderDecodingErrorTest, DecodingError) { + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileReaderDecodingErrorTest_DecodingError"), + 0); + options.enable_blob_files = true; + + ImmutableOptions immutable_options(options); + + constexpr uint32_t column_family_id = 1; + constexpr bool has_ttl = false; + constexpr ExpirationRange expiration_range; + constexpr uint64_t blob_file_number = 1; + constexpr char key[] = "key"; + constexpr char blob[] = "blob"; + + uint64_t blob_offset = 0; + uint64_t blob_size = 0; + + WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range, + expiration_range, blob_file_number, key, blob, kNoCompression, + &blob_offset, &blob_size); + + SyncPoint::GetInstance()->SetCallBack(sync_point_, [](void* arg) { + Slice* const slice = static_cast(arg); + assert(slice); + assert(!slice->empty()); + + slice->remove_prefix(1); + }); + + SyncPoint::GetInstance()->EnableProcessing(); + + constexpr HistogramImpl* blob_file_read_hist = nullptr; + + std::unique_ptr reader; + const ReadOptions read_options; + const Status s = BlobFileReader::Create( + immutable_options, read_options, FileOptions(), column_family_id, + blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader); + + const bool fail_during_create = + sync_point_ != "BlobFileReader::GetBlob:TamperWithResult"; + + if (fail_during_create) { + ASSERT_TRUE(s.IsCorruption()); + } else { + ASSERT_OK(s); + + constexpr FilePrefetchBuffer* prefetch_buffer = nullptr; + constexpr MemoryAllocator* allocator = nullptr; + + std::unique_ptr value; + uint64_t bytes_read = 0; + + ASSERT_TRUE(reader + ->GetBlob(ReadOptions(), key, blob_offset, blob_size, + kNoCompression, prefetch_buffer, allocator, + &value, &bytes_read) + .IsCorruption()); + ASSERT_EQ(value, nullptr); + ASSERT_EQ(bytes_read, 0); + } + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/blob/blob_garbage_meter.cc b/librocksdb-sys/rocksdb/db/blob/blob_garbage_meter.cc new file mode 100644 index 0000000..d328d7f --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_garbage_meter.cc @@ -0,0 +1,100 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_garbage_meter.h" + +#include "db/blob/blob_index.h" +#include "db/blob/blob_log_format.h" +#include "db/dbformat.h" + +namespace ROCKSDB_NAMESPACE { + +Status BlobGarbageMeter::ProcessInFlow(const Slice& key, const Slice& value) { + uint64_t blob_file_number = kInvalidBlobFileNumber; + uint64_t bytes = 0; + + const Status s = Parse(key, value, &blob_file_number, &bytes); + if (!s.ok()) { + return s; + } + + if (blob_file_number == kInvalidBlobFileNumber) { + return Status::OK(); + } + + flows_[blob_file_number].AddInFlow(bytes); + + return Status::OK(); +} + +Status BlobGarbageMeter::ProcessOutFlow(const Slice& key, const Slice& value) { + uint64_t blob_file_number = kInvalidBlobFileNumber; + uint64_t bytes = 0; + + const Status s = Parse(key, value, &blob_file_number, &bytes); + if (!s.ok()) { + return s; + } + + if (blob_file_number == kInvalidBlobFileNumber) { + return Status::OK(); + } + + // Note: in order to measure the amount of additional garbage, we only need to + // track the outflow for preexisting files, i.e. those that also had inflow. + // (Newly written files would only have outflow.) + auto it = flows_.find(blob_file_number); + if (it == flows_.end()) { + return Status::OK(); + } + + it->second.AddOutFlow(bytes); + + return Status::OK(); +} + +Status BlobGarbageMeter::Parse(const Slice& key, const Slice& value, + uint64_t* blob_file_number, uint64_t* bytes) { + assert(blob_file_number); + assert(*blob_file_number == kInvalidBlobFileNumber); + assert(bytes); + assert(*bytes == 0); + + ParsedInternalKey ikey; + + { + constexpr bool log_err_key = false; + const Status s = ParseInternalKey(key, &ikey, log_err_key); + if (!s.ok()) { + return s; + } + } + + if (ikey.type != kTypeBlobIndex) { + return Status::OK(); + } + + BlobIndex blob_index; + + { + const Status s = blob_index.DecodeFrom(value); + if (!s.ok()) { + return s; + } + } + + if (blob_index.IsInlined() || blob_index.HasTTL()) { + return Status::Corruption("Unexpected TTL/inlined blob index"); + } + + *blob_file_number = blob_index.file_number(); + *bytes = + blob_index.size() + + BlobLogRecord::CalculateAdjustmentForRecordHeader(ikey.user_key.size()); + + return Status::OK(); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_garbage_meter.h b/librocksdb-sys/rocksdb/db/blob/blob_garbage_meter.h new file mode 100644 index 0000000..a6c04b0 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_garbage_meter.h @@ -0,0 +1,102 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include + +#include "db/blob/blob_constants.h" +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +class Slice; + +// A class that can be used to compute the amount of additional garbage +// generated by a compaction. It parses the keys and blob references in the +// input and output of a compaction, and aggregates the "inflow" and "outflow" +// on a per-blob file basis. The amount of additional garbage for any given blob +// file can then be computed by subtracting the outflow from the inflow. +class BlobGarbageMeter { + public: + // A class to store the number and total size of blobs on a per-blob file + // basis. + class BlobStats { + public: + void Add(uint64_t bytes) { + ++count_; + bytes_ += bytes; + } + void Add(uint64_t count, uint64_t bytes) { + count_ += count; + bytes_ += bytes; + } + + uint64_t GetCount() const { return count_; } + uint64_t GetBytes() const { return bytes_; } + + private: + uint64_t count_ = 0; + uint64_t bytes_ = 0; + }; + + // A class to keep track of the "inflow" and the "outflow" and to compute the + // amount of additional garbage for a given blob file. + class BlobInOutFlow { + public: + void AddInFlow(uint64_t bytes) { + in_flow_.Add(bytes); + assert(IsValid()); + } + void AddOutFlow(uint64_t bytes) { + out_flow_.Add(bytes); + assert(IsValid()); + } + + const BlobStats& GetInFlow() const { return in_flow_; } + const BlobStats& GetOutFlow() const { return out_flow_; } + + bool IsValid() const { + return in_flow_.GetCount() >= out_flow_.GetCount() && + in_flow_.GetBytes() >= out_flow_.GetBytes(); + } + bool HasGarbage() const { + assert(IsValid()); + return in_flow_.GetCount() > out_flow_.GetCount(); + } + uint64_t GetGarbageCount() const { + assert(IsValid()); + assert(HasGarbage()); + return in_flow_.GetCount() - out_flow_.GetCount(); + } + uint64_t GetGarbageBytes() const { + assert(IsValid()); + assert(HasGarbage()); + return in_flow_.GetBytes() - out_flow_.GetBytes(); + } + + private: + BlobStats in_flow_; + BlobStats out_flow_; + }; + + Status ProcessInFlow(const Slice& key, const Slice& value); + Status ProcessOutFlow(const Slice& key, const Slice& value); + + const std::unordered_map& flows() const { + return flows_; + } + + private: + static Status Parse(const Slice& key, const Slice& value, + uint64_t* blob_file_number, uint64_t* bytes); + + std::unordered_map flows_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_garbage_meter_test.cc b/librocksdb-sys/rocksdb/db/blob/blob_garbage_meter_test.cc new file mode 100644 index 0000000..ba53f06 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_garbage_meter_test.cc @@ -0,0 +1,197 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_garbage_meter.h" + +#include +#include + +#include "db/blob/blob_index.h" +#include "db/blob/blob_log_format.h" +#include "db/dbformat.h" +#include "test_util/testharness.h" + +namespace ROCKSDB_NAMESPACE { + +TEST(BlobGarbageMeterTest, MeasureGarbage) { + BlobGarbageMeter blob_garbage_meter; + + struct BlobDescriptor { + std::string user_key; + uint64_t blob_file_number; + uint64_t offset; + uint64_t size; + CompressionType compression_type; + bool has_in_flow; + bool has_out_flow; + + uint64_t GetExpectedBytes() const { + return size + + BlobLogRecord::CalculateAdjustmentForRecordHeader(user_key.size()); + } + }; + + // Note: blob file 4 has the same inflow and outflow and hence no additional + // garbage. Blob file 5 has less outflow than inflow and thus it does have + // additional garbage. Blob file 6 is a newly written file (i.e. no inflow, + // only outflow) and is thus not tracked by the meter. + std::vector blobs{ + {"key", 4, 1234, 555, kLZ4Compression, true, true}, + {"other_key", 4, 6789, 101010, kLZ4Compression, true, true}, + {"yet_another_key", 5, 22222, 3456, kLZ4Compression, true, true}, + {"foo_key", 5, 77777, 8888, kLZ4Compression, true, true}, + {"bar_key", 5, 999999, 1212, kLZ4Compression, true, false}, + {"baz_key", 5, 1234567, 890, kLZ4Compression, true, false}, + {"new_key", 6, 7777, 9999, kNoCompression, false, true}}; + + for (const auto& blob : blobs) { + constexpr SequenceNumber seq = 123; + const InternalKey key(blob.user_key, seq, kTypeBlobIndex); + const Slice key_slice = key.Encode(); + + std::string value; + BlobIndex::EncodeBlob(&value, blob.blob_file_number, blob.offset, blob.size, + blob.compression_type); + const Slice value_slice(value); + + if (blob.has_in_flow) { + ASSERT_OK(blob_garbage_meter.ProcessInFlow(key_slice, value_slice)); + } + if (blob.has_out_flow) { + ASSERT_OK(blob_garbage_meter.ProcessOutFlow(key_slice, value_slice)); + } + } + + const auto& flows = blob_garbage_meter.flows(); + ASSERT_EQ(flows.size(), 2); + + { + const auto it = flows.find(4); + ASSERT_NE(it, flows.end()); + + const auto& flow = it->second; + + constexpr uint64_t expected_count = 2; + const uint64_t expected_bytes = + blobs[0].GetExpectedBytes() + blobs[1].GetExpectedBytes(); + + const auto& in = flow.GetInFlow(); + ASSERT_EQ(in.GetCount(), expected_count); + ASSERT_EQ(in.GetBytes(), expected_bytes); + + const auto& out = flow.GetOutFlow(); + ASSERT_EQ(out.GetCount(), expected_count); + ASSERT_EQ(out.GetBytes(), expected_bytes); + + ASSERT_TRUE(flow.IsValid()); + ASSERT_FALSE(flow.HasGarbage()); + } + + { + const auto it = flows.find(5); + ASSERT_NE(it, flows.end()); + + const auto& flow = it->second; + + const auto& in = flow.GetInFlow(); + + constexpr uint64_t expected_in_count = 4; + const uint64_t expected_in_bytes = + blobs[2].GetExpectedBytes() + blobs[3].GetExpectedBytes() + + blobs[4].GetExpectedBytes() + blobs[5].GetExpectedBytes(); + + ASSERT_EQ(in.GetCount(), expected_in_count); + ASSERT_EQ(in.GetBytes(), expected_in_bytes); + + const auto& out = flow.GetOutFlow(); + + constexpr uint64_t expected_out_count = 2; + const uint64_t expected_out_bytes = + blobs[2].GetExpectedBytes() + blobs[3].GetExpectedBytes(); + + ASSERT_EQ(out.GetCount(), expected_out_count); + ASSERT_EQ(out.GetBytes(), expected_out_bytes); + + ASSERT_TRUE(flow.IsValid()); + ASSERT_TRUE(flow.HasGarbage()); + ASSERT_EQ(flow.GetGarbageCount(), expected_in_count - expected_out_count); + ASSERT_EQ(flow.GetGarbageBytes(), expected_in_bytes - expected_out_bytes); + } +} + +TEST(BlobGarbageMeterTest, PlainValue) { + constexpr char user_key[] = "user_key"; + constexpr SequenceNumber seq = 123; + + const InternalKey key(user_key, seq, kTypeValue); + const Slice key_slice = key.Encode(); + + constexpr char value[] = "value"; + const Slice value_slice(value); + + BlobGarbageMeter blob_garbage_meter; + + ASSERT_OK(blob_garbage_meter.ProcessInFlow(key_slice, value_slice)); + ASSERT_OK(blob_garbage_meter.ProcessOutFlow(key_slice, value_slice)); + ASSERT_TRUE(blob_garbage_meter.flows().empty()); +} + +TEST(BlobGarbageMeterTest, CorruptInternalKey) { + constexpr char corrupt_key[] = "i_am_corrupt"; + const Slice key_slice(corrupt_key); + + constexpr char value[] = "value"; + const Slice value_slice(value); + + BlobGarbageMeter blob_garbage_meter; + + ASSERT_NOK(blob_garbage_meter.ProcessInFlow(key_slice, value_slice)); + ASSERT_NOK(blob_garbage_meter.ProcessOutFlow(key_slice, value_slice)); +} + +TEST(BlobGarbageMeterTest, CorruptBlobIndex) { + constexpr char user_key[] = "user_key"; + constexpr SequenceNumber seq = 123; + + const InternalKey key(user_key, seq, kTypeBlobIndex); + const Slice key_slice = key.Encode(); + + constexpr char value[] = "i_am_not_a_blob_index"; + const Slice value_slice(value); + + BlobGarbageMeter blob_garbage_meter; + + ASSERT_NOK(blob_garbage_meter.ProcessInFlow(key_slice, value_slice)); + ASSERT_NOK(blob_garbage_meter.ProcessOutFlow(key_slice, value_slice)); +} + +TEST(BlobGarbageMeterTest, InlinedTTLBlobIndex) { + constexpr char user_key[] = "user_key"; + constexpr SequenceNumber seq = 123; + + const InternalKey key(user_key, seq, kTypeBlobIndex); + const Slice key_slice = key.Encode(); + + constexpr uint64_t expiration = 1234567890; + constexpr char inlined_value[] = "inlined"; + + std::string value; + BlobIndex::EncodeInlinedTTL(&value, expiration, inlined_value); + + const Slice value_slice(value); + + BlobGarbageMeter blob_garbage_meter; + + ASSERT_NOK(blob_garbage_meter.ProcessInFlow(key_slice, value_slice)); + ASSERT_NOK(blob_garbage_meter.ProcessOutFlow(key_slice, value_slice)); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/blob/blob_index.h b/librocksdb-sys/rocksdb/db/blob/blob_index.h new file mode 100644 index 0000000..e9944d7 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_index.h @@ -0,0 +1,187 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include +#include + +#include "rocksdb/compression_type.h" +#include "util/coding.h" +#include "util/compression.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +// BlobIndex is a pointer to the blob and metadata of the blob. The index is +// stored in base DB as ValueType::kTypeBlobIndex. +// There are three types of blob index: +// +// kInlinedTTL: +// +------+------------+---------------+ +// | type | expiration | value | +// +------+------------+---------------+ +// | char | varint64 | variable size | +// +------+------------+---------------+ +// +// kBlob: +// +------+-------------+----------+----------+-------------+ +// | type | file number | offset | size | compression | +// +------+-------------+----------+----------+-------------+ +// | char | varint64 | varint64 | varint64 | char | +// +------+-------------+----------+----------+-------------+ +// +// kBlobTTL: +// +------+------------+-------------+----------+----------+-------------+ +// | type | expiration | file number | offset | size | compression | +// +------+------------+-------------+----------+----------+-------------+ +// | char | varint64 | varint64 | varint64 | varint64 | char | +// +------+------------+-------------+----------+----------+-------------+ +// +// There isn't a kInlined (without TTL) type since we can store it as a plain +// value (i.e. ValueType::kTypeValue). +class BlobIndex { + public: + enum class Type : unsigned char { + kInlinedTTL = 0, + kBlob = 1, + kBlobTTL = 2, + kUnknown = 3, + }; + + BlobIndex() : type_(Type::kUnknown) {} + + BlobIndex(const BlobIndex&) = default; + BlobIndex& operator=(const BlobIndex&) = default; + + bool IsInlined() const { return type_ == Type::kInlinedTTL; } + + bool HasTTL() const { + return type_ == Type::kInlinedTTL || type_ == Type::kBlobTTL; + } + + uint64_t expiration() const { + assert(HasTTL()); + return expiration_; + } + + const Slice& value() const { + assert(IsInlined()); + return value_; + } + + uint64_t file_number() const { + assert(!IsInlined()); + return file_number_; + } + + uint64_t offset() const { + assert(!IsInlined()); + return offset_; + } + + uint64_t size() const { + assert(!IsInlined()); + return size_; + } + + CompressionType compression() const { + assert(!IsInlined()); + return compression_; + } + + Status DecodeFrom(Slice slice) { + const char* kErrorMessage = "Error while decoding blob index"; + assert(slice.size() > 0); + type_ = static_cast(*slice.data()); + if (type_ >= Type::kUnknown) { + return Status::Corruption(kErrorMessage, + "Unknown blob index type: " + + std::to_string(static_cast(type_))); + } + slice = Slice(slice.data() + 1, slice.size() - 1); + if (HasTTL()) { + if (!GetVarint64(&slice, &expiration_)) { + return Status::Corruption(kErrorMessage, "Corrupted expiration"); + } + } + if (IsInlined()) { + value_ = slice; + } else { + if (GetVarint64(&slice, &file_number_) && GetVarint64(&slice, &offset_) && + GetVarint64(&slice, &size_) && slice.size() == 1) { + compression_ = static_cast(*slice.data()); + } else { + return Status::Corruption(kErrorMessage, "Corrupted blob offset"); + } + } + return Status::OK(); + } + + std::string DebugString(bool output_hex) const { + std::ostringstream oss; + + if (IsInlined()) { + oss << "[inlined blob] value:" << value_.ToString(output_hex); + } else { + oss << "[blob ref] file:" << file_number_ << " offset:" << offset_ + << " size:" << size_ + << " compression: " << CompressionTypeToString(compression_); + } + + if (HasTTL()) { + oss << " exp:" << expiration_; + } + + return oss.str(); + } + + static void EncodeInlinedTTL(std::string* dst, uint64_t expiration, + const Slice& value) { + assert(dst != nullptr); + dst->clear(); + dst->reserve(1 + kMaxVarint64Length + value.size()); + dst->push_back(static_cast(Type::kInlinedTTL)); + PutVarint64(dst, expiration); + dst->append(value.data(), value.size()); + } + + static void EncodeBlob(std::string* dst, uint64_t file_number, + uint64_t offset, uint64_t size, + CompressionType compression) { + assert(dst != nullptr); + dst->clear(); + dst->reserve(kMaxVarint64Length * 3 + 2); + dst->push_back(static_cast(Type::kBlob)); + PutVarint64(dst, file_number); + PutVarint64(dst, offset); + PutVarint64(dst, size); + dst->push_back(static_cast(compression)); + } + + static void EncodeBlobTTL(std::string* dst, uint64_t expiration, + uint64_t file_number, uint64_t offset, + uint64_t size, CompressionType compression) { + assert(dst != nullptr); + dst->clear(); + dst->reserve(kMaxVarint64Length * 4 + 2); + dst->push_back(static_cast(Type::kBlobTTL)); + PutVarint64(dst, expiration); + PutVarint64(dst, file_number); + PutVarint64(dst, offset); + PutVarint64(dst, size); + dst->push_back(static_cast(compression)); + } + + private: + Type type_ = Type::kUnknown; + uint64_t expiration_ = 0; + Slice value_; + uint64_t file_number_ = 0; + uint64_t offset_ = 0; + uint64_t size_ = 0; + CompressionType compression_ = kNoCompression; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_log_format.cc b/librocksdb-sys/rocksdb/db/blob/blob_log_format.cc new file mode 100644 index 0000000..8e26281 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_log_format.cc @@ -0,0 +1,143 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#include "db/blob/blob_log_format.h" + +#include "util/coding.h" +#include "util/crc32c.h" + +namespace ROCKSDB_NAMESPACE { + +void BlobLogHeader::EncodeTo(std::string* dst) { + assert(dst != nullptr); + dst->clear(); + dst->reserve(BlobLogHeader::kSize); + PutFixed32(dst, kMagicNumber); + PutFixed32(dst, version); + PutFixed32(dst, column_family_id); + unsigned char flags = (has_ttl ? 1 : 0); + dst->push_back(flags); + dst->push_back(compression); + PutFixed64(dst, expiration_range.first); + PutFixed64(dst, expiration_range.second); +} + +Status BlobLogHeader::DecodeFrom(Slice src) { + const char* kErrorMessage = "Error while decoding blob log header"; + if (src.size() != BlobLogHeader::kSize) { + return Status::Corruption(kErrorMessage, + "Unexpected blob file header size"); + } + uint32_t magic_number; + unsigned char flags; + if (!GetFixed32(&src, &magic_number) || !GetFixed32(&src, &version) || + !GetFixed32(&src, &column_family_id)) { + return Status::Corruption( + kErrorMessage, + "Error decoding magic number, version and column family id"); + } + if (magic_number != kMagicNumber) { + return Status::Corruption(kErrorMessage, "Magic number mismatch"); + } + if (version != kVersion1) { + return Status::Corruption(kErrorMessage, "Unknown header version"); + } + flags = src.data()[0]; + compression = static_cast(src.data()[1]); + has_ttl = (flags & 1) == 1; + src.remove_prefix(2); + if (!GetFixed64(&src, &expiration_range.first) || + !GetFixed64(&src, &expiration_range.second)) { + return Status::Corruption(kErrorMessage, "Error decoding expiration range"); + } + return Status::OK(); +} + +void BlobLogFooter::EncodeTo(std::string* dst) { + assert(dst != nullptr); + dst->clear(); + dst->reserve(BlobLogFooter::kSize); + PutFixed32(dst, kMagicNumber); + PutFixed64(dst, blob_count); + PutFixed64(dst, expiration_range.first); + PutFixed64(dst, expiration_range.second); + crc = crc32c::Value(dst->c_str(), dst->size()); + crc = crc32c::Mask(crc); + PutFixed32(dst, crc); +} + +Status BlobLogFooter::DecodeFrom(Slice src) { + const char* kErrorMessage = "Error while decoding blob log footer"; + if (src.size() != BlobLogFooter::kSize) { + return Status::Corruption(kErrorMessage, + "Unexpected blob file footer size"); + } + uint32_t src_crc = 0; + src_crc = crc32c::Value(src.data(), BlobLogFooter::kSize - sizeof(uint32_t)); + src_crc = crc32c::Mask(src_crc); + uint32_t magic_number = 0; + if (!GetFixed32(&src, &magic_number) || !GetFixed64(&src, &blob_count) || + !GetFixed64(&src, &expiration_range.first) || + !GetFixed64(&src, &expiration_range.second) || !GetFixed32(&src, &crc)) { + return Status::Corruption(kErrorMessage, "Error decoding content"); + } + if (magic_number != kMagicNumber) { + return Status::Corruption(kErrorMessage, "Magic number mismatch"); + } + if (src_crc != crc) { + return Status::Corruption(kErrorMessage, "CRC mismatch"); + } + return Status::OK(); +} + +void BlobLogRecord::EncodeHeaderTo(std::string* dst) { + assert(dst != nullptr); + dst->clear(); + dst->reserve(BlobLogRecord::kHeaderSize + key.size() + value.size()); + PutFixed64(dst, key.size()); + PutFixed64(dst, value.size()); + PutFixed64(dst, expiration); + header_crc = crc32c::Value(dst->c_str(), dst->size()); + header_crc = crc32c::Mask(header_crc); + PutFixed32(dst, header_crc); + blob_crc = crc32c::Value(key.data(), key.size()); + blob_crc = crc32c::Extend(blob_crc, value.data(), value.size()); + blob_crc = crc32c::Mask(blob_crc); + PutFixed32(dst, blob_crc); +} + +Status BlobLogRecord::DecodeHeaderFrom(Slice src) { + const char* kErrorMessage = "Error while decoding blob record"; + if (src.size() != BlobLogRecord::kHeaderSize) { + return Status::Corruption(kErrorMessage, + "Unexpected blob record header size"); + } + uint32_t src_crc = 0; + src_crc = crc32c::Value(src.data(), BlobLogRecord::kHeaderSize - 8); + src_crc = crc32c::Mask(src_crc); + if (!GetFixed64(&src, &key_size) || !GetFixed64(&src, &value_size) || + !GetFixed64(&src, &expiration) || !GetFixed32(&src, &header_crc) || + !GetFixed32(&src, &blob_crc)) { + return Status::Corruption(kErrorMessage, "Error decoding content"); + } + if (src_crc != header_crc) { + return Status::Corruption(kErrorMessage, "Header CRC mismatch"); + } + return Status::OK(); +} + +Status BlobLogRecord::CheckBlobCRC() const { + uint32_t expected_crc = 0; + expected_crc = crc32c::Value(key.data(), key.size()); + expected_crc = crc32c::Extend(expected_crc, value.data(), value.size()); + expected_crc = crc32c::Mask(expected_crc); + if (expected_crc != blob_crc) { + return Status::Corruption("Blob CRC mismatch"); + } + return Status::OK(); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_log_format.h b/librocksdb-sys/rocksdb/db/blob/blob_log_format.h new file mode 100644 index 0000000..607db23 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_log_format.h @@ -0,0 +1,164 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Log format information shared by reader and writer. + +#pragma once + +#include +#include + +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "rocksdb/types.h" + +namespace ROCKSDB_NAMESPACE { + +constexpr uint32_t kMagicNumber = 2395959; // 0x00248f37 +constexpr uint32_t kVersion1 = 1; + +using ExpirationRange = std::pair; + +// clang-format off + +// Format of blob log file header (30 bytes): +// +// +--------------+---------+---------+-------+-------------+-------------------+ +// | magic number | version | cf id | flags | compression | expiration range | +// +--------------+---------+---------+-------+-------------+-------------------+ +// | Fixed32 | Fixed32 | Fixed32 | char | char | Fixed64 Fixed64 | +// +--------------+---------+---------+-------+-------------+-------------------+ +// +// List of flags: +// has_ttl: Whether the file contain TTL data. +// +// Expiration range in the header is a rough range based on +// blob_db_options.ttl_range_secs. + +// clang-format on + +struct BlobLogHeader { + static constexpr size_t kSize = 30; + + BlobLogHeader() = default; + BlobLogHeader(uint32_t _column_family_id, CompressionType _compression, + bool _has_ttl, const ExpirationRange& _expiration_range) + : column_family_id(_column_family_id), + compression(_compression), + has_ttl(_has_ttl), + expiration_range(_expiration_range) {} + + uint32_t version = kVersion1; + uint32_t column_family_id = 0; + CompressionType compression = kNoCompression; + bool has_ttl = false; + ExpirationRange expiration_range; + + void EncodeTo(std::string* dst); + + Status DecodeFrom(Slice slice); +}; + +// clang-format off + +// Format of blob log file footer (32 bytes): +// +// +--------------+------------+-------------------+------------+ +// | magic number | blob count | expiration range | footer CRC | +// +--------------+------------+-------------------+------------+ +// | Fixed32 | Fixed64 | Fixed64 + Fixed64 | Fixed32 | +// +--------------+------------+-------------------+------------+ +// +// The footer will be presented only when the blob file is properly closed. +// +// Unlike the same field in file header, expiration range in the footer is the +// range of smallest and largest expiration of the data in this file. + +// clang-format on + +struct BlobLogFooter { + static constexpr size_t kSize = 32; + + uint64_t blob_count = 0; + ExpirationRange expiration_range = std::make_pair(0, 0); + uint32_t crc = 0; + + void EncodeTo(std::string* dst); + + Status DecodeFrom(Slice slice); +}; + +// clang-format off + +// Blob record format (32 bytes header + key + value): +// +// +------------+--------------+------------+------------+----------+---------+-----------+ +// | key length | value length | expiration | header CRC | blob CRC | key | value | +// +------------+--------------+------------+------------+----------+---------+-----------+ +// | Fixed64 | Fixed64 | Fixed64 | Fixed32 | Fixed32 | key len | value len | +// +------------+--------------+------------+------------+----------+---------+-----------+ +// +// If file has has_ttl = false, expiration field is always 0, and the blob +// doesn't has expiration. +// +// Also note that if compression is used, value is compressed value and value +// length is compressed value length. +// +// Header CRC is the checksum of (key_len + val_len + expiration), while +// blob CRC is the checksum of (key + value). +// +// We could use variable length encoding (Varint64) to save more space, but it +// make reader more complicated. + +// clang-format on + +struct BlobLogRecord { + // header include fields up to blob CRC + static constexpr size_t kHeaderSize = 32; + + // Note that the offset field of BlobIndex actually points to the blob value + // as opposed to the start of the blob record. The following method can + // be used to calculate the adjustment needed to read the blob record header. + static constexpr uint64_t CalculateAdjustmentForRecordHeader( + uint64_t key_size) { + return key_size + kHeaderSize; + } + + uint64_t key_size = 0; + uint64_t value_size = 0; + uint64_t expiration = 0; + uint32_t header_crc = 0; + uint32_t blob_crc = 0; + Slice key; + Slice value; + std::unique_ptr key_buf; + std::unique_ptr value_buf; + + uint64_t record_size() const { return kHeaderSize + key_size + value_size; } + + void EncodeHeaderTo(std::string* dst); + + Status DecodeHeaderFrom(Slice src); + + Status CheckBlobCRC() const; +}; + +// Checks whether a blob offset is potentially valid or not. +inline bool IsValidBlobOffset(uint64_t value_offset, uint64_t key_size, + uint64_t value_size, uint64_t file_size) { + if (value_offset < + BlobLogHeader::kSize + BlobLogRecord::kHeaderSize + key_size) { + return false; + } + + if (value_offset + value_size + BlobLogFooter::kSize > file_size) { + return false; + } + + return true; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_log_sequential_reader.cc b/librocksdb-sys/rocksdb/db/blob/blob_log_sequential_reader.cc new file mode 100644 index 0000000..2ed4306 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_log_sequential_reader.cc @@ -0,0 +1,134 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#include "db/blob/blob_log_sequential_reader.h" + +#include "file/random_access_file_reader.h" +#include "monitoring/statistics_impl.h" +#include "util/stop_watch.h" + +namespace ROCKSDB_NAMESPACE { + +BlobLogSequentialReader::BlobLogSequentialReader( + std::unique_ptr&& file_reader, SystemClock* clock, + Statistics* statistics) + : file_(std::move(file_reader)), + clock_(clock), + statistics_(statistics), + next_byte_(0) {} + +BlobLogSequentialReader::~BlobLogSequentialReader() = default; + +Status BlobLogSequentialReader::ReadSlice(uint64_t size, Slice* slice, + char* buf) { + assert(slice); + assert(file_); + + StopWatch read_sw(clock_, statistics_, BLOB_DB_BLOB_FILE_READ_MICROS); + // TODO: rate limit `BlobLogSequentialReader` reads (it appears unused?) + Status s = + file_->Read(IOOptions(), next_byte_, static_cast(size), slice, + buf, nullptr, Env::IO_TOTAL /* rate_limiter_priority */); + next_byte_ += size; + if (!s.ok()) { + return s; + } + RecordTick(statistics_, BLOB_DB_BLOB_FILE_BYTES_READ, slice->size()); + if (slice->size() != size) { + return Status::Corruption("EOF reached while reading record"); + } + return s; +} + +Status BlobLogSequentialReader::ReadHeader(BlobLogHeader* header) { + assert(header); + assert(next_byte_ == 0); + + static_assert(BlobLogHeader::kSize <= sizeof(header_buf_), + "Buffer is smaller than BlobLogHeader::kSize"); + + Status s = ReadSlice(BlobLogHeader::kSize, &buffer_, header_buf_); + if (!s.ok()) { + return s; + } + + if (buffer_.size() != BlobLogHeader::kSize) { + return Status::Corruption("EOF reached before file header"); + } + + return header->DecodeFrom(buffer_); +} + +Status BlobLogSequentialReader::ReadRecord(BlobLogRecord* record, + ReadLevel level, + uint64_t* blob_offset) { + assert(record); + static_assert(BlobLogRecord::kHeaderSize <= sizeof(header_buf_), + "Buffer is smaller than BlobLogRecord::kHeaderSize"); + + Status s = ReadSlice(BlobLogRecord::kHeaderSize, &buffer_, header_buf_); + if (!s.ok()) { + return s; + } + if (buffer_.size() != BlobLogRecord::kHeaderSize) { + return Status::Corruption("EOF reached before record header"); + } + + s = record->DecodeHeaderFrom(buffer_); + if (!s.ok()) { + return s; + } + + uint64_t kb_size = record->key_size + record->value_size; + if (blob_offset != nullptr) { + *blob_offset = next_byte_ + record->key_size; + } + + switch (level) { + case kReadHeader: + next_byte_ += kb_size; + break; + + case kReadHeaderKey: + record->key_buf.reset(new char[record->key_size]); + s = ReadSlice(record->key_size, &record->key, record->key_buf.get()); + next_byte_ += record->value_size; + break; + + case kReadHeaderKeyBlob: + record->key_buf.reset(new char[record->key_size]); + s = ReadSlice(record->key_size, &record->key, record->key_buf.get()); + if (s.ok()) { + record->value_buf.reset(new char[record->value_size]); + s = ReadSlice(record->value_size, &record->value, + record->value_buf.get()); + } + if (s.ok()) { + s = record->CheckBlobCRC(); + } + break; + } + return s; +} + +Status BlobLogSequentialReader::ReadFooter(BlobLogFooter* footer) { + assert(footer); + static_assert(BlobLogFooter::kSize <= sizeof(header_buf_), + "Buffer is smaller than BlobLogFooter::kSize"); + + Status s = ReadSlice(BlobLogFooter::kSize, &buffer_, header_buf_); + if (!s.ok()) { + return s; + } + + if (buffer_.size() != BlobLogFooter::kSize) { + return Status::Corruption("EOF reached before file footer"); + } + + return footer->DecodeFrom(buffer_); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_log_sequential_reader.h b/librocksdb-sys/rocksdb/db/blob/blob_log_sequential_reader.h new file mode 100644 index 0000000..98afa85 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_log_sequential_reader.h @@ -0,0 +1,83 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#pragma once + +#include + +#include "db/blob/blob_log_format.h" +#include "rocksdb/slice.h" + +#define MAX_HEADER_SIZE(a, b, c) (a > b ? (a > c ? a : c) : (b > c ? b : c)) + +namespace ROCKSDB_NAMESPACE { + +class RandomAccessFileReader; +class Env; +class Statistics; +class Status; +class SystemClock; + +/** + * BlobLogSequentialReader is a general purpose log stream reader + * implementation. The actual job of reading from the device is implemented by + * the RandomAccessFileReader interface. + * + * Please see BlobLogWriter for details on the file and record layout. + */ + +class BlobLogSequentialReader { + public: + enum ReadLevel { + kReadHeader, + kReadHeaderKey, + kReadHeaderKeyBlob, + }; + + // Create a reader that will return log records from "*file_reader". + BlobLogSequentialReader(std::unique_ptr&& file_reader, + SystemClock* clock, Statistics* statistics); + + // No copying allowed + BlobLogSequentialReader(const BlobLogSequentialReader&) = delete; + BlobLogSequentialReader& operator=(const BlobLogSequentialReader&) = delete; + + ~BlobLogSequentialReader(); + + Status ReadHeader(BlobLogHeader* header); + + // Read the next record into *record. Returns true if read + // successfully, false if we hit end of the input. The contents filled in + // *record will only be valid until the next mutating operation on this + // reader. + // If blob_offset is non-null, return offset of the blob through it. + Status ReadRecord(BlobLogRecord* record, ReadLevel level = kReadHeader, + uint64_t* blob_offset = nullptr); + + Status ReadFooter(BlobLogFooter* footer); + + void ResetNextByte() { next_byte_ = 0; } + + uint64_t GetNextByte() const { return next_byte_; } + + private: + Status ReadSlice(uint64_t size, Slice* slice, char* buf); + + const std::unique_ptr file_; + SystemClock* clock_; + + Statistics* statistics_; + + Slice buffer_; + char header_buf_[MAX_HEADER_SIZE(BlobLogHeader::kSize, BlobLogFooter::kSize, + BlobLogRecord::kHeaderSize)]; + + // which byte to read next + uint64_t next_byte_; +}; + +} // namespace ROCKSDB_NAMESPACE + +#undef MAX_HEADER_SIZE \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/db/blob/blob_log_writer.cc b/librocksdb-sys/rocksdb/db/blob/blob_log_writer.cc new file mode 100644 index 0000000..bf5ef27 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_log_writer.cc @@ -0,0 +1,178 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_log_writer.h" + +#include +#include + +#include "db/blob/blob_log_format.h" +#include "file/writable_file_writer.h" +#include "monitoring/statistics_impl.h" +#include "rocksdb/system_clock.h" +#include "test_util/sync_point.h" +#include "util/coding.h" +#include "util/stop_watch.h" + +namespace ROCKSDB_NAMESPACE { + +BlobLogWriter::BlobLogWriter(std::unique_ptr&& dest, + SystemClock* clock, Statistics* statistics, + uint64_t log_number, bool use_fs, bool do_flush, + uint64_t boffset) + : dest_(std::move(dest)), + clock_(clock), + statistics_(statistics), + log_number_(log_number), + block_offset_(boffset), + use_fsync_(use_fs), + do_flush_(do_flush), + last_elem_type_(kEtNone) {} + +BlobLogWriter::~BlobLogWriter() = default; + +Status BlobLogWriter::Sync() { + TEST_SYNC_POINT("BlobLogWriter::Sync"); + + StopWatch sync_sw(clock_, statistics_, BLOB_DB_BLOB_FILE_SYNC_MICROS); + Status s = dest_->Sync(use_fsync_); + RecordTick(statistics_, BLOB_DB_BLOB_FILE_SYNCED); + return s; +} + +Status BlobLogWriter::WriteHeader(BlobLogHeader& header) { + assert(block_offset_ == 0); + assert(last_elem_type_ == kEtNone); + std::string str; + header.EncodeTo(&str); + + Status s = dest_->Append(Slice(str)); + if (s.ok()) { + block_offset_ += str.size(); + if (do_flush_) { + s = dest_->Flush(); + } + } + last_elem_type_ = kEtFileHdr; + RecordTick(statistics_, BLOB_DB_BLOB_FILE_BYTES_WRITTEN, + BlobLogHeader::kSize); + return s; +} + +Status BlobLogWriter::AppendFooter(BlobLogFooter& footer, + std::string* checksum_method, + std::string* checksum_value) { + assert(block_offset_ != 0); + assert(last_elem_type_ == kEtFileHdr || last_elem_type_ == kEtRecord); + + std::string str; + footer.EncodeTo(&str); + + Status s; + if (dest_->seen_error()) { + s.PermitUncheckedError(); + return Status::IOError("Seen Error. Skip closing."); + } else { + s = dest_->Append(Slice(str)); + if (s.ok()) { + block_offset_ += str.size(); + + s = Sync(); + + if (s.ok()) { + s = dest_->Close(); + + if (s.ok()) { + assert(!!checksum_method == !!checksum_value); + + if (checksum_method) { + assert(checksum_method->empty()); + + std::string method = dest_->GetFileChecksumFuncName(); + if (method != kUnknownFileChecksumFuncName) { + *checksum_method = std::move(method); + } + } + if (checksum_value) { + assert(checksum_value->empty()); + + std::string value = dest_->GetFileChecksum(); + if (value != kUnknownFileChecksum) { + *checksum_value = std::move(value); + } + } + } + } + } + + dest_.reset(); + } + + last_elem_type_ = kEtFileFooter; + RecordTick(statistics_, BLOB_DB_BLOB_FILE_BYTES_WRITTEN, + BlobLogFooter::kSize); + return s; +} + +Status BlobLogWriter::AddRecord(const Slice& key, const Slice& val, + uint64_t expiration, uint64_t* key_offset, + uint64_t* blob_offset) { + assert(block_offset_ != 0); + assert(last_elem_type_ == kEtFileHdr || last_elem_type_ == kEtRecord); + + std::string buf; + ConstructBlobHeader(&buf, key, val, expiration); + + Status s = EmitPhysicalRecord(buf, key, val, key_offset, blob_offset); + return s; +} + +Status BlobLogWriter::AddRecord(const Slice& key, const Slice& val, + uint64_t* key_offset, uint64_t* blob_offset) { + assert(block_offset_ != 0); + assert(last_elem_type_ == kEtFileHdr || last_elem_type_ == kEtRecord); + + std::string buf; + ConstructBlobHeader(&buf, key, val, 0); + + Status s = EmitPhysicalRecord(buf, key, val, key_offset, blob_offset); + return s; +} + +void BlobLogWriter::ConstructBlobHeader(std::string* buf, const Slice& key, + const Slice& val, uint64_t expiration) { + BlobLogRecord record; + record.key = key; + record.value = val; + record.expiration = expiration; + record.EncodeHeaderTo(buf); +} + +Status BlobLogWriter::EmitPhysicalRecord(const std::string& headerbuf, + const Slice& key, const Slice& val, + uint64_t* key_offset, + uint64_t* blob_offset) { + StopWatch write_sw(clock_, statistics_, BLOB_DB_BLOB_FILE_WRITE_MICROS); + Status s = dest_->Append(Slice(headerbuf)); + if (s.ok()) { + s = dest_->Append(key); + } + if (s.ok()) { + s = dest_->Append(val); + } + if (do_flush_ && s.ok()) { + s = dest_->Flush(); + } + + *key_offset = block_offset_ + BlobLogRecord::kHeaderSize; + *blob_offset = *key_offset + key.size(); + block_offset_ = *blob_offset + val.size(); + last_elem_type_ = kEtRecord; + RecordTick(statistics_, BLOB_DB_BLOB_FILE_BYTES_WRITTEN, + BlobLogRecord::kHeaderSize + key.size() + val.size()); + return s; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_log_writer.h b/librocksdb-sys/rocksdb/db/blob/blob_log_writer.h new file mode 100644 index 0000000..c1f9f31 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_log_writer.h @@ -0,0 +1,83 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include +#include +#include + +#include "db/blob/blob_log_format.h" +#include "rocksdb/slice.h" +#include "rocksdb/statistics.h" +#include "rocksdb/status.h" +#include "rocksdb/types.h" + +namespace ROCKSDB_NAMESPACE { + +class WritableFileWriter; +class SystemClock; +/** + * BlobLogWriter is the blob log stream writer. It provides an append-only + * abstraction for writing blob data. + * + * + * Look at blob_db_format.h to see the details of the record formats. + */ + +class BlobLogWriter { + public: + // Create a writer that will append data to "*dest". + // "*dest" must be initially empty. + // "*dest" must remain live while this BlobLogWriter is in use. + BlobLogWriter(std::unique_ptr&& dest, SystemClock* clock, + Statistics* statistics, uint64_t log_number, bool use_fsync, + bool do_flush, uint64_t boffset = 0); + // No copying allowed + BlobLogWriter(const BlobLogWriter&) = delete; + BlobLogWriter& operator=(const BlobLogWriter&) = delete; + + ~BlobLogWriter(); + + static void ConstructBlobHeader(std::string* buf, const Slice& key, + const Slice& val, uint64_t expiration); + + Status AddRecord(const Slice& key, const Slice& val, uint64_t* key_offset, + uint64_t* blob_offset); + + Status AddRecord(const Slice& key, const Slice& val, uint64_t expiration, + uint64_t* key_offset, uint64_t* blob_offset); + + Status EmitPhysicalRecord(const std::string& headerbuf, const Slice& key, + const Slice& val, uint64_t* key_offset, + uint64_t* blob_offset); + + Status AppendFooter(BlobLogFooter& footer, std::string* checksum_method, + std::string* checksum_value); + + Status WriteHeader(BlobLogHeader& header); + + WritableFileWriter* file() { return dest_.get(); } + + const WritableFileWriter* file() const { return dest_.get(); } + + uint64_t get_log_number() const { return log_number_; } + + Status Sync(); + + private: + std::unique_ptr dest_; + SystemClock* clock_; + Statistics* statistics_; + uint64_t log_number_; + uint64_t block_offset_; // Current offset in block + bool use_fsync_; + bool do_flush_; + + public: + enum ElemType { kEtNone, kEtFileHdr, kEtRecord, kEtFileFooter }; + ElemType last_elem_type_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_read_request.h b/librocksdb-sys/rocksdb/db/blob/blob_read_request.h new file mode 100644 index 0000000..f9668ca --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_read_request.h @@ -0,0 +1,58 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "rocksdb/compression_type.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "util/autovector.h" + +namespace ROCKSDB_NAMESPACE { + +// A read Blob request structure for use in BlobSource::MultiGetBlob and +// BlobFileReader::MultiGetBlob. +struct BlobReadRequest { + // User key to lookup the paired blob + const Slice* user_key = nullptr; + + // File offset in bytes + uint64_t offset = 0; + + // Length to read in bytes + size_t len = 0; + + // Blob compression type + CompressionType compression = kNoCompression; + + // Output parameter set by MultiGetBlob() to point to the data buffer, and + // the number of valid bytes + PinnableSlice* result = nullptr; + + // Status of read + Status* status = nullptr; + + BlobReadRequest(const Slice& _user_key, uint64_t _offset, size_t _len, + CompressionType _compression, PinnableSlice* _result, + Status* _status) + : user_key(&_user_key), + offset(_offset), + len(_len), + compression(_compression), + result(_result), + status(_status) {} + + BlobReadRequest() = default; + BlobReadRequest(const BlobReadRequest& other) = default; + BlobReadRequest& operator=(const BlobReadRequest& other) = default; +}; + +using BlobFileReadRequests = + std::tuple>; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_source.cc b/librocksdb-sys/rocksdb/db/blob/blob_source.cc new file mode 100644 index 0000000..b524982 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_source.cc @@ -0,0 +1,459 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_source.h" + +#include +#include + +#include "cache/cache_reservation_manager.h" +#include "cache/charged_cache.h" +#include "db/blob/blob_contents.h" +#include "db/blob/blob_file_reader.h" +#include "db/blob/blob_log_format.h" +#include "monitoring/statistics_impl.h" +#include "options/cf_options.h" +#include "table/get_context.h" +#include "table/multiget_context.h" + +namespace ROCKSDB_NAMESPACE { + +BlobSource::BlobSource(const ImmutableOptions* immutable_options, + const std::string& db_id, + const std::string& db_session_id, + BlobFileCache* blob_file_cache) + : db_id_(db_id), + db_session_id_(db_session_id), + statistics_(immutable_options->statistics.get()), + blob_file_cache_(blob_file_cache), + blob_cache_(immutable_options->blob_cache), + lowest_used_cache_tier_(immutable_options->lowest_used_cache_tier) { + auto bbto = + immutable_options->table_factory->GetOptions(); + if (bbto && + bbto->cache_usage_options.options_overrides.at(CacheEntryRole::kBlobCache) + .charged == CacheEntryRoleOptions::Decision::kEnabled) { + blob_cache_ = SharedCacheInterface{std::make_shared( + immutable_options->blob_cache, bbto->block_cache)}; + } +} + +BlobSource::~BlobSource() = default; + +Status BlobSource::GetBlobFromCache( + const Slice& cache_key, CacheHandleGuard* cached_blob) const { + assert(blob_cache_); + assert(!cache_key.empty()); + assert(cached_blob); + assert(cached_blob->IsEmpty()); + + Cache::Handle* cache_handle = nullptr; + cache_handle = GetEntryFromCache(cache_key); + if (cache_handle != nullptr) { + *cached_blob = + CacheHandleGuard(blob_cache_.get(), cache_handle); + + assert(cached_blob->GetValue()); + + PERF_COUNTER_ADD(blob_cache_hit_count, 1); + RecordTick(statistics_, BLOB_DB_CACHE_HIT); + RecordTick(statistics_, BLOB_DB_CACHE_BYTES_READ, + cached_blob->GetValue()->size()); + + return Status::OK(); + } + + RecordTick(statistics_, BLOB_DB_CACHE_MISS); + + return Status::NotFound("Blob not found in cache"); +} + +Status BlobSource::PutBlobIntoCache( + const Slice& cache_key, std::unique_ptr* blob, + CacheHandleGuard* cached_blob) const { + assert(blob_cache_); + assert(!cache_key.empty()); + assert(blob); + assert(*blob); + assert(cached_blob); + assert(cached_blob->IsEmpty()); + + TypedHandle* cache_handle = nullptr; + const Status s = InsertEntryIntoCache(cache_key, blob->get(), + &cache_handle, Cache::Priority::BOTTOM); + if (s.ok()) { + blob->release(); + + assert(cache_handle != nullptr); + *cached_blob = + CacheHandleGuard(blob_cache_.get(), cache_handle); + + assert(cached_blob->GetValue()); + + RecordTick(statistics_, BLOB_DB_CACHE_ADD); + RecordTick(statistics_, BLOB_DB_CACHE_BYTES_WRITE, + cached_blob->GetValue()->size()); + + } else { + RecordTick(statistics_, BLOB_DB_CACHE_ADD_FAILURES); + } + + return s; +} + +BlobSource::TypedHandle* BlobSource::GetEntryFromCache(const Slice& key) const { + return blob_cache_.LookupFull(key, nullptr /* context */, + Cache::Priority::BOTTOM, statistics_, + lowest_used_cache_tier_); +} + +void BlobSource::PinCachedBlob(CacheHandleGuard* cached_blob, + PinnableSlice* value) { + assert(cached_blob); + assert(cached_blob->GetValue()); + assert(value); + + // To avoid copying the cached blob into the buffer provided by the + // application, we can simply transfer ownership of the cache handle to + // the target PinnableSlice. This has the potential to save a lot of + // CPU, especially with large blob values. + + value->Reset(); + + constexpr Cleanable* cleanable = nullptr; + value->PinSlice(cached_blob->GetValue()->data(), cleanable); + + cached_blob->TransferTo(value); +} + +void BlobSource::PinOwnedBlob(std::unique_ptr* owned_blob, + PinnableSlice* value) { + assert(owned_blob); + assert(*owned_blob); + assert(value); + + BlobContents* const blob = owned_blob->release(); + assert(blob); + + value->Reset(); + value->PinSlice( + blob->data(), + [](void* arg1, void* /* arg2 */) { + delete static_cast(arg1); + }, + blob, nullptr); +} + +Status BlobSource::InsertEntryIntoCache(const Slice& key, BlobContents* value, + TypedHandle** cache_handle, + Cache::Priority priority) const { + return blob_cache_.InsertFull(key, value, value->ApproximateMemoryUsage(), + cache_handle, priority, + lowest_used_cache_tier_); +} + +Status BlobSource::GetBlob(const ReadOptions& read_options, + const Slice& user_key, uint64_t file_number, + uint64_t offset, uint64_t file_size, + uint64_t value_size, + CompressionType compression_type, + FilePrefetchBuffer* prefetch_buffer, + PinnableSlice* value, uint64_t* bytes_read) { + assert(value); + + Status s; + + const CacheKey cache_key = GetCacheKey(file_number, file_size, offset); + + CacheHandleGuard blob_handle; + + // First, try to get the blob from the cache + // + // If blob cache is enabled, we'll try to read from it. + if (blob_cache_) { + Slice key = cache_key.AsSlice(); + s = GetBlobFromCache(key, &blob_handle); + if (s.ok()) { + PinCachedBlob(&blob_handle, value); + + // For consistency, the size of on-disk (possibly compressed) blob record + // is assigned to bytes_read. + uint64_t adjustment = + read_options.verify_checksums + ? BlobLogRecord::CalculateAdjustmentForRecordHeader( + user_key.size()) + : 0; + assert(offset >= adjustment); + + uint64_t record_size = value_size + adjustment; + if (bytes_read) { + *bytes_read = record_size; + } + return s; + } + } + + assert(blob_handle.IsEmpty()); + + const bool no_io = read_options.read_tier == kBlockCacheTier; + if (no_io) { + s = Status::Incomplete("Cannot read blob(s): no disk I/O allowed"); + return s; + } + + // Can't find the blob from the cache. Since I/O is allowed, read from the + // file. + std::unique_ptr blob_contents; + + { + CacheHandleGuard blob_file_reader; + s = blob_file_cache_->GetBlobFileReader(read_options, file_number, + &blob_file_reader); + if (!s.ok()) { + return s; + } + + assert(blob_file_reader.GetValue()); + + if (compression_type != blob_file_reader.GetValue()->GetCompressionType()) { + return Status::Corruption("Compression type mismatch when reading blob"); + } + + MemoryAllocator* const allocator = + (blob_cache_ && read_options.fill_cache) + ? blob_cache_.get()->memory_allocator() + : nullptr; + + uint64_t read_size = 0; + s = blob_file_reader.GetValue()->GetBlob( + read_options, user_key, offset, value_size, compression_type, + prefetch_buffer, allocator, &blob_contents, &read_size); + if (!s.ok()) { + return s; + } + if (bytes_read) { + *bytes_read = read_size; + } + } + + if (blob_cache_ && read_options.fill_cache) { + // If filling cache is allowed and a cache is configured, try to put the + // blob to the cache. + Slice key = cache_key.AsSlice(); + s = PutBlobIntoCache(key, &blob_contents, &blob_handle); + if (!s.ok()) { + return s; + } + + PinCachedBlob(&blob_handle, value); + } else { + PinOwnedBlob(&blob_contents, value); + } + + assert(s.ok()); + return s; +} + +void BlobSource::MultiGetBlob(const ReadOptions& read_options, + autovector& blob_reqs, + uint64_t* bytes_read) { + assert(blob_reqs.size() > 0); + + uint64_t total_bytes_read = 0; + uint64_t bytes_read_in_file = 0; + + for (auto& [file_number, file_size, blob_reqs_in_file] : blob_reqs) { + // sort blob_reqs_in_file by file offset. + std::sort( + blob_reqs_in_file.begin(), blob_reqs_in_file.end(), + [](const BlobReadRequest& lhs, const BlobReadRequest& rhs) -> bool { + return lhs.offset < rhs.offset; + }); + + MultiGetBlobFromOneFile(read_options, file_number, file_size, + blob_reqs_in_file, &bytes_read_in_file); + + total_bytes_read += bytes_read_in_file; + } + + if (bytes_read) { + *bytes_read = total_bytes_read; + } +} + +void BlobSource::MultiGetBlobFromOneFile(const ReadOptions& read_options, + uint64_t file_number, + uint64_t /*file_size*/, + autovector& blob_reqs, + uint64_t* bytes_read) { + const size_t num_blobs = blob_reqs.size(); + assert(num_blobs > 0); + assert(num_blobs <= MultiGetContext::MAX_BATCH_SIZE); + +#ifndef NDEBUG + for (size_t i = 0; i < num_blobs - 1; ++i) { + assert(blob_reqs[i].offset <= blob_reqs[i + 1].offset); + } +#endif // !NDEBUG + + using Mask = uint64_t; + Mask cache_hit_mask = 0; + + uint64_t total_bytes = 0; + const OffsetableCacheKey base_cache_key(db_id_, db_session_id_, file_number); + + if (blob_cache_) { + size_t cached_blob_count = 0; + for (size_t i = 0; i < num_blobs; ++i) { + auto& req = blob_reqs[i]; + + CacheHandleGuard blob_handle; + const CacheKey cache_key = base_cache_key.WithOffset(req.offset); + const Slice key = cache_key.AsSlice(); + + const Status s = GetBlobFromCache(key, &blob_handle); + + if (s.ok()) { + assert(req.status); + *req.status = s; + + PinCachedBlob(&blob_handle, req.result); + + // Update the counter for the number of valid blobs read from the cache. + ++cached_blob_count; + + // For consistency, the size of each on-disk (possibly compressed) blob + // record is accumulated to total_bytes. + uint64_t adjustment = + read_options.verify_checksums + ? BlobLogRecord::CalculateAdjustmentForRecordHeader( + req.user_key->size()) + : 0; + assert(req.offset >= adjustment); + total_bytes += req.len + adjustment; + cache_hit_mask |= (Mask{1} << i); // cache hit + } + } + + // All blobs were read from the cache. + if (cached_blob_count == num_blobs) { + if (bytes_read) { + *bytes_read = total_bytes; + } + return; + } + } + + const bool no_io = read_options.read_tier == kBlockCacheTier; + if (no_io) { + for (size_t i = 0; i < num_blobs; ++i) { + if (!(cache_hit_mask & (Mask{1} << i))) { + BlobReadRequest& req = blob_reqs[i]; + assert(req.status); + + *req.status = + Status::Incomplete("Cannot read blob(s): no disk I/O allowed"); + } + } + return; + } + + { + // Find the rest of blobs from the file since I/O is allowed. + autovector>> + _blob_reqs; + uint64_t _bytes_read = 0; + + for (size_t i = 0; i < num_blobs; ++i) { + if (!(cache_hit_mask & (Mask{1} << i))) { + _blob_reqs.emplace_back(&blob_reqs[i], std::unique_ptr()); + } + } + + CacheHandleGuard blob_file_reader; + Status s = blob_file_cache_->GetBlobFileReader(read_options, file_number, + &blob_file_reader); + if (!s.ok()) { + for (size_t i = 0; i < _blob_reqs.size(); ++i) { + BlobReadRequest* const req = _blob_reqs[i].first; + assert(req); + assert(req->status); + + *req->status = s; + } + return; + } + + assert(blob_file_reader.GetValue()); + + MemoryAllocator* const allocator = + (blob_cache_ && read_options.fill_cache) + ? blob_cache_.get()->memory_allocator() + : nullptr; + + blob_file_reader.GetValue()->MultiGetBlob(read_options, allocator, + _blob_reqs, &_bytes_read); + + if (blob_cache_ && read_options.fill_cache) { + // If filling cache is allowed and a cache is configured, try to put + // the blob(s) to the cache. + for (auto& [req, blob_contents] : _blob_reqs) { + assert(req); + + if (req->status->ok()) { + CacheHandleGuard blob_handle; + const CacheKey cache_key = base_cache_key.WithOffset(req->offset); + const Slice key = cache_key.AsSlice(); + s = PutBlobIntoCache(key, &blob_contents, &blob_handle); + if (!s.ok()) { + *req->status = s; + } else { + PinCachedBlob(&blob_handle, req->result); + } + } + } + } else { + for (auto& [req, blob_contents] : _blob_reqs) { + assert(req); + + if (req->status->ok()) { + PinOwnedBlob(&blob_contents, req->result); + } + } + } + + total_bytes += _bytes_read; + if (bytes_read) { + *bytes_read = total_bytes; + } + } +} + +bool BlobSource::TEST_BlobInCache(uint64_t file_number, uint64_t file_size, + uint64_t offset, size_t* charge) const { + const CacheKey cache_key = GetCacheKey(file_number, file_size, offset); + const Slice key = cache_key.AsSlice(); + + CacheHandleGuard blob_handle; + const Status s = GetBlobFromCache(key, &blob_handle); + + if (s.ok() && blob_handle.GetValue() != nullptr) { + if (charge) { + const Cache* const cache = blob_handle.GetCache(); + assert(cache); + + Cache::Handle* const handle = blob_handle.GetCacheHandle(); + assert(handle); + + *charge = cache->GetUsage(handle); + } + + return true; + } + + return false; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_source.h b/librocksdb-sys/rocksdb/db/blob/blob_source.h new file mode 100644 index 0000000..d5e009b --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_source.h @@ -0,0 +1,161 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include + +#include "cache/cache_key.h" +#include "cache/typed_cache.h" +#include "db/blob/blob_contents.h" +#include "db/blob/blob_file_cache.h" +#include "db/blob/blob_read_request.h" +#include "rocksdb/cache.h" +#include "rocksdb/rocksdb_namespace.h" +#include "table/block_based/cachable_entry.h" +#include "util/autovector.h" + +namespace ROCKSDB_NAMESPACE { + +struct ImmutableOptions; +class Status; +class FilePrefetchBuffer; +class Slice; + +// BlobSource is a class that provides universal access to blobs, regardless of +// whether they are in the blob cache, secondary cache, or (remote) storage. +// Depending on user settings, it always fetch blobs from multi-tier cache and +// storage with minimal cost. +class BlobSource { + public: + BlobSource(const ImmutableOptions* immutable_options, + const std::string& db_id, const std::string& db_session_id, + BlobFileCache* blob_file_cache); + + BlobSource(const BlobSource&) = delete; + BlobSource& operator=(const BlobSource&) = delete; + + ~BlobSource(); + + // Read a blob from the underlying cache or one blob file. + // + // If successful, returns ok and sets "*value" to the newly retrieved + // uncompressed blob. If there was an error while fetching the blob, sets + // "*value" to empty and returns a non-ok status. + // + // Note: For consistency, whether the blob is found in the cache or on disk, + // sets "*bytes_read" to the size of on-disk (possibly compressed) blob + // record. + Status GetBlob(const ReadOptions& read_options, const Slice& user_key, + uint64_t file_number, uint64_t offset, uint64_t file_size, + uint64_t value_size, CompressionType compression_type, + FilePrefetchBuffer* prefetch_buffer, PinnableSlice* value, + uint64_t* bytes_read); + + // Read multiple blobs from the underlying cache or blob file(s). + // + // If successful, returns ok and sets "result" in the elements of "blob_reqs" + // to the newly retrieved uncompressed blobs. If there was an error while + // fetching one of blobs, sets its "result" to empty and sets its + // corresponding "status" to a non-ok status. + // + // Note: + // - The main difference between this function and MultiGetBlobFromOneFile is + // that this function can read multiple blobs from multiple blob files. + // + // - For consistency, whether the blob is found in the cache or on disk, sets + // "*bytes_read" to the total size of on-disk (possibly compressed) blob + // records. + void MultiGetBlob(const ReadOptions& read_options, + autovector& blob_reqs, + uint64_t* bytes_read); + + // Read multiple blobs from the underlying cache or one blob file. + // + // If successful, returns ok and sets "result" in the elements of "blob_reqs" + // to the newly retrieved uncompressed blobs. If there was an error while + // fetching one of blobs, sets its "result" to empty and sets its + // corresponding "status" to a non-ok status. + // + // Note: + // - The main difference between this function and MultiGetBlob is that this + // function is only used for the case where the demanded blobs are stored in + // one blob file. MultiGetBlob will call this function multiple times if the + // demanded blobs are stored in multiple blob files. + // + // - For consistency, whether the blob is found in the cache or on disk, sets + // "*bytes_read" to the total size of on-disk (possibly compressed) blob + // records. + void MultiGetBlobFromOneFile(const ReadOptions& read_options, + uint64_t file_number, uint64_t file_size, + autovector& blob_reqs, + uint64_t* bytes_read); + + inline Status GetBlobFileReader( + const ReadOptions& read_options, uint64_t blob_file_number, + CacheHandleGuard* blob_file_reader) { + return blob_file_cache_->GetBlobFileReader(read_options, blob_file_number, + blob_file_reader); + } + + inline Cache* GetBlobCache() const { return blob_cache_.get(); } + + bool TEST_BlobInCache(uint64_t file_number, uint64_t file_size, + uint64_t offset, size_t* charge = nullptr) const; + + // For TypedSharedCacheInterface + void Create(BlobContents** out, const char* buf, size_t size, + MemoryAllocator* alloc); + + using SharedCacheInterface = + FullTypedSharedCacheInterface; + using TypedHandle = SharedCacheInterface::TypedHandle; + + private: + Status GetBlobFromCache(const Slice& cache_key, + CacheHandleGuard* cached_blob) const; + + Status PutBlobIntoCache(const Slice& cache_key, + std::unique_ptr* blob, + CacheHandleGuard* cached_blob) const; + + static void PinCachedBlob(CacheHandleGuard* cached_blob, + PinnableSlice* value); + + static void PinOwnedBlob(std::unique_ptr* owned_blob, + PinnableSlice* value); + + TypedHandle* GetEntryFromCache(const Slice& key) const; + + Status InsertEntryIntoCache(const Slice& key, BlobContents* value, + TypedHandle** cache_handle, + Cache::Priority priority) const; + + inline CacheKey GetCacheKey(uint64_t file_number, uint64_t /*file_size*/, + uint64_t offset) const { + OffsetableCacheKey base_cache_key(db_id_, db_session_id_, file_number); + return base_cache_key.WithOffset(offset); + } + + const std::string& db_id_; + const std::string& db_session_id_; + + Statistics* statistics_; + + // A cache to store blob file reader. + BlobFileCache* blob_file_cache_; + + // A cache to store uncompressed blobs. + mutable SharedCacheInterface blob_cache_; + + // The control option of how the cache tiers will be used. Currently rocksdb + // support block/blob cache (volatile tier) and secondary cache (this tier + // isn't strictly speaking a non-volatile tier since the compressed cache in + // this tier is in volatile memory). + const CacheTier lowest_used_cache_tier_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/blob_source_test.cc b/librocksdb-sys/rocksdb/db/blob/blob_source_test.cc new file mode 100644 index 0000000..a977156 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/blob_source_test.cc @@ -0,0 +1,1611 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_source.h" + +#include +#include +#include +#include +#include + +#include "cache/charged_cache.h" +#include "cache/compressed_secondary_cache.h" +#include "db/blob/blob_contents.h" +#include "db/blob/blob_file_cache.h" +#include "db/blob/blob_file_reader.h" +#include "db/blob/blob_log_format.h" +#include "db/blob/blob_log_writer.h" +#include "db/db_test_util.h" +#include "file/filename.h" +#include "file/read_write_util.h" +#include "options/cf_options.h" +#include "rocksdb/options.h" +#include "util/compression.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { + +// Creates a test blob file with `num` blobs in it. +void WriteBlobFile(const ImmutableOptions& immutable_options, + uint32_t column_family_id, bool has_ttl, + const ExpirationRange& expiration_range_header, + const ExpirationRange& expiration_range_footer, + uint64_t blob_file_number, const std::vector& keys, + const std::vector& blobs, CompressionType compression, + std::vector& blob_offsets, + std::vector& blob_sizes) { + assert(!immutable_options.cf_paths.empty()); + size_t num = keys.size(); + assert(num == blobs.size()); + assert(num == blob_offsets.size()); + assert(num == blob_sizes.size()); + + const std::string blob_file_path = + BlobFileName(immutable_options.cf_paths.front().path, blob_file_number); + std::unique_ptr file; + ASSERT_OK(NewWritableFile(immutable_options.fs.get(), blob_file_path, &file, + FileOptions())); + + std::unique_ptr file_writer(new WritableFileWriter( + std::move(file), blob_file_path, FileOptions(), immutable_options.clock)); + + constexpr Statistics* statistics = nullptr; + constexpr bool use_fsync = false; + constexpr bool do_flush = false; + + BlobLogWriter blob_log_writer(std::move(file_writer), immutable_options.clock, + statistics, blob_file_number, use_fsync, + do_flush); + + BlobLogHeader header(column_family_id, compression, has_ttl, + expiration_range_header); + + ASSERT_OK(blob_log_writer.WriteHeader(header)); + + std::vector compressed_blobs(num); + std::vector blobs_to_write(num); + if (kNoCompression == compression) { + for (size_t i = 0; i < num; ++i) { + blobs_to_write[i] = blobs[i]; + blob_sizes[i] = blobs[i].size(); + } + } else { + CompressionOptions opts; + CompressionContext context(compression); + constexpr uint64_t sample_for_compression = 0; + CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(), + compression, sample_for_compression); + + constexpr uint32_t compression_format_version = 2; + + for (size_t i = 0; i < num; ++i) { + ASSERT_TRUE(CompressData(blobs[i], info, compression_format_version, + &compressed_blobs[i])); + blobs_to_write[i] = compressed_blobs[i]; + blob_sizes[i] = compressed_blobs[i].size(); + } + } + + for (size_t i = 0; i < num; ++i) { + uint64_t key_offset = 0; + ASSERT_OK(blob_log_writer.AddRecord(keys[i], blobs_to_write[i], &key_offset, + &blob_offsets[i])); + } + + BlobLogFooter footer; + footer.blob_count = num; + footer.expiration_range = expiration_range_footer; + + std::string checksum_method; + std::string checksum_value; + ASSERT_OK( + blob_log_writer.AppendFooter(footer, &checksum_method, &checksum_value)); +} + +} // anonymous namespace + +class BlobSourceTest : public DBTestBase { + protected: + public: + explicit BlobSourceTest() + : DBTestBase("blob_source_test", /*env_do_fsync=*/true) { + options_.env = env_; + options_.enable_blob_files = true; + options_.create_if_missing = true; + + LRUCacheOptions co; + co.capacity = 8 << 20; + co.num_shard_bits = 2; + co.metadata_charge_policy = kDontChargeCacheMetadata; + co.high_pri_pool_ratio = 0.2; + co.low_pri_pool_ratio = 0.2; + options_.blob_cache = NewLRUCache(co); + options_.lowest_used_cache_tier = CacheTier::kVolatileTier; + + assert(db_->GetDbIdentity(db_id_).ok()); + assert(db_->GetDbSessionId(db_session_id_).ok()); + } + + Options options_; + std::string db_id_; + std::string db_session_id_; +}; + +TEST_F(BlobSourceTest, GetBlobsFromCache) { + options_.cf_paths.emplace_back( + test::PerThreadDBPath(env_, "BlobSourceTest_GetBlobsFromCache"), 0); + + options_.statistics = CreateDBStatistics(); + Statistics* statistics = options_.statistics.get(); + assert(statistics); + + DestroyAndReopen(options_); + + ImmutableOptions immutable_options(options_); + + constexpr uint32_t column_family_id = 1; + constexpr bool has_ttl = false; + constexpr ExpirationRange expiration_range; + constexpr uint64_t blob_file_number = 1; + constexpr size_t num_blobs = 16; + + std::vector key_strs; + std::vector blob_strs; + + for (size_t i = 0; i < num_blobs; ++i) { + key_strs.push_back("key" + std::to_string(i)); + blob_strs.push_back("blob" + std::to_string(i)); + } + + std::vector keys; + std::vector blobs; + + uint64_t file_size = BlobLogHeader::kSize; + for (size_t i = 0; i < num_blobs; ++i) { + keys.push_back({key_strs[i]}); + blobs.push_back({blob_strs[i]}); + file_size += BlobLogRecord::kHeaderSize + keys[i].size() + blobs[i].size(); + } + file_size += BlobLogFooter::kSize; + + std::vector blob_offsets(keys.size()); + std::vector blob_sizes(keys.size()); + + WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range, + expiration_range, blob_file_number, keys, blobs, kNoCompression, + blob_offsets, blob_sizes); + + constexpr size_t capacity = 1024; + std::shared_ptr backing_cache = + NewLRUCache(capacity); // Blob file cache + + FileOptions file_options; + constexpr HistogramImpl* blob_file_read_hist = nullptr; + + std::unique_ptr blob_file_cache = + std::make_unique( + backing_cache.get(), &immutable_options, &file_options, + column_family_id, blob_file_read_hist, nullptr /*IOTracer*/); + + BlobSource blob_source(&immutable_options, db_id_, db_session_id_, + blob_file_cache.get()); + + ReadOptions read_options; + read_options.verify_checksums = true; + + constexpr FilePrefetchBuffer* prefetch_buffer = nullptr; + + { + // GetBlob + std::vector values(keys.size()); + uint64_t bytes_read = 0; + uint64_t blob_bytes = 0; + uint64_t total_bytes = 0; + + read_options.fill_cache = false; + get_perf_context()->Reset(); + + for (size_t i = 0; i < num_blobs; ++i) { + ASSERT_FALSE(blob_source.TEST_BlobInCache(blob_file_number, file_size, + blob_offsets[i])); + + ASSERT_OK(blob_source.GetBlob(read_options, keys[i], blob_file_number, + blob_offsets[i], file_size, blob_sizes[i], + kNoCompression, prefetch_buffer, &values[i], + &bytes_read)); + ASSERT_EQ(values[i], blobs[i]); + ASSERT_TRUE(values[i].IsPinned()); + ASSERT_EQ(bytes_read, + BlobLogRecord::kHeaderSize + keys[i].size() + blob_sizes[i]); + + ASSERT_FALSE(blob_source.TEST_BlobInCache(blob_file_number, file_size, + blob_offsets[i])); + total_bytes += bytes_read; + } + + // Retrieved the blob cache num_blobs * 3 times via TEST_BlobInCache, + // GetBlob, and TEST_BlobInCache. + ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, 0); + ASSERT_EQ((int)get_perf_context()->blob_read_count, num_blobs); + ASSERT_EQ((int)get_perf_context()->blob_read_byte, total_bytes); + ASSERT_GE((int)get_perf_context()->blob_checksum_time, 0); + ASSERT_EQ((int)get_perf_context()->blob_decompress_time, 0); + + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), num_blobs * 3); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), 0); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), 0); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ), 0); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE), 0); + + read_options.fill_cache = true; + blob_bytes = 0; + total_bytes = 0; + get_perf_context()->Reset(); + statistics->Reset().PermitUncheckedError(); + + for (size_t i = 0; i < num_blobs; ++i) { + ASSERT_FALSE(blob_source.TEST_BlobInCache(blob_file_number, file_size, + blob_offsets[i])); + + ASSERT_OK(blob_source.GetBlob(read_options, keys[i], blob_file_number, + blob_offsets[i], file_size, blob_sizes[i], + kNoCompression, prefetch_buffer, &values[i], + &bytes_read)); + ASSERT_EQ(values[i], blobs[i]); + ASSERT_TRUE(values[i].IsPinned()); + ASSERT_EQ(bytes_read, + BlobLogRecord::kHeaderSize + keys[i].size() + blob_sizes[i]); + + blob_bytes += blob_sizes[i]; + total_bytes += bytes_read; + ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, i); + ASSERT_EQ((int)get_perf_context()->blob_read_count, i + 1); + ASSERT_EQ((int)get_perf_context()->blob_read_byte, total_bytes); + + ASSERT_TRUE(blob_source.TEST_BlobInCache(blob_file_number, file_size, + blob_offsets[i])); + + ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, i + 1); + ASSERT_EQ((int)get_perf_context()->blob_read_count, i + 1); + ASSERT_EQ((int)get_perf_context()->blob_read_byte, total_bytes); + } + + ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, num_blobs); + ASSERT_EQ((int)get_perf_context()->blob_read_count, num_blobs); + ASSERT_EQ((int)get_perf_context()->blob_read_byte, total_bytes); + + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), num_blobs * 2); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), num_blobs); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), num_blobs); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ), blob_bytes); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE), + blob_bytes); + + read_options.fill_cache = true; + total_bytes = 0; + blob_bytes = 0; + get_perf_context()->Reset(); + statistics->Reset().PermitUncheckedError(); + + for (size_t i = 0; i < num_blobs; ++i) { + ASSERT_TRUE(blob_source.TEST_BlobInCache(blob_file_number, file_size, + blob_offsets[i])); + + ASSERT_OK(blob_source.GetBlob(read_options, keys[i], blob_file_number, + blob_offsets[i], file_size, blob_sizes[i], + kNoCompression, prefetch_buffer, &values[i], + &bytes_read)); + ASSERT_EQ(values[i], blobs[i]); + ASSERT_TRUE(values[i].IsPinned()); + ASSERT_EQ(bytes_read, + BlobLogRecord::kHeaderSize + keys[i].size() + blob_sizes[i]); + + ASSERT_TRUE(blob_source.TEST_BlobInCache(blob_file_number, file_size, + blob_offsets[i])); + total_bytes += bytes_read; // on-disk blob record size + blob_bytes += blob_sizes[i]; // cached blob value size + } + + // Retrieved the blob cache num_blobs * 3 times via TEST_BlobInCache, + // GetBlob, and TEST_BlobInCache. + ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, num_blobs * 3); + ASSERT_EQ((int)get_perf_context()->blob_read_count, 0); // without i/o + ASSERT_EQ((int)get_perf_context()->blob_read_byte, 0); // without i/o + + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), 0); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), num_blobs * 3); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), 0); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ), + blob_bytes * 3); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE), 0); + + // Cache-only GetBlob + read_options.read_tier = ReadTier::kBlockCacheTier; + total_bytes = 0; + blob_bytes = 0; + get_perf_context()->Reset(); + statistics->Reset().PermitUncheckedError(); + + for (size_t i = 0; i < num_blobs; ++i) { + ASSERT_TRUE(blob_source.TEST_BlobInCache(blob_file_number, file_size, + blob_offsets[i])); + + ASSERT_OK(blob_source.GetBlob(read_options, keys[i], blob_file_number, + blob_offsets[i], file_size, blob_sizes[i], + kNoCompression, prefetch_buffer, &values[i], + &bytes_read)); + ASSERT_EQ(values[i], blobs[i]); + ASSERT_TRUE(values[i].IsPinned()); + ASSERT_EQ(bytes_read, + BlobLogRecord::kHeaderSize + keys[i].size() + blob_sizes[i]); + + ASSERT_TRUE(blob_source.TEST_BlobInCache(blob_file_number, file_size, + blob_offsets[i])); + total_bytes += bytes_read; + blob_bytes += blob_sizes[i]; + } + + // Retrieved the blob cache num_blobs * 3 times via TEST_BlobInCache, + // GetBlob, and TEST_BlobInCache. + ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, num_blobs * 3); + ASSERT_EQ((int)get_perf_context()->blob_read_count, 0); // without i/o + ASSERT_EQ((int)get_perf_context()->blob_read_byte, 0); // without i/o + + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), 0); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), num_blobs * 3); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), 0); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ), + blob_bytes * 3); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE), 0); + } + + options_.blob_cache->EraseUnRefEntries(); + + { + // Cache-only GetBlob + std::vector values(keys.size()); + uint64_t bytes_read = 0; + + read_options.read_tier = ReadTier::kBlockCacheTier; + read_options.fill_cache = true; + get_perf_context()->Reset(); + statistics->Reset().PermitUncheckedError(); + + for (size_t i = 0; i < num_blobs; ++i) { + ASSERT_FALSE(blob_source.TEST_BlobInCache(blob_file_number, file_size, + blob_offsets[i])); + + ASSERT_TRUE(blob_source + .GetBlob(read_options, keys[i], blob_file_number, + blob_offsets[i], file_size, blob_sizes[i], + kNoCompression, prefetch_buffer, &values[i], + &bytes_read) + .IsIncomplete()); + ASSERT_TRUE(values[i].empty()); + ASSERT_FALSE(values[i].IsPinned()); + ASSERT_EQ(bytes_read, 0); + + ASSERT_FALSE(blob_source.TEST_BlobInCache(blob_file_number, file_size, + blob_offsets[i])); + } + + // Retrieved the blob cache num_blobs * 3 times via TEST_BlobInCache, + // GetBlob, and TEST_BlobInCache. + ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, 0); + ASSERT_EQ((int)get_perf_context()->blob_read_count, 0); + ASSERT_EQ((int)get_perf_context()->blob_read_byte, 0); + + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), num_blobs * 3); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), 0); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), 0); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ), 0); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE), 0); + } + + { + // GetBlob from non-existing file + std::vector values(keys.size()); + uint64_t bytes_read = 0; + uint64_t file_number = 100; // non-existing file + + read_options.read_tier = ReadTier::kReadAllTier; + read_options.fill_cache = true; + get_perf_context()->Reset(); + statistics->Reset().PermitUncheckedError(); + + for (size_t i = 0; i < num_blobs; ++i) { + ASSERT_FALSE(blob_source.TEST_BlobInCache(file_number, file_size, + blob_offsets[i])); + + ASSERT_TRUE(blob_source + .GetBlob(read_options, keys[i], file_number, + blob_offsets[i], file_size, blob_sizes[i], + kNoCompression, prefetch_buffer, &values[i], + &bytes_read) + .IsIOError()); + ASSERT_TRUE(values[i].empty()); + ASSERT_FALSE(values[i].IsPinned()); + ASSERT_EQ(bytes_read, 0); + + ASSERT_FALSE(blob_source.TEST_BlobInCache(file_number, file_size, + blob_offsets[i])); + } + + // Retrieved the blob cache num_blobs * 3 times via TEST_BlobInCache, + // GetBlob, and TEST_BlobInCache. + ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, 0); + ASSERT_EQ((int)get_perf_context()->blob_read_count, 0); + ASSERT_EQ((int)get_perf_context()->blob_read_byte, 0); + + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), num_blobs * 3); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), 0); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), 0); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ), 0); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE), 0); + } +} + +TEST_F(BlobSourceTest, GetCompressedBlobs) { + if (!Snappy_Supported()) { + return; + } + + const CompressionType compression = kSnappyCompression; + + options_.cf_paths.emplace_back( + test::PerThreadDBPath(env_, "BlobSourceTest_GetCompressedBlobs"), 0); + + DestroyAndReopen(options_); + + ImmutableOptions immutable_options(options_); + + constexpr uint32_t column_family_id = 1; + constexpr bool has_ttl = false; + constexpr ExpirationRange expiration_range; + constexpr size_t num_blobs = 256; + + std::vector key_strs; + std::vector blob_strs; + + for (size_t i = 0; i < num_blobs; ++i) { + key_strs.push_back("key" + std::to_string(i)); + blob_strs.push_back("blob" + std::to_string(i)); + } + + std::vector keys; + std::vector blobs; + + for (size_t i = 0; i < num_blobs; ++i) { + keys.push_back({key_strs[i]}); + blobs.push_back({blob_strs[i]}); + } + + std::vector blob_offsets(keys.size()); + std::vector blob_sizes(keys.size()); + + constexpr size_t capacity = 1024; + auto backing_cache = NewLRUCache(capacity); // Blob file cache + + FileOptions file_options; + std::unique_ptr blob_file_cache = + std::make_unique( + backing_cache.get(), &immutable_options, &file_options, + column_family_id, nullptr /*HistogramImpl*/, nullptr /*IOTracer*/); + + BlobSource blob_source(&immutable_options, db_id_, db_session_id_, + blob_file_cache.get()); + + ReadOptions read_options; + read_options.verify_checksums = true; + + uint64_t bytes_read = 0; + std::vector values(keys.size()); + + { + // Snappy Compression + const uint64_t file_number = 1; + + read_options.read_tier = ReadTier::kReadAllTier; + + WriteBlobFile(immutable_options, column_family_id, has_ttl, + expiration_range, expiration_range, file_number, keys, blobs, + compression, blob_offsets, blob_sizes); + + CacheHandleGuard blob_file_reader; + ASSERT_OK(blob_source.GetBlobFileReader(read_options, file_number, + &blob_file_reader)); + ASSERT_NE(blob_file_reader.GetValue(), nullptr); + + const uint64_t file_size = blob_file_reader.GetValue()->GetFileSize(); + ASSERT_EQ(blob_file_reader.GetValue()->GetCompressionType(), compression); + + for (size_t i = 0; i < num_blobs; ++i) { + ASSERT_NE(blobs[i].size() /*uncompressed size*/, + blob_sizes[i] /*compressed size*/); + } + + read_options.fill_cache = true; + read_options.read_tier = ReadTier::kReadAllTier; + get_perf_context()->Reset(); + + for (size_t i = 0; i < num_blobs; ++i) { + ASSERT_FALSE(blob_source.TEST_BlobInCache(file_number, file_size, + blob_offsets[i])); + ASSERT_OK(blob_source.GetBlob(read_options, keys[i], file_number, + blob_offsets[i], file_size, blob_sizes[i], + compression, nullptr /*prefetch_buffer*/, + &values[i], &bytes_read)); + ASSERT_EQ(values[i], blobs[i] /*uncompressed blob*/); + ASSERT_NE(values[i].size(), blob_sizes[i] /*compressed size*/); + ASSERT_EQ(bytes_read, + BlobLogRecord::kHeaderSize + keys[i].size() + blob_sizes[i]); + + ASSERT_TRUE(blob_source.TEST_BlobInCache(file_number, file_size, + blob_offsets[i])); + } + + ASSERT_GE((int)get_perf_context()->blob_decompress_time, 0); + + read_options.read_tier = ReadTier::kBlockCacheTier; + get_perf_context()->Reset(); + + for (size_t i = 0; i < num_blobs; ++i) { + ASSERT_TRUE(blob_source.TEST_BlobInCache(file_number, file_size, + blob_offsets[i])); + + // Compressed blob size is passed in GetBlob + ASSERT_OK(blob_source.GetBlob(read_options, keys[i], file_number, + blob_offsets[i], file_size, blob_sizes[i], + compression, nullptr /*prefetch_buffer*/, + &values[i], &bytes_read)); + ASSERT_EQ(values[i], blobs[i] /*uncompressed blob*/); + ASSERT_NE(values[i].size(), blob_sizes[i] /*compressed size*/); + ASSERT_EQ(bytes_read, + BlobLogRecord::kHeaderSize + keys[i].size() + blob_sizes[i]); + + ASSERT_TRUE(blob_source.TEST_BlobInCache(file_number, file_size, + blob_offsets[i])); + } + + ASSERT_EQ((int)get_perf_context()->blob_decompress_time, 0); + } +} + +TEST_F(BlobSourceTest, MultiGetBlobsFromMultiFiles) { + options_.cf_paths.emplace_back( + test::PerThreadDBPath(env_, "BlobSourceTest_MultiGetBlobsFromMultiFiles"), + 0); + + options_.statistics = CreateDBStatistics(); + Statistics* statistics = options_.statistics.get(); + assert(statistics); + + DestroyAndReopen(options_); + + ImmutableOptions immutable_options(options_); + + constexpr uint32_t column_family_id = 1; + constexpr bool has_ttl = false; + constexpr ExpirationRange expiration_range; + constexpr uint64_t blob_files = 2; + constexpr size_t num_blobs = 32; + + std::vector key_strs; + std::vector blob_strs; + + for (size_t i = 0; i < num_blobs; ++i) { + key_strs.push_back("key" + std::to_string(i)); + blob_strs.push_back("blob" + std::to_string(i)); + } + + std::vector keys; + std::vector blobs; + + uint64_t file_size = BlobLogHeader::kSize; + uint64_t blob_value_bytes = 0; + for (size_t i = 0; i < num_blobs; ++i) { + keys.push_back({key_strs[i]}); + blobs.push_back({blob_strs[i]}); + blob_value_bytes += blobs[i].size(); + file_size += BlobLogRecord::kHeaderSize + keys[i].size() + blobs[i].size(); + } + file_size += BlobLogFooter::kSize; + const uint64_t blob_records_bytes = + file_size - BlobLogHeader::kSize - BlobLogFooter::kSize; + + std::vector blob_offsets(keys.size()); + std::vector blob_sizes(keys.size()); + + { + // Write key/blob pairs to multiple blob files. + for (size_t i = 0; i < blob_files; ++i) { + const uint64_t file_number = i + 1; + WriteBlobFile(immutable_options, column_family_id, has_ttl, + expiration_range, expiration_range, file_number, keys, + blobs, kNoCompression, blob_offsets, blob_sizes); + } + } + + constexpr size_t capacity = 10; + std::shared_ptr backing_cache = + NewLRUCache(capacity); // Blob file cache + + FileOptions file_options; + constexpr HistogramImpl* blob_file_read_hist = nullptr; + + std::unique_ptr blob_file_cache = + std::make_unique( + backing_cache.get(), &immutable_options, &file_options, + column_family_id, blob_file_read_hist, nullptr /*IOTracer*/); + + BlobSource blob_source(&immutable_options, db_id_, db_session_id_, + blob_file_cache.get()); + + ReadOptions read_options; + read_options.verify_checksums = true; + + uint64_t bytes_read = 0; + + { + // MultiGetBlob + read_options.fill_cache = true; + read_options.read_tier = ReadTier::kReadAllTier; + + autovector blob_reqs; + std::array, blob_files> blob_reqs_in_file; + std::array value_buf; + std::array statuses_buf; + + for (size_t i = 0; i < blob_files; ++i) { + const uint64_t file_number = i + 1; + for (size_t j = 0; j < num_blobs; ++j) { + blob_reqs_in_file[i].emplace_back( + keys[j], blob_offsets[j], blob_sizes[j], kNoCompression, + &value_buf[i * num_blobs + j], &statuses_buf[i * num_blobs + j]); + } + blob_reqs.emplace_back(file_number, file_size, blob_reqs_in_file[i]); + } + + get_perf_context()->Reset(); + statistics->Reset().PermitUncheckedError(); + + blob_source.MultiGetBlob(read_options, blob_reqs, &bytes_read); + + for (size_t i = 0; i < blob_files; ++i) { + const uint64_t file_number = i + 1; + for (size_t j = 0; j < num_blobs; ++j) { + ASSERT_OK(statuses_buf[i * num_blobs + j]); + ASSERT_EQ(value_buf[i * num_blobs + j], blobs[j]); + ASSERT_TRUE(blob_source.TEST_BlobInCache(file_number, file_size, + blob_offsets[j])); + } + } + + // Retrieved all blobs from 2 blob files twice via MultiGetBlob and + // TEST_BlobInCache. + ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, + num_blobs * blob_files); + ASSERT_EQ((int)get_perf_context()->blob_read_count, + num_blobs * blob_files); // blocking i/o + ASSERT_EQ((int)get_perf_context()->blob_read_byte, + blob_records_bytes * blob_files); // blocking i/o + ASSERT_GE((int)get_perf_context()->blob_checksum_time, 0); + ASSERT_EQ((int)get_perf_context()->blob_decompress_time, 0); + + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), + num_blobs * blob_files); // MultiGetBlob + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), + num_blobs * blob_files); // TEST_BlobInCache + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), + num_blobs * blob_files); // MultiGetBlob + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ), + blob_value_bytes * blob_files); // TEST_BlobInCache + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE), + blob_value_bytes * blob_files); // MultiGetBlob + + get_perf_context()->Reset(); + statistics->Reset().PermitUncheckedError(); + + autovector fake_blob_reqs_in_file; + std::array fake_value_buf; + std::array fake_statuses_buf; + + const uint64_t fake_file_number = 100; + for (size_t i = 0; i < num_blobs; ++i) { + fake_blob_reqs_in_file.emplace_back( + keys[i], blob_offsets[i], blob_sizes[i], kNoCompression, + &fake_value_buf[i], &fake_statuses_buf[i]); + } + + // Add a fake multi-get blob request. + blob_reqs.emplace_back(fake_file_number, file_size, fake_blob_reqs_in_file); + + blob_source.MultiGetBlob(read_options, blob_reqs, &bytes_read); + + // Check the real blob read requests. + for (size_t i = 0; i < blob_files; ++i) { + const uint64_t file_number = i + 1; + for (size_t j = 0; j < num_blobs; ++j) { + ASSERT_OK(statuses_buf[i * num_blobs + j]); + ASSERT_EQ(value_buf[i * num_blobs + j], blobs[j]); + ASSERT_TRUE(blob_source.TEST_BlobInCache(file_number, file_size, + blob_offsets[j])); + } + } + + // Check the fake blob request. + for (size_t i = 0; i < num_blobs; ++i) { + ASSERT_TRUE(fake_statuses_buf[i].IsIOError()); + ASSERT_TRUE(fake_value_buf[i].empty()); + ASSERT_FALSE(blob_source.TEST_BlobInCache(fake_file_number, file_size, + blob_offsets[i])); + } + + // Retrieved all blobs from 3 blob files (including the fake one) twice + // via MultiGetBlob and TEST_BlobInCache. + ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, + num_blobs * blob_files * 2); + ASSERT_EQ((int)get_perf_context()->blob_read_count, + 0); // blocking i/o + ASSERT_EQ((int)get_perf_context()->blob_read_byte, + 0); // blocking i/o + ASSERT_GE((int)get_perf_context()->blob_checksum_time, 0); + ASSERT_EQ((int)get_perf_context()->blob_decompress_time, 0); + + // Fake blob requests: MultiGetBlob and TEST_BlobInCache + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), num_blobs * 2); + // Real blob requests: MultiGetBlob and TEST_BlobInCache + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), + num_blobs * blob_files * 2); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), 0); + // Real blob requests: MultiGetBlob and TEST_BlobInCache + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ), + blob_value_bytes * blob_files * 2); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE), 0); + } +} + +TEST_F(BlobSourceTest, MultiGetBlobsFromCache) { + options_.cf_paths.emplace_back( + test::PerThreadDBPath(env_, "BlobSourceTest_MultiGetBlobsFromCache"), 0); + + options_.statistics = CreateDBStatistics(); + Statistics* statistics = options_.statistics.get(); + assert(statistics); + + DestroyAndReopen(options_); + + ImmutableOptions immutable_options(options_); + + constexpr uint32_t column_family_id = 1; + constexpr bool has_ttl = false; + constexpr ExpirationRange expiration_range; + constexpr uint64_t blob_file_number = 1; + constexpr size_t num_blobs = 16; + + std::vector key_strs; + std::vector blob_strs; + + for (size_t i = 0; i < num_blobs; ++i) { + key_strs.push_back("key" + std::to_string(i)); + blob_strs.push_back("blob" + std::to_string(i)); + } + + std::vector keys; + std::vector blobs; + + uint64_t file_size = BlobLogHeader::kSize; + for (size_t i = 0; i < num_blobs; ++i) { + keys.push_back({key_strs[i]}); + blobs.push_back({blob_strs[i]}); + file_size += BlobLogRecord::kHeaderSize + keys[i].size() + blobs[i].size(); + } + file_size += BlobLogFooter::kSize; + + std::vector blob_offsets(keys.size()); + std::vector blob_sizes(keys.size()); + + WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range, + expiration_range, blob_file_number, keys, blobs, kNoCompression, + blob_offsets, blob_sizes); + + constexpr size_t capacity = 10; + std::shared_ptr backing_cache = + NewLRUCache(capacity); // Blob file cache + + FileOptions file_options; + constexpr HistogramImpl* blob_file_read_hist = nullptr; + + std::unique_ptr blob_file_cache = + std::make_unique( + backing_cache.get(), &immutable_options, &file_options, + column_family_id, blob_file_read_hist, nullptr /*IOTracer*/); + + BlobSource blob_source(&immutable_options, db_id_, db_session_id_, + blob_file_cache.get()); + + ReadOptions read_options; + read_options.verify_checksums = true; + + constexpr FilePrefetchBuffer* prefetch_buffer = nullptr; + + { + // MultiGetBlobFromOneFile + uint64_t bytes_read = 0; + std::array statuses_buf; + std::array value_buf; + autovector blob_reqs; + + for (size_t i = 0; i < num_blobs; i += 2) { // even index + blob_reqs.emplace_back(keys[i], blob_offsets[i], blob_sizes[i], + kNoCompression, &value_buf[i], &statuses_buf[i]); + ASSERT_FALSE(blob_source.TEST_BlobInCache(blob_file_number, file_size, + blob_offsets[i])); + } + + read_options.fill_cache = true; + read_options.read_tier = ReadTier::kReadAllTier; + get_perf_context()->Reset(); + statistics->Reset().PermitUncheckedError(); + + // Get half of blobs + blob_source.MultiGetBlobFromOneFile(read_options, blob_file_number, + file_size, blob_reqs, &bytes_read); + + uint64_t fs_read_bytes = 0; + uint64_t ca_read_bytes = 0; + for (size_t i = 0; i < num_blobs; ++i) { + if (i % 2 == 0) { + ASSERT_OK(statuses_buf[i]); + ASSERT_EQ(value_buf[i], blobs[i]); + ASSERT_TRUE(value_buf[i].IsPinned()); + fs_read_bytes += + blob_sizes[i] + keys[i].size() + BlobLogRecord::kHeaderSize; + ASSERT_TRUE(blob_source.TEST_BlobInCache(blob_file_number, file_size, + blob_offsets[i])); + ca_read_bytes += blob_sizes[i]; + } else { + statuses_buf[i].PermitUncheckedError(); + ASSERT_TRUE(value_buf[i].empty()); + ASSERT_FALSE(value_buf[i].IsPinned()); + ASSERT_FALSE(blob_source.TEST_BlobInCache(blob_file_number, file_size, + blob_offsets[i])); + } + } + + constexpr int num_even_blobs = num_blobs / 2; + ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, num_even_blobs); + ASSERT_EQ((int)get_perf_context()->blob_read_count, + num_even_blobs); // blocking i/o + ASSERT_EQ((int)get_perf_context()->blob_read_byte, + fs_read_bytes); // blocking i/o + ASSERT_GE((int)get_perf_context()->blob_checksum_time, 0); + ASSERT_EQ((int)get_perf_context()->blob_decompress_time, 0); + + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), num_blobs); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), num_even_blobs); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), num_even_blobs); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ), + ca_read_bytes); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE), + ca_read_bytes); + + // Get the rest of blobs + for (size_t i = 1; i < num_blobs; i += 2) { // odd index + ASSERT_FALSE(blob_source.TEST_BlobInCache(blob_file_number, file_size, + blob_offsets[i])); + + ASSERT_OK(blob_source.GetBlob(read_options, keys[i], blob_file_number, + blob_offsets[i], file_size, blob_sizes[i], + kNoCompression, prefetch_buffer, + &value_buf[i], &bytes_read)); + ASSERT_EQ(value_buf[i], blobs[i]); + ASSERT_TRUE(value_buf[i].IsPinned()); + ASSERT_EQ(bytes_read, + BlobLogRecord::kHeaderSize + keys[i].size() + blob_sizes[i]); + + ASSERT_TRUE(blob_source.TEST_BlobInCache(blob_file_number, file_size, + blob_offsets[i])); + } + + // Cache-only MultiGetBlobFromOneFile + read_options.read_tier = ReadTier::kBlockCacheTier; + get_perf_context()->Reset(); + statistics->Reset().PermitUncheckedError(); + + blob_reqs.clear(); + for (size_t i = 0; i < num_blobs; ++i) { + blob_reqs.emplace_back(keys[i], blob_offsets[i], blob_sizes[i], + kNoCompression, &value_buf[i], &statuses_buf[i]); + } + + blob_source.MultiGetBlobFromOneFile(read_options, blob_file_number, + file_size, blob_reqs, &bytes_read); + + uint64_t blob_bytes = 0; + for (size_t i = 0; i < num_blobs; ++i) { + ASSERT_OK(statuses_buf[i]); + ASSERT_EQ(value_buf[i], blobs[i]); + ASSERT_TRUE(value_buf[i].IsPinned()); + ASSERT_TRUE(blob_source.TEST_BlobInCache(blob_file_number, file_size, + blob_offsets[i])); + blob_bytes += blob_sizes[i]; + } + + // Retrieved the blob cache num_blobs * 2 times via GetBlob and + // TEST_BlobInCache. + ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, num_blobs * 2); + ASSERT_EQ((int)get_perf_context()->blob_read_count, 0); // blocking i/o + ASSERT_EQ((int)get_perf_context()->blob_read_byte, 0); // blocking i/o + ASSERT_GE((int)get_perf_context()->blob_checksum_time, 0); + ASSERT_EQ((int)get_perf_context()->blob_decompress_time, 0); + + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), 0); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), num_blobs * 2); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), 0); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ), + blob_bytes * 2); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE), 0); + } + + options_.blob_cache->EraseUnRefEntries(); + + { + // Cache-only MultiGetBlobFromOneFile + uint64_t bytes_read = 0; + read_options.read_tier = ReadTier::kBlockCacheTier; + + std::array statuses_buf; + std::array value_buf; + autovector blob_reqs; + + for (size_t i = 0; i < num_blobs; i++) { + blob_reqs.emplace_back(keys[i], blob_offsets[i], blob_sizes[i], + kNoCompression, &value_buf[i], &statuses_buf[i]); + ASSERT_FALSE(blob_source.TEST_BlobInCache(blob_file_number, file_size, + blob_offsets[i])); + } + + get_perf_context()->Reset(); + statistics->Reset().PermitUncheckedError(); + + blob_source.MultiGetBlobFromOneFile(read_options, blob_file_number, + file_size, blob_reqs, &bytes_read); + + for (size_t i = 0; i < num_blobs; ++i) { + ASSERT_TRUE(statuses_buf[i].IsIncomplete()); + ASSERT_TRUE(value_buf[i].empty()); + ASSERT_FALSE(value_buf[i].IsPinned()); + ASSERT_FALSE(blob_source.TEST_BlobInCache(blob_file_number, file_size, + blob_offsets[i])); + } + + ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, 0); + ASSERT_EQ((int)get_perf_context()->blob_read_count, 0); // blocking i/o + ASSERT_EQ((int)get_perf_context()->blob_read_byte, 0); // blocking i/o + ASSERT_EQ((int)get_perf_context()->blob_checksum_time, 0); + ASSERT_EQ((int)get_perf_context()->blob_decompress_time, 0); + + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), num_blobs * 2); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), 0); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), 0); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ), 0); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE), 0); + } + + { + // MultiGetBlobFromOneFile from non-existing file + uint64_t bytes_read = 0; + uint64_t non_existing_file_number = 100; + read_options.read_tier = ReadTier::kReadAllTier; + + std::array statuses_buf; + std::array value_buf; + autovector blob_reqs; + + for (size_t i = 0; i < num_blobs; i++) { + blob_reqs.emplace_back(keys[i], blob_offsets[i], blob_sizes[i], + kNoCompression, &value_buf[i], &statuses_buf[i]); + ASSERT_FALSE(blob_source.TEST_BlobInCache(non_existing_file_number, + file_size, blob_offsets[i])); + } + + get_perf_context()->Reset(); + statistics->Reset().PermitUncheckedError(); + + blob_source.MultiGetBlobFromOneFile(read_options, non_existing_file_number, + file_size, blob_reqs, &bytes_read); + + for (size_t i = 0; i < num_blobs; ++i) { + ASSERT_TRUE(statuses_buf[i].IsIOError()); + ASSERT_TRUE(value_buf[i].empty()); + ASSERT_FALSE(value_buf[i].IsPinned()); + ASSERT_FALSE(blob_source.TEST_BlobInCache(non_existing_file_number, + file_size, blob_offsets[i])); + } + + ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, 0); + ASSERT_EQ((int)get_perf_context()->blob_read_count, 0); // blocking i/o + ASSERT_EQ((int)get_perf_context()->blob_read_byte, 0); // blocking i/o + ASSERT_EQ((int)get_perf_context()->blob_checksum_time, 0); + ASSERT_EQ((int)get_perf_context()->blob_decompress_time, 0); + + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), num_blobs * 2); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), 0); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), 0); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ), 0); + ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE), 0); + } +} + +class BlobSecondaryCacheTest : public DBTestBase { + protected: + public: + explicit BlobSecondaryCacheTest() + : DBTestBase("blob_secondary_cache_test", /*env_do_fsync=*/true) { + options_.env = env_; + options_.enable_blob_files = true; + options_.create_if_missing = true; + + // Set a small cache capacity to evict entries from the cache, and to test + // that secondary cache is used properly. + lru_cache_opts_.capacity = 1024; + lru_cache_opts_.num_shard_bits = 0; + lru_cache_opts_.strict_capacity_limit = true; + lru_cache_opts_.metadata_charge_policy = kDontChargeCacheMetadata; + lru_cache_opts_.high_pri_pool_ratio = 0.2; + lru_cache_opts_.low_pri_pool_ratio = 0.2; + + secondary_cache_opts_.capacity = 8 << 20; // 8 MB + secondary_cache_opts_.num_shard_bits = 0; + secondary_cache_opts_.metadata_charge_policy = + kDefaultCacheMetadataChargePolicy; + + // Read blobs from the secondary cache if they are not in the primary cache + options_.lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier; + + assert(db_->GetDbIdentity(db_id_).ok()); + assert(db_->GetDbSessionId(db_session_id_).ok()); + } + + Options options_; + + LRUCacheOptions lru_cache_opts_; + CompressedSecondaryCacheOptions secondary_cache_opts_; + + std::string db_id_; + std::string db_session_id_; +}; + +TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) { + if (!Snappy_Supported()) { + return; + } + + secondary_cache_opts_.compression_type = kSnappyCompression; + lru_cache_opts_.secondary_cache = + NewCompressedSecondaryCache(secondary_cache_opts_); + options_.blob_cache = NewLRUCache(lru_cache_opts_); + + options_.cf_paths.emplace_back( + test::PerThreadDBPath( + env_, "BlobSecondaryCacheTest_GetBlobsFromSecondaryCache"), + 0); + + options_.statistics = CreateDBStatistics(); + Statistics* statistics = options_.statistics.get(); + assert(statistics); + + DestroyAndReopen(options_); + + ImmutableOptions immutable_options(options_); + + constexpr uint32_t column_family_id = 1; + constexpr bool has_ttl = false; + constexpr ExpirationRange expiration_range; + constexpr uint64_t file_number = 1; + + Random rnd(301); + + std::vector key_strs{"key0", "key1"}; + std::vector blob_strs{rnd.RandomString(512), + rnd.RandomString(768)}; + + std::vector keys{key_strs[0], key_strs[1]}; + std::vector blobs{blob_strs[0], blob_strs[1]}; + + std::vector blob_offsets(keys.size()); + std::vector blob_sizes(keys.size()); + + WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range, + expiration_range, file_number, keys, blobs, kNoCompression, + blob_offsets, blob_sizes); + + constexpr size_t capacity = 1024; + std::shared_ptr backing_cache = NewLRUCache(capacity); + + FileOptions file_options; + constexpr HistogramImpl* blob_file_read_hist = nullptr; + + std::unique_ptr blob_file_cache(new BlobFileCache( + backing_cache.get(), &immutable_options, &file_options, column_family_id, + blob_file_read_hist, nullptr /*IOTracer*/)); + + BlobSource blob_source(&immutable_options, db_id_, db_session_id_, + blob_file_cache.get()); + + CacheHandleGuard file_reader; + ReadOptions read_options; + ASSERT_OK( + blob_source.GetBlobFileReader(read_options, file_number, &file_reader)); + ASSERT_NE(file_reader.GetValue(), nullptr); + const uint64_t file_size = file_reader.GetValue()->GetFileSize(); + ASSERT_EQ(file_reader.GetValue()->GetCompressionType(), kNoCompression); + + read_options.verify_checksums = true; + + auto blob_cache = options_.blob_cache; + auto secondary_cache = lru_cache_opts_.secondary_cache; + + { + // GetBlob + std::vector values(keys.size()); + + read_options.fill_cache = true; + get_perf_context()->Reset(); + + // key0 should be filled to the primary cache from the blob file. + ASSERT_OK(blob_source.GetBlob(read_options, keys[0], file_number, + blob_offsets[0], file_size, blob_sizes[0], + kNoCompression, nullptr /* prefetch_buffer */, + &values[0], nullptr /* bytes_read */)); + // Release cache handle + values[0].Reset(); + + // key0 should be evicted and key0's dummy item is inserted into secondary + // cache. key1 should be filled to the primary cache from the blob file. + ASSERT_OK(blob_source.GetBlob(read_options, keys[1], file_number, + blob_offsets[1], file_size, blob_sizes[1], + kNoCompression, nullptr /* prefetch_buffer */, + &values[1], nullptr /* bytes_read */)); + + // Release cache handle + values[1].Reset(); + + // key0 should be filled to the primary cache from the blob file. key1 + // should be evicted and key1's dummy item is inserted into secondary cache. + ASSERT_OK(blob_source.GetBlob(read_options, keys[0], file_number, + blob_offsets[0], file_size, blob_sizes[0], + kNoCompression, nullptr /* prefetch_buffer */, + &values[0], nullptr /* bytes_read */)); + ASSERT_EQ(values[0], blobs[0]); + ASSERT_TRUE( + blob_source.TEST_BlobInCache(file_number, file_size, blob_offsets[0])); + + // Release cache handle + values[0].Reset(); + + // key0 should be evicted and is inserted into secondary cache. + // key1 should be filled to the primary cache from the blob file. + ASSERT_OK(blob_source.GetBlob(read_options, keys[1], file_number, + blob_offsets[1], file_size, blob_sizes[1], + kNoCompression, nullptr /* prefetch_buffer */, + &values[1], nullptr /* bytes_read */)); + ASSERT_EQ(values[1], blobs[1]); + ASSERT_TRUE( + blob_source.TEST_BlobInCache(file_number, file_size, blob_offsets[1])); + + // Release cache handle + values[1].Reset(); + + OffsetableCacheKey base_cache_key(db_id_, db_session_id_, file_number); + + // blob_cache here only looks at the primary cache since we didn't provide + // the cache item helper for the secondary cache. However, since key0 is + // demoted to the secondary cache, we shouldn't be able to find it in the + // primary cache. + { + CacheKey cache_key = base_cache_key.WithOffset(blob_offsets[0]); + const Slice key0 = cache_key.AsSlice(); + auto handle0 = blob_cache->BasicLookup(key0, statistics); + ASSERT_EQ(handle0, nullptr); + + // key0's item should be in the secondary cache. + bool kept_in_sec_cache = false; + auto sec_handle0 = secondary_cache->Lookup( + key0, BlobSource::SharedCacheInterface::GetFullHelper(), + /*context*/ nullptr, true, + /*advise_erase=*/true, kept_in_sec_cache); + ASSERT_FALSE(kept_in_sec_cache); + ASSERT_NE(sec_handle0, nullptr); + ASSERT_TRUE(sec_handle0->IsReady()); + auto value = static_cast(sec_handle0->Value()); + ASSERT_NE(value, nullptr); + ASSERT_EQ(value->data(), blobs[0]); + delete value; + + // key0 doesn't exist in the blob cache although key0's dummy + // item exist in the secondary cache. + ASSERT_FALSE(blob_source.TEST_BlobInCache(file_number, file_size, + blob_offsets[0])); + } + + // key1 should exists in the primary cache. key1's dummy item exists + // in the secondary cache. + { + CacheKey cache_key = base_cache_key.WithOffset(blob_offsets[1]); + const Slice key1 = cache_key.AsSlice(); + auto handle1 = blob_cache->BasicLookup(key1, statistics); + ASSERT_NE(handle1, nullptr); + blob_cache->Release(handle1); + + bool kept_in_sec_cache = false; + auto sec_handle1 = secondary_cache->Lookup( + key1, BlobSource::SharedCacheInterface::GetFullHelper(), + /*context*/ nullptr, true, + /*advise_erase=*/true, kept_in_sec_cache); + ASSERT_FALSE(kept_in_sec_cache); + ASSERT_EQ(sec_handle1, nullptr); + + ASSERT_TRUE(blob_source.TEST_BlobInCache(file_number, file_size, + blob_offsets[1])); + } + + { + // fetch key0 from the blob file to the primary cache. + // key1 is evicted and inserted into the secondary cache. + ASSERT_OK(blob_source.GetBlob( + read_options, keys[0], file_number, blob_offsets[0], file_size, + blob_sizes[0], kNoCompression, nullptr /* prefetch_buffer */, + &values[0], nullptr /* bytes_read */)); + ASSERT_EQ(values[0], blobs[0]); + + // Release cache handle + values[0].Reset(); + + // key0 should be in the primary cache. + CacheKey cache_key0 = base_cache_key.WithOffset(blob_offsets[0]); + const Slice key0 = cache_key0.AsSlice(); + auto handle0 = blob_cache->BasicLookup(key0, statistics); + ASSERT_NE(handle0, nullptr); + auto value = static_cast(blob_cache->Value(handle0)); + ASSERT_NE(value, nullptr); + ASSERT_EQ(value->data(), blobs[0]); + blob_cache->Release(handle0); + + // key1 is not in the primary cache and is in the secondary cache. + CacheKey cache_key1 = base_cache_key.WithOffset(blob_offsets[1]); + const Slice key1 = cache_key1.AsSlice(); + auto handle1 = blob_cache->BasicLookup(key1, statistics); + ASSERT_EQ(handle1, nullptr); + + // erase key0 from the primary cache. + blob_cache->Erase(key0); + handle0 = blob_cache->BasicLookup(key0, statistics); + ASSERT_EQ(handle0, nullptr); + + // key1 promotion should succeed due to the primary cache being empty. we + // did't call secondary cache's Lookup() here, because it will remove the + // key but it won't be able to promote the key to the primary cache. + // Instead we use the end-to-end blob source API to read key1. + // In function TEST_BlobInCache, key1's dummy item is inserted into the + // primary cache and a standalone handle is checked by GetValue(). + ASSERT_TRUE(blob_source.TEST_BlobInCache(file_number, file_size, + blob_offsets[1])); + + // key1's dummy handle is in the primary cache and key1's item is still + // in the secondary cache. So, the primary cache's Lookup() without + // secondary cache support cannot see it. (NOTE: The dummy handle used + // to be a leaky abstraction but not anymore.) + handle1 = blob_cache->BasicLookup(key1, statistics); + ASSERT_EQ(handle1, nullptr); + + // But after another access, it is promoted to primary cache + ASSERT_TRUE(blob_source.TEST_BlobInCache(file_number, file_size, + blob_offsets[1])); + + // And Lookup() can find it (without secondary cache support) + handle1 = blob_cache->BasicLookup(key1, statistics); + ASSERT_NE(handle1, nullptr); + ASSERT_NE(blob_cache->Value(handle1), nullptr); + blob_cache->Release(handle1); + } + } +} + +class BlobSourceCacheReservationTest : public DBTestBase { + public: + explicit BlobSourceCacheReservationTest() + : DBTestBase("blob_source_cache_reservation_test", + /*env_do_fsync=*/true) { + options_.env = env_; + options_.enable_blob_files = true; + options_.create_if_missing = true; + + LRUCacheOptions co; + co.capacity = kCacheCapacity; + co.num_shard_bits = kNumShardBits; + co.metadata_charge_policy = kDontChargeCacheMetadata; + + co.high_pri_pool_ratio = 0.0; + co.low_pri_pool_ratio = 0.0; + std::shared_ptr blob_cache = NewLRUCache(co); + + co.high_pri_pool_ratio = 0.5; + co.low_pri_pool_ratio = 0.5; + std::shared_ptr block_cache = NewLRUCache(co); + + options_.blob_cache = blob_cache; + options_.lowest_used_cache_tier = CacheTier::kVolatileTier; + + BlockBasedTableOptions block_based_options; + block_based_options.no_block_cache = false; + block_based_options.block_cache = block_cache; + block_based_options.cache_usage_options.options_overrides.insert( + {CacheEntryRole::kBlobCache, + {/* charged = */ CacheEntryRoleOptions::Decision::kEnabled}}); + options_.table_factory.reset( + NewBlockBasedTableFactory(block_based_options)); + + assert(db_->GetDbIdentity(db_id_).ok()); + assert(db_->GetDbSessionId(db_session_id_).ok()); + } + + void GenerateKeysAndBlobs() { + for (size_t i = 0; i < kNumBlobs; ++i) { + key_strs_.push_back("key" + std::to_string(i)); + blob_strs_.push_back("blob" + std::to_string(i)); + } + + blob_file_size_ = BlobLogHeader::kSize; + for (size_t i = 0; i < kNumBlobs; ++i) { + keys_.push_back({key_strs_[i]}); + blobs_.push_back({blob_strs_[i]}); + blob_file_size_ += + BlobLogRecord::kHeaderSize + keys_[i].size() + blobs_[i].size(); + } + blob_file_size_ += BlobLogFooter::kSize; + } + + static constexpr std::size_t kSizeDummyEntry = CacheReservationManagerImpl< + CacheEntryRole::kBlobCache>::GetDummyEntrySize(); + static constexpr std::size_t kCacheCapacity = 2 * kSizeDummyEntry; + static constexpr int kNumShardBits = 0; // 2^0 shard + + static constexpr uint32_t kColumnFamilyId = 1; + static constexpr bool kHasTTL = false; + static constexpr uint64_t kBlobFileNumber = 1; + static constexpr size_t kNumBlobs = 16; + + std::vector keys_; + std::vector blobs_; + std::vector key_strs_; + std::vector blob_strs_; + uint64_t blob_file_size_; + + Options options_; + std::string db_id_; + std::string db_session_id_; +}; + +TEST_F(BlobSourceCacheReservationTest, SimpleCacheReservation) { + options_.cf_paths.emplace_back( + test::PerThreadDBPath( + env_, "BlobSourceCacheReservationTest_SimpleCacheReservation"), + 0); + + GenerateKeysAndBlobs(); + + DestroyAndReopen(options_); + + ImmutableOptions immutable_options(options_); + + constexpr ExpirationRange expiration_range; + + std::vector blob_offsets(keys_.size()); + std::vector blob_sizes(keys_.size()); + + WriteBlobFile(immutable_options, kColumnFamilyId, kHasTTL, expiration_range, + expiration_range, kBlobFileNumber, keys_, blobs_, + kNoCompression, blob_offsets, blob_sizes); + + constexpr size_t capacity = 10; + std::shared_ptr backing_cache = NewLRUCache(capacity); + + FileOptions file_options; + constexpr HistogramImpl* blob_file_read_hist = nullptr; + + std::unique_ptr blob_file_cache = + std::make_unique( + backing_cache.get(), &immutable_options, &file_options, + kColumnFamilyId, blob_file_read_hist, nullptr /*IOTracer*/); + + BlobSource blob_source(&immutable_options, db_id_, db_session_id_, + blob_file_cache.get()); + + ConcurrentCacheReservationManager* cache_res_mgr = + static_cast(blob_source.GetBlobCache()) + ->TEST_GetCacheReservationManager(); + ASSERT_NE(cache_res_mgr, nullptr); + + ReadOptions read_options; + read_options.verify_checksums = true; + + { + read_options.fill_cache = false; + + std::vector values(keys_.size()); + + for (size_t i = 0; i < kNumBlobs; ++i) { + ASSERT_OK(blob_source.GetBlob( + read_options, keys_[i], kBlobFileNumber, blob_offsets[i], + blob_file_size_, blob_sizes[i], kNoCompression, + nullptr /* prefetch_buffer */, &values[i], nullptr /* bytes_read */)); + ASSERT_EQ(cache_res_mgr->GetTotalReservedCacheSize(), 0); + ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(), 0); + } + } + + { + read_options.fill_cache = true; + + std::vector values(keys_.size()); + + // num_blobs is 16, so the total blob cache usage is less than a single + // dummy entry. Therefore, cache reservation manager only reserves one dummy + // entry here. + uint64_t blob_bytes = 0; + for (size_t i = 0; i < kNumBlobs; ++i) { + ASSERT_OK(blob_source.GetBlob( + read_options, keys_[i], kBlobFileNumber, blob_offsets[i], + blob_file_size_, blob_sizes[i], kNoCompression, + nullptr /* prefetch_buffer */, &values[i], nullptr /* bytes_read */)); + + size_t charge = 0; + ASSERT_TRUE(blob_source.TEST_BlobInCache(kBlobFileNumber, blob_file_size_, + blob_offsets[i], &charge)); + + blob_bytes += charge; + ASSERT_EQ(cache_res_mgr->GetTotalReservedCacheSize(), kSizeDummyEntry); + ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(), blob_bytes); + ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(), + options_.blob_cache->GetUsage()); + } + } + + { + OffsetableCacheKey base_cache_key(db_id_, db_session_id_, kBlobFileNumber); + size_t blob_bytes = options_.blob_cache->GetUsage(); + + for (size_t i = 0; i < kNumBlobs; ++i) { + size_t charge = 0; + ASSERT_TRUE(blob_source.TEST_BlobInCache(kBlobFileNumber, blob_file_size_, + blob_offsets[i], &charge)); + + CacheKey cache_key = base_cache_key.WithOffset(blob_offsets[i]); + // We didn't call options_.blob_cache->Erase() here, this is because + // the cache wrapper's Erase() method must be called to update the + // cache usage after erasing the cache entry. + blob_source.GetBlobCache()->Erase(cache_key.AsSlice()); + if (i == kNumBlobs - 1) { + // All the blobs got removed from the cache. cache_res_mgr should not + // reserve any space for them. + ASSERT_EQ(cache_res_mgr->GetTotalReservedCacheSize(), 0); + } else { + ASSERT_EQ(cache_res_mgr->GetTotalReservedCacheSize(), kSizeDummyEntry); + } + blob_bytes -= charge; + ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(), blob_bytes); + ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(), + options_.blob_cache->GetUsage()); + } + } +} + +TEST_F(BlobSourceCacheReservationTest, IncreaseCacheReservation) { + options_.cf_paths.emplace_back( + test::PerThreadDBPath( + env_, "BlobSourceCacheReservationTest_IncreaseCacheReservation"), + 0); + + GenerateKeysAndBlobs(); + + DestroyAndReopen(options_); + + ImmutableOptions immutable_options(options_); + constexpr size_t blob_size = 24 << 10; // 24KB + for (size_t i = 0; i < kNumBlobs; ++i) { + blob_file_size_ -= blobs_[i].size(); // old blob size + blob_strs_[i].resize(blob_size, '@'); + blobs_[i] = Slice(blob_strs_[i]); + blob_file_size_ += blobs_[i].size(); // new blob size + } + + std::vector blob_offsets(keys_.size()); + std::vector blob_sizes(keys_.size()); + + constexpr ExpirationRange expiration_range; + WriteBlobFile(immutable_options, kColumnFamilyId, kHasTTL, expiration_range, + expiration_range, kBlobFileNumber, keys_, blobs_, + kNoCompression, blob_offsets, blob_sizes); + + constexpr size_t capacity = 10; + std::shared_ptr backing_cache = NewLRUCache(capacity); + + FileOptions file_options; + constexpr HistogramImpl* blob_file_read_hist = nullptr; + + std::unique_ptr blob_file_cache = + std::make_unique( + backing_cache.get(), &immutable_options, &file_options, + kColumnFamilyId, blob_file_read_hist, nullptr /*IOTracer*/); + + BlobSource blob_source(&immutable_options, db_id_, db_session_id_, + blob_file_cache.get()); + + ConcurrentCacheReservationManager* cache_res_mgr = + static_cast(blob_source.GetBlobCache()) + ->TEST_GetCacheReservationManager(); + ASSERT_NE(cache_res_mgr, nullptr); + + ReadOptions read_options; + read_options.verify_checksums = true; + + { + read_options.fill_cache = false; + + std::vector values(keys_.size()); + + for (size_t i = 0; i < kNumBlobs; ++i) { + ASSERT_OK(blob_source.GetBlob( + read_options, keys_[i], kBlobFileNumber, blob_offsets[i], + blob_file_size_, blob_sizes[i], kNoCompression, + nullptr /* prefetch_buffer */, &values[i], nullptr /* bytes_read */)); + ASSERT_EQ(cache_res_mgr->GetTotalReservedCacheSize(), 0); + ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(), 0); + } + } + + { + read_options.fill_cache = true; + + std::vector values(keys_.size()); + + uint64_t blob_bytes = 0; + for (size_t i = 0; i < kNumBlobs; ++i) { + ASSERT_OK(blob_source.GetBlob( + read_options, keys_[i], kBlobFileNumber, blob_offsets[i], + blob_file_size_, blob_sizes[i], kNoCompression, + nullptr /* prefetch_buffer */, &values[i], nullptr /* bytes_read */)); + + // Release cache handle + values[i].Reset(); + + size_t charge = 0; + ASSERT_TRUE(blob_source.TEST_BlobInCache(kBlobFileNumber, blob_file_size_, + blob_offsets[i], &charge)); + + blob_bytes += charge; + + ASSERT_EQ(cache_res_mgr->GetTotalReservedCacheSize(), + (blob_bytes <= kSizeDummyEntry) ? kSizeDummyEntry + : (2 * kSizeDummyEntry)); + ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(), blob_bytes); + ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(), + options_.blob_cache->GetUsage()); + } + } +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/blob/db_blob_basic_test.cc b/librocksdb-sys/rocksdb/db/blob/db_blob_basic_test.cc new file mode 100644 index 0000000..c6c6d2b --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/db_blob_basic_test.cc @@ -0,0 +1,2237 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include +#include +#include + +#include "cache/compressed_secondary_cache.h" +#include "db/blob/blob_index.h" +#include "db/blob/blob_log_format.h" +#include "db/db_test_util.h" +#include "db/db_with_timestamp_test_util.h" +#include "port/stack_trace.h" +#include "test_util/sync_point.h" +#include "utilities/fault_injection_env.h" + +namespace ROCKSDB_NAMESPACE { + +class DBBlobBasicTest : public DBTestBase { + protected: + DBBlobBasicTest() + : DBTestBase("db_blob_basic_test", /* env_do_fsync */ false) {} +}; + +TEST_F(DBBlobBasicTest, GetBlob) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + + Reopen(options); + + constexpr char key[] = "key"; + constexpr char blob_value[] = "blob_value"; + + ASSERT_OK(Put(key, blob_value)); + + ASSERT_OK(Flush()); + + ASSERT_EQ(Get(key), blob_value); + + // Try again with no I/O allowed. The table and the necessary blocks should + // already be in their respective caches; however, the blob itself can only be + // read from the blob file, so the read should return Incomplete. + ReadOptions read_options; + read_options.read_tier = kBlockCacheTier; + + PinnableSlice result; + ASSERT_TRUE(db_->Get(read_options, db_->DefaultColumnFamily(), key, &result) + .IsIncomplete()); +} + +TEST_F(DBBlobBasicTest, GetBlobFromCache) { + Options options = GetDefaultOptions(); + + LRUCacheOptions co; + co.capacity = 2 << 20; // 2MB + co.num_shard_bits = 2; + co.metadata_charge_policy = kDontChargeCacheMetadata; + auto backing_cache = NewLRUCache(co); + + options.enable_blob_files = true; + options.blob_cache = backing_cache; + + BlockBasedTableOptions block_based_options; + block_based_options.no_block_cache = false; + block_based_options.block_cache = backing_cache; + block_based_options.cache_index_and_filter_blocks = true; + options.table_factory.reset(NewBlockBasedTableFactory(block_based_options)); + + Reopen(options); + + constexpr char key[] = "key"; + constexpr char blob_value[] = "blob_value"; + + ASSERT_OK(Put(key, blob_value)); + + ASSERT_OK(Flush()); + + ReadOptions read_options; + + read_options.fill_cache = false; + + { + PinnableSlice result; + + read_options.read_tier = kReadAllTier; + ASSERT_OK(db_->Get(read_options, db_->DefaultColumnFamily(), key, &result)); + ASSERT_EQ(result, blob_value); + + result.Reset(); + read_options.read_tier = kBlockCacheTier; + + // Try again with no I/O allowed. Since we didn't re-fill the cache, the + // blob itself can only be read from the blob file, so the read should + // return Incomplete. + ASSERT_TRUE(db_->Get(read_options, db_->DefaultColumnFamily(), key, &result) + .IsIncomplete()); + ASSERT_TRUE(result.empty()); + } + + read_options.fill_cache = true; + + { + PinnableSlice result; + + read_options.read_tier = kReadAllTier; + ASSERT_OK(db_->Get(read_options, db_->DefaultColumnFamily(), key, &result)); + ASSERT_EQ(result, blob_value); + + result.Reset(); + read_options.read_tier = kBlockCacheTier; + + // Try again with no I/O allowed. The table and the necessary blocks/blobs + // should already be in their respective caches. + ASSERT_OK(db_->Get(read_options, db_->DefaultColumnFamily(), key, &result)); + ASSERT_EQ(result, blob_value); + } +} + +TEST_F(DBBlobBasicTest, IterateBlobsFromCache) { + Options options = GetDefaultOptions(); + + LRUCacheOptions co; + co.capacity = 2 << 20; // 2MB + co.num_shard_bits = 2; + co.metadata_charge_policy = kDontChargeCacheMetadata; + auto backing_cache = NewLRUCache(co); + + options.enable_blob_files = true; + options.blob_cache = backing_cache; + + BlockBasedTableOptions block_based_options; + block_based_options.no_block_cache = false; + block_based_options.block_cache = backing_cache; + block_based_options.cache_index_and_filter_blocks = true; + options.table_factory.reset(NewBlockBasedTableFactory(block_based_options)); + + options.statistics = CreateDBStatistics(); + + Reopen(options); + + int num_blobs = 5; + std::vector keys; + std::vector blobs; + + for (int i = 0; i < num_blobs; ++i) { + keys.push_back("key" + std::to_string(i)); + blobs.push_back("blob" + std::to_string(i)); + ASSERT_OK(Put(keys[i], blobs[i])); + } + ASSERT_OK(Flush()); + + ReadOptions read_options; + + { + read_options.fill_cache = false; + read_options.read_tier = kReadAllTier; + + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_OK(iter->status()); + + int i = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key().ToString(), keys[i]); + ASSERT_EQ(iter->value().ToString(), blobs[i]); + ++i; + } + ASSERT_EQ(i, num_blobs); + ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD), 0); + } + + { + read_options.fill_cache = false; + read_options.read_tier = kBlockCacheTier; + + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_OK(iter->status()); + + // Try again with no I/O allowed. Since we didn't re-fill the cache, + // the blob itself can only be read from the blob file, so iter->Valid() + // should be false. + iter->SeekToFirst(); + ASSERT_NOK(iter->status()); + ASSERT_FALSE(iter->Valid()); + ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD), 0); + } + + { + read_options.fill_cache = true; + read_options.read_tier = kReadAllTier; + + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_OK(iter->status()); + + // Read blobs from the file and refill the cache. + int i = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key().ToString(), keys[i]); + ASSERT_EQ(iter->value().ToString(), blobs[i]); + ++i; + } + ASSERT_EQ(i, num_blobs); + ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD), + num_blobs); + } + + { + read_options.fill_cache = false; + read_options.read_tier = kBlockCacheTier; + + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_OK(iter->status()); + + // Try again with no I/O allowed. The table and the necessary blocks/blobs + // should already be in their respective caches. + int i = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key().ToString(), keys[i]); + ASSERT_EQ(iter->value().ToString(), blobs[i]); + ++i; + } + ASSERT_EQ(i, num_blobs); + ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD), 0); + } +} + +TEST_F(DBBlobBasicTest, IterateBlobsFromCachePinning) { + constexpr size_t min_blob_size = 6; + + Options options = GetDefaultOptions(); + + LRUCacheOptions cache_options; + cache_options.capacity = 2048; + cache_options.num_shard_bits = 0; + cache_options.metadata_charge_policy = kDontChargeCacheMetadata; + + options.blob_cache = NewLRUCache(cache_options); + options.enable_blob_files = true; + options.min_blob_size = min_blob_size; + + Reopen(options); + + // Put then iterate over three key-values. The second value is below the size + // limit and is thus stored inline; the other two are stored separately as + // blobs. We expect to have something pinned in the cache iff we are + // positioned on a blob. + + constexpr char first_key[] = "first_key"; + constexpr char first_value[] = "long_value"; + static_assert(sizeof(first_value) - 1 >= min_blob_size, + "first_value too short to be stored as blob"); + + ASSERT_OK(Put(first_key, first_value)); + + constexpr char second_key[] = "second_key"; + constexpr char second_value[] = "short"; + static_assert(sizeof(second_value) - 1 < min_blob_size, + "second_value too long to be inlined"); + + ASSERT_OK(Put(second_key, second_value)); + + constexpr char third_key[] = "third_key"; + constexpr char third_value[] = "other_long_value"; + static_assert(sizeof(third_value) - 1 >= min_blob_size, + "third_value too short to be stored as blob"); + + ASSERT_OK(Put(third_key, third_value)); + + ASSERT_OK(Flush()); + + { + ReadOptions read_options; + read_options.fill_cache = true; + + std::unique_ptr iter(db_->NewIterator(read_options)); + + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), first_key); + ASSERT_EQ(iter->value(), first_value); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), second_key); + ASSERT_EQ(iter->value(), second_value); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), third_key); + ASSERT_EQ(iter->value(), third_value); + + iter->Next(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + } + + { + ReadOptions read_options; + read_options.fill_cache = false; + read_options.read_tier = kBlockCacheTier; + + std::unique_ptr iter(db_->NewIterator(read_options)); + + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), first_key); + ASSERT_EQ(iter->value(), first_value); + ASSERT_GT(options.blob_cache->GetPinnedUsage(), 0); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), second_key); + ASSERT_EQ(iter->value(), second_value); + ASSERT_EQ(options.blob_cache->GetPinnedUsage(), 0); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), third_key); + ASSERT_EQ(iter->value(), third_value); + ASSERT_GT(options.blob_cache->GetPinnedUsage(), 0); + + iter->Next(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(options.blob_cache->GetPinnedUsage(), 0); + } + + { + ReadOptions read_options; + read_options.fill_cache = false; + read_options.read_tier = kBlockCacheTier; + + std::unique_ptr iter(db_->NewIterator(read_options)); + + iter->SeekToLast(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), third_key); + ASSERT_EQ(iter->value(), third_value); + ASSERT_GT(options.blob_cache->GetPinnedUsage(), 0); + + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), second_key); + ASSERT_EQ(iter->value(), second_value); + ASSERT_EQ(options.blob_cache->GetPinnedUsage(), 0); + + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), first_key); + ASSERT_EQ(iter->value(), first_value); + ASSERT_GT(options.blob_cache->GetPinnedUsage(), 0); + + iter->Prev(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(options.blob_cache->GetPinnedUsage(), 0); + } +} + +TEST_F(DBBlobBasicTest, MultiGetBlobs) { + constexpr size_t min_blob_size = 6; + + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = min_blob_size; + + Reopen(options); + + // Put then retrieve three key-values. The first value is below the size limit + // and is thus stored inline; the other two are stored separately as blobs. + constexpr size_t num_keys = 3; + + constexpr char first_key[] = "first_key"; + constexpr char first_value[] = "short"; + static_assert(sizeof(first_value) - 1 < min_blob_size, + "first_value too long to be inlined"); + + ASSERT_OK(Put(first_key, first_value)); + + constexpr char second_key[] = "second_key"; + constexpr char second_value[] = "long_value"; + static_assert(sizeof(second_value) - 1 >= min_blob_size, + "second_value too short to be stored as blob"); + + ASSERT_OK(Put(second_key, second_value)); + + constexpr char third_key[] = "third_key"; + constexpr char third_value[] = "other_long_value"; + static_assert(sizeof(third_value) - 1 >= min_blob_size, + "third_value too short to be stored as blob"); + + ASSERT_OK(Put(third_key, third_value)); + + ASSERT_OK(Flush()); + + ReadOptions read_options; + + std::array keys{{first_key, second_key, third_key}}; + + { + std::array values; + std::array statuses; + + db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys, &keys[0], + &values[0], &statuses[0]); + + ASSERT_OK(statuses[0]); + ASSERT_EQ(values[0], first_value); + + ASSERT_OK(statuses[1]); + ASSERT_EQ(values[1], second_value); + + ASSERT_OK(statuses[2]); + ASSERT_EQ(values[2], third_value); + } + + // Try again with no I/O allowed. The table and the necessary blocks should + // already be in their respective caches. The first (inlined) value should be + // successfully read; however, the two blob values could only be read from the + // blob file, so for those the read should return Incomplete. + read_options.read_tier = kBlockCacheTier; + + { + std::array values; + std::array statuses; + + db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys, &keys[0], + &values[0], &statuses[0]); + + ASSERT_OK(statuses[0]); + ASSERT_EQ(values[0], first_value); + + ASSERT_TRUE(statuses[1].IsIncomplete()); + + ASSERT_TRUE(statuses[2].IsIncomplete()); + } +} + +TEST_F(DBBlobBasicTest, MultiGetBlobsFromCache) { + Options options = GetDefaultOptions(); + + LRUCacheOptions co; + co.capacity = 2 << 20; // 2MB + co.num_shard_bits = 2; + co.metadata_charge_policy = kDontChargeCacheMetadata; + auto backing_cache = NewLRUCache(co); + + constexpr size_t min_blob_size = 6; + options.min_blob_size = min_blob_size; + options.create_if_missing = true; + options.enable_blob_files = true; + options.blob_cache = backing_cache; + + BlockBasedTableOptions block_based_options; + block_based_options.no_block_cache = false; + block_based_options.block_cache = backing_cache; + block_based_options.cache_index_and_filter_blocks = true; + options.table_factory.reset(NewBlockBasedTableFactory(block_based_options)); + + DestroyAndReopen(options); + + // Put then retrieve three key-values. The first value is below the size limit + // and is thus stored inline; the other two are stored separately as blobs. + constexpr size_t num_keys = 3; + + constexpr char first_key[] = "first_key"; + constexpr char first_value[] = "short"; + static_assert(sizeof(first_value) - 1 < min_blob_size, + "first_value too long to be inlined"); + + ASSERT_OK(Put(first_key, first_value)); + + constexpr char second_key[] = "second_key"; + constexpr char second_value[] = "long_value"; + static_assert(sizeof(second_value) - 1 >= min_blob_size, + "second_value too short to be stored as blob"); + + ASSERT_OK(Put(second_key, second_value)); + + constexpr char third_key[] = "third_key"; + constexpr char third_value[] = "other_long_value"; + static_assert(sizeof(third_value) - 1 >= min_blob_size, + "third_value too short to be stored as blob"); + + ASSERT_OK(Put(third_key, third_value)); + + ASSERT_OK(Flush()); + + ReadOptions read_options; + read_options.fill_cache = false; + + std::array keys{{first_key, second_key, third_key}}; + + { + std::array values; + std::array statuses; + + db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys, &keys[0], + &values[0], &statuses[0]); + + ASSERT_OK(statuses[0]); + ASSERT_EQ(values[0], first_value); + + ASSERT_OK(statuses[1]); + ASSERT_EQ(values[1], second_value); + + ASSERT_OK(statuses[2]); + ASSERT_EQ(values[2], third_value); + } + + // Try again with no I/O allowed. The first (inlined) value should be + // successfully read; however, the two blob values could only be read from the + // blob file, so for those the read should return Incomplete. + read_options.read_tier = kBlockCacheTier; + + { + std::array values; + std::array statuses; + + db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys, &keys[0], + &values[0], &statuses[0]); + + ASSERT_OK(statuses[0]); + ASSERT_EQ(values[0], first_value); + + ASSERT_TRUE(statuses[1].IsIncomplete()); + + ASSERT_TRUE(statuses[2].IsIncomplete()); + } + + // Fill the cache when reading blobs from the blob file. + read_options.read_tier = kReadAllTier; + read_options.fill_cache = true; + + { + std::array values; + std::array statuses; + + db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys, &keys[0], + &values[0], &statuses[0]); + + ASSERT_OK(statuses[0]); + ASSERT_EQ(values[0], first_value); + + ASSERT_OK(statuses[1]); + ASSERT_EQ(values[1], second_value); + + ASSERT_OK(statuses[2]); + ASSERT_EQ(values[2], third_value); + } + + // Try again with no I/O allowed. All blobs should be successfully read from + // the cache. + read_options.read_tier = kBlockCacheTier; + + { + std::array values; + std::array statuses; + + db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys, &keys[0], + &values[0], &statuses[0]); + + ASSERT_OK(statuses[0]); + ASSERT_EQ(values[0], first_value); + + ASSERT_OK(statuses[1]); + ASSERT_EQ(values[1], second_value); + + ASSERT_OK(statuses[2]); + ASSERT_EQ(values[2], third_value); + } +} + +TEST_F(DBBlobBasicTest, MultiGetWithDirectIO) { + Options options = GetDefaultOptions(); + + // First, create an external SST file ["b"]. + const std::string file_path = dbname_ + "/test.sst"; + { + SstFileWriter sst_file_writer(EnvOptions(), GetDefaultOptions()); + Status s = sst_file_writer.Open(file_path); + ASSERT_OK(s); + ASSERT_OK(sst_file_writer.Put("b", "b_value")); + ASSERT_OK(sst_file_writer.Finish()); + } + + options.enable_blob_files = true; + options.min_blob_size = 1000; + options.use_direct_reads = true; + options.allow_ingest_behind = true; + + // Open DB with fixed-prefix sst-partitioner so that compaction will cut + // new table file when encountering a new key whose 1-byte prefix changes. + constexpr size_t key_len = 1; + options.sst_partitioner_factory = + NewSstPartitionerFixedPrefixFactory(key_len); + + Status s = TryReopen(options); + if (s.IsInvalidArgument()) { + ROCKSDB_GTEST_SKIP("This test requires direct IO support"); + return; + } + ASSERT_OK(s); + + constexpr size_t num_keys = 3; + constexpr size_t blob_size = 3000; + + constexpr char first_key[] = "a"; + const std::string first_blob(blob_size, 'a'); + ASSERT_OK(Put(first_key, first_blob)); + + constexpr char second_key[] = "b"; + const std::string second_blob(2 * blob_size, 'b'); + ASSERT_OK(Put(second_key, second_blob)); + + constexpr char third_key[] = "d"; + const std::string third_blob(blob_size, 'd'); + ASSERT_OK(Put(third_key, third_blob)); + + // first_blob, second_blob and third_blob in the same blob file. + // SST Blob file + // L0 ["a", "b", "d"] |'aaaa', 'bbbb', 'dddd'| + // | | | ^ ^ ^ + // | | | | | | + // | | +---------|-------|--------+ + // | +-----------------|-------+ + // +-------------------------+ + ASSERT_OK(Flush()); + + constexpr char fourth_key[] = "c"; + const std::string fourth_blob(blob_size, 'c'); + ASSERT_OK(Put(fourth_key, fourth_blob)); + // fourth_blob in another blob file. + // SST Blob file SST Blob file + // L0 ["a", "b", "d"] |'aaaa', 'bbbb', 'dddd'| ["c"] |'cccc'| + // | | | ^ ^ ^ | ^ + // | | | | | | | | + // | | +---------|-------|--------+ +-------+ + // | +-----------------|-------+ + // +-------------------------+ + ASSERT_OK(Flush()); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr, + /*end=*/nullptr)); + + // Due to the above sst partitioner, we get 4 L1 files. The blob files are + // unchanged. + // |'aaaa', 'bbbb', 'dddd'| |'cccc'| + // ^ ^ ^ ^ + // | | | | + // L0 | | | | + // L1 ["a"] ["b"] ["c"] | | ["d"] | + // | | | | | | + // | | +---------|-------|---------------+ + // | +-----------------|-------+ + // +-------------------------+ + ASSERT_EQ(4, NumTableFilesAtLevel(/*level=*/1)); + + { + // Ingest the external SST file into bottommost level. + std::vector ext_files{file_path}; + IngestExternalFileOptions opts; + opts.ingest_behind = true; + ASSERT_OK( + db_->IngestExternalFile(db_->DefaultColumnFamily(), ext_files, opts)); + } + + // Now the database becomes as follows. + // |'aaaa', 'bbbb', 'dddd'| |'cccc'| + // ^ ^ ^ ^ + // | | | | + // L0 | | | | + // L1 ["a"] ["b"] ["c"] | | ["d"] | + // | | | | | | + // | | +---------|-------|---------------+ + // | +-----------------|-------+ + // +-------------------------+ + // + // L6 ["b"] + + { + // Compact ["b"] to bottommost level. + Slice begin = Slice(second_key); + Slice end = Slice(second_key); + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(cro, &begin, &end)); + } + + // |'aaaa', 'bbbb', 'dddd'| |'cccc'| + // ^ ^ ^ ^ + // | | | | + // L0 | | | | + // L1 ["a"] ["c"] | | ["d"] | + // | | | | | + // | +---------|-------|---------------+ + // | +-----------------|-------+ + // +-------|-----------------+ + // | + // L6 ["b"] + ASSERT_EQ(3, NumTableFilesAtLevel(/*level=*/1)); + ASSERT_EQ(1, NumTableFilesAtLevel(/*level=*/6)); + + bool called = false; + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "RandomAccessFileReader::MultiRead:AlignedReqs", [&](void* arg) { + auto* aligned_reqs = static_cast*>(arg); + assert(aligned_reqs); + ASSERT_EQ(1, aligned_reqs->size()); + called = true; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + std::array keys{{first_key, third_key, second_key}}; + + { + std::array values; + std::array statuses; + + // The MultiGet(), when constructing the KeyContexts, will process the keys + // in such order: a, d, b. The reason is that ["a"] and ["d"] are in L1, + // while ["b"] resides in L6. + // Consequently, the original FSReadRequest list prepared by + // Version::MultiGetblob() will be for "a", "d" and "b". It is unsorted as + // follows: + // + // ["a", offset=30, len=3033], + // ["d", offset=9096, len=3033], + // ["b", offset=3063, len=6033] + // + // If we do not sort them before calling MultiRead() in DirectIO, then the + // underlying IO merging logic will yield two requests. + // + // [offset=0, len=4096] (for "a") + // [offset=0, len=12288] (result of merging the request for "d" and "b") + // + // We need to sort them in Version::MultiGetBlob() so that the underlying + // IO merging logic in DirectIO mode works as expected. The correct + // behavior will be one aligned request: + // + // [offset=0, len=12288] + + db_->MultiGet(ReadOptions(), db_->DefaultColumnFamily(), num_keys, &keys[0], + &values[0], &statuses[0]); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + ASSERT_TRUE(called); + + ASSERT_OK(statuses[0]); + ASSERT_EQ(values[0], first_blob); + + ASSERT_OK(statuses[1]); + ASSERT_EQ(values[1], third_blob); + + ASSERT_OK(statuses[2]); + ASSERT_EQ(values[2], second_blob); + } +} + +TEST_F(DBBlobBasicTest, MultiGetBlobsFromMultipleFiles) { + Options options = GetDefaultOptions(); + + LRUCacheOptions co; + co.capacity = 2 << 20; // 2MB + co.num_shard_bits = 2; + co.metadata_charge_policy = kDontChargeCacheMetadata; + auto backing_cache = NewLRUCache(co); + + options.min_blob_size = 0; + options.create_if_missing = true; + options.enable_blob_files = true; + options.blob_cache = backing_cache; + + BlockBasedTableOptions block_based_options; + block_based_options.no_block_cache = false; + block_based_options.block_cache = backing_cache; + block_based_options.cache_index_and_filter_blocks = true; + options.table_factory.reset(NewBlockBasedTableFactory(block_based_options)); + + Reopen(options); + + constexpr size_t kNumBlobFiles = 3; + constexpr size_t kNumBlobsPerFile = 3; + constexpr size_t kNumKeys = kNumBlobsPerFile * kNumBlobFiles; + + std::vector key_strs; + std::vector value_strs; + for (size_t i = 0; i < kNumBlobFiles; ++i) { + for (size_t j = 0; j < kNumBlobsPerFile; ++j) { + std::string key = "key" + std::to_string(i) + "_" + std::to_string(j); + std::string value = + "value_as_blob" + std::to_string(i) + "_" + std::to_string(j); + ASSERT_OK(Put(key, value)); + key_strs.push_back(key); + value_strs.push_back(value); + } + ASSERT_OK(Flush()); + } + assert(key_strs.size() == kNumKeys); + std::array keys; + for (size_t i = 0; i < keys.size(); ++i) { + keys[i] = key_strs[i]; + } + + ReadOptions read_options; + read_options.read_tier = kReadAllTier; + read_options.fill_cache = false; + + { + std::array values; + std::array statuses; + db_->MultiGet(read_options, db_->DefaultColumnFamily(), kNumKeys, &keys[0], + &values[0], &statuses[0]); + + for (size_t i = 0; i < kNumKeys; ++i) { + ASSERT_OK(statuses[i]); + ASSERT_EQ(value_strs[i], values[i]); + } + } + + read_options.read_tier = kBlockCacheTier; + + { + std::array values; + std::array statuses; + db_->MultiGet(read_options, db_->DefaultColumnFamily(), kNumKeys, &keys[0], + &values[0], &statuses[0]); + + for (size_t i = 0; i < kNumKeys; ++i) { + ASSERT_TRUE(statuses[i].IsIncomplete()); + ASSERT_TRUE(values[i].empty()); + } + } + + read_options.read_tier = kReadAllTier; + read_options.fill_cache = true; + + { + std::array values; + std::array statuses; + db_->MultiGet(read_options, db_->DefaultColumnFamily(), kNumKeys, &keys[0], + &values[0], &statuses[0]); + + for (size_t i = 0; i < kNumKeys; ++i) { + ASSERT_OK(statuses[i]); + ASSERT_EQ(value_strs[i], values[i]); + } + } + + read_options.read_tier = kBlockCacheTier; + + { + std::array values; + std::array statuses; + db_->MultiGet(read_options, db_->DefaultColumnFamily(), kNumKeys, &keys[0], + &values[0], &statuses[0]); + + for (size_t i = 0; i < kNumKeys; ++i) { + ASSERT_OK(statuses[i]); + ASSERT_EQ(value_strs[i], values[i]); + } + } +} + +TEST_F(DBBlobBasicTest, GetBlob_CorruptIndex) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + + Reopen(options); + + constexpr char key[] = "key"; + constexpr char blob[] = "blob"; + + ASSERT_OK(Put(key, blob)); + ASSERT_OK(Flush()); + + SyncPoint::GetInstance()->SetCallBack( + "Version::Get::TamperWithBlobIndex", [](void* arg) { + Slice* const blob_index = static_cast(arg); + assert(blob_index); + assert(!blob_index->empty()); + blob_index->remove_prefix(1); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + PinnableSlice result; + ASSERT_TRUE(db_->Get(ReadOptions(), db_->DefaultColumnFamily(), key, &result) + .IsCorruption()); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(DBBlobBasicTest, MultiGetBlob_CorruptIndex) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + options.create_if_missing = true; + + DestroyAndReopen(options); + + constexpr size_t kNumOfKeys = 3; + std::array key_strs; + std::array value_strs; + std::array keys; + for (size_t i = 0; i < kNumOfKeys; ++i) { + key_strs[i] = "foo" + std::to_string(i); + value_strs[i] = "blob_value" + std::to_string(i); + ASSERT_OK(Put(key_strs[i], value_strs[i])); + keys[i] = key_strs[i]; + } + + constexpr char key[] = "key"; + constexpr char blob[] = "blob"; + ASSERT_OK(Put(key, blob)); + keys[kNumOfKeys] = key; + + ASSERT_OK(Flush()); + + SyncPoint::GetInstance()->SetCallBack( + "Version::MultiGet::TamperWithBlobIndex", [&key](void* arg) { + KeyContext* const key_context = static_cast(arg); + assert(key_context); + assert(key_context->key); + + if (*(key_context->key) == key) { + Slice* const blob_index = key_context->value; + assert(blob_index); + assert(!blob_index->empty()); + blob_index->remove_prefix(1); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + + std::array values; + std::array statuses; + db_->MultiGet(ReadOptions(), dbfull()->DefaultColumnFamily(), kNumOfKeys + 1, + keys.data(), values.data(), statuses.data(), + /*sorted_input=*/false); + for (size_t i = 0; i < kNumOfKeys + 1; ++i) { + if (i != kNumOfKeys) { + ASSERT_OK(statuses[i]); + ASSERT_EQ("blob_value" + std::to_string(i), values[i]); + } else { + ASSERT_TRUE(statuses[i].IsCorruption()); + } + } + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(DBBlobBasicTest, MultiGetBlob_ExceedSoftLimit) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + + Reopen(options); + + constexpr size_t kNumOfKeys = 3; + std::array key_bufs; + std::array value_bufs; + std::array keys; + for (size_t i = 0; i < kNumOfKeys; ++i) { + key_bufs[i] = "foo" + std::to_string(i); + value_bufs[i] = "blob_value" + std::to_string(i); + ASSERT_OK(Put(key_bufs[i], value_bufs[i])); + keys[i] = key_bufs[i]; + } + ASSERT_OK(Flush()); + + std::array values; + std::array statuses; + ReadOptions read_opts; + read_opts.value_size_soft_limit = 1; + db_->MultiGet(read_opts, dbfull()->DefaultColumnFamily(), kNumOfKeys, + keys.data(), values.data(), statuses.data(), + /*sorted_input=*/true); + for (const auto& s : statuses) { + ASSERT_TRUE(s.IsAborted()); + } +} + +TEST_F(DBBlobBasicTest, GetBlob_InlinedTTLIndex) { + constexpr uint64_t min_blob_size = 10; + + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = min_blob_size; + + Reopen(options); + + constexpr char key[] = "key"; + constexpr char blob[] = "short"; + static_assert(sizeof(short) - 1 < min_blob_size, + "Blob too long to be inlined"); + + // Fake an inlined TTL blob index. + std::string blob_index; + + constexpr uint64_t expiration = 1234567890; + + BlobIndex::EncodeInlinedTTL(&blob_index, expiration, blob); + + WriteBatch batch; + ASSERT_OK(WriteBatchInternal::PutBlobIndex(&batch, 0, key, blob_index)); + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + + ASSERT_OK(Flush()); + + PinnableSlice result; + ASSERT_TRUE(db_->Get(ReadOptions(), db_->DefaultColumnFamily(), key, &result) + .IsCorruption()); +} + +TEST_F(DBBlobBasicTest, GetBlob_IndexWithInvalidFileNumber) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + + Reopen(options); + + constexpr char key[] = "key"; + + // Fake a blob index referencing a non-existent blob file. + std::string blob_index; + + constexpr uint64_t blob_file_number = 1000; + constexpr uint64_t offset = 1234; + constexpr uint64_t size = 5678; + + BlobIndex::EncodeBlob(&blob_index, blob_file_number, offset, size, + kNoCompression); + + WriteBatch batch; + ASSERT_OK(WriteBatchInternal::PutBlobIndex(&batch, 0, key, blob_index)); + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + + ASSERT_OK(Flush()); + + PinnableSlice result; + ASSERT_TRUE(db_->Get(ReadOptions(), db_->DefaultColumnFamily(), key, &result) + .IsCorruption()); +} + +TEST_F(DBBlobBasicTest, GenerateIOTracing) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + std::string trace_file = dbname_ + "/io_trace_file"; + + Reopen(options); + { + // Create IO trace file + std::unique_ptr trace_writer; + ASSERT_OK( + NewFileTraceWriter(env_, EnvOptions(), trace_file, &trace_writer)); + ASSERT_OK(db_->StartIOTrace(TraceOptions(), std::move(trace_writer))); + + constexpr char key[] = "key"; + constexpr char blob_value[] = "blob_value"; + + ASSERT_OK(Put(key, blob_value)); + ASSERT_OK(Flush()); + ASSERT_EQ(Get(key), blob_value); + + ASSERT_OK(db_->EndIOTrace()); + ASSERT_OK(env_->FileExists(trace_file)); + } + { + // Parse trace file to check file operations related to blob files are + // recorded. + std::unique_ptr trace_reader; + ASSERT_OK( + NewFileTraceReader(env_, EnvOptions(), trace_file, &trace_reader)); + IOTraceReader reader(std::move(trace_reader)); + + IOTraceHeader header; + ASSERT_OK(reader.ReadHeader(&header)); + ASSERT_EQ(kMajorVersion, static_cast(header.rocksdb_major_version)); + ASSERT_EQ(kMinorVersion, static_cast(header.rocksdb_minor_version)); + + // Read records. + int blob_files_op_count = 0; + Status status; + while (true) { + IOTraceRecord record; + status = reader.ReadIOOp(&record); + if (!status.ok()) { + break; + } + if (record.file_name.find("blob") != std::string::npos) { + blob_files_op_count++; + } + } + // Assuming blob files will have Append, Close and then Read operations. + ASSERT_GT(blob_files_op_count, 2); + } +} + +TEST_F(DBBlobBasicTest, BestEffortsRecovery_MissingNewestBlobFile) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + options.create_if_missing = true; + Reopen(options); + + ASSERT_OK(dbfull()->DisableFileDeletions()); + constexpr int kNumTableFiles = 2; + for (int i = 0; i < kNumTableFiles; ++i) { + for (char ch = 'a'; ch != 'c'; ++ch) { + std::string key(1, ch); + ASSERT_OK(Put(key, "value" + std::to_string(i))); + } + ASSERT_OK(Flush()); + } + + Close(); + + std::vector files; + ASSERT_OK(env_->GetChildren(dbname_, &files)); + std::string blob_file_path; + uint64_t max_blob_file_num = kInvalidBlobFileNumber; + for (const auto& fname : files) { + uint64_t file_num = 0; + FileType type; + if (ParseFileName(fname, &file_num, /*info_log_name_prefix=*/"", &type) && + type == kBlobFile) { + if (file_num > max_blob_file_num) { + max_blob_file_num = file_num; + blob_file_path = dbname_ + "/" + fname; + } + } + } + ASSERT_OK(env_->DeleteFile(blob_file_path)); + + options.best_efforts_recovery = true; + Reopen(options); + std::string value; + ASSERT_OK(db_->Get(ReadOptions(), "a", &value)); + ASSERT_EQ("value" + std::to_string(kNumTableFiles - 2), value); +} + +TEST_F(DBBlobBasicTest, GetMergeBlobWithPut) { + Options options = GetDefaultOptions(); + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + options.enable_blob_files = true; + options.min_blob_size = 0; + + Reopen(options); + + ASSERT_OK(Put("Key1", "v1")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("Key1", "v2")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("Key1", "v3")); + ASSERT_OK(Flush()); + + std::string value; + ASSERT_OK(db_->Get(ReadOptions(), "Key1", &value)); + ASSERT_EQ(Get("Key1"), "v1,v2,v3"); +} + +TEST_F(DBBlobBasicTest, MultiGetMergeBlobWithPut) { + constexpr size_t num_keys = 3; + + Options options = GetDefaultOptions(); + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + options.enable_blob_files = true; + options.min_blob_size = 0; + + Reopen(options); + + ASSERT_OK(Put("Key0", "v0_0")); + ASSERT_OK(Put("Key1", "v1_0")); + ASSERT_OK(Put("Key2", "v2_0")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("Key0", "v0_1")); + ASSERT_OK(Merge("Key1", "v1_1")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("Key0", "v0_2")); + ASSERT_OK(Flush()); + + std::array keys{{"Key0", "Key1", "Key2"}}; + std::array values; + std::array statuses; + + db_->MultiGet(ReadOptions(), db_->DefaultColumnFamily(), num_keys, &keys[0], + &values[0], &statuses[0]); + + ASSERT_OK(statuses[0]); + ASSERT_EQ(values[0], "v0_0,v0_1,v0_2"); + + ASSERT_OK(statuses[1]); + ASSERT_EQ(values[1], "v1_0,v1_1"); + + ASSERT_OK(statuses[2]); + ASSERT_EQ(values[2], "v2_0"); +} + +TEST_F(DBBlobBasicTest, Properties) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + + Reopen(options); + + constexpr char key1[] = "key1"; + constexpr size_t key1_size = sizeof(key1) - 1; + + constexpr char key2[] = "key2"; + constexpr size_t key2_size = sizeof(key2) - 1; + + constexpr char key3[] = "key3"; + constexpr size_t key3_size = sizeof(key3) - 1; + + constexpr char blob[] = "00000000000000"; + constexpr size_t blob_size = sizeof(blob) - 1; + + constexpr char longer_blob[] = "00000000000000000000"; + constexpr size_t longer_blob_size = sizeof(longer_blob) - 1; + + ASSERT_OK(Put(key1, blob)); + ASSERT_OK(Put(key2, longer_blob)); + ASSERT_OK(Flush()); + + constexpr size_t first_blob_file_expected_size = + BlobLogHeader::kSize + + BlobLogRecord::CalculateAdjustmentForRecordHeader(key1_size) + blob_size + + BlobLogRecord::CalculateAdjustmentForRecordHeader(key2_size) + + longer_blob_size + BlobLogFooter::kSize; + + ASSERT_OK(Put(key3, blob)); + ASSERT_OK(Flush()); + + constexpr size_t second_blob_file_expected_size = + BlobLogHeader::kSize + + BlobLogRecord::CalculateAdjustmentForRecordHeader(key3_size) + blob_size + + BlobLogFooter::kSize; + + constexpr size_t total_expected_size = + first_blob_file_expected_size + second_blob_file_expected_size; + + // Number of blob files + uint64_t num_blob_files = 0; + ASSERT_TRUE( + db_->GetIntProperty(DB::Properties::kNumBlobFiles, &num_blob_files)); + ASSERT_EQ(num_blob_files, 2); + + // Total size of live blob files + uint64_t live_blob_file_size = 0; + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kLiveBlobFileSize, + &live_blob_file_size)); + ASSERT_EQ(live_blob_file_size, total_expected_size); + + // Total amount of garbage in live blob files + { + uint64_t live_blob_file_garbage_size = 0; + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kLiveBlobFileGarbageSize, + &live_blob_file_garbage_size)); + ASSERT_EQ(live_blob_file_garbage_size, 0); + } + + // Total size of all blob files across all versions + // Note: this should be the same as above since we only have one + // version at this point. + uint64_t total_blob_file_size = 0; + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kTotalBlobFileSize, + &total_blob_file_size)); + ASSERT_EQ(total_blob_file_size, total_expected_size); + + // Delete key2 to create some garbage + ASSERT_OK(Delete(key2)); + ASSERT_OK(Flush()); + + constexpr Slice* begin = nullptr; + constexpr Slice* end = nullptr; + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), begin, end)); + + constexpr size_t expected_garbage_size = + BlobLogRecord::CalculateAdjustmentForRecordHeader(key2_size) + + longer_blob_size; + + constexpr double expected_space_amp = + static_cast(total_expected_size) / + (total_expected_size - expected_garbage_size); + + // Blob file stats + std::string blob_stats; + ASSERT_TRUE(db_->GetProperty(DB::Properties::kBlobStats, &blob_stats)); + + std::ostringstream oss; + oss << "Number of blob files: 2\nTotal size of blob files: " + << total_expected_size + << "\nTotal size of garbage in blob files: " << expected_garbage_size + << "\nBlob file space amplification: " << expected_space_amp << '\n'; + + ASSERT_EQ(blob_stats, oss.str()); + + // Total amount of garbage in live blob files + { + uint64_t live_blob_file_garbage_size = 0; + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kLiveBlobFileGarbageSize, + &live_blob_file_garbage_size)); + ASSERT_EQ(live_blob_file_garbage_size, expected_garbage_size); + } +} + +TEST_F(DBBlobBasicTest, PropertiesMultiVersion) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + + Reopen(options); + + constexpr char key1[] = "key1"; + constexpr char key2[] = "key2"; + constexpr char key3[] = "key3"; + + constexpr size_t key_size = sizeof(key1) - 1; + static_assert(sizeof(key2) - 1 == key_size, "unexpected size: key2"); + static_assert(sizeof(key3) - 1 == key_size, "unexpected size: key3"); + + constexpr char blob[] = "0000000000"; + constexpr size_t blob_size = sizeof(blob) - 1; + + ASSERT_OK(Put(key1, blob)); + ASSERT_OK(Flush()); + + ASSERT_OK(Put(key2, blob)); + ASSERT_OK(Flush()); + + // Create an iterator to keep the current version alive + std::unique_ptr iter(db_->NewIterator(ReadOptions())); + ASSERT_OK(iter->status()); + + // Note: the Delete and subsequent compaction results in the first blob file + // not making it to the final version. (It is still part of the previous + // version kept alive by the iterator though.) On the other hand, the Put + // results in a third blob file. + ASSERT_OK(Delete(key1)); + ASSERT_OK(Put(key3, blob)); + ASSERT_OK(Flush()); + + constexpr Slice* begin = nullptr; + constexpr Slice* end = nullptr; + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), begin, end)); + + // Total size of all blob files across all versions: between the two versions, + // we should have three blob files of the same size with one blob each. + // The version kept alive by the iterator contains the first and the second + // blob file, while the final version contains the second and the third blob + // file. (The second blob file is thus shared by the two versions but should + // be counted only once.) + uint64_t total_blob_file_size = 0; + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kTotalBlobFileSize, + &total_blob_file_size)); + ASSERT_EQ(total_blob_file_size, + 3 * (BlobLogHeader::kSize + + BlobLogRecord::CalculateAdjustmentForRecordHeader(key_size) + + blob_size + BlobLogFooter::kSize)); +} + +class DBBlobBasicIOErrorTest : public DBBlobBasicTest, + public testing::WithParamInterface { + protected: + DBBlobBasicIOErrorTest() : sync_point_(GetParam()) { + fault_injection_env_.reset(new FaultInjectionTestEnv(env_)); + } + ~DBBlobBasicIOErrorTest() { Close(); } + + std::unique_ptr fault_injection_env_; + std::string sync_point_; +}; + +class DBBlobBasicIOErrorMultiGetTest : public DBBlobBasicIOErrorTest { + public: + DBBlobBasicIOErrorMultiGetTest() : DBBlobBasicIOErrorTest() {} +}; + +INSTANTIATE_TEST_CASE_P(DBBlobBasicTest, DBBlobBasicIOErrorTest, + ::testing::ValuesIn(std::vector{ + "BlobFileReader::OpenFile:NewRandomAccessFile", + "BlobFileReader::GetBlob:ReadFromFile"})); + +INSTANTIATE_TEST_CASE_P(DBBlobBasicTest, DBBlobBasicIOErrorMultiGetTest, + ::testing::ValuesIn(std::vector{ + "BlobFileReader::OpenFile:NewRandomAccessFile", + "BlobFileReader::MultiGetBlob:ReadFromFile"})); + +TEST_P(DBBlobBasicIOErrorTest, GetBlob_IOError) { + Options options; + options.env = fault_injection_env_.get(); + options.enable_blob_files = true; + options.min_blob_size = 0; + + Reopen(options); + + constexpr char key[] = "key"; + constexpr char blob_value[] = "blob_value"; + + ASSERT_OK(Put(key, blob_value)); + + ASSERT_OK(Flush()); + + SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* /* arg */) { + fault_injection_env_->SetFilesystemActive(false, + Status::IOError(sync_point_)); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + PinnableSlice result; + ASSERT_TRUE(db_->Get(ReadOptions(), db_->DefaultColumnFamily(), key, &result) + .IsIOError()); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_P(DBBlobBasicIOErrorMultiGetTest, MultiGetBlobs_IOError) { + Options options = GetDefaultOptions(); + options.env = fault_injection_env_.get(); + options.enable_blob_files = true; + options.min_blob_size = 0; + + Reopen(options); + + constexpr size_t num_keys = 2; + + constexpr char first_key[] = "first_key"; + constexpr char first_value[] = "first_value"; + + ASSERT_OK(Put(first_key, first_value)); + + constexpr char second_key[] = "second_key"; + constexpr char second_value[] = "second_value"; + + ASSERT_OK(Put(second_key, second_value)); + + ASSERT_OK(Flush()); + + std::array keys{{first_key, second_key}}; + std::array values; + std::array statuses; + + SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* /* arg */) { + fault_injection_env_->SetFilesystemActive(false, + Status::IOError(sync_point_)); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + db_->MultiGet(ReadOptions(), db_->DefaultColumnFamily(), num_keys, &keys[0], + &values[0], &statuses[0]); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + ASSERT_TRUE(statuses[0].IsIOError()); + ASSERT_TRUE(statuses[1].IsIOError()); +} + +TEST_P(DBBlobBasicIOErrorMultiGetTest, MultipleBlobFiles) { + Options options = GetDefaultOptions(); + options.env = fault_injection_env_.get(); + options.enable_blob_files = true; + options.min_blob_size = 0; + + Reopen(options); + + constexpr size_t num_keys = 2; + + constexpr char key1[] = "key1"; + constexpr char value1[] = "blob1"; + + ASSERT_OK(Put(key1, value1)); + ASSERT_OK(Flush()); + + constexpr char key2[] = "key2"; + constexpr char value2[] = "blob2"; + + ASSERT_OK(Put(key2, value2)); + ASSERT_OK(Flush()); + + std::array keys{{key1, key2}}; + std::array values; + std::array statuses; + + bool first_blob_file = true; + SyncPoint::GetInstance()->SetCallBack( + sync_point_, [&first_blob_file, this](void* /* arg */) { + if (first_blob_file) { + first_blob_file = false; + return; + } + fault_injection_env_->SetFilesystemActive(false, + Status::IOError(sync_point_)); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + db_->MultiGet(ReadOptions(), db_->DefaultColumnFamily(), num_keys, + keys.data(), values.data(), statuses.data()); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + ASSERT_OK(statuses[0]); + ASSERT_EQ(value1, values[0]); + ASSERT_TRUE(statuses[1].IsIOError()); +} + +namespace { + +class ReadBlobCompactionFilter : public CompactionFilter { + public: + ReadBlobCompactionFilter() = default; + const char* Name() const override { + return "rocksdb.compaction.filter.read.blob"; + } + CompactionFilter::Decision FilterV2( + int /*level*/, const Slice& /*key*/, ValueType value_type, + const Slice& existing_value, std::string* new_value, + std::string* /*skip_until*/) const override { + if (value_type != CompactionFilter::ValueType::kValue) { + return CompactionFilter::Decision::kKeep; + } + assert(new_value); + new_value->assign(existing_value.data(), existing_value.size()); + return CompactionFilter::Decision::kChangeValue; + } +}; + +} // anonymous namespace + +TEST_P(DBBlobBasicIOErrorTest, CompactionFilterReadBlob_IOError) { + Options options = GetDefaultOptions(); + options.env = fault_injection_env_.get(); + options.enable_blob_files = true; + options.min_blob_size = 0; + options.create_if_missing = true; + std::unique_ptr compaction_filter_guard( + new ReadBlobCompactionFilter); + options.compaction_filter = compaction_filter_guard.get(); + + DestroyAndReopen(options); + constexpr char key[] = "foo"; + constexpr char blob_value[] = "foo_blob_value"; + ASSERT_OK(Put(key, blob_value)); + ASSERT_OK(Flush()); + + SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* /* arg */) { + fault_injection_env_->SetFilesystemActive(false, + Status::IOError(sync_point_)); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_TRUE(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr, + /*end=*/nullptr) + .IsIOError()); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(DBBlobBasicTest, WarmCacheWithBlobsDuringFlush) { + Options options = GetDefaultOptions(); + + LRUCacheOptions co; + co.capacity = 1 << 25; + co.num_shard_bits = 2; + co.metadata_charge_policy = kDontChargeCacheMetadata; + auto backing_cache = NewLRUCache(co); + + options.blob_cache = backing_cache; + + BlockBasedTableOptions block_based_options; + block_based_options.no_block_cache = false; + block_based_options.block_cache = backing_cache; + block_based_options.cache_index_and_filter_blocks = true; + options.table_factory.reset(NewBlockBasedTableFactory(block_based_options)); + + options.enable_blob_files = true; + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.enable_blob_garbage_collection = true; + options.blob_garbage_collection_age_cutoff = 1.0; + options.prepopulate_blob_cache = PrepopulateBlobCache::kFlushOnly; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + + DestroyAndReopen(options); + + constexpr size_t kNumBlobs = 10; + constexpr size_t kValueSize = 100; + + std::string value(kValueSize, 'a'); + + for (size_t i = 1; i <= kNumBlobs; i++) { + ASSERT_OK(Put(std::to_string(i), value)); + ASSERT_OK(Put(std::to_string(i + kNumBlobs), value)); // Add some overlap + ASSERT_OK(Flush()); + ASSERT_EQ(i * 2, options.statistics->getTickerCount(BLOB_DB_CACHE_ADD)); + ASSERT_EQ(value, Get(std::to_string(i))); + ASSERT_EQ(value, Get(std::to_string(i + kNumBlobs))); + ASSERT_EQ(0, options.statistics->getTickerCount(BLOB_DB_CACHE_MISS)); + ASSERT_EQ(i * 2, options.statistics->getTickerCount(BLOB_DB_CACHE_HIT)); + } + + // Verify compaction not counted + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr, + /*end=*/nullptr)); + EXPECT_EQ(kNumBlobs * 2, + options.statistics->getTickerCount(BLOB_DB_CACHE_ADD)); +} + +TEST_F(DBBlobBasicTest, DynamicallyWarmCacheDuringFlush) { + Options options = GetDefaultOptions(); + + LRUCacheOptions co; + co.capacity = 1 << 25; + co.num_shard_bits = 2; + co.metadata_charge_policy = kDontChargeCacheMetadata; + auto backing_cache = NewLRUCache(co); + + options.blob_cache = backing_cache; + + BlockBasedTableOptions block_based_options; + block_based_options.no_block_cache = false; + block_based_options.block_cache = backing_cache; + block_based_options.cache_index_and_filter_blocks = true; + options.table_factory.reset(NewBlockBasedTableFactory(block_based_options)); + + options.enable_blob_files = true; + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.enable_blob_garbage_collection = true; + options.blob_garbage_collection_age_cutoff = 1.0; + options.prepopulate_blob_cache = PrepopulateBlobCache::kFlushOnly; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + + DestroyAndReopen(options); + + constexpr size_t kNumBlobs = 10; + constexpr size_t kValueSize = 100; + + std::string value(kValueSize, 'a'); + + for (size_t i = 1; i <= 5; i++) { + ASSERT_OK(Put(std::to_string(i), value)); + ASSERT_OK(Put(std::to_string(i + kNumBlobs), value)); // Add some overlap + ASSERT_OK(Flush()); + ASSERT_EQ(2, options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD)); + + ASSERT_EQ(value, Get(std::to_string(i))); + ASSERT_EQ(value, Get(std::to_string(i + kNumBlobs))); + ASSERT_EQ(0, options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD)); + ASSERT_EQ(0, + options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_MISS)); + ASSERT_EQ(2, options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_HIT)); + } + + ASSERT_OK(dbfull()->SetOptions({{"prepopulate_blob_cache", "kDisable"}})); + + for (size_t i = 6; i <= kNumBlobs; i++) { + ASSERT_OK(Put(std::to_string(i), value)); + ASSERT_OK(Put(std::to_string(i + kNumBlobs), value)); // Add some overlap + ASSERT_OK(Flush()); + ASSERT_EQ(0, options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD)); + + ASSERT_EQ(value, Get(std::to_string(i))); + ASSERT_EQ(value, Get(std::to_string(i + kNumBlobs))); + ASSERT_EQ(2, options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD)); + ASSERT_EQ(2, + options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_MISS)); + ASSERT_EQ(0, options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_HIT)); + } + + // Verify compaction not counted + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr, + /*end=*/nullptr)); + EXPECT_EQ(0, options.statistics->getTickerCount(BLOB_DB_CACHE_ADD)); +} + +TEST_F(DBBlobBasicTest, WarmCacheWithBlobsSecondary) { + CompressedSecondaryCacheOptions secondary_cache_opts; + secondary_cache_opts.capacity = 1 << 20; + secondary_cache_opts.num_shard_bits = 0; + secondary_cache_opts.metadata_charge_policy = kDontChargeCacheMetadata; + secondary_cache_opts.compression_type = kNoCompression; + + LRUCacheOptions primary_cache_opts; + primary_cache_opts.capacity = 1024; + primary_cache_opts.num_shard_bits = 0; + primary_cache_opts.metadata_charge_policy = kDontChargeCacheMetadata; + primary_cache_opts.secondary_cache = + NewCompressedSecondaryCache(secondary_cache_opts); + + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.statistics = CreateDBStatistics(); + options.enable_blob_files = true; + options.blob_cache = NewLRUCache(primary_cache_opts); + options.prepopulate_blob_cache = PrepopulateBlobCache::kFlushOnly; + + DestroyAndReopen(options); + + // Note: only one of the two blobs fit in the primary cache at any given time. + constexpr char first_key[] = "foo"; + constexpr size_t first_blob_size = 512; + const std::string first_blob(first_blob_size, 'a'); + + constexpr char second_key[] = "bar"; + constexpr size_t second_blob_size = 768; + const std::string second_blob(second_blob_size, 'b'); + + // First blob is inserted into primary cache during flush. + ASSERT_OK(Put(first_key, first_blob)); + ASSERT_OK(Flush()); + ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD), 1); + + // Second blob is inserted into primary cache during flush, + // First blob is evicted but only a dummy handle is inserted into secondary + // cache. + ASSERT_OK(Put(second_key, second_blob)); + ASSERT_OK(Flush()); + ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD), 1); + + // First blob is inserted into primary cache. + // Second blob is evicted but only a dummy handle is inserted into secondary + // cache. + ASSERT_EQ(Get(first_key), first_blob); + ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_MISS), 1); + ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_HIT), 0); + ASSERT_EQ(options.statistics->getAndResetTickerCount(SECONDARY_CACHE_HITS), + 0); + // Second blob is inserted into primary cache, + // First blob is evicted and is inserted into secondary cache. + ASSERT_EQ(Get(second_key), second_blob); + ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_MISS), 1); + ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_HIT), 0); + ASSERT_EQ(options.statistics->getAndResetTickerCount(SECONDARY_CACHE_HITS), + 0); + + // First blob's dummy item is inserted into primary cache b/c of lookup. + // Second blob is still in primary cache. + ASSERT_EQ(Get(first_key), first_blob); + ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_MISS), 0); + ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_HIT), 1); + ASSERT_EQ(options.statistics->getAndResetTickerCount(SECONDARY_CACHE_HITS), + 1); + + // First blob's item is inserted into primary cache b/c of lookup. + // Second blob is evicted and inserted into secondary cache. + ASSERT_EQ(Get(first_key), first_blob); + ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_MISS), 0); + ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_HIT), 1); + ASSERT_EQ(options.statistics->getAndResetTickerCount(SECONDARY_CACHE_HITS), + 1); +} + +TEST_F(DBBlobBasicTest, GetEntityBlob) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + + Reopen(options); + + constexpr char key[] = "key"; + constexpr char blob_value[] = "blob_value"; + + constexpr char other_key[] = "other_key"; + constexpr char other_blob_value[] = "other_blob_value"; + + ASSERT_OK(Put(key, blob_value)); + ASSERT_OK(Put(other_key, other_blob_value)); + + ASSERT_OK(Flush()); + + WideColumns expected_columns{{kDefaultWideColumnName, blob_value}}; + WideColumns other_expected_columns{ + {kDefaultWideColumnName, other_blob_value}}; + + { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), key, + &result)); + ASSERT_EQ(result.columns(), expected_columns); + } + + { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + other_key, &result)); + + ASSERT_EQ(result.columns(), other_expected_columns); + } + + { + constexpr size_t num_keys = 2; + + std::array keys{{key, other_key}}; + std::array results; + std::array statuses; + + db_->MultiGetEntity(ReadOptions(), db_->DefaultColumnFamily(), num_keys, + &keys[0], &results[0], &statuses[0]); + + ASSERT_OK(statuses[0]); + ASSERT_EQ(results[0].columns(), expected_columns); + + ASSERT_OK(statuses[1]); + ASSERT_EQ(results[1].columns(), other_expected_columns); + } +} + +class DBBlobWithTimestampTest : public DBBasicTestWithTimestampBase { + protected: + DBBlobWithTimestampTest() + : DBBasicTestWithTimestampBase("db_blob_with_timestamp_test") {} +}; + +TEST_F(DBBlobWithTimestampTest, GetBlob) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.min_blob_size = 0; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + + DestroyAndReopen(options); + WriteOptions write_opts; + const std::string ts = Timestamp(1, 0); + constexpr char key[] = "key"; + constexpr char blob_value[] = "blob_value"; + + ASSERT_OK(db_->Put(write_opts, key, ts, blob_value)); + + ASSERT_OK(Flush()); + + const std::string read_ts = Timestamp(2, 0); + Slice read_ts_slice(read_ts); + ReadOptions read_opts; + read_opts.timestamp = &read_ts_slice; + std::string value; + ASSERT_OK(db_->Get(read_opts, key, &value)); + ASSERT_EQ(value, blob_value); +} + +TEST_F(DBBlobWithTimestampTest, MultiGetBlobs) { + constexpr size_t min_blob_size = 6; + + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = min_blob_size; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + + DestroyAndReopen(options); + + // Put then retrieve three key-values. The first value is below the size limit + // and is thus stored inline; the other two are stored separately as blobs. + constexpr size_t num_keys = 3; + + constexpr char first_key[] = "first_key"; + constexpr char first_value[] = "short"; + static_assert(sizeof(first_value) - 1 < min_blob_size, + "first_value too long to be inlined"); + + DestroyAndReopen(options); + WriteOptions write_opts; + const std::string ts = Timestamp(1, 0); + ASSERT_OK(db_->Put(write_opts, first_key, ts, first_value)); + + constexpr char second_key[] = "second_key"; + constexpr char second_value[] = "long_value"; + static_assert(sizeof(second_value) - 1 >= min_blob_size, + "second_value too short to be stored as blob"); + + ASSERT_OK(db_->Put(write_opts, second_key, ts, second_value)); + + constexpr char third_key[] = "third_key"; + constexpr char third_value[] = "other_long_value"; + static_assert(sizeof(third_value) - 1 >= min_blob_size, + "third_value too short to be stored as blob"); + + ASSERT_OK(db_->Put(write_opts, third_key, ts, third_value)); + + ASSERT_OK(Flush()); + + ReadOptions read_options; + const std::string read_ts = Timestamp(2, 0); + Slice read_ts_slice(read_ts); + read_options.timestamp = &read_ts_slice; + std::array keys{{first_key, second_key, third_key}}; + + { + std::array values; + std::array statuses; + + db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys, &keys[0], + &values[0], &statuses[0]); + + ASSERT_OK(statuses[0]); + ASSERT_EQ(values[0], first_value); + + ASSERT_OK(statuses[1]); + ASSERT_EQ(values[1], second_value); + + ASSERT_OK(statuses[2]); + ASSERT_EQ(values[2], third_value); + } +} + +TEST_F(DBBlobWithTimestampTest, GetMergeBlobWithPut) { + Options options = GetDefaultOptions(); + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + options.enable_blob_files = true; + options.min_blob_size = 0; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + + DestroyAndReopen(options); + + WriteOptions write_opts; + const std::string ts = Timestamp(1, 0); + ASSERT_OK(db_->Put(write_opts, "Key1", ts, "v1")); + ASSERT_OK(Flush()); + ASSERT_OK( + db_->Merge(write_opts, db_->DefaultColumnFamily(), "Key1", ts, "v2")); + ASSERT_OK(Flush()); + ASSERT_OK( + db_->Merge(write_opts, db_->DefaultColumnFamily(), "Key1", ts, "v3")); + ASSERT_OK(Flush()); + + std::string value; + const std::string read_ts = Timestamp(2, 0); + Slice read_ts_slice(read_ts); + ReadOptions read_opts; + read_opts.timestamp = &read_ts_slice; + ASSERT_OK(db_->Get(read_opts, "Key1", &value)); + ASSERT_EQ(value, "v1,v2,v3"); +} + +TEST_F(DBBlobWithTimestampTest, MultiGetMergeBlobWithPut) { + constexpr size_t num_keys = 3; + + Options options = GetDefaultOptions(); + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + options.enable_blob_files = true; + options.min_blob_size = 0; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + + DestroyAndReopen(options); + + WriteOptions write_opts; + const std::string ts = Timestamp(1, 0); + + ASSERT_OK(db_->Put(write_opts, "Key0", ts, "v0_0")); + ASSERT_OK(db_->Put(write_opts, "Key1", ts, "v1_0")); + ASSERT_OK(db_->Put(write_opts, "Key2", ts, "v2_0")); + ASSERT_OK(Flush()); + ASSERT_OK( + db_->Merge(write_opts, db_->DefaultColumnFamily(), "Key0", ts, "v0_1")); + ASSERT_OK( + db_->Merge(write_opts, db_->DefaultColumnFamily(), "Key1", ts, "v1_1")); + ASSERT_OK(Flush()); + ASSERT_OK( + db_->Merge(write_opts, db_->DefaultColumnFamily(), "Key0", ts, "v0_2")); + ASSERT_OK(Flush()); + + const std::string read_ts = Timestamp(2, 0); + Slice read_ts_slice(read_ts); + ReadOptions read_opts; + read_opts.timestamp = &read_ts_slice; + std::array keys{{"Key0", "Key1", "Key2"}}; + std::array values; + std::array statuses; + + db_->MultiGet(read_opts, db_->DefaultColumnFamily(), num_keys, &keys[0], + &values[0], &statuses[0]); + + ASSERT_OK(statuses[0]); + ASSERT_EQ(values[0], "v0_0,v0_1,v0_2"); + + ASSERT_OK(statuses[1]); + ASSERT_EQ(values[1], "v1_0,v1_1"); + + ASSERT_OK(statuses[2]); + ASSERT_EQ(values[2], "v2_0"); +} + +TEST_F(DBBlobWithTimestampTest, IterateBlobs) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + + DestroyAndReopen(options); + + int num_blobs = 5; + std::vector keys; + std::vector blobs; + + WriteOptions write_opts; + std::vector write_timestamps = {Timestamp(1, 0), + Timestamp(2, 0)}; + + // For each key in ["key0", ... "keyi", ...], write two versions: + // Timestamp(1, 0), "blobi0" + // Timestamp(2, 0), "blobi1" + for (int i = 0; i < num_blobs; i++) { + keys.push_back("key" + std::to_string(i)); + blobs.push_back("blob" + std::to_string(i)); + for (size_t j = 0; j < write_timestamps.size(); j++) { + ASSERT_OK(db_->Put(write_opts, keys[i], write_timestamps[j], + blobs[i] + std::to_string(j))); + } + } + ASSERT_OK(Flush()); + + ReadOptions read_options; + std::vector read_timestamps = {Timestamp(0, 0), Timestamp(3, 0)}; + Slice ts_upper_bound(read_timestamps[1]); + read_options.timestamp = &ts_upper_bound; + + auto check_iter_entry = + [](const Iterator* iter, const std::string& expected_key, + const std::string& expected_ts, const std::string& expected_value, + bool key_is_internal = true) { + ASSERT_OK(iter->status()); + if (key_is_internal) { + std::string expected_ukey_and_ts; + expected_ukey_and_ts.assign(expected_key.data(), expected_key.size()); + expected_ukey_and_ts.append(expected_ts.data(), expected_ts.size()); + + ParsedInternalKey parsed_ikey; + ASSERT_OK(ParseInternalKey(iter->key(), &parsed_ikey, + true /* log_err_key */)); + ASSERT_EQ(parsed_ikey.user_key, expected_ukey_and_ts); + } else { + ASSERT_EQ(iter->key(), expected_key); + } + ASSERT_EQ(iter->timestamp(), expected_ts); + ASSERT_EQ(iter->value(), expected_value); + }; + + // Forward iterating one version of each key, get in this order: + // [("key0", Timestamp(2, 0), "blob01"), + // ("key1", Timestamp(2, 0), "blob11")...] + { + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_OK(iter->status()); + + iter->SeekToFirst(); + for (int i = 0; i < num_blobs; i++) { + check_iter_entry(iter.get(), keys[i], write_timestamps[1], + blobs[i] + std::to_string(1), /*key_is_internal*/ false); + iter->Next(); + } + } + + // Forward iteration, then reverse to backward. + { + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_OK(iter->status()); + + iter->SeekToFirst(); + for (int i = 0; i < num_blobs * 2 - 1; i++) { + if (i < num_blobs) { + check_iter_entry(iter.get(), keys[i], write_timestamps[1], + blobs[i] + std::to_string(1), + /*key_is_internal*/ false); + if (i != num_blobs - 1) { + iter->Next(); + } + } else { + if (i != num_blobs) { + check_iter_entry(iter.get(), keys[num_blobs * 2 - 1 - i], + write_timestamps[1], + blobs[num_blobs * 2 - 1 - i] + std::to_string(1), + /*key_is_internal*/ false); + } + iter->Prev(); + } + } + } + + // Backward iterating one versions of each key, get in this order: + // [("key4", Timestamp(2, 0), "blob41"), + // ("key3", Timestamp(2, 0), "blob31")...] + { + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_OK(iter->status()); + + iter->SeekToLast(); + for (int i = 0; i < num_blobs; i++) { + check_iter_entry(iter.get(), keys[num_blobs - 1 - i], write_timestamps[1], + blobs[num_blobs - 1 - i] + std::to_string(1), + /*key_is_internal*/ false); + iter->Prev(); + } + } + + // Backward iteration, then reverse to forward. + { + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_OK(iter->status()); + + iter->SeekToLast(); + for (int i = 0; i < num_blobs * 2 - 1; i++) { + if (i < num_blobs) { + check_iter_entry(iter.get(), keys[num_blobs - 1 - i], + write_timestamps[1], + blobs[num_blobs - 1 - i] + std::to_string(1), + /*key_is_internal*/ false); + if (i != num_blobs - 1) { + iter->Prev(); + } + } else { + if (i != num_blobs) { + check_iter_entry(iter.get(), keys[i - num_blobs], write_timestamps[1], + blobs[i - num_blobs] + std::to_string(1), + /*key_is_internal*/ false); + } + iter->Next(); + } + } + } + + Slice ts_lower_bound(read_timestamps[0]); + read_options.iter_start_ts = &ts_lower_bound; + // Forward iterating multiple versions of the same key, get in this order: + // [("key0", Timestamp(2, 0), "blob01"), + // ("key0", Timestamp(1, 0), "blob00"), + // ("key1", Timestamp(2, 0), "blob11")...] + { + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_OK(iter->status()); + + iter->SeekToFirst(); + for (int i = 0; i < num_blobs; i++) { + for (size_t j = write_timestamps.size(); j > 0; --j) { + check_iter_entry(iter.get(), keys[i], write_timestamps[j - 1], + blobs[i] + std::to_string(j - 1)); + iter->Next(); + } + } + } + + // Backward iterating multiple versions of the same key, get in this order: + // [("key4", Timestamp(1, 0), "blob00"), + // ("key4", Timestamp(2, 0), "blob01"), + // ("key3", Timestamp(1, 0), "blob10")...] + { + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_OK(iter->status()); + + iter->SeekToLast(); + for (int i = num_blobs; i > 0; i--) { + for (size_t j = 0; j < write_timestamps.size(); j++) { + check_iter_entry(iter.get(), keys[i - 1], write_timestamps[j], + blobs[i - 1] + std::to_string(j)); + iter->Prev(); + } + } + } + + int upper_bound_idx = num_blobs - 2; + int lower_bound_idx = 1; + Slice upper_bound_slice(keys[upper_bound_idx]); + Slice lower_bound_slice(keys[lower_bound_idx]); + read_options.iterate_upper_bound = &upper_bound_slice; + read_options.iterate_lower_bound = &lower_bound_slice; + + // Forward iteration with upper and lower bound. + { + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_OK(iter->status()); + + iter->SeekToFirst(); + for (int i = lower_bound_idx; i < upper_bound_idx; i++) { + for (size_t j = write_timestamps.size(); j > 0; --j) { + check_iter_entry(iter.get(), keys[i], write_timestamps[j - 1], + blobs[i] + std::to_string(j - 1)); + iter->Next(); + } + } + } + + // Backward iteration with upper and lower bound. + { + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_OK(iter->status()); + + iter->SeekToLast(); + for (int i = upper_bound_idx; i > lower_bound_idx; i--) { + for (size_t j = 0; j < write_timestamps.size(); j++) { + check_iter_entry(iter.get(), keys[i - 1], write_timestamps[j], + blobs[i - 1] + std::to_string(j)); + iter->Prev(); + } + } + } +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/blob/db_blob_compaction_test.cc b/librocksdb-sys/rocksdb/db/blob/db_blob_compaction_test.cc new file mode 100644 index 0000000..14a3155 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/db_blob_compaction_test.cc @@ -0,0 +1,899 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_index.h" +#include "db/blob/blob_log_format.h" +#include "db/db_test_util.h" +#include "port/stack_trace.h" +#include "test_util/sync_point.h" + +namespace ROCKSDB_NAMESPACE { + +class DBBlobCompactionTest : public DBTestBase { + public: + explicit DBBlobCompactionTest() + : DBTestBase("db_blob_compaction_test", /*env_do_fsync=*/false) {} + + const std::vector& GetCompactionStats() { + VersionSet* const versions = dbfull()->GetVersionSet(); + assert(versions); + assert(versions->GetColumnFamilySet()); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + assert(cfd); + + const InternalStats* const internal_stats = cfd->internal_stats(); + assert(internal_stats); + + return internal_stats->TEST_GetCompactionStats(); + } +}; + +namespace { + +class FilterByKeyLength : public CompactionFilter { + public: + explicit FilterByKeyLength(size_t len) : length_threshold_(len) {} + const char* Name() const override { + return "rocksdb.compaction.filter.by.key.length"; + } + CompactionFilter::Decision FilterBlobByKey( + int /*level*/, const Slice& key, std::string* /*new_value*/, + std::string* /*skip_until*/) const override { + if (key.size() < length_threshold_) { + return CompactionFilter::Decision::kRemove; + } + return CompactionFilter::Decision::kKeep; + } + + private: + size_t length_threshold_; +}; + +class FilterByValueLength : public CompactionFilter { + public: + explicit FilterByValueLength(size_t len) : length_threshold_(len) {} + const char* Name() const override { + return "rocksdb.compaction.filter.by.value.length"; + } + CompactionFilter::Decision FilterV2( + int /*level*/, const Slice& /*key*/, ValueType /*value_type*/, + const Slice& existing_value, std::string* /*new_value*/, + std::string* /*skip_until*/) const override { + if (existing_value.size() < length_threshold_) { + return CompactionFilter::Decision::kRemove; + } + return CompactionFilter::Decision::kKeep; + } + + private: + size_t length_threshold_; +}; + +class BadBlobCompactionFilter : public CompactionFilter { + public: + explicit BadBlobCompactionFilter(std::string prefix, + CompactionFilter::Decision filter_by_key, + CompactionFilter::Decision filter_v2) + : prefix_(std::move(prefix)), + filter_blob_by_key_(filter_by_key), + filter_v2_(filter_v2) {} + const char* Name() const override { return "rocksdb.compaction.filter.bad"; } + CompactionFilter::Decision FilterBlobByKey( + int /*level*/, const Slice& key, std::string* /*new_value*/, + std::string* /*skip_until*/) const override { + if (key.size() >= prefix_.size() && + 0 == strncmp(prefix_.data(), key.data(), prefix_.size())) { + return CompactionFilter::Decision::kUndetermined; + } + return filter_blob_by_key_; + } + CompactionFilter::Decision FilterV2( + int /*level*/, const Slice& /*key*/, ValueType /*value_type*/, + const Slice& /*existing_value*/, std::string* /*new_value*/, + std::string* /*skip_until*/) const override { + return filter_v2_; + } + + private: + const std::string prefix_; + const CompactionFilter::Decision filter_blob_by_key_; + const CompactionFilter::Decision filter_v2_; +}; + +class ValueBlindWriteFilter : public CompactionFilter { + public: + explicit ValueBlindWriteFilter(std::string new_val) + : new_value_(std::move(new_val)) {} + const char* Name() const override { + return "rocksdb.compaction.filter.blind.write"; + } + CompactionFilter::Decision FilterBlobByKey( + int level, const Slice& key, std::string* new_value, + std::string* skip_until) const override; + + private: + const std::string new_value_; +}; + +CompactionFilter::Decision ValueBlindWriteFilter::FilterBlobByKey( + int /*level*/, const Slice& /*key*/, std::string* new_value, + std::string* /*skip_until*/) const { + assert(new_value); + new_value->assign(new_value_); + return CompactionFilter::Decision::kChangeValue; +} + +class ValueMutationFilter : public CompactionFilter { + public: + explicit ValueMutationFilter(std::string padding) + : padding_(std::move(padding)) {} + const char* Name() const override { + return "rocksdb.compaction.filter.value.mutation"; + } + CompactionFilter::Decision FilterV2(int level, const Slice& key, + ValueType value_type, + const Slice& existing_value, + std::string* new_value, + std::string* skip_until) const override; + + private: + const std::string padding_; +}; + +CompactionFilter::Decision ValueMutationFilter::FilterV2( + int /*level*/, const Slice& /*key*/, ValueType value_type, + const Slice& existing_value, std::string* new_value, + std::string* /*skip_until*/) const { + assert(CompactionFilter::ValueType::kBlobIndex != value_type); + if (CompactionFilter::ValueType::kValue != value_type) { + return CompactionFilter::Decision::kKeep; + } + assert(new_value); + new_value->assign(existing_value.data(), existing_value.size()); + new_value->append(padding_); + return CompactionFilter::Decision::kChangeValue; +} + +class AlwaysKeepFilter : public CompactionFilter { + public: + explicit AlwaysKeepFilter() = default; + const char* Name() const override { + return "rocksdb.compaction.filter.always.keep"; + } + CompactionFilter::Decision FilterV2( + int /*level*/, const Slice& /*key*/, ValueType /*value_type*/, + const Slice& /*existing_value*/, std::string* /*new_value*/, + std::string* /*skip_until*/) const override { + return CompactionFilter::Decision::kKeep; + } +}; + +class SkipUntilFilter : public CompactionFilter { + public: + explicit SkipUntilFilter(std::string skip_until) + : skip_until_(std::move(skip_until)) {} + + const char* Name() const override { + return "rocksdb.compaction.filter.skip.until"; + } + + CompactionFilter::Decision FilterV2(int /* level */, const Slice& /* key */, + ValueType /* value_type */, + const Slice& /* existing_value */, + std::string* /* new_value */, + std::string* skip_until) const override { + assert(skip_until); + *skip_until = skip_until_; + + return CompactionFilter::Decision::kRemoveAndSkipUntil; + } + + private: + std::string skip_until_; +}; + +} // anonymous namespace + +class DBBlobBadCompactionFilterTest + : public DBBlobCompactionTest, + public testing::WithParamInterface< + std::tuple> { + public: + explicit DBBlobBadCompactionFilterTest() + : compaction_filter_guard_(new BadBlobCompactionFilter( + std::get<0>(GetParam()), std::get<1>(GetParam()), + std::get<2>(GetParam()))) {} + + protected: + std::unique_ptr compaction_filter_guard_; +}; + +INSTANTIATE_TEST_CASE_P( + BadCompactionFilter, DBBlobBadCompactionFilterTest, + testing::Combine( + testing::Values("a"), + testing::Values(CompactionFilter::Decision::kChangeBlobIndex, + CompactionFilter::Decision::kIOError), + testing::Values(CompactionFilter::Decision::kUndetermined, + CompactionFilter::Decision::kChangeBlobIndex, + CompactionFilter::Decision::kIOError))); + +TEST_F(DBBlobCompactionTest, FilterByKeyLength) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + options.create_if_missing = true; + constexpr size_t kKeyLength = 2; + std::unique_ptr compaction_filter_guard( + new FilterByKeyLength(kKeyLength)); + options.compaction_filter = compaction_filter_guard.get(); + + constexpr char short_key[] = "a"; + constexpr char long_key[] = "abc"; + constexpr char blob_value[] = "value"; + + DestroyAndReopen(options); + ASSERT_OK(Put(short_key, blob_value)); + ASSERT_OK(Put(long_key, blob_value)); + ASSERT_OK(Flush()); + CompactRangeOptions cro; + ASSERT_OK(db_->CompactRange(cro, /*begin=*/nullptr, /*end=*/nullptr)); + std::string value; + ASSERT_TRUE(db_->Get(ReadOptions(), short_key, &value).IsNotFound()); + value.clear(); + ASSERT_OK(db_->Get(ReadOptions(), long_key, &value)); + ASSERT_EQ("value", value); + + const auto& compaction_stats = GetCompactionStats(); + ASSERT_GE(compaction_stats.size(), 2); + + // Filter decides between kKeep and kRemove solely based on key; + // this involves neither reading nor writing blobs + ASSERT_EQ(compaction_stats[1].bytes_read_blob, 0); + ASSERT_EQ(compaction_stats[1].bytes_written_blob, 0); + + Close(); +} + +TEST_F(DBBlobCompactionTest, FilterByValueLength) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 5; + options.create_if_missing = true; + constexpr size_t kValueLength = 5; + std::unique_ptr compaction_filter_guard( + new FilterByValueLength(kValueLength)); + options.compaction_filter = compaction_filter_guard.get(); + + const std::vector short_value_keys = {"a", "e", "j"}; + constexpr char short_value[] = "val"; + const std::vector long_value_keys = {"b", "f", "k"}; + constexpr char long_value[] = "valuevalue"; + + DestroyAndReopen(options); + for (size_t i = 0; i < short_value_keys.size(); ++i) { + ASSERT_OK(Put(short_value_keys[i], short_value)); + } + for (size_t i = 0; i < short_value_keys.size(); ++i) { + ASSERT_OK(Put(long_value_keys[i], long_value)); + } + ASSERT_OK(Flush()); + CompactRangeOptions cro; + ASSERT_OK(db_->CompactRange(cro, /*begin=*/nullptr, /*end=*/nullptr)); + std::string value; + for (size_t i = 0; i < short_value_keys.size(); ++i) { + ASSERT_TRUE( + db_->Get(ReadOptions(), short_value_keys[i], &value).IsNotFound()); + value.clear(); + } + for (size_t i = 0; i < long_value_keys.size(); ++i) { + ASSERT_OK(db_->Get(ReadOptions(), long_value_keys[i], &value)); + ASSERT_EQ(long_value, value); + } + + const auto& compaction_stats = GetCompactionStats(); + ASSERT_GE(compaction_stats.size(), 2); + + // Filter decides between kKeep and kRemove based on value; + // this involves reading but not writing blobs + ASSERT_GT(compaction_stats[1].bytes_read_blob, 0); + ASSERT_EQ(compaction_stats[1].bytes_written_blob, 0); + + Close(); +} + +TEST_F(DBBlobCompactionTest, BlobCompactWithStartingLevel) { + Options options = GetDefaultOptions(); + + options.enable_blob_files = true; + options.min_blob_size = 1000; + options.blob_file_starting_level = 5; + options.create_if_missing = true; + + // Open DB with fixed-prefix sst-partitioner so that compaction will cut + // new table file when encountering a new key whose 1-byte prefix changes. + constexpr size_t key_len = 1; + options.sst_partitioner_factory = + NewSstPartitionerFixedPrefixFactory(key_len); + + ASSERT_OK(TryReopen(options)); + + constexpr size_t blob_size = 3000; + + constexpr char first_key[] = "a"; + const std::string first_blob(blob_size, 'a'); + ASSERT_OK(Put(first_key, first_blob)); + + constexpr char second_key[] = "b"; + const std::string second_blob(2 * blob_size, 'b'); + ASSERT_OK(Put(second_key, second_blob)); + + constexpr char third_key[] = "d"; + const std::string third_blob(blob_size, 'd'); + ASSERT_OK(Put(third_key, third_blob)); + + ASSERT_OK(Flush()); + + constexpr char fourth_key[] = "c"; + const std::string fourth_blob(blob_size, 'c'); + ASSERT_OK(Put(fourth_key, fourth_blob)); + + ASSERT_OK(Flush()); + + ASSERT_EQ(0, GetBlobFileNumbers().size()); + ASSERT_EQ(2, NumTableFilesAtLevel(/*level=*/0)); + ASSERT_EQ(0, NumTableFilesAtLevel(/*level=*/1)); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr, + /*end=*/nullptr)); + + // No blob file should be created since blob_file_starting_level is 5. + ASSERT_EQ(0, GetBlobFileNumbers().size()); + ASSERT_EQ(0, NumTableFilesAtLevel(/*level=*/0)); + ASSERT_EQ(4, NumTableFilesAtLevel(/*level=*/1)); + + { + options.blob_file_starting_level = 1; + DestroyAndReopen(options); + + ASSERT_OK(Put(first_key, first_blob)); + ASSERT_OK(Put(second_key, second_blob)); + ASSERT_OK(Put(third_key, third_blob)); + ASSERT_OK(Flush()); + ASSERT_OK(Put(fourth_key, fourth_blob)); + ASSERT_OK(Flush()); + + ASSERT_EQ(0, GetBlobFileNumbers().size()); + ASSERT_EQ(2, NumTableFilesAtLevel(/*level=*/0)); + ASSERT_EQ(0, NumTableFilesAtLevel(/*level=*/1)); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr, + /*end=*/nullptr)); + // The compaction's output level equals to blob_file_starting_level. + ASSERT_EQ(1, GetBlobFileNumbers().size()); + ASSERT_EQ(0, NumTableFilesAtLevel(/*level=*/0)); + ASSERT_EQ(4, NumTableFilesAtLevel(/*level=*/1)); + } + + Close(); +} + +TEST_F(DBBlobCompactionTest, BlindWriteFilter) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + options.create_if_missing = true; + constexpr char new_blob_value[] = "new_blob_value"; + std::unique_ptr compaction_filter_guard( + new ValueBlindWriteFilter(new_blob_value)); + options.compaction_filter = compaction_filter_guard.get(); + DestroyAndReopen(options); + const std::vector keys = {"a", "b", "c"}; + const std::vector values = {"a_value", "b_value", "c_value"}; + assert(keys.size() == values.size()); + for (size_t i = 0; i < keys.size(); ++i) { + ASSERT_OK(Put(keys[i], values[i])); + } + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr, + /*end=*/nullptr)); + for (const auto& key : keys) { + ASSERT_EQ(new_blob_value, Get(key)); + } + + const auto& compaction_stats = GetCompactionStats(); + ASSERT_GE(compaction_stats.size(), 2); + + // Filter unconditionally changes value in FilterBlobByKey; + // this involves writing but not reading blobs + ASSERT_EQ(compaction_stats[1].bytes_read_blob, 0); + ASSERT_GT(compaction_stats[1].bytes_written_blob, 0); + + Close(); +} + +TEST_F(DBBlobCompactionTest, SkipUntilFilter) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + + std::unique_ptr compaction_filter_guard( + new SkipUntilFilter("z")); + options.compaction_filter = compaction_filter_guard.get(); + + Reopen(options); + + const std::vector keys{"a", "b", "c"}; + const std::vector values{"a_value", "b_value", "c_value"}; + assert(keys.size() == values.size()); + + for (size_t i = 0; i < keys.size(); ++i) { + ASSERT_OK(Put(keys[i], values[i])); + } + + ASSERT_OK(Flush()); + + int process_in_flow_called = 0; + + SyncPoint::GetInstance()->SetCallBack( + "BlobCountingIterator::UpdateAndCountBlobIfNeeded:ProcessInFlow", + [&process_in_flow_called](void* /* arg */) { ++process_in_flow_called; }); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), /* begin */ nullptr, + /* end */ nullptr)); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + for (const auto& key : keys) { + ASSERT_EQ(Get(key), "NOT_FOUND"); + } + + // Make sure SkipUntil was performed using iteration rather than Seek + ASSERT_EQ(process_in_flow_called, keys.size()); + + Close(); +} + +TEST_P(DBBlobBadCompactionFilterTest, BadDecisionFromCompactionFilter) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + options.create_if_missing = true; + options.compaction_filter = compaction_filter_guard_.get(); + DestroyAndReopen(options); + ASSERT_OK(Put("b", "value")); + ASSERT_OK(Flush()); + ASSERT_TRUE(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr, + /*end=*/nullptr) + .IsNotSupported()); + Close(); + + DestroyAndReopen(options); + std::string key(std::get<0>(GetParam())); + ASSERT_OK(Put(key, "value")); + ASSERT_OK(Flush()); + ASSERT_TRUE(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr, + /*end=*/nullptr) + .IsNotSupported()); + Close(); +} + +TEST_F(DBBlobCompactionTest, CompactionFilter_InlinedTTLIndex) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.min_blob_size = 0; + std::unique_ptr compaction_filter_guard( + new ValueMutationFilter("")); + options.compaction_filter = compaction_filter_guard.get(); + DestroyAndReopen(options); + constexpr char key[] = "key"; + constexpr char blob[] = "blob"; + // Fake an inlined TTL blob index. + std::string blob_index; + constexpr uint64_t expiration = 1234567890; + BlobIndex::EncodeInlinedTTL(&blob_index, expiration, blob); + WriteBatch batch; + ASSERT_OK(WriteBatchInternal::PutBlobIndex(&batch, 0, key, blob_index)); + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + ASSERT_OK(Flush()); + ASSERT_TRUE(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr, + /*end=*/nullptr) + .IsCorruption()); + Close(); +} + +TEST_F(DBBlobCompactionTest, CompactionFilter) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.min_blob_size = 0; + constexpr char padding[] = "_delta"; + std::unique_ptr compaction_filter_guard( + new ValueMutationFilter(padding)); + options.compaction_filter = compaction_filter_guard.get(); + DestroyAndReopen(options); + const std::vector> kvs = { + {"a", "a_value"}, {"b", "b_value"}, {"c", "c_value"}}; + for (const auto& kv : kvs) { + ASSERT_OK(Put(kv.first, kv.second)); + } + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr, + /*end=*/nullptr)); + for (const auto& kv : kvs) { + ASSERT_EQ(kv.second + std::string(padding), Get(kv.first)); + } + + const auto& compaction_stats = GetCompactionStats(); + ASSERT_GE(compaction_stats.size(), 2); + + // Filter changes the value using the previous value in FilterV2; + // this involves reading and writing blobs + ASSERT_GT(compaction_stats[1].bytes_read_blob, 0); + ASSERT_GT(compaction_stats[1].bytes_written_blob, 0); + + Close(); +} + +TEST_F(DBBlobCompactionTest, CorruptedBlobIndex) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.min_blob_size = 0; + std::unique_ptr compaction_filter_guard( + new ValueMutationFilter("")); + options.compaction_filter = compaction_filter_guard.get(); + DestroyAndReopen(options); + + constexpr char key[] = "key"; + constexpr char blob[] = "blob"; + + ASSERT_OK(Put(key, blob)); + ASSERT_OK(Flush()); + + SyncPoint::GetInstance()->SetCallBack( + "CompactionIterator::InvokeFilterIfNeeded::TamperWithBlobIndex", + [](void* arg) { + Slice* const blob_index = static_cast(arg); + assert(blob_index); + assert(!blob_index->empty()); + blob_index->remove_prefix(1); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_TRUE(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr, + /*end=*/nullptr) + .IsCorruption()); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + Close(); +} + +TEST_F(DBBlobCompactionTest, CompactionFilterReadBlobAndKeep) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.min_blob_size = 0; + std::unique_ptr compaction_filter_guard( + new AlwaysKeepFilter()); + options.compaction_filter = compaction_filter_guard.get(); + DestroyAndReopen(options); + ASSERT_OK(Put("foo", "foo_value")); + ASSERT_OK(Flush()); + std::vector blob_files = GetBlobFileNumbers(); + ASSERT_EQ(1, blob_files.size()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr, + /*end=*/nullptr)); + ASSERT_EQ(blob_files, GetBlobFileNumbers()); + + const auto& compaction_stats = GetCompactionStats(); + ASSERT_GE(compaction_stats.size(), 2); + + // Filter decides to keep the existing value in FilterV2; + // this involves reading but not writing blobs + ASSERT_GT(compaction_stats[1].bytes_read_blob, 0); + ASSERT_EQ(compaction_stats[1].bytes_written_blob, 0); + + Close(); +} + +TEST_F(DBBlobCompactionTest, TrackGarbage) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + + Reopen(options); + + // First table+blob file pair: 4 blobs with different keys + constexpr char first_key[] = "first_key"; + constexpr char first_value[] = "first_value"; + constexpr char second_key[] = "second_key"; + constexpr char second_value[] = "second_value"; + constexpr char third_key[] = "third_key"; + constexpr char third_value[] = "third_value"; + constexpr char fourth_key[] = "fourth_key"; + constexpr char fourth_value[] = "fourth_value"; + + ASSERT_OK(Put(first_key, first_value)); + ASSERT_OK(Put(second_key, second_value)); + ASSERT_OK(Put(third_key, third_value)); + ASSERT_OK(Put(fourth_key, fourth_value)); + ASSERT_OK(Flush()); + + // Second table+blob file pair: overwrite 2 existing keys + constexpr char new_first_value[] = "new_first_value"; + constexpr char new_second_value[] = "new_second_value"; + + ASSERT_OK(Put(first_key, new_first_value)); + ASSERT_OK(Put(second_key, new_second_value)); + ASSERT_OK(Flush()); + + // Compact them together. The first blob file should have 2 garbage blobs + // corresponding to the 2 overwritten keys. + constexpr Slice* begin = nullptr; + constexpr Slice* end = nullptr; + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), begin, end)); + + VersionSet* const versions = dbfull()->GetVersionSet(); + assert(versions); + assert(versions->GetColumnFamilySet()); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + assert(cfd); + + Version* const current = cfd->current(); + assert(current); + + const VersionStorageInfo* const storage_info = current->storage_info(); + assert(storage_info); + + const auto& blob_files = storage_info->GetBlobFiles(); + ASSERT_EQ(blob_files.size(), 2); + + { + const auto& meta = blob_files.front(); + assert(meta); + + constexpr uint64_t first_expected_bytes = + sizeof(first_value) - 1 + + BlobLogRecord::CalculateAdjustmentForRecordHeader(sizeof(first_key) - + 1); + constexpr uint64_t second_expected_bytes = + sizeof(second_value) - 1 + + BlobLogRecord::CalculateAdjustmentForRecordHeader(sizeof(second_key) - + 1); + constexpr uint64_t third_expected_bytes = + sizeof(third_value) - 1 + + BlobLogRecord::CalculateAdjustmentForRecordHeader(sizeof(third_key) - + 1); + constexpr uint64_t fourth_expected_bytes = + sizeof(fourth_value) - 1 + + BlobLogRecord::CalculateAdjustmentForRecordHeader(sizeof(fourth_key) - + 1); + + ASSERT_EQ(meta->GetTotalBlobCount(), 4); + ASSERT_EQ(meta->GetTotalBlobBytes(), + first_expected_bytes + second_expected_bytes + + third_expected_bytes + fourth_expected_bytes); + ASSERT_EQ(meta->GetGarbageBlobCount(), 2); + ASSERT_EQ(meta->GetGarbageBlobBytes(), + first_expected_bytes + second_expected_bytes); + } + + { + const auto& meta = blob_files.back(); + assert(meta); + + constexpr uint64_t new_first_expected_bytes = + sizeof(new_first_value) - 1 + + BlobLogRecord::CalculateAdjustmentForRecordHeader(sizeof(first_key) - + 1); + constexpr uint64_t new_second_expected_bytes = + sizeof(new_second_value) - 1 + + BlobLogRecord::CalculateAdjustmentForRecordHeader(sizeof(second_key) - + 1); + + ASSERT_EQ(meta->GetTotalBlobCount(), 2); + ASSERT_EQ(meta->GetTotalBlobBytes(), + new_first_expected_bytes + new_second_expected_bytes); + ASSERT_EQ(meta->GetGarbageBlobCount(), 0); + ASSERT_EQ(meta->GetGarbageBlobBytes(), 0); + } +} + +TEST_F(DBBlobCompactionTest, MergeBlobWithBase) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + options.disable_auto_compactions = true; + + Reopen(options); + ASSERT_OK(Put("Key1", "v1_1")); + ASSERT_OK(Put("Key2", "v2_1")); + ASSERT_OK(Flush()); + + ASSERT_OK(Merge("Key1", "v1_2")); + ASSERT_OK(Merge("Key2", "v2_2")); + ASSERT_OK(Flush()); + + ASSERT_OK(Merge("Key1", "v1_3")); + ASSERT_OK(Flush()); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr, + /*end=*/nullptr)); + ASSERT_EQ(Get("Key1"), "v1_1,v1_2,v1_3"); + ASSERT_EQ(Get("Key2"), "v2_1,v2_2"); + Close(); +} + +TEST_F(DBBlobCompactionTest, CompactionReadaheadGarbageCollection) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + options.enable_blob_garbage_collection = true; + options.blob_garbage_collection_age_cutoff = 1.0; + options.blob_compaction_readahead_size = 1 << 10; + options.disable_auto_compactions = true; + + Reopen(options); + + ASSERT_OK(Put("key", "lime")); + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("key", "pie")); + ASSERT_OK(Put("foo", "baz")); + ASSERT_OK(Flush()); + + size_t num_non_prefetch_reads = 0; + SyncPoint::GetInstance()->SetCallBack( + "BlobFileReader::GetBlob:ReadFromFile", + [&num_non_prefetch_reads](void* /* arg */) { ++num_non_prefetch_reads; }); + SyncPoint::GetInstance()->EnableProcessing(); + + constexpr Slice* begin = nullptr; + constexpr Slice* end = nullptr; + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), begin, end)); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + ASSERT_EQ(Get("key"), "pie"); + ASSERT_EQ(Get("foo"), "baz"); + ASSERT_EQ(num_non_prefetch_reads, 0); + + Close(); +} + +TEST_F(DBBlobCompactionTest, CompactionReadaheadFilter) { + Options options = GetDefaultOptions(); + + std::unique_ptr compaction_filter_guard( + new ValueMutationFilter("pie")); + + options.compaction_filter = compaction_filter_guard.get(); + options.enable_blob_files = true; + options.min_blob_size = 0; + options.blob_compaction_readahead_size = 1 << 10; + options.disable_auto_compactions = true; + + Reopen(options); + + ASSERT_OK(Put("key", "lime")); + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Flush()); + + size_t num_non_prefetch_reads = 0; + SyncPoint::GetInstance()->SetCallBack( + "BlobFileReader::GetBlob:ReadFromFile", + [&num_non_prefetch_reads](void* /* arg */) { ++num_non_prefetch_reads; }); + SyncPoint::GetInstance()->EnableProcessing(); + + constexpr Slice* begin = nullptr; + constexpr Slice* end = nullptr; + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), begin, end)); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + ASSERT_EQ(Get("key"), "limepie"); + ASSERT_EQ(Get("foo"), "barpie"); + ASSERT_EQ(num_non_prefetch_reads, 0); + + Close(); +} + +TEST_F(DBBlobCompactionTest, CompactionReadaheadMerge) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + options.blob_compaction_readahead_size = 1 << 10; + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + options.disable_auto_compactions = true; + + Reopen(options); + + ASSERT_OK(Put("key", "lime")); + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Flush()); + + ASSERT_OK(Merge("key", "pie")); + ASSERT_OK(Merge("foo", "baz")); + ASSERT_OK(Flush()); + + size_t num_non_prefetch_reads = 0; + SyncPoint::GetInstance()->SetCallBack( + "BlobFileReader::GetBlob:ReadFromFile", + [&num_non_prefetch_reads](void* /* arg */) { ++num_non_prefetch_reads; }); + SyncPoint::GetInstance()->EnableProcessing(); + + constexpr Slice* begin = nullptr; + constexpr Slice* end = nullptr; + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), begin, end)); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + ASSERT_EQ(Get("key"), "lime,pie"); + ASSERT_EQ(Get("foo"), "bar,baz"); + ASSERT_EQ(num_non_prefetch_reads, 0); + + Close(); +} + +TEST_F(DBBlobCompactionTest, CompactionDoNotFillCache) { + Options options = GetDefaultOptions(); + + options.enable_blob_files = true; + options.min_blob_size = 0; + options.enable_blob_garbage_collection = true; + options.blob_garbage_collection_age_cutoff = 1.0; + options.disable_auto_compactions = true; + options.statistics = CreateDBStatistics(); + + LRUCacheOptions cache_options; + cache_options.capacity = 1 << 20; + cache_options.metadata_charge_policy = kDontChargeCacheMetadata; + + options.blob_cache = NewLRUCache(cache_options); + + Reopen(options); + + ASSERT_OK(Put("key", "lime")); + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("key", "pie")); + ASSERT_OK(Put("foo", "baz")); + ASSERT_OK(Flush()); + + constexpr Slice* begin = nullptr; + constexpr Slice* end = nullptr; + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), begin, end)); + + ASSERT_EQ(options.statistics->getTickerCount(BLOB_DB_CACHE_ADD), 0); + + Close(); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/blob/db_blob_corruption_test.cc b/librocksdb-sys/rocksdb/db/blob/db_blob_corruption_test.cc new file mode 100644 index 0000000..694b25b --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/db_blob_corruption_test.cc @@ -0,0 +1,80 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/db_test_util.h" +#include "port/stack_trace.h" +#include "test_util/sync_point.h" + +namespace ROCKSDB_NAMESPACE { + +class DBBlobCorruptionTest : public DBTestBase { + protected: + DBBlobCorruptionTest() + : DBTestBase("db_blob_corruption_test", /* env_do_fsync */ false) {} + + void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) { + // Pick file to corrupt + std::vector filenames; + ASSERT_OK(env_->GetChildren(dbname_, &filenames)); + uint64_t number; + FileType type; + std::string fname; + uint64_t picked_number = kInvalidBlobFileNumber; + for (size_t i = 0; i < filenames.size(); i++) { + if (ParseFileName(filenames[i], &number, &type) && type == filetype && + number > picked_number) { // Pick latest file + fname = dbname_ + "/" + filenames[i]; + picked_number = number; + } + } + ASSERT_TRUE(!fname.empty()) << filetype; + ASSERT_OK(test::CorruptFile(env_, fname, offset, bytes_to_corrupt)); + } +}; + +TEST_F(DBBlobCorruptionTest, VerifyWholeBlobFileChecksum) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + options.create_if_missing = true; + options.file_checksum_gen_factory = + ROCKSDB_NAMESPACE::GetFileChecksumGenCrc32cFactory(); + Reopen(options); + + ASSERT_OK(Put(Slice("key_1"), Slice("blob_value_1"))); + ASSERT_OK(Flush()); + ASSERT_OK(Put(Slice("key_2"), Slice("blob_value_2"))); + ASSERT_OK(Flush()); + ASSERT_OK(db_->VerifyFileChecksums(ReadOptions())); + Close(); + + Corrupt(kBlobFile, 0, 2); + + ASSERT_OK(TryReopen(options)); + + int count{0}; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::VerifyFullFileChecksum:mismatch", [&](void* arg) { + const Status* s = static_cast(arg); + ASSERT_NE(s, nullptr); + ++count; + ASSERT_NOK(*s); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_TRUE(db_->VerifyFileChecksums(ReadOptions()).IsCorruption()); + ASSERT_EQ(1, count); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/blob/db_blob_index_test.cc b/librocksdb-sys/rocksdb/db/blob/db_blob_index_test.cc new file mode 100644 index 0000000..eabca13 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/db_blob_index_test.cc @@ -0,0 +1,596 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include +#include +#include + +#include "db/arena_wrapped_db_iter.h" +#include "db/blob/blob_index.h" +#include "db/column_family.h" +#include "db/db_iter.h" +#include "db/db_test_util.h" +#include "db/dbformat.h" +#include "db/write_batch_internal.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "util/string_util.h" +#include "utilities/merge_operators.h" + +namespace ROCKSDB_NAMESPACE { + +// kTypeBlobIndex is a value type used by BlobDB only. The base rocksdb +// should accept the value type on write, and report not supported value +// for reads, unless caller request for it explicitly. The base rocksdb +// doesn't understand format of actual blob index (the value). +class DBBlobIndexTest : public DBTestBase { + public: + enum Tier { + kMemtable = 0, + kImmutableMemtables = 1, + kL0SstFile = 2, + kLnSstFile = 3, + }; + const std::vector kAllTiers = {Tier::kMemtable, + Tier::kImmutableMemtables, + Tier::kL0SstFile, Tier::kLnSstFile}; + + DBBlobIndexTest() : DBTestBase("db_blob_index_test", /*env_do_fsync=*/true) {} + + ColumnFamilyHandle* cfh() { return dbfull()->DefaultColumnFamily(); } + + ColumnFamilyData* cfd() { + return static_cast_with_check(cfh())->cfd(); + } + + Status PutBlobIndex(WriteBatch* batch, const Slice& key, + const Slice& blob_index) { + return WriteBatchInternal::PutBlobIndex(batch, cfd()->GetID(), key, + blob_index); + } + + Status Write(WriteBatch* batch) { + return dbfull()->Write(WriteOptions(), batch); + } + + std::string GetImpl(const Slice& key, bool* is_blob_index = nullptr, + const Snapshot* snapshot = nullptr) { + ReadOptions read_options; + read_options.snapshot = snapshot; + PinnableSlice value; + DBImpl::GetImplOptions get_impl_options; + get_impl_options.column_family = cfh(); + get_impl_options.value = &value; + get_impl_options.is_blob_index = is_blob_index; + auto s = dbfull()->GetImpl(read_options, key, get_impl_options); + if (s.IsNotFound()) { + return "NOT_FOUND"; + } + if (s.IsCorruption()) { + return "CORRUPTION"; + } + if (s.IsNotSupported()) { + return "NOT_SUPPORTED"; + } + if (!s.ok()) { + return s.ToString(); + } + return value.ToString(); + } + + std::string GetBlobIndex(const Slice& key, + const Snapshot* snapshot = nullptr) { + bool is_blob_index = false; + std::string value = GetImpl(key, &is_blob_index, snapshot); + if (!is_blob_index) { + return "NOT_BLOB"; + } + return value; + } + + ArenaWrappedDBIter* GetBlobIterator() { + return dbfull()->NewIteratorImpl( + ReadOptions(), cfd(), dbfull()->GetLatestSequenceNumber(), + nullptr /*read_callback*/, true /*expose_blob_index*/); + } + + Options GetTestOptions() { + Options options; + options.env = CurrentOptions().env; + options.create_if_missing = true; + options.num_levels = 2; + options.disable_auto_compactions = true; + // Disable auto flushes. + options.max_write_buffer_number = 10; + options.min_write_buffer_number_to_merge = 10; + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + return options; + } + + void MoveDataTo(Tier tier) { + switch (tier) { + case Tier::kMemtable: + break; + case Tier::kImmutableMemtables: + ASSERT_OK(dbfull()->TEST_SwitchMemtable()); + break; + case Tier::kL0SstFile: + ASSERT_OK(Flush()); + break; + case Tier::kLnSstFile: + ASSERT_OK(Flush()); + ASSERT_OK(Put("a", "dummy")); + ASSERT_OK(Put("z", "dummy")); + ASSERT_OK(Flush()); + ASSERT_OK( + dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("0,1", FilesPerLevel()); + break; + } + } +}; + +// Note: the following test case pertains to the StackableDB-based BlobDB +// implementation. We should be able to write kTypeBlobIndex to memtables and +// SST files. +TEST_F(DBBlobIndexTest, Write) { + for (auto tier : kAllTiers) { + DestroyAndReopen(GetTestOptions()); + + std::vector> key_values; + + constexpr size_t num_key_values = 5; + + key_values.reserve(num_key_values); + + for (size_t i = 1; i <= num_key_values; ++i) { + std::string key = "key" + std::to_string(i); + + std::string blob_index; + BlobIndex::EncodeInlinedTTL(&blob_index, /* expiration */ 9876543210, + "blob" + std::to_string(i)); + + key_values.emplace_back(std::move(key), std::move(blob_index)); + } + + for (const auto& key_value : key_values) { + WriteBatch batch; + ASSERT_OK(PutBlobIndex(&batch, key_value.first, key_value.second)); + ASSERT_OK(Write(&batch)); + } + + MoveDataTo(tier); + + for (const auto& key_value : key_values) { + ASSERT_EQ(GetBlobIndex(key_value.first), key_value.second); + } + } +} + +// Note: the following test case pertains to the StackableDB-based BlobDB +// implementation. Get should be able to return blob index if is_blob_index is +// provided, otherwise it should return Status::NotSupported (when reading from +// memtable) or Status::Corruption (when reading from SST). Reading from SST +// returns Corruption because we can't differentiate between the application +// accidentally opening the base DB of a stacked BlobDB and actual corruption +// when using the integrated BlobDB. +TEST_F(DBBlobIndexTest, Get) { + std::string blob_index; + BlobIndex::EncodeInlinedTTL(&blob_index, /* expiration */ 9876543210, "blob"); + + for (auto tier : kAllTiers) { + DestroyAndReopen(GetTestOptions()); + + WriteBatch batch; + ASSERT_OK(batch.Put("key", "value")); + ASSERT_OK(PutBlobIndex(&batch, "blob_key", blob_index)); + ASSERT_OK(Write(&batch)); + + MoveDataTo(tier); + + // Verify normal value + bool is_blob_index = false; + PinnableSlice value; + ASSERT_EQ("value", Get("key")); + ASSERT_EQ("value", GetImpl("key")); + ASSERT_EQ("value", GetImpl("key", &is_blob_index)); + ASSERT_FALSE(is_blob_index); + + // Verify blob index + if (tier <= kImmutableMemtables) { + ASSERT_TRUE(Get("blob_key", &value).IsNotSupported()); + ASSERT_EQ("NOT_SUPPORTED", GetImpl("blob_key")); + } else { + ASSERT_TRUE(Get("blob_key", &value).IsCorruption()); + ASSERT_EQ("CORRUPTION", GetImpl("blob_key")); + } + ASSERT_EQ(blob_index, GetImpl("blob_key", &is_blob_index)); + ASSERT_TRUE(is_blob_index); + } +} + +// Note: the following test case pertains to the StackableDB-based BlobDB +// implementation. Get should NOT return Status::NotSupported/Status::Corruption +// if blob index is updated with a normal value. See the test case above for +// more details. +TEST_F(DBBlobIndexTest, Updated) { + std::string blob_index; + BlobIndex::EncodeInlinedTTL(&blob_index, /* expiration */ 9876543210, "blob"); + + for (auto tier : kAllTiers) { + DestroyAndReopen(GetTestOptions()); + WriteBatch batch; + for (int i = 0; i < 10; i++) { + ASSERT_OK(PutBlobIndex(&batch, "key" + std::to_string(i), blob_index)); + } + ASSERT_OK(Write(&batch)); + // Avoid blob values from being purged. + const Snapshot* snapshot = dbfull()->GetSnapshot(); + ASSERT_OK(Put("key1", "new_value")); + ASSERT_OK(Merge("key2", "a")); + ASSERT_OK(Merge("key2", "b")); + ASSERT_OK(Merge("key2", "c")); + ASSERT_OK(Delete("key3")); + ASSERT_OK(SingleDelete("key4")); + ASSERT_OK(Delete("key5")); + ASSERT_OK(Merge("key5", "a")); + ASSERT_OK(Merge("key5", "b")); + ASSERT_OK(Merge("key5", "c")); + ASSERT_OK(dbfull()->DeleteRange(WriteOptions(), cfh(), "key6", "key9")); + MoveDataTo(tier); + for (int i = 0; i < 10; i++) { + ASSERT_EQ(blob_index, GetBlobIndex("key" + std::to_string(i), snapshot)); + } + ASSERT_EQ("new_value", Get("key1")); + if (tier <= kImmutableMemtables) { + ASSERT_EQ("NOT_SUPPORTED", GetImpl("key2")); + } else { + ASSERT_EQ("CORRUPTION", GetImpl("key2")); + } + ASSERT_EQ("NOT_FOUND", Get("key3")); + ASSERT_EQ("NOT_FOUND", Get("key4")); + ASSERT_EQ("a,b,c", GetImpl("key5")); + for (int i = 6; i < 9; i++) { + ASSERT_EQ("NOT_FOUND", Get("key" + std::to_string(i))); + } + ASSERT_EQ(blob_index, GetBlobIndex("key9")); + dbfull()->ReleaseSnapshot(snapshot); + } +} + +// Note: the following test case pertains to the StackableDB-based BlobDB +// implementation. When a blob iterator is used, it should set the +// expose_blob_index flag for the underlying DBIter, and retrieve/return the +// corresponding blob value. If a regular DBIter is created (i.e. +// expose_blob_index is not set), it should return Status::Corruption. +TEST_F(DBBlobIndexTest, Iterate) { + const std::vector> data = { + /*00*/ {kTypeValue}, + /*01*/ {kTypeBlobIndex}, + /*02*/ {kTypeValue}, + /*03*/ {kTypeBlobIndex, kTypeValue}, + /*04*/ {kTypeValue}, + /*05*/ {kTypeValue, kTypeBlobIndex}, + /*06*/ {kTypeValue}, + /*07*/ {kTypeDeletion, kTypeBlobIndex}, + /*08*/ {kTypeValue}, + /*09*/ {kTypeSingleDeletion, kTypeBlobIndex}, + /*10*/ {kTypeValue}, + /*11*/ {kTypeMerge, kTypeMerge, kTypeMerge, kTypeBlobIndex}, + /*12*/ {kTypeValue}, + /*13*/ + {kTypeMerge, kTypeMerge, kTypeMerge, kTypeDeletion, kTypeBlobIndex}, + /*14*/ {kTypeValue}, + /*15*/ {kTypeBlobIndex}, + /*16*/ {kTypeValue}, + }; + + auto get_key = [](int index) { + char buf[20]; + snprintf(buf, sizeof(buf), "%02d", index); + return "key" + std::string(buf); + }; + + auto get_value = [&](int index, int version) { + return get_key(index) + "_value" + std::to_string(version); + }; + + auto check_iterator = [&](Iterator* iterator, Status::Code expected_status, + const Slice& expected_value) { + ASSERT_EQ(expected_status, iterator->status().code()); + if (expected_status == Status::kOk) { + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ(expected_value, iterator->value()); + } else { + ASSERT_FALSE(iterator->Valid()); + } + }; + + auto create_normal_iterator = [&]() -> Iterator* { + return dbfull()->NewIterator(ReadOptions()); + }; + + auto create_blob_iterator = [&]() -> Iterator* { return GetBlobIterator(); }; + + auto check_is_blob = [&](bool is_blob) { + return [is_blob](Iterator* iterator) { + ASSERT_EQ(is_blob, + reinterpret_cast(iterator)->IsBlob()); + }; + }; + + auto verify = [&](int index, Status::Code expected_status, + const Slice& forward_value, const Slice& backward_value, + std::function create_iterator, + std::function extra_check = nullptr) { + // Seek + auto* iterator = create_iterator(); + ASSERT_OK(iterator->status()); + ASSERT_OK(iterator->Refresh()); + iterator->Seek(get_key(index)); + check_iterator(iterator, expected_status, forward_value); + if (extra_check) { + extra_check(iterator); + } + delete iterator; + + // Next + iterator = create_iterator(); + ASSERT_OK(iterator->Refresh()); + iterator->Seek(get_key(index - 1)); + ASSERT_TRUE(iterator->Valid()); + ASSERT_OK(iterator->status()); + iterator->Next(); + check_iterator(iterator, expected_status, forward_value); + if (extra_check) { + extra_check(iterator); + } + delete iterator; + + // SeekForPrev + iterator = create_iterator(); + ASSERT_OK(iterator->status()); + ASSERT_OK(iterator->Refresh()); + iterator->SeekForPrev(get_key(index)); + check_iterator(iterator, expected_status, backward_value); + if (extra_check) { + extra_check(iterator); + } + delete iterator; + + // Prev + iterator = create_iterator(); + iterator->Seek(get_key(index + 1)); + ASSERT_TRUE(iterator->Valid()); + ASSERT_OK(iterator->status()); + iterator->Prev(); + check_iterator(iterator, expected_status, backward_value); + if (extra_check) { + extra_check(iterator); + } + delete iterator; + }; + + for (auto tier : {Tier::kMemtable} /*kAllTiers*/) { + // Avoid values from being purged. + std::vector snapshots; + DestroyAndReopen(GetTestOptions()); + + // fill data + for (int i = 0; i < static_cast(data.size()); i++) { + for (int j = static_cast(data[i].size()) - 1; j >= 0; j--) { + std::string key = get_key(i); + std::string value = get_value(i, j); + WriteBatch batch; + switch (data[i][j]) { + case kTypeValue: + ASSERT_OK(Put(key, value)); + break; + case kTypeDeletion: + ASSERT_OK(Delete(key)); + break; + case kTypeSingleDeletion: + ASSERT_OK(SingleDelete(key)); + break; + case kTypeMerge: + ASSERT_OK(Merge(key, value)); + break; + case kTypeBlobIndex: + ASSERT_OK(PutBlobIndex(&batch, key, value)); + ASSERT_OK(Write(&batch)); + break; + default: + FAIL(); + }; + } + snapshots.push_back(dbfull()->GetSnapshot()); + } + ASSERT_OK( + dbfull()->DeleteRange(WriteOptions(), cfh(), get_key(15), get_key(16))); + snapshots.push_back(dbfull()->GetSnapshot()); + MoveDataTo(tier); + + // Normal iterator + verify(1, Status::kCorruption, "", "", create_normal_iterator); + verify(3, Status::kCorruption, "", "", create_normal_iterator); + verify(5, Status::kOk, get_value(5, 0), get_value(5, 0), + create_normal_iterator); + verify(7, Status::kOk, get_value(8, 0), get_value(6, 0), + create_normal_iterator); + verify(9, Status::kOk, get_value(10, 0), get_value(8, 0), + create_normal_iterator); + verify(11, Status::kCorruption, "", "", create_normal_iterator); + verify(13, Status::kOk, + get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0), + get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0), + create_normal_iterator); + verify(15, Status::kOk, get_value(16, 0), get_value(14, 0), + create_normal_iterator); + + // Iterator with blob support + verify(1, Status::kOk, get_value(1, 0), get_value(1, 0), + create_blob_iterator, check_is_blob(true)); + verify(3, Status::kOk, get_value(3, 0), get_value(3, 0), + create_blob_iterator, check_is_blob(true)); + verify(5, Status::kOk, get_value(5, 0), get_value(5, 0), + create_blob_iterator, check_is_blob(false)); + verify(7, Status::kOk, get_value(8, 0), get_value(6, 0), + create_blob_iterator, check_is_blob(false)); + verify(9, Status::kOk, get_value(10, 0), get_value(8, 0), + create_blob_iterator, check_is_blob(false)); + if (tier <= kImmutableMemtables) { + verify(11, Status::kNotSupported, "", "", create_blob_iterator); + } else { + verify(11, Status::kCorruption, "", "", create_blob_iterator); + } + verify(13, Status::kOk, + get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0), + get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0), + create_blob_iterator, check_is_blob(false)); + verify(15, Status::kOk, get_value(16, 0), get_value(14, 0), + create_blob_iterator, check_is_blob(false)); + + // Iterator with blob support and using seek. + ASSERT_OK(dbfull()->SetOptions( + cfh(), {{"max_sequential_skip_in_iterations", "0"}})); + verify(1, Status::kOk, get_value(1, 0), get_value(1, 0), + create_blob_iterator, check_is_blob(true)); + verify(3, Status::kOk, get_value(3, 0), get_value(3, 0), + create_blob_iterator, check_is_blob(true)); + verify(5, Status::kOk, get_value(5, 0), get_value(5, 0), + create_blob_iterator, check_is_blob(false)); + verify(7, Status::kOk, get_value(8, 0), get_value(6, 0), + create_blob_iterator, check_is_blob(false)); + verify(9, Status::kOk, get_value(10, 0), get_value(8, 0), + create_blob_iterator, check_is_blob(false)); + if (tier <= kImmutableMemtables) { + verify(11, Status::kNotSupported, "", "", create_blob_iterator); + } else { + verify(11, Status::kCorruption, "", "", create_blob_iterator); + } + verify(13, Status::kOk, + get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0), + get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0), + create_blob_iterator, check_is_blob(false)); + verify(15, Status::kOk, get_value(16, 0), get_value(14, 0), + create_blob_iterator, check_is_blob(false)); + + for (auto* snapshot : snapshots) { + dbfull()->ReleaseSnapshot(snapshot); + } + } +} + +TEST_F(DBBlobIndexTest, IntegratedBlobIterate) { + const std::vector> data = { + /*00*/ {"Put"}, + /*01*/ {"Put", "Merge", "Merge", "Merge"}, + /*02*/ {"Put"}}; + + auto get_key = [](size_t index) { return ("key" + std::to_string(index)); }; + + auto get_value = [&](size_t index, size_t version) { + return get_key(index) + "_value" + std::to_string(version); + }; + + auto check_iterator = [&](Iterator* iterator, Status expected_status, + const Slice& expected_value) { + ASSERT_EQ(expected_status, iterator->status()); + if (expected_status.ok()) { + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ(expected_value, iterator->value()); + } else { + ASSERT_FALSE(iterator->Valid()); + } + }; + + auto verify = [&](size_t index, Status expected_status, + const Slice& expected_value) { + // Seek + { + Iterator* iterator = db_->NewIterator(ReadOptions()); + std::unique_ptr iterator_guard(iterator); + ASSERT_OK(iterator->status()); + ASSERT_OK(iterator->Refresh()); + iterator->Seek(get_key(index)); + check_iterator(iterator, expected_status, expected_value); + } + // Next + { + Iterator* iterator = db_->NewIterator(ReadOptions()); + std::unique_ptr iterator_guard(iterator); + ASSERT_OK(iterator->Refresh()); + iterator->Seek(get_key(index - 1)); + ASSERT_TRUE(iterator->Valid()); + ASSERT_OK(iterator->status()); + iterator->Next(); + check_iterator(iterator, expected_status, expected_value); + } + // SeekForPrev + { + Iterator* iterator = db_->NewIterator(ReadOptions()); + std::unique_ptr iterator_guard(iterator); + ASSERT_OK(iterator->status()); + ASSERT_OK(iterator->Refresh()); + iterator->SeekForPrev(get_key(index)); + check_iterator(iterator, expected_status, expected_value); + } + // Prev + { + Iterator* iterator = db_->NewIterator(ReadOptions()); + std::unique_ptr iterator_guard(iterator); + iterator->Seek(get_key(index + 1)); + ASSERT_TRUE(iterator->Valid()); + ASSERT_OK(iterator->status()); + iterator->Prev(); + check_iterator(iterator, expected_status, expected_value); + } + }; + + Options options = GetTestOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + + DestroyAndReopen(options); + + // fill data + for (size_t i = 0; i < data.size(); i++) { + for (size_t j = 0; j < data[i].size(); j++) { + std::string key = get_key(i); + std::string value = get_value(i, j); + if (data[i][j] == "Put") { + ASSERT_OK(Put(key, value)); + ASSERT_OK(Flush()); + } else if (data[i][j] == "Merge") { + ASSERT_OK(Merge(key, value)); + ASSERT_OK(Flush()); + } + } + } + + std::string expected_value = get_value(1, 0) + "," + get_value(1, 1) + "," + + get_value(1, 2) + "," + get_value(1, 3); + Status expected_status; + verify(1, expected_status, expected_value); + + // Test DBIter::FindValueForCurrentKeyUsingSeek flow. + ASSERT_OK(dbfull()->SetOptions(cfh(), + {{"max_sequential_skip_in_iterations", "0"}})); + verify(1, expected_status, expected_value); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/blob/prefetch_buffer_collection.cc b/librocksdb-sys/rocksdb/db/blob/prefetch_buffer_collection.cc new file mode 100644 index 0000000..079576f --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/prefetch_buffer_collection.cc @@ -0,0 +1,21 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/prefetch_buffer_collection.h" + +namespace ROCKSDB_NAMESPACE { + +FilePrefetchBuffer* PrefetchBufferCollection::GetOrCreatePrefetchBuffer( + uint64_t file_number) { + auto& prefetch_buffer = prefetch_buffers_[file_number]; + if (!prefetch_buffer) { + prefetch_buffer.reset( + new FilePrefetchBuffer(readahead_size_, readahead_size_)); + } + + return prefetch_buffer.get(); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/blob/prefetch_buffer_collection.h b/librocksdb-sys/rocksdb/db/blob/prefetch_buffer_collection.h new file mode 100644 index 0000000..b973edd --- /dev/null +++ b/librocksdb-sys/rocksdb/db/blob/prefetch_buffer_collection.h @@ -0,0 +1,38 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include +#include + +#include "file/file_prefetch_buffer.h" +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +// A class that owns a collection of FilePrefetchBuffers using the file number +// as key. Used for implementing compaction readahead for blob files. Designed +// to be accessed by a single thread only: every (sub)compaction needs its own +// buffers since they are guaranteed to read different blobs from different +// positions even when reading the same file. +class PrefetchBufferCollection { + public: + explicit PrefetchBufferCollection(uint64_t readahead_size) + : readahead_size_(readahead_size) { + assert(readahead_size_ > 0); + } + + FilePrefetchBuffer* GetOrCreatePrefetchBuffer(uint64_t file_number); + + private: + uint64_t readahead_size_; + std::unordered_map> + prefetch_buffers_; // maps file number to prefetch buffer +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/builder.cc b/librocksdb-sys/rocksdb/db/builder.cc new file mode 100644 index 0000000..a3a6bc4 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/builder.cc @@ -0,0 +1,472 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/builder.h" + +#include +#include +#include + +#include "db/blob/blob_file_builder.h" +#include "db/compaction/compaction_iterator.h" +#include "db/dbformat.h" +#include "db/event_helpers.h" +#include "db/internal_stats.h" +#include "db/merge_helper.h" +#include "db/output_validator.h" +#include "db/range_del_aggregator.h" +#include "db/table_cache.h" +#include "db/version_edit.h" +#include "file/file_util.h" +#include "file/filename.h" +#include "file/read_write_util.h" +#include "file/writable_file_writer.h" +#include "monitoring/iostats_context_imp.h" +#include "monitoring/thread_status_util.h" +#include "options/options_helper.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/iterator.h" +#include "rocksdb/options.h" +#include "rocksdb/table.h" +#include "table/block_based/block_based_table_builder.h" +#include "table/format.h" +#include "table/internal_iterator.h" +#include "table/unique_id_impl.h" +#include "test_util/sync_point.h" +#include "util/stop_watch.h" + +namespace ROCKSDB_NAMESPACE { + +class TableFactory; + +TableBuilder* NewTableBuilder(const TableBuilderOptions& tboptions, + WritableFileWriter* file) { + assert((tboptions.column_family_id == + TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) == + tboptions.column_family_name.empty()); + return tboptions.ioptions.table_factory->NewTableBuilder(tboptions, file); +} + +Status BuildTable( + const std::string& dbname, VersionSet* versions, + const ImmutableDBOptions& db_options, const TableBuilderOptions& tboptions, + const FileOptions& file_options, const ReadOptions& read_options, + TableCache* table_cache, InternalIterator* iter, + std::vector> + range_del_iters, + FileMetaData* meta, std::vector* blob_file_additions, + std::vector snapshots, + SequenceNumber earliest_write_conflict_snapshot, + SequenceNumber job_snapshot, SnapshotChecker* snapshot_checker, + bool paranoid_file_checks, InternalStats* internal_stats, + IOStatus* io_status, const std::shared_ptr& io_tracer, + BlobFileCreationReason blob_creation_reason, + const SeqnoToTimeMapping& seqno_to_time_mapping, EventLogger* event_logger, + int job_id, const Env::IOPriority io_priority, + TableProperties* table_properties, Env::WriteLifeTimeHint write_hint, + const std::string* full_history_ts_low, + BlobFileCompletionCallback* blob_callback, Version* version, + uint64_t* num_input_entries, uint64_t* memtable_payload_bytes, + uint64_t* memtable_garbage_bytes) { + assert((tboptions.column_family_id == + TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) == + tboptions.column_family_name.empty()); + auto& mutable_cf_options = tboptions.moptions; + auto& ioptions = tboptions.ioptions; + // Reports the IOStats for flush for every following bytes. + const size_t kReportFlushIOStatsEvery = 1048576; + OutputValidator output_validator( + tboptions.internal_comparator, + /*enable_order_check=*/ + mutable_cf_options.check_flush_compaction_key_order, + /*enable_hash=*/paranoid_file_checks); + Status s; + meta->fd.file_size = 0; + iter->SeekToFirst(); + std::unique_ptr range_del_agg( + new CompactionRangeDelAggregator(&tboptions.internal_comparator, + snapshots, full_history_ts_low)); + uint64_t num_unfragmented_tombstones = 0; + uint64_t total_tombstone_payload_bytes = 0; + for (auto& range_del_iter : range_del_iters) { + num_unfragmented_tombstones += + range_del_iter->num_unfragmented_tombstones(); + total_tombstone_payload_bytes += + range_del_iter->total_tombstone_payload_bytes(); + range_del_agg->AddTombstones(std::move(range_del_iter)); + } + + std::string fname = TableFileName(ioptions.cf_paths, meta->fd.GetNumber(), + meta->fd.GetPathId()); + std::vector blob_file_paths; + std::string file_checksum = kUnknownFileChecksum; + std::string file_checksum_func_name = kUnknownFileChecksumFuncName; + EventHelpers::NotifyTableFileCreationStarted(ioptions.listeners, dbname, + tboptions.column_family_name, + fname, job_id, tboptions.reason); + Env* env = db_options.env; + assert(env); + FileSystem* fs = db_options.fs.get(); + assert(fs); + + TableProperties tp; + bool table_file_created = false; + if (iter->Valid() || !range_del_agg->IsEmpty()) { + std::unique_ptr compaction_filter; + if (ioptions.compaction_filter_factory != nullptr && + ioptions.compaction_filter_factory->ShouldFilterTableFileCreation( + tboptions.reason)) { + CompactionFilter::Context context; + context.is_full_compaction = false; + context.is_manual_compaction = false; + context.column_family_id = tboptions.column_family_id; + context.reason = tboptions.reason; + compaction_filter = + ioptions.compaction_filter_factory->CreateCompactionFilter(context); + if (compaction_filter != nullptr && + !compaction_filter->IgnoreSnapshots()) { + s.PermitUncheckedError(); + return Status::NotSupported( + "CompactionFilter::IgnoreSnapshots() = false is not supported " + "anymore."); + } + } + + TableBuilder* builder; + std::unique_ptr file_writer; + { + std::unique_ptr file; +#ifndef NDEBUG + bool use_direct_writes = file_options.use_direct_writes; + TEST_SYNC_POINT_CALLBACK("BuildTable:create_file", &use_direct_writes); +#endif // !NDEBUG + IOStatus io_s = NewWritableFile(fs, fname, &file, file_options); + assert(s.ok()); + s = io_s; + if (io_status->ok()) { + *io_status = io_s; + } + if (!s.ok()) { + EventHelpers::LogAndNotifyTableFileCreationFinished( + event_logger, ioptions.listeners, dbname, + tboptions.column_family_name, fname, job_id, meta->fd, + kInvalidBlobFileNumber, tp, tboptions.reason, s, file_checksum, + file_checksum_func_name); + return s; + } + + table_file_created = true; + FileTypeSet tmp_set = ioptions.checksum_handoff_file_types; + file->SetIOPriority(io_priority); + file->SetWriteLifeTimeHint(write_hint); + file_writer.reset(new WritableFileWriter( + std::move(file), fname, file_options, ioptions.clock, io_tracer, + ioptions.stats, ioptions.listeners, + ioptions.file_checksum_gen_factory.get(), + tmp_set.Contains(FileType::kTableFile), false)); + + builder = NewTableBuilder(tboptions, file_writer.get()); + } + + auto ucmp = tboptions.internal_comparator.user_comparator(); + MergeHelper merge( + env, ucmp, ioptions.merge_operator.get(), compaction_filter.get(), + ioptions.logger, true /* internal key corruption is not ok */, + snapshots.empty() ? 0 : snapshots.back(), snapshot_checker); + + std::unique_ptr blob_file_builder( + (mutable_cf_options.enable_blob_files && + tboptions.level_at_creation >= + mutable_cf_options.blob_file_starting_level && + blob_file_additions) + ? new BlobFileBuilder( + versions, fs, &ioptions, &mutable_cf_options, &file_options, + tboptions.db_id, tboptions.db_session_id, job_id, + tboptions.column_family_id, tboptions.column_family_name, + io_priority, write_hint, io_tracer, blob_callback, + blob_creation_reason, &blob_file_paths, blob_file_additions) + : nullptr); + + const std::atomic kManualCompactionCanceledFalse{false}; + CompactionIterator c_iter( + iter, ucmp, &merge, kMaxSequenceNumber, &snapshots, + earliest_write_conflict_snapshot, job_snapshot, snapshot_checker, env, + ShouldReportDetailedTime(env, ioptions.stats), + true /* internal key corruption is not ok */, range_del_agg.get(), + blob_file_builder.get(), ioptions.allow_data_in_errors, + ioptions.enforce_single_del_contracts, + /*manual_compaction_canceled=*/kManualCompactionCanceledFalse, + true /* must_count_input_entries */, + /*compaction=*/nullptr, compaction_filter.get(), + /*shutting_down=*/nullptr, db_options.info_log, full_history_ts_low); + + const size_t ts_sz = ucmp->timestamp_size(); + const bool strip_timestamp = + ts_sz > 0 && !ioptions.persist_user_defined_timestamps; + + std::string key_after_flush_buf; + c_iter.SeekToFirst(); + for (; c_iter.Valid(); c_iter.Next()) { + const Slice& key = c_iter.key(); + const Slice& value = c_iter.value(); + const ParsedInternalKey& ikey = c_iter.ikey(); + Slice key_after_flush = key; + // If user defined timestamps will be stripped from user key after flush, + // the in memory version of the key act logically the same as one with a + // minimum timestamp. We update the timestamp here so file boundary and + // output validator, block builder all see the effect of the stripping. + if (strip_timestamp) { + key_after_flush_buf.clear(); + ReplaceInternalKeyWithMinTimestamp(&key_after_flush_buf, key, ts_sz); + key_after_flush = key_after_flush_buf; + } + + // Generate a rolling 64-bit hash of the key and values + // Note : + // Here "key" integrates 'sequence_number'+'kType'+'user key'. + s = output_validator.Add(key_after_flush, value); + if (!s.ok()) { + break; + } + builder->Add(key_after_flush, value); + + s = meta->UpdateBoundaries(key_after_flush, value, ikey.sequence, + ikey.type); + if (!s.ok()) { + break; + } + + // TODO(noetzli): Update stats after flush, too. + if (io_priority == Env::IO_HIGH && + IOSTATS(bytes_written) >= kReportFlushIOStatsEvery) { + ThreadStatusUtil::SetThreadOperationProperty( + ThreadStatus::FLUSH_BYTES_WRITTEN, IOSTATS(bytes_written)); + } + } + if (!s.ok()) { + c_iter.status().PermitUncheckedError(); + } else if (!c_iter.status().ok()) { + s = c_iter.status(); + } + + if (s.ok()) { + auto range_del_it = range_del_agg->NewIterator(); + Slice last_tombstone_start_user_key{}; + for (range_del_it->SeekToFirst(); range_del_it->Valid(); + range_del_it->Next()) { + auto tombstone = range_del_it->Tombstone(); + auto kv = tombstone.Serialize(); + // TODO(yuzhangyu): handle range deletion for UDT in memtables only. + builder->Add(kv.first.Encode(), kv.second); + InternalKey tombstone_end = tombstone.SerializeEndKey(); + meta->UpdateBoundariesForRange(kv.first, tombstone_end, tombstone.seq_, + tboptions.internal_comparator); + if (version) { + if (last_tombstone_start_user_key.empty() || + ucmp->CompareWithoutTimestamp(last_tombstone_start_user_key, + range_del_it->start_key()) < 0) { + SizeApproximationOptions approx_opts; + approx_opts.files_size_error_margin = 0.1; + meta->compensated_range_deletion_size += versions->ApproximateSize( + approx_opts, read_options, version, kv.first.Encode(), + tombstone_end.Encode(), 0 /* start_level */, -1 /* end_level */, + TableReaderCaller::kFlush); + } + last_tombstone_start_user_key = range_del_it->start_key(); + } + } + } + + TEST_SYNC_POINT("BuildTable:BeforeFinishBuildTable"); + const bool empty = builder->IsEmpty(); + if (num_input_entries != nullptr) { + assert(c_iter.HasNumInputEntryScanned()); + *num_input_entries = + c_iter.NumInputEntryScanned() + num_unfragmented_tombstones; + } + if (!s.ok() || empty) { + builder->Abandon(); + } else { + std::string seqno_time_mapping_str; + seqno_to_time_mapping.Encode( + seqno_time_mapping_str, meta->fd.smallest_seqno, + meta->fd.largest_seqno, meta->file_creation_time); + builder->SetSeqnoTimeTableProperties( + seqno_time_mapping_str, + ioptions.compaction_style == CompactionStyle::kCompactionStyleFIFO + ? meta->file_creation_time + : meta->oldest_ancester_time); + s = builder->Finish(); + } + if (io_status->ok()) { + *io_status = builder->io_status(); + } + + if (s.ok() && !empty) { + uint64_t file_size = builder->FileSize(); + meta->fd.file_size = file_size; + meta->tail_size = builder->GetTailSize(); + meta->marked_for_compaction = builder->NeedCompact(); + meta->user_defined_timestamps_persisted = + ioptions.persist_user_defined_timestamps; + assert(meta->fd.GetFileSize() > 0); + tp = builder + ->GetTableProperties(); // refresh now that builder is finished + if (memtable_payload_bytes != nullptr && + memtable_garbage_bytes != nullptr) { + const CompactionIterationStats& ci_stats = c_iter.iter_stats(); + uint64_t total_payload_bytes = ci_stats.total_input_raw_key_bytes + + ci_stats.total_input_raw_value_bytes + + total_tombstone_payload_bytes; + uint64_t total_payload_bytes_written = + (tp.raw_key_size + tp.raw_value_size); + // Prevent underflow, which may still happen at this point + // since we only support inserts, deletes, and deleteRanges. + if (total_payload_bytes_written <= total_payload_bytes) { + *memtable_payload_bytes = total_payload_bytes; + *memtable_garbage_bytes = + total_payload_bytes - total_payload_bytes_written; + } else { + *memtable_payload_bytes = 0; + *memtable_garbage_bytes = 0; + } + } + if (table_properties) { + *table_properties = tp; + } + } + delete builder; + + // Finish and check for file errors + TEST_SYNC_POINT("BuildTable:BeforeSyncTable"); + if (s.ok() && !empty) { + StopWatch sw(ioptions.clock, ioptions.stats, TABLE_SYNC_MICROS); + *io_status = file_writer->Sync(ioptions.use_fsync); + } + TEST_SYNC_POINT("BuildTable:BeforeCloseTableFile"); + if (s.ok() && io_status->ok() && !empty) { + *io_status = file_writer->Close(); + } + if (s.ok() && io_status->ok() && !empty) { + // Add the checksum information to file metadata. + meta->file_checksum = file_writer->GetFileChecksum(); + meta->file_checksum_func_name = file_writer->GetFileChecksumFuncName(); + file_checksum = meta->file_checksum; + file_checksum_func_name = meta->file_checksum_func_name; + // Set unique_id only if db_id and db_session_id exist + if (!tboptions.db_id.empty() && !tboptions.db_session_id.empty()) { + if (!GetSstInternalUniqueId(tboptions.db_id, tboptions.db_session_id, + meta->fd.GetNumber(), &(meta->unique_id)) + .ok()) { + // if failed to get unique id, just set it Null + meta->unique_id = kNullUniqueId64x2; + } + } + } + + if (s.ok()) { + s = *io_status; + } + + // TODO(yuzhangyu): handle the key copy in the blob when ts should be + // stripped. + if (blob_file_builder) { + if (s.ok()) { + s = blob_file_builder->Finish(); + } else { + blob_file_builder->Abandon(s); + } + blob_file_builder.reset(); + } + + // TODO Also check the IO status when create the Iterator. + + TEST_SYNC_POINT("BuildTable:BeforeOutputValidation"); + if (s.ok() && !empty) { + // Verify that the table is usable + // We set for_compaction to false and don't OptimizeForCompactionTableRead + // here because this is a special case after we finish the table building. + // No matter whether use_direct_io_for_flush_and_compaction is true, + // the goal is to cache it here for further user reads. + std::unique_ptr it(table_cache->NewIterator( + read_options, file_options, tboptions.internal_comparator, *meta, + nullptr /* range_del_agg */, mutable_cf_options.prefix_extractor, + nullptr, + (internal_stats == nullptr) ? nullptr + : internal_stats->GetFileReadHist(0), + TableReaderCaller::kFlush, /*arena=*/nullptr, + /*skip_filter=*/false, tboptions.level_at_creation, + MaxFileSizeForL0MetaPin(mutable_cf_options), + /*smallest_compaction_key=*/nullptr, + /*largest_compaction_key*/ nullptr, + /*allow_unprepared_value*/ false, + mutable_cf_options.block_protection_bytes_per_key)); + s = it->status(); + if (s.ok() && paranoid_file_checks) { + OutputValidator file_validator(tboptions.internal_comparator, + /*enable_order_check=*/true, + /*enable_hash=*/true); + for (it->SeekToFirst(); it->Valid(); it->Next()) { + // Generate a rolling 64-bit hash of the key and values + file_validator.Add(it->key(), it->value()).PermitUncheckedError(); + } + s = it->status(); + if (s.ok() && !output_validator.CompareValidator(file_validator)) { + s = Status::Corruption("Paranoid checksums do not match"); + } + } + } + } + + // Check for input iterator errors + if (!iter->status().ok()) { + s = iter->status(); + } + + if (!s.ok() || meta->fd.GetFileSize() == 0) { + TEST_SYNC_POINT("BuildTable:BeforeDeleteFile"); + + constexpr IODebugContext* dbg = nullptr; + + if (table_file_created) { + Status ignored = fs->DeleteFile(fname, IOOptions(), dbg); + ignored.PermitUncheckedError(); + } + + assert(blob_file_additions || blob_file_paths.empty()); + + if (blob_file_additions) { + for (const std::string& blob_file_path : blob_file_paths) { + Status ignored = DeleteDBFile(&db_options, blob_file_path, dbname, + /*force_bg=*/false, /*force_fg=*/false); + ignored.PermitUncheckedError(); + TEST_SYNC_POINT("BuildTable::AfterDeleteFile"); + } + } + } + + Status status_for_listener = s; + if (meta->fd.GetFileSize() == 0) { + fname = "(nil)"; + if (s.ok()) { + status_for_listener = Status::Aborted("Empty SST file not kept"); + } + } + // Output to event logger and fire events. + EventHelpers::LogAndNotifyTableFileCreationFinished( + event_logger, ioptions.listeners, dbname, tboptions.column_family_name, + fname, job_id, meta->fd, meta->oldest_blob_file_number, tp, + tboptions.reason, status_for_listener, file_checksum, + file_checksum_func_name); + + return s; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/builder.h b/librocksdb-sys/rocksdb/db/builder.h new file mode 100644 index 0000000..6a6a186 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/builder.h @@ -0,0 +1,78 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once +#include +#include +#include + +#include "db/range_tombstone_fragmenter.h" +#include "db/seqno_to_time_mapping.h" +#include "db/table_properties_collector.h" +#include "db/version_set.h" +#include "logging/event_logger.h" +#include "options/cf_options.h" +#include "rocksdb/comparator.h" +#include "rocksdb/env.h" +#include "rocksdb/listener.h" +#include "rocksdb/options.h" +#include "rocksdb/status.h" +#include "rocksdb/table_properties.h" +#include "rocksdb/types.h" +#include "table/scoped_arena_iterator.h" + +namespace ROCKSDB_NAMESPACE { + +struct FileMetaData; + +class VersionSet; +class BlobFileAddition; +class SnapshotChecker; +class TableCache; +class TableBuilder; +class WritableFileWriter; +class InternalStats; +class BlobFileCompletionCallback; + +// Convenience function for NewTableBuilder on the embedded table_factory. +TableBuilder* NewTableBuilder(const TableBuilderOptions& tboptions, + WritableFileWriter* file); + +// Build a Table file from the contents of *iter. The generated file +// will be named according to number specified in meta. On success, the rest of +// *meta will be filled with metadata about the generated table. +// If no data is present in *iter, meta->file_size will be set to +// zero, and no Table file will be produced. +// +// @param column_family_name Name of the column family that is also identified +// by column_family_id, or empty string if unknown. +extern Status BuildTable( + const std::string& dbname, VersionSet* versions, + const ImmutableDBOptions& db_options, const TableBuilderOptions& tboptions, + const FileOptions& file_options, const ReadOptions& read_options, + TableCache* table_cache, InternalIterator* iter, + std::vector> + range_del_iters, + FileMetaData* meta, std::vector* blob_file_additions, + std::vector snapshots, + SequenceNumber earliest_write_conflict_snapshot, + SequenceNumber job_snapshot, SnapshotChecker* snapshot_checker, + bool paranoid_file_checks, InternalStats* internal_stats, + IOStatus* io_status, const std::shared_ptr& io_tracer, + BlobFileCreationReason blob_creation_reason, + const SeqnoToTimeMapping& seqno_to_time_mapping, + EventLogger* event_logger = nullptr, int job_id = 0, + const Env::IOPriority io_priority = Env::IO_HIGH, + TableProperties* table_properties = nullptr, + Env::WriteLifeTimeHint write_hint = Env::WLTH_NOT_SET, + const std::string* full_history_ts_low = nullptr, + BlobFileCompletionCallback* blob_callback = nullptr, + Version* version = nullptr, uint64_t* num_input_entries = nullptr, + uint64_t* memtable_payload_bytes = nullptr, + uint64_t* memtable_garbage_bytes = nullptr); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/c.cc b/librocksdb-sys/rocksdb/db/c.cc new file mode 100644 index 0000000..42ddc52 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/c.cc @@ -0,0 +1,6722 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "rocksdb/c.h" + +#include +#include +#include +#include + +#include "port/port.h" +#include "rocksdb/advanced_cache.h" +#include "rocksdb/compaction_filter.h" +#include "rocksdb/comparator.h" +#include "rocksdb/convenience.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/experimental.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/iterator.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/options.h" +#include "rocksdb/perf_context.h" +#include "rocksdb/rate_limiter.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/statistics.h" +#include "rocksdb/status.h" +#include "rocksdb/table.h" +#include "rocksdb/universal_compaction.h" +#include "rocksdb/utilities/backup_engine.h" +#include "rocksdb/utilities/checkpoint.h" +#include "rocksdb/utilities/db_ttl.h" +#include "rocksdb/utilities/memory_util.h" +#include "rocksdb/utilities/optimistic_transaction_db.h" +#include "rocksdb/utilities/options_util.h" +#include "rocksdb/utilities/table_properties_collectors.h" +#include "rocksdb/utilities/transaction.h" +#include "rocksdb/utilities/transaction_db.h" +#include "rocksdb/utilities/write_batch_with_index.h" +#include "rocksdb/write_batch.h" +#include "utilities/merge_operators.h" + +using ROCKSDB_NAMESPACE::BackupEngine; +using ROCKSDB_NAMESPACE::BackupEngineOptions; +using ROCKSDB_NAMESPACE::BackupID; +using ROCKSDB_NAMESPACE::BackupInfo; +using ROCKSDB_NAMESPACE::BatchResult; +using ROCKSDB_NAMESPACE::BlockBasedTableOptions; +using ROCKSDB_NAMESPACE::BottommostLevelCompaction; +using ROCKSDB_NAMESPACE::BytewiseComparator; +using ROCKSDB_NAMESPACE::Cache; +using ROCKSDB_NAMESPACE::Checkpoint; +using ROCKSDB_NAMESPACE::ColumnFamilyDescriptor; +using ROCKSDB_NAMESPACE::ColumnFamilyHandle; +using ROCKSDB_NAMESPACE::ColumnFamilyMetaData; +using ROCKSDB_NAMESPACE::ColumnFamilyOptions; +using ROCKSDB_NAMESPACE::CompactionFilter; +using ROCKSDB_NAMESPACE::CompactionFilterFactory; +using ROCKSDB_NAMESPACE::CompactionOptionsFIFO; +using ROCKSDB_NAMESPACE::CompactRangeOptions; +using ROCKSDB_NAMESPACE::Comparator; +using ROCKSDB_NAMESPACE::CompressionType; +using ROCKSDB_NAMESPACE::ConfigOptions; +using ROCKSDB_NAMESPACE::CuckooTableOptions; +using ROCKSDB_NAMESPACE::DB; +using ROCKSDB_NAMESPACE::DBOptions; +using ROCKSDB_NAMESPACE::DbPath; +using ROCKSDB_NAMESPACE::Env; +using ROCKSDB_NAMESPACE::EnvOptions; +using ROCKSDB_NAMESPACE::FileLock; +using ROCKSDB_NAMESPACE::FilterPolicy; +using ROCKSDB_NAMESPACE::FlushOptions; +using ROCKSDB_NAMESPACE::HistogramData; +using ROCKSDB_NAMESPACE::HyperClockCacheOptions; +using ROCKSDB_NAMESPACE::InfoLogLevel; +using ROCKSDB_NAMESPACE::IngestExternalFileOptions; +using ROCKSDB_NAMESPACE::Iterator; +using ROCKSDB_NAMESPACE::LevelMetaData; +using ROCKSDB_NAMESPACE::LiveFileMetaData; +using ROCKSDB_NAMESPACE::Logger; +using ROCKSDB_NAMESPACE::LRUCacheOptions; +using ROCKSDB_NAMESPACE::MemoryAllocator; +using ROCKSDB_NAMESPACE::MemoryUtil; +using ROCKSDB_NAMESPACE::MergeOperator; +using ROCKSDB_NAMESPACE::NewBloomFilterPolicy; +using ROCKSDB_NAMESPACE::NewCompactOnDeletionCollectorFactory; +using ROCKSDB_NAMESPACE::NewGenericRateLimiter; +using ROCKSDB_NAMESPACE::NewLRUCache; +using ROCKSDB_NAMESPACE::NewRibbonFilterPolicy; +using ROCKSDB_NAMESPACE::OptimisticTransactionDB; +using ROCKSDB_NAMESPACE::OptimisticTransactionOptions; +using ROCKSDB_NAMESPACE::Options; +using ROCKSDB_NAMESPACE::PerfContext; +using ROCKSDB_NAMESPACE::PerfLevel; +using ROCKSDB_NAMESPACE::PinnableSlice; +using ROCKSDB_NAMESPACE::PrepopulateBlobCache; +using ROCKSDB_NAMESPACE::RandomAccessFile; +using ROCKSDB_NAMESPACE::Range; +using ROCKSDB_NAMESPACE::RateLimiter; +using ROCKSDB_NAMESPACE::ReadOptions; +using ROCKSDB_NAMESPACE::RestoreOptions; +using ROCKSDB_NAMESPACE::SequentialFile; +using ROCKSDB_NAMESPACE::Slice; +using ROCKSDB_NAMESPACE::SliceParts; +using ROCKSDB_NAMESPACE::SliceTransform; +using ROCKSDB_NAMESPACE::Snapshot; +using ROCKSDB_NAMESPACE::SstFileMetaData; +using ROCKSDB_NAMESPACE::SstFileWriter; +using ROCKSDB_NAMESPACE::Status; +using ROCKSDB_NAMESPACE::TablePropertiesCollectorFactory; +using ROCKSDB_NAMESPACE::Transaction; +using ROCKSDB_NAMESPACE::TransactionDB; +using ROCKSDB_NAMESPACE::TransactionDBOptions; +using ROCKSDB_NAMESPACE::TransactionLogIterator; +using ROCKSDB_NAMESPACE::TransactionOptions; +using ROCKSDB_NAMESPACE::WALRecoveryMode; +using ROCKSDB_NAMESPACE::WritableFile; +using ROCKSDB_NAMESPACE::WriteBatch; +using ROCKSDB_NAMESPACE::WriteBatchWithIndex; +using ROCKSDB_NAMESPACE::WriteOptions; + +using std::unordered_set; +using std::vector; + +extern "C" { + +struct rocksdb_t { + DB* rep; +}; +struct rocksdb_backup_engine_t { + BackupEngine* rep; +}; +struct rocksdb_backup_engine_info_t { + std::vector rep; +}; +struct rocksdb_restore_options_t { + RestoreOptions rep; +}; +struct rocksdb_iterator_t { + Iterator* rep; +}; +struct rocksdb_writebatch_t { + WriteBatch rep; +}; +struct rocksdb_writebatch_wi_t { + WriteBatchWithIndex* rep; +}; +struct rocksdb_snapshot_t { + const Snapshot* rep; +}; +struct rocksdb_flushoptions_t { + FlushOptions rep; +}; +struct rocksdb_fifo_compaction_options_t { + CompactionOptionsFIFO rep; +}; +struct rocksdb_readoptions_t { + ReadOptions rep; + // stack variables to set pointers to in ReadOptions + Slice upper_bound; + Slice lower_bound; + Slice timestamp; + Slice iter_start_ts; +}; +struct rocksdb_writeoptions_t { + WriteOptions rep; +}; +struct rocksdb_options_t { + Options rep; +}; +struct rocksdb_compactoptions_t { + CompactRangeOptions rep; + Slice full_history_ts_low; +}; +struct rocksdb_block_based_table_options_t { + BlockBasedTableOptions rep; +}; +struct rocksdb_cuckoo_table_options_t { + CuckooTableOptions rep; +}; +struct rocksdb_seqfile_t { + SequentialFile* rep; +}; +struct rocksdb_randomfile_t { + RandomAccessFile* rep; +}; +struct rocksdb_writablefile_t { + WritableFile* rep; +}; +struct rocksdb_wal_iterator_t { + TransactionLogIterator* rep; +}; +struct rocksdb_wal_readoptions_t { + TransactionLogIterator::ReadOptions rep; +}; +struct rocksdb_filelock_t { + FileLock* rep; +}; +struct rocksdb_logger_t { + std::shared_ptr rep; +}; +struct rocksdb_lru_cache_options_t { + LRUCacheOptions rep; +}; +struct rocksdb_hyper_clock_cache_options_t { + HyperClockCacheOptions rep; +}; +struct rocksdb_memory_allocator_t { + std::shared_ptr rep; +}; +struct rocksdb_cache_t { + std::shared_ptr rep; +}; +struct rocksdb_livefiles_t { + std::vector rep; +}; +struct rocksdb_column_family_handle_t { + ColumnFamilyHandle* rep; +}; +struct rocksdb_column_family_metadata_t { + ColumnFamilyMetaData rep; +}; +struct rocksdb_level_metadata_t { + const LevelMetaData* rep; +}; +struct rocksdb_sst_file_metadata_t { + const SstFileMetaData* rep; +}; +struct rocksdb_envoptions_t { + EnvOptions rep; +}; +struct rocksdb_ingestexternalfileoptions_t { + IngestExternalFileOptions rep; +}; +struct rocksdb_sstfilewriter_t { + SstFileWriter* rep; +}; +struct rocksdb_ratelimiter_t { + std::shared_ptr rep; +}; +struct rocksdb_perfcontext_t { + PerfContext* rep; +}; +struct rocksdb_pinnableslice_t { + PinnableSlice rep; +}; +struct rocksdb_transactiondb_options_t { + TransactionDBOptions rep; +}; +struct rocksdb_transactiondb_t { + TransactionDB* rep; +}; +struct rocksdb_transaction_options_t { + TransactionOptions rep; +}; +struct rocksdb_transaction_t { + Transaction* rep; +}; +struct rocksdb_backup_engine_options_t { + BackupEngineOptions rep; +}; +struct rocksdb_checkpoint_t { + Checkpoint* rep; +}; +struct rocksdb_optimistictransactiondb_t { + OptimisticTransactionDB* rep; +}; +struct rocksdb_optimistictransaction_options_t { + OptimisticTransactionOptions rep; +}; + +struct rocksdb_compactionfiltercontext_t { + CompactionFilter::Context rep; +}; + +struct rocksdb_statistics_histogram_data_t { + rocksdb_statistics_histogram_data_t() : rep() {} + HistogramData rep; +}; + +struct rocksdb_compactionfilter_t : public CompactionFilter { + void* state_; + void (*destructor_)(void*); + unsigned char (*filter_)(void*, int level, const char* key, size_t key_length, + const char* existing_value, size_t value_length, + char** new_value, size_t* new_value_length, + unsigned char* value_changed); + const char* (*name_)(void*); + unsigned char ignore_snapshots_; + + ~rocksdb_compactionfilter_t() override { (*destructor_)(state_); } + + bool Filter(int level, const Slice& key, const Slice& existing_value, + std::string* new_value, bool* value_changed) const override { + char* c_new_value = nullptr; + size_t new_value_length = 0; + unsigned char c_value_changed = 0; + unsigned char result = + (*filter_)(state_, level, key.data(), key.size(), existing_value.data(), + existing_value.size(), &c_new_value, &new_value_length, + &c_value_changed); + if (c_value_changed) { + new_value->assign(c_new_value, new_value_length); + *value_changed = true; + } + return result; + } + + const char* Name() const override { return (*name_)(state_); } + + bool IgnoreSnapshots() const override { return ignore_snapshots_; } +}; + +struct rocksdb_compactionfilterfactory_t : public CompactionFilterFactory { + void* state_; + void (*destructor_)(void*); + rocksdb_compactionfilter_t* (*create_compaction_filter_)( + void*, rocksdb_compactionfiltercontext_t* context); + const char* (*name_)(void*); + + ~rocksdb_compactionfilterfactory_t() override { (*destructor_)(state_); } + + std::unique_ptr CreateCompactionFilter( + const CompactionFilter::Context& context) override { + rocksdb_compactionfiltercontext_t ccontext; + ccontext.rep = context; + CompactionFilter* cf = (*create_compaction_filter_)(state_, &ccontext); + return std::unique_ptr(cf); + } + + const char* Name() const override { return (*name_)(state_); } +}; + +struct rocksdb_comparator_t : public Comparator { + void* state_; + void (*destructor_)(void*); + int (*compare_)(void*, const char* a, size_t alen, const char* b, + size_t blen); + const char* (*name_)(void*); + int (*compare_ts_)(void*, const char* a_ts, size_t a_tslen, const char* b_ts, + size_t b_tslen); + int (*compare_without_ts_)(void*, const char* a, size_t alen, + unsigned char a_has_ts, const char* b, size_t blen, + unsigned char b_has_ts); + + rocksdb_comparator_t() : Comparator() {} + + rocksdb_comparator_t(size_t ts_size) : Comparator(ts_size) {} + + ~rocksdb_comparator_t() override { (*destructor_)(state_); } + + int Compare(const Slice& a, const Slice& b) const override { + return (*compare_)(state_, a.data(), a.size(), b.data(), b.size()); + } + + int CompareTimestamp(const Slice& a_ts, const Slice& b_ts) const override { + if (compare_ts_ == nullptr) { + return 0; + } + return (*compare_ts_)(state_, a_ts.data(), a_ts.size(), b_ts.data(), + b_ts.size()); + } + + int CompareWithoutTimestamp(const Slice& a, bool a_has_ts, const Slice& b, + bool b_has_ts) const override { + if (compare_without_ts_ == nullptr) { + return Compare(a, b); + } + return (*compare_without_ts_)(state_, a.data(), a.size(), a_has_ts, + b.data(), b.size(), b_has_ts); + } + + const char* Name() const override { return (*name_)(state_); } + + // No-ops since the C binding does not support key shortening methods. + void FindShortestSeparator(std::string*, const Slice&) const override {} + void FindShortSuccessor(std::string* /*key*/) const override {} +}; + +struct rocksdb_filterpolicy_t : public FilterPolicy { + void* state_; + void (*destructor_)(void*); + const char* (*name_)(void*); + + ~rocksdb_filterpolicy_t() override { (*destructor_)(state_); } + + const char* Name() const override { return (*name_)(state_); } +}; + +struct rocksdb_mergeoperator_t : public MergeOperator { + void* state_; + void (*destructor_)(void*); + const char* (*name_)(void*); + char* (*full_merge_)(void*, const char* key, size_t key_length, + const char* existing_value, size_t existing_value_length, + const char* const* operands_list, + const size_t* operands_list_length, int num_operands, + unsigned char* success, size_t* new_value_length); + char* (*partial_merge_)(void*, const char* key, size_t key_length, + const char* const* operands_list, + const size_t* operands_list_length, int num_operands, + unsigned char* success, size_t* new_value_length); + void (*delete_value_)(void*, const char* value, size_t value_length); + + ~rocksdb_mergeoperator_t() override { (*destructor_)(state_); } + + const char* Name() const override { return (*name_)(state_); } + + bool FullMergeV2(const MergeOperationInput& merge_in, + MergeOperationOutput* merge_out) const override { + size_t n = merge_in.operand_list.size(); + std::vector operand_pointers(n); + std::vector operand_sizes(n); + for (size_t i = 0; i < n; i++) { + Slice operand(merge_in.operand_list[i]); + operand_pointers[i] = operand.data(); + operand_sizes[i] = operand.size(); + } + + const char* existing_value_data = nullptr; + size_t existing_value_len = 0; + if (merge_in.existing_value != nullptr) { + existing_value_data = merge_in.existing_value->data(); + existing_value_len = merge_in.existing_value->size(); + } + + unsigned char success; + size_t new_value_len; + char* tmp_new_value = (*full_merge_)( + state_, merge_in.key.data(), merge_in.key.size(), existing_value_data, + existing_value_len, &operand_pointers[0], &operand_sizes[0], + static_cast(n), &success, &new_value_len); + merge_out->new_value.assign(tmp_new_value, new_value_len); + + if (delete_value_ != nullptr) { + (*delete_value_)(state_, tmp_new_value, new_value_len); + } else { + free(tmp_new_value); + } + + return success; + } + + bool PartialMergeMulti(const Slice& key, + const std::deque& operand_list, + std::string* new_value, + Logger* /*logger*/) const override { + size_t operand_count = operand_list.size(); + std::vector operand_pointers(operand_count); + std::vector operand_sizes(operand_count); + for (size_t i = 0; i < operand_count; ++i) { + Slice operand(operand_list[i]); + operand_pointers[i] = operand.data(); + operand_sizes[i] = operand.size(); + } + + unsigned char success; + size_t new_value_len; + char* tmp_new_value = (*partial_merge_)( + state_, key.data(), key.size(), &operand_pointers[0], &operand_sizes[0], + static_cast(operand_count), &success, &new_value_len); + new_value->assign(tmp_new_value, new_value_len); + + if (delete_value_ != nullptr) { + (*delete_value_)(state_, tmp_new_value, new_value_len); + } else { + free(tmp_new_value); + } + + return success; + } +}; + +struct rocksdb_dbpath_t { + DbPath rep; +}; + +struct rocksdb_env_t { + Env* rep; + bool is_default; +}; + +struct rocksdb_slicetransform_t : public SliceTransform { + void* state_; + void (*destructor_)(void*); + const char* (*name_)(void*); + char* (*transform_)(void*, const char* key, size_t length, + size_t* dst_length); + unsigned char (*in_domain_)(void*, const char* key, size_t length); + unsigned char (*in_range_)(void*, const char* key, size_t length); + + ~rocksdb_slicetransform_t() override { (*destructor_)(state_); } + + const char* Name() const override { return (*name_)(state_); } + + Slice Transform(const Slice& src) const override { + size_t len; + char* dst = (*transform_)(state_, src.data(), src.size(), &len); + return Slice(dst, len); + } + + bool InDomain(const Slice& src) const override { + return (*in_domain_)(state_, src.data(), src.size()); + } + + bool InRange(const Slice& src) const override { + return (*in_range_)(state_, src.data(), src.size()); + } +}; + +struct rocksdb_universal_compaction_options_t { + ROCKSDB_NAMESPACE::CompactionOptionsUniversal* rep; +}; + +static bool SaveError(char** errptr, const Status& s) { + assert(errptr != nullptr); + if (s.ok()) { + return false; + } else if (*errptr == nullptr) { + *errptr = strdup(s.ToString().c_str()); + } else { + // TODO(sanjay): Merge with existing error? + // This is a bug if *errptr is not created by malloc() + free(*errptr); + *errptr = strdup(s.ToString().c_str()); + } + return true; +} + +static char* CopyString(const std::string& str) { + char* result = reinterpret_cast(malloc(sizeof(char) * str.size())); + memcpy(result, str.data(), sizeof(char) * str.size()); + return result; +} + +rocksdb_t* rocksdb_open(const rocksdb_options_t* options, const char* name, + char** errptr) { + DB* db; + if (SaveError(errptr, DB::Open(options->rep, std::string(name), &db))) { + return nullptr; + } + rocksdb_t* result = new rocksdb_t; + result->rep = db; + return result; +} + +rocksdb_t* rocksdb_open_with_ttl(const rocksdb_options_t* options, + const char* name, int ttl, char** errptr) { + ROCKSDB_NAMESPACE::DBWithTTL* db; + if (SaveError(errptr, ROCKSDB_NAMESPACE::DBWithTTL::Open( + options->rep, std::string(name), &db, ttl))) { + return nullptr; + } + rocksdb_t* result = new rocksdb_t; + result->rep = db; + return result; +} + +rocksdb_t* rocksdb_open_for_read_only(const rocksdb_options_t* options, + const char* name, + unsigned char error_if_wal_file_exists, + char** errptr) { + DB* db; + if (SaveError(errptr, DB::OpenForReadOnly(options->rep, std::string(name), + &db, error_if_wal_file_exists))) { + return nullptr; + } + rocksdb_t* result = new rocksdb_t; + result->rep = db; + return result; +} + +rocksdb_t* rocksdb_open_as_secondary(const rocksdb_options_t* options, + const char* name, + const char* secondary_path, + char** errptr) { + DB* db; + if (SaveError(errptr, + DB::OpenAsSecondary(options->rep, std::string(name), + std::string(secondary_path), &db))) { + return nullptr; + } + rocksdb_t* result = new rocksdb_t; + result->rep = db; + return result; +} + +rocksdb_backup_engine_t* rocksdb_backup_engine_open( + const rocksdb_options_t* options, const char* path, char** errptr) { + BackupEngine* be; + if (SaveError(errptr, BackupEngine::Open( + options->rep.env, + BackupEngineOptions(path, nullptr, true, + options->rep.info_log.get()), + &be))) { + return nullptr; + } + rocksdb_backup_engine_t* result = new rocksdb_backup_engine_t; + result->rep = be; + return result; +} + +rocksdb_backup_engine_t* rocksdb_backup_engine_open_opts( + const rocksdb_backup_engine_options_t* options, rocksdb_env_t* env, + char** errptr) { + BackupEngine* be; + if (SaveError(errptr, BackupEngine::Open(options->rep, env->rep, &be))) { + return nullptr; + } + rocksdb_backup_engine_t* result = new rocksdb_backup_engine_t; + result->rep = be; + return result; +} + +void rocksdb_backup_engine_create_new_backup(rocksdb_backup_engine_t* be, + rocksdb_t* db, char** errptr) { + SaveError(errptr, be->rep->CreateNewBackup(db->rep)); +} + +void rocksdb_backup_engine_create_new_backup_flush( + rocksdb_backup_engine_t* be, rocksdb_t* db, + unsigned char flush_before_backup, char** errptr) { + SaveError(errptr, be->rep->CreateNewBackup(db->rep, flush_before_backup)); +} + +void rocksdb_backup_engine_purge_old_backups(rocksdb_backup_engine_t* be, + uint32_t num_backups_to_keep, + char** errptr) { + SaveError(errptr, be->rep->PurgeOldBackups(num_backups_to_keep)); +} + +rocksdb_restore_options_t* rocksdb_restore_options_create() { + return new rocksdb_restore_options_t; +} + +void rocksdb_restore_options_destroy(rocksdb_restore_options_t* opt) { + delete opt; +} + +void rocksdb_restore_options_set_keep_log_files(rocksdb_restore_options_t* opt, + int v) { + opt->rep.keep_log_files = v; +} + +void rocksdb_backup_engine_verify_backup(rocksdb_backup_engine_t* be, + uint32_t backup_id, char** errptr) { + SaveError(errptr, be->rep->VerifyBackup(static_cast(backup_id))); +} + +void rocksdb_backup_engine_restore_db_from_latest_backup( + rocksdb_backup_engine_t* be, const char* db_dir, const char* wal_dir, + const rocksdb_restore_options_t* restore_options, char** errptr) { + SaveError(errptr, be->rep->RestoreDBFromLatestBackup(std::string(db_dir), + std::string(wal_dir), + restore_options->rep)); +} + +void rocksdb_backup_engine_restore_db_from_backup( + rocksdb_backup_engine_t* be, const char* db_dir, const char* wal_dir, + const rocksdb_restore_options_t* restore_options, const uint32_t backup_id, + char** errptr) { + SaveError(errptr, be->rep->RestoreDBFromBackup(backup_id, std::string(db_dir), + std::string(wal_dir), + restore_options->rep)); +} + +const rocksdb_backup_engine_info_t* rocksdb_backup_engine_get_backup_info( + rocksdb_backup_engine_t* be) { + rocksdb_backup_engine_info_t* result = new rocksdb_backup_engine_info_t; + be->rep->GetBackupInfo(&result->rep); + return result; +} + +int rocksdb_backup_engine_info_count(const rocksdb_backup_engine_info_t* info) { + return static_cast(info->rep.size()); +} + +int64_t rocksdb_backup_engine_info_timestamp( + const rocksdb_backup_engine_info_t* info, int index) { + return info->rep[index].timestamp; +} + +uint32_t rocksdb_backup_engine_info_backup_id( + const rocksdb_backup_engine_info_t* info, int index) { + return info->rep[index].backup_id; +} + +uint64_t rocksdb_backup_engine_info_size( + const rocksdb_backup_engine_info_t* info, int index) { + return info->rep[index].size; +} + +uint32_t rocksdb_backup_engine_info_number_files( + const rocksdb_backup_engine_info_t* info, int index) { + return info->rep[index].number_files; +} + +void rocksdb_backup_engine_info_destroy( + const rocksdb_backup_engine_info_t* info) { + delete info; +} + +void rocksdb_backup_engine_close(rocksdb_backup_engine_t* be) { + delete be->rep; + delete be; +} + +rocksdb_backup_engine_options_t* rocksdb_backup_engine_options_create( + const char* backup_dir) { + return new rocksdb_backup_engine_options_t{ + BackupEngineOptions(std::string(backup_dir))}; +} + +void rocksdb_backup_engine_options_set_backup_dir( + rocksdb_backup_engine_options_t* options, const char* backup_dir) { + options->rep.backup_dir = std::string(backup_dir); +} + +void rocksdb_backup_engine_options_set_env( + rocksdb_backup_engine_options_t* options, rocksdb_env_t* env) { + options->rep.backup_env = (env ? env->rep : nullptr); +} + +void rocksdb_backup_engine_options_set_share_table_files( + rocksdb_backup_engine_options_t* options, unsigned char val) { + options->rep.share_table_files = val; +} + +unsigned char rocksdb_backup_engine_options_get_share_table_files( + rocksdb_backup_engine_options_t* options) { + return options->rep.share_table_files; +} + +void rocksdb_backup_engine_options_set_sync( + rocksdb_backup_engine_options_t* options, unsigned char val) { + options->rep.sync = val; +} + +unsigned char rocksdb_backup_engine_options_get_sync( + rocksdb_backup_engine_options_t* options) { + return options->rep.sync; +} + +void rocksdb_backup_engine_options_set_destroy_old_data( + rocksdb_backup_engine_options_t* options, unsigned char val) { + options->rep.destroy_old_data = val; +} + +unsigned char rocksdb_backup_engine_options_get_destroy_old_data( + rocksdb_backup_engine_options_t* options) { + return options->rep.destroy_old_data; +} + +void rocksdb_backup_engine_options_set_backup_log_files( + rocksdb_backup_engine_options_t* options, unsigned char val) { + options->rep.backup_log_files = val; +} + +unsigned char rocksdb_backup_engine_options_get_backup_log_files( + rocksdb_backup_engine_options_t* options) { + return options->rep.backup_log_files; +} + +void rocksdb_backup_engine_options_set_backup_rate_limit( + rocksdb_backup_engine_options_t* options, uint64_t limit) { + options->rep.backup_rate_limit = limit; +} + +uint64_t rocksdb_backup_engine_options_get_backup_rate_limit( + rocksdb_backup_engine_options_t* options) { + return options->rep.backup_rate_limit; +} + +void rocksdb_backup_engine_options_set_restore_rate_limit( + rocksdb_backup_engine_options_t* options, uint64_t limit) { + options->rep.restore_rate_limit = limit; +} + +uint64_t rocksdb_backup_engine_options_get_restore_rate_limit( + rocksdb_backup_engine_options_t* options) { + return options->rep.restore_rate_limit; +} + +void rocksdb_backup_engine_options_set_max_background_operations( + rocksdb_backup_engine_options_t* options, int val) { + options->rep.max_background_operations = val; +} + +int rocksdb_backup_engine_options_get_max_background_operations( + rocksdb_backup_engine_options_t* options) { + return options->rep.max_background_operations; +} + +void rocksdb_backup_engine_options_set_callback_trigger_interval_size( + rocksdb_backup_engine_options_t* options, uint64_t size) { + options->rep.callback_trigger_interval_size = size; +} + +uint64_t rocksdb_backup_engine_options_get_callback_trigger_interval_size( + rocksdb_backup_engine_options_t* options) { + return options->rep.callback_trigger_interval_size; +} + +void rocksdb_backup_engine_options_set_max_valid_backups_to_open( + rocksdb_backup_engine_options_t* options, int val) { + options->rep.max_valid_backups_to_open = val; +} + +int rocksdb_backup_engine_options_get_max_valid_backups_to_open( + rocksdb_backup_engine_options_t* options) { + return options->rep.max_valid_backups_to_open; +} + +void rocksdb_backup_engine_options_set_share_files_with_checksum_naming( + rocksdb_backup_engine_options_t* options, int val) { + options->rep.share_files_with_checksum_naming = + static_cast(val); +} + +int rocksdb_backup_engine_options_get_share_files_with_checksum_naming( + rocksdb_backup_engine_options_t* options) { + return static_cast(options->rep.share_files_with_checksum_naming); +} + +void rocksdb_backup_engine_options_destroy( + rocksdb_backup_engine_options_t* options) { + delete options; +} + +rocksdb_checkpoint_t* rocksdb_checkpoint_object_create(rocksdb_t* db, + char** errptr) { + Checkpoint* checkpoint; + if (SaveError(errptr, Checkpoint::Create(db->rep, &checkpoint))) { + return nullptr; + } + rocksdb_checkpoint_t* result = new rocksdb_checkpoint_t; + result->rep = checkpoint; + return result; +} + +void rocksdb_checkpoint_create(rocksdb_checkpoint_t* checkpoint, + const char* checkpoint_dir, + uint64_t log_size_for_flush, char** errptr) { + SaveError(errptr, checkpoint->rep->CreateCheckpoint( + std::string(checkpoint_dir), log_size_for_flush)); +} + +void rocksdb_checkpoint_object_destroy(rocksdb_checkpoint_t* checkpoint) { + delete checkpoint->rep; + delete checkpoint; +} + +void rocksdb_close(rocksdb_t* db) { + delete db->rep; + delete db; +} + +void rocksdb_options_set_uint64add_merge_operator(rocksdb_options_t* opt) { + opt->rep.merge_operator = + ROCKSDB_NAMESPACE::MergeOperators::CreateUInt64AddOperator(); +} + +rocksdb_t* rocksdb_open_and_trim_history( + const rocksdb_options_t* db_options, const char* name, + int num_column_families, const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char* trim_ts, + size_t trim_tslen, char** errptr) { + std::vector column_families; + for (int i = 0; i < num_column_families; i++) { + column_families.push_back(ColumnFamilyDescriptor( + std::string(column_family_names[i]), + ColumnFamilyOptions(column_family_options[i]->rep))); + } + + std::string trim_ts_(trim_ts, trim_tslen); + + DB* db; + std::vector handles; + if (SaveError(errptr, DB::OpenAndTrimHistory( + DBOptions(db_options->rep), std::string(name), + column_families, &handles, &db, trim_ts_))) { + return nullptr; + } + + for (size_t i = 0; i < handles.size(); i++) { + rocksdb_column_family_handle_t* c_handle = + new rocksdb_column_family_handle_t; + c_handle->rep = handles[i]; + column_family_handles[i] = c_handle; + } + rocksdb_t* result = new rocksdb_t; + result->rep = db; + return result; +} + +rocksdb_t* rocksdb_open_column_families( + const rocksdb_options_t* db_options, const char* name, + int num_column_families, const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char** errptr) { + std::vector column_families; + for (int i = 0; i < num_column_families; i++) { + column_families.push_back(ColumnFamilyDescriptor( + std::string(column_family_names[i]), + ColumnFamilyOptions(column_family_options[i]->rep))); + } + + DB* db; + std::vector handles; + if (SaveError(errptr, DB::Open(DBOptions(db_options->rep), std::string(name), + column_families, &handles, &db))) { + return nullptr; + } + + for (size_t i = 0; i < handles.size(); i++) { + rocksdb_column_family_handle_t* c_handle = + new rocksdb_column_family_handle_t; + c_handle->rep = handles[i]; + column_family_handles[i] = c_handle; + } + rocksdb_t* result = new rocksdb_t; + result->rep = db; + return result; +} + +rocksdb_t* rocksdb_open_column_families_with_ttl( + const rocksdb_options_t* db_options, const char* name, + int num_column_families, const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, const int* ttls, + char** errptr) { + std::vector ttls_vec; + std::vector column_families; + for (int i = 0; i < num_column_families; i++) { + ttls_vec.push_back(ttls[i]); + + column_families.push_back(ColumnFamilyDescriptor( + std::string(column_family_names[i]), + ColumnFamilyOptions(column_family_options[i]->rep))); + } + + ROCKSDB_NAMESPACE::DBWithTTL* db; + std::vector handles; + if (SaveError(errptr, ROCKSDB_NAMESPACE::DBWithTTL::Open( + DBOptions(db_options->rep), std::string(name), + column_families, &handles, &db, ttls_vec))) { + return nullptr; + } + + for (size_t i = 0; i < handles.size(); i++) { + rocksdb_column_family_handle_t* c_handle = + new rocksdb_column_family_handle_t; + c_handle->rep = handles[i]; + column_family_handles[i] = c_handle; + } + rocksdb_t* result = new rocksdb_t; + result->rep = db; + return result; +} + +rocksdb_t* rocksdb_open_for_read_only_column_families( + const rocksdb_options_t* db_options, const char* name, + int num_column_families, const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, + unsigned char error_if_wal_file_exists, char** errptr) { + std::vector column_families; + for (int i = 0; i < num_column_families; i++) { + column_families.push_back(ColumnFamilyDescriptor( + std::string(column_family_names[i]), + ColumnFamilyOptions(column_family_options[i]->rep))); + } + + DB* db; + std::vector handles; + if (SaveError(errptr, + DB::OpenForReadOnly(DBOptions(db_options->rep), + std::string(name), column_families, + &handles, &db, error_if_wal_file_exists))) { + return nullptr; + } + + for (size_t i = 0; i < handles.size(); i++) { + rocksdb_column_family_handle_t* c_handle = + new rocksdb_column_family_handle_t; + c_handle->rep = handles[i]; + column_family_handles[i] = c_handle; + } + rocksdb_t* result = new rocksdb_t; + result->rep = db; + return result; +} + +rocksdb_t* rocksdb_open_as_secondary_column_families( + const rocksdb_options_t* db_options, const char* name, + const char* secondary_path, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char** errptr) { + std::vector column_families; + for (int i = 0; i != num_column_families; ++i) { + column_families.emplace_back( + std::string(column_family_names[i]), + ColumnFamilyOptions(column_family_options[i]->rep)); + } + DB* db; + std::vector handles; + if (SaveError(errptr, DB::OpenAsSecondary(DBOptions(db_options->rep), + std::string(name), + std::string(secondary_path), + column_families, &handles, &db))) { + return nullptr; + } + for (size_t i = 0; i != handles.size(); ++i) { + rocksdb_column_family_handle_t* c_handle = + new rocksdb_column_family_handle_t; + c_handle->rep = handles[i]; + column_family_handles[i] = c_handle; + } + rocksdb_t* result = new rocksdb_t; + result->rep = db; + return result; +} + +char** rocksdb_list_column_families(const rocksdb_options_t* options, + const char* name, size_t* lencfs, + char** errptr) { + std::vector fams; + SaveError(errptr, DB::ListColumnFamilies(DBOptions(options->rep), + std::string(name), &fams)); + + *lencfs = fams.size(); + char** column_families = + static_cast(malloc(sizeof(char*) * fams.size())); + for (size_t i = 0; i < fams.size(); i++) { + column_families[i] = strdup(fams[i].c_str()); + } + return column_families; +} + +void rocksdb_list_column_families_destroy(char** list, size_t len) { + for (size_t i = 0; i < len; ++i) { + free(list[i]); + } + free(list); +} + +rocksdb_column_family_handle_t* rocksdb_create_column_family( + rocksdb_t* db, const rocksdb_options_t* column_family_options, + const char* column_family_name, char** errptr) { + rocksdb_column_family_handle_t* handle = new rocksdb_column_family_handle_t; + SaveError(errptr, db->rep->CreateColumnFamily( + ColumnFamilyOptions(column_family_options->rep), + std::string(column_family_name), &(handle->rep))); + return handle; +} + +rocksdb_column_family_handle_t** rocksdb_create_column_families( + rocksdb_t* db, const rocksdb_options_t* column_family_options, + int num_column_families, const char* const* column_family_names, + size_t* lencfs, char** errptr) { + std::vector handles; + std::vector names; + for (int i = 0; i != num_column_families; ++i) { + names.push_back(std::string(column_family_names[i])); + } + SaveError(errptr, db->rep->CreateColumnFamilies( + ColumnFamilyOptions(column_family_options->rep), names, + &handles)); + + *lencfs = handles.size(); + rocksdb_column_family_handle_t** c_handles = + static_cast( + malloc(sizeof(rocksdb_column_family_handle_t*) * handles.size())); + for (size_t i = 0; i != handles.size(); ++i) { + c_handles[i] = new rocksdb_column_family_handle_t; + c_handles[i]->rep = handles[i]; + } + + return c_handles; +} + +void rocksdb_create_column_families_destroy( + rocksdb_column_family_handle_t** list) { + free(list); +} + +rocksdb_column_family_handle_t* rocksdb_create_column_family_with_ttl( + rocksdb_t* db, const rocksdb_options_t* column_family_options, + const char* column_family_name, int ttl, char** errptr) { + ROCKSDB_NAMESPACE::DBWithTTL* db_with_ttl = + static_cast(db->rep); + rocksdb_column_family_handle_t* handle = new rocksdb_column_family_handle_t; + SaveError(errptr, db_with_ttl->CreateColumnFamilyWithTtl( + ColumnFamilyOptions(column_family_options->rep), + std::string(column_family_name), &(handle->rep), ttl)); + return handle; +} + +void rocksdb_drop_column_family(rocksdb_t* db, + rocksdb_column_family_handle_t* handle, + char** errptr) { + SaveError(errptr, db->rep->DropColumnFamily(handle->rep)); +} + +uint32_t rocksdb_column_family_handle_get_id( + rocksdb_column_family_handle_t* handle) { + return handle->rep->GetID(); +} + +char* rocksdb_column_family_handle_get_name( + rocksdb_column_family_handle_t* handle, size_t* name_len) { + auto name = handle->rep->GetName(); + *name_len = name.size(); + return CopyString(name); +} + +void rocksdb_column_family_handle_destroy( + rocksdb_column_family_handle_t* handle) { + delete handle->rep; + delete handle; +} + +void rocksdb_put(rocksdb_t* db, const rocksdb_writeoptions_t* options, + const char* key, size_t keylen, const char* val, size_t vallen, + char** errptr) { + SaveError(errptr, + db->rep->Put(options->rep, Slice(key, keylen), Slice(val, vallen))); +} + +void rocksdb_put_cf(rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, + const char* key, size_t keylen, const char* val, + size_t vallen, char** errptr) { + SaveError(errptr, db->rep->Put(options->rep, column_family->rep, + Slice(key, keylen), Slice(val, vallen))); +} + +void rocksdb_put_with_ts(rocksdb_t* db, const rocksdb_writeoptions_t* options, + const char* key, size_t keylen, const char* ts, + size_t tslen, const char* val, size_t vallen, + char** errptr) { + SaveError(errptr, db->rep->Put(options->rep, Slice(key, keylen), + Slice(ts, tslen), Slice(val, vallen))); +} + +void rocksdb_put_cf_with_ts(rocksdb_t* db, + const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, + const char* key, size_t keylen, const char* ts, + size_t tslen, const char* val, size_t vallen, + char** errptr) { + SaveError(errptr, + db->rep->Put(options->rep, column_family->rep, Slice(key, keylen), + Slice(ts, tslen), Slice(val, vallen))); +} + +void rocksdb_delete(rocksdb_t* db, const rocksdb_writeoptions_t* options, + const char* key, size_t keylen, char** errptr) { + SaveError(errptr, db->rep->Delete(options->rep, Slice(key, keylen))); +} + +void rocksdb_delete_cf(rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, + const char* key, size_t keylen, char** errptr) { + SaveError(errptr, db->rep->Delete(options->rep, column_family->rep, + Slice(key, keylen))); +} + +void rocksdb_delete_with_ts(rocksdb_t* db, + const rocksdb_writeoptions_t* options, + const char* key, size_t keylen, const char* ts, + size_t tslen, char** errptr) { + SaveError(errptr, db->rep->Delete(options->rep, Slice(key, keylen), + Slice(ts, tslen))); +} + +void rocksdb_delete_cf_with_ts(rocksdb_t* db, + const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, + const char* key, size_t keylen, const char* ts, + size_t tslen, char** errptr) { + SaveError(errptr, db->rep->Delete(options->rep, column_family->rep, + Slice(key, keylen), Slice(ts, tslen))); +} + +void rocksdb_singledelete(rocksdb_t* db, const rocksdb_writeoptions_t* options, + const char* key, size_t keylen, char** errptr) { + SaveError(errptr, db->rep->SingleDelete(options->rep, Slice(key, keylen))); +} + +void rocksdb_singledelete_cf(rocksdb_t* db, + const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, + const char* key, size_t keylen, char** errptr) { + SaveError(errptr, db->rep->SingleDelete(options->rep, column_family->rep, + Slice(key, keylen))); +} + +void rocksdb_singledelete_with_ts(rocksdb_t* db, + const rocksdb_writeoptions_t* options, + const char* key, size_t keylen, + const char* ts, size_t tslen, char** errptr) { + SaveError(errptr, db->rep->SingleDelete(options->rep, Slice(key, keylen), + Slice(ts, tslen))); +} + +void rocksdb_singledelete_cf_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* ts, size_t tslen, char** errptr) { + SaveError(errptr, + db->rep->SingleDelete(options->rep, column_family->rep, + Slice(key, keylen), Slice(ts, tslen))); +} + +void rocksdb_increase_full_history_ts_low( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* ts_low, size_t ts_lowlen, char** errptr) { + std::string ts(ts_low, ts_lowlen); + SaveError(errptr, db->rep->IncreaseFullHistoryTsLow(column_family->rep, ts)); +} + +char* rocksdb_get_full_history_ts_low( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + size_t* ts_len, char** errptr) { + char* result = nullptr; + std::string tmp; + Status s = db->rep->GetFullHistoryTsLow(column_family->rep, &tmp); + if (s.ok()) { + *ts_len = tmp.size(); + result = CopyString(tmp); + } else { + *ts_len = 0; + SaveError(errptr, s); + } + return result; +} + +void rocksdb_delete_range_cf(rocksdb_t* db, + const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, + const char* end_key, size_t end_key_len, + char** errptr) { + SaveError(errptr, db->rep->DeleteRange(options->rep, column_family->rep, + Slice(start_key, start_key_len), + Slice(end_key, end_key_len))); +} + +void rocksdb_merge(rocksdb_t* db, const rocksdb_writeoptions_t* options, + const char* key, size_t keylen, const char* val, + size_t vallen, char** errptr) { + SaveError(errptr, db->rep->Merge(options->rep, Slice(key, keylen), + Slice(val, vallen))); +} + +void rocksdb_merge_cf(rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, + const char* key, size_t keylen, const char* val, + size_t vallen, char** errptr) { + SaveError(errptr, db->rep->Merge(options->rep, column_family->rep, + Slice(key, keylen), Slice(val, vallen))); +} + +void rocksdb_write(rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_writebatch_t* batch, char** errptr) { + SaveError(errptr, db->rep->Write(options->rep, &batch->rep)); +} + +char* rocksdb_get(rocksdb_t* db, const rocksdb_readoptions_t* options, + const char* key, size_t keylen, size_t* vallen, + char** errptr) { + char* result = nullptr; + std::string tmp; + Status s = db->rep->Get(options->rep, Slice(key, keylen), &tmp); + if (s.ok()) { + *vallen = tmp.size(); + result = CopyString(tmp); + } else { + *vallen = 0; + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + } + return result; +} + +char* rocksdb_get_cf(rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, + const char* key, size_t keylen, size_t* vallen, + char** errptr) { + char* result = nullptr; + std::string tmp; + Status s = + db->rep->Get(options->rep, column_family->rep, Slice(key, keylen), &tmp); + if (s.ok()) { + *vallen = tmp.size(); + result = CopyString(tmp); + } else { + *vallen = 0; + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + } + return result; +} + +char* rocksdb_get_with_ts(rocksdb_t* db, const rocksdb_readoptions_t* options, + const char* key, size_t keylen, size_t* vallen, + char** ts, size_t* tslen, char** errptr) { + char* result = nullptr; + std::string tmp_val; + std::string tmp_ts; + Status s = db->rep->Get(options->rep, Slice(key, keylen), &tmp_val, &tmp_ts); + if (s.ok()) { + *vallen = tmp_val.size(); + result = CopyString(tmp_val); + *tslen = tmp_ts.size(); + *ts = CopyString(tmp_ts); + } else { + *vallen = 0; + *tslen = 0; + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + } + return result; +} + +char* rocksdb_get_cf_with_ts(rocksdb_t* db, + const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, + const char* key, size_t keylen, size_t* vallen, + char** ts, size_t* tslen, char** errptr) { + char* result = nullptr; + std::string tmp; + std::string tmp_ts; + Status s = db->rep->Get(options->rep, column_family->rep, Slice(key, keylen), + &tmp, &tmp_ts); + if (s.ok()) { + *vallen = tmp.size(); + result = CopyString(tmp); + *tslen = tmp_ts.size(); + *ts = CopyString(tmp_ts); + } else { + *vallen = 0; + *tslen = 0; + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + } + return result; +} + +void rocksdb_multi_get(rocksdb_t* db, const rocksdb_readoptions_t* options, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs) { + std::vector keys(num_keys); + for (size_t i = 0; i < num_keys; i++) { + keys[i] = Slice(keys_list[i], keys_list_sizes[i]); + } + std::vector values(num_keys); + std::vector statuses = db->rep->MultiGet(options->rep, keys, &values); + for (size_t i = 0; i < num_keys; i++) { + if (statuses[i].ok()) { + values_list[i] = CopyString(values[i]); + values_list_sizes[i] = values[i].size(); + errs[i] = nullptr; + } else { + values_list[i] = nullptr; + values_list_sizes[i] = 0; + if (!statuses[i].IsNotFound()) { + errs[i] = strdup(statuses[i].ToString().c_str()); + } else { + errs[i] = nullptr; + } + } + } +} + +void rocksdb_multi_get_with_ts(rocksdb_t* db, + const rocksdb_readoptions_t* options, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, + char** values_list, size_t* values_list_sizes, + char** timestamp_list, + size_t* timestamp_list_sizes, char** errs) { + std::vector keys(num_keys); + for (size_t i = 0; i < num_keys; i++) { + keys[i] = Slice(keys_list[i], keys_list_sizes[i]); + } + std::vector values(num_keys); + std::vector timestamps(num_keys); + std::vector statuses = + db->rep->MultiGet(options->rep, keys, &values, ×tamps); + for (size_t i = 0; i < num_keys; i++) { + if (statuses[i].ok()) { + values_list[i] = CopyString(values[i]); + values_list_sizes[i] = values[i].size(); + timestamp_list[i] = CopyString(timestamps[i]); + timestamp_list_sizes[i] = timestamps[i].size(); + errs[i] = nullptr; + } else { + values_list[i] = nullptr; + values_list_sizes[i] = 0; + timestamp_list[i] = nullptr; + timestamp_list_sizes[i] = 0; + if (!statuses[i].IsNotFound()) { + errs[i] = strdup(statuses[i].ToString().c_str()); + } else { + errs[i] = nullptr; + } + } + } +} + +void rocksdb_multi_get_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + const rocksdb_column_family_handle_t* const* column_families, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs) { + std::vector keys(num_keys); + std::vector cfs(num_keys); + for (size_t i = 0; i < num_keys; i++) { + keys[i] = Slice(keys_list[i], keys_list_sizes[i]); + cfs[i] = column_families[i]->rep; + } + std::vector values(num_keys); + std::vector statuses = + db->rep->MultiGet(options->rep, cfs, keys, &values); + for (size_t i = 0; i < num_keys; i++) { + if (statuses[i].ok()) { + values_list[i] = CopyString(values[i]); + values_list_sizes[i] = values[i].size(); + errs[i] = nullptr; + } else { + values_list[i] = nullptr; + values_list_sizes[i] = 0; + if (!statuses[i].IsNotFound()) { + errs[i] = strdup(statuses[i].ToString().c_str()); + } else { + errs[i] = nullptr; + } + } + } +} + +void rocksdb_multi_get_cf_with_ts( + rocksdb_t* db, const rocksdb_readoptions_t* options, + const rocksdb_column_family_handle_t* const* column_families, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** timestamps_list, + size_t* timestamps_list_sizes, char** errs) { + std::vector keys(num_keys); + std::vector cfs(num_keys); + for (size_t i = 0; i < num_keys; i++) { + keys[i] = Slice(keys_list[i], keys_list_sizes[i]); + cfs[i] = column_families[i]->rep; + } + std::vector values(num_keys); + std::vector timestamps(num_keys); + std::vector statuses = + db->rep->MultiGet(options->rep, cfs, keys, &values, ×tamps); + for (size_t i = 0; i < num_keys; i++) { + if (statuses[i].ok()) { + values_list[i] = CopyString(values[i]); + values_list_sizes[i] = values[i].size(); + timestamps_list[i] = CopyString(timestamps[i]); + timestamps_list_sizes[i] = timestamps[i].size(); + errs[i] = nullptr; + } else { + values_list[i] = nullptr; + values_list_sizes[i] = 0; + timestamps_list[i] = nullptr; + timestamps_list_sizes[i] = 0; + if (!statuses[i].IsNotFound()) { + errs[i] = strdup(statuses[i].ToString().c_str()); + } else { + errs[i] = nullptr; + } + } + } +} + +void rocksdb_batched_multi_get_cf(rocksdb_t* db, + const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, + rocksdb_pinnableslice_t** values, char** errs, + const bool sorted_input) { + Slice* key_slices = new Slice[num_keys]; + PinnableSlice* value_slices = new PinnableSlice[num_keys]; + Status* statuses = new Status[num_keys]; + for (size_t i = 0; i < num_keys; ++i) { + key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]); + } + + db->rep->MultiGet(options->rep, column_family->rep, num_keys, key_slices, + value_slices, statuses, sorted_input); + + for (size_t i = 0; i < num_keys; ++i) { + if (statuses[i].ok()) { + values[i] = new (rocksdb_pinnableslice_t); + values[i]->rep = std::move(value_slices[i]); + errs[i] = nullptr; + } else { + values[i] = nullptr; + if (!statuses[i].IsNotFound()) { + errs[i] = strdup(statuses[i].ToString().c_str()); + } else { + errs[i] = nullptr; + } + } + } + + delete[] key_slices; + delete[] value_slices; + delete[] statuses; +} + +unsigned char rocksdb_key_may_exist(rocksdb_t* db, + const rocksdb_readoptions_t* options, + const char* key, size_t key_len, + char** value, size_t* val_len, + const char* timestamp, size_t timestamp_len, + unsigned char* value_found) { + std::string tmp; + std::string time; + if (timestamp) { + time.assign(timestamp, timestamp_len); + } + bool found = false; + const bool result = db->rep->KeyMayExist(options->rep, Slice(key, key_len), + &tmp, timestamp ? &time : nullptr, + value_found ? &found : nullptr); + if (value_found) { + *value_found = found; + if (found) { + *val_len = tmp.size(); + *value = CopyString(tmp); + } + } + return result; +} + +unsigned char rocksdb_key_may_exist_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t key_len, char** value, size_t* val_len, const char* timestamp, + size_t timestamp_len, unsigned char* value_found) { + std::string tmp; + std::string time; + if (timestamp) { + time.assign(timestamp, timestamp_len); + } + bool found = false; + const bool result = db->rep->KeyMayExist( + options->rep, column_family->rep, Slice(key, key_len), &tmp, + timestamp ? &time : nullptr, value_found ? &found : nullptr); + if (value_found) { + *value_found = found; + if (found) { + *val_len = tmp.size(); + *value = CopyString(tmp); + } + } + return result; +} + +rocksdb_iterator_t* rocksdb_create_iterator( + rocksdb_t* db, const rocksdb_readoptions_t* options) { + rocksdb_iterator_t* result = new rocksdb_iterator_t; + result->rep = db->rep->NewIterator(options->rep); + return result; +} + +rocksdb_wal_iterator_t* rocksdb_get_updates_since( + rocksdb_t* db, uint64_t seq_number, + const rocksdb_wal_readoptions_t* options, char** errptr) { + std::unique_ptr iter; + TransactionLogIterator::ReadOptions ro; + if (options != nullptr) { + ro = options->rep; + } + if (SaveError(errptr, db->rep->GetUpdatesSince(seq_number, &iter, ro))) { + return nullptr; + } + rocksdb_wal_iterator_t* result = new rocksdb_wal_iterator_t; + result->rep = iter.release(); + return result; +} + +void rocksdb_wal_iter_next(rocksdb_wal_iterator_t* iter) { iter->rep->Next(); } + +unsigned char rocksdb_wal_iter_valid(const rocksdb_wal_iterator_t* iter) { + return iter->rep->Valid(); +} + +void rocksdb_wal_iter_status(const rocksdb_wal_iterator_t* iter, + char** errptr) { + SaveError(errptr, iter->rep->status()); +} + +void rocksdb_wal_iter_destroy(const rocksdb_wal_iterator_t* iter) { + delete iter->rep; + delete iter; +} + +rocksdb_writebatch_t* rocksdb_wal_iter_get_batch( + const rocksdb_wal_iterator_t* iter, uint64_t* seq) { + rocksdb_writebatch_t* result = rocksdb_writebatch_create(); + BatchResult wal_batch = iter->rep->GetBatch(); + result->rep = std::move(*wal_batch.writeBatchPtr); + if (seq != nullptr) { + *seq = wal_batch.sequence; + } + return result; +} + +uint64_t rocksdb_get_latest_sequence_number(rocksdb_t* db) { + return db->rep->GetLatestSequenceNumber(); +} + +rocksdb_iterator_t* rocksdb_create_iterator_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family) { + rocksdb_iterator_t* result = new rocksdb_iterator_t; + result->rep = db->rep->NewIterator(options->rep, column_family->rep); + return result; +} + +void rocksdb_create_iterators(rocksdb_t* db, rocksdb_readoptions_t* opts, + rocksdb_column_family_handle_t** column_families, + rocksdb_iterator_t** iterators, size_t size, + char** errptr) { + std::vector column_families_vec; + for (size_t i = 0; i < size; i++) { + column_families_vec.push_back(column_families[i]->rep); + } + + std::vector res; + Status status = db->rep->NewIterators(opts->rep, column_families_vec, &res); + assert(res.size() == size); + if (SaveError(errptr, status)) { + return; + } + + for (size_t i = 0; i < size; i++) { + iterators[i] = new rocksdb_iterator_t; + iterators[i]->rep = res[i]; + } +} + +const rocksdb_snapshot_t* rocksdb_create_snapshot(rocksdb_t* db) { + rocksdb_snapshot_t* result = new rocksdb_snapshot_t; + result->rep = db->rep->GetSnapshot(); + return result; +} + +void rocksdb_release_snapshot(rocksdb_t* db, + const rocksdb_snapshot_t* snapshot) { + db->rep->ReleaseSnapshot(snapshot->rep); + delete snapshot; +} + +char* rocksdb_property_value(rocksdb_t* db, const char* propname) { + std::string tmp; + if (db->rep->GetProperty(Slice(propname), &tmp)) { + // We use strdup() since we expect human readable output. + return strdup(tmp.c_str()); + } else { + return nullptr; + } +} + +int rocksdb_property_int(rocksdb_t* db, const char* propname, + uint64_t* out_val) { + if (db->rep->GetIntProperty(Slice(propname), out_val)) { + return 0; + } else { + return -1; + } +} + +int rocksdb_property_int_cf(rocksdb_t* db, + rocksdb_column_family_handle_t* column_family, + const char* propname, uint64_t* out_val) { + if (db->rep->GetIntProperty(column_family->rep, Slice(propname), out_val)) { + return 0; + } else { + return -1; + } +} + +char* rocksdb_property_value_cf(rocksdb_t* db, + rocksdb_column_family_handle_t* column_family, + const char* propname) { + std::string tmp; + if (db->rep->GetProperty(column_family->rep, Slice(propname), &tmp)) { + // We use strdup() since we expect human readable output. + return strdup(tmp.c_str()); + } else { + return nullptr; + } +} + +void rocksdb_approximate_sizes(rocksdb_t* db, int num_ranges, + const char* const* range_start_key, + const size_t* range_start_key_len, + const char* const* range_limit_key, + const size_t* range_limit_key_len, + uint64_t* sizes, char** errptr) { + Range* ranges = new Range[num_ranges]; + for (int i = 0; i < num_ranges; i++) { + ranges[i].start = Slice(range_start_key[i], range_start_key_len[i]); + ranges[i].limit = Slice(range_limit_key[i], range_limit_key_len[i]); + } + Status s = db->rep->GetApproximateSizes(ranges, num_ranges, sizes); + if (!s.ok()) { + SaveError(errptr, s); + } + delete[] ranges; +} + +void rocksdb_approximate_sizes_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + int num_ranges, const char* const* range_start_key, + const size_t* range_start_key_len, const char* const* range_limit_key, + const size_t* range_limit_key_len, uint64_t* sizes, char** errptr) { + Range* ranges = new Range[num_ranges]; + for (int i = 0; i < num_ranges; i++) { + ranges[i].start = Slice(range_start_key[i], range_start_key_len[i]); + ranges[i].limit = Slice(range_limit_key[i], range_limit_key_len[i]); + } + Status s = db->rep->GetApproximateSizes(column_family->rep, ranges, + num_ranges, sizes); + if (!s.ok()) { + SaveError(errptr, s); + } + delete[] ranges; +} + +void rocksdb_delete_file(rocksdb_t* db, const char* name) { + db->rep->DeleteFile(name); +} + +const rocksdb_livefiles_t* rocksdb_livefiles(rocksdb_t* db) { + rocksdb_livefiles_t* result = new rocksdb_livefiles_t; + db->rep->GetLiveFilesMetaData(&result->rep); + return result; +} + +void rocksdb_compact_range(rocksdb_t* db, const char* start_key, + size_t start_key_len, const char* limit_key, + size_t limit_key_len) { + Slice a, b; + db->rep->CompactRange( + CompactRangeOptions(), + // Pass nullptr Slice if corresponding "const char*" is nullptr + (start_key ? (a = Slice(start_key, start_key_len), &a) : nullptr), + (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : nullptr)); +} + +void rocksdb_compact_range_cf(rocksdb_t* db, + rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, + const char* limit_key, size_t limit_key_len) { + Slice a, b; + db->rep->CompactRange( + CompactRangeOptions(), column_family->rep, + // Pass nullptr Slice if corresponding "const char*" is nullptr + (start_key ? (a = Slice(start_key, start_key_len), &a) : nullptr), + (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : nullptr)); +} + +void rocksdb_suggest_compact_range(rocksdb_t* db, const char* start_key, + size_t start_key_len, const char* limit_key, + size_t limit_key_len, char** errptr) { + Slice a, b; + Status s = ROCKSDB_NAMESPACE::experimental::SuggestCompactRange( + db->rep, + (start_key ? (a = Slice(start_key, start_key_len), &a) : nullptr), + (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : nullptr)); + SaveError(errptr, s); +} + +void rocksdb_suggest_compact_range_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* limit_key, + size_t limit_key_len, char** errptr) { + Slice a, b; + Status s = db->rep->SuggestCompactRange( + column_family->rep, + (start_key ? (a = Slice(start_key, start_key_len), &a) : nullptr), + (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : nullptr)); + SaveError(errptr, s); +} + +void rocksdb_compact_range_opt(rocksdb_t* db, rocksdb_compactoptions_t* opt, + const char* start_key, size_t start_key_len, + const char* limit_key, size_t limit_key_len) { + Slice a, b; + db->rep->CompactRange( + opt->rep, + // Pass nullptr Slice if corresponding "const char*" is nullptr + (start_key ? (a = Slice(start_key, start_key_len), &a) : nullptr), + (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : nullptr)); +} + +void rocksdb_compact_range_cf_opt(rocksdb_t* db, + rocksdb_column_family_handle_t* column_family, + rocksdb_compactoptions_t* opt, + const char* start_key, size_t start_key_len, + const char* limit_key, size_t limit_key_len) { + Slice a, b; + db->rep->CompactRange( + opt->rep, column_family->rep, + // Pass nullptr Slice if corresponding "const char*" is nullptr + (start_key ? (a = Slice(start_key, start_key_len), &a) : nullptr), + (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : nullptr)); +} + +void rocksdb_flush(rocksdb_t* db, const rocksdb_flushoptions_t* options, + char** errptr) { + SaveError(errptr, db->rep->Flush(options->rep)); +} + +void rocksdb_flush_cf(rocksdb_t* db, const rocksdb_flushoptions_t* options, + rocksdb_column_family_handle_t* column_family, + char** errptr) { + SaveError(errptr, db->rep->Flush(options->rep, column_family->rep)); +} + +void rocksdb_flush_cfs(rocksdb_t* db, const rocksdb_flushoptions_t* options, + rocksdb_column_family_handle_t** column_families, + int num_column_families, char** errptr) { + std::vector column_family_handles; + for (int i = 0; i < num_column_families; i++) { + column_family_handles.push_back(column_families[i]->rep); + } + + SaveError(errptr, db->rep->Flush(options->rep, column_family_handles)); +} + +void rocksdb_flush_wal(rocksdb_t* db, unsigned char sync, char** errptr) { + SaveError(errptr, db->rep->FlushWAL(sync)); +} + +void rocksdb_disable_file_deletions(rocksdb_t* db, char** errptr) { + SaveError(errptr, db->rep->DisableFileDeletions()); +} + +void rocksdb_enable_file_deletions(rocksdb_t* db, unsigned char force, + char** errptr) { + SaveError(errptr, db->rep->EnableFileDeletions(force)); +} + +void rocksdb_destroy_db(const rocksdb_options_t* options, const char* name, + char** errptr) { + SaveError(errptr, DestroyDB(name, options->rep)); +} + +void rocksdb_repair_db(const rocksdb_options_t* options, const char* name, + char** errptr) { + SaveError(errptr, RepairDB(name, options->rep)); +} + +void rocksdb_iter_destroy(rocksdb_iterator_t* iter) { + delete iter->rep; + delete iter; +} + +unsigned char rocksdb_iter_valid(const rocksdb_iterator_t* iter) { + return iter->rep->Valid(); +} + +void rocksdb_iter_seek_to_first(rocksdb_iterator_t* iter) { + iter->rep->SeekToFirst(); +} + +void rocksdb_iter_seek_to_last(rocksdb_iterator_t* iter) { + iter->rep->SeekToLast(); +} + +void rocksdb_iter_seek(rocksdb_iterator_t* iter, const char* k, size_t klen) { + iter->rep->Seek(Slice(k, klen)); +} + +void rocksdb_iter_seek_for_prev(rocksdb_iterator_t* iter, const char* k, + size_t klen) { + iter->rep->SeekForPrev(Slice(k, klen)); +} + +void rocksdb_iter_next(rocksdb_iterator_t* iter) { iter->rep->Next(); } + +void rocksdb_iter_prev(rocksdb_iterator_t* iter) { iter->rep->Prev(); } + +const char* rocksdb_iter_key(const rocksdb_iterator_t* iter, size_t* klen) { + Slice s = iter->rep->key(); + *klen = s.size(); + return s.data(); +} + +const char* rocksdb_iter_value(const rocksdb_iterator_t* iter, size_t* vlen) { + Slice s = iter->rep->value(); + *vlen = s.size(); + return s.data(); +} + +const char* rocksdb_iter_timestamp(const rocksdb_iterator_t* iter, + size_t* tslen) { + Slice s = iter->rep->timestamp(); + *tslen = s.size(); + return s.data(); +} + +void rocksdb_iter_get_error(const rocksdb_iterator_t* iter, char** errptr) { + SaveError(errptr, iter->rep->status()); +} + +rocksdb_writebatch_t* rocksdb_writebatch_create() { + return new rocksdb_writebatch_t; +} + +rocksdb_writebatch_t* rocksdb_writebatch_create_from(const char* rep, + size_t size) { + rocksdb_writebatch_t* b = new rocksdb_writebatch_t; + b->rep = WriteBatch(std::string(rep, size)); + return b; +} + +void rocksdb_writebatch_destroy(rocksdb_writebatch_t* b) { delete b; } + +void rocksdb_writebatch_clear(rocksdb_writebatch_t* b) { b->rep.Clear(); } + +int rocksdb_writebatch_count(rocksdb_writebatch_t* b) { return b->rep.Count(); } + +void rocksdb_writebatch_put(rocksdb_writebatch_t* b, const char* key, + size_t klen, const char* val, size_t vlen) { + b->rep.Put(Slice(key, klen), Slice(val, vlen)); +} + +void rocksdb_writebatch_put_cf(rocksdb_writebatch_t* b, + rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, + size_t vlen) { + b->rep.Put(column_family->rep, Slice(key, klen), Slice(val, vlen)); +} + +void rocksdb_writebatch_put_cf_with_ts( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* ts, size_t tslen, const char* val, + size_t vlen) { + b->rep.Put(column_family->rep, Slice(key, klen), Slice(ts, tslen), + Slice(val, vlen)); +} + +void rocksdb_writebatch_putv(rocksdb_writebatch_t* b, int num_keys, + const char* const* keys_list, + const size_t* keys_list_sizes, int num_values, + const char* const* values_list, + const size_t* values_list_sizes) { + std::vector key_slices(num_keys); + for (int i = 0; i < num_keys; i++) { + key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]); + } + std::vector value_slices(num_values); + for (int i = 0; i < num_values; i++) { + value_slices[i] = Slice(values_list[i], values_list_sizes[i]); + } + b->rep.Put(SliceParts(key_slices.data(), num_keys), + SliceParts(value_slices.data(), num_values)); +} + +void rocksdb_writebatch_putv_cf(rocksdb_writebatch_t* b, + rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, int num_values, + const char* const* values_list, + const size_t* values_list_sizes) { + std::vector key_slices(num_keys); + for (int i = 0; i < num_keys; i++) { + key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]); + } + std::vector value_slices(num_values); + for (int i = 0; i < num_values; i++) { + value_slices[i] = Slice(values_list[i], values_list_sizes[i]); + } + b->rep.Put(column_family->rep, SliceParts(key_slices.data(), num_keys), + SliceParts(value_slices.data(), num_values)); +} + +void rocksdb_writebatch_merge(rocksdb_writebatch_t* b, const char* key, + size_t klen, const char* val, size_t vlen) { + b->rep.Merge(Slice(key, klen), Slice(val, vlen)); +} + +void rocksdb_writebatch_merge_cf(rocksdb_writebatch_t* b, + rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, + size_t vlen) { + b->rep.Merge(column_family->rep, Slice(key, klen), Slice(val, vlen)); +} + +void rocksdb_writebatch_mergev(rocksdb_writebatch_t* b, int num_keys, + const char* const* keys_list, + const size_t* keys_list_sizes, int num_values, + const char* const* values_list, + const size_t* values_list_sizes) { + std::vector key_slices(num_keys); + for (int i = 0; i < num_keys; i++) { + key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]); + } + std::vector value_slices(num_values); + for (int i = 0; i < num_values; i++) { + value_slices[i] = Slice(values_list[i], values_list_sizes[i]); + } + b->rep.Merge(SliceParts(key_slices.data(), num_keys), + SliceParts(value_slices.data(), num_values)); +} + +void rocksdb_writebatch_mergev_cf(rocksdb_writebatch_t* b, + rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, int num_values, + const char* const* values_list, + const size_t* values_list_sizes) { + std::vector key_slices(num_keys); + for (int i = 0; i < num_keys; i++) { + key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]); + } + std::vector value_slices(num_values); + for (int i = 0; i < num_values; i++) { + value_slices[i] = Slice(values_list[i], values_list_sizes[i]); + } + b->rep.Merge(column_family->rep, SliceParts(key_slices.data(), num_keys), + SliceParts(value_slices.data(), num_values)); +} + +void rocksdb_writebatch_delete(rocksdb_writebatch_t* b, const char* key, + size_t klen) { + b->rep.Delete(Slice(key, klen)); +} + +void rocksdb_writebatch_singledelete(rocksdb_writebatch_t* b, const char* key, + size_t klen) { + b->rep.SingleDelete(Slice(key, klen)); +} + +void rocksdb_writebatch_delete_cf(rocksdb_writebatch_t* b, + rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen) { + b->rep.Delete(column_family->rep, Slice(key, klen)); +} + +void rocksdb_writebatch_delete_cf_with_ts( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* ts, size_t tslen) { + b->rep.Delete(column_family->rep, Slice(key, klen), Slice(ts, tslen)); +} + +void rocksdb_writebatch_singledelete_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen) { + b->rep.SingleDelete(column_family->rep, Slice(key, klen)); +} + +void rocksdb_writebatch_singledelete_cf_with_ts( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* ts, size_t tslen) { + b->rep.SingleDelete(column_family->rep, Slice(key, klen), Slice(ts, tslen)); +} + +void rocksdb_writebatch_deletev(rocksdb_writebatch_t* b, int num_keys, + const char* const* keys_list, + const size_t* keys_list_sizes) { + std::vector key_slices(num_keys); + for (int i = 0; i < num_keys; i++) { + key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]); + } + b->rep.Delete(SliceParts(key_slices.data(), num_keys)); +} + +void rocksdb_writebatch_deletev_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes) { + std::vector key_slices(num_keys); + for (int i = 0; i < num_keys; i++) { + key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]); + } + b->rep.Delete(column_family->rep, SliceParts(key_slices.data(), num_keys)); +} + +void rocksdb_writebatch_delete_range(rocksdb_writebatch_t* b, + const char* start_key, + size_t start_key_len, const char* end_key, + size_t end_key_len) { + b->rep.DeleteRange(Slice(start_key, start_key_len), + Slice(end_key, end_key_len)); +} + +void rocksdb_writebatch_delete_range_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* end_key, + size_t end_key_len) { + b->rep.DeleteRange(column_family->rep, Slice(start_key, start_key_len), + Slice(end_key, end_key_len)); +} + +void rocksdb_writebatch_delete_rangev(rocksdb_writebatch_t* b, int num_keys, + const char* const* start_keys_list, + const size_t* start_keys_list_sizes, + const char* const* end_keys_list, + const size_t* end_keys_list_sizes) { + std::vector start_key_slices(num_keys); + std::vector end_key_slices(num_keys); + for (int i = 0; i < num_keys; i++) { + start_key_slices[i] = Slice(start_keys_list[i], start_keys_list_sizes[i]); + end_key_slices[i] = Slice(end_keys_list[i], end_keys_list_sizes[i]); + } + b->rep.DeleteRange(SliceParts(start_key_slices.data(), num_keys), + SliceParts(end_key_slices.data(), num_keys)); +} + +void rocksdb_writebatch_delete_rangev_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* start_keys_list, + const size_t* start_keys_list_sizes, const char* const* end_keys_list, + const size_t* end_keys_list_sizes) { + std::vector start_key_slices(num_keys); + std::vector end_key_slices(num_keys); + for (int i = 0; i < num_keys; i++) { + start_key_slices[i] = Slice(start_keys_list[i], start_keys_list_sizes[i]); + end_key_slices[i] = Slice(end_keys_list[i], end_keys_list_sizes[i]); + } + b->rep.DeleteRange(column_family->rep, + SliceParts(start_key_slices.data(), num_keys), + SliceParts(end_key_slices.data(), num_keys)); +} + +void rocksdb_writebatch_put_log_data(rocksdb_writebatch_t* b, const char* blob, + size_t len) { + b->rep.PutLogData(Slice(blob, len)); +} + +class H : public WriteBatch::Handler { + public: + void* state_; + void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen); + void (*deleted_)(void*, const char* k, size_t klen); + void Put(const Slice& key, const Slice& value) override { + (*put_)(state_, key.data(), key.size(), value.data(), value.size()); + } + void Delete(const Slice& key) override { + (*deleted_)(state_, key.data(), key.size()); + } +}; + +void rocksdb_writebatch_iterate(rocksdb_writebatch_t* b, void* state, + void (*put)(void*, const char* k, size_t klen, + const char* v, size_t vlen), + void (*deleted)(void*, const char* k, + size_t klen)) { + H handler; + handler.state_ = state; + handler.put_ = put; + handler.deleted_ = deleted; + b->rep.Iterate(&handler); +} + +const char* rocksdb_writebatch_data(rocksdb_writebatch_t* b, size_t* size) { + *size = b->rep.GetDataSize(); + return b->rep.Data().c_str(); +} + +void rocksdb_writebatch_set_save_point(rocksdb_writebatch_t* b) { + b->rep.SetSavePoint(); +} + +void rocksdb_writebatch_rollback_to_save_point(rocksdb_writebatch_t* b, + char** errptr) { + SaveError(errptr, b->rep.RollbackToSavePoint()); +} + +void rocksdb_writebatch_pop_save_point(rocksdb_writebatch_t* b, char** errptr) { + SaveError(errptr, b->rep.PopSavePoint()); +} + +rocksdb_writebatch_wi_t* rocksdb_writebatch_wi_create( + size_t reserved_bytes, unsigned char overwrite_key) { + rocksdb_writebatch_wi_t* b = new rocksdb_writebatch_wi_t; + b->rep = new WriteBatchWithIndex(BytewiseComparator(), reserved_bytes, + overwrite_key); + return b; +} + +void rocksdb_writebatch_wi_destroy(rocksdb_writebatch_wi_t* b) { + if (b->rep) { + delete b->rep; + } + delete b; +} + +void rocksdb_writebatch_wi_clear(rocksdb_writebatch_wi_t* b) { + b->rep->Clear(); +} + +int rocksdb_writebatch_wi_count(rocksdb_writebatch_wi_t* b) { + return b->rep->GetWriteBatch()->Count(); +} + +void rocksdb_writebatch_wi_put(rocksdb_writebatch_wi_t* b, const char* key, + size_t klen, const char* val, size_t vlen) { + b->rep->Put(Slice(key, klen), Slice(val, vlen)); +} + +void rocksdb_writebatch_wi_put_cf(rocksdb_writebatch_wi_t* b, + rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, + size_t vlen) { + b->rep->Put(column_family->rep, Slice(key, klen), Slice(val, vlen)); +} + +void rocksdb_writebatch_wi_putv(rocksdb_writebatch_wi_t* b, int num_keys, + const char* const* keys_list, + const size_t* keys_list_sizes, int num_values, + const char* const* values_list, + const size_t* values_list_sizes) { + std::vector key_slices(num_keys); + for (int i = 0; i < num_keys; i++) { + key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]); + } + std::vector value_slices(num_values); + for (int i = 0; i < num_values; i++) { + value_slices[i] = Slice(values_list[i], values_list_sizes[i]); + } + b->rep->Put(SliceParts(key_slices.data(), num_keys), + SliceParts(value_slices.data(), num_values)); +} + +void rocksdb_writebatch_wi_putv_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes, + int num_values, const char* const* values_list, + const size_t* values_list_sizes) { + std::vector key_slices(num_keys); + for (int i = 0; i < num_keys; i++) { + key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]); + } + std::vector value_slices(num_values); + for (int i = 0; i < num_values; i++) { + value_slices[i] = Slice(values_list[i], values_list_sizes[i]); + } + b->rep->Put(column_family->rep, SliceParts(key_slices.data(), num_keys), + SliceParts(value_slices.data(), num_values)); +} + +void rocksdb_writebatch_wi_merge(rocksdb_writebatch_wi_t* b, const char* key, + size_t klen, const char* val, size_t vlen) { + b->rep->Merge(Slice(key, klen), Slice(val, vlen)); +} + +void rocksdb_writebatch_wi_merge_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen) { + b->rep->Merge(column_family->rep, Slice(key, klen), Slice(val, vlen)); +} + +void rocksdb_writebatch_wi_mergev(rocksdb_writebatch_wi_t* b, int num_keys, + const char* const* keys_list, + const size_t* keys_list_sizes, int num_values, + const char* const* values_list, + const size_t* values_list_sizes) { + std::vector key_slices(num_keys); + for (int i = 0; i < num_keys; i++) { + key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]); + } + std::vector value_slices(num_values); + for (int i = 0; i < num_values; i++) { + value_slices[i] = Slice(values_list[i], values_list_sizes[i]); + } + b->rep->Merge(SliceParts(key_slices.data(), num_keys), + SliceParts(value_slices.data(), num_values)); +} + +void rocksdb_writebatch_wi_mergev_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes, + int num_values, const char* const* values_list, + const size_t* values_list_sizes) { + std::vector key_slices(num_keys); + for (int i = 0; i < num_keys; i++) { + key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]); + } + std::vector value_slices(num_values); + for (int i = 0; i < num_values; i++) { + value_slices[i] = Slice(values_list[i], values_list_sizes[i]); + } + b->rep->Merge(column_family->rep, SliceParts(key_slices.data(), num_keys), + SliceParts(value_slices.data(), num_values)); +} + +void rocksdb_writebatch_wi_delete(rocksdb_writebatch_wi_t* b, const char* key, + size_t klen) { + b->rep->Delete(Slice(key, klen)); +} + +void rocksdb_writebatch_wi_singledelete(rocksdb_writebatch_wi_t* b, + const char* key, size_t klen) { + b->rep->SingleDelete(Slice(key, klen)); +} + +void rocksdb_writebatch_wi_delete_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen) { + b->rep->Delete(column_family->rep, Slice(key, klen)); +} + +void rocksdb_writebatch_wi_singledelete_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen) { + b->rep->SingleDelete(column_family->rep, Slice(key, klen)); +} + +void rocksdb_writebatch_wi_deletev(rocksdb_writebatch_wi_t* b, int num_keys, + const char* const* keys_list, + const size_t* keys_list_sizes) { + std::vector key_slices(num_keys); + for (int i = 0; i < num_keys; i++) { + key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]); + } + b->rep->Delete(SliceParts(key_slices.data(), num_keys)); +} + +void rocksdb_writebatch_wi_deletev_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes) { + std::vector key_slices(num_keys); + for (int i = 0; i < num_keys; i++) { + key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]); + } + b->rep->Delete(column_family->rep, SliceParts(key_slices.data(), num_keys)); +} + +void rocksdb_writebatch_wi_delete_range(rocksdb_writebatch_wi_t* b, + const char* start_key, + size_t start_key_len, + const char* end_key, + size_t end_key_len) { + b->rep->DeleteRange(Slice(start_key, start_key_len), + Slice(end_key, end_key_len)); +} + +void rocksdb_writebatch_wi_delete_range_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* end_key, + size_t end_key_len) { + b->rep->DeleteRange(column_family->rep, Slice(start_key, start_key_len), + Slice(end_key, end_key_len)); +} + +void rocksdb_writebatch_wi_delete_rangev(rocksdb_writebatch_wi_t* b, + int num_keys, + const char* const* start_keys_list, + const size_t* start_keys_list_sizes, + const char* const* end_keys_list, + const size_t* end_keys_list_sizes) { + std::vector start_key_slices(num_keys); + std::vector end_key_slices(num_keys); + for (int i = 0; i < num_keys; i++) { + start_key_slices[i] = Slice(start_keys_list[i], start_keys_list_sizes[i]); + end_key_slices[i] = Slice(end_keys_list[i], end_keys_list_sizes[i]); + } + b->rep->DeleteRange(SliceParts(start_key_slices.data(), num_keys), + SliceParts(end_key_slices.data(), num_keys)); +} + +void rocksdb_writebatch_wi_delete_rangev_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* start_keys_list, + const size_t* start_keys_list_sizes, const char* const* end_keys_list, + const size_t* end_keys_list_sizes) { + std::vector start_key_slices(num_keys); + std::vector end_key_slices(num_keys); + for (int i = 0; i < num_keys; i++) { + start_key_slices[i] = Slice(start_keys_list[i], start_keys_list_sizes[i]); + end_key_slices[i] = Slice(end_keys_list[i], end_keys_list_sizes[i]); + } + b->rep->DeleteRange(column_family->rep, + SliceParts(start_key_slices.data(), num_keys), + SliceParts(end_key_slices.data(), num_keys)); +} + +void rocksdb_writebatch_wi_put_log_data(rocksdb_writebatch_wi_t* b, + const char* blob, size_t len) { + b->rep->PutLogData(Slice(blob, len)); +} + +void rocksdb_writebatch_wi_iterate( + rocksdb_writebatch_wi_t* b, void* state, + void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen), + void (*deleted)(void*, const char* k, size_t klen)) { + H handler; + handler.state_ = state; + handler.put_ = put; + handler.deleted_ = deleted; + b->rep->GetWriteBatch()->Iterate(&handler); +} + +const char* rocksdb_writebatch_wi_data(rocksdb_writebatch_wi_t* b, + size_t* size) { + WriteBatch* wb = b->rep->GetWriteBatch(); + *size = wb->GetDataSize(); + return wb->Data().c_str(); +} + +void rocksdb_writebatch_wi_set_save_point(rocksdb_writebatch_wi_t* b) { + b->rep->SetSavePoint(); +} + +void rocksdb_writebatch_wi_rollback_to_save_point(rocksdb_writebatch_wi_t* b, + char** errptr) { + SaveError(errptr, b->rep->RollbackToSavePoint()); +} + +rocksdb_iterator_t* rocksdb_writebatch_wi_create_iterator_with_base( + rocksdb_writebatch_wi_t* wbwi, rocksdb_iterator_t* base_iterator) { + rocksdb_iterator_t* result = new rocksdb_iterator_t; + result->rep = wbwi->rep->NewIteratorWithBase(base_iterator->rep); + delete base_iterator; + return result; +} + +rocksdb_iterator_t* rocksdb_writebatch_wi_create_iterator_with_base_cf( + rocksdb_writebatch_wi_t* wbwi, rocksdb_iterator_t* base_iterator, + rocksdb_column_family_handle_t* column_family) { + rocksdb_iterator_t* result = new rocksdb_iterator_t; + result->rep = + wbwi->rep->NewIteratorWithBase(column_family->rep, base_iterator->rep); + delete base_iterator; + return result; +} + +char* rocksdb_writebatch_wi_get_from_batch(rocksdb_writebatch_wi_t* wbwi, + const rocksdb_options_t* options, + const char* key, size_t keylen, + size_t* vallen, char** errptr) { + char* result = nullptr; + std::string tmp; + Status s = wbwi->rep->GetFromBatch(options->rep, Slice(key, keylen), &tmp); + if (s.ok()) { + *vallen = tmp.size(); + result = CopyString(tmp); + } else { + *vallen = 0; + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + } + return result; +} + +char* rocksdb_writebatch_wi_get_from_batch_cf( + rocksdb_writebatch_wi_t* wbwi, const rocksdb_options_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** errptr) { + char* result = nullptr; + std::string tmp; + Status s = wbwi->rep->GetFromBatch(column_family->rep, options->rep, + Slice(key, keylen), &tmp); + if (s.ok()) { + *vallen = tmp.size(); + result = CopyString(tmp); + } else { + *vallen = 0; + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + } + return result; +} + +char* rocksdb_writebatch_wi_get_from_batch_and_db( + rocksdb_writebatch_wi_t* wbwi, rocksdb_t* db, + const rocksdb_readoptions_t* options, const char* key, size_t keylen, + size_t* vallen, char** errptr) { + char* result = nullptr; + std::string tmp; + Status s = wbwi->rep->GetFromBatchAndDB(db->rep, options->rep, + Slice(key, keylen), &tmp); + if (s.ok()) { + *vallen = tmp.size(); + result = CopyString(tmp); + } else { + *vallen = 0; + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + } + return result; +} + +char* rocksdb_writebatch_wi_get_from_batch_and_db_cf( + rocksdb_writebatch_wi_t* wbwi, rocksdb_t* db, + const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** errptr) { + char* result = nullptr; + std::string tmp; + Status s = wbwi->rep->GetFromBatchAndDB( + db->rep, options->rep, column_family->rep, Slice(key, keylen), &tmp); + if (s.ok()) { + *vallen = tmp.size(); + result = CopyString(tmp); + } else { + *vallen = 0; + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + } + return result; +} + +void rocksdb_write_writebatch_wi(rocksdb_t* db, + const rocksdb_writeoptions_t* options, + rocksdb_writebatch_wi_t* wbwi, char** errptr) { + WriteBatch* wb = wbwi->rep->GetWriteBatch(); + SaveError(errptr, db->rep->Write(options->rep, wb)); +} + +void rocksdb_load_latest_options( + const char* db_path, rocksdb_env_t* env, bool ignore_unknown_options, + rocksdb_cache_t* cache, rocksdb_options_t** db_options, + size_t* num_column_families, char*** list_column_family_names, + rocksdb_options_t*** list_column_family_options, char** errptr) { + DBOptions db_opt; + std::vector cf_descs; + ConfigOptions config_opts; + config_opts.ignore_unknown_options = ignore_unknown_options; + config_opts.input_strings_escaped = true; + config_opts.env = env->rep; + Status s = LoadLatestOptions(config_opts, std::string(db_path), &db_opt, + &cf_descs, &cache->rep); + if (s.ok()) { + char** cf_names = (char**)malloc(cf_descs.size() * sizeof(char*)); + rocksdb_options_t** cf_options = (rocksdb_options_t**)malloc( + cf_descs.size() * sizeof(rocksdb_options_t*)); + for (size_t i = 0; i < cf_descs.size(); ++i) { + cf_names[i] = strdup(cf_descs[i].name.c_str()); + cf_options[i] = new rocksdb_options_t{ + Options(DBOptions(), std::move(cf_descs[i].options))}; + } + *num_column_families = cf_descs.size(); + *db_options = new rocksdb_options_t{ + Options(std::move(db_opt), ColumnFamilyOptions())}; + *list_column_family_names = cf_names; + *list_column_family_options = cf_options; + } else { + *num_column_families = 0; + *db_options = nullptr; + *list_column_family_names = nullptr; + *list_column_family_options = nullptr; + SaveError(errptr, s); + } +} + +void rocksdb_load_latest_options_destroy( + rocksdb_options_t* db_options, char** list_column_family_names, + rocksdb_options_t** list_column_family_options, size_t len) { + rocksdb_options_destroy(db_options); + if (list_column_family_names) { + for (size_t i = 0; i < len; ++i) { + free(list_column_family_names[i]); + } + free(list_column_family_names); + } + if (list_column_family_options) { + for (size_t i = 0; i < len; ++i) { + rocksdb_options_destroy(list_column_family_options[i]); + } + free(list_column_family_options); + } +} + +rocksdb_block_based_table_options_t* rocksdb_block_based_options_create() { + return new rocksdb_block_based_table_options_t; +} + +void rocksdb_block_based_options_destroy( + rocksdb_block_based_table_options_t* options) { + delete options; +} + +void rocksdb_block_based_options_set_checksum( + rocksdb_block_based_table_options_t* opt, char v) { + opt->rep.checksum = static_cast(v); +} + +void rocksdb_block_based_options_set_block_size( + rocksdb_block_based_table_options_t* options, size_t block_size) { + options->rep.block_size = block_size; +} + +void rocksdb_block_based_options_set_block_size_deviation( + rocksdb_block_based_table_options_t* options, int block_size_deviation) { + options->rep.block_size_deviation = block_size_deviation; +} + +void rocksdb_block_based_options_set_block_restart_interval( + rocksdb_block_based_table_options_t* options, int block_restart_interval) { + options->rep.block_restart_interval = block_restart_interval; +} + +void rocksdb_block_based_options_set_index_block_restart_interval( + rocksdb_block_based_table_options_t* options, + int index_block_restart_interval) { + options->rep.index_block_restart_interval = index_block_restart_interval; +} + +void rocksdb_block_based_options_set_metadata_block_size( + rocksdb_block_based_table_options_t* options, + uint64_t metadata_block_size) { + options->rep.metadata_block_size = metadata_block_size; +} + +void rocksdb_block_based_options_set_partition_filters( + rocksdb_block_based_table_options_t* options, + unsigned char partition_filters) { + options->rep.partition_filters = partition_filters; +} + +void rocksdb_block_based_options_set_optimize_filters_for_memory( + rocksdb_block_based_table_options_t* options, + unsigned char optimize_filters_for_memory) { + options->rep.optimize_filters_for_memory = optimize_filters_for_memory; +} + +void rocksdb_block_based_options_set_use_delta_encoding( + rocksdb_block_based_table_options_t* options, + unsigned char use_delta_encoding) { + options->rep.use_delta_encoding = use_delta_encoding; +} + +void rocksdb_block_based_options_set_filter_policy( + rocksdb_block_based_table_options_t* options, + rocksdb_filterpolicy_t* filter_policy) { + options->rep.filter_policy.reset(filter_policy); +} + +void rocksdb_block_based_options_set_no_block_cache( + rocksdb_block_based_table_options_t* options, + unsigned char no_block_cache) { + options->rep.no_block_cache = no_block_cache; +} + +void rocksdb_block_based_options_set_block_cache( + rocksdb_block_based_table_options_t* options, + rocksdb_cache_t* block_cache) { + if (block_cache) { + options->rep.block_cache = block_cache->rep; + } +} + +void rocksdb_block_based_options_set_whole_key_filtering( + rocksdb_block_based_table_options_t* options, unsigned char v) { + options->rep.whole_key_filtering = v; +} + +void rocksdb_block_based_options_set_format_version( + rocksdb_block_based_table_options_t* options, int v) { + options->rep.format_version = v; +} + +void rocksdb_block_based_options_set_index_type( + rocksdb_block_based_table_options_t* options, int v) { + options->rep.index_type = static_cast(v); +} + +void rocksdb_block_based_options_set_data_block_index_type( + rocksdb_block_based_table_options_t* options, int v) { + options->rep.data_block_index_type = + static_cast(v); +} + +void rocksdb_block_based_options_set_data_block_hash_ratio( + rocksdb_block_based_table_options_t* options, double v) { + options->rep.data_block_hash_table_util_ratio = v; +} + +void rocksdb_block_based_options_set_cache_index_and_filter_blocks( + rocksdb_block_based_table_options_t* options, unsigned char v) { + options->rep.cache_index_and_filter_blocks = v; +} + +void rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority( + rocksdb_block_based_table_options_t* options, unsigned char v) { + options->rep.cache_index_and_filter_blocks_with_high_priority = v; +} + +void rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache( + rocksdb_block_based_table_options_t* options, unsigned char v) { + options->rep.pin_l0_filter_and_index_blocks_in_cache = v; +} + +void rocksdb_block_based_options_set_pin_top_level_index_and_filter( + rocksdb_block_based_table_options_t* options, unsigned char v) { + options->rep.pin_top_level_index_and_filter = v; +} + +void rocksdb_options_set_block_based_table_factory( + rocksdb_options_t* opt, + rocksdb_block_based_table_options_t* table_options) { + if (table_options) { + opt->rep.table_factory.reset( + ROCKSDB_NAMESPACE::NewBlockBasedTableFactory(table_options->rep)); + } +} + +rocksdb_cuckoo_table_options_t* rocksdb_cuckoo_options_create() { + return new rocksdb_cuckoo_table_options_t; +} + +void rocksdb_cuckoo_options_destroy(rocksdb_cuckoo_table_options_t* options) { + delete options; +} + +void rocksdb_cuckoo_options_set_hash_ratio( + rocksdb_cuckoo_table_options_t* options, double v) { + options->rep.hash_table_ratio = v; +} + +void rocksdb_cuckoo_options_set_max_search_depth( + rocksdb_cuckoo_table_options_t* options, uint32_t v) { + options->rep.max_search_depth = v; +} + +void rocksdb_cuckoo_options_set_cuckoo_block_size( + rocksdb_cuckoo_table_options_t* options, uint32_t v) { + options->rep.cuckoo_block_size = v; +} + +void rocksdb_cuckoo_options_set_identity_as_first_hash( + rocksdb_cuckoo_table_options_t* options, unsigned char v) { + options->rep.identity_as_first_hash = v; +} + +void rocksdb_cuckoo_options_set_use_module_hash( + rocksdb_cuckoo_table_options_t* options, unsigned char v) { + options->rep.use_module_hash = v; +} + +void rocksdb_options_set_cuckoo_table_factory( + rocksdb_options_t* opt, rocksdb_cuckoo_table_options_t* table_options) { + if (table_options) { + opt->rep.table_factory.reset( + ROCKSDB_NAMESPACE::NewCuckooTableFactory(table_options->rep)); + } +} + +void rocksdb_set_options(rocksdb_t* db, int count, const char* const keys[], + const char* const values[], char** errptr) { + std::unordered_map options_map; + for (int i = 0; i < count; i++) options_map[keys[i]] = values[i]; + SaveError(errptr, db->rep->SetOptions(options_map)); +} + +void rocksdb_set_options_cf(rocksdb_t* db, + rocksdb_column_family_handle_t* handle, int count, + const char* const keys[], + const char* const values[], char** errptr) { + std::unordered_map options_map; + for (int i = 0; i < count; i++) options_map[keys[i]] = values[i]; + SaveError(errptr, db->rep->SetOptions(handle->rep, options_map)); +} + +rocksdb_options_t* rocksdb_options_create() { return new rocksdb_options_t; } + +void rocksdb_options_destroy(rocksdb_options_t* options) { delete options; } + +rocksdb_options_t* rocksdb_options_create_copy(rocksdb_options_t* options) { + return new rocksdb_options_t(*options); +} + +void rocksdb_options_increase_parallelism(rocksdb_options_t* opt, + int total_threads) { + opt->rep.IncreaseParallelism(total_threads); +} + +void rocksdb_options_optimize_for_point_lookup(rocksdb_options_t* opt, + uint64_t block_cache_size_mb) { + opt->rep.OptimizeForPointLookup(block_cache_size_mb); +} + +void rocksdb_options_optimize_level_style_compaction( + rocksdb_options_t* opt, uint64_t memtable_memory_budget) { + opt->rep.OptimizeLevelStyleCompaction(memtable_memory_budget); +} + +void rocksdb_options_optimize_universal_style_compaction( + rocksdb_options_t* opt, uint64_t memtable_memory_budget) { + opt->rep.OptimizeUniversalStyleCompaction(memtable_memory_budget); +} + +void rocksdb_options_set_allow_ingest_behind(rocksdb_options_t* opt, + unsigned char v) { + opt->rep.allow_ingest_behind = v; +} + +unsigned char rocksdb_options_get_allow_ingest_behind(rocksdb_options_t* opt) { + return opt->rep.allow_ingest_behind; +} + +void rocksdb_options_set_compaction_filter(rocksdb_options_t* opt, + rocksdb_compactionfilter_t* filter) { + opt->rep.compaction_filter = filter; +} + +void rocksdb_options_set_compaction_filter_factory( + rocksdb_options_t* opt, rocksdb_compactionfilterfactory_t* factory) { + opt->rep.compaction_filter_factory = + std::shared_ptr(factory); +} + +void rocksdb_options_compaction_readahead_size(rocksdb_options_t* opt, + size_t s) { + opt->rep.compaction_readahead_size = s; +} + +size_t rocksdb_options_get_compaction_readahead_size(rocksdb_options_t* opt) { + return opt->rep.compaction_readahead_size; +} + +void rocksdb_options_set_comparator(rocksdb_options_t* opt, + rocksdb_comparator_t* cmp) { + opt->rep.comparator = cmp; +} + +void rocksdb_options_set_merge_operator( + rocksdb_options_t* opt, rocksdb_mergeoperator_t* merge_operator) { + opt->rep.merge_operator = std::shared_ptr(merge_operator); +} + +void rocksdb_options_set_create_if_missing(rocksdb_options_t* opt, + unsigned char v) { + opt->rep.create_if_missing = v; +} + +unsigned char rocksdb_options_get_create_if_missing(rocksdb_options_t* opt) { + return opt->rep.create_if_missing; +} + +void rocksdb_options_set_create_missing_column_families(rocksdb_options_t* opt, + unsigned char v) { + opt->rep.create_missing_column_families = v; +} + +unsigned char rocksdb_options_get_create_missing_column_families( + rocksdb_options_t* opt) { + return opt->rep.create_missing_column_families; +} + +void rocksdb_options_set_error_if_exists(rocksdb_options_t* opt, + unsigned char v) { + opt->rep.error_if_exists = v; +} + +unsigned char rocksdb_options_get_error_if_exists(rocksdb_options_t* opt) { + return opt->rep.error_if_exists; +} + +void rocksdb_options_set_paranoid_checks(rocksdb_options_t* opt, + unsigned char v) { + opt->rep.paranoid_checks = v; +} + +unsigned char rocksdb_options_get_paranoid_checks(rocksdb_options_t* opt) { + return opt->rep.paranoid_checks; +} + +void rocksdb_options_set_db_paths(rocksdb_options_t* opt, + const rocksdb_dbpath_t** dbpath_values, + size_t num_paths) { + std::vector db_paths(num_paths); + for (size_t i = 0; i < num_paths; ++i) { + db_paths[i] = dbpath_values[i]->rep; + } + opt->rep.db_paths = db_paths; +} + +void rocksdb_options_set_env(rocksdb_options_t* opt, rocksdb_env_t* env) { + opt->rep.env = (env ? env->rep : nullptr); +} + +void rocksdb_options_set_info_log(rocksdb_options_t* opt, rocksdb_logger_t* l) { + if (l) { + opt->rep.info_log = l->rep; + } +} + +void rocksdb_options_set_info_log_level(rocksdb_options_t* opt, int v) { + opt->rep.info_log_level = static_cast(v); +} + +int rocksdb_options_get_info_log_level(rocksdb_options_t* opt) { + return static_cast(opt->rep.info_log_level); +} + +void rocksdb_options_set_db_write_buffer_size(rocksdb_options_t* opt, + size_t s) { + opt->rep.db_write_buffer_size = s; +} + +size_t rocksdb_options_get_db_write_buffer_size(rocksdb_options_t* opt) { + return opt->rep.db_write_buffer_size; +} + +void rocksdb_options_set_write_buffer_size(rocksdb_options_t* opt, size_t s) { + opt->rep.write_buffer_size = s; +} + +size_t rocksdb_options_get_write_buffer_size(rocksdb_options_t* opt) { + return opt->rep.write_buffer_size; +} + +void rocksdb_options_set_max_open_files(rocksdb_options_t* opt, int n) { + opt->rep.max_open_files = n; +} + +int rocksdb_options_get_max_open_files(rocksdb_options_t* opt) { + return opt->rep.max_open_files; +} + +void rocksdb_options_set_max_file_opening_threads(rocksdb_options_t* opt, + int n) { + opt->rep.max_file_opening_threads = n; +} + +int rocksdb_options_get_max_file_opening_threads(rocksdb_options_t* opt) { + return opt->rep.max_file_opening_threads; +} + +void rocksdb_options_set_max_total_wal_size(rocksdb_options_t* opt, + uint64_t n) { + opt->rep.max_total_wal_size = n; +} + +uint64_t rocksdb_options_get_max_total_wal_size(rocksdb_options_t* opt) { + return opt->rep.max_total_wal_size; +} + +void rocksdb_options_set_target_file_size_base(rocksdb_options_t* opt, + uint64_t n) { + opt->rep.target_file_size_base = n; +} + +uint64_t rocksdb_options_get_target_file_size_base(rocksdb_options_t* opt) { + return opt->rep.target_file_size_base; +} + +void rocksdb_options_set_target_file_size_multiplier(rocksdb_options_t* opt, + int n) { + opt->rep.target_file_size_multiplier = n; +} + +int rocksdb_options_get_target_file_size_multiplier(rocksdb_options_t* opt) { + return opt->rep.target_file_size_multiplier; +} + +void rocksdb_options_set_max_bytes_for_level_base(rocksdb_options_t* opt, + uint64_t n) { + opt->rep.max_bytes_for_level_base = n; +} + +uint64_t rocksdb_options_get_max_bytes_for_level_base(rocksdb_options_t* opt) { + return opt->rep.max_bytes_for_level_base; +} + +void rocksdb_options_set_level_compaction_dynamic_level_bytes( + rocksdb_options_t* opt, unsigned char v) { + opt->rep.level_compaction_dynamic_level_bytes = v; +} + +unsigned char rocksdb_options_get_level_compaction_dynamic_level_bytes( + rocksdb_options_t* opt) { + return opt->rep.level_compaction_dynamic_level_bytes; +} + +void rocksdb_options_set_max_bytes_for_level_multiplier(rocksdb_options_t* opt, + double n) { + opt->rep.max_bytes_for_level_multiplier = n; +} + +double rocksdb_options_get_max_bytes_for_level_multiplier( + rocksdb_options_t* opt) { + return opt->rep.max_bytes_for_level_multiplier; +} + +void rocksdb_options_set_max_compaction_bytes(rocksdb_options_t* opt, + uint64_t n) { + opt->rep.max_compaction_bytes = n; +} + +uint64_t rocksdb_options_get_max_compaction_bytes(rocksdb_options_t* opt) { + return opt->rep.max_compaction_bytes; +} + +void rocksdb_options_set_max_bytes_for_level_multiplier_additional( + rocksdb_options_t* opt, int* level_values, size_t num_levels) { + opt->rep.max_bytes_for_level_multiplier_additional.resize(num_levels); + for (size_t i = 0; i < num_levels; ++i) { + opt->rep.max_bytes_for_level_multiplier_additional[i] = level_values[i]; + } +} + +void rocksdb_options_enable_statistics(rocksdb_options_t* opt) { + opt->rep.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); +} + +void rocksdb_options_set_statistics_level(rocksdb_options_t* opt, int level) { + if (!opt->rep.statistics) { + return; + } + + if (level < rocksdb_statistics_level_disable_all) { + level = rocksdb_statistics_level_disable_all; + } + if (level > rocksdb_statistics_level_all) { + level = rocksdb_statistics_level_all; + } + opt->rep.statistics->set_stats_level( + static_cast(level)); +} + +int rocksdb_options_get_statistics_level(rocksdb_options_t* opt) { + if (!opt->rep.statistics) { + return ROCKSDB_NAMESPACE::StatsLevel::kDisableAll; + } + + return static_cast(opt->rep.statistics->get_stats_level()); +} + +void rocksdb_options_set_skip_stats_update_on_db_open(rocksdb_options_t* opt, + unsigned char val) { + opt->rep.skip_stats_update_on_db_open = val; +} + +unsigned char rocksdb_options_get_skip_stats_update_on_db_open( + rocksdb_options_t* opt) { + return opt->rep.skip_stats_update_on_db_open; +} + +void rocksdb_options_set_skip_checking_sst_file_sizes_on_db_open( + rocksdb_options_t* opt, unsigned char val) { + opt->rep.skip_checking_sst_file_sizes_on_db_open = val; +} + +unsigned char rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open( + rocksdb_options_t* opt) { + return opt->rep.skip_checking_sst_file_sizes_on_db_open; +} + +/* Blob Options Settings */ +void rocksdb_options_set_enable_blob_files(rocksdb_options_t* opt, + unsigned char val) { + opt->rep.enable_blob_files = val; +} +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_enable_blob_files( + rocksdb_options_t* opt) { + return opt->rep.enable_blob_files; +} + +void rocksdb_options_set_min_blob_size(rocksdb_options_t* opt, uint64_t val) { + opt->rep.min_blob_size = val; +} + +uint64_t rocksdb_options_get_min_blob_size(rocksdb_options_t* opt) { + return opt->rep.min_blob_size; +} + +void rocksdb_options_set_blob_file_size(rocksdb_options_t* opt, uint64_t val) { + opt->rep.blob_file_size = val; +} + +uint64_t rocksdb_options_get_blob_file_size(rocksdb_options_t* opt) { + return opt->rep.blob_file_size; +} + +void rocksdb_options_set_blob_compression_type(rocksdb_options_t* opt, + int val) { + opt->rep.blob_compression_type = static_cast(val); +} + +int rocksdb_options_get_blob_compression_type(rocksdb_options_t* opt) { + return opt->rep.blob_compression_type; +} + +void rocksdb_options_set_enable_blob_gc(rocksdb_options_t* opt, + unsigned char val) { + opt->rep.enable_blob_garbage_collection = val; +} + +unsigned char rocksdb_options_get_enable_blob_gc(rocksdb_options_t* opt) { + return opt->rep.enable_blob_garbage_collection; +} + +void rocksdb_options_set_blob_gc_age_cutoff(rocksdb_options_t* opt, + double val) { + opt->rep.blob_garbage_collection_age_cutoff = val; +} + +double rocksdb_options_get_blob_gc_age_cutoff(rocksdb_options_t* opt) { + return opt->rep.blob_garbage_collection_age_cutoff; +} + +void rocksdb_options_set_blob_gc_force_threshold(rocksdb_options_t* opt, + double val) { + opt->rep.blob_garbage_collection_force_threshold = val; +} + +double rocksdb_options_get_blob_gc_force_threshold(rocksdb_options_t* opt) { + return opt->rep.blob_garbage_collection_force_threshold; +} + +void rocksdb_options_set_blob_compaction_readahead_size(rocksdb_options_t* opt, + uint64_t val) { + opt->rep.blob_compaction_readahead_size = val; +} + +uint64_t rocksdb_options_get_blob_compaction_readahead_size( + rocksdb_options_t* opt) { + return opt->rep.blob_compaction_readahead_size; +} + +void rocksdb_options_set_blob_file_starting_level(rocksdb_options_t* opt, + int val) { + opt->rep.blob_file_starting_level = val; +} + +int rocksdb_options_get_blob_file_starting_level(rocksdb_options_t* opt) { + return opt->rep.blob_file_starting_level; +} + +void rocksdb_options_set_blob_cache(rocksdb_options_t* opt, + rocksdb_cache_t* blob_cache) { + opt->rep.blob_cache = blob_cache->rep; +} + +void rocksdb_options_set_prepopulate_blob_cache(rocksdb_options_t* opt, int t) { + opt->rep.prepopulate_blob_cache = static_cast(t); +} + +int rocksdb_options_get_prepopulate_blob_cache(rocksdb_options_t* opt) { + return static_cast(opt->rep.prepopulate_blob_cache); +} + +void rocksdb_options_set_num_levels(rocksdb_options_t* opt, int n) { + opt->rep.num_levels = n; +} + +int rocksdb_options_get_num_levels(rocksdb_options_t* opt) { + return opt->rep.num_levels; +} + +void rocksdb_options_set_level0_file_num_compaction_trigger( + rocksdb_options_t* opt, int n) { + opt->rep.level0_file_num_compaction_trigger = n; +} + +int rocksdb_options_get_level0_file_num_compaction_trigger( + rocksdb_options_t* opt) { + return opt->rep.level0_file_num_compaction_trigger; +} + +void rocksdb_options_set_level0_slowdown_writes_trigger(rocksdb_options_t* opt, + int n) { + opt->rep.level0_slowdown_writes_trigger = n; +} + +int rocksdb_options_get_level0_slowdown_writes_trigger(rocksdb_options_t* opt) { + return opt->rep.level0_slowdown_writes_trigger; +} + +void rocksdb_options_set_level0_stop_writes_trigger(rocksdb_options_t* opt, + int n) { + opt->rep.level0_stop_writes_trigger = n; +} + +int rocksdb_options_get_level0_stop_writes_trigger(rocksdb_options_t* opt) { + return opt->rep.level0_stop_writes_trigger; +} + +void rocksdb_options_set_wal_recovery_mode(rocksdb_options_t* opt, int mode) { + opt->rep.wal_recovery_mode = static_cast(mode); +} + +int rocksdb_options_get_wal_recovery_mode(rocksdb_options_t* opt) { + return static_cast(opt->rep.wal_recovery_mode); +} + +void rocksdb_options_set_compression(rocksdb_options_t* opt, int t) { + opt->rep.compression = static_cast(t); +} + +int rocksdb_options_get_compression(rocksdb_options_t* opt) { + return opt->rep.compression; +} + +void rocksdb_options_set_bottommost_compression(rocksdb_options_t* opt, int t) { + opt->rep.bottommost_compression = static_cast(t); +} + +int rocksdb_options_get_bottommost_compression(rocksdb_options_t* opt) { + return opt->rep.bottommost_compression; +} + +void rocksdb_options_set_compression_per_level(rocksdb_options_t* opt, + const int* level_values, + size_t num_levels) { + opt->rep.compression_per_level.resize(num_levels); + for (size_t i = 0; i < num_levels; ++i) { + opt->rep.compression_per_level[i] = + static_cast(level_values[i]); + } +} + +void rocksdb_options_set_bottommost_compression_options(rocksdb_options_t* opt, + int w_bits, int level, + int strategy, + int max_dict_bytes, + unsigned char enabled) { + opt->rep.bottommost_compression_opts.window_bits = w_bits; + opt->rep.bottommost_compression_opts.level = level; + opt->rep.bottommost_compression_opts.strategy = strategy; + opt->rep.bottommost_compression_opts.max_dict_bytes = max_dict_bytes; + opt->rep.bottommost_compression_opts.enabled = enabled; +} + +void rocksdb_options_set_bottommost_compression_options_zstd_max_train_bytes( + rocksdb_options_t* opt, int zstd_max_train_bytes, unsigned char enabled) { + opt->rep.bottommost_compression_opts.zstd_max_train_bytes = + zstd_max_train_bytes; + opt->rep.bottommost_compression_opts.enabled = enabled; +} + +void rocksdb_options_set_bottommost_compression_options_use_zstd_dict_trainer( + rocksdb_options_t* opt, unsigned char use_zstd_dict_trainer, + unsigned char enabled) { + opt->rep.bottommost_compression_opts.use_zstd_dict_trainer = + use_zstd_dict_trainer; + opt->rep.bottommost_compression_opts.enabled = enabled; +} + +unsigned char +rocksdb_options_get_bottommost_compression_options_use_zstd_dict_trainer( + rocksdb_options_t* opt) { + return opt->rep.bottommost_compression_opts.use_zstd_dict_trainer; +} + +void rocksdb_options_set_bottommost_compression_options_max_dict_buffer_bytes( + rocksdb_options_t* opt, uint64_t max_dict_buffer_bytes, + unsigned char enabled) { + opt->rep.bottommost_compression_opts.max_dict_buffer_bytes = + max_dict_buffer_bytes; + opt->rep.bottommost_compression_opts.enabled = enabled; +} + +void rocksdb_options_set_compression_options(rocksdb_options_t* opt, int w_bits, + int level, int strategy, + int max_dict_bytes) { + opt->rep.compression_opts.window_bits = w_bits; + opt->rep.compression_opts.level = level; + opt->rep.compression_opts.strategy = strategy; + opt->rep.compression_opts.max_dict_bytes = max_dict_bytes; +} + +void rocksdb_options_set_compression_options_zstd_max_train_bytes( + rocksdb_options_t* opt, int zstd_max_train_bytes) { + opt->rep.compression_opts.zstd_max_train_bytes = zstd_max_train_bytes; +} + +int rocksdb_options_get_compression_options_zstd_max_train_bytes( + rocksdb_options_t* opt) { + return opt->rep.compression_opts.zstd_max_train_bytes; +} + +void rocksdb_options_set_compression_options_use_zstd_dict_trainer( + rocksdb_options_t* opt, unsigned char use_zstd_dict_trainer) { + opt->rep.compression_opts.use_zstd_dict_trainer = use_zstd_dict_trainer; +} + +unsigned char rocksdb_options_get_compression_options_use_zstd_dict_trainer( + rocksdb_options_t* opt) { + return opt->rep.compression_opts.use_zstd_dict_trainer; +} + +void rocksdb_options_set_compression_options_parallel_threads( + rocksdb_options_t* opt, int value) { + opt->rep.compression_opts.parallel_threads = value; +} + +int rocksdb_options_get_compression_options_parallel_threads( + rocksdb_options_t* opt) { + return opt->rep.compression_opts.parallel_threads; +} + +void rocksdb_options_set_compression_options_max_dict_buffer_bytes( + rocksdb_options_t* opt, uint64_t max_dict_buffer_bytes) { + opt->rep.compression_opts.max_dict_buffer_bytes = max_dict_buffer_bytes; +} + +uint64_t rocksdb_options_get_compression_options_max_dict_buffer_bytes( + rocksdb_options_t* opt) { + return opt->rep.compression_opts.max_dict_buffer_bytes; +} + +void rocksdb_options_set_prefix_extractor( + rocksdb_options_t* opt, rocksdb_slicetransform_t* prefix_extractor) { + opt->rep.prefix_extractor.reset(prefix_extractor); +} + +void rocksdb_options_set_use_fsync(rocksdb_options_t* opt, int use_fsync) { + opt->rep.use_fsync = use_fsync; +} + +int rocksdb_options_get_use_fsync(rocksdb_options_t* opt) { + return opt->rep.use_fsync; +} + +void rocksdb_options_set_db_log_dir(rocksdb_options_t* opt, + const char* db_log_dir) { + opt->rep.db_log_dir = db_log_dir; +} + +void rocksdb_options_set_wal_dir(rocksdb_options_t* opt, const char* v) { + opt->rep.wal_dir = v; +} + +void rocksdb_options_set_WAL_ttl_seconds(rocksdb_options_t* opt, uint64_t ttl) { + opt->rep.WAL_ttl_seconds = ttl; +} + +uint64_t rocksdb_options_get_WAL_ttl_seconds(rocksdb_options_t* opt) { + return opt->rep.WAL_ttl_seconds; +} + +void rocksdb_options_set_WAL_size_limit_MB(rocksdb_options_t* opt, + uint64_t limit) { + opt->rep.WAL_size_limit_MB = limit; +} + +uint64_t rocksdb_options_get_WAL_size_limit_MB(rocksdb_options_t* opt) { + return opt->rep.WAL_size_limit_MB; +} + +void rocksdb_options_set_manifest_preallocation_size(rocksdb_options_t* opt, + size_t v) { + opt->rep.manifest_preallocation_size = v; +} + +size_t rocksdb_options_get_manifest_preallocation_size(rocksdb_options_t* opt) { + return opt->rep.manifest_preallocation_size; +} + +void rocksdb_options_set_use_direct_reads(rocksdb_options_t* opt, + unsigned char v) { + opt->rep.use_direct_reads = v; +} + +unsigned char rocksdb_options_get_use_direct_reads(rocksdb_options_t* opt) { + return opt->rep.use_direct_reads; +} + +void rocksdb_options_set_use_direct_io_for_flush_and_compaction( + rocksdb_options_t* opt, unsigned char v) { + opt->rep.use_direct_io_for_flush_and_compaction = v; +} + +unsigned char rocksdb_options_get_use_direct_io_for_flush_and_compaction( + rocksdb_options_t* opt) { + return opt->rep.use_direct_io_for_flush_and_compaction; +} + +void rocksdb_options_set_allow_mmap_reads(rocksdb_options_t* opt, + unsigned char v) { + opt->rep.allow_mmap_reads = v; +} + +unsigned char rocksdb_options_get_allow_mmap_reads(rocksdb_options_t* opt) { + return opt->rep.allow_mmap_reads; +} + +void rocksdb_options_set_allow_mmap_writes(rocksdb_options_t* opt, + unsigned char v) { + opt->rep.allow_mmap_writes = v; +} + +unsigned char rocksdb_options_get_allow_mmap_writes(rocksdb_options_t* opt) { + return opt->rep.allow_mmap_writes; +} + +void rocksdb_options_set_is_fd_close_on_exec(rocksdb_options_t* opt, + unsigned char v) { + opt->rep.is_fd_close_on_exec = v; +} + +unsigned char rocksdb_options_get_is_fd_close_on_exec(rocksdb_options_t* opt) { + return opt->rep.is_fd_close_on_exec; +} + +void rocksdb_options_set_stats_dump_period_sec(rocksdb_options_t* opt, + unsigned int v) { + opt->rep.stats_dump_period_sec = v; +} + +unsigned int rocksdb_options_get_stats_dump_period_sec(rocksdb_options_t* opt) { + return opt->rep.stats_dump_period_sec; +} + +void rocksdb_options_set_stats_persist_period_sec(rocksdb_options_t* opt, + unsigned int v) { + opt->rep.stats_persist_period_sec = v; +} + +unsigned int rocksdb_options_get_stats_persist_period_sec( + rocksdb_options_t* opt) { + return opt->rep.stats_persist_period_sec; +} + +void rocksdb_options_set_advise_random_on_open(rocksdb_options_t* opt, + unsigned char v) { + opt->rep.advise_random_on_open = v; +} + +unsigned char rocksdb_options_get_advise_random_on_open( + rocksdb_options_t* opt) { + return opt->rep.advise_random_on_open; +} + +void rocksdb_options_set_access_hint_on_compaction_start(rocksdb_options_t* opt, + int v) { + switch (v) { + case 0: + opt->rep.access_hint_on_compaction_start = + ROCKSDB_NAMESPACE::Options::NONE; + break; + case 1: + opt->rep.access_hint_on_compaction_start = + ROCKSDB_NAMESPACE::Options::NORMAL; + break; + case 2: + opt->rep.access_hint_on_compaction_start = + ROCKSDB_NAMESPACE::Options::SEQUENTIAL; + break; + case 3: + opt->rep.access_hint_on_compaction_start = + ROCKSDB_NAMESPACE::Options::WILLNEED; + break; + default: + assert(0); + } +} + +int rocksdb_options_get_access_hint_on_compaction_start( + rocksdb_options_t* opt) { + return opt->rep.access_hint_on_compaction_start; +} + +void rocksdb_options_set_use_adaptive_mutex(rocksdb_options_t* opt, + unsigned char v) { + opt->rep.use_adaptive_mutex = v; +} + +unsigned char rocksdb_options_get_use_adaptive_mutex(rocksdb_options_t* opt) { + return opt->rep.use_adaptive_mutex; +} + +void rocksdb_options_set_wal_bytes_per_sync(rocksdb_options_t* opt, + uint64_t v) { + opt->rep.wal_bytes_per_sync = v; +} + +uint64_t rocksdb_options_get_wal_bytes_per_sync(rocksdb_options_t* opt) { + return opt->rep.wal_bytes_per_sync; +} + +void rocksdb_options_set_bytes_per_sync(rocksdb_options_t* opt, uint64_t v) { + opt->rep.bytes_per_sync = v; +} + +uint64_t rocksdb_options_get_bytes_per_sync(rocksdb_options_t* opt) { + return opt->rep.bytes_per_sync; +} + +void rocksdb_options_set_writable_file_max_buffer_size(rocksdb_options_t* opt, + uint64_t v) { + opt->rep.writable_file_max_buffer_size = static_cast(v); +} + +uint64_t rocksdb_options_get_writable_file_max_buffer_size( + rocksdb_options_t* opt) { + return opt->rep.writable_file_max_buffer_size; +} + +void rocksdb_options_set_allow_concurrent_memtable_write(rocksdb_options_t* opt, + unsigned char v) { + opt->rep.allow_concurrent_memtable_write = v; +} + +unsigned char rocksdb_options_get_allow_concurrent_memtable_write( + rocksdb_options_t* opt) { + return opt->rep.allow_concurrent_memtable_write; +} + +void rocksdb_options_set_enable_write_thread_adaptive_yield( + rocksdb_options_t* opt, unsigned char v) { + opt->rep.enable_write_thread_adaptive_yield = v; +} + +unsigned char rocksdb_options_get_enable_write_thread_adaptive_yield( + rocksdb_options_t* opt) { + return opt->rep.enable_write_thread_adaptive_yield; +} + +void rocksdb_options_set_max_sequential_skip_in_iterations( + rocksdb_options_t* opt, uint64_t v) { + opt->rep.max_sequential_skip_in_iterations = v; +} + +uint64_t rocksdb_options_get_max_sequential_skip_in_iterations( + rocksdb_options_t* opt) { + return opt->rep.max_sequential_skip_in_iterations; +} + +void rocksdb_options_set_max_write_buffer_number(rocksdb_options_t* opt, + int n) { + opt->rep.max_write_buffer_number = n; +} + +int rocksdb_options_get_max_write_buffer_number(rocksdb_options_t* opt) { + return opt->rep.max_write_buffer_number; +} + +void rocksdb_options_set_min_write_buffer_number_to_merge( + rocksdb_options_t* opt, int n) { + opt->rep.min_write_buffer_number_to_merge = n; +} + +int rocksdb_options_get_min_write_buffer_number_to_merge( + rocksdb_options_t* opt) { + return opt->rep.min_write_buffer_number_to_merge; +} + +void rocksdb_options_set_max_write_buffer_number_to_maintain( + rocksdb_options_t* opt, int n) { + opt->rep.max_write_buffer_number_to_maintain = n; +} + +int rocksdb_options_get_max_write_buffer_number_to_maintain( + rocksdb_options_t* opt) { + return opt->rep.max_write_buffer_number_to_maintain; +} + +void rocksdb_options_set_max_write_buffer_size_to_maintain( + rocksdb_options_t* opt, int64_t n) { + opt->rep.max_write_buffer_size_to_maintain = n; +} + +int64_t rocksdb_options_get_max_write_buffer_size_to_maintain( + rocksdb_options_t* opt) { + return opt->rep.max_write_buffer_size_to_maintain; +} + +void rocksdb_options_set_enable_pipelined_write(rocksdb_options_t* opt, + unsigned char v) { + opt->rep.enable_pipelined_write = v; +} + +unsigned char rocksdb_options_get_enable_pipelined_write( + rocksdb_options_t* opt) { + return opt->rep.enable_pipelined_write; +} + +void rocksdb_options_set_unordered_write(rocksdb_options_t* opt, + unsigned char v) { + opt->rep.unordered_write = v; +} + +unsigned char rocksdb_options_get_unordered_write(rocksdb_options_t* opt) { + return opt->rep.unordered_write; +} + +void rocksdb_options_set_max_subcompactions(rocksdb_options_t* opt, + uint32_t n) { + opt->rep.max_subcompactions = n; +} + +uint32_t rocksdb_options_get_max_subcompactions(rocksdb_options_t* opt) { + return opt->rep.max_subcompactions; +} + +void rocksdb_options_set_max_background_jobs(rocksdb_options_t* opt, int n) { + opt->rep.max_background_jobs = n; +} + +int rocksdb_options_get_max_background_jobs(rocksdb_options_t* opt) { + return opt->rep.max_background_jobs; +} + +void rocksdb_options_set_max_background_compactions(rocksdb_options_t* opt, + int n) { + opt->rep.max_background_compactions = n; +} + +int rocksdb_options_get_max_background_compactions(rocksdb_options_t* opt) { + return opt->rep.max_background_compactions; +} + +void rocksdb_options_set_max_background_flushes(rocksdb_options_t* opt, int n) { + opt->rep.max_background_flushes = n; +} + +int rocksdb_options_get_max_background_flushes(rocksdb_options_t* opt) { + return opt->rep.max_background_flushes; +} + +void rocksdb_options_set_experimental_mempurge_threshold(rocksdb_options_t* opt, + double v) { + opt->rep.experimental_mempurge_threshold = v; +} + +double rocksdb_options_get_experimental_mempurge_threshold( + rocksdb_options_t* opt) { + return opt->rep.experimental_mempurge_threshold; +} + +void rocksdb_options_set_max_log_file_size(rocksdb_options_t* opt, size_t v) { + opt->rep.max_log_file_size = v; +} + +size_t rocksdb_options_get_max_log_file_size(rocksdb_options_t* opt) { + return opt->rep.max_log_file_size; +} + +void rocksdb_options_set_log_file_time_to_roll(rocksdb_options_t* opt, + size_t v) { + opt->rep.log_file_time_to_roll = v; +} + +size_t rocksdb_options_get_log_file_time_to_roll(rocksdb_options_t* opt) { + return opt->rep.log_file_time_to_roll; +} + +void rocksdb_options_set_keep_log_file_num(rocksdb_options_t* opt, size_t v) { + opt->rep.keep_log_file_num = v; +} + +size_t rocksdb_options_get_keep_log_file_num(rocksdb_options_t* opt) { + return opt->rep.keep_log_file_num; +} + +void rocksdb_options_set_recycle_log_file_num(rocksdb_options_t* opt, + size_t v) { + opt->rep.recycle_log_file_num = v; +} + +size_t rocksdb_options_get_recycle_log_file_num(rocksdb_options_t* opt) { + return opt->rep.recycle_log_file_num; +} + +void rocksdb_options_set_soft_pending_compaction_bytes_limit( + rocksdb_options_t* opt, size_t v) { + opt->rep.soft_pending_compaction_bytes_limit = v; +} + +size_t rocksdb_options_get_soft_pending_compaction_bytes_limit( + rocksdb_options_t* opt) { + return opt->rep.soft_pending_compaction_bytes_limit; +} + +void rocksdb_options_set_hard_pending_compaction_bytes_limit( + rocksdb_options_t* opt, size_t v) { + opt->rep.hard_pending_compaction_bytes_limit = v; +} + +size_t rocksdb_options_get_hard_pending_compaction_bytes_limit( + rocksdb_options_t* opt) { + return opt->rep.hard_pending_compaction_bytes_limit; +} + +void rocksdb_options_set_max_manifest_file_size(rocksdb_options_t* opt, + size_t v) { + opt->rep.max_manifest_file_size = v; +} + +size_t rocksdb_options_get_max_manifest_file_size(rocksdb_options_t* opt) { + return opt->rep.max_manifest_file_size; +} + +void rocksdb_options_set_table_cache_numshardbits(rocksdb_options_t* opt, + int v) { + opt->rep.table_cache_numshardbits = v; +} + +int rocksdb_options_get_table_cache_numshardbits(rocksdb_options_t* opt) { + return opt->rep.table_cache_numshardbits; +} + +void rocksdb_options_set_arena_block_size(rocksdb_options_t* opt, size_t v) { + opt->rep.arena_block_size = v; +} + +size_t rocksdb_options_get_arena_block_size(rocksdb_options_t* opt) { + return opt->rep.arena_block_size; +} + +void rocksdb_options_set_disable_auto_compactions(rocksdb_options_t* opt, + int disable) { + opt->rep.disable_auto_compactions = disable; +} + +unsigned char rocksdb_options_get_disable_auto_compactions( + rocksdb_options_t* opt) { + return opt->rep.disable_auto_compactions; +} + +void rocksdb_options_set_optimize_filters_for_hits(rocksdb_options_t* opt, + int v) { + opt->rep.optimize_filters_for_hits = v; +} + +unsigned char rocksdb_options_get_optimize_filters_for_hits( + rocksdb_options_t* opt) { + return opt->rep.optimize_filters_for_hits; +} + +void rocksdb_options_set_delete_obsolete_files_period_micros( + rocksdb_options_t* opt, uint64_t v) { + opt->rep.delete_obsolete_files_period_micros = v; +} + +uint64_t rocksdb_options_get_delete_obsolete_files_period_micros( + rocksdb_options_t* opt) { + return opt->rep.delete_obsolete_files_period_micros; +} + +void rocksdb_options_prepare_for_bulk_load(rocksdb_options_t* opt) { + opt->rep.PrepareForBulkLoad(); +} + +void rocksdb_options_set_memtable_vector_rep(rocksdb_options_t* opt) { + opt->rep.memtable_factory.reset(new ROCKSDB_NAMESPACE::VectorRepFactory); +} + +void rocksdb_options_set_memtable_prefix_bloom_size_ratio( + rocksdb_options_t* opt, double v) { + opt->rep.memtable_prefix_bloom_size_ratio = v; +} + +double rocksdb_options_get_memtable_prefix_bloom_size_ratio( + rocksdb_options_t* opt) { + return opt->rep.memtable_prefix_bloom_size_ratio; +} + +void rocksdb_options_set_memtable_huge_page_size(rocksdb_options_t* opt, + size_t v) { + opt->rep.memtable_huge_page_size = v; +} + +size_t rocksdb_options_get_memtable_huge_page_size(rocksdb_options_t* opt) { + return opt->rep.memtable_huge_page_size; +} + +void rocksdb_options_set_hash_skip_list_rep(rocksdb_options_t* opt, + size_t bucket_count, + int32_t skiplist_height, + int32_t skiplist_branching_factor) { + ROCKSDB_NAMESPACE::MemTableRepFactory* factory = + ROCKSDB_NAMESPACE::NewHashSkipListRepFactory( + bucket_count, skiplist_height, skiplist_branching_factor); + opt->rep.memtable_factory.reset(factory); +} + +void rocksdb_options_set_hash_link_list_rep(rocksdb_options_t* opt, + size_t bucket_count) { + opt->rep.memtable_factory.reset( + ROCKSDB_NAMESPACE::NewHashLinkListRepFactory(bucket_count)); +} + +void rocksdb_options_set_plain_table_factory( + rocksdb_options_t* opt, uint32_t user_key_len, int bloom_bits_per_key, + double hash_table_ratio, size_t index_sparseness, size_t huge_page_tlb_size, + char encoding_type, unsigned char full_scan_mode, + unsigned char store_index_in_file) { + ROCKSDB_NAMESPACE::PlainTableOptions options; + options.user_key_len = user_key_len; + options.bloom_bits_per_key = bloom_bits_per_key; + options.hash_table_ratio = hash_table_ratio; + options.index_sparseness = index_sparseness; + options.huge_page_tlb_size = huge_page_tlb_size; + options.encoding_type = + static_cast(encoding_type); + options.full_scan_mode = full_scan_mode; + options.store_index_in_file = store_index_in_file; + + ROCKSDB_NAMESPACE::TableFactory* factory = + ROCKSDB_NAMESPACE::NewPlainTableFactory(options); + opt->rep.table_factory.reset(factory); +} + +void rocksdb_options_set_max_successive_merges(rocksdb_options_t* opt, + size_t v) { + opt->rep.max_successive_merges = v; +} + +size_t rocksdb_options_get_max_successive_merges(rocksdb_options_t* opt) { + return opt->rep.max_successive_merges; +} + +void rocksdb_options_set_bloom_locality(rocksdb_options_t* opt, uint32_t v) { + opt->rep.bloom_locality = v; +} + +uint32_t rocksdb_options_get_bloom_locality(rocksdb_options_t* opt) { + return opt->rep.bloom_locality; +} + +void rocksdb_options_set_inplace_update_support(rocksdb_options_t* opt, + unsigned char v) { + opt->rep.inplace_update_support = v; +} + +unsigned char rocksdb_options_get_inplace_update_support( + rocksdb_options_t* opt) { + return opt->rep.inplace_update_support; +} + +void rocksdb_options_set_inplace_update_num_locks(rocksdb_options_t* opt, + size_t v) { + opt->rep.inplace_update_num_locks = v; +} + +size_t rocksdb_options_get_inplace_update_num_locks(rocksdb_options_t* opt) { + return opt->rep.inplace_update_num_locks; +} + +void rocksdb_options_set_report_bg_io_stats(rocksdb_options_t* opt, int v) { + opt->rep.report_bg_io_stats = v; +} + +unsigned char rocksdb_options_get_report_bg_io_stats(rocksdb_options_t* opt) { + return opt->rep.report_bg_io_stats; +} + +void rocksdb_options_set_compaction_style(rocksdb_options_t* opt, int style) { + opt->rep.compaction_style = + static_cast(style); +} + +int rocksdb_options_get_compaction_style(rocksdb_options_t* opt) { + return opt->rep.compaction_style; +} + +void rocksdb_options_set_universal_compaction_options( + rocksdb_options_t* opt, rocksdb_universal_compaction_options_t* uco) { + opt->rep.compaction_options_universal = *(uco->rep); +} + +void rocksdb_options_set_fifo_compaction_options( + rocksdb_options_t* opt, rocksdb_fifo_compaction_options_t* fifo) { + opt->rep.compaction_options_fifo = fifo->rep; +} + +char* rocksdb_options_statistics_get_string(rocksdb_options_t* opt) { + ROCKSDB_NAMESPACE::Statistics* statistics = opt->rep.statistics.get(); + if (statistics) { + return strdup(statistics->ToString().c_str()); + } + return nullptr; +} + +uint64_t rocksdb_options_statistics_get_ticker_count(rocksdb_options_t* opt, + uint32_t ticker_type) { + ROCKSDB_NAMESPACE::Statistics* statistics = opt->rep.statistics.get(); + if (statistics) { + return statistics->getTickerCount(ticker_type); + } + return 0; +} + +void rocksdb_options_statistics_get_histogram_data( + rocksdb_options_t* opt, uint32_t type, + rocksdb_statistics_histogram_data_t* const data) { + ROCKSDB_NAMESPACE::Statistics* statistics = opt->rep.statistics.get(); + if (statistics) { + statistics->histogramData(type, &data->rep); + } else { + *data = rocksdb_statistics_histogram_data_t{}; + } +} + +void rocksdb_options_set_ratelimiter(rocksdb_options_t* opt, + rocksdb_ratelimiter_t* limiter) { + if (limiter) { + opt->rep.rate_limiter = limiter->rep; + } +} + +void rocksdb_options_set_atomic_flush(rocksdb_options_t* opt, + unsigned char atomic_flush) { + opt->rep.atomic_flush = atomic_flush; +} + +unsigned char rocksdb_options_get_atomic_flush(rocksdb_options_t* opt) { + return opt->rep.atomic_flush; +} + +void rocksdb_options_set_manual_wal_flush(rocksdb_options_t* opt, + unsigned char manual_wal_flush) { + opt->rep.manual_wal_flush = manual_wal_flush; +} + +unsigned char rocksdb_options_get_manual_wal_flush(rocksdb_options_t* opt) { + return opt->rep.manual_wal_flush; +} + +void rocksdb_options_set_wal_compression(rocksdb_options_t* opt, int val) { + opt->rep.wal_compression = static_cast(val); +} + +int rocksdb_options_get_wal_compression(rocksdb_options_t* opt) { + return opt->rep.wal_compression; +} + +rocksdb_ratelimiter_t* rocksdb_ratelimiter_create(int64_t rate_bytes_per_sec, + int64_t refill_period_us, + int32_t fairness) { + rocksdb_ratelimiter_t* rate_limiter = new rocksdb_ratelimiter_t; + rate_limiter->rep.reset( + NewGenericRateLimiter(rate_bytes_per_sec, refill_period_us, fairness)); + return rate_limiter; +} + +void rocksdb_ratelimiter_destroy(rocksdb_ratelimiter_t* limiter) { + delete limiter; +} + +void rocksdb_options_set_row_cache(rocksdb_options_t* opt, + rocksdb_cache_t* cache) { + if (cache) { + opt->rep.row_cache = cache->rep; + } +} + +void rocksdb_options_add_compact_on_deletion_collector_factory( + rocksdb_options_t* opt, size_t window_size, size_t num_dels_trigger) { + std::shared_ptr + compact_on_del = + NewCompactOnDeletionCollectorFactory(window_size, num_dels_trigger); + opt->rep.table_properties_collector_factories.emplace_back(compact_on_del); +} + +void rocksdb_options_add_compact_on_deletion_collector_factory_del_ratio( + rocksdb_options_t* opt, size_t window_size, size_t num_dels_trigger, + double deletion_ratio) { + std::shared_ptr + compact_on_del = NewCompactOnDeletionCollectorFactory( + window_size, num_dels_trigger, deletion_ratio); + opt->rep.table_properties_collector_factories.emplace_back(compact_on_del); +} + +void rocksdb_set_perf_level(int v) { + PerfLevel level = static_cast(v); + SetPerfLevel(level); +} + +rocksdb_perfcontext_t* rocksdb_perfcontext_create() { + rocksdb_perfcontext_t* context = new rocksdb_perfcontext_t; + context->rep = ROCKSDB_NAMESPACE::get_perf_context(); + return context; +} + +void rocksdb_perfcontext_reset(rocksdb_perfcontext_t* context) { + context->rep->Reset(); +} + +char* rocksdb_perfcontext_report(rocksdb_perfcontext_t* context, + unsigned char exclude_zero_counters) { + return strdup(context->rep->ToString(exclude_zero_counters).c_str()); +} + +uint64_t rocksdb_perfcontext_metric(rocksdb_perfcontext_t* context, + int metric) { + PerfContext* rep = context->rep; + switch (metric) { + case rocksdb_user_key_comparison_count: + return rep->user_key_comparison_count; + case rocksdb_block_cache_hit_count: + return rep->block_cache_hit_count; + case rocksdb_block_read_count: + return rep->block_read_count; + case rocksdb_block_read_byte: + return rep->block_read_byte; + case rocksdb_block_read_time: + return rep->block_read_time; + case rocksdb_block_checksum_time: + return rep->block_checksum_time; + case rocksdb_block_decompress_time: + return rep->block_decompress_time; + case rocksdb_get_read_bytes: + return rep->get_read_bytes; + case rocksdb_multiget_read_bytes: + return rep->multiget_read_bytes; + case rocksdb_iter_read_bytes: + return rep->iter_read_bytes; + case rocksdb_internal_key_skipped_count: + return rep->internal_key_skipped_count; + case rocksdb_internal_delete_skipped_count: + return rep->internal_delete_skipped_count; + case rocksdb_internal_recent_skipped_count: + return rep->internal_recent_skipped_count; + case rocksdb_internal_merge_count: + return rep->internal_merge_count; + case rocksdb_get_snapshot_time: + return rep->get_snapshot_time; + case rocksdb_get_from_memtable_time: + return rep->get_from_memtable_time; + case rocksdb_get_from_memtable_count: + return rep->get_from_memtable_count; + case rocksdb_get_post_process_time: + return rep->get_post_process_time; + case rocksdb_get_from_output_files_time: + return rep->get_from_output_files_time; + case rocksdb_seek_on_memtable_time: + return rep->seek_on_memtable_time; + case rocksdb_seek_on_memtable_count: + return rep->seek_on_memtable_count; + case rocksdb_next_on_memtable_count: + return rep->next_on_memtable_count; + case rocksdb_prev_on_memtable_count: + return rep->prev_on_memtable_count; + case rocksdb_seek_child_seek_time: + return rep->seek_child_seek_time; + case rocksdb_seek_child_seek_count: + return rep->seek_child_seek_count; + case rocksdb_seek_min_heap_time: + return rep->seek_min_heap_time; + case rocksdb_seek_max_heap_time: + return rep->seek_max_heap_time; + case rocksdb_seek_internal_seek_time: + return rep->seek_internal_seek_time; + case rocksdb_find_next_user_entry_time: + return rep->find_next_user_entry_time; + case rocksdb_write_wal_time: + return rep->write_wal_time; + case rocksdb_write_memtable_time: + return rep->write_memtable_time; + case rocksdb_write_delay_time: + return rep->write_delay_time; + case rocksdb_write_pre_and_post_process_time: + return rep->write_pre_and_post_process_time; + case rocksdb_db_mutex_lock_nanos: + return rep->db_mutex_lock_nanos; + case rocksdb_db_condition_wait_nanos: + return rep->db_condition_wait_nanos; + case rocksdb_merge_operator_time_nanos: + return rep->merge_operator_time_nanos; + case rocksdb_read_index_block_nanos: + return rep->read_index_block_nanos; + case rocksdb_read_filter_block_nanos: + return rep->read_filter_block_nanos; + case rocksdb_new_table_block_iter_nanos: + return rep->new_table_block_iter_nanos; + case rocksdb_new_table_iterator_nanos: + return rep->new_table_iterator_nanos; + case rocksdb_block_seek_nanos: + return rep->block_seek_nanos; + case rocksdb_find_table_nanos: + return rep->find_table_nanos; + case rocksdb_bloom_memtable_hit_count: + return rep->bloom_memtable_hit_count; + case rocksdb_bloom_memtable_miss_count: + return rep->bloom_memtable_miss_count; + case rocksdb_bloom_sst_hit_count: + return rep->bloom_sst_hit_count; + case rocksdb_bloom_sst_miss_count: + return rep->bloom_sst_miss_count; + case rocksdb_key_lock_wait_time: + return rep->key_lock_wait_time; + case rocksdb_key_lock_wait_count: + return rep->key_lock_wait_count; + case rocksdb_env_new_sequential_file_nanos: + return rep->env_new_sequential_file_nanos; + case rocksdb_env_new_random_access_file_nanos: + return rep->env_new_random_access_file_nanos; + case rocksdb_env_new_writable_file_nanos: + return rep->env_new_writable_file_nanos; + case rocksdb_env_reuse_writable_file_nanos: + return rep->env_reuse_writable_file_nanos; + case rocksdb_env_new_random_rw_file_nanos: + return rep->env_new_random_rw_file_nanos; + case rocksdb_env_new_directory_nanos: + return rep->env_new_directory_nanos; + case rocksdb_env_file_exists_nanos: + return rep->env_file_exists_nanos; + case rocksdb_env_get_children_nanos: + return rep->env_get_children_nanos; + case rocksdb_env_get_children_file_attributes_nanos: + return rep->env_get_children_file_attributes_nanos; + case rocksdb_env_delete_file_nanos: + return rep->env_delete_file_nanos; + case rocksdb_env_create_dir_nanos: + return rep->env_create_dir_nanos; + case rocksdb_env_create_dir_if_missing_nanos: + return rep->env_create_dir_if_missing_nanos; + case rocksdb_env_delete_dir_nanos: + return rep->env_delete_dir_nanos; + case rocksdb_env_get_file_size_nanos: + return rep->env_get_file_size_nanos; + case rocksdb_env_get_file_modification_time_nanos: + return rep->env_get_file_modification_time_nanos; + case rocksdb_env_rename_file_nanos: + return rep->env_rename_file_nanos; + case rocksdb_env_link_file_nanos: + return rep->env_link_file_nanos; + case rocksdb_env_lock_file_nanos: + return rep->env_lock_file_nanos; + case rocksdb_env_unlock_file_nanos: + return rep->env_unlock_file_nanos; + case rocksdb_env_new_logger_nanos: + return rep->env_new_logger_nanos; + case rocksdb_number_async_seek: + return rep->number_async_seek; + case rocksdb_blob_cache_hit_count: + return rep->blob_cache_hit_count; + case rocksdb_blob_read_count: + return rep->blob_read_count; + case rocksdb_blob_read_byte: + return rep->blob_read_byte; + case rocksdb_blob_read_time: + return rep->blob_read_time; + case rocksdb_blob_checksum_time: + return rep->blob_checksum_time; + case rocksdb_blob_decompress_time: + return rep->blob_decompress_time; + case rocksdb_internal_range_del_reseek_count: + return rep->internal_range_del_reseek_count; + case rocksdb_block_read_cpu_time: + return rep->block_read_cpu_time; + default: + break; + } + return 0; +} + +void rocksdb_perfcontext_destroy(rocksdb_perfcontext_t* context) { + delete context; +} + +/* +TODO: +DB::OpenForReadOnly +DB::KeyMayExist +DB::GetOptions +DB::GetSortedWalFiles +DB::GetLatestSequenceNumber +DB::GetUpdatesSince +DB::GetDbIdentity +DB::RunManualCompaction +custom cache +table_properties_collectors +*/ + +rocksdb_compactionfilter_t* rocksdb_compactionfilter_create( + void* state, void (*destructor)(void*), + unsigned char (*filter)(void*, int level, const char* key, + size_t key_length, const char* existing_value, + size_t value_length, char** new_value, + size_t* new_value_length, + unsigned char* value_changed), + const char* (*name)(void*)) { + rocksdb_compactionfilter_t* result = new rocksdb_compactionfilter_t; + result->state_ = state; + result->destructor_ = destructor; + result->filter_ = filter; + result->ignore_snapshots_ = true; + result->name_ = name; + return result; +} + +void rocksdb_compactionfilter_set_ignore_snapshots( + rocksdb_compactionfilter_t* filter, unsigned char whether_ignore) { + filter->ignore_snapshots_ = whether_ignore; +} + +void rocksdb_compactionfilter_destroy(rocksdb_compactionfilter_t* filter) { + delete filter; +} + +unsigned char rocksdb_compactionfiltercontext_is_full_compaction( + rocksdb_compactionfiltercontext_t* context) { + return context->rep.is_full_compaction; +} + +unsigned char rocksdb_compactionfiltercontext_is_manual_compaction( + rocksdb_compactionfiltercontext_t* context) { + return context->rep.is_manual_compaction; +} + +rocksdb_compactionfilterfactory_t* rocksdb_compactionfilterfactory_create( + void* state, void (*destructor)(void*), + rocksdb_compactionfilter_t* (*create_compaction_filter)( + void*, rocksdb_compactionfiltercontext_t* context), + const char* (*name)(void*)) { + rocksdb_compactionfilterfactory_t* result = + new rocksdb_compactionfilterfactory_t; + result->state_ = state; + result->destructor_ = destructor; + result->create_compaction_filter_ = create_compaction_filter; + result->name_ = name; + return result; +} + +void rocksdb_compactionfilterfactory_destroy( + rocksdb_compactionfilterfactory_t* factory) { + delete factory; +} + +rocksdb_comparator_t* rocksdb_comparator_create( + void* state, void (*destructor)(void*), + int (*compare)(void*, const char* a, size_t alen, const char* b, + size_t blen), + const char* (*name)(void*)) { + rocksdb_comparator_t* result = new rocksdb_comparator_t; + result->state_ = state; + result->destructor_ = destructor; + result->compare_ = compare; + result->name_ = name; + result->compare_ts_ = nullptr; + result->compare_without_ts_ = nullptr; + return result; +} + +void rocksdb_comparator_destroy(rocksdb_comparator_t* cmp) { delete cmp; } + +rocksdb_comparator_t* rocksdb_comparator_with_ts_create( + void* state, void (*destructor)(void*), + int (*compare)(void*, const char* a, size_t alen, const char* b, + size_t blen), + int (*compare_ts)(void*, const char* a_ts, size_t a_tslen, const char* b_ts, + size_t b_tslen), + int (*compare_without_ts)(void*, const char* a, size_t alen, + unsigned char a_has_ts, const char* b, + size_t blen, unsigned char b_has_ts), + const char* (*name)(void*), size_t timestamp_size) { + rocksdb_comparator_t* result = new rocksdb_comparator_t(timestamp_size); + result->state_ = state; + result->destructor_ = destructor; + result->compare_ = compare; + result->compare_ts_ = compare_ts; + result->compare_without_ts_ = compare_without_ts; + result->name_ = name; + return result; +} + +void rocksdb_filterpolicy_destroy(rocksdb_filterpolicy_t* filter) { + delete filter; +} + +rocksdb_filterpolicy_t* rocksdb_filterpolicy_create_bloom_format( + double bits_per_key, bool original_format) { + // Make a rocksdb_filterpolicy_t, but override all of its methods so + // they delegate to a NewBloomFilterPolicy() instead of user + // supplied C functions. + struct Wrapper : public rocksdb_filterpolicy_t { + const FilterPolicy* rep_; + ~Wrapper() override { delete rep_; } + const char* Name() const override { return rep_->Name(); } + const char* CompatibilityName() const override { + return rep_->CompatibilityName(); + } + // No need to override GetFilterBitsBuilder if this one is overridden + ROCKSDB_NAMESPACE::FilterBitsBuilder* GetBuilderWithContext( + const ROCKSDB_NAMESPACE::FilterBuildingContext& context) + const override { + return rep_->GetBuilderWithContext(context); + } + ROCKSDB_NAMESPACE::FilterBitsReader* GetFilterBitsReader( + const Slice& contents) const override { + return rep_->GetFilterBitsReader(contents); + } + static void DoNothing(void*) {} + }; + Wrapper* wrapper = new Wrapper; + wrapper->rep_ = NewBloomFilterPolicy(bits_per_key, original_format); + wrapper->state_ = nullptr; + wrapper->destructor_ = &Wrapper::DoNothing; + return wrapper; +} + +rocksdb_filterpolicy_t* rocksdb_filterpolicy_create_bloom_full( + double bits_per_key) { + return rocksdb_filterpolicy_create_bloom_format(bits_per_key, false); +} + +rocksdb_filterpolicy_t* rocksdb_filterpolicy_create_bloom(double bits_per_key) { + return rocksdb_filterpolicy_create_bloom_format(bits_per_key, true); +} + +rocksdb_filterpolicy_t* rocksdb_filterpolicy_create_ribbon_format( + double bloom_equivalent_bits_per_key, int bloom_before_level) { + // Make a rocksdb_filterpolicy_t, but override all of its methods so + // they delegate to a NewRibbonFilterPolicy() instead of user + // supplied C functions. + struct Wrapper : public rocksdb_filterpolicy_t { + const FilterPolicy* rep_; + ~Wrapper() override { delete rep_; } + const char* Name() const override { return rep_->Name(); } + const char* CompatibilityName() const override { + return rep_->CompatibilityName(); + } + ROCKSDB_NAMESPACE::FilterBitsBuilder* GetBuilderWithContext( + const ROCKSDB_NAMESPACE::FilterBuildingContext& context) + const override { + return rep_->GetBuilderWithContext(context); + } + ROCKSDB_NAMESPACE::FilterBitsReader* GetFilterBitsReader( + const Slice& contents) const override { + return rep_->GetFilterBitsReader(contents); + } + static void DoNothing(void*) {} + }; + Wrapper* wrapper = new Wrapper; + wrapper->rep_ = + NewRibbonFilterPolicy(bloom_equivalent_bits_per_key, bloom_before_level); + wrapper->state_ = nullptr; + wrapper->destructor_ = &Wrapper::DoNothing; + return wrapper; +} + +rocksdb_filterpolicy_t* rocksdb_filterpolicy_create_ribbon( + double bloom_equivalent_bits_per_key) { + return rocksdb_filterpolicy_create_ribbon_format( + bloom_equivalent_bits_per_key, /*bloom_before_level = disabled*/ -1); +} + +rocksdb_filterpolicy_t* rocksdb_filterpolicy_create_ribbon_hybrid( + double bloom_equivalent_bits_per_key, int bloom_before_level) { + return rocksdb_filterpolicy_create_ribbon_format( + bloom_equivalent_bits_per_key, bloom_before_level); +} + +rocksdb_mergeoperator_t* rocksdb_mergeoperator_create( + void* state, void (*destructor)(void*), + char* (*full_merge)(void*, const char* key, size_t key_length, + const char* existing_value, + size_t existing_value_length, + const char* const* operands_list, + const size_t* operands_list_length, int num_operands, + unsigned char* success, size_t* new_value_length), + char* (*partial_merge)(void*, const char* key, size_t key_length, + const char* const* operands_list, + const size_t* operands_list_length, int num_operands, + unsigned char* success, size_t* new_value_length), + void (*delete_value)(void*, const char* value, size_t value_length), + const char* (*name)(void*)) { + rocksdb_mergeoperator_t* result = new rocksdb_mergeoperator_t; + result->state_ = state; + result->destructor_ = destructor; + result->full_merge_ = full_merge; + result->partial_merge_ = partial_merge; + result->delete_value_ = delete_value; + result->name_ = name; + return result; +} + +void rocksdb_mergeoperator_destroy(rocksdb_mergeoperator_t* merge_operator) { + delete merge_operator; +} + +rocksdb_readoptions_t* rocksdb_readoptions_create() { + return new rocksdb_readoptions_t; +} + +void rocksdb_readoptions_destroy(rocksdb_readoptions_t* opt) { delete opt; } + +void rocksdb_readoptions_set_verify_checksums(rocksdb_readoptions_t* opt, + unsigned char v) { + opt->rep.verify_checksums = v; +} + +unsigned char rocksdb_readoptions_get_verify_checksums( + rocksdb_readoptions_t* opt) { + return opt->rep.verify_checksums; +} + +void rocksdb_readoptions_set_fill_cache(rocksdb_readoptions_t* opt, + unsigned char v) { + opt->rep.fill_cache = v; +} + +unsigned char rocksdb_readoptions_get_fill_cache(rocksdb_readoptions_t* opt) { + return opt->rep.fill_cache; +} + +void rocksdb_readoptions_set_snapshot(rocksdb_readoptions_t* opt, + const rocksdb_snapshot_t* snap) { + opt->rep.snapshot = (snap ? snap->rep : nullptr); +} + +void rocksdb_readoptions_set_iterate_upper_bound(rocksdb_readoptions_t* opt, + const char* key, + size_t keylen) { + if (key == nullptr) { + opt->upper_bound = Slice(); + opt->rep.iterate_upper_bound = nullptr; + + } else { + opt->upper_bound = Slice(key, keylen); + opt->rep.iterate_upper_bound = &opt->upper_bound; + } +} + +void rocksdb_readoptions_set_iterate_lower_bound(rocksdb_readoptions_t* opt, + const char* key, + size_t keylen) { + if (key == nullptr) { + opt->lower_bound = Slice(); + opt->rep.iterate_lower_bound = nullptr; + } else { + opt->lower_bound = Slice(key, keylen); + opt->rep.iterate_lower_bound = &opt->lower_bound; + } +} + +void rocksdb_readoptions_set_read_tier(rocksdb_readoptions_t* opt, int v) { + opt->rep.read_tier = static_cast(v); +} + +int rocksdb_readoptions_get_read_tier(rocksdb_readoptions_t* opt) { + return static_cast(opt->rep.read_tier); +} + +void rocksdb_readoptions_set_tailing(rocksdb_readoptions_t* opt, + unsigned char v) { + opt->rep.tailing = v; +} + +unsigned char rocksdb_readoptions_get_tailing(rocksdb_readoptions_t* opt) { + return opt->rep.tailing; +} + +void rocksdb_readoptions_set_managed(rocksdb_readoptions_t* opt, + unsigned char v) { + opt->rep.managed = v; +} + +void rocksdb_readoptions_set_readahead_size(rocksdb_readoptions_t* opt, + size_t v) { + opt->rep.readahead_size = v; +} + +size_t rocksdb_readoptions_get_readahead_size(rocksdb_readoptions_t* opt) { + return opt->rep.readahead_size; +} + +void rocksdb_readoptions_set_prefix_same_as_start(rocksdb_readoptions_t* opt, + unsigned char v) { + opt->rep.prefix_same_as_start = v; +} + +unsigned char rocksdb_readoptions_get_prefix_same_as_start( + rocksdb_readoptions_t* opt) { + return opt->rep.prefix_same_as_start; +} + +void rocksdb_readoptions_set_pin_data(rocksdb_readoptions_t* opt, + unsigned char v) { + opt->rep.pin_data = v; +} + +unsigned char rocksdb_readoptions_get_pin_data(rocksdb_readoptions_t* opt) { + return opt->rep.pin_data; +} + +void rocksdb_readoptions_set_total_order_seek(rocksdb_readoptions_t* opt, + unsigned char v) { + opt->rep.total_order_seek = v; +} + +unsigned char rocksdb_readoptions_get_total_order_seek( + rocksdb_readoptions_t* opt) { + return opt->rep.total_order_seek; +} + +void rocksdb_readoptions_set_max_skippable_internal_keys( + rocksdb_readoptions_t* opt, uint64_t v) { + opt->rep.max_skippable_internal_keys = v; +} + +uint64_t rocksdb_readoptions_get_max_skippable_internal_keys( + rocksdb_readoptions_t* opt) { + return opt->rep.max_skippable_internal_keys; +} + +void rocksdb_readoptions_set_background_purge_on_iterator_cleanup( + rocksdb_readoptions_t* opt, unsigned char v) { + opt->rep.background_purge_on_iterator_cleanup = v; +} + +unsigned char rocksdb_readoptions_get_background_purge_on_iterator_cleanup( + rocksdb_readoptions_t* opt) { + return opt->rep.background_purge_on_iterator_cleanup; +} + +void rocksdb_readoptions_set_ignore_range_deletions(rocksdb_readoptions_t* opt, + unsigned char v) { + opt->rep.ignore_range_deletions = v; +} + +unsigned char rocksdb_readoptions_get_ignore_range_deletions( + rocksdb_readoptions_t* opt) { + return opt->rep.ignore_range_deletions; +} + +void rocksdb_readoptions_set_deadline(rocksdb_readoptions_t* opt, + uint64_t microseconds) { + opt->rep.deadline = std::chrono::microseconds(microseconds); +} + +uint64_t rocksdb_readoptions_get_deadline(rocksdb_readoptions_t* opt) { + return opt->rep.deadline.count(); +} + +void rocksdb_readoptions_set_io_timeout(rocksdb_readoptions_t* opt, + uint64_t microseconds) { + opt->rep.io_timeout = std::chrono::microseconds(microseconds); +} + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_readoptions_get_io_timeout(rocksdb_readoptions_t* opt) { + return opt->rep.io_timeout.count(); +} + +void rocksdb_readoptions_set_async_io(rocksdb_readoptions_t* opt, + unsigned char v) { + opt->rep.async_io = v; +} + +unsigned char rocksdb_readoptions_get_async_io(rocksdb_readoptions_t* opt) { + return opt->rep.async_io; +} + +void rocksdb_readoptions_set_timestamp(rocksdb_readoptions_t* opt, + const char* ts, size_t tslen) { + if (ts == nullptr) { + opt->timestamp = Slice(); + opt->rep.timestamp = nullptr; + } else { + opt->timestamp = Slice(ts, tslen); + opt->rep.timestamp = &opt->timestamp; + } +} + +void rocksdb_readoptions_set_iter_start_ts(rocksdb_readoptions_t* opt, + const char* ts, size_t tslen) { + if (ts == nullptr) { + opt->iter_start_ts = Slice(); + opt->rep.iter_start_ts = nullptr; + } else { + opt->iter_start_ts = Slice(ts, tslen); + opt->rep.iter_start_ts = &opt->iter_start_ts; + } +} + +rocksdb_writeoptions_t* rocksdb_writeoptions_create() { + return new rocksdb_writeoptions_t; +} + +void rocksdb_writeoptions_destroy(rocksdb_writeoptions_t* opt) { delete opt; } + +void rocksdb_writeoptions_set_sync(rocksdb_writeoptions_t* opt, + unsigned char v) { + opt->rep.sync = v; +} + +unsigned char rocksdb_writeoptions_get_sync(rocksdb_writeoptions_t* opt) { + return opt->rep.sync; +} + +void rocksdb_writeoptions_disable_WAL(rocksdb_writeoptions_t* opt, + int disable) { + opt->rep.disableWAL = disable; +} + +unsigned char rocksdb_writeoptions_get_disable_WAL( + rocksdb_writeoptions_t* opt) { + return opt->rep.disableWAL; +} + +void rocksdb_writeoptions_set_ignore_missing_column_families( + rocksdb_writeoptions_t* opt, unsigned char v) { + opt->rep.ignore_missing_column_families = v; +} + +unsigned char rocksdb_writeoptions_get_ignore_missing_column_families( + rocksdb_writeoptions_t* opt) { + return opt->rep.ignore_missing_column_families; +} + +void rocksdb_writeoptions_set_no_slowdown(rocksdb_writeoptions_t* opt, + unsigned char v) { + opt->rep.no_slowdown = v; +} + +unsigned char rocksdb_writeoptions_get_no_slowdown( + rocksdb_writeoptions_t* opt) { + return opt->rep.no_slowdown; +} + +void rocksdb_writeoptions_set_low_pri(rocksdb_writeoptions_t* opt, + unsigned char v) { + opt->rep.low_pri = v; +} + +unsigned char rocksdb_writeoptions_get_low_pri(rocksdb_writeoptions_t* opt) { + return opt->rep.low_pri; +} + +void rocksdb_writeoptions_set_memtable_insert_hint_per_batch( + rocksdb_writeoptions_t* opt, unsigned char v) { + opt->rep.memtable_insert_hint_per_batch = v; +} + +unsigned char rocksdb_writeoptions_get_memtable_insert_hint_per_batch( + rocksdb_writeoptions_t* opt) { + return opt->rep.memtable_insert_hint_per_batch; +} + +rocksdb_compactoptions_t* rocksdb_compactoptions_create() { + return new rocksdb_compactoptions_t; +} + +void rocksdb_compactoptions_destroy(rocksdb_compactoptions_t* opt) { + delete opt; +} + +void rocksdb_compactoptions_set_bottommost_level_compaction( + rocksdb_compactoptions_t* opt, unsigned char v) { + opt->rep.bottommost_level_compaction = + static_cast(v); +} + +unsigned char rocksdb_compactoptions_get_bottommost_level_compaction( + rocksdb_compactoptions_t* opt) { + return static_cast(opt->rep.bottommost_level_compaction); +} + +void rocksdb_compactoptions_set_exclusive_manual_compaction( + rocksdb_compactoptions_t* opt, unsigned char v) { + opt->rep.exclusive_manual_compaction = v; +} + +unsigned char rocksdb_compactoptions_get_exclusive_manual_compaction( + rocksdb_compactoptions_t* opt) { + return opt->rep.exclusive_manual_compaction; +} + +void rocksdb_compactoptions_set_change_level(rocksdb_compactoptions_t* opt, + unsigned char v) { + opt->rep.change_level = v; +} + +unsigned char rocksdb_compactoptions_get_change_level( + rocksdb_compactoptions_t* opt) { + return opt->rep.change_level; +} + +void rocksdb_compactoptions_set_target_level(rocksdb_compactoptions_t* opt, + int n) { + opt->rep.target_level = n; +} + +int rocksdb_compactoptions_get_target_level(rocksdb_compactoptions_t* opt) { + return opt->rep.target_level; +} + +void rocksdb_compactoptions_set_full_history_ts_low( + rocksdb_compactoptions_t* opt, char* ts, size_t tslen) { + if (ts == nullptr) { + opt->full_history_ts_low = Slice(); + opt->rep.full_history_ts_low = nullptr; + } else { + opt->full_history_ts_low = Slice(ts, tslen); + opt->rep.full_history_ts_low = &opt->full_history_ts_low; + } +} + +rocksdb_flushoptions_t* rocksdb_flushoptions_create() { + return new rocksdb_flushoptions_t; +} + +void rocksdb_flushoptions_destroy(rocksdb_flushoptions_t* opt) { delete opt; } + +void rocksdb_flushoptions_set_wait(rocksdb_flushoptions_t* opt, + unsigned char v) { + opt->rep.wait = v; +} + +unsigned char rocksdb_flushoptions_get_wait(rocksdb_flushoptions_t* opt) { + return opt->rep.wait; +} + +rocksdb_memory_allocator_t* rocksdb_jemalloc_nodump_allocator_create( + char** errptr) { + rocksdb_memory_allocator_t* allocator = new rocksdb_memory_allocator_t; + ROCKSDB_NAMESPACE::JemallocAllocatorOptions options; + SaveError(errptr, ROCKSDB_NAMESPACE::NewJemallocNodumpAllocator( + options, &allocator->rep)); + return allocator; +} + +void rocksdb_memory_allocator_destroy(rocksdb_memory_allocator_t* allocator) { + delete allocator; +} + +rocksdb_lru_cache_options_t* rocksdb_lru_cache_options_create() { + return new rocksdb_lru_cache_options_t; +} + +void rocksdb_lru_cache_options_destroy(rocksdb_lru_cache_options_t* opt) { + delete opt; +} + +void rocksdb_lru_cache_options_set_capacity(rocksdb_lru_cache_options_t* opt, + size_t capacity) { + opt->rep.capacity = capacity; +} + +void rocksdb_lru_cache_options_set_num_shard_bits( + rocksdb_lru_cache_options_t* opt, int num_shard_bits) { + opt->rep.num_shard_bits = num_shard_bits; +} + +void rocksdb_lru_cache_options_set_memory_allocator( + rocksdb_lru_cache_options_t* opt, rocksdb_memory_allocator_t* allocator) { + opt->rep.memory_allocator = allocator->rep; +} + +rocksdb_cache_t* rocksdb_cache_create_lru(size_t capacity) { + rocksdb_cache_t* c = new rocksdb_cache_t; + c->rep = NewLRUCache(capacity); + return c; +} + +rocksdb_cache_t* rocksdb_cache_create_lru_with_strict_capacity_limit( + size_t capacity) { + rocksdb_cache_t* c = new rocksdb_cache_t; + c->rep = NewLRUCache(capacity); + c->rep->SetStrictCapacityLimit(true); + return c; +} + +rocksdb_cache_t* rocksdb_cache_create_lru_opts( + const rocksdb_lru_cache_options_t* opt) { + rocksdb_cache_t* c = new rocksdb_cache_t; + c->rep = NewLRUCache(opt->rep); + return c; +} + +rocksdb_hyper_clock_cache_options_t* rocksdb_hyper_clock_cache_options_create( + size_t capacity, size_t estimated_entry_charge) { + return new rocksdb_hyper_clock_cache_options_t{ + HyperClockCacheOptions(capacity, estimated_entry_charge)}; +} + +void rocksdb_hyper_clock_cache_options_destroy( + rocksdb_hyper_clock_cache_options_t* opt) { + delete opt; +} + +void rocksdb_hyper_clock_cache_options_set_capacity( + rocksdb_hyper_clock_cache_options_t* opts, size_t capacity) { + opts->rep.capacity = capacity; +} + +void rocksdb_hyper_clock_cache_options_set_estimated_entry_charge( + rocksdb_hyper_clock_cache_options_t* opts, size_t estimated_entry_charge) { + opts->rep.estimated_entry_charge = estimated_entry_charge; +} + +void rocksdb_hyper_clock_cache_options_set_num_shard_bits( + rocksdb_hyper_clock_cache_options_t* opts, int num_shard_bits) { + opts->rep.num_shard_bits = num_shard_bits; +} + +void rocksdb_hyper_clock_cache_options_set_memory_allocator( + rocksdb_hyper_clock_cache_options_t* opts, + rocksdb_memory_allocator_t* memory_allocator) { + opts->rep.memory_allocator = memory_allocator->rep; +} + +rocksdb_cache_t* rocksdb_cache_create_hyper_clock( + size_t capacity, size_t estimated_entry_charge) { + HyperClockCacheOptions opts(capacity, estimated_entry_charge); + rocksdb_cache_t* c = new rocksdb_cache_t; + c->rep = opts.MakeSharedCache(); + return c; +} + +rocksdb_cache_t* rocksdb_cache_create_hyper_clock_opts( + const rocksdb_hyper_clock_cache_options_t* opts) { + rocksdb_cache_t* c = new rocksdb_cache_t; + c->rep = opts->rep.MakeSharedCache(); + return c; +} + +void rocksdb_cache_destroy(rocksdb_cache_t* cache) { delete cache; } + +void rocksdb_cache_disown_data(rocksdb_cache_t* cache) { + cache->rep->DisownData(); +} + +void rocksdb_cache_set_capacity(rocksdb_cache_t* cache, size_t capacity) { + cache->rep->SetCapacity(capacity); +} + +size_t rocksdb_cache_get_capacity(const rocksdb_cache_t* cache) { + return cache->rep->GetCapacity(); +} + +size_t rocksdb_cache_get_usage(const rocksdb_cache_t* cache) { + return cache->rep->GetUsage(); +} + +size_t rocksdb_cache_get_pinned_usage(const rocksdb_cache_t* cache) { + return cache->rep->GetPinnedUsage(); +} + +size_t rocksdb_cache_get_table_address_count(const rocksdb_cache_t* cache) { + return cache->rep->GetTableAddressCount(); +} + +size_t rocksdb_cache_get_occupancy_count(const rocksdb_cache_t* cache) { + return cache->rep->GetOccupancyCount(); +} + +rocksdb_dbpath_t* rocksdb_dbpath_create(const char* path, + uint64_t target_size) { + rocksdb_dbpath_t* result = new rocksdb_dbpath_t; + result->rep.path = std::string(path); + result->rep.target_size = target_size; + return result; +} + +void rocksdb_dbpath_destroy(rocksdb_dbpath_t* dbpath) { delete dbpath; } + +rocksdb_env_t* rocksdb_create_default_env() { + rocksdb_env_t* result = new rocksdb_env_t; + result->rep = Env::Default(); + result->is_default = true; + return result; +} + +rocksdb_env_t* rocksdb_create_mem_env() { + rocksdb_env_t* result = new rocksdb_env_t; + result->rep = ROCKSDB_NAMESPACE::NewMemEnv(Env::Default()); + result->is_default = false; + return result; +} + +void rocksdb_env_set_background_threads(rocksdb_env_t* env, int n) { + env->rep->SetBackgroundThreads(n); +} + +int rocksdb_env_get_background_threads(rocksdb_env_t* env) { + return env->rep->GetBackgroundThreads(); +} + +void rocksdb_env_set_bottom_priority_background_threads(rocksdb_env_t* env, + int n) { + env->rep->SetBackgroundThreads(n, Env::BOTTOM); +} + +int rocksdb_env_get_bottom_priority_background_threads(rocksdb_env_t* env) { + return env->rep->GetBackgroundThreads(Env::BOTTOM); +} + +void rocksdb_env_set_high_priority_background_threads(rocksdb_env_t* env, + int n) { + env->rep->SetBackgroundThreads(n, Env::HIGH); +} + +int rocksdb_env_get_high_priority_background_threads(rocksdb_env_t* env) { + return env->rep->GetBackgroundThreads(Env::HIGH); +} + +void rocksdb_env_set_low_priority_background_threads(rocksdb_env_t* env, + int n) { + env->rep->SetBackgroundThreads(n, Env::LOW); +} + +int rocksdb_env_get_low_priority_background_threads(rocksdb_env_t* env) { + return env->rep->GetBackgroundThreads(Env::LOW); +} + +void rocksdb_env_join_all_threads(rocksdb_env_t* env) { + env->rep->WaitForJoin(); +} + +void rocksdb_env_lower_thread_pool_io_priority(rocksdb_env_t* env) { + env->rep->LowerThreadPoolIOPriority(); +} + +void rocksdb_env_lower_high_priority_thread_pool_io_priority( + rocksdb_env_t* env) { + env->rep->LowerThreadPoolIOPriority(Env::HIGH); +} + +void rocksdb_env_lower_thread_pool_cpu_priority(rocksdb_env_t* env) { + env->rep->LowerThreadPoolCPUPriority(); +} + +void rocksdb_env_lower_high_priority_thread_pool_cpu_priority( + rocksdb_env_t* env) { + env->rep->LowerThreadPoolCPUPriority(Env::HIGH); +} + +void rocksdb_env_destroy(rocksdb_env_t* env) { + if (!env->is_default) delete env->rep; + delete env; +} + +rocksdb_envoptions_t* rocksdb_envoptions_create() { + rocksdb_envoptions_t* opt = new rocksdb_envoptions_t; + return opt; +} + +void rocksdb_envoptions_destroy(rocksdb_envoptions_t* opt) { delete opt; } + +rocksdb_sstfilewriter_t* rocksdb_sstfilewriter_create( + const rocksdb_envoptions_t* env, const rocksdb_options_t* io_options) { + rocksdb_sstfilewriter_t* writer = new rocksdb_sstfilewriter_t; + writer->rep = new SstFileWriter(env->rep, io_options->rep); + return writer; +} + +void rocksdb_create_dir_if_missing(rocksdb_env_t* env, const char* path, + char** errptr) { + SaveError(errptr, env->rep->CreateDirIfMissing(std::string(path))); +} + +rocksdb_sstfilewriter_t* rocksdb_sstfilewriter_create_with_comparator( + const rocksdb_envoptions_t* env, const rocksdb_options_t* io_options, + const rocksdb_comparator_t* /*comparator*/) { + rocksdb_sstfilewriter_t* writer = new rocksdb_sstfilewriter_t; + writer->rep = new SstFileWriter(env->rep, io_options->rep); + return writer; +} + +void rocksdb_sstfilewriter_open(rocksdb_sstfilewriter_t* writer, + const char* name, char** errptr) { + SaveError(errptr, writer->rep->Open(std::string(name))); +} + +void rocksdb_sstfilewriter_add(rocksdb_sstfilewriter_t* writer, const char* key, + size_t keylen, const char* val, size_t vallen, + char** errptr) { + SaveError(errptr, writer->rep->Put(Slice(key, keylen), Slice(val, vallen))); +} + +void rocksdb_sstfilewriter_put(rocksdb_sstfilewriter_t* writer, const char* key, + size_t keylen, const char* val, size_t vallen, + char** errptr) { + SaveError(errptr, writer->rep->Put(Slice(key, keylen), Slice(val, vallen))); +} + +void rocksdb_sstfilewriter_put_with_ts(rocksdb_sstfilewriter_t* writer, + const char* key, size_t keylen, + const char* ts, size_t tslen, + const char* val, size_t vallen, + char** errptr) { + SaveError(errptr, writer->rep->Put(Slice(key, keylen), Slice(ts, tslen), + Slice(val, vallen))); +} + +void rocksdb_sstfilewriter_merge(rocksdb_sstfilewriter_t* writer, + const char* key, size_t keylen, + const char* val, size_t vallen, + char** errptr) { + SaveError(errptr, writer->rep->Merge(Slice(key, keylen), Slice(val, vallen))); +} + +void rocksdb_sstfilewriter_delete(rocksdb_sstfilewriter_t* writer, + const char* key, size_t keylen, + char** errptr) { + SaveError(errptr, writer->rep->Delete(Slice(key, keylen))); +} + +void rocksdb_sstfilewriter_delete_with_ts(rocksdb_sstfilewriter_t* writer, + const char* key, size_t keylen, + const char* ts, size_t tslen, + char** errptr) { + SaveError(errptr, writer->rep->Delete(Slice(key, keylen), Slice(ts, tslen))); +} + +void rocksdb_sstfilewriter_delete_range(rocksdb_sstfilewriter_t* writer, + const char* begin_key, + size_t begin_keylen, + const char* end_key, size_t end_keylen, + char** errptr) { + SaveError(errptr, writer->rep->DeleteRange(Slice(begin_key, begin_keylen), + Slice(end_key, end_keylen))); +} + +void rocksdb_sstfilewriter_finish(rocksdb_sstfilewriter_t* writer, + char** errptr) { + SaveError(errptr, writer->rep->Finish(nullptr)); +} + +void rocksdb_sstfilewriter_file_size(rocksdb_sstfilewriter_t* writer, + uint64_t* file_size) { + *file_size = writer->rep->FileSize(); +} + +void rocksdb_sstfilewriter_destroy(rocksdb_sstfilewriter_t* writer) { + delete writer->rep; + delete writer; +} + +rocksdb_ingestexternalfileoptions_t* +rocksdb_ingestexternalfileoptions_create() { + rocksdb_ingestexternalfileoptions_t* opt = + new rocksdb_ingestexternalfileoptions_t; + return opt; +} + +void rocksdb_ingestexternalfileoptions_set_move_files( + rocksdb_ingestexternalfileoptions_t* opt, unsigned char move_files) { + opt->rep.move_files = move_files; +} + +void rocksdb_ingestexternalfileoptions_set_snapshot_consistency( + rocksdb_ingestexternalfileoptions_t* opt, + unsigned char snapshot_consistency) { + opt->rep.snapshot_consistency = snapshot_consistency; +} + +void rocksdb_ingestexternalfileoptions_set_allow_global_seqno( + rocksdb_ingestexternalfileoptions_t* opt, + unsigned char allow_global_seqno) { + opt->rep.allow_global_seqno = allow_global_seqno; +} + +void rocksdb_ingestexternalfileoptions_set_allow_blocking_flush( + rocksdb_ingestexternalfileoptions_t* opt, + unsigned char allow_blocking_flush) { + opt->rep.allow_blocking_flush = allow_blocking_flush; +} + +void rocksdb_ingestexternalfileoptions_set_ingest_behind( + rocksdb_ingestexternalfileoptions_t* opt, unsigned char ingest_behind) { + opt->rep.ingest_behind = ingest_behind; +} + +void rocksdb_ingestexternalfileoptions_set_fail_if_not_bottommost_level( + rocksdb_ingestexternalfileoptions_t* opt, + unsigned char fail_if_not_bottommost_level) { + opt->rep.fail_if_not_bottommost_level = fail_if_not_bottommost_level; +} + +void rocksdb_ingestexternalfileoptions_destroy( + rocksdb_ingestexternalfileoptions_t* opt) { + delete opt; +} + +void rocksdb_ingest_external_file( + rocksdb_t* db, const char* const* file_list, const size_t list_len, + const rocksdb_ingestexternalfileoptions_t* opt, char** errptr) { + std::vector files(list_len); + for (size_t i = 0; i < list_len; ++i) { + files[i] = std::string(file_list[i]); + } + SaveError(errptr, db->rep->IngestExternalFile(files, opt->rep)); +} + +void rocksdb_ingest_external_file_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* handle, + const char* const* file_list, const size_t list_len, + const rocksdb_ingestexternalfileoptions_t* opt, char** errptr) { + std::vector files(list_len); + for (size_t i = 0; i < list_len; ++i) { + files[i] = std::string(file_list[i]); + } + SaveError(errptr, db->rep->IngestExternalFile(handle->rep, files, opt->rep)); +} + +void rocksdb_try_catch_up_with_primary(rocksdb_t* db, char** errptr) { + SaveError(errptr, db->rep->TryCatchUpWithPrimary()); +} + +rocksdb_slicetransform_t* rocksdb_slicetransform_create( + void* state, void (*destructor)(void*), + char* (*transform)(void*, const char* key, size_t length, + size_t* dst_length), + unsigned char (*in_domain)(void*, const char* key, size_t length), + unsigned char (*in_range)(void*, const char* key, size_t length), + const char* (*name)(void*)) { + rocksdb_slicetransform_t* result = new rocksdb_slicetransform_t; + result->state_ = state; + result->destructor_ = destructor; + result->transform_ = transform; + result->in_domain_ = in_domain; + result->in_range_ = in_range; + result->name_ = name; + return result; +} + +void rocksdb_slicetransform_destroy(rocksdb_slicetransform_t* st) { delete st; } + +struct SliceTransformWrapper : public rocksdb_slicetransform_t { + const SliceTransform* rep_; + ~SliceTransformWrapper() override { delete rep_; } + const char* Name() const override { return rep_->Name(); } + std::string GetId() const override { return rep_->GetId(); } + Slice Transform(const Slice& src) const override { + return rep_->Transform(src); + } + bool InDomain(const Slice& src) const override { return rep_->InDomain(src); } + bool InRange(const Slice& src) const override { return rep_->InRange(src); } + static void DoNothing(void*) {} +}; + +rocksdb_slicetransform_t* rocksdb_slicetransform_create_fixed_prefix( + size_t prefixLen) { + SliceTransformWrapper* wrapper = new SliceTransformWrapper; + wrapper->rep_ = ROCKSDB_NAMESPACE::NewFixedPrefixTransform(prefixLen); + wrapper->state_ = nullptr; + wrapper->destructor_ = &SliceTransformWrapper::DoNothing; + return wrapper; +} + +rocksdb_slicetransform_t* rocksdb_slicetransform_create_noop() { + SliceTransformWrapper* wrapper = new SliceTransformWrapper; + wrapper->rep_ = ROCKSDB_NAMESPACE::NewNoopTransform(); + wrapper->state_ = nullptr; + wrapper->destructor_ = &SliceTransformWrapper::DoNothing; + return wrapper; +} + +rocksdb_universal_compaction_options_t* +rocksdb_universal_compaction_options_create() { + rocksdb_universal_compaction_options_t* result = + new rocksdb_universal_compaction_options_t; + result->rep = new ROCKSDB_NAMESPACE::CompactionOptionsUniversal; + return result; +} + +void rocksdb_universal_compaction_options_set_size_ratio( + rocksdb_universal_compaction_options_t* uco, int ratio) { + uco->rep->size_ratio = ratio; +} + +int rocksdb_universal_compaction_options_get_size_ratio( + rocksdb_universal_compaction_options_t* uco) { + return uco->rep->size_ratio; +} + +void rocksdb_universal_compaction_options_set_min_merge_width( + rocksdb_universal_compaction_options_t* uco, int w) { + uco->rep->min_merge_width = w; +} + +int rocksdb_universal_compaction_options_get_min_merge_width( + rocksdb_universal_compaction_options_t* uco) { + return uco->rep->min_merge_width; +} + +void rocksdb_universal_compaction_options_set_max_merge_width( + rocksdb_universal_compaction_options_t* uco, int w) { + uco->rep->max_merge_width = w; +} + +int rocksdb_universal_compaction_options_get_max_merge_width( + rocksdb_universal_compaction_options_t* uco) { + return uco->rep->max_merge_width; +} + +void rocksdb_universal_compaction_options_set_max_size_amplification_percent( + rocksdb_universal_compaction_options_t* uco, int p) { + uco->rep->max_size_amplification_percent = p; +} + +int rocksdb_universal_compaction_options_get_max_size_amplification_percent( + rocksdb_universal_compaction_options_t* uco) { + return uco->rep->max_size_amplification_percent; +} + +void rocksdb_universal_compaction_options_set_compression_size_percent( + rocksdb_universal_compaction_options_t* uco, int p) { + uco->rep->compression_size_percent = p; +} + +int rocksdb_universal_compaction_options_get_compression_size_percent( + rocksdb_universal_compaction_options_t* uco) { + return uco->rep->compression_size_percent; +} + +void rocksdb_universal_compaction_options_set_stop_style( + rocksdb_universal_compaction_options_t* uco, int style) { + uco->rep->stop_style = + static_cast(style); +} + +int rocksdb_universal_compaction_options_get_stop_style( + rocksdb_universal_compaction_options_t* uco) { + return static_cast(uco->rep->stop_style); +} + +void rocksdb_universal_compaction_options_destroy( + rocksdb_universal_compaction_options_t* uco) { + delete uco->rep; + delete uco; +} + +rocksdb_fifo_compaction_options_t* rocksdb_fifo_compaction_options_create() { + rocksdb_fifo_compaction_options_t* result = + new rocksdb_fifo_compaction_options_t; + result->rep = CompactionOptionsFIFO(); + return result; +} + +void rocksdb_fifo_compaction_options_set_allow_compaction( + rocksdb_fifo_compaction_options_t* fifo_opts, + unsigned char allow_compaction) { + fifo_opts->rep.allow_compaction = allow_compaction; +} + +unsigned char rocksdb_fifo_compaction_options_get_allow_compaction( + rocksdb_fifo_compaction_options_t* fifo_opts) { + return fifo_opts->rep.allow_compaction; +} + +void rocksdb_fifo_compaction_options_set_max_table_files_size( + rocksdb_fifo_compaction_options_t* fifo_opts, uint64_t size) { + fifo_opts->rep.max_table_files_size = size; +} + +uint64_t rocksdb_fifo_compaction_options_get_max_table_files_size( + rocksdb_fifo_compaction_options_t* fifo_opts) { + return fifo_opts->rep.max_table_files_size; +} + +void rocksdb_fifo_compaction_options_destroy( + rocksdb_fifo_compaction_options_t* fifo_opts) { + delete fifo_opts; +} + +void rocksdb_options_set_min_level_to_compress(rocksdb_options_t* opt, + int level) { + if (level >= 0) { + assert(level <= opt->rep.num_levels); + opt->rep.compression_per_level.resize(opt->rep.num_levels); + for (int i = 0; i < level; i++) { + opt->rep.compression_per_level[i] = ROCKSDB_NAMESPACE::kNoCompression; + } + for (int i = level; i < opt->rep.num_levels; i++) { + opt->rep.compression_per_level[i] = opt->rep.compression; + } + } +} + +int rocksdb_livefiles_count(const rocksdb_livefiles_t* lf) { + return static_cast(lf->rep.size()); +} + +const char* rocksdb_livefiles_column_family_name(const rocksdb_livefiles_t* lf, + int index) { + return lf->rep[index].column_family_name.c_str(); +} + +const char* rocksdb_livefiles_name(const rocksdb_livefiles_t* lf, int index) { + return lf->rep[index].name.c_str(); +} + +int rocksdb_livefiles_level(const rocksdb_livefiles_t* lf, int index) { + return lf->rep[index].level; +} + +size_t rocksdb_livefiles_size(const rocksdb_livefiles_t* lf, int index) { + return lf->rep[index].size; +} + +const char* rocksdb_livefiles_smallestkey(const rocksdb_livefiles_t* lf, + int index, size_t* size) { + *size = lf->rep[index].smallestkey.size(); + return lf->rep[index].smallestkey.data(); +} + +const char* rocksdb_livefiles_largestkey(const rocksdb_livefiles_t* lf, + int index, size_t* size) { + *size = lf->rep[index].largestkey.size(); + return lf->rep[index].largestkey.data(); +} + +uint64_t rocksdb_livefiles_entries(const rocksdb_livefiles_t* lf, int index) { + return lf->rep[index].num_entries; +} + +uint64_t rocksdb_livefiles_deletions(const rocksdb_livefiles_t* lf, int index) { + return lf->rep[index].num_deletions; +} + +extern void rocksdb_livefiles_destroy(const rocksdb_livefiles_t* lf) { + delete lf; +} + +void rocksdb_get_options_from_string(const rocksdb_options_t* base_options, + const char* opts_str, + rocksdb_options_t* new_options, + char** errptr) { + SaveError(errptr, + GetOptionsFromString(base_options->rep, std::string(opts_str), + &new_options->rep)); +} + +void rocksdb_delete_file_in_range(rocksdb_t* db, const char* start_key, + size_t start_key_len, const char* limit_key, + size_t limit_key_len, char** errptr) { + Slice a, b; + SaveError( + errptr, + DeleteFilesInRange( + db->rep, db->rep->DefaultColumnFamily(), + (start_key ? (a = Slice(start_key, start_key_len), &a) : nullptr), + (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : nullptr))); +} + +void rocksdb_delete_file_in_range_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* limit_key, + size_t limit_key_len, char** errptr) { + Slice a, b; + SaveError( + errptr, + DeleteFilesInRange( + db->rep, column_family->rep, + (start_key ? (a = Slice(start_key, start_key_len), &a) : nullptr), + (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : nullptr))); +} + +/* MetaData */ + +rocksdb_column_family_metadata_t* rocksdb_get_column_family_metadata( + rocksdb_t* db) { + rocksdb_column_family_metadata_t* meta = new rocksdb_column_family_metadata_t; + db->rep->GetColumnFamilyMetaData(&meta->rep); + return meta; +} + +rocksdb_column_family_metadata_t* rocksdb_get_column_family_metadata_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family) { + rocksdb_column_family_metadata_t* meta = new rocksdb_column_family_metadata_t; + db->rep->GetColumnFamilyMetaData(column_family->rep, &meta->rep); + return meta; +} + +void rocksdb_column_family_metadata_destroy( + rocksdb_column_family_metadata_t* cf_meta) { + delete cf_meta; +} + +uint64_t rocksdb_column_family_metadata_get_size( + rocksdb_column_family_metadata_t* cf_meta) { + return cf_meta->rep.size; +} + +size_t rocksdb_column_family_metadata_get_file_count( + rocksdb_column_family_metadata_t* cf_meta) { + return cf_meta->rep.file_count; +} + +char* rocksdb_column_family_metadata_get_name( + rocksdb_column_family_metadata_t* cf_meta) { + return strdup(cf_meta->rep.name.c_str()); +} + +size_t rocksdb_column_family_metadata_get_level_count( + rocksdb_column_family_metadata_t* cf_meta) { + return cf_meta->rep.levels.size(); +} + +rocksdb_level_metadata_t* rocksdb_column_family_metadata_get_level_metadata( + rocksdb_column_family_metadata_t* cf_meta, size_t i) { + if (i >= cf_meta->rep.levels.size()) { + return NULL; + } + rocksdb_level_metadata_t* level_meta = + (rocksdb_level_metadata_t*)malloc(sizeof(rocksdb_level_metadata_t)); + level_meta->rep = &cf_meta->rep.levels[i]; + + return level_meta; +} + +void rocksdb_level_metadata_destroy(rocksdb_level_metadata_t* level_meta) { + // Only free the base pointer as its parent rocksdb_column_family_metadata_t + // has the ownership of its rep. + free(level_meta); +} + +int rocksdb_level_metadata_get_level(rocksdb_level_metadata_t* level_meta) { + return level_meta->rep->level; +} + +uint64_t rocksdb_level_metadata_get_size(rocksdb_level_metadata_t* level_meta) { + return level_meta->rep->size; +} + +size_t rocksdb_level_metadata_get_file_count( + rocksdb_level_metadata_t* level_meta) { + return level_meta->rep->files.size(); +} + +rocksdb_sst_file_metadata_t* rocksdb_level_metadata_get_sst_file_metadata( + rocksdb_level_metadata_t* level_meta, size_t i) { + if (i >= level_meta->rep->files.size()) { + return nullptr; + } + rocksdb_sst_file_metadata_t* file_meta = + (rocksdb_sst_file_metadata_t*)malloc(sizeof(rocksdb_sst_file_metadata_t)); + file_meta->rep = &level_meta->rep->files[i]; + return file_meta; +} + +void rocksdb_sst_file_metadata_destroy(rocksdb_sst_file_metadata_t* file_meta) { + // Only free the base pointer as its parent rocksdb_level_metadata_t + // has the ownership of its rep. + free(file_meta); +} + +char* rocksdb_sst_file_metadata_get_relative_filename( + rocksdb_sst_file_metadata_t* file_meta) { + return strdup(file_meta->rep->relative_filename.c_str()); +} + +uint64_t rocksdb_sst_file_metadata_get_size( + rocksdb_sst_file_metadata_t* file_meta) { + return file_meta->rep->size; +} + +char* rocksdb_sst_file_metadata_get_smallestkey( + rocksdb_sst_file_metadata_t* file_meta, size_t* key_len) { + *key_len = file_meta->rep->smallestkey.size(); + return CopyString(file_meta->rep->smallestkey); +} + +char* rocksdb_sst_file_metadata_get_largestkey( + rocksdb_sst_file_metadata_t* file_meta, size_t* key_len) { + *key_len = file_meta->rep->largestkey.size(); + return CopyString(file_meta->rep->largestkey); +} + +/* Transactions */ + +rocksdb_transactiondb_options_t* rocksdb_transactiondb_options_create() { + return new rocksdb_transactiondb_options_t; +} + +void rocksdb_transactiondb_options_destroy( + rocksdb_transactiondb_options_t* opt) { + delete opt; +} + +void rocksdb_transactiondb_options_set_max_num_locks( + rocksdb_transactiondb_options_t* opt, int64_t max_num_locks) { + opt->rep.max_num_locks = max_num_locks; +} + +void rocksdb_transactiondb_options_set_num_stripes( + rocksdb_transactiondb_options_t* opt, size_t num_stripes) { + opt->rep.num_stripes = num_stripes; +} + +void rocksdb_transactiondb_options_set_transaction_lock_timeout( + rocksdb_transactiondb_options_t* opt, int64_t txn_lock_timeout) { + opt->rep.transaction_lock_timeout = txn_lock_timeout; +} + +void rocksdb_transactiondb_options_set_default_lock_timeout( + rocksdb_transactiondb_options_t* opt, int64_t default_lock_timeout) { + opt->rep.default_lock_timeout = default_lock_timeout; +} + +rocksdb_transaction_options_t* rocksdb_transaction_options_create() { + return new rocksdb_transaction_options_t; +} + +void rocksdb_transaction_options_destroy(rocksdb_transaction_options_t* opt) { + delete opt; +} + +void rocksdb_transaction_options_set_set_snapshot( + rocksdb_transaction_options_t* opt, unsigned char v) { + opt->rep.set_snapshot = v; +} + +void rocksdb_transaction_options_set_deadlock_detect( + rocksdb_transaction_options_t* opt, unsigned char v) { + opt->rep.deadlock_detect = v; +} + +void rocksdb_transaction_options_set_lock_timeout( + rocksdb_transaction_options_t* opt, int64_t lock_timeout) { + opt->rep.lock_timeout = lock_timeout; +} + +void rocksdb_transaction_options_set_expiration( + rocksdb_transaction_options_t* opt, int64_t expiration) { + opt->rep.expiration = expiration; +} + +void rocksdb_transaction_options_set_deadlock_detect_depth( + rocksdb_transaction_options_t* opt, int64_t depth) { + opt->rep.deadlock_detect_depth = depth; +} + +void rocksdb_transaction_options_set_max_write_batch_size( + rocksdb_transaction_options_t* opt, size_t size) { + opt->rep.max_write_batch_size = size; +} + +void rocksdb_transaction_options_set_skip_prepare( + rocksdb_transaction_options_t* opt, unsigned char v) { + opt->rep.skip_prepare = v; +} + +rocksdb_optimistictransaction_options_t* +rocksdb_optimistictransaction_options_create() { + return new rocksdb_optimistictransaction_options_t; +} + +void rocksdb_optimistictransaction_options_destroy( + rocksdb_optimistictransaction_options_t* opt) { + delete opt; +} + +void rocksdb_optimistictransaction_options_set_set_snapshot( + rocksdb_optimistictransaction_options_t* opt, unsigned char v) { + opt->rep.set_snapshot = v; +} + +char* rocksdb_optimistictransactiondb_property_value( + rocksdb_optimistictransactiondb_t* db, const char* propname) { + std::string tmp; + if (db->rep->GetProperty(Slice(propname), &tmp)) { + // We use strdup() since we expect human readable output. + return strdup(tmp.c_str()); + } else { + return nullptr; + } +} + +int rocksdb_optimistictransactiondb_property_int( + rocksdb_optimistictransactiondb_t* db, const char* propname, + uint64_t* out_val) { + if (db->rep->GetIntProperty(Slice(propname), out_val)) { + return 0; + } else { + return -1; + } +} + +rocksdb_column_family_handle_t* rocksdb_transactiondb_create_column_family( + rocksdb_transactiondb_t* txn_db, + const rocksdb_options_t* column_family_options, + const char* column_family_name, char** errptr) { + rocksdb_column_family_handle_t* handle = new rocksdb_column_family_handle_t; + SaveError(errptr, txn_db->rep->CreateColumnFamily( + ColumnFamilyOptions(column_family_options->rep), + std::string(column_family_name), &(handle->rep))); + return handle; +} + +rocksdb_transactiondb_t* rocksdb_transactiondb_open( + const rocksdb_options_t* options, + const rocksdb_transactiondb_options_t* txn_db_options, const char* name, + char** errptr) { + TransactionDB* txn_db; + if (SaveError(errptr, TransactionDB::Open(options->rep, txn_db_options->rep, + std::string(name), &txn_db))) { + return nullptr; + } + rocksdb_transactiondb_t* result = new rocksdb_transactiondb_t; + result->rep = txn_db; + return result; +} + +rocksdb_transactiondb_t* rocksdb_transactiondb_open_column_families( + const rocksdb_options_t* options, + const rocksdb_transactiondb_options_t* txn_db_options, const char* name, + int num_column_families, const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char** errptr) { + std::vector column_families; + for (int i = 0; i < num_column_families; i++) { + column_families.push_back(ColumnFamilyDescriptor( + std::string(column_family_names[i]), + ColumnFamilyOptions(column_family_options[i]->rep))); + } + + TransactionDB* txn_db; + std::vector handles; + if (SaveError(errptr, TransactionDB::Open(options->rep, txn_db_options->rep, + std::string(name), column_families, + &handles, &txn_db))) { + return nullptr; + } + + for (size_t i = 0; i < handles.size(); i++) { + rocksdb_column_family_handle_t* c_handle = + new rocksdb_column_family_handle_t; + c_handle->rep = handles[i]; + column_family_handles[i] = c_handle; + } + rocksdb_transactiondb_t* result = new rocksdb_transactiondb_t; + result->rep = txn_db; + return result; +} + +const rocksdb_snapshot_t* rocksdb_transactiondb_create_snapshot( + rocksdb_transactiondb_t* txn_db) { + rocksdb_snapshot_t* result = new rocksdb_snapshot_t; + result->rep = txn_db->rep->GetSnapshot(); + return result; +} + +void rocksdb_transactiondb_release_snapshot( + rocksdb_transactiondb_t* txn_db, const rocksdb_snapshot_t* snapshot) { + txn_db->rep->ReleaseSnapshot(snapshot->rep); + delete snapshot; +} + +char* rocksdb_transactiondb_property_value(rocksdb_transactiondb_t* db, + const char* propname) { + std::string tmp; + if (db->rep->GetProperty(Slice(propname), &tmp)) { + // We use strdup() since we expect human readable output. + return strdup(tmp.c_str()); + } else { + return nullptr; + } +} + +int rocksdb_transactiondb_property_int(rocksdb_transactiondb_t* db, + const char* propname, + uint64_t* out_val) { + if (db->rep->GetIntProperty(Slice(propname), out_val)) { + return 0; + } else { + return -1; + } +} + +rocksdb_t* rocksdb_transactiondb_get_base_db(rocksdb_transactiondb_t* txn_db) { + DB* base_db = txn_db->rep->GetBaseDB(); + + if (base_db != nullptr) { + rocksdb_t* result = new rocksdb_t; + result->rep = base_db; + return result; + } + + return nullptr; +} + +void rocksdb_transactiondb_close_base_db(rocksdb_t* base_db) { delete base_db; } + +rocksdb_transaction_t* rocksdb_transaction_begin( + rocksdb_transactiondb_t* txn_db, + const rocksdb_writeoptions_t* write_options, + const rocksdb_transaction_options_t* txn_options, + rocksdb_transaction_t* old_txn) { + if (old_txn == nullptr) { + rocksdb_transaction_t* result = new rocksdb_transaction_t; + result->rep = txn_db->rep->BeginTransaction(write_options->rep, + txn_options->rep, nullptr); + return result; + } + old_txn->rep = txn_db->rep->BeginTransaction(write_options->rep, + txn_options->rep, old_txn->rep); + return old_txn; +} + +rocksdb_transaction_t** rocksdb_transactiondb_get_prepared_transactions( + rocksdb_transactiondb_t* txn_db, size_t* cnt) { + std::vector txns; + txn_db->rep->GetAllPreparedTransactions(&txns); + *cnt = txns.size(); + if (txns.empty()) { + return nullptr; + } else { + rocksdb_transaction_t** buf = (rocksdb_transaction_t**)malloc( + txns.size() * sizeof(rocksdb_transaction_t*)); + for (size_t i = 0; i < txns.size(); i++) { + buf[i] = new rocksdb_transaction_t; + buf[i]->rep = txns[i]; + } + return buf; + } +} + +void rocksdb_transaction_set_name(rocksdb_transaction_t* txn, const char* name, + size_t name_len, char** errptr) { + std::string str = std::string(name, name_len); + SaveError(errptr, txn->rep->SetName(str)); +} + +char* rocksdb_transaction_get_name(rocksdb_transaction_t* txn, + size_t* name_len) { + auto name = txn->rep->GetName(); + *name_len = name.size(); + return CopyString(name); +} + +void rocksdb_transaction_prepare(rocksdb_transaction_t* txn, char** errptr) { + SaveError(errptr, txn->rep->Prepare()); +} + +rocksdb_writebatch_wi_t* rocksdb_transaction_get_writebatch_wi( + rocksdb_transaction_t* txn) { + rocksdb_writebatch_wi_t* wi = + (rocksdb_writebatch_wi_t*)malloc(sizeof(rocksdb_writebatch_wi_t)); + wi->rep = txn->rep->GetWriteBatch(); + + return wi; +} + +void rocksdb_transaction_rebuild_from_writebatch( + rocksdb_transaction_t* txn, rocksdb_writebatch_t* writebatch, + char** errptr) { + SaveError(errptr, txn->rep->RebuildFromWriteBatch(&writebatch->rep)); +} + +void rocksdb_transaction_rebuild_from_writebatch_wi(rocksdb_transaction_t* txn, + rocksdb_writebatch_wi_t* wi, + char** errptr) { + SaveError(errptr, txn->rep->RebuildFromWriteBatch(wi->rep->GetWriteBatch())); +} + +void rocksdb_transaction_commit(rocksdb_transaction_t* txn, char** errptr) { + SaveError(errptr, txn->rep->Commit()); +} + +void rocksdb_transaction_rollback(rocksdb_transaction_t* txn, char** errptr) { + SaveError(errptr, txn->rep->Rollback()); +} + +void rocksdb_transaction_set_savepoint(rocksdb_transaction_t* txn) { + txn->rep->SetSavePoint(); +} + +void rocksdb_transaction_rollback_to_savepoint(rocksdb_transaction_t* txn, + char** errptr) { + SaveError(errptr, txn->rep->RollbackToSavePoint()); +} + +void rocksdb_transaction_destroy(rocksdb_transaction_t* txn) { + delete txn->rep; + delete txn; +} + +const rocksdb_snapshot_t* rocksdb_transaction_get_snapshot( + rocksdb_transaction_t* txn) { + // This will be freed later on using free, so use malloc here to avoid a + // mismatch + rocksdb_snapshot_t* result = + (rocksdb_snapshot_t*)malloc(sizeof(rocksdb_snapshot_t)); + result->rep = txn->rep->GetSnapshot(); + return result; +} + +// Read a key inside a transaction +char* rocksdb_transaction_get(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options, + const char* key, size_t klen, size_t* vlen, + char** errptr) { + char* result = nullptr; + std::string tmp; + Status s = txn->rep->Get(options->rep, Slice(key, klen), &tmp); + if (s.ok()) { + *vlen = tmp.size(); + result = CopyString(tmp); + } else { + *vlen = 0; + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + } + return result; +} + +rocksdb_pinnableslice_t* rocksdb_transaction_get_pinned( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + const char* key, size_t klen, char** errptr) { + rocksdb_pinnableslice_t* v = new (rocksdb_pinnableslice_t); + Status s = txn->rep->Get(options->rep, Slice(key, klen), &v->rep); + if (!s.ok()) { + delete (v); + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + return nullptr; + } + return v; +} + +char* rocksdb_transaction_get_cf(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, size_t* vlen, + char** errptr) { + char* result = nullptr; + std::string tmp; + Status s = + txn->rep->Get(options->rep, column_family->rep, Slice(key, klen), &tmp); + if (s.ok()) { + *vlen = tmp.size(); + result = CopyString(tmp); + } else { + *vlen = 0; + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + } + return result; +} + +rocksdb_pinnableslice_t* rocksdb_transaction_get_pinned_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, size_t klen, + char** errptr) { + rocksdb_pinnableslice_t* v = new (rocksdb_pinnableslice_t); + Status s = txn->rep->Get(options->rep, column_family->rep, Slice(key, klen), + &v->rep); + if (!s.ok()) { + delete (v); + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + return nullptr; + } + return v; +} + +// Read a key inside a transaction +char* rocksdb_transaction_get_for_update(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options, + const char* key, size_t klen, + size_t* vlen, unsigned char exclusive, + char** errptr) { + char* result = nullptr; + std::string tmp; + Status s = + txn->rep->GetForUpdate(options->rep, Slice(key, klen), &tmp, exclusive); + if (s.ok()) { + *vlen = tmp.size(); + result = CopyString(tmp); + } else { + *vlen = 0; + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + } + return result; +} + +rocksdb_pinnableslice_t* rocksdb_transaction_get_pinned_for_update( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + const char* key, size_t klen, unsigned char exclusive, char** errptr) { + rocksdb_pinnableslice_t* v = new (rocksdb_pinnableslice_t); + Status s = txn->rep->GetForUpdate(options->rep, Slice(key, klen), + v->rep.GetSelf(), exclusive); + v->rep.PinSelf(); + if (!s.ok()) { + delete (v); + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + return nullptr; + } + return v; +} + +char* rocksdb_transaction_get_for_update_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, size_t klen, + size_t* vlen, unsigned char exclusive, char** errptr) { + char* result = nullptr; + std::string tmp; + Status s = txn->rep->GetForUpdate(options->rep, column_family->rep, + Slice(key, klen), &tmp, exclusive); + if (s.ok()) { + *vlen = tmp.size(); + result = CopyString(tmp); + } else { + *vlen = 0; + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + } + return result; +} + +rocksdb_pinnableslice_t* rocksdb_transaction_get_pinned_for_update_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, size_t klen, + unsigned char exclusive, char** errptr) { + rocksdb_pinnableslice_t* v = new (rocksdb_pinnableslice_t); + Status s = txn->rep->GetForUpdate(options->rep, column_family->rep, + Slice(key, klen), &v->rep, exclusive); + if (!s.ok()) { + delete (v); + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + return nullptr; + } + return v; +} + +void rocksdb_transaction_multi_get(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options, + size_t num_keys, + const char* const* keys_list, + const size_t* keys_list_sizes, + char** values_list, + size_t* values_list_sizes, char** errs) { + std::vector keys(num_keys); + for (size_t i = 0; i < num_keys; i++) { + keys[i] = Slice(keys_list[i], keys_list_sizes[i]); + } + std::vector values(num_keys); + std::vector statuses = + txn->rep->MultiGet(options->rep, keys, &values); + for (size_t i = 0; i < num_keys; i++) { + if (statuses[i].ok()) { + values_list[i] = CopyString(values[i]); + values_list_sizes[i] = values[i].size(); + errs[i] = nullptr; + } else { + values_list[i] = nullptr; + values_list_sizes[i] = 0; + if (!statuses[i].IsNotFound()) { + errs[i] = strdup(statuses[i].ToString().c_str()); + } else { + errs[i] = nullptr; + } + } + } +} + +void rocksdb_transaction_multi_get_for_update( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs) { + std::vector keys(num_keys); + for (size_t i = 0; i < num_keys; i++) { + keys[i] = Slice(keys_list[i], keys_list_sizes[i]); + } + std::vector values(num_keys); + std::vector statuses = + txn->rep->MultiGetForUpdate(options->rep, keys, &values); + for (size_t i = 0; i < num_keys; i++) { + if (statuses[i].ok()) { + values_list[i] = CopyString(values[i]); + values_list_sizes[i] = values[i].size(); + errs[i] = nullptr; + } else { + values_list[i] = nullptr; + values_list_sizes[i] = 0; + if (!statuses[i].IsNotFound()) { + errs[i] = strdup(statuses[i].ToString().c_str()); + } else { + errs[i] = nullptr; + } + } + } +} + +void rocksdb_transaction_multi_get_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + const rocksdb_column_family_handle_t* const* column_families, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs) { + std::vector keys(num_keys); + std::vector cfs(num_keys); + for (size_t i = 0; i < num_keys; i++) { + keys[i] = Slice(keys_list[i], keys_list_sizes[i]); + cfs[i] = column_families[i]->rep; + } + std::vector values(num_keys); + std::vector statuses = + txn->rep->MultiGet(options->rep, cfs, keys, &values); + for (size_t i = 0; i < num_keys; i++) { + if (statuses[i].ok()) { + values_list[i] = CopyString(values[i]); + values_list_sizes[i] = values[i].size(); + errs[i] = nullptr; + } else { + values_list[i] = nullptr; + values_list_sizes[i] = 0; + if (!statuses[i].IsNotFound()) { + errs[i] = strdup(statuses[i].ToString().c_str()); + } else { + errs[i] = nullptr; + } + } + } +} + +void rocksdb_transaction_multi_get_for_update_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + const rocksdb_column_family_handle_t* const* column_families, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs) { + std::vector keys(num_keys); + std::vector cfs(num_keys); + for (size_t i = 0; i < num_keys; i++) { + keys[i] = Slice(keys_list[i], keys_list_sizes[i]); + cfs[i] = column_families[i]->rep; + } + std::vector values(num_keys); + std::vector statuses = + txn->rep->MultiGetForUpdate(options->rep, cfs, keys, &values); + for (size_t i = 0; i < num_keys; i++) { + if (statuses[i].ok()) { + values_list[i] = CopyString(values[i]); + values_list_sizes[i] = values[i].size(); + errs[i] = nullptr; + } else { + values_list[i] = nullptr; + values_list_sizes[i] = 0; + if (!statuses[i].IsNotFound()) { + errs[i] = strdup(statuses[i].ToString().c_str()); + } else { + errs[i] = nullptr; + } + } + } +} + +// Read a key outside a transaction +char* rocksdb_transactiondb_get(rocksdb_transactiondb_t* txn_db, + const rocksdb_readoptions_t* options, + const char* key, size_t klen, size_t* vlen, + char** errptr) { + char* result = nullptr; + std::string tmp; + Status s = txn_db->rep->Get(options->rep, Slice(key, klen), &tmp); + if (s.ok()) { + *vlen = tmp.size(); + result = CopyString(tmp); + } else { + *vlen = 0; + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + } + return result; +} + +rocksdb_pinnableslice_t* rocksdb_transactiondb_get_pinned( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + const char* key, size_t klen, char** errptr) { + rocksdb_pinnableslice_t* v = new (rocksdb_pinnableslice_t); + Status s = txn_db->rep->Get(options->rep, txn_db->rep->DefaultColumnFamily(), + Slice(key, klen), &v->rep); + if (!s.ok()) { + delete (v); + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + return nullptr; + } + return v; +} + +char* rocksdb_transactiondb_get_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** errptr) { + char* result = nullptr; + std::string tmp; + Status s = txn_db->rep->Get(options->rep, column_family->rep, + Slice(key, keylen), &tmp); + if (s.ok()) { + *vallen = tmp.size(); + result = CopyString(tmp); + } else { + *vallen = 0; + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + } + return result; +} + +rocksdb_pinnableslice_t* rocksdb_transactiondb_get_pinned_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr) { + rocksdb_pinnableslice_t* v = new (rocksdb_pinnableslice_t); + Status s = txn_db->rep->Get(options->rep, column_family->rep, + Slice(key, keylen), &v->rep); + if (!s.ok()) { + delete (v); + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + return nullptr; + } + return v; +} + +void rocksdb_transactiondb_multi_get(rocksdb_transactiondb_t* txn_db, + const rocksdb_readoptions_t* options, + size_t num_keys, + const char* const* keys_list, + const size_t* keys_list_sizes, + char** values_list, + size_t* values_list_sizes, char** errs) { + std::vector keys(num_keys); + for (size_t i = 0; i < num_keys; i++) { + keys[i] = Slice(keys_list[i], keys_list_sizes[i]); + } + std::vector values(num_keys); + std::vector statuses = + txn_db->rep->MultiGet(options->rep, keys, &values); + for (size_t i = 0; i < num_keys; i++) { + if (statuses[i].ok()) { + values_list[i] = CopyString(values[i]); + values_list_sizes[i] = values[i].size(); + errs[i] = nullptr; + } else { + values_list[i] = nullptr; + values_list_sizes[i] = 0; + if (!statuses[i].IsNotFound()) { + errs[i] = strdup(statuses[i].ToString().c_str()); + } else { + errs[i] = nullptr; + } + } + } +} + +void rocksdb_transactiondb_multi_get_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + const rocksdb_column_family_handle_t* const* column_families, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs) { + std::vector keys(num_keys); + std::vector cfs(num_keys); + for (size_t i = 0; i < num_keys; i++) { + keys[i] = Slice(keys_list[i], keys_list_sizes[i]); + cfs[i] = column_families[i]->rep; + } + std::vector values(num_keys); + std::vector statuses = + txn_db->rep->MultiGet(options->rep, cfs, keys, &values); + for (size_t i = 0; i < num_keys; i++) { + if (statuses[i].ok()) { + values_list[i] = CopyString(values[i]); + values_list_sizes[i] = values[i].size(); + errs[i] = nullptr; + } else { + values_list[i] = nullptr; + values_list_sizes[i] = 0; + if (!statuses[i].IsNotFound()) { + errs[i] = strdup(statuses[i].ToString().c_str()); + } else { + errs[i] = nullptr; + } + } + } +} + +// Put a key inside a transaction +void rocksdb_transaction_put(rocksdb_transaction_t* txn, const char* key, + size_t klen, const char* val, size_t vlen, + char** errptr) { + SaveError(errptr, txn->rep->Put(Slice(key, klen), Slice(val, vlen))); +} + +void rocksdb_transaction_put_cf(rocksdb_transaction_t* txn, + rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, + size_t vlen, char** errptr) { + SaveError(errptr, txn->rep->Put(column_family->rep, Slice(key, klen), + Slice(val, vlen))); +} + +void rocksdb_transaction_set_commit_timestamp(rocksdb_transaction_t* txn, + uint64_t commit_timestamp) { + txn->rep->SetCommitTimestamp(commit_timestamp); +} + +void rocksdb_transaction_set_read_timestamp_for_validation( + rocksdb_transaction_t* txn, uint64_t read_timestamp) { + txn->rep->SetReadTimestampForValidation(read_timestamp); +} + +// Put a key outside a transaction +void rocksdb_transactiondb_put(rocksdb_transactiondb_t* txn_db, + const rocksdb_writeoptions_t* options, + const char* key, size_t klen, const char* val, + size_t vlen, char** errptr) { + SaveError(errptr, + txn_db->rep->Put(options->rep, Slice(key, klen), Slice(val, vlen))); +} + +void rocksdb_transactiondb_put_cf(rocksdb_transactiondb_t* txn_db, + const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, + const char* key, size_t keylen, + const char* val, size_t vallen, + char** errptr) { + SaveError(errptr, txn_db->rep->Put(options->rep, column_family->rep, + Slice(key, keylen), Slice(val, vallen))); +} + +// Write batch into transaction db +void rocksdb_transactiondb_write(rocksdb_transactiondb_t* db, + const rocksdb_writeoptions_t* options, + rocksdb_writebatch_t* batch, char** errptr) { + SaveError(errptr, db->rep->Write(options->rep, &batch->rep)); +} + +// Merge a key inside a transaction +void rocksdb_transaction_merge(rocksdb_transaction_t* txn, const char* key, + size_t klen, const char* val, size_t vlen, + char** errptr) { + SaveError(errptr, txn->rep->Merge(Slice(key, klen), Slice(val, vlen))); +} + +void rocksdb_transaction_merge_cf(rocksdb_transaction_t* txn, + rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, + size_t vlen, char** errptr) { + SaveError(errptr, txn->rep->Merge(column_family->rep, Slice(key, klen), + Slice(val, vlen))); +} + +// Merge a key outside a transaction +void rocksdb_transactiondb_merge(rocksdb_transactiondb_t* txn_db, + const rocksdb_writeoptions_t* options, + const char* key, size_t klen, const char* val, + size_t vlen, char** errptr) { + SaveError(errptr, txn_db->rep->Merge(options->rep, Slice(key, klen), + Slice(val, vlen))); +} + +void rocksdb_transactiondb_merge_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, size_t klen, + const char* val, size_t vlen, char** errptr) { + SaveError(errptr, txn_db->rep->Merge(options->rep, column_family->rep, + Slice(key, klen), Slice(val, vlen))); +} + +// Delete a key inside a transaction +void rocksdb_transaction_delete(rocksdb_transaction_t* txn, const char* key, + size_t klen, char** errptr) { + SaveError(errptr, txn->rep->Delete(Slice(key, klen))); +} + +void rocksdb_transaction_delete_cf( + rocksdb_transaction_t* txn, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, char** errptr) { + SaveError(errptr, txn->rep->Delete(column_family->rep, Slice(key, klen))); +} + +// Delete a key outside a transaction +void rocksdb_transactiondb_delete(rocksdb_transactiondb_t* txn_db, + const rocksdb_writeoptions_t* options, + const char* key, size_t klen, char** errptr) { + SaveError(errptr, txn_db->rep->Delete(options->rep, Slice(key, klen))); +} + +void rocksdb_transactiondb_delete_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr) { + SaveError(errptr, txn_db->rep->Delete(options->rep, column_family->rep, + Slice(key, keylen))); +} + +// Create an iterator inside a transaction +rocksdb_iterator_t* rocksdb_transaction_create_iterator( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options) { + rocksdb_iterator_t* result = new rocksdb_iterator_t; + result->rep = txn->rep->GetIterator(options->rep); + return result; +} + +// Create an iterator inside a transaction with column family +rocksdb_iterator_t* rocksdb_transaction_create_iterator_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family) { + rocksdb_iterator_t* result = new rocksdb_iterator_t; + result->rep = txn->rep->GetIterator(options->rep, column_family->rep); + return result; +} + +// Create an iterator outside a transaction +rocksdb_iterator_t* rocksdb_transactiondb_create_iterator( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options) { + rocksdb_iterator_t* result = new rocksdb_iterator_t; + result->rep = txn_db->rep->NewIterator(options->rep); + return result; +} + +rocksdb_iterator_t* rocksdb_transactiondb_create_iterator_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family) { + rocksdb_iterator_t* result = new rocksdb_iterator_t; + result->rep = txn_db->rep->NewIterator(options->rep, column_family->rep); + return result; +} + +void rocksdb_transactiondb_close(rocksdb_transactiondb_t* txn_db) { + delete txn_db->rep; + delete txn_db; +} + +void rocksdb_transactiondb_flush_wal(rocksdb_transactiondb_t* txn_db, + unsigned char sync, char** errptr) { + SaveError(errptr, txn_db->rep->FlushWAL(sync)); +} + +void rocksdb_transactiondb_flush(rocksdb_transactiondb_t* txn_db, + const rocksdb_flushoptions_t* options, + char** errptr) { + SaveError(errptr, txn_db->rep->Flush(options->rep)); +} + +void rocksdb_transactiondb_flush_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_flushoptions_t* options, + rocksdb_column_family_handle_t* column_family, char** errptr) { + SaveError(errptr, txn_db->rep->Flush(options->rep, column_family->rep)); +} + +void rocksdb_transactiondb_flush_cfs( + rocksdb_transactiondb_t* txn_db, const rocksdb_flushoptions_t* options, + rocksdb_column_family_handle_t** column_families, int num_column_families, + char** errptr) { + std::vector column_family_handles; + for (int i = 0; i < num_column_families; i++) { + column_family_handles.push_back(column_families[i]->rep); + } + + SaveError(errptr, txn_db->rep->Flush(options->rep, column_family_handles)); +} + +rocksdb_checkpoint_t* rocksdb_transactiondb_checkpoint_object_create( + rocksdb_transactiondb_t* txn_db, char** errptr) { + Checkpoint* checkpoint; + if (SaveError(errptr, Checkpoint::Create(txn_db->rep, &checkpoint))) { + return nullptr; + } + rocksdb_checkpoint_t* result = new rocksdb_checkpoint_t; + result->rep = checkpoint; + return result; +} + +rocksdb_optimistictransactiondb_t* rocksdb_optimistictransactiondb_open( + const rocksdb_options_t* options, const char* name, char** errptr) { + OptimisticTransactionDB* otxn_db; + if (SaveError(errptr, OptimisticTransactionDB::Open( + options->rep, std::string(name), &otxn_db))) { + return nullptr; + } + rocksdb_optimistictransactiondb_t* result = + new rocksdb_optimistictransactiondb_t; + result->rep = otxn_db; + return result; +} + +rocksdb_optimistictransactiondb_t* +rocksdb_optimistictransactiondb_open_column_families( + const rocksdb_options_t* db_options, const char* name, + int num_column_families, const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char** errptr) { + std::vector column_families; + for (int i = 0; i < num_column_families; i++) { + column_families.push_back(ColumnFamilyDescriptor( + std::string(column_family_names[i]), + ColumnFamilyOptions(column_family_options[i]->rep))); + } + + OptimisticTransactionDB* otxn_db; + std::vector handles; + if (SaveError(errptr, OptimisticTransactionDB::Open( + DBOptions(db_options->rep), std::string(name), + column_families, &handles, &otxn_db))) { + return nullptr; + } + + for (size_t i = 0; i < handles.size(); i++) { + rocksdb_column_family_handle_t* c_handle = + new rocksdb_column_family_handle_t; + c_handle->rep = handles[i]; + column_family_handles[i] = c_handle; + } + rocksdb_optimistictransactiondb_t* result = + new rocksdb_optimistictransactiondb_t; + result->rep = otxn_db; + return result; +} + +rocksdb_t* rocksdb_optimistictransactiondb_get_base_db( + rocksdb_optimistictransactiondb_t* otxn_db) { + DB* base_db = otxn_db->rep->GetBaseDB(); + + if (base_db != nullptr) { + rocksdb_t* result = new rocksdb_t; + result->rep = base_db; + return result; + } + + return nullptr; +} + +void rocksdb_optimistictransactiondb_close_base_db(rocksdb_t* base_db) { + delete base_db; +} + +rocksdb_transaction_t* rocksdb_optimistictransaction_begin( + rocksdb_optimistictransactiondb_t* otxn_db, + const rocksdb_writeoptions_t* write_options, + const rocksdb_optimistictransaction_options_t* otxn_options, + rocksdb_transaction_t* old_txn) { + if (old_txn == nullptr) { + rocksdb_transaction_t* result = new rocksdb_transaction_t; + result->rep = otxn_db->rep->BeginTransaction(write_options->rep, + otxn_options->rep, nullptr); + return result; + } + old_txn->rep = otxn_db->rep->BeginTransaction( + write_options->rep, otxn_options->rep, old_txn->rep); + return old_txn; +} + +// Write batch into OptimisticTransactionDB +void rocksdb_optimistictransactiondb_write( + rocksdb_optimistictransactiondb_t* otxn_db, + const rocksdb_writeoptions_t* options, rocksdb_writebatch_t* batch, + char** errptr) { + SaveError(errptr, otxn_db->rep->Write(options->rep, &batch->rep)); +} + +void rocksdb_optimistictransactiondb_close( + rocksdb_optimistictransactiondb_t* otxn_db) { + delete otxn_db->rep; + delete otxn_db; +} + +rocksdb_checkpoint_t* rocksdb_optimistictransactiondb_checkpoint_object_create( + rocksdb_optimistictransactiondb_t* otxn_db, char** errptr) { + Checkpoint* checkpoint; + if (SaveError(errptr, Checkpoint::Create(otxn_db->rep, &checkpoint))) { + return nullptr; + } + rocksdb_checkpoint_t* result = new rocksdb_checkpoint_t; + result->rep = checkpoint; + return result; +} + +void rocksdb_free(void* ptr) { free(ptr); } + +rocksdb_pinnableslice_t* rocksdb_get_pinned( + rocksdb_t* db, const rocksdb_readoptions_t* options, const char* key, + size_t keylen, char** errptr) { + rocksdb_pinnableslice_t* v = new (rocksdb_pinnableslice_t); + Status s = db->rep->Get(options->rep, db->rep->DefaultColumnFamily(), + Slice(key, keylen), &v->rep); + if (!s.ok()) { + delete (v); + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + return nullptr; + } + return v; +} + +rocksdb_pinnableslice_t* rocksdb_get_pinned_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr) { + rocksdb_pinnableslice_t* v = new (rocksdb_pinnableslice_t); + Status s = db->rep->Get(options->rep, column_family->rep, Slice(key, keylen), + &v->rep); + if (!s.ok()) { + delete v; + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + return nullptr; + } + return v; +} + +void rocksdb_pinnableslice_destroy(rocksdb_pinnableslice_t* v) { delete v; } + +const char* rocksdb_pinnableslice_value(const rocksdb_pinnableslice_t* v, + size_t* vlen) { + if (!v) { + *vlen = 0; + return nullptr; + } + + *vlen = v->rep.size(); + return v->rep.data(); +} + +// container to keep databases and caches in order to use +// ROCKSDB_NAMESPACE::MemoryUtil +struct rocksdb_memory_consumers_t { + std::vector dbs; + std::unordered_set caches; +}; + +// initializes new container of memory consumers +rocksdb_memory_consumers_t* rocksdb_memory_consumers_create() { + return new rocksdb_memory_consumers_t; +} + +// adds datatabase to the container of memory consumers +void rocksdb_memory_consumers_add_db(rocksdb_memory_consumers_t* consumers, + rocksdb_t* db) { + consumers->dbs.push_back(db); +} + +// adds cache to the container of memory consumers +void rocksdb_memory_consumers_add_cache(rocksdb_memory_consumers_t* consumers, + rocksdb_cache_t* cache) { + consumers->caches.insert(cache); +} + +// deletes container with memory consumers +void rocksdb_memory_consumers_destroy(rocksdb_memory_consumers_t* consumers) { + delete consumers; +} + +// contains memory usage statistics provided by ROCKSDB_NAMESPACE::MemoryUtil +struct rocksdb_memory_usage_t { + uint64_t mem_table_total; + uint64_t mem_table_unflushed; + uint64_t mem_table_readers_total; + uint64_t cache_total; +}; + +// estimates amount of memory occupied by consumers (dbs and caches) +rocksdb_memory_usage_t* rocksdb_approximate_memory_usage_create( + rocksdb_memory_consumers_t* consumers, char** errptr) { + vector dbs; + for (auto db : consumers->dbs) { + dbs.push_back(db->rep); + } + + unordered_set cache_set; + for (auto cache : consumers->caches) { + cache_set.insert(const_cast(cache->rep.get())); + } + + std::map usage_by_type; + + auto status = MemoryUtil::GetApproximateMemoryUsageByType(dbs, cache_set, + &usage_by_type); + if (SaveError(errptr, status)) { + return nullptr; + } + + auto result = new rocksdb_memory_usage_t; + result->mem_table_total = usage_by_type[MemoryUtil::kMemTableTotal]; + result->mem_table_unflushed = usage_by_type[MemoryUtil::kMemTableUnFlushed]; + result->mem_table_readers_total = + usage_by_type[MemoryUtil::kTableReadersTotal]; + result->cache_total = usage_by_type[MemoryUtil::kCacheTotal]; + return result; +} + +uint64_t rocksdb_approximate_memory_usage_get_mem_table_total( + rocksdb_memory_usage_t* memory_usage) { + return memory_usage->mem_table_total; +} + +uint64_t rocksdb_approximate_memory_usage_get_mem_table_unflushed( + rocksdb_memory_usage_t* memory_usage) { + return memory_usage->mem_table_unflushed; +} + +uint64_t rocksdb_approximate_memory_usage_get_mem_table_readers_total( + rocksdb_memory_usage_t* memory_usage) { + return memory_usage->mem_table_readers_total; +} + +uint64_t rocksdb_approximate_memory_usage_get_cache_total( + rocksdb_memory_usage_t* memory_usage) { + return memory_usage->cache_total; +} + +void rocksdb_options_set_dump_malloc_stats(rocksdb_options_t* opt, + unsigned char val) { + opt->rep.dump_malloc_stats = val; +} + +void rocksdb_options_set_memtable_whole_key_filtering(rocksdb_options_t* opt, + unsigned char val) { + opt->rep.memtable_whole_key_filtering = val; +} + +void rocksdb_options_set_avoid_unnecessary_blocking_io(rocksdb_options_t* opt, + unsigned char val) { + opt->rep.avoid_unnecessary_blocking_io = val; +} + +unsigned char rocksdb_options_get_avoid_unnecessary_blocking_io( + rocksdb_options_t* opt) { + return opt->rep.avoid_unnecessary_blocking_io; +} + +// deletes container with memory usage estimates +void rocksdb_approximate_memory_usage_destroy(rocksdb_memory_usage_t* usage) { + delete usage; +} + +void rocksdb_cancel_all_background_work(rocksdb_t* db, unsigned char wait) { + CancelAllBackgroundWork(db->rep, wait); +} + +void rocksdb_disable_manual_compaction(rocksdb_t* db) { + db->rep->DisableManualCompaction(); +} + +void rocksdb_enable_manual_compaction(rocksdb_t* db) { + db->rep->EnableManualCompaction(); +} + +rocksdb_statistics_histogram_data_t* +rocksdb_statistics_histogram_data_create() { + return new rocksdb_statistics_histogram_data_t{}; +} + +void rocksdb_statistics_histogram_data_destroy( + rocksdb_statistics_histogram_data_t* data) { + delete data; +} + +double rocksdb_statistics_histogram_data_get_median( + rocksdb_statistics_histogram_data_t* data) { + return data->rep.median; +} + +double rocksdb_statistics_histogram_data_get_p95( + rocksdb_statistics_histogram_data_t* data) { + return data->rep.percentile95; +} + +double rocksdb_statistics_histogram_data_get_p99( + rocksdb_statistics_histogram_data_t* data) { + return data->rep.percentile99; +} + +double rocksdb_statistics_histogram_data_get_average( + rocksdb_statistics_histogram_data_t* data) { + return data->rep.average; +} + +double rocksdb_statistics_histogram_data_get_std_dev( + rocksdb_statistics_histogram_data_t* data) { + return data->rep.standard_deviation; +} + +double rocksdb_statistics_histogram_data_get_max( + rocksdb_statistics_histogram_data_t* data) { + return data->rep.max; +} + +uint64_t rocksdb_statistics_histogram_data_get_count( + rocksdb_statistics_histogram_data_t* data) { + return data->rep.count; +} + +uint64_t rocksdb_statistics_histogram_data_get_sum( + rocksdb_statistics_histogram_data_t* data) { + return data->rep.sum; +} + +double rocksdb_statistics_histogram_data_get_min( + rocksdb_statistics_histogram_data_t* data) { + return data->rep.min; +} + +} // end extern "C" diff --git a/librocksdb-sys/rocksdb/db/c_test.c b/librocksdb-sys/rocksdb/db/c_test.c new file mode 100644 index 0000000..b21e1ae --- /dev/null +++ b/librocksdb-sys/rocksdb/db/c_test.c @@ -0,0 +1,3707 @@ +/* Copyright (c) 2011 The LevelDB Authors. All rights reserved. + Use of this source code is governed by a BSD-style license that can be + found in the LICENSE file. See the AUTHORS file for names of contributors. */ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +#include "rocksdb/c.h" + +#include +#include +#include +#include +#include +#include +#ifndef OS_WIN +#include +#endif +#include + +// Can not use port/port.h macros as this is a c file +#ifdef OS_WIN +#include + +// Ok for uniqueness +int geteuid() { + int result = 0; + + result = ((int)GetCurrentProcessId() << 16); + result |= (int)GetCurrentThreadId(); + + return result; +} + +#endif + +const char* phase = ""; +static char dbname[200]; +static char sstfilename[200]; +static char dbbackupname[200]; +static char dbcheckpointname[200]; +static char dbpathname[200]; +static char secondary_path[200]; + +static void StartPhase(const char* name) { + fprintf(stderr, "=== Test %s\n", name); + phase = name; +} +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4996) // getenv security warning +#endif +static const char* GetTempDir(void) { + const char* ret = getenv("TEST_TMPDIR"); + if (ret == NULL || ret[0] == '\0') +#ifdef OS_WIN + ret = getenv("TEMP"); +#else + ret = "/tmp"; +#endif + return ret; +} +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +#define CheckNoError(err) \ + if ((err) != NULL) { \ + fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, (err)); \ + abort(); \ + } + +#define CheckCondition(cond) \ + if (!(cond)) { \ + fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, #cond); \ + abort(); \ + } + +static void CheckEqual(const char* expected, const char* v, size_t n) { + if (expected == NULL && v == NULL) { + // ok + } else if (expected != NULL && v != NULL && n == strlen(expected) && + memcmp(expected, v, n) == 0) { + // ok + return; + } else { + fprintf(stderr, "%s: expected '%s', got '%s'\n", phase, + (expected ? expected : "(null)"), (v ? v : "(null)")); + abort(); + } +} + +static void Free(char** ptr) { + if (*ptr) { + free(*ptr); + *ptr = NULL; + } +} + +static void CheckValue(char* err, const char* expected, char** actual, + size_t actual_length) { + CheckNoError(err); + CheckEqual(expected, *actual, actual_length); + Free(actual); +} + +static void CheckGet(rocksdb_t* db, const rocksdb_readoptions_t* options, + const char* key, const char* expected) { + char* err = NULL; + size_t val_len; + char* val; + val = rocksdb_get(db, options, key, strlen(key), &val_len, &err); + CheckNoError(err); + CheckEqual(expected, val, val_len); + Free(&val); +} + +static void CheckGetCF(rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* handle, const char* key, + const char* expected) { + char* err = NULL; + size_t val_len; + char* val; + val = rocksdb_get_cf(db, options, handle, key, strlen(key), &val_len, &err); + CheckNoError(err); + CheckEqual(expected, val, val_len); + Free(&val); +} + +static void CheckPinGet(rocksdb_t* db, const rocksdb_readoptions_t* options, + const char* key, const char* expected) { + char* err = NULL; + size_t val_len; + const char* val; + rocksdb_pinnableslice_t* p; + p = rocksdb_get_pinned(db, options, key, strlen(key), &err); + CheckNoError(err); + val = rocksdb_pinnableslice_value(p, &val_len); + CheckEqual(expected, val, val_len); + rocksdb_pinnableslice_destroy(p); +} + +static void CheckPinGetCF(rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* handle, + const char* key, const char* expected) { + char* err = NULL; + size_t val_len; + const char* val; + rocksdb_pinnableslice_t* p; + p = rocksdb_get_pinned_cf(db, options, handle, key, strlen(key), &err); + CheckNoError(err); + val = rocksdb_pinnableslice_value(p, &val_len); + CheckEqual(expected, val, val_len); + rocksdb_pinnableslice_destroy(p); +} + +static void CheckMultiGetValues(size_t num_keys, char** values, + size_t* values_sizes, char** errs, + const char** expected) { + for (size_t i = 0; i < num_keys; i++) { + CheckNoError(errs[i]); + CheckEqual(expected[i], values[i], values_sizes[i]); + Free(&values[i]); + } +} + +static void CheckIter(rocksdb_iterator_t* iter, const char* key, + const char* val) { + size_t len; + const char* str; + str = rocksdb_iter_key(iter, &len); + CheckEqual(key, str, len); + str = rocksdb_iter_value(iter, &len); + CheckEqual(val, str, len); +} + +// Callback from rocksdb_writebatch_iterate() +static void CheckPut(void* ptr, const char* k, size_t klen, const char* v, + size_t vlen) { + int* state = (int*)ptr; + CheckCondition(*state < 2); + switch (*state) { + case 0: + CheckEqual("bar", k, klen); + CheckEqual("b", v, vlen); + break; + case 1: + CheckEqual("box", k, klen); + CheckEqual("c", v, vlen); + break; + } + (*state)++; +} + +// Callback from rocksdb_writebatch_iterate() +static void CheckDel(void* ptr, const char* k, size_t klen) { + int* state = (int*)ptr; + CheckCondition(*state == 2); + CheckEqual("bar", k, klen); + (*state)++; +} + +static void CmpDestroy(void* arg) { (void)arg; } + +static int CmpCompare(void* arg, const char* a, size_t alen, const char* b, + size_t blen) { + (void)arg; + size_t n = (alen < blen) ? alen : blen; + int r = memcmp(a, b, n); + if (r == 0) { + if (alen < blen) + r = -1; + else if (alen > blen) + r = +1; + } + return r; +} + +static const char* CmpName(void* arg) { + (void)arg; + return "foo"; +} + +// Custom compaction filter +static void CFilterDestroy(void* arg) { (void)arg; } +static const char* CFilterName(void* arg) { + (void)arg; + return "foo"; +} +static unsigned char CFilterFilter(void* arg, int level, const char* key, + size_t key_length, + const char* existing_value, + size_t value_length, char** new_value, + size_t* new_value_length, + unsigned char* value_changed) { + (void)arg; + (void)level; + (void)existing_value; + (void)value_length; + if (key_length == 3) { + if (memcmp(key, "bar", key_length) == 0) { + return 1; + } else if (memcmp(key, "baz", key_length) == 0) { + *value_changed = 1; + *new_value = "newbazvalue"; + *new_value_length = 11; + return 0; + } + } + return 0; +} + +static void CFilterFactoryDestroy(void* arg) { (void)arg; } +static const char* CFilterFactoryName(void* arg) { + (void)arg; + return "foo"; +} +static rocksdb_compactionfilter_t* CFilterCreate( + void* arg, rocksdb_compactionfiltercontext_t* context) { + (void)arg; + (void)context; + return rocksdb_compactionfilter_create(NULL, CFilterDestroy, CFilterFilter, + CFilterName); +} + +void CheckMetaData(rocksdb_column_family_metadata_t* cf_meta, + const char* expected_cf_name) { + char* cf_name = rocksdb_column_family_metadata_get_name(cf_meta); + assert(strcmp(cf_name, expected_cf_name) == 0); + rocksdb_free(cf_name); + + size_t cf_size = rocksdb_column_family_metadata_get_size(cf_meta); + assert(cf_size > 0); + size_t cf_file_count = rocksdb_column_family_metadata_get_size(cf_meta); + assert(cf_file_count > 0); + + uint64_t total_level_size = 0; + size_t total_file_count = 0; + size_t level_count = rocksdb_column_family_metadata_get_level_count(cf_meta); + assert(level_count > 0); + for (size_t l = 0; l < level_count; ++l) { + rocksdb_level_metadata_t* level_meta = + rocksdb_column_family_metadata_get_level_metadata(cf_meta, l); + assert(level_meta); + assert(rocksdb_level_metadata_get_level(level_meta) >= (int)l); + uint64_t level_size = rocksdb_level_metadata_get_size(level_meta); + uint64_t file_size_in_level = 0; + + size_t file_count = rocksdb_level_metadata_get_file_count(level_meta); + total_file_count += file_count; + for (size_t f = 0; f < file_count; ++f) { + rocksdb_sst_file_metadata_t* file_meta = + rocksdb_level_metadata_get_sst_file_metadata(level_meta, f); + assert(file_meta); + + uint64_t file_size = rocksdb_sst_file_metadata_get_size(file_meta); + assert(file_size > 0); + file_size_in_level += file_size; + + char* file_name = + rocksdb_sst_file_metadata_get_relative_filename(file_meta); + assert(file_name); + assert(strlen(file_name) > 0); + rocksdb_free(file_name); + + size_t smallest_key_len; + char* smallest_key = rocksdb_sst_file_metadata_get_smallestkey( + file_meta, &smallest_key_len); + assert(smallest_key); + assert(smallest_key_len > 0); + size_t largest_key_len; + char* largest_key = + rocksdb_sst_file_metadata_get_largestkey(file_meta, &largest_key_len); + assert(largest_key); + assert(largest_key_len > 0); + rocksdb_free(smallest_key); + rocksdb_free(largest_key); + + rocksdb_sst_file_metadata_destroy(file_meta); + } + assert(level_size == file_size_in_level); + total_level_size += level_size; + rocksdb_level_metadata_destroy(level_meta); + } + assert(total_file_count > 0); + assert(cf_size == total_level_size); +} + +void GetAndCheckMetaData(rocksdb_t* db) { + rocksdb_column_family_metadata_t* cf_meta = + rocksdb_get_column_family_metadata(db); + + CheckMetaData(cf_meta, "default"); + + rocksdb_column_family_metadata_destroy(cf_meta); +} + +void GetAndCheckMetaDataCf(rocksdb_t* db, + rocksdb_column_family_handle_t* handle, + const char* cf_name) { + // Compact to make sure we have at least one sst file to obtain datadata. + rocksdb_compact_range_cf(db, handle, NULL, 0, NULL, 0); + + rocksdb_column_family_metadata_t* cf_meta = + rocksdb_get_column_family_metadata_cf(db, handle); + + CheckMetaData(cf_meta, cf_name); + + rocksdb_column_family_metadata_destroy(cf_meta); +} + +static rocksdb_t* CheckCompaction(rocksdb_t* db, rocksdb_options_t* options, + rocksdb_readoptions_t* roptions, + rocksdb_writeoptions_t* woptions) { + char* err = NULL; + db = rocksdb_open(options, dbname, &err); + CheckNoError(err); + rocksdb_put(db, woptions, "foo", 3, "foovalue", 8, &err); + CheckNoError(err); + CheckGet(db, roptions, "foo", "foovalue"); + rocksdb_put(db, woptions, "bar", 3, "barvalue", 8, &err); + CheckNoError(err); + CheckGet(db, roptions, "bar", "barvalue"); + rocksdb_put(db, woptions, "baz", 3, "bazvalue", 8, &err); + CheckNoError(err); + CheckGet(db, roptions, "baz", "bazvalue"); + + // Disable compaction + rocksdb_disable_manual_compaction(db); + rocksdb_compact_range(db, NULL, 0, NULL, 0); + // should not filter anything when disabled + CheckGet(db, roptions, "foo", "foovalue"); + CheckGet(db, roptions, "bar", "barvalue"); + CheckGet(db, roptions, "baz", "bazvalue"); + // Reenable compaction + rocksdb_enable_manual_compaction(db); + + // Force compaction + rocksdb_compact_range(db, NULL, 0, NULL, 0); + // should have filtered bar, but not foo + CheckGet(db, roptions, "foo", "foovalue"); + CheckGet(db, roptions, "bar", NULL); + CheckGet(db, roptions, "baz", "newbazvalue"); + + rocksdb_suggest_compact_range(db, "bar", 3, "foo", 3, &err); + GetAndCheckMetaData(db); + CheckNoError(err); + + return db; +} + +// Custom merge operator +static void MergeOperatorDestroy(void* arg) { (void)arg; } +static const char* MergeOperatorName(void* arg) { + (void)arg; + return "TestMergeOperator"; +} +static char* MergeOperatorFullMerge( + void* arg, const char* key, size_t key_length, const char* existing_value, + size_t existing_value_length, const char* const* operands_list, + const size_t* operands_list_length, int num_operands, + unsigned char* success, size_t* new_value_length) { + (void)arg; + (void)key; + (void)key_length; + (void)existing_value; + (void)existing_value_length; + (void)operands_list; + (void)operands_list_length; + (void)num_operands; + *new_value_length = 4; + *success = 1; + char* result = malloc(4); + memcpy(result, "fake", 4); + return result; +} +static char* MergeOperatorPartialMerge(void* arg, const char* key, + size_t key_length, + const char* const* operands_list, + const size_t* operands_list_length, + int num_operands, unsigned char* success, + size_t* new_value_length) { + (void)arg; + (void)key; + (void)key_length; + (void)operands_list; + (void)operands_list_length; + (void)num_operands; + *new_value_length = 4; + *success = 1; + char* result = malloc(4); + memcpy(result, "fake", 4); + return result; +} + +static void CheckTxnGet(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options, const char* key, + const char* expected) { + char* err = NULL; + size_t val_len; + char* val; + val = rocksdb_transaction_get(txn, options, key, strlen(key), &val_len, &err); + CheckNoError(err); + CheckEqual(expected, val, val_len); + Free(&val); +} + +static void CheckTxnGetCF(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, + const char* key, const char* expected) { + char* err = NULL; + size_t val_len; + char* val; + val = rocksdb_transaction_get_cf(txn, options, column_family, key, + strlen(key), &val_len, &err); + CheckNoError(err); + CheckEqual(expected, val, val_len); + Free(&val); +} + +static void CheckTxnPinGet(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options, + const char* key, const char* expected) { + rocksdb_pinnableslice_t* p = NULL; + const char* val = NULL; + char* err = NULL; + size_t val_len; + p = rocksdb_transaction_get_pinned(txn, options, key, strlen(key), &err); + CheckNoError(err); + val = rocksdb_pinnableslice_value(p, &val_len); + CheckEqual(expected, val, val_len); + rocksdb_pinnableslice_destroy(p); +} + +static void CheckTxnPinGetCF(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, + const char* key, const char* expected) { + rocksdb_pinnableslice_t* p = NULL; + const char* val = NULL; + char* err = NULL; + size_t val_len; + p = rocksdb_transaction_get_pinned_cf(txn, options, column_family, key, + strlen(key), &err); + CheckNoError(err); + val = rocksdb_pinnableslice_value(p, &val_len); + CheckEqual(expected, val, val_len); + rocksdb_pinnableslice_destroy(p); +} + +static void CheckTxnGetForUpdate(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options, + const char* key, const char* expected) { + char* err = NULL; + size_t val_len; + char* val; + val = rocksdb_transaction_get_for_update(txn, options, key, strlen(key), + &val_len, true, &err); + CheckNoError(err); + CheckEqual(expected, val, val_len); + Free(&val); +} + +static void CheckTxnDBGet(rocksdb_transactiondb_t* txn_db, + const rocksdb_readoptions_t* options, const char* key, + const char* expected) { + char* err = NULL; + size_t val_len; + char* val; + val = rocksdb_transactiondb_get(txn_db, options, key, strlen(key), &val_len, + &err); + CheckNoError(err); + CheckEqual(expected, val, val_len); + Free(&val); +} + +static void CheckTxnDBGetCF(rocksdb_transactiondb_t* txn_db, + const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, + const char* key, const char* expected) { + char* err = NULL; + size_t val_len; + char* val; + val = rocksdb_transactiondb_get_cf(txn_db, options, column_family, key, + strlen(key), &val_len, &err); + CheckNoError(err); + CheckEqual(expected, val, val_len); + Free(&val); +} + +static void CheckTxnGetForUpdateCF( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + const char* expected) { + char* err = NULL; + size_t val_len; + char* val; + val = rocksdb_transaction_get_for_update_cf( + txn, options, column_family, key, strlen(key), &val_len, true, &err); + CheckNoError(err); + CheckEqual(expected, val, val_len); + Free(&val); +} + +static void CheckTxnDBPinGet(rocksdb_transactiondb_t* txn_db, + const rocksdb_readoptions_t* options, + const char* key, const char* expected) { + rocksdb_pinnableslice_t* p = NULL; + const char* val = NULL; + char* err = NULL; + size_t val_len; + p = rocksdb_transactiondb_get_pinned(txn_db, options, key, strlen(key), &err); + CheckNoError(err); + val = rocksdb_pinnableslice_value(p, &val_len); + CheckEqual(expected, val, val_len); + rocksdb_pinnableslice_destroy(p); +} + +static void CheckTxnDBPinGetCF(rocksdb_transactiondb_t* txn_db, + const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, + const char* key, const char* expected) { + rocksdb_pinnableslice_t* p = NULL; + const char* val = NULL; + char* err = NULL; + size_t val_len; + p = rocksdb_transactiondb_get_pinned_cf(txn_db, options, column_family, key, + strlen(key), &err); + CheckNoError(err); + val = rocksdb_pinnableslice_value(p, &val_len); + CheckEqual(expected, val, val_len); + rocksdb_pinnableslice_destroy(p); +} + +static void LoadAndCheckLatestOptions(const char* db_name, rocksdb_env_t* env, + bool ignore_unknown_options, + rocksdb_cache_t* cache, + rocksdb_comparator_t* cmp, + const size_t expected_num_column_families, + const char** expected_cf_names, + const char* expected_open_err) { + rocksdb_options_t* db_options; + size_t num_column_families; + char** list_column_family_names; + rocksdb_options_t** list_column_family_options; + char* err = 0; + + // load the latest rocksdb option + rocksdb_load_latest_options(db_name, env, ignore_unknown_options, cache, + &db_options, &num_column_families, + &list_column_family_names, + &list_column_family_options, &err); + assert(num_column_families == expected_num_column_families); + CheckNoError(err); + + // verify the loaded options by opening the db. + rocksdb_options_set_error_if_exists(db_options, 0); + + char** list_const_cf_names = + (char**)malloc(num_column_families * sizeof(char*)); + rocksdb_options_t** list_const_cf_options = (rocksdb_options_t**)malloc( + num_column_families * sizeof(rocksdb_options_t*)); + for (size_t i = 0; i < num_column_families; ++i) { + assert(strcmp(list_column_family_names[i], expected_cf_names[i]) == 0); + list_const_cf_names[i] = list_column_family_names[i]; + if (cmp) { + rocksdb_options_set_comparator(list_column_family_options[i], cmp); + } + list_const_cf_options[i] = list_column_family_options[i]; + } + rocksdb_column_family_handle_t** handles = + (rocksdb_column_family_handle_t**)malloc( + num_column_families * sizeof(rocksdb_column_family_handle_t*)); + + rocksdb_t* db = rocksdb_open_column_families( + db_options, db_name, (int)num_column_families, + (const char* const*)list_const_cf_names, + (const rocksdb_options_t* const*)list_const_cf_options, handles, &err); + if (expected_open_err == NULL) { + CheckNoError(err); + for (size_t i = 0; i < num_column_families; ++i) { + rocksdb_column_family_handle_destroy(handles[i]); + } + free(handles); + rocksdb_close(db); + } else { + assert(err != NULL); + assert(strcmp(err, expected_open_err) == 0); + free(handles); + free(err); + } + + free(list_const_cf_names); + free(list_const_cf_options); + rocksdb_load_latest_options_destroy(db_options, list_column_family_names, + list_column_family_options, + num_column_families); +} + +int main(int argc, char** argv) { + (void)argc; + (void)argv; + rocksdb_t* db; + rocksdb_comparator_t* cmp; + rocksdb_cache_t* cache; + rocksdb_dbpath_t* dbpath; + rocksdb_env_t* env; + rocksdb_options_t* options; + rocksdb_compactoptions_t* coptions; + rocksdb_block_based_table_options_t* table_options; + rocksdb_readoptions_t* roptions; + rocksdb_writeoptions_t* woptions; + rocksdb_ratelimiter_t* rate_limiter; + rocksdb_transactiondb_t* txn_db; + rocksdb_transactiondb_options_t* txn_db_options; + rocksdb_transaction_t* txn; + rocksdb_transaction_options_t* txn_options; + rocksdb_optimistictransactiondb_t* otxn_db; + rocksdb_optimistictransaction_options_t* otxn_options; + char* err = NULL; + int run = -1; + + snprintf(dbname, sizeof(dbname), "%s/rocksdb_c_test-%d", GetTempDir(), + ((int)geteuid())); + + snprintf(dbbackupname, sizeof(dbbackupname), "%s/rocksdb_c_test-%d-backup", + GetTempDir(), ((int)geteuid())); + + snprintf(dbcheckpointname, sizeof(dbcheckpointname), + "%s/rocksdb_c_test-%d-checkpoint", GetTempDir(), ((int)geteuid())); + + snprintf(sstfilename, sizeof(sstfilename), "%s/rocksdb_c_test-%d-sst", + GetTempDir(), ((int)geteuid())); + + snprintf(dbpathname, sizeof(dbpathname), "%s/rocksdb_c_test-%d-dbpath", + GetTempDir(), ((int)geteuid())); + + StartPhase("create_objects"); + cmp = rocksdb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName); + dbpath = rocksdb_dbpath_create(dbpathname, 1024 * 1024); + env = rocksdb_create_default_env(); + + rocksdb_create_dir_if_missing(env, GetTempDir(), &err); + CheckNoError(err); + + cache = rocksdb_cache_create_lru(100000); + + options = rocksdb_options_create(); + rocksdb_options_set_comparator(options, cmp); + rocksdb_options_set_error_if_exists(options, 1); + rocksdb_options_set_env(options, env); + rocksdb_options_set_info_log(options, NULL); + rocksdb_options_set_write_buffer_size(options, 100000); + rocksdb_options_set_paranoid_checks(options, 1); + rocksdb_options_set_max_open_files(options, 10); + + table_options = rocksdb_block_based_options_create(); + rocksdb_block_based_options_set_block_cache(table_options, cache); + rocksdb_block_based_options_set_data_block_index_type(table_options, 1); + rocksdb_block_based_options_set_data_block_hash_ratio(table_options, 0.75); + rocksdb_options_set_block_based_table_factory(options, table_options); + + rocksdb_options_set_compression(options, rocksdb_no_compression); + rocksdb_options_set_compression_options(options, -14, -1, 0, 0); + int compression_levels[] = {rocksdb_no_compression, rocksdb_no_compression, + rocksdb_no_compression, rocksdb_no_compression}; + rocksdb_options_set_compression_per_level(options, compression_levels, 4); + rate_limiter = rocksdb_ratelimiter_create(1000 * 1024 * 1024, 100 * 1000, 10); + rocksdb_options_set_ratelimiter(options, rate_limiter); + rocksdb_ratelimiter_destroy(rate_limiter); + + roptions = rocksdb_readoptions_create(); + rocksdb_readoptions_set_verify_checksums(roptions, 1); + rocksdb_readoptions_set_fill_cache(roptions, 1); + + woptions = rocksdb_writeoptions_create(); + rocksdb_writeoptions_set_sync(woptions, 1); + + coptions = rocksdb_compactoptions_create(); + rocksdb_compactoptions_set_exclusive_manual_compaction(coptions, 1); + + rocksdb_options_add_compact_on_deletion_collector_factory(options, 10000, + 10001); + rocksdb_options_add_compact_on_deletion_collector_factory_del_ratio( + options, 10000, 10001, 0.0); + + StartPhase("destroy"); + rocksdb_destroy_db(options, dbname, &err); + Free(&err); + + StartPhase("open_error"); + rocksdb_open(options, dbname, &err); + CheckCondition(err != NULL); + Free(&err); + + StartPhase("open"); + rocksdb_options_set_create_if_missing(options, 1); + db = rocksdb_open(options, dbname, &err); + CheckNoError(err); + CheckGet(db, roptions, "foo", NULL); + + StartPhase("put"); + rocksdb_put(db, woptions, "foo", 3, "hello", 5, &err); + CheckNoError(err); + CheckGet(db, roptions, "foo", "hello"); + + StartPhase("backup_and_restore"); + { + rocksdb_destroy_db(options, dbbackupname, &err); + CheckNoError(err); + + rocksdb_backup_engine_t* be = + rocksdb_backup_engine_open(options, dbbackupname, &err); + CheckNoError(err); + + rocksdb_backup_engine_create_new_backup(be, db, &err); + CheckNoError(err); + + // need a change to trigger a new backup + rocksdb_delete(db, woptions, "does-not-exist", 14, &err); + CheckNoError(err); + + rocksdb_backup_engine_create_new_backup(be, db, &err); + CheckNoError(err); + + const rocksdb_backup_engine_info_t* bei = + rocksdb_backup_engine_get_backup_info(be); + CheckCondition(rocksdb_backup_engine_info_count(bei) > 1); + rocksdb_backup_engine_info_destroy(bei); + + rocksdb_backup_engine_purge_old_backups(be, 1, &err); + CheckNoError(err); + + bei = rocksdb_backup_engine_get_backup_info(be); + CheckCondition(rocksdb_backup_engine_info_count(bei) == 1); + rocksdb_backup_engine_info_destroy(bei); + + rocksdb_delete(db, woptions, "foo", 3, &err); + CheckNoError(err); + + rocksdb_close(db); + + rocksdb_destroy_db(options, dbname, &err); + CheckNoError(err); + + rocksdb_restore_options_t* restore_options = + rocksdb_restore_options_create(); + rocksdb_restore_options_set_keep_log_files(restore_options, 0); + rocksdb_backup_engine_restore_db_from_latest_backup(be, dbname, dbname, + restore_options, &err); + CheckNoError(err); + rocksdb_restore_options_destroy(restore_options); + + rocksdb_options_set_error_if_exists(options, 0); + db = rocksdb_open(options, dbname, &err); + CheckNoError(err); + rocksdb_options_set_error_if_exists(options, 1); + + CheckGet(db, roptions, "foo", "hello"); + + rocksdb_backup_engine_close(be); + } + + StartPhase("checkpoint"); + { + rocksdb_destroy_db(options, dbcheckpointname, &err); + CheckNoError(err); + + rocksdb_checkpoint_t* checkpoint = + rocksdb_checkpoint_object_create(db, &err); + CheckNoError(err); + + rocksdb_checkpoint_create(checkpoint, dbcheckpointname, 0, &err); + CheckNoError(err); + + // start a new database from the checkpoint + rocksdb_close(db); + rocksdb_options_set_error_if_exists(options, 0); + db = rocksdb_open(options, dbcheckpointname, &err); + CheckNoError(err); + + CheckGet(db, roptions, "foo", "hello"); + + rocksdb_checkpoint_object_destroy(checkpoint); + + rocksdb_close(db); + rocksdb_destroy_db(options, dbcheckpointname, &err); + CheckNoError(err); + + db = rocksdb_open(options, dbname, &err); + CheckNoError(err); + rocksdb_options_set_error_if_exists(options, 1); + } + + StartPhase("compactall"); + rocksdb_compact_range(db, NULL, 0, NULL, 0); + CheckGet(db, roptions, "foo", "hello"); + + StartPhase("compactrange"); + rocksdb_compact_range(db, "a", 1, "z", 1); + CheckGet(db, roptions, "foo", "hello"); + + StartPhase("compactallopt"); + rocksdb_compact_range_opt(db, coptions, NULL, 0, NULL, 0); + CheckGet(db, roptions, "foo", "hello"); + + StartPhase("compactrangeopt"); + rocksdb_compact_range_opt(db, coptions, "a", 1, "z", 1); + CheckGet(db, roptions, "foo", "hello"); + + // Simple check cache usage + StartPhase("cache_usage"); + { + rocksdb_readoptions_set_pin_data(roptions, 1); + rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions); + rocksdb_iter_seek(iter, "foo", 3); + + size_t usage = rocksdb_cache_get_usage(cache); + CheckCondition(usage > 0); + + size_t pin_usage = rocksdb_cache_get_pinned_usage(cache); + CheckCondition(pin_usage > 0); + + rocksdb_iter_next(iter); + rocksdb_iter_destroy(iter); + rocksdb_readoptions_set_pin_data(roptions, 0); + } + + StartPhase("addfile"); + { + rocksdb_envoptions_t* env_opt = rocksdb_envoptions_create(); + rocksdb_options_t* io_options = rocksdb_options_create(); + rocksdb_sstfilewriter_t* writer = + rocksdb_sstfilewriter_create(env_opt, io_options); + + remove(sstfilename); + rocksdb_sstfilewriter_open(writer, sstfilename, &err); + CheckNoError(err); + rocksdb_sstfilewriter_put(writer, "sstk1", 5, "v1", 2, &err); + CheckNoError(err); + rocksdb_sstfilewriter_put(writer, "sstk2", 5, "v2", 2, &err); + CheckNoError(err); + rocksdb_sstfilewriter_put(writer, "sstk3", 5, "v3", 2, &err); + CheckNoError(err); + rocksdb_sstfilewriter_finish(writer, &err); + CheckNoError(err); + + rocksdb_ingestexternalfileoptions_t* ing_opt = + rocksdb_ingestexternalfileoptions_create(); + const char* file_list[1] = {sstfilename}; + rocksdb_ingest_external_file(db, file_list, 1, ing_opt, &err); + CheckNoError(err); + CheckGet(db, roptions, "sstk1", "v1"); + CheckGet(db, roptions, "sstk2", "v2"); + CheckGet(db, roptions, "sstk3", "v3"); + + remove(sstfilename); + rocksdb_sstfilewriter_open(writer, sstfilename, &err); + CheckNoError(err); + rocksdb_sstfilewriter_put(writer, "sstk2", 5, "v4", 2, &err); + CheckNoError(err); + rocksdb_sstfilewriter_put(writer, "sstk22", 6, "v5", 2, &err); + CheckNoError(err); + rocksdb_sstfilewriter_put(writer, "sstk3", 5, "v6", 2, &err); + CheckNoError(err); + rocksdb_sstfilewriter_finish(writer, &err); + CheckNoError(err); + + rocksdb_ingest_external_file(db, file_list, 1, ing_opt, &err); + CheckNoError(err); + CheckGet(db, roptions, "sstk1", "v1"); + CheckGet(db, roptions, "sstk2", "v4"); + CheckGet(db, roptions, "sstk22", "v5"); + CheckGet(db, roptions, "sstk3", "v6"); + + rocksdb_sstfilewriter_open(writer, sstfilename, &err); + CheckNoError(err); + rocksdb_sstfilewriter_put(writer, "abc1", 4, "v7", 2, &err); + CheckNoError(err); + rocksdb_sstfilewriter_put(writer, "abc2", 4, "v8", 2, &err); + CheckNoError(err); + rocksdb_sstfilewriter_put(writer, "abc3", 4, "v9", 2, &err); + CheckNoError(err); + rocksdb_sstfilewriter_put(writer, "abc4", 4, "v10", 3, &err); + CheckNoError(err); + rocksdb_sstfilewriter_delete_range(writer, "abc1", 4, "abc4", 4, &err); + CheckNoError(err); + rocksdb_sstfilewriter_finish(writer, &err); + CheckNoError(err); + + rocksdb_ingestexternalfileoptions_destroy(ing_opt); + rocksdb_sstfilewriter_destroy(writer); + rocksdb_options_destroy(io_options); + rocksdb_envoptions_destroy(env_opt); + + // Delete all keys we just ingested + rocksdb_delete(db, woptions, "sstk1", 5, &err); + CheckNoError(err); + rocksdb_delete(db, woptions, "sstk2", 5, &err); + CheckNoError(err); + rocksdb_delete(db, woptions, "sstk22", 6, &err); + CheckNoError(err); + rocksdb_delete(db, woptions, "sstk3", 5, &err); + CheckNoError(err); + } + + StartPhase("writebatch"); + { + rocksdb_writebatch_t* wb = rocksdb_writebatch_create(); + rocksdb_writebatch_put(wb, "foo", 3, "a", 1); + rocksdb_writebatch_clear(wb); + rocksdb_writebatch_put(wb, "bar", 3, "b", 1); + rocksdb_writebatch_put(wb, "box", 3, "c", 1); + rocksdb_writebatch_delete(wb, "bar", 3); + rocksdb_write(db, woptions, wb, &err); + CheckNoError(err); + CheckGet(db, roptions, "foo", "hello"); + CheckGet(db, roptions, "bar", NULL); + CheckGet(db, roptions, "box", "c"); + int pos = 0; + rocksdb_writebatch_iterate(wb, &pos, CheckPut, CheckDel); + CheckCondition(pos == 3); + rocksdb_writebatch_clear(wb); + rocksdb_writebatch_put(wb, "bar", 3, "b", 1); + rocksdb_writebatch_put(wb, "bay", 3, "d", 1); + rocksdb_writebatch_delete_range(wb, "bar", 3, "bay", 3); + rocksdb_write(db, woptions, wb, &err); + CheckNoError(err); + CheckGet(db, roptions, "bar", NULL); + CheckGet(db, roptions, "bay", "d"); + rocksdb_writebatch_clear(wb); + const char* start_list[1] = {"bay"}; + const size_t start_sizes[1] = {3}; + const char* end_list[1] = {"baz"}; + const size_t end_sizes[1] = {3}; + rocksdb_writebatch_delete_rangev(wb, 1, start_list, start_sizes, end_list, + end_sizes); + rocksdb_write(db, woptions, wb, &err); + CheckNoError(err); + CheckGet(db, roptions, "bay", NULL); + rocksdb_writebatch_destroy(wb); + } + + StartPhase("writebatch_vectors"); + { + rocksdb_writebatch_t* wb = rocksdb_writebatch_create(); + const char* k_list[2] = {"z", "ap"}; + const size_t k_sizes[2] = {1, 2}; + const char* v_list[3] = {"x", "y", "z"}; + const size_t v_sizes[3] = {1, 1, 1}; + rocksdb_writebatch_putv(wb, 2, k_list, k_sizes, 3, v_list, v_sizes); + rocksdb_write(db, woptions, wb, &err); + CheckNoError(err); + CheckGet(db, roptions, "zap", "xyz"); + rocksdb_writebatch_delete(wb, "zap", 3); + rocksdb_write(db, woptions, wb, &err); + CheckNoError(err); + CheckGet(db, roptions, "zap", NULL); + rocksdb_writebatch_destroy(wb); + } + + StartPhase("writebatch_savepoint"); + { + rocksdb_writebatch_t* wb = rocksdb_writebatch_create(); + rocksdb_writebatch_set_save_point(wb); + rocksdb_writebatch_set_save_point(wb); + const char* k_list[2] = {"z", "ap"}; + const size_t k_sizes[2] = {1, 2}; + const char* v_list[3] = {"x", "y", "z"}; + const size_t v_sizes[3] = {1, 1, 1}; + rocksdb_writebatch_pop_save_point(wb, &err); + CheckNoError(err); + rocksdb_writebatch_putv(wb, 2, k_list, k_sizes, 3, v_list, v_sizes); + rocksdb_writebatch_rollback_to_save_point(wb, &err); + CheckNoError(err); + rocksdb_write(db, woptions, wb, &err); + CheckNoError(err); + CheckGet(db, roptions, "zap", NULL); + rocksdb_writebatch_destroy(wb); + } + + StartPhase("writebatch_rep"); + { + rocksdb_writebatch_t* wb1 = rocksdb_writebatch_create(); + rocksdb_writebatch_put(wb1, "baz", 3, "d", 1); + rocksdb_writebatch_put(wb1, "quux", 4, "e", 1); + rocksdb_writebatch_delete(wb1, "quux", 4); + size_t repsize1 = 0; + const char* rep = rocksdb_writebatch_data(wb1, &repsize1); + rocksdb_writebatch_t* wb2 = rocksdb_writebatch_create_from(rep, repsize1); + CheckCondition(rocksdb_writebatch_count(wb1) == + rocksdb_writebatch_count(wb2)); + size_t repsize2 = 0; + CheckCondition( + memcmp(rep, rocksdb_writebatch_data(wb2, &repsize2), repsize1) == 0); + rocksdb_writebatch_destroy(wb1); + rocksdb_writebatch_destroy(wb2); + } + + StartPhase("writebatch_wi"); + { + rocksdb_writebatch_wi_t* wbi = rocksdb_writebatch_wi_create(0, 1); + rocksdb_writebatch_wi_put(wbi, "foo", 3, "a", 1); + rocksdb_writebatch_wi_clear(wbi); + rocksdb_writebatch_wi_put(wbi, "bar", 3, "b", 1); + rocksdb_writebatch_wi_put(wbi, "box", 3, "c", 1); + rocksdb_writebatch_wi_delete(wbi, "bar", 3); + int count = rocksdb_writebatch_wi_count(wbi); + CheckCondition(count == 3); + size_t size; + char* value; + value = rocksdb_writebatch_wi_get_from_batch(wbi, options, "box", 3, &size, + &err); + CheckValue(err, "c", &value, size); + value = rocksdb_writebatch_wi_get_from_batch(wbi, options, "bar", 3, &size, + &err); + CheckValue(err, NULL, &value, size); + value = rocksdb_writebatch_wi_get_from_batch_and_db(wbi, db, roptions, + "foo", 3, &size, &err); + CheckValue(err, "hello", &value, size); + value = rocksdb_writebatch_wi_get_from_batch_and_db(wbi, db, roptions, + "box", 3, &size, &err); + CheckValue(err, "c", &value, size); + rocksdb_write_writebatch_wi(db, woptions, wbi, &err); + CheckNoError(err); + CheckGet(db, roptions, "foo", "hello"); + CheckGet(db, roptions, "bar", NULL); + CheckGet(db, roptions, "box", "c"); + int pos = 0; + rocksdb_writebatch_wi_iterate(wbi, &pos, CheckPut, CheckDel); + CheckCondition(pos == 3); + rocksdb_writebatch_wi_clear(wbi); + rocksdb_writebatch_wi_destroy(wbi); + } + + StartPhase("writebatch_wi_vectors"); + { + rocksdb_writebatch_wi_t* wb = rocksdb_writebatch_wi_create(0, 1); + const char* k_list[2] = {"z", "ap"}; + const size_t k_sizes[2] = {1, 2}; + const char* v_list[3] = {"x", "y", "z"}; + const size_t v_sizes[3] = {1, 1, 1}; + rocksdb_writebatch_wi_putv(wb, 2, k_list, k_sizes, 3, v_list, v_sizes); + rocksdb_write_writebatch_wi(db, woptions, wb, &err); + CheckNoError(err); + CheckGet(db, roptions, "zap", "xyz"); + rocksdb_writebatch_wi_delete(wb, "zap", 3); + rocksdb_write_writebatch_wi(db, woptions, wb, &err); + CheckNoError(err); + CheckGet(db, roptions, "zap", NULL); + rocksdb_writebatch_wi_destroy(wb); + } + + StartPhase("writebatch_wi_savepoint"); + { + rocksdb_writebatch_wi_t* wb = rocksdb_writebatch_wi_create(0, 1); + rocksdb_writebatch_wi_set_save_point(wb); + const char* k_list[2] = {"z", "ap"}; + const size_t k_sizes[2] = {1, 2}; + const char* v_list[3] = {"x", "y", "z"}; + const size_t v_sizes[3] = {1, 1, 1}; + rocksdb_writebatch_wi_putv(wb, 2, k_list, k_sizes, 3, v_list, v_sizes); + rocksdb_writebatch_wi_rollback_to_save_point(wb, &err); + CheckNoError(err); + rocksdb_write_writebatch_wi(db, woptions, wb, &err); + CheckNoError(err); + CheckGet(db, roptions, "zap", NULL); + rocksdb_writebatch_wi_destroy(wb); + } + + StartPhase("iter"); + { + rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions); + CheckCondition(!rocksdb_iter_valid(iter)); + rocksdb_iter_seek_to_first(iter); + CheckCondition(rocksdb_iter_valid(iter)); + CheckIter(iter, "box", "c"); + rocksdb_iter_next(iter); + CheckIter(iter, "foo", "hello"); + rocksdb_iter_prev(iter); + CheckIter(iter, "box", "c"); + rocksdb_iter_prev(iter); + CheckCondition(!rocksdb_iter_valid(iter)); + rocksdb_iter_seek_to_last(iter); + CheckIter(iter, "foo", "hello"); + rocksdb_iter_seek(iter, "b", 1); + CheckIter(iter, "box", "c"); + rocksdb_iter_seek_for_prev(iter, "g", 1); + CheckIter(iter, "foo", "hello"); + rocksdb_iter_seek_for_prev(iter, "box", 3); + CheckIter(iter, "box", "c"); + rocksdb_iter_get_error(iter, &err); + CheckNoError(err); + rocksdb_iter_destroy(iter); + } + + StartPhase("wbwi_iter"); + { + rocksdb_iterator_t* base_iter = rocksdb_create_iterator(db, roptions); + rocksdb_writebatch_wi_t* wbi = rocksdb_writebatch_wi_create(0, 1); + rocksdb_writebatch_wi_put(wbi, "bar", 3, "b", 1); + rocksdb_writebatch_wi_delete(wbi, "foo", 3); + rocksdb_iterator_t* iter = + rocksdb_writebatch_wi_create_iterator_with_base(wbi, base_iter); + CheckCondition(!rocksdb_iter_valid(iter)); + rocksdb_iter_seek_to_first(iter); + CheckCondition(rocksdb_iter_valid(iter)); + CheckIter(iter, "bar", "b"); + rocksdb_iter_next(iter); + CheckIter(iter, "box", "c"); + rocksdb_iter_prev(iter); + CheckIter(iter, "bar", "b"); + rocksdb_iter_prev(iter); + CheckCondition(!rocksdb_iter_valid(iter)); + rocksdb_iter_seek_to_last(iter); + CheckIter(iter, "box", "c"); + rocksdb_iter_seek(iter, "b", 1); + CheckIter(iter, "bar", "b"); + rocksdb_iter_seek_for_prev(iter, "c", 1); + CheckIter(iter, "box", "c"); + rocksdb_iter_seek_for_prev(iter, "box", 3); + CheckIter(iter, "box", "c"); + rocksdb_iter_get_error(iter, &err); + CheckNoError(err); + rocksdb_iter_destroy(iter); + rocksdb_writebatch_wi_destroy(wbi); + } + + StartPhase("multiget"); + { + const char* keys[3] = {"box", "foo", "notfound"}; + const size_t keys_sizes[3] = {3, 3, 8}; + char* vals[3]; + size_t vals_sizes[3]; + char* errs[3]; + const char* expected[3] = {"c", "hello", NULL}; + rocksdb_multi_get(db, roptions, 3, keys, keys_sizes, vals, vals_sizes, + errs); + CheckMultiGetValues(3, vals, vals_sizes, errs, expected); + } + + StartPhase("pin_get"); + { + CheckPinGet(db, roptions, "box", "c"); + CheckPinGet(db, roptions, "foo", "hello"); + CheckPinGet(db, roptions, "notfound", NULL); + } + + StartPhase("approximate_sizes"); + { + int i; + int n = 20000; + char keybuf[100]; + char valbuf[100]; + uint64_t sizes[2]; + const char* start[2] = {"a", "k00000000000000010000"}; + size_t start_len[2] = {1, 21}; + const char* limit[2] = {"k00000000000000010000", "z"}; + size_t limit_len[2] = {21, 1}; + rocksdb_writeoptions_set_sync(woptions, 0); + for (i = 0; i < n; i++) { + snprintf(keybuf, sizeof(keybuf), "k%020d", i); + snprintf(valbuf, sizeof(valbuf), "v%020d", i); + rocksdb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf), + &err); + CheckNoError(err); + } + rocksdb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes, + &err); + CheckNoError(err); + CheckCondition(sizes[0] > 0); + CheckCondition(sizes[1] > 0); + } + + StartPhase("property"); + { + char* prop = rocksdb_property_value(db, "nosuchprop"); + CheckCondition(prop == NULL); + prop = rocksdb_property_value(db, "rocksdb.stats"); + CheckCondition(prop != NULL); + Free(&prop); + } + + StartPhase("snapshot"); + { + const rocksdb_snapshot_t* snap; + snap = rocksdb_create_snapshot(db); + rocksdb_delete(db, woptions, "foo", 3, &err); + CheckNoError(err); + rocksdb_readoptions_set_snapshot(roptions, snap); + CheckGet(db, roptions, "foo", "hello"); + rocksdb_readoptions_set_snapshot(roptions, NULL); + CheckGet(db, roptions, "foo", NULL); + rocksdb_release_snapshot(db, snap); + } + StartPhase("snapshot_with_memtable_inplace_update"); + { + rocksdb_close(db); + const rocksdb_snapshot_t* snap = NULL; + const char* s_key = "foo_snap"; + const char* value1 = "hello_s1"; + const char* value2 = "hello_s2"; + rocksdb_options_set_allow_concurrent_memtable_write(options, 0); + rocksdb_options_set_inplace_update_support(options, 1); + rocksdb_options_set_error_if_exists(options, 0); + db = rocksdb_open(options, dbname, &err); + CheckNoError(err); + rocksdb_put(db, woptions, s_key, 8, value1, 8, &err); + snap = rocksdb_create_snapshot(db); + assert(snap != NULL); + rocksdb_put(db, woptions, s_key, 8, value2, 8, &err); + CheckNoError(err); + rocksdb_readoptions_set_snapshot(roptions, snap); + CheckGet(db, roptions, "foo", NULL); + // snapshot syntax is invalid, because of inplace update supported is set + CheckGet(db, roptions, s_key, value2); + // restore the data and options + rocksdb_delete(db, woptions, s_key, 8, &err); + CheckGet(db, roptions, s_key, NULL); + rocksdb_release_snapshot(db, snap); + rocksdb_readoptions_set_snapshot(roptions, NULL); + rocksdb_options_set_inplace_update_support(options, 0); + rocksdb_options_set_allow_concurrent_memtable_write(options, 1); + rocksdb_options_set_error_if_exists(options, 1); + } + StartPhase("repair"); + { + // If we do not compact here, then the lazy deletion of + // files (https://reviews.facebook.net/D6123) would leave + // around deleted files and the repair process will find + // those files and put them back into the database. + rocksdb_compact_range(db, NULL, 0, NULL, 0); + rocksdb_close(db); + rocksdb_options_set_create_if_missing(options, 0); + rocksdb_options_set_error_if_exists(options, 0); + rocksdb_options_set_wal_recovery_mode(options, 2); + rocksdb_repair_db(options, dbname, &err); + CheckNoError(err); + db = rocksdb_open(options, dbname, &err); + CheckNoError(err); + CheckGet(db, roptions, "foo", NULL); + CheckGet(db, roptions, "bar", NULL); + CheckGet(db, roptions, "box", "c"); + rocksdb_options_set_create_if_missing(options, 1); + rocksdb_options_set_error_if_exists(options, 1); + } + + StartPhase("filter"); + for (run = 1; run <= 4; run++) { + // run=0 uses custom filter (not currently supported) + // run=1 uses old block-based bloom filter + // run=2 run uses full bloom filter + // run=3 uses Ribbon + // run=4 uses Ribbon-Bloom hybrid configuration + CheckNoError(err); + rocksdb_filterpolicy_t* policy; + if (run == 1) { + policy = rocksdb_filterpolicy_create_bloom(8.0); + } else if (run == 2) { + policy = rocksdb_filterpolicy_create_bloom_full(8.0); + } else if (run == 3) { + policy = rocksdb_filterpolicy_create_ribbon(8.0); + } else { + policy = rocksdb_filterpolicy_create_ribbon_hybrid(8.0, 1); + } + rocksdb_block_based_options_set_filter_policy(table_options, policy); + + // Create new database + rocksdb_close(db); + rocksdb_destroy_db(options, dbname, &err); + rocksdb_options_set_block_based_table_factory(options, table_options); + db = rocksdb_open(options, dbname, &err); + CheckNoError(err); + rocksdb_put(db, woptions, "foo", 3, "foovalue", 8, &err); + CheckNoError(err); + rocksdb_put(db, woptions, "bar", 3, "barvalue", 8, &err); + CheckNoError(err); + + { + // Add enough keys to get just one reasonably populated Bloom filter + const int keys_to_add = 1500; + int i; + char keybuf[100]; + for (i = 0; i < keys_to_add; i++) { + snprintf(keybuf, sizeof(keybuf), "yes%020d", i); + rocksdb_put(db, woptions, keybuf, strlen(keybuf), "val", 3, &err); + CheckNoError(err); + } + } + rocksdb_compact_range(db, NULL, 0, NULL, 0); + + CheckGet(db, roptions, "foo", "foovalue"); + CheckGet(db, roptions, "bar", "barvalue"); + + { + // Query some keys not added to identify Bloom filter implementation + // from false positive queries, using perfcontext to detect Bloom + // filter behavior + rocksdb_perfcontext_t* perf = rocksdb_perfcontext_create(); + rocksdb_perfcontext_reset(perf); + + const int keys_to_query = 10000; + int i; + char keybuf[100]; + for (i = 0; i < keys_to_query; i++) { + snprintf(keybuf, sizeof(keybuf), "no%020d", i); + CheckGet(db, roptions, keybuf, NULL); + } + + const int hits = + (int)rocksdb_perfcontext_metric(perf, rocksdb_bloom_sst_hit_count); + if (run == 0) { + // Due to half true, half false with fake filter result + CheckCondition(hits == keys_to_query / 2); + } else if (run == 1 || run == 2 || run == 4) { + // For run == 1, block-based Bloom is no longer available in public + // API; attempting to enable it enables full Bloom instead. + // + // Essentially a fingerprint of full Bloom schema, format_version=5 + CheckCondition(hits == 188); + } else { + // Essentially a fingerprint of Ribbon schema + CheckCondition(hits == 226); + } + CheckCondition( + (keys_to_query - hits) == + (int)rocksdb_perfcontext_metric(perf, rocksdb_bloom_sst_miss_count)); + + rocksdb_perfcontext_destroy(perf); + } + + // Reset the policy + rocksdb_block_based_options_set_filter_policy(table_options, NULL); + rocksdb_options_set_block_based_table_factory(options, table_options); + } + + StartPhase("compaction_filter"); + { + rocksdb_options_t* options_with_filter = rocksdb_options_create(); + rocksdb_options_set_create_if_missing(options_with_filter, 1); + rocksdb_compactionfilter_t* cfilter; + cfilter = rocksdb_compactionfilter_create(NULL, CFilterDestroy, + CFilterFilter, CFilterName); + // Create new database + rocksdb_close(db); + rocksdb_destroy_db(options_with_filter, dbname, &err); + rocksdb_options_set_compaction_filter(options_with_filter, cfilter); + db = CheckCompaction(db, options_with_filter, roptions, woptions); + + rocksdb_options_set_compaction_filter(options_with_filter, NULL); + rocksdb_compactionfilter_destroy(cfilter); + rocksdb_options_destroy(options_with_filter); + } + + StartPhase("compaction_filter_factory"); + { + rocksdb_options_t* options_with_filter_factory = rocksdb_options_create(); + rocksdb_options_set_create_if_missing(options_with_filter_factory, 1); + rocksdb_compactionfilterfactory_t* factory; + factory = rocksdb_compactionfilterfactory_create( + NULL, CFilterFactoryDestroy, CFilterCreate, CFilterFactoryName); + // Create new database + rocksdb_close(db); + rocksdb_destroy_db(options_with_filter_factory, dbname, &err); + rocksdb_options_set_compaction_filter_factory(options_with_filter_factory, + factory); + db = CheckCompaction(db, options_with_filter_factory, roptions, woptions); + + rocksdb_options_set_compaction_filter_factory(options_with_filter_factory, + NULL); + rocksdb_options_destroy(options_with_filter_factory); + } + + StartPhase("merge_operator"); + { + rocksdb_mergeoperator_t* merge_operator; + merge_operator = rocksdb_mergeoperator_create( + NULL, MergeOperatorDestroy, MergeOperatorFullMerge, + MergeOperatorPartialMerge, NULL, MergeOperatorName); + // Create new database + rocksdb_close(db); + rocksdb_destroy_db(options, dbname, &err); + rocksdb_options_set_merge_operator(options, merge_operator); + db = rocksdb_open(options, dbname, &err); + CheckNoError(err); + rocksdb_put(db, woptions, "foo", 3, "foovalue", 8, &err); + CheckNoError(err); + CheckGet(db, roptions, "foo", "foovalue"); + rocksdb_merge(db, woptions, "foo", 3, "barvalue", 8, &err); + CheckNoError(err); + CheckGet(db, roptions, "foo", "fake"); + + // Merge of a non-existing value + rocksdb_merge(db, woptions, "bar", 3, "barvalue", 8, &err); + CheckNoError(err); + CheckGet(db, roptions, "bar", "fake"); + } + + StartPhase("columnfamilies"); + { + rocksdb_close(db); + rocksdb_destroy_db(options, dbname, &err); + CheckNoError(err); + + rocksdb_options_t* db_options = rocksdb_options_create(); + rocksdb_options_set_create_if_missing(db_options, 1); + db = rocksdb_open(db_options, dbname, &err); + CheckNoError(err); + rocksdb_close(db); + { + const char* expected_cf_names[1] = {"default"}; + LoadAndCheckLatestOptions(dbname, env, false, cache, NULL, 1, + expected_cf_names, NULL); + } + + rocksdb_options_set_create_if_missing(db_options, 0); + db = rocksdb_open(db_options, dbname, &err); + rocksdb_column_family_handle_t* cfh; + cfh = rocksdb_create_column_family(db, db_options, "cf1", &err); + rocksdb_column_family_handle_destroy(cfh); + CheckNoError(err); + rocksdb_close(db); + + size_t cflen; + char** column_fams = + rocksdb_list_column_families(db_options, dbname, &cflen, &err); + CheckNoError(err); + CheckEqual("default", column_fams[0], 7); + CheckEqual("cf1", column_fams[1], 3); + CheckCondition(cflen == 2); + rocksdb_list_column_families_destroy(column_fams, cflen); + + rocksdb_options_t* cf_options = rocksdb_options_create(); + + const char* cf_names[2] = {"default", "cf1"}; + const rocksdb_options_t* cf_opts[2] = {cf_options, cf_options}; + rocksdb_column_family_handle_t* handles[2]; + + LoadAndCheckLatestOptions(dbname, env, false, cache, NULL, 2, cf_names, + NULL); + + db = rocksdb_open_column_families(db_options, dbname, 2, cf_names, cf_opts, + handles, &err); + CheckNoError(err); + + rocksdb_put_cf(db, woptions, handles[1], "foo", 3, "hello", 5, &err); + CheckNoError(err); + + rocksdb_put_cf(db, woptions, handles[1], "foobar1", 7, "hello1", 6, &err); + CheckNoError(err); + rocksdb_put_cf(db, woptions, handles[1], "foobar2", 7, "hello2", 6, &err); + CheckNoError(err); + rocksdb_put_cf(db, woptions, handles[1], "foobar3", 7, "hello3", 6, &err); + CheckNoError(err); + rocksdb_put_cf(db, woptions, handles[1], "foobar4", 7, "hello4", 6, &err); + CheckNoError(err); + rocksdb_suggest_compact_range_cf(db, handles[1], "foo", 3, "foobar9", 7, + &err); + CheckNoError(err); + + rocksdb_flushoptions_t* flush_options = rocksdb_flushoptions_create(); + rocksdb_flushoptions_set_wait(flush_options, 1); + rocksdb_flush_cf(db, flush_options, handles[1], &err); + CheckNoError(err) rocksdb_flushoptions_destroy(flush_options); + + CheckGetCF(db, roptions, handles[1], "foo", "hello"); + CheckPinGetCF(db, roptions, handles[1], "foo", "hello"); + + rocksdb_delete_cf(db, woptions, handles[1], "foo", 3, &err); + CheckNoError(err); + + rocksdb_delete_range_cf(db, woptions, handles[1], "foobar2", 7, "foobar4", + 7, &err); + CheckNoError(err); + + CheckGetCF(db, roptions, handles[1], "foo", NULL); + CheckPinGetCF(db, roptions, handles[1], "foo", NULL); + + rocksdb_writebatch_t* wb = rocksdb_writebatch_create(); + rocksdb_writebatch_put_cf(wb, handles[1], "baz", 3, "a", 1); + rocksdb_writebatch_clear(wb); + rocksdb_writebatch_put_cf(wb, handles[1], "bar", 3, "b", 1); + rocksdb_writebatch_put_cf(wb, handles[1], "box", 3, "c", 1); + rocksdb_writebatch_put_cf(wb, handles[1], "buff", 4, "rocksdb", 7); + rocksdb_writebatch_delete_cf(wb, handles[1], "bar", 3); + rocksdb_write(db, woptions, wb, &err); + CheckNoError(err); + CheckGetCF(db, roptions, handles[1], "baz", NULL); + CheckGetCF(db, roptions, handles[1], "bar", NULL); + CheckGetCF(db, roptions, handles[1], "box", "c"); + CheckGetCF(db, roptions, handles[1], "buff", "rocksdb"); + CheckPinGetCF(db, roptions, handles[1], "baz", NULL); + CheckPinGetCF(db, roptions, handles[1], "bar", NULL); + CheckPinGetCF(db, roptions, handles[1], "box", "c"); + CheckPinGetCF(db, roptions, handles[1], "buff", "rocksdb"); + rocksdb_writebatch_destroy(wb); + + rocksdb_flush_wal(db, 1, &err); + CheckNoError(err); + + const char* keys[3] = {"box", "box", "barfooxx"}; + const rocksdb_column_family_handle_t* get_handles[3] = { + handles[0], handles[1], handles[1]}; + const size_t keys_sizes[3] = {3, 3, 8}; + char* vals[3]; + size_t vals_sizes[3]; + char* errs[3]; + rocksdb_multi_get_cf(db, roptions, get_handles, 3, keys, keys_sizes, vals, + vals_sizes, errs); + + int i; + for (i = 0; i < 3; i++) { + CheckEqual(NULL, errs[i], 0); + switch (i) { + case 0: + CheckEqual(NULL, vals[i], vals_sizes[i]); // wrong cf + break; + case 1: + CheckEqual("c", vals[i], vals_sizes[i]); // bingo + break; + case 2: + CheckEqual(NULL, vals[i], vals_sizes[i]); // normal not found + break; + } + Free(&vals[i]); + } + + { + const char* batched_keys[4] = {"box", "buff", "barfooxx", "box"}; + const size_t batched_keys_sizes[4] = {3, 4, 8, 3}; + const char* expected_value[4] = {"c", "rocksdb", NULL, "c"}; + char* batched_errs[4]; + + rocksdb_pinnableslice_t* pvals[4]; + rocksdb_batched_multi_get_cf(db, roptions, handles[1], 4, batched_keys, + batched_keys_sizes, pvals, batched_errs, + false); + const char* val; + size_t val_len; + for (i = 0; i < 4; ++i) { + val = rocksdb_pinnableslice_value(pvals[i], &val_len); + CheckNoError(batched_errs[i]); + CheckEqual(expected_value[i], val, val_len); + rocksdb_pinnableslice_destroy(pvals[i]); + } + } + + { + unsigned char value_found = 0; + + CheckCondition(!rocksdb_key_may_exist(db, roptions, "invalid_key", 11, + NULL, NULL, NULL, 0, NULL)); + CheckCondition(!rocksdb_key_may_exist(db, roptions, "invalid_key", 11, + &vals[0], &vals_sizes[0], NULL, 0, + &value_found)); + if (value_found) { + Free(&vals[0]); + } + + CheckCondition(!rocksdb_key_may_exist_cf(db, roptions, handles[1], + "invalid_key", 11, NULL, NULL, + NULL, 0, NULL)); + CheckCondition(!rocksdb_key_may_exist_cf(db, roptions, handles[1], + "invalid_key", 11, &vals[0], + &vals_sizes[0], NULL, 0, NULL)); + if (value_found) { + Free(&vals[0]); + } + } + + rocksdb_iterator_t* iter = + rocksdb_create_iterator_cf(db, roptions, handles[1]); + CheckCondition(!rocksdb_iter_valid(iter)); + rocksdb_iter_seek_to_first(iter); + CheckCondition(rocksdb_iter_valid(iter)); + + for (i = 0; rocksdb_iter_valid(iter) != 0; rocksdb_iter_next(iter)) { + i++; + } + CheckCondition(i == 4); + rocksdb_iter_get_error(iter, &err); + CheckNoError(err); + rocksdb_iter_destroy(iter); + + rocksdb_column_family_handle_t* iters_cf_handles[2] = {handles[0], + handles[1]}; + rocksdb_iterator_t* iters_handles[2]; + rocksdb_create_iterators(db, roptions, iters_cf_handles, iters_handles, 2, + &err); + CheckNoError(err); + + iter = iters_handles[0]; + CheckCondition(!rocksdb_iter_valid(iter)); + rocksdb_iter_seek_to_first(iter); + CheckCondition(!rocksdb_iter_valid(iter)); + rocksdb_iter_destroy(iter); + + iter = iters_handles[1]; + CheckCondition(!rocksdb_iter_valid(iter)); + rocksdb_iter_seek_to_first(iter); + CheckCondition(rocksdb_iter_valid(iter)); + + for (i = 0; rocksdb_iter_valid(iter) != 0; rocksdb_iter_next(iter)) { + i++; + } + CheckCondition(i == 4); + rocksdb_iter_get_error(iter, &err); + CheckNoError(err); + rocksdb_iter_destroy(iter); + + GetAndCheckMetaDataCf(db, handles[1], cf_names[1]); + + rocksdb_drop_column_family(db, handles[1], &err); + CheckNoError(err); + for (i = 0; i < 2; i++) { + rocksdb_column_family_handle_destroy(handles[i]); + } + rocksdb_close(db); + { + // As column family has been dropped, we expect only one column family. + const char* expected_cf_names[1] = {"default"}; + LoadAndCheckLatestOptions(dbname, env, false, cache, NULL, 1, + expected_cf_names, NULL); + } + rocksdb_destroy_db(options, dbname, &err); + rocksdb_options_destroy(db_options); + rocksdb_options_destroy(cf_options); + } + + StartPhase("prefix"); + { + // Create new database + rocksdb_options_set_allow_mmap_reads(options, 1); + rocksdb_options_set_prefix_extractor( + options, rocksdb_slicetransform_create_fixed_prefix(3)); + rocksdb_options_set_hash_skip_list_rep(options, 5000, 4, 4); + rocksdb_options_set_plain_table_factory(options, 4, 10, 0.75, 16, 0, 0, 0, + 0); + rocksdb_options_set_allow_concurrent_memtable_write(options, 0); + + db = rocksdb_open(options, dbname, &err); + CheckNoError(err); + + rocksdb_put(db, woptions, "foo1", 4, "foo", 3, &err); + CheckNoError(err); + rocksdb_put(db, woptions, "foo2", 4, "foo", 3, &err); + CheckNoError(err); + rocksdb_put(db, woptions, "foo3", 4, "foo", 3, &err); + CheckNoError(err); + rocksdb_put(db, woptions, "bar1", 4, "bar", 3, &err); + CheckNoError(err); + rocksdb_put(db, woptions, "bar2", 4, "bar", 3, &err); + CheckNoError(err); + rocksdb_put(db, woptions, "bar3", 4, "bar", 3, &err); + CheckNoError(err); + + rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions); + CheckCondition(!rocksdb_iter_valid(iter)); + + rocksdb_iter_seek(iter, "bar", 3); + rocksdb_iter_get_error(iter, &err); + CheckNoError(err); + CheckCondition(rocksdb_iter_valid(iter)); + + CheckIter(iter, "bar1", "bar"); + rocksdb_iter_next(iter); + CheckIter(iter, "bar2", "bar"); + rocksdb_iter_next(iter); + CheckIter(iter, "bar3", "bar"); + rocksdb_iter_get_error(iter, &err); + CheckNoError(err); + rocksdb_iter_destroy(iter); + + rocksdb_readoptions_set_total_order_seek(roptions, 1); + iter = rocksdb_create_iterator(db, roptions); + CheckCondition(!rocksdb_iter_valid(iter)); + + rocksdb_iter_seek(iter, "ba", 2); + rocksdb_iter_get_error(iter, &err); + CheckNoError(err); + CheckCondition(rocksdb_iter_valid(iter)); + CheckIter(iter, "bar1", "bar"); + + rocksdb_iter_destroy(iter); + rocksdb_readoptions_set_total_order_seek(roptions, 0); + + rocksdb_close(db); + + { + const char* expected_cf_names[1] = {"default"}; + LoadAndCheckLatestOptions(dbname, env, false, cache, NULL, 1, + expected_cf_names, + "Invalid argument: leveldb.BytewiseComparator: " + "does not match existing comparator foo"); + LoadAndCheckLatestOptions(dbname, env, false, cache, cmp, 1, + expected_cf_names, NULL); + } + rocksdb_destroy_db(options, dbname, &err); + } + + // Check memory usage stats + StartPhase("approximate_memory_usage"); + { + // Create database + db = rocksdb_open(options, dbname, &err); + CheckNoError(err); + + rocksdb_memory_consumers_t* consumers; + consumers = rocksdb_memory_consumers_create(); + rocksdb_memory_consumers_add_db(consumers, db); + rocksdb_memory_consumers_add_cache(consumers, cache); + + // take memory usage report before write-read operation + rocksdb_memory_usage_t* mu1; + mu1 = rocksdb_approximate_memory_usage_create(consumers, &err); + CheckNoError(err); + + // Put data (this should affect memtables) + rocksdb_put(db, woptions, "memory", 6, "test", 4, &err); + CheckNoError(err); + CheckGet(db, roptions, "memory", "test"); + + // take memory usage report after write-read operation + rocksdb_memory_usage_t* mu2; + mu2 = rocksdb_approximate_memory_usage_create(consumers, &err); + CheckNoError(err); + + // amount of memory used within memtables should grow + CheckCondition(rocksdb_approximate_memory_usage_get_mem_table_total(mu2) >= + rocksdb_approximate_memory_usage_get_mem_table_total(mu1)); + CheckCondition( + rocksdb_approximate_memory_usage_get_mem_table_unflushed(mu2) >= + rocksdb_approximate_memory_usage_get_mem_table_unflushed(mu1)); + + rocksdb_memory_consumers_destroy(consumers); + rocksdb_approximate_memory_usage_destroy(mu1); + rocksdb_approximate_memory_usage_destroy(mu2); + rocksdb_close(db); + rocksdb_destroy_db(options, dbname, &err); + CheckNoError(err); + } + + StartPhase("cuckoo_options"); + { + rocksdb_cuckoo_table_options_t* cuckoo_options; + cuckoo_options = rocksdb_cuckoo_options_create(); + rocksdb_cuckoo_options_set_hash_ratio(cuckoo_options, 0.5); + rocksdb_cuckoo_options_set_max_search_depth(cuckoo_options, 200); + rocksdb_cuckoo_options_set_cuckoo_block_size(cuckoo_options, 10); + rocksdb_cuckoo_options_set_identity_as_first_hash(cuckoo_options, 1); + rocksdb_cuckoo_options_set_use_module_hash(cuckoo_options, 0); + rocksdb_options_set_cuckoo_table_factory(options, cuckoo_options); + + db = rocksdb_open(options, dbname, &err); + CheckNoError(err); + + rocksdb_cuckoo_options_destroy(cuckoo_options); + } + + StartPhase("options"); + { + rocksdb_options_t* o; + o = rocksdb_options_create(); + + // Set and check options. + rocksdb_options_set_allow_ingest_behind(o, 1); + CheckCondition(1 == rocksdb_options_get_allow_ingest_behind(o)); + + rocksdb_options_compaction_readahead_size(o, 10); + CheckCondition(10 == rocksdb_options_get_compaction_readahead_size(o)); + + rocksdb_options_set_create_if_missing(o, 1); + CheckCondition(1 == rocksdb_options_get_create_if_missing(o)); + + rocksdb_options_set_create_missing_column_families(o, 1); + CheckCondition(1 == rocksdb_options_get_create_missing_column_families(o)); + + rocksdb_options_set_error_if_exists(o, 1); + CheckCondition(1 == rocksdb_options_get_error_if_exists(o)); + + rocksdb_options_set_paranoid_checks(o, 1); + CheckCondition(1 == rocksdb_options_get_paranoid_checks(o)); + + rocksdb_options_set_info_log_level(o, 3); + CheckCondition(3 == rocksdb_options_get_info_log_level(o)); + + rocksdb_options_set_write_buffer_size(o, 100); + CheckCondition(100 == rocksdb_options_get_write_buffer_size(o)); + + rocksdb_options_set_db_write_buffer_size(o, 1000); + CheckCondition(1000 == rocksdb_options_get_db_write_buffer_size(o)); + + rocksdb_options_set_max_open_files(o, 21); + CheckCondition(21 == rocksdb_options_get_max_open_files(o)); + + rocksdb_options_set_max_file_opening_threads(o, 5); + CheckCondition(5 == rocksdb_options_get_max_file_opening_threads(o)); + + rocksdb_options_set_max_total_wal_size(o, 400); + CheckCondition(400 == rocksdb_options_get_max_total_wal_size(o)); + + rocksdb_options_set_num_levels(o, 7); + CheckCondition(7 == rocksdb_options_get_num_levels(o)); + + rocksdb_options_set_level0_file_num_compaction_trigger(o, 4); + CheckCondition(4 == + rocksdb_options_get_level0_file_num_compaction_trigger(o)); + + rocksdb_options_set_level0_slowdown_writes_trigger(o, 6); + CheckCondition(6 == rocksdb_options_get_level0_slowdown_writes_trigger(o)); + + rocksdb_options_set_level0_stop_writes_trigger(o, 8); + CheckCondition(8 == rocksdb_options_get_level0_stop_writes_trigger(o)); + + rocksdb_options_set_target_file_size_base(o, 256); + CheckCondition(256 == rocksdb_options_get_target_file_size_base(o)); + + rocksdb_options_set_target_file_size_multiplier(o, 3); + CheckCondition(3 == rocksdb_options_get_target_file_size_multiplier(o)); + + rocksdb_options_set_max_bytes_for_level_base(o, 1024); + CheckCondition(1024 == rocksdb_options_get_max_bytes_for_level_base(o)); + + rocksdb_options_set_level_compaction_dynamic_level_bytes(o, 1); + CheckCondition(1 == + rocksdb_options_get_level_compaction_dynamic_level_bytes(o)); + + rocksdb_options_set_max_bytes_for_level_multiplier(o, 2.0); + CheckCondition(2.0 == + rocksdb_options_get_max_bytes_for_level_multiplier(o)); + + rocksdb_options_set_skip_stats_update_on_db_open(o, 1); + CheckCondition(1 == rocksdb_options_get_skip_stats_update_on_db_open(o)); + + rocksdb_options_set_skip_checking_sst_file_sizes_on_db_open(o, 1); + CheckCondition( + 1 == rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open(o)); + + rocksdb_options_set_max_write_buffer_number(o, 97); + CheckCondition(97 == rocksdb_options_get_max_write_buffer_number(o)); + + rocksdb_options_set_min_write_buffer_number_to_merge(o, 23); + CheckCondition(23 == + rocksdb_options_get_min_write_buffer_number_to_merge(o)); + + rocksdb_options_set_max_write_buffer_number_to_maintain(o, 64); + CheckCondition(64 == + rocksdb_options_get_max_write_buffer_number_to_maintain(o)); + + rocksdb_options_set_max_write_buffer_size_to_maintain(o, 50000); + CheckCondition(50000 == + rocksdb_options_get_max_write_buffer_size_to_maintain(o)); + + rocksdb_options_set_enable_pipelined_write(o, 1); + CheckCondition(1 == rocksdb_options_get_enable_pipelined_write(o)); + + rocksdb_options_set_unordered_write(o, 1); + CheckCondition(1 == rocksdb_options_get_unordered_write(o)); + + rocksdb_options_set_max_subcompactions(o, 123456); + CheckCondition(123456 == rocksdb_options_get_max_subcompactions(o)); + + rocksdb_options_set_max_background_jobs(o, 2); + CheckCondition(2 == rocksdb_options_get_max_background_jobs(o)); + + rocksdb_options_set_max_background_compactions(o, 3); + CheckCondition(3 == rocksdb_options_get_max_background_compactions(o)); + + rocksdb_options_set_max_background_flushes(o, 5); + CheckCondition(5 == rocksdb_options_get_max_background_flushes(o)); + + rocksdb_options_set_max_log_file_size(o, 6); + CheckCondition(6 == rocksdb_options_get_max_log_file_size(o)); + + rocksdb_options_set_log_file_time_to_roll(o, 7); + CheckCondition(7 == rocksdb_options_get_log_file_time_to_roll(o)); + + rocksdb_options_set_keep_log_file_num(o, 8); + CheckCondition(8 == rocksdb_options_get_keep_log_file_num(o)); + + rocksdb_options_set_recycle_log_file_num(o, 9); + CheckCondition(9 == rocksdb_options_get_recycle_log_file_num(o)); + + rocksdb_options_set_soft_pending_compaction_bytes_limit(o, 10); + CheckCondition(10 == + rocksdb_options_get_soft_pending_compaction_bytes_limit(o)); + + rocksdb_options_set_hard_pending_compaction_bytes_limit(o, 11); + CheckCondition(11 == + rocksdb_options_get_hard_pending_compaction_bytes_limit(o)); + + rocksdb_options_set_max_manifest_file_size(o, 12); + CheckCondition(12 == rocksdb_options_get_max_manifest_file_size(o)); + + rocksdb_options_set_table_cache_numshardbits(o, 13); + CheckCondition(13 == rocksdb_options_get_table_cache_numshardbits(o)); + + rocksdb_options_set_arena_block_size(o, 14); + CheckCondition(14 == rocksdb_options_get_arena_block_size(o)); + + rocksdb_options_set_use_fsync(o, 1); + CheckCondition(1 == rocksdb_options_get_use_fsync(o)); + + rocksdb_options_set_WAL_ttl_seconds(o, 15); + CheckCondition(15 == rocksdb_options_get_WAL_ttl_seconds(o)); + + rocksdb_options_set_WAL_size_limit_MB(o, 16); + CheckCondition(16 == rocksdb_options_get_WAL_size_limit_MB(o)); + + rocksdb_options_set_manifest_preallocation_size(o, 17); + CheckCondition(17 == rocksdb_options_get_manifest_preallocation_size(o)); + + rocksdb_options_set_allow_mmap_reads(o, 1); + CheckCondition(1 == rocksdb_options_get_allow_mmap_reads(o)); + + rocksdb_options_set_allow_mmap_writes(o, 1); + CheckCondition(1 == rocksdb_options_get_allow_mmap_writes(o)); + + rocksdb_options_set_use_direct_reads(o, 1); + CheckCondition(1 == rocksdb_options_get_use_direct_reads(o)); + + rocksdb_options_set_use_direct_io_for_flush_and_compaction(o, 1); + CheckCondition( + 1 == rocksdb_options_get_use_direct_io_for_flush_and_compaction(o)); + + rocksdb_options_set_is_fd_close_on_exec(o, 1); + CheckCondition(1 == rocksdb_options_get_is_fd_close_on_exec(o)); + + rocksdb_options_set_stats_dump_period_sec(o, 18); + CheckCondition(18 == rocksdb_options_get_stats_dump_period_sec(o)); + + rocksdb_options_set_stats_persist_period_sec(o, 5); + CheckCondition(5 == rocksdb_options_get_stats_persist_period_sec(o)); + + rocksdb_options_set_advise_random_on_open(o, 1); + CheckCondition(1 == rocksdb_options_get_advise_random_on_open(o)); + + rocksdb_options_set_access_hint_on_compaction_start(o, 3); + CheckCondition(3 == rocksdb_options_get_access_hint_on_compaction_start(o)); + + rocksdb_options_set_use_adaptive_mutex(o, 1); + CheckCondition(1 == rocksdb_options_get_use_adaptive_mutex(o)); + + rocksdb_options_set_bytes_per_sync(o, 19); + CheckCondition(19 == rocksdb_options_get_bytes_per_sync(o)); + + rocksdb_options_set_wal_bytes_per_sync(o, 20); + CheckCondition(20 == rocksdb_options_get_wal_bytes_per_sync(o)); + + rocksdb_options_set_writable_file_max_buffer_size(o, 21); + CheckCondition(21 == rocksdb_options_get_writable_file_max_buffer_size(o)); + + rocksdb_options_set_allow_concurrent_memtable_write(o, 1); + CheckCondition(1 == rocksdb_options_get_allow_concurrent_memtable_write(o)); + + rocksdb_options_set_enable_write_thread_adaptive_yield(o, 1); + CheckCondition(1 == + rocksdb_options_get_enable_write_thread_adaptive_yield(o)); + + rocksdb_options_set_max_sequential_skip_in_iterations(o, 22); + CheckCondition(22 == + rocksdb_options_get_max_sequential_skip_in_iterations(o)); + + rocksdb_options_set_disable_auto_compactions(o, 1); + CheckCondition(1 == rocksdb_options_get_disable_auto_compactions(o)); + + rocksdb_options_set_optimize_filters_for_hits(o, 1); + CheckCondition(1 == rocksdb_options_get_optimize_filters_for_hits(o)); + + rocksdb_options_set_delete_obsolete_files_period_micros(o, 23); + CheckCondition(23 == + rocksdb_options_get_delete_obsolete_files_period_micros(o)); + + rocksdb_options_set_memtable_prefix_bloom_size_ratio(o, 2.0); + CheckCondition(2.0 == + rocksdb_options_get_memtable_prefix_bloom_size_ratio(o)); + + rocksdb_options_set_max_compaction_bytes(o, 24); + CheckCondition(24 == rocksdb_options_get_max_compaction_bytes(o)); + + rocksdb_options_set_memtable_huge_page_size(o, 25); + CheckCondition(25 == rocksdb_options_get_memtable_huge_page_size(o)); + + rocksdb_options_set_max_successive_merges(o, 26); + CheckCondition(26 == rocksdb_options_get_max_successive_merges(o)); + + rocksdb_options_set_bloom_locality(o, 27); + CheckCondition(27 == rocksdb_options_get_bloom_locality(o)); + + rocksdb_options_set_inplace_update_support(o, 1); + CheckCondition(1 == rocksdb_options_get_inplace_update_support(o)); + + rocksdb_options_set_inplace_update_num_locks(o, 28); + CheckCondition(28 == rocksdb_options_get_inplace_update_num_locks(o)); + + rocksdb_options_set_report_bg_io_stats(o, 1); + CheckCondition(1 == rocksdb_options_get_report_bg_io_stats(o)); + + rocksdb_options_set_wal_recovery_mode(o, 2); + CheckCondition(2 == rocksdb_options_get_wal_recovery_mode(o)); + + rocksdb_options_set_compression(o, 5); + CheckCondition(5 == rocksdb_options_get_compression(o)); + + rocksdb_options_set_bottommost_compression(o, 4); + CheckCondition(4 == rocksdb_options_get_bottommost_compression(o)); + + rocksdb_options_set_compaction_style(o, 2); + CheckCondition(2 == rocksdb_options_get_compaction_style(o)); + + rocksdb_options_set_atomic_flush(o, 1); + CheckCondition(1 == rocksdb_options_get_atomic_flush(o)); + + rocksdb_options_set_manual_wal_flush(o, 1); + CheckCondition(1 == rocksdb_options_get_manual_wal_flush(o)); + + rocksdb_options_set_wal_compression(o, 1); + CheckCondition(1 == rocksdb_options_get_wal_compression(o)); + + rocksdb_options_set_experimental_mempurge_threshold(o, 29.0); + CheckCondition(29.0 == + rocksdb_options_get_experimental_mempurge_threshold(o)); + + CheckCondition(rocksdb_statistics_level_disable_all == + rocksdb_options_get_statistics_level(o)); + rocksdb_options_enable_statistics(o); + CheckCondition(rocksdb_statistics_level_disable_all != + rocksdb_options_get_statistics_level(o)); + rocksdb_options_set_statistics_level(o, rocksdb_statistics_level_all); + CheckCondition(rocksdb_statistics_level_all == + rocksdb_options_get_statistics_level(o)); + + /* Blob Options */ + rocksdb_options_set_enable_blob_files(o, 1); + CheckCondition(1 == rocksdb_options_get_enable_blob_files(o)); + + rocksdb_options_set_min_blob_size(o, 29); + CheckCondition(29 == rocksdb_options_get_min_blob_size(o)); + + rocksdb_options_set_blob_file_size(o, 30); + CheckCondition(30 == rocksdb_options_get_blob_file_size(o)); + + rocksdb_options_set_blob_compression_type(o, 4); + CheckCondition(4 == rocksdb_options_get_blob_compression_type(o)); + + rocksdb_options_set_enable_blob_gc(o, 1); + CheckCondition(1 == rocksdb_options_get_enable_blob_gc(o)); + + rocksdb_options_set_blob_gc_age_cutoff(o, 0.5); + CheckCondition(0.5 == rocksdb_options_get_blob_gc_age_cutoff(o)); + + rocksdb_options_set_blob_gc_force_threshold(o, 0.75); + CheckCondition(0.75 == rocksdb_options_get_blob_gc_force_threshold(o)); + + rocksdb_options_set_blob_compaction_readahead_size(o, 262144); + CheckCondition(262144 == + rocksdb_options_get_blob_compaction_readahead_size(o)); + + rocksdb_options_set_blob_file_starting_level(o, 5); + CheckCondition(5 == rocksdb_options_get_blob_file_starting_level(o)); + + rocksdb_options_set_prepopulate_blob_cache(o, 1 /* flush only */); + CheckCondition(1 == rocksdb_options_get_prepopulate_blob_cache(o)); + + // Create a copy that should be equal to the original. + rocksdb_options_t* copy; + copy = rocksdb_options_create_copy(o); + + CheckCondition(1 == rocksdb_options_get_allow_ingest_behind(copy)); + CheckCondition(10 == rocksdb_options_get_compaction_readahead_size(copy)); + CheckCondition(1 == rocksdb_options_get_create_if_missing(copy)); + CheckCondition(1 == + rocksdb_options_get_create_missing_column_families(copy)); + CheckCondition(1 == rocksdb_options_get_error_if_exists(copy)); + CheckCondition(1 == rocksdb_options_get_paranoid_checks(copy)); + CheckCondition(3 == rocksdb_options_get_info_log_level(copy)); + CheckCondition(100 == rocksdb_options_get_write_buffer_size(copy)); + CheckCondition(1000 == rocksdb_options_get_db_write_buffer_size(copy)); + CheckCondition(21 == rocksdb_options_get_max_open_files(copy)); + CheckCondition(5 == rocksdb_options_get_max_file_opening_threads(copy)); + CheckCondition(400 == rocksdb_options_get_max_total_wal_size(copy)); + CheckCondition(7 == rocksdb_options_get_num_levels(copy)); + CheckCondition( + 4 == rocksdb_options_get_level0_file_num_compaction_trigger(copy)); + CheckCondition(6 == + rocksdb_options_get_level0_slowdown_writes_trigger(copy)); + CheckCondition(8 == rocksdb_options_get_level0_stop_writes_trigger(copy)); + CheckCondition(256 == rocksdb_options_get_target_file_size_base(copy)); + CheckCondition(3 == rocksdb_options_get_target_file_size_multiplier(copy)); + CheckCondition(1024 == rocksdb_options_get_max_bytes_for_level_base(copy)); + CheckCondition( + 1 == rocksdb_options_get_level_compaction_dynamic_level_bytes(copy)); + CheckCondition(2.0 == + rocksdb_options_get_max_bytes_for_level_multiplier(copy)); + CheckCondition(1 == rocksdb_options_get_skip_stats_update_on_db_open(copy)); + CheckCondition( + 1 == rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open(copy)); + CheckCondition(97 == rocksdb_options_get_max_write_buffer_number(copy)); + CheckCondition(23 == + rocksdb_options_get_min_write_buffer_number_to_merge(copy)); + CheckCondition( + 64 == rocksdb_options_get_max_write_buffer_number_to_maintain(copy)); + CheckCondition(50000 == + rocksdb_options_get_max_write_buffer_size_to_maintain(copy)); + CheckCondition(1 == rocksdb_options_get_enable_pipelined_write(copy)); + CheckCondition(1 == rocksdb_options_get_unordered_write(copy)); + CheckCondition(123456 == rocksdb_options_get_max_subcompactions(copy)); + CheckCondition(2 == rocksdb_options_get_max_background_jobs(copy)); + CheckCondition(3 == rocksdb_options_get_max_background_compactions(copy)); + CheckCondition(5 == rocksdb_options_get_max_background_flushes(copy)); + CheckCondition(6 == rocksdb_options_get_max_log_file_size(copy)); + CheckCondition(7 == rocksdb_options_get_log_file_time_to_roll(copy)); + CheckCondition(8 == rocksdb_options_get_keep_log_file_num(copy)); + CheckCondition(9 == rocksdb_options_get_recycle_log_file_num(copy)); + CheckCondition( + 10 == rocksdb_options_get_soft_pending_compaction_bytes_limit(copy)); + CheckCondition( + 11 == rocksdb_options_get_hard_pending_compaction_bytes_limit(copy)); + CheckCondition(12 == rocksdb_options_get_max_manifest_file_size(copy)); + CheckCondition(13 == rocksdb_options_get_table_cache_numshardbits(copy)); + CheckCondition(14 == rocksdb_options_get_arena_block_size(copy)); + CheckCondition(1 == rocksdb_options_get_use_fsync(copy)); + CheckCondition(15 == rocksdb_options_get_WAL_ttl_seconds(copy)); + CheckCondition(16 == rocksdb_options_get_WAL_size_limit_MB(copy)); + CheckCondition(17 == rocksdb_options_get_manifest_preallocation_size(copy)); + CheckCondition(1 == rocksdb_options_get_allow_mmap_reads(copy)); + CheckCondition(1 == rocksdb_options_get_allow_mmap_writes(copy)); + CheckCondition(1 == rocksdb_options_get_use_direct_reads(copy)); + CheckCondition( + 1 == rocksdb_options_get_use_direct_io_for_flush_and_compaction(copy)); + CheckCondition(1 == rocksdb_options_get_is_fd_close_on_exec(copy)); + CheckCondition(18 == rocksdb_options_get_stats_dump_period_sec(copy)); + CheckCondition(5 == rocksdb_options_get_stats_persist_period_sec(copy)); + CheckCondition(1 == rocksdb_options_get_advise_random_on_open(copy)); + CheckCondition(3 == + rocksdb_options_get_access_hint_on_compaction_start(copy)); + CheckCondition(1 == rocksdb_options_get_use_adaptive_mutex(copy)); + CheckCondition(19 == rocksdb_options_get_bytes_per_sync(copy)); + CheckCondition(20 == rocksdb_options_get_wal_bytes_per_sync(copy)); + CheckCondition(21 == + rocksdb_options_get_writable_file_max_buffer_size(copy)); + CheckCondition(1 == + rocksdb_options_get_allow_concurrent_memtable_write(copy)); + CheckCondition( + 1 == rocksdb_options_get_enable_write_thread_adaptive_yield(copy)); + CheckCondition(22 == + rocksdb_options_get_max_sequential_skip_in_iterations(copy)); + CheckCondition(1 == rocksdb_options_get_disable_auto_compactions(copy)); + CheckCondition(1 == rocksdb_options_get_optimize_filters_for_hits(copy)); + CheckCondition( + 23 == rocksdb_options_get_delete_obsolete_files_period_micros(copy)); + CheckCondition(2.0 == + rocksdb_options_get_memtable_prefix_bloom_size_ratio(copy)); + CheckCondition(24 == rocksdb_options_get_max_compaction_bytes(copy)); + CheckCondition(25 == rocksdb_options_get_memtable_huge_page_size(copy)); + CheckCondition(26 == rocksdb_options_get_max_successive_merges(copy)); + CheckCondition(27 == rocksdb_options_get_bloom_locality(copy)); + CheckCondition(1 == rocksdb_options_get_inplace_update_support(copy)); + CheckCondition(28 == rocksdb_options_get_inplace_update_num_locks(copy)); + CheckCondition(1 == rocksdb_options_get_report_bg_io_stats(copy)); + CheckCondition(2 == rocksdb_options_get_wal_recovery_mode(copy)); + CheckCondition(5 == rocksdb_options_get_compression(copy)); + CheckCondition(4 == rocksdb_options_get_bottommost_compression(copy)); + CheckCondition(2 == rocksdb_options_get_compaction_style(copy)); + CheckCondition(1 == rocksdb_options_get_atomic_flush(copy)); + CheckCondition(29.0 == + rocksdb_options_get_experimental_mempurge_threshold(copy)); + + // Copies should be independent. + rocksdb_options_set_allow_ingest_behind(copy, 0); + CheckCondition(0 == rocksdb_options_get_allow_ingest_behind(copy)); + CheckCondition(1 == rocksdb_options_get_allow_ingest_behind(o)); + + rocksdb_options_compaction_readahead_size(copy, 20); + CheckCondition(20 == rocksdb_options_get_compaction_readahead_size(copy)); + CheckCondition(10 == rocksdb_options_get_compaction_readahead_size(o)); + + rocksdb_options_set_create_if_missing(copy, 0); + CheckCondition(0 == rocksdb_options_get_create_if_missing(copy)); + CheckCondition(1 == rocksdb_options_get_create_if_missing(o)); + + rocksdb_options_set_create_missing_column_families(copy, 0); + CheckCondition(0 == + rocksdb_options_get_create_missing_column_families(copy)); + CheckCondition(1 == rocksdb_options_get_create_missing_column_families(o)); + + rocksdb_options_set_error_if_exists(copy, 0); + CheckCondition(0 == rocksdb_options_get_error_if_exists(copy)); + CheckCondition(1 == rocksdb_options_get_error_if_exists(o)); + + rocksdb_options_set_paranoid_checks(copy, 0); + CheckCondition(0 == rocksdb_options_get_paranoid_checks(copy)); + CheckCondition(1 == rocksdb_options_get_paranoid_checks(o)); + + rocksdb_options_set_info_log_level(copy, 2); + CheckCondition(2 == rocksdb_options_get_info_log_level(copy)); + CheckCondition(3 == rocksdb_options_get_info_log_level(o)); + + rocksdb_options_set_write_buffer_size(copy, 200); + CheckCondition(200 == rocksdb_options_get_write_buffer_size(copy)); + CheckCondition(100 == rocksdb_options_get_write_buffer_size(o)); + + rocksdb_options_set_db_write_buffer_size(copy, 2000); + CheckCondition(2000 == rocksdb_options_get_db_write_buffer_size(copy)); + CheckCondition(1000 == rocksdb_options_get_db_write_buffer_size(o)); + + rocksdb_options_set_max_open_files(copy, 42); + CheckCondition(42 == rocksdb_options_get_max_open_files(copy)); + CheckCondition(21 == rocksdb_options_get_max_open_files(o)); + + rocksdb_options_set_max_file_opening_threads(copy, 3); + CheckCondition(3 == rocksdb_options_get_max_file_opening_threads(copy)); + CheckCondition(5 == rocksdb_options_get_max_file_opening_threads(o)); + + rocksdb_options_set_max_total_wal_size(copy, 4000); + CheckCondition(4000 == rocksdb_options_get_max_total_wal_size(copy)); + CheckCondition(400 == rocksdb_options_get_max_total_wal_size(o)); + + rocksdb_options_set_num_levels(copy, 6); + CheckCondition(6 == rocksdb_options_get_num_levels(copy)); + CheckCondition(7 == rocksdb_options_get_num_levels(o)); + + rocksdb_options_set_level0_file_num_compaction_trigger(copy, 14); + CheckCondition( + 14 == rocksdb_options_get_level0_file_num_compaction_trigger(copy)); + CheckCondition(4 == + rocksdb_options_get_level0_file_num_compaction_trigger(o)); + + rocksdb_options_set_level0_slowdown_writes_trigger(copy, 61); + CheckCondition(61 == + rocksdb_options_get_level0_slowdown_writes_trigger(copy)); + CheckCondition(6 == rocksdb_options_get_level0_slowdown_writes_trigger(o)); + + rocksdb_options_set_level0_stop_writes_trigger(copy, 17); + CheckCondition(17 == rocksdb_options_get_level0_stop_writes_trigger(copy)); + CheckCondition(8 == rocksdb_options_get_level0_stop_writes_trigger(o)); + + rocksdb_options_set_target_file_size_base(copy, 128); + CheckCondition(128 == rocksdb_options_get_target_file_size_base(copy)); + CheckCondition(256 == rocksdb_options_get_target_file_size_base(o)); + + rocksdb_options_set_target_file_size_multiplier(copy, 13); + CheckCondition(13 == rocksdb_options_get_target_file_size_multiplier(copy)); + CheckCondition(3 == rocksdb_options_get_target_file_size_multiplier(o)); + + rocksdb_options_set_max_bytes_for_level_base(copy, 900); + CheckCondition(900 == rocksdb_options_get_max_bytes_for_level_base(copy)); + CheckCondition(1024 == rocksdb_options_get_max_bytes_for_level_base(o)); + + rocksdb_options_set_level_compaction_dynamic_level_bytes(copy, 0); + CheckCondition( + 0 == rocksdb_options_get_level_compaction_dynamic_level_bytes(copy)); + CheckCondition(1 == + rocksdb_options_get_level_compaction_dynamic_level_bytes(o)); + + rocksdb_options_set_max_bytes_for_level_multiplier(copy, 8.0); + CheckCondition(8.0 == + rocksdb_options_get_max_bytes_for_level_multiplier(copy)); + CheckCondition(2.0 == + rocksdb_options_get_max_bytes_for_level_multiplier(o)); + + rocksdb_options_set_skip_stats_update_on_db_open(copy, 0); + CheckCondition(0 == rocksdb_options_get_skip_stats_update_on_db_open(copy)); + CheckCondition(1 == rocksdb_options_get_skip_stats_update_on_db_open(o)); + + rocksdb_options_set_skip_checking_sst_file_sizes_on_db_open(copy, 0); + CheckCondition( + 0 == rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open(copy)); + CheckCondition( + 1 == rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open(o)); + + rocksdb_options_set_max_write_buffer_number(copy, 2000); + CheckCondition(2000 == rocksdb_options_get_max_write_buffer_number(copy)); + CheckCondition(97 == rocksdb_options_get_max_write_buffer_number(o)); + + rocksdb_options_set_min_write_buffer_number_to_merge(copy, 146); + CheckCondition(146 == + rocksdb_options_get_min_write_buffer_number_to_merge(copy)); + CheckCondition(23 == + rocksdb_options_get_min_write_buffer_number_to_merge(o)); + + rocksdb_options_set_max_write_buffer_number_to_maintain(copy, 128); + CheckCondition( + 128 == rocksdb_options_get_max_write_buffer_number_to_maintain(copy)); + CheckCondition(64 == + rocksdb_options_get_max_write_buffer_number_to_maintain(o)); + + rocksdb_options_set_max_write_buffer_size_to_maintain(copy, 9000); + CheckCondition(9000 == + rocksdb_options_get_max_write_buffer_size_to_maintain(copy)); + CheckCondition(50000 == + rocksdb_options_get_max_write_buffer_size_to_maintain(o)); + + rocksdb_options_set_enable_pipelined_write(copy, 0); + CheckCondition(0 == rocksdb_options_get_enable_pipelined_write(copy)); + CheckCondition(1 == rocksdb_options_get_enable_pipelined_write(o)); + + rocksdb_options_set_unordered_write(copy, 0); + CheckCondition(0 == rocksdb_options_get_unordered_write(copy)); + CheckCondition(1 == rocksdb_options_get_unordered_write(o)); + + rocksdb_options_set_max_subcompactions(copy, 90001); + CheckCondition(90001 == rocksdb_options_get_max_subcompactions(copy)); + CheckCondition(123456 == rocksdb_options_get_max_subcompactions(o)); + + rocksdb_options_set_max_background_jobs(copy, 12); + CheckCondition(12 == rocksdb_options_get_max_background_jobs(copy)); + CheckCondition(2 == rocksdb_options_get_max_background_jobs(o)); + + rocksdb_options_set_max_background_compactions(copy, 13); + CheckCondition(13 == rocksdb_options_get_max_background_compactions(copy)); + CheckCondition(3 == rocksdb_options_get_max_background_compactions(o)); + + rocksdb_options_set_max_background_flushes(copy, 15); + CheckCondition(15 == rocksdb_options_get_max_background_flushes(copy)); + CheckCondition(5 == rocksdb_options_get_max_background_flushes(o)); + + rocksdb_options_set_max_log_file_size(copy, 16); + CheckCondition(16 == rocksdb_options_get_max_log_file_size(copy)); + CheckCondition(6 == rocksdb_options_get_max_log_file_size(o)); + + rocksdb_options_set_log_file_time_to_roll(copy, 17); + CheckCondition(17 == rocksdb_options_get_log_file_time_to_roll(copy)); + CheckCondition(7 == rocksdb_options_get_log_file_time_to_roll(o)); + + rocksdb_options_set_keep_log_file_num(copy, 18); + CheckCondition(18 == rocksdb_options_get_keep_log_file_num(copy)); + CheckCondition(8 == rocksdb_options_get_keep_log_file_num(o)); + + rocksdb_options_set_recycle_log_file_num(copy, 19); + CheckCondition(19 == rocksdb_options_get_recycle_log_file_num(copy)); + CheckCondition(9 == rocksdb_options_get_recycle_log_file_num(o)); + + rocksdb_options_set_soft_pending_compaction_bytes_limit(copy, 110); + CheckCondition( + 110 == rocksdb_options_get_soft_pending_compaction_bytes_limit(copy)); + CheckCondition(10 == + rocksdb_options_get_soft_pending_compaction_bytes_limit(o)); + + rocksdb_options_set_hard_pending_compaction_bytes_limit(copy, 111); + CheckCondition( + 111 == rocksdb_options_get_hard_pending_compaction_bytes_limit(copy)); + CheckCondition(11 == + rocksdb_options_get_hard_pending_compaction_bytes_limit(o)); + + rocksdb_options_set_max_manifest_file_size(copy, 112); + CheckCondition(112 == rocksdb_options_get_max_manifest_file_size(copy)); + CheckCondition(12 == rocksdb_options_get_max_manifest_file_size(o)); + + rocksdb_options_set_table_cache_numshardbits(copy, 113); + CheckCondition(113 == rocksdb_options_get_table_cache_numshardbits(copy)); + CheckCondition(13 == rocksdb_options_get_table_cache_numshardbits(o)); + + rocksdb_options_set_arena_block_size(copy, 114); + CheckCondition(114 == rocksdb_options_get_arena_block_size(copy)); + CheckCondition(14 == rocksdb_options_get_arena_block_size(o)); + + rocksdb_options_set_use_fsync(copy, 0); + CheckCondition(0 == rocksdb_options_get_use_fsync(copy)); + CheckCondition(1 == rocksdb_options_get_use_fsync(o)); + + rocksdb_options_set_WAL_ttl_seconds(copy, 115); + CheckCondition(115 == rocksdb_options_get_WAL_ttl_seconds(copy)); + CheckCondition(15 == rocksdb_options_get_WAL_ttl_seconds(o)); + + rocksdb_options_set_WAL_size_limit_MB(copy, 116); + CheckCondition(116 == rocksdb_options_get_WAL_size_limit_MB(copy)); + CheckCondition(16 == rocksdb_options_get_WAL_size_limit_MB(o)); + + rocksdb_options_set_manifest_preallocation_size(copy, 117); + CheckCondition(117 == + rocksdb_options_get_manifest_preallocation_size(copy)); + CheckCondition(17 == rocksdb_options_get_manifest_preallocation_size(o)); + + rocksdb_options_set_allow_mmap_reads(copy, 0); + CheckCondition(0 == rocksdb_options_get_allow_mmap_reads(copy)); + CheckCondition(1 == rocksdb_options_get_allow_mmap_reads(o)); + + rocksdb_options_set_allow_mmap_writes(copy, 0); + CheckCondition(0 == rocksdb_options_get_allow_mmap_writes(copy)); + CheckCondition(1 == rocksdb_options_get_allow_mmap_writes(o)); + + rocksdb_options_set_use_direct_reads(copy, 0); + CheckCondition(0 == rocksdb_options_get_use_direct_reads(copy)); + CheckCondition(1 == rocksdb_options_get_use_direct_reads(o)); + + rocksdb_options_set_use_direct_io_for_flush_and_compaction(copy, 0); + CheckCondition( + 0 == rocksdb_options_get_use_direct_io_for_flush_and_compaction(copy)); + CheckCondition( + 1 == rocksdb_options_get_use_direct_io_for_flush_and_compaction(o)); + + rocksdb_options_set_is_fd_close_on_exec(copy, 0); + CheckCondition(0 == rocksdb_options_get_is_fd_close_on_exec(copy)); + CheckCondition(1 == rocksdb_options_get_is_fd_close_on_exec(o)); + + rocksdb_options_set_stats_dump_period_sec(copy, 218); + CheckCondition(218 == rocksdb_options_get_stats_dump_period_sec(copy)); + CheckCondition(18 == rocksdb_options_get_stats_dump_period_sec(o)); + + rocksdb_options_set_stats_persist_period_sec(copy, 600); + CheckCondition(600 == rocksdb_options_get_stats_persist_period_sec(copy)); + CheckCondition(5 == rocksdb_options_get_stats_persist_period_sec(o)); + + rocksdb_options_set_advise_random_on_open(copy, 0); + CheckCondition(0 == rocksdb_options_get_advise_random_on_open(copy)); + CheckCondition(1 == rocksdb_options_get_advise_random_on_open(o)); + + rocksdb_options_set_access_hint_on_compaction_start(copy, 2); + CheckCondition(2 == + rocksdb_options_get_access_hint_on_compaction_start(copy)); + CheckCondition(3 == rocksdb_options_get_access_hint_on_compaction_start(o)); + + rocksdb_options_set_use_adaptive_mutex(copy, 0); + CheckCondition(0 == rocksdb_options_get_use_adaptive_mutex(copy)); + CheckCondition(1 == rocksdb_options_get_use_adaptive_mutex(o)); + + rocksdb_options_set_bytes_per_sync(copy, 219); + CheckCondition(219 == rocksdb_options_get_bytes_per_sync(copy)); + CheckCondition(19 == rocksdb_options_get_bytes_per_sync(o)); + + rocksdb_options_set_wal_bytes_per_sync(copy, 120); + CheckCondition(120 == rocksdb_options_get_wal_bytes_per_sync(copy)); + CheckCondition(20 == rocksdb_options_get_wal_bytes_per_sync(o)); + + rocksdb_options_set_writable_file_max_buffer_size(copy, 121); + CheckCondition(121 == + rocksdb_options_get_writable_file_max_buffer_size(copy)); + CheckCondition(21 == rocksdb_options_get_writable_file_max_buffer_size(o)); + + rocksdb_options_set_allow_concurrent_memtable_write(copy, 0); + CheckCondition(0 == + rocksdb_options_get_allow_concurrent_memtable_write(copy)); + CheckCondition(1 == rocksdb_options_get_allow_concurrent_memtable_write(o)); + + rocksdb_options_set_enable_write_thread_adaptive_yield(copy, 0); + CheckCondition( + 0 == rocksdb_options_get_enable_write_thread_adaptive_yield(copy)); + CheckCondition(1 == + rocksdb_options_get_enable_write_thread_adaptive_yield(o)); + + rocksdb_options_set_max_sequential_skip_in_iterations(copy, 122); + CheckCondition(122 == + rocksdb_options_get_max_sequential_skip_in_iterations(copy)); + CheckCondition(22 == + rocksdb_options_get_max_sequential_skip_in_iterations(o)); + + rocksdb_options_set_disable_auto_compactions(copy, 0); + CheckCondition(0 == rocksdb_options_get_disable_auto_compactions(copy)); + CheckCondition(1 == rocksdb_options_get_disable_auto_compactions(o)); + + rocksdb_options_set_optimize_filters_for_hits(copy, 0); + CheckCondition(0 == rocksdb_options_get_optimize_filters_for_hits(copy)); + CheckCondition(1 == rocksdb_options_get_optimize_filters_for_hits(o)); + + rocksdb_options_set_delete_obsolete_files_period_micros(copy, 123); + CheckCondition( + 123 == rocksdb_options_get_delete_obsolete_files_period_micros(copy)); + CheckCondition(23 == + rocksdb_options_get_delete_obsolete_files_period_micros(o)); + + rocksdb_options_set_memtable_prefix_bloom_size_ratio(copy, 4.0); + CheckCondition(4.0 == + rocksdb_options_get_memtable_prefix_bloom_size_ratio(copy)); + CheckCondition(2.0 == + rocksdb_options_get_memtable_prefix_bloom_size_ratio(o)); + + rocksdb_options_set_max_compaction_bytes(copy, 124); + CheckCondition(124 == rocksdb_options_get_max_compaction_bytes(copy)); + CheckCondition(24 == rocksdb_options_get_max_compaction_bytes(o)); + + rocksdb_options_set_memtable_huge_page_size(copy, 125); + CheckCondition(125 == rocksdb_options_get_memtable_huge_page_size(copy)); + CheckCondition(25 == rocksdb_options_get_memtable_huge_page_size(o)); + + rocksdb_options_set_max_successive_merges(copy, 126); + CheckCondition(126 == rocksdb_options_get_max_successive_merges(copy)); + CheckCondition(26 == rocksdb_options_get_max_successive_merges(o)); + + rocksdb_options_set_bloom_locality(copy, 127); + CheckCondition(127 == rocksdb_options_get_bloom_locality(copy)); + CheckCondition(27 == rocksdb_options_get_bloom_locality(o)); + + rocksdb_options_set_inplace_update_support(copy, 0); + CheckCondition(0 == rocksdb_options_get_inplace_update_support(copy)); + CheckCondition(1 == rocksdb_options_get_inplace_update_support(o)); + + rocksdb_options_set_inplace_update_num_locks(copy, 128); + CheckCondition(128 == rocksdb_options_get_inplace_update_num_locks(copy)); + CheckCondition(28 == rocksdb_options_get_inplace_update_num_locks(o)); + + rocksdb_options_set_report_bg_io_stats(copy, 0); + CheckCondition(0 == rocksdb_options_get_report_bg_io_stats(copy)); + CheckCondition(1 == rocksdb_options_get_report_bg_io_stats(o)); + + rocksdb_options_set_wal_recovery_mode(copy, 1); + CheckCondition(1 == rocksdb_options_get_wal_recovery_mode(copy)); + CheckCondition(2 == rocksdb_options_get_wal_recovery_mode(o)); + + rocksdb_options_set_compression(copy, 4); + CheckCondition(4 == rocksdb_options_get_compression(copy)); + CheckCondition(5 == rocksdb_options_get_compression(o)); + + rocksdb_options_set_bottommost_compression(copy, 3); + CheckCondition(3 == rocksdb_options_get_bottommost_compression(copy)); + CheckCondition(4 == rocksdb_options_get_bottommost_compression(o)); + + rocksdb_options_set_compaction_style(copy, 1); + CheckCondition(1 == rocksdb_options_get_compaction_style(copy)); + CheckCondition(2 == rocksdb_options_get_compaction_style(o)); + + rocksdb_options_set_atomic_flush(copy, 0); + CheckCondition(0 == rocksdb_options_get_atomic_flush(copy)); + CheckCondition(1 == rocksdb_options_get_atomic_flush(o)); + + rocksdb_options_set_experimental_mempurge_threshold(copy, 229.0); + CheckCondition(229.0 == + rocksdb_options_get_experimental_mempurge_threshold(copy)); + CheckCondition(29.0 == + rocksdb_options_get_experimental_mempurge_threshold(o)); + + rocksdb_options_destroy(copy); + rocksdb_options_destroy(o); + } + + StartPhase("read_options"); + { + rocksdb_readoptions_t* ro; + ro = rocksdb_readoptions_create(); + + rocksdb_readoptions_set_verify_checksums(ro, 1); + CheckCondition(1 == rocksdb_readoptions_get_verify_checksums(ro)); + + rocksdb_readoptions_set_fill_cache(ro, 1); + CheckCondition(1 == rocksdb_readoptions_get_fill_cache(ro)); + + rocksdb_readoptions_set_read_tier(ro, 2); + CheckCondition(2 == rocksdb_readoptions_get_read_tier(ro)); + + rocksdb_readoptions_set_tailing(ro, 1); + CheckCondition(1 == rocksdb_readoptions_get_tailing(ro)); + + rocksdb_readoptions_set_readahead_size(ro, 100); + CheckCondition(100 == rocksdb_readoptions_get_readahead_size(ro)); + + rocksdb_readoptions_set_prefix_same_as_start(ro, 1); + CheckCondition(1 == rocksdb_readoptions_get_prefix_same_as_start(ro)); + + rocksdb_readoptions_set_pin_data(ro, 1); + CheckCondition(1 == rocksdb_readoptions_get_pin_data(ro)); + + rocksdb_readoptions_set_total_order_seek(ro, 1); + CheckCondition(1 == rocksdb_readoptions_get_total_order_seek(ro)); + + rocksdb_readoptions_set_max_skippable_internal_keys(ro, 200); + CheckCondition(200 == + rocksdb_readoptions_get_max_skippable_internal_keys(ro)); + + rocksdb_readoptions_set_background_purge_on_iterator_cleanup(ro, 1); + CheckCondition( + 1 == rocksdb_readoptions_get_background_purge_on_iterator_cleanup(ro)); + + rocksdb_readoptions_set_ignore_range_deletions(ro, 1); + CheckCondition(1 == rocksdb_readoptions_get_ignore_range_deletions(ro)); + + rocksdb_readoptions_set_deadline(ro, 300); + CheckCondition(300 == rocksdb_readoptions_get_deadline(ro)); + + rocksdb_readoptions_set_io_timeout(ro, 400); + CheckCondition(400 == rocksdb_readoptions_get_io_timeout(ro)); + + rocksdb_readoptions_set_async_io(ro, 1); + CheckCondition(1 == rocksdb_readoptions_get_async_io(ro)); + + rocksdb_readoptions_destroy(ro); + } + + StartPhase("write_options"); + { + rocksdb_writeoptions_t* wo; + wo = rocksdb_writeoptions_create(); + + rocksdb_writeoptions_set_sync(wo, 1); + CheckCondition(1 == rocksdb_writeoptions_get_sync(wo)); + + rocksdb_writeoptions_disable_WAL(wo, 1); + CheckCondition(1 == rocksdb_writeoptions_get_disable_WAL(wo)); + + rocksdb_writeoptions_set_ignore_missing_column_families(wo, 1); + CheckCondition(1 == + rocksdb_writeoptions_get_ignore_missing_column_families(wo)); + + rocksdb_writeoptions_set_no_slowdown(wo, 1); + CheckCondition(1 == rocksdb_writeoptions_get_no_slowdown(wo)); + + rocksdb_writeoptions_set_low_pri(wo, 1); + CheckCondition(1 == rocksdb_writeoptions_get_low_pri(wo)); + + rocksdb_writeoptions_set_memtable_insert_hint_per_batch(wo, 1); + CheckCondition(1 == + rocksdb_writeoptions_get_memtable_insert_hint_per_batch(wo)); + + rocksdb_writeoptions_destroy(wo); + } + + StartPhase("compact_options"); + { + rocksdb_compactoptions_t* co; + co = rocksdb_compactoptions_create(); + + rocksdb_compactoptions_set_exclusive_manual_compaction(co, 1); + CheckCondition(1 == + rocksdb_compactoptions_get_exclusive_manual_compaction(co)); + + rocksdb_compactoptions_set_bottommost_level_compaction(co, 1); + CheckCondition(1 == + rocksdb_compactoptions_get_bottommost_level_compaction(co)); + + rocksdb_compactoptions_set_change_level(co, 1); + CheckCondition(1 == rocksdb_compactoptions_get_change_level(co)); + + rocksdb_compactoptions_set_target_level(co, 1); + CheckCondition(1 == rocksdb_compactoptions_get_target_level(co)); + + rocksdb_compactoptions_destroy(co); + } + + StartPhase("flush_options"); + { + rocksdb_flushoptions_t* fo; + fo = rocksdb_flushoptions_create(); + + rocksdb_flushoptions_set_wait(fo, 1); + CheckCondition(1 == rocksdb_flushoptions_get_wait(fo)); + + rocksdb_flushoptions_destroy(fo); + } + + StartPhase("cache_options"); + { + rocksdb_cache_t* co; + co = rocksdb_cache_create_lru(100); + CheckCondition(100 == rocksdb_cache_get_capacity(co)); + + rocksdb_cache_set_capacity(co, 200); + CheckCondition(200 == rocksdb_cache_get_capacity(co)); + + rocksdb_cache_destroy(co); + } + + StartPhase("jemalloc_nodump_allocator"); + { + rocksdb_memory_allocator_t* allocator; + allocator = rocksdb_jemalloc_nodump_allocator_create(&err); + if (err != NULL) { + // not supported on all platforms, allow unsupported error + const char* ni = "Not implemented: "; + size_t ni_len = strlen(ni); + size_t err_len = strlen(err); + + CheckCondition(err_len >= ni_len); + CheckCondition(memcmp(ni, err, ni_len) == 0); + Free(&err); + } else { + rocksdb_cache_t* co; + rocksdb_lru_cache_options_t* copts; + + copts = rocksdb_lru_cache_options_create(); + + rocksdb_lru_cache_options_set_capacity(copts, 100); + rocksdb_lru_cache_options_set_memory_allocator(copts, allocator); + + co = rocksdb_cache_create_lru_opts(copts); + CheckCondition(100 == rocksdb_cache_get_capacity(co)); + + rocksdb_cache_destroy(co); + rocksdb_lru_cache_options_destroy(copts); + } + rocksdb_memory_allocator_destroy(allocator); + } + + StartPhase("env"); + { + rocksdb_env_t* e; + e = rocksdb_create_default_env(); + + rocksdb_env_set_background_threads(e, 10); + CheckCondition(10 == rocksdb_env_get_background_threads(e)); + + rocksdb_env_set_high_priority_background_threads(e, 20); + CheckCondition(20 == rocksdb_env_get_high_priority_background_threads(e)); + + rocksdb_env_set_low_priority_background_threads(e, 30); + CheckCondition(30 == rocksdb_env_get_low_priority_background_threads(e)); + + rocksdb_env_set_bottom_priority_background_threads(e, 40); + CheckCondition(40 == rocksdb_env_get_bottom_priority_background_threads(e)); + + rocksdb_env_destroy(e); + } + + StartPhase("universal_compaction_options"); + { + rocksdb_universal_compaction_options_t* uco; + uco = rocksdb_universal_compaction_options_create(); + + rocksdb_universal_compaction_options_set_size_ratio(uco, 5); + CheckCondition(5 == + rocksdb_universal_compaction_options_get_size_ratio(uco)); + + rocksdb_universal_compaction_options_set_min_merge_width(uco, 15); + CheckCondition( + 15 == rocksdb_universal_compaction_options_get_min_merge_width(uco)); + + rocksdb_universal_compaction_options_set_max_merge_width(uco, 25); + CheckCondition( + 25 == rocksdb_universal_compaction_options_get_max_merge_width(uco)); + + rocksdb_universal_compaction_options_set_max_size_amplification_percent(uco, + 35); + CheckCondition( + 35 == + rocksdb_universal_compaction_options_get_max_size_amplification_percent( + uco)); + + rocksdb_universal_compaction_options_set_compression_size_percent(uco, 45); + CheckCondition( + 45 == + rocksdb_universal_compaction_options_get_compression_size_percent(uco)); + + rocksdb_universal_compaction_options_set_stop_style(uco, 1); + CheckCondition(1 == + rocksdb_universal_compaction_options_get_stop_style(uco)); + + rocksdb_universal_compaction_options_destroy(uco); + } + + StartPhase("fifo_compaction_options"); + { + rocksdb_fifo_compaction_options_t* fco; + fco = rocksdb_fifo_compaction_options_create(); + + rocksdb_fifo_compaction_options_set_max_table_files_size(fco, 100000); + CheckCondition( + 100000 == + rocksdb_fifo_compaction_options_get_max_table_files_size(fco)); + + rocksdb_fifo_compaction_options_destroy(fco); + } + + StartPhase("backup_engine_option"); + { + rocksdb_backup_engine_options_t* bdo; + bdo = rocksdb_backup_engine_options_create("path"); + + rocksdb_backup_engine_options_set_share_table_files(bdo, 1); + CheckCondition(1 == + rocksdb_backup_engine_options_get_share_table_files(bdo)); + + rocksdb_backup_engine_options_set_sync(bdo, 1); + CheckCondition(1 == rocksdb_backup_engine_options_get_sync(bdo)); + + rocksdb_backup_engine_options_set_destroy_old_data(bdo, 1); + CheckCondition(1 == + rocksdb_backup_engine_options_get_destroy_old_data(bdo)); + + rocksdb_backup_engine_options_set_backup_log_files(bdo, 1); + CheckCondition(1 == + rocksdb_backup_engine_options_get_backup_log_files(bdo)); + + rocksdb_backup_engine_options_set_backup_rate_limit(bdo, 123); + CheckCondition(123 == + rocksdb_backup_engine_options_get_backup_rate_limit(bdo)); + + rocksdb_backup_engine_options_set_restore_rate_limit(bdo, 37); + CheckCondition(37 == + rocksdb_backup_engine_options_get_restore_rate_limit(bdo)); + + rocksdb_backup_engine_options_set_max_background_operations(bdo, 20); + CheckCondition( + 20 == rocksdb_backup_engine_options_get_max_background_operations(bdo)); + + rocksdb_backup_engine_options_set_callback_trigger_interval_size(bdo, 9000); + CheckCondition( + 9000 == + rocksdb_backup_engine_options_get_callback_trigger_interval_size(bdo)); + + rocksdb_backup_engine_options_set_max_valid_backups_to_open(bdo, 40); + CheckCondition( + 40 == rocksdb_backup_engine_options_get_max_valid_backups_to_open(bdo)); + + rocksdb_backup_engine_options_set_share_files_with_checksum_naming(bdo, 2); + CheckCondition( + 2 == rocksdb_backup_engine_options_get_share_files_with_checksum_naming( + bdo)); + + rocksdb_backup_engine_options_destroy(bdo); + } + + StartPhase("compression_options"); + { + rocksdb_options_t* co; + co = rocksdb_options_create(); + + rocksdb_options_set_compression_options_zstd_max_train_bytes(co, 100); + CheckCondition( + 100 == + rocksdb_options_get_compression_options_zstd_max_train_bytes(co)); + + rocksdb_options_set_compression_options_parallel_threads(co, 2); + CheckCondition( + 2 == rocksdb_options_get_compression_options_parallel_threads(co)); + + rocksdb_options_set_compression_options_max_dict_buffer_bytes(co, 200); + CheckCondition( + 200 == + rocksdb_options_get_compression_options_max_dict_buffer_bytes(co)); + + rocksdb_options_set_compression_options_use_zstd_dict_trainer(co, 0); + CheckCondition( + 0 == rocksdb_options_get_compression_options_use_zstd_dict_trainer(co)); + rocksdb_options_destroy(co); + } + + StartPhase("iterate_upper_bound"); + { + // Create new empty database + rocksdb_close(db); + rocksdb_destroy_db(options, dbname, &err); + CheckNoError(err); + + rocksdb_options_set_prefix_extractor(options, NULL); + db = rocksdb_open(options, dbname, &err); + CheckNoError(err); + + rocksdb_put(db, woptions, "a", 1, "0", 1, &err); + CheckNoError(err); + rocksdb_put(db, woptions, "foo", 3, "bar", 3, &err); + CheckNoError(err); + rocksdb_put(db, woptions, "foo1", 4, "bar1", 4, &err); + CheckNoError(err); + rocksdb_put(db, woptions, "g1", 2, "0", 1, &err); + CheckNoError(err); + + // testing basic case with no iterate_upper_bound and no prefix_extractor + { + rocksdb_readoptions_set_iterate_upper_bound(roptions, NULL, 0); + rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions); + + rocksdb_iter_seek(iter, "foo", 3); + CheckCondition(rocksdb_iter_valid(iter)); + CheckIter(iter, "foo", "bar"); + + rocksdb_iter_next(iter); + CheckCondition(rocksdb_iter_valid(iter)); + CheckIter(iter, "foo1", "bar1"); + + rocksdb_iter_next(iter); + CheckCondition(rocksdb_iter_valid(iter)); + CheckIter(iter, "g1", "0"); + + rocksdb_iter_destroy(iter); + } + + // testing iterate_upper_bound and forward iterator + // to make sure it stops at bound + { + // iterate_upper_bound points beyond the last expected entry + rocksdb_readoptions_set_iterate_upper_bound(roptions, "foo2", 4); + + rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions); + + rocksdb_iter_seek(iter, "foo", 3); + CheckCondition(rocksdb_iter_valid(iter)); + CheckIter(iter, "foo", "bar"); + + rocksdb_iter_next(iter); + CheckCondition(rocksdb_iter_valid(iter)); + CheckIter(iter, "foo1", "bar1"); + + rocksdb_iter_next(iter); + // should stop here... + CheckCondition(!rocksdb_iter_valid(iter)); + + rocksdb_iter_destroy(iter); + rocksdb_readoptions_set_iterate_upper_bound(roptions, NULL, 0); + } + } + + StartPhase("transactions"); + { + rocksdb_close(db); + rocksdb_destroy_db(options, dbname, &err); + CheckNoError(err); + + // open a TransactionDB + txn_db_options = rocksdb_transactiondb_options_create(); + txn_options = rocksdb_transaction_options_create(); + rocksdb_options_set_create_if_missing(options, 1); + txn_db = rocksdb_transactiondb_open(options, txn_db_options, dbname, &err); + CheckNoError(err); + + // put outside a transaction + rocksdb_transactiondb_put(txn_db, woptions, "foo", 3, "hello", 5, &err); + CheckNoError(err); + CheckTxnDBGet(txn_db, roptions, "foo", "hello"); + CheckTxnDBPinGet(txn_db, roptions, "foo", "hello"); + + // delete from outside transaction + rocksdb_transactiondb_delete(txn_db, woptions, "foo", 3, &err); + CheckNoError(err); + CheckTxnDBGet(txn_db, roptions, "foo", NULL); + CheckTxnDBPinGet(txn_db, roptions, "foo", NULL); + + // write batch into TransactionDB + rocksdb_writebatch_t* wb = rocksdb_writebatch_create(); + rocksdb_writebatch_put(wb, "foo", 3, "a", 1); + rocksdb_writebatch_clear(wb); + rocksdb_writebatch_put(wb, "bar", 3, "b", 1); + rocksdb_writebatch_put(wb, "box", 3, "c", 1); + rocksdb_writebatch_delete(wb, "bar", 3); + rocksdb_transactiondb_write(txn_db, woptions, wb, &err); + rocksdb_writebatch_destroy(wb); + CheckTxnDBGet(txn_db, roptions, "box", "c"); + CheckTxnDBPinGet(txn_db, roptions, "box", "c"); + CheckNoError(err); + + // multi get + { + const char* keys[3] = {"box", "foo", "notfound"}; + const size_t keys_sizes[3] = {3, 3, 8}; + char* vals[3]; + size_t vals_sizes[3]; + char* errs[3]; + const char* expected[3] = {"c", NULL, NULL}; + rocksdb_transactiondb_multi_get(txn_db, roptions, 3, keys, keys_sizes, + vals, vals_sizes, errs); + CheckMultiGetValues(3, vals, vals_sizes, errs, expected); + } + + // begin a transaction + txn = rocksdb_transaction_begin(txn_db, woptions, txn_options, NULL); + // put + rocksdb_transaction_put(txn, "foo", 3, "hello", 5, &err); + CheckNoError(err); + CheckTxnGet(txn, roptions, "foo", "hello"); + CheckTxnPinGet(txn, roptions, "foo", "hello"); + { + const char* keys[3] = {"box", "foo", "notfound"}; + const size_t keys_sizes[3] = {3, 3, 8}; + char* vals[3]; + size_t vals_sizes[3]; + char* errs[3]; + const char* expected[3] = {"c", "hello", NULL}; + rocksdb_transaction_multi_get(txn, roptions, 3, keys, keys_sizes, vals, + vals_sizes, errs); + CheckMultiGetValues(3, vals, vals_sizes, errs, expected); + } + // delete + rocksdb_transaction_delete(txn, "foo", 3, &err); + CheckNoError(err); + CheckTxnGet(txn, roptions, "foo", NULL); + CheckTxnPinGet(txn, roptions, "foo", NULL); + + rocksdb_transaction_put(txn, "foo", 3, "hello", 5, &err); + CheckNoError(err); + + // read from outside transaction, before commit + CheckTxnDBGet(txn_db, roptions, "foo", NULL); + CheckTxnDBPinGet(txn_db, roptions, "foo", NULL); + { + const char* keys[3] = {"box", "foo", "notfound"}; + const size_t keys_sizes[3] = {3, 3, 8}; + char* vals[3]; + size_t vals_sizes[3]; + char* errs[3]; + const char* expected[3] = {"c", NULL, NULL}; + rocksdb_transactiondb_multi_get(txn_db, roptions, 3, keys, keys_sizes, + vals, vals_sizes, errs); + CheckMultiGetValues(3, vals, vals_sizes, errs, expected); + } + + // commit + rocksdb_transaction_commit(txn, &err); + CheckNoError(err); + + // read from outside transaction, after commit + CheckTxnDBGet(txn_db, roptions, "foo", "hello"); + CheckTxnDBPinGet(txn_db, roptions, "foo", "hello"); + { + const char* keys[3] = {"box", "foo", "notfound"}; + const size_t keys_sizes[3] = {3, 3, 8}; + char* vals[3]; + size_t vals_sizes[3]; + char* errs[3]; + const char* expected[3] = {"c", "hello", NULL}; + rocksdb_transactiondb_multi_get(txn_db, roptions, 3, keys, keys_sizes, + vals, vals_sizes, errs); + CheckMultiGetValues(3, vals, vals_sizes, errs, expected); + } + + // reuse old transaction + txn = rocksdb_transaction_begin(txn_db, woptions, txn_options, txn); + + // snapshot + const rocksdb_snapshot_t* snapshot; + snapshot = rocksdb_transactiondb_create_snapshot(txn_db); + rocksdb_readoptions_set_snapshot(roptions, snapshot); + + rocksdb_transactiondb_put(txn_db, woptions, "foo", 3, "hey", 3, &err); + CheckNoError(err); + + CheckTxnDBGet(txn_db, roptions, "foo", "hello"); + CheckTxnDBPinGet(txn_db, roptions, "foo", "hello"); + rocksdb_readoptions_set_snapshot(roptions, NULL); + rocksdb_transactiondb_release_snapshot(txn_db, snapshot); + CheckTxnDBGet(txn_db, roptions, "foo", "hey"); + CheckTxnDBPinGet(txn_db, roptions, "foo", "hey"); + + // iterate + rocksdb_transaction_put(txn, "bar", 3, "hi", 2, &err); + rocksdb_iterator_t* iter = + rocksdb_transaction_create_iterator(txn, roptions); + CheckCondition(!rocksdb_iter_valid(iter)); + rocksdb_iter_seek_to_first(iter); + CheckCondition(rocksdb_iter_valid(iter)); + CheckIter(iter, "bar", "hi"); + rocksdb_iter_get_error(iter, &err); + CheckNoError(err); + rocksdb_iter_destroy(iter); + + // rollback + rocksdb_transaction_rollback(txn, &err); + CheckNoError(err); + CheckTxnDBGet(txn_db, roptions, "bar", NULL); + CheckTxnDBPinGet(txn_db, roptions, "bar", NULL); + + // save point + rocksdb_transaction_put(txn, "foo1", 4, "hi1", 3, &err); + rocksdb_transaction_set_savepoint(txn); + CheckTxnGet(txn, roptions, "foo1", "hi1"); + CheckTxnPinGet(txn, roptions, "foo1", "hi1"); + rocksdb_transaction_put(txn, "foo2", 4, "hi2", 3, &err); + CheckTxnGet(txn, roptions, "foo2", "hi2"); + CheckTxnPinGet(txn, roptions, "foo2", "hi2"); + + // rollback to savepoint + rocksdb_transaction_rollback_to_savepoint(txn, &err); + CheckNoError(err); + CheckTxnGet(txn, roptions, "foo2", NULL); + CheckTxnGet(txn, roptions, "foo1", "hi1"); + CheckTxnPinGet(txn, roptions, "foo2", NULL); + CheckTxnPinGet(txn, roptions, "foo1", "hi1"); + CheckTxnDBGet(txn_db, roptions, "foo1", NULL); + CheckTxnDBGet(txn_db, roptions, "foo2", NULL); + CheckTxnDBPinGet(txn_db, roptions, "foo1", NULL); + CheckTxnDBPinGet(txn_db, roptions, "foo2", NULL); + rocksdb_transaction_commit(txn, &err); + CheckNoError(err); + CheckTxnDBGet(txn_db, roptions, "foo1", "hi1"); + CheckTxnDBGet(txn_db, roptions, "foo2", NULL); + CheckTxnDBPinGet(txn_db, roptions, "foo1", "hi1"); + CheckTxnDBPinGet(txn_db, roptions, "foo2", NULL); + + // Column families. + rocksdb_column_family_handle_t* cfh; + cfh = rocksdb_transactiondb_create_column_family(txn_db, options, + "txn_db_cf", &err); + CheckNoError(err); + + rocksdb_transactiondb_put_cf(txn_db, woptions, cfh, "cf_foo", 6, "cf_hello", + 8, &err); + CheckNoError(err); + CheckTxnDBGetCF(txn_db, roptions, cfh, "cf_foo", "cf_hello"); + CheckTxnDBPinGetCF(txn_db, roptions, cfh, "cf_foo", "cf_hello"); + { + const rocksdb_column_family_handle_t* get_handles[2] = {cfh, cfh}; + const char* keys[2] = {"cf_foo", "notfound"}; + const size_t keys_sizes[2] = {6, 8}; + char* vals[2]; + size_t vals_sizes[2]; + char* errs[2]; + const char* expected[2] = {"cf_hello", NULL}; + rocksdb_transactiondb_multi_get_cf(txn_db, roptions, get_handles, 2, keys, + keys_sizes, vals, vals_sizes, errs); + CheckMultiGetValues(2, vals, vals_sizes, errs, expected); + } + + rocksdb_transactiondb_delete_cf(txn_db, woptions, cfh, "cf_foo", 6, &err); + CheckNoError(err); + CheckTxnDBGetCF(txn_db, roptions, cfh, "cf_foo", NULL); + CheckTxnDBPinGetCF(txn_db, roptions, cfh, "cf_foo", NULL); + + // memory usage + rocksdb_t* base_db = rocksdb_transactiondb_get_base_db(txn_db); + rocksdb_memory_consumers_t* consumers = rocksdb_memory_consumers_create(); + rocksdb_memory_consumers_add_db(consumers, base_db); + rocksdb_memory_usage_t* usage = + rocksdb_approximate_memory_usage_create(consumers, &err); + CheckNoError(err); + rocksdb_approximate_memory_usage_destroy(usage); + rocksdb_memory_consumers_destroy(consumers); + rocksdb_transactiondb_close_base_db(base_db); + + // flush + rocksdb_flushoptions_t* flush_options = rocksdb_flushoptions_create(); + rocksdb_flushoptions_set_wait(flush_options, 1); + rocksdb_transactiondb_flush_wal(txn_db, 1, &err); + CheckNoError(err); + rocksdb_transactiondb_flush_cf(txn_db, flush_options, cfh, &err); + CheckNoError(err); + rocksdb_transactiondb_flush(txn_db, flush_options, &err); + CheckNoError(err); + rocksdb_flushoptions_destroy(flush_options); + + // close and destroy + rocksdb_column_family_handle_destroy(cfh); + rocksdb_transaction_destroy(txn); + rocksdb_transactiondb_close(txn_db); + rocksdb_destroy_db(options, dbname, &err); + CheckNoError(err); + rocksdb_transaction_options_destroy(txn_options); + rocksdb_transactiondb_options_destroy(txn_db_options); + } + + StartPhase("two-phase commit"); + { + // open a TransactionDB + txn_db_options = rocksdb_transactiondb_options_create(); + txn_options = rocksdb_transaction_options_create(); + rocksdb_options_set_create_if_missing(options, 1); + txn_db = rocksdb_transactiondb_open(options, txn_db_options, dbname, &err); + CheckNoError(err); + + rocksdb_transaction_options_set_skip_prepare(txn_options, 0); + txn = rocksdb_transaction_begin(txn_db, woptions, txn_options, NULL); + rocksdb_transaction_commit(txn, &err); + CheckCondition(err != NULL); + Free(&err); + err = NULL; + rocksdb_transaction_prepare(txn, &err); + CheckCondition(err != NULL); + Free(&err); + err = NULL; + rocksdb_transaction_set_name(txn, "txn1", 4, &err); + CheckNoError(err); + rocksdb_transaction_prepare(txn, &err); + CheckNoError(err); + rocksdb_transaction_commit(txn, &err); + CheckNoError(err); + rocksdb_transaction_destroy(txn); + + // prepare 2 transactions and close db. + rocksdb_transaction_t* txn1 = + rocksdb_transaction_begin(txn_db, woptions, txn_options, NULL); + rocksdb_transaction_put(txn1, "bar1", 4, "1", 1, &err); + CheckNoError(err); + rocksdb_transaction_set_name(txn1, "txn1", 4, &err); + CheckNoError(err); + rocksdb_transaction_prepare(txn1, &err); + CheckNoError(err); + rocksdb_transaction_t* txn2 = + rocksdb_transaction_begin(txn_db, woptions, txn_options, NULL); + rocksdb_transaction_put(txn2, "bar2", 4, "2", 1, &err); + CheckNoError(err); + rocksdb_transaction_set_name(txn2, "txn2", 4, &err); + CheckNoError(err); + rocksdb_transaction_prepare(txn2, &err); + CheckNoError(err); + rocksdb_transaction_destroy(txn1); + rocksdb_transaction_destroy(txn2); + rocksdb_transactiondb_close(txn_db); + rocksdb_transaction_options_destroy(txn_options); + rocksdb_transactiondb_options_destroy(txn_db_options); + + // reopen db and get all prepared. + txn_db_options = rocksdb_transactiondb_options_create(); + txn_options = rocksdb_transaction_options_create(); + rocksdb_options_set_error_if_exists(options, 0); + txn_db = rocksdb_transactiondb_open(options, txn_db_options, dbname, &err); + CheckNoError(err); + CheckTxnDBPinGet(txn_db, roptions, "bar1", NULL); + CheckTxnDBPinGet(txn_db, roptions, "bar2", NULL); + size_t cnt; + rocksdb_transaction_t** txns = + rocksdb_transactiondb_get_prepared_transactions(txn_db, &cnt); + CheckCondition(cnt == 2); + size_t i; + for (i = 0; i < cnt; i++) { + txn = txns[i]; + size_t name_len = 0; + char* name = rocksdb_transaction_get_name(txn, &name_len); + CheckCondition(name_len == 4); + if (strncmp(name, "txn1", name_len) == 0) { + rocksdb_transaction_commit(txn, &err); + } else if (strncmp(name, "txn2", name_len) == 0) { + rocksdb_transaction_rollback(txn, &err); + } + rocksdb_free(name); + CheckNoError(err); + rocksdb_transaction_destroy(txn); + } + rocksdb_free(txns); + CheckTxnDBGet(txn_db, roptions, "bar1", "1"); + CheckTxnDBGet(txn_db, roptions, "bar2", NULL); + rocksdb_transactiondb_put(txn_db, woptions, "bar2", 4, "2", 1, &err); + CheckNoError(err); + + // close and destroy + rocksdb_transactiondb_close(txn_db); + rocksdb_destroy_db(options, dbname, &err); + CheckNoError(err); + rocksdb_transaction_options_destroy(txn_options); + rocksdb_transactiondb_options_destroy(txn_db_options); + } + + StartPhase("transactions_multi_get_for_update"); + { + // open a TransactionDB + txn_db_options = rocksdb_transactiondb_options_create(); + rocksdb_transactiondb_options_set_transaction_lock_timeout(txn_db_options, + 0); + txn_options = rocksdb_transaction_options_create(); + rocksdb_options_set_create_if_missing(options, 1); + txn_db = rocksdb_transactiondb_open(options, txn_db_options, dbname, &err); + CheckNoError(err); + + rocksdb_transactiondb_put(txn_db, woptions, "foo", 3, "hey", 3, &err); + CheckNoError(err); + rocksdb_transactiondb_put(txn_db, woptions, "bar", 3, "hello", 5, &err); + CheckNoError(err); + + // begin transactions + txn = rocksdb_transaction_begin(txn_db, woptions, txn_options, NULL); + rocksdb_transaction_t* txn2 = + rocksdb_transaction_begin(txn_db, woptions, txn_options, NULL); + + // multi get + { + const char* keys[2] = {"foo", "bar"}; + const size_t keys_sizes[2] = {3, 3}; + char* vals[2]; + size_t vals_sizes[2]; + char* errs[2]; + const char* expected[2] = {"hey", "hello"}; + rocksdb_transaction_multi_get_for_update( + txn, roptions, 2, keys, keys_sizes, vals, vals_sizes, errs); + CheckMultiGetValues(2, vals, vals_sizes, errs, expected); + } + + char* conflict_err = NULL; + size_t val_len; + rocksdb_transaction_get_for_update(txn2, roptions, "foo", 3, &val_len, true, + &conflict_err); + // get-for-update conflict + CheckCondition(conflict_err != NULL); + Free(&conflict_err); + + // commit + rocksdb_transaction_commit(txn, &err); + CheckNoError(err); + + // should work after first tx is commited + CheckTxnGetForUpdate(txn2, roptions, "foo", "hey"); + + // commit the second one + rocksdb_transaction_commit(txn2, &err); + CheckNoError(err); + + // destroy txns + rocksdb_transaction_destroy(txn); + rocksdb_transaction_destroy(txn2); + + // same for column families + + rocksdb_column_family_handle_t* cfh; + cfh = rocksdb_transactiondb_create_column_family(txn_db, options, + "txn_db_cf", &err); + CheckNoError(err); + + rocksdb_transactiondb_put_cf(txn_db, woptions, cfh, "cf_foo", 6, "cf_hello", + 8, &err); + CheckNoError(err); + rocksdb_transactiondb_put_cf(txn_db, woptions, cfh, "cf_bar", 6, "cf_hey", + 6, &err); + CheckNoError(err); + + txn = rocksdb_transaction_begin(txn_db, woptions, txn_options, NULL); + txn2 = rocksdb_transaction_begin(txn_db, woptions, txn_options, NULL); + + { + const rocksdb_column_family_handle_t* get_handles[2] = {cfh, cfh}; + const char* keys[2] = {"cf_foo", "cf_bar"}; + const size_t keys_sizes[2] = {6, 6}; + char* vals[2]; + size_t vals_sizes[2]; + char* errs[2]; + const char* expected[2] = {"cf_hello", "cf_hey"}; + rocksdb_transaction_multi_get_for_update_cf(txn, roptions, get_handles, 2, + keys, keys_sizes, vals, + vals_sizes, errs); + CheckMultiGetValues(2, vals, vals_sizes, errs, expected); + } + + char* conflict_err_cf = NULL; + size_t val_len_cf; + rocksdb_transaction_get_for_update_cf(txn2, roptions, cfh, "cf_foo", 6, + &val_len_cf, true, &conflict_err_cf); + CheckCondition(conflict_err_cf != NULL); + Free(&conflict_err_cf); + + rocksdb_transaction_commit(txn, &err); + CheckNoError(err); + + CheckTxnGetForUpdateCF(txn2, roptions, cfh, "cf_foo", "cf_hello"); + + rocksdb_transaction_commit(txn2, &err); + CheckNoError(err); + + // close and destroy + rocksdb_column_family_handle_destroy(cfh); + rocksdb_transaction_destroy(txn); + rocksdb_transaction_destroy(txn2); + rocksdb_transactiondb_close(txn_db); + rocksdb_destroy_db(options, dbname, &err); + CheckNoError(err); + rocksdb_transaction_options_destroy(txn_options); + rocksdb_transactiondb_options_destroy(txn_db_options); + } + + StartPhase("optimistic_transactions"); + { + rocksdb_options_t* db_options = rocksdb_options_create(); + rocksdb_options_set_create_if_missing(db_options, 1); + rocksdb_options_set_allow_concurrent_memtable_write(db_options, 1); + otxn_db = rocksdb_optimistictransactiondb_open(db_options, dbname, &err); + otxn_options = rocksdb_optimistictransaction_options_create(); + rocksdb_transaction_t* txn1 = rocksdb_optimistictransaction_begin( + otxn_db, woptions, otxn_options, NULL); + rocksdb_transaction_t* txn2 = rocksdb_optimistictransaction_begin( + otxn_db, woptions, otxn_options, NULL); + rocksdb_transaction_put(txn1, "key", 3, "value", 5, &err); + CheckNoError(err); + rocksdb_transaction_put(txn2, "key1", 4, "value1", 6, &err); + CheckNoError(err); + CheckTxnGet(txn1, roptions, "key", "value"); + CheckTxnPinGet(txn1, roptions, "key", "value"); + rocksdb_transaction_commit(txn1, &err); + CheckNoError(err); + rocksdb_transaction_commit(txn2, &err); + CheckNoError(err); + rocksdb_transaction_destroy(txn1); + rocksdb_transaction_destroy(txn2); + + // Check column family + db = rocksdb_optimistictransactiondb_get_base_db(otxn_db); + rocksdb_put(db, woptions, "key", 3, "value", 5, &err); + CheckNoError(err); + rocksdb_column_family_handle_t *cfh1, *cfh2; + char** list_const_cf_names = (char**)malloc(2 * sizeof(char*)); + list_const_cf_names[0] = "txn_db_cf1"; + list_const_cf_names[1] = "txn_db_cf2"; + size_t cflen; + rocksdb_column_family_handle_t** list_cfh = rocksdb_create_column_families( + db, db_options, 2, (const char* const*)list_const_cf_names, &cflen, + &err); + free(list_const_cf_names); + CheckNoError(err); + assert(cflen == 2); + cfh1 = list_cfh[0]; + cfh2 = list_cfh[1]; + rocksdb_create_column_families_destroy(list_cfh); + txn = rocksdb_optimistictransaction_begin(otxn_db, woptions, otxn_options, + NULL); + rocksdb_transaction_put_cf(txn, cfh1, "key_cf1", 7, "val_cf1", 7, &err); + CheckNoError(err); + rocksdb_transaction_put_cf(txn, cfh2, "key_cf2", 7, "val_cf2", 7, &err); + CheckNoError(err); + rocksdb_transaction_commit(txn, &err); + CheckNoError(err); + txn = rocksdb_optimistictransaction_begin(otxn_db, woptions, otxn_options, + txn); + CheckGetCF(db, roptions, cfh1, "key_cf1", "val_cf1"); + CheckTxnGetCF(txn, roptions, cfh1, "key_cf1", "val_cf1"); + CheckTxnPinGetCF(txn, roptions, cfh1, "key_cf1", "val_cf1"); + { + const rocksdb_column_family_handle_t* get_handles[3] = {cfh1, cfh2, cfh2}; + const char* keys[3] = {"key_cf1", "key_cf2", "notfound"}; + const size_t keys_sizes[3] = {7, 7, 8}; + char* vals[3]; + size_t vals_sizes[3]; + char* errs[3]; + const char* expected[3] = {"val_cf1", "val_cf2", NULL}; + rocksdb_transaction_multi_get_cf(txn, roptions, get_handles, 3, keys, + keys_sizes, vals, vals_sizes, errs); + CheckMultiGetValues(3, vals, vals_sizes, errs, expected); + } + + // Check iterator with column family + rocksdb_transaction_put_cf(txn, cfh1, "key1_cf", 7, "val1_cf", 7, &err); + CheckNoError(err); + rocksdb_iterator_t* iter = + rocksdb_transaction_create_iterator_cf(txn, roptions, cfh1); + CheckCondition(!rocksdb_iter_valid(iter)); + rocksdb_iter_seek_to_first(iter); + CheckCondition(rocksdb_iter_valid(iter)); + CheckIter(iter, "key1_cf", "val1_cf"); + rocksdb_iter_get_error(iter, &err); + CheckNoError(err); + rocksdb_iter_destroy(iter); + + rocksdb_transaction_destroy(txn); + rocksdb_column_family_handle_destroy(cfh1); + rocksdb_column_family_handle_destroy(cfh2); + rocksdb_optimistictransactiondb_close_base_db(db); + rocksdb_optimistictransactiondb_close(otxn_db); + + // Check open optimistic transaction db with column families + size_t cf_len; + char** column_fams = + rocksdb_list_column_families(db_options, dbname, &cf_len, &err); + CheckNoError(err); + CheckEqual("default", column_fams[0], 7); + CheckEqual("txn_db_cf1", column_fams[1], 10); + CheckEqual("txn_db_cf2", column_fams[2], 10); + CheckCondition(cf_len == 3); + rocksdb_list_column_families_destroy(column_fams, cf_len); + + const char* cf_names[3] = {"default", "txn_db_cf1", "txn_db_cf2"}; + rocksdb_options_t* cf_options = rocksdb_options_create(); + const rocksdb_options_t* cf_opts[3] = {cf_options, cf_options, cf_options}; + + rocksdb_options_set_error_if_exists(cf_options, 0); + rocksdb_column_family_handle_t* cf_handles[3]; + otxn_db = rocksdb_optimistictransactiondb_open_column_families( + db_options, dbname, 3, cf_names, cf_opts, cf_handles, &err); + CheckNoError(err); + rocksdb_transaction_t* txn_cf = rocksdb_optimistictransaction_begin( + otxn_db, woptions, otxn_options, NULL); + CheckTxnGetCF(txn_cf, roptions, cf_handles[0], "key", "value"); + CheckTxnGetCF(txn_cf, roptions, cf_handles[1], "key_cf1", "val_cf1"); + CheckTxnGetCF(txn_cf, roptions, cf_handles[2], "key_cf2", "val_cf2"); + CheckTxnPinGetCF(txn_cf, roptions, cf_handles[0], "key", "value"); + CheckTxnPinGetCF(txn_cf, roptions, cf_handles[1], "key_cf1", "val_cf1"); + CheckTxnPinGetCF(txn_cf, roptions, cf_handles[2], "key_cf2", "val_cf2"); + rocksdb_transaction_destroy(txn_cf); + rocksdb_options_destroy(cf_options); + rocksdb_column_family_handle_destroy(cf_handles[0]); + rocksdb_column_family_handle_destroy(cf_handles[1]); + rocksdb_column_family_handle_destroy(cf_handles[2]); + rocksdb_optimistictransactiondb_close(otxn_db); + rocksdb_destroy_db(db_options, dbname, &err); + rocksdb_options_destroy(db_options); + rocksdb_optimistictransaction_options_destroy(otxn_options); + CheckNoError(err); + } + + // Simple sanity check that setting memtable rep works. + StartPhase("memtable_reps"); + { + // Create database with vector memtable. + rocksdb_options_set_memtable_vector_rep(options); + db = rocksdb_open(options, dbname, &err); + CheckNoError(err); + + // Create database with hash skiplist memtable. + rocksdb_close(db); + rocksdb_destroy_db(options, dbname, &err); + CheckNoError(err); + + rocksdb_options_set_hash_skip_list_rep(options, 5000, 4, 4); + db = rocksdb_open(options, dbname, &err); + CheckNoError(err); + } + + // Check that secondary instance works. + StartPhase("open_as_secondary"); + { + rocksdb_close(db); + rocksdb_destroy_db(options, dbname, &err); + + rocksdb_options_t* db_options = rocksdb_options_create(); + rocksdb_options_set_create_if_missing(db_options, 1); + db = rocksdb_open(db_options, dbname, &err); + CheckNoError(err); + rocksdb_t* db1; + rocksdb_options_t* opts = rocksdb_options_create(); + rocksdb_options_set_max_open_files(opts, -1); + rocksdb_options_set_create_if_missing(opts, 1); + snprintf(secondary_path, sizeof(secondary_path), + "%s/rocksdb_c_test_secondary-%d", GetTempDir(), ((int)geteuid())); + db1 = rocksdb_open_as_secondary(opts, dbname, secondary_path, &err); + CheckNoError(err); + + rocksdb_writeoptions_set_sync(woptions, 0); + rocksdb_writeoptions_disable_WAL(woptions, 1); + rocksdb_put(db, woptions, "key0", 4, "value0", 6, &err); + CheckNoError(err); + rocksdb_flushoptions_t* flush_opts = rocksdb_flushoptions_create(); + rocksdb_flushoptions_set_wait(flush_opts, 1); + rocksdb_flush(db, flush_opts, &err); + CheckNoError(err); + rocksdb_try_catch_up_with_primary(db1, &err); + CheckNoError(err); + rocksdb_readoptions_t* ropts = rocksdb_readoptions_create(); + rocksdb_readoptions_set_verify_checksums(ropts, 1); + rocksdb_readoptions_set_snapshot(ropts, NULL); + CheckGet(db, ropts, "key0", "value0"); + CheckGet(db1, ropts, "key0", "value0"); + + rocksdb_writeoptions_disable_WAL(woptions, 0); + rocksdb_put(db, woptions, "key1", 4, "value1", 6, &err); + CheckNoError(err); + rocksdb_try_catch_up_with_primary(db1, &err); + CheckNoError(err); + CheckGet(db1, ropts, "key0", "value0"); + CheckGet(db1, ropts, "key1", "value1"); + + rocksdb_close(db1); + rocksdb_destroy_db(opts, secondary_path, &err); + CheckNoError(err); + + rocksdb_options_destroy(db_options); + rocksdb_options_destroy(opts); + rocksdb_readoptions_destroy(ropts); + rocksdb_flushoptions_destroy(flush_opts); + } + + // Simple sanity check that options setting db_paths work. + StartPhase("open_db_paths"); + { + rocksdb_close(db); + rocksdb_destroy_db(options, dbname, &err); + + const rocksdb_dbpath_t* paths[1] = {dbpath}; + rocksdb_options_set_db_paths(options, paths, 1); + db = rocksdb_open(options, dbname, &err); + CheckNoError(err); + } + + StartPhase("filter_with_prefix_seek"); + { + rocksdb_close(db); + rocksdb_destroy_db(options, dbname, &err); + CheckNoError(err); + + rocksdb_options_set_prefix_extractor( + options, rocksdb_slicetransform_create_fixed_prefix(1)); + rocksdb_filterpolicy_t* filter_policy = + rocksdb_filterpolicy_create_bloom_full(8.0); + rocksdb_block_based_options_set_filter_policy(table_options, filter_policy); + rocksdb_options_set_block_based_table_factory(options, table_options); + + db = rocksdb_open(options, dbname, &err); + CheckNoError(err); + + int i; + for (i = 0; i < 10; ++i) { + char key = '0' + (char)i; + rocksdb_put(db, woptions, &key, 1, "", 1, &err); + CheckNoError(err); + } + + // Flush to generate an L0 so that filter will be used later. + rocksdb_flushoptions_t* flush_options = rocksdb_flushoptions_create(); + rocksdb_flushoptions_set_wait(flush_options, 1); + rocksdb_flush(db, flush_options, &err); + rocksdb_flushoptions_destroy(flush_options); + CheckNoError(err); + + rocksdb_readoptions_t* ropts = rocksdb_readoptions_create(); + rocksdb_iterator_t* iter = rocksdb_create_iterator(db, ropts); + + rocksdb_iter_seek(iter, "0", 1); + int cnt = 0; + while (rocksdb_iter_valid(iter)) { + ++cnt; + rocksdb_iter_next(iter); + } + CheckCondition(10 == cnt); + + rocksdb_iter_destroy(iter); + rocksdb_readoptions_destroy(ropts); + } + + StartPhase("statistics"); + { + const uint32_t BYTES_WRITTEN_TICKER = 40; + const uint32_t DB_WRITE_HIST = 1; + + rocksdb_statistics_histogram_data_t* hist = + rocksdb_statistics_histogram_data_create(); + { + // zero by default + CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_median(hist)); + CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_p95(hist)); + CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_p99(hist)); + CheckCondition(0.0 == + rocksdb_statistics_histogram_data_get_average(hist)); + CheckCondition(0.0 == + rocksdb_statistics_histogram_data_get_std_dev(hist)); + CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_max(hist)); + CheckCondition(0 == rocksdb_statistics_histogram_data_get_count(hist)); + CheckCondition(0 == rocksdb_statistics_histogram_data_get_sum(hist)); + CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_min(hist)); + } + + rocksdb_close(db); + rocksdb_destroy_db(options, dbname, &err); + CheckNoError(err); + + rocksdb_options_enable_statistics(options); + rocksdb_options_set_statistics_level(options, rocksdb_statistics_level_all); + + db = rocksdb_open(options, dbname, &err); + CheckNoError(err); + + CheckCondition(0 == rocksdb_options_statistics_get_ticker_count( + options, BYTES_WRITTEN_TICKER)); + rocksdb_options_statistics_get_histogram_data(options, DB_WRITE_HIST, hist); + CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_median(hist)); + CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_p95(hist)); + CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_p99(hist)); + CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_average(hist)); + CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_std_dev(hist)); + CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_max(hist)); + CheckCondition(0 == rocksdb_statistics_histogram_data_get_count(hist)); + CheckCondition(0 == rocksdb_statistics_histogram_data_get_sum(hist)); + + int i; + for (i = 0; i < 10; ++i) { + char key = '0' + (char)i; + rocksdb_put(db, woptions, &key, 1, "", 1, &err); + CheckNoError(err); + } + CheckCondition(0 != rocksdb_options_statistics_get_ticker_count( + options, BYTES_WRITTEN_TICKER)); + rocksdb_options_statistics_get_histogram_data(options, DB_WRITE_HIST, hist); + CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_median(hist)); + CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_p95(hist)); + CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_p99(hist)); + CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_average(hist)); + CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_std_dev(hist)); + CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_max(hist)); + CheckCondition(0 != rocksdb_statistics_histogram_data_get_count(hist)); + CheckCondition(0 != rocksdb_statistics_histogram_data_get_sum(hist)); + + rocksdb_statistics_histogram_data_destroy(hist); + } + + StartPhase("cancel_all_background_work"); + rocksdb_cancel_all_background_work(db, 1); + + StartPhase("cleanup"); + rocksdb_close(db); + rocksdb_options_destroy(options); + rocksdb_block_based_options_destroy(table_options); + rocksdb_readoptions_destroy(roptions); + rocksdb_writeoptions_destroy(woptions); + rocksdb_compactoptions_destroy(coptions); + rocksdb_cache_destroy(cache); + rocksdb_comparator_destroy(cmp); + rocksdb_dbpath_destroy(dbpath); + rocksdb_env_destroy(env); + + fprintf(stderr, "PASS\n"); + return 0; +} diff --git a/librocksdb-sys/rocksdb/db/column_family.cc b/librocksdb-sys/rocksdb/db/column_family.cc new file mode 100644 index 0000000..185ec72 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/column_family.cc @@ -0,0 +1,1759 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/column_family.h" + +#include +#include +#include +#include +#include +#include + +#include "db/blob/blob_file_cache.h" +#include "db/blob/blob_source.h" +#include "db/compaction/compaction_picker.h" +#include "db/compaction/compaction_picker_fifo.h" +#include "db/compaction/compaction_picker_level.h" +#include "db/compaction/compaction_picker_universal.h" +#include "db/db_impl/db_impl.h" +#include "db/internal_stats.h" +#include "db/job_context.h" +#include "db/range_del_aggregator.h" +#include "db/table_properties_collector.h" +#include "db/version_set.h" +#include "db/write_controller.h" +#include "file/sst_file_manager_impl.h" +#include "logging/logging.h" +#include "monitoring/thread_status_util.h" +#include "options/options_helper.h" +#include "port/port.h" +#include "rocksdb/convenience.h" +#include "rocksdb/table.h" +#include "table/merging_iterator.h" +#include "util/autovector.h" +#include "util/cast_util.h" +#include "util/compression.h" + +namespace ROCKSDB_NAMESPACE { + +ColumnFamilyHandleImpl::ColumnFamilyHandleImpl( + ColumnFamilyData* column_family_data, DBImpl* db, InstrumentedMutex* mutex) + : cfd_(column_family_data), db_(db), mutex_(mutex) { + if (cfd_ != nullptr) { + cfd_->Ref(); + } +} + +ColumnFamilyHandleImpl::~ColumnFamilyHandleImpl() { + if (cfd_ != nullptr) { + for (auto& listener : cfd_->ioptions()->listeners) { + listener->OnColumnFamilyHandleDeletionStarted(this); + } + // Job id == 0 means that this is not our background process, but rather + // user thread + // Need to hold some shared pointers owned by the initial_cf_options + // before final cleaning up finishes. + ColumnFamilyOptions initial_cf_options_copy = cfd_->initial_cf_options(); + JobContext job_context(0); + mutex_->Lock(); + bool dropped = cfd_->IsDropped(); + if (cfd_->UnrefAndTryDelete()) { + if (dropped) { + db_->FindObsoleteFiles(&job_context, false, true); + } + } + mutex_->Unlock(); + if (job_context.HaveSomethingToDelete()) { + bool defer_purge = + db_->immutable_db_options().avoid_unnecessary_blocking_io; + db_->PurgeObsoleteFiles(job_context, defer_purge); + } + job_context.Clean(); + } +} + +uint32_t ColumnFamilyHandleImpl::GetID() const { return cfd()->GetID(); } + +const std::string& ColumnFamilyHandleImpl::GetName() const { + return cfd()->GetName(); +} + +Status ColumnFamilyHandleImpl::GetDescriptor(ColumnFamilyDescriptor* desc) { + // accessing mutable cf-options requires db mutex. + InstrumentedMutexLock l(mutex_); + *desc = ColumnFamilyDescriptor(cfd()->GetName(), cfd()->GetLatestCFOptions()); + return Status::OK(); +} + +const Comparator* ColumnFamilyHandleImpl::GetComparator() const { + return cfd()->user_comparator(); +} + +void GetIntTblPropCollectorFactory( + const ImmutableCFOptions& ioptions, + IntTblPropCollectorFactories* int_tbl_prop_collector_factories) { + assert(int_tbl_prop_collector_factories); + + auto& collector_factories = ioptions.table_properties_collector_factories; + for (size_t i = 0; i < ioptions.table_properties_collector_factories.size(); + ++i) { + assert(collector_factories[i]); + int_tbl_prop_collector_factories->emplace_back( + new UserKeyTablePropertiesCollectorFactory(collector_factories[i])); + } +} + +Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options) { + if (!cf_options.compression_per_level.empty()) { + for (size_t level = 0; level < cf_options.compression_per_level.size(); + ++level) { + if (!CompressionTypeSupported(cf_options.compression_per_level[level])) { + return Status::InvalidArgument( + "Compression type " + + CompressionTypeToString(cf_options.compression_per_level[level]) + + " is not linked with the binary."); + } + } + } else { + if (!CompressionTypeSupported(cf_options.compression)) { + return Status::InvalidArgument( + "Compression type " + + CompressionTypeToString(cf_options.compression) + + " is not linked with the binary."); + } + } + if (cf_options.compression_opts.zstd_max_train_bytes > 0) { + if (cf_options.compression_opts.use_zstd_dict_trainer) { + if (!ZSTD_TrainDictionarySupported()) { + return Status::InvalidArgument( + "zstd dictionary trainer cannot be used because ZSTD 1.1.3+ " + "is not linked with the binary."); + } + } else if (!ZSTD_FinalizeDictionarySupported()) { + return Status::InvalidArgument( + "zstd finalizeDictionary cannot be used because ZSTD 1.4.5+ " + "is not linked with the binary."); + } + if (cf_options.compression_opts.max_dict_bytes == 0) { + return Status::InvalidArgument( + "The dictionary size limit (`CompressionOptions::max_dict_bytes`) " + "should be nonzero if we're using zstd's dictionary generator."); + } + } + + if (!CompressionTypeSupported(cf_options.blob_compression_type)) { + std::ostringstream oss; + oss << "The specified blob compression type " + << CompressionTypeToString(cf_options.blob_compression_type) + << " is not available."; + + return Status::InvalidArgument(oss.str()); + } + + return Status::OK(); +} + +Status CheckConcurrentWritesSupported(const ColumnFamilyOptions& cf_options) { + if (cf_options.inplace_update_support) { + return Status::InvalidArgument( + "In-place memtable updates (inplace_update_support) is not compatible " + "with concurrent writes (allow_concurrent_memtable_write)"); + } + if (!cf_options.memtable_factory->IsInsertConcurrentlySupported()) { + return Status::InvalidArgument( + "Memtable doesn't concurrent writes (allow_concurrent_memtable_write)"); + } + return Status::OK(); +} + +Status CheckCFPathsSupported(const DBOptions& db_options, + const ColumnFamilyOptions& cf_options) { + // More than one cf_paths are supported only in universal + // and level compaction styles. This function also checks the case + // in which cf_paths is not specified, which results in db_paths + // being used. + if ((cf_options.compaction_style != kCompactionStyleUniversal) && + (cf_options.compaction_style != kCompactionStyleLevel)) { + if (cf_options.cf_paths.size() > 1) { + return Status::NotSupported( + "More than one CF paths are only supported in " + "universal and level compaction styles. "); + } else if (cf_options.cf_paths.empty() && db_options.db_paths.size() > 1) { + return Status::NotSupported( + "More than one DB paths are only supported in " + "universal and level compaction styles. "); + } + } + return Status::OK(); +} + +namespace { +const uint64_t kDefaultTtl = 0xfffffffffffffffe; +const uint64_t kDefaultPeriodicCompSecs = 0xfffffffffffffffe; +} // anonymous namespace + +ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options, + const ColumnFamilyOptions& src) { + ColumnFamilyOptions result = src; + size_t clamp_max = std::conditional< + sizeof(size_t) == 4, std::integral_constant, + std::integral_constant>::type::value; + ClipToRange(&result.write_buffer_size, (static_cast(64)) << 10, + clamp_max); + // if user sets arena_block_size, we trust user to use this value. Otherwise, + // calculate a proper value from writer_buffer_size; + if (result.arena_block_size <= 0) { + result.arena_block_size = + std::min(size_t{1024 * 1024}, result.write_buffer_size / 8); + + // Align up to 4k + const size_t align = 4 * 1024; + result.arena_block_size = + ((result.arena_block_size + align - 1) / align) * align; + } + result.min_write_buffer_number_to_merge = + std::min(result.min_write_buffer_number_to_merge, + result.max_write_buffer_number - 1); + if (result.min_write_buffer_number_to_merge < 1) { + result.min_write_buffer_number_to_merge = 1; + } + + if (db_options.atomic_flush && result.min_write_buffer_number_to_merge > 1) { + ROCKS_LOG_WARN( + db_options.logger, + "Currently, if atomic_flush is true, then triggering flush for any " + "column family internally (non-manual flush) will trigger flushing " + "all column families even if the number of memtables is smaller " + "min_write_buffer_number_to_merge. Therefore, configuring " + "min_write_buffer_number_to_merge > 1 is not compatible and should " + "be satinized to 1. Not doing so will lead to data loss and " + "inconsistent state across multiple column families when WAL is " + "disabled, which is a common setting for atomic flush"); + + result.min_write_buffer_number_to_merge = 1; + } + + if (result.num_levels < 1) { + result.num_levels = 1; + } + if (result.compaction_style == kCompactionStyleLevel && + result.num_levels < 2) { + result.num_levels = 2; + } + + if (result.compaction_style == kCompactionStyleUniversal && + db_options.allow_ingest_behind && result.num_levels < 3) { + result.num_levels = 3; + } + + if (result.max_write_buffer_number < 2) { + result.max_write_buffer_number = 2; + } + // fall back max_write_buffer_number_to_maintain if + // max_write_buffer_size_to_maintain is not set + if (result.max_write_buffer_size_to_maintain < 0) { + result.max_write_buffer_size_to_maintain = + result.max_write_buffer_number * + static_cast(result.write_buffer_size); + } else if (result.max_write_buffer_size_to_maintain == 0 && + result.max_write_buffer_number_to_maintain < 0) { + result.max_write_buffer_number_to_maintain = result.max_write_buffer_number; + } + // bloom filter size shouldn't exceed 1/4 of memtable size. + if (result.memtable_prefix_bloom_size_ratio > 0.25) { + result.memtable_prefix_bloom_size_ratio = 0.25; + } else if (result.memtable_prefix_bloom_size_ratio < 0) { + result.memtable_prefix_bloom_size_ratio = 0; + } + + if (!result.prefix_extractor) { + assert(result.memtable_factory); + Slice name = result.memtable_factory->Name(); + if (name.compare("HashSkipListRepFactory") == 0 || + name.compare("HashLinkListRepFactory") == 0) { + result.memtable_factory = std::make_shared(); + } + } + + if (result.compaction_style == kCompactionStyleFIFO) { + // since we delete level0 files in FIFO compaction when there are too many + // of them, these options don't really mean anything + result.level0_slowdown_writes_trigger = std::numeric_limits::max(); + result.level0_stop_writes_trigger = std::numeric_limits::max(); + } + + if (result.max_bytes_for_level_multiplier <= 0) { + result.max_bytes_for_level_multiplier = 1; + } + + if (result.level0_file_num_compaction_trigger == 0) { + ROCKS_LOG_WARN(db_options.logger, + "level0_file_num_compaction_trigger cannot be 0"); + result.level0_file_num_compaction_trigger = 1; + } + + if (result.level0_stop_writes_trigger < + result.level0_slowdown_writes_trigger || + result.level0_slowdown_writes_trigger < + result.level0_file_num_compaction_trigger) { + ROCKS_LOG_WARN(db_options.logger, + "This condition must be satisfied: " + "level0_stop_writes_trigger(%d) >= " + "level0_slowdown_writes_trigger(%d) >= " + "level0_file_num_compaction_trigger(%d)", + result.level0_stop_writes_trigger, + result.level0_slowdown_writes_trigger, + result.level0_file_num_compaction_trigger); + if (result.level0_slowdown_writes_trigger < + result.level0_file_num_compaction_trigger) { + result.level0_slowdown_writes_trigger = + result.level0_file_num_compaction_trigger; + } + if (result.level0_stop_writes_trigger < + result.level0_slowdown_writes_trigger) { + result.level0_stop_writes_trigger = result.level0_slowdown_writes_trigger; + } + ROCKS_LOG_WARN(db_options.logger, + "Adjust the value to " + "level0_stop_writes_trigger(%d)" + "level0_slowdown_writes_trigger(%d)" + "level0_file_num_compaction_trigger(%d)", + result.level0_stop_writes_trigger, + result.level0_slowdown_writes_trigger, + result.level0_file_num_compaction_trigger); + } + + if (result.soft_pending_compaction_bytes_limit == 0) { + result.soft_pending_compaction_bytes_limit = + result.hard_pending_compaction_bytes_limit; + } else if (result.hard_pending_compaction_bytes_limit > 0 && + result.soft_pending_compaction_bytes_limit > + result.hard_pending_compaction_bytes_limit) { + result.soft_pending_compaction_bytes_limit = + result.hard_pending_compaction_bytes_limit; + } + + // When the DB is stopped, it's possible that there are some .trash files that + // were not deleted yet, when we open the DB we will find these .trash files + // and schedule them to be deleted (or delete immediately if SstFileManager + // was not used) + auto sfm = + static_cast(db_options.sst_file_manager.get()); + for (size_t i = 0; i < result.cf_paths.size(); i++) { + DeleteScheduler::CleanupDirectory(db_options.env, sfm, + result.cf_paths[i].path) + .PermitUncheckedError(); + } + + if (result.cf_paths.empty()) { + result.cf_paths = db_options.db_paths; + } + + if (result.level_compaction_dynamic_level_bytes) { + if (result.compaction_style != kCompactionStyleLevel) { + ROCKS_LOG_WARN(db_options.info_log.get(), + "level_compaction_dynamic_level_bytes only makes sense" + "for level-based compaction"); + result.level_compaction_dynamic_level_bytes = false; + } else if (result.cf_paths.size() > 1U) { + // we don't yet know how to make both of this feature and multiple + // DB path work. + ROCKS_LOG_WARN(db_options.info_log.get(), + "multiple cf_paths/db_paths and" + "level_compaction_dynamic_level_bytes" + "can't be used together"); + result.level_compaction_dynamic_level_bytes = false; + } + } + + if (result.max_compaction_bytes == 0) { + result.max_compaction_bytes = result.target_file_size_base * 25; + } + + bool is_block_based_table = (result.table_factory->IsInstanceOf( + TableFactory::kBlockBasedTableName())); + + const uint64_t kAdjustedTtl = 30 * 24 * 60 * 60; + if (result.ttl == kDefaultTtl) { + if (is_block_based_table) { + // FIFO also requires max_open_files=-1, which is checked in + // ValidateOptions(). + result.ttl = kAdjustedTtl; + } else { + result.ttl = 0; + } + } + + const uint64_t kAdjustedPeriodicCompSecs = 30 * 24 * 60 * 60; + if (result.compaction_style == kCompactionStyleLevel) { + if ((result.compaction_filter != nullptr || + result.compaction_filter_factory != nullptr) && + result.periodic_compaction_seconds == kDefaultPeriodicCompSecs && + is_block_based_table) { + result.periodic_compaction_seconds = kAdjustedPeriodicCompSecs; + } + } else if (result.compaction_style == kCompactionStyleUniversal) { + if (result.periodic_compaction_seconds == kDefaultPeriodicCompSecs && + is_block_based_table) { + result.periodic_compaction_seconds = kAdjustedPeriodicCompSecs; + } + } else if (result.compaction_style == kCompactionStyleFIFO) { + if (result.periodic_compaction_seconds != kDefaultPeriodicCompSecs) { + ROCKS_LOG_WARN( + db_options.info_log.get(), + "periodic_compaction_seconds does not support FIFO compaction. You" + "may want to set option TTL instead."); + } + } + + // For universal compaction, `ttl` and `periodic_compaction_seconds` mean the + // same thing, take the stricter value. + if (result.compaction_style == kCompactionStyleUniversal) { + if (result.periodic_compaction_seconds == 0) { + result.periodic_compaction_seconds = result.ttl; + } else if (result.ttl != 0) { + result.periodic_compaction_seconds = + std::min(result.ttl, result.periodic_compaction_seconds); + } + } + + if (result.periodic_compaction_seconds == kDefaultPeriodicCompSecs) { + result.periodic_compaction_seconds = 0; + } + + return result; +} + +int SuperVersion::dummy = 0; +void* const SuperVersion::kSVInUse = &SuperVersion::dummy; +void* const SuperVersion::kSVObsolete = nullptr; + +SuperVersion::~SuperVersion() { + for (auto td : to_delete) { + delete td; + } +} + +SuperVersion* SuperVersion::Ref() { + refs.fetch_add(1, std::memory_order_relaxed); + return this; +} + +bool SuperVersion::Unref() { + // fetch_sub returns the previous value of ref + uint32_t previous_refs = refs.fetch_sub(1); + assert(previous_refs > 0); + return previous_refs == 1; +} + +void SuperVersion::Cleanup() { + assert(refs.load(std::memory_order_relaxed) == 0); + // Since this SuperVersion object is being deleted, + // decrement reference to the immutable MemtableList + // this SV object was pointing to. + imm->Unref(&to_delete); + MemTable* m = mem->Unref(); + if (m != nullptr) { + auto* memory_usage = current->cfd()->imm()->current_memory_usage(); + assert(*memory_usage >= m->ApproximateMemoryUsage()); + *memory_usage -= m->ApproximateMemoryUsage(); + to_delete.push_back(m); + } + current->Unref(); + cfd->UnrefAndTryDelete(); +} + +void SuperVersion::Init(ColumnFamilyData* new_cfd, MemTable* new_mem, + MemTableListVersion* new_imm, Version* new_current) { + cfd = new_cfd; + mem = new_mem; + imm = new_imm; + current = new_current; + cfd->Ref(); + mem->Ref(); + imm->Ref(); + current->Ref(); + refs.store(1, std::memory_order_relaxed); +} + +namespace { +void SuperVersionUnrefHandle(void* ptr) { + // UnrefHandle is called when a thread exits or a ThreadLocalPtr gets + // destroyed. When the former happens, the thread shouldn't see kSVInUse. + // When the latter happens, only super_version_ holds a reference + // to ColumnFamilyData, so no further queries are possible. + SuperVersion* sv = static_cast(ptr); + bool was_last_ref __attribute__((__unused__)); + was_last_ref = sv->Unref(); + // Thread-local SuperVersions can't outlive ColumnFamilyData::super_version_. + // This is important because we can't do SuperVersion cleanup here. + // That would require locking DB mutex, which would deadlock because + // SuperVersionUnrefHandle is called with locked ThreadLocalPtr mutex. + assert(!was_last_ref); +} +} // anonymous namespace + +std::vector ColumnFamilyData::GetDbPaths() const { + std::vector paths; + paths.reserve(ioptions_.cf_paths.size()); + for (const DbPath& db_path : ioptions_.cf_paths) { + paths.emplace_back(db_path.path); + } + return paths; +} + +const uint32_t ColumnFamilyData::kDummyColumnFamilyDataId = + std::numeric_limits::max(); + +ColumnFamilyData::ColumnFamilyData( + uint32_t id, const std::string& name, Version* _dummy_versions, + Cache* _table_cache, WriteBufferManager* write_buffer_manager, + const ColumnFamilyOptions& cf_options, const ImmutableDBOptions& db_options, + const FileOptions* file_options, ColumnFamilySet* column_family_set, + BlockCacheTracer* const block_cache_tracer, + const std::shared_ptr& io_tracer, const std::string& db_id, + const std::string& db_session_id) + : id_(id), + name_(name), + dummy_versions_(_dummy_versions), + current_(nullptr), + refs_(0), + initialized_(false), + dropped_(false), + internal_comparator_(cf_options.comparator), + initial_cf_options_(SanitizeOptions(db_options, cf_options)), + ioptions_(db_options, initial_cf_options_), + mutable_cf_options_(initial_cf_options_), + is_delete_range_supported_( + cf_options.table_factory->IsDeleteRangeSupported()), + write_buffer_manager_(write_buffer_manager), + mem_(nullptr), + imm_(ioptions_.min_write_buffer_number_to_merge, + ioptions_.max_write_buffer_number_to_maintain, + ioptions_.max_write_buffer_size_to_maintain), + super_version_(nullptr), + super_version_number_(0), + local_sv_(new ThreadLocalPtr(&SuperVersionUnrefHandle)), + next_(nullptr), + prev_(nullptr), + log_number_(0), + column_family_set_(column_family_set), + queued_for_flush_(false), + queued_for_compaction_(false), + prev_compaction_needed_bytes_(0), + allow_2pc_(db_options.allow_2pc), + last_memtable_id_(0), + db_paths_registered_(false), + mempurge_used_(false), + next_epoch_number_(1) { + if (id_ != kDummyColumnFamilyDataId) { + // TODO(cc): RegisterDbPaths can be expensive, considering moving it + // outside of this constructor which might be called with db mutex held. + // TODO(cc): considering using ioptions_.fs, currently some tests rely on + // EnvWrapper, that's the main reason why we use env here. + Status s = ioptions_.env->RegisterDbPaths(GetDbPaths()); + if (s.ok()) { + db_paths_registered_ = true; + } else { + ROCKS_LOG_ERROR( + ioptions_.logger, + "Failed to register data paths of column family (id: %d, name: %s)", + id_, name_.c_str()); + } + } + Ref(); + + // Convert user defined table properties collector factories to internal ones. + GetIntTblPropCollectorFactory(ioptions_, &int_tbl_prop_collector_factories_); + + // if _dummy_versions is nullptr, then this is a dummy column family. + if (_dummy_versions != nullptr) { + internal_stats_.reset( + new InternalStats(ioptions_.num_levels, ioptions_.clock, this)); + table_cache_.reset(new TableCache(ioptions_, file_options, _table_cache, + block_cache_tracer, io_tracer, + db_session_id)); + blob_file_cache_.reset( + new BlobFileCache(_table_cache, ioptions(), soptions(), id_, + internal_stats_->GetBlobFileReadHist(), io_tracer)); + blob_source_.reset(new BlobSource(ioptions(), db_id, db_session_id, + blob_file_cache_.get())); + + if (ioptions_.compaction_style == kCompactionStyleLevel) { + compaction_picker_.reset( + new LevelCompactionPicker(ioptions_, &internal_comparator_)); + } else if (ioptions_.compaction_style == kCompactionStyleUniversal) { + compaction_picker_.reset( + new UniversalCompactionPicker(ioptions_, &internal_comparator_)); + } else if (ioptions_.compaction_style == kCompactionStyleFIFO) { + compaction_picker_.reset( + new FIFOCompactionPicker(ioptions_, &internal_comparator_)); + } else if (ioptions_.compaction_style == kCompactionStyleNone) { + compaction_picker_.reset( + new NullCompactionPicker(ioptions_, &internal_comparator_)); + ROCKS_LOG_WARN(ioptions_.logger, + "Column family %s does not use any background compaction. " + "Compactions can only be done via CompactFiles\n", + GetName().c_str()); + } else { + ROCKS_LOG_ERROR(ioptions_.logger, + "Unable to recognize the specified compaction style %d. " + "Column family %s will use kCompactionStyleLevel.\n", + ioptions_.compaction_style, GetName().c_str()); + compaction_picker_.reset( + new LevelCompactionPicker(ioptions_, &internal_comparator_)); + } + + if (column_family_set_->NumberOfColumnFamilies() < 10) { + ROCKS_LOG_INFO(ioptions_.logger, + "--------------- Options for column family [%s]:\n", + name.c_str()); + initial_cf_options_.Dump(ioptions_.logger); + } else { + ROCKS_LOG_INFO(ioptions_.logger, "\t(skipping printing options)\n"); + } + } + + RecalculateWriteStallConditions(mutable_cf_options_); + + if (cf_options.table_factory->IsInstanceOf( + TableFactory::kBlockBasedTableName()) && + cf_options.table_factory->GetOptions()) { + const BlockBasedTableOptions* bbto = + cf_options.table_factory->GetOptions(); + const auto& options_overrides = bbto->cache_usage_options.options_overrides; + const auto file_metadata_charged = + options_overrides.at(CacheEntryRole::kFileMetadata).charged; + if (bbto->block_cache && + file_metadata_charged == CacheEntryRoleOptions::Decision::kEnabled) { + // TODO(hx235): Add a `ConcurrentCacheReservationManager` at DB scope + // responsible for reservation of `ObsoleteFileInfo` so that we can keep + // this `file_metadata_cache_res_mgr_` nonconcurrent + file_metadata_cache_res_mgr_.reset(new ConcurrentCacheReservationManager( + std::make_shared< + CacheReservationManagerImpl>( + bbto->block_cache))); + } + } +} + +// DB mutex held +ColumnFamilyData::~ColumnFamilyData() { + assert(refs_.load(std::memory_order_relaxed) == 0); + // remove from linked list + auto prev = prev_; + auto next = next_; + prev->next_ = next; + next->prev_ = prev; + + if (!dropped_ && column_family_set_ != nullptr) { + // If it's dropped, it's already removed from column family set + // If column_family_set_ == nullptr, this is dummy CFD and not in + // ColumnFamilySet + column_family_set_->RemoveColumnFamily(this); + } + + if (current_ != nullptr) { + current_->Unref(); + } + + // It would be wrong if this ColumnFamilyData is in flush_queue_ or + // compaction_queue_ and we destroyed it + assert(!queued_for_flush_); + assert(!queued_for_compaction_); + assert(super_version_ == nullptr); + + if (dummy_versions_ != nullptr) { + // List must be empty + assert(dummy_versions_->Next() == dummy_versions_); + bool deleted __attribute__((__unused__)); + deleted = dummy_versions_->Unref(); + assert(deleted); + } + + if (mem_ != nullptr) { + delete mem_->Unref(); + } + autovector to_delete; + imm_.current()->Unref(&to_delete); + for (MemTable* m : to_delete) { + delete m; + } + + if (db_paths_registered_) { + // TODO(cc): considering using ioptions_.fs, currently some tests rely on + // EnvWrapper, that's the main reason why we use env here. + Status s = ioptions_.env->UnregisterDbPaths(GetDbPaths()); + if (!s.ok()) { + ROCKS_LOG_ERROR( + ioptions_.logger, + "Failed to unregister data paths of column family (id: %d, name: %s)", + id_, name_.c_str()); + } + } +} + +bool ColumnFamilyData::UnrefAndTryDelete() { + int old_refs = refs_.fetch_sub(1); + assert(old_refs > 0); + + if (old_refs == 1) { + assert(super_version_ == nullptr); + delete this; + return true; + } + + if (old_refs == 2 && super_version_ != nullptr) { + // Only the super_version_ holds me + SuperVersion* sv = super_version_; + super_version_ = nullptr; + + // Release SuperVersion references kept in ThreadLocalPtr. + local_sv_.reset(); + + if (sv->Unref()) { + // Note: sv will delete this ColumnFamilyData during Cleanup() + assert(sv->cfd == this); + sv->Cleanup(); + delete sv; + return true; + } + } + return false; +} + +void ColumnFamilyData::SetDropped() { + // can't drop default CF + assert(id_ != 0); + dropped_ = true; + write_controller_token_.reset(); + + // remove from column_family_set + column_family_set_->RemoveColumnFamily(this); +} + +ColumnFamilyOptions ColumnFamilyData::GetLatestCFOptions() const { + return BuildColumnFamilyOptions(initial_cf_options_, mutable_cf_options_); +} + +uint64_t ColumnFamilyData::OldestLogToKeep() { + auto current_log = GetLogNumber(); + + if (allow_2pc_) { + auto imm_prep_log = imm()->PrecomputeMinLogContainingPrepSection(); + auto mem_prep_log = mem()->GetMinLogContainingPrepSection(); + + if (imm_prep_log > 0 && imm_prep_log < current_log) { + current_log = imm_prep_log; + } + + if (mem_prep_log > 0 && mem_prep_log < current_log) { + current_log = mem_prep_log; + } + } + + return current_log; +} + +const double kIncSlowdownRatio = 0.8; +const double kDecSlowdownRatio = 1 / kIncSlowdownRatio; +const double kNearStopSlowdownRatio = 0.6; +const double kDelayRecoverSlowdownRatio = 1.4; + +namespace { +// If penalize_stop is true, we further reduce slowdown rate. +std::unique_ptr SetupDelay( + WriteController* write_controller, uint64_t compaction_needed_bytes, + uint64_t prev_compaction_need_bytes, bool penalize_stop, + bool auto_compactions_disabled) { + const uint64_t kMinWriteRate = 16 * 1024u; // Minimum write rate 16KB/s. + + uint64_t max_write_rate = write_controller->max_delayed_write_rate(); + uint64_t write_rate = write_controller->delayed_write_rate(); + + if (auto_compactions_disabled) { + // When auto compaction is disabled, always use the value user gave. + write_rate = max_write_rate; + } else if (write_controller->NeedsDelay() && max_write_rate > kMinWriteRate) { + // If user gives rate less than kMinWriteRate, don't adjust it. + // + // If already delayed, need to adjust based on previous compaction debt. + // When there are two or more column families require delay, we always + // increase or reduce write rate based on information for one single + // column family. It is likely to be OK but we can improve if there is a + // problem. + // Ignore compaction_needed_bytes = 0 case because compaction_needed_bytes + // is only available in level-based compaction + // + // If the compaction debt stays the same as previously, we also further slow + // down. It usually means a mem table is full. It's mainly for the case + // where both of flush and compaction are much slower than the speed we + // insert to mem tables, so we need to actively slow down before we get + // feedback signal from compaction and flushes to avoid the full stop + // because of hitting the max write buffer number. + // + // If DB just falled into the stop condition, we need to further reduce + // the write rate to avoid the stop condition. + if (penalize_stop) { + // Penalize the near stop or stop condition by more aggressive slowdown. + // This is to provide the long term slowdown increase signal. + // The penalty is more than the reward of recovering to the normal + // condition. + write_rate = static_cast(static_cast(write_rate) * + kNearStopSlowdownRatio); + if (write_rate < kMinWriteRate) { + write_rate = kMinWriteRate; + } + } else if (prev_compaction_need_bytes > 0 && + prev_compaction_need_bytes <= compaction_needed_bytes) { + write_rate = static_cast(static_cast(write_rate) * + kIncSlowdownRatio); + if (write_rate < kMinWriteRate) { + write_rate = kMinWriteRate; + } + } else if (prev_compaction_need_bytes > compaction_needed_bytes) { + // We are speeding up by ratio of kSlowdownRatio when we have paid + // compaction debt. But we'll never speed up to faster than the write rate + // given by users. + write_rate = static_cast(static_cast(write_rate) * + kDecSlowdownRatio); + if (write_rate > max_write_rate) { + write_rate = max_write_rate; + } + } + } + return write_controller->GetDelayToken(write_rate); +} + +int GetL0ThresholdSpeedupCompaction(int level0_file_num_compaction_trigger, + int level0_slowdown_writes_trigger) { + // SanitizeOptions() ensures it. + assert(level0_file_num_compaction_trigger <= level0_slowdown_writes_trigger); + + if (level0_file_num_compaction_trigger < 0) { + return std::numeric_limits::max(); + } + + const int64_t twice_level0_trigger = + static_cast(level0_file_num_compaction_trigger) * 2; + + const int64_t one_fourth_trigger_slowdown = + static_cast(level0_file_num_compaction_trigger) + + ((level0_slowdown_writes_trigger - level0_file_num_compaction_trigger) / + 4); + + assert(twice_level0_trigger >= 0); + assert(one_fourth_trigger_slowdown >= 0); + + // 1/4 of the way between L0 compaction trigger threshold and slowdown + // condition. + // Or twice as compaction trigger, if it is smaller. + int64_t res = std::min(twice_level0_trigger, one_fourth_trigger_slowdown); + if (res >= std::numeric_limits::max()) { + return std::numeric_limits::max(); + } else { + // res fits in int + return static_cast(res); + } +} +} // anonymous namespace + +std::pair +ColumnFamilyData::GetWriteStallConditionAndCause( + int num_unflushed_memtables, int num_l0_files, + uint64_t num_compaction_needed_bytes, + const MutableCFOptions& mutable_cf_options, + const ImmutableCFOptions& immutable_cf_options) { + if (num_unflushed_memtables >= mutable_cf_options.max_write_buffer_number) { + return {WriteStallCondition::kStopped, WriteStallCause::kMemtableLimit}; + } else if (!mutable_cf_options.disable_auto_compactions && + num_l0_files >= mutable_cf_options.level0_stop_writes_trigger) { + return {WriteStallCondition::kStopped, WriteStallCause::kL0FileCountLimit}; + } else if (!mutable_cf_options.disable_auto_compactions && + mutable_cf_options.hard_pending_compaction_bytes_limit > 0 && + num_compaction_needed_bytes >= + mutable_cf_options.hard_pending_compaction_bytes_limit) { + return {WriteStallCondition::kStopped, + WriteStallCause::kPendingCompactionBytes}; + } else if (mutable_cf_options.max_write_buffer_number > 3 && + num_unflushed_memtables >= + mutable_cf_options.max_write_buffer_number - 1 && + num_unflushed_memtables - 1 >= + immutable_cf_options.min_write_buffer_number_to_merge) { + return {WriteStallCondition::kDelayed, WriteStallCause::kMemtableLimit}; + } else if (!mutable_cf_options.disable_auto_compactions && + mutable_cf_options.level0_slowdown_writes_trigger >= 0 && + num_l0_files >= + mutable_cf_options.level0_slowdown_writes_trigger) { + return {WriteStallCondition::kDelayed, WriteStallCause::kL0FileCountLimit}; + } else if (!mutable_cf_options.disable_auto_compactions && + mutable_cf_options.soft_pending_compaction_bytes_limit > 0 && + num_compaction_needed_bytes >= + mutable_cf_options.soft_pending_compaction_bytes_limit) { + return {WriteStallCondition::kDelayed, + WriteStallCause::kPendingCompactionBytes}; + } + return {WriteStallCondition::kNormal, WriteStallCause::kNone}; +} + +WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions( + const MutableCFOptions& mutable_cf_options) { + auto write_stall_condition = WriteStallCondition::kNormal; + if (current_ != nullptr) { + auto* vstorage = current_->storage_info(); + auto write_controller = column_family_set_->write_controller_; + uint64_t compaction_needed_bytes = + vstorage->estimated_compaction_needed_bytes(); + + auto write_stall_condition_and_cause = GetWriteStallConditionAndCause( + imm()->NumNotFlushed(), vstorage->l0_delay_trigger_count(), + vstorage->estimated_compaction_needed_bytes(), mutable_cf_options, + *ioptions()); + write_stall_condition = write_stall_condition_and_cause.first; + auto write_stall_cause = write_stall_condition_and_cause.second; + + bool was_stopped = write_controller->IsStopped(); + bool needed_delay = write_controller->NeedsDelay(); + + if (write_stall_condition == WriteStallCondition::kStopped && + write_stall_cause == WriteStallCause::kMemtableLimit) { + write_controller_token_ = write_controller->GetStopToken(); + internal_stats_->AddCFStats(InternalStats::MEMTABLE_LIMIT_STOPS, 1); + ROCKS_LOG_WARN( + ioptions_.logger, + "[%s] Stopping writes because we have %d immutable memtables " + "(waiting for flush), max_write_buffer_number is set to %d", + name_.c_str(), imm()->NumNotFlushed(), + mutable_cf_options.max_write_buffer_number); + } else if (write_stall_condition == WriteStallCondition::kStopped && + write_stall_cause == WriteStallCause::kL0FileCountLimit) { + write_controller_token_ = write_controller->GetStopToken(); + internal_stats_->AddCFStats(InternalStats::L0_FILE_COUNT_LIMIT_STOPS, 1); + if (compaction_picker_->IsLevel0CompactionInProgress()) { + internal_stats_->AddCFStats( + InternalStats::L0_FILE_COUNT_LIMIT_STOPS_WITH_ONGOING_COMPACTION, + 1); + } + ROCKS_LOG_WARN(ioptions_.logger, + "[%s] Stopping writes because we have %d level-0 files", + name_.c_str(), vstorage->l0_delay_trigger_count()); + } else if (write_stall_condition == WriteStallCondition::kStopped && + write_stall_cause == WriteStallCause::kPendingCompactionBytes) { + write_controller_token_ = write_controller->GetStopToken(); + internal_stats_->AddCFStats( + InternalStats::PENDING_COMPACTION_BYTES_LIMIT_STOPS, 1); + ROCKS_LOG_WARN( + ioptions_.logger, + "[%s] Stopping writes because of estimated pending compaction " + "bytes %" PRIu64, + name_.c_str(), compaction_needed_bytes); + } else if (write_stall_condition == WriteStallCondition::kDelayed && + write_stall_cause == WriteStallCause::kMemtableLimit) { + write_controller_token_ = + SetupDelay(write_controller, compaction_needed_bytes, + prev_compaction_needed_bytes_, was_stopped, + mutable_cf_options.disable_auto_compactions); + internal_stats_->AddCFStats(InternalStats::MEMTABLE_LIMIT_DELAYS, 1); + ROCKS_LOG_WARN( + ioptions_.logger, + "[%s] Stalling writes because we have %d immutable memtables " + "(waiting for flush), max_write_buffer_number is set to %d " + "rate %" PRIu64, + name_.c_str(), imm()->NumNotFlushed(), + mutable_cf_options.max_write_buffer_number, + write_controller->delayed_write_rate()); + } else if (write_stall_condition == WriteStallCondition::kDelayed && + write_stall_cause == WriteStallCause::kL0FileCountLimit) { + // L0 is the last two files from stopping. + bool near_stop = vstorage->l0_delay_trigger_count() >= + mutable_cf_options.level0_stop_writes_trigger - 2; + write_controller_token_ = + SetupDelay(write_controller, compaction_needed_bytes, + prev_compaction_needed_bytes_, was_stopped || near_stop, + mutable_cf_options.disable_auto_compactions); + internal_stats_->AddCFStats(InternalStats::L0_FILE_COUNT_LIMIT_DELAYS, 1); + if (compaction_picker_->IsLevel0CompactionInProgress()) { + internal_stats_->AddCFStats( + InternalStats::L0_FILE_COUNT_LIMIT_DELAYS_WITH_ONGOING_COMPACTION, + 1); + } + ROCKS_LOG_WARN(ioptions_.logger, + "[%s] Stalling writes because we have %d level-0 files " + "rate %" PRIu64, + name_.c_str(), vstorage->l0_delay_trigger_count(), + write_controller->delayed_write_rate()); + } else if (write_stall_condition == WriteStallCondition::kDelayed && + write_stall_cause == WriteStallCause::kPendingCompactionBytes) { + // If the distance to hard limit is less than 1/4 of the gap between soft + // and + // hard bytes limit, we think it is near stop and speed up the slowdown. + bool near_stop = + mutable_cf_options.hard_pending_compaction_bytes_limit > 0 && + (compaction_needed_bytes - + mutable_cf_options.soft_pending_compaction_bytes_limit) > + 3 * + (mutable_cf_options.hard_pending_compaction_bytes_limit - + mutable_cf_options.soft_pending_compaction_bytes_limit) / + 4; + + write_controller_token_ = + SetupDelay(write_controller, compaction_needed_bytes, + prev_compaction_needed_bytes_, was_stopped || near_stop, + mutable_cf_options.disable_auto_compactions); + internal_stats_->AddCFStats( + InternalStats::PENDING_COMPACTION_BYTES_LIMIT_DELAYS, 1); + ROCKS_LOG_WARN( + ioptions_.logger, + "[%s] Stalling writes because of estimated pending compaction " + "bytes %" PRIu64 " rate %" PRIu64, + name_.c_str(), vstorage->estimated_compaction_needed_bytes(), + write_controller->delayed_write_rate()); + } else { + assert(write_stall_condition == WriteStallCondition::kNormal); + if (vstorage->l0_delay_trigger_count() >= + GetL0ThresholdSpeedupCompaction( + mutable_cf_options.level0_file_num_compaction_trigger, + mutable_cf_options.level0_slowdown_writes_trigger)) { + write_controller_token_ = + write_controller->GetCompactionPressureToken(); + ROCKS_LOG_INFO( + ioptions_.logger, + "[%s] Increasing compaction threads because we have %d level-0 " + "files ", + name_.c_str(), vstorage->l0_delay_trigger_count()); + } else if (vstorage->estimated_compaction_needed_bytes() >= + mutable_cf_options.soft_pending_compaction_bytes_limit / 4) { + // Increase compaction threads if bytes needed for compaction exceeds + // 1/4 of threshold for slowing down. + // If soft pending compaction byte limit is not set, always speed up + // compaction. + write_controller_token_ = + write_controller->GetCompactionPressureToken(); + if (mutable_cf_options.soft_pending_compaction_bytes_limit > 0) { + ROCKS_LOG_INFO( + ioptions_.logger, + "[%s] Increasing compaction threads because of estimated pending " + "compaction " + "bytes %" PRIu64, + name_.c_str(), vstorage->estimated_compaction_needed_bytes()); + } + } else { + write_controller_token_.reset(); + } + // If the DB recovers from delay conditions, we reward with reducing + // double the slowdown ratio. This is to balance the long term slowdown + // increase signal. + if (needed_delay) { + uint64_t write_rate = write_controller->delayed_write_rate(); + write_controller->set_delayed_write_rate(static_cast( + static_cast(write_rate) * kDelayRecoverSlowdownRatio)); + // Set the low pri limit to be 1/4 the delayed write rate. + // Note we don't reset this value even after delay condition is relased. + // Low-pri rate will continue to apply if there is a compaction + // pressure. + write_controller->low_pri_rate_limiter()->SetBytesPerSecond(write_rate / + 4); + } + } + prev_compaction_needed_bytes_ = compaction_needed_bytes; + } + return write_stall_condition; +} + +const FileOptions* ColumnFamilyData::soptions() const { + return &(column_family_set_->file_options_); +} + +void ColumnFamilyData::SetCurrent(Version* current_version) { + current_ = current_version; +} + +uint64_t ColumnFamilyData::GetNumLiveVersions() const { + return VersionSet::GetNumLiveVersions(dummy_versions_); +} + +uint64_t ColumnFamilyData::GetTotalSstFilesSize() const { + return VersionSet::GetTotalSstFilesSize(dummy_versions_); +} + +uint64_t ColumnFamilyData::GetTotalBlobFileSize() const { + return VersionSet::GetTotalBlobFileSize(dummy_versions_); +} + +uint64_t ColumnFamilyData::GetLiveSstFilesSize() const { + return current_->GetSstFilesSize(); +} + +MemTable* ColumnFamilyData::ConstructNewMemtable( + const MutableCFOptions& mutable_cf_options, SequenceNumber earliest_seq) { + return new MemTable(internal_comparator_, ioptions_, mutable_cf_options, + write_buffer_manager_, earliest_seq, id_); +} + +void ColumnFamilyData::CreateNewMemtable( + const MutableCFOptions& mutable_cf_options, SequenceNumber earliest_seq) { + if (mem_ != nullptr) { + delete mem_->Unref(); + } + SetMemtable(ConstructNewMemtable(mutable_cf_options, earliest_seq)); + mem_->Ref(); +} + +bool ColumnFamilyData::NeedsCompaction() const { + return !mutable_cf_options_.disable_auto_compactions && + compaction_picker_->NeedsCompaction(current_->storage_info()); +} + +Compaction* ColumnFamilyData::PickCompaction( + const MutableCFOptions& mutable_options, + const MutableDBOptions& mutable_db_options, LogBuffer* log_buffer) { + auto* result = compaction_picker_->PickCompaction( + GetName(), mutable_options, mutable_db_options, current_->storage_info(), + log_buffer); + if (result != nullptr) { + result->SetInputVersion(current_); + } + return result; +} + +bool ColumnFamilyData::RangeOverlapWithCompaction( + const Slice& smallest_user_key, const Slice& largest_user_key, + int level) const { + return compaction_picker_->RangeOverlapWithCompaction( + smallest_user_key, largest_user_key, level); +} + +Status ColumnFamilyData::RangesOverlapWithMemtables( + const autovector& ranges, SuperVersion* super_version, + bool allow_data_in_errors, bool* overlap) { + assert(overlap != nullptr); + *overlap = false; + // Create an InternalIterator over all unflushed memtables + Arena arena; + // TODO: plumb Env::IOActivity + ReadOptions read_opts; + read_opts.total_order_seek = true; + MergeIteratorBuilder merge_iter_builder(&internal_comparator_, &arena); + merge_iter_builder.AddIterator( + super_version->mem->NewIterator(read_opts, &arena)); + super_version->imm->AddIterators(read_opts, &merge_iter_builder, + false /* add_range_tombstone_iter */); + ScopedArenaIterator memtable_iter(merge_iter_builder.Finish()); + + auto read_seq = super_version->current->version_set()->LastSequence(); + ReadRangeDelAggregator range_del_agg(&internal_comparator_, read_seq); + auto* active_range_del_iter = super_version->mem->NewRangeTombstoneIterator( + read_opts, read_seq, false /* immutable_memtable */); + range_del_agg.AddTombstones( + std::unique_ptr(active_range_del_iter)); + Status status; + status = super_version->imm->AddRangeTombstoneIterators( + read_opts, nullptr /* arena */, &range_del_agg); + // AddRangeTombstoneIterators always return Status::OK. + assert(status.ok()); + + for (size_t i = 0; i < ranges.size() && status.ok() && !*overlap; ++i) { + auto* vstorage = super_version->current->storage_info(); + auto* ucmp = vstorage->InternalComparator()->user_comparator(); + InternalKey range_start(ranges[i].start, kMaxSequenceNumber, + kValueTypeForSeek); + memtable_iter->Seek(range_start.Encode()); + status = memtable_iter->status(); + ParsedInternalKey seek_result; + + if (status.ok() && memtable_iter->Valid()) { + status = ParseInternalKey(memtable_iter->key(), &seek_result, + allow_data_in_errors); + } + + if (status.ok()) { + if (memtable_iter->Valid() && + ucmp->Compare(seek_result.user_key, ranges[i].limit) <= 0) { + *overlap = true; + } else if (range_del_agg.IsRangeOverlapped(ranges[i].start, + ranges[i].limit)) { + *overlap = true; + } + } + } + return status; +} + +const int ColumnFamilyData::kCompactAllLevels = -1; +const int ColumnFamilyData::kCompactToBaseLevel = -2; + +Compaction* ColumnFamilyData::CompactRange( + const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, int input_level, + int output_level, const CompactRangeOptions& compact_range_options, + const InternalKey* begin, const InternalKey* end, + InternalKey** compaction_end, bool* conflict, + uint64_t max_file_num_to_ignore, const std::string& trim_ts) { + auto* result = compaction_picker_->CompactRange( + GetName(), mutable_cf_options, mutable_db_options, + current_->storage_info(), input_level, output_level, + compact_range_options, begin, end, compaction_end, conflict, + max_file_num_to_ignore, trim_ts); + if (result != nullptr) { + result->SetInputVersion(current_); + } + TEST_SYNC_POINT("ColumnFamilyData::CompactRange:Return"); + return result; +} + +SuperVersion* ColumnFamilyData::GetReferencedSuperVersion(DBImpl* db) { + SuperVersion* sv = GetThreadLocalSuperVersion(db); + sv->Ref(); + if (!ReturnThreadLocalSuperVersion(sv)) { + // This Unref() corresponds to the Ref() in GetThreadLocalSuperVersion() + // when the thread-local pointer was populated. So, the Ref() earlier in + // this function still prevents the returned SuperVersion* from being + // deleted out from under the caller. + sv->Unref(); + } + return sv; +} + +SuperVersion* ColumnFamilyData::GetThreadLocalSuperVersion(DBImpl* db) { + // The SuperVersion is cached in thread local storage to avoid acquiring + // mutex when SuperVersion does not change since the last use. When a new + // SuperVersion is installed, the compaction or flush thread cleans up + // cached SuperVersion in all existing thread local storage. To avoid + // acquiring mutex for this operation, we use atomic Swap() on the thread + // local pointer to guarantee exclusive access. If the thread local pointer + // is being used while a new SuperVersion is installed, the cached + // SuperVersion can become stale. In that case, the background thread would + // have swapped in kSVObsolete. We re-check the value at when returning + // SuperVersion back to thread local, with an atomic compare and swap. + // The superversion will need to be released if detected to be stale. + void* ptr = local_sv_->Swap(SuperVersion::kSVInUse); + // Invariant: + // (1) Scrape (always) installs kSVObsolete in ThreadLocal storage + // (2) the Swap above (always) installs kSVInUse, ThreadLocal storage + // should only keep kSVInUse before ReturnThreadLocalSuperVersion call + // (if no Scrape happens). + assert(ptr != SuperVersion::kSVInUse); + SuperVersion* sv = static_cast(ptr); + if (sv == SuperVersion::kSVObsolete) { + RecordTick(ioptions_.stats, NUMBER_SUPERVERSION_ACQUIRES); + db->mutex()->Lock(); + sv = super_version_->Ref(); + db->mutex()->Unlock(); + } + assert(sv != nullptr); + return sv; +} + +bool ColumnFamilyData::ReturnThreadLocalSuperVersion(SuperVersion* sv) { + assert(sv != nullptr); + // Put the SuperVersion back + void* expected = SuperVersion::kSVInUse; + if (local_sv_->CompareAndSwap(static_cast(sv), expected)) { + // When we see kSVInUse in the ThreadLocal, we are sure ThreadLocal + // storage has not been altered and no Scrape has happened. The + // SuperVersion is still current. + return true; + } else { + // ThreadLocal scrape happened in the process of this GetImpl call (after + // thread local Swap() at the beginning and before CompareAndSwap()). + // This means the SuperVersion it holds is obsolete. + assert(expected == SuperVersion::kSVObsolete); + } + return false; +} + +void ColumnFamilyData::InstallSuperVersion(SuperVersionContext* sv_context, + InstrumentedMutex* db_mutex) { + db_mutex->AssertHeld(); + return InstallSuperVersion(sv_context, mutable_cf_options_); +} + +void ColumnFamilyData::InstallSuperVersion( + SuperVersionContext* sv_context, + const MutableCFOptions& mutable_cf_options) { + SuperVersion* new_superversion = sv_context->new_superversion.release(); + new_superversion->mutable_cf_options = mutable_cf_options; + new_superversion->Init(this, mem_, imm_.current(), current_); + SuperVersion* old_superversion = super_version_; + super_version_ = new_superversion; + ++super_version_number_; + super_version_->version_number = super_version_number_; + if (old_superversion == nullptr || old_superversion->current != current() || + old_superversion->mem != mem_ || + old_superversion->imm != imm_.current()) { + // Should not recalculate slow down condition if nothing has changed, since + // currently RecalculateWriteStallConditions() treats it as further slowing + // down is needed. + super_version_->write_stall_condition = + RecalculateWriteStallConditions(mutable_cf_options); + } else { + super_version_->write_stall_condition = + old_superversion->write_stall_condition; + } + if (old_superversion != nullptr) { + // Reset SuperVersions cached in thread local storage. + // This should be done before old_superversion->Unref(). That's to ensure + // that local_sv_ never holds the last reference to SuperVersion, since + // it has no means to safely do SuperVersion cleanup. + ResetThreadLocalSuperVersions(); + + if (old_superversion->mutable_cf_options.write_buffer_size != + mutable_cf_options.write_buffer_size) { + mem_->UpdateWriteBufferSize(mutable_cf_options.write_buffer_size); + } + if (old_superversion->write_stall_condition != + new_superversion->write_stall_condition) { + sv_context->PushWriteStallNotification( + old_superversion->write_stall_condition, + new_superversion->write_stall_condition, GetName(), ioptions()); + } + if (old_superversion->Unref()) { + old_superversion->Cleanup(); + sv_context->superversions_to_free.push_back(old_superversion); + } + } +} + +void ColumnFamilyData::ResetThreadLocalSuperVersions() { + autovector sv_ptrs; + local_sv_->Scrape(&sv_ptrs, SuperVersion::kSVObsolete); + for (auto ptr : sv_ptrs) { + assert(ptr); + if (ptr == SuperVersion::kSVInUse) { + continue; + } + auto sv = static_cast(ptr); + bool was_last_ref __attribute__((__unused__)); + was_last_ref = sv->Unref(); + // sv couldn't have been the last reference because + // ResetThreadLocalSuperVersions() is called before + // unref'ing super_version_. + assert(!was_last_ref); + } +} + +Status ColumnFamilyData::ValidateOptions( + const DBOptions& db_options, const ColumnFamilyOptions& cf_options) { + Status s; + s = CheckCompressionSupported(cf_options); + if (s.ok() && db_options.allow_concurrent_memtable_write) { + s = CheckConcurrentWritesSupported(cf_options); + } + if (s.ok() && db_options.unordered_write && + cf_options.max_successive_merges != 0) { + s = Status::InvalidArgument( + "max_successive_merges > 0 is incompatible with unordered_write"); + } + if (s.ok()) { + s = CheckCFPathsSupported(db_options, cf_options); + } + if (!s.ok()) { + return s; + } + + if (cf_options.ttl > 0 && cf_options.ttl != kDefaultTtl) { + if (!cf_options.table_factory->IsInstanceOf( + TableFactory::kBlockBasedTableName())) { + return Status::NotSupported( + "TTL is only supported in Block-Based Table format. "); + } + } + + if (cf_options.periodic_compaction_seconds > 0 && + cf_options.periodic_compaction_seconds != kDefaultPeriodicCompSecs) { + if (!cf_options.table_factory->IsInstanceOf( + TableFactory::kBlockBasedTableName())) { + return Status::NotSupported( + "Periodic Compaction is only supported in " + "Block-Based Table format. "); + } + } + + const auto* ucmp = cf_options.comparator; + assert(ucmp); + if (ucmp->timestamp_size() > 0 && + !cf_options.persist_user_defined_timestamps) { + if (db_options.atomic_flush) { + return Status::NotSupported( + "Not persisting user-defined timestamps feature is not supported" + "in combination with atomic flush."); + } + if (db_options.allow_concurrent_memtable_write) { + return Status::NotSupported( + "Not persisting user-defined timestamps feature is not supported" + " in combination with concurrent memtable write."); + } + const char* comparator_name = cf_options.comparator->Name(); + size_t name_size = strlen(comparator_name); + const char* suffix = ".u64ts"; + size_t suffix_size = strlen(suffix); + if (name_size <= suffix_size || + strcmp(comparator_name + name_size - suffix_size, suffix) != 0) { + return Status::NotSupported( + "Not persisting user-defined timestamps" + "feature only support user-defined timestamps formatted as " + "uint64_t."); + } + } + + if (cf_options.enable_blob_garbage_collection) { + if (cf_options.blob_garbage_collection_age_cutoff < 0.0 || + cf_options.blob_garbage_collection_age_cutoff > 1.0) { + return Status::InvalidArgument( + "The age cutoff for blob garbage collection should be in the range " + "[0.0, 1.0]."); + } + if (cf_options.blob_garbage_collection_force_threshold < 0.0 || + cf_options.blob_garbage_collection_force_threshold > 1.0) { + return Status::InvalidArgument( + "The garbage ratio threshold for forcing blob garbage collection " + "should be in the range [0.0, 1.0]."); + } + } + + if (cf_options.compaction_style == kCompactionStyleFIFO && + db_options.max_open_files != -1 && cf_options.ttl > 0) { + return Status::NotSupported( + "FIFO compaction only supported with max_open_files = -1."); + } + + std::vector supported{0, 1, 2, 4, 8}; + if (std::find(supported.begin(), supported.end(), + cf_options.memtable_protection_bytes_per_key) == + supported.end()) { + return Status::NotSupported( + "Memtable per key-value checksum protection only supports 0, 1, 2, 4 " + "or 8 bytes per key."); + } + if (std::find(supported.begin(), supported.end(), + cf_options.block_protection_bytes_per_key) == supported.end()) { + return Status::NotSupported( + "Block per key-value checksum protection only supports 0, 1, 2, 4 " + "or 8 bytes per key."); + } + + if (!cf_options.compaction_options_fifo.file_temperature_age_thresholds + .empty()) { + if (cf_options.compaction_style != kCompactionStyleFIFO) { + return Status::NotSupported( + "Option file_temperature_age_thresholds only supports FIFO " + "compaction."); + } else if (cf_options.num_levels > 1) { + return Status::NotSupported( + "Option file_temperature_age_thresholds is only supported when " + "num_levels = 1."); + } else { + const auto& ages = + cf_options.compaction_options_fifo.file_temperature_age_thresholds; + assert(ages.size() >= 1); + // check that age is sorted + for (size_t i = 0; i < ages.size() - 1; ++i) { + if (ages[i].age >= ages[i + 1].age) { + return Status::NotSupported( + "Option file_temperature_age_thresholds requires elements to be " + "sorted in increasing order with respect to `age` field."); + } + } + } + } + return s; +} + +Status ColumnFamilyData::SetOptions( + const DBOptions& db_opts, + const std::unordered_map& options_map) { + ColumnFamilyOptions cf_opts = + BuildColumnFamilyOptions(initial_cf_options_, mutable_cf_options_); + ConfigOptions config_opts; + config_opts.mutable_options_only = true; + Status s = GetColumnFamilyOptionsFromMap(config_opts, cf_opts, options_map, + &cf_opts); + if (s.ok()) { + s = ValidateOptions(db_opts, cf_opts); + } + if (s.ok()) { + mutable_cf_options_ = MutableCFOptions(cf_opts); + mutable_cf_options_.RefreshDerivedOptions(ioptions_); + } + return s; +} + +// REQUIRES: DB mutex held +Env::WriteLifeTimeHint ColumnFamilyData::CalculateSSTWriteHint(int level) { + if (initial_cf_options_.compaction_style != kCompactionStyleLevel) { + return Env::WLTH_NOT_SET; + } + if (level == 0) { + return Env::WLTH_MEDIUM; + } + int base_level = current_->storage_info()->base_level(); + + // L1: medium, L2: long, ... + if (level - base_level >= 2) { + return Env::WLTH_EXTREME; + } else if (level < base_level) { + // There is no restriction which prevents level passed in to be smaller + // than base_level. + return Env::WLTH_MEDIUM; + } + return static_cast( + level - base_level + static_cast(Env::WLTH_MEDIUM)); +} + +Status ColumnFamilyData::AddDirectories( + std::map>* created_dirs) { + Status s; + assert(created_dirs != nullptr); + assert(data_dirs_.empty()); + for (auto& p : ioptions_.cf_paths) { + auto existing_dir = created_dirs->find(p.path); + + if (existing_dir == created_dirs->end()) { + std::unique_ptr path_directory; + s = DBImpl::CreateAndNewDirectory(ioptions_.fs.get(), p.path, + &path_directory); + if (!s.ok()) { + return s; + } + assert(path_directory != nullptr); + data_dirs_.emplace_back(path_directory.release()); + (*created_dirs)[p.path] = data_dirs_.back(); + } else { + data_dirs_.emplace_back(existing_dir->second); + } + } + assert(data_dirs_.size() == ioptions_.cf_paths.size()); + return s; +} + +FSDirectory* ColumnFamilyData::GetDataDir(size_t path_id) const { + if (data_dirs_.empty()) { + return nullptr; + } + + assert(path_id < data_dirs_.size()); + return data_dirs_[path_id].get(); +} + +bool ColumnFamilyData::ShouldPostponeFlushToRetainUDT( + uint64_t max_memtable_id) { + const Comparator* ucmp = user_comparator(); + const size_t ts_sz = ucmp->timestamp_size(); + if (ts_sz == 0 || ioptions_.persist_user_defined_timestamps) { + return false; + } + // If users set the `persist_user_defined_timestamps` flag to false, they + // should also set the `full_history_ts_low` flag to indicate the range of + // user-defined timestamps to retain in memory. Otherwise, we do not + // explicitly postpone flush to retain UDTs. + const std::string& full_history_ts_low = GetFullHistoryTsLow(); + if (full_history_ts_low.empty()) { + return false; + } +#ifndef NDEBUG + Slice last_table_newest_udt; +#endif /* !NDEBUG */ + for (const Slice& table_newest_udt : + imm()->GetTablesNewestUDT(max_memtable_id)) { + assert(table_newest_udt.size() == full_history_ts_low.size()); + assert(last_table_newest_udt.empty() || + ucmp->CompareTimestamp(table_newest_udt, last_table_newest_udt) >= + 0); + // Checking the newest UDT contained in MemTable with ascending ID up to + // `max_memtable_id`. MemTable with bigger ID will have newer UDT, return + // immediately on finding the first MemTable that needs postponing. + if (ucmp->CompareTimestamp(table_newest_udt, full_history_ts_low) >= 0) { + return true; + } +#ifndef NDEBUG + last_table_newest_udt = table_newest_udt; +#endif /* !NDEBUG */ + } + return false; +} + +void ColumnFamilyData::RecoverEpochNumbers() { + assert(current_); + auto* vstorage = current_->storage_info(); + assert(vstorage); + vstorage->RecoverEpochNumbers(this); +} + +ColumnFamilySet::ColumnFamilySet(const std::string& dbname, + const ImmutableDBOptions* db_options, + const FileOptions& file_options, + Cache* table_cache, + WriteBufferManager* _write_buffer_manager, + WriteController* _write_controller, + BlockCacheTracer* const block_cache_tracer, + const std::shared_ptr& io_tracer, + const std::string& db_id, + const std::string& db_session_id) + : max_column_family_(0), + file_options_(file_options), + dummy_cfd_(new ColumnFamilyData( + ColumnFamilyData::kDummyColumnFamilyDataId, "", nullptr, nullptr, + nullptr, ColumnFamilyOptions(), *db_options, &file_options_, nullptr, + block_cache_tracer, io_tracer, db_id, db_session_id)), + default_cfd_cache_(nullptr), + db_name_(dbname), + db_options_(db_options), + table_cache_(table_cache), + write_buffer_manager_(_write_buffer_manager), + write_controller_(_write_controller), + block_cache_tracer_(block_cache_tracer), + io_tracer_(io_tracer), + db_id_(db_id), + db_session_id_(db_session_id) { + // initialize linked list + dummy_cfd_->prev_ = dummy_cfd_; + dummy_cfd_->next_ = dummy_cfd_; +} + +ColumnFamilySet::~ColumnFamilySet() { + while (column_family_data_.size() > 0) { + // cfd destructor will delete itself from column_family_data_ + auto cfd = column_family_data_.begin()->second; + bool last_ref __attribute__((__unused__)); + last_ref = cfd->UnrefAndTryDelete(); + assert(last_ref); + } + bool dummy_last_ref __attribute__((__unused__)); + dummy_last_ref = dummy_cfd_->UnrefAndTryDelete(); + assert(dummy_last_ref); +} + +ColumnFamilyData* ColumnFamilySet::GetDefault() const { + assert(default_cfd_cache_ != nullptr); + return default_cfd_cache_; +} + +ColumnFamilyData* ColumnFamilySet::GetColumnFamily(uint32_t id) const { + auto cfd_iter = column_family_data_.find(id); + if (cfd_iter != column_family_data_.end()) { + return cfd_iter->second; + } else { + return nullptr; + } +} + +ColumnFamilyData* ColumnFamilySet::GetColumnFamily( + const std::string& name) const { + auto cfd_iter = column_families_.find(name); + if (cfd_iter != column_families_.end()) { + auto cfd = GetColumnFamily(cfd_iter->second); + assert(cfd != nullptr); + return cfd; + } else { + return nullptr; + } +} + +uint32_t ColumnFamilySet::GetNextColumnFamilyID() { + return ++max_column_family_; +} + +uint32_t ColumnFamilySet::GetMaxColumnFamily() { return max_column_family_; } + +void ColumnFamilySet::UpdateMaxColumnFamily(uint32_t new_max_column_family) { + max_column_family_ = std::max(new_max_column_family, max_column_family_); +} + +size_t ColumnFamilySet::NumberOfColumnFamilies() const { + return column_families_.size(); +} + +// under a DB mutex AND write thread +ColumnFamilyData* ColumnFamilySet::CreateColumnFamily( + const std::string& name, uint32_t id, Version* dummy_versions, + const ColumnFamilyOptions& options) { + assert(column_families_.find(name) == column_families_.end()); + ColumnFamilyData* new_cfd = new ColumnFamilyData( + id, name, dummy_versions, table_cache_, write_buffer_manager_, options, + *db_options_, &file_options_, this, block_cache_tracer_, io_tracer_, + db_id_, db_session_id_); + column_families_.insert({name, id}); + column_family_data_.insert({id, new_cfd}); + auto ucmp = new_cfd->user_comparator(); + assert(ucmp); + size_t ts_sz = ucmp->timestamp_size(); + running_ts_sz_.insert({id, ts_sz}); + if (ts_sz > 0) { + ts_sz_for_record_.insert({id, ts_sz}); + } + max_column_family_ = std::max(max_column_family_, id); + // add to linked list + new_cfd->next_ = dummy_cfd_; + auto prev = dummy_cfd_->prev_; + new_cfd->prev_ = prev; + prev->next_ = new_cfd; + dummy_cfd_->prev_ = new_cfd; + if (id == 0) { + default_cfd_cache_ = new_cfd; + } + return new_cfd; +} + +// under a DB mutex AND from a write thread +void ColumnFamilySet::RemoveColumnFamily(ColumnFamilyData* cfd) { + uint32_t cf_id = cfd->GetID(); + auto cfd_iter = column_family_data_.find(cf_id); + assert(cfd_iter != column_family_data_.end()); + column_family_data_.erase(cfd_iter); + column_families_.erase(cfd->GetName()); + running_ts_sz_.erase(cf_id); + ts_sz_for_record_.erase(cf_id); +} + +// under a DB mutex OR from a write thread +bool ColumnFamilyMemTablesImpl::Seek(uint32_t column_family_id) { + if (column_family_id == 0) { + // optimization for common case + current_ = column_family_set_->GetDefault(); + } else { + current_ = column_family_set_->GetColumnFamily(column_family_id); + } + handle_.SetCFD(current_); + return current_ != nullptr; +} + +uint64_t ColumnFamilyMemTablesImpl::GetLogNumber() const { + assert(current_ != nullptr); + return current_->GetLogNumber(); +} + +MemTable* ColumnFamilyMemTablesImpl::GetMemTable() const { + assert(current_ != nullptr); + return current_->mem(); +} + +ColumnFamilyHandle* ColumnFamilyMemTablesImpl::GetColumnFamilyHandle() { + assert(current_ != nullptr); + return &handle_; +} + +uint32_t GetColumnFamilyID(ColumnFamilyHandle* column_family) { + uint32_t column_family_id = 0; + if (column_family != nullptr) { + auto cfh = static_cast_with_check(column_family); + column_family_id = cfh->GetID(); + } + return column_family_id; +} + +const Comparator* GetColumnFamilyUserComparator( + ColumnFamilyHandle* column_family) { + if (column_family != nullptr) { + return column_family->GetComparator(); + } + return nullptr; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/column_family.h b/librocksdb-sys/rocksdb/db/column_family.h new file mode 100644 index 0000000..f976c24 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/column_family.h @@ -0,0 +1,876 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include +#include +#include +#include + +#include "cache/cache_reservation_manager.h" +#include "db/memtable_list.h" +#include "db/table_cache.h" +#include "db/table_properties_collector.h" +#include "db/write_batch_internal.h" +#include "db/write_controller.h" +#include "options/cf_options.h" +#include "rocksdb/compaction_job_stats.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "trace_replay/block_cache_tracer.h" +#include "util/hash_containers.h" +#include "util/thread_local.h" + +namespace ROCKSDB_NAMESPACE { + +class Version; +class VersionSet; +class VersionStorageInfo; +class MemTable; +class MemTableListVersion; +class CompactionPicker; +class Compaction; +class InternalKey; +class InternalStats; +class ColumnFamilyData; +class DBImpl; +class LogBuffer; +class InstrumentedMutex; +class InstrumentedMutexLock; +struct SuperVersionContext; +class BlobFileCache; +class BlobSource; + +extern const double kIncSlowdownRatio; +// This file contains a list of data structures for managing column family +// level metadata. +// +// The basic relationships among classes declared here are illustrated as +// following: +// +// +----------------------+ +----------------------+ +--------+ +// +---+ ColumnFamilyHandle 1 | +--+ ColumnFamilyHandle 2 | | DBImpl | +// | +----------------------+ | +----------------------+ +----+---+ +// | +--------------------------+ | +// | | +-----------------------------+ +// | | | +// | | +-----------------------------v-------------------------------+ +// | | | | +// | | | ColumnFamilySet | +// | | | | +// | | +-------------+--------------------------+----------------+---+ +// | | | | | +// | +-------------------------------------+ | | +// | | | | v +// | +-------------v-------------+ +-----v----v---------+ +// | | | | | +// | | ColumnFamilyData 1 | | ColumnFamilyData 2 | ...... +// | | | | | +// +---> | | | +// | +---------+ | | +// | | MemTable| | | +// | | List | | | +// +--------+---+--+-+----+----+ +--------------------++ +// | | | | +// | | | | +// | | | +-----------------------+ +// | | +-----------+ | +// v +--------+ | | +// +--------+--------+ | | | +// | | | | +----------v----------+ +// +---> |SuperVersion 1.a +-----------------> | +// | +------+ | | MemTableListVersion | +// +---+-------------+ | | | | | +// | | | | +----+------------+---+ +// | current | | | | | +// | +-------------+ | |mem | | +// | | | | | | +// +-v---v-------+ +---v--v---+ +-----v----+ +----v-----+ +// | | | | | | | | +// | Version 1.a | | memtable | | memtable | | memtable | +// | | | 1.a | | 1.b | | 1.c | +// +-------------+ | | | | | | +// +----------+ +----------+ +----------+ +// +// DBImpl keeps a ColumnFamilySet, which references to all column families by +// pointing to respective ColumnFamilyData object of each column family. +// This is how DBImpl can list and operate on all the column families. +// ColumnFamilyHandle also points to ColumnFamilyData directly, so that +// when a user executes a query, it can directly find memtables and Version +// as well as SuperVersion to the column family, without going through +// ColumnFamilySet. +// +// ColumnFamilySet points to the latest view of the LSM-tree (list of memtables +// and SST files) indirectly, while ongoing operations may hold references +// to a current or an out-of-date SuperVersion, which in turn points to a +// point-in-time view of the LSM-tree. This guarantees the memtables and SST +// files being operated on will not go away, until the SuperVersion is +// unreferenced to 0 and destoryed. +// +// The following graph illustrates a possible referencing relationships: +// +// Column +--------------+ current +-----------+ +// Family +---->+ +------------------->+ | +// Data | SuperVersion +----------+ | Version A | +// | 3 | imm | | | +// Iter2 +----->+ | +-------v------+ +-----------+ +// +-----+--------+ | MemtableList +----------------> Empty +// | | Version r | +-----------+ +// | +--------------+ | | +// +------------------+ current| Version B | +// +--------------+ | +----->+ | +// | | | | +-----+-----+ +// Compaction +>+ SuperVersion +-------------+ ^ +// Job | 2 +------+ | |current +// | +----+ | | mem | +------------+ +// +--------------+ | | +---------------------> | +// | +------------------------> MemTable a | +// | mem | | | +// +--------------+ | | +------------+ +// | +--------------------------+ +// Iter1 +-----> SuperVersion | | +------------+ +// | 1 +------------------------------>+ | +// | +-+ | mem | MemTable b | +// +--------------+ | | | | +// | | +--------------+ +-----^------+ +// | |imm | MemtableList | | +// | +--->+ Version s +------------+ +// | +--------------+ +// | +--------------+ +// | | MemtableList | +// +------>+ Version t +--------> Empty +// imm +--------------+ +// +// In this example, even if the current LSM-tree consists of Version A and +// memtable a, which is also referenced by SuperVersion, two older SuperVersion +// SuperVersion2 and Superversion1 still exist, and are referenced by a +// compaction job and an old iterator Iter1, respectively. SuperVersion2 +// contains Version B, memtable a and memtable b; SuperVersion1 contains +// Version B and memtable b (mutable). As a result, Version B and memtable b +// are prevented from being destroyed or deleted. + +// ColumnFamilyHandleImpl is the class that clients use to access different +// column families. It has non-trivial destructor, which gets called when client +// is done using the column family +class ColumnFamilyHandleImpl : public ColumnFamilyHandle { + public: + // create while holding the mutex + ColumnFamilyHandleImpl(ColumnFamilyData* cfd, DBImpl* db, + InstrumentedMutex* mutex); + // destroy without mutex + virtual ~ColumnFamilyHandleImpl(); + virtual ColumnFamilyData* cfd() const { return cfd_; } + + virtual uint32_t GetID() const override; + virtual const std::string& GetName() const override; + virtual Status GetDescriptor(ColumnFamilyDescriptor* desc) override; + virtual const Comparator* GetComparator() const override; + + private: + ColumnFamilyData* cfd_; + DBImpl* db_; + InstrumentedMutex* mutex_; +}; + +// Does not ref-count ColumnFamilyData +// We use this dummy ColumnFamilyHandleImpl because sometimes MemTableInserter +// calls DBImpl methods. When this happens, MemTableInserter need access to +// ColumnFamilyHandle (same as the client would need). In that case, we feed +// MemTableInserter dummy ColumnFamilyHandle and enable it to call DBImpl +// methods +class ColumnFamilyHandleInternal : public ColumnFamilyHandleImpl { + public: + ColumnFamilyHandleInternal() + : ColumnFamilyHandleImpl(nullptr, nullptr, nullptr), + internal_cfd_(nullptr) {} + + void SetCFD(ColumnFamilyData* _cfd) { internal_cfd_ = _cfd; } + virtual ColumnFamilyData* cfd() const override { return internal_cfd_; } + + private: + ColumnFamilyData* internal_cfd_; +}; + +// holds references to memtable, all immutable memtables and version +struct SuperVersion { + // Accessing members of this class is not thread-safe and requires external + // synchronization (ie db mutex held or on write thread). + ColumnFamilyData* cfd; + MemTable* mem; + MemTableListVersion* imm; + Version* current; + MutableCFOptions mutable_cf_options; + // Version number of the current SuperVersion + uint64_t version_number; + WriteStallCondition write_stall_condition; + + // should be called outside the mutex + SuperVersion() = default; + ~SuperVersion(); + SuperVersion* Ref(); + // If Unref() returns true, Cleanup() should be called with mutex held + // before deleting this SuperVersion. + bool Unref(); + + // call these two methods with db mutex held + // Cleanup unrefs mem, imm and current. Also, it stores all memtables + // that needs to be deleted in to_delete vector. Unrefing those + // objects needs to be done in the mutex + void Cleanup(); + void Init(ColumnFamilyData* new_cfd, MemTable* new_mem, + MemTableListVersion* new_imm, Version* new_current); + + // The value of dummy is not actually used. kSVInUse takes its address as a + // mark in the thread local storage to indicate the SuperVersion is in use + // by thread. This way, the value of kSVInUse is guaranteed to have no + // conflict with SuperVersion object address and portable on different + // platform. + static int dummy; + static void* const kSVInUse; + static void* const kSVObsolete; + + private: + std::atomic refs; + // We need to_delete because during Cleanup(), imm->Unref() returns + // all memtables that we need to free through this vector. We then + // delete all those memtables outside of mutex, during destruction + autovector to_delete; +}; + +extern Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options); + +extern Status CheckConcurrentWritesSupported( + const ColumnFamilyOptions& cf_options); + +extern Status CheckCFPathsSupported(const DBOptions& db_options, + const ColumnFamilyOptions& cf_options); + +extern ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options, + const ColumnFamilyOptions& src); +// Wrap user defined table properties collector factories `from cf_options` +// into internal ones in int_tbl_prop_collector_factories. Add a system internal +// one too. +extern void GetIntTblPropCollectorFactory( + const ImmutableCFOptions& ioptions, + IntTblPropCollectorFactories* int_tbl_prop_collector_factories); + +class ColumnFamilySet; + +// This class keeps all the data that a column family needs. +// Most methods require DB mutex held, unless otherwise noted +class ColumnFamilyData { + public: + ~ColumnFamilyData(); + + // thread-safe + uint32_t GetID() const { return id_; } + // thread-safe + const std::string& GetName() const { return name_; } + + // Ref() can only be called from a context where the caller can guarantee + // that ColumnFamilyData is alive (while holding a non-zero ref already, + // holding a DB mutex, or as the leader in a write batch group). + void Ref() { refs_.fetch_add(1); } + + // UnrefAndTryDelete() decreases the reference count and do free if needed, + // return true if this is freed else false, UnrefAndTryDelete() can only + // be called while holding a DB mutex, or during single-threaded recovery. + bool UnrefAndTryDelete(); + + // SetDropped() can only be called under following conditions: + // 1) Holding a DB mutex, + // 2) from single-threaded write thread, AND + // 3) from single-threaded VersionSet::LogAndApply() + // After dropping column family no other operation on that column family + // will be executed. All the files and memory will be, however, kept around + // until client drops the column family handle. That way, client can still + // access data from dropped column family. + // Column family can be dropped and still alive. In that state: + // *) Compaction and flush is not executed on the dropped column family. + // *) Client can continue reading from column family. Writes will fail unless + // WriteOptions::ignore_missing_column_families is true + // When the dropped column family is unreferenced, then we: + // *) Remove column family from the linked list maintained by ColumnFamilySet + // *) delete all memory associated with that column family + // *) delete all the files associated with that column family + void SetDropped(); + bool IsDropped() const { return dropped_.load(std::memory_order_relaxed); } + + // thread-safe + int NumberLevels() const { return ioptions_.num_levels; } + + void SetLogNumber(uint64_t log_number) { log_number_ = log_number; } + uint64_t GetLogNumber() const { return log_number_; } + + // thread-safe + const FileOptions* soptions() const; + const ImmutableOptions* ioptions() const { return &ioptions_; } + // REQUIRES: DB mutex held + // This returns the MutableCFOptions used by current SuperVersion + // You should use this API to reference MutableCFOptions most of the time. + const MutableCFOptions* GetCurrentMutableCFOptions() const { + return &(super_version_->mutable_cf_options); + } + // REQUIRES: DB mutex held + // This returns the latest MutableCFOptions, which may be not in effect yet. + const MutableCFOptions* GetLatestMutableCFOptions() const { + return &mutable_cf_options_; + } + + // REQUIRES: DB mutex held + // Build ColumnFamiliesOptions with immutable options and latest mutable + // options. + ColumnFamilyOptions GetLatestCFOptions() const; + + bool is_delete_range_supported() { return is_delete_range_supported_; } + + // Validate CF options against DB options + static Status ValidateOptions(const DBOptions& db_options, + const ColumnFamilyOptions& cf_options); + // REQUIRES: DB mutex held + Status SetOptions( + const DBOptions& db_options, + const std::unordered_map& options_map); + + InternalStats* internal_stats() { return internal_stats_.get(); } + + MemTableList* imm() { return &imm_; } + MemTable* mem() { return mem_; } + + bool IsEmpty() { + return mem()->GetFirstSequenceNumber() == 0 && imm()->NumNotFlushed() == 0; + } + + Version* current() { return current_; } + Version* dummy_versions() { return dummy_versions_; } + void SetCurrent(Version* _current); + uint64_t GetNumLiveVersions() const; // REQUIRE: DB mutex held + uint64_t GetTotalSstFilesSize() const; // REQUIRE: DB mutex held + uint64_t GetLiveSstFilesSize() const; // REQUIRE: DB mutex held + uint64_t GetTotalBlobFileSize() const; // REQUIRE: DB mutex held + void SetMemtable(MemTable* new_mem) { + uint64_t memtable_id = last_memtable_id_.fetch_add(1) + 1; + new_mem->SetID(memtable_id); + mem_ = new_mem; + } + + // calculate the oldest log needed for the durability of this column family + uint64_t OldestLogToKeep(); + + // See Memtable constructor for explanation of earliest_seq param. + MemTable* ConstructNewMemtable(const MutableCFOptions& mutable_cf_options, + SequenceNumber earliest_seq); + void CreateNewMemtable(const MutableCFOptions& mutable_cf_options, + SequenceNumber earliest_seq); + + TableCache* table_cache() const { return table_cache_.get(); } + BlobSource* blob_source() const { return blob_source_.get(); } + + // See documentation in compaction_picker.h + // REQUIRES: DB mutex held + bool NeedsCompaction() const; + // REQUIRES: DB mutex held + Compaction* PickCompaction(const MutableCFOptions& mutable_options, + const MutableDBOptions& mutable_db_options, + LogBuffer* log_buffer); + + // Check if the passed range overlap with any running compactions. + // REQUIRES: DB mutex held + bool RangeOverlapWithCompaction(const Slice& smallest_user_key, + const Slice& largest_user_key, + int level) const; + + // Check if the passed ranges overlap with any unflushed memtables + // (immutable or mutable). + // + // @param super_version A referenced SuperVersion that will be held for the + // duration of this function. + // + // Thread-safe + Status RangesOverlapWithMemtables(const autovector& ranges, + SuperVersion* super_version, + bool allow_data_in_errors, bool* overlap); + + // A flag to tell a manual compaction is to compact all levels together + // instead of a specific level. + static const int kCompactAllLevels; + // A flag to tell a manual compaction's output is base level. + static const int kCompactToBaseLevel; + // REQUIRES: DB mutex held + Compaction* CompactRange(const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, + int input_level, int output_level, + const CompactRangeOptions& compact_range_options, + const InternalKey* begin, const InternalKey* end, + InternalKey** compaction_end, bool* manual_conflict, + uint64_t max_file_num_to_ignore, + const std::string& trim_ts); + + CompactionPicker* compaction_picker() { return compaction_picker_.get(); } + // thread-safe + const Comparator* user_comparator() const { + return internal_comparator_.user_comparator(); + } + // thread-safe + const InternalKeyComparator& internal_comparator() const { + return internal_comparator_; + } + + const IntTblPropCollectorFactories* int_tbl_prop_collector_factories() const { + return &int_tbl_prop_collector_factories_; + } + + SuperVersion* GetSuperVersion() { return super_version_; } + // thread-safe + // Return a already referenced SuperVersion to be used safely. + SuperVersion* GetReferencedSuperVersion(DBImpl* db); + // thread-safe + // Get SuperVersion stored in thread local storage. If it does not exist, + // get a reference from a current SuperVersion. + SuperVersion* GetThreadLocalSuperVersion(DBImpl* db); + // Try to return SuperVersion back to thread local storage. Return true on + // success and false on failure. It fails when the thread local storage + // contains anything other than SuperVersion::kSVInUse flag. + bool ReturnThreadLocalSuperVersion(SuperVersion* sv); + // thread-safe + uint64_t GetSuperVersionNumber() const { + return super_version_number_.load(); + } + // will return a pointer to SuperVersion* if previous SuperVersion + // if its reference count is zero and needs deletion or nullptr if not + // As argument takes a pointer to allocated SuperVersion to enable + // the clients to allocate SuperVersion outside of mutex. + // IMPORTANT: Only call this from DBImpl::InstallSuperVersion() + void InstallSuperVersion(SuperVersionContext* sv_context, + const MutableCFOptions& mutable_cf_options); + void InstallSuperVersion(SuperVersionContext* sv_context, + InstrumentedMutex* db_mutex); + + void ResetThreadLocalSuperVersions(); + + // Protected by DB mutex + void set_queued_for_flush(bool value) { queued_for_flush_ = value; } + void set_queued_for_compaction(bool value) { queued_for_compaction_ = value; } + bool queued_for_flush() { return queued_for_flush_; } + bool queued_for_compaction() { return queued_for_compaction_; } + + static std::pair + GetWriteStallConditionAndCause( + int num_unflushed_memtables, int num_l0_files, + uint64_t num_compaction_needed_bytes, + const MutableCFOptions& mutable_cf_options, + const ImmutableCFOptions& immutable_cf_options); + + // Recalculate some stall conditions, which are changed only during + // compaction, adding new memtable and/or recalculation of compaction score. + WriteStallCondition RecalculateWriteStallConditions( + const MutableCFOptions& mutable_cf_options); + + void set_initialized() { initialized_.store(true); } + + bool initialized() const { return initialized_.load(); } + + const ColumnFamilyOptions& initial_cf_options() { + return initial_cf_options_; + } + + Env::WriteLifeTimeHint CalculateSSTWriteHint(int level); + + // created_dirs remembers directory created, so that we don't need to call + // the same data creation operation again. + Status AddDirectories( + std::map>* created_dirs); + + FSDirectory* GetDataDir(size_t path_id) const; + + // full_history_ts_low_ can only increase. + void SetFullHistoryTsLow(std::string ts_low) { + assert(!ts_low.empty()); + const Comparator* ucmp = user_comparator(); + assert(ucmp); + if (full_history_ts_low_.empty() || + ucmp->CompareTimestamp(ts_low, full_history_ts_low_) > 0) { + full_history_ts_low_ = std::move(ts_low); + } + } + + const std::string& GetFullHistoryTsLow() const { + return full_history_ts_low_; + } + + // REQUIRES: DB mutex held. + // Return true if flushing up to MemTables with ID `max_memtable_id` + // should be postponed to retain user-defined timestamps according to the + // user's setting. Called by background flush job. + bool ShouldPostponeFlushToRetainUDT(uint64_t max_memtable_id); + + ThreadLocalPtr* TEST_GetLocalSV() { return local_sv_.get(); } + WriteBufferManager* write_buffer_mgr() { return write_buffer_manager_; } + std::shared_ptr + GetFileMetadataCacheReservationManager() { + return file_metadata_cache_res_mgr_; + } + + SequenceNumber GetFirstMemtableSequenceNumber() const; + + static const uint32_t kDummyColumnFamilyDataId; + + // Keep track of whether the mempurge feature was ever used. + void SetMempurgeUsed() { mempurge_used_ = true; } + bool GetMempurgeUsed() { return mempurge_used_; } + + // Allocate and return a new epoch number + uint64_t NewEpochNumber() { return next_epoch_number_.fetch_add(1); } + + // Get the next epoch number to be assigned + uint64_t GetNextEpochNumber() const { return next_epoch_number_.load(); } + + // Set the next epoch number to be assigned + void SetNextEpochNumber(uint64_t next_epoch_number) { + next_epoch_number_.store(next_epoch_number); + } + + // Reset the next epoch number to be assigned + void ResetNextEpochNumber() { next_epoch_number_.store(1); } + + // Recover the next epoch number of this CF and epoch number + // of its files (if missing) + void RecoverEpochNumbers(); + + private: + friend class ColumnFamilySet; + ColumnFamilyData(uint32_t id, const std::string& name, + Version* dummy_versions, Cache* table_cache, + WriteBufferManager* write_buffer_manager, + const ColumnFamilyOptions& options, + const ImmutableDBOptions& db_options, + const FileOptions* file_options, + ColumnFamilySet* column_family_set, + BlockCacheTracer* const block_cache_tracer, + const std::shared_ptr& io_tracer, + const std::string& db_id, const std::string& db_session_id); + + std::vector GetDbPaths() const; + + uint32_t id_; + const std::string name_; + Version* dummy_versions_; // Head of circular doubly-linked list of versions. + Version* current_; // == dummy_versions->prev_ + + std::atomic refs_; // outstanding references to ColumnFamilyData + std::atomic initialized_; + std::atomic dropped_; // true if client dropped it + + const InternalKeyComparator internal_comparator_; + IntTblPropCollectorFactories int_tbl_prop_collector_factories_; + + const ColumnFamilyOptions initial_cf_options_; + const ImmutableOptions ioptions_; + MutableCFOptions mutable_cf_options_; + + const bool is_delete_range_supported_; + + std::unique_ptr table_cache_; + std::unique_ptr blob_file_cache_; + std::unique_ptr blob_source_; + + std::unique_ptr internal_stats_; + + WriteBufferManager* write_buffer_manager_; + + MemTable* mem_; + MemTableList imm_; + SuperVersion* super_version_; + + // An ordinal representing the current SuperVersion. Updated by + // InstallSuperVersion(), i.e. incremented every time super_version_ + // changes. + std::atomic super_version_number_; + + // Thread's local copy of SuperVersion pointer + // This needs to be destructed before mutex_ + std::unique_ptr local_sv_; + + // pointers for a circular linked list. we use it to support iterations over + // all column families that are alive (note: dropped column families can also + // be alive as long as client holds a reference) + ColumnFamilyData* next_; + ColumnFamilyData* prev_; + + // This is the earliest log file number that contains data from this + // Column Family. All earlier log files must be ignored and not + // recovered from + uint64_t log_number_; + + // An object that keeps all the compaction stats + // and picks the next compaction + std::unique_ptr compaction_picker_; + + ColumnFamilySet* column_family_set_; + + std::unique_ptr write_controller_token_; + + // If true --> this ColumnFamily is currently present in DBImpl::flush_queue_ + bool queued_for_flush_; + + // If true --> this ColumnFamily is currently present in + // DBImpl::compaction_queue_ + bool queued_for_compaction_; + + uint64_t prev_compaction_needed_bytes_; + + // if the database was opened with 2pc enabled + bool allow_2pc_; + + // Memtable id to track flush. + std::atomic last_memtable_id_; + + // Directories corresponding to cf_paths. + std::vector> data_dirs_; + + bool db_paths_registered_; + + std::string full_history_ts_low_; + + // For charging memory usage of file metadata created for newly added files to + // a Version associated with this CFD + std::shared_ptr file_metadata_cache_res_mgr_; + bool mempurge_used_; + + std::atomic next_epoch_number_; +}; + +// ColumnFamilySet has interesting thread-safety requirements +// * CreateColumnFamily() or RemoveColumnFamily() -- need to be protected by DB +// mutex AND executed in the write thread. +// CreateColumnFamily() should ONLY be called from VersionSet::LogAndApply() AND +// single-threaded write thread. It is also called during Recovery and in +// DumpManifest(). +// RemoveColumnFamily() is only called from SetDropped(). DB mutex needs to be +// held and it needs to be executed from the write thread. SetDropped() also +// guarantees that it will be called only from single-threaded LogAndApply(), +// but this condition is not that important. +// * Iteration -- hold DB mutex. If you want to release the DB mutex in the +// body of the iteration, wrap in a RefedColumnFamilySet. +// * GetDefault() -- thread safe +// * GetColumnFamily() -- either inside of DB mutex or from a write thread +// * GetNextColumnFamilyID(), GetMaxColumnFamily(), UpdateMaxColumnFamily(), +// NumberOfColumnFamilies -- inside of DB mutex +class ColumnFamilySet { + public: + // ColumnFamilySet supports iteration + class iterator { + public: + explicit iterator(ColumnFamilyData* cfd) : current_(cfd) {} + // NOTE: minimum operators for for-loop iteration + iterator& operator++() { + current_ = current_->next_; + return *this; + } + bool operator!=(const iterator& other) const { + return this->current_ != other.current_; + } + ColumnFamilyData* operator*() { return current_; } + + private: + ColumnFamilyData* current_; + }; + + ColumnFamilySet(const std::string& dbname, + const ImmutableDBOptions* db_options, + const FileOptions& file_options, Cache* table_cache, + WriteBufferManager* _write_buffer_manager, + WriteController* _write_controller, + BlockCacheTracer* const block_cache_tracer, + const std::shared_ptr& io_tracer, + const std::string& db_id, const std::string& db_session_id); + ~ColumnFamilySet(); + + ColumnFamilyData* GetDefault() const; + // GetColumnFamily() calls return nullptr if column family is not found + ColumnFamilyData* GetColumnFamily(uint32_t id) const; + ColumnFamilyData* GetColumnFamily(const std::string& name) const; + // this call will return the next available column family ID. it guarantees + // that there is no column family with id greater than or equal to the + // returned value in the current running instance or anytime in RocksDB + // instance history. + uint32_t GetNextColumnFamilyID(); + uint32_t GetMaxColumnFamily(); + void UpdateMaxColumnFamily(uint32_t new_max_column_family); + size_t NumberOfColumnFamilies() const; + + ColumnFamilyData* CreateColumnFamily(const std::string& name, uint32_t id, + Version* dummy_version, + const ColumnFamilyOptions& options); + + const UnorderedMap& GetRunningColumnFamiliesTimestampSize() + const { + return running_ts_sz_; + } + + const UnorderedMap& + GetColumnFamiliesTimestampSizeForRecord() const { + return ts_sz_for_record_; + } + + iterator begin() { return iterator(dummy_cfd_->next_); } + iterator end() { return iterator(dummy_cfd_); } + + Cache* get_table_cache() { return table_cache_; } + + WriteBufferManager* write_buffer_manager() { return write_buffer_manager_; } + + WriteController* write_controller() { return write_controller_; } + + private: + friend class ColumnFamilyData; + // helper function that gets called from cfd destructor + // REQUIRES: DB mutex held + void RemoveColumnFamily(ColumnFamilyData* cfd); + + // column_families_ and column_family_data_ need to be protected: + // * when mutating both conditions have to be satisfied: + // 1. DB mutex locked + // 2. thread currently in single-threaded write thread + // * when reading, at least one condition needs to be satisfied: + // 1. DB mutex locked + // 2. accessed from a single-threaded write thread + UnorderedMap column_families_; + UnorderedMap column_family_data_; + + // Mutating / reading `running_ts_sz_` and `ts_sz_for_record_` follow + // the same requirements as `column_families_` and `column_family_data_`. + // Mapping from column family id to user-defined timestamp size for all + // running column families. + UnorderedMap running_ts_sz_; + // Mapping from column family id to user-defined timestamp size for + // column families with non-zero user-defined timestamp size. + UnorderedMap ts_sz_for_record_; + + uint32_t max_column_family_; + const FileOptions file_options_; + + ColumnFamilyData* dummy_cfd_; + // We don't hold the refcount here, since default column family always exists + // We are also not responsible for cleaning up default_cfd_cache_. This is + // just a cache that makes common case (accessing default column family) + // faster + ColumnFamilyData* default_cfd_cache_; + + const std::string db_name_; + const ImmutableDBOptions* const db_options_; + Cache* table_cache_; + WriteBufferManager* write_buffer_manager_; + WriteController* write_controller_; + BlockCacheTracer* const block_cache_tracer_; + std::shared_ptr io_tracer_; + const std::string& db_id_; + std::string db_session_id_; +}; + +// A wrapper for ColumnFamilySet that supports releasing DB mutex during each +// iteration over the iterator, because the cfd is Refed and Unrefed during +// each iteration to prevent concurrent CF drop from destroying it (until +// Unref). +class RefedColumnFamilySet { + public: + explicit RefedColumnFamilySet(ColumnFamilySet* cfs) : wrapped_(cfs) {} + + class iterator { + public: + explicit iterator(ColumnFamilySet::iterator wrapped) : wrapped_(wrapped) { + MaybeRef(*wrapped_); + } + ~iterator() { MaybeUnref(*wrapped_); } + inline void MaybeRef(ColumnFamilyData* cfd) { + if (cfd->GetID() != ColumnFamilyData::kDummyColumnFamilyDataId) { + cfd->Ref(); + } + } + inline void MaybeUnref(ColumnFamilyData* cfd) { + if (cfd->GetID() != ColumnFamilyData::kDummyColumnFamilyDataId) { + cfd->UnrefAndTryDelete(); + } + } + // NOTE: minimum operators for for-loop iteration + inline iterator& operator++() { + ColumnFamilyData* old = *wrapped_; + ++wrapped_; + // Can only unref & potentially free cfd after accessing its next_ + MaybeUnref(old); + MaybeRef(*wrapped_); + return *this; + } + inline bool operator!=(const iterator& other) const { + return this->wrapped_ != other.wrapped_; + } + inline ColumnFamilyData* operator*() { return *wrapped_; } + + private: + ColumnFamilySet::iterator wrapped_; + }; + + iterator begin() { return iterator(wrapped_->begin()); } + iterator end() { return iterator(wrapped_->end()); } + + private: + ColumnFamilySet* wrapped_; +}; + +// We use ColumnFamilyMemTablesImpl to provide WriteBatch a way to access +// memtables of different column families (specified by ID in the write batch) +class ColumnFamilyMemTablesImpl : public ColumnFamilyMemTables { + public: + explicit ColumnFamilyMemTablesImpl(ColumnFamilySet* column_family_set) + : column_family_set_(column_family_set), current_(nullptr) {} + + // Constructs a ColumnFamilyMemTablesImpl equivalent to one constructed + // with the arguments used to construct *orig. + explicit ColumnFamilyMemTablesImpl(ColumnFamilyMemTablesImpl* orig) + : column_family_set_(orig->column_family_set_), current_(nullptr) {} + + // sets current_ to ColumnFamilyData with column_family_id + // returns false if column family doesn't exist + // REQUIRES: use this function of DBImpl::column_family_memtables_ should be + // under a DB mutex OR from a write thread + bool Seek(uint32_t column_family_id) override; + + // Returns log number of the selected column family + // REQUIRES: under a DB mutex OR from a write thread + uint64_t GetLogNumber() const override; + + // REQUIRES: Seek() called first + // REQUIRES: use this function of DBImpl::column_family_memtables_ should be + // under a DB mutex OR from a write thread + virtual MemTable* GetMemTable() const override; + + // Returns column family handle for the selected column family + // REQUIRES: use this function of DBImpl::column_family_memtables_ should be + // under a DB mutex OR from a write thread + virtual ColumnFamilyHandle* GetColumnFamilyHandle() override; + + // Cannot be called while another thread is calling Seek(). + // REQUIRES: use this function of DBImpl::column_family_memtables_ should be + // under a DB mutex OR from a write thread + virtual ColumnFamilyData* current() override { return current_; } + + private: + ColumnFamilySet* column_family_set_; + ColumnFamilyData* current_; + ColumnFamilyHandleInternal handle_; +}; + +extern uint32_t GetColumnFamilyID(ColumnFamilyHandle* column_family); + +extern const Comparator* GetColumnFamilyUserComparator( + ColumnFamilyHandle* column_family); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/column_family_test.cc b/librocksdb-sys/rocksdb/db/column_family_test.cc new file mode 100644 index 0000000..c0574ee --- /dev/null +++ b/librocksdb-sys/rocksdb/db/column_family_test.cc @@ -0,0 +1,3593 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include +#include +#include + +#include "db/db_impl/db_impl.h" +#include "db/db_test_util.h" +#include "options/options_parser.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/comparator.h" +#include "rocksdb/convenience.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/iterator.h" +#include "rocksdb/utilities/object_registry.h" +#include "test_util/sync_point.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/coding.h" +#include "util/string_util.h" +#include "utilities/fault_injection_env.h" +#include "utilities/merge_operators.h" + +namespace ROCKSDB_NAMESPACE { + +static const int kValueSize = 1000; + +// counts how many operations were performed +class EnvCounter : public SpecialEnv { + public: + explicit EnvCounter(Env* base) + : SpecialEnv(base), num_new_writable_file_(0) {} + int GetNumberOfNewWritableFileCalls() { return num_new_writable_file_; } + Status NewWritableFile(const std::string& f, std::unique_ptr* r, + const EnvOptions& soptions) override { + ++num_new_writable_file_; + return EnvWrapper::NewWritableFile(f, r, soptions); + } + + private: + std::atomic num_new_writable_file_; +}; + +class ColumnFamilyTestBase : public testing::Test { + public: + explicit ColumnFamilyTestBase(uint32_t format) : rnd_(139), format_(format) { + Env* base_env = Env::Default(); + EXPECT_OK( + test::CreateEnvFromSystem(ConfigOptions(), &base_env, &env_guard_)); + EXPECT_NE(nullptr, base_env); + env_ = new EnvCounter(base_env); + env_->skip_fsync_ = true; + dbname_ = test::PerThreadDBPath("column_family_test"); + db_options_.create_if_missing = true; + db_options_.fail_if_options_file_error = true; + db_options_.env = env_; + } + + void SetUp() override { + EXPECT_OK(DestroyDB(dbname_, Options(db_options_, column_family_options_))); + } + + ~ColumnFamilyTestBase() override { + std::vector column_families; + for (auto h : handles_) { + ColumnFamilyDescriptor cfdescriptor; + Status s = h->GetDescriptor(&cfdescriptor); + EXPECT_OK(s); + column_families.push_back(cfdescriptor); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + Destroy(column_families); + delete env_; + } + + BlockBasedTableOptions GetBlockBasedTableOptions() { + BlockBasedTableOptions options; + options.format_version = format_; + return options; + } + + // Return the value to associate with the specified key + Slice Value(int k, std::string* storage) { + if (k == 0) { + // Ugh. Random seed of 0 used to produce no entropy. This code + // preserves the implementation that was in place when all of the + // magic values in this file were picked. + *storage = std::string(kValueSize, ' '); + } else { + Random r(k); + *storage = r.RandomString(kValueSize); + } + return Slice(*storage); + } + + void Build(int base, int n, int flush_every = 0) { + std::string key_space, value_space; + WriteBatch batch; + + for (int i = 0; i < n; i++) { + if (flush_every != 0 && i != 0 && i % flush_every == 0) { + DBImpl* dbi = static_cast_with_check(db_); + dbi->TEST_FlushMemTable(); + } + + int keyi = base + i; + Slice key(DBTestBase::Key(keyi)); + + batch.Clear(); + batch.Put(handles_[0], key, Value(keyi, &value_space)); + batch.Put(handles_[1], key, Value(keyi, &value_space)); + batch.Put(handles_[2], key, Value(keyi, &value_space)); + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + } + } + + void CheckMissed() { + uint64_t next_expected = 0; + uint64_t missed = 0; + int bad_keys = 0; + int bad_values = 0; + int correct = 0; + std::string value_space; + for (int cf = 0; cf < 3; cf++) { + next_expected = 0; + Iterator* iter = db_->NewIterator(ReadOptions(false, true), handles_[cf]); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + uint64_t key; + Slice in(iter->key()); + in.remove_prefix(3); + if (!ConsumeDecimalNumber(&in, &key) || !in.empty() || + key < next_expected) { + bad_keys++; + continue; + } + missed += (key - next_expected); + next_expected = key + 1; + if (iter->value() != Value(static_cast(key), &value_space)) { + bad_values++; + } else { + correct++; + } + } + delete iter; + } + + ASSERT_EQ(0, bad_keys); + ASSERT_EQ(0, bad_values); + ASSERT_EQ(0, missed); + (void)correct; + } + + void Close() { + for (auto h : handles_) { + if (h) { + ASSERT_OK(db_->DestroyColumnFamilyHandle(h)); + } + } + handles_.clear(); + names_.clear(); + delete db_; + db_ = nullptr; + } + + Status TryOpen(std::vector cf, + std::vector options = {}) { + std::vector column_families; + names_.clear(); + for (size_t i = 0; i < cf.size(); ++i) { + column_families.emplace_back( + cf[i], options.size() == 0 ? column_family_options_ : options[i]); + names_.push_back(cf[i]); + } + return DB::Open(db_options_, dbname_, column_families, &handles_, &db_); + } + + Status OpenReadOnly(std::vector cf, + std::vector options = {}) { + std::vector column_families; + names_.clear(); + for (size_t i = 0; i < cf.size(); ++i) { + column_families.emplace_back( + cf[i], options.size() == 0 ? column_family_options_ : options[i]); + names_.push_back(cf[i]); + } + return DB::OpenForReadOnly(db_options_, dbname_, column_families, &handles_, + &db_); + } + + void AssertOpenReadOnly(std::vector cf, + std::vector options = {}) { + ASSERT_OK(OpenReadOnly(cf, options)); + } + + void Open(std::vector cf, + std::vector options = {}) { + ASSERT_OK(TryOpen(cf, options)); + } + + void Open() { Open({"default"}); } + + DBImpl* dbfull() { return static_cast_with_check(db_); } + + int GetProperty(int cf, std::string property) { + std::string value; + EXPECT_TRUE(dbfull()->GetProperty(handles_[cf], property, &value)); +#ifndef CYGWIN + return std::stoi(value); +#else + return std::strtol(value.c_str(), 0 /* off */, 10 /* base */); +#endif + } + + bool IsDbWriteStopped() { + uint64_t v; + EXPECT_TRUE(dbfull()->GetIntProperty("rocksdb.is-write-stopped", &v)); + return (v == 1); + } + + uint64_t GetDbDelayedWriteRate() { + uint64_t v; + EXPECT_TRUE( + dbfull()->GetIntProperty("rocksdb.actual-delayed-write-rate", &v)); + return v; + } + + void Destroy(const std::vector& column_families = + std::vector()) { + Close(); + ASSERT_OK(DestroyDB(dbname_, Options(db_options_, column_family_options_), + column_families)); + } + + void CreateColumnFamilies( + const std::vector& cfs, + const std::vector options = {}) { + int cfi = static_cast(handles_.size()); + handles_.resize(cfi + cfs.size()); + names_.resize(cfi + cfs.size()); + for (size_t i = 0; i < cfs.size(); ++i) { + const auto& current_cf_opt = + options.size() == 0 ? column_family_options_ : options[i]; + ASSERT_OK( + db_->CreateColumnFamily(current_cf_opt, cfs[i], &handles_[cfi])); + names_[cfi] = cfs[i]; + + // Verify the CF options of the returned CF handle. + ColumnFamilyDescriptor desc; + ASSERT_OK(handles_[cfi]->GetDescriptor(&desc)); + // Need to sanitize the default column family options before comparing + // them. + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( + ConfigOptions(), desc.options, + SanitizeOptions(dbfull()->immutable_db_options(), current_cf_opt))); + cfi++; + } + } + + void Reopen(const std::vector options = {}) { + std::vector names; + for (auto name : names_) { + if (name != "") { + names.push_back(name); + } + } + Close(); + assert(options.size() == 0 || names.size() == options.size()); + Open(names, options); + } + + void CreateColumnFamiliesAndReopen(const std::vector& cfs) { + CreateColumnFamilies(cfs); + Reopen(); + } + + void DropColumnFamilies(const std::vector& cfs) { + for (auto cf : cfs) { + ASSERT_OK(db_->DropColumnFamily(handles_[cf])); + ASSERT_OK(db_->DestroyColumnFamilyHandle(handles_[cf])); + handles_[cf] = nullptr; + names_[cf] = ""; + } + } + + void PutRandomData(int cf, int num, int key_value_size, bool save = false) { + if (cf >= static_cast(keys_.size())) { + keys_.resize(cf + 1); + } + for (int i = 0; i < num; ++i) { + // 10 bytes for key, rest is value + if (!save) { + ASSERT_OK(Put(cf, test::RandomKey(&rnd_, 11), + rnd_.RandomString(key_value_size - 10))); + } else { + std::string key = test::RandomKey(&rnd_, 11); + keys_[cf].insert(key); + ASSERT_OK(Put(cf, key, rnd_.RandomString(key_value_size - 10))); + } + } + ASSERT_OK(db_->FlushWAL(/*sync=*/false)); + } + + void WaitForFlush(int cf) { + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[cf])); + } + + void WaitForCompaction() { ASSERT_OK(dbfull()->TEST_WaitForCompact()); } + + uint64_t MaxTotalInMemoryState() { + return dbfull()->TEST_MaxTotalInMemoryState(); + } + + void AssertMaxTotalInMemoryState(uint64_t value) { + ASSERT_EQ(value, MaxTotalInMemoryState()); + } + + Status Put(int cf, const std::string& key, const std::string& value) { + return db_->Put(WriteOptions(), handles_[cf], Slice(key), Slice(value)); + } + Status Merge(int cf, const std::string& key, const std::string& value) { + return db_->Merge(WriteOptions(), handles_[cf], Slice(key), Slice(value)); + } + Status Flush(int cf) { return db_->Flush(FlushOptions(), handles_[cf]); } + + std::string Get(int cf, const std::string& key) { + ReadOptions options; + options.verify_checksums = true; + std::string result; + Status s = db_->Get(options, handles_[cf], Slice(key), &result); + if (s.IsNotFound()) { + result = "NOT_FOUND"; + } else if (!s.ok()) { + result = s.ToString(); + } + return result; + } + + void CompactAll(int cf) { + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), handles_[cf], nullptr, + nullptr)); + } + + void Compact(int cf, const Slice& start, const Slice& limit) { + ASSERT_OK( + db_->CompactRange(CompactRangeOptions(), handles_[cf], &start, &limit)); + } + + int NumTableFilesAtLevel(int level, int cf) { + return GetProperty(cf, + "rocksdb.num-files-at-level" + std::to_string(level)); + } + + // Return spread of files per level + std::string FilesPerLevel(int cf) { + std::string result; + int last_non_zero_offset = 0; + for (int level = 0; level < dbfull()->NumberLevels(handles_[cf]); level++) { + int f = NumTableFilesAtLevel(level, cf); + char buf[100]; + snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f); + result += buf; + if (f > 0) { + last_non_zero_offset = static_cast(result.size()); + } + } + result.resize(last_non_zero_offset); + return result; + } + + void AssertFilesPerLevel(const std::string& value, int cf) { + ASSERT_EQ(value, FilesPerLevel(cf)); + } + + int CountLiveFiles() { + std::vector metadata; + db_->GetLiveFilesMetaData(&metadata); + return static_cast(metadata.size()); + } + + void AssertCountLiveFiles(int expected_value) { + ASSERT_EQ(expected_value, CountLiveFiles()); + } + + // Do n memtable flushes, each of which produces an sstable + // covering the range [small,large]. + void MakeTables(int cf, int n, const std::string& small, + const std::string& large) { + for (int i = 0; i < n; i++) { + ASSERT_OK(Put(cf, small, "begin")); + ASSERT_OK(Put(cf, large, "end")); + ASSERT_OK(db_->Flush(FlushOptions(), handles_[cf])); + } + } + + int CountLiveLogFiles() { + int micros_wait_for_log_deletion = 20000; + env_->SleepForMicroseconds(micros_wait_for_log_deletion); + int ret = 0; + VectorLogPtr wal_files; + Status s; + // GetSortedWalFiles is a flakey function -- it gets all the wal_dir + // children files and then later checks for their existence. if some of the + // log files doesn't exist anymore, it reports an error. it does all of this + // without DB mutex held, so if a background process deletes the log file + // while the function is being executed, it returns an error. We retry the + // function 10 times to avoid the error failing the test + for (int retries = 0; retries < 10; ++retries) { + wal_files.clear(); + s = db_->GetSortedWalFiles(wal_files); + if (s.ok()) { + break; + } + } + EXPECT_OK(s); + for (const auto& wal : wal_files) { + if (wal->Type() == kAliveLogFile) { + ++ret; + } + } + return ret; + return 0; + } + + void AssertCountLiveLogFiles(int value) { + ASSERT_EQ(value, CountLiveLogFiles()); + } + + void AssertNumberOfImmutableMemtables(std::vector num_per_cf) { + assert(num_per_cf.size() == handles_.size()); + + for (size_t i = 0; i < num_per_cf.size(); ++i) { + ASSERT_EQ(num_per_cf[i], GetProperty(static_cast(i), + "rocksdb.num-immutable-mem-table")); + } + } + + void CopyFile(const std::string& source, const std::string& destination, + uint64_t size = 0) { + const EnvOptions soptions; + std::unique_ptr srcfile; + ASSERT_OK(env_->NewSequentialFile(source, &srcfile, soptions)); + std::unique_ptr destfile; + ASSERT_OK(env_->NewWritableFile(destination, &destfile, soptions)); + + if (size == 0) { + // default argument means copy everything + ASSERT_OK(env_->GetFileSize(source, &size)); + } + + char buffer[4096]; + Slice slice; + while (size > 0) { + uint64_t one = std::min(uint64_t(sizeof(buffer)), size); + ASSERT_OK(srcfile->Read(one, &slice, buffer)); + ASSERT_OK(destfile->Append(slice)); + size -= slice.size(); + } + ASSERT_OK(destfile->Close()); + } + + int GetSstFileCount(std::string path) { + std::vector files; + DBTestBase::GetSstFiles(env_, path, &files); + return static_cast(files.size()); + } + + void RecalculateWriteStallConditions( + ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options) { + // add lock to avoid race condition between + // `RecalculateWriteStallConditions` which writes to CFStats and + // background `DBImpl::DumpStats()` threads which read CFStats + dbfull()->TEST_LockMutex(); + cfd->RecalculateWriteStallConditions(mutable_cf_options); + dbfull()->TEST_UnlockMutex(); + } + + std::vector handles_; + std::vector names_; + std::vector> keys_; + ColumnFamilyOptions column_family_options_; + DBOptions db_options_; + std::string dbname_; + DB* db_ = nullptr; + EnvCounter* env_; + std::shared_ptr env_guard_; + Random rnd_; + uint32_t format_; +}; + +class ColumnFamilyTest + : public ColumnFamilyTestBase, + virtual public ::testing::WithParamInterface { + public: + ColumnFamilyTest() : ColumnFamilyTestBase(GetParam()) {} +}; + +INSTANTIATE_TEST_CASE_P(FormatDef, ColumnFamilyTest, + testing::Values(test::kDefaultFormatVersion)); +INSTANTIATE_TEST_CASE_P(FormatLatest, ColumnFamilyTest, + testing::Values(kLatestFormatVersion)); + +TEST_P(ColumnFamilyTest, DontReuseColumnFamilyID) { + for (int iter = 0; iter < 3; ++iter) { + Open(); + CreateColumnFamilies({"one", "two", "three"}); + for (size_t i = 0; i < handles_.size(); ++i) { + auto cfh = static_cast_with_check(handles_[i]); + ASSERT_EQ(i, cfh->GetID()); + } + if (iter == 1) { + Reopen(); + } + DropColumnFamilies({3}); + Reopen(); + if (iter == 2) { + // this tests if max_column_family is correctly persisted with + // WriteSnapshot() + Reopen(); + } + CreateColumnFamilies({"three2"}); + // ID 3 that was used for dropped column family "three" should not be + // reused + auto cfh3 = static_cast_with_check(handles_[3]); + ASSERT_EQ(4U, cfh3->GetID()); + Close(); + Destroy(); + } +} + +TEST_P(ColumnFamilyTest, CreateCFRaceWithGetAggProperty) { + Open(); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::WriteOptionsFile:1", + "ColumnFamilyTest.CreateCFRaceWithGetAggProperty:1"}, + {"ColumnFamilyTest.CreateCFRaceWithGetAggProperty:2", + "DBImpl::WriteOptionsFile:2"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ROCKSDB_NAMESPACE::port::Thread thread( + [&] { CreateColumnFamilies({"one"}); }); + + TEST_SYNC_POINT("ColumnFamilyTest.CreateCFRaceWithGetAggProperty:1"); + uint64_t pv; + db_->GetAggregatedIntProperty(DB::Properties::kEstimateTableReadersMem, &pv); + TEST_SYNC_POINT("ColumnFamilyTest.CreateCFRaceWithGetAggProperty:2"); + + thread.join(); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +class FlushEmptyCFTestWithParam + : public ColumnFamilyTestBase, + virtual public testing::WithParamInterface> { + public: + FlushEmptyCFTestWithParam() + : ColumnFamilyTestBase(std::get<0>(GetParam())), + allow_2pc_(std::get<1>(GetParam())) {} + + // Required if inheriting from testing::WithParamInterface<> + static void SetUpTestCase() {} + static void TearDownTestCase() {} + + bool allow_2pc_; +}; + +TEST_P(FlushEmptyCFTestWithParam, FlushEmptyCFTest) { + std::unique_ptr fault_env( + new FaultInjectionTestEnv(env_)); + db_options_.env = fault_env.get(); + db_options_.allow_2pc = allow_2pc_; + Open(); + CreateColumnFamilies({"one", "two"}); + // Generate log file A. + ASSERT_OK(Put(1, "foo", "v1")); // seqID 1 + + Reopen(); + // Log file A is not dropped after reopening because default column family's + // min log number is 0. + // It flushes to SST file X + ASSERT_OK(Put(1, "foo", "v1")); // seqID 2 + ASSERT_OK(Put(1, "bar", "v2")); // seqID 3 + // Current log file is file B now. While flushing, a new log file C is created + // and is set to current. Boths' min log number is set to file C in memory, so + // after flushing file B is deleted. At the same time, the min log number of + // default CF is not written to manifest. Log file A still remains. + // Flushed to SST file Y. + ASSERT_OK(Flush(1)); + ASSERT_OK(Flush(0)); + ASSERT_OK(Put(1, "bar", "v3")); // seqID 4 + ASSERT_OK(Put(1, "foo", "v4")); // seqID 5 + ASSERT_OK(db_->FlushWAL(/*sync=*/false)); + + // Preserve file system state up to here to simulate a crash condition. + fault_env->SetFilesystemActive(false); + std::vector names; + for (auto name : names_) { + if (name != "") { + names.push_back(name); + } + } + + Close(); + fault_env->ResetState(); + + // Before opening, there are four files: + // Log file A contains seqID 1 + // Log file C contains seqID 4, 5 + // SST file X contains seqID 1 + // SST file Y contains seqID 2, 3 + // Min log number: + // default CF: 0 + // CF one, two: C + // When opening the DB, all the seqID should be preserved. + Open(names, {}); + ASSERT_EQ("v4", Get(1, "foo")); + ASSERT_EQ("v3", Get(1, "bar")); + Close(); + + db_options_.env = env_; +} + +TEST_P(FlushEmptyCFTestWithParam, FlushEmptyCFTest2) { + std::unique_ptr fault_env( + new FaultInjectionTestEnv(env_)); + db_options_.env = fault_env.get(); + db_options_.allow_2pc = allow_2pc_; + Open(); + CreateColumnFamilies({"one", "two"}); + // Generate log file A. + ASSERT_OK(Put(1, "foo", "v1")); // seqID 1 + + Reopen(); + // Log file A is not dropped after reopening because default column family's + // min log number is 0. + // It flushes to SST file X + ASSERT_OK(Put(1, "foo", "v1")); // seqID 2 + ASSERT_OK(Put(1, "bar", "v2")); // seqID 3 + // Current log file is file B now. While flushing, a new log file C is created + // and is set to current. Both CFs' min log number is set to file C so after + // flushing file B is deleted. Log file A still remains. + // Flushed to SST file Y. + ASSERT_OK(Flush(1)); + ASSERT_OK(Put(0, "bar", "v2")); // seqID 4 + ASSERT_OK(Put(2, "bar", "v2")); // seqID 5 + ASSERT_OK(Put(1, "bar", "v3")); // seqID 6 + // Flushing all column families. This forces all CFs' min log to current. This + // is written to the manifest file. Log file C is cleared. + ASSERT_OK(Flush(0)); + ASSERT_OK(Flush(1)); + ASSERT_OK(Flush(2)); + // Write to log file D + ASSERT_OK(Put(1, "bar", "v4")); // seqID 7 + ASSERT_OK(Put(1, "bar", "v5")); // seqID 8 + ASSERT_OK(db_->FlushWAL(/*sync=*/false)); + // Preserve file system state up to here to simulate a crash condition. + fault_env->SetFilesystemActive(false); + std::vector names; + for (auto name : names_) { + if (name != "") { + names.push_back(name); + } + } + + Close(); + fault_env->ResetState(); + // Before opening, there are two logfiles: + // Log file A contains seqID 1 + // Log file D contains seqID 7, 8 + // Min log number: + // default CF: D + // CF one, two: D + // When opening the DB, log file D should be replayed using the seqID + // specified in the file. + Open(names, {}); + ASSERT_EQ("v1", Get(1, "foo")); + ASSERT_EQ("v5", Get(1, "bar")); + Close(); + + db_options_.env = env_; +} + +INSTANTIATE_TEST_CASE_P( + FormatDef, FlushEmptyCFTestWithParam, + testing::Values(std::make_tuple(test::kDefaultFormatVersion, true), + std::make_tuple(test::kDefaultFormatVersion, false))); +INSTANTIATE_TEST_CASE_P( + FormatLatest, FlushEmptyCFTestWithParam, + testing::Values(std::make_tuple(kLatestFormatVersion, true), + std::make_tuple(kLatestFormatVersion, false))); + +TEST_P(ColumnFamilyTest, AddDrop) { + Open(); + CreateColumnFamilies({"one", "two", "three"}); + ASSERT_EQ("NOT_FOUND", Get(1, "fodor")); + ASSERT_EQ("NOT_FOUND", Get(2, "fodor")); + DropColumnFamilies({2}); + ASSERT_EQ("NOT_FOUND", Get(1, "fodor")); + CreateColumnFamilies({"four"}); + ASSERT_EQ("NOT_FOUND", Get(3, "fodor")); + ASSERT_OK(Put(1, "fodor", "mirko")); + ASSERT_EQ("mirko", Get(1, "fodor")); + ASSERT_EQ("NOT_FOUND", Get(3, "fodor")); + Close(); + ASSERT_TRUE(TryOpen({"default"}).IsInvalidArgument()); + Open({"default", "one", "three", "four"}); + DropColumnFamilies({1}); + Reopen(); + Close(); + + std::vector families; + ASSERT_OK(DB::ListColumnFamilies(db_options_, dbname_, &families)); + std::sort(families.begin(), families.end()); + ASSERT_TRUE(families == + std::vector({"default", "four", "three"})); +} + +TEST_P(ColumnFamilyTest, BulkAddDrop) { + constexpr int kNumCF = 1000; + ColumnFamilyOptions cf_options; + WriteOptions write_options; + Open(); + std::vector cf_names; + std::vector cf_handles; + for (int i = 1; i <= kNumCF; i++) { + cf_names.push_back("cf1-" + std::to_string(i)); + } + ASSERT_OK(db_->CreateColumnFamilies(cf_options, cf_names, &cf_handles)); + for (int i = 1; i <= kNumCF; i++) { + ASSERT_OK(db_->Put(write_options, cf_handles[i - 1], "foo", "bar")); + } + ASSERT_OK(db_->DropColumnFamilies(cf_handles)); + std::vector cf_descriptors; + for (auto* handle : cf_handles) { + delete handle; + } + cf_handles.clear(); + for (int i = 1; i <= kNumCF; i++) { + cf_descriptors.emplace_back("cf2-" + std::to_string(i), + ColumnFamilyOptions()); + } + ASSERT_OK(db_->CreateColumnFamilies(cf_descriptors, &cf_handles)); + for (int i = 1; i <= kNumCF; i++) { + ASSERT_OK(db_->Put(write_options, cf_handles[i - 1], "foo", "bar")); + } + ASSERT_OK(db_->DropColumnFamilies(cf_handles)); + for (auto* handle : cf_handles) { + delete handle; + } + Close(); + std::vector families; + ASSERT_OK(DB::ListColumnFamilies(db_options_, dbname_, &families)); + std::sort(families.begin(), families.end()); + ASSERT_TRUE(families == std::vector({"default"})); +} + +TEST_P(ColumnFamilyTest, DropTest) { + // first iteration - don't reopen DB before dropping + // second iteration - reopen DB before dropping + for (int iter = 0; iter < 2; ++iter) { + Open({"default"}); + CreateColumnFamiliesAndReopen({"pikachu"}); + for (int i = 0; i < 100; ++i) { + ASSERT_OK(Put(1, std::to_string(i), "bar" + std::to_string(i))); + } + ASSERT_OK(Flush(1)); + + if (iter == 1) { + Reopen(); + } + ASSERT_EQ("bar1", Get(1, "1")); + + AssertCountLiveFiles(1); + DropColumnFamilies({1}); + // make sure that all files are deleted when we drop the column family + AssertCountLiveFiles(0); + Destroy(); + } +} + +TEST_P(ColumnFamilyTest, WriteBatchFailure) { + Open(); + CreateColumnFamiliesAndReopen({"one", "two"}); + WriteBatch batch; + ASSERT_OK(batch.Put(handles_[0], Slice("existing"), Slice("column-family"))); + ASSERT_OK( + batch.Put(handles_[1], Slice("non-existing"), Slice("column-family"))); + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + DropColumnFamilies({1}); + WriteOptions woptions_ignore_missing_cf; + woptions_ignore_missing_cf.ignore_missing_column_families = true; + ASSERT_OK( + batch.Put(handles_[0], Slice("still here"), Slice("column-family"))); + ASSERT_OK(db_->Write(woptions_ignore_missing_cf, &batch)); + ASSERT_EQ("column-family", Get(0, "still here")); + Status s = db_->Write(WriteOptions(), &batch); + ASSERT_TRUE(s.IsInvalidArgument()); + Close(); +} + +TEST_P(ColumnFamilyTest, ReadWrite) { + Open(); + CreateColumnFamiliesAndReopen({"one", "two"}); + ASSERT_OK(Put(0, "foo", "v1")); + ASSERT_OK(Put(0, "bar", "v2")); + ASSERT_OK(Put(1, "mirko", "v3")); + ASSERT_OK(Put(0, "foo", "v2")); + ASSERT_OK(Put(2, "fodor", "v5")); + + for (int iter = 0; iter <= 3; ++iter) { + ASSERT_EQ("v2", Get(0, "foo")); + ASSERT_EQ("v2", Get(0, "bar")); + ASSERT_EQ("v3", Get(1, "mirko")); + ASSERT_EQ("v5", Get(2, "fodor")); + ASSERT_EQ("NOT_FOUND", Get(0, "fodor")); + ASSERT_EQ("NOT_FOUND", Get(1, "fodor")); + ASSERT_EQ("NOT_FOUND", Get(2, "foo")); + if (iter <= 1) { + Reopen(); + } + } + Close(); +} + +TEST_P(ColumnFamilyTest, IgnoreRecoveredLog) { + std::string backup_logs = dbname_ + "/backup_logs"; + + // delete old files in backup_logs directory + ASSERT_OK(env_->CreateDirIfMissing(dbname_)); + ASSERT_OK(env_->CreateDirIfMissing(backup_logs)); + std::vector old_files; + ASSERT_OK(env_->GetChildren(backup_logs, &old_files)); + for (auto& file : old_files) { + ASSERT_OK(env_->DeleteFile(backup_logs + "/" + file)); + } + + column_family_options_.merge_operator = + MergeOperators::CreateUInt64AddOperator(); + db_options_.wal_dir = dbname_ + "/logs"; + Destroy(); + Open(); + CreateColumnFamilies({"cf1", "cf2"}); + + // fill up the DB + std::string one, two, three; + PutFixed64(&one, 1); + PutFixed64(&two, 2); + PutFixed64(&three, 3); + ASSERT_OK(Merge(0, "foo", one)); + ASSERT_OK(Merge(1, "mirko", one)); + ASSERT_OK(Merge(0, "foo", one)); + ASSERT_OK(Merge(2, "bla", one)); + ASSERT_OK(Merge(2, "fodor", one)); + ASSERT_OK(Merge(0, "bar", one)); + ASSERT_OK(Merge(2, "bla", one)); + ASSERT_OK(Merge(1, "mirko", two)); + ASSERT_OK(Merge(1, "franjo", one)); + + // copy the logs to backup + std::vector logs; + ASSERT_OK(env_->GetChildren(db_options_.wal_dir, &logs)); + for (auto& log : logs) { + CopyFile(db_options_.wal_dir + "/" + log, backup_logs + "/" + log); + } + + // recover the DB + Close(); + + // 1. check consistency + // 2. copy the logs from backup back to WAL dir. if the recovery happens + // again on the same log files, this should lead to incorrect results + // due to applying merge operator twice + // 3. check consistency + for (int iter = 0; iter < 2; ++iter) { + // assert consistency + Open({"default", "cf1", "cf2"}); + ASSERT_EQ(two, Get(0, "foo")); + ASSERT_EQ(one, Get(0, "bar")); + ASSERT_EQ(three, Get(1, "mirko")); + ASSERT_EQ(one, Get(1, "franjo")); + ASSERT_EQ(one, Get(2, "fodor")); + ASSERT_EQ(two, Get(2, "bla")); + Close(); + + if (iter == 0) { + // copy the logs from backup back to wal dir + for (auto& log : logs) { + CopyFile(backup_logs + "/" + log, db_options_.wal_dir + "/" + log); + } + } + } +} + +TEST_P(ColumnFamilyTest, FlushTest) { + Open(); + CreateColumnFamiliesAndReopen({"one", "two"}); + ASSERT_OK(Put(0, "foo", "v1")); + ASSERT_OK(Put(0, "bar", "v2")); + ASSERT_OK(Put(1, "mirko", "v3")); + ASSERT_OK(Put(0, "foo", "v2")); + ASSERT_OK(Put(2, "fodor", "v5")); + + for (int j = 0; j < 2; j++) { + ReadOptions ro; + std::vector iterators; + // Hold super version. + if (j == 0) { + ASSERT_OK(db_->NewIterators(ro, handles_, &iterators)); + } + + for (int i = 0; i < 3; ++i) { + uint64_t max_total_in_memory_state = MaxTotalInMemoryState(); + ASSERT_OK(Flush(i)); + AssertMaxTotalInMemoryState(max_total_in_memory_state); + } + ASSERT_OK(Put(1, "foofoo", "bar")); + ASSERT_OK(Put(0, "foofoo", "bar")); + + for (auto* it : iterators) { + ASSERT_OK(it->status()); + delete it; + } + } + Reopen(); + + for (int iter = 0; iter <= 2; ++iter) { + ASSERT_EQ("v2", Get(0, "foo")); + ASSERT_EQ("v2", Get(0, "bar")); + ASSERT_EQ("v3", Get(1, "mirko")); + ASSERT_EQ("v5", Get(2, "fodor")); + ASSERT_EQ("NOT_FOUND", Get(0, "fodor")); + ASSERT_EQ("NOT_FOUND", Get(1, "fodor")); + ASSERT_EQ("NOT_FOUND", Get(2, "foo")); + if (iter <= 1) { + Reopen(); + } + } + Close(); +} + +// Makes sure that obsolete log files get deleted +TEST_P(ColumnFamilyTest, LogDeletionTest) { + db_options_.max_total_wal_size = std::numeric_limits::max(); + column_family_options_.arena_block_size = 4 * 1024; + column_family_options_.write_buffer_size = 128000; // 128KB + Open(); + CreateColumnFamilies({"one", "two", "three", "four"}); + // Each bracket is one log file. if number is in (), it means + // we don't need it anymore (it's been flushed) + // [] + AssertCountLiveLogFiles(0); + PutRandomData(0, 1, 128); + // [0] + PutRandomData(1, 1, 128); + // [0, 1] + PutRandomData(1, 1000, 128); + WaitForFlush(1); + // [0, (1)] [1] + AssertCountLiveLogFiles(2); + PutRandomData(0, 1, 128); + // [0, (1)] [0, 1] + AssertCountLiveLogFiles(2); + PutRandomData(2, 1, 128); + // [0, (1)] [0, 1, 2] + PutRandomData(2, 1000, 128); + WaitForFlush(2); + // [0, (1)] [0, 1, (2)] [2] + AssertCountLiveLogFiles(3); + PutRandomData(2, 1000, 128); + WaitForFlush(2); + // [0, (1)] [0, 1, (2)] [(2)] [2] + AssertCountLiveLogFiles(4); + PutRandomData(3, 1, 128); + // [0, (1)] [0, 1, (2)] [(2)] [2, 3] + PutRandomData(1, 1, 128); + // [0, (1)] [0, 1, (2)] [(2)] [1, 2, 3] + AssertCountLiveLogFiles(4); + PutRandomData(1, 1000, 128); + WaitForFlush(1); + // [0, (1)] [0, (1), (2)] [(2)] [(1), 2, 3] [1] + AssertCountLiveLogFiles(5); + PutRandomData(0, 1000, 128); + WaitForFlush(0); + // [(0), (1)] [(0), (1), (2)] [(2)] [(1), 2, 3] [1, (0)] [0] + // delete obsolete logs --> + // [(1), 2, 3] [1, (0)] [0] + AssertCountLiveLogFiles(3); + PutRandomData(0, 1000, 128); + WaitForFlush(0); + // [(1), 2, 3] [1, (0)], [(0)] [0] + AssertCountLiveLogFiles(4); + PutRandomData(1, 1000, 128); + WaitForFlush(1); + // [(1), 2, 3] [(1), (0)] [(0)] [0, (1)] [1] + AssertCountLiveLogFiles(5); + PutRandomData(2, 1000, 128); + WaitForFlush(2); + // [(1), (2), 3] [(1), (0)] [(0)] [0, (1)] [1, (2)], [2] + AssertCountLiveLogFiles(6); + PutRandomData(3, 1000, 128); + WaitForFlush(3); + // [(1), (2), (3)] [(1), (0)] [(0)] [0, (1)] [1, (2)], [2, (3)] [3] + // delete obsolete logs --> + // [0, (1)] [1, (2)], [2, (3)] [3] + AssertCountLiveLogFiles(4); + Close(); +} + +TEST_P(ColumnFamilyTest, CrashAfterFlush) { + std::unique_ptr fault_env( + new FaultInjectionTestEnv(env_)); + db_options_.env = fault_env.get(); + Open(); + CreateColumnFamilies({"one"}); + + WriteBatch batch; + ASSERT_OK(batch.Put(handles_[0], Slice("foo"), Slice("bar"))); + ASSERT_OK(batch.Put(handles_[1], Slice("foo"), Slice("bar"))); + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + ASSERT_OK(Flush(0)); + fault_env->SetFilesystemActive(false); + + std::vector names; + for (auto name : names_) { + if (name != "") { + names.push_back(name); + } + } + Close(); + ASSERT_OK(fault_env->DropUnsyncedFileData()); + fault_env->ResetState(); + Open(names, {}); + + // Write batch should be atomic. + ASSERT_EQ(Get(0, "foo"), Get(1, "foo")); + + Close(); + db_options_.env = env_; +} + +TEST_P(ColumnFamilyTest, OpenNonexistentColumnFamily) { + ASSERT_OK(TryOpen({"default"})); + Close(); + ASSERT_TRUE(TryOpen({"default", "dne"}).IsInvalidArgument()); +} + +// Makes sure that obsolete log files get deleted +TEST_P(ColumnFamilyTest, DifferentWriteBufferSizes) { + // disable flushing stale column families + db_options_.max_total_wal_size = std::numeric_limits::max(); + Open(); + CreateColumnFamilies({"one", "two", "three"}); + ColumnFamilyOptions default_cf, one, two, three; + // setup options. all column families have max_write_buffer_number setup to 10 + // "default" -> 100KB memtable, start flushing immediately + // "one" -> 200KB memtable, start flushing with two immutable memtables + // "two" -> 1MB memtable, start flushing with three immutable memtables + // "three" -> 90KB memtable, start flushing with four immutable memtables + default_cf.write_buffer_size = 100000; + default_cf.arena_block_size = 4 * 4096; + default_cf.max_write_buffer_number = 10; + default_cf.min_write_buffer_number_to_merge = 1; + default_cf.max_write_buffer_size_to_maintain = 0; + one.write_buffer_size = 200000; + one.arena_block_size = 4 * 4096; + one.max_write_buffer_number = 10; + one.min_write_buffer_number_to_merge = 2; + one.max_write_buffer_size_to_maintain = + static_cast(one.write_buffer_size); + two.write_buffer_size = 1000000; + two.arena_block_size = 4 * 4096; + two.max_write_buffer_number = 10; + two.min_write_buffer_number_to_merge = 3; + two.max_write_buffer_size_to_maintain = + static_cast(two.write_buffer_size); + three.write_buffer_size = 4096 * 22; + three.arena_block_size = 4096; + three.max_write_buffer_number = 10; + three.min_write_buffer_number_to_merge = 4; + three.max_write_buffer_size_to_maintain = + static_cast(three.write_buffer_size); + + Reopen({default_cf, one, two, three}); + + int micros_wait_for_flush = 10000; + PutRandomData(0, 100, 1000); + WaitForFlush(0); + AssertNumberOfImmutableMemtables({0, 0, 0, 0}); + AssertCountLiveLogFiles(1); + PutRandomData(1, 200, 1000); + env_->SleepForMicroseconds(micros_wait_for_flush); + AssertNumberOfImmutableMemtables({0, 1, 0, 0}); + AssertCountLiveLogFiles(2); + PutRandomData(2, 1000, 1000); + env_->SleepForMicroseconds(micros_wait_for_flush); + AssertNumberOfImmutableMemtables({0, 1, 1, 0}); + AssertCountLiveLogFiles(3); + PutRandomData(2, 1000, 1000); + env_->SleepForMicroseconds(micros_wait_for_flush); + AssertNumberOfImmutableMemtables({0, 1, 2, 0}); + AssertCountLiveLogFiles(4); + PutRandomData(3, 93, 990); + env_->SleepForMicroseconds(micros_wait_for_flush); + AssertNumberOfImmutableMemtables({0, 1, 2, 1}); + AssertCountLiveLogFiles(5); + PutRandomData(3, 88, 990); + env_->SleepForMicroseconds(micros_wait_for_flush); + AssertNumberOfImmutableMemtables({0, 1, 2, 2}); + AssertCountLiveLogFiles(6); + PutRandomData(3, 88, 990); + env_->SleepForMicroseconds(micros_wait_for_flush); + AssertNumberOfImmutableMemtables({0, 1, 2, 3}); + AssertCountLiveLogFiles(7); + PutRandomData(0, 100, 1000); + WaitForFlush(0); + AssertNumberOfImmutableMemtables({0, 1, 2, 3}); + AssertCountLiveLogFiles(8); + PutRandomData(2, 100, 10000); + WaitForFlush(2); + AssertNumberOfImmutableMemtables({0, 1, 0, 3}); + AssertCountLiveLogFiles(9); + PutRandomData(3, 88, 990); + WaitForFlush(3); + AssertNumberOfImmutableMemtables({0, 1, 0, 0}); + AssertCountLiveLogFiles(10); + PutRandomData(3, 88, 990); + env_->SleepForMicroseconds(micros_wait_for_flush); + AssertNumberOfImmutableMemtables({0, 1, 0, 1}); + AssertCountLiveLogFiles(11); + PutRandomData(1, 200, 1000); + WaitForFlush(1); + AssertNumberOfImmutableMemtables({0, 0, 0, 1}); + AssertCountLiveLogFiles(5); + PutRandomData(3, 88 * 3, 990); + WaitForFlush(3); + PutRandomData(3, 88 * 4, 990); + WaitForFlush(3); + AssertNumberOfImmutableMemtables({0, 0, 0, 0}); + AssertCountLiveLogFiles(12); + PutRandomData(0, 100, 1000); + WaitForFlush(0); + AssertNumberOfImmutableMemtables({0, 0, 0, 0}); + AssertCountLiveLogFiles(12); + PutRandomData(2, 3 * 1000, 1000); + WaitForFlush(2); + AssertNumberOfImmutableMemtables({0, 0, 0, 0}); + AssertCountLiveLogFiles(12); + PutRandomData(1, 2 * 200, 1000); + WaitForFlush(1); + AssertNumberOfImmutableMemtables({0, 0, 0, 0}); + AssertCountLiveLogFiles(7); + Close(); +} + +// The test is commented out because we want to test that snapshot is +// not created for memtables not supported it, but There isn't a memtable +// that doesn't support snapshot right now. If we have one later, we can +// re-enable the test. +// +// TEST_P(ColumnFamilyTest, MemtableNotSupportSnapshot) { +// db_options_.allow_concurrent_memtable_write = false; +// Open(); +// auto* s1 = dbfull()->GetSnapshot(); +// ASSERT_TRUE(s1 != nullptr); +// dbfull()->ReleaseSnapshot(s1); + +// // Add a column family that doesn't support snapshot +// ColumnFamilyOptions first; +// first.memtable_factory.reset(new DummyMemtableNotSupportingSnapshot()); +// CreateColumnFamilies({"first"}, {first}); +// auto* s2 = dbfull()->GetSnapshot(); +// ASSERT_TRUE(s2 == nullptr); + +// // Add a column family that supports snapshot. Snapshot stays not +// supported. ColumnFamilyOptions second; CreateColumnFamilies({"second"}, +// {second}); auto* s3 = dbfull()->GetSnapshot(); ASSERT_TRUE(s3 == nullptr); +// Close(); +// } + +class TestComparator : public Comparator { + int Compare(const ROCKSDB_NAMESPACE::Slice& /*a*/, + const ROCKSDB_NAMESPACE::Slice& /*b*/) const override { + return 0; + } + const char* Name() const override { return "Test"; } + void FindShortestSeparator( + std::string* /*start*/, + const ROCKSDB_NAMESPACE::Slice& /*limit*/) const override {} + void FindShortSuccessor(std::string* /*key*/) const override {} +}; + +static TestComparator third_comparator; +static TestComparator fourth_comparator; + +// Test that we can retrieve the comparator from a created CF +TEST_P(ColumnFamilyTest, GetComparator) { + Open(); + // Add a column family with no comparator specified + CreateColumnFamilies({"first"}); + const Comparator* comp = handles_[0]->GetComparator(); + ASSERT_EQ(comp, BytewiseComparator()); + + // Add three column families - one with no comparator and two + // with comparators specified + ColumnFamilyOptions second, third, fourth; + second.comparator = &third_comparator; + third.comparator = &fourth_comparator; + CreateColumnFamilies({"second", "third", "fourth"}, {second, third, fourth}); + ASSERT_EQ(handles_[1]->GetComparator(), BytewiseComparator()); + ASSERT_EQ(handles_[2]->GetComparator(), &third_comparator); + ASSERT_EQ(handles_[3]->GetComparator(), &fourth_comparator); + Close(); +} + +TEST_P(ColumnFamilyTest, DifferentMergeOperators) { + Open(); + CreateColumnFamilies({"first", "second"}); + ColumnFamilyOptions default_cf, first, second; + first.merge_operator = MergeOperators::CreateUInt64AddOperator(); + second.merge_operator = MergeOperators::CreateStringAppendOperator(); + Reopen({default_cf, first, second}); + + std::string one, two, three; + PutFixed64(&one, 1); + PutFixed64(&two, 2); + PutFixed64(&three, 3); + + ASSERT_OK(Put(0, "foo", two)); + ASSERT_OK(Put(0, "foo", one)); + ASSERT_TRUE(Merge(0, "foo", two).IsNotSupported()); + ASSERT_EQ(Get(0, "foo"), one); + + ASSERT_OK(Put(1, "foo", two)); + ASSERT_OK(Put(1, "foo", one)); + ASSERT_OK(Merge(1, "foo", two)); + ASSERT_EQ(Get(1, "foo"), three); + + ASSERT_OK(Put(2, "foo", two)); + ASSERT_OK(Put(2, "foo", one)); + ASSERT_OK(Merge(2, "foo", two)); + ASSERT_EQ(Get(2, "foo"), one + "," + two); + Close(); +} + +TEST_P(ColumnFamilyTest, DifferentCompactionStyles) { + Open(); + CreateColumnFamilies({"one", "two"}); + ColumnFamilyOptions default_cf, one, two; + db_options_.max_open_files = 20; // only 10 files in file cache + + default_cf.level_compaction_dynamic_level_bytes = false; + default_cf.compaction_style = kCompactionStyleLevel; + default_cf.num_levels = 3; + default_cf.write_buffer_size = 64 << 10; // 64KB + default_cf.target_file_size_base = 30 << 10; + default_cf.max_compaction_bytes = static_cast(1) << 60; + + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.no_block_cache = true; + default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + one.compaction_style = kCompactionStyleUniversal; + + one.num_levels = 1; + // trigger compaction if there are >= 4 files + one.level0_file_num_compaction_trigger = 4; + one.write_buffer_size = 120000; + + two.level_compaction_dynamic_level_bytes = false; + two.compaction_style = kCompactionStyleLevel; + two.num_levels = 4; + two.level0_file_num_compaction_trigger = 3; + two.write_buffer_size = 100000; + + Reopen({default_cf, one, two}); + + // SETUP column family "one" -- universal style + for (int i = 0; i < one.level0_file_num_compaction_trigger - 1; ++i) { + PutRandomData(1, 10, 12000); + PutRandomData(1, 1, 10); + WaitForFlush(1); + AssertFilesPerLevel(std::to_string(i + 1), 1); + } + + // SETUP column family "two" -- level style with 4 levels + for (int i = 0; i < two.level0_file_num_compaction_trigger - 1; ++i) { + PutRandomData(2, 10, 12000); + PutRandomData(2, 1, 10); + WaitForFlush(2); + AssertFilesPerLevel(std::to_string(i + 1), 2); + } + + // TRIGGER compaction "one" + PutRandomData(1, 10, 12000); + PutRandomData(1, 1, 10); + + // TRIGGER compaction "two" + PutRandomData(2, 10, 12000); + PutRandomData(2, 1, 10); + + // WAIT for compactions + WaitForCompaction(); + + // VERIFY compaction "one" + AssertFilesPerLevel("1", 1); + + // VERIFY compaction "two" + AssertFilesPerLevel("0,1", 2); + CompactAll(2); + AssertFilesPerLevel("0,1", 2); + + Close(); +} + +// Sync points not supported in RocksDB Lite + +TEST_P(ColumnFamilyTest, MultipleManualCompactions) { + Open(); + CreateColumnFamilies({"one", "two"}); + ColumnFamilyOptions default_cf, one, two; + db_options_.max_open_files = 20; // only 10 files in file cache + db_options_.max_background_compactions = 3; + + default_cf.level_compaction_dynamic_level_bytes = false; + default_cf.compaction_style = kCompactionStyleLevel; + default_cf.num_levels = 3; + default_cf.write_buffer_size = 64 << 10; // 64KB + default_cf.target_file_size_base = 30 << 10; + default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.no_block_cache = true; + default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + one.compaction_style = kCompactionStyleUniversal; + + one.num_levels = 1; + // trigger compaction if there are >= 4 files + one.level0_file_num_compaction_trigger = 4; + one.write_buffer_size = 120000; + + two.level_compaction_dynamic_level_bytes = false; + two.compaction_style = kCompactionStyleLevel; + two.num_levels = 4; + two.level0_file_num_compaction_trigger = 3; + two.write_buffer_size = 100000; + + Reopen({default_cf, one, two}); + + // SETUP column family "one" -- universal style + for (int i = 0; i < one.level0_file_num_compaction_trigger - 2; ++i) { + PutRandomData(1, 10, 12000, true); + PutRandomData(1, 1, 10, true); + WaitForFlush(1); + AssertFilesPerLevel(std::to_string(i + 1), 1); + } + std::atomic_bool cf_1_1{true}; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"ColumnFamilyTest::MultiManual:4", "ColumnFamilyTest::MultiManual:1"}, + {"ColumnFamilyTest::MultiManual:2", "ColumnFamilyTest::MultiManual:5"}, + {"ColumnFamilyTest::MultiManual:2", "ColumnFamilyTest::MultiManual:3"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* /*arg*/) { + if (cf_1_1.exchange(false)) { + TEST_SYNC_POINT("ColumnFamilyTest::MultiManual:4"); + TEST_SYNC_POINT("ColumnFamilyTest::MultiManual:3"); + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + std::vector threads; + threads.emplace_back([&] { + CompactRangeOptions compact_options; + compact_options.exclusive_manual_compaction = false; + ASSERT_OK( + db_->CompactRange(compact_options, handles_[1], nullptr, nullptr)); + }); + + // SETUP column family "two" -- level style with 4 levels + for (int i = 0; i < two.level0_file_num_compaction_trigger - 2; ++i) { + PutRandomData(2, 10, 12000); + PutRandomData(2, 1, 10); + WaitForFlush(2); + AssertFilesPerLevel(std::to_string(i + 1), 2); + } + threads.emplace_back([&] { + TEST_SYNC_POINT("ColumnFamilyTest::MultiManual:1"); + CompactRangeOptions compact_options; + compact_options.exclusive_manual_compaction = false; + ASSERT_OK( + db_->CompactRange(compact_options, handles_[2], nullptr, nullptr)); + TEST_SYNC_POINT("ColumnFamilyTest::MultiManual:2"); + }); + + TEST_SYNC_POINT("ColumnFamilyTest::MultiManual:5"); + for (auto& t : threads) { + t.join(); + } + + // VERIFY compaction "one" + AssertFilesPerLevel("1", 1); + + // VERIFY compaction "two" + AssertFilesPerLevel("0,1", 2); + CompactAll(2); + AssertFilesPerLevel("0,1", 2); + // Compare against saved keys + std::set::iterator key_iter = keys_[1].begin(); + while (key_iter != keys_[1].end()) { + ASSERT_NE("NOT_FOUND", Get(1, *key_iter)); + key_iter++; + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + Close(); +} + +TEST_P(ColumnFamilyTest, AutomaticAndManualCompactions) { + Open(); + CreateColumnFamilies({"one", "two"}); + ColumnFamilyOptions default_cf, one, two; + db_options_.max_open_files = 20; // only 10 files in file cache + db_options_.max_background_compactions = 3; + + default_cf.level_compaction_dynamic_level_bytes = false; + default_cf.compaction_style = kCompactionStyleLevel; + default_cf.num_levels = 3; + default_cf.write_buffer_size = 64 << 10; // 64KB + default_cf.target_file_size_base = 30 << 10; + default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + + table_options.no_block_cache = true; + default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + one.compaction_style = kCompactionStyleUniversal; + + one.num_levels = 1; + // trigger compaction if there are >= 4 files + one.level0_file_num_compaction_trigger = 4; + one.write_buffer_size = 120000; + + two.level_compaction_dynamic_level_bytes = false; + two.compaction_style = kCompactionStyleLevel; + two.num_levels = 4; + two.level0_file_num_compaction_trigger = 3; + two.write_buffer_size = 100000; + + Reopen({default_cf, one, two}); + // make sure all background compaction jobs can be scheduled + auto stop_token = + dbfull()->TEST_write_controler().GetCompactionPressureToken(); + + std::atomic_bool cf_1_1{true}; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"ColumnFamilyTest::AutoManual:4", "ColumnFamilyTest::AutoManual:1"}, + {"ColumnFamilyTest::AutoManual:2", "ColumnFamilyTest::AutoManual:5"}, + {"ColumnFamilyTest::AutoManual:2", "ColumnFamilyTest::AutoManual:3"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* /*arg*/) { + if (cf_1_1.exchange(false)) { + TEST_SYNC_POINT("ColumnFamilyTest::AutoManual:4"); + TEST_SYNC_POINT("ColumnFamilyTest::AutoManual:3"); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + // SETUP column family "one" -- universal style + for (int i = 0; i < one.level0_file_num_compaction_trigger; ++i) { + PutRandomData(1, 10, 12000, true); + PutRandomData(1, 1, 10, true); + WaitForFlush(1); + AssertFilesPerLevel(std::to_string(i + 1), 1); + } + + TEST_SYNC_POINT("ColumnFamilyTest::AutoManual:1"); + + // SETUP column family "two" -- level style with 4 levels + for (int i = 0; i < two.level0_file_num_compaction_trigger - 2; ++i) { + PutRandomData(2, 10, 12000); + PutRandomData(2, 1, 10); + WaitForFlush(2); + AssertFilesPerLevel(std::to_string(i + 1), 2); + } + ROCKSDB_NAMESPACE::port::Thread threads([&] { + CompactRangeOptions compact_options; + compact_options.exclusive_manual_compaction = false; + ASSERT_OK( + db_->CompactRange(compact_options, handles_[2], nullptr, nullptr)); + TEST_SYNC_POINT("ColumnFamilyTest::AutoManual:2"); + }); + + TEST_SYNC_POINT("ColumnFamilyTest::AutoManual:5"); + threads.join(); + + // WAIT for compactions + WaitForCompaction(); + + // VERIFY compaction "one" + AssertFilesPerLevel("1", 1); + + // VERIFY compaction "two" + AssertFilesPerLevel("0,1", 2); + CompactAll(2); + AssertFilesPerLevel("0,1", 2); + // Compare against saved keys + std::set::iterator key_iter = keys_[1].begin(); + while (key_iter != keys_[1].end()) { + ASSERT_NE("NOT_FOUND", Get(1, *key_iter)); + key_iter++; + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_P(ColumnFamilyTest, ManualAndAutomaticCompactions) { + Open(); + CreateColumnFamilies({"one", "two"}); + ColumnFamilyOptions default_cf, one, two; + db_options_.max_open_files = 20; // only 10 files in file cache + db_options_.max_background_compactions = 3; + + default_cf.level_compaction_dynamic_level_bytes = false; + default_cf.compaction_style = kCompactionStyleLevel; + default_cf.num_levels = 3; + default_cf.write_buffer_size = 64 << 10; // 64KB + default_cf.target_file_size_base = 30 << 10; + default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + + table_options.no_block_cache = true; + default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + one.compaction_style = kCompactionStyleUniversal; + + one.num_levels = 1; + // trigger compaction if there are >= 4 files + one.level0_file_num_compaction_trigger = 4; + one.write_buffer_size = 120000; + + two.level_compaction_dynamic_level_bytes = false; + two.compaction_style = kCompactionStyleLevel; + two.num_levels = 4; + two.level0_file_num_compaction_trigger = 3; + two.write_buffer_size = 100000; + + Reopen({default_cf, one, two}); + // make sure all background compaction jobs can be scheduled + auto stop_token = + dbfull()->TEST_write_controler().GetCompactionPressureToken(); + + // SETUP column family "one" -- universal style + for (int i = 0; i < one.level0_file_num_compaction_trigger - 2; ++i) { + PutRandomData(1, 10, 12000, true); + PutRandomData(1, 1, 10, true); + WaitForFlush(1); + AssertFilesPerLevel(std::to_string(i + 1), 1); + } + std::atomic_bool cf_1_1{true}; + std::atomic_bool cf_1_2{true}; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"ColumnFamilyTest::ManualAuto:4", "ColumnFamilyTest::ManualAuto:1"}, + {"ColumnFamilyTest::ManualAuto:5", "ColumnFamilyTest::ManualAuto:2"}, + {"ColumnFamilyTest::ManualAuto:2", "ColumnFamilyTest::ManualAuto:3"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* /*arg*/) { + if (cf_1_1.exchange(false)) { + TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:4"); + TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:3"); + } else if (cf_1_2.exchange(false)) { + TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:2"); + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ROCKSDB_NAMESPACE::port::Thread threads([&] { + CompactRangeOptions compact_options; + compact_options.exclusive_manual_compaction = false; + ASSERT_OK( + db_->CompactRange(compact_options, handles_[1], nullptr, nullptr)); + }); + + TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:1"); + + // SETUP column family "two" -- level style with 4 levels + for (int i = 0; i < two.level0_file_num_compaction_trigger; ++i) { + PutRandomData(2, 10, 12000); + PutRandomData(2, 1, 10); + WaitForFlush(2); + AssertFilesPerLevel(std::to_string(i + 1), 2); + } + TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:5"); + threads.join(); + + // WAIT for compactions + WaitForCompaction(); + + // VERIFY compaction "one" + AssertFilesPerLevel("1", 1); + + // VERIFY compaction "two" + AssertFilesPerLevel("0,1", 2); + CompactAll(2); + AssertFilesPerLevel("0,1", 2); + // Compare against saved keys + std::set::iterator key_iter = keys_[1].begin(); + while (key_iter != keys_[1].end()) { + ASSERT_NE("NOT_FOUND", Get(1, *key_iter)); + key_iter++; + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_P(ColumnFamilyTest, SameCFManualManualCompactions) { + Open(); + CreateColumnFamilies({"one"}); + ColumnFamilyOptions default_cf, one; + db_options_.max_open_files = 20; // only 10 files in file cache + db_options_.max_background_compactions = 3; + + default_cf.compaction_style = kCompactionStyleLevel; + default_cf.num_levels = 3; + default_cf.write_buffer_size = 64 << 10; // 64KB + default_cf.target_file_size_base = 30 << 10; + default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + ; + table_options.no_block_cache = true; + default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + one.compaction_style = kCompactionStyleUniversal; + + one.num_levels = 1; + // trigger compaction if there are >= 4 files + one.level0_file_num_compaction_trigger = 4; + one.write_buffer_size = 120000; + + Reopen({default_cf, one}); + // make sure all background compaction jobs can be scheduled + auto stop_token = + dbfull()->TEST_write_controler().GetCompactionPressureToken(); + + // SETUP column family "one" -- universal style + for (int i = 0; i < one.level0_file_num_compaction_trigger - 2; ++i) { + PutRandomData(1, 10, 12000, true); + PutRandomData(1, 1, 10, true); + WaitForFlush(1); + AssertFilesPerLevel(std::to_string(i + 1), 1); + } + std::atomic_bool cf_1_1{true}; + std::atomic_bool cf_1_2{true}; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"ColumnFamilyTest::ManualManual:4", "ColumnFamilyTest::ManualManual:2"}, + {"ColumnFamilyTest::ManualManual:4", "ColumnFamilyTest::ManualManual:5"}, + {"ColumnFamilyTest::ManualManual:1", "ColumnFamilyTest::ManualManual:2"}, + {"ColumnFamilyTest::ManualManual:1", + "ColumnFamilyTest::ManualManual:3"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* /*arg*/) { + if (cf_1_1.exchange(false)) { + TEST_SYNC_POINT("ColumnFamilyTest::ManualManual:4"); + TEST_SYNC_POINT("ColumnFamilyTest::ManualManual:3"); + } else if (cf_1_2.exchange(false)) { + TEST_SYNC_POINT("ColumnFamilyTest::ManualManual:2"); + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ROCKSDB_NAMESPACE::port::Thread threads([&] { + CompactRangeOptions compact_options; + compact_options.exclusive_manual_compaction = true; + ASSERT_OK( + db_->CompactRange(compact_options, handles_[1], nullptr, nullptr)); + }); + + TEST_SYNC_POINT("ColumnFamilyTest::ManualManual:5"); + + WaitForFlush(1); + + // Add more L0 files and force another manual compaction + for (int i = 0; i < one.level0_file_num_compaction_trigger - 2; ++i) { + PutRandomData(1, 10, 12000, true); + PutRandomData(1, 1, 10, true); + WaitForFlush(1); + AssertFilesPerLevel( + std::to_string(one.level0_file_num_compaction_trigger + i), 1); + } + + ROCKSDB_NAMESPACE::port::Thread threads1([&] { + CompactRangeOptions compact_options; + compact_options.exclusive_manual_compaction = false; + ASSERT_OK( + db_->CompactRange(compact_options, handles_[1], nullptr, nullptr)); + }); + + TEST_SYNC_POINT("ColumnFamilyTest::ManualManual:1"); + + threads.join(); + threads1.join(); + WaitForCompaction(); + // VERIFY compaction "one" + ASSERT_LE(NumTableFilesAtLevel(0, 1), 2); + + // Compare against saved keys + std::set::iterator key_iter = keys_[1].begin(); + while (key_iter != keys_[1].end()) { + ASSERT_NE("NOT_FOUND", Get(1, *key_iter)); + key_iter++; + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_P(ColumnFamilyTest, SameCFManualAutomaticCompactions) { + Open(); + CreateColumnFamilies({"one"}); + ColumnFamilyOptions default_cf, one; + db_options_.max_open_files = 20; // only 10 files in file cache + db_options_.max_background_compactions = 3; + + default_cf.compaction_style = kCompactionStyleLevel; + default_cf.num_levels = 3; + default_cf.write_buffer_size = 64 << 10; // 64KB + default_cf.target_file_size_base = 30 << 10; + default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + ; + table_options.no_block_cache = true; + default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + one.compaction_style = kCompactionStyleUniversal; + + one.num_levels = 1; + // trigger compaction if there are >= 4 files + one.level0_file_num_compaction_trigger = 4; + one.write_buffer_size = 120000; + + Reopen({default_cf, one}); + // make sure all background compaction jobs can be scheduled + auto stop_token = + dbfull()->TEST_write_controler().GetCompactionPressureToken(); + + // SETUP column family "one" -- universal style + for (int i = 0; i < one.level0_file_num_compaction_trigger - 2; ++i) { + PutRandomData(1, 10, 12000, true); + PutRandomData(1, 1, 10, true); + WaitForFlush(1); + AssertFilesPerLevel(std::to_string(i + 1), 1); + } + std::atomic_bool cf_1_1{true}; + std::atomic_bool cf_1_2{true}; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"ColumnFamilyTest::ManualAuto:4", "ColumnFamilyTest::ManualAuto:2"}, + {"ColumnFamilyTest::ManualAuto:4", "ColumnFamilyTest::ManualAuto:5"}, + {"ColumnFamilyTest::ManualAuto:1", "ColumnFamilyTest::ManualAuto:2"}, + {"ColumnFamilyTest::ManualAuto:1", "ColumnFamilyTest::ManualAuto:3"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* /*arg*/) { + if (cf_1_1.exchange(false)) { + TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:4"); + TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:3"); + } else if (cf_1_2.exchange(false)) { + TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:2"); + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ROCKSDB_NAMESPACE::port::Thread threads([&] { + CompactRangeOptions compact_options; + compact_options.exclusive_manual_compaction = false; + ASSERT_OK( + db_->CompactRange(compact_options, handles_[1], nullptr, nullptr)); + }); + + TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:5"); + + WaitForFlush(1); + + // Add more L0 files and force automatic compaction + for (int i = 0; i < one.level0_file_num_compaction_trigger; ++i) { + PutRandomData(1, 10, 12000, true); + PutRandomData(1, 1, 10, true); + WaitForFlush(1); + AssertFilesPerLevel( + std::to_string(one.level0_file_num_compaction_trigger + i), 1); + } + + TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:1"); + + threads.join(); + WaitForCompaction(); + // VERIFY compaction "one" + ASSERT_LE(NumTableFilesAtLevel(0, 1), 2); + + // Compare against saved keys + std::set::iterator key_iter = keys_[1].begin(); + while (key_iter != keys_[1].end()) { + ASSERT_NE("NOT_FOUND", Get(1, *key_iter)); + key_iter++; + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_P(ColumnFamilyTest, SameCFManualAutomaticCompactionsLevel) { + Open(); + CreateColumnFamilies({"one"}); + ColumnFamilyOptions default_cf, one; + db_options_.max_open_files = 20; // only 10 files in file cache + db_options_.max_background_compactions = 3; + + default_cf.compaction_style = kCompactionStyleLevel; + default_cf.num_levels = 3; + default_cf.write_buffer_size = 64 << 10; // 64KB + default_cf.target_file_size_base = 30 << 10; + default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + ; + table_options.no_block_cache = true; + default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + one.compaction_style = kCompactionStyleLevel; + + one.num_levels = 1; + // trigger compaction if there are >= 4 files + one.level0_file_num_compaction_trigger = 3; + one.write_buffer_size = 120000; + + Reopen({default_cf, one}); + // make sure all background compaction jobs can be scheduled + auto stop_token = + dbfull()->TEST_write_controler().GetCompactionPressureToken(); + + // SETUP column family "one" -- level style + for (int i = 0; i < one.level0_file_num_compaction_trigger - 2; ++i) { + PutRandomData(1, 10, 12000, true); + PutRandomData(1, 1, 10, true); + WaitForFlush(1); + AssertFilesPerLevel(std::to_string(i + 1), 1); + } + std::atomic_bool cf_1_1{true}; + std::atomic_bool cf_1_2{true}; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"ColumnFamilyTest::ManualAuto:4", "ColumnFamilyTest::ManualAuto:2"}, + {"ColumnFamilyTest::ManualAuto:4", "ColumnFamilyTest::ManualAuto:5"}, + {"ColumnFamilyTest::ManualAuto:3", "ColumnFamilyTest::ManualAuto:2"}, + {"LevelCompactionPicker::PickCompactionBySize:0", + "ColumnFamilyTest::ManualAuto:3"}, + {"ColumnFamilyTest::ManualAuto:1", "ColumnFamilyTest::ManualAuto:3"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* /*arg*/) { + if (cf_1_1.exchange(false)) { + TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:4"); + TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:3"); + } else if (cf_1_2.exchange(false)) { + TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:2"); + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ROCKSDB_NAMESPACE::port::Thread threads([&] { + CompactRangeOptions compact_options; + compact_options.exclusive_manual_compaction = false; + ASSERT_OK( + db_->CompactRange(compact_options, handles_[1], nullptr, nullptr)); + }); + + TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:5"); + + // Add more L0 files and force automatic compaction + for (int i = 0; i < one.level0_file_num_compaction_trigger; ++i) { + PutRandomData(1, 10, 12000, true); + PutRandomData(1, 1, 10, true); + WaitForFlush(1); + AssertFilesPerLevel( + std::to_string(one.level0_file_num_compaction_trigger + i), 1); + } + + TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:1"); + + threads.join(); + WaitForCompaction(); + // VERIFY compaction "one" + AssertFilesPerLevel("0,1", 1); + + // Compare against saved keys + std::set::iterator key_iter = keys_[1].begin(); + while (key_iter != keys_[1].end()) { + ASSERT_NE("NOT_FOUND", Get(1, *key_iter)); + key_iter++; + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +// In this test, we generate enough files to trigger automatic compactions. +// The automatic compaction waits in NonTrivial:AfterRun +// We generate more files and then trigger an automatic compaction +// This will wait because the automatic compaction has files it needs. +// Once the conflict is hit, the automatic compaction starts and ends +// Then the manual will run and end. +TEST_P(ColumnFamilyTest, SameCFAutomaticManualCompactions) { + Open(); + CreateColumnFamilies({"one"}); + ColumnFamilyOptions default_cf, one; + db_options_.max_open_files = 20; // only 10 files in file cache + db_options_.max_background_compactions = 3; + + default_cf.compaction_style = kCompactionStyleLevel; + default_cf.num_levels = 3; + default_cf.write_buffer_size = 64 << 10; // 64KB + default_cf.target_file_size_base = 30 << 10; + default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + ; + table_options.no_block_cache = true; + default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + one.compaction_style = kCompactionStyleUniversal; + + one.num_levels = 1; + // trigger compaction if there are >= 4 files + one.level0_file_num_compaction_trigger = 4; + one.write_buffer_size = 120000; + + Reopen({default_cf, one}); + // make sure all background compaction jobs can be scheduled + auto stop_token = + dbfull()->TEST_write_controler().GetCompactionPressureToken(); + + std::atomic_bool cf_1_1{true}; + std::atomic_bool cf_1_2{true}; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"ColumnFamilyTest::AutoManual:4", "ColumnFamilyTest::AutoManual:2"}, + {"ColumnFamilyTest::AutoManual:4", "ColumnFamilyTest::AutoManual:5"}, + {"CompactionPicker::CompactRange:Conflict", + "ColumnFamilyTest::AutoManual:3"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* /*arg*/) { + if (cf_1_1.exchange(false)) { + TEST_SYNC_POINT("ColumnFamilyTest::AutoManual:4"); + TEST_SYNC_POINT("ColumnFamilyTest::AutoManual:3"); + } else if (cf_1_2.exchange(false)) { + TEST_SYNC_POINT("ColumnFamilyTest::AutoManual:2"); + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // SETUP column family "one" -- universal style + for (int i = 0; i < one.level0_file_num_compaction_trigger; ++i) { + PutRandomData(1, 10, 12000, true); + PutRandomData(1, 1, 10, true); + WaitForFlush(1); + AssertFilesPerLevel(std::to_string(i + 1), 1); + } + + TEST_SYNC_POINT("ColumnFamilyTest::AutoManual:5"); + + // Add another L0 file and force automatic compaction + for (int i = 0; i < one.level0_file_num_compaction_trigger - 2; ++i) { + PutRandomData(1, 10, 12000, true); + PutRandomData(1, 1, 10, true); + WaitForFlush(1); + } + + CompactRangeOptions compact_options; + compact_options.exclusive_manual_compaction = false; + ASSERT_OK(db_->CompactRange(compact_options, handles_[1], nullptr, nullptr)); + + TEST_SYNC_POINT("ColumnFamilyTest::AutoManual:1"); + + WaitForCompaction(); + // VERIFY compaction "one" + AssertFilesPerLevel("1", 1); + // Compare against saved keys + std::set::iterator key_iter = keys_[1].begin(); + while (key_iter != keys_[1].end()) { + ASSERT_NE("NOT_FOUND", Get(1, *key_iter)); + key_iter++; + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +namespace { +std::string IterStatus(Iterator* iter) { + std::string result; + if (iter->Valid()) { + result = iter->key().ToString() + "->" + iter->value().ToString(); + } else { + EXPECT_OK(iter->status()); + result = "(invalid)"; + } + return result; +} +} // anonymous namespace + +TEST_P(ColumnFamilyTest, NewIteratorsTest) { + // iter == 0 -- no tailing + // iter == 2 -- tailing + for (int iter = 0; iter < 2; ++iter) { + Open(); + CreateColumnFamiliesAndReopen({"one", "two"}); + ASSERT_OK(Put(0, "a", "b")); + ASSERT_OK(Put(1, "b", "a")); + ASSERT_OK(Put(2, "c", "m")); + ASSERT_OK(Put(2, "v", "t")); + std::vector iterators; + ReadOptions options; + options.tailing = (iter == 1); + ASSERT_OK(db_->NewIterators(options, handles_, &iterators)); + + for (auto it : iterators) { + it->SeekToFirst(); + } + ASSERT_EQ(IterStatus(iterators[0]), "a->b"); + ASSERT_EQ(IterStatus(iterators[1]), "b->a"); + ASSERT_EQ(IterStatus(iterators[2]), "c->m"); + + ASSERT_OK(Put(1, "x", "x")); + + for (auto it : iterators) { + it->Next(); + } + + ASSERT_EQ(IterStatus(iterators[0]), "(invalid)"); + if (iter == 0) { + // no tailing + ASSERT_EQ(IterStatus(iterators[1]), "(invalid)"); + } else { + // tailing + ASSERT_EQ(IterStatus(iterators[1]), "x->x"); + } + ASSERT_EQ(IterStatus(iterators[2]), "v->t"); + + for (auto it : iterators) { + delete it; + } + Destroy(); + } +} + +TEST_P(ColumnFamilyTest, ReadOnlyDBTest) { + Open(); + CreateColumnFamiliesAndReopen({"one", "two", "three", "four"}); + ASSERT_OK(Put(0, "a", "b")); + ASSERT_OK(Put(1, "foo", "bla")); + ASSERT_OK(Put(2, "foo", "blabla")); + ASSERT_OK(Put(3, "foo", "blablabla")); + ASSERT_OK(Put(4, "foo", "blablablabla")); + + DropColumnFamilies({2}); + Close(); + // open only a subset of column families + AssertOpenReadOnly({"default", "one", "four"}); + ASSERT_EQ("NOT_FOUND", Get(0, "foo")); + ASSERT_EQ("bla", Get(1, "foo")); + ASSERT_EQ("blablablabla", Get(2, "foo")); + + // test newiterators + { + std::vector iterators; + ASSERT_OK(db_->NewIterators(ReadOptions(), handles_, &iterators)); + for (auto it : iterators) { + it->SeekToFirst(); + } + ASSERT_EQ(IterStatus(iterators[0]), "a->b"); + ASSERT_EQ(IterStatus(iterators[1]), "foo->bla"); + ASSERT_EQ(IterStatus(iterators[2]), "foo->blablablabla"); + for (auto it : iterators) { + it->Next(); + } + ASSERT_EQ(IterStatus(iterators[0]), "(invalid)"); + ASSERT_EQ(IterStatus(iterators[1]), "(invalid)"); + ASSERT_EQ(IterStatus(iterators[2]), "(invalid)"); + + for (auto it : iterators) { + delete it; + } + } + + Close(); + // can't open dropped column family + Status s = OpenReadOnly({"default", "one", "two"}); + ASSERT_TRUE(!s.ok()); + + // Can't open without specifying default column family + s = OpenReadOnly({"one", "four"}); + ASSERT_TRUE(!s.ok()); +} + +TEST_P(ColumnFamilyTest, DontRollEmptyLogs) { + Open(); + CreateColumnFamiliesAndReopen({"one", "two", "three", "four"}); + + for (size_t i = 0; i < handles_.size(); ++i) { + PutRandomData(static_cast(i), 10, 100); + } + int num_writable_file_start = env_->GetNumberOfNewWritableFileCalls(); + // this will trigger the flushes + for (int i = 0; i <= 4; ++i) { + ASSERT_OK(Flush(i)); + } + + for (int i = 0; i < 4; ++i) { + WaitForFlush(i); + } + int total_new_writable_files = + env_->GetNumberOfNewWritableFileCalls() - num_writable_file_start; + ASSERT_EQ(static_cast(total_new_writable_files), handles_.size() + 1); + Close(); +} + +TEST_P(ColumnFamilyTest, FlushStaleColumnFamilies) { + Open(); + CreateColumnFamilies({"one", "two"}); + ColumnFamilyOptions default_cf, one, two; + default_cf.write_buffer_size = 100000; // small write buffer size + default_cf.arena_block_size = 4096; + default_cf.disable_auto_compactions = true; + one.disable_auto_compactions = true; + two.disable_auto_compactions = true; + db_options_.max_total_wal_size = 210000; + + Reopen({default_cf, one, two}); + + PutRandomData(2, 1, 10); // 10 bytes + for (int i = 0; i < 2; ++i) { + PutRandomData(0, 100, 1000); // flush + WaitForFlush(0); + + AssertCountLiveFiles(i + 1); + } + // third flush. now, CF [two] should be detected as stale and flushed + // column family 1 should not be flushed since it's empty + PutRandomData(0, 100, 1000); // flush + WaitForFlush(0); + WaitForFlush(2); + // at least 3 files for default column families, 1 file for column family + // [two], zero files for column family [one], because it's empty + std::vector metadata; + db_->GetLiveFilesMetaData(&metadata); + ASSERT_GE(metadata.size(), 4); + bool has_cf1_sst = false; + bool has_cf2_sst = false; + for (const auto& file : metadata) { + if (file.column_family_name == "one") { + has_cf1_sst = true; + } else if (file.column_family_name == "two") { + has_cf2_sst = true; + } + } + ASSERT_FALSE(has_cf1_sst); + ASSERT_TRUE(has_cf2_sst); + + ASSERT_OK(Flush(0)); + ASSERT_EQ(0, dbfull()->TEST_total_log_size()); + Close(); +} + +TEST_P(ColumnFamilyTest, CreateMissingColumnFamilies) { + Status s = TryOpen({"one", "two"}); + ASSERT_TRUE(!s.ok()); + db_options_.create_missing_column_families = true; + s = TryOpen({"default", "one", "two"}); + ASSERT_TRUE(s.ok()); + Close(); +} + +TEST_P(ColumnFamilyTest, SanitizeOptions) { + DBOptions db_options; + for (int s = kCompactionStyleLevel; s <= kCompactionStyleUniversal; ++s) { + for (int l = 0; l <= 2; l++) { + for (int i = 1; i <= 3; i++) { + for (int j = 1; j <= 3; j++) { + for (int k = 1; k <= 3; k++) { + ColumnFamilyOptions original; + original.compaction_style = static_cast(s); + original.num_levels = l; + original.level0_stop_writes_trigger = i; + original.level0_slowdown_writes_trigger = j; + original.level0_file_num_compaction_trigger = k; + original.write_buffer_size = + l * 4 * 1024 * 1024 + i * 1024 * 1024 + j * 1024 + k; + + ColumnFamilyOptions result = + SanitizeOptions(ImmutableDBOptions(db_options), original); + ASSERT_TRUE(result.level0_stop_writes_trigger >= + result.level0_slowdown_writes_trigger); + ASSERT_TRUE(result.level0_slowdown_writes_trigger >= + result.level0_file_num_compaction_trigger); + ASSERT_TRUE(result.level0_file_num_compaction_trigger == + original.level0_file_num_compaction_trigger); + if (s == kCompactionStyleLevel) { + ASSERT_GE(result.num_levels, 2); + } else { + ASSERT_GE(result.num_levels, 1); + if (original.num_levels >= 1) { + ASSERT_EQ(result.num_levels, original.num_levels); + } + } + + // Make sure Sanitize options sets arena_block_size to 1/8 of + // the write_buffer_size, rounded up to a multiple of 4k. + size_t expected_arena_block_size = + l * 4 * 1024 * 1024 / 8 + i * 1024 * 1024 / 8; + if (j + k != 0) { + // not a multiple of 4k, round up 4k + expected_arena_block_size += 4 * 1024; + } + expected_arena_block_size = + std::min(size_t{1024 * 1024}, expected_arena_block_size); + ASSERT_EQ(expected_arena_block_size, result.arena_block_size); + } + } + } + } + } +} + +TEST_P(ColumnFamilyTest, ReadDroppedColumnFamily) { + // iter 0 -- drop CF, don't reopen + // iter 1 -- delete CF, reopen + for (int iter = 0; iter < 2; ++iter) { + db_options_.create_missing_column_families = true; + db_options_.max_open_files = 20; + // delete obsolete files always + db_options_.delete_obsolete_files_period_micros = 0; + Open({"default", "one", "two"}); + ColumnFamilyOptions options; + options.level0_file_num_compaction_trigger = 100; + options.level0_slowdown_writes_trigger = 200; + options.level0_stop_writes_trigger = 200; + options.write_buffer_size = 100000; // small write buffer size + Reopen({options, options, options}); + + // 1MB should create ~10 files for each CF + int kKeysNum = 10000; + PutRandomData(0, kKeysNum, 100); + PutRandomData(1, kKeysNum, 100); + PutRandomData(2, kKeysNum, 100); + + { + std::unique_ptr iterator( + db_->NewIterator(ReadOptions(), handles_[2])); + iterator->SeekToFirst(); + + if (iter == 0) { + // Drop CF two + ASSERT_OK(db_->DropColumnFamily(handles_[2])); + } else { + // delete CF two + ASSERT_OK(db_->DestroyColumnFamilyHandle(handles_[2])); + handles_[2] = nullptr; + } + // Make sure iterator created can still be used. + int count = 0; + for (; iterator->Valid(); iterator->Next()) { + ASSERT_OK(iterator->status()); + ++count; + } + ASSERT_OK(iterator->status()); + ASSERT_EQ(count, kKeysNum); + } + + // Add bunch more data to other CFs + PutRandomData(0, kKeysNum, 100); + PutRandomData(1, kKeysNum, 100); + + if (iter == 1) { + Reopen(); + } + + // Since we didn't delete CF handle, RocksDB's contract guarantees that + // we're still able to read dropped CF + for (int i = 0; i < 3; ++i) { + std::unique_ptr iterator( + db_->NewIterator(ReadOptions(), handles_[i])); + int count = 0; + for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) { + ASSERT_OK(iterator->status()); + ++count; + } + ASSERT_OK(iterator->status()); + ASSERT_EQ(count, kKeysNum * ((i == 2) ? 1 : 2)); + } + + Close(); + Destroy(); + } +} + +TEST_P(ColumnFamilyTest, LiveIteratorWithDroppedColumnFamily) { + db_options_.create_missing_column_families = true; + db_options_.max_open_files = 20; + // delete obsolete files always + db_options_.delete_obsolete_files_period_micros = 0; + Open({"default", "one", "two"}); + ColumnFamilyOptions options; + options.level0_file_num_compaction_trigger = 100; + options.level0_slowdown_writes_trigger = 200; + options.level0_stop_writes_trigger = 200; + options.write_buffer_size = 100000; // small write buffer size + Reopen({options, options, options}); + + // 1MB should create ~10 files for each CF + int kKeysNum = 10000; + PutRandomData(1, kKeysNum, 100); + { + std::unique_ptr iterator( + db_->NewIterator(ReadOptions(), handles_[1])); + iterator->SeekToFirst(); + + DropColumnFamilies({1}); + + // Make sure iterator created can still be used. + int count = 0; + for (; iterator->Valid(); iterator->Next()) { + ASSERT_OK(iterator->status()); + ++count; + } + ASSERT_OK(iterator->status()); + ASSERT_EQ(count, kKeysNum); + } + + Reopen(); + Close(); + Destroy(); +} + +TEST_P(ColumnFamilyTest, FlushAndDropRaceCondition) { + db_options_.create_missing_column_families = true; + Open({"default", "one"}); + ColumnFamilyOptions options; + options.level0_file_num_compaction_trigger = 100; + options.level0_slowdown_writes_trigger = 200; + options.level0_stop_writes_trigger = 200; + options.max_write_buffer_number = 20; + options.write_buffer_size = 100000; // small write buffer size + Reopen({options, options}); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"VersionSet::LogAndApply::ColumnFamilyDrop:0", + "FlushJob::WriteLevel0Table"}, + {"VersionSet::LogAndApply::ColumnFamilyDrop:1", + "FlushJob::InstallResults"}, + {"FlushJob::InstallResults", + "VersionSet::LogAndApply::ColumnFamilyDrop:2"}}); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + test::SleepingBackgroundTask sleeping_task; + + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task, + Env::Priority::HIGH); + // Make sure the task is sleeping. Otherwise, it might start to execute + // after sleeping_task.WaitUntilDone() and cause TSAN warning. + sleeping_task.WaitUntilSleeping(); + + // 1MB should create ~10 files for each CF + int kKeysNum = 10000; + PutRandomData(1, kKeysNum, 100); + + std::vector threads; + threads.emplace_back([&] { ASSERT_OK(db_->DropColumnFamily(handles_[1])); }); + + sleeping_task.WakeUp(); + sleeping_task.WaitUntilDone(); + sleeping_task.Reset(); + // now we sleep again. this is just so we're certain that flush job finished + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task, + Env::Priority::HIGH); + // Make sure the task is sleeping. Otherwise, it might start to execute + // after sleeping_task.WaitUntilDone() and cause TSAN warning. + sleeping_task.WaitUntilSleeping(); + sleeping_task.WakeUp(); + sleeping_task.WaitUntilDone(); + + { + // Since we didn't delete CF handle, RocksDB's contract guarantees that + // we're still able to read dropped CF + std::unique_ptr iterator( + db_->NewIterator(ReadOptions(), handles_[1])); + int count = 0; + for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) { + ASSERT_OK(iterator->status()); + ++count; + } + ASSERT_OK(iterator->status()); + ASSERT_EQ(count, kKeysNum); + } + for (auto& t : threads) { + t.join(); + } + + Close(); + Destroy(); +} + +namespace { +std::atomic test_stage(0); +std::atomic ordered_by_writethread(false); +const int kMainThreadStartPersistingOptionsFile = 1; +const int kChildThreadFinishDroppingColumnFamily = 2; +void DropSingleColumnFamily(ColumnFamilyTest* cf_test, int cf_id, + std::vector* comparators) { + while (test_stage < kMainThreadStartPersistingOptionsFile && + !ordered_by_writethread) { + Env::Default()->SleepForMicroseconds(100); + } + cf_test->DropColumnFamilies({cf_id}); + if ((*comparators)[cf_id]) { + delete (*comparators)[cf_id]; + (*comparators)[cf_id] = nullptr; + } + test_stage = kChildThreadFinishDroppingColumnFamily; +} +} // anonymous namespace + +TEST_P(ColumnFamilyTest, CreateAndDropRace) { + const int kCfCount = 5; + std::vector cf_opts; + std::vector comparators; + for (int i = 0; i < kCfCount; ++i) { + cf_opts.emplace_back(); + comparators.push_back(new test::SimpleSuffixReverseComparator()); + cf_opts.back().comparator = comparators.back(); + } + db_options_.create_if_missing = true; + db_options_.create_missing_column_families = true; + + auto main_thread_id = std::this_thread::get_id(); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "PersistRocksDBOptions:start", [&](void* /*arg*/) { + auto current_thread_id = std::this_thread::get_id(); + // If it's the main thread hitting this sync-point, then it + // will be blocked until some other thread update the test_stage. + if (main_thread_id == current_thread_id) { + test_stage = kMainThreadStartPersistingOptionsFile; + while (test_stage < kChildThreadFinishDroppingColumnFamily && + !ordered_by_writethread) { + Env::Default()->SleepForMicroseconds(100); + } + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WriteThread::EnterUnbatched:Wait", [&](void* /*arg*/) { + // This means a thread doing DropColumnFamily() is waiting for + // other thread to finish persisting options. + // In such case, we update the test_stage to unblock the main thread. + ordered_by_writethread = true; + }); + + // Create a database with four column families + Open({"default", "one", "two", "three"}, + {cf_opts[0], cf_opts[1], cf_opts[2], cf_opts[3]}); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Start a thread that will drop the first column family + // and its comparator + ROCKSDB_NAMESPACE::port::Thread drop_cf_thread(DropSingleColumnFamily, this, + 1, &comparators); + + DropColumnFamilies({2}); + + drop_cf_thread.join(); + Close(); + Destroy(); + for (auto* comparator : comparators) { + if (comparator) { + delete comparator; + } + } + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_P(ColumnFamilyTest, WriteStallSingleColumnFamily) { + const uint64_t kBaseRate = 800000u; + db_options_.delayed_write_rate = kBaseRate; + db_options_.max_background_compactions = 6; + + Open({"default"}); + ColumnFamilyData* cfd = + static_cast(db_->DefaultColumnFamily())->cfd(); + + VersionStorageInfo* vstorage = cfd->current()->storage_info(); + + MutableCFOptions mutable_cf_options(column_family_options_); + + mutable_cf_options.level0_slowdown_writes_trigger = 20; + mutable_cf_options.level0_stop_writes_trigger = 10000; + mutable_cf_options.soft_pending_compaction_bytes_limit = 200; + mutable_cf_options.hard_pending_compaction_bytes_limit = 2000; + mutable_cf_options.disable_auto_compactions = false; + + vstorage->TEST_set_estimated_compaction_needed_bytes(50); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(201); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate()); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(400); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate()); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(500); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(450); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(205); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(202); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(201); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(198); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(399); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(599); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(2001); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(IsDbWriteStopped()); + ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(3001); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(IsDbWriteStopped()); + ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(390); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(100); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); + + vstorage->set_l0_delay_trigger_count(100); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate()); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); + + vstorage->set_l0_delay_trigger_count(101); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate()); + + vstorage->set_l0_delay_trigger_count(0); + vstorage->TEST_set_estimated_compaction_needed_bytes(300); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate()); + + vstorage->set_l0_delay_trigger_count(101); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate / 1.25 / 1.25 / 1.25, GetDbDelayedWriteRate()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(200); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate()); + + vstorage->set_l0_delay_trigger_count(0); + vstorage->TEST_set_estimated_compaction_needed_bytes(0); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); + + mutable_cf_options.disable_auto_compactions = true; + dbfull()->TEST_write_controler().set_delayed_write_rate(kBaseRate); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); + + vstorage->set_l0_delay_trigger_count(50); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(0, GetDbDelayedWriteRate()); + ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate()); + + vstorage->set_l0_delay_trigger_count(60); + vstorage->TEST_set_estimated_compaction_needed_bytes(300); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(0, GetDbDelayedWriteRate()); + ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate()); + + mutable_cf_options.disable_auto_compactions = false; + vstorage->set_l0_delay_trigger_count(70); + vstorage->TEST_set_estimated_compaction_needed_bytes(500); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate()); + + vstorage->set_l0_delay_trigger_count(71); + vstorage->TEST_set_estimated_compaction_needed_bytes(501); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate()); +} + +TEST_P(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) { + db_options_.max_background_compactions = 6; + Open({"default"}); + ColumnFamilyData* cfd = + static_cast(db_->DefaultColumnFamily())->cfd(); + + VersionStorageInfo* vstorage = cfd->current()->storage_info(); + + MutableCFOptions mutable_cf_options(column_family_options_); + + // Speed up threshold = min(4 * 2, 4 + (36 - 4)/4) = 8 + mutable_cf_options.level0_file_num_compaction_trigger = 4; + mutable_cf_options.level0_slowdown_writes_trigger = 36; + mutable_cf_options.level0_stop_writes_trigger = 50; + // Speedup threshold = 200 / 4 = 50 + mutable_cf_options.soft_pending_compaction_bytes_limit = 200; + mutable_cf_options.hard_pending_compaction_bytes_limit = 2000; + + vstorage->TEST_set_estimated_compaction_needed_bytes(40); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(50); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(300); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(45); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); + + vstorage->set_l0_delay_trigger_count(7); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); + + vstorage->set_l0_delay_trigger_count(9); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); + + vstorage->set_l0_delay_trigger_count(6); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); + + // Speed up threshold = min(4 * 2, 4 + (12 - 4)/4) = 6 + mutable_cf_options.level0_file_num_compaction_trigger = 4; + mutable_cf_options.level0_slowdown_writes_trigger = 16; + mutable_cf_options.level0_stop_writes_trigger = 30; + + vstorage->set_l0_delay_trigger_count(5); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); + + vstorage->set_l0_delay_trigger_count(7); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); + + vstorage->set_l0_delay_trigger_count(3); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); +} + +TEST_P(ColumnFamilyTest, WriteStallTwoColumnFamilies) { + const uint64_t kBaseRate = 810000u; + db_options_.delayed_write_rate = kBaseRate; + Open(); + CreateColumnFamilies({"one"}); + ColumnFamilyData* cfd = + static_cast(db_->DefaultColumnFamily())->cfd(); + VersionStorageInfo* vstorage = cfd->current()->storage_info(); + + ColumnFamilyData* cfd1 = + static_cast(handles_[1])->cfd(); + VersionStorageInfo* vstorage1 = cfd1->current()->storage_info(); + + MutableCFOptions mutable_cf_options(column_family_options_); + mutable_cf_options.level0_slowdown_writes_trigger = 20; + mutable_cf_options.level0_stop_writes_trigger = 10000; + mutable_cf_options.soft_pending_compaction_bytes_limit = 200; + mutable_cf_options.hard_pending_compaction_bytes_limit = 2000; + + MutableCFOptions mutable_cf_options1 = mutable_cf_options; + mutable_cf_options1.soft_pending_compaction_bytes_limit = 500; + + vstorage->TEST_set_estimated_compaction_needed_bytes(50); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); + + vstorage1->TEST_set_estimated_compaction_needed_bytes(201); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); + + vstorage1->TEST_set_estimated_compaction_needed_bytes(600); + RecalculateWriteStallConditions(cfd1, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(70); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate()); + + vstorage1->TEST_set_estimated_compaction_needed_bytes(800); + RecalculateWriteStallConditions(cfd1, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(300); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate()); + + vstorage1->TEST_set_estimated_compaction_needed_bytes(700); + RecalculateWriteStallConditions(cfd1, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(500); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate()); + + vstorage1->TEST_set_estimated_compaction_needed_bytes(600); + RecalculateWriteStallConditions(cfd1, mutable_cf_options); + ASSERT_TRUE(!IsDbWriteStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate()); +} + +TEST_P(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) { + db_options_.max_background_compactions = 6; + column_family_options_.soft_pending_compaction_bytes_limit = 200; + column_family_options_.hard_pending_compaction_bytes_limit = 2000; + Open(); + CreateColumnFamilies({"one"}); + ColumnFamilyData* cfd = + static_cast(db_->DefaultColumnFamily())->cfd(); + VersionStorageInfo* vstorage = cfd->current()->storage_info(); + + ColumnFamilyData* cfd1 = + static_cast(handles_[1])->cfd(); + VersionStorageInfo* vstorage1 = cfd1->current()->storage_info(); + + MutableCFOptions mutable_cf_options(column_family_options_); + // Speed up threshold = min(4 * 2, 4 + (36 - 4)/4) = 8 + mutable_cf_options.level0_file_num_compaction_trigger = 4; + mutable_cf_options.level0_slowdown_writes_trigger = 36; + mutable_cf_options.level0_stop_writes_trigger = 30; + // Speedup threshold = 200 / 4 = 50 + mutable_cf_options.soft_pending_compaction_bytes_limit = 200; + mutable_cf_options.hard_pending_compaction_bytes_limit = 2000; + + MutableCFOptions mutable_cf_options1 = mutable_cf_options; + mutable_cf_options1.level0_slowdown_writes_trigger = 16; + + vstorage->TEST_set_estimated_compaction_needed_bytes(40); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(60); + RecalculateWriteStallConditions(cfd1, mutable_cf_options); + ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); + + vstorage1->TEST_set_estimated_compaction_needed_bytes(30); + RecalculateWriteStallConditions(cfd1, mutable_cf_options); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); + + vstorage1->TEST_set_estimated_compaction_needed_bytes(70); + RecalculateWriteStallConditions(cfd1, mutable_cf_options); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); + + vstorage->TEST_set_estimated_compaction_needed_bytes(20); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); + + vstorage1->TEST_set_estimated_compaction_needed_bytes(3); + RecalculateWriteStallConditions(cfd1, mutable_cf_options); + ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); + + vstorage->set_l0_delay_trigger_count(9); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); + + vstorage1->set_l0_delay_trigger_count(2); + RecalculateWriteStallConditions(cfd1, mutable_cf_options); + ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); + + vstorage->set_l0_delay_trigger_count(0); + RecalculateWriteStallConditions(cfd, mutable_cf_options); + ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); +} + +TEST_P(ColumnFamilyTest, CreateAndDestroyOptions) { + std::unique_ptr cfo(new ColumnFamilyOptions()); + ColumnFamilyHandle* cfh; + Open(); + ASSERT_OK(db_->CreateColumnFamily(*(cfo.get()), "yoyo", &cfh)); + cfo.reset(); + ASSERT_OK(db_->Put(WriteOptions(), cfh, "foo", "bar")); + ASSERT_OK(db_->Flush(FlushOptions(), cfh)); + ASSERT_OK(db_->DropColumnFamily(cfh)); + ASSERT_OK(db_->DestroyColumnFamilyHandle(cfh)); +} + +TEST_P(ColumnFamilyTest, CreateDropAndDestroy) { + ColumnFamilyHandle* cfh; + Open(); + ASSERT_OK(db_->CreateColumnFamily(ColumnFamilyOptions(), "yoyo", &cfh)); + ASSERT_OK(db_->Put(WriteOptions(), cfh, "foo", "bar")); + ASSERT_OK(db_->Flush(FlushOptions(), cfh)); + ASSERT_OK(db_->DropColumnFamily(cfh)); + ASSERT_OK(db_->DestroyColumnFamilyHandle(cfh)); +} + +TEST_P(ColumnFamilyTest, CreateDropAndDestroyWithoutFileDeletion) { + ColumnFamilyHandle* cfh; + Open(); + ASSERT_OK(db_->CreateColumnFamily(ColumnFamilyOptions(), "yoyo", &cfh)); + ASSERT_OK(db_->Put(WriteOptions(), cfh, "foo", "bar")); + ASSERT_OK(db_->Flush(FlushOptions(), cfh)); + ASSERT_OK(db_->DisableFileDeletions()); + ASSERT_OK(db_->DropColumnFamily(cfh)); + ASSERT_OK(db_->DestroyColumnFamilyHandle(cfh)); +} + +TEST_P(ColumnFamilyTest, FlushCloseWALFiles) { + SpecialEnv env(Env::Default()); + db_options_.env = &env; + db_options_.max_background_flushes = 1; + column_family_options_.memtable_factory.reset( + test::NewSpecialSkipListFactory(2)); + Open(); + CreateColumnFamilies({"one"}); + ASSERT_OK(Put(1, "fodor", "mirko")); + ASSERT_OK(Put(0, "fodor", "mirko")); + ASSERT_OK(Put(1, "fodor", "mirko")); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"DBImpl::BGWorkFlush:done", "FlushCloseWALFiles:0"}, + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Block flush jobs from running + test::SleepingBackgroundTask sleeping_task; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task, + Env::Priority::HIGH); + // Make sure the task is sleeping. Otherwise, it might start to execute + // after sleeping_task.WaitUntilDone() and cause TSAN warning. + sleeping_task.WaitUntilSleeping(); + + WriteOptions wo; + wo.sync = true; + ASSERT_OK(db_->Put(wo, handles_[1], "fodor", "mirko")); + + ASSERT_EQ(2, env.num_open_wal_file_.load()); + + sleeping_task.WakeUp(); + sleeping_task.WaitUntilDone(); + TEST_SYNC_POINT("FlushCloseWALFiles:0"); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ASSERT_EQ(1, env.num_open_wal_file_.load()); + + Reopen(); + ASSERT_EQ("mirko", Get(0, "fodor")); + ASSERT_EQ("mirko", Get(1, "fodor")); + db_options_.env = env_; + Close(); +} + +TEST_P(ColumnFamilyTest, IteratorCloseWALFile1) { + SpecialEnv env(Env::Default()); + db_options_.env = &env; + db_options_.max_background_flushes = 1; + column_family_options_.memtable_factory.reset( + test::NewSpecialSkipListFactory(2)); + Open(); + CreateColumnFamilies({"one"}); + ASSERT_OK(Put(1, "fodor", "mirko")); + // Create an iterator holding the current super version. + Iterator* it = db_->NewIterator(ReadOptions(), handles_[1]); + ASSERT_OK(it->status()); + // A flush will make `it` hold the last reference of its super version. + ASSERT_OK(Flush(1)); + + ASSERT_OK(Put(1, "fodor", "mirko")); + ASSERT_OK(Put(0, "fodor", "mirko")); + ASSERT_OK(Put(1, "fodor", "mirko")); + + // Flush jobs will close previous WAL files after finishing. By + // block flush jobs from running, we trigger a condition where + // the iterator destructor should close the WAL files. + test::SleepingBackgroundTask sleeping_task; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task, + Env::Priority::HIGH); + // Make sure the task is sleeping. Otherwise, it might start to execute + // after sleeping_task.WaitUntilDone() and cause TSAN warning. + sleeping_task.WaitUntilSleeping(); + + WriteOptions wo; + wo.sync = true; + ASSERT_OK(db_->Put(wo, handles_[1], "fodor", "mirko")); + + ASSERT_EQ(2, env.num_open_wal_file_.load()); + // Deleting the iterator will clear its super version, triggering + // closing all files + delete it; + ASSERT_EQ(1, env.num_open_wal_file_.load()); + + sleeping_task.WakeUp(); + sleeping_task.WaitUntilDone(); + WaitForFlush(1); + + Reopen(); + ASSERT_EQ("mirko", Get(0, "fodor")); + ASSERT_EQ("mirko", Get(1, "fodor")); + db_options_.env = env_; + Close(); +} + +TEST_P(ColumnFamilyTest, IteratorCloseWALFile2) { + SpecialEnv env(Env::Default()); + // Allow both of flush and purge job to schedule. + env.SetBackgroundThreads(2, Env::HIGH); + db_options_.env = &env; + db_options_.max_background_flushes = 1; + column_family_options_.memtable_factory.reset( + test::NewSpecialSkipListFactory(2)); + Open(); + CreateColumnFamilies({"one"}); + ASSERT_OK(Put(1, "fodor", "mirko")); + // Create an iterator holding the current super version. + ReadOptions ro; + ro.background_purge_on_iterator_cleanup = true; + Iterator* it = db_->NewIterator(ro, handles_[1]); + ASSERT_OK(it->status()); + // A flush will make `it` hold the last reference of its super version. + ASSERT_OK(Flush(1)); + + ASSERT_OK(Put(1, "fodor", "mirko")); + ASSERT_OK(Put(0, "fodor", "mirko")); + ASSERT_OK(Put(1, "fodor", "mirko")); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"ColumnFamilyTest::IteratorCloseWALFile2:0", + "DBImpl::BGWorkPurge:start"}, + {"ColumnFamilyTest::IteratorCloseWALFile2:2", + "DBImpl::BackgroundCallFlush:start"}, + {"DBImpl::BGWorkPurge:end", "ColumnFamilyTest::IteratorCloseWALFile2:1"}, + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + WriteOptions wo; + wo.sync = true; + ASSERT_OK(db_->Put(wo, handles_[1], "fodor", "mirko")); + + ASSERT_EQ(2, env.num_open_wal_file_.load()); + // Deleting the iterator will clear its super version, triggering + // closing all files + delete it; + ASSERT_EQ(2, env.num_open_wal_file_.load()); + + TEST_SYNC_POINT("ColumnFamilyTest::IteratorCloseWALFile2:0"); + TEST_SYNC_POINT("ColumnFamilyTest::IteratorCloseWALFile2:1"); + ASSERT_EQ(1, env.num_open_wal_file_.load()); + TEST_SYNC_POINT("ColumnFamilyTest::IteratorCloseWALFile2:2"); + WaitForFlush(1); + ASSERT_EQ(1, env.num_open_wal_file_.load()); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + Reopen(); + ASSERT_EQ("mirko", Get(0, "fodor")); + ASSERT_EQ("mirko", Get(1, "fodor")); + db_options_.env = env_; + Close(); +} + +TEST_P(ColumnFamilyTest, ForwardIteratorCloseWALFile) { + SpecialEnv env(Env::Default()); + // Allow both of flush and purge job to schedule. + env.SetBackgroundThreads(2, Env::HIGH); + db_options_.env = &env; + db_options_.max_background_flushes = 1; + column_family_options_.memtable_factory.reset( + test::NewSpecialSkipListFactory(3)); + column_family_options_.level0_file_num_compaction_trigger = 2; + Open(); + CreateColumnFamilies({"one"}); + ASSERT_OK(Put(1, "fodor", "mirko")); + ASSERT_OK(Put(1, "fodar2", "mirko")); + ASSERT_OK(Flush(1)); + + // Create an iterator holding the current super version, as well as + // the SST file just flushed. + ReadOptions ro; + ro.tailing = true; + ro.background_purge_on_iterator_cleanup = true; + Iterator* it = db_->NewIterator(ro, handles_[1]); + // A flush will make `it` hold the last reference of its super version. + + ASSERT_OK(Put(1, "fodor", "mirko")); + ASSERT_OK(Put(1, "fodar2", "mirko")); + ASSERT_OK(Flush(1)); + + WaitForCompaction(); + + ASSERT_OK(Put(1, "fodor", "mirko")); + ASSERT_OK(Put(1, "fodor", "mirko")); + ASSERT_OK(Put(0, "fodor", "mirko")); + ASSERT_OK(Put(1, "fodor", "mirko")); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"ColumnFamilyTest::IteratorCloseWALFile2:0", + "DBImpl::BGWorkPurge:start"}, + {"ColumnFamilyTest::IteratorCloseWALFile2:2", + "DBImpl::BackgroundCallFlush:start"}, + {"DBImpl::BGWorkPurge:end", "ColumnFamilyTest::IteratorCloseWALFile2:1"}, + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + WriteOptions wo; + wo.sync = true; + ASSERT_OK(db_->Put(wo, handles_[1], "fodor", "mirko")); + + env.delete_count_.store(0); + ASSERT_EQ(2, env.num_open_wal_file_.load()); + // Deleting the iterator will clear its super version, triggering + // closing all files + it->Seek(""); + ASSERT_OK(it->status()); + + ASSERT_EQ(2, env.num_open_wal_file_.load()); + ASSERT_EQ(0, env.delete_count_.load()); + + TEST_SYNC_POINT("ColumnFamilyTest::IteratorCloseWALFile2:0"); + TEST_SYNC_POINT("ColumnFamilyTest::IteratorCloseWALFile2:1"); + ASSERT_EQ(1, env.num_open_wal_file_.load()); + ASSERT_EQ(1, env.delete_count_.load()); + TEST_SYNC_POINT("ColumnFamilyTest::IteratorCloseWALFile2:2"); + WaitForFlush(1); + ASSERT_EQ(1, env.num_open_wal_file_.load()); + ASSERT_EQ(1, env.delete_count_.load()); + + delete it; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + Reopen(); + ASSERT_EQ("mirko", Get(0, "fodor")); + ASSERT_EQ("mirko", Get(1, "fodor")); + db_options_.env = env_; + Close(); +} + +// Disable on windows because SyncWAL requires env->IsSyncThreadSafe() +// to return true which is not so in unbuffered mode. +#ifndef OS_WIN +TEST_P(ColumnFamilyTest, LogSyncConflictFlush) { + Open(); + CreateColumnFamiliesAndReopen({"one", "two"}); + + ASSERT_OK(Put(0, "", "")); + ASSERT_OK(Put(1, "foo", "bar")); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::SyncWAL:BeforeMarkLogsSynced:1", + "ColumnFamilyTest::LogSyncConflictFlush:1"}, + {"ColumnFamilyTest::LogSyncConflictFlush:2", + "DBImpl::SyncWAL:BeforeMarkLogsSynced:2"}}); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ROCKSDB_NAMESPACE::port::Thread thread([&] { ASSERT_OK(db_->SyncWAL()); }); + + TEST_SYNC_POINT("ColumnFamilyTest::LogSyncConflictFlush:1"); + ASSERT_OK(Flush(1)); + ASSERT_OK(Put(1, "foo", "bar")); + ASSERT_OK(Flush(1)); + + TEST_SYNC_POINT("ColumnFamilyTest::LogSyncConflictFlush:2"); + + thread.join(); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + Close(); +} +#endif + +// this test is placed here, because the infrastructure for Column Family +// test is being used to ensure a roll of wal files. +// Basic idea is to test that WAL truncation is being detected and not +// ignored +TEST_P(ColumnFamilyTest, DISABLED_LogTruncationTest) { + Open(); + CreateColumnFamiliesAndReopen({"one", "two"}); + + Build(0, 100); + + // Flush the 0th column family to force a roll of the wal log + ASSERT_OK(Flush(0)); + + // Add some more entries + Build(100, 100); + + std::vector filenames; + ASSERT_OK(env_->GetChildren(dbname_, &filenames)); + + // collect wal files + std::vector logfs; + for (size_t i = 0; i < filenames.size(); i++) { + uint64_t number; + FileType type; + if (!(ParseFileName(filenames[i], &number, &type))) continue; + + if (type != kWalFile) continue; + + logfs.push_back(filenames[i]); + } + + std::sort(logfs.begin(), logfs.end()); + ASSERT_GE(logfs.size(), 2); + + // Take the last but one file, and truncate it + std::string fpath = dbname_ + "/" + logfs[logfs.size() - 2]; + std::vector names_save = names_; + + uint64_t fsize; + ASSERT_OK(env_->GetFileSize(fpath, &fsize)); + ASSERT_GT(fsize, 0); + + Close(); + + std::string backup_logs = dbname_ + "/backup_logs"; + std::string t_fpath = backup_logs + "/" + logfs[logfs.size() - 2]; + + ASSERT_OK(env_->CreateDirIfMissing(backup_logs)); + // Not sure how easy it is to make this data driven. + // need to read back the WAL file and truncate last 10 + // entries + CopyFile(fpath, t_fpath, fsize - 9180); + + ASSERT_OK(env_->DeleteFile(fpath)); + ASSERT_OK(env_->RenameFile(t_fpath, fpath)); + + db_options_.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery; + + OpenReadOnly(names_save); + + CheckMissed(); + + Close(); + + Open(names_save); + + CheckMissed(); + + Close(); + + // cleanup + ASSERT_OK(env_->DeleteDir(backup_logs)); +} + +TEST_P(ColumnFamilyTest, DefaultCfPathsTest) { + Open(); + // Leave cf_paths for one column families to be empty. + // Files should be generated according to db_paths for that + // column family. + ColumnFamilyOptions cf_opt1, cf_opt2; + cf_opt1.cf_paths.emplace_back(dbname_ + "_one_1", + std::numeric_limits::max()); + CreateColumnFamilies({"one", "two"}, {cf_opt1, cf_opt2}); + Reopen({ColumnFamilyOptions(), cf_opt1, cf_opt2}); + + // Fill Column family 1. + PutRandomData(1, 100, 100); + ASSERT_OK(Flush(1)); + + ASSERT_EQ(1, GetSstFileCount(cf_opt1.cf_paths[0].path)); + ASSERT_EQ(0, GetSstFileCount(dbname_)); + + // Fill column family 2 + PutRandomData(2, 100, 100); + ASSERT_OK(Flush(2)); + + // SST from Column family 2 should be generated in + // db_paths which is dbname_ in this case. + ASSERT_EQ(1, GetSstFileCount(dbname_)); +} + +TEST_P(ColumnFamilyTest, MultipleCFPathsTest) { + Open(); + // Configure Column family specific paths. + ColumnFamilyOptions cf_opt1, cf_opt2; + cf_opt1.cf_paths.emplace_back(dbname_ + "_one_1", + std::numeric_limits::max()); + cf_opt2.cf_paths.emplace_back(dbname_ + "_two_1", + std::numeric_limits::max()); + CreateColumnFamilies({"one", "two"}, {cf_opt1, cf_opt2}); + Reopen({ColumnFamilyOptions(), cf_opt1, cf_opt2}); + + PutRandomData(1, 100, 100, true /* save */); + ASSERT_OK(Flush(1)); + + // Check that files are generated in appropriate paths. + ASSERT_EQ(1, GetSstFileCount(cf_opt1.cf_paths[0].path)); + ASSERT_EQ(0, GetSstFileCount(dbname_)); + + PutRandomData(2, 100, 100, true /* save */); + ASSERT_OK(Flush(2)); + + ASSERT_EQ(1, GetSstFileCount(cf_opt2.cf_paths[0].path)); + ASSERT_EQ(0, GetSstFileCount(dbname_)); + + // Re-open and verify the keys. + Reopen({ColumnFamilyOptions(), cf_opt1, cf_opt2}); + DBImpl* dbi = static_cast_with_check(db_); + for (int cf = 1; cf != 3; ++cf) { + ReadOptions read_options; + read_options.readahead_size = 0; + auto it = dbi->NewIterator(read_options, handles_[cf]); + for (it->SeekToFirst(); it->Valid(); it->Next()) { + ASSERT_OK(it->status()); + Slice key(it->key()); + ASSERT_NE(keys_[cf].end(), keys_[cf].find(key.ToString())); + } + ASSERT_OK(it->status()); + delete it; + + for (const auto& key : keys_[cf]) { + ASSERT_NE("NOT_FOUND", Get(cf, key)); + } + } +} + +TEST(ColumnFamilyTest, ValidateBlobGCCutoff) { + DBOptions db_options; + + ColumnFamilyOptions cf_options; + cf_options.enable_blob_garbage_collection = true; + + cf_options.blob_garbage_collection_age_cutoff = -0.5; + ASSERT_TRUE(ColumnFamilyData::ValidateOptions(db_options, cf_options) + .IsInvalidArgument()); + + cf_options.blob_garbage_collection_age_cutoff = 0.0; + ASSERT_OK(ColumnFamilyData::ValidateOptions(db_options, cf_options)); + + cf_options.blob_garbage_collection_age_cutoff = 0.5; + ASSERT_OK(ColumnFamilyData::ValidateOptions(db_options, cf_options)); + + cf_options.blob_garbage_collection_age_cutoff = 1.0; + ASSERT_OK(ColumnFamilyData::ValidateOptions(db_options, cf_options)); + + cf_options.blob_garbage_collection_age_cutoff = 1.5; + ASSERT_TRUE(ColumnFamilyData::ValidateOptions(db_options, cf_options) + .IsInvalidArgument()); +} + +TEST(ColumnFamilyTest, ValidateBlobGCForceThreshold) { + DBOptions db_options; + + ColumnFamilyOptions cf_options; + cf_options.enable_blob_garbage_collection = true; + + cf_options.blob_garbage_collection_force_threshold = -0.5; + ASSERT_TRUE(ColumnFamilyData::ValidateOptions(db_options, cf_options) + .IsInvalidArgument()); + + cf_options.blob_garbage_collection_force_threshold = 0.0; + ASSERT_OK(ColumnFamilyData::ValidateOptions(db_options, cf_options)); + + cf_options.blob_garbage_collection_force_threshold = 0.5; + ASSERT_OK(ColumnFamilyData::ValidateOptions(db_options, cf_options)); + + cf_options.blob_garbage_collection_force_threshold = 1.0; + ASSERT_OK(ColumnFamilyData::ValidateOptions(db_options, cf_options)); + + cf_options.blob_garbage_collection_force_threshold = 1.5; + ASSERT_TRUE(ColumnFamilyData::ValidateOptions(db_options, cf_options) + .IsInvalidArgument()); +} + +TEST(ColumnFamilyTest, ValidateMemtableKVChecksumOption) { + DBOptions db_options; + + ColumnFamilyOptions cf_options; + ASSERT_OK(ColumnFamilyData::ValidateOptions(db_options, cf_options)); + + cf_options.memtable_protection_bytes_per_key = 5; + ASSERT_TRUE(ColumnFamilyData::ValidateOptions(db_options, cf_options) + .IsNotSupported()); + + cf_options.memtable_protection_bytes_per_key = 1; + ASSERT_OK(ColumnFamilyData::ValidateOptions(db_options, cf_options)); + + cf_options.memtable_protection_bytes_per_key = 16; + ASSERT_TRUE(ColumnFamilyData::ValidateOptions(db_options, cf_options) + .IsNotSupported()); + + cf_options.memtable_protection_bytes_per_key = 0; + ASSERT_OK(ColumnFamilyData::ValidateOptions(db_options, cf_options)); +} + +// Tests the flushing behavior of a column family to retain user-defined +// timestamp when `persist_user_defined_timestamp` is false. +class ColumnFamilyRetainUDTTest : public ColumnFamilyTestBase { + public: + ColumnFamilyRetainUDTTest() : ColumnFamilyTestBase(kLatestFormatVersion) {} + + void SetUp() override { + db_options_.allow_concurrent_memtable_write = false; + column_family_options_.comparator = + test::BytewiseComparatorWithU64TsWrapper(); + column_family_options_.persist_user_defined_timestamps = false; + ColumnFamilyTestBase::SetUp(); + } + + Status Put(int cf, const std::string& key, const std::string& ts, + const std::string& value) { + return db_->Put(WriteOptions(), handles_[cf], Slice(key), Slice(ts), + Slice(value)); + } +}; + +class TestTsComparator : public Comparator { + public: + TestTsComparator() : Comparator(8 /*ts_sz*/) {} + + int Compare(const ROCKSDB_NAMESPACE::Slice& /*a*/, + const ROCKSDB_NAMESPACE::Slice& /*b*/) const override { + return 0; + } + const char* Name() const override { return "TestTs"; } + void FindShortestSeparator( + std::string* /*start*/, + const ROCKSDB_NAMESPACE::Slice& /*limit*/) const override {} + void FindShortSuccessor(std::string* /*key*/) const override {} +}; + +TEST_F(ColumnFamilyRetainUDTTest, SanityCheck) { + Open(); + ColumnFamilyOptions cf_options; + cf_options.persist_user_defined_timestamps = false; + TestTsComparator test_comparator; + cf_options.comparator = &test_comparator; + ColumnFamilyHandle* handle; + // Not persisting user-defined timestamps feature only supports user-defined + // timestamps formatted as uint64_t. + ASSERT_TRUE( + db_->CreateColumnFamily(cf_options, "pikachu", &handle).IsNotSupported()); + + Destroy(); + // Not persisting user-defined timestamps feature doesn't work in combination + // with atomic flush. + db_options_.atomic_flush = true; + ASSERT_TRUE(TryOpen({"default"}).IsNotSupported()); + + // Not persisting user-defined timestamps feature doesn't work in combination + // with concurrent memtable write. + db_options_.atomic_flush = false; + db_options_.allow_concurrent_memtable_write = true; + ASSERT_TRUE(TryOpen({"default"}).IsNotSupported()); + Close(); +} + +TEST_F(ColumnFamilyRetainUDTTest, FullHistoryTsLowNotSet) { + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundFlush:CheckFlushRequest:cb", [&](void* arg) { + ASSERT_NE(nullptr, arg); + auto reschedule_count = *static_cast(arg); + ASSERT_EQ(1, reschedule_count); + }); + + SyncPoint::GetInstance()->EnableProcessing(); + Open(); + std::string write_ts; + PutFixed64(&write_ts, 1); + ASSERT_OK(Put(0, "foo", write_ts, "v1")); + // No `full_history_ts_low` explicitly set by user, flush is continued + // without checking if its UDTs expired. + ASSERT_OK(Flush(0)); + + // After flush, `full_history_ts_low` should be automatically advanced to + // the effective cutoff timestamp: write_ts + 1 + std::string cutoff_ts; + PutFixed64(&cutoff_ts, 2); + std::string effective_full_history_ts_low; + ASSERT_OK( + db_->GetFullHistoryTsLow(handles_[0], &effective_full_history_ts_low)); + ASSERT_EQ(cutoff_ts, effective_full_history_ts_low); + Close(); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(ColumnFamilyRetainUDTTest, AllKeysExpired) { + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundFlush:CheckFlushRequest:cb", [&](void* arg) { + ASSERT_NE(nullptr, arg); + auto reschedule_count = *static_cast(arg); + ASSERT_EQ(1, reschedule_count); + }); + + SyncPoint::GetInstance()->EnableProcessing(); + Open(); + std::string write_ts; + PutFixed64(&write_ts, 1); + ASSERT_OK(Put(0, "foo", write_ts, "v1")); + std::string cutoff_ts; + PutFixed64(&cutoff_ts, 3); + ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], cutoff_ts)); + // All keys expired w.r.t the configured `full_history_ts_low`, flush continue + // without the need for a re-schedule. + ASSERT_OK(Flush(0)); + + // `full_history_ts_low` stays unchanged after flush. + std::string effective_full_history_ts_low; + ASSERT_OK( + db_->GetFullHistoryTsLow(handles_[0], &effective_full_history_ts_low)); + ASSERT_EQ(cutoff_ts, effective_full_history_ts_low); + Close(); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} +TEST_F(ColumnFamilyRetainUDTTest, NotAllKeysExpiredFlushToAvoidWriteStall) { + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundFlush:CheckFlushRequest:cb", [&](void* arg) { + ASSERT_NE(nullptr, arg); + auto reschedule_count = *static_cast(arg); + ASSERT_EQ(1, reschedule_count); + }); + + SyncPoint::GetInstance()->EnableProcessing(); + Open(); + std::string cutoff_ts; + std::string write_ts; + PutFixed64(&write_ts, 1); + ASSERT_OK(Put(0, "foo", write_ts, "v1")); + PutFixed64(&cutoff_ts, 1); + ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], cutoff_ts)); + ASSERT_OK(db_->SetOptions(handles_[0], {{"max_write_buffer_number", "1"}})); + // Not all keys expired, but flush is continued without a re-schedule because + // of risk of write stall. + ASSERT_OK(Flush(0)); + + // After flush, `full_history_ts_low` should be automatically advanced to + // the effective cutoff timestamp: write_ts + 1 + std::string effective_full_history_ts_low; + ASSERT_OK( + db_->GetFullHistoryTsLow(handles_[0], &effective_full_history_ts_low)); + + cutoff_ts.clear(); + PutFixed64(&cutoff_ts, 2); + ASSERT_EQ(cutoff_ts, effective_full_history_ts_low); + Close(); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(ColumnFamilyRetainUDTTest, NotAllKeysExpiredFlushRescheduled) { + std::string cutoff_ts; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::AfterRetainUDTReschedule:cb", [&](void* /*arg*/) { + // Increasing full_history_ts_low so all keys expired after the initial + // FlushRequest is rescheduled + cutoff_ts.clear(); + PutFixed64(&cutoff_ts, 3); + ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], cutoff_ts)); + }); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundFlush:CheckFlushRequest:cb", [&](void* arg) { + ASSERT_NE(nullptr, arg); + auto reschedule_count = *static_cast(arg); + ASSERT_EQ(2, reschedule_count); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + Open(); + std::string write_ts; + PutFixed64(&write_ts, 1); + ASSERT_OK(Put(0, "foo", write_ts, "v1")); + PutFixed64(&cutoff_ts, 1); + ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], cutoff_ts)); + // Not all keys expired, and there is no risk of write stall. Flush is + // rescheduled. The actual flush happens after `full_history_ts_low` is + // increased to mark all keys expired. + ASSERT_OK(Flush(0)); + + std::string effective_full_history_ts_low; + ASSERT_OK( + db_->GetFullHistoryTsLow(handles_[0], &effective_full_history_ts_low)); + // `full_history_ts_low` stays unchanged. + ASSERT_EQ(cutoff_ts, effective_full_history_ts_low); + Close(); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/compact_files_test.cc b/librocksdb-sys/rocksdb/db/compact_files_test.cc new file mode 100644 index 0000000..5ca80d9 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compact_files_test.cc @@ -0,0 +1,495 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + + +#include +#include +#include +#include + +#include "db/db_impl/db_impl.h" +#include "port/port.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "test_util/sync_point.h" +#include "test_util/testharness.h" +#include "util/cast_util.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +class CompactFilesTest : public testing::Test { + public: + CompactFilesTest() { + env_ = Env::Default(); + db_name_ = test::PerThreadDBPath("compact_files_test"); + } + + std::string db_name_; + Env* env_; +}; + +// A class which remembers the name of each flushed file. +class FlushedFileCollector : public EventListener { + public: + FlushedFileCollector() {} + ~FlushedFileCollector() override {} + + void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& info) override { + std::lock_guard lock(mutex_); + flushed_files_.push_back(info.file_path); + } + + std::vector GetFlushedFiles() { + std::lock_guard lock(mutex_); + std::vector result; + for (auto fname : flushed_files_) { + result.push_back(fname); + } + return result; + } + void ClearFlushedFiles() { + std::lock_guard lock(mutex_); + flushed_files_.clear(); + } + + private: + std::vector flushed_files_; + std::mutex mutex_; +}; + +TEST_F(CompactFilesTest, L0ConflictsFiles) { + Options options; + // to trigger compaction more easily + const int kWriteBufferSize = 10000; + const int kLevel0Trigger = 2; + options.create_if_missing = true; + options.level_compaction_dynamic_level_bytes = false; + options.compaction_style = kCompactionStyleLevel; + // Small slowdown and stop trigger for experimental purpose. + options.level0_slowdown_writes_trigger = 20; + options.level0_stop_writes_trigger = 20; + options.level0_stop_writes_trigger = 20; + options.write_buffer_size = kWriteBufferSize; + options.level0_file_num_compaction_trigger = kLevel0Trigger; + options.compression = kNoCompression; + + DB* db = nullptr; + ASSERT_OK(DestroyDB(db_name_, options)); + Status s = DB::Open(options, db_name_, &db); + assert(s.ok()); + assert(db); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"CompactFilesImpl:0", "BackgroundCallCompaction:0"}, + {"BackgroundCallCompaction:1", "CompactFilesImpl:1"}, + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // create couple files + // Background compaction starts and waits in BackgroundCallCompaction:0 + for (int i = 0; i < kLevel0Trigger * 4; ++i) { + ASSERT_OK(db->Put(WriteOptions(), std::to_string(i), "")); + ASSERT_OK(db->Put(WriteOptions(), std::to_string(100 - i), "")); + ASSERT_OK(db->Flush(FlushOptions())); + } + + ROCKSDB_NAMESPACE::ColumnFamilyMetaData meta; + db->GetColumnFamilyMetaData(&meta); + std::string file1; + for (auto& file : meta.levels[0].files) { + ASSERT_EQ(0, meta.levels[0].level); + if (file1 == "") { + file1 = file.db_path + "/" + file.name; + } else { + std::string file2 = file.db_path + "/" + file.name; + // Another thread starts a compact files and creates an L0 compaction + // The background compaction then notices that there is an L0 compaction + // already in progress and doesn't do an L0 compaction + // Once the background compaction finishes, the compact files finishes + ASSERT_OK(db->CompactFiles(ROCKSDB_NAMESPACE::CompactionOptions(), + {file1, file2}, 0)); + break; + } + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + delete db; +} + +TEST_F(CompactFilesTest, MultipleLevel) { + Options options; + options.create_if_missing = true; + // Otherwise background compaction can happen to + // drain unnecessary level + options.level_compaction_dynamic_level_bytes = false; + options.num_levels = 6; + // Add listener + FlushedFileCollector* collector = new FlushedFileCollector(); + options.listeners.emplace_back(collector); + + DB* db = nullptr; + ASSERT_OK(DestroyDB(db_name_, options)); + Status s = DB::Open(options, db_name_, &db); + ASSERT_OK(s); + ASSERT_NE(db, nullptr); + + // create couple files in L0, L3, L4 and L5 + for (int i = 5; i > 2; --i) { + collector->ClearFlushedFiles(); + ASSERT_OK(db->Put(WriteOptions(), std::to_string(i), "")); + ASSERT_OK(db->Flush(FlushOptions())); + // Ensure background work is fully finished including listener callbacks + // before accessing listener state. + ASSERT_OK(static_cast_with_check(db)->TEST_WaitForBackgroundWork()); + auto l0_files = collector->GetFlushedFiles(); + ASSERT_OK(db->CompactFiles(CompactionOptions(), l0_files, i)); + + std::string prop; + ASSERT_TRUE(db->GetProperty( + "rocksdb.num-files-at-level" + std::to_string(i), &prop)); + ASSERT_EQ("1", prop); + } + ASSERT_OK(db->Put(WriteOptions(), std::to_string(0), "")); + ASSERT_OK(db->Flush(FlushOptions())); + + ColumnFamilyMetaData meta; + db->GetColumnFamilyMetaData(&meta); + // Compact files except the file in L3 + std::vector files; + for (int i = 0; i < 6; ++i) { + if (i == 3) continue; + for (auto& file : meta.levels[i].files) { + files.push_back(file.db_path + "/" + file.name); + } + } + + SyncPoint::GetInstance()->LoadDependency({ + {"CompactionJob::Run():Start", "CompactFilesTest.MultipleLevel:0"}, + {"CompactFilesTest.MultipleLevel:1", "CompactFilesImpl:3"}, + }); + SyncPoint::GetInstance()->EnableProcessing(); + + std::thread thread([&] { + TEST_SYNC_POINT("CompactFilesTest.MultipleLevel:0"); + ASSERT_OK(db->Put(WriteOptions(), "bar", "v2")); + ASSERT_OK(db->Put(WriteOptions(), "foo", "v2")); + ASSERT_OK(db->Flush(FlushOptions())); + TEST_SYNC_POINT("CompactFilesTest.MultipleLevel:1"); + }); + + // Compaction cannot move up the data to higher level + // here we have input file from level 5, so the output level has to be >= 5 + for (int invalid_output_level = 0; invalid_output_level < 5; + invalid_output_level++) { + s = db->CompactFiles(CompactionOptions(), files, invalid_output_level); + ASSERT_TRUE(s.IsInvalidArgument()); + } + + ASSERT_OK(db->CompactFiles(CompactionOptions(), files, 5)); + SyncPoint::GetInstance()->DisableProcessing(); + thread.join(); + + delete db; +} + +TEST_F(CompactFilesTest, ObsoleteFiles) { + Options options; + // to trigger compaction more easily + const int kWriteBufferSize = 65536; + options.create_if_missing = true; + // Disable RocksDB background compaction. + options.compaction_style = kCompactionStyleNone; + options.level0_slowdown_writes_trigger = (1 << 30); + options.level0_stop_writes_trigger = (1 << 30); + options.write_buffer_size = kWriteBufferSize; + options.max_write_buffer_number = 2; + options.compression = kNoCompression; + + // Add listener + FlushedFileCollector* collector = new FlushedFileCollector(); + options.listeners.emplace_back(collector); + + DB* db = nullptr; + ASSERT_OK(DestroyDB(db_name_, options)); + Status s = DB::Open(options, db_name_, &db); + ASSERT_OK(s); + ASSERT_NE(db, nullptr); + + // create couple files + for (int i = 1000; i < 2000; ++i) { + ASSERT_OK(db->Put(WriteOptions(), std::to_string(i), + std::string(kWriteBufferSize / 10, 'a' + (i % 26)))); + } + + auto l0_files = collector->GetFlushedFiles(); + ASSERT_OK(db->CompactFiles(CompactionOptions(), l0_files, 1)); + ASSERT_OK(static_cast_with_check(db)->TEST_WaitForCompact()); + + // verify all compaction input files are deleted + for (auto fname : l0_files) { + ASSERT_EQ(Status::NotFound(), env_->FileExists(fname)); + } + delete db; +} + +TEST_F(CompactFilesTest, NotCutOutputOnLevel0) { + Options options; + options.create_if_missing = true; + // Disable RocksDB background compaction. + options.compaction_style = kCompactionStyleNone; + options.level0_slowdown_writes_trigger = 1000; + options.level0_stop_writes_trigger = 1000; + options.write_buffer_size = 65536; + options.max_write_buffer_number = 2; + options.compression = kNoCompression; + options.max_compaction_bytes = 5000; + + // Add listener + FlushedFileCollector* collector = new FlushedFileCollector(); + options.listeners.emplace_back(collector); + + DB* db = nullptr; + ASSERT_OK(DestroyDB(db_name_, options)); + Status s = DB::Open(options, db_name_, &db); + assert(s.ok()); + assert(db); + + // create couple files + for (int i = 0; i < 500; ++i) { + ASSERT_OK(db->Put(WriteOptions(), std::to_string(i), + std::string(1000, 'a' + (i % 26)))); + } + ASSERT_OK(static_cast_with_check(db)->TEST_WaitForFlushMemTable()); + auto l0_files_1 = collector->GetFlushedFiles(); + collector->ClearFlushedFiles(); + for (int i = 0; i < 500; ++i) { + ASSERT_OK(db->Put(WriteOptions(), std::to_string(i), + std::string(1000, 'a' + (i % 26)))); + } + ASSERT_OK(static_cast_with_check(db)->TEST_WaitForFlushMemTable()); + auto l0_files_2 = collector->GetFlushedFiles(); + ASSERT_OK(db->CompactFiles(CompactionOptions(), l0_files_1, 0)); + ASSERT_OK(db->CompactFiles(CompactionOptions(), l0_files_2, 0)); + // no assertion failure + delete db; +} + +TEST_F(CompactFilesTest, CapturingPendingFiles) { + Options options; + options.create_if_missing = true; + // Disable RocksDB background compaction. + options.compaction_style = kCompactionStyleNone; + // Always do full scans for obsolete files (needed to reproduce the issue). + options.delete_obsolete_files_period_micros = 0; + + // Add listener. + FlushedFileCollector* collector = new FlushedFileCollector(); + options.listeners.emplace_back(collector); + + DB* db = nullptr; + ASSERT_OK(DestroyDB(db_name_, options)); + Status s = DB::Open(options, db_name_, &db); + ASSERT_OK(s); + assert(db); + + // Create 5 files. + for (int i = 0; i < 5; ++i) { + ASSERT_OK(db->Put(WriteOptions(), "key" + std::to_string(i), "value")); + ASSERT_OK(db->Flush(FlushOptions())); + } + + // Ensure background work is fully finished including listener callbacks + // before accessing listener state. + ASSERT_OK(static_cast_with_check(db)->TEST_WaitForBackgroundWork()); + auto l0_files = collector->GetFlushedFiles(); + EXPECT_EQ(5, l0_files.size()); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"CompactFilesImpl:2", "CompactFilesTest.CapturingPendingFiles:0"}, + {"CompactFilesTest.CapturingPendingFiles:1", "CompactFilesImpl:3"}, + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Start compacting files. + ROCKSDB_NAMESPACE::port::Thread compaction_thread( + [&] { EXPECT_OK(db->CompactFiles(CompactionOptions(), l0_files, 1)); }); + + // In the meantime flush another file. + TEST_SYNC_POINT("CompactFilesTest.CapturingPendingFiles:0"); + ASSERT_OK(db->Put(WriteOptions(), "key5", "value")); + ASSERT_OK(db->Flush(FlushOptions())); + TEST_SYNC_POINT("CompactFilesTest.CapturingPendingFiles:1"); + + compaction_thread.join(); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + delete db; + + // Make sure we can reopen the DB. + s = DB::Open(options, db_name_, &db); + ASSERT_OK(s); + assert(db); + delete db; +} + +TEST_F(CompactFilesTest, CompactionFilterWithGetSv) { + class FilterWithGet : public CompactionFilter { + public: + bool Filter(int /*level*/, const Slice& /*key*/, const Slice& /*value*/, + std::string* /*new_value*/, + bool* /*value_changed*/) const override { + if (db_ == nullptr) { + return true; + } + std::string res; + db_->Get(ReadOptions(), "", &res); + return true; + } + + void SetDB(DB* db) { db_ = db; } + + const char* Name() const override { return "FilterWithGet"; } + + private: + DB* db_; + }; + + std::shared_ptr cf(new FilterWithGet()); + + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.create_if_missing = true; + options.compaction_filter = cf.get(); + + DB* db = nullptr; + ASSERT_OK(DestroyDB(db_name_, options)); + Status s = DB::Open(options, db_name_, &db); + ASSERT_OK(s); + + cf->SetDB(db); + + // Write one L0 file + ASSERT_OK(db->Put(WriteOptions(), "K1", "V1")); + ASSERT_OK(db->Flush(FlushOptions())); + + // Compact all L0 files using CompactFiles + ROCKSDB_NAMESPACE::ColumnFamilyMetaData meta; + db->GetColumnFamilyMetaData(&meta); + for (auto& file : meta.levels[0].files) { + std::string fname = file.db_path + "/" + file.name; + ASSERT_OK( + db->CompactFiles(ROCKSDB_NAMESPACE::CompactionOptions(), {fname}, 0)); + } + + delete db; +} + +TEST_F(CompactFilesTest, SentinelCompressionType) { + if (!Zlib_Supported()) { + fprintf(stderr, "zlib compression not supported, skip this test\n"); + return; + } + if (!Snappy_Supported()) { + fprintf(stderr, "snappy compression not supported, skip this test\n"); + return; + } + // Check that passing `CompressionType::kDisableCompressionOption` to + // `CompactFiles` causes it to use the column family compression options. + for (auto compaction_style : {CompactionStyle::kCompactionStyleLevel, + CompactionStyle::kCompactionStyleUniversal, + CompactionStyle::kCompactionStyleNone}) { + ASSERT_OK(DestroyDB(db_name_, Options())); + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.compaction_style = compaction_style; + // L0: Snappy, L1: ZSTD, L2: Snappy + options.compression_per_level = {CompressionType::kSnappyCompression, + CompressionType::kZlibCompression, + CompressionType::kSnappyCompression}; + options.create_if_missing = true; + FlushedFileCollector* collector = new FlushedFileCollector(); + options.listeners.emplace_back(collector); + DB* db = nullptr; + ASSERT_OK(DB::Open(options, db_name_, &db)); + + ASSERT_OK(db->Put(WriteOptions(), "key", "val")); + ASSERT_OK(db->Flush(FlushOptions())); + + // Ensure background work is fully finished including listener callbacks + // before accessing listener state. + ASSERT_OK(static_cast_with_check(db)->TEST_WaitForBackgroundWork()); + auto l0_files = collector->GetFlushedFiles(); + ASSERT_EQ(1, l0_files.size()); + + // L0->L1 compaction, so output should be ZSTD-compressed + CompactionOptions compaction_opts; + compaction_opts.compression = CompressionType::kDisableCompressionOption; + ASSERT_OK(db->CompactFiles(compaction_opts, l0_files, 1)); + + ROCKSDB_NAMESPACE::TablePropertiesCollection all_tables_props; + ASSERT_OK(db->GetPropertiesOfAllTables(&all_tables_props)); + for (const auto& name_and_table_props : all_tables_props) { + ASSERT_EQ(CompressionTypeToString(CompressionType::kZlibCompression), + name_and_table_props.second->compression_name); + } + delete db; + } +} + +TEST_F(CompactFilesTest, GetCompactionJobInfo) { + Options options; + options.create_if_missing = true; + // Disable RocksDB background compaction. + options.compaction_style = kCompactionStyleNone; + options.level0_slowdown_writes_trigger = 1000; + options.level0_stop_writes_trigger = 1000; + options.write_buffer_size = 65536; + options.max_write_buffer_number = 2; + options.compression = kNoCompression; + options.max_compaction_bytes = 5000; + + // Add listener + FlushedFileCollector* collector = new FlushedFileCollector(); + options.listeners.emplace_back(collector); + + DB* db = nullptr; + ASSERT_OK(DestroyDB(db_name_, options)); + Status s = DB::Open(options, db_name_, &db); + ASSERT_OK(s); + assert(db); + + // create couple files + for (int i = 0; i < 500; ++i) { + ASSERT_OK(db->Put(WriteOptions(), std::to_string(i), + std::string(1000, 'a' + (i % 26)))); + } + ASSERT_OK(static_cast_with_check(db)->TEST_WaitForFlushMemTable()); + auto l0_files_1 = collector->GetFlushedFiles(); + CompactionOptions co; + co.compression = CompressionType::kLZ4Compression; + CompactionJobInfo compaction_job_info{}; + ASSERT_OK( + db->CompactFiles(co, l0_files_1, 0, -1, nullptr, &compaction_job_info)); + ASSERT_EQ(compaction_job_info.base_input_level, 0); + ASSERT_EQ(compaction_job_info.cf_id, db->DefaultColumnFamily()->GetID()); + ASSERT_EQ(compaction_job_info.cf_name, db->DefaultColumnFamily()->GetName()); + ASSERT_EQ(compaction_job_info.compaction_reason, + CompactionReason::kManualCompaction); + ASSERT_EQ(compaction_job_info.compression, CompressionType::kLZ4Compression); + ASSERT_EQ(compaction_job_info.output_level, 0); + ASSERT_OK(compaction_job_info.status); + // no assertion failure + delete db; +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/librocksdb-sys/rocksdb/db/compaction/clipping_iterator.h b/librocksdb-sys/rocksdb/db/compaction/clipping_iterator.h new file mode 100644 index 0000000..3f50cdd --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/clipping_iterator.h @@ -0,0 +1,281 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "rocksdb/comparator.h" +#include "table/internal_iterator.h" + +namespace ROCKSDB_NAMESPACE { + +// An internal iterator that wraps another one and ensures that any keys +// returned are strictly within a range [start, end). If the underlying +// iterator has already performed the bounds checking, it relies on that result; +// otherwise, it performs the necessary key comparisons itself. Both bounds +// are optional. +class ClippingIterator : public InternalIterator { + public: + ClippingIterator(InternalIterator* iter, const Slice* start, const Slice* end, + const CompareInterface* cmp) + : iter_(iter), start_(start), end_(end), cmp_(cmp), valid_(false) { + assert(iter_); + assert(cmp_); + assert(!start_ || !end_ || cmp_->Compare(*start_, *end_) <= 0); + + UpdateAndEnforceBounds(); + } + + bool Valid() const override { return valid_; } + + void SeekToFirst() override { + if (start_) { + iter_->Seek(*start_); + } else { + iter_->SeekToFirst(); + } + + UpdateAndEnforceUpperBound(); + } + + void SeekToLast() override { + if (end_) { + iter_->SeekForPrev(*end_); + + // Upper bound is exclusive, so we need a key which is strictly smaller + if (iter_->Valid() && cmp_->Compare(iter_->key(), *end_) == 0) { + iter_->Prev(); + } + } else { + iter_->SeekToLast(); + } + + UpdateAndEnforceLowerBound(); + } + + void Seek(const Slice& target) override { + if (start_ && cmp_->Compare(target, *start_) < 0) { + iter_->Seek(*start_); + UpdateAndEnforceUpperBound(); + return; + } + + if (end_ && cmp_->Compare(target, *end_) >= 0) { + valid_ = false; + return; + } + + iter_->Seek(target); + UpdateAndEnforceUpperBound(); + } + + void SeekForPrev(const Slice& target) override { + if (start_ && cmp_->Compare(target, *start_) < 0) { + valid_ = false; + return; + } + + if (end_ && cmp_->Compare(target, *end_) >= 0) { + iter_->SeekForPrev(*end_); + + // Upper bound is exclusive, so we need a key which is strictly smaller + if (iter_->Valid() && cmp_->Compare(iter_->key(), *end_) == 0) { + iter_->Prev(); + } + + UpdateAndEnforceLowerBound(); + return; + } + + iter_->SeekForPrev(target); + UpdateAndEnforceLowerBound(); + } + + void Next() override { + assert(valid_); + iter_->Next(); + UpdateAndEnforceUpperBound(); + } + + bool NextAndGetResult(IterateResult* result) override { + assert(valid_); + assert(result); + + IterateResult res; + valid_ = iter_->NextAndGetResult(&res); + + if (!valid_) { + return false; + } + + if (end_) { + EnforceUpperBoundImpl(res.bound_check_result); + + if (!valid_) { + return false; + } + } + + res.bound_check_result = IterBoundCheck::kInbound; + *result = res; + + return true; + } + + void Prev() override { + assert(valid_); + iter_->Prev(); + UpdateAndEnforceLowerBound(); + } + + Slice key() const override { + assert(valid_); + return iter_->key(); + } + + Slice user_key() const override { + assert(valid_); + return iter_->user_key(); + } + + Slice value() const override { + assert(valid_); + return iter_->value(); + } + + Status status() const override { return iter_->status(); } + + bool PrepareValue() override { + assert(valid_); + + if (iter_->PrepareValue()) { + return true; + } + + assert(!iter_->Valid()); + valid_ = false; + return false; + } + + bool MayBeOutOfLowerBound() override { + assert(valid_); + return false; + } + + IterBoundCheck UpperBoundCheckResult() override { + assert(valid_); + return IterBoundCheck::kInbound; + } + + void SetPinnedItersMgr(PinnedIteratorsManager* pinned_iters_mgr) override { + iter_->SetPinnedItersMgr(pinned_iters_mgr); + } + + bool IsKeyPinned() const override { + assert(valid_); + return iter_->IsKeyPinned(); + } + + bool IsValuePinned() const override { + assert(valid_); + return iter_->IsValuePinned(); + } + + Status GetProperty(std::string prop_name, std::string* prop) override { + return iter_->GetProperty(prop_name, prop); + } + + bool IsDeleteRangeSentinelKey() const override { + assert(valid_); + return iter_->IsDeleteRangeSentinelKey(); + } + + private: + void UpdateValid() { + assert(!iter_->Valid() || iter_->status().ok()); + + valid_ = iter_->Valid(); + } + + void EnforceUpperBoundImpl(IterBoundCheck bound_check_result) { + if (bound_check_result == IterBoundCheck::kInbound) { + return; + } + + if (bound_check_result == IterBoundCheck::kOutOfBound) { + valid_ = false; + return; + } + + assert(bound_check_result == IterBoundCheck::kUnknown); + + if (cmp_->Compare(key(), *end_) >= 0) { + valid_ = false; + } + } + + void EnforceUpperBound() { + if (!valid_) { + return; + } + + if (!end_) { + return; + } + + EnforceUpperBoundImpl(iter_->UpperBoundCheckResult()); + } + + void EnforceLowerBound() { + if (!valid_) { + return; + } + + if (!start_) { + return; + } + + if (!iter_->MayBeOutOfLowerBound()) { + return; + } + + if (cmp_->Compare(key(), *start_) < 0) { + valid_ = false; + } + } + + void AssertBounds() { + assert(!valid_ || !start_ || cmp_->Compare(key(), *start_) >= 0); + assert(!valid_ || !end_ || cmp_->Compare(key(), *end_) < 0); + } + + void UpdateAndEnforceBounds() { + UpdateValid(); + EnforceUpperBound(); + EnforceLowerBound(); + AssertBounds(); + } + + void UpdateAndEnforceUpperBound() { + UpdateValid(); + EnforceUpperBound(); + AssertBounds(); + } + + void UpdateAndEnforceLowerBound() { + UpdateValid(); + EnforceLowerBound(); + AssertBounds(); + } + + InternalIterator* iter_; + const Slice* start_; + const Slice* end_; + const CompareInterface* cmp_; + bool valid_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/clipping_iterator_test.cc b/librocksdb-sys/rocksdb/db/compaction/clipping_iterator_test.cc new file mode 100644 index 0000000..b2b1670 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/clipping_iterator_test.cc @@ -0,0 +1,259 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/compaction/clipping_iterator.h" + +#include +#include +#include +#include + +#include "db/dbformat.h" +#include "rocksdb/comparator.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/vector_iterator.h" + +namespace ROCKSDB_NAMESPACE { + +// A vector iterator which does its own bounds checking. This is for testing the +// optimizations in the clipping iterator where we bypass the bounds checking if +// the input iterator has already performed it. +class BoundsCheckingVectorIterator : public VectorIterator { + public: + BoundsCheckingVectorIterator(const std::vector& keys, + const std::vector& values, + const Slice* start, const Slice* end, + const Comparator* cmp) + : VectorIterator(keys, values, cmp), start_(start), end_(end), cmp_(cmp) { + assert(cmp_); + } + + bool NextAndGetResult(IterateResult* result) override { + assert(Valid()); + assert(result); + + Next(); + + if (!Valid()) { + return false; + } + + result->key = key(); + result->bound_check_result = UpperBoundCheckResult(); + result->value_prepared = true; + + return true; + } + + bool MayBeOutOfLowerBound() override { + assert(Valid()); + + if (!start_) { + return false; + } + + return cmp_->Compare(key(), *start_) < 0; + } + + IterBoundCheck UpperBoundCheckResult() override { + assert(Valid()); + + if (!end_) { + return IterBoundCheck::kInbound; + } + + return cmp_->Compare(key(), *end_) >= 0 ? IterBoundCheck::kOutOfBound + : IterBoundCheck::kInbound; + } + + private: + const Slice* start_; + const Slice* end_; + const Comparator* cmp_; +}; + +class ClippingIteratorTest + : public ::testing::Test, + public ::testing::WithParamInterface> {}; + +TEST_P(ClippingIteratorTest, Clip) { + const std::vector keys{"key0", "key1", "key2", "key3", "key4", + "key5", "key6", "key7", "key8", "key9"}; + const std::vector values{ + "unused0", "value1", "value2", "value3", "unused4", + "unused5", "unused6", "unused7", "unused8", "unused9"}; + + assert(keys.size() == values.size()); + + // Note: the input always contains key1, key2, and key3; however, the clipping + // window is based on the test parameters: its left edge is a value in the + // range [0, 4], and its size is a value in the range [0, 5] + const std::vector input_keys{keys[1], keys[2], keys[3]}; + const std::vector input_values{values[1], values[2], values[3]}; + + const bool use_bounds_checking_vec_it = std::get<0>(GetParam()); + + const size_t clip_start_idx = std::get<1>(GetParam()); + const size_t clip_window_size = std::get<2>(GetParam()); + const size_t clip_end_idx = clip_start_idx + clip_window_size; + + const Slice start(keys[clip_start_idx]); + const Slice end(keys[clip_end_idx]); + + std::unique_ptr input( + use_bounds_checking_vec_it + ? new BoundsCheckingVectorIterator(input_keys, input_values, &start, + &end, BytewiseComparator()) + : new VectorIterator(input_keys, input_values, BytewiseComparator())); + + ClippingIterator clip(input.get(), &start, &end, BytewiseComparator()); + + // The range the clipping iterator should return values from. This is + // essentially the intersection of the input range [1, 4) and the clipping + // window [clip_start_idx, clip_end_idx) + const size_t data_start_idx = + std::max(clip_start_idx, static_cast(1)); + const size_t data_end_idx = std::min(clip_end_idx, static_cast(4)); + + // Range is empty; all Seeks should fail + if (data_start_idx >= data_end_idx) { + clip.SeekToFirst(); + ASSERT_FALSE(clip.Valid()); + + clip.SeekToLast(); + ASSERT_FALSE(clip.Valid()); + + for (size_t i = 0; i < keys.size(); ++i) { + clip.Seek(keys[i]); + ASSERT_FALSE(clip.Valid()); + + clip.SeekForPrev(keys[i]); + ASSERT_FALSE(clip.Valid()); + } + + return; + } + + // Range is non-empty; call SeekToFirst and iterate forward + clip.SeekToFirst(); + ASSERT_TRUE(clip.Valid()); + ASSERT_EQ(clip.key(), keys[data_start_idx]); + ASSERT_EQ(clip.value(), values[data_start_idx]); + ASSERT_FALSE(clip.MayBeOutOfLowerBound()); + ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound); + + for (size_t i = data_start_idx + 1; i < data_end_idx; ++i) { + clip.Next(); + ASSERT_TRUE(clip.Valid()); + ASSERT_EQ(clip.key(), keys[i]); + ASSERT_EQ(clip.value(), values[i]); + ASSERT_FALSE(clip.MayBeOutOfLowerBound()); + ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound); + } + + clip.Next(); + ASSERT_FALSE(clip.Valid()); + + // Do it again using NextAndGetResult + clip.SeekToFirst(); + ASSERT_TRUE(clip.Valid()); + ASSERT_EQ(clip.key(), keys[data_start_idx]); + ASSERT_EQ(clip.value(), values[data_start_idx]); + ASSERT_FALSE(clip.MayBeOutOfLowerBound()); + ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound); + + for (size_t i = data_start_idx + 1; i < data_end_idx; ++i) { + IterateResult result; + ASSERT_TRUE(clip.NextAndGetResult(&result)); + ASSERT_EQ(result.key, keys[i]); + ASSERT_EQ(result.bound_check_result, IterBoundCheck::kInbound); + ASSERT_TRUE(clip.Valid()); + ASSERT_EQ(clip.key(), keys[i]); + ASSERT_EQ(clip.value(), values[i]); + ASSERT_FALSE(clip.MayBeOutOfLowerBound()); + ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound); + } + + IterateResult result; + ASSERT_FALSE(clip.NextAndGetResult(&result)); + ASSERT_FALSE(clip.Valid()); + + // Call SeekToLast and iterate backward + clip.SeekToLast(); + ASSERT_TRUE(clip.Valid()); + ASSERT_EQ(clip.key(), keys[data_end_idx - 1]); + ASSERT_EQ(clip.value(), values[data_end_idx - 1]); + ASSERT_FALSE(clip.MayBeOutOfLowerBound()); + ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound); + + for (size_t i = data_end_idx - 2; i >= data_start_idx; --i) { + clip.Prev(); + ASSERT_TRUE(clip.Valid()); + ASSERT_EQ(clip.key(), keys[i]); + ASSERT_EQ(clip.value(), values[i]); + ASSERT_FALSE(clip.MayBeOutOfLowerBound()); + ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound); + } + + clip.Prev(); + ASSERT_FALSE(clip.Valid()); + + // Call Seek/SeekForPrev for all keys; Seek should return the smallest key + // which is >= the target; SeekForPrev should return the largest key which is + // <= the target + for (size_t i = 0; i < keys.size(); ++i) { + clip.Seek(keys[i]); + + if (i < data_start_idx) { + ASSERT_TRUE(clip.Valid()); + ASSERT_EQ(clip.key(), keys[data_start_idx]); + ASSERT_EQ(clip.value(), values[data_start_idx]); + ASSERT_FALSE(clip.MayBeOutOfLowerBound()); + ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound); + } else if (i < data_end_idx) { + ASSERT_TRUE(clip.Valid()); + ASSERT_EQ(clip.key(), keys[i]); + ASSERT_EQ(clip.value(), values[i]); + ASSERT_FALSE(clip.MayBeOutOfLowerBound()); + ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound); + } else { + ASSERT_FALSE(clip.Valid()); + } + + clip.SeekForPrev(keys[i]); + + if (i < data_start_idx) { + ASSERT_FALSE(clip.Valid()); + } else if (i < data_end_idx) { + ASSERT_TRUE(clip.Valid()); + ASSERT_EQ(clip.key(), keys[i]); + ASSERT_EQ(clip.value(), values[i]); + ASSERT_FALSE(clip.MayBeOutOfLowerBound()); + ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound); + } else { + ASSERT_TRUE(clip.Valid()); + ASSERT_EQ(clip.key(), keys[data_end_idx - 1]); + ASSERT_EQ(clip.value(), values[data_end_idx - 1]); + ASSERT_FALSE(clip.MayBeOutOfLowerBound()); + ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound); + } + } +} + +INSTANTIATE_TEST_CASE_P( + ClippingIteratorTest, ClippingIteratorTest, + ::testing::Combine( + ::testing::Bool(), + ::testing::Range(static_cast(0), static_cast(5)), + ::testing::Range(static_cast(0), static_cast(6)))); + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction.cc b/librocksdb-sys/rocksdb/db/compaction/compaction.cc new file mode 100644 index 0000000..e28257d --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction.cc @@ -0,0 +1,927 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/compaction/compaction.h" + +#include +#include + +#include "db/column_family.h" +#include "logging/logging.h" +#include "rocksdb/compaction_filter.h" +#include "rocksdb/sst_partitioner.h" +#include "test_util/sync_point.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +const uint64_t kRangeTombstoneSentinel = + PackSequenceAndType(kMaxSequenceNumber, kTypeRangeDeletion); + +int sstableKeyCompare(const Comparator* uc, const Slice& a, const Slice& b) { + auto c = uc->CompareWithoutTimestamp(ExtractUserKey(a), ExtractUserKey(b)); + if (c != 0) { + return c; + } + auto a_footer = ExtractInternalKeyFooter(a); + auto b_footer = ExtractInternalKeyFooter(b); + if (a_footer == kRangeTombstoneSentinel) { + if (b_footer != kRangeTombstoneSentinel) { + return -1; + } + } else if (b_footer == kRangeTombstoneSentinel) { + return 1; + } + return 0; +} + +int sstableKeyCompare(const Comparator* user_cmp, const InternalKey* a, + const InternalKey& b) { + if (a == nullptr) { + return -1; + } + return sstableKeyCompare(user_cmp, *a, b); +} + +int sstableKeyCompare(const Comparator* user_cmp, const InternalKey& a, + const InternalKey* b) { + if (b == nullptr) { + return -1; + } + return sstableKeyCompare(user_cmp, a, *b); +} + +uint64_t TotalFileSize(const std::vector& files) { + uint64_t sum = 0; + for (size_t i = 0; i < files.size() && files[i]; i++) { + sum += files[i]->fd.GetFileSize(); + } + return sum; +} + +void Compaction::SetInputVersion(Version* _input_version) { + input_version_ = _input_version; + cfd_ = input_version_->cfd(); + + cfd_->Ref(); + input_version_->Ref(); + edit_.SetColumnFamily(cfd_->GetID()); +} + +void Compaction::GetBoundaryKeys( + VersionStorageInfo* vstorage, + const std::vector& inputs, Slice* smallest_user_key, + Slice* largest_user_key, int exclude_level) { + bool initialized = false; + const Comparator* ucmp = vstorage->InternalComparator()->user_comparator(); + for (size_t i = 0; i < inputs.size(); ++i) { + if (inputs[i].files.empty() || inputs[i].level == exclude_level) { + continue; + } + if (inputs[i].level == 0) { + // we need to consider all files on level 0 + for (const auto* f : inputs[i].files) { + const Slice& start_user_key = f->smallest.user_key(); + if (!initialized || + ucmp->Compare(start_user_key, *smallest_user_key) < 0) { + *smallest_user_key = start_user_key; + } + const Slice& end_user_key = f->largest.user_key(); + if (!initialized || + ucmp->Compare(end_user_key, *largest_user_key) > 0) { + *largest_user_key = end_user_key; + } + initialized = true; + } + } else { + // we only need to consider the first and last file + const Slice& start_user_key = inputs[i].files[0]->smallest.user_key(); + if (!initialized || + ucmp->Compare(start_user_key, *smallest_user_key) < 0) { + *smallest_user_key = start_user_key; + } + const Slice& end_user_key = inputs[i].files.back()->largest.user_key(); + if (!initialized || ucmp->Compare(end_user_key, *largest_user_key) > 0) { + *largest_user_key = end_user_key; + } + initialized = true; + } + } +} + +std::vector Compaction::PopulateWithAtomicBoundaries( + VersionStorageInfo* vstorage, std::vector inputs) { + const Comparator* ucmp = vstorage->InternalComparator()->user_comparator(); + for (size_t i = 0; i < inputs.size(); i++) { + if (inputs[i].level == 0 || inputs[i].files.empty()) { + continue; + } + inputs[i].atomic_compaction_unit_boundaries.reserve(inputs[i].files.size()); + AtomicCompactionUnitBoundary cur_boundary; + size_t first_atomic_idx = 0; + auto add_unit_boundary = [&](size_t to) { + if (first_atomic_idx == to) return; + for (size_t k = first_atomic_idx; k < to; k++) { + inputs[i].atomic_compaction_unit_boundaries.push_back(cur_boundary); + } + first_atomic_idx = to; + }; + for (size_t j = 0; j < inputs[i].files.size(); j++) { + const auto* f = inputs[i].files[j]; + if (j == 0) { + // First file in a level. + cur_boundary.smallest = &f->smallest; + cur_boundary.largest = &f->largest; + } else if (sstableKeyCompare(ucmp, *cur_boundary.largest, f->smallest) == + 0) { + // SSTs overlap but the end key of the previous file was not + // artificially extended by a range tombstone. Extend the current + // boundary. + cur_boundary.largest = &f->largest; + } else { + // Atomic compaction unit has ended. + add_unit_boundary(j); + cur_boundary.smallest = &f->smallest; + cur_boundary.largest = &f->largest; + } + } + add_unit_boundary(inputs[i].files.size()); + assert(inputs[i].files.size() == + inputs[i].atomic_compaction_unit_boundaries.size()); + } + return inputs; +} + +// helper function to determine if compaction is creating files at the +// bottommost level +bool Compaction::IsBottommostLevel( + int output_level, VersionStorageInfo* vstorage, + const std::vector& inputs) { + int output_l0_idx; + if (output_level == 0) { + output_l0_idx = 0; + for (const auto* file : vstorage->LevelFiles(0)) { + if (inputs[0].files.back() == file) { + break; + } + ++output_l0_idx; + } + assert(static_cast(output_l0_idx) < vstorage->LevelFiles(0).size()); + } else { + output_l0_idx = -1; + } + Slice smallest_key, largest_key; + GetBoundaryKeys(vstorage, inputs, &smallest_key, &largest_key); + return !vstorage->RangeMightExistAfterSortedRun(smallest_key, largest_key, + output_level, output_l0_idx); +} + +// test function to validate the functionality of IsBottommostLevel() +// function -- determines if compaction with inputs and storage is bottommost +bool Compaction::TEST_IsBottommostLevel( + int output_level, VersionStorageInfo* vstorage, + const std::vector& inputs) { + return IsBottommostLevel(output_level, vstorage, inputs); +} + +bool Compaction::IsFullCompaction( + VersionStorageInfo* vstorage, + const std::vector& inputs) { + size_t num_files_in_compaction = 0; + size_t total_num_files = 0; + for (int l = 0; l < vstorage->num_levels(); l++) { + total_num_files += vstorage->NumLevelFiles(l); + } + for (size_t i = 0; i < inputs.size(); i++) { + num_files_in_compaction += inputs[i].size(); + } + return num_files_in_compaction == total_num_files; +} + +const TablePropertiesCollection& Compaction::GetTableProperties() { + if (!input_table_properties_initialized_) { + const ReadOptions read_options(Env::IOActivity::kCompaction); + for (size_t i = 0; i < num_input_levels(); ++i) { + for (const FileMetaData* fmd : *(this->inputs(i))) { + std::shared_ptr tp; + std::string file_name = + TableFileName(immutable_options_.cf_paths, fmd->fd.GetNumber(), + fmd->fd.GetPathId()); + Status s = input_version_->GetTableProperties(read_options, &tp, fmd, + &file_name); + if (s.ok()) { + table_properties_[file_name] = tp; + } else { + ROCKS_LOG_ERROR(immutable_options_.info_log, + "Unable to load table properties for file %" PRIu64 + " --- %s\n", + fmd->fd.GetNumber(), s.ToString().c_str()); + } + } + } + + input_table_properties_initialized_ = true; + }; + + return table_properties_; +} + +Compaction::Compaction( + VersionStorageInfo* vstorage, const ImmutableOptions& _immutable_options, + const MutableCFOptions& _mutable_cf_options, + const MutableDBOptions& _mutable_db_options, + std::vector _inputs, int _output_level, + uint64_t _target_file_size, uint64_t _max_compaction_bytes, + uint32_t _output_path_id, CompressionType _compression, + CompressionOptions _compression_opts, Temperature _output_temperature, + uint32_t _max_subcompactions, std::vector _grandparents, + bool _manual_compaction, const std::string& _trim_ts, double _score, + bool _deletion_compaction, bool l0_files_might_overlap, + CompactionReason _compaction_reason, + BlobGarbageCollectionPolicy _blob_garbage_collection_policy, + double _blob_garbage_collection_age_cutoff) + : input_vstorage_(vstorage), + start_level_(_inputs[0].level), + output_level_(_output_level), + target_output_file_size_(_target_file_size), + max_compaction_bytes_(_max_compaction_bytes), + max_subcompactions_(_max_subcompactions), + immutable_options_(_immutable_options), + mutable_cf_options_(_mutable_cf_options), + input_version_(nullptr), + number_levels_(vstorage->num_levels()), + cfd_(nullptr), + output_path_id_(_output_path_id), + output_compression_(_compression), + output_compression_opts_(_compression_opts), + output_temperature_(_output_temperature), + deletion_compaction_(_deletion_compaction), + l0_files_might_overlap_(l0_files_might_overlap), + inputs_(PopulateWithAtomicBoundaries(vstorage, std::move(_inputs))), + grandparents_(std::move(_grandparents)), + score_(_score), + bottommost_level_( + // For simplicity, we don't support the concept of "bottommost level" + // with + // `CompactionReason::kExternalSstIngestion` and + // `CompactionReason::kRefitLevel` + (_compaction_reason == CompactionReason::kExternalSstIngestion || + _compaction_reason == CompactionReason::kRefitLevel) + ? false + : IsBottommostLevel(output_level_, vstorage, inputs_)), + is_full_compaction_(IsFullCompaction(vstorage, inputs_)), + is_manual_compaction_(_manual_compaction), + trim_ts_(_trim_ts), + is_trivial_move_(false), + compaction_reason_(_compaction_reason), + notify_on_compaction_completion_(false), + enable_blob_garbage_collection_( + _blob_garbage_collection_policy == BlobGarbageCollectionPolicy::kForce + ? true + : (_blob_garbage_collection_policy == + BlobGarbageCollectionPolicy::kDisable + ? false + : mutable_cf_options()->enable_blob_garbage_collection)), + blob_garbage_collection_age_cutoff_( + _blob_garbage_collection_age_cutoff < 0 || + _blob_garbage_collection_age_cutoff > 1 + ? mutable_cf_options()->blob_garbage_collection_age_cutoff + : _blob_garbage_collection_age_cutoff), + penultimate_level_( + // For simplicity, we don't support the concept of "penultimate level" + // with `CompactionReason::kExternalSstIngestion` and + // `CompactionReason::kRefitLevel` + _compaction_reason == CompactionReason::kExternalSstIngestion || + _compaction_reason == CompactionReason::kRefitLevel + ? Compaction::kInvalidLevel + : EvaluatePenultimateLevel(vstorage, immutable_options_, + start_level_, output_level_)) { + MarkFilesBeingCompacted(true); + if (is_manual_compaction_) { + compaction_reason_ = CompactionReason::kManualCompaction; + } + if (max_subcompactions_ == 0) { + max_subcompactions_ = _mutable_db_options.max_subcompactions; + } + + // for the non-bottommost levels, it tries to build files match the target + // file size, but not guaranteed. It could be 2x the size of the target size. + max_output_file_size_ = + bottommost_level_ || grandparents_.empty() || + !_immutable_options.level_compaction_dynamic_file_size + ? target_output_file_size_ + : 2 * target_output_file_size_; + +#ifndef NDEBUG + for (size_t i = 1; i < inputs_.size(); ++i) { + assert(inputs_[i].level > inputs_[i - 1].level); + } +#endif + + // setup input_levels_ + { + input_levels_.resize(num_input_levels()); + for (size_t which = 0; which < num_input_levels(); which++) { + DoGenerateLevelFilesBrief(&input_levels_[which], inputs_[which].files, + &arena_); + } + } + + GetBoundaryKeys(vstorage, inputs_, &smallest_user_key_, &largest_user_key_); + + // Every compaction regardless of any compaction reason may respect the + // existing compact cursor in the output level to split output files + output_split_key_ = nullptr; + if (immutable_options_.compaction_style == kCompactionStyleLevel && + immutable_options_.compaction_pri == kRoundRobin) { + const InternalKey* cursor = + &input_vstorage_->GetCompactCursors()[output_level_]; + if (cursor->size() != 0) { + const Slice& cursor_user_key = ExtractUserKey(cursor->Encode()); + auto ucmp = vstorage->InternalComparator()->user_comparator(); + // May split output files according to the cursor if it in the user-key + // range + if (ucmp->CompareWithoutTimestamp(cursor_user_key, smallest_user_key_) > + 0 && + ucmp->CompareWithoutTimestamp(cursor_user_key, largest_user_key_) <= + 0) { + output_split_key_ = cursor; + } + } + } + + PopulatePenultimateLevelOutputRange(); +} + +void Compaction::PopulatePenultimateLevelOutputRange() { + if (!SupportsPerKeyPlacement()) { + return; + } + + // exclude the last level, the range of all input levels is the safe range + // of keys that can be moved up. + int exclude_level = number_levels_ - 1; + penultimate_output_range_type_ = PenultimateOutputRangeType::kNonLastRange; + + // For universal compaction, the penultimate_output_range could be extended if + // all penultimate level files are included in the compaction (which includes + // the case that the penultimate level is empty). + if (immutable_options_.compaction_style == kCompactionStyleUniversal) { + exclude_level = kInvalidLevel; + penultimate_output_range_type_ = PenultimateOutputRangeType::kFullRange; + std::set penultimate_inputs; + for (const auto& input_lvl : inputs_) { + if (input_lvl.level == penultimate_level_) { + for (const auto& file : input_lvl.files) { + penultimate_inputs.emplace(file->fd.GetNumber()); + } + } + } + auto penultimate_files = input_vstorage_->LevelFiles(penultimate_level_); + for (const auto& file : penultimate_files) { + if (penultimate_inputs.find(file->fd.GetNumber()) == + penultimate_inputs.end()) { + exclude_level = number_levels_ - 1; + penultimate_output_range_type_ = + PenultimateOutputRangeType::kNonLastRange; + break; + } + } + } + + GetBoundaryKeys(input_vstorage_, inputs_, + &penultimate_level_smallest_user_key_, + &penultimate_level_largest_user_key_, exclude_level); +} + +Compaction::~Compaction() { + if (input_version_ != nullptr) { + input_version_->Unref(); + } + if (cfd_ != nullptr) { + cfd_->UnrefAndTryDelete(); + } +} + +bool Compaction::SupportsPerKeyPlacement() const { + return penultimate_level_ != kInvalidLevel; +} + +int Compaction::GetPenultimateLevel() const { return penultimate_level_; } + +// smallest_key and largest_key include timestamps if user-defined timestamp is +// enabled. +bool Compaction::OverlapPenultimateLevelOutputRange( + const Slice& smallest_key, const Slice& largest_key) const { + if (!SupportsPerKeyPlacement()) { + return false; + } + const Comparator* ucmp = + input_vstorage_->InternalComparator()->user_comparator(); + + return ucmp->CompareWithoutTimestamp( + smallest_key, penultimate_level_largest_user_key_) <= 0 && + ucmp->CompareWithoutTimestamp( + largest_key, penultimate_level_smallest_user_key_) >= 0; +} + +// key includes timestamp if user-defined timestamp is enabled. +bool Compaction::WithinPenultimateLevelOutputRange(const Slice& key) const { + if (!SupportsPerKeyPlacement()) { + return false; + } + + if (penultimate_level_smallest_user_key_.empty() || + penultimate_level_largest_user_key_.empty()) { + return false; + } + + const Comparator* ucmp = + input_vstorage_->InternalComparator()->user_comparator(); + + return ucmp->CompareWithoutTimestamp( + key, penultimate_level_smallest_user_key_) >= 0 && + ucmp->CompareWithoutTimestamp( + key, penultimate_level_largest_user_key_) <= 0; +} + +bool Compaction::InputCompressionMatchesOutput() const { + int base_level = input_vstorage_->base_level(); + bool matches = + (GetCompressionType(input_vstorage_, mutable_cf_options_, start_level_, + base_level) == output_compression_); + if (matches) { + TEST_SYNC_POINT("Compaction::InputCompressionMatchesOutput:Matches"); + return true; + } + TEST_SYNC_POINT("Compaction::InputCompressionMatchesOutput:DidntMatch"); + return matches; +} + +bool Compaction::IsTrivialMove() const { + // Avoid a move if there is lots of overlapping grandparent data. + // Otherwise, the move could create a parent file that will require + // a very expensive merge later on. + // If start_level_== output_level_, the purpose is to force compaction + // filter to be applied to that level, and thus cannot be a trivial move. + + // Check if start level have files with overlapping ranges + if (start_level_ == 0 && input_vstorage_->level0_non_overlapping() == false && + l0_files_might_overlap_) { + // We cannot move files from L0 to L1 if the L0 files in the LSM-tree are + // overlapping, unless we are sure that files picked in L0 don't overlap. + return false; + } + + if (is_manual_compaction_ && + (immutable_options_.compaction_filter != nullptr || + immutable_options_.compaction_filter_factory != nullptr)) { + // This is a manual compaction and we have a compaction filter that should + // be executed, we cannot do a trivial move + return false; + } + + if (start_level_ == output_level_) { + // It doesn't make sense if compaction picker picks files just to trivial + // move to the same level. + return false; + } + + if (compaction_reason_ == CompactionReason::kChangeTemperature) { + // Changing temperature usually requires rewriting the file. + return false; + } + + // Used in universal compaction, where trivial move can be done if the + // input files are non overlapping + if ((mutable_cf_options_.compaction_options_universal.allow_trivial_move) && + (output_level_ != 0) && + (cfd_->ioptions()->compaction_style == kCompactionStyleUniversal)) { + return is_trivial_move_; + } + + if (!(start_level_ != output_level_ && num_input_levels() == 1 && + input(0, 0)->fd.GetPathId() == output_path_id() && + InputCompressionMatchesOutput())) { + return false; + } + + // assert inputs_.size() == 1 + + if (output_level_ + 1 < number_levels_) { + std::unique_ptr partitioner = CreateSstPartitioner(); + for (const auto& file : inputs_.front().files) { + std::vector file_grand_parents; + input_vstorage_->GetOverlappingInputs(output_level_ + 1, &file->smallest, + &file->largest, + &file_grand_parents); + const auto compaction_size = + file->fd.GetFileSize() + TotalFileSize(file_grand_parents); + if (compaction_size > max_compaction_bytes_) { + return false; + } + + if (partitioner.get() != nullptr) { + if (!partitioner->CanDoTrivialMove(file->smallest.user_key(), + file->largest.user_key())) { + return false; + } + } + } + } + + // PerKeyPlacement compaction should never be trivial move. + if (SupportsPerKeyPlacement()) { + return false; + } + + return true; +} + +void Compaction::AddInputDeletions(VersionEdit* out_edit) { + for (size_t which = 0; which < num_input_levels(); which++) { + for (size_t i = 0; i < inputs_[which].size(); i++) { + out_edit->DeleteFile(level(which), inputs_[which][i]->fd.GetNumber()); + } + } +} + +bool Compaction::KeyNotExistsBeyondOutputLevel( + const Slice& user_key, std::vector* level_ptrs) const { + assert(input_version_ != nullptr); + assert(level_ptrs != nullptr); + assert(level_ptrs->size() == static_cast(number_levels_)); + if (bottommost_level_) { + return true; + } else if (output_level_ != 0 && + cfd_->ioptions()->compaction_style == kCompactionStyleLevel) { + // Maybe use binary search to find right entry instead of linear search? + const Comparator* user_cmp = cfd_->user_comparator(); + for (int lvl = output_level_ + 1; lvl < number_levels_; lvl++) { + const std::vector& files = + input_vstorage_->LevelFiles(lvl); + for (; level_ptrs->at(lvl) < files.size(); level_ptrs->at(lvl)++) { + auto* f = files[level_ptrs->at(lvl)]; + if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) { + // We've advanced far enough + // In the presence of user-defined timestamp, we may need to handle + // the case in which f->smallest.user_key() (including ts) has the + // same user key, but the ts part is smaller. If so, + // Compare(user_key, f->smallest.user_key()) returns -1. + // That's why we need CompareWithoutTimestamp(). + if (user_cmp->CompareWithoutTimestamp(user_key, + f->smallest.user_key()) >= 0) { + // Key falls in this file's range, so it may + // exist beyond output level + return false; + } + break; + } + } + } + return true; + } + return false; +} + +bool Compaction::KeyRangeNotExistsBeyondOutputLevel( + const Slice& begin_key, const Slice& end_key, + std::vector* level_ptrs) const { + assert(input_version_ != nullptr); + assert(level_ptrs != nullptr); + assert(level_ptrs->size() == static_cast(number_levels_)); + assert(cfd_->user_comparator()->CompareWithoutTimestamp(begin_key, end_key) < + 0); + if (bottommost_level_) { + return true /* does not overlap */; + } else if (output_level_ != 0 && + cfd_->ioptions()->compaction_style == kCompactionStyleLevel) { + const Comparator* user_cmp = cfd_->user_comparator(); + for (int lvl = output_level_ + 1; lvl < number_levels_; lvl++) { + const std::vector& files = + input_vstorage_->LevelFiles(lvl); + for (; level_ptrs->at(lvl) < files.size(); level_ptrs->at(lvl)++) { + auto* f = files[level_ptrs->at(lvl)]; + // Advance until the first file with begin_key <= f->largest.user_key() + if (user_cmp->CompareWithoutTimestamp(begin_key, + f->largest.user_key()) > 0) { + continue; + } + // We know that the previous file prev_f, if exists, has + // prev_f->largest.user_key() < begin_key. + if (user_cmp->CompareWithoutTimestamp(end_key, + f->smallest.user_key()) <= 0) { + // not overlapping with this level + break; + } else { + // We have: + // - begin_key < end_key, + // - begin_key <= f->largest.user_key(), and + // - end_key > f->smallest.user_key() + return false /* overlap */; + } + } + } + return true /* does not overlap */; + } + return false /* overlaps */; +}; + +// Mark (or clear) each file that is being compacted +void Compaction::MarkFilesBeingCompacted(bool mark_as_compacted) { + for (size_t i = 0; i < num_input_levels(); i++) { + for (size_t j = 0; j < inputs_[i].size(); j++) { + assert(mark_as_compacted ? !inputs_[i][j]->being_compacted + : inputs_[i][j]->being_compacted); + inputs_[i][j]->being_compacted = mark_as_compacted; + } + } +} + +// Sample output: +// If compacting 3 L0 files, 2 L3 files and 1 L4 file, and outputting to L5, +// print: "3@0 + 2@3 + 1@4 files to L5" +const char* Compaction::InputLevelSummary( + InputLevelSummaryBuffer* scratch) const { + int len = 0; + bool is_first = true; + for (auto& input_level : inputs_) { + if (input_level.empty()) { + continue; + } + if (!is_first) { + len += + snprintf(scratch->buffer + len, sizeof(scratch->buffer) - len, " + "); + len = std::min(len, static_cast(sizeof(scratch->buffer))); + } else { + is_first = false; + } + len += snprintf(scratch->buffer + len, sizeof(scratch->buffer) - len, + "%" ROCKSDB_PRIszt "@%d", input_level.size(), + input_level.level); + len = std::min(len, static_cast(sizeof(scratch->buffer))); + } + snprintf(scratch->buffer + len, sizeof(scratch->buffer) - len, + " files to L%d", output_level()); + + return scratch->buffer; +} + +uint64_t Compaction::CalculateTotalInputSize() const { + uint64_t size = 0; + for (auto& input_level : inputs_) { + for (auto f : input_level.files) { + size += f->fd.GetFileSize(); + } + } + return size; +} + +void Compaction::ReleaseCompactionFiles(Status status) { + MarkFilesBeingCompacted(false); + cfd_->compaction_picker()->ReleaseCompactionFiles(this, status); +} + +void Compaction::ResetNextCompactionIndex() { + assert(input_version_ != nullptr); + input_vstorage_->ResetNextCompactionIndex(start_level_); +} + +namespace { +int InputSummary(const std::vector& files, char* output, + int len) { + *output = '\0'; + int write = 0; + for (size_t i = 0; i < files.size(); i++) { + int sz = len - write; + int ret; + char sztxt[16]; + AppendHumanBytes(files.at(i)->fd.GetFileSize(), sztxt, 16); + ret = snprintf(output + write, sz, "%" PRIu64 "(%s) ", + files.at(i)->fd.GetNumber(), sztxt); + if (ret < 0 || ret >= sz) break; + write += ret; + } + // if files.size() is non-zero, overwrite the last space + return write - !!files.size(); +} +} // namespace + +void Compaction::Summary(char* output, int len) { + int write = + snprintf(output, len, "Base version %" PRIu64 " Base level %d, inputs: [", + input_version_->GetVersionNumber(), start_level_); + if (write < 0 || write >= len) { + return; + } + + for (size_t level_iter = 0; level_iter < num_input_levels(); ++level_iter) { + if (level_iter > 0) { + write += snprintf(output + write, len - write, "], ["); + if (write < 0 || write >= len) { + return; + } + } + write += + InputSummary(inputs_[level_iter].files, output + write, len - write); + if (write < 0 || write >= len) { + return; + } + } + + snprintf(output + write, len - write, "]"); +} + +uint64_t Compaction::OutputFilePreallocationSize() const { + uint64_t preallocation_size = 0; + + for (const auto& level_files : inputs_) { + for (const auto& file : level_files.files) { + preallocation_size += file->fd.GetFileSize(); + } + } + + if (max_output_file_size_ != std::numeric_limits::max() && + (immutable_options_.compaction_style == kCompactionStyleLevel || + output_level() > 0)) { + preallocation_size = std::min(max_output_file_size_, preallocation_size); + } + + // Over-estimate slightly so we don't end up just barely crossing + // the threshold + // No point to preallocate more than 1GB. + return std::min(uint64_t{1073741824}, + preallocation_size + (preallocation_size / 10)); +} + +std::unique_ptr Compaction::CreateCompactionFilter() const { + if (!cfd_->ioptions()->compaction_filter_factory) { + return nullptr; + } + + if (!cfd_->ioptions() + ->compaction_filter_factory->ShouldFilterTableFileCreation( + TableFileCreationReason::kCompaction)) { + return nullptr; + } + + CompactionFilter::Context context; + context.is_full_compaction = is_full_compaction_; + context.is_manual_compaction = is_manual_compaction_; + context.column_family_id = cfd_->GetID(); + context.reason = TableFileCreationReason::kCompaction; + return cfd_->ioptions()->compaction_filter_factory->CreateCompactionFilter( + context); +} + +std::unique_ptr Compaction::CreateSstPartitioner() const { + if (!immutable_options_.sst_partitioner_factory) { + return nullptr; + } + + SstPartitioner::Context context; + context.is_full_compaction = is_full_compaction_; + context.is_manual_compaction = is_manual_compaction_; + context.output_level = output_level_; + context.smallest_user_key = smallest_user_key_; + context.largest_user_key = largest_user_key_; + return immutable_options_.sst_partitioner_factory->CreatePartitioner(context); +} + +bool Compaction::IsOutputLevelEmpty() const { + return inputs_.back().level != output_level_ || inputs_.back().empty(); +} + +bool Compaction::ShouldFormSubcompactions() const { + if (cfd_ == nullptr) { + return false; + } + + // Round-Robin pri under leveled compaction allows subcompactions by default + // and the number of subcompactions can be larger than max_subcompactions_ + if (cfd_->ioptions()->compaction_pri == kRoundRobin && + cfd_->ioptions()->compaction_style == kCompactionStyleLevel) { + return output_level_ > 0; + } + + if (max_subcompactions_ <= 1) { + return false; + } + + if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) { + return (start_level_ == 0 || is_manual_compaction_) && output_level_ > 0; + } else if (cfd_->ioptions()->compaction_style == kCompactionStyleUniversal) { + return number_levels_ > 1 && output_level_ > 0; + } else { + return false; + } +} + +bool Compaction::DoesInputReferenceBlobFiles() const { + assert(input_version_); + + const VersionStorageInfo* storage_info = input_version_->storage_info(); + assert(storage_info); + + if (storage_info->GetBlobFiles().empty()) { + return false; + } + + for (size_t i = 0; i < inputs_.size(); ++i) { + for (const FileMetaData* meta : inputs_[i].files) { + assert(meta); + + if (meta->oldest_blob_file_number != kInvalidBlobFileNumber) { + return true; + } + } + } + + return false; +} + +uint64_t Compaction::MinInputFileOldestAncesterTime( + const InternalKey* start, const InternalKey* end) const { + uint64_t min_oldest_ancester_time = std::numeric_limits::max(); + const InternalKeyComparator& icmp = + column_family_data()->internal_comparator(); + for (const auto& level_files : inputs_) { + for (const auto& file : level_files.files) { + if (start != nullptr && icmp.Compare(file->largest, *start) < 0) { + continue; + } + if (end != nullptr && icmp.Compare(file->smallest, *end) > 0) { + continue; + } + uint64_t oldest_ancester_time = file->TryGetOldestAncesterTime(); + if (oldest_ancester_time != 0) { + min_oldest_ancester_time = + std::min(min_oldest_ancester_time, oldest_ancester_time); + } + } + } + return min_oldest_ancester_time; +} + +uint64_t Compaction::MinInputFileEpochNumber() const { + uint64_t min_epoch_number = std::numeric_limits::max(); + for (const auto& inputs_per_level : inputs_) { + for (const auto& file : inputs_per_level.files) { + min_epoch_number = std::min(min_epoch_number, file->epoch_number); + } + } + return min_epoch_number; +} + +int Compaction::EvaluatePenultimateLevel( + const VersionStorageInfo* vstorage, + const ImmutableOptions& immutable_options, const int start_level, + const int output_level) { + // TODO: currently per_key_placement feature only support level and universal + // compaction + if (immutable_options.compaction_style != kCompactionStyleLevel && + immutable_options.compaction_style != kCompactionStyleUniversal) { + return kInvalidLevel; + } + if (output_level != immutable_options.num_levels - 1) { + return kInvalidLevel; + } + + int penultimate_level = output_level - 1; + assert(penultimate_level < immutable_options.num_levels); + if (penultimate_level <= 0) { + return kInvalidLevel; + } + + // If the penultimate level is not within input level -> output level range + // check if the penultimate output level is empty, if it's empty, it could + // also be locked for the penultimate output. + // TODO: ideally, it only needs to check if there's a file within the + // compaction output key range. For simplicity, it just check if there's any + // file on the penultimate level. + if (start_level == immutable_options.num_levels - 1 && + (immutable_options.compaction_style != kCompactionStyleUniversal || + !vstorage->LevelFiles(penultimate_level).empty())) { + return kInvalidLevel; + } + + bool supports_per_key_placement = + immutable_options.preclude_last_level_data_seconds > 0; + + // it could be overridden by unittest + TEST_SYNC_POINT_CALLBACK("Compaction::SupportsPerKeyPlacement:Enabled", + &supports_per_key_placement); + if (!supports_per_key_placement) { + return kInvalidLevel; + } + + return penultimate_level; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction.h b/librocksdb-sys/rocksdb/db/compaction/compaction.h new file mode 100644 index 0000000..fcb0f30 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction.h @@ -0,0 +1,589 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include "db/version_set.h" +#include "memory/arena.h" +#include "options/cf_options.h" +#include "rocksdb/sst_partitioner.h" +#include "util/autovector.h" + +namespace ROCKSDB_NAMESPACE { +// The file contains class Compaction, as well as some helper functions +// and data structures used by the class. + +// Utility for comparing sstable boundary keys. Returns -1 if either a or b is +// null which provides the property that a==null indicates a key that is less +// than any key and b==null indicates a key that is greater than any key. Note +// that the comparison is performed primarily on the user-key portion of the +// key. If the user-keys compare equal, an additional test is made to sort +// range tombstone sentinel keys before other keys with the same user-key. The +// result is that 2 user-keys will compare equal if they differ purely on +// their sequence number and value, but the range tombstone sentinel for that +// user-key will compare not equal. This is necessary because the range +// tombstone sentinel key is set as the largest key for an sstable even though +// that key never appears in the database. We don't want adjacent sstables to +// be considered overlapping if they are separated by the range tombstone +// sentinel. +int sstableKeyCompare(const Comparator* user_cmp, const Slice&, const Slice&); +inline int sstableKeyCompare(const Comparator* user_cmp, const Slice& a, + const InternalKey& b) { + return sstableKeyCompare(user_cmp, a, b.Encode()); +} +inline int sstableKeyCompare(const Comparator* user_cmp, const InternalKey& a, + const Slice& b) { + return sstableKeyCompare(user_cmp, a.Encode(), b); +} +inline int sstableKeyCompare(const Comparator* user_cmp, const InternalKey& a, + const InternalKey& b) { + return sstableKeyCompare(user_cmp, a.Encode(), b.Encode()); +} +int sstableKeyCompare(const Comparator* user_cmp, const InternalKey* a, + const InternalKey& b); +int sstableKeyCompare(const Comparator* user_cmp, const InternalKey& a, + const InternalKey* b); + +// An AtomicCompactionUnitBoundary represents a range of keys [smallest, +// largest] that exactly spans one ore more neighbouring SSTs on the same +// level. Every pair of SSTs in this range "overlap" (i.e., the largest +// user key of one file is the smallest user key of the next file). These +// boundaries are propagated down to RangeDelAggregator during compaction +// to provide safe truncation boundaries for range tombstones. +struct AtomicCompactionUnitBoundary { + const InternalKey* smallest = nullptr; + const InternalKey* largest = nullptr; +}; + +// The structure that manages compaction input files associated +// with the same physical level. +struct CompactionInputFiles { + int level; + std::vector files; + std::vector atomic_compaction_unit_boundaries; + inline bool empty() const { return files.empty(); } + inline size_t size() const { return files.size(); } + inline void clear() { files.clear(); } + inline FileMetaData* operator[](size_t i) const { return files[i]; } +}; + +class Version; +class ColumnFamilyData; +class VersionStorageInfo; +class CompactionFilter; + +// A Compaction encapsulates metadata about a compaction. +class Compaction { + public: + Compaction(VersionStorageInfo* input_version, + const ImmutableOptions& immutable_options, + const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, + std::vector inputs, int output_level, + uint64_t target_file_size, uint64_t max_compaction_bytes, + uint32_t output_path_id, CompressionType compression, + CompressionOptions compression_opts, + Temperature output_temperature, uint32_t max_subcompactions, + std::vector grandparents, + bool manual_compaction = false, const std::string& trim_ts = "", + double score = -1, bool deletion_compaction = false, + bool l0_files_might_overlap = true, + CompactionReason compaction_reason = CompactionReason::kUnknown, + BlobGarbageCollectionPolicy blob_garbage_collection_policy = + BlobGarbageCollectionPolicy::kUseDefault, + double blob_garbage_collection_age_cutoff = -1); + + // The type of the penultimate level output range + enum class PenultimateOutputRangeType : int { + kNotSupported, // it cannot output to the penultimate level + kFullRange, // any data could be output to the penultimate level + kNonLastRange, // only the keys within non_last_level compaction inputs can + // be outputted to the penultimate level + kDisabled, // no data can be outputted to the penultimate level + }; + + // No copying allowed + Compaction(const Compaction&) = delete; + void operator=(const Compaction&) = delete; + + ~Compaction(); + + // Returns the level associated to the specified compaction input level. + // If compaction_input_level is not specified, then input_level is set to 0. + int level(size_t compaction_input_level = 0) const { + return inputs_[compaction_input_level].level; + } + + int start_level() const { return start_level_; } + + // Outputs will go to this level + int output_level() const { return output_level_; } + + // Returns the number of input levels in this compaction. + size_t num_input_levels() const { return inputs_.size(); } + + // Return the object that holds the edits to the descriptor done + // by this compaction. + VersionEdit* edit() { return &edit_; } + + // Returns the number of input files associated to the specified + // compaction input level. + // The function will return 0 if when "compaction_input_level" < 0 + // or "compaction_input_level" >= "num_input_levels()". + size_t num_input_files(size_t compaction_input_level) const { + if (compaction_input_level < inputs_.size()) { + return inputs_[compaction_input_level].size(); + } + return 0; + } + + // Returns input version of the compaction + Version* input_version() const { return input_version_; } + + // Returns the ColumnFamilyData associated with the compaction. + ColumnFamilyData* column_family_data() const { return cfd_; } + + // Returns the file meta data of the 'i'th input file at the + // specified compaction input level. + // REQUIREMENT: "compaction_input_level" must be >= 0 and + // < "input_levels()" + FileMetaData* input(size_t compaction_input_level, size_t i) const { + assert(compaction_input_level < inputs_.size()); + return inputs_[compaction_input_level][i]; + } + + const std::vector* boundaries( + size_t compaction_input_level) const { + assert(compaction_input_level < inputs_.size()); + return &inputs_[compaction_input_level].atomic_compaction_unit_boundaries; + } + + // Returns the list of file meta data of the specified compaction + // input level. + // REQUIREMENT: "compaction_input_level" must be >= 0 and + // < "input_levels()" + const std::vector* inputs( + size_t compaction_input_level) const { + assert(compaction_input_level < inputs_.size()); + return &inputs_[compaction_input_level].files; + } + + const std::vector* inputs() { return &inputs_; } + + // Returns the LevelFilesBrief of the specified compaction input level. + const LevelFilesBrief* input_levels(size_t compaction_input_level) const { + return &input_levels_[compaction_input_level]; + } + + // Maximum size of files to build during this compaction. + uint64_t max_output_file_size() const { return max_output_file_size_; } + + // Target output file size for this compaction + uint64_t target_output_file_size() const { return target_output_file_size_; } + + // What compression for output + CompressionType output_compression() const { return output_compression_; } + + // What compression options for output + const CompressionOptions& output_compression_opts() const { + return output_compression_opts_; + } + + // Whether need to write output file to second DB path. + uint32_t output_path_id() const { return output_path_id_; } + + // Is this a trivial compaction that can be implemented by just + // moving a single input file to the next level (no merging or splitting) + bool IsTrivialMove() const; + + // The split user key in the output level if this compaction is required to + // split the output files according to the existing cursor in the output + // level under round-robin compaction policy. Empty indicates no required + // splitting key + const InternalKey* GetOutputSplitKey() const { return output_split_key_; } + + // If true, then the compaction can be done by simply deleting input files. + bool deletion_compaction() const { return deletion_compaction_; } + + // Add all inputs to this compaction as delete operations to *edit. + void AddInputDeletions(VersionEdit* edit); + + // Returns true if the available information we have guarantees that + // the input "user_key" does not exist in any level beyond `output_level()`. + bool KeyNotExistsBeyondOutputLevel(const Slice& user_key, + std::vector* level_ptrs) const; + + // Returns true if the user key range [begin_key, end_key) does not exist + // in any level beyond `output_level()`. + // Used for checking range tombstones, so we assume begin_key < end_key. + // begin_key and end_key should include timestamp if enabled. + bool KeyRangeNotExistsBeyondOutputLevel( + const Slice& begin_key, const Slice& end_key, + std::vector* level_ptrs) const; + + // Clear all files to indicate that they are not being compacted + // Delete this compaction from the list of running compactions. + // + // Requirement: DB mutex held + void ReleaseCompactionFiles(Status status); + + // Returns the summary of the compaction in "output" with maximum "len" + // in bytes. The caller is responsible for the memory management of + // "output". + void Summary(char* output, int len); + + // Return the score that was used to pick this compaction run. + double score() const { return score_; } + + // Is this compaction creating a file in the bottom most level? + bool bottommost_level() const { return bottommost_level_; } + + // Is the compaction compact to the last level + bool is_last_level() const { + return output_level_ == immutable_options_.num_levels - 1; + } + + // Does this compaction include all sst files? + bool is_full_compaction() const { return is_full_compaction_; } + + // Was this compaction triggered manually by the client? + bool is_manual_compaction() const { return is_manual_compaction_; } + + std::string trim_ts() const { return trim_ts_; } + + // Used when allow_trivial_move option is set in + // Universal compaction. If all the input files are + // non overlapping, then is_trivial_move_ variable + // will be set true, else false + void set_is_trivial_move(bool trivial_move) { + is_trivial_move_ = trivial_move; + } + + // Used when allow_trivial_move option is set in + // Universal compaction. Returns true, if the input files + // are non-overlapping and can be trivially moved. + bool is_trivial_move() const { return is_trivial_move_; } + + // How many total levels are there? + int number_levels() const { return number_levels_; } + + // Return the ImmutableOptions that should be used throughout the compaction + // procedure + const ImmutableOptions* immutable_options() const { + return &immutable_options_; + } + + // Return the MutableCFOptions that should be used throughout the compaction + // procedure + const MutableCFOptions* mutable_cf_options() const { + return &mutable_cf_options_; + } + + // Returns the size in bytes that the output file should be preallocated to. + // In level compaction, that is max_file_size_. In universal compaction, that + // is the sum of all input file sizes. + uint64_t OutputFilePreallocationSize() const; + + void SetInputVersion(Version* input_version); + + struct InputLevelSummaryBuffer { + char buffer[128]; + }; + + const char* InputLevelSummary(InputLevelSummaryBuffer* scratch) const; + + uint64_t CalculateTotalInputSize() const; + + // In case of compaction error, reset the nextIndex that is used + // to pick up the next file to be compacted from files_by_size_ + void ResetNextCompactionIndex(); + + // Create a CompactionFilter from compaction_filter_factory + std::unique_ptr CreateCompactionFilter() const; + + // Create a SstPartitioner from sst_partitioner_factory + std::unique_ptr CreateSstPartitioner() const; + + // Is the input level corresponding to output_level_ empty? + bool IsOutputLevelEmpty() const; + + // Should this compaction be broken up into smaller ones run in parallel? + bool ShouldFormSubcompactions() const; + + // Returns true iff at least one input file references a blob file. + // + // PRE: input version has been set. + bool DoesInputReferenceBlobFiles() const; + + // test function to validate the functionality of IsBottommostLevel() + // function -- determines if compaction with inputs and storage is bottommost + static bool TEST_IsBottommostLevel( + int output_level, VersionStorageInfo* vstorage, + const std::vector& inputs); + + // If called before a compaction finishes, will return + // table properties of all compaction input files. + // If called after a compaction finished, will return + // table properties of all compaction input and output files. + const TablePropertiesCollection& GetTableProperties(); + + void SetOutputTableProperties( + const std::string& file_name, + const std::shared_ptr& tp) { + table_properties_[file_name] = tp; + } + + Slice GetSmallestUserKey() const { return smallest_user_key_; } + + Slice GetLargestUserKey() const { return largest_user_key_; } + + Slice GetPenultimateLevelSmallestUserKey() const { + return penultimate_level_smallest_user_key_; + } + + Slice GetPenultimateLevelLargestUserKey() const { + return penultimate_level_largest_user_key_; + } + + PenultimateOutputRangeType GetPenultimateOutputRangeType() const { + return penultimate_output_range_type_; + } + + // Return true if the compaction supports per_key_placement + bool SupportsPerKeyPlacement() const; + + // Get per_key_placement penultimate output level, which is `last_level - 1` + // if per_key_placement feature is supported. Otherwise, return -1. + int GetPenultimateLevel() const; + + // Return true if the given range is overlap with penultimate level output + // range. + // Both smallest_key and largest_key include timestamps if user-defined + // timestamp is enabled. + bool OverlapPenultimateLevelOutputRange(const Slice& smallest_key, + const Slice& largest_key) const; + + // Return true if the key is within penultimate level output range for + // per_key_placement feature, which is safe to place the key to the + // penultimate level. different compaction strategy has different rules. + // If per_key_placement is not supported, always return false. + // TODO: currently it doesn't support moving data from the last level to the + // penultimate level + // key includes timestamp if user-defined timestamp is enabled. + bool WithinPenultimateLevelOutputRange(const Slice& key) const; + + CompactionReason compaction_reason() const { return compaction_reason_; } + + const std::vector& grandparents() const { + return grandparents_; + } + + uint64_t max_compaction_bytes() const { return max_compaction_bytes_; } + + Temperature output_temperature() const { return output_temperature_; } + + uint32_t max_subcompactions() const { return max_subcompactions_; } + + bool enable_blob_garbage_collection() const { + return enable_blob_garbage_collection_; + } + + double blob_garbage_collection_age_cutoff() const { + return blob_garbage_collection_age_cutoff_; + } + + // start and end are sub compact range. Null if no boundary. + // This is used to filter out some input files' ancester's time range. + uint64_t MinInputFileOldestAncesterTime(const InternalKey* start, + const InternalKey* end) const; + // Return the minimum epoch number among + // input files' associated with this compaction + uint64_t MinInputFileEpochNumber() const; + + // Called by DBImpl::NotifyOnCompactionCompleted to make sure number of + // compaction begin and compaction completion callbacks match. + void SetNotifyOnCompactionCompleted() { + notify_on_compaction_completion_ = true; + } + + bool ShouldNotifyOnCompactionCompleted() const { + return notify_on_compaction_completion_; + } + + static constexpr int kInvalidLevel = -1; + + // Evaluate penultimate output level. If the compaction supports + // per_key_placement feature, it returns the penultimate level number. + // Otherwise, it's set to kInvalidLevel (-1), which means + // output_to_penultimate_level is not supported. + // Note: even the penultimate level output is supported (PenultimateLevel != + // kInvalidLevel), some key range maybe unsafe to be outputted to the + // penultimate level. The safe key range is populated by + // `PopulatePenultimateLevelOutputRange()`. + // Which could potentially disable all penultimate level output. + static int EvaluatePenultimateLevel(const VersionStorageInfo* vstorage, + const ImmutableOptions& immutable_options, + const int start_level, + const int output_level); + + private: + // mark (or clear) all files that are being compacted + void MarkFilesBeingCompacted(bool mark_as_compacted); + + // get the smallest and largest key present in files to be compacted + static void GetBoundaryKeys(VersionStorageInfo* vstorage, + const std::vector& inputs, + Slice* smallest_key, Slice* largest_key, + int exclude_level = -1); + + // populate penultimate level output range, which will be used to determine if + // a key is safe to output to the penultimate level (details see + // `Compaction::WithinPenultimateLevelOutputRange()`. + void PopulatePenultimateLevelOutputRange(); + + // Get the atomic file boundaries for all files in the compaction. Necessary + // in order to avoid the scenario described in + // https://github.com/facebook/rocksdb/pull/4432#discussion_r221072219 and + // plumb down appropriate key boundaries to RangeDelAggregator during + // compaction. + static std::vector PopulateWithAtomicBoundaries( + VersionStorageInfo* vstorage, std::vector inputs); + + // helper function to determine if compaction with inputs and storage is + // bottommost + static bool IsBottommostLevel( + int output_level, VersionStorageInfo* vstorage, + const std::vector& inputs); + + static bool IsFullCompaction(VersionStorageInfo* vstorage, + const std::vector& inputs); + + VersionStorageInfo* input_vstorage_; + + const int start_level_; // the lowest level to be compacted + const int output_level_; // levels to which output files are stored + uint64_t target_output_file_size_; + uint64_t max_output_file_size_; + uint64_t max_compaction_bytes_; + uint32_t max_subcompactions_; + const ImmutableOptions immutable_options_; + const MutableCFOptions mutable_cf_options_; + Version* input_version_; + VersionEdit edit_; + const int number_levels_; + ColumnFamilyData* cfd_; + Arena arena_; // Arena used to allocate space for file_levels_ + + const uint32_t output_path_id_; + CompressionType output_compression_; + CompressionOptions output_compression_opts_; + Temperature output_temperature_; + // If true, then the compaction can be done by simply deleting input files. + const bool deletion_compaction_; + // should it split the output file using the compact cursor? + const InternalKey* output_split_key_; + + // L0 files in LSM-tree might be overlapping. But the compaction picking + // logic might pick a subset of the files that aren't overlapping. if + // that is the case, set the value to false. Otherwise, set it true. + bool l0_files_might_overlap_; + + // Compaction input files organized by level. Constant after construction + const std::vector inputs_; + + // A copy of inputs_, organized more closely in memory + autovector input_levels_; + + // State used to check for number of overlapping grandparent files + // (grandparent == "output_level_ + 1") + std::vector grandparents_; + const double score_; // score that was used to pick this compaction. + + // Is this compaction creating a file in the bottom most level? + const bool bottommost_level_; + // Does this compaction include all sst files? + const bool is_full_compaction_; + + // Is this compaction requested by the client? + const bool is_manual_compaction_; + + // The data with timestamp > trim_ts_ will be removed + const std::string trim_ts_; + + // True if we can do trivial move in Universal multi level + // compaction + bool is_trivial_move_; + + // Does input compression match the output compression? + bool InputCompressionMatchesOutput() const; + + bool input_table_properties_initialized_ = false; + // table properties of output files + TablePropertiesCollection table_properties_; + + // smallest user keys in compaction + // includes timestamp if user-defined timestamp is enabled. + Slice smallest_user_key_; + + // largest user keys in compaction + // includes timestamp if user-defined timestamp is enabled. + Slice largest_user_key_; + + // Reason for compaction + CompactionReason compaction_reason_; + + // Notify on compaction completion only if listener was notified on compaction + // begin. + bool notify_on_compaction_completion_; + + // Enable/disable GC collection for blobs during compaction. + bool enable_blob_garbage_collection_; + + // Blob garbage collection age cutoff. + double blob_garbage_collection_age_cutoff_; + + // only set when per_key_placement feature is enabled, -1 (kInvalidLevel) + // means not supported. + const int penultimate_level_; + + // Key range for penultimate level output + // includes timestamp if user-defined timestamp is enabled. + // penultimate_output_range_type_ shows the range type + Slice penultimate_level_smallest_user_key_; + Slice penultimate_level_largest_user_key_; + PenultimateOutputRangeType penultimate_output_range_type_ = + PenultimateOutputRangeType::kNotSupported; +}; + +#ifndef NDEBUG +// Helper struct only for tests, which contains the data to decide if a key +// should be output to the penultimate level. +// TODO: remove this when the public feature knob is available +struct PerKeyPlacementContext { + const int level; + const Slice key; + const Slice value; + const SequenceNumber seq_num; + + bool& output_to_penultimate_level; + + PerKeyPlacementContext(int _level, Slice _key, Slice _value, + SequenceNumber _seq_num, + bool& _output_to_penultimate_level) + : level(_level), + key(_key), + value(_value), + seq_num(_seq_num), + output_to_penultimate_level(_output_to_penultimate_level) {} +}; +#endif /* !NDEBUG */ + +// Return sum of sizes of all files in `files`. +extern uint64_t TotalFileSize(const std::vector& files); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_iteration_stats.h b/librocksdb-sys/rocksdb/db/compaction/compaction_iteration_stats.h new file mode 100644 index 0000000..1b1c28b --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_iteration_stats.h @@ -0,0 +1,49 @@ +// Copyright (c) 2016-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +struct CompactionIterationStats { + // Compaction statistics + + // Doesn't include records skipped because of + // CompactionFilter::Decision::kRemoveAndSkipUntil. + int64_t num_record_drop_user = 0; + + int64_t num_record_drop_hidden = 0; + int64_t num_record_drop_obsolete = 0; + int64_t num_record_drop_range_del = 0; + int64_t num_range_del_drop_obsolete = 0; + // Deletions obsoleted before bottom level due to file gap optimization. + int64_t num_optimized_del_drop_obsolete = 0; + uint64_t total_filter_time = 0; + + // Input statistics + // TODO(noetzli): The stats are incomplete. They are lacking everything + // consumed by MergeHelper. + uint64_t num_input_records = 0; + uint64_t num_input_deletion_records = 0; + uint64_t num_input_corrupt_records = 0; + uint64_t total_input_raw_key_bytes = 0; + uint64_t total_input_raw_value_bytes = 0; + + // Single-Delete diagnostics for exceptional situations + uint64_t num_single_del_fallthru = 0; + uint64_t num_single_del_mismatch = 0; + + // Blob related statistics + uint64_t num_blobs_read = 0; + uint64_t total_blob_bytes_read = 0; + uint64_t num_blobs_relocated = 0; + uint64_t total_blob_bytes_relocated = 0; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_iterator.cc b/librocksdb-sys/rocksdb/db/compaction/compaction_iterator.cc new file mode 100644 index 0000000..1c3ca5e --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_iterator.cc @@ -0,0 +1,1446 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/compaction/compaction_iterator.h" + +#include +#include + +#include "db/blob/blob_fetcher.h" +#include "db/blob/blob_file_builder.h" +#include "db/blob/blob_index.h" +#include "db/blob/prefetch_buffer_collection.h" +#include "db/snapshot_checker.h" +#include "db/wide/wide_column_serialization.h" +#include "logging/logging.h" +#include "port/likely.h" +#include "rocksdb/listener.h" +#include "table/internal_iterator.h" +#include "test_util/sync_point.h" + +namespace ROCKSDB_NAMESPACE { +CompactionIterator::CompactionIterator( + InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper, + SequenceNumber last_sequence, std::vector* snapshots, + SequenceNumber earliest_write_conflict_snapshot, + SequenceNumber job_snapshot, const SnapshotChecker* snapshot_checker, + Env* env, bool report_detailed_time, bool expect_valid_internal_key, + CompactionRangeDelAggregator* range_del_agg, + BlobFileBuilder* blob_file_builder, bool allow_data_in_errors, + bool enforce_single_del_contracts, + const std::atomic& manual_compaction_canceled, + bool must_count_input_entries, const Compaction* compaction, + const CompactionFilter* compaction_filter, + const std::atomic* shutting_down, + const std::shared_ptr info_log, + const std::string* full_history_ts_low, + const SequenceNumber preserve_time_min_seqno, + const SequenceNumber preclude_last_level_min_seqno) + : CompactionIterator( + input, cmp, merge_helper, last_sequence, snapshots, + earliest_write_conflict_snapshot, job_snapshot, snapshot_checker, env, + report_detailed_time, expect_valid_internal_key, range_del_agg, + blob_file_builder, allow_data_in_errors, enforce_single_del_contracts, + manual_compaction_canceled, + std::unique_ptr( + compaction ? new RealCompaction(compaction) : nullptr), + must_count_input_entries, compaction_filter, shutting_down, info_log, + full_history_ts_low, preserve_time_min_seqno, + preclude_last_level_min_seqno) {} + +CompactionIterator::CompactionIterator( + InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper, + SequenceNumber /*last_sequence*/, std::vector* snapshots, + SequenceNumber earliest_write_conflict_snapshot, + SequenceNumber job_snapshot, const SnapshotChecker* snapshot_checker, + Env* env, bool report_detailed_time, bool expect_valid_internal_key, + CompactionRangeDelAggregator* range_del_agg, + BlobFileBuilder* blob_file_builder, bool allow_data_in_errors, + bool enforce_single_del_contracts, + const std::atomic& manual_compaction_canceled, + std::unique_ptr compaction, bool must_count_input_entries, + const CompactionFilter* compaction_filter, + const std::atomic* shutting_down, + const std::shared_ptr info_log, + const std::string* full_history_ts_low, + const SequenceNumber preserve_time_min_seqno, + const SequenceNumber preclude_last_level_min_seqno) + : input_(input, cmp, must_count_input_entries), + cmp_(cmp), + merge_helper_(merge_helper), + snapshots_(snapshots), + earliest_write_conflict_snapshot_(earliest_write_conflict_snapshot), + job_snapshot_(job_snapshot), + snapshot_checker_(snapshot_checker), + env_(env), + clock_(env_->GetSystemClock().get()), + report_detailed_time_(report_detailed_time), + expect_valid_internal_key_(expect_valid_internal_key), + range_del_agg_(range_del_agg), + blob_file_builder_(blob_file_builder), + compaction_(std::move(compaction)), + compaction_filter_(compaction_filter), + shutting_down_(shutting_down), + manual_compaction_canceled_(manual_compaction_canceled), + bottommost_level_(!compaction_ ? false + : compaction_->bottommost_level() && + !compaction_->allow_ingest_behind()), + // snapshots_ cannot be nullptr, but we will assert later in the body of + // the constructor. + visible_at_tip_(snapshots_ ? snapshots_->empty() : false), + earliest_snapshot_(!snapshots_ || snapshots_->empty() + ? kMaxSequenceNumber + : snapshots_->at(0)), + info_log_(info_log), + allow_data_in_errors_(allow_data_in_errors), + enforce_single_del_contracts_(enforce_single_del_contracts), + timestamp_size_(cmp_ ? cmp_->timestamp_size() : 0), + full_history_ts_low_(full_history_ts_low), + current_user_key_sequence_(0), + current_user_key_snapshot_(0), + merge_out_iter_(merge_helper_), + blob_garbage_collection_cutoff_file_number_( + ComputeBlobGarbageCollectionCutoffFileNumber(compaction_.get())), + blob_fetcher_(CreateBlobFetcherIfNeeded(compaction_.get())), + prefetch_buffers_( + CreatePrefetchBufferCollectionIfNeeded(compaction_.get())), + current_key_committed_(false), + cmp_with_history_ts_low_(0), + level_(compaction_ == nullptr ? 0 : compaction_->level()), + preserve_time_min_seqno_(preserve_time_min_seqno), + preclude_last_level_min_seqno_(preclude_last_level_min_seqno) { + assert(snapshots_ != nullptr); + assert(preserve_time_min_seqno_ <= preclude_last_level_min_seqno_); + + if (compaction_ != nullptr) { + level_ptrs_ = std::vector(compaction_->number_levels(), 0); + } +#ifndef NDEBUG + // findEarliestVisibleSnapshot assumes this ordering. + for (size_t i = 1; i < snapshots_->size(); ++i) { + assert(snapshots_->at(i - 1) < snapshots_->at(i)); + } + assert(timestamp_size_ == 0 || !full_history_ts_low_ || + timestamp_size_ == full_history_ts_low_->size()); +#endif + input_.SetPinnedItersMgr(&pinned_iters_mgr_); + // The default `merge_until_status_` does not need to be checked since it is + // overwritten as soon as `MergeUntil()` is called + merge_until_status_.PermitUncheckedError(); + TEST_SYNC_POINT_CALLBACK("CompactionIterator:AfterInit", compaction_.get()); +} + +CompactionIterator::~CompactionIterator() { + // input_ Iterator lifetime is longer than pinned_iters_mgr_ lifetime + input_.SetPinnedItersMgr(nullptr); +} + +void CompactionIterator::ResetRecordCounts() { + iter_stats_.num_record_drop_user = 0; + iter_stats_.num_record_drop_hidden = 0; + iter_stats_.num_record_drop_obsolete = 0; + iter_stats_.num_record_drop_range_del = 0; + iter_stats_.num_range_del_drop_obsolete = 0; + iter_stats_.num_optimized_del_drop_obsolete = 0; +} + +void CompactionIterator::SeekToFirst() { + NextFromInput(); + PrepareOutput(); +} + +void CompactionIterator::Next() { + // If there is a merge output, return it before continuing to process the + // input. + if (merge_out_iter_.Valid()) { + merge_out_iter_.Next(); + + // Check if we returned all records of the merge output. + if (merge_out_iter_.Valid()) { + key_ = merge_out_iter_.key(); + value_ = merge_out_iter_.value(); + Status s = ParseInternalKey(key_, &ikey_, allow_data_in_errors_); + // MergeUntil stops when it encounters a corrupt key and does not + // include them in the result, so we expect the keys here to be valid. + if (!s.ok()) { + ROCKS_LOG_FATAL( + info_log_, "Invalid ikey %s in compaction. %s", + allow_data_in_errors_ ? key_.ToString(true).c_str() : "hidden", + s.getState()); + assert(false); + } + + // Keep current_key_ in sync. + if (0 == timestamp_size_) { + current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type); + } else { + Slice ts = ikey_.GetTimestamp(timestamp_size_); + current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type, &ts); + } + key_ = current_key_.GetInternalKey(); + ikey_.user_key = current_key_.GetUserKey(); + validity_info_.SetValid(ValidContext::kMerge1); + } else { + if (merge_until_status_.IsMergeInProgress()) { + // `Status::MergeInProgress()` tells us that the previous `MergeUntil()` + // produced only merge operands. Those merge operands were accessed and + // written out using `merge_out_iter_`. Since `merge_out_iter_` is + // exhausted at this point, all merge operands have been written out. + // + // Still, there may be a base value (PUT, DELETE, SINGLEDEL, etc.) that + // needs to be written out. Normally, `CompactionIterator` would skip it + // on the basis that it has already output something in the same + // snapshot stripe. To prevent this, we reset `has_current_user_key_` to + // trick the future iteration from finding out the snapshot stripe is + // unchanged. + has_current_user_key_ = false; + } + // We consumed all pinned merge operands, release pinned iterators + pinned_iters_mgr_.ReleasePinnedData(); + // MergeHelper moves the iterator to the first record after the merged + // records, so even though we reached the end of the merge output, we do + // not want to advance the iterator. + NextFromInput(); + } + } else { + // Only advance the input iterator if there is no merge output and the + // iterator is not already at the next record. + if (!at_next_) { + AdvanceInputIter(); + } + NextFromInput(); + } + + if (Valid()) { + // Record that we've outputted a record for the current key. + has_outputted_key_ = true; + } + + PrepareOutput(); +} + +bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip, + Slice* skip_until) { + if (!compaction_filter_) { + return true; + } + + if (ikey_.type != kTypeValue && ikey_.type != kTypeBlobIndex && + ikey_.type != kTypeWideColumnEntity) { + return true; + } + + CompactionFilter::Decision decision = + CompactionFilter::Decision::kUndetermined; + CompactionFilter::ValueType value_type = + ikey_.type == kTypeValue ? CompactionFilter::ValueType::kValue + : ikey_.type == kTypeBlobIndex + ? CompactionFilter::ValueType::kBlobIndex + : CompactionFilter::ValueType::kWideColumnEntity; + + // Hack: pass internal key to BlobIndexCompactionFilter since it needs + // to get sequence number. + assert(compaction_filter_); + const Slice& filter_key = + (ikey_.type != kTypeBlobIndex || + !compaction_filter_->IsStackedBlobDbInternalCompactionFilter()) + ? ikey_.user_key + : key_; + + compaction_filter_value_.clear(); + compaction_filter_skip_until_.Clear(); + + std::vector> new_columns; + + { + StopWatchNano timer(clock_, report_detailed_time_); + + if (ikey_.type == kTypeBlobIndex) { + decision = compaction_filter_->FilterBlobByKey( + level_, filter_key, &compaction_filter_value_, + compaction_filter_skip_until_.rep()); + if (decision == CompactionFilter::Decision::kUndetermined && + !compaction_filter_->IsStackedBlobDbInternalCompactionFilter()) { + if (!compaction_) { + status_ = + Status::Corruption("Unexpected blob index outside of compaction"); + validity_info_.Invalidate(); + return false; + } + + TEST_SYNC_POINT_CALLBACK( + "CompactionIterator::InvokeFilterIfNeeded::TamperWithBlobIndex", + &value_); + + // For integrated BlobDB impl, CompactionIterator reads blob value. + // For Stacked BlobDB impl, the corresponding CompactionFilter's + // FilterV2 method should read the blob value. + BlobIndex blob_index; + Status s = blob_index.DecodeFrom(value_); + if (!s.ok()) { + status_ = s; + validity_info_.Invalidate(); + return false; + } + + FilePrefetchBuffer* prefetch_buffer = + prefetch_buffers_ ? prefetch_buffers_->GetOrCreatePrefetchBuffer( + blob_index.file_number()) + : nullptr; + + uint64_t bytes_read = 0; + + assert(blob_fetcher_); + + s = blob_fetcher_->FetchBlob(ikey_.user_key, blob_index, + prefetch_buffer, &blob_value_, + &bytes_read); + if (!s.ok()) { + status_ = s; + validity_info_.Invalidate(); + return false; + } + + ++iter_stats_.num_blobs_read; + iter_stats_.total_blob_bytes_read += bytes_read; + + value_type = CompactionFilter::ValueType::kValue; + } + } + + if (decision == CompactionFilter::Decision::kUndetermined) { + const Slice* existing_val = nullptr; + const WideColumns* existing_col = nullptr; + + WideColumns existing_columns; + + if (ikey_.type != kTypeWideColumnEntity) { + if (!blob_value_.empty()) { + existing_val = &blob_value_; + } else { + existing_val = &value_; + } + } else { + Slice value_copy = value_; + const Status s = + WideColumnSerialization::Deserialize(value_copy, existing_columns); + + if (!s.ok()) { + status_ = s; + validity_info_.Invalidate(); + return false; + } + + existing_col = &existing_columns; + } + + decision = compaction_filter_->FilterV3( + level_, filter_key, value_type, existing_val, existing_col, + &compaction_filter_value_, &new_columns, + compaction_filter_skip_until_.rep()); + } + + iter_stats_.total_filter_time += + env_ != nullptr && report_detailed_time_ ? timer.ElapsedNanos() : 0; + } + + if (decision == CompactionFilter::Decision::kUndetermined) { + // Should not reach here, since FilterV2/FilterV3 should never return + // kUndetermined. + status_ = Status::NotSupported( + "FilterV2/FilterV3 should never return kUndetermined"); + validity_info_.Invalidate(); + return false; + } + + if (decision == CompactionFilter::Decision::kRemoveAndSkipUntil && + cmp_->Compare(*compaction_filter_skip_until_.rep(), ikey_.user_key) <= + 0) { + // Can't skip to a key smaller than the current one. + // Keep the key as per FilterV2/FilterV3 documentation. + decision = CompactionFilter::Decision::kKeep; + } + + if (decision == CompactionFilter::Decision::kRemove) { + // convert the current key to a delete; key_ is pointing into + // current_key_ at this point, so updating current_key_ updates key() + ikey_.type = kTypeDeletion; + current_key_.UpdateInternalKey(ikey_.sequence, kTypeDeletion); + // no value associated with delete + value_.clear(); + iter_stats_.num_record_drop_user++; + } else if (decision == CompactionFilter::Decision::kPurge) { + // convert the current key to a single delete; key_ is pointing into + // current_key_ at this point, so updating current_key_ updates key() + ikey_.type = kTypeSingleDeletion; + current_key_.UpdateInternalKey(ikey_.sequence, kTypeSingleDeletion); + // no value associated with single delete + value_.clear(); + iter_stats_.num_record_drop_user++; + } else if (decision == CompactionFilter::Decision::kChangeValue) { + if (ikey_.type != kTypeValue) { + ikey_.type = kTypeValue; + current_key_.UpdateInternalKey(ikey_.sequence, kTypeValue); + } + + value_ = compaction_filter_value_; + } else if (decision == CompactionFilter::Decision::kRemoveAndSkipUntil) { + *need_skip = true; + compaction_filter_skip_until_.ConvertFromUserKey(kMaxSequenceNumber, + kValueTypeForSeek); + *skip_until = compaction_filter_skip_until_.Encode(); + } else if (decision == CompactionFilter::Decision::kChangeBlobIndex) { + // Only the StackableDB-based BlobDB impl's compaction filter should return + // kChangeBlobIndex. Decision about rewriting blob and changing blob index + // in the integrated BlobDB impl is made in subsequent call to + // PrepareOutput() and its callees. + if (!compaction_filter_->IsStackedBlobDbInternalCompactionFilter()) { + status_ = Status::NotSupported( + "Only stacked BlobDB's internal compaction filter can return " + "kChangeBlobIndex."); + validity_info_.Invalidate(); + return false; + } + + if (ikey_.type != kTypeBlobIndex) { + ikey_.type = kTypeBlobIndex; + current_key_.UpdateInternalKey(ikey_.sequence, kTypeBlobIndex); + } + + value_ = compaction_filter_value_; + } else if (decision == CompactionFilter::Decision::kIOError) { + if (!compaction_filter_->IsStackedBlobDbInternalCompactionFilter()) { + status_ = Status::NotSupported( + "CompactionFilter for integrated BlobDB should not return kIOError"); + validity_info_.Invalidate(); + return false; + } + + status_ = Status::IOError("Failed to access blob during compaction filter"); + validity_info_.Invalidate(); + return false; + } else if (decision == CompactionFilter::Decision::kChangeWideColumnEntity) { + WideColumns sorted_columns; + + sorted_columns.reserve(new_columns.size()); + for (const auto& column : new_columns) { + sorted_columns.emplace_back(column.first, column.second); + } + + std::sort(sorted_columns.begin(), sorted_columns.end(), + [](const WideColumn& lhs, const WideColumn& rhs) { + return lhs.name().compare(rhs.name()) < 0; + }); + + { + const Status s = WideColumnSerialization::Serialize( + sorted_columns, compaction_filter_value_); + if (!s.ok()) { + status_ = s; + validity_info_.Invalidate(); + return false; + } + } + + if (ikey_.type != kTypeWideColumnEntity) { + ikey_.type = kTypeWideColumnEntity; + current_key_.UpdateInternalKey(ikey_.sequence, kTypeWideColumnEntity); + } + + value_ = compaction_filter_value_; + } + + return true; +} + +void CompactionIterator::NextFromInput() { + at_next_ = false; + validity_info_.Invalidate(); + + while (!Valid() && input_.Valid() && !IsPausingManualCompaction() && + !IsShuttingDown()) { + key_ = input_.key(); + value_ = input_.value(); + blob_value_.Reset(); + iter_stats_.num_input_records++; + is_range_del_ = input_.IsDeleteRangeSentinelKey(); + + Status pik_status = ParseInternalKey(key_, &ikey_, allow_data_in_errors_); + if (!pik_status.ok()) { + iter_stats_.num_input_corrupt_records++; + + // If `expect_valid_internal_key_` is false, return the corrupted key + // and let the caller decide what to do with it. + if (expect_valid_internal_key_) { + status_ = pik_status; + return; + } + key_ = current_key_.SetInternalKey(key_); + has_current_user_key_ = false; + current_user_key_sequence_ = kMaxSequenceNumber; + current_user_key_snapshot_ = 0; + validity_info_.SetValid(ValidContext::kParseKeyError); + break; + } + TEST_SYNC_POINT_CALLBACK("CompactionIterator:ProcessKV", &ikey_); + if (is_range_del_) { + validity_info_.SetValid(kRangeDeletion); + break; + } + // Update input statistics + if (ikey_.type == kTypeDeletion || ikey_.type == kTypeSingleDeletion || + ikey_.type == kTypeDeletionWithTimestamp) { + iter_stats_.num_input_deletion_records++; + } + iter_stats_.total_input_raw_key_bytes += key_.size(); + iter_stats_.total_input_raw_value_bytes += value_.size(); + + // If need_skip is true, we should seek the input iterator + // to internal key skip_until and continue from there. + bool need_skip = false; + // Points either into compaction_filter_skip_until_ or into + // merge_helper_->compaction_filter_skip_until_. + Slice skip_until; + + bool user_key_equal_without_ts = false; + int cmp_ts = 0; + if (has_current_user_key_) { + user_key_equal_without_ts = + cmp_->EqualWithoutTimestamp(ikey_.user_key, current_user_key_); + // if timestamp_size_ > 0, then curr_ts_ has been initialized by a + // previous key. + cmp_ts = timestamp_size_ ? cmp_->CompareTimestamp( + ExtractTimestampFromUserKey( + ikey_.user_key, timestamp_size_), + curr_ts_) + : 0; + } + + // Check whether the user key changed. After this if statement current_key_ + // is a copy of the current input key (maybe converted to a delete by the + // compaction filter). ikey_.user_key is pointing to the copy. + if (!has_current_user_key_ || !user_key_equal_without_ts || cmp_ts != 0) { + // First occurrence of this user key + // Copy key for output + key_ = current_key_.SetInternalKey(key_, &ikey_); + + int prev_cmp_with_ts_low = + !full_history_ts_low_ ? 0 + : curr_ts_.empty() + ? 0 + : cmp_->CompareTimestamp(curr_ts_, *full_history_ts_low_); + + // If timestamp_size_ > 0, then copy from ikey_ to curr_ts_ for the use + // in next iteration to compare with the timestamp of next key. + UpdateTimestampAndCompareWithFullHistoryLow(); + + // If + // (1) !has_current_user_key_, OR + // (2) timestamp is disabled, OR + // (3) all history will be preserved, OR + // (4) user key (excluding timestamp) is different from previous key, OR + // (5) timestamp is NO older than *full_history_ts_low_, OR + // (6) timestamp is the largest one older than full_history_ts_low_, + // then current_user_key_ must be treated as a different user key. + // This means, if a user key (excluding ts) is the same as the previous + // user key, and its ts is older than *full_history_ts_low_, then we + // consider this key for GC, e.g. it may be dropped if certain conditions + // match. + if (!has_current_user_key_ || !timestamp_size_ || !full_history_ts_low_ || + !user_key_equal_without_ts || cmp_with_history_ts_low_ >= 0 || + prev_cmp_with_ts_low >= 0) { + // Initialize for future comparison for rule (A) and etc. + current_user_key_sequence_ = kMaxSequenceNumber; + current_user_key_snapshot_ = 0; + has_current_user_key_ = true; + } + current_user_key_ = ikey_.user_key; + + has_outputted_key_ = false; + + last_key_seq_zeroed_ = false; + + current_key_committed_ = KeyCommitted(ikey_.sequence); + + // Apply the compaction filter to the first committed version of the user + // key. + if (current_key_committed_ && + !InvokeFilterIfNeeded(&need_skip, &skip_until)) { + break; + } + } else { + // Update the current key to reflect the new sequence number/type without + // copying the user key. + // TODO(rven): Compaction filter does not process keys in this path + // Need to have the compaction filter process multiple versions + // if we have versions on both sides of a snapshot + current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type); + key_ = current_key_.GetInternalKey(); + ikey_.user_key = current_key_.GetUserKey(); + + // Note that newer version of a key is ordered before older versions. If a + // newer version of a key is committed, so as the older version. No need + // to query snapshot_checker_ in that case. + if (UNLIKELY(!current_key_committed_)) { + assert(snapshot_checker_ != nullptr); + current_key_committed_ = KeyCommitted(ikey_.sequence); + // Apply the compaction filter to the first committed version of the + // user key. + if (current_key_committed_ && + !InvokeFilterIfNeeded(&need_skip, &skip_until)) { + break; + } + } + } + + if (UNLIKELY(!current_key_committed_)) { + assert(snapshot_checker_ != nullptr); + validity_info_.SetValid(ValidContext::kCurrentKeyUncommitted); + break; + } + + // If there are no snapshots, then this kv affect visibility at tip. + // Otherwise, search though all existing snapshots to find the earliest + // snapshot that is affected by this kv. + SequenceNumber last_sequence = current_user_key_sequence_; + current_user_key_sequence_ = ikey_.sequence; + SequenceNumber last_snapshot = current_user_key_snapshot_; + SequenceNumber prev_snapshot = 0; // 0 means no previous snapshot + current_user_key_snapshot_ = + visible_at_tip_ + ? earliest_snapshot_ + : findEarliestVisibleSnapshot(ikey_.sequence, &prev_snapshot); + + if (need_skip) { + // This case is handled below. + } else if (clear_and_output_next_key_) { + // In the previous iteration we encountered a single delete that we could + // not compact out. We will keep this Put, but can drop it's data. + // (See Optimization 3, below.) + if (ikey_.type != kTypeValue && ikey_.type != kTypeBlobIndex && + ikey_.type != kTypeWideColumnEntity) { + ROCKS_LOG_FATAL(info_log_, "Unexpected key %s for compaction output", + ikey_.DebugString(allow_data_in_errors_, true).c_str()); + assert(false); + } + if (current_user_key_snapshot_ < last_snapshot) { + ROCKS_LOG_FATAL(info_log_, + "key %s, current_user_key_snapshot_ (%" PRIu64 + ") < last_snapshot (%" PRIu64 ")", + ikey_.DebugString(allow_data_in_errors_, true).c_str(), + current_user_key_snapshot_, last_snapshot); + assert(false); + } + + if (ikey_.type == kTypeBlobIndex || ikey_.type == kTypeWideColumnEntity) { + ikey_.type = kTypeValue; + current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type); + } + + value_.clear(); + validity_info_.SetValid(ValidContext::kKeepSDAndClearPut); + clear_and_output_next_key_ = false; + } else if (ikey_.type == kTypeSingleDeletion) { + // We can compact out a SingleDelete if: + // 1) We encounter the corresponding PUT -OR- we know that this key + // doesn't appear past this output level + // =AND= + // 2) We've already returned a record in this snapshot -OR- + // there are no earlier earliest_write_conflict_snapshot. + // + // A note about 2) above: + // we try to determine whether there is any earlier write conflict + // checking snapshot by calling DefinitelyInSnapshot() with seq and + // earliest_write_conflict_snapshot as arguments. For write-prepared + // and write-unprepared transactions, if earliest_write_conflict_snapshot + // is evicted from WritePreparedTxnDB::commit_cache, then + // DefinitelyInSnapshot(seq, earliest_write_conflict_snapshot) returns + // false, even if the seq is actually visible within + // earliest_write_conflict_snapshot. Consequently, CompactionIterator + // may try to zero out its sequence number, thus hitting assertion error + // in debug mode or cause incorrect DBIter return result. + // We observe that earliest_write_conflict_snapshot >= earliest_snapshot, + // and the seq zeroing logic depends on + // DefinitelyInSnapshot(seq, earliest_snapshot). Therefore, if we cannot + // determine whether seq is **definitely** in + // earliest_write_conflict_snapshot, then we can additionally check if + // seq is definitely in earliest_snapshot. If the latter holds, then the + // former holds too. + // + // Rule 1 is needed for SingleDelete correctness. Rule 2 is needed to + // allow Transactions to do write-conflict checking (if we compacted away + // all keys, then we wouldn't know that a write happened in this + // snapshot). If there is no earlier snapshot, then we know that there + // are no active transactions that need to know about any writes. + // + // Optimization 3: + // If we encounter a SingleDelete followed by a PUT and Rule 2 is NOT + // true, then we must output a SingleDelete. In this case, we will decide + // to also output the PUT. While we are compacting less by outputting the + // PUT now, hopefully this will lead to better compaction in the future + // when Rule 2 is later true (Ie, We are hoping we can later compact out + // both the SingleDelete and the Put, while we couldn't if we only + // outputted the SingleDelete now). + // In this case, we can save space by removing the PUT's value as it will + // never be read. + // + // Deletes and Merges are not supported on the same key that has a + // SingleDelete as it is not possible to correctly do any partial + // compaction of such a combination of operations. The result of mixing + // those operations for a given key is documented as being undefined. So + // we can choose how to handle such a combinations of operations. We will + // try to compact out as much as we can in these cases. + // We will report counts on these anomalous cases. + // + // Note: If timestamp is enabled, then record will be eligible for + // deletion, only if, along with above conditions (Rule 1 and Rule 2) + // full_history_ts_low_ is specified and timestamp for that key is less + // than *full_history_ts_low_. If it's not eligible for deletion, then we + // will output the SingleDelete. For Optimization 3 also, if + // full_history_ts_low_ is specified and timestamp for the key is less + // than *full_history_ts_low_ then only optimization will be applied. + + // The easiest way to process a SingleDelete during iteration is to peek + // ahead at the next key. + const bool is_timestamp_eligible_for_gc = + (timestamp_size_ == 0 || + (full_history_ts_low_ && cmp_with_history_ts_low_ < 0)); + + ParsedInternalKey next_ikey; + AdvanceInputIter(); + while (input_.Valid() && input_.IsDeleteRangeSentinelKey() && + ParseInternalKey(input_.key(), &next_ikey, allow_data_in_errors_) + .ok() && + cmp_->EqualWithoutTimestamp(ikey_.user_key, next_ikey.user_key)) { + // skip range tombstone start keys with the same user key + // since they are not "real" point keys. + AdvanceInputIter(); + } + + // Check whether the next key exists, is not corrupt, and is the same key + // as the single delete. + if (input_.Valid() && + ParseInternalKey(input_.key(), &next_ikey, allow_data_in_errors_) + .ok() && + cmp_->EqualWithoutTimestamp(ikey_.user_key, next_ikey.user_key)) { + assert(!input_.IsDeleteRangeSentinelKey()); +#ifndef NDEBUG + const Compaction* c = + compaction_ ? compaction_->real_compaction() : nullptr; +#endif + TEST_SYNC_POINT_CALLBACK( + "CompactionIterator::NextFromInput:SingleDelete:1", + const_cast(c)); + if (last_key_seq_zeroed_) { + ++iter_stats_.num_record_drop_hidden; + ++iter_stats_.num_record_drop_obsolete; + assert(bottommost_level_); + AdvanceInputIter(); + } else if (prev_snapshot == 0 || + DefinitelyNotInSnapshot(next_ikey.sequence, prev_snapshot)) { + // Check whether the next key belongs to the same snapshot as the + // SingleDelete. + + TEST_SYNC_POINT_CALLBACK( + "CompactionIterator::NextFromInput:SingleDelete:2", nullptr); + if (next_ikey.type == kTypeSingleDeletion) { + // We encountered two SingleDeletes for same key in a row. This + // could be due to unexpected user input. If write-(un)prepared + // transaction is used, this could also be due to releasing an old + // snapshot between a Put and its matching SingleDelete. + // Skip the first SingleDelete and let the next iteration decide + // how to handle the second SingleDelete. + + // First SingleDelete has been skipped since we already called + // input_.Next(). + ++iter_stats_.num_record_drop_obsolete; + ++iter_stats_.num_single_del_mismatch; + } else if (next_ikey.type == kTypeDeletion) { + std::ostringstream oss; + oss << "Found SD and type: " << static_cast(next_ikey.type) + << " on the same key, violating the contract " + "of SingleDelete. Check your application to make sure the " + "application does not mix SingleDelete and Delete for " + "the same key. If you are using " + "write-prepared/write-unprepared transactions, and use " + "SingleDelete to delete certain keys, then make sure " + "TransactionDBOptions::rollback_deletion_type_callback is " + "configured properly. Mixing SD and DEL can lead to " + "undefined behaviors"; + ++iter_stats_.num_record_drop_obsolete; + ++iter_stats_.num_single_del_mismatch; + if (enforce_single_del_contracts_) { + ROCKS_LOG_ERROR(info_log_, "%s", oss.str().c_str()); + validity_info_.Invalidate(); + status_ = Status::Corruption(oss.str()); + return; + } + ROCKS_LOG_WARN(info_log_, "%s", oss.str().c_str()); + } else if (!is_timestamp_eligible_for_gc) { + // We cannot drop the SingleDelete as timestamp is enabled, and + // timestamp of this key is greater than or equal to + // *full_history_ts_low_. We will output the SingleDelete. + validity_info_.SetValid(ValidContext::kKeepTsHistory); + } else if (has_outputted_key_ || + DefinitelyInSnapshot(ikey_.sequence, + earliest_write_conflict_snapshot_) || + (earliest_snapshot_ < earliest_write_conflict_snapshot_ && + DefinitelyInSnapshot(ikey_.sequence, + earliest_snapshot_))) { + // Found a matching value, we can drop the single delete and the + // value. It is safe to drop both records since we've already + // outputted a key in this snapshot, or there is no earlier + // snapshot (Rule 2 above). + + // Note: it doesn't matter whether the second key is a Put or if it + // is an unexpected Merge or Delete. We will compact it out + // either way. We will maintain counts of how many mismatches + // happened + if (next_ikey.type != kTypeValue && + next_ikey.type != kTypeBlobIndex && + next_ikey.type != kTypeWideColumnEntity) { + ++iter_stats_.num_single_del_mismatch; + } + + ++iter_stats_.num_record_drop_hidden; + ++iter_stats_.num_record_drop_obsolete; + // Already called input_.Next() once. Call it a second time to + // skip past the second key. + AdvanceInputIter(); + } else { + // Found a matching value, but we cannot drop both keys since + // there is an earlier snapshot and we need to leave behind a record + // to know that a write happened in this snapshot (Rule 2 above). + // Clear the value and output the SingleDelete. (The value will be + // outputted on the next iteration.) + + // Setting valid_ to true will output the current SingleDelete + validity_info_.SetValid(ValidContext::kKeepSDForConflictCheck); + + // Set up the Put to be outputted in the next iteration. + // (Optimization 3). + clear_and_output_next_key_ = true; + TEST_SYNC_POINT_CALLBACK( + "CompactionIterator::NextFromInput:KeepSDForWW", + /*arg=*/nullptr); + } + } else { + // We hit the next snapshot without hitting a put, so the iterator + // returns the single delete. + validity_info_.SetValid(ValidContext::kKeepSDForSnapshot); + TEST_SYNC_POINT_CALLBACK( + "CompactionIterator::NextFromInput:SingleDelete:3", + const_cast(c)); + } + } else { + // We are at the end of the input, could not parse the next key, or hit + // a different key. The iterator returns the single delete if the key + // possibly exists beyond the current output level. We set + // has_current_user_key to false so that if the iterator is at the next + // key, we do not compare it again against the previous key at the next + // iteration. If the next key is corrupt, we return before the + // comparison, so the value of has_current_user_key does not matter. + has_current_user_key_ = false; + if (compaction_ != nullptr && + DefinitelyInSnapshot(ikey_.sequence, earliest_snapshot_) && + compaction_->KeyNotExistsBeyondOutputLevel(ikey_.user_key, + &level_ptrs_) && + is_timestamp_eligible_for_gc) { + // Key doesn't exist outside of this range. + // Can compact out this SingleDelete. + ++iter_stats_.num_record_drop_obsolete; + ++iter_stats_.num_single_del_fallthru; + if (!bottommost_level_) { + ++iter_stats_.num_optimized_del_drop_obsolete; + } + } else if (last_key_seq_zeroed_) { + // Skip. + ++iter_stats_.num_record_drop_hidden; + ++iter_stats_.num_record_drop_obsolete; + assert(bottommost_level_); + } else { + // Output SingleDelete + validity_info_.SetValid(ValidContext::kKeepSD); + } + } + + if (Valid()) { + at_next_ = true; + } + } else if (last_snapshot == current_user_key_snapshot_ || + (last_snapshot > 0 && + last_snapshot < current_user_key_snapshot_)) { + // If the earliest snapshot is which this key is visible in + // is the same as the visibility of a previous instance of the + // same key, then this kv is not visible in any snapshot. + // Hidden by an newer entry for same user key + // + // Note: Dropping this key will not affect TransactionDB write-conflict + // checking since there has already been a record returned for this key + // in this snapshot. + if (last_sequence < current_user_key_sequence_) { + ROCKS_LOG_FATAL(info_log_, + "key %s, last_sequence (%" PRIu64 + ") < current_user_key_sequence_ (%" PRIu64 ")", + ikey_.DebugString(allow_data_in_errors_, true).c_str(), + last_sequence, current_user_key_sequence_); + assert(false); + } + + ++iter_stats_.num_record_drop_hidden; // rule (A) + AdvanceInputIter(); + } else if (compaction_ != nullptr && + (ikey_.type == kTypeDeletion || + (ikey_.type == kTypeDeletionWithTimestamp && + cmp_with_history_ts_low_ < 0)) && + DefinitelyInSnapshot(ikey_.sequence, earliest_snapshot_) && + compaction_->KeyNotExistsBeyondOutputLevel(ikey_.user_key, + &level_ptrs_)) { + // TODO(noetzli): This is the only place where we use compaction_ + // (besides the constructor). We should probably get rid of this + // dependency and find a way to do similar filtering during flushes. + // + // For this user key: + // (1) there is no data in higher levels + // (2) data in lower levels will have larger sequence numbers + // (3) data in layers that are being compacted here and have + // smaller sequence numbers will be dropped in the next + // few iterations of this loop (by rule (A) above). + // Therefore this deletion marker is obsolete and can be dropped. + // + // Note: Dropping this Delete will not affect TransactionDB + // write-conflict checking since it is earlier than any snapshot. + // + // It seems that we can also drop deletion later than earliest snapshot + // given that: + // (1) The deletion is earlier than earliest_write_conflict_snapshot, and + // (2) No value exist earlier than the deletion. + // + // Note also that a deletion marker of type kTypeDeletionWithTimestamp + // will be treated as a different user key unless the timestamp is older + // than *full_history_ts_low_. + ++iter_stats_.num_record_drop_obsolete; + if (!bottommost_level_) { + ++iter_stats_.num_optimized_del_drop_obsolete; + } + AdvanceInputIter(); + } else if ((ikey_.type == kTypeDeletion || + (ikey_.type == kTypeDeletionWithTimestamp && + cmp_with_history_ts_low_ < 0)) && + bottommost_level_) { + // Handle the case where we have a delete key at the bottom most level + // We can skip outputting the key iff there are no subsequent puts for + // this key + assert(!compaction_ || compaction_->KeyNotExistsBeyondOutputLevel( + ikey_.user_key, &level_ptrs_)); + ParsedInternalKey next_ikey; + AdvanceInputIter(); +#ifndef NDEBUG + const Compaction* c = + compaction_ ? compaction_->real_compaction() : nullptr; +#endif + TEST_SYNC_POINT_CALLBACK( + "CompactionIterator::NextFromInput:BottommostDelete:1", + const_cast(c)); + // Skip over all versions of this key that happen to occur in the same + // snapshot range as the delete. + // + // Note that a deletion marker of type kTypeDeletionWithTimestamp will be + // considered to have a different user key unless the timestamp is older + // than *full_history_ts_low_. + // + // Range tombstone start keys are skipped as they are not "real" keys. + while (!IsPausingManualCompaction() && !IsShuttingDown() && + input_.Valid() && + (ParseInternalKey(input_.key(), &next_ikey, allow_data_in_errors_) + .ok()) && + cmp_->EqualWithoutTimestamp(ikey_.user_key, next_ikey.user_key) && + (prev_snapshot == 0 || input_.IsDeleteRangeSentinelKey() || + DefinitelyNotInSnapshot(next_ikey.sequence, prev_snapshot))) { + AdvanceInputIter(); + } + // If you find you still need to output a row with this key, we need to + // output the delete too + if (input_.Valid() && + (ParseInternalKey(input_.key(), &next_ikey, allow_data_in_errors_) + .ok()) && + cmp_->EqualWithoutTimestamp(ikey_.user_key, next_ikey.user_key)) { + validity_info_.SetValid(ValidContext::kKeepDel); + at_next_ = true; + } + } else if (ikey_.type == kTypeMerge) { + if (!merge_helper_->HasOperator()) { + status_ = Status::InvalidArgument( + "merge_operator is not properly initialized."); + return; + } + + pinned_iters_mgr_.StartPinning(); + + // We know the merge type entry is not hidden, otherwise we would + // have hit (A) + // We encapsulate the merge related state machine in a different + // object to minimize change to the existing flow. + merge_until_status_ = merge_helper_->MergeUntil( + &input_, range_del_agg_, prev_snapshot, bottommost_level_, + allow_data_in_errors_, blob_fetcher_.get(), full_history_ts_low_, + prefetch_buffers_.get(), &iter_stats_); + merge_out_iter_.SeekToFirst(); + + if (!merge_until_status_.ok() && + !merge_until_status_.IsMergeInProgress()) { + status_ = merge_until_status_; + return; + } else if (merge_out_iter_.Valid()) { + // NOTE: key, value, and ikey_ refer to old entries. + // These will be correctly set below. + key_ = merge_out_iter_.key(); + value_ = merge_out_iter_.value(); + pik_status = ParseInternalKey(key_, &ikey_, allow_data_in_errors_); + // MergeUntil stops when it encounters a corrupt key and does not + // include them in the result, so we expect the keys here to valid. + if (!pik_status.ok()) { + ROCKS_LOG_FATAL( + info_log_, "Invalid key %s in compaction. %s", + allow_data_in_errors_ ? key_.ToString(true).c_str() : "hidden", + pik_status.getState()); + assert(false); + } + // Keep current_key_ in sync. + current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type); + key_ = current_key_.GetInternalKey(); + ikey_.user_key = current_key_.GetUserKey(); + validity_info_.SetValid(ValidContext::kMerge2); + } else { + // all merge operands were filtered out. reset the user key, since the + // batch consumed by the merge operator should not shadow any keys + // coming after the merges + has_current_user_key_ = false; + pinned_iters_mgr_.ReleasePinnedData(); + + if (merge_helper_->FilteredUntil(&skip_until)) { + need_skip = true; + } + } + } else { + // 1. new user key -OR- + // 2. different snapshot stripe + // If user-defined timestamp is enabled, we consider keys for GC if they + // are below history_ts_low_. CompactionRangeDelAggregator::ShouldDelete() + // only considers range deletions that are at or below history_ts_low_ and + // trim_ts_. We drop keys here that are below history_ts_low_ and are + // covered by a range tombstone that is at or below history_ts_low_ and + // trim_ts. + bool should_delete = false; + if (!timestamp_size_ || cmp_with_history_ts_low_ < 0) { + should_delete = range_del_agg_->ShouldDelete( + key_, RangeDelPositioningMode::kForwardTraversal); + } + if (should_delete) { + ++iter_stats_.num_record_drop_hidden; + ++iter_stats_.num_record_drop_range_del; + AdvanceInputIter(); + } else { + validity_info_.SetValid(ValidContext::kNewUserKey); + } + } + + if (need_skip) { + SkipUntil(skip_until); + } + } + + if (!Valid() && IsShuttingDown()) { + status_ = Status::ShutdownInProgress(); + } + + if (IsPausingManualCompaction()) { + status_ = Status::Incomplete(Status::SubCode::kManualCompactionPaused); + } + + // Propagate corruption status from memtable itereator + if (!input_.Valid() && input_.status().IsCorruption()) { + status_ = input_.status(); + } +} + +bool CompactionIterator::ExtractLargeValueIfNeededImpl() { + if (!blob_file_builder_) { + return false; + } + + blob_index_.clear(); + const Status s = blob_file_builder_->Add(user_key(), value_, &blob_index_); + + if (!s.ok()) { + status_ = s; + validity_info_.Invalidate(); + + return false; + } + + if (blob_index_.empty()) { + return false; + } + + value_ = blob_index_; + + return true; +} + +void CompactionIterator::ExtractLargeValueIfNeeded() { + assert(ikey_.type == kTypeValue); + + if (!ExtractLargeValueIfNeededImpl()) { + return; + } + + ikey_.type = kTypeBlobIndex; + current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type); +} + +void CompactionIterator::GarbageCollectBlobIfNeeded() { + assert(ikey_.type == kTypeBlobIndex); + + if (!compaction_) { + return; + } + + // GC for integrated BlobDB + if (compaction_->enable_blob_garbage_collection()) { + TEST_SYNC_POINT_CALLBACK( + "CompactionIterator::GarbageCollectBlobIfNeeded::TamperWithBlobIndex", + &value_); + + BlobIndex blob_index; + + { + const Status s = blob_index.DecodeFrom(value_); + + if (!s.ok()) { + status_ = s; + validity_info_.Invalidate(); + + return; + } + } + + if (blob_index.file_number() >= + blob_garbage_collection_cutoff_file_number_) { + return; + } + + FilePrefetchBuffer* prefetch_buffer = + prefetch_buffers_ ? prefetch_buffers_->GetOrCreatePrefetchBuffer( + blob_index.file_number()) + : nullptr; + + uint64_t bytes_read = 0; + + { + assert(blob_fetcher_); + + const Status s = blob_fetcher_->FetchBlob( + user_key(), blob_index, prefetch_buffer, &blob_value_, &bytes_read); + + if (!s.ok()) { + status_ = s; + validity_info_.Invalidate(); + + return; + } + } + + ++iter_stats_.num_blobs_read; + iter_stats_.total_blob_bytes_read += bytes_read; + + ++iter_stats_.num_blobs_relocated; + iter_stats_.total_blob_bytes_relocated += blob_index.size(); + + value_ = blob_value_; + + if (ExtractLargeValueIfNeededImpl()) { + return; + } + + ikey_.type = kTypeValue; + current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type); + + return; + } + + // GC for stacked BlobDB + if (compaction_filter_ && + compaction_filter_->IsStackedBlobDbInternalCompactionFilter()) { + const auto blob_decision = compaction_filter_->PrepareBlobOutput( + user_key(), value_, &compaction_filter_value_); + + if (blob_decision == CompactionFilter::BlobDecision::kCorruption) { + status_ = + Status::Corruption("Corrupted blob reference encountered during GC"); + validity_info_.Invalidate(); + + return; + } + + if (blob_decision == CompactionFilter::BlobDecision::kIOError) { + status_ = Status::IOError("Could not relocate blob during GC"); + validity_info_.Invalidate(); + + return; + } + + if (blob_decision == CompactionFilter::BlobDecision::kChangeValue) { + value_ = compaction_filter_value_; + + return; + } + } +} + +void CompactionIterator::DecideOutputLevel() { + assert(compaction_->SupportsPerKeyPlacement()); + output_to_penultimate_level_ = false; + // if the key is newer than the cutoff sequence or within the earliest + // snapshot, it should output to the penultimate level. + if (ikey_.sequence > preclude_last_level_min_seqno_ || + ikey_.sequence > earliest_snapshot_) { + output_to_penultimate_level_ = true; + } + +#ifndef NDEBUG + // Could be overridden by unittest + PerKeyPlacementContext context(level_, ikey_.user_key, value_, ikey_.sequence, + output_to_penultimate_level_); + TEST_SYNC_POINT_CALLBACK("CompactionIterator::PrepareOutput.context", + &context); + if (ikey_.sequence > earliest_snapshot_) { + output_to_penultimate_level_ = true; + } +#endif // NDEBUG + + if (output_to_penultimate_level_) { + // If it's decided to output to the penultimate level, but unsafe to do so, + // still output to the last level. For example, moving the data from a lower + // level to a higher level outside of the higher-level input key range is + // considered unsafe, because the key may conflict with higher-level SSTs + // not from this compaction. + // TODO: add statistic for declined output_to_penultimate_level + bool safe_to_penultimate_level = + compaction_->WithinPenultimateLevelOutputRange(ikey_.user_key); + if (!safe_to_penultimate_level) { + output_to_penultimate_level_ = false; + // It could happen when disable/enable `last_level_temperature` while + // holding a snapshot. When `last_level_temperature` is not set + // (==kUnknown), the data newer than any snapshot is pushed to the last + // level, but when the per_key_placement feature is enabled on the fly, + // the data later than the snapshot has to be moved to the penultimate + // level, which may or may not be safe. So the user needs to make sure all + // snapshot is released before enabling `last_level_temperature` feature + // We will migrate the feature to `last_level_temperature` and maybe make + // it not dynamically changeable. + if (ikey_.sequence > earliest_snapshot_) { + status_ = Status::Corruption( + "Unsafe to store Seq later than snapshot in the last level if " + "per_key_placement is enabled"); + } + } + } +} + +void CompactionIterator::PrepareOutput() { + if (Valid()) { + if (LIKELY(!is_range_del_)) { + if (ikey_.type == kTypeValue) { + ExtractLargeValueIfNeeded(); + } else if (ikey_.type == kTypeBlobIndex) { + GarbageCollectBlobIfNeeded(); + } + } + + if (compaction_ != nullptr && compaction_->SupportsPerKeyPlacement()) { + DecideOutputLevel(); + } + + // Zeroing out the sequence number leads to better compression. + // If this is the bottommost level (no files in lower levels) + // and the earliest snapshot is larger than this seqno + // and the userkey differs from the last userkey in compaction + // then we can squash the seqno to zero. + // + // This is safe for TransactionDB write-conflict checking since transactions + // only care about sequence number larger than any active snapshots. + // + // Can we do the same for levels above bottom level as long as + // KeyNotExistsBeyondOutputLevel() return true? + if (Valid() && compaction_ != nullptr && + !compaction_->allow_ingest_behind() && bottommost_level_ && + DefinitelyInSnapshot(ikey_.sequence, earliest_snapshot_) && + ikey_.type != kTypeMerge && current_key_committed_ && + !output_to_penultimate_level_ && + ikey_.sequence < preserve_time_min_seqno_ && !is_range_del_) { + if (ikey_.type == kTypeDeletion || + (ikey_.type == kTypeSingleDeletion && timestamp_size_ == 0)) { + ROCKS_LOG_FATAL( + info_log_, + "Unexpected key %s for seq-zero optimization. " + "earliest_snapshot %" PRIu64 + ", earliest_write_conflict_snapshot %" PRIu64 + " job_snapshot %" PRIu64 + ". timestamp_size: %d full_history_ts_low_ %s. validity %x", + ikey_.DebugString(allow_data_in_errors_, true).c_str(), + earliest_snapshot_, earliest_write_conflict_snapshot_, + job_snapshot_, static_cast(timestamp_size_), + full_history_ts_low_ != nullptr + ? Slice(*full_history_ts_low_).ToString(true).c_str() + : "null", + validity_info_.rep); + assert(false); + } + ikey_.sequence = 0; + last_key_seq_zeroed_ = true; + TEST_SYNC_POINT_CALLBACK("CompactionIterator::PrepareOutput:ZeroingSeq", + &ikey_); + if (!timestamp_size_) { + current_key_.UpdateInternalKey(0, ikey_.type); + } else if (full_history_ts_low_ && cmp_with_history_ts_low_ < 0) { + // We can also zero out timestamp for better compression. + // For the same user key (excluding timestamp), the timestamp-based + // history can be collapsed to save some space if the timestamp is + // older than *full_history_ts_low_. + const std::string kTsMin(timestamp_size_, static_cast(0)); + const Slice ts_slice = kTsMin; + ikey_.SetTimestamp(ts_slice); + current_key_.UpdateInternalKey(0, ikey_.type, &ts_slice); + } + } + } +} + +inline SequenceNumber CompactionIterator::findEarliestVisibleSnapshot( + SequenceNumber in, SequenceNumber* prev_snapshot) { + assert(snapshots_->size()); + if (snapshots_->size() == 0) { + ROCKS_LOG_FATAL(info_log_, + "No snapshot left in findEarliestVisibleSnapshot"); + } + auto snapshots_iter = + std::lower_bound(snapshots_->begin(), snapshots_->end(), in); + assert(prev_snapshot != nullptr); + if (snapshots_iter == snapshots_->begin()) { + *prev_snapshot = 0; + } else { + *prev_snapshot = *std::prev(snapshots_iter); + if (*prev_snapshot >= in) { + ROCKS_LOG_FATAL(info_log_, + "*prev_snapshot (%" PRIu64 ") >= in (%" PRIu64 + ") in findEarliestVisibleSnapshot", + *prev_snapshot, in); + assert(false); + } + } + if (snapshot_checker_ == nullptr) { + return snapshots_iter != snapshots_->end() ? *snapshots_iter + : kMaxSequenceNumber; + } + bool has_released_snapshot = !released_snapshots_.empty(); + for (; snapshots_iter != snapshots_->end(); ++snapshots_iter) { + auto cur = *snapshots_iter; + if (in > cur) { + ROCKS_LOG_FATAL(info_log_, + "in (%" PRIu64 ") > cur (%" PRIu64 + ") in findEarliestVisibleSnapshot", + in, cur); + assert(false); + } + // Skip if cur is in released_snapshots. + if (has_released_snapshot && released_snapshots_.count(cur) > 0) { + continue; + } + auto res = snapshot_checker_->CheckInSnapshot(in, cur); + if (res == SnapshotCheckerResult::kInSnapshot) { + return cur; + } else if (res == SnapshotCheckerResult::kSnapshotReleased) { + released_snapshots_.insert(cur); + } + *prev_snapshot = cur; + } + return kMaxSequenceNumber; +} + +uint64_t CompactionIterator::ComputeBlobGarbageCollectionCutoffFileNumber( + const CompactionProxy* compaction) { + if (!compaction) { + return 0; + } + + if (!compaction->enable_blob_garbage_collection()) { + return 0; + } + + const Version* const version = compaction->input_version(); + assert(version); + + const VersionStorageInfo* const storage_info = version->storage_info(); + assert(storage_info); + + const auto& blob_files = storage_info->GetBlobFiles(); + + const size_t cutoff_index = static_cast( + compaction->blob_garbage_collection_age_cutoff() * blob_files.size()); + + if (cutoff_index >= blob_files.size()) { + return std::numeric_limits::max(); + } + + const auto& meta = blob_files[cutoff_index]; + assert(meta); + + return meta->GetBlobFileNumber(); +} + +std::unique_ptr CompactionIterator::CreateBlobFetcherIfNeeded( + const CompactionProxy* compaction) { + if (!compaction) { + return nullptr; + } + + const Version* const version = compaction->input_version(); + if (!version) { + return nullptr; + } + + ReadOptions read_options; + read_options.io_activity = Env::IOActivity::kCompaction; + read_options.fill_cache = false; + + return std::unique_ptr(new BlobFetcher(version, read_options)); +} + +std::unique_ptr +CompactionIterator::CreatePrefetchBufferCollectionIfNeeded( + const CompactionProxy* compaction) { + if (!compaction) { + return nullptr; + } + + if (!compaction->input_version()) { + return nullptr; + } + + if (compaction->allow_mmap_reads()) { + return nullptr; + } + + const uint64_t readahead_size = compaction->blob_compaction_readahead_size(); + if (!readahead_size) { + return nullptr; + } + + return std::unique_ptr( + new PrefetchBufferCollection(readahead_size)); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_iterator.h b/librocksdb-sys/rocksdb/db/compaction/compaction_iterator.h new file mode 100644 index 0000000..15193b5 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_iterator.h @@ -0,0 +1,541 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "db/compaction/compaction.h" +#include "db/compaction/compaction_iteration_stats.h" +#include "db/merge_helper.h" +#include "db/pinned_iterators_manager.h" +#include "db/range_del_aggregator.h" +#include "db/snapshot_checker.h" +#include "options/cf_options.h" +#include "rocksdb/compaction_filter.h" + +namespace ROCKSDB_NAMESPACE { + +class BlobFileBuilder; +class BlobFetcher; +class PrefetchBufferCollection; + +// A wrapper of internal iterator whose purpose is to count how +// many entries there are in the iterator. +class SequenceIterWrapper : public InternalIterator { + public: + SequenceIterWrapper(InternalIterator* iter, const Comparator* cmp, + bool need_count_entries) + : icmp_(cmp), + inner_iter_(iter), + need_count_entries_(need_count_entries) {} + bool Valid() const override { return inner_iter_->Valid(); } + Status status() const override { return inner_iter_->status(); } + void Next() override { + if (!inner_iter_->IsDeleteRangeSentinelKey()) { + num_itered_++; + } + inner_iter_->Next(); + } + void Seek(const Slice& target) override { + if (!need_count_entries_) { + has_num_itered_ = false; + inner_iter_->Seek(target); + } else { + // Need to count total number of entries, + // so we do Next() rather than Seek(). + while (inner_iter_->Valid() && + icmp_.Compare(inner_iter_->key(), target) < 0) { + Next(); + } + } + } + Slice key() const override { return inner_iter_->key(); } + Slice value() const override { return inner_iter_->value(); } + + // Unused InternalIterator methods + void SeekToFirst() override { assert(false); } + void Prev() override { assert(false); } + void SeekForPrev(const Slice& /* target */) override { assert(false); } + void SeekToLast() override { assert(false); } + + uint64_t NumItered() const { return num_itered_; } + bool HasNumItered() const { return has_num_itered_; } + bool IsDeleteRangeSentinelKey() const override { + assert(Valid()); + return inner_iter_->IsDeleteRangeSentinelKey(); + } + + private: + InternalKeyComparator icmp_; + InternalIterator* inner_iter_; // not owned + uint64_t num_itered_ = 0; + bool need_count_entries_; + bool has_num_itered_ = true; +}; + +class CompactionIterator { + public: + // A wrapper around Compaction. Has a much smaller interface, only what + // CompactionIterator uses. Tests can override it. + class CompactionProxy { + public: + virtual ~CompactionProxy() = default; + + virtual int level() const = 0; + + virtual bool KeyNotExistsBeyondOutputLevel( + const Slice& user_key, std::vector* level_ptrs) const = 0; + + virtual bool bottommost_level() const = 0; + + virtual int number_levels() const = 0; + + // Result includes timestamp if user-defined timestamp is enabled. + virtual Slice GetLargestUserKey() const = 0; + + virtual bool allow_ingest_behind() const = 0; + + virtual bool allow_mmap_reads() const = 0; + + virtual bool enable_blob_garbage_collection() const = 0; + + virtual double blob_garbage_collection_age_cutoff() const = 0; + + virtual uint64_t blob_compaction_readahead_size() const = 0; + + virtual const Version* input_version() const = 0; + + virtual bool DoesInputReferenceBlobFiles() const = 0; + + virtual const Compaction* real_compaction() const = 0; + + virtual bool SupportsPerKeyPlacement() const = 0; + + // `key` includes timestamp if user-defined timestamp is enabled. + virtual bool WithinPenultimateLevelOutputRange(const Slice& key) const = 0; + }; + + class RealCompaction : public CompactionProxy { + public: + explicit RealCompaction(const Compaction* compaction) + : compaction_(compaction) { + assert(compaction_); + assert(compaction_->immutable_options()); + assert(compaction_->mutable_cf_options()); + } + + int level() const override { return compaction_->level(); } + + bool KeyNotExistsBeyondOutputLevel( + const Slice& user_key, std::vector* level_ptrs) const override { + return compaction_->KeyNotExistsBeyondOutputLevel(user_key, level_ptrs); + } + + bool bottommost_level() const override { + return compaction_->bottommost_level(); + } + + int number_levels() const override { return compaction_->number_levels(); } + + // Result includes timestamp if user-defined timestamp is enabled. + Slice GetLargestUserKey() const override { + return compaction_->GetLargestUserKey(); + } + + bool allow_ingest_behind() const override { + return compaction_->immutable_options()->allow_ingest_behind; + } + + bool allow_mmap_reads() const override { + return compaction_->immutable_options()->allow_mmap_reads; + } + + bool enable_blob_garbage_collection() const override { + return compaction_->enable_blob_garbage_collection(); + } + + double blob_garbage_collection_age_cutoff() const override { + return compaction_->blob_garbage_collection_age_cutoff(); + } + + uint64_t blob_compaction_readahead_size() const override { + return compaction_->mutable_cf_options()->blob_compaction_readahead_size; + } + + const Version* input_version() const override { + return compaction_->input_version(); + } + + bool DoesInputReferenceBlobFiles() const override { + return compaction_->DoesInputReferenceBlobFiles(); + } + + const Compaction* real_compaction() const override { return compaction_; } + + bool SupportsPerKeyPlacement() const override { + return compaction_->SupportsPerKeyPlacement(); + } + + // Check if key is within penultimate level output range, to see if it's + // safe to output to the penultimate level for per_key_placement feature. + // `key` includes timestamp if user-defined timestamp is enabled. + bool WithinPenultimateLevelOutputRange(const Slice& key) const override { + return compaction_->WithinPenultimateLevelOutputRange(key); + } + + private: + const Compaction* compaction_; + }; + + // @param must_count_input_entries if true, `NumInputEntryScanned()` will + // return the number of input keys scanned. If false, `NumInputEntryScanned()` + // will return this number if no Seek was called on `input`. User should call + // `HasNumInputEntryScanned()` first in this case. + CompactionIterator( + InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper, + SequenceNumber last_sequence, std::vector* snapshots, + SequenceNumber earliest_write_conflict_snapshot, + SequenceNumber job_snapshot, const SnapshotChecker* snapshot_checker, + Env* env, bool report_detailed_time, bool expect_valid_internal_key, + CompactionRangeDelAggregator* range_del_agg, + BlobFileBuilder* blob_file_builder, bool allow_data_in_errors, + bool enforce_single_del_contracts, + const std::atomic& manual_compaction_canceled, + bool must_count_input_entries, const Compaction* compaction = nullptr, + const CompactionFilter* compaction_filter = nullptr, + const std::atomic* shutting_down = nullptr, + const std::shared_ptr info_log = nullptr, + const std::string* full_history_ts_low = nullptr, + const SequenceNumber preserve_time_min_seqno = kMaxSequenceNumber, + const SequenceNumber preclude_last_level_min_seqno = kMaxSequenceNumber); + + // Constructor with custom CompactionProxy, used for tests. + CompactionIterator( + InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper, + SequenceNumber last_sequence, std::vector* snapshots, + SequenceNumber earliest_write_conflict_snapshot, + SequenceNumber job_snapshot, const SnapshotChecker* snapshot_checker, + Env* env, bool report_detailed_time, bool expect_valid_internal_key, + CompactionRangeDelAggregator* range_del_agg, + BlobFileBuilder* blob_file_builder, bool allow_data_in_errors, + bool enforce_single_del_contracts, + const std::atomic& manual_compaction_canceled, + std::unique_ptr compaction, + bool must_count_input_entries, + const CompactionFilter* compaction_filter = nullptr, + const std::atomic* shutting_down = nullptr, + const std::shared_ptr info_log = nullptr, + const std::string* full_history_ts_low = nullptr, + const SequenceNumber preserve_time_min_seqno = kMaxSequenceNumber, + const SequenceNumber preclude_last_level_min_seqno = kMaxSequenceNumber); + + ~CompactionIterator(); + + void ResetRecordCounts(); + + // Seek to the beginning of the compaction iterator output. + // + // REQUIRED: Call only once. + void SeekToFirst(); + + // Produces the next record in the compaction. + // + // REQUIRED: SeekToFirst() has been called. + void Next(); + + // Getters + const Slice& key() const { return key_; } + const Slice& value() const { return value_; } + const Status& status() const { return status_; } + const ParsedInternalKey& ikey() const { return ikey_; } + inline bool Valid() const { return validity_info_.IsValid(); } + const Slice& user_key() const { + if (UNLIKELY(is_range_del_)) { + return ikey_.user_key; + } + return current_user_key_; + } + const CompactionIterationStats& iter_stats() const { return iter_stats_; } + bool HasNumInputEntryScanned() const { return input_.HasNumItered(); } + uint64_t NumInputEntryScanned() const { return input_.NumItered(); } + // If the current key should be placed on penultimate level, only valid if + // per_key_placement is supported + bool output_to_penultimate_level() const { + return output_to_penultimate_level_; + } + Status InputStatus() const { return input_.status(); } + + bool IsDeleteRangeSentinelKey() const { return is_range_del_; } + + private: + // Processes the input stream to find the next output + void NextFromInput(); + + // Do final preparations before presenting the output to the callee. + void PrepareOutput(); + + // Decide the current key should be output to the last level or penultimate + // level, only call for compaction supports per key placement + void DecideOutputLevel(); + + // Passes the output value to the blob file builder (if any), and replaces it + // with the corresponding blob reference if it has been actually written to a + // blob file (i.e. if it passed the value size check). Returns true if the + // value got extracted to a blob file, false otherwise. + bool ExtractLargeValueIfNeededImpl(); + + // Extracts large values as described above, and updates the internal key's + // type to kTypeBlobIndex if the value got extracted. Should only be called + // for regular values (kTypeValue). + void ExtractLargeValueIfNeeded(); + + // Relocates valid blobs residing in the oldest blob files if garbage + // collection is enabled. Relocated blobs are written to new blob files or + // inlined in the LSM tree depending on the current settings (i.e. + // enable_blob_files and min_blob_size). Should only be called for blob + // references (kTypeBlobIndex). + // + // Note: the stacked BlobDB implementation's compaction filter based GC + // algorithm is also called from here. + void GarbageCollectBlobIfNeeded(); + + // Invoke compaction filter if needed. + // Return true on success, false on failures (e.g.: kIOError). + bool InvokeFilterIfNeeded(bool* need_skip, Slice* skip_until); + + // Given a sequence number, return the sequence number of the + // earliest snapshot that this sequence number is visible in. + // The snapshots themselves are arranged in ascending order of + // sequence numbers. + // Employ a sequential search because the total number of + // snapshots are typically small. + inline SequenceNumber findEarliestVisibleSnapshot( + SequenceNumber in, SequenceNumber* prev_snapshot); + + inline bool KeyCommitted(SequenceNumber sequence) { + return snapshot_checker_ == nullptr || + snapshot_checker_->CheckInSnapshot(sequence, job_snapshot_) == + SnapshotCheckerResult::kInSnapshot; + } + + bool DefinitelyInSnapshot(SequenceNumber seq, SequenceNumber snapshot); + + bool DefinitelyNotInSnapshot(SequenceNumber seq, SequenceNumber snapshot); + + // Extract user-defined timestamp from user key if possible and compare it + // with *full_history_ts_low_ if applicable. + inline void UpdateTimestampAndCompareWithFullHistoryLow() { + if (!timestamp_size_) { + return; + } + Slice ts = ExtractTimestampFromUserKey(ikey_.user_key, timestamp_size_); + curr_ts_.assign(ts.data(), ts.size()); + if (full_history_ts_low_) { + cmp_with_history_ts_low_ = + cmp_->CompareTimestamp(ts, *full_history_ts_low_); + } + } + + static uint64_t ComputeBlobGarbageCollectionCutoffFileNumber( + const CompactionProxy* compaction); + static std::unique_ptr CreateBlobFetcherIfNeeded( + const CompactionProxy* compaction); + static std::unique_ptr + CreatePrefetchBufferCollectionIfNeeded(const CompactionProxy* compaction); + + SequenceIterWrapper input_; + const Comparator* cmp_; + MergeHelper* merge_helper_; + const std::vector* snapshots_; + // List of snapshots released during compaction. + // findEarliestVisibleSnapshot() find them out from return of + // snapshot_checker, and make sure they will not be returned as + // earliest visible snapshot of an older value. + // See WritePreparedTransactionTest::ReleaseSnapshotDuringCompaction3. + std::unordered_set released_snapshots_; + const SequenceNumber earliest_write_conflict_snapshot_; + const SequenceNumber job_snapshot_; + const SnapshotChecker* const snapshot_checker_; + Env* env_; + SystemClock* clock_; + const bool report_detailed_time_; + const bool expect_valid_internal_key_; + CompactionRangeDelAggregator* range_del_agg_; + BlobFileBuilder* blob_file_builder_; + std::unique_ptr compaction_; + const CompactionFilter* compaction_filter_; + const std::atomic* shutting_down_; + const std::atomic& manual_compaction_canceled_; + const bool bottommost_level_; + const bool visible_at_tip_; + const SequenceNumber earliest_snapshot_; + + std::shared_ptr info_log_; + + const bool allow_data_in_errors_; + + const bool enforce_single_del_contracts_; + + // Comes from comparator. + const size_t timestamp_size_; + + // Lower bound timestamp to retain full history in terms of user-defined + // timestamp. If a key's timestamp is older than full_history_ts_low_, then + // the key *may* be eligible for garbage collection (GC). The skipping logic + // is in `NextFromInput()` and `PrepareOutput()`. + // If nullptr, NO GC will be performed and all history will be preserved. + const std::string* const full_history_ts_low_; + + // State + // + enum ValidContext : uint8_t { + kMerge1 = 0, + kMerge2 = 1, + kParseKeyError = 2, + kCurrentKeyUncommitted = 3, + kKeepSDAndClearPut = 4, + kKeepTsHistory = 5, + kKeepSDForConflictCheck = 6, + kKeepSDForSnapshot = 7, + kKeepSD = 8, + kKeepDel = 9, + kNewUserKey = 10, + kRangeDeletion = 11, + }; + + struct ValidityInfo { + inline bool IsValid() const { return rep & 1; } + ValidContext GetContext() const { + return static_cast(rep >> 1); + } + inline void SetValid(uint8_t ctx) { rep = (ctx << 1) | 1; } + inline void Invalidate() { rep = 0; } + + uint8_t rep{0}; + } validity_info_; + + // Points to a copy of the current compaction iterator output (current_key_) + // if valid. + Slice key_; + // Points to the value in the underlying iterator that corresponds to the + // current output. + Slice value_; + // The status is OK unless compaction iterator encounters a merge operand + // while not having a merge operator defined. + Status status_; + // Stores the user key, sequence number and type of the current compaction + // iterator output (or current key in the underlying iterator during + // NextFromInput()). + ParsedInternalKey ikey_; + // Stores whether ikey_.user_key is valid. If set to false, the user key is + // not compared against the current key in the underlying iterator. + bool has_current_user_key_ = false; + // If false, the iterator holds a copy of the current compaction iterator + // output (or current key in the underlying iterator during NextFromInput()). + bool at_next_ = false; + + IterKey current_key_; + Slice current_user_key_; + std::string curr_ts_; + SequenceNumber current_user_key_sequence_; + SequenceNumber current_user_key_snapshot_; + + // True if the iterator has already returned a record for the current key. + bool has_outputted_key_ = false; + + // truncated the value of the next key and output it without applying any + // compaction rules. This is used for outputting a put after a single delete. + bool clear_and_output_next_key_ = false; + + MergeOutputIterator merge_out_iter_; + Status merge_until_status_; + // PinnedIteratorsManager used to pin input_ Iterator blocks while reading + // merge operands and then releasing them after consuming them. + PinnedIteratorsManager pinned_iters_mgr_; + + uint64_t blob_garbage_collection_cutoff_file_number_; + + std::unique_ptr blob_fetcher_; + std::unique_ptr prefetch_buffers_; + + std::string blob_index_; + PinnableSlice blob_value_; + std::string compaction_filter_value_; + InternalKey compaction_filter_skip_until_; + // "level_ptrs" holds indices that remember which file of an associated + // level we were last checking during the last call to compaction-> + // KeyNotExistsBeyondOutputLevel(). This allows future calls to the function + // to pick off where it left off since each subcompaction's key range is + // increasing so a later call to the function must be looking for a key that + // is in or beyond the last file checked during the previous call + std::vector level_ptrs_; + CompactionIterationStats iter_stats_; + + // Used to avoid purging uncommitted values. The application can specify + // uncommitted values by providing a SnapshotChecker object. + bool current_key_committed_; + + // Saved result of ucmp->CompareTimestamp(current_ts_, *full_history_ts_low_) + int cmp_with_history_ts_low_; + + const int level_; + + // True if the previous internal key (same user key)'s sequence number has + // just been zeroed out during bottommost compaction. + bool last_key_seq_zeroed_{false}; + + // True if the current key should be output to the penultimate level if + // possible, compaction logic makes the final decision on which level to + // output to. + bool output_to_penultimate_level_{false}; + + // min seqno for preserving the time information. + const SequenceNumber preserve_time_min_seqno_ = kMaxSequenceNumber; + + // min seqno to preclude the data from the last level, if the key seqno larger + // than this, it will be output to penultimate level + const SequenceNumber preclude_last_level_min_seqno_ = kMaxSequenceNumber; + + void AdvanceInputIter() { input_.Next(); } + + void SkipUntil(const Slice& skip_until) { input_.Seek(skip_until); } + + bool IsShuttingDown() { + // This is a best-effort facility, so memory_order_relaxed is sufficient. + return shutting_down_ && shutting_down_->load(std::memory_order_relaxed); + } + + bool IsPausingManualCompaction() { + // This is a best-effort facility, so memory_order_relaxed is sufficient. + return manual_compaction_canceled_.load(std::memory_order_relaxed); + } + + // Stores whether the current compaction iterator output + // is a range tombstone start key. + bool is_range_del_{false}; +}; + +inline bool CompactionIterator::DefinitelyInSnapshot(SequenceNumber seq, + SequenceNumber snapshot) { + return ((seq) <= (snapshot) && + (snapshot_checker_ == nullptr || + LIKELY(snapshot_checker_->CheckInSnapshot((seq), (snapshot)) == + SnapshotCheckerResult::kInSnapshot))); +} + +inline bool CompactionIterator::DefinitelyNotInSnapshot( + SequenceNumber seq, SequenceNumber snapshot) { + return ((seq) > (snapshot) || + (snapshot_checker_ != nullptr && + UNLIKELY(snapshot_checker_->CheckInSnapshot((seq), (snapshot)) == + SnapshotCheckerResult::kNotInSnapshot))); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_iterator_test.cc b/librocksdb-sys/rocksdb/db/compaction/compaction_iterator_test.cc new file mode 100644 index 0000000..20428a5 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_iterator_test.cc @@ -0,0 +1,1618 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/compaction/compaction_iterator.h" + +#include +#include + +#include "db/dbformat.h" +#include "port/port.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/string_util.h" +#include "util/vector_iterator.h" +#include "utilities/merge_operators.h" + +namespace ROCKSDB_NAMESPACE { + +// Expects no merging attempts. +class NoMergingMergeOp : public MergeOperator { + public: + bool FullMergeV2(const MergeOperationInput& /*merge_in*/, + MergeOperationOutput* /*merge_out*/) const override { + ADD_FAILURE(); + return false; + } + bool PartialMergeMulti(const Slice& /*key*/, + const std::deque& /*operand_list*/, + std::string* /*new_value*/, + Logger* /*logger*/) const override { + ADD_FAILURE(); + return false; + } + const char* Name() const override { + return "CompactionIteratorTest NoMergingMergeOp"; + } +}; + +// Compaction filter that gets stuck when it sees a particular key, +// then gets unstuck when told to. +// Always returns Decision::kRemove. +class StallingFilter : public CompactionFilter { + public: + Decision FilterV2(int /*level*/, const Slice& key, ValueType /*type*/, + const Slice& /*existing_value*/, std::string* /*new_value*/, + std::string* /*skip_until*/) const override { + int k = std::atoi(key.ToString().c_str()); + last_seen.store(k); + while (k >= stall_at.load()) { + std::this_thread::yield(); + } + return Decision::kRemove; + } + + const char* Name() const override { + return "CompactionIteratorTest StallingFilter"; + } + + // Wait until the filter sees a key >= k and stalls at that key. + // If `exact`, asserts that the seen key is equal to k. + void WaitForStall(int k, bool exact = true) { + stall_at.store(k); + while (last_seen.load() < k) { + std::this_thread::yield(); + } + if (exact) { + EXPECT_EQ(k, last_seen.load()); + } + } + + // Filter will stall on key >= stall_at. Advance stall_at to unstall. + mutable std::atomic stall_at{0}; + // Last key the filter was called with. + mutable std::atomic last_seen{0}; +}; + +// Compaction filter that filter out all keys. +class FilterAllKeysCompactionFilter : public CompactionFilter { + public: + Decision FilterV2(int /*level*/, const Slice& /*key*/, ValueType /*type*/, + const Slice& /*existing_value*/, std::string* /*new_value*/, + std::string* /*skip_until*/) const override { + return Decision::kRemove; + } + + const char* Name() const override { return "AllKeysCompactionFilter"; } +}; + +class LoggingForwardVectorIterator : public VectorIterator { + public: + struct Action { + enum class Type { + SEEK_TO_FIRST, + SEEK, + NEXT, + }; + + Type type; + std::string arg; + + explicit Action(Type _type, std::string _arg = "") + : type(_type), arg(_arg) {} + + bool operator==(const Action& rhs) const { + return std::tie(type, arg) == std::tie(rhs.type, rhs.arg); + } + }; + + LoggingForwardVectorIterator(const std::vector& keys, + const std::vector& values) + : VectorIterator(keys, values) { + current_ = keys_.size(); + } + + void SeekToFirst() override { + log.emplace_back(Action::Type::SEEK_TO_FIRST); + VectorIterator::SeekToFirst(); + } + void SeekToLast() override { assert(false); } + + void Seek(const Slice& target) override { + log.emplace_back(Action::Type::SEEK, target.ToString()); + VectorIterator::Seek(target); + } + + void SeekForPrev(const Slice& /*target*/) override { assert(false); } + + void Next() override { + assert(Valid()); + log.emplace_back(Action::Type::NEXT); + VectorIterator::Next(); + } + void Prev() override { assert(false); } + + Slice key() const override { + assert(Valid()); + return VectorIterator::key(); + } + Slice value() const override { + assert(Valid()); + return VectorIterator::value(); + } + + std::vector log; +}; + +class FakeCompaction : public CompactionIterator::CompactionProxy { + public: + int level() const override { return 0; } + + bool KeyNotExistsBeyondOutputLevel( + const Slice& /*user_key*/, + std::vector* /*level_ptrs*/) const override { + return is_bottommost_level || key_not_exists_beyond_output_level; + } + + bool bottommost_level() const override { return is_bottommost_level; } + + int number_levels() const override { return 1; } + + Slice GetLargestUserKey() const override { + return "\xff\xff\xff\xff\xff\xff\xff\xff\xff"; + } + + bool allow_ingest_behind() const override { return is_allow_ingest_behind; } + + bool allow_mmap_reads() const override { return false; } + + bool enable_blob_garbage_collection() const override { return false; } + + double blob_garbage_collection_age_cutoff() const override { return 0.0; } + + uint64_t blob_compaction_readahead_size() const override { return 0; } + + const Version* input_version() const override { return nullptr; } + + bool DoesInputReferenceBlobFiles() const override { return false; } + + const Compaction* real_compaction() const override { return nullptr; } + + bool SupportsPerKeyPlacement() const override { + return supports_per_key_placement; + } + + bool WithinPenultimateLevelOutputRange(const Slice& key) const override { + return (!key.starts_with("unsafe_pb")); + } + + bool key_not_exists_beyond_output_level = false; + + bool is_bottommost_level = false; + + bool is_allow_ingest_behind = false; + + bool supports_per_key_placement = false; +}; + +// A simplified snapshot checker which assumes each snapshot has a global +// last visible sequence. +class TestSnapshotChecker : public SnapshotChecker { + public: + explicit TestSnapshotChecker( + SequenceNumber last_committed_sequence, + const std::unordered_map& snapshots = + {{}}) + : last_committed_sequence_(last_committed_sequence), + snapshots_(snapshots) {} + + SnapshotCheckerResult CheckInSnapshot( + SequenceNumber seq, SequenceNumber snapshot_seq) const override { + if (snapshot_seq == kMaxSequenceNumber) { + return seq <= last_committed_sequence_ + ? SnapshotCheckerResult::kInSnapshot + : SnapshotCheckerResult::kNotInSnapshot; + } + assert(snapshots_.count(snapshot_seq) > 0); + return seq <= snapshots_.at(snapshot_seq) + ? SnapshotCheckerResult::kInSnapshot + : SnapshotCheckerResult::kNotInSnapshot; + } + + private: + SequenceNumber last_committed_sequence_; + // A map of valid snapshot to last visible sequence to the snapshot. + std::unordered_map snapshots_; +}; + +// Test param: +// bool: whether to pass snapshot_checker to compaction iterator. +class CompactionIteratorTest : public testing::TestWithParam { + public: + CompactionIteratorTest() + : cmp_(BytewiseComparator()), icmp_(cmp_), snapshots_({}) {} + + explicit CompactionIteratorTest(const Comparator* ucmp) + : cmp_(ucmp), icmp_(cmp_), snapshots_({}) {} + + void InitIterators( + const std::vector& ks, const std::vector& vs, + const std::vector& range_del_ks, + const std::vector& range_del_vs, + SequenceNumber last_sequence, + SequenceNumber last_committed_sequence = kMaxSequenceNumber, + MergeOperator* merge_op = nullptr, CompactionFilter* filter = nullptr, + bool bottommost_level = false, + SequenceNumber earliest_write_conflict_snapshot = kMaxSequenceNumber, + bool key_not_exists_beyond_output_level = false, + const std::string* full_history_ts_low = nullptr) { + std::unique_ptr unfragmented_range_del_iter( + new VectorIterator(range_del_ks, range_del_vs, &icmp_)); + auto tombstone_list = std::make_shared( + std::move(unfragmented_range_del_iter), icmp_); + std::unique_ptr range_del_iter( + new FragmentedRangeTombstoneIterator(tombstone_list, icmp_, + kMaxSequenceNumber)); + range_del_agg_.reset(new CompactionRangeDelAggregator(&icmp_, snapshots_)); + range_del_agg_->AddTombstones(std::move(range_del_iter)); + + std::unique_ptr compaction; + if (filter || bottommost_level || key_not_exists_beyond_output_level) { + compaction_proxy_ = new FakeCompaction(); + compaction_proxy_->is_bottommost_level = bottommost_level; + compaction_proxy_->is_allow_ingest_behind = AllowIngestBehind(); + compaction_proxy_->key_not_exists_beyond_output_level = + key_not_exists_beyond_output_level; + compaction_proxy_->supports_per_key_placement = SupportsPerKeyPlacement(); + compaction.reset(compaction_proxy_); + } + bool use_snapshot_checker = UseSnapshotChecker() || GetParam(); + if (use_snapshot_checker || last_committed_sequence < kMaxSequenceNumber) { + snapshot_checker_.reset( + new TestSnapshotChecker(last_committed_sequence, snapshot_map_)); + } + merge_helper_.reset( + new MergeHelper(Env::Default(), cmp_, merge_op, filter, nullptr, false, + 0 /*latest_snapshot*/, snapshot_checker_.get(), + 0 /*level*/, nullptr /*statistics*/, &shutting_down_)); + + if (c_iter_) { + // Since iter_ is still used in ~CompactionIterator(), we call + // ~CompactionIterator() first. + c_iter_.reset(); + } + iter_.reset(new LoggingForwardVectorIterator(ks, vs)); + iter_->SeekToFirst(); + c_iter_.reset(new CompactionIterator( + iter_.get(), cmp_, merge_helper_.get(), last_sequence, &snapshots_, + earliest_write_conflict_snapshot, kMaxSequenceNumber, + snapshot_checker_.get(), Env::Default(), + false /* report_detailed_time */, false, range_del_agg_.get(), + nullptr /* blob_file_builder */, true /*allow_data_in_errors*/, + true /*enforce_single_del_contracts*/, + /*manual_compaction_canceled=*/kManualCompactionCanceledFalse_, + std::move(compaction), /*must_count_input_entries=*/false, filter, + &shutting_down_, /*info_log=*/nullptr, full_history_ts_low)); + } + + void AddSnapshot(SequenceNumber snapshot, + SequenceNumber last_visible_seq = kMaxSequenceNumber) { + snapshots_.push_back(snapshot); + snapshot_map_[snapshot] = last_visible_seq; + } + + virtual bool UseSnapshotChecker() const { return false; } + + virtual bool AllowIngestBehind() const { return false; } + + virtual bool SupportsPerKeyPlacement() const { return false; } + + void RunTest( + const std::vector& input_keys, + const std::vector& input_values, + const std::vector& expected_keys, + const std::vector& expected_values, + SequenceNumber last_committed_seq = kMaxSequenceNumber, + MergeOperator* merge_operator = nullptr, + CompactionFilter* compaction_filter = nullptr, + bool bottommost_level = false, + SequenceNumber earliest_write_conflict_snapshot = kMaxSequenceNumber, + bool key_not_exists_beyond_output_level = false, + const std::string* full_history_ts_low = nullptr) { + InitIterators(input_keys, input_values, {}, {}, kMaxSequenceNumber, + last_committed_seq, merge_operator, compaction_filter, + bottommost_level, earliest_write_conflict_snapshot, + key_not_exists_beyond_output_level, full_history_ts_low); + c_iter_->SeekToFirst(); + for (size_t i = 0; i < expected_keys.size(); i++) { + std::string info = "i = " + std::to_string(i); + ASSERT_TRUE(c_iter_->Valid()) << info; + ASSERT_OK(c_iter_->status()) << info; + ASSERT_EQ(expected_keys[i], c_iter_->key().ToString()) << info; + ASSERT_EQ(expected_values[i], c_iter_->value().ToString()) << info; + c_iter_->Next(); + } + ASSERT_OK(c_iter_->status()); + ASSERT_FALSE(c_iter_->Valid()); + } + + void ClearSnapshots() { + snapshots_.clear(); + snapshot_map_.clear(); + } + + const Comparator* cmp_; + const InternalKeyComparator icmp_; + std::vector snapshots_; + // A map of valid snapshot to last visible sequence to the snapshot. + std::unordered_map snapshot_map_; + std::unique_ptr merge_helper_; + std::unique_ptr iter_; + std::unique_ptr c_iter_; + std::unique_ptr range_del_agg_; + std::unique_ptr snapshot_checker_; + std::atomic shutting_down_{false}; + const std::atomic kManualCompactionCanceledFalse_{false}; + FakeCompaction* compaction_proxy_; +}; + +// It is possible that the output of the compaction iterator is empty even if +// the input is not. +TEST_P(CompactionIteratorTest, EmptyResult) { + InitIterators({test::KeyStr("a", 5, kTypeSingleDeletion), + test::KeyStr("a", 3, kTypeValue)}, + {"", "val"}, {}, {}, 5); + c_iter_->SeekToFirst(); + ASSERT_OK(c_iter_->status()); + ASSERT_FALSE(c_iter_->Valid()); +} + +// If there is a corruption after a single deletion, the corrupted key should +// be preserved. +TEST_P(CompactionIteratorTest, CorruptionAfterSingleDeletion) { + InitIterators({test::KeyStr("a", 5, kTypeSingleDeletion), + test::KeyStr("a", 3, kTypeValue, true), + test::KeyStr("b", 10, kTypeValue)}, + {"", "val", "val2"}, {}, {}, 10); + c_iter_->SeekToFirst(); + ASSERT_TRUE(c_iter_->Valid()); + ASSERT_EQ(test::KeyStr("a", 5, kTypeSingleDeletion), + c_iter_->key().ToString()); + c_iter_->Next(); + ASSERT_TRUE(c_iter_->Valid()); + ASSERT_EQ(test::KeyStr("a", 3, kTypeValue, true), c_iter_->key().ToString()); + c_iter_->Next(); + ASSERT_TRUE(c_iter_->Valid()); + ASSERT_EQ(test::KeyStr("b", 10, kTypeValue), c_iter_->key().ToString()); + c_iter_->Next(); + ASSERT_OK(c_iter_->status()); + ASSERT_FALSE(c_iter_->Valid()); +} + +TEST_P(CompactionIteratorTest, SimpleRangeDeletion) { + InitIterators({test::KeyStr("morning", 5, kTypeValue), + test::KeyStr("morning", 2, kTypeValue), + test::KeyStr("night", 3, kTypeValue)}, + {"zao", "zao", "wan"}, + {test::KeyStr("ma", 4, kTypeRangeDeletion)}, {"mz"}, 5); + c_iter_->SeekToFirst(); + ASSERT_TRUE(c_iter_->Valid()); + ASSERT_EQ(test::KeyStr("morning", 5, kTypeValue), c_iter_->key().ToString()); + c_iter_->Next(); + ASSERT_TRUE(c_iter_->Valid()); + ASSERT_EQ(test::KeyStr("night", 3, kTypeValue), c_iter_->key().ToString()); + c_iter_->Next(); + ASSERT_OK(c_iter_->status()); + ASSERT_FALSE(c_iter_->Valid()); +} + +TEST_P(CompactionIteratorTest, RangeDeletionWithSnapshots) { + AddSnapshot(10); + std::vector ks1; + ks1.push_back(test::KeyStr("ma", 28, kTypeRangeDeletion)); + std::vector vs1{"mz"}; + std::vector ks2{test::KeyStr("morning", 15, kTypeValue), + test::KeyStr("morning", 5, kTypeValue), + test::KeyStr("night", 40, kTypeValue), + test::KeyStr("night", 20, kTypeValue)}; + std::vector vs2{"zao 15", "zao 5", "wan 40", "wan 20"}; + InitIterators(ks2, vs2, ks1, vs1, 40); + c_iter_->SeekToFirst(); + ASSERT_TRUE(c_iter_->Valid()); + ASSERT_EQ(test::KeyStr("morning", 5, kTypeValue), c_iter_->key().ToString()); + c_iter_->Next(); + ASSERT_TRUE(c_iter_->Valid()); + ASSERT_EQ(test::KeyStr("night", 40, kTypeValue), c_iter_->key().ToString()); + c_iter_->Next(); + ASSERT_OK(c_iter_->status()); + ASSERT_FALSE(c_iter_->Valid()); +} + +TEST_P(CompactionIteratorTest, CompactionFilterSkipUntil) { + class Filter : public CompactionFilter { + Decision FilterV2(int /*level*/, const Slice& key, ValueType t, + const Slice& existing_value, std::string* /*new_value*/, + std::string* skip_until) const override { + std::string k = key.ToString(); + std::string v = existing_value.ToString(); + // See InitIterators() call below for the sequence of keys and their + // filtering decisions. Here we closely assert that compaction filter is + // called with the expected keys and only them, and with the right values. + if (k == "a") { + EXPECT_EQ(ValueType::kValue, t); + EXPECT_EQ("av50", v); + return Decision::kKeep; + } + if (k == "b") { + EXPECT_EQ(ValueType::kValue, t); + EXPECT_EQ("bv60", v); + *skip_until = "d+"; + return Decision::kRemoveAndSkipUntil; + } + if (k == "e") { + EXPECT_EQ(ValueType::kMergeOperand, t); + EXPECT_EQ("em71", v); + return Decision::kKeep; + } + if (k == "f") { + if (v == "fm65") { + EXPECT_EQ(ValueType::kMergeOperand, t); + *skip_until = "f"; + } else { + EXPECT_EQ("fm30", v); + EXPECT_EQ(ValueType::kMergeOperand, t); + *skip_until = "g+"; + } + return Decision::kRemoveAndSkipUntil; + } + if (k == "h") { + EXPECT_EQ(ValueType::kValue, t); + EXPECT_EQ("hv91", v); + return Decision::kKeep; + } + if (k == "i") { + EXPECT_EQ(ValueType::kMergeOperand, t); + EXPECT_EQ("im95", v); + *skip_until = "z"; + return Decision::kRemoveAndSkipUntil; + } + ADD_FAILURE(); + return Decision::kKeep; + } + + const char* Name() const override { + return "CompactionIteratorTest.CompactionFilterSkipUntil::Filter"; + } + }; + + NoMergingMergeOp merge_op; + Filter filter; + InitIterators( + {test::KeyStr("a", 50, kTypeValue), // keep + test::KeyStr("a", 45, kTypeMerge), + test::KeyStr("b", 60, kTypeValue), // skip to "d+" + test::KeyStr("b", 40, kTypeValue), test::KeyStr("c", 35, kTypeValue), + test::KeyStr("d", 70, kTypeMerge), + test::KeyStr("e", 71, kTypeMerge), // keep + test::KeyStr("f", 65, kTypeMerge), // skip to "f", aka keep + test::KeyStr("f", 30, kTypeMerge), // skip to "g+" + test::KeyStr("f", 25, kTypeValue), test::KeyStr("g", 90, kTypeValue), + test::KeyStr("h", 91, kTypeValue), // keep + test::KeyStr("i", 95, kTypeMerge), // skip to "z" + test::KeyStr("j", 99, kTypeValue)}, + {"av50", "am45", "bv60", "bv40", "cv35", "dm70", "em71", "fm65", "fm30", + "fv25", "gv90", "hv91", "im95", "jv99"}, + {}, {}, kMaxSequenceNumber, kMaxSequenceNumber, &merge_op, &filter); + + // Compaction should output just "a", "e" and "h" keys. + c_iter_->SeekToFirst(); + ASSERT_TRUE(c_iter_->Valid()); + ASSERT_EQ(test::KeyStr("a", 50, kTypeValue), c_iter_->key().ToString()); + ASSERT_EQ("av50", c_iter_->value().ToString()); + c_iter_->Next(); + ASSERT_TRUE(c_iter_->Valid()); + ASSERT_EQ(test::KeyStr("e", 71, kTypeMerge), c_iter_->key().ToString()); + ASSERT_EQ("em71", c_iter_->value().ToString()); + c_iter_->Next(); + ASSERT_TRUE(c_iter_->Valid()); + ASSERT_EQ(test::KeyStr("h", 91, kTypeValue), c_iter_->key().ToString()); + ASSERT_EQ("hv91", c_iter_->value().ToString()); + c_iter_->Next(); + ASSERT_OK(c_iter_->status()); + ASSERT_FALSE(c_iter_->Valid()); + + // Check that the compaction iterator did the correct sequence of calls on + // the underlying iterator. + using A = LoggingForwardVectorIterator::Action; + using T = A::Type; + std::vector expected_actions = { + A(T::SEEK_TO_FIRST), + A(T::NEXT), + A(T::NEXT), + A(T::SEEK, test::KeyStr("d+", kMaxSequenceNumber, kValueTypeForSeek)), + A(T::NEXT), + A(T::NEXT), + A(T::SEEK, test::KeyStr("g+", kMaxSequenceNumber, kValueTypeForSeek)), + A(T::NEXT), + A(T::SEEK, test::KeyStr("z", kMaxSequenceNumber, kValueTypeForSeek))}; + ASSERT_EQ(expected_actions, iter_->log); +} + +TEST_P(CompactionIteratorTest, ShuttingDownInFilter) { + NoMergingMergeOp merge_op; + StallingFilter filter; + InitIterators( + {test::KeyStr("1", 1, kTypeValue), test::KeyStr("2", 2, kTypeValue), + test::KeyStr("3", 3, kTypeValue), test::KeyStr("4", 4, kTypeValue)}, + {"v1", "v2", "v3", "v4"}, {}, {}, kMaxSequenceNumber, kMaxSequenceNumber, + &merge_op, &filter); + // Don't leave tombstones (kTypeDeletion) for filtered keys. + compaction_proxy_->key_not_exists_beyond_output_level = true; + + std::atomic seek_done{false}; + ROCKSDB_NAMESPACE::port::Thread compaction_thread([&] { + c_iter_->SeekToFirst(); + EXPECT_FALSE(c_iter_->Valid()); + EXPECT_TRUE(c_iter_->status().IsShutdownInProgress()); + seek_done.store(true); + }); + + // Let key 1 through. + filter.WaitForStall(1); + + // Shutdown during compaction filter call for key 2. + filter.WaitForStall(2); + shutting_down_.store(true); + EXPECT_FALSE(seek_done.load()); + + // Unstall filter and wait for SeekToFirst() to return. + filter.stall_at.store(3); + compaction_thread.join(); + assert(seek_done.load()); + + // Check that filter was never called again. + EXPECT_EQ(2, filter.last_seen.load()); +} + +// Same as ShuttingDownInFilter, but shutdown happens during filter call for +// a merge operand, not for a value. +TEST_P(CompactionIteratorTest, ShuttingDownInMerge) { + NoMergingMergeOp merge_op; + StallingFilter filter; + InitIterators( + {test::KeyStr("1", 1, kTypeValue), test::KeyStr("2", 2, kTypeMerge), + test::KeyStr("3", 3, kTypeMerge), test::KeyStr("4", 4, kTypeValue)}, + {"v1", "v2", "v3", "v4"}, {}, {}, kMaxSequenceNumber, kMaxSequenceNumber, + &merge_op, &filter); + compaction_proxy_->key_not_exists_beyond_output_level = true; + + std::atomic seek_done{false}; + ROCKSDB_NAMESPACE::port::Thread compaction_thread([&] { + c_iter_->SeekToFirst(); + ASSERT_FALSE(c_iter_->Valid()); + ASSERT_TRUE(c_iter_->status().IsShutdownInProgress()); + seek_done.store(true); + }); + + // Let key 1 through. + filter.WaitForStall(1); + + // Shutdown during compaction filter call for key 2. + filter.WaitForStall(2); + shutting_down_.store(true); + EXPECT_FALSE(seek_done.load()); + + // Unstall filter and wait for SeekToFirst() to return. + filter.stall_at.store(3); + compaction_thread.join(); + assert(seek_done.load()); + + // Check that filter was never called again. + EXPECT_EQ(2, filter.last_seen.load()); +} + +TEST_P(CompactionIteratorTest, SingleMergeOperand) { + class Filter : public CompactionFilter { + Decision FilterV2(int /*level*/, const Slice& key, ValueType t, + const Slice& existing_value, std::string* /*new_value*/, + std::string* /*skip_until*/) const override { + std::string k = key.ToString(); + std::string v = existing_value.ToString(); + + // See InitIterators() call below for the sequence of keys and their + // filtering decisions. Here we closely assert that compaction filter is + // called with the expected keys and only them, and with the right values. + if (k == "a") { + EXPECT_EQ(ValueType::kMergeOperand, t); + EXPECT_EQ("av1", v); + return Decision::kKeep; + } else if (k == "b") { + EXPECT_EQ(ValueType::kMergeOperand, t); + return Decision::kKeep; + } else if (k == "c") { + return Decision::kKeep; + } + + ADD_FAILURE(); + return Decision::kKeep; + } + + const char* Name() const override { + return "CompactionIteratorTest.SingleMergeOperand::Filter"; + } + }; + + class SingleMergeOp : public MergeOperator { + public: + bool FullMergeV2(const MergeOperationInput& merge_in, + MergeOperationOutput* merge_out) const override { + // See InitIterators() call below for why "c" is the only key for which + // FullMergeV2 should be called. + EXPECT_EQ("c", merge_in.key.ToString()); + + std::string temp_value; + if (merge_in.existing_value != nullptr) { + temp_value = merge_in.existing_value->ToString(); + } + + for (auto& operand : merge_in.operand_list) { + temp_value.append(operand.ToString()); + } + merge_out->new_value = temp_value; + + return true; + } + + bool PartialMergeMulti(const Slice& key, + const std::deque& operand_list, + std::string* new_value, + Logger* /*logger*/) const override { + std::string string_key = key.ToString(); + EXPECT_TRUE(string_key == "a" || string_key == "b"); + + if (string_key == "a") { + EXPECT_EQ(1, operand_list.size()); + } else if (string_key == "b") { + EXPECT_EQ(2, operand_list.size()); + } + + std::string temp_value; + for (auto& operand : operand_list) { + temp_value.append(operand.ToString()); + } + swap(temp_value, *new_value); + + return true; + } + + const char* Name() const override { + return "CompactionIteratorTest SingleMergeOp"; + } + + bool AllowSingleOperand() const override { return true; } + }; + + SingleMergeOp merge_op; + Filter filter; + InitIterators( + // a should invoke PartialMergeMulti with a single merge operand. + {test::KeyStr("a", 50, kTypeMerge), + // b should invoke PartialMergeMulti with two operands. + test::KeyStr("b", 70, kTypeMerge), test::KeyStr("b", 60, kTypeMerge), + // c should invoke FullMerge due to kTypeValue at the beginning. + test::KeyStr("c", 90, kTypeMerge), test::KeyStr("c", 80, kTypeValue)}, + {"av1", "bv2", "bv1", "cv2", "cv1"}, {}, {}, kMaxSequenceNumber, + kMaxSequenceNumber, &merge_op, &filter); + + c_iter_->SeekToFirst(); + ASSERT_TRUE(c_iter_->Valid()); + ASSERT_EQ(test::KeyStr("a", 50, kTypeMerge), c_iter_->key().ToString()); + ASSERT_EQ("av1", c_iter_->value().ToString()); + c_iter_->Next(); + ASSERT_TRUE(c_iter_->Valid()); + ASSERT_EQ("bv1bv2", c_iter_->value().ToString()); + c_iter_->Next(); + ASSERT_OK(c_iter_->status()); + ASSERT_EQ("cv1cv2", c_iter_->value().ToString()); +} + +// In bottommost level, values earlier than earliest snapshot can be output +// with sequence = 0. +TEST_P(CompactionIteratorTest, ZeroOutSequenceAtBottomLevel) { + AddSnapshot(1); + RunTest({test::KeyStr("a", 1, kTypeValue), test::KeyStr("b", 2, kTypeValue)}, + {"v1", "v2"}, + {test::KeyStr("a", 0, kTypeValue), test::KeyStr("b", 2, kTypeValue)}, + {"v1", "v2"}, kMaxSequenceNumber /*last_committed_seq*/, + nullptr /*merge_operator*/, nullptr /*compaction_filter*/, + true /*bottommost_level*/); +} + +// In bottommost level, deletions earlier than earliest snapshot can be removed +// permanently. +TEST_P(CompactionIteratorTest, RemoveDeletionAtBottomLevel) { + AddSnapshot(1); + RunTest( + {test::KeyStr("a", 1, kTypeDeletion), test::KeyStr("b", 3, kTypeDeletion), + test::KeyStr("b", 1, kTypeValue)}, + {"", "", ""}, + {test::KeyStr("b", 3, kTypeDeletion), test::KeyStr("b", 0, kTypeValue)}, + {"", ""}, kMaxSequenceNumber /*last_committed_seq*/, + nullptr /*merge_operator*/, nullptr /*compaction_filter*/, + true /*bottommost_level*/); +} + +// In bottommost level, single deletions earlier than earliest snapshot can be +// removed permanently. +TEST_P(CompactionIteratorTest, RemoveSingleDeletionAtBottomLevel) { + AddSnapshot(1); + RunTest({test::KeyStr("a", 1, kTypeSingleDeletion), + test::KeyStr("b", 2, kTypeSingleDeletion)}, + {"", ""}, {test::KeyStr("b", 2, kTypeSingleDeletion)}, {""}, + kMaxSequenceNumber /*last_committed_seq*/, nullptr /*merge_operator*/, + nullptr /*compaction_filter*/, true /*bottommost_level*/); +} + +TEST_P(CompactionIteratorTest, ConvertToPutAtBottom) { + std::shared_ptr merge_op = + MergeOperators::CreateStringAppendOperator(); + RunTest({test::KeyStr("a", 4, kTypeMerge), test::KeyStr("a", 3, kTypeMerge), + test::KeyStr("a", 2, kTypeMerge), test::KeyStr("b", 1, kTypeValue)}, + {"a4", "a3", "a2", "b1"}, + {test::KeyStr("a", 0, kTypeValue), test::KeyStr("b", 0, kTypeValue)}, + {"a2,a3,a4", "b1"}, kMaxSequenceNumber /*last_committed_seq*/, + merge_op.get(), nullptr /*compaction_filter*/, + true /*bottomost_level*/); +} + +INSTANTIATE_TEST_CASE_P(CompactionIteratorTestInstance, CompactionIteratorTest, + testing::Values(true, false)); + +class PerKeyPlacementCompIteratorTest : public CompactionIteratorTest { + public: + bool SupportsPerKeyPlacement() const override { return true; } +}; + +TEST_P(PerKeyPlacementCompIteratorTest, SplitLastLevelData) { + std::atomic_uint64_t latest_cold_seq = 0; + + SyncPoint::GetInstance()->SetCallBack( + "CompactionIterator::PrepareOutput.context", [&](void* arg) { + auto context = static_cast(arg); + context->output_to_penultimate_level = + context->seq_num > latest_cold_seq; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + latest_cold_seq = 5; + + InitIterators( + {test::KeyStr("a", 7, kTypeValue), test::KeyStr("b", 6, kTypeValue), + test::KeyStr("c", 5, kTypeValue)}, + {"vala", "valb", "valc"}, {}, {}, kMaxSequenceNumber, kMaxSequenceNumber, + nullptr, nullptr, true); + c_iter_->SeekToFirst(); + ASSERT_TRUE(c_iter_->Valid()); + + // the first 2 keys are hot, which should has + // `output_to_penultimate_level()==true` and seq num not zeroed out + ASSERT_EQ(test::KeyStr("a", 7, kTypeValue), c_iter_->key().ToString()); + ASSERT_TRUE(c_iter_->output_to_penultimate_level()); + c_iter_->Next(); + ASSERT_TRUE(c_iter_->Valid()); + ASSERT_EQ(test::KeyStr("b", 6, kTypeValue), c_iter_->key().ToString()); + ASSERT_TRUE(c_iter_->output_to_penultimate_level()); + c_iter_->Next(); + ASSERT_TRUE(c_iter_->Valid()); + // `a` is cold data, which should be output to bottommost + ASSERT_EQ(test::KeyStr("c", 0, kTypeValue), c_iter_->key().ToString()); + ASSERT_FALSE(c_iter_->output_to_penultimate_level()); + c_iter_->Next(); + ASSERT_OK(c_iter_->status()); + ASSERT_FALSE(c_iter_->Valid()); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_P(PerKeyPlacementCompIteratorTest, SnapshotData) { + AddSnapshot(5); + + InitIterators( + {test::KeyStr("a", 7, kTypeValue), test::KeyStr("b", 6, kTypeDeletion), + test::KeyStr("b", 5, kTypeValue)}, + {"vala", "", "valb"}, {}, {}, kMaxSequenceNumber, kMaxSequenceNumber, + nullptr, nullptr, true); + c_iter_->SeekToFirst(); + ASSERT_TRUE(c_iter_->Valid()); + + // The first key and the tombstone are within snapshot, which should output + // to the penultimate level (and seq num cannot be zeroed out). + ASSERT_EQ(test::KeyStr("a", 7, kTypeValue), c_iter_->key().ToString()); + ASSERT_TRUE(c_iter_->output_to_penultimate_level()); + c_iter_->Next(); + ASSERT_TRUE(c_iter_->Valid()); + ASSERT_EQ(test::KeyStr("b", 6, kTypeDeletion), c_iter_->key().ToString()); + ASSERT_TRUE(c_iter_->output_to_penultimate_level()); + c_iter_->Next(); + ASSERT_TRUE(c_iter_->Valid()); + // `a` is not protected by the snapshot, the sequence number is zero out and + // should output bottommost + ASSERT_EQ(test::KeyStr("b", 0, kTypeValue), c_iter_->key().ToString()); + ASSERT_FALSE(c_iter_->output_to_penultimate_level()); + c_iter_->Next(); + ASSERT_OK(c_iter_->status()); + ASSERT_FALSE(c_iter_->Valid()); +} + +TEST_P(PerKeyPlacementCompIteratorTest, ConflictWithSnapshot) { + std::atomic_uint64_t latest_cold_seq = 0; + + SyncPoint::GetInstance()->SetCallBack( + "CompactionIterator::PrepareOutput.context", [&](void* arg) { + auto context = static_cast(arg); + context->output_to_penultimate_level = + context->seq_num > latest_cold_seq; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + latest_cold_seq = 6; + + AddSnapshot(5); + + InitIterators({test::KeyStr("a", 7, kTypeValue), + test::KeyStr("unsafe_pb", 6, kTypeValue), + test::KeyStr("c", 5, kTypeValue)}, + {"vala", "valb", "valc"}, {}, {}, kMaxSequenceNumber, + kMaxSequenceNumber, nullptr, nullptr, true); + c_iter_->SeekToFirst(); + ASSERT_TRUE(c_iter_->Valid()); + + ASSERT_EQ(test::KeyStr("a", 7, kTypeValue), c_iter_->key().ToString()); + ASSERT_TRUE(c_iter_->output_to_penultimate_level()); + // the 2nd key is unsafe to output_to_penultimate_level, but it's within + // snapshot so for per_key_placement feature it has to be outputted to the + // penultimate level. which is a corruption. We should never see + // such case as the data with seq num (within snapshot) should always come + // from higher compaction input level, which makes it safe to + // output_to_penultimate_level. + c_iter_->Next(); + ASSERT_TRUE(c_iter_->status().IsCorruption()); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +INSTANTIATE_TEST_CASE_P(PerKeyPlacementCompIteratorTest, + PerKeyPlacementCompIteratorTest, + testing::Values(true, false)); + +// Tests how CompactionIterator work together with SnapshotChecker. +class CompactionIteratorWithSnapshotCheckerTest + : public CompactionIteratorTest { + public: + bool UseSnapshotChecker() const override { return true; } +}; + +// Uncommitted keys (keys with seq > last_committed_seq) should be output as-is +// while committed version of these keys should get compacted as usual. + +TEST_F(CompactionIteratorWithSnapshotCheckerTest, + PreserveUncommittedKeys_Value) { + RunTest( + {test::KeyStr("foo", 3, kTypeValue), test::KeyStr("foo", 2, kTypeValue), + test::KeyStr("foo", 1, kTypeValue)}, + {"v3", "v2", "v1"}, + {test::KeyStr("foo", 3, kTypeValue), test::KeyStr("foo", 2, kTypeValue)}, + {"v3", "v2"}, 2 /*last_committed_seq*/); +} + +TEST_F(CompactionIteratorWithSnapshotCheckerTest, + PreserveUncommittedKeys_Deletion) { + RunTest({test::KeyStr("foo", 2, kTypeDeletion), + test::KeyStr("foo", 1, kTypeValue)}, + {"", "v1"}, + {test::KeyStr("foo", 2, kTypeDeletion), + test::KeyStr("foo", 1, kTypeValue)}, + {"", "v1"}, 1 /*last_committed_seq*/); +} + +TEST_F(CompactionIteratorWithSnapshotCheckerTest, + PreserveUncommittedKeys_Merge) { + auto merge_op = MergeOperators::CreateStringAppendOperator(); + RunTest( + {test::KeyStr("foo", 3, kTypeMerge), test::KeyStr("foo", 2, kTypeMerge), + test::KeyStr("foo", 1, kTypeValue)}, + {"v3", "v2", "v1"}, + {test::KeyStr("foo", 3, kTypeMerge), test::KeyStr("foo", 2, kTypeValue)}, + {"v3", "v1,v2"}, 2 /*last_committed_seq*/, merge_op.get()); +} + +TEST_F(CompactionIteratorWithSnapshotCheckerTest, + PreserveUncommittedKeys_SingleDelete) { + RunTest({test::KeyStr("foo", 2, kTypeSingleDeletion), + test::KeyStr("foo", 1, kTypeValue)}, + {"", "v1"}, + {test::KeyStr("foo", 2, kTypeSingleDeletion), + test::KeyStr("foo", 1, kTypeValue)}, + {"", "v1"}, 1 /*last_committed_seq*/); +} + +TEST_F(CompactionIteratorWithSnapshotCheckerTest, + PreserveUncommittedKeys_BlobIndex) { + RunTest({test::KeyStr("foo", 3, kTypeBlobIndex), + test::KeyStr("foo", 2, kTypeBlobIndex), + test::KeyStr("foo", 1, kTypeBlobIndex)}, + {"v3", "v2", "v1"}, + {test::KeyStr("foo", 3, kTypeBlobIndex), + test::KeyStr("foo", 2, kTypeBlobIndex)}, + {"v3", "v2"}, 2 /*last_committed_seq*/); +} + +// Test compaction iterator dedup keys visible to the same snapshot. + +TEST_F(CompactionIteratorWithSnapshotCheckerTest, DedupSameSnapshot_Value) { + AddSnapshot(2, 1); + RunTest( + {test::KeyStr("foo", 4, kTypeValue), test::KeyStr("foo", 3, kTypeValue), + test::KeyStr("foo", 2, kTypeValue), test::KeyStr("foo", 1, kTypeValue)}, + {"v4", "v3", "v2", "v1"}, + {test::KeyStr("foo", 4, kTypeValue), test::KeyStr("foo", 3, kTypeValue), + test::KeyStr("foo", 1, kTypeValue)}, + {"v4", "v3", "v1"}, 3 /*last_committed_seq*/); +} + +TEST_F(CompactionIteratorWithSnapshotCheckerTest, DedupSameSnapshot_Deletion) { + AddSnapshot(2, 1); + RunTest( + {test::KeyStr("foo", 4, kTypeValue), + test::KeyStr("foo", 3, kTypeDeletion), + test::KeyStr("foo", 2, kTypeValue), test::KeyStr("foo", 1, kTypeValue)}, + {"v4", "", "v2", "v1"}, + {test::KeyStr("foo", 4, kTypeValue), + test::KeyStr("foo", 3, kTypeDeletion), + test::KeyStr("foo", 1, kTypeValue)}, + {"v4", "", "v1"}, 3 /*last_committed_seq*/); +} + +TEST_F(CompactionIteratorWithSnapshotCheckerTest, DedupSameSnapshot_Merge) { + AddSnapshot(2, 1); + AddSnapshot(4, 3); + auto merge_op = MergeOperators::CreateStringAppendOperator(); + RunTest( + {test::KeyStr("foo", 5, kTypeMerge), test::KeyStr("foo", 4, kTypeMerge), + test::KeyStr("foo", 3, kTypeMerge), test::KeyStr("foo", 2, kTypeMerge), + test::KeyStr("foo", 1, kTypeValue)}, + {"v5", "v4", "v3", "v2", "v1"}, + {test::KeyStr("foo", 5, kTypeMerge), test::KeyStr("foo", 4, kTypeMerge), + test::KeyStr("foo", 3, kTypeMerge), test::KeyStr("foo", 1, kTypeValue)}, + {"v5", "v4", "v2,v3", "v1"}, 4 /*last_committed_seq*/, merge_op.get()); +} + +TEST_F(CompactionIteratorWithSnapshotCheckerTest, + DedupSameSnapshot_SingleDeletion) { + AddSnapshot(2, 1); + RunTest( + {test::KeyStr("foo", 4, kTypeValue), + test::KeyStr("foo", 3, kTypeSingleDeletion), + test::KeyStr("foo", 2, kTypeValue), test::KeyStr("foo", 1, kTypeValue)}, + {"v4", "", "v2", "v1"}, + {test::KeyStr("foo", 4, kTypeValue), test::KeyStr("foo", 1, kTypeValue)}, + {"v4", "v1"}, 3 /*last_committed_seq*/); +} + +TEST_F(CompactionIteratorWithSnapshotCheckerTest, DedupSameSnapshot_BlobIndex) { + AddSnapshot(2, 1); + RunTest({test::KeyStr("foo", 4, kTypeBlobIndex), + test::KeyStr("foo", 3, kTypeBlobIndex), + test::KeyStr("foo", 2, kTypeBlobIndex), + test::KeyStr("foo", 1, kTypeBlobIndex)}, + {"v4", "v3", "v2", "v1"}, + {test::KeyStr("foo", 4, kTypeBlobIndex), + test::KeyStr("foo", 3, kTypeBlobIndex), + test::KeyStr("foo", 1, kTypeBlobIndex)}, + {"v4", "v3", "v1"}, 3 /*last_committed_seq*/); +} + +// At bottom level, sequence numbers can be zero out, and deletions can be +// removed, but only when they are visible to earliest snapshot. + +TEST_F(CompactionIteratorWithSnapshotCheckerTest, + NotZeroOutSequenceIfNotVisibleToEarliestSnapshot) { + AddSnapshot(2, 1); + RunTest({test::KeyStr("a", 1, kTypeValue), test::KeyStr("b", 2, kTypeValue), + test::KeyStr("c", 3, kTypeValue)}, + {"v1", "v2", "v3"}, + {test::KeyStr("a", 0, kTypeValue), test::KeyStr("b", 2, kTypeValue), + test::KeyStr("c", 3, kTypeValue)}, + {"v1", "v2", "v3"}, kMaxSequenceNumber /*last_committed_seq*/, + nullptr /*merge_operator*/, nullptr /*compaction_filter*/, + true /*bottommost_level*/); +} + +TEST_F(CompactionIteratorWithSnapshotCheckerTest, + NotRemoveDeletionIfNotVisibleToEarliestSnapshot) { + AddSnapshot(2, 1); + RunTest( + {test::KeyStr("a", 1, kTypeDeletion), test::KeyStr("b", 2, kTypeDeletion), + test::KeyStr("c", 3, kTypeDeletion)}, + {"", "", ""}, {}, {"", ""}, kMaxSequenceNumber /*last_committed_seq*/, + nullptr /*merge_operator*/, nullptr /*compaction_filter*/, + true /*bottommost_level*/); +} + +TEST_F(CompactionIteratorWithSnapshotCheckerTest, + NotRemoveDeletionIfValuePresentToEarlierSnapshot) { + AddSnapshot(2, 1); + RunTest({test::KeyStr("a", 4, kTypeDeletion), + test::KeyStr("a", 1, kTypeValue), test::KeyStr("b", 3, kTypeValue)}, + {"", "", ""}, + {test::KeyStr("a", 4, kTypeDeletion), + test::KeyStr("a", 0, kTypeValue), test::KeyStr("b", 3, kTypeValue)}, + {"", "", ""}, kMaxSequenceNumber /*last_committed_seq*/, + nullptr /*merge_operator*/, nullptr /*compaction_filter*/, + true /*bottommost_level*/); +} + +TEST_F(CompactionIteratorWithSnapshotCheckerTest, + NotRemoveSingleDeletionIfNotVisibleToEarliestSnapshot) { + AddSnapshot(2, 1); + RunTest({test::KeyStr("a", 1, kTypeSingleDeletion), + test::KeyStr("b", 2, kTypeSingleDeletion), + test::KeyStr("c", 3, kTypeSingleDeletion)}, + {"", "", ""}, + {test::KeyStr("b", 2, kTypeSingleDeletion), + test::KeyStr("c", 3, kTypeSingleDeletion)}, + {"", ""}, kMaxSequenceNumber /*last_committed_seq*/, + nullptr /*merge_operator*/, nullptr /*compaction_filter*/, + true /*bottommost_level*/); +} + +// Single delete should not cancel out values that not visible to the +// same set of snapshots +TEST_F(CompactionIteratorWithSnapshotCheckerTest, + SingleDeleteAcrossSnapshotBoundary) { + AddSnapshot(2, 1); + RunTest({test::KeyStr("a", 2, kTypeSingleDeletion), + test::KeyStr("a", 1, kTypeValue)}, + {"", "v1"}, + {test::KeyStr("a", 2, kTypeSingleDeletion), + test::KeyStr("a", 1, kTypeValue)}, + {"", "v1"}, 2 /*last_committed_seq*/); +} + +// Single delete should be kept in case it is not visible to the +// earliest write conflict snapshot. If a single delete is kept for this reason, +// corresponding value can be trimmed to save space. +TEST_F(CompactionIteratorWithSnapshotCheckerTest, + KeepSingleDeletionForWriteConflictChecking) { + AddSnapshot(2, 0); + RunTest({test::KeyStr("a", 2, kTypeSingleDeletion), + test::KeyStr("a", 1, kTypeValue)}, + {"", "v1"}, + {test::KeyStr("a", 2, kTypeSingleDeletion), + test::KeyStr("a", 1, kTypeValue)}, + {"", ""}, 2 /*last_committed_seq*/, nullptr /*merge_operator*/, + nullptr /*compaction_filter*/, false /*bottommost_level*/, + 2 /*earliest_write_conflict_snapshot*/); +} + +// Same as above but with a blob index. In addition to the value getting +// trimmed, the type of the KV is changed to kTypeValue. +TEST_F(CompactionIteratorWithSnapshotCheckerTest, + KeepSingleDeletionForWriteConflictChecking_BlobIndex) { + AddSnapshot(2, 0); + RunTest({test::KeyStr("a", 2, kTypeSingleDeletion), + test::KeyStr("a", 1, kTypeBlobIndex)}, + {"", "fake_blob_index"}, + {test::KeyStr("a", 2, kTypeSingleDeletion), + test::KeyStr("a", 1, kTypeValue)}, + {"", ""}, 2 /*last_committed_seq*/, nullptr /*merge_operator*/, + nullptr /*compaction_filter*/, false /*bottommost_level*/, + 2 /*earliest_write_conflict_snapshot*/); +} + +// Same as above but with a wide-column entity. In addition to the value getting +// trimmed, the type of the KV is changed to kTypeValue. +TEST_F(CompactionIteratorWithSnapshotCheckerTest, + KeepSingleDeletionForWriteConflictChecking_WideColumnEntity) { + AddSnapshot(2, 0); + RunTest({test::KeyStr("a", 2, kTypeSingleDeletion), + test::KeyStr("a", 1, kTypeWideColumnEntity)}, + {"", "fake_entity"}, + {test::KeyStr("a", 2, kTypeSingleDeletion), + test::KeyStr("a", 1, kTypeValue)}, + {"", ""}, 2 /* last_committed_seq */, nullptr /* merge_operator */, + nullptr /* compaction_filter */, false /* bottommost_level */, + 2 /* earliest_write_conflict_snapshot */); +} + +// Compaction filter should keep uncommitted key as-is, and +// * Convert the latest value to deletion, and/or +// * if latest value is a merge, apply filter to all subsequent merges. + +TEST_F(CompactionIteratorWithSnapshotCheckerTest, CompactionFilter_Value) { + std::unique_ptr compaction_filter( + new FilterAllKeysCompactionFilter()); + RunTest( + {test::KeyStr("a", 2, kTypeValue), test::KeyStr("a", 1, kTypeValue), + test::KeyStr("b", 3, kTypeValue), test::KeyStr("c", 1, kTypeValue)}, + {"v2", "v1", "v3", "v4"}, + {test::KeyStr("a", 2, kTypeValue), test::KeyStr("a", 1, kTypeDeletion), + test::KeyStr("b", 3, kTypeValue), test::KeyStr("c", 1, kTypeDeletion)}, + {"v2", "", "v3", ""}, 1 /*last_committed_seq*/, + nullptr /*merge_operator*/, compaction_filter.get()); +} + +TEST_F(CompactionIteratorWithSnapshotCheckerTest, CompactionFilter_Deletion) { + std::unique_ptr compaction_filter( + new FilterAllKeysCompactionFilter()); + RunTest( + {test::KeyStr("a", 2, kTypeDeletion), test::KeyStr("a", 1, kTypeValue)}, + {"", "v1"}, + {test::KeyStr("a", 2, kTypeDeletion), + test::KeyStr("a", 1, kTypeDeletion)}, + {"", ""}, 1 /*last_committed_seq*/, nullptr /*merge_operator*/, + compaction_filter.get()); +} + +TEST_F(CompactionIteratorWithSnapshotCheckerTest, + CompactionFilter_PartialMerge) { + std::shared_ptr merge_op = + MergeOperators::CreateStringAppendOperator(); + std::unique_ptr compaction_filter( + new FilterAllKeysCompactionFilter()); + RunTest({test::KeyStr("a", 3, kTypeMerge), test::KeyStr("a", 2, kTypeMerge), + test::KeyStr("a", 1, kTypeMerge)}, + {"v3", "v2", "v1"}, {test::KeyStr("a", 3, kTypeMerge)}, {"v3"}, + 2 /*last_committed_seq*/, merge_op.get(), compaction_filter.get()); +} + +TEST_F(CompactionIteratorWithSnapshotCheckerTest, CompactionFilter_FullMerge) { + std::shared_ptr merge_op = + MergeOperators::CreateStringAppendOperator(); + std::unique_ptr compaction_filter( + new FilterAllKeysCompactionFilter()); + RunTest( + {test::KeyStr("a", 3, kTypeMerge), test::KeyStr("a", 2, kTypeMerge), + test::KeyStr("a", 1, kTypeValue)}, + {"v3", "v2", "v1"}, + {test::KeyStr("a", 3, kTypeMerge), test::KeyStr("a", 1, kTypeDeletion)}, + {"v3", ""}, 2 /*last_committed_seq*/, merge_op.get(), + compaction_filter.get()); +} + +// Tests how CompactionIterator work together with AllowIngestBehind. +class CompactionIteratorWithAllowIngestBehindTest + : public CompactionIteratorTest { + public: + bool AllowIngestBehind() const override { return true; } +}; + +// When allow_ingest_behind is set, compaction iterator is not targeting +// the bottommost level since there is no guarantee there won't be further +// data ingested under the compaction output in future. +TEST_P(CompactionIteratorWithAllowIngestBehindTest, NoConvertToPutAtBottom) { + std::shared_ptr merge_op = + MergeOperators::CreateStringAppendOperator(); + RunTest({test::KeyStr("a", 4, kTypeMerge), test::KeyStr("a", 3, kTypeMerge), + test::KeyStr("a", 2, kTypeMerge), test::KeyStr("b", 1, kTypeValue)}, + {"a4", "a3", "a2", "b1"}, + {test::KeyStr("a", 4, kTypeMerge), test::KeyStr("b", 1, kTypeValue)}, + {"a2,a3,a4", "b1"}, kMaxSequenceNumber /*last_committed_seq*/, + merge_op.get(), nullptr /*compaction_filter*/, + true /*bottomost_level*/); +} + +TEST_P(CompactionIteratorWithAllowIngestBehindTest, + MergeToPutIfEncounteredPutAtBottom) { + std::shared_ptr merge_op = + MergeOperators::CreateStringAppendOperator(); + RunTest({test::KeyStr("a", 4, kTypeMerge), test::KeyStr("a", 3, kTypeMerge), + test::KeyStr("a", 2, kTypeValue), test::KeyStr("b", 1, kTypeValue)}, + {"a4", "a3", "a2", "b1"}, + {test::KeyStr("a", 4, kTypeValue), test::KeyStr("b", 1, kTypeValue)}, + {"a2,a3,a4", "b1"}, kMaxSequenceNumber /*last_committed_seq*/, + merge_op.get(), nullptr /*compaction_filter*/, + true /*bottomost_level*/); +} + +INSTANTIATE_TEST_CASE_P(CompactionIteratorWithAllowIngestBehindTestInstance, + CompactionIteratorWithAllowIngestBehindTest, + testing::Values(true, false)); + +class CompactionIteratorTsGcTest : public CompactionIteratorTest { + public: + CompactionIteratorTsGcTest() + : CompactionIteratorTest(test::BytewiseComparatorWithU64TsWrapper()) {} +}; + +TEST_P(CompactionIteratorTsGcTest, NoKeyEligibleForGC) { + constexpr char user_key[][2] = {{'a', '\0'}, {'b', '\0'}}; + const std::vector input_keys = { + test::KeyStr(/*ts=*/103, user_key[0], /*seq=*/4, kTypeValue), + test::KeyStr(/*ts=*/102, user_key[0], /*seq=*/3, + kTypeDeletionWithTimestamp), + test::KeyStr(/*ts=*/104, user_key[1], /*seq=*/5, kTypeValue)}; + const std::vector input_values = {"a3", "", "b2"}; + std::string full_history_ts_low; + // All keys' timestamps are newer than or equal to 102, thus none of them + // will be eligible for GC. + PutFixed64(&full_history_ts_low, 102); + const std::vector& expected_keys = input_keys; + const std::vector& expected_values = input_values; + const std::vector> params = { + {false, false}, {false, true}, {true, true}}; + for (const std::pair& param : params) { + const bool bottommost_level = param.first; + const bool key_not_exists_beyond_output_level = param.second; + RunTest(input_keys, input_values, expected_keys, expected_values, + /*last_committed_seq=*/kMaxSequenceNumber, + /*merge_operator=*/nullptr, /*compaction_filter=*/nullptr, + bottommost_level, + /*earliest_write_conflict_snapshot=*/kMaxSequenceNumber, + key_not_exists_beyond_output_level, &full_history_ts_low); + } +} + +TEST_P(CompactionIteratorTsGcTest, NoMergeEligibleForGc) { + constexpr char user_key[] = "a"; + const std::vector input_keys = { + test::KeyStr(10002, user_key, 102, kTypeMerge), + test::KeyStr(10001, user_key, 101, kTypeMerge), + test::KeyStr(10000, user_key, 100, kTypeValue)}; + const std::vector input_values = {"2", "1", "a0"}; + std::shared_ptr merge_op = + MergeOperators::CreateStringAppendTESTOperator(); + const std::vector& expected_keys = input_keys; + const std::vector& expected_values = input_values; + const std::vector> params = { + {false, false}, {false, true}, {true, true}}; + for (const auto& param : params) { + const bool bottommost_level = param.first; + const bool key_not_exists_beyond_output_level = param.second; + RunTest(input_keys, input_values, expected_keys, expected_values, + /*last_committed_seq=*/kMaxSequenceNumber, merge_op.get(), + /*compaction_filter=*/nullptr, bottommost_level, + /*earliest_write_conflict_snapshot=*/kMaxSequenceNumber, + key_not_exists_beyond_output_level, + /*full_history_ts_low=*/nullptr); + } +} + +TEST_P(CompactionIteratorTsGcTest, AllKeysOlderThanThreshold) { + constexpr char user_key[][2] = {{'a', '\0'}, {'b', '\0'}}; + const std::vector input_keys = { + test::KeyStr(/*ts=*/103, user_key[0], /*seq=*/4, + kTypeDeletionWithTimestamp), + test::KeyStr(/*ts=*/102, user_key[0], /*seq=*/3, kTypeValue), + test::KeyStr(/*ts=*/101, user_key[0], /*seq=*/2, kTypeValue), + test::KeyStr(/*ts=*/104, user_key[1], /*seq=*/5, kTypeValue)}; + const std::vector input_values = {"", "a2", "a1", "b5"}; + std::string full_history_ts_low; + PutFixed64(&full_history_ts_low, std::numeric_limits::max()); + { + // With a snapshot at seq 3, both the deletion marker and the key at 3 must + // be preserved. + AddSnapshot(3); + const std::vector expected_keys = { + input_keys[0], input_keys[1], input_keys[3]}; + const std::vector expected_values = {"", "a2", "b5"}; + RunTest(input_keys, input_values, expected_keys, expected_values, + /*last_committed_seq=*/kMaxSequenceNumber, + /*merge_operator=*/nullptr, /*compaction_filter=*/nullptr, + /*bottommost_level=*/false, + /*earliest_write_conflict_snapshot=*/kMaxSequenceNumber, + /*key_not_exists_beyond_output_level=*/false, &full_history_ts_low); + ClearSnapshots(); + } + { + // No snapshot, the deletion marker should be preserved because the user + // key may appear beyond output level. + const std::vector expected_keys = {input_keys[0], + input_keys[3]}; + const std::vector expected_values = {"", "b5"}; + RunTest(input_keys, input_values, expected_keys, expected_values, + /*last_committed_seq=*/kMaxSequenceNumber, + /*merge_operator=*/nullptr, /*compaction_filter=*/nullptr, + /*bottommost_level=*/false, + /*earliest_write_conflict_snapshot=*/kMaxSequenceNumber, + /*key_not_exists_beyond_output_level=*/false, &full_history_ts_low); + } + { + // No snapshot, the deletion marker can be dropped because the user key + // does not appear in higher levels. + const std::vector expected_keys = {input_keys[3]}; + const std::vector expected_values = {"b5"}; + RunTest(input_keys, input_values, expected_keys, expected_values, + /*last_committed_seq=*/kMaxSequenceNumber, + /*merge_operator=*/nullptr, /*compaction_filter=*/nullptr, + /*bottommost_level=*/false, + /*earliest_write_conflict_snapshot=*/kMaxSequenceNumber, + /*key_not_exists_beyond_output_level=*/true, &full_history_ts_low); + } +} + +TEST_P(CompactionIteratorTsGcTest, SomeMergesOlderThanThreshold) { + constexpr char user_key[][2] = {"a", "f"}; + const std::vector input_keys = { + test::KeyStr(/*ts=*/25000, user_key[0], /*seq=*/2500, kTypeMerge), + test::KeyStr(/*ts=*/19000, user_key[0], /*seq=*/2300, kTypeMerge), + test::KeyStr(/*ts=*/18000, user_key[0], /*seq=*/1800, kTypeMerge), + test::KeyStr(/*ts=*/16000, user_key[0], /*seq=*/1600, kTypeValue), + test::KeyStr(/*ts=*/19000, user_key[1], /*seq=*/2000, kTypeMerge), + test::KeyStr(/*ts=*/17000, user_key[1], /*seq=*/1700, kTypeMerge), + test::KeyStr(/*ts=*/15000, user_key[1], /*seq=*/1600, + kTypeDeletionWithTimestamp)}; + const std::vector input_values = {"25", "19", "18", "16", + "19", "17", ""}; + std::shared_ptr merge_op = + MergeOperators::CreateStringAppendTESTOperator(); + std::string full_history_ts_low; + PutFixed64(&full_history_ts_low, 20000); + + const std::vector> params = { + {false, false}, {false, true}, {true, true}}; + + { + AddSnapshot(1600); + AddSnapshot(1900); + const std::vector expected_keys = { + test::KeyStr(/*ts=*/25000, user_key[0], /*seq=*/2500, kTypeMerge), + test::KeyStr(/*ts=*/19000, user_key[0], /*seq=*/2300, kTypeMerge), + test::KeyStr(/*ts=*/18000, user_key[0], /*seq=*/1800, kTypeMerge), + test::KeyStr(/*ts=*/16000, user_key[0], /*seq=*/1600, kTypeValue), + test::KeyStr(/*ts=*/19000, user_key[1], /*seq=*/2000, kTypeMerge), + test::KeyStr(/*ts=*/17000, user_key[1], /*seq=*/1700, kTypeMerge), + test::KeyStr(/*ts=*/15000, user_key[1], /*seq=*/1600, + kTypeDeletionWithTimestamp)}; + const std::vector expected_values = {"25", "19", "18", "16", + "19", "17", ""}; + for (const auto& param : params) { + const bool bottommost_level = param.first; + const bool key_not_exists_beyond_output_level = param.second; + auto expected_keys_copy = expected_keys; + auto expected_values_copy = expected_values; + if (bottommost_level || key_not_exists_beyond_output_level) { + // the kTypeDeletionWithTimestamp will be dropped + expected_keys_copy.pop_back(); + expected_values_copy.pop_back(); + if (bottommost_level) { + // seq zero + expected_keys_copy[3] = + test::KeyStr(/*ts=*/0, user_key[0], /*seq=*/0, kTypeValue); + } + } + RunTest(input_keys, input_values, expected_keys_copy, + expected_values_copy, + /*last_committed_seq=*/kMaxSequenceNumber, merge_op.get(), + /*compaction_filter=*/nullptr, bottommost_level, + /*earliest_write_conflict_snapshot=*/kMaxSequenceNumber, + key_not_exists_beyond_output_level, &full_history_ts_low); + } + ClearSnapshots(); + } + + // No snapshots + { + const std::vector expected_keys = { + test::KeyStr(/*ts=*/25000, user_key[0], /*seq=*/2500, kTypeValue), + test::KeyStr(/*ts=*/19000, user_key[1], /*seq=*/2000, kTypeValue)}; + const std::vector expected_values = {"16,18,19,25", "17,19"}; + for (const auto& param : params) { + const bool bottommost_level = param.first; + const bool key_not_exists_beyond_output_level = param.second; + auto expected_keys_copy = expected_keys; + auto expected_values_copy = expected_values; + if (bottommost_level) { + expected_keys_copy[1] = + test::KeyStr(/*ts=*/0, user_key[1], /*seq=*/0, kTypeValue); + } + RunTest(input_keys, input_values, expected_keys_copy, + expected_values_copy, + /*last_committed_seq=*/kMaxSequenceNumber, merge_op.get(), + /*compaction_filter=*/nullptr, bottommost_level, + /*earliest_write_conflict_snapshot=*/kMaxSequenceNumber, + key_not_exists_beyond_output_level, &full_history_ts_low); + } + } +} + +TEST_P(CompactionIteratorTsGcTest, NewHidesOldSameSnapshot) { + constexpr char user_key[] = "a"; + const std::vector input_keys = { + test::KeyStr(/*ts=*/103, user_key, /*seq=*/4, kTypeDeletionWithTimestamp), + test::KeyStr(/*ts=*/102, user_key, /*seq=*/3, kTypeValue), + test::KeyStr(/*ts=*/101, user_key, /*seq=*/2, kTypeValue), + test::KeyStr(/*ts=*/100, user_key, /*seq=*/1, kTypeValue)}; + const std::vector input_values = {"", "a2", "a1", "a0"}; + { + std::string full_history_ts_low; + // Keys whose timestamps larger than or equal to 102 will be preserved. + PutFixed64(&full_history_ts_low, 102); + const std::vector expected_keys = { + input_keys[0], input_keys[1], input_keys[2]}; + const std::vector expected_values = {"", input_values[1], + input_values[2]}; + RunTest(input_keys, input_values, expected_keys, expected_values, + /*last_committed_seq=*/kMaxSequenceNumber, + /*merge_operator=*/nullptr, /*compaction_filter=*/nullptr, + /*bottommost_level=*/false, + /*earliest_write_conflict_snapshot=*/kMaxSequenceNumber, + /*key_not_exists_beyond_output_level=*/false, &full_history_ts_low); + } +} + +TEST_P(CompactionIteratorTsGcTest, DropTombstones) { + constexpr char user_key[] = "a"; + const std::vector input_keys = { + test::KeyStr(/*ts=*/103, user_key, /*seq=*/4, kTypeDeletionWithTimestamp), + test::KeyStr(/*ts=*/102, user_key, /*seq=*/3, kTypeValue), + test::KeyStr(/*ts=*/101, user_key, /*seq=*/2, kTypeDeletionWithTimestamp), + test::KeyStr(/*ts=*/100, user_key, /*seq=*/1, kTypeValue)}; + const std::vector input_values = {"", "a2", "", "a0"}; + const std::vector expected_keys = {input_keys[0], input_keys[1]}; + const std::vector expected_values = {"", "a2"}; + + // Take a snapshot at seq 2. + AddSnapshot(2); + + { + // Non-bottommost level, but key does not exist beyond output level. + std::string full_history_ts_low; + PutFixed64(&full_history_ts_low, 102); + RunTest(input_keys, input_values, expected_keys, expected_values, + /*last_committed_sequence=*/kMaxSequenceNumber, + /*merge_op=*/nullptr, /*compaction_filter=*/nullptr, + /*bottommost_level=*/false, + /*earliest_write_conflict_snapshot=*/kMaxSequenceNumber, + /*key_not_exists_beyond_output_level=*/true, &full_history_ts_low); + } + { + // Bottommost level + std::string full_history_ts_low; + PutFixed64(&full_history_ts_low, 102); + RunTest(input_keys, input_values, expected_keys, expected_values, + /*last_committed_seq=*/kMaxSequenceNumber, + /*merge_operator=*/nullptr, /*compaction_filter=*/nullptr, + /*bottommost_level=*/true, + /*earliest_write_conflict_snapshot=*/kMaxSequenceNumber, + /*key_not_exists_beyond_output_level=*/false, &full_history_ts_low); + } +} + +TEST_P(CompactionIteratorTsGcTest, RewriteTs) { + constexpr char user_key[] = "a"; + const std::vector input_keys = { + test::KeyStr(/*ts=*/103, user_key, /*seq=*/4, kTypeDeletionWithTimestamp), + test::KeyStr(/*ts=*/102, user_key, /*seq=*/3, kTypeValue), + test::KeyStr(/*ts=*/101, user_key, /*seq=*/2, kTypeDeletionWithTimestamp), + test::KeyStr(/*ts=*/100, user_key, /*seq=*/1, kTypeValue)}; + const std::vector input_values = {"", "a2", "", "a0"}; + const std::vector expected_keys = { + input_keys[0], input_keys[1], input_keys[2], + test::KeyStr(/*ts=*/0, user_key, /*seq=*/0, kTypeValue)}; + const std::vector expected_values = {"", "a2", "", "a0"}; + + AddSnapshot(1); + AddSnapshot(2); + + { + // Bottommost level and need to rewrite both ts and seq. + std::string full_history_ts_low; + PutFixed64(&full_history_ts_low, 102); + RunTest(input_keys, input_values, expected_keys, expected_values, + /*last_committed_seq=*/kMaxSequenceNumber, + /*merge_operator=*/nullptr, /*compaction_filter=*/nullptr, + /*bottommost_level=*/true, + /*earliest_write_conflict_snapshot=*/kMaxSequenceNumber, + /*key_not_exists_beyond_output_level=*/true, &full_history_ts_low); + } +} + +TEST_P(CompactionIteratorTsGcTest, SingleDeleteNoKeyEligibleForGC) { + constexpr char user_key[][2] = {{'a', '\0'}, {'b', '\0'}}; + const std::vector input_keys = { + test::KeyStr(/*ts=*/104, user_key[0], /*seq=*/4, kTypeSingleDeletion), + test::KeyStr(/*ts=*/103, user_key[0], /*seq=*/3, kTypeValue), + test::KeyStr(/*ts=*/102, user_key[1], /*seq=*/2, kTypeValue)}; + const std::vector input_values = {"", "a3", "b2"}; + std::string full_history_ts_low; + // All keys' timestamps are newer than or equal to 102, thus none of them + // will be eligible for GC. + PutFixed64(&full_history_ts_low, 102); + const std::vector& expected_keys = input_keys; + const std::vector& expected_values = input_values; + const std::vector> params = { + {false, false}, {false, true}, {true, true}}; + for (const std::pair& param : params) { + const bool bottommost_level = param.first; + const bool key_not_exists_beyond_output_level = param.second; + RunTest(input_keys, input_values, expected_keys, expected_values, + /*last_committed_seq=*/kMaxSequenceNumber, + /*merge_operator=*/nullptr, /*compaction_filter=*/nullptr, + bottommost_level, + /*earliest_write_conflict_snapshot=*/kMaxSequenceNumber, + key_not_exists_beyond_output_level, &full_history_ts_low); + } +} + +TEST_P(CompactionIteratorTsGcTest, SingleDeleteDropTombstones) { + constexpr char user_key[] = "a"; + const std::vector input_keys = { + test::KeyStr(/*ts=*/103, user_key, /*seq=*/4, kTypeSingleDeletion), + test::KeyStr(/*ts=*/102, user_key, /*seq=*/3, kTypeValue), + test::KeyStr(/*ts=*/101, user_key, /*seq=*/2, kTypeSingleDeletion), + test::KeyStr(/*ts=*/100, user_key, /*seq=*/1, kTypeValue)}; + const std::vector input_values = {"", "a2", "", "a0"}; + const std::vector expected_keys = {input_keys[0], input_keys[1]}; + const std::vector expected_values = {"", "a2"}; + + // Take a snapshot at seq 2. + AddSnapshot(2); + { + const std::vector> params = { + {false, false}, {false, true}, {true, true}}; + for (const std::pair& param : params) { + const bool bottommost_level = param.first; + const bool key_not_exists_beyond_output_level = param.second; + std::string full_history_ts_low; + PutFixed64(&full_history_ts_low, 102); + RunTest(input_keys, input_values, expected_keys, expected_values, + /*last_committed_seq=*/kMaxSequenceNumber, + /*merge_operator=*/nullptr, /*compaction_filter=*/nullptr, + bottommost_level, + /*earliest_write_conflict_snapshot=*/kMaxSequenceNumber, + key_not_exists_beyond_output_level, &full_history_ts_low); + } + } +} + +TEST_P(CompactionIteratorTsGcTest, SingleDeleteAllKeysOlderThanThreshold) { + constexpr char user_key[][2] = {{'a', '\0'}, {'b', '\0'}}; + const std::vector input_keys = { + test::KeyStr(/*ts=*/103, user_key[0], /*seq=*/4, kTypeSingleDeletion), + test::KeyStr(/*ts=*/102, user_key[0], /*seq=*/3, kTypeValue), + test::KeyStr(/*ts=*/104, user_key[1], /*seq=*/5, kTypeValue)}; + const std::vector input_values = {"", "a2", "b5"}; + std::string full_history_ts_low; + PutFixed64(&full_history_ts_low, std::numeric_limits::max()); + { + // With a snapshot at seq 3, both the deletion marker and the key at 3 must + // be preserved. + AddSnapshot(3); + const std::vector expected_keys = { + input_keys[0], input_keys[1], input_keys[2]}; + const std::vector expected_values = {"", "a2", "b5"}; + RunTest(input_keys, input_values, expected_keys, expected_values, + /*last_committed_seq=*/kMaxSequenceNumber, + /*merge_operator=*/nullptr, /*compaction_filter=*/nullptr, + /*bottommost_level=*/false, + /*earliest_write_conflict_snapshot=*/kMaxSequenceNumber, + /*key_not_exists_beyond_output_level=*/false, &full_history_ts_low); + ClearSnapshots(); + } + { + // No snapshot. + const std::vector expected_keys = {input_keys[2]}; + const std::vector expected_values = {"b5"}; + RunTest(input_keys, input_values, expected_keys, expected_values, + /*last_committed_seq=*/kMaxSequenceNumber, + /*merge_operator=*/nullptr, /*compaction_filter=*/nullptr, + /*bottommost_level=*/false, + /*earliest_write_conflict_snapshot=*/kMaxSequenceNumber, + /*key_not_exists_beyond_output_level=*/false, &full_history_ts_low); + } +} + +INSTANTIATE_TEST_CASE_P(CompactionIteratorTsGcTestInstance, + CompactionIteratorTsGcTest, + testing::Values(true, false)); + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_job.cc b/librocksdb-sys/rocksdb/db/compaction/compaction_job.cc new file mode 100644 index 0000000..8ea8068 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_job.cc @@ -0,0 +1,2118 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/compaction/compaction_job.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "db/blob/blob_counting_iterator.h" +#include "db/blob/blob_file_addition.h" +#include "db/blob/blob_file_builder.h" +#include "db/builder.h" +#include "db/compaction/clipping_iterator.h" +#include "db/compaction/compaction_state.h" +#include "db/db_impl/db_impl.h" +#include "db/dbformat.h" +#include "db/error_handler.h" +#include "db/event_helpers.h" +#include "db/history_trimming_iterator.h" +#include "db/log_writer.h" +#include "db/merge_helper.h" +#include "db/range_del_aggregator.h" +#include "db/version_edit.h" +#include "db/version_set.h" +#include "file/filename.h" +#include "file/read_write_util.h" +#include "file/sst_file_manager_impl.h" +#include "file/writable_file_writer.h" +#include "logging/log_buffer.h" +#include "logging/logging.h" +#include "monitoring/iostats_context_imp.h" +#include "monitoring/thread_status_util.h" +#include "options/configurable_helper.h" +#include "options/options_helper.h" +#include "port/port.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "rocksdb/statistics.h" +#include "rocksdb/status.h" +#include "rocksdb/table.h" +#include "rocksdb/utilities/options_type.h" +#include "table/merging_iterator.h" +#include "table/table_builder.h" +#include "table/unique_id_impl.h" +#include "test_util/sync_point.h" +#include "util/stop_watch.h" + +namespace ROCKSDB_NAMESPACE { + +const char* GetCompactionReasonString(CompactionReason compaction_reason) { + switch (compaction_reason) { + case CompactionReason::kUnknown: + return "Unknown"; + case CompactionReason::kLevelL0FilesNum: + return "LevelL0FilesNum"; + case CompactionReason::kLevelMaxLevelSize: + return "LevelMaxLevelSize"; + case CompactionReason::kUniversalSizeAmplification: + return "UniversalSizeAmplification"; + case CompactionReason::kUniversalSizeRatio: + return "UniversalSizeRatio"; + case CompactionReason::kUniversalSortedRunNum: + return "UniversalSortedRunNum"; + case CompactionReason::kFIFOMaxSize: + return "FIFOMaxSize"; + case CompactionReason::kFIFOReduceNumFiles: + return "FIFOReduceNumFiles"; + case CompactionReason::kFIFOTtl: + return "FIFOTtl"; + case CompactionReason::kManualCompaction: + return "ManualCompaction"; + case CompactionReason::kFilesMarkedForCompaction: + return "FilesMarkedForCompaction"; + case CompactionReason::kBottommostFiles: + return "BottommostFiles"; + case CompactionReason::kTtl: + return "Ttl"; + case CompactionReason::kFlush: + return "Flush"; + case CompactionReason::kExternalSstIngestion: + return "ExternalSstIngestion"; + case CompactionReason::kPeriodicCompaction: + return "PeriodicCompaction"; + case CompactionReason::kChangeTemperature: + return "ChangeTemperature"; + case CompactionReason::kForcedBlobGC: + return "ForcedBlobGC"; + case CompactionReason::kRoundRobinTtl: + return "RoundRobinTtl"; + case CompactionReason::kRefitLevel: + return "RefitLevel"; + case CompactionReason::kNumOfReasons: + // fall through + default: + assert(false); + return "Invalid"; + } +} + +const char* GetCompactionPenultimateOutputRangeTypeString( + Compaction::PenultimateOutputRangeType range_type) { + switch (range_type) { + case Compaction::PenultimateOutputRangeType::kNotSupported: + return "NotSupported"; + case Compaction::PenultimateOutputRangeType::kFullRange: + return "FullRange"; + case Compaction::PenultimateOutputRangeType::kNonLastRange: + return "NonLastRange"; + case Compaction::PenultimateOutputRangeType::kDisabled: + return "Disabled"; + default: + assert(false); + return "Invalid"; + } +} + +CompactionJob::CompactionJob( + int job_id, Compaction* compaction, const ImmutableDBOptions& db_options, + const MutableDBOptions& mutable_db_options, const FileOptions& file_options, + VersionSet* versions, const std::atomic* shutting_down, + LogBuffer* log_buffer, FSDirectory* db_directory, + FSDirectory* output_directory, FSDirectory* blob_output_directory, + Statistics* stats, InstrumentedMutex* db_mutex, + ErrorHandler* db_error_handler, + std::vector existing_snapshots, + SequenceNumber earliest_write_conflict_snapshot, + const SnapshotChecker* snapshot_checker, JobContext* job_context, + std::shared_ptr table_cache, EventLogger* event_logger, + bool paranoid_file_checks, bool measure_io_stats, const std::string& dbname, + CompactionJobStats* compaction_job_stats, Env::Priority thread_pri, + const std::shared_ptr& io_tracer, + const std::atomic& manual_compaction_canceled, + const std::string& db_id, const std::string& db_session_id, + std::string full_history_ts_low, std::string trim_ts, + BlobFileCompletionCallback* blob_callback, int* bg_compaction_scheduled, + int* bg_bottom_compaction_scheduled) + : compact_(new CompactionState(compaction)), + compaction_stats_(compaction->compaction_reason(), 1), + db_options_(db_options), + mutable_db_options_copy_(mutable_db_options), + log_buffer_(log_buffer), + output_directory_(output_directory), + stats_(stats), + bottommost_level_(false), + write_hint_(Env::WLTH_NOT_SET), + compaction_job_stats_(compaction_job_stats), + job_id_(job_id), + dbname_(dbname), + db_id_(db_id), + db_session_id_(db_session_id), + file_options_(file_options), + env_(db_options.env), + io_tracer_(io_tracer), + fs_(db_options.fs, io_tracer), + file_options_for_read_( + fs_->OptimizeForCompactionTableRead(file_options, db_options_)), + versions_(versions), + shutting_down_(shutting_down), + manual_compaction_canceled_(manual_compaction_canceled), + db_directory_(db_directory), + blob_output_directory_(blob_output_directory), + db_mutex_(db_mutex), + db_error_handler_(db_error_handler), + existing_snapshots_(std::move(existing_snapshots)), + earliest_write_conflict_snapshot_(earliest_write_conflict_snapshot), + snapshot_checker_(snapshot_checker), + job_context_(job_context), + table_cache_(std::move(table_cache)), + event_logger_(event_logger), + paranoid_file_checks_(paranoid_file_checks), + measure_io_stats_(measure_io_stats), + thread_pri_(thread_pri), + full_history_ts_low_(std::move(full_history_ts_low)), + trim_ts_(std::move(trim_ts)), + blob_callback_(blob_callback), + extra_num_subcompaction_threads_reserved_(0), + bg_compaction_scheduled_(bg_compaction_scheduled), + bg_bottom_compaction_scheduled_(bg_bottom_compaction_scheduled) { + assert(compaction_job_stats_ != nullptr); + assert(log_buffer_ != nullptr); + + const auto* cfd = compact_->compaction->column_family_data(); + ThreadStatusUtil::SetEnableTracking(db_options_.enable_thread_tracking); + ThreadStatusUtil::SetColumnFamily(cfd); + ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_COMPACTION); + ReportStartedCompaction(compaction); +} + +CompactionJob::~CompactionJob() { + assert(compact_ == nullptr); + ThreadStatusUtil::ResetThreadStatus(); +} + +void CompactionJob::ReportStartedCompaction(Compaction* compaction) { + ThreadStatusUtil::SetThreadOperationProperty(ThreadStatus::COMPACTION_JOB_ID, + job_id_); + + ThreadStatusUtil::SetThreadOperationProperty( + ThreadStatus::COMPACTION_INPUT_OUTPUT_LEVEL, + (static_cast(compact_->compaction->start_level()) << 32) + + compact_->compaction->output_level()); + + // In the current design, a CompactionJob is always created + // for non-trivial compaction. + assert(compaction->IsTrivialMove() == false || + compaction->is_manual_compaction() == true); + + ThreadStatusUtil::SetThreadOperationProperty( + ThreadStatus::COMPACTION_PROP_FLAGS, + compaction->is_manual_compaction() + + (compaction->deletion_compaction() << 1)); + + ThreadStatusUtil::SetThreadOperationProperty( + ThreadStatus::COMPACTION_TOTAL_INPUT_BYTES, + compaction->CalculateTotalInputSize()); + + IOSTATS_RESET(bytes_written); + IOSTATS_RESET(bytes_read); + ThreadStatusUtil::SetThreadOperationProperty( + ThreadStatus::COMPACTION_BYTES_WRITTEN, 0); + ThreadStatusUtil::SetThreadOperationProperty( + ThreadStatus::COMPACTION_BYTES_READ, 0); + + // Set the thread operation after operation properties + // to ensure GetThreadList() can always show them all together. + ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_COMPACTION); + + compaction_job_stats_->is_manual_compaction = + compaction->is_manual_compaction(); + compaction_job_stats_->is_full_compaction = compaction->is_full_compaction(); +} + +void CompactionJob::Prepare() { + AutoThreadOperationStageUpdater stage_updater( + ThreadStatus::STAGE_COMPACTION_PREPARE); + + // Generate file_levels_ for compaction before making Iterator + auto* c = compact_->compaction; + ColumnFamilyData* cfd = c->column_family_data(); + assert(cfd != nullptr); + assert(cfd->current()->storage_info()->NumLevelFiles( + compact_->compaction->level()) > 0); + + write_hint_ = cfd->CalculateSSTWriteHint(c->output_level()); + bottommost_level_ = c->bottommost_level(); + + if (c->ShouldFormSubcompactions()) { + StopWatch sw(db_options_.clock, stats_, SUBCOMPACTION_SETUP_TIME); + GenSubcompactionBoundaries(); + } + if (boundaries_.size() >= 1) { + for (size_t i = 0; i <= boundaries_.size(); i++) { + compact_->sub_compact_states.emplace_back( + c, (i != 0) ? std::optional(boundaries_[i - 1]) : std::nullopt, + (i != boundaries_.size()) ? std::optional(boundaries_[i]) + : std::nullopt, + static_cast(i)); + // assert to validate that boundaries don't have same user keys (without + // timestamp part). + assert(i == 0 || i == boundaries_.size() || + cfd->user_comparator()->CompareWithoutTimestamp( + boundaries_[i - 1], boundaries_[i]) < 0); + } + RecordInHistogram(stats_, NUM_SUBCOMPACTIONS_SCHEDULED, + compact_->sub_compact_states.size()); + } else { + compact_->sub_compact_states.emplace_back(c, std::nullopt, std::nullopt, + /*sub_job_id*/ 0); + } + + // collect all seqno->time information from the input files which will be used + // to encode seqno->time to the output files. + uint64_t preserve_time_duration = + std::max(c->immutable_options()->preserve_internal_time_seconds, + c->immutable_options()->preclude_last_level_data_seconds); + + if (preserve_time_duration > 0) { + const ReadOptions read_options(Env::IOActivity::kCompaction); + // setup seqno_time_mapping_ + seqno_time_mapping_.SetMaxTimeDuration(preserve_time_duration); + for (const auto& each_level : *c->inputs()) { + for (const auto& fmd : each_level.files) { + std::shared_ptr tp; + Status s = + cfd->current()->GetTableProperties(read_options, &tp, fmd, nullptr); + if (s.ok()) { + seqno_time_mapping_.Add(tp->seqno_to_time_mapping) + .PermitUncheckedError(); + seqno_time_mapping_.Add(fmd->fd.smallest_seqno, + fmd->oldest_ancester_time); + } + } + } + + auto status = seqno_time_mapping_.Sort(); + if (!status.ok()) { + ROCKS_LOG_WARN(db_options_.info_log, + "Invalid sequence number to time mapping: Status: %s", + status.ToString().c_str()); + } + int64_t _current_time = 0; + status = db_options_.clock->GetCurrentTime(&_current_time); + if (!status.ok()) { + ROCKS_LOG_WARN(db_options_.info_log, + "Failed to get current time in compaction: Status: %s", + status.ToString().c_str()); + // preserve all time information + preserve_time_min_seqno_ = 0; + preclude_last_level_min_seqno_ = 0; + } else { + seqno_time_mapping_.TruncateOldEntries(_current_time); + uint64_t preserve_time = + static_cast(_current_time) > preserve_time_duration + ? _current_time - preserve_time_duration + : 0; + preserve_time_min_seqno_ = + seqno_time_mapping_.GetOldestSequenceNum(preserve_time); + if (c->immutable_options()->preclude_last_level_data_seconds > 0) { + uint64_t preclude_last_level_time = + static_cast(_current_time) > + c->immutable_options()->preclude_last_level_data_seconds + ? _current_time - + c->immutable_options()->preclude_last_level_data_seconds + : 0; + preclude_last_level_min_seqno_ = + seqno_time_mapping_.GetOldestSequenceNum(preclude_last_level_time); + } + } + } +} + +uint64_t CompactionJob::GetSubcompactionsLimit() { + return extra_num_subcompaction_threads_reserved_ + + std::max( + std::uint64_t(1), + static_cast(compact_->compaction->max_subcompactions())); +} + +void CompactionJob::AcquireSubcompactionResources( + int num_extra_required_subcompactions) { + TEST_SYNC_POINT("CompactionJob::AcquireSubcompactionResources:0"); + TEST_SYNC_POINT("CompactionJob::AcquireSubcompactionResources:1"); + int max_db_compactions = + DBImpl::GetBGJobLimits( + mutable_db_options_copy_.max_background_flushes, + mutable_db_options_copy_.max_background_compactions, + mutable_db_options_copy_.max_background_jobs, + versions_->GetColumnFamilySet() + ->write_controller() + ->NeedSpeedupCompaction()) + .max_compactions; + InstrumentedMutexLock l(db_mutex_); + // Apply min function first since We need to compute the extra subcompaction + // against compaction limits. And then try to reserve threads for extra + // subcompactions. The actual number of reserved threads could be less than + // the desired number. + int available_bg_compactions_against_db_limit = + std::max(max_db_compactions - *bg_compaction_scheduled_ - + *bg_bottom_compaction_scheduled_, + 0); + // Reservation only supports backgrdoun threads of which the priority is + // between BOTTOM and HIGH. Need to degrade the priority to HIGH if the + // origin thread_pri_ is higher than that. Similar to ReleaseThreads(). + extra_num_subcompaction_threads_reserved_ = + env_->ReserveThreads(std::min(num_extra_required_subcompactions, + available_bg_compactions_against_db_limit), + std::min(thread_pri_, Env::Priority::HIGH)); + + // Update bg_compaction_scheduled_ or bg_bottom_compaction_scheduled_ + // depending on if this compaction has the bottommost priority + if (thread_pri_ == Env::Priority::BOTTOM) { + *bg_bottom_compaction_scheduled_ += + extra_num_subcompaction_threads_reserved_; + } else { + *bg_compaction_scheduled_ += extra_num_subcompaction_threads_reserved_; + } +} + +void CompactionJob::ShrinkSubcompactionResources(uint64_t num_extra_resources) { + // Do nothing when we have zero resources to shrink + if (num_extra_resources == 0) return; + db_mutex_->Lock(); + // We cannot release threads more than what we reserved before + int extra_num_subcompaction_threads_released = env_->ReleaseThreads( + (int)num_extra_resources, std::min(thread_pri_, Env::Priority::HIGH)); + // Update the number of reserved threads and the number of background + // scheduled compactions for this compaction job + extra_num_subcompaction_threads_reserved_ -= + extra_num_subcompaction_threads_released; + // TODO (zichen): design a test case with new subcompaction partitioning + // when the number of actual partitions is less than the number of planned + // partitions + assert(extra_num_subcompaction_threads_released == (int)num_extra_resources); + // Update bg_compaction_scheduled_ or bg_bottom_compaction_scheduled_ + // depending on if this compaction has the bottommost priority + if (thread_pri_ == Env::Priority::BOTTOM) { + *bg_bottom_compaction_scheduled_ -= + extra_num_subcompaction_threads_released; + } else { + *bg_compaction_scheduled_ -= extra_num_subcompaction_threads_released; + } + db_mutex_->Unlock(); + TEST_SYNC_POINT("CompactionJob::ShrinkSubcompactionResources:0"); +} + +void CompactionJob::ReleaseSubcompactionResources() { + if (extra_num_subcompaction_threads_reserved_ == 0) { + return; + } + { + InstrumentedMutexLock l(db_mutex_); + // The number of reserved threads becomes larger than 0 only if the + // compaction prioity is round robin and there is no sufficient + // sub-compactions available + + // The scheduled compaction must be no less than 1 + extra number + // subcompactions using acquired resources since this compaction job has not + // finished yet + assert(*bg_bottom_compaction_scheduled_ >= + 1 + extra_num_subcompaction_threads_reserved_ || + *bg_compaction_scheduled_ >= + 1 + extra_num_subcompaction_threads_reserved_); + } + ShrinkSubcompactionResources(extra_num_subcompaction_threads_reserved_); +} + +struct RangeWithSize { + Range range; + uint64_t size; + + RangeWithSize(const Slice& a, const Slice& b, uint64_t s = 0) + : range(a, b), size(s) {} +}; + +void CompactionJob::GenSubcompactionBoundaries() { + // The goal is to find some boundary keys so that we can evenly partition + // the compaction input data into max_subcompactions ranges. + // For every input file, we ask TableReader to estimate 128 anchor points + // that evenly partition the input file into 128 ranges and the range + // sizes. This can be calculated by scanning index blocks of the file. + // Once we have the anchor points for all the input files, we merge them + // together and try to find keys dividing ranges evenly. + // For example, if we have two input files, and each returns following + // ranges: + // File1: (a1, 1000), (b1, 1200), (c1, 1100) + // File2: (a2, 1100), (b2, 1000), (c2, 1000) + // We total sort the keys to following: + // (a1, 1000), (a2, 1100), (b1, 1200), (b2, 1000), (c1, 1100), (c2, 1000) + // We calculate the total size by adding up all ranges' size, which is 6400. + // If we would like to partition into 2 subcompactions, the target of the + // range size is 3200. Based on the size, we take "b1" as the partition key + // since the first three ranges would hit 3200. + // + // Note that the ranges are actually overlapping. For example, in the example + // above, the range ending with "b1" is overlapping with the range ending with + // "b2". So the size 1000+1100+1200 is an underestimation of data size up to + // "b1". In extreme cases where we only compact N L0 files, a range can + // overlap with N-1 other ranges. Since we requested a relatively large number + // (128) of ranges from each input files, even N range overlapping would + // cause relatively small inaccuracy. + const ReadOptions read_options(Env::IOActivity::kCompaction); + auto* c = compact_->compaction; + if (c->max_subcompactions() <= 1 && + !(c->immutable_options()->compaction_pri == kRoundRobin && + c->immutable_options()->compaction_style == kCompactionStyleLevel)) { + return; + } + auto* cfd = c->column_family_data(); + const Comparator* cfd_comparator = cfd->user_comparator(); + const InternalKeyComparator& icomp = cfd->internal_comparator(); + + auto* v = compact_->compaction->input_version(); + int base_level = v->storage_info()->base_level(); + InstrumentedMutexUnlock unlock_guard(db_mutex_); + + uint64_t total_size = 0; + std::vector all_anchors; + int start_lvl = c->start_level(); + int out_lvl = c->output_level(); + + for (size_t lvl_idx = 0; lvl_idx < c->num_input_levels(); lvl_idx++) { + int lvl = c->level(lvl_idx); + if (lvl >= start_lvl && lvl <= out_lvl) { + const LevelFilesBrief* flevel = c->input_levels(lvl_idx); + size_t num_files = flevel->num_files; + + if (num_files == 0) { + continue; + } + + for (size_t i = 0; i < num_files; i++) { + FileMetaData* f = flevel->files[i].file_metadata; + std::vector my_anchors; + Status s = cfd->table_cache()->ApproximateKeyAnchors( + read_options, icomp, *f, + c->mutable_cf_options()->block_protection_bytes_per_key, + my_anchors); + if (!s.ok() || my_anchors.empty()) { + my_anchors.emplace_back(f->largest.user_key(), f->fd.GetFileSize()); + } + for (auto& ac : my_anchors) { + // Can be optimize to avoid this loop. + total_size += ac.range_size; + } + + all_anchors.insert(all_anchors.end(), my_anchors.begin(), + my_anchors.end()); + } + } + } + // Here we total sort all the anchor points across all files and go through + // them in the sorted order to find partitioning boundaries. + // Not the most efficient implementation. A much more efficient algorithm + // probably exists. But they are more complex. If performance turns out to + // be a problem, we can optimize. + std::sort( + all_anchors.begin(), all_anchors.end(), + [cfd_comparator](TableReader::Anchor& a, TableReader::Anchor& b) -> bool { + return cfd_comparator->CompareWithoutTimestamp(a.user_key, b.user_key) < + 0; + }); + + // Remove duplicated entries from boundaries. + all_anchors.erase( + std::unique(all_anchors.begin(), all_anchors.end(), + [cfd_comparator](TableReader::Anchor& a, + TableReader::Anchor& b) -> bool { + return cfd_comparator->CompareWithoutTimestamp( + a.user_key, b.user_key) == 0; + }), + all_anchors.end()); + + // Get the number of planned subcompactions, may update reserve threads + // and update extra_num_subcompaction_threads_reserved_ for round-robin + uint64_t num_planned_subcompactions; + if (c->immutable_options()->compaction_pri == kRoundRobin && + c->immutable_options()->compaction_style == kCompactionStyleLevel) { + // For round-robin compaction prioity, we need to employ more + // subcompactions (may exceed the max_subcompaction limit). The extra + // subcompactions will be executed using reserved threads and taken into + // account bg_compaction_scheduled or bg_bottom_compaction_scheduled. + + // Initialized by the number of input files + num_planned_subcompactions = static_cast(c->num_input_files(0)); + uint64_t max_subcompactions_limit = GetSubcompactionsLimit(); + if (max_subcompactions_limit < num_planned_subcompactions) { + // Assert two pointers are not empty so that we can use extra + // subcompactions against db compaction limits + assert(bg_bottom_compaction_scheduled_ != nullptr); + assert(bg_compaction_scheduled_ != nullptr); + // Reserve resources when max_subcompaction is not sufficient + AcquireSubcompactionResources( + (int)(num_planned_subcompactions - max_subcompactions_limit)); + // Subcompactions limit changes after acquiring additional resources. + // Need to call GetSubcompactionsLimit() again to update the number + // of planned subcompactions + num_planned_subcompactions = + std::min(num_planned_subcompactions, GetSubcompactionsLimit()); + } else { + num_planned_subcompactions = max_subcompactions_limit; + } + } else { + num_planned_subcompactions = GetSubcompactionsLimit(); + } + + TEST_SYNC_POINT_CALLBACK("CompactionJob::GenSubcompactionBoundaries:0", + &num_planned_subcompactions); + if (num_planned_subcompactions == 1) return; + + // Group the ranges into subcompactions + uint64_t target_range_size = std::max( + total_size / num_planned_subcompactions, + MaxFileSizeForLevel( + *(c->mutable_cf_options()), out_lvl, + c->immutable_options()->compaction_style, base_level, + c->immutable_options()->level_compaction_dynamic_level_bytes)); + + if (target_range_size >= total_size) { + return; + } + + uint64_t next_threshold = target_range_size; + uint64_t cumulative_size = 0; + uint64_t num_actual_subcompactions = 1U; + for (TableReader::Anchor& anchor : all_anchors) { + cumulative_size += anchor.range_size; + if (cumulative_size > next_threshold) { + next_threshold += target_range_size; + num_actual_subcompactions++; + boundaries_.push_back(anchor.user_key); + } + if (num_actual_subcompactions == num_planned_subcompactions) { + break; + } + } + TEST_SYNC_POINT_CALLBACK("CompactionJob::GenSubcompactionBoundaries:1", + &num_actual_subcompactions); + // Shrink extra subcompactions resources when extra resrouces are acquired + ShrinkSubcompactionResources( + std::min((int)(num_planned_subcompactions - num_actual_subcompactions), + extra_num_subcompaction_threads_reserved_)); +} + +Status CompactionJob::Run() { + AutoThreadOperationStageUpdater stage_updater( + ThreadStatus::STAGE_COMPACTION_RUN); + TEST_SYNC_POINT("CompactionJob::Run():Start"); + log_buffer_->FlushBufferToLog(); + LogCompaction(); + + const size_t num_threads = compact_->sub_compact_states.size(); + assert(num_threads > 0); + const uint64_t start_micros = db_options_.clock->NowMicros(); + + // Launch a thread for each of subcompactions 1...num_threads-1 + std::vector thread_pool; + thread_pool.reserve(num_threads - 1); + for (size_t i = 1; i < compact_->sub_compact_states.size(); i++) { + thread_pool.emplace_back(&CompactionJob::ProcessKeyValueCompaction, this, + &compact_->sub_compact_states[i]); + } + + // Always schedule the first subcompaction (whether or not there are also + // others) in the current thread to be efficient with resources + ProcessKeyValueCompaction(&compact_->sub_compact_states[0]); + + // Wait for all other threads (if there are any) to finish execution + for (auto& thread : thread_pool) { + thread.join(); + } + + compaction_stats_.SetMicros(db_options_.clock->NowMicros() - start_micros); + + for (auto& state : compact_->sub_compact_states) { + compaction_stats_.AddCpuMicros(state.compaction_job_stats.cpu_micros); + state.RemoveLastEmptyOutput(); + } + + RecordTimeToHistogram(stats_, COMPACTION_TIME, + compaction_stats_.stats.micros); + RecordTimeToHistogram(stats_, COMPACTION_CPU_TIME, + compaction_stats_.stats.cpu_micros); + + TEST_SYNC_POINT("CompactionJob::Run:BeforeVerify"); + + // Check if any thread encountered an error during execution + Status status; + IOStatus io_s; + bool wrote_new_blob_files = false; + + for (const auto& state : compact_->sub_compact_states) { + if (!state.status.ok()) { + status = state.status; + io_s = state.io_status; + break; + } + + if (state.Current().HasBlobFileAdditions()) { + wrote_new_blob_files = true; + } + } + + if (io_status_.ok()) { + io_status_ = io_s; + } + if (status.ok()) { + constexpr IODebugContext* dbg = nullptr; + + if (output_directory_) { + io_s = output_directory_->FsyncWithDirOptions( + IOOptions(), dbg, + DirFsyncOptions(DirFsyncOptions::FsyncReason::kNewFileSynced)); + } + + if (io_s.ok() && wrote_new_blob_files && blob_output_directory_ && + blob_output_directory_ != output_directory_) { + io_s = blob_output_directory_->FsyncWithDirOptions( + IOOptions(), dbg, + DirFsyncOptions(DirFsyncOptions::FsyncReason::kNewFileSynced)); + } + } + if (io_status_.ok()) { + io_status_ = io_s; + } + if (status.ok()) { + status = io_s; + } + if (status.ok()) { + thread_pool.clear(); + std::vector files_output; + for (const auto& state : compact_->sub_compact_states) { + for (const auto& output : state.GetOutputs()) { + files_output.emplace_back(&output); + } + } + ColumnFamilyData* cfd = compact_->compaction->column_family_data(); + auto& prefix_extractor = + compact_->compaction->mutable_cf_options()->prefix_extractor; + std::atomic next_file_idx(0); + auto verify_table = [&](Status& output_status) { + while (true) { + size_t file_idx = next_file_idx.fetch_add(1); + if (file_idx >= files_output.size()) { + break; + } + // Verify that the table is usable + // We set for_compaction to false and don't + // OptimizeForCompactionTableRead here because this is a special case + // after we finish the table building No matter whether + // use_direct_io_for_flush_and_compaction is true, we will regard this + // verification as user reads since the goal is to cache it here for + // further user reads + const ReadOptions verify_table_read_options( + Env::IOActivity::kCompaction); + InternalIterator* iter = cfd->table_cache()->NewIterator( + verify_table_read_options, file_options_, + cfd->internal_comparator(), files_output[file_idx]->meta, + /*range_del_agg=*/nullptr, prefix_extractor, + /*table_reader_ptr=*/nullptr, + cfd->internal_stats()->GetFileReadHist( + compact_->compaction->output_level()), + TableReaderCaller::kCompactionRefill, /*arena=*/nullptr, + /*skip_filters=*/false, compact_->compaction->output_level(), + MaxFileSizeForL0MetaPin( + *compact_->compaction->mutable_cf_options()), + /*smallest_compaction_key=*/nullptr, + /*largest_compaction_key=*/nullptr, + /*allow_unprepared_value=*/false, + compact_->compaction->mutable_cf_options() + ->block_protection_bytes_per_key); + auto s = iter->status(); + + if (s.ok() && paranoid_file_checks_) { + OutputValidator validator(cfd->internal_comparator(), + /*_enable_order_check=*/true, + /*_enable_hash=*/true); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + s = validator.Add(iter->key(), iter->value()); + if (!s.ok()) { + break; + } + } + if (s.ok()) { + s = iter->status(); + } + if (s.ok() && + !validator.CompareValidator(files_output[file_idx]->validator)) { + s = Status::Corruption("Paranoid checksums do not match"); + } + } + + delete iter; + + if (!s.ok()) { + output_status = s; + break; + } + } + }; + for (size_t i = 1; i < compact_->sub_compact_states.size(); i++) { + thread_pool.emplace_back( + verify_table, std::ref(compact_->sub_compact_states[i].status)); + } + verify_table(compact_->sub_compact_states[0].status); + for (auto& thread : thread_pool) { + thread.join(); + } + + for (const auto& state : compact_->sub_compact_states) { + if (!state.status.ok()) { + status = state.status; + break; + } + } + } + + ReleaseSubcompactionResources(); + TEST_SYNC_POINT("CompactionJob::ReleaseSubcompactionResources:0"); + TEST_SYNC_POINT("CompactionJob::ReleaseSubcompactionResources:1"); + + TablePropertiesCollection tp; + for (const auto& state : compact_->sub_compact_states) { + for (const auto& output : state.GetOutputs()) { + auto fn = + TableFileName(state.compaction->immutable_options()->cf_paths, + output.meta.fd.GetNumber(), output.meta.fd.GetPathId()); + compact_->compaction->SetOutputTableProperties(fn, + output.table_properties); + } + } + + // Finish up all bookkeeping to unify the subcompaction results. + compact_->AggregateCompactionStats(compaction_stats_, *compaction_job_stats_); + uint64_t num_input_range_del = 0; + bool ok = UpdateCompactionStats(&num_input_range_del); + // (Sub)compactions returned ok, do sanity check on the number of input keys. + if (status.ok() && ok && compaction_job_stats_->has_num_input_records) { + size_t ts_sz = compact_->compaction->column_family_data() + ->user_comparator() + ->timestamp_size(); + // When trim_ts_ is non-empty, CompactionIterator takes + // HistoryTrimmingIterator as input iterator and sees a trimmed view of + // input keys. So the number of keys it processed is not suitable for + // verification here. + // TODO: support verification when trim_ts_ is non-empty. + if (!(ts_sz > 0 && !trim_ts_.empty()) && + db_options_.compaction_verify_record_count) { + assert(compaction_stats_.stats.num_input_records > 0); + // TODO: verify the number of range deletion entries. + uint64_t expected = + compaction_stats_.stats.num_input_records - num_input_range_del; + uint64_t actual = compaction_job_stats_->num_input_records; + if (expected != actual) { + std::string msg = + "Total number of input records: " + std::to_string(expected) + + ", but processed " + std::to_string(actual) + " records."; + ROCKS_LOG_WARN( + db_options_.info_log, "[%s] [JOB %d] Compaction %s", + compact_->compaction->column_family_data()->GetName().c_str(), + job_context_->job_id, msg.c_str()); + status = Status::Corruption( + "Compaction number of input keys does not match number of keys " + "processed."); + } + } + } + RecordCompactionIOStats(); + LogFlush(db_options_.info_log); + TEST_SYNC_POINT("CompactionJob::Run():End"); + compact_->status = status; + TEST_SYNC_POINT_CALLBACK("CompactionJob::Run():EndStatusSet", &status); + return status; +} + +Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) { + assert(compact_); + + AutoThreadOperationStageUpdater stage_updater( + ThreadStatus::STAGE_COMPACTION_INSTALL); + db_mutex_->AssertHeld(); + Status status = compact_->status; + + ColumnFamilyData* cfd = compact_->compaction->column_family_data(); + assert(cfd); + + int output_level = compact_->compaction->output_level(); + cfd->internal_stats()->AddCompactionStats(output_level, thread_pri_, + compaction_stats_); + + if (status.ok()) { + status = InstallCompactionResults(mutable_cf_options); + } + if (!versions_->io_status().ok()) { + io_status_ = versions_->io_status(); + } + + VersionStorageInfo::LevelSummaryStorage tmp; + auto vstorage = cfd->current()->storage_info(); + const auto& stats = compaction_stats_.stats; + + double read_write_amp = 0.0; + double write_amp = 0.0; + double bytes_read_per_sec = 0; + double bytes_written_per_sec = 0; + + const uint64_t bytes_read_non_output_and_blob = + stats.bytes_read_non_output_levels + stats.bytes_read_blob; + const uint64_t bytes_read_all = + stats.bytes_read_output_level + bytes_read_non_output_and_blob; + const uint64_t bytes_written_all = + stats.bytes_written + stats.bytes_written_blob; + + if (bytes_read_non_output_and_blob > 0) { + read_write_amp = (bytes_written_all + bytes_read_all) / + static_cast(bytes_read_non_output_and_blob); + write_amp = + bytes_written_all / static_cast(bytes_read_non_output_and_blob); + } + if (stats.micros > 0) { + bytes_read_per_sec = bytes_read_all / static_cast(stats.micros); + bytes_written_per_sec = + bytes_written_all / static_cast(stats.micros); + } + + const std::string& column_family_name = cfd->GetName(); + + constexpr double kMB = 1048576.0; + + ROCKS_LOG_BUFFER( + log_buffer_, + "[%s] compacted to: %s, MB/sec: %.1f rd, %.1f wr, level %d, " + "files in(%d, %d) out(%d +%d blob) " + "MB in(%.1f, %.1f +%.1f blob) out(%.1f +%.1f blob), " + "read-write-amplify(%.1f) write-amplify(%.1f) %s, records in: %" PRIu64 + ", records dropped: %" PRIu64 " output_compression: %s\n", + column_family_name.c_str(), vstorage->LevelSummary(&tmp), + bytes_read_per_sec, bytes_written_per_sec, + compact_->compaction->output_level(), + stats.num_input_files_in_non_output_levels, + stats.num_input_files_in_output_level, stats.num_output_files, + stats.num_output_files_blob, stats.bytes_read_non_output_levels / kMB, + stats.bytes_read_output_level / kMB, stats.bytes_read_blob / kMB, + stats.bytes_written / kMB, stats.bytes_written_blob / kMB, read_write_amp, + write_amp, status.ToString().c_str(), stats.num_input_records, + stats.num_dropped_records, + CompressionTypeToString(compact_->compaction->output_compression()) + .c_str()); + + const auto& blob_files = vstorage->GetBlobFiles(); + if (!blob_files.empty()) { + assert(blob_files.front()); + assert(blob_files.back()); + + ROCKS_LOG_BUFFER( + log_buffer_, + "[%s] Blob file summary: head=%" PRIu64 ", tail=%" PRIu64 "\n", + column_family_name.c_str(), blob_files.front()->GetBlobFileNumber(), + blob_files.back()->GetBlobFileNumber()); + } + + if (compaction_stats_.has_penultimate_level_output) { + ROCKS_LOG_BUFFER( + log_buffer_, + "[%s] has Penultimate Level output: %" PRIu64 + ", level %d, number of files: %" PRIu64 ", number of records: %" PRIu64, + column_family_name.c_str(), + compaction_stats_.penultimate_level_stats.bytes_written, + compact_->compaction->GetPenultimateLevel(), + compaction_stats_.penultimate_level_stats.num_output_files, + compaction_stats_.penultimate_level_stats.num_output_records); + } + + UpdateCompactionJobStats(stats); + + auto stream = event_logger_->LogToBuffer(log_buffer_, 8192); + stream << "job" << job_id_ << "event" + << "compaction_finished" + << "compaction_time_micros" << stats.micros + << "compaction_time_cpu_micros" << stats.cpu_micros << "output_level" + << compact_->compaction->output_level() << "num_output_files" + << stats.num_output_files << "total_output_size" + << stats.bytes_written; + + if (stats.num_output_files_blob > 0) { + stream << "num_blob_output_files" << stats.num_output_files_blob + << "total_blob_output_size" << stats.bytes_written_blob; + } + + stream << "num_input_records" << stats.num_input_records + << "num_output_records" << stats.num_output_records + << "num_subcompactions" << compact_->sub_compact_states.size() + << "output_compression" + << CompressionTypeToString(compact_->compaction->output_compression()); + + stream << "num_single_delete_mismatches" + << compaction_job_stats_->num_single_del_mismatch; + stream << "num_single_delete_fallthrough" + << compaction_job_stats_->num_single_del_fallthru; + + if (measure_io_stats_) { + stream << "file_write_nanos" << compaction_job_stats_->file_write_nanos; + stream << "file_range_sync_nanos" + << compaction_job_stats_->file_range_sync_nanos; + stream << "file_fsync_nanos" << compaction_job_stats_->file_fsync_nanos; + stream << "file_prepare_write_nanos" + << compaction_job_stats_->file_prepare_write_nanos; + } + + stream << "lsm_state"; + stream.StartArray(); + for (int level = 0; level < vstorage->num_levels(); ++level) { + stream << vstorage->NumLevelFiles(level); + } + stream.EndArray(); + + if (!blob_files.empty()) { + assert(blob_files.front()); + stream << "blob_file_head" << blob_files.front()->GetBlobFileNumber(); + + assert(blob_files.back()); + stream << "blob_file_tail" << blob_files.back()->GetBlobFileNumber(); + } + + if (compaction_stats_.has_penultimate_level_output) { + InternalStats::CompactionStats& pl_stats = + compaction_stats_.penultimate_level_stats; + stream << "penultimate_level_num_output_files" << pl_stats.num_output_files; + stream << "penultimate_level_bytes_written" << pl_stats.bytes_written; + stream << "penultimate_level_num_output_records" + << pl_stats.num_output_records; + stream << "penultimate_level_num_output_files_blob" + << pl_stats.num_output_files_blob; + stream << "penultimate_level_bytes_written_blob" + << pl_stats.bytes_written_blob; + } + + CleanupCompaction(); + return status; +} + +void CompactionJob::NotifyOnSubcompactionBegin( + SubcompactionState* sub_compact) { + Compaction* c = compact_->compaction; + + if (db_options_.listeners.empty()) { + return; + } + if (shutting_down_->load(std::memory_order_acquire)) { + return; + } + if (c->is_manual_compaction() && + manual_compaction_canceled_.load(std::memory_order_acquire)) { + return; + } + + sub_compact->notify_on_subcompaction_completion = true; + + SubcompactionJobInfo info{}; + sub_compact->BuildSubcompactionJobInfo(info); + info.job_id = static_cast(job_id_); + info.thread_id = env_->GetThreadID(); + + for (const auto& listener : db_options_.listeners) { + listener->OnSubcompactionBegin(info); + } + info.status.PermitUncheckedError(); + +} + +void CompactionJob::NotifyOnSubcompactionCompleted( + SubcompactionState* sub_compact) { + + if (db_options_.listeners.empty()) { + return; + } + if (shutting_down_->load(std::memory_order_acquire)) { + return; + } + + if (sub_compact->notify_on_subcompaction_completion == false) { + return; + } + + SubcompactionJobInfo info{}; + sub_compact->BuildSubcompactionJobInfo(info); + info.job_id = static_cast(job_id_); + info.thread_id = env_->GetThreadID(); + + for (const auto& listener : db_options_.listeners) { + listener->OnSubcompactionCompleted(info); + } +} + +void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) { + assert(sub_compact); + assert(sub_compact->compaction); + if (db_options_.compaction_service) { + CompactionServiceJobStatus comp_status = + ProcessKeyValueCompactionWithCompactionService(sub_compact); + if (comp_status == CompactionServiceJobStatus::kSuccess || + comp_status == CompactionServiceJobStatus::kFailure) { + return; + } + // fallback to local compaction + assert(comp_status == CompactionServiceJobStatus::kUseLocal); + } + + uint64_t prev_cpu_micros = db_options_.clock->CPUMicros(); + + ColumnFamilyData* cfd = sub_compact->compaction->column_family_data(); + + // Create compaction filter and fail the compaction if + // IgnoreSnapshots() = false because it is not supported anymore + const CompactionFilter* compaction_filter = + cfd->ioptions()->compaction_filter; + std::unique_ptr compaction_filter_from_factory = nullptr; + if (compaction_filter == nullptr) { + compaction_filter_from_factory = + sub_compact->compaction->CreateCompactionFilter(); + compaction_filter = compaction_filter_from_factory.get(); + } + if (compaction_filter != nullptr && !compaction_filter->IgnoreSnapshots()) { + sub_compact->status = Status::NotSupported( + "CompactionFilter::IgnoreSnapshots() = false is not supported " + "anymore."); + return; + } + + NotifyOnSubcompactionBegin(sub_compact); + + auto range_del_agg = std::make_unique( + &cfd->internal_comparator(), existing_snapshots_, &full_history_ts_low_, + &trim_ts_); + + // TODO: since we already use C++17, should use + // std::optional instead. + const std::optional start = sub_compact->start; + const std::optional end = sub_compact->end; + + std::optional start_without_ts; + std::optional end_without_ts; + + ReadOptions read_options; + read_options.verify_checksums = true; + read_options.fill_cache = false; + read_options.rate_limiter_priority = GetRateLimiterPriority(); + read_options.io_activity = Env::IOActivity::kCompaction; + // Compaction iterators shouldn't be confined to a single prefix. + // Compactions use Seek() for + // (a) concurrent compactions, + // (b) CompactionFilter::Decision::kRemoveAndSkipUntil. + read_options.total_order_seek = true; + + // Remove the timestamps from boundaries because boundaries created in + // GenSubcompactionBoundaries doesn't strip away the timestamp. + size_t ts_sz = cfd->user_comparator()->timestamp_size(); + if (start.has_value()) { + read_options.iterate_lower_bound = &(*start); + if (ts_sz > 0) { + start_without_ts = StripTimestampFromUserKey(*start, ts_sz); + read_options.iterate_lower_bound = &(*start_without_ts); + } + } + if (end.has_value()) { + read_options.iterate_upper_bound = &(*end); + if (ts_sz > 0) { + end_without_ts = StripTimestampFromUserKey(*end, ts_sz); + read_options.iterate_upper_bound = &(*end_without_ts); + } + } + + // Although the v2 aggregator is what the level iterator(s) know about, + // the AddTombstones calls will be propagated down to the v1 aggregator. + std::unique_ptr raw_input(versions_->MakeInputIterator( + read_options, sub_compact->compaction, range_del_agg.get(), + file_options_for_read_, start, end)); + InternalIterator* input = raw_input.get(); + + IterKey start_ikey; + IterKey end_ikey; + Slice start_slice; + Slice end_slice; + Slice start_user_key{}; + Slice end_user_key{}; + + static constexpr char kMaxTs[] = + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"; + Slice ts_slice; + std::string max_ts; + if (ts_sz > 0) { + if (ts_sz <= strlen(kMaxTs)) { + ts_slice = Slice(kMaxTs, ts_sz); + } else { + max_ts = std::string(ts_sz, '\xff'); + ts_slice = Slice(max_ts); + } + } + + if (start.has_value()) { + start_ikey.SetInternalKey(*start, kMaxSequenceNumber, kValueTypeForSeek); + if (ts_sz > 0) { + start_ikey.UpdateInternalKey(kMaxSequenceNumber, kValueTypeForSeek, + &ts_slice); + } + start_slice = start_ikey.GetInternalKey(); + start_user_key = start_ikey.GetUserKey(); + } + if (end.has_value()) { + end_ikey.SetInternalKey(*end, kMaxSequenceNumber, kValueTypeForSeek); + if (ts_sz > 0) { + end_ikey.UpdateInternalKey(kMaxSequenceNumber, kValueTypeForSeek, + &ts_slice); + } + end_slice = end_ikey.GetInternalKey(); + end_user_key = end_ikey.GetUserKey(); + } + + std::unique_ptr clip; + if (start.has_value() || end.has_value()) { + clip = std::make_unique( + raw_input.get(), start.has_value() ? &start_slice : nullptr, + end.has_value() ? &end_slice : nullptr, &cfd->internal_comparator()); + input = clip.get(); + } + + std::unique_ptr blob_counter; + + if (sub_compact->compaction->DoesInputReferenceBlobFiles()) { + BlobGarbageMeter* meter = sub_compact->Current().CreateBlobGarbageMeter(); + blob_counter = std::make_unique(input, meter); + input = blob_counter.get(); + } + + std::unique_ptr trim_history_iter; + if (ts_sz > 0 && !trim_ts_.empty()) { + trim_history_iter = std::make_unique( + input, cfd->user_comparator(), trim_ts_); + input = trim_history_iter.get(); + } + + input->SeekToFirst(); + + AutoThreadOperationStageUpdater stage_updater( + ThreadStatus::STAGE_COMPACTION_PROCESS_KV); + + // I/O measurement variables + PerfLevel prev_perf_level = PerfLevel::kEnableTime; + const uint64_t kRecordStatsEvery = 1000; + uint64_t prev_write_nanos = 0; + uint64_t prev_fsync_nanos = 0; + uint64_t prev_range_sync_nanos = 0; + uint64_t prev_prepare_write_nanos = 0; + uint64_t prev_cpu_write_nanos = 0; + uint64_t prev_cpu_read_nanos = 0; + if (measure_io_stats_) { + prev_perf_level = GetPerfLevel(); + SetPerfLevel(PerfLevel::kEnableTimeAndCPUTimeExceptForMutex); + prev_write_nanos = IOSTATS(write_nanos); + prev_fsync_nanos = IOSTATS(fsync_nanos); + prev_range_sync_nanos = IOSTATS(range_sync_nanos); + prev_prepare_write_nanos = IOSTATS(prepare_write_nanos); + prev_cpu_write_nanos = IOSTATS(cpu_write_nanos); + prev_cpu_read_nanos = IOSTATS(cpu_read_nanos); + } + + MergeHelper merge( + env_, cfd->user_comparator(), cfd->ioptions()->merge_operator.get(), + compaction_filter, db_options_.info_log.get(), + false /* internal key corruption is expected */, + existing_snapshots_.empty() ? 0 : existing_snapshots_.back(), + snapshot_checker_, compact_->compaction->level(), db_options_.stats); + + const MutableCFOptions* mutable_cf_options = + sub_compact->compaction->mutable_cf_options(); + assert(mutable_cf_options); + + std::vector blob_file_paths; + + // TODO: BlobDB to support output_to_penultimate_level compaction, which needs + // 2 builders, so may need to move to `CompactionOutputs` + std::unique_ptr blob_file_builder( + (mutable_cf_options->enable_blob_files && + sub_compact->compaction->output_level() >= + mutable_cf_options->blob_file_starting_level) + ? new BlobFileBuilder( + versions_, fs_.get(), + sub_compact->compaction->immutable_options(), + mutable_cf_options, &file_options_, db_id_, db_session_id_, + job_id_, cfd->GetID(), cfd->GetName(), Env::IOPriority::IO_LOW, + write_hint_, io_tracer_, blob_callback_, + BlobFileCreationReason::kCompaction, &blob_file_paths, + sub_compact->Current().GetBlobFileAdditionsPtr()) + : nullptr); + + TEST_SYNC_POINT("CompactionJob::Run():Inprogress"); + TEST_SYNC_POINT_CALLBACK( + "CompactionJob::Run():PausingManualCompaction:1", + reinterpret_cast( + const_cast*>(&manual_compaction_canceled_))); + + const std::string* const full_history_ts_low = + full_history_ts_low_.empty() ? nullptr : &full_history_ts_low_; + const SequenceNumber job_snapshot_seq = + job_context_ ? job_context_->GetJobSnapshotSequence() + : kMaxSequenceNumber; + + auto c_iter = std::make_unique( + input, cfd->user_comparator(), &merge, versions_->LastSequence(), + &existing_snapshots_, earliest_write_conflict_snapshot_, job_snapshot_seq, + snapshot_checker_, env_, ShouldReportDetailedTime(env_, stats_), + /*expect_valid_internal_key=*/true, range_del_agg.get(), + blob_file_builder.get(), db_options_.allow_data_in_errors, + db_options_.enforce_single_del_contracts, manual_compaction_canceled_, + sub_compact->compaction + ->DoesInputReferenceBlobFiles() /* must_count_input_entries */, + sub_compact->compaction, compaction_filter, shutting_down_, + db_options_.info_log, full_history_ts_low, preserve_time_min_seqno_, + preclude_last_level_min_seqno_); + c_iter->SeekToFirst(); + + // Assign range delete aggregator to the target output level, which makes sure + // it only output to single level + sub_compact->AssignRangeDelAggregator(std::move(range_del_agg)); + + const auto& c_iter_stats = c_iter->iter_stats(); + + // define the open and close functions for the compaction files, which will be + // used open/close output files when needed. + const CompactionFileOpenFunc open_file_func = + [this, sub_compact](CompactionOutputs& outputs) { + return this->OpenCompactionOutputFile(sub_compact, outputs); + }; + + const CompactionFileCloseFunc close_file_func = + [this, sub_compact, start_user_key, end_user_key]( + CompactionOutputs& outputs, const Status& status, + const Slice& next_table_min_key) { + return this->FinishCompactionOutputFile( + status, sub_compact, outputs, next_table_min_key, + sub_compact->start.has_value() ? &start_user_key : nullptr, + sub_compact->end.has_value() ? &end_user_key : nullptr); + }; + + Status status; + TEST_SYNC_POINT_CALLBACK( + "CompactionJob::ProcessKeyValueCompaction()::Processing", + reinterpret_cast( + const_cast(sub_compact->compaction))); + while (status.ok() && !cfd->IsDropped() && c_iter->Valid()) { + // Invariant: c_iter.status() is guaranteed to be OK if c_iter->Valid() + // returns true. + assert(!end.has_value() || + cfd->user_comparator()->Compare(c_iter->user_key(), *end) < 0); + + if (c_iter_stats.num_input_records % kRecordStatsEvery == + kRecordStatsEvery - 1) { + RecordDroppedKeys(c_iter_stats, &sub_compact->compaction_job_stats); + c_iter->ResetRecordCounts(); + RecordCompactionIOStats(); + } + + // Add current compaction_iterator key to target compaction output, if the + // output file needs to be close or open, it will call the `open_file_func` + // and `close_file_func`. + // TODO: it would be better to have the compaction file open/close moved + // into `CompactionOutputs` which has the output file information. + status = sub_compact->AddToOutput(*c_iter, open_file_func, close_file_func); + if (!status.ok()) { + break; + } + + TEST_SYNC_POINT_CALLBACK( + "CompactionJob::Run():PausingManualCompaction:2", + reinterpret_cast( + const_cast*>(&manual_compaction_canceled_))); + c_iter->Next(); + if (c_iter->status().IsManualCompactionPaused()) { + break; + } + +#ifndef NDEBUG + bool stop = false; + TEST_SYNC_POINT_CALLBACK("CompactionJob::ProcessKeyValueCompaction()::stop", + static_cast(&stop)); + if (stop) { + break; + } +#endif // NDEBUG + } + + // This number may not be accurate when CompactionIterator was created + // with `must_count_input_entries=false`. + assert(!sub_compact->compaction->DoesInputReferenceBlobFiles() || + c_iter->HasNumInputEntryScanned()); + sub_compact->compaction_job_stats.has_num_input_records = + c_iter->HasNumInputEntryScanned(); + sub_compact->compaction_job_stats.num_input_records = + c_iter->NumInputEntryScanned(); + sub_compact->compaction_job_stats.num_blobs_read = + c_iter_stats.num_blobs_read; + sub_compact->compaction_job_stats.total_blob_bytes_read = + c_iter_stats.total_blob_bytes_read; + sub_compact->compaction_job_stats.num_input_deletion_records = + c_iter_stats.num_input_deletion_records; + sub_compact->compaction_job_stats.num_corrupt_keys = + c_iter_stats.num_input_corrupt_records; + sub_compact->compaction_job_stats.num_single_del_fallthru = + c_iter_stats.num_single_del_fallthru; + sub_compact->compaction_job_stats.num_single_del_mismatch = + c_iter_stats.num_single_del_mismatch; + sub_compact->compaction_job_stats.total_input_raw_key_bytes += + c_iter_stats.total_input_raw_key_bytes; + sub_compact->compaction_job_stats.total_input_raw_value_bytes += + c_iter_stats.total_input_raw_value_bytes; + + RecordTick(stats_, FILTER_OPERATION_TOTAL_TIME, + c_iter_stats.total_filter_time); + + if (c_iter_stats.num_blobs_relocated > 0) { + RecordTick(stats_, BLOB_DB_GC_NUM_KEYS_RELOCATED, + c_iter_stats.num_blobs_relocated); + } + if (c_iter_stats.total_blob_bytes_relocated > 0) { + RecordTick(stats_, BLOB_DB_GC_BYTES_RELOCATED, + c_iter_stats.total_blob_bytes_relocated); + } + + RecordDroppedKeys(c_iter_stats, &sub_compact->compaction_job_stats); + RecordCompactionIOStats(); + + if (status.ok() && cfd->IsDropped()) { + status = + Status::ColumnFamilyDropped("Column family dropped during compaction"); + } + if ((status.ok() || status.IsColumnFamilyDropped()) && + shutting_down_->load(std::memory_order_relaxed)) { + status = Status::ShutdownInProgress("Database shutdown"); + } + if ((status.ok() || status.IsColumnFamilyDropped()) && + (manual_compaction_canceled_.load(std::memory_order_relaxed))) { + status = Status::Incomplete(Status::SubCode::kManualCompactionPaused); + } + if (status.ok()) { + status = input->status(); + } + if (status.ok()) { + status = c_iter->status(); + } + + // Call FinishCompactionOutputFile() even if status is not ok: it needs to + // close the output files. Open file function is also passed, in case there's + // only range-dels, no file was opened, to save the range-dels, it need to + // create a new output file. + status = sub_compact->CloseCompactionFiles(status, open_file_func, + close_file_func); + + if (blob_file_builder) { + if (status.ok()) { + status = blob_file_builder->Finish(); + } else { + blob_file_builder->Abandon(status); + } + blob_file_builder.reset(); + sub_compact->Current().UpdateBlobStats(); + } + + sub_compact->compaction_job_stats.cpu_micros = + db_options_.clock->CPUMicros() - prev_cpu_micros; + + if (measure_io_stats_) { + sub_compact->compaction_job_stats.file_write_nanos += + IOSTATS(write_nanos) - prev_write_nanos; + sub_compact->compaction_job_stats.file_fsync_nanos += + IOSTATS(fsync_nanos) - prev_fsync_nanos; + sub_compact->compaction_job_stats.file_range_sync_nanos += + IOSTATS(range_sync_nanos) - prev_range_sync_nanos; + sub_compact->compaction_job_stats.file_prepare_write_nanos += + IOSTATS(prepare_write_nanos) - prev_prepare_write_nanos; + sub_compact->compaction_job_stats.cpu_micros -= + (IOSTATS(cpu_write_nanos) - prev_cpu_write_nanos + + IOSTATS(cpu_read_nanos) - prev_cpu_read_nanos) / + 1000; + if (prev_perf_level != PerfLevel::kEnableTimeAndCPUTimeExceptForMutex) { + SetPerfLevel(prev_perf_level); + } + } +#ifdef ROCKSDB_ASSERT_STATUS_CHECKED + if (!status.ok()) { + if (c_iter) { + c_iter->status().PermitUncheckedError(); + } + if (input) { + input->status().PermitUncheckedError(); + } + } +#endif // ROCKSDB_ASSERT_STATUS_CHECKED + + blob_counter.reset(); + clip.reset(); + raw_input.reset(); + sub_compact->status = status; + NotifyOnSubcompactionCompleted(sub_compact); +} + +uint64_t CompactionJob::GetCompactionId(SubcompactionState* sub_compact) const { + return (uint64_t)job_id_ << 32 | sub_compact->sub_job_id; +} + +void CompactionJob::RecordDroppedKeys( + const CompactionIterationStats& c_iter_stats, + CompactionJobStats* compaction_job_stats) { + if (c_iter_stats.num_record_drop_user > 0) { + RecordTick(stats_, COMPACTION_KEY_DROP_USER, + c_iter_stats.num_record_drop_user); + } + if (c_iter_stats.num_record_drop_hidden > 0) { + RecordTick(stats_, COMPACTION_KEY_DROP_NEWER_ENTRY, + c_iter_stats.num_record_drop_hidden); + if (compaction_job_stats) { + compaction_job_stats->num_records_replaced += + c_iter_stats.num_record_drop_hidden; + } + } + if (c_iter_stats.num_record_drop_obsolete > 0) { + RecordTick(stats_, COMPACTION_KEY_DROP_OBSOLETE, + c_iter_stats.num_record_drop_obsolete); + if (compaction_job_stats) { + compaction_job_stats->num_expired_deletion_records += + c_iter_stats.num_record_drop_obsolete; + } + } + if (c_iter_stats.num_record_drop_range_del > 0) { + RecordTick(stats_, COMPACTION_KEY_DROP_RANGE_DEL, + c_iter_stats.num_record_drop_range_del); + } + if (c_iter_stats.num_range_del_drop_obsolete > 0) { + RecordTick(stats_, COMPACTION_RANGE_DEL_DROP_OBSOLETE, + c_iter_stats.num_range_del_drop_obsolete); + } + if (c_iter_stats.num_optimized_del_drop_obsolete > 0) { + RecordTick(stats_, COMPACTION_OPTIMIZED_DEL_DROP_OBSOLETE, + c_iter_stats.num_optimized_del_drop_obsolete); + } +} + +Status CompactionJob::FinishCompactionOutputFile( + const Status& input_status, SubcompactionState* sub_compact, + CompactionOutputs& outputs, const Slice& next_table_min_key, + const Slice* comp_start_user_key, const Slice* comp_end_user_key) { + AutoThreadOperationStageUpdater stage_updater( + ThreadStatus::STAGE_COMPACTION_SYNC_FILE); + assert(sub_compact != nullptr); + assert(outputs.HasBuilder()); + + FileMetaData* meta = outputs.GetMetaData(); + uint64_t output_number = meta->fd.GetNumber(); + assert(output_number != 0); + + ColumnFamilyData* cfd = sub_compact->compaction->column_family_data(); + std::string file_checksum = kUnknownFileChecksum; + std::string file_checksum_func_name = kUnknownFileChecksumFuncName; + + // Check for iterator errors + Status s = input_status; + + // Add range tombstones + auto earliest_snapshot = kMaxSequenceNumber; + if (existing_snapshots_.size() > 0) { + earliest_snapshot = existing_snapshots_[0]; + } + if (s.ok()) { + CompactionIterationStats range_del_out_stats; + // if the compaction supports per_key_placement, only output range dels to + // the penultimate level. + // Note: Use `bottommost_level_ = true` for both bottommost and + // output_to_penultimate_level compaction here, as it's only used to decide + // if range dels could be dropped. + if (outputs.HasRangeDel()) { + s = outputs.AddRangeDels(comp_start_user_key, comp_end_user_key, + range_del_out_stats, bottommost_level_, + cfd->internal_comparator(), earliest_snapshot, + next_table_min_key, full_history_ts_low_); + } + RecordDroppedKeys(range_del_out_stats, &sub_compact->compaction_job_stats); + TEST_SYNC_POINT("CompactionJob::FinishCompactionOutputFile1"); + } + + const uint64_t current_entries = outputs.NumEntries(); + + s = outputs.Finish(s, seqno_time_mapping_); + + if (s.ok()) { + // With accurate smallest and largest key, we can get a slightly more + // accurate oldest ancester time. + // This makes oldest ancester time in manifest more accurate than in + // table properties. Not sure how to resolve it. + if (meta->smallest.size() > 0 && meta->largest.size() > 0) { + uint64_t refined_oldest_ancester_time; + Slice new_smallest = meta->smallest.user_key(); + Slice new_largest = meta->largest.user_key(); + if (!new_largest.empty() && !new_smallest.empty()) { + refined_oldest_ancester_time = + sub_compact->compaction->MinInputFileOldestAncesterTime( + &(meta->smallest), &(meta->largest)); + if (refined_oldest_ancester_time != + std::numeric_limits::max()) { + meta->oldest_ancester_time = refined_oldest_ancester_time; + } + } + } + } + + // Finish and check for file errors + IOStatus io_s = outputs.WriterSyncClose(s, db_options_.clock, stats_, + db_options_.use_fsync); + + if (s.ok() && io_s.ok()) { + file_checksum = meta->file_checksum; + file_checksum_func_name = meta->file_checksum_func_name; + } + + if (s.ok()) { + s = io_s; + } + if (sub_compact->io_status.ok()) { + sub_compact->io_status = io_s; + // Since this error is really a copy of the + // "normal" status, it does not also need to be checked + sub_compact->io_status.PermitUncheckedError(); + } + + TableProperties tp; + if (s.ok()) { + tp = outputs.GetTableProperties(); + } + + if (s.ok() && current_entries == 0 && tp.num_range_deletions == 0) { + // If there is nothing to output, no necessary to generate a sst file. + // This happens when the output level is bottom level, at the same time + // the sub_compact output nothing. + std::string fname = + TableFileName(sub_compact->compaction->immutable_options()->cf_paths, + meta->fd.GetNumber(), meta->fd.GetPathId()); + + // TODO(AR) it is not clear if there are any larger implications if + // DeleteFile fails here + Status ds = env_->DeleteFile(fname); + if (!ds.ok()) { + ROCKS_LOG_WARN( + db_options_.info_log, + "[%s] [JOB %d] Unable to remove SST file for table #%" PRIu64 + " at bottom level%s", + cfd->GetName().c_str(), job_id_, output_number, + meta->marked_for_compaction ? " (need compaction)" : ""); + } + + // Also need to remove the file from outputs, or it will be added to the + // VersionEdit. + outputs.RemoveLastOutput(); + meta = nullptr; + } + + if (s.ok() && (current_entries > 0 || tp.num_range_deletions > 0)) { + // Output to event logger and fire events. + outputs.UpdateTableProperties(); + ROCKS_LOG_INFO(db_options_.info_log, + "[%s] [JOB %d] Generated table #%" PRIu64 ": %" PRIu64 + " keys, %" PRIu64 " bytes%s, temperature: %s", + cfd->GetName().c_str(), job_id_, output_number, + current_entries, meta->fd.file_size, + meta->marked_for_compaction ? " (need compaction)" : "", + temperature_to_string[meta->temperature].c_str()); + } + std::string fname; + FileDescriptor output_fd; + uint64_t oldest_blob_file_number = kInvalidBlobFileNumber; + Status status_for_listener = s; + if (meta != nullptr) { + fname = GetTableFileName(meta->fd.GetNumber()); + output_fd = meta->fd; + oldest_blob_file_number = meta->oldest_blob_file_number; + } else { + fname = "(nil)"; + if (s.ok()) { + status_for_listener = Status::Aborted("Empty SST file not kept"); + } + } + EventHelpers::LogAndNotifyTableFileCreationFinished( + event_logger_, cfd->ioptions()->listeners, dbname_, cfd->GetName(), fname, + job_id_, output_fd, oldest_blob_file_number, tp, + TableFileCreationReason::kCompaction, status_for_listener, file_checksum, + file_checksum_func_name); + + // Report new file to SstFileManagerImpl + auto sfm = + static_cast(db_options_.sst_file_manager.get()); + if (sfm && meta != nullptr && meta->fd.GetPathId() == 0) { + Status add_s = sfm->OnAddFile(fname); + if (!add_s.ok() && s.ok()) { + s = add_s; + } + if (sfm->IsMaxAllowedSpaceReached()) { + // TODO(ajkr): should we return OK() if max space was reached by the final + // compaction output file (similarly to how flush works when full)? + s = Status::SpaceLimit("Max allowed space was reached"); + TEST_SYNC_POINT( + "CompactionJob::FinishCompactionOutputFile:MaxAllowedSpaceReached"); + InstrumentedMutexLock l(db_mutex_); + db_error_handler_->SetBGError(s, BackgroundErrorReason::kCompaction); + } + } + + outputs.ResetBuilder(); + return s; +} + +Status CompactionJob::InstallCompactionResults( + const MutableCFOptions& mutable_cf_options) { + assert(compact_); + + db_mutex_->AssertHeld(); + + const ReadOptions read_options(Env::IOActivity::kCompaction); + auto* compaction = compact_->compaction; + assert(compaction); + + { + Compaction::InputLevelSummaryBuffer inputs_summary; + if (compaction_stats_.has_penultimate_level_output) { + ROCKS_LOG_BUFFER( + log_buffer_, + "[%s] [JOB %d] Compacted %s => output_to_penultimate_level: %" PRIu64 + " bytes + last: %" PRIu64 " bytes. Total: %" PRIu64 " bytes", + compaction->column_family_data()->GetName().c_str(), job_id_, + compaction->InputLevelSummary(&inputs_summary), + compaction_stats_.penultimate_level_stats.bytes_written, + compaction_stats_.stats.bytes_written, + compaction_stats_.TotalBytesWritten()); + } else { + ROCKS_LOG_BUFFER(log_buffer_, + "[%s] [JOB %d] Compacted %s => %" PRIu64 " bytes", + compaction->column_family_data()->GetName().c_str(), + job_id_, compaction->InputLevelSummary(&inputs_summary), + compaction_stats_.TotalBytesWritten()); + } + } + + VersionEdit* const edit = compaction->edit(); + assert(edit); + + // Add compaction inputs + compaction->AddInputDeletions(edit); + + std::unordered_map blob_total_garbage; + + for (const auto& sub_compact : compact_->sub_compact_states) { + sub_compact.AddOutputsEdit(edit); + + for (const auto& blob : sub_compact.Current().GetBlobFileAdditions()) { + edit->AddBlobFile(blob); + } + + if (sub_compact.Current().GetBlobGarbageMeter()) { + const auto& flows = sub_compact.Current().GetBlobGarbageMeter()->flows(); + + for (const auto& pair : flows) { + const uint64_t blob_file_number = pair.first; + const BlobGarbageMeter::BlobInOutFlow& flow = pair.second; + + assert(flow.IsValid()); + if (flow.HasGarbage()) { + blob_total_garbage[blob_file_number].Add(flow.GetGarbageCount(), + flow.GetGarbageBytes()); + } + } + } + } + + for (const auto& pair : blob_total_garbage) { + const uint64_t blob_file_number = pair.first; + const BlobGarbageMeter::BlobStats& stats = pair.second; + + edit->AddBlobFileGarbage(blob_file_number, stats.GetCount(), + stats.GetBytes()); + } + + if ((compaction->compaction_reason() == + CompactionReason::kLevelMaxLevelSize || + compaction->compaction_reason() == CompactionReason::kRoundRobinTtl) && + compaction->immutable_options()->compaction_pri == kRoundRobin) { + int start_level = compaction->start_level(); + if (start_level > 0) { + auto vstorage = compaction->input_version()->storage_info(); + edit->AddCompactCursor(start_level, + vstorage->GetNextCompactCursor( + start_level, compaction->num_input_files(0))); + } + } + + return versions_->LogAndApply(compaction->column_family_data(), + mutable_cf_options, read_options, edit, + db_mutex_, db_directory_); +} + +void CompactionJob::RecordCompactionIOStats() { + RecordTick(stats_, COMPACT_READ_BYTES, IOSTATS(bytes_read)); + RecordTick(stats_, COMPACT_WRITE_BYTES, IOSTATS(bytes_written)); + CompactionReason compaction_reason = + compact_->compaction->compaction_reason(); + if (compaction_reason == CompactionReason::kFilesMarkedForCompaction) { + RecordTick(stats_, COMPACT_READ_BYTES_MARKED, IOSTATS(bytes_read)); + RecordTick(stats_, COMPACT_WRITE_BYTES_MARKED, IOSTATS(bytes_written)); + } else if (compaction_reason == CompactionReason::kPeriodicCompaction) { + RecordTick(stats_, COMPACT_READ_BYTES_PERIODIC, IOSTATS(bytes_read)); + RecordTick(stats_, COMPACT_WRITE_BYTES_PERIODIC, IOSTATS(bytes_written)); + } else if (compaction_reason == CompactionReason::kTtl) { + RecordTick(stats_, COMPACT_READ_BYTES_TTL, IOSTATS(bytes_read)); + RecordTick(stats_, COMPACT_WRITE_BYTES_TTL, IOSTATS(bytes_written)); + } + ThreadStatusUtil::IncreaseThreadOperationProperty( + ThreadStatus::COMPACTION_BYTES_READ, IOSTATS(bytes_read)); + IOSTATS_RESET(bytes_read); + ThreadStatusUtil::IncreaseThreadOperationProperty( + ThreadStatus::COMPACTION_BYTES_WRITTEN, IOSTATS(bytes_written)); + IOSTATS_RESET(bytes_written); +} + +Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact, + CompactionOutputs& outputs) { + assert(sub_compact != nullptr); + + // no need to lock because VersionSet::next_file_number_ is atomic + uint64_t file_number = versions_->NewFileNumber(); + std::string fname = GetTableFileName(file_number); + // Fire events. + ColumnFamilyData* cfd = sub_compact->compaction->column_family_data(); + EventHelpers::NotifyTableFileCreationStarted( + cfd->ioptions()->listeners, dbname_, cfd->GetName(), fname, job_id_, + TableFileCreationReason::kCompaction); + // Make the output file + std::unique_ptr writable_file; +#ifndef NDEBUG + bool syncpoint_arg = file_options_.use_direct_writes; + TEST_SYNC_POINT_CALLBACK("CompactionJob::OpenCompactionOutputFile", + &syncpoint_arg); +#endif + + // Pass temperature of the last level files to FileSystem. + FileOptions fo_copy = file_options_; + Temperature temperature = sub_compact->compaction->output_temperature(); + // only set for the last level compaction and also it's not output to + // penultimate level (when preclude_last_level feature is enabled) + if (temperature == Temperature::kUnknown && + sub_compact->compaction->is_last_level() && + !sub_compact->IsCurrentPenultimateLevel()) { + temperature = + sub_compact->compaction->mutable_cf_options()->last_level_temperature; + } + fo_copy.temperature = temperature; + + Status s; + IOStatus io_s = NewWritableFile(fs_.get(), fname, &writable_file, fo_copy); + s = io_s; + if (sub_compact->io_status.ok()) { + sub_compact->io_status = io_s; + // Since this error is really a copy of the io_s that is checked below as s, + // it does not also need to be checked. + sub_compact->io_status.PermitUncheckedError(); + } + if (!s.ok()) { + ROCKS_LOG_ERROR( + db_options_.info_log, + "[%s] [JOB %d] OpenCompactionOutputFiles for table #%" PRIu64 + " fails at NewWritableFile with status %s", + sub_compact->compaction->column_family_data()->GetName().c_str(), + job_id_, file_number, s.ToString().c_str()); + LogFlush(db_options_.info_log); + EventHelpers::LogAndNotifyTableFileCreationFinished( + event_logger_, cfd->ioptions()->listeners, dbname_, cfd->GetName(), + fname, job_id_, FileDescriptor(), kInvalidBlobFileNumber, + TableProperties(), TableFileCreationReason::kCompaction, s, + kUnknownFileChecksum, kUnknownFileChecksumFuncName); + return s; + } + + // Try to figure out the output file's oldest ancester time. + int64_t temp_current_time = 0; + auto get_time_status = db_options_.clock->GetCurrentTime(&temp_current_time); + // Safe to proceed even if GetCurrentTime fails. So, log and proceed. + if (!get_time_status.ok()) { + ROCKS_LOG_WARN(db_options_.info_log, + "Failed to get current time. Status: %s", + get_time_status.ToString().c_str()); + } + uint64_t current_time = static_cast(temp_current_time); + InternalKey tmp_start, tmp_end; + if (sub_compact->start.has_value()) { + tmp_start.SetMinPossibleForUserKey(*(sub_compact->start)); + } + if (sub_compact->end.has_value()) { + tmp_end.SetMinPossibleForUserKey(*(sub_compact->end)); + } + uint64_t oldest_ancester_time = + sub_compact->compaction->MinInputFileOldestAncesterTime( + sub_compact->start.has_value() ? &tmp_start : nullptr, + sub_compact->end.has_value() ? &tmp_end : nullptr); + if (oldest_ancester_time == std::numeric_limits::max()) { + oldest_ancester_time = current_time; + } + + // Initialize a SubcompactionState::Output and add it to sub_compact->outputs + uint64_t epoch_number = sub_compact->compaction->MinInputFileEpochNumber(); + { + FileMetaData meta; + meta.fd = FileDescriptor(file_number, + sub_compact->compaction->output_path_id(), 0); + meta.oldest_ancester_time = oldest_ancester_time; + meta.file_creation_time = current_time; + meta.epoch_number = epoch_number; + meta.temperature = temperature; + assert(!db_id_.empty()); + assert(!db_session_id_.empty()); + s = GetSstInternalUniqueId(db_id_, db_session_id_, meta.fd.GetNumber(), + &meta.unique_id); + if (!s.ok()) { + ROCKS_LOG_ERROR(db_options_.info_log, + "[%s] [JOB %d] file #%" PRIu64 + " failed to generate unique id: %s.", + cfd->GetName().c_str(), job_id_, meta.fd.GetNumber(), + s.ToString().c_str()); + return s; + } + + outputs.AddOutput(std::move(meta), cfd->internal_comparator(), + sub_compact->compaction->mutable_cf_options() + ->check_flush_compaction_key_order, + paranoid_file_checks_); + } + + writable_file->SetIOPriority(GetRateLimiterPriority()); + writable_file->SetWriteLifeTimeHint(write_hint_); + FileTypeSet tmp_set = db_options_.checksum_handoff_file_types; + writable_file->SetPreallocationBlockSize(static_cast( + sub_compact->compaction->OutputFilePreallocationSize())); + const auto& listeners = + sub_compact->compaction->immutable_options()->listeners; + outputs.AssignFileWriter(new WritableFileWriter( + std::move(writable_file), fname, fo_copy, db_options_.clock, io_tracer_, + db_options_.stats, listeners, db_options_.file_checksum_gen_factory.get(), + tmp_set.Contains(FileType::kTableFile), false)); + + TableBuilderOptions tboptions( + *cfd->ioptions(), *(sub_compact->compaction->mutable_cf_options()), + cfd->internal_comparator(), cfd->int_tbl_prop_collector_factories(), + sub_compact->compaction->output_compression(), + sub_compact->compaction->output_compression_opts(), cfd->GetID(), + cfd->GetName(), sub_compact->compaction->output_level(), + bottommost_level_, TableFileCreationReason::kCompaction, + 0 /* oldest_key_time */, current_time, db_id_, db_session_id_, + sub_compact->compaction->max_output_file_size(), file_number); + + outputs.NewBuilder(tboptions); + + LogFlush(db_options_.info_log); + return s; +} + +void CompactionJob::CleanupCompaction() { + for (SubcompactionState& sub_compact : compact_->sub_compact_states) { + sub_compact.Cleanup(table_cache_.get()); + } + delete compact_; + compact_ = nullptr; +} + +namespace { +void CopyPrefix(const Slice& src, size_t prefix_length, std::string* dst) { + assert(prefix_length > 0); + size_t length = src.size() > prefix_length ? prefix_length : src.size(); + dst->assign(src.data(), length); +} +} // namespace + +bool CompactionJob::UpdateCompactionStats(uint64_t* num_input_range_del) { + assert(compact_); + + Compaction* compaction = compact_->compaction; + compaction_stats_.stats.num_input_files_in_non_output_levels = 0; + compaction_stats_.stats.num_input_files_in_output_level = 0; + + bool has_error = false; + const ReadOptions read_options(Env::IOActivity::kCompaction); + const auto& input_table_properties = compaction->GetTableProperties(); + for (int input_level = 0; + input_level < static_cast(compaction->num_input_levels()); + ++input_level) { + size_t num_input_files = compaction->num_input_files(input_level); + uint64_t* bytes_read; + if (compaction->level(input_level) != compaction->output_level()) { + compaction_stats_.stats.num_input_files_in_non_output_levels += + static_cast(num_input_files); + bytes_read = &compaction_stats_.stats.bytes_read_non_output_levels; + } else { + compaction_stats_.stats.num_input_files_in_output_level += + static_cast(num_input_files); + bytes_read = &compaction_stats_.stats.bytes_read_output_level; + } + for (size_t i = 0; i < num_input_files; ++i) { + const FileMetaData* file_meta = compaction->input(input_level, i); + *bytes_read += file_meta->fd.GetFileSize(); + uint64_t file_input_entries = file_meta->num_entries; + uint64_t file_num_range_del = file_meta->num_range_deletions; + if (file_input_entries == 0) { + uint64_t file_number = file_meta->fd.GetNumber(); + // Try getting info from table property + std::string fn = + TableFileName(compaction->immutable_options()->cf_paths, + file_number, file_meta->fd.GetPathId()); + const auto& tp = input_table_properties.find(fn); + if (tp != input_table_properties.end()) { + file_input_entries = tp->second->num_entries; + file_num_range_del = tp->second->num_range_deletions; + } else { + has_error = true; + } + } + compaction_stats_.stats.num_input_records += file_input_entries; + if (num_input_range_del) { + *num_input_range_del += file_num_range_del; + } + } + } + + assert(compaction_job_stats_); + compaction_stats_.stats.bytes_read_blob = + compaction_job_stats_->total_blob_bytes_read; + + compaction_stats_.stats.num_dropped_records = + compaction_stats_.DroppedRecords(); + return !has_error; +} + +void CompactionJob::UpdateCompactionJobStats( + const InternalStats::CompactionStats& stats) const { + compaction_job_stats_->elapsed_micros = stats.micros; + + // input information + compaction_job_stats_->total_input_bytes = + stats.bytes_read_non_output_levels + stats.bytes_read_output_level; + compaction_job_stats_->num_input_records = stats.num_input_records; + compaction_job_stats_->num_input_files = + stats.num_input_files_in_non_output_levels + + stats.num_input_files_in_output_level; + compaction_job_stats_->num_input_files_at_output_level = + stats.num_input_files_in_output_level; + + // output information + compaction_job_stats_->total_output_bytes = stats.bytes_written; + compaction_job_stats_->total_output_bytes_blob = stats.bytes_written_blob; + compaction_job_stats_->num_output_records = stats.num_output_records; + compaction_job_stats_->num_output_files = stats.num_output_files; + compaction_job_stats_->num_output_files_blob = stats.num_output_files_blob; + + if (stats.num_output_files > 0) { + CopyPrefix(compact_->SmallestUserKey(), + CompactionJobStats::kMaxPrefixLength, + &compaction_job_stats_->smallest_output_key_prefix); + CopyPrefix(compact_->LargestUserKey(), CompactionJobStats::kMaxPrefixLength, + &compaction_job_stats_->largest_output_key_prefix); + } +} + +void CompactionJob::LogCompaction() { + Compaction* compaction = compact_->compaction; + ColumnFamilyData* cfd = compaction->column_family_data(); + + // Let's check if anything will get logged. Don't prepare all the info if + // we're not logging + if (db_options_.info_log_level <= InfoLogLevel::INFO_LEVEL) { + Compaction::InputLevelSummaryBuffer inputs_summary; + ROCKS_LOG_INFO( + db_options_.info_log, "[%s] [JOB %d] Compacting %s, score %.2f", + cfd->GetName().c_str(), job_id_, + compaction->InputLevelSummary(&inputs_summary), compaction->score()); + char scratch[2345]; + compaction->Summary(scratch, sizeof(scratch)); + ROCKS_LOG_INFO(db_options_.info_log, "[%s]: Compaction start summary: %s\n", + cfd->GetName().c_str(), scratch); + // build event logger report + auto stream = event_logger_->Log(); + stream << "job" << job_id_ << "event" + << "compaction_started" + << "compaction_reason" + << GetCompactionReasonString(compaction->compaction_reason()); + for (size_t i = 0; i < compaction->num_input_levels(); ++i) { + stream << ("files_L" + std::to_string(compaction->level(i))); + stream.StartArray(); + for (auto f : *compaction->inputs(i)) { + stream << f->fd.GetNumber(); + } + stream.EndArray(); + } + stream << "score" << compaction->score() << "input_data_size" + << compaction->CalculateTotalInputSize() << "oldest_snapshot_seqno" + << (existing_snapshots_.empty() + ? int64_t{-1} // Use -1 for "none" + : static_cast(existing_snapshots_[0])); + if (compaction->SupportsPerKeyPlacement()) { + stream << "preclude_last_level_min_seqno" + << preclude_last_level_min_seqno_; + stream << "penultimate_output_level" << compaction->GetPenultimateLevel(); + stream << "penultimate_output_range" + << GetCompactionPenultimateOutputRangeTypeString( + compaction->GetPenultimateOutputRangeType()); + + if (compaction->GetPenultimateOutputRangeType() == + Compaction::PenultimateOutputRangeType::kDisabled) { + ROCKS_LOG_WARN( + db_options_.info_log, + "[%s] [JOB %d] Penultimate level output is disabled, likely " + "because of the range conflict in the penultimate level", + cfd->GetName().c_str(), job_id_); + } + } + } +} + +std::string CompactionJob::GetTableFileName(uint64_t file_number) { + return TableFileName(compact_->compaction->immutable_options()->cf_paths, + file_number, compact_->compaction->output_path_id()); +} + +Env::IOPriority CompactionJob::GetRateLimiterPriority() { + if (versions_ && versions_->GetColumnFamilySet() && + versions_->GetColumnFamilySet()->write_controller()) { + WriteController* write_controller = + versions_->GetColumnFamilySet()->write_controller(); + if (write_controller->NeedsDelay() || write_controller->IsStopped()) { + return Env::IO_USER; + } + } + + return Env::IO_LOW; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_job.h b/librocksdb-sys/rocksdb/db/compaction/compaction_job.h new file mode 100644 index 0000000..926f4a8 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_job.h @@ -0,0 +1,515 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "db/blob/blob_file_completion_callback.h" +#include "db/column_family.h" +#include "db/compaction/compaction_iterator.h" +#include "db/compaction/compaction_outputs.h" +#include "db/flush_scheduler.h" +#include "db/internal_stats.h" +#include "db/job_context.h" +#include "db/log_writer.h" +#include "db/memtable_list.h" +#include "db/range_del_aggregator.h" +#include "db/seqno_to_time_mapping.h" +#include "db/version_edit.h" +#include "db/write_controller.h" +#include "db/write_thread.h" +#include "logging/event_logger.h" +#include "options/cf_options.h" +#include "options/db_options.h" +#include "port/port.h" +#include "rocksdb/compaction_filter.h" +#include "rocksdb/compaction_job_stats.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/transaction_log.h" +#include "table/scoped_arena_iterator.h" +#include "util/autovector.h" +#include "util/stop_watch.h" +#include "util/thread_local.h" + +namespace ROCKSDB_NAMESPACE { + +class Arena; +class CompactionState; +class ErrorHandler; +class MemTable; +class SnapshotChecker; +class SystemClock; +class TableCache; +class Version; +class VersionEdit; +class VersionSet; + +class SubcompactionState; + +// CompactionJob is responsible for executing the compaction. Each (manual or +// automated) compaction corresponds to a CompactionJob object, and usually +// goes through the stages of `Prepare()`->`Run()`->`Install()`. CompactionJob +// will divide the compaction into subcompactions and execute them in parallel +// if needed. +// +// CompactionJob has 2 main stats: +// 1. CompactionJobStats compaction_job_stats_ +// CompactionJobStats is a public data structure which is part of Compaction +// event listener that rocksdb share the job stats with the user. +// Internally it's an aggregation of all the compaction_job_stats from each +// `SubcompactionState`: +// +------------------------+ +// | SubcompactionState | +// | | +// +--------->| compaction_job_stats | +// | | | +// | +------------------------+ +// +------------------------+ | +// | CompactionJob | | +------------------------+ +// | | | | SubcompactionState | +// | compaction_job_stats +-----+ | | +// | | +--------->| compaction_job_stats | +// | | | | | +// +------------------------+ | +------------------------+ +// | +// | +------------------------+ +// | | SubcompactionState | +// | | | +// +--------->+ compaction_job_stats | +// | | | +// | +------------------------+ +// | +// | +------------------------+ +// | | ... | +// +--------->+ | +// +------------------------+ +// +// 2. CompactionStatsFull compaction_stats_ +// `CompactionStatsFull` is an internal stats about the compaction, which +// is eventually sent to `ColumnFamilyData::internal_stats_` and used for +// logging and public metrics. +// Internally, it's an aggregation of stats_ from each `SubcompactionState`. +// It has 2 parts, normal stats about the main compaction information and +// the penultimate level output stats. +// `SubcompactionState` maintains the CompactionOutputs for normal output and +// the penultimate level output if exists, the per_level stats is +// stored with the outputs. +// +---------------------------+ +// | SubcompactionState | +// | | +// | +----------------------+ | +// | | CompactionOutputs | | +// | | (normal output) | | +// +---->| stats_ | | +// | | +----------------------+ | +// | | | +// | | +----------------------+ | +// +--------------------------------+ | | | CompactionOutputs | | +// | CompactionJob | | | | (penultimate_level) | | +// | | +--------->| stats_ | | +// | compaction_stats_ | | | | +----------------------+ | +// | +-------------------------+ | | | | | +// | |stats (normal) |------|----+ +---------------------------+ +// | +-------------------------+ | | | +// | | | | +// | +-------------------------+ | | | +---------------------------+ +// | |penultimate_level_stats +------+ | | SubcompactionState | +// | +-------------------------+ | | | | | +// | | | | | +----------------------+ | +// | | | | | | CompactionOutputs | | +// +--------------------------------+ | | | | (normal output) | | +// | +---->| stats_ | | +// | | +----------------------+ | +// | | | +// | | +----------------------+ | +// | | | CompactionOutputs | | +// | | | (penultimate_level) | | +// +--------->| stats_ | | +// | +----------------------+ | +// | | +// +---------------------------+ + +class CompactionJob { + public: + CompactionJob( + int job_id, Compaction* compaction, const ImmutableDBOptions& db_options, + const MutableDBOptions& mutable_db_options, + const FileOptions& file_options, VersionSet* versions, + const std::atomic* shutting_down, LogBuffer* log_buffer, + FSDirectory* db_directory, FSDirectory* output_directory, + FSDirectory* blob_output_directory, Statistics* stats, + InstrumentedMutex* db_mutex, ErrorHandler* db_error_handler, + std::vector existing_snapshots, + SequenceNumber earliest_write_conflict_snapshot, + const SnapshotChecker* snapshot_checker, JobContext* job_context, + std::shared_ptr table_cache, EventLogger* event_logger, + bool paranoid_file_checks, bool measure_io_stats, + const std::string& dbname, CompactionJobStats* compaction_job_stats, + Env::Priority thread_pri, const std::shared_ptr& io_tracer, + const std::atomic& manual_compaction_canceled, + const std::string& db_id = "", const std::string& db_session_id = "", + std::string full_history_ts_low = "", std::string trim_ts = "", + BlobFileCompletionCallback* blob_callback = nullptr, + int* bg_compaction_scheduled = nullptr, + int* bg_bottom_compaction_scheduled = nullptr); + + virtual ~CompactionJob(); + + // no copy/move + CompactionJob(CompactionJob&& job) = delete; + CompactionJob(const CompactionJob& job) = delete; + CompactionJob& operator=(const CompactionJob& job) = delete; + + // REQUIRED: mutex held + // Prepare for the compaction by setting up boundaries for each subcompaction + void Prepare(); + // REQUIRED mutex not held + // Launch threads for each subcompaction and wait for them to finish. After + // that, verify table is usable and finally do bookkeeping to unify + // subcompaction results + Status Run(); + + // REQUIRED: mutex held + // Add compaction input/output to the current version + Status Install(const MutableCFOptions& mutable_cf_options); + + // Return the IO status + IOStatus io_status() const { return io_status_; } + + protected: + // Update the following stats in compaction_stats_.stats + // - num_input_files_in_non_output_levels + // - num_input_files_in_output_level + // - bytes_read_non_output_levels + // - bytes_read_output_level + // - num_input_records + // - bytes_read_blob + // - num_dropped_records + // + // @param num_input_range_del if non-null, will be set to the number of range + // deletion entries in this compaction input. + // + // Returns true iff compaction_stats_.stats.num_input_records and + // num_input_range_del are calculated successfully. + bool UpdateCompactionStats(uint64_t* num_input_range_del = nullptr); + void LogCompaction(); + virtual void RecordCompactionIOStats(); + void CleanupCompaction(); + + // Call compaction filter. Then iterate through input and compact the + // kv-pairs + void ProcessKeyValueCompaction(SubcompactionState* sub_compact); + + CompactionState* compact_; + InternalStats::CompactionStatsFull compaction_stats_; + const ImmutableDBOptions& db_options_; + const MutableDBOptions mutable_db_options_copy_; + LogBuffer* log_buffer_; + FSDirectory* output_directory_; + Statistics* stats_; + // Is this compaction creating a file in the bottom most level? + bool bottommost_level_; + + Env::WriteLifeTimeHint write_hint_; + + IOStatus io_status_; + + CompactionJobStats* compaction_job_stats_; + + private: + friend class CompactionJobTestBase; + + // Generates a histogram representing potential divisions of key ranges from + // the input. It adds the starting and/or ending keys of certain input files + // to the working set and then finds the approximate size of data in between + // each consecutive pair of slices. Then it divides these ranges into + // consecutive groups such that each group has a similar size. + void GenSubcompactionBoundaries(); + + // Get the number of planned subcompactions based on max_subcompactions and + // extra reserved resources + uint64_t GetSubcompactionsLimit(); + + // Additional reserved threads are reserved and the number is stored in + // extra_num_subcompaction_threads_reserved__. For now, this happens only if + // the compaction priority is round-robin and max_subcompactions is not + // sufficient (extra resources may be needed) + void AcquireSubcompactionResources(int num_extra_required_subcompactions); + + // Additional threads may be reserved during IncreaseSubcompactionResources() + // if num_actual_subcompactions is less than num_planned_subcompactions. + // Additional threads will be released and the bg_compaction_scheduled_ or + // bg_bottom_compaction_scheduled_ will be updated if they are used. + // DB Mutex lock is required. + void ShrinkSubcompactionResources(uint64_t num_extra_resources); + + // Release all reserved threads and update the compaction limits. + void ReleaseSubcompactionResources(); + + CompactionServiceJobStatus ProcessKeyValueCompactionWithCompactionService( + SubcompactionState* sub_compact); + + // update the thread status for starting a compaction. + void ReportStartedCompaction(Compaction* compaction); + + Status FinishCompactionOutputFile(const Status& input_status, + SubcompactionState* sub_compact, + CompactionOutputs& outputs, + const Slice& next_table_min_key, + const Slice* comp_start_user_key, + const Slice* comp_end_user_key); + Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options); + Status OpenCompactionOutputFile(SubcompactionState* sub_compact, + CompactionOutputs& outputs); + void UpdateCompactionJobStats( + const InternalStats::CompactionStats& stats) const; + void RecordDroppedKeys(const CompactionIterationStats& c_iter_stats, + CompactionJobStats* compaction_job_stats = nullptr); + + void NotifyOnSubcompactionBegin(SubcompactionState* sub_compact); + + void NotifyOnSubcompactionCompleted(SubcompactionState* sub_compact); + + uint32_t job_id_; + + // DBImpl state + const std::string& dbname_; + const std::string db_id_; + const std::string db_session_id_; + const FileOptions file_options_; + + Env* env_; + std::shared_ptr io_tracer_; + FileSystemPtr fs_; + // env_option optimized for compaction table reads + FileOptions file_options_for_read_; + VersionSet* versions_; + const std::atomic* shutting_down_; + const std::atomic& manual_compaction_canceled_; + FSDirectory* db_directory_; + FSDirectory* blob_output_directory_; + InstrumentedMutex* db_mutex_; + ErrorHandler* db_error_handler_; + // If there were two snapshots with seq numbers s1 and + // s2 and s1 < s2, and if we find two instances of a key k1 then lies + // entirely within s1 and s2, then the earlier version of k1 can be safely + // deleted because that version is not visible in any snapshot. + std::vector existing_snapshots_; + + // This is the earliest snapshot that could be used for write-conflict + // checking by a transaction. For any user-key newer than this snapshot, we + // should make sure not to remove evidence that a write occurred. + SequenceNumber earliest_write_conflict_snapshot_; + + const SnapshotChecker* const snapshot_checker_; + + JobContext* job_context_; + + std::shared_ptr table_cache_; + + EventLogger* event_logger_; + + bool paranoid_file_checks_; + bool measure_io_stats_; + // Stores the Slices that designate the boundaries for each subcompaction + std::vector boundaries_; + Env::Priority thread_pri_; + std::string full_history_ts_low_; + std::string trim_ts_; + BlobFileCompletionCallback* blob_callback_; + + uint64_t GetCompactionId(SubcompactionState* sub_compact) const; + // Stores the number of reserved threads in shared env_ for the number of + // extra subcompaction in kRoundRobin compaction priority + int extra_num_subcompaction_threads_reserved_; + + // Stores the pointer to bg_compaction_scheduled_, + // bg_bottom_compaction_scheduled_ in DBImpl. Mutex is required when accessing + // or updating it. + int* bg_compaction_scheduled_; + int* bg_bottom_compaction_scheduled_; + + // Stores the sequence number to time mapping gathered from all input files + // it also collects the smallest_seqno -> oldest_ancester_time from the SST. + SeqnoToTimeMapping seqno_time_mapping_; + + // Minimal sequence number for preserving the time information. The time info + // older than this sequence number won't be preserved after the compaction and + // if it's bottommost compaction, the seq num will be zeroed out. + SequenceNumber preserve_time_min_seqno_ = kMaxSequenceNumber; + + // Minimal sequence number to preclude the data from the last level. If the + // key has bigger (newer) sequence number than this, it will be precluded from + // the last level (output to penultimate level). + SequenceNumber preclude_last_level_min_seqno_ = kMaxSequenceNumber; + + // Get table file name in where it's outputting to, which should also be in + // `output_directory_`. + virtual std::string GetTableFileName(uint64_t file_number); + // The rate limiter priority (io_priority) is determined dynamically here. + // The Compaction Read and Write priorities are the same for different + // scenarios, such as write stalled. + Env::IOPriority GetRateLimiterPriority(); +}; + +// CompactionServiceInput is used the pass compaction information between two +// db instances. It contains the information needed to do a compaction. It +// doesn't contain the LSM tree information, which is passed though MANIFEST +// file. +struct CompactionServiceInput { + ColumnFamilyDescriptor column_family; + + DBOptions db_options; + + std::vector snapshots; + + // SST files for compaction, it should already be expended to include all the + // files needed for this compaction, for both input level files and output + // level files. + std::vector input_files; + int output_level; + + // db_id is used to generate unique id of sst on the remote compactor + std::string db_id; + + // information for subcompaction + bool has_begin = false; + std::string begin; + bool has_end = false; + std::string end; + + // serialization interface to read and write the object + static Status Read(const std::string& data_str, CompactionServiceInput* obj); + Status Write(std::string* output); + + // Initialize a dummy ColumnFamilyDescriptor + CompactionServiceInput() : column_family("", ColumnFamilyOptions()) {} + +#ifndef NDEBUG + bool TEST_Equals(CompactionServiceInput* other); + bool TEST_Equals(CompactionServiceInput* other, std::string* mismatch); +#endif // NDEBUG +}; + +// CompactionServiceOutputFile is the metadata for the output SST file +struct CompactionServiceOutputFile { + std::string file_name; + SequenceNumber smallest_seqno; + SequenceNumber largest_seqno; + std::string smallest_internal_key; + std::string largest_internal_key; + uint64_t oldest_ancester_time; + uint64_t file_creation_time; + uint64_t epoch_number; + uint64_t paranoid_hash; + bool marked_for_compaction; + UniqueId64x2 unique_id; + + CompactionServiceOutputFile() = default; + CompactionServiceOutputFile( + const std::string& name, SequenceNumber smallest, SequenceNumber largest, + std::string _smallest_internal_key, std::string _largest_internal_key, + uint64_t _oldest_ancester_time, uint64_t _file_creation_time, + uint64_t _epoch_number, uint64_t _paranoid_hash, + bool _marked_for_compaction, UniqueId64x2 _unique_id) + : file_name(name), + smallest_seqno(smallest), + largest_seqno(largest), + smallest_internal_key(std::move(_smallest_internal_key)), + largest_internal_key(std::move(_largest_internal_key)), + oldest_ancester_time(_oldest_ancester_time), + file_creation_time(_file_creation_time), + epoch_number(_epoch_number), + paranoid_hash(_paranoid_hash), + marked_for_compaction(_marked_for_compaction), + unique_id(std::move(_unique_id)) {} +}; + +// CompactionServiceResult contains the compaction result from a different db +// instance, with these information, the primary db instance with write +// permission is able to install the result to the DB. +struct CompactionServiceResult { + Status status; + std::vector output_files; + int output_level; + + // location of the output files + std::string output_path; + + // some statistics about the compaction + uint64_t num_output_records = 0; + uint64_t total_bytes = 0; + uint64_t bytes_read = 0; + uint64_t bytes_written = 0; + CompactionJobStats stats; + + // serialization interface to read and write the object + static Status Read(const std::string& data_str, CompactionServiceResult* obj); + Status Write(std::string* output); + +#ifndef NDEBUG + bool TEST_Equals(CompactionServiceResult* other); + bool TEST_Equals(CompactionServiceResult* other, std::string* mismatch); +#endif // NDEBUG +}; + +// CompactionServiceCompactionJob is an read-only compaction job, it takes +// input information from `compaction_service_input` and put result information +// in `compaction_service_result`, the SST files are generated to `output_path`. +class CompactionServiceCompactionJob : private CompactionJob { + public: + CompactionServiceCompactionJob( + int job_id, Compaction* compaction, const ImmutableDBOptions& db_options, + const MutableDBOptions& mutable_db_options, + const FileOptions& file_options, VersionSet* versions, + const std::atomic* shutting_down, LogBuffer* log_buffer, + FSDirectory* output_directory, Statistics* stats, + InstrumentedMutex* db_mutex, ErrorHandler* db_error_handler, + std::vector existing_snapshots, + std::shared_ptr table_cache, EventLogger* event_logger, + const std::string& dbname, const std::shared_ptr& io_tracer, + const std::atomic& manual_compaction_canceled, + const std::string& db_id, const std::string& db_session_id, + std::string output_path, + const CompactionServiceInput& compaction_service_input, + CompactionServiceResult* compaction_service_result); + + // Run the compaction in current thread and return the result + Status Run(); + + void CleanupCompaction(); + + IOStatus io_status() const { return CompactionJob::io_status(); } + + protected: + void RecordCompactionIOStats() override; + + private: + // Get table file name in output_path + std::string GetTableFileName(uint64_t file_number) override; + // Specific the compaction output path, otherwise it uses default DB path + const std::string output_path_; + + // Compaction job input + const CompactionServiceInput& compaction_input_; + + // Compaction job result + CompactionServiceResult* compaction_result_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_job_stats_test.cc b/librocksdb-sys/rocksdb/db/compaction/compaction_job_stats_test.cc new file mode 100644 index 0000000..56fc51d --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_job_stats_test.cc @@ -0,0 +1,967 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "db/db_impl/db_impl.h" +#include "db/dbformat.h" +#include "db/job_context.h" +#include "db/version_set.h" +#include "db/write_batch_internal.h" +#include "env/mock_env.h" +#include "file/filename.h" +#include "monitoring/statistics_impl.h" +#include "monitoring/thread_status_util.h" +#include "port/stack_trace.h" +#include "rocksdb/cache.h" +#include "rocksdb/compaction_filter.h" +#include "rocksdb/convenience.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/experimental.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/options.h" +#include "rocksdb/perf_context.h" +#include "rocksdb/slice.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/table.h" +#include "rocksdb/table_properties.h" +#include "rocksdb/thread_status.h" +#include "rocksdb/utilities/checkpoint.h" +#include "rocksdb/utilities/write_batch_with_index.h" +#include "table/block_based/block_based_table_factory.h" +#include "table/mock_table.h" +#include "table/plain/plain_table_factory.h" +#include "table/scoped_arena_iterator.h" +#include "test_util/sync_point.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/cast_util.h" +#include "util/compression.h" +#include "util/hash.h" +#include "util/mutexlock.h" +#include "util/rate_limiter_impl.h" +#include "util/string_util.h" +#include "utilities/merge_operators.h" + +#if !defined(IOS_CROSS_COMPILE) +namespace ROCKSDB_NAMESPACE { + +static std::string RandomString(Random* rnd, int len, double ratio) { + std::string r; + test::CompressibleString(rnd, ratio, len, &r); + return r; +} + +std::string Key(uint64_t key, int length) { + const int kBufSize = 1000; + char buf[kBufSize]; + if (length > kBufSize) { + length = kBufSize; + } + snprintf(buf, kBufSize, "%0*" PRIu64, length, key); + return std::string(buf); +} + +class CompactionJobStatsTest : public testing::Test, + public testing::WithParamInterface { + public: + std::string dbname_; + std::string alternative_wal_dir_; + Env* env_; + DB* db_; + std::vector handles_; + uint32_t max_subcompactions_; + + Options last_options_; + + CompactionJobStatsTest() : env_(Env::Default()) { + env_->SetBackgroundThreads(1, Env::LOW); + env_->SetBackgroundThreads(1, Env::HIGH); + dbname_ = test::PerThreadDBPath("compaction_job_stats_test"); + alternative_wal_dir_ = dbname_ + "/wal"; + Options options; + options.create_if_missing = true; + max_subcompactions_ = GetParam(); + options.max_subcompactions = max_subcompactions_; + auto delete_options = options; + delete_options.wal_dir = alternative_wal_dir_; + EXPECT_OK(DestroyDB(dbname_, delete_options)); + // Destroy it for not alternative WAL dir is used. + EXPECT_OK(DestroyDB(dbname_, options)); + db_ = nullptr; + Reopen(options); + } + + ~CompactionJobStatsTest() override { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + Close(); + Options options; + options.db_paths.emplace_back(dbname_, 0); + options.db_paths.emplace_back(dbname_ + "_2", 0); + options.db_paths.emplace_back(dbname_ + "_3", 0); + options.db_paths.emplace_back(dbname_ + "_4", 0); + EXPECT_OK(DestroyDB(dbname_, options)); + } + + // Required if inheriting from testing::WithParamInterface<> + static void SetUpTestCase() {} + static void TearDownTestCase() {} + + DBImpl* dbfull() { return static_cast_with_check(db_); } + + void CreateColumnFamilies(const std::vector& cfs, + const Options& options) { + ColumnFamilyOptions cf_opts(options); + size_t cfi = handles_.size(); + handles_.resize(cfi + cfs.size()); + for (auto cf : cfs) { + ASSERT_OK(db_->CreateColumnFamily(cf_opts, cf, &handles_[cfi++])); + } + } + + void CreateAndReopenWithCF(const std::vector& cfs, + const Options& options) { + CreateColumnFamilies(cfs, options); + std::vector cfs_plus_default = cfs; + cfs_plus_default.insert(cfs_plus_default.begin(), kDefaultColumnFamilyName); + ReopenWithColumnFamilies(cfs_plus_default, options); + } + + void ReopenWithColumnFamilies(const std::vector& cfs, + const std::vector& options) { + ASSERT_OK(TryReopenWithColumnFamilies(cfs, options)); + } + + void ReopenWithColumnFamilies(const std::vector& cfs, + const Options& options) { + ASSERT_OK(TryReopenWithColumnFamilies(cfs, options)); + } + + Status TryReopenWithColumnFamilies(const std::vector& cfs, + const std::vector& options) { + Close(); + EXPECT_EQ(cfs.size(), options.size()); + std::vector column_families; + for (size_t i = 0; i < cfs.size(); ++i) { + column_families.push_back(ColumnFamilyDescriptor(cfs[i], options[i])); + } + DBOptions db_opts = DBOptions(options[0]); + return DB::Open(db_opts, dbname_, column_families, &handles_, &db_); + } + + Status TryReopenWithColumnFamilies(const std::vector& cfs, + const Options& options) { + Close(); + std::vector v_opts(cfs.size(), options); + return TryReopenWithColumnFamilies(cfs, v_opts); + } + + void Reopen(const Options& options) { ASSERT_OK(TryReopen(options)); } + + void Close() { + for (auto h : handles_) { + delete h; + } + handles_.clear(); + delete db_; + db_ = nullptr; + } + + void DestroyAndReopen(const Options& options) { + // Destroy using last options + Destroy(last_options_); + ASSERT_OK(TryReopen(options)); + } + + void Destroy(const Options& options) { + Close(); + ASSERT_OK(DestroyDB(dbname_, options)); + } + + Status ReadOnlyReopen(const Options& options) { + return DB::OpenForReadOnly(options, dbname_, &db_); + } + + Status TryReopen(const Options& options) { + Close(); + last_options_ = options; + return DB::Open(options, dbname_, &db_); + } + + Status Flush(int cf = 0) { + if (cf == 0) { + return db_->Flush(FlushOptions()); + } else { + return db_->Flush(FlushOptions(), handles_[cf]); + } + } + + Status Put(const Slice& k, const Slice& v, WriteOptions wo = WriteOptions()) { + return db_->Put(wo, k, v); + } + + Status Put(int cf, const Slice& k, const Slice& v, + WriteOptions wo = WriteOptions()) { + return db_->Put(wo, handles_[cf], k, v); + } + + Status Delete(const std::string& k) { return db_->Delete(WriteOptions(), k); } + + Status Delete(int cf, const std::string& k) { + return db_->Delete(WriteOptions(), handles_[cf], k); + } + + std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) { + ReadOptions options; + options.verify_checksums = true; + options.snapshot = snapshot; + std::string result; + Status s = db_->Get(options, k, &result); + if (s.IsNotFound()) { + result = "NOT_FOUND"; + } else if (!s.ok()) { + result = s.ToString(); + } + return result; + } + + std::string Get(int cf, const std::string& k, + const Snapshot* snapshot = nullptr) { + ReadOptions options; + options.verify_checksums = true; + options.snapshot = snapshot; + std::string result; + Status s = db_->Get(options, handles_[cf], k, &result); + if (s.IsNotFound()) { + result = "NOT_FOUND"; + } else if (!s.ok()) { + result = s.ToString(); + } + return result; + } + + int NumTableFilesAtLevel(int level, int cf = 0) { + std::string property; + if (cf == 0) { + // default cfd + EXPECT_TRUE(db_->GetProperty( + "rocksdb.num-files-at-level" + std::to_string(level), &property)); + } else { + EXPECT_TRUE(db_->GetProperty( + handles_[cf], "rocksdb.num-files-at-level" + std::to_string(level), + &property)); + } + return atoi(property.c_str()); + } + + // Return spread of files per level + std::string FilesPerLevel(int cf = 0) { + int num_levels = + (cf == 0) ? db_->NumberLevels() : db_->NumberLevels(handles_[1]); + std::string result; + size_t last_non_zero_offset = 0; + for (int level = 0; level < num_levels; level++) { + int f = NumTableFilesAtLevel(level, cf); + char buf[100]; + snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f); + result += buf; + if (f > 0) { + last_non_zero_offset = result.size(); + } + } + result.resize(last_non_zero_offset); + return result; + } + + Status Size(uint64_t* size, const Slice& start, const Slice& limit, + int cf = 0) { + Range r(start, limit); + if (cf == 0) { + return db_->GetApproximateSizes(&r, 1, size); + } else { + return db_->GetApproximateSizes(handles_[1], &r, 1, size); + } + } + + void Compact(int cf, const Slice& start, const Slice& limit, + uint32_t target_path_id) { + CompactRangeOptions compact_options; + compact_options.target_path_id = target_path_id; + ASSERT_OK(db_->CompactRange(compact_options, handles_[cf], &start, &limit)); + } + + void Compact(int cf, const Slice& start, const Slice& limit) { + ASSERT_OK( + db_->CompactRange(CompactRangeOptions(), handles_[cf], &start, &limit)); + } + + void Compact(const Slice& start, const Slice& limit) { + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &start, &limit)); + } + + void TEST_Compact(int level, int cf, const Slice& start, const Slice& limit) { + ASSERT_OK(dbfull()->TEST_CompactRange(level, &start, &limit, handles_[cf], + true /* disallow trivial move */)); + } + + // Do n memtable compactions, each of which produces an sstable + // covering the range [small,large]. + void MakeTables(int n, const std::string& small, const std::string& large, + int cf = 0) { + for (int i = 0; i < n; i++) { + ASSERT_OK(Put(cf, small, "begin")); + ASSERT_OK(Put(cf, large, "end")); + ASSERT_OK(Flush(cf)); + } + } + + static void SetDeletionCompactionStats(CompactionJobStats* stats, + uint64_t input_deletions, + uint64_t expired_deletions, + uint64_t records_replaced) { + stats->num_input_deletion_records = input_deletions; + stats->num_expired_deletion_records = expired_deletions; + stats->num_records_replaced = records_replaced; + } + + void MakeTableWithKeyValues(Random* rnd, uint64_t smallest, uint64_t largest, + int key_size, int value_size, uint64_t interval, + double ratio, int cf = 0) { + for (auto key = smallest; key < largest; key += interval) { + ASSERT_OK(Put(cf, Slice(Key(key, key_size)), + Slice(RandomString(rnd, value_size, ratio)))); + } + ASSERT_OK(Flush(cf)); + } + + // This function behaves with the implicit understanding that two + // rounds of keys are inserted into the database, as per the behavior + // of the DeletionStatsTest. + void SelectivelyDeleteKeys(uint64_t smallest, uint64_t largest, + uint64_t interval, int deletion_interval, + int key_size, uint64_t cutoff_key_num, + CompactionJobStats* stats, int cf = 0) { + // interval needs to be >= 2 so that deletion entries can be inserted + // that are intended to not result in an actual key deletion by using + // an offset of 1 from another existing key + ASSERT_GE(interval, 2); + + uint64_t ctr = 1; + uint32_t deletions_made = 0; + uint32_t num_deleted = 0; + uint32_t num_expired = 0; + for (auto key = smallest; key <= largest; key += interval, ctr++) { + if (ctr % deletion_interval == 0) { + ASSERT_OK(Delete(cf, Key(key, key_size))); + deletions_made++; + num_deleted++; + + if (key > cutoff_key_num) { + num_expired++; + } + } + } + + // Insert some deletions for keys that don't exist that + // are both in and out of the key range + ASSERT_OK(Delete(cf, Key(smallest + 1, key_size))); + deletions_made++; + + ASSERT_OK(Delete(cf, Key(smallest - 1, key_size))); + deletions_made++; + num_expired++; + + ASSERT_OK(Delete(cf, Key(smallest - 9, key_size))); + deletions_made++; + num_expired++; + + ASSERT_OK(Flush(cf)); + SetDeletionCompactionStats(stats, deletions_made, num_expired, num_deleted); + } +}; + +// An EventListener which helps verify the compaction results in +// test CompactionJobStatsTest. +class CompactionJobStatsChecker : public EventListener { + public: + CompactionJobStatsChecker() + : compression_enabled_(false), verify_next_comp_io_stats_(false) {} + + size_t NumberOfUnverifiedStats() { return expected_stats_.size(); } + + void set_verify_next_comp_io_stats(bool v) { verify_next_comp_io_stats_ = v; } + + // Once a compaction completed, this function will verify the returned + // CompactionJobInfo with the oldest CompactionJobInfo added earlier + // in "expected_stats_" which has not yet being used for verification. + void OnCompactionCompleted(DB* /*db*/, const CompactionJobInfo& ci) override { + if (verify_next_comp_io_stats_) { + ASSERT_GT(ci.stats.file_write_nanos, 0); + ASSERT_GT(ci.stats.file_range_sync_nanos, 0); + ASSERT_GT(ci.stats.file_fsync_nanos, 0); + ASSERT_GT(ci.stats.file_prepare_write_nanos, 0); + verify_next_comp_io_stats_ = false; + } + + std::lock_guard lock(mutex_); + if (expected_stats_.size()) { + Verify(ci.stats, expected_stats_.front()); + expected_stats_.pop(); + } + } + + // A helper function which verifies whether two CompactionJobStats + // match. The verification of all compaction stats are done by + // ASSERT_EQ except for the total input / output bytes, which we + // use ASSERT_GE and ASSERT_LE with a reasonable bias --- + // 10% in uncompressed case and 20% when compression is used. + virtual void Verify(const CompactionJobStats& current_stats, + const CompactionJobStats& stats) { + // time + ASSERT_GT(current_stats.elapsed_micros, 0U); + + ASSERT_EQ(current_stats.num_input_records, stats.num_input_records); + ASSERT_EQ(current_stats.num_input_files, stats.num_input_files); + ASSERT_EQ(current_stats.num_input_files_at_output_level, + stats.num_input_files_at_output_level); + + ASSERT_EQ(current_stats.num_output_records, stats.num_output_records); + ASSERT_EQ(current_stats.num_output_files, stats.num_output_files); + + ASSERT_EQ(current_stats.is_full_compaction, stats.is_full_compaction); + ASSERT_EQ(current_stats.is_manual_compaction, stats.is_manual_compaction); + + // file size + double kFileSizeBias = compression_enabled_ ? 0.20 : 0.10; + ASSERT_GE(current_stats.total_input_bytes * (1.00 + kFileSizeBias), + stats.total_input_bytes); + ASSERT_LE(current_stats.total_input_bytes, + stats.total_input_bytes * (1.00 + kFileSizeBias)); + ASSERT_GE(current_stats.total_output_bytes * (1.00 + kFileSizeBias), + stats.total_output_bytes); + ASSERT_LE(current_stats.total_output_bytes, + stats.total_output_bytes * (1.00 + kFileSizeBias)); + ASSERT_EQ(current_stats.total_input_raw_key_bytes, + stats.total_input_raw_key_bytes); + ASSERT_EQ(current_stats.total_input_raw_value_bytes, + stats.total_input_raw_value_bytes); + + ASSERT_EQ(current_stats.num_records_replaced, stats.num_records_replaced); + + ASSERT_EQ(current_stats.num_corrupt_keys, stats.num_corrupt_keys); + + ASSERT_EQ(std::string(current_stats.smallest_output_key_prefix), + std::string(stats.smallest_output_key_prefix)); + ASSERT_EQ(std::string(current_stats.largest_output_key_prefix), + std::string(stats.largest_output_key_prefix)); + } + + // Add an expected compaction stats, which will be used to + // verify the CompactionJobStats returned by the OnCompactionCompleted() + // callback. + void AddExpectedStats(const CompactionJobStats& stats) { + std::lock_guard lock(mutex_); + expected_stats_.push(stats); + } + + void EnableCompression(bool flag) { compression_enabled_ = flag; } + + bool verify_next_comp_io_stats() const { return verify_next_comp_io_stats_; } + + private: + std::mutex mutex_; + std::queue expected_stats_; + bool compression_enabled_; + bool verify_next_comp_io_stats_; +}; + +// An EventListener which helps verify the compaction statistics in +// the test DeletionStatsTest. +class CompactionJobDeletionStatsChecker : public CompactionJobStatsChecker { + public: + // Verifies whether two CompactionJobStats match. + void Verify(const CompactionJobStats& current_stats, + const CompactionJobStats& stats) override { + ASSERT_EQ(current_stats.num_input_deletion_records, + stats.num_input_deletion_records); + ASSERT_EQ(current_stats.num_expired_deletion_records, + stats.num_expired_deletion_records); + ASSERT_EQ(current_stats.num_records_replaced, stats.num_records_replaced); + + ASSERT_EQ(current_stats.num_corrupt_keys, stats.num_corrupt_keys); + } +}; + +namespace { + +uint64_t EstimatedFileSize(uint64_t num_records, size_t key_size, + size_t value_size, double compression_ratio = 1.0, + size_t block_size = 4096, + int bloom_bits_per_key = 10) { + const size_t kPerKeyOverhead = 8; + const size_t kFooterSize = 512; + + uint64_t data_size = static_cast( + num_records * + (key_size + value_size * compression_ratio + kPerKeyOverhead)); + + return data_size + kFooterSize + + num_records * bloom_bits_per_key / 8 // filter block + + data_size * (key_size + 8) / block_size; // index block +} + +namespace { + +void CopyPrefix(const Slice& src, size_t prefix_length, std::string* dst) { + assert(prefix_length > 0); + size_t length = src.size() > prefix_length ? prefix_length : src.size(); + dst->assign(src.data(), length); +} + +} // namespace + +CompactionJobStats NewManualCompactionJobStats( + const std::string& smallest_key, const std::string& largest_key, + size_t num_input_files, size_t num_input_files_at_output_level, + uint64_t num_input_records, size_t key_size, size_t value_size, + size_t num_output_files, uint64_t num_output_records, + double compression_ratio, uint64_t num_records_replaced, + bool is_full = false, bool is_manual = true) { + CompactionJobStats stats; + stats.Reset(); + + stats.num_input_records = num_input_records; + stats.num_input_files = num_input_files; + stats.num_input_files_at_output_level = num_input_files_at_output_level; + + stats.num_output_records = num_output_records; + stats.num_output_files = num_output_files; + + stats.total_input_bytes = + EstimatedFileSize(num_input_records / num_input_files, key_size, + value_size, compression_ratio) * + num_input_files; + stats.total_output_bytes = + EstimatedFileSize(num_output_records / num_output_files, key_size, + value_size, compression_ratio) * + num_output_files; + stats.total_input_raw_key_bytes = num_input_records * (key_size + 8); + stats.total_input_raw_value_bytes = num_input_records * value_size; + + stats.is_full_compaction = is_full; + stats.is_manual_compaction = is_manual; + + stats.num_records_replaced = num_records_replaced; + + CopyPrefix(smallest_key, CompactionJobStats::kMaxPrefixLength, + &stats.smallest_output_key_prefix); + CopyPrefix(largest_key, CompactionJobStats::kMaxPrefixLength, + &stats.largest_output_key_prefix); + + return stats; +} + +CompressionType GetAnyCompression() { + if (Snappy_Supported()) { + return kSnappyCompression; + } else if (Zlib_Supported()) { + return kZlibCompression; + } else if (BZip2_Supported()) { + return kBZip2Compression; + } else if (LZ4_Supported()) { + return kLZ4Compression; + } else if (XPRESS_Supported()) { + return kXpressCompression; + } + + return kNoCompression; +} + +} // namespace + +TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) { + Random rnd(301); + const int kBufSize = 100; + char buf[kBufSize]; + uint64_t key_base = 100000000l; + // Note: key_base must be multiple of num_keys_per_L0_file + int num_keys_per_L0_file = 100; + const int kTestScale = 8; + const int kKeySize = 10; + const int kValueSize = 1000; + const double kCompressionRatio = 0.5; + double compression_ratio = 1.0; + uint64_t key_interval = key_base / num_keys_per_L0_file; + + // Whenever a compaction completes, this listener will try to + // verify whether the returned CompactionJobStats matches + // what we expect. The expected CompactionJobStats is added + // via AddExpectedStats(). + auto* stats_checker = new CompactionJobStatsChecker(); + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.listeners.emplace_back(stats_checker); + options.create_if_missing = true; + // just enough setting to hold off auto-compaction. + options.level0_file_num_compaction_trigger = kTestScale + 1; + options.num_levels = 3; + options.compression = kNoCompression; + options.max_subcompactions = max_subcompactions_; + options.bytes_per_sync = 512 * 1024; + + options.report_bg_io_stats = true; + for (int test = 0; test < 2; ++test) { + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // 1st Phase: generate "num_L0_files" L0 files. + int num_L0_files = 0; + for (uint64_t start_key = key_base; start_key <= key_base * kTestScale; + start_key += key_base) { + MakeTableWithKeyValues(&rnd, start_key, start_key + key_base - 1, + kKeySize, kValueSize, key_interval, + compression_ratio, 1); + snprintf(buf, kBufSize, "%d", ++num_L0_files); + ASSERT_EQ(std::string(buf), FilesPerLevel(1)); + } + ASSERT_EQ(std::to_string(num_L0_files), FilesPerLevel(1)); + + // 2nd Phase: perform L0 -> L1 compaction. + int L0_compaction_count = 6; + int count = 1; + std::string smallest_key; + std::string largest_key; + for (uint64_t start_key = key_base; + start_key <= key_base * L0_compaction_count; + start_key += key_base, count++) { + smallest_key = Key(start_key, 10); + largest_key = Key(start_key + key_base - key_interval, 10); + stats_checker->AddExpectedStats(NewManualCompactionJobStats( + smallest_key, largest_key, 1, 0, num_keys_per_L0_file, kKeySize, + kValueSize, 1, num_keys_per_L0_file, compression_ratio, 0)); + ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 1U); + TEST_Compact(0, 1, smallest_key, largest_key); + snprintf(buf, kBufSize, "%d,%d", num_L0_files - count, count); + ASSERT_EQ(std::string(buf), FilesPerLevel(1)); + } + + // compact two files into one in the last L0 -> L1 compaction + int num_remaining_L0 = num_L0_files - L0_compaction_count; + smallest_key = Key(key_base * (L0_compaction_count + 1), 10); + largest_key = Key(key_base * (kTestScale + 1) - key_interval, 10); + stats_checker->AddExpectedStats(NewManualCompactionJobStats( + smallest_key, largest_key, num_remaining_L0, 0, + num_keys_per_L0_file * num_remaining_L0, kKeySize, kValueSize, 1, + num_keys_per_L0_file * num_remaining_L0, compression_ratio, 0)); + ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 1U); + TEST_Compact(0, 1, smallest_key, largest_key); + + int num_L1_files = num_L0_files - num_remaining_L0 + 1; + num_L0_files = 0; + snprintf(buf, kBufSize, "%d,%d", num_L0_files, num_L1_files); + ASSERT_EQ(std::string(buf), FilesPerLevel(1)); + + // 3rd Phase: generate sparse L0 files (wider key-range, same num of keys) + int sparseness = 2; + for (uint64_t start_key = key_base; start_key <= key_base * kTestScale; + start_key += key_base * sparseness) { + MakeTableWithKeyValues( + &rnd, start_key, start_key + key_base * sparseness - 1, kKeySize, + kValueSize, key_base * sparseness / num_keys_per_L0_file, + compression_ratio, 1); + snprintf(buf, kBufSize, "%d,%d", ++num_L0_files, num_L1_files); + ASSERT_EQ(std::string(buf), FilesPerLevel(1)); + } + + // 4th Phase: perform L0 -> L1 compaction again, expect higher write amp + // When subcompactions are enabled, the number of output files increases + // by 1 because multiple threads are consuming the input and generating + // output files without coordinating to see if the output could fit into + // a smaller number of files like it does when it runs sequentially + int num_output_files = options.max_subcompactions > 1 ? 2 : 1; + for (uint64_t start_key = key_base; num_L0_files > 1; + start_key += key_base * sparseness) { + smallest_key = Key(start_key, 10); + largest_key = Key(start_key + key_base * sparseness - key_interval, 10); + stats_checker->AddExpectedStats(NewManualCompactionJobStats( + smallest_key, largest_key, 3, 2, num_keys_per_L0_file * 3, kKeySize, + kValueSize, num_output_files, + num_keys_per_L0_file * 2, // 1/3 of the data will be updated. + compression_ratio, num_keys_per_L0_file)); + ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 1U); + Compact(1, smallest_key, largest_key); + if (options.max_subcompactions == 1) { + --num_L1_files; + } + snprintf(buf, kBufSize, "%d,%d", --num_L0_files, num_L1_files); + ASSERT_EQ(std::string(buf), FilesPerLevel(1)); + } + + // 5th Phase: Do a full compaction, which involves in two sub-compactions. + // Here we expect to have 1 L0 files and 4 L1 files + // In the first sub-compaction, we expect L0 compaction. + smallest_key = Key(key_base, 10); + largest_key = Key(key_base * (kTestScale + 1) - key_interval, 10); + stats_checker->AddExpectedStats(NewManualCompactionJobStats( + Key(key_base * (kTestScale + 1 - sparseness), 10), largest_key, 2, 1, + num_keys_per_L0_file * 3, kKeySize, kValueSize, 1, + num_keys_per_L0_file * 2, compression_ratio, num_keys_per_L0_file)); + ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 1U); + Compact(1, smallest_key, largest_key); + + num_L1_files = options.max_subcompactions > 1 ? 7 : 4; + char L1_buf[4]; + snprintf(L1_buf, sizeof(L1_buf), "0,%d", num_L1_files); + std::string L1_files(L1_buf); + ASSERT_EQ(L1_files, FilesPerLevel(1)); + options.compression = GetAnyCompression(); + if (options.compression == kNoCompression) { + break; + } + stats_checker->EnableCompression(true); + compression_ratio = kCompressionRatio; + + for (int i = 0; i < 5; i++) { + ASSERT_OK(Put(1, Slice(Key(key_base + i, 10)), + Slice(RandomString(&rnd, 512 * 1024, 1)))); + } + + ASSERT_OK(Flush(1)); + ASSERT_OK(static_cast_with_check(db_)->TEST_WaitForCompact()); + + stats_checker->set_verify_next_comp_io_stats(true); + std::atomic first_prepare_write(true); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WritableFileWriter::Append:BeforePrepareWrite", [&](void* /*arg*/) { + if (first_prepare_write.load()) { + options.env->SleepForMicroseconds(3); + first_prepare_write.store(false); + } + }); + + std::atomic first_flush(true); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WritableFileWriter::Flush:BeforeAppend", [&](void* /*arg*/) { + if (first_flush.load()) { + options.env->SleepForMicroseconds(3); + first_flush.store(false); + } + }); + + std::atomic first_sync(true); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WritableFileWriter::SyncInternal:0", [&](void* /*arg*/) { + if (first_sync.load()) { + options.env->SleepForMicroseconds(3); + first_sync.store(false); + } + }); + + std::atomic first_range_sync(true); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WritableFileWriter::RangeSync:0", [&](void* /*arg*/) { + if (first_range_sync.load()) { + options.env->SleepForMicroseconds(3); + first_range_sync.store(false); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Compact(1, smallest_key, largest_key); + + ASSERT_TRUE(!stats_checker->verify_next_comp_io_stats()); + ASSERT_TRUE(!first_prepare_write.load()); + ASSERT_TRUE(!first_flush.load()); + ASSERT_TRUE(!first_sync.load()); + ASSERT_TRUE(!first_range_sync.load()); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + } + ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 0U); +} + +TEST_P(CompactionJobStatsTest, DeletionStatsTest) { + Random rnd(301); + uint64_t key_base = 100000l; + // Note: key_base must be multiple of num_keys_per_L0_file + int num_keys_per_L0_file = 20; + const int kTestScale = 8; // make sure this is even + const int kKeySize = 10; + const int kValueSize = 100; + double compression_ratio = 1.0; + uint64_t key_interval = key_base / num_keys_per_L0_file; + uint64_t largest_key_num = key_base * (kTestScale + 1) - key_interval; + uint64_t cutoff_key_num = key_base * (kTestScale / 2 + 1) - key_interval; + const std::string smallest_key = Key(key_base - 10, kKeySize); + const std::string largest_key = Key(largest_key_num + 10, kKeySize); + + // Whenever a compaction completes, this listener will try to + // verify whether the returned CompactionJobStats matches + // what we expect. + auto* stats_checker = new CompactionJobDeletionStatsChecker(); + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.listeners.emplace_back(stats_checker); + options.create_if_missing = true; + options.level0_file_num_compaction_trigger = kTestScale + 1; + options.num_levels = 3; + options.compression = kNoCompression; + options.max_bytes_for_level_multiplier = 2; + options.max_subcompactions = max_subcompactions_; + + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // Stage 1: Generate several L0 files and then send them to L2 by + // using CompactRangeOptions and CompactRange(). These files will + // have a strict subset of the keys from the full key-range + for (uint64_t start_key = key_base; start_key <= key_base * kTestScale / 2; + start_key += key_base) { + MakeTableWithKeyValues(&rnd, start_key, start_key + key_base - 1, kKeySize, + kValueSize, key_interval, compression_ratio, 1); + } + + CompactRangeOptions cr_options; + cr_options.change_level = true; + cr_options.target_level = 2; + ASSERT_OK(db_->CompactRange(cr_options, handles_[1], nullptr, nullptr)); + ASSERT_GT(NumTableFilesAtLevel(2, 1), 0); + + // Stage 2: Generate files including keys from the entire key range + for (uint64_t start_key = key_base; start_key <= key_base * kTestScale; + start_key += key_base) { + MakeTableWithKeyValues(&rnd, start_key, start_key + key_base - 1, kKeySize, + kValueSize, key_interval, compression_ratio, 1); + } + + // Send these L0 files to L1 + TEST_Compact(0, 1, smallest_key, largest_key); + ASSERT_GT(NumTableFilesAtLevel(1, 1), 0); + + // Add a new record and flush so now there is a L0 file + // with a value too (not just deletions from the next step) + ASSERT_OK(Put(1, Key(key_base - 6, kKeySize), "test")); + ASSERT_OK(Flush(1)); + + // Stage 3: Generate L0 files with some deletions so now + // there are files with the same key range in L0, L1, and L2 + int deletion_interval = 3; + CompactionJobStats first_compaction_stats; + SelectivelyDeleteKeys(key_base, largest_key_num, key_interval, + deletion_interval, kKeySize, cutoff_key_num, + &first_compaction_stats, 1); + + stats_checker->AddExpectedStats(first_compaction_stats); + + // Stage 4: Trigger compaction and verify the stats + TEST_Compact(0, 1, smallest_key, largest_key); +} + +namespace { +int GetUniversalCompactionInputUnits(uint32_t num_flushes) { + uint32_t compaction_input_units; + for (compaction_input_units = 1; num_flushes >= compaction_input_units; + compaction_input_units *= 2) { + if ((num_flushes & compaction_input_units) != 0) { + return compaction_input_units > 1 ? compaction_input_units : 0; + } + } + return 0; +} +} // namespace + +TEST_P(CompactionJobStatsTest, UniversalCompactionTest) { + Random rnd(301); + uint64_t key_base = 100000000l; + // Note: key_base must be multiple of num_keys_per_L0_file + int num_keys_per_table = 100; + const uint32_t kTestScale = 6; + const int kKeySize = 10; + const int kValueSize = 900; + double compression_ratio = 1.0; + uint64_t key_interval = key_base / num_keys_per_table; + + auto* stats_checker = new CompactionJobStatsChecker(); + Options options; + options.listeners.emplace_back(stats_checker); + options.create_if_missing = true; + options.num_levels = 3; + options.compression = kNoCompression; + options.level0_file_num_compaction_trigger = 2; + options.target_file_size_base = num_keys_per_table * 1000; + options.compaction_style = kCompactionStyleUniversal; + options.compaction_options_universal.size_ratio = 1; + options.compaction_options_universal.max_size_amplification_percent = 1000; + options.max_subcompactions = max_subcompactions_; + + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // Generates the expected CompactionJobStats for each compaction + for (uint32_t num_flushes = 2; num_flushes <= kTestScale; num_flushes++) { + // Here we treat one newly flushed file as an unit. + // + // For example, if a newly flushed file is 100k, and a compaction has + // 4 input units, then this compaction inputs 400k. + uint32_t num_input_units = GetUniversalCompactionInputUnits(num_flushes); + if (num_input_units == 0) { + continue; + } + // A full compaction only happens when the number of flushes equals to + // the number of compaction input runs. + bool is_full = num_flushes == num_input_units; + // The following statement determines the expected smallest key + // based on whether it is a full compaction. + uint64_t smallest_key = is_full ? key_base : key_base * (num_flushes - 1); + + stats_checker->AddExpectedStats(NewManualCompactionJobStats( + Key(smallest_key, 10), + Key(smallest_key + key_base * num_input_units - key_interval, 10), + num_input_units, num_input_units > 2 ? num_input_units / 2 : 0, + num_keys_per_table * num_input_units, kKeySize, kValueSize, + num_input_units, num_keys_per_table * num_input_units, 1.0, 0, is_full, + false)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 3U); + + for (uint64_t start_key = key_base; start_key <= key_base * kTestScale; + start_key += key_base) { + MakeTableWithKeyValues(&rnd, start_key, start_key + key_base - 1, kKeySize, + kValueSize, key_interval, compression_ratio, 1); + ASSERT_OK(static_cast_with_check(db_)->TEST_WaitForCompact()); + } + ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 0U); +} + +INSTANTIATE_TEST_CASE_P(CompactionJobStatsTest, CompactionJobStatsTest, + ::testing::Values(1, 4)); +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + + +#else + +int main(int /*argc*/, char** /*argv*/) { return 0; } +#endif // !defined(IOS_CROSS_COMPILE) diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_job_test.cc b/librocksdb-sys/rocksdb/db/compaction/compaction_job_test.cc new file mode 100644 index 0000000..8f91cc0 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_job_test.cc @@ -0,0 +1,2446 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + + +#include "db/compaction/compaction_job.h" + +#include +#include +#include +#include +#include +#include + +#include "db/blob/blob_index.h" +#include "db/column_family.h" +#include "db/db_impl/db_impl.h" +#include "db/error_handler.h" +#include "db/version_set.h" +#include "file/random_access_file_reader.h" +#include "file/writable_file_writer.h" +#include "options/options_helper.h" +#include "rocksdb/cache.h" +#include "rocksdb/convenience.h" +#include "rocksdb/db.h" +#include "rocksdb/file_system.h" +#include "rocksdb/options.h" +#include "rocksdb/write_buffer_manager.h" +#include "table/mock_table.h" +#include "table/unique_id_impl.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/string_util.h" +#include "utilities/merge_operators.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { + +void VerifyInitializationOfCompactionJobStats( + const CompactionJobStats& compaction_job_stats) { +#if !defined(IOS_CROSS_COMPILE) + ASSERT_EQ(compaction_job_stats.elapsed_micros, 0U); + + ASSERT_EQ(compaction_job_stats.num_input_records, 0U); + ASSERT_EQ(compaction_job_stats.num_input_files, 0U); + ASSERT_EQ(compaction_job_stats.num_input_files_at_output_level, 0U); + + ASSERT_EQ(compaction_job_stats.num_output_records, 0U); + ASSERT_EQ(compaction_job_stats.num_output_files, 0U); + + ASSERT_EQ(compaction_job_stats.is_manual_compaction, true); + + ASSERT_EQ(compaction_job_stats.total_input_bytes, 0U); + ASSERT_EQ(compaction_job_stats.total_output_bytes, 0U); + + ASSERT_EQ(compaction_job_stats.total_input_raw_key_bytes, 0U); + ASSERT_EQ(compaction_job_stats.total_input_raw_value_bytes, 0U); + + ASSERT_EQ(compaction_job_stats.smallest_output_key_prefix[0], 0); + ASSERT_EQ(compaction_job_stats.largest_output_key_prefix[0], 0); + + ASSERT_EQ(compaction_job_stats.num_records_replaced, 0U); + + ASSERT_EQ(compaction_job_stats.num_input_deletion_records, 0U); + ASSERT_EQ(compaction_job_stats.num_expired_deletion_records, 0U); + + ASSERT_EQ(compaction_job_stats.num_corrupt_keys, 0U); +#endif // !defined(IOS_CROSS_COMPILE) +} + +// Mock FSWritableFile for testing io priority. +// Only override the essential functions for testing compaction io priority. +class MockTestWritableFile : public FSWritableFileOwnerWrapper { + public: + MockTestWritableFile(std::unique_ptr&& file, + Env::IOPriority io_priority) + : FSWritableFileOwnerWrapper(std::move(file)), + write_io_priority_(io_priority) {} + IOStatus Append(const Slice& data, const IOOptions& options, + IODebugContext* dbg) override { + EXPECT_EQ(options.rate_limiter_priority, write_io_priority_); + return target()->Append(data, options, dbg); + } + IOStatus Append(const Slice& data, const IOOptions& options, + const DataVerificationInfo& verification_info, + IODebugContext* dbg) override { + EXPECT_EQ(options.rate_limiter_priority, write_io_priority_); + return target()->Append(data, options, verification_info, dbg); + } + IOStatus Close(const IOOptions& options, IODebugContext* dbg) override { + EXPECT_EQ(options.rate_limiter_priority, write_io_priority_); + return target()->Close(options, dbg); + } + IOStatus Flush(const IOOptions& options, IODebugContext* dbg) override { + EXPECT_EQ(options.rate_limiter_priority, write_io_priority_); + return target()->Flush(options, dbg); + } + IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override { + EXPECT_EQ(options.rate_limiter_priority, write_io_priority_); + return target()->Sync(options, dbg); + } + IOStatus Fsync(const IOOptions& options, IODebugContext* dbg) override { + EXPECT_EQ(options.rate_limiter_priority, write_io_priority_); + return target()->Fsync(options, dbg); + } + uint64_t GetFileSize(const IOOptions& options, IODebugContext* dbg) override { + EXPECT_EQ(options.rate_limiter_priority, write_io_priority_); + return target()->GetFileSize(options, dbg); + } + IOStatus RangeSync(uint64_t offset, uint64_t nbytes, const IOOptions& options, + IODebugContext* dbg) override { + EXPECT_EQ(options.rate_limiter_priority, write_io_priority_); + return target()->RangeSync(offset, nbytes, options, dbg); + } + + void PrepareWrite(size_t offset, size_t len, const IOOptions& options, + IODebugContext* dbg) override { + EXPECT_EQ(options.rate_limiter_priority, write_io_priority_); + target()->PrepareWrite(offset, len, options, dbg); + } + + IOStatus Allocate(uint64_t offset, uint64_t len, const IOOptions& options, + IODebugContext* dbg) override { + EXPECT_EQ(options.rate_limiter_priority, write_io_priority_); + return target()->Allocate(offset, len, options, dbg); + } + + private: + Env::IOPriority write_io_priority_; +}; + +// Mock FSRandomAccessFile for testing io priority. +// Only override the essential functions for testing compaction io priority. +class MockTestRandomAccessFile : public FSRandomAccessFileOwnerWrapper { + public: + MockTestRandomAccessFile(std::unique_ptr&& file, + Env::IOPriority io_priority) + : FSRandomAccessFileOwnerWrapper(std::move(file)), + read_io_priority_(io_priority) {} + + IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const override { + EXPECT_EQ(options.rate_limiter_priority, read_io_priority_); + return target()->Read(offset, n, options, result, scratch, dbg); + } + IOStatus Prefetch(uint64_t offset, size_t n, const IOOptions& options, + IODebugContext* dbg) override { + EXPECT_EQ(options.rate_limiter_priority, read_io_priority_); + return target()->Prefetch(offset, n, options, dbg); + } + + private: + Env::IOPriority read_io_priority_; +}; + +// Mock FileSystem for testing io priority. +class MockTestFileSystem : public FileSystemWrapper { + public: + explicit MockTestFileSystem(const std::shared_ptr& base, + Env::IOPriority read_io_priority, + Env::IOPriority write_io_priority) + : FileSystemWrapper(base), + read_io_priority_(read_io_priority), + write_io_priority_(write_io_priority) {} + + static const char* kClassName() { return "MockTestFileSystem"; } + const char* Name() const override { return kClassName(); } + + IOStatus NewRandomAccessFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override { + IOStatus s = target()->NewRandomAccessFile(fname, file_opts, result, dbg); + EXPECT_OK(s); + result->reset( + new MockTestRandomAccessFile(std::move(*result), read_io_priority_)); + return s; + } + IOStatus NewWritableFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override { + IOStatus s = target()->NewWritableFile(fname, file_opts, result, dbg); + EXPECT_OK(s); + result->reset( + new MockTestWritableFile(std::move(*result), write_io_priority_)); + return s; + } + + private: + Env::IOPriority read_io_priority_; + Env::IOPriority write_io_priority_; +}; + +enum TableTypeForTest : uint8_t { kMockTable = 0, kBlockBasedTable = 1 }; + +} // namespace + +class CompactionJobTestBase : public testing::Test { + protected: + CompactionJobTestBase(std::string dbname, const Comparator* ucmp, + std::function encode_u64_ts, + bool test_io_priority, TableTypeForTest table_type) + : dbname_(std::move(dbname)), + ucmp_(ucmp), + db_options_(), + mutable_cf_options_(cf_options_), + mutable_db_options_(), + table_cache_(NewLRUCache(50000, 16)), + write_buffer_manager_(db_options_.db_write_buffer_size), + versions_(new VersionSet( + dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, + /*io_tracer=*/nullptr, /*db_id*/ "", /*db_session_id*/ "")), + shutting_down_(false), + mock_table_factory_(new mock::MockTableFactory()), + error_handler_(nullptr, db_options_, &mutex_), + encode_u64_ts_(std::move(encode_u64_ts)), + test_io_priority_(test_io_priority), + table_type_(table_type) { + Env* base_env = Env::Default(); + EXPECT_OK( + test::CreateEnvFromSystem(ConfigOptions(), &base_env, &env_guard_)); + env_ = base_env; + fs_ = env_->GetFileSystem(); + // set default for the tests + mutable_cf_options_.target_file_size_base = 1024 * 1024; + mutable_cf_options_.max_compaction_bytes = 10 * 1024 * 1024; + } + + void SetUp() override { + EXPECT_OK(env_->CreateDirIfMissing(dbname_)); + db_options_.env = env_; + db_options_.fs = fs_; + db_options_.db_paths.emplace_back(dbname_, + std::numeric_limits::max()); + cf_options_.comparator = ucmp_; + if (table_type_ == TableTypeForTest::kBlockBasedTable) { + BlockBasedTableOptions table_options; + cf_options_.table_factory.reset(NewBlockBasedTableFactory(table_options)); + } else if (table_type_ == TableTypeForTest::kMockTable) { + cf_options_.table_factory = mock_table_factory_; + } else { + assert(false); + } + } + + std::string GenerateFileName(uint64_t file_number) { + FileMetaData meta; + std::vector db_paths; + db_paths.emplace_back(dbname_, std::numeric_limits::max()); + meta.fd = FileDescriptor(file_number, 0, 0); + return TableFileName(db_paths, meta.fd.GetNumber(), meta.fd.GetPathId()); + } + + std::string KeyStr(const std::string& user_key, const SequenceNumber seq_num, + const ValueType t, uint64_t ts = 0) { + std::string user_key_with_ts = user_key + encode_u64_ts_(ts); + return InternalKey(user_key_with_ts, seq_num, t).Encode().ToString(); + } + + static std::string BlobStr(uint64_t blob_file_number, uint64_t offset, + uint64_t size) { + std::string blob_index; + BlobIndex::EncodeBlob(&blob_index, blob_file_number, offset, size, + kNoCompression); + return blob_index; + } + + static std::string BlobStrTTL(uint64_t blob_file_number, uint64_t offset, + uint64_t size, uint64_t expiration) { + std::string blob_index; + BlobIndex::EncodeBlobTTL(&blob_index, expiration, blob_file_number, offset, + size, kNoCompression); + return blob_index; + } + + static std::string BlobStrInlinedTTL(const Slice& value, + uint64_t expiration) { + std::string blob_index; + BlobIndex::EncodeInlinedTTL(&blob_index, expiration, value); + return blob_index; + } + + // Creates a table with the specificied key value pairs. + void CreateTable(const std::string& table_name, + const mock::KVVector& contents, uint64_t& file_size) { + std::unique_ptr file_writer; + Status s = WritableFileWriter::Create(fs_, table_name, FileOptions(), + &file_writer, nullptr); + ASSERT_OK(s); + std::unique_ptr table_builder( + cf_options_.table_factory->NewTableBuilder( + TableBuilderOptions(*cfd_->ioptions(), mutable_cf_options_, + cfd_->internal_comparator(), + cfd_->int_tbl_prop_collector_factories(), + CompressionType::kNoCompression, + CompressionOptions(), 0 /* column_family_id */, + kDefaultColumnFamilyName, -1 /* level */), + file_writer.get())); + // Build table. + for (auto kv : contents) { + std::string key; + std::string value; + std::tie(key, value) = kv; + table_builder->Add(key, value); + } + ASSERT_OK(table_builder->Finish()); + file_size = table_builder->FileSize(); + } + + void AddMockFile(const mock::KVVector& contents, int level = 0) { + assert(contents.size() > 0); + + bool first_key = true; + std::string smallest, largest; + InternalKey smallest_key, largest_key; + SequenceNumber smallest_seqno = kMaxSequenceNumber; + SequenceNumber largest_seqno = 0; + uint64_t oldest_blob_file_number = kInvalidBlobFileNumber; + for (auto kv : contents) { + ParsedInternalKey key; + std::string skey; + std::string value; + std::tie(skey, value) = kv; + const Status pik_status = + ParseInternalKey(skey, &key, true /* log_err_key */); + + smallest_seqno = std::min(smallest_seqno, key.sequence); + largest_seqno = std::max(largest_seqno, key.sequence); + + if (first_key || + cfd_->user_comparator()->Compare(key.user_key, smallest) < 0) { + smallest.assign(key.user_key.data(), key.user_key.size()); + smallest_key.DecodeFrom(skey); + } + if (first_key || + cfd_->user_comparator()->Compare(key.user_key, largest) > 0) { + largest.assign(key.user_key.data(), key.user_key.size()); + largest_key.DecodeFrom(skey); + } + + first_key = false; + + if (pik_status.ok() && key.type == kTypeBlobIndex) { + BlobIndex blob_index; + const Status s = blob_index.DecodeFrom(value); + if (!s.ok()) { + continue; + } + + if (blob_index.IsInlined() || blob_index.HasTTL() || + blob_index.file_number() == kInvalidBlobFileNumber) { + continue; + } + + if (oldest_blob_file_number == kInvalidBlobFileNumber || + oldest_blob_file_number > blob_index.file_number()) { + oldest_blob_file_number = blob_index.file_number(); + } + } + } + + uint64_t file_number = versions_->NewFileNumber(); + + uint64_t file_size = 0; + if (table_type_ == TableTypeForTest::kBlockBasedTable) { + CreateTable(GenerateFileName(file_number), contents, file_size); + } else if (table_type_ == TableTypeForTest::kMockTable) { + file_size = 10; + EXPECT_OK(mock_table_factory_->CreateMockTable( + env_, GenerateFileName(file_number), contents)); + } else { + assert(false); + } + + VersionEdit edit; + edit.AddFile( + level, file_number, 0, file_size, smallest_key, largest_key, + smallest_seqno, largest_seqno, false, Temperature::kUnknown, + oldest_blob_file_number, kUnknownOldestAncesterTime, + kUnknownFileCreationTime, + versions_->GetColumnFamilySet()->GetDefault()->NewEpochNumber(), + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, + /*compensated_range_deletion_size=*/0, /*tail_size=*/0, + /*user_defined_timestamps_persisted=*/true); + + mutex_.Lock(); + EXPECT_OK(versions_->LogAndApply( + versions_->GetColumnFamilySet()->GetDefault(), mutable_cf_options_, + read_options_, &edit, &mutex_, nullptr)); + mutex_.Unlock(); + } + + void VerifyTables(int output_level, + const std::vector& expected_results, + std::vector expected_oldest_blob_file_numbers) { + if (expected_results.empty()) { + ASSERT_EQ(compaction_job_stats_.num_output_files, 0U); + return; + } + int expected_output_file_num = 0; + for (const auto& e : expected_results) { + if (!e.empty()) { + ++expected_output_file_num; + } + } + ASSERT_EQ(expected_output_file_num, compaction_job_stats_.num_output_files); + if (expected_output_file_num == 0) { + return; + } + + if (expected_oldest_blob_file_numbers.empty()) { + expected_oldest_blob_file_numbers.resize(expected_output_file_num, + kInvalidBlobFileNumber); + } + + auto cfd = versions_->GetColumnFamilySet()->GetDefault(); + if (table_type_ == TableTypeForTest::kMockTable) { + ASSERT_EQ(compaction_job_stats_.num_output_files, + expected_results.size()); + mock_table_factory_->AssertLatestFiles(expected_results); + } else { + assert(table_type_ == TableTypeForTest::kBlockBasedTable); + } + + auto output_files = + cfd->current()->storage_info()->LevelFiles(output_level); + ASSERT_EQ(expected_output_file_num, output_files.size()); + + if (table_type_ == TableTypeForTest::kMockTable) { + assert(output_files.size() == + static_cast(expected_output_file_num)); + const FileMetaData* const output_file = output_files[0]; + ASSERT_EQ(output_file->oldest_blob_file_number, + expected_oldest_blob_file_numbers[0]); + return; + } + + for (size_t i = 0; i < expected_results.size(); ++i) { + const FileMetaData* const output_file = output_files[i]; + std::string file_name = GenerateFileName(output_file->fd.GetNumber()); + const auto& fs = env_->GetFileSystem(); + std::unique_ptr freader; + IOStatus ios = RandomAccessFileReader::Create( + fs, file_name, FileOptions(), &freader, nullptr); + ASSERT_OK(ios); + std::unique_ptr table_reader; + uint64_t file_size = output_file->fd.GetFileSize(); + ReadOptions read_opts; + Status s = cf_options_.table_factory->NewTableReader( + read_opts, + TableReaderOptions(*cfd->ioptions(), nullptr, FileOptions(), + cfd_->internal_comparator(), + 0 /* block_protection_bytes_per_key */), + std::move(freader), file_size, &table_reader, false); + ASSERT_OK(s); + assert(table_reader); + std::unique_ptr iiter( + table_reader->NewIterator(read_opts, nullptr, nullptr, true, + TableReaderCaller::kUncategorized)); + assert(iiter); + + mock::KVVector from_db; + for (iiter->SeekToFirst(); iiter->Valid(); iiter->Next()) { + const Slice key = iiter->key(); + const Slice value = iiter->value(); + from_db.emplace_back( + make_pair(key.ToString(false), value.ToString(false))); + } + ASSERT_EQ(expected_results[i], from_db); + } + } + + void SetLastSequence(const SequenceNumber sequence_number) { + versions_->SetLastAllocatedSequence(sequence_number + 1); + versions_->SetLastPublishedSequence(sequence_number + 1); + versions_->SetLastSequence(sequence_number + 1); + } + + // returns expected result after compaction + mock::KVVector CreateTwoFiles(bool gen_corrupted_keys) { + stl_wrappers::KVMap expected_results; + constexpr int kKeysPerFile = 10000; + constexpr int kCorruptKeysPerFile = 200; + constexpr int kMatchingKeys = kKeysPerFile / 2; + SequenceNumber sequence_number = 0; + + auto corrupt_id = [&](int id) { + return gen_corrupted_keys && id > 0 && id <= kCorruptKeysPerFile; + }; + + for (int i = 0; i < 2; ++i) { + auto contents = mock::MakeMockFile(); + for (int k = 0; k < kKeysPerFile; ++k) { + auto key = std::to_string(i * kMatchingKeys + k); + auto value = std::to_string(i * kKeysPerFile + k); + InternalKey internal_key(key, ++sequence_number, kTypeValue); + + // This is how the key will look like once it's written in bottommost + // file + InternalKey bottommost_internal_key(key, 0, kTypeValue); + + if (corrupt_id(k)) { + test::CorruptKeyType(&internal_key); + test::CorruptKeyType(&bottommost_internal_key); + } + contents.push_back({internal_key.Encode().ToString(), value}); + if (i == 1 || k < kMatchingKeys || corrupt_id(k - kMatchingKeys)) { + expected_results.insert( + {bottommost_internal_key.Encode().ToString(), value}); + } + } + mock::SortKVVector(&contents, ucmp_); + + AddMockFile(contents); + } + + SetLastSequence(sequence_number); + + mock::KVVector expected_results_kvvector; + for (auto& kv : expected_results) { + expected_results_kvvector.push_back({kv.first, kv.second}); + } + + return expected_results_kvvector; + } + + void NewDB() { + EXPECT_OK(DestroyDB(dbname_, Options())); + EXPECT_OK(env_->CreateDirIfMissing(dbname_)); + + std::shared_ptr info_log; + DBOptions db_opts = BuildDBOptions(db_options_, mutable_db_options_); + Status s = CreateLoggerFromOptions(dbname_, db_opts, &info_log); + ASSERT_OK(s); + db_options_.info_log = info_log; + + versions_.reset( + new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id*/ "", /*db_session_id*/ "")); + compaction_job_stats_.Reset(); + ASSERT_OK(SetIdentityFile(env_, dbname_)); + + VersionEdit new_db; + new_db.SetLogNumber(0); + new_db.SetNextFile(2); + new_db.SetLastSequence(0); + + const std::string manifest = DescriptorFileName(dbname_, 1); + std::unique_ptr file_writer; + const auto& fs = env_->GetFileSystem(); + s = WritableFileWriter::Create(fs, manifest, + fs->OptimizeForManifestWrite(env_options_), + &file_writer, nullptr); + + ASSERT_OK(s); + { + log::Writer log(std::move(file_writer), 0, false); + std::string record; + new_db.EncodeTo(&record); + s = log.AddRecord(record); + } + ASSERT_OK(s); + // Make "CURRENT" file that points to the new manifest file. + s = SetCurrentFile(fs_.get(), dbname_, 1, nullptr); + + ASSERT_OK(s); + + cf_options_.merge_operator = merge_op_; + cf_options_.compaction_filter = compaction_filter_.get(); + std::vector column_families; + column_families.emplace_back(kDefaultColumnFamilyName, cf_options_); + + ASSERT_OK(versions_->Recover(column_families, false)); + cfd_ = versions_->GetColumnFamilySet()->GetDefault(); + } + + // input_files[i] on input_levels[i] + void RunLastLevelCompaction( + const std::vector>& input_files, + const std::vector input_levels, + std::function&& verify_func, + const std::vector& snapshots = {}) { + const int kLastLevel = cf_options_.num_levels - 1; + verify_per_key_placement_ = std::move(verify_func); + mock::KVVector empty_map; + RunCompaction(input_files, input_levels, {empty_map}, snapshots, + kMaxSequenceNumber, kLastLevel, false); + } + + // input_files[i] on input_levels[i] + void RunCompaction( + const std::vector>& input_files, + const std::vector& input_levels, + const std::vector& expected_results, + const std::vector& snapshots = {}, + SequenceNumber earliest_write_conflict_snapshot = kMaxSequenceNumber, + int output_level = 1, bool verify = true, + std::vector expected_oldest_blob_file_numbers = {}, + bool check_get_priority = false, + Env::IOPriority read_io_priority = Env::IO_TOTAL, + Env::IOPriority write_io_priority = Env::IO_TOTAL, + int max_subcompactions = 0) { + // For compaction, set fs as MockTestFileSystem to check the io_priority. + if (test_io_priority_) { + db_options_.fs.reset( + new MockTestFileSystem(fs_, read_io_priority, write_io_priority)); + } + + auto cfd = versions_->GetColumnFamilySet()->GetDefault(); + + size_t num_input_files = 0; + std::vector compaction_input_files; + for (size_t i = 0; i < input_files.size(); ++i) { + auto level_files = input_files[i]; + CompactionInputFiles compaction_level; + compaction_level.level = input_levels[i]; + compaction_level.files.insert(compaction_level.files.end(), + level_files.begin(), level_files.end()); + compaction_input_files.push_back(compaction_level); + num_input_files += level_files.size(); + } + + std::vector grandparents; + // it should actually be the next non-empty level + const int kGrandparentsLevel = output_level + 1; + if (kGrandparentsLevel < cf_options_.num_levels) { + grandparents = + cfd_->current()->storage_info()->LevelFiles(kGrandparentsLevel); + } + + Compaction compaction( + cfd->current()->storage_info(), *cfd->ioptions(), + *cfd->GetLatestMutableCFOptions(), mutable_db_options_, + compaction_input_files, output_level, + mutable_cf_options_.target_file_size_base, + mutable_cf_options_.max_compaction_bytes, 0, kNoCompression, + cfd->GetLatestMutableCFOptions()->compression_opts, + Temperature::kUnknown, max_subcompactions, grandparents, true); + compaction.SetInputVersion(cfd->current()); + + assert(db_options_.info_log); + LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get()); + mutex_.Lock(); + EventLogger event_logger(db_options_.info_log.get()); + // TODO(yiwu) add a mock snapshot checker and add test for it. + SnapshotChecker* snapshot_checker = nullptr; + ASSERT_TRUE(full_history_ts_low_.empty() || + ucmp_->timestamp_size() == full_history_ts_low_.size()); + const std::atomic kManualCompactionCanceledFalse{false}; + JobContext job_context(1, false /* create_superversion */); + CompactionJob compaction_job( + 0, &compaction, db_options_, mutable_db_options_, env_options_, + versions_.get(), &shutting_down_, &log_buffer, nullptr, nullptr, + nullptr, nullptr, &mutex_, &error_handler_, snapshots, + earliest_write_conflict_snapshot, snapshot_checker, &job_context, + table_cache_, &event_logger, false, false, dbname_, + &compaction_job_stats_, Env::Priority::USER, nullptr /* IOTracer */, + /*manual_compaction_canceled=*/kManualCompactionCanceledFalse, + env_->GenerateUniqueId(), DBImpl::GenerateDbSessionId(nullptr), + full_history_ts_low_); + VerifyInitializationOfCompactionJobStats(compaction_job_stats_); + + compaction_job.Prepare(); + mutex_.Unlock(); + Status s = compaction_job.Run(); + ASSERT_OK(s); + ASSERT_OK(compaction_job.io_status()); + mutex_.Lock(); + ASSERT_OK(compaction_job.Install(*cfd->GetLatestMutableCFOptions())); + ASSERT_OK(compaction_job.io_status()); + mutex_.Unlock(); + log_buffer.FlushBufferToLog(); + + if (verify) { + ASSERT_GE(compaction_job_stats_.elapsed_micros, 0U); + ASSERT_EQ(compaction_job_stats_.num_input_files, num_input_files); + + VerifyTables(output_level, expected_results, + expected_oldest_blob_file_numbers); + } + + if (check_get_priority) { + CheckGetRateLimiterPriority(compaction_job); + } + + if (verify_per_key_placement_) { + // Verify per_key_placement compaction + assert(compaction.SupportsPerKeyPlacement()); + verify_per_key_placement_(compaction); + } + } + + void CheckGetRateLimiterPriority(CompactionJob& compaction_job) { + // When the state from WriteController is normal. + ASSERT_EQ(compaction_job.GetRateLimiterPriority(), Env::IO_LOW); + + WriteController* write_controller = + compaction_job.versions_->GetColumnFamilySet()->write_controller(); + + { + // When the state from WriteController is Delayed. + std::unique_ptr delay_token = + write_controller->GetDelayToken(1000000); + ASSERT_EQ(compaction_job.GetRateLimiterPriority(), Env::IO_USER); + } + + { + // When the state from WriteController is Stopped. + std::unique_ptr stop_token = + write_controller->GetStopToken(); + ASSERT_EQ(compaction_job.GetRateLimiterPriority(), Env::IO_USER); + } + } + + std::shared_ptr env_guard_; + Env* env_; + std::shared_ptr fs_; + std::string dbname_; + const Comparator* const ucmp_; + EnvOptions env_options_; + ImmutableDBOptions db_options_; + ColumnFamilyOptions cf_options_; + MutableCFOptions mutable_cf_options_; + MutableDBOptions mutable_db_options_; + const ReadOptions read_options_; + std::shared_ptr table_cache_; + WriteController write_controller_; + WriteBufferManager write_buffer_manager_; + std::unique_ptr versions_; + InstrumentedMutex mutex_; + std::atomic shutting_down_; + std::shared_ptr mock_table_factory_; + CompactionJobStats compaction_job_stats_; + ColumnFamilyData* cfd_; + std::unique_ptr compaction_filter_; + std::shared_ptr merge_op_; + ErrorHandler error_handler_; + std::string full_history_ts_low_; + const std::function encode_u64_ts_; + const bool test_io_priority_; + std::function verify_per_key_placement_; + const TableTypeForTest table_type_ = kMockTable; +}; + +// TODO(icanadi) Make it simpler once we mock out VersionSet +class CompactionJobTest : public CompactionJobTestBase { + public: + CompactionJobTest() + : CompactionJobTestBase( + test::PerThreadDBPath("compaction_job_test"), BytewiseComparator(), + [](uint64_t /*ts*/) { return ""; }, /*test_io_priority=*/false, + TableTypeForTest::kMockTable) {} +}; + +TEST_F(CompactionJobTest, Simple) { + NewDB(); + + auto expected_results = CreateTwoFiles(false); + auto cfd = versions_->GetColumnFamilySet()->GetDefault(); + constexpr int input_level = 0; + auto files = cfd->current()->storage_info()->LevelFiles(input_level); + ASSERT_EQ(2U, files.size()); + RunCompaction({files}, {input_level}, {expected_results}); +} + +TEST_F(CompactionJobTest, DISABLED_SimpleCorrupted) { + NewDB(); + + auto expected_results = CreateTwoFiles(true); + auto cfd = versions_->GetColumnFamilySet()->GetDefault(); + constexpr int input_level = 0; + auto files = cfd->current()->storage_info()->LevelFiles(input_level); + RunCompaction({files}, {input_level}, {expected_results}); + ASSERT_EQ(compaction_job_stats_.num_corrupt_keys, 400U); +} + +TEST_F(CompactionJobTest, SimpleDeletion) { + NewDB(); + + auto file1 = mock::MakeMockFile({{KeyStr("c", 4U, kTypeDeletion), ""}, + {KeyStr("c", 3U, kTypeValue), "val"}}); + AddMockFile(file1); + + auto file2 = mock::MakeMockFile({{KeyStr("b", 2U, kTypeValue), "val"}, + {KeyStr("b", 1U, kTypeValue), "val"}}); + AddMockFile(file2); + + auto expected_results = + mock::MakeMockFile({{KeyStr("b", 0U, kTypeValue), "val"}}); + + SetLastSequence(4U); + constexpr int input_level = 0; + auto files = cfd_->current()->storage_info()->LevelFiles(input_level); + RunCompaction({files}, {input_level}, {expected_results}); +} + +TEST_F(CompactionJobTest, OutputNothing) { + NewDB(); + + auto file1 = mock::MakeMockFile({{KeyStr("a", 1U, kTypeValue), "val"}}); + + AddMockFile(file1); + + auto file2 = mock::MakeMockFile({{KeyStr("a", 2U, kTypeDeletion), ""}}); + + AddMockFile(file2); + + auto expected_results = mock::MakeMockFile(); + + SetLastSequence(4U); + + constexpr int input_level = 0; + auto files = cfd_->current()->storage_info()->LevelFiles(input_level); + RunCompaction({files}, {input_level}, {expected_results}); +} + +TEST_F(CompactionJobTest, SimpleOverwrite) { + NewDB(); + + auto file1 = mock::MakeMockFile({ + {KeyStr("a", 3U, kTypeValue), "val2"}, + {KeyStr("b", 4U, kTypeValue), "val3"}, + }); + AddMockFile(file1); + + auto file2 = mock::MakeMockFile({{KeyStr("a", 1U, kTypeValue), "val"}, + {KeyStr("b", 2U, kTypeValue), "val"}}); + AddMockFile(file2); + + auto expected_results = + mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), "val2"}, + {KeyStr("b", 0U, kTypeValue), "val3"}}); + + SetLastSequence(4U); + constexpr int input_level = 0; + auto files = cfd_->current()->storage_info()->LevelFiles(input_level); + RunCompaction({files}, {input_level}, {expected_results}); +} + +TEST_F(CompactionJobTest, SimpleNonLastLevel) { + NewDB(); + + auto file1 = mock::MakeMockFile({ + {KeyStr("a", 5U, kTypeValue), "val2"}, + {KeyStr("b", 6U, kTypeValue), "val3"}, + }); + AddMockFile(file1); + + auto file2 = mock::MakeMockFile({{KeyStr("a", 3U, kTypeValue), "val"}, + {KeyStr("b", 4U, kTypeValue), "val"}}); + AddMockFile(file2, 1); + + auto file3 = mock::MakeMockFile({{KeyStr("a", 1U, kTypeValue), "val"}, + {KeyStr("b", 2U, kTypeValue), "val"}}); + AddMockFile(file3, 2); + + // Because level 1 is not the last level, the sequence numbers of a and b + // cannot be set to 0 + auto expected_results = + mock::MakeMockFile({{KeyStr("a", 5U, kTypeValue), "val2"}, + {KeyStr("b", 6U, kTypeValue), "val3"}}); + + SetLastSequence(6U); + const std::vector input_levels = {0, 1}; + auto lvl0_files = + cfd_->current()->storage_info()->LevelFiles(input_levels[0]); + auto lvl1_files = + cfd_->current()->storage_info()->LevelFiles(input_levels[1]); + RunCompaction({lvl0_files, lvl1_files}, input_levels, {expected_results}); +} + +TEST_F(CompactionJobTest, SimpleMerge) { + merge_op_ = MergeOperators::CreateStringAppendOperator(); + NewDB(); + + auto file1 = mock::MakeMockFile({ + {KeyStr("a", 5U, kTypeMerge), "5"}, + {KeyStr("a", 4U, kTypeMerge), "4"}, + {KeyStr("a", 3U, kTypeValue), "3"}, + }); + AddMockFile(file1); + + auto file2 = mock::MakeMockFile( + {{KeyStr("b", 2U, kTypeMerge), "2"}, {KeyStr("b", 1U, kTypeValue), "1"}}); + AddMockFile(file2); + + auto expected_results = + mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), "3,4,5"}, + {KeyStr("b", 0U, kTypeValue), "1,2"}}); + + SetLastSequence(5U); + constexpr int input_level = 0; + auto files = cfd_->current()->storage_info()->LevelFiles(input_level); + RunCompaction({files}, {input_level}, {expected_results}); +} + +TEST_F(CompactionJobTest, NonAssocMerge) { + merge_op_ = MergeOperators::CreateStringAppendTESTOperator(); + NewDB(); + + auto file1 = mock::MakeMockFile({ + {KeyStr("a", 5U, kTypeMerge), "5"}, + {KeyStr("a", 4U, kTypeMerge), "4"}, + {KeyStr("a", 3U, kTypeMerge), "3"}, + }); + AddMockFile(file1); + + auto file2 = mock::MakeMockFile( + {{KeyStr("b", 2U, kTypeMerge), "2"}, {KeyStr("b", 1U, kTypeMerge), "1"}}); + AddMockFile(file2); + + auto expected_results = + mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), "3,4,5"}, + {KeyStr("b", 0U, kTypeValue), "1,2"}}); + + SetLastSequence(5U); + constexpr int input_level = 0; + auto files = cfd_->current()->storage_info()->LevelFiles(input_level); + RunCompaction({files}, {input_level}, {expected_results}); +} + +// Filters merge operands with value 10. +TEST_F(CompactionJobTest, MergeOperandFilter) { + merge_op_ = MergeOperators::CreateUInt64AddOperator(); + compaction_filter_.reset(new test::FilterNumber(10U)); + NewDB(); + + auto file1 = mock::MakeMockFile( + {{KeyStr("a", 5U, kTypeMerge), test::EncodeInt(5U)}, + {KeyStr("a", 4U, kTypeMerge), test::EncodeInt(10U)}, // Filtered + {KeyStr("a", 3U, kTypeMerge), test::EncodeInt(3U)}}); + AddMockFile(file1); + + auto file2 = mock::MakeMockFile({ + {KeyStr("b", 2U, kTypeMerge), test::EncodeInt(2U)}, + {KeyStr("b", 1U, kTypeMerge), test::EncodeInt(10U)} // Filtered + }); + AddMockFile(file2); + + auto expected_results = + mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), test::EncodeInt(8U)}, + {KeyStr("b", 0U, kTypeValue), test::EncodeInt(2U)}}); + + SetLastSequence(5U); + constexpr int input_level = 0; + auto files = cfd_->current()->storage_info()->LevelFiles(input_level); + RunCompaction({files}, {input_level}, {expected_results}); +} + +TEST_F(CompactionJobTest, FilterSomeMergeOperands) { + merge_op_ = MergeOperators::CreateUInt64AddOperator(); + compaction_filter_.reset(new test::FilterNumber(10U)); + NewDB(); + + auto file1 = mock::MakeMockFile( + {{KeyStr("a", 5U, kTypeMerge), test::EncodeInt(5U)}, + {KeyStr("a", 4U, kTypeMerge), test::EncodeInt(10U)}, // Filtered + {KeyStr("a", 3U, kTypeValue), test::EncodeInt(5U)}, + {KeyStr("d", 8U, kTypeMerge), test::EncodeInt(10U)}}); + AddMockFile(file1); + + auto file2 = + mock::MakeMockFile({{KeyStr("b", 2U, kTypeMerge), test::EncodeInt(10U)}, + {KeyStr("b", 1U, kTypeMerge), test::EncodeInt(10U)}, + {KeyStr("c", 2U, kTypeMerge), test::EncodeInt(3U)}, + {KeyStr("c", 1U, kTypeValue), test::EncodeInt(7U)}, + {KeyStr("d", 1U, kTypeValue), test::EncodeInt(6U)}}); + AddMockFile(file2); + + auto file3 = + mock::MakeMockFile({{KeyStr("a", 1U, kTypeMerge), test::EncodeInt(3U)}}); + AddMockFile(file3, 2); + + auto expected_results = mock::MakeMockFile({ + {KeyStr("a", 5U, kTypeValue), test::EncodeInt(10U)}, + {KeyStr("c", 2U, kTypeValue), test::EncodeInt(10U)}, + {KeyStr("d", 1U, kTypeValue), test::EncodeInt(6U)} + // b does not appear because the operands are filtered + }); + + SetLastSequence(5U); + constexpr int input_level = 0; + auto files = cfd_->current()->storage_info()->LevelFiles(input_level); + RunCompaction({files}, {input_level}, {expected_results}); +} + +// Test where all operands/merge results are filtered out. +TEST_F(CompactionJobTest, FilterAllMergeOperands) { + merge_op_ = MergeOperators::CreateUInt64AddOperator(); + compaction_filter_.reset(new test::FilterNumber(10U)); + NewDB(); + + auto file1 = + mock::MakeMockFile({{KeyStr("a", 11U, kTypeMerge), test::EncodeInt(10U)}, + {KeyStr("a", 10U, kTypeMerge), test::EncodeInt(10U)}, + {KeyStr("a", 9U, kTypeMerge), test::EncodeInt(10U)}}); + AddMockFile(file1); + + auto file2 = + mock::MakeMockFile({{KeyStr("b", 8U, kTypeMerge), test::EncodeInt(10U)}, + {KeyStr("b", 7U, kTypeMerge), test::EncodeInt(10U)}, + {KeyStr("b", 6U, kTypeMerge), test::EncodeInt(10U)}, + {KeyStr("b", 5U, kTypeMerge), test::EncodeInt(10U)}, + {KeyStr("b", 4U, kTypeMerge), test::EncodeInt(10U)}, + {KeyStr("b", 3U, kTypeMerge), test::EncodeInt(10U)}, + {KeyStr("b", 2U, kTypeMerge), test::EncodeInt(10U)}, + {KeyStr("c", 2U, kTypeMerge), test::EncodeInt(10U)}, + {KeyStr("c", 1U, kTypeMerge), test::EncodeInt(10U)}}); + AddMockFile(file2); + + auto file3 = + mock::MakeMockFile({{KeyStr("a", 2U, kTypeMerge), test::EncodeInt(10U)}, + {KeyStr("b", 1U, kTypeMerge), test::EncodeInt(10U)}}); + AddMockFile(file3, 2); + + SetLastSequence(11U); + constexpr int input_level = 0; + auto files = cfd_->current()->storage_info()->LevelFiles(input_level); + + mock::KVVector empty_map; + RunCompaction({files}, {input_level}, {empty_map}); +} + +TEST_F(CompactionJobTest, SimpleSingleDelete) { + NewDB(); + + auto file1 = mock::MakeMockFile({ + {KeyStr("a", 5U, kTypeDeletion), ""}, + {KeyStr("b", 6U, kTypeSingleDeletion), ""}, + }); + AddMockFile(file1); + + auto file2 = mock::MakeMockFile({{KeyStr("a", 3U, kTypeValue), "val"}, + {KeyStr("b", 4U, kTypeValue), "val"}}); + AddMockFile(file2); + + auto file3 = mock::MakeMockFile({ + {KeyStr("a", 1U, kTypeValue), "val"}, + }); + AddMockFile(file3, 2); + + auto expected_results = + mock::MakeMockFile({{KeyStr("a", 5U, kTypeDeletion), ""}}); + + SetLastSequence(6U); + constexpr int input_level = 0; + auto files = cfd_->current()->storage_info()->LevelFiles(input_level); + RunCompaction({files}, {input_level}, {expected_results}); +} + +TEST_F(CompactionJobTest, SingleDeleteSnapshots) { + NewDB(); + + auto file1 = mock::MakeMockFile({ + {KeyStr("A", 12U, kTypeSingleDeletion), ""}, + {KeyStr("a", 12U, kTypeSingleDeletion), ""}, + {KeyStr("b", 21U, kTypeSingleDeletion), ""}, + {KeyStr("c", 22U, kTypeSingleDeletion), ""}, + {KeyStr("d", 9U, kTypeSingleDeletion), ""}, + {KeyStr("f", 21U, kTypeSingleDeletion), ""}, + {KeyStr("j", 11U, kTypeSingleDeletion), ""}, + {KeyStr("j", 9U, kTypeSingleDeletion), ""}, + {KeyStr("k", 12U, kTypeSingleDeletion), ""}, + {KeyStr("k", 11U, kTypeSingleDeletion), ""}, + {KeyStr("l", 3U, kTypeSingleDeletion), ""}, + {KeyStr("l", 2U, kTypeSingleDeletion), ""}, + }); + AddMockFile(file1); + + auto file2 = mock::MakeMockFile({ + {KeyStr("0", 2U, kTypeSingleDeletion), ""}, + {KeyStr("a", 11U, kTypeValue), "val1"}, + {KeyStr("b", 11U, kTypeValue), "val2"}, + {KeyStr("c", 21U, kTypeValue), "val3"}, + {KeyStr("d", 8U, kTypeValue), "val4"}, + {KeyStr("e", 2U, kTypeSingleDeletion), ""}, + {KeyStr("f", 1U, kTypeValue), "val1"}, + {KeyStr("g", 11U, kTypeSingleDeletion), ""}, + {KeyStr("h", 2U, kTypeSingleDeletion), ""}, + {KeyStr("m", 12U, kTypeValue), "val1"}, + {KeyStr("m", 11U, kTypeSingleDeletion), ""}, + {KeyStr("m", 8U, kTypeValue), "val2"}, + }); + AddMockFile(file2); + + auto file3 = mock::MakeMockFile({ + {KeyStr("A", 1U, kTypeValue), "val"}, + {KeyStr("e", 1U, kTypeValue), "val"}, + }); + AddMockFile(file3, 2); + + auto expected_results = mock::MakeMockFile({ + {KeyStr("A", 12U, kTypeSingleDeletion), ""}, + {KeyStr("a", 12U, kTypeSingleDeletion), ""}, + {KeyStr("a", 11U, kTypeValue), ""}, + {KeyStr("b", 21U, kTypeSingleDeletion), ""}, + {KeyStr("b", 11U, kTypeValue), "val2"}, + {KeyStr("c", 22U, kTypeSingleDeletion), ""}, + {KeyStr("c", 21U, kTypeValue), ""}, + {KeyStr("e", 2U, kTypeSingleDeletion), ""}, + {KeyStr("f", 21U, kTypeSingleDeletion), ""}, + {KeyStr("f", 1U, kTypeValue), "val1"}, + {KeyStr("g", 11U, kTypeSingleDeletion), ""}, + {KeyStr("j", 11U, kTypeSingleDeletion), ""}, + {KeyStr("k", 11U, kTypeSingleDeletion), ""}, + {KeyStr("m", 12U, kTypeValue), "val1"}, + {KeyStr("m", 11U, kTypeSingleDeletion), ""}, + {KeyStr("m", 8U, kTypeValue), "val2"}, + }); + + SetLastSequence(22U); + constexpr int input_level = 0; + auto files = cfd_->current()->storage_info()->LevelFiles(input_level); + RunCompaction({files}, {input_level}, {expected_results}, {10U, 20U}, 10U); +} + +TEST_F(CompactionJobTest, EarliestWriteConflictSnapshot) { + NewDB(); + + // Test multiple snapshots where the earliest snapshot is not a + // write-conflic-snapshot. + + auto file1 = mock::MakeMockFile({ + {KeyStr("A", 24U, kTypeSingleDeletion), ""}, + {KeyStr("A", 23U, kTypeValue), "val"}, + {KeyStr("B", 24U, kTypeSingleDeletion), ""}, + {KeyStr("B", 23U, kTypeValue), "val"}, + {KeyStr("D", 24U, kTypeSingleDeletion), ""}, + {KeyStr("G", 32U, kTypeSingleDeletion), ""}, + {KeyStr("G", 31U, kTypeValue), "val"}, + {KeyStr("G", 24U, kTypeSingleDeletion), ""}, + {KeyStr("G", 23U, kTypeValue), "val2"}, + {KeyStr("H", 31U, kTypeValue), "val"}, + {KeyStr("H", 24U, kTypeSingleDeletion), ""}, + {KeyStr("H", 23U, kTypeValue), "val"}, + {KeyStr("I", 35U, kTypeSingleDeletion), ""}, + {KeyStr("I", 34U, kTypeValue), "val2"}, + {KeyStr("I", 33U, kTypeSingleDeletion), ""}, + {KeyStr("I", 32U, kTypeValue), "val3"}, + {KeyStr("I", 31U, kTypeSingleDeletion), ""}, + {KeyStr("J", 34U, kTypeValue), "val"}, + {KeyStr("J", 33U, kTypeSingleDeletion), ""}, + {KeyStr("J", 25U, kTypeValue), "val2"}, + {KeyStr("J", 24U, kTypeSingleDeletion), ""}, + }); + AddMockFile(file1); + + auto file2 = mock::MakeMockFile({ + {KeyStr("A", 14U, kTypeSingleDeletion), ""}, + {KeyStr("A", 13U, kTypeValue), "val2"}, + {KeyStr("C", 14U, kTypeSingleDeletion), ""}, + {KeyStr("C", 13U, kTypeValue), "val"}, + {KeyStr("E", 12U, kTypeSingleDeletion), ""}, + {KeyStr("F", 4U, kTypeSingleDeletion), ""}, + {KeyStr("F", 3U, kTypeValue), "val"}, + {KeyStr("G", 14U, kTypeSingleDeletion), ""}, + {KeyStr("G", 13U, kTypeValue), "val3"}, + {KeyStr("H", 14U, kTypeSingleDeletion), ""}, + {KeyStr("H", 13U, kTypeValue), "val2"}, + {KeyStr("I", 13U, kTypeValue), "val4"}, + {KeyStr("I", 12U, kTypeSingleDeletion), ""}, + {KeyStr("I", 11U, kTypeValue), "val5"}, + {KeyStr("J", 15U, kTypeValue), "val3"}, + {KeyStr("J", 14U, kTypeSingleDeletion), ""}, + }); + AddMockFile(file2); + + auto expected_results = mock::MakeMockFile({ + {KeyStr("A", 24U, kTypeSingleDeletion), ""}, + {KeyStr("A", 23U, kTypeValue), ""}, + {KeyStr("B", 24U, kTypeSingleDeletion), ""}, + {KeyStr("B", 23U, kTypeValue), ""}, + {KeyStr("D", 24U, kTypeSingleDeletion), ""}, + {KeyStr("E", 12U, kTypeSingleDeletion), ""}, + {KeyStr("G", 32U, kTypeSingleDeletion), ""}, + {KeyStr("G", 31U, kTypeValue), ""}, + {KeyStr("H", 31U, kTypeValue), "val"}, + {KeyStr("I", 35U, kTypeSingleDeletion), ""}, + {KeyStr("I", 34U, kTypeValue), ""}, + {KeyStr("I", 31U, kTypeSingleDeletion), ""}, + {KeyStr("I", 13U, kTypeValue), "val4"}, + {KeyStr("J", 34U, kTypeValue), "val"}, + {KeyStr("J", 33U, kTypeSingleDeletion), ""}, + {KeyStr("J", 25U, kTypeValue), "val2"}, + {KeyStr("J", 24U, kTypeSingleDeletion), ""}, + {KeyStr("J", 15U, kTypeValue), "val3"}, + {KeyStr("J", 14U, kTypeSingleDeletion), ""}, + }); + + SetLastSequence(24U); + constexpr int input_level = 0; + auto files = cfd_->current()->storage_info()->LevelFiles(input_level); + RunCompaction({files}, {input_level}, {expected_results}, {10U, 20U, 30U}, + 20U); +} + +TEST_F(CompactionJobTest, SingleDeleteZeroSeq) { + NewDB(); + + auto file1 = mock::MakeMockFile({ + {KeyStr("A", 10U, kTypeSingleDeletion), ""}, + {KeyStr("dummy", 5U, kTypeValue), "val2"}, + }); + AddMockFile(file1); + + auto file2 = mock::MakeMockFile({ + {KeyStr("A", 0U, kTypeValue), "val"}, + }); + AddMockFile(file2); + + auto expected_results = mock::MakeMockFile({ + {KeyStr("dummy", 0U, kTypeValue), "val2"}, + }); + + SetLastSequence(22U); + constexpr int input_level = 0; + auto files = cfd_->current()->storage_info()->LevelFiles(input_level); + RunCompaction({files}, {input_level}, {expected_results}, {}); +} + +TEST_F(CompactionJobTest, MultiSingleDelete) { + // Tests three scenarios involving multiple single delete/put pairs: + // + // A: Put Snapshot SDel Put SDel -> Put Snapshot SDel + // B: Snapshot Put SDel Put SDel Snapshot -> Snapshot SDel Snapshot + // C: SDel Put SDel Snapshot Put -> Snapshot Put + // D: (Put) SDel Snapshot Put SDel -> (Put) SDel Snapshot SDel + // E: Put SDel Snapshot Put SDel -> Snapshot SDel + // F: Put SDel Put Sdel Snapshot -> removed + // G: Snapshot SDel Put SDel Put -> Snapshot Put SDel + // H: (Put) Put SDel Put Sdel Snapshot -> Removed + // I: (Put) Snapshot Put SDel Put SDel -> SDel + // J: Put Put SDel Put SDel SDel Snapshot Put Put SDel SDel Put + // -> Snapshot Put + // K: SDel SDel Put SDel Put Put Snapshot SDel Put SDel SDel Put SDel + // -> Snapshot Put Snapshot SDel + // L: SDel Put SDel Put SDel Snapshot SDel Put SDel SDel Put SDel + // -> Snapshot SDel Put SDel + // M: (Put) SDel Put SDel Put SDel Snapshot Put SDel SDel Put SDel SDel + // -> SDel Snapshot Put SDel + NewDB(); + + auto file1 = mock::MakeMockFile({ + {KeyStr("A", 14U, kTypeSingleDeletion), ""}, + {KeyStr("A", 13U, kTypeValue), "val5"}, + {KeyStr("A", 12U, kTypeSingleDeletion), ""}, + {KeyStr("B", 14U, kTypeSingleDeletion), ""}, + {KeyStr("B", 13U, kTypeValue), "val2"}, + {KeyStr("C", 14U, kTypeValue), "val3"}, + {KeyStr("D", 12U, kTypeSingleDeletion), ""}, + {KeyStr("D", 11U, kTypeValue), "val4"}, + {KeyStr("G", 15U, kTypeValue), "val"}, + {KeyStr("G", 14U, kTypeSingleDeletion), ""}, + {KeyStr("G", 13U, kTypeValue), "val"}, + {KeyStr("I", 14U, kTypeSingleDeletion), ""}, + {KeyStr("I", 13U, kTypeValue), "val"}, + {KeyStr("J", 15U, kTypeValue), "val"}, + {KeyStr("J", 14U, kTypeSingleDeletion), ""}, + {KeyStr("J", 13U, kTypeSingleDeletion), ""}, + {KeyStr("J", 12U, kTypeValue), "val"}, + {KeyStr("J", 11U, kTypeValue), "val"}, + {KeyStr("K", 16U, kTypeSingleDeletion), ""}, + {KeyStr("K", 15U, kTypeValue), "val1"}, + {KeyStr("K", 14U, kTypeSingleDeletion), ""}, + {KeyStr("K", 13U, kTypeSingleDeletion), ""}, + {KeyStr("K", 12U, kTypeValue), "val2"}, + {KeyStr("K", 11U, kTypeSingleDeletion), ""}, + {KeyStr("L", 16U, kTypeSingleDeletion), ""}, + {KeyStr("L", 15U, kTypeValue), "val"}, + {KeyStr("L", 14U, kTypeSingleDeletion), ""}, + {KeyStr("L", 13U, kTypeSingleDeletion), ""}, + {KeyStr("L", 12U, kTypeValue), "val"}, + {KeyStr("L", 11U, kTypeSingleDeletion), ""}, + {KeyStr("M", 16U, kTypeSingleDeletion), ""}, + {KeyStr("M", 15U, kTypeSingleDeletion), ""}, + {KeyStr("M", 14U, kTypeValue), "val"}, + {KeyStr("M", 13U, kTypeSingleDeletion), ""}, + {KeyStr("M", 12U, kTypeSingleDeletion), ""}, + {KeyStr("M", 11U, kTypeValue), "val"}, + }); + AddMockFile(file1); + + auto file2 = mock::MakeMockFile({ + {KeyStr("A", 10U, kTypeValue), "val"}, + {KeyStr("B", 12U, kTypeSingleDeletion), ""}, + {KeyStr("B", 11U, kTypeValue), "val2"}, + {KeyStr("C", 10U, kTypeSingleDeletion), ""}, + {KeyStr("C", 9U, kTypeValue), "val6"}, + {KeyStr("C", 8U, kTypeSingleDeletion), ""}, + {KeyStr("D", 10U, kTypeSingleDeletion), ""}, + {KeyStr("E", 12U, kTypeSingleDeletion), ""}, + {KeyStr("E", 11U, kTypeValue), "val"}, + {KeyStr("E", 5U, kTypeSingleDeletion), ""}, + {KeyStr("E", 4U, kTypeValue), "val"}, + {KeyStr("F", 6U, kTypeSingleDeletion), ""}, + {KeyStr("F", 5U, kTypeValue), "val"}, + {KeyStr("F", 4U, kTypeSingleDeletion), ""}, + {KeyStr("F", 3U, kTypeValue), "val"}, + {KeyStr("G", 12U, kTypeSingleDeletion), ""}, + {KeyStr("H", 6U, kTypeSingleDeletion), ""}, + {KeyStr("H", 5U, kTypeValue), "val"}, + {KeyStr("H", 4U, kTypeSingleDeletion), ""}, + {KeyStr("H", 3U, kTypeValue), "val"}, + {KeyStr("I", 12U, kTypeSingleDeletion), ""}, + {KeyStr("I", 11U, kTypeValue), "val"}, + {KeyStr("J", 6U, kTypeSingleDeletion), ""}, + {KeyStr("J", 5U, kTypeSingleDeletion), ""}, + {KeyStr("J", 4U, kTypeValue), "val"}, + {KeyStr("J", 3U, kTypeSingleDeletion), ""}, + {KeyStr("J", 2U, kTypeValue), "val"}, + {KeyStr("K", 8U, kTypeValue), "val3"}, + {KeyStr("K", 7U, kTypeValue), "val4"}, + {KeyStr("K", 6U, kTypeSingleDeletion), ""}, + {KeyStr("K", 5U, kTypeValue), "val5"}, + {KeyStr("K", 2U, kTypeSingleDeletion), ""}, + {KeyStr("K", 1U, kTypeSingleDeletion), ""}, + {KeyStr("L", 5U, kTypeSingleDeletion), ""}, + {KeyStr("L", 4U, kTypeValue), "val"}, + {KeyStr("L", 3U, kTypeSingleDeletion), ""}, + {KeyStr("L", 2U, kTypeValue), "val"}, + {KeyStr("L", 1U, kTypeSingleDeletion), ""}, + {KeyStr("M", 10U, kTypeSingleDeletion), ""}, + {KeyStr("M", 7U, kTypeValue), "val"}, + {KeyStr("M", 5U, kTypeSingleDeletion), ""}, + {KeyStr("M", 4U, kTypeValue), "val"}, + {KeyStr("M", 3U, kTypeSingleDeletion), ""}, + }); + AddMockFile(file2); + + auto file3 = mock::MakeMockFile({ + {KeyStr("D", 1U, kTypeValue), "val"}, + {KeyStr("H", 1U, kTypeValue), "val"}, + {KeyStr("I", 2U, kTypeValue), "val"}, + }); + AddMockFile(file3, 2); + + auto file4 = mock::MakeMockFile({ + {KeyStr("M", 1U, kTypeValue), "val"}, + }); + AddMockFile(file4, 2); + + auto expected_results = + mock::MakeMockFile({{KeyStr("A", 14U, kTypeSingleDeletion), ""}, + {KeyStr("A", 13U, kTypeValue), ""}, + {KeyStr("A", 12U, kTypeSingleDeletion), ""}, + {KeyStr("A", 10U, kTypeValue), "val"}, + {KeyStr("B", 14U, kTypeSingleDeletion), ""}, + {KeyStr("B", 13U, kTypeValue), ""}, + {KeyStr("C", 14U, kTypeValue), "val3"}, + {KeyStr("D", 12U, kTypeSingleDeletion), ""}, + {KeyStr("D", 11U, kTypeValue), ""}, + {KeyStr("D", 10U, kTypeSingleDeletion), ""}, + {KeyStr("E", 12U, kTypeSingleDeletion), ""}, + {KeyStr("E", 11U, kTypeValue), ""}, + {KeyStr("G", 15U, kTypeValue), "val"}, + {KeyStr("G", 12U, kTypeSingleDeletion), ""}, + {KeyStr("I", 14U, kTypeSingleDeletion), ""}, + {KeyStr("I", 13U, kTypeValue), ""}, + {KeyStr("J", 15U, kTypeValue), "val"}, + {KeyStr("K", 16U, kTypeSingleDeletion), ""}, + {KeyStr("K", 15U, kTypeValue), ""}, + {KeyStr("K", 11U, kTypeSingleDeletion), ""}, + {KeyStr("K", 8U, kTypeValue), "val3"}, + {KeyStr("L", 16U, kTypeSingleDeletion), ""}, + {KeyStr("L", 15U, kTypeValue), ""}, + {KeyStr("L", 11U, kTypeSingleDeletion), ""}, + {KeyStr("M", 15U, kTypeSingleDeletion), ""}, + {KeyStr("M", 14U, kTypeValue), ""}, + {KeyStr("M", 3U, kTypeSingleDeletion), ""}}); + + SetLastSequence(22U); + constexpr int input_level = 0; + auto files = cfd_->current()->storage_info()->LevelFiles(input_level); + RunCompaction({files}, {input_level}, {expected_results}, {10U}, 10U); +} + +// This test documents the behavior where a corrupt key follows a deletion or a +// single deletion and the (single) deletion gets removed while the corrupt key +// gets written out. TODO(noetzli): We probably want a better way to treat +// corrupt keys. +TEST_F(CompactionJobTest, DISABLED_CorruptionAfterDeletion) { + NewDB(); + + auto file1 = + mock::MakeMockFile({{test::KeyStr("A", 6U, kTypeValue), "val3"}, + {test::KeyStr("a", 5U, kTypeDeletion), ""}, + {test::KeyStr("a", 4U, kTypeValue, true), "val"}}); + AddMockFile(file1); + + auto file2 = + mock::MakeMockFile({{test::KeyStr("b", 3U, kTypeSingleDeletion), ""}, + {test::KeyStr("b", 2U, kTypeValue, true), "val"}, + {test::KeyStr("c", 1U, kTypeValue), "val2"}}); + AddMockFile(file2); + + auto expected_results = + mock::MakeMockFile({{test::KeyStr("A", 0U, kTypeValue), "val3"}, + {test::KeyStr("a", 0U, kTypeValue, true), "val"}, + {test::KeyStr("b", 0U, kTypeValue, true), "val"}, + {test::KeyStr("c", 0U, kTypeValue), "val2"}}); + + SetLastSequence(6U); + constexpr int input_level = 0; + auto files = cfd_->current()->storage_info()->LevelFiles(input_level); + RunCompaction({files}, {input_level}, {expected_results}); +} + +TEST_F(CompactionJobTest, OldestBlobFileNumber) { + NewDB(); + + // Note: blob1 is inlined TTL, so it will not be considered for the purposes + // of identifying the oldest referenced blob file. Similarly, blob6 will be + // ignored because it has TTL and hence refers to a TTL blob file. + const stl_wrappers::KVMap::value_type blob1( + KeyStr("a", 1U, kTypeBlobIndex), BlobStrInlinedTTL("foo", 1234567890ULL)); + const stl_wrappers::KVMap::value_type blob2(KeyStr("b", 2U, kTypeBlobIndex), + BlobStr(59, 123456, 999)); + const stl_wrappers::KVMap::value_type blob3(KeyStr("c", 3U, kTypeBlobIndex), + BlobStr(138, 1000, 1 << 8)); + auto file1 = mock::MakeMockFile({blob1, blob2, blob3}); + AddMockFile(file1); + + const stl_wrappers::KVMap::value_type blob4(KeyStr("d", 4U, kTypeBlobIndex), + BlobStr(199, 3 << 10, 1 << 20)); + const stl_wrappers::KVMap::value_type blob5(KeyStr("e", 5U, kTypeBlobIndex), + BlobStr(19, 6789, 333)); + const stl_wrappers::KVMap::value_type blob6( + KeyStr("f", 6U, kTypeBlobIndex), + BlobStrTTL(5, 2048, 1 << 7, 1234567890ULL)); + auto file2 = mock::MakeMockFile({blob4, blob5, blob6}); + AddMockFile(file2); + + const stl_wrappers::KVMap::value_type expected_blob1( + KeyStr("a", 0U, kTypeBlobIndex), blob1.second); + const stl_wrappers::KVMap::value_type expected_blob2( + KeyStr("b", 0U, kTypeBlobIndex), blob2.second); + const stl_wrappers::KVMap::value_type expected_blob3( + KeyStr("c", 0U, kTypeBlobIndex), blob3.second); + const stl_wrappers::KVMap::value_type expected_blob4( + KeyStr("d", 0U, kTypeBlobIndex), blob4.second); + const stl_wrappers::KVMap::value_type expected_blob5( + KeyStr("e", 0U, kTypeBlobIndex), blob5.second); + const stl_wrappers::KVMap::value_type expected_blob6( + KeyStr("f", 0U, kTypeBlobIndex), blob6.second); + auto expected_results = + mock::MakeMockFile({expected_blob1, expected_blob2, expected_blob3, + expected_blob4, expected_blob5, expected_blob6}); + + SetLastSequence(6U); + constexpr int input_level = 0; + auto files = cfd_->current()->storage_info()->LevelFiles(input_level); + RunCompaction({files}, {input_level}, {expected_results}, + std::vector(), kMaxSequenceNumber, + /* output_level */ 1, /* verify */ true, + /* expected_oldest_blob_file_numbers */ {19}); +} + +TEST_F(CompactionJobTest, VerifyPenultimateLevelOutput) { + cf_options_.bottommost_temperature = Temperature::kCold; + SyncPoint::GetInstance()->SetCallBack( + "Compaction::SupportsPerKeyPlacement:Enabled", [&](void* arg) { + auto supports_per_key_placement = static_cast(arg); + *supports_per_key_placement = true; + }); + + std::atomic_uint64_t latest_cold_seq = 0; + + SyncPoint::GetInstance()->SetCallBack( + "CompactionIterator::PrepareOutput.context", [&](void* arg) { + auto context = static_cast(arg); + context->output_to_penultimate_level = + context->seq_num > latest_cold_seq; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + NewDB(); + + // Add files on different levels that may overlap + auto file0_1 = mock::MakeMockFile({{KeyStr("z", 12U, kTypeValue), "val"}}); + AddMockFile(file0_1); + + auto file1_1 = mock::MakeMockFile({{KeyStr("b", 10U, kTypeValue), "val"}, + {KeyStr("f", 11U, kTypeValue), "val"}}); + AddMockFile(file1_1, 1); + auto file1_2 = mock::MakeMockFile({{KeyStr("j", 12U, kTypeValue), "val"}, + {KeyStr("k", 13U, kTypeValue), "val"}}); + AddMockFile(file1_2, 1); + auto file1_3 = mock::MakeMockFile({{KeyStr("p", 14U, kTypeValue), "val"}, + {KeyStr("u", 15U, kTypeValue), "val"}}); + AddMockFile(file1_3, 1); + + auto file2_1 = mock::MakeMockFile({{KeyStr("f", 8U, kTypeValue), "val"}, + {KeyStr("h", 9U, kTypeValue), "val"}}); + AddMockFile(file2_1, 2); + auto file2_2 = mock::MakeMockFile({{KeyStr("m", 6U, kTypeValue), "val"}, + {KeyStr("p", 7U, kTypeValue), "val"}}); + AddMockFile(file2_2, 2); + + auto file3_1 = mock::MakeMockFile({{KeyStr("g", 2U, kTypeValue), "val"}, + {KeyStr("k", 3U, kTypeValue), "val"}}); + AddMockFile(file3_1, 3); + auto file3_2 = mock::MakeMockFile({{KeyStr("v", 4U, kTypeValue), "val"}, + {KeyStr("x", 5U, kTypeValue), "val"}}); + AddMockFile(file3_2, 3); + + auto cfd = versions_->GetColumnFamilySet()->GetDefault(); + const std::vector input_levels = {0, 1, 2, 3}; + auto files0 = cfd->current()->storage_info()->LevelFiles(input_levels[0]); + auto files1 = cfd->current()->storage_info()->LevelFiles(input_levels[1]); + auto files2 = cfd->current()->storage_info()->LevelFiles(input_levels[2]); + auto files3 = cfd->current()->storage_info()->LevelFiles(input_levels[3]); + + RunLastLevelCompaction( + {files0, files1, files2, files3}, input_levels, + /*verify_func=*/[&](Compaction& comp) { + for (char c = 'a'; c <= 'z'; c++) { + std::string c_str; + c_str = c; + const Slice key(c_str); + if (c == 'a') { + ASSERT_FALSE(comp.WithinPenultimateLevelOutputRange(key)); + } else { + ASSERT_TRUE(comp.WithinPenultimateLevelOutputRange(key)); + } + } + }); +} + +TEST_F(CompactionJobTest, NoEnforceSingleDeleteContract) { + db_options_.enforce_single_del_contracts = false; + NewDB(); + + auto file = + mock::MakeMockFile({{KeyStr("a", 4U, kTypeSingleDeletion), ""}, + {KeyStr("a", 3U, kTypeDeletion), "dontcare"}}); + AddMockFile(file); + SetLastSequence(4U); + + auto expected_results = mock::MakeMockFile(); + constexpr int input_level = 0; + auto files = cfd_->current()->storage_info()->LevelFiles(input_level); + RunCompaction({files}, {input_level}, {expected_results}); +} + +TEST_F(CompactionJobTest, InputSerialization) { + // Setup a random CompactionServiceInput + CompactionServiceInput input; + const int kStrMaxLen = 1000; + Random rnd(static_cast(time(nullptr))); + Random64 rnd64(time(nullptr)); + input.column_family.name = rnd.RandomString(rnd.Uniform(kStrMaxLen)); + input.column_family.options.comparator = ReverseBytewiseComparator(); + input.column_family.options.max_bytes_for_level_base = + rnd64.Uniform(UINT64_MAX); + input.column_family.options.disable_auto_compactions = rnd.OneIn(2); + input.column_family.options.compression = kZSTD; + input.column_family.options.compression_opts.level = 4; + input.db_options.max_background_flushes = 10; + input.db_options.paranoid_checks = rnd.OneIn(2); + input.db_options.statistics = CreateDBStatistics(); + input.db_options.env = env_; + while (!rnd.OneIn(10)) { + input.snapshots.emplace_back(rnd64.Uniform(UINT64_MAX)); + } + while (!rnd.OneIn(10)) { + input.input_files.emplace_back(rnd.RandomString( + rnd.Uniform(kStrMaxLen - 1) + + 1)); // input file name should have at least one character + } + input.output_level = 4; + input.has_begin = rnd.OneIn(2); + if (input.has_begin) { + input.begin = rnd.RandomBinaryString(rnd.Uniform(kStrMaxLen)); + } + input.has_end = rnd.OneIn(2); + if (input.has_end) { + input.end = rnd.RandomBinaryString(rnd.Uniform(kStrMaxLen)); + } + + std::string output; + ASSERT_OK(input.Write(&output)); + + // Test deserialization + CompactionServiceInput deserialized1; + ASSERT_OK(CompactionServiceInput::Read(output, &deserialized1)); + ASSERT_TRUE(deserialized1.TEST_Equals(&input)); + + // Test mismatch + deserialized1.db_options.max_background_flushes += 10; + std::string mismatch; + ASSERT_FALSE(deserialized1.TEST_Equals(&input, &mismatch)); + ASSERT_EQ(mismatch, "db_options.max_background_flushes"); + + // Test unknown field + CompactionServiceInput deserialized2; + output.clear(); + ASSERT_OK(input.Write(&output)); + output.append("new_field=123;"); + + ASSERT_OK(CompactionServiceInput::Read(output, &deserialized2)); + ASSERT_TRUE(deserialized2.TEST_Equals(&input)); + + // Test missing field + CompactionServiceInput deserialized3; + deserialized3.output_level = 0; + std::string to_remove = "output_level=4;"; + size_t pos = output.find(to_remove); + ASSERT_TRUE(pos != std::string::npos); + output.erase(pos, to_remove.length()); + ASSERT_OK(CompactionServiceInput::Read(output, &deserialized3)); + mismatch.clear(); + ASSERT_FALSE(deserialized3.TEST_Equals(&input, &mismatch)); + ASSERT_EQ(mismatch, "output_level"); + + // manually set the value back, should match the original structure + deserialized3.output_level = 4; + ASSERT_TRUE(deserialized3.TEST_Equals(&input)); + + // Test invalid version + output.clear(); + ASSERT_OK(input.Write(&output)); + + uint32_t data_version = DecodeFixed32(output.data()); + const size_t kDataVersionSize = sizeof(data_version); + ASSERT_EQ(data_version, + 1U); // Update once the default data version is changed + char buf[kDataVersionSize]; + EncodeFixed32(buf, data_version + 10); // make sure it's not valid + output.replace(0, kDataVersionSize, buf, kDataVersionSize); + Status s = CompactionServiceInput::Read(output, &deserialized3); + ASSERT_TRUE(s.IsNotSupported()); +} + +TEST_F(CompactionJobTest, ResultSerialization) { + // Setup a random CompactionServiceResult + CompactionServiceResult result; + const int kStrMaxLen = 1000; + Random rnd(static_cast(time(nullptr))); + Random64 rnd64(time(nullptr)); + std::vector status_list = { + Status::OK(), + Status::InvalidArgument("invalid option"), + Status::Aborted("failed to run"), + Status::NotSupported("not supported option"), + }; + result.status = + status_list.at(rnd.Uniform(static_cast(status_list.size()))); + while (!rnd.OneIn(10)) { + UniqueId64x2 id{rnd64.Uniform(UINT64_MAX), rnd64.Uniform(UINT64_MAX)}; + result.output_files.emplace_back( + rnd.RandomString(rnd.Uniform(kStrMaxLen)), rnd64.Uniform(UINT64_MAX), + rnd64.Uniform(UINT64_MAX), + rnd.RandomBinaryString(rnd.Uniform(kStrMaxLen)), + rnd.RandomBinaryString(rnd.Uniform(kStrMaxLen)), + rnd64.Uniform(UINT64_MAX), rnd64.Uniform(UINT64_MAX), + rnd64.Uniform(UINT64_MAX), rnd64.Uniform(UINT64_MAX), rnd.OneIn(2), id); + } + result.output_level = rnd.Uniform(10); + result.output_path = rnd.RandomString(rnd.Uniform(kStrMaxLen)); + result.num_output_records = rnd64.Uniform(UINT64_MAX); + result.total_bytes = rnd64.Uniform(UINT64_MAX); + result.bytes_read = 123; + result.bytes_written = rnd64.Uniform(UINT64_MAX); + result.stats.elapsed_micros = rnd64.Uniform(UINT64_MAX); + result.stats.num_output_files = rnd.Uniform(1000); + result.stats.is_full_compaction = rnd.OneIn(2); + result.stats.num_single_del_mismatch = rnd64.Uniform(UINT64_MAX); + result.stats.num_input_files = 9; + + std::string output; + ASSERT_OK(result.Write(&output)); + + // Test deserialization + CompactionServiceResult deserialized1; + ASSERT_OK(CompactionServiceResult::Read(output, &deserialized1)); + ASSERT_TRUE(deserialized1.TEST_Equals(&result)); + + // Test mismatch + deserialized1.stats.num_input_files += 10; + std::string mismatch; + ASSERT_FALSE(deserialized1.TEST_Equals(&result, &mismatch)); + ASSERT_EQ(mismatch, "stats.num_input_files"); + + // Test unique id mismatch + if (!result.output_files.empty()) { + CompactionServiceResult deserialized_tmp; + ASSERT_OK(CompactionServiceResult::Read(output, &deserialized_tmp)); + deserialized_tmp.output_files[0].unique_id[0] += 1; + ASSERT_FALSE(deserialized_tmp.TEST_Equals(&result, &mismatch)); + ASSERT_EQ(mismatch, "output_files.unique_id"); + deserialized_tmp.status.PermitUncheckedError(); + } + + // Test unknown field + CompactionServiceResult deserialized2; + output.clear(); + ASSERT_OK(result.Write(&output)); + output.append("new_field=123;"); + + ASSERT_OK(CompactionServiceResult::Read(output, &deserialized2)); + ASSERT_TRUE(deserialized2.TEST_Equals(&result)); + + // Test missing field + CompactionServiceResult deserialized3; + deserialized3.bytes_read = 0; + std::string to_remove = "bytes_read=123;"; + size_t pos = output.find(to_remove); + ASSERT_TRUE(pos != std::string::npos); + output.erase(pos, to_remove.length()); + ASSERT_OK(CompactionServiceResult::Read(output, &deserialized3)); + mismatch.clear(); + ASSERT_FALSE(deserialized3.TEST_Equals(&result, &mismatch)); + ASSERT_EQ(mismatch, "bytes_read"); + + deserialized3.bytes_read = 123; + ASSERT_TRUE(deserialized3.TEST_Equals(&result)); + + // Test invalid version + output.clear(); + ASSERT_OK(result.Write(&output)); + + uint32_t data_version = DecodeFixed32(output.data()); + const size_t kDataVersionSize = sizeof(data_version); + ASSERT_EQ(data_version, + 1U); // Update once the default data version is changed + char buf[kDataVersionSize]; + EncodeFixed32(buf, data_version + 10); // make sure it's not valid + output.replace(0, kDataVersionSize, buf, kDataVersionSize); + Status s = CompactionServiceResult::Read(output, &deserialized3); + ASSERT_TRUE(s.IsNotSupported()); + for (const auto& item : status_list) { + item.PermitUncheckedError(); + } +} + +class CompactionJobDynamicFileSizeTest + : public CompactionJobTestBase, + public ::testing::WithParamInterface { + public: + CompactionJobDynamicFileSizeTest() + : CompactionJobTestBase( + test::PerThreadDBPath("compaction_job_dynamic_file_size_test"), + BytewiseComparator(), [](uint64_t /*ts*/) { return ""; }, + /*test_io_priority=*/false, TableTypeForTest::kMockTable) {} +}; + +TEST_P(CompactionJobDynamicFileSizeTest, CutForMaxCompactionBytes) { + // dynamic_file_size option should have no impact on cutting for max + // compaction bytes. + bool enable_dyanmic_file_size = GetParam(); + cf_options_.level_compaction_dynamic_file_size = enable_dyanmic_file_size; + + NewDB(); + mutable_cf_options_.target_file_size_base = 80; + mutable_cf_options_.max_compaction_bytes = 21; + + auto file1 = mock::MakeMockFile({ + {KeyStr("c", 5U, kTypeValue), "val2"}, + {KeyStr("n", 6U, kTypeValue), "val3"}, + }); + AddMockFile(file1); + + auto file2 = mock::MakeMockFile({{KeyStr("h", 3U, kTypeValue), "val"}, + {KeyStr("j", 4U, kTypeValue), "val"}}); + AddMockFile(file2, 1); + + // Create three L2 files, each size 10. + // max_compaction_bytes 21 means the compaction output in L1 will + // be cut to at least two files. + auto file3 = mock::MakeMockFile({{KeyStr("b", 1U, kTypeValue), "val"}, + {KeyStr("c", 1U, kTypeValue), "val"}, + {KeyStr("c1", 1U, kTypeValue), "val"}, + {KeyStr("c2", 1U, kTypeValue), "val"}, + {KeyStr("c3", 1U, kTypeValue), "val"}, + {KeyStr("c4", 1U, kTypeValue), "val"}, + {KeyStr("d", 1U, kTypeValue), "val"}, + {KeyStr("e", 2U, kTypeValue), "val"}}); + AddMockFile(file3, 2); + + auto file4 = mock::MakeMockFile({{KeyStr("h", 1U, kTypeValue), "val"}, + {KeyStr("i", 1U, kTypeValue), "val"}, + {KeyStr("i1", 1U, kTypeValue), "val"}, + {KeyStr("i2", 1U, kTypeValue), "val"}, + {KeyStr("i3", 1U, kTypeValue), "val"}, + {KeyStr("i4", 1U, kTypeValue), "val"}, + {KeyStr("j", 1U, kTypeValue), "val"}, + {KeyStr("k", 2U, kTypeValue), "val"}}); + AddMockFile(file4, 2); + + auto file5 = mock::MakeMockFile({{KeyStr("l", 1U, kTypeValue), "val"}, + {KeyStr("m", 1U, kTypeValue), "val"}, + {KeyStr("m1", 1U, kTypeValue), "val"}, + {KeyStr("m2", 1U, kTypeValue), "val"}, + {KeyStr("m3", 1U, kTypeValue), "val"}, + {KeyStr("m4", 1U, kTypeValue), "val"}, + {KeyStr("n", 1U, kTypeValue), "val"}, + {KeyStr("o", 2U, kTypeValue), "val"}}); + AddMockFile(file5, 2); + + // The expected output should be: + // L1: [c, h, j] [n] + // L2: [b ... e] [h ... k] [l ... o] + // It's better to have "j" in the first file, because anyway it's overlapping + // with the second file on L2. + // (Note: before this PR, it was cut at "h" because it's using the internal + // comparator which think L1 "h" with seqno 3 is smaller than L2 "h" with + // seqno 1, but actually they're overlapped with the compaction picker). + + auto expected_file1 = + mock::MakeMockFile({{KeyStr("c", 5U, kTypeValue), "val2"}, + {KeyStr("h", 3U, kTypeValue), "val"}, + {KeyStr("j", 4U, kTypeValue), "val"}}); + auto expected_file2 = + mock::MakeMockFile({{KeyStr("n", 6U, kTypeValue), "val3"}}); + + SetLastSequence(6U); + + const std::vector input_levels = {0, 1}; + auto lvl0_files = cfd_->current()->storage_info()->LevelFiles(0); + auto lvl1_files = cfd_->current()->storage_info()->LevelFiles(1); + + RunCompaction({lvl0_files, lvl1_files}, input_levels, + {expected_file1, expected_file2}); +} + +TEST_P(CompactionJobDynamicFileSizeTest, CutToSkipGrandparentFile) { + bool enable_dyanmic_file_size = GetParam(); + cf_options_.level_compaction_dynamic_file_size = enable_dyanmic_file_size; + + NewDB(); + // Make sure the grandparent level file size (10) qualifies skipping. + // Currently, it has to be > 1/8 of target file size. + mutable_cf_options_.target_file_size_base = 70; + + auto file1 = mock::MakeMockFile({ + {KeyStr("a", 5U, kTypeValue), "val2"}, + {KeyStr("z", 6U, kTypeValue), "val3"}, + }); + AddMockFile(file1); + + auto file2 = mock::MakeMockFile({{KeyStr("c", 3U, kTypeValue), "val"}, + {KeyStr("x", 4U, kTypeValue), "val"}}); + AddMockFile(file2, 1); + + auto file3 = mock::MakeMockFile({{KeyStr("b", 1U, kTypeValue), "val"}, + {KeyStr("d", 2U, kTypeValue), "val"}}); + AddMockFile(file3, 2); + + auto file4 = mock::MakeMockFile({{KeyStr("h", 1U, kTypeValue), "val"}, + {KeyStr("i", 2U, kTypeValue), "val"}}); + AddMockFile(file4, 2); + + auto file5 = mock::MakeMockFile({{KeyStr("v", 1U, kTypeValue), "val"}, + {KeyStr("y", 2U, kTypeValue), "val"}}); + AddMockFile(file5, 2); + + auto expected_file1 = + mock::MakeMockFile({{KeyStr("a", 5U, kTypeValue), "val2"}, + {KeyStr("c", 3U, kTypeValue), "val"}}); + auto expected_file2 = + mock::MakeMockFile({{KeyStr("x", 4U, kTypeValue), "val"}, + {KeyStr("z", 6U, kTypeValue), "val3"}}); + + auto expected_file_disable_dynamic_file_size = + mock::MakeMockFile({{KeyStr("a", 5U, kTypeValue), "val2"}, + {KeyStr("c", 3U, kTypeValue), "val"}, + {KeyStr("x", 4U, kTypeValue), "val"}, + {KeyStr("z", 6U, kTypeValue), "val3"}}); + + SetLastSequence(6U); + const std::vector input_levels = {0, 1}; + auto lvl0_files = cfd_->current()->storage_info()->LevelFiles(0); + auto lvl1_files = cfd_->current()->storage_info()->LevelFiles(1); + if (enable_dyanmic_file_size) { + RunCompaction({lvl0_files, lvl1_files}, input_levels, + {expected_file1, expected_file2}); + } else { + RunCompaction({lvl0_files, lvl1_files}, input_levels, + {expected_file_disable_dynamic_file_size}); + } +} + +TEST_P(CompactionJobDynamicFileSizeTest, CutToAlignGrandparentBoundary) { + bool enable_dyanmic_file_size = GetParam(); + cf_options_.level_compaction_dynamic_file_size = enable_dyanmic_file_size; + NewDB(); + + // MockTable has 1 byte per entry by default and each file is 10 bytes. + // When the file size is smaller than 100, it won't cut file earlier to align + // with its grandparent boundary. + const size_t kKeyValueSize = 10000; + mock_table_factory_->SetKeyValueSize(kKeyValueSize); + + mutable_cf_options_.target_file_size_base = 10 * kKeyValueSize; + + mock::KVVector file1; + char ch = 'd'; + // Add value from d -> o + for (char i = 0; i < 12; i++) { + file1.emplace_back(KeyStr(std::string(1, ch + i), i + 10, kTypeValue), + "val" + std::to_string(i)); + } + + AddMockFile(file1); + + auto file2 = mock::MakeMockFile({{KeyStr("e", 3U, kTypeValue), "val"}, + {KeyStr("s", 4U, kTypeValue), "val"}}); + AddMockFile(file2, 1); + + // the 1st grandparent file should be skipped + auto file3 = mock::MakeMockFile({{KeyStr("a", 1U, kTypeValue), "val"}, + {KeyStr("b", 2U, kTypeValue), "val"}}); + AddMockFile(file3, 2); + + auto file4 = mock::MakeMockFile({{KeyStr("c", 1U, kTypeValue), "val"}, + {KeyStr("e", 2U, kTypeValue), "val"}}); + AddMockFile(file4, 2); + + auto file5 = mock::MakeMockFile({{KeyStr("h", 1U, kTypeValue), "val"}, + {KeyStr("j", 2U, kTypeValue), "val"}}); + AddMockFile(file5, 2); + + auto file6 = mock::MakeMockFile({{KeyStr("k", 1U, kTypeValue), "val"}, + {KeyStr("n", 2U, kTypeValue), "val"}}); + AddMockFile(file6, 2); + + auto file7 = mock::MakeMockFile({{KeyStr("q", 1U, kTypeValue), "val"}, + {KeyStr("t", 2U, kTypeValue), "val"}}); + AddMockFile(file7, 2); + + // The expected outputs are: + // L1: [d,e,f,g,h,i,j] [k,l,m,n,o,s] + // L2: [a, b] [c, e] [h, j] [k, n] [q, t] + // The first output cut earlier at "j", so it could be aligned with L2 files. + // If dynamic_file_size is not enabled, it will be cut based on the + // target_file_size + mock::KVVector expected_file1; + for (char i = 0; i < 7; i++) { + expected_file1.emplace_back( + KeyStr(std::string(1, ch + i), i + 10, kTypeValue), + "val" + std::to_string(i)); + } + + mock::KVVector expected_file2; + for (char i = 7; i < 12; i++) { + expected_file2.emplace_back( + KeyStr(std::string(1, ch + i), i + 10, kTypeValue), + "val" + std::to_string(i)); + } + expected_file2.emplace_back(KeyStr("s", 4U, kTypeValue), "val"); + + mock::KVVector expected_file_disable_dynamic_file_size1; + for (char i = 0; i < 10; i++) { + expected_file_disable_dynamic_file_size1.emplace_back( + KeyStr(std::string(1, ch + i), i + 10, kTypeValue), + "val" + std::to_string(i)); + } + + mock::KVVector expected_file_disable_dynamic_file_size2; + for (char i = 10; i < 12; i++) { + expected_file_disable_dynamic_file_size2.emplace_back( + KeyStr(std::string(1, ch + i), i + 10, kTypeValue), + "val" + std::to_string(i)); + } + + expected_file_disable_dynamic_file_size2.emplace_back( + KeyStr("s", 4U, kTypeValue), "val"); + + SetLastSequence(22U); + const std::vector input_levels = {0, 1}; + auto lvl0_files = cfd_->current()->storage_info()->LevelFiles(0); + auto lvl1_files = cfd_->current()->storage_info()->LevelFiles(1); + if (enable_dyanmic_file_size) { + RunCompaction({lvl0_files, lvl1_files}, input_levels, + {expected_file1, expected_file2}); + } else { + RunCompaction({lvl0_files, lvl1_files}, input_levels, + {expected_file_disable_dynamic_file_size1, + expected_file_disable_dynamic_file_size2}); + } +} + +TEST_P(CompactionJobDynamicFileSizeTest, CutToAlignGrandparentBoundarySameKey) { + bool enable_dyanmic_file_size = GetParam(); + cf_options_.level_compaction_dynamic_file_size = enable_dyanmic_file_size; + NewDB(); + + // MockTable has 1 byte per entry by default and each file is 10 bytes. + // When the file size is smaller than 100, it won't cut file earlier to align + // with its grandparent boundary. + const size_t kKeyValueSize = 10000; + mock_table_factory_->SetKeyValueSize(kKeyValueSize); + + mutable_cf_options_.target_file_size_base = 10 * kKeyValueSize; + + mock::KVVector file1; + for (int i = 0; i < 7; i++) { + file1.emplace_back(KeyStr("a", 100 - i, kTypeValue), + "val" + std::to_string(100 - i)); + } + file1.emplace_back(KeyStr("b", 90, kTypeValue), "valb"); + + AddMockFile(file1); + + auto file2 = mock::MakeMockFile({{KeyStr("a", 93U, kTypeValue), "val93"}, + {KeyStr("b", 90U, kTypeValue), "valb"}}); + AddMockFile(file2, 1); + + auto file3 = mock::MakeMockFile({{KeyStr("a", 89U, kTypeValue), "val"}, + {KeyStr("a", 88U, kTypeValue), "val"}}); + AddMockFile(file3, 2); + + auto file4 = mock::MakeMockFile({{KeyStr("a", 87U, kTypeValue), "val"}, + {KeyStr("a", 86U, kTypeValue), "val"}}); + AddMockFile(file4, 2); + + auto file5 = mock::MakeMockFile({{KeyStr("b", 85U, kTypeValue), "val"}, + {KeyStr("b", 84U, kTypeValue), "val"}}); + AddMockFile(file5, 2); + + mock::KVVector expected_file1; + mock::KVVector expected_file_disable_dynamic_file_size; + + for (int i = 0; i < 8; i++) { + expected_file1.emplace_back(KeyStr("a", 100 - i, kTypeValue), + "val" + std::to_string(100 - i)); + expected_file_disable_dynamic_file_size.emplace_back( + KeyStr("a", 100 - i, kTypeValue), "val" + std::to_string(100 - i)); + } + + // make sure `b` is cut in a separated file (so internally it's not using + // internal comparator, which will think the "b:90" (seqno 90) here is smaller + // than "b:85" on L2.) + auto expected_file2 = + mock::MakeMockFile({{KeyStr("b", 90U, kTypeValue), "valb"}}); + + expected_file_disable_dynamic_file_size.emplace_back( + KeyStr("b", 90U, kTypeValue), "valb"); + + SetLastSequence(122U); + const std::vector input_levels = {0, 1}; + auto lvl0_files = cfd_->current()->storage_info()->LevelFiles(0); + auto lvl1_files = cfd_->current()->storage_info()->LevelFiles(1); + + // Just keep all the history + std::vector snapshots; + for (int i = 80; i <= 100; i++) { + snapshots.emplace_back(i); + } + if (enable_dyanmic_file_size) { + RunCompaction({lvl0_files, lvl1_files}, input_levels, + {expected_file1, expected_file2}, snapshots); + } else { + RunCompaction({lvl0_files, lvl1_files}, input_levels, + {expected_file_disable_dynamic_file_size}, snapshots); + } +} + +TEST_P(CompactionJobDynamicFileSizeTest, CutForMaxCompactionBytesSameKey) { + // dynamic_file_size option should have no impact on cutting for max + // compaction bytes. + bool enable_dyanmic_file_size = GetParam(); + cf_options_.level_compaction_dynamic_file_size = enable_dyanmic_file_size; + + NewDB(); + mutable_cf_options_.target_file_size_base = 80; + mutable_cf_options_.max_compaction_bytes = 20; + + auto file1 = mock::MakeMockFile({{KeyStr("a", 104U, kTypeValue), "val1"}, + {KeyStr("b", 103U, kTypeValue), "val"}}); + AddMockFile(file1); + + auto file2 = mock::MakeMockFile({{KeyStr("a", 102U, kTypeValue), "val2"}, + {KeyStr("c", 101U, kTypeValue), "val"}}); + AddMockFile(file2, 1); + + for (int i = 0; i < 10; i++) { + auto file = + mock::MakeMockFile({{KeyStr("a", 100 - (i * 2), kTypeValue), "val"}, + {KeyStr("a", 99 - (i * 2), kTypeValue), "val"}}); + AddMockFile(file, 2); + } + + for (int i = 0; i < 10; i++) { + auto file = + mock::MakeMockFile({{KeyStr("b", 80 - (i * 2), kTypeValue), "val"}, + {KeyStr("b", 79 - (i * 2), kTypeValue), "val"}}); + AddMockFile(file, 2); + } + + auto file5 = mock::MakeMockFile({{KeyStr("c", 60U, kTypeValue), "valc"}, + {KeyStr("c", 59U, kTypeValue), "valc"}}); + + // "a" has 10 overlapped grandparent files (each size 10), which is far + // exceeded the `max_compaction_bytes`, but make sure 2 "a" are not separated, + // as splitting them won't help reducing the compaction size. + // also make sure "b" and "c" are cut separately. + mock::KVVector expected_file1 = + mock::MakeMockFile({{KeyStr("a", 104U, kTypeValue), "val1"}, + {KeyStr("a", 102U, kTypeValue), "val2"}}); + mock::KVVector expected_file2 = + mock::MakeMockFile({{KeyStr("b", 103U, kTypeValue), "val"}}); + mock::KVVector expected_file3 = + mock::MakeMockFile({{KeyStr("c", 101U, kTypeValue), "val"}}); + + SetLastSequence(122U); + const std::vector input_levels = {0, 1}; + auto lvl0_files = cfd_->current()->storage_info()->LevelFiles(0); + auto lvl1_files = cfd_->current()->storage_info()->LevelFiles(1); + + // Just keep all the history + std::vector snapshots; + for (int i = 80; i <= 105; i++) { + snapshots.emplace_back(i); + } + RunCompaction({lvl0_files, lvl1_files}, input_levels, + {expected_file1, expected_file2, expected_file3}, snapshots); +} + +INSTANTIATE_TEST_CASE_P(CompactionJobDynamicFileSizeTest, + CompactionJobDynamicFileSizeTest, testing::Bool()); + +class CompactionJobTimestampTest : public CompactionJobTestBase { + public: + CompactionJobTimestampTest() + : CompactionJobTestBase(test::PerThreadDBPath("compaction_job_ts_test"), + test::BytewiseComparatorWithU64TsWrapper(), + test::EncodeInt, /*test_io_priority=*/false, + TableTypeForTest::kMockTable) {} +}; + +TEST_F(CompactionJobTimestampTest, GCDisabled) { + NewDB(); + + auto file1 = + mock::MakeMockFile({{KeyStr("a", 10, ValueType::kTypeValue, 100), "a10"}, + {KeyStr("a", 9, ValueType::kTypeValue, 99), "a9"}, + {KeyStr("b", 8, ValueType::kTypeValue, 98), "b8"}, + {KeyStr("d", 7, ValueType::kTypeValue, 97), "d7"}}); + + AddMockFile(file1); + + auto file2 = mock::MakeMockFile( + {{KeyStr("b", 6, ValueType::kTypeDeletionWithTimestamp, 96), ""}, + {KeyStr("c", 5, ValueType::kTypeDeletionWithTimestamp, 95), ""}, + {KeyStr("c", 4, ValueType::kTypeValue, 94), "c5"}, + {KeyStr("d", 3, ValueType::kTypeSingleDeletion, 93), ""}}); + AddMockFile(file2); + + SetLastSequence(10); + + auto expected_results = mock::MakeMockFile( + {{KeyStr("a", 10, ValueType::kTypeValue, 100), "a10"}, + {KeyStr("a", 9, ValueType::kTypeValue, 99), "a9"}, + {KeyStr("b", 8, ValueType::kTypeValue, 98), "b8"}, + {KeyStr("b", 6, ValueType::kTypeDeletionWithTimestamp, 96), ""}, + {KeyStr("c", 5, ValueType::kTypeDeletionWithTimestamp, 95), ""}, + {KeyStr("c", 4, ValueType::kTypeValue, 94), "c5"}, + {KeyStr("d", 7, ValueType::kTypeValue, 97), "d7"}, + {KeyStr("d", 3, ValueType::kTypeSingleDeletion, 93), ""}}); + constexpr int input_level = 0; + const auto& files = cfd_->current()->storage_info()->LevelFiles(input_level); + RunCompaction({files}, {input_level}, {expected_results}); +} + +TEST_F(CompactionJobTimestampTest, NoKeyExpired) { + NewDB(); + + auto file1 = + mock::MakeMockFile({{KeyStr("a", 6, ValueType::kTypeValue, 100), "a6"}, + {KeyStr("b", 7, ValueType::kTypeValue, 101), "b7"}, + {KeyStr("c", 5, ValueType::kTypeValue, 99), "c5"}}); + AddMockFile(file1); + + auto file2 = + mock::MakeMockFile({{KeyStr("a", 4, ValueType::kTypeValue, 98), "a4"}, + {KeyStr("c", 3, ValueType::kTypeValue, 97), "c3"}}); + AddMockFile(file2); + + SetLastSequence(101); + + auto expected_results = + mock::MakeMockFile({{KeyStr("a", 6, ValueType::kTypeValue, 100), "a6"}, + {KeyStr("a", 4, ValueType::kTypeValue, 98), "a4"}, + {KeyStr("b", 7, ValueType::kTypeValue, 101), "b7"}, + {KeyStr("c", 5, ValueType::kTypeValue, 99), "c5"}, + {KeyStr("c", 3, ValueType::kTypeValue, 97), "c3"}}); + constexpr int input_level = 0; + const auto& files = cfd_->current()->storage_info()->LevelFiles(input_level); + + full_history_ts_low_ = encode_u64_ts_(0); + RunCompaction({files}, {input_level}, {expected_results}); +} + +TEST_F(CompactionJobTimestampTest, AllKeysExpired) { + NewDB(); + + auto file1 = mock::MakeMockFile( + {{KeyStr("a", 5, ValueType::kTypeDeletionWithTimestamp, 100), ""}, + {KeyStr("b", 6, ValueType::kTypeSingleDeletion, 99), ""}, + {KeyStr("c", 7, ValueType::kTypeValue, 98), "c7"}}); + AddMockFile(file1); + + auto file2 = mock::MakeMockFile( + {{KeyStr("a", 4, ValueType::kTypeValue, 97), "a4"}, + {KeyStr("b", 3, ValueType::kTypeValue, 96), "b3"}, + {KeyStr("c", 2, ValueType::kTypeDeletionWithTimestamp, 95), ""}, + {KeyStr("c", 1, ValueType::kTypeValue, 94), "c1"}}); + AddMockFile(file2); + + SetLastSequence(7); + + auto expected_results = + mock::MakeMockFile({{KeyStr("c", 0, ValueType::kTypeValue, 0), "c7"}}); + constexpr int input_level = 0; + const auto& files = cfd_->current()->storage_info()->LevelFiles(input_level); + + full_history_ts_low_ = encode_u64_ts_(std::numeric_limits::max()); + RunCompaction({files}, {input_level}, {expected_results}); +} + +TEST_F(CompactionJobTimestampTest, SomeKeysExpired) { + NewDB(); + + auto file1 = + mock::MakeMockFile({{KeyStr("a", 5, ValueType::kTypeValue, 50), "a5"}, + {KeyStr("b", 6, ValueType::kTypeValue, 49), "b6"}}); + AddMockFile(file1); + + auto file2 = mock::MakeMockFile( + {{KeyStr("a", 3, ValueType::kTypeValue, 48), "a3"}, + {KeyStr("a", 2, ValueType::kTypeValue, 46), "a2"}, + {KeyStr("b", 4, ValueType::kTypeDeletionWithTimestamp, 47), ""}}); + AddMockFile(file2); + + SetLastSequence(6); + + auto expected_results = + mock::MakeMockFile({{KeyStr("a", 5, ValueType::kTypeValue, 50), "a5"}, + {KeyStr("a", 0, ValueType::kTypeValue, 0), "a3"}, + {KeyStr("b", 6, ValueType::kTypeValue, 49), "b6"}}); + constexpr int input_level = 0; + const auto& files = cfd_->current()->storage_info()->LevelFiles(input_level); + + full_history_ts_low_ = encode_u64_ts_(49); + RunCompaction({files}, {input_level}, {expected_results}); +} + +class CompactionJobTimestampTestWithBbTable : public CompactionJobTestBase { + public: + // Block-based table is needed if we want to test subcompaction partitioning + // with anchors. + explicit CompactionJobTimestampTestWithBbTable() + : CompactionJobTestBase( + test::PerThreadDBPath("compaction_job_ts_bbt_test"), + test::BytewiseComparatorWithU64TsWrapper(), test::EncodeInt, + /*test_io_priority=*/false, TableTypeForTest::kBlockBasedTable) {} +}; + +TEST_F(CompactionJobTimestampTestWithBbTable, SubcompactionAnchorL1) { + cf_options_.target_file_size_base = 20; + mutable_cf_options_.target_file_size_base = 20; + NewDB(); + + const std::vector keys = { + KeyStr("a", 20, ValueType::kTypeValue, 200), + KeyStr("b", 21, ValueType::kTypeValue, 210), + KeyStr("b", 20, ValueType::kTypeValue, 200), + KeyStr("b", 18, ValueType::kTypeValue, 180), + KeyStr("c", 17, ValueType::kTypeValue, 170), + KeyStr("c", 16, ValueType::kTypeValue, 160), + KeyStr("c", 15, ValueType::kTypeValue, 150)}; + const std::vector values = {"a20", "b21", "b20", "b18", + "c17", "c16", "c15"}; + + constexpr int input_level = 1; + + auto file1 = mock::MakeMockFile( + {{keys[0], values[0]}, {keys[1], values[1]}, {keys[2], values[2]}}); + AddMockFile(file1, input_level); + + auto file2 = mock::MakeMockFile( + {{keys[3], values[3]}, {keys[4], values[4]}, {keys[5], values[5]}}); + AddMockFile(file2, input_level); + + auto file3 = mock::MakeMockFile({{keys[6], values[6]}}); + AddMockFile(file3, input_level); + + SetLastSequence(20); + + auto output1 = mock::MakeMockFile({{keys[0], values[0]}}); + auto output2 = mock::MakeMockFile( + {{keys[1], values[1]}, {keys[2], values[2]}, {keys[3], values[3]}}); + auto output3 = mock::MakeMockFile( + {{keys[4], values[4]}, {keys[5], values[5]}, {keys[6], values[6]}}); + + auto expected_results = + std::vector{output1, output2, output3}; + const auto& files = cfd_->current()->storage_info()->LevelFiles(input_level); + + constexpr int output_level = 2; + constexpr int max_subcompactions = 4; + RunCompaction({files}, {input_level}, expected_results, /*snapshots=*/{}, + /*earliest_write_conflict_snapshot=*/kMaxSequenceNumber, + output_level, /*verify=*/true, {kInvalidBlobFileNumber}, + /*check_get_priority=*/false, Env::IO_TOTAL, Env::IO_TOTAL, + max_subcompactions); +} + +TEST_F(CompactionJobTimestampTestWithBbTable, SubcompactionL0) { + cf_options_.target_file_size_base = 20; + mutable_cf_options_.target_file_size_base = 20; + NewDB(); + + const std::vector keys = { + KeyStr("a", 20, ValueType::kTypeValue, 200), + KeyStr("b", 20, ValueType::kTypeValue, 200), + KeyStr("b", 19, ValueType::kTypeValue, 190), + KeyStr("b", 18, ValueType::kTypeValue, 180), + KeyStr("c", 17, ValueType::kTypeValue, 170), + KeyStr("c", 16, ValueType::kTypeValue, 160), + KeyStr("c", 15, ValueType::kTypeValue, 150)}; + const std::vector values = {"a20", "b20", "b19", "b18", + "c17", "c16", "c15"}; + + constexpr int input_level = 0; + + auto file1 = mock::MakeMockFile({{keys[5], values[5]}, {keys[6], values[6]}}); + AddMockFile(file1, input_level); + + auto file2 = mock::MakeMockFile({{keys[3], values[3]}, {keys[4], values[4]}}); + AddMockFile(file2, input_level); + + auto file3 = mock::MakeMockFile( + {{keys[0], values[0]}, {keys[1], values[1]}, {keys[2], values[2]}}); + AddMockFile(file3, input_level); + + SetLastSequence(20); + + auto output1 = mock::MakeMockFile({{keys[0], values[0]}}); + auto output2 = mock::MakeMockFile( + {{keys[1], values[1]}, {keys[2], values[2]}, {keys[3], values[3]}}); + auto output3 = mock::MakeMockFile( + {{keys[4], values[4]}, {keys[5], values[5]}, {keys[6], values[6]}}); + + auto expected_results = + std::vector{output1, output2, output3}; + const auto& files = cfd_->current()->storage_info()->LevelFiles(input_level); + + constexpr int output_level = 1; + constexpr int max_subcompactions = 4; + RunCompaction({files}, {input_level}, expected_results, /*snapshots=*/{}, + /*earliest_write_conflict_snapshot=*/kMaxSequenceNumber, + output_level, /*verify=*/true, {kInvalidBlobFileNumber}, + /*check_get_priority=*/false, Env::IO_TOTAL, Env::IO_TOTAL, + max_subcompactions); +} + +// The io priority of the compaction reads and writes are different from +// other DB reads and writes. To prepare the compaction input files, use the +// default filesystem from Env. To test the io priority of the compaction +// reads and writes, db_options_.fs is set as MockTestFileSystem. +class CompactionJobIOPriorityTest : public CompactionJobTestBase { + public: + CompactionJobIOPriorityTest() + : CompactionJobTestBase( + test::PerThreadDBPath("compaction_job_io_priority_test"), + BytewiseComparator(), [](uint64_t /*ts*/) { return ""; }, + /*test_io_priority=*/true, TableTypeForTest::kBlockBasedTable) {} +}; + +TEST_F(CompactionJobIOPriorityTest, WriteControllerStateNormal) { + // When the state from WriteController is normal. + NewDB(); + mock::KVVector expected_results = CreateTwoFiles(false); + auto cfd = versions_->GetColumnFamilySet()->GetDefault(); + constexpr int input_level = 0; + auto files = cfd->current()->storage_info()->LevelFiles(input_level); + ASSERT_EQ(2U, files.size()); + RunCompaction({files}, {input_level}, {expected_results}, {}, + kMaxSequenceNumber, 1, false, {kInvalidBlobFileNumber}, false, + Env::IO_LOW, Env::IO_LOW); +} + +TEST_F(CompactionJobIOPriorityTest, WriteControllerStateDelayed) { + // When the state from WriteController is Delayed. + NewDB(); + mock::KVVector expected_results = CreateTwoFiles(false); + auto cfd = versions_->GetColumnFamilySet()->GetDefault(); + constexpr int input_level = 0; + auto files = cfd->current()->storage_info()->LevelFiles(input_level); + ASSERT_EQ(2U, files.size()); + { + std::unique_ptr delay_token = + write_controller_.GetDelayToken(1000000); + RunCompaction({files}, {input_level}, {expected_results}, {}, + kMaxSequenceNumber, 1, false, {kInvalidBlobFileNumber}, false, + Env::IO_USER, Env::IO_USER); + } +} + +TEST_F(CompactionJobIOPriorityTest, WriteControllerStateStalled) { + // When the state from WriteController is Stalled. + NewDB(); + mock::KVVector expected_results = CreateTwoFiles(false); + auto cfd = versions_->GetColumnFamilySet()->GetDefault(); + constexpr int input_level = 0; + auto files = cfd->current()->storage_info()->LevelFiles(input_level); + ASSERT_EQ(2U, files.size()); + { + std::unique_ptr stop_token = + write_controller_.GetStopToken(); + RunCompaction({files}, {input_level}, {expected_results}, {}, + kMaxSequenceNumber, 1, false, {kInvalidBlobFileNumber}, false, + Env::IO_USER, Env::IO_USER); + } +} + +TEST_F(CompactionJobIOPriorityTest, GetRateLimiterPriority) { + NewDB(); + mock::KVVector expected_results = CreateTwoFiles(false); + auto cfd = versions_->GetColumnFamilySet()->GetDefault(); + constexpr int input_level = 0; + auto files = cfd->current()->storage_info()->LevelFiles(input_level); + ASSERT_EQ(2U, files.size()); + RunCompaction({files}, {input_level}, {expected_results}, {}, + kMaxSequenceNumber, 1, false, {kInvalidBlobFileNumber}, true, + Env::IO_LOW, Env::IO_LOW); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_outputs.cc b/librocksdb-sys/rocksdb/db/compaction/compaction_outputs.cc new file mode 100644 index 0000000..3e21484 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_outputs.cc @@ -0,0 +1,792 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/compaction/compaction_outputs.h" + +#include "db/builder.h" + +namespace ROCKSDB_NAMESPACE { + +void CompactionOutputs::NewBuilder(const TableBuilderOptions& tboptions) { + builder_.reset(NewTableBuilder(tboptions, file_writer_.get())); +} + +Status CompactionOutputs::Finish(const Status& intput_status, + const SeqnoToTimeMapping& seqno_time_mapping) { + FileMetaData* meta = GetMetaData(); + assert(meta != nullptr); + Status s = intput_status; + if (s.ok()) { + std::string seqno_time_mapping_str; + seqno_time_mapping.Encode(seqno_time_mapping_str, meta->fd.smallest_seqno, + meta->fd.largest_seqno, meta->file_creation_time); + builder_->SetSeqnoTimeTableProperties(seqno_time_mapping_str, + meta->oldest_ancester_time); + s = builder_->Finish(); + + } else { + builder_->Abandon(); + } + Status io_s = builder_->io_status(); + if (s.ok()) { + s = io_s; + } else { + io_s.PermitUncheckedError(); + } + const uint64_t current_bytes = builder_->FileSize(); + if (s.ok()) { + meta->fd.file_size = current_bytes; + meta->tail_size = builder_->GetTailSize(); + meta->marked_for_compaction = builder_->NeedCompact(); + meta->user_defined_timestamps_persisted = static_cast( + builder_->GetTableProperties().user_defined_timestamps_persisted); + } + current_output().finished = true; + stats_.bytes_written += current_bytes; + stats_.num_output_files = outputs_.size(); + + return s; +} + +IOStatus CompactionOutputs::WriterSyncClose(const Status& input_status, + SystemClock* clock, + Statistics* statistics, + bool use_fsync) { + IOStatus io_s; + if (input_status.ok()) { + StopWatch sw(clock, statistics, COMPACTION_OUTFILE_SYNC_MICROS); + io_s = file_writer_->Sync(use_fsync); + } + if (input_status.ok() && io_s.ok()) { + io_s = file_writer_->Close(); + } + + if (input_status.ok() && io_s.ok()) { + FileMetaData* meta = GetMetaData(); + meta->file_checksum = file_writer_->GetFileChecksum(); + meta->file_checksum_func_name = file_writer_->GetFileChecksumFuncName(); + } + + file_writer_.reset(); + + return io_s; +} + +bool CompactionOutputs::UpdateFilesToCutForTTLStates( + const Slice& internal_key) { + if (!files_to_cut_for_ttl_.empty()) { + const InternalKeyComparator* icmp = + &compaction_->column_family_data()->internal_comparator(); + if (cur_files_to_cut_for_ttl_ != -1) { + // Previous key is inside the range of a file + if (icmp->Compare(internal_key, + files_to_cut_for_ttl_[cur_files_to_cut_for_ttl_] + ->largest.Encode()) > 0) { + next_files_to_cut_for_ttl_ = cur_files_to_cut_for_ttl_ + 1; + cur_files_to_cut_for_ttl_ = -1; + return true; + } + } else { + // Look for the key position + while (next_files_to_cut_for_ttl_ < + static_cast(files_to_cut_for_ttl_.size())) { + if (icmp->Compare(internal_key, + files_to_cut_for_ttl_[next_files_to_cut_for_ttl_] + ->smallest.Encode()) >= 0) { + if (icmp->Compare(internal_key, + files_to_cut_for_ttl_[next_files_to_cut_for_ttl_] + ->largest.Encode()) <= 0) { + // With in the current file + cur_files_to_cut_for_ttl_ = next_files_to_cut_for_ttl_; + return true; + } + // Beyond the current file + next_files_to_cut_for_ttl_++; + } else { + // Still fall into the gap + break; + } + } + } + } + return false; +} + +size_t CompactionOutputs::UpdateGrandparentBoundaryInfo( + const Slice& internal_key) { + size_t curr_key_boundary_switched_num = 0; + const std::vector& grandparents = compaction_->grandparents(); + + if (grandparents.empty()) { + return curr_key_boundary_switched_num; + } + const Comparator* ucmp = compaction_->column_family_data()->user_comparator(); + + // Move the grandparent_index_ to the file containing the current user_key. + // If there are multiple files containing the same user_key, make sure the + // index points to the last file containing the key. + while (grandparent_index_ < grandparents.size()) { + if (being_grandparent_gap_) { + if (sstableKeyCompare(ucmp, internal_key, + grandparents[grandparent_index_]->smallest) < 0) { + break; + } + if (seen_key_) { + curr_key_boundary_switched_num++; + grandparent_overlapped_bytes_ += + grandparents[grandparent_index_]->fd.GetFileSize(); + grandparent_boundary_switched_num_++; + } + being_grandparent_gap_ = false; + } else { + int cmp_result = sstableKeyCompare( + ucmp, internal_key, grandparents[grandparent_index_]->largest); + // If it's same key, make sure grandparent_index_ is pointing to the last + // one. + if (cmp_result < 0 || + (cmp_result == 0 && + (grandparent_index_ == grandparents.size() - 1 || + sstableKeyCompare(ucmp, internal_key, + grandparents[grandparent_index_ + 1]->smallest) < + 0))) { + break; + } + if (seen_key_) { + curr_key_boundary_switched_num++; + grandparent_boundary_switched_num_++; + } + being_grandparent_gap_ = true; + grandparent_index_++; + } + } + + // If the first key is in the middle of a grandparent file, adding it to the + // overlap + if (!seen_key_ && !being_grandparent_gap_) { + assert(grandparent_overlapped_bytes_ == 0); + grandparent_overlapped_bytes_ = + GetCurrentKeyGrandparentOverlappedBytes(internal_key); + } + + seen_key_ = true; + return curr_key_boundary_switched_num; +} + +uint64_t CompactionOutputs::GetCurrentKeyGrandparentOverlappedBytes( + const Slice& internal_key) const { + // no overlap with any grandparent file + if (being_grandparent_gap_) { + return 0; + } + uint64_t overlapped_bytes = 0; + + const std::vector& grandparents = compaction_->grandparents(); + const Comparator* ucmp = compaction_->column_family_data()->user_comparator(); + InternalKey ikey; + ikey.DecodeFrom(internal_key); +#ifndef NDEBUG + // make sure the grandparent_index_ is pointing to the last files containing + // the current key. + int cmp_result = + sstableKeyCompare(ucmp, ikey, grandparents[grandparent_index_]->largest); + assert( + cmp_result < 0 || + (cmp_result == 0 && + (grandparent_index_ == grandparents.size() - 1 || + sstableKeyCompare( + ucmp, ikey, grandparents[grandparent_index_ + 1]->smallest) < 0))); + assert(sstableKeyCompare(ucmp, ikey, + grandparents[grandparent_index_]->smallest) >= 0); +#endif + overlapped_bytes += grandparents[grandparent_index_]->fd.GetFileSize(); + + // go backwards to find all overlapped files, one key can overlap multiple + // files. In the following example, if the current output key is `c`, and one + // compaction file was cut before `c`, current `c` can overlap with 3 files: + // [a b] [c... + // [b, b] [c, c] [c, c] [c, d] + for (int64_t i = static_cast(grandparent_index_) - 1; + i >= 0 && sstableKeyCompare(ucmp, ikey, grandparents[i]->largest) == 0; + i--) { + overlapped_bytes += grandparents[i]->fd.GetFileSize(); + } + + return overlapped_bytes; +} + +bool CompactionOutputs::ShouldStopBefore(const CompactionIterator& c_iter) { + assert(c_iter.Valid()); + const Slice& internal_key = c_iter.key(); +#ifndef NDEBUG + bool should_stop = false; + std::pair p{&should_stop, internal_key}; + TEST_SYNC_POINT_CALLBACK( + "CompactionOutputs::ShouldStopBefore::manual_decision", (void*)&p); + if (should_stop) { + return true; + } +#endif // NDEBUG + const uint64_t previous_overlapped_bytes = grandparent_overlapped_bytes_; + const InternalKeyComparator* icmp = + &compaction_->column_family_data()->internal_comparator(); + size_t num_grandparent_boundaries_crossed = 0; + bool should_stop_for_ttl = false; + // Always update grandparent information like overlapped file number, size + // etc., and TTL states. + // If compaction_->output_level() == 0, there is no need to update grandparent + // info, and that `grandparent` should be empty. + if (compaction_->output_level() > 0) { + num_grandparent_boundaries_crossed = + UpdateGrandparentBoundaryInfo(internal_key); + should_stop_for_ttl = UpdateFilesToCutForTTLStates(internal_key); + } + + if (!HasBuilder()) { + return false; + } + + if (should_stop_for_ttl) { + return true; + } + + // If there's user defined partitioner, check that first + if (partitioner_ && partitioner_->ShouldPartition(PartitionerRequest( + last_key_for_partitioner_, c_iter.user_key(), + current_output_file_size_)) == kRequired) { + return true; + } + + // files output to Level 0 won't be split + if (compaction_->output_level() == 0) { + return false; + } + + // reach the max file size + if (current_output_file_size_ >= compaction_->max_output_file_size()) { + return true; + } + + // Check if it needs to split for RoundRobin + // Invalid local_output_split_key indicates that we do not need to split + if (local_output_split_key_ != nullptr && !is_split_) { + // Split occurs when the next key is larger than/equal to the cursor + if (icmp->Compare(internal_key, local_output_split_key_->Encode()) >= 0) { + is_split_ = true; + return true; + } + } + + // only check if the current key is going to cross the grandparents file + // boundary (either the file beginning or ending). + if (num_grandparent_boundaries_crossed > 0) { + // Cut the file before the current key if the size of the current output + // file + its overlapped grandparent files is bigger than + // max_compaction_bytes. Which is to prevent future bigger than + // max_compaction_bytes compaction from the current output level. + if (grandparent_overlapped_bytes_ + current_output_file_size_ > + compaction_->max_compaction_bytes()) { + return true; + } + + // Cut the file if including the key is going to add a skippable file on + // the grandparent level AND its size is reasonably big (1/8 of target file + // size). For example, if it's compacting the files L0 + L1: + // L0: [1, 21] + // L1: [3, 23] + // L2: [2, 4] [11, 15] [22, 24] + // Without this break, it will output as: + // L1: [1,3, 21,23] + // With this break, it will output as (assuming [11, 15] at L2 is bigger + // than 1/8 of target size): + // L1: [1,3] [21,23] + // Then for the future compactions, [11,15] won't be included. + // For random datasets (either evenly distributed or skewed), it rarely + // triggers this condition, but if the user is adding 2 different datasets + // without any overlap, it may likely happen. + // More details, check PR #1963 + const size_t num_skippable_boundaries_crossed = + being_grandparent_gap_ ? 2 : 3; + if (compaction_->immutable_options()->compaction_style == + kCompactionStyleLevel && + compaction_->immutable_options()->level_compaction_dynamic_file_size && + num_grandparent_boundaries_crossed >= + num_skippable_boundaries_crossed && + grandparent_overlapped_bytes_ - previous_overlapped_bytes > + compaction_->target_output_file_size() / 8) { + return true; + } + + // Pre-cut the output file if it's reaching a certain size AND it's at the + // boundary of a grandparent file. It can reduce the future compaction size, + // the cost is having smaller files. + // The pre-cut size threshold is based on how many grandparent boundaries + // it has seen before. Basically, if it has seen no boundary at all, then it + // will pre-cut at 50% target file size. Every boundary it has seen + // increases the threshold by 5%, max at 90%, which it will always cut. + // The idea is based on if it has seen more boundaries before, it will more + // likely to see another boundary (file cutting opportunity) before the + // target file size. The test shows it can generate larger files than a + // static threshold like 75% and has a similar write amplification + // improvement. + if (compaction_->immutable_options()->compaction_style == + kCompactionStyleLevel && + compaction_->immutable_options()->level_compaction_dynamic_file_size && + current_output_file_size_ >= + ((compaction_->target_output_file_size() + 99) / 100) * + (50 + std::min(grandparent_boundary_switched_num_ * 5, + size_t{40}))) { + return true; + } + } + + return false; +} + +Status CompactionOutputs::AddToOutput( + const CompactionIterator& c_iter, + const CompactionFileOpenFunc& open_file_func, + const CompactionFileCloseFunc& close_file_func) { + Status s; + bool is_range_del = c_iter.IsDeleteRangeSentinelKey(); + if (is_range_del && compaction_->bottommost_level()) { + // We don't consider range tombstone for bottommost level since: + // 1. there is no grandparent and hence no overlap to consider + // 2. range tombstone may be dropped at bottommost level. + return s; + } + const Slice& key = c_iter.key(); + if (ShouldStopBefore(c_iter) && HasBuilder()) { + s = close_file_func(*this, c_iter.InputStatus(), key); + if (!s.ok()) { + return s; + } + // reset grandparent information + grandparent_boundary_switched_num_ = 0; + grandparent_overlapped_bytes_ = + GetCurrentKeyGrandparentOverlappedBytes(key); + if (UNLIKELY(is_range_del)) { + // lower bound for this new output file, this is needed as the lower bound + // does not come from the smallest point key in this case. + range_tombstone_lower_bound_.DecodeFrom(key); + } else { + range_tombstone_lower_bound_.Clear(); + } + } + + // Open output file if necessary + if (!HasBuilder()) { + s = open_file_func(*this); + if (!s.ok()) { + return s; + } + } + + // c_iter may emit range deletion keys, so update `last_key_for_partitioner_` + // here before returning below when `is_range_del` is true + if (partitioner_) { + last_key_for_partitioner_.assign(c_iter.user_key().data_, + c_iter.user_key().size_); + } + + if (UNLIKELY(is_range_del)) { + return s; + } + + assert(builder_ != nullptr); + const Slice& value = c_iter.value(); + s = current_output().validator.Add(key, value); + if (!s.ok()) { + return s; + } + builder_->Add(key, value); + + stats_.num_output_records++; + current_output_file_size_ = builder_->EstimatedFileSize(); + + if (blob_garbage_meter_) { + s = blob_garbage_meter_->ProcessOutFlow(key, value); + } + + if (!s.ok()) { + return s; + } + + const ParsedInternalKey& ikey = c_iter.ikey(); + s = current_output().meta.UpdateBoundaries(key, value, ikey.sequence, + ikey.type); + + return s; +} + +namespace { +void SetMaxSeqAndTs(InternalKey& internal_key, const Slice& user_key, + const size_t ts_sz) { + if (ts_sz) { + static constexpr char kTsMax[] = "\xff\xff\xff\xff\xff\xff\xff\xff\xff"; + if (ts_sz <= strlen(kTsMax)) { + internal_key = InternalKey(user_key, kMaxSequenceNumber, + kTypeRangeDeletion, Slice(kTsMax, ts_sz)); + } else { + internal_key = + InternalKey(user_key, kMaxSequenceNumber, kTypeRangeDeletion, + std::string(ts_sz, '\xff')); + } + } else { + internal_key.Set(user_key, kMaxSequenceNumber, kTypeRangeDeletion); + } +} +} // namespace + +Status CompactionOutputs::AddRangeDels( + const Slice* comp_start_user_key, const Slice* comp_end_user_key, + CompactionIterationStats& range_del_out_stats, bool bottommost_level, + const InternalKeyComparator& icmp, SequenceNumber earliest_snapshot, + const Slice& next_table_min_key, const std::string& full_history_ts_low) { + // The following example does not happen since + // CompactionOutput::ShouldStopBefore() always return false for the first + // point key. But we should consider removing this dependency. Suppose for the + // first compaction output file, + // - next_table_min_key.user_key == comp_start_user_key + // - no point key is in the output file + // - there is a range tombstone @seqno to be added that covers + // comp_start_user_key + // Then meta.smallest will be set to comp_start_user_key@seqno + // and meta.largest will be set to comp_start_user_key@kMaxSequenceNumber + // which violates the assumption that meta.smallest should be <= meta.largest. + assert(HasRangeDel()); + FileMetaData& meta = current_output().meta; + const Comparator* ucmp = icmp.user_comparator(); + InternalKey lower_bound_buf, upper_bound_buf; + Slice lower_bound_guard, upper_bound_guard; + std::string smallest_user_key; + const Slice *lower_bound, *upper_bound; + + // We first determine the internal key lower_bound and upper_bound for + // this output file. All and only range tombstones that overlap with + // [lower_bound, upper_bound] should be added to this file. File + // boundaries (meta.smallest/largest) should be updated accordingly when + // extended by range tombstones. + size_t output_size = outputs_.size(); + if (output_size == 1) { + // This is the first file in the subcompaction. + // + // When outputting a range tombstone that spans a subcompaction boundary, + // the files on either side of that boundary need to include that + // boundary's user key. Otherwise, the spanning range tombstone would lose + // coverage. + // + // To achieve this while preventing files from overlapping in internal key + // (an LSM invariant violation), we allow the earlier file to include the + // boundary user key up to `kMaxSequenceNumber,kTypeRangeDeletion`. The + // later file can begin at the boundary user key at the newest key version + // it contains. At this point that version number is unknown since we have + // not processed the range tombstones yet, so permit any version. Same story + // applies to timestamp, and a non-nullptr `comp_start_user_key` should have + // `kMaxTs` here, which similarly permits any timestamp. + if (comp_start_user_key) { + lower_bound_buf.Set(*comp_start_user_key, kMaxSequenceNumber, + kTypeRangeDeletion); + lower_bound_guard = lower_bound_buf.Encode(); + lower_bound = &lower_bound_guard; + } else { + lower_bound = nullptr; + } + } else { + // For subsequent output tables, only include range tombstones from min + // key onwards since the previous file was extended to contain range + // tombstones falling before min key. + if (range_tombstone_lower_bound_.size() > 0) { + assert(meta.smallest.size() == 0 || + icmp.Compare(range_tombstone_lower_bound_, meta.smallest) < 0); + lower_bound_guard = range_tombstone_lower_bound_.Encode(); + } else { + assert(meta.smallest.size() > 0); + lower_bound_guard = meta.smallest.Encode(); + } + lower_bound = &lower_bound_guard; + } + + const size_t ts_sz = ucmp->timestamp_size(); + if (next_table_min_key.empty()) { + // Last file of the subcompaction. + if (comp_end_user_key) { + upper_bound_buf.Set(*comp_end_user_key, kMaxSequenceNumber, + kTypeRangeDeletion); + upper_bound_guard = upper_bound_buf.Encode(); + upper_bound = &upper_bound_guard; + } else { + upper_bound = nullptr; + } + } else { + // There is another file coming whose coverage will begin at + // `next_table_min_key`. The current file needs to extend range tombstone + // coverage through its own keys (through `meta.largest`) and through user + // keys preceding `next_table_min_key`'s user key. + ParsedInternalKey next_table_min_key_parsed; + ParseInternalKey(next_table_min_key, &next_table_min_key_parsed, + false /* log_err_key */) + .PermitUncheckedError(); + assert(next_table_min_key_parsed.sequence < kMaxSequenceNumber); + assert(meta.largest.size() == 0 || + icmp.Compare(meta.largest.Encode(), next_table_min_key) < 0); + assert(!lower_bound || icmp.Compare(*lower_bound, next_table_min_key) <= 0); + if (meta.largest.size() > 0 && + ucmp->EqualWithoutTimestamp(meta.largest.user_key(), + next_table_min_key_parsed.user_key)) { + // Caution: this assumes meta.largest.Encode() lives longer than + // upper_bound, which is only true if meta.largest is never updated. + // This just happens to be the case here since meta.largest serves + // as the upper_bound. + upper_bound_guard = meta.largest.Encode(); + } else { + SetMaxSeqAndTs(upper_bound_buf, next_table_min_key_parsed.user_key, + ts_sz); + upper_bound_guard = upper_bound_buf.Encode(); + } + upper_bound = &upper_bound_guard; + } + if (lower_bound && upper_bound && + icmp.Compare(*lower_bound, *upper_bound) > 0) { + assert(meta.smallest.size() == 0 && + ucmp->EqualWithoutTimestamp(ExtractUserKey(*lower_bound), + ExtractUserKey(*upper_bound))); + // This can only happen when lower_bound have the same user key as + // next_table_min_key and that there is no point key in the current + // compaction output file. + return Status::OK(); + } + // The end key of the subcompaction must be bigger or equal to the upper + // bound. If the end of subcompaction is null or the upper bound is null, + // it means that this file is the last file in the compaction. So there + // will be no overlapping between this file and others. + assert(comp_end_user_key == nullptr || upper_bound == nullptr || + ucmp->CompareWithoutTimestamp(ExtractUserKey(*upper_bound), + *comp_end_user_key) <= 0); + auto it = range_del_agg_->NewIterator(lower_bound, upper_bound); + Slice last_tombstone_start_user_key{}; + bool reached_lower_bound = false; + const ReadOptions read_options(Env::IOActivity::kCompaction); + for (it->SeekToFirst(); it->Valid(); it->Next()) { + auto tombstone = it->Tombstone(); + auto kv = tombstone.Serialize(); + InternalKey tombstone_end = tombstone.SerializeEndKey(); + // TODO: the underlying iterator should support clamping the bounds. + // tombstone_end.Encode is of form user_key@kMaxSeqno + // if it is equal to lower_bound, there is no need to include + // such range tombstone. + if (!reached_lower_bound && lower_bound && + icmp.Compare(tombstone_end.Encode(), *lower_bound) <= 0) { + continue; + } + assert(!lower_bound || + icmp.Compare(*lower_bound, tombstone_end.Encode()) <= 0); + reached_lower_bound = true; + + // Garbage collection for range tombstones. + // If user-defined timestamp is enabled, range tombstones are dropped if + // they are at bottommost_level, below full_history_ts_low and not visible + // in any snapshot. trim_ts_ is passed to the constructor for + // range_del_agg_, and range_del_agg_ internally drops tombstones above + // trim_ts_. + bool consider_drop = + tombstone.seq_ <= earliest_snapshot && + (ts_sz == 0 || + (!full_history_ts_low.empty() && + ucmp->CompareTimestamp(tombstone.ts_, full_history_ts_low) < 0)); + if (consider_drop && bottommost_level) { + // TODO(andrewkr): tombstones that span multiple output files are + // counted for each compaction output file, so lots of double + // counting. + range_del_out_stats.num_range_del_drop_obsolete++; + range_del_out_stats.num_record_drop_obsolete++; + continue; + } + + assert(lower_bound == nullptr || + ucmp->CompareWithoutTimestamp(ExtractUserKey(*lower_bound), + kv.second) < 0); + InternalKey tombstone_start = kv.first; + if (lower_bound && + ucmp->CompareWithoutTimestamp(tombstone_start.user_key(), + ExtractUserKey(*lower_bound)) < 0) { + // This just updates the non-timestamp portion of `tombstone_start`'s user + // key. Ideally there would be a simpler API usage + ParsedInternalKey tombstone_start_parsed; + ParseInternalKey(tombstone_start.Encode(), &tombstone_start_parsed, + false /* log_err_key */) + .PermitUncheckedError(); + // timestamp should be from where sequence number is from, which is from + // tombstone in this case + std::string ts = + tombstone_start_parsed.GetTimestamp(ucmp->timestamp_size()) + .ToString(); + tombstone_start_parsed.user_key = ExtractUserKey(*lower_bound); + tombstone_start.SetFrom(tombstone_start_parsed, ts); + } + if (upper_bound != nullptr && + icmp.Compare(*upper_bound, tombstone_start.Encode()) < 0) { + break; + } + if (lower_bound && + icmp.Compare(tombstone_start.Encode(), *lower_bound) < 0) { + tombstone_start.DecodeFrom(*lower_bound); + } + if (upper_bound && icmp.Compare(*upper_bound, tombstone_end.Encode()) < 0) { + tombstone_end.DecodeFrom(*upper_bound); + } + if (consider_drop && compaction_->KeyRangeNotExistsBeyondOutputLevel( + tombstone_start.user_key(), + tombstone_end.user_key(), &level_ptrs_)) { + range_del_out_stats.num_range_del_drop_obsolete++; + range_del_out_stats.num_record_drop_obsolete++; + continue; + } + // Here we show that *only* range tombstones that overlap with + // [lower_bound, upper_bound] are added to the current file, and + // sanity checking invariants that should hold: + // - [tombstone_start, tombstone_end] overlaps with [lower_bound, + // upper_bound] + // - meta.smallest <= meta.largest + // Corresponding assertions are made, the proof is broken is any of them + // fails. + // TODO: show that *all* range tombstones that overlap with + // [lower_bound, upper_bound] are added. + // TODO: some invariant about boundaries are correctly updated. + // + // Note that `tombstone_start` is updated in the if condition above, we use + // tombstone_start to refer to its initial value, i.e., + // it->Tombstone().first, and use tombstone_start* to refer to its value + // after the update. + // + // To show [lower_bound, upper_bound] overlaps with [tombstone_start, + // tombstone_end]: + // lower_bound <= upper_bound from the if condition right after all + // bounds are initialized. We assume each tombstone fragment has + // start_key.user_key < end_key.user_key, so + // tombstone_start < tombstone_end by + // FragmentedTombstoneIterator::Tombstone(). So these two ranges are both + // non-emtpy. The flag `reached_lower_bound` and the if logic before it + // ensures lower_bound <= tombstone_end. tombstone_start is only updated + // if it has a smaller user_key than lower_bound user_key, so + // tombstone_start <= tombstone_start*. The above if condition implies + // tombstone_start* <= upper_bound. So we have + // tombstone_start <= upper_bound and lower_bound <= tombstone_end + // and the two ranges overlap. + // + // To show meta.smallest <= meta.largest: + // From the implementation of UpdateBoundariesForRange(), it suffices to + // prove that when it is first called in this function, its parameters + // satisfy `start <= end`, where start = max(tombstone_start*, lower_bound) + // and end = min(tombstone_end, upper_bound). From the above proof we have + // lower_bound <= tombstone_end and lower_bound <= upper_bound. We only need + // to show that tombstone_start* <= min(tombstone_end, upper_bound). + // Note that tombstone_start*.user_key = max(tombstone_start.user_key, + // lower_bound.user_key). Assuming tombstone_end always has + // kMaxSequenceNumber and lower_bound.seqno < kMaxSequenceNumber. + // Since lower_bound <= tombstone_end and lower_bound.seqno < + // tombstone_end.seqno (in absolute number order, not internal key order), + // lower_bound.user_key < tombstone_end.user_key. + // Since lower_bound.user_key < tombstone_end.user_key and + // tombstone_start.user_key < tombstone_end.user_key, tombstone_start* < + // tombstone_end. Since tombstone_start* <= upper_bound from the above proof + // and tombstone_start* < tombstone_end, tombstone_start* <= + // min(tombstone_end, upper_bound), so the two ranges overlap. + + // Range tombstone is not supported by output validator yet. + builder_->Add(kv.first.Encode(), kv.second); + assert(icmp.Compare(tombstone_start, tombstone_end) <= 0); + meta.UpdateBoundariesForRange(tombstone_start, tombstone_end, + tombstone.seq_, icmp); + if (!bottommost_level) { + bool start_user_key_changed = + last_tombstone_start_user_key.empty() || + ucmp->CompareWithoutTimestamp(last_tombstone_start_user_key, + it->start_key()) < 0; + last_tombstone_start_user_key = it->start_key(); + if (start_user_key_changed) { + // If tombstone_start >= tombstone_end, then either no key range is + // covered, or that they have the same user key. If they have the same + // user key, then the internal key range should only be within this + // level, and no keys from older levels is covered. + if (ucmp->CompareWithoutTimestamp(tombstone_start.user_key(), + tombstone_end.user_key()) < 0) { + SizeApproximationOptions approx_opts; + approx_opts.files_size_error_margin = 0.1; + auto approximate_covered_size = + compaction_->input_version()->version_set()->ApproximateSize( + approx_opts, read_options, compaction_->input_version(), + tombstone_start.Encode(), tombstone_end.Encode(), + compaction_->output_level() + 1 /* start_level */, + -1 /* end_level */, kCompaction); + meta.compensated_range_deletion_size += approximate_covered_size; + } + } + } + } + return Status::OK(); +} + +void CompactionOutputs::FillFilesToCutForTtl() { + if (compaction_->immutable_options()->compaction_style != + kCompactionStyleLevel || + compaction_->immutable_options()->compaction_pri != + kMinOverlappingRatio || + compaction_->mutable_cf_options()->ttl == 0 || + compaction_->num_input_levels() < 2 || compaction_->bottommost_level()) { + return; + } + + // We define new file with the oldest ancestor time to be younger than 1/4 + // TTL, and an old one to be older than 1/2 TTL time. + int64_t temp_current_time; + auto get_time_status = + compaction_->immutable_options()->clock->GetCurrentTime( + &temp_current_time); + if (!get_time_status.ok()) { + return; + } + + auto current_time = static_cast(temp_current_time); + if (current_time < compaction_->mutable_cf_options()->ttl) { + return; + } + + uint64_t old_age_thres = + current_time - compaction_->mutable_cf_options()->ttl / 2; + const std::vector& olevel = + *(compaction_->inputs(compaction_->num_input_levels() - 1)); + for (FileMetaData* file : olevel) { + // Worth filtering out by start and end? + uint64_t oldest_ancester_time = file->TryGetOldestAncesterTime(); + // We put old files if they are not too small to prevent a flood + // of small files. + if (oldest_ancester_time < old_age_thres && + file->fd.GetFileSize() > + compaction_->mutable_cf_options()->target_file_size_base / 2) { + files_to_cut_for_ttl_.push_back(file); + } + } +} + +CompactionOutputs::CompactionOutputs(const Compaction* compaction, + const bool is_penultimate_level) + : compaction_(compaction), is_penultimate_level_(is_penultimate_level) { + partitioner_ = compaction->output_level() == 0 + ? nullptr + : compaction->CreateSstPartitioner(); + + if (compaction->output_level() != 0) { + FillFilesToCutForTtl(); + } + + level_ptrs_ = std::vector(compaction_->number_levels(), 0); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_outputs.h b/librocksdb-sys/rocksdb/db/compaction/compaction_outputs.h new file mode 100644 index 0000000..6c3e3b6 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_outputs.h @@ -0,0 +1,412 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include "db/blob/blob_garbage_meter.h" +#include "db/compaction/compaction.h" +#include "db/compaction/compaction_iterator.h" +#include "db/internal_stats.h" +#include "db/output_validator.h" + +namespace ROCKSDB_NAMESPACE { + +class CompactionOutputs; +using CompactionFileOpenFunc = std::function; +using CompactionFileCloseFunc = + std::function; + +// Files produced by subcompaction, most of the functions are used by +// compaction_job Open/Close compaction file functions. +class CompactionOutputs { + public: + // compaction output file + struct Output { + Output(FileMetaData&& _meta, const InternalKeyComparator& _icmp, + bool _enable_order_check, bool _enable_hash, bool _finished, + uint64_t precalculated_hash) + : meta(std::move(_meta)), + validator(_icmp, _enable_order_check, _enable_hash, + precalculated_hash), + finished(_finished) {} + FileMetaData meta; + OutputValidator validator; + bool finished; + std::shared_ptr table_properties; + }; + + CompactionOutputs() = delete; + + explicit CompactionOutputs(const Compaction* compaction, + const bool is_penultimate_level); + + // Add generated output to the list + void AddOutput(FileMetaData&& meta, const InternalKeyComparator& icmp, + bool enable_order_check, bool enable_hash, + bool finished = false, uint64_t precalculated_hash = 0) { + outputs_.emplace_back(std::move(meta), icmp, enable_order_check, + enable_hash, finished, precalculated_hash); + } + + // Set new table builder for the current output + void NewBuilder(const TableBuilderOptions& tboptions); + + // Assign a new WritableFileWriter to the current output + void AssignFileWriter(WritableFileWriter* writer) { + file_writer_.reset(writer); + } + + // TODO: Remove it when remote compaction support tiered compaction + void SetTotalBytes(uint64_t bytes) { stats_.bytes_written += bytes; } + void SetNumOutputRecords(uint64_t num) { stats_.num_output_records = num; } + + // TODO: Move the BlobDB builder into CompactionOutputs + const std::vector& GetBlobFileAdditions() const { + if (is_penultimate_level_) { + assert(blob_file_additions_.empty()); + } + return blob_file_additions_; + } + + std::vector* GetBlobFileAdditionsPtr() { + assert(!is_penultimate_level_); + return &blob_file_additions_; + } + + bool HasBlobFileAdditions() const { return !blob_file_additions_.empty(); } + + BlobGarbageMeter* CreateBlobGarbageMeter() { + assert(!is_penultimate_level_); + blob_garbage_meter_ = std::make_unique(); + return blob_garbage_meter_.get(); + } + + BlobGarbageMeter* GetBlobGarbageMeter() const { + if (is_penultimate_level_) { + // blobdb doesn't support per_key_placement yet + assert(blob_garbage_meter_ == nullptr); + return nullptr; + } + return blob_garbage_meter_.get(); + } + + void UpdateBlobStats() { + assert(!is_penultimate_level_); + stats_.num_output_files_blob = blob_file_additions_.size(); + for (const auto& blob : blob_file_additions_) { + stats_.bytes_written_blob += blob.GetTotalBlobBytes(); + } + } + + // Finish the current output file + Status Finish(const Status& intput_status, + const SeqnoToTimeMapping& seqno_time_mapping); + + // Update output table properties from table builder + void UpdateTableProperties() { + current_output().table_properties = + std::make_shared(GetTableProperties()); + } + + IOStatus WriterSyncClose(const Status& intput_status, SystemClock* clock, + Statistics* statistics, bool use_fsync); + + TableProperties GetTableProperties() { + return builder_->GetTableProperties(); + } + + Slice SmallestUserKey() const { + if (!outputs_.empty() && outputs_[0].finished) { + return outputs_[0].meta.smallest.user_key(); + } else { + return Slice{nullptr, 0}; + } + } + + Slice LargestUserKey() const { + if (!outputs_.empty() && outputs_.back().finished) { + return outputs_.back().meta.largest.user_key(); + } else { + return Slice{nullptr, 0}; + } + } + + // In case the last output file is empty, which doesn't need to keep. + void RemoveLastEmptyOutput() { + if (!outputs_.empty() && !outputs_.back().meta.fd.file_size) { + // An error occurred, so ignore the last output. + outputs_.pop_back(); + } + } + + // Remove the last output, for example the last output doesn't have data (no + // entry and no range-dels), but file_size might not be 0, as it has SST + // metadata. + void RemoveLastOutput() { + assert(!outputs_.empty()); + outputs_.pop_back(); + } + + bool HasBuilder() const { return builder_ != nullptr; } + + FileMetaData* GetMetaData() { return ¤t_output().meta; } + + bool HasOutput() const { return !outputs_.empty(); } + + uint64_t NumEntries() const { return builder_->NumEntries(); } + + void ResetBuilder() { + builder_.reset(); + current_output_file_size_ = 0; + } + + // Add range deletions from the range_del_agg_ to the current output file. + // Input parameters, `range_tombstone_lower_bound_` and current output's + // metadata determine the bounds on range deletions to add. Updates output + // file metadata boundary if extended by range tombstones. + // + // @param comp_start_user_key and comp_end_user_key include timestamp if + // user-defined timestamp is enabled. Their timestamp should be max timestamp. + // @param next_table_min_key internal key lower bound for the next compaction + // output. + // @param full_history_ts_low used for range tombstone garbage collection. + Status AddRangeDels(const Slice* comp_start_user_key, + const Slice* comp_end_user_key, + CompactionIterationStats& range_del_out_stats, + bool bottommost_level, const InternalKeyComparator& icmp, + SequenceNumber earliest_snapshot, + const Slice& next_table_min_key, + const std::string& full_history_ts_low); + + // if the outputs have range delete, range delete is also data + bool HasRangeDel() const { + return range_del_agg_ && !range_del_agg_->IsEmpty(); + } + + private: + friend class SubcompactionState; + + void FillFilesToCutForTtl(); + + void SetOutputSlitKey(const std::optional start, + const std::optional end) { + const InternalKeyComparator* icmp = + &compaction_->column_family_data()->internal_comparator(); + + const InternalKey* output_split_key = compaction_->GetOutputSplitKey(); + // Invalid output_split_key indicates that we do not need to split + if (output_split_key != nullptr) { + // We may only split the output when the cursor is in the range. Split + if ((!end.has_value() || + icmp->user_comparator()->Compare( + ExtractUserKey(output_split_key->Encode()), *end) < 0) && + (!start.has_value() || + icmp->user_comparator()->Compare( + ExtractUserKey(output_split_key->Encode()), *start) > 0)) { + local_output_split_key_ = output_split_key; + } + } + } + + // Returns true iff we should stop building the current output + // before processing the current key in compaction iterator. + bool ShouldStopBefore(const CompactionIterator& c_iter); + + void Cleanup() { + if (builder_ != nullptr) { + // May happen if we get a shutdown call in the middle of compaction + builder_->Abandon(); + builder_.reset(); + } + } + + // Updates states related to file cutting for TTL. + // Returns a boolean value indicating whether the current + // compaction output file should be cut before `internal_key`. + // + // @param internal_key the current key to be added to output. + bool UpdateFilesToCutForTTLStates(const Slice& internal_key); + + // update tracked grandparents information like grandparent index, if it's + // in the gap between 2 grandparent files, accumulated grandparent files size + // etc. + // It returns how many boundaries it crosses by including current key. + size_t UpdateGrandparentBoundaryInfo(const Slice& internal_key); + + // helper function to get the overlapped grandparent files size, it's only + // used for calculating the first key's overlap. + uint64_t GetCurrentKeyGrandparentOverlappedBytes( + const Slice& internal_key) const; + + // Add current key from compaction_iterator to the output file. If needed + // close and open new compaction output with the functions provided. + Status AddToOutput(const CompactionIterator& c_iter, + const CompactionFileOpenFunc& open_file_func, + const CompactionFileCloseFunc& close_file_func); + + // Close the current output. `open_file_func` is needed for creating new file + // for range-dels only output file. + Status CloseOutput(const Status& curr_status, + const CompactionFileOpenFunc& open_file_func, + const CompactionFileCloseFunc& close_file_func) { + Status status = curr_status; + // handle subcompaction containing only range deletions + if (status.ok() && !HasBuilder() && !HasOutput() && HasRangeDel()) { + status = open_file_func(*this); + } + if (HasBuilder()) { + const Slice empty_key{}; + Status s = close_file_func(*this, status, empty_key); + if (!s.ok() && status.ok()) { + status = s; + } + } + + return status; + } + + // This subcompaction's output could be empty if compaction was aborted before + // this subcompaction had a chance to generate any output files. When + // subcompactions are executed sequentially this is more likely and will be + // particularly likely for the later subcompactions to be empty. Once they are + // run in parallel however it should be much rarer. + // It's caller's responsibility to make sure it's not empty. + Output& current_output() { + assert(!outputs_.empty()); + return outputs_.back(); + } + + // Assign the range_del_agg to the target output level. There's only one + // range-del-aggregator per compaction outputs, for + // output_to_penultimate_level compaction it is only assigned to the + // penultimate level. + void AssignRangeDelAggregator( + std::unique_ptr&& range_del_agg) { + assert(range_del_agg_ == nullptr); + range_del_agg_ = std::move(range_del_agg); + } + + const Compaction* compaction_; + + // current output builder and writer + std::unique_ptr builder_; + std::unique_ptr file_writer_; + uint64_t current_output_file_size_ = 0; + + // all the compaction outputs so far + std::vector outputs_; + + // BlobDB info + std::vector blob_file_additions_; + std::unique_ptr blob_garbage_meter_; + + // Basic compaction output stats for this level's outputs + InternalStats::CompactionOutputsStats stats_; + + // indicate if this CompactionOutputs obj for penultimate_level, should always + // be false if per_key_placement feature is not enabled. + const bool is_penultimate_level_; + std::unique_ptr range_del_agg_ = nullptr; + + // partitioner information + std::string last_key_for_partitioner_; + std::unique_ptr partitioner_; + + // A flag determines if this subcompaction has been split by the cursor + // for RoundRobin compaction + bool is_split_ = false; + + // We also maintain the output split key for each subcompaction to avoid + // repetitive comparison in ShouldStopBefore() + const InternalKey* local_output_split_key_ = nullptr; + + // Some identified files with old oldest ancester time and the range should be + // isolated out so that the output file(s) in that range can be merged down + // for TTL and clear the timestamps for the range. + std::vector files_to_cut_for_ttl_; + int cur_files_to_cut_for_ttl_ = -1; + int next_files_to_cut_for_ttl_ = 0; + + // An index that used to speed up ShouldStopBefore(). + size_t grandparent_index_ = 0; + + // if the output key is being grandparent files gap, so: + // key > grandparents[grandparent_index_ - 1].largest && + // key < grandparents[grandparent_index_].smallest + bool being_grandparent_gap_ = true; + + // The number of bytes overlapping between the current output and + // grandparent files used in ShouldStopBefore(). + uint64_t grandparent_overlapped_bytes_ = 0; + + // A flag determines whether the key has been seen in ShouldStopBefore() + bool seen_key_ = false; + + // for the current output file, how many file boundaries has it crossed, + // basically number of files overlapped * 2 + size_t grandparent_boundary_switched_num_ = 0; + + // The smallest key of the current output file, this is set when current + // output file's smallest key is a range tombstone start key. + InternalKey range_tombstone_lower_bound_; + + // Used for calls to compaction->KeyRangeNotExistsBeyondOutputLevel() in + // CompactionOutputs::AddRangeDels(). + // level_ptrs_[i] holds index of the file that was checked during the last + // call to compaction->KeyRangeNotExistsBeyondOutputLevel(). This allows + // future calls to the function to pick up where it left off, since each + // range tombstone added to output file within each subcompaction is in + // increasing key range. + std::vector level_ptrs_; +}; + +// helper struct to concatenate the last level and penultimate level outputs +// which could be replaced by std::ranges::join_view() in c++20 +struct OutputIterator { + public: + explicit OutputIterator(const std::vector& a, + const std::vector& b) + : a_(a), b_(b) { + within_a = !a_.empty(); + idx_ = 0; + } + + OutputIterator begin() { return *this; } + + OutputIterator end() { return *this; } + + size_t size() { return a_.size() + b_.size(); } + + const CompactionOutputs::Output& operator*() const { + return within_a ? a_[idx_] : b_[idx_]; + } + + OutputIterator& operator++() { + idx_++; + if (within_a && idx_ >= a_.size()) { + within_a = false; + idx_ = 0; + } + assert(within_a || idx_ <= b_.size()); + return *this; + } + + bool operator!=(const OutputIterator& /*rhs*/) const { + return within_a || idx_ < b_.size(); + } + + private: + const std::vector& a_; + const std::vector& b_; + bool within_a; + size_t idx_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_picker.cc b/librocksdb-sys/rocksdb/db/compaction/compaction_picker.cc new file mode 100644 index 0000000..4d40ab5 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_picker.cc @@ -0,0 +1,1225 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/compaction/compaction_picker.h" + +#include +#include +#include +#include +#include +#include + +#include "db/column_family.h" +#include "file/filename.h" +#include "logging/log_buffer.h" +#include "logging/logging.h" +#include "monitoring/statistics_impl.h" +#include "test_util/sync_point.h" +#include "util/random.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +bool FindIntraL0Compaction(const std::vector& level_files, + size_t min_files_to_compact, + uint64_t max_compact_bytes_per_del_file, + uint64_t max_compaction_bytes, + CompactionInputFiles* comp_inputs) { + TEST_SYNC_POINT("FindIntraL0Compaction"); + + size_t start = 0; + + if (level_files.size() == 0 || level_files[start]->being_compacted) { + return false; + } + + size_t compact_bytes = static_cast(level_files[start]->fd.file_size); + size_t compact_bytes_per_del_file = std::numeric_limits::max(); + // Compaction range will be [start, limit). + size_t limit; + // Pull in files until the amount of compaction work per deleted file begins + // increasing or maximum total compaction size is reached. + size_t new_compact_bytes_per_del_file = 0; + for (limit = start + 1; limit < level_files.size(); ++limit) { + compact_bytes += static_cast(level_files[limit]->fd.file_size); + new_compact_bytes_per_del_file = compact_bytes / (limit - start); + if (level_files[limit]->being_compacted || + new_compact_bytes_per_del_file > compact_bytes_per_del_file || + compact_bytes > max_compaction_bytes) { + break; + } + compact_bytes_per_del_file = new_compact_bytes_per_del_file; + } + + if ((limit - start) >= min_files_to_compact && + compact_bytes_per_del_file < max_compact_bytes_per_del_file) { + assert(comp_inputs != nullptr); + comp_inputs->level = 0; + for (size_t i = start; i < limit; ++i) { + comp_inputs->files.push_back(level_files[i]); + } + return true; + } + return false; +} + +// Determine compression type, based on user options, level of the output +// file and whether compression is disabled. +// If enable_compression is false, then compression is always disabled no +// matter what the values of the other two parameters are. +// Otherwise, the compression type is determined based on options and level. +CompressionType GetCompressionType(const VersionStorageInfo* vstorage, + const MutableCFOptions& mutable_cf_options, + int level, int base_level, + const bool enable_compression) { + if (!enable_compression) { + // disable compression + return kNoCompression; + } + + // If bottommost_compression is set and we are compacting to the + // bottommost level then we should use it. + if (mutable_cf_options.bottommost_compression != kDisableCompressionOption && + level >= (vstorage->num_non_empty_levels() - 1)) { + return mutable_cf_options.bottommost_compression; + } + // If the user has specified a different compression level for each level, + // then pick the compression for that level. + if (!mutable_cf_options.compression_per_level.empty()) { + assert(level == 0 || level >= base_level); + int idx = (level == 0) ? 0 : level - base_level + 1; + + const int n = + static_cast(mutable_cf_options.compression_per_level.size()) - 1; + // It is possible for level_ to be -1; in that case, we use level + // 0's compression. This occurs mostly in backwards compatibility + // situations when the builder doesn't know what level the file + // belongs to. Likewise, if level is beyond the end of the + // specified compression levels, use the last value. + return mutable_cf_options + .compression_per_level[std::max(0, std::min(idx, n))]; + } else { + return mutable_cf_options.compression; + } +} + +CompressionOptions GetCompressionOptions(const MutableCFOptions& cf_options, + const VersionStorageInfo* vstorage, + int level, + const bool enable_compression) { + if (!enable_compression) { + return cf_options.compression_opts; + } + // If bottommost_compression_opts is enabled and we are compacting to the + // bottommost level then we should use the specified compression options. + if (level >= (vstorage->num_non_empty_levels() - 1) && + cf_options.bottommost_compression_opts.enabled) { + return cf_options.bottommost_compression_opts; + } + return cf_options.compression_opts; +} + +CompactionPicker::CompactionPicker(const ImmutableOptions& ioptions, + const InternalKeyComparator* icmp) + : ioptions_(ioptions), icmp_(icmp) {} + +CompactionPicker::~CompactionPicker() {} + +// Delete this compaction from the list of running compactions. +void CompactionPicker::ReleaseCompactionFiles(Compaction* c, Status status) { + UnregisterCompaction(c); + if (!status.ok()) { + c->ResetNextCompactionIndex(); + } +} + +void CompactionPicker::GetRange(const CompactionInputFiles& inputs, + InternalKey* smallest, + InternalKey* largest) const { + const int level = inputs.level; + assert(!inputs.empty()); + smallest->Clear(); + largest->Clear(); + + if (level == 0) { + for (size_t i = 0; i < inputs.size(); i++) { + FileMetaData* f = inputs[i]; + if (i == 0) { + *smallest = f->smallest; + *largest = f->largest; + } else { + if (icmp_->Compare(f->smallest, *smallest) < 0) { + *smallest = f->smallest; + } + if (icmp_->Compare(f->largest, *largest) > 0) { + *largest = f->largest; + } + } + } + } else { + *smallest = inputs[0]->smallest; + *largest = inputs[inputs.size() - 1]->largest; + } +} + +void CompactionPicker::GetRange(const CompactionInputFiles& inputs1, + const CompactionInputFiles& inputs2, + InternalKey* smallest, + InternalKey* largest) const { + assert(!inputs1.empty() || !inputs2.empty()); + if (inputs1.empty()) { + GetRange(inputs2, smallest, largest); + } else if (inputs2.empty()) { + GetRange(inputs1, smallest, largest); + } else { + InternalKey smallest1, smallest2, largest1, largest2; + GetRange(inputs1, &smallest1, &largest1); + GetRange(inputs2, &smallest2, &largest2); + *smallest = + icmp_->Compare(smallest1, smallest2) < 0 ? smallest1 : smallest2; + *largest = icmp_->Compare(largest1, largest2) < 0 ? largest2 : largest1; + } +} + +void CompactionPicker::GetRange(const std::vector& inputs, + InternalKey* smallest, InternalKey* largest, + int exclude_level) const { + InternalKey current_smallest; + InternalKey current_largest; + bool initialized = false; + for (const auto& in : inputs) { + if (in.empty() || in.level == exclude_level) { + continue; + } + GetRange(in, ¤t_smallest, ¤t_largest); + if (!initialized) { + *smallest = current_smallest; + *largest = current_largest; + initialized = true; + } else { + if (icmp_->Compare(current_smallest, *smallest) < 0) { + *smallest = current_smallest; + } + if (icmp_->Compare(current_largest, *largest) > 0) { + *largest = current_largest; + } + } + } + assert(initialized); +} + +bool CompactionPicker::ExpandInputsToCleanCut(const std::string& /*cf_name*/, + VersionStorageInfo* vstorage, + CompactionInputFiles* inputs, + InternalKey** next_smallest) { + // This isn't good compaction + assert(!inputs->empty()); + + const int level = inputs->level; + // GetOverlappingInputs will always do the right thing for level-0. + // So we don't need to do any expansion if level == 0. + if (level == 0) { + return true; + } + + InternalKey smallest, largest; + + // Keep expanding inputs until we are sure that there is a "clean cut" + // boundary between the files in input and the surrounding files. + // This will ensure that no parts of a key are lost during compaction. + int hint_index = -1; + size_t old_size; + do { + old_size = inputs->size(); + GetRange(*inputs, &smallest, &largest); + inputs->clear(); + vstorage->GetOverlappingInputs(level, &smallest, &largest, &inputs->files, + hint_index, &hint_index, true, + next_smallest); + } while (inputs->size() > old_size); + + // we started off with inputs non-empty and the previous loop only grew + // inputs. thus, inputs should be non-empty here + assert(!inputs->empty()); + + // If, after the expansion, there are files that are already under + // compaction, then we must drop/cancel this compaction. + if (AreFilesInCompaction(inputs->files)) { + return false; + } + return true; +} + +bool CompactionPicker::RangeOverlapWithCompaction( + const Slice& smallest_user_key, const Slice& largest_user_key, + int level) const { + const Comparator* ucmp = icmp_->user_comparator(); + for (Compaction* c : compactions_in_progress_) { + if (c->output_level() == level && + ucmp->CompareWithoutTimestamp(smallest_user_key, + c->GetLargestUserKey()) <= 0 && + ucmp->CompareWithoutTimestamp(largest_user_key, + c->GetSmallestUserKey()) >= 0) { + // Overlap + return true; + } + if (c->SupportsPerKeyPlacement()) { + if (c->OverlapPenultimateLevelOutputRange(smallest_user_key, + largest_user_key)) { + return true; + } + } + } + // Did not overlap with any running compaction in level `level` + return false; +} + +bool CompactionPicker::FilesRangeOverlapWithCompaction( + const std::vector& inputs, int level, + int penultimate_level) const { + bool is_empty = true; + for (auto& in : inputs) { + if (!in.empty()) { + is_empty = false; + break; + } + } + if (is_empty) { + // No files in inputs + return false; + } + + // TODO: Intra L0 compactions can have the ranges overlapped, but the input + // files cannot be overlapped in the order of L0 files. + InternalKey smallest, largest; + GetRange(inputs, &smallest, &largest, Compaction::kInvalidLevel); + if (penultimate_level != Compaction::kInvalidLevel) { + if (ioptions_.compaction_style == kCompactionStyleUniversal) { + if (RangeOverlapWithCompaction(smallest.user_key(), largest.user_key(), + penultimate_level)) { + return true; + } + } else { + InternalKey penultimate_smallest, penultimate_largest; + GetRange(inputs, &penultimate_smallest, &penultimate_largest, level); + if (RangeOverlapWithCompaction(penultimate_smallest.user_key(), + penultimate_largest.user_key(), + penultimate_level)) { + return true; + } + } + } + + return RangeOverlapWithCompaction(smallest.user_key(), largest.user_key(), + level); +} + +// Returns true if any one of specified files are being compacted +bool CompactionPicker::AreFilesInCompaction( + const std::vector& files) { + for (size_t i = 0; i < files.size(); i++) { + if (files[i]->being_compacted) { + return true; + } + } + return false; +} + +Compaction* CompactionPicker::CompactFiles( + const CompactionOptions& compact_options, + const std::vector& input_files, int output_level, + VersionStorageInfo* vstorage, const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, uint32_t output_path_id) { +#ifndef NDEBUG + assert(input_files.size()); + // This compaction output should not overlap with a running compaction as + // `SanitizeCompactionInputFiles` should've checked earlier and db mutex + // shouldn't have been released since. + int start_level = Compaction::kInvalidLevel; + for (const auto& in : input_files) { + // input_files should already be sorted by level + if (!in.empty()) { + start_level = in.level; + break; + } + } + assert(output_level == 0 || + !FilesRangeOverlapWithCompaction( + input_files, output_level, + Compaction::EvaluatePenultimateLevel(vstorage, ioptions_, + start_level, output_level))); +#endif /* !NDEBUG */ + + CompressionType compression_type; + if (compact_options.compression == kDisableCompressionOption) { + int base_level; + if (ioptions_.compaction_style == kCompactionStyleLevel) { + base_level = vstorage->base_level(); + } else { + base_level = 1; + } + compression_type = GetCompressionType(vstorage, mutable_cf_options, + output_level, base_level); + } else { + // TODO(ajkr): `CompactionOptions` offers configurable `CompressionType` + // without configurable `CompressionOptions`, which is inconsistent. + compression_type = compact_options.compression; + } + auto c = new Compaction( + vstorage, ioptions_, mutable_cf_options, mutable_db_options, input_files, + output_level, compact_options.output_file_size_limit, + mutable_cf_options.max_compaction_bytes, output_path_id, compression_type, + GetCompressionOptions(mutable_cf_options, vstorage, output_level), + Temperature::kUnknown, compact_options.max_subcompactions, + /* grandparents */ {}, true); + RegisterCompaction(c); + return c; +} + +Status CompactionPicker::GetCompactionInputsFromFileNumbers( + std::vector* input_files, + std::unordered_set* input_set, const VersionStorageInfo* vstorage, + const CompactionOptions& /*compact_options*/) const { + if (input_set->size() == 0U) { + return Status::InvalidArgument( + "Compaction must include at least one file."); + } + assert(input_files); + + std::vector matched_input_files; + matched_input_files.resize(vstorage->num_levels()); + int first_non_empty_level = -1; + int last_non_empty_level = -1; + // TODO(yhchiang): use a lazy-initialized mapping from + // file_number to FileMetaData in Version. + for (int level = 0; level < vstorage->num_levels(); ++level) { + for (auto file : vstorage->LevelFiles(level)) { + auto iter = input_set->find(file->fd.GetNumber()); + if (iter != input_set->end()) { + matched_input_files[level].files.push_back(file); + input_set->erase(iter); + last_non_empty_level = level; + if (first_non_empty_level == -1) { + first_non_empty_level = level; + } + } + } + } + + if (!input_set->empty()) { + std::string message( + "Cannot find matched SST files for the following file numbers:"); + for (auto fn : *input_set) { + message += " "; + message += std::to_string(fn); + } + return Status::InvalidArgument(message); + } + + for (int level = first_non_empty_level; level <= last_non_empty_level; + ++level) { + matched_input_files[level].level = level; + input_files->emplace_back(std::move(matched_input_files[level])); + } + + return Status::OK(); +} + +// Returns true if any one of the parent files are being compacted +bool CompactionPicker::IsRangeInCompaction(VersionStorageInfo* vstorage, + const InternalKey* smallest, + const InternalKey* largest, + int level, int* level_index) { + std::vector inputs; + assert(level < NumberLevels()); + + vstorage->GetOverlappingInputs(level, smallest, largest, &inputs, + level_index ? *level_index : 0, level_index); + return AreFilesInCompaction(inputs); +} + +// Populates the set of inputs of all other levels that overlap with the +// start level. +// Now we assume all levels except start level and output level are empty. +// Will also attempt to expand "start level" if that doesn't expand +// "output level" or cause "level" to include a file for compaction that has an +// overlapping user-key with another file. +// REQUIRES: input_level and output_level are different +// REQUIRES: inputs->empty() == false +// Returns false if files on parent level are currently in compaction, which +// means that we can't compact them +bool CompactionPicker::SetupOtherInputs( + const std::string& cf_name, const MutableCFOptions& mutable_cf_options, + VersionStorageInfo* vstorage, CompactionInputFiles* inputs, + CompactionInputFiles* output_level_inputs, int* parent_index, + int base_index, bool only_expand_towards_right) { + assert(!inputs->empty()); + assert(output_level_inputs->empty()); + const int input_level = inputs->level; + const int output_level = output_level_inputs->level; + if (input_level == output_level) { + // no possibility of conflict + return true; + } + + // For now, we only support merging two levels, start level and output level. + // We need to assert other levels are empty. + for (int l = input_level + 1; l < output_level; l++) { + assert(vstorage->NumLevelFiles(l) == 0); + } + + InternalKey smallest, largest; + + // Get the range one last time. + GetRange(*inputs, &smallest, &largest); + + // Populate the set of next-level files (inputs_GetOutputLevelInputs()) to + // include in compaction + vstorage->GetOverlappingInputs(output_level, &smallest, &largest, + &output_level_inputs->files, *parent_index, + parent_index); + if (AreFilesInCompaction(output_level_inputs->files)) { + return false; + } + if (!output_level_inputs->empty()) { + if (!ExpandInputsToCleanCut(cf_name, vstorage, output_level_inputs)) { + return false; + } + } + + // See if we can further grow the number of inputs in "level" without + // changing the number of "level+1" files we pick up. We also choose NOT + // to expand if this would cause "level" to include some entries for some + // user key, while excluding other entries for the same user key. This + // can happen when one user key spans multiple files. + if (!output_level_inputs->empty()) { + const uint64_t limit = mutable_cf_options.max_compaction_bytes; + const uint64_t output_level_inputs_size = + TotalFileSize(output_level_inputs->files); + const uint64_t inputs_size = TotalFileSize(inputs->files); + bool expand_inputs = false; + + CompactionInputFiles expanded_inputs; + expanded_inputs.level = input_level; + // Get closed interval of output level + InternalKey all_start, all_limit; + GetRange(*inputs, *output_level_inputs, &all_start, &all_limit); + bool try_overlapping_inputs = true; + if (only_expand_towards_right) { + // Round-robin compaction only allows expansion towards the larger side. + vstorage->GetOverlappingInputs(input_level, &smallest, &all_limit, + &expanded_inputs.files, base_index, + nullptr); + } else { + vstorage->GetOverlappingInputs(input_level, &all_start, &all_limit, + &expanded_inputs.files, base_index, + nullptr); + } + uint64_t expanded_inputs_size = TotalFileSize(expanded_inputs.files); + if (!ExpandInputsToCleanCut(cf_name, vstorage, &expanded_inputs)) { + try_overlapping_inputs = false; + } + if (try_overlapping_inputs && expanded_inputs.size() > inputs->size() && + (mutable_cf_options.ignore_max_compaction_bytes_for_input || + output_level_inputs_size + expanded_inputs_size < limit) && + !AreFilesInCompaction(expanded_inputs.files)) { + InternalKey new_start, new_limit; + GetRange(expanded_inputs, &new_start, &new_limit); + CompactionInputFiles expanded_output_level_inputs; + expanded_output_level_inputs.level = output_level; + vstorage->GetOverlappingInputs(output_level, &new_start, &new_limit, + &expanded_output_level_inputs.files, + *parent_index, parent_index); + assert(!expanded_output_level_inputs.empty()); + if (!AreFilesInCompaction(expanded_output_level_inputs.files) && + ExpandInputsToCleanCut(cf_name, vstorage, + &expanded_output_level_inputs) && + expanded_output_level_inputs.size() == output_level_inputs->size()) { + expand_inputs = true; + } + } + if (!expand_inputs) { + vstorage->GetCleanInputsWithinInterval(input_level, &all_start, + &all_limit, &expanded_inputs.files, + base_index, nullptr); + expanded_inputs_size = TotalFileSize(expanded_inputs.files); + if (expanded_inputs.size() > inputs->size() && + (mutable_cf_options.ignore_max_compaction_bytes_for_input || + output_level_inputs_size + expanded_inputs_size < limit) && + !AreFilesInCompaction(expanded_inputs.files)) { + expand_inputs = true; + } + } + if (expand_inputs) { + ROCKS_LOG_INFO(ioptions_.logger, + "[%s] Expanding@%d %" ROCKSDB_PRIszt "+%" ROCKSDB_PRIszt + "(%" PRIu64 "+%" PRIu64 " bytes) to %" ROCKSDB_PRIszt + "+%" ROCKSDB_PRIszt " (%" PRIu64 "+%" PRIu64 " bytes)\n", + cf_name.c_str(), input_level, inputs->size(), + output_level_inputs->size(), inputs_size, + output_level_inputs_size, expanded_inputs.size(), + output_level_inputs->size(), expanded_inputs_size, + output_level_inputs_size); + inputs->files = expanded_inputs.files; + } + } else { + // Likely to be trivial move. Expand files if they are still trivial moves, + // but limit to mutable_cf_options.max_compaction_bytes or 8 files so that + // we don't create too much compaction pressure for the next level. + } + return true; +} + +void CompactionPicker::GetGrandparents( + VersionStorageInfo* vstorage, const CompactionInputFiles& inputs, + const CompactionInputFiles& output_level_inputs, + std::vector* grandparents) { + InternalKey start, limit; + GetRange(inputs, output_level_inputs, &start, &limit); + // Compute the set of grandparent files that overlap this compaction + // (parent == level+1; grandparent == level+2 or the first + // level after that has overlapping files) + for (int level = output_level_inputs.level + 1; level < NumberLevels(); + level++) { + vstorage->GetOverlappingInputs(level, &start, &limit, grandparents); + if (!grandparents->empty()) { + break; + } + } +} + +Compaction* CompactionPicker::CompactRange( + const std::string& cf_name, const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage, + int input_level, int output_level, + const CompactRangeOptions& compact_range_options, const InternalKey* begin, + const InternalKey* end, InternalKey** compaction_end, bool* manual_conflict, + uint64_t max_file_num_to_ignore, const std::string& trim_ts) { + // CompactionPickerFIFO has its own implementation of compact range + assert(ioptions_.compaction_style != kCompactionStyleFIFO); + + if (input_level == ColumnFamilyData::kCompactAllLevels) { + assert(ioptions_.compaction_style == kCompactionStyleUniversal); + + // Universal compaction with more than one level always compacts all the + // files together to the last level. + assert(vstorage->num_levels() > 1); + int max_output_level = + vstorage->MaxOutputLevel(ioptions_.allow_ingest_behind); + // DBImpl::CompactRange() set output level to be the last level + assert(output_level == max_output_level); + // DBImpl::RunManualCompaction will make full range for universal compaction + assert(begin == nullptr); + assert(end == nullptr); + *compaction_end = nullptr; + + int start_level = 0; + for (; start_level <= max_output_level && + vstorage->NumLevelFiles(start_level) == 0; + start_level++) { + } + if (start_level > max_output_level) { + return nullptr; + } + + if ((start_level == 0) && (!level0_compactions_in_progress_.empty())) { + *manual_conflict = true; + // Only one level 0 compaction allowed + return nullptr; + } + + std::vector inputs(max_output_level + 1 - + start_level); + for (int level = start_level; level <= max_output_level; level++) { + inputs[level - start_level].level = level; + auto& files = inputs[level - start_level].files; + for (FileMetaData* f : vstorage->LevelFiles(level)) { + files.push_back(f); + } + if (AreFilesInCompaction(files)) { + *manual_conflict = true; + return nullptr; + } + } + + // 2 non-exclusive manual compactions could run at the same time producing + // overlaping outputs in the same level. + if (FilesRangeOverlapWithCompaction( + inputs, output_level, + Compaction::EvaluatePenultimateLevel(vstorage, ioptions_, + start_level, output_level))) { + // This compaction output could potentially conflict with the output + // of a currently running compaction, we cannot run it. + *manual_conflict = true; + return nullptr; + } + + Compaction* c = new Compaction( + vstorage, ioptions_, mutable_cf_options, mutable_db_options, + std::move(inputs), output_level, + MaxFileSizeForLevel(mutable_cf_options, output_level, + ioptions_.compaction_style), + /* max_compaction_bytes */ LLONG_MAX, + compact_range_options.target_path_id, + GetCompressionType(vstorage, mutable_cf_options, output_level, 1), + GetCompressionOptions(mutable_cf_options, vstorage, output_level), + Temperature::kUnknown, compact_range_options.max_subcompactions, + /* grandparents */ {}, /* is manual */ true, trim_ts, /* score */ -1, + /* deletion_compaction */ false, /* l0_files_might_overlap */ true, + CompactionReason::kUnknown, + compact_range_options.blob_garbage_collection_policy, + compact_range_options.blob_garbage_collection_age_cutoff); + + RegisterCompaction(c); + vstorage->ComputeCompactionScore(ioptions_, mutable_cf_options); + return c; + } + + CompactionInputFiles inputs; + inputs.level = input_level; + bool covering_the_whole_range = true; + + // All files are 'overlapping' in universal style compaction. + // We have to compact the entire range in one shot. + if (ioptions_.compaction_style == kCompactionStyleUniversal) { + begin = nullptr; + end = nullptr; + } + + vstorage->GetOverlappingInputs(input_level, begin, end, &inputs.files); + if (inputs.empty()) { + return nullptr; + } + + if ((input_level == 0) && (!level0_compactions_in_progress_.empty())) { + // Only one level 0 compaction allowed + TEST_SYNC_POINT("CompactionPicker::CompactRange:Conflict"); + *manual_conflict = true; + return nullptr; + } + + // Avoid compacting too much in one shot in case the range is large. + // But we cannot do this for level-0 since level-0 files can overlap + // and we must not pick one file and drop another older file if the + // two files overlap. + if (input_level > 0) { + const uint64_t limit = mutable_cf_options.max_compaction_bytes; + uint64_t input_level_total = 0; + int hint_index = -1; + InternalKey* smallest = nullptr; + InternalKey* largest = nullptr; + for (size_t i = 0; i + 1 < inputs.size(); ++i) { + if (!smallest) { + smallest = &inputs[i]->smallest; + } + largest = &inputs[i]->largest; + + uint64_t input_file_size = inputs[i]->fd.GetFileSize(); + uint64_t output_level_total = 0; + if (output_level < vstorage->num_non_empty_levels()) { + std::vector files; + vstorage->GetOverlappingInputsRangeBinarySearch( + output_level, smallest, largest, &files, hint_index, &hint_index); + for (const auto& file : files) { + output_level_total += file->fd.GetFileSize(); + } + } + + input_level_total += input_file_size; + + if (input_level_total + output_level_total >= limit) { + covering_the_whole_range = false; + // still include the current file, so the compaction could be larger + // than max_compaction_bytes, which is also to make sure the compaction + // can make progress even `max_compaction_bytes` is small (e.g. smaller + // than an SST file). + inputs.files.resize(i + 1); + break; + } + } + } + + assert(compact_range_options.target_path_id < + static_cast(ioptions_.cf_paths.size())); + + // for BOTTOM LEVEL compaction only, use max_file_num_to_ignore to filter out + // files that are created during the current compaction. + if ((compact_range_options.bottommost_level_compaction == + BottommostLevelCompaction::kForceOptimized || + compact_range_options.bottommost_level_compaction == + BottommostLevelCompaction::kIfHaveCompactionFilter) && + max_file_num_to_ignore != std::numeric_limits::max()) { + assert(input_level == output_level); + // inputs_shrunk holds a continuous subset of input files which were all + // created before the current manual compaction + std::vector inputs_shrunk; + size_t skip_input_index = inputs.size(); + for (size_t i = 0; i < inputs.size(); ++i) { + if (inputs[i]->fd.GetNumber() < max_file_num_to_ignore) { + inputs_shrunk.push_back(inputs[i]); + } else if (!inputs_shrunk.empty()) { + // inputs[i] was created during the current manual compaction and + // need to be skipped + skip_input_index = i; + break; + } + } + if (inputs_shrunk.empty()) { + return nullptr; + } + if (inputs.size() != inputs_shrunk.size()) { + inputs.files.swap(inputs_shrunk); + } + // set covering_the_whole_range to false if there is any file that need to + // be compacted in the range of inputs[skip_input_index+1, inputs.size()) + for (size_t i = skip_input_index + 1; i < inputs.size(); ++i) { + if (inputs[i]->fd.GetNumber() < max_file_num_to_ignore) { + covering_the_whole_range = false; + } + } + } + + InternalKey key_storage; + InternalKey* next_smallest = &key_storage; + if (ExpandInputsToCleanCut(cf_name, vstorage, &inputs, &next_smallest) == + false) { + // manual compaction is now multi-threaded, so it can + // happen that ExpandWhileOverlapping fails + // we handle it higher in RunManualCompaction + *manual_conflict = true; + return nullptr; + } + + if (covering_the_whole_range || !next_smallest) { + *compaction_end = nullptr; + } else { + **compaction_end = *next_smallest; + } + + CompactionInputFiles output_level_inputs; + if (output_level == ColumnFamilyData::kCompactToBaseLevel) { + assert(input_level == 0); + output_level = vstorage->base_level(); + assert(output_level > 0); + } + output_level_inputs.level = output_level; + if (input_level != output_level) { + int parent_index = -1; + if (!SetupOtherInputs(cf_name, mutable_cf_options, vstorage, &inputs, + &output_level_inputs, &parent_index, -1)) { + // manual compaction is now multi-threaded, so it can + // happen that SetupOtherInputs fails + // we handle it higher in RunManualCompaction + *manual_conflict = true; + return nullptr; + } + } + + std::vector compaction_inputs({inputs}); + if (!output_level_inputs.empty()) { + compaction_inputs.push_back(output_level_inputs); + } + for (size_t i = 0; i < compaction_inputs.size(); i++) { + if (AreFilesInCompaction(compaction_inputs[i].files)) { + *manual_conflict = true; + return nullptr; + } + } + + // 2 non-exclusive manual compactions could run at the same time producing + // overlaping outputs in the same level. + if (FilesRangeOverlapWithCompaction( + compaction_inputs, output_level, + Compaction::EvaluatePenultimateLevel(vstorage, ioptions_, input_level, + output_level))) { + // This compaction output could potentially conflict with the output + // of a currently running compaction, we cannot run it. + *manual_conflict = true; + return nullptr; + } + + std::vector grandparents; + GetGrandparents(vstorage, inputs, output_level_inputs, &grandparents); + Compaction* compaction = new Compaction( + vstorage, ioptions_, mutable_cf_options, mutable_db_options, + std::move(compaction_inputs), output_level, + MaxFileSizeForLevel(mutable_cf_options, output_level, + ioptions_.compaction_style, vstorage->base_level(), + ioptions_.level_compaction_dynamic_level_bytes), + mutable_cf_options.max_compaction_bytes, + compact_range_options.target_path_id, + GetCompressionType(vstorage, mutable_cf_options, output_level, + vstorage->base_level()), + GetCompressionOptions(mutable_cf_options, vstorage, output_level), + Temperature::kUnknown, compact_range_options.max_subcompactions, + std::move(grandparents), /* is manual */ true, trim_ts, /* score */ -1, + /* deletion_compaction */ false, /* l0_files_might_overlap */ true, + CompactionReason::kUnknown, + compact_range_options.blob_garbage_collection_policy, + compact_range_options.blob_garbage_collection_age_cutoff); + + TEST_SYNC_POINT_CALLBACK("CompactionPicker::CompactRange:Return", compaction); + RegisterCompaction(compaction); + + // Creating a compaction influences the compaction score because the score + // takes running compactions into account (by skipping files that are already + // being compacted). Since we just changed compaction score, we recalculate it + // here + vstorage->ComputeCompactionScore(ioptions_, mutable_cf_options); + + return compaction; +} + +namespace { +// Test whether two files have overlapping key-ranges. +bool HaveOverlappingKeyRanges(const Comparator* c, const SstFileMetaData& a, + const SstFileMetaData& b) { + if (c->CompareWithoutTimestamp(a.smallestkey, b.smallestkey) >= 0) { + if (c->CompareWithoutTimestamp(a.smallestkey, b.largestkey) <= 0) { + // b.smallestkey <= a.smallestkey <= b.largestkey + return true; + } + } else if (c->CompareWithoutTimestamp(a.largestkey, b.smallestkey) >= 0) { + // a.smallestkey < b.smallestkey <= a.largestkey + return true; + } + if (c->CompareWithoutTimestamp(a.largestkey, b.largestkey) <= 0) { + if (c->CompareWithoutTimestamp(a.largestkey, b.smallestkey) >= 0) { + // b.smallestkey <= a.largestkey <= b.largestkey + return true; + } + } else if (c->CompareWithoutTimestamp(a.smallestkey, b.largestkey) <= 0) { + // a.smallestkey <= b.largestkey < a.largestkey + return true; + } + return false; +} +} // namespace + +Status CompactionPicker::SanitizeCompactionInputFilesForAllLevels( + std::unordered_set* input_files, + const ColumnFamilyMetaData& cf_meta, const int output_level) const { + auto& levels = cf_meta.levels; + auto comparator = icmp_->user_comparator(); + + // TODO(yhchiang): add is_adjustable to CompactionOptions + + // the smallest and largest key of the current compaction input + std::string smallestkey; + std::string largestkey; + // a flag for initializing smallest and largest key + bool is_first = false; + const int kNotFound = -1; + + // For each level, it does the following things: + // 1. Find the first and the last compaction input files + // in the current level. + // 2. Include all files between the first and the last + // compaction input files. + // 3. Update the compaction key-range. + // 4. For all remaining levels, include files that have + // overlapping key-range with the compaction key-range. + for (int l = 0; l <= output_level; ++l) { + auto& current_files = levels[l].files; + int first_included = static_cast(current_files.size()); + int last_included = kNotFound; + + // identify the first and the last compaction input files + // in the current level. + for (size_t f = 0; f < current_files.size(); ++f) { + const uint64_t file_number = TableFileNameToNumber(current_files[f].name); + if (input_files->find(file_number) == input_files->end()) { + continue; + } + first_included = std::min(first_included, static_cast(f)); + last_included = std::max(last_included, static_cast(f)); + if (is_first == false) { + smallestkey = current_files[f].smallestkey; + largestkey = current_files[f].largestkey; + is_first = true; + } + } + if (last_included == kNotFound) { + continue; + } + + if (l != 0) { + // expand the compaction input of the current level if it + // has overlapping key-range with other non-compaction input + // files in the same level. + while (first_included > 0) { + if (comparator->CompareWithoutTimestamp( + current_files[first_included - 1].largestkey, + current_files[first_included].smallestkey) < 0) { + break; + } + first_included--; + } + + while (last_included < static_cast(current_files.size()) - 1) { + if (comparator->CompareWithoutTimestamp( + current_files[last_included + 1].smallestkey, + current_files[last_included].largestkey) > 0) { + break; + } + last_included++; + } + } else if (output_level > 0) { + last_included = static_cast(current_files.size() - 1); + } + + // include all files between the first and the last compaction input files. + for (int f = first_included; f <= last_included; ++f) { + if (current_files[f].being_compacted) { + return Status::Aborted("Necessary compaction input file " + + current_files[f].name + + " is currently being compacted."); + } + + input_files->insert(TableFileNameToNumber(current_files[f].name)); + } + + // update smallest and largest key + if (l == 0) { + for (int f = first_included; f <= last_included; ++f) { + if (comparator->CompareWithoutTimestamp( + smallestkey, current_files[f].smallestkey) > 0) { + smallestkey = current_files[f].smallestkey; + } + if (comparator->CompareWithoutTimestamp( + largestkey, current_files[f].largestkey) < 0) { + largestkey = current_files[f].largestkey; + } + } + } else { + if (comparator->CompareWithoutTimestamp( + smallestkey, current_files[first_included].smallestkey) > 0) { + smallestkey = current_files[first_included].smallestkey; + } + if (comparator->CompareWithoutTimestamp( + largestkey, current_files[last_included].largestkey) < 0) { + largestkey = current_files[last_included].largestkey; + } + } + + SstFileMetaData aggregated_file_meta; + aggregated_file_meta.smallestkey = smallestkey; + aggregated_file_meta.largestkey = largestkey; + + // For all lower levels, include all overlapping files. + // We need to add overlapping files from the current level too because even + // if there no input_files in level l, we would still need to add files + // which overlap with the range containing the input_files in levels 0 to l + // Level 0 doesn't need to be handled this way because files are sorted by + // time and not by key + for (int m = std::max(l, 1); m <= output_level; ++m) { + for (auto& next_lv_file : levels[m].files) { + if (HaveOverlappingKeyRanges(comparator, aggregated_file_meta, + next_lv_file)) { + if (next_lv_file.being_compacted) { + return Status::Aborted( + "File " + next_lv_file.name + + " that has overlapping key range with one of the compaction " + " input file is currently being compacted."); + } + input_files->insert(TableFileNameToNumber(next_lv_file.name)); + } + } + } + } + if (RangeOverlapWithCompaction(smallestkey, largestkey, output_level)) { + return Status::Aborted( + "A running compaction is writing to the same output level in an " + "overlapping key range"); + } + return Status::OK(); +} + +Status CompactionPicker::SanitizeCompactionInputFiles( + std::unordered_set* input_files, + const ColumnFamilyMetaData& cf_meta, const int output_level) const { + assert(static_cast(cf_meta.levels.size()) - 1 == + cf_meta.levels[cf_meta.levels.size() - 1].level); + if (output_level >= static_cast(cf_meta.levels.size())) { + return Status::InvalidArgument( + "Output level for column family " + cf_meta.name + + " must between [0, " + + std::to_string(cf_meta.levels[cf_meta.levels.size() - 1].level) + "]."); + } + + if (output_level > MaxOutputLevel()) { + return Status::InvalidArgument( + "Exceed the maximum output level defined by " + "the current compaction algorithm --- " + + std::to_string(MaxOutputLevel())); + } + + if (output_level < 0) { + return Status::InvalidArgument("Output level cannot be negative."); + } + + if (input_files->size() == 0) { + return Status::InvalidArgument( + "A compaction must contain at least one file."); + } + + Status s = SanitizeCompactionInputFilesForAllLevels(input_files, cf_meta, + output_level); + + if (!s.ok()) { + return s; + } + + // for all input files, check whether the file number matches + // any currently-existing files. + for (auto file_num : *input_files) { + bool found = false; + int input_file_level = -1; + for (const auto& level_meta : cf_meta.levels) { + for (const auto& file_meta : level_meta.files) { + if (file_num == TableFileNameToNumber(file_meta.name)) { + if (file_meta.being_compacted) { + return Status::Aborted("Specified compaction input file " + + MakeTableFileName("", file_num) + + " is already being compacted."); + } + found = true; + input_file_level = level_meta.level; + break; + } + } + if (found) { + break; + } + } + if (!found) { + return Status::InvalidArgument( + "Specified compaction input file " + MakeTableFileName("", file_num) + + " does not exist in column family " + cf_meta.name + "."); + } + if (input_file_level > output_level) { + return Status::InvalidArgument( + "Cannot compact file to up level, input file: " + + MakeTableFileName("", file_num) + " level " + + std::to_string(input_file_level) + " > output level " + + std::to_string(output_level)); + } + } + + return Status::OK(); +} + +void CompactionPicker::RegisterCompaction(Compaction* c) { + if (c == nullptr) { + return; + } + assert(ioptions_.compaction_style != kCompactionStyleLevel || + c->output_level() == 0 || + !FilesRangeOverlapWithCompaction(*c->inputs(), c->output_level(), + c->GetPenultimateLevel())); + // CompactionReason::kExternalSstIngestion's start level is just a placeholder + // number without actual meaning as file ingestion technically does not have + // an input level like other compactions + if ((c->start_level() == 0 && + c->compaction_reason() != CompactionReason::kExternalSstIngestion) || + ioptions_.compaction_style == kCompactionStyleUniversal) { + level0_compactions_in_progress_.insert(c); + } + compactions_in_progress_.insert(c); + TEST_SYNC_POINT_CALLBACK("CompactionPicker::RegisterCompaction:Registered", + c); +} + +void CompactionPicker::UnregisterCompaction(Compaction* c) { + if (c == nullptr) { + return; + } + if (c->start_level() == 0 || + ioptions_.compaction_style == kCompactionStyleUniversal) { + level0_compactions_in_progress_.erase(c); + } + compactions_in_progress_.erase(c); +} + +void CompactionPicker::PickFilesMarkedForCompaction( + const std::string& cf_name, VersionStorageInfo* vstorage, int* start_level, + int* output_level, CompactionInputFiles* start_level_inputs) { + if (vstorage->FilesMarkedForCompaction().empty()) { + return; + } + + auto continuation = [&, cf_name](std::pair level_file) { + // If it's being compacted it has nothing to do here. + // If this assert() fails that means that some function marked some + // files as being_compacted, but didn't call ComputeCompactionScore() + assert(!level_file.second->being_compacted); + *start_level = level_file.first; + *output_level = + (*start_level == 0) ? vstorage->base_level() : *start_level + 1; + + if (*start_level == 0 && !level0_compactions_in_progress()->empty()) { + return false; + } + + start_level_inputs->files = {level_file.second}; + start_level_inputs->level = *start_level; + return ExpandInputsToCleanCut(cf_name, vstorage, start_level_inputs); + }; + + // take a chance on a random file first + Random64 rnd(/* seed */ reinterpret_cast(vstorage)); + size_t random_file_index = static_cast(rnd.Uniform( + static_cast(vstorage->FilesMarkedForCompaction().size()))); + TEST_SYNC_POINT_CALLBACK("CompactionPicker::PickFilesMarkedForCompaction", + &random_file_index); + + if (continuation(vstorage->FilesMarkedForCompaction()[random_file_index])) { + // found the compaction! + return; + } + + for (auto& level_file : vstorage->FilesMarkedForCompaction()) { + if (continuation(level_file)) { + // found the compaction! + return; + } + } + start_level_inputs->files.clear(); +} + +bool CompactionPicker::GetOverlappingL0Files( + VersionStorageInfo* vstorage, CompactionInputFiles* start_level_inputs, + int output_level, int* parent_index) { + // Two level 0 compaction won't run at the same time, so don't need to worry + // about files on level 0 being compacted. + assert(level0_compactions_in_progress()->empty()); + InternalKey smallest, largest; + GetRange(*start_level_inputs, &smallest, &largest); + // Note that the next call will discard the file we placed in + // c->inputs_[0] earlier and replace it with an overlapping set + // which will include the picked file. + start_level_inputs->files.clear(); + vstorage->GetOverlappingInputs(0, &smallest, &largest, + &(start_level_inputs->files)); + + // If we include more L0 files in the same compaction run it can + // cause the 'smallest' and 'largest' key to get extended to a + // larger range. So, re-invoke GetRange to get the new key range + GetRange(*start_level_inputs, &smallest, &largest); + if (IsRangeInCompaction(vstorage, &smallest, &largest, output_level, + parent_index)) { + return false; + } + assert(!start_level_inputs->files.empty()); + + return true; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_picker.h b/librocksdb-sys/rocksdb/db/compaction/compaction_picker.h new file mode 100644 index 0000000..0556e99 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_picker.h @@ -0,0 +1,318 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include +#include +#include +#include +#include + +#include "db/compaction/compaction.h" +#include "db/version_set.h" +#include "options/cf_options.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +// The file contains an abstract class CompactionPicker, and its two +// sub-classes LevelCompactionPicker and NullCompactionPicker, as +// well as some helper functions used by them. + +class LogBuffer; +class Compaction; +class VersionStorageInfo; +struct CompactionInputFiles; + +// An abstract class to pick compactions from an existing LSM-tree. +// +// Each compaction style inherits the class and implement the +// interface to form automatic compactions. If NeedCompaction() is true, +// then call PickCompaction() to find what files need to be compacted +// and where to put the output files. +// +// Non-virtual functions CompactRange() and CompactFiles() are used to +// pick files to compact based on users' DB::CompactRange() and +// DB::CompactFiles() requests, respectively. There is little +// compaction style specific logic for them. +class CompactionPicker { + public: + CompactionPicker(const ImmutableOptions& ioptions, + const InternalKeyComparator* icmp); + virtual ~CompactionPicker(); + + // Pick level and inputs for a new compaction. + // + // Returns nullptr if there is no compaction to be done. + // Otherwise returns a pointer to a heap-allocated object that + // describes the compaction. Caller should delete the result. + virtual Compaction* PickCompaction(const std::string& cf_name, + const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, + VersionStorageInfo* vstorage, + LogBuffer* log_buffer) = 0; + + // Return a compaction object for compacting the range [begin,end] in + // the specified level. Returns nullptr if there is nothing in that + // level that overlaps the specified range. Caller should delete + // the result. + // + // The returned Compaction might not include the whole requested range. + // In that case, compaction_end will be set to the next key that needs + // compacting. In case the compaction will compact the whole range, + // compaction_end will be set to nullptr. + // Client is responsible for compaction_end storage -- when called, + // *compaction_end should point to valid InternalKey! + virtual Compaction* CompactRange( + const std::string& cf_name, const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage, + int input_level, int output_level, + const CompactRangeOptions& compact_range_options, + const InternalKey* begin, const InternalKey* end, + InternalKey** compaction_end, bool* manual_conflict, + uint64_t max_file_num_to_ignore, const std::string& trim_ts); + + // The maximum allowed output level. Default value is NumberLevels() - 1. + virtual int MaxOutputLevel() const { return NumberLevels() - 1; } + + virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const = 0; + +// Sanitize the input set of compaction input files. +// When the input parameters do not describe a valid compaction, the +// function will try to fix the input_files by adding necessary +// files. If it's not possible to conver an invalid input_files +// into a valid one by adding more files, the function will return a +// non-ok status with specific reason. +// + Status SanitizeCompactionInputFiles(std::unordered_set* input_files, + const ColumnFamilyMetaData& cf_meta, + const int output_level) const; + + // Free up the files that participated in a compaction + // + // Requirement: DB mutex held + void ReleaseCompactionFiles(Compaction* c, Status status); + + // Returns true if any one of the specified files are being compacted + bool AreFilesInCompaction(const std::vector& files); + + // Takes a list of CompactionInputFiles and returns a (manual) Compaction + // object. + // + // Caller must provide a set of input files that has been passed through + // `SanitizeCompactionInputFiles` earlier. The lock should not be released + // between that call and this one. + Compaction* CompactFiles(const CompactionOptions& compact_options, + const std::vector& input_files, + int output_level, VersionStorageInfo* vstorage, + const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, + uint32_t output_path_id); + + // Converts a set of compaction input file numbers into + // a list of CompactionInputFiles. + Status GetCompactionInputsFromFileNumbers( + std::vector* input_files, + std::unordered_set* input_set, + const VersionStorageInfo* vstorage, + const CompactionOptions& compact_options) const; + + // Is there currently a compaction involving level 0 taking place + bool IsLevel0CompactionInProgress() const { + return !level0_compactions_in_progress_.empty(); + } + + // Return true if the passed key range overlap with a compaction output + // that is currently running. + bool RangeOverlapWithCompaction(const Slice& smallest_user_key, + const Slice& largest_user_key, + int level) const; + + // Stores the minimal range that covers all entries in inputs in + // *smallest, *largest. + // REQUIRES: inputs is not empty + void GetRange(const CompactionInputFiles& inputs, InternalKey* smallest, + InternalKey* largest) const; + + // Stores the minimal range that covers all entries in inputs1 and inputs2 + // in *smallest, *largest. + // REQUIRES: inputs is not empty + void GetRange(const CompactionInputFiles& inputs1, + const CompactionInputFiles& inputs2, InternalKey* smallest, + InternalKey* largest) const; + + // Stores the minimal range that covers all entries in inputs + // in *smallest, *largest. + // REQUIRES: inputs is not empty (at least on entry have one file) + void GetRange(const std::vector& inputs, + InternalKey* smallest, InternalKey* largest, + int exclude_level) const; + + int NumberLevels() const { return ioptions_.num_levels; } + + // Add more files to the inputs on "level" to make sure that + // no newer version of a key is compacted to "level+1" while leaving an older + // version in a "level". Otherwise, any Get() will search "level" first, + // and will likely return an old/stale value for the key, since it always + // searches in increasing order of level to find the value. This could + // also scramble the order of merge operands. This function should be + // called any time a new Compaction is created, and its inputs_[0] are + // populated. + // + // Will return false if it is impossible to apply this compaction. + bool ExpandInputsToCleanCut(const std::string& cf_name, + VersionStorageInfo* vstorage, + CompactionInputFiles* inputs, + InternalKey** next_smallest = nullptr); + + // Returns true if any one of the parent files are being compacted + bool IsRangeInCompaction(VersionStorageInfo* vstorage, + const InternalKey* smallest, + const InternalKey* largest, int level, int* index); + + // Returns true if the key range that `inputs` files cover overlap with the + // key range of a currently running compaction. + bool FilesRangeOverlapWithCompaction( + const std::vector& inputs, int level, + int penultimate_level) const; + + bool SetupOtherInputs(const std::string& cf_name, + const MutableCFOptions& mutable_cf_options, + VersionStorageInfo* vstorage, + CompactionInputFiles* inputs, + CompactionInputFiles* output_level_inputs, + int* parent_index, int base_index, + bool only_expand_towards_right = false); + + void GetGrandparents(VersionStorageInfo* vstorage, + const CompactionInputFiles& inputs, + const CompactionInputFiles& output_level_inputs, + std::vector* grandparents); + + void PickFilesMarkedForCompaction(const std::string& cf_name, + VersionStorageInfo* vstorage, + int* start_level, int* output_level, + CompactionInputFiles* start_level_inputs); + + bool GetOverlappingL0Files(VersionStorageInfo* vstorage, + CompactionInputFiles* start_level_inputs, + int output_level, int* parent_index); + + // Register this compaction in the set of running compactions + void RegisterCompaction(Compaction* c); + + // Remove this compaction from the set of running compactions + void UnregisterCompaction(Compaction* c); + + std::set* level0_compactions_in_progress() { + return &level0_compactions_in_progress_; + } + std::unordered_set* compactions_in_progress() { + return &compactions_in_progress_; + } + + const InternalKeyComparator* icmp() const { return icmp_; } + + protected: + const ImmutableOptions& ioptions_; + +// A helper function to SanitizeCompactionInputFiles() that +// sanitizes "input_files" by adding necessary files. + virtual Status SanitizeCompactionInputFilesForAllLevels( + std::unordered_set* input_files, + const ColumnFamilyMetaData& cf_meta, const int output_level) const; + + // Keeps track of all compactions that are running on Level0. + // Protected by DB mutex + std::set level0_compactions_in_progress_; + + // Keeps track of all compactions that are running. + // Protected by DB mutex + std::unordered_set compactions_in_progress_; + + const InternalKeyComparator* const icmp_; +}; + +// A dummy compaction that never triggers any automatic +// compaction. +class NullCompactionPicker : public CompactionPicker { + public: + NullCompactionPicker(const ImmutableOptions& ioptions, + const InternalKeyComparator* icmp) + : CompactionPicker(ioptions, icmp) {} + virtual ~NullCompactionPicker() {} + + // Always return "nullptr" + Compaction* PickCompaction(const std::string& /*cf_name*/, + const MutableCFOptions& /*mutable_cf_options*/, + const MutableDBOptions& /*mutable_db_options*/, + VersionStorageInfo* /*vstorage*/, + LogBuffer* /* log_buffer */) override { + return nullptr; + } + + // Always return "nullptr" + Compaction* CompactRange(const std::string& /*cf_name*/, + const MutableCFOptions& /*mutable_cf_options*/, + const MutableDBOptions& /*mutable_db_options*/, + VersionStorageInfo* /*vstorage*/, + int /*input_level*/, int /*output_level*/, + const CompactRangeOptions& /*compact_range_options*/, + const InternalKey* /*begin*/, + const InternalKey* /*end*/, + InternalKey** /*compaction_end*/, + bool* /*manual_conflict*/, + uint64_t /*max_file_num_to_ignore*/, + const std::string& /*trim_ts*/) override { + return nullptr; + } + + // Always returns false. + virtual bool NeedsCompaction( + const VersionStorageInfo* /*vstorage*/) const override { + return false; + } +}; + +// Attempts to find an intra L0 compaction conforming to the given parameters. +// +// @param level_files Metadata for L0 files. +// @param min_files_to_compact Minimum number of files required to +// do the compaction. +// @param max_compact_bytes_per_del_file Maximum average size in bytes per +// file that is going to get deleted by +// the compaction. +// @param max_compaction_bytes Maximum total size in bytes (in terms +// of compensated file size) for files +// to be compacted. +// @param [out] comp_inputs If a compaction was found, will be +// initialized with corresponding input +// files. Cannot be nullptr. +// +// @return true iff compaction was found. +bool FindIntraL0Compaction(const std::vector& level_files, + size_t min_files_to_compact, + uint64_t max_compact_bytes_per_del_file, + uint64_t max_compaction_bytes, + CompactionInputFiles* comp_inputs); + +CompressionType GetCompressionType(const VersionStorageInfo* vstorage, + const MutableCFOptions& mutable_cf_options, + int level, int base_level, + const bool enable_compression = true); + +CompressionOptions GetCompressionOptions( + const MutableCFOptions& mutable_cf_options, + const VersionStorageInfo* vstorage, int level, + const bool enable_compression = true); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_picker_fifo.cc b/librocksdb-sys/rocksdb/db/compaction/compaction_picker_fifo.cc new file mode 100644 index 0000000..9aa2430 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_picker_fifo.cc @@ -0,0 +1,471 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/compaction/compaction_picker_fifo.h" + +#include +#include +#include + +#include "db/column_family.h" +#include "logging/log_buffer.h" +#include "logging/logging.h" +#include "options/options_helper.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { +namespace { +uint64_t GetTotalFilesSize(const std::vector& files) { + uint64_t total_size = 0; + for (const auto& f : files) { + total_size += f->fd.file_size; + } + return total_size; +} +} // anonymous namespace + +bool FIFOCompactionPicker::NeedsCompaction( + const VersionStorageInfo* vstorage) const { + const int kLevel0 = 0; + return vstorage->CompactionScore(kLevel0) >= 1; +} + +Compaction* FIFOCompactionPicker::PickTTLCompaction( + const std::string& cf_name, const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage, + LogBuffer* log_buffer) { + assert(mutable_cf_options.ttl > 0); + + const int kLevel0 = 0; + const std::vector& level_files = vstorage->LevelFiles(kLevel0); + uint64_t total_size = GetTotalFilesSize(level_files); + + int64_t _current_time; + auto status = ioptions_.clock->GetCurrentTime(&_current_time); + if (!status.ok()) { + ROCKS_LOG_BUFFER(log_buffer, + "[%s] FIFO compaction: Couldn't get current time: %s. " + "Not doing compactions based on TTL. ", + cf_name.c_str(), status.ToString().c_str()); + return nullptr; + } + const uint64_t current_time = static_cast(_current_time); + + if (!level0_compactions_in_progress_.empty()) { + ROCKS_LOG_BUFFER( + log_buffer, + "[%s] FIFO compaction: Already executing compaction. No need " + "to run parallel compactions since compactions are very fast", + cf_name.c_str()); + return nullptr; + } + + std::vector inputs; + inputs.emplace_back(); + inputs[0].level = 0; + + // avoid underflow + if (current_time > mutable_cf_options.ttl) { + for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) { + FileMetaData* f = *ritr; + assert(f); + if (f->fd.table_reader && f->fd.table_reader->GetTableProperties()) { + uint64_t creation_time = + f->fd.table_reader->GetTableProperties()->creation_time; + if (creation_time == 0 || + creation_time >= (current_time - mutable_cf_options.ttl)) { + break; + } + } + total_size -= f->fd.file_size; + inputs[0].files.push_back(f); + } + } + + // Return a nullptr and proceed to size-based FIFO compaction if: + // 1. there are no files older than ttl OR + // 2. there are a few files older than ttl, but deleting them will not bring + // the total size to be less than max_table_files_size threshold. + if (inputs[0].files.empty() || + total_size > + mutable_cf_options.compaction_options_fifo.max_table_files_size) { + return nullptr; + } + + for (const auto& f : inputs[0].files) { + uint64_t creation_time = 0; + assert(f); + if (f->fd.table_reader && f->fd.table_reader->GetTableProperties()) { + creation_time = f->fd.table_reader->GetTableProperties()->creation_time; + } + ROCKS_LOG_BUFFER(log_buffer, + "[%s] FIFO compaction: picking file %" PRIu64 + " with creation time %" PRIu64 " for deletion", + cf_name.c_str(), f->fd.GetNumber(), creation_time); + } + + Compaction* c = new Compaction( + vstorage, ioptions_, mutable_cf_options, mutable_db_options, + std::move(inputs), 0, 0, 0, 0, kNoCompression, + mutable_cf_options.compression_opts, Temperature::kUnknown, + /* max_subcompactions */ 0, {}, /* is manual */ false, + /* trim_ts */ "", vstorage->CompactionScore(0), + /* is deletion compaction */ true, /* l0_files_might_overlap */ true, + CompactionReason::kFIFOTtl); + return c; +} + +// The size-based compaction picker for FIFO. +// +// When the entire column family size exceeds max_table_files_size, FIFO will +// try to delete the oldest sst file(s) until the resulting column family size +// is smaller than max_table_files_size. +// +// This function also takes care the case where a DB is migrating from level / +// universal compaction to FIFO compaction. During the migration, the column +// family will also have non-L0 files while FIFO can only create L0 files. +// In this case, this function will first purge the sst files in the bottom- +// most non-empty level first, and the DB will eventually converge to the +// regular FIFO case where there're only L0 files. Note that during the +// migration case, the purge order will only be an approximation of "FIFO" +// as entries inside lower-level files might sometimes be newer than some +// entries inside upper-level files. +Compaction* FIFOCompactionPicker::PickSizeCompaction( + const std::string& cf_name, const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage, + LogBuffer* log_buffer) { + // compute the total size and identify the last non-empty level + int last_level = 0; + uint64_t total_size = 0; + for (int level = 0; level < vstorage->num_levels(); ++level) { + auto level_size = GetTotalFilesSize(vstorage->LevelFiles(level)); + total_size += level_size; + if (level_size > 0) { + last_level = level; + } + } + const std::vector& last_level_files = + vstorage->LevelFiles(last_level); + + if (last_level == 0 && + total_size <= + mutable_cf_options.compaction_options_fifo.max_table_files_size) { + // total size not exceeded, try to find intra level 0 compaction if enabled + const std::vector& level0_files = vstorage->LevelFiles(0); + if (mutable_cf_options.compaction_options_fifo.allow_compaction && + level0_files.size() > 0) { + CompactionInputFiles comp_inputs; + // try to prevent same files from being compacted multiple times, which + // could produce large files that may never TTL-expire. Achieve this by + // disallowing compactions with files larger than memtable (inflate its + // size by 10% to account for uncompressed L0 files that may have size + // slightly greater than memtable size limit). + size_t max_compact_bytes_per_del_file = + static_cast(MultiplyCheckOverflow( + static_cast(mutable_cf_options.write_buffer_size), + 1.1)); + if (FindIntraL0Compaction( + level0_files, + mutable_cf_options + .level0_file_num_compaction_trigger /* min_files_to_compact */ + , + max_compact_bytes_per_del_file, + mutable_cf_options.max_compaction_bytes, &comp_inputs)) { + Compaction* c = new Compaction( + vstorage, ioptions_, mutable_cf_options, mutable_db_options, + {comp_inputs}, 0, 16 * 1024 * 1024 /* output file size limit */, + 0 /* max compaction bytes, not applicable */, + 0 /* output path ID */, mutable_cf_options.compression, + mutable_cf_options.compression_opts, Temperature::kUnknown, + 0 /* max_subcompactions */, {}, /* is manual */ false, + /* trim_ts */ "", vstorage->CompactionScore(0), + /* is deletion compaction */ false, + /* l0_files_might_overlap */ true, + CompactionReason::kFIFOReduceNumFiles); + return c; + } + } + + ROCKS_LOG_BUFFER( + log_buffer, + "[%s] FIFO compaction: nothing to do. Total size %" PRIu64 + ", max size %" PRIu64 "\n", + cf_name.c_str(), total_size, + mutable_cf_options.compaction_options_fifo.max_table_files_size); + return nullptr; + } + + if (!level0_compactions_in_progress_.empty()) { + ROCKS_LOG_BUFFER( + log_buffer, + "[%s] FIFO compaction: Already executing compaction. No need " + "to run parallel compactions since compactions are very fast", + cf_name.c_str()); + return nullptr; + } + + std::vector inputs; + inputs.emplace_back(); + inputs[0].level = last_level; + + if (last_level == 0) { + // In L0, right-most files are the oldest files. + for (auto ritr = last_level_files.rbegin(); ritr != last_level_files.rend(); + ++ritr) { + auto f = *ritr; + total_size -= f->fd.file_size; + inputs[0].files.push_back(f); + char tmp_fsize[16]; + AppendHumanBytes(f->fd.GetFileSize(), tmp_fsize, sizeof(tmp_fsize)); + ROCKS_LOG_BUFFER(log_buffer, + "[%s] FIFO compaction: picking file %" PRIu64 + " with size %s for deletion", + cf_name.c_str(), f->fd.GetNumber(), tmp_fsize); + if (total_size <= + mutable_cf_options.compaction_options_fifo.max_table_files_size) { + break; + } + } + } else if (total_size > + mutable_cf_options.compaction_options_fifo.max_table_files_size) { + // If the last level is non-L0, we actually don't know which file is + // logically the oldest since the file creation time only represents + // when this file was compacted to this level, which is independent + // to when the entries in this file were first inserted. + // + // As a result, we delete files from the left instead. This means the sst + // file with the smallest key will be deleted first. This design decision + // better serves a major type of FIFO use cases where smaller keys are + // associated with older data. + for (const auto& f : last_level_files) { + total_size -= f->fd.file_size; + inputs[0].files.push_back(f); + char tmp_fsize[16]; + AppendHumanBytes(f->fd.GetFileSize(), tmp_fsize, sizeof(tmp_fsize)); + ROCKS_LOG_BUFFER( + log_buffer, + "[%s] FIFO compaction: picking file %" PRIu64 + " with size %s for deletion under total size %" PRIu64 + " vs max table files size %" PRIu64, + cf_name.c_str(), f->fd.GetNumber(), tmp_fsize, total_size, + mutable_cf_options.compaction_options_fifo.max_table_files_size); + + if (total_size <= + mutable_cf_options.compaction_options_fifo.max_table_files_size) { + break; + } + } + } else { + ROCKS_LOG_BUFFER( + log_buffer, + "[%s] FIFO compaction: nothing to do. Total size %" PRIu64 + ", max size %" PRIu64 "\n", + cf_name.c_str(), total_size, + mutable_cf_options.compaction_options_fifo.max_table_files_size); + return nullptr; + } + + Compaction* c = new Compaction( + vstorage, ioptions_, mutable_cf_options, mutable_db_options, + std::move(inputs), last_level, + /* target_file_size */ 0, + /* max_compaction_bytes */ 0, + /* output_path_id */ 0, kNoCompression, + mutable_cf_options.compression_opts, Temperature::kUnknown, + /* max_subcompactions */ 0, {}, /* is manual */ false, + /* trim_ts */ "", vstorage->CompactionScore(0), + /* is deletion compaction */ true, + /* l0_files_might_overlap */ true, CompactionReason::kFIFOMaxSize); + return c; +} + +Compaction* FIFOCompactionPicker::PickTemperatureChangeCompaction( + const std::string& cf_name, const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage, + LogBuffer* log_buffer) { + const std::vector& ages = + mutable_cf_options.compaction_options_fifo + .file_temperature_age_thresholds; + if (ages.empty()) { + return nullptr; + } + + // Does not apply to multi-level FIFO. + if (vstorage->num_levels() > 1) { + return nullptr; + } + + const int kLevel0 = 0; + const std::vector& level_files = vstorage->LevelFiles(kLevel0); + if (level_files.empty()) { + return nullptr; + } + + int64_t _current_time; + auto status = ioptions_.clock->GetCurrentTime(&_current_time); + if (!status.ok()) { + ROCKS_LOG_BUFFER( + log_buffer, + "[%s] FIFO compaction: Couldn't get current time: %s. " + "Not doing compactions based on file temperature-age threshold. ", + cf_name.c_str(), status.ToString().c_str()); + return nullptr; + } + const uint64_t current_time = static_cast(_current_time); + + if (!level0_compactions_in_progress_.empty()) { + ROCKS_LOG_BUFFER( + log_buffer, + "[%s] FIFO compaction: Already executing compaction. Parallel " + "compactions are not supported", + cf_name.c_str()); + return nullptr; + } + + std::vector inputs; + inputs.emplace_back(); + inputs[0].level = 0; + + // avoid underflow + uint64_t min_age = ages[0].age; + // kLastTemperature means target temperature is to be determined. + Temperature compaction_target_temp = Temperature::kLastTemperature; + if (current_time > min_age) { + uint64_t create_time_threshold = current_time - min_age; + uint64_t compaction_size = 0; + // We will ideally identify a file qualifying for temperature change by + // knowing the timestamp for the youngest entry in the file. However, right + // now we don't have the information. We infer it by looking at timestamp of + // the previous file's (which is just younger) oldest entry's timestamp. + Temperature cur_target_temp; + // avoid index underflow + assert(level_files.size() >= 1); + for (size_t index = level_files.size() - 1; index >= 1; --index) { + // Try to add cur_file to compaction inputs. + FileMetaData* cur_file = level_files[index]; + // prev_file is just younger than cur_file + FileMetaData* prev_file = level_files[index - 1]; + if (cur_file->being_compacted) { + // Should not happen since we check for + // `level0_compactions_in_progress_` above. Here we simply just don't + // schedule anything. + return nullptr; + } + uint64_t oldest_ancestor_time = prev_file->TryGetOldestAncesterTime(); + if (oldest_ancestor_time == kUnknownOldestAncesterTime) { + // Older files might not have enough information. It is possible to + // handle these files by looking at newer files, but maintaining the + // logic isn't worth it. + break; + } + if (oldest_ancestor_time > create_time_threshold) { + // cur_file is too fresh + break; + } + cur_target_temp = ages[0].temperature; + for (size_t i = 1; i < ages.size(); ++i) { + if (current_time >= ages[i].age && + oldest_ancestor_time <= current_time - ages[i].age) { + cur_target_temp = ages[i].temperature; + } + } + if (cur_file->temperature == cur_target_temp) { + if (inputs[0].empty()) { + continue; + } else { + break; + } + } + + // cur_file needs to change temperature + if (compaction_target_temp == Temperature::kLastTemperature) { + assert(inputs[0].empty()); + compaction_target_temp = cur_target_temp; + } else if (cur_target_temp != compaction_target_temp) { + assert(!inputs[0].empty()); + break; + } + if (inputs[0].empty() || compaction_size + cur_file->fd.GetFileSize() <= + mutable_cf_options.max_compaction_bytes) { + inputs[0].files.push_back(cur_file); + compaction_size += cur_file->fd.GetFileSize(); + ROCKS_LOG_BUFFER( + log_buffer, + "[%s] FIFO compaction: picking file %" PRIu64 + " with next file's oldest time %" PRIu64 " for temperature %s.", + cf_name.c_str(), cur_file->fd.GetNumber(), oldest_ancestor_time, + temperature_to_string[cur_target_temp].c_str()); + } + if (compaction_size > mutable_cf_options.max_compaction_bytes) { + break; + } + } + } + + if (inputs[0].files.empty()) { + return nullptr; + } + + Compaction* c = new Compaction( + vstorage, ioptions_, mutable_cf_options, mutable_db_options, + std::move(inputs), 0, 0 /* output file size limit */, + 0 /* max compaction bytes, not applicable */, 0 /* output path ID */, + mutable_cf_options.compression, mutable_cf_options.compression_opts, + compaction_target_temp, + /* max_subcompactions */ 0, {}, /* is manual */ false, /* trim_ts */ "", + vstorage->CompactionScore(0), + /* is deletion compaction */ false, /* l0_files_might_overlap */ true, + CompactionReason::kChangeTemperature); + return c; +} + +Compaction* FIFOCompactionPicker::PickCompaction( + const std::string& cf_name, const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage, + LogBuffer* log_buffer) { + Compaction* c = nullptr; + if (mutable_cf_options.ttl > 0) { + c = PickTTLCompaction(cf_name, mutable_cf_options, mutable_db_options, + vstorage, log_buffer); + } + if (c == nullptr) { + c = PickSizeCompaction(cf_name, mutable_cf_options, mutable_db_options, + vstorage, log_buffer); + } + if (c == nullptr) { + c = PickTemperatureChangeCompaction( + cf_name, mutable_cf_options, mutable_db_options, vstorage, log_buffer); + } + RegisterCompaction(c); + return c; +} + +Compaction* FIFOCompactionPicker::CompactRange( + const std::string& cf_name, const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage, + int input_level, int output_level, + const CompactRangeOptions& /*compact_range_options*/, + const InternalKey* /*begin*/, const InternalKey* /*end*/, + InternalKey** compaction_end, bool* /*manual_conflict*/, + uint64_t /*max_file_num_to_ignore*/, const std::string& /*trim_ts*/) { +#ifdef NDEBUG + (void)input_level; + (void)output_level; +#endif + assert(input_level == 0); + assert(output_level == 0); + *compaction_end = nullptr; + LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, ioptions_.logger); + Compaction* c = PickCompaction(cf_name, mutable_cf_options, + mutable_db_options, vstorage, &log_buffer); + log_buffer.FlushBufferToLog(); + return c; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_picker_fifo.h b/librocksdb-sys/rocksdb/db/compaction/compaction_picker_fifo.h new file mode 100644 index 0000000..df21a1b --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_picker_fifo.h @@ -0,0 +1,60 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include "db/compaction/compaction_picker.h" + +namespace ROCKSDB_NAMESPACE { +class FIFOCompactionPicker : public CompactionPicker { + public: + FIFOCompactionPicker(const ImmutableOptions& ioptions, + const InternalKeyComparator* icmp) + : CompactionPicker(ioptions, icmp) {} + + virtual Compaction* PickCompaction(const std::string& cf_name, + const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, + VersionStorageInfo* version, + LogBuffer* log_buffer) override; + + virtual Compaction* CompactRange( + const std::string& cf_name, const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage, + int input_level, int output_level, + const CompactRangeOptions& compact_range_options, + const InternalKey* begin, const InternalKey* end, + InternalKey** compaction_end, bool* manual_conflict, + uint64_t max_file_num_to_ignore, const std::string& trim_ts) override; + + // The maximum allowed output level. Always returns 0. + virtual int MaxOutputLevel() const override { return 0; } + + virtual bool NeedsCompaction( + const VersionStorageInfo* vstorage) const override; + + private: + Compaction* PickTTLCompaction(const std::string& cf_name, + const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, + VersionStorageInfo* version, + LogBuffer* log_buffer); + + Compaction* PickSizeCompaction(const std::string& cf_name, + const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, + VersionStorageInfo* version, + LogBuffer* log_buffer); + + Compaction* PickTemperatureChangeCompaction( + const std::string& cf_name, const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage, + LogBuffer* log_buffer); +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_picker_level.cc b/librocksdb-sys/rocksdb/db/compaction/compaction_picker_level.cc new file mode 100644 index 0000000..c436689 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_picker_level.cc @@ -0,0 +1,888 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/compaction/compaction_picker_level.h" + +#include +#include +#include + +#include "db/version_edit.h" +#include "logging/log_buffer.h" +#include "test_util/sync_point.h" + +namespace ROCKSDB_NAMESPACE { + +bool LevelCompactionPicker::NeedsCompaction( + const VersionStorageInfo* vstorage) const { + if (!vstorage->ExpiredTtlFiles().empty()) { + return true; + } + if (!vstorage->FilesMarkedForPeriodicCompaction().empty()) { + return true; + } + if (!vstorage->BottommostFilesMarkedForCompaction().empty()) { + return true; + } + if (!vstorage->FilesMarkedForCompaction().empty()) { + return true; + } + if (!vstorage->FilesMarkedForForcedBlobGC().empty()) { + return true; + } + for (int i = 0; i <= vstorage->MaxInputLevel(); i++) { + if (vstorage->CompactionScore(i) >= 1) { + return true; + } + } + return false; +} + +namespace { +// A class to build a leveled compaction step-by-step. +class LevelCompactionBuilder { + public: + LevelCompactionBuilder(const std::string& cf_name, + VersionStorageInfo* vstorage, + CompactionPicker* compaction_picker, + LogBuffer* log_buffer, + const MutableCFOptions& mutable_cf_options, + const ImmutableOptions& ioptions, + const MutableDBOptions& mutable_db_options) + : cf_name_(cf_name), + vstorage_(vstorage), + compaction_picker_(compaction_picker), + log_buffer_(log_buffer), + mutable_cf_options_(mutable_cf_options), + ioptions_(ioptions), + mutable_db_options_(mutable_db_options) {} + + // Pick and return a compaction. + Compaction* PickCompaction(); + + // Pick the initial files to compact to the next level. (or together + // in Intra-L0 compactions) + void SetupInitialFiles(); + + // If the initial files are from L0 level, pick other L0 + // files if needed. + bool SetupOtherL0FilesIfNeeded(); + + // Compaction with round-robin compaction priority allows more files to be + // picked to form a large compaction + void SetupOtherFilesWithRoundRobinExpansion(); + // Based on initial files, setup other files need to be compacted + // in this compaction, accordingly. + bool SetupOtherInputsIfNeeded(); + + Compaction* GetCompaction(); + + // From `start_level_`, pick files to compact to `output_level_`. + // Returns false if there is no file to compact. + // If it returns true, inputs->files.size() will be exactly one for + // all compaction priorities except round-robin. For round-robin, + // multiple consecutive files may be put into inputs->files. + // If level is 0 and there is already a compaction on that level, this + // function will return false. + bool PickFileToCompact(); + + // Return true if a L0 trivial move is picked up. + bool TryPickL0TrivialMove(); + + // For L0->L0, picks the longest span of files that aren't currently + // undergoing compaction for which work-per-deleted-file decreases. The span + // always starts from the newest L0 file. + // + // Intra-L0 compaction is independent of all other files, so it can be + // performed even when L0->base_level compactions are blocked. + // + // Returns true if `inputs` is populated with a span of files to be compacted; + // otherwise, returns false. + bool PickIntraL0Compaction(); + + // Return true if TrivialMove is extended. `start_index` is the index of + // the initial file picked, which should already be in `start_level_inputs_`. + bool TryExtendNonL0TrivialMove(int start_index, + bool only_expand_right = false); + + // Picks a file from level_files to compact. + // level_files is a vector of (level, file metadata) in ascending order of + // level. If compact_to_next_level is true, compact the file to the next + // level, otherwise, compact to the same level as the input file. + void PickFileToCompact( + const autovector>& level_files, + bool compact_to_next_level); + + const std::string& cf_name_; + VersionStorageInfo* vstorage_; + CompactionPicker* compaction_picker_; + LogBuffer* log_buffer_; + int start_level_ = -1; + int output_level_ = -1; + int parent_index_ = -1; + int base_index_ = -1; + double start_level_score_ = 0; + bool is_manual_ = false; + bool is_l0_trivial_move_ = false; + CompactionInputFiles start_level_inputs_; + std::vector compaction_inputs_; + CompactionInputFiles output_level_inputs_; + std::vector grandparents_; + CompactionReason compaction_reason_ = CompactionReason::kUnknown; + + const MutableCFOptions& mutable_cf_options_; + const ImmutableOptions& ioptions_; + const MutableDBOptions& mutable_db_options_; + // Pick a path ID to place a newly generated file, with its level + static uint32_t GetPathId(const ImmutableCFOptions& ioptions, + const MutableCFOptions& mutable_cf_options, + int level); + + static const int kMinFilesForIntraL0Compaction = 4; +}; + +void LevelCompactionBuilder::PickFileToCompact( + const autovector>& level_files, + bool compact_to_next_level) { + for (auto& level_file : level_files) { + // If it's being compacted it has nothing to do here. + // If this assert() fails that means that some function marked some + // files as being_compacted, but didn't call ComputeCompactionScore() + assert(!level_file.second->being_compacted); + start_level_ = level_file.first; + if ((compact_to_next_level && + start_level_ == vstorage_->num_non_empty_levels() - 1) || + (start_level_ == 0 && + !compaction_picker_->level0_compactions_in_progress()->empty())) { + continue; + } + if (compact_to_next_level) { + output_level_ = + (start_level_ == 0) ? vstorage_->base_level() : start_level_ + 1; + } else { + output_level_ = start_level_; + } + start_level_inputs_.files = {level_file.second}; + start_level_inputs_.level = start_level_; + if (compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_, + &start_level_inputs_)) { + return; + } + } + start_level_inputs_.files.clear(); +} + +void LevelCompactionBuilder::SetupInitialFiles() { + // Find the compactions by size on all levels. + bool skipped_l0_to_base = false; + for (int i = 0; i < compaction_picker_->NumberLevels() - 1; i++) { + start_level_score_ = vstorage_->CompactionScore(i); + start_level_ = vstorage_->CompactionScoreLevel(i); + assert(i == 0 || start_level_score_ <= vstorage_->CompactionScore(i - 1)); + if (start_level_score_ >= 1) { + if (skipped_l0_to_base && start_level_ == vstorage_->base_level()) { + // If L0->base_level compaction is pending, don't schedule further + // compaction from base level. Otherwise L0->base_level compaction + // may starve. + continue; + } + output_level_ = + (start_level_ == 0) ? vstorage_->base_level() : start_level_ + 1; + bool picked_file_to_compact = PickFileToCompact(); + TEST_SYNC_POINT_CALLBACK("PostPickFileToCompact", + &picked_file_to_compact); + if (picked_file_to_compact) { + // found the compaction! + if (start_level_ == 0) { + // L0 score = `num L0 files` / `level0_file_num_compaction_trigger` + compaction_reason_ = CompactionReason::kLevelL0FilesNum; + } else { + // L1+ score = `Level files size` / `MaxBytesForLevel` + compaction_reason_ = CompactionReason::kLevelMaxLevelSize; + } + break; + } else { + // didn't find the compaction, clear the inputs + start_level_inputs_.clear(); + if (start_level_ == 0) { + skipped_l0_to_base = true; + // L0->base_level may be blocked due to ongoing L0->base_level + // compactions. It may also be blocked by an ongoing compaction from + // base_level downwards. + // + // In these cases, to reduce L0 file count and thus reduce likelihood + // of write stalls, we can attempt compacting a span of files within + // L0. + if (PickIntraL0Compaction()) { + output_level_ = 0; + compaction_reason_ = CompactionReason::kLevelL0FilesNum; + break; + } + } + } + } else { + // Compaction scores are sorted in descending order, no further scores + // will be >= 1. + break; + } + } + if (!start_level_inputs_.empty()) { + return; + } + + // if we didn't find a compaction, check if there are any files marked for + // compaction + parent_index_ = base_index_ = -1; + + compaction_picker_->PickFilesMarkedForCompaction( + cf_name_, vstorage_, &start_level_, &output_level_, &start_level_inputs_); + if (!start_level_inputs_.empty()) { + compaction_reason_ = CompactionReason::kFilesMarkedForCompaction; + return; + } + + // Bottommost Files Compaction on deleting tombstones + PickFileToCompact(vstorage_->BottommostFilesMarkedForCompaction(), false); + if (!start_level_inputs_.empty()) { + compaction_reason_ = CompactionReason::kBottommostFiles; + return; + } + + // TTL Compaction + if (ioptions_.compaction_pri == kRoundRobin && + !vstorage_->ExpiredTtlFiles().empty()) { + auto expired_files = vstorage_->ExpiredTtlFiles(); + // the expired files list should already be sorted by level + start_level_ = expired_files.front().first; +#ifndef NDEBUG + for (const auto& file : expired_files) { + assert(start_level_ <= file.first); + } +#endif + if (start_level_ > 0) { + output_level_ = start_level_ + 1; + if (PickFileToCompact()) { + compaction_reason_ = CompactionReason::kRoundRobinTtl; + return; + } + } + } + + PickFileToCompact(vstorage_->ExpiredTtlFiles(), true); + if (!start_level_inputs_.empty()) { + compaction_reason_ = CompactionReason::kTtl; + return; + } + + // Periodic Compaction + PickFileToCompact(vstorage_->FilesMarkedForPeriodicCompaction(), false); + if (!start_level_inputs_.empty()) { + compaction_reason_ = CompactionReason::kPeriodicCompaction; + return; + } + + // Forced blob garbage collection + PickFileToCompact(vstorage_->FilesMarkedForForcedBlobGC(), false); + if (!start_level_inputs_.empty()) { + compaction_reason_ = CompactionReason::kForcedBlobGC; + return; + } +} + +bool LevelCompactionBuilder::SetupOtherL0FilesIfNeeded() { + if (start_level_ == 0 && output_level_ != 0 && !is_l0_trivial_move_) { + return compaction_picker_->GetOverlappingL0Files( + vstorage_, &start_level_inputs_, output_level_, &parent_index_); + } + return true; +} + +void LevelCompactionBuilder::SetupOtherFilesWithRoundRobinExpansion() { + // We only expand when the start level is not L0 under round robin + assert(start_level_ >= 1); + + // For round-robin compaction priority, we have 3 constraints when picking + // multiple files. + // Constraint 1: We can only pick consecutive files + // -> Constraint 1a: When a file is being compacted (or some input files + // are being compacted after expanding, we cannot + // choose it and have to stop choosing more files + // -> Constraint 1b: When we reach the last file (with largest keys), we + // cannot choose more files (the next file will be the + // first one) + // Constraint 2: We should ensure the total compaction bytes (including the + // overlapped files from the next level) is no more than + // mutable_cf_options_.max_compaction_bytes + // Constraint 3: We try our best to pick as many files as possible so that + // the post-compaction level size is less than + // MaxBytesForLevel(start_level_) + // Constraint 4: We do not expand if it is possible to apply a trivial move + // Constraint 5 (TODO): Try to pick minimal files to split into the target + // number of subcompactions + TEST_SYNC_POINT("LevelCompactionPicker::RoundRobin"); + + // Only expand the inputs when we have selected a file in start_level_inputs_ + if (start_level_inputs_.size() == 0) return; + + uint64_t start_lvl_bytes_no_compacting = 0; + uint64_t curr_bytes_to_compact = 0; + uint64_t start_lvl_max_bytes_to_compact = 0; + const std::vector& level_files = + vstorage_->LevelFiles(start_level_); + // Constraint 3 (pre-calculate the ideal max bytes to compact) + for (auto f : level_files) { + if (!f->being_compacted) { + start_lvl_bytes_no_compacting += f->fd.GetFileSize(); + } + } + if (start_lvl_bytes_no_compacting > + vstorage_->MaxBytesForLevel(start_level_)) { + start_lvl_max_bytes_to_compact = start_lvl_bytes_no_compacting - + vstorage_->MaxBytesForLevel(start_level_); + } + + size_t start_index = vstorage_->FilesByCompactionPri(start_level_)[0]; + InternalKey smallest, largest; + // Constraint 4 (No need to check again later) + compaction_picker_->GetRange(start_level_inputs_, &smallest, &largest); + CompactionInputFiles output_level_inputs; + output_level_inputs.level = output_level_; + vstorage_->GetOverlappingInputs(output_level_, &smallest, &largest, + &output_level_inputs.files); + if (output_level_inputs.empty()) { + if (TryExtendNonL0TrivialMove((int)start_index, + true /* only_expand_right */)) { + return; + } + } + // Constraint 3 + if (start_level_inputs_[0]->fd.GetFileSize() >= + start_lvl_max_bytes_to_compact) { + return; + } + CompactionInputFiles tmp_start_level_inputs; + tmp_start_level_inputs = start_level_inputs_; + // TODO (zichen): Future parallel round-robin may also need to update this + // Constraint 1b (only expand till the end) + for (size_t i = start_index + 1; i < level_files.size(); i++) { + auto* f = level_files[i]; + if (f->being_compacted) { + // Constraint 1a + return; + } + + tmp_start_level_inputs.files.push_back(f); + if (!compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_, + &tmp_start_level_inputs) || + compaction_picker_->FilesRangeOverlapWithCompaction( + {tmp_start_level_inputs}, output_level_, + Compaction::EvaluatePenultimateLevel( + vstorage_, ioptions_, start_level_, output_level_))) { + // Constraint 1a + tmp_start_level_inputs.clear(); + return; + } + + curr_bytes_to_compact = 0; + for (auto start_lvl_f : tmp_start_level_inputs.files) { + curr_bytes_to_compact += start_lvl_f->fd.GetFileSize(); + } + + // Check whether any output level files are locked + compaction_picker_->GetRange(tmp_start_level_inputs, &smallest, &largest); + vstorage_->GetOverlappingInputs(output_level_, &smallest, &largest, + &output_level_inputs.files); + if (!output_level_inputs.empty() && + !compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_, + &output_level_inputs)) { + // Constraint 1a + tmp_start_level_inputs.clear(); + return; + } + + uint64_t start_lvl_curr_bytes_to_compact = curr_bytes_to_compact; + for (auto output_lvl_f : output_level_inputs.files) { + curr_bytes_to_compact += output_lvl_f->fd.GetFileSize(); + } + if (curr_bytes_to_compact > mutable_cf_options_.max_compaction_bytes) { + // Constraint 2 + tmp_start_level_inputs.clear(); + return; + } + + start_level_inputs_.files = tmp_start_level_inputs.files; + // Constraint 3 + if (start_lvl_curr_bytes_to_compact > start_lvl_max_bytes_to_compact) { + return; + } + } +} + +bool LevelCompactionBuilder::SetupOtherInputsIfNeeded() { + // Setup input files from output level. For output to L0, we only compact + // spans of files that do not interact with any pending compactions, so don't + // need to consider other levels. + if (output_level_ != 0) { + output_level_inputs_.level = output_level_; + bool round_robin_expanding = + ioptions_.compaction_pri == kRoundRobin && + compaction_reason_ == CompactionReason::kLevelMaxLevelSize; + if (round_robin_expanding) { + SetupOtherFilesWithRoundRobinExpansion(); + } + if (!is_l0_trivial_move_ && + !compaction_picker_->SetupOtherInputs( + cf_name_, mutable_cf_options_, vstorage_, &start_level_inputs_, + &output_level_inputs_, &parent_index_, base_index_, + round_robin_expanding)) { + return false; + } + + compaction_inputs_.push_back(start_level_inputs_); + if (!output_level_inputs_.empty()) { + compaction_inputs_.push_back(output_level_inputs_); + } + + // In some edge cases we could pick a compaction that will be compacting + // a key range that overlap with another running compaction, and both + // of them have the same output level. This could happen if + // (1) we are running a non-exclusive manual compaction + // (2) AddFile ingest a new file into the LSM tree + // We need to disallow this from happening. + if (compaction_picker_->FilesRangeOverlapWithCompaction( + compaction_inputs_, output_level_, + Compaction::EvaluatePenultimateLevel( + vstorage_, ioptions_, start_level_, output_level_))) { + // This compaction output could potentially conflict with the output + // of a currently running compaction, we cannot run it. + return false; + } + if (!is_l0_trivial_move_) { + compaction_picker_->GetGrandparents(vstorage_, start_level_inputs_, + output_level_inputs_, &grandparents_); + } + } else { + compaction_inputs_.push_back(start_level_inputs_); + } + return true; +} + +Compaction* LevelCompactionBuilder::PickCompaction() { + // Pick up the first file to start compaction. It may have been extended + // to a clean cut. + SetupInitialFiles(); + if (start_level_inputs_.empty()) { + return nullptr; + } + assert(start_level_ >= 0 && output_level_ >= 0); + + // If it is a L0 -> base level compaction, we need to set up other L0 + // files if needed. + if (!SetupOtherL0FilesIfNeeded()) { + return nullptr; + } + + // Pick files in the output level and expand more files in the start level + // if needed. + if (!SetupOtherInputsIfNeeded()) { + return nullptr; + } + + // Form a compaction object containing the files we picked. + Compaction* c = GetCompaction(); + + TEST_SYNC_POINT_CALLBACK("LevelCompactionPicker::PickCompaction:Return", c); + + return c; +} + +Compaction* LevelCompactionBuilder::GetCompaction() { + // TryPickL0TrivialMove() does not apply to the case when compacting L0 to an + // empty output level. So L0 files is picked in PickFileToCompact() by + // compaction score. We may still be able to do trivial move when this file + // does not overlap with other L0s. This happens when + // compaction_inputs_[0].size() == 1 since SetupOtherL0FilesIfNeeded() did not + // pull in more L0s. + assert(!compaction_inputs_.empty()); + bool l0_files_might_overlap = + start_level_ == 0 && !is_l0_trivial_move_ && + (compaction_inputs_.size() > 1 || compaction_inputs_[0].size() > 1); + auto c = new Compaction( + vstorage_, ioptions_, mutable_cf_options_, mutable_db_options_, + std::move(compaction_inputs_), output_level_, + MaxFileSizeForLevel(mutable_cf_options_, output_level_, + ioptions_.compaction_style, vstorage_->base_level(), + ioptions_.level_compaction_dynamic_level_bytes), + mutable_cf_options_.max_compaction_bytes, + GetPathId(ioptions_, mutable_cf_options_, output_level_), + GetCompressionType(vstorage_, mutable_cf_options_, output_level_, + vstorage_->base_level()), + GetCompressionOptions(mutable_cf_options_, vstorage_, output_level_), + Temperature::kUnknown, + /* max_subcompactions */ 0, std::move(grandparents_), is_manual_, + /* trim_ts */ "", start_level_score_, false /* deletion_compaction */, + l0_files_might_overlap, compaction_reason_); + + // If it's level 0 compaction, make sure we don't execute any other level 0 + // compactions in parallel + compaction_picker_->RegisterCompaction(c); + + // Creating a compaction influences the compaction score because the score + // takes running compactions into account (by skipping files that are already + // being compacted). Since we just changed compaction score, we recalculate it + // here + vstorage_->ComputeCompactionScore(ioptions_, mutable_cf_options_); + return c; +} + +/* + * Find the optimal path to place a file + * Given a level, finds the path where levels up to it will fit in levels + * up to and including this path + */ +uint32_t LevelCompactionBuilder::GetPathId( + const ImmutableCFOptions& ioptions, + const MutableCFOptions& mutable_cf_options, int level) { + uint32_t p = 0; + assert(!ioptions.cf_paths.empty()); + + // size remaining in the most recent path + uint64_t current_path_size = ioptions.cf_paths[0].target_size; + + uint64_t level_size; + int cur_level = 0; + + // max_bytes_for_level_base denotes L1 size. + // We estimate L0 size to be the same as L1. + level_size = mutable_cf_options.max_bytes_for_level_base; + + // Last path is the fallback + while (p < ioptions.cf_paths.size() - 1) { + if (level_size <= current_path_size) { + if (cur_level == level) { + // Does desired level fit in this path? + return p; + } else { + current_path_size -= level_size; + if (cur_level > 0) { + if (ioptions.level_compaction_dynamic_level_bytes) { + // Currently, level_compaction_dynamic_level_bytes is ignored when + // multiple db paths are specified. https://github.com/facebook/ + // rocksdb/blob/main/db/column_family.cc. + // Still, adding this check to avoid accidentally using + // max_bytes_for_level_multiplier_additional + level_size = static_cast( + level_size * mutable_cf_options.max_bytes_for_level_multiplier); + } else { + level_size = static_cast( + level_size * mutable_cf_options.max_bytes_for_level_multiplier * + mutable_cf_options.MaxBytesMultiplerAdditional(cur_level)); + } + } + cur_level++; + continue; + } + } + p++; + current_path_size = ioptions.cf_paths[p].target_size; + } + return p; +} + +bool LevelCompactionBuilder::TryPickL0TrivialMove() { + if (vstorage_->base_level() <= 0) { + return false; + } + if (start_level_ == 0 && mutable_cf_options_.compression_per_level.empty() && + !vstorage_->LevelFiles(output_level_).empty() && + ioptions_.db_paths.size() <= 1) { + // Try to pick trivial move from L0 to L1. We start from the oldest + // file. We keep expanding to newer files if it would form a + // trivial move. + // For now we don't support it with + // mutable_cf_options_.compression_per_level to prevent the logic + // of determining whether L0 can be trivial moved to the next level. + // We skip the case where output level is empty, since in this case, at + // least the oldest file would qualify for trivial move, and this would + // be a surprising behavior with few benefits. + + // We search from the oldest file from the newest. In theory, there are + // files in the middle can form trivial move too, but it is probably + // uncommon and we ignore these cases for simplicity. + const std::vector& level_files = + vstorage_->LevelFiles(start_level_); + + InternalKey my_smallest, my_largest; + for (auto it = level_files.rbegin(); it != level_files.rend(); ++it) { + CompactionInputFiles output_level_inputs; + output_level_inputs.level = output_level_; + FileMetaData* file = *it; + if (it == level_files.rbegin()) { + my_smallest = file->smallest; + my_largest = file->largest; + } else { + if (compaction_picker_->icmp()->Compare(file->largest, my_smallest) < + 0) { + my_smallest = file->smallest; + } else if (compaction_picker_->icmp()->Compare(file->smallest, + my_largest) > 0) { + my_largest = file->largest; + } else { + break; + } + } + vstorage_->GetOverlappingInputs(output_level_, &my_smallest, &my_largest, + &output_level_inputs.files); + if (output_level_inputs.empty()) { + assert(!file->being_compacted); + start_level_inputs_.files.push_back(file); + } else { + break; + } + } + } + + if (!start_level_inputs_.empty()) { + // Sort files by key range. Not sure it's 100% necessary but it's cleaner + // to always keep files sorted by key the key ranges don't overlap. + std::sort(start_level_inputs_.files.begin(), + start_level_inputs_.files.end(), + [icmp = compaction_picker_->icmp()](FileMetaData* f1, + FileMetaData* f2) -> bool { + return (icmp->Compare(f1->smallest, f2->smallest) < 0); + }); + + is_l0_trivial_move_ = true; + return true; + } + return false; +} + +bool LevelCompactionBuilder::TryExtendNonL0TrivialMove(int start_index, + bool only_expand_right) { + if (start_level_inputs_.size() == 1 && + (ioptions_.db_paths.empty() || ioptions_.db_paths.size() == 1) && + (mutable_cf_options_.compression_per_level.empty())) { + // Only file of `index`, and it is likely a trivial move. Try to + // expand if it is still a trivial move, but not beyond + // max_compaction_bytes or 4 files, so that we don't create too + // much compaction pressure for the next level. + // Ignore if there are more than one DB path, as it would be hard + // to predict whether it is a trivial move. + const std::vector& level_files = + vstorage_->LevelFiles(start_level_); + const size_t kMaxMultiTrivialMove = 4; + FileMetaData* initial_file = start_level_inputs_.files[0]; + size_t total_size = initial_file->fd.GetFileSize(); + CompactionInputFiles output_level_inputs; + output_level_inputs.level = output_level_; + // Expand towards right + for (int i = start_index + 1; + i < static_cast(level_files.size()) && + start_level_inputs_.size() < kMaxMultiTrivialMove; + i++) { + FileMetaData* next_file = level_files[i]; + if (next_file->being_compacted) { + break; + } + vstorage_->GetOverlappingInputs(output_level_, &(initial_file->smallest), + &(next_file->largest), + &output_level_inputs.files); + if (!output_level_inputs.empty()) { + break; + } + if (i < static_cast(level_files.size()) - 1 && + compaction_picker_->icmp() + ->user_comparator() + ->CompareWithoutTimestamp( + next_file->largest.user_key(), + level_files[i + 1]->smallest.user_key()) == 0) { + TEST_SYNC_POINT_CALLBACK( + "LevelCompactionBuilder::TryExtendNonL0TrivialMove:NoCleanCut", + nullptr); + // Not a clean up after adding the next file. Skip. + break; + } + total_size += next_file->fd.GetFileSize(); + if (total_size > mutable_cf_options_.max_compaction_bytes) { + break; + } + start_level_inputs_.files.push_back(next_file); + } + // Expand towards left + if (!only_expand_right) { + for (int i = start_index - 1; + i >= 0 && start_level_inputs_.size() < kMaxMultiTrivialMove; i--) { + FileMetaData* next_file = level_files[i]; + if (next_file->being_compacted) { + break; + } + vstorage_->GetOverlappingInputs(output_level_, &(next_file->smallest), + &(initial_file->largest), + &output_level_inputs.files); + if (!output_level_inputs.empty()) { + break; + } + if (i > 0 && compaction_picker_->icmp() + ->user_comparator() + ->CompareWithoutTimestamp( + next_file->smallest.user_key(), + level_files[i - 1]->largest.user_key()) == 0) { + // Not a clean up after adding the next file. Skip. + break; + } + total_size += next_file->fd.GetFileSize(); + if (total_size > mutable_cf_options_.max_compaction_bytes) { + break; + } + // keep `files` sorted in increasing order by key range + start_level_inputs_.files.insert(start_level_inputs_.files.begin(), + next_file); + } + } + return start_level_inputs_.size() > 1; + } + return false; +} + +bool LevelCompactionBuilder::PickFileToCompact() { + // level 0 files are overlapping. So we cannot pick more + // than one concurrent compactions at this level. This + // could be made better by looking at key-ranges that are + // being compacted at level 0. + if (start_level_ == 0 && + !compaction_picker_->level0_compactions_in_progress()->empty()) { + TEST_SYNC_POINT("LevelCompactionPicker::PickCompactionBySize:0"); + return false; + } + + start_level_inputs_.clear(); + start_level_inputs_.level = start_level_; + + assert(start_level_ >= 0); + + if (TryPickL0TrivialMove()) { + return true; + } + + const std::vector& level_files = + vstorage_->LevelFiles(start_level_); + + // Pick the file with the highest score in this level that is not already + // being compacted. + const std::vector& file_scores = + vstorage_->FilesByCompactionPri(start_level_); + + unsigned int cmp_idx; + for (cmp_idx = vstorage_->NextCompactionIndex(start_level_); + cmp_idx < file_scores.size(); cmp_idx++) { + int index = file_scores[cmp_idx]; + auto* f = level_files[index]; + + // do not pick a file to compact if it is being compacted + // from n-1 level. + if (f->being_compacted) { + if (ioptions_.compaction_pri == kRoundRobin) { + // TODO(zichen): this file may be involved in one compaction from + // an upper level, cannot advance the cursor for round-robin policy. + // Currently, we do not pick any file to compact in this case. We + // should fix this later to ensure a compaction is picked but the + // cursor shall not be advanced. + return false; + } + continue; + } + + start_level_inputs_.files.push_back(f); + if (!compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_, + &start_level_inputs_) || + compaction_picker_->FilesRangeOverlapWithCompaction( + {start_level_inputs_}, output_level_, + Compaction::EvaluatePenultimateLevel( + vstorage_, ioptions_, start_level_, output_level_))) { + // A locked (pending compaction) input-level file was pulled in due to + // user-key overlap. + start_level_inputs_.clear(); + + if (ioptions_.compaction_pri == kRoundRobin) { + return false; + } + continue; + } + + // Now that input level is fully expanded, we check whether any output + // files are locked due to pending compaction. + // + // Note we rely on ExpandInputsToCleanCut() to tell us whether any output- + // level files are locked, not just the extra ones pulled in for user-key + // overlap. + InternalKey smallest, largest; + compaction_picker_->GetRange(start_level_inputs_, &smallest, &largest); + CompactionInputFiles output_level_inputs; + output_level_inputs.level = output_level_; + vstorage_->GetOverlappingInputs(output_level_, &smallest, &largest, + &output_level_inputs.files); + if (output_level_inputs.empty()) { + if (start_level_ > 0 && + TryExtendNonL0TrivialMove(index, + ioptions_.compaction_pri == + kRoundRobin /* only_expand_right */)) { + break; + } + } else { + if (!compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_, + &output_level_inputs)) { + start_level_inputs_.clear(); + if (ioptions_.compaction_pri == kRoundRobin) { + return false; + } + continue; + } + } + + base_index_ = index; + break; + } + + // store where to start the iteration in the next call to PickCompaction + if (ioptions_.compaction_pri != kRoundRobin) { + vstorage_->SetNextCompactionIndex(start_level_, cmp_idx); + } + return start_level_inputs_.size() > 0; +} + +bool LevelCompactionBuilder::PickIntraL0Compaction() { + start_level_inputs_.clear(); + const std::vector& level_files = + vstorage_->LevelFiles(0 /* level */); + if (level_files.size() < + static_cast( + mutable_cf_options_.level0_file_num_compaction_trigger + 2) || + level_files[0]->being_compacted) { + // If L0 isn't accumulating much files beyond the regular trigger, don't + // resort to L0->L0 compaction yet. + return false; + } + return FindIntraL0Compaction(level_files, kMinFilesForIntraL0Compaction, + std::numeric_limits::max(), + mutable_cf_options_.max_compaction_bytes, + &start_level_inputs_); +} +} // namespace + +Compaction* LevelCompactionPicker::PickCompaction( + const std::string& cf_name, const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage, + LogBuffer* log_buffer) { + LevelCompactionBuilder builder(cf_name, vstorage, this, log_buffer, + mutable_cf_options, ioptions_, + mutable_db_options); + return builder.PickCompaction(); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_picker_level.h b/librocksdb-sys/rocksdb/db/compaction/compaction_picker_level.h new file mode 100644 index 0000000..6eb0f58 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_picker_level.h @@ -0,0 +1,33 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include "db/compaction/compaction_picker.h" + +namespace ROCKSDB_NAMESPACE { +// Picking compactions for leveled compaction. See wiki page +// https://github.com/facebook/rocksdb/wiki/Leveled-Compaction +// for description of Leveled compaction. +class LevelCompactionPicker : public CompactionPicker { + public: + LevelCompactionPicker(const ImmutableOptions& ioptions, + const InternalKeyComparator* icmp) + : CompactionPicker(ioptions, icmp) {} + virtual Compaction* PickCompaction(const std::string& cf_name, + const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, + VersionStorageInfo* vstorage, + LogBuffer* log_buffer) override; + + virtual bool NeedsCompaction( + const VersionStorageInfo* vstorage) const override; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_picker_test.cc b/librocksdb-sys/rocksdb/db/compaction/compaction_picker_test.cc new file mode 100644 index 0000000..fd14322 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_picker_test.cc @@ -0,0 +1,4088 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include +#include +#include + +#include "db/compaction/compaction.h" +#include "db/compaction/compaction_picker_fifo.h" +#include "db/compaction/compaction_picker_level.h" +#include "db/compaction/compaction_picker_universal.h" +#include "db/compaction/file_pri.h" +#include "rocksdb/advanced_options.h" +#include "table/unique_id_impl.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +class CountingLogger : public Logger { + public: + using Logger::Logv; + void Logv(const char* /*format*/, va_list /*ap*/) override { log_count++; } + size_t log_count; +}; + +class CompactionPickerTestBase : public testing::Test { + public: + const Comparator* ucmp_; + InternalKeyComparator icmp_; + Options options_; + ImmutableOptions ioptions_; + MutableCFOptions mutable_cf_options_; + MutableDBOptions mutable_db_options_; + LevelCompactionPicker level_compaction_picker; + std::string cf_name_; + CountingLogger logger_; + LogBuffer log_buffer_; + uint32_t file_num_; + CompactionOptionsFIFO fifo_options_; + std::unique_ptr vstorage_; + std::vector> files_; + // does not own FileMetaData + std::unordered_map> file_map_; + // input files to compaction process. + std::vector input_files_; + int compaction_level_start_; + + explicit CompactionPickerTestBase(const Comparator* _ucmp) + : ucmp_(_ucmp), + icmp_(ucmp_), + options_(CreateOptions(ucmp_)), + ioptions_(options_), + mutable_cf_options_(options_), + mutable_db_options_(), + level_compaction_picker(ioptions_, &icmp_), + cf_name_("dummy"), + log_buffer_(InfoLogLevel::INFO_LEVEL, &logger_), + file_num_(1), + vstorage_(nullptr) { + mutable_cf_options_.ttl = 0; + mutable_cf_options_.periodic_compaction_seconds = 0; + // ioptions_.compaction_pri = kMinOverlappingRatio has its own set of + // tests to cover. + ioptions_.compaction_pri = kByCompensatedSize; + fifo_options_.max_table_files_size = 1; + mutable_cf_options_.RefreshDerivedOptions(ioptions_); + ioptions_.cf_paths.emplace_back("dummy", + std::numeric_limits::max()); + // When the default value of this option is true, universal compaction + // tests can encounter assertion failure since SanitizeOption() is + // not run to set this option to false. So we do the sanitization + // here. Tests that test this option set this option to true explicitly. + ioptions_.level_compaction_dynamic_level_bytes = false; + } + + ~CompactionPickerTestBase() override {} + + void NewVersionStorage(int num_levels, CompactionStyle style) { + DeleteVersionStorage(); + options_.num_levels = num_levels; + vstorage_.reset(new VersionStorageInfo( + &icmp_, ucmp_, options_.num_levels, style, nullptr, false, + EpochNumberRequirement::kMustPresent)); + vstorage_->PrepareForVersionAppend(ioptions_, mutable_cf_options_); + } + + // Create a new VersionStorageInfo object so we can add mode files and then + // merge it with the existing VersionStorageInfo + void AddVersionStorage() { + temp_vstorage_.reset(new VersionStorageInfo( + &icmp_, ucmp_, options_.num_levels, ioptions_.compaction_style, + vstorage_.get(), false, EpochNumberRequirement::kMustPresent)); + } + + void DeleteVersionStorage() { + vstorage_.reset(); + temp_vstorage_.reset(); + files_.clear(); + file_map_.clear(); + input_files_.clear(); + } + + // REQUIRES: smallest and largest are c-style strings ending with '\0' + void Add(int level, uint32_t file_number, const char* smallest, + const char* largest, uint64_t file_size = 1, uint32_t path_id = 0, + SequenceNumber smallest_seq = 100, SequenceNumber largest_seq = 100, + size_t compensated_file_size = 0, bool marked_for_compact = false, + Temperature temperature = Temperature::kUnknown, + uint64_t oldest_ancestor_time = kUnknownOldestAncesterTime, + Slice ts_of_smallest = Slice(), Slice ts_of_largest = Slice(), + uint64_t epoch_number = kUnknownEpochNumber) { + assert(ts_of_smallest.size() == ucmp_->timestamp_size()); + assert(ts_of_largest.size() == ucmp_->timestamp_size()); + + VersionStorageInfo* vstorage; + if (temp_vstorage_) { + vstorage = temp_vstorage_.get(); + } else { + vstorage = vstorage_.get(); + } + assert(level < vstorage->num_levels()); + char* smallest_key_buf = nullptr; + char* largest_key_buf = nullptr; + + if (!ts_of_smallest.empty()) { + smallest_key_buf = new char[strlen(smallest) + ucmp_->timestamp_size()]; + memcpy(smallest_key_buf, smallest, strlen(smallest)); + memcpy(smallest_key_buf + strlen(smallest), ts_of_smallest.data(), + ucmp_->timestamp_size()); + largest_key_buf = new char[strlen(largest) + ucmp_->timestamp_size()]; + memcpy(largest_key_buf, largest, strlen(largest)); + memcpy(largest_key_buf + strlen(largest), ts_of_largest.data(), + ucmp_->timestamp_size()); + } + + InternalKey smallest_ikey = InternalKey( + smallest_key_buf ? Slice(smallest_key_buf, + ucmp_->timestamp_size() + strlen(smallest)) + : smallest, + smallest_seq, kTypeValue); + InternalKey largest_ikey = InternalKey( + largest_key_buf + ? Slice(largest_key_buf, ucmp_->timestamp_size() + strlen(largest)) + : largest, + largest_seq, kTypeValue); + + FileMetaData* f = new FileMetaData( + file_number, path_id, file_size, smallest_ikey, largest_ikey, + smallest_seq, largest_seq, marked_for_compact, temperature, + kInvalidBlobFileNumber, kUnknownOldestAncesterTime, + kUnknownFileCreationTime, epoch_number, kUnknownFileChecksum, + kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0, + true /* user_defined_timestamps_persisted */); + f->compensated_file_size = + (compensated_file_size != 0) ? compensated_file_size : file_size; + f->oldest_ancester_time = oldest_ancestor_time; + vstorage->AddFile(level, f); + files_.emplace_back(f); + file_map_.insert({file_number, {f, level}}); + + delete[] smallest_key_buf; + delete[] largest_key_buf; + } + + void SetCompactionInputFilesLevels(int level_count, int start_level) { + input_files_.resize(level_count); + for (int i = 0; i < level_count; ++i) { + input_files_[i].level = start_level + i; + } + compaction_level_start_ = start_level; + } + + void AddToCompactionFiles(uint32_t file_number) { + auto iter = file_map_.find(file_number); + assert(iter != file_map_.end()); + int level = iter->second.second; + assert(level < vstorage_->num_levels()); + input_files_[level - compaction_level_start_].files.emplace_back( + iter->second.first); + } + + void UpdateVersionStorageInfo() { + if (temp_vstorage_) { + VersionBuilder builder(FileOptions(), &ioptions_, nullptr, + vstorage_.get(), nullptr); + ASSERT_OK(builder.SaveTo(temp_vstorage_.get())); + vstorage_ = std::move(temp_vstorage_); + } + vstorage_->PrepareForVersionAppend(ioptions_, mutable_cf_options_); + vstorage_->ComputeCompactionScore(ioptions_, mutable_cf_options_); + vstorage_->SetFinalized(); + } + + private: + Options CreateOptions(const Comparator* ucmp) const { + Options opts; + opts.comparator = ucmp; + return opts; + } + + std::unique_ptr temp_vstorage_; +}; + +class CompactionPickerTest : public CompactionPickerTestBase { + public: + explicit CompactionPickerTest() + : CompactionPickerTestBase(BytewiseComparator()) {} + + ~CompactionPickerTest() override {} +}; + +class CompactionPickerU64TsTest : public CompactionPickerTestBase { + public: + explicit CompactionPickerU64TsTest() + : CompactionPickerTestBase(test::BytewiseComparatorWithU64TsWrapper()) {} + + ~CompactionPickerU64TsTest() override {} +}; + +TEST_F(CompactionPickerTest, Empty) { + NewVersionStorage(6, kCompactionStyleLevel); + UpdateVersionStorageInfo(); + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() == nullptr); +} + +TEST_F(CompactionPickerTest, Single) { + NewVersionStorage(6, kCompactionStyleLevel); + mutable_cf_options_.level0_file_num_compaction_trigger = 2; + Add(0, 1U, "p", "q"); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() == nullptr); +} + +TEST_F(CompactionPickerTest, Level0Trigger) { + NewVersionStorage(6, kCompactionStyleLevel); + mutable_cf_options_.level0_file_num_compaction_trigger = 2; + Add(0, 1U, "150", "200"); + Add(0, 2U, "200", "250"); + + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(2U, compaction->num_input_files(0)); + ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, Level1Trigger) { + NewVersionStorage(6, kCompactionStyleLevel); + Add(1, 66U, "150", "200", 1000000000U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(1U, compaction->num_input_files(0)); + ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, Level1Trigger2) { + mutable_cf_options_.target_file_size_base = 10000000000; + mutable_cf_options_.RefreshDerivedOptions(ioptions_); + NewVersionStorage(6, kCompactionStyleLevel); + Add(1, 66U, "150", "200", 1000000001U); + Add(1, 88U, "201", "300", 1000000000U); + Add(2, 6U, "150", "179", 1000000000U); + Add(2, 7U, "180", "220", 1000000000U); + Add(2, 8U, "221", "300", 1000000000U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(1U, compaction->num_input_files(0)); + ASSERT_EQ(2U, compaction->num_input_files(1)); + ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber()); + ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber()); + ASSERT_EQ(uint64_t{1073741824}, compaction->OutputFilePreallocationSize()); +} + +TEST_F(CompactionPickerTest, LevelMaxScore) { + NewVersionStorage(6, kCompactionStyleLevel); + mutable_cf_options_.target_file_size_base = 10000000; + mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024; + mutable_cf_options_.RefreshDerivedOptions(ioptions_); + Add(0, 1U, "150", "200", 1000000U); + // Level 1 score 1.2 + Add(1, 66U, "150", "200", 6000000U); + Add(1, 88U, "201", "300", 6000000U); + // Level 2 score 1.8. File 7 is the largest. Should be picked + Add(2, 6U, "150", "179", 60000000U); + Add(2, 7U, "180", "220", 60000001U); + Add(2, 8U, "221", "300", 60000000U); + // Level 3 score slightly larger than 1 + Add(3, 26U, "150", "170", 260000000U); + Add(3, 27U, "171", "179", 260000000U); + Add(3, 28U, "191", "220", 260000000U); + Add(3, 29U, "221", "300", 260000000U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(1U, compaction->num_input_files(0)); + ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(mutable_cf_options_.target_file_size_base + + mutable_cf_options_.target_file_size_base / 10, + compaction->OutputFilePreallocationSize()); +} + +TEST_F(CompactionPickerTest, NeedsCompactionLevel) { + const int kLevels = 6; + const int kFileCount = 20; + + for (int level = 0; level < kLevels - 1; ++level) { + NewVersionStorage(kLevels, kCompactionStyleLevel); + uint64_t file_size = vstorage_->MaxBytesForLevel(level) * 2 / kFileCount; + for (int file_count = 1; file_count <= kFileCount; ++file_count) { + // start a brand new version in each test. + NewVersionStorage(kLevels, kCompactionStyleLevel); + for (int i = 0; i < file_count; ++i) { + Add(level, i, std::to_string((i + 100) * 1000).c_str(), + std::to_string((i + 100) * 1000 + 999).c_str(), file_size, 0, + i * 100, i * 100 + 99); + } + UpdateVersionStorageInfo(); + ASSERT_EQ(vstorage_->CompactionScoreLevel(0), level); + ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()), + vstorage_->CompactionScore(0) >= 1); + // release the version storage + DeleteVersionStorage(); + } + } +} + +TEST_F(CompactionPickerTest, Level0TriggerDynamic) { + int num_levels = ioptions_.num_levels; + ioptions_.level_compaction_dynamic_level_bytes = true; + mutable_cf_options_.level0_file_num_compaction_trigger = 2; + mutable_cf_options_.max_bytes_for_level_base = 200; + mutable_cf_options_.max_bytes_for_level_multiplier = 10; + NewVersionStorage(num_levels, kCompactionStyleLevel); + Add(0, 1U, "150", "200"); + Add(0, 2U, "200", "250"); + + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(2U, compaction->num_input_files(0)); + ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); + ASSERT_EQ(1, static_cast(compaction->num_input_levels())); + ASSERT_EQ(num_levels - 1, compaction->output_level()); +} + +TEST_F(CompactionPickerTest, Level0TriggerDynamic2) { + int num_levels = ioptions_.num_levels; + ioptions_.level_compaction_dynamic_level_bytes = true; + mutable_cf_options_.level0_file_num_compaction_trigger = 2; + mutable_cf_options_.max_bytes_for_level_base = 200; + mutable_cf_options_.max_bytes_for_level_multiplier = 10; + NewVersionStorage(num_levels, kCompactionStyleLevel); + Add(0, 1U, "150", "200"); + Add(0, 2U, "200", "250"); + Add(num_levels - 1, 3U, "200", "250", 300U); + + UpdateVersionStorageInfo(); + ASSERT_EQ(vstorage_->base_level(), num_levels - 2); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(2U, compaction->num_input_files(0)); + ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); + ASSERT_EQ(1, static_cast(compaction->num_input_levels())); + ASSERT_EQ(num_levels - 2, compaction->output_level()); +} + +TEST_F(CompactionPickerTest, Level0TriggerDynamic3) { + int num_levels = ioptions_.num_levels; + ioptions_.level_compaction_dynamic_level_bytes = true; + mutable_cf_options_.level0_file_num_compaction_trigger = 2; + mutable_cf_options_.max_bytes_for_level_base = 200; + mutable_cf_options_.max_bytes_for_level_multiplier = 10; + NewVersionStorage(num_levels, kCompactionStyleLevel); + Add(0, 1U, "150", "200"); + Add(0, 2U, "200", "250"); + Add(num_levels - 1, 3U, "200", "250", 300U); + Add(num_levels - 1, 4U, "300", "350", 3000U); + + UpdateVersionStorageInfo(); + ASSERT_EQ(vstorage_->base_level(), num_levels - 3); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(2U, compaction->num_input_files(0)); + ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); + ASSERT_EQ(1, static_cast(compaction->num_input_levels())); + ASSERT_EQ(num_levels - 3, compaction->output_level()); +} + +TEST_F(CompactionPickerTest, Level0TriggerDynamic4) { + int num_levels = ioptions_.num_levels; + ioptions_.level_compaction_dynamic_level_bytes = true; + mutable_cf_options_.level0_file_num_compaction_trigger = 2; + mutable_cf_options_.max_bytes_for_level_base = 200; + mutable_cf_options_.max_bytes_for_level_multiplier = 10; + + NewVersionStorage(num_levels, kCompactionStyleLevel); + Add(0, 1U, "150", "200"); + Add(0, 2U, "200", "250"); + Add(num_levels - 1, 3U, "200", "250", 300U); + Add(num_levels - 1, 4U, "300", "350", 3000U); + Add(num_levels - 3, 5U, "150", "180", 3U); + Add(num_levels - 3, 6U, "181", "300", 3U); + Add(num_levels - 3, 7U, "400", "450", 3U); + + UpdateVersionStorageInfo(); + ASSERT_EQ(vstorage_->base_level(), num_levels - 3); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(2U, compaction->num_input_files(0)); + ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); + ASSERT_EQ(2U, compaction->num_input_files(1)); + ASSERT_EQ(num_levels - 3, compaction->level(1)); + ASSERT_EQ(5U, compaction->input(1, 0)->fd.GetNumber()); + ASSERT_EQ(6U, compaction->input(1, 1)->fd.GetNumber()); + ASSERT_EQ(2, static_cast(compaction->num_input_levels())); + ASSERT_EQ(num_levels - 3, compaction->output_level()); +} + +TEST_F(CompactionPickerTest, LevelTriggerDynamic4) { + int num_levels = ioptions_.num_levels; + ioptions_.level_compaction_dynamic_level_bytes = true; + ioptions_.compaction_pri = kMinOverlappingRatio; + mutable_cf_options_.level0_file_num_compaction_trigger = 2; + mutable_cf_options_.max_bytes_for_level_base = 200; + mutable_cf_options_.max_bytes_for_level_multiplier = 10; + NewVersionStorage(num_levels, kCompactionStyleLevel); + Add(0, 1U, "150", "200"); + Add(num_levels - 1, 2U, "200", "250", 300U); + Add(num_levels - 1, 3U, "300", "350", 3000U); + Add(num_levels - 1, 4U, "400", "450", 3U); + Add(num_levels - 2, 5U, "150", "180", 300U); + Add(num_levels - 2, 6U, "181", "350", 500U); + Add(num_levels - 2, 7U, "400", "450", 200U); + + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(1U, compaction->num_input_files(0)); + ASSERT_EQ(5U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(0, compaction->num_input_files(1)); + ASSERT_EQ(1U, compaction->num_input_levels()); + ASSERT_EQ(num_levels - 1, compaction->output_level()); +} + +TEST_F(CompactionPickerTest, NeedsCompactionUniversal) { + NewVersionStorage(1, kCompactionStyleUniversal); + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + UpdateVersionStorageInfo(); + // must return false when there's no files. + ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()), + false); + + // verify the trigger given different number of L0 files. + for (int i = 1; + i <= mutable_cf_options_.level0_file_num_compaction_trigger * 2; ++i) { + NewVersionStorage(1, kCompactionStyleUniversal); + Add(0, i, std::to_string((i + 100) * 1000).c_str(), + std::to_string((i + 100) * 1000 + 999).c_str(), 1000000, 0, i * 100, + i * 100 + 99); + UpdateVersionStorageInfo(); + ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()), + vstorage_->CompactionScore(0) >= 1); + } +} + +TEST_F(CompactionPickerTest, CompactionUniversalIngestBehindReservedLevel) { + const uint64_t kFileSize = 100000; + NewVersionStorage(3 /* num_levels */, kCompactionStyleUniversal); + ioptions_.allow_ingest_behind = true; + ioptions_.num_levels = 3; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + UpdateVersionStorageInfo(); + // must return false when there's no files. + ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()), + false); + + NewVersionStorage(3, kCompactionStyleUniversal); + + Add(0, 1U, "150", "200", kFileSize, 0, 500, 550); + Add(0, 2U, "201", "250", kFileSize, 0, 401, 450); + Add(0, 4U, "260", "300", kFileSize, 0, 260, 300); + Add(1, 5U, "100", "151", kFileSize, 0, 200, 251); + Add(1, 3U, "301", "350", kFileSize, 0, 101, 150); + Add(2, 6U, "120", "200", kFileSize, 0, 20, 100); + + UpdateVersionStorageInfo(); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + + // output level should be the one above the bottom-most + ASSERT_EQ(1, compaction->output_level()); + + // input should not include the reserved level + const std::vector* inputs = compaction->inputs(); + for (const auto& compaction_input : *inputs) { + if (!compaction_input.empty()) { + ASSERT_LT(compaction_input.level, 2); + } + } +} +// Tests if the files can be trivially moved in multi level +// universal compaction when allow_trivial_move option is set +// In this test as the input files overlaps, they cannot +// be trivially moved. + +TEST_F(CompactionPickerTest, CannotTrivialMoveUniversal) { + const uint64_t kFileSize = 100000; + + mutable_cf_options_.compaction_options_universal.allow_trivial_move = true; + NewVersionStorage(1, kCompactionStyleUniversal); + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + UpdateVersionStorageInfo(); + // must return false when there's no files. + ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()), + false); + + NewVersionStorage(3, kCompactionStyleUniversal); + + Add(0, 1U, "150", "200", kFileSize, 0, 500, 550); + Add(0, 2U, "201", "250", kFileSize, 0, 401, 450); + Add(0, 4U, "260", "300", kFileSize, 0, 260, 300); + Add(1, 5U, "100", "151", kFileSize, 0, 200, 251); + Add(1, 3U, "301", "350", kFileSize, 0, 101, 150); + Add(2, 6U, "120", "200", kFileSize, 0, 20, 100); + + UpdateVersionStorageInfo(); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + + ASSERT_TRUE(!compaction->is_trivial_move()); +} +// Tests if the files can be trivially moved in multi level +// universal compaction when allow_trivial_move option is set +// In this test as the input files doesn't overlaps, they should +// be trivially moved. +TEST_F(CompactionPickerTest, AllowsTrivialMoveUniversal) { + const uint64_t kFileSize = 100000; + + mutable_cf_options_.compaction_options_universal.allow_trivial_move = true; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + + NewVersionStorage(3, kCompactionStyleUniversal); + + Add(0, 1U, "150", "200", kFileSize, 0, 500, 550); + Add(0, 2U, "201", "250", kFileSize, 0, 401, 450); + Add(0, 4U, "260", "300", kFileSize, 0, 260, 300); + Add(1, 5U, "010", "080", kFileSize, 0, 200, 251); + Add(2, 3U, "301", "350", kFileSize, 0, 101, 150); + + UpdateVersionStorageInfo(); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + + ASSERT_TRUE(compaction->is_trivial_move()); +} + +TEST_F(CompactionPickerTest, UniversalPeriodicCompaction1) { + // The case where universal periodic compaction can be picked + // with some newer files being compacted. + const uint64_t kFileSize = 100000; + + mutable_cf_options_.periodic_compaction_seconds = 1000; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + + NewVersionStorage(5, kCompactionStyleUniversal); + + Add(0, 1U, "150", "200", kFileSize, 0, 500, 550); + Add(0, 2U, "201", "250", kFileSize, 0, 401, 450); + Add(0, 4U, "260", "300", kFileSize, 0, 260, 300); + Add(3, 5U, "010", "080", kFileSize, 0, 200, 251); + Add(4, 3U, "301", "350", kFileSize, 0, 101, 150); + Add(4, 6U, "501", "750", kFileSize, 0, 101, 150); + + file_map_[2].first->being_compacted = true; + UpdateVersionStorageInfo(); + vstorage_->TEST_AddFileMarkedForPeriodicCompaction(4, file_map_[3].first); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + + ASSERT_TRUE(compaction); + ASSERT_EQ(4, compaction->output_level()); + ASSERT_EQ(0, compaction->start_level()); + ASSERT_EQ(1U, compaction->num_input_files(0)); +} + +TEST_F(CompactionPickerTest, UniversalPeriodicCompaction2) { + // The case where universal periodic compaction does not + // pick up only level to compact if it doesn't cover + // any file marked as periodic compaction. + const uint64_t kFileSize = 100000; + + mutable_cf_options_.periodic_compaction_seconds = 1000; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + + NewVersionStorage(5, kCompactionStyleUniversal); + + Add(0, 1U, "150", "200", kFileSize, 0, 500, 550); + Add(3, 5U, "010", "080", kFileSize, 0, 200, 251); + Add(4, 3U, "301", "350", kFileSize, 0, 101, 150); + Add(4, 6U, "501", "750", kFileSize, 0, 101, 150); + + file_map_[5].first->being_compacted = true; + UpdateVersionStorageInfo(); + vstorage_->TEST_AddFileMarkedForPeriodicCompaction(0, file_map_[1].first); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + + ASSERT_FALSE(compaction); +} + +TEST_F(CompactionPickerTest, UniversalPeriodicCompaction3) { + // The case where universal periodic compaction does not + // pick up only the last sorted run which is an L0 file if it isn't + // marked as periodic compaction. + const uint64_t kFileSize = 100000; + + mutable_cf_options_.periodic_compaction_seconds = 1000; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + + NewVersionStorage(5, kCompactionStyleUniversal); + + Add(0, 1U, "150", "200", kFileSize, 0, 500, 550); + Add(0, 5U, "010", "080", kFileSize, 0, 200, 251); + Add(0, 6U, "501", "750", kFileSize, 0, 101, 150); + + file_map_[5].first->being_compacted = true; + UpdateVersionStorageInfo(); + vstorage_->TEST_AddFileMarkedForPeriodicCompaction(0, file_map_[1].first); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + + ASSERT_FALSE(compaction); +} + +TEST_F(CompactionPickerTest, UniversalPeriodicCompaction4) { + // The case where universal periodic compaction couldn't form + // a compaction that includes any file marked for periodic compaction. + // Right now we form the compaction anyway if it is more than one + // sorted run. Just put the case here to validate that it doesn't + // crash. + const uint64_t kFileSize = 100000; + + mutable_cf_options_.periodic_compaction_seconds = 1000; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + + NewVersionStorage(5, kCompactionStyleUniversal); + + Add(0, 1U, "150", "200", kFileSize, 0, 500, 550); + Add(2, 2U, "010", "080", kFileSize, 0, 200, 251); + Add(3, 5U, "010", "080", kFileSize, 0, 200, 251); + Add(4, 3U, "301", "350", kFileSize, 0, 101, 150); + Add(4, 6U, "501", "750", kFileSize, 0, 101, 150); + + file_map_[2].first->being_compacted = true; + UpdateVersionStorageInfo(); + vstorage_->TEST_AddFileMarkedForPeriodicCompaction(0, file_map_[2].first); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(!compaction || + compaction->start_level() != compaction->output_level()); +} + +TEST_F(CompactionPickerTest, UniversalPeriodicCompaction5) { + // Test single L0 file periodic compaction triggering. + const uint64_t kFileSize = 100000; + + mutable_cf_options_.periodic_compaction_seconds = 1000; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + + NewVersionStorage(5, kCompactionStyleUniversal); + + Add(0, 6U, "150", "200", kFileSize, 0, 500, 550); + UpdateVersionStorageInfo(); + vstorage_->TEST_AddFileMarkedForPeriodicCompaction(0, file_map_[6].first); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction); + ASSERT_EQ(0, compaction->start_level()); + ASSERT_EQ(1U, compaction->num_input_files(0)); + ASSERT_EQ(6U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(4, compaction->output_level()); +} + +TEST_F(CompactionPickerTest, UniversalPeriodicCompaction6) { + // Test single sorted run non-L0 periodic compaction + const uint64_t kFileSize = 100000; + + mutable_cf_options_.periodic_compaction_seconds = 1000; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + + NewVersionStorage(5, kCompactionStyleUniversal); + + Add(4, 5U, "150", "200", kFileSize, 0, 500, 550); + Add(4, 6U, "350", "400", kFileSize, 0, 500, 550); + UpdateVersionStorageInfo(); + vstorage_->TEST_AddFileMarkedForPeriodicCompaction(4, file_map_[6].first); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction); + ASSERT_EQ(4, compaction->start_level()); + ASSERT_EQ(2U, compaction->num_input_files(0)); + ASSERT_EQ(5U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(6U, compaction->input(0, 1)->fd.GetNumber()); + ASSERT_EQ(4, compaction->output_level()); +} + +TEST_F(CompactionPickerTest, UniversalIncrementalSpace1) { + const uint64_t kFileSize = 100000; + + mutable_cf_options_.max_compaction_bytes = 555555; + mutable_cf_options_.compaction_options_universal.incremental = true; + mutable_cf_options_.compaction_options_universal + .max_size_amplification_percent = 30; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + + NewVersionStorage(5, kCompactionStyleUniversal); + + Add(0, 1U, "150", "200", kFileSize, 0, 500, 550); + Add(2, 2U, "010", "080", kFileSize, 0, 200, 251); + Add(3, 5U, "310", "380", kFileSize, 0, 200, 251); + Add(3, 6U, "410", "880", kFileSize, 0, 200, 251); + Add(3, 7U, "910", "980", 1, 0, 200, 251); + Add(4, 10U, "201", "250", kFileSize, 0, 101, 150); + Add(4, 11U, "301", "350", kFileSize, 0, 101, 150); + Add(4, 12U, "401", "450", kFileSize, 0, 101, 150); + Add(4, 13U, "501", "750", kFileSize, 0, 101, 150); + Add(4, 14U, "801", "850", kFileSize, 0, 101, 150); + Add(4, 15U, "901", "950", kFileSize, 0, 101, 150); + // Add(4, 15U, "960", "970", kFileSize, 0, 101, 150); + + UpdateVersionStorageInfo(); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction); + ASSERT_EQ(4, compaction->output_level()); + ASSERT_EQ(3, compaction->start_level()); + ASSERT_EQ(2U, compaction->num_input_files(0)); + ASSERT_EQ(5U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(6U, compaction->input(0, 1)->fd.GetNumber()); + // ASSERT_EQ(4U, compaction->num_input_files(1)); + ASSERT_EQ(11U, compaction->input(1, 0)->fd.GetNumber()); + ASSERT_EQ(12U, compaction->input(1, 1)->fd.GetNumber()); + ASSERT_EQ(13U, compaction->input(1, 2)->fd.GetNumber()); + ASSERT_EQ(14U, compaction->input(1, 3)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, UniversalIncrementalSpace2) { + const uint64_t kFileSize = 100000; + + mutable_cf_options_.max_compaction_bytes = 400000; + mutable_cf_options_.compaction_options_universal.incremental = true; + mutable_cf_options_.compaction_options_universal + .max_size_amplification_percent = 30; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + + NewVersionStorage(5, kCompactionStyleUniversal); + + Add(0, 1U, "150", "200", kFileSize, 0, 500, 550); + Add(1, 2U, "010", "080", kFileSize, 0, 200, 251); + Add(2, 5U, "310", "380", kFileSize, 0, 200, 251); + Add(2, 6U, "410", "880", kFileSize, 0, 200, 251); + Add(2, 7U, "910", "980", kFileSize, 0, 200, 251); + Add(4, 10U, "201", "250", kFileSize, 0, 101, 150); + Add(4, 11U, "301", "350", kFileSize, 0, 101, 150); + Add(4, 12U, "401", "450", kFileSize, 0, 101, 150); + Add(4, 13U, "501", "750", kFileSize, 0, 101, 150); + Add(4, 14U, "801", "850", kFileSize, 0, 101, 150); + Add(4, 15U, "901", "950", kFileSize, 0, 101, 150); + + UpdateVersionStorageInfo(); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction); + ASSERT_EQ(4, compaction->output_level()); + ASSERT_EQ(2, compaction->start_level()); + ASSERT_EQ(1U, compaction->num_input_files(0)); + ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(1U, compaction->num_input_files(1)); + ASSERT_EQ(15U, compaction->input(1, 0)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, UniversalIncrementalSpace3) { + // Test bottom level files falling between gaps between two upper level + // files + const uint64_t kFileSize = 100000; + + mutable_cf_options_.max_compaction_bytes = 300000; + mutable_cf_options_.compaction_options_universal.incremental = true; + mutable_cf_options_.compaction_options_universal + .max_size_amplification_percent = 30; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + + NewVersionStorage(5, kCompactionStyleUniversal); + + Add(0, 1U, "150", "200", kFileSize, 0, 500, 550); + Add(2, 2U, "010", "080", kFileSize, 0, 200, 251); + Add(3, 5U, "000", "180", kFileSize, 0, 200, 251); + Add(3, 6U, "181", "190", kFileSize, 0, 200, 251); + Add(3, 7U, "710", "810", kFileSize, 0, 200, 251); + Add(3, 8U, "820", "830", kFileSize, 0, 200, 251); + Add(3, 9U, "900", "991", kFileSize, 0, 200, 251); + Add(4, 10U, "201", "250", kFileSize, 0, 101, 150); + Add(4, 11U, "301", "350", kFileSize, 0, 101, 150); + Add(4, 12U, "401", "450", kFileSize, 0, 101, 150); + Add(4, 13U, "501", "750", kFileSize, 0, 101, 150); + Add(4, 14U, "801", "850", kFileSize, 0, 101, 150); + Add(4, 15U, "901", "950", kFileSize, 0, 101, 150); + + UpdateVersionStorageInfo(); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction); + ASSERT_EQ(4, compaction->output_level()); + ASSERT_EQ(2, compaction->start_level()); + ASSERT_EQ(1U, compaction->num_input_files(0)); + ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(2U, compaction->num_input_files(1)); + ASSERT_EQ(5U, compaction->input(1, 0)->fd.GetNumber()); + ASSERT_EQ(6U, compaction->input(1, 1)->fd.GetNumber()); + ASSERT_EQ(0, compaction->num_input_files(2)); +} + +TEST_F(CompactionPickerTest, UniversalIncrementalSpace4) { + // Test compaction candidates always cover many files. + const uint64_t kFileSize = 100000; + + mutable_cf_options_.max_compaction_bytes = 3200000; + mutable_cf_options_.compaction_options_universal.incremental = true; + mutable_cf_options_.compaction_options_universal + .max_size_amplification_percent = 30; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + + NewVersionStorage(5, kCompactionStyleUniversal); + + Add(0, 1U, "150", "200", kFileSize, 0, 500, 550); + Add(2, 2U, "010", "080", kFileSize, 0, 200, 251); + + // Generate files like following: + // L3: (1101, 1180) (1201, 1280) ... (7901, 7908) + // L4: (1130, 1150) (1160, 1210) (1230, 1250) (1260 1310) ... (7960, 8010) + for (int i = 11; i < 79; i++) { + Add(3, 100 + i * 3, std::to_string(i * 100).c_str(), + std::to_string(i * 100 + 80).c_str(), kFileSize, 0, 200, 251); + // Add a tie breaker + if (i == 66) { + Add(3, 10000U, "6690", "6699", kFileSize, 0, 200, 251); + } + + Add(4, 100 + i * 3 + 1, std::to_string(i * 100 + 30).c_str(), + std::to_string(i * 100 + 50).c_str(), kFileSize, 0, 200, 251); + Add(4, 100 + i * 3 + 2, std::to_string(i * 100 + 60).c_str(), + std::to_string(i * 100 + 110).c_str(), kFileSize, 0, 200, 251); + } + UpdateVersionStorageInfo(); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction); + ASSERT_EQ(4, compaction->output_level()); + ASSERT_EQ(3, compaction->start_level()); + ASSERT_EQ(6U, compaction->num_input_files(0)); + ASSERT_EQ(100 + 62U * 3, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(10000U, compaction->input(0, 5)->fd.GetNumber()); + ASSERT_EQ(11, compaction->num_input_files(1)); +} + +TEST_F(CompactionPickerTest, UniversalIncrementalSpace5) { + // Test compaction candidates always cover many files with some single + // files larger than size threshold. + const uint64_t kFileSize = 100000; + + mutable_cf_options_.max_compaction_bytes = 3200000; + mutable_cf_options_.compaction_options_universal.incremental = true; + mutable_cf_options_.compaction_options_universal + .max_size_amplification_percent = 30; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + + NewVersionStorage(5, kCompactionStyleUniversal); + + Add(0, 1U, "150", "200", kFileSize, 0, 500, 550); + Add(2, 2U, "010", "080", kFileSize, 0, 200, 251); + + // Generate files like following: + // L3: (1101, 1180) (1201, 1280) ... (7901, 7908) + // L4: (1130, 1150) (1160, 1210) (1230, 1250) (1260 1310) ... (7960, 8010) + for (int i = 11; i < 70; i++) { + Add(3, 100 + i * 3, std::to_string(i * 100).c_str(), + std::to_string(i * 100 + 80).c_str(), + i % 10 == 9 ? kFileSize * 100 : kFileSize, 0, 200, 251); + + Add(4, 100 + i * 3 + 1, std::to_string(i * 100 + 30).c_str(), + std::to_string(i * 100 + 50).c_str(), kFileSize, 0, 200, 251); + Add(4, 100 + i * 3 + 2, std::to_string(i * 100 + 60).c_str(), + std::to_string(i * 100 + 110).c_str(), kFileSize, 0, 200, 251); + } + UpdateVersionStorageInfo(); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction); + ASSERT_EQ(4, compaction->output_level()); + ASSERT_EQ(3, compaction->start_level()); + ASSERT_EQ(6U, compaction->num_input_files(0)); + ASSERT_EQ(100 + 14 * 3, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(100 + 19 * 3, compaction->input(0, 5)->fd.GetNumber()); + ASSERT_EQ(13, compaction->num_input_files(1)); +} + +TEST_F(CompactionPickerTest, NeedsCompactionFIFO) { + NewVersionStorage(1, kCompactionStyleFIFO); + const int kFileCount = + mutable_cf_options_.level0_file_num_compaction_trigger * 3; + const uint64_t kFileSize = 100000; + const uint64_t kMaxSize = kFileSize * kFileCount / 2; + + fifo_options_.max_table_files_size = kMaxSize; + mutable_cf_options_.compaction_options_fifo = fifo_options_; + FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_); + UpdateVersionStorageInfo(); + // must return false when there's no files. + ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), false); + + // verify whether compaction is needed based on the current + // size of L0 files. + for (int i = 1; i <= kFileCount; ++i) { + NewVersionStorage(1, kCompactionStyleFIFO); + Add(0, i, std::to_string((i + 100) * 1000).c_str(), + std::to_string((i + 100) * 1000 + 999).c_str(), kFileSize, 0, i * 100, + i * 100 + 99); + UpdateVersionStorageInfo(); + ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), + vstorage_->CompactionScore(0) >= 1); + } +} + +TEST_F(CompactionPickerTest, FIFOToCold1) { + NewVersionStorage(1, kCompactionStyleFIFO); + const uint64_t kFileSize = 100000; + const uint64_t kMaxSize = kFileSize * 100000; + uint64_t kColdThreshold = 2000; + + fifo_options_.max_table_files_size = kMaxSize; + fifo_options_.file_temperature_age_thresholds = { + {Temperature::kCold, kColdThreshold}}; + mutable_cf_options_.compaction_options_fifo = fifo_options_; + mutable_cf_options_.level0_file_num_compaction_trigger = 100; + mutable_cf_options_.max_compaction_bytes = kFileSize * 100; + FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_); + + int64_t current_time = 0; + ASSERT_OK(Env::Default()->GetCurrentTime(¤t_time)); + uint64_t threshold_time = + static_cast(current_time) - kColdThreshold; + Add(0 /* level */, 4U /* file_number */, "260", "300", 1 * kFileSize, 0, 2500, + 2600, 0, true, Temperature::kUnknown, + threshold_time - 2000 /* oldest_ancestor_time */); + // Qualifies for compaction to kCold. + Add(0, 3U, "200", "300", 4 * kFileSize, 0, 2300, 2400, 0, true, + Temperature::kUnknown, threshold_time - 3000); + UpdateVersionStorageInfo(); + + ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), true); + std::unique_ptr compaction(fifo_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(compaction->compaction_reason(), + CompactionReason::kChangeTemperature); + ASSERT_EQ(compaction->output_temperature(), Temperature::kCold); + ASSERT_EQ(1U, compaction->num_input_files(0)); + ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, FIFOToCold2) { + NewVersionStorage(1, kCompactionStyleFIFO); + const uint64_t kFileSize = 100000; + const uint64_t kMaxSize = kFileSize * 100000; + uint64_t kColdThreshold = 2000; + + fifo_options_.max_table_files_size = kMaxSize; + fifo_options_.file_temperature_age_thresholds = { + {Temperature::kCold, kColdThreshold}}; + mutable_cf_options_.compaction_options_fifo = fifo_options_; + mutable_cf_options_.level0_file_num_compaction_trigger = 100; + mutable_cf_options_.max_compaction_bytes = kFileSize * 100; + FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_); + + int64_t current_time = 0; + ASSERT_OK(Env::Default()->GetCurrentTime(¤t_time)); + uint64_t threshold_time = + static_cast(current_time) - kColdThreshold; + Add(0, 6U, "240", "290", 2 * kFileSize, 0, 2900, 3000, 0, true, + Temperature::kUnknown, static_cast(current_time) - 100); + Add(0, 4U, "260", "300", 1 * kFileSize, 0, 2500, 2600, 0, true, + Temperature::kUnknown, threshold_time); + // The following two files qualify for compaction to kCold. + Add(0, 3U, "200", "300", 4 * kFileSize, 0, 2300, 2400, 0, true, + Temperature::kUnknown, threshold_time - 3000); + Add(0, 2U, "200", "300", 4 * kFileSize, 0, 2100, 2200, 0, true, + Temperature::kUnknown, threshold_time - 4000); + UpdateVersionStorageInfo(); + + ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), true); + std::unique_ptr compaction(fifo_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(compaction->compaction_reason(), + CompactionReason::kChangeTemperature); + ASSERT_EQ(compaction->output_temperature(), Temperature::kCold); + ASSERT_EQ(2U, compaction->num_input_files(0)); + ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(3U, compaction->input(0, 1)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, FIFOToColdMaxCompactionSize) { + NewVersionStorage(1, kCompactionStyleFIFO); + const uint64_t kFileSize = 100000; + const uint64_t kMaxSize = kFileSize * 100000; + uint64_t kColdThreshold = 2000; + + fifo_options_.max_table_files_size = kMaxSize; + fifo_options_.file_temperature_age_thresholds = { + {Temperature::kCold, kColdThreshold}}; + mutable_cf_options_.compaction_options_fifo = fifo_options_; + mutable_cf_options_.level0_file_num_compaction_trigger = 100; + mutable_cf_options_.max_compaction_bytes = kFileSize * 9; + FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_); + + int64_t current_time = 0; + ASSERT_OK(Env::Default()->GetCurrentTime(¤t_time)); + uint64_t threshold_time = + static_cast(current_time) - kColdThreshold; + Add(0, 6U, "240", "290", 2 * kFileSize, 0, 2900, 3000, 0, true, + Temperature::kUnknown, static_cast(current_time) - 100); + Add(0, 5U, "240", "290", 2 * kFileSize, 0, 2700, 2800, 0, true, + Temperature::kUnknown, threshold_time + 100); + Add(0, 4U, "260", "300", 1 * kFileSize, 0, 2500, 2600, 0, true, + Temperature::kUnknown, threshold_time - 2000); + // The following two files qualify for compaction to kCold. + // But only the last two should be included to respect `max_compaction_bytes`. + Add(0, 3U, "200", "300", 4 * kFileSize, 0, 2300, 2400, 0, true, + Temperature::kUnknown, threshold_time - 3000); + Add(0, 2U, "200", "300", 4 * kFileSize, 0, 2100, 2200, 0, true, + Temperature::kUnknown, threshold_time - 4000); + Add(0, 1U, "200", "300", 4 * kFileSize, 0, 2000, 2100, 0, true, + Temperature::kUnknown, threshold_time - 5000); + UpdateVersionStorageInfo(); + + ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), true); + std::unique_ptr compaction(fifo_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(compaction->compaction_reason(), + CompactionReason::kChangeTemperature); + ASSERT_EQ(compaction->output_temperature(), Temperature::kCold); + ASSERT_EQ(2U, compaction->num_input_files(0)); + ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, FIFOToColdWithExistingCold) { + NewVersionStorage(1, kCompactionStyleFIFO); + const uint64_t kFileSize = 100000; + const uint64_t kMaxSize = kFileSize * 100000; + uint64_t kColdThreshold = 2000; + + fifo_options_.max_table_files_size = kMaxSize; + fifo_options_.file_temperature_age_thresholds = { + {Temperature::kCold, kColdThreshold}}; + mutable_cf_options_.compaction_options_fifo = fifo_options_; + mutable_cf_options_.level0_file_num_compaction_trigger = 100; + mutable_cf_options_.max_compaction_bytes = kFileSize * 100; + FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_); + + int64_t current_time = 0; + ASSERT_OK(Env::Default()->GetCurrentTime(¤t_time)); + uint64_t threshold_time = + static_cast(current_time) - kColdThreshold; + Add(0, 6U, "240", "290", 2 * kFileSize, 0, 2900, 3000, 0, true, + Temperature::kUnknown, static_cast(current_time) - 100); + Add(0, 5U, "240", "290", 2 * kFileSize, 0, 2700, 2800, 0, true, + Temperature::kUnknown, threshold_time + 100); + Add(0, 4U, "260", "300", 1 * kFileSize, 0, 2500, 2600, 0, true, + Temperature::kUnknown, threshold_time - 2000); + // The following two files qualify for compaction to kCold. + Add(0, 3U, "200", "300", 4 * kFileSize, 0, 2300, 2400, 0, true, + Temperature::kUnknown, threshold_time - 3000); + Add(0, 2U, "200", "300", 4 * kFileSize, 0, 2100, 2200, 0, true, + Temperature::kUnknown, threshold_time - 4000); + Add(0, 1U, "200", "300", 4 * kFileSize, 0, 2000, 2100, 0, true, + Temperature::kCold, threshold_time - 5000); + UpdateVersionStorageInfo(); + + ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), true); + std::unique_ptr compaction(fifo_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(compaction->compaction_reason(), + CompactionReason::kChangeTemperature); + ASSERT_EQ(compaction->output_temperature(), Temperature::kCold); + ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(2U, compaction->num_input_files(0)); + ASSERT_EQ(3U, compaction->input(0, 1)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, FIFOToColdWithHotBetweenCold) { + NewVersionStorage(1, kCompactionStyleFIFO); + const uint64_t kFileSize = 100000; + const uint64_t kMaxSize = kFileSize * 100000; + uint64_t kColdThreshold = 2000; + + fifo_options_.max_table_files_size = kMaxSize; + fifo_options_.file_temperature_age_thresholds = { + {Temperature::kCold, kColdThreshold}}; + mutable_cf_options_.compaction_options_fifo = fifo_options_; + mutable_cf_options_.level0_file_num_compaction_trigger = 100; + mutable_cf_options_.max_compaction_bytes = kFileSize * 100; + FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_); + + int64_t current_time = 0; + ASSERT_OK(Env::Default()->GetCurrentTime(¤t_time)); + uint64_t threshold_time = + static_cast(current_time) - kColdThreshold; + Add(0, 6U, "240", "290", 2 * kFileSize, 0, 2900, 3000, 0, true, + Temperature::kUnknown, static_cast(current_time) - 100); + Add(0, 5U, "240", "290", 2 * kFileSize, 0, 2700, 2800, 0, true, + Temperature::kUnknown, threshold_time + 100); + Add(0, 4U, "260", "300", 1 * kFileSize, 0, 2500, 2600, 0, true, + Temperature::kUnknown, threshold_time - 2000); + Add(0, 3U, "200", "300", 4 * kFileSize, 0, 2300, 2400, 0, true, + Temperature::kCold, threshold_time - 3000); + // Qualifies for compaction to kCold. + Add(0, 2U, "200", "300", 4 * kFileSize, 0, 2100, 2200, 0, true, + Temperature::kUnknown, threshold_time - 4000); + Add(0, 1U, "200", "300", 4 * kFileSize, 0, 2000, 2100, 0, true, + Temperature::kCold, threshold_time - 5000); + UpdateVersionStorageInfo(); + + ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), true); + std::unique_ptr compaction(fifo_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(compaction->compaction_reason(), + CompactionReason::kChangeTemperature); + ASSERT_EQ(compaction->output_temperature(), Temperature::kCold); + ASSERT_EQ(1U, compaction->num_input_files(0)); + ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, FIFOToColdAndWarm) { + NewVersionStorage(1, kCompactionStyleFIFO); + const uint64_t kFileSize = 100000; + const uint64_t kMaxSize = kFileSize * 100000; + uint64_t kWarmThreshold = 10000; + uint64_t kHotThreshold = 2000; + + fifo_options_.max_table_files_size = kMaxSize; + // Test that multiple threshold works. + fifo_options_.file_temperature_age_thresholds = { + {Temperature::kHot, kHotThreshold}, {Temperature::kWarm, kWarmThreshold}}; + mutable_cf_options_.compaction_options_fifo = fifo_options_; + mutable_cf_options_.level0_file_num_compaction_trigger = 100; + mutable_cf_options_.max_compaction_bytes = kFileSize * 100; + FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_); + + int64_t current_time = 0; + ASSERT_OK(Env::Default()->GetCurrentTime(¤t_time)); + uint64_t hot_threshold_time = + static_cast(current_time) - kHotThreshold; + uint64_t warm_threshold_time = + static_cast(current_time) - kWarmThreshold; + Add(0, 6U, "240", "290", 2 * kFileSize, 0, 2900, 3000, 0, true, + Temperature::kUnknown, static_cast(current_time) - 100); + Add(0, 5U, "240", "290", 2 * kFileSize, 0, 2700, 2800, 0, true, + Temperature::kUnknown, hot_threshold_time + 100); + Add(0, 4U, "260", "300", 1 * kFileSize, 0, 2500, 2600, 0, true, + Temperature::kUnknown, hot_threshold_time - 200); + // Qualifies for Hot + Add(0, 3U, "200", "300", 4 * kFileSize, 0, 2300, 2400, 0, true, + Temperature::kUnknown, warm_threshold_time - 100); + // Qualifies for Warm + Add(0, 2U, "200", "300", 4 * kFileSize, 0, 2100, 2200, 0, true, + Temperature::kUnknown, warm_threshold_time - 4000); + Add(0, 1U, "200", "300", 4 * kFileSize, 0, 2000, 2100, 0, true, + Temperature::kUnknown, warm_threshold_time - 5000); + UpdateVersionStorageInfo(); + + ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), true); + std::unique_ptr compaction(fifo_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(compaction->compaction_reason(), + CompactionReason::kChangeTemperature); + // Assumes compaction picker picks older files first. + ASSERT_EQ(compaction->output_temperature(), Temperature::kWarm); + ASSERT_EQ(2U, compaction->num_input_files(0)); + ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) { + NewVersionStorage(6, kCompactionStyleLevel); + ioptions_.compaction_pri = kMinOverlappingRatio; + mutable_cf_options_.target_file_size_base = 100000000000; + mutable_cf_options_.target_file_size_multiplier = 10; + mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024; + mutable_cf_options_.RefreshDerivedOptions(ioptions_); + + Add(2, 6U, "150", "179", 50000000U); + Add(2, 7U, "180", "220", 50000000U); + Add(2, 8U, "321", "400", 50000000U); // File not overlapping + Add(2, 9U, "721", "800", 50000000U); + + Add(3, 26U, "150", "170", 260000000U); + Add(3, 27U, "171", "179", 260000000U); + Add(3, 28U, "191", "220", 260000000U); + Add(3, 29U, "221", "300", 260000000U); + Add(3, 30U, "750", "900", 260000000U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(1U, compaction->num_input_files(0)); + // Pick file 8 because it overlaps with 0 files on level 3. + ASSERT_EQ(8U, compaction->input(0, 0)->fd.GetNumber()); + // Compaction input size * 1.1 + ASSERT_GE(uint64_t{55000000}, compaction->OutputFilePreallocationSize()); +} + +TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) { + NewVersionStorage(6, kCompactionStyleLevel); + ioptions_.compaction_pri = kMinOverlappingRatio; + mutable_cf_options_.target_file_size_base = 10000000; + mutable_cf_options_.target_file_size_multiplier = 10; + mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024; + + Add(2, 6U, "150", "175", + 60000000U); // Overlaps with file 26, 27, total size 521M + Add(2, 7U, "176", "200", 60000000U); // Overlaps with file 27, 28, total size + // 520M, the smallest overlapping + Add(2, 8U, "201", "300", + 60000000U); // Overlaps with file 28, 29, total size 521M + + Add(3, 25U, "100", "110", 261000000U); + Add(3, 26U, "150", "170", 261000000U); + Add(3, 27U, "171", "179", 260000000U); + Add(3, 28U, "191", "220", 260000000U); + Add(3, 29U, "221", "300", 261000000U); + Add(3, 30U, "321", "400", 261000000U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(1U, compaction->num_input_files(0)); + // Picking file 7 because overlapping ratio is the biggest. + ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, CompactionPriMinOverlapping3) { + NewVersionStorage(6, kCompactionStyleLevel); + ioptions_.compaction_pri = kMinOverlappingRatio; + mutable_cf_options_.max_bytes_for_level_base = 10000000; + mutable_cf_options_.max_bytes_for_level_multiplier = 10; + + // file 7 and 8 over lap with the same file, but file 8 is smaller so + // it will be picked. + Add(2, 6U, "150", "167", 60000000U); // Overlaps with file 26, 27 + Add(2, 7U, "168", "169", 60000000U); // Overlaps with file 27 + Add(2, 8U, "201", "300", 61000000U); // Overlaps with file 28, but the file + // itself is larger. Should be picked. + + Add(3, 26U, "160", "165", 260000000U); + Add(3, 27U, "166", "170", 260000000U); + Add(3, 28U, "180", "400", 260000000U); + Add(3, 29U, "401", "500", 260000000U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(1U, compaction->num_input_files(0)); + // Picking file 8 because overlapping ratio is the biggest. + ASSERT_EQ(8U, compaction->input(0, 0)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, CompactionPriMinOverlapping4) { + NewVersionStorage(6, kCompactionStyleLevel); + ioptions_.compaction_pri = kMinOverlappingRatio; + mutable_cf_options_.max_bytes_for_level_base = 10000000; + mutable_cf_options_.max_bytes_for_level_multiplier = 10; + mutable_cf_options_.ignore_max_compaction_bytes_for_input = false; + + // file 7 and 8 over lap with the same file, but file 8 is smaller so + // it will be picked. + // Overlaps with file 26, 27. And the file is compensated so will be + // picked up. + Add(2, 6U, "150", "167", 60000000U, 0, 100, 100, 180000000U); + Add(2, 7U, "168", "169", 60000000U); // Overlaps with file 27 + Add(2, 8U, "201", "300", 61000000U); // Overlaps with file 28 + + Add(3, 26U, "160", "165", 60000000U); + // Boosted file size in output level is not considered. + Add(3, 27U, "166", "170", 60000000U, 0, 100, 100, 260000000U); + Add(3, 28U, "180", "400", 60000000U); + Add(3, 29U, "401", "500", 60000000U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(1U, compaction->num_input_files(0)); + // Picking file 8 because overlapping ratio is the biggest. + ASSERT_EQ(6U, compaction->input(0, 0)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, CompactionPriRoundRobin) { + std::vector test_cursors = {InternalKey("249", 100, kTypeValue), + InternalKey("600", 100, kTypeValue), + InternalKey()}; + std::vector selected_files = {8U, 6U, 6U}; + + ioptions_.compaction_pri = kRoundRobin; + mutable_cf_options_.max_bytes_for_level_base = 12000000; + mutable_cf_options_.max_bytes_for_level_multiplier = 10; + for (size_t i = 0; i < test_cursors.size(); i++) { + // start a brand new version in each test. + NewVersionStorage(6, kCompactionStyleLevel); + vstorage_->ResizeCompactCursors(6); + // Set the cursor + vstorage_->AddCursorForOneLevel(2, test_cursors[i]); + Add(2, 6U, "150", "199", 50000000U); // Overlap with 26U, 27U + Add(2, 7U, "200", "249", 50000000U); // File not overlapping + Add(2, 8U, "300", "600", 50000000U); // Overlap with 28U, 29U + + Add(3, 26U, "130", "165", 60000000U); + Add(3, 27U, "166", "170", 60000000U); + Add(3, 28U, "270", "340", 60000000U); + Add(3, 29U, "401", "500", 60000000U); + UpdateVersionStorageInfo(); + LevelCompactionPicker local_level_compaction_picker = + LevelCompactionPicker(ioptions_, &icmp_); + std::unique_ptr compaction( + local_level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + // Since the max bytes for level 2 is 120M, picking one file to compact + // makes the post-compaction level size less than 120M, there is exactly one + // file picked for round-robin compaction + ASSERT_EQ(1U, compaction->num_input_files(0)); + ASSERT_EQ(selected_files[i], compaction->input(0, 0)->fd.GetNumber()); + // release the version storage + DeleteVersionStorage(); + } +} + +TEST_F(CompactionPickerTest, CompactionPriMultipleFilesRoundRobin1) { + ioptions_.compaction_pri = kRoundRobin; + mutable_cf_options_.max_compaction_bytes = 100000000u; + mutable_cf_options_.max_bytes_for_level_base = 120; + mutable_cf_options_.max_bytes_for_level_multiplier = 10; + // start a brand new version in each test. + NewVersionStorage(6, kCompactionStyleLevel); + vstorage_->ResizeCompactCursors(6); + // Set the cursor (file picking should start with 7U) + vstorage_->AddCursorForOneLevel(2, InternalKey("199", 100, kTypeValue)); + Add(2, 6U, "150", "199", 500U); + Add(2, 7U, "200", "249", 500U); + Add(2, 8U, "300", "600", 500U); + Add(2, 9U, "700", "800", 500U); + Add(2, 10U, "850", "950", 500U); + + Add(3, 26U, "130", "165", 600U); + Add(3, 27U, "166", "170", 600U); + Add(3, 28U, "270", "340", 600U); + Add(3, 29U, "401", "500", 600U); + Add(3, 30U, "601", "800", 600U); + Add(3, 31U, "830", "890", 600U); + UpdateVersionStorageInfo(); + LevelCompactionPicker local_level_compaction_picker = + LevelCompactionPicker(ioptions_, &icmp_); + std::unique_ptr compaction( + local_level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + + // The maximum compaction bytes is very large in this case so we can igore its + // constraint in this test case. The maximum bytes for level 2 is 1200 + // bytes, and thus at least 3 files should be picked so that the bytes in + // level 2 is less than the maximum + ASSERT_EQ(3U, compaction->num_input_files(0)); + ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(8U, compaction->input(0, 1)->fd.GetNumber()); + ASSERT_EQ(9U, compaction->input(0, 2)->fd.GetNumber()); + // release the version storage + DeleteVersionStorage(); +} + +TEST_F(CompactionPickerTest, CompactionPriMultipleFilesRoundRobin2) { + ioptions_.compaction_pri = kRoundRobin; + mutable_cf_options_.max_compaction_bytes = 2500u; + mutable_cf_options_.max_bytes_for_level_base = 120; + mutable_cf_options_.max_bytes_for_level_multiplier = 10; + // start a brand new version in each test. + NewVersionStorage(6, kCompactionStyleLevel); + vstorage_->ResizeCompactCursors(6); + // Set the cursor (file picking should start with 6U) + vstorage_->AddCursorForOneLevel(2, InternalKey("1000", 100, kTypeValue)); + Add(2, 6U, "150", "199", 500U); // Overlap with 26U, 27U + Add(2, 7U, "200", "249", 500U); // Overlap with 27U + Add(2, 8U, "300", "600", 500U); // Overlap with 28U, 29U + Add(2, 9U, "700", "800", 500U); + Add(2, 10U, "850", "950", 500U); + + Add(3, 26U, "130", "165", 600U); + Add(3, 27U, "166", "230", 600U); + Add(3, 28U, "270", "340", 600U); + Add(3, 29U, "401", "500", 600U); + Add(3, 30U, "601", "800", 600U); + Add(3, 31U, "830", "890", 600U); + UpdateVersionStorageInfo(); + LevelCompactionPicker local_level_compaction_picker = + LevelCompactionPicker(ioptions_, &icmp_); + std::unique_ptr compaction( + local_level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + + // The maximum compaction bytes is only 2500 bytes now. Even though we are + // required to choose 3 files so that the post-compaction level size is less + // than 1200 bytes. We cannot pick 3 files to compact since the maximum + // compaction size is 2500. After picking files 6U and 7U, the number of + // compaction bytes has reached 2200, and thus no more space to add another + // input file with 50M bytes. + ASSERT_EQ(2U, compaction->num_input_files(0)); + ASSERT_EQ(6U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(7U, compaction->input(0, 1)->fd.GetNumber()); + // release the version storage + DeleteVersionStorage(); +} + +TEST_F(CompactionPickerTest, CompactionPriMultipleFilesRoundRobin3) { + ioptions_.compaction_pri = kRoundRobin; + mutable_cf_options_.max_compaction_bytes = 1000000u; + mutable_cf_options_.max_bytes_for_level_base = 120; + mutable_cf_options_.max_bytes_for_level_multiplier = 10; + // start a brand new version in each test. + NewVersionStorage(6, kCompactionStyleLevel); + vstorage_->ResizeCompactCursors(6); + // Set the cursor (file picking should start with 9U) + vstorage_->AddCursorForOneLevel(2, InternalKey("700", 100, kTypeValue)); + Add(2, 6U, "150", "199", 500U); + Add(2, 7U, "200", "249", 500U); + Add(2, 8U, "300", "600", 500U); + Add(2, 9U, "700", "800", 500U); + Add(2, 10U, "850", "950", 500U); + + Add(3, 26U, "130", "165", 600U); + Add(3, 27U, "166", "170", 600U); + Add(3, 28U, "270", "340", 600U); + Add(3, 29U, "401", "500", 600U); + Add(3, 30U, "601", "800", 600U); + Add(3, 31U, "830", "890", 600U); + UpdateVersionStorageInfo(); + LevelCompactionPicker local_level_compaction_picker = + LevelCompactionPicker(ioptions_, &icmp_); + std::unique_ptr compaction( + local_level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + + // Cannot pick more files since we reach the last file in level 2 + ASSERT_EQ(2U, compaction->num_input_files(0)); + ASSERT_EQ(9U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(10U, compaction->input(0, 1)->fd.GetNumber()); + // release the version storage + DeleteVersionStorage(); +} + +TEST_F(CompactionPickerTest, CompactionPriMinOverlappingManyFiles) { + NewVersionStorage(6, kCompactionStyleLevel); + ioptions_.compaction_pri = kMinOverlappingRatio; + mutable_cf_options_.max_bytes_for_level_base = 15000000; + mutable_cf_options_.max_bytes_for_level_multiplier = 10; + + // file 7 and 8 over lap with the same file, but file 8 is smaller so + // it will be picked. + Add(2, 13U, "010", "011", + 6100U); // Overlaps with a large file. Not picked + Add(2, 14U, "020", "021", + 6100U); // Overlaps with a large file. Not picked + Add(2, 15U, "030", "031", + 6100U); // Overlaps with a large file. Not picked + Add(2, 16U, "040", "041", + 6100U); // Overlaps with a large file. Not picked + Add(2, 17U, "050", "051", + 6100U); // Overlaps with a large file. Not picked + Add(2, 18U, "060", "061", + 6100U); // Overlaps with a large file. Not picked + Add(2, 19U, "070", "071", + 6100U); // Overlaps with a large file. Not picked + Add(2, 20U, "080", "081", + 6100U); // Overlaps with a large file. Not picked + + Add(2, 6U, "150", "167", 60000000U); // Overlaps with file 26, 27 + Add(2, 7U, "168", "169", 60000000U); // Overlaps with file 27 + Add(2, 8U, "201", "300", 61000000U); // Overlaps with file 28, but the file + // itself is larger. Should be picked. + Add(2, 9U, "610", "611", + 6100U); // Overlaps with a large file. Not picked + Add(2, 10U, "620", "621", + 6100U); // Overlaps with a large file. Not picked + Add(2, 11U, "630", "631", + 6100U); // Overlaps with a large file. Not picked + Add(2, 12U, "640", "641", + 6100U); // Overlaps with a large file. Not picked + + Add(3, 31U, "001", "100", 260000000U); + Add(3, 26U, "160", "165", 260000000U); + Add(3, 27U, "166", "170", 260000000U); + Add(3, 28U, "180", "400", 260000000U); + Add(3, 29U, "401", "500", 260000000U); + Add(3, 30U, "601", "700", 260000000U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(1U, compaction->num_input_files(0)); + // Picking file 8 because overlapping ratio is the biggest. + ASSERT_EQ(8U, compaction->input(0, 0)->fd.GetNumber()); +} + +// This test exhibits the bug where we don't properly reset parent_index in +// PickCompaction() +TEST_F(CompactionPickerTest, ParentIndexResetBug) { + int num_levels = ioptions_.num_levels; + mutable_cf_options_.level0_file_num_compaction_trigger = 2; + mutable_cf_options_.max_bytes_for_level_base = 200; + NewVersionStorage(num_levels, kCompactionStyleLevel); + Add(0, 1U, "150", "200"); // <- marked for compaction + Add(1, 3U, "400", "500", 600); // <- this one needs compacting + Add(2, 4U, "150", "200"); + Add(2, 5U, "201", "210"); + Add(2, 6U, "300", "310"); + Add(2, 7U, "400", "500"); // <- being compacted + + vstorage_->LevelFiles(2)[3]->being_compacted = true; + vstorage_->LevelFiles(0)[0]->marked_for_compaction = true; + + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); +} + +// This test checks ExpandWhileOverlapping() by having overlapping user keys +// ranges (with different sequence numbers) in the input files. +TEST_F(CompactionPickerTest, OverlappingUserKeys) { + NewVersionStorage(6, kCompactionStyleLevel); + ioptions_.compaction_pri = kByCompensatedSize; + + Add(1, 1U, "100", "150", 1U); + // Overlapping user keys + Add(1, 2U, "200", "400", 1U); + Add(1, 3U, "400", "500", 1000000000U, 0, 0); + Add(2, 4U, "600", "700", 1U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(1U, compaction->num_input_levels()); + ASSERT_EQ(2U, compaction->num_input_files(0)); + ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(3U, compaction->input(0, 1)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, OverlappingUserKeys2) { + NewVersionStorage(6, kCompactionStyleLevel); + // Overlapping user keys on same level and output level + Add(1, 1U, "200", "400", 1000000000U); + Add(1, 2U, "400", "500", 1U, 0, 0); + Add(2, 3U, "000", "100", 1U); + Add(2, 4U, "100", "600", 1U, 0, 0); + Add(2, 5U, "600", "700", 1U, 0, 0); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(2U, compaction->num_input_levels()); + ASSERT_EQ(2U, compaction->num_input_files(0)); + ASSERT_EQ(3U, compaction->num_input_files(1)); + ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); + ASSERT_EQ(3U, compaction->input(1, 0)->fd.GetNumber()); + ASSERT_EQ(4U, compaction->input(1, 1)->fd.GetNumber()); + ASSERT_EQ(5U, compaction->input(1, 2)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, OverlappingUserKeys3) { + NewVersionStorage(6, kCompactionStyleLevel); + // Chain of overlapping user key ranges (forces ExpandWhileOverlapping() to + // expand multiple times) + Add(1, 1U, "100", "150", 1U); + Add(1, 2U, "150", "200", 1U, 0, 0); + Add(1, 3U, "200", "250", 1000000000U, 0, 0); + Add(1, 4U, "250", "300", 1U, 0, 0); + Add(1, 5U, "300", "350", 1U, 0, 0); + // Output level overlaps with the beginning and the end of the chain + Add(2, 6U, "050", "100", 1U); + Add(2, 7U, "350", "400", 1U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(2U, compaction->num_input_levels()); + ASSERT_EQ(5U, compaction->num_input_files(0)); + ASSERT_EQ(2U, compaction->num_input_files(1)); + ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); + ASSERT_EQ(3U, compaction->input(0, 2)->fd.GetNumber()); + ASSERT_EQ(4U, compaction->input(0, 3)->fd.GetNumber()); + ASSERT_EQ(5U, compaction->input(0, 4)->fd.GetNumber()); + ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber()); + ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, OverlappingUserKeys4) { + NewVersionStorage(6, kCompactionStyleLevel); + mutable_cf_options_.max_bytes_for_level_base = 1000000; + + Add(1, 1U, "100", "150", 1U); + Add(1, 2U, "150", "199", 1U, 0, 0); + Add(1, 3U, "200", "250", 1100000U, 0, 0); + Add(1, 4U, "251", "300", 1U, 0, 0); + Add(1, 5U, "300", "350", 1U, 0, 0); + + Add(2, 6U, "100", "115", 1U); + Add(2, 7U, "125", "325", 1U); + Add(2, 8U, "350", "400", 1U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(2U, compaction->num_input_levels()); + ASSERT_EQ(1U, compaction->num_input_files(0)); + ASSERT_EQ(1U, compaction->num_input_files(1)); + ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(7U, compaction->input(1, 0)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, OverlappingUserKeys5) { + NewVersionStorage(6, kCompactionStyleLevel); + // Overlapping user keys on same level and output level + Add(1, 1U, "200", "400", 1000000000U); + Add(1, 2U, "400", "500", 1U, 0, 0); + Add(2, 3U, "000", "100", 1U); + Add(2, 4U, "100", "600", 1U, 0, 0); + Add(2, 5U, "600", "700", 1U, 0, 0); + + vstorage_->LevelFiles(2)[2]->being_compacted = true; + + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() == nullptr); +} + +TEST_F(CompactionPickerTest, OverlappingUserKeys6) { + NewVersionStorage(6, kCompactionStyleLevel); + // Overlapping user keys on same level and output level + Add(1, 1U, "200", "400", 1U, 0, 0); + Add(1, 2U, "401", "500", 1U, 0, 0); + Add(2, 3U, "000", "100", 1U); + Add(2, 4U, "100", "300", 1U, 0, 0); + Add(2, 5U, "305", "450", 1U, 0, 0); + Add(2, 6U, "460", "600", 1U, 0, 0); + Add(2, 7U, "600", "700", 1U, 0, 0); + + vstorage_->LevelFiles(1)[0]->marked_for_compaction = true; + vstorage_->LevelFiles(1)[1]->marked_for_compaction = true; + + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(2U, compaction->num_input_levels()); + ASSERT_EQ(1U, compaction->num_input_files(0)); + ASSERT_EQ(3U, compaction->num_input_files(1)); +} + +TEST_F(CompactionPickerTest, OverlappingUserKeys7) { + NewVersionStorage(6, kCompactionStyleLevel); + mutable_cf_options_.max_compaction_bytes = 100000000000u; + // Overlapping user keys on same level and output level + Add(1, 1U, "200", "400", 1U, 0, 0); + Add(1, 2U, "401", "500", 1000000000U, 0, 0); + Add(2, 3U, "100", "250", 1U); + Add(2, 4U, "300", "600", 1U, 0, 0); + Add(2, 5U, "600", "800", 1U, 0, 0); + + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(2U, compaction->num_input_levels()); + ASSERT_GE(1U, compaction->num_input_files(0)); + ASSERT_GE(2U, compaction->num_input_files(1)); + // File 5 has to be included in the compaction + ASSERT_EQ(5U, compaction->inputs(1)->back()->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, OverlappingUserKeys8) { + NewVersionStorage(6, kCompactionStyleLevel); + mutable_cf_options_.max_compaction_bytes = 100000000000u; + // grow the number of inputs in "level" without + // changing the number of "level+1" files we pick up + // Expand input level as much as possible + // no overlapping case + Add(1, 1U, "101", "150", 1U); + Add(1, 2U, "151", "200", 1U); + Add(1, 3U, "201", "300", 1000000000U); + Add(1, 4U, "301", "400", 1U); + Add(1, 5U, "401", "500", 1U); + Add(2, 6U, "150", "200", 1U); + Add(2, 7U, "200", "450", 1U, 0, 0); + Add(2, 8U, "500", "600", 1U); + + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(2U, compaction->num_input_levels()); + ASSERT_EQ(3U, compaction->num_input_files(0)); + ASSERT_EQ(2U, compaction->num_input_files(1)); + ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(3U, compaction->input(0, 1)->fd.GetNumber()); + ASSERT_EQ(4U, compaction->input(0, 2)->fd.GetNumber()); + ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber()); + ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, OverlappingUserKeys9) { + NewVersionStorage(6, kCompactionStyleLevel); + mutable_cf_options_.max_compaction_bytes = 100000000000u; + // grow the number of inputs in "level" without + // changing the number of "level+1" files we pick up + // Expand input level as much as possible + // overlapping case + Add(1, 1U, "121", "150", 1U); + Add(1, 2U, "151", "200", 1U); + Add(1, 3U, "201", "300", 1000000000U); + Add(1, 4U, "301", "400", 1U); + Add(1, 5U, "401", "500", 1U); + Add(2, 6U, "100", "120", 1U); + Add(2, 7U, "150", "200", 1U); + Add(2, 8U, "200", "450", 1U, 0, 0); + Add(2, 9U, "501", "600", 1U); + + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(2U, compaction->num_input_levels()); + ASSERT_EQ(5U, compaction->num_input_files(0)); + ASSERT_EQ(2U, compaction->num_input_files(1)); + ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); + ASSERT_EQ(3U, compaction->input(0, 2)->fd.GetNumber()); + ASSERT_EQ(4U, compaction->input(0, 3)->fd.GetNumber()); + ASSERT_EQ(7U, compaction->input(1, 0)->fd.GetNumber()); + ASSERT_EQ(8U, compaction->input(1, 1)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, OverlappingUserKeys10) { + // Locked file encountered when pulling in extra input-level files with same + // user keys. Verify we pick the next-best file from the same input level. + NewVersionStorage(6, kCompactionStyleLevel); + mutable_cf_options_.max_compaction_bytes = 100000000000u; + + // file_number 2U is largest and thus first choice. But it overlaps with + // file_number 1U which is being compacted. So instead we pick the next- + // biggest file, 3U, which is eligible for compaction. + Add(1 /* level */, 1U /* file_number */, "100" /* smallest */, + "150" /* largest */, 1U /* file_size */); + file_map_[1U].first->being_compacted = true; + Add(1 /* level */, 2U /* file_number */, "150" /* smallest */, + "200" /* largest */, 1000000000U /* file_size */, 0 /* smallest_seq */, + 0 /* largest_seq */); + Add(1 /* level */, 3U /* file_number */, "201" /* smallest */, + "250" /* largest */, 900000000U /* file_size */); + Add(2 /* level */, 4U /* file_number */, "100" /* smallest */, + "150" /* largest */, 1U /* file_size */); + Add(2 /* level */, 5U /* file_number */, "151" /* smallest */, + "200" /* largest */, 1U /* file_size */); + Add(2 /* level */, 6U /* file_number */, "201" /* smallest */, + "250" /* largest */, 1U /* file_size */); + + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(2U, compaction->num_input_levels()); + ASSERT_EQ(1U, compaction->num_input_files(0)); + ASSERT_EQ(1U, compaction->num_input_files(1)); + ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, OverlappingUserKeys11) { + // Locked file encountered when pulling in extra output-level files with same + // user keys. Expected to skip that compaction and pick the next-best choice. + NewVersionStorage(6, kCompactionStyleLevel); + mutable_cf_options_.max_compaction_bytes = 100000000000u; + + // score(L1) = 3.7 + // score(L2) = 1.85 + // There is no eligible file in L1 to compact since both candidates pull in + // file_number 5U, which overlaps with a file pending compaction (6U). The + // first eligible compaction is from L2->L3. + Add(1 /* level */, 2U /* file_number */, "151" /* smallest */, + "200" /* largest */, 1000000000U /* file_size */); + Add(1 /* level */, 3U /* file_number */, "201" /* smallest */, + "250" /* largest */, 1U /* file_size */); + Add(2 /* level */, 4U /* file_number */, "100" /* smallest */, + "149" /* largest */, 5000000000U /* file_size */); + Add(2 /* level */, 5U /* file_number */, "150" /* smallest */, + "201" /* largest */, 1U /* file_size */); + Add(2 /* level */, 6U /* file_number */, "201" /* smallest */, + "249" /* largest */, 1U /* file_size */, 0 /* smallest_seq */, + 0 /* largest_seq */); + file_map_[6U].first->being_compacted = true; + Add(3 /* level */, 7U /* file_number */, "100" /* smallest */, + "149" /* largest */, 1U /* file_size */); + + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(2U, compaction->num_input_levels()); + ASSERT_EQ(1U, compaction->num_input_files(0)); + ASSERT_EQ(1U, compaction->num_input_files(1)); + ASSERT_EQ(4U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(7U, compaction->input(1, 0)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, FileTtlBooster) { + // Set TTL to 2048 + // TTL boosting for all levels starts at 1024, + // Whole TTL range is 2048 * 31 / 32 - 1024 = 1984 - 1024 = 960. + // From second last level (L5), range starts at + // 1024 + 480, 1024 + 240, 1024 + 120 (which is L3). + // Boosting step 124 / 16 = 7.75 -> 7 + // + const uint64_t kCurrentTime = 1000000; + FileMetaData meta; + + { + FileTtlBooster booster(kCurrentTime, 2048, 7, 3); + + // Not triggering if the file is younger than ttl/2 + meta.oldest_ancester_time = kCurrentTime - 1023; + ASSERT_EQ(1, booster.GetBoostScore(&meta)); + meta.oldest_ancester_time = kCurrentTime - 1024; + ASSERT_EQ(1, booster.GetBoostScore(&meta)); + meta.oldest_ancester_time = kCurrentTime + 10; + ASSERT_EQ(1, booster.GetBoostScore(&meta)); + + // Within one boosting step + meta.oldest_ancester_time = kCurrentTime - (1024 + 120 + 6); + ASSERT_EQ(1, booster.GetBoostScore(&meta)); + + // One boosting step + meta.oldest_ancester_time = kCurrentTime - (1024 + 120 + 7); + ASSERT_EQ(2, booster.GetBoostScore(&meta)); + meta.oldest_ancester_time = kCurrentTime - (1024 + 120 + 8); + ASSERT_EQ(2, booster.GetBoostScore(&meta)); + + // Multiple boosting steps + meta.oldest_ancester_time = kCurrentTime - (1024 + 120 + 30); + ASSERT_EQ(5, booster.GetBoostScore(&meta)); + + // Very high boosting steps + meta.oldest_ancester_time = kCurrentTime - (1024 + 120 + 700); + ASSERT_EQ(101, booster.GetBoostScore(&meta)); + } + { + // Test second last level + FileTtlBooster booster(kCurrentTime, 2048, 7, 5); + meta.oldest_ancester_time = kCurrentTime - (1024 + 480); + ASSERT_EQ(1, booster.GetBoostScore(&meta)); + meta.oldest_ancester_time = kCurrentTime - (1024 + 480 + 60); + ASSERT_EQ(3, booster.GetBoostScore(&meta)); + } + { + // Test last level + FileTtlBooster booster(kCurrentTime, 2048, 7, 6); + meta.oldest_ancester_time = kCurrentTime - (1024 + 480); + ASSERT_EQ(1, booster.GetBoostScore(&meta)); + meta.oldest_ancester_time = kCurrentTime - (1024 + 480 + 60); + ASSERT_EQ(1, booster.GetBoostScore(&meta)); + meta.oldest_ancester_time = kCurrentTime - 3000; + ASSERT_EQ(1, booster.GetBoostScore(&meta)); + } +} + +TEST_F(CompactionPickerTest, NotScheduleL1IfL0WithHigherPri1) { + NewVersionStorage(6, kCompactionStyleLevel); + mutable_cf_options_.level0_file_num_compaction_trigger = 2; + mutable_cf_options_.max_bytes_for_level_base = 900000000U; + + // 6 L0 files, score 3. + Add(0, 1U, "000", "400", 1U); + Add(0, 2U, "001", "400", 1U, 0, 0); + Add(0, 3U, "001", "400", 1000000000U, 0, 0); + Add(0, 31U, "001", "400", 1000000000U, 0, 0); + Add(0, 32U, "001", "400", 1000000000U, 0, 0); + Add(0, 33U, "001", "400", 1000000000U, 0, 0); + + // L1 total size 2GB, score 2.2. If one file being compacted, score 1.1. + Add(1, 4U, "050", "300", 1000000000U, 0, 0); + file_map_[4u].first->being_compacted = true; + Add(1, 5U, "301", "350", 1000000000U, 0, 0); + + // Output level overlaps with the beginning and the end of the chain + Add(2, 6U, "050", "100", 1U); + Add(2, 7U, "300", "400", 1U); + + // No compaction should be scheduled, if L0 has higher priority than L1 + // but L0->L1 compaction is blocked by a file in L1 being compacted. + UpdateVersionStorageInfo(); + ASSERT_EQ(0, vstorage_->CompactionScoreLevel(0)); + ASSERT_EQ(1, vstorage_->CompactionScoreLevel(1)); + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() == nullptr); +} + +TEST_F(CompactionPickerTest, NotScheduleL1IfL0WithHigherPri2) { + NewVersionStorage(6, kCompactionStyleLevel); + mutable_cf_options_.level0_file_num_compaction_trigger = 2; + mutable_cf_options_.max_bytes_for_level_base = 900000000U; + + // 6 L0 files, score 3. + Add(0, 1U, "000", "400", 1U); + Add(0, 2U, "001", "400", 1U, 0, 0); + Add(0, 3U, "001", "400", 1000000000U, 0, 0); + Add(0, 31U, "001", "400", 1000000000U, 0, 0); + Add(0, 32U, "001", "400", 1000000000U, 0, 0); + Add(0, 33U, "001", "400", 1000000000U, 0, 0); + + // L1 total size 2GB, score 2.2. If one file being compacted, score 1.1. + Add(1, 4U, "050", "300", 1000000000U, 0, 0); + Add(1, 5U, "301", "350", 1000000000U, 0, 0); + + // Output level overlaps with the beginning and the end of the chain + Add(2, 6U, "050", "100", 1U); + Add(2, 7U, "300", "400", 1U); + + // If no file in L1 being compacted, L0->L1 compaction will be scheduled. + UpdateVersionStorageInfo(); // being_compacted flag is cleared here. + ASSERT_EQ(0, vstorage_->CompactionScoreLevel(0)); + ASSERT_EQ(1, vstorage_->CompactionScoreLevel(1)); + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); +} + +TEST_F(CompactionPickerTest, NotScheduleL1IfL0WithHigherPri3) { + NewVersionStorage(6, kCompactionStyleLevel); + mutable_cf_options_.level0_file_num_compaction_trigger = 2; + mutable_cf_options_.max_bytes_for_level_base = 900000000U; + + // 6 L0 files, score 3. + Add(0, 1U, "000", "400", 1U); + Add(0, 2U, "001", "400", 1U, 0, 0); + Add(0, 3U, "001", "400", 1000000000U, 0, 0); + Add(0, 31U, "001", "400", 1000000000U, 0, 0); + Add(0, 32U, "001", "400", 1000000000U, 0, 0); + Add(0, 33U, "001", "400", 1000000000U, 0, 0); + + // L1 score more than 6. + Add(1, 4U, "050", "300", 1000000000U, 0, 0); + file_map_[4u].first->being_compacted = true; + Add(1, 5U, "301", "350", 1000000000U, 0, 0); + Add(1, 51U, "351", "400", 6000000000U, 0, 0); + + // Output level overlaps with the beginning and the end of the chain + Add(2, 6U, "050", "100", 1U); + Add(2, 7U, "300", "400", 1U); + + // If score in L1 is larger than L0, L1 compaction goes through despite + // there is pending L0 compaction. + UpdateVersionStorageInfo(); + ASSERT_EQ(1, vstorage_->CompactionScoreLevel(0)); + ASSERT_EQ(0, vstorage_->CompactionScoreLevel(1)); + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); +} + +TEST_F(CompactionPickerTest, EstimateCompactionBytesNeeded1) { + int num_levels = ioptions_.num_levels; + ioptions_.level_compaction_dynamic_level_bytes = false; + mutable_cf_options_.level0_file_num_compaction_trigger = 4; + mutable_cf_options_.max_bytes_for_level_base = 1000; + mutable_cf_options_.max_bytes_for_level_multiplier = 10; + NewVersionStorage(num_levels, kCompactionStyleLevel); + Add(0, 1U, "150", "200", 200); + Add(0, 2U, "150", "200", 200); + Add(0, 3U, "150", "200", 200); + // Level 1 is over target by 200 + Add(1, 4U, "400", "500", 600); + Add(1, 5U, "600", "700", 600); + // Level 2 is less than target 10000 even added size of level 1 + // Size ratio of L2/L1 is 9600 / 1200 = 8 + Add(2, 6U, "150", "200", 2500); + Add(2, 7U, "201", "210", 2000); + Add(2, 8U, "300", "310", 2600); + Add(2, 9U, "400", "500", 2500); + // Level 3 exceeds target 100,000 of 1000 + Add(3, 10U, "400", "500", 101000); + // Level 4 exceeds target 1,000,000 by 900 after adding size from level 3 + // Size ratio L4/L3 is 9.9 + // After merge from L3, L4 size is 1000900 + Add(4, 11U, "400", "500", 999900); + Add(5, 12U, "400", "500", 8007200); + + UpdateVersionStorageInfo(); + + ASSERT_EQ(200u * 9u + 10900u + 900u * 9, + vstorage_->estimated_compaction_needed_bytes()); +} + +TEST_F(CompactionPickerTest, EstimateCompactionBytesNeeded2) { + int num_levels = ioptions_.num_levels; + ioptions_.level_compaction_dynamic_level_bytes = false; + mutable_cf_options_.level0_file_num_compaction_trigger = 3; + mutable_cf_options_.max_bytes_for_level_base = 1000; + mutable_cf_options_.max_bytes_for_level_multiplier = 10; + NewVersionStorage(num_levels, kCompactionStyleLevel); + Add(0, 1U, "150", "200", 200); + Add(0, 2U, "150", "200", 200); + Add(0, 4U, "150", "200", 200); + Add(0, 5U, "150", "200", 200); + Add(0, 6U, "150", "200", 200); + // Level 1 size will be 1400 after merging with L0 + Add(1, 7U, "400", "500", 200); + Add(1, 8U, "600", "700", 200); + // Level 2 is less than target 10000 even added size of level 1 + Add(2, 9U, "150", "200", 9100); + // Level 3 over the target, but since level 4 is empty, we assume it will be + // a trivial move. + Add(3, 10U, "400", "500", 101000); + + UpdateVersionStorageInfo(); + + // estimated L1->L2 merge: 400 * (9100.0 / 1400.0 + 1.0) + ASSERT_EQ(1400u + 3000u, vstorage_->estimated_compaction_needed_bytes()); +} + +TEST_F(CompactionPickerTest, EstimateCompactionBytesNeeded3) { + int num_levels = ioptions_.num_levels; + ioptions_.level_compaction_dynamic_level_bytes = false; + mutable_cf_options_.level0_file_num_compaction_trigger = 3; + mutable_cf_options_.max_bytes_for_level_base = 1000; + mutable_cf_options_.max_bytes_for_level_multiplier = 10; + NewVersionStorage(num_levels, kCompactionStyleLevel); + Add(0, 1U, "150", "200", 2000); + Add(0, 2U, "150", "200", 2000); + Add(0, 4U, "150", "200", 2000); + Add(0, 5U, "150", "200", 2000); + Add(0, 6U, "150", "200", 1000); + // Level 1 size will be 10000 after merging with L0 + Add(1, 7U, "400", "500", 500); + Add(1, 8U, "600", "700", 500); + + Add(2, 9U, "150", "200", 10000); + + UpdateVersionStorageInfo(); + + ASSERT_EQ(10000u + 18000u, vstorage_->estimated_compaction_needed_bytes()); +} + +TEST_F(CompactionPickerTest, EstimateCompactionBytesNeededDynamicLevel) { + int num_levels = ioptions_.num_levels; + ioptions_.level_compaction_dynamic_level_bytes = true; + mutable_cf_options_.level0_file_num_compaction_trigger = 3; + mutable_cf_options_.max_bytes_for_level_base = 1000; + mutable_cf_options_.max_bytes_for_level_multiplier = 10; + NewVersionStorage(num_levels, kCompactionStyleLevel); + + // Set Last level size 50000 + // num_levels - 1 target 5000 + // num_levels - 2 is base level with target 1000 (rounded up to + // max_bytes_for_level_base). + Add(num_levels - 1, 10U, "400", "500", 50000); + + Add(0, 1U, "150", "200", 200); + Add(0, 2U, "150", "200", 200); + Add(0, 4U, "150", "200", 200); + Add(0, 5U, "150", "200", 200); + Add(0, 6U, "150", "200", 200); + // num_levels - 3 is over target by 100 + 1000 + Add(num_levels - 3, 7U, "400", "500", 550); + Add(num_levels - 3, 8U, "600", "700", 550); + // num_levels - 2 is over target by 1100 + 200 + Add(num_levels - 2, 9U, "150", "200", 5200); + + UpdateVersionStorageInfo(); + + // Merging to the second last level: (5200 / 2100 + 1) * 1100 + // Merging to the last level: (50000 / 6300 + 1) * 1300 + ASSERT_EQ(2100u + 3823u + 11617u, + vstorage_->estimated_compaction_needed_bytes()); +} + +TEST_F(CompactionPickerTest, IsBottommostLevelTest) { + // case 1: Higher levels are empty + NewVersionStorage(6, kCompactionStyleLevel); + Add(0, 1U, "a", "m"); + Add(0, 2U, "c", "z"); + Add(1, 3U, "d", "e"); + Add(1, 4U, "l", "p"); + Add(2, 5U, "g", "i"); + Add(2, 6U, "x", "z"); + UpdateVersionStorageInfo(); + SetCompactionInputFilesLevels(2, 1); + AddToCompactionFiles(3U); + AddToCompactionFiles(5U); + bool result = + Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_); + ASSERT_TRUE(result); + + // case 2: Higher levels have no overlap + NewVersionStorage(6, kCompactionStyleLevel); + Add(0, 1U, "a", "m"); + Add(0, 2U, "c", "z"); + Add(1, 3U, "d", "e"); + Add(1, 4U, "l", "p"); + Add(2, 5U, "g", "i"); + Add(2, 6U, "x", "z"); + Add(3, 7U, "k", "p"); + Add(3, 8U, "t", "w"); + Add(4, 9U, "a", "b"); + Add(5, 10U, "c", "cc"); + UpdateVersionStorageInfo(); + SetCompactionInputFilesLevels(2, 1); + AddToCompactionFiles(3U); + AddToCompactionFiles(5U); + result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_); + ASSERT_TRUE(result); + + // case 3.1: Higher levels (level 3) have overlap + NewVersionStorage(6, kCompactionStyleLevel); + Add(0, 1U, "a", "m"); + Add(0, 2U, "c", "z"); + Add(1, 3U, "d", "e"); + Add(1, 4U, "l", "p"); + Add(2, 5U, "g", "i"); + Add(2, 6U, "x", "z"); + Add(3, 7U, "e", "g"); + Add(3, 8U, "h", "k"); + Add(4, 9U, "a", "b"); + Add(5, 10U, "c", "cc"); + UpdateVersionStorageInfo(); + SetCompactionInputFilesLevels(2, 1); + AddToCompactionFiles(3U); + AddToCompactionFiles(5U); + result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_); + ASSERT_FALSE(result); + + // case 3.2: Higher levels (level 5) have overlap + DeleteVersionStorage(); + NewVersionStorage(6, kCompactionStyleLevel); + Add(0, 1U, "a", "m"); + Add(0, 2U, "c", "z"); + Add(1, 3U, "d", "e"); + Add(1, 4U, "l", "p"); + Add(2, 5U, "g", "i"); + Add(2, 6U, "x", "z"); + Add(3, 7U, "j", "k"); + Add(3, 8U, "l", "m"); + Add(4, 9U, "a", "b"); + Add(5, 10U, "c", "cc"); + Add(5, 11U, "h", "k"); + Add(5, 12U, "y", "yy"); + Add(5, 13U, "z", "zz"); + UpdateVersionStorageInfo(); + SetCompactionInputFilesLevels(2, 1); + AddToCompactionFiles(3U); + AddToCompactionFiles(5U); + result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_); + ASSERT_FALSE(result); + + // case 3.3: Higher levels (level 5) have overlap, but it's only overlapping + // one key ("d") + NewVersionStorage(6, kCompactionStyleLevel); + Add(0, 1U, "a", "m"); + Add(0, 2U, "c", "z"); + Add(1, 3U, "d", "e"); + Add(1, 4U, "l", "p"); + Add(2, 5U, "g", "i"); + Add(2, 6U, "x", "z"); + Add(3, 7U, "j", "k"); + Add(3, 8U, "l", "m"); + Add(4, 9U, "a", "b"); + Add(5, 10U, "c", "cc"); + Add(5, 11U, "ccc", "d"); + Add(5, 12U, "y", "yy"); + Add(5, 13U, "z", "zz"); + UpdateVersionStorageInfo(); + SetCompactionInputFilesLevels(2, 1); + AddToCompactionFiles(3U); + AddToCompactionFiles(5U); + result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_); + ASSERT_FALSE(result); + + // Level 0 files overlap + NewVersionStorage(6, kCompactionStyleLevel); + Add(0, 1U, "s", "t"); + Add(0, 2U, "a", "m"); + Add(0, 3U, "b", "z"); + Add(0, 4U, "e", "f"); + Add(5, 10U, "y", "z"); + UpdateVersionStorageInfo(); + SetCompactionInputFilesLevels(1, 0); + AddToCompactionFiles(1U); + AddToCompactionFiles(2U); + AddToCompactionFiles(3U); + AddToCompactionFiles(4U); + result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_); + ASSERT_FALSE(result); + + // Level 0 files don't overlap + NewVersionStorage(6, kCompactionStyleLevel); + Add(0, 1U, "s", "t"); + Add(0, 2U, "a", "m"); + Add(0, 3U, "b", "k"); + Add(0, 4U, "e", "f"); + Add(5, 10U, "y", "z"); + UpdateVersionStorageInfo(); + SetCompactionInputFilesLevels(1, 0); + AddToCompactionFiles(1U); + AddToCompactionFiles(2U); + AddToCompactionFiles(3U); + AddToCompactionFiles(4U); + result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_); + ASSERT_TRUE(result); + + // Level 1 files overlap + NewVersionStorage(6, kCompactionStyleLevel); + Add(0, 1U, "s", "t"); + Add(0, 2U, "a", "m"); + Add(0, 3U, "b", "k"); + Add(0, 4U, "e", "f"); + Add(1, 5U, "a", "m"); + Add(1, 6U, "n", "o"); + Add(1, 7U, "w", "y"); + Add(5, 10U, "y", "z"); + UpdateVersionStorageInfo(); + SetCompactionInputFilesLevels(2, 0); + AddToCompactionFiles(1U); + AddToCompactionFiles(2U); + AddToCompactionFiles(3U); + AddToCompactionFiles(4U); + AddToCompactionFiles(5U); + AddToCompactionFiles(6U); + AddToCompactionFiles(7U); + result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_); + ASSERT_FALSE(result); + + DeleteVersionStorage(); +} + +TEST_F(CompactionPickerTest, MaxCompactionBytesHit) { + mutable_cf_options_.max_bytes_for_level_base = 1000000u; + mutable_cf_options_.max_compaction_bytes = 800000u; + mutable_cf_options_.ignore_max_compaction_bytes_for_input = false; + ioptions_.level_compaction_dynamic_level_bytes = false; + NewVersionStorage(6, kCompactionStyleLevel); + // A compaction should be triggered and pick file 2 and 5. + // It can expand because adding file 1 and 3, the compaction size will + // exceed mutable_cf_options_.max_bytes_for_level_base. + Add(1, 1U, "100", "150", 300000U); + Add(1, 2U, "151", "200", 300001U, 0, 0); + Add(1, 3U, "201", "250", 300000U, 0, 0); + Add(1, 4U, "251", "300", 300000U, 0, 0); + Add(2, 5U, "100", "256", 1U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(2U, compaction->num_input_levels()); + ASSERT_EQ(1U, compaction->num_input_files(0)); + ASSERT_EQ(1U, compaction->num_input_files(1)); + ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(5U, compaction->input(1, 0)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, MaxCompactionBytesNotHit) { + mutable_cf_options_.max_bytes_for_level_base = 800000u; + mutable_cf_options_.max_compaction_bytes = 1000000u; + mutable_cf_options_.ignore_max_compaction_bytes_for_input = false; + ioptions_.level_compaction_dynamic_level_bytes = false; + NewVersionStorage(6, kCompactionStyleLevel); + // A compaction should be triggered and pick file 2 and 5. + // and it expands to file 1 and 3 too. + Add(1, 1U, "100", "150", 300000U); + Add(1, 2U, "151", "200", 300001U, 0, 0); + Add(1, 3U, "201", "250", 300000U, 0, 0); + Add(1, 4U, "251", "300", 300000U, 0, 0); + Add(2, 5U, "000", "251", 1U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(2U, compaction->num_input_levels()); + ASSERT_EQ(3U, compaction->num_input_files(0)); + ASSERT_EQ(1U, compaction->num_input_files(1)); + ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); + ASSERT_EQ(3U, compaction->input(0, 2)->fd.GetNumber()); + ASSERT_EQ(5U, compaction->input(1, 0)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, IsTrivialMoveOn) { + mutable_cf_options_.max_bytes_for_level_base = 10000u; + mutable_cf_options_.max_compaction_bytes = 10001u; + ioptions_.level_compaction_dynamic_level_bytes = false; + NewVersionStorage(6, kCompactionStyleLevel); + // A compaction should be triggered and pick file 2 + Add(1, 1U, "100", "150", 3000U); + Add(1, 2U, "151", "200", 3001U); + Add(1, 3U, "201", "250", 3000U); + Add(1, 4U, "251", "300", 3000U); + + Add(3, 5U, "120", "130", 7000U); + Add(3, 6U, "170", "180", 7000U); + Add(3, 7U, "220", "230", 7000U); + Add(3, 8U, "270", "280", 7000U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_TRUE(compaction->IsTrivialMove()); +} + +TEST_F(CompactionPickerTest, L0TrivialMove1) { + mutable_cf_options_.max_bytes_for_level_base = 10000000u; + mutable_cf_options_.level0_file_num_compaction_trigger = 4; + mutable_cf_options_.max_compaction_bytes = 10000000u; + ioptions_.level_compaction_dynamic_level_bytes = false; + NewVersionStorage(6, kCompactionStyleLevel); + + Add(0, 1U, "100", "150", 3000U, 0, 710, 800); + Add(0, 2U, "151", "200", 3001U, 0, 610, 700); + Add(0, 3U, "301", "350", 3000U, 0, 510, 600); + Add(0, 4U, "451", "400", 3000U, 0, 410, 500); + + Add(1, 5U, "120", "130", 7000U); + Add(1, 6U, "170", "180", 7000U); + Add(1, 7U, "220", "230", 7000U); + Add(1, 8U, "270", "280", 7000U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(1, compaction->num_input_levels()); + ASSERT_EQ(2, compaction->num_input_files(0)); + ASSERT_EQ(3, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(4, compaction->input(0, 1)->fd.GetNumber()); + ASSERT_TRUE(compaction->IsTrivialMove()); +} + +TEST_F(CompactionPickerTest, L0TrivialMoveOneFile) { + mutable_cf_options_.max_bytes_for_level_base = 10000000u; + mutable_cf_options_.level0_file_num_compaction_trigger = 4; + mutable_cf_options_.max_compaction_bytes = 10000000u; + ioptions_.level_compaction_dynamic_level_bytes = false; + NewVersionStorage(6, kCompactionStyleLevel); + + Add(0, 1U, "100", "150", 3000U, 0, 710, 800); + Add(0, 2U, "551", "600", 3001U, 0, 610, 700); + Add(0, 3U, "101", "150", 3000U, 0, 510, 600); + Add(0, 4U, "451", "400", 3000U, 0, 410, 500); + + Add(1, 5U, "120", "130", 7000U); + Add(1, 6U, "170", "180", 7000U); + Add(1, 7U, "220", "230", 7000U); + Add(1, 8U, "270", "280", 7000U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(1, compaction->num_input_levels()); + ASSERT_EQ(1, compaction->num_input_files(0)); + ASSERT_EQ(4, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_TRUE(compaction->IsTrivialMove()); +} + +TEST_F(CompactionPickerTest, L0TrivialMoveWholeL0) { + mutable_cf_options_.max_bytes_for_level_base = 10000000u; + mutable_cf_options_.level0_file_num_compaction_trigger = 4; + mutable_cf_options_.max_compaction_bytes = 10000000u; + ioptions_.level_compaction_dynamic_level_bytes = false; + NewVersionStorage(6, kCompactionStyleLevel); + + Add(0, 1U, "300", "350", 3000U, 0, 710, 800); + Add(0, 2U, "651", "600", 3001U, 0, 610, 700); + Add(0, 3U, "501", "550", 3000U, 0, 510, 600); + Add(0, 4U, "451", "400", 3000U, 0, 410, 500); + + Add(1, 5U, "120", "130", 7000U); + Add(1, 6U, "970", "980", 7000U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(1, compaction->num_input_levels()); + ASSERT_EQ(4, compaction->num_input_files(0)); + ASSERT_EQ(1, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(4, compaction->input(0, 1)->fd.GetNumber()); + ASSERT_EQ(3, compaction->input(0, 2)->fd.GetNumber()); + ASSERT_EQ(2, compaction->input(0, 3)->fd.GetNumber()); + ASSERT_TRUE(compaction->IsTrivialMove()); +} + +TEST_F(CompactionPickerTest, NonL0TrivialMoveExtendBothDirection) { + mutable_cf_options_.max_bytes_for_level_base = 5000; + mutable_cf_options_.level0_file_num_compaction_trigger = 4; + mutable_cf_options_.max_compaction_bytes = 10000000u; + ioptions_.level_compaction_dynamic_level_bytes = false; + NewVersionStorage(6, kCompactionStyleLevel); + + Add(1, 1U, "300", "350", 3000U, 0, 710, 800, 3000U); + Add(1, 2U, "600", "651", 3001U, 0, 610, 700, 3001U); + Add(1, 3U, "700", "750", 3000U, 0, 500, 550, 3000U); + Add(2, 4U, "800", "850", 4000U, 0, 150, 200, 4000U); + + UpdateVersionStorageInfo(); + // File #2 should be picked first, and expand both directions to include + // files #1 and #3. + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(1, compaction->num_input_levels()); + ASSERT_EQ(3, compaction->num_input_files(0)); + ASSERT_EQ(1, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(2, compaction->input(0, 1)->fd.GetNumber()); + ASSERT_EQ(3, compaction->input(0, 2)->fd.GetNumber()); + ASSERT_TRUE(compaction->IsTrivialMove()); +} + +TEST_F(CompactionPickerTest, L0TrivialMoveToEmptyLevel) { + mutable_cf_options_.max_bytes_for_level_base = 5000; + mutable_cf_options_.level0_file_num_compaction_trigger = 4; + mutable_cf_options_.max_compaction_bytes = 10000000u; + ioptions_.level_compaction_dynamic_level_bytes = false; + NewVersionStorage(6, kCompactionStyleLevel); + + // File 2 will be picked first, which by itself is trivial movable. + // There was a bug before where compaction also picks file 3 and 4, + // (and then file 1 since it overlaps with the key range), + // which makes the compaction not trivial movable. + Add(0, 1U, "450", "599", 3000U, 0, 710, 800, 3000U); + Add(0, 2U, "600", "651", 3001U, 0, 610, 700, 3001U); + Add(0, 3U, "300", "350", 3000U, 0, 500, 550, 3000U); + Add(0, 4U, "500", "550", 2999U, 0, 300, 350, 2999U); + + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(1, compaction->num_input_levels()); + ASSERT_EQ(1, compaction->num_input_files(0)); + ASSERT_EQ(2, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_TRUE(compaction->IsTrivialMove()); +} + +TEST_F(CompactionPickerTest, IsTrivialMoveOffSstPartitioned) { + mutable_cf_options_.max_bytes_for_level_base = 10000u; + mutable_cf_options_.max_compaction_bytes = 10001u; + ioptions_.level_compaction_dynamic_level_bytes = false; + ioptions_.sst_partitioner_factory = NewSstPartitionerFixedPrefixFactory(1); + NewVersionStorage(6, kCompactionStyleLevel); + // A compaction should be triggered and pick file 2 + Add(1, 1U, "100", "150", 3000U); + Add(1, 2U, "151", "200", 3001U); + Add(1, 3U, "201", "250", 3000U); + Add(1, 4U, "251", "300", 3000U); + + Add(3, 5U, "120", "130", 7000U); + Add(3, 6U, "170", "180", 7000U); + Add(3, 7U, "220", "230", 7000U); + Add(3, 8U, "270", "280", 7000U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + // No trivial move, because partitioning is applied + ASSERT_TRUE(!compaction->IsTrivialMove()); +} + +TEST_F(CompactionPickerTest, IsTrivialMoveOff) { + mutable_cf_options_.max_bytes_for_level_base = 1000000u; + mutable_cf_options_.max_compaction_bytes = 10000u; + ioptions_.level_compaction_dynamic_level_bytes = false; + NewVersionStorage(6, kCompactionStyleLevel); + // A compaction should be triggered and pick all files from level 1 + Add(1, 1U, "100", "150", 300000U, 0, 0); + Add(1, 2U, "150", "200", 300000U, 0, 0); + Add(1, 3U, "200", "250", 300000U, 0, 0); + Add(1, 4U, "250", "300", 300000U, 0, 0); + + Add(3, 5U, "120", "130", 6000U); + Add(3, 6U, "140", "150", 6000U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_FALSE(compaction->IsTrivialMove()); +} + +TEST_F(CompactionPickerTest, TrivialMoveMultipleFiles1) { + mutable_cf_options_.max_bytes_for_level_base = 1000u; + mutable_cf_options_.max_compaction_bytes = 10000001u; + ioptions_.level_compaction_dynamic_level_bytes = false; + ioptions_.compaction_pri = kMinOverlappingRatio; + NewVersionStorage(6, kCompactionStyleLevel); + + Add(2, 1U, "100", "150", 3000U); + Add(2, 2U, "151", "200", 3001U); + Add(2, 3U, "301", "350", 3000U); + Add(2, 4U, "451", "400", 3000U); + Add(2, 5U, "551", "500", 3000U); + Add(2, 6U, "651", "600", 3000U); + Add(2, 7U, "751", "700", 3000U); + Add(2, 8U, "851", "900", 3000U); + + Add(3, 15U, "120", "130", 700U); + Add(3, 16U, "170", "180", 700U); + Add(3, 17U, "220", "230", 700U); + Add(3, 18U, "870", "880", 700U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_TRUE(compaction->IsTrivialMove()); + ASSERT_EQ(1, compaction->num_input_levels()); + ASSERT_EQ(4, compaction->num_input_files(0)); + ASSERT_EQ(3, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(4, compaction->input(0, 1)->fd.GetNumber()); + ASSERT_EQ(5, compaction->input(0, 2)->fd.GetNumber()); + ASSERT_EQ(6, compaction->input(0, 3)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, TrivialMoveMultipleFiles2) { + mutable_cf_options_.max_bytes_for_level_base = 1000u; + mutable_cf_options_.max_compaction_bytes = 10000001u; + ioptions_.level_compaction_dynamic_level_bytes = false; + ioptions_.compaction_pri = kMinOverlappingRatio; + NewVersionStorage(6, kCompactionStyleLevel); + + Add(2, 1U, "100", "150", 3000U); + Add(2, 2U, "151", "160", 3001U); + Add(2, 3U, "161", "179", 3000U); + Add(2, 4U, "220", "400", 3000U); + Add(2, 5U, "551", "500", 3000U); + Add(2, 6U, "651", "600", 3000U); + Add(2, 7U, "751", "700", 3000U); + Add(2, 8U, "851", "900", 3000U); + + Add(3, 15U, "120", "130", 700U); + Add(3, 17U, "220", "230", 700U); + Add(3, 18U, "870", "880", 700U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_TRUE(compaction->IsTrivialMove()); + ASSERT_EQ(1, compaction->num_input_levels()); + ASSERT_EQ(2, compaction->num_input_files(0)); + ASSERT_EQ(2, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(3, compaction->input(0, 1)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, TrivialMoveMultipleFiles3) { + mutable_cf_options_.max_bytes_for_level_base = 1000u; + mutable_cf_options_.max_compaction_bytes = 10000001u; + ioptions_.level_compaction_dynamic_level_bytes = false; + ioptions_.compaction_pri = kMinOverlappingRatio; + NewVersionStorage(6, kCompactionStyleLevel); + + // Even if consecutive files can be trivial moved, we don't pick them + // since in case trivial move can't be issued for a reason, we cannot + // fall back to normal compactions. + Add(2, 1U, "100", "150", 3000U); + Add(2, 2U, "151", "160", 3001U); + Add(2, 5U, "551", "500", 3000U); + Add(2, 6U, "651", "600", 3000U); + Add(2, 7U, "751", "700", 3000U); + Add(2, 8U, "851", "900", 3000U); + + Add(3, 15U, "120", "130", 700U); + Add(3, 17U, "220", "230", 700U); + Add(3, 18U, "870", "880", 700U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_TRUE(compaction->IsTrivialMove()); + ASSERT_EQ(1, compaction->num_input_levels()); + ASSERT_EQ(1, compaction->num_input_files(0)); + ASSERT_EQ(2, compaction->input(0, 0)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, TrivialMoveMultipleFiles4) { + mutable_cf_options_.max_bytes_for_level_base = 1000u; + mutable_cf_options_.max_compaction_bytes = 10000001u; + ioptions_.level_compaction_dynamic_level_bytes = false; + ioptions_.compaction_pri = kMinOverlappingRatio; + NewVersionStorage(6, kCompactionStyleLevel); + + Add(2, 1U, "100", "150", 4000U); + Add(2, 2U, "151", "160", 4001U); + Add(2, 3U, "161", "179", 4000U); + + Add(3, 15U, "120", "130", 700U); + Add(3, 17U, "220", "230", 700U); + Add(3, 18U, "870", "880", 700U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_TRUE(compaction->IsTrivialMove()); + ASSERT_EQ(1, compaction->num_input_levels()); + ASSERT_EQ(2, compaction->num_input_files(0)); + ASSERT_EQ(2, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(3, compaction->input(0, 1)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, TrivialMoveMultipleFiles5) { + mutable_cf_options_.max_bytes_for_level_base = 1000u; + mutable_cf_options_.max_compaction_bytes = 10000001u; + ioptions_.level_compaction_dynamic_level_bytes = false; + ioptions_.compaction_pri = kMinOverlappingRatio; + NewVersionStorage(6, kCompactionStyleLevel); + + // File 4 and 5 aren't clean cut, so only 2 and 3 are picked. + Add(2, 1U, "100", "150", 4000U); + Add(2, 2U, "151", "160", 4001U); + Add(2, 3U, "161", "179", 4000U); + Add(2, 4U, "180", "185", 4000U); + Add(2, 5U, "185", "190", 4000U); + + Add(3, 15U, "120", "130", 700U); + Add(3, 17U, "220", "230", 700U); + Add(3, 18U, "870", "880", 700U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_TRUE(compaction->IsTrivialMove()); + ASSERT_EQ(1, compaction->num_input_levels()); + ASSERT_EQ(2, compaction->num_input_files(0)); + ASSERT_EQ(2, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(3, compaction->input(0, 1)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, TrivialMoveMultipleFiles6) { + mutable_cf_options_.max_bytes_for_level_base = 1000u; + mutable_cf_options_.max_compaction_bytes = 10000001u; + ioptions_.level_compaction_dynamic_level_bytes = false; + ioptions_.compaction_pri = kMinOverlappingRatio; + NewVersionStorage(6, kCompactionStyleLevel); + + Add(2, 1U, "100", "150", 3000U); + Add(2, 2U, "151", "200", 3001U); + Add(2, 3U, "301", "350", 3000U); + Add(2, 4U, "451", "400", 3000U); + Add(2, 5U, "551", "500", 3000U); + file_map_[5U].first->being_compacted = true; + Add(2, 6U, "651", "600", 3000U); + Add(2, 7U, "751", "700", 3000U); + Add(2, 8U, "851", "900", 3000U); + + Add(3, 15U, "120", "130", 700U); + Add(3, 16U, "170", "180", 700U); + Add(3, 17U, "220", "230", 700U); + Add(3, 18U, "870", "880", 700U); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_TRUE(compaction->IsTrivialMove()); + ASSERT_EQ(1, compaction->num_input_levels()); + // Since the next file is being compacted. Stopping at 3 and 4. + ASSERT_EQ(2, compaction->num_input_files(0)); + ASSERT_EQ(3, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(4, compaction->input(0, 1)->fd.GetNumber()); +} + +TEST_F(CompactionPickerTest, CacheNextCompactionIndex) { + NewVersionStorage(6, kCompactionStyleLevel); + mutable_cf_options_.max_compaction_bytes = 100000000000u; + + Add(1 /* level */, 1U /* file_number */, "100" /* smallest */, + "149" /* largest */, 1000000000U /* file_size */); + file_map_[1U].first->being_compacted = true; + Add(1 /* level */, 2U /* file_number */, "150" /* smallest */, + "199" /* largest */, 900000000U /* file_size */); + Add(1 /* level */, 3U /* file_number */, "200" /* smallest */, + "249" /* largest */, 800000000U /* file_size */); + Add(1 /* level */, 4U /* file_number */, "250" /* smallest */, + "299" /* largest */, 700000000U /* file_size */); + Add(2 /* level */, 5U /* file_number */, "150" /* smallest */, + "199" /* largest */, 100U /* file_size */); + Add(2 /* level */, 6U /* file_number */, "200" /* smallest */, + "240" /* largest */, 1U /* file_size */); + Add(2 /* level */, 7U /* file_number */, "260" /* smallest */, + "270" /* largest */, 1U /* file_size */); + file_map_[5U].first->being_compacted = true; + + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(2U, compaction->num_input_levels()); + ASSERT_EQ(1U, compaction->num_input_files(0)); + ASSERT_EQ(1U, compaction->num_input_files(1)); + ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(2, vstorage_->NextCompactionIndex(1 /* level */)); + + compaction.reset(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(2U, compaction->num_input_levels()); + ASSERT_EQ(1U, compaction->num_input_files(0)); + ASSERT_EQ(1U, compaction->num_input_files(1)); + ASSERT_EQ(4U, compaction->input(0, 0)->fd.GetNumber()); + ASSERT_EQ(3, vstorage_->NextCompactionIndex(1 /* level */)); + + compaction.reset(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() == nullptr); + ASSERT_EQ(4, vstorage_->NextCompactionIndex(1 /* level */)); +} + +TEST_F(CompactionPickerTest, IntraL0MaxCompactionBytesNotHit) { + // Intra L0 compaction triggers only if there are at least + // level0_file_num_compaction_trigger + 2 L0 files. + mutable_cf_options_.level0_file_num_compaction_trigger = 3; + mutable_cf_options_.max_compaction_bytes = 1000000u; + NewVersionStorage(6, kCompactionStyleLevel); + + // All 5 L0 files will be picked for intra L0 compaction. The one L1 file + // spans entire L0 key range and is marked as being compacted to avoid + // L0->L1 compaction. + Add(0, 1U, "100", "150", 200000U, 0, 100, 101); + Add(0, 2U, "151", "200", 200000U, 0, 102, 103); + Add(0, 3U, "201", "250", 200000U, 0, 104, 105); + Add(0, 4U, "251", "300", 200000U, 0, 106, 107); + Add(0, 5U, "301", "350", 200000U, 0, 108, 109); + Add(1, 6U, "100", "350", 200000U, 0, 110, 111); + vstorage_->LevelFiles(1)[0]->being_compacted = true; + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(1U, compaction->num_input_levels()); + ASSERT_EQ(5U, compaction->num_input_files(0)); + ASSERT_EQ(CompactionReason::kLevelL0FilesNum, + compaction->compaction_reason()); + ASSERT_EQ(0, compaction->output_level()); +} + +TEST_F(CompactionPickerTest, IntraL0MaxCompactionBytesHit) { + // Intra L0 compaction triggers only if there are at least + // level0_file_num_compaction_trigger + 2 L0 files. + mutable_cf_options_.level0_file_num_compaction_trigger = 3; + mutable_cf_options_.max_compaction_bytes = 999999u; + NewVersionStorage(6, kCompactionStyleLevel); + + // 4 out of 5 L0 files will be picked for intra L0 compaction due to + // max_compaction_bytes limit (the minimum number of files for triggering + // intra L0 compaction is 4). The one L1 file spans entire L0 key range and + // is marked as being compacted to avoid L0->L1 compaction. + Add(0, 1U, "100", "150", 200000U, 0, 100, 101); + Add(0, 2U, "151", "200", 200000U, 0, 102, 103); + Add(0, 3U, "201", "250", 200000U, 0, 104, 105); + Add(0, 4U, "251", "300", 200000U, 0, 106, 107); + Add(0, 5U, "301", "350", 200000U, 0, 108, 109); + Add(1, 6U, "100", "350", 200000U, 0, 109, 110); + vstorage_->LevelFiles(1)[0]->being_compacted = true; + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction.get() != nullptr); + ASSERT_EQ(1U, compaction->num_input_levels()); + ASSERT_EQ(4U, compaction->num_input_files(0)); + ASSERT_EQ(CompactionReason::kLevelL0FilesNum, + compaction->compaction_reason()); + ASSERT_EQ(0, compaction->output_level()); +} + +TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap) { + const uint64_t kFileSize = 100000; + + ioptions_.compaction_style = kCompactionStyleUniversal; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + + // This test covers the case where a "regular" universal compaction is + // scheduled first, followed by a delete triggered compaction. The latter + // should fail + NewVersionStorage(5, kCompactionStyleUniversal); + + Add(0, 1U, "150", "200", kFileSize, 0, 500, 550, /*compensated_file_size*/ 0, + /*marked_for_compact*/ false, /* temperature*/ Temperature::kUnknown, + /*oldest_ancestor_time*/ kUnknownOldestAncesterTime, + /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(), + /*epoch_number*/ 3); + Add(0, 2U, "201", "250", 2 * kFileSize, 0, 401, 450, + /*compensated_file_size*/ 0, /*marked_for_compact*/ false, + /* temperature*/ Temperature::kUnknown, + /*oldest_ancestor_time*/ kUnknownOldestAncesterTime, + /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(), + /*epoch_number*/ 2); + Add(0, 4U, "260", "300", 4 * kFileSize, 0, 260, 300, + /*compensated_file_size*/ 0, /*marked_for_compact*/ false, + /* temperature*/ Temperature::kUnknown, + /*oldest_ancestor_time*/ kUnknownOldestAncesterTime, + /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(), + /*epoch_number*/ 1); + Add(3, 5U, "010", "080", 8 * kFileSize, 0, 200, 251); + Add(4, 3U, "301", "350", 8 * kFileSize, 0, 101, 150); + Add(4, 6U, "501", "750", 8 * kFileSize, 0, 101, 150); + + UpdateVersionStorageInfo(); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + + ASSERT_TRUE(compaction); + // Validate that its a compaction to reduce sorted runs + ASSERT_EQ(CompactionReason::kUniversalSortedRunNum, + compaction->compaction_reason()); + ASSERT_EQ(0, compaction->output_level()); + ASSERT_EQ(0, compaction->start_level()); + ASSERT_EQ(2U, compaction->num_input_files(0)); + + AddVersionStorage(); + // Simulate a flush and mark the file for compaction + Add(0, 7U, "150", "200", kFileSize, 0, 551, 600, 0, true, + /* temperature*/ Temperature::kUnknown, + /*oldest_ancestor_time*/ kUnknownOldestAncesterTime, + /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(), + /*epoch_number*/ 4); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction2( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_FALSE(compaction2); +} + +TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap2) { + const uint64_t kFileSize = 100000; + + ioptions_.compaction_style = kCompactionStyleUniversal; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + + // This test covers the case where a delete triggered compaction is + // scheduled first, followed by a "regular" compaction. The latter + // should fail + NewVersionStorage(5, kCompactionStyleUniversal); + + // Mark file number 4 for compaction + Add(0, 4U, "260", "300", 4 * kFileSize, 0, 260, 300, 0, true, + /* temperature*/ Temperature::kUnknown, + /*oldest_ancestor_time*/ kUnknownOldestAncesterTime, + /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(), + /*epoch_number*/ 1); + Add(3, 5U, "240", "290", 8 * kFileSize, 0, 201, 250); + Add(4, 3U, "301", "350", 8 * kFileSize, 0, 101, 150); + Add(4, 6U, "501", "750", 8 * kFileSize, 0, 101, 150); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + + ASSERT_TRUE(compaction); + // Validate that its a delete triggered compaction + ASSERT_EQ(CompactionReason::kFilesMarkedForCompaction, + compaction->compaction_reason()); + ASSERT_EQ(3, compaction->output_level()); + ASSERT_EQ(0, compaction->start_level()); + ASSERT_EQ(1U, compaction->num_input_files(0)); + ASSERT_EQ(1U, compaction->num_input_files(1)); + + AddVersionStorage(); + Add(0, 1U, "150", "200", kFileSize, 0, 500, 550, /*compensated_file_size*/ 0, + /*marked_for_compact*/ false, /* temperature*/ Temperature::kUnknown, + /*oldest_ancestor_time*/ kUnknownOldestAncesterTime, + /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(), + /*epoch_number*/ 3); + Add(0, 2U, "201", "250", 2 * kFileSize, 0, 401, 450, + /*compensated_file_size*/ 0, /*marked_for_compact*/ false, + /* temperature*/ Temperature::kUnknown, + /*oldest_ancestor_time*/ kUnknownOldestAncesterTime, + /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(), + /*epoch_number*/ 2); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction2( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_FALSE(compaction2); +} + +TEST_F(CompactionPickerTest, UniversalMarkedCompactionStartOutputOverlap) { + // The case where universal periodic compaction can be picked + // with some newer files being compacted. + const uint64_t kFileSize = 100000; + + ioptions_.compaction_style = kCompactionStyleUniversal; + + bool input_level_overlap = false; + bool output_level_overlap = false; + // Let's mark 2 files in 2 different levels for compaction. The + // compaction picker will randomly pick one, so use the sync point to + // ensure a deterministic order. Loop until both cases are covered + size_t random_index = 0; + SyncPoint::GetInstance()->SetCallBack( + "CompactionPicker::PickFilesMarkedForCompaction", [&](void* arg) { + size_t* index = static_cast(arg); + *index = random_index; + }); + SyncPoint::GetInstance()->EnableProcessing(); + while (!input_level_overlap || !output_level_overlap) { + // Ensure that the L0 file gets picked first + random_index = !input_level_overlap ? 0 : 1; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + NewVersionStorage(5, kCompactionStyleUniversal); + + Add(0, 1U, "260", "300", 4 * kFileSize, 0, 260, 300, 0, true); + Add(3, 2U, "010", "020", 2 * kFileSize, 0, 201, 248); + Add(3, 3U, "250", "270", 2 * kFileSize, 0, 202, 249); + Add(3, 4U, "290", "310", 2 * kFileSize, 0, 203, 250); + Add(3, 5U, "310", "320", 2 * kFileSize, 0, 204, 251, 0, true); + Add(4, 6U, "301", "350", 8 * kFileSize, 0, 101, 150); + Add(4, 7U, "501", "750", 8 * kFileSize, 0, 101, 150); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + + ASSERT_TRUE(compaction); + // Validate that its a delete triggered compaction + ASSERT_EQ(CompactionReason::kFilesMarkedForCompaction, + compaction->compaction_reason()); + ASSERT_TRUE(compaction->start_level() == 0 || + compaction->start_level() == 3); + if (compaction->start_level() == 0) { + // The L0 file was picked. The next compaction will detect an + // overlap on its input level + input_level_overlap = true; + ASSERT_EQ(3, compaction->output_level()); + ASSERT_EQ(1U, compaction->num_input_files(0)); + ASSERT_EQ(3U, compaction->num_input_files(1)); + } else { + // The level 3 file was picked. The next compaction will pick + // the L0 file and will detect overlap when adding output + // level inputs + output_level_overlap = true; + ASSERT_EQ(4, compaction->output_level()); + ASSERT_EQ(2U, compaction->num_input_files(0)); + ASSERT_EQ(1U, compaction->num_input_files(1)); + } + + vstorage_->ComputeCompactionScore(ioptions_, mutable_cf_options_); + // After recomputing the compaction score, only one marked file will remain + random_index = 0; + std::unique_ptr compaction2( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_FALSE(compaction2); + DeleteVersionStorage(); + } +} + +TEST_F(CompactionPickerTest, UniversalMarkedL0NoOverlap) { + const uint64_t kFileSize = 100000; + + ioptions_.compaction_style = kCompactionStyleUniversal; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + + // This test covers the case where a delete triggered compaction is + // scheduled and should result in a full compaction + NewVersionStorage(1, kCompactionStyleUniversal); + + // Mark file number 4 for compaction + Add(0, 4U, "260", "300", 1 * kFileSize, 0, 260, 300, 0, true); + Add(0, 5U, "240", "290", 2 * kFileSize, 0, 201, 250); + Add(0, 3U, "301", "350", 4 * kFileSize, 0, 101, 150); + Add(0, 6U, "501", "750", 8 * kFileSize, 0, 50, 100); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + + ASSERT_TRUE(compaction); + // Validate that its a delete triggered compaction + ASSERT_EQ(CompactionReason::kFilesMarkedForCompaction, + compaction->compaction_reason()); + ASSERT_EQ(0, compaction->output_level()); + ASSERT_EQ(0, compaction->start_level()); + ASSERT_EQ(4U, compaction->num_input_files(0)); + ASSERT_TRUE(file_map_[4].first->being_compacted); + ASSERT_TRUE(file_map_[5].first->being_compacted); + ASSERT_TRUE(file_map_[3].first->being_compacted); + ASSERT_TRUE(file_map_[6].first->being_compacted); +} + +TEST_F(CompactionPickerTest, UniversalMarkedL0WithOverlap) { + const uint64_t kFileSize = 100000; + + ioptions_.compaction_style = kCompactionStyleUniversal; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + + // This test covers the case where a file is being compacted, and a + // delete triggered compaction is then scheduled. The latter should stop + // at the first file being compacted + NewVersionStorage(1, kCompactionStyleUniversal); + + // Mark file number 4 for compaction + Add(0, 4U, "260", "300", 1 * kFileSize, 0, 260, 300, 0, true); + Add(0, 5U, "240", "290", 2 * kFileSize, 0, 201, 250); + Add(0, 3U, "301", "350", 4 * kFileSize, 0, 101, 150); + Add(0, 6U, "501", "750", 8 * kFileSize, 0, 50, 100); + UpdateVersionStorageInfo(); + file_map_[3].first->being_compacted = true; + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + + ASSERT_TRUE(compaction); + // Validate that its a delete triggered compaction + ASSERT_EQ(CompactionReason::kFilesMarkedForCompaction, + compaction->compaction_reason()); + ASSERT_EQ(0, compaction->output_level()); + ASSERT_EQ(0, compaction->start_level()); + ASSERT_EQ(2U, compaction->num_input_files(0)); + ASSERT_TRUE(file_map_[4].first->being_compacted); + ASSERT_TRUE(file_map_[5].first->being_compacted); +} + +TEST_F(CompactionPickerTest, UniversalMarkedL0Overlap2) { + const uint64_t kFileSize = 100000; + + ioptions_.compaction_style = kCompactionStyleUniversal; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + + // This test covers the case where a delete triggered compaction is + // scheduled first, followed by a "regular" compaction. The latter + // should fail + NewVersionStorage(1, kCompactionStyleUniversal); + + // Mark file number 5 for compaction + Add(0, 4U, "260", "300", 1 * kFileSize, 0, 260, 300, + /*compensated_file_size*/ 0, /*marked_for_compact*/ false, + /* temperature*/ Temperature::kUnknown, + /*oldest_ancestor_time*/ kUnknownOldestAncesterTime, + /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(), + /*epoch_number*/ 4); + Add(0, 5U, "240", "290", 2 * kFileSize, 0, 201, 250, 0, true, + /* temperature*/ Temperature::kUnknown, + /*oldest_ancestor_time*/ kUnknownOldestAncesterTime, + /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(), + /*epoch_number*/ 3); + Add(0, 3U, "301", "350", 4 * kFileSize, 0, 101, 150, + /*compensated_file_size*/ 0, /*marked_for_compact*/ false, + /* temperature*/ Temperature::kUnknown, + /*oldest_ancestor_time*/ kUnknownOldestAncesterTime, + /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(), + /*epoch_number*/ 2); + Add(0, 6U, "501", "750", 8 * kFileSize, 0, 50, 100, + /*compensated_file_size*/ 0, /*marked_for_compact*/ false, + /* temperature*/ Temperature::kUnknown, + /*oldest_ancestor_time*/ kUnknownOldestAncesterTime, + /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(), + /*epoch_number*/ 1); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + + ASSERT_TRUE(compaction); + // Validate that its a delete triggered compaction + ASSERT_EQ(CompactionReason::kFilesMarkedForCompaction, + compaction->compaction_reason()); + ASSERT_EQ(0, compaction->output_level()); + ASSERT_EQ(0, compaction->start_level()); + ASSERT_EQ(3U, compaction->num_input_files(0)); + ASSERT_TRUE(file_map_[5].first->being_compacted); + ASSERT_TRUE(file_map_[3].first->being_compacted); + ASSERT_TRUE(file_map_[6].first->being_compacted); + + AddVersionStorage(); + Add(0, 1U, "150", "200", kFileSize, 0, 500, 550, /*compensated_file_size*/ 0, + /*marked_for_compact*/ false, + /* temperature*/ Temperature::kUnknown, + /*oldest_ancestor_time*/ kUnknownOldestAncesterTime, + /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(), + /*epoch_number*/ 6); + Add(0, 2U, "201", "250", kFileSize, 0, 401, 450, /*compensated_file_size*/ 0, + /*marked_for_compact*/ false, + /* temperature*/ Temperature::kUnknown, + /*oldest_ancestor_time*/ kUnknownOldestAncesterTime, + /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(), + /*epoch_number*/ 5); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction2( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + ASSERT_TRUE(compaction2); + ASSERT_EQ(3U, compaction->num_input_files(0)); + ASSERT_TRUE(file_map_[1].first->being_compacted); + ASSERT_TRUE(file_map_[2].first->being_compacted); + ASSERT_TRUE(file_map_[4].first->being_compacted); +} + +TEST_F(CompactionPickerTest, UniversalMarkedManualCompaction) { + const uint64_t kFileSize = 100000; + const int kNumLevels = 7; + + // This test makes sure the `files_marked_for_compaction_` is updated after + // creating manual compaction. + ioptions_.compaction_style = kCompactionStyleUniversal; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + + NewVersionStorage(kNumLevels, kCompactionStyleUniversal); + + // Add 3 files marked for compaction + Add(0, 3U, "301", "350", 4 * kFileSize, 0, 101, 150, 0, true); + Add(0, 4U, "260", "300", 1 * kFileSize, 0, 260, 300, 0, true); + Add(0, 5U, "240", "290", 2 * kFileSize, 0, 201, 250, 0, true); + UpdateVersionStorageInfo(); + + // All 3 files are marked for compaction + ASSERT_EQ(3U, vstorage_->FilesMarkedForCompaction().size()); + + bool manual_conflict = false; + InternalKey* manual_end = nullptr; + std::unique_ptr compaction( + universal_compaction_picker.CompactRange( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + ColumnFamilyData::kCompactAllLevels, 6, CompactRangeOptions(), + nullptr, nullptr, &manual_end, &manual_conflict, + std::numeric_limits::max(), "")); + + ASSERT_TRUE(compaction); + + ASSERT_EQ(CompactionReason::kManualCompaction, + compaction->compaction_reason()); + ASSERT_EQ(kNumLevels - 1, compaction->output_level()); + ASSERT_EQ(0, compaction->start_level()); + ASSERT_EQ(3U, compaction->num_input_files(0)); + ASSERT_TRUE(file_map_[3].first->being_compacted); + ASSERT_TRUE(file_map_[4].first->being_compacted); + ASSERT_TRUE(file_map_[5].first->being_compacted); + + // After creating the manual compaction, all files should be cleared from + // `FilesMarkedForCompaction`. So they won't be picked by others. + ASSERT_EQ(0U, vstorage_->FilesMarkedForCompaction().size()); +} + +TEST_F(CompactionPickerTest, UniversalSizeAmpTierCompactionNonLastLevel) { + // This test make sure size amplification compaction could still be triggered + // if the last sorted run is not the last level. + const uint64_t kFileSize = 100000; + const int kNumLevels = 7; + const int kLastLevel = kNumLevels - 1; + + ioptions_.compaction_style = kCompactionStyleUniversal; + ioptions_.preclude_last_level_data_seconds = 1000; + mutable_cf_options_.compaction_options_universal + .max_size_amplification_percent = 200; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + + NewVersionStorage(kNumLevels, kCompactionStyleUniversal); + Add(0, 100U, "100", "300", 1 * kFileSize); + Add(0, 101U, "200", "400", 1 * kFileSize); + Add(4, 90U, "100", "600", 4 * kFileSize); + Add(5, 80U, "200", "300", 2 * kFileSize); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + + // Make sure it's a size amp compaction and includes all files + ASSERT_EQ(compaction->compaction_reason(), + CompactionReason::kUniversalSizeAmplification); + ASSERT_EQ(compaction->output_level(), kLastLevel); + ASSERT_EQ(compaction->input_levels(0)->num_files, 2); + ASSERT_EQ(compaction->input_levels(4)->num_files, 1); + ASSERT_EQ(compaction->input_levels(5)->num_files, 1); +} + +TEST_F(CompactionPickerTest, UniversalSizeRatioTierCompactionLastLevel) { + // This test makes sure the size amp calculation skips the last level (L6), so + // size amp compaction is not triggered, instead a size ratio compaction is + // triggered. + const uint64_t kFileSize = 100000; + const int kNumLevels = 7; + const int kLastLevel = kNumLevels - 1; + const int kPenultimateLevel = kLastLevel - 1; + + ioptions_.compaction_style = kCompactionStyleUniversal; + ioptions_.preclude_last_level_data_seconds = 1000; + mutable_cf_options_.compaction_options_universal + .max_size_amplification_percent = 200; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + + NewVersionStorage(kNumLevels, kCompactionStyleUniversal); + Add(0, 100U, "100", "300", 1 * kFileSize); + Add(0, 101U, "200", "400", 1 * kFileSize); + Add(5, 90U, "100", "600", 4 * kFileSize); + Add(6, 80U, "200", "300", 2 * kFileSize); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + + // Internally, size amp compaction is evaluated before size ratio compaction. + // Here to make sure it's size ratio compaction instead of size amp + ASSERT_EQ(compaction->compaction_reason(), + CompactionReason::kUniversalSizeRatio); + ASSERT_EQ(compaction->output_level(), kPenultimateLevel - 1); + ASSERT_EQ(compaction->input_levels(0)->num_files, 2); + ASSERT_EQ(compaction->input_levels(5)->num_files, 0); + ASSERT_EQ(compaction->input_levels(6)->num_files, 0); +} + +TEST_F(CompactionPickerTest, UniversalSizeAmpTierCompactionNotSuport) { + // Tiered compaction only support level_num > 2 (otherwise the penultimate + // level is going to be level 0, which may make thing more complicated), so + // when there's only 2 level, still treating level 1 as the last level for + // size amp compaction + const uint64_t kFileSize = 100000; + const int kNumLevels = 2; + const int kLastLevel = kNumLevels - 1; + + ioptions_.compaction_style = kCompactionStyleUniversal; + ioptions_.preclude_last_level_data_seconds = 1000; + mutable_cf_options_.compaction_options_universal + .max_size_amplification_percent = 200; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + + NewVersionStorage(kNumLevels, kCompactionStyleUniversal); + Add(0, 100U, "100", "300", 1 * kFileSize); + Add(0, 101U, "200", "400", 1 * kFileSize); + Add(0, 90U, "100", "600", 4 * kFileSize); + Add(1, 80U, "200", "300", 2 * kFileSize); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + + // size amp compaction is still triggered even preclude_last_level is set + ASSERT_EQ(compaction->compaction_reason(), + CompactionReason::kUniversalSizeAmplification); + ASSERT_EQ(compaction->output_level(), kLastLevel); + ASSERT_EQ(compaction->input_levels(0)->num_files, 3); + ASSERT_EQ(compaction->input_levels(1)->num_files, 1); +} + +TEST_F(CompactionPickerTest, UniversalSizeAmpTierCompactionLastLevel) { + // This test makes sure the size amp compaction for tiered storage could still + // be triggered, but only for non-last-level files + const uint64_t kFileSize = 100000; + const int kNumLevels = 7; + const int kLastLevel = kNumLevels - 1; + const int kPenultimateLevel = kLastLevel - 1; + + ioptions_.compaction_style = kCompactionStyleUniversal; + ioptions_.preclude_last_level_data_seconds = 1000; + mutable_cf_options_.compaction_options_universal + .max_size_amplification_percent = 200; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + + NewVersionStorage(kNumLevels, kCompactionStyleUniversal); + Add(0, 100U, "100", "300", 3 * kFileSize); + Add(0, 101U, "200", "400", 2 * kFileSize); + Add(5, 90U, "100", "600", 2 * kFileSize); + Add(6, 80U, "200", "300", 2 * kFileSize); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + + // It's a Size Amp compaction, but doesn't include the last level file and + // output to the penultimate level. + ASSERT_EQ(compaction->compaction_reason(), + CompactionReason::kUniversalSizeAmplification); + ASSERT_EQ(compaction->output_level(), kPenultimateLevel); + ASSERT_EQ(compaction->input_levels(0)->num_files, 2); + ASSERT_EQ(compaction->input_levels(5)->num_files, 1); + ASSERT_EQ(compaction->input_levels(6)->num_files, 0); +} + +TEST_F(CompactionPickerU64TsTest, Overlap) { + int num_levels = ioptions_.num_levels; + NewVersionStorage(num_levels, kCompactionStyleLevel); + + constexpr int level = 0; + constexpr uint64_t file_number = 20ULL; + constexpr char smallest[4] = "500"; + constexpr char largest[4] = "600"; + constexpr uint64_t ts_of_smallest = 12345ULL; + constexpr uint64_t ts_of_largest = 56789ULL; + + { + std::string ts1; + PutFixed64(&ts1, ts_of_smallest); + std::string ts2; + PutFixed64(&ts2, ts_of_largest); + Add(level, file_number, smallest, largest, + /*file_size=*/1U, /*path_id=*/0, + /*smallest_seq=*/100, /*largest_seq=*/100, /*compensated_file_size=*/0, + /*marked_for_compact=*/false, /*temperature=*/Temperature::kUnknown, + /*oldest_ancestor_time=*/kUnknownOldestAncesterTime, ts1, ts2); + UpdateVersionStorageInfo(); + } + + std::unordered_set input{file_number}; + + std::vector input_files; + ASSERT_OK(level_compaction_picker.GetCompactionInputsFromFileNumbers( + &input_files, &input, vstorage_.get(), CompactionOptions())); + std::unique_ptr comp1(level_compaction_picker.CompactFiles( + CompactionOptions(), input_files, level, vstorage_.get(), + mutable_cf_options_, mutable_db_options_, /*output_path_id=*/0)); + + { + // [600, ts=50000] to [600, ts=50000] is the range to check. + // ucmp->Compare(smallest_user_key, c->GetLargestUserKey()) > 0, but + // ucmp->CompareWithoutTimestamp(smallest_user_key, + // c->GetLargestUserKey()) == 0. + // Should still be considered overlapping. + std::string user_key_with_ts1(largest); + PutFixed64(&user_key_with_ts1, ts_of_largest - 1); + std::string user_key_with_ts2(largest); + PutFixed64(&user_key_with_ts2, ts_of_largest - 1); + ASSERT_TRUE(level_compaction_picker.RangeOverlapWithCompaction( + user_key_with_ts1, user_key_with_ts2, level)); + } + { + // [500, ts=60000] to [500, ts=60000] is the range to check. + // ucmp->Compare(largest_user_key, c->GetSmallestUserKey()) < 0, but + // ucmp->CompareWithoutTimestamp(largest_user_key, + // c->GetSmallestUserKey()) == 0. + // Should still be considered overlapping. + std::string user_key_with_ts1(smallest); + PutFixed64(&user_key_with_ts1, ts_of_smallest + 1); + std::string user_key_with_ts2(smallest); + PutFixed64(&user_key_with_ts2, ts_of_smallest + 1); + ASSERT_TRUE(level_compaction_picker.RangeOverlapWithCompaction( + user_key_with_ts1, user_key_with_ts2, level)); + } +} + +TEST_F(CompactionPickerU64TsTest, CannotTrivialMoveUniversal) { + constexpr uint64_t kFileSize = 100000; + + mutable_cf_options_.level0_file_num_compaction_trigger = 2; + mutable_cf_options_.compaction_options_universal.allow_trivial_move = true; + NewVersionStorage(1, kCompactionStyleUniversal); + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + UpdateVersionStorageInfo(); + // must return false when there's no files. + ASSERT_FALSE(universal_compaction_picker.NeedsCompaction(vstorage_.get())); + + std::string ts1; + PutFixed64(&ts1, 9000); + std::string ts2; + PutFixed64(&ts2, 8000); + std::string ts3; + PutFixed64(&ts3, 7000); + std::string ts4; + PutFixed64(&ts4, 6000); + + NewVersionStorage(3, kCompactionStyleUniversal); + // A compaction should be triggered and pick file 2 + Add(1, 1U, "150", "150", kFileSize, /*path_id=*/0, /*smallest_seq=*/100, + /*largest_seq=*/100, /*compensated_file_size=*/kFileSize, + /*marked_for_compact=*/false, Temperature::kUnknown, + kUnknownOldestAncesterTime, ts1, ts2); + Add(2, 2U, "150", "150", kFileSize, /*path_id=*/0, /*smallest_seq=*/100, + /*largest_seq=*/100, /*compensated_file_size=*/kFileSize, + /*marked_for_compact=*/false, Temperature::kUnknown, + kUnknownOldestAncesterTime, ts3, ts4); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction( + universal_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + &log_buffer_)); + assert(compaction); + ASSERT_TRUE(!compaction->is_trivial_move()); +} + +class PerKeyPlacementCompactionPickerTest + : public CompactionPickerTest, + public testing::WithParamInterface { + public: + PerKeyPlacementCompactionPickerTest() : CompactionPickerTest() {} + + void SetUp() override { enable_per_key_placement_ = GetParam(); } + + protected: + bool enable_per_key_placement_ = false; +}; + +TEST_P(PerKeyPlacementCompactionPickerTest, OverlapWithNormalCompaction) { + SyncPoint::GetInstance()->SetCallBack( + "Compaction::SupportsPerKeyPlacement:Enabled", [&](void* arg) { + auto supports_per_key_placement = static_cast(arg); + *supports_per_key_placement = enable_per_key_placement_; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + int num_levels = ioptions_.num_levels; + NewVersionStorage(num_levels, kCompactionStyleLevel); + + Add(0, 21U, "100", "150", 60000000U); + Add(0, 22U, "300", "350", 60000000U); + Add(5, 40U, "200", "250", 60000000U); + Add(6, 50U, "101", "351", 60000000U); + UpdateVersionStorageInfo(); + + CompactionOptions comp_options; + std::unordered_set input_set; + input_set.insert(40); + std::vector input_files; + ASSERT_OK(level_compaction_picker.GetCompactionInputsFromFileNumbers( + &input_files, &input_set, vstorage_.get(), comp_options)); + + std::unique_ptr comp1(level_compaction_picker.CompactFiles( + comp_options, input_files, 5, vstorage_.get(), mutable_cf_options_, + mutable_db_options_, 0)); + + input_set.clear(); + input_files.clear(); + input_set.insert(21); + input_set.insert(22); + input_set.insert(50); + ASSERT_OK(level_compaction_picker.GetCompactionInputsFromFileNumbers( + &input_files, &input_set, vstorage_.get(), comp_options)); + + ASSERT_EQ(enable_per_key_placement_, + level_compaction_picker.FilesRangeOverlapWithCompaction( + input_files, 6, + Compaction::EvaluatePenultimateLevel(vstorage_.get(), ioptions_, + 0, 6))); +} + +TEST_P(PerKeyPlacementCompactionPickerTest, NormalCompactionOverlap) { + SyncPoint::GetInstance()->SetCallBack( + "Compaction::SupportsPerKeyPlacement:Enabled", [&](void* arg) { + auto supports_per_key_placement = static_cast(arg); + *supports_per_key_placement = enable_per_key_placement_; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + int num_levels = ioptions_.num_levels; + NewVersionStorage(num_levels, kCompactionStyleLevel); + + Add(0, 21U, "100", "150", 60000000U); + Add(0, 22U, "300", "350", 60000000U); + Add(4, 40U, "200", "220", 60000000U); + Add(4, 41U, "230", "250", 60000000U); + Add(6, 50U, "101", "351", 60000000U); + UpdateVersionStorageInfo(); + + CompactionOptions comp_options; + std::unordered_set input_set; + input_set.insert(21); + input_set.insert(22); + input_set.insert(50); + std::vector input_files; + ASSERT_OK(level_compaction_picker.GetCompactionInputsFromFileNumbers( + &input_files, &input_set, vstorage_.get(), comp_options)); + + std::unique_ptr comp1(level_compaction_picker.CompactFiles( + comp_options, input_files, 6, vstorage_.get(), mutable_cf_options_, + mutable_db_options_, 0)); + + input_set.clear(); + input_files.clear(); + input_set.insert(40); + input_set.insert(41); + ASSERT_OK(level_compaction_picker.GetCompactionInputsFromFileNumbers( + &input_files, &input_set, vstorage_.get(), comp_options)); + + ASSERT_EQ(enable_per_key_placement_, + level_compaction_picker.FilesRangeOverlapWithCompaction( + input_files, 5, Compaction::kInvalidLevel)); +} + +TEST_P(PerKeyPlacementCompactionPickerTest, + OverlapWithNormalCompactionUniveral) { + SyncPoint::GetInstance()->SetCallBack( + "Compaction::SupportsPerKeyPlacement:Enabled", [&](void* arg) { + auto supports_per_key_placement = static_cast(arg); + *supports_per_key_placement = enable_per_key_placement_; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + int num_levels = ioptions_.num_levels; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + NewVersionStorage(num_levels, kCompactionStyleUniversal); + + Add(0, 21U, "100", "150", 60000000U); + Add(0, 22U, "300", "350", 60000000U); + Add(5, 40U, "200", "250", 60000000U); + Add(6, 50U, "101", "351", 60000000U); + UpdateVersionStorageInfo(); + + CompactionOptions comp_options; + std::unordered_set input_set; + input_set.insert(40); + std::vector input_files; + ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers( + &input_files, &input_set, vstorage_.get(), comp_options)); + + std::unique_ptr comp1(universal_compaction_picker.CompactFiles( + comp_options, input_files, 5, vstorage_.get(), mutable_cf_options_, + mutable_db_options_, 0)); + + input_set.clear(); + input_files.clear(); + input_set.insert(21); + input_set.insert(22); + input_set.insert(50); + ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers( + &input_files, &input_set, vstorage_.get(), comp_options)); + + ASSERT_EQ(enable_per_key_placement_, + universal_compaction_picker.FilesRangeOverlapWithCompaction( + input_files, 6, + Compaction::EvaluatePenultimateLevel(vstorage_.get(), ioptions_, + 0, 6))); +} + +TEST_P(PerKeyPlacementCompactionPickerTest, NormalCompactionOverlapUniversal) { + SyncPoint::GetInstance()->SetCallBack( + "Compaction::SupportsPerKeyPlacement:Enabled", [&](void* arg) { + auto supports_per_key_placement = static_cast(arg); + *supports_per_key_placement = enable_per_key_placement_; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + int num_levels = ioptions_.num_levels; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + NewVersionStorage(num_levels, kCompactionStyleUniversal); + + Add(0, 21U, "100", "150", 60000000U); + Add(0, 22U, "300", "350", 60000000U); + Add(4, 40U, "200", "220", 60000000U); + Add(4, 41U, "230", "250", 60000000U); + Add(6, 50U, "101", "351", 60000000U); + UpdateVersionStorageInfo(); + + CompactionOptions comp_options; + std::unordered_set input_set; + input_set.insert(21); + input_set.insert(22); + input_set.insert(50); + std::vector input_files; + ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers( + &input_files, &input_set, vstorage_.get(), comp_options)); + + std::unique_ptr comp1(universal_compaction_picker.CompactFiles( + comp_options, input_files, 6, vstorage_.get(), mutable_cf_options_, + mutable_db_options_, 0)); + + input_set.clear(); + input_files.clear(); + input_set.insert(40); + input_set.insert(41); + ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers( + &input_files, &input_set, vstorage_.get(), comp_options)); + + ASSERT_EQ(enable_per_key_placement_, + universal_compaction_picker.FilesRangeOverlapWithCompaction( + input_files, 5, Compaction::kInvalidLevel)); +} + +TEST_P(PerKeyPlacementCompactionPickerTest, PenultimateOverlapUniversal) { + // This test is make sure the Tiered compaction would lock whole range of + // both output level and penultimate level + if (enable_per_key_placement_) { + ioptions_.preclude_last_level_data_seconds = 10000; + } + + int num_levels = ioptions_.num_levels; + ioptions_.compaction_style = kCompactionStyleUniversal; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + NewVersionStorage(num_levels, kCompactionStyleUniversal); + + // L4: [200, 220] [230, 250] [360, 380] + // L5: + // L6: [101, 351] + Add(4, 40U, "200", "220", 60000000U); + Add(4, 41U, "230", "250", 60000000U); + Add(4, 42U, "360", "380", 60000000U); + Add(6, 60U, "101", "351", 60000000U); + UpdateVersionStorageInfo(); + + // the existing compaction is the 1st L4 file + L6 file + // then compaction of the 2nd L4 file to L5 (penultimate level) is overlapped + // when the tiered compaction feature is on. + CompactionOptions comp_options; + std::unordered_set input_set; + input_set.insert(40); + input_set.insert(60); + std::vector input_files; + ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers( + &input_files, &input_set, vstorage_.get(), comp_options)); + + std::unique_ptr comp1(universal_compaction_picker.CompactFiles( + comp_options, input_files, 6, vstorage_.get(), mutable_cf_options_, + mutable_db_options_, 0)); + + input_set.clear(); + input_files.clear(); + input_set.insert(41); + ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers( + &input_files, &input_set, vstorage_.get(), comp_options)); + + ASSERT_EQ(enable_per_key_placement_, + universal_compaction_picker.FilesRangeOverlapWithCompaction( + input_files, 5, Compaction::kInvalidLevel)); + + // compacting the 3rd L4 file is always safe: + input_set.clear(); + input_files.clear(); + input_set.insert(42); + ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers( + &input_files, &input_set, vstorage_.get(), comp_options)); + + ASSERT_FALSE(universal_compaction_picker.FilesRangeOverlapWithCompaction( + input_files, 5, Compaction::kInvalidLevel)); +} + +TEST_P(PerKeyPlacementCompactionPickerTest, LastLevelOnlyOverlapUniversal) { + if (enable_per_key_placement_) { + ioptions_.preclude_last_level_data_seconds = 10000; + } + + int num_levels = ioptions_.num_levels; + ioptions_.compaction_style = kCompactionStyleUniversal; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + NewVersionStorage(num_levels, kCompactionStyleUniversal); + + // L4: [200, 220] [230, 250] [360, 380] + // L5: + // L6: [101, 351] + Add(4, 40U, "200", "220", 60000000U); + Add(4, 41U, "230", "250", 60000000U); + Add(4, 42U, "360", "380", 60000000U); + Add(6, 60U, "101", "351", 60000000U); + UpdateVersionStorageInfo(); + + CompactionOptions comp_options; + std::unordered_set input_set; + input_set.insert(60); + std::vector input_files; + ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers( + &input_files, &input_set, vstorage_.get(), comp_options)); + + std::unique_ptr comp1(universal_compaction_picker.CompactFiles( + comp_options, input_files, 6, vstorage_.get(), mutable_cf_options_, + mutable_db_options_, 0)); + + // cannot compact file 41 if the preclude_last_level feature is on, otherwise + // compact file 41 is okay. + input_set.clear(); + input_files.clear(); + input_set.insert(41); + ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers( + &input_files, &input_set, vstorage_.get(), comp_options)); + + ASSERT_EQ(enable_per_key_placement_, + universal_compaction_picker.FilesRangeOverlapWithCompaction( + input_files, 5, Compaction::kInvalidLevel)); + + // compacting the 3rd L4 file is always safe: + input_set.clear(); + input_files.clear(); + input_set.insert(42); + ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers( + &input_files, &input_set, vstorage_.get(), comp_options)); + + ASSERT_FALSE(universal_compaction_picker.FilesRangeOverlapWithCompaction( + input_files, 5, Compaction::kInvalidLevel)); +} + +TEST_P(PerKeyPlacementCompactionPickerTest, + LastLevelOnlyFailPenultimateUniversal) { + // This is to test last_level only compaction still unable to do the + // penultimate level compaction if there's already a file in the penultimate + // level. + // This should rarely happen in universal compaction, as the non-empty L5 + // should be included in the compaction. + if (enable_per_key_placement_) { + ioptions_.preclude_last_level_data_seconds = 10000; + } + + int num_levels = ioptions_.num_levels; + ioptions_.compaction_style = kCompactionStyleUniversal; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + NewVersionStorage(num_levels, kCompactionStyleUniversal); + + // L4: [200, 220] + // L5: [230, 250] + // L6: [101, 351] + Add(4, 40U, "200", "220", 60000000U); + Add(5, 50U, "230", "250", 60000000U); + Add(6, 60U, "101", "351", 60000000U); + UpdateVersionStorageInfo(); + + CompactionOptions comp_options; + std::unordered_set input_set; + input_set.insert(60); + std::vector input_files; + ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers( + &input_files, &input_set, vstorage_.get(), comp_options)); + + std::unique_ptr comp1(universal_compaction_picker.CompactFiles( + comp_options, input_files, 6, vstorage_.get(), mutable_cf_options_, + mutable_db_options_, 0)); + + ASSERT_TRUE(comp1); + ASSERT_EQ(comp1->GetPenultimateLevel(), Compaction::kInvalidLevel); + + // As comp1 cannot be output to the penultimate level, compacting file 40 to + // L5 is always safe. + input_set.clear(); + input_files.clear(); + input_set.insert(40); + ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers( + &input_files, &input_set, vstorage_.get(), comp_options)); + + ASSERT_FALSE(universal_compaction_picker.FilesRangeOverlapWithCompaction( + input_files, 5, Compaction::kInvalidLevel)); + + std::unique_ptr comp2(universal_compaction_picker.CompactFiles( + comp_options, input_files, 5, vstorage_.get(), mutable_cf_options_, + mutable_db_options_, 0)); + ASSERT_TRUE(comp2); + ASSERT_EQ(Compaction::kInvalidLevel, comp2->GetPenultimateLevel()); +} + +TEST_P(PerKeyPlacementCompactionPickerTest, + LastLevelOnlyConflictWithOngoingUniversal) { + // This is to test last_level only compaction still unable to do the + // penultimate level compaction if there's already an ongoing compaction to + // the penultimate level + if (enable_per_key_placement_) { + ioptions_.preclude_last_level_data_seconds = 10000; + } + + int num_levels = ioptions_.num_levels; + ioptions_.compaction_style = kCompactionStyleUniversal; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + NewVersionStorage(num_levels, kCompactionStyleUniversal); + + // L4: [200, 220] [230, 250] [360, 380] + // L5: + // L6: [101, 351] + Add(4, 40U, "200", "220", 60000000U); + Add(4, 41U, "230", "250", 60000000U); + Add(4, 42U, "360", "380", 60000000U); + Add(6, 60U, "101", "351", 60000000U); + UpdateVersionStorageInfo(); + + // create an ongoing compaction to L5 (penultimate level) + CompactionOptions comp_options; + std::unordered_set input_set; + input_set.insert(40); + std::vector input_files; + ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers( + &input_files, &input_set, vstorage_.get(), comp_options)); + + std::unique_ptr comp1(universal_compaction_picker.CompactFiles( + comp_options, input_files, 5, vstorage_.get(), mutable_cf_options_, + mutable_db_options_, 0)); + + ASSERT_TRUE(comp1); + ASSERT_EQ(comp1->GetPenultimateLevel(), Compaction::kInvalidLevel); + + input_set.clear(); + input_files.clear(); + input_set.insert(60); + ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers( + &input_files, &input_set, vstorage_.get(), comp_options)); + + ASSERT_EQ(enable_per_key_placement_, + universal_compaction_picker.FilesRangeOverlapWithCompaction( + input_files, 6, + Compaction::EvaluatePenultimateLevel(vstorage_.get(), ioptions_, + 6, 6))); + + if (!enable_per_key_placement_) { + std::unique_ptr comp2(universal_compaction_picker.CompactFiles( + comp_options, input_files, 6, vstorage_.get(), mutable_cf_options_, + mutable_db_options_, 0)); + ASSERT_TRUE(comp2); + ASSERT_EQ(Compaction::kInvalidLevel, comp2->GetPenultimateLevel()); + } +} + +TEST_P(PerKeyPlacementCompactionPickerTest, + LastLevelOnlyNoConflictWithOngoingUniversal) { + // This is similar to `LastLevelOnlyConflictWithOngoingUniversal`, the only + // change is the ongoing compaction to L5 has no overlap with the last level + // compaction, so it's safe to move data from the last level to the + // penultimate level. + if (enable_per_key_placement_) { + ioptions_.preclude_last_level_data_seconds = 10000; + } + + int num_levels = ioptions_.num_levels; + ioptions_.compaction_style = kCompactionStyleUniversal; + UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_); + NewVersionStorage(num_levels, kCompactionStyleUniversal); + + // L4: [200, 220] [230, 250] [360, 380] + // L5: + // L6: [101, 351] + Add(4, 40U, "200", "220", 60000000U); + Add(4, 41U, "230", "250", 60000000U); + Add(4, 42U, "360", "380", 60000000U); + Add(6, 60U, "101", "351", 60000000U); + UpdateVersionStorageInfo(); + + // create an ongoing compaction to L5 (penultimate level) + CompactionOptions comp_options; + std::unordered_set input_set; + input_set.insert(42); + std::vector input_files; + ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers( + &input_files, &input_set, vstorage_.get(), comp_options)); + + std::unique_ptr comp1(universal_compaction_picker.CompactFiles( + comp_options, input_files, 5, vstorage_.get(), mutable_cf_options_, + mutable_db_options_, 0)); + + ASSERT_TRUE(comp1); + ASSERT_EQ(comp1->GetPenultimateLevel(), Compaction::kInvalidLevel); + + input_set.clear(); + input_files.clear(); + input_set.insert(60); + ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers( + &input_files, &input_set, vstorage_.get(), comp_options)); + + // always safe to move data up + ASSERT_FALSE(universal_compaction_picker.FilesRangeOverlapWithCompaction( + input_files, 6, + Compaction::EvaluatePenultimateLevel(vstorage_.get(), ioptions_, 6, 6))); + + // 2 compactions can be run in parallel + std::unique_ptr comp2(universal_compaction_picker.CompactFiles( + comp_options, input_files, 6, vstorage_.get(), mutable_cf_options_, + mutable_db_options_, 0)); + ASSERT_TRUE(comp2); + if (enable_per_key_placement_) { + ASSERT_NE(Compaction::kInvalidLevel, comp2->GetPenultimateLevel()); + } else { + ASSERT_EQ(Compaction::kInvalidLevel, comp2->GetPenultimateLevel()); + } +} + +INSTANTIATE_TEST_CASE_P(PerKeyPlacementCompactionPickerTest, + PerKeyPlacementCompactionPickerTest, ::testing::Bool()); + + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_picker_universal.cc b/librocksdb-sys/rocksdb/db/compaction/compaction_picker_universal.cc new file mode 100644 index 0000000..9eaf395 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_picker_universal.cc @@ -0,0 +1,1445 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/compaction/compaction_picker_universal.h" + +#include +#include +#include +#include +#include + +#include "db/column_family.h" +#include "file/filename.h" +#include "logging/log_buffer.h" +#include "logging/logging.h" +#include "monitoring/statistics_impl.h" +#include "test_util/sync_point.h" +#include "util/random.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { +namespace { +// A helper class that form universal compactions. The class is used by +// UniversalCompactionPicker::PickCompaction(). +// The usage is to create the class, and get the compaction object by calling +// PickCompaction(). +class UniversalCompactionBuilder { + public: + UniversalCompactionBuilder( + const ImmutableOptions& ioptions, const InternalKeyComparator* icmp, + const std::string& cf_name, const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage, + UniversalCompactionPicker* picker, LogBuffer* log_buffer) + : ioptions_(ioptions), + icmp_(icmp), + cf_name_(cf_name), + mutable_cf_options_(mutable_cf_options), + mutable_db_options_(mutable_db_options), + vstorage_(vstorage), + picker_(picker), + log_buffer_(log_buffer) {} + + // Form and return the compaction object. The caller owns return object. + Compaction* PickCompaction(); + + private: + struct SortedRun { + SortedRun(int _level, FileMetaData* _file, uint64_t _size, + uint64_t _compensated_file_size, bool _being_compacted) + : level(_level), + file(_file), + size(_size), + compensated_file_size(_compensated_file_size), + being_compacted(_being_compacted) { + assert(compensated_file_size > 0); + assert(level != 0 || file != nullptr); + } + + void Dump(char* out_buf, size_t out_buf_size, + bool print_path = false) const; + + // sorted_run_count is added into the string to print + void DumpSizeInfo(char* out_buf, size_t out_buf_size, + size_t sorted_run_count) const; + + int level; + // `file` Will be null for level > 0. For level = 0, the sorted run is + // for this file. + FileMetaData* file; + // For level > 0, `size` and `compensated_file_size` are sum of sizes all + // files in the level. `being_compacted` should be the same for all files + // in a non-zero level. Use the value here. + uint64_t size; + uint64_t compensated_file_size; + bool being_compacted; + }; + + // Pick Universal compaction to limit read amplification + Compaction* PickCompactionToReduceSortedRuns( + unsigned int ratio, unsigned int max_number_of_files_to_compact); + + // Pick Universal compaction to limit space amplification. + Compaction* PickCompactionToReduceSizeAmp(); + + // Try to pick incremental compaction to reduce space amplification. + // It will return null if it cannot find a fanout within the threshold. + // Fanout is defined as + // total size of files to compact at output level + // -------------------------------------------------- + // total size of files to compact at other levels + Compaction* PickIncrementalForReduceSizeAmp(double fanout_threshold); + + Compaction* PickDeleteTriggeredCompaction(); + + // Form a compaction from the sorted run indicated by start_index to the + // oldest sorted run. + // The caller is responsible for making sure that those files are not in + // compaction. + Compaction* PickCompactionToOldest(size_t start_index, + CompactionReason compaction_reason); + + Compaction* PickCompactionWithSortedRunRange( + size_t start_index, size_t end_index, CompactionReason compaction_reason); + + // Try to pick periodic compaction. The caller should only call it + // if there is at least one file marked for periodic compaction. + // null will be returned if no such a compaction can be formed + // because some files are being compacted. + Compaction* PickPeriodicCompaction(); + + // Used in universal compaction when the allow_trivial_move + // option is set. Checks whether there are any overlapping files + // in the input. Returns true if the input files are non + // overlapping. + bool IsInputFilesNonOverlapping(Compaction* c); + + uint64_t GetMaxOverlappingBytes() const; + + const ImmutableOptions& ioptions_; + const InternalKeyComparator* icmp_; + double score_; + std::vector sorted_runs_; + const std::string& cf_name_; + const MutableCFOptions& mutable_cf_options_; + const MutableDBOptions& mutable_db_options_; + VersionStorageInfo* vstorage_; + UniversalCompactionPicker* picker_; + LogBuffer* log_buffer_; + + static std::vector CalculateSortedRuns( + const VersionStorageInfo& vstorage, int last_level); + + // Pick a path ID to place a newly generated file, with its estimated file + // size. + static uint32_t GetPathId(const ImmutableCFOptions& ioptions, + const MutableCFOptions& mutable_cf_options, + uint64_t file_size); +}; + +// Used in universal compaction when trivial move is enabled. +// This structure is used for the construction of min heap +// that contains the file meta data, the level of the file +// and the index of the file in that level + +struct InputFileInfo { + InputFileInfo() : f(nullptr), level(0), index(0) {} + + FileMetaData* f; + size_t level; + size_t index; +}; + +// Used in universal compaction when trivial move is enabled. +// This comparator is used for the construction of min heap +// based on the smallest key of the file. +struct SmallestKeyHeapComparator { + explicit SmallestKeyHeapComparator(const Comparator* ucmp) { ucmp_ = ucmp; } + + bool operator()(InputFileInfo i1, InputFileInfo i2) const { + return (ucmp_->CompareWithoutTimestamp(i1.f->smallest.user_key(), + i2.f->smallest.user_key()) > 0); + } + + private: + const Comparator* ucmp_; +}; + +using SmallestKeyHeap = + std::priority_queue, + SmallestKeyHeapComparator>; + +// This function creates the heap that is used to find if the files are +// overlapping during universal compaction when the allow_trivial_move +// is set. +SmallestKeyHeap create_level_heap(Compaction* c, const Comparator* ucmp) { + SmallestKeyHeap smallest_key_priority_q = + SmallestKeyHeap(SmallestKeyHeapComparator(ucmp)); + + InputFileInfo input_file; + + for (size_t l = 0; l < c->num_input_levels(); l++) { + if (c->num_input_files(l) != 0) { + if (l == 0 && c->start_level() == 0) { + for (size_t i = 0; i < c->num_input_files(0); i++) { + input_file.f = c->input(0, i); + input_file.level = 0; + input_file.index = i; + smallest_key_priority_q.push(std::move(input_file)); + } + } else { + input_file.f = c->input(l, 0); + input_file.level = l; + input_file.index = 0; + smallest_key_priority_q.push(std::move(input_file)); + } + } + } + return smallest_key_priority_q; +} + +#ifndef NDEBUG +// smallest_seqno and largest_seqno are set iff. `files` is not empty. +void GetSmallestLargestSeqno(const std::vector& files, + SequenceNumber* smallest_seqno, + SequenceNumber* largest_seqno) { + bool is_first = true; + for (FileMetaData* f : files) { + assert(f->fd.smallest_seqno <= f->fd.largest_seqno); + if (is_first) { + is_first = false; + *smallest_seqno = f->fd.smallest_seqno; + *largest_seqno = f->fd.largest_seqno; + } else { + if (f->fd.smallest_seqno < *smallest_seqno) { + *smallest_seqno = f->fd.smallest_seqno; + } + if (f->fd.largest_seqno > *largest_seqno) { + *largest_seqno = f->fd.largest_seqno; + } + } + } +} +#endif +} // namespace + +// Algorithm that checks to see if there are any overlapping +// files in the input +bool UniversalCompactionBuilder::IsInputFilesNonOverlapping(Compaction* c) { + auto comparator = icmp_->user_comparator(); + int first_iter = 1; + + InputFileInfo prev, curr, next; + + SmallestKeyHeap smallest_key_priority_q = + create_level_heap(c, icmp_->user_comparator()); + + while (!smallest_key_priority_q.empty()) { + curr = smallest_key_priority_q.top(); + smallest_key_priority_q.pop(); + + if (first_iter) { + prev = curr; + first_iter = 0; + } else { + if (comparator->CompareWithoutTimestamp( + prev.f->largest.user_key(), curr.f->smallest.user_key()) >= 0) { + // found overlapping files, return false + return false; + } + assert(comparator->CompareWithoutTimestamp( + curr.f->largest.user_key(), prev.f->largest.user_key()) > 0); + prev = curr; + } + + next.f = nullptr; + + if (c->level(curr.level) != 0 && + curr.index < c->num_input_files(curr.level) - 1) { + next.f = c->input(curr.level, curr.index + 1); + next.level = curr.level; + next.index = curr.index + 1; + } + + if (next.f) { + smallest_key_priority_q.push(std::move(next)); + } + } + return true; +} + +bool UniversalCompactionPicker::NeedsCompaction( + const VersionStorageInfo* vstorage) const { + const int kLevel0 = 0; + if (vstorage->CompactionScore(kLevel0) >= 1) { + return true; + } + if (!vstorage->FilesMarkedForPeriodicCompaction().empty()) { + return true; + } + if (!vstorage->FilesMarkedForCompaction().empty()) { + return true; + } + return false; +} + +Compaction* UniversalCompactionPicker::PickCompaction( + const std::string& cf_name, const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage, + LogBuffer* log_buffer) { + UniversalCompactionBuilder builder(ioptions_, icmp_, cf_name, + mutable_cf_options, mutable_db_options, + vstorage, this, log_buffer); + return builder.PickCompaction(); +} + +void UniversalCompactionBuilder::SortedRun::Dump(char* out_buf, + size_t out_buf_size, + bool print_path) const { + if (level == 0) { + assert(file != nullptr); + if (file->fd.GetPathId() == 0 || !print_path) { + snprintf(out_buf, out_buf_size, "file %" PRIu64, file->fd.GetNumber()); + } else { + snprintf(out_buf, out_buf_size, + "file %" PRIu64 + "(path " + "%" PRIu32 ")", + file->fd.GetNumber(), file->fd.GetPathId()); + } + } else { + snprintf(out_buf, out_buf_size, "level %d", level); + } +} + +void UniversalCompactionBuilder::SortedRun::DumpSizeInfo( + char* out_buf, size_t out_buf_size, size_t sorted_run_count) const { + if (level == 0) { + assert(file != nullptr); + snprintf(out_buf, out_buf_size, + "file %" PRIu64 "[%" ROCKSDB_PRIszt + "] " + "with size %" PRIu64 " (compensated size %" PRIu64 ")", + file->fd.GetNumber(), sorted_run_count, file->fd.GetFileSize(), + file->compensated_file_size); + } else { + snprintf(out_buf, out_buf_size, + "level %d[%" ROCKSDB_PRIszt + "] " + "with size %" PRIu64 " (compensated size %" PRIu64 ")", + level, sorted_run_count, size, compensated_file_size); + } +} + +std::vector +UniversalCompactionBuilder::CalculateSortedRuns( + const VersionStorageInfo& vstorage, int last_level) { + std::vector ret; + for (FileMetaData* f : vstorage.LevelFiles(0)) { + ret.emplace_back(0, f, f->fd.GetFileSize(), f->compensated_file_size, + f->being_compacted); + } + for (int level = 1; level <= last_level; level++) { + uint64_t total_compensated_size = 0U; + uint64_t total_size = 0U; + bool being_compacted = false; + for (FileMetaData* f : vstorage.LevelFiles(level)) { + total_compensated_size += f->compensated_file_size; + total_size += f->fd.GetFileSize(); + // Size amp, read amp and periodic compactions always include all files + // for a non-zero level. However, a delete triggered compaction and + // a trivial move might pick a subset of files in a sorted run. So + // always check all files in a sorted run and mark the entire run as + // being compacted if one or more files are being compacted + if (f->being_compacted) { + being_compacted = f->being_compacted; + } + } + if (total_compensated_size > 0) { + ret.emplace_back(level, nullptr, total_size, total_compensated_size, + being_compacted); + } + } + return ret; +} + +// Universal style of compaction. Pick files that are contiguous in +// time-range to compact. +Compaction* UniversalCompactionBuilder::PickCompaction() { + const int kLevel0 = 0; + score_ = vstorage_->CompactionScore(kLevel0); + int max_output_level = + vstorage_->MaxOutputLevel(ioptions_.allow_ingest_behind); + sorted_runs_ = CalculateSortedRuns(*vstorage_, max_output_level); + + if (sorted_runs_.size() == 0 || + (vstorage_->FilesMarkedForPeriodicCompaction().empty() && + vstorage_->FilesMarkedForCompaction().empty() && + sorted_runs_.size() < (unsigned int)mutable_cf_options_ + .level0_file_num_compaction_trigger)) { + ROCKS_LOG_BUFFER(log_buffer_, "[%s] Universal: nothing to do\n", + cf_name_.c_str()); + TEST_SYNC_POINT_CALLBACK( + "UniversalCompactionBuilder::PickCompaction:Return", nullptr); + return nullptr; + } + VersionStorageInfo::LevelSummaryStorage tmp; + ROCKS_LOG_BUFFER_MAX_SZ( + log_buffer_, 3072, + "[%s] Universal: sorted runs: %" ROCKSDB_PRIszt " files: %s\n", + cf_name_.c_str(), sorted_runs_.size(), vstorage_->LevelSummary(&tmp)); + + Compaction* c = nullptr; + // Periodic compaction has higher priority than other type of compaction + // because it's a hard requirement. + if (!vstorage_->FilesMarkedForPeriodicCompaction().empty()) { + // Always need to do a full compaction for periodic compaction. + c = PickPeriodicCompaction(); + TEST_SYNC_POINT_CALLBACK("PostPickPeriodicCompaction", c); + } + + // Check for size amplification. + if (c == nullptr && + sorted_runs_.size() >= + static_cast( + mutable_cf_options_.level0_file_num_compaction_trigger)) { + if ((c = PickCompactionToReduceSizeAmp()) != nullptr) { + TEST_SYNC_POINT("PickCompactionToReduceSizeAmpReturnNonnullptr"); + ROCKS_LOG_BUFFER(log_buffer_, "[%s] Universal: compacting for size amp\n", + cf_name_.c_str()); + } else { + // Size amplification is within limits. Try reducing read + // amplification while maintaining file size ratios. + unsigned int ratio = + mutable_cf_options_.compaction_options_universal.size_ratio; + + if ((c = PickCompactionToReduceSortedRuns(ratio, UINT_MAX)) != nullptr) { + TEST_SYNC_POINT("PickCompactionToReduceSortedRunsReturnNonnullptr"); + ROCKS_LOG_BUFFER(log_buffer_, + "[%s] Universal: compacting for size ratio\n", + cf_name_.c_str()); + } else { + // Size amplification and file size ratios are within configured limits. + // If max read amplification is exceeding configured limits, then force + // compaction without looking at filesize ratios and try to reduce + // the number of files to fewer than level0_file_num_compaction_trigger. + // This is guaranteed by NeedsCompaction() + assert(sorted_runs_.size() >= + static_cast( + mutable_cf_options_.level0_file_num_compaction_trigger)); + // Get the total number of sorted runs that are not being compacted + int num_sr_not_compacted = 0; + for (size_t i = 0; i < sorted_runs_.size(); i++) { + if (sorted_runs_[i].being_compacted == false) { + num_sr_not_compacted++; + } + } + + // The number of sorted runs that are not being compacted is greater + // than the maximum allowed number of sorted runs + if (num_sr_not_compacted > + mutable_cf_options_.level0_file_num_compaction_trigger) { + unsigned int num_files = + num_sr_not_compacted - + mutable_cf_options_.level0_file_num_compaction_trigger + 1; + if ((c = PickCompactionToReduceSortedRuns(UINT_MAX, num_files)) != + nullptr) { + ROCKS_LOG_BUFFER(log_buffer_, + "[%s] Universal: compacting for file num -- %u\n", + cf_name_.c_str(), num_files); + } + } + } + } + } + + if (c == nullptr) { + if ((c = PickDeleteTriggeredCompaction()) != nullptr) { + TEST_SYNC_POINT("PickDeleteTriggeredCompactionReturnNonnullptr"); + ROCKS_LOG_BUFFER(log_buffer_, + "[%s] Universal: delete triggered compaction\n", + cf_name_.c_str()); + } + } + + if (c == nullptr) { + TEST_SYNC_POINT_CALLBACK( + "UniversalCompactionBuilder::PickCompaction:Return", nullptr); + return nullptr; + } + assert(c->output_level() <= + vstorage_->MaxOutputLevel(ioptions_.allow_ingest_behind)); + + if (mutable_cf_options_.compaction_options_universal.allow_trivial_move == + true && + c->compaction_reason() != CompactionReason::kPeriodicCompaction) { + c->set_is_trivial_move(IsInputFilesNonOverlapping(c)); + } + +// validate that all the chosen files of L0 are non overlapping in time +#ifndef NDEBUG + bool is_first = true; + + size_t level_index = 0U; + if (c->start_level() == 0) { + for (auto f : *c->inputs(0)) { + assert(f->fd.smallest_seqno <= f->fd.largest_seqno); + if (is_first) { + is_first = false; + } + } + level_index = 1U; + } + for (; level_index < c->num_input_levels(); level_index++) { + if (c->num_input_files(level_index) != 0) { + SequenceNumber smallest_seqno = 0U; + SequenceNumber largest_seqno = 0U; + GetSmallestLargestSeqno(*(c->inputs(level_index)), &smallest_seqno, + &largest_seqno); + if (is_first) { + is_first = false; + } + } + } +#endif + // update statistics + size_t num_files = 0; + for (auto& each_level : *c->inputs()) { + num_files += each_level.files.size(); + } + RecordInHistogram(ioptions_.stats, NUM_FILES_IN_SINGLE_COMPACTION, num_files); + + picker_->RegisterCompaction(c); + vstorage_->ComputeCompactionScore(ioptions_, mutable_cf_options_); + + TEST_SYNC_POINT_CALLBACK("UniversalCompactionBuilder::PickCompaction:Return", + c); + return c; +} + +uint32_t UniversalCompactionBuilder::GetPathId( + const ImmutableCFOptions& ioptions, + const MutableCFOptions& mutable_cf_options, uint64_t file_size) { + // Two conditions need to be satisfied: + // (1) the target path needs to be able to hold the file's size + // (2) Total size left in this and previous paths need to be not + // smaller than expected future file size before this new file is + // compacted, which is estimated based on size_ratio. + // For example, if now we are compacting files of size (1, 1, 2, 4, 8), + // we will make sure the target file, probably with size of 16, will be + // placed in a path so that eventually when new files are generated and + // compacted to (1, 1, 2, 4, 8, 16), all those files can be stored in or + // before the path we chose. + // + // TODO(sdong): now the case of multiple column families is not + // considered in this algorithm. So the target size can be violated in + // that case. We need to improve it. + uint64_t accumulated_size = 0; + uint64_t future_size = + file_size * + (100 - mutable_cf_options.compaction_options_universal.size_ratio) / 100; + uint32_t p = 0; + assert(!ioptions.cf_paths.empty()); + for (; p < ioptions.cf_paths.size() - 1; p++) { + uint64_t target_size = ioptions.cf_paths[p].target_size; + if (target_size > file_size && + accumulated_size + (target_size - file_size) > future_size) { + return p; + } + accumulated_size += target_size; + } + return p; +} + +// +// Consider compaction files based on their size differences with +// the next file in time order. +// +Compaction* UniversalCompactionBuilder::PickCompactionToReduceSortedRuns( + unsigned int ratio, unsigned int max_number_of_files_to_compact) { + unsigned int min_merge_width = + mutable_cf_options_.compaction_options_universal.min_merge_width; + unsigned int max_merge_width = + mutable_cf_options_.compaction_options_universal.max_merge_width; + + const SortedRun* sr = nullptr; + bool done = false; + size_t start_index = 0; + unsigned int candidate_count = 0; + + unsigned int max_files_to_compact = + std::min(max_merge_width, max_number_of_files_to_compact); + min_merge_width = std::max(min_merge_width, 2U); + + // Caller checks the size before executing this function. This invariant is + // important because otherwise we may have a possible integer underflow when + // dealing with unsigned types. + assert(sorted_runs_.size() > 0); + + // Considers a candidate file only if it is smaller than the + // total size accumulated so far. + for (size_t loop = 0; loop < sorted_runs_.size(); loop++) { + candidate_count = 0; + + // Skip files that are already being compacted + for (sr = nullptr; loop < sorted_runs_.size(); loop++) { + sr = &sorted_runs_[loop]; + + if (!sr->being_compacted) { + candidate_count = 1; + break; + } + char file_num_buf[kFormatFileNumberBufSize]; + sr->Dump(file_num_buf, sizeof(file_num_buf)); + ROCKS_LOG_BUFFER(log_buffer_, + "[%s] Universal: %s" + "[%d] being compacted, skipping", + cf_name_.c_str(), file_num_buf, loop); + + sr = nullptr; + } + + // This file is not being compacted. Consider it as the + // first candidate to be compacted. + uint64_t candidate_size = sr != nullptr ? sr->compensated_file_size : 0; + if (sr != nullptr) { + char file_num_buf[kFormatFileNumberBufSize]; + sr->Dump(file_num_buf, sizeof(file_num_buf), true); + ROCKS_LOG_BUFFER(log_buffer_, + "[%s] Universal: Possible candidate %s[%d].", + cf_name_.c_str(), file_num_buf, loop); + } + + // Check if the succeeding files need compaction. + for (size_t i = loop + 1; + candidate_count < max_files_to_compact && i < sorted_runs_.size(); + i++) { + const SortedRun* succeeding_sr = &sorted_runs_[i]; + if (succeeding_sr->being_compacted) { + break; + } + // Pick files if the total/last candidate file size (increased by the + // specified ratio) is still larger than the next candidate file. + // candidate_size is the total size of files picked so far with the + // default kCompactionStopStyleTotalSize; with + // kCompactionStopStyleSimilarSize, it's simply the size of the last + // picked file. + double sz = candidate_size * (100.0 + ratio) / 100.0; + if (sz < static_cast(succeeding_sr->size)) { + break; + } + if (mutable_cf_options_.compaction_options_universal.stop_style == + kCompactionStopStyleSimilarSize) { + // Similar-size stopping rule: also check the last picked file isn't + // far larger than the next candidate file. + sz = (succeeding_sr->size * (100.0 + ratio)) / 100.0; + if (sz < static_cast(candidate_size)) { + // If the small file we've encountered begins a run of similar-size + // files, we'll pick them up on a future iteration of the outer + // loop. If it's some lonely straggler, it'll eventually get picked + // by the last-resort read amp strategy which disregards size ratios. + break; + } + candidate_size = succeeding_sr->compensated_file_size; + } else { // default kCompactionStopStyleTotalSize + candidate_size += succeeding_sr->compensated_file_size; + } + candidate_count++; + } + + // Found a series of consecutive files that need compaction. + if (candidate_count >= (unsigned int)min_merge_width) { + start_index = loop; + done = true; + break; + } else { + for (size_t i = loop; + i < loop + candidate_count && i < sorted_runs_.size(); i++) { + const SortedRun* skipping_sr = &sorted_runs_[i]; + char file_num_buf[256]; + skipping_sr->DumpSizeInfo(file_num_buf, sizeof(file_num_buf), loop); + ROCKS_LOG_BUFFER(log_buffer_, "[%s] Universal: Skipping %s", + cf_name_.c_str(), file_num_buf); + } + } + } + if (!done || candidate_count <= 1) { + return nullptr; + } + size_t first_index_after = start_index + candidate_count; + // Compression is enabled if files compacted earlier already reached + // size ratio of compression. + bool enable_compression = true; + int ratio_to_compress = + mutable_cf_options_.compaction_options_universal.compression_size_percent; + if (ratio_to_compress >= 0) { + uint64_t total_size = 0; + for (auto& sorted_run : sorted_runs_) { + total_size += sorted_run.compensated_file_size; + } + + uint64_t older_file_size = 0; + for (size_t i = sorted_runs_.size() - 1; i >= first_index_after; i--) { + older_file_size += sorted_runs_[i].size; + if (older_file_size * 100L >= total_size * (long)ratio_to_compress) { + enable_compression = false; + break; + } + } + } + + uint64_t estimated_total_size = 0; + for (unsigned int i = 0; i < first_index_after; i++) { + estimated_total_size += sorted_runs_[i].size; + } + uint32_t path_id = + GetPathId(ioptions_, mutable_cf_options_, estimated_total_size); + int start_level = sorted_runs_[start_index].level; + int output_level; + // last level is reserved for the files ingested behind + int max_output_level = + vstorage_->MaxOutputLevel(ioptions_.allow_ingest_behind); + if (first_index_after == sorted_runs_.size()) { + output_level = max_output_level; + } else if (sorted_runs_[first_index_after].level == 0) { + output_level = 0; + } else { + output_level = sorted_runs_[first_index_after].level - 1; + } + + std::vector inputs(max_output_level + 1); + for (size_t i = 0; i < inputs.size(); ++i) { + inputs[i].level = start_level + static_cast(i); + } + for (size_t i = start_index; i < first_index_after; i++) { + auto& picking_sr = sorted_runs_[i]; + if (picking_sr.level == 0) { + FileMetaData* picking_file = picking_sr.file; + inputs[0].files.push_back(picking_file); + } else { + auto& files = inputs[picking_sr.level - start_level].files; + for (auto* f : vstorage_->LevelFiles(picking_sr.level)) { + files.push_back(f); + } + } + char file_num_buf[256]; + picking_sr.DumpSizeInfo(file_num_buf, sizeof(file_num_buf), i); + ROCKS_LOG_BUFFER(log_buffer_, "[%s] Universal: Picking %s", + cf_name_.c_str(), file_num_buf); + } + + std::vector grandparents; + // Include grandparents for potential file cutting in incremental + // mode. It is for aligning file cutting boundaries across levels, + // so that subsequent compactions can pick files with aligned + // buffer. + // Single files are only picked up in incremental mode, so that + // there is no need for full range. + if (mutable_cf_options_.compaction_options_universal.incremental && + first_index_after < sorted_runs_.size() && + sorted_runs_[first_index_after].level > 1) { + grandparents = vstorage_->LevelFiles(sorted_runs_[first_index_after].level); + } + + if (output_level != 0 && + picker_->FilesRangeOverlapWithCompaction( + inputs, output_level, + Compaction::EvaluatePenultimateLevel(vstorage_, ioptions_, + start_level, output_level))) { + return nullptr; + } + CompactionReason compaction_reason; + if (max_number_of_files_to_compact == UINT_MAX) { + compaction_reason = CompactionReason::kUniversalSizeRatio; + } else { + compaction_reason = CompactionReason::kUniversalSortedRunNum; + } + return new Compaction(vstorage_, ioptions_, mutable_cf_options_, + mutable_db_options_, std::move(inputs), output_level, + MaxFileSizeForLevel(mutable_cf_options_, output_level, + kCompactionStyleUniversal), + GetMaxOverlappingBytes(), path_id, + GetCompressionType(vstorage_, mutable_cf_options_, + output_level, 1, enable_compression), + GetCompressionOptions(mutable_cf_options_, vstorage_, + output_level, enable_compression), + Temperature::kUnknown, + /* max_subcompactions */ 0, grandparents, + /* is manual */ false, /* trim_ts */ "", score_, + false /* deletion_compaction */, + /* l0_files_might_overlap */ true, compaction_reason); +} + +// Look at overall size amplification. If size amplification +// exceeds the configured value, then do a compaction +// of the candidate files all the way upto the earliest +// base file (overrides configured values of file-size ratios, +// min_merge_width and max_merge_width). +// +Compaction* UniversalCompactionBuilder::PickCompactionToReduceSizeAmp() { + // percentage flexibility while reducing size amplification + uint64_t ratio = mutable_cf_options_.compaction_options_universal + .max_size_amplification_percent; + + unsigned int candidate_count = 0; + uint64_t candidate_size = 0; + size_t start_index = 0; + const SortedRun* sr = nullptr; + + assert(!sorted_runs_.empty()); + if (sorted_runs_.back().being_compacted) { + return nullptr; + } + + // Skip files that are already being compacted + for (size_t loop = 0; loop + 1 < sorted_runs_.size(); loop++) { + sr = &sorted_runs_[loop]; + if (!sr->being_compacted) { + start_index = loop; // Consider this as the first candidate. + break; + } + char file_num_buf[kFormatFileNumberBufSize]; + sr->Dump(file_num_buf, sizeof(file_num_buf), true); + ROCKS_LOG_BUFFER(log_buffer_, + "[%s] Universal: skipping %s[%d] compacted %s", + cf_name_.c_str(), file_num_buf, loop, + " cannot be a candidate to reduce size amp.\n"); + sr = nullptr; + } + + if (sr == nullptr) { + return nullptr; // no candidate files + } + { + char file_num_buf[kFormatFileNumberBufSize]; + sr->Dump(file_num_buf, sizeof(file_num_buf), true); + ROCKS_LOG_BUFFER( + log_buffer_, + "[%s] Universal: First candidate %s[%" ROCKSDB_PRIszt "] %s", + cf_name_.c_str(), file_num_buf, start_index, " to reduce size amp.\n"); + } + + // size of the base sorted run for size amp calculation + uint64_t base_sr_size = sorted_runs_.back().size; + size_t sr_end_idx = sorted_runs_.size() - 1; + // If tiered compaction is enabled and the last sorted run is the last level + if (ioptions_.preclude_last_level_data_seconds > 0 && + ioptions_.num_levels > 2 && + sorted_runs_.back().level == ioptions_.num_levels - 1 && + sorted_runs_.size() > 1) { + sr_end_idx = sorted_runs_.size() - 2; + base_sr_size = sorted_runs_[sr_end_idx].size; + } + + // keep adding up all the remaining files + for (size_t loop = start_index; loop < sr_end_idx; loop++) { + sr = &sorted_runs_[loop]; + if (sr->being_compacted) { + // TODO with incremental compaction is supported, we might want to + // schedule some incremental compactions in parallel if needed. + char file_num_buf[kFormatFileNumberBufSize]; + sr->Dump(file_num_buf, sizeof(file_num_buf), true); + ROCKS_LOG_BUFFER( + log_buffer_, "[%s] Universal: Possible candidate %s[%d] %s", + cf_name_.c_str(), file_num_buf, start_index, + " is already being compacted. No size amp reduction possible.\n"); + return nullptr; + } + candidate_size += sr->compensated_file_size; + candidate_count++; + } + if (candidate_count == 0) { + return nullptr; + } + + // size amplification = percentage of additional size + if (candidate_size * 100 < ratio * base_sr_size) { + ROCKS_LOG_BUFFER( + log_buffer_, + "[%s] Universal: size amp not needed. newer-files-total-size %" PRIu64 + " earliest-file-size %" PRIu64, + cf_name_.c_str(), candidate_size, base_sr_size); + return nullptr; + } else { + ROCKS_LOG_BUFFER( + log_buffer_, + "[%s] Universal: size amp needed. newer-files-total-size %" PRIu64 + " earliest-file-size %" PRIu64, + cf_name_.c_str(), candidate_size, base_sr_size); + } + // Since incremental compaction can't include more than second last + // level, it can introduce penalty, compared to full compaction. We + // hard code the pentalty to be 80%. If we end up with a compaction + // fanout higher than 80% of full level compactions, we fall back + // to full level compaction. + // The 80% threshold is arbitrary and can be adjusted or made + // configurable in the future. + // This also prevent the case when compaction falls behind and we + // need to compact more levels for compactions to catch up. + if (mutable_cf_options_.compaction_options_universal.incremental) { + double fanout_threshold = static_cast(base_sr_size) / + static_cast(candidate_size) * 1.8; + Compaction* picked = PickIncrementalForReduceSizeAmp(fanout_threshold); + if (picked != nullptr) { + // As the feature is still incremental, picking incremental compaction + // might fail and we will fall bck to compacting full level. + return picked; + } + } + return PickCompactionWithSortedRunRange( + start_index, sr_end_idx, CompactionReason::kUniversalSizeAmplification); +} + +Compaction* UniversalCompactionBuilder::PickIncrementalForReduceSizeAmp( + double fanout_threshold) { + // Try find all potential compactions with total size just over + // options.max_compaction_size / 2, and take the one with the lowest + // fanout (defined in declaration of the function). + // This is done by having a sliding window of the files at the second + // lowest level, and keep expanding while finding overlapping in the + // last level. Once total size exceeds the size threshold, calculate + // the fanout value. And then shrinking from the small side of the + // window. Keep doing it until the end. + // Finally, we try to include upper level files if they fall into + // the range. + // + // Note that it is a similar problem as leveled compaction's + // kMinOverlappingRatio priority, but instead of picking single files + // we expand to a target compaction size. The reason is that in + // leveled compaction, actual fanout value tends to high, e.g. 10, so + // even with single file in down merging level, the extra size + // compacted in boundary files is at a lower ratio. But here users + // often have size of second last level size to be 1/4, 1/3 or even + // 1/2 of the bottommost level, so picking single file in second most + // level will cause significant waste, which is not desirable. + // + // This algorithm has lots of room to improve to pick more efficient + // compactions. + assert(sorted_runs_.size() >= 2); + int second_last_level = sorted_runs_[sorted_runs_.size() - 2].level; + if (second_last_level == 0) { + // Can't split Level 0. + return nullptr; + } + int output_level = sorted_runs_.back().level; + const std::vector& bottom_files = + vstorage_->LevelFiles(output_level); + const std::vector& files = + vstorage_->LevelFiles(second_last_level); + assert(!bottom_files.empty()); + assert(!files.empty()); + + // std::unordered_map file_to_order; + + int picked_start_idx = 0; + int picked_end_idx = 0; + double picked_fanout = fanout_threshold; + + // Use half target compaction bytes as anchor to stop growing second most + // level files, and reserve growing space for more overlapping bottom level, + // clean cut, files from other levels, etc. + uint64_t comp_thres_size = mutable_cf_options_.max_compaction_bytes / 2; + int start_idx = 0; + int bottom_end_idx = 0; + int bottom_start_idx = 0; + uint64_t non_bottom_size = 0; + uint64_t bottom_size = 0; + bool end_bottom_size_counted = false; + for (int end_idx = 0; end_idx < static_cast(files.size()); end_idx++) { + FileMetaData* end_file = files[end_idx]; + + // Include bottom most level files smaller than the current second + // last level file. + int num_skipped = 0; + while (bottom_end_idx < static_cast(bottom_files.size()) && + icmp_->Compare(bottom_files[bottom_end_idx]->largest, + end_file->smallest) < 0) { + if (!end_bottom_size_counted) { + bottom_size += bottom_files[bottom_end_idx]->fd.file_size; + } + bottom_end_idx++; + end_bottom_size_counted = false; + num_skipped++; + } + + if (num_skipped > 1) { + // At least a file in the bottom most level falls into the file gap. No + // reason to include the file. We cut the range and start a new sliding + // window. + start_idx = end_idx; + } + + if (start_idx == end_idx) { + // new sliding window. + non_bottom_size = 0; + bottom_size = 0; + bottom_start_idx = bottom_end_idx; + end_bottom_size_counted = false; + } + + non_bottom_size += end_file->fd.file_size; + + // Include all overlapping files in bottom level. + while (bottom_end_idx < static_cast(bottom_files.size()) && + icmp_->Compare(bottom_files[bottom_end_idx]->smallest, + end_file->largest) < 0) { + if (!end_bottom_size_counted) { + bottom_size += bottom_files[bottom_end_idx]->fd.file_size; + end_bottom_size_counted = true; + } + if (icmp_->Compare(bottom_files[bottom_end_idx]->largest, + end_file->largest) > 0) { + // next level file cross large boundary of current file. + break; + } + bottom_end_idx++; + end_bottom_size_counted = false; + } + + if ((non_bottom_size + bottom_size > comp_thres_size || + end_idx == static_cast(files.size()) - 1) && + non_bottom_size > 0) { // Do we alow 0 size file at all? + // If it is a better compaction, remember it in picked* variables. + double fanout = static_cast(bottom_size) / + static_cast(non_bottom_size); + if (fanout < picked_fanout) { + picked_start_idx = start_idx; + picked_end_idx = end_idx; + picked_fanout = fanout; + } + // Shrink from the start end to under comp_thres_size + while (non_bottom_size + bottom_size > comp_thres_size && + start_idx <= end_idx) { + non_bottom_size -= files[start_idx]->fd.file_size; + start_idx++; + if (start_idx < static_cast(files.size())) { + while (bottom_start_idx <= bottom_end_idx && + icmp_->Compare(bottom_files[bottom_start_idx]->largest, + files[start_idx]->smallest) < 0) { + bottom_size -= bottom_files[bottom_start_idx]->fd.file_size; + bottom_start_idx++; + } + } + } + } + } + + if (picked_fanout >= fanout_threshold) { + assert(picked_fanout == fanout_threshold); + return nullptr; + } + + std::vector inputs; + CompactionInputFiles bottom_level_inputs; + CompactionInputFiles second_last_level_inputs; + second_last_level_inputs.level = second_last_level; + bottom_level_inputs.level = output_level; + for (int i = picked_start_idx; i <= picked_end_idx; i++) { + if (files[i]->being_compacted) { + return nullptr; + } + second_last_level_inputs.files.push_back(files[i]); + } + assert(!second_last_level_inputs.empty()); + if (!picker_->ExpandInputsToCleanCut(cf_name_, vstorage_, + &second_last_level_inputs, + /*next_smallest=*/nullptr)) { + return nullptr; + } + // We might be able to avoid this binary search if we save and expand + // from bottom_start_idx and bottom_end_idx, but for now, we use + // SetupOtherInputs() for simplicity. + int parent_index = -1; // Create and use bottom_start_idx? + if (!picker_->SetupOtherInputs(cf_name_, mutable_cf_options_, vstorage_, + &second_last_level_inputs, + &bottom_level_inputs, &parent_index, + /*base_index=*/-1)) { + return nullptr; + } + + // Try to include files in upper levels if they fall into the range. + // Since we need to go from lower level up and this is in the reverse + // order, compared to level order, we first write to an reversed + // data structure and finally copy them to compaction inputs. + InternalKey smallest, largest; + picker_->GetRange(second_last_level_inputs, &smallest, &largest); + std::vector inputs_reverse; + for (auto it = ++(++sorted_runs_.rbegin()); it != sorted_runs_.rend(); it++) { + SortedRun& sr = *it; + if (sr.level == 0) { + break; + } + std::vector level_inputs; + vstorage_->GetCleanInputsWithinInterval(sr.level, &smallest, &largest, + &level_inputs); + if (!level_inputs.empty()) { + inputs_reverse.push_back({}); + inputs_reverse.back().level = sr.level; + inputs_reverse.back().files = level_inputs; + picker_->GetRange(inputs_reverse.back(), &smallest, &largest); + } + } + for (auto it = inputs_reverse.rbegin(); it != inputs_reverse.rend(); it++) { + inputs.push_back(*it); + } + + inputs.push_back(second_last_level_inputs); + inputs.push_back(bottom_level_inputs); + + int start_level = Compaction::kInvalidLevel; + for (const auto& in : inputs) { + if (!in.empty()) { + // inputs should already be sorted by level + start_level = in.level; + break; + } + } + + // intra L0 compactions outputs could have overlap + if (output_level != 0 && + picker_->FilesRangeOverlapWithCompaction( + inputs, output_level, + Compaction::EvaluatePenultimateLevel(vstorage_, ioptions_, + start_level, output_level))) { + return nullptr; + } + + // TODO support multi paths? + uint32_t path_id = 0; + return new Compaction( + vstorage_, ioptions_, mutable_cf_options_, mutable_db_options_, + std::move(inputs), output_level, + MaxFileSizeForLevel(mutable_cf_options_, output_level, + kCompactionStyleUniversal), + GetMaxOverlappingBytes(), path_id, + GetCompressionType(vstorage_, mutable_cf_options_, output_level, 1, + true /* enable_compression */), + GetCompressionOptions(mutable_cf_options_, vstorage_, output_level, + true /* enable_compression */), + Temperature::kUnknown, + /* max_subcompactions */ 0, /* grandparents */ {}, /* is manual */ false, + /* trim_ts */ "", score_, false /* deletion_compaction */, + /* l0_files_might_overlap */ true, + CompactionReason::kUniversalSizeAmplification); +} + +// Pick files marked for compaction. Typically, files are marked by +// CompactOnDeleteCollector due to the presence of tombstones. +Compaction* UniversalCompactionBuilder::PickDeleteTriggeredCompaction() { + CompactionInputFiles start_level_inputs; + int output_level; + std::vector inputs; + std::vector grandparents; + + if (vstorage_->num_levels() == 1) { + // This is single level universal. Since we're basically trying to reclaim + // space by processing files marked for compaction due to high tombstone + // density, let's do the same thing as compaction to reduce size amp which + // has the same goals. + int start_index = -1; + + start_level_inputs.level = 0; + start_level_inputs.files.clear(); + output_level = 0; + // Find the first file marked for compaction. Ignore the last file + for (size_t loop = 0; loop + 1 < sorted_runs_.size(); loop++) { + SortedRun* sr = &sorted_runs_[loop]; + if (sr->being_compacted) { + continue; + } + FileMetaData* f = vstorage_->LevelFiles(0)[loop]; + if (f->marked_for_compaction) { + start_level_inputs.files.push_back(f); + start_index = + static_cast(loop); // Consider this as the first candidate. + break; + } + } + if (start_index < 0) { + // Either no file marked, or they're already being compacted + return nullptr; + } + + for (size_t loop = start_index + 1; loop < sorted_runs_.size(); loop++) { + SortedRun* sr = &sorted_runs_[loop]; + if (sr->being_compacted) { + break; + } + + FileMetaData* f = vstorage_->LevelFiles(0)[loop]; + start_level_inputs.files.push_back(f); + } + if (start_level_inputs.size() <= 1) { + // If only the last file in L0 is marked for compaction, ignore it + return nullptr; + } + inputs.push_back(start_level_inputs); + } else { + int start_level; + + // For multi-level universal, the strategy is to make this look more like + // leveled. We pick one of the files marked for compaction and compact with + // overlapping files in the adjacent level. + picker_->PickFilesMarkedForCompaction(cf_name_, vstorage_, &start_level, + &output_level, &start_level_inputs); + if (start_level_inputs.empty()) { + return nullptr; + } + + int max_output_level = + vstorage_->MaxOutputLevel(ioptions_.allow_ingest_behind); + // Pick the first non-empty level after the start_level + for (output_level = start_level + 1; output_level <= max_output_level; + output_level++) { + if (vstorage_->NumLevelFiles(output_level) != 0) { + break; + } + } + + // If all higher levels are empty, pick the highest level as output level + if (output_level > max_output_level) { + if (start_level == 0) { + output_level = max_output_level; + } else { + // If start level is non-zero and all higher levels are empty, this + // compaction will translate into a trivial move. Since the idea is + // to reclaim space and trivial move doesn't help with that, we + // skip compaction in this case and return nullptr + return nullptr; + } + } + assert(output_level <= max_output_level); + + if (output_level != 0) { + if (start_level == 0) { + if (!picker_->GetOverlappingL0Files(vstorage_, &start_level_inputs, + output_level, nullptr)) { + return nullptr; + } + } + + CompactionInputFiles output_level_inputs; + int parent_index = -1; + + output_level_inputs.level = output_level; + if (!picker_->SetupOtherInputs(cf_name_, mutable_cf_options_, vstorage_, + &start_level_inputs, &output_level_inputs, + &parent_index, -1)) { + return nullptr; + } + inputs.push_back(start_level_inputs); + if (!output_level_inputs.empty()) { + inputs.push_back(output_level_inputs); + } + if (picker_->FilesRangeOverlapWithCompaction( + inputs, output_level, + Compaction::EvaluatePenultimateLevel( + vstorage_, ioptions_, start_level, output_level))) { + return nullptr; + } + + picker_->GetGrandparents(vstorage_, start_level_inputs, + output_level_inputs, &grandparents); + } else { + inputs.push_back(start_level_inputs); + } + } + + uint64_t estimated_total_size = 0; + // Use size of the output level as estimated file size + for (FileMetaData* f : vstorage_->LevelFiles(output_level)) { + estimated_total_size += f->fd.GetFileSize(); + } + uint32_t path_id = + GetPathId(ioptions_, mutable_cf_options_, estimated_total_size); + return new Compaction( + vstorage_, ioptions_, mutable_cf_options_, mutable_db_options_, + std::move(inputs), output_level, + MaxFileSizeForLevel(mutable_cf_options_, output_level, + kCompactionStyleUniversal), + /* max_grandparent_overlap_bytes */ GetMaxOverlappingBytes(), path_id, + GetCompressionType(vstorage_, mutable_cf_options_, output_level, 1), + GetCompressionOptions(mutable_cf_options_, vstorage_, output_level), + Temperature::kUnknown, + /* max_subcompactions */ 0, grandparents, /* is manual */ false, + /* trim_ts */ "", score_, false /* deletion_compaction */, + /* l0_files_might_overlap */ true, + CompactionReason::kFilesMarkedForCompaction); +} + +Compaction* UniversalCompactionBuilder::PickCompactionToOldest( + size_t start_index, CompactionReason compaction_reason) { + return PickCompactionWithSortedRunRange(start_index, sorted_runs_.size() - 1, + compaction_reason); +} + +Compaction* UniversalCompactionBuilder::PickCompactionWithSortedRunRange( + size_t start_index, size_t end_index, CompactionReason compaction_reason) { + assert(start_index < sorted_runs_.size()); + + // Estimate total file size + uint64_t estimated_total_size = 0; + for (size_t loop = start_index; loop <= end_index; loop++) { + estimated_total_size += sorted_runs_[loop].size; + } + uint32_t path_id = + GetPathId(ioptions_, mutable_cf_options_, estimated_total_size); + int start_level = sorted_runs_[start_index].level; + int max_output_level = + vstorage_->MaxOutputLevel(ioptions_.allow_ingest_behind); + std::vector inputs(max_output_level + 1); + for (size_t i = 0; i < inputs.size(); ++i) { + inputs[i].level = start_level + static_cast(i); + } + for (size_t loop = start_index; loop <= end_index; loop++) { + auto& picking_sr = sorted_runs_[loop]; + if (picking_sr.level == 0) { + FileMetaData* f = picking_sr.file; + inputs[0].files.push_back(f); + } else { + auto& files = inputs[picking_sr.level - start_level].files; + for (auto* f : vstorage_->LevelFiles(picking_sr.level)) { + files.push_back(f); + } + } + std::string comp_reason_print_string; + if (compaction_reason == CompactionReason::kPeriodicCompaction) { + comp_reason_print_string = "periodic compaction"; + } else if (compaction_reason == + CompactionReason::kUniversalSizeAmplification) { + comp_reason_print_string = "size amp"; + } else { + assert(false); + comp_reason_print_string = "unknown: "; + comp_reason_print_string.append( + std::to_string(static_cast(compaction_reason))); + } + + char file_num_buf[256]; + picking_sr.DumpSizeInfo(file_num_buf, sizeof(file_num_buf), loop); + ROCKS_LOG_BUFFER(log_buffer_, "[%s] Universal: %s picking %s", + cf_name_.c_str(), comp_reason_print_string.c_str(), + file_num_buf); + } + + int output_level; + if (end_index == sorted_runs_.size() - 1) { + output_level = max_output_level; + } else { + // if it's not including all sorted_runs, it can only output to the level + // above the `end_index + 1` sorted_run. + output_level = sorted_runs_[end_index + 1].level - 1; + } + + // intra L0 compactions outputs could have overlap + if (output_level != 0 && + picker_->FilesRangeOverlapWithCompaction( + inputs, output_level, + Compaction::EvaluatePenultimateLevel(vstorage_, ioptions_, + start_level, output_level))) { + return nullptr; + } + + // We never check size for + // compaction_options_universal.compression_size_percent, + // because we always compact all the files, so always compress. + return new Compaction( + vstorage_, ioptions_, mutable_cf_options_, mutable_db_options_, + std::move(inputs), output_level, + MaxFileSizeForLevel(mutable_cf_options_, output_level, + kCompactionStyleUniversal), + GetMaxOverlappingBytes(), path_id, + GetCompressionType(vstorage_, mutable_cf_options_, output_level, 1, + true /* enable_compression */), + GetCompressionOptions(mutable_cf_options_, vstorage_, output_level, + true /* enable_compression */), + Temperature::kUnknown, + /* max_subcompactions */ 0, /* grandparents */ {}, /* is manual */ false, + /* trim_ts */ "", score_, false /* deletion_compaction */, + /* l0_files_might_overlap */ true, compaction_reason); +} + +Compaction* UniversalCompactionBuilder::PickPeriodicCompaction() { + ROCKS_LOG_BUFFER(log_buffer_, "[%s] Universal: Periodic Compaction", + cf_name_.c_str()); + + // In universal compaction, sorted runs contain older data are almost always + // generated earlier too. To simplify the problem, we just try to trigger + // a full compaction. We start from the oldest sorted run and include + // all sorted runs, until we hit a sorted already being compacted. + // Since usually the largest (which is usually the oldest) sorted run is + // included anyway, doing a full compaction won't increase write + // amplification much. + + // Get some information from marked files to check whether a file is + // included in the compaction. + + size_t start_index = sorted_runs_.size(); + while (start_index > 0 && !sorted_runs_[start_index - 1].being_compacted) { + start_index--; + } + if (start_index == sorted_runs_.size()) { + return nullptr; + } + + // There is a rare corner case where we can't pick up all the files + // because some files are being compacted and we end up with picking files + // but none of them need periodic compaction. Unless we simply recompact + // the last sorted run (either the last level or last L0 file), we would just + // execute the compaction, in order to simplify the logic. + if (start_index == sorted_runs_.size() - 1) { + bool included_file_marked = false; + int start_level = sorted_runs_[start_index].level; + FileMetaData* start_file = sorted_runs_[start_index].file; + for (const std::pair& level_file_pair : + vstorage_->FilesMarkedForPeriodicCompaction()) { + if (start_level != 0) { + // Last sorted run is a level + if (start_level == level_file_pair.first) { + included_file_marked = true; + break; + } + } else { + // Last sorted run is a L0 file. + if (start_file == level_file_pair.second) { + included_file_marked = true; + break; + } + } + } + if (!included_file_marked) { + ROCKS_LOG_BUFFER(log_buffer_, + "[%s] Universal: Cannot form a compaction covering file " + "marked for periodic compaction", + cf_name_.c_str()); + return nullptr; + } + } + + Compaction* c = PickCompactionToOldest(start_index, + CompactionReason::kPeriodicCompaction); + + TEST_SYNC_POINT_CALLBACK( + "UniversalCompactionPicker::PickPeriodicCompaction:Return", c); + + return c; +} + +uint64_t UniversalCompactionBuilder::GetMaxOverlappingBytes() const { + if (!mutable_cf_options_.compaction_options_universal.incremental) { + return std::numeric_limits::max(); + } else { + // Try to align cutting boundary with files at the next level if the + // file isn't end up with 1/2 of target size, or it would overlap + // with two full size files at the next level. + return mutable_cf_options_.target_file_size_base / 2 * 3; + } +} +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_picker_universal.h b/librocksdb-sys/rocksdb/db/compaction/compaction_picker_universal.h new file mode 100644 index 0000000..cb16059 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_picker_universal.h @@ -0,0 +1,30 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include "db/compaction/compaction_picker.h" + +namespace ROCKSDB_NAMESPACE { +class UniversalCompactionPicker : public CompactionPicker { + public: + UniversalCompactionPicker(const ImmutableOptions& ioptions, + const InternalKeyComparator* icmp) + : CompactionPicker(ioptions, icmp) {} + virtual Compaction* PickCompaction(const std::string& cf_name, + const MutableCFOptions& mutable_cf_options, + const MutableDBOptions& mutable_db_options, + VersionStorageInfo* vstorage, + LogBuffer* log_buffer) override; + virtual int MaxOutputLevel() const override { return NumberLevels() - 1; } + + virtual bool NeedsCompaction( + const VersionStorageInfo* vstorage) const override; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_service_job.cc b/librocksdb-sys/rocksdb/db/compaction/compaction_service_job.cc new file mode 100644 index 0000000..3149bb5 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_service_job.cc @@ -0,0 +1,833 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/compaction/compaction_job.h" +#include "db/compaction/compaction_state.h" +#include "logging/logging.h" +#include "monitoring/iostats_context_imp.h" +#include "monitoring/thread_status_util.h" +#include "options/options_helper.h" +#include "rocksdb/utilities/options_type.h" + +namespace ROCKSDB_NAMESPACE { +class SubcompactionState; + +CompactionServiceJobStatus +CompactionJob::ProcessKeyValueCompactionWithCompactionService( + SubcompactionState* sub_compact) { + assert(sub_compact); + assert(sub_compact->compaction); + assert(db_options_.compaction_service); + + const Compaction* compaction = sub_compact->compaction; + CompactionServiceInput compaction_input; + compaction_input.output_level = compaction->output_level(); + compaction_input.db_id = db_id_; + + const std::vector& inputs = + *(compact_->compaction->inputs()); + for (const auto& files_per_level : inputs) { + for (const auto& file : files_per_level.files) { + compaction_input.input_files.emplace_back( + MakeTableFileName(file->fd.GetNumber())); + } + } + compaction_input.column_family.name = + compaction->column_family_data()->GetName(); + compaction_input.column_family.options = + compaction->column_family_data()->GetLatestCFOptions(); + compaction_input.db_options = + BuildDBOptions(db_options_, mutable_db_options_copy_); + compaction_input.snapshots = existing_snapshots_; + compaction_input.has_begin = sub_compact->start.has_value(); + compaction_input.begin = + compaction_input.has_begin ? sub_compact->start->ToString() : ""; + compaction_input.has_end = sub_compact->end.has_value(); + compaction_input.end = + compaction_input.has_end ? sub_compact->end->ToString() : ""; + + std::string compaction_input_binary; + Status s = compaction_input.Write(&compaction_input_binary); + if (!s.ok()) { + sub_compact->status = s; + return CompactionServiceJobStatus::kFailure; + } + + std::ostringstream input_files_oss; + bool is_first_one = true; + for (const auto& file : compaction_input.input_files) { + input_files_oss << (is_first_one ? "" : ", ") << file; + is_first_one = false; + } + + ROCKS_LOG_INFO( + db_options_.info_log, + "[%s] [JOB %d] Starting remote compaction (output level: %d): %s", + compaction_input.column_family.name.c_str(), job_id_, + compaction_input.output_level, input_files_oss.str().c_str()); + CompactionServiceJobInfo info(dbname_, db_id_, db_session_id_, + GetCompactionId(sub_compact), thread_pri_); + CompactionServiceJobStatus compaction_status = + db_options_.compaction_service->StartV2(info, compaction_input_binary); + switch (compaction_status) { + case CompactionServiceJobStatus::kSuccess: + break; + case CompactionServiceJobStatus::kFailure: + sub_compact->status = Status::Incomplete( + "CompactionService failed to start compaction job."); + ROCKS_LOG_WARN(db_options_.info_log, + "[%s] [JOB %d] Remote compaction failed to start.", + compaction_input.column_family.name.c_str(), job_id_); + return compaction_status; + case CompactionServiceJobStatus::kUseLocal: + ROCKS_LOG_INFO( + db_options_.info_log, + "[%s] [JOB %d] Remote compaction fallback to local by API Start.", + compaction_input.column_family.name.c_str(), job_id_); + return compaction_status; + default: + assert(false); // unknown status + break; + } + + ROCKS_LOG_INFO(db_options_.info_log, + "[%s] [JOB %d] Waiting for remote compaction...", + compaction_input.column_family.name.c_str(), job_id_); + std::string compaction_result_binary; + compaction_status = db_options_.compaction_service->WaitForCompleteV2( + info, &compaction_result_binary); + + if (compaction_status == CompactionServiceJobStatus::kUseLocal) { + ROCKS_LOG_INFO(db_options_.info_log, + "[%s] [JOB %d] Remote compaction fallback to local by API " + "WaitForComplete.", + compaction_input.column_family.name.c_str(), job_id_); + return compaction_status; + } + + CompactionServiceResult compaction_result; + s = CompactionServiceResult::Read(compaction_result_binary, + &compaction_result); + + if (compaction_status == CompactionServiceJobStatus::kFailure) { + if (s.ok()) { + if (compaction_result.status.ok()) { + sub_compact->status = Status::Incomplete( + "CompactionService failed to run the compaction job (even though " + "the internal status is okay)."); + } else { + // set the current sub compaction status with the status returned from + // remote + sub_compact->status = compaction_result.status; + } + } else { + sub_compact->status = Status::Incomplete( + "CompactionService failed to run the compaction job (and no valid " + "result is returned)."); + compaction_result.status.PermitUncheckedError(); + } + ROCKS_LOG_WARN(db_options_.info_log, + "[%s] [JOB %d] Remote compaction failed.", + compaction_input.column_family.name.c_str(), job_id_); + return compaction_status; + } + + if (!s.ok()) { + sub_compact->status = s; + compaction_result.status.PermitUncheckedError(); + return CompactionServiceJobStatus::kFailure; + } + sub_compact->status = compaction_result.status; + + std::ostringstream output_files_oss; + is_first_one = true; + for (const auto& file : compaction_result.output_files) { + output_files_oss << (is_first_one ? "" : ", ") << file.file_name; + is_first_one = false; + } + + ROCKS_LOG_INFO(db_options_.info_log, + "[%s] [JOB %d] Receive remote compaction result, output path: " + "%s, files: %s", + compaction_input.column_family.name.c_str(), job_id_, + compaction_result.output_path.c_str(), + output_files_oss.str().c_str()); + + if (!s.ok()) { + sub_compact->status = s; + return CompactionServiceJobStatus::kFailure; + } + + for (const auto& file : compaction_result.output_files) { + uint64_t file_num = versions_->NewFileNumber(); + auto src_file = compaction_result.output_path + "/" + file.file_name; + auto tgt_file = TableFileName(compaction->immutable_options()->cf_paths, + file_num, compaction->output_path_id()); + s = fs_->RenameFile(src_file, tgt_file, IOOptions(), nullptr); + if (!s.ok()) { + sub_compact->status = s; + return CompactionServiceJobStatus::kFailure; + } + + FileMetaData meta; + uint64_t file_size; + s = fs_->GetFileSize(tgt_file, IOOptions(), &file_size, nullptr); + if (!s.ok()) { + sub_compact->status = s; + return CompactionServiceJobStatus::kFailure; + } + meta.fd = FileDescriptor(file_num, compaction->output_path_id(), file_size, + file.smallest_seqno, file.largest_seqno); + meta.smallest.DecodeFrom(file.smallest_internal_key); + meta.largest.DecodeFrom(file.largest_internal_key); + meta.oldest_ancester_time = file.oldest_ancester_time; + meta.file_creation_time = file.file_creation_time; + meta.epoch_number = file.epoch_number; + meta.marked_for_compaction = file.marked_for_compaction; + meta.unique_id = file.unique_id; + + auto cfd = compaction->column_family_data(); + sub_compact->Current().AddOutput(std::move(meta), + cfd->internal_comparator(), false, false, + true, file.paranoid_hash); + } + sub_compact->compaction_job_stats = compaction_result.stats; + sub_compact->Current().SetNumOutputRecords( + compaction_result.num_output_records); + sub_compact->Current().SetTotalBytes(compaction_result.total_bytes); + RecordTick(stats_, REMOTE_COMPACT_READ_BYTES, compaction_result.bytes_read); + RecordTick(stats_, REMOTE_COMPACT_WRITE_BYTES, + compaction_result.bytes_written); + return CompactionServiceJobStatus::kSuccess; +} + +std::string CompactionServiceCompactionJob::GetTableFileName( + uint64_t file_number) { + return MakeTableFileName(output_path_, file_number); +} + +void CompactionServiceCompactionJob::RecordCompactionIOStats() { + compaction_result_->bytes_read += IOSTATS(bytes_read); + compaction_result_->bytes_written += IOSTATS(bytes_written); + CompactionJob::RecordCompactionIOStats(); +} + +CompactionServiceCompactionJob::CompactionServiceCompactionJob( + int job_id, Compaction* compaction, const ImmutableDBOptions& db_options, + const MutableDBOptions& mutable_db_options, const FileOptions& file_options, + VersionSet* versions, const std::atomic* shutting_down, + LogBuffer* log_buffer, FSDirectory* output_directory, Statistics* stats, + InstrumentedMutex* db_mutex, ErrorHandler* db_error_handler, + std::vector existing_snapshots, + std::shared_ptr table_cache, EventLogger* event_logger, + const std::string& dbname, const std::shared_ptr& io_tracer, + const std::atomic& manual_compaction_canceled, + const std::string& db_id, const std::string& db_session_id, + std::string output_path, + const CompactionServiceInput& compaction_service_input, + CompactionServiceResult* compaction_service_result) + : CompactionJob( + job_id, compaction, db_options, mutable_db_options, file_options, + versions, shutting_down, log_buffer, nullptr, output_directory, + nullptr, stats, db_mutex, db_error_handler, + std::move(existing_snapshots), kMaxSequenceNumber, nullptr, nullptr, + std::move(table_cache), event_logger, + compaction->mutable_cf_options()->paranoid_file_checks, + compaction->mutable_cf_options()->report_bg_io_stats, dbname, + &(compaction_service_result->stats), Env::Priority::USER, io_tracer, + manual_compaction_canceled, db_id, db_session_id, + compaction->column_family_data()->GetFullHistoryTsLow()), + output_path_(std::move(output_path)), + compaction_input_(compaction_service_input), + compaction_result_(compaction_service_result) {} + +Status CompactionServiceCompactionJob::Run() { + AutoThreadOperationStageUpdater stage_updater( + ThreadStatus::STAGE_COMPACTION_RUN); + + auto* c = compact_->compaction; + assert(c->column_family_data() != nullptr); + assert(c->column_family_data()->current()->storage_info()->NumLevelFiles( + compact_->compaction->level()) > 0); + + write_hint_ = + c->column_family_data()->CalculateSSTWriteHint(c->output_level()); + bottommost_level_ = c->bottommost_level(); + + Slice begin = compaction_input_.begin; + Slice end = compaction_input_.end; + compact_->sub_compact_states.emplace_back( + c, + compaction_input_.has_begin ? std::optional(begin) + : std::optional(), + compaction_input_.has_end ? std::optional(end) + : std::optional(), + /*sub_job_id*/ 0); + + log_buffer_->FlushBufferToLog(); + LogCompaction(); + const uint64_t start_micros = db_options_.clock->NowMicros(); + // Pick the only sub-compaction we should have + assert(compact_->sub_compact_states.size() == 1); + SubcompactionState* sub_compact = compact_->sub_compact_states.data(); + + ProcessKeyValueCompaction(sub_compact); + + compaction_stats_.stats.micros = + db_options_.clock->NowMicros() - start_micros; + compaction_stats_.stats.cpu_micros = + sub_compact->compaction_job_stats.cpu_micros; + + RecordTimeToHistogram(stats_, COMPACTION_TIME, + compaction_stats_.stats.micros); + RecordTimeToHistogram(stats_, COMPACTION_CPU_TIME, + compaction_stats_.stats.cpu_micros); + + Status status = sub_compact->status; + IOStatus io_s = sub_compact->io_status; + + if (io_status_.ok()) { + io_status_ = io_s; + } + + if (status.ok()) { + constexpr IODebugContext* dbg = nullptr; + + if (output_directory_) { + io_s = output_directory_->FsyncWithDirOptions(IOOptions(), dbg, + DirFsyncOptions()); + } + } + if (io_status_.ok()) { + io_status_ = io_s; + } + if (status.ok()) { + status = io_s; + } + if (status.ok()) { + // TODO: Add verify_table() + } + + // Finish up all book-keeping to unify the subcompaction results + compact_->AggregateCompactionStats(compaction_stats_, *compaction_job_stats_); + UpdateCompactionStats(); + RecordCompactionIOStats(); + + LogFlush(db_options_.info_log); + compact_->status = status; + compact_->status.PermitUncheckedError(); + + // Build compaction result + compaction_result_->output_level = compact_->compaction->output_level(); + compaction_result_->output_path = output_path_; + for (const auto& output_file : sub_compact->GetOutputs()) { + auto& meta = output_file.meta; + compaction_result_->output_files.emplace_back( + MakeTableFileName(meta.fd.GetNumber()), meta.fd.smallest_seqno, + meta.fd.largest_seqno, meta.smallest.Encode().ToString(), + meta.largest.Encode().ToString(), meta.oldest_ancester_time, + meta.file_creation_time, meta.epoch_number, + output_file.validator.GetHash(), meta.marked_for_compaction, + meta.unique_id); + } + InternalStats::CompactionStatsFull compaction_stats; + sub_compact->AggregateCompactionStats(compaction_stats); + compaction_result_->num_output_records = + compaction_stats.stats.num_output_records; + compaction_result_->total_bytes = compaction_stats.TotalBytesWritten(); + + return status; +} + +void CompactionServiceCompactionJob::CleanupCompaction() { + CompactionJob::CleanupCompaction(); +} + +// Internal binary format for the input and result data +enum BinaryFormatVersion : uint32_t { + kOptionsString = 1, // Use string format similar to Option string format +}; + +static std::unordered_map cfd_type_info = { + {"name", + {offsetof(struct ColumnFamilyDescriptor, name), OptionType::kEncodedString, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"options", + {offsetof(struct ColumnFamilyDescriptor, options), + OptionType::kConfigurable, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, + [](const ConfigOptions& opts, const std::string& /*name*/, + const std::string& value, void* addr) { + auto cf_options = static_cast(addr); + return GetColumnFamilyOptionsFromString(opts, ColumnFamilyOptions(), + value, cf_options); + }, + [](const ConfigOptions& opts, const std::string& /*name*/, + const void* addr, std::string* value) { + const auto cf_options = static_cast(addr); + std::string result; + auto status = + GetStringFromColumnFamilyOptions(opts, *cf_options, &result); + *value = "{" + result + "}"; + return status; + }, + [](const ConfigOptions& opts, const std::string& name, const void* addr1, + const void* addr2, std::string* mismatch) { + const auto this_one = static_cast(addr1); + const auto that_one = static_cast(addr2); + auto this_conf = CFOptionsAsConfigurable(*this_one); + auto that_conf = CFOptionsAsConfigurable(*that_one); + std::string mismatch_opt; + bool result = + this_conf->AreEquivalent(opts, that_conf.get(), &mismatch_opt); + if (!result) { + *mismatch = name + "." + mismatch_opt; + } + return result; + }}}, +}; + +static std::unordered_map cs_input_type_info = { + {"column_family", + OptionTypeInfo::Struct( + "column_family", &cfd_type_info, + offsetof(struct CompactionServiceInput, column_family), + OptionVerificationType::kNormal, OptionTypeFlags::kNone)}, + {"db_options", + {offsetof(struct CompactionServiceInput, db_options), + OptionType::kConfigurable, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, + [](const ConfigOptions& opts, const std::string& /*name*/, + const std::string& value, void* addr) { + auto options = static_cast(addr); + return GetDBOptionsFromString(opts, DBOptions(), value, options); + }, + [](const ConfigOptions& opts, const std::string& /*name*/, + const void* addr, std::string* value) { + const auto options = static_cast(addr); + std::string result; + auto status = GetStringFromDBOptions(opts, *options, &result); + *value = "{" + result + "}"; + return status; + }, + [](const ConfigOptions& opts, const std::string& name, const void* addr1, + const void* addr2, std::string* mismatch) { + const auto this_one = static_cast(addr1); + const auto that_one = static_cast(addr2); + auto this_conf = DBOptionsAsConfigurable(*this_one); + auto that_conf = DBOptionsAsConfigurable(*that_one); + std::string mismatch_opt; + bool result = + this_conf->AreEquivalent(opts, that_conf.get(), &mismatch_opt); + if (!result) { + *mismatch = name + "." + mismatch_opt; + } + return result; + }}}, + {"snapshots", OptionTypeInfo::Vector( + offsetof(struct CompactionServiceInput, snapshots), + OptionVerificationType::kNormal, OptionTypeFlags::kNone, + {0, OptionType::kUInt64T})}, + {"input_files", OptionTypeInfo::Vector( + offsetof(struct CompactionServiceInput, input_files), + OptionVerificationType::kNormal, OptionTypeFlags::kNone, + {0, OptionType::kEncodedString})}, + {"output_level", + {offsetof(struct CompactionServiceInput, output_level), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"db_id", + {offsetof(struct CompactionServiceInput, db_id), + OptionType::kEncodedString}}, + {"has_begin", + {offsetof(struct CompactionServiceInput, has_begin), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"begin", + {offsetof(struct CompactionServiceInput, begin), + OptionType::kEncodedString, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"has_end", + {offsetof(struct CompactionServiceInput, has_end), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"end", + {offsetof(struct CompactionServiceInput, end), OptionType::kEncodedString, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, +}; + +static std::unordered_map + cs_output_file_type_info = { + {"file_name", + {offsetof(struct CompactionServiceOutputFile, file_name), + OptionType::kEncodedString, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"smallest_seqno", + {offsetof(struct CompactionServiceOutputFile, smallest_seqno), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"largest_seqno", + {offsetof(struct CompactionServiceOutputFile, largest_seqno), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"smallest_internal_key", + {offsetof(struct CompactionServiceOutputFile, smallest_internal_key), + OptionType::kEncodedString, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"largest_internal_key", + {offsetof(struct CompactionServiceOutputFile, largest_internal_key), + OptionType::kEncodedString, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"oldest_ancester_time", + {offsetof(struct CompactionServiceOutputFile, oldest_ancester_time), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"file_creation_time", + {offsetof(struct CompactionServiceOutputFile, file_creation_time), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"epoch_number", + {offsetof(struct CompactionServiceOutputFile, epoch_number), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"paranoid_hash", + {offsetof(struct CompactionServiceOutputFile, paranoid_hash), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"marked_for_compaction", + {offsetof(struct CompactionServiceOutputFile, marked_for_compaction), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"unique_id", + OptionTypeInfo::Array( + offsetof(struct CompactionServiceOutputFile, unique_id), + OptionVerificationType::kNormal, OptionTypeFlags::kNone, + {0, OptionType::kUInt64T})}, +}; + +static std::unordered_map + compaction_job_stats_type_info = { + {"elapsed_micros", + {offsetof(struct CompactionJobStats, elapsed_micros), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"cpu_micros", + {offsetof(struct CompactionJobStats, cpu_micros), OptionType::kUInt64T, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"num_input_records", + {offsetof(struct CompactionJobStats, num_input_records), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"num_blobs_read", + {offsetof(struct CompactionJobStats, num_blobs_read), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"num_input_files", + {offsetof(struct CompactionJobStats, num_input_files), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"num_input_files_at_output_level", + {offsetof(struct CompactionJobStats, num_input_files_at_output_level), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"num_output_records", + {offsetof(struct CompactionJobStats, num_output_records), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"num_output_files", + {offsetof(struct CompactionJobStats, num_output_files), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"num_output_files_blob", + {offsetof(struct CompactionJobStats, num_output_files_blob), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"is_full_compaction", + {offsetof(struct CompactionJobStats, is_full_compaction), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"is_manual_compaction", + {offsetof(struct CompactionJobStats, is_manual_compaction), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"total_input_bytes", + {offsetof(struct CompactionJobStats, total_input_bytes), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"total_blob_bytes_read", + {offsetof(struct CompactionJobStats, total_blob_bytes_read), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"total_output_bytes", + {offsetof(struct CompactionJobStats, total_output_bytes), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"total_output_bytes_blob", + {offsetof(struct CompactionJobStats, total_output_bytes_blob), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"num_records_replaced", + {offsetof(struct CompactionJobStats, num_records_replaced), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"total_input_raw_key_bytes", + {offsetof(struct CompactionJobStats, total_input_raw_key_bytes), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"total_input_raw_value_bytes", + {offsetof(struct CompactionJobStats, total_input_raw_value_bytes), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"num_input_deletion_records", + {offsetof(struct CompactionJobStats, num_input_deletion_records), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"num_expired_deletion_records", + {offsetof(struct CompactionJobStats, num_expired_deletion_records), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"num_corrupt_keys", + {offsetof(struct CompactionJobStats, num_corrupt_keys), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"file_write_nanos", + {offsetof(struct CompactionJobStats, file_write_nanos), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"file_range_sync_nanos", + {offsetof(struct CompactionJobStats, file_range_sync_nanos), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"file_fsync_nanos", + {offsetof(struct CompactionJobStats, file_fsync_nanos), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"file_prepare_write_nanos", + {offsetof(struct CompactionJobStats, file_prepare_write_nanos), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"smallest_output_key_prefix", + {offsetof(struct CompactionJobStats, smallest_output_key_prefix), + OptionType::kEncodedString, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"largest_output_key_prefix", + {offsetof(struct CompactionJobStats, largest_output_key_prefix), + OptionType::kEncodedString, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"num_single_del_fallthru", + {offsetof(struct CompactionJobStats, num_single_del_fallthru), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"num_single_del_mismatch", + {offsetof(struct CompactionJobStats, num_single_del_mismatch), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, +}; + +namespace { +// this is a helper struct to serialize and deserialize class Status, because +// Status's members are not public. +struct StatusSerializationAdapter { + uint8_t code; + uint8_t subcode; + uint8_t severity; + std::string message; + + StatusSerializationAdapter() = default; + explicit StatusSerializationAdapter(const Status& s) { + code = s.code(); + subcode = s.subcode(); + severity = s.severity(); + auto msg = s.getState(); + message = msg ? msg : ""; + } + + Status GetStatus() const { + return Status{static_cast(code), + static_cast(subcode), + static_cast(severity), message}; + } +}; +} // namespace + +static std::unordered_map + status_adapter_type_info = { + {"code", + {offsetof(struct StatusSerializationAdapter, code), + OptionType::kUInt8T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"subcode", + {offsetof(struct StatusSerializationAdapter, subcode), + OptionType::kUInt8T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"severity", + {offsetof(struct StatusSerializationAdapter, severity), + OptionType::kUInt8T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"message", + {offsetof(struct StatusSerializationAdapter, message), + OptionType::kEncodedString, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, +}; + +static std::unordered_map cs_result_type_info = { + {"status", + {offsetof(struct CompactionServiceResult, status), + OptionType::kCustomizable, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, + [](const ConfigOptions& opts, const std::string& /*name*/, + const std::string& value, void* addr) { + auto status_obj = static_cast(addr); + StatusSerializationAdapter adapter; + Status s = OptionTypeInfo::ParseType( + opts, value, status_adapter_type_info, &adapter); + *status_obj = adapter.GetStatus(); + return s; + }, + [](const ConfigOptions& opts, const std::string& /*name*/, + const void* addr, std::string* value) { + const auto status_obj = static_cast(addr); + StatusSerializationAdapter adapter(*status_obj); + std::string result; + Status s = OptionTypeInfo::SerializeType(opts, status_adapter_type_info, + &adapter, &result); + *value = "{" + result + "}"; + return s; + }, + [](const ConfigOptions& opts, const std::string& /*name*/, + const void* addr1, const void* addr2, std::string* mismatch) { + const auto status1 = static_cast(addr1); + const auto status2 = static_cast(addr2); + + StatusSerializationAdapter adatper1(*status1); + StatusSerializationAdapter adapter2(*status2); + return OptionTypeInfo::TypesAreEqual(opts, status_adapter_type_info, + &adatper1, &adapter2, mismatch); + }}}, + {"output_files", + OptionTypeInfo::Vector( + offsetof(struct CompactionServiceResult, output_files), + OptionVerificationType::kNormal, OptionTypeFlags::kNone, + OptionTypeInfo::Struct("output_files", &cs_output_file_type_info, 0, + OptionVerificationType::kNormal, + OptionTypeFlags::kNone))}, + {"output_level", + {offsetof(struct CompactionServiceResult, output_level), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"output_path", + {offsetof(struct CompactionServiceResult, output_path), + OptionType::kEncodedString, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"num_output_records", + {offsetof(struct CompactionServiceResult, num_output_records), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"total_bytes", + {offsetof(struct CompactionServiceResult, total_bytes), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"bytes_read", + {offsetof(struct CompactionServiceResult, bytes_read), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"bytes_written", + {offsetof(struct CompactionServiceResult, bytes_written), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"stats", OptionTypeInfo::Struct( + "stats", &compaction_job_stats_type_info, + offsetof(struct CompactionServiceResult, stats), + OptionVerificationType::kNormal, OptionTypeFlags::kNone)}, +}; + +Status CompactionServiceInput::Read(const std::string& data_str, + CompactionServiceInput* obj) { + if (data_str.size() <= sizeof(BinaryFormatVersion)) { + return Status::InvalidArgument("Invalid CompactionServiceInput string"); + } + auto format_version = DecodeFixed32(data_str.data()); + if (format_version == kOptionsString) { + ConfigOptions cf; + cf.invoke_prepare_options = false; + cf.ignore_unknown_options = true; + return OptionTypeInfo::ParseType( + cf, data_str.substr(sizeof(BinaryFormatVersion)), cs_input_type_info, + obj); + } else { + return Status::NotSupported( + "Compaction Service Input data version not supported: " + + std::to_string(format_version)); + } +} + +Status CompactionServiceInput::Write(std::string* output) { + char buf[sizeof(BinaryFormatVersion)]; + EncodeFixed32(buf, kOptionsString); + output->append(buf, sizeof(BinaryFormatVersion)); + ConfigOptions cf; + cf.invoke_prepare_options = false; + return OptionTypeInfo::SerializeType(cf, cs_input_type_info, this, output); +} + +Status CompactionServiceResult::Read(const std::string& data_str, + CompactionServiceResult* obj) { + if (data_str.size() <= sizeof(BinaryFormatVersion)) { + return Status::InvalidArgument("Invalid CompactionServiceResult string"); + } + auto format_version = DecodeFixed32(data_str.data()); + if (format_version == kOptionsString) { + ConfigOptions cf; + cf.invoke_prepare_options = false; + cf.ignore_unknown_options = true; + return OptionTypeInfo::ParseType( + cf, data_str.substr(sizeof(BinaryFormatVersion)), cs_result_type_info, + obj); + } else { + return Status::NotSupported( + "Compaction Service Result data version not supported: " + + std::to_string(format_version)); + } +} + +Status CompactionServiceResult::Write(std::string* output) { + char buf[sizeof(BinaryFormatVersion)]; + EncodeFixed32(buf, kOptionsString); + output->append(buf, sizeof(BinaryFormatVersion)); + ConfigOptions cf; + cf.invoke_prepare_options = false; + return OptionTypeInfo::SerializeType(cf, cs_result_type_info, this, output); +} + +#ifndef NDEBUG +bool CompactionServiceResult::TEST_Equals(CompactionServiceResult* other) { + std::string mismatch; + return TEST_Equals(other, &mismatch); +} + +bool CompactionServiceResult::TEST_Equals(CompactionServiceResult* other, + std::string* mismatch) { + ConfigOptions cf; + cf.invoke_prepare_options = false; + return OptionTypeInfo::TypesAreEqual(cf, cs_result_type_info, this, other, + mismatch); +} + +bool CompactionServiceInput::TEST_Equals(CompactionServiceInput* other) { + std::string mismatch; + return TEST_Equals(other, &mismatch); +} + +bool CompactionServiceInput::TEST_Equals(CompactionServiceInput* other, + std::string* mismatch) { + ConfigOptions cf; + cf.invoke_prepare_options = false; + return OptionTypeInfo::TypesAreEqual(cf, cs_input_type_info, this, other, + mismatch); +} +#endif // NDEBUG +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_service_test.cc b/librocksdb-sys/rocksdb/db/compaction/compaction_service_test.cc new file mode 100644 index 0000000..7c87f88 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_service_test.cc @@ -0,0 +1,954 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + + +#include "db/db_test_util.h" +#include "port/stack_trace.h" +#include "table/unique_id_impl.h" + +namespace ROCKSDB_NAMESPACE { + +class MyTestCompactionService : public CompactionService { + public: + MyTestCompactionService( + std::string db_path, Options& options, + std::shared_ptr& statistics, + std::vector>& listeners, + std::vector> + table_properties_collector_factories) + : db_path_(std::move(db_path)), + options_(options), + statistics_(statistics), + start_info_("na", "na", "na", 0, Env::TOTAL), + wait_info_("na", "na", "na", 0, Env::TOTAL), + listeners_(listeners), + table_properties_collector_factories_( + std::move(table_properties_collector_factories)) {} + + static const char* kClassName() { return "MyTestCompactionService"; } + + const char* Name() const override { return kClassName(); } + + CompactionServiceJobStatus StartV2( + const CompactionServiceJobInfo& info, + const std::string& compaction_service_input) override { + InstrumentedMutexLock l(&mutex_); + start_info_ = info; + assert(info.db_name == db_path_); + jobs_.emplace(info.job_id, compaction_service_input); + CompactionServiceJobStatus s = CompactionServiceJobStatus::kSuccess; + if (is_override_start_status_) { + return override_start_status_; + } + return s; + } + + CompactionServiceJobStatus WaitForCompleteV2( + const CompactionServiceJobInfo& info, + std::string* compaction_service_result) override { + std::string compaction_input; + assert(info.db_name == db_path_); + { + InstrumentedMutexLock l(&mutex_); + wait_info_ = info; + auto i = jobs_.find(info.job_id); + if (i == jobs_.end()) { + return CompactionServiceJobStatus::kFailure; + } + compaction_input = std::move(i->second); + jobs_.erase(i); + } + + if (is_override_wait_status_) { + return override_wait_status_; + } + + CompactionServiceOptionsOverride options_override; + options_override.env = options_.env; + options_override.file_checksum_gen_factory = + options_.file_checksum_gen_factory; + options_override.comparator = options_.comparator; + options_override.merge_operator = options_.merge_operator; + options_override.compaction_filter = options_.compaction_filter; + options_override.compaction_filter_factory = + options_.compaction_filter_factory; + options_override.prefix_extractor = options_.prefix_extractor; + options_override.table_factory = options_.table_factory; + options_override.sst_partitioner_factory = options_.sst_partitioner_factory; + options_override.statistics = statistics_; + if (!listeners_.empty()) { + options_override.listeners = listeners_; + } + + if (!table_properties_collector_factories_.empty()) { + options_override.table_properties_collector_factories = + table_properties_collector_factories_; + } + + OpenAndCompactOptions options; + options.canceled = &canceled_; + + Status s = DB::OpenAndCompact( + options, db_path_, db_path_ + "/" + std::to_string(info.job_id), + compaction_input, compaction_service_result, options_override); + if (is_override_wait_result_) { + *compaction_service_result = override_wait_result_; + } + compaction_num_.fetch_add(1); + if (s.ok()) { + return CompactionServiceJobStatus::kSuccess; + } else { + return CompactionServiceJobStatus::kFailure; + } + } + + int GetCompactionNum() { return compaction_num_.load(); } + + CompactionServiceJobInfo GetCompactionInfoForStart() { return start_info_; } + CompactionServiceJobInfo GetCompactionInfoForWait() { return wait_info_; } + + void OverrideStartStatus(CompactionServiceJobStatus s) { + is_override_start_status_ = true; + override_start_status_ = s; + } + + void OverrideWaitStatus(CompactionServiceJobStatus s) { + is_override_wait_status_ = true; + override_wait_status_ = s; + } + + void OverrideWaitResult(std::string str) { + is_override_wait_result_ = true; + override_wait_result_ = std::move(str); + } + + void ResetOverride() { + is_override_wait_result_ = false; + is_override_start_status_ = false; + is_override_wait_status_ = false; + } + + void SetCanceled(bool canceled) { canceled_ = canceled; } + + private: + InstrumentedMutex mutex_; + std::atomic_int compaction_num_{0}; + std::map jobs_; + const std::string db_path_; + Options options_; + std::shared_ptr statistics_; + CompactionServiceJobInfo start_info_; + CompactionServiceJobInfo wait_info_; + bool is_override_start_status_ = false; + CompactionServiceJobStatus override_start_status_ = + CompactionServiceJobStatus::kFailure; + bool is_override_wait_status_ = false; + CompactionServiceJobStatus override_wait_status_ = + CompactionServiceJobStatus::kFailure; + bool is_override_wait_result_ = false; + std::string override_wait_result_; + std::vector> listeners_; + std::vector> + table_properties_collector_factories_; + std::atomic_bool canceled_{false}; +}; + +class CompactionServiceTest : public DBTestBase { + public: + explicit CompactionServiceTest() + : DBTestBase("compaction_service_test", true) {} + + protected: + void ReopenWithCompactionService(Options* options) { + options->env = env_; + primary_statistics_ = CreateDBStatistics(); + options->statistics = primary_statistics_; + compactor_statistics_ = CreateDBStatistics(); + + compaction_service_ = std::make_shared( + dbname_, *options, compactor_statistics_, remote_listeners, + remote_table_properties_collector_factories); + options->compaction_service = compaction_service_; + DestroyAndReopen(*options); + } + + Statistics* GetCompactorStatistics() { return compactor_statistics_.get(); } + + Statistics* GetPrimaryStatistics() { return primary_statistics_.get(); } + + MyTestCompactionService* GetCompactionService() { + CompactionService* cs = compaction_service_.get(); + return static_cast_with_check(cs); + } + + void GenerateTestData() { + // Generate 20 files @ L2 + for (int i = 0; i < 20; i++) { + for (int j = 0; j < 10; j++) { + int key_id = i * 10 + j; + ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id))); + } + ASSERT_OK(Flush()); + } + MoveFilesToLevel(2); + + // Generate 10 files @ L1 overlap with all 20 files @ L2 + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 10; j++) { + int key_id = i * 20 + j * 2; + ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id))); + } + ASSERT_OK(Flush()); + } + MoveFilesToLevel(1); + ASSERT_EQ(FilesPerLevel(), "0,10,20"); + } + + void VerifyTestData() { + for (int i = 0; i < 200; i++) { + auto result = Get(Key(i)); + if (i % 2) { + ASSERT_EQ(result, "value" + std::to_string(i)); + } else { + ASSERT_EQ(result, "value_new" + std::to_string(i)); + } + } + } + + std::vector> remote_listeners; + std::vector> + remote_table_properties_collector_factories; + + private: + std::shared_ptr compactor_statistics_; + std::shared_ptr primary_statistics_; + std::shared_ptr compaction_service_; +}; + +TEST_F(CompactionServiceTest, BasicCompactions) { + Options options = CurrentOptions(); + ReopenWithCompactionService(&options); + + Statistics* primary_statistics = GetPrimaryStatistics(); + Statistics* compactor_statistics = GetCompactorStatistics(); + + for (int i = 0; i < 20; i++) { + for (int j = 0; j < 10; j++) { + int key_id = i * 10 + j; + ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id))); + } + ASSERT_OK(Flush()); + } + + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 10; j++) { + int key_id = i * 20 + j * 2; + ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // verify result + for (int i = 0; i < 200; i++) { + auto result = Get(Key(i)); + if (i % 2) { + ASSERT_EQ(result, "value" + std::to_string(i)); + } else { + ASSERT_EQ(result, "value_new" + std::to_string(i)); + } + } + auto my_cs = GetCompactionService(); + ASSERT_GE(my_cs->GetCompactionNum(), 1); + + // make sure the compaction statistics is only recorded on the remote side + ASSERT_GE(compactor_statistics->getTickerCount(COMPACT_WRITE_BYTES), 1); + ASSERT_GE(compactor_statistics->getTickerCount(COMPACT_READ_BYTES), 1); + ASSERT_EQ(primary_statistics->getTickerCount(COMPACT_WRITE_BYTES), 0); + // even with remote compaction, primary host still needs to read SST files to + // `verify_table()`. + ASSERT_GE(primary_statistics->getTickerCount(COMPACT_READ_BYTES), 1); + // all the compaction write happens on the remote side + ASSERT_EQ(primary_statistics->getTickerCount(REMOTE_COMPACT_WRITE_BYTES), + compactor_statistics->getTickerCount(COMPACT_WRITE_BYTES)); + ASSERT_GE(primary_statistics->getTickerCount(REMOTE_COMPACT_READ_BYTES), 1); + ASSERT_GT(primary_statistics->getTickerCount(COMPACT_READ_BYTES), + primary_statistics->getTickerCount(REMOTE_COMPACT_READ_BYTES)); + // compactor is already the remote side, which doesn't have remote + ASSERT_EQ(compactor_statistics->getTickerCount(REMOTE_COMPACT_READ_BYTES), 0); + ASSERT_EQ(compactor_statistics->getTickerCount(REMOTE_COMPACT_WRITE_BYTES), + 0); + + // Test failed compaction + SyncPoint::GetInstance()->SetCallBack( + "DBImplSecondary::CompactWithoutInstallation::End", [&](void* status) { + // override job status + auto s = static_cast(status); + *s = Status::Aborted("MyTestCompactionService failed to compact!"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + Status s; + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 10; j++) { + int key_id = i * 20 + j * 2; + s = Put(Key(key_id), "value_new" + std::to_string(key_id)); + if (s.IsAborted()) { + break; + } + } + if (s.IsAborted()) { + break; + } + s = Flush(); + if (s.IsAborted()) { + break; + } + s = dbfull()->TEST_WaitForCompact(); + if (s.IsAborted()) { + break; + } + } + ASSERT_TRUE(s.IsAborted()); + + // Test re-open and successful unique id verification + std::atomic_int verify_passed{0}; + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTable::Open::PassedVerifyUniqueId", [&](void* arg) { + // override job status + auto id = static_cast(arg); + assert(*id != kNullUniqueId64x2); + verify_passed++; + }); + Reopen(options); + ASSERT_GT(verify_passed, 0); + Close(); +} + +TEST_F(CompactionServiceTest, ManualCompaction) { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + ReopenWithCompactionService(&options); + GenerateTestData(); + + auto my_cs = GetCompactionService(); + + std::string start_str = Key(15); + std::string end_str = Key(45); + Slice start(start_str); + Slice end(end_str); + uint64_t comp_num = my_cs->GetCompactionNum(); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &start, &end)); + ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1); + VerifyTestData(); + + start_str = Key(120); + start = start_str; + comp_num = my_cs->GetCompactionNum(); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &start, nullptr)); + ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1); + VerifyTestData(); + + end_str = Key(92); + end = end_str; + comp_num = my_cs->GetCompactionNum(); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, &end)); + ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1); + VerifyTestData(); + + comp_num = my_cs->GetCompactionNum(); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1); + VerifyTestData(); +} + +TEST_F(CompactionServiceTest, CancelCompactionOnRemoteSide) { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + ReopenWithCompactionService(&options); + GenerateTestData(); + + auto my_cs = GetCompactionService(); + + std::string start_str = Key(15); + std::string end_str = Key(45); + Slice start(start_str); + Slice end(end_str); + uint64_t comp_num = my_cs->GetCompactionNum(); + + // Test cancel compaction at the beginning + my_cs->SetCanceled(true); + auto s = db_->CompactRange(CompactRangeOptions(), &start, &end); + ASSERT_TRUE(s.IsIncomplete()); + // compaction number is not increased + ASSERT_GE(my_cs->GetCompactionNum(), comp_num); + VerifyTestData(); + + // Test cancel compaction in progress + ReopenWithCompactionService(&options); + GenerateTestData(); + my_cs = GetCompactionService(); + my_cs->SetCanceled(false); + + std::atomic_bool cancel_issued{false}; + SyncPoint::GetInstance()->SetCallBack("CompactionJob::Run():Inprogress", + [&](void* /*arg*/) { + cancel_issued = true; + my_cs->SetCanceled(true); + }); + + SyncPoint::GetInstance()->EnableProcessing(); + + s = db_->CompactRange(CompactRangeOptions(), &start, &end); + ASSERT_TRUE(s.IsIncomplete()); + ASSERT_TRUE(cancel_issued); + // compaction number is not increased + ASSERT_GE(my_cs->GetCompactionNum(), comp_num); + VerifyTestData(); +} + +TEST_F(CompactionServiceTest, FailedToStart) { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + ReopenWithCompactionService(&options); + + GenerateTestData(); + + auto my_cs = GetCompactionService(); + my_cs->OverrideStartStatus(CompactionServiceJobStatus::kFailure); + + std::string start_str = Key(15); + std::string end_str = Key(45); + Slice start(start_str); + Slice end(end_str); + Status s = db_->CompactRange(CompactRangeOptions(), &start, &end); + ASSERT_TRUE(s.IsIncomplete()); +} + +TEST_F(CompactionServiceTest, InvalidResult) { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + ReopenWithCompactionService(&options); + + GenerateTestData(); + + auto my_cs = GetCompactionService(); + my_cs->OverrideWaitResult("Invalid Str"); + + std::string start_str = Key(15); + std::string end_str = Key(45); + Slice start(start_str); + Slice end(end_str); + Status s = db_->CompactRange(CompactRangeOptions(), &start, &end); + ASSERT_FALSE(s.ok()); +} + +TEST_F(CompactionServiceTest, SubCompaction) { + Options options = CurrentOptions(); + options.max_subcompactions = 10; + options.target_file_size_base = 1 << 10; // 1KB + options.disable_auto_compactions = true; + ReopenWithCompactionService(&options); + + GenerateTestData(); + VerifyTestData(); + + auto my_cs = GetCompactionService(); + int compaction_num_before = my_cs->GetCompactionNum(); + + auto cro = CompactRangeOptions(); + cro.max_subcompactions = 10; + Status s = db_->CompactRange(cro, nullptr, nullptr); + ASSERT_OK(s); + VerifyTestData(); + int compaction_num = my_cs->GetCompactionNum() - compaction_num_before; + // make sure there's sub-compaction by checking the compaction number + ASSERT_GE(compaction_num, 2); +} + +class PartialDeleteCompactionFilter : public CompactionFilter { + public: + CompactionFilter::Decision FilterV2( + int /*level*/, const Slice& key, ValueType /*value_type*/, + const Slice& /*existing_value*/, std::string* /*new_value*/, + std::string* /*skip_until*/) const override { + int i = std::stoi(key.ToString().substr(3)); + if (i > 5 && i <= 105) { + return CompactionFilter::Decision::kRemove; + } + return CompactionFilter::Decision::kKeep; + } + + const char* Name() const override { return "PartialDeleteCompactionFilter"; } +}; + +TEST_F(CompactionServiceTest, CompactionFilter) { + Options options = CurrentOptions(); + std::unique_ptr delete_comp_filter( + new PartialDeleteCompactionFilter()); + options.compaction_filter = delete_comp_filter.get(); + ReopenWithCompactionService(&options); + + for (int i = 0; i < 20; i++) { + for (int j = 0; j < 10; j++) { + int key_id = i * 10 + j; + ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id))); + } + ASSERT_OK(Flush()); + } + + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 10; j++) { + int key_id = i * 20 + j * 2; + ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + // verify result + for (int i = 0; i < 200; i++) { + auto result = Get(Key(i)); + if (i > 5 && i <= 105) { + ASSERT_EQ(result, "NOT_FOUND"); + } else if (i % 2) { + ASSERT_EQ(result, "value" + std::to_string(i)); + } else { + ASSERT_EQ(result, "value_new" + std::to_string(i)); + } + } + auto my_cs = GetCompactionService(); + ASSERT_GE(my_cs->GetCompactionNum(), 1); +} + +TEST_F(CompactionServiceTest, Snapshot) { + Options options = CurrentOptions(); + ReopenWithCompactionService(&options); + + ASSERT_OK(Put(Key(1), "value1")); + ASSERT_OK(Put(Key(2), "value1")); + const Snapshot* s1 = db_->GetSnapshot(); + ASSERT_OK(Flush()); + + ASSERT_OK(Put(Key(1), "value2")); + ASSERT_OK(Put(Key(3), "value2")); + ASSERT_OK(Flush()); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + auto my_cs = GetCompactionService(); + ASSERT_GE(my_cs->GetCompactionNum(), 1); + ASSERT_EQ("value1", Get(Key(1), s1)); + ASSERT_EQ("value2", Get(Key(1))); + db_->ReleaseSnapshot(s1); +} + +TEST_F(CompactionServiceTest, ConcurrentCompaction) { + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = 100; + options.max_background_jobs = 20; + ReopenWithCompactionService(&options); + GenerateTestData(); + + ColumnFamilyMetaData meta; + db_->GetColumnFamilyMetaData(&meta); + + std::vector threads; + for (const auto& file : meta.levels[1].files) { + threads.emplace_back(std::thread([&]() { + std::string fname = file.db_path + "/" + file.name; + ASSERT_OK(db_->CompactFiles(CompactionOptions(), {fname}, 2)); + })); + } + + for (auto& thread : threads) { + thread.join(); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // verify result + for (int i = 0; i < 200; i++) { + auto result = Get(Key(i)); + if (i % 2) { + ASSERT_EQ(result, "value" + std::to_string(i)); + } else { + ASSERT_EQ(result, "value_new" + std::to_string(i)); + } + } + auto my_cs = GetCompactionService(); + ASSERT_EQ(my_cs->GetCompactionNum(), 10); + ASSERT_EQ(FilesPerLevel(), "0,0,10"); +} + +TEST_F(CompactionServiceTest, CompactionInfo) { + Options options = CurrentOptions(); + ReopenWithCompactionService(&options); + + for (int i = 0; i < 20; i++) { + for (int j = 0; j < 10; j++) { + int key_id = i * 10 + j; + ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id))); + } + ASSERT_OK(Flush()); + } + + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 10; j++) { + int key_id = i * 20 + j * 2; + ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + auto my_cs = + static_cast_with_check(GetCompactionService()); + uint64_t comp_num = my_cs->GetCompactionNum(); + ASSERT_GE(comp_num, 1); + + CompactionServiceJobInfo info = my_cs->GetCompactionInfoForStart(); + ASSERT_EQ(dbname_, info.db_name); + std::string db_id, db_session_id; + ASSERT_OK(db_->GetDbIdentity(db_id)); + ASSERT_EQ(db_id, info.db_id); + ASSERT_OK(db_->GetDbSessionId(db_session_id)); + ASSERT_EQ(db_session_id, info.db_session_id); + ASSERT_EQ(Env::LOW, info.priority); + info = my_cs->GetCompactionInfoForWait(); + ASSERT_EQ(dbname_, info.db_name); + ASSERT_EQ(db_id, info.db_id); + ASSERT_EQ(db_session_id, info.db_session_id); + ASSERT_EQ(Env::LOW, info.priority); + + // Test priority USER + ColumnFamilyMetaData meta; + db_->GetColumnFamilyMetaData(&meta); + SstFileMetaData file = meta.levels[1].files[0]; + ASSERT_OK(db_->CompactFiles(CompactionOptions(), + {file.db_path + "/" + file.name}, 2)); + info = my_cs->GetCompactionInfoForStart(); + ASSERT_EQ(Env::USER, info.priority); + info = my_cs->GetCompactionInfoForWait(); + ASSERT_EQ(Env::USER, info.priority); + + // Test priority BOTTOM + env_->SetBackgroundThreads(1, Env::BOTTOM); + options.num_levels = 2; + ReopenWithCompactionService(&options); + my_cs = + static_cast_with_check(GetCompactionService()); + + for (int i = 0; i < 20; i++) { + for (int j = 0; j < 10; j++) { + int key_id = i * 10 + j; + ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id))); + } + ASSERT_OK(Flush()); + } + + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 10; j++) { + int key_id = i * 20 + j * 2; + ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + info = my_cs->GetCompactionInfoForStart(); + ASSERT_EQ(Env::BOTTOM, info.priority); + info = my_cs->GetCompactionInfoForWait(); + ASSERT_EQ(Env::BOTTOM, info.priority); +} + +TEST_F(CompactionServiceTest, FallbackLocalAuto) { + Options options = CurrentOptions(); + ReopenWithCompactionService(&options); + + auto my_cs = GetCompactionService(); + Statistics* compactor_statistics = GetCompactorStatistics(); + Statistics* primary_statistics = GetPrimaryStatistics(); + uint64_t compactor_write_bytes = + compactor_statistics->getTickerCount(COMPACT_WRITE_BYTES); + uint64_t primary_write_bytes = + primary_statistics->getTickerCount(COMPACT_WRITE_BYTES); + + my_cs->OverrideStartStatus(CompactionServiceJobStatus::kUseLocal); + + for (int i = 0; i < 20; i++) { + for (int j = 0; j < 10; j++) { + int key_id = i * 10 + j; + ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id))); + } + ASSERT_OK(Flush()); + } + + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 10; j++) { + int key_id = i * 20 + j * 2; + ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // verify result + for (int i = 0; i < 200; i++) { + auto result = Get(Key(i)); + if (i % 2) { + ASSERT_EQ(result, "value" + std::to_string(i)); + } else { + ASSERT_EQ(result, "value_new" + std::to_string(i)); + } + } + + ASSERT_EQ(my_cs->GetCompactionNum(), 0); + + // make sure the compaction statistics is only recorded on the local side + ASSERT_EQ(compactor_statistics->getTickerCount(COMPACT_WRITE_BYTES), + compactor_write_bytes); + ASSERT_GT(primary_statistics->getTickerCount(COMPACT_WRITE_BYTES), + primary_write_bytes); + ASSERT_EQ(primary_statistics->getTickerCount(REMOTE_COMPACT_READ_BYTES), 0); + ASSERT_EQ(primary_statistics->getTickerCount(REMOTE_COMPACT_WRITE_BYTES), 0); +} + +TEST_F(CompactionServiceTest, FallbackLocalManual) { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + ReopenWithCompactionService(&options); + + GenerateTestData(); + VerifyTestData(); + + auto my_cs = GetCompactionService(); + Statistics* compactor_statistics = GetCompactorStatistics(); + Statistics* primary_statistics = GetPrimaryStatistics(); + uint64_t compactor_write_bytes = + compactor_statistics->getTickerCount(COMPACT_WRITE_BYTES); + uint64_t primary_write_bytes = + primary_statistics->getTickerCount(COMPACT_WRITE_BYTES); + + // re-enable remote compaction + my_cs->ResetOverride(); + std::string start_str = Key(15); + std::string end_str = Key(45); + Slice start(start_str); + Slice end(end_str); + uint64_t comp_num = my_cs->GetCompactionNum(); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &start, &end)); + ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1); + // make sure the compaction statistics is only recorded on the remote side + ASSERT_GT(compactor_statistics->getTickerCount(COMPACT_WRITE_BYTES), + compactor_write_bytes); + ASSERT_EQ(primary_statistics->getTickerCount(REMOTE_COMPACT_WRITE_BYTES), + compactor_statistics->getTickerCount(COMPACT_WRITE_BYTES)); + ASSERT_EQ(primary_statistics->getTickerCount(COMPACT_WRITE_BYTES), + primary_write_bytes); + + // return run local again with API WaitForComplete + my_cs->OverrideWaitStatus(CompactionServiceJobStatus::kUseLocal); + start_str = Key(120); + start = start_str; + comp_num = my_cs->GetCompactionNum(); + compactor_write_bytes = + compactor_statistics->getTickerCount(COMPACT_WRITE_BYTES); + primary_write_bytes = primary_statistics->getTickerCount(COMPACT_WRITE_BYTES); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &start, nullptr)); + ASSERT_EQ(my_cs->GetCompactionNum(), + comp_num); // no remote compaction is run + // make sure the compaction statistics is only recorded on the local side + ASSERT_EQ(compactor_statistics->getTickerCount(COMPACT_WRITE_BYTES), + compactor_write_bytes); + ASSERT_GT(primary_statistics->getTickerCount(COMPACT_WRITE_BYTES), + primary_write_bytes); + ASSERT_EQ(primary_statistics->getTickerCount(REMOTE_COMPACT_WRITE_BYTES), + compactor_write_bytes); + + // verify result after 2 manual compactions + VerifyTestData(); +} + +TEST_F(CompactionServiceTest, RemoteEventListener) { + class RemoteEventListenerTest : public EventListener { + public: + const char* Name() const override { return "RemoteEventListenerTest"; } + + void OnSubcompactionBegin(const SubcompactionJobInfo& info) override { + auto result = on_going_compactions.emplace(info.job_id); + ASSERT_TRUE(result.second); // make sure there's no duplication + compaction_num++; + EventListener::OnSubcompactionBegin(info); + } + void OnSubcompactionCompleted(const SubcompactionJobInfo& info) override { + auto num = on_going_compactions.erase(info.job_id); + ASSERT_TRUE(num == 1); // make sure the compaction id exists + EventListener::OnSubcompactionCompleted(info); + } + void OnTableFileCreated(const TableFileCreationInfo& info) override { + ASSERT_EQ(on_going_compactions.count(info.job_id), 1); + file_created++; + EventListener::OnTableFileCreated(info); + } + void OnTableFileCreationStarted( + const TableFileCreationBriefInfo& info) override { + ASSERT_EQ(on_going_compactions.count(info.job_id), 1); + file_creation_started++; + EventListener::OnTableFileCreationStarted(info); + } + + bool ShouldBeNotifiedOnFileIO() override { + file_io_notified++; + return EventListener::ShouldBeNotifiedOnFileIO(); + } + + std::atomic_uint64_t file_io_notified{0}; + std::atomic_uint64_t file_creation_started{0}; + std::atomic_uint64_t file_created{0}; + + std::set on_going_compactions; // store the job_id + std::atomic_uint64_t compaction_num{0}; + }; + + auto listener = new RemoteEventListenerTest(); + remote_listeners.emplace_back(listener); + + Options options = CurrentOptions(); + ReopenWithCompactionService(&options); + + for (int i = 0; i < 20; i++) { + for (int j = 0; j < 10; j++) { + int key_id = i * 10 + j; + ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id))); + } + ASSERT_OK(Flush()); + } + + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 10; j++) { + int key_id = i * 20 + j * 2; + ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // check the events are triggered + ASSERT_TRUE(listener->file_io_notified > 0); + ASSERT_TRUE(listener->file_creation_started > 0); + ASSERT_TRUE(listener->file_created > 0); + ASSERT_TRUE(listener->compaction_num > 0); + ASSERT_TRUE(listener->on_going_compactions.empty()); + + // verify result + for (int i = 0; i < 200; i++) { + auto result = Get(Key(i)); + if (i % 2) { + ASSERT_EQ(result, "value" + std::to_string(i)); + } else { + ASSERT_EQ(result, "value_new" + std::to_string(i)); + } + } +} + +TEST_F(CompactionServiceTest, TablePropertiesCollector) { + const static std::string kUserPropertyName = "TestCount"; + + class TablePropertiesCollectorTest : public TablePropertiesCollector { + public: + Status Finish(UserCollectedProperties* properties) override { + *properties = UserCollectedProperties{ + {kUserPropertyName, std::to_string(count_)}, + }; + return Status::OK(); + } + + UserCollectedProperties GetReadableProperties() const override { + return UserCollectedProperties(); + } + + const char* Name() const override { return "TablePropertiesCollectorTest"; } + + Status AddUserKey(const Slice& /*user_key*/, const Slice& /*value*/, + EntryType /*type*/, SequenceNumber /*seq*/, + uint64_t /*file_size*/) override { + count_++; + return Status::OK(); + } + + private: + uint32_t count_ = 0; + }; + + class TablePropertiesCollectorFactoryTest + : public TablePropertiesCollectorFactory { + public: + TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context /*context*/) override { + return new TablePropertiesCollectorTest(); + } + + const char* Name() const override { + return "TablePropertiesCollectorFactoryTest"; + } + }; + + auto factory = new TablePropertiesCollectorFactoryTest(); + remote_table_properties_collector_factories.emplace_back(factory); + + const int kNumSst = 3; + const int kLevel0Trigger = 4; + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = kLevel0Trigger; + ReopenWithCompactionService(&options); + + // generate a few SSTs locally which should not have user property + for (int i = 0; i < kNumSst; i++) { + for (int j = 0; j < 100; j++) { + ASSERT_OK(Put(Key(i * 10 + j), "value")); + } + ASSERT_OK(Flush()); + } + + TablePropertiesCollection fname_to_props; + ASSERT_OK(db_->GetPropertiesOfAllTables(&fname_to_props)); + for (const auto& file_props : fname_to_props) { + auto properties = file_props.second->user_collected_properties; + auto it = properties.find(kUserPropertyName); + ASSERT_EQ(it, properties.end()); + } + + // trigger compaction + for (int i = kNumSst; i < kLevel0Trigger; i++) { + for (int j = 0; j < 100; j++) { + ASSERT_OK(Put(Key(i * 10 + j), "value")); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_OK(db_->GetPropertiesOfAllTables(&fname_to_props)); + + bool has_user_property = false; + for (const auto& file_props : fname_to_props) { + auto properties = file_props.second->user_collected_properties; + auto it = properties.find(kUserPropertyName); + if (it != properties.end()) { + has_user_property = true; + ASSERT_GT(std::stoi(it->second), 0); + } + } + ASSERT_TRUE(has_user_property); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_state.cc b/librocksdb-sys/rocksdb/db/compaction/compaction_state.cc new file mode 100644 index 0000000..ee4b0c1 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_state.cc @@ -0,0 +1,46 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/compaction/compaction_state.h" + +namespace ROCKSDB_NAMESPACE { + +Slice CompactionState::SmallestUserKey() { + for (const auto& sub_compact_state : sub_compact_states) { + Slice smallest = sub_compact_state.SmallestUserKey(); + if (!smallest.empty()) { + return smallest; + } + } + // If there is no finished output, return an empty slice. + return Slice{nullptr, 0}; +} + +Slice CompactionState::LargestUserKey() { + for (auto it = sub_compact_states.rbegin(); it < sub_compact_states.rend(); + ++it) { + Slice largest = it->LargestUserKey(); + if (!largest.empty()) { + return largest; + } + } + // If there is no finished output, return an empty slice. + return Slice{nullptr, 0}; +} + +void CompactionState::AggregateCompactionStats( + InternalStats::CompactionStatsFull& compaction_stats, + CompactionJobStats& compaction_job_stats) { + for (const auto& sc : sub_compact_states) { + sc.AggregateCompactionStats(compaction_stats); + compaction_job_stats.Add(sc.compaction_job_stats); + } +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/compaction_state.h b/librocksdb-sys/rocksdb/db/compaction/compaction_state.h new file mode 100644 index 0000000..cc5b66c --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/compaction_state.h @@ -0,0 +1,42 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include "db/compaction/compaction.h" +#include "db/compaction/subcompaction_state.h" +#include "db/internal_stats.h" + +// Data structures used for compaction_job and compaction_service_job which has +// the list of sub_compact_states and the aggregated information for the +// compaction. +namespace ROCKSDB_NAMESPACE { + +// Maintains state for the entire compaction +class CompactionState { + public: + Compaction* const compaction; + + // REQUIRED: subcompaction states are stored in order of increasing key-range + std::vector sub_compact_states; + Status status; + + void AggregateCompactionStats( + InternalStats::CompactionStatsFull& compaction_stats, + CompactionJobStats& compaction_job_stats); + + explicit CompactionState(Compaction* c) : compaction(c) {} + + Slice SmallestUserKey(); + + Slice LargestUserKey(); +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/file_pri.h b/librocksdb-sys/rocksdb/db/compaction/file_pri.h new file mode 100644 index 0000000..82dddcf --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/file_pri.h @@ -0,0 +1,92 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#pragma once +#include + +#include "db/version_edit.h" + +namespace ROCKSDB_NAMESPACE { +// We boost files that are closer to TTL limit. This boosting could be +// through FileMetaData.compensated_file_size but this compensated size +// is widely used as something similar to file size so dramatically boost +// the value might cause unintended consequences. +// +// This boosting algorithm can go very fancy, but here we use a simple +// formula which can satisify: +// (1) Different levels are triggered slightly differently to avoid +// too many cascading cases +// (2) Files in the same level get boosting more when TTL gets closer. +// +// Don't do any boosting before TTL has past by half. This is to make +// sure lower write amp for most of the case. And all levels should be +// fully boosted when total TTL compaction threshold triggers. +// Differientiate boosting ranges of each level by 1/2. This will make +// range for each level exponentially increasing. We could do it by +// having them to be equal, or go even fancier. We can adjust it after +// we observe the behavior in production. +// The threshold starting boosting: +// +------------------------------------------------------------------ + +// ^ ^ ^ ^ ^ ^ +// Age 0 ... | | second last level thresold +// | | +// | third last level +// | +// forth last level +// +// We arbitrarily set with 0 when a file is aged boost_age_start and +// grow linearly. The ratio is arbitrarily set so that when the next level +// starts to boost, the previous level's boosting amount is 16. +class FileTtlBooster { + public: + FileTtlBooster(uint64_t current_time, uint64_t ttl, int num_non_empty_levels, + int level) + : current_time_(current_time) { + if (ttl == 0 || level == 0 || level >= num_non_empty_levels - 1) { + enabled_ = false; + boost_age_start_ = 0; + boost_step_ = 1; + } else { + enabled_ = true; + uint64_t all_boost_start_age = ttl / 2; + uint64_t all_boost_age_range = (ttl / 32) * 31 - all_boost_start_age; + uint64_t boost_age_range = + all_boost_age_range >> (num_non_empty_levels - level - 1); + boost_age_start_ = all_boost_start_age + boost_age_range; + const uint64_t kBoostRatio = 16; + // prevent 0 value to avoid divide 0 error. + boost_step_ = std::max(boost_age_range / kBoostRatio, uint64_t{1}); + } + } + + uint64_t GetBoostScore(FileMetaData* f) { + if (!enabled_) { + return 1; + } + uint64_t oldest_ancester_time = f->TryGetOldestAncesterTime(); + if (oldest_ancester_time >= current_time_) { + return 1; + } + uint64_t age = current_time_ - oldest_ancester_time; + if (age > boost_age_start_) { + // Use integer just for convenience. + // We could make all file_to_order double if we want. + // Technically this can overflow if users override timing and + // give a very high current time. Ignore the case for simplicity. + // Boosting is addition to current value, so +1. This will effectively + // make boosting to kick in after the first boost_step_ is reached. + return (age - boost_age_start_) / boost_step_ + 1; + } + return 1; + } + + private: + bool enabled_; + uint64_t current_time_; + uint64_t boost_age_start_; + uint64_t boost_step_; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/sst_partitioner.cc b/librocksdb-sys/rocksdb/db/compaction/sst_partitioner.cc new file mode 100644 index 0000000..2f4d879 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/sst_partitioner.cc @@ -0,0 +1,83 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#include "rocksdb/sst_partitioner.h" + +#include + +#include "rocksdb/utilities/customizable_util.h" +#include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/options_type.h" + +namespace ROCKSDB_NAMESPACE { +static std::unordered_map + sst_fixed_prefix_type_info = { + {"length", + {0, OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, +}; + +SstPartitionerFixedPrefixFactory::SstPartitionerFixedPrefixFactory(size_t len) + : len_(len) { + RegisterOptions("Length", &len_, &sst_fixed_prefix_type_info); +} + +PartitionerResult SstPartitionerFixedPrefix::ShouldPartition( + const PartitionerRequest& request) { + Slice last_key_fixed(*request.prev_user_key); + if (last_key_fixed.size() > len_) { + last_key_fixed.size_ = len_; + } + Slice current_key_fixed(*request.current_user_key); + if (current_key_fixed.size() > len_) { + current_key_fixed.size_ = len_; + } + return last_key_fixed.compare(current_key_fixed) != 0 ? kRequired + : kNotRequired; +} + +bool SstPartitionerFixedPrefix::CanDoTrivialMove( + const Slice& smallest_user_key, const Slice& largest_user_key) { + return ShouldPartition(PartitionerRequest(smallest_user_key, largest_user_key, + 0)) == kNotRequired; +} + +std::unique_ptr +SstPartitionerFixedPrefixFactory::CreatePartitioner( + const SstPartitioner::Context& /* context */) const { + return std::unique_ptr(new SstPartitionerFixedPrefix(len_)); +} + +std::shared_ptr NewSstPartitionerFixedPrefixFactory( + size_t prefix_len) { + return std::make_shared(prefix_len); +} + +namespace { +static int RegisterSstPartitionerFactories(ObjectLibrary& library, + const std::string& /*arg*/) { + library.AddFactory( + SstPartitionerFixedPrefixFactory::kClassName(), + [](const std::string& /*uri*/, + std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new SstPartitionerFixedPrefixFactory(0)); + return guard->get(); + }); + return 1; +} +} // namespace + +Status SstPartitionerFactory::CreateFromString( + const ConfigOptions& options, const std::string& value, + std::shared_ptr* result) { + static std::once_flag once; + std::call_once(once, [&]() { + RegisterSstPartitionerFactories(*(ObjectLibrary::Default().get()), ""); + }); + return LoadSharedObject(options, value, result); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/subcompaction_state.cc b/librocksdb-sys/rocksdb/db/compaction/subcompaction_state.cc new file mode 100644 index 0000000..0c56471 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/subcompaction_state.cc @@ -0,0 +1,106 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/compaction/subcompaction_state.h" + +#include "rocksdb/sst_partitioner.h" + +namespace ROCKSDB_NAMESPACE { +void SubcompactionState::AggregateCompactionStats( + InternalStats::CompactionStatsFull& compaction_stats) const { + compaction_stats.stats.Add(compaction_outputs_.stats_); + if (HasPenultimateLevelOutputs()) { + compaction_stats.has_penultimate_level_output = true; + compaction_stats.penultimate_level_stats.Add( + penultimate_level_outputs_.stats_); + } +} + +OutputIterator SubcompactionState::GetOutputs() const { + return OutputIterator(penultimate_level_outputs_.outputs_, + compaction_outputs_.outputs_); +} + +void SubcompactionState::Cleanup(Cache* cache) { + penultimate_level_outputs_.Cleanup(); + compaction_outputs_.Cleanup(); + + if (!status.ok()) { + for (const auto& out : GetOutputs()) { + // If this file was inserted into the table cache then remove + // them here because this compaction was not committed. + TableCache::Evict(cache, out.meta.fd.GetNumber()); + } + } + // TODO: sub_compact.io_status is not checked like status. Not sure if thats + // intentional. So ignoring the io_status as of now. + io_status.PermitUncheckedError(); +} + +Slice SubcompactionState::SmallestUserKey() const { + if (has_penultimate_level_outputs_) { + Slice a = compaction_outputs_.SmallestUserKey(); + Slice b = penultimate_level_outputs_.SmallestUserKey(); + if (a.empty()) { + return b; + } + if (b.empty()) { + return a; + } + const Comparator* user_cmp = + compaction->column_family_data()->user_comparator(); + if (user_cmp->Compare(a, b) > 0) { + return b; + } else { + return a; + } + } else { + return compaction_outputs_.SmallestUserKey(); + } +} + +Slice SubcompactionState::LargestUserKey() const { + if (has_penultimate_level_outputs_) { + Slice a = compaction_outputs_.LargestUserKey(); + Slice b = penultimate_level_outputs_.LargestUserKey(); + if (a.empty()) { + return b; + } + if (b.empty()) { + return a; + } + const Comparator* user_cmp = + compaction->column_family_data()->user_comparator(); + if (user_cmp->Compare(a, b) < 0) { + return b; + } else { + return a; + } + } else { + return compaction_outputs_.LargestUserKey(); + } +} + +Status SubcompactionState::AddToOutput( + const CompactionIterator& iter, + const CompactionFileOpenFunc& open_file_func, + const CompactionFileCloseFunc& close_file_func) { + // update target output first + is_current_penultimate_level_ = iter.output_to_penultimate_level(); + current_outputs_ = is_current_penultimate_level_ ? &penultimate_level_outputs_ + : &compaction_outputs_; + if (is_current_penultimate_level_) { + has_penultimate_level_outputs_ = true; + } + + return Current().AddToOutput(iter, open_file_func, close_file_func); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/subcompaction_state.h b/librocksdb-sys/rocksdb/db/compaction/subcompaction_state.h new file mode 100644 index 0000000..b933a62 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/subcompaction_state.h @@ -0,0 +1,220 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include + +#include "db/blob/blob_file_addition.h" +#include "db/blob/blob_garbage_meter.h" +#include "db/compaction/compaction.h" +#include "db/compaction/compaction_iterator.h" +#include "db/compaction/compaction_outputs.h" +#include "db/internal_stats.h" +#include "db/output_validator.h" +#include "db/range_del_aggregator.h" + +namespace ROCKSDB_NAMESPACE { + +// Maintains state and outputs for each sub-compaction +// It contains 2 `CompactionOutputs`: +// 1. one for the normal output files +// 2. another for the penultimate level outputs +// a `current` pointer maintains the current output group, when calling +// `AddToOutput()`, it checks the output of the current compaction_iterator key +// and point `current` to the target output group. By default, it just points to +// normal compaction_outputs, if the compaction_iterator key should be placed on +// the penultimate level, `current` is changed to point to +// `penultimate_level_outputs`. +// The later operations uses `Current()` to get the target group. +// +// +----------+ +-----------------------------+ +---------+ +// | *current |--------> | compaction_outputs |----->| output | +// +----------+ +-----------------------------+ +---------+ +// | | output | +// | +---------+ +// | | ... | +// | +// | +-----------------------------+ +---------+ +// +-------------> | penultimate_level_outputs |----->| output | +// +-----------------------------+ +---------+ +// | ... | + +class SubcompactionState { + public: + const Compaction* compaction; + + // The boundaries of the key-range this compaction is interested in. No two + // sub-compactions may have overlapping key-ranges. + // 'start' is inclusive, 'end' is exclusive, and nullptr means unbounded + const std::optional start, end; + + // The return status of this sub-compaction + Status status; + + // The return IO Status of this sub-compaction + IOStatus io_status; + + // Notify on sub-compaction completion only if listener was notified on + // sub-compaction begin. + bool notify_on_subcompaction_completion = false; + + // compaction job stats for this sub-compaction + CompactionJobStats compaction_job_stats; + + // sub-compaction job id, which is used to identify different sub-compaction + // within the same compaction job. + const uint32_t sub_job_id; + + Slice SmallestUserKey() const; + + Slice LargestUserKey() const; + + // Get all outputs from the subcompaction. For per_key_placement compaction, + // it returns both the last level outputs and penultimate level outputs. + OutputIterator GetOutputs() const; + + // Assign range dels aggregator, for each range_del, it can only be assigned + // to one output level, for per_key_placement, it's going to be the + // penultimate level. + // TODO: This does not work for per_key_placement + user-defined timestamp + + // DeleteRange() combo. If user-defined timestamp is enabled, + // it is possible for a range tombstone to belong to bottommost level ( + // seqno < earliest snapshot) without being dropped (garbage collection + // for user-defined timestamp). + void AssignRangeDelAggregator( + std::unique_ptr&& range_del_agg) { + if (compaction->SupportsPerKeyPlacement()) { + penultimate_level_outputs_.AssignRangeDelAggregator( + std::move(range_del_agg)); + } else { + compaction_outputs_.AssignRangeDelAggregator(std::move(range_del_agg)); + } + } + + void RemoveLastEmptyOutput() { + compaction_outputs_.RemoveLastEmptyOutput(); + penultimate_level_outputs_.RemoveLastEmptyOutput(); + } + + void BuildSubcompactionJobInfo( + SubcompactionJobInfo& subcompaction_job_info) const { + const Compaction* c = compaction; + const ColumnFamilyData* cfd = c->column_family_data(); + + subcompaction_job_info.cf_id = cfd->GetID(); + subcompaction_job_info.cf_name = cfd->GetName(); + subcompaction_job_info.status = status; + subcompaction_job_info.subcompaction_job_id = static_cast(sub_job_id); + subcompaction_job_info.base_input_level = c->start_level(); + subcompaction_job_info.output_level = c->output_level(); + subcompaction_job_info.stats = compaction_job_stats; + } + + SubcompactionState() = delete; + SubcompactionState(const SubcompactionState&) = delete; + SubcompactionState& operator=(const SubcompactionState&) = delete; + + SubcompactionState(Compaction* c, const std::optional _start, + const std::optional _end, uint32_t _sub_job_id) + : compaction(c), + start(_start), + end(_end), + sub_job_id(_sub_job_id), + compaction_outputs_(c, /*is_penultimate_level=*/false), + penultimate_level_outputs_(c, /*is_penultimate_level=*/true) { + assert(compaction != nullptr); + // Set output split key (used for RoundRobin feature) only for normal + // compaction_outputs, output to penultimate_level feature doesn't support + // RoundRobin feature (and may never going to be supported, because for + // RoundRobin, the data time is mostly naturally sorted, no need to have + // per-key placement with output_to_penultimate_level). + compaction_outputs_.SetOutputSlitKey(start, end); + } + + SubcompactionState(SubcompactionState&& state) noexcept + : compaction(state.compaction), + start(state.start), + end(state.end), + status(std::move(state.status)), + io_status(std::move(state.io_status)), + notify_on_subcompaction_completion( + state.notify_on_subcompaction_completion), + compaction_job_stats(std::move(state.compaction_job_stats)), + sub_job_id(state.sub_job_id), + compaction_outputs_(std::move(state.compaction_outputs_)), + penultimate_level_outputs_(std::move(state.penultimate_level_outputs_)), + is_current_penultimate_level_(state.is_current_penultimate_level_), + has_penultimate_level_outputs_(state.has_penultimate_level_outputs_) { + current_outputs_ = is_current_penultimate_level_ + ? &penultimate_level_outputs_ + : &compaction_outputs_; + } + + bool HasPenultimateLevelOutputs() const { + return has_penultimate_level_outputs_ || + penultimate_level_outputs_.HasRangeDel(); + } + + bool IsCurrentPenultimateLevel() const { + return is_current_penultimate_level_; + } + + // Add all the new files from this compaction to version_edit + void AddOutputsEdit(VersionEdit* out_edit) const { + for (const auto& file : penultimate_level_outputs_.outputs_) { + out_edit->AddFile(compaction->GetPenultimateLevel(), file.meta); + } + for (const auto& file : compaction_outputs_.outputs_) { + out_edit->AddFile(compaction->output_level(), file.meta); + } + } + + void Cleanup(Cache* cache); + + void AggregateCompactionStats( + InternalStats::CompactionStatsFull& compaction_stats) const; + + CompactionOutputs& Current() const { + assert(current_outputs_); + return *current_outputs_; + } + + // Add compaction_iterator key/value to the `Current` output group. + Status AddToOutput(const CompactionIterator& iter, + const CompactionFileOpenFunc& open_file_func, + const CompactionFileCloseFunc& close_file_func); + + // Close all compaction output files, both output_to_penultimate_level outputs + // and normal outputs. + Status CloseCompactionFiles(const Status& curr_status, + const CompactionFileOpenFunc& open_file_func, + const CompactionFileCloseFunc& close_file_func) { + // Call FinishCompactionOutputFile() even if status is not ok: it needs to + // close the output file. + // CloseOutput() may open new compaction output files. + is_current_penultimate_level_ = true; + Status s = penultimate_level_outputs_.CloseOutput( + curr_status, open_file_func, close_file_func); + is_current_penultimate_level_ = false; + s = compaction_outputs_.CloseOutput(s, open_file_func, close_file_func); + return s; + } + + private: + // State kept for output being generated + CompactionOutputs compaction_outputs_; + CompactionOutputs penultimate_level_outputs_; + CompactionOutputs* current_outputs_ = &compaction_outputs_; + bool is_current_penultimate_level_ = false; + bool has_penultimate_level_outputs_ = false; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/compaction/tiered_compaction_test.cc b/librocksdb-sys/rocksdb/db/compaction/tiered_compaction_test.cc new file mode 100644 index 0000000..d8aa229 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/compaction/tiered_compaction_test.cc @@ -0,0 +1,2149 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/db_test_util.h" +#include "port/stack_trace.h" +#include "rocksdb/iostats_context.h" +#include "rocksdb/listener.h" +#include "rocksdb/utilities/debug.h" +#include "test_util/mock_time_env.h" + +namespace ROCKSDB_NAMESPACE { + + +class TieredCompactionTest : public DBTestBase, + public testing::WithParamInterface { + public: + TieredCompactionTest() + : DBTestBase("tiered_compaction_test", /*env_do_fsync=*/true), + kBasicCompStats(CompactionReason::kUniversalSizeAmplification, 1), + kBasicPerKeyPlacementCompStats( + CompactionReason::kUniversalSizeAmplification, 1), + kBasicFlushStats(CompactionReason::kFlush, 1) { + kBasicCompStats.micros = kHasValue; + kBasicCompStats.cpu_micros = kHasValue; + kBasicCompStats.bytes_read_non_output_levels = kHasValue; + kBasicCompStats.num_input_files_in_non_output_levels = kHasValue; + kBasicCompStats.num_input_records = kHasValue; + kBasicCompStats.num_dropped_records = kHasValue; + + kBasicPerLevelStats.num_output_records = kHasValue; + kBasicPerLevelStats.bytes_written = kHasValue; + kBasicPerLevelStats.num_output_files = kHasValue; + + kBasicPerKeyPlacementCompStats.micros = kHasValue; + kBasicPerKeyPlacementCompStats.cpu_micros = kHasValue; + kBasicPerKeyPlacementCompStats.Add(kBasicPerLevelStats); + + kBasicFlushStats.micros = kHasValue; + kBasicFlushStats.cpu_micros = kHasValue; + kBasicFlushStats.bytes_written = kHasValue; + kBasicFlushStats.num_output_files = kHasValue; + } + + protected: + static constexpr uint8_t kHasValue = 1; + + InternalStats::CompactionStats kBasicCompStats; + InternalStats::CompactionStats kBasicPerKeyPlacementCompStats; + InternalStats::CompactionOutputsStats kBasicPerLevelStats; + InternalStats::CompactionStats kBasicFlushStats; + + std::atomic_bool enable_per_key_placement = true; + + void SetUp() override { + SyncPoint::GetInstance()->SetCallBack( + "Compaction::SupportsPerKeyPlacement:Enabled", [&](void* arg) { + auto supports_per_key_placement = static_cast(arg); + *supports_per_key_placement = enable_per_key_placement; + }); + SyncPoint::GetInstance()->EnableProcessing(); + } + + const std::vector& GetCompactionStats() { + VersionSet* const versions = dbfull()->GetVersionSet(); + assert(versions); + assert(versions->GetColumnFamilySet()); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + assert(cfd); + + const InternalStats* const internal_stats = cfd->internal_stats(); + assert(internal_stats); + + return internal_stats->TEST_GetCompactionStats(); + } + + const InternalStats::CompactionStats& GetPerKeyPlacementCompactionStats() { + VersionSet* const versions = dbfull()->GetVersionSet(); + assert(versions); + assert(versions->GetColumnFamilySet()); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + assert(cfd); + + const InternalStats* const internal_stats = cfd->internal_stats(); + assert(internal_stats); + + return internal_stats->TEST_GetPerKeyPlacementCompactionStats(); + } + + // Verify the compaction stats, the stats are roughly compared + void VerifyCompactionStats( + const std::vector& expect_stats, + const InternalStats::CompactionStats& expect_pl_stats) { + const std::vector& stats = + GetCompactionStats(); + const size_t kLevels = expect_stats.size(); + ASSERT_EQ(kLevels, stats.size()); + + for (auto it = stats.begin(), expect = expect_stats.begin(); + it != stats.end(); it++, expect++) { + VerifyCompactionStats(*it, *expect); + } + + const InternalStats::CompactionStats& pl_stats = + GetPerKeyPlacementCompactionStats(); + VerifyCompactionStats(pl_stats, expect_pl_stats); + } + + void ResetAllStats(std::vector& stats, + InternalStats::CompactionStats& pl_stats) { + ASSERT_OK(dbfull()->ResetStats()); + for (auto& level_stats : stats) { + level_stats.Clear(); + } + pl_stats.Clear(); + } + + // bottommost_temperature is renaming to last_level_temperature, set either + // of them should have the same effect. + void SetColdTemperature(Options& options) { + if (GetParam()) { + options.bottommost_temperature = Temperature::kCold; + } else { + options.last_level_temperature = Temperature::kCold; + } + } + + private: + void CompareStats(uint64_t val, uint64_t expect) { + if (expect > 0) { + ASSERT_TRUE(val > 0); + } else { + ASSERT_EQ(val, 0); + } + } + + void VerifyCompactionStats( + const InternalStats::CompactionStats& stats, + const InternalStats::CompactionStats& expect_stats) { + CompareStats(stats.micros, expect_stats.micros); + CompareStats(stats.cpu_micros, expect_stats.cpu_micros); + CompareStats(stats.bytes_read_non_output_levels, + expect_stats.bytes_read_non_output_levels); + CompareStats(stats.bytes_read_output_level, + expect_stats.bytes_read_output_level); + CompareStats(stats.bytes_read_blob, expect_stats.bytes_read_blob); + CompareStats(stats.bytes_written, expect_stats.bytes_written); + CompareStats(stats.bytes_moved, expect_stats.bytes_moved); + CompareStats(stats.num_input_files_in_non_output_levels, + expect_stats.num_input_files_in_non_output_levels); + CompareStats(stats.num_input_files_in_output_level, + expect_stats.num_input_files_in_output_level); + CompareStats(stats.num_output_files, expect_stats.num_output_files); + CompareStats(stats.num_output_files_blob, + expect_stats.num_output_files_blob); + CompareStats(stats.num_input_records, expect_stats.num_input_records); + CompareStats(stats.num_dropped_records, expect_stats.num_dropped_records); + CompareStats(stats.num_output_records, expect_stats.num_output_records); + ASSERT_EQ(stats.count, expect_stats.count); + for (int i = 0; i < static_cast(CompactionReason::kNumOfReasons); + i++) { + ASSERT_EQ(stats.counts[i], expect_stats.counts[i]); + } + } +}; + +TEST_P(TieredCompactionTest, SequenceBasedTieredStorageUniversal) { + const int kNumTrigger = 4; + const int kNumLevels = 7; + const int kNumKeys = 100; + const int kLastLevel = kNumLevels - 1; + + auto options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + SetColdTemperature(options); + options.level0_file_num_compaction_trigger = kNumTrigger; + options.statistics = CreateDBStatistics(); + options.max_subcompactions = 10; + DestroyAndReopen(options); + + std::atomic_uint64_t latest_cold_seq = 0; + std::vector seq_history; + + SyncPoint::GetInstance()->SetCallBack( + "CompactionIterator::PrepareOutput.context", [&](void* arg) { + auto context = static_cast(arg); + context->output_to_penultimate_level = + context->seq_num > latest_cold_seq; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + std::vector expect_stats(kNumLevels); + InternalStats::CompactionStats& last_stats = expect_stats[kLastLevel]; + InternalStats::CompactionStats expect_pl_stats; + + for (int i = 0; i < kNumTrigger; i++) { + for (int j = 0; j < kNumKeys; j++) { + ASSERT_OK(Put(Key(i * 10 + j), "value" + std::to_string(i))); + } + ASSERT_OK(Flush()); + seq_history.emplace_back(dbfull()->GetLatestSequenceNumber()); + expect_stats[0].Add(kBasicFlushStats); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // the penultimate level file temperature is not cold, all data are output to + // the penultimate level. + ASSERT_EQ("0,0,0,0,0,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0); + + // basic compaction stats are still counted to the last level + expect_stats[kLastLevel].Add(kBasicCompStats); + expect_pl_stats.Add(kBasicPerKeyPlacementCompStats); + + VerifyCompactionStats(expect_stats, expect_pl_stats); + + ResetAllStats(expect_stats, expect_pl_stats); + + // move forward the cold_seq to split the file into 2 levels, so should have + // both the last level stats and the output_to_penultimate_level stats + latest_cold_seq = seq_history[0]; + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel()); + + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + last_stats.Add(kBasicCompStats); + last_stats.ResetCompactionReason(CompactionReason::kManualCompaction); + last_stats.Add(kBasicPerLevelStats); + last_stats.num_dropped_records = 0; + expect_pl_stats.Add(kBasicPerKeyPlacementCompStats); + expect_pl_stats.ResetCompactionReason(CompactionReason::kManualCompaction); + VerifyCompactionStats(expect_stats, expect_pl_stats); + + // delete all cold data, so all data will be on penultimate level + for (int i = 0; i < 10; i++) { + ASSERT_OK(Delete(Key(i))); + } + ASSERT_OK(Flush()); + + ResetAllStats(expect_stats, expect_pl_stats); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0); + + last_stats.Add(kBasicCompStats); + last_stats.ResetCompactionReason(CompactionReason::kManualCompaction); + last_stats.bytes_read_output_level = kHasValue; + last_stats.num_input_files_in_output_level = kHasValue; + expect_pl_stats.Add(kBasicPerKeyPlacementCompStats); + expect_pl_stats.ResetCompactionReason(CompactionReason::kManualCompaction); + VerifyCompactionStats(expect_stats, expect_pl_stats); + + // move forward the cold_seq again with range delete, take a snapshot to keep + // the range dels in both cold and hot SSTs + auto snap = db_->GetSnapshot(); + latest_cold_seq = seq_history[2]; + std::string start = Key(25), end = Key(35); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), start, end)); + ASSERT_OK(Flush()); + + ResetAllStats(expect_stats, expect_pl_stats); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + last_stats.Add(kBasicCompStats); + last_stats.Add(kBasicPerLevelStats); + last_stats.ResetCompactionReason(CompactionReason::kManualCompaction); + expect_pl_stats.Add(kBasicPerKeyPlacementCompStats); + expect_pl_stats.ResetCompactionReason(CompactionReason::kManualCompaction); + VerifyCompactionStats(expect_stats, expect_pl_stats); + + // verify data + std::string value; + for (int i = 0; i < kNumKeys; i++) { + if (i < 10 || (i >= 25 && i < 35)) { + ASSERT_TRUE(db_->Get(ReadOptions(), Key(i), &value).IsNotFound()); + } else { + ASSERT_OK(db_->Get(ReadOptions(), Key(i), &value)); + } + } + + // range delete all hot data + start = Key(30); + end = Key(130); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), start, end)); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + // no range del is dropped because of snapshot + ASSERT_EQ( + options.statistics->getTickerCount(COMPACTION_RANGE_DEL_DROP_OBSOLETE), + 0); + + // release the snapshot and do compaction again should remove all hot data + db_->ReleaseSnapshot(snap); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel()); + ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + // 2 range dels are dropped + ASSERT_EQ( + options.statistics->getTickerCount(COMPACTION_RANGE_DEL_DROP_OBSOLETE), + 3); + + // move backward the cold_seq, for example the user may change the setting of + // hot/cold data, but it won't impact the existing cold data, as the sequence + // number is zeroed out. + latest_cold_seq = seq_history[1]; + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel()); + ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); +} + +TEST_P(TieredCompactionTest, RangeBasedTieredStorageUniversal) { + const int kNumTrigger = 4; + const int kNumLevels = 7; + const int kNumKeys = 100; + const int kLastLevel = kNumLevels - 1; + + auto options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + SetColdTemperature(options); + options.level0_file_num_compaction_trigger = kNumTrigger; + options.statistics = CreateDBStatistics(); + options.max_subcompactions = 10; + DestroyAndReopen(options); + auto cmp = options.comparator; + + port::Mutex mutex; + std::string hot_start = Key(10); + std::string hot_end = Key(50); + + SyncPoint::GetInstance()->SetCallBack( + "CompactionIterator::PrepareOutput.context", [&](void* arg) { + auto context = static_cast(arg); + MutexLock l(&mutex); + context->output_to_penultimate_level = + cmp->Compare(context->key, hot_start) >= 0 && + cmp->Compare(context->key, hot_end) < 0; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + std::vector expect_stats(kNumLevels); + InternalStats::CompactionStats& last_stats = expect_stats[kLastLevel]; + InternalStats::CompactionStats expect_pl_stats; + + for (int i = 0; i < kNumTrigger; i++) { + for (int j = 0; j < kNumKeys; j++) { + ASSERT_OK(Put(Key(j), "value" + std::to_string(j))); + } + ASSERT_OK(Flush()); + expect_stats[0].Add(kBasicFlushStats); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + last_stats.Add(kBasicCompStats); + last_stats.Add(kBasicPerLevelStats); + expect_pl_stats.Add(kBasicPerKeyPlacementCompStats); + VerifyCompactionStats(expect_stats, expect_pl_stats); + + ResetAllStats(expect_stats, expect_pl_stats); + + // change to all cold, no output_to_penultimate_level output + { + MutexLock l(&mutex); + hot_start = Key(100); + hot_end = Key(200); + } + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel()); + ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + last_stats.Add(kBasicCompStats); + last_stats.ResetCompactionReason(CompactionReason::kManualCompaction); + last_stats.Add(kBasicPerLevelStats); + last_stats.num_dropped_records = 0; + last_stats.bytes_read_output_level = kHasValue; + last_stats.num_input_files_in_output_level = kHasValue; + VerifyCompactionStats(expect_stats, expect_pl_stats); + + // change to all hot, universal compaction support moving data to up level if + // it's within compaction level range. + { + MutexLock l(&mutex); + hot_start = Key(0); + hot_end = Key(100); + } + + // No data is moved from cold tier to hot tier because no input files from L5 + // or higher, it's not safe to move data to output_to_penultimate_level level. + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,1", FilesPerLevel()); + + // Add 2 keys in higher level, but in separated files, all keys can be moved + // up if it's hot + ASSERT_OK(Put(Key(0), "value" + std::to_string(0))); + ASSERT_OK(Flush()); + ASSERT_OK(Put(Key(50), "value" + std::to_string(0))); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,1", FilesPerLevel()); + + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0); + + // change to only 1 key cold, to test compaction could stop even it matches + // size amp compaction threshold + { + MutexLock l(&mutex); + hot_start = Key(1); + hot_end = Key(1000); + } + + // generate files just enough to trigger compaction + for (int i = 0; i < kNumTrigger - 1; i++) { + for (int j = 0; j < 1000; j++) { + ASSERT_OK(Put(Key(j), "value" + std::to_string(j))); + } + ASSERT_OK(Flush()); + } + // make sure the compaction is able to finish + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + auto opts = db_->GetOptions(); + auto max_size_amp = + opts.compaction_options_universal.max_size_amplification_percent / 100; + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), + GetSstSizeHelper(Temperature::kCold) * max_size_amp); + + // delete all cold data + ASSERT_OK(Delete(Key(0))); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0); + + // range delete overlap with both hot/cold data, with a snapshot to make sure + // the range del is saved + auto snap = db_->GetSnapshot(); + { + MutexLock l(&mutex); + hot_start = Key(50); + hot_end = Key(100); + } + std::string start = Key(1), end = Key(70); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), start, end)); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + // no range del is dropped until snapshot is released + ASSERT_EQ( + options.statistics->getTickerCount(COMPACTION_RANGE_DEL_DROP_OBSOLETE), + 0); + + // verify data + std::string value; + for (int i = 0; i < kNumKeys; i++) { + if (i < 70) { + ASSERT_TRUE(db_->Get(ReadOptions(), Key(i), &value).IsNotFound()); + } else { + ASSERT_OK(db_->Get(ReadOptions(), Key(i), &value)); + } + } + + db_->ReleaseSnapshot(snap); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + // range del is dropped + ASSERT_EQ( + options.statistics->getTickerCount(COMPACTION_RANGE_DEL_DROP_OBSOLETE), + 1); +} + +TEST_P(TieredCompactionTest, LevelColdRangeDelete) { + const int kNumTrigger = 4; + const int kNumLevels = 7; + const int kNumKeys = 100; + const int kLastLevel = kNumLevels - 1; + + auto options = CurrentOptions(); + SetColdTemperature(options); + options.level0_file_num_compaction_trigger = kNumTrigger; + options.num_levels = kNumLevels; + options.statistics = CreateDBStatistics(); + options.max_subcompactions = 10; + DestroyAndReopen(options); + + std::atomic_uint64_t latest_cold_seq = 0; + + SyncPoint::GetInstance()->SetCallBack( + "CompactionIterator::PrepareOutput.context", [&](void* arg) { + auto context = static_cast(arg); + context->output_to_penultimate_level = + context->seq_num > latest_cold_seq; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(Put(Key(i), "value" + std::to_string(i))); + } + ASSERT_OK(Flush()); + + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,1", + FilesPerLevel()); // bottommost but not last level file is hot + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0); + + // explicitly move the data to the last level + MoveFilesToLevel(kLastLevel); + + ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel()); + + auto snap = db_->GetSnapshot(); + + std::string start = Key(10); + std::string end = Key(50); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), start, end)); + + // 20->30 will be marked as cold data, but it cannot be placed to cold tier + // (bottommost) otherwise, it will be "deleted" by the range del in + // output_to_penultimate_level level verify that these data will be able to + // queried + for (int i = 20; i < 30; i++) { + ASSERT_OK(Put(Key(i), "value" + std::to_string(i))); + } + // make the range tombstone and data after that cold + latest_cold_seq = dbfull()->GetLatestSequenceNumber(); + + // add home hot data, just for test + for (int i = 30; i < 40; i++) { + ASSERT_OK(Put(Key(i), "value" + std::to_string(i))); + } + + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + std::string value; + for (int i = 0; i < kNumKeys; i++) { + auto s = db_->Get(ReadOptions(), Key(i), &value); + if ((i >= 10 && i < 20) || (i >= 40 && i < 50)) { + ASSERT_TRUE(s.IsNotFound()); + } else { + ASSERT_OK(s); + } + } + + db_->ReleaseSnapshot(snap); +} + +// Test SST partitioner cut after every single key +class SingleKeySstPartitioner : public SstPartitioner { + public: + const char* Name() const override { return "SingleKeySstPartitioner"; } + + PartitionerResult ShouldPartition( + const PartitionerRequest& /*request*/) override { + return kRequired; + } + + bool CanDoTrivialMove(const Slice& /*smallest_user_key*/, + const Slice& /*largest_user_key*/) override { + return false; + } +}; + +class SingleKeySstPartitionerFactory : public SstPartitionerFactory { + public: + static const char* kClassName() { return "SingleKeySstPartitionerFactory"; } + const char* Name() const override { return kClassName(); } + + std::unique_ptr CreatePartitioner( + const SstPartitioner::Context& /* context */) const override { + return std::unique_ptr(new SingleKeySstPartitioner()); + } +}; + +TEST_P(TieredCompactionTest, LevelOutofBoundaryRangeDelete) { + const int kNumTrigger = 4; + const int kNumLevels = 3; + const int kNumKeys = 10; + + auto factory = std::make_shared(); + auto options = CurrentOptions(); + SetColdTemperature(options); + options.level0_file_num_compaction_trigger = kNumTrigger; + options.num_levels = kNumLevels; + options.statistics = CreateDBStatistics(); + options.sst_partitioner_factory = factory; + options.max_subcompactions = 10; + DestroyAndReopen(options); + + std::atomic_uint64_t latest_cold_seq = 0; + + SyncPoint::GetInstance()->SetCallBack( + "CompactionIterator::PrepareOutput.context", [&](void* arg) { + auto context = static_cast(arg); + context->output_to_penultimate_level = + context->seq_num > latest_cold_seq; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(Put(Key(i), "value" + std::to_string(i))); + } + ASSERT_OK(Flush()); + + MoveFilesToLevel(kNumLevels - 1); + ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + ASSERT_EQ("0,0,10", FilesPerLevel()); + + auto snap = db_->GetSnapshot(); + + // only range delete + std::string start = Key(3); + std::string end = Key(5); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), start, end)); + ASSERT_OK(Flush()); + + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + // range tombstone is not in cold tier + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + std::vector> level_to_files; + dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(), + &level_to_files); + // range tombstone is in the penultimate level + const int penultimate_level = kNumLevels - 2; + ASSERT_EQ(level_to_files[penultimate_level].size(), 1); + ASSERT_EQ(level_to_files[penultimate_level][0].num_entries, 1); + ASSERT_EQ(level_to_files[penultimate_level][0].num_deletions, 1); + ASSERT_EQ(level_to_files[penultimate_level][0].temperature, + Temperature::kUnknown); + + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + ASSERT_EQ("0,1,10", + FilesPerLevel()); // one file is at the penultimate level which + // only contains a range delete + + // Add 2 hot keys, each is a new SST, they will be placed in the same level as + // range del, but they don't have overlap with range del, make sure the range + // del will still be placed there + latest_cold_seq = dbfull()->GetLatestSequenceNumber(); + ASSERT_OK(Put(Key(0), "new value" + std::to_string(0))); + auto snap2 = db_->GetSnapshot(); + ASSERT_OK(Put(Key(6), "new value" + std::to_string(6))); + ASSERT_OK(Flush()); + + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,2,10", + FilesPerLevel()); // one file is at the penultimate level + // which only contains a range delete + std::vector live_file_meta; + db_->GetLiveFilesMetaData(&live_file_meta); + bool found_sst_with_del = false; + uint64_t sst_with_del_num = 0; + for (const auto& meta : live_file_meta) { + if (meta.num_deletions > 0) { + // found SST with del, which has 2 entries, one for data one for range del + ASSERT_EQ(meta.level, + kNumLevels - 2); // output to penultimate level + ASSERT_EQ(meta.num_entries, 2); + ASSERT_EQ(meta.num_deletions, 1); + found_sst_with_del = true; + sst_with_del_num = meta.file_number; + } + } + ASSERT_TRUE(found_sst_with_del); + + // release the first snapshot and compact, which should compact the range del + // but new inserted key `0` and `6` are still hot data which will be placed on + // the penultimate level + db_->ReleaseSnapshot(snap); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,2,7", FilesPerLevel()); + db_->GetLiveFilesMetaData(&live_file_meta); + found_sst_with_del = false; + for (const auto& meta : live_file_meta) { + // check new SST with del (the old one may not yet be deleted after + // compaction) + if (meta.num_deletions > 0 && meta.file_number != sst_with_del_num) { + found_sst_with_del = true; + } + } + ASSERT_FALSE(found_sst_with_del); + + // Now make all data cold, key 0 will be moved to the last level, but key 6 is + // still in snap2, so it will be kept at the penultimate level + latest_cold_seq = dbfull()->GetLatestSequenceNumber(); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,1,8", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + db_->ReleaseSnapshot(snap2); + + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,8", FilesPerLevel()); + ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); +} + +TEST_P(TieredCompactionTest, UniversalRangeDelete) { + const int kNumTrigger = 4; + const int kNumLevels = 7; + const int kNumKeys = 10; + + auto factory = std::make_shared(); + + auto options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + SetColdTemperature(options); + options.level0_file_num_compaction_trigger = kNumTrigger; + options.statistics = CreateDBStatistics(); + options.sst_partitioner_factory = factory; + options.max_subcompactions = 10; + DestroyAndReopen(options); + + std::atomic_uint64_t latest_cold_seq = 0; + + SyncPoint::GetInstance()->SetCallBack( + "CompactionIterator::PrepareOutput.context", [&](void* arg) { + auto context = static_cast(arg); + context->output_to_penultimate_level = + context->seq_num > latest_cold_seq; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(Put(Key(i), "value" + std::to_string(i))); + } + ASSERT_OK(Flush()); + + // compact to the penultimate level with 10 files + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + ASSERT_EQ("0,0,0,0,0,10", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0); + + // make all data cold + latest_cold_seq = dbfull()->GetLatestSequenceNumber(); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,0,10", FilesPerLevel()); + ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + // range del which considered as hot data, but it will be merged and deleted + // with the last level data + std::string start = Key(3); + std::string end = Key(5); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), start, end)); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + ASSERT_EQ("0,0,0,0,0,0,8", FilesPerLevel()); + + // range del with snapshot should be preserved in the penultimate level + auto snap = db_->GetSnapshot(); + + start = Key(6); + end = Key(8); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), start, end)); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,1,8", FilesPerLevel()); + + // Add 2 hot keys, each is a new SST, they will be placed in the same level as + // range del, but no overlap with range del. + latest_cold_seq = dbfull()->GetLatestSequenceNumber(); + ASSERT_OK(Put(Key(4), "new value" + std::to_string(0))); + auto snap2 = db_->GetSnapshot(); + ASSERT_OK(Put(Key(9), "new value" + std::to_string(6))); + + ASSERT_OK(Flush()); + + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,2,8", FilesPerLevel()); + // find the SST with range del + std::vector live_file_meta; + db_->GetLiveFilesMetaData(&live_file_meta); + bool found_sst_with_del = false; + uint64_t sst_with_del_num = 0; + for (const auto& meta : live_file_meta) { + if (meta.num_deletions > 0) { + // found SST with del, which has 2 entries, one for data one for range del + ASSERT_EQ(meta.level, + kNumLevels - 2); // output_to_penultimate_level level + ASSERT_EQ(meta.num_entries, 2); + ASSERT_EQ(meta.num_deletions, 1); + found_sst_with_del = true; + sst_with_del_num = meta.file_number; + } + } + ASSERT_TRUE(found_sst_with_del); + + // release the first snapshot which should compact the range del, but data on + // the same level is still hot + db_->ReleaseSnapshot(snap); + + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,2,6", FilesPerLevel()); + db_->GetLiveFilesMetaData(&live_file_meta); + // no range del should be found in SST + found_sst_with_del = false; + for (const auto& meta : live_file_meta) { + // check new SST with del (the old one may not yet be deleted after + // compaction) + if (meta.num_deletions > 0 && meta.file_number != sst_with_del_num) { + found_sst_with_del = true; + } + } + ASSERT_FALSE(found_sst_with_del); + + // make all data to cold, but key 6 is still protected by snap2 + latest_cold_seq = dbfull()->GetLatestSequenceNumber(); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,1,7", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + db_->ReleaseSnapshot(snap2); + + // release snapshot, everything go to bottommost + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,0,7", FilesPerLevel()); + ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); +} + +TEST_P(TieredCompactionTest, SequenceBasedTieredStorageLevel) { + const int kNumTrigger = 4; + const int kNumLevels = 7; + const int kNumKeys = 100; + const int kLastLevel = kNumLevels - 1; + + auto options = CurrentOptions(); + SetColdTemperature(options); + options.level0_file_num_compaction_trigger = kNumTrigger; + options.num_levels = kNumLevels; + options.statistics = CreateDBStatistics(); + options.max_subcompactions = 10; + DestroyAndReopen(options); + + std::atomic_uint64_t latest_cold_seq = 0; + std::vector seq_history; + + SyncPoint::GetInstance()->SetCallBack( + "CompactionIterator::PrepareOutput.context", [&](void* arg) { + auto context = static_cast(arg); + context->output_to_penultimate_level = + context->seq_num > latest_cold_seq; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + std::vector expect_stats(kNumLevels); + InternalStats::CompactionStats& last_stats = expect_stats[kLastLevel]; + InternalStats::CompactionStats expect_pl_stats; + + for (int i = 0; i < kNumTrigger; i++) { + for (int j = 0; j < kNumKeys; j++) { + ASSERT_OK(Put(Key(i * 10 + j), "value" + std::to_string(i))); + } + ASSERT_OK(Flush()); + expect_stats[0].Add(kBasicFlushStats); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // non last level is hot + ASSERT_EQ("0,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0); + + expect_stats[1].Add(kBasicCompStats); + expect_stats[1].Add(kBasicPerLevelStats); + expect_stats[1].ResetCompactionReason(CompactionReason::kLevelL0FilesNum); + VerifyCompactionStats(expect_stats, expect_pl_stats); + + // move all data to the last level + MoveFilesToLevel(kLastLevel); + + ResetAllStats(expect_stats, expect_pl_stats); + + // The compaction won't move the data up + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel()); + ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + last_stats.Add(kBasicCompStats); + last_stats.Add(kBasicPerLevelStats); + last_stats.num_dropped_records = 0; + last_stats.bytes_read_non_output_levels = 0; + last_stats.num_input_files_in_non_output_levels = 0; + last_stats.bytes_read_output_level = kHasValue; + last_stats.num_input_files_in_output_level = kHasValue; + last_stats.ResetCompactionReason(CompactionReason::kManualCompaction); + VerifyCompactionStats(expect_stats, expect_pl_stats); + + // Add new data, which is all hot and overriding all existing data + for (int i = 0; i < kNumTrigger; i++) { + for (int j = 0; j < kNumKeys; j++) { + ASSERT_OK(Put(Key(i * 10 + j), "value" + std::to_string(i))); + } + ASSERT_OK(Flush()); + seq_history.emplace_back(dbfull()->GetLatestSequenceNumber()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,1,0,0,0,0,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + ResetAllStats(expect_stats, expect_pl_stats); + + // after compaction, all data are hot + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0); + + for (int level = 2; level < kNumLevels - 1; level++) { + expect_stats[level].bytes_moved = kHasValue; + } + + last_stats.Add(kBasicCompStats); + last_stats.bytes_read_output_level = kHasValue; + last_stats.num_input_files_in_output_level = kHasValue; + last_stats.ResetCompactionReason(CompactionReason::kManualCompaction); + expect_pl_stats.Add(kBasicPerKeyPlacementCompStats); + expect_pl_stats.ResetCompactionReason(CompactionReason::kManualCompaction); + VerifyCompactionStats(expect_stats, expect_pl_stats); + + // move forward the cold_seq, try to split the data into cold and hot, but in + // this case it's unsafe to split the data + // because it's non-last-level but bottommost file, the sequence number will + // be zeroed out and lost the time information (with + // `level_compaction_dynamic_level_bytes` or Universal Compaction, it should + // be rare.) + // TODO(zjay): ideally we should avoid zero out non-last-level bottommost file + latest_cold_seq = seq_history[1]; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0); + + seq_history.clear(); + + // manually move all data (cold) to last level + MoveFilesToLevel(kLastLevel); + seq_history.clear(); + // Add new data once again + for (int i = 0; i < kNumTrigger; i++) { + for (int j = 0; j < kNumKeys; j++) { + ASSERT_OK(Put(Key(i * 10 + j), "value" + std::to_string(i))); + } + ASSERT_OK(Flush()); + seq_history.emplace_back(dbfull()->GetLatestSequenceNumber()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + latest_cold_seq = seq_history[0]; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + // delete all cold data + for (int i = 0; i < 10; i++) { + ASSERT_OK(Delete(Key(i))); + } + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0); + + latest_cold_seq = seq_history[2]; + + MoveFilesToLevel(kLastLevel); + + // move forward the cold_seq again with range delete, take a snapshot to keep + // the range dels in bottommost + auto snap = db_->GetSnapshot(); + + std::string start = Key(25), end = Key(35); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), start, end)); + // add one small key and large key in the input level, to make sure it's able + // to move hot data to input level within that range + ASSERT_OK(Put(Key(0), "value" + std::to_string(0))); + ASSERT_OK(Put(Key(100), "value" + std::to_string(0))); + + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + // verify data + std::string value; + for (int i = 1; i < 130; i++) { + if (i < 10 || (i >= 25 && i < 35)) { + ASSERT_TRUE(db_->Get(ReadOptions(), Key(i), &value).IsNotFound()); + } else { + ASSERT_OK(db_->Get(ReadOptions(), Key(i), &value)); + } + } + + // delete all hot data + ASSERT_OK(Delete(Key(0))); + start = Key(30); + end = Key(101); // range [101, 130] is cold, because it's not in input range + // in previous compaction + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), start, end)); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + // no range del is dropped because of snapshot + ASSERT_EQ( + options.statistics->getTickerCount(COMPACTION_RANGE_DEL_DROP_OBSOLETE), + 0); + + db_->ReleaseSnapshot(snap); + + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + // 3 range dels dropped, the first one is double counted as expected, which is + // spread into 2 SST files + ASSERT_EQ( + options.statistics->getTickerCount(COMPACTION_RANGE_DEL_DROP_OBSOLETE), + 3); + + // move backward of cold_seq, which might happen when the user change the + // setting. the hot data won't move up, just to make sure it still runs + // fine, which is because: + // 1. sequence number is zeroed out, so no time information + // 2. leveled compaction only support move data up within the higher level + // input range + latest_cold_seq = seq_history[1]; + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); +} + +TEST_P(TieredCompactionTest, RangeBasedTieredStorageLevel) { + const int kNumTrigger = 4; + const int kNumLevels = 7; + const int kNumKeys = 100; + + auto options = CurrentOptions(); + SetColdTemperature(options); + options.level0_file_num_compaction_trigger = kNumTrigger; + options.level_compaction_dynamic_level_bytes = true; + options.num_levels = kNumLevels; + options.statistics = CreateDBStatistics(); + options.max_subcompactions = 10; + options.preclude_last_level_data_seconds = 10000; + DestroyAndReopen(options); + auto cmp = options.comparator; + + port::Mutex mutex; + std::string hot_start = Key(10); + std::string hot_end = Key(50); + + SyncPoint::GetInstance()->SetCallBack( + "CompactionIterator::PrepareOutput.context", [&](void* arg) { + auto context = static_cast(arg); + MutexLock l(&mutex); + context->output_to_penultimate_level = + cmp->Compare(context->key, hot_start) >= 0 && + cmp->Compare(context->key, hot_end) < 0; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + for (int i = 0; i < kNumTrigger; i++) { + for (int j = 0; j < kNumKeys; j++) { + ASSERT_OK(Put(Key(j), "value" + std::to_string(j))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + // change to all cold + { + MutexLock l(&mutex); + hot_start = Key(100); + hot_end = Key(200); + } + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel()); + ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + // change to all hot, but level compaction only support move cold to hot + // within it's higher level input range. + { + MutexLock l(&mutex); + hot_start = Key(0); + hot_end = Key(100); + } + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel()); + ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + // with mixed hot/cold data + { + MutexLock l(&mutex); + hot_start = Key(50); + hot_end = Key(100); + } + ASSERT_OK(Put(Key(0), "value" + std::to_string(0))); + ASSERT_OK(Put(Key(100), "value" + std::to_string(100))); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + // delete all hot data, but with snapshot to keep the range del + auto snap = db_->GetSnapshot(); + std::string start = Key(50); + std::string end = Key(100); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), start, end)); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + // no range del is dropped because of snapshot + ASSERT_EQ( + options.statistics->getTickerCount(COMPACTION_RANGE_DEL_DROP_OBSOLETE), + 0); + + // release the snapshot and do compaction again should remove all hot data + db_->ReleaseSnapshot(snap); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel()); + ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + ASSERT_EQ( + options.statistics->getTickerCount(COMPACTION_RANGE_DEL_DROP_OBSOLETE), + 1); +} + +INSTANTIATE_TEST_CASE_P(TieredCompactionTest, TieredCompactionTest, + testing::Bool()); + +class PrecludeLastLevelTest : public DBTestBase { + public: + PrecludeLastLevelTest() + : DBTestBase("preclude_last_level_test", /*env_do_fsync=*/false) { + mock_clock_ = std::make_shared(env_->GetSystemClock()); + mock_env_ = std::make_unique(env_, mock_clock_); + } + + protected: + std::unique_ptr mock_env_; + std::shared_ptr mock_clock_; + + void SetUp() override { + mock_clock_->InstallTimedWaitFixCallback(); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::StartPeriodicTaskScheduler:Init", [&](void* arg) { + auto periodic_task_scheduler_ptr = + reinterpret_cast(arg); + periodic_task_scheduler_ptr->TEST_OverrideTimer(mock_clock_.get()); + }); + mock_clock_->SetCurrentTime(0); + } +}; + +TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeManualCompaction) { + const int kNumTrigger = 4; + const int kNumLevels = 7; + const int kNumKeys = 100; + const int kKeyPerSec = 10; + + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.preserve_internal_time_seconds = 10000; + options.env = mock_env_.get(); + options.level0_file_num_compaction_trigger = kNumTrigger; + options.num_levels = kNumLevels; + DestroyAndReopen(options); + + // pass some time first, otherwise the first a few keys write time are going + // to be zero, and internally zero has special meaning: kUnknownSeqnoTime + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); + + int sst_num = 0; + // Write files that are overlap and enough to trigger compaction + for (; sst_num < kNumTrigger; sst_num++) { + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun([&] { + mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); + }); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // all data is pushed to the last level + ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel()); + + // enable preclude feature + options.preclude_last_level_data_seconds = 10000; + options.last_level_temperature = Temperature::kCold; + Reopen(options); + + // all data is hot, even they're in the last level + ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + + // Generate a sstable and trigger manual compaction + ASSERT_OK(Put(Key(10), "value")); + ASSERT_OK(Flush()); + + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + // all data is moved up to the penultimate level + ASSERT_EQ("0,0,0,0,0,1", FilesPerLevel()); + ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + + // close explicitly, because the env is local variable which will be released + // first. + Close(); +} + +TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeAutoCompaction) { + const int kNumTrigger = 4; + const int kNumLevels = 7; + const int kNumKeys = 100; + const int kKeyPerSec = 10; + + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.preserve_internal_time_seconds = 10000; + options.env = mock_env_.get(); + options.level0_file_num_compaction_trigger = kNumTrigger; + options.num_levels = kNumLevels; + DestroyAndReopen(options); + + // pass some time first, otherwise the first a few keys write time are going + // to be zero, and internally zero has special meaning: kUnknownSeqnoTime + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); + + int sst_num = 0; + // Write files that are overlap and enough to trigger compaction + for (; sst_num < kNumTrigger; sst_num++) { + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun([&] { + mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); + }); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // all data is pushed to the last level + ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel()); + + // enable preclude feature + options.preclude_last_level_data_seconds = 10000; + options.last_level_temperature = Temperature::kCold; + // make sure it won't trigger Size Amp compaction, unlike normal Size Amp + // compaction which is typically a last level compaction, when tiered Storage + // ("preclude_last_level") is enabled, size amp won't include the last level. + // As the last level would be in cold tier and the size would not be a + // problem, which also avoid frequent hot to cold storage compaction. + options.compaction_options_universal.max_size_amplification_percent = 400; + Reopen(options); + + // all data is hot, even they're in the last level + ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + + // Write more data, but still all hot until the 10th SST, as: + // write a key every 10 seconds, 100 keys per SST, each SST takes 1000 seconds + // The preclude_last_level_data_seconds is 10k + Random rnd(301); + for (; sst_num < kNumTrigger * 2 - 1; sst_num++) { + for (int i = 0; i < kNumKeys; i++) { + // the value needs to be big enough to trigger full compaction + ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), rnd.RandomString(100))); + dbfull()->TEST_WaitForPeriodicTaskRun([&] { + mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); + }); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + + // all data is moved up to the penultimate level + ASSERT_EQ("0,0,0,0,0,1", FilesPerLevel()); + ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + + // close explicitly, because the env is local variable which will be released + // first. + Close(); +} + +TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimePartial) { + const int kNumTrigger = 4; + const int kNumLevels = 7; + const int kNumKeys = 100; + const int kKeyPerSec = 10; + + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.preserve_internal_time_seconds = 2000; + options.env = mock_env_.get(); + options.level0_file_num_compaction_trigger = kNumTrigger; + options.num_levels = kNumLevels; + DestroyAndReopen(options); + + // pass some time first, otherwise the first a few keys write time are going + // to be zero, and internally zero has special meaning: kUnknownSeqnoTime + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); + + int sst_num = 0; + // Write files that are overlap and enough to trigger compaction + for (; sst_num < kNumTrigger; sst_num++) { + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun([&] { + mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); + }); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // all data is pushed to the last level + ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel()); + + std::vector key_versions; + ASSERT_OK(GetAllKeyVersions(db_, Slice(), Slice(), + std::numeric_limits::max(), + &key_versions)); + + // make sure there're more than 300 keys and first 100 keys are having seqno + // zeroed out, the last 100 key seqno not zeroed out + ASSERT_GT(key_versions.size(), 300); + for (int i = 0; i < 100; i++) { + ASSERT_EQ(key_versions[i].sequence, 0); + } + auto rit = key_versions.rbegin(); + for (int i = 0; i < 100; i++) { + ASSERT_GT(rit->sequence, 0); + rit++; + } + + // enable preclude feature + options.preclude_last_level_data_seconds = 2000; + options.last_level_temperature = Temperature::kCold; + Reopen(options); + + // Generate a sstable and trigger manual compaction + ASSERT_OK(Put(Key(10), "value")); + ASSERT_OK(Flush()); + + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + // some data are moved up, some are not + ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + + Close(); +} + +TEST_F(PrecludeLastLevelTest, SmallPrecludeTime) { + const int kNumTrigger = 4; + const int kNumLevels = 7; + const int kNumKeys = 100; + + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.preclude_last_level_data_seconds = 60; + options.preserve_internal_time_seconds = 0; + options.env = mock_env_.get(); + options.level0_file_num_compaction_trigger = kNumTrigger; + options.num_levels = kNumLevels; + options.last_level_temperature = Temperature::kCold; + DestroyAndReopen(options); + + Random rnd(301); + + dbfull()->TEST_WaitForPeriodicTaskRun([&] { + mock_clock_->MockSleepForSeconds(static_cast(rnd.Uniform(10) + 1)); + }); + + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(100))); + dbfull()->TEST_WaitForPeriodicTaskRun([&] { + mock_clock_->MockSleepForSeconds(static_cast(rnd.Uniform(2))); + }); + } + ASSERT_OK(Flush()); + + TablePropertiesCollection tables_props; + ASSERT_OK(dbfull()->GetPropertiesOfAllTables(&tables_props)); + ASSERT_EQ(tables_props.size(), 1); + ASSERT_FALSE(tables_props.begin()->second->seqno_to_time_mapping.empty()); + SeqnoToTimeMapping tp_mapping; + ASSERT_OK( + tp_mapping.Add(tables_props.begin()->second->seqno_to_time_mapping)); + ASSERT_OK(tp_mapping.Sort()); + ASSERT_FALSE(tp_mapping.Empty()); + auto seqs = tp_mapping.TEST_GetInternalMapping(); + ASSERT_FALSE(seqs.empty()); + + // Wait more than preclude_last_level time, then make sure all the data is + // compacted to the last level even there's no write (no seqno -> time + // information was flushed to any SST). + mock_clock_->MockSleepForSeconds(100); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel()); + ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + Close(); +} + +TEST_F(PrecludeLastLevelTest, LastLevelOnlyCompactionPartial) { + const int kNumTrigger = 4; + const int kNumLevels = 7; + const int kNumKeys = 100; + const int kKeyPerSec = 10; + + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.preserve_internal_time_seconds = 2000; + options.env = mock_env_.get(); + options.level0_file_num_compaction_trigger = kNumTrigger; + options.num_levels = kNumLevels; + DestroyAndReopen(options); + + // pass some time first, otherwise the first a few keys write time are going + // to be zero, and internally zero has special meaning: kUnknownSeqnoTime + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); + + int sst_num = 0; + // Write files that are overlap and enough to trigger compaction + for (; sst_num < kNumTrigger; sst_num++) { + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun([&] { + mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); + }); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // all data is pushed to the last level + ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel()); + + // enable preclude feature + options.preclude_last_level_data_seconds = 2000; + options.last_level_temperature = Temperature::kCold; + Reopen(options); + + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + // some data are moved up, some are not + ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + + std::vector key_versions; + ASSERT_OK(GetAllKeyVersions(db_, Slice(), Slice(), + std::numeric_limits::max(), + &key_versions)); + + // make sure there're more than 300 keys and first 100 keys are having seqno + // zeroed out, the last 100 key seqno not zeroed out + ASSERT_GT(key_versions.size(), 300); + for (int i = 0; i < 100; i++) { + ASSERT_EQ(key_versions[i].sequence, 0); + } + auto rit = key_versions.rbegin(); + for (int i = 0; i < 100; i++) { + ASSERT_GT(rit->sequence, 0); + rit++; + } + + Close(); +} + +class PrecludeLastLevelTestWithParms + : public PrecludeLastLevelTest, + public testing::WithParamInterface { + public: + PrecludeLastLevelTestWithParms() : PrecludeLastLevelTest() {} +}; + +TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) { + const int kNumTrigger = 4; + const int kNumLevels = 7; + const int kNumKeys = 100; + const int kKeyPerSec = 10; + + bool enable_preclude_last_level = GetParam(); + + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.preserve_internal_time_seconds = 2000; + options.env = mock_env_.get(); + options.level0_file_num_compaction_trigger = kNumTrigger; + options.num_levels = kNumLevels; + DestroyAndReopen(options); + + // pass some time first, otherwise the first a few keys write time are going + // to be zero, and internally zero has special meaning: kUnknownSeqnoTime + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); + + Random rnd(301); + int sst_num = 0; + // Write files that are overlap and enough to trigger compaction + for (; sst_num < kNumTrigger; sst_num++) { + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), rnd.RandomString(100))); + dbfull()->TEST_WaitForPeriodicTaskRun([&] { + mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); + }); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // all data is pushed to the last level + ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel()); + + std::atomic_bool is_manual_compaction_running = false; + std::atomic_bool verified_compaction_order = false; + + // Make sure the manual compaction is in progress and try to trigger a + // SizeRatio compaction by flushing 4 files to L0. The compaction will try to + // compact 4 files at L0 to L5 (the last empty level). + // If the preclude_last_feature is enabled, the auto triggered compaction + // cannot be picked. Otherwise, the auto triggered compaction can run in + // parallel with the last level compaction. + // L0: [a] [b] [c] [d] + // L5: (locked if preclude_last_level is enabled) + // L6: [z] (locked: manual compaction in progress) + // TODO: in this case, L0 files should just be compacted to L4, so the 2 + // compactions won't be overlapped. + SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::ProcessKeyValueCompaction()::Processing", [&](void* arg) { + auto compaction = static_cast(arg); + if (compaction->is_manual_compaction()) { + is_manual_compaction_running = true; + TEST_SYNC_POINT( + "PrecludeLastLevelTest::LastLevelOnlyCompactionConflit:" + "ManualCompaction1"); + TEST_SYNC_POINT( + "PrecludeLastLevelTest::LastLevelOnlyCompactionConflit:" + "ManualCompaction2"); + is_manual_compaction_running = false; + } + }); + + SyncPoint::GetInstance()->SetCallBack( + "UniversalCompactionBuilder::PickCompaction:Return", [&](void* arg) { + auto compaction = static_cast(arg); + if (enable_preclude_last_level && is_manual_compaction_running) { + ASSERT_TRUE(compaction == nullptr); + verified_compaction_order = true; + } else { + ASSERT_TRUE(compaction != nullptr); + verified_compaction_order = true; + } + if (!compaction || !compaction->is_manual_compaction()) { + TEST_SYNC_POINT( + "PrecludeLastLevelTest::LastLevelOnlyCompactionConflit:" + "AutoCompactionPicked"); + } + }); + + SyncPoint::GetInstance()->LoadDependency({ + {"PrecludeLastLevelTest::LastLevelOnlyCompactionConflit:" + "ManualCompaction1", + "PrecludeLastLevelTest::LastLevelOnlyCompactionConflit:StartWrite"}, + {"PrecludeLastLevelTest::LastLevelOnlyCompactionConflit:" + "AutoCompactionPicked", + "PrecludeLastLevelTest::LastLevelOnlyCompactionConflit:" + "ManualCompaction2"}, + }); + + SyncPoint::GetInstance()->EnableProcessing(); + + // only enable if the Parameter is true + if (enable_preclude_last_level) { + options.preclude_last_level_data_seconds = 2000; + } + options.max_background_jobs = 8; + options.last_level_temperature = Temperature::kCold; + Reopen(options); + + auto manual_compaction_thread = port::Thread([this]() { + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + cro.exclusive_manual_compaction = false; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + }); + + TEST_SYNC_POINT( + "PrecludeLastLevelTest::LastLevelOnlyCompactionConflit:StartWrite"); + auto stop_token = + dbfull()->TEST_write_controler().GetCompactionPressureToken(); + + for (; sst_num < kNumTrigger * 2; sst_num++) { + for (int i = 0; i < kNumKeys; i++) { + // the value needs to be big enough to trigger full compaction + ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun([&] { + mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); + }); + } + ASSERT_OK(Flush()); + } + + manual_compaction_thread.join(); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + if (enable_preclude_last_level) { + ASSERT_NE("0,0,0,0,0,1,1", FilesPerLevel()); + } else { + ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel()); + } + ASSERT_TRUE(verified_compaction_order); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + stop_token.reset(); + + Close(); +} + +TEST_P(PrecludeLastLevelTestWithParms, PeriodicCompactionToPenultimateLevel) { + // Test the last level only periodic compaction should also be blocked by an + // ongoing compaction in penultimate level if tiered compaction is enabled + // otherwise, the periodic compaction should just run for the last level. + const int kNumTrigger = 4; + const int kNumLevels = 7; + const int kPenultimateLevel = kNumLevels - 2; + const int kKeyPerSec = 1; + const int kNumKeys = 100; + + bool enable_preclude_last_level = GetParam(); + + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.preserve_internal_time_seconds = 20000; + options.env = mock_env_.get(); + options.level0_file_num_compaction_trigger = kNumTrigger; + options.num_levels = kNumLevels; + options.ignore_max_compaction_bytes_for_input = false; + options.periodic_compaction_seconds = 10000; + DestroyAndReopen(options); + + Random rnd(301); + + for (int i = 0; i < 3 * kNumKeys; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(100))); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kKeyPerSec); }); + } + ASSERT_OK(Flush()); + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + // make sure all data is compacted to the last level + ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel()); + + // enable preclude feature + if (enable_preclude_last_level) { + options.preclude_last_level_data_seconds = 20000; + } + options.max_background_jobs = 8; + options.last_level_temperature = Temperature::kCold; + Reopen(options); + + std::atomic_bool is_size_ratio_compaction_running = false; + std::atomic_bool verified_last_level_compaction = false; + + SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::ProcessKeyValueCompaction()::Processing", [&](void* arg) { + auto compaction = static_cast(arg); + if (compaction->output_level() == kPenultimateLevel) { + is_size_ratio_compaction_running = true; + TEST_SYNC_POINT( + "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:" + "SizeRatioCompaction1"); + TEST_SYNC_POINT( + "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:" + "SizeRatioCompaction2"); + is_size_ratio_compaction_running = false; + } + }); + + SyncPoint::GetInstance()->SetCallBack( + "UniversalCompactionBuilder::PickCompaction:Return", [&](void* arg) { + auto compaction = static_cast(arg); + + if (is_size_ratio_compaction_running) { + if (enable_preclude_last_level) { + ASSERT_TRUE(compaction == nullptr); + } else { + ASSERT_TRUE(compaction != nullptr); + ASSERT_EQ(compaction->compaction_reason(), + CompactionReason::kPeriodicCompaction); + ASSERT_EQ(compaction->start_level(), kNumLevels - 1); + } + verified_last_level_compaction = true; + } + TEST_SYNC_POINT( + "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:" + "AutoCompactionPicked"); + }); + + SyncPoint::GetInstance()->LoadDependency({ + {"PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:" + "SizeRatioCompaction1", + "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:DoneWrite"}, + {"PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:" + "AutoCompactionPicked", + "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:" + "SizeRatioCompaction2"}, + }); + + auto stop_token = + dbfull()->TEST_write_controler().GetCompactionPressureToken(); + + for (int i = 0; i < kNumTrigger - 1; i++) { + for (int j = 0; j < kNumKeys; j++) { + ASSERT_OK(Put(Key(i * (kNumKeys - 1) + i), rnd.RandomString(10))); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kKeyPerSec); }); + } + ASSERT_OK(Flush()); + } + + TEST_SYNC_POINT( + "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:DoneWrite"); + + // wait for periodic compaction time and flush to trigger the periodic + // compaction, which should be blocked by ongoing compaction in the + // penultimate level + mock_clock_->MockSleepForSeconds(10000); + for (int i = 0; i < 3 * kNumKeys; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(10))); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kKeyPerSec); }); + } + ASSERT_OK(Flush()); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + stop_token.reset(); + + Close(); +} + +INSTANTIATE_TEST_CASE_P(PrecludeLastLevelTestWithParms, + PrecludeLastLevelTestWithParms, testing::Bool()); + +// partition the SST into 3 ranges [0, 19] [20, 39] [40, ...] +class ThreeRangesPartitioner : public SstPartitioner { + public: + const char* Name() const override { return "SingleKeySstPartitioner"; } + + PartitionerResult ShouldPartition( + const PartitionerRequest& request) override { + if ((cmp->CompareWithoutTimestamp(*request.current_user_key, + DBTestBase::Key(20)) >= 0 && + cmp->CompareWithoutTimestamp(*request.prev_user_key, + DBTestBase::Key(20)) < 0) || + (cmp->CompareWithoutTimestamp(*request.current_user_key, + DBTestBase::Key(40)) >= 0 && + cmp->CompareWithoutTimestamp(*request.prev_user_key, + DBTestBase::Key(40)) < 0)) { + return kRequired; + } else { + return kNotRequired; + } + } + + bool CanDoTrivialMove(const Slice& /*smallest_user_key*/, + const Slice& /*largest_user_key*/) override { + return false; + } + + const Comparator* cmp = BytewiseComparator(); +}; + +class ThreeRangesPartitionerFactory : public SstPartitionerFactory { + public: + static const char* kClassName() { + return "TombstoneTestSstPartitionerFactory"; + } + const char* Name() const override { return kClassName(); } + + std::unique_ptr CreatePartitioner( + const SstPartitioner::Context& /* context */) const override { + return std::unique_ptr(new ThreeRangesPartitioner()); + } +}; + +TEST_F(PrecludeLastLevelTest, PartialPenultimateLevelCompaction) { + const int kNumTrigger = 4; + const int kNumLevels = 7; + const int kKeyPerSec = 10; + + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.env = mock_env_.get(); + options.level0_file_num_compaction_trigger = kNumTrigger; + options.preserve_internal_time_seconds = 10000; + options.num_levels = kNumLevels; + DestroyAndReopen(options); + + // pass some time first, otherwise the first a few keys write time are going + // to be zero, and internally zero has special meaning: kUnknownSeqnoTime + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(10)); }); + + Random rnd(301); + + for (int i = 0; i < 300; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(100))); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kKeyPerSec); }); + } + ASSERT_OK(Flush()); + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + // make sure all data is compacted to the last level + ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel()); + + // Create 3 L5 files + auto factory = std::make_shared(); + options.sst_partitioner_factory = factory; + + Reopen(options); + + for (int i = 0; i < kNumTrigger - 1; i++) { + for (int j = 0; j < 100; j++) { + ASSERT_OK(Put(Key(i * 100 + j), rnd.RandomString(10))); + } + ASSERT_OK(Flush()); + } + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // L5: [0,19] [20,39] [40,299] + // L6: [0, 299] + ASSERT_EQ("0,0,0,0,0,3,1", FilesPerLevel()); + + // enable tiered storage feature + options.preclude_last_level_data_seconds = 10000; + options.last_level_temperature = Temperature::kCold; + options.statistics = CreateDBStatistics(); + Reopen(options); + + ColumnFamilyMetaData meta; + db_->GetColumnFamilyMetaData(&meta); + ASSERT_EQ(meta.levels[5].files.size(), 3); + ASSERT_EQ(meta.levels[6].files.size(), 1); + ASSERT_EQ(meta.levels[6].files[0].smallestkey, Key(0)); + ASSERT_EQ(meta.levels[6].files[0].largestkey, Key(299)); + + std::string file_path = meta.levels[5].files[1].db_path; + std::vector files; + // pick 3rd file @L5 + file@L6 for compaction + files.push_back(file_path + "/" + meta.levels[5].files[2].name); + files.push_back(file_path + "/" + meta.levels[6].files[0].name); + ASSERT_OK(db_->CompactFiles(CompactionOptions(), files, 6)); + + // The compaction only moved partial of the hot data to hot tier, range[0,39] + // is unsafe to move up, otherwise, they will be overlapped with the existing + // files@L5. + // The output should be: + // L5: [0,19] [20,39] [40,299] <-- Temperature::kUnknown + // L6: [0,19] [20,39] <-- Temperature::kCold + // L6 file is split because of the customized partitioner + ASSERT_EQ("0,0,0,0,0,3,2", FilesPerLevel()); + + // even all the data is hot, but not all data are moved to the hot tier + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + + db_->GetColumnFamilyMetaData(&meta); + ASSERT_EQ(meta.levels[5].files.size(), 3); + ASSERT_EQ(meta.levels[6].files.size(), 2); + for (const auto& file : meta.levels[5].files) { + ASSERT_EQ(file.temperature, Temperature::kUnknown); + } + for (const auto& file : meta.levels[6].files) { + ASSERT_EQ(file.temperature, Temperature::kCold); + } + ASSERT_EQ(meta.levels[6].files[0].smallestkey, Key(0)); + ASSERT_EQ(meta.levels[6].files[0].largestkey, Key(19)); + ASSERT_EQ(meta.levels[6].files[1].smallestkey, Key(20)); + ASSERT_EQ(meta.levels[6].files[1].largestkey, Key(39)); + + Close(); +} + +TEST_F(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) { + const int kNumLevels = 7; + const int kSecondsPerKey = 10; + const int kNumFiles = 3; + const int kValueBytes = 4 << 10; + const int kFileBytes = 4 * kValueBytes; + // `kNumKeysPerFile == 5` is determined by the current file cutting heuristics + // for this choice of `kValueBytes` and `kFileBytes`. + const int kNumKeysPerFile = 5; + const int kNumKeys = kNumFiles * kNumKeysPerFile; + + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.env = mock_env_.get(); + options.last_level_temperature = Temperature::kCold; + options.preserve_internal_time_seconds = 600; + options.preclude_last_level_data_seconds = 1; + options.num_levels = kNumLevels; + options.target_file_size_base = kFileBytes; + DestroyAndReopen(options); + + // pass some time first, otherwise the first a few keys write time are going + // to be zero, and internally zero has special meaning: kUnknownSeqnoTime + dbfull()->TEST_WaitForPeriodicTaskRun([&] { + mock_clock_->MockSleepForSeconds(static_cast(kSecondsPerKey)); + }); + + // Flush an L0 file with the following contents (new to old): + // + // Range deletions [4, 6) [7, 8) [9, 11) + // --- snap2 --- + // Key(0) .. Key(14) + // --- snap1 --- + // Key(3) .. Key(17) + const auto verify_db = [&]() { + for (int i = 0; i < kNumKeys; i++) { + std::string value; + auto s = db_->Get(ReadOptions(), Key(i), &value); + if (i == 4 || i == 5 || i == 7 || i == 9 || i == 10) { + ASSERT_TRUE(s.IsNotFound()); + } else { + ASSERT_OK(s); + } + } + }; + Random rnd(301); + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(Put(Key(i + 3), rnd.RandomString(kValueBytes))); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kSecondsPerKey); }); + } + auto* snap1 = db_->GetSnapshot(); + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(kValueBytes))); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kSecondsPerKey); }); + } + auto* snap2 = db_->GetSnapshot(); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(kNumKeysPerFile - 1), + Key(kNumKeysPerFile + 1))); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(kNumKeysPerFile + 2), + Key(kNumKeysPerFile + 3))); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(2 * kNumKeysPerFile - 1), + Key(2 * kNumKeysPerFile + 1))); + ASSERT_OK(Flush()); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kSecondsPerKey); }); + verify_db(); + + // Count compactions supporting per-key placement + std::atomic_int per_key_comp_num = 0; + SyncPoint::GetInstance()->SetCallBack( + "UniversalCompactionBuilder::PickCompaction:Return", [&](void* arg) { + auto compaction = static_cast(arg); + if (compaction->SupportsPerKeyPlacement()) { + ASSERT_EQ(compaction->GetPenultimateOutputRangeType(), + Compaction::PenultimateOutputRangeType::kNonLastRange); + per_key_comp_num++; + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + + // The `CompactRange()` writes the following files to L5. + // + // [key000000#16,kTypeValue, + // key000005#kMaxSequenceNumber,kTypeRangeDeletion] + // [key000005#21,kTypeValue, + // key000010#kMaxSequenceNumber,kTypeRangeDeletion] + // [key000010#26,kTypeValue, key000014#30,kTypeValue] + // + // And it writes the following files to L6. + // + // [key000003#1,kTypeValue, key000007#5,kTypeValue] + // [key000008#6,kTypeValue, key000012#10,kTypeValue] + // [key000013#11,kTypeValue, key000017#15,kTypeValue] + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,3,3", FilesPerLevel()); + verify_db(); + + // Rewrite the middle file only. File endpoints should not change. + std::string begin_key_buf = Key(kNumKeysPerFile + 1), + end_key_buf = Key(kNumKeysPerFile + 2); + Slice begin_key(begin_key_buf), end_key(end_key_buf); + ASSERT_OK(db_->SuggestCompactRange(db_->DefaultColumnFamily(), &begin_key, + &end_key)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,0,0,0,0,3,3", FilesPerLevel()); + ASSERT_EQ(1, per_key_comp_num); + verify_db(); + + // Rewrite the middle file again after releasing snap2. Still file endpoints + // should not change. + db_->ReleaseSnapshot(snap2); + ASSERT_OK(db_->SuggestCompactRange(db_->DefaultColumnFamily(), &begin_key, + &end_key)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,0,0,0,0,3,3", FilesPerLevel()); + ASSERT_EQ(2, per_key_comp_num); + verify_db(); + + // Middle file once more after releasing snap1. This time the data in the + // middle L5 file can all be compacted to the last level. + db_->ReleaseSnapshot(snap1); + ASSERT_OK(db_->SuggestCompactRange(db_->DefaultColumnFamily(), &begin_key, + &end_key)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,0,0,0,0,2,3", FilesPerLevel()); + ASSERT_EQ(3, per_key_comp_num); + verify_db(); + + // Finish off the penultimate level. + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,0,0,3", FilesPerLevel()); + verify_db(); + + Close(); +} + + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/comparator_db_test.cc b/librocksdb-sys/rocksdb/db/comparator_db_test.cc new file mode 100644 index 0000000..e5e3493 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/comparator_db_test.cc @@ -0,0 +1,678 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#include +#include +#include + +#include "memtable/stl_wrappers.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/hash.h" +#include "util/kv_map.h" +#include "util/random.h" +#include "util/string_util.h" +#include "utilities/merge_operators.h" + +namespace ROCKSDB_NAMESPACE { +namespace { + +static const Comparator* kTestComparator = nullptr; + +class KVIter : public Iterator { + public: + explicit KVIter(const stl_wrappers::KVMap* map) + : map_(map), iter_(map_->end()) {} + bool Valid() const override { return iter_ != map_->end(); } + void SeekToFirst() override { iter_ = map_->begin(); } + void SeekToLast() override { + if (map_->empty()) { + iter_ = map_->end(); + } else { + iter_ = map_->find(map_->rbegin()->first); + } + } + void Seek(const Slice& k) override { + iter_ = map_->lower_bound(k.ToString()); + } + void SeekForPrev(const Slice& k) override { + iter_ = map_->upper_bound(k.ToString()); + Prev(); + } + void Next() override { ++iter_; } + void Prev() override { + if (iter_ == map_->begin()) { + iter_ = map_->end(); + return; + } + --iter_; + } + + Slice key() const override { return iter_->first; } + Slice value() const override { return iter_->second; } + Status status() const override { return Status::OK(); } + + private: + const stl_wrappers::KVMap* const map_; + stl_wrappers::KVMap::const_iterator iter_; +}; + +void AssertItersEqual(Iterator* iter1, Iterator* iter2) { + ASSERT_EQ(iter1->Valid(), iter2->Valid()); + if (iter1->Valid()) { + ASSERT_EQ(iter1->key().ToString(), iter2->key().ToString()); + ASSERT_EQ(iter1->value().ToString(), iter2->value().ToString()); + } +} + +// Measuring operations on DB (expect to be empty). +// source_strings are candidate keys +void DoRandomIteraratorTest(DB* db, std::vector source_strings, + Random* rnd, int num_writes, int num_iter_ops, + int num_trigger_flush) { + stl_wrappers::KVMap map((stl_wrappers::LessOfComparator(kTestComparator))); + + for (int i = 0; i < num_writes; i++) { + if (num_trigger_flush > 0 && i != 0 && i % num_trigger_flush == 0) { + db->Flush(FlushOptions()); + } + + int type = rnd->Uniform(2); + int index = rnd->Uniform(static_cast(source_strings.size())); + auto& key = source_strings[index]; + switch (type) { + case 0: + // put + map[key] = key; + ASSERT_OK(db->Put(WriteOptions(), key, key)); + break; + case 1: + // delete + if (map.find(key) != map.end()) { + map.erase(key); + } + ASSERT_OK(db->Delete(WriteOptions(), key)); + break; + default: + assert(false); + } + } + + std::unique_ptr iter(db->NewIterator(ReadOptions())); + std::unique_ptr result_iter(new KVIter(&map)); + + bool is_valid = false; + for (int i = 0; i < num_iter_ops; i++) { + // Random walk and make sure iter and result_iter returns the + // same key and value + int type = rnd->Uniform(6); + ASSERT_OK(iter->status()); + switch (type) { + case 0: + // Seek to First + iter->SeekToFirst(); + result_iter->SeekToFirst(); + break; + case 1: + // Seek to last + iter->SeekToLast(); + result_iter->SeekToLast(); + break; + case 2: { + // Seek to random key + auto key_idx = rnd->Uniform(static_cast(source_strings.size())); + auto key = source_strings[key_idx]; + iter->Seek(key); + result_iter->Seek(key); + break; + } + case 3: + // Next + if (is_valid) { + iter->Next(); + result_iter->Next(); + } else { + continue; + } + break; + case 4: + // Prev + if (is_valid) { + iter->Prev(); + result_iter->Prev(); + } else { + continue; + } + break; + default: { + assert(type == 5); + auto key_idx = rnd->Uniform(static_cast(source_strings.size())); + auto key = source_strings[key_idx]; + std::string result; + auto status = db->Get(ReadOptions(), key, &result); + if (map.find(key) == map.end()) { + ASSERT_TRUE(status.IsNotFound()); + } else { + ASSERT_EQ(map[key], result); + } + break; + } + } + AssertItersEqual(iter.get(), result_iter.get()); + is_valid = iter->Valid(); + } +} + +class DoubleComparator : public Comparator { + public: + DoubleComparator() {} + + const char* Name() const override { return "DoubleComparator"; } + + int Compare(const Slice& a, const Slice& b) const override { +#ifndef CYGWIN + double da = std::stod(a.ToString()); + double db = std::stod(b.ToString()); +#else + double da = std::strtod(a.ToString().c_str(), 0 /* endptr */); + double db = std::strtod(a.ToString().c_str(), 0 /* endptr */); +#endif + if (da == db) { + return a.compare(b); + } else if (da > db) { + return 1; + } else { + return -1; + } + } + void FindShortestSeparator(std::string* /*start*/, + const Slice& /*limit*/) const override {} + + void FindShortSuccessor(std::string* /*key*/) const override {} +}; + +class HashComparator : public Comparator { + public: + HashComparator() {} + + const char* Name() const override { return "HashComparator"; } + + int Compare(const Slice& a, const Slice& b) const override { + uint32_t ha = Hash(a.data(), a.size(), 66); + uint32_t hb = Hash(b.data(), b.size(), 66); + if (ha == hb) { + return a.compare(b); + } else if (ha > hb) { + return 1; + } else { + return -1; + } + } + void FindShortestSeparator(std::string* /*start*/, + const Slice& /*limit*/) const override {} + + void FindShortSuccessor(std::string* /*key*/) const override {} +}; + +class TwoStrComparator : public Comparator { + public: + TwoStrComparator() {} + + const char* Name() const override { return "TwoStrComparator"; } + + int Compare(const Slice& a, const Slice& b) const override { + assert(a.size() >= 2); + assert(b.size() >= 2); + size_t size_a1 = static_cast(a[0]); + size_t size_b1 = static_cast(b[0]); + size_t size_a2 = static_cast(a[1]); + size_t size_b2 = static_cast(b[1]); + assert(size_a1 + size_a2 + 2 == a.size()); + assert(size_b1 + size_b2 + 2 == b.size()); + + Slice a1 = Slice(a.data() + 2, size_a1); + Slice b1 = Slice(b.data() + 2, size_b1); + Slice a2 = Slice(a.data() + 2 + size_a1, size_a2); + Slice b2 = Slice(b.data() + 2 + size_b1, size_b2); + + if (a1 != b1) { + return a1.compare(b1); + } + return a2.compare(b2); + } + void FindShortestSeparator(std::string* /*start*/, + const Slice& /*limit*/) const override {} + + void FindShortSuccessor(std::string* /*key*/) const override {} +}; +} // anonymous namespace + +class ComparatorDBTest + : public testing::Test, + virtual public ::testing::WithParamInterface { + private: + std::string dbname_; + Env* env_; + DB* db_; + Options last_options_; + std::unique_ptr comparator_guard; + + public: + ComparatorDBTest() : env_(Env::Default()), db_(nullptr) { + kTestComparator = BytewiseComparator(); + dbname_ = test::PerThreadDBPath("comparator_db_test"); + BlockBasedTableOptions toptions; + toptions.format_version = GetParam(); + last_options_.table_factory.reset( + ROCKSDB_NAMESPACE::NewBlockBasedTableFactory(toptions)); + EXPECT_OK(DestroyDB(dbname_, last_options_)); + } + + ~ComparatorDBTest() override { + delete db_; + EXPECT_OK(DestroyDB(dbname_, last_options_)); + kTestComparator = BytewiseComparator(); + } + + DB* GetDB() { return db_; } + + void SetOwnedComparator(const Comparator* cmp, bool owner = true) { + if (owner) { + comparator_guard.reset(cmp); + } else { + comparator_guard.reset(); + } + kTestComparator = cmp; + last_options_.comparator = cmp; + } + + // Return the current option configuration. + Options* GetOptions() { return &last_options_; } + + void DestroyAndReopen() { + // Destroy using last options + Destroy(); + ASSERT_OK(TryReopen()); + } + + void Destroy() { + delete db_; + db_ = nullptr; + ASSERT_OK(DestroyDB(dbname_, last_options_)); + } + + Status TryReopen() { + delete db_; + db_ = nullptr; + last_options_.create_if_missing = true; + + return DB::Open(last_options_, dbname_, &db_); + } +}; + +INSTANTIATE_TEST_CASE_P(FormatDef, ComparatorDBTest, + testing::Values(test::kDefaultFormatVersion)); +INSTANTIATE_TEST_CASE_P(FormatLatest, ComparatorDBTest, + testing::Values(kLatestFormatVersion)); + +TEST_P(ComparatorDBTest, Bytewise) { + for (int rand_seed = 301; rand_seed < 306; rand_seed++) { + DestroyAndReopen(); + Random rnd(rand_seed); + DoRandomIteraratorTest(GetDB(), + {"a", "b", "c", "d", "e", "f", "g", "h", "i"}, &rnd, + 8, 100, 3); + } +} + +TEST_P(ComparatorDBTest, SimpleSuffixReverseComparator) { + SetOwnedComparator(new test::SimpleSuffixReverseComparator()); + + for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) { + Options* opt = GetOptions(); + opt->comparator = kTestComparator; + DestroyAndReopen(); + Random rnd(rnd_seed); + + std::vector source_strings; + std::vector source_prefixes; + // Randomly generate 5 prefixes + for (int i = 0; i < 5; i++) { + source_prefixes.push_back(rnd.HumanReadableString(8)); + } + for (int j = 0; j < 20; j++) { + int prefix_index = rnd.Uniform(static_cast(source_prefixes.size())); + std::string key = source_prefixes[prefix_index] + + rnd.HumanReadableString(rnd.Uniform(8)); + source_strings.push_back(key); + } + + DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 30, 600, 66); + } +} + +TEST_P(ComparatorDBTest, Uint64Comparator) { + SetOwnedComparator(test::Uint64Comparator(), false /* owner */); + + for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) { + Options* opt = GetOptions(); + opt->comparator = kTestComparator; + DestroyAndReopen(); + Random rnd(rnd_seed); + Random64 rnd64(rnd_seed); + + std::vector source_strings; + // Randomly generate source keys + for (int i = 0; i < 100; i++) { + uint64_t r = rnd64.Next(); + std::string str; + str.resize(8); + memcpy(&str[0], static_cast(&r), 8); + source_strings.push_back(str); + } + + DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66); + } +} + +TEST_P(ComparatorDBTest, DoubleComparator) { + SetOwnedComparator(new DoubleComparator()); + + for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) { + Options* opt = GetOptions(); + opt->comparator = kTestComparator; + DestroyAndReopen(); + Random rnd(rnd_seed); + + std::vector source_strings; + // Randomly generate source keys + for (int i = 0; i < 100; i++) { + uint32_t r = rnd.Next(); + uint32_t divide_order = rnd.Uniform(8); + double to_divide = 1.0; + for (uint32_t j = 0; j < divide_order; j++) { + to_divide *= 10.0; + } + source_strings.push_back(std::to_string(r / to_divide)); + } + + DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66); + } +} + +TEST_P(ComparatorDBTest, HashComparator) { + SetOwnedComparator(new HashComparator()); + + for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) { + Options* opt = GetOptions(); + opt->comparator = kTestComparator; + DestroyAndReopen(); + Random rnd(rnd_seed); + + std::vector source_strings; + // Randomly generate source keys + for (int i = 0; i < 100; i++) { + source_strings.push_back(test::RandomKey(&rnd, 8)); + } + + DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66); + } +} + +TEST_P(ComparatorDBTest, TwoStrComparator) { + SetOwnedComparator(new TwoStrComparator()); + + for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) { + Options* opt = GetOptions(); + opt->comparator = kTestComparator; + DestroyAndReopen(); + Random rnd(rnd_seed); + + std::vector source_strings; + // Randomly generate source keys + for (int i = 0; i < 100; i++) { + std::string str; + uint32_t size1 = rnd.Uniform(8); + uint32_t size2 = rnd.Uniform(8); + str.append(1, static_cast(size1)); + str.append(1, static_cast(size2)); + str.append(test::RandomKey(&rnd, size1)); + str.append(test::RandomKey(&rnd, size2)); + source_strings.push_back(str); + } + + DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66); + } +} + +namespace { +void VerifyNotSuccessor(const Slice& s, const Slice& t) { + auto bc = BytewiseComparator(); + auto rbc = ReverseBytewiseComparator(); + ASSERT_FALSE(bc->IsSameLengthImmediateSuccessor(s, t)); + ASSERT_FALSE(rbc->IsSameLengthImmediateSuccessor(s, t)); + ASSERT_FALSE(bc->IsSameLengthImmediateSuccessor(t, s)); + ASSERT_FALSE(rbc->IsSameLengthImmediateSuccessor(t, s)); +} + +void VerifySuccessor(const Slice& s, const Slice& t) { + auto bc = BytewiseComparator(); + auto rbc = ReverseBytewiseComparator(); + ASSERT_TRUE(bc->IsSameLengthImmediateSuccessor(s, t)); + ASSERT_FALSE(rbc->IsSameLengthImmediateSuccessor(s, t)); + ASSERT_FALSE(bc->IsSameLengthImmediateSuccessor(t, s)); + // Should be true but that increases exposure to a design bug in + // auto_prefix_mode, so currently set to FALSE + ASSERT_FALSE(rbc->IsSameLengthImmediateSuccessor(t, s)); +} + +} // anonymous namespace + +TEST_P(ComparatorDBTest, IsSameLengthImmediateSuccessor) { + { + // different length + Slice s("abcxy"); + Slice t("abcxyz"); + VerifyNotSuccessor(s, t); + } + { + Slice s("abcxyz"); + Slice t("abcxy"); + VerifyNotSuccessor(s, t); + } + { + // not last byte different + Slice s("abc1xyz"); + Slice t("abc2xyz"); + VerifyNotSuccessor(s, t); + } + { + // same string + Slice s("abcxyz"); + Slice t("abcxyz"); + VerifyNotSuccessor(s, t); + } + { + Slice s("abcxy"); + Slice t("abcxz"); + VerifySuccessor(s, t); + } + { + const char s_array[] = "\x50\x8a\xac"; + const char t_array[] = "\x50\x8a\xad"; + Slice s(s_array); + Slice t(t_array); + VerifySuccessor(s, t); + } + { + const char s_array[] = "\x50\x8a\xff"; + const char t_array[] = "\x50\x8b\x00"; + Slice s(s_array, 3); + Slice t(t_array, 3); + VerifySuccessor(s, t); + } + { + const char s_array[] = "\x50\x8a\xff\xff"; + const char t_array[] = "\x50\x8b\x00\x00"; + Slice s(s_array, 4); + Slice t(t_array, 4); + VerifySuccessor(s, t); + } + { + const char s_array[] = "\x50\x8a\xff\xff"; + const char t_array[] = "\x50\x8b\x00\x01"; + Slice s(s_array, 4); + Slice t(t_array, 4); + VerifyNotSuccessor(s, t); + } +} + +TEST_P(ComparatorDBTest, FindShortestSeparator) { + std::string s1 = "abc1xyz"; + std::string s2 = "abc3xy"; + + BytewiseComparator()->FindShortestSeparator(&s1, s2); + ASSERT_EQ("abc2", s1); + + s1 = "abc5xyztt"; + + ReverseBytewiseComparator()->FindShortestSeparator(&s1, s2); + ASSERT_EQ("abc5", s1); + + s1 = "abc3"; + s2 = "abc2xy"; + ReverseBytewiseComparator()->FindShortestSeparator(&s1, s2); + ASSERT_EQ("abc3", s1); + + s1 = "abc3xyz"; + s2 = "abc2xy"; + ReverseBytewiseComparator()->FindShortestSeparator(&s1, s2); + ASSERT_EQ("abc3", s1); + + s1 = "abc3xyz"; + s2 = "abc2"; + ReverseBytewiseComparator()->FindShortestSeparator(&s1, s2); + ASSERT_EQ("abc3", s1); + + std::string old_s1 = s1 = "abc2xy"; + s2 = "abc2"; + ReverseBytewiseComparator()->FindShortestSeparator(&s1, s2); + ASSERT_TRUE(old_s1 >= s1); + ASSERT_TRUE(s1 > s2); +} + +TEST_P(ComparatorDBTest, SeparatorSuccessorRandomizeTest) { + // Char list for boundary cases. + std::array char_list{{0, 1, 2, 253, 254, 255}}; + Random rnd(301); + + for (int attempts = 0; attempts < 1000; attempts++) { + uint32_t size1 = rnd.Skewed(4); + uint32_t size2; + + if (rnd.OneIn(2)) { + // size2 to be random size + size2 = rnd.Skewed(4); + } else { + // size1 is within [-2, +2] of size1 + int diff = static_cast(rnd.Uniform(5)) - 2; + int tmp_size2 = static_cast(size1) + diff; + if (tmp_size2 < 0) { + tmp_size2 = 0; + } + size2 = static_cast(tmp_size2); + } + + std::string s1; + std::string s2; + for (uint32_t i = 0; i < size1; i++) { + if (rnd.OneIn(2)) { + // Use random byte + s1 += static_cast(rnd.Uniform(256)); + } else { + // Use one byte in char_list + char c = static_cast(char_list[rnd.Uniform(sizeof(char_list))]); + s1 += c; + } + } + + // First set s2 to be the same as s1, and then modify s2. + s2 = s1; + s2.resize(size2); + // We start from the back of the string + if (size2 > 0) { + uint32_t pos = size2 - 1; + do { + if (pos >= size1 || rnd.OneIn(4)) { + // For 1/4 chance, use random byte + s2[pos] = static_cast(rnd.Uniform(256)); + } else if (rnd.OneIn(4)) { + // In 1/4 chance, stop here. + break; + } else { + // Create a char within [-2, +2] of the matching char of s1. + int diff = static_cast(rnd.Uniform(5)) - 2; + // char may be signed or unsigned based on platform. + int s1_char = static_cast(static_cast(s1[pos])); + int s2_char = s1_char + diff; + if (s2_char < 0) { + s2_char = 0; + } + if (s2_char > 255) { + s2_char = 255; + } + s2[pos] = static_cast(s2_char); + } + } while (pos-- != 0); + } + + // Test separators + for (int rev = 0; rev < 2; rev++) { + if (rev == 1) { + // switch s1 and s2 + std::string t = s1; + s1 = s2; + s2 = t; + } + std::string separator = s1; + BytewiseComparator()->FindShortestSeparator(&separator, s2); + std::string rev_separator = s1; + ReverseBytewiseComparator()->FindShortestSeparator(&rev_separator, s2); + + if (s1 == s2) { + ASSERT_EQ(s1, separator); + ASSERT_EQ(s2, rev_separator); + } else if (s1 < s2) { + ASSERT_TRUE(s1 <= separator); + ASSERT_TRUE(s2 > separator); + ASSERT_LE(separator.size(), std::max(s1.size(), s2.size())); + ASSERT_EQ(s1, rev_separator); + } else { + ASSERT_TRUE(s1 >= rev_separator); + ASSERT_TRUE(s2 < rev_separator); + ASSERT_LE(rev_separator.size(), std::max(s1.size(), s2.size())); + ASSERT_EQ(s1, separator); + } + } + + // Test successors + std::string succ = s1; + BytewiseComparator()->FindShortSuccessor(&succ); + ASSERT_TRUE(succ >= s1); + + succ = s1; + ReverseBytewiseComparator()->FindShortSuccessor(&succ); + ASSERT_TRUE(succ <= s1); + } +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/convenience.cc b/librocksdb-sys/rocksdb/db/convenience.cc new file mode 100644 index 0000000..32cdfaf --- /dev/null +++ b/librocksdb-sys/rocksdb/db/convenience.cc @@ -0,0 +1,81 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + + +#include "rocksdb/convenience.h" + +#include "db/db_impl/db_impl.h" +#include "util/cast_util.h" + +namespace ROCKSDB_NAMESPACE { + +void CancelAllBackgroundWork(DB* db, bool wait) { + (static_cast_with_check(db->GetRootDB())) + ->CancelAllBackgroundWork(wait); +} + +Status DeleteFilesInRange(DB* db, ColumnFamilyHandle* column_family, + const Slice* begin, const Slice* end, + bool include_end) { + RangePtr range(begin, end); + return DeleteFilesInRanges(db, column_family, &range, 1, include_end); +} + +Status DeleteFilesInRanges(DB* db, ColumnFamilyHandle* column_family, + const RangePtr* ranges, size_t n, bool include_end) { + return (static_cast_with_check(db->GetRootDB())) + ->DeleteFilesInRanges(column_family, ranges, n, include_end); +} + +Status VerifySstFileChecksum(const Options& options, + const EnvOptions& env_options, + const std::string& file_path) { + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + return VerifySstFileChecksum(options, env_options, read_options, file_path); +} +Status VerifySstFileChecksum(const Options& options, + const EnvOptions& env_options, + const ReadOptions& read_options, + const std::string& file_path, + const SequenceNumber& largest_seqno) { + std::unique_ptr file; + uint64_t file_size; + InternalKeyComparator internal_comparator(options.comparator); + ImmutableOptions ioptions(options); + + Status s = ioptions.fs->NewRandomAccessFile( + file_path, FileOptions(env_options), &file, nullptr); + if (s.ok()) { + s = ioptions.fs->GetFileSize(file_path, IOOptions(), &file_size, nullptr); + } else { + return s; + } + std::unique_ptr table_reader; + std::unique_ptr file_reader( + new RandomAccessFileReader( + std::move(file), file_path, ioptions.clock, nullptr /* io_tracer */, + ioptions.stats /* stats */, + Histograms::SST_READ_MICROS /* hist_type */, + nullptr /* file_read_hist */, ioptions.rate_limiter.get())); + const bool kImmortal = true; + auto reader_options = TableReaderOptions( + ioptions, options.prefix_extractor, env_options, internal_comparator, + options.block_protection_bytes_per_key, false /* skip_filters */, + !kImmortal, false /* force_direct_prefetch */, -1 /* level */); + reader_options.largest_seqno = largest_seqno; + s = ioptions.table_factory->NewTableReader( + reader_options, std::move(file_reader), file_size, &table_reader, + false /* prefetch_index_and_filter_in_cache */); + if (!s.ok()) { + return s; + } + s = table_reader->VerifyChecksum(read_options, + TableReaderCaller::kUserVerifyChecksum); + return s; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/corruption_test.cc b/librocksdb-sys/rocksdb/db/corruption_test.cc new file mode 100644 index 0000000..d1cb022 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/corruption_test.cc @@ -0,0 +1,1687 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "rocksdb/options.h" + +#include +#include +#include + +#include + +#include "db/db_impl/db_impl.h" +#include "db/db_test_util.h" +#include "db/log_format.h" +#include "db/version_set.h" +#include "file/filename.h" +#include "port/stack_trace.h" +#include "rocksdb/cache.h" +#include "rocksdb/convenience.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/table.h" +#include "rocksdb/utilities/transaction_db.h" +#include "rocksdb/write_batch.h" +#include "table/block_based/block_based_table_builder.h" +#include "table/meta_blocks.h" +#include "table/mock_table.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/cast_util.h" +#include "util/random.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +static constexpr int kValueSize = 1000; +namespace { +// A wrapper that allows injection of errors. +class ErrorFS : public FileSystemWrapper { + public: + bool writable_file_error_; + int num_writable_file_errors_; + + explicit ErrorFS(const std::shared_ptr& _target) + : FileSystemWrapper(_target), + writable_file_error_(false), + num_writable_file_errors_(0) {} + const char* Name() const override { return "ErrorEnv"; } + + virtual IOStatus NewWritableFile(const std::string& fname, + const FileOptions& opts, + std::unique_ptr* result, + IODebugContext* dbg) override { + result->reset(); + if (writable_file_error_) { + ++num_writable_file_errors_; + return IOStatus::IOError(fname, "fake error"); + } + return target()->NewWritableFile(fname, opts, result, dbg); + } +}; +} // anonymous namespace +class CorruptionTest : public testing::Test { + public: + std::shared_ptr env_guard_; + std::shared_ptr fs_; + std::unique_ptr env_; + Env* base_env_; + std::string dbname_; + std::shared_ptr tiny_cache_; + Options options_; + DB* db_; + + CorruptionTest() { + // If LRU cache shard bit is smaller than 2 (or -1 which will automatically + // set it to 0), test SequenceNumberRecovery will fail, likely because of a + // bug in recovery code. Keep it 4 for now to make the test passes. + tiny_cache_ = NewLRUCache(100, 4); + base_env_ = Env::Default(); + EXPECT_OK( + test::CreateEnvFromSystem(ConfigOptions(), &base_env_, &env_guard_)); + EXPECT_NE(base_env_, nullptr); + fs_.reset(new ErrorFS(base_env_->GetFileSystem())); + env_ = NewCompositeEnv(fs_); + options_.wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords; + options_.env = env_.get(); + dbname_ = test::PerThreadDBPath(env_.get(), "corruption_test"); + Status s = DestroyDB(dbname_, options_); + EXPECT_OK(s); + + db_ = nullptr; + options_.create_if_missing = true; + BlockBasedTableOptions table_options; + table_options.block_size_deviation = 0; // make unit test pass for now + options_.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(); + options_.create_if_missing = false; + } + + ~CorruptionTest() override { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->LoadDependency({}); + SyncPoint::GetInstance()->ClearAllCallBacks(); + delete db_; + db_ = nullptr; + if (getenv("KEEP_DB")) { + fprintf(stdout, "db is still at %s\n", dbname_.c_str()); + } else { + Options opts; + opts.env = base_env_; + EXPECT_OK(DestroyDB(dbname_, opts)); + } + } + + void CloseDb() { + delete db_; + db_ = nullptr; + } + + Status TryReopen(Options* options = nullptr) { + delete db_; + db_ = nullptr; + Options opt = (options ? *options : options_); + if (opt.env == Options().env) { + // If env is not overridden, replace it with ErrorEnv. + // Otherwise, the test already uses a non-default Env. + opt.env = env_.get(); + } + opt.arena_block_size = 4096; + BlockBasedTableOptions table_options; + table_options.block_cache = tiny_cache_; + table_options.block_size_deviation = 0; + opt.table_factory.reset(NewBlockBasedTableFactory(table_options)); + return DB::Open(opt, dbname_, &db_); + } + + void Reopen(Options* options = nullptr) { ASSERT_OK(TryReopen(options)); } + + void RepairDB() { + delete db_; + db_ = nullptr; + ASSERT_OK(::ROCKSDB_NAMESPACE::RepairDB(dbname_, options_)); + } + + void Build(int n, int start, int flush_every) { + std::string key_space, value_space; + WriteBatch batch; + for (int i = 0; i < n; i++) { + if (flush_every != 0 && i != 0 && i % flush_every == 0) { + DBImpl* dbi = static_cast_with_check(db_); + ASSERT_OK(dbi->TEST_FlushMemTable()); + } + // if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n); + Slice key = Key(i + start, &key_space); + batch.Clear(); + ASSERT_OK(batch.Put(key, Value(i + start, &value_space))); + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + } + } + + void Build(int n, int flush_every = 0) { Build(n, 0, flush_every); } + + void Check(int min_expected, int max_expected) { + Check(min_expected, max_expected, ReadOptions(false, true)); + } + + void Check(int min_expected, int max_expected, ReadOptions read_options) { + uint64_t next_expected = 0; + uint64_t missed = 0; + int bad_keys = 0; + int bad_values = 0; + int correct = 0; + std::string value_space; + // Do not verify checksums. If we verify checksums then the + // db itself will raise errors because data is corrupted. + // Instead, we want the reads to be successful and this test + // will detect whether the appropriate corruptions have + // occurred. + Iterator* iter = db_->NewIterator(read_options); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_OK(iter->status()); + uint64_t key; + Slice in(iter->key()); + if (!ConsumeDecimalNumber(&in, &key) || !in.empty() || + key < next_expected) { + bad_keys++; + continue; + } + missed += (key - next_expected); + next_expected = key + 1; + if (iter->value() != Value(static_cast(key), &value_space)) { + bad_values++; + } else { + correct++; + } + } + iter->status().PermitUncheckedError(); + delete iter; + + fprintf( + stderr, + "expected=%d..%d; got=%d; bad_keys=%d; bad_values=%d; missed=%llu\n", + min_expected, max_expected, correct, bad_keys, bad_values, + static_cast(missed)); + ASSERT_LE(min_expected, correct); + ASSERT_GE(max_expected, correct); + } + + void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) { + // Pick file to corrupt + std::vector filenames; + ASSERT_OK(env_->GetChildren(dbname_, &filenames)); + uint64_t number; + FileType type; + std::string fname; + int picked_number = -1; + for (size_t i = 0; i < filenames.size(); i++) { + if (ParseFileName(filenames[i], &number, &type) && type == filetype && + static_cast(number) > picked_number) { // Pick latest file + fname = dbname_ + "/" + filenames[i]; + picked_number = static_cast(number); + } + } + ASSERT_TRUE(!fname.empty()) << filetype; + + ASSERT_OK(test::CorruptFile(env_.get(), fname, offset, bytes_to_corrupt, + /*verify_checksum*/ filetype == kTableFile)); + } + + // corrupts exactly one file at level `level`. if no file found at level, + // asserts + void CorruptTableFileAtLevel(int level, int offset, int bytes_to_corrupt) { + std::vector metadata; + db_->GetLiveFilesMetaData(&metadata); + for (const auto& m : metadata) { + if (m.level == level) { + ASSERT_OK(test::CorruptFile(env_.get(), dbname_ + "/" + m.name, offset, + bytes_to_corrupt)); + return; + } + } + FAIL() << "no file found at level"; + } + + int Property(const std::string& name) { + std::string property; + int result; + if (db_->GetProperty(name, &property) && + sscanf(property.c_str(), "%d", &result) == 1) { + return result; + } else { + return -1; + } + } + + // Return the ith key + Slice Key(int i, std::string* storage) { + char buf[100]; + snprintf(buf, sizeof(buf), "%016d", i); + storage->assign(buf, strlen(buf)); + return Slice(*storage); + } + + // Return the value to associate with the specified key + Slice Value(int k, std::string* storage) { + if (k == 0) { + // Ugh. Random seed of 0 used to produce no entropy. This code + // preserves the implementation that was in place when all of the + // magic values in this file were picked. + *storage = std::string(kValueSize, ' '); + } else { + Random r(k); + *storage = r.RandomString(kValueSize); + } + return Slice(*storage); + } + + void GetSortedWalFiles(std::vector& file_nums) { + std::vector tmp_files; + ASSERT_OK(env_->GetChildren(dbname_, &tmp_files)); + FileType type = kWalFile; + for (const auto& file : tmp_files) { + uint64_t number = 0; + if (ParseFileName(file, &number, &type) && type == kWalFile) { + file_nums.push_back(number); + } + } + std::sort(file_nums.begin(), file_nums.end()); + } + + void CorruptFileWithTruncation(FileType file, uint64_t number, + uint64_t bytes_to_truncate = 0) { + std::string path; + switch (file) { + case FileType::kWalFile: + path = LogFileName(dbname_, number); + break; + // TODO: Add other file types as this method is being used for those file + // types. + default: + return; + } + uint64_t old_size = 0; + ASSERT_OK(env_->GetFileSize(path, &old_size)); + assert(old_size > bytes_to_truncate); + uint64_t new_size = old_size - bytes_to_truncate; + // If bytes_to_truncate == 0, it will do full truncation. + if (bytes_to_truncate == 0) { + new_size = 0; + } + ASSERT_OK(test::TruncateFile(env_.get(), path, new_size)); + } +}; + +TEST_F(CorruptionTest, Recovery) { + Build(100); + Check(100, 100); +#ifdef OS_WIN + // On Wndows OS Disk cache does not behave properly + // We do not call FlushBuffers on every Flush. If we do not close + // the log file prior to the corruption we end up with the first + // block not corrupted but only the second. However, under the debugger + // things work just fine but never pass when running normally + // For that reason people may want to run with unbuffered I/O. That option + // is not available for WAL though. + CloseDb(); +#endif + Corrupt(kWalFile, 19, 1); // WriteBatch tag for first record + Corrupt(kWalFile, log::kBlockSize + 1000, 1); // Somewhere in second block + ASSERT_TRUE(!TryReopen().ok()); + options_.paranoid_checks = false; + Reopen(&options_); + + // The 64 records in the first two log blocks are completely lost. + Check(36, 36); +} + +TEST_F(CorruptionTest, PostPITRCorruptionWALsRetained) { + // Repro for bug where WALs following the point-in-time recovery were not + // retained leading to the next recovery failing. + CloseDb(); + + options_.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery; + + const std::string test_cf_name = "test_cf"; + std::vector cf_descs; + cf_descs.emplace_back(kDefaultColumnFamilyName, ColumnFamilyOptions()); + cf_descs.emplace_back(test_cf_name, ColumnFamilyOptions()); + + uint64_t log_num; + { + options_.create_missing_column_families = true; + std::vector cfhs; + ASSERT_OK(DB::Open(options_, dbname_, cf_descs, &cfhs, &db_)); + assert(db_ != nullptr); // suppress false clang-analyze report + + ASSERT_OK(db_->Put(WriteOptions(), cfhs[0], "k", "v")); + ASSERT_OK(db_->Put(WriteOptions(), cfhs[1], "k", "v")); + ASSERT_OK(db_->Put(WriteOptions(), cfhs[0], "k2", "v2")); + std::vector file_nums; + GetSortedWalFiles(file_nums); + log_num = file_nums.back(); + for (auto* cfh : cfhs) { + delete cfh; + } + CloseDb(); + } + + CorruptFileWithTruncation(FileType::kWalFile, log_num, + /*bytes_to_truncate=*/1); + + { + // Recover "k" -> "v" for both CFs. "k2" -> "v2" is lost due to truncation. + options_.avoid_flush_during_recovery = true; + std::vector cfhs; + ASSERT_OK(DB::Open(options_, dbname_, cf_descs, &cfhs, &db_)); + assert(db_ != nullptr); // suppress false clang-analyze report + + // Flush one but not both CFs and write some data so there's a seqno gap + // between the PITR corruption and the next DB session's first WAL. + ASSERT_OK(db_->Put(WriteOptions(), cfhs[1], "k2", "v2")); + ASSERT_OK(db_->Flush(FlushOptions(), cfhs[1])); + + for (auto* cfh : cfhs) { + delete cfh; + } + CloseDb(); + } + + // With the bug, this DB open would remove the WALs following the PITR + // corruption. Then, the next recovery would fail. + for (int i = 0; i < 2; ++i) { + std::vector cfhs; + ASSERT_OK(DB::Open(options_, dbname_, cf_descs, &cfhs, &db_)); + assert(db_ != nullptr); // suppress false clang-analyze report + + for (auto* cfh : cfhs) { + delete cfh; + } + CloseDb(); + } +} + +TEST_F(CorruptionTest, RecoverWriteError) { + fs_->writable_file_error_ = true; + Status s = TryReopen(); + ASSERT_TRUE(!s.ok()); +} + +TEST_F(CorruptionTest, NewFileErrorDuringWrite) { + // Do enough writing to force minor compaction + fs_->writable_file_error_ = true; + const int num = + static_cast(3 + (Options().write_buffer_size / kValueSize)); + std::string value_storage; + Status s; + bool failed = false; + for (int i = 0; i < num; i++) { + WriteBatch batch; + ASSERT_OK(batch.Put("a", Value(100, &value_storage))); + s = db_->Write(WriteOptions(), &batch); + if (!s.ok()) { + failed = true; + } + ASSERT_TRUE(!failed || !s.ok()); + } + ASSERT_TRUE(!s.ok()); + ASSERT_GE(fs_->num_writable_file_errors_, 1); + fs_->writable_file_error_ = false; + Reopen(); +} + +TEST_F(CorruptionTest, TableFile) { + Build(100); + DBImpl* dbi = static_cast_with_check(db_); + ASSERT_OK(dbi->TEST_FlushMemTable()); + ASSERT_OK(dbi->TEST_CompactRange(0, nullptr, nullptr)); + ASSERT_OK(dbi->TEST_CompactRange(1, nullptr, nullptr)); + + Corrupt(kTableFile, 100, 1); + Check(99, 99); + ASSERT_NOK(dbi->VerifyChecksum()); +} + +TEST_F(CorruptionTest, VerifyChecksumReadahead) { + Options options; + options.level_compaction_dynamic_level_bytes = false; + SpecialEnv senv(base_env_); + options.env = &senv; + // Disable block cache as we are going to check checksum for + // the same file twice and measure number of reads. + BlockBasedTableOptions table_options_no_bc; + table_options_no_bc.no_block_cache = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options_no_bc)); + + Reopen(&options); + + Build(10000); + DBImpl* dbi = static_cast_with_check(db_); + ASSERT_OK(dbi->TEST_FlushMemTable()); + ASSERT_OK(dbi->TEST_CompactRange(0, nullptr, nullptr)); + ASSERT_OK(dbi->TEST_CompactRange(1, nullptr, nullptr)); + + senv.count_random_reads_ = true; + senv.random_read_counter_.Reset(); + ASSERT_OK(dbi->VerifyChecksum()); + + // Make sure the counter is enabled. + ASSERT_GT(senv.random_read_counter_.Read(), 0); + + // The SST file is about 10MB. Default readahead size is 256KB. + // Give a conservative 20 reads for metadata blocks, The number + // of random reads should be within 10 MB / 256KB + 20 = 60. + ASSERT_LT(senv.random_read_counter_.Read(), 60); + + senv.random_read_bytes_counter_ = 0; + ReadOptions ro; + ro.readahead_size = size_t{32 * 1024}; + ASSERT_OK(dbi->VerifyChecksum(ro)); + // The SST file is about 10MB. We set readahead size to 32KB. + // Give 0 to 20 reads for metadata blocks, and allow real read + // to range from 24KB to 48KB. The lower bound would be: + // 10MB / 48KB + 0 = 213 + // The higher bound is + // 10MB / 24KB + 20 = 447. + ASSERT_GE(senv.random_read_counter_.Read(), 213); + ASSERT_LE(senv.random_read_counter_.Read(), 447); + + // Test readahead shouldn't break mmap mode (where it should be + // disabled). + options.allow_mmap_reads = true; + Reopen(&options); + dbi = static_cast(db_); + ASSERT_OK(dbi->VerifyChecksum(ro)); + + CloseDb(); +} + +TEST_F(CorruptionTest, TableFileIndexData) { + Options options; + options.level_compaction_dynamic_level_bytes = false; + // very big, we'll trigger flushes manually + options.write_buffer_size = 100 * 1024 * 1024; + Reopen(&options); + // build 2 tables, flush at 5000 + Build(10000, 5000); + DBImpl* dbi = static_cast_with_check(db_); + ASSERT_OK(dbi->TEST_FlushMemTable()); + + // corrupt an index block of an entire file + Corrupt(kTableFile, -2000, 500); + options.paranoid_checks = false; + Reopen(&options); + dbi = static_cast_with_check(db_); + // one full file may be readable, since only one was corrupted + // the other file should be fully non-readable, since index was corrupted + Check(0, 5000, ReadOptions(true, true)); + ASSERT_NOK(dbi->VerifyChecksum()); + + // In paranoid mode, the db cannot be opened due to the corrupted file. + ASSERT_TRUE(TryReopen().IsCorruption()); +} + +TEST_F(CorruptionTest, TableFileFooterMagic) { + Build(100); + DBImpl* dbi = static_cast_with_check(db_); + ASSERT_OK(dbi->TEST_FlushMemTable()); + Check(100, 100); + // Corrupt the whole footer + Corrupt(kTableFile, -100, 100); + Status s = TryReopen(); + ASSERT_TRUE(s.IsCorruption()); + // Contains useful message, and magic number should be the first thing + // reported as corrupt. + ASSERT_TRUE(s.ToString().find("magic number") != std::string::npos); + // with file name + ASSERT_TRUE(s.ToString().find(".sst") != std::string::npos); +} + +TEST_F(CorruptionTest, TableFileFooterNotMagic) { + Build(100); + DBImpl* dbi = static_cast_with_check(db_); + ASSERT_OK(dbi->TEST_FlushMemTable()); + Check(100, 100); + // Corrupt footer except magic number + Corrupt(kTableFile, -100, 92); + Status s = TryReopen(); + ASSERT_TRUE(s.IsCorruption()); + // The next thing checked after magic number is format_version + ASSERT_TRUE(s.ToString().find("format_version") != std::string::npos); + // with file name + ASSERT_TRUE(s.ToString().find(".sst") != std::string::npos); +} + +TEST_F(CorruptionTest, TableFileWrongSize) { + Build(100); + DBImpl* dbi = static_cast_with_check(db_); + ASSERT_OK(dbi->TEST_FlushMemTable()); + Check(100, 100); + + // ******************************************** + // Make the file bigger by appending to it + std::vector metadata; + db_->GetLiveFilesMetaData(&metadata); + ASSERT_EQ(1U, metadata.size()); + std::string filename = dbname_ + metadata[0].name; + const auto& fs = options_.env->GetFileSystem(); + { + std::unique_ptr f; + ASSERT_OK(fs->ReopenWritableFile(filename, FileOptions(), &f, nullptr)); + ASSERT_OK(f->Append("blahblah", IOOptions(), nullptr)); + ASSERT_OK(f->Close(IOOptions(), nullptr)); + } + + // DB actually accepts this without paranoid checks, relying on size + // recorded in manifest to locate the SST footer. + options_.paranoid_checks = false; + options_.skip_checking_sst_file_sizes_on_db_open = false; + Reopen(); + Check(100, 100); + + // But reports the issue with paranoid checks + options_.paranoid_checks = true; + Status s = TryReopen(); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(s.ToString().find("file size mismatch") != std::string::npos); + + // ******************************************** + // Make the file smaller with truncation. + // First leaving a partial footer, and then completely removing footer. + for (size_t bytes_lost : {8, 100}) { + ASSERT_OK(test::TruncateFile(env_.get(), filename, + metadata[0].size - bytes_lost)); + + // Reported well with paranoid checks + options_.paranoid_checks = true; + s = TryReopen(); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(s.ToString().find("file size mismatch") != std::string::npos); + + // Without paranoid checks, not reported until read + options_.paranoid_checks = false; + Reopen(); + Check(0, 0); // Missing data + } +} + +TEST_F(CorruptionTest, MissingDescriptor) { + Build(1000); + RepairDB(); + Reopen(); + Check(1000, 1000); +} + +TEST_F(CorruptionTest, SequenceNumberRecovery) { + ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1")); + ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2")); + ASSERT_OK(db_->Put(WriteOptions(), "foo", "v3")); + ASSERT_OK(db_->Put(WriteOptions(), "foo", "v4")); + ASSERT_OK(db_->Put(WriteOptions(), "foo", "v5")); + RepairDB(); + Reopen(); + std::string v; + ASSERT_OK(db_->Get(ReadOptions(), "foo", &v)); + ASSERT_EQ("v5", v); + // Write something. If sequence number was not recovered properly, + // it will be hidden by an earlier write. + ASSERT_OK(db_->Put(WriteOptions(), "foo", "v6")); + ASSERT_OK(db_->Get(ReadOptions(), "foo", &v)); + ASSERT_EQ("v6", v); + Reopen(); + ASSERT_OK(db_->Get(ReadOptions(), "foo", &v)); + ASSERT_EQ("v6", v); +} + +TEST_F(CorruptionTest, CorruptedDescriptor) { + ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello")); + DBImpl* dbi = static_cast_with_check(db_); + ASSERT_OK(dbi->TEST_FlushMemTable()); + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK( + dbi->CompactRange(cro, dbi->DefaultColumnFamily(), nullptr, nullptr)); + + Corrupt(kDescriptorFile, 0, 1000); + Status s = TryReopen(); + ASSERT_TRUE(!s.ok()); + + RepairDB(); + Reopen(); + std::string v; + ASSERT_OK(db_->Get(ReadOptions(), "foo", &v)); + ASSERT_EQ("hello", v); +} + +TEST_F(CorruptionTest, CompactionInputError) { + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.env = env_.get(); + Reopen(&options); + Build(10); + DBImpl* dbi = static_cast_with_check(db_); + ASSERT_OK(dbi->TEST_FlushMemTable()); + ASSERT_OK(dbi->TEST_CompactRange(0, nullptr, nullptr)); + ASSERT_OK(dbi->TEST_CompactRange(1, nullptr, nullptr)); + ASSERT_EQ(1, Property("rocksdb.num-files-at-level2")); + + Corrupt(kTableFile, 100, 1); + Check(9, 9); + ASSERT_NOK(dbi->VerifyChecksum()); + + // Force compactions by writing lots of values + Build(10000); + Check(10000, 10000); + ASSERT_NOK(dbi->VerifyChecksum()); +} + +TEST_F(CorruptionTest, CompactionInputErrorParanoid) { + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.env = env_.get(); + options.paranoid_checks = true; + options.write_buffer_size = 131072; + options.max_write_buffer_number = 2; + Reopen(&options); + DBImpl* dbi = static_cast_with_check(db_); + + // Fill levels >= 1 + for (int level = 1; level < dbi->NumberLevels(); level++) { + ASSERT_OK(dbi->Put(WriteOptions(), "", "begin")); + ASSERT_OK(dbi->Put(WriteOptions(), "~", "end")); + ASSERT_OK(dbi->TEST_FlushMemTable()); + for (int comp_level = 0; comp_level < dbi->NumberLevels() - level; + ++comp_level) { + ASSERT_OK(dbi->TEST_CompactRange(comp_level, nullptr, nullptr)); + } + } + + Reopen(&options); + + dbi = static_cast_with_check(db_); + Build(10); + ASSERT_OK(dbi->TEST_FlushMemTable()); + ASSERT_OK(dbi->TEST_WaitForCompact()); + ASSERT_EQ(1, Property("rocksdb.num-files-at-level0")); + + CorruptTableFileAtLevel(0, 100, 1); + Check(9, 9); + ASSERT_NOK(dbi->VerifyChecksum()); + + // Write must eventually fail because of corrupted table + Status s; + std::string tmp1, tmp2; + bool failed = false; + for (int i = 0; i < 10000; i++) { + s = db_->Put(WriteOptions(), Key(i, &tmp1), Value(i, &tmp2)); + if (!s.ok()) { + failed = true; + } + // if one write failed, every subsequent write must fail, too + ASSERT_TRUE(!failed || !s.ok()) << "write did not fail in a corrupted db"; + } + ASSERT_TRUE(!s.ok()) << "write did not fail in corrupted paranoid db"; +} + +TEST_F(CorruptionTest, UnrelatedKeys) { + Build(10); + DBImpl* dbi = static_cast_with_check(db_); + ASSERT_OK(dbi->TEST_FlushMemTable()); + Corrupt(kTableFile, 100, 1); + ASSERT_NOK(dbi->VerifyChecksum()); + + std::string tmp1, tmp2; + ASSERT_OK(db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2))); + std::string v; + ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v)); + ASSERT_EQ(Value(1000, &tmp2).ToString(), v); + ASSERT_OK(dbi->TEST_FlushMemTable()); + ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v)); + ASSERT_EQ(Value(1000, &tmp2).ToString(), v); +} + +TEST_F(CorruptionTest, RangeDeletionCorrupted) { + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "b")); + ASSERT_OK(db_->Flush(FlushOptions())); + std::vector metadata; + db_->GetLiveFilesMetaData(&metadata); + ASSERT_EQ(static_cast(1), metadata.size()); + std::string filename = dbname_ + metadata[0].name; + + FileOptions file_opts; + const auto& fs = options_.env->GetFileSystem(); + std::unique_ptr file_reader; + ASSERT_OK(RandomAccessFileReader::Create(fs, filename, file_opts, + &file_reader, nullptr)); + + uint64_t file_size; + ASSERT_OK( + fs->GetFileSize(filename, file_opts.io_options, &file_size, nullptr)); + + BlockHandle range_del_handle; + const ReadOptions read_options; + ASSERT_OK(FindMetaBlockInFile(file_reader.get(), file_size, + kBlockBasedTableMagicNumber, + ImmutableOptions(options_), read_options, + kRangeDelBlockName, &range_del_handle)); + + ASSERT_OK(TryReopen()); + ASSERT_OK(test::CorruptFile(env_.get(), filename, + static_cast(range_del_handle.offset()), 1)); + ASSERT_TRUE(TryReopen().IsCorruption()); +} + +TEST_F(CorruptionTest, FileSystemStateCorrupted) { + for (int iter = 0; iter < 2; ++iter) { + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.env = env_.get(); + options.paranoid_checks = true; + options.create_if_missing = true; + Reopen(&options); + Build(10); + ASSERT_OK(db_->Flush(FlushOptions())); + DBImpl* dbi = static_cast_with_check(db_); + std::vector metadata; + dbi->GetLiveFilesMetaData(&metadata); + ASSERT_GT(metadata.size(), 0); + std::string filename = dbname_ + metadata[0].name; + + delete db_; + db_ = nullptr; + + if (iter == 0) { // corrupt file size + std::unique_ptr file; + ASSERT_OK(env_->NewWritableFile(filename, &file, EnvOptions())); + ASSERT_OK(file->Append(Slice("corrupted sst"))); + file.reset(); + Status x = TryReopen(&options); + ASSERT_TRUE(x.IsCorruption()); + } else { // delete the file + ASSERT_OK(env_->DeleteFile(filename)); + Status x = TryReopen(&options); + ASSERT_TRUE(x.IsCorruption()); + } + + ASSERT_OK(DestroyDB(dbname_, options_)); + } +} + +static const auto& corruption_modes = { + mock::MockTableFactory::kCorruptNone, mock::MockTableFactory::kCorruptKey, + mock::MockTableFactory::kCorruptValue, + mock::MockTableFactory::kCorruptReorderKey}; + +TEST_F(CorruptionTest, ParanoidFileChecksOnFlush) { + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.env = env_.get(); + options.check_flush_compaction_key_order = false; + options.paranoid_file_checks = true; + options.create_if_missing = true; + Status s; + for (const auto& mode : corruption_modes) { + delete db_; + db_ = nullptr; + s = DestroyDB(dbname_, options); + ASSERT_OK(s); + std::shared_ptr mock = + std::make_shared(); + options.table_factory = mock; + mock->SetCorruptionMode(mode); + ASSERT_OK(DB::Open(options, dbname_, &db_)); + assert(db_ != nullptr); // suppress false clang-analyze report + Build(10); + s = db_->Flush(FlushOptions()); + if (mode == mock::MockTableFactory::kCorruptNone) { + ASSERT_OK(s); + } else { + ASSERT_NOK(s); + } + } +} + +TEST_F(CorruptionTest, ParanoidFileChecksOnCompact) { + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.env = env_.get(); + options.paranoid_file_checks = true; + options.create_if_missing = true; + options.check_flush_compaction_key_order = false; + Status s; + for (const auto& mode : corruption_modes) { + delete db_; + db_ = nullptr; + s = DestroyDB(dbname_, options); + ASSERT_OK(s); + std::shared_ptr mock = + std::make_shared(); + options.table_factory = mock; + ASSERT_OK(DB::Open(options, dbname_, &db_)); + assert(db_ != nullptr); // suppress false clang-analyze report + Build(100, 2); + // ASSERT_OK(db_->Flush(FlushOptions())); + DBImpl* dbi = static_cast_with_check(db_); + ASSERT_OK(dbi->TEST_FlushMemTable()); + mock->SetCorruptionMode(mode); + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + s = dbi->CompactRange(cro, dbi->DefaultColumnFamily(), nullptr, nullptr); + if (mode == mock::MockTableFactory::kCorruptNone) { + ASSERT_OK(s); + } else { + ASSERT_NOK(s); + } + } +} + +TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRangeFirst) { + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.env = env_.get(); + options.check_flush_compaction_key_order = false; + options.paranoid_file_checks = true; + options.create_if_missing = true; + for (bool do_flush : {true, false}) { + delete db_; + db_ = nullptr; + ASSERT_OK(DestroyDB(dbname_, options)); + ASSERT_OK(DB::Open(options, dbname_, &db_)); + std::string start, end; + assert(db_ != nullptr); // suppress false clang-analyze report + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(3, &start), Key(7, &end))); + auto snap = db_->GetSnapshot(); + ASSERT_NE(snap, nullptr); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(8, &start), Key(9, &end))); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(2, &start), Key(5, &end))); + Build(10); + if (do_flush) { + ASSERT_OK(db_->Flush(FlushOptions())); + } else { + DBImpl* dbi = static_cast_with_check(db_); + ASSERT_OK(dbi->TEST_FlushMemTable()); + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK( + dbi->CompactRange(cro, dbi->DefaultColumnFamily(), nullptr, nullptr)); + } + db_->ReleaseSnapshot(snap); + } +} + +TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRange) { + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.env = env_.get(); + options.check_flush_compaction_key_order = false; + options.paranoid_file_checks = true; + options.create_if_missing = true; + for (bool do_flush : {true, false}) { + delete db_; + db_ = nullptr; + ASSERT_OK(DestroyDB(dbname_, options)); + ASSERT_OK(DB::Open(options, dbname_, &db_)); + assert(db_ != nullptr); // suppress false clang-analyze report + Build(10, 0, 0); + std::string start, end; + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(5, &start), Key(15, &end))); + auto snap = db_->GetSnapshot(); + ASSERT_NE(snap, nullptr); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(8, &start), Key(9, &end))); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(12, &start), Key(17, &end))); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(2, &start), Key(4, &end))); + Build(10, 10, 0); + if (do_flush) { + ASSERT_OK(db_->Flush(FlushOptions())); + } else { + DBImpl* dbi = static_cast_with_check(db_); + ASSERT_OK(dbi->TEST_FlushMemTable()); + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK( + dbi->CompactRange(cro, dbi->DefaultColumnFamily(), nullptr, nullptr)); + } + db_->ReleaseSnapshot(snap); + } +} + +TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRangeLast) { + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.env = env_.get(); + options.check_flush_compaction_key_order = false; + options.paranoid_file_checks = true; + options.create_if_missing = true; + for (bool do_flush : {true, false}) { + delete db_; + db_ = nullptr; + ASSERT_OK(DestroyDB(dbname_, options)); + ASSERT_OK(DB::Open(options, dbname_, &db_)); + assert(db_ != nullptr); // suppress false clang-analyze report + std::string start, end; + Build(10); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(3, &start), Key(7, &end))); + auto snap = db_->GetSnapshot(); + ASSERT_NE(snap, nullptr); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(6, &start), Key(8, &end))); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(2, &start), Key(5, &end))); + if (do_flush) { + ASSERT_OK(db_->Flush(FlushOptions())); + } else { + DBImpl* dbi = static_cast_with_check(db_); + ASSERT_OK(dbi->TEST_FlushMemTable()); + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK( + dbi->CompactRange(cro, dbi->DefaultColumnFamily(), nullptr, nullptr)); + } + db_->ReleaseSnapshot(snap); + } +} + +TEST_F(CorruptionTest, LogCorruptionErrorsInCompactionIterator) { + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.env = env_.get(); + options.create_if_missing = true; + options.allow_data_in_errors = true; + auto mode = mock::MockTableFactory::kCorruptKey; + delete db_; + db_ = nullptr; + ASSERT_OK(DestroyDB(dbname_, options)); + + std::shared_ptr mock = + std::make_shared(); + mock->SetCorruptionMode(mode); + options.table_factory = mock; + + ASSERT_OK(DB::Open(options, dbname_, &db_)); + assert(db_ != nullptr); // suppress false clang-analyze report + Build(100, 2); + + DBImpl* dbi = static_cast_with_check(db_); + ASSERT_OK(dbi->TEST_FlushMemTable()); + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + Status s = + dbi->CompactRange(cro, dbi->DefaultColumnFamily(), nullptr, nullptr); + ASSERT_NOK(s); + ASSERT_TRUE(s.IsCorruption()); +} + +TEST_F(CorruptionTest, CompactionKeyOrderCheck) { + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.env = env_.get(); + options.paranoid_file_checks = false; + options.create_if_missing = true; + options.check_flush_compaction_key_order = false; + delete db_; + db_ = nullptr; + ASSERT_OK(DestroyDB(dbname_, options)); + std::shared_ptr mock = + std::make_shared(); + options.table_factory = mock; + ASSERT_OK(DB::Open(options, dbname_, &db_)); + assert(db_ != nullptr); // suppress false clang-analyze report + mock->SetCorruptionMode(mock::MockTableFactory::kCorruptReorderKey); + Build(100, 2); + DBImpl* dbi = static_cast_with_check(db_); + ASSERT_OK(dbi->TEST_FlushMemTable()); + + mock->SetCorruptionMode(mock::MockTableFactory::kCorruptNone); + ASSERT_OK(db_->SetOptions({{"check_flush_compaction_key_order", "true"}})); + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_NOK( + dbi->CompactRange(cro, dbi->DefaultColumnFamily(), nullptr, nullptr)); +} + +TEST_F(CorruptionTest, FlushKeyOrderCheck) { + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.env = env_.get(); + options.paranoid_file_checks = false; + options.create_if_missing = true; + ASSERT_OK(db_->SetOptions({{"check_flush_compaction_key_order", "true"}})); + + ASSERT_OK(db_->Put(WriteOptions(), "foo1", "v1")); + ASSERT_OK(db_->Put(WriteOptions(), "foo2", "v1")); + ASSERT_OK(db_->Put(WriteOptions(), "foo3", "v1")); + ASSERT_OK(db_->Put(WriteOptions(), "foo4", "v1")); + + int cnt = 0; + // Generate some out of order keys from the memtable + SyncPoint::GetInstance()->SetCallBack( + "MemTableIterator::Next:0", [&](void* arg) { + MemTableRep::Iterator* mem_iter = + static_cast(arg); + if (++cnt == 3) { + mem_iter->Prev(); + mem_iter->Prev(); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + Status s = static_cast_with_check(db_)->TEST_FlushMemTable(); + ASSERT_NOK(s); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(CorruptionTest, DisableKeyOrderCheck) { + ASSERT_OK(db_->SetOptions({{"check_flush_compaction_key_order", "false"}})); + DBImpl* dbi = static_cast_with_check(db_); + + SyncPoint::GetInstance()->SetCallBack( + "OutputValidator::Add:order_check", + [&](void* /*arg*/) { ASSERT_TRUE(false); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(db_->Put(WriteOptions(), "foo1", "v1")); + ASSERT_OK(db_->Put(WriteOptions(), "foo3", "v1")); + ASSERT_OK(dbi->TEST_FlushMemTable()); + ASSERT_OK(db_->Put(WriteOptions(), "foo2", "v1")); + ASSERT_OK(db_->Put(WriteOptions(), "foo4", "v1")); + ASSERT_OK(dbi->TEST_FlushMemTable()); + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK( + dbi->CompactRange(cro, dbi->DefaultColumnFamily(), nullptr, nullptr)); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(CorruptionTest, VerifyWholeTableChecksum) { + CloseDb(); + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.env = env_.get(); + ASSERT_OK(DestroyDB(dbname_, options)); + options.create_if_missing = true; + options.file_checksum_gen_factory = + ROCKSDB_NAMESPACE::GetFileChecksumGenCrc32cFactory(); + Reopen(&options); + + Build(10, 5); + + ASSERT_OK(db_->VerifyFileChecksums(ReadOptions())); + CloseDb(); + + // Corrupt the first byte of each table file, this must be data block. + Corrupt(kTableFile, 0, 1); + + ASSERT_OK(TryReopen(&options)); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + int count{0}; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::VerifyFullFileChecksum:mismatch", [&](void* arg) { + auto* s = reinterpret_cast(arg); + ASSERT_NE(s, nullptr); + ++count; + ASSERT_NOK(*s); + }); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_TRUE(db_->VerifyFileChecksums(ReadOptions()).IsCorruption()); + ASSERT_EQ(1, count); +} + +class CrashDuringRecoveryWithCorruptionTest + : public CorruptionTest, + public testing::WithParamInterface> { + public: + explicit CrashDuringRecoveryWithCorruptionTest() + : CorruptionTest(), + avoid_flush_during_recovery_(std::get<0>(GetParam())), + track_and_verify_wals_in_manifest_(std::get<1>(GetParam())) {} + + protected: + const bool avoid_flush_during_recovery_; + const bool track_and_verify_wals_in_manifest_; +}; + +INSTANTIATE_TEST_CASE_P(CorruptionTest, CrashDuringRecoveryWithCorruptionTest, + ::testing::Values(std::make_tuple(true, false), + std::make_tuple(false, false), + std::make_tuple(true, true), + std::make_tuple(false, true))); + +// In case of non-TransactionDB with avoid_flush_during_recovery = true, RocksDB +// won't flush the data from WAL to L0 for all column families if possible. As a +// result, not all column families can increase their log_numbers, and +// min_log_number_to_keep won't change. +// It may prematurely persist a new MANIFEST even before we can declare the DB +// is in consistent state after recovery (this is when the new WAL is synced) +// and advances log_numbers for some column families. +// +// If there is power failure before we sync the new WAL, we will end up in +// a situation in which after persisting the MANIFEST, RocksDB will see some +// column families' log_numbers larger than the corrupted wal, and +// "Column family inconsistency: SST file contains data beyond the point of +// corruption" error will be hit, causing recovery to fail. +// +// After adding the fix, only after new WAL is synced, RocksDB persist a new +// MANIFEST with column families to ensure RocksDB is in consistent state. +// RocksDB writes an empty WriteBatch as a sentinel to the new WAL which is +// synced immediately afterwards. The sequence number of the sentinel +// WriteBatch will be the next sequence number immediately after the largest +// sequence number recovered from previous WALs and MANIFEST because of which DB +// will be in consistent state. +// If a future recovery starts from the new MANIFEST, then it means the new WAL +// is successfully synced. Due to the sentinel empty write batch at the +// beginning, kPointInTimeRecovery of WAL is guaranteed to go after this point. +// If future recovery starts from the old MANIFEST, it means the writing the new +// MANIFEST failed. It won't have the "SST ahead of WAL" error. +// +// The combination of corrupting a WAL and injecting an error during subsequent +// re-open exposes the bug of prematurely persisting a new MANIFEST with +// advanced ColumnFamilyData::log_number. +TEST_P(CrashDuringRecoveryWithCorruptionTest, CrashDuringRecovery) { + CloseDb(); + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.track_and_verify_wals_in_manifest = + track_and_verify_wals_in_manifest_; + options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery; + options.avoid_flush_during_recovery = false; + options.env = env_.get(); + ASSERT_OK(DestroyDB(dbname_, options)); + options.create_if_missing = true; + options.max_write_buffer_number = 8; + + Reopen(&options); + Status s; + const std::string test_cf_name = "test_cf"; + ColumnFamilyHandle* cfh = nullptr; + s = db_->CreateColumnFamily(options, test_cf_name, &cfh); + ASSERT_OK(s); + delete cfh; + CloseDb(); + + std::vector cf_descs; + cf_descs.emplace_back(kDefaultColumnFamilyName, options); + cf_descs.emplace_back(test_cf_name, options); + std::vector handles; + + // 1. Open and populate the DB. Write and flush default_cf several times to + // advance wal number so that some column families have advanced log_number + // while other don't. + { + ASSERT_OK(DB::Open(options, dbname_, cf_descs, &handles, &db_)); + auto* dbimpl = static_cast_with_check(db_); + assert(dbimpl); + + // Write one key to test_cf. + ASSERT_OK(db_->Put(WriteOptions(), handles[1], "old_key", "dontcare")); + ASSERT_OK(db_->Flush(FlushOptions(), handles[1])); + + // Write to default_cf and flush this cf several times to advance wal + // number. TEST_SwitchMemtable makes sure WALs are not synced and test can + // corrupt un-sync WAL. + for (int i = 0; i < 2; ++i) { + ASSERT_OK(db_->Put(WriteOptions(), "key" + std::to_string(i), + "value" + std::to_string(i))); + ASSERT_OK(dbimpl->TEST_SwitchMemtable()); + } + + for (auto* h : handles) { + delete h; + } + handles.clear(); + CloseDb(); + } + + // 2. Corrupt second last un-syned wal file to emulate power reset which + // caused the DB to lose the un-synced WAL. + { + std::vector file_nums; + GetSortedWalFiles(file_nums); + size_t size = file_nums.size(); + assert(size >= 2); + uint64_t log_num = file_nums[size - 2]; + CorruptFileWithTruncation(FileType::kWalFile, log_num, + /*bytes_to_truncate=*/8); + } + + // 3. After first crash reopen the DB which contains corrupted WAL. Default + // family has higher log number than corrupted wal number. + // + // Case1: If avoid_flush_during_recovery = true, RocksDB won't flush the data + // from WAL to L0 for all column families (test_cf_name in this case). As a + // result, not all column families can increase their log_numbers, and + // min_log_number_to_keep won't change. + // + // Case2: If avoid_flush_during_recovery = false, all column families have + // flushed their data from WAL to L0 during recovery, and none of them will + // ever need to read the WALs again. + + // 4. Fault is injected to fail the recovery. + { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::GetLogSizeAndMaybeTruncate:0", [&](void* arg) { + auto* tmp_s = reinterpret_cast(arg); + assert(tmp_s); + *tmp_s = Status::IOError("Injected"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + handles.clear(); + options.avoid_flush_during_recovery = true; + s = DB::Open(options, dbname_, cf_descs, &handles, &db_); + ASSERT_TRUE(s.IsIOError()); + ASSERT_EQ("IO error: Injected", s.ToString()); + for (auto* h : handles) { + delete h; + } + CloseDb(); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + } + + // 5. After second crash reopen the db with second corruption. Default family + // has higher log number than corrupted wal number. + // + // Case1: If avoid_flush_during_recovery = true, we persist a new + // MANIFEST with advanced log_numbers for some column families only after + // syncing the WAL. So during second crash, RocksDB will skip the corrupted + // WAL files as they have been moved to different folder. Since newly synced + // WAL file's sequence number (sentinel WriteBatch) will be the next + // sequence number immediately after the largest sequence number recovered + // from previous WALs and MANIFEST, db will be in consistent state and opens + // successfully. + // + // Case2: If avoid_flush_during_recovery = false, the corrupted WAL is below + // this number. So during a second crash after persisting the new MANIFEST, + // RocksDB will skip the corrupted WAL(s) because they are all below this + // bound. Therefore, we won't hit the "column family inconsistency" error + // message. + { + options.avoid_flush_during_recovery = avoid_flush_during_recovery_; + ASSERT_OK(DB::Open(options, dbname_, cf_descs, &handles, &db_)); + + // Verify that data is not lost. + { + std::string v; + ASSERT_OK(db_->Get(ReadOptions(), handles[1], "old_key", &v)); + ASSERT_EQ("dontcare", v); + + v.clear(); + ASSERT_OK(db_->Get(ReadOptions(), "key" + std::to_string(0), &v)); + ASSERT_EQ("value" + std::to_string(0), v); + + // Since it's corrupting second last wal, below key is not found. + v.clear(); + ASSERT_EQ(db_->Get(ReadOptions(), "key" + std::to_string(1), &v), + Status::NotFound()); + } + + for (auto* h : handles) { + delete h; + } + handles.clear(); + CloseDb(); + } +} + +// In case of TransactionDB, it enables two-phase-commit. The prepare section of +// an uncommitted transaction always need to be kept. Even if we perform flush +// during recovery, we may still need to hold an old WAL. The +// min_log_number_to_keep won't change, and "Column family inconsistency: SST +// file contains data beyond the point of corruption" error will be hit, causing +// recovery to fail. +// +// After adding the fix, only after new WAL is synced, RocksDB persist a new +// MANIFEST with column families to ensure RocksDB is in consistent state. +// RocksDB writes an empty WriteBatch as a sentinel to the new WAL which is +// synced immediately afterwards. The sequence number of the sentinel +// WriteBatch will be the next sequence number immediately after the largest +// sequence number recovered from previous WALs and MANIFEST because of which DB +// will be in consistent state. +// If a future recovery starts from the new MANIFEST, then it means the new WAL +// is successfully synced. Due to the sentinel empty write batch at the +// beginning, kPointInTimeRecovery of WAL is guaranteed to go after this point. +// If future recovery starts from the old MANIFEST, it means the writing the new +// MANIFEST failed. It won't have the "SST ahead of WAL" error. +// +// The combination of corrupting a WAL and injecting an error during subsequent +// re-open exposes the bug of prematurely persisting a new MANIFEST with +// advanced ColumnFamilyData::log_number. +TEST_P(CrashDuringRecoveryWithCorruptionTest, TxnDbCrashDuringRecovery) { + CloseDb(); + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery; + options.track_and_verify_wals_in_manifest = + track_and_verify_wals_in_manifest_; + options.avoid_flush_during_recovery = false; + options.env = env_.get(); + ASSERT_OK(DestroyDB(dbname_, options)); + options.create_if_missing = true; + options.max_write_buffer_number = 3; + Reopen(&options); + + // Create cf test_cf_name. + ColumnFamilyHandle* cfh = nullptr; + const std::string test_cf_name = "test_cf"; + Status s = db_->CreateColumnFamily(options, test_cf_name, &cfh); + ASSERT_OK(s); + delete cfh; + CloseDb(); + + std::vector cf_descs; + cf_descs.emplace_back(kDefaultColumnFamilyName, options); + cf_descs.emplace_back(test_cf_name, options); + std::vector handles; + + TransactionDB* txn_db = nullptr; + TransactionDBOptions txn_db_opts; + + // 1. Open and populate the DB. Write and flush default_cf several times to + // advance wal number so that some column families have advanced log_number + // while other don't. + { + ASSERT_OK(TransactionDB::Open(options, txn_db_opts, dbname_, cf_descs, + &handles, &txn_db)); + + auto* txn = txn_db->BeginTransaction(WriteOptions(), TransactionOptions()); + // Put cf1 + ASSERT_OK(txn->Put(handles[1], "foo", "value")); + ASSERT_OK(txn->SetName("txn0")); + ASSERT_OK(txn->Prepare()); + ASSERT_OK(txn_db->Flush(FlushOptions())); + + delete txn; + txn = nullptr; + + auto* dbimpl = static_cast_with_check(txn_db->GetRootDB()); + assert(dbimpl); + + // Put and flush cf0 + for (int i = 0; i < 2; ++i) { + ASSERT_OK(txn_db->Put(WriteOptions(), "key" + std::to_string(i), + "value" + std::to_string(i))); + ASSERT_OK(dbimpl->TEST_SwitchMemtable()); + } + + // Put cf1 + txn = txn_db->BeginTransaction(WriteOptions(), TransactionOptions()); + ASSERT_OK(txn->Put(handles[1], "foo1", "value1")); + ASSERT_OK(txn->Commit()); + + delete txn; + txn = nullptr; + + for (auto* h : handles) { + delete h; + } + handles.clear(); + delete txn_db; + } + + // 2. Corrupt second last wal to emulate power reset which caused the DB to + // lose the un-synced WAL. + { + std::vector file_nums; + GetSortedWalFiles(file_nums); + size_t size = file_nums.size(); + assert(size >= 2); + uint64_t log_num = file_nums[size - 2]; + CorruptFileWithTruncation(FileType::kWalFile, log_num, + /*bytes_to_truncate=*/8); + } + + // 3. After first crash reopen the DB which contains corrupted WAL. Default + // family has higher log number than corrupted wal number. There may be old + // WAL files that it must not delete because they can contain data of + // uncommitted transactions. As a result, min_log_number_to_keep won't change. + + { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::Open::BeforeSyncWAL", [&](void* arg) { + auto* tmp_s = reinterpret_cast(arg); + assert(tmp_s); + *tmp_s = Status::IOError("Injected"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + handles.clear(); + s = TransactionDB::Open(options, txn_db_opts, dbname_, cf_descs, &handles, + &txn_db); + ASSERT_TRUE(s.IsIOError()); + ASSERT_EQ("IO error: Injected", s.ToString()); + for (auto* h : handles) { + delete h; + } + CloseDb(); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + } + + // 4. Corrupt max_wal_num. + { + std::vector file_nums; + GetSortedWalFiles(file_nums); + size_t size = file_nums.size(); + uint64_t log_num = file_nums[size - 1]; + CorruptFileWithTruncation(FileType::kWalFile, log_num); + } + + // 5. After second crash reopen the db with second corruption. Default family + // has higher log number than corrupted wal number. + // We persist a new MANIFEST with advanced log_numbers for some column + // families only after syncing the WAL. So during second crash, RocksDB will + // skip the corrupted WAL files as they have been moved to different folder. + // Since newly synced WAL file's sequence number (sentinel WriteBatch) will be + // the next sequence number immediately after the largest sequence number + // recovered from previous WALs and MANIFEST, db will be in consistent state + // and opens successfully. + { + ASSERT_OK(TransactionDB::Open(options, txn_db_opts, dbname_, cf_descs, + &handles, &txn_db)); + + // Verify that data is not lost. + { + std::string v; + // Key not visible since it's not committed. + ASSERT_EQ(txn_db->Get(ReadOptions(), handles[1], "foo", &v), + Status::NotFound()); + + v.clear(); + ASSERT_OK(txn_db->Get(ReadOptions(), "key" + std::to_string(0), &v)); + ASSERT_EQ("value" + std::to_string(0), v); + + // Last WAL is corrupted which contains two keys below. + v.clear(); + ASSERT_EQ(txn_db->Get(ReadOptions(), "key" + std::to_string(1), &v), + Status::NotFound()); + v.clear(); + ASSERT_EQ(txn_db->Get(ReadOptions(), handles[1], "foo1", &v), + Status::NotFound()); + } + + for (auto* h : handles) { + delete h; + } + delete txn_db; + } +} + +// This test is similar to +// CrashDuringRecoveryWithCorruptionTest.CrashDuringRecovery except it calls +// flush and corrupts Last WAL. It calls flush to sync some of the WALs and +// remaining are unsyned one of which is then corrupted to simulate crash. +// +// In case of non-TransactionDB with avoid_flush_during_recovery = true, RocksDB +// won't flush the data from WAL to L0 for all column families if possible. As a +// result, not all column families can increase their log_numbers, and +// min_log_number_to_keep won't change. +// It may prematurely persist a new MANIFEST even before we can declare the DB +// is in consistent state after recovery (this is when the new WAL is synced) +// and advances log_numbers for some column families. +// +// If there is power failure before we sync the new WAL, we will end up in +// a situation in which after persisting the MANIFEST, RocksDB will see some +// column families' log_numbers larger than the corrupted wal, and +// "Column family inconsistency: SST file contains data beyond the point of +// corruption" error will be hit, causing recovery to fail. +// +// After adding the fix, only after new WAL is synced, RocksDB persist a new +// MANIFEST with column families to ensure RocksDB is in consistent state. +// RocksDB writes an empty WriteBatch as a sentinel to the new WAL which is +// synced immediately afterwards. The sequence number of the sentinel +// WriteBatch will be the next sequence number immediately after the largest +// sequence number recovered from previous WALs and MANIFEST because of which DB +// will be in consistent state. +// If a future recovery starts from the new MANIFEST, then it means the new WAL +// is successfully synced. Due to the sentinel empty write batch at the +// beginning, kPointInTimeRecovery of WAL is guaranteed to go after this point. +// If future recovery starts from the old MANIFEST, it means the writing the new +// MANIFEST failed. It won't have the "SST ahead of WAL" error. + +// The combination of corrupting a WAL and injecting an error during subsequent +// re-open exposes the bug of prematurely persisting a new MANIFEST with +// advanced ColumnFamilyData::log_number. +TEST_P(CrashDuringRecoveryWithCorruptionTest, CrashDuringRecoveryWithFlush) { + CloseDb(); + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery; + options.avoid_flush_during_recovery = false; + options.env = env_.get(); + options.create_if_missing = true; + + ASSERT_OK(DestroyDB(dbname_, options)); + Reopen(&options); + + ColumnFamilyHandle* cfh = nullptr; + const std::string test_cf_name = "test_cf"; + Status s = db_->CreateColumnFamily(options, test_cf_name, &cfh); + ASSERT_OK(s); + delete cfh; + + CloseDb(); + + std::vector cf_descs; + cf_descs.emplace_back(kDefaultColumnFamilyName, options); + cf_descs.emplace_back(test_cf_name, options); + std::vector handles; + + { + ASSERT_OK(DB::Open(options, dbname_, cf_descs, &handles, &db_)); + + // Write one key to test_cf. + ASSERT_OK(db_->Put(WriteOptions(), handles[1], "old_key", "dontcare")); + + // Write to default_cf and flush this cf several times to advance wal + // number. + for (int i = 0; i < 2; ++i) { + ASSERT_OK(db_->Put(WriteOptions(), "key" + std::to_string(i), + "value" + std::to_string(i))); + ASSERT_OK(db_->Flush(FlushOptions())); + } + + ASSERT_OK(db_->Put(WriteOptions(), handles[1], "dontcare", "dontcare")); + for (auto* h : handles) { + delete h; + } + handles.clear(); + CloseDb(); + } + + // Corrupt second last un-syned wal file to emulate power reset which + // caused the DB to lose the un-synced WAL. + { + std::vector file_nums; + GetSortedWalFiles(file_nums); + size_t size = file_nums.size(); + uint64_t log_num = file_nums[size - 1]; + CorruptFileWithTruncation(FileType::kWalFile, log_num, + /*bytes_to_truncate=*/8); + } + + // Fault is injected to fail the recovery. + { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::GetLogSizeAndMaybeTruncate:0", [&](void* arg) { + auto* tmp_s = reinterpret_cast(arg); + assert(tmp_s); + *tmp_s = Status::IOError("Injected"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + handles.clear(); + options.avoid_flush_during_recovery = true; + s = DB::Open(options, dbname_, cf_descs, &handles, &db_); + ASSERT_TRUE(s.IsIOError()); + ASSERT_EQ("IO error: Injected", s.ToString()); + for (auto* h : handles) { + delete h; + } + CloseDb(); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + } + + // Reopen db again + { + options.avoid_flush_during_recovery = avoid_flush_during_recovery_; + ASSERT_OK(DB::Open(options, dbname_, cf_descs, &handles, &db_)); + + // Verify that data is not lost. + { + std::string v; + ASSERT_OK(db_->Get(ReadOptions(), handles[1], "old_key", &v)); + ASSERT_EQ("dontcare", v); + + for (int i = 0; i < 2; ++i) { + v.clear(); + ASSERT_OK(db_->Get(ReadOptions(), "key" + std::to_string(i), &v)); + ASSERT_EQ("value" + std::to_string(i), v); + } + + // Since it's corrupting last wal after Flush, below key is not found. + v.clear(); + ASSERT_EQ(db_->Get(ReadOptions(), handles[1], "dontcare", &v), + Status::NotFound()); + } + + for (auto* h : handles) { + delete h; + } + } +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/cuckoo_table_db_test.cc b/librocksdb-sys/rocksdb/db/cuckoo_table_db_test.cc new file mode 100644 index 0000000..dec5c05 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/cuckoo_table_db_test.cc @@ -0,0 +1,352 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + + +#include "db/db_impl/db_impl.h" +#include "db/db_test_util.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "table/cuckoo/cuckoo_table_factory.h" +#include "table/cuckoo/cuckoo_table_reader.h" +#include "table/meta_blocks.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/cast_util.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +class CuckooTableDBTest : public testing::Test { + private: + std::string dbname_; + Env* env_; + DB* db_; + + public: + CuckooTableDBTest() : env_(Env::Default()) { + dbname_ = test::PerThreadDBPath("cuckoo_table_db_test"); + EXPECT_OK(DestroyDB(dbname_, Options())); + db_ = nullptr; + Reopen(); + } + + ~CuckooTableDBTest() override { + delete db_; + EXPECT_OK(DestroyDB(dbname_, Options())); + } + + Options CurrentOptions() { + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.table_factory.reset(NewCuckooTableFactory()); + options.memtable_factory.reset(NewHashLinkListRepFactory(4, 0, 3, true)); + options.allow_mmap_reads = true; + options.create_if_missing = true; + options.allow_concurrent_memtable_write = false; + return options; + } + + DBImpl* dbfull() { return static_cast_with_check(db_); } + + // The following util methods are copied from plain_table_db_test. + void Reopen(Options* options = nullptr) { + delete db_; + db_ = nullptr; + Options opts; + if (options != nullptr) { + opts = *options; + } else { + opts = CurrentOptions(); + opts.create_if_missing = true; + } + ASSERT_OK(DB::Open(opts, dbname_, &db_)); + } + + void DestroyAndReopen(Options* options) { + assert(options); + ASSERT_OK(db_->Close()); + delete db_; + db_ = nullptr; + ASSERT_OK(DestroyDB(dbname_, *options)); + Reopen(options); + } + + Status Put(const Slice& k, const Slice& v) { + return db_->Put(WriteOptions(), k, v); + } + + Status Delete(const std::string& k) { return db_->Delete(WriteOptions(), k); } + + std::string Get(const std::string& k) { + ReadOptions options; + std::string result; + Status s = db_->Get(options, k, &result); + if (s.IsNotFound()) { + result = "NOT_FOUND"; + } else if (!s.ok()) { + result = s.ToString(); + } + return result; + } + + int NumTableFilesAtLevel(int level) { + std::string property; + EXPECT_TRUE(db_->GetProperty( + "rocksdb.num-files-at-level" + std::to_string(level), &property)); + return atoi(property.c_str()); + } + + // Return spread of files per level + std::string FilesPerLevel() { + std::string result; + size_t last_non_zero_offset = 0; + for (int level = 0; level < db_->NumberLevels(); level++) { + int f = NumTableFilesAtLevel(level); + char buf[100]; + snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f); + result += buf; + if (f > 0) { + last_non_zero_offset = result.size(); + } + } + result.resize(last_non_zero_offset); + return result; + } +}; + +TEST_F(CuckooTableDBTest, Flush) { + // Try with empty DB first. + ASSERT_TRUE(dbfull() != nullptr); + ASSERT_EQ("NOT_FOUND", Get("key2")); + + // Add some values to db. + Options options = CurrentOptions(); + Reopen(&options); + + ASSERT_OK(Put("key1", "v1")); + ASSERT_OK(Put("key2", "v2")); + ASSERT_OK(Put("key3", "v3")); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + + TablePropertiesCollection ptc; + ASSERT_OK(reinterpret_cast(dbfull())->GetPropertiesOfAllTables(&ptc)); + VerifySstUniqueIds(ptc); + ASSERT_EQ(1U, ptc.size()); + ASSERT_EQ(3U, ptc.begin()->second->num_entries); + ASSERT_EQ("1", FilesPerLevel()); + + ASSERT_EQ("v1", Get("key1")); + ASSERT_EQ("v2", Get("key2")); + ASSERT_EQ("v3", Get("key3")); + ASSERT_EQ("NOT_FOUND", Get("key4")); + + // Now add more keys and flush. + ASSERT_OK(Put("key4", "v4")); + ASSERT_OK(Put("key5", "v5")); + ASSERT_OK(Put("key6", "v6")); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + + ASSERT_OK(reinterpret_cast(dbfull())->GetPropertiesOfAllTables(&ptc)); + VerifySstUniqueIds(ptc); + ASSERT_EQ(2U, ptc.size()); + auto row = ptc.begin(); + ASSERT_EQ(3U, row->second->num_entries); + ASSERT_EQ(3U, (++row)->second->num_entries); + ASSERT_EQ("2", FilesPerLevel()); + ASSERT_EQ("v1", Get("key1")); + ASSERT_EQ("v2", Get("key2")); + ASSERT_EQ("v3", Get("key3")); + ASSERT_EQ("v4", Get("key4")); + ASSERT_EQ("v5", Get("key5")); + ASSERT_EQ("v6", Get("key6")); + + ASSERT_OK(Delete("key6")); + ASSERT_OK(Delete("key5")); + ASSERT_OK(Delete("key4")); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + ASSERT_OK(reinterpret_cast(dbfull())->GetPropertiesOfAllTables(&ptc)); + VerifySstUniqueIds(ptc); + ASSERT_EQ(3U, ptc.size()); + row = ptc.begin(); + ASSERT_EQ(3U, row->second->num_entries); + ASSERT_EQ(3U, (++row)->second->num_entries); + ASSERT_EQ(3U, (++row)->second->num_entries); + ASSERT_EQ("3", FilesPerLevel()); + ASSERT_EQ("v1", Get("key1")); + ASSERT_EQ("v2", Get("key2")); + ASSERT_EQ("v3", Get("key3")); + ASSERT_EQ("NOT_FOUND", Get("key4")); + ASSERT_EQ("NOT_FOUND", Get("key5")); + ASSERT_EQ("NOT_FOUND", Get("key6")); +} + +TEST_F(CuckooTableDBTest, FlushWithDuplicateKeys) { + Options options = CurrentOptions(); + Reopen(&options); + ASSERT_OK(Put("key1", "v1")); + ASSERT_OK(Put("key2", "v2")); + ASSERT_OK(Put("key1", "v3")); // Duplicate + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + + TablePropertiesCollection ptc; + ASSERT_OK(reinterpret_cast(dbfull())->GetPropertiesOfAllTables(&ptc)); + VerifySstUniqueIds(ptc); + ASSERT_EQ(1U, ptc.size()); + ASSERT_EQ(2U, ptc.begin()->second->num_entries); + ASSERT_EQ("1", FilesPerLevel()); + ASSERT_EQ("v3", Get("key1")); + ASSERT_EQ("v2", Get("key2")); +} + +namespace { +static std::string Key(int i) { + char buf[100]; + snprintf(buf, sizeof(buf), "key_______%06d", i); + return std::string(buf); +} +static std::string Uint64Key(uint64_t i) { + std::string str; + str.resize(8); + memcpy(&str[0], static_cast(&i), 8); + return str; +} +} // namespace. + +TEST_F(CuckooTableDBTest, Uint64Comparator) { + Options options = CurrentOptions(); + options.comparator = test::Uint64Comparator(); + DestroyAndReopen(&options); + + ASSERT_OK(Put(Uint64Key(1), "v1")); + ASSERT_OK(Put(Uint64Key(2), "v2")); + ASSERT_OK(Put(Uint64Key(3), "v3")); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + + ASSERT_EQ("v1", Get(Uint64Key(1))); + ASSERT_EQ("v2", Get(Uint64Key(2))); + ASSERT_EQ("v3", Get(Uint64Key(3))); + ASSERT_EQ("NOT_FOUND", Get(Uint64Key(4))); + + // Add more keys. + ASSERT_OK(Delete(Uint64Key(2))); // Delete. + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + ASSERT_OK(Put(Uint64Key(3), "v0")); // Update. + ASSERT_OK(Put(Uint64Key(4), "v4")); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + ASSERT_EQ("v1", Get(Uint64Key(1))); + ASSERT_EQ("NOT_FOUND", Get(Uint64Key(2))); + ASSERT_EQ("v0", Get(Uint64Key(3))); + ASSERT_EQ("v4", Get(Uint64Key(4))); +} + +TEST_F(CuckooTableDBTest, CompactionIntoMultipleFiles) { + // Create a big L0 file and check it compacts into multiple files in L1. + Options options = CurrentOptions(); + options.write_buffer_size = 270 << 10; + // Two SST files should be created, each containing 14 keys. + // Number of buckets will be 16. Total size ~156 KB. + options.target_file_size_base = 160 << 10; + Reopen(&options); + + // Write 28 values, each 10016 B ~ 10KB + for (int idx = 0; idx < 28; ++idx) { + ASSERT_OK(Put(Key(idx), std::string(10000, 'a' + char(idx)))); + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ("1", FilesPerLevel()); + + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr, + true /* disallow trivial move */)); + ASSERT_EQ("0,2", FilesPerLevel()); + for (int idx = 0; idx < 28; ++idx) { + ASSERT_EQ(std::string(10000, 'a' + char(idx)), Get(Key(idx))); + } +} + +TEST_F(CuckooTableDBTest, SameKeyInsertedInTwoDifferentFilesAndCompacted) { + // Insert same key twice so that they go to different SST files. Then wait for + // compaction and check if the latest value is stored and old value removed. + Options options = CurrentOptions(); + options.write_buffer_size = 100 << 10; // 100KB + options.level0_file_num_compaction_trigger = 2; + Reopen(&options); + + // Write 11 values, each 10016 B + for (int idx = 0; idx < 11; ++idx) { + ASSERT_OK(Put(Key(idx), std::string(10000, 'a'))); + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ("1", FilesPerLevel()); + + // Generate one more file in level-0, and should trigger level-0 compaction + for (int idx = 0; idx < 11; ++idx) { + ASSERT_OK(Put(Key(idx), std::string(10000, 'a' + char(idx)))); + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr)); + + ASSERT_EQ("0,1", FilesPerLevel()); + for (int idx = 0; idx < 11; ++idx) { + ASSERT_EQ(std::string(10000, 'a' + char(idx)), Get(Key(idx))); + } +} + +TEST_F(CuckooTableDBTest, AdaptiveTable) { + Options options = CurrentOptions(); + + // Ensure options compatible with PlainTable + options.prefix_extractor.reset(NewCappedPrefixTransform(8)); + + // Write some keys using cuckoo table. + options.table_factory.reset(NewCuckooTableFactory()); + Reopen(&options); + + ASSERT_OK(Put("key1", "v1")); + ASSERT_OK(Put("key2", "v2")); + ASSERT_OK(Put("key3", "v3")); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + + // Write some keys using plain table. + std::shared_ptr block_based_factory( + NewBlockBasedTableFactory()); + std::shared_ptr plain_table_factory(NewPlainTableFactory()); + std::shared_ptr cuckoo_table_factory(NewCuckooTableFactory()); + options.create_if_missing = false; + options.table_factory.reset( + NewAdaptiveTableFactory(plain_table_factory, block_based_factory, + plain_table_factory, cuckoo_table_factory)); + Reopen(&options); + ASSERT_OK(Put("key4", "v4")); + ASSERT_OK(Put("key1", "v5")); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + + // Write some keys using block based table. + options.table_factory.reset( + NewAdaptiveTableFactory(block_based_factory, block_based_factory, + plain_table_factory, cuckoo_table_factory)); + Reopen(&options); + ASSERT_OK(Put("key5", "v6")); + ASSERT_OK(Put("key2", "v7")); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + + ASSERT_EQ("v5", Get("key1")); + ASSERT_EQ("v7", Get("key2")); + ASSERT_EQ("v3", Get("key3")); + ASSERT_EQ("v4", Get("key4")); + ASSERT_EQ("v6", Get("key5")); +} +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + if (ROCKSDB_NAMESPACE::port::kLittleEndian) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); + } else { + fprintf(stderr, "SKIPPED as Cuckoo table doesn't support Big Endian\n"); + return 0; + } +} + diff --git a/librocksdb-sys/rocksdb/db/db_basic_test.cc b/librocksdb-sys/rocksdb/db/db_basic_test.cc new file mode 100644 index 0000000..063b998 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_basic_test.cc @@ -0,0 +1,4777 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include + +#include "db/db_test_util.h" +#include "options/options_helper.h" +#include "port/stack_trace.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/flush_block_policy.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/perf_context.h" +#include "rocksdb/table.h" +#include "rocksdb/utilities/debug.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/block_based/block_builder.h" +#include "test_util/sync_point.h" +#include "util/file_checksum_helper.h" +#include "util/random.h" +#include "utilities/counted_fs.h" +#include "utilities/fault_injection_env.h" +#include "utilities/merge_operators.h" +#include "utilities/merge_operators/string_append/stringappend.h" + +namespace ROCKSDB_NAMESPACE { + +static bool enable_io_uring = true; +extern "C" bool RocksDbIOUringEnable() { return enable_io_uring; } + +class DBBasicTest : public DBTestBase { + public: + DBBasicTest() : DBTestBase("db_basic_test", /*env_do_fsync=*/false) {} +}; + +TEST_F(DBBasicTest, OpenWhenOpen) { + Options options = CurrentOptions(); + options.env = env_; + DB* db2 = nullptr; + Status s = DB::Open(options, dbname_, &db2); + ASSERT_NOK(s) << [db2]() { + delete db2; + return "db2 open: ok"; + }(); + ASSERT_EQ(Status::Code::kIOError, s.code()); + ASSERT_EQ(Status::SubCode::kNone, s.subcode()); + ASSERT_TRUE(strstr(s.getState(), "lock ") != nullptr); + + delete db2; +} + +TEST_F(DBBasicTest, EnableDirectIOWithZeroBuf) { + if (!IsDirectIOSupported()) { + ROCKSDB_GTEST_BYPASS("Direct IO not supported"); + return; + } + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.use_direct_io_for_flush_and_compaction = true; + options.writable_file_max_buffer_size = 0; + ASSERT_TRUE(TryReopen(options).IsInvalidArgument()); + + options.writable_file_max_buffer_size = 1024; + Reopen(options); + const std::unordered_map new_db_opts = { + {"writable_file_max_buffer_size", "0"}}; + ASSERT_TRUE(db_->SetDBOptions(new_db_opts).IsInvalidArgument()); +} + +TEST_F(DBBasicTest, UniqueSession) { + Options options = CurrentOptions(); + std::string sid1, sid2, sid3, sid4; + + ASSERT_OK(db_->GetDbSessionId(sid1)); + Reopen(options); + ASSERT_OK(db_->GetDbSessionId(sid2)); + ASSERT_OK(Put("foo", "v1")); + ASSERT_OK(db_->GetDbSessionId(sid4)); + Reopen(options); + ASSERT_OK(db_->GetDbSessionId(sid3)); + + ASSERT_NE(sid1, sid2); + ASSERT_NE(sid1, sid3); + ASSERT_NE(sid2, sid3); + + ASSERT_EQ(sid2, sid4); + + // Expected compact format for session ids (see notes in implementation) + TestRegex expected("[0-9A-Z]{20}"); + EXPECT_MATCHES_REGEX(sid1, expected); + EXPECT_MATCHES_REGEX(sid2, expected); + EXPECT_MATCHES_REGEX(sid3, expected); + + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + ASSERT_OK(db_->GetDbSessionId(sid1)); + // Test uniqueness between readonly open (sid1) and regular open (sid3) + ASSERT_NE(sid1, sid3); + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + ASSERT_OK(db_->GetDbSessionId(sid2)); + ASSERT_EQ("v1", Get("foo")); + ASSERT_OK(db_->GetDbSessionId(sid3)); + + ASSERT_NE(sid1, sid2); + + ASSERT_EQ(sid2, sid3); + + CreateAndReopenWithCF({"goku"}, options); + ASSERT_OK(db_->GetDbSessionId(sid1)); + ASSERT_OK(Put("bar", "e1")); + ASSERT_OK(db_->GetDbSessionId(sid2)); + ASSERT_EQ("e1", Get("bar")); + ASSERT_OK(db_->GetDbSessionId(sid3)); + ReopenWithColumnFamilies({"default", "goku"}, options); + ASSERT_OK(db_->GetDbSessionId(sid4)); + + ASSERT_EQ(sid1, sid2); + ASSERT_EQ(sid2, sid3); + + ASSERT_NE(sid1, sid4); +} + +TEST_F(DBBasicTest, ReadOnlyDB) { + ASSERT_OK(Put("foo", "v1")); + ASSERT_OK(Put("bar", "v2")); + ASSERT_OK(Put("foo", "v3")); + Close(); + + auto verify_one_iter = [&](Iterator* iter) { + int count = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_OK(iter->status()); + ++count; + } + // Always expect two keys: "foo" and "bar" + ASSERT_EQ(count, 2); + }; + + auto verify_all_iters = [&]() { + Iterator* iter = db_->NewIterator(ReadOptions()); + verify_one_iter(iter); + delete iter; + + std::vector iters; + ASSERT_OK(db_->NewIterators(ReadOptions(), + {dbfull()->DefaultColumnFamily()}, &iters)); + ASSERT_EQ(static_cast(1), iters.size()); + verify_one_iter(iters[0]); + delete iters[0]; + }; + + auto options = CurrentOptions(); + assert(options.env == env_); + ASSERT_OK(ReadOnlyReopen(options)); + ASSERT_EQ("v3", Get("foo")); + ASSERT_EQ("v2", Get("bar")); + verify_all_iters(); + Close(); + + // Reopen and flush memtable. + Reopen(options); + ASSERT_OK(Flush()); + Close(); + // Now check keys in read only mode. + ASSERT_OK(ReadOnlyReopen(options)); + ASSERT_EQ("v3", Get("foo")); + ASSERT_EQ("v2", Get("bar")); + verify_all_iters(); + ASSERT_TRUE(db_->SyncWAL().IsNotSupported()); +} + +// TODO akanksha: Update the test to check that combination +// does not actually write to FS (use open read-only with +// CompositeEnvWrapper+ReadOnlyFileSystem). +TEST_F(DBBasicTest, DISABLED_ReadOnlyDBWithWriteDBIdToManifestSet) { + ASSERT_OK(Put("foo", "v1")); + ASSERT_OK(Put("bar", "v2")); + ASSERT_OK(Put("foo", "v3")); + Close(); + + auto options = CurrentOptions(); + options.write_dbid_to_manifest = true; + assert(options.env == env_); + ASSERT_OK(ReadOnlyReopen(options)); + std::string db_id1; + ASSERT_OK(db_->GetDbIdentity(db_id1)); + ASSERT_EQ("v3", Get("foo")); + ASSERT_EQ("v2", Get("bar")); + Iterator* iter = db_->NewIterator(ReadOptions()); + int count = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_OK(iter->status()); + ++count; + } + ASSERT_EQ(count, 2); + delete iter; + Close(); + + // Reopen and flush memtable. + Reopen(options); + ASSERT_OK(Flush()); + Close(); + // Now check keys in read only mode. + ASSERT_OK(ReadOnlyReopen(options)); + ASSERT_EQ("v3", Get("foo")); + ASSERT_EQ("v2", Get("bar")); + ASSERT_TRUE(db_->SyncWAL().IsNotSupported()); + std::string db_id2; + ASSERT_OK(db_->GetDbIdentity(db_id2)); + ASSERT_EQ(db_id1, db_id2); +} + +TEST_F(DBBasicTest, CompactedDB) { + const uint64_t kFileSize = 1 << 20; + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.write_buffer_size = kFileSize; + options.target_file_size_base = kFileSize; + options.max_bytes_for_level_base = 1 << 30; + options.compression = kNoCompression; + Reopen(options); + // 1 L0 file, use CompactedDB if max_open_files = -1 + ASSERT_OK(Put("aaa", DummyString(kFileSize / 2, '1'))); + ASSERT_OK(Flush()); + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + Status s = Put("new", "value"); + ASSERT_EQ(s.ToString(), + "Not implemented: Not supported operation in read only mode."); + ASSERT_EQ(DummyString(kFileSize / 2, '1'), Get("aaa")); + Close(); + options.max_open_files = -1; + ASSERT_OK(ReadOnlyReopen(options)); + s = Put("new", "value"); + ASSERT_EQ(s.ToString(), + "Not implemented: Not supported in compacted db mode."); + ASSERT_EQ(DummyString(kFileSize / 2, '1'), Get("aaa")); + Close(); + Reopen(options); + // Add more L0 files + ASSERT_OK(Put("bbb", DummyString(kFileSize / 2, '2'))); + ASSERT_OK(Flush()); + ASSERT_OK(Put("aaa", DummyString(kFileSize / 2, 'a'))); + ASSERT_OK(Flush()); + ASSERT_OK(Put("bbb", DummyString(kFileSize / 2, 'b'))); + ASSERT_OK(Put("eee", DummyString(kFileSize / 2, 'e'))); + ASSERT_OK(Flush()); + ASSERT_OK(Put("something_not_flushed", "x")); + Close(); + + ASSERT_OK(ReadOnlyReopen(options)); + // Fallback to read-only DB + s = Put("new", "value"); + ASSERT_EQ(s.ToString(), + "Not implemented: Not supported operation in read only mode."); + + // TODO: validate that other write ops return NotImplemented + // (DBImplReadOnly is missing some overrides) + + // Ensure no deadlock on flush triggered by another API function + // (Old deadlock bug depends on something_not_flushed above.) + std::vector files; + uint64_t manifest_file_size; + ASSERT_OK(db_->GetLiveFiles(files, &manifest_file_size, /*flush*/ true)); + LiveFilesStorageInfoOptions lfsi_opts; + lfsi_opts.wal_size_for_flush = 0; // always + std::vector files2; + ASSERT_OK(db_->GetLiveFilesStorageInfo(lfsi_opts, &files2)); + + Close(); + + // Full compaction + Reopen(options); + // Add more keys + ASSERT_OK(Put("fff", DummyString(kFileSize / 2, 'f'))); + ASSERT_OK(Put("hhh", DummyString(kFileSize / 2, 'h'))); + ASSERT_OK(Put("iii", DummyString(kFileSize / 2, 'i'))); + ASSERT_OK(Put("jjj", DummyString(kFileSize / 2, 'j'))); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(3, NumTableFilesAtLevel(1)); + Close(); + + // CompactedDB + ASSERT_OK(ReadOnlyReopen(options)); + s = Put("new", "value"); + ASSERT_EQ(s.ToString(), + "Not implemented: Not supported in compacted db mode."); + ASSERT_EQ("NOT_FOUND", Get("abc")); + ASSERT_EQ(DummyString(kFileSize / 2, 'a'), Get("aaa")); + ASSERT_EQ(DummyString(kFileSize / 2, 'b'), Get("bbb")); + ASSERT_EQ("NOT_FOUND", Get("ccc")); + ASSERT_EQ(DummyString(kFileSize / 2, 'e'), Get("eee")); + ASSERT_EQ(DummyString(kFileSize / 2, 'f'), Get("fff")); + ASSERT_EQ("NOT_FOUND", Get("ggg")); + ASSERT_EQ(DummyString(kFileSize / 2, 'h'), Get("hhh")); + ASSERT_EQ(DummyString(kFileSize / 2, 'i'), Get("iii")); + ASSERT_EQ(DummyString(kFileSize / 2, 'j'), Get("jjj")); + ASSERT_EQ("NOT_FOUND", Get("kkk")); + + // TODO: validate that other write ops return NotImplemented + // (CompactedDB is missing some overrides) + + // Ensure no deadlock on flush triggered by another API function + ASSERT_OK(db_->GetLiveFiles(files, &manifest_file_size, /*flush*/ true)); + ASSERT_OK(db_->GetLiveFilesStorageInfo(lfsi_opts, &files2)); + + // MultiGet + std::vector values; + std::vector status_list = dbfull()->MultiGet( + ReadOptions(), + std::vector({Slice("aaa"), Slice("ccc"), Slice("eee"), + Slice("ggg"), Slice("iii"), Slice("kkk")}), + &values); + ASSERT_EQ(status_list.size(), static_cast(6)); + ASSERT_EQ(values.size(), static_cast(6)); + ASSERT_OK(status_list[0]); + ASSERT_EQ(DummyString(kFileSize / 2, 'a'), values[0]); + ASSERT_TRUE(status_list[1].IsNotFound()); + ASSERT_OK(status_list[2]); + ASSERT_EQ(DummyString(kFileSize / 2, 'e'), values[2]); + ASSERT_TRUE(status_list[3].IsNotFound()); + ASSERT_OK(status_list[4]); + ASSERT_EQ(DummyString(kFileSize / 2, 'i'), values[4]); + ASSERT_TRUE(status_list[5].IsNotFound()); + + Reopen(options); + // Add a key + ASSERT_OK(Put("fff", DummyString(kFileSize / 2, 'f'))); + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + s = Put("new", "value"); + ASSERT_EQ(s.ToString(), + "Not implemented: Not supported operation in read only mode."); +} + +TEST_F(DBBasicTest, LevelLimitReopen) { + Options options = CurrentOptions(); + CreateAndReopenWithCF({"pikachu"}, options); + + const std::string value(1024 * 1024, ' '); + int i = 0; + while (NumTableFilesAtLevel(2, 1) == 0) { + ASSERT_OK(Put(1, Key(i++), value)); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + + options.num_levels = 1; + options.max_bytes_for_level_multiplier_additional.resize(1, 1); + Status s = TryReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_EQ(s.IsInvalidArgument(), true); + ASSERT_EQ(s.ToString(), + "Invalid argument: db has more levels than options.num_levels"); + + options.num_levels = 10; + options.max_bytes_for_level_multiplier_additional.resize(10, 1); + ASSERT_OK(TryReopenWithColumnFamilies({"default", "pikachu"}, options)); +} + +TEST_F(DBBasicTest, PutDeleteGet) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "foo", "v1")); + ASSERT_EQ("v1", Get(1, "foo")); + ASSERT_OK(Put(1, "foo", "v2")); + ASSERT_EQ("v2", Get(1, "foo")); + ASSERT_OK(Delete(1, "foo")); + ASSERT_EQ("NOT_FOUND", Get(1, "foo")); + } while (ChangeOptions()); +} + +TEST_F(DBBasicTest, PutSingleDeleteGet) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "foo", "v1")); + ASSERT_EQ("v1", Get(1, "foo")); + ASSERT_OK(Put(1, "foo2", "v2")); + ASSERT_EQ("v2", Get(1, "foo2")); + ASSERT_OK(SingleDelete(1, "foo")); + ASSERT_EQ("NOT_FOUND", Get(1, "foo")); + // Ski FIFO and universal compaction because they do not apply to the test + // case. Skip MergePut because single delete does not get removed when it + // encounters a merge. + } while (ChangeOptions(kSkipFIFOCompaction | kSkipUniversalCompaction | + kSkipMergePut)); +} + +TEST_F(DBBasicTest, EmptyFlush) { + // It is possible to produce empty flushes when using single deletes. Tests + // whether empty flushes cause issues. + do { + Random rnd(301); + + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(1, "a", Slice())); + ASSERT_OK(SingleDelete(1, "a")); + ASSERT_OK(Flush(1)); + + ASSERT_EQ("[ ]", AllEntriesFor("a", 1)); + // Skip FIFO and universal compaction as they do not apply to the test + // case. Skip MergePut because merges cannot be combined with single + // deletions. + } while (ChangeOptions(kSkipFIFOCompaction | kSkipUniversalCompaction | + kSkipMergePut)); +} + +TEST_F(DBBasicTest, GetFromVersions) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "foo", "v1")); + ASSERT_OK(Flush(1)); + ASSERT_EQ("v1", Get(1, "foo")); + ASSERT_EQ("NOT_FOUND", Get(0, "foo")); + } while (ChangeOptions()); +} + +TEST_F(DBBasicTest, GetSnapshot) { + anon::OptionsOverride options_override; + options_override.skip_policy = kSkipNoSnapshot; + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions(options_override)); + // Try with both a short key and a long key + for (int i = 0; i < 2; i++) { + std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x'); + ASSERT_OK(Put(1, key, "v1")); + const Snapshot* s1 = db_->GetSnapshot(); + ASSERT_OK(Put(1, key, "v2")); + ASSERT_EQ("v2", Get(1, key)); + ASSERT_EQ("v1", Get(1, key, s1)); + ASSERT_OK(Flush(1)); + ASSERT_EQ("v2", Get(1, key)); + ASSERT_EQ("v1", Get(1, key, s1)); + db_->ReleaseSnapshot(s1); + } + } while (ChangeOptions()); +} + +TEST_F(DBBasicTest, CheckLock) { + do { + DB* localdb = nullptr; + Options options = CurrentOptions(); + ASSERT_OK(TryReopen(options)); + + // second open should fail + Status s = DB::Open(options, dbname_, &localdb); + ASSERT_NOK(s) << [localdb]() { + delete localdb; + return "localdb open: ok"; + }(); +#ifdef OS_LINUX + ASSERT_TRUE(s.ToString().find("lock ") != std::string::npos); +#endif // OS_LINUX + } while (ChangeCompactOptions()); +} + +TEST_F(DBBasicTest, FlushMultipleMemtable) { + do { + Options options = CurrentOptions(); + WriteOptions writeOpt = WriteOptions(); + writeOpt.disableWAL = true; + options.max_write_buffer_number = 4; + options.min_write_buffer_number_to_merge = 3; + options.max_write_buffer_size_to_maintain = -1; + CreateAndReopenWithCF({"pikachu"}, options); + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v1")); + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1")); + + ASSERT_EQ("v1", Get(1, "foo")); + ASSERT_EQ("v1", Get(1, "bar")); + ASSERT_OK(Flush(1)); + } while (ChangeCompactOptions()); +} + +TEST_F(DBBasicTest, FlushEmptyColumnFamily) { + // Block flush thread and disable compaction thread + env_->SetBackgroundThreads(1, Env::HIGH); + env_->SetBackgroundThreads(1, Env::LOW); + test::SleepingBackgroundTask sleeping_task_low; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::LOW); + test::SleepingBackgroundTask sleeping_task_high; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, + &sleeping_task_high, Env::Priority::HIGH); + + Options options = CurrentOptions(); + // disable compaction + options.disable_auto_compactions = true; + WriteOptions writeOpt = WriteOptions(); + writeOpt.disableWAL = true; + options.max_write_buffer_number = 2; + options.min_write_buffer_number_to_merge = 1; + options.max_write_buffer_size_to_maintain = + static_cast(options.write_buffer_size); + CreateAndReopenWithCF({"pikachu"}, options); + + // Compaction can still go through even if no thread can flush the + // mem table. + ASSERT_OK(Flush(0)); + ASSERT_OK(Flush(1)); + + // Insert can go through + ASSERT_OK(dbfull()->Put(writeOpt, handles_[0], "foo", "v1")); + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1")); + + ASSERT_EQ("v1", Get(0, "foo")); + ASSERT_EQ("v1", Get(1, "bar")); + + sleeping_task_high.WakeUp(); + sleeping_task_high.WaitUntilDone(); + + // Flush can still go through. + ASSERT_OK(Flush(0)); + ASSERT_OK(Flush(1)); + + sleeping_task_low.WakeUp(); + sleeping_task_low.WaitUntilDone(); +} + +TEST_F(DBBasicTest, Flush) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + WriteOptions writeOpt = WriteOptions(); + writeOpt.disableWAL = true; + SetPerfLevel(kEnableTime); + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v1")); + // this will now also flush the last 2 writes + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1")); + + get_perf_context()->Reset(); + Get(1, "foo"); + ASSERT_TRUE((int)get_perf_context()->get_from_output_files_time > 0); + ASSERT_EQ(2, (int)get_perf_context()->get_read_bytes); + + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + ASSERT_EQ("v1", Get(1, "foo")); + ASSERT_EQ("v1", Get(1, "bar")); + + writeOpt.disableWAL = true; + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v2")); + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v2")); + ASSERT_OK(Flush(1)); + + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + ASSERT_EQ("v2", Get(1, "bar")); + get_perf_context()->Reset(); + ASSERT_EQ("v2", Get(1, "foo")); + ASSERT_TRUE((int)get_perf_context()->get_from_output_files_time > 0); + + writeOpt.disableWAL = false; + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v3")); + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v3")); + ASSERT_OK(Flush(1)); + + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + // 'foo' should be there because its put + // has WAL enabled. + ASSERT_EQ("v3", Get(1, "foo")); + ASSERT_EQ("v3", Get(1, "bar")); + + SetPerfLevel(kDisable); + } while (ChangeCompactOptions()); +} + +TEST_F(DBBasicTest, ManifestRollOver) { + do { + Options options; + options.max_manifest_file_size = 10; // 10 bytes + options = CurrentOptions(options); + CreateAndReopenWithCF({"pikachu"}, options); + { + ASSERT_OK(Put(1, "manifest_key1", std::string(1000, '1'))); + ASSERT_OK(Put(1, "manifest_key2", std::string(1000, '2'))); + ASSERT_OK(Put(1, "manifest_key3", std::string(1000, '3'))); + uint64_t manifest_before_flush = dbfull()->TEST_Current_Manifest_FileNo(); + ASSERT_OK(Flush(1)); // This should trigger LogAndApply. + uint64_t manifest_after_flush = dbfull()->TEST_Current_Manifest_FileNo(); + ASSERT_GT(manifest_after_flush, manifest_before_flush); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_GT(dbfull()->TEST_Current_Manifest_FileNo(), manifest_after_flush); + // check if a new manifest file got inserted or not. + ASSERT_EQ(std::string(1000, '1'), Get(1, "manifest_key1")); + ASSERT_EQ(std::string(1000, '2'), Get(1, "manifest_key2")); + ASSERT_EQ(std::string(1000, '3'), Get(1, "manifest_key3")); + } + } while (ChangeCompactOptions()); +} + +TEST_F(DBBasicTest, IdentityAcrossRestarts) { + constexpr size_t kMinIdSize = 10; + do { + for (bool with_manifest : {false, true}) { + std::string idfilename = IdentityFileName(dbname_); + std::string id1, tmp; + ASSERT_OK(db_->GetDbIdentity(id1)); + ASSERT_GE(id1.size(), kMinIdSize); + + Options options = CurrentOptions(); + options.write_dbid_to_manifest = with_manifest; + Reopen(options); + std::string id2; + ASSERT_OK(db_->GetDbIdentity(id2)); + // id2 should match id1 because identity was not regenerated + ASSERT_EQ(id1, id2); + ASSERT_OK(ReadFileToString(env_, idfilename, &tmp)); + ASSERT_EQ(tmp, id2); + + // Recover from deleted/missing IDENTITY + ASSERT_OK(env_->DeleteFile(idfilename)); + Reopen(options); + std::string id3; + ASSERT_OK(db_->GetDbIdentity(id3)); + if (with_manifest) { + // id3 should match id1 because identity was restored from manifest + ASSERT_EQ(id1, id3); + } else { + // id3 should NOT match id1 because identity was regenerated + ASSERT_NE(id1, id3); + ASSERT_GE(id3.size(), kMinIdSize); + } + ASSERT_OK(ReadFileToString(env_, idfilename, &tmp)); + ASSERT_EQ(tmp, id3); + + // Recover from truncated IDENTITY + { + std::unique_ptr w; + ASSERT_OK(env_->NewWritableFile(idfilename, &w, EnvOptions())); + ASSERT_OK(w->Close()); + } + Reopen(options); + std::string id4; + ASSERT_OK(db_->GetDbIdentity(id4)); + if (with_manifest) { + // id4 should match id1 because identity was restored from manifest + ASSERT_EQ(id1, id4); + } else { + // id4 should NOT match id1 because identity was regenerated + ASSERT_NE(id1, id4); + ASSERT_GE(id4.size(), kMinIdSize); + } + ASSERT_OK(ReadFileToString(env_, idfilename, &tmp)); + ASSERT_EQ(tmp, id4); + + // Recover from overwritten IDENTITY + std::string silly_id = "asdf123456789"; + { + std::unique_ptr w; + ASSERT_OK(env_->NewWritableFile(idfilename, &w, EnvOptions())); + ASSERT_OK(w->Append(silly_id)); + ASSERT_OK(w->Close()); + } + Reopen(options); + std::string id5; + ASSERT_OK(db_->GetDbIdentity(id5)); + if (with_manifest) { + // id4 should match id1 because identity was restored from manifest + ASSERT_EQ(id1, id5); + } else { + ASSERT_EQ(id5, silly_id); + } + ASSERT_OK(ReadFileToString(env_, idfilename, &tmp)); + ASSERT_EQ(tmp, id5); + } + } while (ChangeCompactOptions()); +} + +TEST_F(DBBasicTest, LockFileRecovery) { + Options options = CurrentOptions(); + // Regardless of best_efforts_recovery + for (bool ber : {false, true}) { + options.best_efforts_recovery = ber; + DestroyAndReopen(options); + std::string id1, id2; + ASSERT_OK(db_->GetDbIdentity(id1)); + Close(); + + // Should be OK to re-open DB after lock file deleted + std::string lockfilename = LockFileName(dbname_); + ASSERT_OK(env_->DeleteFile(lockfilename)); + Reopen(options); + + // Should be same DB as before + ASSERT_OK(db_->GetDbIdentity(id2)); + ASSERT_EQ(id1, id2); + } +} + +TEST_F(DBBasicTest, Snapshot) { + env_->SetMockSleep(); + anon::OptionsOverride options_override; + options_override.skip_policy = kSkipNoSnapshot; + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions(options_override)); + ASSERT_OK(Put(0, "foo", "0v1")); + ASSERT_OK(Put(1, "foo", "1v1")); + + const Snapshot* s1 = db_->GetSnapshot(); + ASSERT_EQ(1U, GetNumSnapshots()); + uint64_t time_snap1 = GetTimeOldestSnapshots(); + ASSERT_GT(time_snap1, 0U); + ASSERT_EQ(GetSequenceOldestSnapshots(), s1->GetSequenceNumber()); + ASSERT_OK(Put(0, "foo", "0v2")); + ASSERT_OK(Put(1, "foo", "1v2")); + + env_->MockSleepForSeconds(1); + + const Snapshot* s2 = db_->GetSnapshot(); + ASSERT_EQ(2U, GetNumSnapshots()); + ASSERT_EQ(time_snap1, GetTimeOldestSnapshots()); + ASSERT_EQ(GetSequenceOldestSnapshots(), s1->GetSequenceNumber()); + ASSERT_OK(Put(0, "foo", "0v3")); + ASSERT_OK(Put(1, "foo", "1v3")); + + { + ManagedSnapshot s3(db_); + ASSERT_EQ(3U, GetNumSnapshots()); + ASSERT_EQ(time_snap1, GetTimeOldestSnapshots()); + ASSERT_EQ(GetSequenceOldestSnapshots(), s1->GetSequenceNumber()); + + ASSERT_OK(Put(0, "foo", "0v4")); + ASSERT_OK(Put(1, "foo", "1v4")); + ASSERT_EQ("0v1", Get(0, "foo", s1)); + ASSERT_EQ("1v1", Get(1, "foo", s1)); + ASSERT_EQ("0v2", Get(0, "foo", s2)); + ASSERT_EQ("1v2", Get(1, "foo", s2)); + ASSERT_EQ("0v3", Get(0, "foo", s3.snapshot())); + ASSERT_EQ("1v3", Get(1, "foo", s3.snapshot())); + ASSERT_EQ("0v4", Get(0, "foo")); + ASSERT_EQ("1v4", Get(1, "foo")); + } + + ASSERT_EQ(2U, GetNumSnapshots()); + ASSERT_EQ(time_snap1, GetTimeOldestSnapshots()); + ASSERT_EQ(GetSequenceOldestSnapshots(), s1->GetSequenceNumber()); + ASSERT_EQ("0v1", Get(0, "foo", s1)); + ASSERT_EQ("1v1", Get(1, "foo", s1)); + ASSERT_EQ("0v2", Get(0, "foo", s2)); + ASSERT_EQ("1v2", Get(1, "foo", s2)); + ASSERT_EQ("0v4", Get(0, "foo")); + ASSERT_EQ("1v4", Get(1, "foo")); + + db_->ReleaseSnapshot(s1); + ASSERT_EQ("0v2", Get(0, "foo", s2)); + ASSERT_EQ("1v2", Get(1, "foo", s2)); + ASSERT_EQ("0v4", Get(0, "foo")); + ASSERT_EQ("1v4", Get(1, "foo")); + ASSERT_EQ(1U, GetNumSnapshots()); + ASSERT_LT(time_snap1, GetTimeOldestSnapshots()); + ASSERT_EQ(GetSequenceOldestSnapshots(), s2->GetSequenceNumber()); + + db_->ReleaseSnapshot(s2); + ASSERT_EQ(0U, GetNumSnapshots()); + ASSERT_EQ(GetSequenceOldestSnapshots(), 0); + ASSERT_EQ("0v4", Get(0, "foo")); + ASSERT_EQ("1v4", Get(1, "foo")); + } while (ChangeOptions()); +} + + +class DBBasicMultiConfigs : public DBBasicTest, + public ::testing::WithParamInterface { + public: + DBBasicMultiConfigs() { option_config_ = GetParam(); } + + static std::vector GenerateOptionConfigs() { + std::vector option_configs; + for (int option_config = kDefault; option_config < kEnd; ++option_config) { + if (!ShouldSkipOptions(option_config, kSkipFIFOCompaction)) { + option_configs.push_back(option_config); + } + } + return option_configs; + } +}; + +TEST_P(DBBasicMultiConfigs, CompactBetweenSnapshots) { + anon::OptionsOverride options_override; + options_override.skip_policy = kSkipNoSnapshot; + Options options = CurrentOptions(options_override); + options.disable_auto_compactions = true; + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + Random rnd(301); + FillLevels("a", "z", 1); + + ASSERT_OK(Put(1, "foo", "first")); + const Snapshot* snapshot1 = db_->GetSnapshot(); + ASSERT_OK(Put(1, "foo", "second")); + ASSERT_OK(Put(1, "foo", "third")); + ASSERT_OK(Put(1, "foo", "fourth")); + const Snapshot* snapshot2 = db_->GetSnapshot(); + ASSERT_OK(Put(1, "foo", "fifth")); + ASSERT_OK(Put(1, "foo", "sixth")); + + // All entries (including duplicates) exist + // before any compaction or flush is triggered. + ASSERT_EQ(AllEntriesFor("foo", 1), + "[ sixth, fifth, fourth, third, second, first ]"); + ASSERT_EQ("sixth", Get(1, "foo")); + ASSERT_EQ("fourth", Get(1, "foo", snapshot2)); + ASSERT_EQ("first", Get(1, "foo", snapshot1)); + + // After a flush, "second", "third" and "fifth" should + // be removed + ASSERT_OK(Flush(1)); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth, fourth, first ]"); + + // after we release the snapshot1, only two values left + db_->ReleaseSnapshot(snapshot1); + FillLevels("a", "z", 1); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr, + nullptr)); + + // We have only one valid snapshot snapshot2. Since snapshot1 is + // not valid anymore, "first" should be removed by a compaction. + ASSERT_EQ("sixth", Get(1, "foo")); + ASSERT_EQ("fourth", Get(1, "foo", snapshot2)); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth, fourth ]"); + + // after we release the snapshot2, only one value should be left + db_->ReleaseSnapshot(snapshot2); + FillLevels("a", "z", 1); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr, + nullptr)); + ASSERT_EQ("sixth", Get(1, "foo")); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth ]"); +} + +INSTANTIATE_TEST_CASE_P( + DBBasicMultiConfigs, DBBasicMultiConfigs, + ::testing::ValuesIn(DBBasicMultiConfigs::GenerateOptionConfigs())); + +TEST_F(DBBasicTest, DBOpen_Options) { + Options options = CurrentOptions(); + Close(); + Destroy(options); + + // Does not exist, and create_if_missing == false: error + DB* db = nullptr; + options.create_if_missing = false; + Status s = DB::Open(options, dbname_, &db); + ASSERT_TRUE(strstr(s.ToString().c_str(), "does not exist") != nullptr); + ASSERT_TRUE(db == nullptr); + + // Does not exist, and create_if_missing == true: OK + options.create_if_missing = true; + s = DB::Open(options, dbname_, &db); + ASSERT_OK(s); + ASSERT_TRUE(db != nullptr); + + delete db; + db = nullptr; + + // Does exist, and error_if_exists == true: error + options.create_if_missing = false; + options.error_if_exists = true; + s = DB::Open(options, dbname_, &db); + ASSERT_TRUE(strstr(s.ToString().c_str(), "exists") != nullptr); + ASSERT_TRUE(db == nullptr); + + // Does exist, and error_if_exists == false: OK + options.create_if_missing = true; + options.error_if_exists = false; + s = DB::Open(options, dbname_, &db); + ASSERT_OK(s); + ASSERT_TRUE(db != nullptr); + + delete db; + db = nullptr; +} + +TEST_F(DBBasicTest, CompactOnFlush) { + anon::OptionsOverride options_override; + options_override.skip_policy = kSkipNoSnapshot; + do { + Options options = CurrentOptions(options_override); + options.disable_auto_compactions = true; + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(1, "foo", "v1")); + ASSERT_OK(Flush(1)); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ v1 ]"); + + // Write two new keys + ASSERT_OK(Put(1, "a", "begin")); + ASSERT_OK(Put(1, "z", "end")); + ASSERT_OK(Flush(1)); + + // Case1: Delete followed by a put + ASSERT_OK(Delete(1, "foo")); + ASSERT_OK(Put(1, "foo", "v2")); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, DEL, v1 ]"); + + // After the current memtable is flushed, the DEL should + // have been removed + ASSERT_OK(Flush(1)); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, v1 ]"); + + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), handles_[1], + nullptr, nullptr)); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2 ]"); + + // Case 2: Delete followed by another delete + ASSERT_OK(Delete(1, "foo")); + ASSERT_OK(Delete(1, "foo")); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, DEL, v2 ]"); + ASSERT_OK(Flush(1)); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v2 ]"); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), handles_[1], + nullptr, nullptr)); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]"); + + // Case 3: Put followed by a delete + ASSERT_OK(Put(1, "foo", "v3")); + ASSERT_OK(Delete(1, "foo")); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v3 ]"); + ASSERT_OK(Flush(1)); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL ]"); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), handles_[1], + nullptr, nullptr)); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]"); + + // Case 4: Put followed by another Put + ASSERT_OK(Put(1, "foo", "v4")); + ASSERT_OK(Put(1, "foo", "v5")); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ v5, v4 ]"); + ASSERT_OK(Flush(1)); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ v5 ]"); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), handles_[1], + nullptr, nullptr)); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ v5 ]"); + + // clear database + ASSERT_OK(Delete(1, "foo")); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), handles_[1], + nullptr, nullptr)); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]"); + + // Case 5: Put followed by snapshot followed by another Put + // Both puts should remain. + ASSERT_OK(Put(1, "foo", "v6")); + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK(Put(1, "foo", "v7")); + ASSERT_OK(Flush(1)); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ v7, v6 ]"); + db_->ReleaseSnapshot(snapshot); + + // clear database + ASSERT_OK(Delete(1, "foo")); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), handles_[1], + nullptr, nullptr)); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]"); + + // Case 5: snapshot followed by a put followed by another Put + // Only the last put should remain. + const Snapshot* snapshot1 = db_->GetSnapshot(); + ASSERT_OK(Put(1, "foo", "v8")); + ASSERT_OK(Put(1, "foo", "v9")); + ASSERT_OK(Flush(1)); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ v9 ]"); + db_->ReleaseSnapshot(snapshot1); + } while (ChangeCompactOptions()); +} + +TEST_F(DBBasicTest, FlushOneColumnFamily) { + Options options = CurrentOptions(); + CreateAndReopenWithCF({"pikachu", "ilya", "muromec", "dobrynia", "nikitich", + "alyosha", "popovich"}, + options); + + ASSERT_OK(Put(0, "Default", "Default")); + ASSERT_OK(Put(1, "pikachu", "pikachu")); + ASSERT_OK(Put(2, "ilya", "ilya")); + ASSERT_OK(Put(3, "muromec", "muromec")); + ASSERT_OK(Put(4, "dobrynia", "dobrynia")); + ASSERT_OK(Put(5, "nikitich", "nikitich")); + ASSERT_OK(Put(6, "alyosha", "alyosha")); + ASSERT_OK(Put(7, "popovich", "popovich")); + + for (int i = 0; i < 8; ++i) { + ASSERT_OK(Flush(i)); + auto tables = ListTableFiles(env_, dbname_); + ASSERT_EQ(tables.size(), i + 1U); + } +} + +TEST_F(DBBasicTest, MultiGetSimple) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + SetPerfLevel(kEnableCount); + ASSERT_OK(Put(1, "k1", "v1")); + ASSERT_OK(Put(1, "k2", "v2")); + ASSERT_OK(Put(1, "k3", "v3")); + ASSERT_OK(Put(1, "k4", "v4")); + ASSERT_OK(Delete(1, "k4")); + ASSERT_OK(Put(1, "k5", "v5")); + ASSERT_OK(Delete(1, "no_key")); + + std::vector keys({"k1", "k2", "k3", "k4", "k5", "no_key"}); + + std::vector values(20, "Temporary data to be overwritten"); + std::vector cfs(keys.size(), handles_[1]); + + get_perf_context()->Reset(); + std::vector s = db_->MultiGet(ReadOptions(), cfs, keys, &values); + ASSERT_EQ(values.size(), keys.size()); + ASSERT_EQ(values[0], "v1"); + ASSERT_EQ(values[1], "v2"); + ASSERT_EQ(values[2], "v3"); + ASSERT_EQ(values[4], "v5"); + // four kv pairs * two bytes per value + ASSERT_EQ(8, (int)get_perf_context()->multiget_read_bytes); + + ASSERT_OK(s[0]); + ASSERT_OK(s[1]); + ASSERT_OK(s[2]); + ASSERT_TRUE(s[3].IsNotFound()); + ASSERT_OK(s[4]); + ASSERT_TRUE(s[5].IsNotFound()); + SetPerfLevel(kDisable); + } while (ChangeCompactOptions()); +} + +TEST_F(DBBasicTest, MultiGetEmpty) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + // Empty Key Set + std::vector keys; + std::vector values; + std::vector cfs; + std::vector s = db_->MultiGet(ReadOptions(), cfs, keys, &values); + ASSERT_EQ(s.size(), 0U); + + // Empty Database, Empty Key Set + Options options = CurrentOptions(); + options.create_if_missing = true; + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + s = db_->MultiGet(ReadOptions(), cfs, keys, &values); + ASSERT_EQ(s.size(), 0U); + + // Empty Database, Search for Keys + keys.resize(2); + keys[0] = "a"; + keys[1] = "b"; + cfs.push_back(handles_[0]); + cfs.push_back(handles_[1]); + s = db_->MultiGet(ReadOptions(), cfs, keys, &values); + ASSERT_EQ(static_cast(s.size()), 2); + ASSERT_TRUE(s[0].IsNotFound() && s[1].IsNotFound()); + } while (ChangeCompactOptions()); +} + +class DBBlockChecksumTest : public DBBasicTest, + public testing::WithParamInterface {}; + +INSTANTIATE_TEST_CASE_P(FormatVersions, DBBlockChecksumTest, + testing::ValuesIn(test::kFooterFormatVersionsToTest)); + +TEST_P(DBBlockChecksumTest, BlockChecksumTest) { + BlockBasedTableOptions table_options; + table_options.format_version = GetParam(); + Options options = CurrentOptions(); + const int kNumPerFile = 2; + + const auto algs = GetSupportedChecksums(); + const int algs_size = static_cast(algs.size()); + + // generate one table with each type of checksum + for (int i = 0; i < algs_size; ++i) { + table_options.checksum = algs[i]; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + for (int j = 0; j < kNumPerFile; ++j) { + ASSERT_OK(Put(Key(i * kNumPerFile + j), Key(i * kNumPerFile + j))); + } + ASSERT_OK(Flush()); + } + + // with each valid checksum type setting... + for (int i = 0; i < algs_size; ++i) { + table_options.checksum = algs[i]; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + // verify every type of checksum (should be regardless of that setting) + for (int j = 0; j < algs_size * kNumPerFile; ++j) { + ASSERT_EQ(Key(j), Get(Key(j))); + } + } + + // Now test invalid checksum type + table_options.checksum = static_cast(123); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + ASSERT_TRUE(TryReopen(options).IsInvalidArgument()); +} + +// On Windows you can have either memory mapped file or a file +// with unbuffered access. So this asserts and does not make +// sense to run +#ifndef OS_WIN +TEST_F(DBBasicTest, MmapAndBufferOptions) { + if (!IsMemoryMappedAccessSupported()) { + return; + } + Options options = CurrentOptions(); + + options.use_direct_reads = true; + options.allow_mmap_reads = true; + ASSERT_NOK(TryReopen(options)); + + // All other combinations are acceptable + options.use_direct_reads = false; + ASSERT_OK(TryReopen(options)); + + if (IsDirectIOSupported()) { + options.use_direct_reads = true; + options.allow_mmap_reads = false; + ASSERT_OK(TryReopen(options)); + } + + options.use_direct_reads = false; + ASSERT_OK(TryReopen(options)); +} +#endif + +class TestEnv : public EnvWrapper { + public: + explicit TestEnv(Env* base_env) : EnvWrapper(base_env), close_count(0) {} + static const char* kClassName() { return "TestEnv"; } + const char* Name() const override { return kClassName(); } + + class TestLogger : public Logger { + public: + using Logger::Logv; + explicit TestLogger(TestEnv* env_ptr) : Logger() { env = env_ptr; } + ~TestLogger() override { + if (!closed_) { + CloseHelper().PermitUncheckedError(); + } + } + void Logv(const char* /*format*/, va_list /*ap*/) override {} + + protected: + Status CloseImpl() override { return CloseHelper(); } + + private: + Status CloseHelper() { + env->CloseCountInc(); + ; + return Status::IOError(); + } + TestEnv* env; + }; + + void CloseCountInc() { close_count++; } + + int GetCloseCount() { return close_count; } + + Status NewLogger(const std::string& /*fname*/, + std::shared_ptr* result) override { + result->reset(new TestLogger(this)); + return Status::OK(); + } + + private: + int close_count; +}; + +TEST_F(DBBasicTest, DBClose) { + Options options = GetDefaultOptions(); + std::string dbname = test::PerThreadDBPath("db_close_test"); + ASSERT_OK(DestroyDB(dbname, options)); + + DB* db = nullptr; + TestEnv* env = new TestEnv(env_); + std::unique_ptr local_env_guard(env); + options.create_if_missing = true; + options.env = env; + Status s = DB::Open(options, dbname, &db); + ASSERT_OK(s); + ASSERT_TRUE(db != nullptr); + + s = db->Close(); + ASSERT_EQ(env->GetCloseCount(), 1); + ASSERT_EQ(s, Status::IOError()); + + delete db; + ASSERT_EQ(env->GetCloseCount(), 1); + + // Do not call DB::Close() and ensure our logger Close() still gets called + s = DB::Open(options, dbname, &db); + ASSERT_OK(s); + ASSERT_TRUE(db != nullptr); + delete db; + ASSERT_EQ(env->GetCloseCount(), 2); + + // Provide our own logger and ensure DB::Close() does not close it + options.info_log.reset(new TestEnv::TestLogger(env)); + options.create_if_missing = false; + s = DB::Open(options, dbname, &db); + ASSERT_OK(s); + ASSERT_TRUE(db != nullptr); + + s = db->Close(); + ASSERT_EQ(s, Status::OK()); + delete db; + ASSERT_EQ(env->GetCloseCount(), 2); + options.info_log.reset(); + ASSERT_EQ(env->GetCloseCount(), 3); +} + +TEST_F(DBBasicTest, DBCloseAllDirectoryFDs) { + Options options = GetDefaultOptions(); + std::string dbname = test::PerThreadDBPath("db_close_all_dir_fds_test"); + // Configure a specific WAL directory + options.wal_dir = dbname + "_wal_dir"; + // Configure 3 different data directories + options.db_paths.emplace_back(dbname + "_1", 512 * 1024); + options.db_paths.emplace_back(dbname + "_2", 4 * 1024 * 1024); + options.db_paths.emplace_back(dbname + "_3", 1024 * 1024 * 1024); + + ASSERT_OK(DestroyDB(dbname, options)); + + DB* db = nullptr; + std::unique_ptr env = NewCompositeEnv( + std::make_shared(FileSystem::Default())); + options.create_if_missing = true; + options.env = env.get(); + Status s = DB::Open(options, dbname, &db); + ASSERT_OK(s); + ASSERT_TRUE(db != nullptr); + + // Explicitly close the database to ensure the open and close counter for + // directories are equivalent + s = db->Close(); + auto* counted_fs = + options.env->GetFileSystem()->CheckedCast(); + ASSERT_TRUE(counted_fs != nullptr); + ASSERT_EQ(counted_fs->counters()->dir_opens, + counted_fs->counters()->dir_closes); + ASSERT_OK(s); + delete db; +} + +TEST_F(DBBasicTest, DBCloseFlushError) { + std::unique_ptr fault_injection_env( + new FaultInjectionTestEnv(env_)); + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.manual_wal_flush = true; + options.write_buffer_size = 100; + options.env = fault_injection_env.get(); + + Reopen(options); + ASSERT_OK(Put("key1", "value1")); + ASSERT_OK(Put("key2", "value2")); + ASSERT_OK(dbfull()->TEST_SwitchMemtable()); + ASSERT_OK(Put("key3", "value3")); + fault_injection_env->SetFilesystemActive(false); + Status s = dbfull()->Close(); + ASSERT_NE(s, Status::OK()); + // retry should return the same error + s = dbfull()->Close(); + ASSERT_NE(s, Status::OK()); + fault_injection_env->SetFilesystemActive(true); + // retry close() is no-op even the system is back. Could be improved if + // Close() is retry-able: #9029 + s = dbfull()->Close(); + ASSERT_NE(s, Status::OK()); + Destroy(options); +} + +class DBMultiGetTestWithParam + : public DBBasicTest, + public testing::WithParamInterface> {}; + +TEST_P(DBMultiGetTestWithParam, MultiGetMultiCF) { +#ifndef USE_COROUTINES + if (std::get<1>(GetParam())) { + ROCKSDB_GTEST_SKIP("This test requires coroutine support"); + return; + } +#endif // USE_COROUTINES + Options options = CurrentOptions(); + CreateAndReopenWithCF({"pikachu", "ilya", "muromec", "dobrynia", "nikitich", + "alyosha", "popovich"}, + options); + // tuples + std::vector> cf_kv_vec; + static const int num_keys = 24; + cf_kv_vec.reserve(num_keys); + + for (int i = 0; i < num_keys; ++i) { + int cf = i / 3; + int cf_key = 1 % 3; + cf_kv_vec.emplace_back(std::make_tuple( + cf, "cf" + std::to_string(cf) + "_key_" + std::to_string(cf_key), + "cf" + std::to_string(cf) + "_val_" + std::to_string(cf_key))); + ASSERT_OK(Put(std::get<0>(cf_kv_vec[i]), std::get<1>(cf_kv_vec[i]), + std::get<2>(cf_kv_vec[i]))); + } + + int get_sv_count = 0; + ROCKSDB_NAMESPACE::DBImpl* db = static_cast_with_check(db_); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::MultiGet::AfterRefSV", [&](void* /*arg*/) { + if (++get_sv_count == 2) { + // After MultiGet refs a couple of CFs, flush all CFs so MultiGet + // is forced to repeat the process + for (int i = 0; i < num_keys; ++i) { + int cf = i / 3; + int cf_key = i % 8; + if (cf_key == 0) { + ASSERT_OK(Flush(cf)); + } + ASSERT_OK(Put(std::get<0>(cf_kv_vec[i]), std::get<1>(cf_kv_vec[i]), + std::get<2>(cf_kv_vec[i]) + "_2")); + } + } + if (get_sv_count == 11) { + for (int i = 0; i < 8; ++i) { + auto* cfd = static_cast_with_check( + db->GetColumnFamilyHandle(i)) + ->cfd(); + ASSERT_EQ(cfd->TEST_GetLocalSV()->Get(), SuperVersion::kSVInUse); + } + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + std::vector cfs; + std::vector keys; + std::vector values; + + for (int i = 0; i < num_keys; ++i) { + cfs.push_back(std::get<0>(cf_kv_vec[i])); + keys.push_back(std::get<1>(cf_kv_vec[i])); + } + + values = MultiGet(cfs, keys, nullptr, std::get<0>(GetParam()), + std::get<1>(GetParam())); + ASSERT_EQ(values.size(), num_keys); + for (unsigned int j = 0; j < values.size(); ++j) { + ASSERT_EQ(values[j], std::get<2>(cf_kv_vec[j]) + "_2"); + } + + keys.clear(); + cfs.clear(); + cfs.push_back(std::get<0>(cf_kv_vec[0])); + keys.push_back(std::get<1>(cf_kv_vec[0])); + cfs.push_back(std::get<0>(cf_kv_vec[3])); + keys.push_back(std::get<1>(cf_kv_vec[3])); + cfs.push_back(std::get<0>(cf_kv_vec[4])); + keys.push_back(std::get<1>(cf_kv_vec[4])); + values = MultiGet(cfs, keys, nullptr, std::get<0>(GetParam()), + std::get<1>(GetParam())); + ASSERT_EQ(values[0], std::get<2>(cf_kv_vec[0]) + "_2"); + ASSERT_EQ(values[1], std::get<2>(cf_kv_vec[3]) + "_2"); + ASSERT_EQ(values[2], std::get<2>(cf_kv_vec[4]) + "_2"); + + keys.clear(); + cfs.clear(); + cfs.push_back(std::get<0>(cf_kv_vec[7])); + keys.push_back(std::get<1>(cf_kv_vec[7])); + cfs.push_back(std::get<0>(cf_kv_vec[6])); + keys.push_back(std::get<1>(cf_kv_vec[6])); + cfs.push_back(std::get<0>(cf_kv_vec[1])); + keys.push_back(std::get<1>(cf_kv_vec[1])); + values = MultiGet(cfs, keys, nullptr, std::get<0>(GetParam()), + std::get<1>(GetParam())); + ASSERT_EQ(values[0], std::get<2>(cf_kv_vec[7]) + "_2"); + ASSERT_EQ(values[1], std::get<2>(cf_kv_vec[6]) + "_2"); + ASSERT_EQ(values[2], std::get<2>(cf_kv_vec[1]) + "_2"); + + for (int cf = 0; cf < 8; ++cf) { + auto* cfd = + static_cast_with_check( + static_cast_with_check(db_)->GetColumnFamilyHandle(cf)) + ->cfd(); + ASSERT_NE(cfd->TEST_GetLocalSV()->Get(), SuperVersion::kSVInUse); + ASSERT_NE(cfd->TEST_GetLocalSV()->Get(), SuperVersion::kSVObsolete); + } +} + +TEST_P(DBMultiGetTestWithParam, MultiGetMultiCFMutex) { +#ifndef USE_COROUTINES + if (std::get<1>(GetParam())) { + ROCKSDB_GTEST_SKIP("This test requires coroutine support"); + return; + } +#endif // USE_COROUTINES + Options options = CurrentOptions(); + CreateAndReopenWithCF({"pikachu", "ilya", "muromec", "dobrynia", "nikitich", + "alyosha", "popovich"}, + options); + + for (int i = 0; i < 8; ++i) { + ASSERT_OK(Put(i, "cf" + std::to_string(i) + "_key", + "cf" + std::to_string(i) + "_val")); + } + + int get_sv_count = 0; + int retries = 0; + bool last_try = false; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::MultiGet::LastTry", [&](void* /*arg*/) { + last_try = true; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::MultiGet::AfterRefSV", [&](void* /*arg*/) { + if (last_try) { + return; + } + if (++get_sv_count == 2) { + ++retries; + get_sv_count = 0; + for (int i = 0; i < 8; ++i) { + ASSERT_OK(Flush(i)); + ASSERT_OK(Put( + i, "cf" + std::to_string(i) + "_key", + "cf" + std::to_string(i) + "_val" + std::to_string(retries))); + } + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + std::vector cfs; + std::vector keys; + std::vector values; + + for (int i = 0; i < 8; ++i) { + cfs.push_back(i); + keys.push_back("cf" + std::to_string(i) + "_key"); + } + + values = MultiGet(cfs, keys, nullptr, std::get<0>(GetParam()), + std::get<1>(GetParam())); + ASSERT_TRUE(last_try); + ASSERT_EQ(values.size(), 8); + for (unsigned int j = 0; j < values.size(); ++j) { + ASSERT_EQ(values[j], + "cf" + std::to_string(j) + "_val" + std::to_string(retries)); + } + for (int i = 0; i < 8; ++i) { + auto* cfd = + static_cast_with_check( + static_cast_with_check(db_)->GetColumnFamilyHandle(i)) + ->cfd(); + ASSERT_NE(cfd->TEST_GetLocalSV()->Get(), SuperVersion::kSVInUse); + } +} + +TEST_P(DBMultiGetTestWithParam, MultiGetMultiCFSnapshot) { +#ifndef USE_COROUTINES + if (std::get<1>(GetParam())) { + ROCKSDB_GTEST_SKIP("This test requires coroutine support"); + return; + } +#endif // USE_COROUTINES + Options options = CurrentOptions(); + CreateAndReopenWithCF({"pikachu", "ilya", "muromec", "dobrynia", "nikitich", + "alyosha", "popovich"}, + options); + + for (int i = 0; i < 8; ++i) { + ASSERT_OK(Put(i, "cf" + std::to_string(i) + "_key", + "cf" + std::to_string(i) + "_val")); + } + + int get_sv_count = 0; + ROCKSDB_NAMESPACE::DBImpl* db = static_cast_with_check(db_); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::MultiGet::AfterRefSV", [&](void* /*arg*/) { + if (++get_sv_count == 2) { + for (int i = 0; i < 8; ++i) { + ASSERT_OK(Flush(i)); + ASSERT_OK(Put(i, "cf" + std::to_string(i) + "_key", + "cf" + std::to_string(i) + "_val2")); + } + } + if (get_sv_count == 8) { + for (int i = 0; i < 8; ++i) { + auto* cfd = static_cast_with_check( + db->GetColumnFamilyHandle(i)) + ->cfd(); + ASSERT_TRUE( + (cfd->TEST_GetLocalSV()->Get() == SuperVersion::kSVInUse) || + (cfd->TEST_GetLocalSV()->Get() == SuperVersion::kSVObsolete)); + } + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + std::vector cfs; + std::vector keys; + std::vector values; + + for (int i = 0; i < 8; ++i) { + cfs.push_back(i); + keys.push_back("cf" + std::to_string(i) + "_key"); + } + + const Snapshot* snapshot = db_->GetSnapshot(); + values = MultiGet(cfs, keys, snapshot, std::get<0>(GetParam()), + std::get<1>(GetParam())); + db_->ReleaseSnapshot(snapshot); + ASSERT_EQ(values.size(), 8); + for (unsigned int j = 0; j < values.size(); ++j) { + ASSERT_EQ(values[j], "cf" + std::to_string(j) + "_val"); + } + for (int i = 0; i < 8; ++i) { + auto* cfd = + static_cast_with_check( + static_cast_with_check(db_)->GetColumnFamilyHandle(i)) + ->cfd(); + ASSERT_NE(cfd->TEST_GetLocalSV()->Get(), SuperVersion::kSVInUse); + } +} + +TEST_P(DBMultiGetTestWithParam, MultiGetMultiCFUnsorted) { +#ifndef USE_COROUTINES + if (std::get<1>(GetParam())) { + ROCKSDB_GTEST_SKIP("This test requires coroutine support"); + return; + } +#endif // USE_COROUTINES + Options options = CurrentOptions(); + CreateAndReopenWithCF({"one", "two"}, options); + + ASSERT_OK(Put(1, "foo", "bar")); + ASSERT_OK(Put(2, "baz", "xyz")); + ASSERT_OK(Put(1, "abc", "def")); + + // Note: keys for the same CF do not form a consecutive range + std::vector cfs{1, 2, 1}; + std::vector keys{"foo", "baz", "abc"}; + std::vector values; + + values = MultiGet(cfs, keys, /* snapshot */ nullptr, + /* batched */ std::get<0>(GetParam()), + /* async */ std::get<1>(GetParam())); + + ASSERT_EQ(values.size(), 3); + ASSERT_EQ(values[0], "bar"); + ASSERT_EQ(values[1], "xyz"); + ASSERT_EQ(values[2], "def"); +} + +TEST_P(DBMultiGetTestWithParam, MultiGetBatchedSimpleUnsorted) { +#ifndef USE_COROUTINES + if (std::get<1>(GetParam())) { + ROCKSDB_GTEST_SKIP("This test requires coroutine support"); + return; + } +#endif // USE_COROUTINES + // Skip for unbatched MultiGet + if (!std::get<0>(GetParam())) { + ROCKSDB_GTEST_BYPASS("This test is only for batched MultiGet"); + return; + } + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + SetPerfLevel(kEnableCount); + ASSERT_OK(Put(1, "k1", "v1")); + ASSERT_OK(Put(1, "k2", "v2")); + ASSERT_OK(Put(1, "k3", "v3")); + ASSERT_OK(Put(1, "k4", "v4")); + ASSERT_OK(Delete(1, "k4")); + ASSERT_OK(Put(1, "k5", "v5")); + ASSERT_OK(Delete(1, "no_key")); + + get_perf_context()->Reset(); + + std::vector keys({"no_key", "k5", "k4", "k3", "k2", "k1"}); + std::vector values(keys.size()); + std::vector cfs(keys.size(), handles_[1]); + std::vector s(keys.size()); + + ReadOptions ro; + ro.async_io = std::get<1>(GetParam()); + db_->MultiGet(ro, handles_[1], keys.size(), keys.data(), values.data(), + s.data(), false); + + ASSERT_EQ(values.size(), keys.size()); + ASSERT_EQ(std::string(values[5].data(), values[5].size()), "v1"); + ASSERT_EQ(std::string(values[4].data(), values[4].size()), "v2"); + ASSERT_EQ(std::string(values[3].data(), values[3].size()), "v3"); + ASSERT_EQ(std::string(values[1].data(), values[1].size()), "v5"); + // four kv pairs * two bytes per value + ASSERT_EQ(8, (int)get_perf_context()->multiget_read_bytes); + + ASSERT_TRUE(s[0].IsNotFound()); + ASSERT_OK(s[1]); + ASSERT_TRUE(s[2].IsNotFound()); + ASSERT_OK(s[3]); + ASSERT_OK(s[4]); + ASSERT_OK(s[5]); + + SetPerfLevel(kDisable); + } while (ChangeCompactOptions()); +} + +TEST_P(DBMultiGetTestWithParam, MultiGetBatchedSortedMultiFile) { +#ifndef USE_COROUTINES + if (std::get<1>(GetParam())) { + ROCKSDB_GTEST_SKIP("This test requires coroutine support"); + return; + } +#endif // USE_COROUTINES + // Skip for unbatched MultiGet + if (!std::get<0>(GetParam())) { + ROCKSDB_GTEST_BYPASS("This test is only for batched MultiGet"); + return; + } + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + SetPerfLevel(kEnableCount); + // To expand the power of this test, generate > 1 table file and + // mix with memtable + ASSERT_OK(Put(1, "k1", "v1")); + ASSERT_OK(Put(1, "k2", "v2")); + ASSERT_OK(Flush(1)); + ASSERT_OK(Put(1, "k3", "v3")); + ASSERT_OK(Put(1, "k4", "v4")); + ASSERT_OK(Flush(1)); + ASSERT_OK(Delete(1, "k4")); + ASSERT_OK(Put(1, "k5", "v5")); + ASSERT_OK(Delete(1, "no_key")); + + get_perf_context()->Reset(); + + std::vector keys({"k1", "k2", "k3", "k4", "k5", "no_key"}); + std::vector values(keys.size()); + std::vector cfs(keys.size(), handles_[1]); + std::vector s(keys.size()); + + ReadOptions ro; + ro.async_io = std::get<1>(GetParam()); + db_->MultiGet(ro, handles_[1], keys.size(), keys.data(), values.data(), + s.data(), true); + + ASSERT_EQ(values.size(), keys.size()); + ASSERT_EQ(std::string(values[0].data(), values[0].size()), "v1"); + ASSERT_EQ(std::string(values[1].data(), values[1].size()), "v2"); + ASSERT_EQ(std::string(values[2].data(), values[2].size()), "v3"); + ASSERT_EQ(std::string(values[4].data(), values[4].size()), "v5"); + // four kv pairs * two bytes per value + ASSERT_EQ(8, (int)get_perf_context()->multiget_read_bytes); + + ASSERT_OK(s[0]); + ASSERT_OK(s[1]); + ASSERT_OK(s[2]); + ASSERT_TRUE(s[3].IsNotFound()); + ASSERT_OK(s[4]); + ASSERT_TRUE(s[5].IsNotFound()); + + SetPerfLevel(kDisable); + } while (ChangeOptions()); +} + +TEST_P(DBMultiGetTestWithParam, MultiGetBatchedDuplicateKeys) { +#ifndef USE_COROUTINES + if (std::get<1>(GetParam())) { + ROCKSDB_GTEST_SKIP("This test requires coroutine support"); + return; + } +#endif // USE_COROUTINES + // Skip for unbatched MultiGet + if (!std::get<0>(GetParam())) { + ROCKSDB_GTEST_BYPASS("This test is only for batched MultiGet"); + return; + } + Options opts = CurrentOptions(); + opts.merge_operator = MergeOperators::CreateStringAppendOperator(); + CreateAndReopenWithCF({"pikachu"}, opts); + SetPerfLevel(kEnableCount); + // To expand the power of this test, generate > 1 table file and + // mix with memtable + ASSERT_OK(Merge(1, "k1", "v1")); + ASSERT_OK(Merge(1, "k2", "v2")); + ASSERT_OK(Flush(1)); + MoveFilesToLevel(2, 1); + ASSERT_OK(Merge(1, "k3", "v3")); + ASSERT_OK(Merge(1, "k4", "v4")); + ASSERT_OK(Flush(1)); + MoveFilesToLevel(2, 1); + ASSERT_OK(Merge(1, "k4", "v4_2")); + ASSERT_OK(Merge(1, "k6", "v6")); + ASSERT_OK(Flush(1)); + MoveFilesToLevel(2, 1); + ASSERT_OK(Merge(1, "k7", "v7")); + ASSERT_OK(Merge(1, "k8", "v8")); + ASSERT_OK(Flush(1)); + MoveFilesToLevel(2, 1); + + get_perf_context()->Reset(); + + std::vector keys({"k8", "k8", "k8", "k4", "k4", "k1", "k3"}); + std::vector values(keys.size()); + std::vector cfs(keys.size(), handles_[1]); + std::vector s(keys.size()); + + ReadOptions ro; + ro.async_io = std::get<1>(GetParam()); + db_->MultiGet(ro, handles_[1], keys.size(), keys.data(), values.data(), + s.data(), false); + + ASSERT_EQ(values.size(), keys.size()); + ASSERT_EQ(std::string(values[0].data(), values[0].size()), "v8"); + ASSERT_EQ(std::string(values[1].data(), values[1].size()), "v8"); + ASSERT_EQ(std::string(values[2].data(), values[2].size()), "v8"); + ASSERT_EQ(std::string(values[3].data(), values[3].size()), "v4,v4_2"); + ASSERT_EQ(std::string(values[4].data(), values[4].size()), "v4,v4_2"); + ASSERT_EQ(std::string(values[5].data(), values[5].size()), "v1"); + ASSERT_EQ(std::string(values[6].data(), values[6].size()), "v3"); + ASSERT_EQ(24, (int)get_perf_context()->multiget_read_bytes); + + for (Status& status : s) { + ASSERT_OK(status); + } + + SetPerfLevel(kDisable); +} + +TEST_P(DBMultiGetTestWithParam, MultiGetBatchedMultiLevel) { +#ifndef USE_COROUTINES + if (std::get<1>(GetParam())) { + ROCKSDB_GTEST_SKIP("This test requires coroutine support"); + return; + } +#endif // USE_COROUTINES + // Skip for unbatched MultiGet + if (!std::get<0>(GetParam())) { + ROCKSDB_GTEST_BYPASS("This test is only for batched MultiGet"); + return; + } + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + Reopen(options); + int num_keys = 0; + + for (int i = 0; i < 128; ++i) { + ASSERT_OK(Put("key_" + std::to_string(i), "val_l2_" + std::to_string(i))); + num_keys++; + if (num_keys == 8) { + ASSERT_OK(Flush()); + num_keys = 0; + } + } + if (num_keys > 0) { + ASSERT_OK(Flush()); + num_keys = 0; + } + MoveFilesToLevel(2); + + for (int i = 0; i < 128; i += 3) { + ASSERT_OK(Put("key_" + std::to_string(i), "val_l1_" + std::to_string(i))); + num_keys++; + if (num_keys == 8) { + ASSERT_OK(Flush()); + num_keys = 0; + } + } + if (num_keys > 0) { + ASSERT_OK(Flush()); + num_keys = 0; + } + MoveFilesToLevel(1); + + for (int i = 0; i < 128; i += 5) { + ASSERT_OK(Put("key_" + std::to_string(i), "val_l0_" + std::to_string(i))); + num_keys++; + if (num_keys == 8) { + ASSERT_OK(Flush()); + num_keys = 0; + } + } + if (num_keys > 0) { + ASSERT_OK(Flush()); + num_keys = 0; + } + ASSERT_EQ(0, num_keys); + + for (int i = 0; i < 128; i += 9) { + ASSERT_OK(Put("key_" + std::to_string(i), "val_mem_" + std::to_string(i))); + } + + std::vector keys; + std::vector values; + + for (int i = 64; i < 80; ++i) { + keys.push_back("key_" + std::to_string(i)); + } + + values = MultiGet(keys, nullptr, std::get<1>(GetParam())); + ASSERT_EQ(values.size(), 16); + for (unsigned int j = 0; j < values.size(); ++j) { + int key = j + 64; + if (key % 9 == 0) { + ASSERT_EQ(values[j], "val_mem_" + std::to_string(key)); + } else if (key % 5 == 0) { + ASSERT_EQ(values[j], "val_l0_" + std::to_string(key)); + } else if (key % 3 == 0) { + ASSERT_EQ(values[j], "val_l1_" + std::to_string(key)); + } else { + ASSERT_EQ(values[j], "val_l2_" + std::to_string(key)); + } + } +} + +TEST_P(DBMultiGetTestWithParam, MultiGetBatchedMultiLevelMerge) { +#ifndef USE_COROUTINES + if (std::get<1>(GetParam())) { + ROCKSDB_GTEST_SKIP("This test requires coroutine support"); + return; + } +#endif // USE_COROUTINES + // Skip for unbatched MultiGet + if (!std::get<0>(GetParam())) { + ROCKSDB_GTEST_BYPASS("This test is only for batched MultiGet"); + return; + } + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(NewBloomFilterPolicy(10, false)); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + Reopen(options); + int num_keys = 0; + + for (int i = 0; i < 128; ++i) { + ASSERT_OK(Put("key_" + std::to_string(i), "val_l2_" + std::to_string(i))); + num_keys++; + if (num_keys == 8) { + ASSERT_OK(Flush()); + num_keys = 0; + } + } + if (num_keys > 0) { + ASSERT_OK(Flush()); + num_keys = 0; + } + MoveFilesToLevel(2); + + for (int i = 0; i < 128; i += 3) { + ASSERT_OK(Merge("key_" + std::to_string(i), "val_l1_" + std::to_string(i))); + num_keys++; + if (num_keys == 8) { + ASSERT_OK(Flush()); + num_keys = 0; + } + } + if (num_keys > 0) { + ASSERT_OK(Flush()); + num_keys = 0; + } + MoveFilesToLevel(1); + + for (int i = 0; i < 128; i += 5) { + ASSERT_OK(Merge("key_" + std::to_string(i), "val_l0_" + std::to_string(i))); + num_keys++; + if (num_keys == 8) { + ASSERT_OK(Flush()); + num_keys = 0; + } + } + if (num_keys > 0) { + ASSERT_OK(Flush()); + num_keys = 0; + } + ASSERT_EQ(0, num_keys); + + for (int i = 0; i < 128; i += 9) { + ASSERT_OK( + Merge("key_" + std::to_string(i), "val_mem_" + std::to_string(i))); + } + + std::vector keys; + std::vector values; + + for (int i = 32; i < 80; ++i) { + keys.push_back("key_" + std::to_string(i)); + } + + values = MultiGet(keys, nullptr, std::get<1>(GetParam())); + ASSERT_EQ(values.size(), keys.size()); + for (unsigned int j = 0; j < 48; ++j) { + int key = j + 32; + std::string value; + value.append("val_l2_" + std::to_string(key)); + if (key % 3 == 0) { + value.append(","); + value.append("val_l1_" + std::to_string(key)); + } + if (key % 5 == 0) { + value.append(","); + value.append("val_l0_" + std::to_string(key)); + } + if (key % 9 == 0) { + value.append(","); + value.append("val_mem_" + std::to_string(key)); + } + ASSERT_EQ(values[j], value); + } +} + +TEST_P(DBMultiGetTestWithParam, MultiGetBatchedValueSizeInMemory) { +#ifndef USE_COROUTINES + if (std::get<1>(GetParam())) { + ROCKSDB_GTEST_SKIP("This test requires coroutine support"); + return; + } +#endif // USE_COROUTINES + // Skip for unbatched MultiGet + if (!std::get<0>(GetParam())) { + ROCKSDB_GTEST_BYPASS("This test is only for batched MultiGet"); + return; + } + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + SetPerfLevel(kEnableCount); + ASSERT_OK(Put(1, "k1", "v_1")); + ASSERT_OK(Put(1, "k2", "v_2")); + ASSERT_OK(Put(1, "k3", "v_3")); + ASSERT_OK(Put(1, "k4", "v_4")); + ASSERT_OK(Put(1, "k5", "v_5")); + ASSERT_OK(Put(1, "k6", "v_6")); + std::vector keys = {"k1", "k2", "k3", "k4", "k5", "k6"}; + std::vector values(keys.size()); + std::vector s(keys.size()); + std::vector cfs(keys.size(), handles_[1]); + + get_perf_context()->Reset(); + ReadOptions ro; + ro.value_size_soft_limit = 11; + ro.async_io = std::get<1>(GetParam()); + db_->MultiGet(ro, handles_[1], keys.size(), keys.data(), values.data(), + s.data(), false); + + ASSERT_EQ(values.size(), keys.size()); + for (unsigned int i = 0; i < 4; i++) { + ASSERT_EQ(std::string(values[i].data(), values[i].size()), + "v_" + std::to_string(i + 1)); + } + + for (unsigned int i = 4; i < 6; i++) { + ASSERT_TRUE(s[i].IsAborted()); + } + + ASSERT_EQ(12, (int)get_perf_context()->multiget_read_bytes); + SetPerfLevel(kDisable); +} + +TEST_P(DBMultiGetTestWithParam, MultiGetBatchedValueSize) { +#ifndef USE_COROUTINES + if (std::get<1>(GetParam())) { + ROCKSDB_GTEST_SKIP("This test requires coroutine support"); + return; + } +#endif // USE_COROUTINES + // Skip for unbatched MultiGet + if (!std::get<0>(GetParam())) { + return; + } + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + SetPerfLevel(kEnableCount); + + ASSERT_OK(Put(1, "k6", "v6")); + ASSERT_OK(Put(1, "k7", "v7_")); + ASSERT_OK(Put(1, "k3", "v3_")); + ASSERT_OK(Put(1, "k4", "v4")); + ASSERT_OK(Flush(1)); + ASSERT_OK(Delete(1, "k4")); + ASSERT_OK(Put(1, "k11", "v11")); + ASSERT_OK(Delete(1, "no_key")); + ASSERT_OK(Put(1, "k8", "v8_")); + ASSERT_OK(Put(1, "k13", "v13")); + ASSERT_OK(Put(1, "k14", "v14")); + ASSERT_OK(Put(1, "k15", "v15")); + ASSERT_OK(Put(1, "k16", "v16")); + ASSERT_OK(Put(1, "k17", "v17")); + ASSERT_OK(Flush(1)); + + ASSERT_OK(Put(1, "k1", "v1_")); + ASSERT_OK(Put(1, "k2", "v2_")); + ASSERT_OK(Put(1, "k5", "v5_")); + ASSERT_OK(Put(1, "k9", "v9_")); + ASSERT_OK(Put(1, "k10", "v10")); + ASSERT_OK(Delete(1, "k2")); + ASSERT_OK(Delete(1, "k6")); + + get_perf_context()->Reset(); + + std::vector keys({"k1", "k10", "k11", "k12", "k13", "k14", "k15", + "k16", "k17", "k2", "k3", "k4", "k5", "k6", "k7", + "k8", "k9", "no_key"}); + std::vector values(keys.size()); + std::vector cfs(keys.size(), handles_[1]); + std::vector s(keys.size()); + + ReadOptions ro; + ro.value_size_soft_limit = 20; + ro.async_io = std::get<1>(GetParam()); + db_->MultiGet(ro, handles_[1], keys.size(), keys.data(), values.data(), + s.data(), false); + + ASSERT_EQ(values.size(), keys.size()); + + // In memory keys + ASSERT_EQ(std::string(values[0].data(), values[0].size()), "v1_"); + ASSERT_EQ(std::string(values[1].data(), values[1].size()), "v10"); + ASSERT_TRUE(s[9].IsNotFound()); // k2 + ASSERT_EQ(std::string(values[12].data(), values[12].size()), "v5_"); + ASSERT_TRUE(s[13].IsNotFound()); // k6 + ASSERT_EQ(std::string(values[16].data(), values[16].size()), "v9_"); + + // In sst files + ASSERT_EQ(std::string(values[2].data(), values[1].size()), "v11"); + ASSERT_EQ(std::string(values[4].data(), values[4].size()), "v13"); + ASSERT_EQ(std::string(values[5].data(), values[5].size()), "v14"); + + // Remaining aborted after value_size exceeds. + ASSERT_TRUE(s[3].IsAborted()); + ASSERT_TRUE(s[6].IsAborted()); + ASSERT_TRUE(s[7].IsAborted()); + ASSERT_TRUE(s[8].IsAborted()); + ASSERT_TRUE(s[10].IsAborted()); + ASSERT_TRUE(s[11].IsAborted()); + ASSERT_TRUE(s[14].IsAborted()); + ASSERT_TRUE(s[15].IsAborted()); + ASSERT_TRUE(s[17].IsAborted()); + + // 6 kv pairs * 3 bytes per value (i.e. 18) + ASSERT_EQ(21, (int)get_perf_context()->multiget_read_bytes); + SetPerfLevel(kDisable); + } while (ChangeCompactOptions()); +} + +TEST_P(DBMultiGetTestWithParam, MultiGetBatchedValueSizeMultiLevelMerge) { + if (std::get<1>(GetParam())) { + ROCKSDB_GTEST_BYPASS("This test needs to be fixed for async IO"); + return; + } + // Skip for unbatched MultiGet + if (!std::get<0>(GetParam())) { + ROCKSDB_GTEST_BYPASS("This test is only for batched MultiGet"); + return; + } + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(NewBloomFilterPolicy(10, false)); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + Reopen(options); + int num_keys = 0; + + for (int i = 0; i < 64; ++i) { + ASSERT_OK(Put("key_" + std::to_string(i), "val_l2_" + std::to_string(i))); + num_keys++; + if (num_keys == 8) { + ASSERT_OK(Flush()); + num_keys = 0; + } + } + if (num_keys > 0) { + ASSERT_OK(Flush()); + num_keys = 0; + } + MoveFilesToLevel(2); + + for (int i = 0; i < 64; i += 3) { + ASSERT_OK(Merge("key_" + std::to_string(i), "val_l1_" + std::to_string(i))); + num_keys++; + if (num_keys == 8) { + ASSERT_OK(Flush()); + num_keys = 0; + } + } + if (num_keys > 0) { + ASSERT_OK(Flush()); + num_keys = 0; + } + MoveFilesToLevel(1); + + for (int i = 0; i < 64; i += 5) { + ASSERT_OK(Merge("key_" + std::to_string(i), "val_l0_" + std::to_string(i))); + num_keys++; + if (num_keys == 8) { + ASSERT_OK(Flush()); + num_keys = 0; + } + } + if (num_keys > 0) { + ASSERT_OK(Flush()); + num_keys = 0; + } + ASSERT_EQ(0, num_keys); + + for (int i = 0; i < 64; i += 9) { + ASSERT_OK( + Merge("key_" + std::to_string(i), "val_mem_" + std::to_string(i))); + } + + std::vector keys_str; + for (int i = 10; i < 50; ++i) { + keys_str.push_back("key_" + std::to_string(i)); + } + + std::vector keys(keys_str.size()); + for (int i = 0; i < 40; i++) { + keys[i] = Slice(keys_str[i]); + } + + std::vector values(keys_str.size()); + std::vector statuses(keys_str.size()); + ReadOptions read_options; + read_options.verify_checksums = true; + read_options.value_size_soft_limit = 380; + read_options.async_io = std::get<1>(GetParam()); + db_->MultiGet(read_options, dbfull()->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data()); + + ASSERT_EQ(values.size(), keys.size()); + + for (unsigned int j = 0; j < 26; ++j) { + int key = j + 10; + std::string value; + value.append("val_l2_" + std::to_string(key)); + if (key % 3 == 0) { + value.append(","); + value.append("val_l1_" + std::to_string(key)); + } + if (key % 5 == 0) { + value.append(","); + value.append("val_l0_" + std::to_string(key)); + } + if (key % 9 == 0) { + value.append(","); + value.append("val_mem_" + std::to_string(key)); + } + ASSERT_EQ(values[j], value); + ASSERT_OK(statuses[j]); + } + + // All remaning keys status is set Status::Abort + for (unsigned int j = 26; j < 40; j++) { + ASSERT_TRUE(statuses[j].IsAborted()); + } +} + +INSTANTIATE_TEST_CASE_P(DBMultiGetTestWithParam, DBMultiGetTestWithParam, + testing::Combine(testing::Bool(), testing::Bool())); + +#if USE_COROUTINES +class DBMultiGetAsyncIOTest : public DBBasicTest, + public ::testing::WithParamInterface { + public: + DBMultiGetAsyncIOTest() + : DBBasicTest(), statistics_(ROCKSDB_NAMESPACE::CreateDBStatistics()) { + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(NewBloomFilterPolicy(10)); + options_ = CurrentOptions(); + options_.disable_auto_compactions = true; + options_.statistics = statistics_; + options_.table_factory.reset(NewBlockBasedTableFactory(bbto)); + options_.env = Env::Default(); + Reopen(options_); + int num_keys = 0; + + // Put all keys in the bottommost level, and overwrite some keys + // in L0 and L1 + for (int i = 0; i < 256; ++i) { + EXPECT_OK(Put(Key(i), "val_l2_" + std::to_string(i))); + num_keys++; + if (num_keys == 8) { + EXPECT_OK(Flush()); + num_keys = 0; + } + } + if (num_keys > 0) { + EXPECT_OK(Flush()); + num_keys = 0; + } + MoveFilesToLevel(2); + + for (int i = 0; i < 128; i += 3) { + EXPECT_OK(Put(Key(i), "val_l1_" + std::to_string(i))); + num_keys++; + if (num_keys == 8) { + EXPECT_OK(Flush()); + num_keys = 0; + } + } + if (num_keys > 0) { + EXPECT_OK(Flush()); + num_keys = 0; + } + // Put some range deletes in L1 + for (int i = 128; i < 256; i += 32) { + std::string range_begin = Key(i); + std::string range_end = Key(i + 16); + EXPECT_OK(dbfull()->DeleteRange(WriteOptions(), + dbfull()->DefaultColumnFamily(), + range_begin, range_end)); + // Also do some Puts to force creation of bloom filter + for (int j = i + 16; j < i + 32; ++j) { + if (j % 3 == 0) { + EXPECT_OK(Put(Key(j), "val_l1_" + std::to_string(j))); + } + } + EXPECT_OK(Flush()); + } + MoveFilesToLevel(1); + + for (int i = 0; i < 128; i += 5) { + EXPECT_OK(Put(Key(i), "val_l0_" + std::to_string(i))); + num_keys++; + if (num_keys == 8) { + EXPECT_OK(Flush()); + num_keys = 0; + } + } + if (num_keys > 0) { + EXPECT_OK(Flush()); + num_keys = 0; + } + EXPECT_EQ(0, num_keys); + } + + const std::shared_ptr& statistics() { return statistics_; } + + protected: + void PrepareDBForTest() { +#ifdef ROCKSDB_IOURING_PRESENT + Reopen(options_); +#else // ROCKSDB_IOURING_PRESENT + // Warm up the block cache so we don't need to use the IO uring + Iterator* iter = dbfull()->NewIterator(ReadOptions()); + for (iter->SeekToFirst(); iter->Valid() && iter->status().ok(); + iter->Next()) + ; + EXPECT_OK(iter->status()); + delete iter; +#endif // ROCKSDB_IOURING_PRESENT + } + + void ReopenDB() { Reopen(options_); } + + private: + std::shared_ptr statistics_; + Options options_; +}; + +TEST_P(DBMultiGetAsyncIOTest, GetFromL0) { + // All 3 keys in L0. The L0 files should be read serially. + std::vector key_strs{Key(0), Key(40), Key(80)}; + std::vector keys{key_strs[0], key_strs[1], key_strs[2]}; + std::vector values(key_strs.size()); + std::vector statuses(key_strs.size()); + + PrepareDBForTest(); + + ReadOptions ro; + ro.async_io = true; + ro.optimize_multiget_for_io = GetParam(); + dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data()); + ASSERT_EQ(values.size(), 3); + ASSERT_OK(statuses[0]); + ASSERT_OK(statuses[1]); + ASSERT_OK(statuses[2]); + ASSERT_EQ(values[0], "val_l0_" + std::to_string(0)); + ASSERT_EQ(values[1], "val_l0_" + std::to_string(40)); + ASSERT_EQ(values[2], "val_l0_" + std::to_string(80)); + + HistogramData multiget_io_batch_size; + + statistics()->histogramData(MULTIGET_IO_BATCH_SIZE, &multiget_io_batch_size); + + // With async IO, lookups will happen in parallel for each key +#ifdef ROCKSDB_IOURING_PRESENT + if (GetParam()) { + ASSERT_EQ(multiget_io_batch_size.count, 1); + ASSERT_EQ(multiget_io_batch_size.max, 3); + ASSERT_EQ(statistics()->getTickerCount(MULTIGET_COROUTINE_COUNT), 3); + } else { + // Without Async IO, MultiGet will call MultiRead 3 times, once for each + // L0 file + ASSERT_EQ(multiget_io_batch_size.count, 3); + } +#else // ROCKSDB_IOURING_PRESENT + ASSERT_EQ(statistics()->getTickerCount(MULTIGET_COROUTINE_COUNT), 0); +#endif // ROCKSDB_IOURING_PRESENT +} + +TEST_P(DBMultiGetAsyncIOTest, GetFromL1) { + std::vector key_strs; + std::vector keys; + std::vector values; + std::vector statuses; + + key_strs.push_back(Key(33)); + key_strs.push_back(Key(54)); + key_strs.push_back(Key(102)); + keys.push_back(key_strs[0]); + keys.push_back(key_strs[1]); + keys.push_back(key_strs[2]); + values.resize(keys.size()); + statuses.resize(keys.size()); + + PrepareDBForTest(); + + ReadOptions ro; + ro.async_io = true; + ro.optimize_multiget_for_io = GetParam(); + dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data()); + ASSERT_EQ(values.size(), 3); + ASSERT_EQ(statuses[0], Status::OK()); + ASSERT_EQ(statuses[1], Status::OK()); + ASSERT_EQ(statuses[2], Status::OK()); + ASSERT_EQ(values[0], "val_l1_" + std::to_string(33)); + ASSERT_EQ(values[1], "val_l1_" + std::to_string(54)); + ASSERT_EQ(values[2], "val_l1_" + std::to_string(102)); + + HistogramData multiget_io_batch_size; + + statistics()->histogramData(MULTIGET_IO_BATCH_SIZE, &multiget_io_batch_size); + +#ifdef ROCKSDB_IOURING_PRESENT + // A batch of 3 async IOs is expected, one for each overlapping file in L1 + ASSERT_EQ(multiget_io_batch_size.count, 1); + ASSERT_EQ(multiget_io_batch_size.max, 3); + ASSERT_EQ(statistics()->getTickerCount(MULTIGET_COROUTINE_COUNT), 3); +#else // ROCKSDB_IOURING_PRESENT + ASSERT_EQ(statistics()->getTickerCount(MULTIGET_COROUTINE_COUNT), 0); +#endif // ROCKSDB_IOURING_PRESENT +} + +#ifdef ROCKSDB_IOURING_PRESENT +TEST_P(DBMultiGetAsyncIOTest, GetFromL1Error) { + std::vector key_strs; + std::vector keys; + std::vector values; + std::vector statuses; + + key_strs.push_back(Key(33)); + key_strs.push_back(Key(54)); + key_strs.push_back(Key(102)); + keys.push_back(key_strs[0]); + keys.push_back(key_strs[1]); + keys.push_back(key_strs[2]); + values.resize(keys.size()); + statuses.resize(keys.size()); + + int count = 0; + SyncPoint::GetInstance()->SetCallBack( + "TableCache::GetTableReader:BeforeOpenFile", [&](void* status) { + count++; + // Fail the last table reader open, which is the 6th SST file + // since 3 overlapping L0 files + 3 L1 files containing the keys + if (count == 6) { + Status* s = static_cast(status); + *s = Status::IOError(); + } + }); + // DB open will create table readers unless we reduce the table cache + // capacity. + // SanitizeOptions will set max_open_files to minimum of 20. Table cache + // is allocated with max_open_files - 10 as capacity. So override + // max_open_files to 11 so table cache capacity will become 1. This will + // prevent file open during DB open and force the file to be opened + // during MultiGet + SyncPoint::GetInstance()->SetCallBack( + "SanitizeOptions::AfterChangeMaxOpenFiles", [&](void* arg) { + int* max_open_files = (int*)arg; + *max_open_files = 11; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + PrepareDBForTest(); + + ReadOptions ro; + ro.async_io = true; + ro.optimize_multiget_for_io = GetParam(); + dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data()); + SyncPoint::GetInstance()->DisableProcessing(); + ASSERT_EQ(values.size(), 3); + ASSERT_EQ(statuses[0], Status::OK()); + ASSERT_EQ(statuses[1], Status::OK()); + ASSERT_EQ(statuses[2], Status::IOError()); + + HistogramData multiget_io_batch_size; + + statistics()->histogramData(MULTIGET_IO_BATCH_SIZE, &multiget_io_batch_size); + + // A batch of 3 async IOs is expected, one for each overlapping file in L1 + ASSERT_EQ(multiget_io_batch_size.count, 1); + ASSERT_EQ(multiget_io_batch_size.max, 2); + ASSERT_EQ(statistics()->getTickerCount(MULTIGET_COROUTINE_COUNT), 2); +} +#endif // ROCKSDB_IOURING_PRESENT + +TEST_P(DBMultiGetAsyncIOTest, LastKeyInFile) { + std::vector key_strs; + std::vector keys; + std::vector values; + std::vector statuses; + + // 21 is the last key in the first L1 file + key_strs.push_back(Key(21)); + key_strs.push_back(Key(54)); + key_strs.push_back(Key(102)); + keys.push_back(key_strs[0]); + keys.push_back(key_strs[1]); + keys.push_back(key_strs[2]); + values.resize(keys.size()); + statuses.resize(keys.size()); + + PrepareDBForTest(); + + ReadOptions ro; + ro.async_io = true; + ro.optimize_multiget_for_io = GetParam(); + dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data()); + ASSERT_EQ(values.size(), 3); + ASSERT_EQ(statuses[0], Status::OK()); + ASSERT_EQ(statuses[1], Status::OK()); + ASSERT_EQ(statuses[2], Status::OK()); + ASSERT_EQ(values[0], "val_l1_" + std::to_string(21)); + ASSERT_EQ(values[1], "val_l1_" + std::to_string(54)); + ASSERT_EQ(values[2], "val_l1_" + std::to_string(102)); + +#ifdef ROCKSDB_IOURING_PRESENT + HistogramData multiget_io_batch_size; + + statistics()->histogramData(MULTIGET_IO_BATCH_SIZE, &multiget_io_batch_size); + + // Since the first MultiGet key is the last key in a file, the MultiGet is + // expected to lookup in that file first, before moving on to other files. + // So the first file lookup will issue one async read, and the next lookup + // will lookup 2 files in parallel and issue 2 async reads + ASSERT_EQ(multiget_io_batch_size.count, 2); + ASSERT_EQ(multiget_io_batch_size.max, 2); +#endif // ROCKSDB_IOURING_PRESENT +} + +TEST_P(DBMultiGetAsyncIOTest, GetFromL1AndL2) { + std::vector key_strs; + std::vector keys; + std::vector values; + std::vector statuses; + + // 33 and 102 are in L1, and 56 is in L2 + key_strs.push_back(Key(33)); + key_strs.push_back(Key(56)); + key_strs.push_back(Key(102)); + keys.push_back(key_strs[0]); + keys.push_back(key_strs[1]); + keys.push_back(key_strs[2]); + values.resize(keys.size()); + statuses.resize(keys.size()); + + PrepareDBForTest(); + + ReadOptions ro; + ro.async_io = true; + ro.optimize_multiget_for_io = GetParam(); + dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data()); + ASSERT_EQ(values.size(), 3); + ASSERT_EQ(statuses[0], Status::OK()); + ASSERT_EQ(statuses[1], Status::OK()); + ASSERT_EQ(statuses[2], Status::OK()); + ASSERT_EQ(values[0], "val_l1_" + std::to_string(33)); + ASSERT_EQ(values[1], "val_l2_" + std::to_string(56)); + ASSERT_EQ(values[2], "val_l1_" + std::to_string(102)); + +#ifdef ROCKSDB_IOURING_PRESENT + HistogramData multiget_io_batch_size; + + statistics()->histogramData(MULTIGET_IO_BATCH_SIZE, &multiget_io_batch_size); + + // There are 2 keys in L1 in twp separate files, and 1 in L2. With + // optimize_multiget_for_io, all three lookups will happen in parallel. + // Otherwise, the L2 lookup will happen after L1. + ASSERT_EQ(multiget_io_batch_size.count, GetParam() ? 1 : 2); + ASSERT_EQ(multiget_io_batch_size.max, GetParam() ? 3 : 2); +#endif // ROCKSDB_IOURING_PRESENT +} + +TEST_P(DBMultiGetAsyncIOTest, GetFromL2WithRangeOverlapL0L1) { + std::vector key_strs; + std::vector keys; + std::vector values; + std::vector statuses; + + // 19 and 26 are in L2, but overlap with L0 and L1 file ranges + key_strs.push_back(Key(19)); + key_strs.push_back(Key(26)); + keys.push_back(key_strs[0]); + keys.push_back(key_strs[1]); + values.resize(keys.size()); + statuses.resize(keys.size()); + + PrepareDBForTest(); + + ReadOptions ro; + ro.async_io = true; + ro.optimize_multiget_for_io = GetParam(); + dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data()); + ASSERT_EQ(values.size(), 2); + ASSERT_EQ(statuses[0], Status::OK()); + ASSERT_EQ(statuses[1], Status::OK()); + ASSERT_EQ(values[0], "val_l2_" + std::to_string(19)); + ASSERT_EQ(values[1], "val_l2_" + std::to_string(26)); + +#ifdef ROCKSDB_IOURING_PRESENT + // Bloom filters in L0/L1 will avoid the coroutine calls in those levels + ASSERT_EQ(statistics()->getTickerCount(MULTIGET_COROUTINE_COUNT), 2); +#else // ROCKSDB_IOURING_PRESENT + ASSERT_EQ(statistics()->getTickerCount(MULTIGET_COROUTINE_COUNT), 0); +#endif // ROCKSDB_IOURING_PRESENT +} + +#ifdef ROCKSDB_IOURING_PRESENT +TEST_P(DBMultiGetAsyncIOTest, GetFromL2WithRangeDelInL1) { + std::vector key_strs; + std::vector keys; + std::vector values; + std::vector statuses; + + // 139 and 163 are in L2, but overlap with a range deletes in L1 + key_strs.push_back(Key(139)); + key_strs.push_back(Key(163)); + keys.push_back(key_strs[0]); + keys.push_back(key_strs[1]); + values.resize(keys.size()); + statuses.resize(keys.size()); + + PrepareDBForTest(); + + ReadOptions ro; + ro.async_io = true; + ro.optimize_multiget_for_io = GetParam(); + dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data()); + ASSERT_EQ(values.size(), 2); + ASSERT_EQ(statuses[0], Status::NotFound()); + ASSERT_EQ(statuses[1], Status::NotFound()); + + // Bloom filters in L0/L1 will avoid the coroutine calls in those levels + ASSERT_EQ(statistics()->getTickerCount(MULTIGET_COROUTINE_COUNT), 2); +} + +TEST_P(DBMultiGetAsyncIOTest, GetFromL1AndL2WithRangeDelInL1) { + std::vector key_strs; + std::vector keys; + std::vector values; + std::vector statuses; + + // 139 and 163 are in L2, but overlap with a range deletes in L1 + key_strs.push_back(Key(139)); + key_strs.push_back(Key(144)); + key_strs.push_back(Key(163)); + keys.push_back(key_strs[0]); + keys.push_back(key_strs[1]); + keys.push_back(key_strs[2]); + values.resize(keys.size()); + statuses.resize(keys.size()); + + PrepareDBForTest(); + + ReadOptions ro; + ro.async_io = true; + ro.optimize_multiget_for_io = GetParam(); + dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data()); + ASSERT_EQ(values.size(), keys.size()); + ASSERT_EQ(statuses[0], Status::NotFound()); + ASSERT_EQ(statuses[1], Status::OK()); + ASSERT_EQ(values[1], "val_l1_" + std::to_string(144)); + ASSERT_EQ(statuses[2], Status::NotFound()); + + // Bloom filters in L0/L1 will avoid the coroutine calls in those levels + ASSERT_EQ(statistics()->getTickerCount(MULTIGET_COROUTINE_COUNT), 3); +} +#endif // ROCKSDB_IOURING_PRESENT + +TEST_P(DBMultiGetAsyncIOTest, GetNoIOUring) { + std::vector key_strs; + std::vector keys; + std::vector values; + std::vector statuses; + + key_strs.push_back(Key(33)); + key_strs.push_back(Key(54)); + key_strs.push_back(Key(102)); + keys.push_back(key_strs[0]); + keys.push_back(key_strs[1]); + keys.push_back(key_strs[2]); + values.resize(keys.size()); + statuses.resize(keys.size()); + + enable_io_uring = false; + ReopenDB(); + + ReadOptions ro; + ro.async_io = true; + ro.optimize_multiget_for_io = GetParam(); + dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data()); + ASSERT_EQ(values.size(), 3); + ASSERT_EQ(statuses[0], Status::OK()); + ASSERT_EQ(statuses[1], Status::OK()); + ASSERT_EQ(statuses[2], Status::OK()); + + HistogramData async_read_bytes; + + statistics()->histogramData(ASYNC_READ_BYTES, &async_read_bytes); + + // A batch of 3 async IOs is expected, one for each overlapping file in L1 + ASSERT_EQ(async_read_bytes.count, 0); + ASSERT_EQ(statistics()->getTickerCount(MULTIGET_COROUTINE_COUNT), 0); +} + +INSTANTIATE_TEST_CASE_P(DBMultiGetAsyncIOTest, DBMultiGetAsyncIOTest, + testing::Bool()); +#endif // USE_COROUTINES + +TEST_F(DBBasicTest, MultiGetStats) { + Options options; + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.env = env_; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + BlockBasedTableOptions table_options; + table_options.block_size = 1; + table_options.index_type = + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + table_options.partition_filters = true; + table_options.no_block_cache = true; + table_options.cache_index_and_filter_blocks = false; + table_options.filter_policy.reset(NewBloomFilterPolicy(10, false)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + CreateAndReopenWithCF({"pikachu"}, options); + + int total_keys = 2000; + std::vector keys_str(total_keys); + std::vector keys(total_keys); + static size_t kMultiGetBatchSize = 100; + std::vector values(kMultiGetBatchSize); + std::vector s(kMultiGetBatchSize); + ReadOptions read_opts; + + Random rnd(309); + // Create Multiple SST files at multiple levels. + for (int i = 0; i < 500; ++i) { + keys_str[i] = "k" + std::to_string(i); + keys[i] = Slice(keys_str[i]); + ASSERT_OK(Put(1, "k" + std::to_string(i), rnd.RandomString(1000))); + if (i % 100 == 0) { + ASSERT_OK(Flush(1)); + } + } + ASSERT_OK(Flush(1)); + MoveFilesToLevel(2, 1); + + for (int i = 501; i < 1000; ++i) { + keys_str[i] = "k" + std::to_string(i); + keys[i] = Slice(keys_str[i]); + ASSERT_OK(Put(1, "k" + std::to_string(i), rnd.RandomString(1000))); + if (i % 100 == 0) { + ASSERT_OK(Flush(1)); + } + } + + ASSERT_OK(Flush(1)); + MoveFilesToLevel(2, 1); + + for (int i = 1001; i < total_keys; ++i) { + keys_str[i] = "k" + std::to_string(i); + keys[i] = Slice(keys_str[i]); + ASSERT_OK(Put(1, "k" + std::to_string(i), rnd.RandomString(1000))); + if (i % 100 == 0) { + ASSERT_OK(Flush(1)); + } + } + ASSERT_OK(Flush(1)); + MoveFilesToLevel(1, 1); + Close(); + + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_OK(options.statistics->Reset()); + + db_->MultiGet(read_opts, handles_[1], kMultiGetBatchSize, &keys[1250], + values.data(), s.data(), false); + + ASSERT_EQ(values.size(), kMultiGetBatchSize); + HistogramData hist_level; + HistogramData hist_index_and_filter_blocks; + HistogramData hist_sst; + + options.statistics->histogramData(NUM_LEVEL_READ_PER_MULTIGET, &hist_level); + options.statistics->histogramData(NUM_INDEX_AND_FILTER_BLOCKS_READ_PER_LEVEL, + &hist_index_and_filter_blocks); + options.statistics->histogramData(NUM_SST_READ_PER_LEVEL, &hist_sst); + + // Maximum number of blocks read from a file system in a level. + ASSERT_EQ(hist_level.max, 1); + ASSERT_GT(hist_index_and_filter_blocks.max, 0); + // Maximum number of sst files read from file system in a level. + ASSERT_EQ(hist_sst.max, 2); + + // Minimun number of blocks read in a level. + ASSERT_EQ(hist_level.min, 1); + ASSERT_GT(hist_index_and_filter_blocks.min, 0); + // Minimun number of sst files read in a level. + ASSERT_EQ(hist_sst.min, 1); + + for (PinnableSlice& value : values) { + value.Reset(); + } + for (Status& status : s) { + status = Status::OK(); + } + db_->MultiGet(read_opts, handles_[1], kMultiGetBatchSize, &keys[950], + values.data(), s.data(), false); + options.statistics->histogramData(NUM_LEVEL_READ_PER_MULTIGET, &hist_level); + ASSERT_EQ(hist_level.max, 2); +} + +// Test class for batched MultiGet with prefix extractor +// Param bool - If true, use partitioned filters +// If false, use full filter block +class MultiGetPrefixExtractorTest : public DBBasicTest, + public ::testing::WithParamInterface { +}; + +TEST_P(MultiGetPrefixExtractorTest, Batched) { + Options options = CurrentOptions(); + options.prefix_extractor.reset(NewFixedPrefixTransform(2)); + options.memtable_prefix_bloom_size_ratio = 10; + BlockBasedTableOptions bbto; + if (GetParam()) { + bbto.index_type = BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + bbto.partition_filters = true; + } + bbto.filter_policy.reset(NewBloomFilterPolicy(10, false)); + bbto.whole_key_filtering = false; + bbto.cache_index_and_filter_blocks = false; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + Reopen(options); + + SetPerfLevel(kEnableCount); + get_perf_context()->Reset(); + + ASSERT_OK(Put("k", "v0")); + ASSERT_OK(Put("kk1", "v1")); + ASSERT_OK(Put("kk2", "v2")); + ASSERT_OK(Put("kk3", "v3")); + ASSERT_OK(Put("kk4", "v4")); + std::vector keys( + {"k", "kk1", "kk2", "kk3", "kk4", "rofl", "lmho"}); + std::vector expected( + {"v0", "v1", "v2", "v3", "v4", "NOT_FOUND", "NOT_FOUND"}); + std::vector values; + values = MultiGet(keys, nullptr); + ASSERT_EQ(values, expected); + // One key ("k") is not queried against the filter because it is outside + // the prefix_extractor domain, leaving 6 keys with queried prefixes. + ASSERT_EQ(get_perf_context()->bloom_memtable_miss_count, 2); + ASSERT_EQ(get_perf_context()->bloom_memtable_hit_count, 4); + ASSERT_OK(Flush()); + + get_perf_context()->Reset(); + values = MultiGet(keys, nullptr); + ASSERT_EQ(values, expected); + ASSERT_EQ(get_perf_context()->bloom_sst_miss_count, 2); + ASSERT_EQ(get_perf_context()->bloom_sst_hit_count, 4); + + // Also check Get stat + get_perf_context()->Reset(); + for (size_t i = 0; i < keys.size(); ++i) { + values[i] = Get(keys[i]); + } + ASSERT_EQ(values, expected); + ASSERT_EQ(get_perf_context()->bloom_sst_miss_count, 2); + ASSERT_EQ(get_perf_context()->bloom_sst_hit_count, 4); +} + +INSTANTIATE_TEST_CASE_P(MultiGetPrefix, MultiGetPrefixExtractorTest, + ::testing::Bool()); + +class DBMultiGetRowCacheTest : public DBBasicTest, + public ::testing::WithParamInterface {}; + +TEST_P(DBMultiGetRowCacheTest, MultiGetBatched) { + do { + option_config_ = kRowCache; + Options options = CurrentOptions(); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + CreateAndReopenWithCF({"pikachu"}, options); + SetPerfLevel(kEnableCount); + ASSERT_OK(Put(1, "k1", "v1")); + ASSERT_OK(Put(1, "k2", "v2")); + ASSERT_OK(Put(1, "k3", "v3")); + ASSERT_OK(Put(1, "k4", "v4")); + ASSERT_OK(Flush(1)); + ASSERT_OK(Put(1, "k5", "v5")); + const Snapshot* snap1 = dbfull()->GetSnapshot(); + ASSERT_OK(Delete(1, "k4")); + ASSERT_OK(Flush(1)); + const Snapshot* snap2 = dbfull()->GetSnapshot(); + + get_perf_context()->Reset(); + + std::vector keys({"no_key", "k5", "k4", "k3", "k1"}); + std::vector values(keys.size()); + std::vector cfs(keys.size(), handles_[1]); + std::vector s(keys.size()); + + ReadOptions ro; + bool use_snapshots = GetParam(); + if (use_snapshots) { + ro.snapshot = snap2; + } + db_->MultiGet(ro, handles_[1], keys.size(), keys.data(), values.data(), + s.data(), false); + + ASSERT_EQ(values.size(), keys.size()); + ASSERT_EQ(std::string(values[4].data(), values[4].size()), "v1"); + ASSERT_EQ(std::string(values[3].data(), values[3].size()), "v3"); + ASSERT_EQ(std::string(values[1].data(), values[1].size()), "v5"); + // four kv pairs * two bytes per value + ASSERT_EQ(6, (int)get_perf_context()->multiget_read_bytes); + + ASSERT_TRUE(s[0].IsNotFound()); + ASSERT_OK(s[1]); + ASSERT_TRUE(s[2].IsNotFound()); + ASSERT_OK(s[3]); + ASSERT_OK(s[4]); + + // Call MultiGet() again with some intersection with the previous set of + // keys. Those should already be in the row cache. + keys.assign({"no_key", "k5", "k3", "k2"}); + for (size_t i = 0; i < keys.size(); ++i) { + values[i].Reset(); + s[i] = Status::OK(); + } + get_perf_context()->Reset(); + + if (use_snapshots) { + ro.snapshot = snap1; + } + db_->MultiGet(ReadOptions(), handles_[1], keys.size(), keys.data(), + values.data(), s.data(), false); + + ASSERT_EQ(std::string(values[3].data(), values[3].size()), "v2"); + ASSERT_EQ(std::string(values[2].data(), values[2].size()), "v3"); + ASSERT_EQ(std::string(values[1].data(), values[1].size()), "v5"); + // four kv pairs * two bytes per value + ASSERT_EQ(6, (int)get_perf_context()->multiget_read_bytes); + + ASSERT_TRUE(s[0].IsNotFound()); + ASSERT_OK(s[1]); + ASSERT_OK(s[2]); + ASSERT_OK(s[3]); + if (use_snapshots) { + // Only reads from the first SST file would have been cached, since + // snapshot seq no is > fd.largest_seqno + ASSERT_EQ(1, TestGetTickerCount(options, ROW_CACHE_HIT)); + } else { + ASSERT_EQ(2, TestGetTickerCount(options, ROW_CACHE_HIT)); + } + + SetPerfLevel(kDisable); + dbfull()->ReleaseSnapshot(snap1); + dbfull()->ReleaseSnapshot(snap2); + } while (ChangeCompactOptions()); +} + +INSTANTIATE_TEST_CASE_P(DBMultiGetRowCacheTest, DBMultiGetRowCacheTest, + testing::Values(true, false)); + +TEST_F(DBBasicTest, GetAllKeyVersions) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.disable_auto_compactions = true; + CreateAndReopenWithCF({"pikachu"}, options); + ASSERT_EQ(2, handles_.size()); + const size_t kNumInserts = 4; + const size_t kNumDeletes = 4; + const size_t kNumUpdates = 4; + + // Check default column family + for (size_t i = 0; i != kNumInserts; ++i) { + ASSERT_OK(Put(std::to_string(i), "value")); + } + for (size_t i = 0; i != kNumUpdates; ++i) { + ASSERT_OK(Put(std::to_string(i), "value1")); + } + for (size_t i = 0; i != kNumDeletes; ++i) { + ASSERT_OK(Delete(std::to_string(i))); + } + std::vector key_versions; + ASSERT_OK(GetAllKeyVersions(db_, Slice(), Slice(), + std::numeric_limits::max(), + &key_versions)); + ASSERT_EQ(kNumInserts + kNumDeletes + kNumUpdates, key_versions.size()); + for (size_t i = 0; i < kNumInserts + kNumDeletes + kNumUpdates; i++) { + if (i % 3 == 0) { + ASSERT_EQ(key_versions[i].GetTypeName(), "TypeDeletion"); + } else { + ASSERT_EQ(key_versions[i].GetTypeName(), "TypeValue"); + } + } + ASSERT_OK(GetAllKeyVersions(db_, handles_[0], Slice(), Slice(), + std::numeric_limits::max(), + &key_versions)); + ASSERT_EQ(kNumInserts + kNumDeletes + kNumUpdates, key_versions.size()); + + // Check non-default column family + for (size_t i = 0; i + 1 != kNumInserts; ++i) { + ASSERT_OK(Put(1, std::to_string(i), "value")); + } + for (size_t i = 0; i + 1 != kNumUpdates; ++i) { + ASSERT_OK(Put(1, std::to_string(i), "value1")); + } + for (size_t i = 0; i + 1 != kNumDeletes; ++i) { + ASSERT_OK(Delete(1, std::to_string(i))); + } + ASSERT_OK(GetAllKeyVersions(db_, handles_[1], Slice(), Slice(), + std::numeric_limits::max(), + &key_versions)); + ASSERT_EQ(kNumInserts + kNumDeletes + kNumUpdates - 3, key_versions.size()); +} + +TEST_F(DBBasicTest, ValueTypeString) { + KeyVersion key_version; + // when adding new type, please also update `value_type_string_map` + for (unsigned char i = ValueType::kTypeDeletion; i < ValueType::kTypeMaxValid; + i++) { + key_version.type = i; + ASSERT_TRUE(key_version.GetTypeName() != "Invalid"); + } +} + +TEST_F(DBBasicTest, MultiGetIOBufferOverrun) { + Options options = CurrentOptions(); + Random rnd(301); + BlockBasedTableOptions table_options; + table_options.pin_l0_filter_and_index_blocks_in_cache = true; + table_options.block_size = 16 * 1024; + ASSERT_TRUE(table_options.block_size > + BlockBasedTable::kMultiGetReadStackBufSize); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + + std::string zero_str(128, '\0'); + for (int i = 0; i < 100; ++i) { + // Make the value compressible. A purely random string doesn't compress + // and the resultant data block will not be compressed + std::string value(rnd.RandomString(128) + zero_str); + assert(Put(Key(i), value) == Status::OK()); + } + ASSERT_OK(Flush()); + + std::vector key_data(10); + std::vector keys; + // We cannot resize a PinnableSlice vector, so just set initial size to + // largest we think we will need + std::vector values(10); + std::vector statuses; + ReadOptions ro; + + // Warm up the cache first + key_data.emplace_back(Key(0)); + keys.emplace_back(Slice(key_data.back())); + key_data.emplace_back(Key(50)); + keys.emplace_back(Slice(key_data.back())); + statuses.resize(keys.size()); + + dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data(), true); +} + +TEST_F(DBBasicTest, IncrementalRecoveryNoCorrupt) { + Options options = CurrentOptions(); + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu", "eevee"}, options); + size_t num_cfs = handles_.size(); + ASSERT_EQ(3, num_cfs); + WriteOptions write_opts; + write_opts.disableWAL = true; + for (size_t cf = 0; cf != num_cfs; ++cf) { + for (size_t i = 0; i != 10000; ++i) { + std::string key_str = Key(static_cast(i)); + std::string value_str = std::to_string(cf) + "_" + std::to_string(i); + + ASSERT_OK(Put(static_cast(cf), key_str, value_str)); + if (0 == (i % 1000)) { + ASSERT_OK(Flush(static_cast(cf))); + } + } + } + for (size_t cf = 0; cf != num_cfs; ++cf) { + ASSERT_OK(Flush(static_cast(cf))); + } + Close(); + options.best_efforts_recovery = true; + ReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu", "eevee"}, + options); + num_cfs = handles_.size(); + ASSERT_EQ(3, num_cfs); + for (size_t cf = 0; cf != num_cfs; ++cf) { + for (int i = 0; i != 10000; ++i) { + std::string key_str = Key(static_cast(i)); + std::string expected_value_str = + std::to_string(cf) + "_" + std::to_string(i); + ASSERT_EQ(expected_value_str, Get(static_cast(cf), key_str)); + } + } +} + +TEST_F(DBBasicTest, BestEffortsRecoveryWithVersionBuildingFailure) { + Options options = CurrentOptions(); + DestroyAndReopen(options); + ASSERT_OK(Put("foo", "value")); + ASSERT_OK(Flush()); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "VersionBuilder::CheckConsistencyBeforeReturn", [&](void* arg) { + ASSERT_NE(nullptr, arg); + *(reinterpret_cast(arg)) = + Status::Corruption("Inject corruption"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + options.best_efforts_recovery = true; + Status s = TryReopen(options); + ASSERT_TRUE(s.IsCorruption()); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +namespace { +class TableFileListener : public EventListener { + public: + void OnTableFileCreated(const TableFileCreationInfo& info) override { + InstrumentedMutexLock lock(&mutex_); + cf_to_paths_[info.cf_name].push_back(info.file_path); + } + std::vector& GetFiles(const std::string& cf_name) { + InstrumentedMutexLock lock(&mutex_); + return cf_to_paths_[cf_name]; + } + + private: + InstrumentedMutex mutex_; + std::unordered_map> cf_to_paths_; +}; +} // anonymous namespace + +TEST_F(DBBasicTest, LastSstFileNotInManifest) { + // If the last sst file is not tracked in MANIFEST, + // or the VersionEdit for the last sst file is not synced, + // on recovery, the last sst file should be deleted, + // and new sst files shouldn't reuse its file number. + Options options = CurrentOptions(); + DestroyAndReopen(options); + Close(); + + // Manually add a sst file. + constexpr uint64_t kSstFileNumber = 100; + const std::string kSstFile = MakeTableFileName(dbname_, kSstFileNumber); + ASSERT_OK(WriteStringToFile(env_, /* data = */ "bad sst file content", + /* fname = */ kSstFile, + /* should_sync = */ true)); + ASSERT_OK(env_->FileExists(kSstFile)); + + TableFileListener* listener = new TableFileListener(); + options.listeners.emplace_back(listener); + Reopen(options); + // kSstFile should already be deleted. + ASSERT_TRUE(env_->FileExists(kSstFile).IsNotFound()); + + ASSERT_OK(Put("k", "v")); + ASSERT_OK(Flush()); + // New sst file should have file number > kSstFileNumber. + std::vector& files = + listener->GetFiles(kDefaultColumnFamilyName); + ASSERT_EQ(files.size(), 1); + const std::string fname = files[0].erase(0, (dbname_ + "/").size()); + uint64_t number = 0; + FileType type = kTableFile; + ASSERT_TRUE(ParseFileName(fname, &number, &type)); + ASSERT_EQ(type, kTableFile); + ASSERT_GT(number, kSstFileNumber); +} + +TEST_F(DBBasicTest, RecoverWithMissingFiles) { + Options options = CurrentOptions(); + DestroyAndReopen(options); + TableFileListener* listener = new TableFileListener(); + // Disable auto compaction to simplify SST file name tracking. + options.disable_auto_compactions = true; + options.listeners.emplace_back(listener); + CreateAndReopenWithCF({"pikachu", "eevee"}, options); + std::vector all_cf_names = {kDefaultColumnFamilyName, "pikachu", + "eevee"}; + size_t num_cfs = handles_.size(); + ASSERT_EQ(3, num_cfs); + for (size_t cf = 0; cf != num_cfs; ++cf) { + ASSERT_OK(Put(static_cast(cf), "a", "0_value")); + ASSERT_OK(Flush(static_cast(cf))); + ASSERT_OK(Put(static_cast(cf), "b", "0_value")); + ASSERT_OK(Flush(static_cast(cf))); + ASSERT_OK(Put(static_cast(cf), "c", "0_value")); + ASSERT_OK(Flush(static_cast(cf))); + } + + // Delete and corrupt files + for (size_t i = 0; i < all_cf_names.size(); ++i) { + std::vector& files = listener->GetFiles(all_cf_names[i]); + ASSERT_EQ(3, files.size()); + std::string corrupted_data; + ASSERT_OK(ReadFileToString(env_, files[files.size() - 1], &corrupted_data)); + ASSERT_OK(WriteStringToFile( + env_, corrupted_data.substr(0, corrupted_data.size() - 2), + files[files.size() - 1], /*should_sync=*/true)); + for (int j = static_cast(files.size() - 2); j >= static_cast(i); + --j) { + ASSERT_OK(env_->DeleteFile(files[j])); + } + } + options.best_efforts_recovery = true; + ReopenWithColumnFamilies(all_cf_names, options); + // Verify data + ReadOptions read_opts; + read_opts.total_order_seek = true; + { + std::unique_ptr iter(db_->NewIterator(read_opts, handles_[0])); + iter->SeekToFirst(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + iter.reset(db_->NewIterator(read_opts, handles_[1])); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("a", iter->key()); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + iter.reset(db_->NewIterator(read_opts, handles_[2])); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("a", iter->key()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("b", iter->key()); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + } +} + +TEST_F(DBBasicTest, BestEffortsRecoveryTryMultipleManifests) { + Options options = CurrentOptions(); + options.env = env_; + DestroyAndReopen(options); + ASSERT_OK(Put("foo", "value0")); + ASSERT_OK(Flush()); + Close(); + { + // Hack by adding a new MANIFEST with high file number + std::string garbage(10, '\0'); + ASSERT_OK(WriteStringToFile(env_, garbage, dbname_ + "/MANIFEST-001000", + /*should_sync=*/true)); + } + { + // Hack by adding a corrupted SST not referenced by any MANIFEST + std::string garbage(10, '\0'); + ASSERT_OK(WriteStringToFile(env_, garbage, dbname_ + "/001001.sst", + /*should_sync=*/true)); + } + + options.best_efforts_recovery = true; + + Reopen(options); + ASSERT_OK(Put("bar", "value")); +} + +TEST_F(DBBasicTest, RecoverWithNoCurrentFile) { + Options options = CurrentOptions(); + options.env = env_; + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + options.best_efforts_recovery = true; + ReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu"}, options); + ASSERT_EQ(2, handles_.size()); + ASSERT_OK(Put("foo", "value")); + ASSERT_OK(Put(1, "bar", "value")); + ASSERT_OK(Flush()); + ASSERT_OK(Flush(1)); + Close(); + ASSERT_OK(env_->DeleteFile(CurrentFileName(dbname_))); + ReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu"}, options); + std::vector cf_names; + ASSERT_OK(DB::ListColumnFamilies(DBOptions(options), dbname_, &cf_names)); + ASSERT_EQ(2, cf_names.size()); + for (const auto& name : cf_names) { + ASSERT_TRUE(name == kDefaultColumnFamilyName || name == "pikachu"); + } +} + +TEST_F(DBBasicTest, RecoverWithNoManifest) { + Options options = CurrentOptions(); + options.env = env_; + DestroyAndReopen(options); + ASSERT_OK(Put("foo", "value")); + ASSERT_OK(Flush()); + Close(); + { + // Delete all MANIFEST. + std::vector files; + ASSERT_OK(env_->GetChildren(dbname_, &files)); + for (const auto& file : files) { + uint64_t number = 0; + FileType type = kWalFile; + if (ParseFileName(file, &number, &type) && type == kDescriptorFile) { + ASSERT_OK(env_->DeleteFile(dbname_ + "/" + file)); + } + } + } + options.best_efforts_recovery = true; + options.create_if_missing = false; + Status s = TryReopen(options); + ASSERT_TRUE(s.IsInvalidArgument()); + options.create_if_missing = true; + Reopen(options); + // Since no MANIFEST exists, best-efforts recovery creates a new, empty db. + ASSERT_EQ("NOT_FOUND", Get("foo")); +} + +TEST_F(DBBasicTest, SkipWALIfMissingTableFiles) { + Options options = CurrentOptions(); + DestroyAndReopen(options); + TableFileListener* listener = new TableFileListener(); + options.listeners.emplace_back(listener); + CreateAndReopenWithCF({"pikachu"}, options); + std::vector kAllCfNames = {kDefaultColumnFamilyName, "pikachu"}; + size_t num_cfs = handles_.size(); + ASSERT_EQ(2, num_cfs); + for (int cf = 0; cf < static_cast(kAllCfNames.size()); ++cf) { + ASSERT_OK(Put(cf, "a", "0_value")); + ASSERT_OK(Flush(cf)); + ASSERT_OK(Put(cf, "b", "0_value")); + } + // Delete files + for (size_t i = 0; i < kAllCfNames.size(); ++i) { + std::vector& files = listener->GetFiles(kAllCfNames[i]); + ASSERT_EQ(1, files.size()); + for (int j = static_cast(files.size() - 1); j >= static_cast(i); + --j) { + ASSERT_OK(env_->DeleteFile(files[j])); + } + } + options.best_efforts_recovery = true; + ReopenWithColumnFamilies(kAllCfNames, options); + // Verify WAL is not applied + ReadOptions read_opts; + read_opts.total_order_seek = true; + std::unique_ptr iter(db_->NewIterator(read_opts, handles_[0])); + iter->SeekToFirst(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + iter.reset(db_->NewIterator(read_opts, handles_[1])); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("a", iter->key()); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); +} + +TEST_F(DBBasicTest, DisableTrackWal) { + // If WAL tracking was enabled, and then disabled during reopen, + // the previously tracked WALs should be removed from MANIFEST. + + Options options = CurrentOptions(); + options.track_and_verify_wals_in_manifest = true; + // extremely small write buffer size, + // so that new WALs are created more frequently. + options.write_buffer_size = 100; + options.env = env_; + DestroyAndReopen(options); + for (int i = 0; i < 100; i++) { + ASSERT_OK(Put("foo" + std::to_string(i), "value" + std::to_string(i))); + } + ASSERT_OK(dbfull()->TEST_SwitchMemtable()); + ASSERT_OK(db_->SyncWAL()); + // Some WALs are tracked. + ASSERT_FALSE(dbfull()->GetVersionSet()->GetWalSet().GetWals().empty()); + Close(); + + // Disable WAL tracking. + options.track_and_verify_wals_in_manifest = false; + options.create_if_missing = false; + ASSERT_OK(TryReopen(options)); + // Previously tracked WALs are cleared. + ASSERT_TRUE(dbfull()->GetVersionSet()->GetWalSet().GetWals().empty()); + Close(); + + // Re-enable WAL tracking again. + options.track_and_verify_wals_in_manifest = true; + options.create_if_missing = false; + ASSERT_OK(TryReopen(options)); + ASSERT_TRUE(dbfull()->GetVersionSet()->GetWalSet().GetWals().empty()); + Close(); +} + +TEST_F(DBBasicTest, ManifestChecksumMismatch) { + Options options = CurrentOptions(); + DestroyAndReopen(options); + ASSERT_OK(Put("bar", "value")); + ASSERT_OK(Flush()); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "LogWriter::EmitPhysicalRecord:BeforeEncodeChecksum", [&](void* arg) { + auto* crc = reinterpret_cast(arg); + *crc = *crc + 1; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + WriteOptions write_opts; + write_opts.disableWAL = true; + Status s = db_->Put(write_opts, "foo", "value"); + ASSERT_OK(s); + ASSERT_OK(Flush()); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + ASSERT_OK(Put("foo", "value1")); + ASSERT_OK(Flush()); + s = TryReopen(options); + ASSERT_TRUE(s.IsCorruption()); +} + +TEST_F(DBBasicTest, ConcurrentlyCloseDB) { + Options options = CurrentOptions(); + DestroyAndReopen(options); + std::vector workers; + for (int i = 0; i < 10; i++) { + workers.push_back(std::thread([&]() { + auto s = db_->Close(); + ASSERT_OK(s); + })); + } + for (auto& w : workers) { + w.join(); + } +} + +class DBBasicTestTrackWal : public DBTestBase, + public testing::WithParamInterface { + public: + DBBasicTestTrackWal() + : DBTestBase("db_basic_test_track_wal", /*env_do_fsync=*/false) {} + + int CountWalFiles() { + VectorLogPtr log_files; + EXPECT_OK(dbfull()->GetSortedWalFiles(log_files)); + return static_cast(log_files.size()); + }; +}; + +TEST_P(DBBasicTestTrackWal, DoNotTrackObsoleteWal) { + // If a WAL becomes obsolete after flushing, but is not deleted from disk yet, + // then if SyncWAL is called afterwards, the obsolete WAL should not be + // tracked in MANIFEST. + + Options options = CurrentOptions(); + options.create_if_missing = true; + options.track_and_verify_wals_in_manifest = true; + options.atomic_flush = GetParam(); + + DestroyAndReopen(options); + CreateAndReopenWithCF({"cf"}, options); + ASSERT_EQ(handles_.size(), 2); // default, cf + // Do not delete WALs. + ASSERT_OK(db_->DisableFileDeletions()); + constexpr int n = 10; + std::vector> wals(n); + for (size_t i = 0; i < n; i++) { + // Generate a new WAL for each key-value. + const int cf = i % 2; + ASSERT_OK(db_->GetCurrentWalFile(&wals[i])); + ASSERT_OK(Put(cf, "k" + std::to_string(i), "v" + std::to_string(i))); + ASSERT_OK(Flush({0, 1})); + } + ASSERT_EQ(CountWalFiles(), n); + // Since all WALs are obsolete, no WAL should be tracked in MANIFEST. + ASSERT_OK(db_->SyncWAL()); + + // Manually delete all WALs. + Close(); + for (const auto& wal : wals) { + ASSERT_OK(env_->DeleteFile(LogFileName(dbname_, wal->LogNumber()))); + } + + // If SyncWAL tracks the obsolete WALs in MANIFEST, + // reopen will fail because the WALs are missing from disk. + ASSERT_OK(TryReopenWithColumnFamilies({"default", "cf"}, options)); + Destroy(options); +} + +INSTANTIATE_TEST_CASE_P(DBBasicTestTrackWal, DBBasicTestTrackWal, + testing::Bool()); + +class DBBasicTestMultiGet : public DBTestBase { + public: + DBBasicTestMultiGet(std::string test_dir, int num_cfs, + bool uncompressed_cache, bool _compression_enabled, + bool _fill_cache, uint32_t compression_parallel_threads) + : DBTestBase(test_dir, /*env_do_fsync=*/false) { + compression_enabled_ = _compression_enabled; + fill_cache_ = _fill_cache; + + if (uncompressed_cache) { + std::shared_ptr cache = NewLRUCache(1048576); + uncompressed_cache_ = std::make_shared(cache); + } + + env_->count_random_reads_ = true; + + Options options = CurrentOptions(); + Random rnd(301); + BlockBasedTableOptions table_options; + + if (compression_enabled_) { + std::vector compression_types; + compression_types = GetSupportedCompressions(); + // Not every platform may have compression libraries available, so + // dynamically pick based on what's available + CompressionType tmp_type = kNoCompression; + for (auto c_type : compression_types) { + if (c_type != kNoCompression) { + tmp_type = c_type; + break; + } + } + if (tmp_type != kNoCompression) { + options.compression = tmp_type; + } else { + compression_enabled_ = false; + } + } + + table_options.block_cache = uncompressed_cache_; + if (table_options.block_cache == nullptr) { + table_options.no_block_cache = true; + } else { + table_options.pin_l0_filter_and_index_blocks_in_cache = true; + } + table_options.flush_block_policy_factory.reset( + new MyFlushBlockPolicyFactory()); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + if (!compression_enabled_) { + options.compression = kNoCompression; + } else { + options.compression_opts.parallel_threads = compression_parallel_threads; + } + options_ = options; + Reopen(options); + + if (num_cfs > 1) { + for (int cf = 0; cf < num_cfs; ++cf) { + cf_names_.emplace_back("cf" + std::to_string(cf)); + } + CreateColumnFamilies(cf_names_, options); + cf_names_.emplace_back("default"); + } + + std::string zero_str(128, '\0'); + for (int cf = 0; cf < num_cfs; ++cf) { + for (int i = 0; i < 100; ++i) { + // Make the value compressible. A purely random string doesn't compress + // and the resultant data block will not be compressed + values_.emplace_back(rnd.RandomString(128) + zero_str); + assert(((num_cfs == 1) ? Put(Key(i), values_[i]) + : Put(cf, Key(i), values_[i])) == Status::OK()); + } + if (num_cfs == 1) { + EXPECT_OK(Flush()); + } else { + EXPECT_OK(dbfull()->Flush(FlushOptions(), handles_[cf])); + } + + for (int i = 0; i < 100; ++i) { + // block cannot gain space by compression + uncompressable_values_.emplace_back(rnd.RandomString(256) + '\0'); + std::string tmp_key = "a" + Key(i); + assert(((num_cfs == 1) ? Put(tmp_key, uncompressable_values_[i]) + : Put(cf, tmp_key, uncompressable_values_[i])) == + Status::OK()); + } + if (num_cfs == 1) { + EXPECT_OK(Flush()); + } else { + EXPECT_OK(dbfull()->Flush(FlushOptions(), handles_[cf])); + } + } + // Clear compressed cache, which is always pre-populated + if (compressed_cache_) { + compressed_cache_->SetCapacity(0); + compressed_cache_->SetCapacity(1048576); + } + } + + bool CheckValue(int i, const std::string& value) { + if (values_[i].compare(value) == 0) { + return true; + } + return false; + } + + bool CheckUncompressableValue(int i, const std::string& value) { + if (uncompressable_values_[i].compare(value) == 0) { + return true; + } + return false; + } + + const std::vector& GetCFNames() const { return cf_names_; } + + int num_lookups() { return uncompressed_cache_->num_lookups(); } + int num_found() { return uncompressed_cache_->num_found(); } + int num_inserts() { return uncompressed_cache_->num_inserts(); } + + int num_lookups_compressed() { return compressed_cache_->num_lookups(); } + int num_found_compressed() { return compressed_cache_->num_found(); } + int num_inserts_compressed() { return compressed_cache_->num_inserts(); } + + bool fill_cache() { return fill_cache_; } + bool compression_enabled() { return compression_enabled_; } + bool has_compressed_cache() { return compressed_cache_ != nullptr; } + bool has_uncompressed_cache() { return uncompressed_cache_ != nullptr; } + Options get_options() { return options_; } + + static void SetUpTestCase() {} + static void TearDownTestCase() {} + + protected: + class MyFlushBlockPolicyFactory : public FlushBlockPolicyFactory { + public: + MyFlushBlockPolicyFactory() {} + + virtual const char* Name() const override { + return "MyFlushBlockPolicyFactory"; + } + + virtual FlushBlockPolicy* NewFlushBlockPolicy( + const BlockBasedTableOptions& /*table_options*/, + const BlockBuilder& data_block_builder) const override { + return new MyFlushBlockPolicy(data_block_builder); + } + }; + + class MyFlushBlockPolicy : public FlushBlockPolicy { + public: + explicit MyFlushBlockPolicy(const BlockBuilder& data_block_builder) + : num_keys_(0), data_block_builder_(data_block_builder) {} + + bool Update(const Slice& /*key*/, const Slice& /*value*/) override { + if (data_block_builder_.empty()) { + // First key in this block + num_keys_ = 1; + return false; + } + // Flush every 10 keys + if (num_keys_ == 10) { + num_keys_ = 1; + return true; + } + num_keys_++; + return false; + } + + private: + int num_keys_; + const BlockBuilder& data_block_builder_; + }; + + class MyBlockCache : public CacheWrapper { + public: + explicit MyBlockCache(std::shared_ptr target) + : CacheWrapper(target), + num_lookups_(0), + num_found_(0), + num_inserts_(0) {} + + const char* Name() const override { return "MyBlockCache"; } + + Status Insert(const Slice& key, Cache::ObjectPtr value, + const CacheItemHelper* helper, size_t charge, + Handle** handle = nullptr, + Priority priority = Priority::LOW) override { + num_inserts_++; + return target_->Insert(key, value, helper, charge, handle, priority); + } + + Handle* Lookup(const Slice& key, const CacheItemHelper* helper, + CreateContext* create_context, + Priority priority = Priority::LOW, + Statistics* stats = nullptr) override { + num_lookups_++; + Handle* handle = + target_->Lookup(key, helper, create_context, priority, stats); + if (handle != nullptr) { + num_found_++; + } + return handle; + } + + int num_lookups() { return num_lookups_; } + + int num_found() { return num_found_; } + + int num_inserts() { return num_inserts_; } + + private: + int num_lookups_; + int num_found_; + int num_inserts_; + }; + + std::shared_ptr compressed_cache_; + std::shared_ptr uncompressed_cache_; + Options options_; + bool compression_enabled_; + std::vector values_; + std::vector uncompressable_values_; + bool fill_cache_; + std::vector cf_names_; +}; + +class DBBasicTestWithParallelIO : public DBBasicTestMultiGet, + public testing::WithParamInterface< + std::tuple> { + public: + DBBasicTestWithParallelIO() + : DBBasicTestMultiGet("/db_basic_test_with_parallel_io", 1, + std::get<0>(GetParam()), std::get<1>(GetParam()), + std::get<2>(GetParam()), std::get<3>(GetParam())) {} +}; + +TEST_P(DBBasicTestWithParallelIO, MultiGet) { + std::vector key_data(10); + std::vector keys; + // We cannot resize a PinnableSlice vector, so just set initial size to + // largest we think we will need + std::vector values(10); + std::vector statuses; + ReadOptions ro; + ro.fill_cache = fill_cache(); + + // Warm up the cache first + key_data.emplace_back(Key(0)); + keys.emplace_back(Slice(key_data.back())); + key_data.emplace_back(Key(50)); + keys.emplace_back(Slice(key_data.back())); + statuses.resize(keys.size()); + + dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data(), true); + ASSERT_TRUE(CheckValue(0, values[0].ToString())); + ASSERT_TRUE(CheckValue(50, values[1].ToString())); + + int random_reads = env_->random_read_counter_.Read(); + key_data[0] = Key(1); + key_data[1] = Key(51); + keys[0] = Slice(key_data[0]); + keys[1] = Slice(key_data[1]); + values[0].Reset(); + values[1].Reset(); + dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data(), true); + ASSERT_TRUE(CheckValue(1, values[0].ToString())); + ASSERT_TRUE(CheckValue(51, values[1].ToString())); + + bool read_from_cache = false; + if (fill_cache()) { + if (has_uncompressed_cache()) { + read_from_cache = true; + } else if (has_compressed_cache() && compression_enabled()) { + read_from_cache = true; + } + } + + int expected_reads = random_reads + (read_from_cache ? 0 : 2); + ASSERT_EQ(env_->random_read_counter_.Read(), expected_reads); + + keys.resize(10); + statuses.resize(10); + std::vector key_ints{1, 2, 15, 16, 55, 81, 82, 83, 84, 85}; + for (size_t i = 0; i < key_ints.size(); ++i) { + key_data[i] = Key(key_ints[i]); + keys[i] = Slice(key_data[i]); + statuses[i] = Status::OK(); + values[i].Reset(); + } + dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data(), true); + for (size_t i = 0; i < key_ints.size(); ++i) { + ASSERT_OK(statuses[i]); + ASSERT_TRUE(CheckValue(key_ints[i], values[i].ToString())); + } + if (compression_enabled() && !has_compressed_cache()) { + expected_reads += (read_from_cache ? 2 : 3); + } else { + expected_reads += (read_from_cache ? 2 : 4); + } + ASSERT_EQ(env_->random_read_counter_.Read(), expected_reads); + + keys.resize(10); + statuses.resize(10); + std::vector key_uncmp{1, 2, 15, 16, 55, 81, 82, 83, 84, 85}; + for (size_t i = 0; i < key_uncmp.size(); ++i) { + key_data[i] = "a" + Key(key_uncmp[i]); + keys[i] = Slice(key_data[i]); + statuses[i] = Status::OK(); + values[i].Reset(); + } + dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data(), true); + for (size_t i = 0; i < key_uncmp.size(); ++i) { + ASSERT_OK(statuses[i]); + ASSERT_TRUE(CheckUncompressableValue(key_uncmp[i], values[i].ToString())); + } + if (compression_enabled() && !has_compressed_cache()) { + expected_reads += (read_from_cache ? 3 : 3); + } else { + expected_reads += (read_from_cache ? 4 : 4); + } + ASSERT_EQ(env_->random_read_counter_.Read(), expected_reads); + + keys.resize(5); + statuses.resize(5); + std::vector key_tr{1, 2, 15, 16, 55}; + for (size_t i = 0; i < key_tr.size(); ++i) { + key_data[i] = "a" + Key(key_tr[i]); + keys[i] = Slice(key_data[i]); + statuses[i] = Status::OK(); + values[i].Reset(); + } + dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data(), true); + for (size_t i = 0; i < key_tr.size(); ++i) { + ASSERT_OK(statuses[i]); + ASSERT_TRUE(CheckUncompressableValue(key_tr[i], values[i].ToString())); + } + if (compression_enabled() && !has_compressed_cache()) { + expected_reads += (read_from_cache ? 0 : 2); + ASSERT_EQ(env_->random_read_counter_.Read(), expected_reads); + } else { + if (has_uncompressed_cache()) { + expected_reads += (read_from_cache ? 0 : 3); + ASSERT_EQ(env_->random_read_counter_.Read(), expected_reads); + } else { + // A rare case, even we enable the block compression but some of data + // blocks are not compressed due to content. If user only enable the + // compressed cache, the uncompressed blocks will not tbe cached, and + // block reads will be triggered. The number of reads is related to + // the compression algorithm. + ASSERT_TRUE(env_->random_read_counter_.Read() >= expected_reads); + } + } +} + +TEST_P(DBBasicTestWithParallelIO, MultiGetDirectIO) { + class FakeDirectIOEnv : public EnvWrapper { + class FakeDirectIOSequentialFile; + class FakeDirectIORandomAccessFile; + + public: + FakeDirectIOEnv(Env* env) : EnvWrapper(env) {} + static const char* kClassName() { return "FakeDirectIOEnv"; } + const char* Name() const override { return kClassName(); } + + Status NewRandomAccessFile(const std::string& fname, + std::unique_ptr* result, + const EnvOptions& options) override { + std::unique_ptr file; + assert(options.use_direct_reads); + EnvOptions opts = options; + opts.use_direct_reads = false; + Status s = target()->NewRandomAccessFile(fname, &file, opts); + if (!s.ok()) { + return s; + } + result->reset(new FakeDirectIORandomAccessFile(std::move(file))); + return s; + } + + private: + class FakeDirectIOSequentialFile : public SequentialFileWrapper { + public: + FakeDirectIOSequentialFile(std::unique_ptr&& file) + : SequentialFileWrapper(file.get()), file_(std::move(file)) {} + ~FakeDirectIOSequentialFile() {} + + bool use_direct_io() const override { return true; } + size_t GetRequiredBufferAlignment() const override { return 1; } + + private: + std::unique_ptr file_; + }; + + class FakeDirectIORandomAccessFile : public RandomAccessFileWrapper { + public: + FakeDirectIORandomAccessFile(std::unique_ptr&& file) + : RandomAccessFileWrapper(file.get()), file_(std::move(file)) {} + ~FakeDirectIORandomAccessFile() {} + + bool use_direct_io() const override { return true; } + size_t GetRequiredBufferAlignment() const override { return 1; } + + private: + std::unique_ptr file_; + }; + }; + + std::unique_ptr env(new FakeDirectIOEnv(env_)); + Options opts = get_options(); + opts.env = env.get(); + opts.use_direct_reads = true; + Reopen(opts); + + std::vector key_data(10); + std::vector keys; + // We cannot resize a PinnableSlice vector, so just set initial size to + // largest we think we will need + std::vector values(10); + std::vector statuses; + ReadOptions ro; + ro.fill_cache = fill_cache(); + + // Warm up the cache first + key_data.emplace_back(Key(0)); + keys.emplace_back(Slice(key_data.back())); + key_data.emplace_back(Key(50)); + keys.emplace_back(Slice(key_data.back())); + statuses.resize(keys.size()); + + dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data(), true); + ASSERT_TRUE(CheckValue(0, values[0].ToString())); + ASSERT_TRUE(CheckValue(50, values[1].ToString())); + + int random_reads = env_->random_read_counter_.Read(); + key_data[0] = Key(1); + key_data[1] = Key(51); + keys[0] = Slice(key_data[0]); + keys[1] = Slice(key_data[1]); + values[0].Reset(); + values[1].Reset(); + if (uncompressed_cache_) { + uncompressed_cache_->SetCapacity(0); + uncompressed_cache_->SetCapacity(1048576); + } + dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data(), true); + ASSERT_TRUE(CheckValue(1, values[0].ToString())); + ASSERT_TRUE(CheckValue(51, values[1].ToString())); + + bool read_from_cache = false; + if (fill_cache()) { + if (has_uncompressed_cache()) { + read_from_cache = true; + } else if (has_compressed_cache() && compression_enabled()) { + read_from_cache = true; + } + } + + int expected_reads = random_reads; + if (!compression_enabled() || !has_compressed_cache()) { + expected_reads += 2; + } else { + expected_reads += (read_from_cache ? 0 : 2); + } + if (env_->random_read_counter_.Read() != expected_reads) { + ASSERT_EQ(env_->random_read_counter_.Read(), expected_reads); + } + Close(); +} + +TEST_P(DBBasicTestWithParallelIO, MultiGetWithChecksumMismatch) { + std::vector key_data(10); + std::vector keys; + // We cannot resize a PinnableSlice vector, so just set initial size to + // largest we think we will need + std::vector values(10); + std::vector statuses; + int read_count = 0; + ReadOptions ro; + ro.fill_cache = fill_cache(); + + SyncPoint::GetInstance()->SetCallBack( + "RetrieveMultipleBlocks:VerifyChecksum", [&](void* status) { + Status* s = static_cast(status); + read_count++; + if (read_count == 2) { + *s = Status::Corruption(); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + + // Warm up the cache first + key_data.emplace_back(Key(0)); + keys.emplace_back(Slice(key_data.back())); + key_data.emplace_back(Key(50)); + keys.emplace_back(Slice(key_data.back())); + statuses.resize(keys.size()); + + dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data(), true); + ASSERT_TRUE(CheckValue(0, values[0].ToString())); + // ASSERT_TRUE(CheckValue(50, values[1].ToString())); + ASSERT_EQ(statuses[0], Status::OK()); + ASSERT_EQ(statuses[1], Status::Corruption()); + + SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_P(DBBasicTestWithParallelIO, MultiGetWithMissingFile) { + std::vector key_data(10); + std::vector keys; + // We cannot resize a PinnableSlice vector, so just set initial size to + // largest we think we will need + std::vector values(10); + std::vector statuses; + ReadOptions ro; + ro.fill_cache = fill_cache(); + + SyncPoint::GetInstance()->SetCallBack( + "TableCache::MultiGet:FindTable", [&](void* status) { + Status* s = static_cast(status); + *s = Status::IOError(); + }); + // DB open will create table readers unless we reduce the table cache + // capacity. + // SanitizeOptions will set max_open_files to minimum of 20. Table cache + // is allocated with max_open_files - 10 as capacity. So override + // max_open_files to 11 so table cache capacity will become 1. This will + // prevent file open during DB open and force the file to be opened + // during MultiGet + SyncPoint::GetInstance()->SetCallBack( + "SanitizeOptions::AfterChangeMaxOpenFiles", [&](void* arg) { + int* max_open_files = (int*)arg; + *max_open_files = 11; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + Reopen(CurrentOptions()); + + // Warm up the cache first + key_data.emplace_back(Key(0)); + keys.emplace_back(Slice(key_data.back())); + key_data.emplace_back(Key(50)); + keys.emplace_back(Slice(key_data.back())); + statuses.resize(keys.size()); + + dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data(), true); + ASSERT_EQ(statuses[0], Status::IOError()); + ASSERT_EQ(statuses[1], Status::IOError()); + + SyncPoint::GetInstance()->DisableProcessing(); +} + +INSTANTIATE_TEST_CASE_P(ParallelIO, DBBasicTestWithParallelIO, + // Params are as follows - + // Param 0 - Uncompressed cache enabled + // Param 1 - Data compression enabled + // Param 2 - ReadOptions::fill_cache + // Param 3 - CompressionOptions::parallel_threads + ::testing::Combine(::testing::Bool(), ::testing::Bool(), + ::testing::Bool(), + ::testing::Values(1, 4))); + +// Forward declaration +class DeadlineFS; + +class DeadlineRandomAccessFile : public FSRandomAccessFileOwnerWrapper { + public: + DeadlineRandomAccessFile(DeadlineFS& fs, + std::unique_ptr& file) + : FSRandomAccessFileOwnerWrapper(std::move(file)), fs_(fs) {} + + IOStatus Read(uint64_t offset, size_t len, const IOOptions& opts, + Slice* result, char* scratch, + IODebugContext* dbg) const override; + + IOStatus MultiRead(FSReadRequest* reqs, size_t num_reqs, + const IOOptions& options, IODebugContext* dbg) override; + + IOStatus ReadAsync(FSReadRequest& req, const IOOptions& opts, + std::function cb, + void* cb_arg, void** io_handle, IOHandleDeleter* del_fn, + IODebugContext* dbg) override; + + private: + DeadlineFS& fs_; + std::unique_ptr file_; +}; + +class DeadlineFS : public FileSystemWrapper { + public: + // The error_on_delay parameter specifies whether a IOStatus::TimedOut() + // status should be returned after delaying the IO to exceed the timeout, + // or to simply delay but return success anyway. The latter mimics the + // behavior of PosixFileSystem, which does not enforce any timeout + explicit DeadlineFS(SpecialEnv* env, bool error_on_delay) + : FileSystemWrapper(env->GetFileSystem()), + deadline_(std::chrono::microseconds::zero()), + io_timeout_(std::chrono::microseconds::zero()), + env_(env), + timedout_(false), + ignore_deadline_(false), + error_on_delay_(error_on_delay) {} + + static const char* kClassName() { return "DeadlineFileSystem"; } + const char* Name() const override { return kClassName(); } + + IOStatus NewRandomAccessFile(const std::string& fname, + const FileOptions& opts, + std::unique_ptr* result, + IODebugContext* dbg) override { + std::unique_ptr file; + IOStatus s = target()->NewRandomAccessFile(fname, opts, &file, dbg); + EXPECT_OK(s); + result->reset(new DeadlineRandomAccessFile(*this, file)); + + const std::chrono::microseconds deadline = GetDeadline(); + const std::chrono::microseconds io_timeout = GetIOTimeout(); + if (deadline.count() || io_timeout.count()) { + AssertDeadline(deadline, io_timeout, opts.io_options); + } + return ShouldDelay(opts.io_options); + } + + // Set a vector of {IO counter, delay in microseconds, return status} tuples + // that control when to inject a delay and duration of the delay + void SetDelayTrigger(const std::chrono::microseconds deadline, + const std::chrono::microseconds io_timeout, + const int trigger) { + delay_trigger_ = trigger; + io_count_ = 0; + deadline_ = deadline; + io_timeout_ = io_timeout; + timedout_ = false; + } + + // Increment the IO counter and return a delay in microseconds + IOStatus ShouldDelay(const IOOptions& opts) { + if (timedout_) { + return IOStatus::TimedOut(); + } else if (!deadline_.count() && !io_timeout_.count()) { + return IOStatus::OK(); + } + if (!ignore_deadline_ && delay_trigger_ == io_count_++) { + env_->SleepForMicroseconds(static_cast(opts.timeout.count() + 1)); + timedout_ = true; + if (error_on_delay_) { + return IOStatus::TimedOut(); + } + } + return IOStatus::OK(); + } + + const std::chrono::microseconds GetDeadline() { + return ignore_deadline_ ? std::chrono::microseconds::zero() : deadline_; + } + + const std::chrono::microseconds GetIOTimeout() { + return ignore_deadline_ ? std::chrono::microseconds::zero() : io_timeout_; + } + + bool TimedOut() { return timedout_; } + + void IgnoreDeadline(bool ignore) { ignore_deadline_ = ignore; } + + void AssertDeadline(const std::chrono::microseconds deadline, + const std::chrono::microseconds io_timeout, + const IOOptions& opts) const { + // Give a leeway of +- 10us as it can take some time for the Get/ + // MultiGet call to reach here, in order to avoid false alarms + std::chrono::microseconds now = + std::chrono::microseconds(env_->NowMicros()); + std::chrono::microseconds timeout; + if (deadline.count()) { + timeout = deadline - now; + if (io_timeout.count()) { + timeout = std::min(timeout, io_timeout); + } + } else { + timeout = io_timeout; + } + if (opts.timeout != timeout) { + ASSERT_EQ(timeout, opts.timeout); + } + } + + private: + // The number of IOs to trigger the delay after + int delay_trigger_; + // Current IO count + int io_count_; + // ReadOptions deadline for the Get/MultiGet/Iterator + std::chrono::microseconds deadline_; + // ReadOptions io_timeout for the Get/MultiGet/Iterator + std::chrono::microseconds io_timeout_; + SpecialEnv* env_; + // Flag to indicate whether we injected a delay + bool timedout_; + // Temporarily ignore deadlines/timeouts + bool ignore_deadline_; + // Return IOStatus::TimedOut() or IOStatus::OK() + bool error_on_delay_; +}; + +IOStatus DeadlineRandomAccessFile::Read(uint64_t offset, size_t len, + const IOOptions& opts, Slice* result, + char* scratch, + IODebugContext* dbg) const { + const std::chrono::microseconds deadline = fs_.GetDeadline(); + const std::chrono::microseconds io_timeout = fs_.GetIOTimeout(); + IOStatus s; + if (deadline.count() || io_timeout.count()) { + fs_.AssertDeadline(deadline, io_timeout, opts); + } + if (s.ok()) { + s = FSRandomAccessFileWrapper::Read(offset, len, opts, result, scratch, + dbg); + } + if (s.ok()) { + s = fs_.ShouldDelay(opts); + } + return s; +} + +IOStatus DeadlineRandomAccessFile::ReadAsync( + FSReadRequest& req, const IOOptions& opts, + std::function cb, void* cb_arg, + void** io_handle, IOHandleDeleter* del_fn, IODebugContext* dbg) { + const std::chrono::microseconds deadline = fs_.GetDeadline(); + const std::chrono::microseconds io_timeout = fs_.GetIOTimeout(); + IOStatus s; + if (deadline.count() || io_timeout.count()) { + fs_.AssertDeadline(deadline, io_timeout, opts); + } + if (s.ok()) { + s = FSRandomAccessFileWrapper::ReadAsync(req, opts, cb, cb_arg, io_handle, + del_fn, dbg); + } + if (s.ok()) { + s = fs_.ShouldDelay(opts); + } + return s; +} + +IOStatus DeadlineRandomAccessFile::MultiRead(FSReadRequest* reqs, + size_t num_reqs, + const IOOptions& options, + IODebugContext* dbg) { + const std::chrono::microseconds deadline = fs_.GetDeadline(); + const std::chrono::microseconds io_timeout = fs_.GetIOTimeout(); + IOStatus s; + if (deadline.count() || io_timeout.count()) { + fs_.AssertDeadline(deadline, io_timeout, options); + } + if (s.ok()) { + s = FSRandomAccessFileWrapper::MultiRead(reqs, num_reqs, options, dbg); + } + if (s.ok()) { + s = fs_.ShouldDelay(options); + } + return s; +} + +// A test class for intercepting random reads and injecting artificial +// delays. Used for testing the MultiGet deadline feature +class DBBasicTestMultiGetDeadline : public DBBasicTestMultiGet, + public testing::WithParamInterface { + public: + DBBasicTestMultiGetDeadline() + : DBBasicTestMultiGet( + "db_basic_test_multiget_deadline" /*Test dir*/, + 10 /*# of column families*/, true /*uncompressed cache enabled*/, + true /*compression enabled*/, true /*ReadOptions.fill_cache*/, + 1 /*# of parallel compression threads*/) {} + + inline void CheckStatus(std::vector& statuses, size_t num_ok) { + for (size_t i = 0; i < statuses.size(); ++i) { + if (i < num_ok) { + EXPECT_OK(statuses[i]); + } else { + if (statuses[i] != Status::TimedOut()) { + EXPECT_EQ(statuses[i], Status::TimedOut()); + } + } + } + } +}; + +TEST_P(DBBasicTestMultiGetDeadline, MultiGetDeadlineExceeded) { +#ifndef USE_COROUTINES + if (GetParam()) { + ROCKSDB_GTEST_SKIP("This test requires coroutine support"); + return; + } +#endif // USE_COROUTINES + std::shared_ptr fs = std::make_shared(env_, false); + std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); + Options options = CurrentOptions(); + + std::shared_ptr cache = NewLRUCache(1048576); + BlockBasedTableOptions table_options; + table_options.block_cache = cache; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.env = env.get(); + SetTimeElapseOnlySleepOnReopen(&options); + ReopenWithColumnFamilies(GetCFNames(), options); + + // Test the non-batched version of MultiGet with multiple column + // families + std::vector key_str; + size_t i; + for (i = 0; i < 5; ++i) { + key_str.emplace_back(Key(static_cast(i))); + } + std::vector cfs(key_str.size()); + ; + std::vector keys(key_str.size()); + std::vector values(key_str.size()); + for (i = 0; i < key_str.size(); ++i) { + cfs[i] = handles_[i]; + keys[i] = Slice(key_str[i].data(), key_str[i].size()); + } + + ReadOptions ro; + ro.deadline = std::chrono::microseconds{env->NowMicros() + 10000}; + ro.async_io = GetParam(); + // Delay the first IO + fs->SetDelayTrigger(ro.deadline, ro.io_timeout, 0); + + std::vector statuses = dbfull()->MultiGet(ro, cfs, keys, &values); + // The first key is successful because we check after the lookup, but + // subsequent keys fail due to deadline exceeded + CheckStatus(statuses, 1); + + // Clear the cache + cache->SetCapacity(0); + cache->SetCapacity(1048576); + // Test non-batched Multiget with multiple column families and + // introducing an IO delay in one of the middle CFs + key_str.clear(); + for (i = 0; i < 10; ++i) { + key_str.emplace_back(Key(static_cast(i))); + } + cfs.resize(key_str.size()); + keys.resize(key_str.size()); + values.resize(key_str.size()); + for (i = 0; i < key_str.size(); ++i) { + // 2 keys per CF + cfs[i] = handles_[i / 2]; + keys[i] = Slice(key_str[i].data(), key_str[i].size()); + } + ro.deadline = std::chrono::microseconds{env->NowMicros() + 10000}; + fs->SetDelayTrigger(ro.deadline, ro.io_timeout, 1); + statuses = dbfull()->MultiGet(ro, cfs, keys, &values); + CheckStatus(statuses, 3); + + // Test batched MultiGet with an IO delay in the first data block read. + // Both keys in the first CF should succeed as they're in the same data + // block and would form one batch, and we check for deadline between + // batches. + std::vector pin_values(keys.size()); + cache->SetCapacity(0); + cache->SetCapacity(1048576); + statuses.clear(); + statuses.resize(keys.size()); + ro.deadline = std::chrono::microseconds{env->NowMicros() + 10000}; + fs->SetDelayTrigger(ro.deadline, ro.io_timeout, 0); + dbfull()->MultiGet(ro, keys.size(), cfs.data(), keys.data(), + pin_values.data(), statuses.data()); + CheckStatus(statuses, 2); + + // Similar to the previous one, but an IO delay in the third CF data block + // read + for (PinnableSlice& value : pin_values) { + value.Reset(); + } + cache->SetCapacity(0); + cache->SetCapacity(1048576); + statuses.clear(); + statuses.resize(keys.size()); + ro.deadline = std::chrono::microseconds{env->NowMicros() + 10000}; + fs->SetDelayTrigger(ro.deadline, ro.io_timeout, 2); + dbfull()->MultiGet(ro, keys.size(), cfs.data(), keys.data(), + pin_values.data(), statuses.data()); + CheckStatus(statuses, 6); + + // Similar to the previous one, but an IO delay in the last but one CF + for (PinnableSlice& value : pin_values) { + value.Reset(); + } + cache->SetCapacity(0); + cache->SetCapacity(1048576); + statuses.clear(); + statuses.resize(keys.size()); + ro.deadline = std::chrono::microseconds{env->NowMicros() + 10000}; + fs->SetDelayTrigger(ro.deadline, ro.io_timeout, 3); + dbfull()->MultiGet(ro, keys.size(), cfs.data(), keys.data(), + pin_values.data(), statuses.data()); + CheckStatus(statuses, 8); + + // Test batched MultiGet with single CF and lots of keys. Inject delay + // into the second batch of keys. As each batch is 32, the first 64 keys, + // i.e first two batches, should succeed and the rest should time out + for (PinnableSlice& value : pin_values) { + value.Reset(); + } + cache->SetCapacity(0); + cache->SetCapacity(1048576); + key_str.clear(); + for (i = 0; i < 100; ++i) { + key_str.emplace_back(Key(static_cast(i))); + } + keys.resize(key_str.size()); + pin_values.clear(); + pin_values.resize(key_str.size()); + for (i = 0; i < key_str.size(); ++i) { + keys[i] = Slice(key_str[i].data(), key_str[i].size()); + } + statuses.clear(); + statuses.resize(keys.size()); + ro.deadline = std::chrono::microseconds{env->NowMicros() + 10000}; + fs->SetDelayTrigger(ro.deadline, ro.io_timeout, 1); + dbfull()->MultiGet(ro, handles_[0], keys.size(), keys.data(), + pin_values.data(), statuses.data()); + CheckStatus(statuses, 64); + Close(); +} + +INSTANTIATE_TEST_CASE_P(DeadlineIO, DBBasicTestMultiGetDeadline, + ::testing::Bool()); + +TEST_F(DBBasicTest, ManifestWriteFailure) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.env = env_; + DestroyAndReopen(options); + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Flush()); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::ProcessManifestWrites:AfterSyncManifest", [&](void* arg) { + ASSERT_NE(nullptr, arg); + auto* s = reinterpret_cast(arg); + ASSERT_OK(*s); + // Manually overwrite return status + *s = Status::IOError(); + }); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(Put("key", "value")); + ASSERT_NOK(Flush()); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->EnableProcessing(); + Reopen(options); +} + +TEST_F(DBBasicTest, DestroyDefaultCfHandle) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + for (const auto* h : handles_) { + ASSERT_NE(db_->DefaultColumnFamily(), h); + } + + // We have two handles to the default column family. The two handles point to + // different ColumnFamilyHandle objects. + assert(db_->DefaultColumnFamily()); + ASSERT_EQ(0U, db_->DefaultColumnFamily()->GetID()); + assert(handles_[0]); + ASSERT_EQ(0U, handles_[0]->GetID()); + + // You can destroy handles_[...]. + for (auto* h : handles_) { + ASSERT_OK(db_->DestroyColumnFamilyHandle(h)); + } + handles_.clear(); + + // But you should not destroy db_->DefaultColumnFamily(), since it's going to + // be deleted in `DBImpl::CloseHelper()`. Before that, it may be used + // elsewhere internally too. + ColumnFamilyHandle* default_cf = db_->DefaultColumnFamily(); + ASSERT_TRUE(db_->DestroyColumnFamilyHandle(default_cf).IsInvalidArgument()); +} + +TEST_F(DBBasicTest, FailOpenIfLoggerCreationFail) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "rocksdb::CreateLoggerFromOptions:AfterGetPath", [&](void* arg) { + auto* s = reinterpret_cast(arg); + assert(s); + *s = Status::IOError("Injected"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + Status s = TryReopen(options); + ASSERT_EQ(nullptr, options.info_log); + ASSERT_TRUE(s.IsIOError()); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(DBBasicTest, VerifyFileChecksums) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.env = env_; + DestroyAndReopen(options); + ASSERT_OK(Put("a", "value")); + ASSERT_OK(Flush()); + ASSERT_TRUE(db_->VerifyFileChecksums(ReadOptions()).IsInvalidArgument()); + + options.file_checksum_gen_factory = GetFileChecksumGenCrc32cFactory(); + Reopen(options); + ASSERT_OK(db_->VerifyFileChecksums(ReadOptions())); + + // Write an L0 with checksum computed. + ASSERT_OK(Put("b", "value")); + ASSERT_OK(Flush()); + + ASSERT_OK(db_->VerifyFileChecksums(ReadOptions())); + + // Does the right thing but with the wrong name -- using it should lead to an + // error. + class MisnamedFileChecksumGenerator : public FileChecksumGenCrc32c { + public: + MisnamedFileChecksumGenerator(const FileChecksumGenContext& context) + : FileChecksumGenCrc32c(context) {} + + const char* Name() const override { return "sha1"; } + }; + + class MisnamedFileChecksumGenFactory : public FileChecksumGenCrc32cFactory { + public: + std::unique_ptr CreateFileChecksumGenerator( + const FileChecksumGenContext& context) override { + return std::unique_ptr( + new MisnamedFileChecksumGenerator(context)); + } + }; + + options.file_checksum_gen_factory.reset(new MisnamedFileChecksumGenFactory()); + Reopen(options); + ASSERT_TRUE(db_->VerifyFileChecksums(ReadOptions()).IsInvalidArgument()); +} + +TEST_F(DBBasicTest, VerifyFileChecksumsReadahead) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.env = env_; + options.file_checksum_gen_factory = GetFileChecksumGenCrc32cFactory(); + DestroyAndReopen(options); + + Random rnd(301); + int alignment = 256 * 1024; + for (int i = 0; i < 16; ++i) { + ASSERT_OK(Put("key" + std::to_string(i), rnd.RandomString(alignment))); + } + ASSERT_OK(Flush()); + + std::vector filenames; + int sst_cnt = 0; + std::string sst_name; + uint64_t sst_size; + uint64_t number; + FileType type; + ASSERT_OK(env_->GetChildren(dbname_, &filenames)); + for (auto name : filenames) { + if (ParseFileName(name, &number, &type)) { + if (type == kTableFile) { + sst_cnt++; + sst_name = name; + } + } + } + ASSERT_EQ(sst_cnt, 1); + ASSERT_OK(env_->GetFileSize(dbname_ + '/' + sst_name, &sst_size)); + + bool last_read = false; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "GenerateOneFileChecksum::Chunk:0", [&](void* /*arg*/) { + if (env_->random_read_bytes_counter_.load() == sst_size) { + EXPECT_FALSE(last_read); + last_read = true; + } else { + ASSERT_EQ(env_->random_read_bytes_counter_.load() & (alignment - 1), + 0); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + env_->count_random_reads_ = true; + env_->random_read_bytes_counter_ = 0; + env_->random_read_counter_.Reset(); + + ReadOptions ro; + ro.readahead_size = alignment; + ASSERT_OK(db_->VerifyFileChecksums(ro)); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ASSERT_TRUE(last_read); + ASSERT_EQ(env_->random_read_counter_.Read(), + (sst_size + alignment - 1) / (alignment)); +} + +// TODO: re-enable after we provide finer-grained control for WAL tracking to +// meet the needs of different use cases, durability levels and recovery modes. +TEST_F(DBBasicTest, DISABLED_ManualWalSync) { + Options options = CurrentOptions(); + options.track_and_verify_wals_in_manifest = true; + options.wal_recovery_mode = WALRecoveryMode::kAbsoluteConsistency; + DestroyAndReopen(options); + + ASSERT_OK(Put("x", "y")); + // This does not create a new WAL. + ASSERT_OK(db_->SyncWAL()); + EXPECT_FALSE(dbfull()->GetVersionSet()->GetWalSet().GetWals().empty()); + + std::unique_ptr wal; + Status s = db_->GetCurrentWalFile(&wal); + ASSERT_OK(s); + Close(); + + EXPECT_OK(env_->DeleteFile(LogFileName(dbname_, wal->LogNumber()))); + + ASSERT_TRUE(TryReopen(options).IsCorruption()); +} + +// A test class for intercepting random reads and injecting artificial +// delays. Used for testing the deadline/timeout feature +class DBBasicTestDeadline + : public DBBasicTest, + public testing::WithParamInterface> {}; + +TEST_P(DBBasicTestDeadline, PointLookupDeadline) { + std::shared_ptr fs = std::make_shared(env_, true); + std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); + bool set_deadline = std::get<0>(GetParam()); + bool set_timeout = std::get<1>(GetParam()); + + for (int option_config = kDefault; option_config < kEnd; ++option_config) { + if (ShouldSkipOptions(option_config, kSkipPlainTable | kSkipMmapReads)) { + continue; + } + option_config_ = option_config; + Options options = CurrentOptions(); + if (options.use_direct_reads) { + continue; + } + options.env = env.get(); + options.disable_auto_compactions = true; + Cache* block_cache = nullptr; + // Fileter block reads currently don't cause the request to get + // aborted on a read timeout, so its possible those block reads + // may get issued even if the deadline is past + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTable::Get:BeforeFilterMatch", + [&](void* /*arg*/) { fs->IgnoreDeadline(true); }); + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTable::Get:AfterFilterMatch", + [&](void* /*arg*/) { fs->IgnoreDeadline(false); }); + // DB open will create table readers unless we reduce the table cache + // capacity. + // SanitizeOptions will set max_open_files to minimum of 20. Table cache + // is allocated with max_open_files - 10 as capacity. So override + // max_open_files to 11 so table cache capacity will become 1. This will + // prevent file open during DB open and force the file to be opened + // during MultiGet + SyncPoint::GetInstance()->SetCallBack( + "SanitizeOptions::AfterChangeMaxOpenFiles", [&](void* arg) { + int* max_open_files = (int*)arg; + *max_open_files = 11; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + SetTimeElapseOnlySleepOnReopen(&options); + Reopen(options); + + if (options.table_factory) { + block_cache = options.table_factory->GetOptions( + TableFactory::kBlockCacheOpts()); + } + + Random rnd(301); + for (int i = 0; i < 400; ++i) { + std::string key = "k" + std::to_string(i); + ASSERT_OK(Put(key, rnd.RandomString(100))); + } + ASSERT_OK(Flush()); + + bool timedout = true; + // A timeout will be forced when the IO counter reaches this value + int io_deadline_trigger = 0; + // Keep incrementing io_deadline_trigger and call Get() until there is an + // iteration that doesn't cause a timeout. This ensures that we cover + // all file reads in the point lookup path that can potentially timeout + // and cause the Get() to fail. + while (timedout) { + ReadOptions ro; + if (set_deadline) { + ro.deadline = std::chrono::microseconds{env->NowMicros() + 10000}; + } + if (set_timeout) { + ro.io_timeout = std::chrono::microseconds{5000}; + } + fs->SetDelayTrigger(ro.deadline, ro.io_timeout, io_deadline_trigger); + + block_cache->SetCapacity(0); + block_cache->SetCapacity(1048576); + + std::string value; + Status s = dbfull()->Get(ro, "k50", &value); + if (fs->TimedOut()) { + ASSERT_EQ(s, Status::TimedOut()); + } else { + timedout = false; + ASSERT_OK(s); + } + io_deadline_trigger++; + } + // Reset the delay sequence in order to avoid false alarms during Reopen + fs->SetDelayTrigger(std::chrono::microseconds::zero(), + std::chrono::microseconds::zero(), 0); + } + Close(); +} + +TEST_P(DBBasicTestDeadline, IteratorDeadline) { + std::shared_ptr fs = std::make_shared(env_, true); + std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); + bool set_deadline = std::get<0>(GetParam()); + bool set_timeout = std::get<1>(GetParam()); + + for (int option_config = kDefault; option_config < kEnd; ++option_config) { + if (ShouldSkipOptions(option_config, kSkipPlainTable | kSkipMmapReads)) { + continue; + } + Options options = CurrentOptions(); + if (options.use_direct_reads) { + continue; + } + options.env = env.get(); + options.disable_auto_compactions = true; + Cache* block_cache = nullptr; + // DB open will create table readers unless we reduce the table cache + // capacity. + // SanitizeOptions will set max_open_files to minimum of 20. Table cache + // is allocated with max_open_files - 10 as capacity. So override + // max_open_files to 11 so table cache capacity will become 1. This will + // prevent file open during DB open and force the file to be opened + // during MultiGet + SyncPoint::GetInstance()->SetCallBack( + "SanitizeOptions::AfterChangeMaxOpenFiles", [&](void* arg) { + int* max_open_files = (int*)arg; + *max_open_files = 11; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + SetTimeElapseOnlySleepOnReopen(&options); + Reopen(options); + + if (options.table_factory) { + block_cache = options.table_factory->GetOptions( + TableFactory::kBlockCacheOpts()); + } + + Random rnd(301); + for (int i = 0; i < 400; ++i) { + std::string key = "k" + std::to_string(i); + ASSERT_OK(Put(key, rnd.RandomString(100))); + } + ASSERT_OK(Flush()); + + bool timedout = true; + // A timeout will be forced when the IO counter reaches this value + int io_deadline_trigger = 0; + // Keep incrementing io_deadline_trigger and call Get() until there is an + // iteration that doesn't cause a timeout. This ensures that we cover + // all file reads in the point lookup path that can potentially timeout + while (timedout) { + ReadOptions ro; + if (set_deadline) { + ro.deadline = std::chrono::microseconds{env->NowMicros() + 10000}; + } + if (set_timeout) { + ro.io_timeout = std::chrono::microseconds{5000}; + } + fs->SetDelayTrigger(ro.deadline, ro.io_timeout, io_deadline_trigger); + + block_cache->SetCapacity(0); + block_cache->SetCapacity(1048576); + + Iterator* iter = dbfull()->NewIterator(ro); + int count = 0; + iter->Seek("k50"); + while (iter->Valid() && count++ < 100) { + iter->Next(); + } + if (fs->TimedOut()) { + ASSERT_FALSE(iter->Valid()); + ASSERT_EQ(iter->status(), Status::TimedOut()); + } else { + timedout = false; + ASSERT_OK(iter->status()); + } + delete iter; + io_deadline_trigger++; + } + // Reset the delay sequence in order to avoid false alarms during Reopen + fs->SetDelayTrigger(std::chrono::microseconds::zero(), + std::chrono::microseconds::zero(), 0); + } + Close(); +} + +// Param 0: If true, set read_options.deadline +// Param 1: If true, set read_options.io_timeout +INSTANTIATE_TEST_CASE_P(DBBasicTestDeadline, DBBasicTestDeadline, + ::testing::Values(std::make_tuple(true, false), + std::make_tuple(false, true), + std::make_tuple(true, true))); +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_block_cache_test.cc b/librocksdb-sys/rocksdb/db/db_block_cache_test.cc new file mode 100644 index 0000000..83a027f --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_block_cache_test.cc @@ -0,0 +1,1972 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include +#include +#include +#include + +#include "cache/cache_entry_roles.h" +#include "cache/cache_key.h" +#include "cache/lru_cache.h" +#include "cache/typed_cache.h" +#include "db/column_family.h" +#include "db/db_impl/db_impl.h" +#include "db/db_test_util.h" +#include "env/unique_id_gen.h" +#include "port/stack_trace.h" +#include "rocksdb/persistent_cache.h" +#include "rocksdb/statistics.h" +#include "rocksdb/table.h" +#include "rocksdb/table_properties.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/unique_id_impl.h" +#include "util/compression.h" +#include "util/defer.h" +#include "util/hash.h" +#include "util/math.h" +#include "util/random.h" +#include "utilities/fault_injection_fs.h" + +namespace ROCKSDB_NAMESPACE { + +class DBBlockCacheTest : public DBTestBase { + private: + size_t miss_count_ = 0; + size_t hit_count_ = 0; + size_t insert_count_ = 0; + size_t failure_count_ = 0; + size_t compression_dict_miss_count_ = 0; + size_t compression_dict_hit_count_ = 0; + size_t compression_dict_insert_count_ = 0; + + public: + const size_t kNumBlocks = 10; + const size_t kValueSize = 100; + + DBBlockCacheTest() + : DBTestBase("db_block_cache_test", /*env_do_fsync=*/true) {} + + BlockBasedTableOptions GetTableOptions() { + BlockBasedTableOptions table_options; + // Set a small enough block size so that each key-value get its own block. + table_options.block_size = 1; + return table_options; + } + + Options GetOptions(const BlockBasedTableOptions& table_options) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.avoid_flush_during_recovery = false; + // options.compression = kNoCompression; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + return options; + } + + void InitTable(const Options& /*options*/) { + std::string value(kValueSize, 'a'); + for (size_t i = 0; i < kNumBlocks; i++) { + ASSERT_OK(Put(std::to_string(i), value.c_str())); + } + } + + void RecordCacheCounters(const Options& options) { + miss_count_ = TestGetTickerCount(options, BLOCK_CACHE_MISS); + hit_count_ = TestGetTickerCount(options, BLOCK_CACHE_HIT); + insert_count_ = TestGetTickerCount(options, BLOCK_CACHE_ADD); + failure_count_ = TestGetTickerCount(options, BLOCK_CACHE_ADD_FAILURES); + } + + void RecordCacheCountersForCompressionDict(const Options& options) { + compression_dict_miss_count_ = + TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_MISS); + compression_dict_hit_count_ = + TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_HIT); + compression_dict_insert_count_ = + TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_ADD); + } + + void CheckCacheCounters(const Options& options, size_t expected_misses, + size_t expected_hits, size_t expected_inserts, + size_t expected_failures) { + size_t new_miss_count = TestGetTickerCount(options, BLOCK_CACHE_MISS); + size_t new_hit_count = TestGetTickerCount(options, BLOCK_CACHE_HIT); + size_t new_insert_count = TestGetTickerCount(options, BLOCK_CACHE_ADD); + size_t new_failure_count = + TestGetTickerCount(options, BLOCK_CACHE_ADD_FAILURES); + ASSERT_EQ(miss_count_ + expected_misses, new_miss_count); + ASSERT_EQ(hit_count_ + expected_hits, new_hit_count); + ASSERT_EQ(insert_count_ + expected_inserts, new_insert_count); + ASSERT_EQ(failure_count_ + expected_failures, new_failure_count); + miss_count_ = new_miss_count; + hit_count_ = new_hit_count; + insert_count_ = new_insert_count; + failure_count_ = new_failure_count; + } + + void CheckCacheCountersForCompressionDict( + const Options& options, size_t expected_compression_dict_misses, + size_t expected_compression_dict_hits, + size_t expected_compression_dict_inserts) { + size_t new_compression_dict_miss_count = + TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_MISS); + size_t new_compression_dict_hit_count = + TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_HIT); + size_t new_compression_dict_insert_count = + TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_ADD); + ASSERT_EQ(compression_dict_miss_count_ + expected_compression_dict_misses, + new_compression_dict_miss_count); + ASSERT_EQ(compression_dict_hit_count_ + expected_compression_dict_hits, + new_compression_dict_hit_count); + ASSERT_EQ( + compression_dict_insert_count_ + expected_compression_dict_inserts, + new_compression_dict_insert_count); + compression_dict_miss_count_ = new_compression_dict_miss_count; + compression_dict_hit_count_ = new_compression_dict_hit_count; + compression_dict_insert_count_ = new_compression_dict_insert_count; + } + + const std::array GetCacheEntryRoleCountsBg() { + // Verify in cache entry role stats + std::array cache_entry_role_counts; + std::map values; + EXPECT_TRUE(db_->GetMapProperty(DB::Properties::kFastBlockCacheEntryStats, + &values)); + for (size_t i = 0; i < kNumCacheEntryRoles; ++i) { + auto role = static_cast(i); + cache_entry_role_counts[i] = + ParseSizeT(values[BlockCacheEntryStatsMapKeys::EntryCount(role)]); + } + return cache_entry_role_counts; + } +}; + +TEST_F(DBBlockCacheTest, IteratorBlockCacheUsage) { + ReadOptions read_options; + read_options.fill_cache = false; + auto table_options = GetTableOptions(); + auto options = GetOptions(table_options); + InitTable(options); + + LRUCacheOptions co; + co.capacity = 0; + co.num_shard_bits = 0; + co.strict_capacity_limit = false; + // Needed not to count entry stats collector + co.metadata_charge_policy = kDontChargeCacheMetadata; + std::shared_ptr cache = NewLRUCache(co); + table_options.block_cache = cache; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + RecordCacheCounters(options); + + std::vector> iterators(kNumBlocks - 1); + Iterator* iter = nullptr; + + ASSERT_EQ(0, cache->GetUsage()); + iter = db_->NewIterator(read_options); + iter->Seek(std::to_string(0)); + ASSERT_LT(0, cache->GetUsage()); + delete iter; + iter = nullptr; + ASSERT_EQ(0, cache->GetUsage()); +} + +TEST_F(DBBlockCacheTest, TestWithoutCompressedBlockCache) { + ReadOptions read_options; + auto table_options = GetTableOptions(); + auto options = GetOptions(table_options); + InitTable(options); + + LRUCacheOptions co; + co.capacity = 0; + co.num_shard_bits = 0; + co.strict_capacity_limit = false; + // Needed not to count entry stats collector + co.metadata_charge_policy = kDontChargeCacheMetadata; + std::shared_ptr cache = NewLRUCache(co); + table_options.block_cache = cache; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + RecordCacheCounters(options); + + std::vector> iterators(kNumBlocks - 1); + Iterator* iter = nullptr; + + // Load blocks into cache. + for (size_t i = 0; i + 1 < kNumBlocks; i++) { + iter = db_->NewIterator(read_options); + iter->Seek(std::to_string(i)); + ASSERT_OK(iter->status()); + CheckCacheCounters(options, 1, 0, 1, 0); + iterators[i].reset(iter); + } + size_t usage = cache->GetUsage(); + ASSERT_LT(0, usage); + cache->SetCapacity(usage); + ASSERT_EQ(usage, cache->GetPinnedUsage()); + + // Test with strict capacity limit. + cache->SetStrictCapacityLimit(true); + iter = db_->NewIterator(read_options); + iter->Seek(std::to_string(kNumBlocks - 1)); + ASSERT_TRUE(iter->status().IsMemoryLimit()); + CheckCacheCounters(options, 1, 0, 0, 1); + delete iter; + iter = nullptr; + + // Release iterators and access cache again. + for (size_t i = 0; i + 1 < kNumBlocks; i++) { + iterators[i].reset(); + CheckCacheCounters(options, 0, 0, 0, 0); + } + ASSERT_EQ(0, cache->GetPinnedUsage()); + for (size_t i = 0; i + 1 < kNumBlocks; i++) { + iter = db_->NewIterator(read_options); + iter->Seek(std::to_string(i)); + ASSERT_OK(iter->status()); + CheckCacheCounters(options, 0, 1, 0, 0); + iterators[i].reset(iter); + } +} + +#ifdef SNAPPY + +namespace { +class PersistentCacheFromCache : public PersistentCache { + public: + PersistentCacheFromCache(std::shared_ptr cache, bool read_only) + : cache_(cache), read_only_(read_only) {} + + Status Insert(const Slice& key, const char* data, + const size_t size) override { + if (read_only_) { + return Status::NotSupported(); + } + std::unique_ptr copy{new char[size]}; + std::copy_n(data, size, copy.get()); + Status s = cache_.Insert(key, copy.get(), size); + if (s.ok()) { + copy.release(); + } + return s; + } + + Status Lookup(const Slice& key, std::unique_ptr* data, + size_t* size) override { + auto handle = cache_.Lookup(key); + if (handle) { + char* ptr = cache_.Value(handle); + *size = cache_.get()->GetCharge(handle); + data->reset(new char[*size]); + std::copy_n(ptr, *size, data->get()); + cache_.Release(handle); + return Status::OK(); + } else { + return Status::NotFound(); + } + } + + bool IsCompressed() override { return false; } + + StatsType Stats() override { return StatsType(); } + + std::string GetPrintableOptions() const override { return ""; } + + uint64_t NewId() override { return cache_.get()->NewId(); } + + private: + BasicTypedSharedCacheInterface cache_; + bool read_only_; +}; + +class ReadOnlyCacheWrapper : public CacheWrapper { + public: + using CacheWrapper::CacheWrapper; + + const char* Name() const override { return "ReadOnlyCacheWrapper"; } + + Status Insert(const Slice& /*key*/, Cache::ObjectPtr /*value*/, + const CacheItemHelper* /*helper*/, size_t /*charge*/, + Handle** /*handle*/, Priority /*priority*/) override { + return Status::NotSupported(); + } +}; + +} // anonymous namespace +#endif // SNAPPY + + +// Make sure that when options.block_cache is set, after a new table is +// created its index/filter blocks are added to block cache. +TEST_F(DBBlockCacheTest, IndexAndFilterBlocksOfNewTableAddedToCache) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + BlockBasedTableOptions table_options; + table_options.cache_index_and_filter_blocks = true; + table_options.filter_policy.reset(NewBloomFilterPolicy(20)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(1, "key", "val")); + // Create a new table. + ASSERT_OK(Flush(1)); + + // index/filter blocks added to block cache right after table creation. + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(2, /* only index/filter were added */ + TestGetTickerCount(options, BLOCK_CACHE_ADD)); + ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_MISS)); + uint64_t int_num; + ASSERT_TRUE( + dbfull()->GetIntProperty("rocksdb.estimate-table-readers-mem", &int_num)); + ASSERT_EQ(int_num, 0U); + + // Make sure filter block is in cache. + std::string value; + ReadOptions ropt; + db_->KeyMayExist(ReadOptions(), handles_[1], "key", &value); + + // Miss count should remain the same. + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT)); + + db_->KeyMayExist(ReadOptions(), handles_[1], "key", &value); + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT)); + + // Make sure index block is in cache. + auto index_block_hit = TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT); + value = Get(1, "key"); + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); + ASSERT_EQ(index_block_hit + 1, + TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT)); + + value = Get(1, "key"); + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); + ASSERT_EQ(index_block_hit + 2, + TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT)); +} + +// With fill_cache = false, fills up the cache, then iterates over the entire +// db, verify dummy entries inserted in `BlockBasedTable::NewDataBlockIterator` +// does not cause heap-use-after-free errors in COMPILE_WITH_ASAN=1 runs +TEST_F(DBBlockCacheTest, FillCacheAndIterateDB) { + ReadOptions read_options; + read_options.fill_cache = false; + auto table_options = GetTableOptions(); + auto options = GetOptions(table_options); + InitTable(options); + + std::shared_ptr cache = NewLRUCache(10, 0, true); + table_options.block_cache = cache; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + ASSERT_OK(Put("key1", "val1")); + ASSERT_OK(Put("key2", "val2")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("key3", "val3")); + ASSERT_OK(Put("key4", "val4")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("key5", "val5")); + ASSERT_OK(Put("key6", "val6")); + ASSERT_OK(Flush()); + + Iterator* iter = nullptr; + + iter = db_->NewIterator(read_options); + iter->Seek(std::to_string(0)); + while (iter->Valid()) { + iter->Next(); + } + delete iter; + iter = nullptr; +} + +TEST_F(DBBlockCacheTest, IndexAndFilterBlocksStats) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + BlockBasedTableOptions table_options; + table_options.cache_index_and_filter_blocks = true; + LRUCacheOptions co; + // 500 bytes are enough to hold the first two blocks + co.capacity = 500; + co.num_shard_bits = 0; + co.strict_capacity_limit = false; + co.metadata_charge_policy = kDontChargeCacheMetadata; + std::shared_ptr cache = NewLRUCache(co); + table_options.block_cache = cache; + table_options.filter_policy.reset(NewBloomFilterPolicy(20, true)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(1, "longer_key", "val")); + // Create a new table + ASSERT_OK(Flush(1)); + size_t index_bytes_insert = + TestGetTickerCount(options, BLOCK_CACHE_INDEX_BYTES_INSERT); + size_t filter_bytes_insert = + TestGetTickerCount(options, BLOCK_CACHE_FILTER_BYTES_INSERT); + ASSERT_GT(index_bytes_insert, 0); + ASSERT_GT(filter_bytes_insert, 0); + ASSERT_EQ(cache->GetUsage(), index_bytes_insert + filter_bytes_insert); + // set the cache capacity to the current usage + cache->SetCapacity(index_bytes_insert + filter_bytes_insert); + // Note that the second key needs to be no longer than the first one. + // Otherwise the second index block may not fit in cache. + ASSERT_OK(Put(1, "key", "val")); + // Create a new table + ASSERT_OK(Flush(1)); + // cache evicted old index and block entries + ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_INDEX_BYTES_INSERT), + index_bytes_insert); + ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_FILTER_BYTES_INSERT), + filter_bytes_insert); +} + +#if (defined OS_LINUX || defined OS_WIN) +TEST_F(DBBlockCacheTest, WarmCacheWithDataBlocksDuringFlush) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + + BlockBasedTableOptions table_options; + table_options.block_cache = NewLRUCache(1 << 25, 0, false); + table_options.cache_index_and_filter_blocks = false; + table_options.prepopulate_block_cache = + BlockBasedTableOptions::PrepopulateBlockCache::kFlushOnly; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DestroyAndReopen(options); + + std::string value(kValueSize, 'a'); + for (size_t i = 1; i <= kNumBlocks; i++) { + ASSERT_OK(Put(std::to_string(i), value)); + ASSERT_OK(Flush()); + ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_DATA_ADD)); + ASSERT_EQ(value, Get(std::to_string(i))); + ASSERT_EQ(0, options.statistics->getTickerCount(BLOCK_CACHE_DATA_MISS)); + ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_DATA_HIT)); + } + // Verify compaction not counted + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr, + /*end=*/nullptr)); + EXPECT_EQ(kNumBlocks, + options.statistics->getTickerCount(BLOCK_CACHE_DATA_ADD)); +} + +// This test cache data, index and filter blocks during flush. +class DBBlockCacheTest1 : public DBTestBase, + public ::testing::WithParamInterface { + public: + const size_t kNumBlocks = 10; + const size_t kValueSize = 100; + DBBlockCacheTest1() : DBTestBase("db_block_cache_test1", true) {} +}; + +INSTANTIATE_TEST_CASE_P(DBBlockCacheTest1, DBBlockCacheTest1, + ::testing::Values(1, 2)); + +TEST_P(DBBlockCacheTest1, WarmCacheWithBlocksDuringFlush) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + + BlockBasedTableOptions table_options; + table_options.block_cache = NewLRUCache(1 << 25, 0, false); + + uint32_t filter_type = GetParam(); + switch (filter_type) { + case 1: // partition_filter + table_options.partition_filters = true; + table_options.index_type = + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + table_options.filter_policy.reset(NewBloomFilterPolicy(10)); + break; + case 2: // full filter + table_options.filter_policy.reset(NewBloomFilterPolicy(10)); + break; + default: + assert(false); + } + + table_options.cache_index_and_filter_blocks = true; + table_options.prepopulate_block_cache = + BlockBasedTableOptions::PrepopulateBlockCache::kFlushOnly; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DestroyAndReopen(options); + + std::string value(kValueSize, 'a'); + for (size_t i = 1; i <= kNumBlocks; i++) { + ASSERT_OK(Put(std::to_string(i), value)); + ASSERT_OK(Flush()); + ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_DATA_ADD)); + if (filter_type == 1) { + ASSERT_EQ(2 * i, + options.statistics->getTickerCount(BLOCK_CACHE_INDEX_ADD)); + ASSERT_EQ(2 * i, + options.statistics->getTickerCount(BLOCK_CACHE_FILTER_ADD)); + } else { + ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_INDEX_ADD)); + ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_FILTER_ADD)); + } + ASSERT_EQ(value, Get(std::to_string(i))); + + ASSERT_EQ(0, options.statistics->getTickerCount(BLOCK_CACHE_DATA_MISS)); + ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_DATA_HIT)); + + ASSERT_EQ(0, options.statistics->getTickerCount(BLOCK_CACHE_INDEX_MISS)); + ASSERT_EQ(i * 3, options.statistics->getTickerCount(BLOCK_CACHE_INDEX_HIT)); + if (filter_type == 1) { + ASSERT_EQ(i * 3, + options.statistics->getTickerCount(BLOCK_CACHE_FILTER_HIT)); + } else { + ASSERT_EQ(i * 2, + options.statistics->getTickerCount(BLOCK_CACHE_FILTER_HIT)); + } + ASSERT_EQ(0, options.statistics->getTickerCount(BLOCK_CACHE_FILTER_MISS)); + } + + // Verify compaction not counted + CompactRangeOptions cro; + // Ensure files are rewritten, not just trivially moved. + cro.bottommost_level_compaction = BottommostLevelCompaction::kForceOptimized; + ASSERT_OK(db_->CompactRange(cro, /*begin=*/nullptr, /*end=*/nullptr)); + EXPECT_EQ(kNumBlocks, + options.statistics->getTickerCount(BLOCK_CACHE_DATA_ADD)); + // Index and filter blocks are automatically warmed when the new table file + // is automatically opened at the end of compaction. This is not easily + // disabled so results in the new index and filter blocks being warmed. + if (filter_type == 1) { + EXPECT_EQ(2 * (1 + kNumBlocks), + options.statistics->getTickerCount(BLOCK_CACHE_INDEX_ADD)); + EXPECT_EQ(2 * (1 + kNumBlocks), + options.statistics->getTickerCount(BLOCK_CACHE_FILTER_ADD)); + } else { + EXPECT_EQ(1 + kNumBlocks, + options.statistics->getTickerCount(BLOCK_CACHE_INDEX_ADD)); + EXPECT_EQ(1 + kNumBlocks, + options.statistics->getTickerCount(BLOCK_CACHE_FILTER_ADD)); + } +} + +TEST_F(DBBlockCacheTest, DynamicallyWarmCacheDuringFlush) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + + BlockBasedTableOptions table_options; + table_options.block_cache = NewLRUCache(1 << 25, 0, false); + table_options.cache_index_and_filter_blocks = false; + table_options.prepopulate_block_cache = + BlockBasedTableOptions::PrepopulateBlockCache::kFlushOnly; + + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DestroyAndReopen(options); + + std::string value(kValueSize, 'a'); + + for (size_t i = 1; i <= 5; i++) { + ASSERT_OK(Put(std::to_string(i), value)); + ASSERT_OK(Flush()); + ASSERT_EQ(1, + options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD)); + + ASSERT_EQ(value, Get(std::to_string(i))); + ASSERT_EQ(0, + options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD)); + ASSERT_EQ( + 0, options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_MISS)); + ASSERT_EQ(1, + options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_HIT)); + } + + ASSERT_OK(dbfull()->SetOptions( + {{"block_based_table_factory", "{prepopulate_block_cache=kDisable;}"}})); + + for (size_t i = 6; i <= kNumBlocks; i++) { + ASSERT_OK(Put(std::to_string(i), value)); + ASSERT_OK(Flush()); + ASSERT_EQ(0, + options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD)); + + ASSERT_EQ(value, Get(std::to_string(i))); + ASSERT_EQ(1, + options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD)); + ASSERT_EQ( + 1, options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_MISS)); + ASSERT_EQ(0, + options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_HIT)); + } +} +#endif + +namespace { + +// A mock cache wraps LRUCache, and record how many entries have been +// inserted for each priority. +class MockCache : public LRUCache { + public: + static uint32_t high_pri_insert_count; + static uint32_t low_pri_insert_count; + + MockCache() + : LRUCache(LRUCacheOptions( + size_t{1} << 25 /*capacity*/, 0 /*num_shard_bits*/, + false /*strict_capacity_limit*/, 0.0 /*high_pri_pool_ratio*/)) {} + + using ShardedCache::Insert; + + Status Insert(const Slice& key, Cache::ObjectPtr value, + const Cache::CacheItemHelper* helper, size_t charge, + Handle** handle, Priority priority) override { + if (priority == Priority::LOW) { + low_pri_insert_count++; + } else { + high_pri_insert_count++; + } + return LRUCache::Insert(key, value, helper, charge, handle, priority); + } +}; + +uint32_t MockCache::high_pri_insert_count = 0; +uint32_t MockCache::low_pri_insert_count = 0; + +} // anonymous namespace + +TEST_F(DBBlockCacheTest, IndexAndFilterBlocksCachePriority) { + for (auto priority : {Cache::Priority::LOW, Cache::Priority::HIGH}) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + BlockBasedTableOptions table_options; + table_options.cache_index_and_filter_blocks = true; + table_options.block_cache.reset(new MockCache()); + table_options.filter_policy.reset(NewBloomFilterPolicy(20)); + table_options.cache_index_and_filter_blocks_with_high_priority = + priority == Cache::Priority::HIGH ? true : false; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DestroyAndReopen(options); + + MockCache::high_pri_insert_count = 0; + MockCache::low_pri_insert_count = 0; + + // Create a new table. + ASSERT_OK(Put("foo", "value")); + ASSERT_OK(Put("bar", "value")); + ASSERT_OK(Flush()); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + + // index/filter blocks added to block cache right after table creation. + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(2, /* only index/filter were added */ + TestGetTickerCount(options, BLOCK_CACHE_ADD)); + ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_MISS)); + if (priority == Cache::Priority::LOW) { + ASSERT_EQ(0u, MockCache::high_pri_insert_count); + ASSERT_EQ(2u, MockCache::low_pri_insert_count); + } else { + ASSERT_EQ(2u, MockCache::high_pri_insert_count); + ASSERT_EQ(0u, MockCache::low_pri_insert_count); + } + + // Access data block. + ASSERT_EQ("value", Get("foo")); + + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(3, /*adding data block*/ + TestGetTickerCount(options, BLOCK_CACHE_ADD)); + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_MISS)); + + // Data block should be inserted with low priority. + if (priority == Cache::Priority::LOW) { + ASSERT_EQ(0u, MockCache::high_pri_insert_count); + ASSERT_EQ(3u, MockCache::low_pri_insert_count); + } else { + ASSERT_EQ(2u, MockCache::high_pri_insert_count); + ASSERT_EQ(1u, MockCache::low_pri_insert_count); + } + } +} + +namespace { + +// An LRUCache wrapper that can falsely report "not found" on Lookup. +// This allows us to manipulate BlockBasedTableReader into thinking +// another thread inserted the data in between Lookup and Insert, +// while mostly preserving the LRUCache interface/behavior. +class LookupLiarCache : public CacheWrapper { + int nth_lookup_not_found_ = 0; + + public: + explicit LookupLiarCache(std::shared_ptr target) + : CacheWrapper(std::move(target)) {} + + const char* Name() const override { return "LookupLiarCache"; } + + Handle* Lookup(const Slice& key, const CacheItemHelper* helper = nullptr, + CreateContext* create_context = nullptr, + Priority priority = Priority::LOW, + Statistics* stats = nullptr) override { + if (nth_lookup_not_found_ == 1) { + nth_lookup_not_found_ = 0; + return nullptr; + } + if (nth_lookup_not_found_ > 1) { + --nth_lookup_not_found_; + } + return CacheWrapper::Lookup(key, helper, create_context, priority, stats); + } + + // 1 == next lookup, 2 == after next, etc. + void SetNthLookupNotFound(int n) { nth_lookup_not_found_ = n; } +}; + +} // anonymous namespace + +TEST_F(DBBlockCacheTest, AddRedundantStats) { + const size_t capacity = size_t{1} << 25; + const int num_shard_bits = 0; // 1 shard + int iterations_tested = 0; + for (std::shared_ptr base_cache : + {NewLRUCache(capacity, num_shard_bits), + HyperClockCacheOptions( + capacity, + BlockBasedTableOptions().block_size /*estimated_value_size*/, + num_shard_bits) + .MakeSharedCache()}) { + if (!base_cache) { + // Skip clock cache when not supported + continue; + } + ++iterations_tested; + Options options = CurrentOptions(); + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + + std::shared_ptr cache = + std::make_shared(base_cache); + + BlockBasedTableOptions table_options; + table_options.cache_index_and_filter_blocks = true; + table_options.block_cache = cache; + table_options.filter_policy.reset(NewBloomFilterPolicy(50)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DestroyAndReopen(options); + + // Create a new table. + ASSERT_OK(Put("foo", "value")); + ASSERT_OK(Put("bar", "value")); + ASSERT_OK(Flush()); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + + // Normal access filter+index+data. + ASSERT_EQ("value", Get("foo")); + + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD)); + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD)); + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD)); + // -------- + ASSERT_EQ(3, TestGetTickerCount(options, BLOCK_CACHE_ADD)); + + ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT)); + ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT)); + ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT)); + // -------- + ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT)); + + // Againt access filter+index+data, but force redundant load+insert on index + cache->SetNthLookupNotFound(2); + ASSERT_EQ("value", Get("bar")); + + ASSERT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD)); + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD)); + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD)); + // -------- + ASSERT_EQ(4, TestGetTickerCount(options, BLOCK_CACHE_ADD)); + + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT)); + ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT)); + ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT)); + // -------- + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT)); + + // Access just filter (with high probability), and force redundant + // load+insert + cache->SetNthLookupNotFound(1); + ASSERT_EQ("NOT_FOUND", Get("this key was not added")); + + EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD)); + EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD)); + EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD)); + // -------- + EXPECT_EQ(5, TestGetTickerCount(options, BLOCK_CACHE_ADD)); + + EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT)); + EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT)); + EXPECT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT)); + // -------- + EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT)); + + // Access just data, forcing redundant load+insert + ReadOptions read_options; + std::unique_ptr iter{db_->NewIterator(read_options)}; + cache->SetNthLookupNotFound(1); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), "bar"); + + EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD)); + EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD)); + EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD)); + // -------- + EXPECT_EQ(6, TestGetTickerCount(options, BLOCK_CACHE_ADD)); + + EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT)); + EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT)); + EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT)); + // -------- + EXPECT_EQ(3, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT)); + } + EXPECT_GE(iterations_tested, 1); +} + +TEST_F(DBBlockCacheTest, ParanoidFileChecks) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.level0_file_num_compaction_trigger = 2; + options.paranoid_file_checks = true; + BlockBasedTableOptions table_options; + table_options.cache_index_and_filter_blocks = false; + table_options.filter_policy.reset(NewBloomFilterPolicy(20)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(1, "1_key", "val")); + ASSERT_OK(Put(1, "9_key", "val")); + // Create a new table. + ASSERT_OK(Flush(1)); + ASSERT_EQ(1, /* read and cache data block */ + TestGetTickerCount(options, BLOCK_CACHE_ADD)); + + ASSERT_OK(Put(1, "1_key2", "val2")); + ASSERT_OK(Put(1, "9_key2", "val2")); + // Create a new SST file. This will further trigger a compaction + // and generate another file. + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(3, /* Totally 3 files created up to now */ + TestGetTickerCount(options, BLOCK_CACHE_ADD)); + + // After disabling options.paranoid_file_checks. NO further block + // is added after generating a new file. + ASSERT_OK( + dbfull()->SetOptions(handles_[1], {{"paranoid_file_checks", "false"}})); + + ASSERT_OK(Put(1, "1_key3", "val3")); + ASSERT_OK(Put(1, "9_key3", "val3")); + ASSERT_OK(Flush(1)); + ASSERT_OK(Put(1, "1_key4", "val4")); + ASSERT_OK(Put(1, "9_key4", "val4")); + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(3, /* Totally 3 files created up to now */ + TestGetTickerCount(options, BLOCK_CACHE_ADD)); +} + +TEST_F(DBBlockCacheTest, CacheCompressionDict) { + const int kNumFiles = 4; + const int kNumEntriesPerFile = 128; + const int kNumBytesPerEntry = 1024; + + // Try all the available libraries that support dictionary compression + std::vector compression_types; + if (Zlib_Supported()) { + compression_types.push_back(kZlibCompression); + } + if (LZ4_Supported()) { + compression_types.push_back(kLZ4Compression); + compression_types.push_back(kLZ4HCCompression); + } + if (ZSTD_Supported()) { + compression_types.push_back(kZSTD); + } else if (ZSTDNotFinal_Supported()) { + compression_types.push_back(kZSTDNotFinalCompression); + } + Random rnd(301); + for (auto compression_type : compression_types) { + Options options = CurrentOptions(); + options.bottommost_compression = compression_type; + options.bottommost_compression_opts.max_dict_bytes = 4096; + options.bottommost_compression_opts.enabled = true; + options.create_if_missing = true; + options.num_levels = 2; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.target_file_size_base = kNumEntriesPerFile * kNumBytesPerEntry; + BlockBasedTableOptions table_options; + table_options.cache_index_and_filter_blocks = true; + table_options.block_cache.reset(new MockCache()); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DestroyAndReopen(options); + + RecordCacheCountersForCompressionDict(options); + + for (int i = 0; i < kNumFiles; ++i) { + ASSERT_EQ(i, NumTableFilesAtLevel(0, 0)); + for (int j = 0; j < kNumEntriesPerFile; ++j) { + std::string value = rnd.RandomString(kNumBytesPerEntry); + ASSERT_OK(Put(Key(j * kNumFiles + i), value.c_str())); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_EQ(kNumFiles, NumTableFilesAtLevel(1)); + + // Compression dictionary blocks are preloaded. + CheckCacheCountersForCompressionDict( + options, kNumFiles /* expected_compression_dict_misses */, + 0 /* expected_compression_dict_hits */, + kNumFiles /* expected_compression_dict_inserts */); + + // Seek to a key in a file. It should cause the SST's dictionary meta-block + // to be read. + RecordCacheCounters(options); + RecordCacheCountersForCompressionDict(options); + ReadOptions read_options; + ASSERT_NE("NOT_FOUND", Get(Key(kNumFiles * kNumEntriesPerFile - 1))); + // Two block hits: index and dictionary since they are prefetched + // One block missed/added: data block + CheckCacheCounters(options, 1 /* expected_misses */, 2 /* expected_hits */, + 1 /* expected_inserts */, 0 /* expected_failures */); + CheckCacheCountersForCompressionDict( + options, 0 /* expected_compression_dict_misses */, + 1 /* expected_compression_dict_hits */, + 0 /* expected_compression_dict_inserts */); + } +} + +static void ClearCache(Cache* cache) { + std::deque keys; + Cache::ApplyToAllEntriesOptions opts; + auto callback = [&](const Slice& key, Cache::ObjectPtr, size_t /*charge*/, + const Cache::CacheItemHelper* helper) { + if (helper && helper->role == CacheEntryRole::kMisc) { + // Keep the stats collector + return; + } + keys.push_back(key.ToString()); + }; + cache->ApplyToAllEntries(callback, opts); + for (auto& k : keys) { + cache->Erase(k); + } +} + +TEST_F(DBBlockCacheTest, CacheEntryRoleStats) { + const size_t capacity = size_t{1} << 25; + int iterations_tested = 0; + for (bool partition : {false, true}) { + SCOPED_TRACE("Partition? " + std::to_string(partition)); + for (std::shared_ptr cache : + {NewLRUCache(capacity), + HyperClockCacheOptions( + capacity, + BlockBasedTableOptions().block_size /*estimated_value_size*/) + .MakeSharedCache()}) { + SCOPED_TRACE(std::string("Cache: ") + cache->Name()); + ++iterations_tested; + + Options options = CurrentOptions(); + SetTimeElapseOnlySleepOnReopen(&options); + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.max_open_files = 13; + options.table_cache_numshardbits = 0; + // If this wakes up, it could interfere with test + options.stats_dump_period_sec = 0; + + BlockBasedTableOptions table_options; + table_options.block_cache = cache; + table_options.cache_index_and_filter_blocks = true; + table_options.filter_policy.reset(NewBloomFilterPolicy(50)); + if (partition) { + table_options.index_type = BlockBasedTableOptions::kTwoLevelIndexSearch; + table_options.partition_filters = true; + } + table_options.metadata_cache_options.top_level_index_pinning = + PinningTier::kNone; + table_options.metadata_cache_options.partition_pinning = + PinningTier::kNone; + table_options.metadata_cache_options.unpartitioned_pinning = + PinningTier::kNone; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DestroyAndReopen(options); + + // Create a new table. + ASSERT_OK(Put("foo", "value")); + ASSERT_OK(Put("bar", "value")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("zfoo", "value")); + ASSERT_OK(Put("zbar", "value")); + ASSERT_OK(Flush()); + + ASSERT_EQ(2, NumTableFilesAtLevel(0)); + + // Fresh cache + ClearCache(cache.get()); + + std::array expected{}; + // For CacheEntryStatsCollector + expected[static_cast(CacheEntryRole::kMisc)] = 1; + EXPECT_EQ(expected, GetCacheEntryRoleCountsBg()); + + std::array prev_expected = expected; + + // First access only filters + ASSERT_EQ("NOT_FOUND", Get("different from any key added")); + expected[static_cast(CacheEntryRole::kFilterBlock)] += 2; + if (partition) { + expected[static_cast(CacheEntryRole::kFilterMetaBlock)] += 2; + } + // Within some time window, we will get cached entry stats + EXPECT_EQ(prev_expected, GetCacheEntryRoleCountsBg()); + // Not enough to force a miss + env_->MockSleepForSeconds(45); + EXPECT_EQ(prev_expected, GetCacheEntryRoleCountsBg()); + // Enough to force a miss + env_->MockSleepForSeconds(601); + EXPECT_EQ(expected, GetCacheEntryRoleCountsBg()); + + // Now access index and data block + ASSERT_EQ("value", Get("foo")); + expected[static_cast(CacheEntryRole::kIndexBlock)]++; + if (partition) { + // top-level + expected[static_cast(CacheEntryRole::kIndexBlock)]++; + } + expected[static_cast(CacheEntryRole::kDataBlock)]++; + // Enough to force a miss + env_->MockSleepForSeconds(601); + // But inject a simulated long scan so that we need a longer + // interval to force a miss next time. + SyncPoint::GetInstance()->SetCallBack( + "CacheEntryStatsCollector::GetStats:AfterApplyToAllEntries", + [this](void*) { + // To spend no more than 0.2% of time scanning, we would need + // interval of at least 10000s + env_->MockSleepForSeconds(20); + }); + SyncPoint::GetInstance()->EnableProcessing(); + EXPECT_EQ(expected, GetCacheEntryRoleCountsBg()); + prev_expected = expected; + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + // The same for other file + ASSERT_EQ("value", Get("zfoo")); + expected[static_cast(CacheEntryRole::kIndexBlock)]++; + if (partition) { + // top-level + expected[static_cast(CacheEntryRole::kIndexBlock)]++; + } + expected[static_cast(CacheEntryRole::kDataBlock)]++; + // Because of the simulated long scan, this is not enough to force + // a miss + env_->MockSleepForSeconds(601); + EXPECT_EQ(prev_expected, GetCacheEntryRoleCountsBg()); + // But this is enough + env_->MockSleepForSeconds(10000); + EXPECT_EQ(expected, GetCacheEntryRoleCountsBg()); + prev_expected = expected; + + // Also check the GetProperty interface + std::map values; + ASSERT_TRUE( + db_->GetMapProperty(DB::Properties::kBlockCacheEntryStats, &values)); + + for (size_t i = 0; i < kNumCacheEntryRoles; ++i) { + auto role = static_cast(i); + EXPECT_EQ(std::to_string(expected[i]), + values[BlockCacheEntryStatsMapKeys::EntryCount(role)]); + } + + // Add one for kWriteBuffer + { + WriteBufferManager wbm(size_t{1} << 20, cache); + wbm.ReserveMem(1024); + expected[static_cast(CacheEntryRole::kWriteBuffer)]++; + // Now we check that the GetProperty interface is more agressive about + // re-scanning stats, but not totally aggressive. + // Within some time window, we will get cached entry stats + env_->MockSleepForSeconds(1); + EXPECT_EQ(std::to_string(prev_expected[static_cast( + CacheEntryRole::kWriteBuffer)]), + values[BlockCacheEntryStatsMapKeys::EntryCount( + CacheEntryRole::kWriteBuffer)]); + // Not enough for a "background" miss but enough for a "foreground" miss + env_->MockSleepForSeconds(45); + + ASSERT_TRUE(db_->GetMapProperty(DB::Properties::kBlockCacheEntryStats, + &values)); + EXPECT_EQ( + std::to_string( + expected[static_cast(CacheEntryRole::kWriteBuffer)]), + values[BlockCacheEntryStatsMapKeys::EntryCount( + CacheEntryRole::kWriteBuffer)]); + } + prev_expected = expected; + + // With collector pinned in cache, we should be able to hit + // even if the cache is full + ClearCache(cache.get()); + Cache::Handle* h = nullptr; + if (strcmp(cache->Name(), "LRUCache") == 0) { + ASSERT_OK(cache->Insert("Fill-it-up", nullptr, &kNoopCacheItemHelper, + capacity + 1, &h, Cache::Priority::HIGH)); + } else { + // For ClockCache we use a 16-byte key. + ASSERT_OK(cache->Insert("Fill-it-up-xxxxx", nullptr, + &kNoopCacheItemHelper, capacity + 1, &h, + Cache::Priority::HIGH)); + } + ASSERT_GT(cache->GetUsage(), cache->GetCapacity()); + expected = {}; + // For CacheEntryStatsCollector + expected[static_cast(CacheEntryRole::kMisc)] = 1; + // For Fill-it-up + expected[static_cast(CacheEntryRole::kMisc)]++; + // Still able to hit on saved stats + EXPECT_EQ(prev_expected, GetCacheEntryRoleCountsBg()); + // Enough to force a miss + env_->MockSleepForSeconds(1000); + EXPECT_EQ(expected, GetCacheEntryRoleCountsBg()); + + cache->Release(h); + + // Now we test that the DB mutex is not held during scans, for the ways + // we know how to (possibly) trigger them. Without a better good way to + // check this, we simply inject an acquire & release of the DB mutex + // deep in the stat collection code. If we were already holding the + // mutex, that is UB that would at least be found by TSAN. + int scan_count = 0; + SyncPoint::GetInstance()->SetCallBack( + "CacheEntryStatsCollector::GetStats:AfterApplyToAllEntries", + [this, &scan_count](void*) { + dbfull()->TEST_LockMutex(); + dbfull()->TEST_UnlockMutex(); + ++scan_count; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + // Different things that might trigger a scan, with mock sleeps to + // force a miss. + env_->MockSleepForSeconds(10000); + dbfull()->DumpStats(); + ASSERT_EQ(scan_count, 1); + + env_->MockSleepForSeconds(60); + ASSERT_TRUE(db_->GetMapProperty(DB::Properties::kFastBlockCacheEntryStats, + &values)); + ASSERT_EQ(scan_count, 1); + ASSERT_TRUE( + db_->GetMapProperty(DB::Properties::kBlockCacheEntryStats, &values)); + ASSERT_EQ(scan_count, 2); + + env_->MockSleepForSeconds(10000); + ASSERT_TRUE(db_->GetMapProperty(DB::Properties::kFastBlockCacheEntryStats, + &values)); + ASSERT_EQ(scan_count, 3); + + env_->MockSleepForSeconds(60); + std::string value_str; + ASSERT_TRUE(db_->GetProperty(DB::Properties::kFastBlockCacheEntryStats, + &value_str)); + ASSERT_EQ(scan_count, 3); + ASSERT_TRUE( + db_->GetProperty(DB::Properties::kBlockCacheEntryStats, &value_str)); + ASSERT_EQ(scan_count, 4); + + env_->MockSleepForSeconds(10000); + ASSERT_TRUE(db_->GetProperty(DB::Properties::kFastBlockCacheEntryStats, + &value_str)); + ASSERT_EQ(scan_count, 5); + + ASSERT_TRUE(db_->GetProperty(DB::Properties::kCFStats, &value_str)); + // To match historical speed, querying this property no longer triggers + // a scan, even if results are old. But periodic dump stats should keep + // things reasonably updated. + ASSERT_EQ(scan_count, /*unchanged*/ 5); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + } + EXPECT_GE(iterations_tested, 1); + } +} + +namespace { + +void DummyFillCache(Cache& cache, size_t entry_size, + std::vector>& handles) { + // fprintf(stderr, "Entry size: %zu\n", entry_size); + handles.clear(); + cache.EraseUnRefEntries(); + void* fake_value = &cache; + size_t capacity = cache.GetCapacity(); + OffsetableCacheKey ck{"abc", "abc", 42}; + for (size_t my_usage = 0; my_usage < capacity;) { + size_t charge = std::min(entry_size, capacity - my_usage); + Cache::Handle* handle; + Status st = cache.Insert(ck.WithOffset(my_usage).AsSlice(), fake_value, + &kNoopCacheItemHelper, charge, &handle); + ASSERT_OK(st); + handles.emplace_back(&cache, handle); + my_usage += charge; + } +} + +class CountingLogger : public Logger { + public: + ~CountingLogger() override {} + using Logger::Logv; + void Logv(const InfoLogLevel log_level, const char* format, + va_list /*ap*/) override { + if (std::strstr(format, "HyperClockCache") == nullptr) { + // Not a match + return; + } + // static StderrLogger debug; + // debug.Logv(log_level, format, ap); + if (log_level == InfoLogLevel::INFO_LEVEL) { + ++info_count_; + } else if (log_level == InfoLogLevel::WARN_LEVEL) { + ++warn_count_; + } else if (log_level == InfoLogLevel::ERROR_LEVEL) { + ++error_count_; + } + } + + std::array PopCounts() { + std::array rv{{info_count_, warn_count_, error_count_}}; + info_count_ = warn_count_ = error_count_ = 0; + return rv; + } + + private: + int info_count_{}; + int warn_count_{}; + int error_count_{}; +}; + +} // namespace + +TEST_F(DBBlockCacheTest, HyperClockCacheReportProblems) { + size_t capacity = 1024 * 1024; + size_t value_size_est = 8 * 1024; + HyperClockCacheOptions hcc_opts{capacity, value_size_est}; + hcc_opts.num_shard_bits = 2; // 4 shards + hcc_opts.metadata_charge_policy = kDontChargeCacheMetadata; + hcc_opts.hash_seed = 0; // deterministic hashing + std::shared_ptr cache = hcc_opts.MakeSharedCache(); + std::shared_ptr logger = std::make_shared(); + + auto table_options = GetTableOptions(); + auto options = GetOptions(table_options); + table_options.block_cache = cache; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.info_log = logger; + // Going to sample more directly + options.stats_dump_period_sec = 0; + Reopen(options); + + std::vector> handles; + + // Clear anything from DB startup + logger->PopCounts(); + + // Fill cache based on expected size and check that when we + // don't report anything relevant in periodic stats dump + DummyFillCache(*cache, value_size_est, handles); + dbfull()->DumpStats(); + EXPECT_EQ(logger->PopCounts(), (std::array{{0, 0, 0}})); + + // Same, within reasonable bounds + DummyFillCache(*cache, value_size_est - value_size_est / 4, handles); + dbfull()->DumpStats(); + EXPECT_EQ(logger->PopCounts(), (std::array{{0, 0, 0}})); + + DummyFillCache(*cache, value_size_est + value_size_est / 3, handles); + dbfull()->DumpStats(); + EXPECT_EQ(logger->PopCounts(), (std::array{{0, 0, 0}})); + + // Estimate too high (value size too low) eventually reports ERROR + DummyFillCache(*cache, value_size_est / 2, handles); + dbfull()->DumpStats(); + EXPECT_EQ(logger->PopCounts(), (std::array{{0, 1, 0}})); + + DummyFillCache(*cache, value_size_est / 3, handles); + dbfull()->DumpStats(); + EXPECT_EQ(logger->PopCounts(), (std::array{{0, 0, 1}})); + + // Estimate too low (value size too high) starts with INFO + // and is only WARNING in the worst case + DummyFillCache(*cache, value_size_est * 2, handles); + dbfull()->DumpStats(); + EXPECT_EQ(logger->PopCounts(), (std::array{{1, 0, 0}})); + + DummyFillCache(*cache, value_size_est * 3, handles); + dbfull()->DumpStats(); + EXPECT_EQ(logger->PopCounts(), (std::array{{0, 1, 0}})); + + DummyFillCache(*cache, value_size_est * 20, handles); + dbfull()->DumpStats(); + EXPECT_EQ(logger->PopCounts(), (std::array{{0, 1, 0}})); +} + + +class DBBlockCacheKeyTest + : public DBTestBase, + public testing::WithParamInterface> { + public: + DBBlockCacheKeyTest() + : DBTestBase("db_block_cache_test", /*env_do_fsync=*/false) {} + + void SetUp() override { + use_compressed_cache_ = std::get<0>(GetParam()); + exclude_file_numbers_ = std::get<1>(GetParam()); + } + + bool use_compressed_cache_; + bool exclude_file_numbers_; +}; + +// Disable LinkFile so that we can physically copy a DB using Checkpoint. +// Disable file GetUniqueId to enable stable cache keys. +class StableCacheKeyTestFS : public FaultInjectionTestFS { + public: + explicit StableCacheKeyTestFS(const std::shared_ptr& base) + : FaultInjectionTestFS(base) { + SetFailGetUniqueId(true); + } + + virtual ~StableCacheKeyTestFS() override {} + + IOStatus LinkFile(const std::string&, const std::string&, const IOOptions&, + IODebugContext*) override { + return IOStatus::NotSupported("Disabled"); + } +}; + +TEST_P(DBBlockCacheKeyTest, StableCacheKeys) { + std::shared_ptr test_fs{ + new StableCacheKeyTestFS(env_->GetFileSystem())}; + std::unique_ptr test_env{ + new CompositeEnvWrapper(env_, test_fs)}; + + Options options = CurrentOptions(); + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.env = test_env.get(); + + // Corrupting the table properties corrupts the unique id. + // Ignore the unique id recorded in the manifest. + options.verify_sst_unique_id_in_manifest = false; + + BlockBasedTableOptions table_options; + + int key_count = 0; + uint64_t expected_stat = 0; + + std::function verify_stats; + table_options.cache_index_and_filter_blocks = true; + table_options.block_cache = NewLRUCache(1 << 25, 0, false); + verify_stats = [&options, &expected_stat] { + ASSERT_EQ(expected_stat, + options.statistics->getTickerCount(BLOCK_CACHE_DATA_ADD)); + ASSERT_EQ(expected_stat, + options.statistics->getTickerCount(BLOCK_CACHE_INDEX_ADD)); + ASSERT_EQ(expected_stat, + options.statistics->getTickerCount(BLOCK_CACHE_FILTER_ADD)); + }; + + table_options.filter_policy.reset(NewBloomFilterPolicy(10, false)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + CreateAndReopenWithCF({"koko"}, options); + + if (exclude_file_numbers_) { + // Simulate something like old behavior without file numbers in properties. + // This is a "control" side of the test that also ensures safely degraded + // behavior on old files. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTableBuilder::BlockBasedTableBuilder:PreSetupBaseCacheKey", + [&](void* arg) { + TableProperties* props = reinterpret_cast(arg); + props->orig_file_number = 0; + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + } + + std::function perform_gets = [&key_count, &expected_stat, this]() { + if (exclude_file_numbers_) { + // No cache key reuse should happen, because we can't rely on current + // file number being stable + expected_stat += key_count; + } else { + // Cache keys should be stable + expected_stat = key_count; + } + for (int i = 0; i < key_count; ++i) { + ASSERT_EQ(Get(1, Key(i)), "abc"); + } + }; + + // Ordinary SST files with same session id + const std::string something_compressible(500U, 'x'); + for (int i = 0; i < 2; ++i) { + ASSERT_OK(Put(1, Key(key_count), "abc")); + ASSERT_OK(Put(1, Key(key_count) + "a", something_compressible)); + ASSERT_OK(Flush(1)); + ++key_count; + } + + // Save an export of those ordinary SST files for later + std::string export_files_dir = dbname_ + "/exported"; + ExportImportFilesMetaData* metadata_ptr_ = nullptr; + Checkpoint* checkpoint; + ASSERT_OK(Checkpoint::Create(db_, &checkpoint)); + ASSERT_OK(checkpoint->ExportColumnFamily(handles_[1], export_files_dir, + &metadata_ptr_)); + ASSERT_NE(metadata_ptr_, nullptr); + delete checkpoint; + checkpoint = nullptr; + + // External SST files with same session id + SstFileWriter sst_file_writer(EnvOptions(), options); + std::vector external; + for (int i = 0; i < 2; ++i) { + std::string f = dbname_ + "/external" + std::to_string(i) + ".sst"; + external.push_back(f); + ASSERT_OK(sst_file_writer.Open(f)); + ASSERT_OK(sst_file_writer.Put(Key(key_count), "abc")); + ASSERT_OK( + sst_file_writer.Put(Key(key_count) + "a", something_compressible)); + ++key_count; + ExternalSstFileInfo external_info; + ASSERT_OK(sst_file_writer.Finish(&external_info)); + IngestExternalFileOptions ingest_opts; + ASSERT_OK(db_->IngestExternalFile(handles_[1], {f}, ingest_opts)); + } + + perform_gets(); + verify_stats(); + + // Make sure we can cache hit after re-open + ReopenWithColumnFamilies({"default", "koko"}, options); + + perform_gets(); + verify_stats(); + + // Make sure we can cache hit even on a full copy of the DB. Using + // StableCacheKeyTestFS, Checkpoint will resort to full copy not hard link. + // (Checkpoint not available in LITE mode to test this.) + auto db_copy_name = dbname_ + "-copy"; + ASSERT_OK(Checkpoint::Create(db_, &checkpoint)); + ASSERT_OK(checkpoint->CreateCheckpoint(db_copy_name)); + delete checkpoint; + + Close(); + Destroy(options); + + // Switch to the DB copy + SaveAndRestore save_dbname(&dbname_, db_copy_name); + ReopenWithColumnFamilies({"default", "koko"}, options); + + perform_gets(); + verify_stats(); + + // And ensure that re-importing + ingesting the same files into a + // different DB uses same cache keys + DestroyAndReopen(options); + + ColumnFamilyHandle* cfh = nullptr; + ASSERT_OK(db_->CreateColumnFamilyWithImport(ColumnFamilyOptions(), "yoyo", + ImportColumnFamilyOptions(), + *metadata_ptr_, &cfh)); + ASSERT_NE(cfh, nullptr); + delete cfh; + cfh = nullptr; + delete metadata_ptr_; + metadata_ptr_ = nullptr; + + ASSERT_OK(DestroyDB(export_files_dir, options)); + + ReopenWithColumnFamilies({"default", "yoyo"}, options); + + IngestExternalFileOptions ingest_opts; + ASSERT_OK(db_->IngestExternalFile(handles_[1], {external}, ingest_opts)); + + perform_gets(); + verify_stats(); + + Close(); + Destroy(options); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +class CacheKeyTest : public testing::Test { + public: + CacheKey GetBaseCacheKey() { + CacheKey rv = GetOffsetableCacheKey(0, /*min file_number*/ 1).WithOffset(0); + // Correct for file_number_ == 1 + *reinterpret_cast(&rv) ^= ReverseBits(uint64_t{1}); + return rv; + } + CacheKey GetCacheKey(uint64_t session_counter, uint64_t file_number, + uint64_t offset) { + OffsetableCacheKey offsetable = + GetOffsetableCacheKey(session_counter, file_number); + // * 4 to counteract optimization that strips lower 2 bits in encoding + // the offset in BlockBasedTable::GetCacheKey (which we prefer to include + // in unit tests to maximize functional coverage). + EXPECT_GE(offset * 4, offset); // no overflow + return BlockBasedTable::GetCacheKey(offsetable, + BlockHandle(offset * 4, /*size*/ 5)); + } + + protected: + OffsetableCacheKey GetOffsetableCacheKey(uint64_t session_counter, + uint64_t file_number) { + // Like SemiStructuredUniqueIdGen::GenerateNext + tp_.db_session_id = EncodeSessionId(base_session_upper_, + base_session_lower_ ^ session_counter); + tp_.db_id = std::to_string(db_id_); + tp_.orig_file_number = file_number; + bool is_stable; + std::string cur_session_id = ""; // ignored + uint64_t cur_file_number = 42; // ignored + OffsetableCacheKey rv; + BlockBasedTable::SetupBaseCacheKey(&tp_, cur_session_id, cur_file_number, + &rv, &is_stable); + EXPECT_TRUE(is_stable); + EXPECT_TRUE(!rv.IsEmpty()); + // BEGIN some assertions in relation to SST unique IDs + std::string external_unique_id_str; + EXPECT_OK(GetUniqueIdFromTableProperties(tp_, &external_unique_id_str)); + UniqueId64x2 sst_unique_id = {}; + EXPECT_OK(DecodeUniqueIdBytes(external_unique_id_str, &sst_unique_id)); + ExternalUniqueIdToInternal(&sst_unique_id); + OffsetableCacheKey ock = + OffsetableCacheKey::FromInternalUniqueId(&sst_unique_id); + EXPECT_EQ(rv.WithOffset(0).AsSlice(), ock.WithOffset(0).AsSlice()); + EXPECT_EQ(ock.ToInternalUniqueId(), sst_unique_id); + // END some assertions in relation to SST unique IDs + return rv; + } + + TableProperties tp_; + uint64_t base_session_upper_ = 0; + uint64_t base_session_lower_ = 0; + uint64_t db_id_ = 0; +}; + +TEST_F(CacheKeyTest, DBImplSessionIdStructure) { + // We have to generate our own session IDs for simulation purposes in other + // tests. Here we verify that the DBImpl implementation seems to match + // our construction here, by using lowest XORed-in bits for "session + // counter." + std::string session_id1 = DBImpl::GenerateDbSessionId(/*env*/ nullptr); + std::string session_id2 = DBImpl::GenerateDbSessionId(/*env*/ nullptr); + uint64_t upper1, upper2, lower1, lower2; + ASSERT_OK(DecodeSessionId(session_id1, &upper1, &lower1)); + ASSERT_OK(DecodeSessionId(session_id2, &upper2, &lower2)); + // Because generated in same process + ASSERT_EQ(upper1, upper2); + // Unless we generate > 4 billion session IDs in this process... + ASSERT_EQ(Upper32of64(lower1), Upper32of64(lower2)); + // But they must be different somewhere + ASSERT_NE(Lower32of64(lower1), Lower32of64(lower2)); +} + +namespace { +// Deconstruct cache key, based on knowledge of implementation details. +void DeconstructNonemptyCacheKey(const CacheKey& key, uint64_t* file_num_etc64, + uint64_t* offset_etc64) { + *file_num_etc64 = *reinterpret_cast(key.AsSlice().data()); + *offset_etc64 = *reinterpret_cast(key.AsSlice().data() + 8); + assert(*file_num_etc64 != 0); + if (*offset_etc64 == 0) { + std::swap(*file_num_etc64, *offset_etc64); + } + assert(*offset_etc64 != 0); +} + +// Make a bit mask of 0 to 64 bits +uint64_t MakeMask64(int bits) { + if (bits >= 64) { + return uint64_t{0} - 1; + } else { + return (uint64_t{1} << bits) - 1; + } +} + +// See CacheKeyTest::Encodings +struct CacheKeyDecoder { + // Inputs + uint64_t base_file_num_etc64, base_offset_etc64; + int session_counter_bits, file_number_bits, offset_bits; + + // Derived + uint64_t session_counter_mask, file_number_mask, offset_mask; + + // Outputs + uint64_t decoded_session_counter, decoded_file_num, decoded_offset; + + void SetBaseCacheKey(const CacheKey& base) { + DeconstructNonemptyCacheKey(base, &base_file_num_etc64, &base_offset_etc64); + } + + void SetRanges(int _session_counter_bits, int _file_number_bits, + int _offset_bits) { + session_counter_bits = _session_counter_bits; + session_counter_mask = MakeMask64(session_counter_bits); + file_number_bits = _file_number_bits; + file_number_mask = MakeMask64(file_number_bits); + offset_bits = _offset_bits; + offset_mask = MakeMask64(offset_bits); + } + + void Decode(const CacheKey& key) { + uint64_t file_num_etc64, offset_etc64; + DeconstructNonemptyCacheKey(key, &file_num_etc64, &offset_etc64); + + // First decode session counter + if (offset_bits + session_counter_bits <= 64) { + // fully recoverable from offset_etc64 + decoded_session_counter = + ReverseBits((offset_etc64 ^ base_offset_etc64)) & + session_counter_mask; + } else if (file_number_bits + session_counter_bits <= 64) { + // fully recoverable from file_num_etc64 + decoded_session_counter = DownwardInvolution( + (file_num_etc64 ^ base_file_num_etc64) & session_counter_mask); + } else { + // Need to combine parts from each word. + // Piece1 will contain some correct prefix of the bottom bits of + // session counter. + uint64_t piece1 = + ReverseBits((offset_etc64 ^ base_offset_etc64) & ~offset_mask); + int piece1_bits = 64 - offset_bits; + // Piece2 will contain involuded bits that we can combine with piece1 + // to infer rest of session counter + int piece2_bits = std::min(64 - file_number_bits, 64 - piece1_bits); + ASSERT_LT(piece2_bits, 64); + uint64_t piece2_mask = MakeMask64(piece2_bits); + uint64_t piece2 = (file_num_etc64 ^ base_file_num_etc64) & piece2_mask; + + // Cancel out the part of piece2 that we can infer from piece1 + // (DownwardInvolution distributes over xor) + piece2 ^= DownwardInvolution(piece1) & piece2_mask; + + // Now we need to solve for the unknown original bits in higher + // positions than piece1 provides. We use Gaussian elimination + // because we know that a piece2_bits X piece2_bits submatrix of + // the matrix underlying DownwardInvolution times the vector of + // unknown original bits equals piece2. + // + // Build an augmented row matrix for that submatrix, built column by + // column. + std::array aug_rows{}; + for (int i = 0; i < piece2_bits; ++i) { // over columns + uint64_t col_i = DownwardInvolution(uint64_t{1} << piece1_bits << i); + ASSERT_NE(col_i & 1U, 0); + for (int j = 0; j < piece2_bits; ++j) { // over rows + aug_rows[j] |= (col_i & 1U) << i; + col_i >>= 1; + } + } + // Augment with right hand side + for (int j = 0; j < piece2_bits; ++j) { // over rows + aug_rows[j] |= (piece2 & 1U) << piece2_bits; + piece2 >>= 1; + } + // Run Gaussian elimination + for (int i = 0; i < piece2_bits; ++i) { // over columns + // Find a row that can be used to cancel others + uint64_t canceller = 0; + // Note: Rows 0 through i-1 contain 1s in columns already eliminated + for (int j = i; j < piece2_bits; ++j) { // over rows + if (aug_rows[j] & (uint64_t{1} << i)) { + // Swap into appropriate row + std::swap(aug_rows[i], aug_rows[j]); + // Keep a handy copy for row reductions + canceller = aug_rows[i]; + break; + } + } + ASSERT_NE(canceller, 0); + for (int j = 0; j < piece2_bits; ++j) { // over rows + if (i != j && ((aug_rows[j] >> i) & 1) != 0) { + // Row reduction + aug_rows[j] ^= canceller; + } + } + } + // Extract result + decoded_session_counter = piece1; + for (int j = 0; j < piece2_bits; ++j) { // over rows + ASSERT_EQ(aug_rows[j] & piece2_mask, uint64_t{1} << j); + decoded_session_counter |= aug_rows[j] >> piece2_bits << piece1_bits + << j; + } + } + + decoded_offset = + offset_etc64 ^ base_offset_etc64 ^ ReverseBits(decoded_session_counter); + + decoded_file_num = ReverseBits(file_num_etc64 ^ base_file_num_etc64 ^ + DownwardInvolution(decoded_session_counter)); + } +}; +} // anonymous namespace + +TEST_F(CacheKeyTest, Encodings) { + // This test primarily verifies this claim from cache_key.cc: + // // In fact, if DB ids were not involved, we would be guaranteed unique + // // cache keys for files generated in a single process until total bits for + // // biggest session_id_counter, orig_file_number, and offset_in_file + // // reach 128 bits. + // + // To demonstrate this, CacheKeyDecoder can reconstruct the structured inputs + // to the cache key when provided an output cache key, the unstructured + // inputs, and bounds on the structured inputs. + // + // See OffsetableCacheKey comments in cache_key.cc. + + // We are going to randomly initialize some values that *should* not affect + // result + Random64 r{std::random_device{}()}; + + CacheKeyDecoder decoder; + db_id_ = r.Next(); + base_session_upper_ = r.Next(); + base_session_lower_ = r.Next(); + if (base_session_lower_ == 0) { + base_session_lower_ = 1; + } + + decoder.SetBaseCacheKey(GetBaseCacheKey()); + + // Loop over configurations and test those + for (int session_counter_bits = 0; session_counter_bits <= 64; + ++session_counter_bits) { + for (int file_number_bits = 1; file_number_bits <= 64; ++file_number_bits) { + // 62 bits max because unoptimized offset will be 64 bits in that case + for (int offset_bits = 0; offset_bits <= 62; ++offset_bits) { + if (session_counter_bits + file_number_bits + offset_bits > 128) { + break; + } + + decoder.SetRanges(session_counter_bits, file_number_bits, offset_bits); + + uint64_t session_counter = r.Next() & decoder.session_counter_mask; + uint64_t file_number = r.Next() & decoder.file_number_mask; + if (file_number == 0) { + // Minimum + file_number = 1; + } + uint64_t offset = r.Next() & decoder.offset_mask; + decoder.Decode(GetCacheKey(session_counter, file_number, offset)); + + EXPECT_EQ(decoder.decoded_session_counter, session_counter); + EXPECT_EQ(decoder.decoded_file_num, file_number); + EXPECT_EQ(decoder.decoded_offset, offset); + } + } + } +} + +INSTANTIATE_TEST_CASE_P(DBBlockCacheKeyTest, DBBlockCacheKeyTest, + ::testing::Combine(::testing::Bool(), + ::testing::Bool())); + +class DBBlockCachePinningTest + : public DBTestBase, + public testing::WithParamInterface< + std::tuple> { + public: + DBBlockCachePinningTest() + : DBTestBase("db_block_cache_test", /*env_do_fsync=*/false) {} + + void SetUp() override { + partition_index_and_filters_ = std::get<0>(GetParam()); + top_level_index_pinning_ = std::get<1>(GetParam()); + partition_pinning_ = std::get<2>(GetParam()); + unpartitioned_pinning_ = std::get<3>(GetParam()); + } + + bool partition_index_and_filters_; + PinningTier top_level_index_pinning_; + PinningTier partition_pinning_; + PinningTier unpartitioned_pinning_; +}; + +TEST_P(DBBlockCachePinningTest, TwoLevelDB) { + // Creates one file in L0 and one file in L1. Both files have enough data that + // their index and filter blocks are partitioned. The L1 file will also have + // a compression dictionary (those are trained only during compaction), which + // must be unpartitioned. + const int kKeySize = 32; + const int kBlockSize = 128; + const int kNumBlocksPerFile = 128; + const int kNumKeysPerFile = kBlockSize * kNumBlocksPerFile / kKeySize; + + Options options = CurrentOptions(); + // `kNoCompression` makes the unit test more portable. But it relies on the + // current behavior of persisting/accessing dictionary even when there's no + // (de)compression happening, which seems fairly likely to change over time. + options.compression = kNoCompression; + options.compression_opts.max_dict_bytes = 4 << 10; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + BlockBasedTableOptions table_options; + table_options.block_cache = NewLRUCache(1 << 20 /* capacity */); + table_options.block_size = kBlockSize; + table_options.metadata_block_size = kBlockSize; + table_options.cache_index_and_filter_blocks = true; + table_options.metadata_cache_options.top_level_index_pinning = + top_level_index_pinning_; + table_options.metadata_cache_options.partition_pinning = partition_pinning_; + table_options.metadata_cache_options.unpartitioned_pinning = + unpartitioned_pinning_; + table_options.filter_policy.reset( + NewBloomFilterPolicy(10 /* bits_per_key */)); + if (partition_index_and_filters_) { + table_options.index_type = + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + table_options.partition_filters = true; + } + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + + Random rnd(301); + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK(Put(Key(i * kNumKeysPerFile + j), rnd.RandomString(kKeySize))); + } + ASSERT_OK(Flush()); + if (i == 0) { + // Prevent trivial move so file will be rewritten with dictionary and + // reopened with L1's pinning settings. + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + } + } + + // Clear all unpinned blocks so unpinned blocks will show up as cache misses + // when reading a key from a file. + table_options.block_cache->EraseUnRefEntries(); + + // Get base cache values + uint64_t filter_misses = TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS); + uint64_t index_misses = TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS); + uint64_t compression_dict_misses = + TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_MISS); + + // Read a key from the L0 file + Get(Key(kNumKeysPerFile)); + uint64_t expected_filter_misses = filter_misses; + uint64_t expected_index_misses = index_misses; + uint64_t expected_compression_dict_misses = compression_dict_misses; + if (partition_index_and_filters_) { + if (top_level_index_pinning_ == PinningTier::kNone) { + ++expected_filter_misses; + ++expected_index_misses; + } + if (partition_pinning_ == PinningTier::kNone) { + ++expected_filter_misses; + ++expected_index_misses; + } + } else { + if (unpartitioned_pinning_ == PinningTier::kNone) { + ++expected_filter_misses; + ++expected_index_misses; + } + } + if (unpartitioned_pinning_ == PinningTier::kNone) { + ++expected_compression_dict_misses; + } + ASSERT_EQ(expected_filter_misses, + TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(expected_index_misses, + TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); + ASSERT_EQ(expected_compression_dict_misses, + TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_MISS)); + + // Clear all unpinned blocks so unpinned blocks will show up as cache misses + // when reading a key from a file. + table_options.block_cache->EraseUnRefEntries(); + + // Read a key from the L1 file + Get(Key(0)); + if (partition_index_and_filters_) { + if (top_level_index_pinning_ == PinningTier::kNone || + top_level_index_pinning_ == PinningTier::kFlushedAndSimilar) { + ++expected_filter_misses; + ++expected_index_misses; + } + if (partition_pinning_ == PinningTier::kNone || + partition_pinning_ == PinningTier::kFlushedAndSimilar) { + ++expected_filter_misses; + ++expected_index_misses; + } + } else { + if (unpartitioned_pinning_ == PinningTier::kNone || + unpartitioned_pinning_ == PinningTier::kFlushedAndSimilar) { + ++expected_filter_misses; + ++expected_index_misses; + } + } + if (unpartitioned_pinning_ == PinningTier::kNone || + unpartitioned_pinning_ == PinningTier::kFlushedAndSimilar) { + ++expected_compression_dict_misses; + } + ASSERT_EQ(expected_filter_misses, + TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(expected_index_misses, + TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); + ASSERT_EQ(expected_compression_dict_misses, + TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_MISS)); +} + +INSTANTIATE_TEST_CASE_P( + DBBlockCachePinningTest, DBBlockCachePinningTest, + ::testing::Combine( + ::testing::Bool(), + ::testing::Values(PinningTier::kNone, PinningTier::kFlushedAndSimilar, + PinningTier::kAll), + ::testing::Values(PinningTier::kNone, PinningTier::kFlushedAndSimilar, + PinningTier::kAll), + ::testing::Values(PinningTier::kNone, PinningTier::kFlushedAndSimilar, + PinningTier::kAll))); + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_bloom_filter_test.cc b/librocksdb-sys/rocksdb/db/db_bloom_filter_test.cc new file mode 100644 index 0000000..69face5 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_bloom_filter_test.cc @@ -0,0 +1,3513 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include +#include +#include + +#include "cache/cache_entry_roles.h" +#include "cache/cache_reservation_manager.h" +#include "db/db_test_util.h" +#include "options/options_helper.h" +#include "port/stack_trace.h" +#include "rocksdb/advanced_options.h" +#include "rocksdb/convenience.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/perf_context.h" +#include "rocksdb/statistics.h" +#include "rocksdb/table.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/block_based/filter_policy_internal.h" +#include "table/format.h" +#include "test_util/testutil.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { +std::shared_ptr Create(double bits_per_key, + const std::string& name) { + return BloomLikeFilterPolicy::Create(name, bits_per_key); +} +const std::string kLegacyBloom = test::LegacyBloomFilterPolicy::kClassName(); +const std::string kFastLocalBloom = + test::FastLocalBloomFilterPolicy::kClassName(); +const std::string kStandard128Ribbon = + test::Standard128RibbonFilterPolicy::kClassName(); +const std::string kAutoBloom = BloomFilterPolicy::kClassName(); +const std::string kAutoRibbon = RibbonFilterPolicy::kClassName(); + +template +T Pop(T& var) { + auto rv = var; + var = 0; + return rv; +} +PerfContextByLevel& GetLevelPerfContext(uint32_t level) { + return (*(get_perf_context()->level_to_perf_context))[level]; +} +} // anonymous namespace + +// DB tests related to bloom filter. + +class DBBloomFilterTest : public DBTestBase { + public: + DBBloomFilterTest() + : DBTestBase("db_bloom_filter_test", /*env_do_fsync=*/true) {} +}; + +class DBBloomFilterTestWithParam + : public DBTestBase, + public testing::WithParamInterface< + std::tuple> { + // public testing::WithParamInterface { + protected: + std::string bfp_impl_; + bool partition_filters_; + uint32_t format_version_; + + public: + DBBloomFilterTestWithParam() + : DBTestBase("db_bloom_filter_tests", /*env_do_fsync=*/true) {} + + ~DBBloomFilterTestWithParam() override {} + + void SetUp() override { + bfp_impl_ = std::get<0>(GetParam()); + partition_filters_ = std::get<1>(GetParam()); + format_version_ = std::get<2>(GetParam()); + } +}; + +class DBBloomFilterTestDefFormatVersion : public DBBloomFilterTestWithParam {}; + +class SliceTransformLimitedDomainGeneric : public SliceTransform { + const char* Name() const override { + return "SliceTransformLimitedDomainGeneric"; + } + + Slice Transform(const Slice& src) const override { + return Slice(src.data(), 5); + } + + bool InDomain(const Slice& src) const override { + // prefix will be x???? + return src.size() >= 5; + } + + bool InRange(const Slice& dst) const override { + // prefix will be x???? + return dst.size() == 5; + } +}; + +// KeyMayExist can lead to a few false positives, but not false negatives. +// To make test deterministic, use a much larger number of bits per key-20 than +// bits in the key, so that false positives are eliminated +TEST_P(DBBloomFilterTestDefFormatVersion, KeyMayExist) { + do { + ReadOptions ropts; + std::string value; + anon::OptionsOverride options_override; + options_override.filter_policy = Create(20, bfp_impl_); + options_override.partition_filters = partition_filters_; + options_override.metadata_block_size = 32; + options_override.full_block_cache = true; + Options options = CurrentOptions(options_override); + if (partition_filters_) { + auto* table_options = + options.table_factory->GetOptions(); + if (table_options != nullptr && + table_options->index_type != + BlockBasedTableOptions::kTwoLevelIndexSearch) { + // In the current implementation partitioned filters depend on + // partitioned indexes + continue; + } + } + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_TRUE(!db_->KeyMayExist(ropts, handles_[1], "a", &value)); + + ASSERT_OK(Put(1, "a", "b")); + bool value_found = false; + ASSERT_TRUE( + db_->KeyMayExist(ropts, handles_[1], "a", &value, &value_found)); + ASSERT_TRUE(value_found); + ASSERT_EQ("b", value); + + ASSERT_OK(Flush(1)); + value.clear(); + + uint64_t numopen = TestGetTickerCount(options, NO_FILE_OPENS); + uint64_t cache_added = TestGetTickerCount(options, BLOCK_CACHE_ADD); + ASSERT_TRUE( + db_->KeyMayExist(ropts, handles_[1], "a", &value, &value_found)); + ASSERT_TRUE(!value_found); + // assert that no new files were opened and no new blocks were + // read into block cache. + ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS)); + ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD)); + + ASSERT_OK(Delete(1, "a")); + + numopen = TestGetTickerCount(options, NO_FILE_OPENS); + cache_added = TestGetTickerCount(options, BLOCK_CACHE_ADD); + ASSERT_TRUE(!db_->KeyMayExist(ropts, handles_[1], "a", &value)); + ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS)); + ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD)); + + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1], + true /* disallow trivial move */)); + + numopen = TestGetTickerCount(options, NO_FILE_OPENS); + cache_added = TestGetTickerCount(options, BLOCK_CACHE_ADD); + ASSERT_TRUE(!db_->KeyMayExist(ropts, handles_[1], "a", &value)); + ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS)); + ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD)); + + ASSERT_OK(Delete(1, "c")); + + numopen = TestGetTickerCount(options, NO_FILE_OPENS); + cache_added = TestGetTickerCount(options, BLOCK_CACHE_ADD); + ASSERT_TRUE(!db_->KeyMayExist(ropts, handles_[1], "c", &value)); + ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS)); + ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD)); + + // KeyMayExist function only checks data in block caches, which is not used + // by plain table format. + } while ( + ChangeOptions(kSkipPlainTable | kSkipHashIndex | kSkipFIFOCompaction)); +} + +TEST_F(DBBloomFilterTest, GetFilterByPrefixBloomCustomPrefixExtractor) { + for (bool partition_filters : {true, false}) { + Options options = last_options_; + options.prefix_extractor = + std::make_shared(); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + get_perf_context()->EnablePerLevelPerfContext(); + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(NewBloomFilterPolicy(10)); + if (partition_filters) { + bbto.partition_filters = true; + bbto.index_type = BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + } + bbto.whole_key_filtering = false; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + DestroyAndReopen(options); + + WriteOptions wo; + ReadOptions ro; + FlushOptions fo; + fo.wait = true; + std::string value; + + ASSERT_OK(dbfull()->Put(wo, "barbarbar", "foo")); + ASSERT_OK(dbfull()->Put(wo, "barbarbar2", "foo2")); + ASSERT_OK(dbfull()->Put(wo, "foofoofoo", "bar")); + + ASSERT_OK(dbfull()->Flush(fo)); + + ASSERT_EQ("foo", Get("barbarbar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0); + + ASSERT_EQ("foo2", Get("barbarbar2")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0); + + ASSERT_EQ("NOT_FOUND", Get("barbarbar3")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0); + + ASSERT_EQ("NOT_FOUND", Get("barfoofoo")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1); + EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1); + + ASSERT_EQ("NOT_FOUND", Get("foobarbar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1); + EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1); + + ro.total_order_seek = true; + // NOTE: total_order_seek no longer affects Get() + ASSERT_EQ("NOT_FOUND", Get("foobarbar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1); + EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1); + + // No bloom on extractor changed + ASSERT_OK(db_->SetOptions({{"prefix_extractor", "capped:10"}})); + ASSERT_EQ("NOT_FOUND", Get("foobarbar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0); + + // No bloom on extractor changed, after re-open + options.prefix_extractor.reset(NewCappedPrefixTransform(10)); + Reopen(options); + ASSERT_EQ("NOT_FOUND", Get("foobarbar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0); + + get_perf_context()->Reset(); + } +} + +TEST_F(DBBloomFilterTest, GetFilterByPrefixBloom) { + for (bool partition_filters : {true, false}) { + Options options = last_options_; + options.prefix_extractor.reset(NewFixedPrefixTransform(8)); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + get_perf_context()->EnablePerLevelPerfContext(); + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(NewBloomFilterPolicy(10)); + if (partition_filters) { + bbto.partition_filters = true; + bbto.index_type = BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + } + bbto.whole_key_filtering = false; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + DestroyAndReopen(options); + + WriteOptions wo; + ReadOptions ro; + FlushOptions fo; + fo.wait = true; + std::string value; + + ASSERT_OK(dbfull()->Put(wo, "barbarbar", "foo")); + ASSERT_OK(dbfull()->Put(wo, "barbarbar2", "foo2")); + ASSERT_OK(dbfull()->Put(wo, "foofoofoo", "bar")); + + ASSERT_OK(dbfull()->Flush(fo)); + + ASSERT_EQ("foo", Get("barbarbar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + ASSERT_EQ("foo2", Get("barbarbar2")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + ASSERT_EQ("NOT_FOUND", Get("barbarbar3")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0); + + ASSERT_EQ("NOT_FOUND", Get("barfoofoo")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1); + EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1); + + ASSERT_EQ("NOT_FOUND", Get("foobarbar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1); + EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1); + + ro.total_order_seek = true; + // NOTE: total_order_seek no longer affects Get() + ASSERT_EQ("NOT_FOUND", Get("foobarbar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1); + EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 1); + + // No bloom on extractor changed + ASSERT_OK(db_->SetOptions({{"prefix_extractor", "capped:10"}})); + ASSERT_EQ("NOT_FOUND", Get("foobarbar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(Pop(GetLevelPerfContext(0).bloom_filter_useful), 0); + + get_perf_context()->Reset(); + } +} + +TEST_F(DBBloomFilterTest, WholeKeyFilterProp) { + for (bool partition_filters : {true, false}) { + Options options = last_options_; + options.prefix_extractor.reset(NewFixedPrefixTransform(3)); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + get_perf_context()->EnablePerLevelPerfContext(); + + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(NewBloomFilterPolicy(10)); + bbto.whole_key_filtering = false; + if (partition_filters) { + bbto.partition_filters = true; + bbto.index_type = BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + } + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + DestroyAndReopen(options); + + WriteOptions wo; + ReadOptions ro; + FlushOptions fo; + fo.wait = true; + std::string value; + + ASSERT_OK(dbfull()->Put(wo, "foobar", "foo")); + // Needs insert some keys to make sure files are not filtered out by key + // ranges. + ASSERT_OK(dbfull()->Put(wo, "aaa", "")); + ASSERT_OK(dbfull()->Put(wo, "zzz", "")); + ASSERT_OK(dbfull()->Flush(fo)); + + Reopen(options); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + ASSERT_EQ("NOT_FOUND", Get("foo")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + ASSERT_EQ("NOT_FOUND", Get("bar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + ASSERT_EQ("foo", Get("foobar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + + // Reopen with whole key filtering enabled and prefix extractor + // NULL. Bloom filter should be off for both of whole key and + // prefix bloom. + bbto.whole_key_filtering = true; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + options.prefix_extractor.reset(); + Reopen(options); + + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + ASSERT_EQ("NOT_FOUND", Get("foo")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + ASSERT_EQ("NOT_FOUND", Get("bar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + ASSERT_EQ("foo", Get("foobar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + // Write DB with only full key filtering. + ASSERT_OK(dbfull()->Put(wo, "foobar", "foo")); + // Needs insert some keys to make sure files are not filtered out by key + // ranges. + ASSERT_OK(dbfull()->Put(wo, "aaa", "")); + ASSERT_OK(dbfull()->Put(wo, "zzz", "")); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + // Reopen with both of whole key off and prefix extractor enabled. + // Still no bloom filter should be used. + options.prefix_extractor.reset(NewFixedPrefixTransform(3)); + bbto.whole_key_filtering = false; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + Reopen(options); + + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + ASSERT_EQ("NOT_FOUND", Get("foo")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + ASSERT_EQ("NOT_FOUND", Get("bar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + ASSERT_EQ("foo", Get("foobar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + + // Try to create a DB with mixed files: + ASSERT_OK(dbfull()->Put(wo, "foobar", "foo")); + // Needs insert some keys to make sure files are not filtered out by key + // ranges. + ASSERT_OK(dbfull()->Put(wo, "aaa", "")); + ASSERT_OK(dbfull()->Put(wo, "zzz", "")); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + options.prefix_extractor.reset(); + bbto.whole_key_filtering = true; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + Reopen(options); + + // Try to create a DB with mixed files. + ASSERT_OK(dbfull()->Put(wo, "barfoo", "bar")); + // In this case needs insert some keys to make sure files are + // not filtered out by key ranges. + ASSERT_OK(dbfull()->Put(wo, "aaa", "")); + ASSERT_OK(dbfull()->Put(wo, "zzz", "")); + ASSERT_OK(Flush()); + + // Now we have two files: + // File 1: An older file with prefix bloom (disabled) + // File 2: A newer file with whole bloom filter. + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + ASSERT_EQ("NOT_FOUND", Get("foo")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1); + ASSERT_EQ("NOT_FOUND", Get("bar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1); + ASSERT_EQ("foo", Get("foobar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1); + ASSERT_EQ("bar", Get("barfoo")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + + // Reopen with the same setting: only whole key is used + Reopen(options); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + ASSERT_EQ("NOT_FOUND", Get("foo")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1); + ASSERT_EQ("NOT_FOUND", Get("bar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1); + ASSERT_EQ("foo", Get("foobar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1); + ASSERT_EQ("bar", Get("barfoo")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + + // Restart with both filters are allowed + options.prefix_extractor.reset(NewFixedPrefixTransform(3)); + bbto.whole_key_filtering = true; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + Reopen(options); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + // File 1 will has it filtered out. + // File 2 will not, as prefix `foo` exists in the file. + ASSERT_EQ("NOT_FOUND", Get("foo")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1); + ASSERT_EQ("NOT_FOUND", Get("bar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1); + ASSERT_EQ("foo", Get("foobar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1); + ASSERT_EQ("bar", Get("barfoo")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + + // Restart with only prefix bloom is allowed. + options.prefix_extractor.reset(NewFixedPrefixTransform(3)); + bbto.whole_key_filtering = false; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + Reopen(options); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + ASSERT_EQ("NOT_FOUND", Get("foo")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + ASSERT_EQ("NOT_FOUND", Get("bar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 1); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + ASSERT_EQ("foo", Get("foobar")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + ASSERT_EQ("bar", Get("barfoo")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); + uint64_t bloom_filter_useful_all_levels = 0; + for (auto& kv : (*(get_perf_context()->level_to_perf_context))) { + if (kv.second.bloom_filter_useful > 0) { + bloom_filter_useful_all_levels += kv.second.bloom_filter_useful; + } + } + ASSERT_EQ(12, bloom_filter_useful_all_levels); + get_perf_context()->Reset(); + } +} + +TEST_P(DBBloomFilterTestWithParam, BloomFilter) { + do { + Options options = CurrentOptions(); + env_->count_random_reads_ = true; + options.env = env_; + // ChangeCompactOptions() only changes compaction style, which does not + // trigger reset of table_factory + BlockBasedTableOptions table_options; + table_options.no_block_cache = true; + table_options.filter_policy = Create(10, bfp_impl_); + table_options.partition_filters = partition_filters_; + if (partition_filters_) { + table_options.index_type = + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + } + table_options.format_version = format_version_; + if (format_version_ >= 4) { + // value delta encoding challenged more with index interval > 1 + table_options.index_block_restart_interval = 8; + } + table_options.metadata_block_size = 32; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + CreateAndReopenWithCF({"pikachu"}, options); + + // Populate multiple layers + const int N = 10000; + for (int i = 0; i < N; i++) { + ASSERT_OK(Put(1, Key(i), Key(i))); + } + Compact(1, "a", "z"); + for (int i = 0; i < N; i += 100) { + ASSERT_OK(Put(1, Key(i), Key(i))); + } + ASSERT_OK(Flush(1)); + + // Prevent auto compactions triggered by seeks + env_->delay_sstable_sync_.store(true, std::memory_order_release); + + // Lookup present keys. Should rarely read from small sstable. + env_->random_read_counter_.Reset(); + for (int i = 0; i < N; i++) { + ASSERT_EQ(Key(i), Get(1, Key(i))); + } + int reads = env_->random_read_counter_.Read(); + fprintf(stderr, "%d present => %d reads\n", N, reads); + ASSERT_GE(reads, N); + if (partition_filters_) { + // Without block cache, we read an extra partition filter per each + // level*read and a partition index per each read + ASSERT_LE(reads, 4 * N + 2 * N / 100); + } else { + ASSERT_LE(reads, N + 2 * N / 100); + } + + // Lookup present keys. Should rarely read from either sstable. + env_->random_read_counter_.Reset(); + for (int i = 0; i < N; i++) { + ASSERT_EQ("NOT_FOUND", Get(1, Key(i) + ".missing")); + } + reads = env_->random_read_counter_.Read(); + fprintf(stderr, "%d missing => %d reads\n", N, reads); + if (partition_filters_) { + // With partitioned filter we read one extra filter per level per each + // missed read. + ASSERT_LE(reads, 2 * N + 3 * N / 100); + } else { + ASSERT_LE(reads, 3 * N / 100); + } + + // Sanity check some table properties + std::map props; + ASSERT_TRUE(db_->GetMapProperty( + handles_[1], DB::Properties::kAggregatedTableProperties, &props)); + uint64_t nkeys = N + N / 100; + uint64_t filter_size = ParseUint64(props["filter_size"]); + EXPECT_LE(filter_size, + (partition_filters_ ? 12 : 11) * nkeys / /*bits / byte*/ 8); + if (bfp_impl_ == kAutoRibbon) { + // Sometimes using Ribbon filter which is more space-efficient + EXPECT_GE(filter_size, 7 * nkeys / /*bits / byte*/ 8); + } else { + // Always Bloom + EXPECT_GE(filter_size, 10 * nkeys / /*bits / byte*/ 8); + } + + uint64_t num_filter_entries = ParseUint64(props["num_filter_entries"]); + EXPECT_EQ(num_filter_entries, nkeys); + + env_->delay_sstable_sync_.store(false, std::memory_order_release); + Close(); + } while (ChangeCompactOptions()); +} + +namespace { + +class AlwaysTrueBitsBuilder : public FilterBitsBuilder { + public: + void AddKey(const Slice&) override {} + size_t EstimateEntriesAdded() override { return 0U; } + Slice Finish(std::unique_ptr* /* buf */) override { + // Interpreted as "always true" filter (0 probes over 1 byte of + // payload, 5 bytes metadata) + return Slice("\0\0\0\0\0\0", 6); + } + using FilterBitsBuilder::Finish; + size_t ApproximateNumEntries(size_t) override { return SIZE_MAX; } +}; + +class AlwaysTrueFilterPolicy : public ReadOnlyBuiltinFilterPolicy { + public: + explicit AlwaysTrueFilterPolicy(bool skip) : skip_(skip) {} + + FilterBitsBuilder* GetBuilderWithContext( + const FilterBuildingContext&) const override { + if (skip_) { + return nullptr; + } else { + return new AlwaysTrueBitsBuilder(); + } + } + + private: + bool skip_; +}; + +} // anonymous namespace + +TEST_P(DBBloomFilterTestWithParam, SkipFilterOnEssentiallyZeroBpk) { + constexpr int maxKey = 10; + auto PutFn = [&]() { + int i; + // Put + for (i = 0; i < maxKey; i++) { + ASSERT_OK(Put(Key(i), Key(i))); + } + Flush(); + }; + auto GetFn = [&]() { + int i; + // Get OK + for (i = 0; i < maxKey; i++) { + ASSERT_EQ(Key(i), Get(Key(i))); + } + // Get NotFound + for (; i < maxKey * 2; i++) { + ASSERT_EQ(Get(Key(i)), "NOT_FOUND"); + } + }; + auto PutAndGetFn = [&]() { + PutFn(); + GetFn(); + }; + std::map props; + const auto& kAggTableProps = DB::Properties::kAggregatedTableProperties; + + Options options = CurrentOptions(); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + BlockBasedTableOptions table_options; + table_options.partition_filters = partition_filters_; + if (partition_filters_) { + table_options.index_type = + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + } + table_options.format_version = format_version_; + + // Test 1: bits per key < 0.5 means skip filters -> no filter + // constructed or read. + table_options.filter_policy = Create(0.4, bfp_impl_); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DestroyAndReopen(options); + PutAndGetFn(); + + // Verify no filter access nor contruction + EXPECT_EQ(TestGetTickerCount(options, BLOOM_FILTER_FULL_POSITIVE), 0); + EXPECT_EQ(TestGetTickerCount(options, BLOOM_FILTER_FULL_TRUE_POSITIVE), 0); + + props.clear(); + ASSERT_TRUE(db_->GetMapProperty(kAggTableProps, &props)); + EXPECT_EQ(props["filter_size"], "0"); + + // Test 2: use custom API to skip filters -> no filter constructed + // or read. + table_options.filter_policy.reset( + new AlwaysTrueFilterPolicy(/* skip */ true)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DestroyAndReopen(options); + PutAndGetFn(); + + // Verify no filter access nor construction + EXPECT_EQ(TestGetTickerCount(options, BLOOM_FILTER_FULL_POSITIVE), 0); + EXPECT_EQ(TestGetTickerCount(options, BLOOM_FILTER_FULL_TRUE_POSITIVE), 0); + + props.clear(); + ASSERT_TRUE(db_->GetMapProperty(kAggTableProps, &props)); + EXPECT_EQ(props["filter_size"], "0"); + + // Control test: using an actual filter with 100% FP rate -> the filter + // is constructed and checked on read. + table_options.filter_policy.reset( + new AlwaysTrueFilterPolicy(/* skip */ false)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DestroyAndReopen(options); + PutAndGetFn(); + + // Verify filter is accessed (and constructed) + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_FULL_POSITIVE), maxKey * 2); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_FULL_TRUE_POSITIVE), maxKey); + props.clear(); + ASSERT_TRUE(db_->GetMapProperty(kAggTableProps, &props)); + EXPECT_NE(props["filter_size"], "0"); + + // Test 3 (options test): Able to read existing filters with longstanding + // generated options file entry `filter_policy=rocksdb.BuiltinBloomFilter` + ASSERT_OK(FilterPolicy::CreateFromString(ConfigOptions(), + "rocksdb.BuiltinBloomFilter", + &table_options.filter_policy)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + GetFn(); + + // Verify filter is accessed + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_FULL_POSITIVE), maxKey * 2); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_FULL_TRUE_POSITIVE), maxKey); + + // But new filters are not generated (configuration details unknown) + DestroyAndReopen(options); + PutAndGetFn(); + + // Verify no filter access nor construction + EXPECT_EQ(TestGetTickerCount(options, BLOOM_FILTER_FULL_POSITIVE), 0); + EXPECT_EQ(TestGetTickerCount(options, BLOOM_FILTER_FULL_TRUE_POSITIVE), 0); + + props.clear(); + ASSERT_TRUE(db_->GetMapProperty(kAggTableProps, &props)); + EXPECT_EQ(props["filter_size"], "0"); +} + +#if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) +INSTANTIATE_TEST_CASE_P( + FormatDef, DBBloomFilterTestDefFormatVersion, + ::testing::Values( + std::make_tuple(kAutoBloom, true, test::kDefaultFormatVersion), + std::make_tuple(kAutoBloom, false, test::kDefaultFormatVersion), + std::make_tuple(kAutoRibbon, false, test::kDefaultFormatVersion))); + +INSTANTIATE_TEST_CASE_P( + FormatDef, DBBloomFilterTestWithParam, + ::testing::Values( + std::make_tuple(kAutoBloom, true, test::kDefaultFormatVersion), + std::make_tuple(kAutoBloom, false, test::kDefaultFormatVersion), + std::make_tuple(kAutoRibbon, false, test::kDefaultFormatVersion))); + +INSTANTIATE_TEST_CASE_P( + FormatLatest, DBBloomFilterTestWithParam, + ::testing::Values(std::make_tuple(kAutoBloom, true, kLatestFormatVersion), + std::make_tuple(kAutoBloom, false, kLatestFormatVersion), + std::make_tuple(kAutoRibbon, false, + kLatestFormatVersion))); +#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) + +TEST_F(DBBloomFilterTest, BloomFilterRate) { + while (ChangeFilterOptions()) { + Options options = CurrentOptions(); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + get_perf_context()->EnablePerLevelPerfContext(); + CreateAndReopenWithCF({"pikachu"}, options); + + const int maxKey = 10000; + for (int i = 0; i < maxKey; i++) { + ASSERT_OK(Put(1, Key(i), Key(i))); + } + // Add a large key to make the file contain wide range + ASSERT_OK(Put(1, Key(maxKey + 55555), Key(maxKey + 55555))); + Flush(1); + + // Check if they can be found + for (int i = 0; i < maxKey; i++) { + ASSERT_EQ(Key(i), Get(1, Key(i))); + } + ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0); + + // Check if filter is useful + for (int i = 0; i < maxKey; i++) { + ASSERT_EQ("NOT_FOUND", Get(1, Key(i + 33333))); + } + ASSERT_GE(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), maxKey * 0.98); + ASSERT_GE(GetLevelPerfContext(0).bloom_filter_useful, maxKey * 0.98); + get_perf_context()->Reset(); + } +} + +namespace { +struct CompatibilityConfig { + std::shared_ptr policy; + bool partitioned; + uint32_t format_version; + + void SetInTableOptions(BlockBasedTableOptions* table_options) { + table_options->filter_policy = policy; + table_options->partition_filters = partitioned; + if (partitioned) { + table_options->index_type = + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + } else { + table_options->index_type = + BlockBasedTableOptions::IndexType::kBinarySearch; + } + table_options->format_version = format_version; + } +}; +// High bits per key -> almost no FPs +std::shared_ptr kCompatibilityBloomPolicy{ + NewBloomFilterPolicy(20)}; +// bloom_before_level=-1 -> always use Ribbon +std::shared_ptr kCompatibilityRibbonPolicy{ + NewRibbonFilterPolicy(20, -1)}; + +std::vector kCompatibilityConfigs = { + {kCompatibilityBloomPolicy, false, BlockBasedTableOptions().format_version}, + {kCompatibilityBloomPolicy, true, BlockBasedTableOptions().format_version}, + {kCompatibilityBloomPolicy, false, /* legacy Bloom */ 4U}, + {kCompatibilityRibbonPolicy, false, + BlockBasedTableOptions().format_version}, + {kCompatibilityRibbonPolicy, true, BlockBasedTableOptions().format_version}, +}; +} // anonymous namespace + +TEST_F(DBBloomFilterTest, BloomFilterCompatibility) { + Options options = CurrentOptions(); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.level0_file_num_compaction_trigger = + static_cast(kCompatibilityConfigs.size()) + 1; + options.max_open_files = -1; + + Close(); + + // Create one file for each kind of filter. Each file covers a distinct key + // range. + for (size_t i = 0; i < kCompatibilityConfigs.size(); ++i) { + BlockBasedTableOptions table_options; + kCompatibilityConfigs[i].SetInTableOptions(&table_options); + ASSERT_TRUE(table_options.filter_policy != nullptr); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + + std::string prefix = std::to_string(i) + "_"; + ASSERT_OK(Put(prefix + "A", "val")); + ASSERT_OK(Put(prefix + "Z", "val")); + ASSERT_OK(Flush()); + } + + // Test filter is used between each pair of {reader,writer} configurations, + // because any built-in FilterPolicy should be able to read filters from any + // other built-in FilterPolicy + for (size_t i = 0; i < kCompatibilityConfigs.size(); ++i) { + BlockBasedTableOptions table_options; + kCompatibilityConfigs[i].SetInTableOptions(&table_options); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + for (size_t j = 0; j < kCompatibilityConfigs.size(); ++j) { + std::string prefix = std::to_string(j) + "_"; + ASSERT_EQ("val", Get(prefix + "A")); // Filter positive + ASSERT_EQ("val", Get(prefix + "Z")); // Filter positive + // Filter negative, with high probability + ASSERT_EQ("NOT_FOUND", Get(prefix + "Q")); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_FULL_POSITIVE), 2); + EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 1); + } + } +} + +// To align with the type of hash entry being reserved in implementation. +using FilterConstructionReserveMemoryHash = uint64_t; + +class ChargeFilterConstructionTestWithParam + : public DBTestBase, + public testing::WithParamInterface> { + public: + ChargeFilterConstructionTestWithParam() + : DBTestBase("db_bloom_filter_tests", + /*env_do_fsync=*/true), + num_key_(0), + charge_filter_construction_(std::get<0>(GetParam())), + policy_(std::get<1>(GetParam())), + partition_filters_(std::get<2>(GetParam())), + detect_filter_construct_corruption_(std::get<3>(GetParam())) { + if (charge_filter_construction_ == + CacheEntryRoleOptions::Decision::kDisabled || + policy_ == kLegacyBloom) { + // For these cases, we only interested in whether filter construction + // cache charging happens instead of its accuracy. Therefore we don't + // need many keys. + num_key_ = 5; + } else if (partition_filters_) { + // For PartitionFilter case, since we set + // table_options.metadata_block_size big enough such that each partition + // trigger at least 1 dummy entry reservation each for hash entries and + // final filter, we need a large number of keys to ensure we have at least + // two partitions. + num_key_ = 18 * + CacheReservationManagerImpl< + CacheEntryRole::kFilterConstruction>::GetDummyEntrySize() / + sizeof(FilterConstructionReserveMemoryHash); + } else if (policy_ == kFastLocalBloom) { + // For Bloom Filter + FullFilter case, since we design the num_key_ to + // make hash entry cache charging be a multiple of dummy entries, the + // correct behavior of charging final filter on top of it will trigger at + // least another dummy entry insertion. Therefore we can assert that + // behavior and we don't need a large number of keys to verify we + // indeed charge the final filter for in cache, even though final + // filter is a lot smaller than hash entries. + num_key_ = 1 * + CacheReservationManagerImpl< + CacheEntryRole::kFilterConstruction>::GetDummyEntrySize() / + sizeof(FilterConstructionReserveMemoryHash); + } else { + // For Ribbon Filter + FullFilter case, we need a large enough number of + // keys so that charging final filter after releasing the hash entries + // reservation will trigger at least another dummy entry (or equivalently + // to saying, causing another peak in cache charging) as banding + // reservation might not be a multiple of dummy entry. + num_key_ = 12 * + CacheReservationManagerImpl< + CacheEntryRole::kFilterConstruction>::GetDummyEntrySize() / + sizeof(FilterConstructionReserveMemoryHash); + } + } + + BlockBasedTableOptions GetBlockBasedTableOptions() { + BlockBasedTableOptions table_options; + + // We set cache capacity big enough to prevent cache full for convenience in + // calculation. + constexpr std::size_t kCacheCapacity = 100 * 1024 * 1024; + + table_options.cache_usage_options.options_overrides.insert( + {CacheEntryRole::kFilterConstruction, + {/*.charged = */ charge_filter_construction_}}); + table_options.filter_policy = Create(10, policy_); + table_options.partition_filters = partition_filters_; + if (table_options.partition_filters) { + table_options.index_type = + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + // We set table_options.metadata_block_size big enough so that each + // partition trigger at least 1 dummy entry insertion each for hash + // entries and final filter. + table_options.metadata_block_size = 409000; + } + table_options.detect_filter_construct_corruption = + detect_filter_construct_corruption_; + + LRUCacheOptions lo; + lo.capacity = kCacheCapacity; + lo.num_shard_bits = 0; // 2^0 shard + lo.strict_capacity_limit = true; + cache_ = std::make_shared< + TargetCacheChargeTrackingCache>( + (NewLRUCache(lo))); + table_options.block_cache = cache_; + + return table_options; + } + + std::size_t GetNumKey() { return num_key_; } + + CacheEntryRoleOptions::Decision ChargeFilterConstructMemory() { + return charge_filter_construction_; + } + + std::string GetFilterPolicy() { return policy_; } + + bool PartitionFilters() { return partition_filters_; } + + std::shared_ptr< + TargetCacheChargeTrackingCache> + GetCache() { + return cache_; + } + + private: + std::size_t num_key_; + CacheEntryRoleOptions::Decision charge_filter_construction_; + std::string policy_; + bool partition_filters_; + std::shared_ptr< + TargetCacheChargeTrackingCache> + cache_; + bool detect_filter_construct_corruption_; +}; + +INSTANTIATE_TEST_CASE_P( + ChargeFilterConstructionTestWithParam, + ChargeFilterConstructionTestWithParam, + ::testing::Values( + std::make_tuple(CacheEntryRoleOptions::Decision::kDisabled, + kFastLocalBloom, false, false), + + std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled, + kFastLocalBloom, false, false), + std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled, + kFastLocalBloom, false, true), + std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled, + kFastLocalBloom, true, false), + std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled, + kFastLocalBloom, true, true), + + std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled, + kStandard128Ribbon, false, false), + std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled, + kStandard128Ribbon, false, true), + std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled, + kStandard128Ribbon, true, false), + std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled, + kStandard128Ribbon, true, true), + + std::make_tuple(CacheEntryRoleOptions::Decision::kEnabled, kLegacyBloom, + false, false))); + +// TODO: Speed up this test, and reduce disk space usage (~700MB) +// The current test inserts many keys (on the scale of dummy entry size) +// in order to make small memory user (e.g, final filter, partitioned hash +// entries/filter/banding) , which is proportional to the number of +// keys, big enough so that its cache charging triggers dummy entry insertion +// and becomes observable in the test. +// +// However, inserting that many keys slows down this test and leaves future +// developers an opportunity to speed it up. +// +// Possible approaches & challenges: +// 1. Use sync point during cache charging of filter construction +// +// Benefit: It does not rely on triggering dummy entry insertion +// but the sync point to verify small memory user is charged correctly. +// +// Challenge: this approach is intrusive. +// +// 2. Make dummy entry size configurable and set it small in the test +// +// Benefit: It increases the precision of cache charging and therefore +// small memory usage can still trigger insertion of dummy entry. +// +// Challenge: change CacheReservationManager related APIs and a hack +// might be needed to control the size of dummmy entry of +// CacheReservationManager used in filter construction for testing +// since CacheReservationManager is not exposed at the high level. +// +TEST_P(ChargeFilterConstructionTestWithParam, Basic) { + Options options = CurrentOptions(); + // We set write_buffer_size big enough so that in the case where there is + // filter construction cache charging, flush won't be triggered before we + // manually trigger it for clean testing + options.write_buffer_size = 640 << 20; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + std::shared_ptr< + TargetCacheChargeTrackingCache> + cache = GetCache(); + options.create_if_missing = true; + // Disable auto compaction to prevent its unexpected side effect + // to the number of keys per partition designed by us in the test + options.disable_auto_compactions = true; + DestroyAndReopen(options); + int num_key = static_cast(GetNumKey()); + for (int i = 0; i < num_key; i++) { + ASSERT_OK(Put(Key(i), Key(i))); + } + + ASSERT_EQ(cache->GetChargedCacheIncrementSum(), 0) + << "Flush was triggered too early in the test case with filter " + "construction cache charging - please make sure no flush triggered " + "during the key insertions above"; + + ASSERT_OK(Flush()); + + bool charge_filter_construction = (ChargeFilterConstructMemory() == + CacheEntryRoleOptions::Decision::kEnabled); + std::string policy = GetFilterPolicy(); + bool partition_filters = PartitionFilters(); + bool detect_filter_construct_corruption = + table_options.detect_filter_construct_corruption; + + std::deque filter_construction_cache_res_peaks = + cache->GetChargedCachePeaks(); + std::size_t filter_construction_cache_res_increments_sum = + cache->GetChargedCacheIncrementSum(); + + if (!charge_filter_construction) { + EXPECT_EQ(filter_construction_cache_res_peaks.size(), 0); + return; + } + + if (policy == kLegacyBloom) { + EXPECT_EQ(filter_construction_cache_res_peaks.size(), 0) + << "There shouldn't be filter construction cache charging as this " + "feature does not support kLegacyBloom"; + return; + } + + const std::size_t kDummyEntrySize = CacheReservationManagerImpl< + CacheEntryRole::kFilterConstruction>::GetDummyEntrySize(); + + const std::size_t predicted_hash_entries_cache_res = + num_key * sizeof(FilterConstructionReserveMemoryHash); + ASSERT_EQ(predicted_hash_entries_cache_res % kDummyEntrySize, 0) + << "It's by this test's design that predicted_hash_entries_cache_res is " + "a multipe of dummy entry"; + + const std::size_t predicted_hash_entries_cache_res_dummy_entry_num = + predicted_hash_entries_cache_res / kDummyEntrySize; + const std::size_t predicted_final_filter_cache_res = + static_cast( + std::ceil(1.0 * predicted_hash_entries_cache_res_dummy_entry_num / 6 * + (policy == kStandard128Ribbon ? 0.7 : 1))) * + kDummyEntrySize; + const std::size_t predicted_banding_cache_res = + static_cast( + std::ceil(predicted_hash_entries_cache_res_dummy_entry_num * 2.5)) * + kDummyEntrySize; + + if (policy == kFastLocalBloom) { + /* kFastLocalBloom + FullFilter + * p0 + * / \ + * b / \ + * / \ + * / \ + * 0/ \ + * hash entries = b - 0, final filter = p0 - b + * p0 = hash entries + final filter + * + * The test is designed in a way such that the reservation for b is a + * multiple of dummy entries so that reservation for (p0 - b) + * will trigger at least another dummy entry insertion. + * + * kFastLocalBloom + FullFilter + + * detect_filter_construct_corruption + * The peak p0 stays the same as + * (kFastLocalBloom + FullFilter) but just lasts + * longer since we release hash entries reservation later. + * + * kFastLocalBloom + PartitionedFilter + * p1 + * / \ + * p0 b'/ \ + * / \ / \ + * b / \ / \ + * / \ / \ + * / a \ + * 0/ \ + * partitioned hash entries1 = b - 0, partitioned hash entries1 = b' - a + * parittioned final filter1 = p0 - b, parittioned final filter2 = p1 - b' + * + * (increment p0 - 0) + (increment p1 - a) + * = partitioned hash entries1 + partitioned hash entries2 + * + parittioned final filter1 + parittioned final filter2 + * = hash entries + final filter + * + * kFastLocalBloom + PartitionedFilter + + * detect_filter_construct_corruption + * The peak p0, p1 stay the same as + * (kFastLocalBloom + PartitionedFilter) but just + * last longer since we release hash entries reservation later. + * + */ + if (!partition_filters) { + EXPECT_EQ(filter_construction_cache_res_peaks.size(), 1) + << "Filter construction cache charging should have only 1 peak in " + "case: kFastLocalBloom + FullFilter"; + std::size_t filter_construction_cache_res_peak = + filter_construction_cache_res_peaks[0]; + EXPECT_GT(filter_construction_cache_res_peak, + predicted_hash_entries_cache_res) + << "The testing number of hash entries is designed to make hash " + "entries cache charging be multiples of dummy entries" + " so the correct behavior of charging final filter on top of it" + " should've triggered at least another dummy entry insertion"; + + std::size_t predicted_filter_construction_cache_res_peak = + predicted_hash_entries_cache_res + predicted_final_filter_cache_res; + EXPECT_GE(filter_construction_cache_res_peak, + predicted_filter_construction_cache_res_peak * 0.9); + EXPECT_LE(filter_construction_cache_res_peak, + predicted_filter_construction_cache_res_peak * 1.1); + return; + } else { + EXPECT_GE(filter_construction_cache_res_peaks.size(), 2) + << "Filter construction cache charging should have multiple peaks " + "in case: kFastLocalBloom + " + "PartitionedFilter"; + std::size_t predicted_filter_construction_cache_res_increments_sum = + predicted_hash_entries_cache_res + predicted_final_filter_cache_res; + EXPECT_GE(filter_construction_cache_res_increments_sum, + predicted_filter_construction_cache_res_increments_sum * 0.9); + EXPECT_LE(filter_construction_cache_res_increments_sum, + predicted_filter_construction_cache_res_increments_sum * 1.1); + return; + } + } + + if (policy == kStandard128Ribbon) { + /* kStandard128Ribbon + FullFilter + * p0 + * / \ p1 + * / \/\ + * b / b' \ + * / \ + * 0/ \ + * hash entries = b - 0, banding = p0 - b, final filter = p1 - b' + * p0 = hash entries + banding + * + * The test is designed in a way such that the reservation for (p1 - b') + * will trigger at least another dummy entry insertion + * (or equivalently to saying, creating another peak). + * + * kStandard128Ribbon + FullFilter + + * detect_filter_construct_corruption + * + * new p0 + * / \ + * / \ + * pre p0 \ + * / \ + * / \ + * b / \ + * / \ + * 0/ \ + * hash entries = b - 0, banding = pre p0 - b, + * final filter = new p0 - pre p0 + * new p0 = hash entries + banding + final filter + * + * The previous p0 will no longer be a peak since under + * detect_filter_construct_corruption == true, we do not release hash + * entries reserveration (like p0 - b' previously) until after final filter + * creation and post-verification + * + * kStandard128Ribbon + PartitionedFilter + * p3 + * p0 /\ p4 + * / \ p1 / \ /\ + * / \/\ b''/ a' \ + * b / b' \ / \ + * / \ / \ + * 0/ a \ + * partitioned hash entries1 = b - 0, partitioned hash entries2 = b'' - a + * partitioned banding1 = p0 - b, partitioned banding2 = p3 - b'' + * parittioned final filter1 = p1 - b',parittioned final filter2 = p4 - a' + * + * (increment p0 - 0) + (increment p1 - b') + * + (increment p3 - a) + (increment p4 - a') + * = partitioned hash entries1 + partitioned hash entries2 + * + parittioned banding1 + parittioned banding2 + * + parittioned final filter1 + parittioned final filter2 + * = hash entries + banding + final filter + * + * kStandard128Ribbon + PartitionedFilter + + * detect_filter_construct_corruption + * + * new p3 + * / \ + * pre p3 \ + * new p0 / \ + * / \ / \ + * pre p0 \ / \ + * / \ b'/ \ + * / \ / \ + * b / \ / \ + * / \a \ + * 0/ \ + * partitioned hash entries1 = b - 0, partitioned hash entries2 = b' - a + * partitioned banding1 = pre p0 - b, partitioned banding2 = pre p3 - b' + * parittioned final filter1 = new p0 - pre p0, + * parittioned final filter2 = new p3 - pre p3 + * + * The previous p0 and p3 will no longer be a peak since under + * detect_filter_construct_corruption == true, we do not release hash + * entries reserveration (like p0 - b', p3 - a' previously) until after + * parittioned final filter creation and post-verification + * + * However, increments sum stay the same as shown below: + * (increment new p0 - 0) + (increment new p3 - a) + * = partitioned hash entries1 + partitioned hash entries2 + * + parittioned banding1 + parittioned banding2 + * + parittioned final filter1 + parittioned final filter2 + * = hash entries + banding + final filter + * + */ + if (!partition_filters) { + ASSERT_GE( + std::floor( + 1.0 * predicted_final_filter_cache_res / + CacheReservationManagerImpl< + CacheEntryRole::kFilterConstruction>::GetDummyEntrySize()), + 1) + << "Final filter cache charging too small for this test - please " + "increase the number of keys"; + if (!detect_filter_construct_corruption) { + EXPECT_EQ(filter_construction_cache_res_peaks.size(), 2) + << "Filter construction cache charging should have 2 peaks in " + "case: kStandard128Ribbon + " + "FullFilter. " + "The second peak is resulted from charging the final filter " + "after " + "decreasing the hash entry reservation since the testing final " + "filter reservation is designed to be at least 1 dummy entry " + "size"; + + std::size_t filter_construction_cache_res_peak = + filter_construction_cache_res_peaks[0]; + std::size_t predicted_filter_construction_cache_res_peak = + predicted_hash_entries_cache_res + predicted_banding_cache_res; + EXPECT_GE(filter_construction_cache_res_peak, + predicted_filter_construction_cache_res_peak * 0.9); + EXPECT_LE(filter_construction_cache_res_peak, + predicted_filter_construction_cache_res_peak * 1.1); + } else { + EXPECT_EQ(filter_construction_cache_res_peaks.size(), 1) + << "Filter construction cache charging should have 1 peaks in " + "case: kStandard128Ribbon + FullFilter " + "+ detect_filter_construct_corruption. " + "The previous second peak now disappears since we don't " + "decrease the hash entry reservation" + "until after final filter reservation and post-verification"; + + std::size_t filter_construction_cache_res_peak = + filter_construction_cache_res_peaks[0]; + std::size_t predicted_filter_construction_cache_res_peak = + predicted_hash_entries_cache_res + predicted_banding_cache_res + + predicted_final_filter_cache_res; + EXPECT_GE(filter_construction_cache_res_peak, + predicted_filter_construction_cache_res_peak * 0.9); + EXPECT_LE(filter_construction_cache_res_peak, + predicted_filter_construction_cache_res_peak * 1.1); + } + return; + } else { + if (!detect_filter_construct_corruption) { + EXPECT_GE(filter_construction_cache_res_peaks.size(), 3) + << "Filter construction cache charging should have more than 3 " + "peaks " + "in case: kStandard128Ribbon + " + "PartitionedFilter"; + } else { + EXPECT_GE(filter_construction_cache_res_peaks.size(), 2) + << "Filter construction cache charging should have more than 2 " + "peaks " + "in case: kStandard128Ribbon + " + "PartitionedFilter + detect_filter_construct_corruption"; + } + std::size_t predicted_filter_construction_cache_res_increments_sum = + predicted_hash_entries_cache_res + predicted_banding_cache_res + + predicted_final_filter_cache_res; + EXPECT_GE(filter_construction_cache_res_increments_sum, + predicted_filter_construction_cache_res_increments_sum * 0.9); + EXPECT_LE(filter_construction_cache_res_increments_sum, + predicted_filter_construction_cache_res_increments_sum * 1.1); + return; + } + } +} + +class DBFilterConstructionCorruptionTestWithParam + : public DBTestBase, + public testing::WithParamInterface< + std::tuple> { + public: + DBFilterConstructionCorruptionTestWithParam() + : DBTestBase("db_bloom_filter_tests", + /*env_do_fsync=*/true) {} + + BlockBasedTableOptions GetBlockBasedTableOptions() { + BlockBasedTableOptions table_options; + table_options.detect_filter_construct_corruption = std::get<0>(GetParam()); + table_options.filter_policy = Create(10, std::get<1>(GetParam())); + table_options.partition_filters = std::get<2>(GetParam()); + if (table_options.partition_filters) { + table_options.index_type = + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + // We set table_options.metadata_block_size small enough so we can + // trigger filter partitioning with GetNumKey() amount of keys + table_options.metadata_block_size = 10; + } + + return table_options; + } + + // Return an appropriate amount of keys for testing + // to generate a long filter (i.e, size >= 8 + kMetadataLen) + std::size_t GetNumKey() { return 5000; } +}; + +INSTANTIATE_TEST_CASE_P( + DBFilterConstructionCorruptionTestWithParam, + DBFilterConstructionCorruptionTestWithParam, + ::testing::Values(std::make_tuple(false, kFastLocalBloom, false), + std::make_tuple(true, kFastLocalBloom, false), + std::make_tuple(true, kFastLocalBloom, true), + std::make_tuple(true, kStandard128Ribbon, false), + std::make_tuple(true, kStandard128Ribbon, true))); + +TEST_P(DBFilterConstructionCorruptionTestWithParam, DetectCorruption) { + Options options = CurrentOptions(); + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.create_if_missing = true; + options.disable_auto_compactions = true; + + DestroyAndReopen(options); + int num_key = static_cast(GetNumKey()); + Status s; + + // Case 1: No corruption in filter construction + for (int i = 0; i < num_key; i++) { + ASSERT_OK(Put(Key(i), Key(i))); + } + s = Flush(); + EXPECT_TRUE(s.ok()); + + // Case 2: Corruption of hash entries in filter construction + for (int i = 0; i < num_key; i++) { + ASSERT_OK(Put(Key(i), Key(i))); + } + + SyncPoint::GetInstance()->SetCallBack( + "XXPH3FilterBitsBuilder::Finish::TamperHashEntries", [&](void* arg) { + std::deque* hash_entries_to_corrupt = + (std::deque*)arg; + assert(!hash_entries_to_corrupt->empty()); + *(hash_entries_to_corrupt->begin()) = + *(hash_entries_to_corrupt->begin()) ^ uint64_t { 1 }; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + s = Flush(); + + if (table_options.detect_filter_construct_corruption) { + EXPECT_TRUE(s.IsCorruption()); + EXPECT_TRUE( + s.ToString().find("Filter's hash entries checksum mismatched") != + std::string::npos); + } else { + EXPECT_TRUE(s.ok()); + } + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearCallBack( + "XXPH3FilterBitsBuilder::Finish::" + "TamperHashEntries"); + + // Case 3: Corruption of filter content in filter construction + DestroyAndReopen(options); + + for (int i = 0; i < num_key; i++) { + ASSERT_OK(Put(Key(i), Key(i))); + } + + SyncPoint::GetInstance()->SetCallBack( + "XXPH3FilterBitsBuilder::Finish::TamperFilter", [&](void* arg) { + std::pair*, std::size_t>* TEST_arg_pair = + (std::pair*, std::size_t>*)arg; + std::size_t filter_size = TEST_arg_pair->second; + // 5 is the kMetadataLen and + assert(filter_size >= 8 + 5); + std::unique_ptr* filter_content_to_corrupt = + TEST_arg_pair->first; + std::memset(filter_content_to_corrupt->get(), '\0', 8); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + s = Flush(); + + if (table_options.detect_filter_construct_corruption) { + EXPECT_TRUE(s.IsCorruption()); + EXPECT_TRUE(s.ToString().find("Corrupted filter content") != + std::string::npos); + } else { + EXPECT_TRUE(s.ok()); + } + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearCallBack( + "XXPH3FilterBitsBuilder::Finish::" + "TamperFilter"); +} + +// RocksDB lite does not support dynamic options +TEST_P(DBFilterConstructionCorruptionTestWithParam, + DynamicallyTurnOnAndOffDetectConstructCorruption) { + Options options = CurrentOptions(); + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + // We intend to turn on + // table_options.detect_filter_construct_corruption dynamically + // therefore we override this test parmater's value + table_options.detect_filter_construct_corruption = false; + + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.create_if_missing = true; + + int num_key = static_cast(GetNumKey()); + Status s; + + DestroyAndReopen(options); + + // Case 1: !table_options.detect_filter_construct_corruption + for (int i = 0; i < num_key; i++) { + ASSERT_OK(Put(Key(i), Key(i))); + } + + SyncPoint::GetInstance()->SetCallBack( + "XXPH3FilterBitsBuilder::Finish::TamperHashEntries", [&](void* arg) { + std::deque* hash_entries_to_corrupt = + (std::deque*)arg; + assert(!hash_entries_to_corrupt->empty()); + *(hash_entries_to_corrupt->begin()) = + *(hash_entries_to_corrupt->begin()) ^ uint64_t { 1 }; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + s = Flush(); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearCallBack( + "XXPH3FilterBitsBuilder::Finish::" + "TamperHashEntries"); + + ASSERT_FALSE(table_options.detect_filter_construct_corruption); + EXPECT_TRUE(s.ok()); + + // Case 2: dynamically turn on + // table_options.detect_filter_construct_corruption + ASSERT_OK(db_->SetOptions({{"block_based_table_factory", + "{detect_filter_construct_corruption=true;}"}})); + + for (int i = 0; i < num_key; i++) { + ASSERT_OK(Put(Key(i), Key(i))); + } + + SyncPoint::GetInstance()->SetCallBack( + "XXPH3FilterBitsBuilder::Finish::TamperHashEntries", [&](void* arg) { + std::deque* hash_entries_to_corrupt = + (std::deque*)arg; + assert(!hash_entries_to_corrupt->empty()); + *(hash_entries_to_corrupt->begin()) = + *(hash_entries_to_corrupt->begin()) ^ uint64_t { 1 }; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + s = Flush(); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearCallBack( + "XXPH3FilterBitsBuilder::Finish::" + "TamperHashEntries"); + + auto updated_table_options = + db_->GetOptions().table_factory->GetOptions(); + EXPECT_TRUE(updated_table_options->detect_filter_construct_corruption); + EXPECT_TRUE(s.IsCorruption()); + EXPECT_TRUE(s.ToString().find("Filter's hash entries checksum mismatched") != + std::string::npos); + + // Case 3: dynamically turn off + // table_options.detect_filter_construct_corruption + ASSERT_OK(db_->SetOptions({{"block_based_table_factory", + "{detect_filter_construct_corruption=false;}"}})); + updated_table_options = + db_->GetOptions().table_factory->GetOptions(); + EXPECT_FALSE(updated_table_options->detect_filter_construct_corruption); +} + +namespace { +// NOTE: This class is referenced by HISTORY.md as a model for a wrapper +// FilterPolicy selecting among configurations based on context. +class LevelAndStyleCustomFilterPolicy : public FilterPolicy { + public: + explicit LevelAndStyleCustomFilterPolicy(int bpk_fifo, int bpk_l0_other, + int bpk_otherwise) + : policy_fifo_(NewBloomFilterPolicy(bpk_fifo)), + policy_l0_other_(NewBloomFilterPolicy(bpk_l0_other)), + policy_otherwise_(NewBloomFilterPolicy(bpk_otherwise)) {} + + const char* Name() const override { + return "LevelAndStyleCustomFilterPolicy"; + } + + // OK to use built-in policy name because we are deferring to a + // built-in builder. We aren't changing the serialized format. + const char* CompatibilityName() const override { + return policy_fifo_->CompatibilityName(); + } + + FilterBitsBuilder* GetBuilderWithContext( + const FilterBuildingContext& context) const override { + if (context.compaction_style == kCompactionStyleFIFO) { + return policy_fifo_->GetBuilderWithContext(context); + } else if (context.level_at_creation == 0) { + return policy_l0_other_->GetBuilderWithContext(context); + } else { + return policy_otherwise_->GetBuilderWithContext(context); + } + } + + FilterBitsReader* GetFilterBitsReader(const Slice& contents) const override { + // OK to defer to any of them; they all can parse built-in filters + // from any settings. + return policy_fifo_->GetFilterBitsReader(contents); + } + + private: + const std::unique_ptr policy_fifo_; + const std::unique_ptr policy_l0_other_; + const std::unique_ptr policy_otherwise_; +}; + +static std::map + table_file_creation_reason_to_string{ + {TableFileCreationReason::kCompaction, "kCompaction"}, + {TableFileCreationReason::kFlush, "kFlush"}, + {TableFileCreationReason::kMisc, "kMisc"}, + {TableFileCreationReason::kRecovery, "kRecovery"}, + }; + +class TestingContextCustomFilterPolicy + : public LevelAndStyleCustomFilterPolicy { + public: + explicit TestingContextCustomFilterPolicy(int bpk_fifo, int bpk_l0_other, + int bpk_otherwise) + : LevelAndStyleCustomFilterPolicy(bpk_fifo, bpk_l0_other, bpk_otherwise) { + } + + FilterBitsBuilder* GetBuilderWithContext( + const FilterBuildingContext& context) const override { + test_report_ += "cf="; + test_report_ += context.column_family_name; + test_report_ += ",s="; + test_report_ += + OptionsHelper::compaction_style_to_string[context.compaction_style]; + test_report_ += ",n="; + test_report_ += std::to_string(context.num_levels); + test_report_ += ",l="; + test_report_ += std::to_string(context.level_at_creation); + test_report_ += ",b="; + test_report_ += std::to_string(int{context.is_bottommost}); + test_report_ += ",r="; + test_report_ += table_file_creation_reason_to_string[context.reason]; + test_report_ += "\n"; + + return LevelAndStyleCustomFilterPolicy::GetBuilderWithContext(context); + } + + std::string DumpTestReport() { + std::string rv; + std::swap(rv, test_report_); + return rv; + } + + private: + mutable std::string test_report_; +}; +} // anonymous namespace + +TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) { + auto policy = std::make_shared(15, 8, 5); + Options options; + for (bool fifo : {true, false}) { + options = CurrentOptions(); + options.max_open_files = fifo ? -1 : options.max_open_files; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.compaction_style = + fifo ? kCompactionStyleFIFO : kCompactionStyleLevel; + + BlockBasedTableOptions table_options; + table_options.filter_policy = policy; + table_options.format_version = 5; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + TryReopen(options); + CreateAndReopenWithCF({fifo ? "abe" : "bob"}, options); + + const int maxKey = 10000; + for (int i = 0; i < maxKey / 2; i++) { + ASSERT_OK(Put(1, Key(i), Key(i))); + } + // Add a large key to make the file contain wide range + ASSERT_OK(Put(1, Key(maxKey + 55555), Key(maxKey + 55555))); + Flush(1); + EXPECT_EQ(policy->DumpTestReport(), + fifo ? "cf=abe,s=kCompactionStyleFIFO,n=7,l=0,b=0,r=kFlush\n" + : "cf=bob,s=kCompactionStyleLevel,n=7,l=0,b=0,r=kFlush\n"); + + for (int i = maxKey / 2; i < maxKey; i++) { + ASSERT_OK(Put(1, Key(i), Key(i))); + } + Flush(1); + EXPECT_EQ(policy->DumpTestReport(), + fifo ? "cf=abe,s=kCompactionStyleFIFO,n=7,l=0,b=0,r=kFlush\n" + : "cf=bob,s=kCompactionStyleLevel,n=7,l=0,b=0,r=kFlush\n"); + + // Check that they can be found + for (int i = 0; i < maxKey; i++) { + ASSERT_EQ(Key(i), Get(1, Key(i))); + } + // Since we have two tables / two filters, we might have Bloom checks on + // our queries, but no more than one "useful" per query on a found key. + EXPECT_LE(PopTicker(options, BLOOM_FILTER_USEFUL), maxKey); + + // Check that we have two filters, each about + // fifo: 0.12% FP rate (15 bits per key) + // level: 2.3% FP rate (8 bits per key) + for (int i = 0; i < maxKey; i++) { + ASSERT_EQ("NOT_FOUND", Get(1, Key(i + 33333))); + } + { + auto useful_count = PopTicker(options, BLOOM_FILTER_USEFUL); + EXPECT_GE(useful_count, maxKey * 2 * (fifo ? 0.9980 : 0.975)); + EXPECT_LE(useful_count, maxKey * 2 * (fifo ? 0.9995 : 0.98)); + } + + if (!fifo) { // FIFO doesn't fully support CompactRange + // Full compaction + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), handles_[1], nullptr, + nullptr)); + EXPECT_EQ(policy->DumpTestReport(), + "cf=bob,s=kCompactionStyleLevel,n=7,l=1,b=1,r=kCompaction\n"); + + // Check that we now have one filter, about 9.2% FP rate (5 bits per key) + for (int i = 0; i < maxKey; i++) { + ASSERT_EQ("NOT_FOUND", Get(1, Key(i + 33333))); + } + { + auto useful_count = PopTicker(options, BLOOM_FILTER_USEFUL); + EXPECT_GE(useful_count, maxKey * 0.90); + EXPECT_LE(useful_count, maxKey * 0.91); + } + } else { + // Also try external SST file + { + std::string file_path = dbname_ + "/external.sst"; + SstFileWriter sst_file_writer(EnvOptions(), options, handles_[1]); + ASSERT_OK(sst_file_writer.Open(file_path)); + ASSERT_OK(sst_file_writer.Put("key", "value")); + ASSERT_OK(sst_file_writer.Finish()); + } + // Note: kCompactionStyleLevel is default, ignored if num_levels == -1 + EXPECT_EQ(policy->DumpTestReport(), + "cf=abe,s=kCompactionStyleLevel,n=-1,l=-1,b=0,r=kMisc\n"); + } + + // Destroy + ASSERT_OK(dbfull()->DropColumnFamily(handles_[1])); + ASSERT_OK(dbfull()->DestroyColumnFamilyHandle(handles_[1])); + handles_[1] = nullptr; + } +} + +class SliceTransformLimitedDomain : public SliceTransform { + const char* Name() const override { return "SliceTransformLimitedDomain"; } + + Slice Transform(const Slice& src) const override { + return Slice(src.data(), 5); + } + + bool InDomain(const Slice& src) const override { + // prefix will be x???? + return src.size() >= 5 && src[0] == 'x'; + } + + bool InRange(const Slice& dst) const override { + // prefix will be x???? + return dst.size() == 5 && dst[0] == 'x'; + } +}; + +TEST_F(DBBloomFilterTest, PrefixExtractorWithFilter1) { + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10)); + bbto.whole_key_filtering = false; + + Options options = CurrentOptions(); + options.prefix_extractor = std::make_shared(); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + + DestroyAndReopen(options); + + ASSERT_OK(Put("x1111_AAAA", "val1")); + ASSERT_OK(Put("x1112_AAAA", "val2")); + ASSERT_OK(Put("x1113_AAAA", "val3")); + ASSERT_OK(Put("x1114_AAAA", "val4")); + // Not in domain, wont be added to filter + ASSERT_OK(Put("zzzzz_AAAA", "val5")); + + ASSERT_OK(Flush()); + + ASSERT_EQ(Get("x1111_AAAA"), "val1"); + ASSERT_EQ(Get("x1112_AAAA"), "val2"); + ASSERT_EQ(Get("x1113_AAAA"), "val3"); + ASSERT_EQ(Get("x1114_AAAA"), "val4"); + // Was not added to filter but rocksdb will try to read it from the filter + ASSERT_EQ(Get("zzzzz_AAAA"), "val5"); +} + +TEST_F(DBBloomFilterTest, PrefixExtractorWithFilter2) { + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10)); + + Options options = CurrentOptions(); + options.prefix_extractor = std::make_shared(); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + + DestroyAndReopen(options); + + ASSERT_OK(Put("x1113_AAAA", "val3")); + ASSERT_OK(Put("x1114_AAAA", "val4")); + // Not in domain, wont be added to filter + ASSERT_OK(Put("zzzzz_AAAA", "val1")); + ASSERT_OK(Put("zzzzz_AAAB", "val2")); + ASSERT_OK(Put("zzzzz_AAAC", "val3")); + ASSERT_OK(Put("zzzzz_AAAD", "val4")); + + ASSERT_OK(Flush()); + + std::vector iter_res; + auto iter = db_->NewIterator(ReadOptions()); + // Seek to a key that was not in Domain + for (iter->Seek("zzzzz_AAAA"); iter->Valid(); iter->Next()) { + iter_res.emplace_back(iter->value().ToString()); + } + + std::vector expected_res = {"val1", "val2", "val3", "val4"}; + ASSERT_EQ(iter_res, expected_res); + delete iter; +} + +TEST_F(DBBloomFilterTest, MemtableWholeKeyBloomFilter) { + // regression test for #2743. the range delete tombstones in memtable should + // be added even when Get() skips searching due to its prefix bloom filter + const int kMemtableSize = 1 << 20; // 1MB + const int kMemtablePrefixFilterSize = 1 << 13; // 8KB + const int kPrefixLen = 4; + Options options = CurrentOptions(); + options.memtable_prefix_bloom_size_ratio = + static_cast(kMemtablePrefixFilterSize) / kMemtableSize; + options.prefix_extractor.reset( + ROCKSDB_NAMESPACE::NewFixedPrefixTransform(kPrefixLen)); + options.write_buffer_size = kMemtableSize; + options.memtable_whole_key_filtering = false; + Reopen(options); + std::string key1("AAAABBBB"); + std::string key2("AAAACCCC"); // not in DB + std::string key3("AAAADDDD"); + std::string key4("AAAAEEEE"); + std::string value1("Value1"); + std::string value3("Value3"); + std::string value4("Value4"); + + ASSERT_OK(Put(key1, value1, WriteOptions())); + + // check memtable bloom stats + ASSERT_EQ("NOT_FOUND", Get(key2)); + ASSERT_EQ(0, get_perf_context()->bloom_memtable_miss_count); + // same prefix, bloom filter false positive + ASSERT_EQ(1, get_perf_context()->bloom_memtable_hit_count); + + // enable whole key bloom filter + options.memtable_whole_key_filtering = true; + Reopen(options); + // check memtable bloom stats + ASSERT_OK(Put(key3, value3, WriteOptions())); + ASSERT_EQ("NOT_FOUND", Get(key2)); + // whole key bloom filter kicks in and determines it's a miss + ASSERT_EQ(1, get_perf_context()->bloom_memtable_miss_count); + ASSERT_EQ(1, get_perf_context()->bloom_memtable_hit_count); + + // verify whole key filtering does not depend on prefix_extractor + options.prefix_extractor.reset(); + Reopen(options); + // check memtable bloom stats + ASSERT_OK(Put(key4, value4, WriteOptions())); + ASSERT_EQ("NOT_FOUND", Get(key2)); + // whole key bloom filter kicks in and determines it's a miss + ASSERT_EQ(2, get_perf_context()->bloom_memtable_miss_count); + ASSERT_EQ(1, get_perf_context()->bloom_memtable_hit_count); +} + +TEST_F(DBBloomFilterTest, MemtableWholeKeyBloomFilterMultiGet) { + Options options = CurrentOptions(); + options.memtable_prefix_bloom_size_ratio = 0.015; + options.memtable_whole_key_filtering = true; + Reopen(options); + std::string key1("AA"); + std::string key2("BB"); + std::string key3("CC"); + std::string key4("DD"); + std::string key_not("EE"); + std::string value1("Value1"); + std::string value2("Value2"); + std::string value3("Value3"); + std::string value4("Value4"); + + ASSERT_OK(Put(key1, value1, WriteOptions())); + ASSERT_OK(Put(key2, value2, WriteOptions())); + ASSERT_OK(Flush()); + ASSERT_OK(Put(key3, value3, WriteOptions())); + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK(Put(key4, value4, WriteOptions())); + + // Delete key2 and key3 + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "BA", "CZ")); + + // Read without snapshot + auto results = MultiGet({key_not, key1, key2, key3, key4}); + ASSERT_EQ(results[0], "NOT_FOUND"); + ASSERT_EQ(results[1], value1); + ASSERT_EQ(results[2], "NOT_FOUND"); + ASSERT_EQ(results[3], "NOT_FOUND"); + ASSERT_EQ(results[4], value4); + + // Also check Get + ASSERT_EQ(Get(key1), value1); + ASSERT_EQ(Get(key2), "NOT_FOUND"); + ASSERT_EQ(Get(key3), "NOT_FOUND"); + ASSERT_EQ(Get(key4), value4); + + // Read with snapshot + results = MultiGet({key_not, key1, key2, key3, key4}, snapshot); + ASSERT_EQ(results[0], "NOT_FOUND"); + ASSERT_EQ(results[1], value1); + ASSERT_EQ(results[2], value2); + ASSERT_EQ(results[3], value3); + ASSERT_EQ(results[4], "NOT_FOUND"); + + // Also check Get + ASSERT_EQ(Get(key1, snapshot), value1); + ASSERT_EQ(Get(key2, snapshot), value2); + ASSERT_EQ(Get(key3, snapshot), value3); + ASSERT_EQ(Get(key4, snapshot), "NOT_FOUND"); + + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(DBBloomFilterTest, MemtablePrefixBloomOutOfDomain) { + constexpr size_t kPrefixSize = 8; + const std::string kKey = "key"; + assert(kKey.size() < kPrefixSize); + Options options = CurrentOptions(); + options.prefix_extractor.reset(NewFixedPrefixTransform(kPrefixSize)); + options.memtable_prefix_bloom_size_ratio = 0.25; + Reopen(options); + ASSERT_OK(Put(kKey, "v")); + ASSERT_EQ("v", Get(kKey)); + std::unique_ptr iter(dbfull()->NewIterator(ReadOptions())); + iter->Seek(kKey); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(kKey, iter->key()); + iter->SeekForPrev(kKey); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(kKey, iter->key()); +} + +class DBBloomFilterTestVaryPrefixAndFormatVer + : public DBTestBase, + public testing::WithParamInterface> { + protected: + bool use_prefix_; + uint32_t format_version_; + + public: + DBBloomFilterTestVaryPrefixAndFormatVer() + : DBTestBase("db_bloom_filter_tests", /*env_do_fsync=*/true) {} + + ~DBBloomFilterTestVaryPrefixAndFormatVer() override {} + + void SetUp() override { + use_prefix_ = std::get<0>(GetParam()); + format_version_ = std::get<1>(GetParam()); + } + + static std::string UKey(uint32_t i) { return Key(static_cast(i)); } +}; + +TEST_P(DBBloomFilterTestVaryPrefixAndFormatVer, PartitionedMultiGet) { + Options options = CurrentOptions(); + if (use_prefix_) { + // Entire key from UKey() + options.prefix_extractor.reset(NewCappedPrefixTransform(9)); + } + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(NewBloomFilterPolicy(20)); + bbto.partition_filters = true; + bbto.index_type = BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + bbto.whole_key_filtering = !use_prefix_; + if (use_prefix_) { // (not related to prefix, just alternating between) + // Make sure code appropriately deals with metadata block size setting + // that is "too small" (smaller than minimum size for filter builder) + bbto.metadata_block_size = 63; + } else { + // Make sure the test will work even on platforms with large minimum + // filter size, due to large cache line size. + // (Largest cache line size + 10+% overhead.) + bbto.metadata_block_size = 290; + } + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + DestroyAndReopen(options); + ReadOptions ropts; + + constexpr uint32_t N = 12000; + // Add N/2 evens + for (uint32_t i = 0; i < N; i += 2) { + ASSERT_OK(Put(UKey(i), UKey(i))); + } + ASSERT_OK(Flush()); + ASSERT_EQ(TotalTableFiles(), 1); + + constexpr uint32_t Q = 29; + // MultiGet In + std::array keys; + std::array key_slices; + std::array column_families; + // MultiGet Out + std::array statuses; + std::array values; + + PopTicker(options, BLOCK_CACHE_FILTER_HIT); + PopTicker(options, BLOCK_CACHE_FILTER_MISS); + PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL); + PopTicker(options, BLOOM_FILTER_USEFUL); + PopTicker(options, BLOOM_FILTER_PREFIX_CHECKED); + PopTicker(options, BLOOM_FILTER_FULL_POSITIVE); + PopTicker(options, BLOOM_FILTER_FULL_TRUE_POSITIVE); + PopTicker(options, BLOOM_FILTER_PREFIX_TRUE_POSITIVE); + + // Check that initial clump of keys only loads one partition filter from + // block cache. + // And that spread out keys load many partition filters. + // In both cases, mix present vs. not present keys. + for (uint32_t stride : {uint32_t{1}, (N / Q) | 1}) { + for (uint32_t i = 0; i < Q; ++i) { + keys[i] = UKey(i * stride); + key_slices[i] = Slice(keys[i]); + column_families[i] = db_->DefaultColumnFamily(); + statuses[i] = Status(); + values[i] = PinnableSlice(); + } + + db_->MultiGet(ropts, Q, &column_families[0], &key_slices[0], &values[0], + /*timestamps=*/nullptr, &statuses[0], true); + + // Confirm correct status results + uint32_t number_not_found = 0; + for (uint32_t i = 0; i < Q; ++i) { + if ((i * stride % 2) == 0) { + ASSERT_OK(statuses[i]); + } else { + ASSERT_TRUE(statuses[i].IsNotFound()); + ++number_not_found; + } + } + + // Confirm correct Bloom stats (no FPs) + uint64_t filter_useful = + PopTicker(options, use_prefix_ ? BLOOM_FILTER_PREFIX_USEFUL + : BLOOM_FILTER_USEFUL); + uint64_t filter_checked = + PopTicker(options, use_prefix_ ? BLOOM_FILTER_PREFIX_CHECKED + : BLOOM_FILTER_FULL_POSITIVE) + + (use_prefix_ ? 0 : filter_useful); + EXPECT_EQ(filter_useful, number_not_found); + EXPECT_EQ(filter_checked, Q); + EXPECT_EQ(PopTicker(options, use_prefix_ ? BLOOM_FILTER_PREFIX_TRUE_POSITIVE + : BLOOM_FILTER_FULL_TRUE_POSITIVE), + Q - number_not_found); + + // Confirm no duplicate loading same filter partition + uint64_t filter_accesses = PopTicker(options, BLOCK_CACHE_FILTER_HIT) + + PopTicker(options, BLOCK_CACHE_FILTER_MISS); + if (stride == 1) { + EXPECT_EQ(filter_accesses, 1); + } else { + // for large stride + EXPECT_GE(filter_accesses, Q / 2 + 1); + } + } + + // Check that a clump of keys (present and not) works when spanning + // two partitions + int found_spanning = 0; + for (uint32_t start = 0; start < N / 2;) { + for (uint32_t i = 0; i < Q; ++i) { + keys[i] = UKey(start + i); + key_slices[i] = Slice(keys[i]); + column_families[i] = db_->DefaultColumnFamily(); + statuses[i] = Status(); + values[i] = PinnableSlice(); + } + + db_->MultiGet(ropts, Q, &column_families[0], &key_slices[0], &values[0], + /*timestamps=*/nullptr, &statuses[0], true); + + // Confirm correct status results + uint32_t number_not_found = 0; + for (uint32_t i = 0; i < Q; ++i) { + if (((start + i) % 2) == 0) { + ASSERT_OK(statuses[i]); + } else { + ASSERT_TRUE(statuses[i].IsNotFound()); + ++number_not_found; + } + } + + // Confirm correct Bloom stats (might see some FPs) + uint64_t filter_useful = + PopTicker(options, use_prefix_ ? BLOOM_FILTER_PREFIX_USEFUL + : BLOOM_FILTER_USEFUL); + uint64_t filter_checked = + PopTicker(options, use_prefix_ ? BLOOM_FILTER_PREFIX_CHECKED + : BLOOM_FILTER_FULL_POSITIVE) + + (use_prefix_ ? 0 : filter_useful); + EXPECT_GE(filter_useful, number_not_found - 2); // possible FP + EXPECT_EQ(filter_checked, Q); + EXPECT_EQ(PopTicker(options, use_prefix_ ? BLOOM_FILTER_PREFIX_TRUE_POSITIVE + : BLOOM_FILTER_FULL_TRUE_POSITIVE), + Q - number_not_found); + + // Confirm no duplicate loading of same filter partition + uint64_t filter_accesses = PopTicker(options, BLOCK_CACHE_FILTER_HIT) + + PopTicker(options, BLOCK_CACHE_FILTER_MISS); + if (filter_accesses == 2) { + // Spanned across partitions. + ++found_spanning; + if (found_spanning >= 2) { + break; + } else { + // Ensure that at least once we have at least one present and + // one non-present key on both sides of partition boundary. + start += 2; + } + } else { + EXPECT_EQ(filter_accesses, 1); + // See explanation at "start += 2" + start += Q - 4; + } + } + EXPECT_TRUE(found_spanning >= 2); +} + +INSTANTIATE_TEST_CASE_P(DBBloomFilterTestVaryPrefixAndFormatVer, + DBBloomFilterTestVaryPrefixAndFormatVer, + ::testing::Values( + // (use_prefix, format_version) + std::make_tuple(false, 2), + std::make_tuple(false, 3), + std::make_tuple(false, 4), + std::make_tuple(false, 5), std::make_tuple(true, 2), + std::make_tuple(true, 3), std::make_tuple(true, 4), + std::make_tuple(true, 5))); + +namespace { +static const std::string kPlainTable = "test_PlainTableBloom"; +} // anonymous namespace + +class BloomStatsTestWithParam + : public DBBloomFilterTest, + public testing::WithParamInterface> { + public: + BloomStatsTestWithParam() { + bfp_impl_ = std::get<0>(GetParam()); + partition_filters_ = std::get<1>(GetParam()); + + options_.create_if_missing = true; + options_.prefix_extractor.reset( + ROCKSDB_NAMESPACE::NewFixedPrefixTransform(4)); + options_.memtable_prefix_bloom_size_ratio = + 8.0 * 1024.0 / static_cast(options_.write_buffer_size); + if (bfp_impl_ == kPlainTable) { + assert(!partition_filters_); // not supported in plain table + PlainTableOptions table_options; + options_.table_factory.reset(NewPlainTableFactory(table_options)); + } else { + BlockBasedTableOptions table_options; + if (partition_filters_) { + table_options.partition_filters = partition_filters_; + table_options.index_type = + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + } + table_options.filter_policy = Create(10, bfp_impl_); + options_.table_factory.reset(NewBlockBasedTableFactory(table_options)); + } + options_.env = env_; + + get_perf_context()->Reset(); + DestroyAndReopen(options_); + } + + ~BloomStatsTestWithParam() override { + get_perf_context()->Reset(); + Destroy(options_); + } + + // Required if inheriting from testing::WithParamInterface<> + static void SetUpTestCase() {} + static void TearDownTestCase() {} + + std::string bfp_impl_; + bool partition_filters_; + Options options_; +}; + +// 1 Insert 2 K-V pairs into DB +// 2 Call Get() for both keys - expext memtable bloom hit stat to be 2 +// 3 Call Get() for nonexisting key - expect memtable bloom miss stat to be 1 +// 4 Call Flush() to create SST +// 5 Call Get() for both keys - expext SST bloom hit stat to be 2 +// 6 Call Get() for nonexisting key - expect SST bloom miss stat to be 1 +// Test both: block and plain SST +TEST_P(BloomStatsTestWithParam, BloomStatsTest) { + std::string key1("AAAA"); + std::string key2("RXDB"); // not in DB + std::string key3("ZBRA"); + std::string value1("Value1"); + std::string value3("Value3"); + + ASSERT_OK(Put(key1, value1, WriteOptions())); + ASSERT_OK(Put(key3, value3, WriteOptions())); + + // check memtable bloom stats + ASSERT_EQ(value1, Get(key1)); + ASSERT_EQ(1, get_perf_context()->bloom_memtable_hit_count); + ASSERT_EQ(value3, Get(key3)); + ASSERT_EQ(2, get_perf_context()->bloom_memtable_hit_count); + ASSERT_EQ(0, get_perf_context()->bloom_memtable_miss_count); + + ASSERT_EQ("NOT_FOUND", Get(key2)); + ASSERT_EQ(1, get_perf_context()->bloom_memtable_miss_count); + ASSERT_EQ(2, get_perf_context()->bloom_memtable_hit_count); + + // sanity checks + ASSERT_EQ(0, get_perf_context()->bloom_sst_hit_count); + ASSERT_EQ(0, get_perf_context()->bloom_sst_miss_count); + + Flush(); + + // sanity checks + ASSERT_EQ(0, get_perf_context()->bloom_sst_hit_count); + ASSERT_EQ(0, get_perf_context()->bloom_sst_miss_count); + + // check SST bloom stats + ASSERT_EQ(value1, Get(key1)); + ASSERT_EQ(1, get_perf_context()->bloom_sst_hit_count); + ASSERT_EQ(value3, Get(key3)); + ASSERT_EQ(2, get_perf_context()->bloom_sst_hit_count); + + ASSERT_EQ("NOT_FOUND", Get(key2)); + ASSERT_EQ(1, get_perf_context()->bloom_sst_miss_count); +} + +// Same scenario as in BloomStatsTest but using an iterator +TEST_P(BloomStatsTestWithParam, BloomStatsTestWithIter) { + std::string key1("AAAA"); + std::string key2("RXDB"); // not in DB + std::string key3("ZBRA"); + std::string value1("Value1"); + std::string value3("Value3"); + + ASSERT_OK(Put(key1, value1, WriteOptions())); + ASSERT_OK(Put(key3, value3, WriteOptions())); + + std::unique_ptr iter(dbfull()->NewIterator(ReadOptions())); + + // check memtable bloom stats + iter->Seek(key1); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(value1, iter->value().ToString()); + ASSERT_EQ(1, get_perf_context()->bloom_memtable_hit_count); + ASSERT_EQ(0, get_perf_context()->bloom_memtable_miss_count); + + iter->Seek(key3); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(value3, iter->value().ToString()); + ASSERT_EQ(2, get_perf_context()->bloom_memtable_hit_count); + ASSERT_EQ(0, get_perf_context()->bloom_memtable_miss_count); + + iter->Seek(key2); + ASSERT_OK(iter->status()); + ASSERT_TRUE(!iter->Valid()); + ASSERT_EQ(1, get_perf_context()->bloom_memtable_miss_count); + ASSERT_EQ(2, get_perf_context()->bloom_memtable_hit_count); + + Flush(); + + iter.reset(dbfull()->NewIterator(ReadOptions())); + + // Check SST bloom stats + iter->Seek(key1); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(value1, iter->value().ToString()); + ASSERT_EQ(1, get_perf_context()->bloom_sst_hit_count); + + iter->Seek(key3); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(value3, iter->value().ToString()); + uint64_t expected_hits = 2; + ASSERT_EQ(expected_hits, get_perf_context()->bloom_sst_hit_count); + + iter->Seek(key2); + ASSERT_OK(iter->status()); + ASSERT_TRUE(!iter->Valid()); + ASSERT_EQ(1, get_perf_context()->bloom_sst_miss_count); + ASSERT_EQ(expected_hits, get_perf_context()->bloom_sst_hit_count); +} + +INSTANTIATE_TEST_CASE_P( + BloomStatsTestWithParam, BloomStatsTestWithParam, + ::testing::Values(std::make_tuple(kLegacyBloom, false), + std::make_tuple(kLegacyBloom, true), + std::make_tuple(kFastLocalBloom, false), + std::make_tuple(kFastLocalBloom, true), + std::make_tuple(kPlainTable, false))); + +namespace { +void PrefixScanInit(DBBloomFilterTest* dbtest) { + char buf[100]; + std::string keystr; + const int small_range_sstfiles = 5; + const int big_range_sstfiles = 5; + + // Generate 11 sst files with the following prefix ranges. + // GROUP 0: [0,10] (level 1) + // GROUP 1: [1,2], [2,3], [3,4], [4,5], [5, 6] (level 0) + // GROUP 2: [0,6], [0,7], [0,8], [0,9], [0,10] (level 0) + // + // A seek with the previous API would do 11 random I/Os (to all the + // files). With the new API and a prefix filter enabled, we should + // only do 2 random I/O, to the 2 files containing the key. + + // GROUP 0 + snprintf(buf, sizeof(buf), "%02d______:start", 0); + keystr = std::string(buf); + ASSERT_OK(dbtest->Put(keystr, keystr)); + snprintf(buf, sizeof(buf), "%02d______:end", 10); + keystr = std::string(buf); + ASSERT_OK(dbtest->Put(keystr, keystr)); + ASSERT_OK(dbtest->Flush()); + ASSERT_OK(dbtest->dbfull()->CompactRange(CompactRangeOptions(), nullptr, + nullptr)); // move to level 1 + + // GROUP 1 + for (int i = 1; i <= small_range_sstfiles; i++) { + snprintf(buf, sizeof(buf), "%02d______:start", i); + keystr = std::string(buf); + ASSERT_OK(dbtest->Put(keystr, keystr)); + snprintf(buf, sizeof(buf), "%02d______:end", i + 1); + keystr = std::string(buf); + ASSERT_OK(dbtest->Put(keystr, keystr)); + dbtest->Flush(); + } + + // GROUP 2 + for (int i = 1; i <= big_range_sstfiles; i++) { + snprintf(buf, sizeof(buf), "%02d______:start", 0); + keystr = std::string(buf); + ASSERT_OK(dbtest->Put(keystr, keystr)); + snprintf(buf, sizeof(buf), "%02d______:end", small_range_sstfiles + i + 1); + keystr = std::string(buf); + ASSERT_OK(dbtest->Put(keystr, keystr)); + dbtest->Flush(); + } +} +} // anonymous namespace + +TEST_F(DBBloomFilterTest, PrefixScan) { + while (ChangeFilterOptions()) { + int count; + Slice prefix; + Slice key; + char buf[100]; + Iterator* iter; + snprintf(buf, sizeof(buf), "03______:"); + prefix = Slice(buf, 8); + key = Slice(buf, 9); + ASSERT_EQ(key.difference_offset(prefix), 8); + ASSERT_EQ(prefix.difference_offset(key), 8); + // db configs + env_->count_random_reads_ = true; + Options options = CurrentOptions(); + options.env = env_; + options.prefix_extractor.reset(NewFixedPrefixTransform(8)); + options.disable_auto_compactions = true; + options.max_background_compactions = 2; + options.create_if_missing = true; + options.memtable_factory.reset(NewHashSkipListRepFactory(16)); + assert(!options.unordered_write); + // It is incompatible with allow_concurrent_memtable_write=false + options.allow_concurrent_memtable_write = false; + + BlockBasedTableOptions table_options; + table_options.no_block_cache = true; + table_options.filter_policy.reset(NewBloomFilterPolicy(10)); + table_options.whole_key_filtering = false; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + // 11 RAND I/Os + DestroyAndReopen(options); + PrefixScanInit(this); + count = 0; + env_->random_read_counter_.Reset(); + iter = db_->NewIterator(ReadOptions()); + for (iter->Seek(prefix); iter->Valid(); iter->Next()) { + if (!iter->key().starts_with(prefix)) { + break; + } + count++; + } + ASSERT_OK(iter->status()); + delete iter; + ASSERT_EQ(count, 2); + ASSERT_EQ(env_->random_read_counter_.Read(), 2); + Close(); + } // end of while +} + +TEST_F(DBBloomFilterTest, OptimizeFiltersForHits) { + const int kNumKeysPerFlush = 1000; + + Options options = CurrentOptions(); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFlush)); + options.target_file_size_base = 64 * 1024; + options.level0_file_num_compaction_trigger = 2; + options.level0_slowdown_writes_trigger = 2; + options.level0_stop_writes_trigger = 4; + options.max_bytes_for_level_base = 256 * 1024; + options.max_write_buffer_number = 2; + options.max_background_compactions = 8; + options.max_background_flushes = 8; + options.compression = kNoCompression; + options.compaction_style = kCompactionStyleLevel; + options.level_compaction_dynamic_level_bytes = true; + BlockBasedTableOptions bbto; + bbto.cache_index_and_filter_blocks = true; + bbto.filter_policy.reset(NewBloomFilterPolicy(10)); + bbto.whole_key_filtering = true; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + options.optimize_filters_for_hits = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + get_perf_context()->Reset(); + get_perf_context()->EnablePerLevelPerfContext(); + CreateAndReopenWithCF({"mypikachu"}, options); + + int numkeys = 200000; + + // Generate randomly shuffled keys, so the updates are almost + // random. + std::vector keys; + keys.reserve(numkeys); + for (int i = 0; i < numkeys; i += 2) { + keys.push_back(i); + } + RandomShuffle(std::begin(keys), std::end(keys), /*seed*/ 42); + int num_inserted = 0; + for (int key : keys) { + ASSERT_OK(Put(1, Key(key), "val")); + num_inserted++; + // The write after each `kNumKeysPerFlush` keys triggers a flush. Always + // wait for that flush and any follow-on compactions for deterministic LSM + // shape. + if (num_inserted > kNumKeysPerFlush && + num_inserted % kNumKeysPerFlush == 1) { + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + } + ASSERT_OK(Put(1, Key(0), "val")); + ASSERT_OK(Put(1, Key(numkeys), "val")); + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + if (NumTableFilesAtLevel(0, 1) == 0) { + // No Level 0 file. Create one. + ASSERT_OK(Put(1, Key(0), "val")); + ASSERT_OK(Put(1, Key(numkeys), "val")); + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + + for (int i = 1; i < numkeys; i += 2) { + ASSERT_EQ(Get(1, Key(i)), "NOT_FOUND"); + } + + ASSERT_EQ(0, TestGetTickerCount(options, GET_HIT_L0)); + ASSERT_EQ(0, TestGetTickerCount(options, GET_HIT_L1)); + ASSERT_EQ(0, TestGetTickerCount(options, GET_HIT_L2_AND_UP)); + + // Now we have three sorted run, L0, L5 and L6 with most files in L6 have + // no bloom filter. Most keys be checked bloom filters twice. + ASSERT_GT(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 65000 * 2); + ASSERT_LT(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 120000 * 2); + uint64_t bloom_filter_useful_all_levels = 0; + for (auto& kv : (*(get_perf_context()->level_to_perf_context))) { + if (kv.second.bloom_filter_useful > 0) { + bloom_filter_useful_all_levels += kv.second.bloom_filter_useful; + } + } + ASSERT_GT(bloom_filter_useful_all_levels, 65000 * 2); + ASSERT_LT(bloom_filter_useful_all_levels, 120000 * 2); + + for (int i = 0; i < numkeys; i += 2) { + ASSERT_EQ(Get(1, Key(i)), "val"); + } + + // Part 2 (read path): rewrite last level with blooms, then verify they get + // cached only if !optimize_filters_for_hits + options.disable_auto_compactions = true; + options.num_levels = 9; + options.optimize_filters_for_hits = false; + options.statistics = CreateDBStatistics(); + bbto.block_cache.reset(); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + + ReopenWithColumnFamilies({"default", "mypikachu"}, options); + MoveFilesToLevel(7 /* level */, 1 /* column family index */); + + std::string value = Get(1, Key(0)); + uint64_t prev_cache_filter_hits = + TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT); + value = Get(1, Key(0)); + ASSERT_EQ(prev_cache_filter_hits + 1, + TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT)); + + // Now that we know the filter blocks exist in the last level files, see if + // filter caching is skipped for this optimization + options.optimize_filters_for_hits = true; + options.statistics = CreateDBStatistics(); + bbto.block_cache.reset(); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + + ReopenWithColumnFamilies({"default", "mypikachu"}, options); + + value = Get(1, Key(0)); + ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT)); + ASSERT_EQ(2 /* index and data block */, + TestGetTickerCount(options, BLOCK_CACHE_ADD)); + + // Check filter block ignored for files preloaded during DB::Open() + options.max_open_files = -1; + options.statistics = CreateDBStatistics(); + bbto.block_cache.reset(); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + + ReopenWithColumnFamilies({"default", "mypikachu"}, options); + + uint64_t prev_cache_filter_misses = + TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS); + prev_cache_filter_hits = TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT); + Get(1, Key(0)); + ASSERT_EQ(prev_cache_filter_misses, + TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(prev_cache_filter_hits, + TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT)); + + // Check filter block ignored for file trivially-moved to bottom level + bbto.block_cache.reset(); + options.max_open_files = 100; // setting > -1 makes it not preload all files + options.statistics = CreateDBStatistics(); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + + ReopenWithColumnFamilies({"default", "mypikachu"}, options); + + ASSERT_OK(Put(1, Key(numkeys + 1), "val")); + ASSERT_OK(Flush(1)); + + int32_t trivial_move = 0; + int32_t non_trivial_move = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:TrivialMove", + [&](void* /*arg*/) { trivial_move++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial", + [&](void* /*arg*/) { non_trivial_move++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + CompactRangeOptions compact_options; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kSkip; + compact_options.change_level = true; + compact_options.target_level = 7; + ASSERT_OK(db_->CompactRange(compact_options, handles_[1], nullptr, nullptr)); + + ASSERT_GE(trivial_move, 1); + ASSERT_EQ(non_trivial_move, 0); + + prev_cache_filter_hits = TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT); + prev_cache_filter_misses = + TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS); + value = Get(1, Key(numkeys + 1)); + ASSERT_EQ(prev_cache_filter_hits, + TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT)); + ASSERT_EQ(prev_cache_filter_misses, + TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + + // Check filter block not cached for iterator + bbto.block_cache.reset(); + options.statistics = CreateDBStatistics(); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + + ReopenWithColumnFamilies({"default", "mypikachu"}, options); + + std::unique_ptr iter(db_->NewIterator(ReadOptions(), handles_[1])); + iter->SeekToFirst(); + ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT)); + ASSERT_EQ(2 /* index and data block */, + TestGetTickerCount(options, BLOCK_CACHE_ADD)); + get_perf_context()->Reset(); +} + +int CountIter(std::unique_ptr& iter, const Slice& key) { + int count = 0; + for (iter->Seek(key); iter->Valid(); iter->Next()) { + count++; + // Access key & value as if we were using them + (void)iter->key(); + (void)iter->value(); + } + EXPECT_OK(iter->status()); + return count; +} + +// use iterate_upper_bound to hint compatiability of existing bloom filters. +// The BF is considered compatible if 1) upper bound and seek key transform +// into the same string, or 2) the transformed seek key is of the same length +// as the upper bound and two keys are adjacent according to the comparator. +TEST_F(DBBloomFilterTest, DynamicBloomFilterUpperBound) { + for (const auto& bfp_impl : BloomLikeFilterPolicy::GetAllFixedImpls()) { + Options options; + options.create_if_missing = true; + options.env = CurrentOptions().env; + options.prefix_extractor.reset(NewCappedPrefixTransform(4)); + options.disable_auto_compactions = true; + options.statistics = CreateDBStatistics(); + // Enable prefix bloom for SST files + BlockBasedTableOptions table_options; + table_options.cache_index_and_filter_blocks = true; + table_options.filter_policy = Create(10, bfp_impl); + table_options.index_shortening = BlockBasedTableOptions:: + IndexShorteningMode::kShortenSeparatorsAndSuccessor; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DestroyAndReopen(options); + + ASSERT_OK(Put("abcdxxx0", "val1")); + ASSERT_OK(Put("abcdxxx1", "val2")); + ASSERT_OK(Put("abcdxxx2", "val3")); + ASSERT_OK(Put("abcdxxx3", "val4")); + ASSERT_OK(dbfull()->Flush(FlushOptions())); + { + // prefix_extractor has not changed, BF will always be read + Slice upper_bound("abce"); + ReadOptions read_options; + read_options.prefix_same_as_start = true; + read_options.iterate_upper_bound = &upper_bound; + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_EQ(CountIter(iter, "abcd0000"), 4); + ASSERT_EQ(TestGetTickerCount(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), + 1); + ASSERT_EQ(TestGetTickerCount(options, NON_LAST_LEVEL_SEEK_FILTERED), 0); + ASSERT_EQ(TestGetTickerCount( + options, NON_LAST_LEVEL_SEEK_DATA_USEFUL_FILTER_MATCH), + 1); + } + { + Slice upper_bound("abcdzzzz"); + ReadOptions read_options; + read_options.prefix_same_as_start = true; + read_options.iterate_upper_bound = &upper_bound; + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_EQ(CountIter(iter, "abcd0000"), 4); + ASSERT_EQ(TestGetTickerCount(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), + 2); + ASSERT_EQ(TestGetTickerCount(options, NON_LAST_LEVEL_SEEK_FILTERED), 0); + } + ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "fixed:5"}})); + ASSERT_EQ(dbfull()->GetOptions().prefix_extractor->AsString(), + "rocksdb.FixedPrefix.5"); + { + // BF changed, [abcdxx00, abce) is a valid bound, will trigger BF read + Slice upper_bound("abce"); + ReadOptions read_options; + read_options.prefix_same_as_start = true; + read_options.iterate_upper_bound = &upper_bound; + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_EQ(CountIter(iter, "abcdxx00"), 4); + // should check bloom filter since upper bound meets requirement + ASSERT_EQ(TestGetTickerCount(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), + 3); + ASSERT_EQ(TestGetTickerCount(options, NON_LAST_LEVEL_SEEK_FILTERED), 0); + } + { + // [abcdxx01, abcey) is not valid bound since upper bound is too long for + // the BF in SST (capped:4) + Slice upper_bound("abcey"); + ReadOptions read_options; + read_options.prefix_same_as_start = true; + read_options.iterate_upper_bound = &upper_bound; + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_EQ(CountIter(iter, "abcdxx01"), 4); + // should skip bloom filter since upper bound is too long + ASSERT_EQ(TestGetTickerCount(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), + 3); + ASSERT_EQ(TestGetTickerCount(options, NON_LAST_LEVEL_SEEK_FILTERED), 0); + } + { + // [abcdxx02, abcdy) is a valid bound since the prefix is the same + Slice upper_bound("abcdy"); + ReadOptions read_options; + read_options.prefix_same_as_start = true; + read_options.iterate_upper_bound = &upper_bound; + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_EQ(CountIter(iter, "abcdxx02"), 4); + // should check bloom filter since upper bound matches transformed seek + // key + ASSERT_EQ(TestGetTickerCount(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), + 4); + ASSERT_EQ(TestGetTickerCount(options, NON_LAST_LEVEL_SEEK_FILTERED), 0); + } + { + // [aaaaaaaa, abce) is not a valid bound since 1) they don't share the + // same prefix, 2) the prefixes are not consecutive + Slice upper_bound("abce"); + ReadOptions read_options; + read_options.prefix_same_as_start = true; + read_options.iterate_upper_bound = &upper_bound; + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_EQ(CountIter(iter, "aaaaaaaa"), 0); + // should skip bloom filter since mismatch is found + ASSERT_EQ(TestGetTickerCount(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), + 4); + ASSERT_EQ(TestGetTickerCount(options, NON_LAST_LEVEL_SEEK_FILTERED), 0); + } + ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "fixed:3"}})); + { + // [abc, abd) is not a valid bound since the upper bound is too short + // for BF (capped:4) + Slice upper_bound("abd"); + ReadOptions read_options; + read_options.prefix_same_as_start = true; + read_options.iterate_upper_bound = &upper_bound; + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_EQ(CountIter(iter, "abc"), 4); + ASSERT_EQ(TestGetTickerCount(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), + 4); + ASSERT_EQ(TestGetTickerCount(options, NON_LAST_LEVEL_SEEK_FILTERED), 0); + } + // Same with re-open + options.prefix_extractor.reset(NewFixedPrefixTransform(3)); + Reopen(options); + { + Slice upper_bound("abd"); + ReadOptions read_options; + read_options.prefix_same_as_start = true; + read_options.iterate_upper_bound = &upper_bound; + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_EQ(CountIter(iter, "abc"), 4); + ASSERT_EQ(TestGetTickerCount(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), + 4); + ASSERT_EQ(TestGetTickerCount(options, NON_LAST_LEVEL_SEEK_FILTERED), 0); + } + // Set back to capped:4 and verify BF is always read + options.prefix_extractor.reset(NewCappedPrefixTransform(4)); + Reopen(options); + { + Slice upper_bound("abd"); + ReadOptions read_options; + read_options.prefix_same_as_start = true; + read_options.iterate_upper_bound = &upper_bound; + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_EQ(CountIter(iter, "abc"), 0); + ASSERT_EQ(TestGetTickerCount(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), + 4); + ASSERT_EQ(TestGetTickerCount(options, NON_LAST_LEVEL_SEEK_FILTERED), 1); + } + // Same if there's a problem initally loading prefix transform + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTable::Open::ForceNullTablePrefixExtractor", + [&](void* arg) { *static_cast(arg) = true; }); + SyncPoint::GetInstance()->EnableProcessing(); + Reopen(options); + { + Slice upper_bound("abd"); + ReadOptions read_options; + read_options.prefix_same_as_start = true; + read_options.iterate_upper_bound = &upper_bound; + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_EQ(CountIter(iter, "abc"), 0); + ASSERT_EQ(TestGetTickerCount(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), + 4); + ASSERT_EQ(TestGetTickerCount(options, NON_LAST_LEVEL_SEEK_FILTERED), 2); + } + SyncPoint::GetInstance()->DisableProcessing(); + } +} + +// Create multiple SST files each with a different prefix_extractor config, +// verify iterators can read all SST files using the latest config. +TEST_F(DBBloomFilterTest, DynamicBloomFilterMultipleSST) { + for (const auto& bfp_impl : BloomLikeFilterPolicy::GetAllFixedImpls()) { + Options options; + options.env = CurrentOptions().env; + options.create_if_missing = true; + options.prefix_extractor.reset(NewFixedPrefixTransform(1)); + options.disable_auto_compactions = true; + options.statistics = CreateDBStatistics(); + // Enable prefix bloom for SST files + BlockBasedTableOptions table_options; + table_options.filter_policy = Create(10, bfp_impl); + table_options.cache_index_and_filter_blocks = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DestroyAndReopen(options); + + Slice upper_bound("foz90000"); + ReadOptions read_options; + read_options.prefix_same_as_start = true; + + // first SST with fixed:1 BF + ASSERT_OK(Put("foo2", "bar2")); + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Put("foq1", "bar1")); + ASSERT_OK(Put("fpa", "0")); + dbfull()->Flush(FlushOptions()); + std::unique_ptr iter_old(db_->NewIterator(read_options)); + ASSERT_EQ(CountIter(iter_old, "foo"), 4); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTERED), 0); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), 1); + + ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "capped:3"}})); + ASSERT_EQ(dbfull()->GetOptions().prefix_extractor->AsString(), + "rocksdb.CappedPrefix.3"); + read_options.iterate_upper_bound = &upper_bound; + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_EQ(CountIter(iter, "foo"), 2); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTERED), 0); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), 1); + ASSERT_EQ(CountIter(iter, "gpk"), 0); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTERED), 0); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), 0); + + // second SST with capped:3 BF + ASSERT_OK(Put("foo3", "bar3")); + ASSERT_OK(Put("foo4", "bar4")); + ASSERT_OK(Put("foq5", "bar5")); + ASSERT_OK(Put("fpb", "1")); + ASSERT_OK(dbfull()->Flush(FlushOptions())); + { + // BF is cappped:3 now + std::unique_ptr iter_tmp(db_->NewIterator(read_options)); + ASSERT_EQ(CountIter(iter_tmp, "foo"), 4); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTERED), 0); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), 2); + ASSERT_EQ(CountIter(iter_tmp, "gpk"), 0); + // both counters are incremented because BF is "not changed" for 1 of the + // 2 SST files, so filter is checked once and found no match. + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTERED), 1); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), 0); + } + + ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "fixed:2"}})); + ASSERT_EQ(dbfull()->GetOptions().prefix_extractor->AsString(), + "rocksdb.FixedPrefix.2"); + // third SST with fixed:2 BF + ASSERT_OK(Put("foo6", "bar6")); + ASSERT_OK(Put("foo7", "bar7")); + ASSERT_OK(Put("foq8", "bar8")); + ASSERT_OK(Put("fpc", "2")); + ASSERT_OK(dbfull()->Flush(FlushOptions())); + { + // BF is fixed:2 now + std::unique_ptr iter_tmp(db_->NewIterator(read_options)); + ASSERT_EQ(CountIter(iter_tmp, "foo"), 9); + // the first and last BF are checked + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTERED), 0); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), 2); + ASSERT_EQ(CountIter(iter_tmp, "gpk"), 0); + // only last BF is checked and not found + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTERED), 1); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), 0); + } + + // iter_old can only see the first SST + ASSERT_EQ(CountIter(iter_old, "foo"), 4); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTERED), 0); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), 1); + // same with iter, but different prefix extractor + ASSERT_EQ(CountIter(iter, "foo"), 2); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTERED), 0); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), 1); + + { + // keys in all three SSTs are visible to iterator + // The range of [foo, foz90000] is compatible with (fixed:1) and (fixed:2) + std::unique_ptr iter_all(db_->NewIterator(read_options)); + ASSERT_EQ(CountIter(iter_all, "foo"), 9); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTERED), 0); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), 2); + ASSERT_EQ(CountIter(iter_all, "gpk"), 0); + // FIXME? isn't seek key out of SST range? + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTERED), 1); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), 0); + } + ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "capped:3"}})); + ASSERT_EQ(dbfull()->GetOptions().prefix_extractor->AsString(), + "rocksdb.CappedPrefix.3"); + { + std::unique_ptr iter_all(db_->NewIterator(read_options)); + ASSERT_EQ(CountIter(iter_all, "foo"), 6); + // all three SST are checked because the current options has the same as + // the remaining SST (capped:3) + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTERED), 0); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), 3); + ASSERT_EQ(CountIter(iter_all, "gpk"), 0); + // FIXME? isn't seek key out of SST range? + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTERED), 1); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), 0); + } + // TODO(Zhongyi): Maybe also need to add Get calls to test point look up? + } +} + +// Create a new column family in a running DB, change prefix_extractor +// dynamically, verify the iterator created on the new column family behaves +// as expected +TEST_F(DBBloomFilterTest, DynamicBloomFilterNewColumnFamily) { + int iteration = 0; + for (const auto& bfp_impl : BloomLikeFilterPolicy::GetAllFixedImpls()) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.prefix_extractor.reset(NewFixedPrefixTransform(1)); + options.disable_auto_compactions = true; + options.statistics = CreateDBStatistics(); + // Enable prefix bloom for SST files + BlockBasedTableOptions table_options; + table_options.cache_index_and_filter_blocks = true; + table_options.filter_policy = Create(10, bfp_impl); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + CreateAndReopenWithCF({"pikachu" + std::to_string(iteration)}, options); + ReadOptions read_options; + read_options.prefix_same_as_start = true; + // create a new CF and set prefix_extractor dynamically + options.prefix_extractor.reset(NewCappedPrefixTransform(3)); + CreateColumnFamilies({"ramen_dojo_" + std::to_string(iteration)}, options); + ASSERT_EQ(dbfull()->GetOptions(handles_[2]).prefix_extractor->AsString(), + "rocksdb.CappedPrefix.3"); + ASSERT_OK(Put(2, "foo3", "bar3")); + ASSERT_OK(Put(2, "foo4", "bar4")); + ASSERT_OK(Put(2, "foo5", "bar5")); + ASSERT_OK(Put(2, "foq6", "bar6")); + ASSERT_OK(Put(2, "fpq7", "bar7")); + dbfull()->Flush(FlushOptions()); + { + std::unique_ptr iter( + db_->NewIterator(read_options, handles_[2])); + ASSERT_EQ(CountIter(iter, "foo"), 3); + ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED), 0); + ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + } + ASSERT_OK( + dbfull()->SetOptions(handles_[2], {{"prefix_extractor", "fixed:2"}})); + ASSERT_EQ(dbfull()->GetOptions(handles_[2]).prefix_extractor->AsString(), + "rocksdb.FixedPrefix.2"); + { + std::unique_ptr iter( + db_->NewIterator(read_options, handles_[2])); + ASSERT_EQ(CountIter(iter, "foo"), 4); + ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED), 0); + ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 0); + } + ASSERT_OK(dbfull()->DropColumnFamily(handles_[2])); + ASSERT_OK(dbfull()->DestroyColumnFamilyHandle(handles_[2])); + handles_[2] = nullptr; + ASSERT_OK(dbfull()->DropColumnFamily(handles_[1])); + ASSERT_OK(dbfull()->DestroyColumnFamilyHandle(handles_[1])); + handles_[1] = nullptr; + iteration++; + } +} + +// Verify it's possible to change prefix_extractor at runtime and iterators +// behaves as expected +TEST_F(DBBloomFilterTest, DynamicBloomFilterOptions) { + for (const auto& bfp_impl : BloomLikeFilterPolicy::GetAllFixedImpls()) { + Options options; + options.env = CurrentOptions().env; + options.create_if_missing = true; + options.prefix_extractor.reset(NewFixedPrefixTransform(1)); + options.disable_auto_compactions = true; + options.statistics = CreateDBStatistics(); + // Enable prefix bloom for SST files + BlockBasedTableOptions table_options; + table_options.cache_index_and_filter_blocks = true; + table_options.filter_policy = Create(10, bfp_impl); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DestroyAndReopen(options); + + ASSERT_OK(Put("foo2", "bar2")); + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Put("foo1", "bar1")); + ASSERT_OK(Put("fpa", "0")); + dbfull()->Flush(FlushOptions()); + ASSERT_OK(Put("foo3", "bar3")); + ASSERT_OK(Put("foo4", "bar4")); + ASSERT_OK(Put("foo5", "bar5")); + ASSERT_OK(Put("fpb", "1")); + dbfull()->Flush(FlushOptions()); + ASSERT_OK(Put("foo6", "bar6")); + ASSERT_OK(Put("foo7", "bar7")); + ASSERT_OK(Put("foo8", "bar8")); + ASSERT_OK(Put("fpc", "2")); + dbfull()->Flush(FlushOptions()); + + ReadOptions read_options; + read_options.prefix_same_as_start = true; + { + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_EQ(CountIter(iter, "foo"), 12); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTERED), 0); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), 3); + } + std::unique_ptr iter_old(db_->NewIterator(read_options)); + ASSERT_EQ(CountIter(iter_old, "foo"), 12); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTERED), 0); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), 3); + + ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "capped:3"}})); + ASSERT_EQ(dbfull()->GetOptions().prefix_extractor->AsString(), + "rocksdb.CappedPrefix.3"); + { + std::unique_ptr iter(db_->NewIterator(read_options)); + // "fp*" should be skipped + ASSERT_EQ(CountIter(iter, "foo"), 9); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTERED), 0); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), 0); + } + + // iterator created before should not be affected and see all keys + ASSERT_EQ(CountIter(iter_old, "foo"), 12); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTERED), 0); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), 3); + ASSERT_EQ(CountIter(iter_old, "abc"), 0); + // FIXME? isn't seek key out of SST range? + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTERED), 3); + EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH), 0); + } +} + +TEST_F(DBBloomFilterTest, SeekForPrevWithPartitionedFilters) { + Options options = CurrentOptions(); + constexpr size_t kNumKeys = 10000; + static_assert(kNumKeys <= 10000, "kNumKeys have to be <= 10000"); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeys + 10)); + options.create_if_missing = true; + constexpr size_t kPrefixLength = 4; + options.prefix_extractor.reset(NewFixedPrefixTransform(kPrefixLength)); + options.compression = kNoCompression; + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(NewBloomFilterPolicy(50)); + bbto.index_shortening = + BlockBasedTableOptions::IndexShorteningMode::kNoShortening; + bbto.block_size = 128; + bbto.metadata_block_size = 128; + bbto.partition_filters = true; + bbto.index_type = BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + DestroyAndReopen(options); + + const std::string value(64, '\0'); + + WriteOptions write_opts; + write_opts.disableWAL = true; + for (size_t i = 0; i < kNumKeys; ++i) { + std::ostringstream oss; + oss << std::setfill('0') << std::setw(4) << std::fixed << i; + ASSERT_OK(db_->Put(write_opts, oss.str(), value)); + } + ASSERT_OK(Flush()); + + ReadOptions read_opts; + // Use legacy, implicit prefix seek + read_opts.total_order_seek = false; + read_opts.auto_prefix_mode = false; + std::unique_ptr it(db_->NewIterator(read_opts)); + for (size_t i = 0; i < kNumKeys; ++i) { + // Seek with a key after each one added but with same prefix. One will + // surely cross a partition boundary. + std::ostringstream oss; + oss << std::setfill('0') << std::setw(4) << std::fixed << i << "a"; + it->SeekForPrev(oss.str()); + ASSERT_OK(it->status()); + ASSERT_TRUE(it->Valid()); + } + it.reset(); +} + +namespace { +class BackwardBytewiseComparator : public Comparator { + public: + const char* Name() const override { return "BackwardBytewiseComparator"; } + + int Compare(const Slice& a, const Slice& b) const override { + int min_size_neg = -static_cast(std::min(a.size(), b.size())); + const char* a_end = a.data() + a.size(); + const char* b_end = b.data() + b.size(); + for (int i = -1; i >= min_size_neg; --i) { + if (a_end[i] != b_end[i]) { + if (static_cast(a_end[i]) < + static_cast(b_end[i])) { + return -1; + } else { + return 1; + } + } + } + return static_cast(a.size()) - static_cast(b.size()); + } + + void FindShortestSeparator(std::string* /*start*/, + const Slice& /*limit*/) const override {} + + void FindShortSuccessor(std::string* /*key*/) const override {} +}; + +const BackwardBytewiseComparator kBackwardBytewiseComparator{}; + +class FixedSuffix4Transform : public SliceTransform { + const char* Name() const override { return "FixedSuffixTransform"; } + + Slice Transform(const Slice& src) const override { + return Slice(src.data() + src.size() - 4, 4); + } + + bool InDomain(const Slice& src) const override { return src.size() >= 4; } +}; + +std::pair GetBloomStat(const Options& options, bool sst) { + if (sst) { + return {options.statistics->getAndResetTickerCount( + NON_LAST_LEVEL_SEEK_FILTER_MATCH), + options.statistics->getAndResetTickerCount( + NON_LAST_LEVEL_SEEK_FILTERED)}; + } else { + auto hit = std::exchange(get_perf_context()->bloom_memtable_hit_count, 0); + auto miss = std::exchange(get_perf_context()->bloom_memtable_miss_count, 0); + return {hit, miss}; + } +} + +std::pair HitAndMiss(uint64_t hits, uint64_t misses) { + return {hits, misses}; +} +} // anonymous namespace + +// This uses a prefix_extractor + comparator combination that violates +// one of the old obsolete, unnecessary axioms of prefix extraction: +// * key.starts_with(prefix(key)) +// This axiom is not really needed, and we validate that here. +TEST_F(DBBloomFilterTest, WeirdPrefixExtractorWithFilter1) { + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10)); + bbto.whole_key_filtering = false; + + Options options = CurrentOptions(); + options.comparator = &kBackwardBytewiseComparator; + options.prefix_extractor = std::make_shared(); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + options.memtable_prefix_bloom_size_ratio = 0.1; + options.statistics = CreateDBStatistics(); + + DestroyAndReopen(options); + + ASSERT_OK(Put("321aaaa", "val1")); + ASSERT_OK(Put("112aaaa", "val2")); + ASSERT_OK(Put("009aaaa", "val3")); + ASSERT_OK(Put("baa", "val4")); // out of domain + ASSERT_OK(Put("321abaa", "val5")); + ASSERT_OK(Put("zzz", "val6")); // out of domain + + for (auto flushed : {false, true}) { + SCOPED_TRACE("flushed=" + std::to_string(flushed)); + if (flushed) { + ASSERT_OK(Flush()); + } + ReadOptions read_options; + if (flushed) { // TODO: support auto_prefix_mode in memtable? + read_options.auto_prefix_mode = true; + } + EXPECT_EQ(GetBloomStat(options, flushed), HitAndMiss(0, 0)); + { + Slice ub("999aaaa"); + read_options.iterate_upper_bound = &ub; + std::unique_ptr iter(db_->NewIterator(read_options)); + EXPECT_EQ(CountIter(iter, "aaaa"), 3); + EXPECT_EQ(GetBloomStat(options, flushed), HitAndMiss(1, 0)); + } + { + Slice ub("999abaa"); + read_options.iterate_upper_bound = &ub; + std::unique_ptr iter(db_->NewIterator(read_options)); + EXPECT_EQ(CountIter(iter, "abaa"), 1); + EXPECT_EQ(GetBloomStat(options, flushed), HitAndMiss(1, 0)); + } + { + Slice ub("999acaa"); + read_options.iterate_upper_bound = &ub; + std::unique_ptr iter(db_->NewIterator(read_options)); + EXPECT_EQ(CountIter(iter, "acaa"), 0); + EXPECT_EQ(GetBloomStat(options, flushed), HitAndMiss(0, 1)); + } + { + Slice ub("zzzz"); + read_options.iterate_upper_bound = &ub; + std::unique_ptr iter(db_->NewIterator(read_options)); + EXPECT_EQ(CountIter(iter, "baa"), 3); + if (flushed) { // TODO: fix memtable case + EXPECT_EQ(GetBloomStat(options, flushed), HitAndMiss(0, 0)); + } + } + } +} + +// This uses a prefix_extractor + comparator combination that violates +// one of the old obsolete, unnecessary axioms of prefix extraction: +// * Compare(prefix(key), key) <= 0 +// This axiom is not really needed, and we validate that here. +TEST_F(DBBloomFilterTest, WeirdPrefixExtractorWithFilter2) { + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10)); + bbto.whole_key_filtering = false; + + Options options = CurrentOptions(); + options.comparator = ReverseBytewiseComparator(); + options.prefix_extractor.reset(NewFixedPrefixTransform(4)); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + options.memtable_prefix_bloom_size_ratio = 0.1; + options.statistics = CreateDBStatistics(); + + DestroyAndReopen(options); + + ASSERT_OK(Put("aaaa123", "val1")); + ASSERT_OK(Put("aaaa211", "val2")); + ASSERT_OK(Put("aaaa900", "val3")); + ASSERT_OK(Put("aab", "val4")); // out of domain + ASSERT_OK(Put("aaba123", "val5")); + ASSERT_OK(Put("qqqq123", "val7")); + ASSERT_OK(Put("qqqq", "val8")); + ASSERT_OK(Put("zzz", "val8")); // out of domain + + for (auto flushed : {false, true}) { + SCOPED_TRACE("flushed=" + std::to_string(flushed)); + if (flushed) { + ASSERT_OK(Flush()); + } + ReadOptions read_options; + if (flushed) { // TODO: support auto_prefix_mode in memtable? + read_options.auto_prefix_mode = true; + } else { + // TODO: why needed? + get_perf_context()->bloom_memtable_hit_count = 0; + get_perf_context()->bloom_memtable_miss_count = 0; + } + EXPECT_EQ(GetBloomStat(options, flushed), HitAndMiss(0, 0)); + { + Slice ub("aaaa000"); + read_options.iterate_upper_bound = &ub; + std::unique_ptr iter(db_->NewIterator(read_options)); + EXPECT_EQ(CountIter(iter, "aaaa999"), 3); + EXPECT_EQ(GetBloomStat(options, flushed), HitAndMiss(1, 0)); + } + { + // Note: prefix does work as upper bound + Slice ub("aaaa"); + read_options.iterate_upper_bound = &ub; + std::unique_ptr iter(db_->NewIterator(read_options)); + EXPECT_EQ(CountIter(iter, "aaaa999"), 3); + EXPECT_EQ(GetBloomStat(options, flushed), HitAndMiss(1, 0)); + } + { + // Note: prefix does not work here as seek key + Slice ub("aaaa500"); + read_options.iterate_upper_bound = &ub; + std::unique_ptr iter(db_->NewIterator(read_options)); + EXPECT_EQ(CountIter(iter, "aaaa"), 0); + EXPECT_EQ(GetBloomStat(options, flushed), HitAndMiss(1, 0)); + } + { + Slice ub("aaba000"); + read_options.iterate_upper_bound = &ub; + std::unique_ptr iter(db_->NewIterator(read_options)); + EXPECT_EQ(CountIter(iter, "aaba999"), 1); + EXPECT_EQ(GetBloomStat(options, flushed), HitAndMiss(1, 0)); + } + { + Slice ub("aaca000"); + read_options.iterate_upper_bound = &ub; + std::unique_ptr iter(db_->NewIterator(read_options)); + EXPECT_EQ(CountIter(iter, "aaca999"), 0); + EXPECT_EQ(GetBloomStat(options, flushed), HitAndMiss(0, 1)); + } + { + Slice ub("aaaz"); + read_options.iterate_upper_bound = &ub; + std::unique_ptr iter(db_->NewIterator(read_options)); + EXPECT_EQ(CountIter(iter, "zzz"), 5); + EXPECT_EQ(GetBloomStat(options, flushed), HitAndMiss(0, 0)); + } + { + // Note: prefix does work here as seek key, but only finds key equal + // to prefix (others with same prefix are less) + read_options.auto_prefix_mode = false; + read_options.iterate_upper_bound = nullptr; + read_options.prefix_same_as_start = true; + std::unique_ptr iter(db_->NewIterator(read_options)); + EXPECT_EQ(CountIter(iter, "qqqq"), 1); + EXPECT_EQ(GetBloomStat(options, flushed), HitAndMiss(1, 0)); + } + } +} + +namespace { +// A weird comparator that in combination with NonIdempotentFixed4Transform +// breaks an old axiom of prefix filtering. +class WeirdComparator : public Comparator { + public: + const char* Name() const override { return "WeirdComparator"; } + + int Compare(const Slice& a, const Slice& b) const override { + bool a_in = a.size() >= 5; + bool b_in = b.size() >= 5; + if (a_in != b_in) { + // Order keys after prefixes + return a_in - b_in; + } + if (a_in) { + return BytewiseComparator()->Compare(a, b); + } else { + // Different ordering on the prefixes + return ReverseBytewiseComparator()->Compare(a, b); + } + } + + void FindShortestSeparator(std::string* /*start*/, + const Slice& /*limit*/) const override {} + + void FindShortSuccessor(std::string* /*key*/) const override {} +}; +const WeirdComparator kWeirdComparator{}; + +// Non-idempotentent because prefix is always 4 bytes, but this is +// out-of-domain for keys to be assigned prefixes (>= 5 bytes) +class NonIdempotentFixed4Transform : public SliceTransform { + const char* Name() const override { return "NonIdempotentFixed4Transform"; } + + Slice Transform(const Slice& src) const override { + return Slice(src.data(), 4); + } + + bool InDomain(const Slice& src) const override { return src.size() >= 5; } +}; +} // anonymous namespace + +// This uses a prefix_extractor + comparator combination that violates +// two of the old obsolete, unnecessary axioms of prefix extraction: +// * prefix(prefix(key)) == prefix(key) +// * If Compare(k1, k2) <= 0, then Compare(prefix(k1), prefix(k2)) <= 0 +// This axiom is not really needed, and we validate that here. +TEST_F(DBBloomFilterTest, WeirdPrefixExtractorWithFilter3) { + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10)); + bbto.whole_key_filtering = false; + + Options options = CurrentOptions(); + options.prefix_extractor = std::make_shared(); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + options.memtable_prefix_bloom_size_ratio = 0.1; + options.statistics = CreateDBStatistics(); + + for (auto weird_comparator : {false, true}) { + if (weird_comparator) { + options.comparator = &kWeirdComparator; + } + DestroyAndReopen(options); + + ASSERT_OK(Put("aaaa123", "val1")); + ASSERT_OK(Put("aaaa211", "val2")); + ASSERT_OK(Put("aaaa900", "val3")); + ASSERT_OK(Put("aab", "val4")); // out of domain + ASSERT_OK(Put("aaba123", "val5")); + ASSERT_OK(Put("qqqq123", "val7")); + ASSERT_OK(Put("qqqq", "val8")); // out of domain + ASSERT_OK(Put("zzzz", "val8")); // out of domain + + for (auto flushed : {false, true}) { + SCOPED_TRACE("flushed=" + std::to_string(flushed)); + if (flushed) { + ASSERT_OK(Flush()); + } + ReadOptions read_options; + if (flushed) { // TODO: support auto_prefix_mode in memtable? + read_options.auto_prefix_mode = true; + } else { + // TODO: why needed? + get_perf_context()->bloom_memtable_hit_count = 0; + get_perf_context()->bloom_memtable_miss_count = 0; + } + EXPECT_EQ(GetBloomStat(options, flushed), HitAndMiss(0, 0)); + { + Slice ub("aaaa999"); + read_options.iterate_upper_bound = &ub; + std::unique_ptr iter(db_->NewIterator(read_options)); + EXPECT_EQ(CountIter(iter, "aaaa000"), 3); + EXPECT_EQ(GetBloomStat(options, flushed), HitAndMiss(1, 0)); + } + { + // Note: prefix as seek key is not bloom-optimized + // Note: the count works with weird_comparator because "aaaa" is + // ordered as the last of the prefixes + Slice ub("aaaa999"); + read_options.iterate_upper_bound = &ub; + std::unique_ptr iter(db_->NewIterator(read_options)); + EXPECT_EQ(CountIter(iter, "aaaa"), 3); + EXPECT_EQ(GetBloomStat(options, flushed), HitAndMiss(0, 0)); + } + { + Slice ub("aaba9"); + read_options.iterate_upper_bound = &ub; + std::unique_ptr iter(db_->NewIterator(read_options)); + EXPECT_EQ(CountIter(iter, "aaba0"), 1); + EXPECT_EQ(GetBloomStat(options, flushed), HitAndMiss(1, 0)); + } + { + Slice ub("aaca9"); + read_options.iterate_upper_bound = &ub; + std::unique_ptr iter(db_->NewIterator(read_options)); + EXPECT_EQ(CountIter(iter, "aaca0"), 0); + EXPECT_EQ(GetBloomStat(options, flushed), HitAndMiss(0, 1)); + } + { + Slice ub("qqqq9"); + read_options.iterate_upper_bound = &ub; + std::unique_ptr iter(db_->NewIterator(read_options)); + EXPECT_EQ(CountIter(iter, "qqqq0"), 1); + EXPECT_EQ(GetBloomStat(options, flushed), HitAndMiss(1, 0)); + } + { + // Note: prefix as seek key is not bloom-optimized + Slice ub("qqqq9"); + read_options.iterate_upper_bound = &ub; + std::unique_ptr iter(db_->NewIterator(read_options)); + EXPECT_EQ(CountIter(iter, "qqqq"), weird_comparator ? 7 : 2); + EXPECT_EQ(GetBloomStat(options, flushed), HitAndMiss(0, 0)); + } + { + // Note: prefix as seek key is not bloom-optimized + Slice ub("zzzz9"); + read_options.iterate_upper_bound = &ub; + std::unique_ptr iter(db_->NewIterator(read_options)); + EXPECT_EQ(CountIter(iter, "zzzz"), weird_comparator ? 8 : 1); + EXPECT_EQ(GetBloomStat(options, flushed), HitAndMiss(0, 0)); + } + { + Slice ub("zzzz9"); + read_options.iterate_upper_bound = &ub; + std::unique_ptr iter(db_->NewIterator(read_options)); + EXPECT_EQ(CountIter(iter, "aab"), weird_comparator ? 6 : 5); + EXPECT_EQ(GetBloomStat(options, flushed), HitAndMiss(0, 0)); + } + } + } +} + + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_clip_test.cc b/librocksdb-sys/rocksdb/db/db_clip_test.cc new file mode 100644 index 0000000..fd0bb57 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_clip_test.cc @@ -0,0 +1,142 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/db_test_util.h" +#include "port/port.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +class DBClipTest : public DBTestBase { + public: + DBClipTest() : DBTestBase("db_clip_test", /*env_do_fsync=*/true) {} +}; + +TEST_F(DBClipTest, TestClipRange) { + Options options = CurrentOptions(); + options.write_buffer_size = 10 * 1024 * 1024; + options.max_bytes_for_level_multiplier = 2; + options.num_levels = 3; + options.max_background_compactions = 3; + options.disable_auto_compactions = true; + options.statistics = CreateDBStatistics(); + + DestroyAndReopen(options); + int32_t value_size = 10 * 1024; // 10 KB + + Random rnd(301); + std::map values; + + // file [0 => 100), [100 => 200), ... [900, 1000) + for (auto i = 0; i < 10; i++) { + for (auto j = 0; j < 100; j++) { + auto k = i * 100 + j; + values[k] = rnd.RandomString(value_size); + ASSERT_OK(Put(Key(k), values[k])); + } + ASSERT_OK(Flush()); + } + ASSERT_EQ("10", FilesPerLevel(0)); + auto begin_key = Key(251), end_key = Key(751); + ASSERT_OK( + db_->ClipColumnFamily(db_->DefaultColumnFamily(), begin_key, end_key)); + + for (auto i = 0; i < 251; i++) { + ReadOptions ropts; + std::string result; + auto s = db_->Get(ropts, Key(i), &result); + ASSERT_TRUE(s.IsNotFound()); + } + for (auto i = 251; i < 751; i++) { + ASSERT_EQ(Get(Key(i)), values[i]); + } + for (auto i = 751; i < 1000; i++) { + ReadOptions ropts; + std::string result; + auto s = db_->Get(ropts, Key(i), &result); + ASSERT_TRUE(s.IsNotFound()); + } + + std::vector all_metadata; + db_->GetLiveFilesMetaData(&all_metadata); + for (auto& md : all_metadata) { + // make sure clip_begin_key <= file_smallestkey <= file_largestkey <= + // clip_end_key + bool in_range = false; + + if (options.comparator->Compare(begin_key, md.smallestkey) <= 0 && + options.comparator->Compare(end_key, md.largestkey) > 0) { + in_range = true; + } + ASSERT_TRUE(in_range); + } + + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 2; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + ASSERT_EQ("0,0,3", FilesPerLevel(0)); + + for (auto i = 0; i < 10; i += 2) { + for (auto j = 0; j < 100; j++) { + auto k = i * 100 + j; + ASSERT_OK(Put(Key(k), values[k])); + } + ASSERT_OK(Flush()); + } + ASSERT_EQ("5,0,3", FilesPerLevel(0)); + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr)); + ASSERT_EQ("0,5,3", FilesPerLevel(0)); + + for (auto i = 1; i < 10; i += 2) { + for (auto j = 0; j < 100; j++) { + auto k = i * 100 + j; + ASSERT_OK(Put(Key(k), values[k])); + } + ASSERT_OK(Flush()); + } + ASSERT_EQ("5,5,3", FilesPerLevel(0)); + + auto begin_key_2 = Key(222), end_key_2 = Key(888); + + ASSERT_OK(db_->ClipColumnFamily(db_->DefaultColumnFamily(), begin_key_2, + end_key_2)); + + for (auto i = 0; i < 222; i++) { + ReadOptions ropts; + std::string result; + auto s = db_->Get(ropts, Key(i), &result); + ASSERT_TRUE(s.IsNotFound()); + } + for (auto i = 222; i < 888; i++) { + ASSERT_EQ(Get(Key(i)), values[i]); + } + for (auto i = 888; i < 1000; i++) { + ReadOptions ropts; + std::string result; + auto s = db_->Get(ropts, Key(i), &result); + ASSERT_TRUE(s.IsNotFound()); + } + + std::vector all_metadata_2; + db_->GetLiveFilesMetaData(&all_metadata_2); + for (auto& md : all_metadata_2) { + // make sure clip_begin_key <= file_smallestkey <= file_largestkey <= + // clip_end_key + bool in_range = false; + if (begin_key_2.compare(md.smallestkey) <= 0 && + end_key_2.compare(md.largestkey) > 0) { + in_range = true; + } + ASSERT_TRUE(in_range); + } +} +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/db/db_compaction_filter_test.cc b/librocksdb-sys/rocksdb/db/db_compaction_filter_test.cc new file mode 100644 index 0000000..596dfef --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_compaction_filter_test.cc @@ -0,0 +1,1030 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/db_test_util.h" +#include "port/stack_trace.h" + +namespace ROCKSDB_NAMESPACE { + +static int cfilter_count = 0; +static int cfilter_skips = 0; + +// This is a static filter used for filtering +// kvs during the compaction process. +static std::string NEW_VALUE = "NewValue"; + +class DBTestCompactionFilter : public DBTestBase { + public: + DBTestCompactionFilter() + : DBTestBase("db_compaction_filter_test", /*env_do_fsync=*/true) {} +}; + +// Param variant of DBTestBase::ChangeCompactOptions +class DBTestCompactionFilterWithCompactParam + : public DBTestCompactionFilter, + public ::testing::WithParamInterface { + public: + DBTestCompactionFilterWithCompactParam() : DBTestCompactionFilter() { + option_config_ = GetParam(); + Destroy(last_options_); + auto options = CurrentOptions(); + if (option_config_ == kDefault || option_config_ == kUniversalCompaction || + option_config_ == kUniversalCompactionMultiLevel) { + options.create_if_missing = true; + } + if (option_config_ == kLevelSubcompactions || + option_config_ == kUniversalSubcompactions) { + assert(options.max_subcompactions > 1); + } + Reopen(options); + } +}; + +#if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) +INSTANTIATE_TEST_CASE_P( + CompactionFilterWithOption, DBTestCompactionFilterWithCompactParam, + ::testing::Values(DBTestBase::OptionConfig::kDefault, + DBTestBase::OptionConfig::kUniversalCompaction, + DBTestBase::OptionConfig::kUniversalCompactionMultiLevel, + DBTestBase::OptionConfig::kLevelSubcompactions, + DBTestBase::OptionConfig::kUniversalSubcompactions)); +#else +// Run fewer cases in non-full valgrind to save time. +INSTANTIATE_TEST_CASE_P(CompactionFilterWithOption, + DBTestCompactionFilterWithCompactParam, + ::testing::Values(DBTestBase::OptionConfig::kDefault)); +#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) + +class KeepFilter : public CompactionFilter { + public: + bool Filter(int /*level*/, const Slice& /*key*/, const Slice& /*value*/, + std::string* /*new_value*/, + bool* /*value_changed*/) const override { + cfilter_count++; + return false; + } + + const char* Name() const override { return "KeepFilter"; } +}; + +class DeleteFilter : public CompactionFilter { + public: + bool Filter(int /*level*/, const Slice& /*key*/, const Slice& /*value*/, + std::string* /*new_value*/, + bool* /*value_changed*/) const override { + cfilter_count++; + return true; + } + + bool FilterMergeOperand(int /*level*/, const Slice& /*key*/, + const Slice& /*operand*/) const override { + return true; + } + + const char* Name() const override { return "DeleteFilter"; } +}; + +class DeleteISFilter : public CompactionFilter { + public: + bool Filter(int /*level*/, const Slice& key, const Slice& /*value*/, + std::string* /*new_value*/, + bool* /*value_changed*/) const override { + cfilter_count++; + int i = std::stoi(key.ToString()); + if (i > 5 && i <= 105) { + return true; + } + return false; + } + + bool IgnoreSnapshots() const override { return true; } + + const char* Name() const override { return "DeleteFilter"; } +}; + +// Skip x if floor(x/10) is even, use range skips. Requires that keys are +// zero-padded to length 10. +class SkipEvenFilter : public CompactionFilter { + public: + Decision FilterV2(int /*level*/, const Slice& key, ValueType /*value_type*/, + const Slice& /*existing_value*/, std::string* /*new_value*/, + std::string* skip_until) const override { + cfilter_count++; + int i = std::stoi(key.ToString()); + if (i / 10 % 2 == 0) { + char key_str[100]; + snprintf(key_str, sizeof(key_str), "%010d", i / 10 * 10 + 10); + *skip_until = key_str; + ++cfilter_skips; + return Decision::kRemoveAndSkipUntil; + } + return Decision::kKeep; + } + + bool IgnoreSnapshots() const override { return true; } + + const char* Name() const override { return "DeleteFilter"; } +}; + +class ConditionalFilter : public CompactionFilter { + public: + explicit ConditionalFilter(const std::string* filtered_value) + : filtered_value_(filtered_value) {} + bool Filter(int /*level*/, const Slice& /*key*/, const Slice& value, + std::string* /*new_value*/, + bool* /*value_changed*/) const override { + return value.ToString() == *filtered_value_; + } + + const char* Name() const override { return "ConditionalFilter"; } + + private: + const std::string* filtered_value_; +}; + +class ChangeFilter : public CompactionFilter { + public: + explicit ChangeFilter() {} + + bool Filter(int /*level*/, const Slice& /*key*/, const Slice& /*value*/, + std::string* new_value, bool* value_changed) const override { + assert(new_value != nullptr); + *new_value = NEW_VALUE; + *value_changed = true; + return false; + } + + const char* Name() const override { return "ChangeFilter"; } +}; + +class KeepFilterFactory : public CompactionFilterFactory { + public: + explicit KeepFilterFactory(bool check_context = false, + bool check_context_cf_id = false) + : check_context_(check_context), + check_context_cf_id_(check_context_cf_id), + compaction_filter_created_(false) {} + + std::unique_ptr CreateCompactionFilter( + const CompactionFilter::Context& context) override { + if (check_context_) { + EXPECT_EQ(expect_full_compaction_.load(), context.is_full_compaction); + EXPECT_EQ(expect_manual_compaction_.load(), context.is_manual_compaction); + } + if (check_context_cf_id_) { + EXPECT_EQ(expect_cf_id_.load(), context.column_family_id); + } + compaction_filter_created_ = true; + return std::unique_ptr(new KeepFilter()); + } + + bool compaction_filter_created() const { return compaction_filter_created_; } + + const char* Name() const override { return "KeepFilterFactory"; } + bool check_context_; + bool check_context_cf_id_; + std::atomic_bool expect_full_compaction_; + std::atomic_bool expect_manual_compaction_; + std::atomic expect_cf_id_; + bool compaction_filter_created_; +}; + +// This filter factory is configured with a `TableFileCreationReason`. Only +// table files created for that reason will undergo filtering. This +// configurability makes it useful to tests for filtering non-compaction table +// files, such as "CompactionFilterFlush" and "CompactionFilterRecovery". +class DeleteFilterFactory : public CompactionFilterFactory { + public: + explicit DeleteFilterFactory(TableFileCreationReason reason) + : reason_(reason) {} + + std::unique_ptr CreateCompactionFilter( + const CompactionFilter::Context& context) override { + EXPECT_EQ(reason_, context.reason); + if (context.reason == TableFileCreationReason::kCompaction && + !context.is_manual_compaction) { + // Table files created by automatic compaction do not undergo filtering. + // Presumably some tests rely on this. + return std::unique_ptr(nullptr); + } + return std::unique_ptr(new DeleteFilter()); + } + + bool ShouldFilterTableFileCreation( + TableFileCreationReason reason) const override { + return reason_ == reason; + } + + const char* Name() const override { return "DeleteFilterFactory"; } + + private: + const TableFileCreationReason reason_; +}; + +// Delete Filter Factory which ignores snapshots +class DeleteISFilterFactory : public CompactionFilterFactory { + public: + std::unique_ptr CreateCompactionFilter( + const CompactionFilter::Context& context) override { + if (context.is_manual_compaction) { + return std::unique_ptr(new DeleteISFilter()); + } else { + return std::unique_ptr(nullptr); + } + } + + const char* Name() const override { return "DeleteFilterFactory"; } +}; + +class SkipEvenFilterFactory : public CompactionFilterFactory { + public: + std::unique_ptr CreateCompactionFilter( + const CompactionFilter::Context& context) override { + if (context.is_manual_compaction) { + return std::unique_ptr(new SkipEvenFilter()); + } else { + return std::unique_ptr(nullptr); + } + } + + const char* Name() const override { return "SkipEvenFilterFactory"; } +}; + +class ConditionalFilterFactory : public CompactionFilterFactory { + public: + explicit ConditionalFilterFactory(const Slice& filtered_value) + : filtered_value_(filtered_value.ToString()) {} + + std::unique_ptr CreateCompactionFilter( + const CompactionFilter::Context& /*context*/) override { + return std::unique_ptr( + new ConditionalFilter(&filtered_value_)); + } + + const char* Name() const override { return "ConditionalFilterFactory"; } + + private: + std::string filtered_value_; +}; + +class ChangeFilterFactory : public CompactionFilterFactory { + public: + explicit ChangeFilterFactory() {} + + std::unique_ptr CreateCompactionFilter( + const CompactionFilter::Context& /*context*/) override { + return std::unique_ptr(new ChangeFilter()); + } + + const char* Name() const override { return "ChangeFilterFactory"; } +}; + +TEST_F(DBTestCompactionFilter, CompactionFilter) { + Options options = CurrentOptions(); + options.max_open_files = -1; + options.num_levels = 3; + options.compaction_filter_factory = std::make_shared(); + options = CurrentOptions(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // Write 100K keys, these are written to a few files in L0. + const std::string value(10, 'x'); + for (int i = 0; i < 100000; i++) { + char key[100]; + snprintf(key, sizeof(key), "B%010d", i); + ASSERT_OK(Put(1, key, value)); + } + ASSERT_OK(Flush(1)); + + // Push all files to the highest level L2. Verify that + // the compaction is each level invokes the filter for + // all the keys in that level. + cfilter_count = 0; + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1])); + ASSERT_EQ(cfilter_count, 100000); + cfilter_count = 0; + ASSERT_OK(dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1])); + ASSERT_EQ(cfilter_count, 100000); + + ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0); + ASSERT_EQ(NumTableFilesAtLevel(1, 1), 0); + ASSERT_NE(NumTableFilesAtLevel(2, 1), 0); + cfilter_count = 0; + + // All the files are in the lowest level. + // Verify that all but the 100001st record + // has sequence number zero. The 100001st record + // is at the tip of this snapshot and cannot + // be zeroed out. + int count = 0; + int total = 0; + Arena arena; + { + InternalKeyComparator icmp(options.comparator); + ReadOptions read_options; + ScopedArenaIterator iter(dbfull()->NewInternalIterator( + read_options, &arena, kMaxSequenceNumber, handles_[1])); + iter->SeekToFirst(); + ASSERT_OK(iter->status()); + while (iter->Valid()) { + ParsedInternalKey ikey(Slice(), 0, kTypeValue); + ASSERT_OK(ParseInternalKey(iter->key(), &ikey, true /* log_err_key */)); + total++; + if (ikey.sequence != 0) { + count++; + } + iter->Next(); + } + ASSERT_OK(iter->status()); + } + ASSERT_EQ(total, 100000); + ASSERT_EQ(count, 0); + + // overwrite all the 100K keys once again. + for (int i = 0; i < 100000; i++) { + char key[100]; + snprintf(key, sizeof(key), "B%010d", i); + ASSERT_OK(Put(1, key, value)); + } + ASSERT_OK(Flush(1)); + + // push all files to the highest level L2. This + // means that all keys should pass at least once + // via the compaction filter + cfilter_count = 0; + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1])); + ASSERT_EQ(cfilter_count, 100000); + cfilter_count = 0; + ASSERT_OK(dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1])); + ASSERT_EQ(cfilter_count, 100000); + ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0); + ASSERT_EQ(NumTableFilesAtLevel(1, 1), 0); + ASSERT_NE(NumTableFilesAtLevel(2, 1), 0); + + // create a new database with the compaction + // filter in such a way that it deletes all keys + options.compaction_filter_factory = std::make_shared( + TableFileCreationReason::kCompaction); + options.create_if_missing = true; + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // write all the keys once again. + for (int i = 0; i < 100000; i++) { + char key[100]; + snprintf(key, sizeof(key), "B%010d", i); + ASSERT_OK(Put(1, key, value)); + } + ASSERT_OK(Flush(1)); + ASSERT_NE(NumTableFilesAtLevel(0, 1), 0); + ASSERT_EQ(NumTableFilesAtLevel(1, 1), 0); + ASSERT_EQ(NumTableFilesAtLevel(2, 1), 0); + + // Push all files to the highest level L2. This + // triggers the compaction filter to delete all keys, + // verify that at the end of the compaction process, + // nothing is left. + cfilter_count = 0; + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1])); + ASSERT_EQ(cfilter_count, 100000); + cfilter_count = 0; + ASSERT_OK(dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1])); + ASSERT_EQ(cfilter_count, 0); + ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0); + ASSERT_EQ(NumTableFilesAtLevel(1, 1), 0); + + { + // Scan the entire database to ensure that nothing is left + std::unique_ptr iter( + db_->NewIterator(ReadOptions(), handles_[1])); + iter->SeekToFirst(); + count = 0; + while (iter->Valid()) { + count++; + iter->Next(); + } + ASSERT_OK(iter->status()); + ASSERT_EQ(count, 0); + } + + // The sequence number of the remaining record + // is not zeroed out even though it is at the + // level Lmax because this record is at the tip + count = 0; + { + InternalKeyComparator icmp(options.comparator); + ReadOptions read_options; + ScopedArenaIterator iter(dbfull()->NewInternalIterator( + read_options, &arena, kMaxSequenceNumber, handles_[1])); + iter->SeekToFirst(); + ASSERT_OK(iter->status()); + while (iter->Valid()) { + ParsedInternalKey ikey(Slice(), 0, kTypeValue); + ASSERT_OK(ParseInternalKey(iter->key(), &ikey, true /* log_err_key */)); + ASSERT_NE(ikey.sequence, (unsigned)0); + count++; + iter->Next(); + } + ASSERT_EQ(count, 0); + } +} + +// Tests the edge case where compaction does not produce any output -- all +// entries are deleted. The compaction should create bunch of 'DeleteFile' +// entries in VersionEdit, but none of the 'AddFile's. +TEST_F(DBTestCompactionFilter, CompactionFilterDeletesAll) { + Options options = CurrentOptions(); + options.compaction_filter_factory = std::make_shared( + TableFileCreationReason::kCompaction); + options.disable_auto_compactions = true; + options.create_if_missing = true; + DestroyAndReopen(options); + + // put some data + for (int table = 0; table < 4; ++table) { + for (int i = 0; i < 10 + table; ++i) { + ASSERT_OK(Put(std::to_string(table * 100 + i), "val")); + } + ASSERT_OK(Flush()); + } + + // this will produce empty file (delete compaction filter) + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(0U, CountLiveFiles()); + + Reopen(options); + + Iterator* itr = db_->NewIterator(ReadOptions()); + itr->SeekToFirst(); + ASSERT_OK(itr->status()); + // empty db + ASSERT_TRUE(!itr->Valid()); + + delete itr; +} + +TEST_F(DBTestCompactionFilter, CompactionFilterFlush) { + // Tests a `CompactionFilterFactory` that filters when table file is created + // by flush. + Options options = CurrentOptions(); + options.compaction_filter_factory = + std::make_shared(TableFileCreationReason::kFlush); + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + Reopen(options); + + // Puts and Merges are purged in flush. + ASSERT_OK(Put("a", "v")); + ASSERT_OK(Merge("b", "v")); + ASSERT_OK(Flush()); + ASSERT_EQ("NOT_FOUND", Get("a")); + ASSERT_EQ("NOT_FOUND", Get("b")); + + // However, Puts and Merges are preserved by recovery. + ASSERT_OK(Put("a", "v")); + ASSERT_OK(Merge("b", "v")); + Reopen(options); + ASSERT_EQ("v", Get("a")); + ASSERT_EQ("v", Get("b")); + + // Likewise, compaction does not apply filtering. + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("v", Get("a")); + ASSERT_EQ("v", Get("b")); +} + +TEST_F(DBTestCompactionFilter, CompactionFilterRecovery) { + // Tests a `CompactionFilterFactory` that filters when table file is created + // by recovery. + Options options = CurrentOptions(); + options.compaction_filter_factory = + std::make_shared(TableFileCreationReason::kRecovery); + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + Reopen(options); + + // Puts and Merges are purged in recovery. + ASSERT_OK(Put("a", "v")); + ASSERT_OK(Merge("b", "v")); + Reopen(options); + ASSERT_EQ("NOT_FOUND", Get("a")); + ASSERT_EQ("NOT_FOUND", Get("b")); + + // However, Puts and Merges are preserved by flush. + ASSERT_OK(Put("a", "v")); + ASSERT_OK(Merge("b", "v")); + ASSERT_OK(Flush()); + ASSERT_EQ("v", Get("a")); + ASSERT_EQ("v", Get("b")); + + // Likewise, compaction does not apply filtering. + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("v", Get("a")); + ASSERT_EQ("v", Get("b")); +} + +TEST_P(DBTestCompactionFilterWithCompactParam, + CompactionFilterWithValueChange) { + Options options = CurrentOptions(); + options.num_levels = 3; + options.compaction_filter_factory = std::make_shared(); + CreateAndReopenWithCF({"pikachu"}, options); + + // Write 100K+1 keys, these are written to a few files + // in L0. We do this so that the current snapshot points + // to the 100001 key.The compaction filter is not invoked + // on keys that are visible via a snapshot because we + // anyways cannot delete it. + const std::string value(10, 'x'); + for (int i = 0; i < 100001; i++) { + char key[100]; + snprintf(key, sizeof(key), "B%010d", i); + ASSERT_OK(Put(1, key, value)); + } + + // push all files to lower levels + ASSERT_OK(Flush(1)); + if (option_config_ != kUniversalCompactionMultiLevel && + option_config_ != kUniversalSubcompactions) { + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1])); + ASSERT_OK(dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1])); + } else { + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), handles_[1], + nullptr, nullptr)); + } + + // re-write all data again + for (int i = 0; i < 100001; i++) { + char key[100]; + snprintf(key, sizeof(key), "B%010d", i); + ASSERT_OK(Put(1, key, value)); + } + + // push all files to lower levels. This should + // invoke the compaction filter for all 100000 keys. + ASSERT_OK(Flush(1)); + if (option_config_ != kUniversalCompactionMultiLevel && + option_config_ != kUniversalSubcompactions) { + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1])); + ASSERT_OK(dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1])); + } else { + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), handles_[1], + nullptr, nullptr)); + } + + // verify that all keys now have the new value that + // was set by the compaction process. + for (int i = 0; i < 100001; i++) { + char key[100]; + snprintf(key, sizeof(key), "B%010d", i); + std::string newvalue = Get(1, key); + ASSERT_EQ(newvalue.compare(NEW_VALUE), 0); + } +} + +TEST_F(DBTestCompactionFilter, CompactionFilterWithMergeOperator) { + std::string one, two, three, four; + PutFixed64(&one, 1); + PutFixed64(&two, 2); + PutFixed64(&three, 3); + PutFixed64(&four, 4); + + Options options = CurrentOptions(); + options.create_if_missing = true; + options.merge_operator = MergeOperators::CreateUInt64AddOperator(); + options.num_levels = 3; + // Filter out keys with value is 2. + options.compaction_filter_factory = + std::make_shared(two); + DestroyAndReopen(options); + + // In the same compaction, a value type needs to be deleted based on + // compaction filter, and there is a merge type for the key. compaction + // filter result is ignored. + ASSERT_OK(db_->Put(WriteOptions(), "foo", two)); + ASSERT_OK(Flush()); + ASSERT_OK(db_->Merge(WriteOptions(), "foo", one)); + ASSERT_OK(Flush()); + std::string newvalue = Get("foo"); + ASSERT_EQ(newvalue, three); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + newvalue = Get("foo"); + ASSERT_EQ(newvalue, three); + + // value key can be deleted based on compaction filter, leaving only + // merge keys. + ASSERT_OK(db_->Put(WriteOptions(), "bar", two)); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + newvalue = Get("bar"); + ASSERT_EQ("NOT_FOUND", newvalue); + ASSERT_OK(db_->Merge(WriteOptions(), "bar", two)); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + newvalue = Get("bar"); + ASSERT_EQ(two, two); + + // Compaction filter never applies to merge keys. + ASSERT_OK(db_->Put(WriteOptions(), "foobar", one)); + ASSERT_OK(Flush()); + ASSERT_OK(db_->Merge(WriteOptions(), "foobar", two)); + ASSERT_OK(Flush()); + newvalue = Get("foobar"); + ASSERT_EQ(newvalue, three); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + newvalue = Get("foobar"); + ASSERT_EQ(newvalue, three); + + // In the same compaction, both of value type and merge type keys need to be + // deleted based on compaction filter, and there is a merge type for the key. + // For both keys, compaction filter results are ignored. + ASSERT_OK(db_->Put(WriteOptions(), "barfoo", two)); + ASSERT_OK(Flush()); + ASSERT_OK(db_->Merge(WriteOptions(), "barfoo", two)); + ASSERT_OK(Flush()); + newvalue = Get("barfoo"); + ASSERT_EQ(newvalue, four); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + newvalue = Get("barfoo"); + ASSERT_EQ(newvalue, four); +} + +TEST_F(DBTestCompactionFilter, CompactionFilterContextManual) { + KeepFilterFactory* filter = new KeepFilterFactory(true, true); + + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.compaction_filter_factory.reset(filter); + options.compression = kNoCompression; + options.level0_file_num_compaction_trigger = 8; + Reopen(options); + int num_keys_per_file = 400; + for (int j = 0; j < 3; j++) { + // Write several keys. + const std::string value(10, 'x'); + for (int i = 0; i < num_keys_per_file; i++) { + char key[100]; + snprintf(key, sizeof(key), "B%08d%02d", i, j); + ASSERT_OK(Put(key, value)); + } + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + // Make sure next file is much smaller so automatic compaction will not + // be triggered. + num_keys_per_file /= 2; + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // Force a manual compaction + cfilter_count = 0; + filter->expect_manual_compaction_.store(true); + filter->expect_full_compaction_.store(true); + filter->expect_cf_id_.store(0); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(cfilter_count, 700); + ASSERT_EQ(NumSortedRuns(0), 1); + ASSERT_TRUE(filter->compaction_filter_created()); + + // Verify total number of keys is correct after manual compaction. + { + int count = 0; + int total = 0; + Arena arena; + InternalKeyComparator icmp(options.comparator); + ReadOptions read_options; + ScopedArenaIterator iter(dbfull()->NewInternalIterator(read_options, &arena, + kMaxSequenceNumber)); + iter->SeekToFirst(); + ASSERT_OK(iter->status()); + while (iter->Valid()) { + ParsedInternalKey ikey(Slice(), 0, kTypeValue); + ASSERT_OK(ParseInternalKey(iter->key(), &ikey, true /* log_err_key */)); + total++; + if (ikey.sequence != 0) { + count++; + } + iter->Next(); + } + ASSERT_EQ(total, 700); + ASSERT_EQ(count, 0); + } +} + +TEST_F(DBTestCompactionFilter, CompactionFilterContextCfId) { + KeepFilterFactory* filter = new KeepFilterFactory(false, true); + filter->expect_cf_id_.store(1); + + Options options = CurrentOptions(); + options.compaction_filter_factory.reset(filter); + options.compression = kNoCompression; + options.level0_file_num_compaction_trigger = 2; + CreateAndReopenWithCF({"pikachu"}, options); + + int num_keys_per_file = 400; + for (int j = 0; j < 3; j++) { + // Write several keys. + const std::string value(10, 'x'); + for (int i = 0; i < num_keys_per_file; i++) { + char key[100]; + snprintf(key, sizeof(key), "B%08d%02d", i, j); + ASSERT_OK(Put(1, key, value)); + } + ASSERT_OK(Flush(1)); + // Make sure next file is much smaller so automatic compaction will not + // be triggered. + num_keys_per_file /= 2; + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_TRUE(filter->compaction_filter_created()); +} + +// Compaction filters applies to all records, regardless snapshots. +TEST_F(DBTestCompactionFilter, CompactionFilterIgnoreSnapshot) { + std::string five = std::to_string(5); + Options options = CurrentOptions(); + options.compaction_filter_factory = std::make_shared(); + options.disable_auto_compactions = true; + options.create_if_missing = true; + DestroyAndReopen(options); + + // Put some data. + const Snapshot* snapshot = nullptr; + for (int table = 0; table < 4; ++table) { + for (int i = 0; i < 10; ++i) { + ASSERT_OK(Put(std::to_string(table * 100 + i), "val")); + } + ASSERT_OK(Flush()); + + if (table == 0) { + snapshot = db_->GetSnapshot(); + } + } + assert(snapshot != nullptr); + + cfilter_count = 0; + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + // The filter should delete 40 records. + ASSERT_EQ(40, cfilter_count); + + { + // Scan the entire database as of the snapshot to ensure + // that nothing is left + ReadOptions read_options; + read_options.snapshot = snapshot; + std::unique_ptr iter(db_->NewIterator(read_options)); + iter->SeekToFirst(); + ASSERT_OK(iter->status()); + int count = 0; + while (iter->Valid()) { + count++; + iter->Next(); + } + ASSERT_EQ(count, 6); + read_options.snapshot = nullptr; + std::unique_ptr iter1(db_->NewIterator(read_options)); + ASSERT_OK(iter1->status()); + iter1->SeekToFirst(); + count = 0; + while (iter1->Valid()) { + count++; + iter1->Next(); + } + // We have deleted 10 keys from 40 using the compaction filter + // Keys 6-9 before the snapshot and 100-105 after the snapshot + ASSERT_EQ(count, 30); + } + + // Release the snapshot and compact again -> now all records should be + // removed. + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(DBTestCompactionFilter, SkipUntil) { + Options options = CurrentOptions(); + options.compaction_filter_factory = std::make_shared(); + options.disable_auto_compactions = true; + options.create_if_missing = true; + DestroyAndReopen(options); + + // Write 100K keys, these are written to a few files in L0. + for (int table = 0; table < 4; ++table) { + // Key ranges in tables are [0, 38], [106, 149], [212, 260], [318, 371]. + for (int i = table * 6; i < 39 + table * 11; ++i) { + char key[100]; + snprintf(key, sizeof(key), "%010d", table * 100 + i); + ASSERT_OK(Put(key, std::to_string(table * 1000 + i))); + } + ASSERT_OK(Flush()); + } + + cfilter_skips = 0; + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + // Number of skips in tables: 2, 3, 3, 3. + ASSERT_EQ(11, cfilter_skips); + + for (int table = 0; table < 4; ++table) { + for (int i = table * 6; i < 39 + table * 11; ++i) { + int k = table * 100 + i; + char key[100]; + snprintf(key, sizeof(key), "%010d", table * 100 + i); + auto expected = std::to_string(table * 1000 + i); + std::string val; + Status s = db_->Get(ReadOptions(), key, &val); + if (k / 10 % 2 == 0) { + ASSERT_TRUE(s.IsNotFound()); + } else { + ASSERT_OK(s); + ASSERT_EQ(expected, val); + } + } + } +} + +TEST_F(DBTestCompactionFilter, SkipUntilWithBloomFilter) { + BlockBasedTableOptions table_options; + table_options.whole_key_filtering = false; + table_options.filter_policy.reset(NewBloomFilterPolicy(100, false)); + + Options options = CurrentOptions(); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.prefix_extractor.reset(NewCappedPrefixTransform(9)); + options.compaction_filter_factory = std::make_shared(); + options.disable_auto_compactions = true; + options.create_if_missing = true; + DestroyAndReopen(options); + + ASSERT_OK(Put("0000000010", "v10")); + ASSERT_OK(Put("0000000020", "v20")); // skipped + ASSERT_OK(Put("0000000050", "v50")); + ASSERT_OK(Flush()); + + cfilter_skips = 0; + EXPECT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + EXPECT_EQ(1, cfilter_skips); + + Status s; + std::string val; + + s = db_->Get(ReadOptions(), "0000000010", &val); + ASSERT_OK(s); + EXPECT_EQ("v10", val); + + s = db_->Get(ReadOptions(), "0000000020", &val); + EXPECT_TRUE(s.IsNotFound()); + + s = db_->Get(ReadOptions(), "0000000050", &val); + ASSERT_OK(s); + EXPECT_EQ("v50", val); +} + +class TestNotSupportedFilter : public CompactionFilter { + public: + bool Filter(int /*level*/, const Slice& /*key*/, const Slice& /*value*/, + std::string* /*new_value*/, + bool* /*value_changed*/) const override { + return true; + } + + const char* Name() const override { return "NotSupported"; } + bool IgnoreSnapshots() const override { return false; } +}; + +TEST_F(DBTestCompactionFilter, IgnoreSnapshotsFalse) { + Options options = CurrentOptions(); + options.compaction_filter = new TestNotSupportedFilter(); + DestroyAndReopen(options); + + ASSERT_OK(Put("a", "v10")); + ASSERT_OK(Put("z", "v20")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("a", "v10")); + ASSERT_OK(Put("z", "v20")); + ASSERT_OK(Flush()); + + // Comapction should fail because IgnoreSnapshots() = false + EXPECT_TRUE(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr) + .IsNotSupported()); + + delete options.compaction_filter; +} + +class TestNotSupportedFilterFactory : public CompactionFilterFactory { + public: + explicit TestNotSupportedFilterFactory(TableFileCreationReason reason) + : reason_(reason) {} + + bool ShouldFilterTableFileCreation( + TableFileCreationReason reason) const override { + return reason_ == reason; + } + + std::unique_ptr CreateCompactionFilter( + const CompactionFilter::Context& /* context */) override { + return std::unique_ptr(new TestNotSupportedFilter()); + } + + const char* Name() const override { return "TestNotSupportedFilterFactory"; } + + private: + const TableFileCreationReason reason_; +}; + +TEST_F(DBTestCompactionFilter, IgnoreSnapshotsFalseDuringFlush) { + Options options = CurrentOptions(); + options.compaction_filter_factory = + std::make_shared( + TableFileCreationReason::kFlush); + Reopen(options); + + ASSERT_OK(Put("a", "v10")); + ASSERT_TRUE(Flush().IsNotSupported()); +} + +TEST_F(DBTestCompactionFilter, IgnoreSnapshotsFalseRecovery) { + Options options = CurrentOptions(); + options.compaction_filter_factory = + std::make_shared( + TableFileCreationReason::kRecovery); + Reopen(options); + + ASSERT_OK(Put("a", "v10")); + ASSERT_TRUE(TryReopen(options).IsNotSupported()); +} + +TEST_F(DBTestCompactionFilter, DropKeyWithSingleDelete) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + + Reopen(options); + + ASSERT_OK(Put("a", "v0")); + ASSERT_OK(Put("b", "v0")); + const Snapshot* snapshot = db_->GetSnapshot(); + + ASSERT_OK(SingleDelete("b")); + ASSERT_OK(Flush()); + + { + CompactRangeOptions cro; + cro.change_level = true; + cro.target_level = options.num_levels - 1; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + } + + db_->ReleaseSnapshot(snapshot); + Close(); + + class DeleteFilterV2 : public CompactionFilter { + public: + Decision FilterV2(int /*level*/, const Slice& key, ValueType /*value_type*/, + const Slice& /*existing_value*/, + std::string* /*new_value*/, + std::string* /*skip_until*/) const override { + if (key.starts_with("b")) { + return Decision::kPurge; + } + return Decision::kRemove; + } + + const char* Name() const override { return "DeleteFilterV2"; } + } delete_filter_v2; + + options.compaction_filter = &delete_filter_v2; + options.level0_file_num_compaction_trigger = 2; + Reopen(options); + + ASSERT_OK(Put("b", "v1")); + ASSERT_OK(Put("x", "v1")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("r", "v1")); + ASSERT_OK(Put("z", "v1")); + ASSERT_OK(Flush()); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + Close(); + + options.compaction_filter = nullptr; + Reopen(options); + ASSERT_OK(SingleDelete("b")); + ASSERT_OK(Flush()); + { + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + } +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_compaction_test.cc b/librocksdb-sys/rocksdb/db/db_compaction_test.cc new file mode 100644 index 0000000..3e56510 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_compaction_test.cc @@ -0,0 +1,9853 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include + +#include "compaction/compaction_picker_universal.h" +#include "db/blob/blob_index.h" +#include "db/db_test_util.h" +#include "db/dbformat.h" +#include "env/mock_env.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/concurrent_task_limiter.h" +#include "rocksdb/experimental.h" +#include "rocksdb/sst_file_writer.h" +#include "rocksdb/utilities/convenience.h" +#include "test_util/sync_point.h" +#include "test_util/testutil.h" +#include "util/concurrent_task_limiter_impl.h" +#include "util/random.h" +#include "utilities/fault_injection_env.h" +#include "utilities/fault_injection_fs.h" + +namespace ROCKSDB_NAMESPACE { + +// SYNC_POINT is not supported in released Windows mode. + +class CompactionStatsCollector : public EventListener { + public: + CompactionStatsCollector() + : compaction_completed_( + static_cast(CompactionReason::kNumOfReasons)) { + for (auto& v : compaction_completed_) { + v.store(0); + } + } + + ~CompactionStatsCollector() override {} + + void OnCompactionCompleted(DB* /* db */, + const CompactionJobInfo& info) override { + int k = static_cast(info.compaction_reason); + int num_of_reasons = static_cast(CompactionReason::kNumOfReasons); + assert(k >= 0 && k < num_of_reasons); + compaction_completed_[k]++; + } + + void OnExternalFileIngested( + DB* /* db */, const ExternalFileIngestionInfo& /* info */) override { + int k = static_cast(CompactionReason::kExternalSstIngestion); + compaction_completed_[k]++; + } + + void OnFlushCompleted(DB* /* db */, const FlushJobInfo& /* info */) override { + int k = static_cast(CompactionReason::kFlush); + compaction_completed_[k]++; + } + + int NumberOfCompactions(CompactionReason reason) const { + int num_of_reasons = static_cast(CompactionReason::kNumOfReasons); + int k = static_cast(reason); + assert(k >= 0 && k < num_of_reasons); + return compaction_completed_.at(k).load(); + } + + private: + std::vector> compaction_completed_; +}; + +class DBCompactionTest : public DBTestBase { + public: + DBCompactionTest() + : DBTestBase("db_compaction_test", /*env_do_fsync=*/true) {} + + protected: + /* + * Verifies compaction stats of cfd are valid. + * + * For each level of cfd, its compaction stats are valid if + * 1) sum(stat.counts) == stat.count, and + * 2) stat.counts[i] == collector.NumberOfCompactions(i) + */ + void VerifyCompactionStats(ColumnFamilyData& cfd, + const CompactionStatsCollector& collector) { +#ifndef NDEBUG + InternalStats* internal_stats_ptr = cfd.internal_stats(); + ASSERT_NE(internal_stats_ptr, nullptr); + const std::vector& comp_stats = + internal_stats_ptr->TEST_GetCompactionStats(); + const int num_of_reasons = + static_cast(CompactionReason::kNumOfReasons); + std::vector counts(num_of_reasons, 0); + // Count the number of compactions caused by each CompactionReason across + // all levels. + for (const auto& stat : comp_stats) { + int sum = 0; + for (int i = 0; i < num_of_reasons; i++) { + counts[i] += stat.counts[i]; + sum += stat.counts[i]; + } + ASSERT_EQ(sum, stat.count); + } + // Verify InternalStats bookkeeping matches that of + // CompactionStatsCollector, assuming that all compactions complete. + for (int i = 0; i < num_of_reasons; i++) { + ASSERT_EQ(collector.NumberOfCompactions(static_cast(i)), + counts[i]); + } +#endif /* NDEBUG */ + } +}; + +class DBCompactionTestWithParam + : public DBTestBase, + public testing::WithParamInterface> { + public: + DBCompactionTestWithParam() + : DBTestBase("db_compaction_test", /*env_do_fsync=*/true) { + max_subcompactions_ = std::get<0>(GetParam()); + exclusive_manual_compaction_ = std::get<1>(GetParam()); + } + + // Required if inheriting from testing::WithParamInterface<> + static void SetUpTestCase() {} + static void TearDownTestCase() {} + + uint32_t max_subcompactions_; + bool exclusive_manual_compaction_; +}; + +class DBCompactionTestWithBottommostParam + : public DBTestBase, + public testing::WithParamInterface< + std::tuple> { + public: + DBCompactionTestWithBottommostParam() + : DBTestBase("db_compaction_test", /*env_do_fsync=*/true) { + bottommost_level_compaction_ = std::get<0>(GetParam()); + } + + BottommostLevelCompaction bottommost_level_compaction_; +}; + +class DBCompactionDirectIOTest : public DBCompactionTest, + public ::testing::WithParamInterface { + public: + DBCompactionDirectIOTest() : DBCompactionTest() {} +}; + +class DBCompactionWaitForCompactTest + : public DBTestBase, + public testing::WithParamInterface> { + public: + DBCompactionWaitForCompactTest() + : DBTestBase("db_compaction_test", /*env_do_fsync=*/true) { + abort_on_pause_ = std::get<0>(GetParam()); + flush_ = std::get<1>(GetParam()); + } + bool abort_on_pause_; + bool flush_; + Options options_; + WaitForCompactOptions wait_for_compact_options_; + + void SetUp() override { + // This test sets up a scenario that one more L0 file will trigger a + // compaction + const int kNumKeysPerFile = 4; + const int kNumFiles = 2; + + options_ = CurrentOptions(); + options_.level0_file_num_compaction_trigger = kNumFiles + 1; + + wait_for_compact_options_ = WaitForCompactOptions(); + wait_for_compact_options_.abort_on_pause = abort_on_pause_; + wait_for_compact_options_.flush = flush_; + + DestroyAndReopen(options_); + + Random rnd(301); + for (int i = 0; i < kNumFiles; ++i) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK( + Put(Key(i * kNumKeysPerFile + j), rnd.RandomString(100 /* len */))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("2", FilesPerLevel()); + } +}; + +// Param = true : target level is non-empty +// Param = false: level between target level and source level +// is not empty. +class ChangeLevelConflictsWithAuto + : public DBCompactionTest, + public ::testing::WithParamInterface { + public: + ChangeLevelConflictsWithAuto() : DBCompactionTest() {} +}; + +// Param = true: grab the compaction pressure token (enable +// parallel compactions) +// Param = false: Not grab the token (no parallel compactions) +class RoundRobinSubcompactionsAgainstPressureToken + : public DBCompactionTest, + public ::testing::WithParamInterface { + public: + RoundRobinSubcompactionsAgainstPressureToken() { + grab_pressure_token_ = GetParam(); + } + bool grab_pressure_token_; +}; + +class RoundRobinSubcompactionsAgainstResources + : public DBCompactionTest, + public ::testing::WithParamInterface> { + public: + RoundRobinSubcompactionsAgainstResources() { + total_low_pri_threads_ = std::get<0>(GetParam()); + max_compaction_limits_ = std::get<1>(GetParam()); + } + int total_low_pri_threads_; + int max_compaction_limits_; +}; + +namespace { +class FlushedFileCollector : public EventListener { + public: + FlushedFileCollector() {} + ~FlushedFileCollector() override {} + + void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& info) override { + std::lock_guard lock(mutex_); + flushed_files_.push_back(info.file_path); + } + + std::vector GetFlushedFiles() { + std::lock_guard lock(mutex_); + std::vector result; + for (auto fname : flushed_files_) { + result.push_back(fname); + } + return result; + } + + void ClearFlushedFiles() { flushed_files_.clear(); } + + private: + std::vector flushed_files_; + std::mutex mutex_; +}; + +class SstStatsCollector : public EventListener { + public: + SstStatsCollector() : num_ssts_creation_started_(0) {} + + void OnTableFileCreationStarted( + const TableFileCreationBriefInfo& /* info */) override { + ++num_ssts_creation_started_; + } + + int num_ssts_creation_started() { return num_ssts_creation_started_; } + + private: + std::atomic num_ssts_creation_started_; +}; + +static const int kCDTValueSize = 1000; +static const int kCDTKeysPerBuffer = 4; +static const int kCDTNumLevels = 8; +Options DeletionTriggerOptions(Options options) { + options.compression = kNoCompression; + options.write_buffer_size = kCDTKeysPerBuffer * (kCDTValueSize + 24); + options.min_write_buffer_number_to_merge = 1; + options.max_write_buffer_size_to_maintain = 0; + options.num_levels = kCDTNumLevels; + options.level0_file_num_compaction_trigger = 1; + options.target_file_size_base = options.write_buffer_size * 2; + options.target_file_size_multiplier = 2; + options.max_bytes_for_level_base = + options.target_file_size_base * options.target_file_size_multiplier; + options.max_bytes_for_level_multiplier = 2; + options.disable_auto_compactions = false; + options.compaction_options_universal.max_size_amplification_percent = 100; + return options; +} + +bool HaveOverlappingKeyRanges(const Comparator* c, const SstFileMetaData& a, + const SstFileMetaData& b) { + if (c->CompareWithoutTimestamp(a.smallestkey, b.smallestkey) >= 0) { + if (c->CompareWithoutTimestamp(a.smallestkey, b.largestkey) <= 0) { + // b.smallestkey <= a.smallestkey <= b.largestkey + return true; + } + } else if (c->CompareWithoutTimestamp(a.largestkey, b.smallestkey) >= 0) { + // a.smallestkey < b.smallestkey <= a.largestkey + return true; + } + if (c->CompareWithoutTimestamp(a.largestkey, b.largestkey) <= 0) { + if (c->CompareWithoutTimestamp(a.largestkey, b.smallestkey) >= 0) { + // b.smallestkey <= a.largestkey <= b.largestkey + return true; + } + } else if (c->CompareWithoutTimestamp(a.smallestkey, b.largestkey) <= 0) { + // a.smallestkey <= b.largestkey < a.largestkey + return true; + } + return false; +} + +// Identifies all files between level "min_level" and "max_level" +// which has overlapping key range with "input_file_meta". +void GetOverlappingFileNumbersForLevelCompaction( + const ColumnFamilyMetaData& cf_meta, const Comparator* comparator, + int min_level, int max_level, const SstFileMetaData* input_file_meta, + std::set* overlapping_file_names) { + std::set overlapping_files; + overlapping_files.insert(input_file_meta); + for (int m = min_level; m <= max_level; ++m) { + for (auto& file : cf_meta.levels[m].files) { + for (auto* included_file : overlapping_files) { + if (HaveOverlappingKeyRanges(comparator, *included_file, file)) { + overlapping_files.insert(&file); + overlapping_file_names->insert(file.name); + break; + } + } + } + } +} + +void VerifyCompactionResult( + const ColumnFamilyMetaData& cf_meta, + const std::set& overlapping_file_numbers) { +#ifndef NDEBUG + for (auto& level : cf_meta.levels) { + for (auto& file : level.files) { + assert(overlapping_file_numbers.find(file.name) == + overlapping_file_numbers.end()); + } + } +#endif +} + +const SstFileMetaData* PickFileRandomly(const ColumnFamilyMetaData& cf_meta, + Random* rand, int* level = nullptr) { + auto file_id = rand->Uniform(static_cast(cf_meta.file_count)) + 1; + for (auto& level_meta : cf_meta.levels) { + if (file_id <= level_meta.files.size()) { + if (level != nullptr) { + *level = level_meta.level; + } + auto result = rand->Uniform(file_id); + return &(level_meta.files[result]); + } + file_id -= static_cast(level_meta.files.size()); + } + assert(false); + return nullptr; +} +} // anonymous namespace + +#if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) +// All the TEST_P tests run once with sub_compactions disabled (i.e. +// options.max_subcompactions = 1) and once with it enabled +TEST_P(DBCompactionTestWithParam, CompactionDeletionTrigger) { + for (int tid = 0; tid < 3; ++tid) { + uint64_t db_size[2]; + Options options = DeletionTriggerOptions(CurrentOptions()); + options.max_subcompactions = max_subcompactions_; + + if (tid == 1) { + // the following only disable stats update in DB::Open() + // and should not affect the result of this test. + options.skip_stats_update_on_db_open = true; + } else if (tid == 2) { + // third pass with universal compaction + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = 1; + } + + DestroyAndReopen(options); + Random rnd(301); + + const int kTestSize = kCDTKeysPerBuffer * 1024; + std::vector values; + for (int k = 0; k < kTestSize; ++k) { + values.push_back(rnd.RandomString(kCDTValueSize)); + ASSERT_OK(Put(Key(k), values[k])); + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_OK(Size(Key(0), Key(kTestSize - 1), &db_size[0])); + + for (int k = 0; k < kTestSize; ++k) { + ASSERT_OK(Delete(Key(k))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_OK(Size(Key(0), Key(kTestSize - 1), &db_size[1])); + + if (options.compaction_style == kCompactionStyleUniversal) { + // Claim: in universal compaction none of the original data will remain + // once compactions settle. + // + // Proof: The compensated size of the file containing the most tombstones + // is enough on its own to trigger size amp compaction. Size amp + // compaction is a full compaction, so all tombstones meet the obsolete + // keys they cover. + ASSERT_EQ(0, db_size[1]); + } else { + // Claim: in level compaction at most `db_size[0] / 2` of the original + // data will remain once compactions settle. + // + // Proof: Assume the original data is all in the bottom level. If it were + // not, it would meet its tombstone sooner. The original data size is + // large enough to require fanout to bottom level to be greater than + // `max_bytes_for_level_multiplier == 2`. In the level just above, + // tombstones must cover less than `db_size[0] / 4` bytes since fanout >= + // 2 and file size is compensated by doubling the size of values we expect + // are covered (`kDeletionWeightOnCompaction == 2`). The tombstones in + // levels above must cover less than `db_size[0] / 8` bytes of original + // data, `db_size[0] / 16`, and so on. + ASSERT_GT(db_size[0] / 2, db_size[1]); + } + } +} +#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) + +TEST_F(DBCompactionTest, SkipStatsUpdateTest) { + // This test verify UpdateAccumulatedStats is not on + // if options.skip_stats_update_on_db_open = true + // The test will need to be updated if the internal behavior changes. + + Options options = DeletionTriggerOptions(CurrentOptions()); + options.disable_auto_compactions = true; + options.env = env_; + DestroyAndReopen(options); + Random rnd(301); + + const int kTestSize = kCDTKeysPerBuffer * 512; + std::vector values; + for (int k = 0; k < kTestSize; ++k) { + values.push_back(rnd.RandomString(kCDTValueSize)); + ASSERT_OK(Put(Key(k), values[k])); + } + + ASSERT_OK(Flush()); + + Close(); + + int update_acc_stats_called = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "VersionStorageInfo::UpdateAccumulatedStats", + [&](void* /* arg */) { ++update_acc_stats_called; }); + SyncPoint::GetInstance()->EnableProcessing(); + + // Reopen the DB with stats-update disabled + options.skip_stats_update_on_db_open = true; + options.max_open_files = 20; + Reopen(options); + + ASSERT_EQ(update_acc_stats_called, 0); + + // Repeat the reopen process, but this time we enable + // stats-update. + options.skip_stats_update_on_db_open = false; + Reopen(options); + + ASSERT_GT(update_acc_stats_called, 0); + + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBCompactionTest, TestTableReaderForCompaction) { + Options options = CurrentOptions(); + options.env = env_; + options.max_open_files = 20; + options.level0_file_num_compaction_trigger = 3; + // Avoid many shards with small max_open_files, where as little as + // two table insertions could lead to an LRU eviction, depending on + // hash values. + options.table_cache_numshardbits = 2; + DestroyAndReopen(options); + Random rnd(301); + + int num_table_cache_lookup = 0; + int num_new_table_reader = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "TableCache::FindTable:0", [&](void* arg) { + assert(arg != nullptr); + bool no_io = *(reinterpret_cast(arg)); + if (!no_io) { + // filter out cases for table properties queries. + num_table_cache_lookup++; + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "TableCache::GetTableReader:0", + [&](void* /*arg*/) { num_new_table_reader++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + for (int k = 0; k < options.level0_file_num_compaction_trigger; ++k) { + ASSERT_OK(Put(Key(k), Key(k))); + ASSERT_OK(Put(Key(10 - k), "bar")); + if (k < options.level0_file_num_compaction_trigger - 1) { + num_table_cache_lookup = 0; + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // preloading iterator issues one table cache lookup and create + // a new table reader, if not preloaded. + int old_num_table_cache_lookup = num_table_cache_lookup; + ASSERT_GE(num_table_cache_lookup, 1); + ASSERT_EQ(num_new_table_reader, 1); + + num_table_cache_lookup = 0; + num_new_table_reader = 0; + ASSERT_EQ(Key(k), Get(Key(k))); + // lookup iterator from table cache and no need to create a new one. + ASSERT_EQ(old_num_table_cache_lookup + num_table_cache_lookup, 2); + ASSERT_EQ(num_new_table_reader, 0); + } + } + + num_table_cache_lookup = 0; + num_new_table_reader = 0; + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // Preloading iterator issues one table cache lookup and creates + // a new table reader. One file is created for flush and one for compaction. + // Compaction inputs make no table cache look-up for data/range deletion + // iterators + // May preload table cache too. + ASSERT_GE(num_table_cache_lookup, 2); + int old_num_table_cache_lookup2 = num_table_cache_lookup; + + // Create new iterator for: + // (1) 1 for verifying flush results + // (2) 1 for verifying compaction results. + // (3) New TableReaders will not be created for compaction inputs + ASSERT_EQ(num_new_table_reader, 2); + + num_table_cache_lookup = 0; + num_new_table_reader = 0; + ASSERT_EQ(Key(1), Get(Key(1))); + ASSERT_EQ(num_table_cache_lookup + old_num_table_cache_lookup2, 5); + ASSERT_EQ(num_new_table_reader, 0); + + num_table_cache_lookup = 0; + num_new_table_reader = 0; + CompactRangeOptions cro; + cro.change_level = true; + cro.target_level = 2; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForceOptimized; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + // Only verifying compaction outputs issues one table cache lookup + // for both data block and range deletion block). + // May preload table cache too. + ASSERT_GE(num_table_cache_lookup, 1); + old_num_table_cache_lookup2 = num_table_cache_lookup; + // One for verifying compaction results. + // No new iterator created for compaction. + ASSERT_EQ(num_new_table_reader, 1); + + num_table_cache_lookup = 0; + num_new_table_reader = 0; + ASSERT_EQ(Key(1), Get(Key(1))); + ASSERT_EQ(num_table_cache_lookup + old_num_table_cache_lookup2, 3); + ASSERT_EQ(num_new_table_reader, 0); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_P(DBCompactionTestWithParam, CompactionDeletionTriggerReopen) { + for (int tid = 0; tid < 2; ++tid) { + uint64_t db_size[3]; + Options options = DeletionTriggerOptions(CurrentOptions()); + options.max_subcompactions = max_subcompactions_; + + if (tid == 1) { + // second pass with universal compaction + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = 1; + } + + DestroyAndReopen(options); + Random rnd(301); + + // round 1 --- insert key/value pairs. + const int kTestSize = kCDTKeysPerBuffer * 512; + std::vector values; + for (int k = 0; k < kTestSize; ++k) { + values.push_back(rnd.RandomString(kCDTValueSize)); + ASSERT_OK(Put(Key(k), values[k])); + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_OK(Size(Key(0), Key(kTestSize - 1), &db_size[0])); + Close(); + + // round 2 --- disable auto-compactions and issue deletions. + options.create_if_missing = false; + options.disable_auto_compactions = true; + Reopen(options); + + for (int k = 0; k < kTestSize; ++k) { + ASSERT_OK(Delete(Key(k))); + } + ASSERT_OK(Size(Key(0), Key(kTestSize - 1), &db_size[1])); + Close(); + // as auto_compaction is off, we shouldn't see any reduction in db size. + ASSERT_LE(db_size[0], db_size[1]); + + // round 3 --- reopen db with auto_compaction on and see if + // deletion compensation still work. + options.disable_auto_compactions = false; + Reopen(options); + // insert relatively small amount of data to trigger auto compaction. + for (int k = 0; k < kTestSize / 10; ++k) { + ASSERT_OK(Put(Key(k), values[k])); + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_OK(Size(Key(0), Key(kTestSize - 1), &db_size[2])); + // this time we're expecting significant drop in size. + // + // See "CompactionDeletionTrigger" test for proof that at most + // `db_size[0] / 2` of the original data remains. In addition to that, this + // test inserts `db_size[0] / 10` to push the tombstones into SST files and + // then through automatic compactions. So in total `3 * db_size[0] / 5` of + // the original data may remain. + ASSERT_GT(3 * db_size[0] / 5, db_size[2]); + } +} + +TEST_F(DBCompactionTest, CompactRangeBottomPri) { + ASSERT_OK(Put(Key(50), "")); + ASSERT_OK(Flush()); + ASSERT_OK(Put(Key(100), "")); + ASSERT_OK(Flush()); + ASSERT_OK(Put(Key(200), "")); + ASSERT_OK(Flush()); + + { + CompactRangeOptions cro; + cro.change_level = true; + cro.target_level = 2; + ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr)); + } + ASSERT_EQ("0,0,3", FilesPerLevel(0)); + + ASSERT_OK(Put(Key(1), "")); + ASSERT_OK(Put(Key(199), "")); + ASSERT_OK(Flush()); + ASSERT_OK(Put(Key(2), "")); + ASSERT_OK(Put(Key(199), "")); + ASSERT_OK(Flush()); + ASSERT_EQ("2,0,3", FilesPerLevel(0)); + + // Now we have 2 L0 files, and 3 L2 files, and a manual compaction will + // be triggered. + // Two compaction jobs will run. One compacts 2 L0 files in Low Pri Pool + // and one compact to L2 in bottom pri pool. + int low_pri_count = 0; + int bottom_pri_count = 0; + SyncPoint::GetInstance()->SetCallBack( + "ThreadPoolImpl::Impl::BGThread:BeforeRun", [&](void* arg) { + Env::Priority* pri = reinterpret_cast(arg); + // First time is low pri pool in the test case. + if (low_pri_count == 0 && bottom_pri_count == 0) { + ASSERT_EQ(Env::Priority::LOW, *pri); + } + if (*pri == Env::Priority::LOW) { + low_pri_count++; + } else { + bottom_pri_count++; + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + env_->SetBackgroundThreads(1, Env::Priority::BOTTOM); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(1, low_pri_count); + ASSERT_EQ(1, bottom_pri_count); + ASSERT_EQ("0,0,2", FilesPerLevel(0)); + + // Recompact bottom most level uses bottom pool + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ(1, low_pri_count); + ASSERT_EQ(2, bottom_pri_count); + + env_->SetBackgroundThreads(0, Env::Priority::BOTTOM); + ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr)); + // Low pri pool is used if bottom pool has size 0. + ASSERT_EQ(2, low_pri_count); + ASSERT_EQ(2, bottom_pri_count); + + SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBCompactionTest, DisableStatsUpdateReopen) { + uint64_t db_size[3]; + for (int test = 0; test < 2; ++test) { + Options options = DeletionTriggerOptions(CurrentOptions()); + options.skip_stats_update_on_db_open = (test == 0); + + env_->random_read_counter_.Reset(); + DestroyAndReopen(options); + Random rnd(301); + + // round 1 --- insert key/value pairs. + const int kTestSize = kCDTKeysPerBuffer * 512; + std::vector values; + for (int k = 0; k < kTestSize; ++k) { + values.push_back(rnd.RandomString(kCDTValueSize)); + ASSERT_OK(Put(Key(k), values[k])); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // L1 and L2 can fit deletions iff size compensation does not take effect, + // i.e., when `skip_stats_update_on_db_open == true`. Move any remaining + // files at or above L2 down to L3 to ensure obsolete data does not + // accidentally meet its tombstone above L3. This makes the final size more + // deterministic and easy to see whether size compensation for deletions + // took effect. + MoveFilesToLevel(3 /* level */); + ASSERT_OK(Size(Key(0), Key(kTestSize - 1), &db_size[0])); + Close(); + + // round 2 --- disable auto-compactions and issue deletions. + options.create_if_missing = false; + options.disable_auto_compactions = true; + + env_->random_read_counter_.Reset(); + Reopen(options); + + for (int k = 0; k < kTestSize; ++k) { + ASSERT_OK(Delete(Key(k))); + } + ASSERT_OK(Size(Key(0), Key(kTestSize - 1), &db_size[1])); + Close(); + // as auto_compaction is off, we shouldn't see any reduction in db size. + ASSERT_LE(db_size[0], db_size[1]); + + // round 3 --- reopen db with auto_compaction on and see if + // deletion compensation still work. + options.disable_auto_compactions = false; + Reopen(options); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_OK(Size(Key(0), Key(kTestSize - 1), &db_size[2])); + + if (options.skip_stats_update_on_db_open) { + // If update stats on DB::Open is disable, we don't expect + // deletion entries taking effect. + // + // The deletions are small enough to fit in L1 and L2, and obsolete keys + // were moved to L3+, so none of the original data should have been + // dropped. + ASSERT_LE(db_size[0], db_size[2]); + } else { + // Otherwise, we should see a significant drop in db size. + // + // See "CompactionDeletionTrigger" test for proof that at most + // `db_size[0] / 2` of the original data remains. + ASSERT_GT(db_size[0] / 2, db_size[2]); + } + } +} + +TEST_P(DBCompactionTestWithParam, CompactionTrigger) { + const int kNumKeysPerFile = 100; + + Options options = CurrentOptions(); + options.write_buffer_size = 110 << 10; // 110KB + options.arena_block_size = 4 << 10; + options.num_levels = 3; + options.level0_file_num_compaction_trigger = 3; + options.max_subcompactions = max_subcompactions_; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + CreateAndReopenWithCF({"pikachu"}, options); + + Random rnd(301); + + for (int num = 0; num < options.level0_file_num_compaction_trigger - 1; + num++) { + std::vector values; + // Write 100KB (100 values, each 1K) + for (int i = 0; i < kNumKeysPerFile; i++) { + values.push_back(rnd.RandomString(990)); + ASSERT_OK(Put(1, Key(i), values[i])); + } + // put extra key to trigger flush + ASSERT_OK(Put(1, "", "")); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); + ASSERT_EQ(NumTableFilesAtLevel(0, 1), num + 1); + } + + // generate one more file in level-0, and should trigger level-0 compaction + std::vector values; + for (int i = 0; i < kNumKeysPerFile; i++) { + values.push_back(rnd.RandomString(990)); + ASSERT_OK(Put(1, Key(i), values[i])); + } + // put extra key to trigger flush + ASSERT_OK(Put(1, "", "")); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0); + ASSERT_EQ(NumTableFilesAtLevel(1, 1), 1); +} + +TEST_F(DBCompactionTest, BGCompactionsAllowed) { + // Create several column families. Make compaction triggers in all of them + // and see number of compactions scheduled to be less than allowed. + const int kNumKeysPerFile = 100; + + Options options = CurrentOptions(); + options.write_buffer_size = 110 << 10; // 110KB + options.arena_block_size = 4 << 10; + options.num_levels = 3; + // Should speed up compaction when there are 4 files. + options.level0_file_num_compaction_trigger = 2; + options.level0_slowdown_writes_trigger = 20; + options.soft_pending_compaction_bytes_limit = 1 << 30; // Infinitely large + options.max_background_compactions = 3; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + + // Block all threads in thread pool. + const size_t kTotalTasks = 4; + env_->SetBackgroundThreads(4, Env::LOW); + test::SleepingBackgroundTask sleeping_tasks[kTotalTasks]; + for (size_t i = 0; i < kTotalTasks; i++) { + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, + &sleeping_tasks[i], Env::Priority::LOW); + sleeping_tasks[i].WaitUntilSleeping(); + } + + CreateAndReopenWithCF({"one", "two", "three"}, options); + + Random rnd(301); + for (int cf = 0; cf < 4; cf++) { + for (int num = 0; num < options.level0_file_num_compaction_trigger; num++) { + for (int i = 0; i < kNumKeysPerFile; i++) { + ASSERT_OK(Put(cf, Key(i), "")); + } + // put extra key to trigger flush + ASSERT_OK(Put(cf, "", "")); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[cf])); + ASSERT_EQ(NumTableFilesAtLevel(0, cf), num + 1); + } + } + + // Now all column families qualify compaction but only one should be + // scheduled, because no column family hits speed up condition. + ASSERT_EQ(1u, env_->GetThreadPoolQueueLen(Env::Priority::LOW)); + + // Create two more files for one column family, which triggers speed up + // condition, three compactions will be scheduled. + for (int num = 0; num < options.level0_file_num_compaction_trigger; num++) { + for (int i = 0; i < kNumKeysPerFile; i++) { + ASSERT_OK(Put(2, Key(i), "")); + } + // put extra key to trigger flush + ASSERT_OK(Put(2, "", "")); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[2])); + ASSERT_EQ(options.level0_file_num_compaction_trigger + num + 1, + NumTableFilesAtLevel(0, 2)); + } + ASSERT_EQ(3U, env_->GetThreadPoolQueueLen(Env::Priority::LOW)); + + // Unblock all threads to unblock all compactions. + for (size_t i = 0; i < kTotalTasks; i++) { + sleeping_tasks[i].WakeUp(); + sleeping_tasks[i].WaitUntilDone(); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // Verify number of compactions allowed will come back to 1. + + for (size_t i = 0; i < kTotalTasks; i++) { + sleeping_tasks[i].Reset(); + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, + &sleeping_tasks[i], Env::Priority::LOW); + sleeping_tasks[i].WaitUntilSleeping(); + } + for (int cf = 0; cf < 4; cf++) { + for (int num = 0; num < options.level0_file_num_compaction_trigger; num++) { + for (int i = 0; i < kNumKeysPerFile; i++) { + ASSERT_OK(Put(cf, Key(i), "")); + } + // put extra key to trigger flush + ASSERT_OK(Put(cf, "", "")); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[cf])); + ASSERT_EQ(NumTableFilesAtLevel(0, cf), num + 1); + } + } + + // Now all column families qualify compaction but only one should be + // scheduled, because no column family hits speed up condition. + ASSERT_EQ(1U, env_->GetThreadPoolQueueLen(Env::Priority::LOW)); + + for (size_t i = 0; i < kTotalTasks; i++) { + sleeping_tasks[i].WakeUp(); + sleeping_tasks[i].WaitUntilDone(); + } +} + +TEST_P(DBCompactionTestWithParam, CompactionsGenerateMultipleFiles) { + Options options = CurrentOptions(); + options.write_buffer_size = 100000000; // Large write buffer + options.max_subcompactions = max_subcompactions_; + CreateAndReopenWithCF({"pikachu"}, options); + + Random rnd(301); + + // Write 8MB (80 values, each 100K) + ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0); + std::vector values; + for (int i = 0; i < 80; i++) { + values.push_back(rnd.RandomString(100000)); + ASSERT_OK(Put(1, Key(i), values[i])); + } + + // Reopening moves updates to level-0 + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1], + true /* disallow trivial move */)); + + ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0); + ASSERT_GT(NumTableFilesAtLevel(1, 1), 1); + for (int i = 0; i < 80; i++) { + ASSERT_EQ(Get(1, Key(i)), values[i]); + } +} + +TEST_F(DBCompactionTest, MinorCompactionsHappen) { + do { + Options options = CurrentOptions(); + options.write_buffer_size = 10000; + CreateAndReopenWithCF({"pikachu"}, options); + + const int N = 500; + + int starting_num_tables = TotalTableFiles(1); + for (int i = 0; i < N; i++) { + ASSERT_OK(Put(1, Key(i), Key(i) + std::string(1000, 'v'))); + } + int ending_num_tables = TotalTableFiles(1); + ASSERT_GT(ending_num_tables, starting_num_tables); + + for (int i = 0; i < N; i++) { + ASSERT_EQ(Key(i) + std::string(1000, 'v'), Get(1, Key(i))); + } + + ReopenWithColumnFamilies({"default", "pikachu"}, options); + + for (int i = 0; i < N; i++) { + ASSERT_EQ(Key(i) + std::string(1000, 'v'), Get(1, Key(i))); + } + } while (ChangeCompactOptions()); +} + +TEST_F(DBCompactionTest, UserKeyCrossFile1) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleLevel; + options.level0_file_num_compaction_trigger = 3; + + DestroyAndReopen(options); + + // create first file and flush to l0 + ASSERT_OK(Put("4", "A")); + ASSERT_OK(Put("3", "A")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + + ASSERT_OK(Put("2", "A")); + ASSERT_OK(Delete("3")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ("NOT_FOUND", Get("3")); + + // move both files down to l1 + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("NOT_FOUND", Get("3")); + + for (int i = 0; i < 3; i++) { + ASSERT_OK(Put("2", "B")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ("NOT_FOUND", Get("3")); +} + +TEST_F(DBCompactionTest, UserKeyCrossFile2) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleLevel; + options.level0_file_num_compaction_trigger = 3; + + DestroyAndReopen(options); + + // create first file and flush to l0 + ASSERT_OK(Put("4", "A")); + ASSERT_OK(Put("3", "A")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + + ASSERT_OK(Put("2", "A")); + ASSERT_OK(SingleDelete("3")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ("NOT_FOUND", Get("3")); + + // move both files down to l1 + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("NOT_FOUND", Get("3")); + + for (int i = 0; i < 3; i++) { + ASSERT_OK(Put("2", "B")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ("NOT_FOUND", Get("3")); +} + +TEST_F(DBCompactionTest, CompactionSstPartitioner) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleLevel; + options.level0_file_num_compaction_trigger = 3; + std::shared_ptr factory( + NewSstPartitionerFixedPrefixFactory(4)); + options.sst_partitioner_factory = factory; + + DestroyAndReopen(options); + + // create first file and flush to l0 + ASSERT_OK(Put("aaaa1", "A")); + ASSERT_OK(Put("bbbb1", "B")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + + ASSERT_OK(Put("aaaa1", "A2")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + + // move both files down to l1 + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + std::vector files; + dbfull()->GetLiveFilesMetaData(&files); + ASSERT_EQ(2, files.size()); + ASSERT_EQ("A2", Get("aaaa1")); + ASSERT_EQ("B", Get("bbbb1")); +} + +TEST_F(DBCompactionTest, CompactionSstPartitionWithManualCompaction) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleLevel; + options.level0_file_num_compaction_trigger = 3; + + DestroyAndReopen(options); + + // create first file and flush to l0 + ASSERT_OK(Put("000015", "A")); + ASSERT_OK(Put("000025", "B")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + + // create second file and flush to l0 + ASSERT_OK(Put("000015", "A2")); + ASSERT_OK(Put("000025", "B2")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + + // CONTROL 1: compact without partitioner + CompactRangeOptions compact_options; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForceOptimized; + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + // Check (compacted but no partitioning yet) + std::vector files; + dbfull()->GetLiveFilesMetaData(&files); + ASSERT_EQ(1, files.size()); + + // Install partitioner + std::shared_ptr factory( + NewSstPartitionerFixedPrefixFactory(5)); + options.sst_partitioner_factory = factory; + Reopen(options); + + // CONTROL 2: request compaction on range with no partition boundary and no + // overlap with actual entries + Slice from("000017"); + Slice to("000019"); + ASSERT_OK(dbfull()->CompactRange(compact_options, &from, &to)); + + // Check (no partitioning yet) + files.clear(); + dbfull()->GetLiveFilesMetaData(&files); + ASSERT_EQ(1, files.size()); + ASSERT_EQ("A2", Get("000015")); + ASSERT_EQ("B2", Get("000025")); + + // TEST: request compaction overlapping with partition boundary but no + // actual entries + // NOTE: `to` is INCLUSIVE + from = Slice("000019"); + to = Slice("000020"); + ASSERT_OK(dbfull()->CompactRange(compact_options, &from, &to)); + + // Check (must be partitioned) + files.clear(); + dbfull()->GetLiveFilesMetaData(&files); + ASSERT_EQ(2, files.size()); + ASSERT_EQ("A2", Get("000015")); + ASSERT_EQ("B2", Get("000025")); +} + +TEST_F(DBCompactionTest, CompactionSstPartitionerNonTrivial) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleLevel; + options.level0_file_num_compaction_trigger = 1; + std::shared_ptr factory( + NewSstPartitionerFixedPrefixFactory(4)); + options.sst_partitioner_factory = factory; + + DestroyAndReopen(options); + + // create first file and flush to l0 + ASSERT_OK(Put("aaaa1", "A")); + ASSERT_OK(Put("bbbb1", "B")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + std::vector files; + dbfull()->GetLiveFilesMetaData(&files); + ASSERT_EQ(2, files.size()); + ASSERT_EQ("A", Get("aaaa1")); + ASSERT_EQ("B", Get("bbbb1")); +} + +TEST_F(DBCompactionTest, ZeroSeqIdCompaction) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleLevel; + options.level0_file_num_compaction_trigger = 3; + + FlushedFileCollector* collector = new FlushedFileCollector(); + options.listeners.emplace_back(collector); + + // compaction options + CompactionOptions compact_opt; + compact_opt.compression = kNoCompression; + compact_opt.output_file_size_limit = 4096; + const size_t key_len = + static_cast(compact_opt.output_file_size_limit) / 5; + + DestroyAndReopen(options); + + std::vector snaps; + + // create first file and flush to l0 + for (auto& key : {"1", "2", "3", "3", "3", "3"}) { + ASSERT_OK(Put(key, std::string(key_len, 'A'))); + snaps.push_back(dbfull()->GetSnapshot()); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + + // create second file and flush to l0 + for (auto& key : {"3", "4", "5", "6", "7", "8"}) { + ASSERT_OK(Put(key, std::string(key_len, 'A'))); + snaps.push_back(dbfull()->GetSnapshot()); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + + // move both files down to l1 + ASSERT_OK( + dbfull()->CompactFiles(compact_opt, collector->GetFlushedFiles(), 1)); + + // release snap so that first instance of key(3) can have seqId=0 + for (auto snap : snaps) { + dbfull()->ReleaseSnapshot(snap); + } + + // create 3 files in l0 so to trigger compaction + for (int i = 0; i < options.level0_file_num_compaction_trigger; i++) { + ASSERT_OK(Put("2", std::string(1, 'A'))); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_OK(Put("", "")); +} + +TEST_F(DBCompactionTest, ManualCompactionUnknownOutputSize) { + // github issue #2249 + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleLevel; + options.level0_file_num_compaction_trigger = 3; + DestroyAndReopen(options); + + // create two files in l1 that we can compact + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < options.level0_file_num_compaction_trigger; j++) { + ASSERT_OK(Put(std::to_string(2 * i), std::string(1, 'A'))); + ASSERT_OK(Put(std::to_string(2 * i + 1), std::string(1, 'A'))); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + ASSERT_OK( + dbfull()->SetOptions({{"level0_file_num_compaction_trigger", "2"}})); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumTableFilesAtLevel(0, 0), 0); + ASSERT_EQ(NumTableFilesAtLevel(1, 0), 2); + ASSERT_OK( + dbfull()->SetOptions({{"level0_file_num_compaction_trigger", "3"}})); + + ColumnFamilyMetaData cf_meta; + dbfull()->GetColumnFamilyMetaData(dbfull()->DefaultColumnFamily(), &cf_meta); + ASSERT_EQ(2, cf_meta.levels[1].files.size()); + std::vector input_filenames; + for (const auto& sst_file : cf_meta.levels[1].files) { + input_filenames.push_back(sst_file.name); + } + + // note CompactionOptions::output_file_size_limit is unset. + CompactionOptions compact_opt; + compact_opt.compression = kNoCompression; + ASSERT_OK(dbfull()->CompactFiles(compact_opt, input_filenames, 1)); +} + +// Check that writes done during a memtable compaction are recovered +// if the database is shutdown during the memtable compaction. +TEST_F(DBCompactionTest, RecoverDuringMemtableCompaction) { + do { + Options options = CurrentOptions(); + options.env = env_; + CreateAndReopenWithCF({"pikachu"}, options); + + // Trigger a long memtable compaction and reopen the database during it + ASSERT_OK(Put(1, "foo", "v1")); // Goes to 1st log file + ASSERT_OK(Put(1, "big1", std::string(10000000, 'x'))); // Fills memtable + ASSERT_OK(Put(1, "big2", std::string(1000, 'y'))); // Triggers compaction + ASSERT_OK(Put(1, "bar", "v2")); // Goes to new log file + + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_EQ("v1", Get(1, "foo")); + ASSERT_EQ("v2", Get(1, "bar")); + ASSERT_EQ(std::string(10000000, 'x'), Get(1, "big1")); + ASSERT_EQ(std::string(1000, 'y'), Get(1, "big2")); + } while (ChangeOptions()); +} + +TEST_P(DBCompactionTestWithParam, TrivialMoveOneFile) { + int32_t trivial_move = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:TrivialMove", + [&](void* /*arg*/) { trivial_move++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Options options = CurrentOptions(); + options.write_buffer_size = 100000000; + options.max_subcompactions = max_subcompactions_; + DestroyAndReopen(options); + + int32_t num_keys = 80; + int32_t value_size = 100 * 1024; // 100 KB + + Random rnd(301); + std::vector values; + for (int i = 0; i < num_keys; i++) { + values.push_back(rnd.RandomString(value_size)); + ASSERT_OK(Put(Key(i), values[i])); + } + + // Reopening moves updates to L0 + Reopen(options); + ASSERT_EQ(NumTableFilesAtLevel(0, 0), 1); // 1 file in L0 + ASSERT_EQ(NumTableFilesAtLevel(1, 0), 0); // 0 files in L1 + + std::vector metadata; + db_->GetLiveFilesMetaData(&metadata); + ASSERT_EQ(metadata.size(), 1U); + LiveFileMetaData level0_file = metadata[0]; // L0 file meta + + CompactRangeOptions cro; + cro.exclusive_manual_compaction = exclusive_manual_compaction_; + + // Compaction will initiate a trivial move from L0 to L1 + ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr)); + + // File moved From L0 to L1 + ASSERT_EQ(NumTableFilesAtLevel(0, 0), 0); // 0 files in L0 + ASSERT_EQ(NumTableFilesAtLevel(1, 0), 1); // 1 file in L1 + + metadata.clear(); + db_->GetLiveFilesMetaData(&metadata); + ASSERT_EQ(metadata.size(), 1U); + ASSERT_EQ(metadata[0].name /* level1_file.name */, level0_file.name); + ASSERT_EQ(metadata[0].size /* level1_file.size */, level0_file.size); + + for (int i = 0; i < num_keys; i++) { + ASSERT_EQ(Get(Key(i)), values[i]); + } + + ASSERT_EQ(trivial_move, 1); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_P(DBCompactionTestWithParam, TrivialMoveNonOverlappingFiles) { + int32_t trivial_move = 0; + int32_t non_trivial_move = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:TrivialMove", + [&](void* /*arg*/) { trivial_move++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial", + [&](void* /*arg*/) { non_trivial_move++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.write_buffer_size = 10 * 1024 * 1024; + options.max_subcompactions = max_subcompactions_; + + DestroyAndReopen(options); + // non overlapping ranges + std::vector> ranges = { + {100, 199}, {300, 399}, {0, 99}, {200, 299}, + {600, 699}, {400, 499}, {500, 550}, {551, 599}, + }; + int32_t value_size = 10 * 1024; // 10 KB + + Random rnd(301); + std::map values; + for (size_t i = 0; i < ranges.size(); i++) { + for (int32_t j = ranges[i].first; j <= ranges[i].second; j++) { + values[j] = rnd.RandomString(value_size); + ASSERT_OK(Put(Key(j), values[j])); + } + ASSERT_OK(Flush()); + } + + int32_t level0_files = NumTableFilesAtLevel(0, 0); + ASSERT_EQ(level0_files, ranges.size()); // Multiple files in L0 + ASSERT_EQ(NumTableFilesAtLevel(1, 0), 0); // No files in L1 + + CompactRangeOptions cro; + cro.exclusive_manual_compaction = exclusive_manual_compaction_; + + // Since data is non-overlapping we expect compaction to initiate + // a trivial move + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + // We expect that all the files were trivially moved from L0 to L1 + ASSERT_EQ(NumTableFilesAtLevel(0, 0), 0); + ASSERT_EQ(NumTableFilesAtLevel(1, 0) /* level1_files */, level0_files); + + for (size_t i = 0; i < ranges.size(); i++) { + for (int32_t j = ranges[i].first; j <= ranges[i].second; j++) { + ASSERT_EQ(Get(Key(j)), values[j]); + } + } + + ASSERT_EQ(trivial_move, 1); + ASSERT_EQ(non_trivial_move, 0); + + trivial_move = 0; + non_trivial_move = 0; + values.clear(); + DestroyAndReopen(options); + // Same ranges as above but overlapping + ranges = { + {100, 199}, + {300, 399}, + {0, 99}, + {200, 299}, + {600, 699}, + {400, 499}, + {500, 560}, // this range overlap with the next + // one + {551, 599}, + }; + for (size_t i = 0; i < ranges.size(); i++) { + for (int32_t j = ranges[i].first; j <= ranges[i].second; j++) { + values[j] = rnd.RandomString(value_size); + ASSERT_OK(Put(Key(j), values[j])); + } + ASSERT_OK(Flush()); + } + + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + for (size_t i = 0; i < ranges.size(); i++) { + for (int32_t j = ranges[i].first; j <= ranges[i].second; j++) { + ASSERT_EQ(Get(Key(j)), values[j]); + } + } + ASSERT_EQ(trivial_move, 0); + ASSERT_EQ(non_trivial_move, 1); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_P(DBCompactionTestWithParam, TrivialMoveTargetLevel) { + int32_t trivial_move = 0; + int32_t non_trivial_move = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:TrivialMove", + [&](void* /*arg*/) { trivial_move++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial", + [&](void* /*arg*/) { non_trivial_move++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.write_buffer_size = 10 * 1024 * 1024; + options.num_levels = 7; + options.max_subcompactions = max_subcompactions_; + + DestroyAndReopen(options); + int32_t value_size = 10 * 1024; // 10 KB + + // Add 2 non-overlapping files + Random rnd(301); + std::map values; + + // file 1 [0 => 300] + for (int32_t i = 0; i <= 300; i++) { + values[i] = rnd.RandomString(value_size); + ASSERT_OK(Put(Key(i), values[i])); + } + ASSERT_OK(Flush()); + + // file 2 [600 => 700] + for (int32_t i = 600; i <= 700; i++) { + values[i] = rnd.RandomString(value_size); + ASSERT_OK(Put(Key(i), values[i])); + } + ASSERT_OK(Flush()); + + // 2 files in L0 + ASSERT_EQ("2", FilesPerLevel(0)); + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 6; + compact_options.exclusive_manual_compaction = exclusive_manual_compaction_; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + // 2 files in L6 + ASSERT_EQ("0,0,0,0,0,0,2", FilesPerLevel(0)); + + ASSERT_EQ(trivial_move, 1); + ASSERT_EQ(non_trivial_move, 0); + + for (int32_t i = 0; i <= 300; i++) { + ASSERT_EQ(Get(Key(i)), values[i]); + } + for (int32_t i = 600; i <= 700; i++) { + ASSERT_EQ(Get(Key(i)), values[i]); + } +} + +TEST_P(DBCompactionTestWithParam, PartialOverlappingL0) { + class SubCompactionEventListener : public EventListener { + public: + void OnSubcompactionCompleted(const SubcompactionJobInfo&) override { + sub_compaction_finished_++; + } + std::atomic sub_compaction_finished_{0}; + }; + + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.write_buffer_size = 10 * 1024 * 1024; + options.max_subcompactions = max_subcompactions_; + SubCompactionEventListener* listener = new SubCompactionEventListener(); + options.listeners.emplace_back(listener); + + DestroyAndReopen(options); + + // For subcompactino to trigger, output level needs to be non-empty. + ASSERT_OK(Put("key", "")); + ASSERT_OK(Put("kez", "")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("key", "")); + ASSERT_OK(Put("kez", "")); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + // Ranges that are only briefly overlapping so that they won't be trivially + // moved but subcompaction ranges would only contain a subset of files. + std::vector> ranges = { + {100, 199}, {198, 399}, {397, 600}, {598, 800}, {799, 900}, {895, 999}, + }; + int32_t value_size = 10 * 1024; // 10 KB + + Random rnd(301); + std::map values; + for (size_t i = 0; i < ranges.size(); i++) { + for (int32_t j = ranges[i].first; j <= ranges[i].second; j++) { + values[j] = rnd.RandomString(value_size); + ASSERT_OK(Put(Key(j), values[j])); + } + ASSERT_OK(Flush()); + } + + int32_t level0_files = NumTableFilesAtLevel(0, 0); + ASSERT_EQ(level0_files, ranges.size()); // Multiple files in L0 + ASSERT_EQ(NumTableFilesAtLevel(1, 0), 1); // One file in L1 + + listener->sub_compaction_finished_ = 0; + ASSERT_OK(db_->EnableAutoCompaction({db_->DefaultColumnFamily()})); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + if (max_subcompactions_ > 3) { + // RocksDB might not generate the exact number of sub compactions. + // Here we validate that at least subcompaction happened. + ASSERT_GT(listener->sub_compaction_finished_.load(), 2); + } + + // We expect that all the files were compacted to L1 + ASSERT_EQ(NumTableFilesAtLevel(0, 0), 0); + ASSERT_GT(NumTableFilesAtLevel(1, 0), 1); + + for (size_t i = 0; i < ranges.size(); i++) { + for (int32_t j = ranges[i].first; j <= ranges[i].second; j++) { + ASSERT_EQ(Get(Key(j)), values[j]); + } + } +} + +TEST_P(DBCompactionTestWithParam, ManualCompactionPartial) { + int32_t trivial_move = 0; + int32_t non_trivial_move = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:TrivialMove", + [&](void* /*arg*/) { trivial_move++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial", + [&](void* /*arg*/) { non_trivial_move++; }); + bool first = true; + // Purpose of dependencies: + // 4 -> 1: ensure the order of two non-trivial compactions + // 5 -> 2 and 5 -> 3: ensure we do a check before two non-trivial compactions + // are installed + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBCompaction::ManualPartial:4", "DBCompaction::ManualPartial:1"}, + {"DBCompaction::ManualPartial:5", "DBCompaction::ManualPartial:2"}, + {"DBCompaction::ManualPartial:5", "DBCompaction::ManualPartial:3"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* /*arg*/) { + if (first) { + first = false; + TEST_SYNC_POINT("DBCompaction::ManualPartial:4"); + TEST_SYNC_POINT("DBCompaction::ManualPartial:3"); + } else { // second non-trivial compaction + TEST_SYNC_POINT("DBCompaction::ManualPartial:2"); + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Options options = CurrentOptions(); + options.write_buffer_size = 10 * 1024 * 1024; + options.num_levels = 7; + options.max_subcompactions = max_subcompactions_; + options.level0_file_num_compaction_trigger = 3; + options.max_background_compactions = 3; + options.target_file_size_base = 1 << 23; // 8 MB + + DestroyAndReopen(options); + int32_t value_size = 10 * 1024; // 10 KB + + // Add 2 non-overlapping files + Random rnd(301); + std::map values; + + // file 1 [0 => 100] + for (int32_t i = 0; i < 100; i++) { + values[i] = rnd.RandomString(value_size); + ASSERT_OK(Put(Key(i), values[i])); + } + ASSERT_OK(Flush()); + + // file 2 [100 => 300] + for (int32_t i = 100; i < 300; i++) { + values[i] = rnd.RandomString(value_size); + ASSERT_OK(Put(Key(i), values[i])); + } + ASSERT_OK(Flush()); + + // 2 files in L0 + ASSERT_EQ("2", FilesPerLevel(0)); + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 6; + compact_options.exclusive_manual_compaction = exclusive_manual_compaction_; + // Trivial move the two non-overlapping files to level 6 + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + // 2 files in L6 + ASSERT_EQ("0,0,0,0,0,0,2", FilesPerLevel(0)); + + ASSERT_EQ(trivial_move, 1); + ASSERT_EQ(non_trivial_move, 0); + + // file 3 [ 0 => 200] + for (int32_t i = 0; i < 200; i++) { + values[i] = rnd.RandomString(value_size); + ASSERT_OK(Put(Key(i), values[i])); + } + ASSERT_OK(Flush()); + + // 1 files in L0 + ASSERT_EQ("1,0,0,0,0,0,2", FilesPerLevel(0)); + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr, false)); + ASSERT_OK(dbfull()->TEST_CompactRange(1, nullptr, nullptr, nullptr, false)); + ASSERT_OK(dbfull()->TEST_CompactRange(2, nullptr, nullptr, nullptr, false)); + ASSERT_OK(dbfull()->TEST_CompactRange(3, nullptr, nullptr, nullptr, false)); + ASSERT_OK(dbfull()->TEST_CompactRange(4, nullptr, nullptr, nullptr, false)); + // 2 files in L6, 1 file in L5 + ASSERT_EQ("0,0,0,0,0,1,2", FilesPerLevel(0)); + + ASSERT_EQ(trivial_move, 6); + ASSERT_EQ(non_trivial_move, 0); + + ROCKSDB_NAMESPACE::port::Thread threads([&] { + compact_options.change_level = false; + compact_options.exclusive_manual_compaction = false; + std::string begin_string = Key(0); + std::string end_string = Key(199); + Slice begin(begin_string); + Slice end(end_string); + // First non-trivial compaction is triggered + ASSERT_OK(db_->CompactRange(compact_options, &begin, &end)); + }); + + TEST_SYNC_POINT("DBCompaction::ManualPartial:1"); + // file 4 [300 => 400) + for (int32_t i = 300; i <= 400; i++) { + values[i] = rnd.RandomString(value_size); + ASSERT_OK(Put(Key(i), values[i])); + } + ASSERT_OK(Flush()); + + // file 5 [400 => 500) + for (int32_t i = 400; i <= 500; i++) { + values[i] = rnd.RandomString(value_size); + ASSERT_OK(Put(Key(i), values[i])); + } + ASSERT_OK(Flush()); + + // file 6 [500 => 600) + for (int32_t i = 500; i <= 600; i++) { + values[i] = rnd.RandomString(value_size); + ASSERT_OK(Put(Key(i), values[i])); + } + // Second non-trivial compaction is triggered + ASSERT_OK(Flush()); + + // Before two non-trivial compactions are installed, there are 3 files in L0 + ASSERT_EQ("3,0,0,0,0,1,2", FilesPerLevel(0)); + TEST_SYNC_POINT("DBCompaction::ManualPartial:5"); + + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // After two non-trivial compactions are installed, there is 1 file in L6, and + // 1 file in L1 + ASSERT_EQ("0,1,0,0,0,0,1", FilesPerLevel(0)); + threads.join(); + + for (int32_t i = 0; i < 600; i++) { + ASSERT_EQ(Get(Key(i)), values[i]); + } +} + +// Disable as the test is flaky. +TEST_F(DBCompactionTest, DISABLED_ManualPartialFill) { + int32_t trivial_move = 0; + int32_t non_trivial_move = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:TrivialMove", + [&](void* /*arg*/) { trivial_move++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial", + [&](void* /*arg*/) { non_trivial_move++; }); + bool first = true; + bool second = true; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBCompaction::PartialFill:4", "DBCompaction::PartialFill:1"}, + {"DBCompaction::PartialFill:2", "DBCompaction::PartialFill:3"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* /*arg*/) { + if (first) { + TEST_SYNC_POINT("DBCompaction::PartialFill:4"); + first = false; + TEST_SYNC_POINT("DBCompaction::PartialFill:3"); + } else if (second) { + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Options options = CurrentOptions(); + options.write_buffer_size = 10 * 1024 * 1024; + options.max_bytes_for_level_multiplier = 2; + options.num_levels = 4; + options.level0_file_num_compaction_trigger = 3; + options.max_background_compactions = 3; + + DestroyAndReopen(options); + // make sure all background compaction jobs can be scheduled + auto stop_token = + dbfull()->TEST_write_controler().GetCompactionPressureToken(); + int32_t value_size = 10 * 1024; // 10 KB + + // Add 2 non-overlapping files + Random rnd(301); + std::map values; + + // file 1 [0 => 100] + for (int32_t i = 0; i < 100; i++) { + values[i] = rnd.RandomString(value_size); + ASSERT_OK(Put(Key(i), values[i])); + } + ASSERT_OK(Flush()); + + // file 2 [100 => 300] + for (int32_t i = 100; i < 300; i++) { + values[i] = rnd.RandomString(value_size); + ASSERT_OK(Put(Key(i), values[i])); + } + ASSERT_OK(Flush()); + + // 2 files in L0 + ASSERT_EQ("2", FilesPerLevel(0)); + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 2; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + // 2 files in L2 + ASSERT_EQ("0,0,2", FilesPerLevel(0)); + + ASSERT_EQ(trivial_move, 1); + ASSERT_EQ(non_trivial_move, 0); + + // file 3 [ 0 => 200] + for (int32_t i = 0; i < 200; i++) { + values[i] = rnd.RandomString(value_size); + ASSERT_OK(Put(Key(i), values[i])); + } + ASSERT_OK(Flush()); + + // 2 files in L2, 1 in L0 + ASSERT_EQ("1,0,2", FilesPerLevel(0)); + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr, false)); + // 2 files in L2, 1 in L1 + ASSERT_EQ("0,1,2", FilesPerLevel(0)); + + ASSERT_EQ(trivial_move, 2); + ASSERT_EQ(non_trivial_move, 0); + + ROCKSDB_NAMESPACE::port::Thread threads([&] { + compact_options.change_level = false; + compact_options.exclusive_manual_compaction = false; + std::string begin_string = Key(0); + std::string end_string = Key(199); + Slice begin(begin_string); + Slice end(end_string); + ASSERT_OK(db_->CompactRange(compact_options, &begin, &end)); + }); + + TEST_SYNC_POINT("DBCompaction::PartialFill:1"); + // Many files 4 [300 => 4300) + for (int32_t i = 0; i <= 5; i++) { + for (int32_t j = 300; j < 4300; j++) { + if (j == 2300) { + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + values[j] = rnd.RandomString(value_size); + ASSERT_OK(Put(Key(j), values[j])); + } + } + + // Verify level sizes + uint64_t target_size = 4 * options.max_bytes_for_level_base; + for (int32_t i = 1; i < options.num_levels; i++) { + ASSERT_LE(SizeAtLevel(i), target_size); + target_size = static_cast(target_size * + options.max_bytes_for_level_multiplier); + } + + TEST_SYNC_POINT("DBCompaction::PartialFill:2"); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + threads.join(); + + for (int32_t i = 0; i < 4300; i++) { + ASSERT_EQ(Get(Key(i)), values[i]); + } +} + +TEST_F(DBCompactionTest, ManualCompactionWithUnorderedWrite) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::WriteImpl:UnorderedWriteAfterWriteWAL", + "DBCompactionTest::ManualCompactionWithUnorderedWrite:WaitWriteWAL"}, + {"DBImpl::WaitForPendingWrites:BeforeBlock", + "DBImpl::WriteImpl:BeforeUnorderedWriteMemtable"}}); + + Options options = CurrentOptions(); + options.unordered_write = true; + DestroyAndReopen(options); + ASSERT_OK(Put("foo", "v1")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("bar", "v1")); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + port::Thread writer([&]() { ASSERT_OK(Put("foo", "v2")); }); + + TEST_SYNC_POINT( + "DBCompactionTest::ManualCompactionWithUnorderedWrite:WaitWriteWAL"); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + writer.join(); + ASSERT_EQ(Get("foo"), "v2"); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + Reopen(options); + ASSERT_EQ(Get("foo"), "v2"); +} + +TEST_F(DBCompactionTest, DeleteFileRange) { + Options options = CurrentOptions(); + options.write_buffer_size = 10 * 1024 * 1024; + options.max_bytes_for_level_multiplier = 2; + options.num_levels = 4; + options.level0_file_num_compaction_trigger = 3; + options.max_background_compactions = 3; + + DestroyAndReopen(options); + int32_t value_size = 10 * 1024; // 10 KB + + // Add 2 non-overlapping files + Random rnd(301); + std::map values; + + // file 1 [0 => 100] + for (int32_t i = 0; i < 100; i++) { + values[i] = rnd.RandomString(value_size); + ASSERT_OK(Put(Key(i), values[i])); + } + ASSERT_OK(Flush()); + + // file 2 [100 => 300] + for (int32_t i = 100; i < 300; i++) { + values[i] = rnd.RandomString(value_size); + ASSERT_OK(Put(Key(i), values[i])); + } + ASSERT_OK(Flush()); + + // 2 files in L0 + ASSERT_EQ("2", FilesPerLevel(0)); + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 2; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + // 2 files in L2 + ASSERT_EQ("0,0,2", FilesPerLevel(0)); + + // file 3 [ 0 => 200] + for (int32_t i = 0; i < 200; i++) { + values[i] = rnd.RandomString(value_size); + ASSERT_OK(Put(Key(i), values[i])); + } + ASSERT_OK(Flush()); + + // Many files 4 [300 => 4300) + for (int32_t i = 0; i <= 5; i++) { + for (int32_t j = 300; j < 4300; j++) { + if (j == 2300) { + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + values[j] = rnd.RandomString(value_size); + ASSERT_OK(Put(Key(j), values[j])); + } + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // Verify level sizes + uint64_t target_size = 4 * options.max_bytes_for_level_base; + for (int32_t i = 1; i < options.num_levels; i++) { + ASSERT_LE(SizeAtLevel(i), target_size); + target_size = static_cast(target_size * + options.max_bytes_for_level_multiplier); + } + + const size_t old_num_files = CountFiles(); + std::string begin_string = Key(1000); + std::string end_string = Key(2000); + Slice begin(begin_string); + Slice end(end_string); + ASSERT_OK(DeleteFilesInRange(db_, db_->DefaultColumnFamily(), &begin, &end)); + + int32_t deleted_count = 0; + for (int32_t i = 0; i < 4300; i++) { + if (i < 1000 || i > 2000) { + ASSERT_EQ(Get(Key(i)), values[i]); + } else { + ReadOptions roptions; + std::string result; + Status s = db_->Get(roptions, Key(i), &result); + ASSERT_TRUE(s.IsNotFound() || s.ok()); + if (s.IsNotFound()) { + deleted_count++; + } + } + } + ASSERT_GT(deleted_count, 0); + begin_string = Key(5000); + end_string = Key(6000); + Slice begin1(begin_string); + Slice end1(end_string); + // Try deleting files in range which contain no keys + ASSERT_OK( + DeleteFilesInRange(db_, db_->DefaultColumnFamily(), &begin1, &end1)); + + // Push data from level 0 to level 1 to force all data to be deleted + // Note that we don't delete level 0 files + compact_options.change_level = true; + compact_options.target_level = 1; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_OK( + DeleteFilesInRange(db_, db_->DefaultColumnFamily(), nullptr, nullptr)); + + int32_t deleted_count2 = 0; + for (int32_t i = 0; i < 4300; i++) { + ReadOptions roptions; + std::string result; + ASSERT_TRUE(db_->Get(roptions, Key(i), &result).IsNotFound()); + deleted_count2++; + } + ASSERT_GT(deleted_count2, deleted_count); + const size_t new_num_files = CountFiles(); + ASSERT_GT(old_num_files, new_num_files); +} + +TEST_F(DBCompactionTest, DeleteFilesInRanges) { + Options options = CurrentOptions(); + options.write_buffer_size = 10 * 1024 * 1024; + options.max_bytes_for_level_multiplier = 2; + options.num_levels = 4; + options.max_background_compactions = 3; + options.disable_auto_compactions = true; + + DestroyAndReopen(options); + int32_t value_size = 10 * 1024; // 10 KB + + Random rnd(301); + std::map values; + + // file [0 => 100), [100 => 200), ... [900, 1000) + for (auto i = 0; i < 10; i++) { + for (auto j = 0; j < 100; j++) { + auto k = i * 100 + j; + values[k] = rnd.RandomString(value_size); + ASSERT_OK(Put(Key(k), values[k])); + } + ASSERT_OK(Flush()); + } + ASSERT_EQ("10", FilesPerLevel(0)); + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 2; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + ASSERT_EQ("0,0,10", FilesPerLevel(0)); + + // file [0 => 100), [200 => 300), ... [800, 900) + for (auto i = 0; i < 10; i += 2) { + for (auto j = 0; j < 100; j++) { + auto k = i * 100 + j; + ASSERT_OK(Put(Key(k), values[k])); + } + ASSERT_OK(Flush()); + } + ASSERT_EQ("5,0,10", FilesPerLevel(0)); + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr)); + ASSERT_EQ("0,5,10", FilesPerLevel(0)); + + // Delete files in range [0, 299] (inclusive) + { + auto begin_str1 = Key(0), end_str1 = Key(100); + auto begin_str2 = Key(100), end_str2 = Key(200); + auto begin_str3 = Key(200), end_str3 = Key(299); + Slice begin1(begin_str1), end1(end_str1); + Slice begin2(begin_str2), end2(end_str2); + Slice begin3(begin_str3), end3(end_str3); + std::vector ranges; + ranges.push_back(RangePtr(&begin1, &end1)); + ranges.push_back(RangePtr(&begin2, &end2)); + ranges.push_back(RangePtr(&begin3, &end3)); + ASSERT_OK(DeleteFilesInRanges(db_, db_->DefaultColumnFamily(), + ranges.data(), ranges.size())); + ASSERT_EQ("0,3,7", FilesPerLevel(0)); + + // Keys [0, 300) should not exist. + for (auto i = 0; i < 300; i++) { + ReadOptions ropts; + std::string result; + auto s = db_->Get(ropts, Key(i), &result); + ASSERT_TRUE(s.IsNotFound()); + } + for (auto i = 300; i < 1000; i++) { + ASSERT_EQ(Get(Key(i)), values[i]); + } + } + + // Delete files in range [600, 999) (exclusive) + { + auto begin_str1 = Key(600), end_str1 = Key(800); + auto begin_str2 = Key(700), end_str2 = Key(900); + auto begin_str3 = Key(800), end_str3 = Key(999); + Slice begin1(begin_str1), end1(end_str1); + Slice begin2(begin_str2), end2(end_str2); + Slice begin3(begin_str3), end3(end_str3); + std::vector ranges; + ranges.push_back(RangePtr(&begin1, &end1)); + ranges.push_back(RangePtr(&begin2, &end2)); + ranges.push_back(RangePtr(&begin3, &end3)); + ASSERT_OK(DeleteFilesInRanges(db_, db_->DefaultColumnFamily(), + ranges.data(), ranges.size(), false)); + ASSERT_EQ("0,1,4", FilesPerLevel(0)); + + // Keys [600, 900) should not exist. + for (auto i = 600; i < 900; i++) { + ReadOptions ropts; + std::string result; + auto s = db_->Get(ropts, Key(i), &result); + ASSERT_TRUE(s.IsNotFound()); + } + for (auto i = 300; i < 600; i++) { + ASSERT_EQ(Get(Key(i)), values[i]); + } + for (auto i = 900; i < 1000; i++) { + ASSERT_EQ(Get(Key(i)), values[i]); + } + } + + // Delete all files. + { + RangePtr range; + ASSERT_OK(DeleteFilesInRanges(db_, db_->DefaultColumnFamily(), &range, 1)); + ASSERT_EQ("", FilesPerLevel(0)); + + for (auto i = 0; i < 1000; i++) { + ReadOptions ropts; + std::string result; + auto s = db_->Get(ropts, Key(i), &result); + ASSERT_TRUE(s.IsNotFound()); + } + } +} + +TEST_F(DBCompactionTest, DeleteFileRangeFileEndpointsOverlapBug) { + // regression test for #2833: groups of files whose user-keys overlap at the + // endpoints could be split by `DeleteFilesInRange`. This caused old data to + // reappear, either because a new version of the key was removed, or a range + // deletion was partially dropped. It could also cause non-overlapping + // invariant to be violated if the files dropped by DeleteFilesInRange were + // a subset of files that a range deletion spans. + const int kNumL0Files = 2; + const int kValSize = 8 << 10; // 8KB + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = kNumL0Files; + options.target_file_size_base = 1 << 10; // 1KB + DestroyAndReopen(options); + + // The snapshot prevents key 1 from having its old version dropped. The low + // `target_file_size_base` ensures two keys will be in each output file. + const Snapshot* snapshot = nullptr; + Random rnd(301); + // The value indicates which flush the key belonged to, which is enough + // for us to determine the keys' relative ages. After L0 flushes finish, + // files look like: + // + // File 0: 0 -> vals[0], 1 -> vals[0] + // File 1: 1 -> vals[1], 2 -> vals[1] + // + // Then L0->L1 compaction happens, which outputs keys as follows: + // + // File 0: 0 -> vals[0], 1 -> vals[1] + // File 1: 1 -> vals[0], 2 -> vals[1] + // + // DeleteFilesInRange shouldn't be allowed to drop just file 0, as that + // would cause `1 -> vals[0]` (an older key) to reappear. + std::string vals[kNumL0Files]; + for (int i = 0; i < kNumL0Files; ++i) { + vals[i] = rnd.RandomString(kValSize); + ASSERT_OK(Put(Key(i), vals[i])); + ASSERT_OK(Put(Key(i + 1), vals[i])); + ASSERT_OK(Flush()); + if (i == 0) { + snapshot = db_->GetSnapshot(); + } + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // Verify `DeleteFilesInRange` can't drop only file 0 which would cause + // "1 -> vals[0]" to reappear. + std::string begin_str = Key(0), end_str = Key(1); + Slice begin = begin_str, end = end_str; + ASSERT_OK(DeleteFilesInRange(db_, db_->DefaultColumnFamily(), &begin, &end)); + ASSERT_EQ(vals[1], Get(Key(1))); + + db_->ReleaseSnapshot(snapshot); +} + +TEST_P(DBCompactionTestWithParam, TrivialMoveToLastLevelWithFiles) { + int32_t trivial_move = 0; + int32_t non_trivial_move = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:TrivialMove", + [&](void* /*arg*/) { trivial_move++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial", + [&](void* /*arg*/) { non_trivial_move++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Options options = CurrentOptions(); + options.write_buffer_size = 100000000; + options.max_subcompactions = max_subcompactions_; + DestroyAndReopen(options); + + int32_t value_size = 10 * 1024; // 10 KB + + Random rnd(301); + std::vector values; + // File with keys [ 0 => 99 ] + for (int i = 0; i < 100; i++) { + values.push_back(rnd.RandomString(value_size)); + ASSERT_OK(Put(Key(i), values[i])); + } + ASSERT_OK(Flush()); + + ASSERT_EQ("1", FilesPerLevel(0)); + // Compaction will do L0=>L1 (trivial move) then move L1 files to L3 + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 3; + compact_options.exclusive_manual_compaction = exclusive_manual_compaction_; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + ASSERT_EQ("0,0,0,1", FilesPerLevel(0)); + ASSERT_EQ(trivial_move, 1); + ASSERT_EQ(non_trivial_move, 0); + + // File with keys [ 100 => 199 ] + for (int i = 100; i < 200; i++) { + values.push_back(rnd.RandomString(value_size)); + ASSERT_OK(Put(Key(i), values[i])); + } + ASSERT_OK(Flush()); + + ASSERT_EQ("1,0,0,1", FilesPerLevel(0)); + CompactRangeOptions cro; + cro.exclusive_manual_compaction = exclusive_manual_compaction_; + // Compaction will do L0=>L1 L1=>L2 L2=>L3 (3 trivial moves) + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,0,2", FilesPerLevel(0)); + ASSERT_EQ(trivial_move, 4); + ASSERT_EQ(non_trivial_move, 0); + + for (int i = 0; i < 200; i++) { + ASSERT_EQ(Get(Key(i)), values[i]); + } + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_P(DBCompactionTestWithParam, LevelCompactionThirdPath) { + Options options = CurrentOptions(); + options.db_paths.emplace_back(dbname_, 500 * 1024); + options.db_paths.emplace_back(dbname_ + "_2", 4 * 1024 * 1024); + options.db_paths.emplace_back(dbname_ + "_3", 1024 * 1024 * 1024); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + options.compaction_style = kCompactionStyleLevel; + options.write_buffer_size = 110 << 10; // 110KB + options.arena_block_size = 4 << 10; + options.level0_file_num_compaction_trigger = 2; + options.num_levels = 4; + options.max_bytes_for_level_base = 400 * 1024; + options.max_subcompactions = max_subcompactions_; + + DestroyAndReopen(options); + + Random rnd(301); + int key_idx = 0; + + // First three 110KB files are not going to second path. + // After that, (100K, 200K) + for (int num = 0; num < 3; num++) { + GenerateNewFile(&rnd, &key_idx); + } + + // Another 110KB triggers a compaction to 400K file to fill up first path + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(3, GetSstFileCount(options.db_paths[1].path)); + + // (1, 4) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,4", FilesPerLevel(0)); + ASSERT_EQ(4, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(1, GetSstFileCount(dbname_)); + + // (1, 4, 1) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,4,1", FilesPerLevel(0)); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[2].path)); + ASSERT_EQ(4, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(1, GetSstFileCount(dbname_)); + + // (1, 4, 2) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,4,2", FilesPerLevel(0)); + ASSERT_EQ(2, GetSstFileCount(options.db_paths[2].path)); + ASSERT_EQ(4, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(1, GetSstFileCount(dbname_)); + + // (1, 4, 3) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,4,3", FilesPerLevel(0)); + ASSERT_EQ(3, GetSstFileCount(options.db_paths[2].path)); + ASSERT_EQ(4, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(1, GetSstFileCount(dbname_)); + + // (1, 4, 4) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,4,4", FilesPerLevel(0)); + ASSERT_EQ(4, GetSstFileCount(options.db_paths[2].path)); + ASSERT_EQ(4, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(1, GetSstFileCount(dbname_)); + + // (1, 4, 5) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,4,5", FilesPerLevel(0)); + ASSERT_EQ(5, GetSstFileCount(options.db_paths[2].path)); + ASSERT_EQ(4, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(1, GetSstFileCount(dbname_)); + + // (1, 4, 6) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,4,6", FilesPerLevel(0)); + ASSERT_EQ(6, GetSstFileCount(options.db_paths[2].path)); + ASSERT_EQ(4, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(1, GetSstFileCount(dbname_)); + + // (1, 4, 7) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,4,7", FilesPerLevel(0)); + ASSERT_EQ(7, GetSstFileCount(options.db_paths[2].path)); + ASSERT_EQ(4, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(1, GetSstFileCount(dbname_)); + + // (1, 4, 8) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,4,8", FilesPerLevel(0)); + ASSERT_EQ(8, GetSstFileCount(options.db_paths[2].path)); + ASSERT_EQ(4, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(1, GetSstFileCount(dbname_)); + + for (int i = 0; i < key_idx; i++) { + auto v = Get(Key(i)); + ASSERT_NE(v, "NOT_FOUND"); + ASSERT_TRUE(v.size() == 1 || v.size() == 990); + } + + Reopen(options); + + for (int i = 0; i < key_idx; i++) { + auto v = Get(Key(i)); + ASSERT_NE(v, "NOT_FOUND"); + ASSERT_TRUE(v.size() == 1 || v.size() == 990); + } + + Destroy(options); +} + +TEST_P(DBCompactionTestWithParam, LevelCompactionPathUse) { + Options options = CurrentOptions(); + options.db_paths.emplace_back(dbname_, 500 * 1024); + options.db_paths.emplace_back(dbname_ + "_2", 4 * 1024 * 1024); + options.db_paths.emplace_back(dbname_ + "_3", 1024 * 1024 * 1024); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + options.compaction_style = kCompactionStyleLevel; + options.write_buffer_size = 110 << 10; // 110KB + options.arena_block_size = 4 << 10; + options.level0_file_num_compaction_trigger = 2; + options.num_levels = 4; + options.max_bytes_for_level_base = 400 * 1024; + options.max_subcompactions = max_subcompactions_; + + DestroyAndReopen(options); + + Random rnd(301); + int key_idx = 0; + + // Always gets compacted into 1 Level1 file, + // 0/1 Level 0 file + for (int num = 0; num < 3; num++) { + key_idx = 0; + GenerateNewFile(&rnd, &key_idx); + } + + key_idx = 0; + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); + + key_idx = 0; + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,1", FilesPerLevel(0)); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(1, GetSstFileCount(dbname_)); + + key_idx = 0; + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("0,1", FilesPerLevel(0)); + ASSERT_EQ(0, GetSstFileCount(options.db_paths[2].path)); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(0, GetSstFileCount(dbname_)); + + key_idx = 0; + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,1", FilesPerLevel(0)); + ASSERT_EQ(0, GetSstFileCount(options.db_paths[2].path)); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(1, GetSstFileCount(dbname_)); + + key_idx = 0; + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("0,1", FilesPerLevel(0)); + ASSERT_EQ(0, GetSstFileCount(options.db_paths[2].path)); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(0, GetSstFileCount(dbname_)); + + key_idx = 0; + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,1", FilesPerLevel(0)); + ASSERT_EQ(0, GetSstFileCount(options.db_paths[2].path)); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(1, GetSstFileCount(dbname_)); + + key_idx = 0; + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("0,1", FilesPerLevel(0)); + ASSERT_EQ(0, GetSstFileCount(options.db_paths[2].path)); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(0, GetSstFileCount(dbname_)); + + key_idx = 0; + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,1", FilesPerLevel(0)); + ASSERT_EQ(0, GetSstFileCount(options.db_paths[2].path)); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(1, GetSstFileCount(dbname_)); + + key_idx = 0; + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("0,1", FilesPerLevel(0)); + ASSERT_EQ(0, GetSstFileCount(options.db_paths[2].path)); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(0, GetSstFileCount(dbname_)); + + key_idx = 0; + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,1", FilesPerLevel(0)); + ASSERT_EQ(0, GetSstFileCount(options.db_paths[2].path)); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(1, GetSstFileCount(dbname_)); + + for (int i = 0; i < key_idx; i++) { + auto v = Get(Key(i)); + ASSERT_NE(v, "NOT_FOUND"); + ASSERT_TRUE(v.size() == 1 || v.size() == 990); + } + + Reopen(options); + + for (int i = 0; i < key_idx; i++) { + auto v = Get(Key(i)); + ASSERT_NE(v, "NOT_FOUND"); + ASSERT_TRUE(v.size() == 1 || v.size() == 990); + } + + Destroy(options); +} + +TEST_P(DBCompactionTestWithParam, LevelCompactionCFPathUse) { + Options options = CurrentOptions(); + options.db_paths.emplace_back(dbname_, 500 * 1024); + options.db_paths.emplace_back(dbname_ + "_2", 4 * 1024 * 1024); + options.db_paths.emplace_back(dbname_ + "_3", 1024 * 1024 * 1024); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + options.compaction_style = kCompactionStyleLevel; + options.write_buffer_size = 110 << 10; // 110KB + options.arena_block_size = 4 << 10; + options.level0_file_num_compaction_trigger = 2; + options.num_levels = 4; + options.max_bytes_for_level_base = 400 * 1024; + options.max_subcompactions = max_subcompactions_; + + std::vector option_vector; + option_vector.emplace_back(options); + ColumnFamilyOptions cf_opt1(options), cf_opt2(options); + // Configure CF1 specific paths. + cf_opt1.cf_paths.emplace_back(dbname_ + "cf1", 500 * 1024); + cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_2", 4 * 1024 * 1024); + cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_3", 1024 * 1024 * 1024); + option_vector.emplace_back(DBOptions(options), cf_opt1); + CreateColumnFamilies({"one"}, option_vector[1]); + + // Configure CF2 specific paths. + cf_opt2.cf_paths.emplace_back(dbname_ + "cf2", 500 * 1024); + cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_2", 4 * 1024 * 1024); + cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_3", 1024 * 1024 * 1024); + option_vector.emplace_back(DBOptions(options), cf_opt2); + CreateColumnFamilies({"two"}, option_vector[2]); + + ReopenWithColumnFamilies({"default", "one", "two"}, option_vector); + + Random rnd(301); + int key_idx = 0; + int key_idx1 = 0; + int key_idx2 = 0; + + auto generate_file = [&]() { + GenerateNewFile(0, &rnd, &key_idx); + GenerateNewFile(1, &rnd, &key_idx1); + GenerateNewFile(2, &rnd, &key_idx2); + }; + + auto check_sstfilecount = [&](int path_id, int expected) { + ASSERT_EQ(expected, GetSstFileCount(options.db_paths[path_id].path)); + ASSERT_EQ(expected, GetSstFileCount(cf_opt1.cf_paths[path_id].path)); + ASSERT_EQ(expected, GetSstFileCount(cf_opt2.cf_paths[path_id].path)); + }; + + auto check_filesperlevel = [&](const std::string& expected) { + ASSERT_EQ(expected, FilesPerLevel(0)); + ASSERT_EQ(expected, FilesPerLevel(1)); + ASSERT_EQ(expected, FilesPerLevel(2)); + }; + + auto check_getvalues = [&]() { + for (int i = 0; i < key_idx; i++) { + auto v = Get(0, Key(i)); + ASSERT_NE(v, "NOT_FOUND"); + ASSERT_TRUE(v.size() == 1 || v.size() == 990); + } + + for (int i = 0; i < key_idx1; i++) { + auto v = Get(1, Key(i)); + ASSERT_NE(v, "NOT_FOUND"); + ASSERT_TRUE(v.size() == 1 || v.size() == 990); + } + + for (int i = 0; i < key_idx2; i++) { + auto v = Get(2, Key(i)); + ASSERT_NE(v, "NOT_FOUND"); + ASSERT_TRUE(v.size() == 1 || v.size() == 990); + } + }; + + // Check that default column family uses db_paths. + // And Column family "one" uses cf_paths. + + // The compaction in level0 outputs the sst files in level1. + // The first path cannot hold level1's data(400KB+400KB > 500KB), + // so every compaction move a sst file to second path. Please + // refer to LevelCompactionBuilder::GetPathId. + for (int num = 0; num < 3; num++) { + generate_file(); + } + check_sstfilecount(0, 1); + check_sstfilecount(1, 2); + + generate_file(); + check_sstfilecount(1, 3); + + // (1, 4) + generate_file(); + check_filesperlevel("1,4"); + check_sstfilecount(1, 4); + check_sstfilecount(0, 1); + + // (1, 4, 1) + generate_file(); + check_filesperlevel("1,4,1"); + check_sstfilecount(2, 1); + check_sstfilecount(1, 4); + check_sstfilecount(0, 1); + + // (1, 4, 2) + generate_file(); + check_filesperlevel("1,4,2"); + check_sstfilecount(2, 2); + check_sstfilecount(1, 4); + check_sstfilecount(0, 1); + + check_getvalues(); + + { // Also verify GetLiveFilesStorageInfo with db_paths / cf_paths + std::vector new_infos; + LiveFilesStorageInfoOptions lfsio; + lfsio.wal_size_for_flush = UINT64_MAX; // no flush + ASSERT_OK(db_->GetLiveFilesStorageInfo(lfsio, &new_infos)); + std::unordered_map live_sst_by_dir; + for (auto& info : new_infos) { + if (info.file_type == kTableFile) { + live_sst_by_dir[info.directory]++; + // Verify file on disk (no directory confusion) + uint64_t size; + ASSERT_OK(env_->GetFileSize( + info.directory + "/" + info.relative_filename, &size)); + ASSERT_EQ(info.size, size); + } + } + ASSERT_EQ(3U * 3U, live_sst_by_dir.size()); + for (auto& paths : {options.db_paths, cf_opt1.cf_paths, cf_opt2.cf_paths}) { + ASSERT_EQ(1, live_sst_by_dir[paths[0].path]); + ASSERT_EQ(4, live_sst_by_dir[paths[1].path]); + ASSERT_EQ(2, live_sst_by_dir[paths[2].path]); + } + } + + ReopenWithColumnFamilies({"default", "one", "two"}, option_vector); + + check_getvalues(); + + Destroy(options, true); +} + +TEST_P(DBCompactionTestWithParam, ConvertCompactionStyle) { + Random rnd(301); + int max_key_level_insert = 200; + int max_key_universal_insert = 600; + + // Stage 1: generate a db with level compaction + Options options = CurrentOptions(); + options.write_buffer_size = 110 << 10; // 110KB + options.arena_block_size = 4 << 10; + options.num_levels = 4; + options.level0_file_num_compaction_trigger = 3; + options.max_bytes_for_level_base = 500 << 10; // 500KB + options.max_bytes_for_level_multiplier = 1; + options.target_file_size_base = 200 << 10; // 200KB + options.target_file_size_multiplier = 1; + options.max_subcompactions = max_subcompactions_; + CreateAndReopenWithCF({"pikachu"}, options); + + for (int i = 0; i <= max_key_level_insert; i++) { + // each value is 10K + ASSERT_OK(Put(1, Key(i), rnd.RandomString(10000))); + } + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_GT(TotalTableFiles(1, 4), 1); + int non_level0_num_files = 0; + for (int i = 1; i < options.num_levels; i++) { + non_level0_num_files += NumTableFilesAtLevel(i, 1); + } + ASSERT_GT(non_level0_num_files, 0); + + // Stage 2: reopen with universal compaction - should fail + options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = 1; + options = CurrentOptions(options); + Status s = TryReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_TRUE(s.IsInvalidArgument()); + + // Stage 3: compact into a single file and move the file to level 0 + options = CurrentOptions(); + options.disable_auto_compactions = true; + options.target_file_size_base = INT_MAX; + options.target_file_size_multiplier = 1; + options.max_bytes_for_level_base = INT_MAX; + options.max_bytes_for_level_multiplier = 1; + options.num_levels = 4; + options = CurrentOptions(options); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 0; + // cannot use kForceOptimized here because the compaction here is expected + // to generate one output file + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + compact_options.exclusive_manual_compaction = exclusive_manual_compaction_; + ASSERT_OK( + dbfull()->CompactRange(compact_options, handles_[1], nullptr, nullptr)); + + // Only 1 file in L0 + ASSERT_EQ("1", FilesPerLevel(1)); + + // Stage 4: re-open in universal compaction style and do some db operations + options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = 4; + options.write_buffer_size = 110 << 10; // 110KB + options.arena_block_size = 4 << 10; + options.level0_file_num_compaction_trigger = 3; + options = CurrentOptions(options); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + + options.num_levels = 1; + ReopenWithColumnFamilies({"default", "pikachu"}, options); + + for (int i = max_key_level_insert / 2; i <= max_key_universal_insert; i++) { + ASSERT_OK(Put(1, Key(i), rnd.RandomString(10000))); + } + ASSERT_OK(dbfull()->Flush(FlushOptions())); + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + for (int i = 1; i < options.num_levels; i++) { + ASSERT_EQ(NumTableFilesAtLevel(i, 1), 0); + } + + // verify keys inserted in both level compaction style and universal + // compaction style + std::string keys_in_db; + Iterator* iter = dbfull()->NewIterator(ReadOptions(), handles_[1]); + ASSERT_OK(iter->status()); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + keys_in_db.append(iter->key().ToString()); + keys_in_db.push_back(','); + } + delete iter; + + std::string expected_keys; + for (int i = 0; i <= max_key_universal_insert; i++) { + expected_keys.append(Key(i)); + expected_keys.push_back(','); + } + + ASSERT_EQ(keys_in_db, expected_keys); +} + +TEST_F(DBCompactionTest, L0_CompactionBug_Issue44_a) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "b", "v")); + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + ASSERT_OK(Delete(1, "b")); + ASSERT_OK(Delete(1, "a")); + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + ASSERT_OK(Delete(1, "a")); + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "a", "v")); + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + ASSERT_EQ("(a->v)", Contents(1)); + env_->SleepForMicroseconds(1000000); // Wait for compaction to finish + ASSERT_EQ("(a->v)", Contents(1)); + } while (ChangeCompactOptions()); +} + +TEST_F(DBCompactionTest, L0_CompactionBug_Issue44_b) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "", "")); + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + ASSERT_OK(Delete(1, "e")); + ASSERT_OK(Put(1, "", "")); + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "c", "cv")); + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "", "")); + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "", "")); + env_->SleepForMicroseconds(1000000); // Wait for compaction to finish + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "d", "dv")); + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "", "")); + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + ASSERT_OK(Delete(1, "d")); + ASSERT_OK(Delete(1, "b")); + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + ASSERT_EQ("(->)(c->cv)", Contents(1)); + env_->SleepForMicroseconds(1000000); // Wait for compaction to finish + ASSERT_EQ("(->)(c->cv)", Contents(1)); + } while (ChangeCompactOptions()); +} + +TEST_F(DBCompactionTest, ManualAutoRace) { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BGWorkCompaction", "DBCompactionTest::ManualAutoRace:1"}, + {"DBImpl::RunManualCompaction:WaitScheduled", + "BackgroundCallCompaction:0"}}); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put(1, "foo", "")); + ASSERT_OK(Put(1, "bar", "")); + ASSERT_OK(Flush(1)); + ASSERT_OK(Put(1, "foo", "")); + ASSERT_OK(Put(1, "bar", "")); + // Generate four files in CF 0, which should trigger an auto compaction + ASSERT_OK(Put("foo", "")); + ASSERT_OK(Put("bar", "")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("foo", "")); + ASSERT_OK(Put("bar", "")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("foo", "")); + ASSERT_OK(Put("bar", "")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("foo", "")); + ASSERT_OK(Put("bar", "")); + ASSERT_OK(Flush()); + + // The auto compaction is scheduled but waited until here + TEST_SYNC_POINT("DBCompactionTest::ManualAutoRace:1"); + // The auto compaction will wait until the manual compaction is registerd + // before processing so that it will be cancelled. + CompactRangeOptions cro; + cro.exclusive_manual_compaction = true; + ASSERT_OK(dbfull()->CompactRange(cro, handles_[1], nullptr, nullptr)); + ASSERT_EQ("0,1", FilesPerLevel(1)); + + // Eventually the cancelled compaction will be rescheduled and executed. + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,1", FilesPerLevel(0)); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_P(DBCompactionTestWithParam, ManualCompaction) { + Options options = CurrentOptions(); + options.max_subcompactions = max_subcompactions_; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + CreateAndReopenWithCF({"pikachu"}, options); + + // iter - 0 with 7 levels + // iter - 1 with 3 levels + for (int iter = 0; iter < 2; ++iter) { + MakeTables(3, "p", "q", 1); + ASSERT_EQ("1,1,1", FilesPerLevel(1)); + + // Compaction range falls before files + Compact(1, "", "c"); + ASSERT_EQ("1,1,1", FilesPerLevel(1)); + + // Compaction range falls after files + Compact(1, "r", "z"); + ASSERT_EQ("1,1,1", FilesPerLevel(1)); + + // Compaction range overlaps files + Compact(1, "p", "q"); + ASSERT_EQ("0,0,1", FilesPerLevel(1)); + + // Populate a different range + MakeTables(3, "c", "e", 1); + ASSERT_EQ("1,1,2", FilesPerLevel(1)); + + // Compact just the new range + Compact(1, "b", "f"); + ASSERT_EQ("0,0,2", FilesPerLevel(1)); + + // Compact all + MakeTables(1, "a", "z", 1); + ASSERT_EQ("1,0,2", FilesPerLevel(1)); + + uint64_t prev_block_cache_add = + options.statistics->getTickerCount(BLOCK_CACHE_ADD); + CompactRangeOptions cro; + cro.exclusive_manual_compaction = exclusive_manual_compaction_; + ASSERT_OK(db_->CompactRange(cro, handles_[1], nullptr, nullptr)); + // Verify manual compaction doesn't fill block cache + ASSERT_EQ(prev_block_cache_add, + options.statistics->getTickerCount(BLOCK_CACHE_ADD)); + + ASSERT_EQ("0,0,1", FilesPerLevel(1)); + + if (iter == 0) { + options = CurrentOptions(); + options.num_levels = 3; + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + } + } +} + +TEST_P(DBCompactionTestWithParam, ManualLevelCompactionOutputPathId) { + Options options = CurrentOptions(); + options.db_paths.emplace_back(dbname_ + "_2", 2 * 10485760); + options.db_paths.emplace_back(dbname_ + "_3", 100 * 10485760); + options.db_paths.emplace_back(dbname_ + "_4", 120 * 10485760); + options.max_subcompactions = max_subcompactions_; + CreateAndReopenWithCF({"pikachu"}, options); + + // iter - 0 with 7 levels + // iter - 1 with 3 levels + for (int iter = 0; iter < 2; ++iter) { + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Put(1, "p", "begin")); + ASSERT_OK(Put(1, "q", "end")); + ASSERT_OK(Flush(1)); + } + ASSERT_EQ("3", FilesPerLevel(1)); + ASSERT_EQ(3, GetSstFileCount(options.db_paths[0].path)); + ASSERT_EQ(0, GetSstFileCount(dbname_)); + + // Compaction range falls before files + Compact(1, "", "c"); + ASSERT_EQ("3", FilesPerLevel(1)); + + // Compaction range falls after files + Compact(1, "r", "z"); + ASSERT_EQ("3", FilesPerLevel(1)); + + // Compaction range overlaps files + Compact(1, "p", "q", 1); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,1", FilesPerLevel(1)); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(0, GetSstFileCount(options.db_paths[0].path)); + ASSERT_EQ(0, GetSstFileCount(dbname_)); + + // Populate a different range + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Put(1, "c", "begin")); + ASSERT_OK(Put(1, "e", "end")); + ASSERT_OK(Flush(1)); + } + ASSERT_EQ("3,1", FilesPerLevel(1)); + + // Compact just the new range + Compact(1, "b", "f", 1); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,2", FilesPerLevel(1)); + ASSERT_EQ(2, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(0, GetSstFileCount(options.db_paths[0].path)); + ASSERT_EQ(0, GetSstFileCount(dbname_)); + + // Compact all + ASSERT_OK(Put(1, "a", "begin")); + ASSERT_OK(Put(1, "z", "end")); + ASSERT_OK(Flush(1)); + ASSERT_EQ("1,2", FilesPerLevel(1)); + ASSERT_EQ(2, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[0].path)); + CompactRangeOptions compact_options; + compact_options.target_path_id = 1; + compact_options.exclusive_manual_compaction = exclusive_manual_compaction_; + ASSERT_OK( + db_->CompactRange(compact_options, handles_[1], nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ("0,1", FilesPerLevel(1)); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(0, GetSstFileCount(options.db_paths[0].path)); + ASSERT_EQ(0, GetSstFileCount(dbname_)); + + if (iter == 0) { + DestroyAndReopen(options); + options = CurrentOptions(); + options.db_paths.emplace_back(dbname_ + "_2", 2 * 10485760); + options.db_paths.emplace_back(dbname_ + "_3", 100 * 10485760); + options.db_paths.emplace_back(dbname_ + "_4", 120 * 10485760); + options.max_background_flushes = 1; + options.num_levels = 3; + options.create_if_missing = true; + CreateAndReopenWithCF({"pikachu"}, options); + } + } +} + +TEST_F(DBCompactionTest, FilesDeletedAfterCompaction) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "foo", "v2")); + Compact(1, "a", "z"); + const size_t num_files = CountLiveFiles(); + for (int i = 0; i < 10; i++) { + ASSERT_OK(Put(1, "foo", "v2")); + Compact(1, "a", "z"); + } + ASSERT_EQ(CountLiveFiles(), num_files); + } while (ChangeCompactOptions()); +} + +// Check level comapction with compact files +TEST_P(DBCompactionTestWithParam, DISABLED_CompactFilesOnLevelCompaction) { + const int kTestKeySize = 16; + const int kTestValueSize = 984; + const int kEntrySize = kTestKeySize + kTestValueSize; + const int kEntriesPerBuffer = 100; + Options options; + options.create_if_missing = true; + options.write_buffer_size = kEntrySize * kEntriesPerBuffer; + options.compaction_style = kCompactionStyleLevel; + options.target_file_size_base = options.write_buffer_size; + options.max_bytes_for_level_base = options.target_file_size_base * 2; + options.level0_stop_writes_trigger = 2; + options.max_bytes_for_level_multiplier = 2; + options.compression = kNoCompression; + options.max_subcompactions = max_subcompactions_; + options = CurrentOptions(options); + CreateAndReopenWithCF({"pikachu"}, options); + + Random rnd(301); + for (int key = 64 * kEntriesPerBuffer; key >= 0; --key) { + ASSERT_OK(Put(1, std::to_string(key), rnd.RandomString(kTestValueSize))); + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ColumnFamilyMetaData cf_meta; + dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta); + int output_level = static_cast(cf_meta.levels.size()) - 1; + for (int file_picked = 5; file_picked > 0; --file_picked) { + std::set overlapping_file_names; + std::vector compaction_input_file_names; + for (int f = 0; f < file_picked; ++f) { + int level = 0; + auto file_meta = PickFileRandomly(cf_meta, &rnd, &level); + compaction_input_file_names.push_back(file_meta->name); + GetOverlappingFileNumbersForLevelCompaction( + cf_meta, options.comparator, level, output_level, file_meta, + &overlapping_file_names); + } + + ASSERT_OK(dbfull()->CompactFiles(CompactionOptions(), handles_[1], + compaction_input_file_names, + output_level)); + + // Make sure all overlapping files do not exist after compaction + dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta); + VerifyCompactionResult(cf_meta, overlapping_file_names); + } + + // make sure all key-values are still there. + for (int key = 64 * kEntriesPerBuffer; key >= 0; --key) { + ASSERT_NE(Get(1, std::to_string(key)), "NOT_FOUND"); + } +} + +TEST_P(DBCompactionTestWithParam, PartialCompactionFailure) { + Options options; + const int kKeySize = 16; + const int kKvSize = 1000; + const int kKeysPerBuffer = 100; + const int kNumL1Files = 5; + options.create_if_missing = true; + options.write_buffer_size = kKeysPerBuffer * kKvSize; + options.max_write_buffer_number = 2; + options.target_file_size_base = + options.write_buffer_size * (options.max_write_buffer_number - 1); + options.level0_file_num_compaction_trigger = kNumL1Files; + options.max_bytes_for_level_base = + options.level0_file_num_compaction_trigger * + options.target_file_size_base; + options.max_bytes_for_level_multiplier = 2; + options.compression = kNoCompression; + options.max_subcompactions = max_subcompactions_; + + env_->SetBackgroundThreads(1, Env::HIGH); + env_->SetBackgroundThreads(1, Env::LOW); + // stop the compaction thread until we simulate the file creation failure. + test::SleepingBackgroundTask sleeping_task_low; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::LOW); + + options.env = env_; + + DestroyAndReopen(options); + + const int kNumInsertedKeys = options.level0_file_num_compaction_trigger * + (options.max_write_buffer_number - 1) * + kKeysPerBuffer; + + Random rnd(301); + std::vector keys; + std::vector values; + for (int k = 0; k < kNumInsertedKeys; ++k) { + keys.emplace_back(rnd.RandomString(kKeySize)); + values.emplace_back(rnd.RandomString(kKvSize - kKeySize)); + ASSERT_OK(Put(Slice(keys[k]), Slice(values[k]))); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + + ASSERT_OK(dbfull()->TEST_FlushMemTable(true)); + // Make sure the number of L0 files can trigger compaction. + ASSERT_GE(NumTableFilesAtLevel(0), + options.level0_file_num_compaction_trigger); + + auto previous_num_level0_files = NumTableFilesAtLevel(0); + + // Fail the first file creation. + env_->non_writable_count_ = 1; + sleeping_task_low.WakeUp(); + sleeping_task_low.WaitUntilDone(); + + // Expect compaction to fail here as one file will fail its + // creation. + ASSERT_TRUE(!dbfull()->TEST_WaitForCompact().ok()); + + // Verify L0 -> L1 compaction does fail. + ASSERT_EQ(NumTableFilesAtLevel(1), 0); + + // Verify all L0 files are still there. + ASSERT_EQ(NumTableFilesAtLevel(0), previous_num_level0_files); + + // All key-values must exist after compaction fails. + for (int k = 0; k < kNumInsertedKeys; ++k) { + ASSERT_EQ(values[k], Get(keys[k])); + } + + env_->non_writable_count_ = 0; + + // Make sure RocksDB will not get into corrupted state. + Reopen(options); + + // Verify again after reopen. + for (int k = 0; k < kNumInsertedKeys; ++k) { + ASSERT_EQ(values[k], Get(keys[k])); + } +} + +TEST_P(DBCompactionTestWithParam, DeleteMovedFileAfterCompaction) { + // iter 1 -- delete_obsolete_files_period_micros == 0 + for (int iter = 0; iter < 2; ++iter) { + // This test triggers move compaction and verifies that the file is not + // deleted when it's part of move compaction + Options options = CurrentOptions(); + options.env = env_; + if (iter == 1) { + options.delete_obsolete_files_period_micros = 0; + } + options.create_if_missing = true; + options.level0_file_num_compaction_trigger = + 2; // trigger compaction when we have 2 files + OnFileDeletionListener* listener = new OnFileDeletionListener(); + options.listeners.emplace_back(listener); + options.max_subcompactions = max_subcompactions_; + DestroyAndReopen(options); + + Random rnd(301); + // Create two 1MB sst files + for (int i = 0; i < 2; ++i) { + // Create 1MB sst file + for (int j = 0; j < 100; ++j) { + ASSERT_OK(Put(Key(i * 50 + j), rnd.RandomString(10 * 1024))); + } + ASSERT_OK(Flush()); + } + // this should execute L0->L1 + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,1", FilesPerLevel(0)); + + // block compactions + test::SleepingBackgroundTask sleeping_task; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task, + Env::Priority::LOW); + + options.max_bytes_for_level_base = 1024 * 1024; // 1 MB + Reopen(options); + std::unique_ptr iterator(db_->NewIterator(ReadOptions())); + ASSERT_EQ("0,1", FilesPerLevel(0)); + // let compactions go + sleeping_task.WakeUp(); + sleeping_task.WaitUntilDone(); + + // this should execute L1->L2 (move) + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ("0,0,1", FilesPerLevel(0)); + + std::vector metadata; + db_->GetLiveFilesMetaData(&metadata); + ASSERT_EQ(metadata.size(), 1U); + auto moved_file_name = metadata[0].name; + + // Create two more 1MB sst files + for (int i = 0; i < 2; ++i) { + // Create 1MB sst file + for (int j = 0; j < 100; ++j) { + ASSERT_OK(Put(Key(i * 50 + j + 100), rnd.RandomString(10 * 1024))); + } + ASSERT_OK(Flush()); + } + // this should execute both L0->L1 and L1->L2 (merge with previous file) + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ("0,0,2", FilesPerLevel(0)); + + // iterator is holding the file + ASSERT_OK(env_->FileExists(dbname_ + moved_file_name)); + + listener->SetExpectedFileName(dbname_ + moved_file_name); + ASSERT_OK(iterator->status()); + iterator.reset(); + + // this file should have been compacted away + ASSERT_NOK(env_->FileExists(dbname_ + moved_file_name)); + listener->VerifyMatchedCount(1); + } +} + +TEST_P(DBCompactionTestWithParam, CompressLevelCompaction) { + if (!Zlib_Supported()) { + return; + } + Options options = CurrentOptions(); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + options.compaction_style = kCompactionStyleLevel; + options.write_buffer_size = 110 << 10; // 110KB + options.arena_block_size = 4 << 10; + options.level0_file_num_compaction_trigger = 2; + options.num_levels = 4; + options.max_bytes_for_level_base = 400 * 1024; + options.max_subcompactions = max_subcompactions_; + // First two levels have no compression, so that a trivial move between + // them will be allowed. Level 2 has Zlib compression so that a trivial + // move to level 3 will not be allowed + options.compression_per_level = {kNoCompression, kNoCompression, + kZlibCompression}; + int matches = 0, didnt_match = 0, trivial_move = 0, non_trivial = 0; + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "Compaction::InputCompressionMatchesOutput:Matches", + [&](void* /*arg*/) { matches++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "Compaction::InputCompressionMatchesOutput:DidntMatch", + [&](void* /*arg*/) { didnt_match++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial", + [&](void* /*arg*/) { non_trivial++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:TrivialMove", + [&](void* /*arg*/) { trivial_move++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Reopen(options); + + Random rnd(301); + int key_idx = 0; + + // First three 110KB files are going to level 0 + // After that, (100K, 200K) + for (int num = 0; num < 3; num++) { + GenerateNewFile(&rnd, &key_idx); + } + + // Another 110KB triggers a compaction to 400K file to fill up level 0 + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(4, GetSstFileCount(dbname_)); + + // (1, 4) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,4", FilesPerLevel(0)); + + // (1, 4, 1) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,4,1", FilesPerLevel(0)); + + // (1, 4, 2) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,4,2", FilesPerLevel(0)); + + // (1, 4, 3) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,4,3", FilesPerLevel(0)); + + // (1, 4, 4) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,4,4", FilesPerLevel(0)); + + // (1, 4, 5) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,4,5", FilesPerLevel(0)); + + // (1, 4, 6) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,4,6", FilesPerLevel(0)); + + // (1, 4, 7) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,4,7", FilesPerLevel(0)); + + // (1, 4, 8) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,4,8", FilesPerLevel(0)); + + ASSERT_EQ(matches, 12); + // Currently, the test relies on the number of calls to + // InputCompressionMatchesOutput() per compaction. + const int kCallsToInputCompressionMatch = 2; + ASSERT_EQ(didnt_match, 8 * kCallsToInputCompressionMatch); + ASSERT_EQ(trivial_move, 12); + ASSERT_EQ(non_trivial, 8); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + for (int i = 0; i < key_idx; i++) { + auto v = Get(Key(i)); + ASSERT_NE(v, "NOT_FOUND"); + ASSERT_TRUE(v.size() == 1 || v.size() == 990); + } + + Reopen(options); + + for (int i = 0; i < key_idx; i++) { + auto v = Get(Key(i)); + ASSERT_NE(v, "NOT_FOUND"); + ASSERT_TRUE(v.size() == 1 || v.size() == 990); + } + + Destroy(options); +} + +TEST_F(DBCompactionTest, SanitizeCompactionOptionsTest) { + Options options = CurrentOptions(); + options.max_background_compactions = 5; + options.soft_pending_compaction_bytes_limit = 0; + options.hard_pending_compaction_bytes_limit = 100; + options.create_if_missing = true; + DestroyAndReopen(options); + ASSERT_EQ(100, db_->GetOptions().soft_pending_compaction_bytes_limit); + + options.max_background_compactions = 3; + options.soft_pending_compaction_bytes_limit = 200; + options.hard_pending_compaction_bytes_limit = 150; + DestroyAndReopen(options); + ASSERT_EQ(150, db_->GetOptions().soft_pending_compaction_bytes_limit); +} + +// This tests for a bug that could cause two level0 compactions running +// concurrently +// TODO(aekmekji): Make sure that the reason this fails when run with +// max_subcompactions > 1 is not a correctness issue but just inherent to +// running parallel L0-L1 compactions +TEST_F(DBCompactionTest, SuggestCompactRangeNoTwoLevel0Compactions) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleLevel; + options.write_buffer_size = 110 << 10; + options.arena_block_size = 4 << 10; + options.level0_file_num_compaction_trigger = 4; + options.num_levels = 4; + options.compression = kNoCompression; + options.max_bytes_for_level_base = 450 << 10; + options.target_file_size_base = 98 << 10; + options.max_write_buffer_number = 2; + options.max_background_compactions = 2; + + DestroyAndReopen(options); + + // fill up the DB + Random rnd(301); + for (int num = 0; num < 10; num++) { + GenerateNewRandomFile(&rnd); + } + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"CompactionJob::Run():Start", + "DBCompactionTest::SuggestCompactRangeNoTwoLevel0Compactions:1"}, + {"DBCompactionTest::SuggestCompactRangeNoTwoLevel0Compactions:2", + "CompactionJob::Run():End"}}); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // trigger L0 compaction + for (int num = 0; num < options.level0_file_num_compaction_trigger + 1; + num++) { + GenerateNewRandomFile(&rnd, /* nowait */ true); + ASSERT_OK(Flush()); + } + + TEST_SYNC_POINT( + "DBCompactionTest::SuggestCompactRangeNoTwoLevel0Compactions:1"); + + GenerateNewRandomFile(&rnd, /* nowait */ true); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(experimental::SuggestCompactRange(db_, nullptr, nullptr)); + for (int num = 0; num < options.level0_file_num_compaction_trigger + 1; + num++) { + GenerateNewRandomFile(&rnd, /* nowait */ true); + ASSERT_OK(Flush()); + } + + TEST_SYNC_POINT( + "DBCompactionTest::SuggestCompactRangeNoTwoLevel0Compactions:2"); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +INSTANTIATE_TEST_CASE_P(DBCompactionWaitForCompactTest, + DBCompactionWaitForCompactTest, + ::testing::Values(std::make_tuple(false, false), + std::make_tuple(false, true), + std::make_tuple(true, false), + std::make_tuple(true, true))); + +TEST_P(DBCompactionWaitForCompactTest, + WaitForCompactWaitsOnCompactionToFinish) { + // Triggers a compaction. Before the compaction finishes, test + // closes the DB Upon reopen, wait for the compaction to finish and checks for + // the number of compaction finished + + int compaction_finished = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::Run():EndStatusSet", [&](void* arg) { + auto status = static_cast(arg); + if (status->ok()) { + compaction_finished++; + } + }); + // To make sure there's a flush/compaction debt + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::MaybeScheduleFlushOrCompaction:BeforeSchedule", [&](void* arg) { + auto unscheduled_flushes = *static_cast(arg); + ASSERT_GT(unscheduled_flushes, 0); + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBCompactionTest::WaitForCompactWaitsOnCompactionToFinish", + "DBImpl::MaybeScheduleFlushOrCompaction:BeforeSchedule"}}); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // create compaction debt by adding one more L0 file then closing + Random rnd(123); + GenerateNewRandomFile(&rnd, /* nowait */ true); + ASSERT_EQ(0, compaction_finished); + Close(); + TEST_SYNC_POINT("DBCompactionTest::WaitForCompactWaitsOnCompactionToFinish"); + ASSERT_EQ(0, compaction_finished); + + // Reopen the db and we expect the compaction to be triggered. + Reopen(options_); + + // Wait for compaction to finish + ASSERT_OK(dbfull()->WaitForCompact(wait_for_compact_options_)); + ASSERT_GT(compaction_finished, 0); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_P(DBCompactionWaitForCompactTest, WaitForCompactAbortOnPause) { + // Triggers a compaction. Before the compaction finishes, test + // pauses the compaction. Calling WaitForCompact() with option + // abort_on_pause=true should return Status::Aborted Or + // ContinueBackgroundWork() must be called + + // Now trigger L0 compaction by adding a file + Random rnd(123); + GenerateNewRandomFile(&rnd, /* nowait */ true); + ASSERT_OK(Flush()); + + // Pause the background jobs. + ASSERT_OK(dbfull()->PauseBackgroundWork()); + + // If not abort_on_pause_ continue the background jobs. + if (!abort_on_pause_) { + ASSERT_OK(dbfull()->ContinueBackgroundWork()); + } + + Status s = dbfull()->WaitForCompact(wait_for_compact_options_); + if (abort_on_pause_) { + ASSERT_NOK(s); + ASSERT_TRUE(s.IsAborted()); + } else { + ASSERT_OK(s); + } +} + +TEST_P(DBCompactionWaitForCompactTest, WaitForCompactShutdownWhileWaiting) { + // Triggers a compaction. Before the compaction finishes, db + // shuts down (by calling CancelAllBackgroundWork()). Calling WaitForCompact() + // should return Status::IsShutdownInProgress() + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"CompactionJob::Run():Start", + "DBCompactionTest::WaitForCompactShutdownWhileWaiting:0"}, + {"DBImpl::WaitForCompact:StartWaiting", + "DBCompactionTest::WaitForCompactShutdownWhileWaiting:1"}, + {"DBImpl::~DBImpl:WaitJob", "CompactionJob::Run():End"}, + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Now trigger L0 compaction by adding a file + Random rnd(123); + GenerateNewRandomFile(&rnd, /* nowait */ true); + ASSERT_OK(Flush()); + // Wait for compaction to start + TEST_SYNC_POINT("DBCompactionTest::WaitForCompactShutdownWhileWaiting:0"); + + // Wait for Compaction in another thread + auto waiting_for_compaction_thread = port::Thread([this]() { + Status s = dbfull()->WaitForCompact(wait_for_compact_options_); + ASSERT_NOK(s); + ASSERT_TRUE(s.IsShutdownInProgress()); + }); + TEST_SYNC_POINT("DBCompactionTest::WaitForCompactShutdownWhileWaiting:1"); + // Shutdown after wait started, but before the compaction finishes + auto closing_thread = port::Thread([this]() { ASSERT_OK(db_->Close()); }); + + waiting_for_compaction_thread.join(); + closing_thread.join(); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_P(DBCompactionWaitForCompactTest, WaitForCompactWithOptionToFlush) { + // After creating enough L0 files that one more file will trigger the + // compaction, write some data in memtable. Calls WaitForCompact with option + // to flush. This will flush the memtable to a new L0 file which will trigger + // compaction. Lastly check for expected number of files, closing + reopening + // DB won't trigger any flush or compaction + + int compaction_finished = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:AfterCompaction", + [&](void*) { compaction_finished++; }); + + int flush_finished = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "FlushJob::End", [&](void*) { flush_finished++; }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // write to memtable (overlapping key with first L0 file), but no flush is + // needed at this point. + ASSERT_OK(Put(Key(0), "some random string")); + ASSERT_EQ(0, compaction_finished); + ASSERT_EQ(0, flush_finished); + ASSERT_EQ("2", FilesPerLevel()); + + ASSERT_OK(dbfull()->WaitForCompact(wait_for_compact_options_)); + if (flush_) { + ASSERT_EQ("1,2", FilesPerLevel()); + ASSERT_EQ(1, compaction_finished); + ASSERT_EQ(1, flush_finished); + } else { + ASSERT_EQ(0, compaction_finished); + ASSERT_EQ(0, flush_finished); + ASSERT_EQ("2", FilesPerLevel()); + } + + compaction_finished = 0; + flush_finished = 0; + Close(); + Reopen(options_); + + ASSERT_EQ(0, flush_finished); + if (flush_) { + // if flushed already prior to close and reopen, expect there's no + // additional compaction needed + ASSERT_EQ(0, compaction_finished); + } else { + // if not flushed prior to close and reopen, expect L0 file creation from + // WAL when reopening which will trigger the compaction. + ASSERT_OK(dbfull()->WaitForCompact(wait_for_compact_options_)); + ASSERT_EQ(1, compaction_finished); + } + + ASSERT_EQ("1,2", FilesPerLevel()); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +static std::string ShortKey(int i) { + assert(i < 10000); + char buf[100]; + snprintf(buf, sizeof(buf), "key%04d", i); + return std::string(buf); +} + +TEST_P(DBCompactionTestWithParam, ForceBottommostLevelCompaction) { + int32_t trivial_move = 0; + int32_t non_trivial_move = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:TrivialMove", + [&](void* /*arg*/) { trivial_move++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial", + [&](void* /*arg*/) { non_trivial_move++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // The key size is guaranteed to be <= 8 + class ShortKeyComparator : public Comparator { + int Compare(const ROCKSDB_NAMESPACE::Slice& a, + const ROCKSDB_NAMESPACE::Slice& b) const override { + assert(a.size() <= 8); + assert(b.size() <= 8); + return BytewiseComparator()->Compare(a, b); + } + const char* Name() const override { return "ShortKeyComparator"; } + void FindShortestSeparator( + std::string* start, + const ROCKSDB_NAMESPACE::Slice& limit) const override { + return BytewiseComparator()->FindShortestSeparator(start, limit); + } + void FindShortSuccessor(std::string* key) const override { + return BytewiseComparator()->FindShortSuccessor(key); + } + } short_key_cmp; + Options options = CurrentOptions(); + options.target_file_size_base = 100000000; + options.write_buffer_size = 100000000; + options.max_subcompactions = max_subcompactions_; + options.comparator = &short_key_cmp; + DestroyAndReopen(options); + + int32_t value_size = 10 * 1024; // 10 KB + + Random rnd(301); + std::vector values; + // File with keys [ 0 => 99 ] + for (int i = 0; i < 100; i++) { + values.push_back(rnd.RandomString(value_size)); + ASSERT_OK(Put(ShortKey(i), values[i])); + } + ASSERT_OK(Flush()); + + ASSERT_EQ("1", FilesPerLevel(0)); + // Compaction will do L0=>L1 (trivial move) then move L1 files to L3 + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 3; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + ASSERT_EQ("0,0,0,1", FilesPerLevel(0)); + ASSERT_EQ(trivial_move, 1); + ASSERT_EQ(non_trivial_move, 0); + + // File with keys [ 100 => 199 ] + for (int i = 100; i < 200; i++) { + values.push_back(rnd.RandomString(value_size)); + ASSERT_OK(Put(ShortKey(i), values[i])); + } + ASSERT_OK(Flush()); + + ASSERT_EQ("1,0,0,1", FilesPerLevel(0)); + // Compaction will do L0=>L1 L1=>L2 L2=>L3 (3 trivial moves) + // then compacte the bottommost level L3=>L3 (non trivial move) + compact_options = CompactRangeOptions(); + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForceOptimized; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + ASSERT_EQ("0,0,0,1", FilesPerLevel(0)); + ASSERT_EQ(trivial_move, 4); + ASSERT_EQ(non_trivial_move, 1); + + // File with keys [ 200 => 299 ] + for (int i = 200; i < 300; i++) { + values.push_back(rnd.RandomString(value_size)); + ASSERT_OK(Put(ShortKey(i), values[i])); + } + ASSERT_OK(Flush()); + + ASSERT_EQ("1,0,0,1", FilesPerLevel(0)); + trivial_move = 0; + non_trivial_move = 0; + compact_options = CompactRangeOptions(); + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kSkip; + // Compaction will do L0=>L1 L1=>L2 L2=>L3 (3 trivial moves) + // and will skip bottommost level compaction + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + ASSERT_EQ("0,0,0,2", FilesPerLevel(0)); + ASSERT_EQ(trivial_move, 3); + ASSERT_EQ(non_trivial_move, 0); + + for (int i = 0; i < 300; i++) { + ASSERT_EQ(Get(ShortKey(i)), values[i]); + } + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_P(DBCompactionTestWithParam, IntraL0Compaction) { + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.level0_file_num_compaction_trigger = 5; + options.max_background_compactions = 2; + options.max_subcompactions = max_subcompactions_; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.write_buffer_size = 2 << 20; // 2MB + + BlockBasedTableOptions table_options; + table_options.block_cache = NewLRUCache(64 << 20); // 64MB + table_options.cache_index_and_filter_blocks = true; + table_options.pin_l0_filter_and_index_blocks_in_cache = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + DestroyAndReopen(options); + + const size_t kValueSize = 1 << 20; + Random rnd(301); + std::string value(rnd.RandomString(kValueSize)); + + // The L0->L1 must be picked before we begin flushing files to trigger + // intra-L0 compaction, and must not finish until after an intra-L0 + // compaction has been picked. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"LevelCompactionPicker::PickCompaction:Return", + "DBCompactionTest::IntraL0Compaction:L0ToL1Ready"}, + {"LevelCompactionPicker::PickCompactionBySize:0", + "CompactionJob::Run():Start"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // index: 0 1 2 3 4 5 6 7 8 9 + // size: 1MB 1MB 1MB 1MB 1MB 2MB 1MB 1MB 1MB 1MB + // score: 1.5 1.3 1.5 2.0 inf + // + // Files 0-4 will be included in an L0->L1 compaction. + // + // L0->L0 will be triggered since the sync points guarantee compaction to base + // level is still blocked when files 5-9 trigger another compaction. + // + // Files 6-9 are the longest span of available files for which + // work-per-deleted-file decreases (see "score" row above). + for (int i = 0; i < 10; ++i) { + ASSERT_OK(Put(Key(0), "")); // prevents trivial move + if (i == 5) { + TEST_SYNC_POINT("DBCompactionTest::IntraL0Compaction:L0ToL1Ready"); + ASSERT_OK(Put(Key(i + 1), value + value)); + } else { + ASSERT_OK(Put(Key(i + 1), value)); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + std::vector> level_to_files; + dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(), + &level_to_files); + ASSERT_GE(level_to_files.size(), 2); // at least L0 and L1 + // L0 has the 2MB file (not compacted) and 4MB file (output of L0->L0) + ASSERT_EQ(2, level_to_files[0].size()); + ASSERT_GT(level_to_files[1].size(), 0); + for (int i = 0; i < 2; ++i) { + ASSERT_GE(level_to_files[0][i].fd.file_size, 1 << 21); + } + + // The index/filter in the file produced by intra-L0 should not be pinned. + // That means clearing unref'd entries in block cache and re-accessing the + // file produced by intra-L0 should bump the index block miss count. + uint64_t prev_index_misses = + TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS); + table_options.block_cache->EraseUnRefEntries(); + ASSERT_EQ("", Get(Key(0))); + ASSERT_EQ(prev_index_misses + 1, + TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); +} + +TEST_P(DBCompactionTestWithParam, IntraL0CompactionDoesNotObsoleteDeletions) { + // regression test for issue #2722: L0->L0 compaction can resurrect deleted + // keys from older L0 files if L1+ files' key-ranges do not include the key. + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.level0_file_num_compaction_trigger = 5; + options.max_background_compactions = 2; + options.max_subcompactions = max_subcompactions_; + DestroyAndReopen(options); + + const size_t kValueSize = 1 << 20; + Random rnd(301); + std::string value(rnd.RandomString(kValueSize)); + + // The L0->L1 must be picked before we begin flushing files to trigger + // intra-L0 compaction, and must not finish until after an intra-L0 + // compaction has been picked. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"LevelCompactionPicker::PickCompaction:Return", + "DBCompactionTest::IntraL0CompactionDoesNotObsoleteDeletions:" + "L0ToL1Ready"}, + {"LevelCompactionPicker::PickCompactionBySize:0", + "CompactionJob::Run():Start"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // index: 0 1 2 3 4 5 6 7 8 9 + // size: 1MB 1MB 1MB 1MB 1MB 1MB 1MB 1MB 1MB 1MB + // score: 1.25 1.33 1.5 2.0 inf + // + // Files 0-4 will be included in an L0->L1 compaction. + // + // L0->L0 will be triggered since the sync points guarantee compaction to base + // level is still blocked when files 5-9 trigger another compaction. All files + // 5-9 are included in the L0->L0 due to work-per-deleted file decreasing. + // + // Put a key-value in files 0-4. Delete that key in files 5-9. Verify the + // L0->L0 preserves the deletion such that the key remains deleted. + for (int i = 0; i < 10; ++i) { + // key 0 serves both to prevent trivial move and as the key we want to + // verify is not resurrected by L0->L0 compaction. + if (i < 5) { + ASSERT_OK(Put(Key(0), "")); + } else { + ASSERT_OK(Delete(Key(0))); + } + if (i == 5) { + TEST_SYNC_POINT( + "DBCompactionTest::IntraL0CompactionDoesNotObsoleteDeletions:" + "L0ToL1Ready"); + } + ASSERT_OK(Put(Key(i + 1), value)); + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + std::vector> level_to_files; + dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(), + &level_to_files); + ASSERT_GE(level_to_files.size(), 2); // at least L0 and L1 + // L0 has a single output file from L0->L0 + ASSERT_EQ(1, level_to_files[0].size()); + ASSERT_GT(level_to_files[1].size(), 0); + ASSERT_GE(level_to_files[0][0].fd.file_size, 1 << 22); + + ReadOptions roptions; + std::string result; + ASSERT_TRUE(db_->Get(roptions, Key(0), &result).IsNotFound()); +} + +TEST_P(DBCompactionTestWithParam, FullCompactionInBottomPriThreadPool) { + const int kNumFilesTrigger = 3; + Env::Default()->SetBackgroundThreads(1, Env::Priority::BOTTOM); + for (bool use_universal_compaction : {false, true}) { + Options options = CurrentOptions(); + if (use_universal_compaction) { + options.compaction_style = kCompactionStyleUniversal; + } else { + options.compaction_style = kCompactionStyleLevel; + options.level_compaction_dynamic_level_bytes = true; + } + options.num_levels = 4; + options.write_buffer_size = 100 << 10; // 100KB + options.target_file_size_base = 32 << 10; // 32KB + options.level0_file_num_compaction_trigger = kNumFilesTrigger; + // Trigger compaction if size amplification exceeds 110% + options.compaction_options_universal.max_size_amplification_percent = 110; + DestroyAndReopen(options); + + int num_bottom_pri_compactions = 0; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BGWorkBottomCompaction", + [&](void* /*arg*/) { ++num_bottom_pri_compactions; }); + SyncPoint::GetInstance()->EnableProcessing(); + + Random rnd(301); + for (int num = 0; num < kNumFilesTrigger; num++) { + ASSERT_EQ(NumSortedRuns(), num); + int key_idx = 0; + GenerateNewFile(&rnd, &key_idx); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ(1, num_bottom_pri_compactions); + + // Verify that size amplification did occur + ASSERT_EQ(NumSortedRuns(), 1); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + } + Env::Default()->SetBackgroundThreads(0, Env::Priority::BOTTOM); +} + +TEST_F(DBCompactionTest, CancelCompactionWaitingOnConflict) { + // This test verifies cancellation of a compaction waiting to be scheduled due + // to conflict with a running compaction. + // + // A `CompactRange()` in universal compacts all files, waiting for files to + // become available if they are locked for another compaction. This test + // triggers an automatic compaction that blocks a `CompactRange()`, and + // verifies that `DisableManualCompaction()` can successfully cancel the + // `CompactRange()` without waiting for the automatic compaction to finish. + const int kNumSortedRuns = 4; + + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.level0_file_num_compaction_trigger = kNumSortedRuns; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + Reopen(options); + + test::SleepingBackgroundTask auto_compaction_sleeping_task; + // Block automatic compaction when it runs in the callback + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::Run():Start", + [&](void* /*arg*/) { auto_compaction_sleeping_task.DoSleep(); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Fill overlapping files in L0 to trigger an automatic compaction + Random rnd(301); + for (int i = 0; i < kNumSortedRuns; ++i) { + int key_idx = 0; + // We hold the compaction from happening, so when generating the last SST + // file, we cannot wait. Otherwise, we'll hit a deadlock. + GenerateNewFile(&rnd, &key_idx, + (i == kNumSortedRuns - 1) ? true : false /* nowait */); + } + auto_compaction_sleeping_task.WaitUntilSleeping(); + + // Make sure the manual compaction has seen the conflict before being canceled + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"ColumnFamilyData::CompactRange:Return", + "DBCompactionTest::CancelCompactionWaitingOnConflict:" + "PreDisableManualCompaction"}}); + auto manual_compaction_thread = port::Thread([this]() { + ASSERT_TRUE(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr) + .IsIncomplete()); + }); + + // Cancel it. Thread should be joinable, i.e., manual compaction was unblocked + // despite finding a conflict with an automatic compaction that is still + // running + TEST_SYNC_POINT( + "DBCompactionTest::CancelCompactionWaitingOnConflict:" + "PreDisableManualCompaction"); + db_->DisableManualCompaction(); + manual_compaction_thread.join(); +} + +TEST_F(DBCompactionTest, OptimizedDeletionObsoleting) { + // Deletions can be dropped when compacted to non-last level if they fall + // outside the lower-level files' key-ranges. + const int kNumL0Files = 4; + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = kNumL0Files; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + DestroyAndReopen(options); + + // put key 1 and 3 in separate L1, L2 files. + // So key 0, 2, and 4+ fall outside these levels' key-ranges. + for (int level = 2; level >= 1; --level) { + for (int i = 0; i < 2; ++i) { + ASSERT_OK(Put(Key(2 * i + 1), "val")); + ASSERT_OK(Flush()); + } + MoveFilesToLevel(level); + ASSERT_EQ(2, NumTableFilesAtLevel(level)); + } + + // Delete keys in range [1, 4]. These L0 files will be compacted with L1: + // - Tombstones for keys 2 and 4 can be dropped early. + // - Tombstones for keys 1 and 3 must be kept due to L2 files' key-ranges. + for (int i = 0; i < kNumL0Files; ++i) { + ASSERT_OK(Put(Key(0), "val")); // sentinel to prevent trivial move + ASSERT_OK(Delete(Key(i + 1))); + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + for (int i = 0; i < kNumL0Files; ++i) { + std::string value; + ASSERT_TRUE(db_->Get(ReadOptions(), Key(i + 1), &value).IsNotFound()); + } + ASSERT_EQ(2, options.statistics->getTickerCount( + COMPACTION_OPTIMIZED_DEL_DROP_OBSOLETE)); + ASSERT_EQ(2, + options.statistics->getTickerCount(COMPACTION_KEY_DROP_OBSOLETE)); +} + +TEST_F(DBCompactionTest, CompactFilesPendingL0Bug) { + // https://www.facebook.com/groups/rocksdb.dev/permalink/1389452781153232/ + // CompactFiles() had a bug where it failed to pick a compaction when an L0 + // compaction existed, but marked it as scheduled anyways. It'd never be + // unmarked as scheduled, so future compactions or DB close could hang. + const int kNumL0Files = 5; + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = kNumL0Files - 1; + options.max_background_compactions = 2; + DestroyAndReopen(options); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"LevelCompactionPicker::PickCompaction:Return", + "DBCompactionTest::CompactFilesPendingL0Bug:Picked"}, + {"DBCompactionTest::CompactFilesPendingL0Bug:ManualCompacted", + "DBImpl::BackgroundCompaction:NonTrivial:AfterRun"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + auto schedule_multi_compaction_token = + dbfull()->TEST_write_controler().GetCompactionPressureToken(); + + // Files 0-3 will be included in an L0->L1 compaction. + // + // File 4 will be included in a call to CompactFiles() while the first + // compaction is running. + for (int i = 0; i < kNumL0Files - 1; ++i) { + ASSERT_OK(Put(Key(0), "val")); // sentinel to prevent trivial move + ASSERT_OK(Put(Key(i + 1), "val")); + ASSERT_OK(Flush()); + } + TEST_SYNC_POINT("DBCompactionTest::CompactFilesPendingL0Bug:Picked"); + // file 4 flushed after 0-3 picked + ASSERT_OK(Put(Key(kNumL0Files), "val")); + ASSERT_OK(Flush()); + + // previously DB close would hang forever as this situation caused scheduled + // compactions count to never decrement to zero. + ColumnFamilyMetaData cf_meta; + dbfull()->GetColumnFamilyMetaData(dbfull()->DefaultColumnFamily(), &cf_meta); + ASSERT_EQ(kNumL0Files, cf_meta.levels[0].files.size()); + std::vector input_filenames; + input_filenames.push_back(cf_meta.levels[0].files.front().name); + ASSERT_OK(dbfull()->CompactFiles(CompactionOptions(), input_filenames, + 0 /* output_level */)); + TEST_SYNC_POINT("DBCompactionTest::CompactFilesPendingL0Bug:ManualCompacted"); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBCompactionTest, CompactFilesOverlapInL0Bug) { + // Regression test for bug of not pulling in L0 files that overlap the user- + // specified input files in time- and key-ranges. + ASSERT_OK(Put(Key(0), "old_val")); + ASSERT_OK(Flush()); + ASSERT_OK(Put(Key(0), "new_val")); + ASSERT_OK(Flush()); + + ColumnFamilyMetaData cf_meta; + dbfull()->GetColumnFamilyMetaData(dbfull()->DefaultColumnFamily(), &cf_meta); + ASSERT_GE(cf_meta.levels.size(), 2); + ASSERT_EQ(2, cf_meta.levels[0].files.size()); + + // Compacting {new L0 file, L1 file} should pull in the old L0 file since it + // overlaps in key-range and time-range. + std::vector input_filenames; + input_filenames.push_back(cf_meta.levels[0].files.front().name); + ASSERT_OK(dbfull()->CompactFiles(CompactionOptions(), input_filenames, + 1 /* output_level */)); + ASSERT_EQ("new_val", Get(Key(0))); +} + +TEST_F(DBCompactionTest, DeleteFilesInRangeConflictWithCompaction) { + Options options = CurrentOptions(); + DestroyAndReopen(options); + const Snapshot* snapshot = nullptr; + const int kMaxKey = 10; + + for (int i = 0; i < kMaxKey; i++) { + ASSERT_OK(Put(Key(i), Key(i))); + ASSERT_OK(Delete(Key(i))); + if (!snapshot) { + snapshot = db_->GetSnapshot(); + } + } + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + ASSERT_OK(Put(Key(kMaxKey), Key(kMaxKey))); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // test DeleteFilesInRange() deletes the files already picked for compaction + SyncPoint::GetInstance()->LoadDependency( + {{"VersionSet::LogAndApply:WriteManifestStart", + "BackgroundCallCompaction:0"}, + {"DBImpl::BackgroundCompaction:Finish", + "VersionSet::LogAndApply:WriteManifestDone"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + // release snapshot which mark bottommost file for compaction + db_->ReleaseSnapshot(snapshot); + std::string begin_string = Key(0); + std::string end_string = Key(kMaxKey + 1); + Slice begin(begin_string); + Slice end(end_string); + ASSERT_OK(DeleteFilesInRange(db_, db_->DefaultColumnFamily(), &begin, &end)); + SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBCompactionTest, CompactBottomLevelFilesWithDeletions) { + // bottom-level files may contain deletions due to snapshots protecting the + // deleted keys. Once the snapshot is released, we should see files with many + // such deletions undergo single-file compactions. + const int kNumKeysPerFile = 1024; + const int kNumLevelFiles = 4; + const int kValueSize = 128; + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.level0_file_num_compaction_trigger = kNumLevelFiles; + // inflate it a bit to account for key/metadata overhead + options.target_file_size_base = 120 * kNumKeysPerFile * kValueSize / 100; + CreateAndReopenWithCF({"one"}, options); + + Random rnd(301); + const Snapshot* snapshot = nullptr; + for (int i = 0; i < kNumLevelFiles; ++i) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK( + Put(Key(i * kNumKeysPerFile + j), rnd.RandomString(kValueSize))); + } + if (i == kNumLevelFiles - 1) { + snapshot = db_->GetSnapshot(); + // delete every other key after grabbing a snapshot, so these deletions + // and the keys they cover can't be dropped until after the snapshot is + // released. + for (int j = 0; j < kNumLevelFiles * kNumKeysPerFile; j += 2) { + ASSERT_OK(Delete(Key(j))); + } + } + ASSERT_OK(Flush()); + if (i < kNumLevelFiles - 1) { + ASSERT_EQ(i + 1, NumTableFilesAtLevel(0)); + } + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(kNumLevelFiles, NumTableFilesAtLevel(1)); + + std::vector pre_release_metadata, post_release_metadata; + db_->GetLiveFilesMetaData(&pre_release_metadata); + // just need to bump seqnum so ReleaseSnapshot knows the newest key in the SST + // files does not need to be preserved in case of a future snapshot. + ASSERT_OK(Put(Key(0), "val")); + ASSERT_NE(kMaxSequenceNumber, dbfull()->bottommost_files_mark_threshold_); + // release snapshot and wait for compactions to finish. Single-file + // compactions should be triggered, which reduce the size of each bottom-level + // file without changing file count. + db_->ReleaseSnapshot(snapshot); + ASSERT_EQ(kMaxSequenceNumber, dbfull()->bottommost_files_mark_threshold_); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) { + Compaction* compaction = reinterpret_cast(arg); + ASSERT_TRUE(compaction->compaction_reason() == + CompactionReason::kBottommostFiles); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + db_->GetLiveFilesMetaData(&post_release_metadata); + ASSERT_EQ(pre_release_metadata.size(), post_release_metadata.size()); + + for (size_t i = 0; i < pre_release_metadata.size(); ++i) { + const auto& pre_file = pre_release_metadata[i]; + const auto& post_file = post_release_metadata[i]; + ASSERT_EQ(1, pre_file.level); + ASSERT_EQ(1, post_file.level); + // each file is smaller than it was before as it was rewritten without + // deletion markers/deleted keys. + ASSERT_LT(post_file.size, pre_file.size); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBCompactionTest, NoCompactBottomLevelFilesWithDeletions) { + // bottom-level files may contain deletions due to snapshots protecting the + // deleted keys. Once the snapshot is released, we should see files with many + // such deletions undergo single-file compactions. But when disabling auto + // compactions, it shouldn't be triggered which may causing too many + // background jobs. + const int kNumKeysPerFile = 1024; + const int kNumLevelFiles = 4; + const int kValueSize = 128; + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + options.level0_file_num_compaction_trigger = kNumLevelFiles; + // inflate it a bit to account for key/metadata overhead + options.target_file_size_base = 120 * kNumKeysPerFile * kValueSize / 100; + Reopen(options); + + Random rnd(301); + const Snapshot* snapshot = nullptr; + for (int i = 0; i < kNumLevelFiles; ++i) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK( + Put(Key(i * kNumKeysPerFile + j), rnd.RandomString(kValueSize))); + } + if (i == kNumLevelFiles - 1) { + snapshot = db_->GetSnapshot(); + // delete every other key after grabbing a snapshot, so these deletions + // and the keys they cover can't be dropped until after the snapshot is + // released. + for (int j = 0; j < kNumLevelFiles * kNumKeysPerFile; j += 2) { + ASSERT_OK(Delete(Key(j))); + } + } + ASSERT_OK(Flush()); + if (i < kNumLevelFiles - 1) { + ASSERT_EQ(i + 1, NumTableFilesAtLevel(0)); + } + } + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr)); + ASSERT_EQ(kNumLevelFiles, NumTableFilesAtLevel(1)); + + std::vector pre_release_metadata, post_release_metadata; + db_->GetLiveFilesMetaData(&pre_release_metadata); + // just need to bump seqnum so ReleaseSnapshot knows the newest key in the SST + // files does not need to be preserved in case of a future snapshot. + ASSERT_OK(Put(Key(0), "val")); + + // release snapshot and no compaction should be triggered. + std::atomic num_compactions{0}; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:Start", + [&](void* /*arg*/) { num_compactions.fetch_add(1); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + db_->ReleaseSnapshot(snapshot); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(0, num_compactions); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + db_->GetLiveFilesMetaData(&post_release_metadata); + ASSERT_EQ(pre_release_metadata.size(), post_release_metadata.size()); + for (size_t i = 0; i < pre_release_metadata.size(); ++i) { + const auto& pre_file = pre_release_metadata[i]; + const auto& post_file = post_release_metadata[i]; + ASSERT_EQ(1, pre_file.level); + ASSERT_EQ(1, post_file.level); + // each file is same as before with deletion markers/deleted keys. + ASSERT_EQ(post_file.size, pre_file.size); + } +} + +TEST_F(DBCompactionTest, RoundRobinTtlCompactionNormal) { + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.level0_file_num_compaction_trigger = 20; + options.ttl = 24 * 60 * 60; // 24 hours + options.compaction_pri = kRoundRobin; + env_->now_cpu_count_.store(0); + env_->SetMockSleep(); + options.env = env_; + + // add a small second for each wait time, to make sure the file is expired + int small_seconds = 1; + + std::atomic_int ttl_compactions{0}; + std::atomic_int round_robin_ttl_compactions{0}; + std::atomic_int other_compactions{0}; + + SyncPoint::GetInstance()->SetCallBack( + "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) { + Compaction* compaction = reinterpret_cast(arg); + auto compaction_reason = compaction->compaction_reason(); + if (compaction_reason == CompactionReason::kTtl) { + ttl_compactions++; + } else if (compaction_reason == CompactionReason::kRoundRobinTtl) { + round_robin_ttl_compactions++; + } else { + other_compactions++; + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + + DestroyAndReopen(options); + + // Setup the files from lower level to up level, each file is 1 hour's older + // than the next one. + // create 10 files on the last level (L6) + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 100; j++) { + ASSERT_OK(Put(Key(i * 100 + j), "value" + std::to_string(i * 100 + j))); + } + ASSERT_OK(Flush()); + env_->MockSleepForSeconds(60 * 60); // generate 1 file per hour + } + MoveFilesToLevel(6); + + // create 5 files on L5 + for (int i = 0; i < 5; i++) { + for (int j = 0; j < 200; j++) { + ASSERT_OK(Put(Key(i * 200 + j), "value" + std::to_string(i * 200 + j))); + } + ASSERT_OK(Flush()); + env_->MockSleepForSeconds(60 * 60); + } + MoveFilesToLevel(5); + + // create 3 files on L4 + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 300; j++) { + ASSERT_OK(Put(Key(i * 300 + j), "value" + std::to_string(i * 300 + j))); + } + ASSERT_OK(Flush()); + env_->MockSleepForSeconds(60 * 60); + } + MoveFilesToLevel(4); + + // The LSM tree should be like: + // L4: [0, 299], [300, 599], [600, 899] + // L5: [0, 199] [200, 399]...............[800, 999] + // L6: [0,99][100,199][200,299][300,399]...............[800,899][900,999] + ASSERT_EQ("0,0,0,0,3,5,10", FilesPerLevel()); + + // make sure the first L5 file is expired + env_->MockSleepForSeconds(16 * 60 * 60 + small_seconds++); + + // trigger TTL compaction + ASSERT_OK(Put(Key(4), "value" + std::to_string(1))); + ASSERT_OK(Put(Key(5), "value" + std::to_string(1))); + ASSERT_OK(Flush()); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // verify there's a RoundRobin TTL compaction + ASSERT_EQ(1, round_robin_ttl_compactions); + round_robin_ttl_compactions = 0; + + // expire 2 more files + env_->MockSleepForSeconds(2 * 60 * 60 + small_seconds++); + // trigger TTL compaction + ASSERT_OK(Put(Key(4), "value" + std::to_string(2))); + ASSERT_OK(Put(Key(5), "value" + std::to_string(2))); + ASSERT_OK(Flush()); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ(2, round_robin_ttl_compactions); + round_robin_ttl_compactions = 0; + + // expire 4 more files, 2 out of 3 files on L4 are expired + env_->MockSleepForSeconds(4 * 60 * 60 + small_seconds++); + // trigger TTL compaction + ASSERT_OK(Put(Key(6), "value" + std::to_string(3))); + ASSERT_OK(Put(Key(7), "value" + std::to_string(3))); + ASSERT_OK(Flush()); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ(1, NumTableFilesAtLevel(4)); + ASSERT_EQ(0, NumTableFilesAtLevel(5)); + + ASSERT_GT(round_robin_ttl_compactions, 0); + round_robin_ttl_compactions = 0; + + // make the first L0 file expired, which triggers a normal TTL compaction + // instead of roundrobin TTL compaction, it will also include an extra file + // from L0 because of overlap + ASSERT_EQ(0, ttl_compactions); + env_->MockSleepForSeconds(19 * 60 * 60 + small_seconds++); + + // trigger TTL compaction + ASSERT_OK(Put(Key(6), "value" + std::to_string(4))); + ASSERT_OK(Put(Key(7), "value" + std::to_string(4))); + ASSERT_OK(Flush()); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // L0 -> L1 compaction is normal TTL compaction, L1 -> next levels compactions + // are RoundRobin TTL compaction. + ASSERT_GT(ttl_compactions, 0); + ttl_compactions = 0; + ASSERT_GT(round_robin_ttl_compactions, 0); + round_robin_ttl_compactions = 0; + + // All files are expired, so only the last level has data + env_->MockSleepForSeconds(24 * 60 * 60); + // trigger TTL compaction + ASSERT_OK(Put(Key(6), "value" + std::to_string(4))); + ASSERT_OK(Put(Key(7), "value" + std::to_string(4))); + ASSERT_OK(Flush()); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,0,0,0,0,0,2", FilesPerLevel()); + + ASSERT_GT(ttl_compactions, 0); + ttl_compactions = 0; + ASSERT_GT(round_robin_ttl_compactions, 0); + round_robin_ttl_compactions = 0; + + ASSERT_EQ(0, other_compactions); +} + +TEST_F(DBCompactionTest, RoundRobinTtlCompactionUnsortedTime) { + // This is to test the case that the RoundRobin compaction cursor not pointing + // to the oldest file, RoundRobin compaction should still compact the file + // after cursor until all expired files are compacted. + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.level0_file_num_compaction_trigger = 20; + options.ttl = 24 * 60 * 60; // 24 hours + options.compaction_pri = kRoundRobin; + env_->now_cpu_count_.store(0); + env_->SetMockSleep(); + options.env = env_; + + std::atomic_int ttl_compactions{0}; + std::atomic_int round_robin_ttl_compactions{0}; + std::atomic_int other_compactions{0}; + + SyncPoint::GetInstance()->SetCallBack( + "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) { + Compaction* compaction = reinterpret_cast(arg); + auto compaction_reason = compaction->compaction_reason(); + if (compaction_reason == CompactionReason::kTtl) { + ttl_compactions++; + } else if (compaction_reason == CompactionReason::kRoundRobinTtl) { + round_robin_ttl_compactions++; + } else { + other_compactions++; + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + + DestroyAndReopen(options); + + // create 10 files on the last level (L6) + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 100; j++) { + ASSERT_OK(Put(Key(i * 100 + j), "value" + std::to_string(i * 100 + j))); + } + ASSERT_OK(Flush()); + env_->MockSleepForSeconds(60 * 60); // generate 1 file per hour + } + MoveFilesToLevel(6); + + // create 5 files on L5 + for (int i = 0; i < 5; i++) { + for (int j = 0; j < 200; j++) { + ASSERT_OK(Put(Key(i * 200 + j), "value" + std::to_string(i * 200 + j))); + } + ASSERT_OK(Flush()); + env_->MockSleepForSeconds(60 * 60); // 1 hour + } + MoveFilesToLevel(5); + + // The LSM tree should be like: + // L5: [0, 199] [200, 399] [400,599] [600,799] [800, 999] + // L6: [0,99][100,199][200,299][300,399]....................[800,899][900,999] + ASSERT_EQ("0,0,0,0,0,5,10", FilesPerLevel()); + + // point the compaction cursor to the 4th file on L5 + VersionSet* const versions = dbfull()->GetVersionSet(); + assert(versions); + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(cfd, nullptr); + Version* const current = cfd->current(); + ASSERT_NE(current, nullptr); + VersionStorageInfo* storage_info = current->storage_info(); + ASSERT_NE(storage_info, nullptr); + const InternalKey split_cursor = InternalKey(Key(600), 100000, kTypeValue); + storage_info->AddCursorForOneLevel(5, split_cursor); + + // make the first file on L5 expired, there should be 3 TTL compactions: + // 4th one, 5th one, then 1st one. + env_->MockSleepForSeconds(19 * 60 * 60 + 1); + // trigger TTL compaction + ASSERT_OK(Put(Key(6), "value" + std::to_string(4))); + ASSERT_OK(Put(Key(7), "value" + std::to_string(4))); + ASSERT_OK(Flush()); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(2, NumTableFilesAtLevel(5)); + + ASSERT_EQ(3, round_robin_ttl_compactions); + ASSERT_EQ(0, ttl_compactions); + ASSERT_EQ(0, other_compactions); +} + +TEST_F(DBCompactionTest, LevelCompactExpiredTtlFiles) { + const int kNumKeysPerFile = 32; + const int kNumLevelFiles = 2; + const int kValueSize = 1024; + + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.ttl = 24 * 60 * 60; // 24 hours + options.max_open_files = -1; + env_->SetMockSleep(); + options.env = env_; + + // NOTE: Presumed unnecessary and removed: resetting mock time in env + + DestroyAndReopen(options); + + Random rnd(301); + for (int i = 0; i < kNumLevelFiles; ++i) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK( + Put(Key(i * kNumKeysPerFile + j), rnd.RandomString(kValueSize))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + MoveFilesToLevel(3); + ASSERT_EQ("0,0,0,2", FilesPerLevel()); + + // Delete previously written keys. + for (int i = 0; i < kNumLevelFiles; ++i) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK(Delete(Key(i * kNumKeysPerFile + j))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("2,0,0,2", FilesPerLevel()); + MoveFilesToLevel(1); + ASSERT_EQ("0,2,0,2", FilesPerLevel()); + + env_->MockSleepForSeconds(36 * 60 * 60); // 36 hours + ASSERT_EQ("0,2,0,2", FilesPerLevel()); + + // Just do a simple write + flush so that the Ttl expired files get + // compacted. + ASSERT_OK(Put("a", "1")); + ASSERT_OK(Flush()); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) { + Compaction* compaction = reinterpret_cast(arg); + ASSERT_TRUE(compaction->compaction_reason() == CompactionReason::kTtl); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // All non-L0 files are deleted, as they contained only deleted data. + ASSERT_EQ("1", FilesPerLevel()); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + // Test dynamically changing ttl. + + // NOTE: Presumed unnecessary and removed: resetting mock time in env + + DestroyAndReopen(options); + + for (int i = 0; i < kNumLevelFiles; ++i) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK( + Put(Key(i * kNumKeysPerFile + j), rnd.RandomString(kValueSize))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + MoveFilesToLevel(3); + ASSERT_EQ("0,0,0,2", FilesPerLevel()); + + // Delete previously written keys. + for (int i = 0; i < kNumLevelFiles; ++i) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK(Delete(Key(i * kNumKeysPerFile + j))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("2,0,0,2", FilesPerLevel()); + MoveFilesToLevel(1); + ASSERT_EQ("0,2,0,2", FilesPerLevel()); + + // Move time forward by 12 hours, and make sure that compaction still doesn't + // trigger as ttl is set to 24 hours. + env_->MockSleepForSeconds(12 * 60 * 60); + ASSERT_OK(Put("a", "1")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("1,2,0,2", FilesPerLevel()); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) { + Compaction* compaction = reinterpret_cast(arg); + ASSERT_TRUE(compaction->compaction_reason() == CompactionReason::kTtl); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Dynamically change ttl to 10 hours. + // This should trigger a ttl compaction, as 12 hours have already passed. + ASSERT_OK(dbfull()->SetOptions({{"ttl", "36000"}})); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // All non-L0 files are deleted, as they contained only deleted data. + ASSERT_EQ("1", FilesPerLevel()); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBCompactionTest, LevelTtlCompactionOutputCuttingIteractingWithOther) { + // This test is for a bug fix in CompactionOutputs::ShouldStopBefore() where + // TTL states were not being updated for keys that ShouldStopBefore() would + // return true for reasons other than TTL. + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.ttl = 24 * 60 * 60; // 24 hours + options.max_open_files = -1; + options.compaction_pri = kMinOverlappingRatio; + env_->SetMockSleep(); + options.env = env_; + options.target_file_size_base = 4 << 10; + options.disable_auto_compactions = true; + options.level_compaction_dynamic_file_size = false; + + DestroyAndReopen(options); + Random rnd(301); + + // This makes sure the manual compaction below + // is not a bottommost compaction as TTL is only + // for non-bottommost compactions. + ASSERT_OK(Put(Key(3), rnd.RandomString(1 << 10))); + ASSERT_OK(Put(Key(0), rnd.RandomString(1 << 10))); + ASSERT_OK(Flush()); + MoveFilesToLevel(6); + + // L2: + ASSERT_OK(Put(Key(2), rnd.RandomString(4 << 10))); + ASSERT_OK(Put(Key(3), rnd.RandomString(4 << 10))); + ASSERT_OK(Flush()); + MoveFilesToLevel(2); + + // L1, overlaps in range with the file in L2 so + // that they compact together. + ASSERT_OK(Put(Key(0), rnd.RandomString(4 << 10))); + ASSERT_OK(Put(Key(1), rnd.RandomString(4 << 10))); + ASSERT_OK(Put(Key(3), rnd.RandomString(4 << 10))); + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + + ASSERT_EQ("0,1,1,0,0,0,1", FilesPerLevel()); + // 36 hours so that the file in L2 is eligible for TTL + env_->MockSleepForSeconds(36 * 60 * 60); + + CompactRangeOptions compact_range_opts; + + ASSERT_OK(dbfull()->RunManualCompaction( + static_cast_with_check(db_->DefaultColumnFamily()) + ->cfd(), + 1 /* input_level */, 2 /* output_level */, compact_range_opts, + nullptr /* begin */, nullptr /* end */, true /* exclusive */, + true /* disallow_trivial_move */, + std::numeric_limits::max() /*max_file_num_to_ignore*/, + "" /*trim_ts*/)); + + // L2 should have 2 files: + // file 1: Key(0), Key(1) + // ShouldStopBefore(Key(2)) return true due to TTL or output file size + // file 2: Key(2), Key(3) + // + // Before the fix in this PR, L2 would have 3 files: + // file 1: Key(0), Key(1) + // CompactionOutputs::ShouldStopBefore(Key(2)) returns true due to output file + // size. + // file 2: Key(2) + // CompactionOutput::ShouldStopBefore(Key(3)) returns true + // due to TTL cutting and that TTL states were not updated + // for Key(2). + // file 3: Key(3) + ASSERT_EQ("0,0,2,0,0,0,1", FilesPerLevel()); +} + +TEST_F(DBCompactionTest, LevelTtlCascadingCompactions) { + env_->SetMockSleep(); + const int kValueSize = 100; + + for (bool if_restart : {false, true}) { + for (bool if_open_all_files : {false, true}) { + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.ttl = 24 * 60 * 60; // 24 hours + if (if_open_all_files) { + options.max_open_files = -1; + } else { + options.max_open_files = 20; + } + // RocksDB sanitize max open files to at least 20. Modify it back. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SanitizeOptions::AfterChangeMaxOpenFiles", [&](void* arg) { + int* max_open_files = static_cast(arg); + *max_open_files = 2; + }); + // In the case where all files are opened and doing DB restart + // forcing the oldest ancester time in manifest file to be 0 to + // simulate the case of reading from an old version. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "VersionEdit::EncodeTo:VarintOldestAncesterTime", [&](void* arg) { + if (if_restart && if_open_all_files) { + std::string* encoded_fieled = static_cast(arg); + *encoded_fieled = ""; + PutVarint64(encoded_fieled, 0); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + options.env = env_; + + // NOTE: Presumed unnecessary and removed: resetting mock time in env + + DestroyAndReopen(options); + + int ttl_compactions = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) { + Compaction* compaction = reinterpret_cast(arg); + auto compaction_reason = compaction->compaction_reason(); + if (compaction_reason == CompactionReason::kTtl) { + ttl_compactions++; + } + }); + + // Add two L6 files with key ranges: [1 .. 100], [101 .. 200]. + Random rnd(301); + for (int i = 1; i <= 100; ++i) { + ASSERT_OK(Put(Key(i), rnd.RandomString(kValueSize))); + } + ASSERT_OK(Flush()); + // Get the first file's creation time. This will be the oldest file in the + // DB. Compactions inolving this file's descendents should keep getting + // this time. + std::vector> level_to_files; + dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(), + &level_to_files); + uint64_t oldest_time = level_to_files[0][0].oldest_ancester_time; + // Add 1 hour and do another flush. + env_->MockSleepForSeconds(1 * 60 * 60); + for (int i = 101; i <= 200; ++i) { + ASSERT_OK(Put(Key(i), rnd.RandomString(kValueSize))); + } + ASSERT_OK(Flush()); + MoveFilesToLevel(6); + ASSERT_EQ("0,0,0,0,0,0,2", FilesPerLevel()); + + env_->MockSleepForSeconds(1 * 60 * 60); + // Add two L4 files with key ranges: [1 .. 50], [51 .. 150]. + for (int i = 1; i <= 50; ++i) { + ASSERT_OK(Put(Key(i), rnd.RandomString(kValueSize))); + } + ASSERT_OK(Flush()); + env_->MockSleepForSeconds(1 * 60 * 60); + for (int i = 51; i <= 150; ++i) { + ASSERT_OK(Put(Key(i), rnd.RandomString(kValueSize))); + } + ASSERT_OK(Flush()); + MoveFilesToLevel(4); + ASSERT_EQ("0,0,0,0,2,0,2", FilesPerLevel()); + + env_->MockSleepForSeconds(1 * 60 * 60); + // Add one L1 file with key range: [26, 75]. + for (int i = 26; i <= 75; ++i) { + ASSERT_OK(Put(Key(i), rnd.RandomString(kValueSize))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + MoveFilesToLevel(1); + ASSERT_EQ("0,1,0,0,2,0,2", FilesPerLevel()); + + // LSM tree: + // L1: [26 .. 75] + // L4: [1 .. 50][51 ..... 150] + // L6: [1 ........ 100][101 .... 200] + // + // On TTL expiry, TTL compaction should be initiated on L1 file, and the + // compactions should keep going on until the key range hits bottom level. + // In other words: the compaction on this data range "cascasdes" until + // reaching the bottom level. + // + // Order of events on TTL expiry: + // 1. L1 file falls to L3 via 2 trivial moves which are initiated by the + // ttl + // compaction. + // 2. A TTL compaction happens between L3 and L4 files. Output file in L4. + // 3. The new output file from L4 falls to L5 via 1 trival move initiated + // by the ttl compaction. + // 4. A TTL compaction happens between L5 and L6 files. Ouptut in L6. + + // Add 25 hours and do a write + env_->MockSleepForSeconds(25 * 60 * 60); + + ASSERT_OK(Put(Key(1), "1")); + if (if_restart) { + Reopen(options); + } else { + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("1,0,0,0,0,0,1", FilesPerLevel()); + ASSERT_EQ(5, ttl_compactions); + + dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(), + &level_to_files); + ASSERT_EQ(oldest_time, level_to_files[6][0].oldest_ancester_time); + + env_->MockSleepForSeconds(25 * 60 * 60); + ASSERT_OK(Put(Key(2), "1")); + if (if_restart) { + Reopen(options); + } else { + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("1,0,0,0,0,0,1", FilesPerLevel()); + ASSERT_GE(ttl_compactions, 6); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + } + } +} + +TEST_F(DBCompactionTest, LevelPeriodicCompaction) { + env_->SetMockSleep(); + const int kNumKeysPerFile = 32; + const int kNumLevelFiles = 2; + const int kValueSize = 100; + + for (bool if_restart : {false, true}) { + for (bool if_open_all_files : {false, true}) { + Options options = CurrentOptions(); + options.periodic_compaction_seconds = 48 * 60 * 60; // 2 days + if (if_open_all_files) { + options.max_open_files = -1; // needed for ttl compaction + } else { + options.max_open_files = 20; + } + // RocksDB sanitize max open files to at least 20. Modify it back. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SanitizeOptions::AfterChangeMaxOpenFiles", [&](void* arg) { + int* max_open_files = static_cast(arg); + *max_open_files = 0; + }); + // In the case where all files are opened and doing DB restart + // forcing the file creation time in manifest file to be 0 to + // simulate the case of reading from an old version. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "VersionEdit::EncodeTo:VarintFileCreationTime", [&](void* arg) { + if (if_restart && if_open_all_files) { + std::string* encoded_fieled = static_cast(arg); + *encoded_fieled = ""; + PutVarint64(encoded_fieled, 0); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + options.env = env_; + + // NOTE: Presumed unnecessary and removed: resetting mock time in env + + DestroyAndReopen(options); + + int periodic_compactions = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) { + Compaction* compaction = reinterpret_cast(arg); + auto compaction_reason = compaction->compaction_reason(); + if (compaction_reason == CompactionReason::kPeriodicCompaction) { + periodic_compactions++; + } + }); + + Random rnd(301); + for (int i = 0; i < kNumLevelFiles; ++i) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK( + Put(Key(i * kNumKeysPerFile + j), rnd.RandomString(kValueSize))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ("2", FilesPerLevel()); + ASSERT_EQ(0, periodic_compactions); + + // Add 50 hours and do a write + env_->MockSleepForSeconds(50 * 60 * 60); + ASSERT_OK(Put("a", "1")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // Assert that the files stay in the same level + ASSERT_EQ("3", FilesPerLevel()); + // The two old files go through the periodic compaction process + ASSERT_EQ(2, periodic_compactions); + + MoveFilesToLevel(1); + ASSERT_EQ("0,3", FilesPerLevel()); + + // Add another 50 hours and do another write + env_->MockSleepForSeconds(50 * 60 * 60); + ASSERT_OK(Put("b", "2")); + if (if_restart) { + Reopen(options); + } else { + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("1,3", FilesPerLevel()); + // The three old files now go through the periodic compaction process. 2 + // + 3. + ASSERT_EQ(5, periodic_compactions); + + // Add another 50 hours and do another write + env_->MockSleepForSeconds(50 * 60 * 60); + ASSERT_OK(Put("c", "3")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("2,3", FilesPerLevel()); + // The four old files now go through the periodic compaction process. 5 + // + 4. + ASSERT_EQ(9, periodic_compactions); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + } + } +} + +TEST_F(DBCompactionTest, LevelPeriodicCompactionWithOldDB) { + // This test makes sure that periodic compactions are working with a DB + // where file_creation_time of some files is 0. + // After compactions the new files are created with a valid file_creation_time + + const int kNumKeysPerFile = 32; + const int kNumFiles = 4; + const int kValueSize = 100; + + Options options = CurrentOptions(); + env_->SetMockSleep(); + options.env = env_; + + // NOTE: Presumed unnecessary and removed: resetting mock time in env + + DestroyAndReopen(options); + + int periodic_compactions = 0; + bool set_file_creation_time_to_zero = true; + bool set_creation_time_to_zero = true; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) { + Compaction* compaction = reinterpret_cast(arg); + auto compaction_reason = compaction->compaction_reason(); + if (compaction_reason == CompactionReason::kPeriodicCompaction) { + periodic_compactions++; + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "PropertyBlockBuilder::AddTableProperty:Start", [&](void* arg) { + TableProperties* props = reinterpret_cast(arg); + if (set_file_creation_time_to_zero) { + props->file_creation_time = 0; + } + if (set_creation_time_to_zero) { + props->creation_time = 0; + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Random rnd(301); + for (int i = 0; i < kNumFiles; ++i) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK( + Put(Key(i * kNumKeysPerFile + j), rnd.RandomString(kValueSize))); + } + ASSERT_OK(Flush()); + // Move the first two files to L2. + if (i == 1) { + MoveFilesToLevel(2); + set_creation_time_to_zero = false; + } + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ("2,0,2", FilesPerLevel()); + ASSERT_EQ(0, periodic_compactions); + + Close(); + + set_file_creation_time_to_zero = false; + // Forward the clock by 2 days. + env_->MockSleepForSeconds(2 * 24 * 60 * 60); + options.periodic_compaction_seconds = 1 * 24 * 60 * 60; // 1 day + + Reopen(options); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("2,0,2", FilesPerLevel()); + // Make sure that all files go through periodic compaction. + ASSERT_EQ(kNumFiles, periodic_compactions); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBCompactionTest, LevelPeriodicAndTtlCompaction) { + const int kNumKeysPerFile = 32; + const int kNumLevelFiles = 2; + const int kValueSize = 100; + + Options options = CurrentOptions(); + options.ttl = 10 * 60 * 60; // 10 hours + options.periodic_compaction_seconds = 48 * 60 * 60; // 2 days + options.max_open_files = -1; // needed for both periodic and ttl compactions + env_->SetMockSleep(); + options.env = env_; + + // NOTE: Presumed unnecessary and removed: resetting mock time in env + + DestroyAndReopen(options); + + int periodic_compactions = 0; + int ttl_compactions = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) { + Compaction* compaction = reinterpret_cast(arg); + auto compaction_reason = compaction->compaction_reason(); + if (compaction_reason == CompactionReason::kPeriodicCompaction) { + periodic_compactions++; + } else if (compaction_reason == CompactionReason::kTtl) { + ttl_compactions++; + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Random rnd(301); + for (int i = 0; i < kNumLevelFiles; ++i) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK( + Put(Key(i * kNumKeysPerFile + j), rnd.RandomString(kValueSize))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + MoveFilesToLevel(3); + + ASSERT_EQ("0,0,0,2", FilesPerLevel()); + ASSERT_EQ(0, periodic_compactions); + ASSERT_EQ(0, ttl_compactions); + + // Add some time greater than periodic_compaction_time. + env_->MockSleepForSeconds(50 * 60 * 60); + ASSERT_OK(Put("a", "1")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // Files in the bottom level go through periodic compactions. + ASSERT_EQ("1,0,0,2", FilesPerLevel()); + ASSERT_EQ(2, periodic_compactions); + ASSERT_EQ(0, ttl_compactions); + + // Add a little more time than ttl + env_->MockSleepForSeconds(11 * 60 * 60); + ASSERT_OK(Put("b", "1")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // Notice that the previous file in level 1 falls down to the bottom level + // due to ttl compactions, one level at a time. + // And bottom level files don't get picked up for ttl compactions. + ASSERT_EQ("1,0,0,3", FilesPerLevel()); + ASSERT_EQ(2, periodic_compactions); + ASSERT_EQ(3, ttl_compactions); + + // Add some time greater than periodic_compaction_time. + env_->MockSleepForSeconds(50 * 60 * 60); + ASSERT_OK(Put("c", "1")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // Previous L0 file falls one level at a time to bottom level due to ttl. + // And all 4 bottom files go through periodic compactions. + ASSERT_EQ("1,0,0,4", FilesPerLevel()); + ASSERT_EQ(6, periodic_compactions); + ASSERT_EQ(6, ttl_compactions); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBCompactionTest, LevelTtlBooster) { + const int kNumKeysPerFile = 32; + const int kNumLevelFiles = 3; + const int kValueSize = 1000; + + Options options = CurrentOptions(); + options.ttl = 10 * 60 * 60; // 10 hours + options.periodic_compaction_seconds = 480 * 60 * 60; // very long + options.level0_file_num_compaction_trigger = 2; + options.max_bytes_for_level_base = 5 * uint64_t{kNumKeysPerFile * kValueSize}; + options.max_open_files = -1; // needed for both periodic and ttl compactions + options.compaction_pri = CompactionPri::kMinOverlappingRatio; + env_->SetMockSleep(); + options.env = env_; + + // NOTE: Presumed unnecessary and removed: resetting mock time in env + + DestroyAndReopen(options); + + Random rnd(301); + for (int i = 0; i < kNumLevelFiles; ++i) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK( + Put(Key(i * kNumKeysPerFile + j), rnd.RandomString(kValueSize))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + MoveFilesToLevel(2); + + ASSERT_EQ("0,0,3", FilesPerLevel()); + + // Create some files for L1 + for (int i = 0; i < 2; i++) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK(Put(Key(2 * j + i), rnd.RandomString(kValueSize))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + + ASSERT_EQ("0,1,3", FilesPerLevel()); + + // Make the new L0 files qualify TTL boosting and generate one more to trigger + // L1 -> L2 compaction. Old files will be picked even if their priority is + // lower without boosting. + env_->MockSleepForSeconds(8 * 60 * 60); + for (int i = 0; i < 2; i++) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK(Put(Key(kNumKeysPerFile * 2 + 2 * j + i), + rnd.RandomString(kValueSize * 2))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + // Force files to be compacted to L1 + ASSERT_OK( + dbfull()->SetOptions({{"level0_file_num_compaction_trigger", "1"}})); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,1,2", FilesPerLevel()); + ASSERT_OK( + dbfull()->SetOptions({{"level0_file_num_compaction_trigger", "2"}})); + + ASSERT_GT(SizeAtLevel(1), kNumKeysPerFile * 4 * kValueSize); +} + +TEST_F(DBCompactionTest, LevelPeriodicCompactionWithCompactionFilters) { + class TestCompactionFilter : public CompactionFilter { + const char* Name() const override { return "TestCompactionFilter"; } + }; + class TestCompactionFilterFactory : public CompactionFilterFactory { + const char* Name() const override { return "TestCompactionFilterFactory"; } + std::unique_ptr CreateCompactionFilter( + const CompactionFilter::Context& /*context*/) override { + return std::unique_ptr(new TestCompactionFilter()); + } + }; + + const int kNumKeysPerFile = 32; + const int kNumLevelFiles = 2; + const int kValueSize = 100; + + Random rnd(301); + + Options options = CurrentOptions(); + TestCompactionFilter test_compaction_filter; + env_->SetMockSleep(); + options.env = env_; + + // NOTE: Presumed unnecessary and removed: resetting mock time in env + + enum CompactionFilterType { + kUseCompactionFilter, + kUseCompactionFilterFactory + }; + + for (CompactionFilterType comp_filter_type : + {kUseCompactionFilter, kUseCompactionFilterFactory}) { + // Assert that periodic compactions are not enabled. + ASSERT_EQ(std::numeric_limits::max() - 1, + options.periodic_compaction_seconds); + + if (comp_filter_type == kUseCompactionFilter) { + options.compaction_filter = &test_compaction_filter; + options.compaction_filter_factory.reset(); + } else if (comp_filter_type == kUseCompactionFilterFactory) { + options.compaction_filter = nullptr; + options.compaction_filter_factory.reset( + new TestCompactionFilterFactory()); + } + DestroyAndReopen(options); + + // periodic_compaction_seconds should be set to the sanitized value when + // a compaction filter or a compaction filter factory is used. + ASSERT_EQ(30 * 24 * 60 * 60, + dbfull()->GetOptions().periodic_compaction_seconds); + + int periodic_compactions = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) { + Compaction* compaction = reinterpret_cast(arg); + auto compaction_reason = compaction->compaction_reason(); + if (compaction_reason == CompactionReason::kPeriodicCompaction) { + periodic_compactions++; + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + for (int i = 0; i < kNumLevelFiles; ++i) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK( + Put(Key(i * kNumKeysPerFile + j), rnd.RandomString(kValueSize))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ("2", FilesPerLevel()); + ASSERT_EQ(0, periodic_compactions); + + // Add 31 days and do a write + env_->MockSleepForSeconds(31 * 24 * 60 * 60); + ASSERT_OK(Put("a", "1")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // Assert that the files stay in the same level + ASSERT_EQ("3", FilesPerLevel()); + // The two old files go through the periodic compaction process + ASSERT_EQ(2, periodic_compactions); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + } +} + +TEST_F(DBCompactionTest, CompactRangeDelayedByL0FileCount) { + // Verify that, when `CompactRangeOptions::allow_write_stall == false`, manual + // compaction only triggers flush after it's sure stall won't be triggered for + // L0 file count going too high. + const int kNumL0FilesTrigger = 4; + const int kNumL0FilesLimit = 8; + // i == 0: verifies normal case where stall is avoided by delay + // i == 1: verifies no delay in edge case where stall trigger is same as + // compaction trigger, so stall can't be avoided + for (int i = 0; i < 2; ++i) { + Options options = CurrentOptions(); + options.level0_slowdown_writes_trigger = kNumL0FilesLimit; + if (i == 0) { + options.level0_file_num_compaction_trigger = kNumL0FilesTrigger; + } else { + options.level0_file_num_compaction_trigger = kNumL0FilesLimit; + } + Reopen(options); + + if (i == 0) { + // ensure the auto compaction doesn't finish until manual compaction has + // had a chance to be delayed. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::WaitUntilFlushWouldNotStallWrites:StallWait", + "CompactionJob::Run():End"}}); + } else { + // ensure the auto-compaction doesn't finish until manual compaction has + // continued without delay. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::FlushMemTable:StallWaitDone", + "CompactionJob::Run():End"}}); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Random rnd(301); + for (int j = 0; j < kNumL0FilesLimit - 1; ++j) { + for (int k = 0; k < 2; ++k) { + ASSERT_OK(Put(Key(k), rnd.RandomString(1024))); + } + ASSERT_OK(Flush()); + } + auto manual_compaction_thread = port::Thread([this]() { + CompactRangeOptions cro; + cro.allow_write_stall = false; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + }); + + manual_compaction_thread.join(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_GT(NumTableFilesAtLevel(1), 0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + } +} + +TEST_F(DBCompactionTest, CompactRangeDelayedByImmMemTableCount) { + // Verify that, when `CompactRangeOptions::allow_write_stall == false`, manual + // compaction only triggers flush after it's sure stall won't be triggered for + // immutable memtable count going too high. + const int kNumImmMemTableLimit = 8; + // i == 0: verifies normal case where stall is avoided by delay + // i == 1: verifies no delay in edge case where stall trigger is same as flush + // trigger, so stall can't be avoided + for (int i = 0; i < 2; ++i) { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + // the delay limit is one less than the stop limit. This test focuses on + // avoiding delay limit, but this option sets stop limit, so add one. + options.max_write_buffer_number = kNumImmMemTableLimit + 1; + if (i == 1) { + options.min_write_buffer_number_to_merge = kNumImmMemTableLimit; + } + Reopen(options); + + if (i == 0) { + // ensure the flush doesn't finish until manual compaction has had a + // chance to be delayed. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::WaitUntilFlushWouldNotStallWrites:StallWait", + "FlushJob::WriteLevel0Table"}}); + } else { + // ensure the flush doesn't finish until manual compaction has continued + // without delay. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::FlushMemTable:StallWaitDone", + "FlushJob::WriteLevel0Table"}}); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Random rnd(301); + for (int j = 0; j < kNumImmMemTableLimit - 1; ++j) { + ASSERT_OK(Put(Key(0), rnd.RandomString(1024))); + FlushOptions flush_opts; + flush_opts.wait = false; + flush_opts.allow_write_stall = true; + ASSERT_OK(dbfull()->Flush(flush_opts)); + } + + auto manual_compaction_thread = port::Thread([this]() { + // Write something to make the current Memtable non-empty, so an extra + // immutable Memtable will be created upon manual flush requested by + // CompactRange, triggering a write stall mode to be entered because of + // accumulation of write buffers due to manual flush. + Random compact_rnd(301); + ASSERT_OK(Put(Key(0), compact_rnd.RandomString(1024))); + CompactRangeOptions cro; + cro.allow_write_stall = false; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + }); + + manual_compaction_thread.join(); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_GT(NumTableFilesAtLevel(1), 0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + } +} + +TEST_F(DBCompactionTest, CompactRangeShutdownWhileDelayed) { + // Verify that, when `CompactRangeOptions::allow_write_stall == false`, delay + // does not hang if CF is dropped or DB is closed + const int kNumL0FilesTrigger = 4; + const int kNumL0FilesLimit = 8; + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = kNumL0FilesTrigger; + options.level0_slowdown_writes_trigger = kNumL0FilesLimit; + // i == 0: DB::DropColumnFamily() on CompactRange's target CF unblocks it + // i == 1: DB::CancelAllBackgroundWork() unblocks CompactRange. This is to + // simulate what happens during Close as we can't call Close (it + // blocks on the auto-compaction, making a cycle). + for (int i = 0; i < 2; ++i) { + CreateAndReopenWithCF({"one"}, options); + // The calls to close CF/DB wait until the manual compaction stalls. + // The auto-compaction waits until the manual compaction finishes to ensure + // the signal comes from closing CF/DB, not from compaction making progress. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::WaitUntilFlushWouldNotStallWrites:StallWait", + "DBCompactionTest::CompactRangeShutdownWhileDelayed:PreShutdown"}, + {"DBCompactionTest::CompactRangeShutdownWhileDelayed:PostManual", + "CompactionJob::Run():End"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Random rnd(301); + for (int j = 0; j < kNumL0FilesLimit - 1; ++j) { + for (int k = 0; k < 2; ++k) { + ASSERT_OK(Put(1, Key(k), rnd.RandomString(1024))); + } + ASSERT_OK(Flush(1)); + } + auto manual_compaction_thread = port::Thread([this, i]() { + CompactRangeOptions cro; + cro.allow_write_stall = false; + if (i == 0) { + ASSERT_TRUE(db_->CompactRange(cro, handles_[1], nullptr, nullptr) + .IsColumnFamilyDropped()); + } else { + ASSERT_TRUE(db_->CompactRange(cro, handles_[1], nullptr, nullptr) + .IsShutdownInProgress()); + } + }); + + TEST_SYNC_POINT( + "DBCompactionTest::CompactRangeShutdownWhileDelayed:PreShutdown"); + if (i == 0) { + ASSERT_OK(db_->DropColumnFamily(handles_[1])); + } else { + dbfull()->CancelAllBackgroundWork(false /* wait */); + } + manual_compaction_thread.join(); + TEST_SYNC_POINT( + "DBCompactionTest::CompactRangeShutdownWhileDelayed:PostManual"); + if (i == 0) { + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } else { + ASSERT_NOK(dbfull()->TEST_WaitForCompact()); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + } +} + +TEST_F(DBCompactionTest, CompactRangeSkipFlushAfterDelay) { + // Verify that, when `CompactRangeOptions::allow_write_stall == false`, + // CompactRange skips its flush if the delay is long enough that the memtables + // existing at the beginning of the call have already been flushed. + const int kNumL0FilesTrigger = 4; + const int kNumL0FilesLimit = 8; + Options options = CurrentOptions(); + options.level0_slowdown_writes_trigger = kNumL0FilesLimit; + options.level0_file_num_compaction_trigger = kNumL0FilesTrigger; + Reopen(options); + + Random rnd(301); + // The manual flush includes the memtable that was active when CompactRange + // began. So it unblocks CompactRange and precludes its flush. Throughout the + // test, stall conditions are upheld via high L0 file count. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::WaitUntilFlushWouldNotStallWrites:StallWait", + "DBCompactionTest::CompactRangeSkipFlushAfterDelay:PreFlush"}, + {"DBCompactionTest::CompactRangeSkipFlushAfterDelay:PostFlush", + "DBImpl::FlushMemTable:StallWaitDone"}, + {"DBImpl::FlushMemTable:StallWaitDone", "CompactionJob::Run():End"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // used for the delayable flushes + FlushOptions flush_opts; + flush_opts.allow_write_stall = true; + for (int i = 0; i < kNumL0FilesLimit - 1; ++i) { + for (int j = 0; j < 2; ++j) { + ASSERT_OK(Put(Key(j), rnd.RandomString(1024))); + } + ASSERT_OK(dbfull()->Flush(flush_opts)); + } + auto manual_compaction_thread = port::Thread([this]() { + CompactRangeOptions cro; + cro.allow_write_stall = false; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + }); + + TEST_SYNC_POINT("DBCompactionTest::CompactRangeSkipFlushAfterDelay:PreFlush"); + ASSERT_OK(Put(std::to_string(0), rnd.RandomString(1024))); + ASSERT_OK(dbfull()->Flush(flush_opts)); + ASSERT_OK(Put(std::to_string(0), rnd.RandomString(1024))); + TEST_SYNC_POINT( + "DBCompactionTest::CompactRangeSkipFlushAfterDelay:PostFlush"); + manual_compaction_thread.join(); + + // If CompactRange's flush was skipped, the final Put above will still be + // in the active memtable. + std::string num_keys_in_memtable; + ASSERT_TRUE(db_->GetProperty(DB::Properties::kNumEntriesActiveMemTable, + &num_keys_in_memtable)); + ASSERT_EQ(std::to_string(1), num_keys_in_memtable); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBCompactionTest, CompactRangeFlushOverlappingMemtable) { + // Verify memtable only gets flushed if it contains data overlapping the range + // provided to `CompactRange`. Tests all kinds of overlap/non-overlap. + const int kNumEndpointKeys = 5; + std::string keys[kNumEndpointKeys] = {"a", "b", "c", "d", "e"}; + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + Reopen(options); + + // One extra iteration for nullptr, which means left side of interval is + // unbounded. + for (int i = 0; i <= kNumEndpointKeys; ++i) { + Slice begin; + Slice* begin_ptr; + if (i == 0) { + begin_ptr = nullptr; + } else { + begin = keys[i - 1]; + begin_ptr = &begin; + } + // Start at `i` so right endpoint comes after left endpoint. One extra + // iteration for nullptr, which means right side of interval is unbounded. + for (int j = std::max(0, i - 1); j <= kNumEndpointKeys; ++j) { + Slice end; + Slice* end_ptr; + if (j == kNumEndpointKeys) { + end_ptr = nullptr; + } else { + end = keys[j]; + end_ptr = &end; + } + ASSERT_OK(Put("b", "val")); + ASSERT_OK(Put("d", "val")); + CompactRangeOptions compact_range_opts; + ASSERT_OK(db_->CompactRange(compact_range_opts, begin_ptr, end_ptr)); + + uint64_t get_prop_tmp, num_memtable_entries = 0; + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kNumEntriesImmMemTables, + &get_prop_tmp)); + num_memtable_entries += get_prop_tmp; + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kNumEntriesActiveMemTable, + &get_prop_tmp)); + num_memtable_entries += get_prop_tmp; + if (begin_ptr == nullptr || end_ptr == nullptr || + (i <= 4 && j >= 1 && (begin != "c" || end != "c"))) { + // In this case `CompactRange`'s range overlapped in some way with the + // memtable's range, so flush should've happened. Then "b" and "d" won't + // be in the memtable. + ASSERT_EQ(0, num_memtable_entries); + } else { + ASSERT_EQ(2, num_memtable_entries); + // flush anyways to prepare for next iteration + ASSERT_OK(db_->Flush(FlushOptions())); + } + } + } +} + +TEST_F(DBCompactionTest, CompactionStatsTest) { + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = 2; + CompactionStatsCollector* collector = new CompactionStatsCollector(); + options.listeners.emplace_back(collector); + DestroyAndReopen(options); + + for (int i = 0; i < 32; i++) { + for (int j = 0; j < 5000; j++) { + ASSERT_OK(Put(std::to_string(j), std::string(1, 'A'))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ColumnFamilyHandleImpl* cfh = + static_cast(dbfull()->DefaultColumnFamily()); + ColumnFamilyData* cfd = cfh->cfd(); + + VerifyCompactionStats(*cfd, *collector); +} + +TEST_F(DBCompactionTest, SubcompactionEvent) { + class SubCompactionEventListener : public EventListener { + public: + void OnCompactionBegin(DB* /*db*/, const CompactionJobInfo& ci) override { + InstrumentedMutexLock l(&mutex_); + ASSERT_EQ(running_compactions_.find(ci.job_id), + running_compactions_.end()); + running_compactions_.emplace(ci.job_id, std::unordered_set()); + } + + void OnCompactionCompleted(DB* /*db*/, + const CompactionJobInfo& ci) override { + InstrumentedMutexLock l(&mutex_); + auto it = running_compactions_.find(ci.job_id); + ASSERT_NE(it, running_compactions_.end()); + ASSERT_EQ(it->second.size(), 0); + running_compactions_.erase(it); + } + + void OnSubcompactionBegin(const SubcompactionJobInfo& si) override { + InstrumentedMutexLock l(&mutex_); + auto it = running_compactions_.find(si.job_id); + ASSERT_NE(it, running_compactions_.end()); + auto r = it->second.insert(si.subcompaction_job_id); + ASSERT_TRUE(r.second); // each subcompaction_job_id should be different + total_subcompaction_cnt_++; + } + + void OnSubcompactionCompleted(const SubcompactionJobInfo& si) override { + InstrumentedMutexLock l(&mutex_); + auto it = running_compactions_.find(si.job_id); + ASSERT_NE(it, running_compactions_.end()); + auto r = it->second.erase(si.subcompaction_job_id); + ASSERT_EQ(r, 1); + } + + size_t GetRunningCompactionCount() { + InstrumentedMutexLock l(&mutex_); + return running_compactions_.size(); + } + + size_t GetTotalSubcompactionCount() { + InstrumentedMutexLock l(&mutex_); + return total_subcompaction_cnt_; + } + + private: + InstrumentedMutex mutex_; + std::unordered_map> running_compactions_; + size_t total_subcompaction_cnt_ = 0; + }; + + Options options = CurrentOptions(); + options.target_file_size_base = 1024; + options.level0_file_num_compaction_trigger = 10; + auto* listener = new SubCompactionEventListener(); + options.listeners.emplace_back(listener); + + DestroyAndReopen(options); + + // generate 4 files @ L2 + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 10; j++) { + int key_id = i * 10 + j; + ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id))); + } + ASSERT_OK(Flush()); + } + MoveFilesToLevel(2); + + // generate 2 files @ L1 which overlaps with L2 files + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 10; j++) { + int key_id = i * 20 + j * 2; + ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id))); + } + ASSERT_OK(Flush()); + } + MoveFilesToLevel(1); + ASSERT_EQ(FilesPerLevel(), "0,2,4"); + + CompactRangeOptions comp_opts; + comp_opts.max_subcompactions = 4; + Status s = dbfull()->CompactRange(comp_opts, nullptr, nullptr); + ASSERT_OK(s); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // make sure there's no running compaction + ASSERT_EQ(listener->GetRunningCompactionCount(), 0); + // and sub compaction is triggered + ASSERT_GT(listener->GetTotalSubcompactionCount(), 0); +} + +TEST_F(DBCompactionTest, CompactFilesOutputRangeConflict) { + // LSM setup: + // L1: [ba bz] + // L2: [a b] [c d] + // L3: [a b] [c d] + // + // Thread 1: Thread 2: + // Begin compacting all L2->L3 + // Compact [ba bz] L1->L3 + // End compacting all L2->L3 + // + // The compaction operation in thread 2 should be disallowed because the range + // overlaps with the compaction in thread 1, which also covers that range in + // L3. + Options options = CurrentOptions(); + FlushedFileCollector* collector = new FlushedFileCollector(); + options.listeners.emplace_back(collector); + Reopen(options); + + for (int level = 3; level >= 2; --level) { + ASSERT_OK(Put("a", "val")); + ASSERT_OK(Put("b", "val")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("c", "val")); + ASSERT_OK(Put("d", "val")); + ASSERT_OK(Flush()); + MoveFilesToLevel(level); + } + ASSERT_OK(Put("ba", "val")); + ASSERT_OK(Put("bz", "val")); + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + + SyncPoint::GetInstance()->LoadDependency({ + {"CompactFilesImpl:0", + "DBCompactionTest::CompactFilesOutputRangeConflict:Thread2Begin"}, + {"DBCompactionTest::CompactFilesOutputRangeConflict:Thread2End", + "CompactFilesImpl:1"}, + }); + SyncPoint::GetInstance()->EnableProcessing(); + + auto bg_thread = port::Thread([&]() { + // Thread 1 + std::vector filenames = collector->GetFlushedFiles(); + filenames.pop_back(); + ASSERT_OK(db_->CompactFiles(CompactionOptions(), filenames, + 3 /* output_level */)); + }); + + // Thread 2 + TEST_SYNC_POINT( + "DBCompactionTest::CompactFilesOutputRangeConflict:Thread2Begin"); + std::string filename = collector->GetFlushedFiles().back(); + ASSERT_FALSE( + db_->CompactFiles(CompactionOptions(), {filename}, 3 /* output_level */) + .ok()); + TEST_SYNC_POINT( + "DBCompactionTest::CompactFilesOutputRangeConflict:Thread2End"); + + bg_thread.join(); +} + +TEST_F(DBCompactionTest, CompactionHasEmptyOutput) { + Options options = CurrentOptions(); + SstStatsCollector* collector = new SstStatsCollector(); + options.level0_file_num_compaction_trigger = 2; + options.listeners.emplace_back(collector); + Reopen(options); + + // Make sure the L0 files overlap to prevent trivial move. + ASSERT_OK(Put("a", "val")); + ASSERT_OK(Put("b", "val")); + ASSERT_OK(Flush()); + ASSERT_OK(Delete("a")); + ASSERT_OK(Delete("b")); + ASSERT_OK(Flush()); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + ASSERT_EQ(NumTableFilesAtLevel(1), 0); + + // Expect one file creation to start for each flush, and zero for compaction + // since no keys are written. + ASSERT_EQ(2, collector->num_ssts_creation_started()); +} + +TEST_F(DBCompactionTest, CompactionLimiter) { + const int kNumKeysPerFile = 10; + const int kMaxBackgroundThreads = 64; + + struct CompactionLimiter { + std::string name; + int limit_tasks; + int max_tasks; + int tasks; + std::shared_ptr limiter; + }; + + std::vector limiter_settings; + limiter_settings.push_back({"limiter_1", 1, 0, 0, nullptr}); + limiter_settings.push_back({"limiter_2", 2, 0, 0, nullptr}); + limiter_settings.push_back({"limiter_3", 3, 0, 0, nullptr}); + + for (auto& ls : limiter_settings) { + ls.limiter.reset(NewConcurrentTaskLimiter(ls.name, ls.limit_tasks)); + } + + std::shared_ptr unique_limiter( + NewConcurrentTaskLimiter("unique_limiter", -1)); + + const char* cf_names[] = {"default", "0", "1", "2", "3", "4", "5", "6", "7", + "8", "9", "a", "b", "c", "d", "e", "f"}; + const unsigned int cf_count = sizeof cf_names / sizeof cf_names[0]; + + std::unordered_map cf_to_limiter; + + Options options = CurrentOptions(); + options.write_buffer_size = 110 * 1024; // 110KB + options.arena_block_size = 4096; + options.num_levels = 3; + options.level0_file_num_compaction_trigger = 4; + options.level0_slowdown_writes_trigger = 64; + options.level0_stop_writes_trigger = 64; + options.max_background_jobs = kMaxBackgroundThreads; // Enough threads + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + options.max_write_buffer_number = 10; // Enough memtables + DestroyAndReopen(options); + + std::vector option_vector; + option_vector.reserve(cf_count); + + for (unsigned int cf = 0; cf < cf_count; cf++) { + ColumnFamilyOptions cf_opt(options); + if (cf == 0) { + // "Default" CF does't use compaction limiter + cf_opt.compaction_thread_limiter = nullptr; + } else if (cf == 1) { + // "1" CF uses bypass compaction limiter + unique_limiter->SetMaxOutstandingTask(-1); + cf_opt.compaction_thread_limiter = unique_limiter; + } else { + // Assign limiter by mod + auto& ls = limiter_settings[cf % 3]; + cf_opt.compaction_thread_limiter = ls.limiter; + cf_to_limiter[cf_names[cf]] = &ls; + } + option_vector.emplace_back(DBOptions(options), cf_opt); + } + + for (unsigned int cf = 1; cf < cf_count; cf++) { + CreateColumnFamilies({cf_names[cf]}, option_vector[cf]); + } + + ReopenWithColumnFamilies( + std::vector(cf_names, cf_names + cf_count), option_vector); + + port::Mutex mutex; + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:BeforeCompaction", [&](void* arg) { + const auto& cf_name = static_cast(arg)->GetName(); + auto iter = cf_to_limiter.find(cf_name); + if (iter != cf_to_limiter.end()) { + MutexLock l(&mutex); + ASSERT_GE(iter->second->limit_tasks, ++iter->second->tasks); + iter->second->max_tasks = + std::max(iter->second->max_tasks, iter->second->limit_tasks); + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:AfterCompaction", [&](void* arg) { + const auto& cf_name = static_cast(arg)->GetName(); + auto iter = cf_to_limiter.find(cf_name); + if (iter != cf_to_limiter.end()) { + MutexLock l(&mutex); + ASSERT_GE(--iter->second->tasks, 0); + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Block all compact threads in thread pool. + const size_t kTotalFlushTasks = kMaxBackgroundThreads / 4; + const size_t kTotalCompactTasks = kMaxBackgroundThreads - kTotalFlushTasks; + env_->SetBackgroundThreads((int)kTotalFlushTasks, Env::HIGH); + env_->SetBackgroundThreads((int)kTotalCompactTasks, Env::LOW); + + test::SleepingBackgroundTask sleeping_compact_tasks[kTotalCompactTasks]; + + // Block all compaction threads in thread pool. + for (size_t i = 0; i < kTotalCompactTasks; i++) { + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, + &sleeping_compact_tasks[i], Env::LOW); + sleeping_compact_tasks[i].WaitUntilSleeping(); + } + + int keyIndex = 0; + + for (int n = 0; n < options.level0_file_num_compaction_trigger; n++) { + for (unsigned int cf = 0; cf < cf_count; cf++) { + // All L0s should overlap with each other + for (int i = 0; i < kNumKeysPerFile; i++) { + ASSERT_OK(Put(cf, Key(i), "")); + } + // put extra key to trigger flush + ASSERT_OK(Put(cf, "", "")); + } + + for (unsigned int cf = 0; cf < cf_count; cf++) { + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[cf])); + } + } + + // Enough L0 files to trigger compaction + for (unsigned int cf = 0; cf < cf_count; cf++) { + ASSERT_EQ(NumTableFilesAtLevel(0, cf), + options.level0_file_num_compaction_trigger); + } + + // Create more files for one column family, which triggers speed up + // condition, all compactions will be scheduled. + for (int num = 0; num < options.level0_file_num_compaction_trigger; num++) { + for (int i = 0; i < kNumKeysPerFile; i++) { + ASSERT_OK(Put(0, Key(i), "")); + } + // put extra key to trigger flush + ASSERT_OK(Put(0, "", "")); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[0])); + ASSERT_EQ(options.level0_file_num_compaction_trigger + num + 1, + NumTableFilesAtLevel(0, 0)); + } + + // All CFs are pending compaction + ASSERT_EQ(cf_count, env_->GetThreadPoolQueueLen(Env::LOW)); + + // Unblock all compaction threads + for (size_t i = 0; i < kTotalCompactTasks; i++) { + sleeping_compact_tasks[i].WakeUp(); + sleeping_compact_tasks[i].WaitUntilDone(); + } + + for (unsigned int cf = 0; cf < cf_count; cf++) { + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[cf])); + } + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // Max outstanding compact tasks reached limit + for (auto& ls : limiter_settings) { + ASSERT_EQ(ls.limit_tasks, ls.max_tasks); + ASSERT_EQ(0, ls.limiter->GetOutstandingTask()); + } + + // test manual compaction under a fully throttled limiter + int cf_test = 1; + unique_limiter->SetMaxOutstandingTask(0); + + // flush one more file to cf 1 + for (int i = 0; i < kNumKeysPerFile; i++) { + ASSERT_OK(Put(cf_test, Key(keyIndex++), "")); + } + // put extra key to trigger flush + ASSERT_OK(Put(cf_test, "", "")); + + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[cf_test])); + ASSERT_EQ(1, NumTableFilesAtLevel(0, cf_test)); + + Compact(cf_test, Key(0), Key(keyIndex)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); +} + +INSTANTIATE_TEST_CASE_P(DBCompactionTestWithParam, DBCompactionTestWithParam, + ::testing::Values(std::make_tuple(1, true), + std::make_tuple(1, false), + std::make_tuple(4, true), + std::make_tuple(4, false))); + +TEST_P(DBCompactionDirectIOTest, DirectIO) { + Options options = CurrentOptions(); + Destroy(options); + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.use_direct_io_for_flush_and_compaction = GetParam(); + options.env = MockEnv::Create(Env::Default()); + Reopen(options); + bool readahead = false; + SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::OpenCompactionOutputFile", [&](void* arg) { + bool* use_direct_writes = static_cast(arg); + ASSERT_EQ(*use_direct_writes, + options.use_direct_io_for_flush_and_compaction); + }); + if (options.use_direct_io_for_flush_and_compaction) { + SyncPoint::GetInstance()->SetCallBack( + "SanitizeOptions:direct_io", [&](void* /*arg*/) { readahead = true; }); + } + SyncPoint::GetInstance()->EnableProcessing(); + CreateAndReopenWithCF({"pikachu"}, options); + MakeTables(3, "p", "q", 1); + ASSERT_EQ("1,1,1", FilesPerLevel(1)); + Compact(1, "p", "q"); + ASSERT_EQ(readahead, options.use_direct_reads); + ASSERT_EQ("0,0,1", FilesPerLevel(1)); + Destroy(options); + delete options.env; +} + +INSTANTIATE_TEST_CASE_P(DBCompactionDirectIOTest, DBCompactionDirectIOTest, + testing::Bool()); + +class CompactionPriTest : public DBTestBase, + public testing::WithParamInterface { + public: + CompactionPriTest() + : DBTestBase("compaction_pri_test", /*env_do_fsync=*/true) { + compaction_pri_ = GetParam(); + } + + // Required if inheriting from testing::WithParamInterface<> + static void SetUpTestCase() {} + static void TearDownTestCase() {} + + uint32_t compaction_pri_; +}; + +TEST_P(CompactionPriTest, Test) { + Options options = CurrentOptions(); + options.write_buffer_size = 16 * 1024; + options.compaction_pri = static_cast(compaction_pri_); + options.hard_pending_compaction_bytes_limit = 256 * 1024; + options.max_bytes_for_level_base = 64 * 1024; + options.max_bytes_for_level_multiplier = 4; + options.compression = kNoCompression; + + DestroyAndReopen(options); + + Random rnd(301); + const int kNKeys = 5000; + int keys[kNKeys]; + for (int i = 0; i < kNKeys; i++) { + keys[i] = i; + } + RandomShuffle(std::begin(keys), std::end(keys), rnd.Next()); + + for (int i = 0; i < kNKeys; i++) { + ASSERT_OK(Put(Key(keys[i]), rnd.RandomString(102))); + } + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + for (int i = 0; i < kNKeys; i++) { + ASSERT_NE("NOT_FOUND", Get(Key(i))); + } +} + +INSTANTIATE_TEST_CASE_P( + CompactionPriTest, CompactionPriTest, + ::testing::Values(CompactionPri::kByCompensatedSize, + CompactionPri::kOldestLargestSeqFirst, + CompactionPri::kOldestSmallestSeqFirst, + CompactionPri::kMinOverlappingRatio, + CompactionPri::kRoundRobin)); + +TEST_F(DBCompactionTest, PersistRoundRobinCompactCursor) { + Options options = CurrentOptions(); + options.write_buffer_size = 16 * 1024; + options.max_bytes_for_level_base = 128 * 1024; + options.target_file_size_base = 64 * 1024; + options.level0_file_num_compaction_trigger = 4; + options.compaction_pri = CompactionPri::kRoundRobin; + options.max_bytes_for_level_multiplier = 4; + options.num_levels = 3; + options.compression = kNoCompression; + + DestroyAndReopen(options); + + Random rnd(301); + + // 30 Files in L0 to trigger compactions between L1 and L2 + for (int i = 0; i < 30; i++) { + for (int j = 0; j < 16; j++) { + ASSERT_OK(Put(rnd.RandomString(24), rnd.RandomString(1000))); + } + ASSERT_OK(Flush()); + } + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + VersionSet* const versions = dbfull()->GetVersionSet(); + assert(versions); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(cfd, nullptr); + + Version* const current = cfd->current(); + ASSERT_NE(current, nullptr); + + const VersionStorageInfo* const storage_info = current->storage_info(); + ASSERT_NE(storage_info, nullptr); + + const std::vector compact_cursors = + storage_info->GetCompactCursors(); + + Reopen(options); + + VersionSet* const reopened_versions = dbfull()->GetVersionSet(); + assert(reopened_versions); + + ColumnFamilyData* const reopened_cfd = + reopened_versions->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(reopened_cfd, nullptr); + + Version* const reopened_current = reopened_cfd->current(); + ASSERT_NE(reopened_current, nullptr); + + const VersionStorageInfo* const reopened_storage_info = + reopened_current->storage_info(); + ASSERT_NE(reopened_storage_info, nullptr); + + const std::vector reopened_compact_cursors = + reopened_storage_info->GetCompactCursors(); + const auto icmp = reopened_storage_info->InternalComparator(); + ASSERT_EQ(compact_cursors.size(), reopened_compact_cursors.size()); + for (size_t i = 0; i < compact_cursors.size(); i++) { + if (compact_cursors[i].Valid()) { + ASSERT_EQ(0, + icmp->Compare(compact_cursors[i], reopened_compact_cursors[i])); + } else { + ASSERT_TRUE(!reopened_compact_cursors[i].Valid()); + } + } +} + +TEST_P(RoundRobinSubcompactionsAgainstPressureToken, PressureTokenTest) { + const int kKeysPerBuffer = 100; + Options options = CurrentOptions(); + options.num_levels = 4; + options.max_bytes_for_level_multiplier = 2; + options.level0_file_num_compaction_trigger = 4; + options.target_file_size_base = kKeysPerBuffer * 1024; + options.compaction_pri = CompactionPri::kRoundRobin; + options.max_bytes_for_level_base = 8 * kKeysPerBuffer * 1024; + options.disable_auto_compactions = true; + // Setup 7 threads but limited subcompactions so that + // RoundRobin requires extra compactions from reserved threads + options.max_subcompactions = 1; + options.max_background_compactions = 7; + options.max_compaction_bytes = 100000000; + DestroyAndReopen(options); + env_->SetBackgroundThreads(7, Env::LOW); + + Random rnd(301); + const std::vector files_per_level = {0, 15, 25}; + for (int lvl = 2; lvl > 0; lvl--) { + for (int i = 0; i < files_per_level[lvl]; i++) { + for (int j = 0; j < kKeysPerBuffer; j++) { + // Add (lvl-1) to ensure nearly equivallent number of files + // in L2 are overlapped with fils selected to compact from + // L1 + ASSERT_OK(Put(Key(2 * i * kKeysPerBuffer + 2 * j + (lvl - 1)), + rnd.RandomString(1010))); + } + ASSERT_OK(Flush()); + } + MoveFilesToLevel(lvl); + ASSERT_EQ(files_per_level[lvl], NumTableFilesAtLevel(lvl, 0)); + } + // 15 files in L1; 25 files in L2 + + // This is a variable for making sure the following callback is called + // and the assertions in it are indeed excuted. + bool num_planned_subcompactions_verified = false; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::GenSubcompactionBoundaries:0", [&](void* arg) { + uint64_t num_planned_subcompactions = *(static_cast(arg)); + if (grab_pressure_token_) { + // 7 files are selected for round-robin under auto + // compaction. The number of planned subcompaction is restricted by + // the limited number of max_background_compactions + ASSERT_EQ(num_planned_subcompactions, 7); + } else { + ASSERT_EQ(num_planned_subcompactions, 1); + } + num_planned_subcompactions_verified = true; + }); + + // The following 3 dependencies have to be added to ensure the auto + // compaction and the pressure token is correctly enabled. Same for + // RoundRobinSubcompactionsUsingResources and + // DBCompactionTest.RoundRobinSubcompactionsShrinkResources + SyncPoint::GetInstance()->LoadDependency( + {{"RoundRobinSubcompactionsAgainstPressureToken:0", + "BackgroundCallCompaction:0"}, + {"CompactionJob::AcquireSubcompactionResources:0", + "RoundRobinSubcompactionsAgainstPressureToken:1"}, + {"RoundRobinSubcompactionsAgainstPressureToken:2", + "CompactionJob::AcquireSubcompactionResources:1"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(dbfull()->EnableAutoCompaction({dbfull()->DefaultColumnFamily()})); + TEST_SYNC_POINT("RoundRobinSubcompactionsAgainstPressureToken:0"); + TEST_SYNC_POINT("RoundRobinSubcompactionsAgainstPressureToken:1"); + std::unique_ptr pressure_token; + if (grab_pressure_token_) { + pressure_token = + dbfull()->TEST_write_controler().GetCompactionPressureToken(); + } + TEST_SYNC_POINT("RoundRobinSubcompactionsAgainstPressureToken:2"); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_TRUE(num_planned_subcompactions_verified); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +INSTANTIATE_TEST_CASE_P(RoundRobinSubcompactionsAgainstPressureToken, + RoundRobinSubcompactionsAgainstPressureToken, + testing::Bool()); + +TEST_P(RoundRobinSubcompactionsAgainstResources, SubcompactionsUsingResources) { + const int kKeysPerBuffer = 200; + Options options = CurrentOptions(); + options.num_levels = 4; + options.level0_file_num_compaction_trigger = 3; + options.target_file_size_base = kKeysPerBuffer * 1024; + options.compaction_pri = CompactionPri::kRoundRobin; + options.max_bytes_for_level_base = 30 * kKeysPerBuffer * 1024; + options.disable_auto_compactions = true; + options.max_subcompactions = 1; + options.max_background_compactions = max_compaction_limits_; + // Set a large number for max_compaction_bytes so that one round-robin + // compaction is enough to make post-compaction L1 size less than + // the maximum size (this test assumes only one round-robin compaction + // is triggered by kLevelMaxLevelSize) + options.max_compaction_bytes = 100000000; + + DestroyAndReopen(options); + env_->SetBackgroundThreads(total_low_pri_threads_, Env::LOW); + + Random rnd(301); + const std::vector files_per_level = {0, 40, 100}; + for (int lvl = 2; lvl > 0; lvl--) { + for (int i = 0; i < files_per_level[lvl]; i++) { + for (int j = 0; j < kKeysPerBuffer; j++) { + // Add (lvl-1) to ensure nearly equivallent number of files + // in L2 are overlapped with fils selected to compact from + // L1 + ASSERT_OK(Put(Key(2 * i * kKeysPerBuffer + 2 * j + (lvl - 1)), + rnd.RandomString(1010))); + } + ASSERT_OK(Flush()); + } + MoveFilesToLevel(lvl); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(files_per_level[lvl], NumTableFilesAtLevel(lvl, 0)); + } + + // 40 files in L1; 100 files in L2 + // This is a variable for making sure the following callback is called + // and the assertions in it are indeed excuted. + bool num_planned_subcompactions_verified = false; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::GenSubcompactionBoundaries:0", [&](void* arg) { + uint64_t num_planned_subcompactions = *(static_cast(arg)); + // More than 10 files are selected for round-robin under auto + // compaction. The number of planned subcompaction is restricted by + // the minimum number between available threads and compaction limits + ASSERT_EQ(num_planned_subcompactions - options.max_subcompactions, + std::min(total_low_pri_threads_, max_compaction_limits_) - 1); + num_planned_subcompactions_verified = true; + }); + SyncPoint::GetInstance()->LoadDependency( + {{"RoundRobinSubcompactionsAgainstResources:0", + "BackgroundCallCompaction:0"}, + {"CompactionJob::AcquireSubcompactionResources:0", + "RoundRobinSubcompactionsAgainstResources:1"}, + {"RoundRobinSubcompactionsAgainstResources:2", + "CompactionJob::AcquireSubcompactionResources:1"}, + {"CompactionJob::ReleaseSubcompactionResources:0", + "RoundRobinSubcompactionsAgainstResources:3"}, + {"RoundRobinSubcompactionsAgainstResources:4", + "CompactionJob::ReleaseSubcompactionResources:1"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_OK(dbfull()->EnableAutoCompaction({dbfull()->DefaultColumnFamily()})); + TEST_SYNC_POINT("RoundRobinSubcompactionsAgainstResources:0"); + TEST_SYNC_POINT("RoundRobinSubcompactionsAgainstResources:1"); + auto pressure_token = + dbfull()->TEST_write_controler().GetCompactionPressureToken(); + + TEST_SYNC_POINT("RoundRobinSubcompactionsAgainstResources:2"); + TEST_SYNC_POINT("RoundRobinSubcompactionsAgainstResources:3"); + // We can reserve more threads now except one is being used + ASSERT_EQ(total_low_pri_threads_ - 1, + env_->ReserveThreads(total_low_pri_threads_, Env::Priority::LOW)); + ASSERT_EQ( + total_low_pri_threads_ - 1, + env_->ReleaseThreads(total_low_pri_threads_ - 1, Env::Priority::LOW)); + TEST_SYNC_POINT("RoundRobinSubcompactionsAgainstResources:4"); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_TRUE(num_planned_subcompactions_verified); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +INSTANTIATE_TEST_CASE_P(RoundRobinSubcompactionsAgainstResources, + RoundRobinSubcompactionsAgainstResources, + ::testing::Values(std::make_tuple(1, 5), + std::make_tuple(5, 1), + std::make_tuple(10, 5), + std::make_tuple(5, 10), + std::make_tuple(10, 10))); + +TEST_P(DBCompactionTestWithParam, RoundRobinWithoutAdditionalResources) { + const int kKeysPerBuffer = 200; + Options options = CurrentOptions(); + options.num_levels = 4; + options.level0_file_num_compaction_trigger = 3; + options.target_file_size_base = kKeysPerBuffer * 1024; + options.compaction_pri = CompactionPri::kRoundRobin; + options.max_bytes_for_level_base = 30 * kKeysPerBuffer * 1024; + options.disable_auto_compactions = true; + options.max_subcompactions = max_subcompactions_; + options.max_background_compactions = 1; + options.max_compaction_bytes = 100000000; + // Similar experiment setting as above except the max_subcompactions + // is given by max_subcompactions_ (1 or 4), and we fix the + // additional resources as (1, 1) and thus no more extra resources + // can be used + DestroyAndReopen(options); + env_->SetBackgroundThreads(1, Env::LOW); + + Random rnd(301); + const std::vector files_per_level = {0, 33, 100}; + for (int lvl = 2; lvl > 0; lvl--) { + for (int i = 0; i < files_per_level[lvl]; i++) { + for (int j = 0; j < kKeysPerBuffer; j++) { + // Add (lvl-1) to ensure nearly equivallent number of files + // in L2 are overlapped with fils selected to compact from + // L1 + ASSERT_OK(Put(Key(2 * i * kKeysPerBuffer + 2 * j + (lvl - 1)), + rnd.RandomString(1010))); + } + ASSERT_OK(Flush()); + } + MoveFilesToLevel(lvl); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(files_per_level[lvl], NumTableFilesAtLevel(lvl, 0)); + } + + // 33 files in L1; 100 files in L2 + // This is a variable for making sure the following callback is called + // and the assertions in it are indeed excuted. + bool num_planned_subcompactions_verified = false; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::GenSubcompactionBoundaries:0", [&](void* arg) { + uint64_t num_planned_subcompactions = *(static_cast(arg)); + // At most 4 files are selected for round-robin under auto + // compaction. The number of planned subcompaction is restricted by + // the max_subcompactions since no extra resources can be used + ASSERT_EQ(num_planned_subcompactions, options.max_subcompactions); + num_planned_subcompactions_verified = true; + }); + // No need to setup dependency for pressure token since + // AcquireSubcompactionResources may not be called and it anyway cannot + // reserve any additional resources + SyncPoint::GetInstance()->LoadDependency( + {{"DBCompactionTest::RoundRobinWithoutAdditionalResources:0", + "BackgroundCallCompaction:0"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_OK(dbfull()->EnableAutoCompaction({dbfull()->DefaultColumnFamily()})); + TEST_SYNC_POINT("DBCompactionTest::RoundRobinWithoutAdditionalResources:0"); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_TRUE(num_planned_subcompactions_verified); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(DBCompactionTest, RoundRobinCutOutputAtCompactCursor) { + Options options = CurrentOptions(); + options.num_levels = 3; + options.compression = kNoCompression; + options.write_buffer_size = 4 * 1024; + options.max_bytes_for_level_base = 64 * 1024; + options.max_bytes_for_level_multiplier = 4; + options.level0_file_num_compaction_trigger = 4; + options.compaction_pri = CompactionPri::kRoundRobin; + + DestroyAndReopen(options); + + VersionSet* const versions = dbfull()->GetVersionSet(); + assert(versions); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(cfd, nullptr); + + Version* const current = cfd->current(); + ASSERT_NE(current, nullptr); + + VersionStorageInfo* storage_info = current->storage_info(); + ASSERT_NE(storage_info, nullptr); + + const InternalKey split_cursor = InternalKey(Key(600), 100, kTypeValue); + storage_info->AddCursorForOneLevel(2, split_cursor); + + Random rnd(301); + + for (int i = 0; i < 50; i++) { + for (int j = 0; j < 50; j++) { + ASSERT_OK(Put(Key(j * 2 + i * 100), rnd.RandomString(102))); + } + } + // Add more overlapping files (avoid trivial move) to trigger compaction that + // output files in L2. Note that trivial move does not trigger compaction and + // in that case the cursor is not necessarily the boundary of file. + for (int i = 0; i < 50; i++) { + for (int j = 0; j < 50; j++) { + ASSERT_OK(Put(Key(j * 2 + 1 + i * 100), rnd.RandomString(1014))); + } + } + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + std::vector> level_to_files; + dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(), + &level_to_files); + const auto icmp = cfd->current()->storage_info()->InternalComparator(); + // Files in level 2 should be split by the cursor + for (const auto& file : level_to_files[2]) { + ASSERT_TRUE( + icmp->Compare(file.smallest.Encode(), split_cursor.Encode()) >= 0 || + icmp->Compare(file.largest.Encode(), split_cursor.Encode()) < 0); + } +} + +class NoopMergeOperator : public MergeOperator { + public: + NoopMergeOperator() {} + + bool FullMergeV2(const MergeOperationInput& /*merge_in*/, + MergeOperationOutput* merge_out) const override { + std::string val("bar"); + merge_out->new_value = val; + return true; + } + + const char* Name() const override { return "Noop"; } +}; + +TEST_F(DBCompactionTest, PartialManualCompaction) { + Options opts = CurrentOptions(); + opts.num_levels = 3; + opts.level0_file_num_compaction_trigger = 10; + opts.compression = kNoCompression; + opts.merge_operator.reset(new NoopMergeOperator()); + opts.target_file_size_base = 10240; + DestroyAndReopen(opts); + + Random rnd(301); + for (auto i = 0; i < 8; ++i) { + for (auto j = 0; j < 10; ++j) { + ASSERT_OK(Merge("foo", rnd.RandomString(1024))); + } + ASSERT_OK(Flush()); + } + + MoveFilesToLevel(2); + + std::string prop; + EXPECT_TRUE(dbfull()->GetProperty(DB::Properties::kLiveSstFilesSize, &prop)); + uint64_t max_compaction_bytes = atoi(prop.c_str()) / 2; + ASSERT_OK(dbfull()->SetOptions( + {{"max_compaction_bytes", std::to_string(max_compaction_bytes)}})); + + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForceOptimized; + ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr)); +} + +TEST_F(DBCompactionTest, ManualCompactionFailsInReadOnlyMode) { + // Regression test for bug where manual compaction hangs forever when the DB + // is in read-only mode. Verify it now at least returns, despite failing. + const int kNumL0Files = 4; + std::unique_ptr mock_env( + new FaultInjectionTestEnv(env_)); + Options opts = CurrentOptions(); + opts.disable_auto_compactions = true; + opts.env = mock_env.get(); + DestroyAndReopen(opts); + + Random rnd(301); + for (int i = 0; i < kNumL0Files; ++i) { + // Make sure files are overlapping in key-range to prevent trivial move. + ASSERT_OK(Put("key1", rnd.RandomString(1024))); + ASSERT_OK(Put("key2", rnd.RandomString(1024))); + ASSERT_OK(Flush()); + } + ASSERT_EQ(kNumL0Files, NumTableFilesAtLevel(0)); + + // Enter read-only mode by failing a write. + mock_env->SetFilesystemActive(false); + // Make sure this is outside `CompactRange`'s range so that it doesn't fail + // early trying to flush memtable. + ASSERT_NOK(Put("key3", rnd.RandomString(1024))); + + // In the bug scenario, the first manual compaction would fail and forget to + // unregister itself, causing the second one to hang forever due to conflict + // with a non-running compaction. + CompactRangeOptions cro; + cro.exclusive_manual_compaction = false; + Slice begin_key("key1"); + Slice end_key("key2"); + ASSERT_NOK(dbfull()->CompactRange(cro, &begin_key, &end_key)); + ASSERT_NOK(dbfull()->CompactRange(cro, &begin_key, &end_key)); + + // Close before mock_env destruct. + Close(); +} + +// ManualCompactionBottomLevelOptimization tests the bottom level manual +// compaction optimization to skip recompacting files created by Ln-1 to Ln +// compaction +TEST_F(DBCompactionTest, ManualCompactionBottomLevelOptimized) { + Options opts = CurrentOptions(); + opts.num_levels = 3; + opts.level0_file_num_compaction_trigger = 5; + opts.compression = kNoCompression; + opts.merge_operator.reset(new NoopMergeOperator()); + opts.target_file_size_base = 1024; + opts.max_bytes_for_level_multiplier = 2; + opts.disable_auto_compactions = true; + DestroyAndReopen(opts); + ColumnFamilyHandleImpl* cfh = + static_cast(dbfull()->DefaultColumnFamily()); + ColumnFamilyData* cfd = cfh->cfd(); + InternalStats* internal_stats_ptr = cfd->internal_stats(); + ASSERT_NE(internal_stats_ptr, nullptr); + + Random rnd(301); + for (auto i = 0; i < 8; ++i) { + for (auto j = 0; j < 10; ++j) { + ASSERT_OK( + Put("foo" + std::to_string(i * 10 + j), rnd.RandomString(1024))); + } + ASSERT_OK(Flush()); + } + + MoveFilesToLevel(2); + + for (auto i = 0; i < 8; ++i) { + for (auto j = 0; j < 10; ++j) { + ASSERT_OK( + Put("bar" + std::to_string(i * 10 + j), rnd.RandomString(1024))); + } + ASSERT_OK(Flush()); + } + const std::vector& comp_stats = + internal_stats_ptr->TEST_GetCompactionStats(); + int num = comp_stats[2].num_input_files_in_output_level; + ASSERT_EQ(num, 0); + + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForceOptimized; + ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr)); + + const std::vector& comp_stats2 = + internal_stats_ptr->TEST_GetCompactionStats(); + num = comp_stats2[2].num_input_files_in_output_level; + ASSERT_EQ(num, 0); +} + +TEST_F(DBCompactionTest, ManualCompactionMax) { + uint64_t l1_avg_size = 0, l2_avg_size = 0; + auto generate_sst_func = [&]() { + Random rnd(301); + for (auto i = 0; i < 100; i++) { + for (auto j = 0; j < 10; j++) { + ASSERT_OK(Put(Key(i * 10 + j), rnd.RandomString(1024))); + } + ASSERT_OK(Flush()); + } + MoveFilesToLevel(2); + + for (auto i = 0; i < 10; i++) { + for (auto j = 0; j < 10; j++) { + ASSERT_OK(Put(Key(i * 100 + j * 10), rnd.RandomString(1024))); + } + ASSERT_OK(Flush()); + } + MoveFilesToLevel(1); + + std::vector> level_to_files; + dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(), + &level_to_files); + + uint64_t total = 0; + for (const auto& file : level_to_files[1]) { + total += file.compensated_file_size; + } + l1_avg_size = total / level_to_files[1].size(); + + total = 0; + for (const auto& file : level_to_files[2]) { + total += file.compensated_file_size; + } + l2_avg_size = total / level_to_files[2].size(); + }; + + std::atomic_int num_compactions(0); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BGWorkCompaction", [&](void* /*arg*/) { ++num_compactions; }); + SyncPoint::GetInstance()->EnableProcessing(); + + Options opts = CurrentOptions(); + opts.disable_auto_compactions = true; + + // with default setting (1.6G by default), it should cover all files in 1 + // compaction + DestroyAndReopen(opts); + generate_sst_func(); + num_compactions.store(0); + CompactRangeOptions cro; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_TRUE(num_compactions.load() == 1); + + // split the compaction to 5 + int num_split = 5; + DestroyAndReopen(opts); + generate_sst_func(); + uint64_t total_size = (l1_avg_size * 10) + (l2_avg_size * 100); + opts.max_compaction_bytes = total_size / num_split; + opts.target_file_size_base = total_size / num_split; + Reopen(opts); + num_compactions.store(0); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_TRUE(num_compactions.load() == num_split); + + // very small max_compaction_bytes, it should still move forward + opts.max_compaction_bytes = l1_avg_size / 2; + opts.target_file_size_base = l1_avg_size / 2; + DestroyAndReopen(opts); + generate_sst_func(); + num_compactions.store(0); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_TRUE(num_compactions.load() > 10); + + // dynamically set the option + num_split = 2; + opts.max_compaction_bytes = 0; + DestroyAndReopen(opts); + generate_sst_func(); + total_size = (l1_avg_size * 10) + (l2_avg_size * 100); + Status s = db_->SetOptions( + {{"max_compaction_bytes", std::to_string(total_size / num_split)}, + {"target_file_size_base", std::to_string(total_size / num_split)}}); + ASSERT_OK(s); + + num_compactions.store(0); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_TRUE(num_compactions.load() == num_split); +} + +TEST_F(DBCompactionTest, CompactionDuringShutdown) { + Options opts = CurrentOptions(); + opts.level0_file_num_compaction_trigger = 2; + opts.disable_auto_compactions = true; + DestroyAndReopen(opts); + ColumnFamilyHandleImpl* cfh = + static_cast(dbfull()->DefaultColumnFamily()); + ColumnFamilyData* cfd = cfh->cfd(); + InternalStats* internal_stats_ptr = cfd->internal_stats(); + ASSERT_NE(internal_stats_ptr, nullptr); + + Random rnd(301); + for (auto i = 0; i < 2; ++i) { + for (auto j = 0; j < 10; ++j) { + ASSERT_OK( + Put("foo" + std::to_string(i * 10 + j), rnd.RandomString(1024))); + } + ASSERT_OK(Flush()); + } + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial:BeforeRun", + [&](void* /*arg*/) { dbfull()->shutting_down_.store(true); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + Status s = dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr); + ASSERT_TRUE(s.ok() || s.IsShutdownInProgress()); + ASSERT_OK(dbfull()->error_handler_.GetBGError()); +} + +// FixFileIngestionCompactionDeadlock tests and verifies that compaction and +// file ingestion do not cause deadlock in the event of write stall triggered +// by number of L0 files reaching level0_stop_writes_trigger. +TEST_P(DBCompactionTestWithParam, FixFileIngestionCompactionDeadlock) { + const int kNumKeysPerFile = 100; + // Generate SST files. + Options options = CurrentOptions(); + + // Generate an external SST file containing a single key, i.e. 99 + std::string sst_files_dir = dbname_ + "/sst_files/"; + ASSERT_OK(DestroyDir(env_, sst_files_dir)); + ASSERT_OK(env_->CreateDir(sst_files_dir)); + SstFileWriter sst_writer(EnvOptions(), options); + const std::string sst_file_path = sst_files_dir + "test.sst"; + ASSERT_OK(sst_writer.Open(sst_file_path)); + ASSERT_OK(sst_writer.Put(Key(kNumKeysPerFile - 1), "value")); + ASSERT_OK(sst_writer.Finish()); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->LoadDependency({ + {"DBImpl::IngestExternalFile:AfterIncIngestFileCounter", + "BackgroundCallCompaction:0"}, + }); + SyncPoint::GetInstance()->EnableProcessing(); + + options.write_buffer_size = 110 << 10; // 110KB + options.level0_file_num_compaction_trigger = + options.level0_stop_writes_trigger; + options.max_subcompactions = max_subcompactions_; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + Random rnd(301); + + // Generate level0_stop_writes_trigger L0 files to trigger write stop + for (int i = 0; i != options.level0_file_num_compaction_trigger; ++i) { + for (int j = 0; j != kNumKeysPerFile; ++j) { + ASSERT_OK(Put(Key(j), rnd.RandomString(990))); + } + if (i > 0) { + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(NumTableFilesAtLevel(0 /*level*/, 0 /*cf*/), i); + } + } + // When we reach this point, there will be level0_stop_writes_trigger L0 + // files and one extra key (99) in memory, which overlaps with the external + // SST file. Write stall triggers, and can be cleared only after compaction + // reduces the number of L0 files. + + // Compaction will also be triggered since we have reached the threshold for + // auto compaction. Note that compaction may begin after the following file + // ingestion thread and waits for ingestion to finish. + + // Thread to ingest file with overlapping key range with the current + // memtable. Consequently ingestion will trigger a flush. The flush MUST + // proceed without waiting for the write stall condition to clear, otherwise + // deadlock can happen. + port::Thread ingestion_thr([&]() { + IngestExternalFileOptions ifo; + Status s = db_->IngestExternalFile({sst_file_path}, ifo); + ASSERT_OK(s); + }); + + // More write to trigger write stop + ingestion_thr.join(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + Close(); +} + +class DBCompactionTestWithOngoingFileIngestionParam + : public DBCompactionTest, + public testing::WithParamInterface { + public: + DBCompactionTestWithOngoingFileIngestionParam() : DBCompactionTest() { + compaction_path_to_test_ = GetParam(); + } + void SetupOptions() { + options_ = CurrentOptions(); + options_.create_if_missing = true; + + if (compaction_path_to_test_ == "RefitLevelCompactRange") { + options_.num_levels = 7; + } else { + options_.num_levels = 3; + } + options_.compaction_style = CompactionStyle::kCompactionStyleLevel; + if (compaction_path_to_test_ == "AutoCompaction") { + options_.disable_auto_compactions = false; + options_.level0_file_num_compaction_trigger = 1; + } else { + options_.disable_auto_compactions = true; + } + } + + void PauseCompactionThread() { + sleeping_task_.reset(new test::SleepingBackgroundTask()); + env_->SetBackgroundThreads(1, Env::LOW); + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, + sleeping_task_.get(), Env::Priority::LOW); + sleeping_task_->WaitUntilSleeping(); + } + + void ResumeCompactionThread() { + if (sleeping_task_) { + sleeping_task_->WakeUp(); + sleeping_task_->WaitUntilDone(); + } + } + + void SetupFilesToForceFutureFilesIngestedToCertainLevel() { + SstFileWriter sst_file_writer(EnvOptions(), options_); + std::string dummy = dbname_ + "/dummy.sst"; + ASSERT_OK(sst_file_writer.Open(dummy)); + ASSERT_OK(sst_file_writer.Put("k2", "dummy")); + ASSERT_OK(sst_file_writer.Finish()); + ASSERT_OK(db_->IngestExternalFile({dummy}, IngestExternalFileOptions())); + // L2 is made to contain a file overlapped with files to be ingested in + // later steps on key "k2". This will force future files ingested to L1 or + // above. + ASSERT_EQ("0,0,1", FilesPerLevel(0)); + } + + void SetupSyncPoints() { + if (compaction_path_to_test_ == "AutoCompaction") { + SyncPoint::GetInstance()->SetCallBack( + "ExternalSstFileIngestionJob::Run", [&](void*) { + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BackgroundCompaction():AfterPickCompaction", + "VersionSet::LogAndApply:WriteManifest"}}); + }); + } else if (compaction_path_to_test_ == "NonRefitLevelCompactRange") { + SyncPoint::GetInstance()->SetCallBack( + "ExternalSstFileIngestionJob::Run", [&](void*) { + SyncPoint::GetInstance()->LoadDependency( + {{"ColumnFamilyData::CompactRange:Return", + "VersionSet::LogAndApply:WriteManifest"}}); + }); + } else if (compaction_path_to_test_ == "RefitLevelCompactRange") { + SyncPoint::GetInstance()->SetCallBack( + "ExternalSstFileIngestionJob::Run", [&](void*) { + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::CompactRange:PostRefitLevel", + "VersionSet::LogAndApply:WriteManifest"}}); + }); + } else if (compaction_path_to_test_ == "CompactFiles") { + SyncPoint::GetInstance()->SetCallBack( + "ExternalSstFileIngestionJob::Run", [&](void*) { + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::CompactFilesImpl::PostSanitizeCompactionInputFiles", + "VersionSet::LogAndApply:WriteManifest"}}); + }); + } else { + assert(false); + } + SyncPoint::GetInstance()->LoadDependency( + {{"ExternalSstFileIngestionJob::Run", "PreCompaction"}}); + SyncPoint::GetInstance()->EnableProcessing(); + } + + void RunCompactionOverlappedWithFileIngestion() { + if (compaction_path_to_test_ == "AutoCompaction") { + TEST_SYNC_POINT("PreCompaction"); + ResumeCompactionThread(); + // Without proper range conflict check, + // this would have been `Status::Corruption` about overlapping ranges + Status s = dbfull()->TEST_WaitForCompact(); + EXPECT_OK(s); + } else if (compaction_path_to_test_ == "NonRefitLevelCompactRange") { + CompactRangeOptions cro; + cro.change_level = false; + std::string start_key = "k1"; + Slice start(start_key); + std::string end_key = "k4"; + Slice end(end_key); + TEST_SYNC_POINT("PreCompaction"); + // Without proper range conflict check, + // this would have been `Status::Corruption` about overlapping ranges + Status s = dbfull()->CompactRange(cro, &start, &end); + EXPECT_OK(s); + } else if (compaction_path_to_test_ == "RefitLevelCompactRange") { + CompactRangeOptions cro; + cro.change_level = true; + cro.target_level = 5; + std::string start_key = "k1"; + Slice start(start_key); + std::string end_key = "k4"; + Slice end(end_key); + TEST_SYNC_POINT("PreCompaction"); + Status s = dbfull()->CompactRange(cro, &start, &end); + // Without proper range conflict check, + // this would have been `Status::Corruption` about overlapping ranges + // To see this, remove the fix AND replace + // `DBImpl::CompactRange:PostRefitLevel` in sync point dependency with + // `DBImpl::ReFitLevel:PostRegisterCompaction` + EXPECT_TRUE(s.IsNotSupported()); + EXPECT_TRUE(s.ToString().find("some ongoing compaction's output") != + std::string::npos); + } else if (compaction_path_to_test_ == "CompactFiles") { + ColumnFamilyMetaData cf_meta_data; + db_->GetColumnFamilyMetaData(&cf_meta_data); + ASSERT_EQ(cf_meta_data.levels[0].files.size(), 1); + std::vector input_files; + for (const auto& file : cf_meta_data.levels[0].files) { + input_files.push_back(file.name); + } + TEST_SYNC_POINT("PreCompaction"); + Status s = db_->CompactFiles(CompactionOptions(), input_files, 1); + // Without proper range conflict check, + // this would have been `Status::Corruption` about overlapping ranges + EXPECT_TRUE(s.IsAborted()); + EXPECT_TRUE( + s.ToString().find( + "A running compaction is writing to the same output level") != + std::string::npos); + } else { + assert(false); + } + } + + void DisableSyncPoints() { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + } + + protected: + std::string compaction_path_to_test_; + Options options_; + std::shared_ptr sleeping_task_; +}; + +INSTANTIATE_TEST_CASE_P(DBCompactionTestWithOngoingFileIngestionParam, + DBCompactionTestWithOngoingFileIngestionParam, + ::testing::Values("AutoCompaction", + "NonRefitLevelCompactRange", + "RefitLevelCompactRange", + "CompactFiles")); + +TEST_P(DBCompactionTestWithOngoingFileIngestionParam, RangeConflictCheck) { + SetupOptions(); + DestroyAndReopen(options_); + + if (compaction_path_to_test_ == "AutoCompaction") { + PauseCompactionThread(); + } + + if (compaction_path_to_test_ != "RefitLevelCompactRange") { + SetupFilesToForceFutureFilesIngestedToCertainLevel(); + } + + // Create s1 + ASSERT_OK(Put("k1", "v")); + ASSERT_OK(Put("k4", "v")); + ASSERT_OK(Flush()); + if (compaction_path_to_test_ == "RefitLevelCompactRange") { + MoveFilesToLevel(6 /* level */); + ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel(0)); + } else { + ASSERT_EQ("1,0,1", FilesPerLevel(0)); + } + + // To coerce following sequence of events + // Timeline Thread 1 (Ingest s2) Thread 2 (Compact s1) + // t0 | Decide to output to Lk + // t1 | Release lock in LogAndApply() + // t2 | Acquire lock + // t3 | Decides to compact to Lk + // | Expected to fail due to range + // | conflict check with file + // | ingestion + // t4 | Release lock in LogAndApply() + // t5 | Acquire lock again and finish + // t6 | Acquire lock again and finish + SetupSyncPoints(); + + // Ingest s2 + port::Thread thread1([&] { + SstFileWriter sst_file_writer(EnvOptions(), options_); + std::string s2 = dbname_ + "/ingested_s2.sst"; + ASSERT_OK(sst_file_writer.Open(s2)); + ASSERT_OK(sst_file_writer.Put("k2", "v2")); + ASSERT_OK(sst_file_writer.Put("k3", "v2")); + ASSERT_OK(sst_file_writer.Finish()); + ASSERT_OK(db_->IngestExternalFile({s2}, IngestExternalFileOptions())); + }); + + // Compact s1. Without proper range conflict check, + // this will encounter overlapping file corruption. + port::Thread thread2([&] { RunCompactionOverlappedWithFileIngestion(); }); + + thread1.join(); + thread2.join(); + DisableSyncPoints(); +} + +TEST_F(DBCompactionTest, ConsistencyFailTest) { + Options options = CurrentOptions(); + options.force_consistency_checks = true; + DestroyAndReopen(options); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "VersionBuilder::CheckConsistency0", [&](void* arg) { + auto p = + reinterpret_cast*>(arg); + // just swap the two FileMetaData so that we hit error + // in CheckConsistency funcion + FileMetaData* temp = *(p->first); + *(p->first) = *(p->second); + *(p->second) = temp; + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + for (int k = 0; k < 2; ++k) { + ASSERT_OK(Put("foo", "bar")); + Status s = Flush(); + if (k < 1) { + ASSERT_OK(s); + } else { + ASSERT_TRUE(s.IsCorruption()); + } + } + + ASSERT_NOK(Put("foo", "bar")); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(DBCompactionTest, ConsistencyFailTest2) { + Options options = CurrentOptions(); + options.force_consistency_checks = true; + options.target_file_size_base = 1000; + options.level0_file_num_compaction_trigger = 2; + BlockBasedTableOptions bbto; + bbto.block_size = 400; // small block size + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + DestroyAndReopen(options); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "VersionBuilder::CheckConsistency1", [&](void* arg) { + auto p = + reinterpret_cast*>(arg); + // just swap the two FileMetaData so that we hit error + // in CheckConsistency funcion + FileMetaData* temp = *(p->first); + *(p->first) = *(p->second); + *(p->second) = temp; + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Random rnd(301); + std::string value = rnd.RandomString(1000); + + ASSERT_OK(Put("foo1", value)); + ASSERT_OK(Put("z", "")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("foo2", value)); + ASSERT_OK(Put("z", "")); + Status s = Flush(); + ASSERT_TRUE(s.ok() || s.IsCorruption()); + + // This probably returns non-OK, but we rely on the next Put() + // to determine the DB is frozen. + ASSERT_NOK(dbfull()->TEST_WaitForCompact()); + ASSERT_NOK(Put("foo", "bar")); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +void IngestOneKeyValue(DBImpl* db, const std::string& key, + const std::string& value, const Options& options) { + ExternalSstFileInfo info; + std::string f = test::PerThreadDBPath("sst_file" + key); + EnvOptions env; + ROCKSDB_NAMESPACE::SstFileWriter writer(env, options); + auto s = writer.Open(f); + ASSERT_OK(s); + // ASSERT_OK(writer.Put(Key(), "")); + ASSERT_OK(writer.Put(key, value)); + + ASSERT_OK(writer.Finish(&info)); + IngestExternalFileOptions ingest_opt; + + ASSERT_OK(db->IngestExternalFile({info.file_path}, ingest_opt)); +} + +class DBCompactionTestL0FilesMisorderCorruption : public DBCompactionTest { + public: + DBCompactionTestL0FilesMisorderCorruption() : DBCompactionTest() {} + void SetupOptions(const CompactionStyle compaciton_style, + const std::string& compaction_path_to_test = "") { + options_ = CurrentOptions(); + options_.create_if_missing = true; + options_.compression = kNoCompression; + + options_.force_consistency_checks = true; + options_.compaction_style = compaciton_style; + + if (compaciton_style == CompactionStyle::kCompactionStyleLevel) { + options_.num_levels = 7; + // Level compaction's PickIntraL0Compaction() impl detail requires + // `options.level0_file_num_compaction_trigger` to be + // at least 2 files less than the actual number of level 0 files + // (i.e, 7 by design in this test) + options_.level0_file_num_compaction_trigger = 5; + options_.max_background_compactions = 2; + options_.write_buffer_size = 2 << 20; + options_.max_write_buffer_number = 6; + } else if (compaciton_style == CompactionStyle::kCompactionStyleUniversal) { + // TODO: expand test coverage to num_lvels > 1 for universal compacion, + // which requires careful unit test design to compact to level 0 despite + // num_levels > 1 + options_.num_levels = 1; + options_.level0_file_num_compaction_trigger = 5; + + CompactionOptionsUniversal universal_options; + if (compaction_path_to_test == "PickCompactionToReduceSizeAmp") { + universal_options.max_size_amplification_percent = 50; + } else if (compaction_path_to_test == + "PickCompactionToReduceSortedRuns") { + universal_options.max_size_amplification_percent = 400; + } else if (compaction_path_to_test == "PickDeleteTriggeredCompaction") { + universal_options.max_size_amplification_percent = 400; + universal_options.min_merge_width = 6; + } + options_.compaction_options_universal = universal_options; + } else if (compaciton_style == CompactionStyle::kCompactionStyleFIFO) { + options_.max_open_files = -1; + options_.num_levels = 1; + options_.level0_file_num_compaction_trigger = 3; + + CompactionOptionsFIFO fifo_options; + if (compaction_path_to_test == "FindIntraL0Compaction" || + compaction_path_to_test == "CompactRange") { + fifo_options.allow_compaction = true; + } else if (compaction_path_to_test == "CompactFile") { + fifo_options.allow_compaction = false; + } + options_.compaction_options_fifo = fifo_options; + } + + if (compaction_path_to_test == "CompactFile" || + compaction_path_to_test == "CompactRange") { + options_.disable_auto_compactions = true; + } else { + options_.disable_auto_compactions = false; + } + } + + void Destroy(const Options& options) { + if (snapshot_) { + assert(db_); + db_->ReleaseSnapshot(snapshot_); + snapshot_ = nullptr; + } + DBTestBase::Destroy(options); + } + + void Reopen(const Options& options) { + DBTestBase::Reopen(options); + if (options.compaction_style != CompactionStyle::kCompactionStyleLevel) { + // To force assigning the global seqno to ingested file + // for our test purpose. + assert(snapshot_ == nullptr); + snapshot_ = db_->GetSnapshot(); + } + } + + void DestroyAndReopen(Options& options) { + Destroy(options); + Reopen(options); + } + + void PauseCompactionThread() { + sleeping_task_.reset(new test::SleepingBackgroundTask()); + env_->SetBackgroundThreads(1, Env::LOW); + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, + sleeping_task_.get(), Env::Priority::LOW); + sleeping_task_->WaitUntilSleeping(); + } + + void ResumeCompactionThread() { + if (sleeping_task_) { + sleeping_task_->WakeUp(); + sleeping_task_->WaitUntilDone(); + } + } + + void AddFilesMarkedForPeriodicCompaction(const size_t num_files) { + assert(options_.compaction_style == + CompactionStyle::kCompactionStyleUniversal); + VersionSet* const versions = dbfull()->GetVersionSet(); + assert(versions); + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + assert(cfd); + Version* const current = cfd->current(); + assert(current); + + VersionStorageInfo* const storage_info = current->storage_info(); + assert(storage_info); + + const std::vector level0_files = storage_info->LevelFiles(0); + assert(level0_files.size() == num_files); + + for (FileMetaData* f : level0_files) { + storage_info->TEST_AddFileMarkedForPeriodicCompaction(0, f); + } + } + + void AddFilesMarkedForCompaction(const size_t num_files) { + assert(options_.compaction_style == + CompactionStyle::kCompactionStyleUniversal); + VersionSet* const versions = dbfull()->GetVersionSet(); + assert(versions); + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + assert(cfd); + Version* const current = cfd->current(); + assert(current); + + VersionStorageInfo* const storage_info = current->storage_info(); + assert(storage_info); + + const std::vector level0_files = storage_info->LevelFiles(0); + assert(level0_files.size() == num_files); + + for (FileMetaData* f : level0_files) { + storage_info->TEST_AddFileMarkedForCompaction(0, f); + } + } + + void SetupSyncPoints(const std::string& compaction_path_to_test) { + compaction_path_sync_point_called_.store(false); + if (compaction_path_to_test == "FindIntraL0Compaction" && + options_.compaction_style == CompactionStyle::kCompactionStyleLevel) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "PostPickFileToCompact", [&](void* arg) { + bool* picked_file_to_compact = (bool*)arg; + // To trigger intra-L0 compaction specifically, + // we mock PickFileToCompact()'s result to be false + *picked_file_to_compact = false; + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "FindIntraL0Compaction", [&](void* /*arg*/) { + compaction_path_sync_point_called_.store(true); + }); + + } else if (compaction_path_to_test == "PickPeriodicCompaction") { + assert(options_.compaction_style == + CompactionStyle::kCompactionStyleUniversal); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "PostPickPeriodicCompaction", [&](void* compaction_arg) { + Compaction* compaction = (Compaction*)compaction_arg; + if (compaction != nullptr) { + compaction_path_sync_point_called_.store(true); + } + }); + } else if (compaction_path_to_test == "PickCompactionToReduceSizeAmp") { + assert(options_.compaction_style == + CompactionStyle::kCompactionStyleUniversal); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "PickCompactionToReduceSizeAmpReturnNonnullptr", [&](void* /*arg*/) { + compaction_path_sync_point_called_.store(true); + }); + } else if (compaction_path_to_test == "PickCompactionToReduceSortedRuns") { + assert(options_.compaction_style == + CompactionStyle::kCompactionStyleUniversal); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "PickCompactionToReduceSortedRunsReturnNonnullptr", + [&](void* /*arg*/) { + compaction_path_sync_point_called_.store(true); + }); + } else if (compaction_path_to_test == "PickDeleteTriggeredCompaction") { + assert(options_.compaction_style == + CompactionStyle::kCompactionStyleUniversal); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "PickDeleteTriggeredCompactionReturnNonnullptr", [&](void* /*arg*/) { + compaction_path_sync_point_called_.store(true); + }); + } else if ((compaction_path_to_test == "FindIntraL0Compaction" || + compaction_path_to_test == "CompactRange") && + options_.compaction_style == + CompactionStyle::kCompactionStyleFIFO) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "FindIntraL0Compaction", [&](void* /*arg*/) { + compaction_path_sync_point_called_.store(true); + }); + } + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + } + + bool SyncPointsCalled() { return compaction_path_sync_point_called_.load(); } + + void DisableSyncPoints() { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + } + + // Return the largest seqno of the latest L0 file based on file number + SequenceNumber GetLatestL0FileLargestSeqnoHelper() { + VersionSet* const versions = dbfull()->GetVersionSet(); + assert(versions); + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + assert(cfd); + Version* const current = cfd->current(); + assert(current); + VersionStorageInfo* const storage_info = current->storage_info(); + assert(storage_info); + const std::vector level0_files = storage_info->LevelFiles(0); + assert(level0_files.size() >= 1); + + uint64_t latest_file_num = 0; + uint64_t latest_file_largest_seqno = 0; + for (FileMetaData* f : level0_files) { + if (f->fd.GetNumber() > latest_file_num) { + latest_file_num = f->fd.GetNumber(); + latest_file_largest_seqno = f->fd.largest_seqno; + } + } + + return latest_file_largest_seqno; + } + + protected: + Options options_; + + private: + const Snapshot* snapshot_ = nullptr; + std::atomic compaction_path_sync_point_called_; + std::shared_ptr sleeping_task_; +}; + +TEST_F(DBCompactionTestL0FilesMisorderCorruption, + FlushAfterIntraL0LevelCompactionWithIngestedFile) { + SetupOptions(CompactionStyle::kCompactionStyleLevel, ""); + DestroyAndReopen(options_); + // Prevents trivial move + for (int i = 0; i < 10; ++i) { + ASSERT_OK(Put(Key(i), "")); // Prevents trivial move + } + ASSERT_OK(Flush()); + Compact("", Key(99)); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + + // To get accurate NumTableFilesAtLevel(0) when the number reaches + // options_.level0_file_num_compaction_trigger + PauseCompactionThread(); + + // To create below LSM tree + // (key:value@n indicates key-value pair has seqno "n", L0 is sorted): + // + // memtable: m1[ 5:new@12 .. 1:new@8, 0:new@7] + // L0: s6[6:new@13], s5[5:old@6] ... s1[1:old@2],s0[0:old@1] + // + // (1) Make 6 L0 sst (i.e, s0 - s5) + for (int i = 0; i < 6; ++i) { + if (i % 2 == 0) { + IngestOneKeyValue(dbfull(), Key(i), "old", options_); + } else { + ASSERT_OK(Put(Key(i), "old")); + ASSERT_OK(Flush()); + } + } + ASSERT_EQ(6, NumTableFilesAtLevel(0)); + + // (2) Create m1 + for (int i = 0; i < 6; ++i) { + ASSERT_OK(Put(Key(i), "new")); + } + ASSERT_EQ(6, NumTableFilesAtLevel(0)); + + // (3) Ingest file (i.e, s6) to trigger IntraL0Compaction() + for (int i = 6; i < 7; ++i) { + ASSERT_EQ(i, NumTableFilesAtLevel(0)); + IngestOneKeyValue(dbfull(), Key(i), "new", options_); + } + + SetupSyncPoints("FindIntraL0Compaction"); + ResumeCompactionThread(); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_TRUE(SyncPointsCalled()); + DisableSyncPoints(); + + // After compaction, we have LSM tree: + // + // memtable: m1[ 5:new@12 .. 1:new@8, 0:new@7] + // L0: s7[6:new@13, 5:old@6 .. 0:old@1] + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + SequenceNumber compact_output_file_largest_seqno = + GetLatestL0FileLargestSeqnoHelper(); + + ASSERT_OK(Flush()); + // After flush, we have LSM tree: + // + // L0: s8[5:new@12 .. 0:new@7],s7[6:new@13, 5:old@5 .. 0:old@1] + ASSERT_EQ(2, NumTableFilesAtLevel(0)); + SequenceNumber flushed_file_largest_seqno = + GetLatestL0FileLargestSeqnoHelper(); + + // To verify there isn't any file misorder leading to returning a old value + // of Key(0) - Key(5) , which is caused by flushed table s8 has a + // smaller largest seqno than the compaction output file s7's largest seqno + // while the flushed table has the newer version of the values than the + // compaction output file's. + ASSERT_TRUE(flushed_file_largest_seqno < compact_output_file_largest_seqno); + for (int i = 0; i < 6; ++i) { + ASSERT_EQ("new", Get(Key(i))); + } + for (int i = 6; i < 7; ++i) { + ASSERT_EQ("new", Get(Key(i))); + } +} + +TEST_F(DBCompactionTestL0FilesMisorderCorruption, + FlushAfterIntraL0UniversalCompactionWithIngestedFile) { + for (const std::string compaction_path_to_test : + {"PickPeriodicCompaction", "PickCompactionToReduceSizeAmp", + "PickCompactionToReduceSortedRuns", "PickDeleteTriggeredCompaction"}) { + SetupOptions(CompactionStyle::kCompactionStyleUniversal, + compaction_path_to_test); + DestroyAndReopen(options_); + + // To get accurate NumTableFilesAtLevel(0) when the number reaches + // options_.level0_file_num_compaction_trigger + PauseCompactionThread(); + + // To create below LSM tree + // (key:value@n indicates key-value pair has seqno "n", L0 is sorted): + // + // memtable: m1 [ k2:new@8, k1:new@7] + // L0: s4[k9:dummy@10], s3[k8:dummy@9], + // s2[k7:old@6, k6:old@5].. s0[k3:old@2, k1:old@1] + // + // (1) Create 3 existing SST file (i.e, s0 - s2) + ASSERT_OK(Put("k1", "old")); + ASSERT_OK(Put("k3", "old")); + ASSERT_OK(Flush()); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + ASSERT_OK(Put("k4", "old")); + ASSERT_OK(Put("k5", "old")); + ASSERT_OK(Flush()); + ASSERT_EQ(2, NumTableFilesAtLevel(0)); + ASSERT_OK(Put("k6", "old")); + ASSERT_OK(Put("k7", "old")); + ASSERT_OK(Flush()); + ASSERT_EQ(3, NumTableFilesAtLevel(0)); + + // (2) Create m1. Noted that it contains a overlaped key with s0 + ASSERT_OK(Put("k1", "new")); // overlapped key + ASSERT_OK(Put("k2", "new")); + + // (3) Ingest two SST files s3, s4 + IngestOneKeyValue(dbfull(), "k8", "dummy", options_); + IngestOneKeyValue(dbfull(), "k9", "dummy", options_); + // Up to now, L0 contains s0 - s4 + ASSERT_EQ(5, NumTableFilesAtLevel(0)); + + if (compaction_path_to_test == "PickPeriodicCompaction") { + AddFilesMarkedForPeriodicCompaction(5); + } else if (compaction_path_to_test == "PickDeleteTriggeredCompaction") { + AddFilesMarkedForCompaction(5); + } + + SetupSyncPoints(compaction_path_to_test); + ResumeCompactionThread(); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_TRUE(SyncPointsCalled()) + << "failed for compaction path to test: " << compaction_path_to_test; + DisableSyncPoints(); + + // After compaction, we have LSM tree: + // + // memtable: m1[ k2:new@8, k1:new@7] + // L0: s5[k9:dummy@10, k8@dummy@9, k7:old@6 .. k3:old@2, k1:old@1] + ASSERT_EQ(1, NumTableFilesAtLevel(0)) + << "failed for compaction path to test: " << compaction_path_to_test; + SequenceNumber compact_output_file_largest_seqno = + GetLatestL0FileLargestSeqnoHelper(); + + ASSERT_OK(Flush()) << "failed for compaction path to test: " + << compaction_path_to_test; + // After flush, we have LSM tree: + // + // L0: s6[k2:new@8, k1:new@7], + // s5[k9:dummy@10, k8@dummy@9, k7:old@6 .. k3:old@2, k1:old@1] + ASSERT_EQ(2, NumTableFilesAtLevel(0)) + << "failed for compaction path to test: " << compaction_path_to_test; + SequenceNumber flushed_file_largest_seqno = + GetLatestL0FileLargestSeqnoHelper(); + + // To verify there isn't any file misorder leading to returning a old + // value of "k1" , which is caused by flushed table s6 has a + // smaller largest seqno than the compaction output file s5's largest seqno + // while the flushed table has the newer version of the value + // than the compaction output file's. + ASSERT_TRUE(flushed_file_largest_seqno < compact_output_file_largest_seqno) + << "failed for compaction path to test: " << compaction_path_to_test; + EXPECT_EQ(Get("k1"), "new") + << "failed for compaction path to test: " << compaction_path_to_test; + } + + Destroy(options_); +} + +TEST_F(DBCompactionTestL0FilesMisorderCorruption, + FlushAfterIntraL0FIFOCompactionWithIngestedFile) { + for (const std::string compaction_path_to_test : {"FindIntraL0Compaction"}) { + SetupOptions(CompactionStyle::kCompactionStyleFIFO, + compaction_path_to_test); + DestroyAndReopen(options_); + + // To create below LSM tree + // (key:value@n indicates key-value pair has seqno "n", L0 is sorted): + // + // memtable: m1 [ k2:new@4, k1:new@3] + // L0: s2[k5:dummy@6], s1[k4:dummy@5], s0[k3:old@2, k1:old@1] + // + // (1) Create an existing SST file s0 + ASSERT_OK(Put("k1", "old")); + ASSERT_OK(Put("k3", "old")); + ASSERT_OK(Flush()); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + + // (2) Create memtable m1. Noted that it contains a overlaped key with s0 + ASSERT_OK(Put("k1", "new")); // overlapped key + ASSERT_OK(Put("k2", "new")); + + // To get accurate NumTableFilesAtLevel(0) when the number reaches + // options_.level0_file_num_compaction_trigger + PauseCompactionThread(); + + // (3) Ingest two SST files s1, s2 + IngestOneKeyValue(dbfull(), "k4", "dummy", options_); + IngestOneKeyValue(dbfull(), "k5", "dummy", options_); + // Up to now, L0 contains s0, s1, s2 + ASSERT_EQ(3, NumTableFilesAtLevel(0)); + + SetupSyncPoints(compaction_path_to_test); + ResumeCompactionThread(); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_TRUE(SyncPointsCalled()) + << "failed for compaction path to test: " << compaction_path_to_test; + DisableSyncPoints(); + // After compaction, we have LSM tree: + // + // memtable: m1 [ k2:new@4, k1:new@3] + // L0: s3[k5:dummy@6, k4:dummy@5, k3:old@2, k1:old@1] + ASSERT_EQ(1, NumTableFilesAtLevel(0)) + << "failed for compaction path to test: " << compaction_path_to_test; + SequenceNumber compact_output_file_largest_seqno = + GetLatestL0FileLargestSeqnoHelper(); + + ASSERT_OK(Flush()) << "failed for compaction path to test: " + << compaction_path_to_test; + // After flush, we have LSM tree: + // + // L0: s4[k2:new@4, k1:new@3], s3[k5:dummy@6, k4:dummy@5, k3:old@2, + // k1:old@1] + ASSERT_EQ(2, NumTableFilesAtLevel(0)) + << "failed for compaction path to test: " << compaction_path_to_test; + SequenceNumber flushed_file_largest_seqno = + GetLatestL0FileLargestSeqnoHelper(); + + // To verify there isn't any file misorder leading to returning a old + // value of "k1" , which is caused by flushed table s4 has a + // smaller largest seqno than the compaction output file s3's largest seqno + // while the flushed table has the newer version of the value + // than the compaction output file's. + ASSERT_TRUE(flushed_file_largest_seqno < compact_output_file_largest_seqno) + << "failed for compaction path to test: " << compaction_path_to_test; + EXPECT_EQ(Get("k1"), "new") + << "failed for compaction path to test: " << compaction_path_to_test; + } + + Destroy(options_); +} + +class DBCompactionTestL0FilesMisorderCorruptionWithParam + : public DBCompactionTestL0FilesMisorderCorruption, + public testing::WithParamInterface { + public: + DBCompactionTestL0FilesMisorderCorruptionWithParam() + : DBCompactionTestL0FilesMisorderCorruption() {} +}; + +// TODO: add `CompactionStyle::kCompactionStyleLevel` to testing parameter, +// which requires careful unit test +// design for ingesting file to L0 and CompactRange()/CompactFile() to L0 +INSTANTIATE_TEST_CASE_P( + DBCompactionTestL0FilesMisorderCorruptionWithParam, + DBCompactionTestL0FilesMisorderCorruptionWithParam, + ::testing::Values(CompactionStyle::kCompactionStyleUniversal, + CompactionStyle::kCompactionStyleFIFO)); + +TEST_P(DBCompactionTestL0FilesMisorderCorruptionWithParam, + FlushAfterIntraL0CompactFileWithIngestedFile) { + SetupOptions(GetParam(), "CompactFile"); + DestroyAndReopen(options_); + + // To create below LSM tree + // (key:value@n indicates key-value pair has seqno "n", L0 is sorted): + // + // memtable: m1 [ k2:new@4, k1:new@3] + // L0: s2[k5:dummy@6], s1[k4:dummy@5], s0[k3:old@2, k1:old@1] + // + // (1) Create an existing SST file s0 + ASSERT_OK(Put("k1", "old")); + ASSERT_OK(Put("k3", "old")); + ASSERT_OK(Flush()); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + + // (2) Create memtable m1. Noted that it contains a overlaped key with s0 + ASSERT_OK(Put("k1", "new")); // overlapped key + ASSERT_OK(Put("k2", "new")); + + // (3) Ingest two SST files s1, s2 + IngestOneKeyValue(dbfull(), "k4", "dummy", options_); + IngestOneKeyValue(dbfull(), "k5", "dummy", options_); + // Up to now, L0 contains s0, s1, s2 + ASSERT_EQ(3, NumTableFilesAtLevel(0)); + + ColumnFamilyMetaData cf_meta_data; + db_->GetColumnFamilyMetaData(&cf_meta_data); + ASSERT_EQ(cf_meta_data.levels[0].files.size(), 3); + std::vector input_files; + for (const auto& file : cf_meta_data.levels[0].files) { + input_files.push_back(file.name); + } + ASSERT_EQ(input_files.size(), 3); + + Status s = db_->CompactFiles(CompactionOptions(), input_files, 0); + // After compaction, we have LSM tree: + // + // memtable: m1 [ k2:new@4, k1:new@3] + // L0: s3[k5:dummy@6, k4:dummy@5, k3:old@2, k1:old@1] + ASSERT_OK(s); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + SequenceNumber compact_output_file_largest_seqno = + GetLatestL0FileLargestSeqnoHelper(); + + ASSERT_OK(Flush()); + // After flush, we have LSM tree: + // + // L0: s4[k2:new@4, k1:new@3], s3[k5:dummy@6, k4:dummy@5, k3:old@2, + // k1:old@1] + ASSERT_EQ(2, NumTableFilesAtLevel(0)); + SequenceNumber flushed_file_largest_seqno = + GetLatestL0FileLargestSeqnoHelper(); + + // To verify there isn't any file misorder leading to returning a old value + // of "1" , which is caused by flushed table s4 has a smaller + // largest seqno than the compaction output file s3's largest seqno while the + // flushed table has the newer version of the value than the + // compaction output file's. + ASSERT_TRUE(flushed_file_largest_seqno < compact_output_file_largest_seqno); + EXPECT_EQ(Get("k1"), "new"); + + Destroy(options_); +} + +TEST_P(DBCompactionTestL0FilesMisorderCorruptionWithParam, + FlushAfterIntraL0CompactRangeWithIngestedFile) { + SetupOptions(GetParam(), "CompactRange"); + DestroyAndReopen(options_); + + // To create below LSM tree + // (key:value@n indicates key-value pair has seqno "n", L0 is sorted): + // + // memtable: m1 [ k2:new@4, k1:new@3] + // L0: s2[k5:dummy@6], s1[k4:dummy@5], s0[k3:old@2, k1:old@1] + // + // (1) Create an existing SST file s0 + ASSERT_OK(Put("k1", "old")); + ASSERT_OK(Put("k3", "old")); + ASSERT_OK(Flush()); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + + // (2) Create memtable m1. Noted that it contains a overlaped key with s0 + ASSERT_OK(Put("k1", "new")); // overlapped key + ASSERT_OK(Put("k2", "new")); + + // (3) Ingest two SST files s1, s2 + IngestOneKeyValue(dbfull(), "k4", "dummy", options_); + IngestOneKeyValue(dbfull(), "k5", "dummy", options_); + // Up to now, L0 contains s0, s1, s2 + ASSERT_EQ(3, NumTableFilesAtLevel(0)); + + if (options_.compaction_style == CompactionStyle::kCompactionStyleFIFO) { + SetupSyncPoints("CompactRange"); + } + // `start` and `end` is carefully chosen so that compact range: + // (1) doesn't overlap with memtable therefore the memtable won't be flushed + // (2) should target at compacting s0 with s1 and s2 + Slice start("k3"), end("k5"); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &start, &end)); + // After compaction, we have LSM tree: + // + // memtable: m1 [ k2:new@4, k1:new@3] + // L0: s3[k5:dummy@6, k4:dummy@5, k3:old@2, k1:old@1] + if (options_.compaction_style == CompactionStyle::kCompactionStyleFIFO) { + ASSERT_TRUE(SyncPointsCalled()); + DisableSyncPoints(); + } + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + SequenceNumber compact_output_file_largest_seqno = + GetLatestL0FileLargestSeqnoHelper(); + + ASSERT_OK(Flush()); + // After flush, we have LSM tree: + // + // L0: s4[k2:new@4, k1:new@3], s3[k5:dummy@6, k4:dummy@5, k3:old@2, + // k1:old@1] + ASSERT_EQ(2, NumTableFilesAtLevel(0)); + SequenceNumber flushed_file_largest_seqno = + GetLatestL0FileLargestSeqnoHelper(); + + // To verify there isn't any file misorder leading to returning a old value + // of "k1" , which is caused by flushed table s4 has a smaller + // largest seqno than the compaction output file s3's largest seqno while the + // flushed table has the newer version of the value than the + // compaction output file's. + ASSERT_TRUE(flushed_file_largest_seqno < compact_output_file_largest_seqno); + EXPECT_EQ(Get("k1"), "new"); + + Destroy(options_); +} + +TEST_F(DBCompactionTest, SingleLevelUniveresal) { + // Tests that manual compaction works with single level universal compaction. + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.disable_auto_compactions = true; + options.num_levels = 1; + DestroyAndReopen(options); + + Random rnd(31); + for (int i = 0; i < 10; ++i) { + for (int j = 0; j < 50; ++j) { + ASSERT_OK(Put(Key(i * 100 + j), rnd.RandomString(50))); + } + ASSERT_OK(Flush()); + } + ASSERT_EQ(NumTableFilesAtLevel(0), 10); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(NumTableFilesAtLevel(0), 1); +} + +TEST_F(DBCompactionTest, SingleOverlappingNonL0BottommostManualCompaction) { + // Tests that manual compact will rewrite bottommost level + // when there is only a single non-L0 level that overlaps with + // manual compaction range. + constexpr int kSstNum = 10; + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.num_levels = 7; + for (auto b : {BottommostLevelCompaction::kForce, + BottommostLevelCompaction::kForceOptimized}) { + DestroyAndReopen(options); + + // Generate some sst files on level 0 with sequence keys (no overlap) + for (int i = 0; i < kSstNum; i++) { + for (int j = 1; j < UCHAR_MAX; j++) { + auto key = std::string(kSstNum, '\0'); + key[kSstNum - i] += static_cast(j); + ASSERT_OK(Put(key, std::string(i % 1000, 'A'))); + } + ASSERT_OK(Flush()); + } + MoveFilesToLevel(4); + ASSERT_EQ(NumTableFilesAtLevel(4), kSstNum); + CompactRangeOptions cro; + cro.bottommost_level_compaction = b; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ(NumTableFilesAtLevel(4), 1); + } +} + +TEST_P(DBCompactionTestWithBottommostParam, SequenceKeysManualCompaction) { + constexpr int kSstNum = 10; + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.num_levels = 7; + const bool dynamic_level = std::get<1>(GetParam()); + options.level_compaction_dynamic_level_bytes = dynamic_level; + DestroyAndReopen(options); + + // Generate some sst files on level 0 with sequence keys (no overlap) + for (int i = 0; i < kSstNum; i++) { + for (int j = 1; j < UCHAR_MAX; j++) { + auto key = std::string(kSstNum, '\0'); + key[kSstNum - i] += static_cast(j); + ASSERT_OK(Put(key, std::string(i % 1000, 'A'))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + + ASSERT_EQ(std::to_string(kSstNum), FilesPerLevel(0)); + + auto cro = CompactRangeOptions(); + cro.bottommost_level_compaction = bottommost_level_compaction_; + bool trivial_moved = false; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:TrivialMove", + [&](void* /*arg*/) { trivial_moved = true; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + // All bottommost_level_compaction options should allow l0 -> l1 trivial move. + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_TRUE(trivial_moved); + if (bottommost_level_compaction_ == BottommostLevelCompaction::kForce || + bottommost_level_compaction_ == + BottommostLevelCompaction::kForceOptimized) { + // bottommost level should go through intra-level compaction + // and has only 1 file + if (dynamic_level) { + ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel(0)); + } else { + ASSERT_EQ("0,1", FilesPerLevel(0)); + } + } else { + // Just trivial move from level 0 -> 1/base + if (dynamic_level) { + ASSERT_EQ("0,0,0,0,0,0," + std::to_string(kSstNum), FilesPerLevel(0)); + } else { + ASSERT_EQ("0," + std::to_string(kSstNum), FilesPerLevel(0)); + } + } +} + +INSTANTIATE_TEST_CASE_P( + DBCompactionTestWithBottommostParam, DBCompactionTestWithBottommostParam, + ::testing::Combine( + ::testing::Values(BottommostLevelCompaction::kSkip, + BottommostLevelCompaction::kIfHaveCompactionFilter, + BottommostLevelCompaction::kForce, + BottommostLevelCompaction::kForceOptimized), + ::testing::Bool())); + +TEST_F(DBCompactionTest, UpdateLevelSubCompactionTest) { + Options options = CurrentOptions(); + options.max_subcompactions = 10; + options.target_file_size_base = 1 << 10; // 1KB + DestroyAndReopen(options); + + bool has_compaction = false; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) { + Compaction* compaction = reinterpret_cast(arg); + ASSERT_TRUE(compaction->max_subcompactions() == 10); + has_compaction = true; + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_TRUE(dbfull()->GetDBOptions().max_subcompactions == 10); + // Trigger compaction + for (int i = 0; i < 32; i++) { + for (int j = 0; j < 5000; j++) { + ASSERT_OK(Put(std::to_string(j), std::string(1, 'A'))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_TRUE(has_compaction); + + has_compaction = false; + ASSERT_OK(dbfull()->SetDBOptions({{"max_subcompactions", "2"}})); + ASSERT_TRUE(dbfull()->GetDBOptions().max_subcompactions == 2); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) { + Compaction* compaction = reinterpret_cast(arg); + ASSERT_TRUE(compaction->max_subcompactions() == 2); + has_compaction = true; + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Trigger compaction + for (int i = 0; i < 32; i++) { + for (int j = 0; j < 5000; j++) { + ASSERT_OK(Put(std::to_string(j), std::string(1, 'A'))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_TRUE(has_compaction); +} + +TEST_F(DBCompactionTest, UpdateUniversalSubCompactionTest) { + Options options = CurrentOptions(); + options.max_subcompactions = 10; + options.compaction_style = kCompactionStyleUniversal; + options.target_file_size_base = 1 << 10; // 1KB + DestroyAndReopen(options); + + bool has_compaction = false; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "UniversalCompactionBuilder::PickCompaction:Return", [&](void* arg) { + Compaction* compaction = reinterpret_cast(arg); + ASSERT_TRUE(compaction->max_subcompactions() == 10); + has_compaction = true; + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Trigger compaction + for (int i = 0; i < 32; i++) { + for (int j = 0; j < 5000; j++) { + ASSERT_OK(Put(std::to_string(j), std::string(1, 'A'))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_TRUE(has_compaction); + has_compaction = false; + + ASSERT_OK(dbfull()->SetDBOptions({{"max_subcompactions", "2"}})); + ASSERT_TRUE(dbfull()->GetDBOptions().max_subcompactions == 2); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "UniversalCompactionBuilder::PickCompaction:Return", [&](void* arg) { + Compaction* compaction = reinterpret_cast(arg); + ASSERT_TRUE(compaction->max_subcompactions() == 2); + has_compaction = true; + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Trigger compaction + for (int i = 0; i < 32; i++) { + for (int j = 0; j < 5000; j++) { + ASSERT_OK(Put(std::to_string(j), std::string(1, 'A'))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_TRUE(has_compaction); +} + +TEST_P(ChangeLevelConflictsWithAuto, TestConflict) { + // A `CompactRange()` may race with an automatic compaction, we'll need + // to make sure it doesn't corrupte the data. + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = 2; + Reopen(options); + + ASSERT_OK(Put("foo", "v1")); + ASSERT_OK(Put("bar", "v1")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + + { + CompactRangeOptions cro; + cro.change_level = true; + cro.target_level = 2; + ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr)); + } + ASSERT_EQ("0,0,1", FilesPerLevel(0)); + + // Run a qury to refitting to level 1 while another thread writing to + // the same level. + SyncPoint::GetInstance()->LoadDependency({ + // The first two dependencies ensure the foreground creates an L0 file + // between the background compaction's L0->L1 and its L1->L2. + { + "DBImpl::CompactRange:BeforeRefit:1", + "AutoCompactionFinished1", + }, + { + "AutoCompactionFinished2", + "DBImpl::CompactRange:BeforeRefit:2", + }, + }); + SyncPoint::GetInstance()->EnableProcessing(); + + std::thread auto_comp([&] { + TEST_SYNC_POINT("AutoCompactionFinished1"); + ASSERT_OK(Put("bar", "v2")); + ASSERT_OK(Put("foo", "v2")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("bar", "v3")); + ASSERT_OK(Put("foo", "v3")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + TEST_SYNC_POINT("AutoCompactionFinished2"); + }); + + { + CompactRangeOptions cro; + cro.change_level = true; + cro.target_level = GetParam() ? 1 : 0; + // This should return non-OK, but it's more important for the test to + // make sure that the DB is not corrupted. + ASSERT_NOK(dbfull()->CompactRange(cro, nullptr, nullptr)); + } + auto_comp.join(); + // Refitting didn't happen. + SyncPoint::GetInstance()->DisableProcessing(); + + // Write something to DB just make sure that consistency check didn't + // fail and make the DB readable. +} + +INSTANTIATE_TEST_CASE_P(ChangeLevelConflictsWithAuto, + ChangeLevelConflictsWithAuto, testing::Bool()); + +TEST_F(DBCompactionTest, ChangeLevelCompactRangeConflictsWithManual) { + // A `CompactRange()` with `change_level == true` needs to execute its final + // step, `ReFitLevel()`, in isolation. Previously there was a bug where + // refitting could target the same level as an ongoing manual compaction, + // leading to overlapping files in that level. + // + // This test ensures that case is not possible by verifying any manual + // compaction issued during the `ReFitLevel()` phase fails with + // `Status::Incomplete`. + Options options = CurrentOptions(); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + options.level0_file_num_compaction_trigger = 2; + options.num_levels = 3; + Reopen(options); + + // Setup an LSM with three levels populated. + Random rnd(301); + int key_idx = 0; + GenerateNewFile(&rnd, &key_idx); + { + CompactRangeOptions cro; + cro.change_level = true; + cro.target_level = 2; + ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr)); + } + ASSERT_EQ("0,0,2", FilesPerLevel(0)); + + GenerateNewFile(&rnd, &key_idx); + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,1,2", FilesPerLevel(0)); + + // The background thread will refit L2->L1 while the + // foreground thread will try to simultaneously compact L0->L1. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + // The first two dependencies ensure the foreground creates an L0 file + // between the background compaction's L0->L1 and its L1->L2. + { + "DBImpl::RunManualCompaction()::1", + "DBCompactionTest::ChangeLevelCompactRangeConflictsWithManual:" + "PutFG", + }, + { + "DBCompactionTest::ChangeLevelCompactRangeConflictsWithManual:" + "FlushedFG", + "DBImpl::RunManualCompaction()::2", + }, + // The next two dependencies ensure the foreground invokes + // `CompactRange()` while the background is refitting. The + // foreground's `CompactRange()` is guaranteed to attempt an L0->L1 + // as we set it up with an empty memtable and a new L0 file. + { + "DBImpl::CompactRange:PreRefitLevel", + "DBCompactionTest::ChangeLevelCompactRangeConflictsWithManual:" + "CompactFG", + }, + { + "DBCompactionTest::ChangeLevelCompactRangeConflictsWithManual:" + "CompactedFG", + "DBImpl::CompactRange:PostRefitLevel", + }, + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ROCKSDB_NAMESPACE::port::Thread refit_level_thread([&] { + CompactRangeOptions cro; + cro.change_level = true; + cro.target_level = 1; + ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr)); + }); + + TEST_SYNC_POINT( + "DBCompactionTest::ChangeLevelCompactRangeConflictsWithManual:PutFG"); + // Make sure we have something new to compact in the foreground. + // Note key 1 is carefully chosen as it ensures the file we create here + // overlaps with one of the files being refitted L2->L1 in the background. + // If we chose key 0, the file created here would not overlap. + ASSERT_OK(Put(Key(1), "val")); + ASSERT_OK(Flush()); + TEST_SYNC_POINT( + "DBCompactionTest::ChangeLevelCompactRangeConflictsWithManual:FlushedFG"); + + TEST_SYNC_POINT( + "DBCompactionTest::ChangeLevelCompactRangeConflictsWithManual:CompactFG"); + ASSERT_TRUE(dbfull() + ->CompactRange(CompactRangeOptions(), nullptr, nullptr) + .IsIncomplete()); + TEST_SYNC_POINT( + "DBCompactionTest::ChangeLevelCompactRangeConflictsWithManual:" + "CompactedFG"); + refit_level_thread.join(); +} + +TEST_F(DBCompactionTest, ChangeLevelErrorPathTest) { + // This test is added to ensure that RefitLevel() error paths are clearing + // internal flags and to test that subsequent valid RefitLevel() calls + // succeeds + Options options = CurrentOptions(); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + options.level0_file_num_compaction_trigger = 2; + options.num_levels = 3; + Reopen(options); + + ASSERT_EQ("", FilesPerLevel(0)); + + // Setup an LSM with three levels populated. + Random rnd(301); + int key_idx = 0; + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1", FilesPerLevel(0)); + { + CompactRangeOptions cro; + cro.change_level = true; + cro.target_level = 2; + ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr)); + } + ASSERT_EQ("0,0,2", FilesPerLevel(0)); + + auto start_idx = key_idx; + GenerateNewFile(&rnd, &key_idx); + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ("1,1,2", FilesPerLevel(0)); + + MoveFilesToLevel(1); + ASSERT_EQ("0,2,2", FilesPerLevel(0)); + + // The next CompactRange() call is used to test exercise error paths within + // RefitLevel() before triggering a valid RefitLevel() call + // + // Try a refit from L2->L1 - this should fail and exercise error paths in + // RefitLevel() + { + // Select key range that matches the bottom most level (L2) + std::string begin_string = Key(0); + std::string end_string = Key(start_idx - 1); + Slice begin(begin_string); + Slice end(end_string); + + CompactRangeOptions cro; + cro.change_level = true; + cro.target_level = 1; + ASSERT_NOK(dbfull()->CompactRange(cro, &begin, &end)); + } + ASSERT_EQ("0,2,2", FilesPerLevel(0)); + + // Try a valid Refit request to ensure, the path is still working + { + CompactRangeOptions cro; + cro.change_level = true; + cro.target_level = 1; + ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr)); + } + ASSERT_EQ("0,5", FilesPerLevel(0)); +} + +TEST_F(DBCompactionTest, CompactionWithBlob) { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + Reopen(options); + + constexpr char first_key[] = "first_key"; + constexpr char second_key[] = "second_key"; + constexpr char first_value[] = "first_value"; + constexpr char second_value[] = "second_value"; + constexpr char third_value[] = "third_value"; + + ASSERT_OK(Put(first_key, first_value)); + ASSERT_OK(Put(second_key, first_value)); + ASSERT_OK(Flush()); + + ASSERT_OK(Put(first_key, second_value)); + ASSERT_OK(Put(second_key, second_value)); + ASSERT_OK(Flush()); + + ASSERT_OK(Put(first_key, third_value)); + ASSERT_OK(Put(second_key, third_value)); + ASSERT_OK(Flush()); + + options.enable_blob_files = true; + + Reopen(options); + + constexpr Slice* begin = nullptr; + constexpr Slice* end = nullptr; + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), begin, end)); + + ASSERT_EQ(Get(first_key), third_value); + ASSERT_EQ(Get(second_key), third_value); + + VersionSet* const versions = dbfull()->GetVersionSet(); + assert(versions); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(cfd, nullptr); + + Version* const current = cfd->current(); + ASSERT_NE(current, nullptr); + + const VersionStorageInfo* const storage_info = current->storage_info(); + ASSERT_NE(storage_info, nullptr); + + const auto& l1_files = storage_info->LevelFiles(1); + ASSERT_EQ(l1_files.size(), 1); + + const FileMetaData* const table_file = l1_files[0]; + ASSERT_NE(table_file, nullptr); + + const auto& blob_files = storage_info->GetBlobFiles(); + ASSERT_EQ(blob_files.size(), 1); + + const auto& blob_file = blob_files.front(); + ASSERT_NE(blob_file, nullptr); + + ASSERT_EQ(table_file->smallest.user_key(), first_key); + ASSERT_EQ(table_file->largest.user_key(), second_key); + ASSERT_EQ(table_file->oldest_blob_file_number, + blob_file->GetBlobFileNumber()); + + ASSERT_EQ(blob_file->GetTotalBlobCount(), 2); + + const InternalStats* const internal_stats = cfd->internal_stats(); + ASSERT_NE(internal_stats, nullptr); + + const auto& compaction_stats = internal_stats->TEST_GetCompactionStats(); + ASSERT_GE(compaction_stats.size(), 2); + ASSERT_EQ(compaction_stats[1].bytes_read_blob, 0); + ASSERT_EQ(compaction_stats[1].bytes_written, table_file->fd.GetFileSize()); + ASSERT_EQ(compaction_stats[1].bytes_written_blob, + blob_file->GetTotalBlobBytes()); + ASSERT_EQ(compaction_stats[1].num_output_files, 1); + ASSERT_EQ(compaction_stats[1].num_output_files_blob, 1); +} + +class DBCompactionTestBlobError + : public DBCompactionTest, + public testing::WithParamInterface { + public: + DBCompactionTestBlobError() : sync_point_(GetParam()) {} + + std::string sync_point_; +}; + +INSTANTIATE_TEST_CASE_P(DBCompactionTestBlobError, DBCompactionTestBlobError, + ::testing::ValuesIn(std::vector{ + "BlobFileBuilder::WriteBlobToFile:AddRecord", + "BlobFileBuilder::WriteBlobToFile:AppendFooter"})); + +TEST_P(DBCompactionTestBlobError, CompactionError) { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + Reopen(options); + + constexpr char first_key[] = "first_key"; + constexpr char second_key[] = "second_key"; + constexpr char first_value[] = "first_value"; + constexpr char second_value[] = "second_value"; + constexpr char third_value[] = "third_value"; + + ASSERT_OK(Put(first_key, first_value)); + ASSERT_OK(Put(second_key, first_value)); + ASSERT_OK(Flush()); + + ASSERT_OK(Put(first_key, second_value)); + ASSERT_OK(Put(second_key, second_value)); + ASSERT_OK(Flush()); + + ASSERT_OK(Put(first_key, third_value)); + ASSERT_OK(Put(second_key, third_value)); + ASSERT_OK(Flush()); + + options.enable_blob_files = true; + + Reopen(options); + + SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* arg) { + Status* const s = static_cast(arg); + assert(s); + + (*s) = Status::IOError(sync_point_); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + constexpr Slice* begin = nullptr; + constexpr Slice* end = nullptr; + + ASSERT_TRUE(db_->CompactRange(CompactRangeOptions(), begin, end).IsIOError()); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + VersionSet* const versions = dbfull()->GetVersionSet(); + assert(versions); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(cfd, nullptr); + + Version* const current = cfd->current(); + ASSERT_NE(current, nullptr); + + const VersionStorageInfo* const storage_info = current->storage_info(); + ASSERT_NE(storage_info, nullptr); + + const auto& l1_files = storage_info->LevelFiles(1); + ASSERT_TRUE(l1_files.empty()); + + const auto& blob_files = storage_info->GetBlobFiles(); + ASSERT_TRUE(blob_files.empty()); + + const InternalStats* const internal_stats = cfd->internal_stats(); + ASSERT_NE(internal_stats, nullptr); + + const auto& compaction_stats = internal_stats->TEST_GetCompactionStats(); + ASSERT_GE(compaction_stats.size(), 2); + + if (sync_point_ == "BlobFileBuilder::WriteBlobToFile:AddRecord") { + ASSERT_EQ(compaction_stats[1].bytes_read_blob, 0); + ASSERT_EQ(compaction_stats[1].bytes_written, 0); + ASSERT_EQ(compaction_stats[1].bytes_written_blob, 0); + ASSERT_EQ(compaction_stats[1].num_output_files, 0); + ASSERT_EQ(compaction_stats[1].num_output_files_blob, 0); + } else { + // SST file writing succeeded; blob file writing failed (during Finish) + ASSERT_EQ(compaction_stats[1].bytes_read_blob, 0); + ASSERT_GT(compaction_stats[1].bytes_written, 0); + ASSERT_EQ(compaction_stats[1].bytes_written_blob, 0); + ASSERT_EQ(compaction_stats[1].num_output_files, 1); + ASSERT_EQ(compaction_stats[1].num_output_files_blob, 0); + } +} + +class DBCompactionTestBlobGC + : public DBCompactionTest, + public testing::WithParamInterface> { + public: + DBCompactionTestBlobGC() + : blob_gc_age_cutoff_(std::get<0>(GetParam())), + updated_enable_blob_files_(std::get<1>(GetParam())) {} + + double blob_gc_age_cutoff_; + bool updated_enable_blob_files_; +}; + +INSTANTIATE_TEST_CASE_P(DBCompactionTestBlobGC, DBCompactionTestBlobGC, + ::testing::Combine(::testing::Values(0.0, 0.5, 1.0), + ::testing::Bool())); + +TEST_P(DBCompactionTestBlobGC, CompactionWithBlobGCOverrides) { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.enable_blob_files = true; + options.blob_file_size = 32; // one blob per file + options.enable_blob_garbage_collection = true; + options.blob_garbage_collection_age_cutoff = 0; + + DestroyAndReopen(options); + + for (int i = 0; i < 128; i += 2) { + ASSERT_OK(Put("key" + std::to_string(i), "value" + std::to_string(i))); + ASSERT_OK( + Put("key" + std::to_string(i + 1), "value" + std::to_string(i + 1))); + ASSERT_OK(Flush()); + } + + std::vector original_blob_files = GetBlobFileNumbers(); + ASSERT_EQ(original_blob_files.size(), 128); + + // Note: turning off enable_blob_files before the compaction results in + // garbage collected values getting inlined. + ASSERT_OK(db_->SetOptions({{"enable_blob_files", "false"}})); + + CompactRangeOptions cro; + cro.blob_garbage_collection_policy = BlobGarbageCollectionPolicy::kForce; + cro.blob_garbage_collection_age_cutoff = blob_gc_age_cutoff_; + + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + // Check that the GC stats are correct + { + VersionSet* const versions = dbfull()->GetVersionSet(); + assert(versions); + assert(versions->GetColumnFamilySet()); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + assert(cfd); + + const InternalStats* const internal_stats = cfd->internal_stats(); + assert(internal_stats); + + const auto& compaction_stats = internal_stats->TEST_GetCompactionStats(); + ASSERT_GE(compaction_stats.size(), 2); + + ASSERT_GE(compaction_stats[1].bytes_read_blob, 0); + ASSERT_EQ(compaction_stats[1].bytes_written_blob, 0); + } + + const size_t cutoff_index = static_cast( + cro.blob_garbage_collection_age_cutoff * original_blob_files.size()); + const size_t expected_num_files = original_blob_files.size() - cutoff_index; + + const std::vector new_blob_files = GetBlobFileNumbers(); + + ASSERT_EQ(new_blob_files.size(), expected_num_files); + + // Original blob files below the cutoff should be gone, original blob files + // at or above the cutoff should be still there + for (size_t i = cutoff_index; i < original_blob_files.size(); ++i) { + ASSERT_EQ(new_blob_files[i - cutoff_index], original_blob_files[i]); + } + + for (size_t i = 0; i < 128; ++i) { + ASSERT_EQ(Get("key" + std::to_string(i)), "value" + std::to_string(i)); + } +} + +TEST_P(DBCompactionTestBlobGC, CompactionWithBlobGC) { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.enable_blob_files = true; + options.blob_file_size = 32; // one blob per file + options.enable_blob_garbage_collection = true; + options.blob_garbage_collection_age_cutoff = blob_gc_age_cutoff_; + + Reopen(options); + + constexpr char first_key[] = "first_key"; + constexpr char first_value[] = "first_value"; + constexpr char second_key[] = "second_key"; + constexpr char second_value[] = "second_value"; + + ASSERT_OK(Put(first_key, first_value)); + ASSERT_OK(Put(second_key, second_value)); + ASSERT_OK(Flush()); + + constexpr char third_key[] = "third_key"; + constexpr char third_value[] = "third_value"; + constexpr char fourth_key[] = "fourth_key"; + constexpr char fourth_value[] = "fourth_value"; + + ASSERT_OK(Put(third_key, third_value)); + ASSERT_OK(Put(fourth_key, fourth_value)); + ASSERT_OK(Flush()); + + const std::vector original_blob_files = GetBlobFileNumbers(); + + ASSERT_EQ(original_blob_files.size(), 4); + + const size_t cutoff_index = static_cast( + options.blob_garbage_collection_age_cutoff * original_blob_files.size()); + + // Note: turning off enable_blob_files before the compaction results in + // garbage collected values getting inlined. + size_t expected_number_of_files = original_blob_files.size(); + + if (!updated_enable_blob_files_) { + ASSERT_OK(db_->SetOptions({{"enable_blob_files", "false"}})); + + expected_number_of_files -= cutoff_index; + } + + constexpr Slice* begin = nullptr; + constexpr Slice* end = nullptr; + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), begin, end)); + + ASSERT_EQ(Get(first_key), first_value); + ASSERT_EQ(Get(second_key), second_value); + ASSERT_EQ(Get(third_key), third_value); + ASSERT_EQ(Get(fourth_key), fourth_value); + + const std::vector new_blob_files = GetBlobFileNumbers(); + + ASSERT_EQ(new_blob_files.size(), expected_number_of_files); + + // Original blob files below the cutoff should be gone, original blob files at + // or above the cutoff should be still there + for (size_t i = cutoff_index; i < original_blob_files.size(); ++i) { + ASSERT_EQ(new_blob_files[i - cutoff_index], original_blob_files[i]); + } + + VersionSet* const versions = dbfull()->GetVersionSet(); + assert(versions); + assert(versions->GetColumnFamilySet()); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + assert(cfd); + + const InternalStats* const internal_stats = cfd->internal_stats(); + assert(internal_stats); + + const auto& compaction_stats = internal_stats->TEST_GetCompactionStats(); + ASSERT_GE(compaction_stats.size(), 2); + + if (blob_gc_age_cutoff_ > 0.0) { + ASSERT_GT(compaction_stats[1].bytes_read_blob, 0); + + if (updated_enable_blob_files_) { + // GC relocated some blobs to new blob files + ASSERT_GT(compaction_stats[1].bytes_written_blob, 0); + ASSERT_EQ(compaction_stats[1].bytes_read_blob, + compaction_stats[1].bytes_written_blob); + } else { + // GC moved some blobs back to the LSM, no new blob files + ASSERT_EQ(compaction_stats[1].bytes_written_blob, 0); + } + } else { + ASSERT_EQ(compaction_stats[1].bytes_read_blob, 0); + ASSERT_EQ(compaction_stats[1].bytes_written_blob, 0); + } +} + +TEST_F(DBCompactionTest, CompactionWithBlobGCError_CorruptIndex) { + Options options; + options.env = env_; + options.disable_auto_compactions = true; + options.enable_blob_files = true; + options.enable_blob_garbage_collection = true; + options.blob_garbage_collection_age_cutoff = 1.0; + + Reopen(options); + + constexpr char first_key[] = "first_key"; + constexpr char first_value[] = "first_value"; + ASSERT_OK(Put(first_key, first_value)); + + constexpr char second_key[] = "second_key"; + constexpr char second_value[] = "second_value"; + ASSERT_OK(Put(second_key, second_value)); + + ASSERT_OK(Flush()); + + constexpr char third_key[] = "third_key"; + constexpr char third_value[] = "third_value"; + ASSERT_OK(Put(third_key, third_value)); + + constexpr char fourth_key[] = "fourth_key"; + constexpr char fourth_value[] = "fourth_value"; + ASSERT_OK(Put(fourth_key, fourth_value)); + + ASSERT_OK(Flush()); + + SyncPoint::GetInstance()->SetCallBack( + "CompactionIterator::GarbageCollectBlobIfNeeded::TamperWithBlobIndex", + [](void* arg) { + Slice* const blob_index = static_cast(arg); + assert(blob_index); + assert(!blob_index->empty()); + blob_index->remove_prefix(1); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + constexpr Slice* begin = nullptr; + constexpr Slice* end = nullptr; + + ASSERT_TRUE( + db_->CompactRange(CompactRangeOptions(), begin, end).IsCorruption()); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(DBCompactionTest, CompactionWithBlobGCError_InlinedTTLIndex) { + constexpr uint64_t min_blob_size = 10; + + Options options; + options.env = env_; + options.disable_auto_compactions = true; + options.enable_blob_files = true; + options.min_blob_size = min_blob_size; + options.enable_blob_garbage_collection = true; + options.blob_garbage_collection_age_cutoff = 1.0; + + Reopen(options); + + constexpr char first_key[] = "first_key"; + constexpr char first_value[] = "first_value"; + ASSERT_OK(Put(first_key, first_value)); + + constexpr char second_key[] = "second_key"; + constexpr char second_value[] = "second_value"; + ASSERT_OK(Put(second_key, second_value)); + + ASSERT_OK(Flush()); + + constexpr char third_key[] = "third_key"; + constexpr char third_value[] = "third_value"; + ASSERT_OK(Put(third_key, third_value)); + + constexpr char fourth_key[] = "fourth_key"; + constexpr char blob[] = "short"; + static_assert(sizeof(short) - 1 < min_blob_size, + "Blob too long to be inlined"); + + // Fake an inlined TTL blob index. + std::string blob_index; + + constexpr uint64_t expiration = 1234567890; + + BlobIndex::EncodeInlinedTTL(&blob_index, expiration, blob); + + WriteBatch batch; + ASSERT_OK( + WriteBatchInternal::PutBlobIndex(&batch, 0, fourth_key, blob_index)); + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + + ASSERT_OK(Flush()); + + constexpr Slice* begin = nullptr; + constexpr Slice* end = nullptr; + + ASSERT_TRUE( + db_->CompactRange(CompactRangeOptions(), begin, end).IsCorruption()); +} + +TEST_F(DBCompactionTest, CompactionWithBlobGCError_IndexWithInvalidFileNumber) { + Options options; + options.env = env_; + options.disable_auto_compactions = true; + options.enable_blob_files = true; + options.enable_blob_garbage_collection = true; + options.blob_garbage_collection_age_cutoff = 1.0; + + Reopen(options); + + constexpr char first_key[] = "first_key"; + constexpr char first_value[] = "first_value"; + ASSERT_OK(Put(first_key, first_value)); + + constexpr char second_key[] = "second_key"; + constexpr char second_value[] = "second_value"; + ASSERT_OK(Put(second_key, second_value)); + + ASSERT_OK(Flush()); + + constexpr char third_key[] = "third_key"; + constexpr char third_value[] = "third_value"; + ASSERT_OK(Put(third_key, third_value)); + + constexpr char fourth_key[] = "fourth_key"; + + // Fake a blob index referencing a non-existent blob file. + std::string blob_index; + + constexpr uint64_t blob_file_number = 1000; + constexpr uint64_t offset = 1234; + constexpr uint64_t size = 5678; + + BlobIndex::EncodeBlob(&blob_index, blob_file_number, offset, size, + kNoCompression); + + WriteBatch batch; + ASSERT_OK( + WriteBatchInternal::PutBlobIndex(&batch, 0, fourth_key, blob_index)); + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + + ASSERT_OK(Flush()); + + constexpr Slice* begin = nullptr; + constexpr Slice* end = nullptr; + + ASSERT_TRUE( + db_->CompactRange(CompactRangeOptions(), begin, end).IsCorruption()); +} + +TEST_F(DBCompactionTest, CompactionWithChecksumHandoff1) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_SKIP("Test requires non-mem or non-encrypted environment"); + return; + } + std::shared_ptr fault_fs( + new FaultInjectionTestFS(FileSystem::Default())); + std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = 2; + options.num_levels = 3; + options.env = fault_fs_env.get(); + options.create_if_missing = true; + options.checksum_handoff_file_types.Add(FileType::kTableFile); + Status s; + Reopen(options); + + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kCRC32c); + ASSERT_OK(Put(Key(0), "value1")); + ASSERT_OK(Put(Key(2), "value2")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + ASSERT_OK(Put(Key(1), "value3")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + s = dbfull()->TEST_WaitForCompact(); + ASSERT_EQ(s, Status::OK()); + Destroy(options); + Reopen(options); + + // The hash does not match, compaction write fails + // fault_fs->SetChecksumHandoffFuncType(ChecksumType::kxxHash); + // Since the file system returns IOStatus::Corruption, it is an + // unrecoverable error. + ASSERT_OK(Put(Key(0), "value1")); + ASSERT_OK(Put(Key(2), "value2")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::FlushMemTable:FlushMemTableFinished", + "BackgroundCallCompaction:0"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "BackgroundCallCompaction:0", [&](void*) { + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kxxHash); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(Put(Key(1), "value3")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + s = dbfull()->TEST_WaitForCompact(); + ASSERT_EQ(s.severity(), + ROCKSDB_NAMESPACE::Status::Severity::kUnrecoverableError); + SyncPoint::GetInstance()->DisableProcessing(); + Destroy(options); + Reopen(options); + + // The file system does not support checksum handoff. The check + // will be ignored. + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kNoChecksum); + ASSERT_OK(Put(Key(0), "value1")); + ASSERT_OK(Put(Key(2), "value2")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + ASSERT_OK(Put(Key(1), "value3")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + s = dbfull()->TEST_WaitForCompact(); + ASSERT_EQ(s, Status::OK()); + + // Each write will be similated as corrupted. + // Since the file system returns IOStatus::Corruption, it is an + // unrecoverable error. + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kCRC32c); + ASSERT_OK(Put(Key(0), "value1")); + ASSERT_OK(Put(Key(2), "value2")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::FlushMemTable:FlushMemTableFinished", + "BackgroundCallCompaction:0"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "BackgroundCallCompaction:0", + [&](void*) { fault_fs->IngestDataCorruptionBeforeWrite(); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(Put(Key(1), "value3")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + s = dbfull()->TEST_WaitForCompact(); + ASSERT_EQ(s.severity(), + ROCKSDB_NAMESPACE::Status::Severity::kUnrecoverableError); + SyncPoint::GetInstance()->DisableProcessing(); + + Destroy(options); +} + +TEST_F(DBCompactionTest, CompactionWithChecksumHandoff2) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_SKIP("Test requires non-mem or non-encrypted environment"); + return; + } + std::shared_ptr fault_fs( + new FaultInjectionTestFS(FileSystem::Default())); + std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = 2; + options.num_levels = 3; + options.env = fault_fs_env.get(); + options.create_if_missing = true; + Status s; + Reopen(options); + + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kCRC32c); + ASSERT_OK(Put(Key(0), "value1")); + ASSERT_OK(Put(Key(2), "value2")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + ASSERT_OK(Put(Key(1), "value3")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + s = dbfull()->TEST_WaitForCompact(); + ASSERT_EQ(s, Status::OK()); + Destroy(options); + Reopen(options); + + // options is not set, the checksum handoff will not be triggered + ASSERT_OK(Put(Key(0), "value1")); + ASSERT_OK(Put(Key(2), "value2")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::FlushMemTable:FlushMemTableFinished", + "BackgroundCallCompaction:0"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "BackgroundCallCompaction:0", [&](void*) { + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kxxHash); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(Put(Key(1), "value3")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + s = dbfull()->TEST_WaitForCompact(); + ASSERT_EQ(s, Status::OK()); + SyncPoint::GetInstance()->DisableProcessing(); + Destroy(options); + Reopen(options); + + // The file system does not support checksum handoff. The check + // will be ignored. + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kNoChecksum); + ASSERT_OK(Put(Key(0), "value1")); + ASSERT_OK(Put(Key(2), "value2")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + ASSERT_OK(Put(Key(1), "value3")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + s = dbfull()->TEST_WaitForCompact(); + ASSERT_EQ(s, Status::OK()); + + // options is not set, the checksum handoff will not be triggered + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kCRC32c); + ASSERT_OK(Put(Key(0), "value1")); + ASSERT_OK(Put(Key(2), "value2")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::FlushMemTable:FlushMemTableFinished", + "BackgroundCallCompaction:0"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "BackgroundCallCompaction:0", + [&](void*) { fault_fs->IngestDataCorruptionBeforeWrite(); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(Put(Key(1), "value3")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + s = dbfull()->TEST_WaitForCompact(); + ASSERT_EQ(s, Status::OK()); + + Destroy(options); +} + +TEST_F(DBCompactionTest, CompactionWithChecksumHandoffManifest1) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_SKIP("Test requires non-mem or non-encrypted environment"); + return; + } + std::shared_ptr fault_fs( + new FaultInjectionTestFS(FileSystem::Default())); + std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = 2; + options.num_levels = 3; + options.env = fault_fs_env.get(); + options.create_if_missing = true; + options.checksum_handoff_file_types.Add(FileType::kDescriptorFile); + Status s; + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kCRC32c); + Reopen(options); + + ASSERT_OK(Put(Key(0), "value1")); + ASSERT_OK(Put(Key(2), "value2")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + ASSERT_OK(Put(Key(1), "value3")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + s = dbfull()->TEST_WaitForCompact(); + ASSERT_EQ(s, Status::OK()); + Destroy(options); + Reopen(options); + + // The hash does not match, compaction write fails + // fault_fs->SetChecksumHandoffFuncType(ChecksumType::kxxHash); + // Since the file system returns IOStatus::Corruption, it is mapped to + // kFatalError error. + ASSERT_OK(Put(Key(0), "value1")); + ASSERT_OK(Put(Key(2), "value2")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::FlushMemTable:FlushMemTableFinished", + "BackgroundCallCompaction:0"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "BackgroundCallCompaction:0", [&](void*) { + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kxxHash); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(Put(Key(1), "value3")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + s = dbfull()->TEST_WaitForCompact(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kFatalError); + SyncPoint::GetInstance()->DisableProcessing(); + Destroy(options); +} + +TEST_F(DBCompactionTest, CompactionWithChecksumHandoffManifest2) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_SKIP("Test requires non-mem or non-encrypted environment"); + return; + } + std::shared_ptr fault_fs( + new FaultInjectionTestFS(FileSystem::Default())); + std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = 2; + options.num_levels = 3; + options.env = fault_fs_env.get(); + options.create_if_missing = true; + options.checksum_handoff_file_types.Add(FileType::kDescriptorFile); + Status s; + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kNoChecksum); + Reopen(options); + + // The file system does not support checksum handoff. The check + // will be ignored. + ASSERT_OK(Put(Key(0), "value1")); + ASSERT_OK(Put(Key(2), "value2")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + ASSERT_OK(Put(Key(1), "value3")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + s = dbfull()->TEST_WaitForCompact(); + ASSERT_EQ(s, Status::OK()); + + // Each write will be similated as corrupted. + // Since the file system returns IOStatus::Corruption, it is mapped to + // kFatalError error. + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kCRC32c); + ASSERT_OK(Put(Key(0), "value1")); + ASSERT_OK(Put(Key(2), "value2")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::FlushMemTable:FlushMemTableFinished", + "BackgroundCallCompaction:0"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "BackgroundCallCompaction:0", + [&](void*) { fault_fs->IngestDataCorruptionBeforeWrite(); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(Put(Key(1), "value3")); + s = Flush(); + ASSERT_EQ(s, Status::OK()); + s = dbfull()->TEST_WaitForCompact(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kFatalError); + SyncPoint::GetInstance()->DisableProcessing(); + + Destroy(options); +} + +TEST_F(DBCompactionTest, FIFOChangeTemperature) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleFIFO; + options.num_levels = 1; + options.max_open_files = -1; + options.level0_file_num_compaction_trigger = 2; + options.create_if_missing = true; + CompactionOptionsFIFO fifo_options; + fifo_options.file_temperature_age_thresholds = {{Temperature::kCold, 1000}}; + fifo_options.max_table_files_size = 100000000; + options.compaction_options_fifo = fifo_options; + env_->SetMockSleep(); + Reopen(options); + + int total_cold = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "NewWritableFile::FileOptions.temperature", [&](void* arg) { + Temperature temperature = *(static_cast(arg)); + if (temperature == Temperature::kCold) { + total_cold++; + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // The file system does not support checksum handoff. The check + // will be ignored. + ASSERT_OK(Put(Key(0), "value1")); + env_->MockSleepForSeconds(800); + ASSERT_OK(Put(Key(2), "value2")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put(Key(0), "value1")); + env_->MockSleepForSeconds(800); + ASSERT_OK(Put(Key(2), "value2")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put(Key(0), "value1")); + env_->MockSleepForSeconds(800); + ASSERT_OK(Put(Key(2), "value2")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_OK(Put(Key(0), "value1")); + env_->MockSleepForSeconds(800); + ASSERT_OK(Put(Key(2), "value2")); + ASSERT_OK(Flush()); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + ColumnFamilyMetaData metadata; + db_->GetColumnFamilyMetaData(&metadata); + ASSERT_EQ(4, metadata.file_count); + ASSERT_EQ(Temperature::kUnknown, metadata.levels[0].files[0].temperature); + ASSERT_EQ(Temperature::kUnknown, metadata.levels[0].files[1].temperature); + ASSERT_EQ(Temperature::kCold, metadata.levels[0].files[2].temperature); + ASSERT_EQ(Temperature::kCold, metadata.levels[0].files[3].temperature); + ASSERT_EQ(2, total_cold); + + Destroy(options); +} + +TEST_F(DBCompactionTest, DisableMultiManualCompaction) { + const int kNumL0Files = 10; + + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = kNumL0Files; + Reopen(options); + + // Generate 2 levels of file to make sure the manual compaction is not skipped + for (int i = 0; i < 10; i++) { + ASSERT_OK(Put(Key(i), "value")); + if (i % 2) { + ASSERT_OK(Flush()); + } + } + MoveFilesToLevel(2); + + for (int i = 0; i < 10; i++) { + ASSERT_OK(Put(Key(i), "value")); + if (i % 2) { + ASSERT_OK(Flush()); + } + } + MoveFilesToLevel(1); + + // Block compaction queue + test::SleepingBackgroundTask sleeping_task_low; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::LOW); + + port::Thread compact_thread1([&]() { + CompactRangeOptions cro; + cro.exclusive_manual_compaction = false; + std::string begin_str = Key(0); + std::string end_str = Key(3); + Slice b = begin_str; + Slice e = end_str; + auto s = db_->CompactRange(cro, &b, &e); + ASSERT_TRUE(s.IsIncomplete()); + }); + + port::Thread compact_thread2([&]() { + CompactRangeOptions cro; + cro.exclusive_manual_compaction = false; + std::string begin_str = Key(4); + std::string end_str = Key(7); + Slice b = begin_str; + Slice e = end_str; + auto s = db_->CompactRange(cro, &b, &e); + ASSERT_TRUE(s.IsIncomplete()); + }); + + // Disable manual compaction should cancel both manual compactions and both + // compaction should return incomplete. + db_->DisableManualCompaction(); + + compact_thread1.join(); + compact_thread2.join(); + + sleeping_task_low.WakeUp(); + sleeping_task_low.WaitUntilDone(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); +} + +TEST_F(DBCompactionTest, DisableJustStartedManualCompaction) { + const int kNumL0Files = 4; + + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = kNumL0Files; + Reopen(options); + + // generate files, but avoid trigger auto compaction + for (int i = 0; i < kNumL0Files / 2; i++) { + ASSERT_OK(Put(Key(1), "value1")); + ASSERT_OK(Put(Key(2), "value2")); + ASSERT_OK(Flush()); + } + + // make sure the manual compaction background is started but not yet set the + // status to in_progress, then cancel the manual compaction, which should not + // result in segfault + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BGWorkCompaction", + "DBCompactionTest::DisableJustStartedManualCompaction:" + "PreDisableManualCompaction"}, + {"DBImpl::RunManualCompaction:Unscheduled", + "BackgroundCallCompaction:0"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + port::Thread compact_thread([&]() { + CompactRangeOptions cro; + cro.exclusive_manual_compaction = true; + auto s = db_->CompactRange(cro, nullptr, nullptr); + ASSERT_TRUE(s.IsIncomplete()); + }); + TEST_SYNC_POINT( + "DBCompactionTest::DisableJustStartedManualCompaction:" + "PreDisableManualCompaction"); + db_->DisableManualCompaction(); + + compact_thread.join(); +} + +TEST_F(DBCompactionTest, DisableInProgressManualCompaction) { + const int kNumL0Files = 4; + + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = kNumL0Files; + Reopen(options); + + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BackgroundCompaction:InProgress", + "DBCompactionTest::DisableInProgressManualCompaction:" + "PreDisableManualCompaction"}, + {"DBImpl::RunManualCompaction:Unscheduled", + "CompactionJob::Run():Start"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + // generate files, but avoid trigger auto compaction + for (int i = 0; i < kNumL0Files / 2; i++) { + ASSERT_OK(Put(Key(1), "value1")); + ASSERT_OK(Put(Key(2), "value2")); + ASSERT_OK(Flush()); + } + + port::Thread compact_thread([&]() { + CompactRangeOptions cro; + cro.exclusive_manual_compaction = true; + auto s = db_->CompactRange(cro, nullptr, nullptr); + ASSERT_TRUE(s.IsIncomplete()); + }); + + TEST_SYNC_POINT( + "DBCompactionTest::DisableInProgressManualCompaction:" + "PreDisableManualCompaction"); + db_->DisableManualCompaction(); + + compact_thread.join(); +} + +TEST_F(DBCompactionTest, DisableManualCompactionThreadQueueFull) { + const int kNumL0Files = 4; + + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::RunManualCompaction:Scheduled", + "DBCompactionTest::DisableManualCompactionThreadQueueFull:" + "PreDisableManualCompaction"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = kNumL0Files; + Reopen(options); + + // Block compaction queue + test::SleepingBackgroundTask sleeping_task_low; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::LOW); + + // generate files, but avoid trigger auto compaction + for (int i = 0; i < kNumL0Files / 2; i++) { + ASSERT_OK(Put(Key(1), "value1")); + ASSERT_OK(Put(Key(2), "value2")); + ASSERT_OK(Flush()); + } + + port::Thread compact_thread([&]() { + CompactRangeOptions cro; + cro.exclusive_manual_compaction = true; + auto s = db_->CompactRange(cro, nullptr, nullptr); + ASSERT_TRUE(s.IsIncomplete()); + }); + + TEST_SYNC_POINT( + "DBCompactionTest::DisableManualCompactionThreadQueueFull:" + "PreDisableManualCompaction"); + + // Generate more files to trigger auto compaction which is scheduled after + // manual compaction. Has to generate 4 more files because existing files are + // pending compaction + for (int i = 0; i < kNumL0Files; i++) { + ASSERT_OK(Put(Key(1), "value1")); + ASSERT_OK(Put(Key(2), "value2")); + ASSERT_OK(Flush()); + } + ASSERT_EQ(std::to_string(kNumL0Files + (kNumL0Files / 2)), FilesPerLevel(0)); + + db_->DisableManualCompaction(); + + // CompactRange should return before the compaction has the chance to run + compact_thread.join(); + + sleeping_task_low.WakeUp(); + sleeping_task_low.WaitUntilDone(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,1", FilesPerLevel(0)); +} + +TEST_F(DBCompactionTest, DisableManualCompactionThreadQueueFullDBClose) { + const int kNumL0Files = 4; + + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::RunManualCompaction:Scheduled", + "DBCompactionTest::DisableManualCompactionThreadQueueFullDBClose:" + "PreDisableManualCompaction"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = kNumL0Files; + Reopen(options); + + // Block compaction queue + test::SleepingBackgroundTask sleeping_task_low; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::LOW); + + // generate files, but avoid trigger auto compaction + for (int i = 0; i < kNumL0Files / 2; i++) { + ASSERT_OK(Put(Key(1), "value1")); + ASSERT_OK(Put(Key(2), "value2")); + ASSERT_OK(Flush()); + } + + port::Thread compact_thread([&]() { + CompactRangeOptions cro; + cro.exclusive_manual_compaction = true; + auto s = db_->CompactRange(cro, nullptr, nullptr); + ASSERT_TRUE(s.IsIncomplete()); + }); + + TEST_SYNC_POINT( + "DBCompactionTest::DisableManualCompactionThreadQueueFullDBClose:" + "PreDisableManualCompaction"); + + // Generate more files to trigger auto compaction which is scheduled after + // manual compaction. Has to generate 4 more files because existing files are + // pending compaction + for (int i = 0; i < kNumL0Files; i++) { + ASSERT_OK(Put(Key(1), "value1")); + ASSERT_OK(Put(Key(2), "value2")); + ASSERT_OK(Flush()); + } + ASSERT_EQ(std::to_string(kNumL0Files + (kNumL0Files / 2)), FilesPerLevel(0)); + + db_->DisableManualCompaction(); + + // CompactRange should return before the compaction has the chance to run + compact_thread.join(); + + // Try close DB while manual compaction is canceled but still in the queue. + // And an auto-triggered compaction is also in the queue. + auto s = db_->Close(); + ASSERT_OK(s); + + sleeping_task_low.WakeUp(); + sleeping_task_low.WaitUntilDone(); +} + +TEST_F(DBCompactionTest, DBCloseWithManualCompaction) { + const int kNumL0Files = 4; + + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::RunManualCompaction:Scheduled", + "DBCompactionTest::DisableManualCompactionThreadQueueFullDBClose:" + "PreDisableManualCompaction"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = kNumL0Files; + Reopen(options); + + // Block compaction queue + test::SleepingBackgroundTask sleeping_task_low; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::LOW); + + // generate files, but avoid trigger auto compaction + for (int i = 0; i < kNumL0Files / 2; i++) { + ASSERT_OK(Put(Key(1), "value1")); + ASSERT_OK(Put(Key(2), "value2")); + ASSERT_OK(Flush()); + } + + port::Thread compact_thread([&]() { + CompactRangeOptions cro; + cro.exclusive_manual_compaction = true; + auto s = db_->CompactRange(cro, nullptr, nullptr); + ASSERT_TRUE(s.IsIncomplete()); + }); + + TEST_SYNC_POINT( + "DBCompactionTest::DisableManualCompactionThreadQueueFullDBClose:" + "PreDisableManualCompaction"); + + // Generate more files to trigger auto compaction which is scheduled after + // manual compaction. Has to generate 4 more files because existing files are + // pending compaction + for (int i = 0; i < kNumL0Files; i++) { + ASSERT_OK(Put(Key(1), "value1")); + ASSERT_OK(Put(Key(2), "value2")); + ASSERT_OK(Flush()); + } + ASSERT_EQ(std::to_string(kNumL0Files + (kNumL0Files / 2)), FilesPerLevel(0)); + + // Close DB with manual compaction and auto triggered compaction in the queue. + auto s = db_->Close(); + ASSERT_OK(s); + + // manual compaction thread should return with Incomplete(). + compact_thread.join(); + + sleeping_task_low.WakeUp(); + sleeping_task_low.WaitUntilDone(); +} + +TEST_F(DBCompactionTest, + DisableManualCompactionDoesNotWaitForDrainingAutomaticCompaction) { + // When `CompactRangeOptions::exclusive_manual_compaction == true`, we wait + // for automatic compactions to drain before starting the manual compaction. + // This test verifies `DisableManualCompaction()` can cancel such a compaction + // without waiting for the drain to complete. + const int kNumL0Files = 4; + + // Enforces manual compaction enters wait loop due to pending automatic + // compaction. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BGWorkCompaction", "DBImpl::RunManualCompaction:NotScheduled"}, + {"DBImpl::RunManualCompaction:WaitScheduled", + "BackgroundCallCompaction:0"}}); + // The automatic compaction will cancel the waiting manual compaction. + // Completing this implies the cancellation did not wait on automatic + // compactions to finish. + bool callback_completed = false; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "BackgroundCallCompaction:0", [&](void* /*arg*/) { + db_->DisableManualCompaction(); + callback_completed = true; + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = kNumL0Files; + Reopen(options); + + for (int i = 0; i < kNumL0Files; ++i) { + ASSERT_OK(Put(Key(1), "value1")); + ASSERT_OK(Put(Key(2), "value2")); + ASSERT_OK(Flush()); + } + + CompactRangeOptions cro; + cro.exclusive_manual_compaction = true; + ASSERT_TRUE(db_->CompactRange(cro, nullptr, nullptr).IsIncomplete()); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_TRUE(callback_completed); +} + +TEST_F(DBCompactionTest, ChangeLevelConflictsWithManual) { + Options options = CurrentOptions(); + options.num_levels = 3; + Reopen(options); + + // Setup an LSM with L2 populated. + Random rnd(301); + ASSERT_OK(Put(Key(0), rnd.RandomString(990))); + ASSERT_OK(Put(Key(1), rnd.RandomString(990))); + { + CompactRangeOptions cro; + cro.change_level = true; + cro.target_level = 2; + ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr)); + } + ASSERT_EQ("0,0,1", FilesPerLevel(0)); + + // The background thread will refit L2->L1 while the foreground thread will + // attempt to run a compaction on new data. The following dependencies + // ensure the background manual compaction's refitting phase disables manual + // compaction immediately before the foreground manual compaction can register + // itself. Manual compaction is kept disabled until the foreground manual + // checks for the failure once. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + // Only do Put()s for foreground CompactRange() once the background + // CompactRange() has reached the refitting phase. + { + "DBImpl::CompactRange:BeforeRefit:1", + "DBCompactionTest::ChangeLevelConflictsWithManual:" + "PreForegroundCompactRange", + }, + // Right before we register the manual compaction, proceed with + // the refitting phase so manual compactions are disabled. Stay in + // the refitting phase with manual compactions disabled until it is + // noticed. + { + "DBImpl::RunManualCompaction:0", + "DBImpl::CompactRange:BeforeRefit:2", + }, + { + "DBImpl::CompactRange:PreRefitLevel", + "DBImpl::RunManualCompaction:1", + }, + { + "DBImpl::RunManualCompaction:PausedAtStart", + "DBImpl::CompactRange:PostRefitLevel", + }, + // If compaction somehow were scheduled, let's let it run after reenabling + // manual compactions. This dependency is not expected to be hit but is + // here for speculatively coercing future bugs. + { + "DBImpl::CompactRange:PostRefitLevel:ManualCompactionEnabled", + "BackgroundCallCompaction:0", + }, + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ROCKSDB_NAMESPACE::port::Thread refit_level_thread([&] { + CompactRangeOptions cro; + cro.change_level = true; + cro.target_level = 1; + ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr)); + }); + + TEST_SYNC_POINT( + "DBCompactionTest::ChangeLevelConflictsWithManual:" + "PreForegroundCompactRange"); + ASSERT_OK(Put(Key(0), rnd.RandomString(990))); + ASSERT_OK(Put(Key(1), rnd.RandomString(990))); + ASSERT_TRUE(dbfull() + ->CompactRange(CompactRangeOptions(), nullptr, nullptr) + .IsIncomplete()); + + refit_level_thread.join(); +} + +TEST_F(DBCompactionTest, BottomPriCompactionCountsTowardConcurrencyLimit) { + // Flushes several files to trigger compaction while lock is released during + // a bottom-pri compaction. Verifies it does not get scheduled to thread pool + // because per-DB limit for compaction parallelism is one (default). + const int kNumL0Files = 4; + const int kNumLevels = 3; + + env_->SetBackgroundThreads(1, Env::Priority::BOTTOM); + + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = kNumL0Files; + options.num_levels = kNumLevels; + DestroyAndReopen(options); + + // Setup last level to be non-empty since it's a bit unclear whether + // compaction to an empty level would be considered "bottommost". + ASSERT_OK(Put(Key(0), "val")); + ASSERT_OK(Flush()); + MoveFilesToLevel(kNumLevels - 1); + + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BGWorkBottomCompaction", + "DBCompactionTest::BottomPriCompactionCountsTowardConcurrencyLimit:" + "PreTriggerCompaction"}, + {"DBCompactionTest::BottomPriCompactionCountsTowardConcurrencyLimit:" + "PostTriggerCompaction", + "BackgroundCallCompaction:0"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + port::Thread compact_range_thread([&] { + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + cro.exclusive_manual_compaction = false; + ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr)); + }); + + // Sleep in the low-pri thread so any newly scheduled compaction will be + // queued. Otherwise it might finish before we check its existence. + test::SleepingBackgroundTask sleeping_task_low; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::LOW); + + TEST_SYNC_POINT( + "DBCompactionTest::BottomPriCompactionCountsTowardConcurrencyLimit:" + "PreTriggerCompaction"); + for (int i = 0; i < kNumL0Files; ++i) { + ASSERT_OK(Put(Key(0), "val")); + ASSERT_OK(Flush()); + } + ASSERT_EQ(0u, env_->GetThreadPoolQueueLen(Env::Priority::LOW)); + TEST_SYNC_POINT( + "DBCompactionTest::BottomPriCompactionCountsTowardConcurrencyLimit:" + "PostTriggerCompaction"); + + sleeping_task_low.WakeUp(); + sleeping_task_low.WaitUntilDone(); + compact_range_thread.join(); +} + +TEST_F(DBCompactionTest, BottommostFileCompactionAllowIngestBehind) { + // allow_ingest_behind prevents seqnum zeroing, and could cause + // compaction loop with reason kBottommostFiles. + Options options = CurrentOptions(); + options.env = env_; + options.compaction_style = kCompactionStyleLevel; + options.allow_ingest_behind = true; + options.comparator = BytewiseComparator(); + DestroyAndReopen(options); + + WriteOptions write_opts; + ASSERT_OK(db_->Put(write_opts, "infinite", "compaction loop")); + ASSERT_OK(db_->Put(write_opts, "infinite", "loop")); + + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + ASSERT_OK(db_->Put(write_opts, "bumpseqnum", "")); + ASSERT_OK(Flush()); + auto snapshot = db_->GetSnapshot(); + // Bump up oldest_snapshot_seqnum_ in VersionStorageInfo. + db_->ReleaseSnapshot(snapshot); + bool compacted = false; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "LevelCompactionPicker::PickCompaction:Return", [&](void* /* arg */) { + // There should not be a compaction. + compacted = true; + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + // Wait for compaction to be scheduled. + env_->SleepForMicroseconds(2000000); + ASSERT_FALSE(compacted); + // The following assert can be used to check for compaction loop: + // it used to wait forever before the fix. + // ASSERT_OK(dbfull()->TEST_WaitForCompact(true /* wait_unscheduled */)); +} + +TEST_F(DBCompactionTest, TurnOnLevelCompactionDynamicLevelBytes) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleLevel; + options.allow_ingest_behind = false; + options.level_compaction_dynamic_level_bytes = false; + options.num_levels = 6; + options.compression = kNoCompression; + options.max_bytes_for_level_base = 1 << 20; + options.max_bytes_for_level_multiplier = 10; + DestroyAndReopen(options); + + // put files in L0, L1 and L2 + WriteOptions write_opts; + ASSERT_OK(db_->Put(write_opts, Key(1), "val1")); + Random rnd(33); + // Fill L2 with size larger than max_bytes_for_level_base, + // so the level above it won't be drained. + for (int i = 2; i <= (1 << 10); ++i) { + ASSERT_OK(db_->Put(write_opts, Key(i), rnd.RandomString(2 << 10))); + } + ASSERT_OK(Flush()); + MoveFilesToLevel(2); + ASSERT_OK(db_->Put(write_opts, Key(2), "val2")); + ASSERT_OK(Flush()); + MoveFilesToLevel(2); + ASSERT_OK(db_->Put(write_opts, Key(1), "new_val1")); + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + ASSERT_OK(db_->Put(write_opts, Key(3), "val3")); + ASSERT_OK(Flush()); + ASSERT_EQ("1,1,2", FilesPerLevel()); + auto verify_db = [&]() { + ASSERT_EQ(Get(Key(1)), "new_val1"); + ASSERT_EQ(Get(Key(2)), "val2"); + ASSERT_EQ(Get(Key(3)), "val3"); + }; + verify_db(); + + options.level_compaction_dynamic_level_bytes = true; + Reopen(options); + // except for L0, files should be pushed down as much as possible + ASSERT_EQ("1,0,0,0,1,2", FilesPerLevel()); + verify_db(); + + // turning the options on and off should be safe + options.level_compaction_dynamic_level_bytes = false; + Reopen(options); + MoveFilesToLevel(1); + ASSERT_EQ("0,1,0,0,1,2", FilesPerLevel()); + verify_db(); + + // newly flushed file is also pushed down + options.level_compaction_dynamic_level_bytes = true; + Reopen(options); + // Files in L1 should be trivially moved down during DB opening. + // The file should be moved to L3, and then may be drained and compacted to + // L4. So we just check L1 and L2 here. + ASSERT_EQ(0, NumTableFilesAtLevel(1)); + ASSERT_EQ(0, NumTableFilesAtLevel(2)); + verify_db(); +} + +TEST_F(DBCompactionTest, TurnOnLevelCompactionDynamicLevelBytesUCToLC) { + // Basic test for migrating from UC to LC. + // DB has non-empty L1 that should be pushed down to last level (L49). + Options options = CurrentOptions(); + options.compaction_style = CompactionStyle::kCompactionStyleUniversal; + options.allow_ingest_behind = false; + options.level_compaction_dynamic_level_bytes = false; + options.num_levels = 50; + CreateAndReopenWithCF({"pikachu"}, options); + + Random rnd(33); + for (int f = 0; f < 10; ++f) { + ASSERT_OK(Put(1, Key(f), rnd.RandomString(1000))); + ASSERT_OK(Flush(1)); + } + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 1; + ASSERT_OK(db_->CompactRange(compact_options, handles_[1], nullptr, nullptr)); + ASSERT_EQ("0,1", FilesPerLevel(1)); + + options.compaction_style = CompactionStyle::kCompactionStyleLevel; + options.level_compaction_dynamic_level_bytes = true; + ReopenWithColumnFamilies({"default", "pikachu"}, options); + std::string expected_lsm = ""; + for (int i = 0; i < 49; ++i) { + expected_lsm += "0,"; + } + expected_lsm += "1"; + ASSERT_EQ(expected_lsm, FilesPerLevel(1)); + + // Tests that entries for trial move in MANIFEST should be valid + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_EQ(expected_lsm, FilesPerLevel(1)); +} + +TEST_F(DBCompactionTest, DrainUnnecessaryLevelsAfterMultiplierChanged) { + // When the level size multiplier increases such that fewer levels become + // necessary, unnecessary levels should to be drained. + const int kBaseLevelBytes = 256 << 10; // 256KB + const int kFileBytes = 64 << 10; // 64KB + const int kInitMultiplier = 2, kChangedMultiplier = 10; + const int kNumFiles = 32; + const int kNumLevels = 5; + const int kValueBytes = 1 << 10; // 1KB + + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.level_compaction_dynamic_level_bytes = true; + options.max_bytes_for_level_base = kBaseLevelBytes; + options.max_bytes_for_level_multiplier = kInitMultiplier; + options.num_levels = kNumLevels; + Reopen(options); + + // Initially we setup the LSM to look roughly as follows: + // + // L0: empty + // L1: 256KB + // ... + // L4: 1MB + Random rnd(301); + for (int file = 0; file < kNumFiles; ++file) { + for (int i = 0; i < kFileBytes / kValueBytes; ++i) { + ASSERT_OK(Put(Key(file * kFileBytes / kValueBytes + i), + rnd.RandomString(kValueBytes))); + } + ASSERT_OK(Flush()); + } + + int init_num_nonempty = 0; + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + for (int level = 1; level < kNumLevels; ++level) { + if (NumTableFilesAtLevel(level) > 0) { + ++init_num_nonempty; + } + } + + // After increasing the multiplier and running compaction fewer levels are + // needed to hold all the data. Unnecessary levels should be drained. + ASSERT_OK(db_->SetOptions({{"max_bytes_for_level_multiplier", + std::to_string(kChangedMultiplier)}})); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + int final_num_nonempty = 0; + for (int level = 1; level < kNumLevels; ++level) { + if (NumTableFilesAtLevel(level) > 0) { + ++final_num_nonempty; + } + } + ASSERT_GT(init_num_nonempty, final_num_nonempty); +} + +TEST_F(DBCompactionTest, DrainUnnecessaryLevelsAfterDBBecomesSmall) { + // When the DB size is smaller, e.g., large chunk of data deleted by + // DeleteRange(), unnecessary levels should to be drained. + const int kBaseLevelBytes = 256 << 10; // 256KB + const int kFileBytes = 64 << 10; // 64KB + const int kMultiplier = 2; + const int kNumFiles = 32; + const int kNumLevels = 5; + const int kValueBytes = 1 << 10; // 1KB + const int kDeleteFileNum = 8; + + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.level_compaction_dynamic_level_bytes = true; + options.max_bytes_for_level_base = kBaseLevelBytes; + options.max_bytes_for_level_multiplier = kMultiplier; + options.num_levels = kNumLevels; + Reopen(options); + + // Initially we setup the LSM to look roughly as follows: + // + // L0: empty + // L1: 256KB + // ... + // L4: 1MB + Random rnd(301); + for (int file = 0; file < kNumFiles; ++file) { + for (int i = 0; i < kFileBytes / kValueBytes; ++i) { + ASSERT_OK(Put(Key(file * kFileBytes / kValueBytes + i), + rnd.RandomString(kValueBytes))); + } + ASSERT_OK(Flush()); + if (file == kDeleteFileNum) { + // Ensure the DeleteRange() call below only delete data from last level + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(NumTableFilesAtLevel(kNumLevels - 1), kDeleteFileNum + 1); + } + } + + int init_num_nonempty = 0; + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + for (int level = 1; level < kNumLevels; ++level) { + if (NumTableFilesAtLevel(level) > 0) { + ++init_num_nonempty; + } + } + + // Disable auto compaction CompactRange() below + ASSERT_OK(dbfull()->SetOptions({{"disable_auto_compactions", "true"}})); + // Delete keys within first (kDeleteFileNum + 1) files' key ranges. + // This should reduce DB size enough such that there is now + // an unneeded level. + std::string begin = Key(0); + std::string end = Key(kDeleteFileNum * kFileBytes / kValueBytes); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), begin, end)); + Slice begin_slice = begin; + Slice end_slice = end; + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &begin_slice, &end_slice)); + int after_delete_range_nonempty = 0; + for (int level = 1; level < kNumLevels; ++level) { + if (NumTableFilesAtLevel(level) > 0) { + ++after_delete_range_nonempty; + } + } + ASSERT_OK(dbfull()->SetOptions({{"disable_auto_compactions", "false"}})); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + int final_num_nonempty = 0; + for (int level = 1; level < kNumLevels; ++level) { + if (NumTableFilesAtLevel(level) > 0) { + ++final_num_nonempty; + } + } + ASSERT_GE(init_num_nonempty, after_delete_range_nonempty); + ASSERT_GT(after_delete_range_nonempty, final_num_nonempty); +} + +TEST_F(DBCompactionTest, ManualCompactionCompactAllKeysInRange) { + // CompactRange() used to pre-compute target level to compact to + // before running compactions. However, the files at target level + // could be trivially moved down by some background compaction. This means + // some keys in the manual compaction key range may not be compacted + // during the manual compaction. This unit test tests this scenario. + // A fix has been applied for this scenario to always compact + // to the bottommost level. + const int kBaseLevelBytes = 8 << 20; // 8MB + const int kMultiplier = 2; + Options options = CurrentOptions(); + options.num_levels = 7; + options.level_compaction_dynamic_level_bytes = false; + options.compaction_style = kCompactionStyleLevel; + options.max_bytes_for_level_base = kBaseLevelBytes; + options.max_bytes_for_level_multiplier = kMultiplier; + options.compression = kNoCompression; + options.target_file_size_base = 2 * kBaseLevelBytes; + + DestroyAndReopen(options); + Random rnd(301); + // Populate L2 so that manual compaction will compact to at least L2. + // Otherwise, there is still a possibility of race condition where + // the manual compaction thread believes that max non-empty level is L1 + // while there is some auto compaction that moves some files from L1 to L2. + ASSERT_OK(db_->Put(WriteOptions(), Key(1000), rnd.RandomString(100))); + ASSERT_OK(Flush()); + MoveFilesToLevel(2); + ASSERT_EQ(1, NumTableFilesAtLevel(2)); + + // one file in L1: [Key(5), Key(6)] + ASSERT_OK( + db_->Put(WriteOptions(), Key(5), rnd.RandomString(kBaseLevelBytes / 3))); + ASSERT_OK( + db_->Put(WriteOptions(), Key(6), rnd.RandomString(kBaseLevelBytes / 3))); + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + + ASSERT_OK( + db_->Put(WriteOptions(), Key(1), rnd.RandomString(kBaseLevelBytes / 2))); + // We now do manual compaction for key range [Key(1), Key(6)]. + // First it compacts file [Key(1)] to L1. + // L1 will have two files [Key(1)], and [Key(5), Key(6)]. + // After L0 -> L1 manual compaction, an automatic compaction will trivially + // move both files from L1 to L2. Here the dependency makes manual compaction + // wait for auto-compaction to pick a compaction before proceeding. Manual + // compaction should not stop at L1 and keep compacting L2. With kForce + // specified, expected output is that manual compaction compacts to L2 and L2 + // will contain 2 files: one for Key(1000) and one for Key(1), Key(5) and + // Key(6). + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BackgroundCompaction():AfterPickCompaction", + "DBImpl::RunManualCompaction()::1"}}); + SyncPoint::GetInstance()->EnableProcessing(); + std::string begin_str = Key(1); + std::string end_str = Key(6); + Slice begin_slice = begin_str; + Slice end_slice = end_str; + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(cro, &begin_slice, &end_slice)); + + ASSERT_EQ(NumTableFilesAtLevel(2), 2); +} + +TEST_F(DBCompactionTest, + ManualCompactionCompactAllKeysInRangeDynamicLevelBytes) { + // Similar to the test above (ManualCompactionCompactAllKeysInRange), but with + // level_compaction_dynamic_level_bytes = true. + const int kBaseLevelBytes = 8 << 20; // 8MB + const int kMultiplier = 2; + Options options = CurrentOptions(); + options.num_levels = 7; + options.level_compaction_dynamic_level_bytes = true; + options.compaction_style = kCompactionStyleLevel; + options.max_bytes_for_level_base = kBaseLevelBytes; + options.max_bytes_for_level_multiplier = kMultiplier; + options.compression = kNoCompression; + options.target_file_size_base = 2 * kBaseLevelBytes; + DestroyAndReopen(options); + + Random rnd(301); + ASSERT_OK(db_->Put(WriteOptions(), Key(5), + rnd.RandomString(3 * kBaseLevelBytes / 2))); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(1, NumTableFilesAtLevel(6)); + // L6 now has one file with size ~ 3/2 * kBaseLevelBytes. + // L5 is the new base level, with target size ~ 3/4 * kBaseLevelBytes. + + ASSERT_OK( + db_->Put(WriteOptions(), Key(3), rnd.RandomString(kBaseLevelBytes / 3))); + ASSERT_OK( + db_->Put(WriteOptions(), Key(4), rnd.RandomString(kBaseLevelBytes / 3))); + ASSERT_OK(Flush()); + + MoveFilesToLevel(5); + ASSERT_EQ(1, NumTableFilesAtLevel(5)); + // L5 now has one file with size ~ 2/3 * kBaseLevelBytes, which is below its + // target size. + + ASSERT_OK( + db_->Put(WriteOptions(), Key(1), rnd.RandomString(kBaseLevelBytes / 3))); + ASSERT_OK( + db_->Put(WriteOptions(), Key(2), rnd.RandomString(kBaseLevelBytes / 3))); + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BackgroundCompaction():AfterPickCompaction", + "DBImpl::RunManualCompaction()::1"}}); + SyncPoint::GetInstance()->EnableProcessing(); + // After compacting the file with [Key(1), Key(2)] to L5, + // L5 has size ~ 4/3 * kBaseLevelBytes > its target size. + // We let manual compaction wait for an auto-compaction to pick + // a compaction before proceeding. The auto-compaction would + // trivially move both files in L5 down to L6. If manual compaction + // works correctly with kForce specified, it should rewrite the two files in + // L6 into a single file. + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + std::string begin_str = Key(1); + std::string end_str = Key(4); + Slice begin_slice = begin_str; + Slice end_slice = end_str; + ASSERT_OK(db_->CompactRange(cro, &begin_slice, &end_slice)); + ASSERT_EQ(2, NumTableFilesAtLevel(6)); + ASSERT_EQ(0, NumTableFilesAtLevel(5)); +} + +TEST_F(DBCompactionTest, NumberOfSubcompactions) { + // Tests that expected number of subcompactions are created. + class SubCompactionEventListener : public EventListener { + public: + void OnSubcompactionCompleted(const SubcompactionJobInfo&) override { + sub_compaction_finished_++; + } + void OnCompactionCompleted(DB*, const CompactionJobInfo&) override { + compaction_finished_++; + } + std::atomic sub_compaction_finished_{0}; + std::atomic compaction_finished_{0}; + }; + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleLevel; + options.compression = kNoCompression; + const int kFileSize = 100 << 10; // 100KB + options.target_file_size_base = kFileSize; + const int kLevel0CompactTrigger = 2; + options.level0_file_num_compaction_trigger = kLevel0CompactTrigger; + Destroy(options); + Random rnd(301); + + // Exposing internal implementation detail here where the + // number of subcompactions depends on the size of data + // being compacted. In particular, to enable x subcompactions, + // we need to compact at least x * target file size amount + // of data. + // + // Will write two files below to avoid trivial move. + // Size written in total: 500 * 1000 * 2 ~ 10MB ~ 100 * target file size. + const int kValueSize = 500; + const int kNumKeyPerFile = 1000; + for (int i = 1; i <= 8; ++i) { + options.max_subcompactions = i; + SubCompactionEventListener* listener = new SubCompactionEventListener(); + options.listeners.clear(); + options.listeners.emplace_back(listener); + TryReopen(options); + + for (int file = 0; file < kLevel0CompactTrigger; ++file) { + for (int key = file; key < 2 * kNumKeyPerFile; key += 2) { + ASSERT_OK(Put(Key(key), rnd.RandomString(kValueSize))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(listener->compaction_finished_, 1); + EXPECT_EQ(listener->sub_compaction_finished_, i); + Destroy(options); + } +} + +TEST_F(DBCompactionTest, VerifyRecordCount) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleLevel; + options.level0_file_num_compaction_trigger = 3; + options.compaction_verify_record_count = true; + DestroyAndReopen(options); + Random rnd(301); + + // Create 2 overlapping L0 files + for (int i = 1; i < 20; i += 2) { + ASSERT_OK(Put(Key(i), rnd.RandomString(100))); + } + ASSERT_OK(Flush()); + + for (int i = 0; i < 20; i += 2) { + ASSERT_OK(Put(Key(i), rnd.RandomString(100))); + } + ASSERT_OK(Flush()); + + // Only iterator through 10 keys and force compaction to finish. + int num_iter = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::ProcessKeyValueCompaction()::stop", [&](void* stop_ptr) { + num_iter++; + if (num_iter == 10) { + *(bool*)stop_ptr = true; + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + + Status s = db_->CompactRange(CompactRangeOptions(), nullptr, nullptr); + ASSERT_TRUE(s.IsCorruption()); + const char* expect = + "Compaction number of input keys does not match number of keys " + "processed."; + ASSERT_TRUE(std::strstr(s.getState(), expect)); +} +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_dynamic_level_test.cc b/librocksdb-sys/rocksdb/db/db_dynamic_level_test.cc new file mode 100644 index 0000000..a1c2fa9 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_dynamic_level_test.cc @@ -0,0 +1,499 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +// Introduction of SyncPoint effectively disabled building and running this test +// in Release build. +// which is a pity, it is a good test + +#include "db/db_test_util.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/env.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { +class DBTestDynamicLevel : public DBTestBase { + public: + DBTestDynamicLevel() + : DBTestBase("db_dynamic_level_test", /*env_do_fsync=*/true) {} +}; + +TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase) { + if (!Snappy_Supported() || !LZ4_Supported()) { + return; + } + // Use InMemoryEnv, or it would be too slow. + std::unique_ptr env(NewMemEnv(env_)); + + const int kNKeys = 1000; + int keys[kNKeys]; + + auto verify_func = [&]() { + for (int i = 0; i < kNKeys; i++) { + ASSERT_NE("NOT_FOUND", Get(Key(i))); + ASSERT_NE("NOT_FOUND", Get(Key(kNKeys * 2 + i))); + if (i < kNKeys / 10) { + ASSERT_EQ("NOT_FOUND", Get(Key(kNKeys + keys[i]))); + } else { + ASSERT_NE("NOT_FOUND", Get(Key(kNKeys + keys[i]))); + } + } + }; + + Random rnd(301); + for (int ordered_insert = 0; ordered_insert <= 1; ordered_insert++) { + for (int i = 0; i < kNKeys; i++) { + keys[i] = i; + } + if (ordered_insert == 0) { + RandomShuffle(std::begin(keys), std::end(keys), rnd.Next()); + } + for (int max_background_compactions = 1; max_background_compactions < 4; + max_background_compactions += 2) { + Options options; + options.env = env.get(); + options.create_if_missing = true; + options.write_buffer_size = 2048; + options.max_write_buffer_number = 2; + options.level0_file_num_compaction_trigger = 2; + options.level0_slowdown_writes_trigger = 2; + options.level0_stop_writes_trigger = 2; + options.target_file_size_base = 2048; + options.level_compaction_dynamic_level_bytes = true; + options.max_bytes_for_level_base = 10240; + options.max_bytes_for_level_multiplier = 4; + options.max_background_compactions = max_background_compactions; + options.num_levels = 5; + + options.compression_per_level.resize(3); + options.compression_per_level[0] = kNoCompression; + options.compression_per_level[1] = kLZ4Compression; + options.compression_per_level[2] = kSnappyCompression; + options.env = env_; + + DestroyAndReopen(options); + + for (int i = 0; i < kNKeys; i++) { + int key = keys[i]; + ASSERT_OK(Put(Key(kNKeys + key), rnd.RandomString(102))); + ASSERT_OK(Put(Key(key), rnd.RandomString(102))); + ASSERT_OK(Put(Key(kNKeys * 2 + key), rnd.RandomString(102))); + ASSERT_OK(Delete(Key(kNKeys + keys[i / 10]))); + env_->SleepForMicroseconds(5000); + } + + uint64_t int_prop; + ASSERT_TRUE(db_->GetIntProperty("rocksdb.background-errors", &int_prop)); + ASSERT_EQ(0U, int_prop); + + // Verify DB + for (int j = 0; j < 2; j++) { + verify_func(); + if (j == 0) { + Reopen(options); + } + } + + // Test compact range works + ASSERT_OK( + dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + // All data should be in the last level. + ColumnFamilyMetaData cf_meta; + db_->GetColumnFamilyMetaData(&cf_meta); + ASSERT_EQ(5U, cf_meta.levels.size()); + for (int i = 0; i < 4; i++) { + ASSERT_EQ(0U, cf_meta.levels[i].files.size()); + } + ASSERT_GT(cf_meta.levels[4U].files.size(), 0U); + verify_func(); + + Close(); + } + } + + env_->SetBackgroundThreads(1, Env::LOW); + env_->SetBackgroundThreads(1, Env::HIGH); +} + +// Test specific cases in dynamic max bytes +TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase2) { + Random rnd(301); + int kMaxKey = 1000000; + + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.create_if_missing = true; + options.write_buffer_size = 20480; + options.max_write_buffer_number = 2; + options.level0_file_num_compaction_trigger = 2; + options.level0_slowdown_writes_trigger = 9999; + options.level0_stop_writes_trigger = 9999; + options.target_file_size_base = 9102; + options.level_compaction_dynamic_level_bytes = true; + options.max_bytes_for_level_base = 40960; + options.max_bytes_for_level_multiplier = 4; + options.max_background_compactions = 2; + options.num_levels = 5; + options.max_compaction_bytes = 0; // Force not expanding in compactions + options.db_host_id = ""; // Setting this messes up the file size calculation + BlockBasedTableOptions table_options; + table_options.block_size = 1024; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + DestroyAndReopen(options); + ASSERT_OK(dbfull()->SetOptions({ + {"disable_auto_compactions", "true"}, + })); + + uint64_t int_prop; + std::string str_prop; + + // Initial base level is the last level + ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop)); + ASSERT_EQ(4U, int_prop); + + // Put about 28K to L0 + for (int i = 0; i < 70; i++) { + ASSERT_OK(Put(Key(static_cast(rnd.Uniform(kMaxKey))), + rnd.RandomString(380))); + } + ASSERT_OK(dbfull()->SetOptions({ + {"disable_auto_compactions", "false"}, + })); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop)); + ASSERT_EQ(4U, int_prop); + + // Insert extra about 28K to L0. After they are compacted to L4, the base + // level should be changed to L3. + ASSERT_OK(dbfull()->SetOptions({ + {"disable_auto_compactions", "true"}, + })); + for (int i = 0; i < 70; i++) { + ASSERT_OK(Put(Key(static_cast(rnd.Uniform(kMaxKey))), + rnd.RandomString(380))); + } + + ASSERT_OK(dbfull()->SetOptions({ + {"disable_auto_compactions", "false"}, + })); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop)); + ASSERT_EQ(3U, int_prop); + ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level1", &str_prop)); + ASSERT_EQ("0", str_prop); + ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level2", &str_prop)); + ASSERT_EQ("0", str_prop); + + // Write even more data while leaving the base level at L3. + ASSERT_OK(dbfull()->SetOptions({ + {"disable_auto_compactions", "true"}, + })); + // Write about 40K more + for (int i = 0; i < 100; i++) { + ASSERT_OK(Put(Key(static_cast(rnd.Uniform(kMaxKey))), + rnd.RandomString(380))); + } + ASSERT_OK(dbfull()->SetOptions({ + {"disable_auto_compactions", "false"}, + })); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop)); + ASSERT_EQ(3U, int_prop); + + // Fill up L0, and then run an (auto) L0->Lmax compaction to raise the base + // level to 2. + ASSERT_OK(dbfull()->SetOptions({ + {"disable_auto_compactions", "true"}, + })); + // Write about 650K more. + // Each file is about 11KB, with 9KB of data. + for (int i = 0; i < 1300; i++) { + ASSERT_OK(Put(Key(static_cast(rnd.Uniform(kMaxKey))), + rnd.RandomString(380))); + } + + // Make sure that the compaction starts before the last bit of data is + // flushed, so that the base level isn't raised to L1. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"CompactionJob::Run():Start", "DynamicLevelMaxBytesBase2:0"}, + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(dbfull()->SetOptions({ + {"disable_auto_compactions", "false"}, + })); + + TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:0"); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop)); + ASSERT_EQ(2U, int_prop); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + + // Write more data until the base level changes to L1. There will be + // a manual compaction going on at the same time. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"CompactionJob::Run():Start", "DynamicLevelMaxBytesBase2:1"}, + {"DynamicLevelMaxBytesBase2:2", "CompactionJob::Run():End"}, + {"DynamicLevelMaxBytesBase2:compact_range_finish", + "FlushJob::WriteLevel0Table"}, + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ROCKSDB_NAMESPACE::port::Thread thread([this] { + TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:compact_range_start"); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:compact_range_finish"); + }); + + TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:1"); + for (int i = 0; i < 2; i++) { + ASSERT_OK(Put(Key(static_cast(rnd.Uniform(kMaxKey))), + rnd.RandomString(380))); + } + TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:2"); + + ASSERT_OK(Flush()); + + thread.join(); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + + ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop)); + ASSERT_EQ(1U, int_prop); +} + +// Test specific cases in dynamic max bytes +TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesCompactRange) { + Random rnd(301); + int kMaxKey = 1000000; + + Options options = CurrentOptions(); + options.create_if_missing = true; + options.write_buffer_size = 2048; + options.max_write_buffer_number = 2; + options.level0_file_num_compaction_trigger = 2; + options.level0_slowdown_writes_trigger = 9999; + options.level0_stop_writes_trigger = 9999; + options.target_file_size_base = 2; + options.level_compaction_dynamic_level_bytes = true; + options.max_bytes_for_level_base = 10240; + options.max_bytes_for_level_multiplier = 4; + options.max_background_compactions = 1; + const int kNumLevels = 5; + options.num_levels = kNumLevels; + options.max_compaction_bytes = 1; // Force not expanding in compactions + BlockBasedTableOptions table_options; + table_options.block_size = 1024; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + DestroyAndReopen(options); + + // Compact against empty DB + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + uint64_t int_prop; + std::string str_prop; + + // Initial base level is the last level + ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop)); + ASSERT_EQ(4U, int_prop); + + // Put about 7K to L0 + for (int i = 0; i < 140; i++) { + ASSERT_OK( + Put(Key(static_cast(rnd.Uniform(kMaxKey))), rnd.RandomString(80))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + if (NumTableFilesAtLevel(0) == 0) { + // Make sure level 0 is not empty + ASSERT_OK( + Put(Key(static_cast(rnd.Uniform(kMaxKey))), rnd.RandomString(80))); + ASSERT_OK(Flush()); + } + + ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop)); + ASSERT_EQ(3U, int_prop); + ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level1", &str_prop)); + ASSERT_EQ("0", str_prop); + ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level2", &str_prop)); + ASSERT_EQ("0", str_prop); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + + std::set output_levels; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionPicker::CompactRange:Return", [&](void* arg) { + Compaction* compaction = reinterpret_cast(arg); + output_levels.insert(compaction->output_level()); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(output_levels.size(), 2); + ASSERT_TRUE(output_levels.find(3) != output_levels.end()); + ASSERT_TRUE(output_levels.find(4) != output_levels.end()); + ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level0", &str_prop)); + ASSERT_EQ("0", str_prop); + ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level3", &str_prop)); + ASSERT_EQ("0", str_prop); + // Base level is still level 3. + ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop)); + ASSERT_EQ(3U, int_prop); +} + +TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBaseInc) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.write_buffer_size = 2048; + options.max_write_buffer_number = 2; + options.level0_file_num_compaction_trigger = 2; + options.level0_slowdown_writes_trigger = 2; + options.level0_stop_writes_trigger = 2; + options.target_file_size_base = 2048; + options.level_compaction_dynamic_level_bytes = true; + options.max_bytes_for_level_base = 10240; + options.max_bytes_for_level_multiplier = 4; + options.max_background_compactions = 2; + options.num_levels = 5; + options.max_compaction_bytes = 100000000; + + DestroyAndReopen(options); + + int non_trivial = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial", + [&](void* /*arg*/) { non_trivial++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Random rnd(301); + const int total_keys = 3000; + const int random_part_size = 100; + for (int i = 0; i < total_keys; i++) { + std::string value = rnd.RandomString(random_part_size); + PutFixed32(&value, static_cast(i)); + ASSERT_OK(Put(Key(i), value)); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + ASSERT_EQ(non_trivial, 0); + + for (int i = 0; i < total_keys; i++) { + std::string value = Get(Key(i)); + ASSERT_EQ(DecodeFixed32(value.c_str() + random_part_size), + static_cast(i)); + } + + env_->SetBackgroundThreads(1, Env::LOW); + env_->SetBackgroundThreads(1, Env::HIGH); +} + +TEST_F(DBTestDynamicLevel, DISABLED_MigrateToDynamicLevelMaxBytesBase) { + Random rnd(301); + const int kMaxKey = 2000; + + Options options; + options.create_if_missing = true; + options.write_buffer_size = 2048; + options.max_write_buffer_number = 8; + options.level0_file_num_compaction_trigger = 4; + options.level0_slowdown_writes_trigger = 4; + options.level0_stop_writes_trigger = 8; + options.target_file_size_base = 2048; + options.level_compaction_dynamic_level_bytes = false; + options.max_bytes_for_level_base = 10240; + options.max_bytes_for_level_multiplier = 4; + options.num_levels = 8; + + DestroyAndReopen(options); + + auto verify_func = [&](int num_keys, bool if_sleep) { + for (int i = 0; i < num_keys; i++) { + ASSERT_NE("NOT_FOUND", Get(Key(kMaxKey + i))); + if (i < num_keys / 10) { + ASSERT_EQ("NOT_FOUND", Get(Key(i))); + } else { + ASSERT_NE("NOT_FOUND", Get(Key(i))); + } + if (if_sleep && i % 1000 == 0) { + // Without it, valgrind may choose not to give another + // thread a chance to run before finishing the function, + // causing the test to be extremely slow. + env_->SleepForMicroseconds(1); + } + } + }; + + int total_keys = 1000; + for (int i = 0; i < total_keys; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(102))); + ASSERT_OK(Put(Key(kMaxKey + i), rnd.RandomString(102))); + ASSERT_OK(Delete(Key(i / 10))); + } + verify_func(total_keys, false); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + options.level_compaction_dynamic_level_bytes = true; + options.disable_auto_compactions = true; + Reopen(options); + verify_func(total_keys, false); + + std::atomic_bool compaction_finished; + compaction_finished = false; + // Issue manual compaction in one thread and still verify DB state + // in main thread. + ROCKSDB_NAMESPACE::port::Thread t([&]() { + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = options.num_levels - 1; + ASSERT_OK(dbfull()->CompactRange(compact_options, nullptr, nullptr)); + compaction_finished.store(true); + }); + do { + verify_func(total_keys, true); + } while (!compaction_finished.load()); + t.join(); + + ASSERT_OK(dbfull()->SetOptions({ + {"disable_auto_compactions", "false"}, + })); + + int total_keys2 = 2000; + for (int i = total_keys; i < total_keys2; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(102))); + ASSERT_OK(Put(Key(kMaxKey + i), rnd.RandomString(102))); + ASSERT_OK(Delete(Key(i / 10))); + } + + verify_func(total_keys2, false); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + verify_func(total_keys2, false); + + // Base level is not level 1 + ASSERT_EQ(NumTableFilesAtLevel(1), 0); + ASSERT_EQ(NumTableFilesAtLevel(2), 0); +} +} // namespace ROCKSDB_NAMESPACE + + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_encryption_test.cc b/librocksdb-sys/rocksdb/db/db_encryption_test.cc new file mode 100644 index 0000000..fc8be5b --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_encryption_test.cc @@ -0,0 +1,126 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#include "db/db_test_util.h" +#include "port/stack_trace.h" +#include "rocksdb/perf_context.h" +#include "test_util/sync_point.h" +#include +#include + +namespace ROCKSDB_NAMESPACE { + +class DBEncryptionTest : public DBTestBase { + public: + DBEncryptionTest() + : DBTestBase("db_encryption_test", /*env_do_fsync=*/true) {} + Env* GetTargetEnv() { + if (encrypted_env_ != nullptr) { + return (static_cast(encrypted_env_))->target(); + } else { + return env_; + } + } +}; + + +TEST_F(DBEncryptionTest, CheckEncrypted) { + ASSERT_OK(Put("foo567", "v1.fetdq")); + ASSERT_OK(Put("bar123", "v2.dfgkjdfghsd")); + Close(); + + // Open all files and look for the values we've put in there. + // They should not be found if encrypted, otherwise + // they should be found. + std::vector fileNames; + auto status = env_->GetChildren(dbname_, &fileNames); + ASSERT_OK(status); + + Env* target = GetTargetEnv(); + int hits = 0; + for (auto it = fileNames.begin(); it != fileNames.end(); ++it) { + if (*it == "LOCK") { + continue; + } + auto filePath = dbname_ + "/" + *it; + std::unique_ptr seqFile; + auto envOptions = EnvOptions(CurrentOptions()); + status = target->NewSequentialFile(filePath, &seqFile, envOptions); + ASSERT_OK(status); + + uint64_t fileSize; + status = target->GetFileSize(filePath, &fileSize); + ASSERT_OK(status); + + std::string scratch; + scratch.reserve(fileSize); + Slice data; + status = seqFile->Read(fileSize, &data, (char*)scratch.data()); + ASSERT_OK(status); + + if (data.ToString().find("foo567") != std::string::npos) { + hits++; + // std::cout << "Hit in " << filePath << "\n"; + } + if (data.ToString().find("v1.fetdq") != std::string::npos) { + hits++; + // std::cout << "Hit in " << filePath << "\n"; + } + if (data.ToString().find("bar123") != std::string::npos) { + hits++; + // std::cout << "Hit in " << filePath << "\n"; + } + if (data.ToString().find("v2.dfgkjdfghsd") != std::string::npos) { + hits++; + // std::cout << "Hit in " << filePath << "\n"; + } + if (data.ToString().find("dfgk") != std::string::npos) { + hits++; + // std::cout << "Hit in " << filePath << "\n"; + } + } + if (encrypted_env_) { + ASSERT_EQ(hits, 0); + } else { + ASSERT_GE(hits, 4); + } +} + +TEST_F(DBEncryptionTest, ReadEmptyFile) { + auto defaultEnv = GetTargetEnv(); + + // create empty file for reading it back in later + auto envOptions = EnvOptions(CurrentOptions()); + auto filePath = dbname_ + "/empty.empty"; + + Status status; + { + std::unique_ptr writableFile; + status = defaultEnv->NewWritableFile(filePath, &writableFile, envOptions); + ASSERT_OK(status); + } + + std::unique_ptr seqFile; + status = defaultEnv->NewSequentialFile(filePath, &seqFile, envOptions); + ASSERT_OK(status); + + std::string scratch; + Slice data; + // reading back 16 bytes from the empty file shouldn't trigger an assertion. + // it should just work and return an empty string + status = seqFile->Read(16, &data, (char*)scratch.data()); + ASSERT_OK(status); + + ASSERT_TRUE(data.empty()); +} + + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_filesnapshot.cc b/librocksdb-sys/rocksdb/db/db_filesnapshot.cc new file mode 100644 index 0000000..cb95a16 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_filesnapshot.cc @@ -0,0 +1,409 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + + +#include +#include +#include +#include +#include + +#include "db/db_impl/db_impl.h" +#include "db/job_context.h" +#include "db/version_set.h" +#include "file/file_util.h" +#include "file/filename.h" +#include "logging/logging.h" +#include "port/port.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/metadata.h" +#include "rocksdb/types.h" +#include "test_util/sync_point.h" +#include "util/file_checksum_helper.h" +#include "util/mutexlock.h" + +namespace ROCKSDB_NAMESPACE { + +Status DBImpl::FlushForGetLiveFiles() { + return DBImpl::FlushAllColumnFamilies(FlushOptions(), + FlushReason::kGetLiveFiles); +} + +Status DBImpl::GetLiveFiles(std::vector& ret, + uint64_t* manifest_file_size, bool flush_memtable) { + *manifest_file_size = 0; + + mutex_.Lock(); + + if (flush_memtable) { + Status status = FlushForGetLiveFiles(); + if (!status.ok()) { + mutex_.Unlock(); + ROCKS_LOG_ERROR(immutable_db_options_.info_log, "Cannot Flush data %s\n", + status.ToString().c_str()); + return status; + } + } + + // Make a set of all of the live table and blob files + std::vector live_table_files; + std::vector live_blob_files; + for (auto cfd : *versions_->GetColumnFamilySet()) { + if (cfd->IsDropped()) { + continue; + } + cfd->current()->AddLiveFiles(&live_table_files, &live_blob_files); + } + + ret.clear(); + ret.reserve(live_table_files.size() + live_blob_files.size() + + 3); // for CURRENT + MANIFEST + OPTIONS + + // create names of the live files. The names are not absolute + // paths, instead they are relative to dbname_. + for (const auto& table_file_number : live_table_files) { + ret.emplace_back(MakeTableFileName("", table_file_number)); + } + + for (const auto& blob_file_number : live_blob_files) { + ret.emplace_back(BlobFileName("", blob_file_number)); + } + + ret.emplace_back(CurrentFileName("")); + ret.emplace_back(DescriptorFileName("", versions_->manifest_file_number())); + // The OPTIONS file number is zero in read-write mode when OPTIONS file + // writing failed and the DB was configured with + // `fail_if_options_file_error == false`. In read-only mode the OPTIONS file + // number is zero when no OPTIONS file exist at all. In those cases we do not + // record any OPTIONS file in the live file list. + if (versions_->options_file_number() != 0) { + ret.emplace_back(OptionsFileName("", versions_->options_file_number())); + } + + // find length of manifest file while holding the mutex lock + *manifest_file_size = versions_->manifest_file_size(); + + mutex_.Unlock(); + return Status::OK(); +} + +Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) { + // Record tracked WALs as a (minimum) cross-check for directory scan + std::vector required_by_manifest; + + // If caller disabled deletions, this function should return files that are + // guaranteed not to be deleted until deletions are re-enabled. We need to + // wait for pending purges to finish since WalManager doesn't know which + // files are going to be purged. Additional purges won't be scheduled as + // long as deletions are disabled (so the below loop must terminate). + // Also note that we disable deletions anyway to avoid the case where a + // file is deleted in the middle of the scan, causing IO error. + Status deletions_disabled = DisableFileDeletions(); + { + InstrumentedMutexLock l(&mutex_); + while (pending_purge_obsolete_files_ > 0 || bg_purge_scheduled_ > 0) { + bg_cv_.Wait(); + } + + // Record tracked WALs as a (minimum) cross-check for directory scan + const auto& manifest_wals = versions_->GetWalSet().GetWals(); + required_by_manifest.reserve(manifest_wals.size()); + for (const auto& wal : manifest_wals) { + required_by_manifest.push_back(wal.first); + } + } + + Status s = wal_manager_.GetSortedWalFiles(files); + + // DisableFileDeletions / EnableFileDeletions not supported in read-only DB + if (deletions_disabled.ok()) { + Status s2 = EnableFileDeletions(/*force*/ false); + assert(s2.ok()); + s2.PermitUncheckedError(); + } else { + assert(deletions_disabled.IsNotSupported()); + } + + if (s.ok()) { + // Verify includes those required by manifest (one sorted list is superset + // of the other) + auto required = required_by_manifest.begin(); + auto included = files.begin(); + + while (required != required_by_manifest.end()) { + if (included == files.end() || *required < (*included)->LogNumber()) { + // FAIL - did not find + return Status::Corruption( + "WAL file " + std::to_string(*required) + + " required by manifest but not in directory list"); + } + if (*required == (*included)->LogNumber()) { + ++required; + ++included; + } else { + assert(*required > (*included)->LogNumber()); + ++included; + } + } + } + + return s; +} + +Status DBImpl::GetCurrentWalFile(std::unique_ptr* current_log_file) { + uint64_t current_logfile_number; + { + InstrumentedMutexLock l(&mutex_); + current_logfile_number = logfile_number_; + } + + return wal_manager_.GetLiveWalFile(current_logfile_number, current_log_file); +} + +Status DBImpl::GetLiveFilesStorageInfo( + const LiveFilesStorageInfoOptions& opts, + std::vector* files) { + // To avoid returning partial results, only move results to files on success. + assert(files); + files->clear(); + std::vector results; + + // NOTE: This implementation was largely migrated from Checkpoint. + + Status s; + VectorLogPtr live_wal_files; + bool flush_memtable = true; + if (!immutable_db_options_.allow_2pc) { + if (opts.wal_size_for_flush == std::numeric_limits::max()) { + flush_memtable = false; + } else if (opts.wal_size_for_flush > 0) { + // If the outstanding log files are small, we skip the flush. + s = GetSortedWalFiles(live_wal_files); + + if (!s.ok()) { + return s; + } + + // Don't flush column families if total log size is smaller than + // log_size_for_flush. We copy the log files instead. + // We may be able to cover 2PC case too. + uint64_t total_wal_size = 0; + for (auto& wal : live_wal_files) { + total_wal_size += wal->SizeFileBytes(); + } + if (total_wal_size < opts.wal_size_for_flush) { + flush_memtable = false; + } + live_wal_files.clear(); + } + } + + // This is a modified version of GetLiveFiles, to get access to more + // metadata. + mutex_.Lock(); + if (flush_memtable) { + Status status = FlushForGetLiveFiles(); + if (!status.ok()) { + mutex_.Unlock(); + ROCKS_LOG_ERROR(immutable_db_options_.info_log, "Cannot Flush data %s\n", + status.ToString().c_str()); + return status; + } + } + + // Make a set of all of the live table and blob files + for (auto cfd : *versions_->GetColumnFamilySet()) { + if (cfd->IsDropped()) { + continue; + } + VersionStorageInfo& vsi = *cfd->current()->storage_info(); + auto& cf_paths = cfd->ioptions()->cf_paths; + + auto GetDir = [&](size_t path_id) { + // Matching TableFileName() behavior + if (path_id >= cf_paths.size()) { + assert(false); + return cf_paths.back().path; + } else { + return cf_paths[path_id].path; + } + }; + + for (int level = 0; level < vsi.num_levels(); ++level) { + const auto& level_files = vsi.LevelFiles(level); + for (const auto& meta : level_files) { + assert(meta); + + results.emplace_back(); + LiveFileStorageInfo& info = results.back(); + + info.relative_filename = MakeTableFileName(meta->fd.GetNumber()); + info.directory = GetDir(meta->fd.GetPathId()); + info.file_number = meta->fd.GetNumber(); + info.file_type = kTableFile; + info.size = meta->fd.GetFileSize(); + if (opts.include_checksum_info) { + info.file_checksum_func_name = meta->file_checksum_func_name; + info.file_checksum = meta->file_checksum; + if (info.file_checksum_func_name.empty()) { + info.file_checksum_func_name = kUnknownFileChecksumFuncName; + info.file_checksum = kUnknownFileChecksum; + } + } + info.temperature = meta->temperature; + } + } + const auto& blob_files = vsi.GetBlobFiles(); + for (const auto& meta : blob_files) { + assert(meta); + + results.emplace_back(); + LiveFileStorageInfo& info = results.back(); + + info.relative_filename = BlobFileName(meta->GetBlobFileNumber()); + info.directory = GetDir(/* path_id */ 0); + info.file_number = meta->GetBlobFileNumber(); + info.file_type = kBlobFile; + info.size = meta->GetBlobFileSize(); + if (opts.include_checksum_info) { + info.file_checksum_func_name = meta->GetChecksumMethod(); + info.file_checksum = meta->GetChecksumValue(); + if (info.file_checksum_func_name.empty()) { + info.file_checksum_func_name = kUnknownFileChecksumFuncName; + info.file_checksum = kUnknownFileChecksum; + } + } + // TODO?: info.temperature + } + } + + // Capture some final info before releasing mutex + const uint64_t manifest_number = versions_->manifest_file_number(); + const uint64_t manifest_size = versions_->manifest_file_size(); + const uint64_t options_number = versions_->options_file_number(); + const uint64_t options_size = versions_->options_file_size_; + const uint64_t min_log_num = MinLogNumberToKeep(); + + mutex_.Unlock(); + + std::string manifest_fname = DescriptorFileName(manifest_number); + { // MANIFEST + results.emplace_back(); + LiveFileStorageInfo& info = results.back(); + + info.relative_filename = manifest_fname; + info.directory = GetName(); + info.file_number = manifest_number; + info.file_type = kDescriptorFile; + info.size = manifest_size; + info.trim_to_size = true; + if (opts.include_checksum_info) { + info.file_checksum_func_name = kUnknownFileChecksumFuncName; + info.file_checksum = kUnknownFileChecksum; + } + } + + { // CURRENT + results.emplace_back(); + LiveFileStorageInfo& info = results.back(); + + info.relative_filename = kCurrentFileName; + info.directory = GetName(); + info.file_type = kCurrentFile; + // CURRENT could be replaced so we have to record the contents as needed. + info.replacement_contents = manifest_fname + "\n"; + info.size = manifest_fname.size() + 1; + if (opts.include_checksum_info) { + info.file_checksum_func_name = kUnknownFileChecksumFuncName; + info.file_checksum = kUnknownFileChecksum; + } + } + + // The OPTIONS file number is zero in read-write mode when OPTIONS file + // writing failed and the DB was configured with + // `fail_if_options_file_error == false`. In read-only mode the OPTIONS file + // number is zero when no OPTIONS file exist at all. In those cases we do not + // record any OPTIONS file in the live file list. + if (options_number != 0) { + results.emplace_back(); + LiveFileStorageInfo& info = results.back(); + + info.relative_filename = OptionsFileName(options_number); + info.directory = GetName(); + info.file_number = options_number; + info.file_type = kOptionsFile; + info.size = options_size; + if (opts.include_checksum_info) { + info.file_checksum_func_name = kUnknownFileChecksumFuncName; + info.file_checksum = kUnknownFileChecksum; + } + } + + // Some legacy testing stuff TODO: carefully clean up obsolete parts + TEST_SYNC_POINT("CheckpointImpl::CreateCheckpoint:FlushDone"); + + TEST_SYNC_POINT("CheckpointImpl::CreateCheckpoint:SavedLiveFiles1"); + TEST_SYNC_POINT("CheckpointImpl::CreateCheckpoint:SavedLiveFiles2"); + + if (s.ok()) { + // To maximize the effectiveness of track_and_verify_wals_in_manifest, + // sync WAL when it is enabled. + s = FlushWAL( + immutable_db_options_.track_and_verify_wals_in_manifest /* sync */); + if (s.IsNotSupported()) { // read-only DB or similar + s = Status::OK(); + } + } + + TEST_SYNC_POINT("CheckpointImpl::CreateCustomCheckpoint:AfterGetLive1"); + TEST_SYNC_POINT("CheckpointImpl::CreateCustomCheckpoint:AfterGetLive2"); + + // If we have more than one column family, we also need to get WAL files. + if (s.ok()) { + s = GetSortedWalFiles(live_wal_files); + } + if (!s.ok()) { + return s; + } + + size_t wal_size = live_wal_files.size(); + + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "Number of log files %" ROCKSDB_PRIszt, live_wal_files.size()); + + // Link WAL files. Copy exact size of last one because it is the only one + // that has changes after the last flush. + auto wal_dir = immutable_db_options_.GetWalDir(); + for (size_t i = 0; s.ok() && i < wal_size; ++i) { + if ((live_wal_files[i]->Type() == kAliveLogFile) && + (!flush_memtable || live_wal_files[i]->LogNumber() >= min_log_num)) { + results.emplace_back(); + LiveFileStorageInfo& info = results.back(); + auto f = live_wal_files[i]->PathName(); + assert(!f.empty() && f[0] == '/'); + info.relative_filename = f.substr(1); + info.directory = wal_dir; + info.file_number = live_wal_files[i]->LogNumber(); + info.file_type = kWalFile; + info.size = live_wal_files[i]->SizeFileBytes(); + // Only last should need to be trimmed + info.trim_to_size = (i + 1 == wal_size); + if (opts.include_checksum_info) { + info.file_checksum_func_name = kUnknownFileChecksumFuncName; + info.file_checksum = kUnknownFileChecksum; + } + } + } + + if (s.ok()) { + // Only move results to output on success. + *files = std::move(results); + } + return s; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/db_flush_test.cc b/librocksdb-sys/rocksdb/db/db_flush_test.cc new file mode 100644 index 0000000..0b2e7ab --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_flush_test.cc @@ -0,0 +1,3202 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include + +#include "db/db_impl/db_impl.h" +#include "db/db_test_util.h" +#include "env/mock_env.h" +#include "file/filename.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/utilities/transaction_db.h" +#include "test_util/sync_point.h" +#include "test_util/testutil.h" +#include "util/cast_util.h" +#include "util/mutexlock.h" +#include "utilities/fault_injection_env.h" +#include "utilities/fault_injection_fs.h" + +namespace ROCKSDB_NAMESPACE { + +// This is a static filter used for filtering +// kvs during the compaction process. +static std::string NEW_VALUE = "NewValue"; + +class DBFlushTest : public DBTestBase { + public: + DBFlushTest() : DBTestBase("db_flush_test", /*env_do_fsync=*/true) {} +}; + +class DBFlushDirectIOTest : public DBFlushTest, + public ::testing::WithParamInterface { + public: + DBFlushDirectIOTest() : DBFlushTest() {} +}; + +class DBAtomicFlushTest : public DBFlushTest, + public ::testing::WithParamInterface { + public: + DBAtomicFlushTest() : DBFlushTest() {} +}; + +// We had issue when two background threads trying to flush at the same time, +// only one of them get committed. The test verifies the issue is fixed. +TEST_F(DBFlushTest, FlushWhileWritingManifest) { + Options options; + options.disable_auto_compactions = true; + options.max_background_flushes = 2; + options.env = env_; + Reopen(options); + FlushOptions no_wait; + no_wait.wait = false; + no_wait.allow_write_stall = true; + + SyncPoint::GetInstance()->LoadDependency( + {{"VersionSet::LogAndApply:WriteManifest", + "DBFlushTest::FlushWhileWritingManifest:1"}, + {"MemTableList::TryInstallMemtableFlushResults:InProgress", + "VersionSet::LogAndApply:WriteManifestDone"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put("foo", "v")); + ASSERT_OK(dbfull()->Flush(no_wait)); + TEST_SYNC_POINT("DBFlushTest::FlushWhileWritingManifest:1"); + ASSERT_OK(Put("bar", "v")); + ASSERT_OK(dbfull()->Flush(no_wait)); + // If the issue is hit we will wait here forever. + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(2, TotalTableFiles()); +} + +// Disable this test temporarily on Travis as it fails intermittently. +// Github issue: #4151 +TEST_F(DBFlushTest, SyncFail) { + std::unique_ptr fault_injection_env( + new FaultInjectionTestEnv(env_)); + Options options; + options.disable_auto_compactions = true; + options.env = fault_injection_env.get(); + + SyncPoint::GetInstance()->LoadDependency( + {{"DBFlushTest::SyncFail:1", "DBImpl::SyncClosedLogs:Start"}, + {"DBImpl::SyncClosedLogs:Failed", "DBFlushTest::SyncFail:2"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + CreateAndReopenWithCF({"pikachu"}, options); + ASSERT_OK(Put("key", "value")); + FlushOptions flush_options; + flush_options.wait = false; + ASSERT_OK(dbfull()->Flush(flush_options)); + // Flush installs a new super-version. Get the ref count after that. + fault_injection_env->SetFilesystemActive(false); + TEST_SYNC_POINT("DBFlushTest::SyncFail:1"); + TEST_SYNC_POINT("DBFlushTest::SyncFail:2"); + fault_injection_env->SetFilesystemActive(true); + // Now the background job will do the flush; wait for it. + // Returns the IO error happend during flush. + ASSERT_NOK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ("", FilesPerLevel()); // flush failed. + Destroy(options); +} + +TEST_F(DBFlushTest, SyncSkip) { + Options options = CurrentOptions(); + + SyncPoint::GetInstance()->LoadDependency( + {{"DBFlushTest::SyncSkip:1", "DBImpl::SyncClosedLogs:Skip"}, + {"DBImpl::SyncClosedLogs:Skip", "DBFlushTest::SyncSkip:2"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + Reopen(options); + ASSERT_OK(Put("key", "value")); + + FlushOptions flush_options; + flush_options.wait = false; + ASSERT_OK(dbfull()->Flush(flush_options)); + + TEST_SYNC_POINT("DBFlushTest::SyncSkip:1"); + TEST_SYNC_POINT("DBFlushTest::SyncSkip:2"); + + // Now the background job will do the flush; wait for it. + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + + Destroy(options); +} + +TEST_F(DBFlushTest, FlushInLowPriThreadPool) { + // Verify setting an empty high-pri (flush) thread pool causes flushes to be + // scheduled in the low-pri (compaction) thread pool. + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = 4; + options.memtable_factory.reset(test::NewSpecialSkipListFactory(1)); + Reopen(options); + env_->SetBackgroundThreads(0, Env::HIGH); + + std::thread::id tid; + int num_flushes = 0, num_compactions = 0; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BGWorkFlush", [&](void* /*arg*/) { + if (tid == std::thread::id()) { + tid = std::this_thread::get_id(); + } else { + ASSERT_EQ(tid, std::this_thread::get_id()); + } + ++num_flushes; + }); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BGWorkCompaction", [&](void* /*arg*/) { + ASSERT_EQ(tid, std::this_thread::get_id()); + ++num_compactions; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put("key", "val")); + for (int i = 0; i < 4; ++i) { + ASSERT_OK(Put("key", "val")); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(4, num_flushes); + ASSERT_EQ(1, num_compactions); +} + +// Test when flush job is submitted to low priority thread pool and when DB is +// closed in the meanwhile, CloseHelper doesn't hang. +TEST_F(DBFlushTest, CloseDBWhenFlushInLowPri) { + Options options = CurrentOptions(); + options.max_background_flushes = 1; + options.max_total_wal_size = 8192; + + DestroyAndReopen(options); + CreateColumnFamilies({"cf1", "cf2"}, options); + + env_->SetBackgroundThreads(0, Env::HIGH); + env_->SetBackgroundThreads(1, Env::LOW); + test::SleepingBackgroundTask sleeping_task_low; + int num_flushes = 0; + + SyncPoint::GetInstance()->SetCallBack("DBImpl::BGWorkFlush", + [&](void* /*arg*/) { ++num_flushes; }); + + int num_low_flush_unscheduled = 0; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::UnscheduleLowFlushCallback", [&](void* /*arg*/) { + num_low_flush_unscheduled++; + // There should be one flush job in low pool that needs to be + // unscheduled + ASSERT_EQ(num_low_flush_unscheduled, 1); + }); + + int num_high_flush_unscheduled = 0; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::UnscheduleHighFlushCallback", [&](void* /*arg*/) { + num_high_flush_unscheduled++; + // There should be no flush job in high pool + ASSERT_EQ(num_high_flush_unscheduled, 0); + }); + + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put(0, "key1", DummyString(8192))); + // Block thread so that flush cannot be run and can be removed from the queue + // when called Unschedule. + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::LOW); + sleeping_task_low.WaitUntilSleeping(); + + // Trigger flush and flush job will be scheduled to LOW priority thread. + ASSERT_OK(Put(0, "key2", DummyString(8192))); + + // Close DB and flush job in low priority queue will be removed without + // running. + Close(); + sleeping_task_low.WakeUp(); + sleeping_task_low.WaitUntilDone(); + ASSERT_EQ(0, num_flushes); + + TryReopenWithColumnFamilies({"default", "cf1", "cf2"}, options); + ASSERT_OK(Put(0, "key3", DummyString(8192))); + ASSERT_OK(Flush(0)); + ASSERT_EQ(1, num_flushes); +} + +TEST_F(DBFlushTest, ManualFlushWithMinWriteBufferNumberToMerge) { + Options options = CurrentOptions(); + options.write_buffer_size = 100; + options.max_write_buffer_number = 4; + options.min_write_buffer_number_to_merge = 3; + Reopen(options); + + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BGWorkFlush", + "DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge:1"}, + {"DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge:2", + "FlushJob::WriteLevel0Table"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put("key1", "value1")); + + port::Thread t([&]() { + // The call wait for flush to finish, i.e. with flush_options.wait = true. + ASSERT_OK(Flush()); + }); + + // Wait for flush start. + TEST_SYNC_POINT("DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge:1"); + // Insert a second memtable before the manual flush finish. + // At the end of the manual flush job, it will check if further flush + // is needed, but it will not trigger flush of the second memtable because + // min_write_buffer_number_to_merge is not reached. + ASSERT_OK(Put("key2", "value2")); + ASSERT_OK(dbfull()->TEST_SwitchMemtable()); + TEST_SYNC_POINT("DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge:2"); + + // Manual flush should return, without waiting for flush indefinitely. + t.join(); +} + +TEST_F(DBFlushTest, ScheduleOnlyOneBgThread) { + Options options = CurrentOptions(); + Reopen(options); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + int called = 0; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::MaybeScheduleFlushOrCompaction:AfterSchedule:0", [&](void* arg) { + ASSERT_NE(nullptr, arg); + auto unscheduled_flushes = *reinterpret_cast(arg); + ASSERT_EQ(0, unscheduled_flushes); + ++called; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put("a", "foo")); + FlushOptions flush_opts; + ASSERT_OK(dbfull()->Flush(flush_opts)); + ASSERT_EQ(1, called); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +// The following 3 tests are designed for testing garbage statistics at flush +// time. +// +// ======= General Information ======= (from GitHub Wiki). +// There are three scenarios where memtable flush can be triggered: +// +// 1 - Memtable size exceeds ColumnFamilyOptions::write_buffer_size +// after a write. +// 2 - Total memtable size across all column families exceeds +// DBOptions::db_write_buffer_size, +// or DBOptions::write_buffer_manager signals a flush. In this scenario +// the largest memtable will be flushed. +// 3 - Total WAL file size exceeds DBOptions::max_total_wal_size. +// In this scenario the memtable with the oldest data will be flushed, +// in order to allow the WAL file with data from this memtable to be +// purged. +// +// As a result, a memtable can be flushed before it is full. This is one +// reason the generated SST file can be smaller than the corresponding +// memtable. Compression is another factor to make SST file smaller than +// corresponding memtable, since data in memtable is uncompressed. + +TEST_F(DBFlushTest, StatisticsGarbageBasic) { + Options options = CurrentOptions(); + + // The following options are used to enforce several values that + // may already exist as default values to make this test resilient + // to default value updates in the future. + options.statistics = CreateDBStatistics(); + + // Record all statistics. + options.statistics->set_stats_level(StatsLevel::kAll); + + // create the DB if it's not already present + options.create_if_missing = true; + + // Useful for now as we are trying to compare uncompressed data savings on + // flush(). + options.compression = kNoCompression; + + // Prevent memtable in place updates. Should already be disabled + // (from Wiki: + // In place updates can be enabled by toggling on the bool + // inplace_update_support flag. However, this flag is by default set to + // false + // because this thread-safe in-place update support is not compatible + // with concurrent memtable writes. Note that the bool + // allow_concurrent_memtable_write is set to true by default ) + options.inplace_update_support = false; + options.allow_concurrent_memtable_write = true; + + // Enforce size of a single MemTable to 64MB (64MB = 67108864 bytes). + options.write_buffer_size = 64 << 20; + + ASSERT_OK(TryReopen(options)); + + // Put multiple times the same key-values. + // The encoded length of a db entry in the memtable is + // defined in db/memtable.cc (MemTable::Add) as the variable: + // encoded_len= VarintLength(internal_key_size) --> = + // log_256(internal_key). + // Min # of bytes + // necessary to + // store + // internal_key_size. + // + internal_key_size --> = actual key string, + // (size key_size: w/o term null char) + // + 8 bytes for + // fixed uint64 "seq + // number + // + + // insertion type" + // + VarintLength(val_size) --> = min # of bytes to + // store val_size + // + val_size --> = actual value + // string + // For example, in our situation, "key1" : size 4, "value1" : size 6 + // (the terminating null characters are not copied over to the memtable). + // And therefore encoded_len = 1 + (4+8) + 1 + 6 = 20 bytes per entry. + // However in terms of raw data contained in the memtable, and written + // over to the SSTable, we only count internal_key_size and val_size, + // because this is the only raw chunk of bytes that contains everything + // necessary to reconstruct a user entry: sequence number, insertion type, + // key, and value. + + // To test the relevance of our Memtable garbage statistics, + // namely MEMTABLE_PAYLOAD_BYTES_AT_FLUSH and MEMTABLE_GARBAGE_BYTES_AT_FLUSH, + // we insert K-V pairs with 3 distinct keys (of length 4), + // and random values of arbitrary length RAND_VALUES_LENGTH, + // and we repeat this step NUM_REPEAT times total. + // At the end, we insert 3 final K-V pairs with the same 3 keys + // and known values (these will be the final values, of length 6). + // I chose NUM_REPEAT=2,000 such that no automatic flush is + // triggered (the number of bytes in the memtable is therefore + // well below any meaningful heuristic for a memtable of size 64MB). + // As a result, since each K-V pair is inserted as a payload + // of N meaningful bytes (sequence number, insertion type, + // key, and value = 8 + 4 + RAND_VALUE_LENGTH), + // MEMTABLE_GARBAGE_BYTES_AT_FLUSH should be equal to 2,000 * N bytes + // and MEMTABLE_PAYLAOD_BYTES_AT_FLUSH = MEMTABLE_GARBAGE_BYTES_AT_FLUSH + + // (3*(8 + 4 + 6)) bytes. For RAND_VALUE_LENGTH = 172 (arbitrary value), we + // expect: + // N = 8 + 4 + 172 = 184 bytes + // MEMTABLE_GARBAGE_BYTES_AT_FLUSH = 2,000 * 184 = 368,000 bytes. + // MEMTABLE_PAYLOAD_BYTES_AT_FLUSH = 368,000 + 3*18 = 368,054 bytes. + + const size_t NUM_REPEAT = 2000; + const size_t RAND_VALUES_LENGTH = 172; + const std::string KEY1 = "key1"; + const std::string KEY2 = "key2"; + const std::string KEY3 = "key3"; + const std::string VALUE1 = "value1"; + const std::string VALUE2 = "value2"; + const std::string VALUE3 = "value3"; + uint64_t EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH = 0; + uint64_t EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH = 0; + + Random rnd(301); + // Insertion of of K-V pairs, multiple times. + for (size_t i = 0; i < NUM_REPEAT; i++) { + // Create value strings of arbitrary length RAND_VALUES_LENGTH bytes. + std::string p_v1 = rnd.RandomString(RAND_VALUES_LENGTH); + std::string p_v2 = rnd.RandomString(RAND_VALUES_LENGTH); + std::string p_v3 = rnd.RandomString(RAND_VALUES_LENGTH); + ASSERT_OK(Put(KEY1, p_v1)); + ASSERT_OK(Put(KEY2, p_v2)); + ASSERT_OK(Put(KEY3, p_v3)); + EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH += + KEY1.size() + p_v1.size() + sizeof(uint64_t); + EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH += + KEY2.size() + p_v2.size() + sizeof(uint64_t); + EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH += + KEY3.size() + p_v3.size() + sizeof(uint64_t); + } + + // The memtable data bytes includes the "garbage" + // bytes along with the useful payload. + EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH = + EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH; + + ASSERT_OK(Put(KEY1, VALUE1)); + ASSERT_OK(Put(KEY2, VALUE2)); + ASSERT_OK(Put(KEY3, VALUE3)); + + // Add useful payload to the memtable data bytes: + EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH += + KEY1.size() + VALUE1.size() + KEY2.size() + VALUE2.size() + KEY3.size() + + VALUE3.size() + 3 * sizeof(uint64_t); + + // We assert that the last K-V pairs have been successfully inserted, + // and that the valid values are VALUE1, VALUE2, VALUE3. + PinnableSlice value; + ASSERT_OK(Get(KEY1, &value)); + ASSERT_EQ(value.ToString(), VALUE1); + ASSERT_OK(Get(KEY2, &value)); + ASSERT_EQ(value.ToString(), VALUE2); + ASSERT_OK(Get(KEY3, &value)); + ASSERT_EQ(value.ToString(), VALUE3); + + // Force flush to SST. Increments the statistics counter. + ASSERT_OK(Flush()); + + // Collect statistics. + uint64_t mem_data_bytes = + TestGetTickerCount(options, MEMTABLE_PAYLOAD_BYTES_AT_FLUSH); + uint64_t mem_garbage_bytes = + TestGetTickerCount(options, MEMTABLE_GARBAGE_BYTES_AT_FLUSH); + + EXPECT_EQ(mem_data_bytes, EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH); + EXPECT_EQ(mem_garbage_bytes, EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH); + + Close(); +} + +TEST_F(DBFlushTest, StatisticsGarbageInsertAndDeletes) { + Options options = CurrentOptions(); + options.statistics = CreateDBStatistics(); + options.statistics->set_stats_level(StatsLevel::kAll); + options.create_if_missing = true; + options.compression = kNoCompression; + options.inplace_update_support = false; + options.allow_concurrent_memtable_write = true; + options.write_buffer_size = 67108864; + + ASSERT_OK(TryReopen(options)); + + const size_t NUM_REPEAT = 2000; + const size_t RAND_VALUES_LENGTH = 37; + const std::string KEY1 = "key1"; + const std::string KEY2 = "key2"; + const std::string KEY3 = "key3"; + const std::string KEY4 = "key4"; + const std::string KEY5 = "key5"; + const std::string KEY6 = "key6"; + + uint64_t EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH = 0; + uint64_t EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH = 0; + + WriteBatch batch; + + Random rnd(301); + // Insertion of of K-V pairs, multiple times. + for (size_t i = 0; i < NUM_REPEAT; i++) { + // Create value strings of arbitrary length RAND_VALUES_LENGTH bytes. + std::string p_v1 = rnd.RandomString(RAND_VALUES_LENGTH); + std::string p_v2 = rnd.RandomString(RAND_VALUES_LENGTH); + std::string p_v3 = rnd.RandomString(RAND_VALUES_LENGTH); + ASSERT_OK(Put(KEY1, p_v1)); + ASSERT_OK(Put(KEY2, p_v2)); + ASSERT_OK(Put(KEY3, p_v3)); + EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH += + KEY1.size() + p_v1.size() + sizeof(uint64_t); + EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH += + KEY2.size() + p_v2.size() + sizeof(uint64_t); + EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH += + KEY3.size() + p_v3.size() + sizeof(uint64_t); + ASSERT_OK(Delete(KEY1)); + ASSERT_OK(Delete(KEY2)); + ASSERT_OK(Delete(KEY3)); + EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH += + KEY1.size() + KEY2.size() + KEY3.size() + 3 * sizeof(uint64_t); + } + + // The memtable data bytes includes the "garbage" + // bytes along with the useful payload. + EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH = + EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH; + + // Note : one set of delete for KEY1, KEY2, KEY3 is written to + // SSTable to propagate the delete operations to K-V pairs + // that could have been inserted into the database during past Flush + // opeartions. + EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH -= + KEY1.size() + KEY2.size() + KEY3.size() + 3 * sizeof(uint64_t); + + // Additional useful paylaod. + ASSERT_OK(Delete(KEY4)); + ASSERT_OK(Delete(KEY5)); + ASSERT_OK(Delete(KEY6)); + + // // Add useful payload to the memtable data bytes: + EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH += + KEY4.size() + KEY5.size() + KEY6.size() + 3 * sizeof(uint64_t); + + // We assert that the K-V pairs have been successfully deleted. + PinnableSlice value; + ASSERT_NOK(Get(KEY1, &value)); + ASSERT_NOK(Get(KEY2, &value)); + ASSERT_NOK(Get(KEY3, &value)); + + // Force flush to SST. Increments the statistics counter. + ASSERT_OK(Flush()); + + // Collect statistics. + uint64_t mem_data_bytes = + TestGetTickerCount(options, MEMTABLE_PAYLOAD_BYTES_AT_FLUSH); + uint64_t mem_garbage_bytes = + TestGetTickerCount(options, MEMTABLE_GARBAGE_BYTES_AT_FLUSH); + + EXPECT_EQ(mem_data_bytes, EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH); + EXPECT_EQ(mem_garbage_bytes, EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH); + + Close(); +} + +TEST_F(DBFlushTest, StatisticsGarbageRangeDeletes) { + Options options = CurrentOptions(); + options.statistics = CreateDBStatistics(); + options.statistics->set_stats_level(StatsLevel::kAll); + options.create_if_missing = true; + options.compression = kNoCompression; + options.inplace_update_support = false; + options.allow_concurrent_memtable_write = true; + options.write_buffer_size = 67108864; + + ASSERT_OK(TryReopen(options)); + + const size_t NUM_REPEAT = 1000; + const size_t RAND_VALUES_LENGTH = 42; + const std::string KEY1 = "key1"; + const std::string KEY2 = "key2"; + const std::string KEY3 = "key3"; + const std::string KEY4 = "key4"; + const std::string KEY5 = "key5"; + const std::string KEY6 = "key6"; + const std::string VALUE3 = "value3"; + + uint64_t EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH = 0; + uint64_t EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH = 0; + + Random rnd(301); + // Insertion of of K-V pairs, multiple times. + // Also insert DeleteRange + for (size_t i = 0; i < NUM_REPEAT; i++) { + // Create value strings of arbitrary length RAND_VALUES_LENGTH bytes. + std::string p_v1 = rnd.RandomString(RAND_VALUES_LENGTH); + std::string p_v2 = rnd.RandomString(RAND_VALUES_LENGTH); + std::string p_v3 = rnd.RandomString(RAND_VALUES_LENGTH); + ASSERT_OK(Put(KEY1, p_v1)); + ASSERT_OK(Put(KEY2, p_v2)); + ASSERT_OK(Put(KEY3, p_v3)); + EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH += + KEY1.size() + p_v1.size() + sizeof(uint64_t); + EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH += + KEY2.size() + p_v2.size() + sizeof(uint64_t); + EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH += + KEY3.size() + p_v3.size() + sizeof(uint64_t); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), KEY1, + KEY2)); + // Note: DeleteRange have an exclusive upper bound, e.g. here: [KEY2,KEY3) + // is deleted. + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), KEY2, + KEY3)); + // Delete ranges are stored as a regular K-V pair, with key=STARTKEY, + // value=ENDKEY. + EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH += + (KEY1.size() + KEY2.size() + sizeof(uint64_t)) + + (KEY2.size() + KEY3.size() + sizeof(uint64_t)); + } + + // The memtable data bytes includes the "garbage" + // bytes along with the useful payload. + EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH = + EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH; + + // Note : one set of deleteRange for (KEY1, KEY2) and (KEY2, KEY3) is written + // to SSTable to propagate the deleteRange operations to K-V pairs that could + // have been inserted into the database during past Flush opeartions. + EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH -= + (KEY1.size() + KEY2.size() + sizeof(uint64_t)) + + (KEY2.size() + KEY3.size() + sizeof(uint64_t)); + + // Overwrite KEY3 with known value (VALUE3) + // Note that during the whole time KEY3 has never been deleted + // by the RangeDeletes. + ASSERT_OK(Put(KEY3, VALUE3)); + EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH += + KEY3.size() + VALUE3.size() + sizeof(uint64_t); + + // Additional useful paylaod. + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), KEY4, KEY5)); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), KEY5, KEY6)); + + // Add useful payload to the memtable data bytes: + EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH += + (KEY4.size() + KEY5.size() + sizeof(uint64_t)) + + (KEY5.size() + KEY6.size() + sizeof(uint64_t)); + + // We assert that the K-V pairs have been successfully deleted. + PinnableSlice value; + ASSERT_NOK(Get(KEY1, &value)); + ASSERT_NOK(Get(KEY2, &value)); + // And that KEY3's value is correct. + ASSERT_OK(Get(KEY3, &value)); + ASSERT_EQ(value, VALUE3); + + // Force flush to SST. Increments the statistics counter. + ASSERT_OK(Flush()); + + // Collect statistics. + uint64_t mem_data_bytes = + TestGetTickerCount(options, MEMTABLE_PAYLOAD_BYTES_AT_FLUSH); + uint64_t mem_garbage_bytes = + TestGetTickerCount(options, MEMTABLE_GARBAGE_BYTES_AT_FLUSH); + + EXPECT_EQ(mem_data_bytes, EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH); + EXPECT_EQ(mem_garbage_bytes, EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH); + + Close(); +} + +// This simple Listener can only handle one flush at a time. +class TestFlushListener : public EventListener { + public: + TestFlushListener(Env* env, DBFlushTest* test) + : slowdown_count(0), stop_count(0), db_closed(), env_(env), test_(test) { + db_closed = false; + } + + ~TestFlushListener() override { + prev_fc_info_.status.PermitUncheckedError(); // Ignore the status + } + + void OnTableFileCreated(const TableFileCreationInfo& info) override { + // remember the info for later checking the FlushJobInfo. + prev_fc_info_ = info; + ASSERT_GT(info.db_name.size(), 0U); + ASSERT_GT(info.cf_name.size(), 0U); + ASSERT_GT(info.file_path.size(), 0U); + ASSERT_GT(info.job_id, 0); + ASSERT_GT(info.table_properties.data_size, 0U); + ASSERT_GT(info.table_properties.raw_key_size, 0U); + ASSERT_GT(info.table_properties.raw_value_size, 0U); + ASSERT_GT(info.table_properties.num_data_blocks, 0U); + ASSERT_GT(info.table_properties.num_entries, 0U); + ASSERT_EQ(info.file_checksum, kUnknownFileChecksum); + ASSERT_EQ(info.file_checksum_func_name, kUnknownFileChecksumFuncName); + } + + void OnFlushCompleted(DB* db, const FlushJobInfo& info) override { + flushed_dbs_.push_back(db); + flushed_column_family_names_.push_back(info.cf_name); + if (info.triggered_writes_slowdown) { + slowdown_count++; + } + if (info.triggered_writes_stop) { + stop_count++; + } + // verify whether the previously created file matches the flushed file. + ASSERT_EQ(prev_fc_info_.db_name, db->GetName()); + ASSERT_EQ(prev_fc_info_.cf_name, info.cf_name); + ASSERT_EQ(prev_fc_info_.job_id, info.job_id); + ASSERT_EQ(prev_fc_info_.file_path, info.file_path); + ASSERT_EQ(TableFileNameToNumber(info.file_path), info.file_number); + + // Note: the following chunk relies on the notification pertaining to the + // database pointed to by DBTestBase::db_, and is thus bypassed when + // that assumption does not hold (see the test case MultiDBMultiListeners + // below). + ASSERT_TRUE(test_); + if (db == test_->db_) { + std::vector> files_by_level; + test_->dbfull()->TEST_GetFilesMetaData(db->DefaultColumnFamily(), + &files_by_level); + + ASSERT_FALSE(files_by_level.empty()); + auto it = std::find_if(files_by_level[0].begin(), files_by_level[0].end(), + [&](const FileMetaData& meta) { + return meta.fd.GetNumber() == info.file_number; + }); + ASSERT_NE(it, files_by_level[0].end()); + ASSERT_EQ(info.oldest_blob_file_number, it->oldest_blob_file_number); + } + + ASSERT_EQ(db->GetEnv()->GetThreadID(), info.thread_id); + ASSERT_GT(info.thread_id, 0U); + } + + std::vector flushed_column_family_names_; + std::vector flushed_dbs_; + int slowdown_count; + int stop_count; + bool db_closing; + std::atomic_bool db_closed; + TableFileCreationInfo prev_fc_info_; + + protected: + Env* env_; + DBFlushTest* test_; +}; + +TEST_F( + DBFlushTest, + FixUnrecoverableWriteDuringAtomicFlushWaitUntilFlushWouldNotStallWrites) { + Options options = CurrentOptions(); + options.atomic_flush = true; + + // To simulate a real-life crash where we can't flush during db's shutdown + options.avoid_flush_during_shutdown = true; + + // Set 3 low thresholds (while `disable_auto_compactions=false`) here so flush + // adding one more L0 file during `GetLiveFiles()` will have to wait till such + // flush will not stall writes + options.level0_stop_writes_trigger = 2; + options.level0_slowdown_writes_trigger = 2; + // Disable level-0 compaction triggered by number of files to avoid + // stalling check being skipped (resulting in the flush mentioned above didn't + // wait) + options.level0_file_num_compaction_trigger = -1; + + CreateAndReopenWithCF({"cf1"}, options); + + // Manually pause compaction thread to ensure enough L0 files as + // `disable_auto_compactions=false`is needed, in order to meet the 3 low + // thresholds above + std::unique_ptr sleeping_task_; + sleeping_task_.reset(new test::SleepingBackgroundTask()); + env_->SetBackgroundThreads(1, Env::LOW); + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, + sleeping_task_.get(), Env::Priority::LOW); + sleeping_task_->WaitUntilSleeping(); + + // Create some initial file to help meet the 3 low thresholds above + ASSERT_OK(Put(1, "dontcare", "dontcare")); + ASSERT_OK(Flush(1)); + + // Insert some initial data so we have something to atomic-flush later + // triggered by `GetLiveFiles()` + WriteOptions write_opts; + write_opts.disableWAL = true; + ASSERT_OK(Put(1, "k1", "v1", write_opts)); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({{ + "DBImpl::WaitUntilFlushWouldNotStallWrites:StallWait", + "DBFlushTest::" + "UnrecoverableWriteInAtomicFlushWaitUntilFlushWouldNotStallWrites::Write", + }}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Write to db when atomic flush releases the lock to wait on write stall + // condition to be gone in `WaitUntilFlushWouldNotStallWrites()` + port::Thread write_thread([&] { + TEST_SYNC_POINT( + "DBFlushTest::" + "UnrecoverableWriteInAtomicFlushWaitUntilFlushWouldNotStallWrites::" + "Write"); + // Before the fix, the empty default CF would've been prematurely excluded + // from this atomic flush. The following two writes together make default CF + // later contain data that should've been included in the atomic flush. + ASSERT_OK(Put(0, "k2", "v2", write_opts)); + // The following write increases the max seqno of this atomic flush to be 3, + // which is greater than the seqno of default CF's data. This then violates + // the invariant that all entries of seqno less than the max seqno + // of this atomic flush should've been flushed by the time of this atomic + // flush finishes. + ASSERT_OK(Put(1, "k3", "v3", write_opts)); + + // Resume compaction threads and reduce L0 files so `GetLiveFiles()` can + // resume from the wait + sleeping_task_->WakeUp(); + sleeping_task_->WaitUntilDone(); + MoveFilesToLevel(1, 1); + }); + + // Trigger an atomic flush by `GetLiveFiles()` + std::vector files; + uint64_t manifest_file_size; + ASSERT_OK(db_->GetLiveFiles(files, &manifest_file_size, /*flush*/ true)); + + write_thread.join(); + + ReopenWithColumnFamilies({"default", "cf1"}, options); + + ASSERT_EQ(Get(1, "k3"), "v3"); + // Prior to the fix, `Get()` will return `NotFound as "k2" entry in default CF + // can't be recovered from a crash right after the atomic flush finishes, + // resulting in a "recovery hole" as "k3" can be recovered. It's due to the + // invariant violation described above. + ASSERT_EQ(Get(0, "k2"), "v2"); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBFlushTest, FixFlushReasonRaceFromConcurrentFlushes) { + Options options = CurrentOptions(); + options.atomic_flush = true; + options.disable_auto_compactions = true; + CreateAndReopenWithCF({"cf1"}, options); + + for (int idx = 0; idx < 1; ++idx) { + ASSERT_OK(Put(0, Key(idx), std::string(1, 'v'))); + ASSERT_OK(Put(1, Key(idx), std::string(1, 'v'))); + } + + // To coerce a manual flush happenning in the middle of GetLiveFiles's flush, + // we need to pause background flush thread and enable it later. + std::shared_ptr sleeping_task = + std::make_shared(); + env_->SetBackgroundThreads(1, Env::HIGH); + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, + sleeping_task.get(), Env::Priority::HIGH); + sleeping_task->WaitUntilSleeping(); + + // Coerce a manual flush happenning in the middle of GetLiveFiles's flush + bool get_live_files_paused_at_sync_point = false; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::AtomicFlushMemTables:AfterScheduleFlush", [&](void* /* arg */) { + if (get_live_files_paused_at_sync_point) { + // To prevent non-GetLiveFiles() flush from pausing at this sync point + return; + } + get_live_files_paused_at_sync_point = true; + + FlushOptions fo; + fo.wait = false; + fo.allow_write_stall = true; + ASSERT_OK(dbfull()->Flush(fo)); + + // Resume background flush thread so GetLiveFiles() can finish + sleeping_task->WakeUp(); + sleeping_task->WaitUntilDone(); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + std::vector files; + uint64_t manifest_file_size; + // Before the fix, a race condition on default cf's flush reason due to + // concurrent GetLiveFiles's flush and manual flush will fail + // an internal assertion. + // After the fix, such race condition is fixed and there is no assertion + // failure. + ASSERT_OK(db_->GetLiveFiles(files, &manifest_file_size, /*flush*/ true)); + ASSERT_TRUE(get_live_files_paused_at_sync_point); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBFlushTest, MemPurgeBasic) { + Options options = CurrentOptions(); + + // The following options are used to enforce several values that + // may already exist as default values to make this test resilient + // to default value updates in the future. + options.statistics = CreateDBStatistics(); + + // Record all statistics. + options.statistics->set_stats_level(StatsLevel::kAll); + + // create the DB if it's not already present + options.create_if_missing = true; + + // Useful for now as we are trying to compare uncompressed data savings on + // flush(). + options.compression = kNoCompression; + + // Prevent memtable in place updates. Should already be disabled + // (from Wiki: + // In place updates can be enabled by toggling on the bool + // inplace_update_support flag. However, this flag is by default set to + // false + // because this thread-safe in-place update support is not compatible + // with concurrent memtable writes. Note that the bool + // allow_concurrent_memtable_write is set to true by default ) + options.inplace_update_support = false; + options.allow_concurrent_memtable_write = true; + + // Enforce size of a single MemTable to 64MB (64MB = 67108864 bytes). + options.write_buffer_size = 1 << 20; + // Initially deactivate the MemPurge prototype. + options.experimental_mempurge_threshold = 0.0; + TestFlushListener* listener = new TestFlushListener(options.env, this); + options.listeners.emplace_back(listener); + ASSERT_OK(TryReopen(options)); + + // RocksDB lite does not support dynamic options + // Dynamically activate the MemPurge prototype without restarting the DB. + ColumnFamilyHandle* cfh = db_->DefaultColumnFamily(); + ASSERT_OK(db_->SetOptions(cfh, {{"experimental_mempurge_threshold", "1.0"}})); + + std::atomic mempurge_count{0}; + std::atomic sst_count{0}; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::FlushJob:MemPurgeSuccessful", + [&](void* /*arg*/) { mempurge_count++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::FlushJob:SSTFileCreated", [&](void* /*arg*/) { sst_count++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + std::string KEY1 = "IamKey1"; + std::string KEY2 = "IamKey2"; + std::string KEY3 = "IamKey3"; + std::string KEY4 = "IamKey4"; + std::string KEY5 = "IamKey5"; + std::string KEY6 = "IamKey6"; + std::string KEY7 = "IamKey7"; + std::string KEY8 = "IamKey8"; + std::string KEY9 = "IamKey9"; + std::string RNDKEY1, RNDKEY2, RNDKEY3; + const std::string NOT_FOUND = "NOT_FOUND"; + + // Heavy overwrite workload, + // more than would fit in maximum allowed memtables. + Random rnd(719); + const size_t NUM_REPEAT = 100; + const size_t RAND_KEYS_LENGTH = 57; + const size_t RAND_VALUES_LENGTH = 10240; + std::string p_v1, p_v2, p_v3, p_v4, p_v5, p_v6, p_v7, p_v8, p_v9, p_rv1, + p_rv2, p_rv3; + + // Insert a very first set of keys that will be + // mempurged at least once. + p_v1 = rnd.RandomString(RAND_VALUES_LENGTH); + p_v2 = rnd.RandomString(RAND_VALUES_LENGTH); + p_v3 = rnd.RandomString(RAND_VALUES_LENGTH); + p_v4 = rnd.RandomString(RAND_VALUES_LENGTH); + ASSERT_OK(Put(KEY1, p_v1)); + ASSERT_OK(Put(KEY2, p_v2)); + ASSERT_OK(Put(KEY3, p_v3)); + ASSERT_OK(Put(KEY4, p_v4)); + ASSERT_EQ(Get(KEY1), p_v1); + ASSERT_EQ(Get(KEY2), p_v2); + ASSERT_EQ(Get(KEY3), p_v3); + ASSERT_EQ(Get(KEY4), p_v4); + + // Insertion of of K-V pairs, multiple times (overwrites). + for (size_t i = 0; i < NUM_REPEAT; i++) { + // Create value strings of arbitrary length RAND_VALUES_LENGTH bytes. + p_v5 = rnd.RandomString(RAND_VALUES_LENGTH); + p_v6 = rnd.RandomString(RAND_VALUES_LENGTH); + p_v7 = rnd.RandomString(RAND_VALUES_LENGTH); + p_v8 = rnd.RandomString(RAND_VALUES_LENGTH); + p_v9 = rnd.RandomString(RAND_VALUES_LENGTH); + + ASSERT_OK(Put(KEY5, p_v5)); + ASSERT_OK(Put(KEY6, p_v6)); + ASSERT_OK(Put(KEY7, p_v7)); + ASSERT_OK(Put(KEY8, p_v8)); + ASSERT_OK(Put(KEY9, p_v9)); + + ASSERT_EQ(Get(KEY1), p_v1); + ASSERT_EQ(Get(KEY2), p_v2); + ASSERT_EQ(Get(KEY3), p_v3); + ASSERT_EQ(Get(KEY4), p_v4); + ASSERT_EQ(Get(KEY5), p_v5); + ASSERT_EQ(Get(KEY6), p_v6); + ASSERT_EQ(Get(KEY7), p_v7); + ASSERT_EQ(Get(KEY8), p_v8); + ASSERT_EQ(Get(KEY9), p_v9); + } + + // Check that there was at least one mempurge + const uint32_t EXPECTED_MIN_MEMPURGE_COUNT = 1; + // Check that there was no SST files created during flush. + const uint32_t EXPECTED_SST_COUNT = 0; + + EXPECT_GE(mempurge_count.exchange(0), EXPECTED_MIN_MEMPURGE_COUNT); + EXPECT_EQ(sst_count.exchange(0), EXPECTED_SST_COUNT); + + // Insertion of of K-V pairs, no overwrites. + for (size_t i = 0; i < NUM_REPEAT; i++) { + // Create value strings of arbitrary length RAND_VALUES_LENGTH bytes. + RNDKEY1 = rnd.RandomString(RAND_KEYS_LENGTH); + RNDKEY2 = rnd.RandomString(RAND_KEYS_LENGTH); + RNDKEY3 = rnd.RandomString(RAND_KEYS_LENGTH); + p_rv1 = rnd.RandomString(RAND_VALUES_LENGTH); + p_rv2 = rnd.RandomString(RAND_VALUES_LENGTH); + p_rv3 = rnd.RandomString(RAND_VALUES_LENGTH); + + ASSERT_OK(Put(RNDKEY1, p_rv1)); + ASSERT_OK(Put(RNDKEY2, p_rv2)); + ASSERT_OK(Put(RNDKEY3, p_rv3)); + + ASSERT_EQ(Get(KEY1), p_v1); + ASSERT_EQ(Get(KEY2), p_v2); + ASSERT_EQ(Get(KEY3), p_v3); + ASSERT_EQ(Get(KEY4), p_v4); + ASSERT_EQ(Get(KEY5), p_v5); + ASSERT_EQ(Get(KEY6), p_v6); + ASSERT_EQ(Get(KEY7), p_v7); + ASSERT_EQ(Get(KEY8), p_v8); + ASSERT_EQ(Get(KEY9), p_v9); + ASSERT_EQ(Get(RNDKEY1), p_rv1); + ASSERT_EQ(Get(RNDKEY2), p_rv2); + ASSERT_EQ(Get(RNDKEY3), p_rv3); + } + + // Assert that at least one flush to storage has been performed + EXPECT_GT(sst_count.exchange(0), EXPECTED_SST_COUNT); + // (which will consequently increase the number of mempurges recorded too). + EXPECT_GE(mempurge_count.exchange(0), EXPECTED_MIN_MEMPURGE_COUNT); + + // Assert that there is no data corruption, even with + // a flush to storage. + ASSERT_EQ(Get(KEY1), p_v1); + ASSERT_EQ(Get(KEY2), p_v2); + ASSERT_EQ(Get(KEY3), p_v3); + ASSERT_EQ(Get(KEY4), p_v4); + ASSERT_EQ(Get(KEY5), p_v5); + ASSERT_EQ(Get(KEY6), p_v6); + ASSERT_EQ(Get(KEY7), p_v7); + ASSERT_EQ(Get(KEY8), p_v8); + ASSERT_EQ(Get(KEY9), p_v9); + ASSERT_EQ(Get(RNDKEY1), p_rv1); + ASSERT_EQ(Get(RNDKEY2), p_rv2); + ASSERT_EQ(Get(RNDKEY3), p_rv3); + + Close(); +} + +// RocksDB lite does not support dynamic options +TEST_F(DBFlushTest, MemPurgeBasicToggle) { + Options options = CurrentOptions(); + + // The following options are used to enforce several values that + // may already exist as default values to make this test resilient + // to default value updates in the future. + options.statistics = CreateDBStatistics(); + + // Record all statistics. + options.statistics->set_stats_level(StatsLevel::kAll); + + // create the DB if it's not already present + options.create_if_missing = true; + + // Useful for now as we are trying to compare uncompressed data savings on + // flush(). + options.compression = kNoCompression; + + // Prevent memtable in place updates. Should already be disabled + // (from Wiki: + // In place updates can be enabled by toggling on the bool + // inplace_update_support flag. However, this flag is by default set to + // false + // because this thread-safe in-place update support is not compatible + // with concurrent memtable writes. Note that the bool + // allow_concurrent_memtable_write is set to true by default ) + options.inplace_update_support = false; + options.allow_concurrent_memtable_write = true; + + // Enforce size of a single MemTable to 64MB (64MB = 67108864 bytes). + options.write_buffer_size = 1 << 20; + // Initially deactivate the MemPurge prototype. + // (negative values are equivalent to 0.0). + options.experimental_mempurge_threshold = -25.3; + TestFlushListener* listener = new TestFlushListener(options.env, this); + options.listeners.emplace_back(listener); + + ASSERT_OK(TryReopen(options)); + // Dynamically activate the MemPurge prototype without restarting the DB. + ColumnFamilyHandle* cfh = db_->DefaultColumnFamily(); + // Values greater than 1.0 are equivalent to 1.0 + ASSERT_OK( + db_->SetOptions(cfh, {{"experimental_mempurge_threshold", "3.7898"}})); + std::atomic mempurge_count{0}; + std::atomic sst_count{0}; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::FlushJob:MemPurgeSuccessful", + [&](void* /*arg*/) { mempurge_count++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::FlushJob:SSTFileCreated", [&](void* /*arg*/) { sst_count++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + const size_t KVSIZE = 3; + std::vector KEYS(KVSIZE); + for (size_t k = 0; k < KVSIZE; k++) { + KEYS[k] = "IamKey" + std::to_string(k); + } + + std::vector RNDVALS(KVSIZE); + const std::string NOT_FOUND = "NOT_FOUND"; + + // Heavy overwrite workload, + // more than would fit in maximum allowed memtables. + Random rnd(719); + const size_t NUM_REPEAT = 100; + const size_t RAND_VALUES_LENGTH = 10240; + + // Insertion of of K-V pairs, multiple times (overwrites). + for (size_t i = 0; i < NUM_REPEAT; i++) { + for (size_t j = 0; j < KEYS.size(); j++) { + RNDVALS[j] = rnd.RandomString(RAND_VALUES_LENGTH); + ASSERT_OK(Put(KEYS[j], RNDVALS[j])); + ASSERT_EQ(Get(KEYS[j]), RNDVALS[j]); + } + for (size_t j = 0; j < KEYS.size(); j++) { + ASSERT_EQ(Get(KEYS[j]), RNDVALS[j]); + } + } + + // Check that there was at least one mempurge + const uint32_t EXPECTED_MIN_MEMPURGE_COUNT = 1; + // Check that there was no SST files created during flush. + const uint32_t EXPECTED_SST_COUNT = 0; + + EXPECT_GE(mempurge_count.exchange(0), EXPECTED_MIN_MEMPURGE_COUNT); + EXPECT_EQ(sst_count.exchange(0), EXPECTED_SST_COUNT); + + // Dynamically deactivate MemPurge. + ASSERT_OK( + db_->SetOptions(cfh, {{"experimental_mempurge_threshold", "-1023.0"}})); + + // Insertion of of K-V pairs, multiple times (overwrites). + for (size_t i = 0; i < NUM_REPEAT; i++) { + for (size_t j = 0; j < KEYS.size(); j++) { + RNDVALS[j] = rnd.RandomString(RAND_VALUES_LENGTH); + ASSERT_OK(Put(KEYS[j], RNDVALS[j])); + ASSERT_EQ(Get(KEYS[j]), RNDVALS[j]); + } + for (size_t j = 0; j < KEYS.size(); j++) { + ASSERT_EQ(Get(KEYS[j]), RNDVALS[j]); + } + } + + // Check that there was at least one mempurge + const uint32_t ZERO = 0; + // Assert that at least one flush to storage has been performed + EXPECT_GT(sst_count.exchange(0), EXPECTED_SST_COUNT); + // The mempurge count is expected to be set to 0 when the options are updated. + // We expect no mempurge at all. + EXPECT_EQ(mempurge_count.exchange(0), ZERO); + + Close(); +} +// End of MemPurgeBasicToggle, which is not +// supported with RocksDB LITE because it +// relies on dynamically changing the option +// flag experimental_mempurge_threshold. + +// At the moment, MemPurge feature is deactivated +// when atomic_flush is enabled. This is because the level +// of garbage between Column Families is not guaranteed to +// be consistent, therefore a CF could hypothetically +// trigger a MemPurge while another CF would trigger +// a regular Flush. +TEST_F(DBFlushTest, MemPurgeWithAtomicFlush) { + Options options = CurrentOptions(); + + // The following options are used to enforce several values that + // may already exist as default values to make this test resilient + // to default value updates in the future. + options.statistics = CreateDBStatistics(); + + // Record all statistics. + options.statistics->set_stats_level(StatsLevel::kAll); + + // create the DB if it's not already present + options.create_if_missing = true; + + // Useful for now as we are trying to compare uncompressed data savings on + // flush(). + options.compression = kNoCompression; + + // Prevent memtable in place updates. Should already be disabled + // (from Wiki: + // In place updates can be enabled by toggling on the bool + // inplace_update_support flag. However, this flag is by default set to + // false + // because this thread-safe in-place update support is not compatible + // with concurrent memtable writes. Note that the bool + // allow_concurrent_memtable_write is set to true by default ) + options.inplace_update_support = false; + options.allow_concurrent_memtable_write = true; + + // Enforce size of a single MemTable to 64KB (64KB = 65,536 bytes). + options.write_buffer_size = 1 << 20; + // Activate the MemPurge prototype. + options.experimental_mempurge_threshold = 153.245; + // Activate atomic_flush. + options.atomic_flush = true; + + const std::vector new_cf_names = {"pikachu", "eevie"}; + CreateColumnFamilies(new_cf_names, options); + + Close(); + + // 3 CFs: default will be filled with overwrites (would normally trigger + // mempurge) + // new_cf_names[1] will be filled with random values (would trigger + // flush) new_cf_names[2] not filled with anything. + ReopenWithColumnFamilies( + {kDefaultColumnFamilyName, new_cf_names[0], new_cf_names[1]}, options); + size_t num_cfs = handles_.size(); + ASSERT_EQ(3, num_cfs); + ASSERT_OK(Put(1, "foo", "bar")); + ASSERT_OK(Put(2, "bar", "baz")); + + std::atomic mempurge_count{0}; + std::atomic sst_count{0}; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::FlushJob:MemPurgeSuccessful", + [&](void* /*arg*/) { mempurge_count++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::FlushJob:SSTFileCreated", [&](void* /*arg*/) { sst_count++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + const size_t KVSIZE = 3; + std::vector KEYS(KVSIZE); + for (size_t k = 0; k < KVSIZE; k++) { + KEYS[k] = "IamKey" + std::to_string(k); + } + + std::string RNDKEY; + std::vector RNDVALS(KVSIZE); + const std::string NOT_FOUND = "NOT_FOUND"; + + // Heavy overwrite workload, + // more than would fit in maximum allowed memtables. + Random rnd(106); + const size_t NUM_REPEAT = 100; + const size_t RAND_KEY_LENGTH = 128; + const size_t RAND_VALUES_LENGTH = 10240; + + // Insertion of of K-V pairs, multiple times (overwrites). + for (size_t i = 0; i < NUM_REPEAT; i++) { + for (size_t j = 0; j < KEYS.size(); j++) { + RNDKEY = rnd.RandomString(RAND_KEY_LENGTH); + RNDVALS[j] = rnd.RandomString(RAND_VALUES_LENGTH); + ASSERT_OK(Put(KEYS[j], RNDVALS[j])); + ASSERT_OK(Put(1, RNDKEY, RNDVALS[j])); + ASSERT_EQ(Get(KEYS[j]), RNDVALS[j]); + ASSERT_EQ(Get(1, RNDKEY), RNDVALS[j]); + } + } + + // Check that there was no mempurge because atomic_flush option is true. + const uint32_t EXPECTED_MIN_MEMPURGE_COUNT = 0; + // Check that there was at least one SST files created during flush. + const uint32_t EXPECTED_SST_COUNT = 1; + + EXPECT_EQ(mempurge_count.exchange(0), EXPECTED_MIN_MEMPURGE_COUNT); + EXPECT_GE(sst_count.exchange(0), EXPECTED_SST_COUNT); + + Close(); +} + +TEST_F(DBFlushTest, MemPurgeDeleteAndDeleteRange) { + Options options = CurrentOptions(); + + options.statistics = CreateDBStatistics(); + options.statistics->set_stats_level(StatsLevel::kAll); + options.create_if_missing = true; + options.compression = kNoCompression; + options.inplace_update_support = false; + options.allow_concurrent_memtable_write = true; + TestFlushListener* listener = new TestFlushListener(options.env, this); + options.listeners.emplace_back(listener); + // Enforce size of a single MemTable to 64MB (64MB = 67108864 bytes). + options.write_buffer_size = 1 << 20; + // Activate the MemPurge prototype. + options.experimental_mempurge_threshold = 15.0; + + ASSERT_OK(TryReopen(options)); + + std::atomic mempurge_count{0}; + std::atomic sst_count{0}; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::FlushJob:MemPurgeSuccessful", + [&](void* /*arg*/) { mempurge_count++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::FlushJob:SSTFileCreated", [&](void* /*arg*/) { sst_count++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + std::string KEY1 = "ThisIsKey1"; + std::string KEY2 = "ThisIsKey2"; + std::string KEY3 = "ThisIsKey3"; + std::string KEY4 = "ThisIsKey4"; + std::string KEY5 = "ThisIsKey5"; + const std::string NOT_FOUND = "NOT_FOUND"; + + Random rnd(117); + const size_t NUM_REPEAT = 100; + const size_t RAND_VALUES_LENGTH = 10240; + + std::string key, value, p_v1, p_v2, p_v3, p_v3b, p_v4, p_v5; + int count = 0; + const int EXPECTED_COUNT_FORLOOP = 3; + const int EXPECTED_COUNT_END = 4; + + ReadOptions ropt; + ropt.pin_data = true; + ropt.total_order_seek = true; + Iterator* iter = nullptr; + + // Insertion of of K-V pairs, multiple times. + // Also insert DeleteRange + for (size_t i = 0; i < NUM_REPEAT; i++) { + // Create value strings of arbitrary length RAND_VALUES_LENGTH bytes. + p_v1 = rnd.RandomString(RAND_VALUES_LENGTH); + p_v2 = rnd.RandomString(RAND_VALUES_LENGTH); + p_v3 = rnd.RandomString(RAND_VALUES_LENGTH); + p_v3b = rnd.RandomString(RAND_VALUES_LENGTH); + p_v4 = rnd.RandomString(RAND_VALUES_LENGTH); + p_v5 = rnd.RandomString(RAND_VALUES_LENGTH); + ASSERT_OK(Put(KEY1, p_v1)); + ASSERT_OK(Put(KEY2, p_v2)); + ASSERT_OK(Put(KEY3, p_v3)); + ASSERT_OK(Put(KEY4, p_v4)); + ASSERT_OK(Put(KEY5, p_v5)); + ASSERT_OK(Delete(KEY2)); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), KEY2, + KEY4)); + ASSERT_OK(Put(KEY3, p_v3b)); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), KEY1, + KEY3)); + ASSERT_OK(Delete(KEY1)); + + ASSERT_EQ(Get(KEY1), NOT_FOUND); + ASSERT_EQ(Get(KEY2), NOT_FOUND); + ASSERT_EQ(Get(KEY3), p_v3b); + ASSERT_EQ(Get(KEY4), p_v4); + ASSERT_EQ(Get(KEY5), p_v5); + + iter = db_->NewIterator(ropt); + iter->SeekToFirst(); + count = 0; + for (; iter->Valid(); iter->Next()) { + ASSERT_OK(iter->status()); + key = (iter->key()).ToString(false); + value = (iter->value()).ToString(false); + if (key.compare(KEY3) == 0) + ASSERT_EQ(value, p_v3b); + else if (key.compare(KEY4) == 0) + ASSERT_EQ(value, p_v4); + else if (key.compare(KEY5) == 0) + ASSERT_EQ(value, p_v5); + else + ASSERT_EQ(value, NOT_FOUND); + count++; + } + + // Expected count here is 3: KEY3, KEY4, KEY5. + ASSERT_EQ(count, EXPECTED_COUNT_FORLOOP); + if (iter) { + delete iter; + } + } + + // Check that there was at least one mempurge + const uint32_t EXPECTED_MIN_MEMPURGE_COUNT = 1; + // Check that there was no SST files created during flush. + const uint32_t EXPECTED_SST_COUNT = 0; + + EXPECT_GE(mempurge_count.exchange(0), EXPECTED_MIN_MEMPURGE_COUNT); + EXPECT_EQ(sst_count.exchange(0), EXPECTED_SST_COUNT); + + // Additional test for the iterator+memPurge. + ASSERT_OK(Put(KEY2, p_v2)); + iter = db_->NewIterator(ropt); + iter->SeekToFirst(); + ASSERT_OK(Put(KEY4, p_v4)); + count = 0; + for (; iter->Valid(); iter->Next()) { + ASSERT_OK(iter->status()); + key = (iter->key()).ToString(false); + value = (iter->value()).ToString(false); + if (key.compare(KEY2) == 0) + ASSERT_EQ(value, p_v2); + else if (key.compare(KEY3) == 0) + ASSERT_EQ(value, p_v3b); + else if (key.compare(KEY4) == 0) + ASSERT_EQ(value, p_v4); + else if (key.compare(KEY5) == 0) + ASSERT_EQ(value, p_v5); + else + ASSERT_EQ(value, NOT_FOUND); + count++; + } + + // Expected count here is 4: KEY2, KEY3, KEY4, KEY5. + ASSERT_EQ(count, EXPECTED_COUNT_END); + if (iter) delete iter; + + Close(); +} + +// Create a Compaction Fitler that will be invoked +// at flush time and will update the value of a KV pair +// if the key string is "lower" than the filter_key_ string. +class ConditionalUpdateFilter : public CompactionFilter { + public: + explicit ConditionalUpdateFilter(const std::string* filtered_key) + : filtered_key_(filtered_key) {} + bool Filter(int /*level*/, const Slice& key, const Slice& /*value*/, + std::string* new_value, bool* value_changed) const override { + // If key CreateCompactionFilter( + const CompactionFilter::Context& /*context*/) override { + return std::unique_ptr( + new ConditionalUpdateFilter(&filtered_key_)); + } + + const char* Name() const override { return "ConditionalUpdateFilterFactory"; } + + bool ShouldFilterTableFileCreation( + TableFileCreationReason reason) const override { + // This compaction filter will be invoked + // at flush time (and therefore at MemPurge time). + return (reason == TableFileCreationReason::kFlush); + } + + private: + std::string filtered_key_; +}; + +TEST_F(DBFlushTest, MemPurgeAndCompactionFilter) { + Options options = CurrentOptions(); + + std::string KEY1 = "ThisIsKey1"; + std::string KEY2 = "ThisIsKey2"; + std::string KEY3 = "ThisIsKey3"; + std::string KEY4 = "ThisIsKey4"; + std::string KEY5 = "ThisIsKey5"; + std::string KEY6 = "ThisIsKey6"; + std::string KEY7 = "ThisIsKey7"; + std::string KEY8 = "ThisIsKey8"; + std::string KEY9 = "ThisIsKey9"; + const std::string NOT_FOUND = "NOT_FOUND"; + + options.statistics = CreateDBStatistics(); + options.statistics->set_stats_level(StatsLevel::kAll); + options.create_if_missing = true; + options.compression = kNoCompression; + options.inplace_update_support = false; + options.allow_concurrent_memtable_write = true; + TestFlushListener* listener = new TestFlushListener(options.env, this); + options.listeners.emplace_back(listener); + // Create a ConditionalUpdate compaction filter + // that will update all the values of the KV pairs + // where the keys are "lower" than KEY4. + options.compaction_filter_factory = + std::make_shared(KEY4); + + // Enforce size of a single MemTable to 64MB (64MB = 67108864 bytes). + options.write_buffer_size = 1 << 20; + // Activate the MemPurge prototype. + options.experimental_mempurge_threshold = 26.55; + + ASSERT_OK(TryReopen(options)); + + std::atomic mempurge_count{0}; + std::atomic sst_count{0}; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::FlushJob:MemPurgeSuccessful", + [&](void* /*arg*/) { mempurge_count++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::FlushJob:SSTFileCreated", [&](void* /*arg*/) { sst_count++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Random rnd(53); + const size_t NUM_REPEAT = 1000; + const size_t RAND_VALUES_LENGTH = 10240; + std::string p_v1, p_v2, p_v3, p_v4, p_v5, p_v6, p_v7, p_v8, p_v9; + + p_v1 = rnd.RandomString(RAND_VALUES_LENGTH); + p_v2 = rnd.RandomString(RAND_VALUES_LENGTH); + p_v3 = rnd.RandomString(RAND_VALUES_LENGTH); + p_v4 = rnd.RandomString(RAND_VALUES_LENGTH); + p_v5 = rnd.RandomString(RAND_VALUES_LENGTH); + ASSERT_OK(Put(KEY1, p_v1)); + ASSERT_OK(Put(KEY2, p_v2)); + ASSERT_OK(Put(KEY3, p_v3)); + ASSERT_OK(Put(KEY4, p_v4)); + ASSERT_OK(Put(KEY5, p_v5)); + ASSERT_OK(Delete(KEY1)); + + // Insertion of of K-V pairs, multiple times. + for (size_t i = 0; i < NUM_REPEAT; i++) { + // Create value strings of arbitrary + // length RAND_VALUES_LENGTH bytes. + p_v6 = rnd.RandomString(RAND_VALUES_LENGTH); + p_v7 = rnd.RandomString(RAND_VALUES_LENGTH); + p_v8 = rnd.RandomString(RAND_VALUES_LENGTH); + p_v9 = rnd.RandomString(RAND_VALUES_LENGTH); + ASSERT_OK(Put(KEY6, p_v6)); + ASSERT_OK(Put(KEY7, p_v7)); + ASSERT_OK(Put(KEY8, p_v8)); + ASSERT_OK(Put(KEY9, p_v9)); + + ASSERT_OK(Delete(KEY7)); + } + + // Check that there was at least one mempurge + const uint32_t EXPECTED_MIN_MEMPURGE_COUNT = 1; + // Check that there was no SST files created during flush. + const uint32_t EXPECTED_SST_COUNT = 0; + + EXPECT_GE(mempurge_count.exchange(0), EXPECTED_MIN_MEMPURGE_COUNT); + EXPECT_EQ(sst_count.exchange(0), EXPECTED_SST_COUNT); + + // Verify that the ConditionalUpdateCompactionFilter + // updated the values of KEY2 and KEY3, and not KEY4 and KEY5. + ASSERT_EQ(Get(KEY1), NOT_FOUND); + ASSERT_EQ(Get(KEY2), NEW_VALUE); + ASSERT_EQ(Get(KEY3), NEW_VALUE); + ASSERT_EQ(Get(KEY4), p_v4); + ASSERT_EQ(Get(KEY5), p_v5); +} + +TEST_F(DBFlushTest, DISABLED_MemPurgeWALSupport) { + Options options = CurrentOptions(); + + options.statistics = CreateDBStatistics(); + options.statistics->set_stats_level(StatsLevel::kAll); + options.create_if_missing = true; + options.compression = kNoCompression; + options.inplace_update_support = false; + options.allow_concurrent_memtable_write = true; + + // Enforce size of a single MemTable to 128KB. + options.write_buffer_size = 128 << 10; + // Activate the MemPurge prototype + // (values >1.0 are equivalent to 1.0). + options.experimental_mempurge_threshold = 2.5; + + ASSERT_OK(TryReopen(options)); + + const size_t KVSIZE = 10; + + do { + CreateAndReopenWithCF({"pikachu"}, options); + ASSERT_OK(Put(1, "foo", "v1")); + ASSERT_OK(Put(1, "baz", "v5")); + + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_EQ("v1", Get(1, "foo")); + + ASSERT_EQ("v1", Get(1, "foo")); + ASSERT_EQ("v5", Get(1, "baz")); + ASSERT_OK(Put(0, "bar", "v2")); + ASSERT_OK(Put(1, "bar", "v2")); + ASSERT_OK(Put(1, "foo", "v3")); + std::atomic mempurge_count{0}; + std::atomic sst_count{0}; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::FlushJob:MemPurgeSuccessful", + [&](void* /*arg*/) { mempurge_count++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::FlushJob:SSTFileCreated", [&](void* /*arg*/) { sst_count++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + std::vector keys; + for (size_t k = 0; k < KVSIZE; k++) { + keys.push_back("IamKey" + std::to_string(k)); + } + + std::string RNDKEY, RNDVALUE; + const std::string NOT_FOUND = "NOT_FOUND"; + + // Heavy overwrite workload, + // more than would fit in maximum allowed memtables. + Random rnd(719); + const size_t NUM_REPEAT = 100; + const size_t RAND_KEY_LENGTH = 4096; + const size_t RAND_VALUES_LENGTH = 1024; + std::vector values_default(KVSIZE), values_pikachu(KVSIZE); + + // Insert a very first set of keys that will be + // mempurged at least once. + for (size_t k = 0; k < KVSIZE / 2; k++) { + values_default[k] = rnd.RandomString(RAND_VALUES_LENGTH); + values_pikachu[k] = rnd.RandomString(RAND_VALUES_LENGTH); + } + + // Insert keys[0:KVSIZE/2] to + // both 'default' and 'pikachu' CFs. + for (size_t k = 0; k < KVSIZE / 2; k++) { + ASSERT_OK(Put(0, keys[k], values_default[k])); + ASSERT_OK(Put(1, keys[k], values_pikachu[k])); + } + + // Check that the insertion was seamless. + for (size_t k = 0; k < KVSIZE / 2; k++) { + ASSERT_EQ(Get(0, keys[k]), values_default[k]); + ASSERT_EQ(Get(1, keys[k]), values_pikachu[k]); + } + + // Insertion of of K-V pairs, multiple times (overwrites) + // into 'default' CF. Will trigger mempurge. + for (size_t j = 0; j < NUM_REPEAT; j++) { + // Create value strings of arbitrary length RAND_VALUES_LENGTH bytes. + for (size_t k = KVSIZE / 2; k < KVSIZE; k++) { + values_default[k] = rnd.RandomString(RAND_VALUES_LENGTH); + } + + // Insert K-V into default CF. + for (size_t k = KVSIZE / 2; k < KVSIZE; k++) { + ASSERT_OK(Put(0, keys[k], values_default[k])); + } + + // Check key validity, for all keys, both in + // default and pikachu CFs. + for (size_t k = 0; k < KVSIZE; k++) { + ASSERT_EQ(Get(0, keys[k]), values_default[k]); + } + // Note that at this point, only keys[0:KVSIZE/2] + // have been inserted into Pikachu. + for (size_t k = 0; k < KVSIZE / 2; k++) { + ASSERT_EQ(Get(1, keys[k]), values_pikachu[k]); + } + } + + // Insertion of of K-V pairs, multiple times (overwrites) + // into 'pikachu' CF. Will trigger mempurge. + // Check that we keep the older logs for 'default' imm(). + for (size_t j = 0; j < NUM_REPEAT; j++) { + // Create value strings of arbitrary length RAND_VALUES_LENGTH bytes. + for (size_t k = KVSIZE / 2; k < KVSIZE; k++) { + values_pikachu[k] = rnd.RandomString(RAND_VALUES_LENGTH); + } + + // Insert K-V into pikachu CF. + for (size_t k = KVSIZE / 2; k < KVSIZE; k++) { + ASSERT_OK(Put(1, keys[k], values_pikachu[k])); + } + + // Check key validity, for all keys, + // both in default and pikachu. + for (size_t k = 0; k < KVSIZE; k++) { + ASSERT_EQ(Get(0, keys[k]), values_default[k]); + ASSERT_EQ(Get(1, keys[k]), values_pikachu[k]); + } + } + + // Check that there was at least one mempurge + const uint32_t EXPECTED_MIN_MEMPURGE_COUNT = 1; + // Check that there was no SST files created during flush. + const uint32_t EXPECTED_SST_COUNT = 0; + + EXPECT_GE(mempurge_count.exchange(0), EXPECTED_MIN_MEMPURGE_COUNT); + if (options.experimental_mempurge_threshold == + std::numeric_limits::max()) { + EXPECT_EQ(sst_count.exchange(0), EXPECTED_SST_COUNT); + } + + ReopenWithColumnFamilies({"default", "pikachu"}, options); + // Check that there was no data corruption anywhere, + // not in 'default' nor in 'Pikachu' CFs. + ASSERT_EQ("v3", Get(1, "foo")); + ASSERT_OK(Put(1, "foo", "v4")); + ASSERT_EQ("v4", Get(1, "foo")); + ASSERT_EQ("v2", Get(1, "bar")); + ASSERT_EQ("v5", Get(1, "baz")); + // Check keys in 'Default' and 'Pikachu'. + // keys[0:KVSIZE/2] were for sure contained + // in the imm() at Reopen/recovery time. + for (size_t k = 0; k < KVSIZE; k++) { + ASSERT_EQ(Get(0, keys[k]), values_default[k]); + ASSERT_EQ(Get(1, keys[k]), values_pikachu[k]); + } + // Insertion of random K-V pairs to trigger + // a flush in the Pikachu CF. + for (size_t j = 0; j < NUM_REPEAT; j++) { + RNDKEY = rnd.RandomString(RAND_KEY_LENGTH); + RNDVALUE = rnd.RandomString(RAND_VALUES_LENGTH); + ASSERT_OK(Put(1, RNDKEY, RNDVALUE)); + } + // ASsert than there was at least one flush to storage. + EXPECT_GT(sst_count.exchange(0), EXPECTED_SST_COUNT); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_EQ("v4", Get(1, "foo")); + ASSERT_EQ("v2", Get(1, "bar")); + ASSERT_EQ("v5", Get(1, "baz")); + // Since values in default are held in mutable mem() + // and imm(), check if the flush in pikachu didn't + // affect these values. + for (size_t k = 0; k < KVSIZE; k++) { + ASSERT_EQ(Get(0, keys[k]), values_default[k]); + ASSERT_EQ(Get(1, keys[k]), values_pikachu[k]); + } + ASSERT_EQ(Get(1, RNDKEY), RNDVALUE); + } while (ChangeWalOptions()); +} + +TEST_F(DBFlushTest, MemPurgeCorrectLogNumberAndSSTFileCreation) { + // Before our bug fix, we noticed that when 2 memtables were + // being flushed (with one memtable being the output of a + // previous MemPurge and one memtable being a newly-sealed memtable), + // the SST file created was not properly added to the DB version + // (via the VersionEdit obj), leading to data loss (the SST file + // was later being purged as an obsolete file). + // Therefore, we reproduce this scenario to test our fix. + Options options = CurrentOptions(); + + options.create_if_missing = true; + options.compression = kNoCompression; + options.inplace_update_support = false; + options.allow_concurrent_memtable_write = true; + + // Enforce size of a single MemTable to 1MB (64MB = 1048576 bytes). + options.write_buffer_size = 1 << 20; + // Activate the MemPurge prototype. + options.experimental_mempurge_threshold = 1.0; + + // Force to have more than one memtable to trigger a flush. + // For some reason this option does not seem to be enforced, + // so the following test is designed to make sure that we + // are testing the correct test case. + options.min_write_buffer_number_to_merge = 3; + options.max_write_buffer_number = 5; + options.max_write_buffer_size_to_maintain = 2 * (options.write_buffer_size); + options.disable_auto_compactions = true; + ASSERT_OK(TryReopen(options)); + + std::atomic mempurge_count{0}; + std::atomic sst_count{0}; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::FlushJob:MemPurgeSuccessful", + [&](void* /*arg*/) { mempurge_count++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::FlushJob:SSTFileCreated", [&](void* /*arg*/) { sst_count++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Dummy variable used for the following callback function. + uint64_t ZERO = 0; + // We will first execute mempurge operations exclusively. + // Therefore, when the first flush is triggered, we want to make + // sure there is at least 2 memtables being flushed: one output + // from a previous mempurge, and one newly sealed memtable. + // This is when we observed in the past that some SST files created + // were not properly added to the DB version (via the VersionEdit obj). + std::atomic num_memtable_at_first_flush(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "FlushJob::WriteLevel0Table:num_memtables", [&](void* arg) { + uint64_t* mems_size = reinterpret_cast(arg); + // atomic_compare_exchange_strong sometimes updates the value + // of ZERO (the "expected" object), so we make sure ZERO is indeed... + // zero. + ZERO = 0; + std::atomic_compare_exchange_strong(&num_memtable_at_first_flush, &ZERO, + *mems_size); + }); + + const std::vector KEYS = { + "ThisIsKey1", "ThisIsKey2", "ThisIsKey3", "ThisIsKey4", "ThisIsKey5", + "ThisIsKey6", "ThisIsKey7", "ThisIsKey8", "ThisIsKey9"}; + const std::string NOT_FOUND = "NOT_FOUND"; + + Random rnd(117); + const uint64_t NUM_REPEAT_OVERWRITES = 100; + const uint64_t NUM_RAND_INSERTS = 500; + const uint64_t RAND_VALUES_LENGTH = 10240; + + std::string key, value; + std::vector values(9, ""); + + // Keys used to check that no SST file disappeared. + for (uint64_t k = 0; k < 5; k++) { + values[k] = rnd.RandomString(RAND_VALUES_LENGTH); + ASSERT_OK(Put(KEYS[k], values[k])); + } + + // Insertion of of K-V pairs, multiple times. + // Trigger at least one mempurge and no SST file creation. + for (size_t i = 0; i < NUM_REPEAT_OVERWRITES; i++) { + // Create value strings of arbitrary length RAND_VALUES_LENGTH bytes. + for (uint64_t k = 5; k < values.size(); k++) { + values[k] = rnd.RandomString(RAND_VALUES_LENGTH); + ASSERT_OK(Put(KEYS[k], values[k])); + } + // Check database consistency. + for (uint64_t k = 0; k < values.size(); k++) { + ASSERT_EQ(Get(KEYS[k]), values[k]); + } + } + + // Check that there was at least one mempurge + uint32_t expected_min_mempurge_count = 1; + // Check that there was no SST files created during flush. + uint32_t expected_sst_count = 0; + EXPECT_GE(mempurge_count.load(), expected_min_mempurge_count); + EXPECT_EQ(sst_count.load(), expected_sst_count); + + // Trigger an SST file creation and no mempurge. + for (size_t i = 0; i < NUM_RAND_INSERTS; i++) { + key = rnd.RandomString(RAND_VALUES_LENGTH); + // Create value strings of arbitrary length RAND_VALUES_LENGTH bytes. + value = rnd.RandomString(RAND_VALUES_LENGTH); + ASSERT_OK(Put(key, value)); + // Check database consistency. + for (uint64_t k = 0; k < values.size(); k++) { + ASSERT_EQ(Get(KEYS[k]), values[k]); + } + ASSERT_EQ(Get(key), value); + } + + // Check that there was at least one SST files created during flush. + expected_sst_count = 1; + EXPECT_GE(sst_count.load(), expected_sst_count); + + // Oddly enough, num_memtable_at_first_flush is not enforced to be + // equal to min_write_buffer_number_to_merge. So by asserting that + // the first SST file creation comes from one output memtable + // from a previous mempurge, and one newly sealed memtable. This + // is the scenario where we observed that some SST files created + // were not properly added to the DB version before our bug fix. + ASSERT_GE(num_memtable_at_first_flush.load(), 2); + + // Check that no data was lost after SST file creation. + for (uint64_t k = 0; k < values.size(); k++) { + ASSERT_EQ(Get(KEYS[k]), values[k]); + } + // Extra check of database consistency. + ASSERT_EQ(Get(key), value); + + Close(); +} + +TEST_P(DBFlushDirectIOTest, DirectIO) { + Options options; + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.max_background_flushes = 2; + options.use_direct_io_for_flush_and_compaction = GetParam(); + options.env = MockEnv::Create(Env::Default()); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:create_file", [&](void* arg) { + bool* use_direct_writes = static_cast(arg); + ASSERT_EQ(*use_direct_writes, + options.use_direct_io_for_flush_and_compaction); + }); + + SyncPoint::GetInstance()->EnableProcessing(); + Reopen(options); + ASSERT_OK(Put("foo", "v")); + FlushOptions flush_options; + flush_options.wait = true; + ASSERT_OK(dbfull()->Flush(flush_options)); + Destroy(options); + delete options.env; +} + +TEST_F(DBFlushTest, FlushError) { + Options options; + std::unique_ptr fault_injection_env( + new FaultInjectionTestEnv(env_)); + options.write_buffer_size = 100; + options.max_write_buffer_number = 4; + options.min_write_buffer_number_to_merge = 3; + options.disable_auto_compactions = true; + options.env = fault_injection_env.get(); + Reopen(options); + + ASSERT_OK(Put("key1", "value1")); + ASSERT_OK(Put("key2", "value2")); + fault_injection_env->SetFilesystemActive(false); + Status s = dbfull()->TEST_SwitchMemtable(); + fault_injection_env->SetFilesystemActive(true); + Destroy(options); + ASSERT_NE(s, Status::OK()); +} + +TEST_F(DBFlushTest, ManualFlushFailsInReadOnlyMode) { + // Regression test for bug where manual flush hangs forever when the DB + // is in read-only mode. Verify it now at least returns, despite failing. + Options options; + std::unique_ptr fault_injection_env( + new FaultInjectionTestEnv(env_)); + options.env = fault_injection_env.get(); + options.max_write_buffer_number = 2; + Reopen(options); + + // Trigger a first flush but don't let it run + ASSERT_OK(db_->PauseBackgroundWork()); + ASSERT_OK(Put("key1", "value1")); + FlushOptions flush_opts; + flush_opts.wait = false; + ASSERT_OK(db_->Flush(flush_opts)); + + // Write a key to the second memtable so we have something to flush later + // after the DB is in read-only mode. + ASSERT_OK(Put("key2", "value2")); + + // Let the first flush continue, hit an error, and put the DB in read-only + // mode. + fault_injection_env->SetFilesystemActive(false); + ASSERT_OK(db_->ContinueBackgroundWork()); + // We ingested the error to env, so the returned status is not OK. + ASSERT_NOK(dbfull()->TEST_WaitForFlushMemTable()); + uint64_t num_bg_errors; + ASSERT_TRUE( + db_->GetIntProperty(DB::Properties::kBackgroundErrors, &num_bg_errors)); + ASSERT_GT(num_bg_errors, 0); + + // In the bug scenario, triggering another flush would cause the second flush + // to hang forever. After the fix we expect it to return an error. + ASSERT_NOK(db_->Flush(FlushOptions())); + + Close(); +} + +TEST_F(DBFlushTest, CFDropRaceWithWaitForFlushMemTables) { + Options options = CurrentOptions(); + options.create_if_missing = true; + CreateAndReopenWithCF({"pikachu"}, options); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::FlushMemTable:AfterScheduleFlush", + "DBFlushTest::CFDropRaceWithWaitForFlushMemTables:BeforeDrop"}, + {"DBFlushTest::CFDropRaceWithWaitForFlushMemTables:AfterFree", + "DBImpl::BackgroundCallFlush:start"}, + {"DBImpl::BackgroundCallFlush:start", + "DBImpl::FlushMemTable:BeforeWaitForBgFlush"}}); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_EQ(2, handles_.size()); + ASSERT_OK(Put(1, "key", "value")); + auto* cfd = static_cast(handles_[1])->cfd(); + port::Thread drop_cf_thr([&]() { + TEST_SYNC_POINT( + "DBFlushTest::CFDropRaceWithWaitForFlushMemTables:BeforeDrop"); + ASSERT_OK(dbfull()->DropColumnFamily(handles_[1])); + ASSERT_OK(dbfull()->DestroyColumnFamilyHandle(handles_[1])); + handles_.resize(1); + TEST_SYNC_POINT( + "DBFlushTest::CFDropRaceWithWaitForFlushMemTables:AfterFree"); + }); + FlushOptions flush_opts; + flush_opts.allow_write_stall = true; + ASSERT_NOK(dbfull()->TEST_FlushMemTable(cfd, flush_opts)); + drop_cf_thr.join(); + Close(); + SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBFlushTest, FireOnFlushCompletedAfterCommittedResult) { + class TestListener : public EventListener { + public: + void OnFlushCompleted(DB* db, const FlushJobInfo& info) override { + // There's only one key in each flush. + ASSERT_EQ(info.smallest_seqno, info.largest_seqno); + ASSERT_NE(0, info.smallest_seqno); + if (info.smallest_seqno == seq1) { + // First flush completed + ASSERT_FALSE(completed1); + completed1 = true; + CheckFlushResultCommitted(db, seq1); + } else { + // Second flush completed + ASSERT_FALSE(completed2); + completed2 = true; + ASSERT_EQ(info.smallest_seqno, seq2); + CheckFlushResultCommitted(db, seq2); + } + } + + void CheckFlushResultCommitted(DB* db, SequenceNumber seq) { + DBImpl* db_impl = static_cast_with_check(db); + InstrumentedMutex* mutex = db_impl->mutex(); + mutex->Lock(); + auto* cfd = static_cast_with_check( + db->DefaultColumnFamily()) + ->cfd(); + ASSERT_LT(seq, cfd->imm()->current()->GetEarliestSequenceNumber()); + mutex->Unlock(); + } + + std::atomic seq1{0}; + std::atomic seq2{0}; + std::atomic completed1{false}; + std::atomic completed2{false}; + }; + std::shared_ptr listener = std::make_shared(); + + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::FlushMemTableToOutputFile:AfterPickMemtables", + "DBFlushTest::FireOnFlushCompletedAfterCommittedResult:WaitFirst"}, + {"DBImpl::FlushMemTableToOutputFile:Finish", + "DBFlushTest::FireOnFlushCompletedAfterCommittedResult:WaitSecond"}}); + SyncPoint::GetInstance()->SetCallBack( + "FlushJob::WriteLevel0Table", [&listener](void* arg) { + // Wait for the second flush finished, out of mutex. + auto* mems = reinterpret_cast*>(arg); + if (mems->front()->GetEarliestSequenceNumber() == listener->seq1 - 1) { + TEST_SYNC_POINT( + "DBFlushTest::FireOnFlushCompletedAfterCommittedResult:" + "WaitSecond"); + } + }); + + Options options = CurrentOptions(); + options.create_if_missing = true; + options.listeners.push_back(listener); + // Setting max_flush_jobs = max_background_jobs / 4 = 2. + options.max_background_jobs = 8; + // Allow 2 immutable memtables. + options.max_write_buffer_number = 3; + Reopen(options); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(Put("foo", "v")); + listener->seq1 = db_->GetLatestSequenceNumber(); + // t1 will wait for the second flush complete before committing flush result. + auto t1 = port::Thread([&]() { + // flush_opts.wait = true + ASSERT_OK(db_->Flush(FlushOptions())); + }); + // Wait for first flush started. + TEST_SYNC_POINT( + "DBFlushTest::FireOnFlushCompletedAfterCommittedResult:WaitFirst"); + // The second flush will exit early without commit its result. The work + // is delegated to the first flush. + ASSERT_OK(Put("bar", "v")); + listener->seq2 = db_->GetLatestSequenceNumber(); + FlushOptions flush_opts; + flush_opts.wait = false; + ASSERT_OK(db_->Flush(flush_opts)); + t1.join(); + // Ensure background work is fully finished including listener callbacks + // before accessing listener state. + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + ASSERT_TRUE(listener->completed1); + ASSERT_TRUE(listener->completed2); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(DBFlushTest, FlushWithBlob) { + constexpr uint64_t min_blob_size = 10; + + Options options; + options.enable_blob_files = true; + options.min_blob_size = min_blob_size; + options.disable_auto_compactions = true; + options.env = env_; + + Reopen(options); + + constexpr char short_value[] = "short"; + static_assert(sizeof(short_value) - 1 < min_blob_size, + "short_value too long"); + + constexpr char long_value[] = "long_value"; + static_assert(sizeof(long_value) - 1 >= min_blob_size, + "long_value too short"); + + ASSERT_OK(Put("key1", short_value)); + ASSERT_OK(Put("key2", long_value)); + + ASSERT_OK(Flush()); + + ASSERT_EQ(Get("key1"), short_value); + ASSERT_EQ(Get("key2"), long_value); + + VersionSet* const versions = dbfull()->GetVersionSet(); + assert(versions); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + assert(cfd); + + Version* const current = cfd->current(); + assert(current); + + const VersionStorageInfo* const storage_info = current->storage_info(); + assert(storage_info); + + const auto& l0_files = storage_info->LevelFiles(0); + ASSERT_EQ(l0_files.size(), 1); + + const FileMetaData* const table_file = l0_files[0]; + assert(table_file); + + const auto& blob_files = storage_info->GetBlobFiles(); + ASSERT_EQ(blob_files.size(), 1); + + const auto& blob_file = blob_files.front(); + assert(blob_file); + + ASSERT_EQ(table_file->smallest.user_key(), "key1"); + ASSERT_EQ(table_file->largest.user_key(), "key2"); + ASSERT_EQ(table_file->fd.smallest_seqno, 1); + ASSERT_EQ(table_file->fd.largest_seqno, 2); + ASSERT_EQ(table_file->oldest_blob_file_number, + blob_file->GetBlobFileNumber()); + + ASSERT_EQ(blob_file->GetTotalBlobCount(), 1); + + const InternalStats* const internal_stats = cfd->internal_stats(); + assert(internal_stats); + + const auto& compaction_stats = internal_stats->TEST_GetCompactionStats(); + ASSERT_FALSE(compaction_stats.empty()); + ASSERT_EQ(compaction_stats[0].bytes_written, table_file->fd.GetFileSize()); + ASSERT_EQ(compaction_stats[0].bytes_written_blob, + blob_file->GetTotalBlobBytes()); + ASSERT_EQ(compaction_stats[0].num_output_files, 1); + ASSERT_EQ(compaction_stats[0].num_output_files_blob, 1); + + const uint64_t* const cf_stats_value = internal_stats->TEST_GetCFStatsValue(); + ASSERT_EQ(cf_stats_value[InternalStats::BYTES_FLUSHED], + compaction_stats[0].bytes_written + + compaction_stats[0].bytes_written_blob); +} + +TEST_F(DBFlushTest, FlushWithChecksumHandoff1) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_SKIP("Test requires non-mem or non-encrypted environment"); + return; + } + std::shared_ptr fault_fs( + new FaultInjectionTestFS(FileSystem::Default())); + std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); + Options options = CurrentOptions(); + options.write_buffer_size = 100; + options.max_write_buffer_number = 4; + options.min_write_buffer_number_to_merge = 3; + options.disable_auto_compactions = true; + options.env = fault_fs_env.get(); + options.checksum_handoff_file_types.Add(FileType::kTableFile); + Reopen(options); + + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kCRC32c); + ASSERT_OK(Put("key1", "value1")); + ASSERT_OK(Put("key2", "value2")); + ASSERT_OK(dbfull()->TEST_SwitchMemtable()); + + // The hash does not match, write fails + // fault_fs->SetChecksumHandoffFuncType(ChecksumType::kxxHash); + // Since the file system returns IOStatus::Corruption, it is an + // unrecoverable error. + SyncPoint::GetInstance()->SetCallBack("FlushJob::Start", [&](void*) { + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kxxHash); + }); + ASSERT_OK(Put("key3", "value3")); + ASSERT_OK(Put("key4", "value4")); + SyncPoint::GetInstance()->EnableProcessing(); + Status s = Flush(); + ASSERT_EQ(s.severity(), + ROCKSDB_NAMESPACE::Status::Severity::kUnrecoverableError); + SyncPoint::GetInstance()->DisableProcessing(); + Destroy(options); + Reopen(options); + + // The file system does not support checksum handoff. The check + // will be ignored. + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kNoChecksum); + ASSERT_OK(Put("key5", "value5")); + ASSERT_OK(Put("key6", "value6")); + ASSERT_OK(dbfull()->TEST_SwitchMemtable()); + + // Each write will be similated as corrupted. + // Since the file system returns IOStatus::Corruption, it is an + // unrecoverable error. + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kCRC32c); + SyncPoint::GetInstance()->SetCallBack("FlushJob::Start", [&](void*) { + fault_fs->IngestDataCorruptionBeforeWrite(); + }); + ASSERT_OK(Put("key7", "value7")); + ASSERT_OK(Put("key8", "value8")); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), + ROCKSDB_NAMESPACE::Status::Severity::kUnrecoverableError); + SyncPoint::GetInstance()->DisableProcessing(); + + Destroy(options); +} + +TEST_F(DBFlushTest, FlushWithChecksumHandoff2) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_SKIP("Test requires non-mem or non-encrypted environment"); + return; + } + std::shared_ptr fault_fs( + new FaultInjectionTestFS(FileSystem::Default())); + std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); + Options options = CurrentOptions(); + options.write_buffer_size = 100; + options.max_write_buffer_number = 4; + options.min_write_buffer_number_to_merge = 3; + options.disable_auto_compactions = true; + options.env = fault_fs_env.get(); + Reopen(options); + + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kCRC32c); + ASSERT_OK(Put("key1", "value1")); + ASSERT_OK(Put("key2", "value2")); + ASSERT_OK(Flush()); + + // options is not set, the checksum handoff will not be triggered + SyncPoint::GetInstance()->SetCallBack("FlushJob::Start", [&](void*) { + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kxxHash); + }); + ASSERT_OK(Put("key3", "value3")); + ASSERT_OK(Put("key4", "value4")); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(Flush()); + SyncPoint::GetInstance()->DisableProcessing(); + Destroy(options); + Reopen(options); + + // The file system does not support checksum handoff. The check + // will be ignored. + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kNoChecksum); + ASSERT_OK(Put("key5", "value5")); + ASSERT_OK(Put("key6", "value6")); + ASSERT_OK(Flush()); + + // options is not set, the checksum handoff will not be triggered + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kCRC32c); + SyncPoint::GetInstance()->SetCallBack("FlushJob::Start", [&](void*) { + fault_fs->IngestDataCorruptionBeforeWrite(); + }); + ASSERT_OK(Put("key7", "value7")); + ASSERT_OK(Put("key8", "value8")); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(Flush()); + SyncPoint::GetInstance()->DisableProcessing(); + + Destroy(options); +} + +TEST_F(DBFlushTest, FlushWithChecksumHandoffManifest1) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_SKIP("Test requires non-mem or non-encrypted environment"); + return; + } + std::shared_ptr fault_fs( + new FaultInjectionTestFS(FileSystem::Default())); + std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); + Options options = CurrentOptions(); + options.write_buffer_size = 100; + options.max_write_buffer_number = 4; + options.min_write_buffer_number_to_merge = 3; + options.disable_auto_compactions = true; + options.env = fault_fs_env.get(); + options.checksum_handoff_file_types.Add(FileType::kDescriptorFile); + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kCRC32c); + Reopen(options); + + ASSERT_OK(Put("key1", "value1")); + ASSERT_OK(Put("key2", "value2")); + ASSERT_OK(Flush()); + + // The hash does not match, write fails + // fault_fs->SetChecksumHandoffFuncType(ChecksumType::kxxHash); + // Since the file system returns IOStatus::Corruption, it is mapped to + // kFatalError error. + ASSERT_OK(Put("key3", "value3")); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:WriteManifest", [&](void*) { + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kxxHash); + }); + ASSERT_OK(Put("key3", "value3")); + ASSERT_OK(Put("key4", "value4")); + SyncPoint::GetInstance()->EnableProcessing(); + Status s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kFatalError); + SyncPoint::GetInstance()->DisableProcessing(); + Destroy(options); +} + +TEST_F(DBFlushTest, FlushWithChecksumHandoffManifest2) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_SKIP("Test requires non-mem or non-encrypted environment"); + return; + } + std::shared_ptr fault_fs( + new FaultInjectionTestFS(FileSystem::Default())); + std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); + Options options = CurrentOptions(); + options.write_buffer_size = 100; + options.max_write_buffer_number = 4; + options.min_write_buffer_number_to_merge = 3; + options.disable_auto_compactions = true; + options.env = fault_fs_env.get(); + options.checksum_handoff_file_types.Add(FileType::kDescriptorFile); + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kNoChecksum); + Reopen(options); + // The file system does not support checksum handoff. The check + // will be ignored. + ASSERT_OK(Put("key5", "value5")); + ASSERT_OK(Put("key6", "value6")); + ASSERT_OK(Flush()); + + // Each write will be similated as corrupted. + // Since the file system returns IOStatus::Corruption, it is mapped to + // kFatalError error. + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kCRC32c); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:WriteManifest", + [&](void*) { fault_fs->IngestDataCorruptionBeforeWrite(); }); + ASSERT_OK(Put("key7", "value7")); + ASSERT_OK(Put("key8", "value8")); + SyncPoint::GetInstance()->EnableProcessing(); + Status s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kFatalError); + SyncPoint::GetInstance()->DisableProcessing(); + + Destroy(options); +} + +TEST_F(DBFlushTest, PickRightMemtables) { + Options options = CurrentOptions(); + DestroyAndReopen(options); + options.create_if_missing = true; + + const std::string test_cf_name = "test_cf"; + options.max_write_buffer_number = 128; + CreateColumnFamilies({test_cf_name}, options); + + Close(); + + ReopenWithColumnFamilies({kDefaultColumnFamilyName, test_cf_name}, options); + + ASSERT_OK(db_->Put(WriteOptions(), "key", "value")); + + ASSERT_OK(db_->Put(WriteOptions(), handles_[1], "key", "value")); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::SyncClosedLogs:BeforeReLock", [&](void* /*arg*/) { + ASSERT_OK(db_->Put(WriteOptions(), handles_[1], "what", "v")); + auto* cfhi = + static_cast_with_check(handles_[1]); + assert(cfhi); + ASSERT_OK(dbfull()->TEST_SwitchMemtable(cfhi->cfd())); + }); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::FlushMemTableToOutputFile:AfterPickMemtables", [&](void* arg) { + auto* job = reinterpret_cast(arg); + assert(job); + const auto& mems = job->GetMemTables(); + assert(mems.size() == 1); + assert(mems[0]); + ASSERT_EQ(1, mems[0]->GetID()); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(db_->Flush(FlushOptions(), handles_[1])); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +class DBFlushTestBlobError : public DBFlushTest, + public testing::WithParamInterface { + public: + DBFlushTestBlobError() : sync_point_(GetParam()) {} + + std::string sync_point_; +}; + +INSTANTIATE_TEST_CASE_P(DBFlushTestBlobError, DBFlushTestBlobError, + ::testing::ValuesIn(std::vector{ + "BlobFileBuilder::WriteBlobToFile:AddRecord", + "BlobFileBuilder::WriteBlobToFile:AppendFooter"})); + +TEST_P(DBFlushTestBlobError, FlushError) { + Options options; + options.enable_blob_files = true; + options.disable_auto_compactions = true; + options.env = env_; + + Reopen(options); + + ASSERT_OK(Put("key", "blob")); + + SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* arg) { + Status* const s = static_cast(arg); + assert(s); + + (*s) = Status::IOError(sync_point_); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_NOK(Flush()); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + VersionSet* const versions = dbfull()->GetVersionSet(); + assert(versions); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + assert(cfd); + + Version* const current = cfd->current(); + assert(current); + + const VersionStorageInfo* const storage_info = current->storage_info(); + assert(storage_info); + + const auto& l0_files = storage_info->LevelFiles(0); + ASSERT_TRUE(l0_files.empty()); + + const auto& blob_files = storage_info->GetBlobFiles(); + ASSERT_TRUE(blob_files.empty()); + + // Make sure the files generated by the failed job have been deleted + std::vector files; + ASSERT_OK(env_->GetChildren(dbname_, &files)); + for (const auto& file : files) { + uint64_t number = 0; + FileType type = kTableFile; + + if (!ParseFileName(file, &number, &type)) { + continue; + } + + ASSERT_NE(type, kTableFile); + ASSERT_NE(type, kBlobFile); + } + + const InternalStats* const internal_stats = cfd->internal_stats(); + assert(internal_stats); + + const auto& compaction_stats = internal_stats->TEST_GetCompactionStats(); + ASSERT_FALSE(compaction_stats.empty()); + + if (sync_point_ == "BlobFileBuilder::WriteBlobToFile:AddRecord") { + ASSERT_EQ(compaction_stats[0].bytes_written, 0); + ASSERT_EQ(compaction_stats[0].bytes_written_blob, 0); + ASSERT_EQ(compaction_stats[0].num_output_files, 0); + ASSERT_EQ(compaction_stats[0].num_output_files_blob, 0); + } else { + // SST file writing succeeded; blob file writing failed (during Finish) + ASSERT_GT(compaction_stats[0].bytes_written, 0); + ASSERT_EQ(compaction_stats[0].bytes_written_blob, 0); + ASSERT_EQ(compaction_stats[0].num_output_files, 1); + ASSERT_EQ(compaction_stats[0].num_output_files_blob, 0); + } + + const uint64_t* const cf_stats_value = internal_stats->TEST_GetCFStatsValue(); + ASSERT_EQ(cf_stats_value[InternalStats::BYTES_FLUSHED], + compaction_stats[0].bytes_written + + compaction_stats[0].bytes_written_blob); +} + +TEST_F(DBFlushTest, TombstoneVisibleInSnapshot) { + class SimpleTestFlushListener : public EventListener { + public: + explicit SimpleTestFlushListener(DBFlushTest* _test) : test_(_test) {} + ~SimpleTestFlushListener() override {} + + void OnFlushBegin(DB* db, const FlushJobInfo& info) override { + ASSERT_EQ(static_cast(0), info.cf_id); + + ASSERT_OK(db->Delete(WriteOptions(), "foo")); + snapshot_ = db->GetSnapshot(); + ASSERT_OK(db->Put(WriteOptions(), "foo", "value")); + + auto* dbimpl = static_cast_with_check(db); + assert(dbimpl); + + ColumnFamilyHandle* cfh = db->DefaultColumnFamily(); + auto* cfhi = static_cast_with_check(cfh); + assert(cfhi); + ASSERT_OK(dbimpl->TEST_SwitchMemtable(cfhi->cfd())); + } + + DBFlushTest* test_ = nullptr; + const Snapshot* snapshot_ = nullptr; + }; + + Options options = CurrentOptions(); + options.create_if_missing = true; + auto* listener = new SimpleTestFlushListener(this); + options.listeners.emplace_back(listener); + DestroyAndReopen(options); + + ASSERT_OK(db_->Put(WriteOptions(), "foo", "value0")); + + ManagedSnapshot snapshot_guard(db_); + + ColumnFamilyHandle* default_cf = db_->DefaultColumnFamily(); + ASSERT_OK(db_->Flush(FlushOptions(), default_cf)); + + const Snapshot* snapshot = listener->snapshot_; + assert(snapshot); + + ReadOptions read_opts; + read_opts.snapshot = snapshot; + + // Using snapshot should not see "foo". + { + std::string value; + Status s = db_->Get(read_opts, "foo", &value); + ASSERT_TRUE(s.IsNotFound()); + } + + db_->ReleaseSnapshot(snapshot); +} + +TEST_P(DBAtomicFlushTest, ManualFlushUnder2PC) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.allow_2pc = true; + options.atomic_flush = GetParam(); + // 64MB so that memtable flush won't be trigger by the small writes. + options.write_buffer_size = (static_cast(64) << 20); + auto flush_listener = std::make_shared(); + flush_listener->expected_flush_reason = FlushReason::kManualFlush; + options.listeners.push_back(flush_listener); + // Destroy the DB to recreate as a TransactionDB. + Close(); + Destroy(options, true); + + // Create a TransactionDB. + TransactionDB* txn_db = nullptr; + TransactionDBOptions txn_db_opts; + txn_db_opts.write_policy = TxnDBWritePolicy::WRITE_COMMITTED; + ASSERT_OK(TransactionDB::Open(options, txn_db_opts, dbname_, &txn_db)); + ASSERT_NE(txn_db, nullptr); + db_ = txn_db; + + // Create two more columns other than default CF. + std::vector cfs = {"puppy", "kitty"}; + CreateColumnFamilies(cfs, options); + ASSERT_EQ(handles_.size(), 2); + ASSERT_EQ(handles_[0]->GetName(), cfs[0]); + ASSERT_EQ(handles_[1]->GetName(), cfs[1]); + const size_t kNumCfToFlush = options.atomic_flush ? 2 : 1; + + WriteOptions wopts; + TransactionOptions txn_opts; + // txn1 only prepare, but does not commit. + // The WAL containing the prepared but uncommitted data must be kept. + Transaction* txn1 = txn_db->BeginTransaction(wopts, txn_opts, nullptr); + // txn2 not only prepare, but also commit. + Transaction* txn2 = txn_db->BeginTransaction(wopts, txn_opts, nullptr); + ASSERT_NE(txn1, nullptr); + ASSERT_NE(txn2, nullptr); + for (size_t i = 0; i < kNumCfToFlush; i++) { + ASSERT_OK(txn1->Put(handles_[i], "k1", "v1")); + ASSERT_OK(txn2->Put(handles_[i], "k2", "v2")); + } + // A txn must be named before prepare. + ASSERT_OK(txn1->SetName("txn1")); + ASSERT_OK(txn2->SetName("txn2")); + // Prepare writes to WAL, but not to memtable. (WriteCommitted) + ASSERT_OK(txn1->Prepare()); + ASSERT_OK(txn2->Prepare()); + // Commit writes to memtable. + ASSERT_OK(txn2->Commit()); + delete txn1; + delete txn2; + + // There are still data in memtable not flushed. + // But since data is small enough to reside in the active memtable, + // there are no immutable memtable. + for (size_t i = 0; i < kNumCfToFlush; i++) { + auto cfh = static_cast(handles_[i]); + ASSERT_EQ(0, cfh->cfd()->imm()->NumNotFlushed()); + ASSERT_FALSE(cfh->cfd()->mem()->IsEmpty()); + } + + // Atomic flush memtables, + // the min log with prepared data should be written to MANIFEST. + std::vector cfs_to_flush(kNumCfToFlush); + for (size_t i = 0; i < kNumCfToFlush; i++) { + cfs_to_flush[i] = handles_[i]; + } + ASSERT_OK(txn_db->Flush(FlushOptions(), cfs_to_flush)); + + // There are no remaining data in memtable after flush. + for (size_t i = 0; i < kNumCfToFlush; i++) { + auto cfh = static_cast(handles_[i]); + ASSERT_EQ(0, cfh->cfd()->imm()->NumNotFlushed()); + ASSERT_TRUE(cfh->cfd()->mem()->IsEmpty()); + } + + // The recovered min log number with prepared data should be non-zero. + // In 2pc mode, MinLogNumberToKeep returns the + // VersionSet::min_log_number_to_keep recovered from MANIFEST, if it's 0, + // it means atomic flush didn't write the min_log_number_to_keep to MANIFEST. + cfs.push_back(kDefaultColumnFamilyName); + ASSERT_OK(TryReopenWithColumnFamilies(cfs, options)); + DBImpl* db_impl = reinterpret_cast(db_); + ASSERT_TRUE(db_impl->allow_2pc()); + ASSERT_NE(db_impl->MinLogNumberToKeep(), 0); +} + +TEST_P(DBAtomicFlushTest, ManualAtomicFlush) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.atomic_flush = GetParam(); + options.write_buffer_size = (static_cast(64) << 20); + auto flush_listener = std::make_shared(); + flush_listener->expected_flush_reason = FlushReason::kManualFlush; + options.listeners.push_back(flush_listener); + + CreateAndReopenWithCF({"pikachu", "eevee"}, options); + size_t num_cfs = handles_.size(); + ASSERT_EQ(3, num_cfs); + WriteOptions wopts; + wopts.disableWAL = true; + for (size_t i = 0; i != num_cfs; ++i) { + ASSERT_OK(Put(static_cast(i) /*cf*/, "key", "value", wopts)); + } + + for (size_t i = 0; i != num_cfs; ++i) { + auto cfh = static_cast(handles_[i]); + ASSERT_EQ(0, cfh->cfd()->imm()->NumNotFlushed()); + ASSERT_FALSE(cfh->cfd()->mem()->IsEmpty()); + } + + std::vector cf_ids; + for (size_t i = 0; i != num_cfs; ++i) { + cf_ids.emplace_back(static_cast(i)); + } + ASSERT_OK(Flush(cf_ids)); + + for (size_t i = 0; i != num_cfs; ++i) { + auto cfh = static_cast(handles_[i]); + ASSERT_EQ(0, cfh->cfd()->imm()->NumNotFlushed()); + ASSERT_TRUE(cfh->cfd()->mem()->IsEmpty()); + } +} + +TEST_P(DBAtomicFlushTest, PrecomputeMinLogNumberToKeepNon2PC) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.atomic_flush = GetParam(); + options.write_buffer_size = (static_cast(64) << 20); + CreateAndReopenWithCF({"pikachu"}, options); + + const size_t num_cfs = handles_.size(); + ASSERT_EQ(num_cfs, 2); + WriteOptions wopts; + for (size_t i = 0; i != num_cfs; ++i) { + ASSERT_OK(Put(static_cast(i) /*cf*/, "key", "value", wopts)); + } + + { + // Flush the default CF only. + std::vector cf_ids{0}; + ASSERT_OK(Flush(cf_ids)); + + autovector flushed_cfds; + autovector> flush_edits; + auto flushed_cfh = static_cast(handles_[0]); + flushed_cfds.push_back(flushed_cfh->cfd()); + flush_edits.push_back({}); + auto unflushed_cfh = static_cast(handles_[1]); + + ASSERT_EQ(PrecomputeMinLogNumberToKeepNon2PC(dbfull()->GetVersionSet(), + flushed_cfds, flush_edits), + unflushed_cfh->cfd()->GetLogNumber()); + } + + { + // Flush all CFs. + std::vector cf_ids; + for (size_t i = 0; i != num_cfs; ++i) { + cf_ids.emplace_back(static_cast(i)); + } + ASSERT_OK(Flush(cf_ids)); + uint64_t log_num_after_flush = dbfull()->TEST_GetCurrentLogNumber(); + + uint64_t min_log_number_to_keep = std::numeric_limits::max(); + autovector flushed_cfds; + autovector> flush_edits; + for (size_t i = 0; i != num_cfs; ++i) { + auto cfh = static_cast(handles_[i]); + flushed_cfds.push_back(cfh->cfd()); + flush_edits.push_back({}); + min_log_number_to_keep = + std::min(min_log_number_to_keep, cfh->cfd()->GetLogNumber()); + } + ASSERT_EQ(min_log_number_to_keep, log_num_after_flush); + ASSERT_EQ(PrecomputeMinLogNumberToKeepNon2PC(dbfull()->GetVersionSet(), + flushed_cfds, flush_edits), + min_log_number_to_keep); + } +} + +TEST_P(DBAtomicFlushTest, AtomicFlushTriggeredByMemTableFull) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.atomic_flush = GetParam(); + // 4KB so that we can easily trigger auto flush. + options.write_buffer_size = 4096; + + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BackgroundCallFlush:FlushFinish:0", + "DBAtomicFlushTest::AtomicFlushTriggeredByMemTableFull:BeforeCheck"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + CreateAndReopenWithCF({"pikachu", "eevee"}, options); + size_t num_cfs = handles_.size(); + ASSERT_EQ(3, num_cfs); + WriteOptions wopts; + wopts.disableWAL = true; + for (size_t i = 0; i != num_cfs; ++i) { + ASSERT_OK(Put(static_cast(i) /*cf*/, "key", "value", wopts)); + } + // Keep writing to one of them column families to trigger auto flush. + for (int i = 0; i != 4000; ++i) { + ASSERT_OK(Put(static_cast(num_cfs) - 1 /*cf*/, + "key" + std::to_string(i), "value" + std::to_string(i), + wopts)); + } + + TEST_SYNC_POINT( + "DBAtomicFlushTest::AtomicFlushTriggeredByMemTableFull:BeforeCheck"); + if (options.atomic_flush) { + for (size_t i = 0; i + 1 != num_cfs; ++i) { + auto cfh = static_cast(handles_[i]); + ASSERT_EQ(0, cfh->cfd()->imm()->NumNotFlushed()); + ASSERT_TRUE(cfh->cfd()->mem()->IsEmpty()); + } + } else { + for (size_t i = 0; i + 1 != num_cfs; ++i) { + auto cfh = static_cast(handles_[i]); + ASSERT_EQ(0, cfh->cfd()->imm()->NumNotFlushed()); + ASSERT_FALSE(cfh->cfd()->mem()->IsEmpty()); + } + } + SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_P(DBAtomicFlushTest, AtomicFlushRollbackSomeJobs) { + bool atomic_flush = GetParam(); + if (!atomic_flush) { + return; + } + std::unique_ptr fault_injection_env( + new FaultInjectionTestEnv(env_)); + Options options = CurrentOptions(); + options.create_if_missing = true; + options.atomic_flush = atomic_flush; + options.env = fault_injection_env.get(); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::AtomicFlushMemTablesToOutputFiles:SomeFlushJobsComplete:1", + "DBAtomicFlushTest::AtomicFlushRollbackSomeJobs:1"}, + {"DBAtomicFlushTest::AtomicFlushRollbackSomeJobs:2", + "DBImpl::AtomicFlushMemTablesToOutputFiles:SomeFlushJobsComplete:2"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + CreateAndReopenWithCF({"pikachu", "eevee"}, options); + size_t num_cfs = handles_.size(); + ASSERT_EQ(3, num_cfs); + WriteOptions wopts; + wopts.disableWAL = true; + for (size_t i = 0; i != num_cfs; ++i) { + int cf_id = static_cast(i); + ASSERT_OK(Put(cf_id, "key", "value", wopts)); + } + FlushOptions flush_opts; + flush_opts.wait = false; + ASSERT_OK(dbfull()->Flush(flush_opts, handles_)); + TEST_SYNC_POINT("DBAtomicFlushTest::AtomicFlushRollbackSomeJobs:1"); + fault_injection_env->SetFilesystemActive(false); + TEST_SYNC_POINT("DBAtomicFlushTest::AtomicFlushRollbackSomeJobs:2"); + for (auto* cfh : handles_) { + // Returns the IO error happend during flush. + ASSERT_NOK(dbfull()->TEST_WaitForFlushMemTable(cfh)); + } + for (size_t i = 0; i != num_cfs; ++i) { + auto cfh = static_cast(handles_[i]); + ASSERT_EQ(1, cfh->cfd()->imm()->NumNotFlushed()); + ASSERT_TRUE(cfh->cfd()->mem()->IsEmpty()); + } + fault_injection_env->SetFilesystemActive(true); + Destroy(options); +} + +TEST_P(DBAtomicFlushTest, FlushMultipleCFs_DropSomeBeforeRequestFlush) { + bool atomic_flush = GetParam(); + if (!atomic_flush) { + return; + } + Options options = CurrentOptions(); + options.create_if_missing = true; + options.atomic_flush = atomic_flush; + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->EnableProcessing(); + + CreateAndReopenWithCF({"pikachu", "eevee"}, options); + size_t num_cfs = handles_.size(); + ASSERT_EQ(3, num_cfs); + WriteOptions wopts; + wopts.disableWAL = true; + std::vector cf_ids; + for (size_t i = 0; i != num_cfs; ++i) { + int cf_id = static_cast(i); + ASSERT_OK(Put(cf_id, "key", "value", wopts)); + cf_ids.push_back(cf_id); + } + ASSERT_OK(dbfull()->DropColumnFamily(handles_[1])); + ASSERT_TRUE(Flush(cf_ids).IsColumnFamilyDropped()); + Destroy(options); +} + +TEST_P(DBAtomicFlushTest, + FlushMultipleCFs_DropSomeAfterScheduleFlushBeforeFlushJobRun) { + bool atomic_flush = GetParam(); + if (!atomic_flush) { + return; + } + Options options = CurrentOptions(); + options.create_if_missing = true; + options.atomic_flush = atomic_flush; + + CreateAndReopenWithCF({"pikachu", "eevee"}, options); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::AtomicFlushMemTables:AfterScheduleFlush", + "DBAtomicFlushTest::BeforeDropCF"}, + {"DBAtomicFlushTest::AfterDropCF", + "DBImpl::BackgroundCallFlush:start"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + size_t num_cfs = handles_.size(); + ASSERT_EQ(3, num_cfs); + WriteOptions wopts; + wopts.disableWAL = true; + for (size_t i = 0; i != num_cfs; ++i) { + int cf_id = static_cast(i); + ASSERT_OK(Put(cf_id, "key", "value", wopts)); + } + port::Thread user_thread([&]() { + TEST_SYNC_POINT("DBAtomicFlushTest::BeforeDropCF"); + ASSERT_OK(dbfull()->DropColumnFamily(handles_[1])); + TEST_SYNC_POINT("DBAtomicFlushTest::AfterDropCF"); + }); + FlushOptions flush_opts; + flush_opts.wait = true; + ASSERT_OK(dbfull()->Flush(flush_opts, handles_)); + user_thread.join(); + for (size_t i = 0; i != num_cfs; ++i) { + int cf_id = static_cast(i); + ASSERT_EQ("value", Get(cf_id, "key")); + } + + ReopenWithColumnFamilies({kDefaultColumnFamilyName, "eevee"}, options); + num_cfs = handles_.size(); + ASSERT_EQ(2, num_cfs); + for (size_t i = 0; i != num_cfs; ++i) { + int cf_id = static_cast(i); + ASSERT_EQ("value", Get(cf_id, "key")); + } + Destroy(options); +} + +TEST_P(DBAtomicFlushTest, TriggerFlushAndClose) { + bool atomic_flush = GetParam(); + if (!atomic_flush) { + return; + } + const int kNumKeysTriggerFlush = 4; + Options options = CurrentOptions(); + options.create_if_missing = true; + options.atomic_flush = atomic_flush; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysTriggerFlush)); + CreateAndReopenWithCF({"pikachu"}, options); + + for (int i = 0; i != kNumKeysTriggerFlush; ++i) { + ASSERT_OK(Put(0, "key" + std::to_string(i), "value" + std::to_string(i))); + } + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(Put(0, "key", "value")); + Close(); + + ReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu"}, options); + ASSERT_EQ("value", Get(0, "key")); +} + +TEST_P(DBAtomicFlushTest, PickMemtablesRaceWithBackgroundFlush) { + bool atomic_flush = GetParam(); + Options options = CurrentOptions(); + options.create_if_missing = true; + options.atomic_flush = atomic_flush; + options.max_write_buffer_number = 4; + // Set min_write_buffer_number_to_merge to be greater than 1, so that + // a column family with one memtable in the imm will not cause IsFlushPending + // to return true when flush_requested_ is false. + options.min_write_buffer_number_to_merge = 2; + CreateAndReopenWithCF({"pikachu"}, options); + ASSERT_EQ(2, handles_.size()); + ASSERT_OK(dbfull()->PauseBackgroundWork()); + ASSERT_OK(Put(0, "key00", "value00")); + ASSERT_OK(Put(1, "key10", "value10")); + FlushOptions flush_opts; + flush_opts.wait = false; + ASSERT_OK(dbfull()->Flush(flush_opts, handles_)); + ASSERT_OK(Put(0, "key01", "value01")); + // Since max_write_buffer_number is 4, the following flush won't cause write + // stall. + ASSERT_OK(dbfull()->Flush(flush_opts)); + ASSERT_OK(dbfull()->DropColumnFamily(handles_[1])); + ASSERT_OK(dbfull()->DestroyColumnFamilyHandle(handles_[1])); + handles_[1] = nullptr; + ASSERT_OK(dbfull()->ContinueBackgroundWork()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[0])); + delete handles_[0]; + handles_.clear(); +} + +TEST_P(DBAtomicFlushTest, CFDropRaceWithWaitForFlushMemTables) { + bool atomic_flush = GetParam(); + if (!atomic_flush) { + return; + } + Options options = CurrentOptions(); + options.create_if_missing = true; + options.atomic_flush = atomic_flush; + CreateAndReopenWithCF({"pikachu"}, options); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::AtomicFlushMemTables:AfterScheduleFlush", + "DBAtomicFlushTest::CFDropRaceWithWaitForFlushMemTables:BeforeDrop"}, + {"DBAtomicFlushTest::CFDropRaceWithWaitForFlushMemTables:AfterFree", + "DBImpl::BackgroundCallFlush:start"}, + {"DBImpl::BackgroundCallFlush:start", + "DBImpl::AtomicFlushMemTables:BeforeWaitForBgFlush"}}); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_EQ(2, handles_.size()); + ASSERT_OK(Put(0, "key", "value")); + ASSERT_OK(Put(1, "key", "value")); + auto* cfd_default = + static_cast(dbfull()->DefaultColumnFamily()) + ->cfd(); + auto* cfd_pikachu = static_cast(handles_[1])->cfd(); + port::Thread drop_cf_thr([&]() { + TEST_SYNC_POINT( + "DBAtomicFlushTest::CFDropRaceWithWaitForFlushMemTables:BeforeDrop"); + ASSERT_OK(dbfull()->DropColumnFamily(handles_[1])); + delete handles_[1]; + handles_.resize(1); + TEST_SYNC_POINT( + "DBAtomicFlushTest::CFDropRaceWithWaitForFlushMemTables:AfterFree"); + }); + FlushOptions flush_opts; + flush_opts.allow_write_stall = true; + ASSERT_OK(dbfull()->TEST_AtomicFlushMemTables({cfd_default, cfd_pikachu}, + flush_opts)); + drop_cf_thr.join(); + Close(); + SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_P(DBAtomicFlushTest, RollbackAfterFailToInstallResults) { + bool atomic_flush = GetParam(); + if (!atomic_flush) { + return; + } + auto fault_injection_env = std::make_shared(env_); + Options options = CurrentOptions(); + options.env = fault_injection_env.get(); + options.create_if_missing = true; + options.atomic_flush = atomic_flush; + CreateAndReopenWithCF({"pikachu"}, options); + ASSERT_EQ(2, handles_.size()); + for (size_t cf = 0; cf < handles_.size(); ++cf) { + ASSERT_OK(Put(static_cast(cf), "a", "value")); + } + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::ProcessManifestWrites:BeforeWriteLastVersionEdit:0", + [&](void* /*arg*/) { fault_injection_env->SetFilesystemActive(false); }); + SyncPoint::GetInstance()->EnableProcessing(); + FlushOptions flush_opts; + Status s = db_->Flush(flush_opts, handles_); + ASSERT_NOK(s); + fault_injection_env->SetFilesystemActive(true); + Close(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +// In atomic flush, concurrent bg flush threads commit to the MANIFEST in +// serial, in the order of their picked memtables for each column family. +// Only when a bg flush thread finds out that its memtables are the earliest +// unflushed ones for all the included column families will this bg flush +// thread continue to commit to MANIFEST. +// This unit test uses sync point to coordinate the execution of two bg threads +// executing the same sequence of functions. The interleaving are as follows. +// time bg1 bg2 +// | pick memtables to flush +// | flush memtables cf1_m1, cf2_m1 +// | join MANIFEST write queue +// | pick memtabls to flush +// | flush memtables cf1_(m1+1) +// | join MANIFEST write queue +// | wait to write MANIFEST +// | write MANIFEST +// | IO error +// | detect IO error and stop waiting +// V +TEST_P(DBAtomicFlushTest, BgThreadNoWaitAfterManifestError) { + bool atomic_flush = GetParam(); + if (!atomic_flush) { + return; + } + auto fault_injection_env = std::make_shared(env_); + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.atomic_flush = true; + options.env = fault_injection_env.get(); + // Set a larger value than default so that RocksDB can schedule concurrent + // background flush threads. + options.max_background_jobs = 8; + options.max_write_buffer_number = 8; + CreateAndReopenWithCF({"pikachu"}, options); + + assert(2 == handles_.size()); + + WriteOptions write_opts; + write_opts.disableWAL = true; + + ASSERT_OK(Put(0, "a", "v_0_a", write_opts)); + ASSERT_OK(Put(1, "a", "v_1_a", write_opts)); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + SyncPoint::GetInstance()->LoadDependency({ + {"BgFlushThr2:WaitToCommit", "BgFlushThr1:BeforeWriteManifest"}, + }); + + std::thread::id bg_flush_thr1, bg_flush_thr2; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCallFlush:start", [&](void*) { + if (bg_flush_thr1 == std::thread::id()) { + bg_flush_thr1 = std::this_thread::get_id(); + } else if (bg_flush_thr2 == std::thread::id()) { + bg_flush_thr2 = std::this_thread::get_id(); + } + }); + + int called = 0; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::AtomicFlushMemTablesToOutputFiles:WaitToCommit", [&](void* arg) { + if (std::this_thread::get_id() == bg_flush_thr2) { + const auto* ptr = reinterpret_cast*>(arg); + assert(ptr); + if (0 == called) { + // When bg flush thread 2 reaches here for the first time. + ASSERT_OK(ptr->first); + ASSERT_TRUE(ptr->second); + } else if (1 == called) { + // When bg flush thread 2 reaches here for the second time. + ASSERT_TRUE(ptr->first.IsIOError()); + ASSERT_FALSE(ptr->second); + } + ++called; + TEST_SYNC_POINT("BgFlushThr2:WaitToCommit"); + } + }); + + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::ProcessManifestWrites:BeforeWriteLastVersionEdit:0", + [&](void*) { + if (std::this_thread::get_id() == bg_flush_thr1) { + TEST_SYNC_POINT("BgFlushThr1:BeforeWriteManifest"); + } + }); + + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:WriteManifest", [&](void*) { + if (std::this_thread::get_id() != bg_flush_thr1) { + return; + } + ASSERT_OK(db_->Put(write_opts, "b", "v_1_b")); + + FlushOptions flush_opts; + flush_opts.wait = false; + std::vector cfhs(1, db_->DefaultColumnFamily()); + ASSERT_OK(dbfull()->Flush(flush_opts, cfhs)); + }); + + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::ProcessManifestWrites:AfterSyncManifest", [&](void* arg) { + auto* ptr = reinterpret_cast(arg); + assert(ptr); + *ptr = IOStatus::IOError("Injected failure"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_TRUE(dbfull()->Flush(FlushOptions(), handles_).IsIOError()); + + Close(); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_P(DBAtomicFlushTest, NoWaitWhenWritesStopped) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.atomic_flush = GetParam(); + options.max_write_buffer_number = 2; + options.memtable_factory.reset(test::NewSpecialSkipListFactory(1)); + + Reopen(options); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::DelayWrite:Start", + "DBAtomicFlushTest::NoWaitWhenWritesStopped:0"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(dbfull()->PauseBackgroundWork()); + for (int i = 0; i < options.max_write_buffer_number; ++i) { + ASSERT_OK(Put("k" + std::to_string(i), "v" + std::to_string(i))); + } + std::thread stalled_writer([&]() { ASSERT_OK(Put("k", "v")); }); + + TEST_SYNC_POINT("DBAtomicFlushTest::NoWaitWhenWritesStopped:0"); + + { + FlushOptions flush_opts; + flush_opts.wait = false; + flush_opts.allow_write_stall = true; + ASSERT_TRUE(db_->Flush(flush_opts).IsTryAgain()); + } + + ASSERT_OK(dbfull()->ContinueBackgroundWork()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + + stalled_writer.join(); + + SyncPoint::GetInstance()->DisableProcessing(); +} + +INSTANTIATE_TEST_CASE_P(DBFlushDirectIOTest, DBFlushDirectIOTest, + testing::Bool()); + +INSTANTIATE_TEST_CASE_P(DBAtomicFlushTest, DBAtomicFlushTest, testing::Bool()); + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_impl/compacted_db_impl.cc b/librocksdb-sys/rocksdb/db/db_impl/compacted_db_impl.cc new file mode 100644 index 0000000..3d824ba --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_impl/compacted_db_impl.cc @@ -0,0 +1,260 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/db_impl/compacted_db_impl.h" + +#include "db/db_impl/db_impl.h" +#include "db/version_set.h" +#include "logging/logging.h" +#include "table/get_context.h" +#include "util/cast_util.h" + +namespace ROCKSDB_NAMESPACE { + +extern void MarkKeyMayExist(void* arg); +extern bool SaveValue(void* arg, const ParsedInternalKey& parsed_key, + const Slice& v, bool hit_and_return); + +CompactedDBImpl::CompactedDBImpl(const DBOptions& options, + const std::string& dbname) + : DBImpl(options, dbname, /*seq_per_batch*/ false, +/*batch_per_txn*/ true, + /*read_only*/ true), + cfd_(nullptr), + version_(nullptr), + user_comparator_(nullptr) {} + +CompactedDBImpl::~CompactedDBImpl() {} + +size_t CompactedDBImpl::FindFile(const Slice& key) { + size_t right = files_.num_files - 1; + auto cmp = [&](const FdWithKeyRange& f, const Slice& k) -> bool { + return user_comparator_->Compare(ExtractUserKey(f.largest_key), k) < 0; + }; + return static_cast( + std::lower_bound(files_.files, files_.files + right, key, cmp) - + files_.files); +} + +Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*, + const Slice& key, PinnableSlice* value) { + return Get(options, /*column_family*/ nullptr, key, value, + /*timestamp*/ nullptr); +} + +Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*, + const Slice& key, PinnableSlice* value, + std::string* timestamp) { + if (options.io_activity != Env::IOActivity::kUnknown) { + return Status::InvalidArgument( + "Cannot call Get with `ReadOptions::io_activity` != " + "`Env::IOActivity::kUnknown`"); + } + assert(user_comparator_); + if (options.timestamp) { + const Status s = FailIfTsMismatchCf( + DefaultColumnFamily(), *(options.timestamp), /*ts_for_read=*/true); + if (!s.ok()) { + return s; + } + } else { + const Status s = FailIfCfHasTs(DefaultColumnFamily()); + if (!s.ok()) { + return s; + } + } + + // Clear the timestamps for returning results so that we can distinguish + // between tombstone or key that has never been written + if (timestamp) { + timestamp->clear(); + } + + GetWithTimestampReadCallback read_cb(kMaxSequenceNumber); + std::string* ts = + user_comparator_->timestamp_size() > 0 ? timestamp : nullptr; + LookupKey lkey(key, kMaxSequenceNumber, options.timestamp); + GetContext get_context(user_comparator_, nullptr, nullptr, nullptr, + GetContext::kNotFound, lkey.user_key(), value, + /*columns=*/nullptr, ts, nullptr, nullptr, true, + nullptr, nullptr, nullptr, nullptr, &read_cb); + + const FdWithKeyRange& f = files_.files[FindFile(lkey.user_key())]; + if (user_comparator_->CompareWithoutTimestamp( + key, /*a_has_ts=*/false, + ExtractUserKeyAndStripTimestamp(f.smallest_key, + user_comparator_->timestamp_size()), + /*b_has_ts=*/false) < 0) { + return Status::NotFound(); + } + Status s = f.fd.table_reader->Get(options, lkey.internal_key(), &get_context, + nullptr); + if (!s.ok() && !s.IsNotFound()) { + return s; + } + if (get_context.State() == GetContext::kFound) { + return Status::OK(); + } + return Status::NotFound(); +} + +std::vector CompactedDBImpl::MultiGet( + const ReadOptions& options, const std::vector&, + const std::vector& keys, std::vector* values) { + return MultiGet(options, keys, values, /*timestamps*/ nullptr); +} + +std::vector CompactedDBImpl::MultiGet( + const ReadOptions& options, const std::vector&, + const std::vector& keys, std::vector* values, + std::vector* timestamps) { + assert(user_comparator_); + size_t num_keys = keys.size(); + + if (options.timestamp) { + Status s = FailIfTsMismatchCf(DefaultColumnFamily(), *(options.timestamp), + /*ts_for_read=*/true); + if (!s.ok()) { + return std::vector(num_keys, s); + } + } else { + Status s = FailIfCfHasTs(DefaultColumnFamily()); + if (!s.ok()) { + return std::vector(num_keys, s); + } + } + + // Clear the timestamps for returning results so that we can distinguish + // between tombstone or key that has never been written + if (timestamps) { + for (auto& ts : *timestamps) { + ts.clear(); + } + } + + GetWithTimestampReadCallback read_cb(kMaxSequenceNumber); + autovector reader_list; + for (const auto& key : keys) { + LookupKey lkey(key, kMaxSequenceNumber, options.timestamp); + const FdWithKeyRange& f = files_.files[FindFile(lkey.user_key())]; + if (user_comparator_->CompareWithoutTimestamp( + key, /*a_has_ts=*/false, + ExtractUserKeyAndStripTimestamp(f.smallest_key, + user_comparator_->timestamp_size()), + /*b_has_ts=*/false) < 0) { + reader_list.push_back(nullptr); + } else { + f.fd.table_reader->Prepare(lkey.internal_key()); + reader_list.push_back(f.fd.table_reader); + } + } + std::vector statuses(num_keys, Status::NotFound()); + values->resize(num_keys); + if (timestamps) { + timestamps->resize(num_keys); + } + int idx = 0; + for (auto* r : reader_list) { + if (r != nullptr) { + PinnableSlice pinnable_val; + std::string& value = (*values)[idx]; + LookupKey lkey(keys[idx], kMaxSequenceNumber, options.timestamp); + std::string* timestamp = timestamps ? &(*timestamps)[idx] : nullptr; + GetContext get_context( + user_comparator_, nullptr, nullptr, nullptr, GetContext::kNotFound, + lkey.user_key(), &pinnable_val, /*columns=*/nullptr, + user_comparator_->timestamp_size() > 0 ? timestamp : nullptr, nullptr, + nullptr, true, nullptr, nullptr, nullptr, nullptr, &read_cb); + Status s = r->Get(options, lkey.internal_key(), &get_context, nullptr); + assert(static_cast(idx) < statuses.size()); + if (!s.ok() && !s.IsNotFound()) { + statuses[idx] = s; + } else { + value.assign(pinnable_val.data(), pinnable_val.size()); + if (get_context.State() == GetContext::kFound) { + statuses[idx] = Status::OK(); + } + } + } + ++idx; + } + return statuses; +} + +Status CompactedDBImpl::Init(const Options& options) { + SuperVersionContext sv_context(/* create_superversion */ true); + mutex_.Lock(); + ColumnFamilyDescriptor cf(kDefaultColumnFamilyName, + ColumnFamilyOptions(options)); + Status s = Recover({cf}, true /* read only */, false, true); + if (s.ok()) { + cfd_ = static_cast_with_check(DefaultColumnFamily()) + ->cfd(); + cfd_->InstallSuperVersion(&sv_context, &mutex_); + } + mutex_.Unlock(); + sv_context.Clean(); + if (!s.ok()) { + return s; + } + NewThreadStatusCfInfo(cfd_); + version_ = cfd_->GetSuperVersion()->current; + user_comparator_ = cfd_->user_comparator(); + auto* vstorage = version_->storage_info(); + if (vstorage->num_non_empty_levels() == 0) { + return Status::NotSupported("no file exists"); + } + const LevelFilesBrief& l0 = vstorage->LevelFilesBrief(0); + // L0 should not have files + if (l0.num_files > 1) { + return Status::NotSupported("L0 contain more than 1 file"); + } + if (l0.num_files == 1) { + if (vstorage->num_non_empty_levels() > 1) { + return Status::NotSupported("Both L0 and other level contain files"); + } + files_ = l0; + return Status::OK(); + } + + for (int i = 1; i < vstorage->num_non_empty_levels() - 1; ++i) { + if (vstorage->LevelFilesBrief(i).num_files > 0) { + return Status::NotSupported("Other levels also contain files"); + } + } + + int level = vstorage->num_non_empty_levels() - 1; + if (vstorage->LevelFilesBrief(level).num_files > 0) { + files_ = vstorage->LevelFilesBrief(level); + return Status::OK(); + } + return Status::NotSupported("no file exists"); +} + +Status CompactedDBImpl::Open(const Options& options, const std::string& dbname, + DB** dbptr) { + *dbptr = nullptr; + + if (options.max_open_files != -1) { + return Status::InvalidArgument("require max_open_files = -1"); + } + if (options.merge_operator.get() != nullptr) { + return Status::InvalidArgument("merge operator is not supported"); + } + DBOptions db_options(options); + std::unique_ptr db(new CompactedDBImpl(db_options, dbname)); + Status s = db->Init(options); + if (s.ok()) { + s = db->StartPeriodicTaskScheduler(); + } + if (s.ok()) { + ROCKS_LOG_INFO(db->immutable_db_options_.info_log, + "Opened the db as fully compacted mode"); + LogFlush(db->immutable_db_options_.info_log); + *dbptr = db.release(); + } + return s; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/db_impl/compacted_db_impl.h b/librocksdb-sys/rocksdb/db/db_impl/compacted_db_impl.h new file mode 100644 index 0000000..ad192b4 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_impl/compacted_db_impl.h @@ -0,0 +1,158 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once +#include +#include + +#include "db/db_impl/db_impl.h" + +namespace ROCKSDB_NAMESPACE { + +// TODO: Share common structure with DBImplSecondary and DBImplReadOnly +class CompactedDBImpl : public DBImpl { + public: + CompactedDBImpl(const DBOptions& options, const std::string& dbname); + // No copying allowed + CompactedDBImpl(const CompactedDBImpl&) = delete; + void operator=(const CompactedDBImpl&) = delete; + + ~CompactedDBImpl() override; + + static Status Open(const Options& options, const std::string& dbname, + DB** dbptr); + + // Implementations of the DB interface + using DB::Get; + virtual Status Get(const ReadOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value) override; + + Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family, + const Slice& key, PinnableSlice* value, + std::string* timestamp) override; + + using DB::MultiGet; + // Note that CompactedDBImpl::MultiGet is not the optimized version of + // MultiGet to use. + // TODO: optimize CompactedDBImpl::MultiGet, see DBImpl::MultiGet for details. + virtual std::vector MultiGet( + const ReadOptions& options, const std::vector&, + const std::vector& keys, + std::vector* values) override; + + std::vector MultiGet(const ReadOptions& options, + const std::vector&, + const std::vector& keys, + std::vector* values, + std::vector* timestamps) override; + + using DBImpl::Put; + virtual Status Put(const WriteOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/, + const Slice& /*key*/, const Slice& /*value*/) override { + return Status::NotSupported("Not supported in compacted db mode."); + } + + using DBImpl::PutEntity; + Status PutEntity(const WriteOptions& /* options */, + ColumnFamilyHandle* /* column_family */, + const Slice& /* key */, + const WideColumns& /* columns */) override { + return Status::NotSupported("Not supported in compacted db mode."); + } + + using DBImpl::Merge; + virtual Status Merge(const WriteOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/, + const Slice& /*key*/, const Slice& /*value*/) override { + return Status::NotSupported("Not supported in compacted db mode."); + } + + using DBImpl::Delete; + virtual Status Delete(const WriteOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/, + const Slice& /*key*/) override { + return Status::NotSupported("Not supported in compacted db mode."); + } + virtual Status Write(const WriteOptions& /*options*/, + WriteBatch* /*updates*/) override { + return Status::NotSupported("Not supported in compacted db mode."); + } + using DBImpl::CompactRange; + virtual Status CompactRange(const CompactRangeOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/, + const Slice* /*begin*/, + const Slice* /*end*/) override { + return Status::NotSupported("Not supported in compacted db mode."); + } + + virtual Status DisableFileDeletions() override { + return Status::NotSupported("Not supported in compacted db mode."); + } + virtual Status EnableFileDeletions(bool /*force*/) override { + return Status::NotSupported("Not supported in compacted db mode."); + } + virtual Status GetLiveFiles(std::vector& ret, + uint64_t* manifest_file_size, + bool /*flush_memtable*/) override { + return DBImpl::GetLiveFiles(ret, manifest_file_size, + false /* flush_memtable */); + } + using DBImpl::Flush; + virtual Status Flush(const FlushOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/) override { + return Status::NotSupported("Not supported in compacted db mode."); + } + + virtual Status SyncWAL() override { + return Status::NotSupported("Not supported in compacted db mode."); + } + + using DB::IngestExternalFile; + virtual Status IngestExternalFile( + ColumnFamilyHandle* /*column_family*/, + const std::vector& /*external_files*/, + const IngestExternalFileOptions& /*ingestion_options*/) override { + return Status::NotSupported("Not supported in compacted db mode."); + } + + using DB::CreateColumnFamilyWithImport; + virtual Status CreateColumnFamilyWithImport( + const ColumnFamilyOptions& /*options*/, + const std::string& /*column_family_name*/, + const ImportColumnFamilyOptions& /*import_options*/, + const std::vector& /*metadatas*/, + ColumnFamilyHandle** /*handle*/) override { + return Status::NotSupported("Not supported in compacted db mode."); + } + + using DB::ClipColumnFamily; + virtual Status ClipColumnFamily(ColumnFamilyHandle* /*column_family*/, + const Slice& /*begin*/, + const Slice& /*end*/) override { + return Status::NotSupported("Not supported in compacted db mode."); + } + + // FIXME: some missing overrides for more "write" functions + // Share with DBImplReadOnly? + + protected: + Status FlushForGetLiveFiles() override { + // No-op for read-only DB + return Status::OK(); + } + + private: + friend class DB; + inline size_t FindFile(const Slice& key); + Status Init(const Options& options); + + ColumnFamilyData* cfd_; + Version* version_; + const Comparator* user_comparator_; + LevelFilesBrief files_; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/db_impl/db_impl.cc b/librocksdb-sys/rocksdb/db/db_impl/db_impl.cc new file mode 100644 index 0000000..5dee778 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_impl/db_impl.cc @@ -0,0 +1,6145 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "db/db_impl/db_impl.h" + +#include +#ifdef OS_SOLARIS +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "db/arena_wrapped_db_iter.h" +#include "db/builder.h" +#include "db/compaction/compaction_job.h" +#include "db/db_info_dumper.h" +#include "db/db_iter.h" +#include "db/dbformat.h" +#include "db/error_handler.h" +#include "db/event_helpers.h" +#include "db/external_sst_file_ingestion_job.h" +#include "db/flush_job.h" +#include "db/forward_iterator.h" +#include "db/import_column_family_job.h" +#include "db/job_context.h" +#include "db/log_reader.h" +#include "db/log_writer.h" +#include "db/malloc_stats.h" +#include "db/memtable.h" +#include "db/memtable_list.h" +#include "db/merge_context.h" +#include "db/merge_helper.h" +#include "db/periodic_task_scheduler.h" +#include "db/range_tombstone_fragmenter.h" +#include "db/table_cache.h" +#include "db/table_properties_collector.h" +#include "db/transaction_log_impl.h" +#include "db/version_set.h" +#include "db/write_batch_internal.h" +#include "db/write_callback.h" +#include "env/unique_id_gen.h" +#include "file/file_util.h" +#include "file/filename.h" +#include "file/random_access_file_reader.h" +#include "file/sst_file_manager_impl.h" +#include "logging/auto_roll_logger.h" +#include "logging/log_buffer.h" +#include "logging/logging.h" +#include "monitoring/in_memory_stats_history.h" +#include "monitoring/instrumented_mutex.h" +#include "monitoring/iostats_context_imp.h" +#include "monitoring/perf_context_imp.h" +#include "monitoring/persistent_stats_history.h" +#include "monitoring/thread_status_updater.h" +#include "monitoring/thread_status_util.h" +#include "options/cf_options.h" +#include "options/options_helper.h" +#include "options/options_parser.h" +#include "port/port.h" +#include "rocksdb/cache.h" +#include "rocksdb/compaction_filter.h" +#include "rocksdb/convenience.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/statistics.h" +#include "rocksdb/stats_history.h" +#include "rocksdb/status.h" +#include "rocksdb/table.h" +#include "rocksdb/version.h" +#include "rocksdb/write_buffer_manager.h" +#include "table/block_based/block.h" +#include "table/block_based/block_based_table_factory.h" +#include "table/get_context.h" +#include "table/merging_iterator.h" +#include "table/multiget_context.h" +#include "table/sst_file_dumper.h" +#include "table/table_builder.h" +#include "table/two_level_iterator.h" +#include "table/unique_id_impl.h" +#include "test_util/sync_point.h" +#include "trace_replay/trace_replay.h" +#include "util/autovector.h" +#include "util/cast_util.h" +#include "util/coding.h" +#include "util/compression.h" +#include "util/crc32c.h" +#include "util/defer.h" +#include "util/distributed_mutex.h" +#include "util/hash_containers.h" +#include "util/mutexlock.h" +#include "util/stop_watch.h" +#include "util/string_util.h" +#include "utilities/trace/replayer_impl.h" + +namespace ROCKSDB_NAMESPACE { + +const std::string kDefaultColumnFamilyName("default"); +const std::string kPersistentStatsColumnFamilyName( + "___rocksdb_stats_history___"); +void DumpRocksDBBuildVersion(Logger* log); + +CompressionType GetCompressionFlush( + const ImmutableCFOptions& ioptions, + const MutableCFOptions& mutable_cf_options) { + // Compressing memtable flushes might not help unless the sequential load + // optimization is used for leveled compaction. Otherwise the CPU and + // latency overhead is not offset by saving much space. + if (ioptions.compaction_style == kCompactionStyleUniversal && + mutable_cf_options.compaction_options_universal + .compression_size_percent >= 0) { + return kNoCompression; + } + if (mutable_cf_options.compression_per_level.empty()) { + return mutable_cf_options.compression; + } else { + // For leveled compress when min_level_to_compress != 0. + return mutable_cf_options.compression_per_level[0]; + } +} + +namespace { +void DumpSupportInfo(Logger* logger) { + ROCKS_LOG_HEADER(logger, "Compression algorithms supported:"); + for (auto& compression : OptionsHelper::compression_type_string_map) { + if (compression.second != kNoCompression && + compression.second != kDisableCompressionOption) { + ROCKS_LOG_HEADER(logger, "\t%s supported: %d", compression.first.c_str(), + CompressionTypeSupported(compression.second)); + } + } + ROCKS_LOG_HEADER(logger, "Fast CRC32 supported: %s", + crc32c::IsFastCrc32Supported().c_str()); + + ROCKS_LOG_HEADER(logger, "DMutex implementation: %s", DMutex::kName()); +} +} // namespace + +DBImpl::DBImpl(const DBOptions& options, const std::string& dbname, + const bool seq_per_batch, const bool batch_per_txn, + bool read_only) + : dbname_(dbname), + own_info_log_(options.info_log == nullptr), + init_logger_creation_s_(), + initial_db_options_(SanitizeOptions(dbname, options, read_only, + &init_logger_creation_s_)), + env_(initial_db_options_.env), + io_tracer_(std::make_shared()), + immutable_db_options_(initial_db_options_), + fs_(immutable_db_options_.fs, io_tracer_), + mutable_db_options_(initial_db_options_), + stats_(immutable_db_options_.stats), +#ifdef COERCE_CONTEXT_SWITCH + mutex_(stats_, immutable_db_options_.clock, DB_MUTEX_WAIT_MICROS, &bg_cv_, + immutable_db_options_.use_adaptive_mutex), +#else // COERCE_CONTEXT_SWITCH + mutex_(stats_, immutable_db_options_.clock, DB_MUTEX_WAIT_MICROS, + immutable_db_options_.use_adaptive_mutex), +#endif // COERCE_CONTEXT_SWITCH + default_cf_handle_(nullptr), + error_handler_(this, immutable_db_options_, &mutex_), + event_logger_(immutable_db_options_.info_log.get()), + max_total_in_memory_state_(0), + file_options_(BuildDBOptions(immutable_db_options_, mutable_db_options_)), + file_options_for_compaction_(fs_->OptimizeForCompactionTableWrite( + file_options_, immutable_db_options_)), + seq_per_batch_(seq_per_batch), + batch_per_txn_(batch_per_txn), + next_job_id_(1), + shutting_down_(false), + db_lock_(nullptr), + manual_compaction_paused_(false), + bg_cv_(&mutex_), + logfile_number_(0), + log_dir_synced_(false), + log_empty_(true), + persist_stats_cf_handle_(nullptr), + log_sync_cv_(&log_write_mutex_), + total_log_size_(0), + is_snapshot_supported_(true), + write_buffer_manager_(immutable_db_options_.write_buffer_manager.get()), + write_thread_(immutable_db_options_), + nonmem_write_thread_(immutable_db_options_), + write_controller_(mutable_db_options_.delayed_write_rate), + last_batch_group_size_(0), + unscheduled_flushes_(0), + unscheduled_compactions_(0), + bg_bottom_compaction_scheduled_(0), + bg_compaction_scheduled_(0), + num_running_compactions_(0), + bg_flush_scheduled_(0), + num_running_flushes_(0), + bg_purge_scheduled_(0), + disable_delete_obsolete_files_(0), + pending_purge_obsolete_files_(0), + delete_obsolete_files_last_run_(immutable_db_options_.clock->NowMicros()), + last_stats_dump_time_microsec_(0), + has_unpersisted_data_(false), + unable_to_release_oldest_log_(false), + num_running_ingest_file_(0), + wal_manager_(immutable_db_options_, file_options_, io_tracer_, + seq_per_batch), + bg_work_paused_(0), + bg_compaction_paused_(0), + refitting_level_(false), + opened_successfully_(false), + periodic_task_scheduler_(), + two_write_queues_(options.two_write_queues), + manual_wal_flush_(options.manual_wal_flush), + // last_sequencee_ is always maintained by the main queue that also writes + // to the memtable. When two_write_queues_ is disabled last seq in + // memtable is the same as last seq published to the readers. When it is + // enabled but seq_per_batch_ is disabled, last seq in memtable still + // indicates last published seq since wal-only writes that go to the 2nd + // queue do not consume a sequence number. Otherwise writes performed by + // the 2nd queue could change what is visible to the readers. In this + // cases, last_seq_same_as_publish_seq_==false, the 2nd queue maintains a + // separate variable to indicate the last published sequence. + last_seq_same_as_publish_seq_( + !(seq_per_batch && options.two_write_queues)), + // Since seq_per_batch_ is currently set only by WritePreparedTxn which + // requires a custom gc for compaction, we use that to set use_custom_gc_ + // as well. + use_custom_gc_(seq_per_batch), + shutdown_initiated_(false), + own_sfm_(options.sst_file_manager == nullptr), + closed_(false), + atomic_flush_install_cv_(&mutex_), + blob_callback_(immutable_db_options_.sst_file_manager.get(), &mutex_, + &error_handler_, &event_logger_, + immutable_db_options_.listeners, dbname_), + lock_wal_count_(0) { + // !batch_per_trx_ implies seq_per_batch_ because it is only unset for + // WriteUnprepared, which should use seq_per_batch_. + assert(batch_per_txn_ || seq_per_batch_); + + // Reserve ten files or so for other uses and give the rest to TableCache. + // Give a large number for setting of "infinite" open files. + const int table_cache_size = (mutable_db_options_.max_open_files == -1) + ? TableCache::kInfiniteCapacity + : mutable_db_options_.max_open_files - 10; + LRUCacheOptions co; + co.capacity = table_cache_size; + co.num_shard_bits = immutable_db_options_.table_cache_numshardbits; + co.metadata_charge_policy = kDontChargeCacheMetadata; + // TODO: Consider a non-fixed seed once test fallout (prefetch_test) is + // dealt with + co.hash_seed = 0; + table_cache_ = NewLRUCache(co); + SetDbSessionId(); + assert(!db_session_id_.empty()); + + periodic_task_functions_.emplace(PeriodicTaskType::kDumpStats, + [this]() { this->DumpStats(); }); + periodic_task_functions_.emplace(PeriodicTaskType::kPersistStats, + [this]() { this->PersistStats(); }); + periodic_task_functions_.emplace(PeriodicTaskType::kFlushInfoLog, + [this]() { this->FlushInfoLog(); }); + periodic_task_functions_.emplace( + PeriodicTaskType::kRecordSeqnoTime, + [this]() { this->RecordSeqnoToTimeMapping(); }); + + versions_.reset(new VersionSet(dbname_, &immutable_db_options_, file_options_, + table_cache_.get(), write_buffer_manager_, + &write_controller_, &block_cache_tracer_, + io_tracer_, db_id_, db_session_id_)); + column_family_memtables_.reset( + new ColumnFamilyMemTablesImpl(versions_->GetColumnFamilySet())); + + DumpRocksDBBuildVersion(immutable_db_options_.info_log.get()); + DumpDBFileSummary(immutable_db_options_, dbname_, db_session_id_); + immutable_db_options_.Dump(immutable_db_options_.info_log.get()); + mutable_db_options_.Dump(immutable_db_options_.info_log.get()); + DumpSupportInfo(immutable_db_options_.info_log.get()); + + max_total_wal_size_.store(mutable_db_options_.max_total_wal_size, + std::memory_order_relaxed); + if (write_buffer_manager_) { + wbm_stall_.reset(new WBMStallInterface()); + } +} + +Status DBImpl::Resume() { + ROCKS_LOG_INFO(immutable_db_options_.info_log, "Resuming DB"); + + InstrumentedMutexLock db_mutex(&mutex_); + + if (!error_handler_.IsDBStopped() && !error_handler_.IsBGWorkStopped()) { + // Nothing to do + return Status::OK(); + } + + if (error_handler_.IsRecoveryInProgress()) { + // Don't allow a mix of manual and automatic recovery + return Status::Busy(); + } + + mutex_.Unlock(); + Status s = error_handler_.RecoverFromBGError(true); + mutex_.Lock(); + return s; +} + +// This function implements the guts of recovery from a background error. It +// is eventually called for both manual as well as automatic recovery. It does +// the following - +// 1. Wait for currently scheduled background flush/compaction to exit, in +// order to inadvertently causing an error and thinking recovery failed +// 2. Flush memtables if there's any data for all the CFs. This may result +// another error, which will be saved by error_handler_ and reported later +// as the recovery status +// 3. Find and delete any obsolete files +// 4. Schedule compactions if needed for all the CFs. This is needed as the +// flush in the prior step might have been a no-op for some CFs, which +// means a new super version wouldn't have been installed +Status DBImpl::ResumeImpl(DBRecoverContext context) { + mutex_.AssertHeld(); + + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + WaitForBackgroundWork(); + + Status s; + if (shutdown_initiated_) { + // Returning shutdown status to SFM during auto recovery will cause it + // to abort the recovery and allow the shutdown to progress + s = Status::ShutdownInProgress(); + } + + if (s.ok()) { + Status bg_error = error_handler_.GetBGError(); + if (bg_error.severity() > Status::Severity::kHardError) { + ROCKS_LOG_INFO( + immutable_db_options_.info_log, + "DB resume requested but failed due to Fatal/Unrecoverable error"); + s = bg_error; + } + } + + // Make sure the IO Status stored in version set is set to OK. + bool file_deletion_disabled = !IsFileDeletionsEnabled(); + if (s.ok()) { + IOStatus io_s = versions_->io_status(); + if (io_s.IsIOError()) { + // If resuming from IOError resulted from MANIFEST write, then assert + // that we must have already set the MANIFEST writer to nullptr during + // clean-up phase MANIFEST writing. We must have also disabled file + // deletions. + assert(!versions_->descriptor_log_); + assert(file_deletion_disabled); + // Since we are trying to recover from MANIFEST write error, we need to + // switch to a new MANIFEST anyway. The old MANIFEST can be corrupted. + // Therefore, force writing a dummy version edit because we do not know + // whether there are flush jobs with non-empty data to flush, triggering + // appends to MANIFEST. + VersionEdit edit; + auto cfh = + static_cast_with_check(default_cf_handle_); + assert(cfh); + ColumnFamilyData* cfd = cfh->cfd(); + const MutableCFOptions& cf_opts = *cfd->GetLatestMutableCFOptions(); + s = versions_->LogAndApply(cfd, cf_opts, read_options, &edit, &mutex_, + directories_.GetDbDir()); + if (!s.ok()) { + io_s = versions_->io_status(); + if (!io_s.ok()) { + s = error_handler_.SetBGError(io_s, + BackgroundErrorReason::kManifestWrite); + } + } + } + } + + // We cannot guarantee consistency of the WAL. So force flush Memtables of + // all the column families + if (s.ok()) { + FlushOptions flush_opts; + // We allow flush to stall write since we are trying to resume from error. + flush_opts.allow_write_stall = true; + if (immutable_db_options_.atomic_flush) { + mutex_.Unlock(); + s = AtomicFlushMemTables(flush_opts, context.flush_reason); + mutex_.Lock(); + } else { + for (auto cfd : versions_->GetRefedColumnFamilySet()) { + if (cfd->IsDropped()) { + continue; + } + InstrumentedMutexUnlock u(&mutex_); + s = FlushMemTable(cfd, flush_opts, context.flush_reason); + if (!s.ok()) { + break; + } + } + } + if (!s.ok()) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "DB resume requested but failed due to Flush failure [%s]", + s.ToString().c_str()); + } + } + + JobContext job_context(0); + FindObsoleteFiles(&job_context, true); + mutex_.Unlock(); + + job_context.manifest_file_number = 1; + if (job_context.HaveSomethingToDelete()) { + PurgeObsoleteFiles(job_context); + } + job_context.Clean(); + + if (s.ok()) { + assert(versions_->io_status().ok()); + // If we reach here, we should re-enable file deletions if it was disabled + // during previous error handling. + if (file_deletion_disabled) { + // Always return ok + s = EnableFileDeletions(/*force=*/true); + if (!s.ok()) { + ROCKS_LOG_INFO( + immutable_db_options_.info_log, + "DB resume requested but could not enable file deletions [%s]", + s.ToString().c_str()); + assert(false); + } + } + } + + mutex_.Lock(); + if (s.ok()) { + // This will notify and unblock threads waiting for error recovery to + // finish. Those previouly waiting threads can now proceed, which may + // include closing the db. + s = error_handler_.ClearBGError(); + } else { + // NOTE: this is needed to pass ASSERT_STATUS_CHECKED + // in the DBSSTTest.DBWithMaxSpaceAllowedRandomized test. + // See https://github.com/facebook/rocksdb/pull/7715#issuecomment-754947952 + error_handler_.GetRecoveryError().PermitUncheckedError(); + } + + if (s.ok()) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, "Successfully resumed DB"); + } else { + ROCKS_LOG_INFO(immutable_db_options_.info_log, "Failed to resume DB [%s]", + s.ToString().c_str()); + } + + // Check for shutdown again before scheduling further compactions, + // since we released and re-acquired the lock above + if (shutdown_initiated_) { + s = Status::ShutdownInProgress(); + } + if (s.ok()) { + for (auto cfd : *versions_->GetColumnFamilySet()) { + SchedulePendingCompaction(cfd); + } + MaybeScheduleFlushOrCompaction(); + } + + // Wake up any waiters - in this case, it could be the shutdown thread + bg_cv_.SignalAll(); + + // No need to check BGError again. If something happened, event listener would + // be notified and the operation causing it would have failed + return s; +} + +void DBImpl::WaitForBackgroundWork() { + // Wait for background work to finish + while (bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ || + bg_flush_scheduled_) { + bg_cv_.Wait(); + } +} + +// Will lock the mutex_, will wait for completion if wait is true +void DBImpl::CancelAllBackgroundWork(bool wait) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "Shutdown: canceling all background work"); + + for (uint8_t task_type = 0; + task_type < static_cast(PeriodicTaskType::kMax); task_type++) { + Status s = periodic_task_scheduler_.Unregister( + static_cast(task_type)); + if (!s.ok()) { + ROCKS_LOG_WARN(immutable_db_options_.info_log, + "Failed to unregister periodic task %d, status: %s", + task_type, s.ToString().c_str()); + } + } + + InstrumentedMutexLock l(&mutex_); + if (!shutting_down_.load(std::memory_order_acquire) && + has_unpersisted_data_.load(std::memory_order_relaxed) && + !mutable_db_options_.avoid_flush_during_shutdown) { + if (immutable_db_options_.atomic_flush) { + mutex_.Unlock(); + Status s = AtomicFlushMemTables(FlushOptions(), FlushReason::kShutDown); + s.PermitUncheckedError(); //**TODO: What to do on error? + mutex_.Lock(); + } else { + for (auto cfd : versions_->GetRefedColumnFamilySet()) { + if (!cfd->IsDropped() && cfd->initialized() && !cfd->mem()->IsEmpty()) { + InstrumentedMutexUnlock u(&mutex_); + Status s = FlushMemTable(cfd, FlushOptions(), FlushReason::kShutDown); + s.PermitUncheckedError(); //**TODO: What to do on error? + } + } + } + } + + shutting_down_.store(true, std::memory_order_release); + bg_cv_.SignalAll(); + if (!wait) { + return; + } + WaitForBackgroundWork(); +} + +Status DBImpl::MaybeReleaseTimestampedSnapshotsAndCheck() { + size_t num_snapshots = 0; + ReleaseTimestampedSnapshotsOlderThan(std::numeric_limits::max(), + &num_snapshots); + + // If there is unreleased snapshot, fail the close call + if (num_snapshots > 0) { + return Status::Aborted("Cannot close DB with unreleased snapshot."); + } + + return Status::OK(); +} + +Status DBImpl::CloseHelper() { + // Guarantee that there is no background error recovery in progress before + // continuing with the shutdown + mutex_.Lock(); + shutdown_initiated_ = true; + error_handler_.CancelErrorRecovery(); + while (error_handler_.IsRecoveryInProgress()) { + bg_cv_.Wait(); + } + mutex_.Unlock(); + + // Below check is added as recovery_error_ is not checked and it causes crash + // in DBSSTTest.DBWithMaxSpaceAllowedWithBlobFiles when space limit is + // reached. + error_handler_.GetRecoveryError().PermitUncheckedError(); + + // CancelAllBackgroundWork called with false means we just set the shutdown + // marker. After this we do a variant of the waiting and unschedule work + // (to consider: moving all the waiting into CancelAllBackgroundWork(true)) + CancelAllBackgroundWork(false); + + // Cancel manual compaction if there's any + if (HasPendingManualCompaction()) { + DisableManualCompaction(); + } + mutex_.Lock(); + // Unschedule all tasks for this DB + for (uint8_t i = 0; i < static_cast(TaskType::kCount); i++) { + env_->UnSchedule(GetTaskTag(i), Env::Priority::BOTTOM); + env_->UnSchedule(GetTaskTag(i), Env::Priority::LOW); + env_->UnSchedule(GetTaskTag(i), Env::Priority::HIGH); + } + + Status ret = Status::OK(); + + // Wait for background work to finish + while (bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ || + bg_flush_scheduled_ || bg_purge_scheduled_ || + pending_purge_obsolete_files_ || + error_handler_.IsRecoveryInProgress()) { + TEST_SYNC_POINT("DBImpl::~DBImpl:WaitJob"); + bg_cv_.Wait(); + } + TEST_SYNC_POINT_CALLBACK("DBImpl::CloseHelper:PendingPurgeFinished", + &files_grabbed_for_purge_); + EraseThreadStatusDbInfo(); + flush_scheduler_.Clear(); + trim_history_scheduler_.Clear(); + + while (!flush_queue_.empty()) { + const FlushRequest& flush_req = PopFirstFromFlushQueue(); + for (const auto& iter : flush_req.cfd_to_max_mem_id_to_persist) { + iter.first->UnrefAndTryDelete(); + } + } + + while (!compaction_queue_.empty()) { + auto cfd = PopFirstFromCompactionQueue(); + cfd->UnrefAndTryDelete(); + } + + if (default_cf_handle_ != nullptr || persist_stats_cf_handle_ != nullptr) { + // we need to delete handle outside of lock because it does its own locking + mutex_.Unlock(); + if (default_cf_handle_) { + delete default_cf_handle_; + default_cf_handle_ = nullptr; + } + if (persist_stats_cf_handle_) { + delete persist_stats_cf_handle_; + persist_stats_cf_handle_ = nullptr; + } + mutex_.Lock(); + } + + // Clean up obsolete files due to SuperVersion release. + // (1) Need to delete to obsolete files before closing because RepairDB() + // scans all existing files in the file system and builds manifest file. + // Keeping obsolete files confuses the repair process. + // (2) Need to check if we Open()/Recover() the DB successfully before + // deleting because if VersionSet recover fails (may be due to corrupted + // manifest file), it is not able to identify live files correctly. As a + // result, all "live" files can get deleted by accident. However, corrupted + // manifest is recoverable by RepairDB(). + if (opened_successfully_) { + JobContext job_context(next_job_id_.fetch_add(1)); + FindObsoleteFiles(&job_context, true); + + mutex_.Unlock(); + // manifest number starting from 2 + job_context.manifest_file_number = 1; + if (job_context.HaveSomethingToDelete()) { + PurgeObsoleteFiles(job_context); + } + job_context.Clean(); + mutex_.Lock(); + } + { + InstrumentedMutexLock lock(&log_write_mutex_); + for (auto l : logs_to_free_) { + delete l; + } + for (auto& log : logs_) { + uint64_t log_number = log.writer->get_log_number(); + Status s = log.ClearWriter(); + if (!s.ok()) { + ROCKS_LOG_WARN( + immutable_db_options_.info_log, + "Unable to Sync WAL file %s with error -- %s", + LogFileName(immutable_db_options_.GetWalDir(), log_number).c_str(), + s.ToString().c_str()); + // Retain the first error + if (ret.ok()) { + ret = s; + } + } + } + logs_.clear(); + } + + // Table cache may have table handles holding blocks from the block cache. + // We need to release them before the block cache is destroyed. The block + // cache may be destroyed inside versions_.reset(), when column family data + // list is destroyed, so leaving handles in table cache after + // versions_.reset() may cause issues. + // Here we clean all unreferenced handles in table cache. + // Now we assume all user queries have finished, so only version set itself + // can possibly hold the blocks from block cache. After releasing unreferenced + // handles here, only handles held by version set left and inside + // versions_.reset(), we will release them. There, we need to make sure every + // time a handle is released, we erase it from the cache too. By doing that, + // we can guarantee that after versions_.reset(), table cache is empty + // so the cache can be safely destroyed. + table_cache_->EraseUnRefEntries(); + + for (auto& txn_entry : recovered_transactions_) { + delete txn_entry.second; + } + + // versions need to be destroyed before table_cache since it can hold + // references to table_cache. + versions_.reset(); + mutex_.Unlock(); + if (db_lock_ != nullptr) { + // TODO: Check for unlock error + env_->UnlockFile(db_lock_).PermitUncheckedError(); + } + + ROCKS_LOG_INFO(immutable_db_options_.info_log, "Shutdown complete"); + LogFlush(immutable_db_options_.info_log); + + // If the sst_file_manager was allocated by us during DB::Open(), ccall + // Close() on it before closing the info_log. Otherwise, background thread + // in SstFileManagerImpl might try to log something + if (immutable_db_options_.sst_file_manager && own_sfm_) { + auto sfm = static_cast( + immutable_db_options_.sst_file_manager.get()); + sfm->Close(); + } + + if (immutable_db_options_.info_log && own_info_log_) { + Status s = immutable_db_options_.info_log->Close(); + if (!s.ok() && !s.IsNotSupported() && ret.ok()) { + ret = s; + } + } + + if (write_buffer_manager_ && wbm_stall_) { + write_buffer_manager_->RemoveDBFromQueue(wbm_stall_.get()); + } + + IOStatus io_s = directories_.Close(IOOptions(), nullptr /* dbg */); + if (!io_s.ok()) { + ret = io_s; + } + if (ret.IsAborted()) { + // Reserve IsAborted() error for those where users didn't release + // certain resource and they can release them and come back and + // retry. In this case, we wrap this exception to something else. + return Status::Incomplete(ret.ToString()); + } + + return ret; +} + +Status DBImpl::CloseImpl() { return CloseHelper(); } + +DBImpl::~DBImpl() { + // TODO: remove this. + init_logger_creation_s_.PermitUncheckedError(); + + InstrumentedMutexLock closing_lock_guard(&closing_mutex_); + if (closed_) { + return; + } + + closed_ = true; + + { + const Status s = MaybeReleaseTimestampedSnapshotsAndCheck(); + s.PermitUncheckedError(); + } + + closing_status_ = CloseImpl(); + closing_status_.PermitUncheckedError(); +} + +void DBImpl::MaybeIgnoreError(Status* s) const { + if (s->ok() || immutable_db_options_.paranoid_checks) { + // No change needed + } else { + ROCKS_LOG_WARN(immutable_db_options_.info_log, "Ignoring error %s", + s->ToString().c_str()); + *s = Status::OK(); + } +} + +const Status DBImpl::CreateArchivalDirectory() { + if (immutable_db_options_.WAL_ttl_seconds > 0 || + immutable_db_options_.WAL_size_limit_MB > 0) { + std::string archivalPath = + ArchivalDirectory(immutable_db_options_.GetWalDir()); + return env_->CreateDirIfMissing(archivalPath); + } + return Status::OK(); +} + +void DBImpl::PrintStatistics() { + auto dbstats = immutable_db_options_.stats; + if (dbstats) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, "STATISTICS:\n %s", + dbstats->ToString().c_str()); + } +} + +Status DBImpl::StartPeriodicTaskScheduler() { + +#ifndef NDEBUG + // It only used by test to disable scheduler + bool disable_scheduler = false; + TEST_SYNC_POINT_CALLBACK( + "DBImpl::StartPeriodicTaskScheduler:DisableScheduler", + &disable_scheduler); + if (disable_scheduler) { + return Status::OK(); + } + + { + InstrumentedMutexLock l(&mutex_); + TEST_SYNC_POINT_CALLBACK("DBImpl::StartPeriodicTaskScheduler:Init", + &periodic_task_scheduler_); + } + +#endif // !NDEBUG + if (mutable_db_options_.stats_dump_period_sec > 0) { + Status s = periodic_task_scheduler_.Register( + PeriodicTaskType::kDumpStats, + periodic_task_functions_.at(PeriodicTaskType::kDumpStats), + mutable_db_options_.stats_dump_period_sec); + if (!s.ok()) { + return s; + } + } + if (mutable_db_options_.stats_persist_period_sec > 0) { + Status s = periodic_task_scheduler_.Register( + PeriodicTaskType::kPersistStats, + periodic_task_functions_.at(PeriodicTaskType::kPersistStats), + mutable_db_options_.stats_persist_period_sec); + if (!s.ok()) { + return s; + } + } + + Status s = periodic_task_scheduler_.Register( + PeriodicTaskType::kFlushInfoLog, + periodic_task_functions_.at(PeriodicTaskType::kFlushInfoLog)); + + return s; +} + +Status DBImpl::RegisterRecordSeqnoTimeWorker() { + uint64_t min_time_duration = std::numeric_limits::max(); + uint64_t max_time_duration = std::numeric_limits::min(); + { + InstrumentedMutexLock l(&mutex_); + + for (auto cfd : *versions_->GetColumnFamilySet()) { + // preserve time is the max of 2 options. + uint64_t preserve_time_duration = + std::max(cfd->ioptions()->preserve_internal_time_seconds, + cfd->ioptions()->preclude_last_level_data_seconds); + if (!cfd->IsDropped() && preserve_time_duration > 0) { + min_time_duration = std::min(preserve_time_duration, min_time_duration); + max_time_duration = std::max(preserve_time_duration, max_time_duration); + } + } + if (min_time_duration == std::numeric_limits::max()) { + seqno_time_mapping_.Resize(0, 0); + } else { + seqno_time_mapping_.Resize(min_time_duration, max_time_duration); + } + } + + uint64_t seqno_time_cadence = 0; + if (min_time_duration != std::numeric_limits::max()) { + // round up to 1 when the time_duration is smaller than + // kMaxSeqnoTimePairsPerCF + seqno_time_cadence = + (min_time_duration + SeqnoToTimeMapping::kMaxSeqnoTimePairsPerCF - 1) / + SeqnoToTimeMapping::kMaxSeqnoTimePairsPerCF; + } + + Status s; + if (seqno_time_cadence == 0) { + s = periodic_task_scheduler_.Unregister(PeriodicTaskType::kRecordSeqnoTime); + } else { + s = periodic_task_scheduler_.Register( + PeriodicTaskType::kRecordSeqnoTime, + periodic_task_functions_.at(PeriodicTaskType::kRecordSeqnoTime), + seqno_time_cadence); + } + + return s; +} + +// esitmate the total size of stats_history_ +size_t DBImpl::EstimateInMemoryStatsHistorySize() const { + size_t size_total = + sizeof(std::map>); + if (stats_history_.size() == 0) return size_total; + size_t size_per_slice = + sizeof(uint64_t) + sizeof(std::map); + // non-empty map, stats_history_.begin() guaranteed to exist + for (const auto& pairs : stats_history_.begin()->second) { + size_per_slice += + pairs.first.capacity() + sizeof(pairs.first) + sizeof(pairs.second); + } + size_total = size_per_slice * stats_history_.size(); + return size_total; +} + +void DBImpl::PersistStats() { + TEST_SYNC_POINT("DBImpl::PersistStats:Entry"); + if (shutdown_initiated_) { + return; + } + TEST_SYNC_POINT("DBImpl::PersistStats:StartRunning"); + uint64_t now_seconds = + immutable_db_options_.clock->NowMicros() / kMicrosInSecond; + + Statistics* statistics = immutable_db_options_.stats; + if (!statistics) { + return; + } + size_t stats_history_size_limit = 0; + { + InstrumentedMutexLock l(&mutex_); + stats_history_size_limit = mutable_db_options_.stats_history_buffer_size; + } + + std::map stats_map; + if (!statistics->getTickerMap(&stats_map)) { + return; + } + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "------- PERSISTING STATS -------"); + + if (immutable_db_options_.persist_stats_to_disk) { + WriteBatch batch; + Status s = Status::OK(); + if (stats_slice_initialized_) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "Reading %" ROCKSDB_PRIszt " stats from statistics\n", + stats_slice_.size()); + for (const auto& stat : stats_map) { + if (s.ok()) { + char key[100]; + int length = + EncodePersistentStatsKey(now_seconds, stat.first, 100, key); + // calculate the delta from last time + if (stats_slice_.find(stat.first) != stats_slice_.end()) { + uint64_t delta = stat.second - stats_slice_[stat.first]; + s = batch.Put(persist_stats_cf_handle_, + Slice(key, std::min(100, length)), + std::to_string(delta)); + } + } + } + } + stats_slice_initialized_ = true; + std::swap(stats_slice_, stats_map); + if (s.ok()) { + WriteOptions wo; + wo.low_pri = true; + wo.no_slowdown = true; + wo.sync = false; + s = Write(wo, &batch); + } + if (!s.ok()) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "Writing to persistent stats CF failed -- %s", + s.ToString().c_str()); + } else { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "Writing %" ROCKSDB_PRIszt " stats with timestamp %" PRIu64 + " to persistent stats CF succeeded", + stats_slice_.size(), now_seconds); + } + // TODO(Zhongyi): add purging for persisted data + } else { + InstrumentedMutexLock l(&stats_history_mutex_); + // calculate the delta from last time + if (stats_slice_initialized_) { + std::map stats_delta; + for (const auto& stat : stats_map) { + if (stats_slice_.find(stat.first) != stats_slice_.end()) { + stats_delta[stat.first] = stat.second - stats_slice_[stat.first]; + } + } + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "Storing %" ROCKSDB_PRIszt " stats with timestamp %" PRIu64 + " to in-memory stats history", + stats_slice_.size(), now_seconds); + stats_history_[now_seconds] = stats_delta; + } + stats_slice_initialized_ = true; + std::swap(stats_slice_, stats_map); + TEST_SYNC_POINT("DBImpl::PersistStats:StatsCopied"); + + // delete older stats snapshots to control memory consumption + size_t stats_history_size = EstimateInMemoryStatsHistorySize(); + bool purge_needed = stats_history_size > stats_history_size_limit; + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "[Pre-GC] In-memory stats history size: %" ROCKSDB_PRIszt + " bytes, slice count: %" ROCKSDB_PRIszt, + stats_history_size, stats_history_.size()); + while (purge_needed && !stats_history_.empty()) { + stats_history_.erase(stats_history_.begin()); + purge_needed = + EstimateInMemoryStatsHistorySize() > stats_history_size_limit; + } + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "[Post-GC] In-memory stats history size: %" ROCKSDB_PRIszt + " bytes, slice count: %" ROCKSDB_PRIszt, + stats_history_size, stats_history_.size()); + } + TEST_SYNC_POINT("DBImpl::PersistStats:End"); +} + +bool DBImpl::FindStatsByTime(uint64_t start_time, uint64_t end_time, + uint64_t* new_time, + std::map* stats_map) { + assert(new_time); + assert(stats_map); + if (!new_time || !stats_map) return false; + // lock when search for start_time + { + InstrumentedMutexLock l(&stats_history_mutex_); + auto it = stats_history_.lower_bound(start_time); + if (it != stats_history_.end() && it->first < end_time) { + // make a copy for timestamp and stats_map + *new_time = it->first; + *stats_map = it->second; + return true; + } else { + return false; + } + } +} + +Status DBImpl::GetStatsHistory( + uint64_t start_time, uint64_t end_time, + std::unique_ptr* stats_iterator) { + if (!stats_iterator) { + return Status::InvalidArgument("stats_iterator not preallocated."); + } + if (immutable_db_options_.persist_stats_to_disk) { + stats_iterator->reset( + new PersistentStatsHistoryIterator(start_time, end_time, this)); + } else { + stats_iterator->reset( + new InMemoryStatsHistoryIterator(start_time, end_time, this)); + } + return (*stats_iterator)->status(); +} + +void DBImpl::DumpStats() { + TEST_SYNC_POINT("DBImpl::DumpStats:1"); + std::string stats; + if (shutdown_initiated_) { + return; + } + + // Also probe block cache(s) for problems, dump to info log + UnorderedSet probed_caches; + TEST_SYNC_POINT("DBImpl::DumpStats:StartRunning"); + { + InstrumentedMutexLock l(&mutex_); + for (auto cfd : versions_->GetRefedColumnFamilySet()) { + if (!cfd->initialized()) { + continue; + } + + // Release DB mutex for gathering cache entry stats. Pass over all + // column families for this first so that other stats are dumped + // near-atomically. + InstrumentedMutexUnlock u(&mutex_); + cfd->internal_stats()->CollectCacheEntryStats(/*foreground=*/false); + + // Probe block cache for problems (if not already via another CF) + if (immutable_db_options_.info_log) { + auto* table_factory = cfd->ioptions()->table_factory.get(); + assert(table_factory != nullptr); + Cache* cache = + table_factory->GetOptions(TableFactory::kBlockCacheOpts()); + if (cache && probed_caches.insert(cache).second) { + cache->ReportProblems(immutable_db_options_.info_log); + } + } + } + + const std::string* property = &DB::Properties::kDBStats; + const DBPropertyInfo* property_info = GetPropertyInfo(*property); + assert(property_info != nullptr); + assert(!property_info->need_out_of_mutex); + default_cf_internal_stats_->GetStringProperty(*property_info, *property, + &stats); + + property = &InternalStats::kPeriodicCFStats; + property_info = GetPropertyInfo(*property); + assert(property_info != nullptr); + assert(!property_info->need_out_of_mutex); + for (auto cfd : *versions_->GetColumnFamilySet()) { + if (cfd->initialized()) { + cfd->internal_stats()->GetStringProperty(*property_info, *property, + &stats); + } + } + } + TEST_SYNC_POINT("DBImpl::DumpStats:2"); + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "------- DUMPING STATS -------"); + ROCKS_LOG_INFO(immutable_db_options_.info_log, "%s", stats.c_str()); + if (immutable_db_options_.dump_malloc_stats) { + stats.clear(); + DumpMallocStats(&stats); + if (!stats.empty()) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "------- Malloc STATS -------"); + ROCKS_LOG_INFO(immutable_db_options_.info_log, "%s", stats.c_str()); + } + } + + PrintStatistics(); +} + +// Periodically flush info log out of application buffer at a low frequency. +// This improves debuggability in case of RocksDB hanging since it ensures the +// log messages leading up to the hang will eventually become visible in the +// log. +void DBImpl::FlushInfoLog() { + if (shutdown_initiated_) { + return; + } + TEST_SYNC_POINT("DBImpl::FlushInfoLog:StartRunning"); + LogFlush(immutable_db_options_.info_log); +} + +Status DBImpl::TablesRangeTombstoneSummary(ColumnFamilyHandle* column_family, + int max_entries_to_print, + std::string* out_str) { + auto* cfh = static_cast_with_check(column_family); + ColumnFamilyData* cfd = cfh->cfd(); + + SuperVersion* super_version = cfd->GetReferencedSuperVersion(this); + Version* version = super_version->current; + + Status s = + version->TablesRangeTombstoneSummary(max_entries_to_print, out_str); + + CleanupSuperVersion(super_version); + return s; +} + +void DBImpl::ScheduleBgLogWriterClose(JobContext* job_context) { + mutex_.AssertHeld(); + if (!job_context->logs_to_free.empty()) { + for (auto l : job_context->logs_to_free) { + AddToLogsToFreeQueue(l); + } + job_context->logs_to_free.clear(); + } +} + +FSDirectory* DBImpl::GetDataDir(ColumnFamilyData* cfd, size_t path_id) const { + assert(cfd); + FSDirectory* ret_dir = cfd->GetDataDir(path_id); + if (ret_dir == nullptr) { + return directories_.GetDataDir(path_id); + } + return ret_dir; +} + +Status DBImpl::SetOptions( + ColumnFamilyHandle* column_family, + const std::unordered_map& options_map) { + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + auto* cfd = + static_cast_with_check(column_family)->cfd(); + if (options_map.empty()) { + ROCKS_LOG_WARN(immutable_db_options_.info_log, + "SetOptions() on column family [%s], empty input", + cfd->GetName().c_str()); + return Status::InvalidArgument("empty input"); + } + + MutableCFOptions new_options; + Status s; + Status persist_options_status; + SuperVersionContext sv_context(/* create_superversion */ true); + { + auto db_options = GetDBOptions(); + InstrumentedMutexLock l(&mutex_); + s = cfd->SetOptions(db_options, options_map); + if (s.ok()) { + new_options = *cfd->GetLatestMutableCFOptions(); + // Append new version to recompute compaction score. + VersionEdit dummy_edit; + s = versions_->LogAndApply(cfd, new_options, read_options, &dummy_edit, + &mutex_, directories_.GetDbDir()); + // Trigger possible flush/compactions. This has to be before we persist + // options to file, otherwise there will be a deadlock with writer + // thread. + InstallSuperVersionAndScheduleWork(cfd, &sv_context, new_options); + + persist_options_status = WriteOptionsFile( + false /*need_mutex_lock*/, true /*need_enter_write_thread*/); + bg_cv_.SignalAll(); + } + } + sv_context.Clean(); + + ROCKS_LOG_INFO( + immutable_db_options_.info_log, + "SetOptions() on column family [%s], inputs:", cfd->GetName().c_str()); + for (const auto& o : options_map) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, "%s: %s\n", o.first.c_str(), + o.second.c_str()); + } + if (s.ok()) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "[%s] SetOptions() succeeded", cfd->GetName().c_str()); + new_options.Dump(immutable_db_options_.info_log.get()); + if (!persist_options_status.ok()) { + // NOTE: WriteOptionsFile already logs on failure + s = persist_options_status; + } + } else { + persist_options_status.PermitUncheckedError(); // less important + ROCKS_LOG_WARN(immutable_db_options_.info_log, "[%s] SetOptions() failed", + cfd->GetName().c_str()); + } + LogFlush(immutable_db_options_.info_log); + return s; +} + +Status DBImpl::SetDBOptions( + const std::unordered_map& options_map) { + if (options_map.empty()) { + ROCKS_LOG_WARN(immutable_db_options_.info_log, + "SetDBOptions(), empty input."); + return Status::InvalidArgument("empty input"); + } + + MutableDBOptions new_options; + Status s; + Status persist_options_status = Status::OK(); + bool wal_changed = false; + WriteContext write_context; + { + InstrumentedMutexLock l(&mutex_); + s = GetMutableDBOptionsFromStrings(mutable_db_options_, options_map, + &new_options); + + if (new_options.bytes_per_sync == 0) { + new_options.bytes_per_sync = 1024 * 1024; + } + + if (MutableDBOptionsAreEqual(mutable_db_options_, new_options)) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "SetDBOptions(), input option value is not changed, " + "skipping updating."); + persist_options_status.PermitUncheckedError(); + return s; + } + + DBOptions new_db_options = + BuildDBOptions(immutable_db_options_, new_options); + if (s.ok()) { + s = ValidateOptions(new_db_options); + } + if (s.ok()) { + for (auto c : *versions_->GetColumnFamilySet()) { + if (!c->IsDropped()) { + auto cf_options = c->GetLatestCFOptions(); + s = ColumnFamilyData::ValidateOptions(new_db_options, cf_options); + if (!s.ok()) { + break; + } + } + } + } + if (s.ok()) { + const BGJobLimits current_bg_job_limits = + GetBGJobLimits(mutable_db_options_.max_background_flushes, + mutable_db_options_.max_background_compactions, + mutable_db_options_.max_background_jobs, + /* parallelize_compactions */ true); + const BGJobLimits new_bg_job_limits = GetBGJobLimits( + new_options.max_background_flushes, + new_options.max_background_compactions, + new_options.max_background_jobs, /* parallelize_compactions */ true); + + const bool max_flushes_increased = + new_bg_job_limits.max_flushes > current_bg_job_limits.max_flushes; + const bool max_compactions_increased = + new_bg_job_limits.max_compactions > + current_bg_job_limits.max_compactions; + + if (max_flushes_increased || max_compactions_increased) { + if (max_flushes_increased) { + env_->IncBackgroundThreadsIfNeeded(new_bg_job_limits.max_flushes, + Env::Priority::HIGH); + } + + if (max_compactions_increased) { + env_->IncBackgroundThreadsIfNeeded(new_bg_job_limits.max_compactions, + Env::Priority::LOW); + } + + MaybeScheduleFlushOrCompaction(); + } + + mutex_.Unlock(); + if (new_options.stats_dump_period_sec == 0) { + s = periodic_task_scheduler_.Unregister(PeriodicTaskType::kDumpStats); + } else { + s = periodic_task_scheduler_.Register( + PeriodicTaskType::kDumpStats, + periodic_task_functions_.at(PeriodicTaskType::kDumpStats), + new_options.stats_dump_period_sec); + } + if (new_options.max_total_wal_size != + mutable_db_options_.max_total_wal_size) { + max_total_wal_size_.store(new_options.max_total_wal_size, + std::memory_order_release); + } + if (s.ok()) { + if (new_options.stats_persist_period_sec == 0) { + s = periodic_task_scheduler_.Unregister( + PeriodicTaskType::kPersistStats); + } else { + s = periodic_task_scheduler_.Register( + PeriodicTaskType::kPersistStats, + periodic_task_functions_.at(PeriodicTaskType::kPersistStats), + new_options.stats_persist_period_sec); + } + } + mutex_.Lock(); + if (!s.ok()) { + return s; + } + + write_controller_.set_max_delayed_write_rate( + new_options.delayed_write_rate); + table_cache_.get()->SetCapacity(new_options.max_open_files == -1 + ? TableCache::kInfiniteCapacity + : new_options.max_open_files - 10); + wal_changed = mutable_db_options_.wal_bytes_per_sync != + new_options.wal_bytes_per_sync; + mutable_db_options_ = new_options; + file_options_for_compaction_ = FileOptions(new_db_options); + file_options_for_compaction_ = fs_->OptimizeForCompactionTableWrite( + file_options_for_compaction_, immutable_db_options_); + versions_->ChangeFileOptions(mutable_db_options_); + // TODO(xiez): clarify why apply optimize for read to write options + file_options_for_compaction_ = fs_->OptimizeForCompactionTableRead( + file_options_for_compaction_, immutable_db_options_); + file_options_for_compaction_.compaction_readahead_size = + mutable_db_options_.compaction_readahead_size; + WriteThread::Writer w; + write_thread_.EnterUnbatched(&w, &mutex_); + if (total_log_size_ > GetMaxTotalWalSize() || wal_changed) { + Status purge_wal_status = SwitchWAL(&write_context); + if (!purge_wal_status.ok()) { + ROCKS_LOG_WARN(immutable_db_options_.info_log, + "Unable to purge WAL files in SetDBOptions() -- %s", + purge_wal_status.ToString().c_str()); + } + } + persist_options_status = WriteOptionsFile( + false /*need_mutex_lock*/, false /*need_enter_write_thread*/); + write_thread_.ExitUnbatched(&w); + } else { + // To get here, we must have had invalid options and will not attempt to + // persist the options, which means the status is "OK/Uninitialized. + persist_options_status.PermitUncheckedError(); + } + } + ROCKS_LOG_INFO(immutable_db_options_.info_log, "SetDBOptions(), inputs:"); + for (const auto& o : options_map) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, "%s: %s\n", o.first.c_str(), + o.second.c_str()); + } + if (s.ok()) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, "SetDBOptions() succeeded"); + new_options.Dump(immutable_db_options_.info_log.get()); + if (!persist_options_status.ok()) { + if (immutable_db_options_.fail_if_options_file_error) { + s = Status::IOError( + "SetDBOptions() succeeded, but unable to persist options", + persist_options_status.ToString()); + } + ROCKS_LOG_WARN(immutable_db_options_.info_log, + "Unable to persist options in SetDBOptions() -- %s", + persist_options_status.ToString().c_str()); + } + } else { + ROCKS_LOG_WARN(immutable_db_options_.info_log, "SetDBOptions failed"); + } + LogFlush(immutable_db_options_.info_log); + return s; +} + +// return the same level if it cannot be moved +int DBImpl::FindMinimumEmptyLevelFitting( + ColumnFamilyData* cfd, const MutableCFOptions& /*mutable_cf_options*/, + int level) { + mutex_.AssertHeld(); + const auto* vstorage = cfd->current()->storage_info(); + int minimum_level = level; + for (int i = level - 1; i > 0; --i) { + // stop if level i is not empty + if (vstorage->NumLevelFiles(i) > 0) break; + // stop if level i is too small (cannot fit the level files) + if (vstorage->MaxBytesForLevel(i) < vstorage->NumLevelBytes(level)) { + break; + } + + minimum_level = i; + } + return minimum_level; +} + +Status DBImpl::FlushWAL(bool sync) { + if (manual_wal_flush_) { + IOStatus io_s; + { + // We need to lock log_write_mutex_ since logs_ might change concurrently + InstrumentedMutexLock wl(&log_write_mutex_); + log::Writer* cur_log_writer = logs_.back().writer; + io_s = cur_log_writer->WriteBuffer(); + } + if (!io_s.ok()) { + ROCKS_LOG_ERROR(immutable_db_options_.info_log, "WAL flush error %s", + io_s.ToString().c_str()); + // In case there is a fs error we should set it globally to prevent the + // future writes + IOStatusCheck(io_s); + // whether sync or not, we should abort the rest of function upon error + return static_cast(io_s); + } + if (!sync) { + ROCKS_LOG_DEBUG(immutable_db_options_.info_log, "FlushWAL sync=false"); + return static_cast(io_s); + } + } + if (!sync) { + return Status::OK(); + } + // sync = true + ROCKS_LOG_DEBUG(immutable_db_options_.info_log, "FlushWAL sync=true"); + return SyncWAL(); +} + +bool DBImpl::WALBufferIsEmpty() { + InstrumentedMutexLock l(&log_write_mutex_); + log::Writer* cur_log_writer = logs_.back().writer; + auto res = cur_log_writer->BufferIsEmpty(); + return res; +} + +Status DBImpl::SyncWAL() { + TEST_SYNC_POINT("DBImpl::SyncWAL:Begin"); + autovector logs_to_sync; + bool need_log_dir_sync; + uint64_t current_log_number; + + { + InstrumentedMutexLock l(&log_write_mutex_); + assert(!logs_.empty()); + + // This SyncWAL() call only cares about logs up to this number. + current_log_number = logfile_number_; + + while (logs_.front().number <= current_log_number && + logs_.front().IsSyncing()) { + log_sync_cv_.Wait(); + } + // First check that logs are safe to sync in background. + for (auto it = logs_.begin(); + it != logs_.end() && it->number <= current_log_number; ++it) { + if (!it->writer->file()->writable_file()->IsSyncThreadSafe()) { + return Status::NotSupported( + "SyncWAL() is not supported for this implementation of WAL file", + immutable_db_options_.allow_mmap_writes + ? "try setting Options::allow_mmap_writes to false" + : Slice()); + } + } + for (auto it = logs_.begin(); + it != logs_.end() && it->number <= current_log_number; ++it) { + auto& log = *it; + log.PrepareForSync(); + logs_to_sync.push_back(log.writer); + } + + need_log_dir_sync = !log_dir_synced_; + } + + TEST_SYNC_POINT("DBWALTest::SyncWALNotWaitWrite:1"); + RecordTick(stats_, WAL_FILE_SYNCED); + Status status; + IOStatus io_s; + for (log::Writer* log : logs_to_sync) { + io_s = log->file()->SyncWithoutFlush(immutable_db_options_.use_fsync); + if (!io_s.ok()) { + status = io_s; + break; + } + } + if (!io_s.ok()) { + ROCKS_LOG_ERROR(immutable_db_options_.info_log, "WAL Sync error %s", + io_s.ToString().c_str()); + // In case there is a fs error we should set it globally to prevent the + // future writes + IOStatusCheck(io_s); + } + if (status.ok() && need_log_dir_sync) { + status = directories_.GetWalDir()->FsyncWithDirOptions( + IOOptions(), nullptr, + DirFsyncOptions(DirFsyncOptions::FsyncReason::kNewFileSynced)); + } + TEST_SYNC_POINT("DBWALTest::SyncWALNotWaitWrite:2"); + + TEST_SYNC_POINT("DBImpl::SyncWAL:BeforeMarkLogsSynced:1"); + VersionEdit synced_wals; + { + InstrumentedMutexLock l(&log_write_mutex_); + if (status.ok()) { + MarkLogsSynced(current_log_number, need_log_dir_sync, &synced_wals); + } else { + MarkLogsNotSynced(current_log_number); + } + } + if (status.ok() && synced_wals.IsWalAddition()) { + InstrumentedMutexLock l(&mutex_); + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + status = ApplyWALToManifest(read_options, &synced_wals); + } + + TEST_SYNC_POINT("DBImpl::SyncWAL:BeforeMarkLogsSynced:2"); + + return status; +} + +Status DBImpl::ApplyWALToManifest(const ReadOptions& read_options, + VersionEdit* synced_wals) { + // not empty, write to MANIFEST. + mutex_.AssertHeld(); + + Status status = versions_->LogAndApplyToDefaultColumnFamily( + read_options, synced_wals, &mutex_, directories_.GetDbDir()); + if (!status.ok() && versions_->io_status().IsIOError()) { + status = error_handler_.SetBGError(versions_->io_status(), + BackgroundErrorReason::kManifestWrite); + } + return status; +} + +Status DBImpl::LockWAL() { + { + InstrumentedMutexLock lock(&mutex_); + if (lock_wal_count_ > 0) { + assert(lock_wal_write_token_); + ++lock_wal_count_; + } else { + // NOTE: this will "unnecessarily" wait for other non-LockWAL() write + // stalls to clear before LockWAL returns, however fixing that would + // not be simple because if we notice the primary queue is already + // stalled, that stall might clear while we release DB mutex in + // EnterUnbatched() for the nonmem queue. And if we work around that in + // the naive way, we could deadlock by locking the two queues in different + // orders. + + WriteThread::Writer w; + write_thread_.EnterUnbatched(&w, &mutex_); + WriteThread::Writer nonmem_w; + if (two_write_queues_) { + nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_); + } + + // NOTE: releasing mutex in EnterUnbatched might mean we are actually + // now lock_wal_count > 0 + if (lock_wal_count_ == 0) { + assert(!lock_wal_write_token_); + lock_wal_write_token_ = write_controller_.GetStopToken(); + } + ++lock_wal_count_; + + if (two_write_queues_) { + nonmem_write_thread_.ExitUnbatched(&nonmem_w); + } + write_thread_.ExitUnbatched(&w); + } + } + // NOTE: avoid I/O holding DB mutex + Status s = FlushWAL(/*sync=*/false); + if (!s.ok()) { + // Non-OK return should not be in locked state + UnlockWAL().PermitUncheckedError(); + } + return s; +} + +Status DBImpl::UnlockWAL() { + bool signal = false; + uint64_t maybe_stall_begun_count = 0; + uint64_t nonmem_maybe_stall_begun_count = 0; + { + InstrumentedMutexLock lock(&mutex_); + if (lock_wal_count_ == 0) { + return Status::Aborted("No LockWAL() in effect"); + } + --lock_wal_count_; + if (lock_wal_count_ == 0) { + lock_wal_write_token_.reset(); + signal = true; + // For the last UnlockWAL, we don't want to return from UnlockWAL() + // until the thread(s) that called BeginWriteStall() have had a chance to + // call EndWriteStall(), so that no_slowdown writes after UnlockWAL() are + // guaranteed to succeed if there's no other source of stall. + maybe_stall_begun_count = write_thread_.GetBegunCountOfOutstandingStall(); + if (two_write_queues_) { + nonmem_maybe_stall_begun_count = + nonmem_write_thread_.GetBegunCountOfOutstandingStall(); + } + } + } + if (signal) { + // SignalAll outside of mutex for efficiency + bg_cv_.SignalAll(); + } + // Ensure stalls have cleared + if (maybe_stall_begun_count) { + write_thread_.WaitForStallEndedCount(maybe_stall_begun_count); + } + if (nonmem_maybe_stall_begun_count) { + nonmem_write_thread_.WaitForStallEndedCount(nonmem_maybe_stall_begun_count); + } + return Status::OK(); +} + +void DBImpl::MarkLogsSynced(uint64_t up_to, bool synced_dir, + VersionEdit* synced_wals) { + log_write_mutex_.AssertHeld(); + if (synced_dir && logfile_number_ == up_to) { + log_dir_synced_ = true; + } + for (auto it = logs_.begin(); it != logs_.end() && it->number <= up_to;) { + auto& wal = *it; + assert(wal.IsSyncing()); + + if (wal.number < logs_.back().number) { + // Inactive WAL + if (immutable_db_options_.track_and_verify_wals_in_manifest && + wal.GetPreSyncSize() > 0) { + synced_wals->AddWal(wal.number, WalMetadata(wal.GetPreSyncSize())); + } + if (wal.GetPreSyncSize() == wal.writer->file()->GetFlushedSize()) { + // Fully synced + logs_to_free_.push_back(wal.ReleaseWriter()); + it = logs_.erase(it); + } else { + assert(wal.GetPreSyncSize() < wal.writer->file()->GetFlushedSize()); + wal.FinishSync(); + ++it; + } + } else { + assert(wal.number == logs_.back().number); + // Active WAL + wal.FinishSync(); + ++it; + } + } + log_sync_cv_.SignalAll(); +} + +void DBImpl::MarkLogsNotSynced(uint64_t up_to) { + log_write_mutex_.AssertHeld(); + for (auto it = logs_.begin(); it != logs_.end() && it->number <= up_to; + ++it) { + auto& wal = *it; + wal.FinishSync(); + } + log_sync_cv_.SignalAll(); +} + +SequenceNumber DBImpl::GetLatestSequenceNumber() const { + return versions_->LastSequence(); +} + +void DBImpl::SetLastPublishedSequence(SequenceNumber seq) { + versions_->SetLastPublishedSequence(seq); +} + +Status DBImpl::GetFullHistoryTsLow(ColumnFamilyHandle* column_family, + std::string* ts_low) { + if (ts_low == nullptr) { + return Status::InvalidArgument("ts_low is nullptr"); + } + ColumnFamilyData* cfd = nullptr; + if (column_family == nullptr) { + cfd = default_cf_handle_->cfd(); + } else { + auto cfh = static_cast_with_check(column_family); + assert(cfh != nullptr); + cfd = cfh->cfd(); + } + assert(cfd != nullptr && cfd->user_comparator() != nullptr); + if (cfd->user_comparator()->timestamp_size() == 0) { + return Status::InvalidArgument( + "Timestamp is not enabled in this column family"); + } + InstrumentedMutexLock l(&mutex_); + *ts_low = cfd->GetFullHistoryTsLow(); + assert(cfd->user_comparator()->timestamp_size() == ts_low->size()); + return Status::OK(); +} + +InternalIterator* DBImpl::NewInternalIterator(const ReadOptions& read_options, + Arena* arena, + SequenceNumber sequence, + ColumnFamilyHandle* column_family, + bool allow_unprepared_value) { + ColumnFamilyData* cfd; + if (column_family == nullptr) { + cfd = default_cf_handle_->cfd(); + } else { + auto cfh = static_cast_with_check(column_family); + cfd = cfh->cfd(); + } + + mutex_.Lock(); + SuperVersion* super_version = cfd->GetSuperVersion()->Ref(); + mutex_.Unlock(); + return NewInternalIterator(read_options, cfd, super_version, arena, sequence, + allow_unprepared_value); +} + +void DBImpl::SchedulePurge() { + mutex_.AssertHeld(); + assert(opened_successfully_); + + // Purge operations are put into High priority queue + bg_purge_scheduled_++; + env_->Schedule(&DBImpl::BGWorkPurge, this, Env::Priority::HIGH, nullptr); +} + +void DBImpl::BackgroundCallPurge() { + mutex_.Lock(); + + while (!logs_to_free_queue_.empty()) { + assert(!logs_to_free_queue_.empty()); + log::Writer* log_writer = *(logs_to_free_queue_.begin()); + logs_to_free_queue_.pop_front(); + mutex_.Unlock(); + delete log_writer; + mutex_.Lock(); + } + while (!superversions_to_free_queue_.empty()) { + assert(!superversions_to_free_queue_.empty()); + SuperVersion* sv = superversions_to_free_queue_.front(); + superversions_to_free_queue_.pop_front(); + mutex_.Unlock(); + delete sv; + mutex_.Lock(); + } + + assert(bg_purge_scheduled_ > 0); + + // Can't use iterator to go over purge_files_ because inside the loop we're + // unlocking the mutex that protects purge_files_. + while (!purge_files_.empty()) { + auto it = purge_files_.begin(); + // Need to make a copy of the PurgeFilesInfo before unlocking the mutex. + PurgeFileInfo purge_file = it->second; + + const std::string& fname = purge_file.fname; + const std::string& dir_to_sync = purge_file.dir_to_sync; + FileType type = purge_file.type; + uint64_t number = purge_file.number; + int job_id = purge_file.job_id; + + purge_files_.erase(it); + + mutex_.Unlock(); + DeleteObsoleteFileImpl(job_id, fname, dir_to_sync, type, number); + mutex_.Lock(); + } + + bg_purge_scheduled_--; + + bg_cv_.SignalAll(); + // IMPORTANT:there should be no code after calling SignalAll. This call may + // signal the DB destructor that it's OK to proceed with destruction. In + // that case, all DB variables will be dealloacated and referencing them + // will cause trouble. + mutex_.Unlock(); +} + +namespace { + +// A `SuperVersionHandle` holds a non-null `SuperVersion*` pointing at a +// `SuperVersion` referenced once for this object. It also contains the state +// needed to clean up the `SuperVersion` reference from outside of `DBImpl` +// using `CleanupSuperVersionHandle()`. +struct SuperVersionHandle { + // `_super_version` must be non-nullptr and `Ref()`'d once as long as the + // `SuperVersionHandle` may use it. + SuperVersionHandle(DBImpl* _db, InstrumentedMutex* _mu, + SuperVersion* _super_version, bool _background_purge) + : db(_db), + mu(_mu), + super_version(_super_version), + background_purge(_background_purge) {} + + DBImpl* db; + InstrumentedMutex* mu; + SuperVersion* super_version; + bool background_purge; +}; + +static void CleanupSuperVersionHandle(void* arg1, void* /*arg2*/) { + SuperVersionHandle* sv_handle = reinterpret_cast(arg1); + + if (sv_handle->super_version->Unref()) { + // Job id == 0 means that this is not our background process, but rather + // user thread + JobContext job_context(0); + + sv_handle->mu->Lock(); + sv_handle->super_version->Cleanup(); + sv_handle->db->FindObsoleteFiles(&job_context, false, true); + if (sv_handle->background_purge) { + sv_handle->db->ScheduleBgLogWriterClose(&job_context); + sv_handle->db->AddSuperVersionsToFreeQueue(sv_handle->super_version); + sv_handle->db->SchedulePurge(); + } + sv_handle->mu->Unlock(); + + if (!sv_handle->background_purge) { + delete sv_handle->super_version; + } + if (job_context.HaveSomethingToDelete()) { + sv_handle->db->PurgeObsoleteFiles(job_context, + sv_handle->background_purge); + } + job_context.Clean(); + } + + delete sv_handle; +} + +struct GetMergeOperandsState { + MergeContext merge_context; + PinnedIteratorsManager pinned_iters_mgr; + SuperVersionHandle* sv_handle; +}; + +static void CleanupGetMergeOperandsState(void* arg1, void* /*arg2*/) { + GetMergeOperandsState* state = static_cast(arg1); + CleanupSuperVersionHandle(state->sv_handle /* arg1 */, nullptr /* arg2 */); + delete state; +} + +} // namespace + +InternalIterator* DBImpl::NewInternalIterator( + const ReadOptions& read_options, ColumnFamilyData* cfd, + SuperVersion* super_version, Arena* arena, SequenceNumber sequence, + bool allow_unprepared_value, ArenaWrappedDBIter* db_iter) { + InternalIterator* internal_iter; + assert(arena != nullptr); + // Need to create internal iterator from the arena. + MergeIteratorBuilder merge_iter_builder( + &cfd->internal_comparator(), arena, + !read_options.total_order_seek && + super_version->mutable_cf_options.prefix_extractor != nullptr, + read_options.iterate_upper_bound); + // Collect iterator for mutable memtable + auto mem_iter = super_version->mem->NewIterator(read_options, arena); + Status s; + if (!read_options.ignore_range_deletions) { + TruncatedRangeDelIterator* mem_tombstone_iter = nullptr; + auto range_del_iter = super_version->mem->NewRangeTombstoneIterator( + read_options, sequence, false /* immutable_memtable */); + if (range_del_iter == nullptr || range_del_iter->empty()) { + delete range_del_iter; + } else { + mem_tombstone_iter = new TruncatedRangeDelIterator( + std::unique_ptr(range_del_iter), + &cfd->ioptions()->internal_comparator, nullptr /* smallest */, + nullptr /* largest */); + } + merge_iter_builder.AddPointAndTombstoneIterator(mem_iter, + mem_tombstone_iter); + } else { + merge_iter_builder.AddIterator(mem_iter); + } + + // Collect all needed child iterators for immutable memtables + if (s.ok()) { + super_version->imm->AddIterators(read_options, &merge_iter_builder, + !read_options.ignore_range_deletions); + } + TEST_SYNC_POINT_CALLBACK("DBImpl::NewInternalIterator:StatusCallback", &s); + if (s.ok()) { + // Collect iterators for files in L0 - Ln + if (read_options.read_tier != kMemtableTier) { + super_version->current->AddIterators(read_options, file_options_, + &merge_iter_builder, + allow_unprepared_value); + } + internal_iter = merge_iter_builder.Finish( + read_options.ignore_range_deletions ? nullptr : db_iter); + SuperVersionHandle* cleanup = new SuperVersionHandle( + this, &mutex_, super_version, + read_options.background_purge_on_iterator_cleanup || + immutable_db_options_.avoid_unnecessary_blocking_io); + internal_iter->RegisterCleanup(CleanupSuperVersionHandle, cleanup, nullptr); + + return internal_iter; + } else { + CleanupSuperVersion(super_version); + } + return NewErrorInternalIterator(s, arena); +} + +ColumnFamilyHandle* DBImpl::DefaultColumnFamily() const { + return default_cf_handle_; +} + +ColumnFamilyHandle* DBImpl::PersistentStatsColumnFamily() const { + return persist_stats_cf_handle_; +} + +Status DBImpl::Get(const ReadOptions& read_options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value) { + return Get(read_options, column_family, key, value, /*timestamp=*/nullptr); +} + +Status DBImpl::Get(const ReadOptions& read_options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value, std::string* timestamp) { + assert(value != nullptr); + value->Reset(); + GetImplOptions get_impl_options; + get_impl_options.column_family = column_family; + get_impl_options.value = value; + get_impl_options.timestamp = timestamp; + Status s = GetImpl(read_options, key, get_impl_options); + return s; +} + +Status DBImpl::GetEntity(const ReadOptions& read_options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableWideColumns* columns) { + if (!column_family) { + return Status::InvalidArgument( + "Cannot call GetEntity without a column family handle"); + } + + if (!columns) { + return Status::InvalidArgument( + "Cannot call GetEntity without a PinnableWideColumns object"); + } + + if (read_options.io_activity != Env::IOActivity::kUnknown) { + return Status::InvalidArgument( + "Cannot call GetEntity with `ReadOptions::io_activity` != " + "`Env::IOActivity::kUnknown`"); + } + + columns->Reset(); + + GetImplOptions get_impl_options; + get_impl_options.column_family = column_family; + get_impl_options.columns = columns; + + return GetImpl(read_options, key, get_impl_options); +} + +bool DBImpl::ShouldReferenceSuperVersion(const MergeContext& merge_context) { + // If both thresholds are reached, a function returning merge operands as + // `PinnableSlice`s should reference the `SuperVersion` to avoid large and/or + // numerous `memcpy()`s. + // + // The below constants enable the optimization conservatively. They are + // verified to not regress `GetMergeOperands()` latency in the following + // scenarios. + // + // - CPU: two socket Intel(R) Xeon(R) Gold 6138 CPU @ 2.00GHz + // - `GetMergeOperands()` threads: 1 - 32 + // - Entry size: 32 bytes - 4KB + // - Merges per key: 1 - 16K + // - LSM component: memtable + // + // TODO(ajkr): expand measurement to SST files. + static const size_t kNumBytesForSvRef = 32768; + static const size_t kLog2AvgBytesForSvRef = 8; // 256 bytes + + size_t num_bytes = 0; + for (const Slice& sl : merge_context.GetOperands()) { + num_bytes += sl.size(); + } + return num_bytes >= kNumBytesForSvRef && + (num_bytes >> kLog2AvgBytesForSvRef) >= + merge_context.GetOperands().size(); +} + +Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key, + GetImplOptions& get_impl_options) { + assert(get_impl_options.value != nullptr || + get_impl_options.merge_operands != nullptr || + get_impl_options.columns != nullptr); + + assert(get_impl_options.column_family); + + if (read_options.io_activity != Env::IOActivity::kUnknown) { + return Status::InvalidArgument( + "Cannot call Get with `ReadOptions::io_activity` != " + "`Env::IOActivity::kUnknown`"); + } + + if (read_options.timestamp) { + const Status s = FailIfTsMismatchCf(get_impl_options.column_family, + *(read_options.timestamp), + /*ts_for_read=*/true); + if (!s.ok()) { + return s; + } + } else { + const Status s = FailIfCfHasTs(get_impl_options.column_family); + if (!s.ok()) { + return s; + } + } + + // Clear the timestamps for returning results so that we can distinguish + // between tombstone or key that has never been written + if (get_impl_options.timestamp) { + get_impl_options.timestamp->clear(); + } + + GetWithTimestampReadCallback read_cb(0); // Will call Refresh + + PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock); + StopWatch sw(immutable_db_options_.clock, stats_, DB_GET); + PERF_TIMER_GUARD(get_snapshot_time); + + auto cfh = static_cast_with_check( + get_impl_options.column_family); + auto cfd = cfh->cfd(); + + if (tracer_) { + // TODO: This mutex should be removed later, to improve performance when + // tracing is enabled. + InstrumentedMutexLock lock(&trace_mutex_); + if (tracer_) { + // TODO: maybe handle the tracing status? + tracer_->Get(get_impl_options.column_family, key).PermitUncheckedError(); + } + } + + if (get_impl_options.get_merge_operands_options != nullptr) { + for (int i = 0; i < get_impl_options.get_merge_operands_options + ->expected_max_number_of_operands; + ++i) { + get_impl_options.merge_operands[i].Reset(); + } + } + + // Acquire SuperVersion + SuperVersion* sv = GetAndRefSuperVersion(cfd); + + TEST_SYNC_POINT("DBImpl::GetImpl:1"); + TEST_SYNC_POINT("DBImpl::GetImpl:2"); + + SequenceNumber snapshot; + if (read_options.snapshot != nullptr) { + if (get_impl_options.callback) { + // Already calculated based on read_options.snapshot + snapshot = get_impl_options.callback->max_visible_seq(); + } else { + snapshot = + reinterpret_cast(read_options.snapshot)->number_; + } + } else { + // Note that the snapshot is assigned AFTER referencing the super + // version because otherwise a flush happening in between may compact away + // data for the snapshot, so the reader would see neither data that was be + // visible to the snapshot before compaction nor the newer data inserted + // afterwards. + snapshot = GetLastPublishedSequence(); + if (get_impl_options.callback) { + // The unprep_seqs are not published for write unprepared, so it could be + // that max_visible_seq is larger. Seek to the std::max of the two. + // However, we still want our callback to contain the actual snapshot so + // that it can do the correct visibility filtering. + get_impl_options.callback->Refresh(snapshot); + + // Internally, WriteUnpreparedTxnReadCallback::Refresh would set + // max_visible_seq = max(max_visible_seq, snapshot) + // + // Currently, the commented out assert is broken by + // InvalidSnapshotReadCallback, but if write unprepared recovery followed + // the regular transaction flow, then this special read callback would not + // be needed. + // + // assert(callback->max_visible_seq() >= snapshot); + snapshot = get_impl_options.callback->max_visible_seq(); + } + } + // If timestamp is used, we use read callback to ensure is returned + // only if t <= read_opts.timestamp and s <= snapshot. + // HACK: temporarily overwrite input struct field but restore + SaveAndRestore restore_callback(&get_impl_options.callback); + const Comparator* ucmp = get_impl_options.column_family->GetComparator(); + assert(ucmp); + if (ucmp->timestamp_size() > 0) { + assert(!get_impl_options + .callback); // timestamp with callback is not supported + read_cb.Refresh(snapshot); + get_impl_options.callback = &read_cb; + } + TEST_SYNC_POINT("DBImpl::GetImpl:3"); + TEST_SYNC_POINT("DBImpl::GetImpl:4"); + + // Prepare to store a list of merge operations if merge occurs. + MergeContext merge_context; + SequenceNumber max_covering_tombstone_seq = 0; + + Status s; + // First look in the memtable, then in the immutable memtable (if any). + // s is both in/out. When in, s could either be OK or MergeInProgress. + // merge_operands will contain the sequence of merges in the latter case. + LookupKey lkey(key, snapshot, read_options.timestamp); + PERF_TIMER_STOP(get_snapshot_time); + + bool skip_memtable = (read_options.read_tier == kPersistedTier && + has_unpersisted_data_.load(std::memory_order_relaxed)); + bool done = false; + std::string* timestamp = + ucmp->timestamp_size() > 0 ? get_impl_options.timestamp : nullptr; + if (!skip_memtable) { + // Get value associated with key + if (get_impl_options.get_value) { + if (sv->mem->Get( + lkey, + get_impl_options.value ? get_impl_options.value->GetSelf() + : nullptr, + get_impl_options.columns, timestamp, &s, &merge_context, + &max_covering_tombstone_seq, read_options, + false /* immutable_memtable */, get_impl_options.callback, + get_impl_options.is_blob_index)) { + done = true; + + if (get_impl_options.value) { + get_impl_options.value->PinSelf(); + } + + RecordTick(stats_, MEMTABLE_HIT); + } else if ((s.ok() || s.IsMergeInProgress()) && + sv->imm->Get(lkey, + get_impl_options.value + ? get_impl_options.value->GetSelf() + : nullptr, + get_impl_options.columns, timestamp, &s, + &merge_context, &max_covering_tombstone_seq, + read_options, get_impl_options.callback, + get_impl_options.is_blob_index)) { + done = true; + + if (get_impl_options.value) { + get_impl_options.value->PinSelf(); + } + + RecordTick(stats_, MEMTABLE_HIT); + } + } else { + // Get Merge Operands associated with key, Merge Operands should not be + // merged and raw values should be returned to the user. + if (sv->mem->Get(lkey, /*value=*/nullptr, /*columns=*/nullptr, + /*timestamp=*/nullptr, &s, &merge_context, + &max_covering_tombstone_seq, read_options, + false /* immutable_memtable */, nullptr, nullptr, + false)) { + done = true; + RecordTick(stats_, MEMTABLE_HIT); + } else if ((s.ok() || s.IsMergeInProgress()) && + sv->imm->GetMergeOperands(lkey, &s, &merge_context, + &max_covering_tombstone_seq, + read_options)) { + done = true; + RecordTick(stats_, MEMTABLE_HIT); + } + } + if (!done && !s.ok() && !s.IsMergeInProgress()) { + ReturnAndCleanupSuperVersion(cfd, sv); + return s; + } + } + TEST_SYNC_POINT("DBImpl::GetImpl:PostMemTableGet:0"); + TEST_SYNC_POINT("DBImpl::GetImpl:PostMemTableGet:1"); + PinnedIteratorsManager pinned_iters_mgr; + if (!done) { + PERF_TIMER_GUARD(get_from_output_files_time); + sv->current->Get( + read_options, lkey, get_impl_options.value, get_impl_options.columns, + timestamp, &s, &merge_context, &max_covering_tombstone_seq, + &pinned_iters_mgr, + get_impl_options.get_value ? get_impl_options.value_found : nullptr, + nullptr, nullptr, + get_impl_options.get_value ? get_impl_options.callback : nullptr, + get_impl_options.get_value ? get_impl_options.is_blob_index : nullptr, + get_impl_options.get_value); + RecordTick(stats_, MEMTABLE_MISS); + } + + { + PERF_TIMER_GUARD(get_post_process_time); + + RecordTick(stats_, NUMBER_KEYS_READ); + size_t size = 0; + if (s.ok()) { + if (get_impl_options.get_value) { + if (get_impl_options.value) { + size = get_impl_options.value->size(); + } else if (get_impl_options.columns) { + size = get_impl_options.columns->serialized_size(); + } + } else { + // Return all merge operands for get_impl_options.key + *get_impl_options.number_of_operands = + static_cast(merge_context.GetNumOperands()); + if (*get_impl_options.number_of_operands > + get_impl_options.get_merge_operands_options + ->expected_max_number_of_operands) { + s = Status::Incomplete( + Status::SubCode::KMergeOperandsInsufficientCapacity); + } else { + // Each operand depends on one of the following resources: `sv`, + // `pinned_iters_mgr`, or `merge_context`. It would be crazy expensive + // to reference `sv` for each operand relying on it because `sv` is + // (un)ref'd in all threads using the DB. Furthermore, we do not track + // on which resource each operand depends. + // + // To solve this, we bundle the resources in a `GetMergeOperandsState` + // and manage them with a `SharedCleanablePtr` shared among the + // `PinnableSlice`s we return. This bundle includes one `sv` reference + // and ownership of the `merge_context` and `pinned_iters_mgr` + // objects. + bool ref_sv = ShouldReferenceSuperVersion(merge_context); + if (ref_sv) { + assert(!merge_context.GetOperands().empty()); + SharedCleanablePtr shared_cleanable; + GetMergeOperandsState* state = nullptr; + state = new GetMergeOperandsState(); + state->merge_context = std::move(merge_context); + state->pinned_iters_mgr = std::move(pinned_iters_mgr); + + sv->Ref(); + + state->sv_handle = new SuperVersionHandle( + this, &mutex_, sv, + immutable_db_options_.avoid_unnecessary_blocking_io); + + shared_cleanable.Allocate(); + shared_cleanable->RegisterCleanup(CleanupGetMergeOperandsState, + state /* arg1 */, + nullptr /* arg2 */); + for (size_t i = 0; i < state->merge_context.GetOperands().size(); + ++i) { + const Slice& sl = state->merge_context.GetOperands()[i]; + size += sl.size(); + + get_impl_options.merge_operands->PinSlice( + sl, nullptr /* cleanable */); + if (i == state->merge_context.GetOperands().size() - 1) { + shared_cleanable.MoveAsCleanupTo( + get_impl_options.merge_operands); + } else { + shared_cleanable.RegisterCopyWith( + get_impl_options.merge_operands); + } + get_impl_options.merge_operands++; + } + } else { + for (const Slice& sl : merge_context.GetOperands()) { + size += sl.size(); + get_impl_options.merge_operands->PinSelf(sl); + get_impl_options.merge_operands++; + } + } + } + } + RecordTick(stats_, BYTES_READ, size); + PERF_COUNTER_ADD(get_read_bytes, size); + } + + ReturnAndCleanupSuperVersion(cfd, sv); + + RecordInHistogram(stats_, BYTES_PER_READ, size); + } + return s; +} + +std::vector DBImpl::MultiGet( + const ReadOptions& read_options, + const std::vector& column_family, + const std::vector& keys, std::vector* values) { + return MultiGet(read_options, column_family, keys, values, + /*timestamps=*/nullptr); +} + +std::vector DBImpl::MultiGet( + const ReadOptions& read_options, + const std::vector& column_family, + const std::vector& keys, std::vector* values, + std::vector* timestamps) { + PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock); + StopWatch sw(immutable_db_options_.clock, stats_, DB_MULTIGET); + PERF_TIMER_GUARD(get_snapshot_time); + + size_t num_keys = keys.size(); + assert(column_family.size() == num_keys); + std::vector stat_list(num_keys); + + bool should_fail = false; + for (size_t i = 0; i < num_keys; ++i) { + assert(column_family[i]); + if (read_options.timestamp) { + stat_list[i] = FailIfTsMismatchCf( + column_family[i], *(read_options.timestamp), /*ts_for_read=*/true); + if (!stat_list[i].ok()) { + should_fail = true; + } + } else { + stat_list[i] = FailIfCfHasTs(column_family[i]); + if (!stat_list[i].ok()) { + should_fail = true; + } + } + } + + if (should_fail) { + for (auto& s : stat_list) { + if (s.ok()) { + s = Status::Incomplete( + "DB not queried due to invalid argument(s) in the same MultiGet"); + } + } + return stat_list; + } + + if (tracer_) { + // TODO: This mutex should be removed later, to improve performance when + // tracing is enabled. + InstrumentedMutexLock lock(&trace_mutex_); + if (tracer_) { + // TODO: maybe handle the tracing status? + tracer_->MultiGet(column_family, keys).PermitUncheckedError(); + } + } + + SequenceNumber consistent_seqnum; + + UnorderedMap multiget_cf_data( + column_family.size()); + for (auto cf : column_family) { + auto cfh = static_cast_with_check(cf); + auto cfd = cfh->cfd(); + if (multiget_cf_data.find(cfd->GetID()) == multiget_cf_data.end()) { + multiget_cf_data.emplace(cfd->GetID(), + MultiGetColumnFamilyData(cfh, nullptr)); + } + } + + std::function::iterator&)> + iter_deref_lambda = + [](UnorderedMap::iterator& + cf_iter) { return &cf_iter->second; }; + + bool unref_only = + MultiCFSnapshot>( + read_options, nullptr, iter_deref_lambda, &multiget_cf_data, + &consistent_seqnum); + + TEST_SYNC_POINT("DBImpl::MultiGet:AfterGetSeqNum1"); + TEST_SYNC_POINT("DBImpl::MultiGet:AfterGetSeqNum2"); + + // Contain a list of merge operations if merge occurs. + MergeContext merge_context; + + // Note: this always resizes the values array + values->resize(num_keys); + if (timestamps) { + timestamps->resize(num_keys); + } + + // Keep track of bytes that we read for statistics-recording later + uint64_t bytes_read = 0; + PERF_TIMER_STOP(get_snapshot_time); + + // For each of the given keys, apply the entire "get" process as follows: + // First look in the memtable, then in the immutable memtable (if any). + // s is both in/out. When in, s could either be OK or MergeInProgress. + // merge_operands will contain the sequence of merges in the latter case. + size_t num_found = 0; + size_t keys_read; + uint64_t curr_value_size = 0; + + GetWithTimestampReadCallback timestamp_read_callback(0); + ReadCallback* read_callback = nullptr; + if (read_options.timestamp && read_options.timestamp->size() > 0) { + timestamp_read_callback.Refresh(consistent_seqnum); + read_callback = ×tamp_read_callback; + } + + for (keys_read = 0; keys_read < num_keys; ++keys_read) { + merge_context.Clear(); + Status& s = stat_list[keys_read]; + std::string* value = &(*values)[keys_read]; + std::string* timestamp = timestamps ? &(*timestamps)[keys_read] : nullptr; + + LookupKey lkey(keys[keys_read], consistent_seqnum, read_options.timestamp); + auto cfh = static_cast_with_check( + column_family[keys_read]); + SequenceNumber max_covering_tombstone_seq = 0; + auto mgd_iter = multiget_cf_data.find(cfh->cfd()->GetID()); + assert(mgd_iter != multiget_cf_data.end()); + auto mgd = mgd_iter->second; + auto super_version = mgd.super_version; + bool skip_memtable = + (read_options.read_tier == kPersistedTier && + has_unpersisted_data_.load(std::memory_order_relaxed)); + bool done = false; + if (!skip_memtable) { + if (super_version->mem->Get( + lkey, value, /*columns=*/nullptr, timestamp, &s, &merge_context, + &max_covering_tombstone_seq, read_options, + false /* immutable_memtable */, read_callback)) { + done = true; + RecordTick(stats_, MEMTABLE_HIT); + } else if (super_version->imm->Get(lkey, value, /*columns=*/nullptr, + timestamp, &s, &merge_context, + &max_covering_tombstone_seq, + read_options, read_callback)) { + done = true; + RecordTick(stats_, MEMTABLE_HIT); + } + } + if (!done) { + PinnableSlice pinnable_val; + PERF_TIMER_GUARD(get_from_output_files_time); + PinnedIteratorsManager pinned_iters_mgr; + super_version->current->Get(read_options, lkey, &pinnable_val, + /*columns=*/nullptr, timestamp, &s, + &merge_context, &max_covering_tombstone_seq, + &pinned_iters_mgr, /*value_found=*/nullptr, + /*key_exists=*/nullptr, + /*seq=*/nullptr, read_callback); + value->assign(pinnable_val.data(), pinnable_val.size()); + RecordTick(stats_, MEMTABLE_MISS); + } + + if (s.ok()) { + bytes_read += value->size(); + num_found++; + curr_value_size += value->size(); + if (curr_value_size > read_options.value_size_soft_limit) { + while (++keys_read < num_keys) { + stat_list[keys_read] = Status::Aborted(); + } + break; + } + } + if (read_options.deadline.count() && + immutable_db_options_.clock->NowMicros() > + static_cast(read_options.deadline.count())) { + break; + } + } + + if (keys_read < num_keys) { + // The only reason to break out of the loop is when the deadline is + // exceeded + assert(immutable_db_options_.clock->NowMicros() > + static_cast(read_options.deadline.count())); + for (++keys_read; keys_read < num_keys; ++keys_read) { + stat_list[keys_read] = Status::TimedOut(); + } + } + + // Post processing (decrement reference counts and record statistics) + PERF_TIMER_GUARD(get_post_process_time); + + for (auto mgd_iter : multiget_cf_data) { + auto mgd = mgd_iter.second; + if (!unref_only) { + ReturnAndCleanupSuperVersion(mgd.cfd, mgd.super_version); + } else { + mgd.cfd->GetSuperVersion()->Unref(); + } + } + RecordTick(stats_, NUMBER_MULTIGET_CALLS); + RecordTick(stats_, NUMBER_MULTIGET_KEYS_READ, num_keys); + RecordTick(stats_, NUMBER_MULTIGET_KEYS_FOUND, num_found); + RecordTick(stats_, NUMBER_MULTIGET_BYTES_READ, bytes_read); + RecordInHistogram(stats_, BYTES_PER_MULTIGET, bytes_read); + PERF_COUNTER_ADD(multiget_read_bytes, bytes_read); + PERF_TIMER_STOP(get_post_process_time); + + return stat_list; +} + +template +bool DBImpl::MultiCFSnapshot( + const ReadOptions& read_options, ReadCallback* callback, + std::function& + iter_deref_func, + T* cf_list, SequenceNumber* snapshot) { + PERF_TIMER_GUARD(get_snapshot_time); + + bool last_try = false; + if (cf_list->size() == 1) { + // Fast path for a single column family. We can simply get the thread loca + // super version + auto cf_iter = cf_list->begin(); + auto node = iter_deref_func(cf_iter); + node->super_version = GetAndRefSuperVersion(node->cfd); + if (read_options.snapshot != nullptr) { + // Note: In WritePrepared txns this is not necessary but not harmful + // either. Because prep_seq > snapshot => commit_seq > snapshot so if + // a snapshot is specified we should be fine with skipping seq numbers + // that are greater than that. + // + // In WriteUnprepared, we cannot set snapshot in the lookup key because we + // may skip uncommitted data that should be visible to the transaction for + // reading own writes. + *snapshot = + static_cast(read_options.snapshot)->number_; + if (callback) { + *snapshot = std::max(*snapshot, callback->max_visible_seq()); + } + } else { + // Since we get and reference the super version before getting + // the snapshot number, without a mutex protection, it is possible + // that a memtable switch happened in the middle and not all the + // data for this snapshot is available. But it will contain all + // the data available in the super version we have, which is also + // a valid snapshot to read from. + // We shouldn't get snapshot before finding and referencing the super + // version because a flush happening in between may compact away data for + // the snapshot, but the snapshot is earlier than the data overwriting it, + // so users may see wrong results. + *snapshot = GetLastPublishedSequence(); + } + } else { + // If we end up with the same issue of memtable geting sealed during 2 + // consecutive retries, it means the write rate is very high. In that case + // its probably ok to take the mutex on the 3rd try so we can succeed for + // sure + constexpr int num_retries = 3; + for (int i = 0; i < num_retries; ++i) { + last_try = (i == num_retries - 1); + bool retry = false; + + if (i > 0) { + for (auto cf_iter = cf_list->begin(); cf_iter != cf_list->end(); + ++cf_iter) { + auto node = iter_deref_func(cf_iter); + SuperVersion* super_version = node->super_version; + ColumnFamilyData* cfd = node->cfd; + if (super_version != nullptr) { + ReturnAndCleanupSuperVersion(cfd, super_version); + } + node->super_version = nullptr; + } + } + if (read_options.snapshot == nullptr) { + if (last_try) { + TEST_SYNC_POINT("DBImpl::MultiGet::LastTry"); + // We're close to max number of retries. For the last retry, + // acquire the lock so we're sure to succeed + mutex_.Lock(); + } + *snapshot = GetLastPublishedSequence(); + } else { + *snapshot = + static_cast_with_check(read_options.snapshot) + ->number_; + } + for (auto cf_iter = cf_list->begin(); cf_iter != cf_list->end(); + ++cf_iter) { + auto node = iter_deref_func(cf_iter); + if (!last_try) { + node->super_version = GetAndRefSuperVersion(node->cfd); + } else { + node->super_version = node->cfd->GetSuperVersion()->Ref(); + } + TEST_SYNC_POINT("DBImpl::MultiGet::AfterRefSV"); + if (read_options.snapshot != nullptr || last_try) { + // If user passed a snapshot, then we don't care if a memtable is + // sealed or compaction happens because the snapshot would ensure + // that older key versions are kept around. If this is the last + // retry, then we have the lock so nothing bad can happen + continue; + } + // We could get the earliest sequence number for the whole list of + // memtables, which will include immutable memtables as well, but that + // might be tricky to maintain in case we decide, in future, to do + // memtable compaction. + if (!last_try) { + SequenceNumber seq = + node->super_version->mem->GetEarliestSequenceNumber(); + if (seq > *snapshot) { + retry = true; + break; + } + } + } + if (!retry) { + if (last_try) { + mutex_.Unlock(); + } + break; + } + } + } + + // Keep track of bytes that we read for statistics-recording later + PERF_TIMER_STOP(get_snapshot_time); + + return last_try; +} + +void DBImpl::MultiGet(const ReadOptions& read_options, const size_t num_keys, + ColumnFamilyHandle** column_families, const Slice* keys, + PinnableSlice* values, Status* statuses, + const bool sorted_input) { + MultiGet(read_options, num_keys, column_families, keys, values, + /* timestamps */ nullptr, statuses, sorted_input); +} + +void DBImpl::MultiGet(const ReadOptions& read_options, const size_t num_keys, + ColumnFamilyHandle** column_families, const Slice* keys, + PinnableSlice* values, std::string* timestamps, + Status* statuses, const bool sorted_input) { + MultiGetCommon(read_options, num_keys, column_families, keys, values, + /* columns */ nullptr, timestamps, statuses, sorted_input); +} + +void DBImpl::MultiGetCommon(const ReadOptions& read_options, + const size_t num_keys, + ColumnFamilyHandle** column_families, + const Slice* keys, PinnableSlice* values, + PinnableWideColumns* columns, + std::string* timestamps, Status* statuses, + const bool sorted_input) { + if (num_keys == 0) { + return; + } + + bool should_fail = false; + for (size_t i = 0; i < num_keys; ++i) { + ColumnFamilyHandle* cfh = column_families[i]; + assert(cfh); + if (read_options.timestamp) { + statuses[i] = FailIfTsMismatchCf(cfh, *(read_options.timestamp), + /*ts_for_read=*/true); + if (!statuses[i].ok()) { + should_fail = true; + } + } else { + statuses[i] = FailIfCfHasTs(cfh); + if (!statuses[i].ok()) { + should_fail = true; + } + } + } + if (should_fail) { + for (size_t i = 0; i < num_keys; ++i) { + if (statuses[i].ok()) { + statuses[i] = Status::Incomplete( + "DB not queried due to invalid argument(s) in the same MultiGet"); + } + } + return; + } + + if (tracer_) { + // TODO: This mutex should be removed later, to improve performance when + // tracing is enabled. + InstrumentedMutexLock lock(&trace_mutex_); + if (tracer_) { + // TODO: maybe handle the tracing status? + tracer_->MultiGet(num_keys, column_families, keys).PermitUncheckedError(); + } + } + + autovector key_context; + autovector sorted_keys; + sorted_keys.resize(num_keys); + for (size_t i = 0; i < num_keys; ++i) { + PinnableSlice* val = nullptr; + PinnableWideColumns* col = nullptr; + + if (values) { + val = &values[i]; + val->Reset(); + } else { + assert(columns); + + col = &columns[i]; + col->Reset(); + } + + key_context.emplace_back(column_families[i], keys[i], val, col, + timestamps ? ×tamps[i] : nullptr, + &statuses[i]); + } + for (size_t i = 0; i < num_keys; ++i) { + sorted_keys[i] = &key_context[i]; + } + PrepareMultiGetKeys(num_keys, sorted_input, &sorted_keys); + + autovector + multiget_cf_data; + size_t cf_start = 0; + ColumnFamilyHandle* cf = sorted_keys[0]->column_family; + + for (size_t i = 0; i < num_keys; ++i) { + KeyContext* key_ctx = sorted_keys[i]; + if (key_ctx->column_family != cf) { + multiget_cf_data.emplace_back(cf, cf_start, i - cf_start, nullptr); + cf_start = i; + cf = key_ctx->column_family; + } + } + + multiget_cf_data.emplace_back(cf, cf_start, num_keys - cf_start, nullptr); + + std::function::iterator&)> + iter_deref_lambda = + [](autovector::iterator& cf_iter) { + return &(*cf_iter); + }; + + SequenceNumber consistent_seqnum; + bool unref_only = MultiCFSnapshot< + autovector>( + read_options, nullptr, iter_deref_lambda, &multiget_cf_data, + &consistent_seqnum); + + GetWithTimestampReadCallback timestamp_read_callback(0); + ReadCallback* read_callback = nullptr; + if (read_options.timestamp && read_options.timestamp->size() > 0) { + timestamp_read_callback.Refresh(consistent_seqnum); + read_callback = ×tamp_read_callback; + } + + Status s; + auto cf_iter = multiget_cf_data.begin(); + for (; cf_iter != multiget_cf_data.end(); ++cf_iter) { + s = MultiGetImpl(read_options, cf_iter->start, cf_iter->num_keys, + &sorted_keys, cf_iter->super_version, consistent_seqnum, + read_callback); + if (!s.ok()) { + break; + } + } + if (!s.ok()) { + assert(s.IsTimedOut() || s.IsAborted()); + for (++cf_iter; cf_iter != multiget_cf_data.end(); ++cf_iter) { + for (size_t i = cf_iter->start; i < cf_iter->start + cf_iter->num_keys; + ++i) { + *sorted_keys[i]->s = s; + } + } + } + + for (const auto& iter : multiget_cf_data) { + if (!unref_only) { + ReturnAndCleanupSuperVersion(iter.cfd, iter.super_version); + } else { + iter.cfd->GetSuperVersion()->Unref(); + } + } +} + +namespace { +// Order keys by CF ID, followed by key contents +struct CompareKeyContext { + inline bool operator()(const KeyContext* lhs, const KeyContext* rhs) { + ColumnFamilyHandleImpl* cfh = + static_cast(lhs->column_family); + uint32_t cfd_id1 = cfh->cfd()->GetID(); + const Comparator* comparator = cfh->cfd()->user_comparator(); + cfh = static_cast(rhs->column_family); + uint32_t cfd_id2 = cfh->cfd()->GetID(); + + if (cfd_id1 < cfd_id2) { + return true; + } else if (cfd_id1 > cfd_id2) { + return false; + } + + // Both keys are from the same column family + int cmp = comparator->CompareWithoutTimestamp( + *(lhs->key), /*a_has_ts=*/false, *(rhs->key), /*b_has_ts=*/false); + if (cmp < 0) { + return true; + } + return false; + } +}; + +} // anonymous namespace + +void DBImpl::PrepareMultiGetKeys( + size_t num_keys, bool sorted_input, + autovector* sorted_keys) { + if (sorted_input) { +#ifndef NDEBUG + assert(std::is_sorted(sorted_keys->begin(), sorted_keys->end(), + CompareKeyContext())); +#endif + return; + } + + std::sort(sorted_keys->begin(), sorted_keys->begin() + num_keys, + CompareKeyContext()); +} + +void DBImpl::MultiGet(const ReadOptions& read_options, + ColumnFamilyHandle* column_family, const size_t num_keys, + const Slice* keys, PinnableSlice* values, + Status* statuses, const bool sorted_input) { + MultiGet(read_options, column_family, num_keys, keys, values, + /* timestamps */ nullptr, statuses, sorted_input); +} + +void DBImpl::MultiGet(const ReadOptions& read_options, + ColumnFamilyHandle* column_family, const size_t num_keys, + const Slice* keys, PinnableSlice* values, + std::string* timestamps, Status* statuses, + const bool sorted_input) { + MultiGetCommon(read_options, column_family, num_keys, keys, values, + /* columns */ nullptr, timestamps, statuses, sorted_input); +} + +void DBImpl::MultiGetCommon(const ReadOptions& read_options, + ColumnFamilyHandle* column_family, + const size_t num_keys, const Slice* keys, + PinnableSlice* values, PinnableWideColumns* columns, + std::string* timestamps, Status* statuses, + bool sorted_input) { + if (tracer_) { + // TODO: This mutex should be removed later, to improve performance when + // tracing is enabled. + InstrumentedMutexLock lock(&trace_mutex_); + if (tracer_) { + // TODO: maybe handle the tracing status? + tracer_->MultiGet(num_keys, column_family, keys).PermitUncheckedError(); + } + } + autovector key_context; + autovector sorted_keys; + sorted_keys.resize(num_keys); + for (size_t i = 0; i < num_keys; ++i) { + PinnableSlice* val = nullptr; + PinnableWideColumns* col = nullptr; + + if (values) { + val = &values[i]; + val->Reset(); + } else { + assert(columns); + + col = &columns[i]; + col->Reset(); + } + + key_context.emplace_back(column_family, keys[i], val, col, + timestamps ? ×tamps[i] : nullptr, + &statuses[i]); + } + for (size_t i = 0; i < num_keys; ++i) { + sorted_keys[i] = &key_context[i]; + } + PrepareMultiGetKeys(num_keys, sorted_input, &sorted_keys); + MultiGetWithCallback(read_options, column_family, nullptr, &sorted_keys); +} + +void DBImpl::MultiGetWithCallback( + const ReadOptions& read_options, ColumnFamilyHandle* column_family, + ReadCallback* callback, + autovector* sorted_keys) { + std::array multiget_cf_data; + multiget_cf_data[0] = MultiGetColumnFamilyData(column_family, nullptr); + std::function::iterator&)> + iter_deref_lambda = + [](std::array::iterator& cf_iter) { + return &(*cf_iter); + }; + + size_t num_keys = sorted_keys->size(); + SequenceNumber consistent_seqnum; + bool unref_only = MultiCFSnapshot>( + read_options, callback, iter_deref_lambda, &multiget_cf_data, + &consistent_seqnum); +#ifndef NDEBUG + assert(!unref_only); +#else + // Silence unused variable warning + (void)unref_only; +#endif // NDEBUG + + if (callback && read_options.snapshot == nullptr) { + // The unprep_seqs are not published for write unprepared, so it could be + // that max_visible_seq is larger. Seek to the std::max of the two. + // However, we still want our callback to contain the actual snapshot so + // that it can do the correct visibility filtering. + callback->Refresh(consistent_seqnum); + + // Internally, WriteUnpreparedTxnReadCallback::Refresh would set + // max_visible_seq = max(max_visible_seq, snapshot) + // + // Currently, the commented out assert is broken by + // InvalidSnapshotReadCallback, but if write unprepared recovery followed + // the regular transaction flow, then this special read callback would not + // be needed. + // + // assert(callback->max_visible_seq() >= snapshot); + consistent_seqnum = callback->max_visible_seq(); + } + + GetWithTimestampReadCallback timestamp_read_callback(0); + ReadCallback* read_callback = callback; + if (read_options.timestamp && read_options.timestamp->size() > 0) { + assert(!read_callback); // timestamp with callback is not supported + timestamp_read_callback.Refresh(consistent_seqnum); + read_callback = ×tamp_read_callback; + } + + Status s = MultiGetImpl(read_options, 0, num_keys, sorted_keys, + multiget_cf_data[0].super_version, consistent_seqnum, + read_callback); + assert(s.ok() || s.IsTimedOut() || s.IsAborted()); + ReturnAndCleanupSuperVersion(multiget_cf_data[0].cfd, + multiget_cf_data[0].super_version); +} + +// The actual implementation of batched MultiGet. Parameters - +// start_key - Index in the sorted_keys vector to start processing from +// num_keys - Number of keys to lookup, starting with sorted_keys[start_key] +// sorted_keys - The entire batch of sorted keys for this CF +// +// The per key status is returned in the KeyContext structures pointed to by +// sorted_keys. An overall Status is also returned, with the only possible +// values being Status::OK() and Status::TimedOut(). The latter indicates +// that the call exceeded read_options.deadline +Status DBImpl::MultiGetImpl( + const ReadOptions& read_options, size_t start_key, size_t num_keys, + autovector* sorted_keys, + SuperVersion* super_version, SequenceNumber snapshot, + ReadCallback* callback) { + if (read_options.io_activity != Env::IOActivity::kUnknown) { + return Status::InvalidArgument( + "Cannot call MultiGet with `ReadOptions::io_activity` != " + "`Env::IOActivity::kUnknown`"); + } + PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock); + StopWatch sw(immutable_db_options_.clock, stats_, DB_MULTIGET); + + assert(sorted_keys); + // Clear the timestamps for returning results so that we can distinguish + // between tombstone or key that has never been written + for (auto* kctx : *sorted_keys) { + assert(kctx); + if (kctx->timestamp) { + kctx->timestamp->clear(); + } + } + + // For each of the given keys, apply the entire "get" process as follows: + // First look in the memtable, then in the immutable memtable (if any). + // s is both in/out. When in, s could either be OK or MergeInProgress. + // merge_operands will contain the sequence of merges in the latter case. + size_t keys_left = num_keys; + Status s; + uint64_t curr_value_size = 0; + while (keys_left) { + if (read_options.deadline.count() && + immutable_db_options_.clock->NowMicros() > + static_cast(read_options.deadline.count())) { + s = Status::TimedOut(); + break; + } + + size_t batch_size = (keys_left > MultiGetContext::MAX_BATCH_SIZE) + ? MultiGetContext::MAX_BATCH_SIZE + : keys_left; + MultiGetContext ctx(sorted_keys, start_key + num_keys - keys_left, + batch_size, snapshot, read_options, GetFileSystem(), + stats_); + MultiGetRange range = ctx.GetMultiGetRange(); + range.AddValueSize(curr_value_size); + bool lookup_current = false; + + keys_left -= batch_size; + for (auto mget_iter = range.begin(); mget_iter != range.end(); + ++mget_iter) { + mget_iter->merge_context.Clear(); + *mget_iter->s = Status::OK(); + } + + bool skip_memtable = + (read_options.read_tier == kPersistedTier && + has_unpersisted_data_.load(std::memory_order_relaxed)); + if (!skip_memtable) { + super_version->mem->MultiGet(read_options, &range, callback, + false /* immutable_memtable */); + if (!range.empty()) { + super_version->imm->MultiGet(read_options, &range, callback); + } + if (!range.empty()) { + lookup_current = true; + uint64_t left = range.KeysLeft(); + RecordTick(stats_, MEMTABLE_MISS, left); + } + } + if (lookup_current) { + PERF_TIMER_GUARD(get_from_output_files_time); + super_version->current->MultiGet(read_options, &range, callback); + } + curr_value_size = range.GetValueSize(); + if (curr_value_size > read_options.value_size_soft_limit) { + s = Status::Aborted(); + break; + } + } + + // Post processing (decrement reference counts and record statistics) + PERF_TIMER_GUARD(get_post_process_time); + size_t num_found = 0; + uint64_t bytes_read = 0; + for (size_t i = start_key; i < start_key + num_keys - keys_left; ++i) { + KeyContext* key = (*sorted_keys)[i]; + assert(key); + assert(key->s); + + if (key->s->ok()) { + if (key->value) { + bytes_read += key->value->size(); + } else { + assert(key->columns); + bytes_read += key->columns->serialized_size(); + } + + num_found++; + } + } + if (keys_left) { + assert(s.IsTimedOut() || s.IsAborted()); + for (size_t i = start_key + num_keys - keys_left; i < start_key + num_keys; + ++i) { + KeyContext* key = (*sorted_keys)[i]; + *key->s = s; + } + } + + RecordTick(stats_, NUMBER_MULTIGET_CALLS); + RecordTick(stats_, NUMBER_MULTIGET_KEYS_READ, num_keys); + RecordTick(stats_, NUMBER_MULTIGET_KEYS_FOUND, num_found); + RecordTick(stats_, NUMBER_MULTIGET_BYTES_READ, bytes_read); + RecordInHistogram(stats_, BYTES_PER_MULTIGET, bytes_read); + PERF_COUNTER_ADD(multiget_read_bytes, bytes_read); + PERF_TIMER_STOP(get_post_process_time); + + return s; +} + +void DBImpl::MultiGetEntity(const ReadOptions& options, size_t num_keys, + ColumnFamilyHandle** column_families, + const Slice* keys, PinnableWideColumns* results, + Status* statuses, bool sorted_input) { + MultiGetCommon(options, num_keys, column_families, keys, /* values */ nullptr, + results, /* timestamps */ nullptr, statuses, sorted_input); +} + +void DBImpl::MultiGetEntity(const ReadOptions& options, + ColumnFamilyHandle* column_family, size_t num_keys, + const Slice* keys, PinnableWideColumns* results, + Status* statuses, bool sorted_input) { + MultiGetCommon(options, column_family, num_keys, keys, /* values */ nullptr, + results, /* timestamps */ nullptr, statuses, sorted_input); +} + +Status DBImpl::CreateColumnFamily(const ColumnFamilyOptions& cf_options, + const std::string& column_family, + ColumnFamilyHandle** handle) { + assert(handle != nullptr); + Status s = CreateColumnFamilyImpl(cf_options, column_family, handle); + if (s.ok()) { + s = WriteOptionsFile(true /*need_mutex_lock*/, + true /*need_enter_write_thread*/); + } + return s; +} + +Status DBImpl::CreateColumnFamilies( + const ColumnFamilyOptions& cf_options, + const std::vector& column_family_names, + std::vector* handles) { + assert(handles != nullptr); + handles->clear(); + size_t num_cf = column_family_names.size(); + Status s; + bool success_once = false; + for (size_t i = 0; i < num_cf; i++) { + ColumnFamilyHandle* handle; + s = CreateColumnFamilyImpl(cf_options, column_family_names[i], &handle); + if (!s.ok()) { + break; + } + handles->push_back(handle); + success_once = true; + } + if (success_once) { + Status persist_options_status = WriteOptionsFile( + true /*need_mutex_lock*/, true /*need_enter_write_thread*/); + if (s.ok() && !persist_options_status.ok()) { + s = persist_options_status; + } + } + return s; +} + +Status DBImpl::CreateColumnFamilies( + const std::vector& column_families, + std::vector* handles) { + assert(handles != nullptr); + handles->clear(); + size_t num_cf = column_families.size(); + Status s; + bool success_once = false; + for (size_t i = 0; i < num_cf; i++) { + ColumnFamilyHandle* handle; + s = CreateColumnFamilyImpl(column_families[i].options, + column_families[i].name, &handle); + if (!s.ok()) { + break; + } + handles->push_back(handle); + success_once = true; + } + if (success_once) { + Status persist_options_status = WriteOptionsFile( + true /*need_mutex_lock*/, true /*need_enter_write_thread*/); + if (s.ok() && !persist_options_status.ok()) { + s = persist_options_status; + } + } + return s; +} + +Status DBImpl::CreateColumnFamilyImpl(const ColumnFamilyOptions& cf_options, + const std::string& column_family_name, + ColumnFamilyHandle** handle) { + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + Status s; + *handle = nullptr; + + DBOptions db_options = + BuildDBOptions(immutable_db_options_, mutable_db_options_); + s = ColumnFamilyData::ValidateOptions(db_options, cf_options); + if (s.ok()) { + for (auto& cf_path : cf_options.cf_paths) { + s = env_->CreateDirIfMissing(cf_path.path); + if (!s.ok()) { + break; + } + } + } + if (!s.ok()) { + return s; + } + + SuperVersionContext sv_context(/* create_superversion */ true); + { + InstrumentedMutexLock l(&mutex_); + + if (versions_->GetColumnFamilySet()->GetColumnFamily(column_family_name) != + nullptr) { + return Status::InvalidArgument("Column family already exists"); + } + VersionEdit edit; + edit.AddColumnFamily(column_family_name); + uint32_t new_id = versions_->GetColumnFamilySet()->GetNextColumnFamilyID(); + edit.SetColumnFamily(new_id); + edit.SetLogNumber(logfile_number_); + edit.SetComparatorName(cf_options.comparator->Name()); + edit.SetPersistUserDefinedTimestamps( + cf_options.persist_user_defined_timestamps); + + // LogAndApply will both write the creation in MANIFEST and create + // ColumnFamilyData object + { // write thread + WriteThread::Writer w; + write_thread_.EnterUnbatched(&w, &mutex_); + // LogAndApply will both write the creation in MANIFEST and create + // ColumnFamilyData object + s = versions_->LogAndApply(nullptr, MutableCFOptions(cf_options), + read_options, &edit, &mutex_, + directories_.GetDbDir(), false, &cf_options); + write_thread_.ExitUnbatched(&w); + } + if (s.ok()) { + auto* cfd = + versions_->GetColumnFamilySet()->GetColumnFamily(column_family_name); + assert(cfd != nullptr); + std::map> dummy_created_dirs; + s = cfd->AddDirectories(&dummy_created_dirs); + } + if (s.ok()) { + auto* cfd = + versions_->GetColumnFamilySet()->GetColumnFamily(column_family_name); + assert(cfd != nullptr); + InstallSuperVersionAndScheduleWork(cfd, &sv_context, + *cfd->GetLatestMutableCFOptions()); + + if (!cfd->mem()->IsSnapshotSupported()) { + is_snapshot_supported_ = false; + } + + cfd->set_initialized(); + + *handle = new ColumnFamilyHandleImpl(cfd, this, &mutex_); + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "Created column family [%s] (ID %u)", + column_family_name.c_str(), (unsigned)cfd->GetID()); + } else { + ROCKS_LOG_ERROR(immutable_db_options_.info_log, + "Creating column family [%s] FAILED -- %s", + column_family_name.c_str(), s.ToString().c_str()); + } + } // InstrumentedMutexLock l(&mutex_) + + if (cf_options.preserve_internal_time_seconds > 0 || + cf_options.preclude_last_level_data_seconds > 0) { + s = RegisterRecordSeqnoTimeWorker(); + } + sv_context.Clean(); + // this is outside the mutex + if (s.ok()) { + NewThreadStatusCfInfo( + static_cast_with_check(*handle)->cfd()); + } + return s; +} + +Status DBImpl::DropColumnFamily(ColumnFamilyHandle* column_family) { + assert(column_family != nullptr); + Status s = DropColumnFamilyImpl(column_family); + if (s.ok()) { + s = WriteOptionsFile(true /*need_mutex_lock*/, + true /*need_enter_write_thread*/); + } + return s; +} + +Status DBImpl::DropColumnFamilies( + const std::vector& column_families) { + Status s; + bool success_once = false; + for (auto* handle : column_families) { + s = DropColumnFamilyImpl(handle); + if (!s.ok()) { + break; + } + success_once = true; + } + if (success_once) { + Status persist_options_status = WriteOptionsFile( + true /*need_mutex_lock*/, true /*need_enter_write_thread*/); + if (s.ok() && !persist_options_status.ok()) { + s = persist_options_status; + } + } + return s; +} + +Status DBImpl::DropColumnFamilyImpl(ColumnFamilyHandle* column_family) { + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + auto cfh = static_cast_with_check(column_family); + auto cfd = cfh->cfd(); + if (cfd->GetID() == 0) { + return Status::InvalidArgument("Can't drop default column family"); + } + + bool cf_support_snapshot = cfd->mem()->IsSnapshotSupported(); + + VersionEdit edit; + edit.DropColumnFamily(); + edit.SetColumnFamily(cfd->GetID()); + + Status s; + { + InstrumentedMutexLock l(&mutex_); + if (cfd->IsDropped()) { + s = Status::InvalidArgument("Column family already dropped!\n"); + } + if (s.ok()) { + // we drop column family from a single write thread + WriteThread::Writer w; + write_thread_.EnterUnbatched(&w, &mutex_); + s = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(), + read_options, &edit, &mutex_, + directories_.GetDbDir()); + write_thread_.ExitUnbatched(&w); + } + if (s.ok()) { + auto* mutable_cf_options = cfd->GetLatestMutableCFOptions(); + max_total_in_memory_state_ -= mutable_cf_options->write_buffer_size * + mutable_cf_options->max_write_buffer_number; + } + + if (!cf_support_snapshot) { + // Dropped Column Family doesn't support snapshot. Need to recalculate + // is_snapshot_supported_. + bool new_is_snapshot_supported = true; + for (auto c : *versions_->GetColumnFamilySet()) { + if (!c->IsDropped() && !c->mem()->IsSnapshotSupported()) { + new_is_snapshot_supported = false; + break; + } + } + is_snapshot_supported_ = new_is_snapshot_supported; + } + bg_cv_.SignalAll(); + } + + if (cfd->ioptions()->preserve_internal_time_seconds > 0 || + cfd->ioptions()->preclude_last_level_data_seconds > 0) { + s = RegisterRecordSeqnoTimeWorker(); + } + + if (s.ok()) { + // Note that here we erase the associated cf_info of the to-be-dropped + // cfd before its ref-count goes to zero to avoid having to erase cf_info + // later inside db_mutex. + EraseThreadStatusCfInfo(cfd); + assert(cfd->IsDropped()); + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "Dropped column family with id %u\n", cfd->GetID()); + } else { + ROCKS_LOG_ERROR(immutable_db_options_.info_log, + "Dropping column family with id %u FAILED -- %s\n", + cfd->GetID(), s.ToString().c_str()); + } + + return s; +} + +bool DBImpl::KeyMayExist(const ReadOptions& read_options, + ColumnFamilyHandle* column_family, const Slice& key, + std::string* value, std::string* timestamp, + bool* value_found) { + assert(value != nullptr); + assert(read_options.io_activity == Env::IOActivity::kUnknown); + + if (value_found != nullptr) { + // falsify later if key-may-exist but can't fetch value + *value_found = true; + } + // TODO: plumb Env::IOActivity + ReadOptions roptions = read_options; + roptions.read_tier = kBlockCacheTier; // read from block cache only + PinnableSlice pinnable_val; + GetImplOptions get_impl_options; + get_impl_options.column_family = column_family; + get_impl_options.value = &pinnable_val; + get_impl_options.value_found = value_found; + get_impl_options.timestamp = timestamp; + auto s = GetImpl(roptions, key, get_impl_options); + value->assign(pinnable_val.data(), pinnable_val.size()); + + // If block_cache is enabled and the index block of the table didn't + // not present in block_cache, the return value will be Status::Incomplete. + // In this case, key may still exist in the table. + return s.ok() || s.IsIncomplete(); +} + +Iterator* DBImpl::NewIterator(const ReadOptions& read_options, + ColumnFamilyHandle* column_family) { + if (read_options.managed) { + return NewErrorIterator( + Status::NotSupported("Managed iterator is not supported anymore.")); + } + Iterator* result = nullptr; + if (read_options.read_tier == kPersistedTier) { + return NewErrorIterator(Status::NotSupported( + "ReadTier::kPersistedData is not yet supported in iterators.")); + } + if (read_options.io_activity != Env::IOActivity::kUnknown) { + return NewErrorIterator(Status::InvalidArgument( + "Cannot call NewIterator with `ReadOptions::io_activity` != " + "`Env::IOActivity::kUnknown`")); + } + + assert(column_family); + + if (read_options.timestamp) { + const Status s = FailIfTsMismatchCf( + column_family, *(read_options.timestamp), /*ts_for_read=*/true); + if (!s.ok()) { + return NewErrorIterator(s); + } + } else { + const Status s = FailIfCfHasTs(column_family); + if (!s.ok()) { + return NewErrorIterator(s); + } + } + + auto cfh = static_cast_with_check(column_family); + ColumnFamilyData* cfd = cfh->cfd(); + assert(cfd != nullptr); + ReadCallback* read_callback = nullptr; // No read callback provided. + if (read_options.tailing) { + SuperVersion* sv = cfd->GetReferencedSuperVersion(this); + auto iter = new ForwardIterator(this, read_options, cfd, sv, + /* allow_unprepared_value */ true); + result = NewDBIterator( + env_, read_options, *cfd->ioptions(), sv->mutable_cf_options, + cfd->user_comparator(), iter, sv->current, kMaxSequenceNumber, + sv->mutable_cf_options.max_sequential_skip_in_iterations, read_callback, + this, cfd); + } else { + // Note: no need to consider the special case of + // last_seq_same_as_publish_seq_==false since NewIterator is overridden in + // WritePreparedTxnDB + result = NewIteratorImpl(read_options, cfd, + (read_options.snapshot != nullptr) + ? read_options.snapshot->GetSequenceNumber() + : kMaxSequenceNumber, + read_callback); + } + return result; +} + +ArenaWrappedDBIter* DBImpl::NewIteratorImpl(const ReadOptions& read_options, + ColumnFamilyData* cfd, + SequenceNumber snapshot, + ReadCallback* read_callback, + bool expose_blob_index, + bool allow_refresh) { + SuperVersion* sv = cfd->GetReferencedSuperVersion(this); + + TEST_SYNC_POINT("DBImpl::NewIterator:1"); + TEST_SYNC_POINT("DBImpl::NewIterator:2"); + + if (snapshot == kMaxSequenceNumber) { + // Note that the snapshot is assigned AFTER referencing the super + // version because otherwise a flush happening in between may compact away + // data for the snapshot, so the reader would see neither data that was be + // visible to the snapshot before compaction nor the newer data inserted + // afterwards. + // Note that the super version might not contain all the data available + // to this snapshot, but in that case it can see all the data in the + // super version, which is a valid consistent state after the user + // calls NewIterator(). + snapshot = versions_->LastSequence(); + TEST_SYNC_POINT("DBImpl::NewIterator:3"); + TEST_SYNC_POINT("DBImpl::NewIterator:4"); + } + + // Try to generate a DB iterator tree in continuous memory area to be + // cache friendly. Here is an example of result: + // +-------------------------------+ + // | | + // | ArenaWrappedDBIter | + // | + | + // | +---> Inner Iterator ------------+ + // | | | | + // | | +-- -- -- -- -- -- -- --+ | + // | +--- | Arena | | + // | | | | + // | Allocated Memory: | | + // | | +-------------------+ | + // | | | DBIter | <---+ + // | | + | + // | | | +-> iter_ ------------+ + // | | | | | + // | | +-------------------+ | + // | | | MergingIterator | <---+ + // | | + | + // | | | +->child iter1 ------------+ + // | | | | | | + // | | +->child iter2 ----------+ | + // | | | | | | | + // | | | +->child iter3 --------+ | | + // | | | | | | + // | | +-------------------+ | | | + // | | | Iterator1 | <--------+ + // | | +-------------------+ | | + // | | | Iterator2 | <------+ + // | | +-------------------+ | + // | | | Iterator3 | <----+ + // | | +-------------------+ + // | | | + // +-------+-----------------------+ + // + // ArenaWrappedDBIter inlines an arena area where all the iterators in + // the iterator tree are allocated in the order of being accessed when + // querying. + // Laying out the iterators in the order of being accessed makes it more + // likely that any iterator pointer is close to the iterator it points to so + // that they are likely to be in the same cache line and/or page. + ArenaWrappedDBIter* db_iter = NewArenaWrappedDbIterator( + env_, read_options, *cfd->ioptions(), sv->mutable_cf_options, sv->current, + snapshot, sv->mutable_cf_options.max_sequential_skip_in_iterations, + sv->version_number, read_callback, this, cfd, expose_blob_index, + read_options.snapshot != nullptr ? false : allow_refresh); + + InternalIterator* internal_iter = NewInternalIterator( + db_iter->GetReadOptions(), cfd, sv, db_iter->GetArena(), snapshot, + /* allow_unprepared_value */ true, db_iter); + db_iter->SetIterUnderDBIter(internal_iter); + + return db_iter; +} + +Status DBImpl::NewIterators( + const ReadOptions& read_options, + const std::vector& column_families, + std::vector* iterators) { + if (read_options.managed) { + return Status::NotSupported("Managed iterator is not supported anymore."); + } + if (read_options.read_tier == kPersistedTier) { + return Status::NotSupported( + "ReadTier::kPersistedData is not yet supported in iterators."); + } + if (read_options.io_activity != Env::IOActivity::kUnknown) { + return Status::InvalidArgument( + "Cannot call NewIterators with `ReadOptions::io_activity` != " + "`Env::IOActivity::kUnknown`"); + } + + if (read_options.timestamp) { + for (auto* cf : column_families) { + assert(cf); + const Status s = FailIfTsMismatchCf(cf, *(read_options.timestamp), + /*ts_for_read=*/true); + if (!s.ok()) { + return s; + } + } + } else { + for (auto* cf : column_families) { + assert(cf); + const Status s = FailIfCfHasTs(cf); + if (!s.ok()) { + return s; + } + } + } + + ReadCallback* read_callback = nullptr; // No read callback provided. + iterators->clear(); + iterators->reserve(column_families.size()); + if (read_options.tailing) { + for (auto cfh : column_families) { + auto cfd = static_cast_with_check(cfh)->cfd(); + SuperVersion* sv = cfd->GetReferencedSuperVersion(this); + auto iter = new ForwardIterator(this, read_options, cfd, sv, + /* allow_unprepared_value */ true); + iterators->push_back(NewDBIterator( + env_, read_options, *cfd->ioptions(), sv->mutable_cf_options, + cfd->user_comparator(), iter, sv->current, kMaxSequenceNumber, + sv->mutable_cf_options.max_sequential_skip_in_iterations, + read_callback, this, cfd)); + } + } else { + // Note: no need to consider the special case of + // last_seq_same_as_publish_seq_==false since NewIterators is overridden in + // WritePreparedTxnDB + auto snapshot = read_options.snapshot != nullptr + ? read_options.snapshot->GetSequenceNumber() + : versions_->LastSequence(); + for (size_t i = 0; i < column_families.size(); ++i) { + auto* cfd = + static_cast_with_check(column_families[i]) + ->cfd(); + iterators->push_back( + NewIteratorImpl(read_options, cfd, snapshot, read_callback)); + } + } + + return Status::OK(); +} + +const Snapshot* DBImpl::GetSnapshot() { return GetSnapshotImpl(false); } + +const Snapshot* DBImpl::GetSnapshotForWriteConflictBoundary() { + return GetSnapshotImpl(true); +} + +std::pair> +DBImpl::CreateTimestampedSnapshot(SequenceNumber snapshot_seq, uint64_t ts) { + assert(ts != std::numeric_limits::max()); + + auto ret = CreateTimestampedSnapshotImpl(snapshot_seq, ts, /*lock=*/true); + return ret; +} + +std::shared_ptr DBImpl::GetTimestampedSnapshot( + uint64_t ts) const { + InstrumentedMutexLock lock_guard(&mutex_); + return timestamped_snapshots_.GetSnapshot(ts); +} + +void DBImpl::ReleaseTimestampedSnapshotsOlderThan(uint64_t ts, + size_t* remaining_total_ss) { + autovector> snapshots_to_release; + { + InstrumentedMutexLock lock_guard(&mutex_); + timestamped_snapshots_.ReleaseSnapshotsOlderThan(ts, snapshots_to_release); + } + snapshots_to_release.clear(); + + if (remaining_total_ss) { + InstrumentedMutexLock lock_guard(&mutex_); + *remaining_total_ss = static_cast(snapshots_.count()); + } +} + +Status DBImpl::GetTimestampedSnapshots( + uint64_t ts_lb, uint64_t ts_ub, + std::vector>& timestamped_snapshots) const { + if (ts_lb >= ts_ub) { + return Status::InvalidArgument( + "timestamp lower bound must be smaller than upper bound"); + } + timestamped_snapshots.clear(); + InstrumentedMutexLock lock_guard(&mutex_); + timestamped_snapshots_.GetSnapshots(ts_lb, ts_ub, timestamped_snapshots); + return Status::OK(); +} + +SnapshotImpl* DBImpl::GetSnapshotImpl(bool is_write_conflict_boundary, + bool lock) { + int64_t unix_time = 0; + immutable_db_options_.clock->GetCurrentTime(&unix_time) + .PermitUncheckedError(); // Ignore error + SnapshotImpl* s = new SnapshotImpl; + + if (lock) { + mutex_.Lock(); + } else { + mutex_.AssertHeld(); + } + // returns null if the underlying memtable does not support snapshot. + if (!is_snapshot_supported_) { + if (lock) { + mutex_.Unlock(); + } + delete s; + return nullptr; + } + auto snapshot_seq = GetLastPublishedSequence(); + SnapshotImpl* snapshot = + snapshots_.New(s, snapshot_seq, unix_time, is_write_conflict_boundary); + if (lock) { + mutex_.Unlock(); + } + return snapshot; +} + +std::pair> +DBImpl::CreateTimestampedSnapshotImpl(SequenceNumber snapshot_seq, uint64_t ts, + bool lock) { + int64_t unix_time = 0; + immutable_db_options_.clock->GetCurrentTime(&unix_time) + .PermitUncheckedError(); // Ignore error + SnapshotImpl* s = new SnapshotImpl; + + const bool need_update_seq = (snapshot_seq != kMaxSequenceNumber); + + if (lock) { + mutex_.Lock(); + } else { + mutex_.AssertHeld(); + } + // returns null if the underlying memtable does not support snapshot. + if (!is_snapshot_supported_) { + if (lock) { + mutex_.Unlock(); + } + delete s; + return std::make_pair( + Status::NotSupported("Memtable does not support snapshot"), nullptr); + } + + // Caller is not write thread, thus didn't provide a valid snapshot_seq. + // Obtain seq from db. + if (!need_update_seq) { + snapshot_seq = GetLastPublishedSequence(); + } + + std::shared_ptr latest = + timestamped_snapshots_.GetSnapshot(std::numeric_limits::max()); + + // If there is already a latest timestamped snapshot, then we need to do some + // checks. + if (latest) { + uint64_t latest_snap_ts = latest->GetTimestamp(); + SequenceNumber latest_snap_seq = latest->GetSequenceNumber(); + assert(latest_snap_seq <= snapshot_seq); + bool needs_create_snap = true; + Status status; + std::shared_ptr ret; + if (latest_snap_ts > ts) { + // A snapshot created later cannot have smaller timestamp than a previous + // timestamped snapshot. + needs_create_snap = false; + std::ostringstream oss; + oss << "snapshot exists with larger timestamp " << latest_snap_ts << " > " + << ts; + status = Status::InvalidArgument(oss.str()); + } else if (latest_snap_ts == ts) { + if (latest_snap_seq == snapshot_seq) { + // We are requesting the same sequence number and timestamp, thus can + // safely reuse (share) the current latest timestamped snapshot. + needs_create_snap = false; + ret = latest; + } else if (latest_snap_seq < snapshot_seq) { + // There may have been writes to the database since the latest + // timestamped snapshot, yet we are still requesting the same + // timestamp. In this case, we cannot create the new timestamped + // snapshot. + needs_create_snap = false; + std::ostringstream oss; + oss << "Allocated seq is " << snapshot_seq + << ", while snapshot exists with smaller seq " << latest_snap_seq + << " but same timestamp " << ts; + status = Status::InvalidArgument(oss.str()); + } + } + if (!needs_create_snap) { + if (lock) { + mutex_.Unlock(); + } + delete s; + return std::make_pair(status, ret); + } else { + status.PermitUncheckedError(); + } + } + + SnapshotImpl* snapshot = + snapshots_.New(s, snapshot_seq, unix_time, + /*is_write_conflict_boundary=*/true, ts); + + std::shared_ptr ret( + snapshot, + std::bind(&DBImpl::ReleaseSnapshot, this, std::placeholders::_1)); + timestamped_snapshots_.AddSnapshot(ret); + + // Caller is from write thread, and we need to update database's sequence + // number. + if (need_update_seq) { + assert(versions_); + if (last_seq_same_as_publish_seq_) { + versions_->SetLastSequence(snapshot_seq); + } else { + // TODO: support write-prepared/write-unprepared transactions with two + // write queues. + assert(false); + } + } + + if (lock) { + mutex_.Unlock(); + } + return std::make_pair(Status::OK(), ret); +} + +namespace { +using CfdList = autovector; +bool CfdListContains(const CfdList& list, ColumnFamilyData* cfd) { + for (const ColumnFamilyData* t : list) { + if (t == cfd) { + return true; + } + } + return false; +} +} // namespace + +void DBImpl::ReleaseSnapshot(const Snapshot* s) { + if (s == nullptr) { + // DBImpl::GetSnapshot() can return nullptr when snapshot + // not supported by specifying the condition: + // inplace_update_support enabled. + return; + } + const SnapshotImpl* casted_s = reinterpret_cast(s); + { + InstrumentedMutexLock l(&mutex_); + snapshots_.Delete(casted_s); + uint64_t oldest_snapshot; + if (snapshots_.empty()) { + oldest_snapshot = GetLastPublishedSequence(); + } else { + oldest_snapshot = snapshots_.oldest()->number_; + } + // Avoid to go through every column family by checking a global threshold + // first. + if (oldest_snapshot > bottommost_files_mark_threshold_) { + CfdList cf_scheduled; + for (auto* cfd : *versions_->GetColumnFamilySet()) { + if (!cfd->ioptions()->allow_ingest_behind) { + cfd->current()->storage_info()->UpdateOldestSnapshot(oldest_snapshot); + if (!cfd->current() + ->storage_info() + ->BottommostFilesMarkedForCompaction() + .empty()) { + SchedulePendingCompaction(cfd); + MaybeScheduleFlushOrCompaction(); + cf_scheduled.push_back(cfd); + } + } + } + + // Calculate a new threshold, skipping those CFs where compactions are + // scheduled. We do not do the same pass as the previous loop because + // mutex might be unlocked during the loop, making the result inaccurate. + SequenceNumber new_bottommost_files_mark_threshold = kMaxSequenceNumber; + for (auto* cfd : *versions_->GetColumnFamilySet()) { + if (CfdListContains(cf_scheduled, cfd) || + cfd->ioptions()->allow_ingest_behind) { + continue; + } + new_bottommost_files_mark_threshold = std::min( + new_bottommost_files_mark_threshold, + cfd->current()->storage_info()->bottommost_files_mark_threshold()); + } + bottommost_files_mark_threshold_ = new_bottommost_files_mark_threshold; + } + } + delete casted_s; +} + +Status DBImpl::GetPropertiesOfAllTables(ColumnFamilyHandle* column_family, + TablePropertiesCollection* props) { + auto cfh = static_cast_with_check(column_family); + auto cfd = cfh->cfd(); + + // Increment the ref count + mutex_.Lock(); + auto version = cfd->current(); + version->Ref(); + mutex_.Unlock(); + + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + auto s = version->GetPropertiesOfAllTables(read_options, props); + + // Decrement the ref count + mutex_.Lock(); + version->Unref(); + mutex_.Unlock(); + + return s; +} + +Status DBImpl::GetPropertiesOfTablesInRange(ColumnFamilyHandle* column_family, + const Range* range, std::size_t n, + TablePropertiesCollection* props) { + auto cfh = static_cast_with_check(column_family); + auto cfd = cfh->cfd(); + + // Increment the ref count + mutex_.Lock(); + auto version = cfd->current(); + version->Ref(); + mutex_.Unlock(); + + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + auto s = version->GetPropertiesOfTablesInRange(read_options, range, n, props); + + // Decrement the ref count + mutex_.Lock(); + version->Unref(); + mutex_.Unlock(); + + return s; +} + + +const std::string& DBImpl::GetName() const { return dbname_; } + +Env* DBImpl::GetEnv() const { return env_; } + +FileSystem* DB::GetFileSystem() const { + const auto& fs = GetEnv()->GetFileSystem(); + return fs.get(); +} + +FileSystem* DBImpl::GetFileSystem() const { + return immutable_db_options_.fs.get(); +} + +SystemClock* DBImpl::GetSystemClock() const { + return immutable_db_options_.clock; +} + + +Status DBImpl::StartIOTrace(const TraceOptions& trace_options, + std::unique_ptr&& trace_writer) { + assert(trace_writer != nullptr); + return io_tracer_->StartIOTrace(GetSystemClock(), trace_options, + std::move(trace_writer)); +} + +Status DBImpl::EndIOTrace() { + io_tracer_->EndIOTrace(); + return Status::OK(); +} + + +Options DBImpl::GetOptions(ColumnFamilyHandle* column_family) const { + InstrumentedMutexLock l(&mutex_); + auto cfh = static_cast_with_check(column_family); + return Options(BuildDBOptions(immutable_db_options_, mutable_db_options_), + cfh->cfd()->GetLatestCFOptions()); +} + +DBOptions DBImpl::GetDBOptions() const { + InstrumentedMutexLock l(&mutex_); + return BuildDBOptions(immutable_db_options_, mutable_db_options_); +} + +bool DBImpl::GetProperty(ColumnFamilyHandle* column_family, + const Slice& property, std::string* value) { + const DBPropertyInfo* property_info = GetPropertyInfo(property); + value->clear(); + auto cfd = + static_cast_with_check(column_family)->cfd(); + if (property_info == nullptr) { + return false; + } else if (property_info->handle_int) { + uint64_t int_value; + bool ret_value = + GetIntPropertyInternal(cfd, *property_info, false, &int_value); + if (ret_value) { + *value = std::to_string(int_value); + } + return ret_value; + } else if (property_info->handle_string) { + if (property_info->need_out_of_mutex) { + return cfd->internal_stats()->GetStringProperty(*property_info, property, + value); + } else { + InstrumentedMutexLock l(&mutex_); + return cfd->internal_stats()->GetStringProperty(*property_info, property, + value); + } + } else if (property_info->handle_string_dbimpl) { + if (property_info->need_out_of_mutex) { + return (this->*(property_info->handle_string_dbimpl))(value); + } else { + InstrumentedMutexLock l(&mutex_); + return (this->*(property_info->handle_string_dbimpl))(value); + } + } + // Shouldn't reach here since exactly one of handle_string and handle_int + // should be non-nullptr. + assert(false); + return false; +} + +bool DBImpl::GetMapProperty(ColumnFamilyHandle* column_family, + const Slice& property, + std::map* value) { + const DBPropertyInfo* property_info = GetPropertyInfo(property); + value->clear(); + auto cfd = + static_cast_with_check(column_family)->cfd(); + if (property_info == nullptr) { + return false; + } else if (property_info->handle_map) { + if (property_info->need_out_of_mutex) { + return cfd->internal_stats()->GetMapProperty(*property_info, property, + value); + } else { + InstrumentedMutexLock l(&mutex_); + return cfd->internal_stats()->GetMapProperty(*property_info, property, + value); + } + } + // If we reach this point it means that handle_map is not provided for the + // requested property + return false; +} + +bool DBImpl::GetIntProperty(ColumnFamilyHandle* column_family, + const Slice& property, uint64_t* value) { + const DBPropertyInfo* property_info = GetPropertyInfo(property); + if (property_info == nullptr || property_info->handle_int == nullptr) { + return false; + } + auto cfd = + static_cast_with_check(column_family)->cfd(); + return GetIntPropertyInternal(cfd, *property_info, false, value); +} + +bool DBImpl::GetIntPropertyInternal(ColumnFamilyData* cfd, + const DBPropertyInfo& property_info, + bool is_locked, uint64_t* value) { + assert(property_info.handle_int != nullptr); + if (!property_info.need_out_of_mutex) { + if (is_locked) { + mutex_.AssertHeld(); + return cfd->internal_stats()->GetIntProperty(property_info, value, this); + } else { + InstrumentedMutexLock l(&mutex_); + return cfd->internal_stats()->GetIntProperty(property_info, value, this); + } + } else { + SuperVersion* sv = nullptr; + if (is_locked) { + mutex_.Unlock(); + } + sv = GetAndRefSuperVersion(cfd); + + bool ret = cfd->internal_stats()->GetIntPropertyOutOfMutex( + property_info, sv->current, value); + + ReturnAndCleanupSuperVersion(cfd, sv); + if (is_locked) { + mutex_.Lock(); + } + + return ret; + } +} + +bool DBImpl::GetPropertyHandleOptionsStatistics(std::string* value) { + assert(value != nullptr); + Statistics* statistics = immutable_db_options_.stats; + if (!statistics) { + return false; + } + *value = statistics->ToString(); + return true; +} + +Status DBImpl::ResetStats() { + InstrumentedMutexLock l(&mutex_); + for (auto* cfd : *versions_->GetColumnFamilySet()) { + if (cfd->initialized()) { + cfd->internal_stats()->Clear(); + } + } + return Status::OK(); +} + +bool DBImpl::GetAggregatedIntProperty(const Slice& property, + uint64_t* aggregated_value) { + const DBPropertyInfo* property_info = GetPropertyInfo(property); + if (property_info == nullptr || property_info->handle_int == nullptr) { + return false; + } + + uint64_t sum = 0; + bool ret = true; + { + // Needs mutex to protect the list of column families. + InstrumentedMutexLock l(&mutex_); + uint64_t value; + for (auto* cfd : versions_->GetRefedColumnFamilySet()) { + if (!cfd->initialized()) { + continue; + } + ret = GetIntPropertyInternal(cfd, *property_info, true, &value); + // GetIntPropertyInternal may release db mutex and re-acquire it. + mutex_.AssertHeld(); + if (ret) { + sum += value; + } else { + ret = false; + break; + } + } + } + *aggregated_value = sum; + return ret; +} + +SuperVersion* DBImpl::GetAndRefSuperVersion(ColumnFamilyData* cfd) { + // TODO(ljin): consider using GetReferencedSuperVersion() directly + return cfd->GetThreadLocalSuperVersion(this); +} + +// REQUIRED: this function should only be called on the write thread or if the +// mutex is held. +SuperVersion* DBImpl::GetAndRefSuperVersion(uint32_t column_family_id) { + auto column_family_set = versions_->GetColumnFamilySet(); + auto cfd = column_family_set->GetColumnFamily(column_family_id); + if (!cfd) { + return nullptr; + } + + return GetAndRefSuperVersion(cfd); +} + +void DBImpl::CleanupSuperVersion(SuperVersion* sv) { + // Release SuperVersion + if (sv->Unref()) { + bool defer_purge = immutable_db_options().avoid_unnecessary_blocking_io; + { + InstrumentedMutexLock l(&mutex_); + sv->Cleanup(); + if (defer_purge) { + AddSuperVersionsToFreeQueue(sv); + SchedulePurge(); + } + } + if (!defer_purge) { + delete sv; + } + RecordTick(stats_, NUMBER_SUPERVERSION_CLEANUPS); + } + RecordTick(stats_, NUMBER_SUPERVERSION_RELEASES); +} + +void DBImpl::ReturnAndCleanupSuperVersion(ColumnFamilyData* cfd, + SuperVersion* sv) { + if (!cfd->ReturnThreadLocalSuperVersion(sv)) { + CleanupSuperVersion(sv); + } +} + +// REQUIRED: this function should only be called on the write thread. +void DBImpl::ReturnAndCleanupSuperVersion(uint32_t column_family_id, + SuperVersion* sv) { + auto column_family_set = versions_->GetColumnFamilySet(); + auto cfd = column_family_set->GetColumnFamily(column_family_id); + + // If SuperVersion is held, and we successfully fetched a cfd using + // GetAndRefSuperVersion(), it must still exist. + assert(cfd != nullptr); + ReturnAndCleanupSuperVersion(cfd, sv); +} + +// REQUIRED: this function should only be called on the write thread or if the +// mutex is held. +ColumnFamilyHandle* DBImpl::GetColumnFamilyHandle(uint32_t column_family_id) { + ColumnFamilyMemTables* cf_memtables = column_family_memtables_.get(); + + if (!cf_memtables->Seek(column_family_id)) { + return nullptr; + } + + return cf_memtables->GetColumnFamilyHandle(); +} + +// REQUIRED: mutex is NOT held. +std::unique_ptr DBImpl::GetColumnFamilyHandleUnlocked( + uint32_t column_family_id) { + InstrumentedMutexLock l(&mutex_); + + auto* cfd = + versions_->GetColumnFamilySet()->GetColumnFamily(column_family_id); + if (cfd == nullptr) { + return nullptr; + } + + return std::unique_ptr( + new ColumnFamilyHandleImpl(cfd, this, &mutex_)); +} + +void DBImpl::GetApproximateMemTableStats(ColumnFamilyHandle* column_family, + const Range& range, + uint64_t* const count, + uint64_t* const size) { + ColumnFamilyHandleImpl* cfh = + static_cast_with_check(column_family); + ColumnFamilyData* cfd = cfh->cfd(); + SuperVersion* sv = GetAndRefSuperVersion(cfd); + + // Convert user_key into a corresponding internal key. + InternalKey k1(range.start, kMaxSequenceNumber, kValueTypeForSeek); + InternalKey k2(range.limit, kMaxSequenceNumber, kValueTypeForSeek); + MemTable::MemTableStats memStats = + sv->mem->ApproximateStats(k1.Encode(), k2.Encode()); + MemTable::MemTableStats immStats = + sv->imm->ApproximateStats(k1.Encode(), k2.Encode()); + *count = memStats.count + immStats.count; + *size = memStats.size + immStats.size; + + ReturnAndCleanupSuperVersion(cfd, sv); +} + +Status DBImpl::GetApproximateSizes(const SizeApproximationOptions& options, + ColumnFamilyHandle* column_family, + const Range* range, int n, uint64_t* sizes) { + if (!options.include_memtables && !options.include_files) { + return Status::InvalidArgument("Invalid options"); + } + + const Comparator* const ucmp = column_family->GetComparator(); + assert(ucmp); + size_t ts_sz = ucmp->timestamp_size(); + + Version* v; + auto cfh = static_cast_with_check(column_family); + auto cfd = cfh->cfd(); + SuperVersion* sv = GetAndRefSuperVersion(cfd); + v = sv->current; + + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + for (int i = 0; i < n; i++) { + Slice start = range[i].start; + Slice limit = range[i].limit; + + // Add timestamp if needed + std::string start_with_ts, limit_with_ts; + if (ts_sz > 0) { + // Maximum timestamp means including all key with any timestamp + AppendKeyWithMaxTimestamp(&start_with_ts, start, ts_sz); + // Append a maximum timestamp as the range limit is exclusive: + // [start, limit) + AppendKeyWithMaxTimestamp(&limit_with_ts, limit, ts_sz); + start = start_with_ts; + limit = limit_with_ts; + } + // Convert user_key into a corresponding internal key. + InternalKey k1(start, kMaxSequenceNumber, kValueTypeForSeek); + InternalKey k2(limit, kMaxSequenceNumber, kValueTypeForSeek); + sizes[i] = 0; + if (options.include_files) { + sizes[i] += versions_->ApproximateSize( + options, read_options, v, k1.Encode(), k2.Encode(), /*start_level=*/0, + /*end_level=*/-1, TableReaderCaller::kUserApproximateSize); + } + if (options.include_memtables) { + sizes[i] += sv->mem->ApproximateStats(k1.Encode(), k2.Encode()).size; + sizes[i] += sv->imm->ApproximateStats(k1.Encode(), k2.Encode()).size; + } + } + + ReturnAndCleanupSuperVersion(cfd, sv); + return Status::OK(); +} + +std::list::iterator +DBImpl::CaptureCurrentFileNumberInPendingOutputs() { + // We need to remember the iterator of our insert, because after the + // background job is done, we need to remove that element from + // pending_outputs_. + pending_outputs_.push_back(versions_->current_next_file_number()); + auto pending_outputs_inserted_elem = pending_outputs_.end(); + --pending_outputs_inserted_elem; + return pending_outputs_inserted_elem; +} + +void DBImpl::ReleaseFileNumberFromPendingOutputs( + std::unique_ptr::iterator>& v) { + if (v.get() != nullptr) { + pending_outputs_.erase(*v.get()); + v.reset(); + } +} + +Status DBImpl::GetUpdatesSince( + SequenceNumber seq, std::unique_ptr* iter, + const TransactionLogIterator::ReadOptions& read_options) { + RecordTick(stats_, GET_UPDATES_SINCE_CALLS); + if (seq_per_batch_) { + return Status::NotSupported( + "This API is not yet compatible with write-prepared/write-unprepared " + "transactions"); + } + if (seq > versions_->LastSequence()) { + return Status::NotFound("Requested sequence not yet written in the db"); + } + return wal_manager_.GetUpdatesSince(seq, iter, read_options, versions_.get()); +} + +Status DBImpl::DeleteFile(std::string name) { + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + uint64_t number; + FileType type; + WalFileType log_type; + if (!ParseFileName(name, &number, &type, &log_type) || + (type != kTableFile && type != kWalFile)) { + ROCKS_LOG_ERROR(immutable_db_options_.info_log, "DeleteFile %s failed.\n", + name.c_str()); + return Status::InvalidArgument("Invalid file name"); + } + + if (type == kWalFile) { + // Only allow deleting archived log files + if (log_type != kArchivedLogFile) { + ROCKS_LOG_ERROR(immutable_db_options_.info_log, + "DeleteFile %s failed - not archived log.\n", + name.c_str()); + return Status::NotSupported("Delete only supported for archived logs"); + } + Status status = wal_manager_.DeleteFile(name, number); + if (!status.ok()) { + ROCKS_LOG_ERROR(immutable_db_options_.info_log, + "DeleteFile %s failed -- %s.\n", name.c_str(), + status.ToString().c_str()); + } + return status; + } + + Status status; + int level; + FileMetaData* metadata; + ColumnFamilyData* cfd; + VersionEdit edit; + JobContext job_context(next_job_id_.fetch_add(1), true); + { + InstrumentedMutexLock l(&mutex_); + status = versions_->GetMetadataForFile(number, &level, &metadata, &cfd); + if (!status.ok()) { + ROCKS_LOG_WARN(immutable_db_options_.info_log, + "DeleteFile %s failed. File not found\n", name.c_str()); + job_context.Clean(); + return Status::InvalidArgument("File not found"); + } + assert(level < cfd->NumberLevels()); + + // If the file is being compacted no need to delete. + if (metadata->being_compacted) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "DeleteFile %s Skipped. File about to be compacted\n", + name.c_str()); + job_context.Clean(); + return Status::OK(); + } + + // Only the files in the last level can be deleted externally. + // This is to make sure that any deletion tombstones are not + // lost. Check that the level passed is the last level. + auto* vstoreage = cfd->current()->storage_info(); + for (int i = level + 1; i < cfd->NumberLevels(); i++) { + if (vstoreage->NumLevelFiles(i) != 0) { + ROCKS_LOG_WARN(immutable_db_options_.info_log, + "DeleteFile %s FAILED. File not in last level\n", + name.c_str()); + job_context.Clean(); + return Status::InvalidArgument("File not in last level"); + } + } + // if level == 0, it has to be the oldest file + if (level == 0 && + vstoreage->LevelFiles(0).back()->fd.GetNumber() != number) { + ROCKS_LOG_WARN(immutable_db_options_.info_log, + "DeleteFile %s failed ---" + " target file in level 0 must be the oldest.", + name.c_str()); + job_context.Clean(); + return Status::InvalidArgument("File in level 0, but not oldest"); + } + edit.SetColumnFamily(cfd->GetID()); + edit.DeleteFile(level, number); + status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(), + read_options, &edit, &mutex_, + directories_.GetDbDir()); + if (status.ok()) { + InstallSuperVersionAndScheduleWork(cfd, + &job_context.superversion_contexts[0], + *cfd->GetLatestMutableCFOptions()); + } + FindObsoleteFiles(&job_context, false); + } // lock released here + + LogFlush(immutable_db_options_.info_log); + // remove files outside the db-lock + if (job_context.HaveSomethingToDelete()) { + // Call PurgeObsoleteFiles() without holding mutex. + PurgeObsoleteFiles(job_context); + } + job_context.Clean(); + return status; +} + +Status DBImpl::DeleteFilesInRanges(ColumnFamilyHandle* column_family, + const RangePtr* ranges, size_t n, + bool include_end) { + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + Status status = Status::OK(); + auto cfh = static_cast_with_check(column_family); + ColumnFamilyData* cfd = cfh->cfd(); + VersionEdit edit; + std::set deleted_files; + JobContext job_context(next_job_id_.fetch_add(1), true); + { + InstrumentedMutexLock l(&mutex_); + Version* input_version = cfd->current(); + + auto* vstorage = input_version->storage_info(); + for (size_t r = 0; r < n; r++) { + auto begin = ranges[r].start, end = ranges[r].limit; + for (int i = 1; i < cfd->NumberLevels(); i++) { + if (vstorage->LevelFiles(i).empty() || + !vstorage->OverlapInLevel(i, begin, end)) { + continue; + } + std::vector level_files; + InternalKey begin_storage, end_storage, *begin_key, *end_key; + if (begin == nullptr) { + begin_key = nullptr; + } else { + begin_storage.SetMinPossibleForUserKey(*begin); + begin_key = &begin_storage; + } + if (end == nullptr) { + end_key = nullptr; + } else { + end_storage.SetMaxPossibleForUserKey(*end); + end_key = &end_storage; + } + + vstorage->GetCleanInputsWithinInterval( + i, begin_key, end_key, &level_files, -1 /* hint_index */, + nullptr /* file_index */); + FileMetaData* level_file; + for (uint32_t j = 0; j < level_files.size(); j++) { + level_file = level_files[j]; + if (level_file->being_compacted) { + continue; + } + if (deleted_files.find(level_file) != deleted_files.end()) { + continue; + } + if (!include_end && end != nullptr && + cfd->user_comparator()->Compare(level_file->largest.user_key(), + *end) == 0) { + continue; + } + edit.SetColumnFamily(cfd->GetID()); + edit.DeleteFile(i, level_file->fd.GetNumber()); + deleted_files.insert(level_file); + level_file->being_compacted = true; + } + vstorage->ComputeCompactionScore(*cfd->ioptions(), + *cfd->GetLatestMutableCFOptions()); + } + } + if (edit.GetDeletedFiles().empty()) { + job_context.Clean(); + return status; + } + input_version->Ref(); + status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(), + read_options, &edit, &mutex_, + directories_.GetDbDir()); + if (status.ok()) { + InstallSuperVersionAndScheduleWork(cfd, + &job_context.superversion_contexts[0], + *cfd->GetLatestMutableCFOptions()); + } + for (auto* deleted_file : deleted_files) { + deleted_file->being_compacted = false; + } + input_version->Unref(); + FindObsoleteFiles(&job_context, false); + } // lock released here + + LogFlush(immutable_db_options_.info_log); + // remove files outside the db-lock + if (job_context.HaveSomethingToDelete()) { + // Call PurgeObsoleteFiles() without holding mutex. + PurgeObsoleteFiles(job_context); + } + job_context.Clean(); + return status; +} + +void DBImpl::GetLiveFilesMetaData(std::vector* metadata) { + InstrumentedMutexLock l(&mutex_); + versions_->GetLiveFilesMetaData(metadata); +} + +Status DBImpl::GetLiveFilesChecksumInfo(FileChecksumList* checksum_list) { + InstrumentedMutexLock l(&mutex_); + return versions_->GetLiveFilesChecksumInfo(checksum_list); +} + +void DBImpl::GetColumnFamilyMetaData(ColumnFamilyHandle* column_family, + ColumnFamilyMetaData* cf_meta) { + assert(column_family); + auto* cfd = + static_cast_with_check(column_family)->cfd(); + auto* sv = GetAndRefSuperVersion(cfd); + { + // Without mutex, Version::GetColumnFamilyMetaData will have data race with + // Compaction::MarkFilesBeingCompacted. One solution is to use mutex, but + // this may cause regression. An alternative is to make + // FileMetaData::being_compacted atomic, but it will make FileMetaData + // non-copy-able. Another option is to separate these variables from + // original FileMetaData struct, and this requires re-organization of data + // structures. For now, we take the easy approach. If + // DB::GetColumnFamilyMetaData is not called frequently, the regression + // should not be big. We still need to keep an eye on it. + InstrumentedMutexLock l(&mutex_); + sv->current->GetColumnFamilyMetaData(cf_meta); + } + ReturnAndCleanupSuperVersion(cfd, sv); +} + +void DBImpl::GetAllColumnFamilyMetaData( + std::vector* metadata) { + InstrumentedMutexLock l(&mutex_); + for (auto cfd : *(versions_->GetColumnFamilySet())) { + { + metadata->emplace_back(); + cfd->current()->GetColumnFamilyMetaData(&metadata->back()); + } + } +} + +Status DBImpl::CheckConsistency() { + mutex_.AssertHeld(); + std::vector metadata; + versions_->GetLiveFilesMetaData(&metadata); + TEST_SYNC_POINT("DBImpl::CheckConsistency:AfterGetLiveFilesMetaData"); + + std::string corruption_messages; + + if (immutable_db_options_.skip_checking_sst_file_sizes_on_db_open) { + // Instead of calling GetFileSize() for each expected file, call + // GetChildren() for the DB directory and check that all expected files + // are listed, without checking their sizes. + // Since sst files might be in different directories, do it for each + // directory separately. + std::map> files_by_directory; + for (const auto& md : metadata) { + // md.name has a leading "/". Remove it. + std::string fname = md.name; + if (!fname.empty() && fname[0] == '/') { + fname = fname.substr(1); + } + files_by_directory[md.db_path].push_back(fname); + } + + IOOptions io_opts; + io_opts.do_not_recurse = true; + for (const auto& dir_files : files_by_directory) { + std::string directory = dir_files.first; + std::vector existing_files; + Status s = fs_->GetChildren(directory, io_opts, &existing_files, + /*IODebugContext*=*/nullptr); + if (!s.ok()) { + corruption_messages += + "Can't list files in " + directory + ": " + s.ToString() + "\n"; + continue; + } + std::sort(existing_files.begin(), existing_files.end()); + + for (const std::string& fname : dir_files.second) { + if (!std::binary_search(existing_files.begin(), existing_files.end(), + fname) && + !std::binary_search(existing_files.begin(), existing_files.end(), + Rocks2LevelTableFileName(fname))) { + corruption_messages += + "Missing sst file " + fname + " in " + directory + "\n"; + } + } + } + } else { + for (const auto& md : metadata) { + // md.name has a leading "/". + std::string file_path = md.db_path + md.name; + + uint64_t fsize = 0; + TEST_SYNC_POINT("DBImpl::CheckConsistency:BeforeGetFileSize"); + Status s = env_->GetFileSize(file_path, &fsize); + if (!s.ok() && + env_->GetFileSize(Rocks2LevelTableFileName(file_path), &fsize).ok()) { + s = Status::OK(); + } + if (!s.ok()) { + corruption_messages += + "Can't access " + md.name + ": " + s.ToString() + "\n"; + } else if (fsize != md.size) { + corruption_messages += "Sst file size mismatch: " + file_path + + ". Size recorded in manifest " + + std::to_string(md.size) + ", actual size " + + std::to_string(fsize) + "\n"; + } + } + } + + if (corruption_messages.size() == 0) { + return Status::OK(); + } else { + return Status::Corruption(corruption_messages); + } +} + +Status DBImpl::GetDbIdentity(std::string& identity) const { + identity.assign(db_id_); + return Status::OK(); +} + +Status DBImpl::GetDbIdentityFromIdentityFile(std::string* identity) const { + std::string idfilename = IdentityFileName(dbname_); + const FileOptions soptions; + + Status s = ReadFileToString(fs_.get(), idfilename, identity); + if (!s.ok()) { + return s; + } + + // If last character is '\n' remove it from identity. (Old implementations + // of Env::GenerateUniqueId() would include a trailing '\n'.) + if (identity->size() > 0 && identity->back() == '\n') { + identity->pop_back(); + } + return s; +} + +Status DBImpl::GetDbSessionId(std::string& session_id) const { + session_id.assign(db_session_id_); + return Status::OK(); +} + +namespace { +SemiStructuredUniqueIdGen* DbSessionIdGen() { + static SemiStructuredUniqueIdGen gen; + return &gen; +} +} // namespace + +void DBImpl::TEST_ResetDbSessionIdGen() { DbSessionIdGen()->Reset(); } + +std::string DBImpl::GenerateDbSessionId(Env*) { + // See SemiStructuredUniqueIdGen for its desirable properties. + auto gen = DbSessionIdGen(); + + uint64_t lo, hi; + gen->GenerateNext(&hi, &lo); + if (lo == 0) { + // Avoid emitting session ID with lo==0, so that SST unique + // IDs can be more easily ensured non-zero + gen->GenerateNext(&hi, &lo); + assert(lo != 0); + } + return EncodeSessionId(hi, lo); +} + +void DBImpl::SetDbSessionId() { + db_session_id_ = GenerateDbSessionId(env_); + TEST_SYNC_POINT_CALLBACK("DBImpl::SetDbSessionId", &db_session_id_); +} + +// Default implementation -- returns not supported status +Status DB::CreateColumnFamily(const ColumnFamilyOptions& /*cf_options*/, + const std::string& /*column_family_name*/, + ColumnFamilyHandle** /*handle*/) { + return Status::NotSupported(""); +} + +Status DB::CreateColumnFamilies( + const ColumnFamilyOptions& /*cf_options*/, + const std::vector& /*column_family_names*/, + std::vector* /*handles*/) { + return Status::NotSupported(""); +} + +Status DB::CreateColumnFamilies( + const std::vector& /*column_families*/, + std::vector* /*handles*/) { + return Status::NotSupported(""); +} + +Status DB::DropColumnFamily(ColumnFamilyHandle* /*column_family*/) { + return Status::NotSupported(""); +} + +Status DB::DropColumnFamilies( + const std::vector& /*column_families*/) { + return Status::NotSupported(""); +} + +Status DB::DestroyColumnFamilyHandle(ColumnFamilyHandle* column_family) { + if (DefaultColumnFamily() == column_family) { + return Status::InvalidArgument( + "Cannot destroy the handle returned by DefaultColumnFamily()"); + } + delete column_family; + return Status::OK(); +} + +DB::~DB() {} + +Status DBImpl::Close() { + InstrumentedMutexLock closing_lock_guard(&closing_mutex_); + if (closed_) { + return closing_status_; + } + + { + const Status s = MaybeReleaseTimestampedSnapshotsAndCheck(); + if (!s.ok()) { + return s; + } + } + + closing_status_ = CloseImpl(); + closed_ = true; + return closing_status_; +} + +Status DB::ListColumnFamilies(const DBOptions& db_options, + const std::string& name, + std::vector* column_families) { + const std::shared_ptr& fs = db_options.env->GetFileSystem(); + return VersionSet::ListColumnFamilies(column_families, name, fs.get()); +} + +Snapshot::~Snapshot() {} + +Status DestroyDB(const std::string& dbname, const Options& options, + const std::vector& column_families) { + ImmutableDBOptions soptions(SanitizeOptions(dbname, options)); + Env* env = soptions.env; + std::vector filenames; + bool wal_in_db_path = soptions.IsWalDirSameAsDBPath(); + + // Reset the logger because it holds a handle to the + // log file and prevents cleanup and directory removal + soptions.info_log.reset(); + IOOptions io_opts; + // Ignore error in case directory does not exist + soptions.fs + ->GetChildren(dbname, io_opts, &filenames, + /*IODebugContext*=*/nullptr) + .PermitUncheckedError(); + + FileLock* lock; + const std::string lockname = LockFileName(dbname); + Status result = env->LockFile(lockname, &lock); + if (result.ok()) { + uint64_t number; + FileType type; + InfoLogPrefix info_log_prefix(!soptions.db_log_dir.empty(), dbname); + for (const auto& fname : filenames) { + if (ParseFileName(fname, &number, info_log_prefix.prefix, &type) && + type != kDBLockFile) { // Lock file will be deleted at end + Status del; + std::string path_to_delete = dbname + "/" + fname; + if (type == kMetaDatabase) { + del = DestroyDB(path_to_delete, options); + } else if (type == kTableFile || type == kWalFile || + type == kBlobFile) { + del = DeleteDBFile( + &soptions, path_to_delete, dbname, + /*force_bg=*/false, + /*force_fg=*/(type == kWalFile) ? !wal_in_db_path : false); + } else { + del = env->DeleteFile(path_to_delete); + } + if (!del.ok() && result.ok()) { + result = del; + } + } + } + + std::set paths; + for (const DbPath& db_path : options.db_paths) { + paths.insert(db_path.path); + } + for (const ColumnFamilyDescriptor& cf : column_families) { + for (const DbPath& cf_path : cf.options.cf_paths) { + paths.insert(cf_path.path); + } + } + + for (const auto& path : paths) { + if (soptions.fs + ->GetChildren(path, io_opts, &filenames, + /*IODebugContext*=*/nullptr) + .ok()) { + for (const auto& fname : filenames) { + if (ParseFileName(fname, &number, &type) && + (type == kTableFile || + type == kBlobFile)) { // Lock file will be deleted at end + std::string file_path = path + "/" + fname; + Status del = DeleteDBFile(&soptions, file_path, dbname, + /*force_bg=*/false, /*force_fg=*/false); + if (!del.ok() && result.ok()) { + result = del; + } + } + } + // TODO: Should we return an error if we cannot delete the directory? + env->DeleteDir(path).PermitUncheckedError(); + } + } + + std::vector walDirFiles; + std::string archivedir = ArchivalDirectory(dbname); + bool wal_dir_exists = false; + if (!soptions.IsWalDirSameAsDBPath(dbname)) { + wal_dir_exists = + soptions.fs + ->GetChildren(soptions.wal_dir, io_opts, &walDirFiles, + /*IODebugContext*=*/nullptr) + .ok(); + archivedir = ArchivalDirectory(soptions.wal_dir); + } + + // Archive dir may be inside wal dir or dbname and should be + // processed and removed before those otherwise we have issues + // removing them + std::vector archiveFiles; + if (soptions.fs + ->GetChildren(archivedir, io_opts, &archiveFiles, + /*IODebugContext*=*/nullptr) + .ok()) { + // Delete archival files. + for (const auto& file : archiveFiles) { + if (ParseFileName(file, &number, &type) && type == kWalFile) { + Status del = + DeleteDBFile(&soptions, archivedir + "/" + file, archivedir, + /*force_bg=*/false, /*force_fg=*/!wal_in_db_path); + if (!del.ok() && result.ok()) { + result = del; + } + } + } + // Ignore error in case dir contains other files + env->DeleteDir(archivedir).PermitUncheckedError(); + } + + // Delete log files in the WAL dir + if (wal_dir_exists) { + for (const auto& file : walDirFiles) { + if (ParseFileName(file, &number, &type) && type == kWalFile) { + Status del = + DeleteDBFile(&soptions, LogFileName(soptions.wal_dir, number), + soptions.wal_dir, /*force_bg=*/false, + /*force_fg=*/!wal_in_db_path); + if (!del.ok() && result.ok()) { + result = del; + } + } + } + // Ignore error in case dir contains other files + env->DeleteDir(soptions.wal_dir).PermitUncheckedError(); + } + + // Ignore error since state is already gone + env->UnlockFile(lock).PermitUncheckedError(); + env->DeleteFile(lockname).PermitUncheckedError(); + + // sst_file_manager holds a ref to the logger. Make sure the logger is + // gone before trying to remove the directory. + soptions.sst_file_manager.reset(); + + // Ignore error in case dir contains other files + env->DeleteDir(dbname).PermitUncheckedError(); + ; + } + return result; +} + +Status DBImpl::WriteOptionsFile(bool need_mutex_lock, + bool need_enter_write_thread) { + WriteThread::Writer w; + if (need_mutex_lock) { + mutex_.Lock(); + } else { + mutex_.AssertHeld(); + } + if (need_enter_write_thread) { + write_thread_.EnterUnbatched(&w, &mutex_); + } + + std::vector cf_names; + std::vector cf_opts; + + // This part requires mutex to protect the column family options + for (auto cfd : *versions_->GetColumnFamilySet()) { + if (cfd->IsDropped()) { + continue; + } + cf_names.push_back(cfd->GetName()); + cf_opts.push_back(cfd->GetLatestCFOptions()); + } + + // Unlock during expensive operations. New writes cannot get here + // because the single write thread ensures all new writes get queued. + DBOptions db_options = + BuildDBOptions(immutable_db_options_, mutable_db_options_); + mutex_.Unlock(); + + TEST_SYNC_POINT("DBImpl::WriteOptionsFile:1"); + TEST_SYNC_POINT("DBImpl::WriteOptionsFile:2"); + TEST_SYNC_POINT_CALLBACK("DBImpl::WriteOptionsFile:PersistOptions", + &db_options); + + std::string file_name = + TempOptionsFileName(GetName(), versions_->NewFileNumber()); + Status s = PersistRocksDBOptions(db_options, cf_names, cf_opts, file_name, + fs_.get()); + + if (s.ok()) { + s = RenameTempFileToOptionsFile(file_name); + } + + if (!s.ok() && GetEnv()->FileExists(file_name).ok()) { + if (!GetEnv()->DeleteFile(file_name).ok()) { + ROCKS_LOG_WARN(immutable_db_options_.info_log, + "Unable to delete temp options file %s", + file_name.c_str()); + } + } + + // restore lock + if (!need_mutex_lock) { + mutex_.Lock(); + } + if (need_enter_write_thread) { + write_thread_.ExitUnbatched(&w); + } + if (!s.ok()) { + ROCKS_LOG_WARN(immutable_db_options_.info_log, + "Unnable to persist options -- %s", s.ToString().c_str()); + if (immutable_db_options_.fail_if_options_file_error) { + return Status::IOError("Unable to persist options.", + s.ToString().c_str()); + } + } + return Status::OK(); +} + +namespace { +void DeleteOptionsFilesHelper(const std::map& filenames, + const size_t num_files_to_keep, + const std::shared_ptr& info_log, + Env* env) { + if (filenames.size() <= num_files_to_keep) { + return; + } + for (auto iter = std::next(filenames.begin(), num_files_to_keep); + iter != filenames.end(); ++iter) { + if (!env->DeleteFile(iter->second).ok()) { + ROCKS_LOG_WARN(info_log, "Unable to delete options file %s", + iter->second.c_str()); + } + } +} +} // namespace + +Status DBImpl::DeleteObsoleteOptionsFiles() { + std::vector filenames; + // use ordered map to store keep the filenames sorted from the newest + // to the oldest. + std::map options_filenames; + Status s; + IOOptions io_opts; + io_opts.do_not_recurse = true; + s = fs_->GetChildren(GetName(), io_opts, &filenames, + /*IODebugContext*=*/nullptr); + if (!s.ok()) { + return s; + } + for (auto& filename : filenames) { + uint64_t file_number; + FileType type; + if (ParseFileName(filename, &file_number, &type) && type == kOptionsFile) { + options_filenames.insert( + {std::numeric_limits::max() - file_number, + GetName() + "/" + filename}); + } + } + + // Keeps the latest 2 Options file + const size_t kNumOptionsFilesKept = 2; + DeleteOptionsFilesHelper(options_filenames, kNumOptionsFilesKept, + immutable_db_options_.info_log, GetEnv()); + return Status::OK(); +} + +Status DBImpl::RenameTempFileToOptionsFile(const std::string& file_name) { + Status s; + + uint64_t options_file_number = versions_->NewFileNumber(); + std::string options_file_name = + OptionsFileName(GetName(), options_file_number); + uint64_t options_file_size = 0; + s = GetEnv()->GetFileSize(file_name, &options_file_size); + if (s.ok()) { + // Retry if the file name happen to conflict with an existing one. + s = GetEnv()->RenameFile(file_name, options_file_name); + std::unique_ptr dir_obj; + if (s.ok()) { + s = fs_->NewDirectory(GetName(), IOOptions(), &dir_obj, nullptr); + } + if (s.ok()) { + s = dir_obj->FsyncWithDirOptions(IOOptions(), nullptr, + DirFsyncOptions(options_file_name)); + } + if (s.ok()) { + Status temp_s = dir_obj->Close(IOOptions(), nullptr); + // The default Close() could return "NotSupproted" and we bypass it + // if it is not impelmented. Detailed explanations can be found in + // db/db_impl/db_impl.h + if (!temp_s.ok()) { + if (temp_s.IsNotSupported()) { + temp_s.PermitUncheckedError(); + } else { + s = temp_s; + } + } + } + } + if (s.ok()) { + InstrumentedMutexLock l(&mutex_); + versions_->options_file_number_ = options_file_number; + versions_->options_file_size_ = options_file_size; + } + + if (0 == disable_delete_obsolete_files_) { + // TODO: Should we check for errors here? + DeleteObsoleteOptionsFiles().PermitUncheckedError(); + } + return s; +} + +#ifdef ROCKSDB_USING_THREAD_STATUS + +void DBImpl::NewThreadStatusCfInfo(ColumnFamilyData* cfd) const { + if (immutable_db_options_.enable_thread_tracking) { + ThreadStatusUtil::NewColumnFamilyInfo(this, cfd, cfd->GetName(), + cfd->ioptions()->env); + } +} + +void DBImpl::EraseThreadStatusCfInfo(ColumnFamilyData* cfd) const { + if (immutable_db_options_.enable_thread_tracking) { + ThreadStatusUtil::EraseColumnFamilyInfo(cfd); + } +} + +void DBImpl::EraseThreadStatusDbInfo() const { + if (immutable_db_options_.enable_thread_tracking) { + ThreadStatusUtil::EraseDatabaseInfo(this); + } +} + +#else +void DBImpl::NewThreadStatusCfInfo(ColumnFamilyData* /*cfd*/) const {} + +void DBImpl::EraseThreadStatusCfInfo(ColumnFamilyData* /*cfd*/) const {} + +void DBImpl::EraseThreadStatusDbInfo() const {} +#endif // ROCKSDB_USING_THREAD_STATUS + +// +// A global method that can dump out the build version +void DumpRocksDBBuildVersion(Logger* log) { + ROCKS_LOG_HEADER(log, "RocksDB version: %s\n", + GetRocksVersionAsString().c_str()); + const auto& props = GetRocksBuildProperties(); + const auto& sha = props.find("rocksdb_build_git_sha"); + if (sha != props.end()) { + ROCKS_LOG_HEADER(log, "Git sha %s", sha->second.c_str()); + } + const auto date = props.find("rocksdb_build_date"); + if (date != props.end()) { + ROCKS_LOG_HEADER(log, "Compile date %s", date->second.c_str()); + } +} + +SequenceNumber DBImpl::GetEarliestMemTableSequenceNumber(SuperVersion* sv, + bool include_history) { + // Find the earliest sequence number that we know we can rely on reading + // from the memtable without needing to check sst files. + SequenceNumber earliest_seq = + sv->imm->GetEarliestSequenceNumber(include_history); + if (earliest_seq == kMaxSequenceNumber) { + earliest_seq = sv->mem->GetEarliestSequenceNumber(); + } + assert(sv->mem->GetEarliestSequenceNumber() >= earliest_seq); + + return earliest_seq; +} + +Status DBImpl::GetLatestSequenceForKey( + SuperVersion* sv, const Slice& key, bool cache_only, + SequenceNumber lower_bound_seq, SequenceNumber* seq, std::string* timestamp, + bool* found_record_for_key, bool* is_blob_index) { + Status s; + MergeContext merge_context; + SequenceNumber max_covering_tombstone_seq = 0; + + // TODO: plumb Env::IOActivity + ReadOptions read_options; + SequenceNumber current_seq = versions_->LastSequence(); + + ColumnFamilyData* cfd = sv->cfd; + assert(cfd); + const Comparator* const ucmp = cfd->user_comparator(); + assert(ucmp); + size_t ts_sz = ucmp->timestamp_size(); + std::string ts_buf; + if (ts_sz > 0) { + assert(timestamp); + ts_buf.assign(ts_sz, '\xff'); + } else { + assert(!timestamp); + } + Slice ts(ts_buf); + + LookupKey lkey(key, current_seq, ts_sz == 0 ? nullptr : &ts); + + *seq = kMaxSequenceNumber; + *found_record_for_key = false; + + // Check if there is a record for this key in the latest memtable + sv->mem->Get(lkey, /*value=*/nullptr, /*columns=*/nullptr, timestamp, &s, + &merge_context, &max_covering_tombstone_seq, seq, read_options, + false /* immutable_memtable */, nullptr /*read_callback*/, + is_blob_index); + + if (!(s.ok() || s.IsNotFound() || s.IsMergeInProgress())) { + // unexpected error reading memtable. + ROCKS_LOG_ERROR(immutable_db_options_.info_log, + "Unexpected status returned from MemTable::Get: %s\n", + s.ToString().c_str()); + + return s; + } + assert(!ts_sz || + (*seq != kMaxSequenceNumber && + *timestamp != std::string(ts_sz, '\xff')) || + (*seq == kMaxSequenceNumber && timestamp->empty())); + + TEST_SYNC_POINT_CALLBACK("DBImpl::GetLatestSequenceForKey:mem", timestamp); + + if (*seq != kMaxSequenceNumber) { + // Found a sequence number, no need to check immutable memtables + *found_record_for_key = true; + return Status::OK(); + } + + SequenceNumber lower_bound_in_mem = sv->mem->GetEarliestSequenceNumber(); + if (lower_bound_in_mem != kMaxSequenceNumber && + lower_bound_in_mem < lower_bound_seq) { + *found_record_for_key = false; + return Status::OK(); + } + + // Check if there is a record for this key in the immutable memtables + sv->imm->Get(lkey, /*value=*/nullptr, /*columns=*/nullptr, timestamp, &s, + &merge_context, &max_covering_tombstone_seq, seq, read_options, + nullptr /*read_callback*/, is_blob_index); + + if (!(s.ok() || s.IsNotFound() || s.IsMergeInProgress())) { + // unexpected error reading memtable. + ROCKS_LOG_ERROR(immutable_db_options_.info_log, + "Unexpected status returned from MemTableList::Get: %s\n", + s.ToString().c_str()); + + return s; + } + + assert(!ts_sz || + (*seq != kMaxSequenceNumber && + *timestamp != std::string(ts_sz, '\xff')) || + (*seq == kMaxSequenceNumber && timestamp->empty())); + + if (*seq != kMaxSequenceNumber) { + // Found a sequence number, no need to check memtable history + *found_record_for_key = true; + return Status::OK(); + } + + SequenceNumber lower_bound_in_imm = sv->imm->GetEarliestSequenceNumber(); + if (lower_bound_in_imm != kMaxSequenceNumber && + lower_bound_in_imm < lower_bound_seq) { + *found_record_for_key = false; + return Status::OK(); + } + + // Check if there is a record for this key in the immutable memtables + sv->imm->GetFromHistory(lkey, /*value=*/nullptr, /*columns=*/nullptr, + timestamp, &s, &merge_context, + &max_covering_tombstone_seq, seq, read_options, + is_blob_index); + + if (!(s.ok() || s.IsNotFound() || s.IsMergeInProgress())) { + // unexpected error reading memtable. + ROCKS_LOG_ERROR( + immutable_db_options_.info_log, + "Unexpected status returned from MemTableList::GetFromHistory: %s\n", + s.ToString().c_str()); + + return s; + } + + assert(!ts_sz || + (*seq != kMaxSequenceNumber && + *timestamp != std::string(ts_sz, '\xff')) || + (*seq == kMaxSequenceNumber && timestamp->empty())); + + if (*seq != kMaxSequenceNumber) { + // Found a sequence number, no need to check SST files + assert(0 == ts_sz || *timestamp != std::string(ts_sz, '\xff')); + *found_record_for_key = true; + return Status::OK(); + } + + // We could do a sv->imm->GetEarliestSequenceNumber(/*include_history*/ true) + // check here to skip the history if possible. But currently the caller + // already does that. Maybe we should move the logic here later. + + // TODO(agiardullo): possible optimization: consider checking cached + // SST files if cache_only=true? + if (!cache_only) { + // Check tables + PinnedIteratorsManager pinned_iters_mgr; + sv->current->Get(read_options, lkey, /*value=*/nullptr, /*columns=*/nullptr, + timestamp, &s, &merge_context, &max_covering_tombstone_seq, + &pinned_iters_mgr, nullptr /* value_found */, + found_record_for_key, seq, nullptr /*read_callback*/, + is_blob_index); + + if (!(s.ok() || s.IsNotFound() || s.IsMergeInProgress())) { + // unexpected error reading SST files + ROCKS_LOG_ERROR(immutable_db_options_.info_log, + "Unexpected status returned from Version::Get: %s\n", + s.ToString().c_str()); + } + } + + return s; +} + +Status DBImpl::IngestExternalFile( + ColumnFamilyHandle* column_family, + const std::vector& external_files, + const IngestExternalFileOptions& ingestion_options) { + IngestExternalFileArg arg; + arg.column_family = column_family; + arg.external_files = external_files; + arg.options = ingestion_options; + return IngestExternalFiles({arg}); +} + +Status DBImpl::IngestExternalFiles( + const std::vector& args) { + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + if (args.empty()) { + return Status::InvalidArgument("ingestion arg list is empty"); + } + { + std::unordered_set unique_cfhs; + for (const auto& arg : args) { + if (arg.column_family == nullptr) { + return Status::InvalidArgument("column family handle is null"); + } else if (unique_cfhs.count(arg.column_family) > 0) { + return Status::InvalidArgument( + "ingestion args have duplicate column families"); + } + unique_cfhs.insert(arg.column_family); + } + } + // Ingest multiple external SST files atomically. + const size_t num_cfs = args.size(); + for (size_t i = 0; i != num_cfs; ++i) { + if (args[i].external_files.empty()) { + char err_msg[128] = {0}; + snprintf(err_msg, 128, "external_files[%zu] is empty", i); + return Status::InvalidArgument(err_msg); + } + } + for (const auto& arg : args) { + const IngestExternalFileOptions& ingest_opts = arg.options; + if (ingest_opts.ingest_behind && + !immutable_db_options_.allow_ingest_behind) { + return Status::InvalidArgument( + "can't ingest_behind file in DB with allow_ingest_behind=false"); + } + } + + // TODO (yanqin) maybe handle the case in which column_families have + // duplicates + std::unique_ptr::iterator> pending_output_elem; + size_t total = 0; + for (const auto& arg : args) { + total += arg.external_files.size(); + } + uint64_t next_file_number = 0; + Status status = ReserveFileNumbersBeforeIngestion( + static_cast(args[0].column_family)->cfd(), total, + pending_output_elem, &next_file_number); + if (!status.ok()) { + InstrumentedMutexLock l(&mutex_); + ReleaseFileNumberFromPendingOutputs(pending_output_elem); + return status; + } + + std::vector ingestion_jobs; + for (const auto& arg : args) { + auto* cfd = static_cast(arg.column_family)->cfd(); + ingestion_jobs.emplace_back(versions_.get(), cfd, immutable_db_options_, + mutable_db_options_, file_options_, &snapshots_, + arg.options, &directories_, &event_logger_, + io_tracer_); + } + + // TODO(yanqin) maybe make jobs run in parallel + uint64_t start_file_number = next_file_number; + for (size_t i = 1; i != num_cfs; ++i) { + start_file_number += args[i - 1].external_files.size(); + auto* cfd = + static_cast(args[i].column_family)->cfd(); + SuperVersion* super_version = cfd->GetReferencedSuperVersion(this); + Status es = ingestion_jobs[i].Prepare( + args[i].external_files, args[i].files_checksums, + args[i].files_checksum_func_names, args[i].file_temperature, + start_file_number, super_version); + // capture first error only + if (!es.ok() && status.ok()) { + status = es; + } + CleanupSuperVersion(super_version); + } + TEST_SYNC_POINT("DBImpl::IngestExternalFiles:BeforeLastJobPrepare:0"); + TEST_SYNC_POINT("DBImpl::IngestExternalFiles:BeforeLastJobPrepare:1"); + { + auto* cfd = + static_cast(args[0].column_family)->cfd(); + SuperVersion* super_version = cfd->GetReferencedSuperVersion(this); + Status es = ingestion_jobs[0].Prepare( + args[0].external_files, args[0].files_checksums, + args[0].files_checksum_func_names, args[0].file_temperature, + next_file_number, super_version); + if (!es.ok()) { + status = es; + } + CleanupSuperVersion(super_version); + } + if (!status.ok()) { + for (size_t i = 0; i != num_cfs; ++i) { + ingestion_jobs[i].Cleanup(status); + } + InstrumentedMutexLock l(&mutex_); + ReleaseFileNumberFromPendingOutputs(pending_output_elem); + return status; + } + + std::vector sv_ctxs; + for (size_t i = 0; i != num_cfs; ++i) { + sv_ctxs.emplace_back(true /* create_superversion */); + } + TEST_SYNC_POINT("DBImpl::IngestExternalFiles:BeforeJobsRun:0"); + TEST_SYNC_POINT("DBImpl::IngestExternalFiles:BeforeJobsRun:1"); + TEST_SYNC_POINT("DBImpl::AddFile:Start"); + { + InstrumentedMutexLock l(&mutex_); + TEST_SYNC_POINT("DBImpl::AddFile:MutexLock"); + + // Stop writes to the DB by entering both write threads + WriteThread::Writer w; + write_thread_.EnterUnbatched(&w, &mutex_); + WriteThread::Writer nonmem_w; + if (two_write_queues_) { + nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_); + } + + // When unordered_write is enabled, the keys are writing to memtable in an + // unordered way. If the ingestion job checks memtable key range before the + // key landing in memtable, the ingestion job may skip the necessary + // memtable flush. + // So wait here to ensure there is no pending write to memtable. + WaitForPendingWrites(); + + num_running_ingest_file_ += static_cast(num_cfs); + TEST_SYNC_POINT("DBImpl::IngestExternalFile:AfterIncIngestFileCounter"); + + bool at_least_one_cf_need_flush = false; + std::vector need_flush(num_cfs, false); + for (size_t i = 0; i != num_cfs; ++i) { + auto* cfd = + static_cast(args[i].column_family)->cfd(); + if (cfd->IsDropped()) { + // TODO (yanqin) investigate whether we should abort ingestion or + // proceed with other non-dropped column families. + status = Status::InvalidArgument( + "cannot ingest an external file into a dropped CF"); + break; + } + bool tmp = false; + status = ingestion_jobs[i].NeedsFlush(&tmp, cfd->GetSuperVersion()); + need_flush[i] = tmp; + at_least_one_cf_need_flush = (at_least_one_cf_need_flush || tmp); + if (!status.ok()) { + break; + } + } + TEST_SYNC_POINT_CALLBACK("DBImpl::IngestExternalFile:NeedFlush", + &at_least_one_cf_need_flush); + + if (status.ok() && at_least_one_cf_need_flush) { + FlushOptions flush_opts; + flush_opts.allow_write_stall = true; + if (immutable_db_options_.atomic_flush) { + mutex_.Unlock(); + status = AtomicFlushMemTables( + flush_opts, FlushReason::kExternalFileIngestion, + {} /* provided_candidate_cfds */, true /* entered_write_thread */); + mutex_.Lock(); + } else { + for (size_t i = 0; i != num_cfs; ++i) { + if (need_flush[i]) { + mutex_.Unlock(); + auto* cfd = + static_cast(args[i].column_family) + ->cfd(); + status = FlushMemTable(cfd, flush_opts, + FlushReason::kExternalFileIngestion, + true /* entered_write_thread */); + mutex_.Lock(); + if (!status.ok()) { + break; + } + } + } + } + } + // Run ingestion jobs. + if (status.ok()) { + for (size_t i = 0; i != num_cfs; ++i) { + mutex_.AssertHeld(); + status = ingestion_jobs[i].Run(); + if (!status.ok()) { + break; + } + ingestion_jobs[i].RegisterRange(); + } + } + if (status.ok()) { + autovector cfds_to_commit; + autovector mutable_cf_options_list; + autovector> edit_lists; + uint32_t num_entries = 0; + for (size_t i = 0; i != num_cfs; ++i) { + auto* cfd = + static_cast(args[i].column_family)->cfd(); + if (cfd->IsDropped()) { + continue; + } + cfds_to_commit.push_back(cfd); + mutable_cf_options_list.push_back(cfd->GetLatestMutableCFOptions()); + autovector edit_list; + edit_list.push_back(ingestion_jobs[i].edit()); + edit_lists.push_back(edit_list); + ++num_entries; + } + // Mark the version edits as an atomic group if the number of version + // edits exceeds 1. + if (cfds_to_commit.size() > 1) { + for (auto& edits : edit_lists) { + assert(edits.size() == 1); + edits[0]->MarkAtomicGroup(--num_entries); + } + assert(0 == num_entries); + } + status = versions_->LogAndApply(cfds_to_commit, mutable_cf_options_list, + read_options, edit_lists, &mutex_, + directories_.GetDbDir()); + // It is safe to update VersionSet last seqno here after LogAndApply since + // LogAndApply persists last sequence number from VersionEdits, + // which are from file's largest seqno and not from VersionSet. + // + // It is necessary to update last seqno here since LogAndApply releases + // mutex when persisting MANIFEST file, and the snapshots taken during + // that period will not be stable if VersionSet last seqno is updated + // before LogAndApply. + int consumed_seqno_count = + ingestion_jobs[0].ConsumedSequenceNumbersCount(); + for (size_t i = 1; i != num_cfs; ++i) { + consumed_seqno_count = + std::max(consumed_seqno_count, + ingestion_jobs[i].ConsumedSequenceNumbersCount()); + } + if (consumed_seqno_count > 0) { + const SequenceNumber last_seqno = versions_->LastSequence(); + versions_->SetLastAllocatedSequence(last_seqno + consumed_seqno_count); + versions_->SetLastPublishedSequence(last_seqno + consumed_seqno_count); + versions_->SetLastSequence(last_seqno + consumed_seqno_count); + } + } + + for (auto& job : ingestion_jobs) { + job.UnregisterRange(); + } + + if (status.ok()) { + for (size_t i = 0; i != num_cfs; ++i) { + auto* cfd = + static_cast(args[i].column_family)->cfd(); + if (!cfd->IsDropped()) { + InstallSuperVersionAndScheduleWork(cfd, &sv_ctxs[i], + *cfd->GetLatestMutableCFOptions()); +#ifndef NDEBUG + if (0 == i && num_cfs > 1) { + TEST_SYNC_POINT( + "DBImpl::IngestExternalFiles:InstallSVForFirstCF:0"); + TEST_SYNC_POINT( + "DBImpl::IngestExternalFiles:InstallSVForFirstCF:1"); + } +#endif // !NDEBUG + } + } + } else if (versions_->io_status().IsIOError()) { + // Error while writing to MANIFEST. + // In fact, versions_->io_status() can also be the result of renaming + // CURRENT file. With current code, it's just difficult to tell. So just + // be pessimistic and try write to a new MANIFEST. + // TODO: distinguish between MANIFEST write and CURRENT renaming + const IOStatus& io_s = versions_->io_status(); + // Should handle return error? + error_handler_.SetBGError(io_s, BackgroundErrorReason::kManifestWrite); + } + + // Resume writes to the DB + if (two_write_queues_) { + nonmem_write_thread_.ExitUnbatched(&nonmem_w); + } + write_thread_.ExitUnbatched(&w); + + if (status.ok()) { + for (auto& job : ingestion_jobs) { + job.UpdateStats(); + } + } + ReleaseFileNumberFromPendingOutputs(pending_output_elem); + num_running_ingest_file_ -= static_cast(num_cfs); + if (0 == num_running_ingest_file_) { + bg_cv_.SignalAll(); + } + TEST_SYNC_POINT("DBImpl::AddFile:MutexUnlock"); + } + // mutex_ is unlocked here + + // Cleanup + for (size_t i = 0; i != num_cfs; ++i) { + sv_ctxs[i].Clean(); + // This may rollback jobs that have completed successfully. This is + // intended for atomicity. + ingestion_jobs[i].Cleanup(status); + } + if (status.ok()) { + for (size_t i = 0; i != num_cfs; ++i) { + auto* cfd = + static_cast(args[i].column_family)->cfd(); + if (!cfd->IsDropped()) { + NotifyOnExternalFileIngested(cfd, ingestion_jobs[i]); + } + } + } + return status; +} + +Status DBImpl::CreateColumnFamilyWithImport( + const ColumnFamilyOptions& options, const std::string& column_family_name, + const ImportColumnFamilyOptions& import_options, + const std::vector& metadatas, + ColumnFamilyHandle** handle) { + assert(handle != nullptr); + assert(*handle == nullptr); + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + std::string cf_comparator_name = options.comparator->Name(); + + size_t total_file_num = 0; + std::vector> metadata_files(metadatas.size()); + for (size_t i = 0; i < metadatas.size(); i++) { + if (cf_comparator_name != metadatas[i]->db_comparator_name) { + return Status::InvalidArgument("Comparator name mismatch"); + } + for (auto& file : metadatas[i]->files) { + metadata_files[i].push_back((LiveFileMetaData*)&file); + } + total_file_num += metadatas[i]->files.size(); + } + + // Create column family. + auto status = CreateColumnFamily(options, column_family_name, handle); + if (!status.ok()) { + return status; + } + + // Import sst files from metadata. + auto cfh = static_cast_with_check(*handle); + auto cfd = cfh->cfd(); + ImportColumnFamilyJob import_job(versions_.get(), cfd, immutable_db_options_, + file_options_, import_options, + metadata_files, io_tracer_); + + SuperVersionContext dummy_sv_ctx(/* create_superversion */ true); + VersionEdit dummy_edit; + uint64_t next_file_number = 0; + std::unique_ptr::iterator> pending_output_elem; + { + // Lock db mutex + InstrumentedMutexLock l(&mutex_); + if (error_handler_.IsDBStopped()) { + // Don't import files when there is a bg_error + status = error_handler_.GetBGError(); + } + + // Make sure that bg cleanup wont delete the files that we are importing + pending_output_elem.reset(new std::list::iterator( + CaptureCurrentFileNumberInPendingOutputs())); + + if (status.ok()) { + // If crash happen after a hard link established, Recover function may + // reuse the file number that has already assigned to the internal file, + // and this will overwrite the external file. To protect the external + // file, we have to make sure the file number will never being reused. + next_file_number = versions_->FetchAddFileNumber(total_file_num); + auto cf_options = cfd->GetLatestMutableCFOptions(); + status = + versions_->LogAndApply(cfd, *cf_options, read_options, &dummy_edit, + &mutex_, directories_.GetDbDir()); + if (status.ok()) { + InstallSuperVersionAndScheduleWork(cfd, &dummy_sv_ctx, *cf_options); + } + } + } + dummy_sv_ctx.Clean(); + + if (status.ok()) { + SuperVersion* sv = cfd->GetReferencedSuperVersion(this); + status = import_job.Prepare(next_file_number, sv); + CleanupSuperVersion(sv); + } + + if (status.ok()) { + SuperVersionContext sv_context(true /*create_superversion*/); + { + // Lock db mutex + InstrumentedMutexLock l(&mutex_); + + // Stop writes to the DB by entering both write threads + WriteThread::Writer w; + write_thread_.EnterUnbatched(&w, &mutex_); + WriteThread::Writer nonmem_w; + if (two_write_queues_) { + nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_); + } + + num_running_ingest_file_++; + assert(!cfd->IsDropped()); + mutex_.AssertHeld(); + status = import_job.Run(); + + // Install job edit [Mutex will be unlocked here] + if (status.ok()) { + auto cf_options = cfd->GetLatestMutableCFOptions(); + status = versions_->LogAndApply(cfd, *cf_options, read_options, + import_job.edit(), &mutex_, + directories_.GetDbDir()); + if (status.ok()) { + InstallSuperVersionAndScheduleWork(cfd, &sv_context, *cf_options); + } + } + + // Resume writes to the DB + if (two_write_queues_) { + nonmem_write_thread_.ExitUnbatched(&nonmem_w); + } + write_thread_.ExitUnbatched(&w); + + num_running_ingest_file_--; + if (num_running_ingest_file_ == 0) { + bg_cv_.SignalAll(); + } + } + // mutex_ is unlocked here + + sv_context.Clean(); + } + + { + InstrumentedMutexLock l(&mutex_); + ReleaseFileNumberFromPendingOutputs(pending_output_elem); + } + + import_job.Cleanup(status); + if (!status.ok()) { + Status temp_s = DropColumnFamily(*handle); + if (!temp_s.ok()) { + ROCKS_LOG_ERROR(immutable_db_options_.info_log, + "DropColumnFamily failed with error %s", + temp_s.ToString().c_str()); + } + // Always returns Status::OK() + temp_s = DestroyColumnFamilyHandle(*handle); + assert(temp_s.ok()); + *handle = nullptr; + } + return status; +} + +Status DBImpl::ClipColumnFamily(ColumnFamilyHandle* column_family, + const Slice& begin_key, const Slice& end_key) { + assert(column_family); + Status status; + // Flush memtable + FlushOptions flush_opts; + flush_opts.allow_write_stall = true; + auto* cfd = + static_cast_with_check(column_family)->cfd(); + if (immutable_db_options_.atomic_flush) { + status = AtomicFlushMemTables(flush_opts, FlushReason::kDeleteFiles, + {} /* provided_candidate_cfds */, + false /* entered_write_thread */); + } else { + status = FlushMemTable(cfd, flush_opts, FlushReason::kDeleteFiles, + false /* entered_write_thread */); + } + + if (status.ok()) { + // DeleteFilesInRanges non-overlap files except L0 + std::vector ranges; + ranges.push_back(RangePtr(nullptr, &begin_key)); + ranges.push_back(RangePtr(&end_key, nullptr)); + status = DeleteFilesInRanges(column_family, ranges.data(), ranges.size()); + } + + // DeleteRange the remaining overlapping keys + bool empty_after_delete = false; + if (status.ok()) { + Slice smallest_user_key, largest_user_key; + { + // Lock db mutex + InstrumentedMutexLock l(&mutex_); + cfd->current()->GetSstFilesBoundaryKeys(&smallest_user_key, + &largest_user_key); + } + // all the files has been deleted after DeleteFilesInRanges; + if (smallest_user_key.empty() && largest_user_key.empty()) { + empty_after_delete = true; + } else { + const Comparator* const ucmp = column_family->GetComparator(); + WriteOptions wo; + // Delete [smallest_user_key, clip_begin_key) + if (ucmp->Compare(smallest_user_key, begin_key) < 0) { + status = DeleteRange(wo, column_family, smallest_user_key, begin_key); + } + + if (status.ok()) { + // Delete [clip_end_key, largest_use_key] + if (ucmp->Compare(end_key, largest_user_key) < 0) { + status = DeleteRange(wo, column_family, end_key, largest_user_key); + if (status.ok()) { + status = Delete(wo, column_family, largest_user_key); + } + } + } + } + } + + if (status.ok() && !empty_after_delete) { + // CompactRange delete all the tombstones + CompactRangeOptions compact_options; + compact_options.exclusive_manual_compaction = true; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForceOptimized; + // We could just compact the ranges [null, clip_begin_key] and + // [clip_end_key, null]. But due to how manual compaction calculates the + // last level to compact to and that range tombstones are not dropped + // during non-bottommost compactions, calling CompactRange() on these two + // ranges may not clear all range tombstones. + status = CompactRange(compact_options, nullptr, nullptr); + } + return status; +} + +Status DBImpl::VerifyFileChecksums(const ReadOptions& read_options) { + return VerifyChecksumInternal(read_options, /*use_file_checksum=*/true); +} + +Status DBImpl::VerifyChecksum(const ReadOptions& read_options) { + return VerifyChecksumInternal(read_options, /*use_file_checksum=*/false); +} + +Status DBImpl::VerifyChecksumInternal(const ReadOptions& read_options, + bool use_file_checksum) { + // `bytes_read` stat is enabled based on compile-time support and cannot + // be dynamically toggled. So we do not need to worry about `PerfLevel` + // here, unlike many other `IOStatsContext` / `PerfContext` stats. + uint64_t prev_bytes_read = IOSTATS(bytes_read); + + Status s; + + if (read_options.io_activity != Env::IOActivity::kUnknown) { + s = Status::InvalidArgument( + "Cannot verify file checksum with `ReadOptions::io_activity` != " + "`Env::IOActivity::kUnknown`"); + return s; + } + if (use_file_checksum) { + FileChecksumGenFactory* const file_checksum_gen_factory = + immutable_db_options_.file_checksum_gen_factory.get(); + if (!file_checksum_gen_factory) { + s = Status::InvalidArgument( + "Cannot verify file checksum if options.file_checksum_gen_factory is " + "null"); + return s; + } + } + // FIXME? What does it mean if read_options.verify_checksums == false? + + // TODO: simplify using GetRefedColumnFamilySet? + std::vector cfd_list; + { + InstrumentedMutexLock l(&mutex_); + for (auto cfd : *versions_->GetColumnFamilySet()) { + if (!cfd->IsDropped() && cfd->initialized()) { + cfd->Ref(); + cfd_list.push_back(cfd); + } + } + } + std::vector sv_list; + for (auto cfd : cfd_list) { + sv_list.push_back(cfd->GetReferencedSuperVersion(this)); + } + + for (auto& sv : sv_list) { + VersionStorageInfo* vstorage = sv->current->storage_info(); + ColumnFamilyData* cfd = sv->current->cfd(); + Options opts; + if (!use_file_checksum) { + InstrumentedMutexLock l(&mutex_); + opts = Options(BuildDBOptions(immutable_db_options_, mutable_db_options_), + cfd->GetLatestCFOptions()); + } + for (int i = 0; i < vstorage->num_non_empty_levels() && s.ok(); i++) { + for (size_t j = 0; j < vstorage->LevelFilesBrief(i).num_files && s.ok(); + j++) { + const auto& fd_with_krange = vstorage->LevelFilesBrief(i).files[j]; + const auto& fd = fd_with_krange.fd; + const FileMetaData* fmeta = fd_with_krange.file_metadata; + assert(fmeta); + std::string fname = TableFileName(cfd->ioptions()->cf_paths, + fd.GetNumber(), fd.GetPathId()); + if (use_file_checksum) { + s = VerifyFullFileChecksum(fmeta->file_checksum, + fmeta->file_checksum_func_name, fname, + read_options); + } else { + s = ROCKSDB_NAMESPACE::VerifySstFileChecksum( + opts, file_options_, read_options, fname, fd.largest_seqno); + } + RecordTick(stats_, VERIFY_CHECKSUM_READ_BYTES, + IOSTATS(bytes_read) - prev_bytes_read); + prev_bytes_read = IOSTATS(bytes_read); + } + } + + if (s.ok() && use_file_checksum) { + const auto& blob_files = vstorage->GetBlobFiles(); + for (const auto& meta : blob_files) { + assert(meta); + + const uint64_t blob_file_number = meta->GetBlobFileNumber(); + + const std::string blob_file_name = BlobFileName( + cfd->ioptions()->cf_paths.front().path, blob_file_number); + s = VerifyFullFileChecksum(meta->GetChecksumValue(), + meta->GetChecksumMethod(), blob_file_name, + read_options); + RecordTick(stats_, VERIFY_CHECKSUM_READ_BYTES, + IOSTATS(bytes_read) - prev_bytes_read); + prev_bytes_read = IOSTATS(bytes_read); + if (!s.ok()) { + break; + } + } + } + if (!s.ok()) { + break; + } + } + + bool defer_purge = immutable_db_options().avoid_unnecessary_blocking_io; + { + InstrumentedMutexLock l(&mutex_); + for (auto sv : sv_list) { + if (sv && sv->Unref()) { + sv->Cleanup(); + if (defer_purge) { + AddSuperVersionsToFreeQueue(sv); + } else { + delete sv; + } + } + } + if (defer_purge) { + SchedulePurge(); + } + for (auto cfd : cfd_list) { + cfd->UnrefAndTryDelete(); + } + } + RecordTick(stats_, VERIFY_CHECKSUM_READ_BYTES, + IOSTATS(bytes_read) - prev_bytes_read); + return s; +} + +Status DBImpl::VerifyFullFileChecksum(const std::string& file_checksum_expected, + const std::string& func_name_expected, + const std::string& fname, + const ReadOptions& read_options) { + if (read_options.io_activity != Env::IOActivity::kUnknown) { + return Status::InvalidArgument( + "Cannot call VerifyChecksum with `ReadOptions::io_activity` != " + "`Env::IOActivity::kUnknown`"); + } + + Status s; + if (file_checksum_expected == kUnknownFileChecksum) { + return s; + } + std::string file_checksum; + std::string func_name; + s = ROCKSDB_NAMESPACE::GenerateOneFileChecksum( + fs_.get(), fname, immutable_db_options_.file_checksum_gen_factory.get(), + func_name_expected, &file_checksum, &func_name, + read_options.readahead_size, immutable_db_options_.allow_mmap_reads, + io_tracer_, immutable_db_options_.rate_limiter.get(), + read_options.rate_limiter_priority); + if (s.ok()) { + assert(func_name_expected == func_name); + if (file_checksum != file_checksum_expected) { + std::ostringstream oss; + oss << fname << " file checksum mismatch, "; + oss << "expecting " + << Slice(file_checksum_expected).ToString(/*hex=*/true); + oss << ", but actual " << Slice(file_checksum).ToString(/*hex=*/true); + s = Status::Corruption(oss.str()); + TEST_SYNC_POINT_CALLBACK("DBImpl::VerifyFullFileChecksum:mismatch", &s); + } + } + return s; +} + +void DBImpl::NotifyOnExternalFileIngested( + ColumnFamilyData* cfd, const ExternalSstFileIngestionJob& ingestion_job) { + if (immutable_db_options_.listeners.empty()) { + return; + } + + for (const IngestedFileInfo& f : ingestion_job.files_to_ingest()) { + ExternalFileIngestionInfo info; + info.cf_name = cfd->GetName(); + info.external_file_path = f.external_file_path; + info.internal_file_path = f.internal_file_path; + info.global_seqno = f.assigned_seqno; + info.table_properties = f.table_properties; + for (auto listener : immutable_db_options_.listeners) { + listener->OnExternalFileIngested(this, info); + } + } +} + +Status DBImpl::StartTrace(const TraceOptions& trace_options, + std::unique_ptr&& trace_writer) { + InstrumentedMutexLock lock(&trace_mutex_); + tracer_.reset(new Tracer(immutable_db_options_.clock, trace_options, + std::move(trace_writer))); + return Status::OK(); +} + +Status DBImpl::EndTrace() { + InstrumentedMutexLock lock(&trace_mutex_); + Status s; + if (tracer_ != nullptr) { + s = tracer_->Close(); + tracer_.reset(); + } else { + s = Status::IOError("No trace file to close"); + } + return s; +} + +Status DBImpl::NewDefaultReplayer( + const std::vector& handles, + std::unique_ptr&& reader, + std::unique_ptr* replayer) { + replayer->reset(new ReplayerImpl(this, handles, std::move(reader))); + return Status::OK(); +} + +Status DBImpl::StartBlockCacheTrace( + const TraceOptions& trace_options, + std::unique_ptr&& trace_writer) { + BlockCacheTraceOptions block_trace_opts; + block_trace_opts.sampling_frequency = trace_options.sampling_frequency; + + BlockCacheTraceWriterOptions trace_writer_opt; + trace_writer_opt.max_trace_file_size = trace_options.max_trace_file_size; + + std::unique_ptr block_cache_trace_writer = + NewBlockCacheTraceWriter(env_->GetSystemClock().get(), trace_writer_opt, + std::move(trace_writer)); + + return block_cache_tracer_.StartTrace(block_trace_opts, + std::move(block_cache_trace_writer)); +} + +Status DBImpl::StartBlockCacheTrace( + const BlockCacheTraceOptions& trace_options, + std::unique_ptr&& trace_writer) { + return block_cache_tracer_.StartTrace(trace_options, std::move(trace_writer)); +} + +Status DBImpl::EndBlockCacheTrace() { + block_cache_tracer_.EndTrace(); + return Status::OK(); +} + +Status DBImpl::TraceIteratorSeek(const uint32_t& cf_id, const Slice& key, + const Slice& lower_bound, + const Slice upper_bound) { + Status s; + if (tracer_) { + InstrumentedMutexLock lock(&trace_mutex_); + if (tracer_) { + s = tracer_->IteratorSeek(cf_id, key, lower_bound, upper_bound); + } + } + return s; +} + +Status DBImpl::TraceIteratorSeekForPrev(const uint32_t& cf_id, const Slice& key, + const Slice& lower_bound, + const Slice upper_bound) { + Status s; + if (tracer_) { + InstrumentedMutexLock lock(&trace_mutex_); + if (tracer_) { + s = tracer_->IteratorSeekForPrev(cf_id, key, lower_bound, upper_bound); + } + } + return s; +} + +Status DBImpl::ReserveFileNumbersBeforeIngestion( + ColumnFamilyData* cfd, uint64_t num, + std::unique_ptr::iterator>& pending_output_elem, + uint64_t* next_file_number) { + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + Status s; + SuperVersionContext dummy_sv_ctx(true /* create_superversion */); + assert(nullptr != next_file_number); + InstrumentedMutexLock l(&mutex_); + if (error_handler_.IsDBStopped()) { + // Do not ingest files when there is a bg_error + return error_handler_.GetBGError(); + } + pending_output_elem.reset(new std::list::iterator( + CaptureCurrentFileNumberInPendingOutputs())); + *next_file_number = versions_->FetchAddFileNumber(static_cast(num)); + auto cf_options = cfd->GetLatestMutableCFOptions(); + VersionEdit dummy_edit; + // If crash happen after a hard link established, Recover function may + // reuse the file number that has already assigned to the internal file, + // and this will overwrite the external file. To protect the external + // file, we have to make sure the file number will never being reused. + s = versions_->LogAndApply(cfd, *cf_options, read_options, &dummy_edit, + &mutex_, directories_.GetDbDir()); + if (s.ok()) { + InstallSuperVersionAndScheduleWork(cfd, &dummy_sv_ctx, *cf_options); + } + dummy_sv_ctx.Clean(); + return s; +} + +Status DBImpl::GetCreationTimeOfOldestFile(uint64_t* creation_time) { + if (mutable_db_options_.max_open_files == -1) { + uint64_t oldest_time = std::numeric_limits::max(); + for (auto cfd : *versions_->GetColumnFamilySet()) { + if (!cfd->IsDropped()) { + uint64_t ctime; + { + SuperVersion* sv = GetAndRefSuperVersion(cfd); + Version* version = sv->current; + version->GetCreationTimeOfOldestFile(&ctime); + ReturnAndCleanupSuperVersion(cfd, sv); + } + + if (ctime < oldest_time) { + oldest_time = ctime; + } + if (oldest_time == 0) { + break; + } + } + } + *creation_time = oldest_time; + return Status::OK(); + } else { + return Status::NotSupported("This API only works if max_open_files = -1"); + } +} + +void DBImpl::RecordSeqnoToTimeMapping() { + // Get time first then sequence number, so the actual time of seqno is <= + // unix_time recorded + int64_t unix_time = 0; + immutable_db_options_.clock->GetCurrentTime(&unix_time) + .PermitUncheckedError(); // Ignore error + SequenceNumber seqno = GetLatestSequenceNumber(); + bool appended = false; + { + InstrumentedMutexLock l(&mutex_); + appended = seqno_time_mapping_.Append(seqno, unix_time); + } + if (!appended) { + ROCKS_LOG_WARN(immutable_db_options_.info_log, + "Failed to insert sequence number to time entry: %" PRIu64 + " -> %" PRIu64, + seqno, unix_time); + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/db_impl/db_impl.h b/librocksdb-sys/rocksdb/db/db_impl/db_impl.h new file mode 100644 index 0000000..0c65403 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_impl/db_impl.h @@ -0,0 +1,2850 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "db/column_family.h" +#include "db/compaction/compaction_iterator.h" +#include "db/compaction/compaction_job.h" +#include "db/error_handler.h" +#include "db/event_helpers.h" +#include "db/external_sst_file_ingestion_job.h" +#include "db/flush_job.h" +#include "db/flush_scheduler.h" +#include "db/import_column_family_job.h" +#include "db/internal_stats.h" +#include "db/log_writer.h" +#include "db/logs_with_prep_tracker.h" +#include "db/memtable_list.h" +#include "db/periodic_task_scheduler.h" +#include "db/post_memtable_callback.h" +#include "db/pre_release_callback.h" +#include "db/range_del_aggregator.h" +#include "db/read_callback.h" +#include "db/seqno_to_time_mapping.h" +#include "db/snapshot_checker.h" +#include "db/snapshot_impl.h" +#include "db/trim_history_scheduler.h" +#include "db/version_edit.h" +#include "db/wal_manager.h" +#include "db/write_controller.h" +#include "db/write_thread.h" +#include "logging/event_logger.h" +#include "monitoring/instrumented_mutex.h" +#include "options/db_options.h" +#include "port/port.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/status.h" +#include "rocksdb/trace_reader_writer.h" +#include "rocksdb/transaction_log.h" +#include "rocksdb/utilities/replayer.h" +#include "rocksdb/write_buffer_manager.h" +#include "table/merging_iterator.h" +#include "table/scoped_arena_iterator.h" +#include "util/autovector.h" +#include "util/hash.h" +#include "util/repeatable_thread.h" +#include "util/stop_watch.h" +#include "util/thread_local.h" + +namespace ROCKSDB_NAMESPACE { + +class Arena; +class ArenaWrappedDBIter; +class InMemoryStatsHistoryIterator; +class MemTable; +class PersistentStatsHistoryIterator; +class TableCache; +class TaskLimiterToken; +class Version; +class VersionEdit; +class VersionSet; +class WriteCallback; +struct JobContext; +struct ExternalSstFileInfo; +struct MemTableInfo; + +// Class to maintain directories for all database paths other than main one. +class Directories { + public: + IOStatus SetDirectories(FileSystem* fs, const std::string& dbname, + const std::string& wal_dir, + const std::vector& data_paths); + + FSDirectory* GetDataDir(size_t path_id) const { + assert(path_id < data_dirs_.size()); + FSDirectory* ret_dir = data_dirs_[path_id].get(); + if (ret_dir == nullptr) { + // Should use db_dir_ + return db_dir_.get(); + } + return ret_dir; + } + + FSDirectory* GetWalDir() { + if (wal_dir_) { + return wal_dir_.get(); + } + return db_dir_.get(); + } + + FSDirectory* GetDbDir() { return db_dir_.get(); } + + IOStatus Close(const IOOptions& options, IODebugContext* dbg) { + // close all directories for all database paths + IOStatus s = IOStatus::OK(); + + // The default implementation for Close() in Directory/FSDirectory class + // "NotSupported" status, the upper level interface should be able to + // handle this error so that Close() does not fail after upgrading when + // run on FileSystems that have not implemented `Directory::Close()` or + // `FSDirectory::Close()` yet + + if (db_dir_) { + IOStatus temp_s = db_dir_->Close(options, dbg); + if (!temp_s.ok() && !temp_s.IsNotSupported() && s.ok()) { + s = std::move(temp_s); + } + } + + // Attempt to close everything even if one fails + s.PermitUncheckedError(); + + if (wal_dir_) { + IOStatus temp_s = wal_dir_->Close(options, dbg); + if (!temp_s.ok() && !temp_s.IsNotSupported() && s.ok()) { + s = std::move(temp_s); + } + } + + s.PermitUncheckedError(); + + for (auto& data_dir_ptr : data_dirs_) { + if (data_dir_ptr) { + IOStatus temp_s = data_dir_ptr->Close(options, dbg); + if (!temp_s.ok() && !temp_s.IsNotSupported() && s.ok()) { + s = std::move(temp_s); + } + } + } + + // Ready for caller + s.MustCheck(); + return s; + } + + private: + std::unique_ptr db_dir_; + std::vector> data_dirs_; + std::unique_ptr wal_dir_; +}; + +// While DB is the public interface of RocksDB, and DBImpl is the actual +// class implementing it. It's the entrance of the core RocksdB engine. +// All other DB implementations, e.g. TransactionDB, BlobDB, etc, wrap a +// DBImpl internally. +// Other than functions implementing the DB interface, some public +// functions are there for other internal components to call. For +// example, TransactionDB directly calls DBImpl::WriteImpl() and +// BlobDB directly calls DBImpl::GetImpl(). Some other functions +// are for sub-components to call. For example, ColumnFamilyHandleImpl +// calls DBImpl::FindObsoleteFiles(). +// +// Since it's a very large class, the definition of the functions is +// divided in several db_impl_*.cc files, besides db_impl.cc. +class DBImpl : public DB { + public: + DBImpl(const DBOptions& options, const std::string& dbname, + const bool seq_per_batch = false, const bool batch_per_txn = true, + bool read_only = false); + // No copying allowed + DBImpl(const DBImpl&) = delete; + void operator=(const DBImpl&) = delete; + + virtual ~DBImpl(); + + // ---- Implementations of the DB interface ---- + + using DB::Resume; + Status Resume() override; + + using DB::Put; + Status Put(const WriteOptions& options, ColumnFamilyHandle* column_family, + const Slice& key, const Slice& value) override; + Status Put(const WriteOptions& options, ColumnFamilyHandle* column_family, + const Slice& key, const Slice& ts, const Slice& value) override; + + using DB::PutEntity; + Status PutEntity(const WriteOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + const WideColumns& columns) override; + + using DB::Merge; + Status Merge(const WriteOptions& options, ColumnFamilyHandle* column_family, + const Slice& key, const Slice& value) override; + Status Merge(const WriteOptions& options, ColumnFamilyHandle* column_family, + const Slice& key, const Slice& ts, const Slice& value) override; + + using DB::Delete; + Status Delete(const WriteOptions& options, ColumnFamilyHandle* column_family, + const Slice& key) override; + Status Delete(const WriteOptions& options, ColumnFamilyHandle* column_family, + const Slice& key, const Slice& ts) override; + + using DB::SingleDelete; + Status SingleDelete(const WriteOptions& options, + ColumnFamilyHandle* column_family, + const Slice& key) override; + Status SingleDelete(const WriteOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + const Slice& ts) override; + + using DB::DeleteRange; + Status DeleteRange(const WriteOptions& options, + ColumnFamilyHandle* column_family, const Slice& begin_key, + const Slice& end_key) override; + Status DeleteRange(const WriteOptions& options, + ColumnFamilyHandle* column_family, const Slice& begin_key, + const Slice& end_key, const Slice& ts) override; + + using DB::Write; + virtual Status Write(const WriteOptions& options, + WriteBatch* updates) override; + + using DB::Get; + virtual Status Get(const ReadOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value) override; + virtual Status Get(const ReadOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value, std::string* timestamp) override; + + using DB::GetEntity; + Status GetEntity(const ReadOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableWideColumns* columns) override; + + using DB::GetMergeOperands; + Status GetMergeOperands(const ReadOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* merge_operands, + GetMergeOperandsOptions* get_merge_operands_options, + int* number_of_operands) override { + GetImplOptions get_impl_options; + get_impl_options.column_family = column_family; + get_impl_options.merge_operands = merge_operands; + get_impl_options.get_merge_operands_options = get_merge_operands_options; + get_impl_options.number_of_operands = number_of_operands; + get_impl_options.get_value = false; + return GetImpl(options, key, get_impl_options); + } + + using DB::MultiGet; + virtual std::vector MultiGet( + const ReadOptions& options, + const std::vector& column_family, + const std::vector& keys, + std::vector* values) override; + virtual std::vector MultiGet( + const ReadOptions& options, + const std::vector& column_family, + const std::vector& keys, std::vector* values, + std::vector* timestamps) override; + + // This MultiGet is a batched version, which may be faster than calling Get + // multiple times, especially if the keys have some spatial locality that + // enables them to be queried in the same SST files/set of files. The larger + // the batch size, the more scope for batching and performance improvement + // The values and statuses parameters are arrays with number of elements + // equal to keys.size(). This allows the storage for those to be alloacted + // by the caller on the stack for small batches + void MultiGet(const ReadOptions& options, ColumnFamilyHandle* column_family, + const size_t num_keys, const Slice* keys, PinnableSlice* values, + Status* statuses, const bool sorted_input = false) override; + void MultiGet(const ReadOptions& options, ColumnFamilyHandle* column_family, + const size_t num_keys, const Slice* keys, PinnableSlice* values, + std::string* timestamps, Status* statuses, + const bool sorted_input = false) override; + + void MultiGet(const ReadOptions& options, const size_t num_keys, + ColumnFamilyHandle** column_families, const Slice* keys, + PinnableSlice* values, Status* statuses, + const bool sorted_input = false) override; + void MultiGet(const ReadOptions& options, const size_t num_keys, + ColumnFamilyHandle** column_families, const Slice* keys, + PinnableSlice* values, std::string* timestamps, + Status* statuses, const bool sorted_input = false) override; + + void MultiGetWithCallback( + const ReadOptions& options, ColumnFamilyHandle* column_family, + ReadCallback* callback, + autovector* sorted_keys); + + using DB::MultiGetEntity; + + void MultiGetEntity(const ReadOptions& options, + ColumnFamilyHandle* column_family, size_t num_keys, + const Slice* keys, PinnableWideColumns* results, + Status* statuses, bool sorted_input) override; + + void MultiGetEntity(const ReadOptions& options, size_t num_keys, + ColumnFamilyHandle** column_families, const Slice* keys, + PinnableWideColumns* results, Status* statuses, + bool sorted_input) override; + + virtual Status CreateColumnFamily(const ColumnFamilyOptions& cf_options, + const std::string& column_family, + ColumnFamilyHandle** handle) override; + virtual Status CreateColumnFamilies( + const ColumnFamilyOptions& cf_options, + const std::vector& column_family_names, + std::vector* handles) override; + virtual Status CreateColumnFamilies( + const std::vector& column_families, + std::vector* handles) override; + virtual Status DropColumnFamily(ColumnFamilyHandle* column_family) override; + virtual Status DropColumnFamilies( + const std::vector& column_families) override; + + // Returns false if key doesn't exist in the database and true if it may. + // If value_found is not passed in as null, then return the value if found in + // memory. On return, if value was found, then value_found will be set to true + // , otherwise false. + using DB::KeyMayExist; + virtual bool KeyMayExist(const ReadOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + std::string* value, std::string* timestamp, + bool* value_found = nullptr) override; + + using DB::NewIterator; + virtual Iterator* NewIterator(const ReadOptions& options, + ColumnFamilyHandle* column_family) override; + virtual Status NewIterators( + const ReadOptions& options, + const std::vector& column_families, + std::vector* iterators) override; + + virtual const Snapshot* GetSnapshot() override; + virtual void ReleaseSnapshot(const Snapshot* snapshot) override; + // Create a timestamped snapshot. This snapshot can be shared by multiple + // readers. If any of them uses it for write conflict checking, then + // is_write_conflict_boundary is true. For simplicity, set it to true by + // default. + std::pair> CreateTimestampedSnapshot( + SequenceNumber snapshot_seq, uint64_t ts); + std::shared_ptr GetTimestampedSnapshot(uint64_t ts) const; + void ReleaseTimestampedSnapshotsOlderThan( + uint64_t ts, size_t* remaining_total_ss = nullptr); + Status GetTimestampedSnapshots(uint64_t ts_lb, uint64_t ts_ub, + std::vector>& + timestamped_snapshots) const; + + using DB::GetProperty; + virtual bool GetProperty(ColumnFamilyHandle* column_family, + const Slice& property, std::string* value) override; + using DB::GetMapProperty; + virtual bool GetMapProperty( + ColumnFamilyHandle* column_family, const Slice& property, + std::map* value) override; + using DB::GetIntProperty; + virtual bool GetIntProperty(ColumnFamilyHandle* column_family, + const Slice& property, uint64_t* value) override; + using DB::GetAggregatedIntProperty; + virtual bool GetAggregatedIntProperty(const Slice& property, + uint64_t* aggregated_value) override; + using DB::GetApproximateSizes; + virtual Status GetApproximateSizes(const SizeApproximationOptions& options, + ColumnFamilyHandle* column_family, + const Range* range, int n, + uint64_t* sizes) override; + using DB::GetApproximateMemTableStats; + virtual void GetApproximateMemTableStats(ColumnFamilyHandle* column_family, + const Range& range, + uint64_t* const count, + uint64_t* const size) override; + using DB::CompactRange; + virtual Status CompactRange(const CompactRangeOptions& options, + ColumnFamilyHandle* column_family, + const Slice* begin, const Slice* end) override; + + using DB::CompactFiles; + virtual Status CompactFiles( + const CompactionOptions& compact_options, + ColumnFamilyHandle* column_family, + const std::vector& input_file_names, const int output_level, + const int output_path_id = -1, + std::vector* const output_file_names = nullptr, + CompactionJobInfo* compaction_job_info = nullptr) override; + + virtual Status PauseBackgroundWork() override; + virtual Status ContinueBackgroundWork() override; + + virtual Status EnableAutoCompaction( + const std::vector& column_family_handles) override; + + virtual void EnableManualCompaction() override; + virtual void DisableManualCompaction() override; + + using DB::SetOptions; + Status SetOptions( + ColumnFamilyHandle* column_family, + const std::unordered_map& options_map) override; + + virtual Status SetDBOptions( + const std::unordered_map& options_map) override; + + using DB::NumberLevels; + virtual int NumberLevels(ColumnFamilyHandle* column_family) override; + using DB::MaxMemCompactionLevel; + virtual int MaxMemCompactionLevel(ColumnFamilyHandle* column_family) override; + using DB::Level0StopWriteTrigger; + virtual int Level0StopWriteTrigger( + ColumnFamilyHandle* column_family) override; + virtual const std::string& GetName() const override; + virtual Env* GetEnv() const override; + virtual FileSystem* GetFileSystem() const override; + using DB::GetOptions; + virtual Options GetOptions(ColumnFamilyHandle* column_family) const override; + using DB::GetDBOptions; + virtual DBOptions GetDBOptions() const override; + using DB::Flush; + virtual Status Flush(const FlushOptions& options, + ColumnFamilyHandle* column_family) override; + virtual Status Flush( + const FlushOptions& options, + const std::vector& column_families) override; + virtual Status FlushWAL(bool sync) override; + bool WALBufferIsEmpty(); + virtual Status SyncWAL() override; + virtual Status LockWAL() override; + virtual Status UnlockWAL() override; + + virtual SequenceNumber GetLatestSequenceNumber() const override; + + // IncreaseFullHistoryTsLow(ColumnFamilyHandle*, std::string) will acquire + // and release db_mutex + Status IncreaseFullHistoryTsLow(ColumnFamilyHandle* column_family, + std::string ts_low) override; + + // GetFullHistoryTsLow(ColumnFamilyHandle*, std::string*) will acquire and + // release db_mutex + Status GetFullHistoryTsLow(ColumnFamilyHandle* column_family, + std::string* ts_low) override; + + virtual Status GetDbIdentity(std::string& identity) const override; + + virtual Status GetDbIdentityFromIdentityFile(std::string* identity) const; + + virtual Status GetDbSessionId(std::string& session_id) const override; + + ColumnFamilyHandle* DefaultColumnFamily() const override; + + ColumnFamilyHandle* PersistentStatsColumnFamily() const; + + virtual Status Close() override; + + virtual Status DisableFileDeletions() override; + + virtual Status EnableFileDeletions(bool force) override; + + virtual bool IsFileDeletionsEnabled() const; + + Status GetStatsHistory( + uint64_t start_time, uint64_t end_time, + std::unique_ptr* stats_iterator) override; + + using DB::ResetStats; + virtual Status ResetStats() override; + // All the returned filenames start with "/" + virtual Status GetLiveFiles(std::vector&, + uint64_t* manifest_file_size, + bool flush_memtable = true) override; + virtual Status GetSortedWalFiles(VectorLogPtr& files) override; + virtual Status GetCurrentWalFile( + std::unique_ptr* current_log_file) override; + virtual Status GetCreationTimeOfOldestFile( + uint64_t* creation_time) override; + + virtual Status GetUpdatesSince( + SequenceNumber seq_number, std::unique_ptr* iter, + const TransactionLogIterator::ReadOptions& read_options = + TransactionLogIterator::ReadOptions()) override; + virtual Status DeleteFile(std::string name) override; + Status DeleteFilesInRanges(ColumnFamilyHandle* column_family, + const RangePtr* ranges, size_t n, + bool include_end = true); + + virtual void GetLiveFilesMetaData( + std::vector* metadata) override; + + virtual Status GetLiveFilesChecksumInfo( + FileChecksumList* checksum_list) override; + + virtual Status GetLiveFilesStorageInfo( + const LiveFilesStorageInfoOptions& opts, + std::vector* files) override; + + // Obtains the meta data of the specified column family of the DB. + // TODO(yhchiang): output parameter is placed in the end in this codebase. + virtual void GetColumnFamilyMetaData(ColumnFamilyHandle* column_family, + ColumnFamilyMetaData* metadata) override; + + void GetAllColumnFamilyMetaData( + std::vector* metadata) override; + + Status SuggestCompactRange(ColumnFamilyHandle* column_family, + const Slice* begin, const Slice* end) override; + + Status PromoteL0(ColumnFamilyHandle* column_family, + int target_level) override; + + using DB::IngestExternalFile; + virtual Status IngestExternalFile( + ColumnFamilyHandle* column_family, + const std::vector& external_files, + const IngestExternalFileOptions& ingestion_options) override; + + using DB::IngestExternalFiles; + virtual Status IngestExternalFiles( + const std::vector& args) override; + + using DB::CreateColumnFamilyWithImport; + virtual Status CreateColumnFamilyWithImport( + const ColumnFamilyOptions& options, const std::string& column_family_name, + const ImportColumnFamilyOptions& import_options, + const std::vector& metadatas, + ColumnFamilyHandle** handle) override; + + using DB::ClipColumnFamily; + virtual Status ClipColumnFamily(ColumnFamilyHandle* column_family, + const Slice& begin_key, + const Slice& end_key) override; + + using DB::VerifyFileChecksums; + Status VerifyFileChecksums(const ReadOptions& read_options) override; + + using DB::VerifyChecksum; + virtual Status VerifyChecksum(const ReadOptions& /*read_options*/) override; + // Verify the checksums of files in db. Currently only tables are checked. + // + // read_options: controls file I/O behavior, e.g. read ahead size while + // reading all the live table files. + // + // use_file_checksum: if false, verify the block checksums of all live table + // in db. Otherwise, obtain the file checksums and compare + // with the MANIFEST. Currently, file checksums are + // recomputed by reading all table files. + // + // Returns: OK if there is no file whose file or block checksum mismatches. + Status VerifyChecksumInternal(const ReadOptions& read_options, + bool use_file_checksum); + + Status VerifyFullFileChecksum(const std::string& file_checksum_expected, + const std::string& func_name_expected, + const std::string& fpath, + const ReadOptions& read_options); + + using DB::StartTrace; + virtual Status StartTrace( + const TraceOptions& options, + std::unique_ptr&& trace_writer) override; + + using DB::EndTrace; + virtual Status EndTrace() override; + + using DB::NewDefaultReplayer; + virtual Status NewDefaultReplayer( + const std::vector& handles, + std::unique_ptr&& reader, + std::unique_ptr* replayer) override; + + using DB::StartBlockCacheTrace; + Status StartBlockCacheTrace( + const TraceOptions& trace_options, + std::unique_ptr&& trace_writer) override; + + Status StartBlockCacheTrace( + const BlockCacheTraceOptions& options, + std::unique_ptr&& trace_writer) override; + + using DB::EndBlockCacheTrace; + Status EndBlockCacheTrace() override; + + using DB::StartIOTrace; + Status StartIOTrace(const TraceOptions& options, + std::unique_ptr&& trace_writer) override; + + using DB::EndIOTrace; + Status EndIOTrace() override; + + using DB::GetPropertiesOfAllTables; + virtual Status GetPropertiesOfAllTables( + ColumnFamilyHandle* column_family, + TablePropertiesCollection* props) override; + virtual Status GetPropertiesOfTablesInRange( + ColumnFamilyHandle* column_family, const Range* range, std::size_t n, + TablePropertiesCollection* props) override; + + + // ---- End of implementations of the DB interface ---- + SystemClock* GetSystemClock() const; + + struct GetImplOptions { + ColumnFamilyHandle* column_family = nullptr; + PinnableSlice* value = nullptr; + PinnableWideColumns* columns = nullptr; + std::string* timestamp = nullptr; + bool* value_found = nullptr; + ReadCallback* callback = nullptr; + bool* is_blob_index = nullptr; + // If true return value associated with key via value pointer else return + // all merge operands for key via merge_operands pointer + bool get_value = true; + // Pointer to an array of size + // get_merge_operands_options.expected_max_number_of_operands allocated by + // user + PinnableSlice* merge_operands = nullptr; + GetMergeOperandsOptions* get_merge_operands_options = nullptr; + int* number_of_operands = nullptr; + }; + + // Function that Get and KeyMayExist call with no_io true or false + // Note: 'value_found' from KeyMayExist propagates here + // This function is also called by GetMergeOperands + // If get_impl_options.get_value = true get value associated with + // get_impl_options.key via get_impl_options.value + // If get_impl_options.get_value = false get merge operands associated with + // get_impl_options.key via get_impl_options.merge_operands + Status GetImpl(const ReadOptions& options, const Slice& key, + GetImplOptions& get_impl_options); + + // If `snapshot` == kMaxSequenceNumber, set a recent one inside the file. + ArenaWrappedDBIter* NewIteratorImpl(const ReadOptions& options, + ColumnFamilyData* cfd, + SequenceNumber snapshot, + ReadCallback* read_callback, + bool expose_blob_index = false, + bool allow_refresh = true); + + virtual SequenceNumber GetLastPublishedSequence() const { + if (last_seq_same_as_publish_seq_) { + return versions_->LastSequence(); + } else { + return versions_->LastPublishedSequence(); + } + } + + // REQUIRES: joined the main write queue if two_write_queues is disabled, and + // the second write queue otherwise. + virtual void SetLastPublishedSequence(SequenceNumber seq); + // Returns LastSequence in last_seq_same_as_publish_seq_ + // mode and LastAllocatedSequence otherwise. This is useful when visiblility + // depends also on data written to the WAL but not to the memtable. + SequenceNumber TEST_GetLastVisibleSequence() const; + + // Similar to Write() but will call the callback once on the single write + // thread to determine whether it is safe to perform the write. + virtual Status WriteWithCallback(const WriteOptions& write_options, + WriteBatch* my_batch, + WriteCallback* callback); + + // Returns the sequence number that is guaranteed to be smaller than or equal + // to the sequence number of any key that could be inserted into the current + // memtables. It can then be assumed that any write with a larger(or equal) + // sequence number will be present in this memtable or a later memtable. + // + // If the earliest sequence number could not be determined, + // kMaxSequenceNumber will be returned. + // + // If include_history=true, will also search Memtables in MemTableList + // History. + SequenceNumber GetEarliestMemTableSequenceNumber(SuperVersion* sv, + bool include_history); + + // For a given key, check to see if there are any records for this key + // in the memtables, including memtable history. If cache_only is false, + // SST files will also be checked. + // + // `key` should NOT have user-defined timestamp appended to user key even if + // timestamp is enabled. + // + // If a key is found, *found_record_for_key will be set to true and + // *seq will be set to the stored sequence number for the latest + // operation on this key or kMaxSequenceNumber if unknown. If user-defined + // timestamp is enabled for this column family and timestamp is not nullptr, + // then *timestamp will be set to the stored timestamp for the latest + // operation on this key. + // If no key is found, *found_record_for_key will be set to false. + // + // Note: If cache_only=false, it is possible for *seq to be set to 0 if + // the sequence number has been cleared from the record. If the caller is + // holding an active db snapshot, we know the missing sequence must be less + // than the snapshot's sequence number (sequence numbers are only cleared + // when there are no earlier active snapshots). + // + // If NotFound is returned and found_record_for_key is set to false, then no + // record for this key was found. If the caller is holding an active db + // snapshot, we know that no key could have existing after this snapshot + // (since we do not compact keys that have an earlier snapshot). + // + // Only records newer than or at `lower_bound_seq` are guaranteed to be + // returned. Memtables and files may not be checked if it only contains data + // older than `lower_bound_seq`. + // + // Returns OK or NotFound on success, + // other status on unexpected error. + // TODO(andrewkr): this API need to be aware of range deletion operations + Status GetLatestSequenceForKey(SuperVersion* sv, const Slice& key, + bool cache_only, + SequenceNumber lower_bound_seq, + SequenceNumber* seq, std::string* timestamp, + bool* found_record_for_key, + bool* is_blob_index); + + Status TraceIteratorSeek(const uint32_t& cf_id, const Slice& key, + const Slice& lower_bound, const Slice upper_bound); + Status TraceIteratorSeekForPrev(const uint32_t& cf_id, const Slice& key, + const Slice& lower_bound, + const Slice upper_bound); + + // Similar to GetSnapshot(), but also lets the db know that this snapshot + // will be used for transaction write-conflict checking. The DB can then + // make sure not to compact any keys that would prevent a write-conflict from + // being detected. + const Snapshot* GetSnapshotForWriteConflictBoundary(); + + // checks if all live files exist on file system and that their file sizes + // match to our in-memory records + virtual Status CheckConsistency(); + + // max_file_num_to_ignore allows bottom level compaction to filter out newly + // compacted SST files. Setting max_file_num_to_ignore to kMaxUint64 will + // disable the filtering + // If `final_output_level` is not nullptr, it is set to manual compaction's + // output level if returned status is OK, and it may or may not be set to + // manual compaction's output level if returned status is not OK. + Status RunManualCompaction(ColumnFamilyData* cfd, int input_level, + int output_level, + const CompactRangeOptions& compact_range_options, + const Slice* begin, const Slice* end, + bool exclusive, bool disallow_trivial_move, + uint64_t max_file_num_to_ignore, + const std::string& trim_ts, + int* final_output_level = nullptr); + + // Return an internal iterator over the current state of the database. + // The keys of this iterator are internal keys (see format.h). + // The returned iterator should be deleted when no longer needed. + // If allow_unprepared_value is true, the returned iterator may defer reading + // the value and so will require PrepareValue() to be called before value(); + // allow_unprepared_value = false is convenient when this optimization is not + // useful, e.g. when reading the whole column family. + // + // read_options.ignore_range_deletions determines whether range tombstones are + // processed in the returned interator internally, i.e., whether range + // tombstone covered keys are in this iterator's output. + // @param read_options Must outlive the returned iterator. + InternalIterator* NewInternalIterator( + const ReadOptions& read_options, Arena* arena, SequenceNumber sequence, + ColumnFamilyHandle* column_family = nullptr, + bool allow_unprepared_value = false); + + // Note: to support DB iterator refresh, memtable range tombstones in the + // underlying merging iterator needs to be refreshed. If db_iter is not + // nullptr, db_iter->SetMemtableRangetombstoneIter() is called with the + // memtable range tombstone iterator used by the underlying merging iterator. + // This range tombstone iterator can be refreshed later by db_iter. + // @param read_options Must outlive the returned iterator. + InternalIterator* NewInternalIterator(const ReadOptions& read_options, + ColumnFamilyData* cfd, + SuperVersion* super_version, + Arena* arena, SequenceNumber sequence, + bool allow_unprepared_value, + ArenaWrappedDBIter* db_iter = nullptr); + + LogsWithPrepTracker* logs_with_prep_tracker() { + return &logs_with_prep_tracker_; + } + + struct BGJobLimits { + int max_flushes; + int max_compactions; + }; + // Returns maximum background flushes and compactions allowed to be scheduled + BGJobLimits GetBGJobLimits() const; + // Need a static version that can be called during SanitizeOptions(). + static BGJobLimits GetBGJobLimits(int max_background_flushes, + int max_background_compactions, + int max_background_jobs, + bool parallelize_compactions); + + // move logs pending closing from job_context to the DB queue and + // schedule a purge + void ScheduleBgLogWriterClose(JobContext* job_context); + + uint64_t MinLogNumberToKeep(); + + // Returns the lower bound file number for SSTs that won't be deleted, even if + // they're obsolete. This lower bound is used internally to prevent newly + // created flush/compaction output files from being deleted before they're + // installed. This technique avoids the need for tracking the exact numbers of + // files pending creation, although it prevents more files than necessary from + // being deleted. + uint64_t MinObsoleteSstNumberToKeep(); + + uint64_t GetObsoleteSstFilesSize(); + + // Returns the list of live files in 'live' and the list + // of all files in the filesystem in 'candidate_files'. + // If force == false and the last call was less than + // db_options_.delete_obsolete_files_period_micros microseconds ago, + // it will not fill up the job_context + void FindObsoleteFiles(JobContext* job_context, bool force, + bool no_full_scan = false); + + // Diffs the files listed in filenames and those that do not + // belong to live files are possibly removed. Also, removes all the + // files in sst_delete_files and log_delete_files. + // It is not necessary to hold the mutex when invoking this method. + // If FindObsoleteFiles() was run, we need to also run + // PurgeObsoleteFiles(), even if disable_delete_obsolete_files_ is true + void PurgeObsoleteFiles(JobContext& background_contet, + bool schedule_only = false); + + // Schedule a background job to actually delete obsolete files. + void SchedulePurge(); + + const SnapshotList& snapshots() const { return snapshots_; } + + // load list of snapshots to `snap_vector` that is no newer than `max_seq` + // in ascending order. + // `oldest_write_conflict_snapshot` is filled with the oldest snapshot + // which satisfies SnapshotImpl.is_write_conflict_boundary_ = true. + void LoadSnapshots(std::vector* snap_vector, + SequenceNumber* oldest_write_conflict_snapshot, + const SequenceNumber& max_seq) const { + InstrumentedMutexLock l(mutex()); + snapshots().GetAll(snap_vector, oldest_write_conflict_snapshot, max_seq); + } + + const ImmutableDBOptions& immutable_db_options() const { + return immutable_db_options_; + } + + // Cancel all background jobs, including flush, compaction, background + // purging, stats dumping threads, etc. If `wait` = true, wait for the + // running jobs to abort or finish before returning. Otherwise, only + // sends the signals. + void CancelAllBackgroundWork(bool wait); + + // Find Super version and reference it. Based on options, it might return + // the thread local cached one. + // Call ReturnAndCleanupSuperVersion() when it is no longer needed. + SuperVersion* GetAndRefSuperVersion(ColumnFamilyData* cfd); + + // Similar to the previous function but looks up based on a column family id. + // nullptr will be returned if this column family no longer exists. + // REQUIRED: this function should only be called on the write thread or if the + // mutex is held. + SuperVersion* GetAndRefSuperVersion(uint32_t column_family_id); + + // Un-reference the super version and clean it up if it is the last reference. + void CleanupSuperVersion(SuperVersion* sv); + + // Un-reference the super version and return it to thread local cache if + // needed. If it is the last reference of the super version. Clean it up + // after un-referencing it. + void ReturnAndCleanupSuperVersion(ColumnFamilyData* cfd, SuperVersion* sv); + + // Similar to the previous function but looks up based on a column family id. + // nullptr will be returned if this column family no longer exists. + // REQUIRED: this function should only be called on the write thread. + void ReturnAndCleanupSuperVersion(uint32_t colun_family_id, SuperVersion* sv); + + // REQUIRED: this function should only be called on the write thread or if the + // mutex is held. Return value only valid until next call to this function or + // mutex is released. + ColumnFamilyHandle* GetColumnFamilyHandle(uint32_t column_family_id); + + // Same as above, should called without mutex held and not on write thread. + std::unique_ptr GetColumnFamilyHandleUnlocked( + uint32_t column_family_id); + + // Returns the number of currently running flushes. + // REQUIREMENT: mutex_ must be held when calling this function. + int num_running_flushes() { + mutex_.AssertHeld(); + return num_running_flushes_; + } + + // Returns the number of currently running compactions. + // REQUIREMENT: mutex_ must be held when calling this function. + int num_running_compactions() { + mutex_.AssertHeld(); + return num_running_compactions_; + } + + const WriteController& write_controller() { return write_controller_; } + + // hollow transactions shell used for recovery. + // these will then be passed to TransactionDB so that + // locks can be reacquired before writing can resume. + struct RecoveredTransaction { + std::string name_; + bool unprepared_; + + struct BatchInfo { + uint64_t log_number_; + // TODO(lth): For unprepared, the memory usage here can be big for + // unprepared transactions. This is only useful for rollbacks, and we + // can in theory just keep keyset for that. + WriteBatch* batch_; + // Number of sub-batches. A new sub-batch is created if txn attempts to + // insert a duplicate key,seq to memtable. This is currently used in + // WritePreparedTxn/WriteUnpreparedTxn. + size_t batch_cnt_; + }; + + // This maps the seq of the first key in the batch to BatchInfo, which + // contains WriteBatch and other information relevant to the batch. + // + // For WriteUnprepared, batches_ can have size greater than 1, but for + // other write policies, it must be of size 1. + std::map batches_; + + explicit RecoveredTransaction(const uint64_t log, const std::string& name, + WriteBatch* batch, SequenceNumber seq, + size_t batch_cnt, bool unprepared) + : name_(name), unprepared_(unprepared) { + batches_[seq] = {log, batch, batch_cnt}; + } + + ~RecoveredTransaction() { + for (auto& it : batches_) { + delete it.second.batch_; + } + } + + void AddBatch(SequenceNumber seq, uint64_t log_number, WriteBatch* batch, + size_t batch_cnt, bool unprepared) { + assert(batches_.count(seq) == 0); + batches_[seq] = {log_number, batch, batch_cnt}; + // Prior state must be unprepared, since the prepare batch must be the + // last batch. + assert(unprepared_); + unprepared_ = unprepared; + } + }; + + bool allow_2pc() const { return immutable_db_options_.allow_2pc; } + + std::unordered_map + recovered_transactions() { + return recovered_transactions_; + } + + RecoveredTransaction* GetRecoveredTransaction(const std::string& name) { + auto it = recovered_transactions_.find(name); + if (it == recovered_transactions_.end()) { + return nullptr; + } else { + return it->second; + } + } + + void InsertRecoveredTransaction(const uint64_t log, const std::string& name, + WriteBatch* batch, SequenceNumber seq, + size_t batch_cnt, bool unprepared_batch) { + // For WriteUnpreparedTxn, InsertRecoveredTransaction is called multiple + // times for every unprepared batch encountered during recovery. + // + // If the transaction is prepared, then the last call to + // InsertRecoveredTransaction will have unprepared_batch = false. + auto rtxn = recovered_transactions_.find(name); + if (rtxn == recovered_transactions_.end()) { + recovered_transactions_[name] = new RecoveredTransaction( + log, name, batch, seq, batch_cnt, unprepared_batch); + } else { + rtxn->second->AddBatch(seq, log, batch, batch_cnt, unprepared_batch); + } + logs_with_prep_tracker_.MarkLogAsContainingPrepSection(log); + } + + void DeleteRecoveredTransaction(const std::string& name) { + auto it = recovered_transactions_.find(name); + assert(it != recovered_transactions_.end()); + auto* trx = it->second; + recovered_transactions_.erase(it); + for (const auto& info : trx->batches_) { + logs_with_prep_tracker_.MarkLogAsHavingPrepSectionFlushed( + info.second.log_number_); + } + delete trx; + } + + void DeleteAllRecoveredTransactions() { + for (auto it = recovered_transactions_.begin(); + it != recovered_transactions_.end(); ++it) { + delete it->second; + } + recovered_transactions_.clear(); + } + + void AddToLogsToFreeQueue(log::Writer* log_writer) { + mutex_.AssertHeld(); + logs_to_free_queue_.push_back(log_writer); + } + + void AddSuperVersionsToFreeQueue(SuperVersion* sv) { + superversions_to_free_queue_.push_back(sv); + } + + void SetSnapshotChecker(SnapshotChecker* snapshot_checker); + + // Fill JobContext with snapshot information needed by flush and compaction. + void GetSnapshotContext(JobContext* job_context, + std::vector* snapshot_seqs, + SequenceNumber* earliest_write_conflict_snapshot, + SnapshotChecker** snapshot_checker); + + // Not thread-safe. + void SetRecoverableStatePreReleaseCallback(PreReleaseCallback* callback); + + InstrumentedMutex* mutex() const { return &mutex_; } + + // Initialize a brand new DB. The DB directory is expected to be empty before + // calling it. Push new manifest file name into `new_filenames`. + Status NewDB(std::vector* new_filenames); + + // This is to be used only by internal rocksdb classes. + static Status Open(const DBOptions& db_options, const std::string& name, + const std::vector& column_families, + std::vector* handles, DB** dbptr, + const bool seq_per_batch, const bool batch_per_txn); + + static IOStatus CreateAndNewDirectory( + FileSystem* fs, const std::string& dirname, + std::unique_ptr* directory); + + // find stats map from stats_history_ with smallest timestamp in + // the range of [start_time, end_time) + bool FindStatsByTime(uint64_t start_time, uint64_t end_time, + uint64_t* new_time, + std::map* stats_map); + + // Print information of all tombstones of all iterators to the std::string + // This is only used by ldb. The output might be capped. Tombstones + // printed out are not guaranteed to be in any order. + Status TablesRangeTombstoneSummary(ColumnFamilyHandle* column_family, + int max_entries_to_print, + std::string* out_str); + + VersionSet* GetVersionSet() const { return versions_.get(); } + + Status WaitForCompact( + const WaitForCompactOptions& wait_for_compact_options) override; + +#ifndef NDEBUG + // Compact any files in the named level that overlap [*begin, *end] + Status TEST_CompactRange(int level, const Slice* begin, const Slice* end, + ColumnFamilyHandle* column_family = nullptr, + bool disallow_trivial_move = false); + + Status TEST_SwitchWAL(); + + bool TEST_UnableToReleaseOldestLog() { return unable_to_release_oldest_log_; } + + bool TEST_IsLogGettingFlushed() { + return alive_log_files_.begin()->getting_flushed; + } + + Status TEST_SwitchMemtable(ColumnFamilyData* cfd = nullptr); + + // Force current memtable contents to be flushed. + Status TEST_FlushMemTable(bool wait = true, bool allow_write_stall = false, + ColumnFamilyHandle* cfh = nullptr); + + Status TEST_FlushMemTable(ColumnFamilyData* cfd, + const FlushOptions& flush_opts); + + // Flush (multiple) ColumnFamilyData without using ColumnFamilyHandle. This + // is because in certain cases, we can flush column families, wait for the + // flush to complete, but delete the column family handle before the wait + // finishes. For example in CompactRange. + Status TEST_AtomicFlushMemTables( + const autovector& provided_candidate_cfds, + const FlushOptions& flush_opts); + + // Wait for background threads to complete scheduled work. + Status TEST_WaitForBackgroundWork(); + + // Wait for memtable compaction + Status TEST_WaitForFlushMemTable(ColumnFamilyHandle* column_family = nullptr); + + Status TEST_WaitForCompact(); + Status TEST_WaitForCompact( + const WaitForCompactOptions& wait_for_compact_options); + + // Wait for any background purge + Status TEST_WaitForPurge(); + + // Get the background error status + Status TEST_GetBGError(); + + // Return the maximum overlapping data (in bytes) at next level for any + // file at a level >= 1. + uint64_t TEST_MaxNextLevelOverlappingBytes( + ColumnFamilyHandle* column_family = nullptr); + + // Return the current manifest file no. + uint64_t TEST_Current_Manifest_FileNo(); + + // Returns the number that'll be assigned to the next file that's created. + uint64_t TEST_Current_Next_FileNo(); + + // get total level0 file size. Only for testing. + uint64_t TEST_GetLevel0TotalSize(); + + void TEST_GetFilesMetaData( + ColumnFamilyHandle* column_family, + std::vector>* metadata, + std::vector>* blob_metadata = nullptr); + + void TEST_LockMutex(); + + void TEST_UnlockMutex(); + + void TEST_SignalAllBgCv(); + + // REQUIRES: mutex locked + void* TEST_BeginWrite(); + + // REQUIRES: mutex locked + // pass the pointer that you got from TEST_BeginWrite() + void TEST_EndWrite(void* w); + + uint64_t TEST_MaxTotalInMemoryState() const { + return max_total_in_memory_state_; + } + + size_t TEST_LogsToFreeSize(); + + uint64_t TEST_LogfileNumber(); + + uint64_t TEST_total_log_size() const { return total_log_size_; } + + // Returns column family name to ImmutableCFOptions map. + Status TEST_GetAllImmutableCFOptions( + std::unordered_map* iopts_map); + + // Return the lastest MutableCFOptions of a column family + Status TEST_GetLatestMutableCFOptions(ColumnFamilyHandle* column_family, + MutableCFOptions* mutable_cf_options); + + Cache* TEST_table_cache() { return table_cache_.get(); } + + WriteController& TEST_write_controler() { return write_controller_; } + + uint64_t TEST_FindMinLogContainingOutstandingPrep(); + uint64_t TEST_FindMinPrepLogReferencedByMemTable(); + size_t TEST_PreparedSectionCompletedSize(); + size_t TEST_LogsWithPrepSize(); + + int TEST_BGCompactionsAllowed() const; + int TEST_BGFlushesAllowed() const; + size_t TEST_GetWalPreallocateBlockSize(uint64_t write_buffer_size) const; + void TEST_WaitForPeriodicTaskRun(std::function callback) const; + SeqnoToTimeMapping TEST_GetSeqnoToTimeMapping() const; + size_t TEST_EstimateInMemoryStatsHistorySize() const; + + uint64_t TEST_GetCurrentLogNumber() const { + InstrumentedMutexLock l(mutex()); + assert(!logs_.empty()); + return logs_.back().number; + } + + const std::unordered_set& TEST_GetFilesGrabbedForPurge() const { + return files_grabbed_for_purge_; + } + + const PeriodicTaskScheduler& TEST_GetPeriodicTaskScheduler() const; + +#endif // NDEBUG + + // persist stats to column family "_persistent_stats" + void PersistStats(); + + // dump rocksdb.stats to LOG + void DumpStats(); + + // flush LOG out of application buffer + void FlushInfoLog(); + + // record current sequence number to time mapping + void RecordSeqnoToTimeMapping(); + + // Interface to block and signal the DB in case of stalling writes by + // WriteBufferManager. Each DBImpl object contains ptr to WBMStallInterface. + // When DB needs to be blocked or signalled by WriteBufferManager, + // state_ is changed accordingly. + class WBMStallInterface : public StallInterface { + public: + enum State { + BLOCKED = 0, + RUNNING, + }; + + WBMStallInterface() : state_cv_(&state_mutex_) { + MutexLock lock(&state_mutex_); + state_ = State::RUNNING; + } + + void SetState(State state) { + MutexLock lock(&state_mutex_); + state_ = state; + } + + // Change the state_ to State::BLOCKED and wait until its state is + // changed by WriteBufferManager. When stall is cleared, Signal() is + // called to change the state and unblock the DB. + void Block() override { + MutexLock lock(&state_mutex_); + while (state_ == State::BLOCKED) { + TEST_SYNC_POINT("WBMStallInterface::BlockDB"); + state_cv_.Wait(); + } + } + + // Called from WriteBufferManager. This function changes the state_ + // to State::RUNNING indicating the stall is cleared and DB can proceed. + void Signal() override { + { + MutexLock lock(&state_mutex_); + state_ = State::RUNNING; + } + state_cv_.Signal(); + } + + private: + // Conditional variable and mutex to block and + // signal the DB during stalling process. + port::Mutex state_mutex_; + port::CondVar state_cv_; + // state represting whether DB is running or blocked because of stall by + // WriteBufferManager. + State state_; + }; + + static void TEST_ResetDbSessionIdGen(); + static std::string GenerateDbSessionId(Env* env); + + bool seq_per_batch() const { return seq_per_batch_; } + + protected: + const std::string dbname_; + // TODO(peterd): unify with VersionSet::db_id_ + std::string db_id_; + // db_session_id_ is an identifier that gets reset + // every time the DB is opened + std::string db_session_id_; + std::unique_ptr versions_; + // Flag to check whether we allocated and own the info log file + bool own_info_log_; + Status init_logger_creation_s_; + const DBOptions initial_db_options_; + Env* const env_; + std::shared_ptr io_tracer_; + const ImmutableDBOptions immutable_db_options_; + FileSystemPtr fs_; + MutableDBOptions mutable_db_options_; + Statistics* stats_; + std::unordered_map + recovered_transactions_; + std::unique_ptr tracer_; + InstrumentedMutex trace_mutex_; + BlockCacheTracer block_cache_tracer_; + + // constant false canceled flag, used when the compaction is not manual + const std::atomic kManualCompactionCanceledFalse_{false}; + + // State below is protected by mutex_ + // With two_write_queues enabled, some of the variables that accessed during + // WriteToWAL need different synchronization: log_empty_, alive_log_files_, + // logs_, logfile_number_. Refer to the definition of each variable below for + // more description. + // + // `mutex_` can be a hot lock in some workloads, so it deserves dedicated + // cachelines. + mutable CacheAlignedInstrumentedMutex mutex_; + + ColumnFamilyHandleImpl* default_cf_handle_; + InternalStats* default_cf_internal_stats_; + + // table_cache_ provides its own synchronization + std::shared_ptr table_cache_; + + ErrorHandler error_handler_; + + // Unified interface for logging events + EventLogger event_logger_; + + // only used for dynamically adjusting max_total_wal_size. it is a sum of + // [write_buffer_size * max_write_buffer_number] over all column families + std::atomic max_total_in_memory_state_; + + // The options to access storage files + const FileOptions file_options_; + + // Additonal options for compaction and flush + FileOptions file_options_for_compaction_; + + std::unique_ptr column_family_memtables_; + + // Increase the sequence number after writing each batch, whether memtable is + // disabled for that or not. Otherwise the sequence number is increased after + // writing each key into memtable. This implies that when disable_memtable is + // set, the seq is not increased at all. + // + // Default: false + const bool seq_per_batch_; + // This determines during recovery whether we expect one writebatch per + // recovered transaction, or potentially multiple writebatches per + // transaction. For WriteUnprepared, this is set to false, since multiple + // batches can exist per transaction. + // + // Default: true + const bool batch_per_txn_; + + // Each flush or compaction gets its own job id. this counter makes sure + // they're unique + std::atomic next_job_id_; + + std::atomic shutting_down_; + + // RecoveryContext struct stores the context about version edits along + // with corresponding column_family_data and column_family_options. + class RecoveryContext { + public: + ~RecoveryContext() { + for (auto& edit_list : edit_lists_) { + for (auto* edit : edit_list) { + delete edit; + } + } + } + + void UpdateVersionEdits(ColumnFamilyData* cfd, const VersionEdit& edit) { + assert(cfd != nullptr); + if (map_.find(cfd->GetID()) == map_.end()) { + uint32_t size = static_cast(map_.size()); + map_.emplace(cfd->GetID(), size); + cfds_.emplace_back(cfd); + mutable_cf_opts_.emplace_back(cfd->GetLatestMutableCFOptions()); + edit_lists_.emplace_back(autovector()); + } + uint32_t i = map_[cfd->GetID()]; + edit_lists_[i].emplace_back(new VersionEdit(edit)); + } + + std::unordered_map map_; // cf_id to index; + autovector cfds_; + autovector mutable_cf_opts_; + autovector> edit_lists_; + // files_to_delete_ contains sst files + std::unordered_set files_to_delete_; + }; + + // Except in DB::Open(), WriteOptionsFile can only be called when: + // Persist options to options file. + // If need_mutex_lock = false, the method will lock DB mutex. + // If need_enter_write_thread = false, the method will enter write thread. + Status WriteOptionsFile(bool need_mutex_lock, bool need_enter_write_thread); + + Status CompactRangeInternal(const CompactRangeOptions& options, + ColumnFamilyHandle* column_family, + const Slice* begin, const Slice* end, + const std::string& trim_ts); + + // The following two functions can only be called when: + // 1. WriteThread::Writer::EnterUnbatched() is used. + // 2. db_mutex is NOT held + Status RenameTempFileToOptionsFile(const std::string& file_name); + Status DeleteObsoleteOptionsFiles(); + + void NotifyOnFlushBegin(ColumnFamilyData* cfd, FileMetaData* file_meta, + const MutableCFOptions& mutable_cf_options, + int job_id, FlushReason flush_reason); + + void NotifyOnFlushCompleted( + ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options, + std::list>* flush_jobs_info); + + void NotifyOnCompactionBegin(ColumnFamilyData* cfd, Compaction* c, + const Status& st, + const CompactionJobStats& job_stats, int job_id); + + void NotifyOnCompactionCompleted(ColumnFamilyData* cfd, Compaction* c, + const Status& st, + const CompactionJobStats& job_stats, + int job_id); + void NotifyOnMemTableSealed(ColumnFamilyData* cfd, + const MemTableInfo& mem_table_info); + + void NotifyOnExternalFileIngested( + ColumnFamilyData* cfd, const ExternalSstFileIngestionJob& ingestion_job); + + Status FlushAllColumnFamilies(const FlushOptions& flush_options, + FlushReason flush_reason); + + virtual Status FlushForGetLiveFiles(); + + void NewThreadStatusCfInfo(ColumnFamilyData* cfd) const; + + void EraseThreadStatusCfInfo(ColumnFamilyData* cfd) const; + + void EraseThreadStatusDbInfo() const; + + // If disable_memtable is set the application logic must guarantee that the + // batch will still be skipped from memtable during the recovery. An excption + // to this is seq_per_batch_ mode, in which since each batch already takes one + // seq, it is ok for the batch to write to memtable during recovery as long as + // it only takes one sequence number: i.e., no duplicate keys. + // In WriteCommitted it is guarnateed since disable_memtable is used for + // prepare batch which will be written to memtable later during the commit, + // and in WritePrepared it is guaranteed since it will be used only for WAL + // markers which will never be written to memtable. If the commit marker is + // accompanied with CommitTimeWriteBatch that is not written to memtable as + // long as it has no duplicate keys, it does not violate the one-seq-per-batch + // policy. + // batch_cnt is expected to be non-zero in seq_per_batch mode and + // indicates the number of sub-patches. A sub-patch is a subset of the write + // batch that does not have duplicate keys. + Status WriteImpl(const WriteOptions& options, WriteBatch* updates, + WriteCallback* callback = nullptr, + uint64_t* log_used = nullptr, uint64_t log_ref = 0, + bool disable_memtable = false, uint64_t* seq_used = nullptr, + size_t batch_cnt = 0, + PreReleaseCallback* pre_release_callback = nullptr, + PostMemTableCallback* post_memtable_callback = nullptr); + + Status PipelinedWriteImpl(const WriteOptions& options, WriteBatch* updates, + WriteCallback* callback = nullptr, + uint64_t* log_used = nullptr, uint64_t log_ref = 0, + bool disable_memtable = false, + uint64_t* seq_used = nullptr); + + // Write only to memtables without joining any write queue + Status UnorderedWriteMemtable(const WriteOptions& write_options, + WriteBatch* my_batch, WriteCallback* callback, + uint64_t log_ref, SequenceNumber seq, + const size_t sub_batch_cnt); + + // Whether the batch requires to be assigned with an order + enum AssignOrder : bool { kDontAssignOrder, kDoAssignOrder }; + // Whether it requires publishing last sequence or not + enum PublishLastSeq : bool { kDontPublishLastSeq, kDoPublishLastSeq }; + + // Join the write_thread to write the batch only to the WAL. It is the + // responsibility of the caller to also write the write batch to the memtable + // if it required. + // + // sub_batch_cnt is expected to be non-zero when assign_order = kDoAssignOrder + // indicating the number of sub-batches in my_batch. A sub-patch is a subset + // of the write batch that does not have duplicate keys. When seq_per_batch is + // not set, each key is a separate sub_batch. Otherwise each duplicate key + // marks start of a new sub-batch. + Status WriteImplWALOnly( + WriteThread* write_thread, const WriteOptions& options, + WriteBatch* updates, WriteCallback* callback, uint64_t* log_used, + const uint64_t log_ref, uint64_t* seq_used, const size_t sub_batch_cnt, + PreReleaseCallback* pre_release_callback, const AssignOrder assign_order, + const PublishLastSeq publish_last_seq, const bool disable_memtable); + + // write cached_recoverable_state_ to memtable if it is not empty + // The writer must be the leader in write_thread_ and holding mutex_ + Status WriteRecoverableState(); + + // Actual implementation of Close() + Status CloseImpl(); + + // Recover the descriptor from persistent storage. May do a significant + // amount of work to recover recently logged updates. Any changes to + // be made to the descriptor are added to *edit. + // recovered_seq is set to less than kMaxSequenceNumber if the log's tail is + // skipped. + // recovery_ctx stores the context about version edits and all those + // edits are persisted to new Manifest after successfully syncing the new WAL. + virtual Status Recover( + const std::vector& column_families, + bool read_only = false, bool error_if_wal_file_exists = false, + bool error_if_data_exists_in_wals = false, + uint64_t* recovered_seq = nullptr, + RecoveryContext* recovery_ctx = nullptr); + + virtual bool OwnTablesAndLogs() const { return true; } + + // Setup DB identity file, and write DB ID to manifest if necessary. + Status SetupDBId(bool read_only, RecoveryContext* recovery_ctx); + // Assign db_id_ and write DB ID to manifest if necessary. + void SetDBId(std::string&& id, bool read_only, RecoveryContext* recovery_ctx); + + // REQUIRES: db mutex held when calling this function, but the db mutex can + // be released and re-acquired. Db mutex will be held when the function + // returns. + // After recovery, there may be SST files in db/cf paths that are + // not referenced in the MANIFEST (e.g. + // 1. It's best effort recovery; + // 2. The VersionEdits referencing the SST files are appended to + // RecoveryContext, DB crashes when syncing the MANIFEST, the VersionEdits are + // still not synced to MANIFEST during recovery.) + // It stores the SST files to be deleted in RecoveryContext. In the + // meantime, we find out the largest file number present in the paths, and + // bump up the version set's next_file_number_ to be 1 + largest_file_number. + // recovery_ctx stores the context about version edits and files to be + // deleted. All those edits are persisted to new Manifest after successfully + // syncing the new WAL. + Status DeleteUnreferencedSstFiles(RecoveryContext* recovery_ctx); + + // SetDbSessionId() should be called in the constuctor DBImpl() + // to ensure that db_session_id_ gets updated every time the DB is opened + void SetDbSessionId(); + + Status FailIfCfHasTs(const ColumnFamilyHandle* column_family) const; + Status FailIfTsMismatchCf(ColumnFamilyHandle* column_family, const Slice& ts, + bool ts_for_read) const; + + // recovery_ctx stores the context about version edits and + // LogAndApplyForRecovery persist all those edits to new Manifest after + // successfully syncing new WAL. + // LogAndApplyForRecovery should be called only once during recovery and it + // should be called when RocksDB writes to a first new MANIFEST since this + // recovery. + Status LogAndApplyForRecovery(const RecoveryContext& recovery_ctx); + + void InvokeWalFilterIfNeededOnColumnFamilyToWalNumberMap(); + + // Return true to proceed with current WAL record whose content is stored in + // `batch`. Return false to skip current WAL record. + bool InvokeWalFilterIfNeededOnWalRecord(uint64_t wal_number, + const std::string& wal_fname, + log::Reader::Reporter& reporter, + Status& status, bool& stop_replay, + WriteBatch& batch); + + private: + friend class DB; + friend class ErrorHandler; + friend class InternalStats; + friend class PessimisticTransaction; + friend class TransactionBaseImpl; + friend class WriteCommittedTxn; + friend class WritePreparedTxn; + friend class WritePreparedTxnDB; + friend class WriteBatchWithIndex; + friend class WriteUnpreparedTxnDB; + friend class WriteUnpreparedTxn; + + friend class ForwardIterator; + friend struct SuperVersion; + friend class CompactedDBImpl; + friend class DBTest_ConcurrentFlushWAL_Test; + friend class DBTest_MixedSlowdownOptionsStop_Test; + friend class DBCompactionTest_CompactBottomLevelFilesWithDeletions_Test; + friend class DBCompactionTest_CompactionDuringShutdown_Test; + friend class StatsHistoryTest_PersistentStatsCreateColumnFamilies_Test; +#ifndef NDEBUG + friend class DBTest2_ReadCallbackTest_Test; + friend class WriteCallbackPTest_WriteWithCallbackTest_Test; + friend class XFTransactionWriteHandler; + friend class DBBlobIndexTest; + friend class WriteUnpreparedTransactionTest_RecoveryTest_Test; +#endif + + struct CompactionState; + struct PrepickedCompaction; + struct PurgeFileInfo; + + struct WriteContext { + SuperVersionContext superversion_context; + autovector memtables_to_free_; + + explicit WriteContext(bool create_superversion = false) + : superversion_context(create_superversion) {} + + ~WriteContext() { + superversion_context.Clean(); + for (auto& m : memtables_to_free_) { + delete m; + } + } + }; + + struct LogFileNumberSize { + explicit LogFileNumberSize(uint64_t _number) : number(_number) {} + LogFileNumberSize() {} + void AddSize(uint64_t new_size) { size += new_size; } + uint64_t number; + uint64_t size = 0; + bool getting_flushed = false; + }; + + struct LogWriterNumber { + // pass ownership of _writer + LogWriterNumber(uint64_t _number, log::Writer* _writer) + : number(_number), writer(_writer) {} + + log::Writer* ReleaseWriter() { + auto* w = writer; + writer = nullptr; + return w; + } + Status ClearWriter() { + Status s = writer->WriteBuffer(); + delete writer; + writer = nullptr; + return s; + } + + bool IsSyncing() { return getting_synced; } + + uint64_t GetPreSyncSize() { + assert(getting_synced); + return pre_sync_size; + } + + void PrepareForSync() { + assert(!getting_synced); + // Size is expected to be monotonically increasing. + assert(writer->file()->GetFlushedSize() >= pre_sync_size); + getting_synced = true; + pre_sync_size = writer->file()->GetFlushedSize(); + } + + void FinishSync() { + assert(getting_synced); + getting_synced = false; + } + + uint64_t number; + // Visual Studio doesn't support deque's member to be noncopyable because + // of a std::unique_ptr as a member. + log::Writer* writer; // own + + private: + // true for some prefix of logs_ + bool getting_synced = false; + // The size of the file before the sync happens. This amount is guaranteed + // to be persisted even if appends happen during sync so it can be used for + // tracking the synced size in MANIFEST. + uint64_t pre_sync_size = 0; + }; + + struct LogContext { + explicit LogContext(bool need_sync = false) + : need_log_sync(need_sync), need_log_dir_sync(need_sync) {} + bool need_log_sync = false; + bool need_log_dir_sync = false; + log::Writer* writer = nullptr; + LogFileNumberSize* log_file_number_size = nullptr; + }; + + // PurgeFileInfo is a structure to hold information of files to be deleted in + // purge_files_ + struct PurgeFileInfo { + std::string fname; + std::string dir_to_sync; + FileType type; + uint64_t number; + int job_id; + PurgeFileInfo(std::string fn, std::string d, FileType t, uint64_t num, + int jid) + : fname(fn), dir_to_sync(d), type(t), number(num), job_id(jid) {} + }; + + // Argument required by background flush thread. + struct BGFlushArg { + BGFlushArg() + : cfd_(nullptr), + max_memtable_id_(0), + superversion_context_(nullptr), + flush_reason_(FlushReason::kOthers) {} + BGFlushArg(ColumnFamilyData* cfd, uint64_t max_memtable_id, + SuperVersionContext* superversion_context, + FlushReason flush_reason) + : cfd_(cfd), + max_memtable_id_(max_memtable_id), + superversion_context_(superversion_context), + flush_reason_(flush_reason) {} + + // Column family to flush. + ColumnFamilyData* cfd_; + // Maximum ID of memtable to flush. In this column family, memtables with + // IDs smaller than this value must be flushed before this flush completes. + uint64_t max_memtable_id_; + // Pointer to a SuperVersionContext object. After flush completes, RocksDB + // installs a new superversion for the column family. This operation + // requires a SuperVersionContext object (currently embedded in JobContext). + SuperVersionContext* superversion_context_; + FlushReason flush_reason_; + }; + + // Argument passed to flush thread. + struct FlushThreadArg { + DBImpl* db_; + + Env::Priority thread_pri_; + }; + + // Information for a manual compaction + struct ManualCompactionState { + ManualCompactionState(ColumnFamilyData* _cfd, int _input_level, + int _output_level, uint32_t _output_path_id, + bool _exclusive, bool _disallow_trivial_move, + std::atomic* _canceled) + : cfd(_cfd), + input_level(_input_level), + output_level(_output_level), + output_path_id(_output_path_id), + exclusive(_exclusive), + disallow_trivial_move(_disallow_trivial_move), + canceled(_canceled ? *_canceled : canceled_internal_storage) {} + // When _canceled is not provided by ther user, we assign the reference of + // canceled_internal_storage to it to consolidate canceled and + // manual_compaction_paused since DisableManualCompaction() might be + // called + + ColumnFamilyData* cfd; + int input_level; + int output_level; + uint32_t output_path_id; + Status status; + bool done = false; + bool in_progress = false; // compaction request being processed? + bool incomplete = false; // only part of requested range compacted + bool exclusive; // current behavior of only one manual + bool disallow_trivial_move; // Force actual compaction to run + const InternalKey* begin = nullptr; // nullptr means beginning of key range + const InternalKey* end = nullptr; // nullptr means end of key range + InternalKey* manual_end = nullptr; // how far we are compacting + InternalKey tmp_storage; // Used to keep track of compaction progress + InternalKey tmp_storage1; // Used to keep track of compaction progress + + // When the user provides a canceled pointer in CompactRangeOptions, the + // above varaibe is the reference of the user-provided + // `canceled`, otherwise, it is the reference of canceled_internal_storage + std::atomic canceled_internal_storage = false; + std::atomic& canceled; // Compaction canceled pointer reference + }; + struct PrepickedCompaction { + // background compaction takes ownership of `compaction`. + Compaction* compaction; + // caller retains ownership of `manual_compaction_state` as it is reused + // across background compactions. + ManualCompactionState* manual_compaction_state; // nullptr if non-manual + // task limiter token is requested during compaction picking. + std::unique_ptr task_token; + }; + + struct CompactionArg { + // caller retains ownership of `db`. + DBImpl* db; + // background compaction takes ownership of `prepicked_compaction`. + PrepickedCompaction* prepicked_compaction; + Env::Priority compaction_pri_; + }; + + // Initialize the built-in column family for persistent stats. Depending on + // whether on-disk persistent stats have been enabled before, it may either + // create a new column family and column family handle or just a column family + // handle. + // Required: DB mutex held + Status InitPersistStatsColumnFamily(); + + // Persistent Stats column family has two format version key which are used + // for compatibility check. Write format version if it's created for the + // first time, read format version and check compatibility if recovering + // from disk. This function requires DB mutex held at entrance but may + // release and re-acquire DB mutex in the process. + // Required: DB mutex held + Status PersistentStatsProcessFormatVersion(); + + Status ResumeImpl(DBRecoverContext context); + + void MaybeIgnoreError(Status* s) const; + + const Status CreateArchivalDirectory(); + + Status CreateColumnFamilyImpl(const ColumnFamilyOptions& cf_options, + const std::string& cf_name, + ColumnFamilyHandle** handle); + + Status DropColumnFamilyImpl(ColumnFamilyHandle* column_family); + + // Delete any unneeded files and stale in-memory entries. + void DeleteObsoleteFiles(); + // Delete obsolete files and log status and information of file deletion + void DeleteObsoleteFileImpl(int job_id, const std::string& fname, + const std::string& path_to_sync, FileType type, + uint64_t number); + + // Background process needs to call + // auto x = CaptureCurrentFileNumberInPendingOutputs() + // auto file_num = versions_->NewFileNumber(); + // + // ReleaseFileNumberFromPendingOutputs(x) + // This will protect any file with number `file_num` or greater from being + // deleted while is running. + // ----------- + // This function will capture current file number and append it to + // pending_outputs_. This will prevent any background process to delete any + // file created after this point. + std::list::iterator CaptureCurrentFileNumberInPendingOutputs(); + // This function should be called with the result of + // CaptureCurrentFileNumberInPendingOutputs(). It then marks that any file + // created between the calls CaptureCurrentFileNumberInPendingOutputs() and + // ReleaseFileNumberFromPendingOutputs() can now be deleted (if it's not live + // and blocked by any other pending_outputs_ calls) + void ReleaseFileNumberFromPendingOutputs( + std::unique_ptr::iterator>& v); + + IOStatus SyncClosedLogs(JobContext* job_context, VersionEdit* synced_wals); + + // Flush the in-memory write buffer to storage. Switches to a new + // log-file/memtable and writes a new descriptor iff successful. Then + // installs a new super version for the column family. + Status FlushMemTableToOutputFile( + ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options, + bool* madeProgress, JobContext* job_context, FlushReason flush_reason, + SuperVersionContext* superversion_context, + std::vector& snapshot_seqs, + SequenceNumber earliest_write_conflict_snapshot, + SnapshotChecker* snapshot_checker, LogBuffer* log_buffer, + Env::Priority thread_pri); + + // Flush the memtables of (multiple) column families to multiple files on + // persistent storage. + Status FlushMemTablesToOutputFiles( + const autovector& bg_flush_args, bool* made_progress, + JobContext* job_context, LogBuffer* log_buffer, Env::Priority thread_pri); + + Status AtomicFlushMemTablesToOutputFiles( + const autovector& bg_flush_args, bool* made_progress, + JobContext* job_context, LogBuffer* log_buffer, Env::Priority thread_pri); + + // REQUIRES: log_numbers are sorted in ascending order + // corrupted_log_found is set to true if we recover from a corrupted log file. + Status RecoverLogFiles(const std::vector& log_numbers, + SequenceNumber* next_sequence, bool read_only, + bool* corrupted_log_found, + RecoveryContext* recovery_ctx); + + // The following two methods are used to flush a memtable to + // storage. The first one is used at database RecoveryTime (when the + // database is opened) and is heavyweight because it holds the mutex + // for the entire period. The second method WriteLevel0Table supports + // concurrent flush memtables to storage. + Status WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd, + MemTable* mem, VersionEdit* edit); + + // Get the size of a log file and, if truncate is true, truncate the + // log file to its actual size, thereby freeing preallocated space. + // Return success even if truncate fails + Status GetLogSizeAndMaybeTruncate(uint64_t wal_number, bool truncate, + LogFileNumberSize* log); + + // Restore alive_log_files_ and total_log_size_ after recovery. + // It needs to run only when there's no flush during recovery + // (e.g. avoid_flush_during_recovery=true). May also trigger flush + // in case total_log_size > max_total_wal_size. + Status RestoreAliveLogFiles(const std::vector& log_numbers); + + // num_bytes: for slowdown case, delay time is calculated based on + // `num_bytes` going through. + Status DelayWrite(uint64_t num_bytes, WriteThread& write_thread, + const WriteOptions& write_options); + + // Begin stalling of writes when memory usage increases beyond a certain + // threshold. + void WriteBufferManagerStallWrites(); + + Status ThrottleLowPriWritesIfNeeded(const WriteOptions& write_options, + WriteBatch* my_batch); + + // REQUIRES: mutex locked and in write thread. + Status ScheduleFlushes(WriteContext* context); + + void MaybeFlushStatsCF(autovector* cfds); + + Status TrimMemtableHistory(WriteContext* context); + + Status SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context); + + // Select and output column families qualified for atomic flush in + // `selected_cfds`. If `provided_candidate_cfds` is non-empty, it will be used + // as candidate CFs to select qualified ones from. Otherwise, all column + // families are used as candidate to select from. + // + // REQUIRES: mutex held + void SelectColumnFamiliesForAtomicFlush( + autovector* selected_cfds, + const autovector& provided_candidate_cfds = {}); + + // Force current memtable contents to be flushed. + Status FlushMemTable(ColumnFamilyData* cfd, const FlushOptions& options, + FlushReason flush_reason, + bool entered_write_thread = false); + + // Atomic-flush memtables from quanlified CFs among `provided_candidate_cfds` + // (if non-empty) or amomg all column families and atomically record the + // result to the MANIFEST. + Status AtomicFlushMemTables( + const FlushOptions& options, FlushReason flush_reason, + const autovector& provided_candidate_cfds = {}, + bool entered_write_thread = false); + + // Wait until flushing this column family won't stall writes + Status WaitUntilFlushWouldNotStallWrites(ColumnFamilyData* cfd, + bool* flush_needed); + + // Wait for memtable flushed. + // If flush_memtable_id is non-null, wait until the memtable with the ID + // gets flush. Otherwise, wait until the column family don't have any + // memtable pending flush. + // resuming_from_bg_err indicates whether the caller is attempting to resume + // from background error. + Status WaitForFlushMemTable(ColumnFamilyData* cfd, + const uint64_t* flush_memtable_id = nullptr, + bool resuming_from_bg_err = false) { + return WaitForFlushMemTables({cfd}, {flush_memtable_id}, + resuming_from_bg_err); + } + // Wait for memtables to be flushed for multiple column families. + Status WaitForFlushMemTables( + const autovector& cfds, + const autovector& flush_memtable_ids, + bool resuming_from_bg_err); + + inline void WaitForPendingWrites() { + mutex_.AssertHeld(); + TEST_SYNC_POINT("DBImpl::WaitForPendingWrites:BeforeBlock"); + // In case of pipelined write is enabled, wait for all pending memtable + // writers. + if (immutable_db_options_.enable_pipelined_write) { + // Memtable writers may call DB::Get in case max_successive_merges > 0, + // which may lock mutex. Unlocking mutex here to avoid deadlock. + mutex_.Unlock(); + write_thread_.WaitForMemTableWriters(); + mutex_.Lock(); + } + + if (!immutable_db_options_.unordered_write) { + // Then the writes are finished before the next write group starts + return; + } + + // Wait for the ones who already wrote to the WAL to finish their + // memtable write. + if (pending_memtable_writes_.load() != 0) { + std::unique_lock guard(switch_mutex_); + switch_cv_.wait(guard, + [&] { return pending_memtable_writes_.load() == 0; }); + } + } + + // TaskType is used to identify tasks in thread-pool, currently only + // differentiate manual compaction, which could be unscheduled from the + // thread-pool. + enum class TaskType : uint8_t { + kDefault = 0, + kManualCompaction = 1, + kCount = 2, + }; + + // Task tag is used to identity tasks in thread-pool, which is + // dbImpl obj address + type + inline void* GetTaskTag(TaskType type) { + return GetTaskTag(static_cast(type)); + } + + inline void* GetTaskTag(uint8_t type) { + return static_cast(static_cast(this)) + type; + } + + // REQUIRES: mutex locked and in write thread. + void AssignAtomicFlushSeq(const autovector& cfds); + + // REQUIRES: mutex locked and in write thread. + Status SwitchWAL(WriteContext* write_context); + + // REQUIRES: mutex locked and in write thread. + Status HandleWriteBufferManagerFlush(WriteContext* write_context); + + // REQUIRES: mutex locked + Status PreprocessWrite(const WriteOptions& write_options, + LogContext* log_context, WriteContext* write_context); + + // Merge write batches in the write group into merged_batch. + // Returns OK if merge is successful. + // Returns Corruption if corruption in write batch is detected. + Status MergeBatch(const WriteThread::WriteGroup& write_group, + WriteBatch* tmp_batch, WriteBatch** merged_batch, + size_t* write_with_wal, WriteBatch** to_be_cached_state); + + // rate_limiter_priority is used to charge `DBOptions::rate_limiter` + // for automatic WAL flush (`Options::manual_wal_flush` == false) + // associated with this WriteToWAL + IOStatus WriteToWAL(const WriteBatch& merged_batch, log::Writer* log_writer, + uint64_t* log_used, uint64_t* log_size, + Env::IOPriority rate_limiter_priority, + LogFileNumberSize& log_file_number_size); + + IOStatus WriteToWAL(const WriteThread::WriteGroup& write_group, + log::Writer* log_writer, uint64_t* log_used, + bool need_log_sync, bool need_log_dir_sync, + SequenceNumber sequence, + LogFileNumberSize& log_file_number_size); + + IOStatus ConcurrentWriteToWAL(const WriteThread::WriteGroup& write_group, + uint64_t* log_used, + SequenceNumber* last_sequence, size_t seq_inc); + + // Used by WriteImpl to update bg_error_ if paranoid check is enabled. + // Caller must hold mutex_. + void WriteStatusCheckOnLocked(const Status& status); + + // Used by WriteImpl to update bg_error_ if paranoid check is enabled. + void WriteStatusCheck(const Status& status); + + // Used by WriteImpl to update bg_error_ when IO error happens, e.g., write + // WAL, sync WAL fails, if paranoid check is enabled. + void IOStatusCheck(const IOStatus& status); + + // Used by WriteImpl to update bg_error_ in case of memtable insert error. + void MemTableInsertStatusCheck(const Status& memtable_insert_status); + + Status CompactFilesImpl(const CompactionOptions& compact_options, + ColumnFamilyData* cfd, Version* version, + const std::vector& input_file_names, + std::vector* const output_file_names, + const int output_level, int output_path_id, + JobContext* job_context, LogBuffer* log_buffer, + CompactionJobInfo* compaction_job_info); + + ColumnFamilyData* GetColumnFamilyDataByName(const std::string& cf_name); + + void MaybeScheduleFlushOrCompaction(); + + struct FlushRequest { + FlushReason flush_reason; + // A map from column family to flush to largest memtable id to persist for + // each column family. Once all the memtables whose IDs are smaller than or + // equal to this per-column-family specified value, this flush request is + // considered to have completed its work of flushing this column family. + // After completing the work for all column families in this request, this + // flush is considered complete. + std::unordered_map + cfd_to_max_mem_id_to_persist; + +#ifndef NDEBUG + int reschedule_count = 1; +#endif /* !NDEBUG */ + }; + + void GenerateFlushRequest(const autovector& cfds, + FlushReason flush_reason, FlushRequest* req); + + void SchedulePendingFlush(const FlushRequest& req); + + void SchedulePendingCompaction(ColumnFamilyData* cfd); + void SchedulePendingPurge(std::string fname, std::string dir_to_sync, + FileType type, uint64_t number, int job_id); + static void BGWorkCompaction(void* arg); + // Runs a pre-chosen universal compaction involving bottom level in a + // separate, bottom-pri thread pool. + static void BGWorkBottomCompaction(void* arg); + static void BGWorkFlush(void* arg); + static void BGWorkPurge(void* arg); + static void UnscheduleCompactionCallback(void* arg); + static void UnscheduleFlushCallback(void* arg); + void BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction, + Env::Priority thread_pri); + void BackgroundCallFlush(Env::Priority thread_pri); + void BackgroundCallPurge(); + Status BackgroundCompaction(bool* madeProgress, JobContext* job_context, + LogBuffer* log_buffer, + PrepickedCompaction* prepicked_compaction, + Env::Priority thread_pri); + Status BackgroundFlush(bool* madeProgress, JobContext* job_context, + LogBuffer* log_buffer, FlushReason* reason, + bool* flush_rescheduled_to_retain_udt, + Env::Priority thread_pri); + + bool EnoughRoomForCompaction(ColumnFamilyData* cfd, + const std::vector& inputs, + bool* sfm_bookkeeping, LogBuffer* log_buffer); + + // Request compaction tasks token from compaction thread limiter. + // It always succeeds if force = true or limiter is disable. + bool RequestCompactionToken(ColumnFamilyData* cfd, bool force, + std::unique_ptr* token, + LogBuffer* log_buffer); + + // Return true if the `FlushRequest` can be rescheduled to retain the UDT. + // Only true if there are user-defined timestamps in the involved MemTables + // with newer than cutoff timestamp `full_history_ts_low` and not flushing + // immediately will not cause entering write stall mode. + bool ShouldRescheduleFlushRequestToRetainUDT(const FlushRequest& flush_req); + + // Schedule background tasks + Status StartPeriodicTaskScheduler(); + + Status RegisterRecordSeqnoTimeWorker(); + + void PrintStatistics(); + + size_t EstimateInMemoryStatsHistorySize() const; + + // Return the minimum empty level that could hold the total data in the + // input level. Return the input level, if such level could not be found. + int FindMinimumEmptyLevelFitting(ColumnFamilyData* cfd, + const MutableCFOptions& mutable_cf_options, + int level); + + // Move the files in the input level to the target level. + // If target_level < 0, automatically calculate the minimum level that could + // hold the data set. + Status ReFitLevel(ColumnFamilyData* cfd, int level, int target_level = -1); + + // helper functions for adding and removing from flush & compaction queues + void AddToCompactionQueue(ColumnFamilyData* cfd); + ColumnFamilyData* PopFirstFromCompactionQueue(); + FlushRequest PopFirstFromFlushQueue(); + + // Pick the first unthrottled compaction with task token from queue. + ColumnFamilyData* PickCompactionFromQueue( + std::unique_ptr* token, LogBuffer* log_buffer); + + // helper function to call after some of the logs_ were synced + void MarkLogsSynced(uint64_t up_to, bool synced_dir, VersionEdit* edit); + Status ApplyWALToManifest(const ReadOptions& read_options, VersionEdit* edit); + // WALs with log number up to up_to are not synced successfully. + void MarkLogsNotSynced(uint64_t up_to); + + SnapshotImpl* GetSnapshotImpl(bool is_write_conflict_boundary, + bool lock = true); + + // If snapshot_seq != kMaxSequenceNumber, then this function can only be + // called from the write thread that publishes sequence numbers to readers. + // For 1) write-committed, or 2) write-prepared + one-write-queue, this will + // be the write thread performing memtable writes. For write-prepared with + // two write queues, this will be the write thread writing commit marker to + // the WAL. + // If snapshot_seq == kMaxSequenceNumber, this function is called by a caller + // ensuring no writes to the database. + std::pair> + CreateTimestampedSnapshotImpl(SequenceNumber snapshot_seq, uint64_t ts, + bool lock = true); + + uint64_t GetMaxTotalWalSize() const; + + FSDirectory* GetDataDir(ColumnFamilyData* cfd, size_t path_id) const; + + Status MaybeReleaseTimestampedSnapshotsAndCheck(); + + Status CloseHelper(); + + void WaitForBackgroundWork(); + + // Background threads call this function, which is just a wrapper around + // the InstallSuperVersion() function. Background threads carry + // sv_context which can have new_superversion already + // allocated. + // All ColumnFamily state changes go through this function. Here we analyze + // the new state and we schedule background work if we detect that the new + // state needs flush or compaction. + void InstallSuperVersionAndScheduleWork( + ColumnFamilyData* cfd, SuperVersionContext* sv_context, + const MutableCFOptions& mutable_cf_options); + + bool GetIntPropertyInternal(ColumnFamilyData* cfd, + const DBPropertyInfo& property_info, + bool is_locked, uint64_t* value); + bool GetPropertyHandleOptionsStatistics(std::string* value); + + bool HasPendingManualCompaction(); + bool HasExclusiveManualCompaction(); + void AddManualCompaction(ManualCompactionState* m); + void RemoveManualCompaction(ManualCompactionState* m); + bool ShouldntRunManualCompaction(ManualCompactionState* m); + bool HaveManualCompaction(ColumnFamilyData* cfd); + bool MCOverlap(ManualCompactionState* m, ManualCompactionState* m1); + void BuildCompactionJobInfo(const ColumnFamilyData* cfd, Compaction* c, + const Status& st, + const CompactionJobStats& compaction_job_stats, + const int job_id, + CompactionJobInfo* compaction_job_info) const; + // Reserve the next 'num' file numbers for to-be-ingested external SST files, + // and return the current file_number in 'next_file_number'. + // Write a version edit to the MANIFEST. + Status ReserveFileNumbersBeforeIngestion( + ColumnFamilyData* cfd, uint64_t num, + std::unique_ptr::iterator>& pending_output_elem, + uint64_t* next_file_number); + + bool ShouldPurge(uint64_t file_number) const; + void MarkAsGrabbedForPurge(uint64_t file_number); + + size_t GetWalPreallocateBlockSize(uint64_t write_buffer_size) const; + Env::WriteLifeTimeHint CalculateWALWriteHint() { return Env::WLTH_SHORT; } + + IOStatus CreateWAL(uint64_t log_file_num, uint64_t recycle_log_number, + size_t preallocate_block_size, log::Writer** new_log); + + // Validate self-consistency of DB options + static Status ValidateOptions(const DBOptions& db_options); + // Validate self-consistency of DB options and its consistency with cf options + static Status ValidateOptions( + const DBOptions& db_options, + const std::vector& column_families); + + // Utility function to do some debug validation and sort the given vector + // of MultiGet keys + void PrepareMultiGetKeys( + const size_t num_keys, bool sorted, + autovector* key_ptrs); + + void MultiGetCommon(const ReadOptions& options, + ColumnFamilyHandle* column_family, const size_t num_keys, + const Slice* keys, PinnableSlice* values, + PinnableWideColumns* columns, std::string* timestamps, + Status* statuses, bool sorted_input); + + void MultiGetCommon(const ReadOptions& options, const size_t num_keys, + ColumnFamilyHandle** column_families, const Slice* keys, + PinnableSlice* values, PinnableWideColumns* columns, + std::string* timestamps, Status* statuses, + bool sorted_input); + + // A structure to hold the information required to process MultiGet of keys + // belonging to one column family. For a multi column family MultiGet, there + // will be a container of these objects. + struct MultiGetColumnFamilyData { + ColumnFamilyHandle* cf; + ColumnFamilyData* cfd; + + // For the batched MultiGet which relies on sorted keys, start specifies + // the index of first key belonging to this column family in the sorted + // list. + size_t start; + + // For the batched MultiGet case, num_keys specifies the number of keys + // belonging to this column family in the sorted list + size_t num_keys; + + // SuperVersion for the column family obtained in a manner that ensures a + // consistent view across all column families in the DB + SuperVersion* super_version; + MultiGetColumnFamilyData(ColumnFamilyHandle* column_family, + SuperVersion* sv) + : cf(column_family), + cfd(static_cast(cf)->cfd()), + start(0), + num_keys(0), + super_version(sv) {} + + MultiGetColumnFamilyData(ColumnFamilyHandle* column_family, size_t first, + size_t count, SuperVersion* sv) + : cf(column_family), + cfd(static_cast(cf)->cfd()), + start(first), + num_keys(count), + super_version(sv) {} + + MultiGetColumnFamilyData() = default; + }; + + // A common function to obtain a consistent snapshot, which can be implicit + // if the user doesn't specify a snapshot in read_options, across + // multiple column families for MultiGet. It will attempt to get an implicit + // snapshot without acquiring the db_mutes, but will give up after a few + // tries and acquire the mutex if a memtable flush happens. The template + // allows both the batched and non-batched MultiGet to call this with + // either an std::unordered_map or autovector of column families. + // + // If callback is non-null, the callback is refreshed with the snapshot + // sequence number + // + // A return value of true indicates that the SuperVersions were obtained + // from the ColumnFamilyData, whereas false indicates they are thread + // local + template + bool MultiCFSnapshot( + const ReadOptions& read_options, ReadCallback* callback, + std::function& + iter_deref_func, + T* cf_list, SequenceNumber* snapshot); + + // The actual implementation of the batching MultiGet. The caller is expected + // to have acquired the SuperVersion and pass in a snapshot sequence number + // in order to construct the LookupKeys. The start_key and num_keys specify + // the range of keys in the sorted_keys vector for a single column family. + Status MultiGetImpl( + const ReadOptions& read_options, size_t start_key, size_t num_keys, + autovector* sorted_keys, + SuperVersion* sv, SequenceNumber snap_seqnum, ReadCallback* callback); + + Status DisableFileDeletionsWithLock(); + + Status IncreaseFullHistoryTsLowImpl(ColumnFamilyData* cfd, + std::string ts_low); + + bool ShouldReferenceSuperVersion(const MergeContext& merge_context); + + // Lock over the persistent DB state. Non-nullptr iff successfully acquired. + FileLock* db_lock_; + + // In addition to mutex_, log_write_mutex_ protected writes to stats_history_ + InstrumentedMutex stats_history_mutex_; + // In addition to mutex_, log_write_mutex_ protected writes to logs_ and + // logfile_number_. With two_write_queues it also protects alive_log_files_, + // and log_empty_. Refer to the definition of each variable below for more + // details. + // Note: to avoid deadlock, if needed to acquire both log_write_mutex_ and + // mutex_, the order should be first mutex_ and then log_write_mutex_. + InstrumentedMutex log_write_mutex_; + + // If zero, manual compactions are allowed to proceed. If non-zero, manual + // compactions may still be running, but will quickly fail with + // `Status::Incomplete`. The value indicates how many threads have paused + // manual compactions. It is accessed in read mode outside the DB mutex in + // compaction code paths. + std::atomic manual_compaction_paused_; + + // This condition variable is signaled on these conditions: + // * whenever bg_compaction_scheduled_ goes down to 0 + // * if AnyManualCompaction, whenever a compaction finishes, even if it hasn't + // made any progress + // * whenever a compaction made any progress + // * whenever bg_flush_scheduled_ or bg_purge_scheduled_ value decreases + // (i.e. whenever a flush is done, even if it didn't make any progress) + // * whenever there is an error in background purge, flush or compaction + // * whenever num_running_ingest_file_ goes to 0. + // * whenever pending_purge_obsolete_files_ goes to 0. + // * whenever disable_delete_obsolete_files_ goes to 0. + // * whenever SetOptions successfully updates options. + // * whenever a column family is dropped. + InstrumentedCondVar bg_cv_; + // Writes are protected by locking both mutex_ and log_write_mutex_, and reads + // must be under either mutex_ or log_write_mutex_. Since after ::Open, + // logfile_number_ is currently updated only in write_thread_, it can be read + // from the same write_thread_ without any locks. + uint64_t logfile_number_; + // Log files that we can recycle. Must be protected by db mutex_. + std::deque log_recycle_files_; + // Protected by log_write_mutex_. + bool log_dir_synced_; + // Without two_write_queues, read and writes to log_empty_ are protected by + // mutex_. Since it is currently updated/read only in write_thread_, it can be + // accessed from the same write_thread_ without any locks. With + // two_write_queues writes, where it can be updated in different threads, + // read and writes are protected by log_write_mutex_ instead. This is to avoid + // expensive mutex_ lock during WAL write, which update log_empty_. + bool log_empty_; + + ColumnFamilyHandleImpl* persist_stats_cf_handle_; + + bool persistent_stats_cfd_exists_ = true; + + // alive_log_files_ is protected by mutex_ and log_write_mutex_ with details + // as follows: + // 1. read by FindObsoleteFiles() which can be called in either application + // thread or RocksDB bg threads, both mutex_ and log_write_mutex_ are + // held. + // 2. pop_front() by FindObsoleteFiles(), both mutex_ and log_write_mutex_ + // are held. + // 3. push_back() by DBImpl::Open() and DBImpl::RestoreAliveLogFiles() + // (actually called by Open()), only mutex_ is held because at this point, + // the DB::Open() call has not returned success to application, and the + // only other thread(s) that can conflict are bg threads calling + // FindObsoleteFiles() which ensure that both mutex_ and log_write_mutex_ + // are held when accessing alive_log_files_. + // 4. read by DBImpl::Open() is protected by mutex_. + // 5. push_back() by SwitchMemtable(). Both mutex_ and log_write_mutex_ are + // held. This is done by the write group leader. Note that in the case of + // two-write-queues, another WAL-only write thread can be writing to the + // WAL concurrently. See 9. + // 6. read by SwitchWAL() with both mutex_ and log_write_mutex_ held. This is + // done by write group leader. + // 7. read by ConcurrentWriteToWAL() by the write group leader in the case of + // two-write-queues. Only log_write_mutex_ is held to protect concurrent + // pop_front() by FindObsoleteFiles(). + // 8. read by PreprocessWrite() by the write group leader. log_write_mutex_ + // is held to protect the data structure from concurrent pop_front() by + // FindObsoleteFiles(). + // 9. read by ConcurrentWriteToWAL() by a WAL-only write thread in the case + // of two-write-queues. Only log_write_mutex_ is held. This suffices to + // protect the data structure from concurrent push_back() by current + // write group leader as well as pop_front() by FindObsoleteFiles(). + std::deque alive_log_files_; + + // Log files that aren't fully synced, and the current log file. + // Synchronization: + // 1. read by FindObsoleteFiles() which can be called either in application + // thread or RocksDB bg threads. log_write_mutex_ is always held, while + // some reads are performed without mutex_. + // 2. pop_front() by FindObsoleteFiles() with only log_write_mutex_ held. + // 3. read by DBImpl::Open() with both mutex_ and log_write_mutex_. + // 4. emplace_back() by DBImpl::Open() with both mutex_ and log_write_mutex. + // Note that at this point, DB::Open() has not returned success to + // application, thus the only other thread(s) that can conflict are bg + // threads calling FindObsoleteFiles(). See 1. + // 5. iteration and clear() from CloseHelper() always hold log_write_mutex + // and mutex_. + // 6. back() called by APIs FlushWAL() and LockWAL() are protected by only + // log_write_mutex_. These two can be called by application threads after + // DB::Open() returns success to applications. + // 7. read by SyncWAL(), another API, protected by only log_write_mutex_. + // 8. read by MarkLogsNotSynced() and MarkLogsSynced() are protected by + // log_write_mutex_. + // 9. erase() by MarkLogsSynced() protected by log_write_mutex_. + // 10. read by SyncClosedLogs() protected by only log_write_mutex_. This can + // happen in bg flush threads after DB::Open() returns success to + // applications. + // 11. reads, e.g. front(), iteration, and back() called by PreprocessWrite() + // holds only the log_write_mutex_. This is done by the write group + // leader. A bg thread calling FindObsoleteFiles() or MarkLogsSynced() + // can happen concurrently. This is fine because log_write_mutex_ is used + // by all parties. See 2, 5, 9. + // 12. reads, empty(), back() called by SwitchMemtable() hold both mutex_ and + // log_write_mutex_. This happens in the write group leader. + // 13. emplace_back() by SwitchMemtable() hold both mutex_ and + // log_write_mutex_. This happens in the write group leader. Can conflict + // with bg threads calling FindObsoleteFiles(), MarkLogsSynced(), + // SyncClosedLogs(), etc. as well as application threads calling + // FlushWAL(), SyncWAL(), LockWAL(). This is fine because all parties + // require at least log_write_mutex_. + // 14. iteration called in WriteToWAL(write_group) protected by + // log_write_mutex_. This is done by write group leader when + // two-write-queues is disabled and write needs to sync logs. + // 15. back() called in ConcurrentWriteToWAL() protected by log_write_mutex_. + // This can be done by the write group leader if two-write-queues is + // enabled. It can also be done by another WAL-only write thread. + // + // Other observations: + // - back() and items with getting_synced=true are not popped, + // - The same thread that sets getting_synced=true will reset it. + // - it follows that the object referred by back() can be safely read from + // the write_thread_ without using mutex. Note that calling back() without + // mutex may be unsafe because different implementations of deque::back() may + // access other member variables of deque, causing undefined behaviors. + // Generally, do not access stl containers without proper synchronization. + // - it follows that the items with getting_synced=true can be safely read + // from the same thread that has set getting_synced=true + std::deque logs_; + + // Signaled when getting_synced becomes false for some of the logs_. + InstrumentedCondVar log_sync_cv_; + // This is the app-level state that is written to the WAL but will be used + // only during recovery. Using this feature enables not writing the state to + // memtable on normal writes and hence improving the throughput. Each new + // write of the state will replace the previous state entirely even if the + // keys in the two consecutive states do not overlap. + // It is protected by log_write_mutex_ when two_write_queues_ is enabled. + // Otherwise only the heaad of write_thread_ can access it. + WriteBatch cached_recoverable_state_; + std::atomic cached_recoverable_state_empty_ = {true}; + std::atomic total_log_size_; + + // If this is non-empty, we need to delete these log files in background + // threads. Protected by log_write_mutex_. + autovector logs_to_free_; + + bool is_snapshot_supported_; + + std::map> stats_history_; + + std::map stats_slice_; + + bool stats_slice_initialized_ = false; + + Directories directories_; + + WriteBufferManager* write_buffer_manager_; + + WriteThread write_thread_; + WriteBatch tmp_batch_; + // The write thread when the writers have no memtable write. This will be used + // in 2PC to batch the prepares separately from the serial commit. + WriteThread nonmem_write_thread_; + + WriteController write_controller_; + + // Size of the last batch group. In slowdown mode, next write needs to + // sleep if it uses up the quota. + // Note: This is to protect memtable and compaction. If the batch only writes + // to the WAL its size need not to be included in this. + uint64_t last_batch_group_size_; + + FlushScheduler flush_scheduler_; + + TrimHistoryScheduler trim_history_scheduler_; + + SnapshotList snapshots_; + + TimestampedSnapshotList timestamped_snapshots_; + + // For each background job, pending_outputs_ keeps the current file number at + // the time that background job started. + // FindObsoleteFiles()/PurgeObsoleteFiles() never deletes any file that has + // number bigger than any of the file number in pending_outputs_. Since file + // numbers grow monotonically, this also means that pending_outputs_ is always + // sorted. After a background job is done executing, its file number is + // deleted from pending_outputs_, which allows PurgeObsoleteFiles() to clean + // it up. + // State is protected with db mutex. + std::list pending_outputs_; + + // flush_queue_ and compaction_queue_ hold column families that we need to + // flush and compact, respectively. + // A column family is inserted into flush_queue_ when it satisfies condition + // cfd->imm()->IsFlushPending() + // A column family is inserted into compaction_queue_ when it satisfied + // condition cfd->NeedsCompaction() + // Column families in this list are all Ref()-erenced + // TODO(icanadi) Provide some kind of ReferencedColumnFamily class that will + // do RAII on ColumnFamilyData + // Column families are in this queue when they need to be flushed or + // compacted. Consumers of these queues are flush and compaction threads. When + // column family is put on this queue, we increase unscheduled_flushes_ and + // unscheduled_compactions_. When these variables are bigger than zero, that + // means we need to schedule background threads for flush and compaction. + // Once the background threads are scheduled, we decrease unscheduled_flushes_ + // and unscheduled_compactions_. That way we keep track of number of + // compaction and flush threads we need to schedule. This scheduling is done + // in MaybeScheduleFlushOrCompaction() + // invariant(column family present in flush_queue_ <==> + // ColumnFamilyData::pending_flush_ == true) + std::deque flush_queue_; + // invariant(column family present in compaction_queue_ <==> + // ColumnFamilyData::pending_compaction_ == true) + std::deque compaction_queue_; + + // A map to store file numbers and filenames of the files to be purged + std::unordered_map purge_files_; + + // A vector to store the file numbers that have been assigned to certain + // JobContext. Current implementation tracks table and blob files only. + std::unordered_set files_grabbed_for_purge_; + + // A queue to store log writers to close. Protected by db mutex_. + std::deque logs_to_free_queue_; + + std::deque superversions_to_free_queue_; + + int unscheduled_flushes_; + + int unscheduled_compactions_; + + // count how many background compactions are running or have been scheduled in + // the BOTTOM pool + int bg_bottom_compaction_scheduled_; + + // count how many background compactions are running or have been scheduled + int bg_compaction_scheduled_; + + // stores the number of compactions are currently running + int num_running_compactions_; + + // number of background memtable flush jobs, submitted to the HIGH pool + int bg_flush_scheduled_; + + // stores the number of flushes are currently running + int num_running_flushes_; + + // number of background obsolete file purge jobs, submitted to the HIGH pool + int bg_purge_scheduled_; + + std::deque manual_compaction_dequeue_; + + // shall we disable deletion of obsolete files + // if 0 the deletion is enabled. + // if non-zero, files will not be getting deleted + // This enables two different threads to call + // EnableFileDeletions() and DisableFileDeletions() + // without any synchronization + int disable_delete_obsolete_files_; + + // Number of times FindObsoleteFiles has found deletable files and the + // corresponding call to PurgeObsoleteFiles has not yet finished. + int pending_purge_obsolete_files_; + + // last time when DeleteObsoleteFiles with full scan was executed. Originally + // initialized with startup time. + uint64_t delete_obsolete_files_last_run_; + + // last time stats were dumped to LOG + std::atomic last_stats_dump_time_microsec_; + + // The thread that wants to switch memtable, can wait on this cv until the + // pending writes to memtable finishes. + std::condition_variable switch_cv_; + // The mutex used by switch_cv_. mutex_ should be acquired beforehand. + std::mutex switch_mutex_; + // Number of threads intending to write to memtable + std::atomic pending_memtable_writes_ = {}; + + // A flag indicating whether the current rocksdb database has any + // data that is not yet persisted into either WAL or SST file. + // Used when disableWAL is true. + std::atomic has_unpersisted_data_; + + // if an attempt was made to flush all column families that + // the oldest log depends on but uncommitted data in the oldest + // log prevents the log from being released. + // We must attempt to free the dependent memtables again + // at a later time after the transaction in the oldest + // log is fully commited. + bool unable_to_release_oldest_log_; + + // Number of running IngestExternalFile() or CreateColumnFamilyWithImport() + // calls. + // REQUIRES: mutex held + int num_running_ingest_file_; + + WalManager wal_manager_; + + // A value of > 0 temporarily disables scheduling of background work + int bg_work_paused_; + + // A value of > 0 temporarily disables scheduling of background compaction + int bg_compaction_paused_; + + // Guard against multiple concurrent refitting + bool refitting_level_; + + // Indicate DB was opened successfully + bool opened_successfully_; + + // The min threshold to triggere bottommost compaction for removing + // garbages, among all column families. + SequenceNumber bottommost_files_mark_threshold_ = kMaxSequenceNumber; + + LogsWithPrepTracker logs_with_prep_tracker_; + + // Callback for compaction to check if a key is visible to a snapshot. + // REQUIRES: mutex held + std::unique_ptr snapshot_checker_; + + // Callback for when the cached_recoverable_state_ is written to memtable + // Only to be set during initialization + std::unique_ptr recoverable_state_pre_release_callback_; + + // Scheduler to run DumpStats(), PersistStats(), and FlushInfoLog(). + // Currently, internally it has a global timer instance for running the tasks. + PeriodicTaskScheduler periodic_task_scheduler_; + + // It contains the implementations for each periodic task. + std::map periodic_task_functions_; + + // When set, we use a separate queue for writes that don't write to memtable. + // In 2PC these are the writes at Prepare phase. + const bool two_write_queues_; + const bool manual_wal_flush_; + + // LastSequence also indicates last published sequence visibile to the + // readers. Otherwise LastPublishedSequence should be used. + const bool last_seq_same_as_publish_seq_; + // It indicates that a customized gc algorithm must be used for + // flush/compaction and if it is not provided vis SnapshotChecker, we should + // disable gc to be safe. + const bool use_custom_gc_; + // Flag to indicate that the DB instance shutdown has been initiated. This + // different from shutting_down_ atomic in that it is set at the beginning + // of shutdown sequence, specifically in order to prevent any background + // error recovery from going on in parallel. The latter, shutting_down_, + // is set a little later during the shutdown after scheduling memtable + // flushes + std::atomic shutdown_initiated_; + // Flag to indicate whether sst_file_manager object was allocated in + // DB::Open() or passed to us + bool own_sfm_; + + // Flag to check whether Close() has been called on this DB + bool closed_; + // save the closing status, for re-calling the close() + Status closing_status_; + // mutex for DB::Close() + InstrumentedMutex closing_mutex_; + + // Conditional variable to coordinate installation of atomic flush results. + // With atomic flush, each bg thread installs the result of flushing multiple + // column families, and different threads can flush different column + // families. It's difficult to rely on one thread to perform batch + // installation for all threads. This is different from the non-atomic flush + // case. + // atomic_flush_install_cv_ makes sure that threads install atomic flush + // results sequentially. Flush results of memtables with lower IDs get + // installed to MANIFEST first. + InstrumentedCondVar atomic_flush_install_cv_; + + bool wal_in_db_path_; + std::atomic max_total_wal_size_; + + BlobFileCompletionCallback blob_callback_; + + // Pointer to WriteBufferManager stalling interface. + std::unique_ptr wbm_stall_; + + // seqno_time_mapping_ stores the sequence number to time mapping, it's not + // thread safe, both read and write need db mutex hold. + SeqnoToTimeMapping seqno_time_mapping_; + + // Stop write token that is acquired when first LockWAL() is called. + // Destroyed when last UnlockWAL() is called. Controlled by DB mutex. + // See lock_wal_count_ + std::unique_ptr lock_wal_write_token_; + + // The number of LockWAL called without matching UnlockWAL call. + // See also lock_wal_write_token_ + uint32_t lock_wal_count_; +}; + +class GetWithTimestampReadCallback : public ReadCallback { + public: + explicit GetWithTimestampReadCallback(SequenceNumber seq) + : ReadCallback(seq) {} + bool IsVisibleFullCheck(SequenceNumber seq) override { + return seq <= max_visible_seq_; + } +}; + +extern Options SanitizeOptions(const std::string& db, const Options& src, + bool read_only = false, + Status* logger_creation_s = nullptr); + +extern DBOptions SanitizeOptions(const std::string& db, const DBOptions& src, + bool read_only = false, + Status* logger_creation_s = nullptr); + +extern CompressionType GetCompressionFlush( + const ImmutableCFOptions& ioptions, + const MutableCFOptions& mutable_cf_options); + +// Return the earliest log file to keep after the memtable flush is +// finalized. +// `cfd_to_flush` is the column family whose memtable (specified in +// `memtables_to_flush`) will be flushed and thus will not depend on any WAL +// file. +// The function is only applicable to 2pc mode. +extern uint64_t PrecomputeMinLogNumberToKeep2PC( + VersionSet* vset, const ColumnFamilyData& cfd_to_flush, + const autovector& edit_list, + const autovector& memtables_to_flush, + LogsWithPrepTracker* prep_tracker); +// For atomic flush. +extern uint64_t PrecomputeMinLogNumberToKeep2PC( + VersionSet* vset, const autovector& cfds_to_flush, + const autovector>& edit_lists, + const autovector*>& memtables_to_flush, + LogsWithPrepTracker* prep_tracker); + +// In non-2PC mode, WALs with log number < the returned number can be +// deleted after the cfd_to_flush column family is flushed successfully. +extern uint64_t PrecomputeMinLogNumberToKeepNon2PC( + VersionSet* vset, const ColumnFamilyData& cfd_to_flush, + const autovector& edit_list); +// For atomic flush. +extern uint64_t PrecomputeMinLogNumberToKeepNon2PC( + VersionSet* vset, const autovector& cfds_to_flush, + const autovector>& edit_lists); + +// `cfd_to_flush` is the column family whose memtable will be flushed and thus +// will not depend on any WAL file. nullptr means no memtable is being flushed. +// The function is only applicable to 2pc mode. +extern uint64_t FindMinPrepLogReferencedByMemTable( + VersionSet* vset, const autovector& memtables_to_flush); +// For atomic flush. +extern uint64_t FindMinPrepLogReferencedByMemTable( + VersionSet* vset, + const autovector*>& memtables_to_flush); + +// Fix user-supplied options to be reasonable +template +static void ClipToRange(T* ptr, V minvalue, V maxvalue) { + if (static_cast(*ptr) > maxvalue) *ptr = maxvalue; + if (static_cast(*ptr) < minvalue) *ptr = minvalue; +} + +inline Status DBImpl::FailIfCfHasTs( + const ColumnFamilyHandle* column_family) const { + column_family = column_family ? column_family : DefaultColumnFamily(); + assert(column_family); + const Comparator* const ucmp = column_family->GetComparator(); + assert(ucmp); + if (ucmp->timestamp_size() > 0) { + std::ostringstream oss; + oss << "cannot call this method on column family " + << column_family->GetName() << " that enables timestamp"; + return Status::InvalidArgument(oss.str()); + } + return Status::OK(); +} + +inline Status DBImpl::FailIfTsMismatchCf(ColumnFamilyHandle* column_family, + const Slice& ts, + bool ts_for_read) const { + if (!column_family) { + return Status::InvalidArgument("column family handle cannot be null"); + } + assert(column_family); + const Comparator* const ucmp = column_family->GetComparator(); + assert(ucmp); + if (0 == ucmp->timestamp_size()) { + std::stringstream oss; + oss << "cannot call this method on column family " + << column_family->GetName() << " that does not enable timestamp"; + return Status::InvalidArgument(oss.str()); + } + const size_t ts_sz = ts.size(); + if (ts_sz != ucmp->timestamp_size()) { + std::stringstream oss; + oss << "Timestamp sizes mismatch: expect " << ucmp->timestamp_size() << ", " + << ts_sz << " given"; + return Status::InvalidArgument(oss.str()); + } + if (ts_for_read) { + auto cfh = static_cast_with_check(column_family); + auto cfd = cfh->cfd(); + std::string current_ts_low = cfd->GetFullHistoryTsLow(); + if (!current_ts_low.empty() && + ucmp->CompareTimestamp(ts, current_ts_low) < 0) { + std::stringstream oss; + oss << "Read timestamp: " << ts.ToString(true) + << " is smaller than full_history_ts_low: " + << Slice(current_ts_low).ToString(true) << std::endl; + return Status::InvalidArgument(oss.str()); + } + } + return Status::OK(); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/db_impl/db_impl_compaction_flush.cc b/librocksdb-sys/rocksdb/db/db_impl/db_impl_compaction_flush.cc new file mode 100644 index 0000000..9cde560 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_impl/db_impl_compaction_flush.cc @@ -0,0 +1,4118 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include +#include + +#include "db/builder.h" +#include "db/db_impl/db_impl.h" +#include "db/error_handler.h" +#include "db/event_helpers.h" +#include "file/sst_file_manager_impl.h" +#include "logging/logging.h" +#include "monitoring/iostats_context_imp.h" +#include "monitoring/perf_context_imp.h" +#include "monitoring/thread_status_updater.h" +#include "monitoring/thread_status_util.h" +#include "test_util/sync_point.h" +#include "util/cast_util.h" +#include "util/coding.h" +#include "util/concurrent_task_limiter_impl.h" + +namespace ROCKSDB_NAMESPACE { + +bool DBImpl::EnoughRoomForCompaction( + ColumnFamilyData* cfd, const std::vector& inputs, + bool* sfm_reserved_compact_space, LogBuffer* log_buffer) { + // Check if we have enough room to do the compaction + bool enough_room = true; + auto sfm = static_cast( + immutable_db_options_.sst_file_manager.get()); + if (sfm) { + // Pass the current bg_error_ to SFM so it can decide what checks to + // perform. If this DB instance hasn't seen any error yet, the SFM can be + // optimistic and not do disk space checks + Status bg_error = error_handler_.GetBGError(); + enough_room = sfm->EnoughRoomForCompaction(cfd, inputs, bg_error); + bg_error.PermitUncheckedError(); // bg_error is just a copy of the Status + // from the error_handler_ + if (enough_room) { + *sfm_reserved_compact_space = true; + } + } + if (!enough_room) { + // Just in case tests want to change the value of enough_room + TEST_SYNC_POINT_CALLBACK( + "DBImpl::BackgroundCompaction():CancelledCompaction", &enough_room); + ROCKS_LOG_BUFFER(log_buffer, + "Cancelled compaction because not enough room"); + RecordTick(stats_, COMPACTION_CANCELLED, 1); + } + return enough_room; +} + +bool DBImpl::RequestCompactionToken(ColumnFamilyData* cfd, bool force, + std::unique_ptr* token, + LogBuffer* log_buffer) { + assert(*token == nullptr); + auto limiter = static_cast( + cfd->ioptions()->compaction_thread_limiter.get()); + if (limiter == nullptr) { + return true; + } + *token = limiter->GetToken(force); + if (*token != nullptr) { + ROCKS_LOG_BUFFER(log_buffer, + "Thread limiter [%s] increase [%s] compaction task, " + "force: %s, tasks after: %d", + limiter->GetName().c_str(), cfd->GetName().c_str(), + force ? "true" : "false", limiter->GetOutstandingTask()); + return true; + } + return false; +} + +bool DBImpl::ShouldRescheduleFlushRequestToRetainUDT( + const FlushRequest& flush_req) { + mutex_.AssertHeld(); + assert(flush_req.cfd_to_max_mem_id_to_persist.size() == 1); + ColumnFamilyData* cfd = flush_req.cfd_to_max_mem_id_to_persist.begin()->first; + uint64_t max_memtable_id = + flush_req.cfd_to_max_mem_id_to_persist.begin()->second; + if (cfd->IsDropped() || + !cfd->ShouldPostponeFlushToRetainUDT(max_memtable_id)) { + return false; + } + // Check if holding on the flush will cause entering write stall mode. + // Write stall entered because of the accumulation of write buffers can be + // alleviated if we continue with the flush instead of postponing it. + const auto& mutable_cf_options = *cfd->GetLatestMutableCFOptions(); + + // Taking the status of the active Memtable into consideration so that we are + // not just checking if DB is currently already in write stall mode. + int mem_to_flush = cfd->mem()->ApproximateMemoryUsageFast() >= + cfd->mem()->write_buffer_size() / 2 + ? 1 + : 0; + WriteStallCondition write_stall = + ColumnFamilyData::GetWriteStallConditionAndCause( + cfd->imm()->NumNotFlushed() + mem_to_flush, /*num_l0_files=*/0, + /*num_compaction_needed_bytes=*/0, mutable_cf_options, + *cfd->ioptions()) + .first; + if (write_stall != WriteStallCondition::kNormal) { + return false; + } + return true; +} + +IOStatus DBImpl::SyncClosedLogs(JobContext* job_context, + VersionEdit* synced_wals) { + TEST_SYNC_POINT("DBImpl::SyncClosedLogs:Start"); + InstrumentedMutexLock l(&log_write_mutex_); + autovector logs_to_sync; + uint64_t current_log_number = logfile_number_; + while (logs_.front().number < current_log_number && + logs_.front().IsSyncing()) { + log_sync_cv_.Wait(); + } + for (auto it = logs_.begin(); + it != logs_.end() && it->number < current_log_number; ++it) { + auto& log = *it; + log.PrepareForSync(); + logs_to_sync.push_back(log.writer); + } + + IOStatus io_s; + if (!logs_to_sync.empty()) { + log_write_mutex_.Unlock(); + + assert(job_context); + + for (log::Writer* log : logs_to_sync) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "[JOB %d] Syncing log #%" PRIu64, job_context->job_id, + log->get_log_number()); + if (error_handler_.IsRecoveryInProgress()) { + log->file()->reset_seen_error(); + } + io_s = log->file()->Sync(immutable_db_options_.use_fsync); + if (!io_s.ok()) { + break; + } + + if (immutable_db_options_.recycle_log_file_num > 0) { + if (error_handler_.IsRecoveryInProgress()) { + log->file()->reset_seen_error(); + } + io_s = log->Close(); + if (!io_s.ok()) { + break; + } + } + } + if (io_s.ok()) { + io_s = directories_.GetWalDir()->FsyncWithDirOptions( + IOOptions(), nullptr, + DirFsyncOptions(DirFsyncOptions::FsyncReason::kNewFileSynced)); + } + + TEST_SYNC_POINT_CALLBACK("DBImpl::SyncClosedLogs:BeforeReLock", + /*arg=*/nullptr); + log_write_mutex_.Lock(); + + // "number <= current_log_number - 1" is equivalent to + // "number < current_log_number". + if (io_s.ok()) { + MarkLogsSynced(current_log_number - 1, true, synced_wals); + } else { + MarkLogsNotSynced(current_log_number - 1); + } + if (!io_s.ok()) { + TEST_SYNC_POINT("DBImpl::SyncClosedLogs:Failed"); + return io_s; + } + } + TEST_SYNC_POINT("DBImpl::SyncClosedLogs:end"); + return io_s; +} + +Status DBImpl::FlushMemTableToOutputFile( + ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options, + bool* made_progress, JobContext* job_context, FlushReason flush_reason, + SuperVersionContext* superversion_context, + std::vector& snapshot_seqs, + SequenceNumber earliest_write_conflict_snapshot, + SnapshotChecker* snapshot_checker, LogBuffer* log_buffer, + Env::Priority thread_pri) { + mutex_.AssertHeld(); + assert(cfd); + assert(cfd->imm()); + assert(cfd->imm()->NumNotFlushed() != 0); + assert(cfd->imm()->IsFlushPending()); + assert(versions_); + assert(versions_->GetColumnFamilySet()); + // If there are more than one column families, we need to make sure that + // all the log files except the most recent one are synced. Otherwise if + // the host crashes after flushing and before WAL is persistent, the + // flushed SST may contain data from write batches whose updates to + // other (unflushed) column families are missing. + const bool needs_to_sync_closed_wals = + logfile_number_ > 0 && + versions_->GetColumnFamilySet()->NumberOfColumnFamilies() > 1; + + // If needs_to_sync_closed_wals is true, we need to record the current + // maximum memtable ID of this column family so that a later PickMemtables() + // call will not pick memtables whose IDs are higher. This is due to the fact + // that SyncClosedLogs() may release the db mutex, and memtable switch can + // happen for this column family in the meantime. The newly created memtables + // have their data backed by unsynced WALs, thus they cannot be included in + // this flush job. + // Another reason why we must record the current maximum memtable ID of this + // column family: SyncClosedLogs() may release db mutex, thus it's possible + // for application to continue to insert into memtables increasing db's + // sequence number. The application may take a snapshot, but this snapshot is + // not included in `snapshot_seqs` which will be passed to flush job because + // `snapshot_seqs` has already been computed before this function starts. + // Recording the max memtable ID ensures that the flush job does not flush + // a memtable without knowing such snapshot(s). + uint64_t max_memtable_id = needs_to_sync_closed_wals + ? cfd->imm()->GetLatestMemTableID() + : std::numeric_limits::max(); + + // If needs_to_sync_closed_wals is false, then the flush job will pick ALL + // existing memtables of the column family when PickMemTable() is called + // later. Although we won't call SyncClosedLogs() in this case, we may still + // call the callbacks of the listeners, i.e. NotifyOnFlushBegin() which also + // releases and re-acquires the db mutex. In the meantime, the application + // can still insert into the memtables and increase the db's sequence number. + // The application can take a snapshot, hoping that the latest visible state + // to this snapshto is preserved. This is hard to guarantee since db mutex + // not held. This newly-created snapshot is not included in `snapshot_seqs` + // and the flush job is unaware of its presence. Consequently, the flush job + // may drop certain keys when generating the L0, causing incorrect data to be + // returned for snapshot read using this snapshot. + // To address this, we make sure NotifyOnFlushBegin() executes after memtable + // picking so that no new snapshot can be taken between the two functions. + + FlushJob flush_job( + dbname_, cfd, immutable_db_options_, mutable_cf_options, max_memtable_id, + file_options_for_compaction_, versions_.get(), &mutex_, &shutting_down_, + snapshot_seqs, earliest_write_conflict_snapshot, snapshot_checker, + job_context, flush_reason, log_buffer, directories_.GetDbDir(), + GetDataDir(cfd, 0U), + GetCompressionFlush(*cfd->ioptions(), mutable_cf_options), stats_, + &event_logger_, mutable_cf_options.report_bg_io_stats, + true /* sync_output_directory */, true /* write_manifest */, thread_pri, + io_tracer_, seqno_time_mapping_, db_id_, db_session_id_, + cfd->GetFullHistoryTsLow(), &blob_callback_); + FileMetaData file_meta; + + Status s; + bool need_cancel = false; + IOStatus log_io_s = IOStatus::OK(); + if (needs_to_sync_closed_wals) { + // SyncClosedLogs() may unlock and re-lock the log_write_mutex multiple + // times. + VersionEdit synced_wals; + mutex_.Unlock(); + log_io_s = SyncClosedLogs(job_context, &synced_wals); + mutex_.Lock(); + if (log_io_s.ok() && synced_wals.IsWalAddition()) { + const ReadOptions read_options(Env::IOActivity::kFlush); + log_io_s = + status_to_io_status(ApplyWALToManifest(read_options, &synced_wals)); + TEST_SYNC_POINT_CALLBACK("DBImpl::FlushMemTableToOutputFile:CommitWal:1", + nullptr); + } + + if (!log_io_s.ok() && !log_io_s.IsShutdownInProgress() && + !log_io_s.IsColumnFamilyDropped()) { + error_handler_.SetBGError(log_io_s, BackgroundErrorReason::kFlush); + } + } else { + TEST_SYNC_POINT("DBImpl::SyncClosedLogs:Skip"); + } + s = log_io_s; + + // If the log sync failed, we do not need to pick memtable. Otherwise, + // num_flush_not_started_ needs to be rollback. + TEST_SYNC_POINT("DBImpl::FlushMemTableToOutputFile:BeforePickMemtables"); + if (s.ok()) { + flush_job.PickMemTable(); + need_cancel = true; + } + TEST_SYNC_POINT_CALLBACK( + "DBImpl::FlushMemTableToOutputFile:AfterPickMemtables", &flush_job); + + // may temporarily unlock and lock the mutex. + NotifyOnFlushBegin(cfd, &file_meta, mutable_cf_options, job_context->job_id, + flush_reason); + + bool switched_to_mempurge = false; + // Within flush_job.Run, rocksdb may call event listener to notify + // file creation and deletion. + // + // Note that flush_job.Run will unlock and lock the db_mutex, + // and EventListener callback will be called when the db_mutex + // is unlocked by the current thread. + if (s.ok()) { + s = flush_job.Run(&logs_with_prep_tracker_, &file_meta, + &switched_to_mempurge); + need_cancel = false; + } + + if (!s.ok() && need_cancel) { + flush_job.Cancel(); + } + + if (s.ok()) { + InstallSuperVersionAndScheduleWork(cfd, superversion_context, + mutable_cf_options); + if (made_progress) { + *made_progress = true; + } + + const std::string& column_family_name = cfd->GetName(); + + Version* const current = cfd->current(); + assert(current); + + const VersionStorageInfo* const storage_info = current->storage_info(); + assert(storage_info); + + VersionStorageInfo::LevelSummaryStorage tmp; + ROCKS_LOG_BUFFER(log_buffer, "[%s] Level summary: %s\n", + column_family_name.c_str(), + storage_info->LevelSummary(&tmp)); + + const auto& blob_files = storage_info->GetBlobFiles(); + if (!blob_files.empty()) { + assert(blob_files.front()); + assert(blob_files.back()); + + ROCKS_LOG_BUFFER( + log_buffer, + "[%s] Blob file summary: head=%" PRIu64 ", tail=%" PRIu64 "\n", + column_family_name.c_str(), blob_files.front()->GetBlobFileNumber(), + blob_files.back()->GetBlobFileNumber()); + } + } + + if (!s.ok() && !s.IsShutdownInProgress() && !s.IsColumnFamilyDropped()) { + if (log_io_s.ok()) { + // Error while writing to MANIFEST. + // In fact, versions_->io_status() can also be the result of renaming + // CURRENT file. With current code, it's just difficult to tell. So just + // be pessimistic and try write to a new MANIFEST. + // TODO: distinguish between MANIFEST write and CURRENT renaming + if (!versions_->io_status().ok()) { + // If WAL sync is successful (either WAL size is 0 or there is no IO + // error), all the Manifest write will be map to soft error. + // TODO: kManifestWriteNoWAL and kFlushNoWAL are misleading. Refactor is + // needed. + error_handler_.SetBGError(s, + BackgroundErrorReason::kManifestWriteNoWAL); + } else { + // If WAL sync is successful (either WAL size is 0 or there is no IO + // error), all the other SST file write errors will be set as + // kFlushNoWAL. + error_handler_.SetBGError(s, BackgroundErrorReason::kFlushNoWAL); + } + } else { + assert(s == log_io_s); + Status new_bg_error = s; + error_handler_.SetBGError(new_bg_error, BackgroundErrorReason::kFlush); + } + } + // If flush ran smoothly and no mempurge happened + // install new SST file path. + if (s.ok() && (!switched_to_mempurge)) { + // may temporarily unlock and lock the mutex. + NotifyOnFlushCompleted(cfd, mutable_cf_options, + flush_job.GetCommittedFlushJobsInfo()); + auto sfm = static_cast( + immutable_db_options_.sst_file_manager.get()); + if (sfm) { + // Notify sst_file_manager that a new file was added + std::string file_path = MakeTableFileName( + cfd->ioptions()->cf_paths[0].path, file_meta.fd.GetNumber()); + // TODO (PR7798). We should only add the file to the FileManager if it + // exists. Otherwise, some tests may fail. Ignore the error in the + // interim. + sfm->OnAddFile(file_path).PermitUncheckedError(); + if (sfm->IsMaxAllowedSpaceReached()) { + Status new_bg_error = + Status::SpaceLimit("Max allowed space was reached"); + TEST_SYNC_POINT_CALLBACK( + "DBImpl::FlushMemTableToOutputFile:MaxAllowedSpaceReached", + &new_bg_error); + error_handler_.SetBGError(new_bg_error, BackgroundErrorReason::kFlush); + } + } + } + TEST_SYNC_POINT("DBImpl::FlushMemTableToOutputFile:Finish"); + return s; +} + +Status DBImpl::FlushMemTablesToOutputFiles( + const autovector& bg_flush_args, bool* made_progress, + JobContext* job_context, LogBuffer* log_buffer, Env::Priority thread_pri) { + if (immutable_db_options_.atomic_flush) { + return AtomicFlushMemTablesToOutputFiles( + bg_flush_args, made_progress, job_context, log_buffer, thread_pri); + } + assert(bg_flush_args.size() == 1); + std::vector snapshot_seqs; + SequenceNumber earliest_write_conflict_snapshot; + SnapshotChecker* snapshot_checker; + GetSnapshotContext(job_context, &snapshot_seqs, + &earliest_write_conflict_snapshot, &snapshot_checker); + const auto& bg_flush_arg = bg_flush_args[0]; + ColumnFamilyData* cfd = bg_flush_arg.cfd_; + // intentional infrequent copy for each flush + MutableCFOptions mutable_cf_options_copy = *cfd->GetLatestMutableCFOptions(); + SuperVersionContext* superversion_context = + bg_flush_arg.superversion_context_; + FlushReason flush_reason = bg_flush_arg.flush_reason_; + Status s = FlushMemTableToOutputFile( + cfd, mutable_cf_options_copy, made_progress, job_context, flush_reason, + superversion_context, snapshot_seqs, earliest_write_conflict_snapshot, + snapshot_checker, log_buffer, thread_pri); + return s; +} + +/* + * Atomically flushes multiple column families. + * + * For each column family, all memtables with ID smaller than or equal to the + * ID specified in bg_flush_args will be flushed. Only after all column + * families finish flush will this function commit to MANIFEST. If any of the + * column families are not flushed successfully, this function does not have + * any side-effect on the state of the database. + */ +Status DBImpl::AtomicFlushMemTablesToOutputFiles( + const autovector& bg_flush_args, bool* made_progress, + JobContext* job_context, LogBuffer* log_buffer, Env::Priority thread_pri) { + mutex_.AssertHeld(); + + autovector cfds; + for (const auto& arg : bg_flush_args) { + cfds.emplace_back(arg.cfd_); + } + +#ifndef NDEBUG + for (const auto cfd : cfds) { + assert(cfd->imm()->NumNotFlushed() != 0); + assert(cfd->imm()->IsFlushPending()); + } + for (const auto& bg_flush_arg : bg_flush_args) { + assert(bg_flush_arg.flush_reason_ == bg_flush_args[0].flush_reason_); + } +#endif /* !NDEBUG */ + + std::vector snapshot_seqs; + SequenceNumber earliest_write_conflict_snapshot; + SnapshotChecker* snapshot_checker; + GetSnapshotContext(job_context, &snapshot_seqs, + &earliest_write_conflict_snapshot, &snapshot_checker); + + autovector distinct_output_dirs; + autovector distinct_output_dir_paths; + std::vector> jobs; + std::vector all_mutable_cf_options; + int num_cfs = static_cast(cfds.size()); + all_mutable_cf_options.reserve(num_cfs); + for (int i = 0; i < num_cfs; ++i) { + auto cfd = cfds[i]; + FSDirectory* data_dir = GetDataDir(cfd, 0U); + const std::string& curr_path = cfd->ioptions()->cf_paths[0].path; + + // Add to distinct output directories if eligible. Use linear search. Since + // the number of elements in the vector is not large, performance should be + // tolerable. + bool found = false; + for (const auto& path : distinct_output_dir_paths) { + if (path == curr_path) { + found = true; + break; + } + } + if (!found) { + distinct_output_dir_paths.emplace_back(curr_path); + distinct_output_dirs.emplace_back(data_dir); + } + + all_mutable_cf_options.emplace_back(*cfd->GetLatestMutableCFOptions()); + const MutableCFOptions& mutable_cf_options = all_mutable_cf_options.back(); + uint64_t max_memtable_id = bg_flush_args[i].max_memtable_id_; + FlushReason flush_reason = bg_flush_args[i].flush_reason_; + jobs.emplace_back(new FlushJob( + dbname_, cfd, immutable_db_options_, mutable_cf_options, + max_memtable_id, file_options_for_compaction_, versions_.get(), &mutex_, + &shutting_down_, snapshot_seqs, earliest_write_conflict_snapshot, + snapshot_checker, job_context, flush_reason, log_buffer, + directories_.GetDbDir(), data_dir, + GetCompressionFlush(*cfd->ioptions(), mutable_cf_options), stats_, + &event_logger_, mutable_cf_options.report_bg_io_stats, + false /* sync_output_directory */, false /* write_manifest */, + thread_pri, io_tracer_, seqno_time_mapping_, db_id_, db_session_id_, + cfd->GetFullHistoryTsLow(), &blob_callback_)); + } + + std::vector file_meta(num_cfs); + // Use of deque because vector + // is specific and doesn't allow &v[i]. + std::deque switched_to_mempurge(num_cfs, false); + Status s; + IOStatus log_io_s = IOStatus::OK(); + assert(num_cfs == static_cast(jobs.size())); + + for (int i = 0; i != num_cfs; ++i) { + const MutableCFOptions& mutable_cf_options = all_mutable_cf_options.at(i); + // may temporarily unlock and lock the mutex. + FlushReason flush_reason = bg_flush_args[i].flush_reason_; + NotifyOnFlushBegin(cfds[i], &file_meta[i], mutable_cf_options, + job_context->job_id, flush_reason); + } + + if (logfile_number_ > 0) { + // TODO (yanqin) investigate whether we should sync the closed logs for + // single column family case. + VersionEdit synced_wals; + mutex_.Unlock(); + log_io_s = SyncClosedLogs(job_context, &synced_wals); + mutex_.Lock(); + if (log_io_s.ok() && synced_wals.IsWalAddition()) { + const ReadOptions read_options(Env::IOActivity::kFlush); + log_io_s = + status_to_io_status(ApplyWALToManifest(read_options, &synced_wals)); + } + + if (!log_io_s.ok() && !log_io_s.IsShutdownInProgress() && + !log_io_s.IsColumnFamilyDropped()) { + if (total_log_size_ > 0) { + error_handler_.SetBGError(log_io_s, BackgroundErrorReason::kFlush); + } else { + // If the WAL is empty, we use different error reason + error_handler_.SetBGError(log_io_s, BackgroundErrorReason::kFlushNoWAL); + } + } + } + s = log_io_s; + + // exec_status stores the execution status of flush_jobs as + // + autovector> exec_status; + std::vector pick_status; + for (int i = 0; i != num_cfs; ++i) { + // Initially all jobs are not executed, with status OK. + exec_status.emplace_back(false, Status::OK()); + pick_status.push_back(false); + } + + if (s.ok()) { + for (int i = 0; i != num_cfs; ++i) { + jobs[i]->PickMemTable(); + pick_status[i] = true; + } + } + + if (s.ok()) { + assert(switched_to_mempurge.size() == + static_cast(num_cfs)); + // TODO (yanqin): parallelize jobs with threads. + for (int i = 1; i != num_cfs; ++i) { + exec_status[i].second = + jobs[i]->Run(&logs_with_prep_tracker_, &file_meta[i], + &(switched_to_mempurge.at(i))); + exec_status[i].first = true; + } + if (num_cfs > 1) { + TEST_SYNC_POINT( + "DBImpl::AtomicFlushMemTablesToOutputFiles:SomeFlushJobsComplete:1"); + TEST_SYNC_POINT( + "DBImpl::AtomicFlushMemTablesToOutputFiles:SomeFlushJobsComplete:2"); + } + assert(exec_status.size() > 0); + assert(!file_meta.empty()); + exec_status[0].second = jobs[0]->Run( + &logs_with_prep_tracker_, file_meta.data() /* &file_meta[0] */, + switched_to_mempurge.empty() ? nullptr : &(switched_to_mempurge.at(0))); + exec_status[0].first = true; + + Status error_status; + for (const auto& e : exec_status) { + if (!e.second.ok()) { + s = e.second; + if (!e.second.IsShutdownInProgress() && + !e.second.IsColumnFamilyDropped()) { + // If a flush job did not return OK, and the CF is not dropped, and + // the DB is not shutting down, then we have to return this result to + // caller later. + error_status = e.second; + } + } + } + + s = error_status.ok() ? s : error_status; + } + + if (s.IsColumnFamilyDropped()) { + s = Status::OK(); + } + + if (s.ok() || s.IsShutdownInProgress()) { + // Sync on all distinct output directories. + for (auto dir : distinct_output_dirs) { + if (dir != nullptr) { + Status error_status = dir->FsyncWithDirOptions( + IOOptions(), nullptr, + DirFsyncOptions(DirFsyncOptions::FsyncReason::kNewFileSynced)); + if (!error_status.ok()) { + s = error_status; + break; + } + } + } + } else { + // Need to undo atomic flush if something went wrong, i.e. s is not OK and + // it is not because of CF drop. + // Have to cancel the flush jobs that have NOT executed because we need to + // unref the versions. + for (int i = 0; i != num_cfs; ++i) { + if (pick_status[i] && !exec_status[i].first) { + jobs[i]->Cancel(); + } + } + for (int i = 0; i != num_cfs; ++i) { + if (exec_status[i].second.ok() && exec_status[i].first) { + auto& mems = jobs[i]->GetMemTables(); + cfds[i]->imm()->RollbackMemtableFlush(mems, + file_meta[i].fd.GetNumber()); + } + } + } + + if (s.ok()) { + const auto wait_to_install_func = + [&]() -> std::pair { + if (!versions_->io_status().ok()) { + // Something went wrong elsewhere, we cannot count on waiting for our + // turn to write/sync to MANIFEST or CURRENT. Just return. + return std::make_pair(versions_->io_status(), false); + } else if (shutting_down_.load(std::memory_order_acquire)) { + return std::make_pair(Status::ShutdownInProgress(), false); + } + bool ready = true; + for (size_t i = 0; i != cfds.size(); ++i) { + const auto& mems = jobs[i]->GetMemTables(); + if (cfds[i]->IsDropped()) { + // If the column family is dropped, then do not wait. + continue; + } else if (!mems.empty() && + cfds[i]->imm()->GetEarliestMemTableID() < mems[0]->GetID()) { + // If a flush job needs to install the flush result for mems and + // mems[0] is not the earliest memtable, it means another thread must + // be installing flush results for the same column family, then the + // current thread needs to wait. + ready = false; + break; + } else if (mems.empty() && cfds[i]->imm()->GetEarliestMemTableID() <= + bg_flush_args[i].max_memtable_id_) { + // If a flush job does not need to install flush results, then it has + // to wait until all memtables up to max_memtable_id_ (inclusive) are + // installed. + ready = false; + break; + } + } + return std::make_pair(Status::OK(), !ready); + }; + + bool resuming_from_bg_err = + error_handler_.IsDBStopped() || + (bg_flush_args[0].flush_reason_ == FlushReason::kErrorRecovery || + bg_flush_args[0].flush_reason_ == + FlushReason::kErrorRecoveryRetryFlush); + while ((!resuming_from_bg_err || error_handler_.GetRecoveryError().ok())) { + std::pair res = wait_to_install_func(); + + TEST_SYNC_POINT_CALLBACK( + "DBImpl::AtomicFlushMemTablesToOutputFiles:WaitToCommit", &res); + + if (!res.first.ok()) { + s = res.first; + break; + } else if (!res.second) { + break; + } + atomic_flush_install_cv_.Wait(); + + resuming_from_bg_err = + error_handler_.IsDBStopped() || + (bg_flush_args[0].flush_reason_ == FlushReason::kErrorRecovery || + bg_flush_args[0].flush_reason_ == + FlushReason::kErrorRecoveryRetryFlush); + } + + if (!resuming_from_bg_err) { + // If not resuming from bg err, then we determine future action based on + // whether we hit background error. + if (s.ok()) { + s = error_handler_.GetBGError(); + } + } else if (s.ok()) { + // If resuming from bg err, we still rely on wait_to_install_func()'s + // result to determine future action. If wait_to_install_func() returns + // non-ok already, then we should not proceed to flush result + // installation. + s = error_handler_.GetRecoveryError(); + } + } + + if (s.ok()) { + autovector tmp_cfds; + autovector*> mems_list; + autovector mutable_cf_options_list; + autovector tmp_file_meta; + autovector>*> + committed_flush_jobs_info; + for (int i = 0; i != num_cfs; ++i) { + const auto& mems = jobs[i]->GetMemTables(); + if (!cfds[i]->IsDropped() && !mems.empty()) { + tmp_cfds.emplace_back(cfds[i]); + mems_list.emplace_back(&mems); + mutable_cf_options_list.emplace_back(&all_mutable_cf_options[i]); + tmp_file_meta.emplace_back(&file_meta[i]); + committed_flush_jobs_info.emplace_back( + jobs[i]->GetCommittedFlushJobsInfo()); + } + } + + s = InstallMemtableAtomicFlushResults( + nullptr /* imm_lists */, tmp_cfds, mutable_cf_options_list, mems_list, + versions_.get(), &logs_with_prep_tracker_, &mutex_, tmp_file_meta, + committed_flush_jobs_info, &job_context->memtables_to_free, + directories_.GetDbDir(), log_buffer); + } + + if (s.ok()) { + assert(num_cfs == + static_cast(job_context->superversion_contexts.size())); + for (int i = 0; i != num_cfs; ++i) { + assert(cfds[i]); + + if (cfds[i]->IsDropped()) { + continue; + } + InstallSuperVersionAndScheduleWork(cfds[i], + &job_context->superversion_contexts[i], + all_mutable_cf_options[i]); + + const std::string& column_family_name = cfds[i]->GetName(); + + Version* const current = cfds[i]->current(); + assert(current); + + const VersionStorageInfo* const storage_info = current->storage_info(); + assert(storage_info); + + VersionStorageInfo::LevelSummaryStorage tmp; + ROCKS_LOG_BUFFER(log_buffer, "[%s] Level summary: %s\n", + column_family_name.c_str(), + storage_info->LevelSummary(&tmp)); + + const auto& blob_files = storage_info->GetBlobFiles(); + if (!blob_files.empty()) { + assert(blob_files.front()); + assert(blob_files.back()); + + ROCKS_LOG_BUFFER( + log_buffer, + "[%s] Blob file summary: head=%" PRIu64 ", tail=%" PRIu64 "\n", + column_family_name.c_str(), blob_files.front()->GetBlobFileNumber(), + blob_files.back()->GetBlobFileNumber()); + } + } + if (made_progress) { + *made_progress = true; + } + auto sfm = static_cast( + immutable_db_options_.sst_file_manager.get()); + assert(all_mutable_cf_options.size() == static_cast(num_cfs)); + for (int i = 0; s.ok() && i != num_cfs; ++i) { + // If mempurge happened instead of Flush, + // no NotifyOnFlushCompleted call (no SST file created). + if (switched_to_mempurge[i]) { + continue; + } + if (cfds[i]->IsDropped()) { + continue; + } + NotifyOnFlushCompleted(cfds[i], all_mutable_cf_options[i], + jobs[i]->GetCommittedFlushJobsInfo()); + if (sfm) { + std::string file_path = MakeTableFileName( + cfds[i]->ioptions()->cf_paths[0].path, file_meta[i].fd.GetNumber()); + // TODO (PR7798). We should only add the file to the FileManager if it + // exists. Otherwise, some tests may fail. Ignore the error in the + // interim. + sfm->OnAddFile(file_path).PermitUncheckedError(); + if (sfm->IsMaxAllowedSpaceReached() && + error_handler_.GetBGError().ok()) { + Status new_bg_error = + Status::SpaceLimit("Max allowed space was reached"); + error_handler_.SetBGError(new_bg_error, + BackgroundErrorReason::kFlush); + } + } + } + } + + // Need to undo atomic flush if something went wrong, i.e. s is not OK and + // it is not because of CF drop. + if (!s.ok() && !s.IsColumnFamilyDropped()) { + if (log_io_s.ok()) { + // Error while writing to MANIFEST. + // In fact, versions_->io_status() can also be the result of renaming + // CURRENT file. With current code, it's just difficult to tell. So just + // be pessimistic and try write to a new MANIFEST. + // TODO: distinguish between MANIFEST write and CURRENT renaming + if (!versions_->io_status().ok()) { + // If WAL sync is successful (either WAL size is 0 or there is no IO + // error), all the Manifest write will be map to soft error. + // TODO: kManifestWriteNoWAL and kFlushNoWAL are misleading. Refactor + // is needed. + error_handler_.SetBGError(s, + BackgroundErrorReason::kManifestWriteNoWAL); + } else { + // If WAL sync is successful (either WAL size is 0 or there is no IO + // error), all the other SST file write errors will be set as + // kFlushNoWAL. + error_handler_.SetBGError(s, BackgroundErrorReason::kFlushNoWAL); + } + } else { + assert(s == log_io_s); + Status new_bg_error = s; + error_handler_.SetBGError(new_bg_error, BackgroundErrorReason::kFlush); + } + } + + return s; +} + +void DBImpl::NotifyOnFlushBegin(ColumnFamilyData* cfd, FileMetaData* file_meta, + const MutableCFOptions& mutable_cf_options, + int job_id, FlushReason flush_reason) { + if (immutable_db_options_.listeners.size() == 0U) { + return; + } + mutex_.AssertHeld(); + if (shutting_down_.load(std::memory_order_acquire)) { + return; + } + bool triggered_writes_slowdown = + (cfd->current()->storage_info()->NumLevelFiles(0) >= + mutable_cf_options.level0_slowdown_writes_trigger); + bool triggered_writes_stop = + (cfd->current()->storage_info()->NumLevelFiles(0) >= + mutable_cf_options.level0_stop_writes_trigger); + // release lock while notifying events + mutex_.Unlock(); + { + FlushJobInfo info{}; + info.cf_id = cfd->GetID(); + info.cf_name = cfd->GetName(); + // TODO(yhchiang): make db_paths dynamic in case flush does not + // go to L0 in the future. + const uint64_t file_number = file_meta->fd.GetNumber(); + info.file_path = + MakeTableFileName(cfd->ioptions()->cf_paths[0].path, file_number); + info.file_number = file_number; + info.thread_id = env_->GetThreadID(); + info.job_id = job_id; + info.triggered_writes_slowdown = triggered_writes_slowdown; + info.triggered_writes_stop = triggered_writes_stop; + info.smallest_seqno = file_meta->fd.smallest_seqno; + info.largest_seqno = file_meta->fd.largest_seqno; + info.flush_reason = flush_reason; + for (auto listener : immutable_db_options_.listeners) { + listener->OnFlushBegin(this, info); + } + } + mutex_.Lock(); + // no need to signal bg_cv_ as it will be signaled at the end of the + // flush process. +} + +void DBImpl::NotifyOnFlushCompleted( + ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options, + std::list>* flush_jobs_info) { + assert(flush_jobs_info != nullptr); + if (immutable_db_options_.listeners.size() == 0U) { + return; + } + mutex_.AssertHeld(); + if (shutting_down_.load(std::memory_order_acquire)) { + return; + } + bool triggered_writes_slowdown = + (cfd->current()->storage_info()->NumLevelFiles(0) >= + mutable_cf_options.level0_slowdown_writes_trigger); + bool triggered_writes_stop = + (cfd->current()->storage_info()->NumLevelFiles(0) >= + mutable_cf_options.level0_stop_writes_trigger); + // release lock while notifying events + mutex_.Unlock(); + { + for (auto& info : *flush_jobs_info) { + info->triggered_writes_slowdown = triggered_writes_slowdown; + info->triggered_writes_stop = triggered_writes_stop; + for (auto listener : immutable_db_options_.listeners) { + listener->OnFlushCompleted(this, *info); + } + TEST_SYNC_POINT( + "DBImpl::NotifyOnFlushCompleted::PostAllOnFlushCompleted"); + } + flush_jobs_info->clear(); + } + mutex_.Lock(); + // no need to signal bg_cv_ as it will be signaled at the end of the + // flush process. +} + +Status DBImpl::CompactRange(const CompactRangeOptions& options, + ColumnFamilyHandle* column_family, + const Slice* begin_without_ts, + const Slice* end_without_ts) { + if (manual_compaction_paused_.load(std::memory_order_acquire) > 0) { + return Status::Incomplete(Status::SubCode::kManualCompactionPaused); + } + + if (options.canceled && options.canceled->load(std::memory_order_acquire)) { + return Status::Incomplete(Status::SubCode::kManualCompactionPaused); + } + + const Comparator* const ucmp = column_family->GetComparator(); + assert(ucmp); + size_t ts_sz = ucmp->timestamp_size(); + if (ts_sz == 0) { + return CompactRangeInternal(options, column_family, begin_without_ts, + end_without_ts, "" /*trim_ts*/); + } + + std::string begin_str; + std::string end_str; + + // CompactRange compact all keys: [begin, end] inclusively. Add maximum + // timestamp to include all `begin` keys, and add minimal timestamp to include + // all `end` keys. + if (begin_without_ts != nullptr) { + AppendKeyWithMaxTimestamp(&begin_str, *begin_without_ts, ts_sz); + } + if (end_without_ts != nullptr) { + AppendKeyWithMinTimestamp(&end_str, *end_without_ts, ts_sz); + } + Slice begin(begin_str); + Slice end(end_str); + + Slice* begin_with_ts = begin_without_ts ? &begin : nullptr; + Slice* end_with_ts = end_without_ts ? &end : nullptr; + + return CompactRangeInternal(options, column_family, begin_with_ts, + end_with_ts, "" /*trim_ts*/); +} + +Status DBImpl::IncreaseFullHistoryTsLow(ColumnFamilyHandle* column_family, + std::string ts_low) { + ColumnFamilyData* cfd = nullptr; + if (column_family == nullptr) { + cfd = default_cf_handle_->cfd(); + } else { + auto cfh = static_cast_with_check(column_family); + assert(cfh != nullptr); + cfd = cfh->cfd(); + } + assert(cfd != nullptr && cfd->user_comparator() != nullptr); + if (cfd->user_comparator()->timestamp_size() == 0) { + return Status::InvalidArgument( + "Timestamp is not enabled in this column family"); + } + if (cfd->user_comparator()->timestamp_size() != ts_low.size()) { + return Status::InvalidArgument("ts_low size mismatch"); + } + return IncreaseFullHistoryTsLowImpl(cfd, ts_low); +} + +Status DBImpl::IncreaseFullHistoryTsLowImpl(ColumnFamilyData* cfd, + std::string ts_low) { + VersionEdit edit; + edit.SetColumnFamily(cfd->GetID()); + edit.SetFullHistoryTsLow(ts_low); + + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + TEST_SYNC_POINT_CALLBACK("DBImpl::IncreaseFullHistoryTsLowImpl:BeforeEdit", + &edit); + + InstrumentedMutexLock l(&mutex_); + std::string current_ts_low = cfd->GetFullHistoryTsLow(); + const Comparator* ucmp = cfd->user_comparator(); + assert(ucmp->timestamp_size() == ts_low.size() && !ts_low.empty()); + if (!current_ts_low.empty() && + ucmp->CompareTimestamp(ts_low, current_ts_low) < 0) { + return Status::InvalidArgument("Cannot decrease full_history_ts_low"); + } + + Status s = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(), + read_options, &edit, &mutex_, + directories_.GetDbDir()); + if (!s.ok()) { + return s; + } + current_ts_low = cfd->GetFullHistoryTsLow(); + if (!current_ts_low.empty() && + ucmp->CompareTimestamp(current_ts_low, ts_low) > 0) { + std::stringstream oss; + oss << "full_history_ts_low: " << Slice(current_ts_low).ToString(true) + << " is set to be higher than the requested " + "timestamp: " + << Slice(ts_low).ToString(true) << std::endl; + return Status::TryAgain(oss.str()); + } + return Status::OK(); +} + +Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options, + ColumnFamilyHandle* column_family, + const Slice* begin, const Slice* end, + const std::string& trim_ts) { + auto cfh = static_cast_with_check(column_family); + auto cfd = cfh->cfd(); + + if (options.target_path_id >= cfd->ioptions()->cf_paths.size()) { + return Status::InvalidArgument("Invalid target path ID"); + } + + bool flush_needed = true; + + // Update full_history_ts_low if it's set + if (options.full_history_ts_low != nullptr && + !options.full_history_ts_low->empty()) { + std::string ts_low = options.full_history_ts_low->ToString(); + if (begin != nullptr || end != nullptr) { + return Status::InvalidArgument( + "Cannot specify compaction range with full_history_ts_low"); + } + Status s = IncreaseFullHistoryTsLowImpl(cfd, ts_low); + if (!s.ok()) { + LogFlush(immutable_db_options_.info_log); + return s; + } + } + + Status s; + if (begin != nullptr && end != nullptr) { + // TODO(ajkr): We could also optimize away the flush in certain cases where + // one/both sides of the interval are unbounded. But it requires more + // changes to RangesOverlapWithMemtables. + Range range(*begin, *end); + SuperVersion* super_version = cfd->GetReferencedSuperVersion(this); + s = cfd->RangesOverlapWithMemtables( + {range}, super_version, immutable_db_options_.allow_data_in_errors, + &flush_needed); + CleanupSuperVersion(super_version); + } + + if (s.ok() && flush_needed) { + FlushOptions fo; + fo.allow_write_stall = options.allow_write_stall; + if (immutable_db_options_.atomic_flush) { + s = AtomicFlushMemTables(fo, FlushReason::kManualCompaction); + } else { + s = FlushMemTable(cfd, fo, FlushReason::kManualCompaction); + } + if (!s.ok()) { + LogFlush(immutable_db_options_.info_log); + return s; + } + } + + constexpr int kInvalidLevel = -1; + int final_output_level = kInvalidLevel; + bool exclusive = options.exclusive_manual_compaction; + if (cfd->ioptions()->compaction_style == kCompactionStyleUniversal && + cfd->NumberLevels() > 1) { + // Always compact all files together. + final_output_level = cfd->NumberLevels() - 1; + // if bottom most level is reserved + if (immutable_db_options_.allow_ingest_behind) { + final_output_level--; + } + s = RunManualCompaction(cfd, ColumnFamilyData::kCompactAllLevels, + final_output_level, options, begin, end, exclusive, + false /* disable_trivial_move */, + std::numeric_limits::max(), trim_ts); + } else { + int first_overlapped_level = kInvalidLevel; + { + SuperVersion* super_version = cfd->GetReferencedSuperVersion(this); + Version* current_version = super_version->current; + + // Might need to query the partitioner + SstPartitionerFactory* partitioner_factory = + current_version->cfd()->ioptions()->sst_partitioner_factory.get(); + std::unique_ptr partitioner; + if (partitioner_factory && begin != nullptr && end != nullptr) { + SstPartitioner::Context context; + context.is_full_compaction = false; + context.is_manual_compaction = true; + context.output_level = /*unknown*/ -1; + // Small lies about compaction range + context.smallest_user_key = *begin; + context.largest_user_key = *end; + partitioner = partitioner_factory->CreatePartitioner(context); + } + + ReadOptions ro; + ro.total_order_seek = true; + ro.io_activity = Env::IOActivity::kCompaction; + bool overlap; + for (int level = 0; + level < current_version->storage_info()->num_non_empty_levels(); + level++) { + overlap = true; + + // Whether to look at specific keys within files for overlap with + // compaction range, other than largest and smallest keys of the file + // known in Version metadata. + bool check_overlap_within_file = false; + if (begin != nullptr && end != nullptr) { + // Typically checking overlap within files in this case + check_overlap_within_file = true; + // WART: Not known why we don't check within file in one-sided bound + // cases + if (partitioner) { + // Especially if the partitioner is new, the manual compaction + // might be used to enforce the partitioning. Checking overlap + // within files might miss cases where compaction is needed to + // partition the files, as in this example: + // * File has two keys "001" and "111" + // * Compaction range is ["011", "101") + // * Partition boundary at "100" + // In cases like this, file-level overlap with the compaction + // range is sufficient to force any partitioning that is needed + // within the compaction range. + // + // But if there's no partitioning boundary within the compaction + // range, we can be sure there's no need to fix partitioning + // within that range, thus safe to check overlap within file. + // + // Use a hypothetical trivial move query to check for partition + // boundary in range. (NOTE: in defiance of all conventions, + // `begin` and `end` here are both INCLUSIVE bounds, which makes + // this analogy to CanDoTrivialMove() accurate even when `end` is + // the first key in a partition.) + if (!partitioner->CanDoTrivialMove(*begin, *end)) { + check_overlap_within_file = false; + } + } + } + if (check_overlap_within_file) { + Status status = current_version->OverlapWithLevelIterator( + ro, file_options_, *begin, *end, level, &overlap); + if (!status.ok()) { + check_overlap_within_file = false; + } + } + if (!check_overlap_within_file) { + overlap = current_version->storage_info()->OverlapInLevel(level, + begin, end); + } + if (overlap) { + first_overlapped_level = level; + break; + } + } + CleanupSuperVersion(super_version); + } + if (s.ok() && first_overlapped_level != kInvalidLevel) { + if (cfd->ioptions()->compaction_style == kCompactionStyleUniversal || + cfd->ioptions()->compaction_style == kCompactionStyleFIFO) { + assert(first_overlapped_level == 0); + s = RunManualCompaction( + cfd, first_overlapped_level, first_overlapped_level, options, begin, + end, exclusive, true /* disallow_trivial_move */, + std::numeric_limits::max() /* max_file_num_to_ignore */, + trim_ts); + final_output_level = first_overlapped_level; + } else { + assert(cfd->ioptions()->compaction_style == kCompactionStyleLevel); + uint64_t next_file_number = versions_->current_next_file_number(); + // Start compaction from `first_overlapped_level`, one level down at a + // time, until output level >= max_overlapped_level. + // When max_overlapped_level == 0, we will still compact from L0 -> L1 + // (or LBase), and followed by a bottommost level intra-level compaction + // at L1 (or LBase), if applicable. + int level = first_overlapped_level; + final_output_level = level; + int output_level = 0, base_level = 0; + for (;;) { + // Always allow L0 -> L1 compaction + if (level > 0) { + if (cfd->ioptions()->level_compaction_dynamic_level_bytes) { + assert(final_output_level < cfd->ioptions()->num_levels); + if (final_output_level + 1 == cfd->ioptions()->num_levels) { + break; + } + } else { + // TODO(cbi): there is still a race condition here where + // if a background compaction compacts some file beyond + // current()->storage_info()->num_non_empty_levels() right after + // the check here.This should happen very infrequently and should + // not happen once a user populates the last level of the LSM. + InstrumentedMutexLock l(&mutex_); + // num_non_empty_levels may be lower after a compaction, so + // we check for >= here. + if (final_output_level + 1 >= + cfd->current()->storage_info()->num_non_empty_levels()) { + break; + } + } + } + output_level = level + 1; + if (cfd->ioptions()->level_compaction_dynamic_level_bytes && + level == 0) { + output_level = ColumnFamilyData::kCompactToBaseLevel; + } + // Use max value for `max_file_num_to_ignore` to always compact + // files down. + s = RunManualCompaction( + cfd, level, output_level, options, begin, end, exclusive, + !trim_ts.empty() /* disallow_trivial_move */, + std::numeric_limits::max() /* max_file_num_to_ignore */, + trim_ts, + output_level == ColumnFamilyData::kCompactToBaseLevel + ? &base_level + : nullptr); + if (!s.ok()) { + break; + } + if (output_level == ColumnFamilyData::kCompactToBaseLevel) { + assert(base_level > 0); + level = base_level; + } else { + ++level; + } + final_output_level = level; + TEST_SYNC_POINT("DBImpl::RunManualCompaction()::1"); + TEST_SYNC_POINT("DBImpl::RunManualCompaction()::2"); + } + if (s.ok()) { + assert(final_output_level > 0); + // bottommost level intra-level compaction + if ((options.bottommost_level_compaction == + BottommostLevelCompaction::kIfHaveCompactionFilter && + (cfd->ioptions()->compaction_filter != nullptr || + cfd->ioptions()->compaction_filter_factory != nullptr)) || + options.bottommost_level_compaction == + BottommostLevelCompaction::kForceOptimized || + options.bottommost_level_compaction == + BottommostLevelCompaction::kForce) { + // Use `next_file_number` as `max_file_num_to_ignore` to avoid + // rewriting newly compacted files when it is kForceOptimized + // or kIfHaveCompactionFilter with compaction filter set. + s = RunManualCompaction( + cfd, final_output_level, final_output_level, options, begin, + end, exclusive, true /* disallow_trivial_move */, + next_file_number /* max_file_num_to_ignore */, trim_ts); + } + } + } + } + } + if (!s.ok() || final_output_level == kInvalidLevel) { + LogFlush(immutable_db_options_.info_log); + return s; + } + + if (options.change_level) { + TEST_SYNC_POINT("DBImpl::CompactRange:BeforeRefit:1"); + TEST_SYNC_POINT("DBImpl::CompactRange:BeforeRefit:2"); + + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "[RefitLevel] waiting for background threads to stop"); + // TODO(hx235): remove `Enable/DisableManualCompaction` and + // `Continue/PauseBackgroundWork` once we ensure registering RefitLevel()'s + // range is sufficient (if not, what else is needed) for avoiding range + // conflicts with other activities (e.g, compaction, flush) that are + // currently avoided by `Enable/DisableManualCompaction` and + // `Continue/PauseBackgroundWork`. + DisableManualCompaction(); + s = PauseBackgroundWork(); + if (s.ok()) { + TEST_SYNC_POINT("DBImpl::CompactRange:PreRefitLevel"); + s = ReFitLevel(cfd, final_output_level, options.target_level); + TEST_SYNC_POINT("DBImpl::CompactRange:PostRefitLevel"); + // ContinueBackgroundWork always return Status::OK(). + Status temp_s = ContinueBackgroundWork(); + assert(temp_s.ok()); + } + EnableManualCompaction(); + TEST_SYNC_POINT( + "DBImpl::CompactRange:PostRefitLevel:ManualCompactionEnabled"); + } + LogFlush(immutable_db_options_.info_log); + + { + InstrumentedMutexLock l(&mutex_); + // an automatic compaction that has been scheduled might have been + // preempted by the manual compactions. Need to schedule it back. + MaybeScheduleFlushOrCompaction(); + } + + return s; +} + +Status DBImpl::CompactFiles(const CompactionOptions& compact_options, + ColumnFamilyHandle* column_family, + const std::vector& input_file_names, + const int output_level, const int output_path_id, + std::vector* const output_file_names, + CompactionJobInfo* compaction_job_info) { + if (column_family == nullptr) { + return Status::InvalidArgument("ColumnFamilyHandle must be non-null."); + } + + auto cfd = + static_cast_with_check(column_family)->cfd(); + assert(cfd); + + Status s; + JobContext job_context(next_job_id_.fetch_add(1), true); + LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, + immutable_db_options_.info_log.get()); + + // Perform CompactFiles + TEST_SYNC_POINT("TestCompactFiles::IngestExternalFile2"); + TEST_SYNC_POINT_CALLBACK( + "TestCompactFiles:PausingManualCompaction:3", + reinterpret_cast( + const_cast*>(&manual_compaction_paused_))); + { + InstrumentedMutexLock l(&mutex_); + auto* current = cfd->current(); + current->Ref(); + + s = CompactFilesImpl(compact_options, cfd, current, input_file_names, + output_file_names, output_level, output_path_id, + &job_context, &log_buffer, compaction_job_info); + + current->Unref(); + } + + // Find and delete obsolete files + { + InstrumentedMutexLock l(&mutex_); + // If !s.ok(), this means that Compaction failed. In that case, we want + // to delete all obsolete files we might have created and we force + // FindObsoleteFiles(). This is because job_context does not + // catch all created files if compaction failed. + FindObsoleteFiles(&job_context, !s.ok()); + } // release the mutex + + // delete unnecessary files if any, this is done outside the mutex + if (job_context.HaveSomethingToClean() || + job_context.HaveSomethingToDelete() || !log_buffer.IsEmpty()) { + // Have to flush the info logs before bg_compaction_scheduled_-- + // because if bg_flush_scheduled_ becomes 0 and the lock is + // released, the deconstructor of DB can kick in and destroy all the + // states of DB so info_log might not be available after that point. + // It also applies to access other states that DB owns. + log_buffer.FlushBufferToLog(); + if (job_context.HaveSomethingToDelete()) { + // no mutex is locked here. No need to Unlock() and Lock() here. + PurgeObsoleteFiles(job_context); + } + job_context.Clean(); + } + + return s; +} + +Status DBImpl::CompactFilesImpl( + const CompactionOptions& compact_options, ColumnFamilyData* cfd, + Version* version, const std::vector& input_file_names, + std::vector* const output_file_names, const int output_level, + int output_path_id, JobContext* job_context, LogBuffer* log_buffer, + CompactionJobInfo* compaction_job_info) { + mutex_.AssertHeld(); + + if (shutting_down_.load(std::memory_order_acquire)) { + return Status::ShutdownInProgress(); + } + if (manual_compaction_paused_.load(std::memory_order_acquire) > 0) { + return Status::Incomplete(Status::SubCode::kManualCompactionPaused); + } + + std::unordered_set input_set; + for (const auto& file_name : input_file_names) { + input_set.insert(TableFileNameToNumber(file_name)); + } + + ColumnFamilyMetaData cf_meta; + // TODO(yhchiang): can directly use version here if none of the + // following functions call is pluggable to external developers. + version->GetColumnFamilyMetaData(&cf_meta); + + if (output_path_id < 0) { + if (cfd->ioptions()->cf_paths.size() == 1U) { + output_path_id = 0; + } else { + return Status::NotSupported( + "Automatic output path selection is not " + "yet supported in CompactFiles()"); + } + } + + if (cfd->ioptions()->allow_ingest_behind && + output_level >= cfd->ioptions()->num_levels - 1) { + return Status::InvalidArgument( + "Exceed the maximum output level defined by " + "the current compaction algorithm with ingest_behind --- " + + std::to_string(cfd->ioptions()->num_levels - 1)); + } + + Status s = cfd->compaction_picker()->SanitizeCompactionInputFiles( + &input_set, cf_meta, output_level); + TEST_SYNC_POINT("DBImpl::CompactFilesImpl::PostSanitizeCompactionInputFiles"); + if (!s.ok()) { + return s; + } + + std::vector input_files; + s = cfd->compaction_picker()->GetCompactionInputsFromFileNumbers( + &input_files, &input_set, version->storage_info(), compact_options); + if (!s.ok()) { + return s; + } + + for (const auto& inputs : input_files) { + if (cfd->compaction_picker()->AreFilesInCompaction(inputs.files)) { + return Status::Aborted( + "Some of the necessary compaction input " + "files are already being compacted"); + } + } + bool sfm_reserved_compact_space = false; + // First check if we have enough room to do the compaction + bool enough_room = EnoughRoomForCompaction( + cfd, input_files, &sfm_reserved_compact_space, log_buffer); + + if (!enough_room) { + // m's vars will get set properly at the end of this function, + // as long as status == CompactionTooLarge + return Status::CompactionTooLarge(); + } + + // At this point, CompactFiles will be run. + bg_compaction_scheduled_++; + + std::unique_ptr c; + assert(cfd->compaction_picker()); + c.reset(cfd->compaction_picker()->CompactFiles( + compact_options, input_files, output_level, version->storage_info(), + *cfd->GetLatestMutableCFOptions(), mutable_db_options_, output_path_id)); + // we already sanitized the set of input files and checked for conflicts + // without releasing the lock, so we're guaranteed a compaction can be formed. + assert(c != nullptr); + + c->SetInputVersion(version); + // deletion compaction currently not allowed in CompactFiles. + assert(!c->deletion_compaction()); + + std::vector snapshot_seqs; + SequenceNumber earliest_write_conflict_snapshot; + SnapshotChecker* snapshot_checker; + GetSnapshotContext(job_context, &snapshot_seqs, + &earliest_write_conflict_snapshot, &snapshot_checker); + + std::unique_ptr::iterator> pending_outputs_inserted_elem( + new std::list::iterator( + CaptureCurrentFileNumberInPendingOutputs())); + + assert(is_snapshot_supported_ || snapshots_.empty()); + CompactionJobStats compaction_job_stats; + CompactionJob compaction_job( + job_context->job_id, c.get(), immutable_db_options_, mutable_db_options_, + file_options_for_compaction_, versions_.get(), &shutting_down_, + log_buffer, directories_.GetDbDir(), + GetDataDir(c->column_family_data(), c->output_path_id()), + GetDataDir(c->column_family_data(), 0), stats_, &mutex_, &error_handler_, + snapshot_seqs, earliest_write_conflict_snapshot, snapshot_checker, + job_context, table_cache_, &event_logger_, + c->mutable_cf_options()->paranoid_file_checks, + c->mutable_cf_options()->report_bg_io_stats, dbname_, + &compaction_job_stats, Env::Priority::USER, io_tracer_, + kManualCompactionCanceledFalse_, db_id_, db_session_id_, + c->column_family_data()->GetFullHistoryTsLow(), c->trim_ts(), + &blob_callback_, &bg_compaction_scheduled_, + &bg_bottom_compaction_scheduled_); + + // Creating a compaction influences the compaction score because the score + // takes running compactions into account (by skipping files that are already + // being compacted). Since we just changed compaction score, we recalculate it + // here. + version->storage_info()->ComputeCompactionScore(*cfd->ioptions(), + *c->mutable_cf_options()); + + compaction_job.Prepare(); + + mutex_.Unlock(); + TEST_SYNC_POINT("CompactFilesImpl:0"); + TEST_SYNC_POINT("CompactFilesImpl:1"); + // Ignore the status here, as it will be checked in the Install down below... + compaction_job.Run().PermitUncheckedError(); + TEST_SYNC_POINT("CompactFilesImpl:2"); + TEST_SYNC_POINT("CompactFilesImpl:3"); + mutex_.Lock(); + + Status status = compaction_job.Install(*c->mutable_cf_options()); + if (status.ok()) { + assert(compaction_job.io_status().ok()); + InstallSuperVersionAndScheduleWork(c->column_family_data(), + &job_context->superversion_contexts[0], + *c->mutable_cf_options()); + } + // status above captures any error during compaction_job.Install, so its ok + // not check compaction_job.io_status() explicitly if we're not calling + // SetBGError + compaction_job.io_status().PermitUncheckedError(); + c->ReleaseCompactionFiles(s); + // Need to make sure SstFileManager does its bookkeeping + auto sfm = static_cast( + immutable_db_options_.sst_file_manager.get()); + if (sfm && sfm_reserved_compact_space) { + sfm->OnCompactionCompletion(c.get()); + } + + ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem); + + if (compaction_job_info != nullptr) { + BuildCompactionJobInfo(cfd, c.get(), s, compaction_job_stats, + job_context->job_id, compaction_job_info); + } + + if (status.ok()) { + // Done + } else if (status.IsColumnFamilyDropped() || status.IsShutdownInProgress()) { + // Ignore compaction errors found during shutting down + } else if (status.IsManualCompactionPaused()) { + // Don't report stopping manual compaction as error + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "[%s] [JOB %d] Stopping manual compaction", + c->column_family_data()->GetName().c_str(), + job_context->job_id); + } else { + ROCKS_LOG_WARN(immutable_db_options_.info_log, + "[%s] [JOB %d] Compaction error: %s", + c->column_family_data()->GetName().c_str(), + job_context->job_id, status.ToString().c_str()); + IOStatus io_s = compaction_job.io_status(); + if (!io_s.ok()) { + error_handler_.SetBGError(io_s, BackgroundErrorReason::kCompaction); + } else { + error_handler_.SetBGError(status, BackgroundErrorReason::kCompaction); + } + } + + if (output_file_names != nullptr) { + for (const auto& newf : c->edit()->GetNewFiles()) { + output_file_names->push_back(TableFileName( + c->immutable_options()->cf_paths, newf.second.fd.GetNumber(), + newf.second.fd.GetPathId())); + } + + for (const auto& blob_file : c->edit()->GetBlobFileAdditions()) { + output_file_names->push_back( + BlobFileName(c->immutable_options()->cf_paths.front().path, + blob_file.GetBlobFileNumber())); + } + } + + c.reset(); + + bg_compaction_scheduled_--; + if (bg_compaction_scheduled_ == 0) { + bg_cv_.SignalAll(); + } + MaybeScheduleFlushOrCompaction(); + TEST_SYNC_POINT("CompactFilesImpl:End"); + + return status; +} + +Status DBImpl::PauseBackgroundWork() { + InstrumentedMutexLock guard_lock(&mutex_); + bg_compaction_paused_++; + while (bg_bottom_compaction_scheduled_ > 0 || bg_compaction_scheduled_ > 0 || + bg_flush_scheduled_ > 0) { + bg_cv_.Wait(); + } + bg_work_paused_++; + return Status::OK(); +} + +Status DBImpl::ContinueBackgroundWork() { + InstrumentedMutexLock guard_lock(&mutex_); + if (bg_work_paused_ == 0) { + return Status::InvalidArgument(); + } + assert(bg_work_paused_ > 0); + assert(bg_compaction_paused_ > 0); + bg_compaction_paused_--; + bg_work_paused_--; + // It's sufficient to check just bg_work_paused_ here since + // bg_work_paused_ is always no greater than bg_compaction_paused_ + if (bg_work_paused_ == 0) { + MaybeScheduleFlushOrCompaction(); + } + return Status::OK(); +} + +void DBImpl::NotifyOnCompactionBegin(ColumnFamilyData* cfd, Compaction* c, + const Status& st, + const CompactionJobStats& job_stats, + int job_id) { + if (immutable_db_options_.listeners.empty()) { + return; + } + mutex_.AssertHeld(); + if (shutting_down_.load(std::memory_order_acquire)) { + return; + } + if (c->is_manual_compaction() && + manual_compaction_paused_.load(std::memory_order_acquire) > 0) { + return; + } + + c->SetNotifyOnCompactionCompleted(); + // release lock while notifying events + mutex_.Unlock(); + TEST_SYNC_POINT("DBImpl::NotifyOnCompactionBegin::UnlockMutex"); + { + CompactionJobInfo info{}; + BuildCompactionJobInfo(cfd, c, st, job_stats, job_id, &info); + for (auto listener : immutable_db_options_.listeners) { + listener->OnCompactionBegin(this, info); + } + info.status.PermitUncheckedError(); + } + mutex_.Lock(); +} + +void DBImpl::NotifyOnCompactionCompleted( + ColumnFamilyData* cfd, Compaction* c, const Status& st, + const CompactionJobStats& compaction_job_stats, const int job_id) { + if (immutable_db_options_.listeners.size() == 0U) { + return; + } + mutex_.AssertHeld(); + if (shutting_down_.load(std::memory_order_acquire)) { + return; + } + + if (c->ShouldNotifyOnCompactionCompleted() == false) { + return; + } + + // release lock while notifying events + mutex_.Unlock(); + TEST_SYNC_POINT("DBImpl::NotifyOnCompactionCompleted::UnlockMutex"); + { + CompactionJobInfo info{}; + BuildCompactionJobInfo(cfd, c, st, compaction_job_stats, job_id, &info); + for (auto listener : immutable_db_options_.listeners) { + listener->OnCompactionCompleted(this, info); + } + } + mutex_.Lock(); + // no need to signal bg_cv_ as it will be signaled at the end of the + // flush process. +} + +// REQUIREMENT: block all background work by calling PauseBackgroundWork() +// before calling this function +Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) { + assert(level < cfd->NumberLevels()); + if (target_level >= cfd->NumberLevels()) { + return Status::InvalidArgument("Target level exceeds number of levels"); + } + + const ReadOptions read_options(Env::IOActivity::kCompaction); + + SuperVersionContext sv_context(/* create_superversion */ true); + + InstrumentedMutexLock guard_lock(&mutex_); + + auto* vstorage = cfd->current()->storage_info(); + if (vstorage->LevelFiles(level).empty()) { + return Status::OK(); + } + // only allow one thread refitting + if (refitting_level_) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "[ReFitLevel] another thread is refitting"); + return Status::NotSupported("another thread is refitting"); + } + refitting_level_ = true; + + const MutableCFOptions mutable_cf_options = *cfd->GetLatestMutableCFOptions(); + // move to a smaller level + int to_level = target_level; + if (target_level < 0) { + to_level = FindMinimumEmptyLevelFitting(cfd, mutable_cf_options, level); + } + + if (to_level != level) { + std::vector input(1); + input[0].level = level; + for (auto& f : vstorage->LevelFiles(level)) { + input[0].files.push_back(f); + } + InternalKey refit_level_smallest; + InternalKey refit_level_largest; + cfd->compaction_picker()->GetRange(input[0], &refit_level_smallest, + &refit_level_largest); + if (to_level > level) { + if (level == 0) { + refitting_level_ = false; + return Status::NotSupported( + "Cannot change from level 0 to other levels."); + } + // Check levels are empty for a trivial move + for (int l = level + 1; l <= to_level; l++) { + if (vstorage->NumLevelFiles(l) > 0) { + refitting_level_ = false; + return Status::NotSupported( + "Levels between source and target are not empty for a move."); + } + if (cfd->RangeOverlapWithCompaction(refit_level_smallest.user_key(), + refit_level_largest.user_key(), + l)) { + refitting_level_ = false; + return Status::NotSupported( + "Levels between source and target " + "will have some ongoing compaction's output."); + } + } + } else { + // to_level < level + // Check levels are empty for a trivial move + for (int l = to_level; l < level; l++) { + if (vstorage->NumLevelFiles(l) > 0) { + refitting_level_ = false; + return Status::NotSupported( + "Levels between source and target are not empty for a move."); + } + if (cfd->RangeOverlapWithCompaction(refit_level_smallest.user_key(), + refit_level_largest.user_key(), + l)) { + refitting_level_ = false; + return Status::NotSupported( + "Levels between source and target " + "will have some ongoing compaction's output."); + } + } + } + ROCKS_LOG_DEBUG(immutable_db_options_.info_log, + "[%s] Before refitting:\n%s", cfd->GetName().c_str(), + cfd->current()->DebugString().data()); + + std::unique_ptr c(new Compaction( + vstorage, *cfd->ioptions(), mutable_cf_options, mutable_db_options_, + {input}, to_level, + MaxFileSizeForLevel( + mutable_cf_options, to_level, + cfd->ioptions() + ->compaction_style) /* output file size limit, not applicable */ + , + LLONG_MAX /* max compaction bytes, not applicable */, + 0 /* output path ID, not applicable */, mutable_cf_options.compression, + mutable_cf_options.compression_opts, Temperature::kUnknown, + 0 /* max_subcompactions, not applicable */, + {} /* grandparents, not applicable */, false /* is manual */, + "" /* trim_ts */, -1 /* score, not applicable */, + false /* is deletion compaction, not applicable */, + false /* l0_files_might_overlap, not applicable */, + CompactionReason::kRefitLevel)); + cfd->compaction_picker()->RegisterCompaction(c.get()); + TEST_SYNC_POINT("DBImpl::ReFitLevel:PostRegisterCompaction"); + VersionEdit edit; + edit.SetColumnFamily(cfd->GetID()); + + for (const auto& f : vstorage->LevelFiles(level)) { + edit.DeleteFile(level, f->fd.GetNumber()); + edit.AddFile( + to_level, f->fd.GetNumber(), f->fd.GetPathId(), f->fd.GetFileSize(), + f->smallest, f->largest, f->fd.smallest_seqno, f->fd.largest_seqno, + f->marked_for_compaction, f->temperature, f->oldest_blob_file_number, + f->oldest_ancester_time, f->file_creation_time, f->epoch_number, + f->file_checksum, f->file_checksum_func_name, f->unique_id, + f->compensated_range_deletion_size, f->tail_size, + f->user_defined_timestamps_persisted); + } + ROCKS_LOG_DEBUG(immutable_db_options_.info_log, + "[%s] Apply version edit:\n%s", cfd->GetName().c_str(), + edit.DebugString().data()); + + Status status = + versions_->LogAndApply(cfd, mutable_cf_options, read_options, &edit, + &mutex_, directories_.GetDbDir()); + + cfd->compaction_picker()->UnregisterCompaction(c.get()); + c.reset(); + + InstallSuperVersionAndScheduleWork(cfd, &sv_context, mutable_cf_options); + + ROCKS_LOG_DEBUG(immutable_db_options_.info_log, "[%s] LogAndApply: %s\n", + cfd->GetName().c_str(), status.ToString().data()); + + if (status.ok()) { + ROCKS_LOG_DEBUG(immutable_db_options_.info_log, + "[%s] After refitting:\n%s", cfd->GetName().c_str(), + cfd->current()->DebugString().data()); + } + sv_context.Clean(); + refitting_level_ = false; + + return status; + } + + refitting_level_ = false; + return Status::OK(); +} + +int DBImpl::NumberLevels(ColumnFamilyHandle* column_family) { + auto cfh = static_cast_with_check(column_family); + return cfh->cfd()->NumberLevels(); +} + +int DBImpl::MaxMemCompactionLevel(ColumnFamilyHandle* /*column_family*/) { + return 0; +} + +int DBImpl::Level0StopWriteTrigger(ColumnFamilyHandle* column_family) { + auto cfh = static_cast_with_check(column_family); + InstrumentedMutexLock l(&mutex_); + return cfh->cfd() + ->GetSuperVersion() + ->mutable_cf_options.level0_stop_writes_trigger; +} + +Status DBImpl::FlushAllColumnFamilies(const FlushOptions& flush_options, + FlushReason flush_reason) { + mutex_.AssertHeld(); + Status status; + if (immutable_db_options_.atomic_flush) { + mutex_.Unlock(); + status = AtomicFlushMemTables(flush_options, flush_reason); + if (status.IsColumnFamilyDropped()) { + status = Status::OK(); + } + mutex_.Lock(); + } else { + for (auto cfd : versions_->GetRefedColumnFamilySet()) { + if (cfd->IsDropped()) { + continue; + } + mutex_.Unlock(); + status = FlushMemTable(cfd, flush_options, flush_reason); + TEST_SYNC_POINT("DBImpl::FlushAllColumnFamilies:1"); + TEST_SYNC_POINT("DBImpl::FlushAllColumnFamilies:2"); + mutex_.Lock(); + if (!status.ok() && !status.IsColumnFamilyDropped()) { + break; + } else if (status.IsColumnFamilyDropped()) { + status = Status::OK(); + } + } + } + return status; +} + +Status DBImpl::Flush(const FlushOptions& flush_options, + ColumnFamilyHandle* column_family) { + auto cfh = static_cast_with_check(column_family); + ROCKS_LOG_INFO(immutable_db_options_.info_log, "[%s] Manual flush start.", + cfh->GetName().c_str()); + Status s; + if (immutable_db_options_.atomic_flush) { + s = AtomicFlushMemTables(flush_options, FlushReason::kManualFlush, + {cfh->cfd()}); + } else { + s = FlushMemTable(cfh->cfd(), flush_options, FlushReason::kManualFlush); + } + + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "[%s] Manual flush finished, status: %s\n", + cfh->GetName().c_str(), s.ToString().c_str()); + return s; +} + +Status DBImpl::Flush(const FlushOptions& flush_options, + const std::vector& column_families) { + Status s; + if (!immutable_db_options_.atomic_flush) { + for (auto cfh : column_families) { + s = Flush(flush_options, cfh); + if (!s.ok()) { + break; + } + } + } else { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "Manual atomic flush start.\n" + "=====Column families:====="); + for (auto cfh : column_families) { + auto cfhi = static_cast(cfh); + ROCKS_LOG_INFO(immutable_db_options_.info_log, "%s", + cfhi->GetName().c_str()); + } + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "=====End of column families list====="); + autovector cfds; + std::for_each(column_families.begin(), column_families.end(), + [&cfds](ColumnFamilyHandle* elem) { + auto cfh = static_cast(elem); + cfds.emplace_back(cfh->cfd()); + }); + s = AtomicFlushMemTables(flush_options, FlushReason::kManualFlush, cfds); + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "Manual atomic flush finished, status: %s\n" + "=====Column families:=====", + s.ToString().c_str()); + for (auto cfh : column_families) { + auto cfhi = static_cast(cfh); + ROCKS_LOG_INFO(immutable_db_options_.info_log, "%s", + cfhi->GetName().c_str()); + } + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "=====End of column families list====="); + } + return s; +} + +Status DBImpl::RunManualCompaction( + ColumnFamilyData* cfd, int input_level, int output_level, + const CompactRangeOptions& compact_range_options, const Slice* begin, + const Slice* end, bool exclusive, bool disallow_trivial_move, + uint64_t max_file_num_to_ignore, const std::string& trim_ts, + int* final_output_level) { + assert(input_level == ColumnFamilyData::kCompactAllLevels || + input_level >= 0); + + InternalKey begin_storage, end_storage; + CompactionArg* ca = nullptr; + + bool scheduled = false; + bool unscheduled = false; + Env::Priority thread_pool_priority = Env::Priority::TOTAL; + bool manual_conflict = false; + + ManualCompactionState manual( + cfd, input_level, output_level, compact_range_options.target_path_id, + exclusive, disallow_trivial_move, compact_range_options.canceled); + // For universal compaction, we enforce every manual compaction to compact + // all files. + if (begin == nullptr || + cfd->ioptions()->compaction_style == kCompactionStyleUniversal || + cfd->ioptions()->compaction_style == kCompactionStyleFIFO) { + manual.begin = nullptr; + } else { + begin_storage.SetMinPossibleForUserKey(*begin); + manual.begin = &begin_storage; + } + if (end == nullptr || + cfd->ioptions()->compaction_style == kCompactionStyleUniversal || + cfd->ioptions()->compaction_style == kCompactionStyleFIFO) { + manual.end = nullptr; + } else { + end_storage.SetMaxPossibleForUserKey(*end); + manual.end = &end_storage; + } + + TEST_SYNC_POINT("DBImpl::RunManualCompaction:0"); + TEST_SYNC_POINT("DBImpl::RunManualCompaction:1"); + InstrumentedMutexLock l(&mutex_); + + if (manual_compaction_paused_ > 0) { + // Does not make sense to `AddManualCompaction()` in this scenario since + // `DisableManualCompaction()` just waited for the manual compaction queue + // to drain. So return immediately. + TEST_SYNC_POINT("DBImpl::RunManualCompaction:PausedAtStart"); + manual.status = + Status::Incomplete(Status::SubCode::kManualCompactionPaused); + manual.done = true; + return manual.status; + } + + // When a manual compaction arrives, temporarily disable scheduling of + // non-manual compactions and wait until the number of scheduled compaction + // jobs drops to zero. This used to be needed to ensure that this manual + // compaction can compact any range of keys/files. Now it is optional + // (see `CompactRangeOptions::exclusive_manual_compaction`). The use case for + // `exclusive_manual_compaction=true` is unclear beyond not trusting the code. + // + // HasPendingManualCompaction() is true when at least one thread is inside + // RunManualCompaction(), i.e. during that time no other compaction will + // get scheduled (see MaybeScheduleFlushOrCompaction). + // + // Note that the following loop doesn't stop more that one thread calling + // RunManualCompaction() from getting to the second while loop below. + // However, only one of them will actually schedule compaction, while + // others will wait on a condition variable until it completes. + + AddManualCompaction(&manual); + TEST_SYNC_POINT_CALLBACK("DBImpl::RunManualCompaction:NotScheduled", &mutex_); + if (exclusive) { + // Limitation: there's no way to wake up the below loop when user sets + // `*manual.canceled`. So `CompactRangeOptions::exclusive_manual_compaction` + // and `CompactRangeOptions::canceled` might not work well together. + while (bg_bottom_compaction_scheduled_ > 0 || + bg_compaction_scheduled_ > 0) { + if (manual_compaction_paused_ > 0 || manual.canceled == true) { + // Pretend the error came from compaction so the below cleanup/error + // handling code can process it. + manual.done = true; + manual.status = + Status::Incomplete(Status::SubCode::kManualCompactionPaused); + break; + } + TEST_SYNC_POINT("DBImpl::RunManualCompaction:WaitScheduled"); + ROCKS_LOG_INFO( + immutable_db_options_.info_log, + "[%s] Manual compaction waiting for all other scheduled background " + "compactions to finish", + cfd->GetName().c_str()); + bg_cv_.Wait(); + } + } + + LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, + immutable_db_options_.info_log.get()); + + ROCKS_LOG_BUFFER(&log_buffer, "[%s] Manual compaction starting", + cfd->GetName().c_str()); + + // We don't check bg_error_ here, because if we get the error in compaction, + // the compaction will set manual.status to bg_error_ and set manual.done to + // true. + while (!manual.done) { + assert(HasPendingManualCompaction()); + manual_conflict = false; + Compaction* compaction = nullptr; + if (ShouldntRunManualCompaction(&manual) || (manual.in_progress == true) || + scheduled || + (((manual.manual_end = &manual.tmp_storage1) != nullptr) && + ((compaction = manual.cfd->CompactRange( + *manual.cfd->GetLatestMutableCFOptions(), mutable_db_options_, + manual.input_level, manual.output_level, compact_range_options, + manual.begin, manual.end, &manual.manual_end, &manual_conflict, + max_file_num_to_ignore, trim_ts)) == nullptr && + manual_conflict))) { + if (!scheduled) { + // There is a conflicting compaction + if (manual_compaction_paused_ > 0 || manual.canceled == true) { + // Stop waiting since it was canceled. Pretend the error came from + // compaction so the below cleanup/error handling code can process it. + manual.done = true; + manual.status = + Status::Incomplete(Status::SubCode::kManualCompactionPaused); + } + } + if (!manual.done) { + bg_cv_.Wait(); + } + if (manual_compaction_paused_ > 0 && scheduled && !unscheduled) { + assert(thread_pool_priority != Env::Priority::TOTAL); + // unschedule all manual compactions + auto unscheduled_task_num = env_->UnSchedule( + GetTaskTag(TaskType::kManualCompaction), thread_pool_priority); + if (unscheduled_task_num > 0) { + ROCKS_LOG_INFO( + immutable_db_options_.info_log, + "[%s] Unscheduled %d number of manual compactions from the " + "thread-pool", + cfd->GetName().c_str(), unscheduled_task_num); + // it may unschedule other manual compactions, notify others. + bg_cv_.SignalAll(); + } + unscheduled = true; + TEST_SYNC_POINT("DBImpl::RunManualCompaction:Unscheduled"); + } + if (scheduled && manual.incomplete == true) { + assert(!manual.in_progress); + scheduled = false; + manual.incomplete = false; + } + } else if (!scheduled) { + if (compaction == nullptr) { + manual.done = true; + if (final_output_level) { + // No compaction needed or there is a conflicting compaction. + // Still set `final_output_level` to the level where we would + // have compacted to. + *final_output_level = output_level; + if (output_level == ColumnFamilyData::kCompactToBaseLevel) { + *final_output_level = cfd->current()->storage_info()->base_level(); + } + } + bg_cv_.SignalAll(); + continue; + } + ca = new CompactionArg; + ca->db = this; + ca->prepicked_compaction = new PrepickedCompaction; + ca->prepicked_compaction->manual_compaction_state = &manual; + ca->prepicked_compaction->compaction = compaction; + if (!RequestCompactionToken( + cfd, true, &ca->prepicked_compaction->task_token, &log_buffer)) { + // Don't throttle manual compaction, only count outstanding tasks. + assert(false); + } + manual.incomplete = false; + if (compaction->bottommost_level() && + env_->GetBackgroundThreads(Env::Priority::BOTTOM) > 0) { + bg_bottom_compaction_scheduled_++; + ca->compaction_pri_ = Env::Priority::BOTTOM; + env_->Schedule(&DBImpl::BGWorkBottomCompaction, ca, + Env::Priority::BOTTOM, + GetTaskTag(TaskType::kManualCompaction), + &DBImpl::UnscheduleCompactionCallback); + thread_pool_priority = Env::Priority::BOTTOM; + } else { + bg_compaction_scheduled_++; + ca->compaction_pri_ = Env::Priority::LOW; + env_->Schedule(&DBImpl::BGWorkCompaction, ca, Env::Priority::LOW, + GetTaskTag(TaskType::kManualCompaction), + &DBImpl::UnscheduleCompactionCallback); + thread_pool_priority = Env::Priority::LOW; + } + scheduled = true; + TEST_SYNC_POINT("DBImpl::RunManualCompaction:Scheduled"); + if (final_output_level) { + *final_output_level = compaction->output_level(); + } + } + } + + log_buffer.FlushBufferToLog(); + assert(!manual.in_progress); + assert(HasPendingManualCompaction()); + RemoveManualCompaction(&manual); + // if the manual job is unscheduled, try schedule other jobs in case there's + // any unscheduled compaction job which was blocked by exclusive manual + // compaction. + if (manual.status.IsIncomplete() && + manual.status.subcode() == Status::SubCode::kManualCompactionPaused) { + MaybeScheduleFlushOrCompaction(); + } + bg_cv_.SignalAll(); + return manual.status; +} + +void DBImpl::GenerateFlushRequest(const autovector& cfds, + FlushReason flush_reason, FlushRequest* req) { + assert(req != nullptr); + req->flush_reason = flush_reason; + req->cfd_to_max_mem_id_to_persist.reserve(cfds.size()); + for (const auto cfd : cfds) { + if (nullptr == cfd) { + // cfd may be null, see DBImpl::ScheduleFlushes + continue; + } + uint64_t max_memtable_id = cfd->imm()->GetLatestMemTableID(); + req->cfd_to_max_mem_id_to_persist.emplace(cfd, max_memtable_id); + } +} + +Status DBImpl::FlushMemTable(ColumnFamilyData* cfd, + const FlushOptions& flush_options, + FlushReason flush_reason, + bool entered_write_thread) { + // This method should not be called if atomic_flush is true. + assert(!immutable_db_options_.atomic_flush); + if (!flush_options.wait && write_controller_.IsStopped()) { + std::ostringstream oss; + oss << "Writes have been stopped, thus unable to perform manual flush. " + "Please try again later after writes are resumed"; + return Status::TryAgain(oss.str()); + } + Status s; + if (!flush_options.allow_write_stall) { + bool flush_needed = true; + s = WaitUntilFlushWouldNotStallWrites(cfd, &flush_needed); + TEST_SYNC_POINT("DBImpl::FlushMemTable:StallWaitDone"); + if (!s.ok() || !flush_needed) { + return s; + } + } + + const bool needs_to_join_write_thread = !entered_write_thread; + autovector flush_reqs; + autovector memtable_ids_to_wait; + { + WriteContext context; + InstrumentedMutexLock guard_lock(&mutex_); + + WriteThread::Writer w; + WriteThread::Writer nonmem_w; + if (needs_to_join_write_thread) { + write_thread_.EnterUnbatched(&w, &mutex_); + if (two_write_queues_) { + nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_); + } + } + WaitForPendingWrites(); + + if (flush_reason != FlushReason::kErrorRecoveryRetryFlush && + (!cfd->mem()->IsEmpty() || !cached_recoverable_state_empty_.load())) { + // Note that, when flush reason is kErrorRecoveryRetryFlush, during the + // auto retry resume, we want to avoid creating new small memtables. + // Therefore, SwitchMemtable will not be called. Also, since ResumeImpl + // will iterate through all the CFs and call FlushMemtable during auto + // retry resume, it is possible that in some CFs, + // cfd->imm()->NumNotFlushed() = 0. In this case, so no flush request will + // be created and scheduled, status::OK() will be returned. + s = SwitchMemtable(cfd, &context); + } + const uint64_t flush_memtable_id = std::numeric_limits::max(); + if (s.ok()) { + if (cfd->imm()->NumNotFlushed() != 0 || !cfd->mem()->IsEmpty() || + !cached_recoverable_state_empty_.load()) { + FlushRequest req{flush_reason, {{cfd, flush_memtable_id}}}; + flush_reqs.emplace_back(std::move(req)); + memtable_ids_to_wait.emplace_back(cfd->imm()->GetLatestMemTableID()); + } + if (immutable_db_options_.persist_stats_to_disk && + flush_reason != FlushReason::kErrorRecoveryRetryFlush) { + ColumnFamilyData* cfd_stats = + versions_->GetColumnFamilySet()->GetColumnFamily( + kPersistentStatsColumnFamilyName); + if (cfd_stats != nullptr && cfd_stats != cfd && + !cfd_stats->mem()->IsEmpty()) { + // only force flush stats CF when it will be the only CF lagging + // behind after the current flush + bool stats_cf_flush_needed = true; + for (auto* loop_cfd : *versions_->GetColumnFamilySet()) { + if (loop_cfd == cfd_stats || loop_cfd == cfd) { + continue; + } + if (loop_cfd->GetLogNumber() <= cfd_stats->GetLogNumber()) { + stats_cf_flush_needed = false; + } + } + if (stats_cf_flush_needed) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "Force flushing stats CF with manual flush of %s " + "to avoid holding old logs", + cfd->GetName().c_str()); + s = SwitchMemtable(cfd_stats, &context); + FlushRequest req{flush_reason, {{cfd_stats, flush_memtable_id}}}; + flush_reqs.emplace_back(std::move(req)); + memtable_ids_to_wait.emplace_back( + cfd_stats->imm()->GetLatestMemTableID()); + } + } + } + } + + if (s.ok() && !flush_reqs.empty()) { + for (const auto& req : flush_reqs) { + assert(req.cfd_to_max_mem_id_to_persist.size() == 1); + ColumnFamilyData* loop_cfd = + req.cfd_to_max_mem_id_to_persist.begin()->first; + loop_cfd->imm()->FlushRequested(); + } + // If the caller wants to wait for this flush to complete, it indicates + // that the caller expects the ColumnFamilyData not to be free'ed by + // other threads which may drop the column family concurrently. + // Therefore, we increase the cfd's ref count. + if (flush_options.wait) { + for (const auto& req : flush_reqs) { + assert(req.cfd_to_max_mem_id_to_persist.size() == 1); + ColumnFamilyData* loop_cfd = + req.cfd_to_max_mem_id_to_persist.begin()->first; + loop_cfd->Ref(); + } + } + for (const auto& req : flush_reqs) { + SchedulePendingFlush(req); + } + MaybeScheduleFlushOrCompaction(); + } + + if (needs_to_join_write_thread) { + write_thread_.ExitUnbatched(&w); + if (two_write_queues_) { + nonmem_write_thread_.ExitUnbatched(&nonmem_w); + } + } + } + TEST_SYNC_POINT("DBImpl::FlushMemTable:AfterScheduleFlush"); + TEST_SYNC_POINT("DBImpl::FlushMemTable:BeforeWaitForBgFlush"); + if (s.ok() && flush_options.wait) { + autovector cfds; + autovector flush_memtable_ids; + assert(flush_reqs.size() == memtable_ids_to_wait.size()); + for (size_t i = 0; i < flush_reqs.size(); ++i) { + assert(flush_reqs[i].cfd_to_max_mem_id_to_persist.size() == 1); + cfds.push_back(flush_reqs[i].cfd_to_max_mem_id_to_persist.begin()->first); + flush_memtable_ids.push_back(&(memtable_ids_to_wait[i])); + } + s = WaitForFlushMemTables( + cfds, flush_memtable_ids, + (flush_reason == FlushReason::kErrorRecovery || + flush_reason == FlushReason::kErrorRecoveryRetryFlush)); + InstrumentedMutexLock lock_guard(&mutex_); + for (auto* tmp_cfd : cfds) { + tmp_cfd->UnrefAndTryDelete(); + } + } + TEST_SYNC_POINT("DBImpl::FlushMemTable:FlushMemTableFinished"); + return s; +} + +Status DBImpl::AtomicFlushMemTables( + const FlushOptions& flush_options, FlushReason flush_reason, + const autovector& provided_candidate_cfds, + bool entered_write_thread) { + assert(immutable_db_options_.atomic_flush); + if (!flush_options.wait && write_controller_.IsStopped()) { + std::ostringstream oss; + oss << "Writes have been stopped, thus unable to perform manual flush. " + "Please try again later after writes are resumed"; + return Status::TryAgain(oss.str()); + } + Status s; + autovector candidate_cfds; + if (provided_candidate_cfds.empty()) { + // Generate candidate cfds if not provided + { + InstrumentedMutexLock l(&mutex_); + for (ColumnFamilyData* cfd : *versions_->GetColumnFamilySet()) { + if (!cfd->IsDropped() && cfd->initialized()) { + cfd->Ref(); + candidate_cfds.push_back(cfd); + } + } + } + } else { + candidate_cfds = provided_candidate_cfds; + } + + if (!flush_options.allow_write_stall) { + int num_cfs_to_flush = 0; + for (auto cfd : candidate_cfds) { + bool flush_needed = true; + s = WaitUntilFlushWouldNotStallWrites(cfd, &flush_needed); + if (!s.ok()) { + // Unref the newly generated candidate cfds (when not provided) in + // `candidate_cfds` + if (provided_candidate_cfds.empty()) { + for (auto candidate_cfd : candidate_cfds) { + candidate_cfd->UnrefAndTryDelete(); + } + } + return s; + } else if (flush_needed) { + ++num_cfs_to_flush; + } + } + if (0 == num_cfs_to_flush) { + // Unref the newly generated candidate cfds (when not provided) in + // `candidate_cfds` + if (provided_candidate_cfds.empty()) { + for (auto candidate_cfd : candidate_cfds) { + candidate_cfd->UnrefAndTryDelete(); + } + } + return s; + } + } + const bool needs_to_join_write_thread = !entered_write_thread; + FlushRequest flush_req; + autovector cfds; + { + WriteContext context; + InstrumentedMutexLock guard_lock(&mutex_); + + WriteThread::Writer w; + WriteThread::Writer nonmem_w; + if (needs_to_join_write_thread) { + write_thread_.EnterUnbatched(&w, &mutex_); + if (two_write_queues_) { + nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_); + } + } + WaitForPendingWrites(); + + SelectColumnFamiliesForAtomicFlush(&cfds, candidate_cfds); + + // Unref the newly generated candidate cfds (when not provided) in + // `candidate_cfds` + if (provided_candidate_cfds.empty()) { + for (auto candidate_cfd : candidate_cfds) { + candidate_cfd->UnrefAndTryDelete(); + } + } + + for (auto cfd : cfds) { + if ((cfd->mem()->IsEmpty() && cached_recoverable_state_empty_.load()) || + flush_reason == FlushReason::kErrorRecoveryRetryFlush) { + continue; + } + cfd->Ref(); + s = SwitchMemtable(cfd, &context); + cfd->UnrefAndTryDelete(); + if (!s.ok()) { + break; + } + } + if (s.ok()) { + AssignAtomicFlushSeq(cfds); + for (auto cfd : cfds) { + cfd->imm()->FlushRequested(); + } + // If the caller wants to wait for this flush to complete, it indicates + // that the caller expects the ColumnFamilyData not to be free'ed by + // other threads which may drop the column family concurrently. + // Therefore, we increase the cfd's ref count. + if (flush_options.wait) { + for (auto cfd : cfds) { + cfd->Ref(); + } + } + GenerateFlushRequest(cfds, flush_reason, &flush_req); + SchedulePendingFlush(flush_req); + MaybeScheduleFlushOrCompaction(); + } + + if (needs_to_join_write_thread) { + write_thread_.ExitUnbatched(&w); + if (two_write_queues_) { + nonmem_write_thread_.ExitUnbatched(&nonmem_w); + } + } + } + TEST_SYNC_POINT("DBImpl::AtomicFlushMemTables:AfterScheduleFlush"); + TEST_SYNC_POINT("DBImpl::AtomicFlushMemTables:BeforeWaitForBgFlush"); + if (s.ok() && flush_options.wait) { + autovector flush_memtable_ids; + for (auto& iter : flush_req.cfd_to_max_mem_id_to_persist) { + flush_memtable_ids.push_back(&(iter.second)); + } + s = WaitForFlushMemTables( + cfds, flush_memtable_ids, + (flush_reason == FlushReason::kErrorRecovery || + flush_reason == FlushReason::kErrorRecoveryRetryFlush)); + InstrumentedMutexLock lock_guard(&mutex_); + for (auto* cfd : cfds) { + cfd->UnrefAndTryDelete(); + } + } + return s; +} + +// Calling FlushMemTable(), whether from DB::Flush() or from Backup Engine, can +// cause write stall, for example if one memtable is being flushed already. +// This method tries to avoid write stall (similar to CompactRange() behavior) +// it emulates how the SuperVersion / LSM would change if flush happens, checks +// it against various constrains and delays flush if it'd cause write stall. +// Caller should check status and flush_needed to see if flush already happened. +Status DBImpl::WaitUntilFlushWouldNotStallWrites(ColumnFamilyData* cfd, + bool* flush_needed) { + { + *flush_needed = true; + InstrumentedMutexLock l(&mutex_); + uint64_t orig_active_memtable_id = cfd->mem()->GetID(); + WriteStallCondition write_stall_condition = WriteStallCondition::kNormal; + do { + if (write_stall_condition != WriteStallCondition::kNormal) { + // Same error handling as user writes: Don't wait if there's a + // background error, even if it's a soft error. We might wait here + // indefinitely as the pending flushes/compactions may never finish + // successfully, resulting in the stall condition lasting indefinitely + if (error_handler_.IsBGWorkStopped()) { + return error_handler_.GetBGError(); + } + + TEST_SYNC_POINT("DBImpl::WaitUntilFlushWouldNotStallWrites:StallWait"); + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "[%s] WaitUntilFlushWouldNotStallWrites" + " waiting on stall conditions to clear", + cfd->GetName().c_str()); + bg_cv_.Wait(); + } + if (cfd->IsDropped()) { + return Status::ColumnFamilyDropped(); + } + if (shutting_down_.load(std::memory_order_acquire)) { + return Status::ShutdownInProgress(); + } + + uint64_t earliest_memtable_id = + std::min(cfd->mem()->GetID(), cfd->imm()->GetEarliestMemTableID()); + if (earliest_memtable_id > orig_active_memtable_id) { + // We waited so long that the memtable we were originally waiting on was + // flushed. + *flush_needed = false; + return Status::OK(); + } + + const auto& mutable_cf_options = *cfd->GetLatestMutableCFOptions(); + const auto* vstorage = cfd->current()->storage_info(); + + // Skip stalling check if we're below auto-flush and auto-compaction + // triggers. If it stalled in these conditions, that'd mean the stall + // triggers are so low that stalling is needed for any background work. In + // that case we shouldn't wait since background work won't be scheduled. + if (cfd->imm()->NumNotFlushed() < + cfd->ioptions()->min_write_buffer_number_to_merge && + vstorage->l0_delay_trigger_count() < + mutable_cf_options.level0_file_num_compaction_trigger) { + break; + } + + // check whether one extra immutable memtable or an extra L0 file would + // cause write stalling mode to be entered. It could still enter stall + // mode due to pending compaction bytes, but that's less common + // No extra immutable Memtable will be created if the current Memtable is + // empty. + int mem_to_flush = cfd->mem()->IsEmpty() ? 0 : 1; + write_stall_condition = ColumnFamilyData::GetWriteStallConditionAndCause( + cfd->imm()->NumNotFlushed() + mem_to_flush, + vstorage->l0_delay_trigger_count() + 1, + vstorage->estimated_compaction_needed_bytes(), + mutable_cf_options, *cfd->ioptions()) + .first; + } while (write_stall_condition != WriteStallCondition::kNormal); + } + return Status::OK(); +} + +// Wait for memtables to be flushed for multiple column families. +// let N = cfds.size() +// for i in [0, N), +// 1) if flush_memtable_ids[i] is not null, then the memtables with lower IDs +// have to be flushed for THIS column family; +// 2) if flush_memtable_ids[i] is null, then all memtables in THIS column +// family have to be flushed. +// Finish waiting when ALL column families finish flushing memtables. +// resuming_from_bg_err indicates whether the caller is trying to resume from +// background error or in normal processing. +Status DBImpl::WaitForFlushMemTables( + const autovector& cfds, + const autovector& flush_memtable_ids, + bool resuming_from_bg_err) { + int num = static_cast(cfds.size()); + // Wait until the compaction completes + InstrumentedMutexLock l(&mutex_); + Status s; + // If the caller is trying to resume from bg error, then + // error_handler_.IsDBStopped() is true. + while (resuming_from_bg_err || !error_handler_.IsDBStopped()) { + if (shutting_down_.load(std::memory_order_acquire)) { + s = Status::ShutdownInProgress(); + return s; + } + // If an error has occurred during resumption, then no need to wait. + // But flush operation may fail because of this error, so need to + // return the status. + if (!error_handler_.GetRecoveryError().ok()) { + s = error_handler_.GetRecoveryError(); + break; + } + // If BGWorkStopped, which indicate that there is a BG error and + // 1) soft error but requires no BG work, 2) no in auto_recovery_ + if (!resuming_from_bg_err && error_handler_.IsBGWorkStopped() && + error_handler_.GetBGError().severity() < Status::Severity::kHardError) { + s = error_handler_.GetBGError(); + return s; + } + + // Number of column families that have been dropped. + int num_dropped = 0; + // Number of column families that have finished flush. + int num_finished = 0; + for (int i = 0; i < num; ++i) { + if (cfds[i]->IsDropped()) { + ++num_dropped; + } else if (cfds[i]->imm()->NumNotFlushed() == 0 || + (flush_memtable_ids[i] != nullptr && + cfds[i]->imm()->GetEarliestMemTableID() > + *flush_memtable_ids[i])) { + ++num_finished; + } + } + if (1 == num_dropped && 1 == num) { + s = Status::ColumnFamilyDropped(); + return s; + } + // Column families involved in this flush request have either been dropped + // or finished flush. Then it's time to finish waiting. + if (num_dropped + num_finished == num) { + break; + } + bg_cv_.Wait(); + } + // If not resuming from bg error, and an error has caused the DB to stop, + // then report the bg error to caller. + if (!resuming_from_bg_err && error_handler_.IsDBStopped()) { + s = error_handler_.GetBGError(); + } + return s; +} + +Status DBImpl::EnableAutoCompaction( + const std::vector& column_family_handles) { + Status s; + for (auto cf_ptr : column_family_handles) { + Status status = + this->SetOptions(cf_ptr, {{"disable_auto_compactions", "false"}}); + if (!status.ok()) { + s = status; + } + } + + return s; +} + +// NOTE: Calling DisableManualCompaction() may overwrite the +// user-provided canceled variable in CompactRangeOptions +void DBImpl::DisableManualCompaction() { + InstrumentedMutexLock l(&mutex_); + manual_compaction_paused_.fetch_add(1, std::memory_order_release); + + // Mark the canceled as true when the cancellation is triggered by + // manual_compaction_paused (may overwrite user-provided `canceled`) + for (const auto& manual_compaction : manual_compaction_dequeue_) { + manual_compaction->canceled = true; + } + + // Wake up manual compactions waiting to start. + bg_cv_.SignalAll(); + + // Wait for any pending manual compactions to finish (typically through + // failing with `Status::Incomplete`) prior to returning. This way we are + // guaranteed no pending manual compaction will commit while manual + // compactions are "disabled". + while (HasPendingManualCompaction()) { + bg_cv_.Wait(); + } +} + +// NOTE: In contrast to DisableManualCompaction(), calling +// EnableManualCompaction() does NOT overwrite the user-provided *canceled +// variable to be false since there is NO CHANCE a canceled compaction +// is uncanceled. In other words, a canceled compaction must have been +// dropped out of the manual compaction queue, when we disable it. +void DBImpl::EnableManualCompaction() { + InstrumentedMutexLock l(&mutex_); + assert(manual_compaction_paused_ > 0); + manual_compaction_paused_.fetch_sub(1, std::memory_order_release); +} + +void DBImpl::MaybeScheduleFlushOrCompaction() { + mutex_.AssertHeld(); + if (!opened_successfully_) { + // Compaction may introduce data race to DB open + return; + } + if (bg_work_paused_ > 0) { + // we paused the background work + return; + } else if (error_handler_.IsBGWorkStopped() && + !error_handler_.IsRecoveryInProgress()) { + // There has been a hard error and this call is not part of the recovery + // sequence. Bail out here so we don't get into an endless loop of + // scheduling BG work which will again call this function + return; + } else if (shutting_down_.load(std::memory_order_acquire)) { + // DB is being deleted; no more background compactions + return; + } + auto bg_job_limits = GetBGJobLimits(); + bool is_flush_pool_empty = + env_->GetBackgroundThreads(Env::Priority::HIGH) == 0; + while (!is_flush_pool_empty && unscheduled_flushes_ > 0 && + bg_flush_scheduled_ < bg_job_limits.max_flushes) { + TEST_SYNC_POINT_CALLBACK( + "DBImpl::MaybeScheduleFlushOrCompaction:BeforeSchedule", + &unscheduled_flushes_); + bg_flush_scheduled_++; + FlushThreadArg* fta = new FlushThreadArg; + fta->db_ = this; + fta->thread_pri_ = Env::Priority::HIGH; + env_->Schedule(&DBImpl::BGWorkFlush, fta, Env::Priority::HIGH, this, + &DBImpl::UnscheduleFlushCallback); + --unscheduled_flushes_; + TEST_SYNC_POINT_CALLBACK( + "DBImpl::MaybeScheduleFlushOrCompaction:AfterSchedule:0", + &unscheduled_flushes_); + } + + // special case -- if high-pri (flush) thread pool is empty, then schedule + // flushes in low-pri (compaction) thread pool. + if (is_flush_pool_empty) { + while (unscheduled_flushes_ > 0 && + bg_flush_scheduled_ + bg_compaction_scheduled_ < + bg_job_limits.max_flushes) { + bg_flush_scheduled_++; + FlushThreadArg* fta = new FlushThreadArg; + fta->db_ = this; + fta->thread_pri_ = Env::Priority::LOW; + env_->Schedule(&DBImpl::BGWorkFlush, fta, Env::Priority::LOW, this, + &DBImpl::UnscheduleFlushCallback); + --unscheduled_flushes_; + } + } + + if (bg_compaction_paused_ > 0) { + // we paused the background compaction + return; + } else if (error_handler_.IsBGWorkStopped()) { + // Compaction is not part of the recovery sequence from a hard error. We + // might get here because recovery might do a flush and install a new + // super version, which will try to schedule pending compactions. Bail + // out here and let the higher level recovery handle compactions + return; + } + + if (HasExclusiveManualCompaction()) { + // only manual compactions are allowed to run. don't schedule automatic + // compactions + TEST_SYNC_POINT("DBImpl::MaybeScheduleFlushOrCompaction:Conflict"); + return; + } + + while (bg_compaction_scheduled_ + bg_bottom_compaction_scheduled_ < + bg_job_limits.max_compactions && + unscheduled_compactions_ > 0) { + CompactionArg* ca = new CompactionArg; + ca->db = this; + ca->compaction_pri_ = Env::Priority::LOW; + ca->prepicked_compaction = nullptr; + bg_compaction_scheduled_++; + unscheduled_compactions_--; + env_->Schedule(&DBImpl::BGWorkCompaction, ca, Env::Priority::LOW, this, + &DBImpl::UnscheduleCompactionCallback); + } +} + +DBImpl::BGJobLimits DBImpl::GetBGJobLimits() const { + mutex_.AssertHeld(); + return GetBGJobLimits(mutable_db_options_.max_background_flushes, + mutable_db_options_.max_background_compactions, + mutable_db_options_.max_background_jobs, + write_controller_.NeedSpeedupCompaction()); +} + +DBImpl::BGJobLimits DBImpl::GetBGJobLimits(int max_background_flushes, + int max_background_compactions, + int max_background_jobs, + bool parallelize_compactions) { + BGJobLimits res; + if (max_background_flushes == -1 && max_background_compactions == -1) { + // for our first stab implementing max_background_jobs, simply allocate a + // quarter of the threads to flushes. + res.max_flushes = std::max(1, max_background_jobs / 4); + res.max_compactions = std::max(1, max_background_jobs - res.max_flushes); + } else { + // compatibility code in case users haven't migrated to max_background_jobs, + // which automatically computes flush/compaction limits + res.max_flushes = std::max(1, max_background_flushes); + res.max_compactions = std::max(1, max_background_compactions); + } + if (!parallelize_compactions) { + // throttle background compactions until we deem necessary + res.max_compactions = 1; + } + return res; +} + +void DBImpl::AddToCompactionQueue(ColumnFamilyData* cfd) { + assert(!cfd->queued_for_compaction()); + cfd->Ref(); + compaction_queue_.push_back(cfd); + cfd->set_queued_for_compaction(true); +} + +ColumnFamilyData* DBImpl::PopFirstFromCompactionQueue() { + assert(!compaction_queue_.empty()); + auto cfd = *compaction_queue_.begin(); + compaction_queue_.pop_front(); + assert(cfd->queued_for_compaction()); + cfd->set_queued_for_compaction(false); + return cfd; +} + +DBImpl::FlushRequest DBImpl::PopFirstFromFlushQueue() { + assert(!flush_queue_.empty()); + FlushRequest flush_req = std::move(flush_queue_.front()); + flush_queue_.pop_front(); + if (!immutable_db_options_.atomic_flush) { + assert(flush_req.cfd_to_max_mem_id_to_persist.size() == 1); + } + for (const auto& elem : flush_req.cfd_to_max_mem_id_to_persist) { + if (!immutable_db_options_.atomic_flush) { + ColumnFamilyData* cfd = elem.first; + assert(cfd); + assert(cfd->queued_for_flush()); + cfd->set_queued_for_flush(false); + } + } + return flush_req; +} + +ColumnFamilyData* DBImpl::PickCompactionFromQueue( + std::unique_ptr* token, LogBuffer* log_buffer) { + assert(!compaction_queue_.empty()); + assert(*token == nullptr); + autovector throttled_candidates; + ColumnFamilyData* cfd = nullptr; + while (!compaction_queue_.empty()) { + auto first_cfd = *compaction_queue_.begin(); + compaction_queue_.pop_front(); + assert(first_cfd->queued_for_compaction()); + if (!RequestCompactionToken(first_cfd, false, token, log_buffer)) { + throttled_candidates.push_back(first_cfd); + continue; + } + cfd = first_cfd; + cfd->set_queued_for_compaction(false); + break; + } + // Add throttled compaction candidates back to queue in the original order. + for (auto iter = throttled_candidates.rbegin(); + iter != throttled_candidates.rend(); ++iter) { + compaction_queue_.push_front(*iter); + } + return cfd; +} + +void DBImpl::SchedulePendingFlush(const FlushRequest& flush_req) { + mutex_.AssertHeld(); + if (flush_req.cfd_to_max_mem_id_to_persist.empty()) { + return; + } + if (!immutable_db_options_.atomic_flush) { + // For the non-atomic flush case, we never schedule multiple column + // families in the same flush request. + assert(flush_req.cfd_to_max_mem_id_to_persist.size() == 1); + ColumnFamilyData* cfd = + flush_req.cfd_to_max_mem_id_to_persist.begin()->first; + assert(cfd); + + if (!cfd->queued_for_flush() && cfd->imm()->IsFlushPending()) { + cfd->Ref(); + cfd->set_queued_for_flush(true); + ++unscheduled_flushes_; + flush_queue_.push_back(flush_req); + } + } else { + for (auto& iter : flush_req.cfd_to_max_mem_id_to_persist) { + ColumnFamilyData* cfd = iter.first; + cfd->Ref(); + } + ++unscheduled_flushes_; + flush_queue_.push_back(flush_req); + } +} + +void DBImpl::SchedulePendingCompaction(ColumnFamilyData* cfd) { + mutex_.AssertHeld(); + if (!cfd->queued_for_compaction() && cfd->NeedsCompaction()) { + AddToCompactionQueue(cfd); + ++unscheduled_compactions_; + } +} + +void DBImpl::SchedulePendingPurge(std::string fname, std::string dir_to_sync, + FileType type, uint64_t number, int job_id) { + mutex_.AssertHeld(); + PurgeFileInfo file_info(fname, dir_to_sync, type, number, job_id); + purge_files_.insert({{number, std::move(file_info)}}); +} + +void DBImpl::BGWorkFlush(void* arg) { + FlushThreadArg fta = *(reinterpret_cast(arg)); + delete reinterpret_cast(arg); + + IOSTATS_SET_THREAD_POOL_ID(fta.thread_pri_); + TEST_SYNC_POINT("DBImpl::BGWorkFlush"); + static_cast_with_check(fta.db_)->BackgroundCallFlush(fta.thread_pri_); + TEST_SYNC_POINT("DBImpl::BGWorkFlush:done"); +} + +void DBImpl::BGWorkCompaction(void* arg) { + CompactionArg ca = *(reinterpret_cast(arg)); + delete reinterpret_cast(arg); + IOSTATS_SET_THREAD_POOL_ID(Env::Priority::LOW); + TEST_SYNC_POINT("DBImpl::BGWorkCompaction"); + auto prepicked_compaction = + static_cast(ca.prepicked_compaction); + static_cast_with_check(ca.db)->BackgroundCallCompaction( + prepicked_compaction, Env::Priority::LOW); + delete prepicked_compaction; +} + +void DBImpl::BGWorkBottomCompaction(void* arg) { + CompactionArg ca = *(static_cast(arg)); + delete static_cast(arg); + IOSTATS_SET_THREAD_POOL_ID(Env::Priority::BOTTOM); + TEST_SYNC_POINT("DBImpl::BGWorkBottomCompaction"); + auto* prepicked_compaction = ca.prepicked_compaction; + assert(prepicked_compaction && prepicked_compaction->compaction); + ca.db->BackgroundCallCompaction(prepicked_compaction, Env::Priority::BOTTOM); + delete prepicked_compaction; +} + +void DBImpl::BGWorkPurge(void* db) { + IOSTATS_SET_THREAD_POOL_ID(Env::Priority::HIGH); + TEST_SYNC_POINT("DBImpl::BGWorkPurge:start"); + reinterpret_cast(db)->BackgroundCallPurge(); + TEST_SYNC_POINT("DBImpl::BGWorkPurge:end"); +} + +void DBImpl::UnscheduleCompactionCallback(void* arg) { + CompactionArg* ca_ptr = reinterpret_cast(arg); + Env::Priority compaction_pri = ca_ptr->compaction_pri_; + if (Env::Priority::BOTTOM == compaction_pri) { + // Decrement bg_bottom_compaction_scheduled_ if priority is BOTTOM + ca_ptr->db->bg_bottom_compaction_scheduled_--; + } else if (Env::Priority::LOW == compaction_pri) { + // Decrement bg_compaction_scheduled_ if priority is LOW + ca_ptr->db->bg_compaction_scheduled_--; + } + CompactionArg ca = *(ca_ptr); + delete reinterpret_cast(arg); + if (ca.prepicked_compaction != nullptr) { + // if it's a manual compaction, set status to ManualCompactionPaused + if (ca.prepicked_compaction->manual_compaction_state) { + ca.prepicked_compaction->manual_compaction_state->done = true; + ca.prepicked_compaction->manual_compaction_state->status = + Status::Incomplete(Status::SubCode::kManualCompactionPaused); + } + if (ca.prepicked_compaction->compaction != nullptr) { + ca.prepicked_compaction->compaction->ReleaseCompactionFiles( + Status::Incomplete(Status::SubCode::kManualCompactionPaused)); + delete ca.prepicked_compaction->compaction; + } + delete ca.prepicked_compaction; + } + TEST_SYNC_POINT("DBImpl::UnscheduleCompactionCallback"); +} + +void DBImpl::UnscheduleFlushCallback(void* arg) { + // Decrement bg_flush_scheduled_ in flush callback + reinterpret_cast(arg)->db_->bg_flush_scheduled_--; + Env::Priority flush_pri = reinterpret_cast(arg)->thread_pri_; + if (Env::Priority::LOW == flush_pri) { + TEST_SYNC_POINT("DBImpl::UnscheduleLowFlushCallback"); + } else if (Env::Priority::HIGH == flush_pri) { + TEST_SYNC_POINT("DBImpl::UnscheduleHighFlushCallback"); + } + delete reinterpret_cast(arg); + TEST_SYNC_POINT("DBImpl::UnscheduleFlushCallback"); +} + +Status DBImpl::BackgroundFlush(bool* made_progress, JobContext* job_context, + LogBuffer* log_buffer, FlushReason* reason, + bool* flush_rescheduled_to_retain_udt, + Env::Priority thread_pri) { + mutex_.AssertHeld(); + + Status status; + *reason = FlushReason::kOthers; + // If BG work is stopped due to an error, but a recovery is in progress, + // that means this flush is part of the recovery. So allow it to go through + if (!error_handler_.IsBGWorkStopped()) { + if (shutting_down_.load(std::memory_order_acquire)) { + status = Status::ShutdownInProgress(); + } + } else if (!error_handler_.IsRecoveryInProgress()) { + status = error_handler_.GetBGError(); + } + + if (!status.ok()) { + return status; + } + + autovector bg_flush_args; + std::vector& superversion_contexts = + job_context->superversion_contexts; + autovector column_families_not_to_flush; + while (!flush_queue_.empty()) { + // This cfd is already referenced + FlushRequest flush_req = PopFirstFromFlushQueue(); + FlushReason flush_reason = flush_req.flush_reason; + if (!immutable_db_options_.atomic_flush && + ShouldRescheduleFlushRequestToRetainUDT(flush_req)) { + assert(flush_req.cfd_to_max_mem_id_to_persist.size() == 1); + ColumnFamilyData* cfd = + flush_req.cfd_to_max_mem_id_to_persist.begin()->first; + if (cfd->UnrefAndTryDelete()) { + return Status::OK(); + } + ROCKS_LOG_BUFFER(log_buffer, + "FlushRequest for column family %s is re-scheduled to " + "retain user-defined timestamps.", + cfd->GetName().c_str()); + // Reschedule the `FlushRequest` as is without checking dropped column + // family etc. The follow-up job will do the check anyways, so save the + // duplication. Column family is deduplicated by `SchdulePendingFlush` and + // `PopFirstFromFlushQueue` contains at flush request enqueueing and + // dequeueing time. + // This flush request is rescheduled right after it's popped from the + // queue while the db mutex is held, so there should be no other + // FlushRequest for the same column family with higher `max_memtable_id` + // in the queue to block the reschedule from succeeding. +#ifndef NDEBUG + flush_req.reschedule_count += 1; +#endif /* !NDEBUG */ + SchedulePendingFlush(flush_req); + *reason = flush_reason; + *flush_rescheduled_to_retain_udt = true; + return Status::TryAgain(); + } + superversion_contexts.clear(); + superversion_contexts.reserve( + flush_req.cfd_to_max_mem_id_to_persist.size()); + + for (const auto& [cfd, max_memtable_id] : + flush_req.cfd_to_max_mem_id_to_persist) { + if (cfd->GetMempurgeUsed()) { + // If imm() contains silent memtables (e.g.: because + // MemPurge was activated), requesting a flush will + // mark the imm_needed as true. + cfd->imm()->FlushRequested(); + } + + if (cfd->IsDropped() || !cfd->imm()->IsFlushPending()) { + // can't flush this CF, try next one + column_families_not_to_flush.push_back(cfd); + continue; + } + superversion_contexts.emplace_back(SuperVersionContext(true)); + bg_flush_args.emplace_back(cfd, max_memtable_id, + &(superversion_contexts.back()), flush_reason); + } + // `MaybeScheduleFlushOrCompaction` schedules as many `BackgroundCallFlush` + // jobs as the number of `FlushRequest` in the `flush_queue_`, a.k.a + // `unscheduled_flushes_`. So it's sufficient to make each `BackgroundFlush` + // handle one `FlushRequest` and each have a Status returned. + if (!bg_flush_args.empty() || !column_families_not_to_flush.empty()) { + TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundFlush:CheckFlushRequest:cb", + const_cast(&flush_req.reschedule_count)); + break; + } + } + + if (!bg_flush_args.empty()) { + auto bg_job_limits = GetBGJobLimits(); + for (const auto& arg : bg_flush_args) { + ColumnFamilyData* cfd = arg.cfd_; + ROCKS_LOG_BUFFER( + log_buffer, + "Calling FlushMemTableToOutputFile with column " + "family [%s], flush slots available %d, compaction slots available " + "%d, " + "flush slots scheduled %d, compaction slots scheduled %d", + cfd->GetName().c_str(), bg_job_limits.max_flushes, + bg_job_limits.max_compactions, bg_flush_scheduled_, + bg_compaction_scheduled_); + } + status = FlushMemTablesToOutputFiles(bg_flush_args, made_progress, + job_context, log_buffer, thread_pri); + TEST_SYNC_POINT("DBImpl::BackgroundFlush:BeforeFlush"); +// All the CFD/bg_flush_arg in the FlushReq must have the same flush reason, so +// just grab the first one +#ifndef NDEBUG + for (const auto& bg_flush_arg : bg_flush_args) { + assert(bg_flush_arg.flush_reason_ == bg_flush_args[0].flush_reason_); + } +#endif /* !NDEBUG */ + *reason = bg_flush_args[0].flush_reason_; + for (auto& arg : bg_flush_args) { + ColumnFamilyData* cfd = arg.cfd_; + if (cfd->UnrefAndTryDelete()) { + arg.cfd_ = nullptr; + } + } + } + for (auto cfd : column_families_not_to_flush) { + cfd->UnrefAndTryDelete(); + } + return status; +} + +void DBImpl::BackgroundCallFlush(Env::Priority thread_pri) { + bool made_progress = false; + JobContext job_context(next_job_id_.fetch_add(1), true); + + TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundCallFlush:start", nullptr); + + LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, + immutable_db_options_.info_log.get()); + TEST_SYNC_POINT("DBImpl::BackgroundCallFlush:Start:1"); + TEST_SYNC_POINT("DBImpl::BackgroundCallFlush:Start:2"); + { + InstrumentedMutexLock l(&mutex_); + assert(bg_flush_scheduled_); + num_running_flushes_++; + + std::unique_ptr::iterator> + pending_outputs_inserted_elem(new std::list::iterator( + CaptureCurrentFileNumberInPendingOutputs())); + FlushReason reason; + bool flush_rescheduled_to_retain_udt = false; + Status s = + BackgroundFlush(&made_progress, &job_context, &log_buffer, &reason, + &flush_rescheduled_to_retain_udt, thread_pri); + if (s.IsTryAgain() && flush_rescheduled_to_retain_udt) { + bg_cv_.SignalAll(); // In case a waiter can proceed despite the error + mutex_.Unlock(); + TEST_SYNC_POINT_CALLBACK("DBImpl::AfterRetainUDTReschedule:cb", nullptr); + immutable_db_options_.clock->SleepForMicroseconds( + 100000); // prevent hot loop + mutex_.Lock(); + } else if (!s.ok() && !s.IsShutdownInProgress() && + !s.IsColumnFamilyDropped() && + reason != FlushReason::kErrorRecovery) { + // Wait a little bit before retrying background flush in + // case this is an environmental problem and we do not want to + // chew up resources for failed flushes for the duration of + // the problem. + uint64_t error_cnt = + default_cf_internal_stats_->BumpAndGetBackgroundErrorCount(); + bg_cv_.SignalAll(); // In case a waiter can proceed despite the error + mutex_.Unlock(); + ROCKS_LOG_ERROR(immutable_db_options_.info_log, + "Waiting after background flush error: %s" + "Accumulated background error counts: %" PRIu64, + s.ToString().c_str(), error_cnt); + log_buffer.FlushBufferToLog(); + LogFlush(immutable_db_options_.info_log); + immutable_db_options_.clock->SleepForMicroseconds(1000000); + mutex_.Lock(); + } + + TEST_SYNC_POINT("DBImpl::BackgroundCallFlush:FlushFinish:0"); + ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem); + // There is no need to do these clean up if the flush job is rescheduled + // to retain user-defined timestamps because the job doesn't get to the + // stage of actually flushing the MemTables. + if (!flush_rescheduled_to_retain_udt) { + // If flush failed, we want to delete all temporary files that we might + // have created. Thus, we force full scan in FindObsoleteFiles() + FindObsoleteFiles(&job_context, !s.ok() && !s.IsShutdownInProgress() && + !s.IsColumnFamilyDropped()); + // delete unnecessary files if any, this is done outside the mutex + if (job_context.HaveSomethingToClean() || + job_context.HaveSomethingToDelete() || !log_buffer.IsEmpty()) { + mutex_.Unlock(); + TEST_SYNC_POINT("DBImpl::BackgroundCallFlush:FilesFound"); + // Have to flush the info logs before bg_flush_scheduled_-- + // because if bg_flush_scheduled_ becomes 0 and the lock is + // released, the deconstructor of DB can kick in and destroy all the + // states of DB so info_log might not be available after that point. + // It also applies to access other states that DB owns. + log_buffer.FlushBufferToLog(); + if (job_context.HaveSomethingToDelete()) { + PurgeObsoleteFiles(job_context); + } + job_context.Clean(); + mutex_.Lock(); + } + TEST_SYNC_POINT("DBImpl::BackgroundCallFlush:ContextCleanedUp"); + } + + assert(num_running_flushes_ > 0); + num_running_flushes_--; + bg_flush_scheduled_--; + // See if there's more work to be done + MaybeScheduleFlushOrCompaction(); + atomic_flush_install_cv_.SignalAll(); + bg_cv_.SignalAll(); + // IMPORTANT: there should be no code after calling SignalAll. This call may + // signal the DB destructor that it's OK to proceed with destruction. In + // that case, all DB variables will be dealloacated and referencing them + // will cause trouble. + } +} + +void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction, + Env::Priority bg_thread_pri) { + bool made_progress = false; + JobContext job_context(next_job_id_.fetch_add(1), true); + TEST_SYNC_POINT("BackgroundCallCompaction:0"); + LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, + immutable_db_options_.info_log.get()); + { + InstrumentedMutexLock l(&mutex_); + + num_running_compactions_++; + + std::unique_ptr::iterator> + pending_outputs_inserted_elem(new std::list::iterator( + CaptureCurrentFileNumberInPendingOutputs())); + + assert((bg_thread_pri == Env::Priority::BOTTOM && + bg_bottom_compaction_scheduled_) || + (bg_thread_pri == Env::Priority::LOW && bg_compaction_scheduled_)); + Status s = BackgroundCompaction(&made_progress, &job_context, &log_buffer, + prepicked_compaction, bg_thread_pri); + TEST_SYNC_POINT("BackgroundCallCompaction:1"); + if (s.IsBusy()) { + bg_cv_.SignalAll(); // In case a waiter can proceed despite the error + mutex_.Unlock(); + immutable_db_options_.clock->SleepForMicroseconds( + 10000); // prevent hot loop + mutex_.Lock(); + } else if (!s.ok() && !s.IsShutdownInProgress() && + !s.IsManualCompactionPaused() && !s.IsColumnFamilyDropped()) { + // Wait a little bit before retrying background compaction in + // case this is an environmental problem and we do not want to + // chew up resources for failed compactions for the duration of + // the problem. + uint64_t error_cnt = + default_cf_internal_stats_->BumpAndGetBackgroundErrorCount(); + bg_cv_.SignalAll(); // In case a waiter can proceed despite the error + mutex_.Unlock(); + log_buffer.FlushBufferToLog(); + ROCKS_LOG_ERROR(immutable_db_options_.info_log, + "Waiting after background compaction error: %s, " + "Accumulated background error counts: %" PRIu64, + s.ToString().c_str(), error_cnt); + LogFlush(immutable_db_options_.info_log); + immutable_db_options_.clock->SleepForMicroseconds(1000000); + mutex_.Lock(); + } else if (s.IsManualCompactionPaused()) { + assert(prepicked_compaction); + ManualCompactionState* m = prepicked_compaction->manual_compaction_state; + assert(m); + ROCKS_LOG_BUFFER(&log_buffer, "[%s] [JOB %d] Manual compaction paused", + m->cfd->GetName().c_str(), job_context.job_id); + } + + ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem); + + // If compaction failed, we want to delete all temporary files that we + // might have created (they might not be all recorded in job_context in + // case of a failure). Thus, we force full scan in FindObsoleteFiles() + FindObsoleteFiles(&job_context, !s.ok() && !s.IsShutdownInProgress() && + !s.IsManualCompactionPaused() && + !s.IsColumnFamilyDropped() && + !s.IsBusy()); + TEST_SYNC_POINT("DBImpl::BackgroundCallCompaction:FoundObsoleteFiles"); + + // delete unnecessary files if any, this is done outside the mutex + if (job_context.HaveSomethingToClean() || + job_context.HaveSomethingToDelete() || !log_buffer.IsEmpty()) { + mutex_.Unlock(); + // Have to flush the info logs before bg_compaction_scheduled_-- + // because if bg_flush_scheduled_ becomes 0 and the lock is + // released, the deconstructor of DB can kick in and destroy all the + // states of DB so info_log might not be available after that point. + // It also applies to access other states that DB owns. + log_buffer.FlushBufferToLog(); + if (job_context.HaveSomethingToDelete()) { + PurgeObsoleteFiles(job_context); + TEST_SYNC_POINT("DBImpl::BackgroundCallCompaction:PurgedObsoleteFiles"); + } + job_context.Clean(); + mutex_.Lock(); + } + + assert(num_running_compactions_ > 0); + num_running_compactions_--; + + if (bg_thread_pri == Env::Priority::LOW) { + bg_compaction_scheduled_--; + } else { + assert(bg_thread_pri == Env::Priority::BOTTOM); + bg_bottom_compaction_scheduled_--; + } + + // See if there's more work to be done + MaybeScheduleFlushOrCompaction(); + + if (prepicked_compaction != nullptr && + prepicked_compaction->task_token != nullptr) { + // Releasing task tokens affects (and asserts on) the DB state, so + // must be done before we potentially signal the DB close process to + // proceed below. + prepicked_compaction->task_token.reset(); + } + + if (made_progress || + (bg_compaction_scheduled_ == 0 && + bg_bottom_compaction_scheduled_ == 0) || + HasPendingManualCompaction() || unscheduled_compactions_ == 0) { + // signal if + // * made_progress -- need to wakeup DelayWrite + // * bg_{bottom,}_compaction_scheduled_ == 0 -- need to wakeup ~DBImpl + // * HasPendingManualCompaction -- need to wakeup RunManualCompaction + // If none of this is true, there is no need to signal since nobody is + // waiting for it + bg_cv_.SignalAll(); + } + // IMPORTANT: there should be no code after calling SignalAll. This call may + // signal the DB destructor that it's OK to proceed with destruction. In + // that case, all DB variables will be dealloacated and referencing them + // will cause trouble. + } +} + +Status DBImpl::BackgroundCompaction(bool* made_progress, + JobContext* job_context, + LogBuffer* log_buffer, + PrepickedCompaction* prepicked_compaction, + Env::Priority thread_pri) { + ManualCompactionState* manual_compaction = + prepicked_compaction == nullptr + ? nullptr + : prepicked_compaction->manual_compaction_state; + *made_progress = false; + mutex_.AssertHeld(); + TEST_SYNC_POINT("DBImpl::BackgroundCompaction:Start"); + + const ReadOptions read_options(Env::IOActivity::kCompaction); + + bool is_manual = (manual_compaction != nullptr); + std::unique_ptr c; + if (prepicked_compaction != nullptr && + prepicked_compaction->compaction != nullptr) { + c.reset(prepicked_compaction->compaction); + } + bool is_prepicked = is_manual || c; + + // (manual_compaction->in_progress == false); + bool trivial_move_disallowed = + is_manual && manual_compaction->disallow_trivial_move; + + CompactionJobStats compaction_job_stats; + Status status; + if (!error_handler_.IsBGWorkStopped()) { + if (shutting_down_.load(std::memory_order_acquire)) { + status = Status::ShutdownInProgress(); + } else if (is_manual && + manual_compaction->canceled.load(std::memory_order_acquire)) { + status = Status::Incomplete(Status::SubCode::kManualCompactionPaused); + } + } else { + status = error_handler_.GetBGError(); + // If we get here, it means a hard error happened after this compaction + // was scheduled by MaybeScheduleFlushOrCompaction(), but before it got + // a chance to execute. Since we didn't pop a cfd from the compaction + // queue, increment unscheduled_compactions_ + unscheduled_compactions_++; + } + + if (!status.ok()) { + if (is_manual) { + manual_compaction->status = status; + manual_compaction->done = true; + manual_compaction->in_progress = false; + manual_compaction = nullptr; + } + if (c) { + c->ReleaseCompactionFiles(status); + c.reset(); + } + return status; + } + + if (is_manual) { + // another thread cannot pick up the same work + manual_compaction->in_progress = true; + } + + TEST_SYNC_POINT("DBImpl::BackgroundCompaction:InProgress"); + + std::unique_ptr task_token; + + // InternalKey manual_end_storage; + // InternalKey* manual_end = &manual_end_storage; + bool sfm_reserved_compact_space = false; + if (is_manual) { + ManualCompactionState* m = manual_compaction; + assert(m->in_progress); + if (!c) { + m->done = true; + m->manual_end = nullptr; + ROCKS_LOG_BUFFER( + log_buffer, + "[%s] Manual compaction from level-%d from %s .. " + "%s; nothing to do\n", + m->cfd->GetName().c_str(), m->input_level, + (m->begin ? m->begin->DebugString(true).c_str() : "(begin)"), + (m->end ? m->end->DebugString(true).c_str() : "(end)")); + } else { + // First check if we have enough room to do the compaction + bool enough_room = EnoughRoomForCompaction( + m->cfd, *(c->inputs()), &sfm_reserved_compact_space, log_buffer); + + if (!enough_room) { + // Then don't do the compaction + c->ReleaseCompactionFiles(status); + c.reset(); + // m's vars will get set properly at the end of this function, + // as long as status == CompactionTooLarge + status = Status::CompactionTooLarge(); + } else { + ROCKS_LOG_BUFFER( + log_buffer, + "[%s] Manual compaction from level-%d to level-%d from %s .. " + "%s; will stop at %s\n", + m->cfd->GetName().c_str(), m->input_level, c->output_level(), + (m->begin ? m->begin->DebugString(true).c_str() : "(begin)"), + (m->end ? m->end->DebugString(true).c_str() : "(end)"), + ((m->done || m->manual_end == nullptr) + ? "(end)" + : m->manual_end->DebugString(true).c_str())); + } + } + } else if (!is_prepicked && !compaction_queue_.empty()) { + if (HasExclusiveManualCompaction()) { + // Can't compact right now, but try again later + TEST_SYNC_POINT("DBImpl::BackgroundCompaction()::Conflict"); + + // Stay in the compaction queue. + unscheduled_compactions_++; + + return Status::OK(); + } + + auto cfd = PickCompactionFromQueue(&task_token, log_buffer); + if (cfd == nullptr) { + // Can't find any executable task from the compaction queue. + // All tasks have been throttled by compaction thread limiter. + ++unscheduled_compactions_; + return Status::Busy(); + } + + // We unreference here because the following code will take a Ref() on + // this cfd if it is going to use it (Compaction class holds a + // reference). + // This will all happen under a mutex so we don't have to be afraid of + // somebody else deleting it. + if (cfd->UnrefAndTryDelete()) { + // This was the last reference of the column family, so no need to + // compact. + return Status::OK(); + } + + // Pick up latest mutable CF Options and use it throughout the + // compaction job + // Compaction makes a copy of the latest MutableCFOptions. It should be used + // throughout the compaction procedure to make sure consistency. It will + // eventually be installed into SuperVersion + auto* mutable_cf_options = cfd->GetLatestMutableCFOptions(); + if (!mutable_cf_options->disable_auto_compactions && !cfd->IsDropped()) { + // NOTE: try to avoid unnecessary copy of MutableCFOptions if + // compaction is not necessary. Need to make sure mutex is held + // until we make a copy in the following code + TEST_SYNC_POINT("DBImpl::BackgroundCompaction():BeforePickCompaction"); + c.reset(cfd->PickCompaction(*mutable_cf_options, mutable_db_options_, + log_buffer)); + TEST_SYNC_POINT("DBImpl::BackgroundCompaction():AfterPickCompaction"); + + if (c != nullptr) { + bool enough_room = EnoughRoomForCompaction( + cfd, *(c->inputs()), &sfm_reserved_compact_space, log_buffer); + + if (!enough_room) { + // Then don't do the compaction + c->ReleaseCompactionFiles(status); + c->column_family_data() + ->current() + ->storage_info() + ->ComputeCompactionScore(*(c->immutable_options()), + *(c->mutable_cf_options())); + AddToCompactionQueue(cfd); + ++unscheduled_compactions_; + + c.reset(); + // Don't need to sleep here, because BackgroundCallCompaction + // will sleep if !s.ok() + status = Status::CompactionTooLarge(); + } else { + // update statistics + size_t num_files = 0; + for (auto& each_level : *c->inputs()) { + num_files += each_level.files.size(); + } + RecordInHistogram(stats_, NUM_FILES_IN_SINGLE_COMPACTION, num_files); + + // There are three things that can change compaction score: + // 1) When flush or compaction finish. This case is covered by + // InstallSuperVersionAndScheduleWork + // 2) When MutableCFOptions changes. This case is also covered by + // InstallSuperVersionAndScheduleWork, because this is when the new + // options take effect. + // 3) When we Pick a new compaction, we "remove" those files being + // compacted from the calculation, which then influences compaction + // score. Here we check if we need the new compaction even without the + // files that are currently being compacted. If we need another + // compaction, we might be able to execute it in parallel, so we add + // it to the queue and schedule a new thread. + if (cfd->NeedsCompaction()) { + // Yes, we need more compactions! + AddToCompactionQueue(cfd); + ++unscheduled_compactions_; + MaybeScheduleFlushOrCompaction(); + } + } + } + } + } + + IOStatus io_s; + if (!c) { + // Nothing to do + ROCKS_LOG_BUFFER(log_buffer, "Compaction nothing to do"); + } else if (c->deletion_compaction()) { + // TODO(icanadi) Do we want to honor snapshots here? i.e. not delete old + // file if there is alive snapshot pointing to it + TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundCompaction:BeforeCompaction", + c->column_family_data()); + assert(c->num_input_files(1) == 0); + assert(c->column_family_data()->ioptions()->compaction_style == + kCompactionStyleFIFO); + + compaction_job_stats.num_input_files = c->num_input_files(0); + + NotifyOnCompactionBegin(c->column_family_data(), c.get(), status, + compaction_job_stats, job_context->job_id); + + for (const auto& f : *c->inputs(0)) { + c->edit()->DeleteFile(c->level(), f->fd.GetNumber()); + } + status = versions_->LogAndApply( + c->column_family_data(), *c->mutable_cf_options(), read_options, + c->edit(), &mutex_, directories_.GetDbDir()); + io_s = versions_->io_status(); + InstallSuperVersionAndScheduleWork(c->column_family_data(), + &job_context->superversion_contexts[0], + *c->mutable_cf_options()); + ROCKS_LOG_BUFFER(log_buffer, "[%s] Deleted %d files\n", + c->column_family_data()->GetName().c_str(), + c->num_input_files(0)); + *made_progress = true; + TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundCompaction:AfterCompaction", + c->column_family_data()); + } else if (!trivial_move_disallowed && c->IsTrivialMove()) { + TEST_SYNC_POINT("DBImpl::BackgroundCompaction:TrivialMove"); + TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundCompaction:BeforeCompaction", + c->column_family_data()); + // Instrument for event update + // TODO(yhchiang): add op details for showing trivial-move. + ThreadStatusUtil::SetColumnFamily(c->column_family_data()); + ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_COMPACTION); + + compaction_job_stats.num_input_files = c->num_input_files(0); + + NotifyOnCompactionBegin(c->column_family_data(), c.get(), status, + compaction_job_stats, job_context->job_id); + + // Move files to next level + int32_t moved_files = 0; + int64_t moved_bytes = 0; + for (unsigned int l = 0; l < c->num_input_levels(); l++) { + if (c->level(l) == c->output_level()) { + continue; + } + for (size_t i = 0; i < c->num_input_files(l); i++) { + FileMetaData* f = c->input(l, i); + c->edit()->DeleteFile(c->level(l), f->fd.GetNumber()); + c->edit()->AddFile( + c->output_level(), f->fd.GetNumber(), f->fd.GetPathId(), + f->fd.GetFileSize(), f->smallest, f->largest, f->fd.smallest_seqno, + f->fd.largest_seqno, f->marked_for_compaction, f->temperature, + f->oldest_blob_file_number, f->oldest_ancester_time, + f->file_creation_time, f->epoch_number, f->file_checksum, + f->file_checksum_func_name, f->unique_id, + f->compensated_range_deletion_size, f->tail_size, + f->user_defined_timestamps_persisted); + + ROCKS_LOG_BUFFER( + log_buffer, + "[%s] Moving #%" PRIu64 " to level-%d %" PRIu64 " bytes\n", + c->column_family_data()->GetName().c_str(), f->fd.GetNumber(), + c->output_level(), f->fd.GetFileSize()); + ++moved_files; + moved_bytes += f->fd.GetFileSize(); + } + } + if (c->compaction_reason() == CompactionReason::kLevelMaxLevelSize && + c->immutable_options()->compaction_pri == kRoundRobin) { + int start_level = c->start_level(); + if (start_level > 0) { + auto vstorage = c->input_version()->storage_info(); + c->edit()->AddCompactCursor( + start_level, + vstorage->GetNextCompactCursor(start_level, c->num_input_files(0))); + } + } + status = versions_->LogAndApply( + c->column_family_data(), *c->mutable_cf_options(), read_options, + c->edit(), &mutex_, directories_.GetDbDir()); + io_s = versions_->io_status(); + // Use latest MutableCFOptions + InstallSuperVersionAndScheduleWork(c->column_family_data(), + &job_context->superversion_contexts[0], + *c->mutable_cf_options()); + + VersionStorageInfo::LevelSummaryStorage tmp; + c->column_family_data()->internal_stats()->IncBytesMoved(c->output_level(), + moved_bytes); + { + event_logger_.LogToBuffer(log_buffer) + << "job" << job_context->job_id << "event" + << "trivial_move" + << "destination_level" << c->output_level() << "files" << moved_files + << "total_files_size" << moved_bytes; + } + ROCKS_LOG_BUFFER( + log_buffer, + "[%s] Moved #%d files to level-%d %" PRIu64 " bytes %s: %s\n", + c->column_family_data()->GetName().c_str(), moved_files, + c->output_level(), moved_bytes, status.ToString().c_str(), + c->column_family_data()->current()->storage_info()->LevelSummary(&tmp)); + *made_progress = true; + + // Clear Instrument + ThreadStatusUtil::ResetThreadStatus(); + TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundCompaction:AfterCompaction", + c->column_family_data()); + } else if (!is_prepicked && c->output_level() > 0 && + c->output_level() == + c->column_family_data() + ->current() + ->storage_info() + ->MaxOutputLevel( + immutable_db_options_.allow_ingest_behind) && + env_->GetBackgroundThreads(Env::Priority::BOTTOM) > 0) { + // Forward compactions involving last level to the bottom pool if it exists, + // such that compactions unlikely to contribute to write stalls can be + // delayed or deprioritized. + TEST_SYNC_POINT("DBImpl::BackgroundCompaction:ForwardToBottomPriPool"); + CompactionArg* ca = new CompactionArg; + ca->db = this; + ca->compaction_pri_ = Env::Priority::BOTTOM; + ca->prepicked_compaction = new PrepickedCompaction; + ca->prepicked_compaction->compaction = c.release(); + ca->prepicked_compaction->manual_compaction_state = nullptr; + // Transfer requested token, so it doesn't need to do it again. + ca->prepicked_compaction->task_token = std::move(task_token); + ++bg_bottom_compaction_scheduled_; + env_->Schedule(&DBImpl::BGWorkBottomCompaction, ca, Env::Priority::BOTTOM, + this, &DBImpl::UnscheduleCompactionCallback); + } else { + TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundCompaction:BeforeCompaction", + c->column_family_data()); + int output_level __attribute__((__unused__)); + output_level = c->output_level(); + TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundCompaction:NonTrivial", + &output_level); + std::vector snapshot_seqs; + SequenceNumber earliest_write_conflict_snapshot; + SnapshotChecker* snapshot_checker; + GetSnapshotContext(job_context, &snapshot_seqs, + &earliest_write_conflict_snapshot, &snapshot_checker); + assert(is_snapshot_supported_ || snapshots_.empty()); + + CompactionJob compaction_job( + job_context->job_id, c.get(), immutable_db_options_, + mutable_db_options_, file_options_for_compaction_, versions_.get(), + &shutting_down_, log_buffer, directories_.GetDbDir(), + GetDataDir(c->column_family_data(), c->output_path_id()), + GetDataDir(c->column_family_data(), 0), stats_, &mutex_, + &error_handler_, snapshot_seqs, earliest_write_conflict_snapshot, + snapshot_checker, job_context, table_cache_, &event_logger_, + c->mutable_cf_options()->paranoid_file_checks, + c->mutable_cf_options()->report_bg_io_stats, dbname_, + &compaction_job_stats, thread_pri, io_tracer_, + is_manual ? manual_compaction->canceled + : kManualCompactionCanceledFalse_, + db_id_, db_session_id_, c->column_family_data()->GetFullHistoryTsLow(), + c->trim_ts(), &blob_callback_, &bg_compaction_scheduled_, + &bg_bottom_compaction_scheduled_); + compaction_job.Prepare(); + + NotifyOnCompactionBegin(c->column_family_data(), c.get(), status, + compaction_job_stats, job_context->job_id); + mutex_.Unlock(); + TEST_SYNC_POINT_CALLBACK( + "DBImpl::BackgroundCompaction:NonTrivial:BeforeRun", nullptr); + // Should handle error? + compaction_job.Run().PermitUncheckedError(); + TEST_SYNC_POINT("DBImpl::BackgroundCompaction:NonTrivial:AfterRun"); + mutex_.Lock(); + + status = compaction_job.Install(*c->mutable_cf_options()); + io_s = compaction_job.io_status(); + if (status.ok()) { + InstallSuperVersionAndScheduleWork(c->column_family_data(), + &job_context->superversion_contexts[0], + *c->mutable_cf_options()); + } + *made_progress = true; + TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundCompaction:AfterCompaction", + c->column_family_data()); + } + + if (status.ok() && !io_s.ok()) { + status = io_s; + } else { + io_s.PermitUncheckedError(); + } + + if (c != nullptr) { + c->ReleaseCompactionFiles(status); + *made_progress = true; + + // Need to make sure SstFileManager does its bookkeeping + auto sfm = static_cast( + immutable_db_options_.sst_file_manager.get()); + if (sfm && sfm_reserved_compact_space) { + sfm->OnCompactionCompletion(c.get()); + } + + NotifyOnCompactionCompleted(c->column_family_data(), c.get(), status, + compaction_job_stats, job_context->job_id); + } + + if (status.ok() || status.IsCompactionTooLarge() || + status.IsManualCompactionPaused()) { + // Done + } else if (status.IsColumnFamilyDropped() || status.IsShutdownInProgress()) { + // Ignore compaction errors found during shutting down + } else { + ROCKS_LOG_WARN(immutable_db_options_.info_log, "Compaction error: %s", + status.ToString().c_str()); + if (!io_s.ok()) { + // Error while writing to MANIFEST. + // In fact, versions_->io_status() can also be the result of renaming + // CURRENT file. With current code, it's just difficult to tell. So just + // be pessimistic and try write to a new MANIFEST. + // TODO: distinguish between MANIFEST write and CURRENT renaming + auto err_reason = versions_->io_status().ok() + ? BackgroundErrorReason::kCompaction + : BackgroundErrorReason::kManifestWrite; + error_handler_.SetBGError(io_s, err_reason); + } else { + error_handler_.SetBGError(status, BackgroundErrorReason::kCompaction); + } + if (c != nullptr && !is_manual && !error_handler_.IsBGWorkStopped()) { + // Put this cfd back in the compaction queue so we can retry after some + // time + auto cfd = c->column_family_data(); + assert(cfd != nullptr); + // Since this compaction failed, we need to recompute the score so it + // takes the original input files into account + c->column_family_data() + ->current() + ->storage_info() + ->ComputeCompactionScore(*(c->immutable_options()), + *(c->mutable_cf_options())); + if (!cfd->queued_for_compaction()) { + AddToCompactionQueue(cfd); + ++unscheduled_compactions_; + } + } + } + // this will unref its input_version and column_family_data + c.reset(); + + if (is_manual) { + ManualCompactionState* m = manual_compaction; + if (!status.ok()) { + m->status = status; + m->done = true; + } + // For universal compaction: + // Because universal compaction always happens at level 0, so one + // compaction will pick up all overlapped files. No files will be + // filtered out due to size limit and left for a successive compaction. + // So we can safely conclude the current compaction. + // + // Also note that, if we don't stop here, then the current compaction + // writes a new file back to level 0, which will be used in successive + // compaction. Hence the manual compaction will never finish. + // + // Stop the compaction if manual_end points to nullptr -- this means + // that we compacted the whole range. manual_end should always point + // to nullptr in case of universal compaction + if (m->manual_end == nullptr) { + m->done = true; + } + if (!m->done) { + // We only compacted part of the requested range. Update *m + // to the range that is left to be compacted. + // Universal and FIFO compactions should always compact the whole range + assert(m->cfd->ioptions()->compaction_style != + kCompactionStyleUniversal || + m->cfd->ioptions()->num_levels > 1); + assert(m->cfd->ioptions()->compaction_style != kCompactionStyleFIFO); + m->tmp_storage = *m->manual_end; + m->begin = &m->tmp_storage; + m->incomplete = true; + } + m->in_progress = false; // not being processed anymore + } + TEST_SYNC_POINT("DBImpl::BackgroundCompaction:Finish"); + return status; +} + +bool DBImpl::HasPendingManualCompaction() { + return (!manual_compaction_dequeue_.empty()); +} + +void DBImpl::AddManualCompaction(DBImpl::ManualCompactionState* m) { + assert(manual_compaction_paused_ == 0); + manual_compaction_dequeue_.push_back(m); +} + +void DBImpl::RemoveManualCompaction(DBImpl::ManualCompactionState* m) { + // Remove from queue + std::deque::iterator it = + manual_compaction_dequeue_.begin(); + while (it != manual_compaction_dequeue_.end()) { + if (m == (*it)) { + it = manual_compaction_dequeue_.erase(it); + return; + } + ++it; + } + assert(false); + return; +} + +bool DBImpl::ShouldntRunManualCompaction(ManualCompactionState* m) { + if (m->exclusive) { + return (bg_bottom_compaction_scheduled_ > 0 || + bg_compaction_scheduled_ > 0); + } + std::deque::iterator it = + manual_compaction_dequeue_.begin(); + bool seen = false; + while (it != manual_compaction_dequeue_.end()) { + if (m == (*it)) { + ++it; + seen = true; + continue; + } else if (MCOverlap(m, (*it)) && (!seen && !(*it)->in_progress)) { + // Consider the other manual compaction *it, conflicts if: + // overlaps with m + // and (*it) is ahead in the queue and is not yet in progress + return true; + } + ++it; + } + return false; +} + +bool DBImpl::HaveManualCompaction(ColumnFamilyData* cfd) { + // Remove from priority queue + std::deque::iterator it = + manual_compaction_dequeue_.begin(); + while (it != manual_compaction_dequeue_.end()) { + if ((*it)->exclusive) { + return true; + } + if ((cfd == (*it)->cfd) && (!((*it)->in_progress || (*it)->done))) { + // Allow automatic compaction if manual compaction is + // in progress + return true; + } + ++it; + } + return false; +} + +bool DBImpl::HasExclusiveManualCompaction() { + // Remove from priority queue + std::deque::iterator it = + manual_compaction_dequeue_.begin(); + while (it != manual_compaction_dequeue_.end()) { + if ((*it)->exclusive) { + return true; + } + ++it; + } + return false; +} + +bool DBImpl::MCOverlap(ManualCompactionState* m, ManualCompactionState* m1) { + if ((m->exclusive) || (m1->exclusive)) { + return true; + } + if (m->cfd != m1->cfd) { + return false; + } + return false; +} + +void DBImpl::BuildCompactionJobInfo( + const ColumnFamilyData* cfd, Compaction* c, const Status& st, + const CompactionJobStats& compaction_job_stats, const int job_id, + CompactionJobInfo* compaction_job_info) const { + assert(compaction_job_info != nullptr); + compaction_job_info->cf_id = cfd->GetID(); + compaction_job_info->cf_name = cfd->GetName(); + compaction_job_info->status = st; + compaction_job_info->thread_id = env_->GetThreadID(); + compaction_job_info->job_id = job_id; + compaction_job_info->base_input_level = c->start_level(); + compaction_job_info->output_level = c->output_level(); + compaction_job_info->stats = compaction_job_stats; + compaction_job_info->table_properties = c->GetTableProperties(); + compaction_job_info->compaction_reason = c->compaction_reason(); + compaction_job_info->compression = c->output_compression(); + + const ReadOptions read_options(Env::IOActivity::kCompaction); + for (size_t i = 0; i < c->num_input_levels(); ++i) { + for (const auto fmd : *c->inputs(i)) { + const FileDescriptor& desc = fmd->fd; + const uint64_t file_number = desc.GetNumber(); + auto fn = TableFileName(c->immutable_options()->cf_paths, file_number, + desc.GetPathId()); + compaction_job_info->input_files.push_back(fn); + compaction_job_info->input_file_infos.push_back(CompactionFileInfo{ + static_cast(i), file_number, fmd->oldest_blob_file_number}); + } + } + + for (const auto& newf : c->edit()->GetNewFiles()) { + const FileMetaData& meta = newf.second; + const FileDescriptor& desc = meta.fd; + const uint64_t file_number = desc.GetNumber(); + compaction_job_info->output_files.push_back(TableFileName( + c->immutable_options()->cf_paths, file_number, desc.GetPathId())); + compaction_job_info->output_file_infos.push_back(CompactionFileInfo{ + newf.first, file_number, meta.oldest_blob_file_number}); + } + compaction_job_info->blob_compression_type = + c->mutable_cf_options()->blob_compression_type; + + // Update BlobFilesInfo. + for (const auto& blob_file : c->edit()->GetBlobFileAdditions()) { + BlobFileAdditionInfo blob_file_addition_info( + BlobFileName(c->immutable_options()->cf_paths.front().path, + blob_file.GetBlobFileNumber()) /*blob_file_path*/, + blob_file.GetBlobFileNumber(), blob_file.GetTotalBlobCount(), + blob_file.GetTotalBlobBytes()); + compaction_job_info->blob_file_addition_infos.emplace_back( + std::move(blob_file_addition_info)); + } + + // Update BlobFilesGarbageInfo. + for (const auto& blob_file : c->edit()->GetBlobFileGarbages()) { + BlobFileGarbageInfo blob_file_garbage_info( + BlobFileName(c->immutable_options()->cf_paths.front().path, + blob_file.GetBlobFileNumber()) /*blob_file_path*/, + blob_file.GetBlobFileNumber(), blob_file.GetGarbageBlobCount(), + blob_file.GetGarbageBlobBytes()); + compaction_job_info->blob_file_garbage_infos.emplace_back( + std::move(blob_file_garbage_info)); + } +} + +// SuperVersionContext gets created and destructed outside of the lock -- +// we use this conveniently to: +// * malloc one SuperVersion() outside of the lock -- new_superversion +// * delete SuperVersion()s outside of the lock -- superversions_to_free +// +// However, if InstallSuperVersionAndScheduleWork() gets called twice with the +// same sv_context, we can't reuse the SuperVersion() that got +// malloced because +// first call already used it. In that rare case, we take a hit and create a +// new SuperVersion() inside of the mutex. We do similar thing +// for superversion_to_free + +void DBImpl::InstallSuperVersionAndScheduleWork( + ColumnFamilyData* cfd, SuperVersionContext* sv_context, + const MutableCFOptions& mutable_cf_options) { + mutex_.AssertHeld(); + + // Update max_total_in_memory_state_ + size_t old_memtable_size = 0; + auto* old_sv = cfd->GetSuperVersion(); + if (old_sv) { + old_memtable_size = old_sv->mutable_cf_options.write_buffer_size * + old_sv->mutable_cf_options.max_write_buffer_number; + } + + // this branch is unlikely to step in + if (UNLIKELY(sv_context->new_superversion == nullptr)) { + sv_context->NewSuperVersion(); + } + cfd->InstallSuperVersion(sv_context, mutable_cf_options); + + // There may be a small data race here. The snapshot tricking bottommost + // compaction may already be released here. But assuming there will always be + // newer snapshot created and released frequently, the compaction will be + // triggered soon anyway. + bottommost_files_mark_threshold_ = kMaxSequenceNumber; + for (auto* my_cfd : *versions_->GetColumnFamilySet()) { + if (!my_cfd->ioptions()->allow_ingest_behind) { + bottommost_files_mark_threshold_ = std::min( + bottommost_files_mark_threshold_, + my_cfd->current()->storage_info()->bottommost_files_mark_threshold()); + } + } + + // Whenever we install new SuperVersion, we might need to issue new flushes or + // compactions. + SchedulePendingCompaction(cfd); + MaybeScheduleFlushOrCompaction(); + + // Update max_total_in_memory_state_ + max_total_in_memory_state_ = max_total_in_memory_state_ - old_memtable_size + + mutable_cf_options.write_buffer_size * + mutable_cf_options.max_write_buffer_number; +} + +// ShouldPurge is called by FindObsoleteFiles when doing a full scan, +// and db mutex (mutex_) should already be held. +// Actually, the current implementation of FindObsoleteFiles with +// full_scan=true can issue I/O requests to obtain list of files in +// directories, e.g. env_->getChildren while holding db mutex. +bool DBImpl::ShouldPurge(uint64_t file_number) const { + return files_grabbed_for_purge_.find(file_number) == + files_grabbed_for_purge_.end() && + purge_files_.find(file_number) == purge_files_.end(); +} + +// MarkAsGrabbedForPurge is called by FindObsoleteFiles, and db mutex +// (mutex_) should already be held. +void DBImpl::MarkAsGrabbedForPurge(uint64_t file_number) { + files_grabbed_for_purge_.insert(file_number); +} + +void DBImpl::SetSnapshotChecker(SnapshotChecker* snapshot_checker) { + InstrumentedMutexLock l(&mutex_); + // snapshot_checker_ should only set once. If we need to set it multiple + // times, we need to make sure the old one is not deleted while it is still + // using by a compaction job. + assert(!snapshot_checker_); + snapshot_checker_.reset(snapshot_checker); +} + +void DBImpl::GetSnapshotContext( + JobContext* job_context, std::vector* snapshot_seqs, + SequenceNumber* earliest_write_conflict_snapshot, + SnapshotChecker** snapshot_checker_ptr) { + mutex_.AssertHeld(); + assert(job_context != nullptr); + assert(snapshot_seqs != nullptr); + assert(earliest_write_conflict_snapshot != nullptr); + assert(snapshot_checker_ptr != nullptr); + + *snapshot_checker_ptr = snapshot_checker_.get(); + if (use_custom_gc_ && *snapshot_checker_ptr == nullptr) { + *snapshot_checker_ptr = DisableGCSnapshotChecker::Instance(); + } + if (*snapshot_checker_ptr != nullptr) { + // If snapshot_checker is used, that means the flush/compaction may + // contain values not visible to snapshot taken after + // flush/compaction job starts. Take a snapshot and it will appear + // in snapshot_seqs and force compaction iterator to consider such + // snapshots. + const Snapshot* job_snapshot = + GetSnapshotImpl(false /*write_conflict_boundary*/, false /*lock*/); + job_context->job_snapshot.reset(new ManagedSnapshot(this, job_snapshot)); + } + *snapshot_seqs = snapshots_.GetAll(earliest_write_conflict_snapshot); +} + +Status DBImpl::WaitForCompact( + const WaitForCompactOptions& wait_for_compact_options) { + InstrumentedMutexLock l(&mutex_); + if (wait_for_compact_options.flush) { + Status s = DBImpl::FlushAllColumnFamilies(FlushOptions(), + FlushReason::kManualFlush); + if (!s.ok()) { + return s; + } + } + TEST_SYNC_POINT("DBImpl::WaitForCompact:StartWaiting"); + for (;;) { + if (shutting_down_.load(std::memory_order_acquire)) { + return Status::ShutdownInProgress(); + } + if (bg_work_paused_ && wait_for_compact_options.abort_on_pause) { + return Status::Aborted(); + } + if ((bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ || + bg_flush_scheduled_ || unscheduled_compactions_ || + unscheduled_flushes_) && + (error_handler_.GetBGError().ok())) { + bg_cv_.Wait(); + } else { + return error_handler_.GetBGError(); + } + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/db_impl/db_impl_debug.cc b/librocksdb-sys/rocksdb/db/db_impl/db_impl_debug.cc new file mode 100644 index 0000000..be63637 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_impl/db_impl_debug.cc @@ -0,0 +1,317 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef NDEBUG + +#include "db/column_family.h" +#include "db/db_impl/db_impl.h" +#include "db/error_handler.h" +#include "db/periodic_task_scheduler.h" +#include "monitoring/thread_status_updater.h" +#include "util/cast_util.h" + +namespace ROCKSDB_NAMESPACE { +uint64_t DBImpl::TEST_GetLevel0TotalSize() { + InstrumentedMutexLock l(&mutex_); + return default_cf_handle_->cfd()->current()->storage_info()->NumLevelBytes(0); +} + +Status DBImpl::TEST_SwitchWAL() { + WriteContext write_context; + InstrumentedMutexLock l(&mutex_); + void* writer = TEST_BeginWrite(); + auto s = SwitchWAL(&write_context); + TEST_EndWrite(writer); + return s; +} + +uint64_t DBImpl::TEST_MaxNextLevelOverlappingBytes( + ColumnFamilyHandle* column_family) { + ColumnFamilyData* cfd; + if (column_family == nullptr) { + cfd = default_cf_handle_->cfd(); + } else { + auto cfh = static_cast_with_check(column_family); + cfd = cfh->cfd(); + } + InstrumentedMutexLock l(&mutex_); + return cfd->current()->storage_info()->MaxNextLevelOverlappingBytes(); +} + +void DBImpl::TEST_GetFilesMetaData( + ColumnFamilyHandle* column_family, + std::vector>* metadata, + std::vector>* blob_metadata) { + assert(metadata); + + auto cfh = static_cast_with_check(column_family); + assert(cfh); + + auto cfd = cfh->cfd(); + assert(cfd); + + InstrumentedMutexLock l(&mutex_); + + const auto* current = cfd->current(); + assert(current); + + const auto* vstorage = current->storage_info(); + assert(vstorage); + + metadata->resize(NumberLevels()); + + for (int level = 0; level < NumberLevels(); ++level) { + const std::vector& files = vstorage->LevelFiles(level); + + (*metadata)[level].clear(); + (*metadata)[level].reserve(files.size()); + + for (const auto& f : files) { + (*metadata)[level].push_back(*f); + } + } + + if (blob_metadata) { + *blob_metadata = vstorage->GetBlobFiles(); + } +} + +uint64_t DBImpl::TEST_Current_Manifest_FileNo() { + return versions_->manifest_file_number(); +} + +uint64_t DBImpl::TEST_Current_Next_FileNo() { + return versions_->current_next_file_number(); +} + +Status DBImpl::TEST_CompactRange(int level, const Slice* begin, + const Slice* end, + ColumnFamilyHandle* column_family, + bool disallow_trivial_move) { + ColumnFamilyData* cfd; + if (column_family == nullptr) { + cfd = default_cf_handle_->cfd(); + } else { + auto cfh = static_cast_with_check(column_family); + cfd = cfh->cfd(); + } + int output_level = + (cfd->ioptions()->compaction_style == kCompactionStyleUniversal || + cfd->ioptions()->compaction_style == kCompactionStyleFIFO) + ? level + : level + 1; + return RunManualCompaction( + cfd, level, output_level, CompactRangeOptions(), begin, end, true, + disallow_trivial_move, + std::numeric_limits::max() /*max_file_num_to_ignore*/, + "" /*trim_ts*/); +} + +Status DBImpl::TEST_SwitchMemtable(ColumnFamilyData* cfd) { + WriteContext write_context; + InstrumentedMutexLock l(&mutex_); + if (cfd == nullptr) { + cfd = default_cf_handle_->cfd(); + } + + Status s; + void* writer = TEST_BeginWrite(); + if (two_write_queues_) { + WriteThread::Writer nonmem_w; + nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_); + s = SwitchMemtable(cfd, &write_context); + nonmem_write_thread_.ExitUnbatched(&nonmem_w); + } else { + s = SwitchMemtable(cfd, &write_context); + } + TEST_EndWrite(writer); + return s; +} + +Status DBImpl::TEST_FlushMemTable(bool wait, bool allow_write_stall, + ColumnFamilyHandle* cfh) { + FlushOptions fo; + fo.wait = wait; + fo.allow_write_stall = allow_write_stall; + ColumnFamilyData* cfd; + if (cfh == nullptr) { + cfd = default_cf_handle_->cfd(); + } else { + auto cfhi = static_cast_with_check(cfh); + cfd = cfhi->cfd(); + } + return FlushMemTable(cfd, fo, FlushReason::kTest); +} + +Status DBImpl::TEST_FlushMemTable(ColumnFamilyData* cfd, + const FlushOptions& flush_opts) { + return FlushMemTable(cfd, flush_opts, FlushReason::kTest); +} + +Status DBImpl::TEST_AtomicFlushMemTables( + const autovector& provided_candidate_cfds, + const FlushOptions& flush_opts) { + return AtomicFlushMemTables(flush_opts, FlushReason::kTest, + provided_candidate_cfds); +} + +Status DBImpl::TEST_WaitForBackgroundWork() { + InstrumentedMutexLock l(&mutex_); + WaitForBackgroundWork(); + return error_handler_.GetBGError(); +} + +Status DBImpl::TEST_WaitForFlushMemTable(ColumnFamilyHandle* column_family) { + ColumnFamilyData* cfd; + if (column_family == nullptr) { + cfd = default_cf_handle_->cfd(); + } else { + auto cfh = static_cast_with_check(column_family); + cfd = cfh->cfd(); + } + return WaitForFlushMemTable(cfd, nullptr, false); +} + +Status DBImpl::TEST_WaitForCompact() { + return WaitForCompact(WaitForCompactOptions()); +} +Status DBImpl::TEST_WaitForCompact( + const WaitForCompactOptions& wait_for_compact_options) { + return WaitForCompact(wait_for_compact_options); +} + +Status DBImpl::TEST_WaitForPurge() { + InstrumentedMutexLock l(&mutex_); + while (bg_purge_scheduled_ && error_handler_.GetBGError().ok()) { + bg_cv_.Wait(); + } + return error_handler_.GetBGError(); +} + +Status DBImpl::TEST_GetBGError() { + InstrumentedMutexLock l(&mutex_); + return error_handler_.GetBGError(); +} + +void DBImpl::TEST_LockMutex() { mutex_.Lock(); } + +void DBImpl::TEST_UnlockMutex() { mutex_.Unlock(); } + +void DBImpl::TEST_SignalAllBgCv() { bg_cv_.SignalAll(); } + +void* DBImpl::TEST_BeginWrite() { + auto w = new WriteThread::Writer(); + write_thread_.EnterUnbatched(w, &mutex_); + return reinterpret_cast(w); +} + +void DBImpl::TEST_EndWrite(void* w) { + auto writer = reinterpret_cast(w); + write_thread_.ExitUnbatched(writer); + delete writer; +} + +size_t DBImpl::TEST_LogsToFreeSize() { + InstrumentedMutexLock l(&log_write_mutex_); + return logs_to_free_.size(); +} + +uint64_t DBImpl::TEST_LogfileNumber() { + InstrumentedMutexLock l(&mutex_); + return logfile_number_; +} + +Status DBImpl::TEST_GetAllImmutableCFOptions( + std::unordered_map* iopts_map) { + std::vector cf_names; + std::vector iopts; + { + InstrumentedMutexLock l(&mutex_); + for (auto cfd : *versions_->GetColumnFamilySet()) { + cf_names.push_back(cfd->GetName()); + iopts.push_back(cfd->ioptions()); + } + } + iopts_map->clear(); + for (size_t i = 0; i < cf_names.size(); ++i) { + iopts_map->insert({cf_names[i], iopts[i]}); + } + + return Status::OK(); +} + +uint64_t DBImpl::TEST_FindMinLogContainingOutstandingPrep() { + return logs_with_prep_tracker_.FindMinLogContainingOutstandingPrep(); +} + +size_t DBImpl::TEST_PreparedSectionCompletedSize() { + return logs_with_prep_tracker_.TEST_PreparedSectionCompletedSize(); +} + +size_t DBImpl::TEST_LogsWithPrepSize() { + return logs_with_prep_tracker_.TEST_LogsWithPrepSize(); +} + +uint64_t DBImpl::TEST_FindMinPrepLogReferencedByMemTable() { + autovector empty_list; + return FindMinPrepLogReferencedByMemTable(versions_.get(), empty_list); +} + +Status DBImpl::TEST_GetLatestMutableCFOptions( + ColumnFamilyHandle* column_family, MutableCFOptions* mutable_cf_options) { + InstrumentedMutexLock l(&mutex_); + + auto cfh = static_cast_with_check(column_family); + *mutable_cf_options = *cfh->cfd()->GetLatestMutableCFOptions(); + return Status::OK(); +} + +int DBImpl::TEST_BGCompactionsAllowed() const { + InstrumentedMutexLock l(&mutex_); + return GetBGJobLimits().max_compactions; +} + +int DBImpl::TEST_BGFlushesAllowed() const { + InstrumentedMutexLock l(&mutex_); + return GetBGJobLimits().max_flushes; +} + +SequenceNumber DBImpl::TEST_GetLastVisibleSequence() const { + if (last_seq_same_as_publish_seq_) { + return versions_->LastSequence(); + } else { + return versions_->LastAllocatedSequence(); + } +} + +size_t DBImpl::TEST_GetWalPreallocateBlockSize( + uint64_t write_buffer_size) const { + InstrumentedMutexLock l(&mutex_); + return GetWalPreallocateBlockSize(write_buffer_size); +} + +void DBImpl::TEST_WaitForPeriodicTaskRun(std::function callback) const { + periodic_task_scheduler_.TEST_WaitForRun(callback); +} + +const PeriodicTaskScheduler& DBImpl::TEST_GetPeriodicTaskScheduler() const { + return periodic_task_scheduler_; +} + +SeqnoToTimeMapping DBImpl::TEST_GetSeqnoToTimeMapping() const { + InstrumentedMutexLock l(&mutex_); + return seqno_time_mapping_; +} + + +size_t DBImpl::TEST_EstimateInMemoryStatsHistorySize() const { + return EstimateInMemoryStatsHistorySize(); +} +} // namespace ROCKSDB_NAMESPACE +#endif // NDEBUG diff --git a/librocksdb-sys/rocksdb/db/db_impl/db_impl_experimental.cc b/librocksdb-sys/rocksdb/db/db_impl/db_impl_experimental.cc new file mode 100644 index 0000000..442cb47 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_impl/db_impl_experimental.cc @@ -0,0 +1,160 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include + +#include "db/column_family.h" +#include "db/db_impl/db_impl.h" +#include "db/job_context.h" +#include "db/version_set.h" +#include "logging/logging.h" +#include "rocksdb/status.h" +#include "util/cast_util.h" + +namespace ROCKSDB_NAMESPACE { + +Status DBImpl::SuggestCompactRange(ColumnFamilyHandle* column_family, + const Slice* begin, const Slice* end) { + auto cfh = static_cast_with_check(column_family); + auto cfd = cfh->cfd(); + InternalKey start_key, end_key; + if (begin != nullptr) { + start_key.SetMinPossibleForUserKey(*begin); + } + if (end != nullptr) { + end_key.SetMaxPossibleForUserKey(*end); + } + { + InstrumentedMutexLock l(&mutex_); + auto vstorage = cfd->current()->storage_info(); + for (int level = 0; level < vstorage->num_non_empty_levels() - 1; ++level) { + std::vector inputs; + vstorage->GetOverlappingInputs( + level, begin == nullptr ? nullptr : &start_key, + end == nullptr ? nullptr : &end_key, &inputs); + for (auto f : inputs) { + f->marked_for_compaction = true; + } + } + // Since we have some more files to compact, we should also recompute + // compaction score + vstorage->ComputeCompactionScore(*cfd->ioptions(), + *cfd->GetLatestMutableCFOptions()); + SchedulePendingCompaction(cfd); + MaybeScheduleFlushOrCompaction(); + } + return Status::OK(); +} + +Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) { + assert(column_family); + + if (target_level < 1) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "PromoteL0 FAILED. Invalid target level %d\n", target_level); + return Status::InvalidArgument("Invalid target level"); + } + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + Status status; + VersionEdit edit; + JobContext job_context(next_job_id_.fetch_add(1), true); + { + InstrumentedMutexLock l(&mutex_); + auto* cfd = static_cast(column_family)->cfd(); + const auto* vstorage = cfd->current()->storage_info(); + + if (target_level >= vstorage->num_levels()) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "PromoteL0 FAILED. Target level %d does not exist\n", + target_level); + job_context.Clean(); + status = Status::InvalidArgument("Target level does not exist"); + return status; + } + + // Sort L0 files by range. + const InternalKeyComparator* icmp = &cfd->internal_comparator(); + auto l0_files = vstorage->LevelFiles(0); + std::sort(l0_files.begin(), l0_files.end(), + [icmp](FileMetaData* f1, FileMetaData* f2) { + return icmp->Compare(f1->largest, f2->largest) < 0; + }); + + // Check that no L0 file is being compacted and that they have + // non-overlapping ranges. + for (size_t i = 0; i < l0_files.size(); ++i) { + auto f = l0_files[i]; + if (f->being_compacted) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "PromoteL0 FAILED. File %" PRIu64 " being compacted\n", + f->fd.GetNumber()); + job_context.Clean(); + status = + Status::InvalidArgument("PromoteL0 called during L0 compaction"); + return status; + } + + if (i == 0) continue; + auto prev_f = l0_files[i - 1]; + if (icmp->Compare(prev_f->largest, f->smallest) >= 0) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "PromoteL0 FAILED. Files %" PRIu64 " and %" PRIu64 + " have overlapping ranges\n", + prev_f->fd.GetNumber(), f->fd.GetNumber()); + job_context.Clean(); + status = Status::InvalidArgument("L0 has overlapping files"); + return status; + } + } + + // Check that all levels up to target_level are empty. + for (int level = 1; level <= target_level; ++level) { + if (vstorage->NumLevelFiles(level) > 0) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "PromoteL0 FAILED. Level %d not empty\n", level); + job_context.Clean(); + status = Status::InvalidArgument( + "All levels up to target_level " + "must be empty"); + return status; + } + } + + edit.SetColumnFamily(cfd->GetID()); + for (const auto& f : l0_files) { + edit.DeleteFile(0, f->fd.GetNumber()); + edit.AddFile(target_level, f->fd.GetNumber(), f->fd.GetPathId(), + f->fd.GetFileSize(), f->smallest, f->largest, + f->fd.smallest_seqno, f->fd.largest_seqno, + f->marked_for_compaction, f->temperature, + f->oldest_blob_file_number, f->oldest_ancester_time, + f->file_creation_time, f->epoch_number, f->file_checksum, + f->file_checksum_func_name, f->unique_id, + f->compensated_range_deletion_size, f->tail_size, + f->user_defined_timestamps_persisted); + } + + status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(), + read_options, &edit, &mutex_, + directories_.GetDbDir()); + if (status.ok()) { + InstallSuperVersionAndScheduleWork(cfd, + &job_context.superversion_contexts[0], + *cfd->GetLatestMutableCFOptions()); + } + } // lock released here + LogFlush(immutable_db_options_.info_log); + job_context.Clean(); + + return status; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/db_impl/db_impl_files.cc b/librocksdb-sys/rocksdb/db/db_impl/db_impl_files.cc new file mode 100644 index 0000000..9fc9bb2 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_impl/db_impl_files.cc @@ -0,0 +1,1019 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include +#include +#include + +#include "db/db_impl/db_impl.h" +#include "db/event_helpers.h" +#include "db/memtable_list.h" +#include "file/file_util.h" +#include "file/filename.h" +#include "file/sst_file_manager_impl.h" +#include "logging/logging.h" +#include "port/port.h" +#include "util/autovector.h" +#include "util/defer.h" + +namespace ROCKSDB_NAMESPACE { + +uint64_t DBImpl::MinLogNumberToKeep() { + return versions_->min_log_number_to_keep(); +} + +uint64_t DBImpl::MinObsoleteSstNumberToKeep() { + mutex_.AssertHeld(); + if (!pending_outputs_.empty()) { + return *pending_outputs_.begin(); + } + return std::numeric_limits::max(); +} + +uint64_t DBImpl::GetObsoleteSstFilesSize() { + mutex_.AssertHeld(); + return versions_->GetObsoleteSstFilesSize(); +} + +Status DBImpl::DisableFileDeletions() { + Status s; + int my_disable_delete_obsolete_files; + { + InstrumentedMutexLock l(&mutex_); + s = DisableFileDeletionsWithLock(); + my_disable_delete_obsolete_files = disable_delete_obsolete_files_; + } + if (my_disable_delete_obsolete_files == 1) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, "File Deletions Disabled"); + } else { + ROCKS_LOG_WARN(immutable_db_options_.info_log, + "File Deletions Disabled, but already disabled. Counter: %d", + my_disable_delete_obsolete_files); + } + return s; +} + +// FIXME: can be inconsistent with DisableFileDeletions in cases like +// DBImplReadOnly +Status DBImpl::DisableFileDeletionsWithLock() { + mutex_.AssertHeld(); + ++disable_delete_obsolete_files_; + return Status::OK(); +} + +Status DBImpl::EnableFileDeletions(bool force) { + // Job id == 0 means that this is not our background process, but rather + // user thread + JobContext job_context(0); + int saved_counter; // initialize on all paths + { + InstrumentedMutexLock l(&mutex_); + if (force) { + // if force, we need to enable file deletions right away + disable_delete_obsolete_files_ = 0; + } else if (disable_delete_obsolete_files_ > 0) { + --disable_delete_obsolete_files_; + } + saved_counter = disable_delete_obsolete_files_; + if (saved_counter == 0) { + FindObsoleteFiles(&job_context, true); + bg_cv_.SignalAll(); + } + } + if (saved_counter == 0) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, "File Deletions Enabled"); + if (job_context.HaveSomethingToDelete()) { + PurgeObsoleteFiles(job_context); + } + } else { + ROCKS_LOG_WARN(immutable_db_options_.info_log, + "File Deletions Enable, but not really enabled. Counter: %d", + saved_counter); + } + job_context.Clean(); + LogFlush(immutable_db_options_.info_log); + return Status::OK(); +} + +bool DBImpl::IsFileDeletionsEnabled() const { + return 0 == disable_delete_obsolete_files_; +} + +// * Returns the list of live files in 'sst_live' and 'blob_live'. +// If it's doing full scan: +// * Returns the list of all files in the filesystem in +// 'full_scan_candidate_files'. +// Otherwise, gets obsolete files from VersionSet. +// no_full_scan = true -- never do the full scan using GetChildren() +// force = false -- don't force the full scan, except every +// mutable_db_options_.delete_obsolete_files_period_micros +// force = true -- force the full scan +void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force, + bool no_full_scan) { + mutex_.AssertHeld(); + + // if deletion is disabled, do nothing + if (disable_delete_obsolete_files_ > 0) { + return; + } + + bool doing_the_full_scan = false; + + // logic for figuring out if we're doing the full scan + if (no_full_scan) { + doing_the_full_scan = false; + } else if (force || + mutable_db_options_.delete_obsolete_files_period_micros == 0) { + doing_the_full_scan = true; + } else { + const uint64_t now_micros = immutable_db_options_.clock->NowMicros(); + if ((delete_obsolete_files_last_run_ + + mutable_db_options_.delete_obsolete_files_period_micros) < + now_micros) { + doing_the_full_scan = true; + delete_obsolete_files_last_run_ = now_micros; + } + } + + // don't delete files that might be currently written to from compaction + // threads + // Since job_context->min_pending_output is set, until file scan finishes, + // mutex_ cannot be released. Otherwise, we might see no min_pending_output + // here but later find newer generated unfinalized files while scanning. + job_context->min_pending_output = MinObsoleteSstNumberToKeep(); + + // Get obsolete files. This function will also update the list of + // pending files in VersionSet(). + versions_->GetObsoleteFiles( + &job_context->sst_delete_files, &job_context->blob_delete_files, + &job_context->manifest_delete_files, job_context->min_pending_output); + + // Mark the elements in job_context->sst_delete_files and + // job_context->blob_delete_files as "grabbed for purge" so that other threads + // calling FindObsoleteFiles with full_scan=true will not add these files to + // candidate list for purge. + for (const auto& sst_to_del : job_context->sst_delete_files) { + MarkAsGrabbedForPurge(sst_to_del.metadata->fd.GetNumber()); + } + + for (const auto& blob_file : job_context->blob_delete_files) { + MarkAsGrabbedForPurge(blob_file.GetBlobFileNumber()); + } + + // store the current filenum, lognum, etc + job_context->manifest_file_number = versions_->manifest_file_number(); + job_context->pending_manifest_file_number = + versions_->pending_manifest_file_number(); + job_context->log_number = MinLogNumberToKeep(); + job_context->prev_log_number = versions_->prev_log_number(); + + if (doing_the_full_scan) { + versions_->AddLiveFiles(&job_context->sst_live, &job_context->blob_live); + InfoLogPrefix info_log_prefix(!immutable_db_options_.db_log_dir.empty(), + dbname_); + std::set paths; + for (size_t path_id = 0; path_id < immutable_db_options_.db_paths.size(); + path_id++) { + paths.insert(immutable_db_options_.db_paths[path_id].path); + } + + // Note that if cf_paths is not specified in the ColumnFamilyOptions + // of a particular column family, we use db_paths as the cf_paths + // setting. Hence, there can be multiple duplicates of files from db_paths + // in the following code. The duplicate are removed while identifying + // unique files in PurgeObsoleteFiles. + for (auto cfd : *versions_->GetColumnFamilySet()) { + for (size_t path_id = 0; path_id < cfd->ioptions()->cf_paths.size(); + path_id++) { + auto& path = cfd->ioptions()->cf_paths[path_id].path; + + if (paths.find(path) == paths.end()) { + paths.insert(path); + } + } + } + + IOOptions io_opts; + io_opts.do_not_recurse = true; + for (auto& path : paths) { + // set of all files in the directory. We'll exclude files that are still + // alive in the subsequent processings. + std::vector files; + Status s = immutable_db_options_.fs->GetChildren( + path, io_opts, &files, /*IODebugContext*=*/nullptr); + s.PermitUncheckedError(); // TODO: What should we do on error? + for (const std::string& file : files) { + uint64_t number; + FileType type; + // 1. If we cannot parse the file name, we skip; + // 2. If the file with file_number equals number has already been + // grabbed for purge by another compaction job, or it has already been + // schedule for purge, we also skip it if we + // are doing full scan in order to avoid double deletion of the same + // file under race conditions. See + // https://github.com/facebook/rocksdb/issues/3573 + if (!ParseFileName(file, &number, info_log_prefix.prefix, &type) || + !ShouldPurge(number)) { + continue; + } + + // TODO(icanadi) clean up this mess to avoid having one-off "/" + // prefixes + job_context->full_scan_candidate_files.emplace_back("/" + file, path); + } + } + + // Add log files in wal_dir + if (!immutable_db_options_.IsWalDirSameAsDBPath(dbname_)) { + std::vector log_files; + Status s = immutable_db_options_.fs->GetChildren( + immutable_db_options_.wal_dir, io_opts, &log_files, + /*IODebugContext*=*/nullptr); + s.PermitUncheckedError(); // TODO: What should we do on error? + for (const std::string& log_file : log_files) { + job_context->full_scan_candidate_files.emplace_back( + log_file, immutable_db_options_.wal_dir); + } + } + + // Add info log files in db_log_dir + if (!immutable_db_options_.db_log_dir.empty() && + immutable_db_options_.db_log_dir != dbname_) { + std::vector info_log_files; + Status s = immutable_db_options_.fs->GetChildren( + immutable_db_options_.db_log_dir, io_opts, &info_log_files, + /*IODebugContext*=*/nullptr); + s.PermitUncheckedError(); // TODO: What should we do on error? + for (std::string& log_file : info_log_files) { + job_context->full_scan_candidate_files.emplace_back( + log_file, immutable_db_options_.db_log_dir); + } + } + } else { + // Instead of filling ob_context->sst_live and job_context->blob_live, + // directly remove files that show up in any Version. This is because + // candidate files tend to be a small percentage of all files, so it is + // usually cheaper to check them against every version, compared to + // building a map for all files. + versions_->RemoveLiveFiles(job_context->sst_delete_files, + job_context->blob_delete_files); + } + + // Before potentially releasing mutex and waiting on condvar, increment + // pending_purge_obsolete_files_ so that another thread executing + // `GetSortedWals` will wait until this thread finishes execution since the + // other thread will be waiting for `pending_purge_obsolete_files_`. + // pending_purge_obsolete_files_ MUST be decremented if there is nothing to + // delete. + ++pending_purge_obsolete_files_; + + Defer cleanup([job_context, this]() { + assert(job_context != nullptr); + if (!job_context->HaveSomethingToDelete()) { + mutex_.AssertHeld(); + --pending_purge_obsolete_files_; + } + }); + + // logs_ is empty when called during recovery, in which case there can't yet + // be any tracked obsolete logs + log_write_mutex_.Lock(); + + if (alive_log_files_.empty() || logs_.empty()) { + mutex_.AssertHeld(); + // We may reach here if the db is DBImplSecondary + log_write_mutex_.Unlock(); + return; + } + + bool mutex_unlocked = false; + if (!alive_log_files_.empty() && !logs_.empty()) { + uint64_t min_log_number = job_context->log_number; + size_t num_alive_log_files = alive_log_files_.size(); + // find newly obsoleted log files + while (alive_log_files_.begin()->number < min_log_number) { + auto& earliest = *alive_log_files_.begin(); + if (immutable_db_options_.recycle_log_file_num > + log_recycle_files_.size()) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "adding log %" PRIu64 " to recycle list\n", + earliest.number); + log_recycle_files_.push_back(earliest.number); + } else { + job_context->log_delete_files.push_back(earliest.number); + } + if (job_context->size_log_to_delete == 0) { + job_context->prev_total_log_size = total_log_size_; + job_context->num_alive_log_files = num_alive_log_files; + } + job_context->size_log_to_delete += earliest.size; + total_log_size_ -= earliest.size; + alive_log_files_.pop_front(); + + // Current log should always stay alive since it can't have + // number < MinLogNumber(). + assert(alive_log_files_.size()); + } + log_write_mutex_.Unlock(); + mutex_.Unlock(); + mutex_unlocked = true; + TEST_SYNC_POINT_CALLBACK("FindObsoleteFiles::PostMutexUnlock", nullptr); + log_write_mutex_.Lock(); + while (!logs_.empty() && logs_.front().number < min_log_number) { + auto& log = logs_.front(); + if (log.IsSyncing()) { + log_sync_cv_.Wait(); + // logs_ could have changed while we were waiting. + continue; + } + logs_to_free_.push_back(log.ReleaseWriter()); + logs_.pop_front(); + } + // Current log cannot be obsolete. + assert(!logs_.empty()); + } + + // We're just cleaning up for DB::Write(). + assert(job_context->logs_to_free.empty()); + job_context->logs_to_free = logs_to_free_; + + logs_to_free_.clear(); + log_write_mutex_.Unlock(); + if (mutex_unlocked) { + mutex_.Lock(); + } + job_context->log_recycle_files.assign(log_recycle_files_.begin(), + log_recycle_files_.end()); +} + +// Delete obsolete files and log status and information of file deletion +void DBImpl::DeleteObsoleteFileImpl(int job_id, const std::string& fname, + const std::string& path_to_sync, + FileType type, uint64_t number) { + TEST_SYNC_POINT_CALLBACK("DBImpl::DeleteObsoleteFileImpl::BeforeDeletion", + const_cast(&fname)); + + Status file_deletion_status; + if (type == kTableFile || type == kBlobFile || type == kWalFile) { + // Rate limit WAL deletion only if its in the DB dir + file_deletion_status = DeleteDBFile( + &immutable_db_options_, fname, path_to_sync, + /*force_bg=*/false, + /*force_fg=*/(type == kWalFile) ? !wal_in_db_path_ : false); + } else { + file_deletion_status = env_->DeleteFile(fname); + } + TEST_SYNC_POINT_CALLBACK("DBImpl::DeleteObsoleteFileImpl:AfterDeletion", + &file_deletion_status); + if (file_deletion_status.ok()) { + ROCKS_LOG_DEBUG(immutable_db_options_.info_log, + "[JOB %d] Delete %s type=%d #%" PRIu64 " -- %s\n", job_id, + fname.c_str(), type, number, + file_deletion_status.ToString().c_str()); + } else if (env_->FileExists(fname).IsNotFound()) { + ROCKS_LOG_INFO( + immutable_db_options_.info_log, + "[JOB %d] Tried to delete a non-existing file %s type=%d #%" PRIu64 + " -- %s\n", + job_id, fname.c_str(), type, number, + file_deletion_status.ToString().c_str()); + } else { + ROCKS_LOG_ERROR(immutable_db_options_.info_log, + "[JOB %d] Failed to delete %s type=%d #%" PRIu64 " -- %s\n", + job_id, fname.c_str(), type, number, + file_deletion_status.ToString().c_str()); + } + if (type == kTableFile) { + EventHelpers::LogAndNotifyTableFileDeletion( + &event_logger_, job_id, number, fname, file_deletion_status, GetName(), + immutable_db_options_.listeners); + } + if (type == kBlobFile) { + EventHelpers::LogAndNotifyBlobFileDeletion( + &event_logger_, immutable_db_options_.listeners, job_id, number, fname, + file_deletion_status, GetName()); + } +} + +// Diffs the files listed in filenames and those that do not +// belong to live files are possibly removed. Also, removes all the +// files in sst_delete_files and log_delete_files. +// It is not necessary to hold the mutex when invoking this method. +void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) { + TEST_SYNC_POINT("DBImpl::PurgeObsoleteFiles:Begin"); + // we'd better have sth to delete + assert(state.HaveSomethingToDelete()); + + // FindObsoleteFiles() should've populated this so nonzero + assert(state.manifest_file_number != 0); + + // Now, convert lists to unordered sets, WITHOUT mutex held; set is slow. + std::unordered_set sst_live_set(state.sst_live.begin(), + state.sst_live.end()); + std::unordered_set blob_live_set(state.blob_live.begin(), + state.blob_live.end()); + std::unordered_set log_recycle_files_set( + state.log_recycle_files.begin(), state.log_recycle_files.end()); + + auto candidate_files = state.full_scan_candidate_files; + candidate_files.reserve( + candidate_files.size() + state.sst_delete_files.size() + + state.blob_delete_files.size() + state.log_delete_files.size() + + state.manifest_delete_files.size()); + // We may ignore the dbname when generating the file names. + for (auto& file : state.sst_delete_files) { + if (!file.only_delete_metadata) { + candidate_files.emplace_back( + MakeTableFileName(file.metadata->fd.GetNumber()), file.path); + } + if (file.metadata->table_reader_handle) { + table_cache_->Release(file.metadata->table_reader_handle); + } + file.DeleteMetadata(); + } + + for (const auto& blob_file : state.blob_delete_files) { + candidate_files.emplace_back(BlobFileName(blob_file.GetBlobFileNumber()), + blob_file.GetPath()); + } + + auto wal_dir = immutable_db_options_.GetWalDir(); + for (auto file_num : state.log_delete_files) { + if (file_num > 0) { + candidate_files.emplace_back(LogFileName(file_num), wal_dir); + } + } + for (const auto& filename : state.manifest_delete_files) { + candidate_files.emplace_back(filename, dbname_); + } + + // dedup state.candidate_files so we don't try to delete the same + // file twice + std::sort(candidate_files.begin(), candidate_files.end(), + [](const JobContext::CandidateFileInfo& lhs, + const JobContext::CandidateFileInfo& rhs) { + if (lhs.file_name > rhs.file_name) { + return true; + } else if (lhs.file_name < rhs.file_name) { + return false; + } else { + return (lhs.file_path > rhs.file_path); + } + }); + candidate_files.erase( + std::unique(candidate_files.begin(), candidate_files.end()), + candidate_files.end()); + + if (state.prev_total_log_size > 0) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "[JOB %d] Try to delete WAL files size %" PRIu64 + ", prev total WAL file size %" PRIu64 + ", number of live WAL files %" ROCKSDB_PRIszt ".\n", + state.job_id, state.size_log_to_delete, + state.prev_total_log_size, state.num_alive_log_files); + } + + std::vector old_info_log_files; + InfoLogPrefix info_log_prefix(!immutable_db_options_.db_log_dir.empty(), + dbname_); + + // File numbers of most recent two OPTIONS file in candidate_files (found in + // previos FindObsoleteFiles(full_scan=true)) + // At this point, there must not be any duplicate file numbers in + // candidate_files. + uint64_t optsfile_num1 = std::numeric_limits::min(); + uint64_t optsfile_num2 = std::numeric_limits::min(); + for (const auto& candidate_file : candidate_files) { + const std::string& fname = candidate_file.file_name; + uint64_t number; + FileType type; + if (!ParseFileName(fname, &number, info_log_prefix.prefix, &type) || + type != kOptionsFile) { + continue; + } + if (number > optsfile_num1) { + optsfile_num2 = optsfile_num1; + optsfile_num1 = number; + } else if (number > optsfile_num2) { + optsfile_num2 = number; + } + } + + // Close WALs before trying to delete them. + for (const auto w : state.logs_to_free) { + // TODO: maybe check the return value of Close. + auto s = w->Close(); + s.PermitUncheckedError(); + } + + bool own_files = OwnTablesAndLogs(); + std::unordered_set files_to_del; + for (const auto& candidate_file : candidate_files) { + const std::string& to_delete = candidate_file.file_name; + uint64_t number; + FileType type; + // Ignore file if we cannot recognize it. + if (!ParseFileName(to_delete, &number, info_log_prefix.prefix, &type)) { + continue; + } + + bool keep = true; + switch (type) { + case kWalFile: + keep = ((number >= state.log_number) || + (number == state.prev_log_number) || + (log_recycle_files_set.find(number) != + log_recycle_files_set.end())); + break; + case kDescriptorFile: + // Keep my manifest file, and any newer incarnations' + // (can happen during manifest roll) + keep = (number >= state.manifest_file_number); + break; + case kTableFile: + // If the second condition is not there, this makes + // DontDeletePendingOutputs fail + keep = (sst_live_set.find(number) != sst_live_set.end()) || + number >= state.min_pending_output; + if (!keep) { + files_to_del.insert(number); + } + break; + case kBlobFile: + keep = number >= state.min_pending_output || + (blob_live_set.find(number) != blob_live_set.end()); + if (!keep) { + files_to_del.insert(number); + } + break; + case kTempFile: + // Any temp files that are currently being written to must + // be recorded in pending_outputs_, which is inserted into "live". + // Also, SetCurrentFile creates a temp file when writing out new + // manifest, which is equal to state.pending_manifest_file_number. We + // should not delete that file + // + // TODO(yhchiang): carefully modify the third condition to safely + // remove the temp options files. + keep = (sst_live_set.find(number) != sst_live_set.end()) || + (blob_live_set.find(number) != blob_live_set.end()) || + (number == state.pending_manifest_file_number) || + (to_delete.find(kOptionsFileNamePrefix) != std::string::npos); + break; + case kInfoLogFile: + keep = true; + if (number != 0) { + old_info_log_files.push_back(to_delete); + } + break; + case kOptionsFile: + keep = (number >= optsfile_num2); + break; + case kCurrentFile: + case kDBLockFile: + case kIdentityFile: + case kMetaDatabase: + keep = true; + break; + } + + if (keep) { + continue; + } + + std::string fname; + std::string dir_to_sync; + if (type == kTableFile) { + // evict from cache + TableCache::Evict(table_cache_.get(), number); + fname = MakeTableFileName(candidate_file.file_path, number); + dir_to_sync = candidate_file.file_path; + } else if (type == kBlobFile) { + fname = BlobFileName(candidate_file.file_path, number); + dir_to_sync = candidate_file.file_path; + } else { + dir_to_sync = (type == kWalFile) ? wal_dir : dbname_; + fname = dir_to_sync + + ((!dir_to_sync.empty() && dir_to_sync.back() == '/') || + (!to_delete.empty() && to_delete.front() == '/') + ? "" + : "/") + + to_delete; + } + + if (type == kWalFile && (immutable_db_options_.WAL_ttl_seconds > 0 || + immutable_db_options_.WAL_size_limit_MB > 0)) { + wal_manager_.ArchiveWALFile(fname, number); + continue; + } + + // If I do not own these files, e.g. secondary instance with max_open_files + // = -1, then no need to delete or schedule delete these files since they + // will be removed by their owner, e.g. the primary instance. + if (!own_files) { + continue; + } + if (schedule_only) { + InstrumentedMutexLock guard_lock(&mutex_); + SchedulePendingPurge(fname, dir_to_sync, type, number, state.job_id); + } else { + DeleteObsoleteFileImpl(state.job_id, fname, dir_to_sync, type, number); + } + } + + { + // After purging obsolete files, remove them from files_grabbed_for_purge_. + InstrumentedMutexLock guard_lock(&mutex_); + autovector to_be_removed; + for (auto fn : files_grabbed_for_purge_) { + if (files_to_del.count(fn) != 0) { + to_be_removed.emplace_back(fn); + } + } + for (auto fn : to_be_removed) { + files_grabbed_for_purge_.erase(fn); + } + } + + // Delete old info log files. + size_t old_info_log_file_count = old_info_log_files.size(); + if (old_info_log_file_count != 0 && + old_info_log_file_count >= immutable_db_options_.keep_log_file_num) { + std::sort(old_info_log_files.begin(), old_info_log_files.end()); + size_t end = + old_info_log_file_count - immutable_db_options_.keep_log_file_num; + for (unsigned int i = 0; i <= end; i++) { + std::string& to_delete = old_info_log_files.at(i); + std::string full_path_to_delete = + (immutable_db_options_.db_log_dir.empty() + ? dbname_ + : immutable_db_options_.db_log_dir) + + "/" + to_delete; + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "[JOB %d] Delete info log file %s\n", state.job_id, + full_path_to_delete.c_str()); + Status s = env_->DeleteFile(full_path_to_delete); + if (!s.ok()) { + if (env_->FileExists(full_path_to_delete).IsNotFound()) { + ROCKS_LOG_INFO( + immutable_db_options_.info_log, + "[JOB %d] Tried to delete non-existing info log file %s FAILED " + "-- %s\n", + state.job_id, to_delete.c_str(), s.ToString().c_str()); + } else { + ROCKS_LOG_ERROR(immutable_db_options_.info_log, + "[JOB %d] Delete info log file %s FAILED -- %s\n", + state.job_id, to_delete.c_str(), + s.ToString().c_str()); + } + } + } + } + wal_manager_.PurgeObsoleteWALFiles(); + LogFlush(immutable_db_options_.info_log); + InstrumentedMutexLock l(&mutex_); + --pending_purge_obsolete_files_; + assert(pending_purge_obsolete_files_ >= 0); + if (schedule_only) { + // Must change from pending_purge_obsolete_files_ to bg_purge_scheduled_ + // while holding mutex (for GetSortedWalFiles() etc.) + SchedulePurge(); + } + if (pending_purge_obsolete_files_ == 0) { + bg_cv_.SignalAll(); + } + TEST_SYNC_POINT("DBImpl::PurgeObsoleteFiles:End"); +} + +void DBImpl::DeleteObsoleteFiles() { + mutex_.AssertHeld(); + JobContext job_context(next_job_id_.fetch_add(1)); + FindObsoleteFiles(&job_context, true); + + mutex_.Unlock(); + if (job_context.HaveSomethingToDelete()) { + bool defer_purge = immutable_db_options_.avoid_unnecessary_blocking_io; + PurgeObsoleteFiles(job_context, defer_purge); + } + job_context.Clean(); + mutex_.Lock(); +} + +uint64_t FindMinPrepLogReferencedByMemTable( + VersionSet* vset, const autovector& memtables_to_flush) { + uint64_t min_log = 0; + + // we must look through the memtables for two phase transactions + // that have been committed but not yet flushed + std::unordered_set memtables_to_flush_set( + memtables_to_flush.begin(), memtables_to_flush.end()); + for (auto loop_cfd : *vset->GetColumnFamilySet()) { + if (loop_cfd->IsDropped()) { + continue; + } + + auto log = loop_cfd->imm()->PrecomputeMinLogContainingPrepSection( + &memtables_to_flush_set); + + if (log > 0 && (min_log == 0 || log < min_log)) { + min_log = log; + } + + log = loop_cfd->mem()->GetMinLogContainingPrepSection(); + + if (log > 0 && (min_log == 0 || log < min_log)) { + min_log = log; + } + } + + return min_log; +} + +uint64_t FindMinPrepLogReferencedByMemTable( + VersionSet* vset, + const autovector*>& memtables_to_flush) { + uint64_t min_log = 0; + + std::unordered_set memtables_to_flush_set; + for (const autovector* memtables : memtables_to_flush) { + memtables_to_flush_set.insert(memtables->begin(), memtables->end()); + } + for (auto loop_cfd : *vset->GetColumnFamilySet()) { + if (loop_cfd->IsDropped()) { + continue; + } + + auto log = loop_cfd->imm()->PrecomputeMinLogContainingPrepSection( + &memtables_to_flush_set); + if (log > 0 && (min_log == 0 || log < min_log)) { + min_log = log; + } + + log = loop_cfd->mem()->GetMinLogContainingPrepSection(); + if (log > 0 && (min_log == 0 || log < min_log)) { + min_log = log; + } + } + + return min_log; +} + +uint64_t PrecomputeMinLogNumberToKeepNon2PC( + VersionSet* vset, const ColumnFamilyData& cfd_to_flush, + const autovector& edit_list) { + assert(vset != nullptr); + + // Precompute the min log number containing unflushed data for the column + // family being flushed (`cfd_to_flush`). + uint64_t cf_min_log_number_to_keep = 0; + for (auto& e : edit_list) { + if (e->HasLogNumber()) { + cf_min_log_number_to_keep = + std::max(cf_min_log_number_to_keep, e->GetLogNumber()); + } + } + if (cf_min_log_number_to_keep == 0) { + // No version edit contains information on log number. The log number + // for this column family should stay the same as it is. + cf_min_log_number_to_keep = cfd_to_flush.GetLogNumber(); + } + + // Get min log number containing unflushed data for other column families. + uint64_t min_log_number_to_keep = + vset->PreComputeMinLogNumberWithUnflushedData(&cfd_to_flush); + if (cf_min_log_number_to_keep != 0) { + min_log_number_to_keep = + std::min(cf_min_log_number_to_keep, min_log_number_to_keep); + } + return min_log_number_to_keep; +} + +uint64_t PrecomputeMinLogNumberToKeepNon2PC( + VersionSet* vset, const autovector& cfds_to_flush, + const autovector>& edit_lists) { + assert(vset != nullptr); + assert(!cfds_to_flush.empty()); + assert(cfds_to_flush.size() == edit_lists.size()); + + uint64_t min_log_number_to_keep = std::numeric_limits::max(); + for (const auto& edit_list : edit_lists) { + uint64_t log = 0; + for (const auto& e : edit_list) { + if (e->HasLogNumber()) { + log = std::max(log, e->GetLogNumber()); + } + } + if (log != 0) { + min_log_number_to_keep = std::min(min_log_number_to_keep, log); + } + } + if (min_log_number_to_keep == std::numeric_limits::max()) { + min_log_number_to_keep = cfds_to_flush[0]->GetLogNumber(); + for (size_t i = 1; i < cfds_to_flush.size(); i++) { + min_log_number_to_keep = + std::min(min_log_number_to_keep, cfds_to_flush[i]->GetLogNumber()); + } + } + + std::unordered_set flushed_cfds( + cfds_to_flush.begin(), cfds_to_flush.end()); + min_log_number_to_keep = + std::min(min_log_number_to_keep, + vset->PreComputeMinLogNumberWithUnflushedData(flushed_cfds)); + + return min_log_number_to_keep; +} + +uint64_t PrecomputeMinLogNumberToKeep2PC( + VersionSet* vset, const ColumnFamilyData& cfd_to_flush, + const autovector& edit_list, + const autovector& memtables_to_flush, + LogsWithPrepTracker* prep_tracker) { + assert(vset != nullptr); + assert(prep_tracker != nullptr); + // Calculate updated min_log_number_to_keep + // Since the function should only be called in 2pc mode, log number in + // the version edit should be sufficient. + + uint64_t min_log_number_to_keep = + PrecomputeMinLogNumberToKeepNon2PC(vset, cfd_to_flush, edit_list); + + // if are 2pc we must consider logs containing prepared + // sections of outstanding transactions. + // + // We must check min logs with outstanding prep before we check + // logs references by memtables because a log referenced by the + // first data structure could transition to the second under us. + // + // TODO: iterating over all column families under db mutex. + // should find more optimal solution + auto min_log_in_prep_heap = + prep_tracker->FindMinLogContainingOutstandingPrep(); + + if (min_log_in_prep_heap != 0 && + min_log_in_prep_heap < min_log_number_to_keep) { + min_log_number_to_keep = min_log_in_prep_heap; + } + + uint64_t min_log_refed_by_mem = + FindMinPrepLogReferencedByMemTable(vset, memtables_to_flush); + + if (min_log_refed_by_mem != 0 && + min_log_refed_by_mem < min_log_number_to_keep) { + min_log_number_to_keep = min_log_refed_by_mem; + } + return min_log_number_to_keep; +} + +uint64_t PrecomputeMinLogNumberToKeep2PC( + VersionSet* vset, const autovector& cfds_to_flush, + const autovector>& edit_lists, + const autovector*>& memtables_to_flush, + LogsWithPrepTracker* prep_tracker) { + assert(vset != nullptr); + assert(prep_tracker != nullptr); + assert(cfds_to_flush.size() == edit_lists.size()); + assert(cfds_to_flush.size() == memtables_to_flush.size()); + + uint64_t min_log_number_to_keep = + PrecomputeMinLogNumberToKeepNon2PC(vset, cfds_to_flush, edit_lists); + + uint64_t min_log_in_prep_heap = + prep_tracker->FindMinLogContainingOutstandingPrep(); + + if (min_log_in_prep_heap != 0 && + min_log_in_prep_heap < min_log_number_to_keep) { + min_log_number_to_keep = min_log_in_prep_heap; + } + + uint64_t min_log_refed_by_mem = + FindMinPrepLogReferencedByMemTable(vset, memtables_to_flush); + + if (min_log_refed_by_mem != 0 && + min_log_refed_by_mem < min_log_number_to_keep) { + min_log_number_to_keep = min_log_refed_by_mem; + } + + return min_log_number_to_keep; +} + +void DBImpl::SetDBId(std::string&& id, bool read_only, + RecoveryContext* recovery_ctx) { + assert(db_id_.empty()); + assert(!id.empty()); + db_id_ = std::move(id); + if (!read_only && immutable_db_options_.write_dbid_to_manifest) { + assert(recovery_ctx != nullptr); + assert(versions_->GetColumnFamilySet() != nullptr); + VersionEdit edit; + edit.SetDBId(db_id_); + versions_->db_id_ = db_id_; + recovery_ctx->UpdateVersionEdits( + versions_->GetColumnFamilySet()->GetDefault(), edit); + } +} + +Status DBImpl::SetupDBId(bool read_only, RecoveryContext* recovery_ctx) { + Status s; + // Check for the IDENTITY file and create it if not there or + // broken or not matching manifest + std::string db_id_in_file; + s = fs_->FileExists(IdentityFileName(dbname_), IOOptions(), nullptr); + if (s.ok()) { + s = GetDbIdentityFromIdentityFile(&db_id_in_file); + if (s.ok() && !db_id_in_file.empty()) { + if (db_id_.empty()) { + // Loaded from file and wasn't already known from manifest + SetDBId(std::move(db_id_in_file), read_only, recovery_ctx); + return s; + } else if (db_id_ == db_id_in_file) { + // Loaded from file and matches manifest + return s; + } + } + } + if (s.IsNotFound()) { + s = Status::OK(); + } + if (!s.ok()) { + assert(s.IsIOError()); + return s; + } + // Otherwise IDENTITY file is missing or no good. + // Generate new id if needed + if (db_id_.empty()) { + SetDBId(env_->GenerateUniqueId(), read_only, recovery_ctx); + } + // Persist it to IDENTITY file if allowed + if (!read_only) { + s = SetIdentityFile(env_, dbname_, db_id_); + } + return s; +} + +Status DBImpl::DeleteUnreferencedSstFiles(RecoveryContext* recovery_ctx) { + mutex_.AssertHeld(); + std::vector paths; + paths.push_back(NormalizePath(dbname_ + std::string(1, kFilePathSeparator))); + for (const auto& db_path : immutable_db_options_.db_paths) { + paths.push_back( + NormalizePath(db_path.path + std::string(1, kFilePathSeparator))); + } + for (const auto* cfd : *versions_->GetColumnFamilySet()) { + for (const auto& cf_path : cfd->ioptions()->cf_paths) { + paths.push_back( + NormalizePath(cf_path.path + std::string(1, kFilePathSeparator))); + } + } + // Dedup paths + std::sort(paths.begin(), paths.end()); + paths.erase(std::unique(paths.begin(), paths.end()), paths.end()); + + uint64_t next_file_number = versions_->current_next_file_number(); + uint64_t largest_file_number = next_file_number; + Status s; + for (const auto& path : paths) { + std::vector files; + s = env_->GetChildren(path, &files); + if (!s.ok()) { + break; + } + for (const auto& fname : files) { + uint64_t number = 0; + FileType type; + if (!ParseFileName(fname, &number, &type)) { + continue; + } + // path ends with '/' or '\\' + const std::string normalized_fpath = path + fname; + largest_file_number = std::max(largest_file_number, number); + if (type == kTableFile && number >= next_file_number && + recovery_ctx->files_to_delete_.find(normalized_fpath) == + recovery_ctx->files_to_delete_.end()) { + recovery_ctx->files_to_delete_.emplace(normalized_fpath); + } + } + } + if (!s.ok()) { + return s; + } + + if (largest_file_number >= next_file_number) { + versions_->next_file_number_.store(largest_file_number + 1); + } + + VersionEdit edit; + edit.SetNextFile(versions_->next_file_number_.load()); + assert(versions_->GetColumnFamilySet()); + ColumnFamilyData* default_cfd = versions_->GetColumnFamilySet()->GetDefault(); + assert(default_cfd); + recovery_ctx->UpdateVersionEdits(default_cfd, edit); + return s; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/db_impl/db_impl_open.cc b/librocksdb-sys/rocksdb/db/db_impl/db_impl_open.cc new file mode 100644 index 0000000..e6d97b1 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_impl/db_impl_open.cc @@ -0,0 +1,2238 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include + +#include "db/builder.h" +#include "db/db_impl/db_impl.h" +#include "db/error_handler.h" +#include "db/periodic_task_scheduler.h" +#include "env/composite_env_wrapper.h" +#include "file/filename.h" +#include "file/read_write_util.h" +#include "file/sst_file_manager_impl.h" +#include "file/writable_file_writer.h" +#include "logging/logging.h" +#include "monitoring/persistent_stats_history.h" +#include "monitoring/thread_status_util.h" +#include "options/options_helper.h" +#include "rocksdb/table.h" +#include "rocksdb/wal_filter.h" +#include "test_util/sync_point.h" +#include "util/rate_limiter_impl.h" +#include "util/udt_util.h" + +namespace ROCKSDB_NAMESPACE { +Options SanitizeOptions(const std::string& dbname, const Options& src, + bool read_only, Status* logger_creation_s) { + auto db_options = + SanitizeOptions(dbname, DBOptions(src), read_only, logger_creation_s); + ImmutableDBOptions immutable_db_options(db_options); + auto cf_options = + SanitizeOptions(immutable_db_options, ColumnFamilyOptions(src)); + return Options(db_options, cf_options); +} + +DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src, + bool read_only, Status* logger_creation_s) { + DBOptions result(src); + + if (result.env == nullptr) { + result.env = Env::Default(); + } + + // result.max_open_files means an "infinite" open files. + if (result.max_open_files != -1) { + int max_max_open_files = port::GetMaxOpenFiles(); + if (max_max_open_files == -1) { + max_max_open_files = 0x400000; + } + ClipToRange(&result.max_open_files, 20, max_max_open_files); + TEST_SYNC_POINT_CALLBACK("SanitizeOptions::AfterChangeMaxOpenFiles", + &result.max_open_files); + } + + if (result.info_log == nullptr && !read_only) { + Status s = CreateLoggerFromOptions(dbname, result, &result.info_log); + if (!s.ok()) { + // No place suitable for logging + result.info_log = nullptr; + if (logger_creation_s) { + *logger_creation_s = s; + } + } + } + + if (!result.write_buffer_manager) { + result.write_buffer_manager.reset( + new WriteBufferManager(result.db_write_buffer_size)); + } + auto bg_job_limits = DBImpl::GetBGJobLimits( + result.max_background_flushes, result.max_background_compactions, + result.max_background_jobs, true /* parallelize_compactions */); + result.env->IncBackgroundThreadsIfNeeded(bg_job_limits.max_compactions, + Env::Priority::LOW); + result.env->IncBackgroundThreadsIfNeeded(bg_job_limits.max_flushes, + Env::Priority::HIGH); + + if (result.rate_limiter.get() != nullptr) { + if (result.bytes_per_sync == 0) { + result.bytes_per_sync = 1024 * 1024; + } + } + + if (result.delayed_write_rate == 0) { + if (result.rate_limiter.get() != nullptr) { + result.delayed_write_rate = result.rate_limiter->GetBytesPerSecond(); + } + if (result.delayed_write_rate == 0) { + result.delayed_write_rate = 16 * 1024 * 1024; + } + } + + if (result.WAL_ttl_seconds > 0 || result.WAL_size_limit_MB > 0) { + result.recycle_log_file_num = false; + } + + if (result.recycle_log_file_num && + (result.wal_recovery_mode == + WALRecoveryMode::kTolerateCorruptedTailRecords || + result.wal_recovery_mode == WALRecoveryMode::kPointInTimeRecovery || + result.wal_recovery_mode == WALRecoveryMode::kAbsoluteConsistency)) { + // - kTolerateCorruptedTailRecords is inconsistent with recycle log file + // feature. WAL recycling expects recovery success upon encountering a + // corrupt record at the point where new data ends and recycled data + // remains at the tail. However, `kTolerateCorruptedTailRecords` must fail + // upon encountering any such corrupt record, as it cannot differentiate + // between this and a real corruption, which would cause committed updates + // to be truncated -- a violation of the recovery guarantee. + // - kPointInTimeRecovery and kAbsoluteConsistency are incompatible with + // recycle log file feature temporarily due to a bug found introducing a + // hole in the recovered data + // (https://github.com/facebook/rocksdb/pull/7252#issuecomment-673766236). + // Besides this bug, we believe the features are fundamentally compatible. + result.recycle_log_file_num = 0; + } + + if (result.db_paths.size() == 0) { + result.db_paths.emplace_back(dbname, std::numeric_limits::max()); + } else if (result.wal_dir.empty()) { + // Use dbname as default + result.wal_dir = dbname; + } + if (!result.wal_dir.empty()) { + // If there is a wal_dir already set, check to see if the wal_dir is the + // same as the dbname AND the same as the db_path[0] (which must exist from + // a few lines ago). If the wal_dir matches both of these values, then clear + // the wal_dir value, which will make wal_dir == dbname. Most likely this + // condition was the result of reading an old options file where we forced + // wal_dir to be set (to dbname). + auto npath = NormalizePath(dbname + "/"); + if (npath == NormalizePath(result.wal_dir + "/") && + npath == NormalizePath(result.db_paths[0].path + "/")) { + result.wal_dir.clear(); + } + } + + if (!result.wal_dir.empty() && result.wal_dir.back() == '/') { + result.wal_dir = result.wal_dir.substr(0, result.wal_dir.size() - 1); + } + + if (result.compaction_readahead_size == 0) { + if (result.use_direct_reads) { + TEST_SYNC_POINT_CALLBACK("SanitizeOptions:direct_io", nullptr); + } + result.compaction_readahead_size = 1024 * 1024 * 2; + } + + // Force flush on DB open if 2PC is enabled, since with 2PC we have no + // guarantee that consecutive log files have consecutive sequence id, which + // make recovery complicated. + if (result.allow_2pc) { + result.avoid_flush_during_recovery = false; + } + + ImmutableDBOptions immutable_db_options(result); + if (!immutable_db_options.IsWalDirSameAsDBPath()) { + // Either the WAL dir and db_paths[0]/db_name are not the same, or we + // cannot tell for sure. In either case, assume they're different and + // explicitly cleanup the trash log files (bypass DeleteScheduler) + // Do this first so even if we end up calling + // DeleteScheduler::CleanupDirectory on the same dir later, it will be + // safe + std::vector filenames; + IOOptions io_opts; + io_opts.do_not_recurse = true; + auto wal_dir = immutable_db_options.GetWalDir(); + Status s = immutable_db_options.fs->GetChildren( + wal_dir, io_opts, &filenames, /*IODebugContext*=*/nullptr); + s.PermitUncheckedError(); //**TODO: What to do on error? + for (std::string& filename : filenames) { + if (filename.find(".log.trash", filename.length() - + std::string(".log.trash").length()) != + std::string::npos) { + std::string trash_file = wal_dir + "/" + filename; + result.env->DeleteFile(trash_file).PermitUncheckedError(); + } + } + } + // When the DB is stopped, it's possible that there are some .trash files that + // were not deleted yet, when we open the DB we will find these .trash files + // and schedule them to be deleted (or delete immediately if SstFileManager + // was not used) + auto sfm = static_cast(result.sst_file_manager.get()); + for (size_t i = 0; i < result.db_paths.size(); i++) { + DeleteScheduler::CleanupDirectory(result.env, sfm, result.db_paths[i].path) + .PermitUncheckedError(); + } + + // Create a default SstFileManager for purposes of tracking compaction size + // and facilitating recovery from out of space errors. + if (result.sst_file_manager.get() == nullptr) { + std::shared_ptr sst_file_manager( + NewSstFileManager(result.env, result.info_log)); + result.sst_file_manager = sst_file_manager; + } + + // Supported wal compression types + if (!StreamingCompressionTypeSupported(result.wal_compression)) { + result.wal_compression = kNoCompression; + ROCKS_LOG_WARN(result.info_log, + "wal_compression is disabled since only zstd is supported"); + } + + if (!result.paranoid_checks) { + result.skip_checking_sst_file_sizes_on_db_open = true; + ROCKS_LOG_INFO(result.info_log, + "file size check will be skipped during open."); + } + + return result; +} + +namespace { +Status ValidateOptionsByTable( + const DBOptions& db_opts, + const std::vector& column_families) { + Status s; + for (auto& cf : column_families) { + s = ValidateOptions(db_opts, cf.options); + if (!s.ok()) { + return s; + } + } + return Status::OK(); +} +} // namespace + +Status DBImpl::ValidateOptions( + const DBOptions& db_options, + const std::vector& column_families) { + Status s; + for (auto& cfd : column_families) { + s = ColumnFamilyData::ValidateOptions(db_options, cfd.options); + if (!s.ok()) { + return s; + } + } + s = ValidateOptions(db_options); + return s; +} + +Status DBImpl::ValidateOptions(const DBOptions& db_options) { + if (db_options.db_paths.size() > 4) { + return Status::NotSupported( + "More than four DB paths are not supported yet. "); + } + + if (db_options.allow_mmap_reads && db_options.use_direct_reads) { + // Protect against assert in PosixMMapReadableFile constructor + return Status::NotSupported( + "If memory mapped reads (allow_mmap_reads) are enabled " + "then direct I/O reads (use_direct_reads) must be disabled. "); + } + + if (db_options.allow_mmap_writes && + db_options.use_direct_io_for_flush_and_compaction) { + return Status::NotSupported( + "If memory mapped writes (allow_mmap_writes) are enabled " + "then direct I/O writes (use_direct_io_for_flush_and_compaction) must " + "be disabled. "); + } + + if (db_options.keep_log_file_num == 0) { + return Status::InvalidArgument("keep_log_file_num must be greater than 0"); + } + + if (db_options.unordered_write && + !db_options.allow_concurrent_memtable_write) { + return Status::InvalidArgument( + "unordered_write is incompatible with " + "!allow_concurrent_memtable_write"); + } + + if (db_options.unordered_write && db_options.enable_pipelined_write) { + return Status::InvalidArgument( + "unordered_write is incompatible with enable_pipelined_write"); + } + + if (db_options.atomic_flush && db_options.enable_pipelined_write) { + return Status::InvalidArgument( + "atomic_flush is incompatible with enable_pipelined_write"); + } + + // TODO remove this restriction + if (db_options.atomic_flush && db_options.best_efforts_recovery) { + return Status::InvalidArgument( + "atomic_flush is currently incompatible with best-efforts recovery"); + } + + if (db_options.use_direct_io_for_flush_and_compaction && + 0 == db_options.writable_file_max_buffer_size) { + return Status::InvalidArgument( + "writes in direct IO require writable_file_max_buffer_size > 0"); + } + + return Status::OK(); +} + +Status DBImpl::NewDB(std::vector* new_filenames) { + VersionEdit new_db; + Status s = SetIdentityFile(env_, dbname_); + if (!s.ok()) { + return s; + } + if (immutable_db_options_.write_dbid_to_manifest) { + std::string temp_db_id; + GetDbIdentityFromIdentityFile(&temp_db_id); + new_db.SetDBId(temp_db_id); + } + new_db.SetLogNumber(0); + new_db.SetNextFile(2); + new_db.SetLastSequence(0); + + ROCKS_LOG_INFO(immutable_db_options_.info_log, "Creating manifest 1 \n"); + const std::string manifest = DescriptorFileName(dbname_, 1); + { + if (fs_->FileExists(manifest, IOOptions(), nullptr).ok()) { + fs_->DeleteFile(manifest, IOOptions(), nullptr).PermitUncheckedError(); + } + std::unique_ptr file; + FileOptions file_options = fs_->OptimizeForManifestWrite(file_options_); + s = NewWritableFile(fs_.get(), manifest, &file, file_options); + if (!s.ok()) { + return s; + } + FileTypeSet tmp_set = immutable_db_options_.checksum_handoff_file_types; + file->SetPreallocationBlockSize( + immutable_db_options_.manifest_preallocation_size); + std::unique_ptr file_writer(new WritableFileWriter( + std::move(file), manifest, file_options, immutable_db_options_.clock, + io_tracer_, nullptr /* stats */, immutable_db_options_.listeners, + nullptr, tmp_set.Contains(FileType::kDescriptorFile), + tmp_set.Contains(FileType::kDescriptorFile))); + log::Writer log(std::move(file_writer), 0, false); + std::string record; + new_db.EncodeTo(&record); + s = log.AddRecord(record); + if (s.ok()) { + s = SyncManifest(&immutable_db_options_, log.file()); + } + } + if (s.ok()) { + // Make "CURRENT" file that points to the new manifest file. + s = SetCurrentFile(fs_.get(), dbname_, 1, directories_.GetDbDir()); + if (new_filenames) { + new_filenames->emplace_back( + manifest.substr(manifest.find_last_of("/\\") + 1)); + } + } else { + fs_->DeleteFile(manifest, IOOptions(), nullptr).PermitUncheckedError(); + } + return s; +} + +IOStatus DBImpl::CreateAndNewDirectory( + FileSystem* fs, const std::string& dirname, + std::unique_ptr* directory) { + // We call CreateDirIfMissing() as the directory may already exist (if we + // are reopening a DB), when this happens we don't want creating the + // directory to cause an error. However, we need to check if creating the + // directory fails or else we may get an obscure message about the lock + // file not existing. One real-world example of this occurring is if + // env->CreateDirIfMissing() doesn't create intermediate directories, e.g. + // when dbname_ is "dir/db" but when "dir" doesn't exist. + IOStatus io_s = fs->CreateDirIfMissing(dirname, IOOptions(), nullptr); + if (!io_s.ok()) { + return io_s; + } + return fs->NewDirectory(dirname, IOOptions(), directory, nullptr); +} + +IOStatus Directories::SetDirectories(FileSystem* fs, const std::string& dbname, + const std::string& wal_dir, + const std::vector& data_paths) { + IOStatus io_s = DBImpl::CreateAndNewDirectory(fs, dbname, &db_dir_); + if (!io_s.ok()) { + return io_s; + } + if (!wal_dir.empty() && dbname != wal_dir) { + io_s = DBImpl::CreateAndNewDirectory(fs, wal_dir, &wal_dir_); + if (!io_s.ok()) { + return io_s; + } + } + + data_dirs_.clear(); + for (auto& p : data_paths) { + const std::string db_path = p.path; + if (db_path == dbname) { + data_dirs_.emplace_back(nullptr); + } else { + std::unique_ptr path_directory; + io_s = DBImpl::CreateAndNewDirectory(fs, db_path, &path_directory); + if (!io_s.ok()) { + return io_s; + } + data_dirs_.emplace_back(path_directory.release()); + } + } + assert(data_dirs_.size() == data_paths.size()); + return IOStatus::OK(); +} + +Status DBImpl::Recover( + const std::vector& column_families, bool read_only, + bool error_if_wal_file_exists, bool error_if_data_exists_in_wals, + uint64_t* recovered_seq, RecoveryContext* recovery_ctx) { + mutex_.AssertHeld(); + + bool is_new_db = false; + assert(db_lock_ == nullptr); + std::vector files_in_dbname; + if (!read_only) { + Status s = directories_.SetDirectories(fs_.get(), dbname_, + immutable_db_options_.wal_dir, + immutable_db_options_.db_paths); + if (!s.ok()) { + return s; + } + + s = env_->LockFile(LockFileName(dbname_), &db_lock_); + if (!s.ok()) { + return s; + } + + std::string current_fname = CurrentFileName(dbname_); + // Path to any MANIFEST file in the db dir. It does not matter which one. + // Since best-efforts recovery ignores CURRENT file, existence of a + // MANIFEST indicates the recovery to recover existing db. If no MANIFEST + // can be found, a new db will be created. + std::string manifest_path; + if (!immutable_db_options_.best_efforts_recovery) { + s = env_->FileExists(current_fname); + } else { + s = Status::NotFound(); + IOOptions io_opts; + io_opts.do_not_recurse = true; + Status io_s = immutable_db_options_.fs->GetChildren( + dbname_, io_opts, &files_in_dbname, /*IODebugContext*=*/nullptr); + if (!io_s.ok()) { + s = io_s; + files_in_dbname.clear(); + } + for (const std::string& file : files_in_dbname) { + uint64_t number = 0; + FileType type = kWalFile; // initialize + if (ParseFileName(file, &number, &type) && type == kDescriptorFile) { + uint64_t bytes; + s = env_->GetFileSize(DescriptorFileName(dbname_, number), &bytes); + if (s.ok() && bytes != 0) { + // Found non-empty MANIFEST (descriptor log), thus best-efforts + // recovery does not have to treat the db as empty. + manifest_path = dbname_ + "/" + file; + break; + } + } + } + } + if (s.IsNotFound()) { + if (immutable_db_options_.create_if_missing) { + s = NewDB(&files_in_dbname); + is_new_db = true; + if (!s.ok()) { + return s; + } + } else { + return Status::InvalidArgument( + current_fname, "does not exist (create_if_missing is false)"); + } + } else if (s.ok()) { + if (immutable_db_options_.error_if_exists) { + return Status::InvalidArgument(dbname_, + "exists (error_if_exists is true)"); + } + } else { + // Unexpected error reading file + assert(s.IsIOError()); + return s; + } + // Verify compatibility of file_options_ and filesystem + { + std::unique_ptr idfile; + FileOptions customized_fs(file_options_); + customized_fs.use_direct_reads |= + immutable_db_options_.use_direct_io_for_flush_and_compaction; + const std::string& fname = + manifest_path.empty() ? current_fname : manifest_path; + s = fs_->NewRandomAccessFile(fname, customized_fs, &idfile, nullptr); + if (!s.ok()) { + std::string error_str = s.ToString(); + // Check if unsupported Direct I/O is the root cause + customized_fs.use_direct_reads = false; + s = fs_->NewRandomAccessFile(fname, customized_fs, &idfile, nullptr); + if (s.ok()) { + return Status::InvalidArgument( + "Direct I/O is not supported by the specified DB."); + } else { + return Status::InvalidArgument( + "Found options incompatible with filesystem", error_str.c_str()); + } + } + } + } else if (immutable_db_options_.best_efforts_recovery) { + assert(files_in_dbname.empty()); + IOOptions io_opts; + io_opts.do_not_recurse = true; + Status s = immutable_db_options_.fs->GetChildren( + dbname_, io_opts, &files_in_dbname, /*IODebugContext*=*/nullptr); + if (s.IsNotFound()) { + return Status::InvalidArgument(dbname_, + "does not exist (open for read only)"); + } else if (s.IsIOError()) { + return s; + } + assert(s.ok()); + } + assert(db_id_.empty()); + Status s; + bool missing_table_file = false; + if (!immutable_db_options_.best_efforts_recovery) { + s = versions_->Recover(column_families, read_only, &db_id_); + } else { + assert(!files_in_dbname.empty()); + s = versions_->TryRecover(column_families, read_only, files_in_dbname, + &db_id_, &missing_table_file); + if (s.ok()) { + // TryRecover may delete previous column_family_set_. + column_family_memtables_.reset( + new ColumnFamilyMemTablesImpl(versions_->GetColumnFamilySet())); + } + } + if (!s.ok()) { + return s; + } + if (s.ok() && !read_only) { + for (auto cfd : *versions_->GetColumnFamilySet()) { + // Try to trivially move files down the LSM tree to start from bottommost + // level when level_compaction_dynamic_level_bytes is enabled. This should + // only be useful when user is migrating to turning on this option. + // If a user is migrating from Level Compaction with a smaller level + // multiplier or from Universal Compaction, there may be too many + // non-empty levels and the trivial moves here are not sufficed for + // migration. Additional compactions are needed to drain unnecessary + // levels. + // + // Note that this step moves files down LSM without consulting + // SSTPartitioner. Further compactions are still needed if + // the user wants to partition SST files. + // Note that files moved in this step may not respect the compression + // option in target level. + if (cfd->ioptions()->compaction_style == + CompactionStyle::kCompactionStyleLevel && + cfd->ioptions()->level_compaction_dynamic_level_bytes && + !cfd->GetLatestMutableCFOptions()->disable_auto_compactions) { + int to_level = cfd->ioptions()->num_levels - 1; + // last level is reserved + // allow_ingest_behind does not support Level Compaction, + // and per_key_placement can have infinite compaction loop for Level + // Compaction. Adjust to_level here just to be safe. + if (cfd->ioptions()->allow_ingest_behind || + cfd->ioptions()->preclude_last_level_data_seconds > 0) { + to_level -= 1; + } + // Whether this column family has a level trivially moved + bool moved = false; + // Fill the LSM starting from to_level and going up one level at a time. + // Some loop invariants (when last level is not reserved): + // - levels in (from_level, to_level] are empty, and + // - levels in (to_level, last_level] are non-empty. + for (int from_level = to_level; from_level >= 0; --from_level) { + const std::vector& level_files = + cfd->current()->storage_info()->LevelFiles(from_level); + if (level_files.empty() || from_level == 0) { + continue; + } + assert(from_level <= to_level); + // Trivial move files from `from_level` to `to_level` + if (from_level < to_level) { + if (!moved) { + // lsm_state will look like "[1,2,3,4,5,6,0]" for an LSM with + // 7 levels + std::string lsm_state = "["; + for (int i = 0; i < cfd->ioptions()->num_levels; ++i) { + lsm_state += std::to_string( + cfd->current()->storage_info()->NumLevelFiles(i)); + if (i < cfd->ioptions()->num_levels - 1) { + lsm_state += ","; + } + } + lsm_state += "]"; + ROCKS_LOG_WARN(immutable_db_options_.info_log, + "[%s] Trivially move files down the LSM when open " + "with level_compaction_dynamic_level_bytes=true," + " lsm_state: %s (Files are moved only if DB " + "Recovery is successful).", + cfd->GetName().c_str(), lsm_state.c_str()); + moved = true; + } + ROCKS_LOG_WARN( + immutable_db_options_.info_log, + "[%s] Moving %zu files from from_level-%d to from_level-%d", + cfd->GetName().c_str(), level_files.size(), from_level, + to_level); + VersionEdit edit; + edit.SetColumnFamily(cfd->GetID()); + for (const FileMetaData* f : level_files) { + edit.DeleteFile(from_level, f->fd.GetNumber()); + edit.AddFile(to_level, f->fd.GetNumber(), f->fd.GetPathId(), + f->fd.GetFileSize(), f->smallest, f->largest, + f->fd.smallest_seqno, f->fd.largest_seqno, + f->marked_for_compaction, + f->temperature, // this can be different from + // `last_level_temperature` + f->oldest_blob_file_number, f->oldest_ancester_time, + f->file_creation_time, f->epoch_number, + f->file_checksum, f->file_checksum_func_name, + f->unique_id, f->compensated_range_deletion_size, + f->tail_size, f->user_defined_timestamps_persisted); + ROCKS_LOG_WARN(immutable_db_options_.info_log, + "[%s] Moving #%" PRIu64 + " from from_level-%d to from_level-%d %" PRIu64 + " bytes\n", + cfd->GetName().c_str(), f->fd.GetNumber(), + from_level, to_level, f->fd.GetFileSize()); + } + recovery_ctx->UpdateVersionEdits(cfd, edit); + } + --to_level; + } + } + } + } + s = SetupDBId(read_only, recovery_ctx); + ROCKS_LOG_INFO(immutable_db_options_.info_log, "DB ID: %s\n", db_id_.c_str()); + if (s.ok() && !read_only) { + s = DeleteUnreferencedSstFiles(recovery_ctx); + } + + if (immutable_db_options_.paranoid_checks && s.ok()) { + s = CheckConsistency(); + } + if (s.ok() && !read_only) { + // TODO: share file descriptors (FSDirectory) with SetDirectories above + std::map> created_dirs; + for (auto cfd : *versions_->GetColumnFamilySet()) { + s = cfd->AddDirectories(&created_dirs); + if (!s.ok()) { + return s; + } + } + } + + std::vector files_in_wal_dir; + if (s.ok()) { + // Initial max_total_in_memory_state_ before recovery wals. Log recovery + // may check this value to decide whether to flush. + max_total_in_memory_state_ = 0; + for (auto cfd : *versions_->GetColumnFamilySet()) { + auto* mutable_cf_options = cfd->GetLatestMutableCFOptions(); + max_total_in_memory_state_ += mutable_cf_options->write_buffer_size * + mutable_cf_options->max_write_buffer_number; + } + + SequenceNumber next_sequence(kMaxSequenceNumber); + default_cf_handle_ = new ColumnFamilyHandleImpl( + versions_->GetColumnFamilySet()->GetDefault(), this, &mutex_); + default_cf_internal_stats_ = default_cf_handle_->cfd()->internal_stats(); + + // Recover from all newer log files than the ones named in the + // descriptor (new log files may have been added by the previous + // incarnation without registering them in the descriptor). + // + // Note that prev_log_number() is no longer used, but we pay + // attention to it in case we are recovering a database + // produced by an older version of rocksdb. + auto wal_dir = immutable_db_options_.GetWalDir(); + if (!immutable_db_options_.best_efforts_recovery) { + IOOptions io_opts; + io_opts.do_not_recurse = true; + s = immutable_db_options_.fs->GetChildren( + wal_dir, io_opts, &files_in_wal_dir, /*IODebugContext*=*/nullptr); + } + if (s.IsNotFound()) { + return Status::InvalidArgument("wal_dir not found", wal_dir); + } else if (!s.ok()) { + return s; + } + + std::unordered_map wal_files; + for (const auto& file : files_in_wal_dir) { + uint64_t number; + FileType type; + if (ParseFileName(file, &number, &type) && type == kWalFile) { + if (is_new_db) { + return Status::Corruption( + "While creating a new Db, wal_dir contains " + "existing log file: ", + file); + } else { + wal_files[number] = LogFileName(wal_dir, number); + } + } + } + + if (immutable_db_options_.track_and_verify_wals_in_manifest) { + if (!immutable_db_options_.best_efforts_recovery) { + // Verify WALs in MANIFEST. + s = versions_->GetWalSet().CheckWals(env_, wal_files); + } // else since best effort recovery does not recover from WALs, no need + // to check WALs. + } else if (!versions_->GetWalSet().GetWals().empty()) { + // Tracking is disabled, clear previously tracked WALs from MANIFEST, + // otherwise, in the future, if WAL tracking is enabled again, + // since the WALs deleted when WAL tracking is disabled are not persisted + // into MANIFEST, WAL check may fail. + VersionEdit edit; + WalNumber max_wal_number = + versions_->GetWalSet().GetWals().rbegin()->first; + edit.DeleteWalsBefore(max_wal_number + 1); + assert(recovery_ctx != nullptr); + assert(versions_->GetColumnFamilySet() != nullptr); + recovery_ctx->UpdateVersionEdits( + versions_->GetColumnFamilySet()->GetDefault(), edit); + } + if (!s.ok()) { + return s; + } + + if (!wal_files.empty()) { + if (error_if_wal_file_exists) { + return Status::Corruption( + "The db was opened in readonly mode with error_if_wal_file_exists" + "flag but a WAL file already exists"); + } else if (error_if_data_exists_in_wals) { + for (auto& wal_file : wal_files) { + uint64_t bytes; + s = env_->GetFileSize(wal_file.second, &bytes); + if (s.ok()) { + if (bytes > 0) { + return Status::Corruption( + "error_if_data_exists_in_wals is set but there are data " + " in WAL files."); + } + } + } + } + } + + if (!wal_files.empty()) { + // Recover in the order in which the wals were generated + std::vector wals; + wals.reserve(wal_files.size()); + for (const auto& wal_file : wal_files) { + wals.push_back(wal_file.first); + } + std::sort(wals.begin(), wals.end()); + + bool corrupted_wal_found = false; + s = RecoverLogFiles(wals, &next_sequence, read_only, &corrupted_wal_found, + recovery_ctx); + if (corrupted_wal_found && recovered_seq != nullptr) { + *recovered_seq = next_sequence; + } + if (!s.ok()) { + // Clear memtables if recovery failed + for (auto cfd : *versions_->GetColumnFamilySet()) { + cfd->CreateNewMemtable(*cfd->GetLatestMutableCFOptions(), + kMaxSequenceNumber); + } + } + } + } + + if (read_only) { + // If we are opening as read-only, we need to update options_file_number_ + // to reflect the most recent OPTIONS file. It does not matter for regular + // read-write db instance because options_file_number_ will later be + // updated to versions_->NewFileNumber() in RenameTempFileToOptionsFile. + std::vector filenames; + if (s.ok()) { + const std::string normalized_dbname = NormalizePath(dbname_); + const std::string normalized_wal_dir = + NormalizePath(immutable_db_options_.GetWalDir()); + if (immutable_db_options_.best_efforts_recovery) { + filenames = std::move(files_in_dbname); + } else if (normalized_dbname == normalized_wal_dir) { + filenames = std::move(files_in_wal_dir); + } else { + IOOptions io_opts; + io_opts.do_not_recurse = true; + s = immutable_db_options_.fs->GetChildren( + GetName(), io_opts, &filenames, /*IODebugContext*=*/nullptr); + } + } + if (s.ok()) { + uint64_t number = 0; + uint64_t options_file_number = 0; + FileType type; + for (const auto& fname : filenames) { + if (ParseFileName(fname, &number, &type) && type == kOptionsFile) { + options_file_number = std::max(number, options_file_number); + } + } + versions_->options_file_number_ = options_file_number; + uint64_t options_file_size = 0; + if (options_file_number > 0) { + s = env_->GetFileSize(OptionsFileName(GetName(), options_file_number), + &options_file_size); + } + versions_->options_file_size_ = options_file_size; + } + } + return s; +} + +Status DBImpl::PersistentStatsProcessFormatVersion() { + mutex_.AssertHeld(); + Status s; + // persist version when stats CF doesn't exist + bool should_persist_format_version = !persistent_stats_cfd_exists_; + mutex_.Unlock(); + if (persistent_stats_cfd_exists_) { + // Check persistent stats format version compatibility. Drop and recreate + // persistent stats CF if format version is incompatible + uint64_t format_version_recovered = 0; + Status s_format = DecodePersistentStatsVersionNumber( + this, StatsVersionKeyType::kFormatVersion, &format_version_recovered); + uint64_t compatible_version_recovered = 0; + Status s_compatible = DecodePersistentStatsVersionNumber( + this, StatsVersionKeyType::kCompatibleVersion, + &compatible_version_recovered); + // abort reading from existing stats CF if any of following is true: + // 1. failed to read format version or compatible version from disk + // 2. sst's format version is greater than current format version, meaning + // this sst is encoded with a newer RocksDB release, and current compatible + // version is below the sst's compatible version + if (!s_format.ok() || !s_compatible.ok() || + (kStatsCFCurrentFormatVersion < format_version_recovered && + kStatsCFCompatibleFormatVersion < compatible_version_recovered)) { + if (!s_format.ok() || !s_compatible.ok()) { + ROCKS_LOG_WARN( + immutable_db_options_.info_log, + "Recreating persistent stats column family since reading " + "persistent stats version key failed. Format key: %s, compatible " + "key: %s", + s_format.ToString().c_str(), s_compatible.ToString().c_str()); + } else { + ROCKS_LOG_WARN( + immutable_db_options_.info_log, + "Recreating persistent stats column family due to corrupted or " + "incompatible format version. Recovered format: %" PRIu64 + "; recovered format compatible since: %" PRIu64 "\n", + format_version_recovered, compatible_version_recovered); + } + s = DropColumnFamily(persist_stats_cf_handle_); + if (s.ok()) { + s = DestroyColumnFamilyHandle(persist_stats_cf_handle_); + } + ColumnFamilyHandle* handle = nullptr; + if (s.ok()) { + ColumnFamilyOptions cfo; + OptimizeForPersistentStats(&cfo); + s = CreateColumnFamily(cfo, kPersistentStatsColumnFamilyName, &handle); + } + if (s.ok()) { + persist_stats_cf_handle_ = static_cast(handle); + // should also persist version here because old stats CF is discarded + should_persist_format_version = true; + } + } + } + if (should_persist_format_version) { + // Persistent stats CF being created for the first time, need to write + // format version key + WriteBatch batch; + if (s.ok()) { + s = batch.Put(persist_stats_cf_handle_, kFormatVersionKeyString, + std::to_string(kStatsCFCurrentFormatVersion)); + } + if (s.ok()) { + s = batch.Put(persist_stats_cf_handle_, kCompatibleVersionKeyString, + std::to_string(kStatsCFCompatibleFormatVersion)); + } + if (s.ok()) { + WriteOptions wo; + wo.low_pri = true; + wo.no_slowdown = true; + wo.sync = false; + s = Write(wo, &batch); + } + } + mutex_.Lock(); + return s; +} + +Status DBImpl::InitPersistStatsColumnFamily() { + mutex_.AssertHeld(); + assert(!persist_stats_cf_handle_); + ColumnFamilyData* persistent_stats_cfd = + versions_->GetColumnFamilySet()->GetColumnFamily( + kPersistentStatsColumnFamilyName); + persistent_stats_cfd_exists_ = persistent_stats_cfd != nullptr; + + Status s; + if (persistent_stats_cfd != nullptr) { + // We are recovering from a DB which already contains persistent stats CF, + // the CF is already created in VersionSet::ApplyOneVersionEdit, but + // column family handle was not. Need to explicitly create handle here. + persist_stats_cf_handle_ = + new ColumnFamilyHandleImpl(persistent_stats_cfd, this, &mutex_); + } else { + mutex_.Unlock(); + ColumnFamilyHandle* handle = nullptr; + ColumnFamilyOptions cfo; + OptimizeForPersistentStats(&cfo); + s = CreateColumnFamily(cfo, kPersistentStatsColumnFamilyName, &handle); + persist_stats_cf_handle_ = static_cast(handle); + mutex_.Lock(); + } + return s; +} + +Status DBImpl::LogAndApplyForRecovery(const RecoveryContext& recovery_ctx) { + mutex_.AssertHeld(); + assert(versions_->descriptor_log_ == nullptr); + const ReadOptions read_options(Env::IOActivity::kDBOpen); + Status s = versions_->LogAndApply( + recovery_ctx.cfds_, recovery_ctx.mutable_cf_opts_, read_options, + recovery_ctx.edit_lists_, &mutex_, directories_.GetDbDir()); + if (s.ok() && !(recovery_ctx.files_to_delete_.empty())) { + mutex_.Unlock(); + for (const auto& fname : recovery_ctx.files_to_delete_) { + s = env_->DeleteFile(fname); + if (!s.ok()) { + break; + } + } + mutex_.Lock(); + } + return s; +} + +void DBImpl::InvokeWalFilterIfNeededOnColumnFamilyToWalNumberMap() { + if (immutable_db_options_.wal_filter == nullptr) { + return; + } + assert(immutable_db_options_.wal_filter != nullptr); + WalFilter& wal_filter = *(immutable_db_options_.wal_filter); + + std::map cf_name_id_map; + std::map cf_lognumber_map; + assert(versions_); + assert(versions_->GetColumnFamilySet()); + for (auto cfd : *versions_->GetColumnFamilySet()) { + assert(cfd); + cf_name_id_map.insert(std::make_pair(cfd->GetName(), cfd->GetID())); + cf_lognumber_map.insert(std::make_pair(cfd->GetID(), cfd->GetLogNumber())); + } + + wal_filter.ColumnFamilyLogNumberMap(cf_lognumber_map, cf_name_id_map); +} + +bool DBImpl::InvokeWalFilterIfNeededOnWalRecord(uint64_t wal_number, + const std::string& wal_fname, + log::Reader::Reporter& reporter, + Status& status, + bool& stop_replay, + WriteBatch& batch) { + if (immutable_db_options_.wal_filter == nullptr) { + return true; + } + assert(immutable_db_options_.wal_filter != nullptr); + WalFilter& wal_filter = *(immutable_db_options_.wal_filter); + + WriteBatch new_batch; + bool batch_changed = false; + + bool process_current_record = true; + + WalFilter::WalProcessingOption wal_processing_option = + wal_filter.LogRecordFound(wal_number, wal_fname, batch, &new_batch, + &batch_changed); + + switch (wal_processing_option) { + case WalFilter::WalProcessingOption::kContinueProcessing: + // do nothing, proceeed normally + break; + case WalFilter::WalProcessingOption::kIgnoreCurrentRecord: + // skip current record + process_current_record = false; + break; + case WalFilter::WalProcessingOption::kStopReplay: + // skip current record and stop replay + process_current_record = false; + stop_replay = true; + break; + case WalFilter::WalProcessingOption::kCorruptedRecord: { + status = Status::Corruption("Corruption reported by Wal Filter ", + wal_filter.Name()); + MaybeIgnoreError(&status); + if (!status.ok()) { + process_current_record = false; + reporter.Corruption(batch.GetDataSize(), status); + } + break; + } + default: { + // logical error which should not happen. If RocksDB throws, we would + // just do `throw std::logic_error`. + assert(false); + status = Status::NotSupported( + "Unknown WalProcessingOption returned by Wal Filter ", + wal_filter.Name()); + MaybeIgnoreError(&status); + if (!status.ok()) { + // Ignore the error with current record processing. + stop_replay = true; + } + break; + } + } + + if (!process_current_record) { + return false; + } + + if (batch_changed) { + // Make sure that the count in the new batch is + // within the orignal count. + int new_count = WriteBatchInternal::Count(&new_batch); + int original_count = WriteBatchInternal::Count(&batch); + if (new_count > original_count) { + ROCKS_LOG_FATAL( + immutable_db_options_.info_log, + "Recovering log #%" PRIu64 + " mode %d log filter %s returned " + "more records (%d) than original (%d) which is not allowed. " + "Aborting recovery.", + wal_number, static_cast(immutable_db_options_.wal_recovery_mode), + wal_filter.Name(), new_count, original_count); + status = Status::NotSupported( + "More than original # of records " + "returned by Wal Filter ", + wal_filter.Name()); + return false; + } + // Set the same sequence number in the new_batch + // as the original batch. + WriteBatchInternal::SetSequence(&new_batch, + WriteBatchInternal::Sequence(&batch)); + batch = new_batch; + } + return true; +} + +// REQUIRES: wal_numbers are sorted in ascending order +Status DBImpl::RecoverLogFiles(const std::vector& wal_numbers, + SequenceNumber* next_sequence, bool read_only, + bool* corrupted_wal_found, + RecoveryContext* recovery_ctx) { + struct LogReporter : public log::Reader::Reporter { + Env* env; + Logger* info_log; + const char* fname; + Status* status; // nullptr if immutable_db_options_.paranoid_checks==false + void Corruption(size_t bytes, const Status& s) override { + ROCKS_LOG_WARN(info_log, "%s%s: dropping %d bytes; %s", + (status == nullptr ? "(ignoring error) " : ""), fname, + static_cast(bytes), s.ToString().c_str()); + if (status != nullptr && status->ok()) { + *status = s; + } + } + }; + + mutex_.AssertHeld(); + Status status; + std::unordered_map version_edits; + // no need to refcount because iteration is under mutex + for (auto cfd : *versions_->GetColumnFamilySet()) { + VersionEdit edit; + edit.SetColumnFamily(cfd->GetID()); + version_edits.insert({cfd->GetID(), edit}); + } + int job_id = next_job_id_.fetch_add(1); + { + auto stream = event_logger_.Log(); + stream << "job" << job_id << "event" + << "recovery_started"; + stream << "wal_files"; + stream.StartArray(); + for (auto wal_number : wal_numbers) { + stream << wal_number; + } + stream.EndArray(); + } + + // No-op for immutable_db_options_.wal_filter == nullptr. + InvokeWalFilterIfNeededOnColumnFamilyToWalNumberMap(); + + bool stop_replay_by_wal_filter = false; + bool stop_replay_for_corruption = false; + bool flushed = false; + uint64_t corrupted_wal_number = kMaxSequenceNumber; + uint64_t min_wal_number = MinLogNumberToKeep(); + if (!allow_2pc()) { + // In non-2pc mode, we skip WALs that do not back unflushed data. + min_wal_number = + std::max(min_wal_number, versions_->MinLogNumberWithUnflushedData()); + } + for (auto wal_number : wal_numbers) { + if (wal_number < min_wal_number) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "Skipping log #%" PRIu64 + " since it is older than min log to keep #%" PRIu64, + wal_number, min_wal_number); + continue; + } + // The previous incarnation may not have written any MANIFEST + // records after allocating this log number. So we manually + // update the file number allocation counter in VersionSet. + versions_->MarkFileNumberUsed(wal_number); + // Open the log file + std::string fname = + LogFileName(immutable_db_options_.GetWalDir(), wal_number); + + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "Recovering log #%" PRIu64 " mode %d", wal_number, + static_cast(immutable_db_options_.wal_recovery_mode)); + auto logFileDropped = [this, &fname]() { + uint64_t bytes; + if (env_->GetFileSize(fname, &bytes).ok()) { + auto info_log = immutable_db_options_.info_log.get(); + ROCKS_LOG_WARN(info_log, "%s: dropping %d bytes", fname.c_str(), + static_cast(bytes)); + } + }; + if (stop_replay_by_wal_filter) { + logFileDropped(); + continue; + } + + std::unique_ptr file_reader; + { + std::unique_ptr file; + status = fs_->NewSequentialFile( + fname, fs_->OptimizeForLogRead(file_options_), &file, nullptr); + if (!status.ok()) { + MaybeIgnoreError(&status); + if (!status.ok()) { + return status; + } else { + // Fail with one log file, but that's ok. + // Try next one. + continue; + } + } + file_reader.reset(new SequentialFileReader( + std::move(file), fname, immutable_db_options_.log_readahead_size, + io_tracer_)); + } + + // Create the log reader. + LogReporter reporter; + reporter.env = env_; + reporter.info_log = immutable_db_options_.info_log.get(); + reporter.fname = fname.c_str(); + if (!immutable_db_options_.paranoid_checks || + immutable_db_options_.wal_recovery_mode == + WALRecoveryMode::kSkipAnyCorruptedRecords) { + reporter.status = nullptr; + } else { + reporter.status = &status; + } + // We intentially make log::Reader do checksumming even if + // paranoid_checks==false so that corruptions cause entire commits + // to be skipped instead of propagating bad information (like overly + // large sequence numbers). + log::Reader reader(immutable_db_options_.info_log, std::move(file_reader), + &reporter, true /*checksum*/, wal_number); + + // Determine if we should tolerate incomplete records at the tail end of the + // Read all the records and add to a memtable + std::string scratch; + Slice record; + + const UnorderedMap& running_ts_sz = + versions_->GetRunningColumnFamiliesTimestampSize(); + + TEST_SYNC_POINT_CALLBACK("DBImpl::RecoverLogFiles:BeforeReadWal", + /*arg=*/nullptr); + uint64_t record_checksum; + while (!stop_replay_by_wal_filter && + reader.ReadRecord(&record, &scratch, + immutable_db_options_.wal_recovery_mode, + &record_checksum) && + status.ok()) { + if (record.size() < WriteBatchInternal::kHeader) { + reporter.Corruption(record.size(), + Status::Corruption("log record too small")); + continue; + } + // We create a new batch and initialize with a valid prot_info_ to store + // the data checksums + WriteBatch batch; + std::unique_ptr new_batch; + + status = WriteBatchInternal::SetContents(&batch, record); + if (!status.ok()) { + return status; + } + + const UnorderedMap& record_ts_sz = + reader.GetRecordedTimestampSize(); + status = HandleWriteBatchTimestampSizeDifference( + &batch, running_ts_sz, record_ts_sz, + TimestampSizeConsistencyMode::kReconcileInconsistency, &new_batch); + if (!status.ok()) { + return status; + } + + bool batch_updated = new_batch != nullptr; + WriteBatch* batch_to_use = batch_updated ? new_batch.get() : &batch; + TEST_SYNC_POINT_CALLBACK( + "DBImpl::RecoverLogFiles:BeforeUpdateProtectionInfo:batch", + batch_to_use); + TEST_SYNC_POINT_CALLBACK( + "DBImpl::RecoverLogFiles:BeforeUpdateProtectionInfo:checksum", + &record_checksum); + status = WriteBatchInternal::UpdateProtectionInfo( + batch_to_use, 8 /* bytes_per_key */, + batch_updated ? nullptr : &record_checksum); + if (!status.ok()) { + return status; + } + + SequenceNumber sequence = WriteBatchInternal::Sequence(batch_to_use); + + if (immutable_db_options_.wal_recovery_mode == + WALRecoveryMode::kPointInTimeRecovery) { + // In point-in-time recovery mode, if sequence id of log files are + // consecutive, we continue recovery despite corruption. This could + // happen when we open and write to a corrupted DB, where sequence id + // will start from the last sequence id we recovered. + if (sequence == *next_sequence) { + stop_replay_for_corruption = false; + } + if (stop_replay_for_corruption) { + logFileDropped(); + break; + } + } + + // For the default case of wal_filter == nullptr, always performs no-op + // and returns true. + if (!InvokeWalFilterIfNeededOnWalRecord(wal_number, fname, reporter, + status, stop_replay_by_wal_filter, + *batch_to_use)) { + continue; + } + + // If column family was not found, it might mean that the WAL write + // batch references to the column family that was dropped after the + // insert. We don't want to fail the whole write batch in that case -- + // we just ignore the update. + // That's why we set ignore missing column families to true + bool has_valid_writes = false; + status = WriteBatchInternal::InsertInto( + batch_to_use, column_family_memtables_.get(), &flush_scheduler_, + &trim_history_scheduler_, true, wal_number, this, + false /* concurrent_memtable_writes */, next_sequence, + &has_valid_writes, seq_per_batch_, batch_per_txn_); + MaybeIgnoreError(&status); + if (!status.ok()) { + // We are treating this as a failure while reading since we read valid + // blocks that do not form coherent data + reporter.Corruption(record.size(), status); + continue; + } + + if (has_valid_writes && !read_only) { + // we can do this because this is called before client has access to the + // DB and there is only a single thread operating on DB + ColumnFamilyData* cfd; + + while ((cfd = flush_scheduler_.TakeNextColumnFamily()) != nullptr) { + cfd->UnrefAndTryDelete(); + // If this asserts, it means that InsertInto failed in + // filtering updates to already-flushed column families + assert(cfd->GetLogNumber() <= wal_number); + auto iter = version_edits.find(cfd->GetID()); + assert(iter != version_edits.end()); + VersionEdit* edit = &iter->second; + status = WriteLevel0TableForRecovery(job_id, cfd, cfd->mem(), edit); + if (!status.ok()) { + // Reflect errors immediately so that conditions like full + // file-systems cause the DB::Open() to fail. + return status; + } + flushed = true; + + cfd->CreateNewMemtable(*cfd->GetLatestMutableCFOptions(), + *next_sequence); + } + } + } + + if (!status.ok()) { + if (status.IsNotSupported()) { + // We should not treat NotSupported as corruption. It is rather a clear + // sign that we are processing a WAL that is produced by an incompatible + // version of the code. + return status; + } + if (immutable_db_options_.wal_recovery_mode == + WALRecoveryMode::kSkipAnyCorruptedRecords) { + // We should ignore all errors unconditionally + status = Status::OK(); + } else if (immutable_db_options_.wal_recovery_mode == + WALRecoveryMode::kPointInTimeRecovery) { + if (status.IsIOError()) { + ROCKS_LOG_ERROR(immutable_db_options_.info_log, + "IOError during point-in-time reading log #%" PRIu64 + " seq #%" PRIu64 + ". %s. This likely mean loss of synced WAL, " + "thus recovery fails.", + wal_number, *next_sequence, + status.ToString().c_str()); + return status; + } + // We should ignore the error but not continue replaying + status = Status::OK(); + stop_replay_for_corruption = true; + corrupted_wal_number = wal_number; + if (corrupted_wal_found != nullptr) { + *corrupted_wal_found = true; + } + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "Point in time recovered to log #%" PRIu64 + " seq #%" PRIu64, + wal_number, *next_sequence); + } else { + assert(immutable_db_options_.wal_recovery_mode == + WALRecoveryMode::kTolerateCorruptedTailRecords || + immutable_db_options_.wal_recovery_mode == + WALRecoveryMode::kAbsoluteConsistency); + return status; + } + } + + flush_scheduler_.Clear(); + trim_history_scheduler_.Clear(); + auto last_sequence = *next_sequence - 1; + if ((*next_sequence != kMaxSequenceNumber) && + (versions_->LastSequence() <= last_sequence)) { + versions_->SetLastAllocatedSequence(last_sequence); + versions_->SetLastPublishedSequence(last_sequence); + versions_->SetLastSequence(last_sequence); + } + } + // Compare the corrupted log number to all columnfamily's current log number. + // Abort Open() if any column family's log number is greater than + // the corrupted log number, which means CF contains data beyond the point of + // corruption. This could during PIT recovery when the WAL is corrupted and + // some (but not all) CFs are flushed + // Exclude the PIT case where no log is dropped after the corruption point. + // This is to cover the case for empty wals after corrupted log, in which we + // don't reset stop_replay_for_corruption. + if (stop_replay_for_corruption == true && + (immutable_db_options_.wal_recovery_mode == + WALRecoveryMode::kPointInTimeRecovery || + immutable_db_options_.wal_recovery_mode == + WALRecoveryMode::kTolerateCorruptedTailRecords)) { + for (auto cfd : *versions_->GetColumnFamilySet()) { + // One special case cause cfd->GetLogNumber() > corrupted_wal_number but + // the CF is still consistent: If a new column family is created during + // the flush and the WAL sync fails at the same time, the new CF points to + // the new WAL but the old WAL is curropted. Since the new CF is empty, it + // is still consistent. We add the check of CF sst file size to avoid the + // false positive alert. + + // Note that, the check of (cfd->GetLiveSstFilesSize() > 0) may leads to + // the ignorance of a very rare inconsistency case caused in data + // canclation. One CF is empty due to KV deletion. But those operations + // are in the WAL. If the WAL is corrupted, the status of this CF might + // not be consistent with others. However, the consistency check will be + // bypassed due to empty CF. + // TODO: a better and complete implementation is needed to ensure strict + // consistency check in WAL recovery including hanlding the tailing + // issues. + if (cfd->GetLogNumber() > corrupted_wal_number && + cfd->GetLiveSstFilesSize() > 0) { + ROCKS_LOG_ERROR(immutable_db_options_.info_log, + "Column family inconsistency: SST file contains data" + " beyond the point of corruption."); + return Status::Corruption("SST file is ahead of WALs in CF " + + cfd->GetName()); + } + } + } + + // True if there's any data in the WALs; if not, we can skip re-processing + // them later + bool data_seen = false; + if (!read_only) { + // no need to refcount since client still doesn't have access + // to the DB and can not drop column families while we iterate + const WalNumber max_wal_number = wal_numbers.back(); + for (auto cfd : *versions_->GetColumnFamilySet()) { + auto iter = version_edits.find(cfd->GetID()); + assert(iter != version_edits.end()); + VersionEdit* edit = &iter->second; + + if (cfd->GetLogNumber() > max_wal_number) { + // Column family cfd has already flushed the data + // from all wals. Memtable has to be empty because + // we filter the updates based on wal_number + // (in WriteBatch::InsertInto) + assert(cfd->mem()->GetFirstSequenceNumber() == 0); + assert(edit->NumEntries() == 0); + continue; + } + + TEST_SYNC_POINT_CALLBACK( + "DBImpl::RecoverLogFiles:BeforeFlushFinalMemtable", /*arg=*/nullptr); + + // flush the final memtable (if non-empty) + if (cfd->mem()->GetFirstSequenceNumber() != 0) { + // If flush happened in the middle of recovery (e.g. due to memtable + // being full), we flush at the end. Otherwise we'll need to record + // where we were on last flush, which make the logic complicated. + if (flushed || !immutable_db_options_.avoid_flush_during_recovery) { + status = WriteLevel0TableForRecovery(job_id, cfd, cfd->mem(), edit); + if (!status.ok()) { + // Recovery failed + break; + } + flushed = true; + + cfd->CreateNewMemtable(*cfd->GetLatestMutableCFOptions(), + versions_->LastSequence()); + } + data_seen = true; + } + + // Update the log number info in the version edit corresponding to this + // column family. Note that the version edits will be written to MANIFEST + // together later. + // writing wal_number in the manifest means that any log file + // with number strongly less than (wal_number + 1) is already + // recovered and should be ignored on next reincarnation. + // Since we already recovered max_wal_number, we want all wals + // with numbers `<= max_wal_number` (includes this one) to be ignored + if (flushed || cfd->mem()->GetFirstSequenceNumber() == 0) { + edit->SetLogNumber(max_wal_number + 1); + } + } + if (status.ok()) { + // we must mark the next log number as used, even though it's + // not actually used. that is because VersionSet assumes + // VersionSet::next_file_number_ always to be strictly greater than any + // log number + versions_->MarkFileNumberUsed(max_wal_number + 1); + assert(recovery_ctx != nullptr); + + for (auto* cfd : *versions_->GetColumnFamilySet()) { + auto iter = version_edits.find(cfd->GetID()); + assert(iter != version_edits.end()); + recovery_ctx->UpdateVersionEdits(cfd, iter->second); + } + + if (flushed || !data_seen) { + VersionEdit wal_deletion; + if (immutable_db_options_.track_and_verify_wals_in_manifest) { + wal_deletion.DeleteWalsBefore(max_wal_number + 1); + } + if (!allow_2pc()) { + // In non-2pc mode, flushing the memtables of the column families + // means we can advance min_log_number_to_keep. + wal_deletion.SetMinLogNumberToKeep(max_wal_number + 1); + } + assert(versions_->GetColumnFamilySet() != nullptr); + recovery_ctx->UpdateVersionEdits( + versions_->GetColumnFamilySet()->GetDefault(), wal_deletion); + } + } + } + + if (status.ok()) { + if (data_seen && !flushed) { + status = RestoreAliveLogFiles(wal_numbers); + } else if (!wal_numbers.empty()) { // If there's no data in the WAL, or we + // flushed all the data, still + // truncate the log file. If the process goes into a crash loop before + // the file is deleted, the preallocated space will never get freed. + const bool truncate = !read_only; + GetLogSizeAndMaybeTruncate(wal_numbers.back(), truncate, nullptr) + .PermitUncheckedError(); + } + } + + event_logger_.Log() << "job" << job_id << "event" + << "recovery_finished"; + + return status; +} + +Status DBImpl::GetLogSizeAndMaybeTruncate(uint64_t wal_number, bool truncate, + LogFileNumberSize* log_ptr) { + LogFileNumberSize log(wal_number); + std::string fname = + LogFileName(immutable_db_options_.GetWalDir(), wal_number); + Status s; + // This gets the appear size of the wals, not including preallocated space. + s = env_->GetFileSize(fname, &log.size); + TEST_SYNC_POINT_CALLBACK("DBImpl::GetLogSizeAndMaybeTruncate:0", /*arg=*/&s); + if (s.ok() && truncate) { + std::unique_ptr last_log; + Status truncate_status = fs_->ReopenWritableFile( + fname, + fs_->OptimizeForLogWrite( + file_options_, + BuildDBOptions(immutable_db_options_, mutable_db_options_)), + &last_log, nullptr); + if (truncate_status.ok()) { + truncate_status = last_log->Truncate(log.size, IOOptions(), nullptr); + } + if (truncate_status.ok()) { + truncate_status = last_log->Close(IOOptions(), nullptr); + } + // Not a critical error if fail to truncate. + if (!truncate_status.ok() && !truncate_status.IsNotSupported()) { + ROCKS_LOG_WARN(immutable_db_options_.info_log, + "Failed to truncate log #%" PRIu64 ": %s", wal_number, + truncate_status.ToString().c_str()); + } + } + if (log_ptr) { + *log_ptr = log; + } + return s; +} + +Status DBImpl::RestoreAliveLogFiles(const std::vector& wal_numbers) { + if (wal_numbers.empty()) { + return Status::OK(); + } + Status s; + mutex_.AssertHeld(); + assert(immutable_db_options_.avoid_flush_during_recovery); + // Mark these as alive so they'll be considered for deletion later by + // FindObsoleteFiles() + total_log_size_ = 0; + log_empty_ = false; + uint64_t min_wal_with_unflushed_data = + versions_->MinLogNumberWithUnflushedData(); + for (auto wal_number : wal_numbers) { + if (!allow_2pc() && wal_number < min_wal_with_unflushed_data) { + // In non-2pc mode, the WAL files not backing unflushed data are not + // alive, thus should not be added to the alive_log_files_. + continue; + } + // We preallocate space for wals, but then after a crash and restart, those + // preallocated space are not needed anymore. It is likely only the last + // log has such preallocated space, so we only truncate for the last log. + LogFileNumberSize log; + s = GetLogSizeAndMaybeTruncate( + wal_number, /*truncate=*/(wal_number == wal_numbers.back()), &log); + if (!s.ok()) { + break; + } + total_log_size_ += log.size; + alive_log_files_.push_back(log); + } + return s; +} + +Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd, + MemTable* mem, VersionEdit* edit) { + mutex_.AssertHeld(); + assert(cfd); + assert(cfd->imm()); + // The immutable memtable list must be empty. + assert(std::numeric_limits::max() == + cfd->imm()->GetEarliestMemTableID()); + + const uint64_t start_micros = immutable_db_options_.clock->NowMicros(); + + FileMetaData meta; + std::vector blob_file_additions; + + std::unique_ptr::iterator> pending_outputs_inserted_elem( + new std::list::iterator( + CaptureCurrentFileNumberInPendingOutputs())); + meta.fd = FileDescriptor(versions_->NewFileNumber(), 0, 0); + ReadOptions ro; + ro.total_order_seek = true; + ro.io_activity = Env::IOActivity::kDBOpen; + Arena arena; + Status s; + TableProperties table_properties; + { + ScopedArenaIterator iter(mem->NewIterator(ro, &arena)); + ROCKS_LOG_DEBUG(immutable_db_options_.info_log, + "[%s] [WriteLevel0TableForRecovery]" + " Level-0 table #%" PRIu64 ": started", + cfd->GetName().c_str(), meta.fd.GetNumber()); + + // Get the latest mutable cf options while the mutex is still locked + const MutableCFOptions mutable_cf_options = + *cfd->GetLatestMutableCFOptions(); + bool paranoid_file_checks = + cfd->GetLatestMutableCFOptions()->paranoid_file_checks; + + int64_t _current_time = 0; + immutable_db_options_.clock->GetCurrentTime(&_current_time) + .PermitUncheckedError(); // ignore error + const uint64_t current_time = static_cast(_current_time); + meta.oldest_ancester_time = current_time; + meta.epoch_number = cfd->NewEpochNumber(); + { + auto write_hint = cfd->CalculateSSTWriteHint(0); + mutex_.Unlock(); + + SequenceNumber earliest_write_conflict_snapshot; + std::vector snapshot_seqs = + snapshots_.GetAll(&earliest_write_conflict_snapshot); + auto snapshot_checker = snapshot_checker_.get(); + if (use_custom_gc_ && snapshot_checker == nullptr) { + snapshot_checker = DisableGCSnapshotChecker::Instance(); + } + std::vector> + range_del_iters; + auto range_del_iter = + // This is called during recovery, where a live memtable is flushed + // directly. In this case, no fragmented tombstone list is cached in + // this memtable yet. + mem->NewRangeTombstoneIterator(ro, kMaxSequenceNumber, + false /* immutable_memtable */); + if (range_del_iter != nullptr) { + range_del_iters.emplace_back(range_del_iter); + } + + IOStatus io_s; + TableBuilderOptions tboptions( + *cfd->ioptions(), mutable_cf_options, cfd->internal_comparator(), + cfd->int_tbl_prop_collector_factories(), + GetCompressionFlush(*cfd->ioptions(), mutable_cf_options), + mutable_cf_options.compression_opts, cfd->GetID(), cfd->GetName(), + 0 /* level */, false /* is_bottommost */, + TableFileCreationReason::kRecovery, 0 /* oldest_key_time */, + 0 /* file_creation_time */, db_id_, db_session_id_, + 0 /* target_file_size */, meta.fd.GetNumber()); + SeqnoToTimeMapping empty_seqno_time_mapping; + Version* version = cfd->current(); + version->Ref(); + const ReadOptions read_option(Env::IOActivity::kDBOpen); + uint64_t num_input_entries = 0; + s = BuildTable( + dbname_, versions_.get(), immutable_db_options_, tboptions, + file_options_for_compaction_, read_option, cfd->table_cache(), + iter.get(), std::move(range_del_iters), &meta, &blob_file_additions, + snapshot_seqs, earliest_write_conflict_snapshot, kMaxSequenceNumber, + snapshot_checker, paranoid_file_checks, cfd->internal_stats(), &io_s, + io_tracer_, BlobFileCreationReason::kRecovery, + empty_seqno_time_mapping, &event_logger_, job_id, Env::IO_HIGH, + nullptr /* table_properties */, write_hint, + nullptr /*full_history_ts_low*/, &blob_callback_, version, + &num_input_entries); + version->Unref(); + LogFlush(immutable_db_options_.info_log); + ROCKS_LOG_DEBUG(immutable_db_options_.info_log, + "[%s] [WriteLevel0TableForRecovery]" + " Level-0 table #%" PRIu64 ": %" PRIu64 " bytes %s", + cfd->GetName().c_str(), meta.fd.GetNumber(), + meta.fd.GetFileSize(), s.ToString().c_str()); + mutex_.Lock(); + + // TODO(AR) is this ok? + if (!io_s.ok() && s.ok()) { + s = io_s; + } + + uint64_t total_num_entries = mem->num_entries(); + if (s.ok() && total_num_entries != num_input_entries) { + std::string msg = "Expected " + std::to_string(total_num_entries) + + " entries in memtable, but read " + + std::to_string(num_input_entries); + ROCKS_LOG_WARN(immutable_db_options_.info_log, + "[%s] [JOB %d] Level-0 flush during recover: %s", + cfd->GetName().c_str(), job_id, msg.c_str()); + if (immutable_db_options_.flush_verify_memtable_count) { + s = Status::Corruption(msg); + } + } + } + } + ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem); + + // Note that if file_size is zero, the file has been deleted and + // should not be added to the manifest. + const bool has_output = meta.fd.GetFileSize() > 0; + + constexpr int level = 0; + + if (s.ok() && has_output) { + edit->AddFile(level, meta.fd.GetNumber(), meta.fd.GetPathId(), + meta.fd.GetFileSize(), meta.smallest, meta.largest, + meta.fd.smallest_seqno, meta.fd.largest_seqno, + meta.marked_for_compaction, meta.temperature, + meta.oldest_blob_file_number, meta.oldest_ancester_time, + meta.file_creation_time, meta.epoch_number, + meta.file_checksum, meta.file_checksum_func_name, + meta.unique_id, meta.compensated_range_deletion_size, + meta.tail_size, meta.user_defined_timestamps_persisted); + + for (const auto& blob : blob_file_additions) { + edit->AddBlobFile(blob); + } + } + + InternalStats::CompactionStats stats(CompactionReason::kFlush, 1); + stats.micros = immutable_db_options_.clock->NowMicros() - start_micros; + + if (has_output) { + stats.bytes_written = meta.fd.GetFileSize(); + stats.num_output_files = 1; + } + + const auto& blobs = edit->GetBlobFileAdditions(); + for (const auto& blob : blobs) { + stats.bytes_written_blob += blob.GetTotalBlobBytes(); + } + + stats.num_output_files_blob = static_cast(blobs.size()); + + cfd->internal_stats()->AddCompactionStats(level, Env::Priority::USER, stats); + cfd->internal_stats()->AddCFStats( + InternalStats::BYTES_FLUSHED, + stats.bytes_written + stats.bytes_written_blob); + RecordTick(stats_, COMPACT_WRITE_BYTES, meta.fd.GetFileSize()); + return s; +} + +Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) { + DBOptions db_options(options); + ColumnFamilyOptions cf_options(options); + std::vector column_families; + column_families.push_back( + ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options)); + if (db_options.persist_stats_to_disk) { + column_families.push_back( + ColumnFamilyDescriptor(kPersistentStatsColumnFamilyName, cf_options)); + } + std::vector handles; + Status s = DB::Open(db_options, dbname, column_families, &handles, dbptr); + if (s.ok()) { + if (db_options.persist_stats_to_disk) { + assert(handles.size() == 2); + } else { + assert(handles.size() == 1); + } + // i can delete the handle since DBImpl is always holding a reference to + // default column family + if (db_options.persist_stats_to_disk && handles[1] != nullptr) { + delete handles[1]; + } + delete handles[0]; + } + return s; +} + +Status DB::Open(const DBOptions& db_options, const std::string& dbname, + const std::vector& column_families, + std::vector* handles, DB** dbptr) { + const bool kSeqPerBatch = true; + const bool kBatchPerTxn = true; + ThreadStatusUtil::SetEnableTracking(db_options.enable_thread_tracking); + ThreadStatusUtil::SetThreadOperation(ThreadStatus::OperationType::OP_DBOPEN); + Status s = DBImpl::Open(db_options, dbname, column_families, handles, dbptr, + !kSeqPerBatch, kBatchPerTxn); + ThreadStatusUtil::ResetThreadStatus(); + return s; +} + +// TODO: Implement the trimming in flush code path. +// TODO: Perform trimming before inserting into memtable during recovery. +// TODO: Pick files with max_timestamp > trim_ts by each file's timestamp meta +// info, and handle only these files to reduce io. +Status DB::OpenAndTrimHistory( + const DBOptions& db_options, const std::string& dbname, + const std::vector& column_families, + std::vector* handles, DB** dbptr, + std::string trim_ts) { + assert(dbptr != nullptr); + assert(handles != nullptr); + auto validate_options = [&db_options] { + if (db_options.avoid_flush_during_recovery) { + return Status::InvalidArgument( + "avoid_flush_during_recovery incompatible with " + "OpenAndTrimHistory"); + } + return Status::OK(); + }; + auto s = validate_options(); + if (!s.ok()) { + return s; + } + + DB* db = nullptr; + s = DB::Open(db_options, dbname, column_families, handles, &db); + if (!s.ok()) { + return s; + } + assert(db); + CompactRangeOptions options; + options.bottommost_level_compaction = + BottommostLevelCompaction::kForceOptimized; + auto db_impl = static_cast_with_check(db); + for (auto handle : *handles) { + assert(handle != nullptr); + auto cfh = static_cast_with_check(handle); + auto cfd = cfh->cfd(); + assert(cfd != nullptr); + // Only compact column families with timestamp enabled + if (cfd->user_comparator() != nullptr && + cfd->user_comparator()->timestamp_size() > 0) { + s = db_impl->CompactRangeInternal(options, handle, nullptr, nullptr, + trim_ts); + if (!s.ok()) { + break; + } + } + } + auto clean_op = [&handles, &db] { + for (auto handle : *handles) { + auto temp_s = db->DestroyColumnFamilyHandle(handle); + assert(temp_s.ok()); + } + handles->clear(); + delete db; + }; + if (!s.ok()) { + clean_op(); + return s; + } + + *dbptr = db; + return s; +} + +IOStatus DBImpl::CreateWAL(uint64_t log_file_num, uint64_t recycle_log_number, + size_t preallocate_block_size, + log::Writer** new_log) { + IOStatus io_s; + std::unique_ptr lfile; + + DBOptions db_options = + BuildDBOptions(immutable_db_options_, mutable_db_options_); + FileOptions opt_file_options = + fs_->OptimizeForLogWrite(file_options_, db_options); + std::string wal_dir = immutable_db_options_.GetWalDir(); + std::string log_fname = LogFileName(wal_dir, log_file_num); + + if (recycle_log_number) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "reusing log %" PRIu64 " from recycle list\n", + recycle_log_number); + std::string old_log_fname = LogFileName(wal_dir, recycle_log_number); + TEST_SYNC_POINT("DBImpl::CreateWAL:BeforeReuseWritableFile1"); + TEST_SYNC_POINT("DBImpl::CreateWAL:BeforeReuseWritableFile2"); + io_s = fs_->ReuseWritableFile(log_fname, old_log_fname, opt_file_options, + &lfile, /*dbg=*/nullptr); + } else { + io_s = NewWritableFile(fs_.get(), log_fname, &lfile, opt_file_options); + } + + if (io_s.ok()) { + lfile->SetWriteLifeTimeHint(CalculateWALWriteHint()); + lfile->SetPreallocationBlockSize(preallocate_block_size); + + const auto& listeners = immutable_db_options_.listeners; + FileTypeSet tmp_set = immutable_db_options_.checksum_handoff_file_types; + std::unique_ptr file_writer(new WritableFileWriter( + std::move(lfile), log_fname, opt_file_options, + immutable_db_options_.clock, io_tracer_, nullptr /* stats */, listeners, + nullptr, tmp_set.Contains(FileType::kWalFile), + tmp_set.Contains(FileType::kWalFile))); + *new_log = new log::Writer(std::move(file_writer), log_file_num, + immutable_db_options_.recycle_log_file_num > 0, + immutable_db_options_.manual_wal_flush, + immutable_db_options_.wal_compression); + io_s = (*new_log)->AddCompressionTypeRecord(); + } + return io_s; +} + +Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname, + const std::vector& column_families, + std::vector* handles, DB** dbptr, + const bool seq_per_batch, const bool batch_per_txn) { + Status s = ValidateOptionsByTable(db_options, column_families); + if (!s.ok()) { + return s; + } + + s = ValidateOptions(db_options, column_families); + if (!s.ok()) { + return s; + } + + *dbptr = nullptr; + assert(handles); + handles->clear(); + + size_t max_write_buffer_size = 0; + for (auto cf : column_families) { + max_write_buffer_size = + std::max(max_write_buffer_size, cf.options.write_buffer_size); + } + + DBImpl* impl = new DBImpl(db_options, dbname, seq_per_batch, batch_per_txn); + if (!impl->immutable_db_options_.info_log) { + s = impl->init_logger_creation_s_; + delete impl; + return s; + } else { + assert(impl->init_logger_creation_s_.ok()); + } + s = impl->env_->CreateDirIfMissing(impl->immutable_db_options_.GetWalDir()); + if (s.ok()) { + std::vector paths; + for (auto& db_path : impl->immutable_db_options_.db_paths) { + paths.emplace_back(db_path.path); + } + for (auto& cf : column_families) { + for (auto& cf_path : cf.options.cf_paths) { + paths.emplace_back(cf_path.path); + } + } + for (auto& path : paths) { + s = impl->env_->CreateDirIfMissing(path); + if (!s.ok()) { + break; + } + } + + // For recovery from NoSpace() error, we can only handle + // the case where the database is stored in a single path + if (paths.size() <= 1) { + impl->error_handler_.EnableAutoRecovery(); + } + } + if (s.ok()) { + s = impl->CreateArchivalDirectory(); + } + if (!s.ok()) { + delete impl; + return s; + } + + impl->wal_in_db_path_ = impl->immutable_db_options_.IsWalDirSameAsDBPath(); + RecoveryContext recovery_ctx; + impl->mutex_.Lock(); + + // Handles create_if_missing, error_if_exists + uint64_t recovered_seq(kMaxSequenceNumber); + s = impl->Recover(column_families, false /* read_only */, + false /* error_if_wal_file_exists */, + false /* error_if_data_exists_in_wals */, &recovered_seq, + &recovery_ctx); + if (s.ok()) { + uint64_t new_log_number = impl->versions_->NewFileNumber(); + log::Writer* new_log = nullptr; + const size_t preallocate_block_size = + impl->GetWalPreallocateBlockSize(max_write_buffer_size); + s = impl->CreateWAL(new_log_number, 0 /*recycle_log_number*/, + preallocate_block_size, &new_log); + if (s.ok()) { + InstrumentedMutexLock wl(&impl->log_write_mutex_); + impl->logfile_number_ = new_log_number; + assert(new_log != nullptr); + assert(impl->logs_.empty()); + impl->logs_.emplace_back(new_log_number, new_log); + } + + if (s.ok()) { + impl->alive_log_files_.push_back( + DBImpl::LogFileNumberSize(impl->logfile_number_)); + // In WritePrepared there could be gap in sequence numbers. This breaks + // the trick we use in kPointInTimeRecovery which assumes the first seq in + // the log right after the corrupted log is one larger than the last seq + // we read from the wals. To let this trick keep working, we add a dummy + // entry with the expected sequence to the first log right after recovery. + // In non-WritePrepared case also the new log after recovery could be + // empty, and thus missing the consecutive seq hint to distinguish + // middle-log corruption to corrupted-log-remained-after-recovery. This + // case also will be addressed by a dummy write. + if (recovered_seq != kMaxSequenceNumber) { + WriteBatch empty_batch; + WriteBatchInternal::SetSequence(&empty_batch, recovered_seq); + WriteOptions write_options; + uint64_t log_used, log_size; + log::Writer* log_writer = impl->logs_.back().writer; + LogFileNumberSize& log_file_number_size = impl->alive_log_files_.back(); + + assert(log_writer->get_log_number() == log_file_number_size.number); + impl->mutex_.AssertHeld(); + s = impl->WriteToWAL(empty_batch, log_writer, &log_used, &log_size, + Env::IO_TOTAL, log_file_number_size); + if (s.ok()) { + // Need to fsync, otherwise it might get lost after a power reset. + s = impl->FlushWAL(false); + TEST_SYNC_POINT_CALLBACK("DBImpl::Open::BeforeSyncWAL", /*arg=*/&s); + if (s.ok()) { + s = log_writer->file()->Sync(impl->immutable_db_options_.use_fsync); + } + } + } + } + } + if (s.ok()) { + s = impl->LogAndApplyForRecovery(recovery_ctx); + } + + if (s.ok() && impl->immutable_db_options_.persist_stats_to_disk) { + impl->mutex_.AssertHeld(); + s = impl->InitPersistStatsColumnFamily(); + } + + if (s.ok()) { + // set column family handles + for (auto cf : column_families) { + auto cfd = + impl->versions_->GetColumnFamilySet()->GetColumnFamily(cf.name); + if (cfd != nullptr) { + handles->push_back( + new ColumnFamilyHandleImpl(cfd, impl, &impl->mutex_)); + impl->NewThreadStatusCfInfo(cfd); + } else { + if (db_options.create_missing_column_families) { + // missing column family, create it + ColumnFamilyHandle* handle = nullptr; + impl->mutex_.Unlock(); + s = impl->CreateColumnFamily(cf.options, cf.name, &handle); + impl->mutex_.Lock(); + if (s.ok()) { + handles->push_back(handle); + } else { + break; + } + } else { + s = Status::InvalidArgument("Column family not found", cf.name); + break; + } + } + } + } + + if (s.ok()) { + SuperVersionContext sv_context(/* create_superversion */ true); + for (auto cfd : *impl->versions_->GetColumnFamilySet()) { + impl->InstallSuperVersionAndScheduleWork( + cfd, &sv_context, *cfd->GetLatestMutableCFOptions()); + } + sv_context.Clean(); + } + + if (s.ok() && impl->immutable_db_options_.persist_stats_to_disk) { + // try to read format version + s = impl->PersistentStatsProcessFormatVersion(); + } + + if (s.ok()) { + for (auto cfd : *impl->versions_->GetColumnFamilySet()) { + if (!cfd->mem()->IsSnapshotSupported()) { + impl->is_snapshot_supported_ = false; + } + if (cfd->ioptions()->merge_operator != nullptr && + !cfd->mem()->IsMergeOperatorSupported()) { + s = Status::InvalidArgument( + "The memtable of column family %s does not support merge operator " + "its options.merge_operator is non-null", + cfd->GetName().c_str()); + } + if (!s.ok()) { + break; + } + } + } + TEST_SYNC_POINT("DBImpl::Open:Opened"); + Status persist_options_status; + if (s.ok()) { + // Persist RocksDB Options before scheduling the compaction. + // The WriteOptionsFile() will release and lock the mutex internally. + persist_options_status = impl->WriteOptionsFile( + false /*need_mutex_lock*/, false /*need_enter_write_thread*/); + + *dbptr = impl; + impl->opened_successfully_ = true; + impl->DeleteObsoleteFiles(); + TEST_SYNC_POINT("DBImpl::Open:AfterDeleteFiles"); + impl->MaybeScheduleFlushOrCompaction(); + } else { + persist_options_status.PermitUncheckedError(); + } + impl->mutex_.Unlock(); + + auto sfm = static_cast( + impl->immutable_db_options_.sst_file_manager.get()); + if (s.ok() && sfm) { + // Set Statistics ptr for SstFileManager to dump the stats of + // DeleteScheduler. + sfm->SetStatisticsPtr(impl->immutable_db_options_.statistics); + ROCKS_LOG_INFO(impl->immutable_db_options_.info_log, + "SstFileManager instance %p", sfm); + + // Notify SstFileManager about all sst files that already exist in + // db_paths[0] and cf_paths[0] when the DB is opened. + + // SstFileManagerImpl needs to know sizes of the files. For files whose size + // we already know (sst files that appear in manifest - typically that's the + // vast majority of all files), we'll pass the size to SstFileManager. + // For all other files SstFileManager will query the size from filesystem. + + std::vector metadata; + impl->GetAllColumnFamilyMetaData(&metadata); + + std::unordered_map known_file_sizes; + for (const auto& md : metadata) { + for (const auto& lmd : md.levels) { + for (const auto& fmd : lmd.files) { + known_file_sizes[fmd.relative_filename] = fmd.size; + } + } + for (const auto& bmd : md.blob_files) { + std::string name = bmd.blob_file_name; + // The BlobMetaData.blob_file_name may start with "/". + if (!name.empty() && name[0] == '/') { + name = name.substr(1); + } + known_file_sizes[name] = bmd.blob_file_size; + } + } + + std::vector paths; + paths.emplace_back(impl->immutable_db_options_.db_paths[0].path); + for (auto& cf : column_families) { + if (!cf.options.cf_paths.empty()) { + paths.emplace_back(cf.options.cf_paths[0].path); + } + } + // Remove duplicate paths. + std::sort(paths.begin(), paths.end()); + paths.erase(std::unique(paths.begin(), paths.end()), paths.end()); + IOOptions io_opts; + io_opts.do_not_recurse = true; + for (auto& path : paths) { + std::vector existing_files; + impl->immutable_db_options_.fs + ->GetChildren(path, io_opts, &existing_files, + /*IODebugContext*=*/nullptr) + .PermitUncheckedError(); //**TODO: What do to on error? + for (auto& file_name : existing_files) { + uint64_t file_number; + FileType file_type; + std::string file_path = path + "/" + file_name; + if (ParseFileName(file_name, &file_number, &file_type) && + (file_type == kTableFile || file_type == kBlobFile)) { + // TODO: Check for errors from OnAddFile? + if (known_file_sizes.count(file_name)) { + // We're assuming that each sst file name exists in at most one of + // the paths. + sfm->OnAddFile(file_path, known_file_sizes.at(file_name)) + .PermitUncheckedError(); + } else { + sfm->OnAddFile(file_path).PermitUncheckedError(); + } + } + } + } + + // Reserve some disk buffer space. This is a heuristic - when we run out + // of disk space, this ensures that there is atleast write_buffer_size + // amount of free space before we resume DB writes. In low disk space + // conditions, we want to avoid a lot of small L0 files due to frequent + // WAL write failures and resultant forced flushes + sfm->ReserveDiskBuffer(max_write_buffer_size, + impl->immutable_db_options_.db_paths[0].path); + } + + + if (s.ok()) { + ROCKS_LOG_HEADER(impl->immutable_db_options_.info_log, "DB pointer %p", + impl); + LogFlush(impl->immutable_db_options_.info_log); + if (!impl->WALBufferIsEmpty()) { + s = impl->FlushWAL(false); + if (s.ok()) { + // Sync is needed otherwise WAL buffered data might get lost after a + // power reset. + log::Writer* log_writer = impl->logs_.back().writer; + s = log_writer->file()->Sync(impl->immutable_db_options_.use_fsync); + } + } + if (s.ok() && !persist_options_status.ok()) { + s = Status::IOError( + "DB::Open() failed --- Unable to persist Options file", + persist_options_status.ToString()); + } + } + if (!s.ok()) { + ROCKS_LOG_WARN(impl->immutable_db_options_.info_log, + "DB::Open() failed: %s", s.ToString().c_str()); + } + if (s.ok()) { + s = impl->StartPeriodicTaskScheduler(); + } + + if (s.ok()) { + s = impl->RegisterRecordSeqnoTimeWorker(); + } + if (!s.ok()) { + for (auto* h : *handles) { + delete h; + } + handles->clear(); + delete impl; + *dbptr = nullptr; + } + return s; +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/db_impl/db_impl_readonly.cc b/librocksdb-sys/rocksdb/db/db_impl/db_impl_readonly.cc new file mode 100644 index 0000000..6f7e95f --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_impl/db_impl_readonly.cc @@ -0,0 +1,335 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/db_impl/db_impl_readonly.h" + +#include "db/arena_wrapped_db_iter.h" +#include "db/db_impl/compacted_db_impl.h" +#include "db/db_impl/db_impl.h" +#include "db/db_iter.h" +#include "db/merge_context.h" +#include "logging/logging.h" +#include "monitoring/perf_context_imp.h" +#include "util/cast_util.h" + +namespace ROCKSDB_NAMESPACE { + + +DBImplReadOnly::DBImplReadOnly(const DBOptions& db_options, + const std::string& dbname) + : DBImpl(db_options, dbname, /*seq_per_batch*/ false, + /*batch_per_txn*/ true, /*read_only*/ true) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "Opening the db in read only mode"); + LogFlush(immutable_db_options_.info_log); +} + +DBImplReadOnly::~DBImplReadOnly() {} + +// Implementations of the DB interface +Status DBImplReadOnly::Get(const ReadOptions& read_options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* pinnable_val) { + return Get(read_options, column_family, key, pinnable_val, + /*timestamp*/ nullptr); +} + +Status DBImplReadOnly::Get(const ReadOptions& read_options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* pinnable_val, + std::string* timestamp) { + if (read_options.io_activity != Env::IOActivity::kUnknown) { + return Status::InvalidArgument( + "Cannot call Get with `ReadOptions::io_activity` != " + "`Env::IOActivity::kUnknown`"); + } + assert(pinnable_val != nullptr); + PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock); + StopWatch sw(immutable_db_options_.clock, stats_, DB_GET); + PERF_TIMER_GUARD(get_snapshot_time); + + assert(column_family); + if (read_options.timestamp) { + const Status s = FailIfTsMismatchCf( + column_family, *(read_options.timestamp), /*ts_for_read=*/true); + if (!s.ok()) { + return s; + } + } else { + const Status s = FailIfCfHasTs(column_family); + if (!s.ok()) { + return s; + } + } + + // Clear the timestamps for returning results so that we can distinguish + // between tombstone or key that has never been written + if (timestamp) { + timestamp->clear(); + } + + const Comparator* ucmp = column_family->GetComparator(); + assert(ucmp); + std::string* ts = ucmp->timestamp_size() > 0 ? timestamp : nullptr; + + Status s; + SequenceNumber snapshot = versions_->LastSequence(); + GetWithTimestampReadCallback read_cb(snapshot); + auto cfh = static_cast_with_check(column_family); + auto cfd = cfh->cfd(); + if (tracer_) { + InstrumentedMutexLock lock(&trace_mutex_); + if (tracer_) { + tracer_->Get(column_family, key); + } + } + SuperVersion* super_version = cfd->GetSuperVersion(); + MergeContext merge_context; + SequenceNumber max_covering_tombstone_seq = 0; + LookupKey lkey(key, snapshot, read_options.timestamp); + PERF_TIMER_STOP(get_snapshot_time); + if (super_version->mem->Get(lkey, pinnable_val->GetSelf(), + /*columns=*/nullptr, ts, &s, &merge_context, + &max_covering_tombstone_seq, read_options, + false /* immutable_memtable */, &read_cb)) { + pinnable_val->PinSelf(); + RecordTick(stats_, MEMTABLE_HIT); + } else { + PERF_TIMER_GUARD(get_from_output_files_time); + PinnedIteratorsManager pinned_iters_mgr; + super_version->current->Get( + read_options, lkey, pinnable_val, /*columns=*/nullptr, ts, &s, + &merge_context, &max_covering_tombstone_seq, &pinned_iters_mgr, + /*value_found*/ nullptr, + /*key_exists*/ nullptr, /*seq*/ nullptr, &read_cb, + /*is_blob*/ nullptr, + /*do_merge*/ true); + RecordTick(stats_, MEMTABLE_MISS); + } + RecordTick(stats_, NUMBER_KEYS_READ); + size_t size = pinnable_val->size(); + RecordTick(stats_, BYTES_READ, size); + RecordInHistogram(stats_, BYTES_PER_READ, size); + PERF_COUNTER_ADD(get_read_bytes, size); + return s; +} + +Iterator* DBImplReadOnly::NewIterator(const ReadOptions& read_options, + ColumnFamilyHandle* column_family) { + if (read_options.io_activity != Env::IOActivity::kUnknown) { + return NewErrorIterator(Status::InvalidArgument( + "Cannot call NewIterator with `ReadOptions::io_activity` != " + "`Env::IOActivity::kUnknown`")); + } + assert(column_family); + if (read_options.timestamp) { + const Status s = FailIfTsMismatchCf( + column_family, *(read_options.timestamp), /*ts_for_read=*/true); + if (!s.ok()) { + return NewErrorIterator(s); + } + } else { + const Status s = FailIfCfHasTs(column_family); + if (!s.ok()) { + return NewErrorIterator(s); + } + } + auto cfh = static_cast_with_check(column_family); + auto cfd = cfh->cfd(); + SuperVersion* super_version = cfd->GetSuperVersion()->Ref(); + SequenceNumber latest_snapshot = versions_->LastSequence(); + SequenceNumber read_seq = + read_options.snapshot != nullptr + ? reinterpret_cast(read_options.snapshot) + ->number_ + : latest_snapshot; + ReadCallback* read_callback = nullptr; // No read callback provided. + auto db_iter = NewArenaWrappedDbIterator( + env_, read_options, *cfd->ioptions(), super_version->mutable_cf_options, + super_version->current, read_seq, + super_version->mutable_cf_options.max_sequential_skip_in_iterations, + super_version->version_number, read_callback); + auto internal_iter = NewInternalIterator( + db_iter->GetReadOptions(), cfd, super_version, db_iter->GetArena(), + read_seq, /* allow_unprepared_value */ true, db_iter); + db_iter->SetIterUnderDBIter(internal_iter); + return db_iter; +} + +Status DBImplReadOnly::NewIterators( + const ReadOptions& read_options, + const std::vector& column_families, + std::vector* iterators) { + if (read_options.timestamp) { + for (auto* cf : column_families) { + assert(cf); + const Status s = FailIfTsMismatchCf(cf, *(read_options.timestamp), + /*ts_for_read=*/true); + if (!s.ok()) { + return s; + } + } + } else { + for (auto* cf : column_families) { + assert(cf); + const Status s = FailIfCfHasTs(cf); + if (!s.ok()) { + return s; + } + } + } + + ReadCallback* read_callback = nullptr; // No read callback provided. + if (iterators == nullptr) { + return Status::InvalidArgument("iterators not allowed to be nullptr"); + } + iterators->clear(); + iterators->reserve(column_families.size()); + SequenceNumber latest_snapshot = versions_->LastSequence(); + SequenceNumber read_seq = + read_options.snapshot != nullptr + ? reinterpret_cast(read_options.snapshot) + ->number_ + : latest_snapshot; + + for (auto cfh : column_families) { + auto* cfd = static_cast_with_check(cfh)->cfd(); + auto* sv = cfd->GetSuperVersion()->Ref(); + auto* db_iter = NewArenaWrappedDbIterator( + env_, read_options, *cfd->ioptions(), sv->mutable_cf_options, + sv->current, read_seq, + sv->mutable_cf_options.max_sequential_skip_in_iterations, + sv->version_number, read_callback); + auto* internal_iter = NewInternalIterator( + db_iter->GetReadOptions(), cfd, sv, db_iter->GetArena(), read_seq, + /* allow_unprepared_value */ true, db_iter); + db_iter->SetIterUnderDBIter(internal_iter); + iterators->push_back(db_iter); + } + + return Status::OK(); +} + +namespace { +// Return OK if dbname exists in the file system or create it if +// create_if_missing +Status OpenForReadOnlyCheckExistence(const DBOptions& db_options, + const std::string& dbname) { + Status s; + if (!db_options.create_if_missing) { + // Attempt to read "CURRENT" file + const std::shared_ptr& fs = db_options.env->GetFileSystem(); + std::string manifest_path; + uint64_t manifest_file_number; + s = VersionSet::GetCurrentManifestPath(dbname, fs.get(), &manifest_path, + &manifest_file_number); + } else { + // Historic behavior that doesn't necessarily make sense + s = db_options.env->CreateDirIfMissing(dbname); + } + return s; +} +} // namespace + +Status DB::OpenForReadOnly(const Options& options, const std::string& dbname, + DB** dbptr, bool /*error_if_wal_file_exists*/) { + Status s = OpenForReadOnlyCheckExistence(options, dbname); + if (!s.ok()) { + return s; + } + + *dbptr = nullptr; + + // Try to first open DB as fully compacted DB + s = CompactedDBImpl::Open(options, dbname, dbptr); + if (s.ok()) { + return s; + } + + DBOptions db_options(options); + ColumnFamilyOptions cf_options(options); + std::vector column_families; + column_families.push_back( + ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options)); + std::vector handles; + + s = DBImplReadOnly::OpenForReadOnlyWithoutCheck( + db_options, dbname, column_families, &handles, dbptr); + if (s.ok()) { + assert(handles.size() == 1); + // i can delete the handle since DBImpl is always holding a + // reference to default column family + delete handles[0]; + } + return s; +} + +Status DB::OpenForReadOnly( + const DBOptions& db_options, const std::string& dbname, + const std::vector& column_families, + std::vector* handles, DB** dbptr, + bool error_if_wal_file_exists) { + // If dbname does not exist in the file system, should not do anything + Status s = OpenForReadOnlyCheckExistence(db_options, dbname); + if (!s.ok()) { + return s; + } + + return DBImplReadOnly::OpenForReadOnlyWithoutCheck( + db_options, dbname, column_families, handles, dbptr, + error_if_wal_file_exists); +} + +Status DBImplReadOnly::OpenForReadOnlyWithoutCheck( + const DBOptions& db_options, const std::string& dbname, + const std::vector& column_families, + std::vector* handles, DB** dbptr, + bool error_if_wal_file_exists) { + *dbptr = nullptr; + handles->clear(); + + SuperVersionContext sv_context(/* create_superversion */ true); + DBImplReadOnly* impl = new DBImplReadOnly(db_options, dbname); + impl->mutex_.Lock(); + Status s = impl->Recover(column_families, true /* read only */, + error_if_wal_file_exists); + if (s.ok()) { + // set column family handles + for (auto cf : column_families) { + auto cfd = + impl->versions_->GetColumnFamilySet()->GetColumnFamily(cf.name); + if (cfd == nullptr) { + s = Status::InvalidArgument("Column family not found", cf.name); + break; + } + handles->push_back(new ColumnFamilyHandleImpl(cfd, impl, &impl->mutex_)); + } + } + if (s.ok()) { + for (auto cfd : *impl->versions_->GetColumnFamilySet()) { + sv_context.NewSuperVersion(); + cfd->InstallSuperVersion(&sv_context, &impl->mutex_); + } + } + impl->mutex_.Unlock(); + sv_context.Clean(); + if (s.ok()) { + *dbptr = impl; + for (auto* h : *handles) { + impl->NewThreadStatusCfInfo( + static_cast_with_check(h)->cfd()); + } + } else { + for (auto h : *handles) { + delete h; + } + handles->clear(); + delete impl; + } + return s; +} + + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/db_impl/db_impl_readonly.h b/librocksdb-sys/rocksdb/db/db_impl/db_impl_readonly.h new file mode 100644 index 0000000..ccd52e4 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_impl/db_impl_readonly.h @@ -0,0 +1,182 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + + +#include +#include + +#include "db/db_impl/db_impl.h" + +namespace ROCKSDB_NAMESPACE { + +// TODO: Share common structure with CompactedDBImpl and DBImplSecondary +class DBImplReadOnly : public DBImpl { + public: + DBImplReadOnly(const DBOptions& options, const std::string& dbname); + // No copying allowed + DBImplReadOnly(const DBImplReadOnly&) = delete; + void operator=(const DBImplReadOnly&) = delete; + + virtual ~DBImplReadOnly(); + + // Implementations of the DB interface + using DB::Get; + virtual Status Get(const ReadOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value) override; + Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family, + const Slice& key, PinnableSlice* value, + std::string* timestamp) override; + + // TODO: Implement ReadOnly MultiGet? + + using DBImpl::NewIterator; + virtual Iterator* NewIterator(const ReadOptions&, + ColumnFamilyHandle* column_family) override; + + virtual Status NewIterators( + const ReadOptions& options, + const std::vector& column_families, + std::vector* iterators) override; + + using DBImpl::Put; + virtual Status Put(const WriteOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/, + const Slice& /*key*/, const Slice& /*value*/) override { + return Status::NotSupported("Not supported operation in read only mode."); + } + + using DBImpl::PutEntity; + Status PutEntity(const WriteOptions& /* options */, + ColumnFamilyHandle* /* column_family */, + const Slice& /* key */, + const WideColumns& /* columns */) override { + return Status::NotSupported("Not supported operation in read only mode."); + } + + using DBImpl::Merge; + virtual Status Merge(const WriteOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/, + const Slice& /*key*/, const Slice& /*value*/) override { + return Status::NotSupported("Not supported operation in read only mode."); + } + using DBImpl::Delete; + virtual Status Delete(const WriteOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/, + const Slice& /*key*/) override { + return Status::NotSupported("Not supported operation in read only mode."); + } + using DBImpl::SingleDelete; + virtual Status SingleDelete(const WriteOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/, + const Slice& /*key*/) override { + return Status::NotSupported("Not supported operation in read only mode."); + } + virtual Status Write(const WriteOptions& /*options*/, + WriteBatch* /*updates*/) override { + return Status::NotSupported("Not supported operation in read only mode."); + } + using DBImpl::CompactRange; + virtual Status CompactRange(const CompactRangeOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/, + const Slice* /*begin*/, + const Slice* /*end*/) override { + return Status::NotSupported("Not supported operation in read only mode."); + } + + using DBImpl::CompactFiles; + virtual Status CompactFiles( + const CompactionOptions& /*compact_options*/, + ColumnFamilyHandle* /*column_family*/, + const std::vector& /*input_file_names*/, + const int /*output_level*/, const int /*output_path_id*/ = -1, + std::vector* const /*output_file_names*/ = nullptr, + CompactionJobInfo* /*compaction_job_info*/ = nullptr) override { + return Status::NotSupported("Not supported operation in read only mode."); + } + + virtual Status DisableFileDeletions() override { + return Status::NotSupported("Not supported operation in read only mode."); + } + + virtual Status EnableFileDeletions(bool /*force*/) override { + return Status::NotSupported("Not supported operation in read only mode."); + } + virtual Status GetLiveFiles(std::vector& ret, + uint64_t* manifest_file_size, + bool /*flush_memtable*/) override { + return DBImpl::GetLiveFiles(ret, manifest_file_size, + false /* flush_memtable */); + } + + using DBImpl::Flush; + virtual Status Flush(const FlushOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/) override { + return Status::NotSupported("Not supported operation in read only mode."); + } + + using DBImpl::SyncWAL; + virtual Status SyncWAL() override { + return Status::NotSupported("Not supported operation in read only mode."); + } + + using DB::IngestExternalFile; + virtual Status IngestExternalFile( + ColumnFamilyHandle* /*column_family*/, + const std::vector& /*external_files*/, + const IngestExternalFileOptions& /*ingestion_options*/) override { + return Status::NotSupported("Not supported operation in read only mode."); + } + + using DB::CreateColumnFamilyWithImport; + virtual Status CreateColumnFamilyWithImport( + const ColumnFamilyOptions& /*options*/, + const std::string& /*column_family_name*/, + const ImportColumnFamilyOptions& /*import_options*/, + const ExportImportFilesMetaData& /*metadata*/, + ColumnFamilyHandle** /*handle*/) override { + return Status::NotSupported("Not supported operation in read only mode."); + } + + virtual Status CreateColumnFamilyWithImport( + const ColumnFamilyOptions& /*options*/, + const std::string& /*column_family_name*/, + const ImportColumnFamilyOptions& /*import_options*/, + const std::vector& /*metadatas*/, + ColumnFamilyHandle** /*handle*/) override { + return Status::NotSupported("Not supported operation in read only mode."); + } + + using DB::ClipColumnFamily; + virtual Status ClipColumnFamily(ColumnFamilyHandle* /*column_family*/, + const Slice& /*begin*/, + const Slice& /*end*/) override { + return Status::NotSupported("Not supported operation in read only mode."); + } + + // FIXME: some missing overrides for more "write" functions + + protected: + Status FlushForGetLiveFiles() override { + // No-op for read-only DB + return Status::OK(); + } + + private: + // A "helper" function for DB::OpenForReadOnly without column families + // to reduce unnecessary I/O + // It has the same functionality as DB::OpenForReadOnly with column families + // but does not check the existence of dbname in the file system + static Status OpenForReadOnlyWithoutCheck( + const DBOptions& db_options, const std::string& dbname, + const std::vector& column_families, + std::vector* handles, DB** dbptr, + bool error_if_wal_file_exists = false); + friend class DB; +}; +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/db/db_impl/db_impl_secondary.cc b/librocksdb-sys/rocksdb/db/db_impl/db_impl_secondary.cc new file mode 100644 index 0000000..de35368 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_impl/db_impl_secondary.cc @@ -0,0 +1,976 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/db_impl/db_impl_secondary.h" + +#include + +#include "db/arena_wrapped_db_iter.h" +#include "db/merge_context.h" +#include "logging/auto_roll_logger.h" +#include "logging/logging.h" +#include "monitoring/perf_context_imp.h" +#include "rocksdb/configurable.h" +#include "util/cast_util.h" +#include "util/write_batch_util.h" + +namespace ROCKSDB_NAMESPACE { + +DBImplSecondary::DBImplSecondary(const DBOptions& db_options, + const std::string& dbname, + std::string secondary_path) + : DBImpl(db_options, dbname, false, true, true), + secondary_path_(std::move(secondary_path)) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "Opening the db in secondary mode"); + LogFlush(immutable_db_options_.info_log); +} + +DBImplSecondary::~DBImplSecondary() {} + +Status DBImplSecondary::Recover( + const std::vector& column_families, + bool /*readonly*/, bool /*error_if_wal_file_exists*/, + bool /*error_if_data_exists_in_wals*/, uint64_t*, + RecoveryContext* /*recovery_ctx*/) { + mutex_.AssertHeld(); + + JobContext job_context(0); + Status s; + s = static_cast(versions_.get()) + ->Recover(column_families, &manifest_reader_, &manifest_reporter_, + &manifest_reader_status_); + if (!s.ok()) { + if (manifest_reader_status_) { + manifest_reader_status_->PermitUncheckedError(); + } + return s; + } + if (immutable_db_options_.paranoid_checks && s.ok()) { + s = CheckConsistency(); + } + // Initial max_total_in_memory_state_ before recovery logs. + max_total_in_memory_state_ = 0; + for (auto cfd : *versions_->GetColumnFamilySet()) { + auto* mutable_cf_options = cfd->GetLatestMutableCFOptions(); + max_total_in_memory_state_ += mutable_cf_options->write_buffer_size * + mutable_cf_options->max_write_buffer_number; + } + if (s.ok()) { + default_cf_handle_ = new ColumnFamilyHandleImpl( + versions_->GetColumnFamilySet()->GetDefault(), this, &mutex_); + default_cf_internal_stats_ = default_cf_handle_->cfd()->internal_stats(); + + std::unordered_set cfds_changed; + s = FindAndRecoverLogFiles(&cfds_changed, &job_context); + } + + if (s.IsPathNotFound()) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "Secondary tries to read WAL, but WAL file(s) have already " + "been purged by primary."); + s = Status::OK(); + } + // TODO: update options_file_number_ needed? + + job_context.Clean(); + return s; +} + +// find new WAL and apply them in order to the secondary instance +Status DBImplSecondary::FindAndRecoverLogFiles( + std::unordered_set* cfds_changed, + JobContext* job_context) { + assert(nullptr != cfds_changed); + assert(nullptr != job_context); + Status s; + std::vector logs; + s = FindNewLogNumbers(&logs); + if (s.ok() && !logs.empty()) { + SequenceNumber next_sequence(kMaxSequenceNumber); + s = RecoverLogFiles(logs, &next_sequence, cfds_changed, job_context); + } + return s; +} + +// List wal_dir and find all new WALs, return these log numbers +Status DBImplSecondary::FindNewLogNumbers(std::vector* logs) { + assert(logs != nullptr); + std::vector filenames; + Status s; + IOOptions io_opts; + io_opts.do_not_recurse = true; + s = immutable_db_options_.fs->GetChildren(immutable_db_options_.GetWalDir(), + io_opts, &filenames, + /*IODebugContext*=*/nullptr); + if (s.IsNotFound()) { + return Status::InvalidArgument("Failed to open wal_dir", + immutable_db_options_.GetWalDir()); + } else if (!s.ok()) { + return s; + } + + // if log_readers_ is non-empty, it means we have applied all logs with log + // numbers smaller than the smallest log in log_readers_, so there is no + // need to pass these logs to RecoverLogFiles + uint64_t log_number_min = 0; + if (!log_readers_.empty()) { + log_number_min = log_readers_.begin()->first; + } + for (size_t i = 0; i < filenames.size(); i++) { + uint64_t number; + FileType type; + if (ParseFileName(filenames[i], &number, &type) && type == kWalFile && + number >= log_number_min) { + logs->push_back(number); + } + } + // Recover logs in the order that they were generated + if (!logs->empty()) { + std::sort(logs->begin(), logs->end()); + } + return s; +} + +Status DBImplSecondary::MaybeInitLogReader( + uint64_t log_number, log::FragmentBufferedReader** log_reader) { + auto iter = log_readers_.find(log_number); + // make sure the log file is still present + if (iter == log_readers_.end() || + iter->second->reader_->GetLogNumber() != log_number) { + // delete the obsolete log reader if log number mismatch + if (iter != log_readers_.end()) { + log_readers_.erase(iter); + } + // initialize log reader from log_number + // TODO: min_log_number_to_keep_2pc check needed? + // Open the log file + std::string fname = + LogFileName(immutable_db_options_.GetWalDir(), log_number); + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "Recovering log #%" PRIu64 " mode %d", log_number, + static_cast(immutable_db_options_.wal_recovery_mode)); + + std::unique_ptr file_reader; + { + std::unique_ptr file; + Status status = fs_->NewSequentialFile( + fname, fs_->OptimizeForLogRead(file_options_), &file, nullptr); + if (!status.ok()) { + *log_reader = nullptr; + return status; + } + file_reader.reset(new SequentialFileReader( + std::move(file), fname, immutable_db_options_.log_readahead_size, + io_tracer_)); + } + + // Create the log reader. + LogReaderContainer* log_reader_container = new LogReaderContainer( + env_, immutable_db_options_.info_log, std::move(fname), + std::move(file_reader), log_number); + log_readers_.insert(std::make_pair( + log_number, std::unique_ptr(log_reader_container))); + } + iter = log_readers_.find(log_number); + assert(iter != log_readers_.end()); + *log_reader = iter->second->reader_; + return Status::OK(); +} + +// After manifest recovery, replay WALs and refresh log_readers_ if necessary +// REQUIRES: log_numbers are sorted in ascending order +Status DBImplSecondary::RecoverLogFiles( + const std::vector& log_numbers, SequenceNumber* next_sequence, + std::unordered_set* cfds_changed, + JobContext* job_context) { + assert(nullptr != cfds_changed); + assert(nullptr != job_context); + mutex_.AssertHeld(); + Status status; + for (auto log_number : log_numbers) { + log::FragmentBufferedReader* reader = nullptr; + status = MaybeInitLogReader(log_number, &reader); + if (!status.ok()) { + return status; + } + assert(reader != nullptr); + } + + const UnorderedMap& running_ts_sz = + versions_->GetRunningColumnFamiliesTimestampSize(); + for (auto log_number : log_numbers) { + auto it = log_readers_.find(log_number); + assert(it != log_readers_.end()); + log::FragmentBufferedReader* reader = it->second->reader_; + Status* wal_read_status = it->second->status_; + assert(wal_read_status); + // Manually update the file number allocation counter in VersionSet. + versions_->MarkFileNumberUsed(log_number); + + // Determine if we should tolerate incomplete records at the tail end of the + // Read all the records and add to a memtable + std::string scratch; + Slice record; + WriteBatch batch; + + while (reader->ReadRecord(&record, &scratch, + immutable_db_options_.wal_recovery_mode) && + wal_read_status->ok() && status.ok()) { + if (record.size() < WriteBatchInternal::kHeader) { + reader->GetReporter()->Corruption( + record.size(), Status::Corruption("log record too small")); + continue; + } + status = WriteBatchInternal::SetContents(&batch, record); + if (!status.ok()) { + break; + } + const UnorderedMap& record_ts_sz = + reader->GetRecordedTimestampSize(); + status = HandleWriteBatchTimestampSizeDifference( + &batch, running_ts_sz, record_ts_sz, + TimestampSizeConsistencyMode::kVerifyConsistency); + if (!status.ok()) { + break; + } + SequenceNumber seq_of_batch = WriteBatchInternal::Sequence(&batch); + std::vector column_family_ids; + status = CollectColumnFamilyIdsFromWriteBatch(batch, &column_family_ids); + if (status.ok()) { + for (const auto id : column_family_ids) { + ColumnFamilyData* cfd = + versions_->GetColumnFamilySet()->GetColumnFamily(id); + if (cfd == nullptr) { + continue; + } + if (cfds_changed->count(cfd) == 0) { + cfds_changed->insert(cfd); + } + const std::vector& l0_files = + cfd->current()->storage_info()->LevelFiles(0); + SequenceNumber seq = + l0_files.empty() ? 0 : l0_files.back()->fd.largest_seqno; + // If the write batch's sequence number is smaller than the last + // sequence number of the largest sequence persisted for this column + // family, then its data must reside in an SST that has already been + // added in the prior MANIFEST replay. + if (seq_of_batch <= seq) { + continue; + } + auto curr_log_num = std::numeric_limits::max(); + if (cfd_to_current_log_.count(cfd) > 0) { + curr_log_num = cfd_to_current_log_[cfd]; + } + // If the active memtable contains records added by replaying an + // earlier WAL, then we need to seal the memtable, add it to the + // immutable memtable list and create a new active memtable. + if (!cfd->mem()->IsEmpty() && + (curr_log_num == std::numeric_limits::max() || + curr_log_num != log_number)) { + const MutableCFOptions mutable_cf_options = + *cfd->GetLatestMutableCFOptions(); + MemTable* new_mem = + cfd->ConstructNewMemtable(mutable_cf_options, seq_of_batch); + cfd->mem()->SetNextLogNumber(log_number); + cfd->mem()->ConstructFragmentedRangeTombstones(); + cfd->imm()->Add(cfd->mem(), &job_context->memtables_to_free); + new_mem->Ref(); + cfd->SetMemtable(new_mem); + } + } + bool has_valid_writes = false; + status = WriteBatchInternal::InsertInto( + &batch, column_family_memtables_.get(), + nullptr /* flush_scheduler */, nullptr /* trim_history_scheduler*/, + true, log_number, this, false /* concurrent_memtable_writes */, + next_sequence, &has_valid_writes, seq_per_batch_, batch_per_txn_); + } + // If column family was not found, it might mean that the WAL write + // batch references to the column family that was dropped after the + // insert. We don't want to fail the whole write batch in that case -- + // we just ignore the update. + // That's why we set ignore missing column families to true + // passing null flush_scheduler will disable memtable flushing which is + // needed for secondary instances + if (status.ok()) { + for (const auto id : column_family_ids) { + ColumnFamilyData* cfd = + versions_->GetColumnFamilySet()->GetColumnFamily(id); + if (cfd == nullptr) { + continue; + } + std::unordered_map::iterator iter = + cfd_to_current_log_.find(cfd); + if (iter == cfd_to_current_log_.end()) { + cfd_to_current_log_.insert({cfd, log_number}); + } else if (log_number > iter->second) { + iter->second = log_number; + } + } + auto last_sequence = *next_sequence - 1; + if ((*next_sequence != kMaxSequenceNumber) && + (versions_->LastSequence() <= last_sequence)) { + versions_->SetLastAllocatedSequence(last_sequence); + versions_->SetLastPublishedSequence(last_sequence); + versions_->SetLastSequence(last_sequence); + } + } else { + // We are treating this as a failure while reading since we read valid + // blocks that do not form coherent data + reader->GetReporter()->Corruption(record.size(), status); + } + } + if (status.ok() && !wal_read_status->ok()) { + status = *wal_read_status; + } + if (!status.ok()) { + return status; + } + } + // remove logreaders from map after successfully recovering the WAL + if (log_readers_.size() > 1) { + auto erase_iter = log_readers_.begin(); + std::advance(erase_iter, log_readers_.size() - 1); + log_readers_.erase(log_readers_.begin(), erase_iter); + } + return status; +} + +// Implementation of the DB interface +Status DBImplSecondary::Get(const ReadOptions& read_options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value) { + return GetImpl(read_options, column_family, key, value, + /*timestamp*/ nullptr); +} + +Status DBImplSecondary::Get(const ReadOptions& read_options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value, std::string* timestamp) { + return GetImpl(read_options, column_family, key, value, timestamp); +} + +Status DBImplSecondary::GetImpl(const ReadOptions& read_options, + ColumnFamilyHandle* column_family, + const Slice& key, PinnableSlice* pinnable_val, + std::string* timestamp) { + if (read_options.io_activity != Env::IOActivity::kUnknown) { + return Status::InvalidArgument( + "Cannot call Get with `ReadOptions::io_activity` != " + "`Env::IOActivity::kUnknown`"); + } + assert(pinnable_val != nullptr); + PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock); + StopWatch sw(immutable_db_options_.clock, stats_, DB_GET); + PERF_TIMER_GUARD(get_snapshot_time); + + assert(column_family); + if (read_options.timestamp) { + const Status s = FailIfTsMismatchCf( + column_family, *(read_options.timestamp), /*ts_for_read=*/true); + if (!s.ok()) { + return s; + } + } else { + const Status s = FailIfCfHasTs(column_family); + if (!s.ok()) { + return s; + } + } + + // Clear the timestamp for returning results so that we can distinguish + // between tombstone or key that has never been written later. + if (timestamp) { + timestamp->clear(); + } + + auto cfh = static_cast(column_family); + ColumnFamilyData* cfd = cfh->cfd(); + if (tracer_) { + InstrumentedMutexLock lock(&trace_mutex_); + if (tracer_) { + tracer_->Get(column_family, key); + } + } + // Acquire SuperVersion + SuperVersion* super_version = GetAndRefSuperVersion(cfd); + SequenceNumber snapshot = versions_->LastSequence(); + GetWithTimestampReadCallback read_cb(snapshot); + MergeContext merge_context; + SequenceNumber max_covering_tombstone_seq = 0; + Status s; + LookupKey lkey(key, snapshot, read_options.timestamp); + PERF_TIMER_STOP(get_snapshot_time); + + bool done = false; + const Comparator* ucmp = column_family->GetComparator(); + assert(ucmp); + std::string* ts = ucmp->timestamp_size() > 0 ? timestamp : nullptr; + if (super_version->mem->Get(lkey, pinnable_val->GetSelf(), + /*columns=*/nullptr, ts, &s, &merge_context, + &max_covering_tombstone_seq, read_options, + false /* immutable_memtable */, &read_cb)) { + done = true; + pinnable_val->PinSelf(); + RecordTick(stats_, MEMTABLE_HIT); + } else if ((s.ok() || s.IsMergeInProgress()) && + super_version->imm->Get( + lkey, pinnable_val->GetSelf(), /*columns=*/nullptr, ts, &s, + &merge_context, &max_covering_tombstone_seq, read_options, + &read_cb)) { + done = true; + pinnable_val->PinSelf(); + RecordTick(stats_, MEMTABLE_HIT); + } + if (!done && !s.ok() && !s.IsMergeInProgress()) { + ReturnAndCleanupSuperVersion(cfd, super_version); + return s; + } + if (!done) { + PERF_TIMER_GUARD(get_from_output_files_time); + PinnedIteratorsManager pinned_iters_mgr; + super_version->current->Get( + read_options, lkey, pinnable_val, /*columns=*/nullptr, ts, &s, + &merge_context, &max_covering_tombstone_seq, &pinned_iters_mgr, + /*value_found*/ nullptr, + /*key_exists*/ nullptr, /*seq*/ nullptr, &read_cb, /*is_blob*/ nullptr, + /*do_merge*/ true); + RecordTick(stats_, MEMTABLE_MISS); + } + { + PERF_TIMER_GUARD(get_post_process_time); + ReturnAndCleanupSuperVersion(cfd, super_version); + RecordTick(stats_, NUMBER_KEYS_READ); + size_t size = pinnable_val->size(); + RecordTick(stats_, BYTES_READ, size); + RecordTimeToHistogram(stats_, BYTES_PER_READ, size); + PERF_COUNTER_ADD(get_read_bytes, size); + } + return s; +} + +Iterator* DBImplSecondary::NewIterator(const ReadOptions& read_options, + ColumnFamilyHandle* column_family) { + if (read_options.managed) { + return NewErrorIterator( + Status::NotSupported("Managed iterator is not supported anymore.")); + } + if (read_options.read_tier == kPersistedTier) { + return NewErrorIterator(Status::NotSupported( + "ReadTier::kPersistedData is not yet supported in iterators.")); + } + if (read_options.io_activity != Env::IOActivity::kUnknown) { + return NewErrorIterator(Status::InvalidArgument( + "Cannot call NewIterator with `ReadOptions::io_activity` != " + "`Env::IOActivity::kUnknown`")); + } + + assert(column_family); + if (read_options.timestamp) { + const Status s = FailIfTsMismatchCf( + column_family, *(read_options.timestamp), /*ts_for_read=*/true); + if (!s.ok()) { + return NewErrorIterator(s); + } + } else { + const Status s = FailIfCfHasTs(column_family); + if (!s.ok()) { + return NewErrorIterator(s); + } + } + + Iterator* result = nullptr; + auto cfh = static_cast_with_check(column_family); + auto cfd = cfh->cfd(); + ReadCallback* read_callback = nullptr; // No read callback provided. + if (read_options.tailing) { + return NewErrorIterator(Status::NotSupported( + "tailing iterator not supported in secondary mode")); + } else if (read_options.snapshot != nullptr) { + // TODO (yanqin) support snapshot. + return NewErrorIterator( + Status::NotSupported("snapshot not supported in secondary mode")); + } else { + SequenceNumber snapshot(kMaxSequenceNumber); + result = NewIteratorImpl(read_options, cfd, snapshot, read_callback); + } + return result; +} + +ArenaWrappedDBIter* DBImplSecondary::NewIteratorImpl( + const ReadOptions& read_options, ColumnFamilyData* cfd, + SequenceNumber snapshot, ReadCallback* read_callback, + bool expose_blob_index, bool allow_refresh) { + assert(nullptr != cfd); + SuperVersion* super_version = cfd->GetReferencedSuperVersion(this); + assert(snapshot == kMaxSequenceNumber); + snapshot = versions_->LastSequence(); + assert(snapshot != kMaxSequenceNumber); + auto db_iter = NewArenaWrappedDbIterator( + env_, read_options, *cfd->ioptions(), super_version->mutable_cf_options, + super_version->current, snapshot, + super_version->mutable_cf_options.max_sequential_skip_in_iterations, + super_version->version_number, read_callback, this, cfd, + expose_blob_index, read_options.snapshot ? false : allow_refresh); + auto internal_iter = NewInternalIterator( + db_iter->GetReadOptions(), cfd, super_version, db_iter->GetArena(), + snapshot, /* allow_unprepared_value */ true, db_iter); + db_iter->SetIterUnderDBIter(internal_iter); + return db_iter; +} + +Status DBImplSecondary::NewIterators( + const ReadOptions& read_options, + const std::vector& column_families, + std::vector* iterators) { + if (read_options.managed) { + return Status::NotSupported("Managed iterator is not supported anymore."); + } + if (read_options.read_tier == kPersistedTier) { + return Status::NotSupported( + "ReadTier::kPersistedData is not yet supported in iterators."); + } + if (read_options.io_activity != Env::IOActivity::kUnknown) { + return Status::InvalidArgument( + "Cannot call NewIterators with `ReadOptions::io_activity` != " + "`Env::IOActivity::kUnknown`"); + } + ReadCallback* read_callback = nullptr; // No read callback provided. + if (iterators == nullptr) { + return Status::InvalidArgument("iterators not allowed to be nullptr"); + } + + if (read_options.timestamp) { + for (auto* cf : column_families) { + assert(cf); + const Status s = FailIfTsMismatchCf(cf, *(read_options.timestamp), + /*ts_for_read=*/true); + if (!s.ok()) { + return s; + } + } + } else { + for (auto* cf : column_families) { + assert(cf); + const Status s = FailIfCfHasTs(cf); + if (!s.ok()) { + return s; + } + } + } + iterators->clear(); + iterators->reserve(column_families.size()); + if (read_options.tailing) { + return Status::NotSupported( + "tailing iterator not supported in secondary mode"); + } else if (read_options.snapshot != nullptr) { + // TODO (yanqin) support snapshot. + return Status::NotSupported("snapshot not supported in secondary mode"); + } else { + SequenceNumber read_seq(kMaxSequenceNumber); + for (auto cfh : column_families) { + ColumnFamilyData* cfd = static_cast(cfh)->cfd(); + iterators->push_back( + NewIteratorImpl(read_options, cfd, read_seq, read_callback)); + } + } + return Status::OK(); +} + +Status DBImplSecondary::CheckConsistency() { + mutex_.AssertHeld(); + Status s = DBImpl::CheckConsistency(); + // If DBImpl::CheckConsistency() which is stricter returns success, then we + // do not need to give a second chance. + if (s.ok()) { + return s; + } + // It's possible that DBImpl::CheckConssitency() can fail because the primary + // may have removed certain files, causing the GetFileSize(name) call to + // fail and returning a PathNotFound. In this case, we take a best-effort + // approach and just proceed. + TEST_SYNC_POINT_CALLBACK( + "DBImplSecondary::CheckConsistency:AfterFirstAttempt", &s); + + if (immutable_db_options_.skip_checking_sst_file_sizes_on_db_open) { + return Status::OK(); + } + + std::vector metadata; + versions_->GetLiveFilesMetaData(&metadata); + + std::string corruption_messages; + for (const auto& md : metadata) { + // md.name has a leading "/". + std::string file_path = md.db_path + md.name; + + uint64_t fsize = 0; + s = env_->GetFileSize(file_path, &fsize); + if (!s.ok() && + (env_->GetFileSize(Rocks2LevelTableFileName(file_path), &fsize).ok() || + s.IsPathNotFound())) { + s = Status::OK(); + } + if (!s.ok()) { + corruption_messages += + "Can't access " + md.name + ": " + s.ToString() + "\n"; + } + } + return corruption_messages.empty() ? Status::OK() + : Status::Corruption(corruption_messages); +} + +Status DBImplSecondary::TryCatchUpWithPrimary() { + assert(versions_.get() != nullptr); + assert(manifest_reader_.get() != nullptr); + Status s; + // read the manifest and apply new changes to the secondary instance + std::unordered_set cfds_changed; + JobContext job_context(0, true /*create_superversion*/); + { + InstrumentedMutexLock lock_guard(&mutex_); + s = static_cast_with_check(versions_.get()) + ->ReadAndApply(&mutex_, &manifest_reader_, + manifest_reader_status_.get(), &cfds_changed); + + ROCKS_LOG_INFO(immutable_db_options_.info_log, "Last sequence is %" PRIu64, + static_cast(versions_->LastSequence())); + for (ColumnFamilyData* cfd : cfds_changed) { + if (cfd->IsDropped()) { + ROCKS_LOG_DEBUG(immutable_db_options_.info_log, "[%s] is dropped\n", + cfd->GetName().c_str()); + continue; + } + VersionStorageInfo::LevelSummaryStorage tmp; + ROCKS_LOG_DEBUG(immutable_db_options_.info_log, + "[%s] Level summary: %s\n", cfd->GetName().c_str(), + cfd->current()->storage_info()->LevelSummary(&tmp)); + } + + // list wal_dir to discover new WALs and apply new changes to the secondary + // instance + if (s.ok()) { + s = FindAndRecoverLogFiles(&cfds_changed, &job_context); + } + if (s.IsPathNotFound()) { + ROCKS_LOG_INFO( + immutable_db_options_.info_log, + "Secondary tries to read WAL, but WAL file(s) have already " + "been purged by primary."); + s = Status::OK(); + } + if (s.ok()) { + for (auto cfd : cfds_changed) { + cfd->imm()->RemoveOldMemTables(cfd->GetLogNumber(), + &job_context.memtables_to_free); + auto& sv_context = job_context.superversion_contexts.back(); + cfd->InstallSuperVersion(&sv_context, &mutex_); + sv_context.NewSuperVersion(); + } + } + } + job_context.Clean(); + + // Cleanup unused, obsolete files. + JobContext purge_files_job_context(0); + { + InstrumentedMutexLock lock_guard(&mutex_); + // Currently, secondary instance does not own the database files, thus it + // is unnecessary for the secondary to force full scan. + FindObsoleteFiles(&purge_files_job_context, /*force=*/false); + } + if (purge_files_job_context.HaveSomethingToDelete()) { + PurgeObsoleteFiles(purge_files_job_context); + } + purge_files_job_context.Clean(); + return s; +} + +Status DB::OpenAsSecondary(const Options& options, const std::string& dbname, + const std::string& secondary_path, DB** dbptr) { + *dbptr = nullptr; + + DBOptions db_options(options); + ColumnFamilyOptions cf_options(options); + std::vector column_families; + column_families.emplace_back(kDefaultColumnFamilyName, cf_options); + std::vector handles; + + Status s = DB::OpenAsSecondary(db_options, dbname, secondary_path, + column_families, &handles, dbptr); + if (s.ok()) { + assert(handles.size() == 1); + delete handles[0]; + } + return s; +} + +Status DB::OpenAsSecondary( + const DBOptions& db_options, const std::string& dbname, + const std::string& secondary_path, + const std::vector& column_families, + std::vector* handles, DB** dbptr) { + *dbptr = nullptr; + + DBOptions tmp_opts(db_options); + Status s; + if (nullptr == tmp_opts.info_log) { + s = CreateLoggerFromOptions(secondary_path, tmp_opts, &tmp_opts.info_log); + if (!s.ok()) { + tmp_opts.info_log = nullptr; + return s; + } + } + + assert(tmp_opts.info_log != nullptr); + if (db_options.max_open_files != -1) { + std::ostringstream oss; + oss << "The primary instance may delete all types of files after they " + "become obsolete. The application can coordinate the primary and " + "secondary so that primary does not delete/rename files that are " + "currently being used by the secondary. Alternatively, a custom " + "Env/FS can be provided such that files become inaccessible only " + "after all primary and secondaries indicate that they are obsolete " + "and deleted. If the above two are not possible, you can open the " + "secondary instance with `max_open_files==-1` so that secondary " + "will eagerly keep all table files open. Even if a file is deleted, " + "its content can still be accessed via a prior open file " + "descriptor. This is a hacky workaround for only table files. If " + "none of the above is done, then point lookup or " + "range scan via the secondary instance can result in IOError: file " + "not found. This can be resolved by retrying " + "TryCatchUpWithPrimary()."; + ROCKS_LOG_WARN(tmp_opts.info_log, "%s", oss.str().c_str()); + } + + handles->clear(); + DBImplSecondary* impl = new DBImplSecondary(tmp_opts, dbname, secondary_path); + impl->versions_.reset(new ReactiveVersionSet( + dbname, &impl->immutable_db_options_, impl->file_options_, + impl->table_cache_.get(), impl->write_buffer_manager_, + &impl->write_controller_, impl->io_tracer_)); + impl->column_family_memtables_.reset( + new ColumnFamilyMemTablesImpl(impl->versions_->GetColumnFamilySet())); + impl->wal_in_db_path_ = impl->immutable_db_options_.IsWalDirSameAsDBPath(); + + impl->mutex_.Lock(); + s = impl->Recover(column_families, true, false, false); + if (s.ok()) { + for (auto cf : column_families) { + auto cfd = + impl->versions_->GetColumnFamilySet()->GetColumnFamily(cf.name); + if (nullptr == cfd) { + s = Status::InvalidArgument("Column family not found", cf.name); + break; + } + handles->push_back(new ColumnFamilyHandleImpl(cfd, impl, &impl->mutex_)); + } + } + SuperVersionContext sv_context(true /* create_superversion */); + if (s.ok()) { + for (auto cfd : *impl->versions_->GetColumnFamilySet()) { + sv_context.NewSuperVersion(); + cfd->InstallSuperVersion(&sv_context, &impl->mutex_); + } + } + impl->mutex_.Unlock(); + sv_context.Clean(); + if (s.ok()) { + *dbptr = impl; + for (auto h : *handles) { + impl->NewThreadStatusCfInfo( + static_cast_with_check(h)->cfd()); + } + } else { + for (auto h : *handles) { + delete h; + } + handles->clear(); + delete impl; + } + return s; +} + +Status DBImplSecondary::CompactWithoutInstallation( + const OpenAndCompactOptions& options, ColumnFamilyHandle* cfh, + const CompactionServiceInput& input, CompactionServiceResult* result) { + if (options.canceled && options.canceled->load(std::memory_order_acquire)) { + return Status::Incomplete(Status::SubCode::kManualCompactionPaused); + } + InstrumentedMutexLock l(&mutex_); + auto cfd = static_cast_with_check(cfh)->cfd(); + if (!cfd) { + return Status::InvalidArgument("Cannot find column family" + + cfh->GetName()); + } + + std::unordered_set input_set; + for (const auto& file_name : input.input_files) { + input_set.insert(TableFileNameToNumber(file_name)); + } + + auto* version = cfd->current(); + + ColumnFamilyMetaData cf_meta; + version->GetColumnFamilyMetaData(&cf_meta); + + const MutableCFOptions* mutable_cf_options = cfd->GetLatestMutableCFOptions(); + ColumnFamilyOptions cf_options = cfd->GetLatestCFOptions(); + VersionStorageInfo* vstorage = version->storage_info(); + + // Use comp_options to reuse some CompactFiles functions + CompactionOptions comp_options; + comp_options.compression = kDisableCompressionOption; + comp_options.output_file_size_limit = MaxFileSizeForLevel( + *mutable_cf_options, input.output_level, cf_options.compaction_style, + vstorage->base_level(), cf_options.level_compaction_dynamic_level_bytes); + + std::vector input_files; + Status s = cfd->compaction_picker()->GetCompactionInputsFromFileNumbers( + &input_files, &input_set, vstorage, comp_options); + if (!s.ok()) { + return s; + } + + std::unique_ptr c; + assert(cfd->compaction_picker()); + c.reset(cfd->compaction_picker()->CompactFiles( + comp_options, input_files, input.output_level, vstorage, + *mutable_cf_options, mutable_db_options_, 0)); + assert(c != nullptr); + + c->SetInputVersion(version); + + // Create output directory if it's not existed yet + std::unique_ptr output_dir; + s = CreateAndNewDirectory(fs_.get(), secondary_path_, &output_dir); + if (!s.ok()) { + return s; + } + + LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, + immutable_db_options_.info_log.get()); + + const int job_id = next_job_id_.fetch_add(1); + + // use primary host's db_id for running the compaction, but db_session_id is + // using the local one, which is to make sure the unique id is unique from + // the remote compactors. Because the id is generated from db_id, + // db_session_id and orig_file_number, unlike the local compaction, remote + // compaction cannot guarantee the uniqueness of orig_file_number, the file + // number is only assigned when compaction is done. + CompactionServiceCompactionJob compaction_job( + job_id, c.get(), immutable_db_options_, mutable_db_options_, + file_options_for_compaction_, versions_.get(), &shutting_down_, + &log_buffer, output_dir.get(), stats_, &mutex_, &error_handler_, + input.snapshots, table_cache_, &event_logger_, dbname_, io_tracer_, + options.canceled ? *options.canceled : kManualCompactionCanceledFalse_, + input.db_id, db_session_id_, secondary_path_, input, result); + + mutex_.Unlock(); + s = compaction_job.Run(); + mutex_.Lock(); + + // clean up + compaction_job.io_status().PermitUncheckedError(); + compaction_job.CleanupCompaction(); + c->ReleaseCompactionFiles(s); + c.reset(); + + TEST_SYNC_POINT_CALLBACK("DBImplSecondary::CompactWithoutInstallation::End", + &s); + result->status = s; + return s; +} + +Status DB::OpenAndCompact( + const OpenAndCompactOptions& options, const std::string& name, + const std::string& output_directory, const std::string& input, + std::string* output, + const CompactionServiceOptionsOverride& override_options) { + if (options.canceled && options.canceled->load(std::memory_order_acquire)) { + return Status::Incomplete(Status::SubCode::kManualCompactionPaused); + } + CompactionServiceInput compaction_input; + Status s = CompactionServiceInput::Read(input, &compaction_input); + if (!s.ok()) { + return s; + } + + compaction_input.db_options.max_open_files = -1; + compaction_input.db_options.compaction_service = nullptr; + if (compaction_input.db_options.statistics) { + compaction_input.db_options.statistics.reset(); + } + compaction_input.db_options.env = override_options.env; + compaction_input.db_options.file_checksum_gen_factory = + override_options.file_checksum_gen_factory; + compaction_input.db_options.statistics = override_options.statistics; + compaction_input.column_family.options.comparator = + override_options.comparator; + compaction_input.column_family.options.merge_operator = + override_options.merge_operator; + compaction_input.column_family.options.compaction_filter = + override_options.compaction_filter; + compaction_input.column_family.options.compaction_filter_factory = + override_options.compaction_filter_factory; + compaction_input.column_family.options.prefix_extractor = + override_options.prefix_extractor; + compaction_input.column_family.options.table_factory = + override_options.table_factory; + compaction_input.column_family.options.sst_partitioner_factory = + override_options.sst_partitioner_factory; + compaction_input.column_family.options.table_properties_collector_factories = + override_options.table_properties_collector_factories; + compaction_input.db_options.listeners = override_options.listeners; + + std::vector column_families; + column_families.push_back(compaction_input.column_family); + // TODO: we have to open default CF, because of an implementation limitation, + // currently we just use the same CF option from input, which is not collect + // and open may fail. + if (compaction_input.column_family.name != kDefaultColumnFamilyName) { + column_families.emplace_back(kDefaultColumnFamilyName, + compaction_input.column_family.options); + } + + DB* db; + std::vector handles; + + s = DB::OpenAsSecondary(compaction_input.db_options, name, output_directory, + column_families, &handles, &db); + if (!s.ok()) { + return s; + } + + CompactionServiceResult compaction_result; + DBImplSecondary* db_secondary = static_cast_with_check(db); + assert(handles.size() > 0); + s = db_secondary->CompactWithoutInstallation( + options, handles[0], compaction_input, &compaction_result); + + Status serialization_status = compaction_result.Write(output); + + for (auto& handle : handles) { + delete handle; + } + delete db; + if (s.ok()) { + return serialization_status; + } + return s; +} + +Status DB::OpenAndCompact( + const std::string& name, const std::string& output_directory, + const std::string& input, std::string* output, + const CompactionServiceOptionsOverride& override_options) { + return OpenAndCompact(OpenAndCompactOptions(), name, output_directory, input, + output, override_options); +} + + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/db_impl/db_impl_secondary.h b/librocksdb-sys/rocksdb/db/db_impl/db_impl_secondary.h new file mode 100644 index 0000000..faaa987 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_impl/db_impl_secondary.h @@ -0,0 +1,327 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + + +#include +#include + +#include "db/db_impl/db_impl.h" +#include "logging/logging.h" + +namespace ROCKSDB_NAMESPACE { + +// A wrapper class to hold log reader, log reporter, log status. +class LogReaderContainer { + public: + LogReaderContainer() + : reader_(nullptr), reporter_(nullptr), status_(nullptr) {} + LogReaderContainer(Env* env, std::shared_ptr info_log, + std::string fname, + std::unique_ptr&& file_reader, + uint64_t log_number) { + LogReporter* reporter = new LogReporter(); + status_ = new Status(); + reporter->env = env; + reporter->info_log = info_log.get(); + reporter->fname = std::move(fname); + reporter->status = status_; + reporter_ = reporter; + // We intentially make log::Reader do checksumming even if + // paranoid_checks==false so that corruptions cause entire commits + // to be skipped instead of propagating bad information (like overly + // large sequence numbers). + reader_ = new log::FragmentBufferedReader(info_log, std::move(file_reader), + reporter, true /*checksum*/, + log_number); + } + log::FragmentBufferedReader* reader_; + log::Reader::Reporter* reporter_; + Status* status_; + ~LogReaderContainer() { + delete reader_; + delete reporter_; + delete status_; + } + + private: + struct LogReporter : public log::Reader::Reporter { + Env* env; + Logger* info_log; + std::string fname; + Status* status; // nullptr if immutable_db_options_.paranoid_checks==false + void Corruption(size_t bytes, const Status& s) override { + ROCKS_LOG_WARN(info_log, "%s%s: dropping %d bytes; %s", + (this->status == nullptr ? "(ignoring error) " : ""), + fname.c_str(), static_cast(bytes), + s.ToString().c_str()); + if (this->status != nullptr && this->status->ok()) { + *this->status = s; + } + } + }; +}; + +// The secondary instance shares access to the storage as the primary. +// The secondary is able to read and replay changes described in both the +// MANIFEST and the WAL files without coordination with the primary. +// The secondary instance can be opened using `DB::OpenAsSecondary`. After +// that, it can call `DBImplSecondary::TryCatchUpWithPrimary` to make best +// effort attempts to catch up with the primary. +// TODO: Share common structure with CompactedDBImpl and DBImplReadOnly +class DBImplSecondary : public DBImpl { + public: + DBImplSecondary(const DBOptions& options, const std::string& dbname, + std::string secondary_path); + ~DBImplSecondary() override; + + // Recover by replaying MANIFEST and WAL. Also initialize manifest_reader_ + // and log_readers_ to facilitate future operations. + Status Recover(const std::vector& column_families, + bool read_only, bool error_if_wal_file_exists, + bool error_if_data_exists_in_wals, uint64_t* = nullptr, + RecoveryContext* recovery_ctx = nullptr) override; + + // Implementations of the DB interface. + using DB::Get; + // Can return IOError due to files being deleted by the primary. To avoid + // IOError in this case, application can coordinate between primary and + // secondaries so that primary will not delete files that are currently being + // used by the secondaries. The application can also provide a custom FS/Env + // implementation so that files will remain present until all primary and + // secondaries indicate that they can be deleted. As a partial hacky + // workaround, the secondaries can be opened with `max_open_files=-1` so that + // it eagerly keeps all talbe files open and is able to access the contents of + // deleted files via prior open fd. + Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family, + const Slice& key, PinnableSlice* value) override; + + Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family, + const Slice& key, PinnableSlice* value, + std::string* timestamp) override; + + Status GetImpl(const ReadOptions& options, ColumnFamilyHandle* column_family, + const Slice& key, PinnableSlice* value, + std::string* timestamp); + + using DBImpl::NewIterator; + // Operations on the created iterators can return IOError due to files being + // deleted by the primary. To avoid IOError in this case, application can + // coordinate between primary and secondaries so that primary will not delete + // files that are currently being used by the secondaries. The application can + // also provide a custom FS/Env implementation so that files will remain + // present until all primary and secondaries indicate that they can be + // deleted. As a partial hacky workaround, the secondaries can be opened with + // `max_open_files=-1` so that it eagerly keeps all talbe files open and is + // able to access the contents of deleted files via prior open fd. + Iterator* NewIterator(const ReadOptions&, + ColumnFamilyHandle* column_family) override; + + ArenaWrappedDBIter* NewIteratorImpl(const ReadOptions& read_options, + ColumnFamilyData* cfd, + SequenceNumber snapshot, + ReadCallback* read_callback, + bool expose_blob_index = false, + bool allow_refresh = true); + + Status NewIterators(const ReadOptions& options, + const std::vector& column_families, + std::vector* iterators) override; + + using DBImpl::Put; + Status Put(const WriteOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/, const Slice& /*key*/, + const Slice& /*value*/) override { + return Status::NotSupported("Not supported operation in secondary mode."); + } + + using DBImpl::PutEntity; + Status PutEntity(const WriteOptions& /* options */, + ColumnFamilyHandle* /* column_family */, + const Slice& /* key */, + const WideColumns& /* columns */) override { + return Status::NotSupported("Not supported operation in secondary mode."); + } + + using DBImpl::Merge; + Status Merge(const WriteOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/, const Slice& /*key*/, + const Slice& /*value*/) override { + return Status::NotSupported("Not supported operation in secondary mode."); + } + + using DBImpl::Delete; + Status Delete(const WriteOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/, + const Slice& /*key*/) override { + return Status::NotSupported("Not supported operation in secondary mode."); + } + + using DBImpl::SingleDelete; + Status SingleDelete(const WriteOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/, + const Slice& /*key*/) override { + return Status::NotSupported("Not supported operation in secondary mode."); + } + + Status Write(const WriteOptions& /*options*/, + WriteBatch* /*updates*/) override { + return Status::NotSupported("Not supported operation in secondary mode."); + } + + using DBImpl::CompactRange; + Status CompactRange(const CompactRangeOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/, + const Slice* /*begin*/, const Slice* /*end*/) override { + return Status::NotSupported("Not supported operation in secondary mode."); + } + + using DBImpl::CompactFiles; + Status CompactFiles( + const CompactionOptions& /*compact_options*/, + ColumnFamilyHandle* /*column_family*/, + const std::vector& /*input_file_names*/, + const int /*output_level*/, const int /*output_path_id*/ = -1, + std::vector* const /*output_file_names*/ = nullptr, + CompactionJobInfo* /*compaction_job_info*/ = nullptr) override { + return Status::NotSupported("Not supported operation in secondary mode."); + } + + Status DisableFileDeletions() override { + return Status::NotSupported("Not supported operation in secondary mode."); + } + + Status EnableFileDeletions(bool /*force*/) override { + return Status::NotSupported("Not supported operation in secondary mode."); + } + + Status GetLiveFiles(std::vector&, + uint64_t* /*manifest_file_size*/, + bool /*flush_memtable*/ = true) override { + return Status::NotSupported("Not supported operation in secondary mode."); + } + + using DBImpl::Flush; + Status Flush(const FlushOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/) override { + return Status::NotSupported("Not supported operation in secondary mode."); + } + + using DBImpl::SetDBOptions; + Status SetDBOptions(const std::unordered_map& + /*options_map*/) override { + // Currently not supported because changing certain options may cause + // flush/compaction. + return Status::NotSupported("Not supported operation in secondary mode."); + } + + using DBImpl::SetOptions; + Status SetOptions( + ColumnFamilyHandle* /*cfd*/, + const std::unordered_map& /*options_map*/) + override { + // Currently not supported because changing certain options may cause + // flush/compaction and/or write to MANIFEST. + return Status::NotSupported("Not supported operation in secondary mode."); + } + + using DBImpl::SyncWAL; + Status SyncWAL() override { + return Status::NotSupported("Not supported operation in secondary mode."); + } + + using DB::IngestExternalFile; + Status IngestExternalFile( + ColumnFamilyHandle* /*column_family*/, + const std::vector& /*external_files*/, + const IngestExternalFileOptions& /*ingestion_options*/) override { + return Status::NotSupported("Not supported operation in secondary mode."); + } + + // Try to catch up with the primary by reading as much as possible from the + // log files until there is nothing more to read or encounters an error. If + // the amount of information in the log files to process is huge, this + // method can take long time due to all the I/O and CPU costs. + Status TryCatchUpWithPrimary() override; + + // Try to find log reader using log_number from log_readers_ map, initialize + // if it doesn't exist + Status MaybeInitLogReader(uint64_t log_number, + log::FragmentBufferedReader** log_reader); + + // Check if all live files exist on file system and that their file sizes + // matche to the in-memory records. It is possible that some live files may + // have been deleted by the primary. In this case, CheckConsistency() does + // not flag the missing file as inconsistency. + Status CheckConsistency() override; + +#ifndef NDEBUG + Status TEST_CompactWithoutInstallation(const OpenAndCompactOptions& options, + ColumnFamilyHandle* cfh, + const CompactionServiceInput& input, + CompactionServiceResult* result) { + return CompactWithoutInstallation(options, cfh, input, result); + } +#endif // NDEBUG + + protected: + Status FlushForGetLiveFiles() override { + // No-op for read-only DB + return Status::OK(); + } + + bool OwnTablesAndLogs() const override { + // Currently, the secondary instance does not own the database files. It + // simply opens the files of the primary instance and tracks their file + // descriptors until they become obsolete. In the future, the secondary may + // create links to database files. OwnTablesAndLogs will return true then. + return false; + } + + private: + friend class DB; + + // No copying allowed + DBImplSecondary(const DBImplSecondary&); + void operator=(const DBImplSecondary&); + + using DBImpl::Recover; + + Status FindAndRecoverLogFiles( + std::unordered_set* cfds_changed, + JobContext* job_context); + Status FindNewLogNumbers(std::vector* logs); + // After manifest recovery, replay WALs and refresh log_readers_ if necessary + // REQUIRES: log_numbers are sorted in ascending order + Status RecoverLogFiles(const std::vector& log_numbers, + SequenceNumber* next_sequence, + std::unordered_set* cfds_changed, + JobContext* job_context); + + // Run compaction without installation, the output files will be placed in the + // secondary DB path. The LSM tree won't be changed, the secondary DB is still + // in read-only mode. + Status CompactWithoutInstallation(const OpenAndCompactOptions& options, + ColumnFamilyHandle* cfh, + const CompactionServiceInput& input, + CompactionServiceResult* result); + + std::unique_ptr manifest_reader_; + std::unique_ptr manifest_reporter_; + std::unique_ptr manifest_reader_status_; + + // Cache log readers for each log number, used for continue WAL replay + // after recovery + std::map> log_readers_; + + // Current WAL number replayed for each column family. + std::unordered_map cfd_to_current_log_; + + const std::string secondary_path_; +}; + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/db/db_impl/db_impl_write.cc b/librocksdb-sys/rocksdb/db/db_impl/db_impl_write.cc new file mode 100644 index 0000000..8a1a6ce --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_impl/db_impl_write.cc @@ -0,0 +1,2493 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include + +#include "db/db_impl/db_impl.h" +#include "db/error_handler.h" +#include "db/event_helpers.h" +#include "logging/logging.h" +#include "monitoring/perf_context_imp.h" +#include "options/options_helper.h" +#include "test_util/sync_point.h" +#include "util/cast_util.h" + +namespace ROCKSDB_NAMESPACE { +// Convenience methods +Status DBImpl::Put(const WriteOptions& o, ColumnFamilyHandle* column_family, + const Slice& key, const Slice& val) { + const Status s = FailIfCfHasTs(column_family); + if (!s.ok()) { + return s; + } + return DB::Put(o, column_family, key, val); +} + +Status DBImpl::Put(const WriteOptions& o, ColumnFamilyHandle* column_family, + const Slice& key, const Slice& ts, const Slice& val) { + const Status s = FailIfTsMismatchCf(column_family, ts, /*ts_for_read=*/false); + if (!s.ok()) { + return s; + } + return DB::Put(o, column_family, key, ts, val); +} + +Status DBImpl::PutEntity(const WriteOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + const WideColumns& columns) { + const Status s = FailIfCfHasTs(column_family); + if (!s.ok()) { + return s; + } + + return DB::PutEntity(options, column_family, key, columns); +} + +Status DBImpl::Merge(const WriteOptions& o, ColumnFamilyHandle* column_family, + const Slice& key, const Slice& val) { + const Status s = FailIfCfHasTs(column_family); + if (!s.ok()) { + return s; + } + auto cfh = static_cast_with_check(column_family); + if (!cfh->cfd()->ioptions()->merge_operator) { + return Status::NotSupported("Provide a merge_operator when opening DB"); + } else { + return DB::Merge(o, column_family, key, val); + } +} + +Status DBImpl::Merge(const WriteOptions& o, ColumnFamilyHandle* column_family, + const Slice& key, const Slice& ts, const Slice& val) { + const Status s = FailIfTsMismatchCf(column_family, ts, /*ts_for_read=*/false); + if (!s.ok()) { + return s; + } + return DB::Merge(o, column_family, key, ts, val); +} + +Status DBImpl::Delete(const WriteOptions& write_options, + ColumnFamilyHandle* column_family, const Slice& key) { + const Status s = FailIfCfHasTs(column_family); + if (!s.ok()) { + return s; + } + return DB::Delete(write_options, column_family, key); +} + +Status DBImpl::Delete(const WriteOptions& write_options, + ColumnFamilyHandle* column_family, const Slice& key, + const Slice& ts) { + const Status s = FailIfTsMismatchCf(column_family, ts, /*ts_for_read=*/false); + if (!s.ok()) { + return s; + } + return DB::Delete(write_options, column_family, key, ts); +} + +Status DBImpl::SingleDelete(const WriteOptions& write_options, + ColumnFamilyHandle* column_family, + const Slice& key) { + const Status s = FailIfCfHasTs(column_family); + if (!s.ok()) { + return s; + } + return DB::SingleDelete(write_options, column_family, key); +} + +Status DBImpl::SingleDelete(const WriteOptions& write_options, + ColumnFamilyHandle* column_family, const Slice& key, + const Slice& ts) { + const Status s = FailIfTsMismatchCf(column_family, ts, /*ts_for_read=*/false); + if (!s.ok()) { + return s; + } + return DB::SingleDelete(write_options, column_family, key, ts); +} + +Status DBImpl::DeleteRange(const WriteOptions& write_options, + ColumnFamilyHandle* column_family, + const Slice& begin_key, const Slice& end_key) { + const Status s = FailIfCfHasTs(column_family); + if (!s.ok()) { + return s; + } + return DB::DeleteRange(write_options, column_family, begin_key, end_key); +} + +Status DBImpl::DeleteRange(const WriteOptions& write_options, + ColumnFamilyHandle* column_family, + const Slice& begin_key, const Slice& end_key, + const Slice& ts) { + const Status s = FailIfTsMismatchCf(column_family, ts, /*ts_for_read=*/false); + if (!s.ok()) { + return s; + } + return DB::DeleteRange(write_options, column_family, begin_key, end_key, ts); +} + +void DBImpl::SetRecoverableStatePreReleaseCallback( + PreReleaseCallback* callback) { + recoverable_state_pre_release_callback_.reset(callback); +} + +Status DBImpl::Write(const WriteOptions& write_options, WriteBatch* my_batch) { + Status s; + if (write_options.protection_bytes_per_key > 0) { + s = WriteBatchInternal::UpdateProtectionInfo( + my_batch, write_options.protection_bytes_per_key); + } + if (s.ok()) { + s = WriteImpl(write_options, my_batch, /*callback=*/nullptr, + /*log_used=*/nullptr); + } + return s; +} + +Status DBImpl::WriteWithCallback(const WriteOptions& write_options, + WriteBatch* my_batch, + WriteCallback* callback) { + Status s; + if (write_options.protection_bytes_per_key > 0) { + s = WriteBatchInternal::UpdateProtectionInfo( + my_batch, write_options.protection_bytes_per_key); + } + if (s.ok()) { + s = WriteImpl(write_options, my_batch, callback, nullptr); + } + return s; +} + +// The main write queue. This is the only write queue that updates LastSequence. +// When using one write queue, the same sequence also indicates the last +// published sequence. +Status DBImpl::WriteImpl(const WriteOptions& write_options, + WriteBatch* my_batch, WriteCallback* callback, + uint64_t* log_used, uint64_t log_ref, + bool disable_memtable, uint64_t* seq_used, + size_t batch_cnt, + PreReleaseCallback* pre_release_callback, + PostMemTableCallback* post_memtable_callback) { + assert(!seq_per_batch_ || batch_cnt != 0); + assert(my_batch == nullptr || my_batch->Count() == 0 || + write_options.protection_bytes_per_key == 0 || + write_options.protection_bytes_per_key == + my_batch->GetProtectionBytesPerKey()); + if (my_batch == nullptr) { + return Status::InvalidArgument("Batch is nullptr!"); + } else if (!disable_memtable && + WriteBatchInternal::TimestampsUpdateNeeded(*my_batch)) { + // If writing to memtable, then we require the caller to set/update the + // timestamps for the keys in the write batch. + // Otherwise, it means we are just writing to the WAL, and we allow + // timestamps unset for the keys in the write batch. This can happen if we + // use TransactionDB with write-committed policy, and we currently do not + // support user-defined timestamp with other policies. + // In the prepare phase, a transaction can write the batch to the WAL + // without inserting to memtable. The keys in the batch do not have to be + // assigned timestamps because they will be used only during recovery if + // there is a commit marker which includes their commit timestamp. + return Status::InvalidArgument("write batch must have timestamp(s) set"); + } else if (write_options.rate_limiter_priority != Env::IO_TOTAL && + write_options.rate_limiter_priority != Env::IO_USER) { + return Status::InvalidArgument( + "WriteOptions::rate_limiter_priority only allows " + "Env::IO_TOTAL and Env::IO_USER due to implementation constraints"); + } else if (write_options.rate_limiter_priority != Env::IO_TOTAL && + (write_options.disableWAL || manual_wal_flush_)) { + return Status::InvalidArgument( + "WriteOptions::rate_limiter_priority currently only supports " + "rate-limiting automatic WAL flush, which requires " + "`WriteOptions::disableWAL` and " + "`DBOptions::manual_wal_flush` both set to false"); + } else if (write_options.protection_bytes_per_key != 0 && + write_options.protection_bytes_per_key != 8) { + return Status::InvalidArgument( + "`WriteOptions::protection_bytes_per_key` must be zero or eight"); + } + // TODO: this use of operator bool on `tracer_` can avoid unnecessary lock + // grabs but does not seem thread-safe. + if (tracer_) { + InstrumentedMutexLock lock(&trace_mutex_); + if (tracer_ && !tracer_->IsWriteOrderPreserved()) { + // We don't have to preserve write order so can trace anywhere. It's more + // efficient to trace here than to add latency to a phase of the log/apply + // pipeline. + // TODO: maybe handle the tracing status? + tracer_->Write(my_batch).PermitUncheckedError(); + } + } + if (write_options.sync && write_options.disableWAL) { + return Status::InvalidArgument("Sync writes has to enable WAL."); + } + if (two_write_queues_ && immutable_db_options_.enable_pipelined_write) { + return Status::NotSupported( + "pipelined_writes is not compatible with concurrent prepares"); + } + if (seq_per_batch_ && immutable_db_options_.enable_pipelined_write) { + // TODO(yiwu): update pipeline write with seq_per_batch and batch_cnt + return Status::NotSupported( + "pipelined_writes is not compatible with seq_per_batch"); + } + if (immutable_db_options_.unordered_write && + immutable_db_options_.enable_pipelined_write) { + return Status::NotSupported( + "pipelined_writes is not compatible with unordered_write"); + } + if (immutable_db_options_.enable_pipelined_write && + post_memtable_callback != nullptr) { + return Status::NotSupported( + "pipelined write currently does not honor post_memtable_callback"); + } + if (seq_per_batch_ && post_memtable_callback != nullptr) { + return Status::NotSupported( + "seq_per_batch currently does not honor post_memtable_callback"); + } + // Otherwise IsLatestPersistentState optimization does not make sense + assert(!WriteBatchInternal::IsLatestPersistentState(my_batch) || + disable_memtable); + + if (write_options.low_pri) { + Status s = ThrottleLowPriWritesIfNeeded(write_options, my_batch); + if (!s.ok()) { + return s; + } + } + + if (two_write_queues_ && disable_memtable) { + AssignOrder assign_order = + seq_per_batch_ ? kDoAssignOrder : kDontAssignOrder; + // Otherwise it is WAL-only Prepare batches in WriteCommitted policy and + // they don't consume sequence. + return WriteImplWALOnly(&nonmem_write_thread_, write_options, my_batch, + callback, log_used, log_ref, seq_used, batch_cnt, + pre_release_callback, assign_order, + kDontPublishLastSeq, disable_memtable); + } + + if (immutable_db_options_.unordered_write) { + const size_t sub_batch_cnt = batch_cnt != 0 + ? batch_cnt + // every key is a sub-batch consuming a seq + : WriteBatchInternal::Count(my_batch); + uint64_t seq = 0; + // Use a write thread to i) optimize for WAL write, ii) publish last + // sequence in in increasing order, iii) call pre_release_callback serially + Status status = WriteImplWALOnly( + &write_thread_, write_options, my_batch, callback, log_used, log_ref, + &seq, sub_batch_cnt, pre_release_callback, kDoAssignOrder, + kDoPublishLastSeq, disable_memtable); + TEST_SYNC_POINT("DBImpl::WriteImpl:UnorderedWriteAfterWriteWAL"); + if (!status.ok()) { + return status; + } + if (seq_used) { + *seq_used = seq; + } + if (!disable_memtable) { + TEST_SYNC_POINT("DBImpl::WriteImpl:BeforeUnorderedWriteMemtable"); + status = UnorderedWriteMemtable(write_options, my_batch, callback, + log_ref, seq, sub_batch_cnt); + } + return status; + } + + if (immutable_db_options_.enable_pipelined_write) { + return PipelinedWriteImpl(write_options, my_batch, callback, log_used, + log_ref, disable_memtable, seq_used); + } + + PERF_TIMER_GUARD(write_pre_and_post_process_time); + WriteThread::Writer w(write_options, my_batch, callback, log_ref, + disable_memtable, batch_cnt, pre_release_callback, + post_memtable_callback); + StopWatch write_sw(immutable_db_options_.clock, stats_, DB_WRITE); + + write_thread_.JoinBatchGroup(&w); + if (w.state == WriteThread::STATE_PARALLEL_MEMTABLE_WRITER) { + // we are a non-leader in a parallel group + + if (w.ShouldWriteToMemtable()) { + PERF_TIMER_STOP(write_pre_and_post_process_time); + PERF_TIMER_GUARD(write_memtable_time); + + ColumnFamilyMemTablesImpl column_family_memtables( + versions_->GetColumnFamilySet()); + w.status = WriteBatchInternal::InsertInto( + &w, w.sequence, &column_family_memtables, &flush_scheduler_, + &trim_history_scheduler_, + write_options.ignore_missing_column_families, 0 /*log_number*/, this, + true /*concurrent_memtable_writes*/, seq_per_batch_, w.batch_cnt, + batch_per_txn_, write_options.memtable_insert_hint_per_batch); + + PERF_TIMER_START(write_pre_and_post_process_time); + } + + if (write_thread_.CompleteParallelMemTableWriter(&w)) { + // we're responsible for exit batch group + // TODO(myabandeh): propagate status to write_group + auto last_sequence = w.write_group->last_sequence; + for (auto* tmp_w : *(w.write_group)) { + assert(tmp_w); + if (tmp_w->post_memtable_callback) { + Status tmp_s = + (*tmp_w->post_memtable_callback)(last_sequence, disable_memtable); + // TODO: propagate the execution status of post_memtable_callback to + // caller. + assert(tmp_s.ok()); + } + } + versions_->SetLastSequence(last_sequence); + MemTableInsertStatusCheck(w.status); + write_thread_.ExitAsBatchGroupFollower(&w); + } + assert(w.state == WriteThread::STATE_COMPLETED); + // STATE_COMPLETED conditional below handles exit + } + if (w.state == WriteThread::STATE_COMPLETED) { + if (log_used != nullptr) { + *log_used = w.log_used; + } + if (seq_used != nullptr) { + *seq_used = w.sequence; + } + // write is complete and leader has updated sequence + return w.FinalStatus(); + } + // else we are the leader of the write batch group + assert(w.state == WriteThread::STATE_GROUP_LEADER); + Status status; + // Once reaches this point, the current writer "w" will try to do its write + // job. It may also pick up some of the remaining writers in the "writers_" + // when it finds suitable, and finish them in the same write batch. + // This is how a write job could be done by the other writer. + WriteContext write_context; + LogContext log_context(write_options.sync); + WriteThread::WriteGroup write_group; + bool in_parallel_group = false; + uint64_t last_sequence = kMaxSequenceNumber; + + assert(!two_write_queues_ || !disable_memtable); + { + // With concurrent writes we do preprocess only in the write thread that + // also does write to memtable to avoid sync issue on shared data structure + // with the other thread + + // PreprocessWrite does its own perf timing. + PERF_TIMER_STOP(write_pre_and_post_process_time); + + status = PreprocessWrite(write_options, &log_context, &write_context); + if (!two_write_queues_) { + // Assign it after ::PreprocessWrite since the sequence might advance + // inside it by WriteRecoverableState + last_sequence = versions_->LastSequence(); + } + + PERF_TIMER_START(write_pre_and_post_process_time); + } + + // Add to log and apply to memtable. We can release the lock + // during this phase since &w is currently responsible for logging + // and protects against concurrent loggers and concurrent writes + // into memtables + + TEST_SYNC_POINT("DBImpl::WriteImpl:BeforeLeaderEnters"); + last_batch_group_size_ = + write_thread_.EnterAsBatchGroupLeader(&w, &write_group); + + IOStatus io_s; + Status pre_release_cb_status; + if (status.ok()) { + // TODO: this use of operator bool on `tracer_` can avoid unnecessary lock + // grabs but does not seem thread-safe. + if (tracer_) { + InstrumentedMutexLock lock(&trace_mutex_); + if (tracer_ && tracer_->IsWriteOrderPreserved()) { + for (auto* writer : write_group) { + // TODO: maybe handle the tracing status? + tracer_->Write(writer->batch).PermitUncheckedError(); + } + } + } + // Rules for when we can update the memtable concurrently + // 1. supported by memtable + // 2. Puts are not okay if inplace_update_support + // 3. Merges are not okay + // + // Rules 1..2 are enforced by checking the options + // during startup (CheckConcurrentWritesSupported), so if + // options.allow_concurrent_memtable_write is true then they can be + // assumed to be true. Rule 3 is checked for each batch. We could + // relax rules 2 if we could prevent write batches from referring + // more than once to a particular key. + bool parallel = immutable_db_options_.allow_concurrent_memtable_write && + write_group.size > 1; + size_t total_count = 0; + size_t valid_batches = 0; + size_t total_byte_size = 0; + size_t pre_release_callback_cnt = 0; + for (auto* writer : write_group) { + assert(writer); + if (writer->CheckCallback(this)) { + valid_batches += writer->batch_cnt; + if (writer->ShouldWriteToMemtable()) { + total_count += WriteBatchInternal::Count(writer->batch); + parallel = parallel && !writer->batch->HasMerge(); + } + total_byte_size = WriteBatchInternal::AppendedByteSize( + total_byte_size, WriteBatchInternal::ByteSize(writer->batch)); + if (writer->pre_release_callback) { + pre_release_callback_cnt++; + } + } + } + // Note about seq_per_batch_: either disableWAL is set for the entire write + // group or not. In either case we inc seq for each write batch with no + // failed callback. This means that there could be a batch with + // disalbe_memtable in between; although we do not write this batch to + // memtable it still consumes a seq. Otherwise, if !seq_per_batch_, we inc + // the seq per valid written key to mem. + size_t seq_inc = seq_per_batch_ ? valid_batches : total_count; + + const bool concurrent_update = two_write_queues_; + // Update stats while we are an exclusive group leader, so we know + // that nobody else can be writing to these particular stats. + // We're optimistic, updating the stats before we successfully + // commit. That lets us release our leader status early. + auto stats = default_cf_internal_stats_; + stats->AddDBStats(InternalStats::kIntStatsNumKeysWritten, total_count, + concurrent_update); + RecordTick(stats_, NUMBER_KEYS_WRITTEN, total_count); + stats->AddDBStats(InternalStats::kIntStatsBytesWritten, total_byte_size, + concurrent_update); + RecordTick(stats_, BYTES_WRITTEN, total_byte_size); + stats->AddDBStats(InternalStats::kIntStatsWriteDoneBySelf, 1, + concurrent_update); + RecordTick(stats_, WRITE_DONE_BY_SELF); + auto write_done_by_other = write_group.size - 1; + if (write_done_by_other > 0) { + stats->AddDBStats(InternalStats::kIntStatsWriteDoneByOther, + write_done_by_other, concurrent_update); + RecordTick(stats_, WRITE_DONE_BY_OTHER, write_done_by_other); + } + RecordInHistogram(stats_, BYTES_PER_WRITE, total_byte_size); + + if (write_options.disableWAL) { + has_unpersisted_data_.store(true, std::memory_order_relaxed); + } + + PERF_TIMER_STOP(write_pre_and_post_process_time); + + if (!two_write_queues_) { + if (status.ok() && !write_options.disableWAL) { + assert(log_context.log_file_number_size); + LogFileNumberSize& log_file_number_size = + *(log_context.log_file_number_size); + PERF_TIMER_GUARD(write_wal_time); + io_s = + WriteToWAL(write_group, log_context.writer, log_used, + log_context.need_log_sync, log_context.need_log_dir_sync, + last_sequence + 1, log_file_number_size); + } + } else { + if (status.ok() && !write_options.disableWAL) { + PERF_TIMER_GUARD(write_wal_time); + // LastAllocatedSequence is increased inside WriteToWAL under + // wal_write_mutex_ to ensure ordered events in WAL + io_s = ConcurrentWriteToWAL(write_group, log_used, &last_sequence, + seq_inc); + } else { + // Otherwise we inc seq number for memtable writes + last_sequence = versions_->FetchAddLastAllocatedSequence(seq_inc); + } + } + status = io_s; + assert(last_sequence != kMaxSequenceNumber); + const SequenceNumber current_sequence = last_sequence + 1; + last_sequence += seq_inc; + + // PreReleaseCallback is called after WAL write and before memtable write + if (status.ok()) { + SequenceNumber next_sequence = current_sequence; + size_t index = 0; + // Note: the logic for advancing seq here must be consistent with the + // logic in WriteBatchInternal::InsertInto(write_group...) as well as + // with WriteBatchInternal::InsertInto(write_batch...) that is called on + // the merged batch during recovery from the WAL. + for (auto* writer : write_group) { + if (writer->CallbackFailed()) { + continue; + } + writer->sequence = next_sequence; + if (writer->pre_release_callback) { + Status ws = writer->pre_release_callback->Callback( + writer->sequence, disable_memtable, writer->log_used, index++, + pre_release_callback_cnt); + if (!ws.ok()) { + status = pre_release_cb_status = ws; + break; + } + } + if (seq_per_batch_) { + assert(writer->batch_cnt); + next_sequence += writer->batch_cnt; + } else if (writer->ShouldWriteToMemtable()) { + next_sequence += WriteBatchInternal::Count(writer->batch); + } + } + } + + if (status.ok()) { + PERF_TIMER_GUARD(write_memtable_time); + + if (!parallel) { + // w.sequence will be set inside InsertInto + w.status = WriteBatchInternal::InsertInto( + write_group, current_sequence, column_family_memtables_.get(), + &flush_scheduler_, &trim_history_scheduler_, + write_options.ignore_missing_column_families, + 0 /*recovery_log_number*/, this, parallel, seq_per_batch_, + batch_per_txn_); + } else { + write_group.last_sequence = last_sequence; + write_thread_.LaunchParallelMemTableWriters(&write_group); + in_parallel_group = true; + + // Each parallel follower is doing each own writes. The leader should + // also do its own. + if (w.ShouldWriteToMemtable()) { + ColumnFamilyMemTablesImpl column_family_memtables( + versions_->GetColumnFamilySet()); + assert(w.sequence == current_sequence); + w.status = WriteBatchInternal::InsertInto( + &w, w.sequence, &column_family_memtables, &flush_scheduler_, + &trim_history_scheduler_, + write_options.ignore_missing_column_families, 0 /*log_number*/, + this, true /*concurrent_memtable_writes*/, seq_per_batch_, + w.batch_cnt, batch_per_txn_, + write_options.memtable_insert_hint_per_batch); + } + } + if (seq_used != nullptr) { + *seq_used = w.sequence; + } + } + } + PERF_TIMER_START(write_pre_and_post_process_time); + + if (!io_s.ok()) { + // Check WriteToWAL status + IOStatusCheck(io_s); + } + if (!w.CallbackFailed()) { + if (!io_s.ok()) { + assert(pre_release_cb_status.ok()); + } else { + WriteStatusCheck(pre_release_cb_status); + } + } else { + assert(pre_release_cb_status.ok()); + } + + if (log_context.need_log_sync) { + VersionEdit synced_wals; + log_write_mutex_.Lock(); + if (status.ok()) { + MarkLogsSynced(logfile_number_, log_context.need_log_dir_sync, + &synced_wals); + } else { + MarkLogsNotSynced(logfile_number_); + } + log_write_mutex_.Unlock(); + if (status.ok() && synced_wals.IsWalAddition()) { + InstrumentedMutexLock l(&mutex_); + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + status = ApplyWALToManifest(read_options, &synced_wals); + } + + // Requesting sync with two_write_queues_ is expected to be very rare. We + // hence provide a simple implementation that is not necessarily efficient. + if (two_write_queues_) { + if (manual_wal_flush_) { + status = FlushWAL(true); + } else { + status = SyncWAL(); + } + } + } + + bool should_exit_batch_group = true; + if (in_parallel_group) { + // CompleteParallelWorker returns true if this thread should + // handle exit, false means somebody else did + should_exit_batch_group = write_thread_.CompleteParallelMemTableWriter(&w); + } + if (should_exit_batch_group) { + if (status.ok()) { + for (auto* tmp_w : write_group) { + assert(tmp_w); + if (tmp_w->post_memtable_callback) { + Status tmp_s = + (*tmp_w->post_memtable_callback)(last_sequence, disable_memtable); + // TODO: propagate the execution status of post_memtable_callback to + // caller. + assert(tmp_s.ok()); + } + } + // Note: if we are to resume after non-OK statuses we need to revisit how + // we reacts to non-OK statuses here. + versions_->SetLastSequence(last_sequence); + } + MemTableInsertStatusCheck(w.status); + write_thread_.ExitAsBatchGroupLeader(write_group, status); + } + + if (status.ok()) { + status = w.FinalStatus(); + } + return status; +} + +Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options, + WriteBatch* my_batch, WriteCallback* callback, + uint64_t* log_used, uint64_t log_ref, + bool disable_memtable, uint64_t* seq_used) { + PERF_TIMER_GUARD(write_pre_and_post_process_time); + StopWatch write_sw(immutable_db_options_.clock, stats_, DB_WRITE); + + WriteContext write_context; + + WriteThread::Writer w(write_options, my_batch, callback, log_ref, + disable_memtable, /*_batch_cnt=*/0, + /*_pre_release_callback=*/nullptr); + write_thread_.JoinBatchGroup(&w); + TEST_SYNC_POINT("DBImplWrite::PipelinedWriteImpl:AfterJoinBatchGroup"); + if (w.state == WriteThread::STATE_GROUP_LEADER) { + WriteThread::WriteGroup wal_write_group; + if (w.callback && !w.callback->AllowWriteBatching()) { + write_thread_.WaitForMemTableWriters(); + } + LogContext log_context(!write_options.disableWAL && write_options.sync); + // PreprocessWrite does its own perf timing. + PERF_TIMER_STOP(write_pre_and_post_process_time); + w.status = PreprocessWrite(write_options, &log_context, &write_context); + PERF_TIMER_START(write_pre_and_post_process_time); + + // This can set non-OK status if callback fail. + last_batch_group_size_ = + write_thread_.EnterAsBatchGroupLeader(&w, &wal_write_group); + const SequenceNumber current_sequence = + write_thread_.UpdateLastSequence(versions_->LastSequence()) + 1; + size_t total_count = 0; + size_t total_byte_size = 0; + + if (w.status.ok()) { + // TODO: this use of operator bool on `tracer_` can avoid unnecessary lock + // grabs but does not seem thread-safe. + if (tracer_) { + InstrumentedMutexLock lock(&trace_mutex_); + if (tracer_ != nullptr && tracer_->IsWriteOrderPreserved()) { + for (auto* writer : wal_write_group) { + // TODO: maybe handle the tracing status? + tracer_->Write(writer->batch).PermitUncheckedError(); + } + } + } + SequenceNumber next_sequence = current_sequence; + for (auto* writer : wal_write_group) { + assert(writer); + if (writer->CheckCallback(this)) { + if (writer->ShouldWriteToMemtable()) { + writer->sequence = next_sequence; + size_t count = WriteBatchInternal::Count(writer->batch); + next_sequence += count; + total_count += count; + } + total_byte_size = WriteBatchInternal::AppendedByteSize( + total_byte_size, WriteBatchInternal::ByteSize(writer->batch)); + } + } + if (w.disable_wal) { + has_unpersisted_data_.store(true, std::memory_order_relaxed); + } + write_thread_.UpdateLastSequence(current_sequence + total_count - 1); + } + + auto stats = default_cf_internal_stats_; + stats->AddDBStats(InternalStats::kIntStatsNumKeysWritten, total_count); + RecordTick(stats_, NUMBER_KEYS_WRITTEN, total_count); + stats->AddDBStats(InternalStats::kIntStatsBytesWritten, total_byte_size); + RecordTick(stats_, BYTES_WRITTEN, total_byte_size); + RecordInHistogram(stats_, BYTES_PER_WRITE, total_byte_size); + + PERF_TIMER_STOP(write_pre_and_post_process_time); + + IOStatus io_s; + io_s.PermitUncheckedError(); // Allow io_s to be uninitialized + + if (w.status.ok() && !write_options.disableWAL) { + PERF_TIMER_GUARD(write_wal_time); + stats->AddDBStats(InternalStats::kIntStatsWriteDoneBySelf, 1); + RecordTick(stats_, WRITE_DONE_BY_SELF, 1); + if (wal_write_group.size > 1) { + stats->AddDBStats(InternalStats::kIntStatsWriteDoneByOther, + wal_write_group.size - 1); + RecordTick(stats_, WRITE_DONE_BY_OTHER, wal_write_group.size - 1); + } + assert(log_context.log_file_number_size); + LogFileNumberSize& log_file_number_size = + *(log_context.log_file_number_size); + io_s = + WriteToWAL(wal_write_group, log_context.writer, log_used, + log_context.need_log_sync, log_context.need_log_dir_sync, + current_sequence, log_file_number_size); + w.status = io_s; + } + + if (!io_s.ok()) { + // Check WriteToWAL status + IOStatusCheck(io_s); + } else if (!w.CallbackFailed()) { + WriteStatusCheck(w.status); + } + + VersionEdit synced_wals; + if (log_context.need_log_sync) { + InstrumentedMutexLock l(&log_write_mutex_); + if (w.status.ok()) { + MarkLogsSynced(logfile_number_, log_context.need_log_dir_sync, + &synced_wals); + } else { + MarkLogsNotSynced(logfile_number_); + } + } + if (w.status.ok() && synced_wals.IsWalAddition()) { + InstrumentedMutexLock l(&mutex_); + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + w.status = ApplyWALToManifest(read_options, &synced_wals); + } + write_thread_.ExitAsBatchGroupLeader(wal_write_group, w.status); + } + + // NOTE: the memtable_write_group is declared before the following + // `if` statement because its lifetime needs to be longer + // that the inner context of the `if` as a reference to it + // may be used further below within the outer _write_thread + WriteThread::WriteGroup memtable_write_group; + + if (w.state == WriteThread::STATE_MEMTABLE_WRITER_LEADER) { + PERF_TIMER_GUARD(write_memtable_time); + assert(w.ShouldWriteToMemtable()); + write_thread_.EnterAsMemTableWriter(&w, &memtable_write_group); + if (memtable_write_group.size > 1 && + immutable_db_options_.allow_concurrent_memtable_write) { + write_thread_.LaunchParallelMemTableWriters(&memtable_write_group); + } else { + memtable_write_group.status = WriteBatchInternal::InsertInto( + memtable_write_group, w.sequence, column_family_memtables_.get(), + &flush_scheduler_, &trim_history_scheduler_, + write_options.ignore_missing_column_families, 0 /*log_number*/, this, + false /*concurrent_memtable_writes*/, seq_per_batch_, batch_per_txn_); + versions_->SetLastSequence(memtable_write_group.last_sequence); + write_thread_.ExitAsMemTableWriter(&w, memtable_write_group); + } + } else { + // NOTE: the memtable_write_group is never really used, + // so we need to set its status to pass ASSERT_STATUS_CHECKED + memtable_write_group.status.PermitUncheckedError(); + } + + if (w.state == WriteThread::STATE_PARALLEL_MEMTABLE_WRITER) { + assert(w.ShouldWriteToMemtable()); + ColumnFamilyMemTablesImpl column_family_memtables( + versions_->GetColumnFamilySet()); + w.status = WriteBatchInternal::InsertInto( + &w, w.sequence, &column_family_memtables, &flush_scheduler_, + &trim_history_scheduler_, write_options.ignore_missing_column_families, + 0 /*log_number*/, this, true /*concurrent_memtable_writes*/, + false /*seq_per_batch*/, 0 /*batch_cnt*/, true /*batch_per_txn*/, + write_options.memtable_insert_hint_per_batch); + if (write_thread_.CompleteParallelMemTableWriter(&w)) { + MemTableInsertStatusCheck(w.status); + versions_->SetLastSequence(w.write_group->last_sequence); + write_thread_.ExitAsMemTableWriter(&w, *w.write_group); + } + } + if (seq_used != nullptr) { + *seq_used = w.sequence; + } + + assert(w.state == WriteThread::STATE_COMPLETED); + return w.FinalStatus(); +} + +Status DBImpl::UnorderedWriteMemtable(const WriteOptions& write_options, + WriteBatch* my_batch, + WriteCallback* callback, uint64_t log_ref, + SequenceNumber seq, + const size_t sub_batch_cnt) { + PERF_TIMER_GUARD(write_pre_and_post_process_time); + StopWatch write_sw(immutable_db_options_.clock, stats_, DB_WRITE); + + WriteThread::Writer w(write_options, my_batch, callback, log_ref, + false /*disable_memtable*/); + + if (w.CheckCallback(this) && w.ShouldWriteToMemtable()) { + w.sequence = seq; + size_t total_count = WriteBatchInternal::Count(my_batch); + InternalStats* stats = default_cf_internal_stats_; + stats->AddDBStats(InternalStats::kIntStatsNumKeysWritten, total_count); + RecordTick(stats_, NUMBER_KEYS_WRITTEN, total_count); + + ColumnFamilyMemTablesImpl column_family_memtables( + versions_->GetColumnFamilySet()); + w.status = WriteBatchInternal::InsertInto( + &w, w.sequence, &column_family_memtables, &flush_scheduler_, + &trim_history_scheduler_, write_options.ignore_missing_column_families, + 0 /*log_number*/, this, true /*concurrent_memtable_writes*/, + seq_per_batch_, sub_batch_cnt, true /*batch_per_txn*/, + write_options.memtable_insert_hint_per_batch); + if (write_options.disableWAL) { + has_unpersisted_data_.store(true, std::memory_order_relaxed); + } + } + + size_t pending_cnt = pending_memtable_writes_.fetch_sub(1) - 1; + if (pending_cnt == 0) { + // switch_cv_ waits until pending_memtable_writes_ = 0. Locking its mutex + // before notify ensures that cv is in waiting state when it is notified + // thus not missing the update to pending_memtable_writes_ even though it is + // not modified under the mutex. + std::lock_guard lck(switch_mutex_); + switch_cv_.notify_all(); + } + WriteStatusCheck(w.status); + + if (!w.FinalStatus().ok()) { + return w.FinalStatus(); + } + return Status::OK(); +} + +// The 2nd write queue. If enabled it will be used only for WAL-only writes. +// This is the only queue that updates LastPublishedSequence which is only +// applicable in a two-queue setting. +Status DBImpl::WriteImplWALOnly( + WriteThread* write_thread, const WriteOptions& write_options, + WriteBatch* my_batch, WriteCallback* callback, uint64_t* log_used, + const uint64_t log_ref, uint64_t* seq_used, const size_t sub_batch_cnt, + PreReleaseCallback* pre_release_callback, const AssignOrder assign_order, + const PublishLastSeq publish_last_seq, const bool disable_memtable) { + PERF_TIMER_GUARD(write_pre_and_post_process_time); + WriteThread::Writer w(write_options, my_batch, callback, log_ref, + disable_memtable, sub_batch_cnt, pre_release_callback); + StopWatch write_sw(immutable_db_options_.clock, stats_, DB_WRITE); + + write_thread->JoinBatchGroup(&w); + assert(w.state != WriteThread::STATE_PARALLEL_MEMTABLE_WRITER); + if (w.state == WriteThread::STATE_COMPLETED) { + if (log_used != nullptr) { + *log_used = w.log_used; + } + if (seq_used != nullptr) { + *seq_used = w.sequence; + } + return w.FinalStatus(); + } + // else we are the leader of the write batch group + assert(w.state == WriteThread::STATE_GROUP_LEADER); + + if (publish_last_seq == kDoPublishLastSeq) { + Status status; + + // Currently we only use kDoPublishLastSeq in unordered_write + assert(immutable_db_options_.unordered_write); + WriteContext write_context; + if (error_handler_.IsDBStopped()) { + status = error_handler_.GetBGError(); + } + // TODO(myabandeh): Make preliminary checks thread-safe so we could do them + // without paying the cost of obtaining the mutex. + if (status.ok()) { + LogContext log_context; + status = PreprocessWrite(write_options, &log_context, &write_context); + WriteStatusCheckOnLocked(status); + } + if (!status.ok()) { + WriteThread::WriteGroup write_group; + write_thread->EnterAsBatchGroupLeader(&w, &write_group); + write_thread->ExitAsBatchGroupLeader(write_group, status); + return status; + } + } else { + InstrumentedMutexLock lock(&mutex_); + Status status = + DelayWrite(/*num_bytes=*/0ull, *write_thread, write_options); + if (!status.ok()) { + WriteThread::WriteGroup write_group; + write_thread->EnterAsBatchGroupLeader(&w, &write_group); + write_thread->ExitAsBatchGroupLeader(write_group, status); + return status; + } + } + + WriteThread::WriteGroup write_group; + uint64_t last_sequence; + write_thread->EnterAsBatchGroupLeader(&w, &write_group); + // Note: no need to update last_batch_group_size_ here since the batch writes + // to WAL only + // TODO: this use of operator bool on `tracer_` can avoid unnecessary lock + // grabs but does not seem thread-safe. + if (tracer_) { + InstrumentedMutexLock lock(&trace_mutex_); + if (tracer_ != nullptr && tracer_->IsWriteOrderPreserved()) { + for (auto* writer : write_group) { + // TODO: maybe handle the tracing status? + tracer_->Write(writer->batch).PermitUncheckedError(); + } + } + } + + size_t pre_release_callback_cnt = 0; + size_t total_byte_size = 0; + for (auto* writer : write_group) { + assert(writer); + if (writer->CheckCallback(this)) { + total_byte_size = WriteBatchInternal::AppendedByteSize( + total_byte_size, WriteBatchInternal::ByteSize(writer->batch)); + if (writer->pre_release_callback) { + pre_release_callback_cnt++; + } + } + } + + const bool concurrent_update = true; + // Update stats while we are an exclusive group leader, so we know + // that nobody else can be writing to these particular stats. + // We're optimistic, updating the stats before we successfully + // commit. That lets us release our leader status early. + auto stats = default_cf_internal_stats_; + stats->AddDBStats(InternalStats::kIntStatsBytesWritten, total_byte_size, + concurrent_update); + RecordTick(stats_, BYTES_WRITTEN, total_byte_size); + stats->AddDBStats(InternalStats::kIntStatsWriteDoneBySelf, 1, + concurrent_update); + RecordTick(stats_, WRITE_DONE_BY_SELF); + auto write_done_by_other = write_group.size - 1; + if (write_done_by_other > 0) { + stats->AddDBStats(InternalStats::kIntStatsWriteDoneByOther, + write_done_by_other, concurrent_update); + RecordTick(stats_, WRITE_DONE_BY_OTHER, write_done_by_other); + } + RecordInHistogram(stats_, BYTES_PER_WRITE, total_byte_size); + + PERF_TIMER_STOP(write_pre_and_post_process_time); + + PERF_TIMER_GUARD(write_wal_time); + // LastAllocatedSequence is increased inside WriteToWAL under + // wal_write_mutex_ to ensure ordered events in WAL + size_t seq_inc = 0 /* total_count */; + if (assign_order == kDoAssignOrder) { + size_t total_batch_cnt = 0; + for (auto* writer : write_group) { + assert(writer->batch_cnt || !seq_per_batch_); + if (!writer->CallbackFailed()) { + total_batch_cnt += writer->batch_cnt; + } + } + seq_inc = total_batch_cnt; + } + Status status; + if (!write_options.disableWAL) { + IOStatus io_s = + ConcurrentWriteToWAL(write_group, log_used, &last_sequence, seq_inc); + status = io_s; + // last_sequence may not be set if there is an error + // This error checking and return is moved up to avoid using uninitialized + // last_sequence. + if (!io_s.ok()) { + IOStatusCheck(io_s); + write_thread->ExitAsBatchGroupLeader(write_group, status); + return status; + } + } else { + // Otherwise we inc seq number to do solely the seq allocation + last_sequence = versions_->FetchAddLastAllocatedSequence(seq_inc); + } + + size_t memtable_write_cnt = 0; + auto curr_seq = last_sequence + 1; + for (auto* writer : write_group) { + if (writer->CallbackFailed()) { + continue; + } + writer->sequence = curr_seq; + if (assign_order == kDoAssignOrder) { + assert(writer->batch_cnt || !seq_per_batch_); + curr_seq += writer->batch_cnt; + } + if (!writer->disable_memtable) { + memtable_write_cnt++; + } + // else seq advances only by memtable writes + } + if (status.ok() && write_options.sync) { + assert(!write_options.disableWAL); + // Requesting sync with two_write_queues_ is expected to be very rare. We + // hance provide a simple implementation that is not necessarily efficient. + if (manual_wal_flush_) { + status = FlushWAL(true); + } else { + status = SyncWAL(); + } + } + PERF_TIMER_START(write_pre_and_post_process_time); + + if (!w.CallbackFailed()) { + WriteStatusCheck(status); + } + if (status.ok()) { + size_t index = 0; + for (auto* writer : write_group) { + if (!writer->CallbackFailed() && writer->pre_release_callback) { + assert(writer->sequence != kMaxSequenceNumber); + Status ws = writer->pre_release_callback->Callback( + writer->sequence, disable_memtable, writer->log_used, index++, + pre_release_callback_cnt); + if (!ws.ok()) { + status = ws; + break; + } + } + } + } + if (publish_last_seq == kDoPublishLastSeq) { + versions_->SetLastSequence(last_sequence + seq_inc); + // Currently we only use kDoPublishLastSeq in unordered_write + assert(immutable_db_options_.unordered_write); + } + if (immutable_db_options_.unordered_write && status.ok()) { + pending_memtable_writes_ += memtable_write_cnt; + } + write_thread->ExitAsBatchGroupLeader(write_group, status); + if (status.ok()) { + status = w.FinalStatus(); + } + if (seq_used != nullptr) { + *seq_used = w.sequence; + } + return status; +} + +void DBImpl::WriteStatusCheckOnLocked(const Status& status) { + // Is setting bg_error_ enough here? This will at least stop + // compaction and fail any further writes. + InstrumentedMutexLock l(&mutex_); + assert(!status.IsIOFenced() || !error_handler_.GetBGError().ok()); + if (immutable_db_options_.paranoid_checks && !status.ok() && + !status.IsBusy() && !status.IsIncomplete()) { + // Maybe change the return status to void? + error_handler_.SetBGError(status, BackgroundErrorReason::kWriteCallback); + } +} + +void DBImpl::WriteStatusCheck(const Status& status) { + // Is setting bg_error_ enough here? This will at least stop + // compaction and fail any further writes. + assert(!status.IsIOFenced() || !error_handler_.GetBGError().ok()); + if (immutable_db_options_.paranoid_checks && !status.ok() && + !status.IsBusy() && !status.IsIncomplete()) { + mutex_.Lock(); + // Maybe change the return status to void? + error_handler_.SetBGError(status, BackgroundErrorReason::kWriteCallback); + mutex_.Unlock(); + } +} + +void DBImpl::IOStatusCheck(const IOStatus& io_status) { + // Is setting bg_error_ enough here? This will at least stop + // compaction and fail any further writes. + if ((immutable_db_options_.paranoid_checks && !io_status.ok() && + !io_status.IsBusy() && !io_status.IsIncomplete()) || + io_status.IsIOFenced()) { + mutex_.Lock(); + // Maybe change the return status to void? + error_handler_.SetBGError(io_status, BackgroundErrorReason::kWriteCallback); + mutex_.Unlock(); + } else { + // Force writable file to be continue writable. + logs_.back().writer->file()->reset_seen_error(); + } +} + +void DBImpl::MemTableInsertStatusCheck(const Status& status) { + // A non-OK status here indicates that the state implied by the + // WAL has diverged from the in-memory state. This could be + // because of a corrupt write_batch (very bad), or because the + // client specified an invalid column family and didn't specify + // ignore_missing_column_families. + if (!status.ok()) { + mutex_.Lock(); + assert(!error_handler_.IsBGWorkStopped()); + // Maybe change the return status to void? + error_handler_.SetBGError(status, BackgroundErrorReason::kMemTable) + .PermitUncheckedError(); + mutex_.Unlock(); + } +} + +Status DBImpl::PreprocessWrite(const WriteOptions& write_options, + LogContext* log_context, + WriteContext* write_context) { + assert(write_context != nullptr && log_context != nullptr); + Status status; + + if (error_handler_.IsDBStopped()) { + InstrumentedMutexLock l(&mutex_); + status = error_handler_.GetBGError(); + } + + PERF_TIMER_GUARD(write_scheduling_flushes_compactions_time); + + if (UNLIKELY(status.ok() && total_log_size_ > GetMaxTotalWalSize())) { + assert(versions_); + InstrumentedMutexLock l(&mutex_); + const ColumnFamilySet* const column_families = + versions_->GetColumnFamilySet(); + assert(column_families); + size_t num_cfs = column_families->NumberOfColumnFamilies(); + assert(num_cfs >= 1); + if (num_cfs > 1) { + WaitForPendingWrites(); + status = SwitchWAL(write_context); + } + } + + if (UNLIKELY(status.ok() && write_buffer_manager_->ShouldFlush())) { + // Before a new memtable is added in SwitchMemtable(), + // write_buffer_manager_->ShouldFlush() will keep returning true. If another + // thread is writing to another DB with the same write buffer, they may also + // be flushed. We may end up with flushing much more DBs than needed. It's + // suboptimal but still correct. + InstrumentedMutexLock l(&mutex_); + WaitForPendingWrites(); + status = HandleWriteBufferManagerFlush(write_context); + } + + if (UNLIKELY(status.ok() && !trim_history_scheduler_.Empty())) { + InstrumentedMutexLock l(&mutex_); + status = TrimMemtableHistory(write_context); + } + + if (UNLIKELY(status.ok() && !flush_scheduler_.Empty())) { + InstrumentedMutexLock l(&mutex_); + WaitForPendingWrites(); + status = ScheduleFlushes(write_context); + } + + PERF_TIMER_STOP(write_scheduling_flushes_compactions_time); + PERF_TIMER_GUARD(write_pre_and_post_process_time); + + if (UNLIKELY(status.ok() && (write_controller_.IsStopped() || + write_controller_.NeedsDelay()))) { + PERF_TIMER_STOP(write_pre_and_post_process_time); + PERF_TIMER_GUARD(write_delay_time); + // We don't know size of curent batch so that we always use the size + // for previous one. It might create a fairness issue that expiration + // might happen for smaller writes but larger writes can go through. + // Can optimize it if it is an issue. + InstrumentedMutexLock l(&mutex_); + status = DelayWrite(last_batch_group_size_, write_thread_, write_options); + PERF_TIMER_START(write_pre_and_post_process_time); + } + + // If memory usage exceeded beyond a certain threshold, + // write_buffer_manager_->ShouldStall() returns true to all threads writing to + // all DBs and writers will be stalled. + // It does soft checking because WriteBufferManager::buffer_limit_ has already + // exceeded at this point so no new write (including current one) will go + // through until memory usage is decreased. + if (UNLIKELY(status.ok() && write_buffer_manager_->ShouldStall())) { + default_cf_internal_stats_->AddDBStats( + InternalStats::kIntStatsWriteBufferManagerLimitStopsCounts, 1, + true /* concurrent */); + if (write_options.no_slowdown) { + status = Status::Incomplete("Write stall"); + } else { + InstrumentedMutexLock l(&mutex_); + WriteBufferManagerStallWrites(); + } + } + InstrumentedMutexLock l(&log_write_mutex_); + if (status.ok() && log_context->need_log_sync) { + // Wait until the parallel syncs are finished. Any sync process has to sync + // the front log too so it is enough to check the status of front() + // We do a while loop since log_sync_cv_ is signalled when any sync is + // finished + // Note: there does not seem to be a reason to wait for parallel sync at + // this early step but it is not important since parallel sync (SyncWAL) and + // need_log_sync are usually not used together. + while (logs_.front().IsSyncing()) { + log_sync_cv_.Wait(); + } + for (auto& log : logs_) { + // This is just to prevent the logs to be synced by a parallel SyncWAL + // call. We will do the actual syncing later after we will write to the + // WAL. + // Note: there does not seem to be a reason to set this early before we + // actually write to the WAL + log.PrepareForSync(); + } + } else { + log_context->need_log_sync = false; + } + log_context->writer = logs_.back().writer; + log_context->need_log_dir_sync = + log_context->need_log_dir_sync && !log_dir_synced_; + log_context->log_file_number_size = std::addressof(alive_log_files_.back()); + + return status; +} + +Status DBImpl::MergeBatch(const WriteThread::WriteGroup& write_group, + WriteBatch* tmp_batch, WriteBatch** merged_batch, + size_t* write_with_wal, + WriteBatch** to_be_cached_state) { + assert(write_with_wal != nullptr); + assert(tmp_batch != nullptr); + assert(*to_be_cached_state == nullptr); + *write_with_wal = 0; + auto* leader = write_group.leader; + assert(!leader->disable_wal); // Same holds for all in the batch group + if (write_group.size == 1 && !leader->CallbackFailed() && + leader->batch->GetWalTerminationPoint().is_cleared()) { + // we simply write the first WriteBatch to WAL if the group only + // contains one batch, that batch should be written to the WAL, + // and the batch is not wanting to be truncated + *merged_batch = leader->batch; + if (WriteBatchInternal::IsLatestPersistentState(*merged_batch)) { + *to_be_cached_state = *merged_batch; + } + *write_with_wal = 1; + } else { + // WAL needs all of the batches flattened into a single batch. + // We could avoid copying here with an iov-like AddRecord + // interface + *merged_batch = tmp_batch; + for (auto writer : write_group) { + if (!writer->CallbackFailed()) { + Status s = WriteBatchInternal::Append(*merged_batch, writer->batch, + /*WAL_only*/ true); + if (!s.ok()) { + tmp_batch->Clear(); + return s; + } + if (WriteBatchInternal::IsLatestPersistentState(writer->batch)) { + // We only need to cache the last of such write batch + *to_be_cached_state = writer->batch; + } + (*write_with_wal)++; + } + } + } + // return merged_batch; + return Status::OK(); +} + +// When two_write_queues_ is disabled, this function is called from the only +// write thread. Otherwise this must be called holding log_write_mutex_. +IOStatus DBImpl::WriteToWAL(const WriteBatch& merged_batch, + log::Writer* log_writer, uint64_t* log_used, + uint64_t* log_size, + Env::IOPriority rate_limiter_priority, + LogFileNumberSize& log_file_number_size) { + assert(log_size != nullptr); + + Slice log_entry = WriteBatchInternal::Contents(&merged_batch); + TEST_SYNC_POINT_CALLBACK("DBImpl::WriteToWAL:log_entry", &log_entry); + auto s = merged_batch.VerifyChecksum(); + if (!s.ok()) { + return status_to_io_status(std::move(s)); + } + *log_size = log_entry.size(); + // When two_write_queues_ WriteToWAL has to be protected from concurretn calls + // from the two queues anyway and log_write_mutex_ is already held. Otherwise + // if manual_wal_flush_ is enabled we need to protect log_writer->AddRecord + // from possible concurrent calls via the FlushWAL by the application. + const bool needs_locking = manual_wal_flush_ && !two_write_queues_; + // Due to performance cocerns of missed branch prediction penalize the new + // manual_wal_flush_ feature (by UNLIKELY) instead of the more common case + // when we do not need any locking. + if (UNLIKELY(needs_locking)) { + log_write_mutex_.Lock(); + } + IOStatus io_s = log_writer->MaybeAddUserDefinedTimestampSizeRecord( + versions_->GetColumnFamiliesTimestampSizeForRecord(), + rate_limiter_priority); + if (!io_s.ok()) { + return io_s; + } + io_s = log_writer->AddRecord(log_entry, rate_limiter_priority); + + if (UNLIKELY(needs_locking)) { + log_write_mutex_.Unlock(); + } + if (log_used != nullptr) { + *log_used = logfile_number_; + } + total_log_size_ += log_entry.size(); + log_file_number_size.AddSize(*log_size); + log_empty_ = false; + return io_s; +} + +IOStatus DBImpl::WriteToWAL(const WriteThread::WriteGroup& write_group, + log::Writer* log_writer, uint64_t* log_used, + bool need_log_sync, bool need_log_dir_sync, + SequenceNumber sequence, + LogFileNumberSize& log_file_number_size) { + IOStatus io_s; + assert(!two_write_queues_); + assert(!write_group.leader->disable_wal); + // Same holds for all in the batch group + size_t write_with_wal = 0; + WriteBatch* to_be_cached_state = nullptr; + WriteBatch* merged_batch; + io_s = status_to_io_status(MergeBatch(write_group, &tmp_batch_, &merged_batch, + &write_with_wal, &to_be_cached_state)); + if (UNLIKELY(!io_s.ok())) { + return io_s; + } + + if (merged_batch == write_group.leader->batch) { + write_group.leader->log_used = logfile_number_; + } else if (write_with_wal > 1) { + for (auto writer : write_group) { + writer->log_used = logfile_number_; + } + } + + WriteBatchInternal::SetSequence(merged_batch, sequence); + + uint64_t log_size; + io_s = WriteToWAL(*merged_batch, log_writer, log_used, &log_size, + write_group.leader->rate_limiter_priority, + log_file_number_size); + if (to_be_cached_state) { + cached_recoverable_state_ = *to_be_cached_state; + cached_recoverable_state_empty_ = false; + } + + if (io_s.ok() && need_log_sync) { + StopWatch sw(immutable_db_options_.clock, stats_, WAL_FILE_SYNC_MICROS); + // It's safe to access logs_ with unlocked mutex_ here because: + // - we've set getting_synced=true for all logs, + // so other threads won't pop from logs_ while we're here, + // - only writer thread can push to logs_, and we're in + // writer thread, so no one will push to logs_, + // - as long as other threads don't modify it, it's safe to read + // from std::deque from multiple threads concurrently. + // + // Sync operation should work with locked log_write_mutex_, because: + // when DBOptions.manual_wal_flush_ is set, + // FlushWAL function will be invoked by another thread. + // if without locked log_write_mutex_, the log file may get data + // corruption + + const bool needs_locking = manual_wal_flush_ && !two_write_queues_; + if (UNLIKELY(needs_locking)) { + log_write_mutex_.Lock(); + } + + for (auto& log : logs_) { + io_s = log.writer->file()->Sync(immutable_db_options_.use_fsync); + if (!io_s.ok()) { + break; + } + } + + if (UNLIKELY(needs_locking)) { + log_write_mutex_.Unlock(); + } + + if (io_s.ok() && need_log_dir_sync) { + // We only sync WAL directory the first time WAL syncing is + // requested, so that in case users never turn on WAL sync, + // we can avoid the disk I/O in the write code path. + io_s = directories_.GetWalDir()->FsyncWithDirOptions( + IOOptions(), nullptr, + DirFsyncOptions(DirFsyncOptions::FsyncReason::kNewFileSynced)); + } + } + + if (merged_batch == &tmp_batch_) { + tmp_batch_.Clear(); + } + if (io_s.ok()) { + auto stats = default_cf_internal_stats_; + if (need_log_sync) { + stats->AddDBStats(InternalStats::kIntStatsWalFileSynced, 1); + RecordTick(stats_, WAL_FILE_SYNCED); + } + stats->AddDBStats(InternalStats::kIntStatsWalFileBytes, log_size); + RecordTick(stats_, WAL_FILE_BYTES, log_size); + stats->AddDBStats(InternalStats::kIntStatsWriteWithWal, write_with_wal); + RecordTick(stats_, WRITE_WITH_WAL, write_with_wal); + } + return io_s; +} + +IOStatus DBImpl::ConcurrentWriteToWAL( + const WriteThread::WriteGroup& write_group, uint64_t* log_used, + SequenceNumber* last_sequence, size_t seq_inc) { + IOStatus io_s; + + assert(two_write_queues_ || immutable_db_options_.unordered_write); + assert(!write_group.leader->disable_wal); + // Same holds for all in the batch group + WriteBatch tmp_batch; + size_t write_with_wal = 0; + WriteBatch* to_be_cached_state = nullptr; + WriteBatch* merged_batch; + io_s = status_to_io_status(MergeBatch(write_group, &tmp_batch, &merged_batch, + &write_with_wal, &to_be_cached_state)); + if (UNLIKELY(!io_s.ok())) { + return io_s; + } + + // We need to lock log_write_mutex_ since logs_ and alive_log_files might be + // pushed back concurrently + log_write_mutex_.Lock(); + if (merged_batch == write_group.leader->batch) { + write_group.leader->log_used = logfile_number_; + } else if (write_with_wal > 1) { + for (auto writer : write_group) { + writer->log_used = logfile_number_; + } + } + *last_sequence = versions_->FetchAddLastAllocatedSequence(seq_inc); + auto sequence = *last_sequence + 1; + WriteBatchInternal::SetSequence(merged_batch, sequence); + + log::Writer* log_writer = logs_.back().writer; + LogFileNumberSize& log_file_number_size = alive_log_files_.back(); + + assert(log_writer->get_log_number() == log_file_number_size.number); + + uint64_t log_size; + io_s = WriteToWAL(*merged_batch, log_writer, log_used, &log_size, + write_group.leader->rate_limiter_priority, + log_file_number_size); + if (to_be_cached_state) { + cached_recoverable_state_ = *to_be_cached_state; + cached_recoverable_state_empty_ = false; + } + log_write_mutex_.Unlock(); + + if (io_s.ok()) { + const bool concurrent = true; + auto stats = default_cf_internal_stats_; + stats->AddDBStats(InternalStats::kIntStatsWalFileBytes, log_size, + concurrent); + RecordTick(stats_, WAL_FILE_BYTES, log_size); + stats->AddDBStats(InternalStats::kIntStatsWriteWithWal, write_with_wal, + concurrent); + RecordTick(stats_, WRITE_WITH_WAL, write_with_wal); + } + return io_s; +} + +Status DBImpl::WriteRecoverableState() { + mutex_.AssertHeld(); + if (!cached_recoverable_state_empty_) { + bool dont_care_bool; + SequenceNumber next_seq; + if (two_write_queues_) { + log_write_mutex_.Lock(); + } + SequenceNumber seq; + if (two_write_queues_) { + seq = versions_->FetchAddLastAllocatedSequence(0); + } else { + seq = versions_->LastSequence(); + } + WriteBatchInternal::SetSequence(&cached_recoverable_state_, seq + 1); + auto status = WriteBatchInternal::InsertInto( + &cached_recoverable_state_, column_family_memtables_.get(), + &flush_scheduler_, &trim_history_scheduler_, true, + 0 /*recovery_log_number*/, this, false /* concurrent_memtable_writes */, + &next_seq, &dont_care_bool, seq_per_batch_); + auto last_seq = next_seq - 1; + if (two_write_queues_) { + versions_->FetchAddLastAllocatedSequence(last_seq - seq); + versions_->SetLastPublishedSequence(last_seq); + } + versions_->SetLastSequence(last_seq); + if (two_write_queues_) { + log_write_mutex_.Unlock(); + } + if (status.ok() && recoverable_state_pre_release_callback_) { + const bool DISABLE_MEMTABLE = true; + for (uint64_t sub_batch_seq = seq + 1; + sub_batch_seq < next_seq && status.ok(); sub_batch_seq++) { + uint64_t const no_log_num = 0; + // Unlock it since the callback might end up locking mutex. e.g., + // AddCommitted -> AdvanceMaxEvictedSeq -> GetSnapshotListFromDB + mutex_.Unlock(); + status = recoverable_state_pre_release_callback_->Callback( + sub_batch_seq, !DISABLE_MEMTABLE, no_log_num, 0, 1); + mutex_.Lock(); + } + } + if (status.ok()) { + cached_recoverable_state_.Clear(); + cached_recoverable_state_empty_ = true; + } + return status; + } + return Status::OK(); +} + +void DBImpl::SelectColumnFamiliesForAtomicFlush( + autovector* selected_cfds, + const autovector& provided_candidate_cfds) { + mutex_.AssertHeld(); + assert(selected_cfds); + + autovector candidate_cfds; + + // Generate candidate cfds if not provided + if (provided_candidate_cfds.empty()) { + for (ColumnFamilyData* cfd : *versions_->GetColumnFamilySet()) { + if (!cfd->IsDropped() && cfd->initialized()) { + cfd->Ref(); + candidate_cfds.push_back(cfd); + } + } + } else { + candidate_cfds = provided_candidate_cfds; + } + + for (ColumnFamilyData* cfd : candidate_cfds) { + if (cfd->IsDropped()) { + continue; + } + if (cfd->imm()->NumNotFlushed() != 0 || !cfd->mem()->IsEmpty() || + !cached_recoverable_state_empty_.load()) { + selected_cfds->push_back(cfd); + } + } + + // Unref the newly generated candidate cfds (when not provided) in + // `candidate_cfds` + if (provided_candidate_cfds.empty()) { + for (auto candidate_cfd : candidate_cfds) { + candidate_cfd->UnrefAndTryDelete(); + } + } +} + +// Assign sequence number for atomic flush. +void DBImpl::AssignAtomicFlushSeq(const autovector& cfds) { + assert(immutable_db_options_.atomic_flush); + auto seq = versions_->LastSequence(); + for (auto cfd : cfds) { + cfd->imm()->AssignAtomicFlushSeq(seq); + } +} + +Status DBImpl::SwitchWAL(WriteContext* write_context) { + mutex_.AssertHeld(); + assert(write_context != nullptr); + Status status; + + if (alive_log_files_.begin()->getting_flushed) { + return status; + } + + auto oldest_alive_log = alive_log_files_.begin()->number; + bool flush_wont_release_oldest_log = false; + if (allow_2pc()) { + auto oldest_log_with_uncommitted_prep = + logs_with_prep_tracker_.FindMinLogContainingOutstandingPrep(); + + assert(oldest_log_with_uncommitted_prep == 0 || + oldest_log_with_uncommitted_prep >= oldest_alive_log); + if (oldest_log_with_uncommitted_prep > 0 && + oldest_log_with_uncommitted_prep == oldest_alive_log) { + if (unable_to_release_oldest_log_) { + // we already attempted to flush all column families dependent on + // the oldest alive log but the log still contained uncommitted + // transactions so there is still nothing that we can do. + return status; + } else { + ROCKS_LOG_WARN( + immutable_db_options_.info_log, + "Unable to release oldest log due to uncommitted transaction"); + unable_to_release_oldest_log_ = true; + flush_wont_release_oldest_log = true; + } + } + } + if (!flush_wont_release_oldest_log) { + // we only mark this log as getting flushed if we have successfully + // flushed all data in this log. If this log contains outstanding prepared + // transactions then we cannot flush this log until those transactions are + // commited. + unable_to_release_oldest_log_ = false; + alive_log_files_.begin()->getting_flushed = true; + } + + ROCKS_LOG_INFO( + immutable_db_options_.info_log, + "Flushing all column families with data in WAL number %" PRIu64 + ". Total log size is %" PRIu64 " while max_total_wal_size is %" PRIu64, + oldest_alive_log, total_log_size_.load(), GetMaxTotalWalSize()); + // no need to refcount because drop is happening in write thread, so can't + // happen while we're in the write thread + autovector cfds; + if (immutable_db_options_.atomic_flush) { + SelectColumnFamiliesForAtomicFlush(&cfds); + } else { + for (auto cfd : *versions_->GetColumnFamilySet()) { + if (cfd->IsDropped()) { + continue; + } + if (cfd->OldestLogToKeep() <= oldest_alive_log) { + cfds.push_back(cfd); + } + } + MaybeFlushStatsCF(&cfds); + } + WriteThread::Writer nonmem_w; + if (two_write_queues_) { + nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_); + } + + for (const auto cfd : cfds) { + cfd->Ref(); + status = SwitchMemtable(cfd, write_context); + cfd->UnrefAndTryDelete(); + if (!status.ok()) { + break; + } + } + if (two_write_queues_) { + nonmem_write_thread_.ExitUnbatched(&nonmem_w); + } + + if (status.ok()) { + if (immutable_db_options_.atomic_flush) { + AssignAtomicFlushSeq(cfds); + } + for (auto cfd : cfds) { + cfd->imm()->FlushRequested(); + if (!immutable_db_options_.atomic_flush) { + FlushRequest flush_req; + GenerateFlushRequest({cfd}, FlushReason::kWalFull, &flush_req); + SchedulePendingFlush(flush_req); + } + } + if (immutable_db_options_.atomic_flush) { + FlushRequest flush_req; + GenerateFlushRequest(cfds, FlushReason::kWalFull, &flush_req); + SchedulePendingFlush(flush_req); + } + MaybeScheduleFlushOrCompaction(); + } + return status; +} + +Status DBImpl::HandleWriteBufferManagerFlush(WriteContext* write_context) { + mutex_.AssertHeld(); + assert(write_context != nullptr); + Status status; + + // Before a new memtable is added in SwitchMemtable(), + // write_buffer_manager_->ShouldFlush() will keep returning true. If another + // thread is writing to another DB with the same write buffer, they may also + // be flushed. We may end up with flushing much more DBs than needed. It's + // suboptimal but still correct. + // no need to refcount because drop is happening in write thread, so can't + // happen while we're in the write thread + autovector cfds; + if (immutable_db_options_.atomic_flush) { + SelectColumnFamiliesForAtomicFlush(&cfds); + } else { + ColumnFamilyData* cfd_picked = nullptr; + SequenceNumber seq_num_for_cf_picked = kMaxSequenceNumber; + + for (auto cfd : *versions_->GetColumnFamilySet()) { + if (cfd->IsDropped()) { + continue; + } + if (!cfd->mem()->IsEmpty() && !cfd->imm()->IsFlushPendingOrRunning()) { + // We only consider flush on CFs with bytes in the mutable memtable, + // and no immutable memtables for which flush has yet to finish. If + // we triggered flush on CFs already trying to flush, we would risk + // creating too many immutable memtables leading to write stalls. + uint64_t seq = cfd->mem()->GetCreationSeq(); + if (cfd_picked == nullptr || seq < seq_num_for_cf_picked) { + cfd_picked = cfd; + seq_num_for_cf_picked = seq; + } + } + } + if (cfd_picked != nullptr) { + cfds.push_back(cfd_picked); + } + MaybeFlushStatsCF(&cfds); + } + if (!cfds.empty()) { + ROCKS_LOG_INFO( + immutable_db_options_.info_log, + "Flushing triggered to alleviate write buffer memory usage. Write " + "buffer is using %" ROCKSDB_PRIszt + " bytes out of a total of %" ROCKSDB_PRIszt ".", + write_buffer_manager_->memory_usage(), + write_buffer_manager_->buffer_size()); + } + + WriteThread::Writer nonmem_w; + if (two_write_queues_) { + nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_); + } + for (const auto cfd : cfds) { + if (cfd->mem()->IsEmpty()) { + continue; + } + cfd->Ref(); + status = SwitchMemtable(cfd, write_context); + cfd->UnrefAndTryDelete(); + if (!status.ok()) { + break; + } + } + if (two_write_queues_) { + nonmem_write_thread_.ExitUnbatched(&nonmem_w); + } + + if (status.ok()) { + if (immutable_db_options_.atomic_flush) { + AssignAtomicFlushSeq(cfds); + } + for (const auto cfd : cfds) { + cfd->imm()->FlushRequested(); + if (!immutable_db_options_.atomic_flush) { + FlushRequest flush_req; + GenerateFlushRequest({cfd}, FlushReason::kWriteBufferManager, + &flush_req); + SchedulePendingFlush(flush_req); + } + } + if (immutable_db_options_.atomic_flush) { + FlushRequest flush_req; + GenerateFlushRequest(cfds, FlushReason::kWriteBufferManager, &flush_req); + SchedulePendingFlush(flush_req); + } + MaybeScheduleFlushOrCompaction(); + } + return status; +} + +uint64_t DBImpl::GetMaxTotalWalSize() const { + uint64_t max_total_wal_size = + max_total_wal_size_.load(std::memory_order_acquire); + if (max_total_wal_size > 0) { + return max_total_wal_size; + } + return 4 * max_total_in_memory_state_.load(std::memory_order_acquire); +} + +// REQUIRES: mutex_ is held +// REQUIRES: this thread is currently at the leader for write_thread +Status DBImpl::DelayWrite(uint64_t num_bytes, WriteThread& write_thread, + const WriteOptions& write_options) { + mutex_.AssertHeld(); + uint64_t time_delayed = 0; + bool delayed = false; + { + StopWatch sw(immutable_db_options_.clock, stats_, WRITE_STALL, + Histograms::HISTOGRAM_ENUM_MAX, &time_delayed); + // To avoid parallel timed delays (bad throttling), only support them + // on the primary write queue. + uint64_t delay; + if (&write_thread == &write_thread_) { + delay = + write_controller_.GetDelay(immutable_db_options_.clock, num_bytes); + } else { + assert(num_bytes == 0); + delay = 0; + } + TEST_SYNC_POINT("DBImpl::DelayWrite:Start"); + if (delay > 0) { + if (write_options.no_slowdown) { + return Status::Incomplete("Write stall"); + } + TEST_SYNC_POINT("DBImpl::DelayWrite:Sleep"); + + // Notify write_thread about the stall so it can setup a barrier and + // fail any pending writers with no_slowdown + write_thread.BeginWriteStall(); + mutex_.Unlock(); + TEST_SYNC_POINT("DBImpl::DelayWrite:BeginWriteStallDone"); + // We will delay the write until we have slept for `delay` microseconds + // or we don't need a delay anymore. We check for cancellation every 1ms + // (slightly longer because WriteController minimum delay is 1ms, in + // case of sleep imprecision, rounding, etc.) + const uint64_t kDelayInterval = 1001; + uint64_t stall_end = sw.start_time() + delay; + while (write_controller_.NeedsDelay()) { + if (immutable_db_options_.clock->NowMicros() >= stall_end) { + // We already delayed this write `delay` microseconds + break; + } + + delayed = true; + // Sleep for 0.001 seconds + immutable_db_options_.clock->SleepForMicroseconds(kDelayInterval); + } + mutex_.Lock(); + write_thread.EndWriteStall(); + } + + // Don't wait if there's a background error, even if its a soft error. We + // might wait here indefinitely as the background compaction may never + // finish successfully, resulting in the stall condition lasting + // indefinitely + while (error_handler_.GetBGError().ok() && write_controller_.IsStopped() && + !shutting_down_.load(std::memory_order_relaxed)) { + if (write_options.no_slowdown) { + return Status::Incomplete("Write stall"); + } + delayed = true; + + // Notify write_thread about the stall so it can setup a barrier and + // fail any pending writers with no_slowdown + write_thread.BeginWriteStall(); + if (&write_thread == &write_thread_) { + TEST_SYNC_POINT("DBImpl::DelayWrite:Wait"); + } else { + TEST_SYNC_POINT("DBImpl::DelayWrite:NonmemWait"); + } + bg_cv_.Wait(); + TEST_SYNC_POINT_CALLBACK("DBImpl::DelayWrite:AfterWait", &mutex_); + write_thread.EndWriteStall(); + } + } + assert(!delayed || !write_options.no_slowdown); + if (delayed) { + default_cf_internal_stats_->AddDBStats( + InternalStats::kIntStatsWriteStallMicros, time_delayed); + RecordTick(stats_, STALL_MICROS, time_delayed); + } + + // If DB is not in read-only mode and write_controller is not stopping + // writes, we can ignore any background errors and allow the write to + // proceed + Status s; + if (write_controller_.IsStopped()) { + if (!shutting_down_.load(std::memory_order_relaxed)) { + // If writes are still stopped and db not shutdown, it means we bailed + // due to a background error + s = Status::Incomplete(error_handler_.GetBGError().ToString()); + } else { + s = Status::ShutdownInProgress("stalled writes"); + } + } + if (error_handler_.IsDBStopped()) { + s = error_handler_.GetBGError(); + } + return s; +} + +// REQUIRES: mutex_ is held +// REQUIRES: this thread is currently at the front of the writer queue +void DBImpl::WriteBufferManagerStallWrites() { + mutex_.AssertHeld(); + // First block future writer threads who want to add themselves to the queue + // of WriteThread. + write_thread_.BeginWriteStall(); + mutex_.Unlock(); + + // Change the state to State::Blocked. + static_cast(wbm_stall_.get()) + ->SetState(WBMStallInterface::State::BLOCKED); + // Then WriteBufferManager will add DB instance to its queue + // and block this thread by calling WBMStallInterface::Block(). + write_buffer_manager_->BeginWriteStall(wbm_stall_.get()); + wbm_stall_->Block(); + + mutex_.Lock(); + // Stall has ended. Signal writer threads so that they can add + // themselves to the WriteThread queue for writes. + write_thread_.EndWriteStall(); +} + +Status DBImpl::ThrottleLowPriWritesIfNeeded(const WriteOptions& write_options, + WriteBatch* my_batch) { + assert(write_options.low_pri); + // This is called outside the DB mutex. Although it is safe to make the call, + // the consistency condition is not guaranteed to hold. It's OK to live with + // it in this case. + // If we need to speed compaction, it means the compaction is left behind + // and we start to limit low pri writes to a limit. + if (write_controller_.NeedSpeedupCompaction()) { + if (allow_2pc() && (my_batch->HasCommit() || my_batch->HasRollback())) { + // For 2PC, we only rate limit prepare, not commit. + return Status::OK(); + } + if (write_options.no_slowdown) { + return Status::Incomplete("Low priority write stall"); + } else { + assert(my_batch != nullptr); + // Rate limit those writes. The reason that we don't completely wait + // is that in case the write is heavy, low pri writes may never have + // a chance to run. Now we guarantee we are still slowly making + // progress. + PERF_TIMER_GUARD(write_delay_time); + write_controller_.low_pri_rate_limiter()->Request( + my_batch->GetDataSize(), Env::IO_HIGH, nullptr /* stats */, + RateLimiter::OpType::kWrite); + } + } + return Status::OK(); +} + +void DBImpl::MaybeFlushStatsCF(autovector* cfds) { + assert(cfds != nullptr); + if (!cfds->empty() && immutable_db_options_.persist_stats_to_disk) { + ColumnFamilyData* cfd_stats = + versions_->GetColumnFamilySet()->GetColumnFamily( + kPersistentStatsColumnFamilyName); + if (cfd_stats != nullptr && !cfd_stats->mem()->IsEmpty()) { + for (ColumnFamilyData* cfd : *cfds) { + if (cfd == cfd_stats) { + // stats CF already included in cfds + return; + } + } + // force flush stats CF when its log number is less than all other CF's + // log numbers + bool force_flush_stats_cf = true; + for (auto* loop_cfd : *versions_->GetColumnFamilySet()) { + if (loop_cfd == cfd_stats) { + continue; + } + if (loop_cfd->GetLogNumber() <= cfd_stats->GetLogNumber()) { + force_flush_stats_cf = false; + } + } + if (force_flush_stats_cf) { + cfds->push_back(cfd_stats); + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "Force flushing stats CF with automated flush " + "to avoid holding old logs"); + } + } + } +} + +Status DBImpl::TrimMemtableHistory(WriteContext* context) { + autovector cfds; + ColumnFamilyData* tmp_cfd; + while ((tmp_cfd = trim_history_scheduler_.TakeNextColumnFamily()) != + nullptr) { + cfds.push_back(tmp_cfd); + } + for (auto& cfd : cfds) { + autovector to_delete; + bool trimmed = cfd->imm()->TrimHistory(&context->memtables_to_free_, + cfd->mem()->MemoryAllocatedBytes()); + if (trimmed) { + context->superversion_context.NewSuperVersion(); + assert(context->superversion_context.new_superversion.get() != nullptr); + cfd->InstallSuperVersion(&context->superversion_context, &mutex_); + } + + if (cfd->UnrefAndTryDelete()) { + cfd = nullptr; + } + } + return Status::OK(); +} + +Status DBImpl::ScheduleFlushes(WriteContext* context) { + autovector cfds; + if (immutable_db_options_.atomic_flush) { + SelectColumnFamiliesForAtomicFlush(&cfds); + for (auto cfd : cfds) { + cfd->Ref(); + } + flush_scheduler_.Clear(); + } else { + ColumnFamilyData* tmp_cfd; + while ((tmp_cfd = flush_scheduler_.TakeNextColumnFamily()) != nullptr) { + cfds.push_back(tmp_cfd); + } + MaybeFlushStatsCF(&cfds); + } + Status status; + WriteThread::Writer nonmem_w; + if (two_write_queues_) { + nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_); + } + + for (auto& cfd : cfds) { + if (!cfd->mem()->IsEmpty()) { + status = SwitchMemtable(cfd, context); + } + if (cfd->UnrefAndTryDelete()) { + cfd = nullptr; + } + if (!status.ok()) { + break; + } + } + + if (two_write_queues_) { + nonmem_write_thread_.ExitUnbatched(&nonmem_w); + } + + if (status.ok()) { + if (immutable_db_options_.atomic_flush) { + AssignAtomicFlushSeq(cfds); + FlushRequest flush_req; + GenerateFlushRequest(cfds, FlushReason::kWriteBufferFull, &flush_req); + SchedulePendingFlush(flush_req); + } else { + for (auto* cfd : cfds) { + FlushRequest flush_req; + GenerateFlushRequest({cfd}, FlushReason::kWriteBufferFull, &flush_req); + SchedulePendingFlush(flush_req); + } + } + MaybeScheduleFlushOrCompaction(); + } + return status; +} + +void DBImpl::NotifyOnMemTableSealed(ColumnFamilyData* /*cfd*/, + const MemTableInfo& mem_table_info) { + if (immutable_db_options_.listeners.size() == 0U) { + return; + } + if (shutting_down_.load(std::memory_order_acquire)) { + return; + } + + mutex_.Unlock(); + for (auto listener : immutable_db_options_.listeners) { + listener->OnMemTableSealed(mem_table_info); + } + mutex_.Lock(); +} + +// REQUIRES: mutex_ is held +// REQUIRES: this thread is currently at the front of the writer queue +// REQUIRES: this thread is currently at the front of the 2nd writer queue if +// two_write_queues_ is true (This is to simplify the reasoning.) +Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) { + mutex_.AssertHeld(); + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + log::Writer* new_log = nullptr; + MemTable* new_mem = nullptr; + IOStatus io_s; + + // Recoverable state is persisted in WAL. After memtable switch, WAL might + // be deleted, so we write the state to memtable to be persisted as well. + Status s = WriteRecoverableState(); + if (!s.ok()) { + return s; + } + + // Attempt to switch to a new memtable and trigger flush of old. + // Do this without holding the dbmutex lock. + assert(versions_->prev_log_number() == 0); + if (two_write_queues_) { + log_write_mutex_.Lock(); + } + bool creating_new_log = !log_empty_; + if (two_write_queues_) { + log_write_mutex_.Unlock(); + } + uint64_t recycle_log_number = 0; + if (creating_new_log && immutable_db_options_.recycle_log_file_num && + !log_recycle_files_.empty()) { + recycle_log_number = log_recycle_files_.front(); + } + uint64_t new_log_number = + creating_new_log ? versions_->NewFileNumber() : logfile_number_; + const MutableCFOptions mutable_cf_options = *cfd->GetLatestMutableCFOptions(); + + // Set memtable_info for memtable sealed callback + MemTableInfo memtable_info; + memtable_info.cf_name = cfd->GetName(); + memtable_info.first_seqno = cfd->mem()->GetFirstSequenceNumber(); + memtable_info.earliest_seqno = cfd->mem()->GetEarliestSequenceNumber(); + memtable_info.num_entries = cfd->mem()->num_entries(); + memtable_info.num_deletes = cfd->mem()->num_deletes(); + // Log this later after lock release. It may be outdated, e.g., if background + // flush happens before logging, but that should be ok. + int num_imm_unflushed = cfd->imm()->NumNotFlushed(); + const auto preallocate_block_size = + GetWalPreallocateBlockSize(mutable_cf_options.write_buffer_size); + mutex_.Unlock(); + if (creating_new_log) { + // TODO: Write buffer size passed in should be max of all CF's instead + // of mutable_cf_options.write_buffer_size. + io_s = CreateWAL(new_log_number, recycle_log_number, preallocate_block_size, + &new_log); + if (s.ok()) { + s = io_s; + } + } + if (s.ok()) { + SequenceNumber seq = versions_->LastSequence(); + new_mem = cfd->ConstructNewMemtable(mutable_cf_options, seq); + context->superversion_context.NewSuperVersion(); + } + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "[%s] New memtable created with log file: #%" PRIu64 + ". Immutable memtables: %d.\n", + cfd->GetName().c_str(), new_log_number, num_imm_unflushed); + // There should be no concurrent write as the thread is at the front of + // writer queue + cfd->mem()->ConstructFragmentedRangeTombstones(); + + mutex_.Lock(); + if (recycle_log_number != 0) { + // Since renaming the file is done outside DB mutex, we need to ensure + // concurrent full purges don't delete the file while we're recycling it. + // To achieve that we hold the old log number in the recyclable list until + // after it has been renamed. + assert(log_recycle_files_.front() == recycle_log_number); + log_recycle_files_.pop_front(); + } + if (s.ok() && creating_new_log) { + InstrumentedMutexLock l(&log_write_mutex_); + assert(new_log != nullptr); + if (!logs_.empty()) { + // Alway flush the buffer of the last log before switching to a new one + log::Writer* cur_log_writer = logs_.back().writer; + if (error_handler_.IsRecoveryInProgress()) { + // In recovery path, we force another try of writing WAL buffer. + cur_log_writer->file()->reset_seen_error(); + } + io_s = cur_log_writer->WriteBuffer(); + if (s.ok()) { + s = io_s; + } + if (!s.ok()) { + ROCKS_LOG_WARN(immutable_db_options_.info_log, + "[%s] Failed to switch from #%" PRIu64 " to #%" PRIu64 + " WAL file\n", + cfd->GetName().c_str(), cur_log_writer->get_log_number(), + new_log_number); + } + } + if (s.ok()) { + logfile_number_ = new_log_number; + log_empty_ = true; + log_dir_synced_ = false; + logs_.emplace_back(logfile_number_, new_log); + alive_log_files_.push_back(LogFileNumberSize(logfile_number_)); + } + } + + if (!s.ok()) { + // how do we fail if we're not creating new log? + assert(creating_new_log); + delete new_mem; + delete new_log; + context->superversion_context.new_superversion.reset(); + // We may have lost data from the WritableFileBuffer in-memory buffer for + // the current log, so treat it as a fatal error and set bg_error + if (!io_s.ok()) { + error_handler_.SetBGError(io_s, BackgroundErrorReason::kMemTable); + } else { + error_handler_.SetBGError(s, BackgroundErrorReason::kMemTable); + } + // Read back bg_error in order to get the right severity + s = error_handler_.GetBGError(); + return s; + } + + bool empty_cf_updated = false; + if (immutable_db_options_.track_and_verify_wals_in_manifest && + !immutable_db_options_.allow_2pc && creating_new_log) { + // In non-2pc mode, WALs become obsolete if they do not contain unflushed + // data. Updating the empty CF's log number might cause some WALs to become + // obsolete. So we should track the WAL obsoletion event before actually + // updating the empty CF's log number. + uint64_t min_wal_number_to_keep = + versions_->PreComputeMinLogNumberWithUnflushedData(logfile_number_); + if (min_wal_number_to_keep > + versions_->GetWalSet().GetMinWalNumberToKeep()) { + // Get a snapshot of the empty column families. + // LogAndApply may release and reacquire db + // mutex, during that period, column family may become empty (e.g. its + // flush succeeds), then it affects the computed min_log_number_to_keep, + // so we take a snapshot for consistency of column family data + // status. If a column family becomes non-empty afterwards, its active log + // should still be the created new log, so the min_log_number_to_keep is + // not affected. + autovector empty_cfs; + for (auto cf : *versions_->GetColumnFamilySet()) { + if (cf->IsEmpty()) { + empty_cfs.push_back(cf); + } + } + + VersionEdit wal_deletion; + wal_deletion.DeleteWalsBefore(min_wal_number_to_keep); + s = versions_->LogAndApplyToDefaultColumnFamily( + read_options, &wal_deletion, &mutex_, directories_.GetDbDir()); + if (!s.ok() && versions_->io_status().IsIOError()) { + s = error_handler_.SetBGError(versions_->io_status(), + BackgroundErrorReason::kManifestWrite); + } + if (!s.ok()) { + return s; + } + + for (auto cf : empty_cfs) { + if (cf->IsEmpty()) { + cf->SetLogNumber(logfile_number_); + // MEMPURGE: No need to change this, because new adds + // should still receive new sequence numbers. + cf->mem()->SetCreationSeq(versions_->LastSequence()); + } // cf may become non-empty. + } + empty_cf_updated = true; + } + } + if (!empty_cf_updated) { + for (auto cf : *versions_->GetColumnFamilySet()) { + // all this is just optimization to delete logs that + // are no longer needed -- if CF is empty, that means it + // doesn't need that particular log to stay alive, so we just + // advance the log number. no need to persist this in the manifest + if (cf->IsEmpty()) { + if (creating_new_log) { + cf->SetLogNumber(logfile_number_); + } + cf->mem()->SetCreationSeq(versions_->LastSequence()); + } + } + } + + cfd->mem()->SetNextLogNumber(logfile_number_); + assert(new_mem != nullptr); + cfd->imm()->Add(cfd->mem(), &context->memtables_to_free_); + new_mem->Ref(); + cfd->SetMemtable(new_mem); + InstallSuperVersionAndScheduleWork(cfd, &context->superversion_context, + mutable_cf_options); + + // Notify client that memtable is sealed, now that we have successfully + // installed a new memtable + NotifyOnMemTableSealed(cfd, memtable_info); + // It is possible that we got here without checking the value of i_os, but + // that is okay. If we did, it most likely means that s was already an error. + // In any case, ignore any unchecked error for i_os here. + io_s.PermitUncheckedError(); + return s; +} + +size_t DBImpl::GetWalPreallocateBlockSize(uint64_t write_buffer_size) const { + mutex_.AssertHeld(); + size_t bsize = + static_cast(write_buffer_size / 10 + write_buffer_size); + // Some users might set very high write_buffer_size and rely on + // max_total_wal_size or other parameters to control the WAL size. + if (mutable_db_options_.max_total_wal_size > 0) { + bsize = std::min( + bsize, static_cast(mutable_db_options_.max_total_wal_size)); + } + if (immutable_db_options_.db_write_buffer_size > 0) { + bsize = std::min(bsize, immutable_db_options_.db_write_buffer_size); + } + if (immutable_db_options_.write_buffer_manager && + immutable_db_options_.write_buffer_manager->enabled()) { + bsize = std::min( + bsize, immutable_db_options_.write_buffer_manager->buffer_size()); + } + + return bsize; +} + +// Default implementations of convenience methods that subclasses of DB +// can call if they wish +Status DB::Put(const WriteOptions& opt, ColumnFamilyHandle* column_family, + const Slice& key, const Slice& value) { + // Pre-allocate size of write batch conservatively. + // 8 bytes are taken by header, 4 bytes for count, 1 byte for type, + // and we allocate 11 extra bytes for key length, as well as value length. + WriteBatch batch(key.size() + value.size() + 24, 0 /* max_bytes */, + opt.protection_bytes_per_key, 0 /* default_cf_ts_sz */); + Status s = batch.Put(column_family, key, value); + if (!s.ok()) { + return s; + } + return Write(opt, &batch); +} + +Status DB::Put(const WriteOptions& opt, ColumnFamilyHandle* column_family, + const Slice& key, const Slice& ts, const Slice& value) { + ColumnFamilyHandle* default_cf = DefaultColumnFamily(); + assert(default_cf); + const Comparator* const default_cf_ucmp = default_cf->GetComparator(); + assert(default_cf_ucmp); + WriteBatch batch(0 /* reserved_bytes */, 0 /* max_bytes */, + opt.protection_bytes_per_key, + default_cf_ucmp->timestamp_size()); + Status s = batch.Put(column_family, key, ts, value); + if (!s.ok()) { + return s; + } + return Write(opt, &batch); +} + +Status DB::PutEntity(const WriteOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + const WideColumns& columns) { + const ColumnFamilyHandle* const default_cf = DefaultColumnFamily(); + assert(default_cf); + + const Comparator* const default_cf_ucmp = default_cf->GetComparator(); + assert(default_cf_ucmp); + + WriteBatch batch(/* reserved_bytes */ 0, /* max_bytes */ 0, + options.protection_bytes_per_key, + default_cf_ucmp->timestamp_size()); + + const Status s = batch.PutEntity(column_family, key, columns); + if (!s.ok()) { + return s; + } + + return Write(options, &batch); +} + +Status DB::Delete(const WriteOptions& opt, ColumnFamilyHandle* column_family, + const Slice& key) { + WriteBatch batch(0 /* reserved_bytes */, 0 /* max_bytes */, + opt.protection_bytes_per_key, 0 /* default_cf_ts_sz */); + Status s = batch.Delete(column_family, key); + if (!s.ok()) { + return s; + } + return Write(opt, &batch); +} + +Status DB::Delete(const WriteOptions& opt, ColumnFamilyHandle* column_family, + const Slice& key, const Slice& ts) { + ColumnFamilyHandle* default_cf = DefaultColumnFamily(); + assert(default_cf); + const Comparator* const default_cf_ucmp = default_cf->GetComparator(); + assert(default_cf_ucmp); + WriteBatch batch(0 /* reserved_bytes */, 0 /* max_bytes */, + opt.protection_bytes_per_key, + default_cf_ucmp->timestamp_size()); + Status s = batch.Delete(column_family, key, ts); + if (!s.ok()) { + return s; + } + return Write(opt, &batch); +} + +Status DB::SingleDelete(const WriteOptions& opt, + ColumnFamilyHandle* column_family, const Slice& key) { + WriteBatch batch(0 /* reserved_bytes */, 0 /* max_bytes */, + opt.protection_bytes_per_key, 0 /* default_cf_ts_sz */); + Status s = batch.SingleDelete(column_family, key); + if (!s.ok()) { + return s; + } + return Write(opt, &batch); +} + +Status DB::SingleDelete(const WriteOptions& opt, + ColumnFamilyHandle* column_family, const Slice& key, + const Slice& ts) { + ColumnFamilyHandle* default_cf = DefaultColumnFamily(); + assert(default_cf); + const Comparator* const default_cf_ucmp = default_cf->GetComparator(); + assert(default_cf_ucmp); + WriteBatch batch(0 /* reserved_bytes */, 0 /* max_bytes */, + opt.protection_bytes_per_key, + default_cf_ucmp->timestamp_size()); + Status s = batch.SingleDelete(column_family, key, ts); + if (!s.ok()) { + return s; + } + return Write(opt, &batch); +} + +Status DB::DeleteRange(const WriteOptions& opt, + ColumnFamilyHandle* column_family, + const Slice& begin_key, const Slice& end_key) { + WriteBatch batch(0 /* reserved_bytes */, 0 /* max_bytes */, + opt.protection_bytes_per_key, 0 /* default_cf_ts_sz */); + Status s = batch.DeleteRange(column_family, begin_key, end_key); + if (!s.ok()) { + return s; + } + return Write(opt, &batch); +} + +Status DB::DeleteRange(const WriteOptions& opt, + ColumnFamilyHandle* column_family, + const Slice& begin_key, const Slice& end_key, + const Slice& ts) { + ColumnFamilyHandle* default_cf = DefaultColumnFamily(); + assert(default_cf); + const Comparator* const default_cf_ucmp = default_cf->GetComparator(); + assert(default_cf_ucmp); + WriteBatch batch(0 /* reserved_bytes */, 0 /* max_bytes */, + opt.protection_bytes_per_key, + default_cf_ucmp->timestamp_size()); + Status s = batch.DeleteRange(column_family, begin_key, end_key, ts); + if (!s.ok()) { + return s; + } + return Write(opt, &batch); +} + +Status DB::Merge(const WriteOptions& opt, ColumnFamilyHandle* column_family, + const Slice& key, const Slice& value) { + WriteBatch batch(0 /* reserved_bytes */, 0 /* max_bytes */, + opt.protection_bytes_per_key, 0 /* default_cf_ts_sz */); + Status s = batch.Merge(column_family, key, value); + if (!s.ok()) { + return s; + } + return Write(opt, &batch); +} + +Status DB::Merge(const WriteOptions& opt, ColumnFamilyHandle* column_family, + const Slice& key, const Slice& ts, const Slice& value) { + ColumnFamilyHandle* default_cf = DefaultColumnFamily(); + assert(default_cf); + const Comparator* const default_cf_ucmp = default_cf->GetComparator(); + assert(default_cf_ucmp); + WriteBatch batch(0 /* reserved_bytes */, 0 /* max_bytes */, + opt.protection_bytes_per_key, + default_cf_ucmp->timestamp_size()); + Status s = batch.Merge(column_family, key, ts, value); + if (!s.ok()) { + return s; + } + return Write(opt, &batch); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/db_info_dumper.cc b/librocksdb-sys/rocksdb/db/db_info_dumper.cc new file mode 100644 index 0000000..be8d5be --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_info_dumper.cc @@ -0,0 +1,147 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/db_info_dumper.h" + +#include + +#include +#include +#include +#include + +#include "file/filename.h" +#include "rocksdb/env.h" + +namespace ROCKSDB_NAMESPACE { + +void DumpDBFileSummary(const ImmutableDBOptions& options, + const std::string& dbname, + const std::string& session_id) { + if (options.info_log == nullptr) { + return; + } + + auto* env = options.env; + uint64_t number = 0; + FileType type = kInfoLogFile; + + std::vector files; + uint64_t file_num = 0; + uint64_t file_size; + std::string file_info, wal_info; + + Header(options.info_log, "DB SUMMARY\n"); + Header(options.info_log, "DB Session ID: %s\n", session_id.c_str()); + + Status s; + // Get files in dbname dir + s = env->GetChildren(dbname, &files); + if (!s.ok()) { + Error(options.info_log, "Error when reading %s dir %s\n", dbname.c_str(), + s.ToString().c_str()); + } + std::sort(files.begin(), files.end()); + for (const std::string& file : files) { + if (!ParseFileName(file, &number, &type)) { + continue; + } + switch (type) { + case kCurrentFile: + Header(options.info_log, "CURRENT file: %s\n", file.c_str()); + break; + case kIdentityFile: + Header(options.info_log, "IDENTITY file: %s\n", file.c_str()); + break; + case kDescriptorFile: + s = env->GetFileSize(dbname + "/" + file, &file_size); + if (s.ok()) { + Header(options.info_log, + "MANIFEST file: %s size: %" PRIu64 " Bytes\n", file.c_str(), + file_size); + } else { + Error(options.info_log, + "Error when reading MANIFEST file: %s/%s %s\n", dbname.c_str(), + file.c_str(), s.ToString().c_str()); + } + break; + case kWalFile: + s = env->GetFileSize(dbname + "/" + file, &file_size); + if (s.ok()) { + wal_info.append(file) + .append(" size: ") + .append(std::to_string(file_size)) + .append(" ; "); + } else { + Error(options.info_log, "Error when reading LOG file: %s/%s %s\n", + dbname.c_str(), file.c_str(), s.ToString().c_str()); + } + break; + case kTableFile: + if (++file_num < 10) { + file_info.append(file).append(" "); + } + break; + default: + break; + } + } + + // Get sst files in db_path dir + for (auto& db_path : options.db_paths) { + if (dbname.compare(db_path.path) != 0) { + s = env->GetChildren(db_path.path, &files); + if (!s.ok()) { + Error(options.info_log, "Error when reading %s dir %s\n", + db_path.path.c_str(), s.ToString().c_str()); + continue; + } + std::sort(files.begin(), files.end()); + for (const std::string& file : files) { + if (ParseFileName(file, &number, &type)) { + if (type == kTableFile && ++file_num < 10) { + file_info.append(file).append(" "); + } + } + } + } + Header(options.info_log, + "SST files in %s dir, Total Num: %" PRIu64 ", files: %s\n", + db_path.path.c_str(), file_num, file_info.c_str()); + file_num = 0; + file_info.clear(); + } + + // Get wal file in wal_dir + const auto& wal_dir = options.GetWalDir(dbname); + if (!options.IsWalDirSameAsDBPath(dbname)) { + s = env->GetChildren(wal_dir, &files); + if (!s.ok()) { + Error(options.info_log, "Error when reading %s dir %s\n", wal_dir.c_str(), + s.ToString().c_str()); + return; + } + wal_info.clear(); + for (const std::string& file : files) { + if (ParseFileName(file, &number, &type)) { + if (type == kWalFile) { + s = env->GetFileSize(wal_dir + "/" + file, &file_size); + if (s.ok()) { + wal_info.append(file) + .append(" size: ") + .append(std::to_string(file_size)) + .append(" ; "); + } else { + Error(options.info_log, "Error when reading LOG file %s/%s %s\n", + wal_dir.c_str(), file.c_str(), s.ToString().c_str()); + } + } + } + } + } + Header(options.info_log, "Write Ahead Log file in %s: %s\n", wal_dir.c_str(), + wal_info.c_str()); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/db_info_dumper.h b/librocksdb-sys/rocksdb/db/db_info_dumper.h new file mode 100644 index 0000000..f518e84 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_info_dumper.h @@ -0,0 +1,15 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include + +#include "options/db_options.h" + +namespace ROCKSDB_NAMESPACE { +void DumpDBFileSummary(const ImmutableDBOptions& options, + const std::string& dbname, + const std::string& session_id = ""); +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/db_inplace_update_test.cc b/librocksdb-sys/rocksdb/db/db_inplace_update_test.cc new file mode 100644 index 0000000..3921a3b --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_inplace_update_test.cc @@ -0,0 +1,262 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "db/db_test_util.h" +#include "port/stack_trace.h" + +namespace ROCKSDB_NAMESPACE { + +class DBTestInPlaceUpdate : public DBTestBase { + public: + DBTestInPlaceUpdate() + : DBTestBase("db_inplace_update_test", /*env_do_fsync=*/true) {} +}; + +TEST_F(DBTestInPlaceUpdate, InPlaceUpdate) { + do { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.inplace_update_support = true; + options.env = env_; + options.write_buffer_size = 100000; + options.allow_concurrent_memtable_write = false; + Reopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // Update key with values of smaller size + int numValues = 10; + for (int i = numValues; i > 0; i--) { + std::string value = DummyString(i, 'a'); + ASSERT_OK(Put(1, "key", value)); + ASSERT_EQ(value, Get(1, "key")); + } + + // Only 1 instance for that key. + validateNumberOfEntries(1, 1); + } while (ChangeCompactOptions()); +} + +TEST_F(DBTestInPlaceUpdate, InPlaceUpdateLargeNewValue) { + do { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.inplace_update_support = true; + options.env = env_; + options.write_buffer_size = 100000; + options.allow_concurrent_memtable_write = false; + Reopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // Update key with values of larger size + int numValues = 10; + for (int i = 0; i < numValues; i++) { + std::string value = DummyString(i, 'a'); + ASSERT_OK(Put(1, "key", value)); + ASSERT_EQ(value, Get(1, "key")); + } + + // All 10 updates exist in the internal iterator + validateNumberOfEntries(numValues, 1); + } while (ChangeCompactOptions()); +} + +TEST_F(DBTestInPlaceUpdate, InPlaceUpdateEntitySmallerNewValue) { + do { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.inplace_update_support = true; + options.env = env_; + options.allow_concurrent_memtable_write = false; + + Reopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // Update key with values of smaller size + constexpr int num_values = 10; + for (int i = num_values; i > 0; --i) { + constexpr char key[] = "key"; + const std::string value = DummyString(i, 'a'); + WideColumns wide_columns{{"attr", value}}; + + ASSERT_OK(db_->PutEntity(WriteOptions(), handles_[1], key, wide_columns)); + // TODO: use Get to check entity once it's supported + } + + // Only 1 instance for that key. + validateNumberOfEntries(1, 1); + } while (ChangeCompactOptions()); +} + +TEST_F(DBTestInPlaceUpdate, InPlaceUpdateEntityLargerNewValue) { + do { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.inplace_update_support = true; + options.env = env_; + options.allow_concurrent_memtable_write = false; + + Reopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // Update key with values of larger size + constexpr int num_values = 10; + for (int i = 0; i < num_values; ++i) { + constexpr char key[] = "key"; + const std::string value = DummyString(i, 'a'); + WideColumns wide_columns{{"attr", value}}; + + ASSERT_OK(db_->PutEntity(WriteOptions(), handles_[1], key, wide_columns)); + // TODO: use Get to check entity once it's supported + } + + // All 10 updates exist in the internal iterator + validateNumberOfEntries(num_values, 1); + } while (ChangeCompactOptions()); +} + +TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerSize) { + do { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.inplace_update_support = true; + + options.env = env_; + options.write_buffer_size = 100000; + options.inplace_callback = + ROCKSDB_NAMESPACE::DBTestInPlaceUpdate::updateInPlaceSmallerSize; + options.allow_concurrent_memtable_write = false; + Reopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // Update key with values of smaller size + int numValues = 10; + ASSERT_OK(Put(1, "key", DummyString(numValues, 'a'))); + ASSERT_EQ(DummyString(numValues, 'c'), Get(1, "key")); + + for (int i = numValues; i > 0; i--) { + ASSERT_OK(Put(1, "key", DummyString(i, 'a'))); + ASSERT_EQ(DummyString(i - 1, 'b'), Get(1, "key")); + } + + // Only 1 instance for that key. + validateNumberOfEntries(1, 1); + } while (ChangeCompactOptions()); +} + +TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerVarintSize) { + do { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.inplace_update_support = true; + + options.env = env_; + options.write_buffer_size = 100000; + options.inplace_callback = + ROCKSDB_NAMESPACE::DBTestInPlaceUpdate::updateInPlaceSmallerVarintSize; + options.allow_concurrent_memtable_write = false; + Reopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // Update key with values of smaller varint size + int numValues = 265; + ASSERT_OK(Put(1, "key", DummyString(numValues, 'a'))); + ASSERT_EQ(DummyString(numValues, 'c'), Get(1, "key")); + + for (int i = numValues; i > 0; i--) { + ASSERT_OK(Put(1, "key", DummyString(i, 'a'))); + ASSERT_EQ(DummyString(1, 'b'), Get(1, "key")); + } + + // Only 1 instance for that key. + validateNumberOfEntries(1, 1); + } while (ChangeCompactOptions()); +} + +TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackLargeNewValue) { + do { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.inplace_update_support = true; + + options.env = env_; + options.write_buffer_size = 100000; + options.inplace_callback = + ROCKSDB_NAMESPACE::DBTestInPlaceUpdate::updateInPlaceLargerSize; + options.allow_concurrent_memtable_write = false; + Reopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // Update key with values of larger size + int numValues = 10; + for (int i = 0; i < numValues; i++) { + ASSERT_OK(Put(1, "key", DummyString(i, 'a'))); + ASSERT_EQ(DummyString(i, 'c'), Get(1, "key")); + } + + // No inplace updates. All updates are puts with new seq number + // All 10 updates exist in the internal iterator + validateNumberOfEntries(numValues, 1); + } while (ChangeCompactOptions()); +} + +TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackNoAction) { + do { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.inplace_update_support = true; + + options.env = env_; + options.write_buffer_size = 100000; + options.inplace_callback = + ROCKSDB_NAMESPACE::DBTestInPlaceUpdate::updateInPlaceNoAction; + options.allow_concurrent_memtable_write = false; + Reopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // Callback function requests no actions from db + ASSERT_OK(Put(1, "key", DummyString(1, 'a'))); + ASSERT_EQ(Get(1, "key"), "NOT_FOUND"); + } while (ChangeCompactOptions()); +} + +TEST_F(DBTestInPlaceUpdate, InPlaceUpdateAndSnapshot) { + do { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.inplace_update_support = true; + options.env = env_; + options.write_buffer_size = 100000; + options.allow_concurrent_memtable_write = false; + Reopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // Update key with values of smaller size, and + // run GetSnapshot and ReleaseSnapshot + int numValues = 2; + for (int i = numValues; i > 0; i--) { + const Snapshot* s = db_->GetSnapshot(); + ASSERT_EQ(nullptr, s); + std::string value = DummyString(i, 'a'); + ASSERT_OK(Put(1, "key", value)); + ASSERT_EQ(value, Get(1, "key")); + // release s (nullptr) + db_->ReleaseSnapshot(s); + } + + // Only 1 instance for that key. + validateNumberOfEntries(1, 1); + } while (ChangeCompactOptions()); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_io_failure_test.cc b/librocksdb-sys/rocksdb/db/db_io_failure_test.cc new file mode 100644 index 0000000..e79272e --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_io_failure_test.cc @@ -0,0 +1,589 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/db_test_util.h" +#include "port/stack_trace.h" +#include "test_util/testutil.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +class DBIOFailureTest : public DBTestBase { + public: + DBIOFailureTest() : DBTestBase("db_io_failure_test", /*env_do_fsync=*/true) {} +}; + +// Check that number of files does not grow when writes are dropped +TEST_F(DBIOFailureTest, DropWrites) { + do { + Options options = CurrentOptions(); + options.env = env_; + options.paranoid_checks = false; + Reopen(options); + + ASSERT_OK(Put("foo", "v1")); + ASSERT_EQ("v1", Get("foo")); + Compact("a", "z"); + const size_t num_files = CountFiles(); + // Force out-of-space errors + env_->drop_writes_.store(true, std::memory_order_release); + env_->sleep_counter_.Reset(); + env_->SetMockSleep(); + for (int i = 0; i < 5; i++) { + if (option_config_ != kUniversalCompactionMultiLevel && + option_config_ != kUniversalSubcompactions) { + for (int level = 0; level < dbfull()->NumberLevels(); level++) { + if (level > 0 && level == dbfull()->NumberLevels() - 1) { + break; + } + Status s = + dbfull()->TEST_CompactRange(level, nullptr, nullptr, nullptr, + true /* disallow trivial move */); + ASSERT_TRUE(s.ok() || s.IsCorruption()); + } + } else { + Status s = + dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr); + ASSERT_TRUE(s.ok() || s.IsCorruption()); + } + } + + std::string property_value; + ASSERT_TRUE(db_->GetProperty("rocksdb.background-errors", &property_value)); + ASSERT_EQ("5", property_value); + + env_->drop_writes_.store(false, std::memory_order_release); + const size_t count = CountFiles(); + ASSERT_LT(count, num_files + 3); + + // Check that compaction attempts slept after errors + // TODO @krad: Figure out why ASSERT_EQ 5 keeps failing in certain compiler + // versions + ASSERT_GE(env_->sleep_counter_.Read(), 4); + } while (ChangeCompactOptions()); +} + +// Check background error counter bumped on flush failures. +TEST_F(DBIOFailureTest, DropWritesFlush) { + do { + Options options = CurrentOptions(); + options.env = env_; + options.max_background_flushes = 1; + Reopen(options); + + ASSERT_OK(Put("foo", "v1")); + // Force out-of-space errors + env_->drop_writes_.store(true, std::memory_order_release); + + std::string property_value; + // Background error count is 0 now. + ASSERT_TRUE(db_->GetProperty("rocksdb.background-errors", &property_value)); + ASSERT_EQ("0", property_value); + + // ASSERT file is too short + ASSERT_TRUE(dbfull()->TEST_FlushMemTable(true).IsCorruption()); + + ASSERT_TRUE(db_->GetProperty("rocksdb.background-errors", &property_value)); + ASSERT_EQ("1", property_value); + + env_->drop_writes_.store(false, std::memory_order_release); + } while (ChangeCompactOptions()); +} + +// Check that CompactRange() returns failure if there is not enough space left +// on device +TEST_F(DBIOFailureTest, NoSpaceCompactRange) { + do { + Options options = CurrentOptions(); + options.env = env_; + options.disable_auto_compactions = true; + Reopen(options); + + // generate 5 tables + for (int i = 0; i < 5; ++i) { + ASSERT_OK(Put(Key(i), Key(i) + "v")); + ASSERT_OK(Flush()); + } + + // Force out-of-space errors + env_->no_space_.store(true, std::memory_order_release); + + Status s = dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr, + true /* disallow trivial move */); + ASSERT_TRUE(s.IsIOError()); + ASSERT_TRUE(s.IsNoSpace()); + + env_->no_space_.store(false, std::memory_order_release); + } while (ChangeCompactOptions()); +} + +TEST_F(DBIOFailureTest, NonWritableFileSystem) { + do { + Options options = CurrentOptions(); + options.write_buffer_size = 4096; + options.arena_block_size = 4096; + options.env = env_; + Reopen(options); + ASSERT_OK(Put("foo", "v1")); + env_->non_writeable_rate_.store(100); + std::string big(100000, 'x'); + int errors = 0; + for (int i = 0; i < 20; i++) { + if (!Put("foo", big).ok()) { + errors++; + env_->SleepForMicroseconds(100000); + } + } + ASSERT_GT(errors, 0); + env_->non_writeable_rate_.store(0); + } while (ChangeCompactOptions()); +} + +TEST_F(DBIOFailureTest, ManifestWriteError) { + // Test for the following problem: + // (a) Compaction produces file F + // (b) Log record containing F is written to MANIFEST file, but Sync() fails + // (c) GC deletes F + // (d) After reopening DB, reads fail since deleted F is named in log record + + // We iterate twice. In the second iteration, everything is the + // same except the log record never makes it to the MANIFEST file. + for (int iter = 0; iter < 2; iter++) { + std::atomic* error_type = (iter == 0) ? &env_->manifest_sync_error_ + : &env_->manifest_write_error_; + + // Insert foo=>bar mapping + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.error_if_exists = false; + options.paranoid_checks = true; + DestroyAndReopen(options); + ASSERT_OK(Put("foo", "bar")); + ASSERT_EQ("bar", Get("foo")); + + // Memtable compaction (will succeed) + ASSERT_OK(Flush()); + ASSERT_EQ("bar", Get("foo")); + const int last = 2; + MoveFilesToLevel(2); + ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo=>bar is now in last level + + // Merging compaction (will fail) + error_type->store(true, std::memory_order_release); + ASSERT_NOK( + dbfull()->TEST_CompactRange(last, nullptr, nullptr)); // Should fail + ASSERT_EQ("bar", Get("foo")); + + error_type->store(false, std::memory_order_release); + + // Since paranoid_checks=true, writes should fail + ASSERT_NOK(Put("foo2", "bar2")); + + // Recovery: should not lose data + ASSERT_EQ("bar", Get("foo")); + + // Try again with paranoid_checks=false + Close(); + options.paranoid_checks = false; + Reopen(options); + + // Merging compaction (will fail) + error_type->store(true, std::memory_order_release); + Status s = + dbfull()->TEST_CompactRange(last, nullptr, nullptr); // Should fail + if (iter == 0) { + ASSERT_OK(s); + } else { + ASSERT_TRUE(s.IsIOError()); + } + ASSERT_EQ("bar", Get("foo")); + + // Recovery: should not lose data + error_type->store(false, std::memory_order_release); + Reopen(options); + ASSERT_EQ("bar", Get("foo")); + + // Since paranoid_checks=false, writes should succeed + ASSERT_OK(Put("foo2", "bar2")); + ASSERT_EQ("bar", Get("foo")); + ASSERT_EQ("bar2", Get("foo2")); + } +} + +TEST_F(DBIOFailureTest, PutFailsParanoid) { + // Test the following: + // (a) A random put fails in paranoid mode (simulate by sync fail) + // (b) All other puts have to fail, even if writes would succeed + // (c) All of that should happen ONLY if paranoid_checks = true + + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.error_if_exists = false; + options.paranoid_checks = true; + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(1, "foo", "bar")); + ASSERT_OK(Put(1, "foo1", "bar1")); + // simulate error + env_->log_write_error_.store(true, std::memory_order_release); + ASSERT_NOK(Put(1, "foo2", "bar2")); + env_->log_write_error_.store(false, std::memory_order_release); + // the next put should fail, too + ASSERT_NOK(Put(1, "foo3", "bar3")); + // but we're still able to read + ASSERT_EQ("bar", Get(1, "foo")); + + // do the same thing with paranoid checks off + options.paranoid_checks = false; + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(1, "foo", "bar")); + ASSERT_OK(Put(1, "foo1", "bar1")); + // simulate error + env_->log_write_error_.store(true, std::memory_order_release); + ASSERT_NOK(Put(1, "foo2", "bar2")); + env_->log_write_error_.store(false, std::memory_order_release); + // the next put should NOT fail + ASSERT_OK(Put(1, "foo3", "bar3")); +} +#if !(defined NDEBUG) || !defined(OS_WIN) +TEST_F(DBIOFailureTest, FlushSstRangeSyncError) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.error_if_exists = false; + options.paranoid_checks = true; + options.write_buffer_size = 256 * 1024 * 1024; + options.writable_file_max_buffer_size = 128 * 1024; + options.bytes_per_sync = 128 * 1024; + options.level0_file_num_compaction_trigger = 4; + options.memtable_factory.reset(test::NewSpecialSkipListFactory(10)); + BlockBasedTableOptions table_options; + table_options.filter_policy.reset(NewBloomFilterPolicy(10)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + const char* io_error_msg = "range sync dummy error"; + std::atomic range_sync_called(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SpecialEnv::SStableFile::RangeSync", [&](void* arg) { + if (range_sync_called.fetch_add(1) == 0) { + Status* st = static_cast(arg); + *st = Status::IOError(io_error_msg); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Random rnd(301); + std::string rnd_str = + rnd.RandomString(static_cast(options.bytes_per_sync / 2)); + std::string rnd_str_512kb = rnd.RandomString(512 * 1024); + + ASSERT_OK(Put(1, "foo", "bar")); + // First 1MB doesn't get range synced + ASSERT_OK(Put(1, "foo0_0", rnd_str_512kb)); + ASSERT_OK(Put(1, "foo0_1", rnd_str_512kb)); + ASSERT_OK(Put(1, "foo1_1", rnd_str)); + ASSERT_OK(Put(1, "foo1_2", rnd_str)); + ASSERT_OK(Put(1, "foo1_3", rnd_str)); + ASSERT_OK(Put(1, "foo2", "bar")); + ASSERT_OK(Put(1, "foo3_1", rnd_str)); + ASSERT_OK(Put(1, "foo3_2", rnd_str)); + ASSERT_OK(Put(1, "foo3_3", rnd_str)); + ASSERT_OK(Put(1, "foo4", "bar")); + Status s = dbfull()->TEST_WaitForFlushMemTable(handles_[1]); + ASSERT_TRUE(s.IsIOError()); + ASSERT_STREQ(s.getState(), io_error_msg); + + // Following writes should fail as flush failed. + ASSERT_NOK(Put(1, "foo2", "bar3")); + ASSERT_EQ("bar", Get(1, "foo")); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ASSERT_GE(1, range_sync_called.load()); + + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_EQ("bar", Get(1, "foo")); +} + +TEST_F(DBIOFailureTest, CompactSstRangeSyncError) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.error_if_exists = false; + options.paranoid_checks = true; + options.write_buffer_size = 256 * 1024 * 1024; + options.writable_file_max_buffer_size = 128 * 1024; + options.bytes_per_sync = 128 * 1024; + options.level0_file_num_compaction_trigger = 2; + options.target_file_size_base = 256 * 1024 * 1024; + options.disable_auto_compactions = true; + BlockBasedTableOptions table_options; + table_options.filter_policy.reset(NewBloomFilterPolicy(10)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + Random rnd(301); + std::string rnd_str = + rnd.RandomString(static_cast(options.bytes_per_sync / 2)); + std::string rnd_str_512kb = rnd.RandomString(512 * 1024); + + ASSERT_OK(Put(1, "foo", "bar")); + // First 1MB doesn't get range synced + ASSERT_OK(Put(1, "foo0_0", rnd_str_512kb)); + ASSERT_OK(Put(1, "foo0_1", rnd_str_512kb)); + ASSERT_OK(Put(1, "foo1_1", rnd_str)); + ASSERT_OK(Put(1, "foo1_2", rnd_str)); + ASSERT_OK(Put(1, "foo1_3", rnd_str)); + ASSERT_OK(Flush(1)); + ASSERT_OK(Put(1, "foo", "bar")); + ASSERT_OK(Put(1, "foo3_1", rnd_str)); + ASSERT_OK(Put(1, "foo3_2", rnd_str)); + ASSERT_OK(Put(1, "foo3_3", rnd_str)); + ASSERT_OK(Put(1, "foo4", "bar")); + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); + + const char* io_error_msg = "range sync dummy error"; + std::atomic range_sync_called(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SpecialEnv::SStableFile::RangeSync", [&](void* arg) { + if (range_sync_called.fetch_add(1) == 0) { + Status* st = static_cast(arg); + *st = Status::IOError(io_error_msg); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(dbfull()->SetOptions(handles_[1], + { + {"disable_auto_compactions", "false"}, + })); + Status s = dbfull()->TEST_WaitForCompact(); + ASSERT_TRUE(s.IsIOError()); + ASSERT_STREQ(s.getState(), io_error_msg); + + // Following writes should fail as flush failed. + ASSERT_NOK(Put(1, "foo2", "bar3")); + ASSERT_EQ("bar", Get(1, "foo")); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ASSERT_GE(1, range_sync_called.load()); + + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_EQ("bar", Get(1, "foo")); +} + +TEST_F(DBIOFailureTest, FlushSstCloseError) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.error_if_exists = false; + options.paranoid_checks = true; + options.level0_file_num_compaction_trigger = 4; + options.memtable_factory.reset(test::NewSpecialSkipListFactory(2)); + + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + const char* io_error_msg = "close dummy error"; + std::atomic close_called(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SpecialEnv::SStableFile::Close", [&](void* arg) { + if (close_called.fetch_add(1) == 0) { + Status* st = static_cast(arg); + *st = Status::IOError(io_error_msg); + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put(1, "foo", "bar")); + ASSERT_OK(Put(1, "foo1", "bar1")); + ASSERT_OK(Put(1, "foo", "bar2")); + Status s = dbfull()->TEST_WaitForFlushMemTable(handles_[1]); + ASSERT_TRUE(s.IsIOError()); + ASSERT_STREQ(s.getState(), io_error_msg); + + // Following writes should fail as flush failed. + ASSERT_NOK(Put(1, "foo2", "bar3")); + ASSERT_EQ("bar2", Get(1, "foo")); + ASSERT_EQ("bar1", Get(1, "foo1")); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_EQ("bar2", Get(1, "foo")); + ASSERT_EQ("bar1", Get(1, "foo1")); +} + +TEST_F(DBIOFailureTest, CompactionSstCloseError) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.error_if_exists = false; + options.paranoid_checks = true; + options.level0_file_num_compaction_trigger = 2; + options.disable_auto_compactions = true; + + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(1, "foo", "bar")); + ASSERT_OK(Put(1, "foo2", "bar")); + ASSERT_OK(Flush(1)); + ASSERT_OK(Put(1, "foo", "bar2")); + ASSERT_OK(Put(1, "foo2", "bar")); + ASSERT_OK(Flush(1)); + ASSERT_OK(Put(1, "foo", "bar3")); + ASSERT_OK(Put(1, "foo2", "bar")); + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + const char* io_error_msg = "close dummy error"; + std::atomic close_called(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SpecialEnv::SStableFile::Close", [&](void* arg) { + if (close_called.fetch_add(1) == 0) { + Status* st = static_cast(arg); + *st = Status::IOError(io_error_msg); + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(dbfull()->SetOptions(handles_[1], + { + {"disable_auto_compactions", "false"}, + })); + Status s = dbfull()->TEST_WaitForCompact(); + ASSERT_TRUE(s.IsIOError()); + ASSERT_STREQ(s.getState(), io_error_msg); + + // Following writes should fail as compaction failed. + ASSERT_NOK(Put(1, "foo2", "bar3")); + ASSERT_EQ("bar3", Get(1, "foo")); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_EQ("bar3", Get(1, "foo")); +} + +TEST_F(DBIOFailureTest, FlushSstSyncError) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.error_if_exists = false; + options.paranoid_checks = true; + options.use_fsync = false; + options.level0_file_num_compaction_trigger = 4; + options.memtable_factory.reset(test::NewSpecialSkipListFactory(2)); + + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + const char* io_error_msg = "sync dummy error"; + std::atomic sync_called(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SpecialEnv::SStableFile::Sync", [&](void* arg) { + if (sync_called.fetch_add(1) == 0) { + Status* st = static_cast(arg); + *st = Status::IOError(io_error_msg); + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put(1, "foo", "bar")); + ASSERT_OK(Put(1, "foo1", "bar1")); + ASSERT_OK(Put(1, "foo", "bar2")); + Status s = dbfull()->TEST_WaitForFlushMemTable(handles_[1]); + ASSERT_TRUE(s.IsIOError()); + ASSERT_STREQ(s.getState(), io_error_msg); + + // Following writes should fail as flush failed. + ASSERT_NOK(Put(1, "foo2", "bar3")); + ASSERT_EQ("bar2", Get(1, "foo")); + ASSERT_EQ("bar1", Get(1, "foo1")); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_EQ("bar2", Get(1, "foo")); + ASSERT_EQ("bar1", Get(1, "foo1")); +} + +TEST_F(DBIOFailureTest, CompactionSstSyncError) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.error_if_exists = false; + options.paranoid_checks = true; + options.level0_file_num_compaction_trigger = 2; + options.disable_auto_compactions = true; + options.use_fsync = false; + + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(1, "foo", "bar")); + ASSERT_OK(Put(1, "foo2", "bar")); + ASSERT_OK(Flush(1)); + ASSERT_OK(Put(1, "foo", "bar2")); + ASSERT_OK(Put(1, "foo2", "bar")); + ASSERT_OK(Flush(1)); + ASSERT_OK(Put(1, "foo", "bar3")); + ASSERT_OK(Put(1, "foo2", "bar")); + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + const char* io_error_msg = "sync dummy error"; + std::atomic sync_called(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SpecialEnv::SStableFile::Sync", [&](void* arg) { + if (sync_called.fetch_add(1) == 0) { + Status* st = static_cast(arg); + *st = Status::IOError(io_error_msg); + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(dbfull()->SetOptions(handles_[1], + { + {"disable_auto_compactions", "false"}, + })); + Status s = dbfull()->TEST_WaitForCompact(); + ASSERT_TRUE(s.IsIOError()); + ASSERT_STREQ(s.getState(), io_error_msg); + + // Following writes should fail as compaction failed. + ASSERT_NOK(Put(1, "foo2", "bar3")); + ASSERT_EQ("bar3", Get(1, "foo")); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_EQ("bar3", Get(1, "foo")); +} +#endif // !(defined NDEBUG) || !defined(OS_WIN) +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_iter.cc b/librocksdb-sys/rocksdb/db/db_iter.cc new file mode 100644 index 0000000..3d980c8 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_iter.cc @@ -0,0 +1,1708 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/db_iter.h" + +#include +#include +#include + +#include "db/dbformat.h" +#include "db/merge_context.h" +#include "db/merge_helper.h" +#include "db/pinned_iterators_manager.h" +#include "db/wide/wide_column_serialization.h" +#include "file/filename.h" +#include "logging/logging.h" +#include "memory/arena.h" +#include "monitoring/perf_context_imp.h" +#include "rocksdb/env.h" +#include "rocksdb/iterator.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/options.h" +#include "rocksdb/system_clock.h" +#include "table/internal_iterator.h" +#include "table/iterator_wrapper.h" +#include "trace_replay/trace_replay.h" +#include "util/mutexlock.h" +#include "util/string_util.h" +#include "util/user_comparator_wrapper.h" + +namespace ROCKSDB_NAMESPACE { + +DBIter::DBIter(Env* _env, const ReadOptions& read_options, + const ImmutableOptions& ioptions, + const MutableCFOptions& mutable_cf_options, + const Comparator* cmp, InternalIterator* iter, + const Version* version, SequenceNumber s, bool arena_mode, + uint64_t max_sequential_skip_in_iterations, + ReadCallback* read_callback, DBImpl* db_impl, + ColumnFamilyData* cfd, bool expose_blob_index) + : prefix_extractor_(mutable_cf_options.prefix_extractor.get()), + env_(_env), + clock_(ioptions.clock), + logger_(ioptions.logger), + user_comparator_(cmp), + merge_operator_(ioptions.merge_operator.get()), + iter_(iter), + version_(version), + read_callback_(read_callback), + sequence_(s), + statistics_(ioptions.stats), + max_skip_(max_sequential_skip_in_iterations), + max_skippable_internal_keys_(read_options.max_skippable_internal_keys), + num_internal_keys_skipped_(0), + iterate_lower_bound_(read_options.iterate_lower_bound), + iterate_upper_bound_(read_options.iterate_upper_bound), + direction_(kForward), + valid_(false), + current_entry_is_merged_(false), + is_key_seqnum_zero_(false), + prefix_same_as_start_(mutable_cf_options.prefix_extractor + ? read_options.prefix_same_as_start + : false), + pin_thru_lifetime_(read_options.pin_data), + expect_total_order_inner_iter_(prefix_extractor_ == nullptr || + read_options.total_order_seek || + read_options.auto_prefix_mode), + read_tier_(read_options.read_tier), + fill_cache_(read_options.fill_cache), + verify_checksums_(read_options.verify_checksums), + expose_blob_index_(expose_blob_index), + is_blob_(false), + arena_mode_(arena_mode), + db_impl_(db_impl), + cfd_(cfd), + timestamp_ub_(read_options.timestamp), + timestamp_lb_(read_options.iter_start_ts), + timestamp_size_(timestamp_ub_ ? timestamp_ub_->size() : 0) { + RecordTick(statistics_, NO_ITERATOR_CREATED); + if (pin_thru_lifetime_) { + pinned_iters_mgr_.StartPinning(); + } + if (iter_.iter()) { + iter_.iter()->SetPinnedItersMgr(&pinned_iters_mgr_); + } + status_.PermitUncheckedError(); + assert(timestamp_size_ == + user_comparator_.user_comparator()->timestamp_size()); +} + +Status DBIter::GetProperty(std::string prop_name, std::string* prop) { + if (prop == nullptr) { + return Status::InvalidArgument("prop is nullptr"); + } + if (prop_name == "rocksdb.iterator.super-version-number") { + // First try to pass the value returned from inner iterator. + return iter_.iter()->GetProperty(prop_name, prop); + } else if (prop_name == "rocksdb.iterator.is-key-pinned") { + if (valid_) { + *prop = (pin_thru_lifetime_ && saved_key_.IsKeyPinned()) ? "1" : "0"; + } else { + *prop = "Iterator is not valid."; + } + return Status::OK(); + } else if (prop_name == "rocksdb.iterator.internal-key") { + *prop = saved_key_.GetUserKey().ToString(); + return Status::OK(); + } + return Status::InvalidArgument("Unidentified property."); +} + +bool DBIter::ParseKey(ParsedInternalKey* ikey) { + Status s = ParseInternalKey(iter_.key(), ikey, false /* log_err_key */); + if (!s.ok()) { + status_ = Status::Corruption("In DBIter: ", s.getState()); + valid_ = false; + ROCKS_LOG_ERROR(logger_, "In DBIter: %s", status_.getState()); + return false; + } else { + return true; + } +} + +void DBIter::Next() { + assert(valid_); + assert(status_.ok()); + + PERF_COUNTER_ADD(iter_next_count, 1); + PERF_CPU_TIMER_GUARD(iter_next_cpu_nanos, clock_); + // Release temporarily pinned blocks from last operation + ReleaseTempPinnedData(); + ResetBlobValue(); + ResetValueAndColumns(); + local_stats_.skip_count_ += num_internal_keys_skipped_; + local_stats_.skip_count_--; + num_internal_keys_skipped_ = 0; + bool ok = true; + if (direction_ == kReverse) { + is_key_seqnum_zero_ = false; + if (!ReverseToForward()) { + ok = false; + } + } else if (!current_entry_is_merged_) { + // If the current value is not a merge, the iter position is the + // current key, which is already returned. We can safely issue a + // Next() without checking the current key. + // If the current key is a merge, very likely iter already points + // to the next internal position. + assert(iter_.Valid()); + iter_.Next(); + PERF_COUNTER_ADD(internal_key_skipped_count, 1); + } + + local_stats_.next_count_++; + if (ok && iter_.Valid()) { + ClearSavedValue(); + + if (prefix_same_as_start_) { + assert(prefix_extractor_ != nullptr); + const Slice prefix = prefix_.GetUserKey(); + FindNextUserEntry(true /* skipping the current user key */, &prefix); + } else { + FindNextUserEntry(true /* skipping the current user key */, nullptr); + } + } else { + is_key_seqnum_zero_ = false; + valid_ = false; + } + if (statistics_ != nullptr && valid_) { + local_stats_.next_found_count_++; + local_stats_.bytes_read_ += (key().size() + value().size()); + } +} + +bool DBIter::SetBlobValueIfNeeded(const Slice& user_key, + const Slice& blob_index) { + assert(!is_blob_); + assert(blob_value_.empty()); + + if (expose_blob_index_) { // Stacked BlobDB implementation + is_blob_ = true; + return true; + } + + if (!version_) { + status_ = Status::Corruption("Encountered unexpected blob index."); + valid_ = false; + return false; + } + + // TODO: consider moving ReadOptions from ArenaWrappedDBIter to DBIter to + // avoid having to copy options back and forth. + // TODO: plumb Env::IOActivity + ReadOptions read_options; + read_options.read_tier = read_tier_; + read_options.fill_cache = fill_cache_; + read_options.verify_checksums = verify_checksums_; + + constexpr FilePrefetchBuffer* prefetch_buffer = nullptr; + constexpr uint64_t* bytes_read = nullptr; + + const Status s = version_->GetBlob(read_options, user_key, blob_index, + prefetch_buffer, &blob_value_, bytes_read); + + if (!s.ok()) { + status_ = s; + valid_ = false; + return false; + } + + is_blob_ = true; + return true; +} + +bool DBIter::SetValueAndColumnsFromEntity(Slice slice) { + assert(value_.empty()); + assert(wide_columns_.empty()); + + const Status s = WideColumnSerialization::Deserialize(slice, wide_columns_); + + if (!s.ok()) { + status_ = s; + valid_ = false; + return false; + } + + if (!wide_columns_.empty() && + wide_columns_[0].name() == kDefaultWideColumnName) { + value_ = wide_columns_[0].value(); + } + + return true; +} + +// PRE: saved_key_ has the current user key if skipping_saved_key +// POST: saved_key_ should have the next user key if valid_, +// if the current entry is a result of merge +// current_entry_is_merged_ => true +// saved_value_ => the merged value +// +// NOTE: In between, saved_key_ can point to a user key that has +// a delete marker or a sequence number higher than sequence_ +// saved_key_ MUST have a proper user_key before calling this function +// +// The prefix parameter, if not null, indicates that we need to iterate +// within the prefix, and the iterator needs to be made invalid, if no +// more entry for the prefix can be found. +bool DBIter::FindNextUserEntry(bool skipping_saved_key, const Slice* prefix) { + PERF_TIMER_GUARD(find_next_user_entry_time); + return FindNextUserEntryInternal(skipping_saved_key, prefix); +} + +// Actual implementation of DBIter::FindNextUserEntry() +bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key, + const Slice* prefix) { + // Loop until we hit an acceptable entry to yield + assert(iter_.Valid()); + assert(status_.ok()); + assert(direction_ == kForward); + current_entry_is_merged_ = false; + + // How many times in a row we have skipped an entry with user key less than + // or equal to saved_key_. We could skip these entries either because + // sequence numbers were too high or because skipping_saved_key = true. + // What saved_key_ contains throughout this method: + // - if skipping_saved_key : saved_key_ contains the key that we need + // to skip, and we haven't seen any keys greater + // than that, + // - if num_skipped > 0 : saved_key_ contains the key that we have skipped + // num_skipped times, and we haven't seen any keys + // greater than that, + // - none of the above : saved_key_ can contain anything, it doesn't + // matter. + uint64_t num_skipped = 0; + // For write unprepared, the target sequence number in reseek could be larger + // than the snapshot, and thus needs to be skipped again. This could result in + // an infinite loop of reseeks. To avoid that, we limit the number of reseeks + // to one. + bool reseek_done = false; + + do { + // Will update is_key_seqnum_zero_ as soon as we parsed the current key + // but we need to save the previous value to be used in the loop. + bool is_prev_key_seqnum_zero = is_key_seqnum_zero_; + if (!ParseKey(&ikey_)) { + is_key_seqnum_zero_ = false; + return false; + } + Slice user_key_without_ts = + StripTimestampFromUserKey(ikey_.user_key, timestamp_size_); + + is_key_seqnum_zero_ = (ikey_.sequence == 0); + + assert(iterate_upper_bound_ == nullptr || + iter_.UpperBoundCheckResult() != IterBoundCheck::kInbound || + user_comparator_.CompareWithoutTimestamp( + user_key_without_ts, /*a_has_ts=*/false, *iterate_upper_bound_, + /*b_has_ts=*/false) < 0); + if (iterate_upper_bound_ != nullptr && + iter_.UpperBoundCheckResult() != IterBoundCheck::kInbound && + user_comparator_.CompareWithoutTimestamp( + user_key_without_ts, /*a_has_ts=*/false, *iterate_upper_bound_, + /*b_has_ts=*/false) >= 0) { + break; + } + + assert(prefix == nullptr || prefix_extractor_ != nullptr); + if (prefix != nullptr && + prefix_extractor_->Transform(user_key_without_ts).compare(*prefix) != + 0) { + assert(prefix_same_as_start_); + break; + } + + if (TooManyInternalKeysSkipped()) { + return false; + } + + assert(ikey_.user_key.size() >= timestamp_size_); + Slice ts = timestamp_size_ > 0 ? ExtractTimestampFromUserKey( + ikey_.user_key, timestamp_size_) + : Slice(); + bool more_recent = false; + if (IsVisible(ikey_.sequence, ts, &more_recent)) { + // If the previous entry is of seqnum 0, the current entry will not + // possibly be skipped. This condition can potentially be relaxed to + // prev_key.seq <= ikey_.sequence. We are cautious because it will be more + // prone to bugs causing the same user key with the same sequence number. + // Note that with current timestamp implementation, the same user key can + // have different timestamps and zero sequence number on the bottommost + // level. This may change in the future. + if ((!is_prev_key_seqnum_zero || timestamp_size_ > 0) && + skipping_saved_key && + CompareKeyForSkip(ikey_.user_key, saved_key_.GetUserKey()) <= 0) { + num_skipped++; // skip this entry + PERF_COUNTER_ADD(internal_key_skipped_count, 1); + } else { + assert(!skipping_saved_key || + CompareKeyForSkip(ikey_.user_key, saved_key_.GetUserKey()) > 0); + num_skipped = 0; + reseek_done = false; + switch (ikey_.type) { + case kTypeDeletion: + case kTypeDeletionWithTimestamp: + case kTypeSingleDeletion: + // Arrange to skip all upcoming entries for this key since + // they are hidden by this deletion. + if (timestamp_lb_) { + saved_key_.SetInternalKey(ikey_); + valid_ = true; + return true; + } else { + saved_key_.SetUserKey( + ikey_.user_key, !pin_thru_lifetime_ || + !iter_.iter()->IsKeyPinned() /* copy */); + skipping_saved_key = true; + PERF_COUNTER_ADD(internal_delete_skipped_count, 1); + } + break; + case kTypeValue: + case kTypeBlobIndex: + case kTypeWideColumnEntity: + if (!iter_.PrepareValue()) { + assert(!iter_.status().ok()); + valid_ = false; + return false; + } + if (timestamp_lb_) { + saved_key_.SetInternalKey(ikey_); + } else { + saved_key_.SetUserKey( + ikey_.user_key, !pin_thru_lifetime_ || + !iter_.iter()->IsKeyPinned() /* copy */); + } + + if (ikey_.type == kTypeBlobIndex) { + if (!SetBlobValueIfNeeded(ikey_.user_key, iter_.value())) { + return false; + } + + SetValueAndColumnsFromPlain(expose_blob_index_ ? iter_.value() + : blob_value_); + } else if (ikey_.type == kTypeWideColumnEntity) { + if (!SetValueAndColumnsFromEntity(iter_.value())) { + return false; + } + } else { + assert(ikey_.type == kTypeValue); + SetValueAndColumnsFromPlain(iter_.value()); + } + + valid_ = true; + return true; + break; + case kTypeMerge: + if (!iter_.PrepareValue()) { + assert(!iter_.status().ok()); + valid_ = false; + return false; + } + saved_key_.SetUserKey( + ikey_.user_key, + !pin_thru_lifetime_ || !iter_.iter()->IsKeyPinned() /* copy */); + // By now, we are sure the current ikey is going to yield a value + current_entry_is_merged_ = true; + valid_ = true; + return MergeValuesNewToOld(); // Go to a different state machine + break; + default: + valid_ = false; + status_ = Status::Corruption( + "Unknown value type: " + + std::to_string(static_cast(ikey_.type))); + return false; + } + } + } else { + if (more_recent) { + PERF_COUNTER_ADD(internal_recent_skipped_count, 1); + } + + // This key was inserted after our snapshot was taken or skipped by + // timestamp range. If this happens too many times in a row for the same + // user key, we want to seek to the target sequence number. + int cmp = user_comparator_.CompareWithoutTimestamp( + ikey_.user_key, saved_key_.GetUserKey()); + if (cmp == 0 || (skipping_saved_key && cmp < 0)) { + num_skipped++; + } else { + saved_key_.SetUserKey( + ikey_.user_key, + !iter_.iter()->IsKeyPinned() || !pin_thru_lifetime_ /* copy */); + skipping_saved_key = false; + num_skipped = 0; + reseek_done = false; + } + } + + // If we have sequentially iterated via numerous equal keys, then it's + // better to seek so that we can avoid too many key comparisons. + // + // To avoid infinite loops, do not reseek if we have already attempted to + // reseek previously. + // + // TODO(lth): If we reseek to sequence number greater than ikey_.sequence, + // then it does not make sense to reseek as we would actually land further + // away from the desired key. There is opportunity for optimization here. + if (num_skipped > max_skip_ && !reseek_done) { + is_key_seqnum_zero_ = false; + num_skipped = 0; + reseek_done = true; + std::string last_key; + if (skipping_saved_key) { + // We're looking for the next user-key but all we see are the same + // user-key with decreasing sequence numbers. Fast forward to + // sequence number 0 and type deletion (the smallest type). + if (timestamp_size_ == 0) { + AppendInternalKey( + &last_key, + ParsedInternalKey(saved_key_.GetUserKey(), 0, kTypeDeletion)); + } else { + const std::string kTsMin(timestamp_size_, '\0'); + AppendInternalKeyWithDifferentTimestamp( + &last_key, + ParsedInternalKey(saved_key_.GetUserKey(), 0, kTypeDeletion), + kTsMin); + } + // Don't set skipping_saved_key = false because we may still see more + // user-keys equal to saved_key_. + } else { + // We saw multiple entries with this user key and sequence numbers + // higher than sequence_. Fast forward to sequence_. + // Note that this only covers a case when a higher key was overwritten + // many times since our snapshot was taken, not the case when a lot of + // different keys were inserted after our snapshot was taken. + if (timestamp_size_ == 0) { + AppendInternalKey( + &last_key, ParsedInternalKey(saved_key_.GetUserKey(), sequence_, + kValueTypeForSeek)); + } else { + AppendInternalKeyWithDifferentTimestamp( + &last_key, + ParsedInternalKey(saved_key_.GetUserKey(), sequence_, + kValueTypeForSeek), + *timestamp_ub_); + } + } + iter_.Seek(last_key); + RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION); + } else { + iter_.Next(); + } + } while (iter_.Valid()); + + valid_ = false; + return iter_.status().ok(); +} + +// Merge values of the same user key starting from the current iter_ position +// Scan from the newer entries to older entries. +// PRE: iter_.key() points to the first merge type entry +// saved_key_ stores the user key +// iter_.PrepareValue() has been called +// POST: saved_value_ has the merged value for the user key +// iter_ points to the next entry (or invalid) +bool DBIter::MergeValuesNewToOld() { + if (!merge_operator_) { + ROCKS_LOG_ERROR(logger_, "Options::merge_operator is null."); + status_ = Status::InvalidArgument("merge_operator_ must be set."); + valid_ = false; + return false; + } + + // Temporarily pin the blocks that hold merge operands + TempPinData(); + merge_context_.Clear(); + // Start the merge process by pushing the first operand + merge_context_.PushOperand( + iter_.value(), iter_.iter()->IsValuePinned() /* operand_pinned */); + PERF_COUNTER_ADD(internal_merge_count, 1); + + TEST_SYNC_POINT("DBIter::MergeValuesNewToOld:PushedFirstOperand"); + + ParsedInternalKey ikey; + for (iter_.Next(); iter_.Valid(); iter_.Next()) { + TEST_SYNC_POINT("DBIter::MergeValuesNewToOld:SteppedToNextOperand"); + if (!ParseKey(&ikey)) { + return false; + } + + if (!user_comparator_.EqualWithoutTimestamp(ikey.user_key, + saved_key_.GetUserKey())) { + // hit the next user key, stop right here + break; + } + if (kTypeDeletion == ikey.type || kTypeSingleDeletion == ikey.type || + kTypeDeletionWithTimestamp == ikey.type) { + // hit a delete with the same user key, stop right here + // iter_ is positioned after delete + iter_.Next(); + break; + } + if (!iter_.PrepareValue()) { + valid_ = false; + return false; + } + + if (kTypeValue == ikey.type) { + // hit a put, merge the put value with operands and store the + // final result in saved_value_. We are done! + const Slice val = iter_.value(); + if (!Merge(&val, ikey.user_key)) { + return false; + } + // iter_ is positioned after put + iter_.Next(); + if (!iter_.status().ok()) { + valid_ = false; + return false; + } + return true; + } else if (kTypeMerge == ikey.type) { + // hit a merge, add the value as an operand and run associative merge. + // when complete, add result to operands and continue. + merge_context_.PushOperand( + iter_.value(), iter_.iter()->IsValuePinned() /* operand_pinned */); + PERF_COUNTER_ADD(internal_merge_count, 1); + } else if (kTypeBlobIndex == ikey.type) { + if (expose_blob_index_) { + status_ = + Status::NotSupported("BlobDB does not support merge operator."); + valid_ = false; + return false; + } + // hit a put, merge the put value with operands and store the + // final result in saved_value_. We are done! + if (!SetBlobValueIfNeeded(ikey.user_key, iter_.value())) { + return false; + } + valid_ = true; + if (!Merge(&blob_value_, ikey.user_key)) { + return false; + } + + ResetBlobValue(); + + // iter_ is positioned after put + iter_.Next(); + if (!iter_.status().ok()) { + valid_ = false; + return false; + } + return true; + } else if (kTypeWideColumnEntity == ikey.type) { + if (!MergeEntity(iter_.value(), ikey.user_key)) { + return false; + } + + // iter_ is positioned after put + iter_.Next(); + if (!iter_.status().ok()) { + valid_ = false; + return false; + } + + return true; + } else { + valid_ = false; + status_ = Status::Corruption( + "Unrecognized value type: " + + std::to_string(static_cast(ikey.type))); + return false; + } + } + + if (!iter_.status().ok()) { + valid_ = false; + return false; + } + + // we either exhausted all internal keys under this user key, or hit + // a deletion marker. + // feed null as the existing value to the merge operator, such that + // client can differentiate this scenario and do things accordingly. + if (!Merge(nullptr, saved_key_.GetUserKey())) { + return false; + } + assert(status_.ok()); + return true; +} + +void DBIter::Prev() { + assert(valid_); + assert(status_.ok()); + + PERF_COUNTER_ADD(iter_prev_count, 1); + PERF_CPU_TIMER_GUARD(iter_prev_cpu_nanos, clock_); + ReleaseTempPinnedData(); + ResetBlobValue(); + ResetValueAndColumns(); + ResetInternalKeysSkippedCounter(); + bool ok = true; + if (direction_ == kForward) { + if (!ReverseToBackward()) { + ok = false; + } + } + if (ok) { + ClearSavedValue(); + + Slice prefix; + if (prefix_same_as_start_) { + assert(prefix_extractor_ != nullptr); + prefix = prefix_.GetUserKey(); + } + PrevInternal(prefix_same_as_start_ ? &prefix : nullptr); + } + + if (statistics_ != nullptr) { + local_stats_.prev_count_++; + if (valid_) { + local_stats_.prev_found_count_++; + local_stats_.bytes_read_ += (key().size() + value().size()); + } + } +} + +bool DBIter::ReverseToForward() { + assert(iter_.status().ok()); + + // When moving backwards, iter_ is positioned on _previous_ key, which may + // not exist or may have different prefix than the current key(). + // If that's the case, seek iter_ to current key. + if (!expect_total_order_inner_iter() || !iter_.Valid()) { + IterKey last_key; + ParsedInternalKey pikey(saved_key_.GetUserKey(), kMaxSequenceNumber, + kValueTypeForSeek); + if (timestamp_size_ > 0) { + // TODO: pre-create kTsMax. + const std::string kTsMax(timestamp_size_, '\xff'); + pikey.SetTimestamp(kTsMax); + } + last_key.SetInternalKey(pikey); + iter_.Seek(last_key.GetInternalKey()); + RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION); + } + + direction_ = kForward; + // Skip keys less than the current key() (a.k.a. saved_key_). + while (iter_.Valid()) { + ParsedInternalKey ikey; + if (!ParseKey(&ikey)) { + return false; + } + if (user_comparator_.Compare(ikey.user_key, saved_key_.GetUserKey()) >= 0) { + return true; + } + iter_.Next(); + } + + if (!iter_.status().ok()) { + valid_ = false; + return false; + } + + return true; +} + +// Move iter_ to the key before saved_key_. +bool DBIter::ReverseToBackward() { + assert(iter_.status().ok()); + + // When current_entry_is_merged_ is true, iter_ may be positioned on the next + // key, which may not exist or may have prefix different from current. + // If that's the case, seek to saved_key_. + if (current_entry_is_merged_ && + (!expect_total_order_inner_iter() || !iter_.Valid())) { + IterKey last_key; + // Using kMaxSequenceNumber and kValueTypeForSeek + // (not kValueTypeForSeekForPrev) to seek to a key strictly smaller + // than saved_key_. + last_key.SetInternalKey(ParsedInternalKey( + saved_key_.GetUserKey(), kMaxSequenceNumber, kValueTypeForSeek)); + if (!expect_total_order_inner_iter()) { + iter_.SeekForPrev(last_key.GetInternalKey()); + } else { + // Some iterators may not support SeekForPrev(), so we avoid using it + // when prefix seek mode is disabled. This is somewhat expensive + // (an extra Prev(), as well as an extra change of direction of iter_), + // so we may need to reconsider it later. + iter_.Seek(last_key.GetInternalKey()); + if (!iter_.Valid() && iter_.status().ok()) { + iter_.SeekToLast(); + } + } + RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION); + } + + direction_ = kReverse; + return FindUserKeyBeforeSavedKey(); +} + +void DBIter::PrevInternal(const Slice* prefix) { + while (iter_.Valid()) { + saved_key_.SetUserKey( + ExtractUserKey(iter_.key()), + !iter_.iter()->IsKeyPinned() || !pin_thru_lifetime_ /* copy */); + + assert(prefix == nullptr || prefix_extractor_ != nullptr); + if (prefix != nullptr && + prefix_extractor_ + ->Transform(StripTimestampFromUserKey(saved_key_.GetUserKey(), + timestamp_size_)) + .compare(*prefix) != 0) { + assert(prefix_same_as_start_); + // Current key does not have the same prefix as start + valid_ = false; + return; + } + + assert(iterate_lower_bound_ == nullptr || iter_.MayBeOutOfLowerBound() || + user_comparator_.CompareWithoutTimestamp( + saved_key_.GetUserKey(), /*a_has_ts=*/true, + *iterate_lower_bound_, /*b_has_ts=*/false) >= 0); + if (iterate_lower_bound_ != nullptr && iter_.MayBeOutOfLowerBound() && + user_comparator_.CompareWithoutTimestamp( + saved_key_.GetUserKey(), /*a_has_ts=*/true, *iterate_lower_bound_, + /*b_has_ts=*/false) < 0) { + // We've iterated earlier than the user-specified lower bound. + valid_ = false; + return; + } + + if (!FindValueForCurrentKey()) { // assigns valid_ + return; + } + + // Whether or not we found a value for current key, we need iter_ to end up + // on a smaller key. + if (!FindUserKeyBeforeSavedKey()) { + return; + } + + if (valid_) { + // Found the value. + return; + } + + if (TooManyInternalKeysSkipped(false)) { + return; + } + } + + // We haven't found any key - iterator is not valid + valid_ = false; +} + +// Used for backwards iteration. +// Looks at the entries with user key saved_key_ and finds the most up-to-date +// value for it, or executes a merge, or determines that the value was deleted. +// Sets valid_ to true if the value is found and is ready to be presented to +// the user through value(). +// Sets valid_ to false if the value was deleted, and we should try another key. +// Returns false if an error occurred, and !status().ok() and !valid_. +// +// PRE: iter_ is positioned on the last entry with user key equal to saved_key_. +// POST: iter_ is positioned on one of the entries equal to saved_key_, or on +// the entry just before them, or on the entry just after them. +bool DBIter::FindValueForCurrentKey() { + assert(iter_.Valid()); + merge_context_.Clear(); + current_entry_is_merged_ = false; + // last entry before merge (could be kTypeDeletion, + // kTypeDeletionWithTimestamp, kTypeSingleDeletion, kTypeValue, + // kTypeBlobIndex, or kTypeWideColumnEntity) + ValueType last_not_merge_type = kTypeDeletion; + ValueType last_key_entry_type = kTypeDeletion; + + // If false, it indicates that we have not seen any valid entry, even though + // last_key_entry_type is initialized to kTypeDeletion. + bool valid_entry_seen = false; + + // Temporarily pin blocks that hold (merge operands / the value) + ReleaseTempPinnedData(); + TempPinData(); + size_t num_skipped = 0; + while (iter_.Valid()) { + ParsedInternalKey ikey; + if (!ParseKey(&ikey)) { + return false; + } + + if (!user_comparator_.EqualWithoutTimestamp(ikey.user_key, + saved_key_.GetUserKey())) { + // Found a smaller user key, thus we are done with current user key. + break; + } + + assert(ikey.user_key.size() >= timestamp_size_); + Slice ts; + if (timestamp_size_ > 0) { + ts = Slice(ikey.user_key.data() + ikey.user_key.size() - timestamp_size_, + timestamp_size_); + } + + bool visible = IsVisible(ikey.sequence, ts); + if (!visible && + (timestamp_lb_ == nullptr || + user_comparator_.CompareTimestamp(ts, *timestamp_ub_) > 0)) { + // Found an invisible version of the current user key, and it must have + // a higher sequence number or timestamp. Therefore, we are done with the + // current user key. + break; + } + + if (!ts.empty()) { + saved_timestamp_.assign(ts.data(), ts.size()); + } + + if (TooManyInternalKeysSkipped()) { + return false; + } + + // This user key has lots of entries. + // We're going from old to new, and it's taking too long. Let's do a Seek() + // and go from new to old. This helps when a key was overwritten many times. + if (num_skipped >= max_skip_) { + return FindValueForCurrentKeyUsingSeek(); + } + + if (!iter_.PrepareValue()) { + valid_ = false; + return false; + } + + if (timestamp_lb_ != nullptr) { + // Only needed when timestamp_lb_ is not null + [[maybe_unused]] const bool ret = ParseKey(&ikey_); + // Since the preceding ParseKey(&ikey) succeeds, so must this. + assert(ret); + saved_key_.SetInternalKey(ikey); + } else if (user_comparator_.Compare(ikey.user_key, + saved_key_.GetUserKey()) < 0) { + saved_key_.SetUserKey( + ikey.user_key, + !pin_thru_lifetime_ || !iter_.iter()->IsKeyPinned() /* copy */); + } + + valid_entry_seen = true; + last_key_entry_type = ikey.type; + switch (last_key_entry_type) { + case kTypeValue: + case kTypeBlobIndex: + case kTypeWideColumnEntity: + if (iter_.iter()->IsValuePinned()) { + pinned_value_ = iter_.value(); + } else { + valid_ = false; + status_ = Status::NotSupported( + "Backward iteration not supported if underlying iterator's value " + "cannot be pinned."); + } + merge_context_.Clear(); + last_not_merge_type = last_key_entry_type; + if (!status_.ok()) { + return false; + } + break; + case kTypeDeletion: + case kTypeDeletionWithTimestamp: + case kTypeSingleDeletion: + merge_context_.Clear(); + last_not_merge_type = last_key_entry_type; + PERF_COUNTER_ADD(internal_delete_skipped_count, 1); + break; + case kTypeMerge: { + assert(merge_operator_ != nullptr); + merge_context_.PushOperandBack( + iter_.value(), iter_.iter()->IsValuePinned() /* operand_pinned */); + PERF_COUNTER_ADD(internal_merge_count, 1); + } break; + default: + valid_ = false; + status_ = Status::Corruption( + "Unknown value type: " + + std::to_string(static_cast(last_key_entry_type))); + return false; + } + + PERF_COUNTER_ADD(internal_key_skipped_count, 1); + iter_.Prev(); + ++num_skipped; + + if (visible && timestamp_lb_ != nullptr) { + // If timestamp_lb_ is not nullptr, we do not have to look further for + // another internal key. We can return this current internal key. Yet we + // still keep the invariant that iter_ is positioned before the returned + // key. + break; + } + } + + if (!iter_.status().ok()) { + valid_ = false; + return false; + } + + if (!valid_entry_seen) { + // Since we haven't seen any valid entry, last_key_entry_type remains + // unchanged and the same as its initial value. + assert(last_key_entry_type == kTypeDeletion); + assert(last_not_merge_type == kTypeDeletion); + valid_ = false; + return true; + } + + if (timestamp_lb_ != nullptr) { + assert(last_key_entry_type == ikey_.type); + } + + switch (last_key_entry_type) { + case kTypeDeletion: + case kTypeDeletionWithTimestamp: + case kTypeSingleDeletion: + if (timestamp_lb_ == nullptr) { + valid_ = false; + } else { + valid_ = true; + } + return true; + case kTypeMerge: + current_entry_is_merged_ = true; + if (last_not_merge_type == kTypeDeletion || + last_not_merge_type == kTypeSingleDeletion || + last_not_merge_type == kTypeDeletionWithTimestamp) { + if (!Merge(nullptr, saved_key_.GetUserKey())) { + return false; + } + return true; + } else if (last_not_merge_type == kTypeBlobIndex) { + if (expose_blob_index_) { + status_ = + Status::NotSupported("BlobDB does not support merge operator."); + valid_ = false; + return false; + } + if (!SetBlobValueIfNeeded(saved_key_.GetUserKey(), pinned_value_)) { + return false; + } + valid_ = true; + if (!Merge(&blob_value_, saved_key_.GetUserKey())) { + return false; + } + + ResetBlobValue(); + + return true; + } else if (last_not_merge_type == kTypeWideColumnEntity) { + if (!MergeEntity(pinned_value_, saved_key_.GetUserKey())) { + return false; + } + + return true; + } else { + assert(last_not_merge_type == kTypeValue); + if (!Merge(&pinned_value_, saved_key_.GetUserKey())) { + return false; + } + return true; + } + break; + case kTypeValue: + SetValueAndColumnsFromPlain(pinned_value_); + + break; + case kTypeBlobIndex: + if (!SetBlobValueIfNeeded(saved_key_.GetUserKey(), pinned_value_)) { + return false; + } + + SetValueAndColumnsFromPlain(expose_blob_index_ ? pinned_value_ + : blob_value_); + + break; + case kTypeWideColumnEntity: + if (!SetValueAndColumnsFromEntity(pinned_value_)) { + return false; + } + break; + default: + valid_ = false; + status_ = Status::Corruption( + "Unknown value type: " + + std::to_string(static_cast(last_key_entry_type))); + return false; + } + valid_ = true; + return true; +} + +// This function is used in FindValueForCurrentKey. +// We use Seek() function instead of Prev() to find necessary value +// TODO: This is very similar to FindNextUserEntry() and MergeValuesNewToOld(). +// Would be nice to reuse some code. +bool DBIter::FindValueForCurrentKeyUsingSeek() { + // FindValueForCurrentKey will enable pinning before calling + // FindValueForCurrentKeyUsingSeek() + assert(pinned_iters_mgr_.PinningEnabled()); + std::string last_key; + if (0 == timestamp_size_) { + AppendInternalKey(&last_key, + ParsedInternalKey(saved_key_.GetUserKey(), sequence_, + kValueTypeForSeek)); + } else { + AppendInternalKeyWithDifferentTimestamp( + &last_key, + ParsedInternalKey(saved_key_.GetUserKey(), sequence_, + kValueTypeForSeek), + timestamp_lb_ == nullptr ? *timestamp_ub_ : *timestamp_lb_); + } + iter_.Seek(last_key); + RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION); + + // In case read_callback presents, the value we seek to may not be visible. + // Find the next value that's visible. + ParsedInternalKey ikey; + + while (true) { + if (!iter_.Valid()) { + valid_ = false; + return iter_.status().ok(); + } + + if (!ParseKey(&ikey)) { + return false; + } + assert(ikey.user_key.size() >= timestamp_size_); + Slice ts; + if (timestamp_size_ > 0) { + ts = Slice(ikey.user_key.data() + ikey.user_key.size() - timestamp_size_, + timestamp_size_); + } + + if (!user_comparator_.EqualWithoutTimestamp(ikey.user_key, + saved_key_.GetUserKey())) { + // No visible values for this key, even though FindValueForCurrentKey() + // has seen some. This is possible if we're using a tailing iterator, and + // the entries were discarded in a compaction. + valid_ = false; + return true; + } + + if (IsVisible(ikey.sequence, ts)) { + break; + } + + iter_.Next(); + } + + if (ikey.type == kTypeDeletion || ikey.type == kTypeSingleDeletion || + kTypeDeletionWithTimestamp == ikey.type) { + if (timestamp_lb_ == nullptr) { + valid_ = false; + } else { + valid_ = true; + saved_key_.SetInternalKey(ikey); + } + return true; + } + if (!iter_.PrepareValue()) { + valid_ = false; + return false; + } + if (timestamp_size_ > 0) { + Slice ts = ExtractTimestampFromUserKey(ikey.user_key, timestamp_size_); + saved_timestamp_.assign(ts.data(), ts.size()); + } + if (ikey.type == kTypeValue || ikey.type == kTypeBlobIndex || + ikey.type == kTypeWideColumnEntity) { + assert(iter_.iter()->IsValuePinned()); + pinned_value_ = iter_.value(); + if (ikey.type == kTypeBlobIndex) { + if (!SetBlobValueIfNeeded(ikey.user_key, pinned_value_)) { + return false; + } + + SetValueAndColumnsFromPlain(expose_blob_index_ ? pinned_value_ + : blob_value_); + } else if (ikey.type == kTypeWideColumnEntity) { + if (!SetValueAndColumnsFromEntity(pinned_value_)) { + return false; + } + } else { + assert(ikey.type == kTypeValue); + SetValueAndColumnsFromPlain(pinned_value_); + } + + if (timestamp_lb_ != nullptr) { + saved_key_.SetInternalKey(ikey); + } + + valid_ = true; + return true; + } + + // kTypeMerge. We need to collect all kTypeMerge values and save them + // in operands + assert(ikey.type == kTypeMerge); + current_entry_is_merged_ = true; + merge_context_.Clear(); + merge_context_.PushOperand( + iter_.value(), iter_.iter()->IsValuePinned() /* operand_pinned */); + PERF_COUNTER_ADD(internal_merge_count, 1); + + while (true) { + iter_.Next(); + + if (!iter_.Valid()) { + if (!iter_.status().ok()) { + valid_ = false; + return false; + } + break; + } + if (!ParseKey(&ikey)) { + return false; + } + if (!user_comparator_.EqualWithoutTimestamp(ikey.user_key, + saved_key_.GetUserKey())) { + break; + } + if (ikey.type == kTypeDeletion || ikey.type == kTypeSingleDeletion || + ikey.type == kTypeDeletionWithTimestamp) { + break; + } + if (!iter_.PrepareValue()) { + valid_ = false; + return false; + } + + if (ikey.type == kTypeValue) { + const Slice val = iter_.value(); + if (!Merge(&val, saved_key_.GetUserKey())) { + return false; + } + return true; + } else if (ikey.type == kTypeMerge) { + merge_context_.PushOperand( + iter_.value(), iter_.iter()->IsValuePinned() /* operand_pinned */); + PERF_COUNTER_ADD(internal_merge_count, 1); + } else if (ikey.type == kTypeBlobIndex) { + if (expose_blob_index_) { + status_ = + Status::NotSupported("BlobDB does not support merge operator."); + valid_ = false; + return false; + } + if (!SetBlobValueIfNeeded(ikey.user_key, iter_.value())) { + return false; + } + valid_ = true; + if (!Merge(&blob_value_, saved_key_.GetUserKey())) { + return false; + } + + ResetBlobValue(); + + return true; + } else if (ikey.type == kTypeWideColumnEntity) { + if (!MergeEntity(iter_.value(), saved_key_.GetUserKey())) { + return false; + } + + return true; + } else { + valid_ = false; + status_ = Status::Corruption( + "Unknown value type: " + + std::to_string(static_cast(ikey.type))); + return false; + } + } + + if (!Merge(nullptr, saved_key_.GetUserKey())) { + return false; + } + + // Make sure we leave iter_ in a good state. If it's valid and we don't care + // about prefixes, that's already good enough. Otherwise it needs to be + // seeked to the current key. + if (!expect_total_order_inner_iter() || !iter_.Valid()) { + if (!expect_total_order_inner_iter()) { + iter_.SeekForPrev(last_key); + } else { + iter_.Seek(last_key); + if (!iter_.Valid() && iter_.status().ok()) { + iter_.SeekToLast(); + } + } + RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION); + } + + valid_ = true; + return true; +} + +bool DBIter::Merge(const Slice* val, const Slice& user_key) { + // `op_failure_scope` (an output parameter) is not provided (set to nullptr) + // since a failure must be propagated regardless of its value. + Status s = MergeHelper::TimedFullMerge( + merge_operator_, user_key, val, merge_context_.GetOperands(), + &saved_value_, logger_, statistics_, clock_, &pinned_value_, + /* update_num_ops_stats */ true, + /* op_failure_scope */ nullptr); + if (!s.ok()) { + valid_ = false; + status_ = s; + return false; + } + + SetValueAndColumnsFromPlain(pinned_value_.data() ? pinned_value_ + : saved_value_); + + valid_ = true; + return true; +} + +bool DBIter::MergeEntity(const Slice& entity, const Slice& user_key) { + // `op_failure_scope` (an output parameter) is not provided (set to nullptr) + // since a failure must be propagated regardless of its value. + Status s = MergeHelper::TimedFullMergeWithEntity( + merge_operator_, user_key, entity, merge_context_.GetOperands(), + &saved_value_, logger_, statistics_, clock_, + /* update_num_ops_stats */ true, + /* op_failure_scope */ nullptr); + if (!s.ok()) { + valid_ = false; + status_ = s; + return false; + } + + if (!SetValueAndColumnsFromEntity(saved_value_)) { + return false; + } + + valid_ = true; + return true; +} + +// Move backwards until the key smaller than saved_key_. +// Changes valid_ only if return value is false. +bool DBIter::FindUserKeyBeforeSavedKey() { + assert(status_.ok()); + size_t num_skipped = 0; + while (iter_.Valid()) { + ParsedInternalKey ikey; + if (!ParseKey(&ikey)) { + return false; + } + + if (CompareKeyForSkip(ikey.user_key, saved_key_.GetUserKey()) < 0) { + return true; + } + + if (TooManyInternalKeysSkipped()) { + return false; + } + + assert(ikey.sequence != kMaxSequenceNumber); + assert(ikey.user_key.size() >= timestamp_size_); + Slice ts; + if (timestamp_size_ > 0) { + ts = Slice(ikey.user_key.data() + ikey.user_key.size() - timestamp_size_, + timestamp_size_); + } + if (!IsVisible(ikey.sequence, ts)) { + PERF_COUNTER_ADD(internal_recent_skipped_count, 1); + } else { + PERF_COUNTER_ADD(internal_key_skipped_count, 1); + } + + if (num_skipped >= max_skip_) { + num_skipped = 0; + IterKey last_key; + ParsedInternalKey pikey(saved_key_.GetUserKey(), kMaxSequenceNumber, + kValueTypeForSeek); + if (timestamp_size_ > 0) { + // TODO: pre-create kTsMax. + const std::string kTsMax(timestamp_size_, '\xff'); + pikey.SetTimestamp(kTsMax); + } + last_key.SetInternalKey(pikey); + // It would be more efficient to use SeekForPrev() here, but some + // iterators may not support it. + iter_.Seek(last_key.GetInternalKey()); + RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION); + if (!iter_.Valid()) { + break; + } + } else { + ++num_skipped; + } + + iter_.Prev(); + } + + if (!iter_.status().ok()) { + valid_ = false; + return false; + } + + return true; +} + +bool DBIter::TooManyInternalKeysSkipped(bool increment) { + if ((max_skippable_internal_keys_ > 0) && + (num_internal_keys_skipped_ > max_skippable_internal_keys_)) { + valid_ = false; + status_ = Status::Incomplete("Too many internal keys skipped."); + return true; + } else if (increment) { + num_internal_keys_skipped_++; + } + return false; +} + +bool DBIter::IsVisible(SequenceNumber sequence, const Slice& ts, + bool* more_recent) { + // Remember that comparator orders preceding timestamp as larger. + // TODO(yanqin): support timestamp in read_callback_. + bool visible_by_seq = (read_callback_ == nullptr) + ? sequence <= sequence_ + : read_callback_->IsVisible(sequence); + + bool visible_by_ts = + (timestamp_ub_ == nullptr || + user_comparator_.CompareTimestamp(ts, *timestamp_ub_) <= 0) && + (timestamp_lb_ == nullptr || + user_comparator_.CompareTimestamp(ts, *timestamp_lb_) >= 0); + + if (more_recent) { + *more_recent = !visible_by_seq; + } + return visible_by_seq && visible_by_ts; +} + +void DBIter::SetSavedKeyToSeekTarget(const Slice& target) { + is_key_seqnum_zero_ = false; + SequenceNumber seq = sequence_; + saved_key_.Clear(); + saved_key_.SetInternalKey(target, seq, kValueTypeForSeek, timestamp_ub_); + + if (iterate_lower_bound_ != nullptr && + user_comparator_.CompareWithoutTimestamp( + saved_key_.GetUserKey(), /*a_has_ts=*/true, *iterate_lower_bound_, + /*b_has_ts=*/false) < 0) { + // Seek key is smaller than the lower bound. + saved_key_.Clear(); + saved_key_.SetInternalKey(*iterate_lower_bound_, seq, kValueTypeForSeek, + timestamp_ub_); + } +} + +void DBIter::SetSavedKeyToSeekForPrevTarget(const Slice& target) { + is_key_seqnum_zero_ = false; + saved_key_.Clear(); + // now saved_key is used to store internal key. + saved_key_.SetInternalKey(target, 0 /* sequence_number */, + kValueTypeForSeekForPrev, timestamp_ub_); + + if (timestamp_size_ > 0) { + const std::string kTsMin(timestamp_size_, '\0'); + Slice ts = kTsMin; + saved_key_.UpdateInternalKey( + /*seq=*/0, kValueTypeForSeekForPrev, + timestamp_lb_ == nullptr ? &ts : timestamp_lb_); + } + + if (iterate_upper_bound_ != nullptr && + user_comparator_.CompareWithoutTimestamp( + saved_key_.GetUserKey(), /*a_has_ts=*/true, *iterate_upper_bound_, + /*b_has_ts=*/false) >= 0) { + saved_key_.Clear(); + saved_key_.SetInternalKey(*iterate_upper_bound_, kMaxSequenceNumber, + kValueTypeForSeekForPrev, timestamp_ub_); + if (timestamp_size_ > 0) { + const std::string kTsMax(timestamp_size_, '\xff'); + Slice ts = kTsMax; + saved_key_.UpdateInternalKey(kMaxSequenceNumber, kValueTypeForSeekForPrev, + &ts); + } + } +} + +void DBIter::Seek(const Slice& target) { + PERF_COUNTER_ADD(iter_seek_count, 1); + PERF_CPU_TIMER_GUARD(iter_seek_cpu_nanos, clock_); + StopWatch sw(clock_, statistics_, DB_SEEK); + + if (db_impl_ != nullptr && cfd_ != nullptr) { + // TODO: What do we do if this returns an error? + Slice lower_bound, upper_bound; + if (iterate_lower_bound_ != nullptr) { + lower_bound = *iterate_lower_bound_; + } else { + lower_bound = Slice(""); + } + if (iterate_upper_bound_ != nullptr) { + upper_bound = *iterate_upper_bound_; + } else { + upper_bound = Slice(""); + } + db_impl_->TraceIteratorSeek(cfd_->GetID(), target, lower_bound, upper_bound) + .PermitUncheckedError(); + } + + status_ = Status::OK(); + ReleaseTempPinnedData(); + ResetBlobValue(); + ResetValueAndColumns(); + ResetInternalKeysSkippedCounter(); + + // Seek the inner iterator based on the target key. + { + PERF_TIMER_GUARD(seek_internal_seek_time); + + SetSavedKeyToSeekTarget(target); + iter_.Seek(saved_key_.GetInternalKey()); + + RecordTick(statistics_, NUMBER_DB_SEEK); + } + if (!iter_.Valid()) { + valid_ = false; + return; + } + direction_ = kForward; + + // Now the inner iterator is placed to the target position. From there, + // we need to find out the next key that is visible to the user. + ClearSavedValue(); + if (prefix_same_as_start_) { + // The case where the iterator needs to be invalidated if it has exhausted + // keys within the same prefix of the seek key. + assert(prefix_extractor_ != nullptr); + Slice target_prefix = prefix_extractor_->Transform(target); + FindNextUserEntry(false /* not skipping saved_key */, + &target_prefix /* prefix */); + if (valid_) { + // Remember the prefix of the seek key for the future Next() call to + // check. + prefix_.SetUserKey(target_prefix); + } + } else { + FindNextUserEntry(false /* not skipping saved_key */, nullptr); + } + if (!valid_) { + return; + } + + // Updating stats and perf context counters. + if (statistics_ != nullptr) { + // Decrement since we don't want to count this key as skipped + RecordTick(statistics_, NUMBER_DB_SEEK_FOUND); + RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size()); + } + PERF_COUNTER_ADD(iter_read_bytes, key().size() + value().size()); +} + +void DBIter::SeekForPrev(const Slice& target) { + PERF_COUNTER_ADD(iter_seek_count, 1); + PERF_CPU_TIMER_GUARD(iter_seek_cpu_nanos, clock_); + StopWatch sw(clock_, statistics_, DB_SEEK); + + if (db_impl_ != nullptr && cfd_ != nullptr) { + // TODO: What do we do if this returns an error? + Slice lower_bound, upper_bound; + if (iterate_lower_bound_ != nullptr) { + lower_bound = *iterate_lower_bound_; + } else { + lower_bound = Slice(""); + } + if (iterate_upper_bound_ != nullptr) { + upper_bound = *iterate_upper_bound_; + } else { + upper_bound = Slice(""); + } + db_impl_ + ->TraceIteratorSeekForPrev(cfd_->GetID(), target, lower_bound, + upper_bound) + .PermitUncheckedError(); + } + + status_ = Status::OK(); + ReleaseTempPinnedData(); + ResetBlobValue(); + ResetValueAndColumns(); + ResetInternalKeysSkippedCounter(); + + // Seek the inner iterator based on the target key. + { + PERF_TIMER_GUARD(seek_internal_seek_time); + SetSavedKeyToSeekForPrevTarget(target); + iter_.SeekForPrev(saved_key_.GetInternalKey()); + RecordTick(statistics_, NUMBER_DB_SEEK); + } + if (!iter_.Valid()) { + valid_ = false; + return; + } + direction_ = kReverse; + + // Now the inner iterator is placed to the target position. From there, + // we need to find out the first key that is visible to the user in the + // backward direction. + ClearSavedValue(); + if (prefix_same_as_start_) { + // The case where the iterator needs to be invalidated if it has exhausted + // keys within the same prefix of the seek key. + assert(prefix_extractor_ != nullptr); + Slice target_prefix = prefix_extractor_->Transform(target); + PrevInternal(&target_prefix); + if (valid_) { + // Remember the prefix of the seek key for the future Prev() call to + // check. + prefix_.SetUserKey(target_prefix); + } + } else { + PrevInternal(nullptr); + } + + // Report stats and perf context. + if (statistics_ != nullptr && valid_) { + RecordTick(statistics_, NUMBER_DB_SEEK_FOUND); + RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size()); + PERF_COUNTER_ADD(iter_read_bytes, key().size() + value().size()); + } +} + +void DBIter::SeekToFirst() { + if (iterate_lower_bound_ != nullptr) { + Seek(*iterate_lower_bound_); + return; + } + PERF_COUNTER_ADD(iter_seek_count, 1); + PERF_CPU_TIMER_GUARD(iter_seek_cpu_nanos, clock_); + // Don't use iter_::Seek() if we set a prefix extractor + // because prefix seek will be used. + if (!expect_total_order_inner_iter()) { + max_skip_ = std::numeric_limits::max(); + } + status_ = Status::OK(); + // if iterator is empty, this status_ could be unchecked. + status_.PermitUncheckedError(); + direction_ = kForward; + ReleaseTempPinnedData(); + ResetBlobValue(); + ResetValueAndColumns(); + ResetInternalKeysSkippedCounter(); + ClearSavedValue(); + is_key_seqnum_zero_ = false; + + { + PERF_TIMER_GUARD(seek_internal_seek_time); + iter_.SeekToFirst(); + } + + RecordTick(statistics_, NUMBER_DB_SEEK); + if (iter_.Valid()) { + saved_key_.SetUserKey( + ExtractUserKey(iter_.key()), + !iter_.iter()->IsKeyPinned() || !pin_thru_lifetime_ /* copy */); + FindNextUserEntry(false /* not skipping saved_key */, + nullptr /* no prefix check */); + if (statistics_ != nullptr) { + if (valid_) { + RecordTick(statistics_, NUMBER_DB_SEEK_FOUND); + RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size()); + PERF_COUNTER_ADD(iter_read_bytes, key().size() + value().size()); + } + } + } else { + valid_ = false; + } + if (valid_ && prefix_same_as_start_) { + assert(prefix_extractor_ != nullptr); + prefix_.SetUserKey(prefix_extractor_->Transform( + StripTimestampFromUserKey(saved_key_.GetUserKey(), timestamp_size_))); + } +} + +void DBIter::SeekToLast() { + if (iterate_upper_bound_ != nullptr) { + // Seek to last key strictly less than ReadOptions.iterate_upper_bound. + SeekForPrev(*iterate_upper_bound_); +#ifndef NDEBUG + Slice k = Valid() ? key() : Slice(); + if (Valid() && timestamp_size_ > 0 && timestamp_lb_) { + k.remove_suffix(kNumInternalBytes + timestamp_size_); + } + assert(!Valid() || user_comparator_.CompareWithoutTimestamp( + k, /*a_has_ts=*/false, *iterate_upper_bound_, + /*b_has_ts=*/false) < 0); +#endif + return; + } + + PERF_COUNTER_ADD(iter_seek_count, 1); + PERF_CPU_TIMER_GUARD(iter_seek_cpu_nanos, clock_); + // Don't use iter_::Seek() if we set a prefix extractor + // because prefix seek will be used. + if (!expect_total_order_inner_iter()) { + max_skip_ = std::numeric_limits::max(); + } + status_ = Status::OK(); + // if iterator is empty, this status_ could be unchecked. + status_.PermitUncheckedError(); + direction_ = kReverse; + ReleaseTempPinnedData(); + ResetBlobValue(); + ResetValueAndColumns(); + ResetInternalKeysSkippedCounter(); + ClearSavedValue(); + is_key_seqnum_zero_ = false; + + { + PERF_TIMER_GUARD(seek_internal_seek_time); + iter_.SeekToLast(); + } + PrevInternal(nullptr); + if (statistics_ != nullptr) { + RecordTick(statistics_, NUMBER_DB_SEEK); + if (valid_) { + RecordTick(statistics_, NUMBER_DB_SEEK_FOUND); + RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size()); + PERF_COUNTER_ADD(iter_read_bytes, key().size() + value().size()); + } + } + if (valid_ && prefix_same_as_start_) { + assert(prefix_extractor_ != nullptr); + prefix_.SetUserKey(prefix_extractor_->Transform( + StripTimestampFromUserKey(saved_key_.GetUserKey(), timestamp_size_))); + } +} + +Iterator* NewDBIterator(Env* env, const ReadOptions& read_options, + const ImmutableOptions& ioptions, + const MutableCFOptions& mutable_cf_options, + const Comparator* user_key_comparator, + InternalIterator* internal_iter, const Version* version, + const SequenceNumber& sequence, + uint64_t max_sequential_skip_in_iterations, + ReadCallback* read_callback, DBImpl* db_impl, + ColumnFamilyData* cfd, bool expose_blob_index) { + DBIter* db_iter = + new DBIter(env, read_options, ioptions, mutable_cf_options, + user_key_comparator, internal_iter, version, sequence, false, + max_sequential_skip_in_iterations, read_callback, db_impl, cfd, + expose_blob_index); + return db_iter; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/db_iter.h b/librocksdb-sys/rocksdb/db/db_iter.h new file mode 100644 index 0000000..e1663bb --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_iter.h @@ -0,0 +1,411 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include +#include + +#include "db/db_impl/db_impl.h" +#include "db/range_del_aggregator.h" +#include "memory/arena.h" +#include "options/cf_options.h" +#include "rocksdb/db.h" +#include "rocksdb/iterator.h" +#include "rocksdb/wide_columns.h" +#include "table/iterator_wrapper.h" +#include "util/autovector.h" + +namespace ROCKSDB_NAMESPACE { +class Version; + +// This file declares the factory functions of DBIter, in its original form +// or a wrapped form with class ArenaWrappedDBIter, which is defined here. +// Class DBIter, which is declared and implemented inside db_iter.cc, is +// an iterator that converts internal keys (yielded by an InternalIterator) +// that were live at the specified sequence number into appropriate user +// keys. +// Each internal key consists of a user key, a sequence number, and a value +// type. DBIter deals with multiple key versions, tombstones, merge operands, +// etc, and exposes an Iterator. +// For example, DBIter may wrap following InternalIterator: +// user key: AAA value: v3 seqno: 100 type: Put +// user key: AAA value: v2 seqno: 97 type: Put +// user key: AAA value: v1 seqno: 95 type: Put +// user key: BBB value: v1 seqno: 90 type: Put +// user key: BBC value: N/A seqno: 98 type: Delete +// user key: BBC value: v1 seqno: 95 type: Put +// If the snapshot passed in is 102, then the DBIter is expected to +// expose the following iterator: +// key: AAA value: v3 +// key: BBB value: v1 +// If the snapshot passed in is 96, then it should expose: +// key: AAA value: v1 +// key: BBB value: v1 +// key: BBC value: v1 +// + +// Memtables and sstables that make the DB representation contain +// (userkey,seq,type) => uservalue entries. DBIter +// combines multiple entries for the same userkey found in the DB +// representation into a single entry while accounting for sequence +// numbers, deletion markers, overwrites, etc. +class DBIter final : public Iterator { + public: + // The following is grossly complicated. TODO: clean it up + // Which direction is the iterator currently moving? + // (1) When moving forward: + // (1a) if current_entry_is_merged_ = false, the internal iterator is + // positioned at the exact entry that yields this->key(), this->value() + // (1b) if current_entry_is_merged_ = true, the internal iterator is + // positioned immediately after the last entry that contributed to the + // current this->value(). That entry may or may not have key equal to + // this->key(). + // (2) When moving backwards, the internal iterator is positioned + // just before all entries whose user key == this->key(). + enum Direction : uint8_t { kForward, kReverse }; + + // LocalStatistics contain Statistics counters that will be aggregated per + // each iterator instance and then will be sent to the global statistics when + // the iterator is destroyed. + // + // The purpose of this approach is to avoid perf regression happening + // when multiple threads bump the atomic counters from a DBIter::Next(). + struct LocalStatistics { + explicit LocalStatistics() { ResetCounters(); } + + void ResetCounters() { + next_count_ = 0; + next_found_count_ = 0; + prev_count_ = 0; + prev_found_count_ = 0; + bytes_read_ = 0; + skip_count_ = 0; + } + + void BumpGlobalStatistics(Statistics* global_statistics) { + RecordTick(global_statistics, NUMBER_DB_NEXT, next_count_); + RecordTick(global_statistics, NUMBER_DB_NEXT_FOUND, next_found_count_); + RecordTick(global_statistics, NUMBER_DB_PREV, prev_count_); + RecordTick(global_statistics, NUMBER_DB_PREV_FOUND, prev_found_count_); + RecordTick(global_statistics, ITER_BYTES_READ, bytes_read_); + RecordTick(global_statistics, NUMBER_ITER_SKIP, skip_count_); + PERF_COUNTER_ADD(iter_read_bytes, bytes_read_); + ResetCounters(); + } + + // Map to Tickers::NUMBER_DB_NEXT + uint64_t next_count_; + // Map to Tickers::NUMBER_DB_NEXT_FOUND + uint64_t next_found_count_; + // Map to Tickers::NUMBER_DB_PREV + uint64_t prev_count_; + // Map to Tickers::NUMBER_DB_PREV_FOUND + uint64_t prev_found_count_; + // Map to Tickers::ITER_BYTES_READ + uint64_t bytes_read_; + // Map to Tickers::NUMBER_ITER_SKIP + uint64_t skip_count_; + }; + + DBIter(Env* _env, const ReadOptions& read_options, + const ImmutableOptions& ioptions, + const MutableCFOptions& mutable_cf_options, const Comparator* cmp, + InternalIterator* iter, const Version* version, SequenceNumber s, + bool arena_mode, uint64_t max_sequential_skip_in_iterations, + ReadCallback* read_callback, DBImpl* db_impl, ColumnFamilyData* cfd, + bool expose_blob_index); + + // No copying allowed + DBIter(const DBIter&) = delete; + void operator=(const DBIter&) = delete; + + ~DBIter() override { + // Release pinned data if any + if (pinned_iters_mgr_.PinningEnabled()) { + pinned_iters_mgr_.ReleasePinnedData(); + } + RecordTick(statistics_, NO_ITERATOR_DELETED); + ResetInternalKeysSkippedCounter(); + local_stats_.BumpGlobalStatistics(statistics_); + iter_.DeleteIter(arena_mode_); + } + void SetIter(InternalIterator* iter) { + assert(iter_.iter() == nullptr); + iter_.Set(iter); + iter_.iter()->SetPinnedItersMgr(&pinned_iters_mgr_); + } + + bool Valid() const override { +#ifdef ROCKSDB_ASSERT_STATUS_CHECKED + if (valid_) { + status_.PermitUncheckedError(); + } +#endif // ROCKSDB_ASSERT_STATUS_CHECKED + return valid_; + } + Slice key() const override { + assert(valid_); + if (timestamp_lb_) { + return saved_key_.GetInternalKey(); + } else { + const Slice ukey_and_ts = saved_key_.GetUserKey(); + return Slice(ukey_and_ts.data(), ukey_and_ts.size() - timestamp_size_); + } + } + Slice value() const override { + assert(valid_); + + return value_; + } + + const WideColumns& columns() const override { + assert(valid_); + + return wide_columns_; + } + + Status status() const override { + if (status_.ok()) { + return iter_.status(); + } else { + assert(!valid_); + return status_; + } + } + Slice timestamp() const override { + assert(valid_); + assert(timestamp_size_ > 0); + if (direction_ == kReverse) { + return saved_timestamp_; + } + const Slice ukey_and_ts = saved_key_.GetUserKey(); + assert(timestamp_size_ < ukey_and_ts.size()); + return ExtractTimestampFromUserKey(ukey_and_ts, timestamp_size_); + } + bool IsBlob() const { + assert(valid_); + return is_blob_; + } + + Status GetProperty(std::string prop_name, std::string* prop) override; + + void Next() final override; + void Prev() final override; + // 'target' does not contain timestamp, even if user timestamp feature is + // enabled. + void Seek(const Slice& target) final override; + void SeekForPrev(const Slice& target) final override; + void SeekToFirst() final override; + void SeekToLast() final override; + Env* env() const { return env_; } + void set_sequence(uint64_t s) { + sequence_ = s; + if (read_callback_) { + read_callback_->Refresh(s); + } + } + void set_valid(bool v) { valid_ = v; } + + private: + // For all methods in this block: + // PRE: iter_->Valid() && status_.ok() + // Return false if there was an error, and status() is non-ok, valid_ = false; + // in this case callers would usually stop what they were doing and return. + bool ReverseToForward(); + bool ReverseToBackward(); + // Set saved_key_ to the seek key to target, with proper sequence number set. + // It might get adjusted if the seek key is smaller than iterator lower bound. + // target does not have timestamp. + void SetSavedKeyToSeekTarget(const Slice& target); + // Set saved_key_ to the seek key to target, with proper sequence number set. + // It might get adjusted if the seek key is larger than iterator upper bound. + // target does not have timestamp. + void SetSavedKeyToSeekForPrevTarget(const Slice& target); + bool FindValueForCurrentKey(); + bool FindValueForCurrentKeyUsingSeek(); + bool FindUserKeyBeforeSavedKey(); + // If `skipping_saved_key` is true, the function will keep iterating until it + // finds a user key that is larger than `saved_key_`. + // If `prefix` is not null, the iterator needs to stop when all keys for the + // prefix are exhausted and the iterator is set to invalid. + bool FindNextUserEntry(bool skipping_saved_key, const Slice* prefix); + // Internal implementation of FindNextUserEntry(). + bool FindNextUserEntryInternal(bool skipping_saved_key, const Slice* prefix); + bool ParseKey(ParsedInternalKey* key); + bool MergeValuesNewToOld(); + + // If prefix is not null, we need to set the iterator to invalid if no more + // entry can be found within the prefix. + void PrevInternal(const Slice* prefix); + bool TooManyInternalKeysSkipped(bool increment = true); + bool IsVisible(SequenceNumber sequence, const Slice& ts, + bool* more_recent = nullptr); + + // Temporarily pin the blocks that we encounter until ReleaseTempPinnedData() + // is called + void TempPinData() { + if (!pin_thru_lifetime_) { + pinned_iters_mgr_.StartPinning(); + } + } + + // Release blocks pinned by TempPinData() + void ReleaseTempPinnedData() { + if (!pin_thru_lifetime_ && pinned_iters_mgr_.PinningEnabled()) { + pinned_iters_mgr_.ReleasePinnedData(); + } + } + + inline void ClearSavedValue() { + if (saved_value_.capacity() > 1048576) { + std::string empty; + swap(empty, saved_value_); + } else { + saved_value_.clear(); + } + } + + inline void ResetInternalKeysSkippedCounter() { + local_stats_.skip_count_ += num_internal_keys_skipped_; + if (valid_) { + local_stats_.skip_count_--; + } + num_internal_keys_skipped_ = 0; + } + + bool expect_total_order_inner_iter() { + assert(expect_total_order_inner_iter_ || prefix_extractor_ != nullptr); + return expect_total_order_inner_iter_; + } + + // If lower bound of timestamp is given by ReadOptions.iter_start_ts, we need + // to return versions of the same key. We cannot just skip if the key value + // is the same but timestamps are different but fall in timestamp range. + inline int CompareKeyForSkip(const Slice& a, const Slice& b) { + return timestamp_lb_ != nullptr + ? user_comparator_.Compare(a, b) + : user_comparator_.CompareWithoutTimestamp(a, b); + } + + // Retrieves the blob value for the specified user key using the given blob + // index when using the integrated BlobDB implementation. + bool SetBlobValueIfNeeded(const Slice& user_key, const Slice& blob_index); + + void ResetBlobValue() { + is_blob_ = false; + blob_value_.Reset(); + } + + void SetValueAndColumnsFromPlain(const Slice& slice) { + assert(value_.empty()); + assert(wide_columns_.empty()); + + value_ = slice; + wide_columns_.emplace_back(kDefaultWideColumnName, slice); + } + + bool SetValueAndColumnsFromEntity(Slice slice); + + void ResetValueAndColumns() { + value_.clear(); + wide_columns_.clear(); + } + + // If user-defined timestamp is enabled, `user_key` includes timestamp. + bool Merge(const Slice* val, const Slice& user_key); + bool MergeEntity(const Slice& entity, const Slice& user_key); + + const SliceTransform* prefix_extractor_; + Env* const env_; + SystemClock* clock_; + Logger* logger_; + UserComparatorWrapper user_comparator_; + const MergeOperator* const merge_operator_; + IteratorWrapper iter_; + const Version* version_; + ReadCallback* read_callback_; + // Max visible sequence number. It is normally the snapshot seq unless we have + // uncommitted data in db as in WriteUnCommitted. + SequenceNumber sequence_; + + IterKey saved_key_; + // Reusable internal key data structure. This is only used inside one function + // and should not be used across functions. Reusing this object can reduce + // overhead of calling construction of the function if creating it each time. + ParsedInternalKey ikey_; + std::string saved_value_; + Slice pinned_value_; + // for prefix seek mode to support prev() + PinnableSlice blob_value_; + // Value of the default column + Slice value_; + // All columns (i.e. name-value pairs) + WideColumns wide_columns_; + Statistics* statistics_; + uint64_t max_skip_; + uint64_t max_skippable_internal_keys_; + uint64_t num_internal_keys_skipped_; + const Slice* iterate_lower_bound_; + const Slice* iterate_upper_bound_; + + // The prefix of the seek key. It is only used when prefix_same_as_start_ + // is true and prefix extractor is not null. In Next() or Prev(), current keys + // will be checked against this prefix, so that the iterator can be + // invalidated if the keys in this prefix has been exhausted. Set it using + // SetUserKey() and use it using GetUserKey(). + IterKey prefix_; + + Status status_; + Direction direction_; + bool valid_; + bool current_entry_is_merged_; + // True if we know that the current entry's seqnum is 0. + // This information is used as that the next entry will be for another + // user key. + bool is_key_seqnum_zero_; + const bool prefix_same_as_start_; + // Means that we will pin all data blocks we read as long the Iterator + // is not deleted, will be true if ReadOptions::pin_data is true + const bool pin_thru_lifetime_; + // Expect the inner iterator to maintain a total order. + // prefix_extractor_ must be non-NULL if the value is false. + const bool expect_total_order_inner_iter_; + ReadTier read_tier_; + bool fill_cache_; + bool verify_checksums_; + // Whether the iterator is allowed to expose blob references. Set to true when + // the stacked BlobDB implementation is used, false otherwise. + bool expose_blob_index_; + bool is_blob_; + bool arena_mode_; + // List of operands for merge operator. + MergeContext merge_context_; + LocalStatistics local_stats_; + PinnedIteratorsManager pinned_iters_mgr_; + DBImpl* db_impl_; + ColumnFamilyData* cfd_; + const Slice* const timestamp_ub_; + const Slice* const timestamp_lb_; + const size_t timestamp_size_; + std::string saved_timestamp_; +}; + +// Return a new iterator that converts internal keys (yielded by +// "*internal_iter") that were live at the specified `sequence` number +// into appropriate user keys. +extern Iterator* NewDBIterator( + Env* env, const ReadOptions& read_options, const ImmutableOptions& ioptions, + const MutableCFOptions& mutable_cf_options, + const Comparator* user_key_comparator, InternalIterator* internal_iter, + const Version* version, const SequenceNumber& sequence, + uint64_t max_sequential_skip_in_iterations, ReadCallback* read_callback, + DBImpl* db_impl = nullptr, ColumnFamilyData* cfd = nullptr, + bool expose_blob_index = false); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/db_iter_stress_test.cc b/librocksdb-sys/rocksdb/db/db_iter_stress_test.cc new file mode 100644 index 0000000..872f7e6 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_iter_stress_test.cc @@ -0,0 +1,658 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/db_iter.h" +#include "db/dbformat.h" +#include "rocksdb/comparator.h" +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "test_util/testharness.h" +#include "util/random.h" +#include "util/string_util.h" +#include "utilities/merge_operators.h" + +#ifdef GFLAGS + +#include "util/gflags_compat.h" + +using GFLAGS_NAMESPACE::ParseCommandLineFlags; + +DEFINE_bool(verbose, false, + "Print huge, detailed trace. Intended for debugging failures."); + +#else + +void ParseCommandLineFlags(int*, char***, bool) {} +bool FLAGS_verbose = false; + +#endif + +namespace ROCKSDB_NAMESPACE { + +class DBIteratorStressTest : public testing::Test { + public: + Env* env_; + + DBIteratorStressTest() : env_(Env::Default()) {} +}; + +namespace { + +struct Entry { + std::string key; + ValueType type; // kTypeValue, kTypeDeletion, kTypeMerge + uint64_t sequence; + std::string ikey; // internal key, made from `key`, `sequence` and `type` + std::string value; + // If false, we'll pretend that this entry doesn't exist. + bool visible = true; + + bool operator<(const Entry& e) const { + if (key != e.key) return key < e.key; + return std::tie(sequence, type) > std::tie(e.sequence, e.type); + } +}; + +struct Data { + std::vector entries; + + // Indices in `entries` with `visible` = false. + std::vector hidden; + // Keys of entries whose `visible` changed since the last seek of iterators. + std::set recently_touched_keys; +}; + +struct StressTestIterator : public InternalIterator { + Data* data; + Random64* rnd; + InternalKeyComparator cmp; + + // Each operation will return error with this probability... + double error_probability = 0; + // ... and add/remove entries with this probability. + double mutation_probability = 0; + // The probability of adding vs removing entries will be chosen so that the + // amount of removed entries stays somewhat close to this number. + double target_hidden_fraction = 0; + // If true, print all mutations to stdout for debugging. + bool trace = false; + + int iter = -1; + Status status_; + + StressTestIterator(Data* _data, Random64* _rnd, const Comparator* _cmp) + : data(_data), rnd(_rnd), cmp(_cmp) {} + + bool Valid() const override { + if (iter >= 0 && iter < (int)data->entries.size()) { + assert(status_.ok()); + return true; + } + return false; + } + + Status status() const override { return status_; } + + bool MaybeFail() { + if (rnd->Next() >= + static_cast(std::numeric_limits::max()) * + error_probability) { + return false; + } + if (rnd->Next() % 2) { + status_ = Status::Incomplete("test"); + } else { + status_ = Status::IOError("test"); + } + if (trace) { + std::cout << "injecting " << status_.ToString() << std::endl; + } + iter = -1; + return true; + } + + void MaybeMutate() { + if (rnd->Next() >= + static_cast(std::numeric_limits::max()) * + mutation_probability) { + return; + } + do { + // If too many entries are hidden, hide less, otherwise hide more. + double hide_probability = + data->hidden.size() > data->entries.size() * target_hidden_fraction + ? 1. / 3 + : 2. / 3; + if (data->hidden.empty()) { + hide_probability = 1; + } + bool do_hide = rnd->Next() < + static_cast(std::numeric_limits::max()) * + hide_probability; + if (do_hide) { + // Hide a random entry. + size_t idx = rnd->Next() % data->entries.size(); + Entry& e = data->entries[idx]; + if (e.visible) { + if (trace) { + std::cout << "hiding idx " << idx << std::endl; + } + e.visible = false; + data->hidden.push_back(idx); + data->recently_touched_keys.insert(e.key); + } else { + // Already hidden. Let's go unhide something instead, just because + // it's easy and it doesn't really matter what we do. + do_hide = false; + } + } + if (!do_hide) { + // Unhide a random entry. + size_t hi = rnd->Next() % data->hidden.size(); + size_t idx = data->hidden[hi]; + if (trace) { + std::cout << "unhiding idx " << idx << std::endl; + } + Entry& e = data->entries[idx]; + assert(!e.visible); + e.visible = true; + data->hidden[hi] = data->hidden.back(); + data->hidden.pop_back(); + data->recently_touched_keys.insert(e.key); + } + } while (rnd->Next() % 3 != 0); // do 3 mutations on average + } + + void SkipForward() { + while (iter < (int)data->entries.size() && !data->entries[iter].visible) { + ++iter; + } + } + void SkipBackward() { + while (iter >= 0 && !data->entries[iter].visible) { + --iter; + } + } + + void SeekToFirst() override { + if (MaybeFail()) return; + MaybeMutate(); + + status_ = Status::OK(); + iter = 0; + SkipForward(); + } + void SeekToLast() override { + if (MaybeFail()) return; + MaybeMutate(); + + status_ = Status::OK(); + iter = (int)data->entries.size() - 1; + SkipBackward(); + } + + void Seek(const Slice& target) override { + if (MaybeFail()) return; + MaybeMutate(); + + status_ = Status::OK(); + // Binary search. + auto it = std::partition_point( + data->entries.begin(), data->entries.end(), + [&](const Entry& e) { return cmp.Compare(e.ikey, target) < 0; }); + iter = (int)(it - data->entries.begin()); + SkipForward(); + } + void SeekForPrev(const Slice& target) override { + if (MaybeFail()) return; + MaybeMutate(); + + status_ = Status::OK(); + // Binary search. + auto it = std::partition_point( + data->entries.begin(), data->entries.end(), + [&](const Entry& e) { return cmp.Compare(e.ikey, target) <= 0; }); + iter = (int)(it - data->entries.begin()); + --iter; + SkipBackward(); + } + + void Next() override { + assert(Valid()); + if (MaybeFail()) return; + MaybeMutate(); + ++iter; + SkipForward(); + } + void Prev() override { + assert(Valid()); + if (MaybeFail()) return; + MaybeMutate(); + --iter; + SkipBackward(); + } + + Slice key() const override { + assert(Valid()); + return data->entries[iter].ikey; + } + Slice value() const override { + assert(Valid()); + return data->entries[iter].value; + } + + bool IsKeyPinned() const override { return true; } + bool IsValuePinned() const override { return true; } +}; + +// A small reimplementation of DBIter, supporting only some of the features, +// and doing everything in O(log n). +// Skips all keys that are in recently_touched_keys. +struct ReferenceIterator { + Data* data; + uint64_t sequence; // ignore entries with sequence number below this + + bool valid = false; + std::string key; + std::string value; + + ReferenceIterator(Data* _data, uint64_t _sequence) + : data(_data), sequence(_sequence) {} + + bool Valid() const { return valid; } + + // Finds the first entry with key + // greater/less/greater-or-equal/less-or-equal than `key`, depending on + // arguments: if `skip`, inequality is strict; if `forward`, it's + // greater/greater-or-equal, otherwise less/less-or-equal. + // Sets `key` to the result. + // If no such key exists, returns false. Doesn't check `visible`. + bool FindNextKey(bool skip, bool forward) { + valid = false; + auto it = std::partition_point(data->entries.begin(), data->entries.end(), + [&](const Entry& e) { + if (forward != skip) { + return e.key < key; + } else { + return e.key <= key; + } + }); + if (forward) { + if (it != data->entries.end()) { + key = it->key; + return true; + } + } else { + if (it != data->entries.begin()) { + --it; + key = it->key; + return true; + } + } + return false; + } + + bool FindValueForCurrentKey() { + if (data->recently_touched_keys.count(key)) { + return false; + } + + // Find the first entry for the key. The caller promises that it exists. + auto it = std::partition_point(data->entries.begin(), data->entries.end(), + [&](const Entry& e) { + if (e.key != key) { + return e.key < key; + } + return e.sequence > sequence; + }); + + // Find the first visible entry. + for (;; ++it) { + if (it == data->entries.end()) { + return false; + } + Entry& e = *it; + if (e.key != key) { + return false; + } + assert(e.sequence <= sequence); + if (!e.visible) continue; + if (e.type == kTypeDeletion) { + return false; + } + if (e.type == kTypeValue) { + value = e.value; + valid = true; + return true; + } + assert(e.type == kTypeMerge); + break; + } + + // Collect merge operands. + std::vector operands; + for (; it != data->entries.end(); ++it) { + Entry& e = *it; + if (e.key != key) { + break; + } + assert(e.sequence <= sequence); + if (!e.visible) continue; + if (e.type == kTypeDeletion) { + break; + } + operands.push_back(e.value); + if (e.type == kTypeValue) { + break; + } + } + + // Do a merge. + value = operands.back().ToString(); + for (int i = (int)operands.size() - 2; i >= 0; --i) { + value.append(","); + value.append(operands[i].data(), operands[i].size()); + } + + valid = true; + return true; + } + + // Start at `key` and move until we encounter a valid value. + // `forward` defines the direction of movement. + // If `skip` is true, we're looking for key not equal to `key`. + void DoTheThing(bool skip, bool forward) { + while (FindNextKey(skip, forward) && !FindValueForCurrentKey()) { + skip = true; + } + } + + void Seek(const Slice& target) { + key = target.ToString(); + DoTheThing(false, true); + } + void SeekForPrev(const Slice& target) { + key = target.ToString(); + DoTheThing(false, false); + } + void SeekToFirst() { Seek(""); } + void SeekToLast() { + key = data->entries.back().key; + DoTheThing(false, false); + } + void Next() { + assert(Valid()); + DoTheThing(true, true); + } + void Prev() { + assert(Valid()); + DoTheThing(true, false); + } +}; + +} // anonymous namespace + +// Use an internal iterator that sometimes returns errors and sometimes +// adds/removes entries on the fly. Do random operations on a DBIter and +// check results. +// TODO: can be improved for more coverage: +// * Override IsKeyPinned() and IsValuePinned() to actually use +// PinnedIteratorManager and check that there's no use-after free. +// * Try different combinations of prefix_extractor, total_order_seek, +// prefix_same_as_start, iterate_lower_bound, iterate_upper_bound. +TEST_F(DBIteratorStressTest, StressTest) { + // We use a deterministic RNG, and everything happens in a single thread. + Random64 rnd(826909345792864532ll); + + auto gen_key = [&](int max_key) { + assert(max_key > 0); + int len = 0; + int a = max_key; + while (a) { + a /= 10; + ++len; + } + std::string s = std::to_string(rnd.Next() % static_cast(max_key)); + s.insert(0, len - (int)s.size(), '0'); + return s; + }; + + Options options; + options.merge_operator = MergeOperators::CreateFromStringId("stringappend"); + ReadOptions ropt; + + size_t num_matching = 0; + size_t num_at_end = 0; + size_t num_not_ok = 0; + size_t num_recently_removed = 0; + + // Number of iterations for each combination of parameters + // (there are ~250 of those). + // Tweak this to change the test run time. + // As of the time of writing, the test takes ~4 seconds for value of 5000. + const int num_iterations = 5000; + // Enable this to print all the operations for debugging. + bool trace = FLAGS_verbose; + + for (int num_entries : {5, 10, 100}) { + for (double key_space : {0.1, 1.0, 3.0}) { + for (ValueType prevalent_entry_type : + {kTypeValue, kTypeDeletion, kTypeMerge}) { + for (double error_probability : {0.01, 0.1}) { + for (double mutation_probability : {0.01, 0.5}) { + for (double target_hidden_fraction : {0.1, 0.5}) { + std::string trace_str = + "entries: " + std::to_string(num_entries) + + ", key_space: " + std::to_string(key_space) + + ", error_probability: " + std::to_string(error_probability) + + ", mutation_probability: " + + std::to_string(mutation_probability) + + ", target_hidden_fraction: " + + std::to_string(target_hidden_fraction); + SCOPED_TRACE(trace_str); + if (trace) { + std::cout << trace_str << std::endl; + } + + // Generate data. + Data data; + int max_key = (int)(num_entries * key_space) + 1; + for (int i = 0; i < num_entries; ++i) { + Entry e; + e.key = gen_key(max_key); + if (rnd.Next() % 10 != 0) { + e.type = prevalent_entry_type; + } else { + const ValueType types[] = {kTypeValue, kTypeDeletion, + kTypeMerge}; + e.type = + types[rnd.Next() % (sizeof(types) / sizeof(types[0]))]; + } + e.sequence = i; + e.value = "v" + std::to_string(i); + ParsedInternalKey internal_key(e.key, e.sequence, e.type); + AppendInternalKey(&e.ikey, internal_key); + + data.entries.push_back(e); + } + std::sort(data.entries.begin(), data.entries.end()); + if (trace) { + std::cout << "entries:"; + for (size_t i = 0; i < data.entries.size(); ++i) { + Entry& e = data.entries[i]; + std::cout << "\n idx " << i << ": \"" << e.key << "\": \"" + << e.value << "\" seq: " << e.sequence << " type: " + << (e.type == kTypeValue ? "val" + : e.type == kTypeDeletion ? "del" + : "merge"); + } + std::cout << std::endl; + } + + std::unique_ptr db_iter; + std::unique_ptr ref_iter; + for (int iteration = 0; iteration < num_iterations; ++iteration) { + SCOPED_TRACE(iteration); + // Create a new iterator every ~30 operations. + if (db_iter == nullptr || rnd.Next() % 30 == 0) { + uint64_t sequence = rnd.Next() % (data.entries.size() + 2); + ref_iter.reset(new ReferenceIterator(&data, sequence)); + if (trace) { + std::cout << "new iterator, seq: " << sequence << std::endl; + } + + auto internal_iter = + new StressTestIterator(&data, &rnd, BytewiseComparator()); + internal_iter->error_probability = error_probability; + internal_iter->mutation_probability = mutation_probability; + internal_iter->target_hidden_fraction = + target_hidden_fraction; + internal_iter->trace = trace; + db_iter.reset(NewDBIterator( + env_, ropt, ImmutableOptions(options), + MutableCFOptions(options), BytewiseComparator(), + internal_iter, nullptr /* version */, sequence, + options.max_sequential_skip_in_iterations, + nullptr /*read_callback*/)); + } + + // Do a random operation. It's important to do it on ref_it + // later than on db_iter to make sure ref_it sees the correct + // recently_touched_keys. + std::string old_key; + bool forward = rnd.Next() % 2 > 0; + // Do Next()/Prev() ~90% of the time. + bool seek = !ref_iter->Valid() || rnd.Next() % 10 == 0; + if (trace) { + std::cout << iteration << ": "; + } + + if (!seek) { + assert(db_iter->Valid()); + old_key = ref_iter->key; + if (trace) { + std::cout << (forward ? "Next" : "Prev") << std::endl; + } + + if (forward) { + db_iter->Next(); + ref_iter->Next(); + } else { + db_iter->Prev(); + ref_iter->Prev(); + } + } else { + data.recently_touched_keys.clear(); + // Do SeekToFirst less often than Seek. + if (rnd.Next() % 4 == 0) { + if (trace) { + std::cout << (forward ? "SeekToFirst" : "SeekToLast") + << std::endl; + } + + if (forward) { + old_key = ""; + db_iter->SeekToFirst(); + ref_iter->SeekToFirst(); + } else { + old_key = data.entries.back().key; + db_iter->SeekToLast(); + ref_iter->SeekToLast(); + } + } else { + old_key = gen_key(max_key); + if (trace) { + std::cout << (forward ? "Seek" : "SeekForPrev") << " \"" + << old_key << '"' << std::endl; + } + if (forward) { + db_iter->Seek(old_key); + ref_iter->Seek(old_key); + } else { + db_iter->SeekForPrev(old_key); + ref_iter->SeekForPrev(old_key); + } + } + } + + // Check the result. + if (db_iter->Valid()) { + ASSERT_TRUE(db_iter->status().ok()); + if (data.recently_touched_keys.count( + db_iter->key().ToString())) { + // Ended on a key that may have been mutated during the + // operation. Reference iterator skips such keys, so we + // can't check the exact result. + + // Check that the key moved in the right direction. + if (forward) { + if (seek) + ASSERT_GE(db_iter->key().ToString(), old_key); + else + ASSERT_GT(db_iter->key().ToString(), old_key); + } else { + if (seek) + ASSERT_LE(db_iter->key().ToString(), old_key); + else + ASSERT_LT(db_iter->key().ToString(), old_key); + } + + if (ref_iter->Valid()) { + // Check that DBIter didn't miss any non-mutated key. + if (forward) { + ASSERT_LT(db_iter->key().ToString(), ref_iter->key); + } else { + ASSERT_GT(db_iter->key().ToString(), ref_iter->key); + } + } + // Tell the next iteration of the loop to reseek the + // iterators. + ref_iter->valid = false; + + ++num_recently_removed; + } else { + ASSERT_TRUE(ref_iter->Valid()); + ASSERT_EQ(ref_iter->key, db_iter->key().ToString()); + ASSERT_EQ(ref_iter->value, db_iter->value()); + ++num_matching; + } + } else if (db_iter->status().ok()) { + ASSERT_FALSE(ref_iter->Valid()); + ++num_at_end; + } else { + // Non-ok status. Nothing to check here. + // Tell the next iteration of the loop to reseek the + // iterators. + ref_iter->valid = false; + ++num_not_ok; + } + } + } + } + } + } + } + } + + // Check that all cases were hit many times. + EXPECT_GT(num_matching, 10000); + EXPECT_GT(num_at_end, 10000); + EXPECT_GT(num_not_ok, 10000); + EXPECT_GT(num_recently_removed, 10000); + + std::cout << "stats:\n exact matches: " << num_matching + << "\n end reached: " << num_at_end + << "\n non-ok status: " << num_not_ok + << "\n mutated on the fly: " << num_recently_removed << std::endl; +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + ParseCommandLineFlags(&argc, &argv, true); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_iter_test.cc b/librocksdb-sys/rocksdb/db/db_iter_test.cc new file mode 100644 index 0000000..65290bf --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_iter_test.cc @@ -0,0 +1,3195 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/db_iter.h" + +#include +#include +#include +#include + +#include "db/dbformat.h" +#include "rocksdb/comparator.h" +#include "rocksdb/options.h" +#include "rocksdb/perf_context.h" +#include "rocksdb/slice.h" +#include "rocksdb/statistics.h" +#include "table/iterator_wrapper.h" +#include "table/merging_iterator.h" +#include "test_util/sync_point.h" +#include "test_util/testharness.h" +#include "util/string_util.h" +#include "utilities/merge_operators.h" + +namespace ROCKSDB_NAMESPACE { + +static uint64_t TestGetTickerCount(const Options& options, + Tickers ticker_type) { + return options.statistics->getTickerCount(ticker_type); +} + +class TestIterator : public InternalIterator { + public: + explicit TestIterator(const Comparator* comparator) + : initialized_(false), + valid_(false), + sequence_number_(0), + iter_(0), + cmp(comparator) { + data_.reserve(16); + } + + void AddPut(std::string argkey, std::string argvalue) { + Add(argkey, kTypeValue, argvalue); + } + + void AddDeletion(std::string argkey) { + Add(argkey, kTypeDeletion, std::string()); + } + + void AddSingleDeletion(std::string argkey) { + Add(argkey, kTypeSingleDeletion, std::string()); + } + + void AddMerge(std::string argkey, std::string argvalue) { + Add(argkey, kTypeMerge, argvalue); + } + + void Add(std::string argkey, ValueType type, std::string argvalue) { + Add(argkey, type, argvalue, sequence_number_++); + } + + void Add(std::string argkey, ValueType type, std::string argvalue, + size_t seq_num, bool update_iter = false) { + valid_ = true; + ParsedInternalKey internal_key(argkey, seq_num, type); + data_.push_back( + std::pair(std::string(), argvalue)); + AppendInternalKey(&data_.back().first, internal_key); + if (update_iter && valid_ && cmp.Compare(data_.back().first, key()) < 0) { + // insert a key smaller than current key + Finish(); + // data_[iter_] is not anymore the current element of the iterator. + // Increment it to reposition it to the right position. + iter_++; + } + } + + // should be called before operations with iterator + void Finish() { + initialized_ = true; + std::sort(data_.begin(), data_.end(), + [this](std::pair a, + std::pair b) { + return (cmp.Compare(a.first, b.first) < 0); + }); + } + + // Removes the key from the set of keys over which this iterator iterates. + // Not to be confused with AddDeletion(). + // If the iterator is currently positioned on this key, the deletion will + // apply next time the iterator moves. + // Used for simulating ForwardIterator updating to a new version that doesn't + // have some of the keys (e.g. after compaction with a filter). + void Vanish(std::string _key) { + if (valid_ && data_[iter_].first == _key) { + delete_current_ = true; + return; + } + for (auto it = data_.begin(); it != data_.end(); ++it) { + ParsedInternalKey ikey; + Status pik_status = + ParseInternalKey(it->first, &ikey, true /* log_err_key */); + pik_status.PermitUncheckedError(); + assert(pik_status.ok()); + if (!pik_status.ok() || ikey.user_key != _key) { + continue; + } + if (valid_ && data_.begin() + iter_ > it) { + --iter_; + } + data_.erase(it); + return; + } + assert(false); + } + + // Number of operations done on this iterator since construction. + size_t steps() const { return steps_; } + + bool Valid() const override { + assert(initialized_); + return valid_; + } + + void SeekToFirst() override { + assert(initialized_); + ++steps_; + DeleteCurrentIfNeeded(); + valid_ = (data_.size() > 0); + iter_ = 0; + } + + void SeekToLast() override { + assert(initialized_); + ++steps_; + DeleteCurrentIfNeeded(); + valid_ = (data_.size() > 0); + iter_ = data_.size() - 1; + } + + void Seek(const Slice& target) override { + assert(initialized_); + SeekToFirst(); + ++steps_; + if (!valid_) { + return; + } + while (iter_ < data_.size() && + (cmp.Compare(data_[iter_].first, target) < 0)) { + ++iter_; + } + + if (iter_ == data_.size()) { + valid_ = false; + } + } + + void SeekForPrev(const Slice& target) override { + assert(initialized_); + DeleteCurrentIfNeeded(); + SeekForPrevImpl(target, &cmp); + } + + void Next() override { + assert(initialized_); + assert(valid_); + assert(iter_ < data_.size()); + + ++steps_; + if (delete_current_) { + DeleteCurrentIfNeeded(); + } else { + ++iter_; + } + valid_ = iter_ < data_.size(); + } + + void Prev() override { + assert(initialized_); + assert(valid_); + assert(iter_ < data_.size()); + + ++steps_; + DeleteCurrentIfNeeded(); + if (iter_ == 0) { + valid_ = false; + } else { + --iter_; + } + } + + Slice key() const override { + assert(initialized_); + return data_[iter_].first; + } + + Slice value() const override { + assert(initialized_); + return data_[iter_].second; + } + + Status status() const override { + assert(initialized_); + return Status::OK(); + } + + bool IsKeyPinned() const override { return true; } + bool IsValuePinned() const override { return true; } + + private: + bool initialized_; + bool valid_; + size_t sequence_number_; + size_t iter_; + size_t steps_ = 0; + + InternalKeyComparator cmp; + std::vector> data_; + bool delete_current_ = false; + + void DeleteCurrentIfNeeded() { + if (!delete_current_) { + return; + } + data_.erase(data_.begin() + iter_); + delete_current_ = false; + } +}; + +class DBIteratorTest : public testing::Test { + public: + Env* env_; + + DBIteratorTest() : env_(Env::Default()) {} +}; + +TEST_F(DBIteratorTest, DBIteratorPrevNext) { + Options options; + ImmutableOptions ioptions = ImmutableOptions(options); + MutableCFOptions mutable_cf_options = MutableCFOptions(options); + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddDeletion("a"); + internal_iter->AddDeletion("a"); + internal_iter->AddDeletion("a"); + internal_iter->AddDeletion("a"); + internal_iter->AddPut("a", "val_a"); + + internal_iter->AddPut("b", "val_b"); + internal_iter->Finish(); + + ReadOptions ro; + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 10 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "val_b"); + + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "val_a"); + + db_iter->Next(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "val_b"); + + db_iter->Next(); + ASSERT_TRUE(!db_iter->Valid()); + } + // Test to check the SeekToLast() with iterate_upper_bound not set + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "val_a"); + internal_iter->AddPut("b", "val_b"); + internal_iter->AddPut("b", "val_b"); + internal_iter->AddPut("c", "val_c"); + internal_iter->Finish(); + + ReadOptions ro; + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 10 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + } + + // Test to check the SeekToLast() with iterate_upper_bound set + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + + internal_iter->AddPut("a", "val_a"); + internal_iter->AddPut("b", "val_b"); + internal_iter->AddPut("c", "val_c"); + internal_iter->AddPut("d", "val_d"); + internal_iter->AddPut("e", "val_e"); + internal_iter->AddPut("f", "val_f"); + internal_iter->Finish(); + + Slice prefix("d"); + + ReadOptions ro; + ro.iterate_upper_bound = &prefix; + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 10 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + + db_iter->Next(); + ASSERT_TRUE(!db_iter->Valid()); + + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + } + // Test to check the SeekToLast() iterate_upper_bound set to a key that + // is not Put yet + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + + internal_iter->AddPut("a", "val_a"); + internal_iter->AddPut("a", "val_a"); + internal_iter->AddPut("b", "val_b"); + internal_iter->AddPut("c", "val_c"); + internal_iter->AddPut("d", "val_d"); + internal_iter->Finish(); + + Slice prefix("z"); + + ReadOptions ro; + ro.iterate_upper_bound = &prefix; + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 10 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "d"); + + db_iter->Next(); + ASSERT_TRUE(!db_iter->Valid()); + + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "d"); + + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + } + // Test to check the SeekToLast() with iterate_upper_bound set to the + // first key + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "val_a"); + internal_iter->AddPut("a", "val_a"); + internal_iter->AddPut("a", "val_a"); + internal_iter->AddPut("b", "val_b"); + internal_iter->AddPut("b", "val_b"); + internal_iter->Finish(); + + Slice prefix("a"); + + ReadOptions ro; + ro.iterate_upper_bound = &prefix; + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 10 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->SeekToLast(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); + } + // Test case to check SeekToLast with iterate_upper_bound set + // (same key put may times - SeekToLast should start with the + // maximum sequence id of the upper bound) + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "val_a"); + internal_iter->AddPut("b", "val_b"); + internal_iter->AddPut("c", "val_c"); + internal_iter->AddPut("c", "val_c"); + internal_iter->AddPut("c", "val_c"); + internal_iter->AddPut("c", "val_c"); + internal_iter->AddPut("c", "val_c"); + internal_iter->AddPut("c", "val_c"); + internal_iter->AddPut("c", "val_c"); + internal_iter->Finish(); + + Slice prefix("c"); + + ReadOptions ro; + ro.iterate_upper_bound = &prefix; + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 7 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + SetPerfLevel(kEnableCount); + ASSERT_TRUE(GetPerfLevel() == kEnableCount); + + get_perf_context()->Reset(); + db_iter->SeekToLast(); + + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(static_cast(get_perf_context()->internal_key_skipped_count), + 1); + ASSERT_EQ(db_iter->key().ToString(), "b"); + + SetPerfLevel(kDisable); + } + // Test to check the SeekToLast() with the iterate_upper_bound set + // (Checking the value of the key which has sequence ids greater than + // and less that the iterator's sequence id) + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + + internal_iter->AddPut("a", "val_a1"); + internal_iter->AddPut("a", "val_a2"); + internal_iter->AddPut("b", "val_b1"); + internal_iter->AddPut("c", "val_c1"); + internal_iter->AddPut("c", "val_c2"); + internal_iter->AddPut("c", "val_c3"); + internal_iter->AddPut("b", "val_b2"); + internal_iter->AddPut("d", "val_d1"); + internal_iter->Finish(); + + Slice prefix("c"); + + ReadOptions ro; + ro.iterate_upper_bound = &prefix; + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 4 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "val_b1"); + } + + // Test to check the SeekToLast() with the iterate_upper_bound set to the + // key that is deleted + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "val_a"); + internal_iter->AddDeletion("a"); + internal_iter->AddPut("b", "val_b"); + internal_iter->AddPut("c", "val_c"); + internal_iter->Finish(); + + Slice prefix("a"); + + ReadOptions ro; + ro.iterate_upper_bound = &prefix; + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 10 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->SeekToLast(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); + } + // Test to check the SeekToLast() with the iterate_upper_bound set + // (Deletion cases) + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "val_a"); + internal_iter->AddPut("b", "val_b"); + internal_iter->AddDeletion("b"); + internal_iter->AddPut("c", "val_c"); + internal_iter->Finish(); + + Slice prefix("c"); + + ReadOptions ro; + ro.iterate_upper_bound = &prefix; + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 10 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + + db_iter->Next(); + ASSERT_TRUE(!db_iter->Valid()); + + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + } + // Test to check the SeekToLast() with iterate_upper_bound set + // (Deletion cases - Lot of internal keys after the upper_bound + // is deleted) + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "val_a"); + internal_iter->AddPut("b", "val_b"); + internal_iter->AddDeletion("c"); + internal_iter->AddDeletion("d"); + internal_iter->AddDeletion("e"); + internal_iter->AddDeletion("f"); + internal_iter->AddDeletion("g"); + internal_iter->AddDeletion("h"); + internal_iter->Finish(); + + Slice prefix("c"); + + ReadOptions ro; + ro.iterate_upper_bound = &prefix; + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 7 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + SetPerfLevel(kEnableCount); + ASSERT_TRUE(GetPerfLevel() == kEnableCount); + + get_perf_context()->Reset(); + db_iter->SeekToLast(); + + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ( + static_cast(get_perf_context()->internal_delete_skipped_count), 0); + ASSERT_EQ(db_iter->key().ToString(), "b"); + + SetPerfLevel(kDisable); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddDeletion("a"); + internal_iter->AddDeletion("a"); + internal_iter->AddDeletion("a"); + internal_iter->AddDeletion("a"); + internal_iter->AddPut("a", "val_a"); + + internal_iter->AddPut("b", "val_b"); + internal_iter->Finish(); + + ReadOptions ro; + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 10 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->SeekToFirst(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "val_a"); + + db_iter->Next(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "val_b"); + + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "val_a"); + + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "val_a"); + internal_iter->AddPut("b", "val_b"); + + internal_iter->AddPut("a", "val_a"); + internal_iter->AddPut("b", "val_b"); + + internal_iter->AddPut("a", "val_a"); + internal_iter->AddPut("b", "val_b"); + + internal_iter->AddPut("a", "val_a"); + internal_iter->AddPut("b", "val_b"); + + internal_iter->AddPut("a", "val_a"); + internal_iter->AddPut("b", "val_b"); + internal_iter->Finish(); + + ReadOptions ro; + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 2 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "val_b"); + + db_iter->Next(); + ASSERT_TRUE(!db_iter->Valid()); + + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "val_b"); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "val_a"); + internal_iter->AddPut("a", "val_a"); + internal_iter->AddPut("a", "val_a"); + internal_iter->AddPut("a", "val_a"); + internal_iter->AddPut("a", "val_a"); + + internal_iter->AddPut("b", "val_b"); + + internal_iter->AddPut("c", "val_c"); + internal_iter->Finish(); + + ReadOptions ro; + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 10 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "val_c"); + + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "val_b"); + + db_iter->Next(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "val_c"); + } +} + +TEST_F(DBIteratorTest, DBIteratorEmpty) { + Options options; + ImmutableOptions ioptions = ImmutableOptions(options); + MutableCFOptions mutable_cf_options = MutableCFOptions(options); + ReadOptions ro; + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 0 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 0 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToFirst(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); + } +} + +TEST_F(DBIteratorTest, DBIteratorUseSkipCountSkips) { + ReadOptions ro; + Options options; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.merge_operator = MergeOperators::CreateFromStringId("stringappend"); + + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + for (size_t i = 0; i < 200; ++i) { + internal_iter->AddPut("a", "a"); + internal_iter->AddPut("b", "b"); + internal_iter->AddPut("c", "c"); + } + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ImmutableOptions(options), MutableCFOptions(options), + BytewiseComparator(), internal_iter, nullptr /* version */, + 2 /* sequence */, options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "c"); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 1u); + + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "b"); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 2u); + + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "a"); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 3u); + + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 3u); +} + +TEST_F(DBIteratorTest, DBIteratorUseSkip) { + ReadOptions ro; + Options options; + options.merge_operator = MergeOperators::CreateFromStringId("stringappend"); + ImmutableOptions ioptions = ImmutableOptions(options); + MutableCFOptions mutable_cf_options = MutableCFOptions(options); + + { + for (size_t i = 0; i < 200; ++i) { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("b", "merge_1"); + internal_iter->AddMerge("a", "merge_2"); + for (size_t k = 0; k < 200; ++k) { + internal_iter->AddPut("c", std::to_string(k)); + } + internal_iter->Finish(); + + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, i + 2 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), std::to_string(i)); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_2"); + db_iter->Prev(); + + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); + } + } + + { + for (size_t i = 0; i < 200; ++i) { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("b", "merge_1"); + internal_iter->AddMerge("a", "merge_2"); + for (size_t k = 0; k < 200; ++k) { + internal_iter->AddDeletion("c"); + } + internal_iter->AddPut("c", "200"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, i + 2 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_2"); + db_iter->Prev(); + + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("b", "merge_1"); + internal_iter->AddMerge("a", "merge_2"); + for (size_t i = 0; i < 200; ++i) { + internal_iter->AddDeletion("c"); + } + internal_iter->AddPut("c", "200"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 202 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "200"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_2"); + db_iter->Prev(); + + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); + } + } + + { + for (size_t i = 0; i < 200; ++i) { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + for (size_t k = 0; k < 200; ++k) { + internal_iter->AddDeletion("c"); + } + internal_iter->AddPut("c", "200"); + internal_iter->Finish(); + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, i /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); + + db_iter->SeekToFirst(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); + } + + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + for (size_t i = 0; i < 200; ++i) { + internal_iter->AddDeletion("c"); + } + internal_iter->AddPut("c", "200"); + internal_iter->Finish(); + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 200 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "200"); + + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); + + db_iter->SeekToFirst(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "200"); + + db_iter->Next(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); + } + + { + for (size_t i = 0; i < 200; ++i) { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("b", "merge_1"); + internal_iter->AddMerge("a", "merge_2"); + for (size_t k = 0; k < 200; ++k) { + internal_iter->AddPut("d", std::to_string(k)); + } + + for (size_t k = 0; k < 200; ++k) { + internal_iter->AddPut("c", std::to_string(k)); + } + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, i + 2 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "d"); + ASSERT_EQ(db_iter->value().ToString(), std::to_string(i)); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_2"); + db_iter->Prev(); + + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); + } + } + + { + for (size_t i = 0; i < 200; ++i) { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("b", "b"); + internal_iter->AddMerge("a", "a"); + for (size_t k = 0; k < 200; ++k) { + internal_iter->AddMerge("c", std::to_string(k)); + } + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, i + 2 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "c"); + std::string merge_result = "0"; + for (size_t j = 1; j <= i; ++j) { + merge_result += "," + std::to_string(j); + } + ASSERT_EQ(db_iter->value().ToString(), merge_result); + + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "b"); + + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "a"); + + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); + } + } +} + +TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) { + Options options; + ImmutableOptions ioptions = ImmutableOptions(options); + MutableCFOptions mutable_cf_options = MutableCFOptions(options); + ReadOptions ro; + + // Basic test case ... Make sure explicityly passing the default value works. + // Skipping internal keys is disabled by default, when the value is 0. + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "val_a"); + internal_iter->AddDeletion("b"); + internal_iter->AddDeletion("b"); + internal_iter->AddPut("c", "val_c"); + internal_iter->AddPut("c", "val_c"); + internal_iter->AddDeletion("c"); + internal_iter->AddPut("d", "val_d"); + internal_iter->Finish(); + + ro.max_skippable_internal_keys = 0; + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 10 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->SeekToFirst(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "val_a"); + + db_iter->Next(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "d"); + ASSERT_EQ(db_iter->value().ToString(), "val_d"); + + db_iter->Next(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_TRUE(db_iter->status().ok()); + + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "d"); + ASSERT_EQ(db_iter->value().ToString(), "val_d"); + + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "val_a"); + + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_OK(db_iter->status()); + } + + // Test to make sure that the request will *not* fail as incomplete if + // num_internal_keys_skipped is *equal* to max_skippable_internal_keys + // threshold. (It will fail as incomplete only when the threshold is + // exceeded.) + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "val_a"); + internal_iter->AddDeletion("b"); + internal_iter->AddDeletion("b"); + internal_iter->AddPut("c", "val_c"); + internal_iter->Finish(); + + ro.max_skippable_internal_keys = 2; + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 10 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->SeekToFirst(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "val_a"); + + db_iter->Next(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "val_c"); + + db_iter->Next(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_TRUE(db_iter->status().ok()); + + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "val_c"); + + db_iter->Prev(); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "val_a"); + + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_TRUE(db_iter->status().ok()); + } + + // Fail the request as incomplete when num_internal_keys_skipped > + // max_skippable_internal_keys + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "val_a"); + internal_iter->AddDeletion("b"); + internal_iter->AddDeletion("b"); + internal_iter->AddDeletion("b"); + internal_iter->AddPut("c", "val_c"); + internal_iter->Finish(); + + ro.max_skippable_internal_keys = 2; + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 10 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->SeekToFirst(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "val_a"); + + db_iter->Next(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_TRUE(db_iter->status().IsIncomplete()); + + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "val_c"); + + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_TRUE(db_iter->status().IsIncomplete()); + } + + // Test that the num_internal_keys_skipped counter resets after a successful + // read. + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "val_a"); + internal_iter->AddDeletion("b"); + internal_iter->AddDeletion("b"); + internal_iter->AddPut("c", "val_c"); + internal_iter->AddDeletion("d"); + internal_iter->AddDeletion("d"); + internal_iter->AddDeletion("d"); + internal_iter->AddPut("e", "val_e"); + internal_iter->Finish(); + + ro.max_skippable_internal_keys = 2; + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 10 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->SeekToFirst(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "val_a"); + + db_iter->Next(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "val_c"); + + db_iter->Next(); // num_internal_keys_skipped counter resets here. + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_TRUE(db_iter->status().IsIncomplete()); + } + + // Test that the num_internal_keys_skipped counter resets after a successful + // read. + // Reverse direction + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "val_a"); + internal_iter->AddDeletion("b"); + internal_iter->AddDeletion("b"); + internal_iter->AddDeletion("b"); + internal_iter->AddPut("c", "val_c"); + internal_iter->AddDeletion("d"); + internal_iter->AddDeletion("d"); + internal_iter->AddPut("e", "val_e"); + internal_iter->Finish(); + + ro.max_skippable_internal_keys = 2; + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 10 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "e"); + ASSERT_EQ(db_iter->value().ToString(), "val_e"); + + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "val_c"); + + db_iter->Prev(); // num_internal_keys_skipped counter resets here. + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_TRUE(db_iter->status().IsIncomplete()); + } + + // Test that skipping separate keys is handled + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "val_a"); + internal_iter->AddDeletion("b"); + internal_iter->AddDeletion("c"); + internal_iter->AddDeletion("d"); + internal_iter->AddPut("e", "val_e"); + internal_iter->Finish(); + + ro.max_skippable_internal_keys = 2; + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 10 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->SeekToFirst(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "val_a"); + + db_iter->Next(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_TRUE(db_iter->status().IsIncomplete()); + + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "e"); + ASSERT_EQ(db_iter->value().ToString(), "val_e"); + + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_TRUE(db_iter->status().IsIncomplete()); + } + + // Test if alternating puts and deletes of the same key are handled correctly. + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "val_a"); + internal_iter->AddPut("b", "val_b"); + internal_iter->AddDeletion("b"); + internal_iter->AddPut("c", "val_c"); + internal_iter->AddDeletion("c"); + internal_iter->AddPut("d", "val_d"); + internal_iter->AddDeletion("d"); + internal_iter->AddPut("e", "val_e"); + internal_iter->Finish(); + + ro.max_skippable_internal_keys = 2; + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 10 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->SeekToFirst(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "val_a"); + + db_iter->Next(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_TRUE(db_iter->status().IsIncomplete()); + + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "e"); + ASSERT_EQ(db_iter->value().ToString(), "val_e"); + + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_TRUE(db_iter->status().IsIncomplete()); + } + + // Test for large number of skippable internal keys with *default* + // max_sequential_skip_in_iterations. + { + for (size_t i = 1; i <= 200; ++i) { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "val_a"); + for (size_t j = 1; j <= i; ++j) { + internal_iter->AddPut("b", "val_b"); + internal_iter->AddDeletion("b"); + } + internal_iter->AddPut("c", "val_c"); + internal_iter->Finish(); + + ro.max_skippable_internal_keys = i; + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 2 * i + 1 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->SeekToFirst(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "val_a"); + + db_iter->Next(); + if ((options.max_sequential_skip_in_iterations + 1) >= + ro.max_skippable_internal_keys) { + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_TRUE(db_iter->status().IsIncomplete()); + } else { + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "val_c"); + } + + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "val_c"); + + db_iter->Prev(); + if ((options.max_sequential_skip_in_iterations + 1) >= + ro.max_skippable_internal_keys) { + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_TRUE(db_iter->status().IsIncomplete()); + } else { + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "val_a"); + } + } + } + + // Test for large number of skippable internal keys with a *non-default* + // max_sequential_skip_in_iterations. + { + for (size_t i = 1; i <= 200; ++i) { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "val_a"); + for (size_t j = 1; j <= i; ++j) { + internal_iter->AddPut("b", "val_b"); + internal_iter->AddDeletion("b"); + } + internal_iter->AddPut("c", "val_c"); + internal_iter->Finish(); + + options.max_sequential_skip_in_iterations = 1000; + ro.max_skippable_internal_keys = i; + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 2 * i + 1 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->SeekToFirst(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "val_a"); + + db_iter->Next(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_TRUE(db_iter->status().IsIncomplete()); + + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "val_c"); + + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + ASSERT_TRUE(db_iter->status().IsIncomplete()); + } + } +} + +TEST_F(DBIteratorTest, DBIterator1) { + ReadOptions ro; + Options options; + options.merge_operator = MergeOperators::CreateFromStringId("stringappend"); + + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "0"); + internal_iter->AddPut("b", "0"); + internal_iter->AddDeletion("b"); + internal_iter->AddMerge("a", "1"); + internal_iter->AddMerge("b", "2"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ImmutableOptions(options), MutableCFOptions(options), + BytewiseComparator(), internal_iter, nullptr /* version */, + 1 /* sequence */, options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToFirst(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "0"); + db_iter->Next(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + db_iter->Next(); + ASSERT_FALSE(db_iter->Valid()); +} + +TEST_F(DBIteratorTest, DBIterator2) { + ReadOptions ro; + Options options; + options.merge_operator = MergeOperators::CreateFromStringId("stringappend"); + + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "0"); + internal_iter->AddPut("b", "0"); + internal_iter->AddDeletion("b"); + internal_iter->AddMerge("a", "1"); + internal_iter->AddMerge("b", "2"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ImmutableOptions(options), MutableCFOptions(options), + BytewiseComparator(), internal_iter, nullptr /* version */, + 0 /* sequence */, options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToFirst(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "0"); + db_iter->Next(); + ASSERT_TRUE(!db_iter->Valid()); +} + +TEST_F(DBIteratorTest, DBIterator3) { + ReadOptions ro; + Options options; + options.merge_operator = MergeOperators::CreateFromStringId("stringappend"); + + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "0"); + internal_iter->AddPut("b", "0"); + internal_iter->AddDeletion("b"); + internal_iter->AddMerge("a", "1"); + internal_iter->AddMerge("b", "2"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ImmutableOptions(options), MutableCFOptions(options), + BytewiseComparator(), internal_iter, nullptr /* version */, + 2 /* sequence */, options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToFirst(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "0"); + db_iter->Next(); + ASSERT_TRUE(!db_iter->Valid()); +} + +TEST_F(DBIteratorTest, DBIterator4) { + ReadOptions ro; + Options options; + options.merge_operator = MergeOperators::CreateFromStringId("stringappend"); + + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "0"); + internal_iter->AddPut("b", "0"); + internal_iter->AddDeletion("b"); + internal_iter->AddMerge("a", "1"); + internal_iter->AddMerge("b", "2"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ImmutableOptions(options), MutableCFOptions(options), + BytewiseComparator(), internal_iter, nullptr /* version */, + 4 /* sequence */, options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToFirst(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "0,1"); + db_iter->Next(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "2"); + db_iter->Next(); + ASSERT_TRUE(!db_iter->Valid()); +} + +TEST_F(DBIteratorTest, DBIterator5) { + ReadOptions ro; + Options options; + options.merge_operator = MergeOperators::CreateFromStringId("stringappend"); + ImmutableOptions ioptions = ImmutableOptions(options); + MutableCFOptions mutable_cf_options = MutableCFOptions(options); + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddMerge("a", "merge_2"); + internal_iter->AddMerge("a", "merge_3"); + internal_iter->AddPut("a", "put_1"); + internal_iter->AddMerge("a", "merge_4"); + internal_iter->AddMerge("a", "merge_5"); + internal_iter->AddMerge("a", "merge_6"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 0 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddMerge("a", "merge_2"); + internal_iter->AddMerge("a", "merge_3"); + internal_iter->AddPut("a", "put_1"); + internal_iter->AddMerge("a", "merge_4"); + internal_iter->AddMerge("a", "merge_5"); + internal_iter->AddMerge("a", "merge_6"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 1 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddMerge("a", "merge_2"); + internal_iter->AddMerge("a", "merge_3"); + internal_iter->AddPut("a", "put_1"); + internal_iter->AddMerge("a", "merge_4"); + internal_iter->AddMerge("a", "merge_5"); + internal_iter->AddMerge("a", "merge_6"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 2 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2,merge_3"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddMerge("a", "merge_2"); + internal_iter->AddMerge("a", "merge_3"); + internal_iter->AddPut("a", "put_1"); + internal_iter->AddMerge("a", "merge_4"); + internal_iter->AddMerge("a", "merge_5"); + internal_iter->AddMerge("a", "merge_6"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 3 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "put_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddMerge("a", "merge_2"); + internal_iter->AddMerge("a", "merge_3"); + internal_iter->AddPut("a", "put_1"); + internal_iter->AddMerge("a", "merge_4"); + internal_iter->AddMerge("a", "merge_5"); + internal_iter->AddMerge("a", "merge_6"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 4 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "put_1,merge_4"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddMerge("a", "merge_2"); + internal_iter->AddMerge("a", "merge_3"); + internal_iter->AddPut("a", "put_1"); + internal_iter->AddMerge("a", "merge_4"); + internal_iter->AddMerge("a", "merge_5"); + internal_iter->AddMerge("a", "merge_6"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 5 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "put_1,merge_4,merge_5"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddMerge("a", "merge_2"); + internal_iter->AddMerge("a", "merge_3"); + internal_iter->AddPut("a", "put_1"); + internal_iter->AddMerge("a", "merge_4"); + internal_iter->AddMerge("a", "merge_5"); + internal_iter->AddMerge("a", "merge_6"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 6 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "put_1,merge_4,merge_5,merge_6"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + // put, singledelete, merge + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "val_a"); + internal_iter->AddSingleDeletion("a"); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddMerge("a", "merge_2"); + internal_iter->AddPut("b", "val_b"); + internal_iter->Finish(); + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 10 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->Seek("b"); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + } +} + +TEST_F(DBIteratorTest, DBIterator6) { + ReadOptions ro; + Options options; + options.merge_operator = MergeOperators::CreateFromStringId("stringappend"); + ImmutableOptions ioptions = ImmutableOptions(options); + MutableCFOptions mutable_cf_options = MutableCFOptions(options); + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddMerge("a", "merge_2"); + internal_iter->AddMerge("a", "merge_3"); + internal_iter->AddDeletion("a"); + internal_iter->AddMerge("a", "merge_4"); + internal_iter->AddMerge("a", "merge_5"); + internal_iter->AddMerge("a", "merge_6"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 0 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddMerge("a", "merge_2"); + internal_iter->AddMerge("a", "merge_3"); + internal_iter->AddDeletion("a"); + internal_iter->AddMerge("a", "merge_4"); + internal_iter->AddMerge("a", "merge_5"); + internal_iter->AddMerge("a", "merge_6"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 1 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddMerge("a", "merge_2"); + internal_iter->AddMerge("a", "merge_3"); + internal_iter->AddDeletion("a"); + internal_iter->AddMerge("a", "merge_4"); + internal_iter->AddMerge("a", "merge_5"); + internal_iter->AddMerge("a", "merge_6"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 2 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2,merge_3"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddMerge("a", "merge_2"); + internal_iter->AddMerge("a", "merge_3"); + internal_iter->AddDeletion("a"); + internal_iter->AddMerge("a", "merge_4"); + internal_iter->AddMerge("a", "merge_5"); + internal_iter->AddMerge("a", "merge_6"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 3 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddMerge("a", "merge_2"); + internal_iter->AddMerge("a", "merge_3"); + internal_iter->AddDeletion("a"); + internal_iter->AddMerge("a", "merge_4"); + internal_iter->AddMerge("a", "merge_5"); + internal_iter->AddMerge("a", "merge_6"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 4 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_4"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddMerge("a", "merge_2"); + internal_iter->AddMerge("a", "merge_3"); + internal_iter->AddDeletion("a"); + internal_iter->AddMerge("a", "merge_4"); + internal_iter->AddMerge("a", "merge_5"); + internal_iter->AddMerge("a", "merge_6"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 5 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddMerge("a", "merge_2"); + internal_iter->AddMerge("a", "merge_3"); + internal_iter->AddDeletion("a"); + internal_iter->AddMerge("a", "merge_4"); + internal_iter->AddMerge("a", "merge_5"); + internal_iter->AddMerge("a", "merge_6"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 6 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5,merge_6"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } +} + +TEST_F(DBIteratorTest, DBIterator7) { + ReadOptions ro; + Options options; + options.merge_operator = MergeOperators::CreateFromStringId("stringappend"); + ImmutableOptions ioptions = ImmutableOptions(options); + MutableCFOptions mutable_cf_options = MutableCFOptions(options); + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddPut("b", "val"); + internal_iter->AddMerge("b", "merge_2"); + + internal_iter->AddDeletion("b"); + internal_iter->AddMerge("b", "merge_3"); + + internal_iter->AddMerge("c", "merge_4"); + internal_iter->AddMerge("c", "merge_5"); + + internal_iter->AddDeletion("b"); + internal_iter->AddMerge("b", "merge_6"); + internal_iter->AddMerge("b", "merge_7"); + internal_iter->AddMerge("b", "merge_8"); + internal_iter->AddMerge("b", "merge_9"); + internal_iter->AddMerge("b", "merge_10"); + internal_iter->AddMerge("b", "merge_11"); + + internal_iter->AddDeletion("c"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 0 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddPut("b", "val"); + internal_iter->AddMerge("b", "merge_2"); + + internal_iter->AddDeletion("b"); + internal_iter->AddMerge("b", "merge_3"); + + internal_iter->AddMerge("c", "merge_4"); + internal_iter->AddMerge("c", "merge_5"); + + internal_iter->AddDeletion("b"); + internal_iter->AddMerge("b", "merge_6"); + internal_iter->AddMerge("b", "merge_7"); + internal_iter->AddMerge("b", "merge_8"); + internal_iter->AddMerge("b", "merge_9"); + internal_iter->AddMerge("b", "merge_10"); + internal_iter->AddMerge("b", "merge_11"); + + internal_iter->AddDeletion("c"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 2 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "val,merge_2"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddPut("b", "val"); + internal_iter->AddMerge("b", "merge_2"); + + internal_iter->AddDeletion("b"); + internal_iter->AddMerge("b", "merge_3"); + + internal_iter->AddMerge("c", "merge_4"); + internal_iter->AddMerge("c", "merge_5"); + + internal_iter->AddDeletion("b"); + internal_iter->AddMerge("b", "merge_6"); + internal_iter->AddMerge("b", "merge_7"); + internal_iter->AddMerge("b", "merge_8"); + internal_iter->AddMerge("b", "merge_9"); + internal_iter->AddMerge("b", "merge_10"); + internal_iter->AddMerge("b", "merge_11"); + + internal_iter->AddDeletion("c"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 4 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "merge_3"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddPut("b", "val"); + internal_iter->AddMerge("b", "merge_2"); + + internal_iter->AddDeletion("b"); + internal_iter->AddMerge("b", "merge_3"); + + internal_iter->AddMerge("c", "merge_4"); + internal_iter->AddMerge("c", "merge_5"); + + internal_iter->AddDeletion("b"); + internal_iter->AddMerge("b", "merge_6"); + internal_iter->AddMerge("b", "merge_7"); + internal_iter->AddMerge("b", "merge_8"); + internal_iter->AddMerge("b", "merge_9"); + internal_iter->AddMerge("b", "merge_10"); + internal_iter->AddMerge("b", "merge_11"); + + internal_iter->AddDeletion("c"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 5 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "merge_4"); + db_iter->Prev(); + + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "merge_3"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddPut("b", "val"); + internal_iter->AddMerge("b", "merge_2"); + + internal_iter->AddDeletion("b"); + internal_iter->AddMerge("b", "merge_3"); + + internal_iter->AddMerge("c", "merge_4"); + internal_iter->AddMerge("c", "merge_5"); + + internal_iter->AddDeletion("b"); + internal_iter->AddMerge("b", "merge_6"); + internal_iter->AddMerge("b", "merge_7"); + internal_iter->AddMerge("b", "merge_8"); + internal_iter->AddMerge("b", "merge_9"); + internal_iter->AddMerge("b", "merge_10"); + internal_iter->AddMerge("b", "merge_11"); + + internal_iter->AddDeletion("c"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 6 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "merge_3"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddPut("b", "val"); + internal_iter->AddMerge("b", "merge_2"); + + internal_iter->AddDeletion("b"); + internal_iter->AddMerge("b", "merge_3"); + + internal_iter->AddMerge("c", "merge_4"); + internal_iter->AddMerge("c", "merge_5"); + + internal_iter->AddDeletion("b"); + internal_iter->AddMerge("b", "merge_6"); + internal_iter->AddMerge("b", "merge_7"); + internal_iter->AddMerge("b", "merge_8"); + internal_iter->AddMerge("b", "merge_9"); + internal_iter->AddMerge("b", "merge_10"); + internal_iter->AddMerge("b", "merge_11"); + + internal_iter->AddDeletion("c"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 7 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddPut("b", "val"); + internal_iter->AddMerge("b", "merge_2"); + + internal_iter->AddDeletion("b"); + internal_iter->AddMerge("b", "merge_3"); + + internal_iter->AddMerge("c", "merge_4"); + internal_iter->AddMerge("c", "merge_5"); + + internal_iter->AddDeletion("b"); + internal_iter->AddMerge("b", "merge_6"); + internal_iter->AddMerge("b", "merge_7"); + internal_iter->AddMerge("b", "merge_8"); + internal_iter->AddMerge("b", "merge_9"); + internal_iter->AddMerge("b", "merge_10"); + internal_iter->AddMerge("b", "merge_11"); + + internal_iter->AddDeletion("c"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 9 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "merge_6,merge_7"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddPut("b", "val"); + internal_iter->AddMerge("b", "merge_2"); + + internal_iter->AddDeletion("b"); + internal_iter->AddMerge("b", "merge_3"); + + internal_iter->AddMerge("c", "merge_4"); + internal_iter->AddMerge("c", "merge_5"); + + internal_iter->AddDeletion("b"); + internal_iter->AddMerge("b", "merge_6"); + internal_iter->AddMerge("b", "merge_7"); + internal_iter->AddMerge("b", "merge_8"); + internal_iter->AddMerge("b", "merge_9"); + internal_iter->AddMerge("b", "merge_10"); + internal_iter->AddMerge("b", "merge_11"); + + internal_iter->AddDeletion("c"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 13 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), + "merge_6,merge_7,merge_8,merge_9,merge_10,merge_11"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } + + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddPut("b", "val"); + internal_iter->AddMerge("b", "merge_2"); + + internal_iter->AddDeletion("b"); + internal_iter->AddMerge("b", "merge_3"); + + internal_iter->AddMerge("c", "merge_4"); + internal_iter->AddMerge("c", "merge_5"); + + internal_iter->AddDeletion("b"); + internal_iter->AddMerge("b", "merge_6"); + internal_iter->AddMerge("b", "merge_7"); + internal_iter->AddMerge("b", "merge_8"); + internal_iter->AddMerge("b", "merge_9"); + internal_iter->AddMerge("b", "merge_10"); + internal_iter->AddMerge("b", "merge_11"); + + internal_iter->AddDeletion("c"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ioptions, mutable_cf_options, BytewiseComparator(), + internal_iter, nullptr /* version */, 14 /* sequence */, + options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), + "merge_6,merge_7,merge_8,merge_9,merge_10,merge_11"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1"); + db_iter->Prev(); + ASSERT_TRUE(!db_iter->Valid()); + } +} + +TEST_F(DBIteratorTest, DBIterator8) { + ReadOptions ro; + Options options; + options.merge_operator = MergeOperators::CreateFromStringId("stringappend"); + + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddDeletion("a"); + internal_iter->AddPut("a", "0"); + internal_iter->AddPut("b", "0"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ImmutableOptions(options), MutableCFOptions(options), + BytewiseComparator(), internal_iter, nullptr /* version */, + 10 /* sequence */, options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "0"); + + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "0"); +} + +// TODO(3.13): fix the issue of Seek() then Prev() which might not necessary +// return the biggest element smaller than the seek key. +TEST_F(DBIteratorTest, DBIterator9) { + ReadOptions ro; + Options options; + options.merge_operator = MergeOperators::CreateFromStringId("stringappend"); + { + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddMerge("a", "merge_1"); + internal_iter->AddMerge("a", "merge_2"); + internal_iter->AddMerge("b", "merge_3"); + internal_iter->AddMerge("b", "merge_4"); + internal_iter->AddMerge("d", "merge_5"); + internal_iter->AddMerge("d", "merge_6"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ImmutableOptions(options), MutableCFOptions(options), + BytewiseComparator(), internal_iter, nullptr /* version */, + 10 /* sequence */, options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "merge_3,merge_4"); + db_iter->Next(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "d"); + ASSERT_EQ(db_iter->value().ToString(), "merge_5,merge_6"); + + db_iter->Seek("b"); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "merge_3,merge_4"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2"); + + db_iter->SeekForPrev("b"); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "merge_3,merge_4"); + db_iter->Next(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "d"); + ASSERT_EQ(db_iter->value().ToString(), "merge_5,merge_6"); + + db_iter->Seek("c"); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "d"); + ASSERT_EQ(db_iter->value().ToString(), "merge_5,merge_6"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "merge_3,merge_4"); + + db_iter->SeekForPrev("c"); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "merge_3,merge_4"); + db_iter->Next(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "d"); + ASSERT_EQ(db_iter->value().ToString(), "merge_5,merge_6"); + } +} + +// TODO(3.13): fix the issue of Seek() then Prev() which might not necessary +// return the biggest element smaller than the seek key. +TEST_F(DBIteratorTest, DBIterator10) { + ReadOptions ro; + Options options; + + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "1"); + internal_iter->AddPut("b", "2"); + internal_iter->AddPut("c", "3"); + internal_iter->AddPut("d", "4"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ImmutableOptions(options), MutableCFOptions(options), + BytewiseComparator(), internal_iter, nullptr /* version */, + 10 /* sequence */, options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->Seek("c"); + ASSERT_TRUE(db_iter->Valid()); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "2"); + + db_iter->Next(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "3"); + + db_iter->SeekForPrev("c"); + ASSERT_TRUE(db_iter->Valid()); + db_iter->Next(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "d"); + ASSERT_EQ(db_iter->value().ToString(), "4"); + + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "3"); +} + +TEST_F(DBIteratorTest, SeekToLastOccurrenceSeq0) { + ReadOptions ro; + Options options; + options.merge_operator = nullptr; + + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "1"); + internal_iter->AddPut("b", "2"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ImmutableOptions(options), MutableCFOptions(options), + BytewiseComparator(), internal_iter, nullptr /* version */, + 10 /* sequence */, 0 /* force seek */, nullptr /* read_callback */)); + db_iter->SeekToFirst(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "1"); + db_iter->Next(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "2"); + db_iter->Next(); + ASSERT_FALSE(db_iter->Valid()); +} + +TEST_F(DBIteratorTest, DBIterator11) { + ReadOptions ro; + Options options; + options.merge_operator = MergeOperators::CreateFromStringId("stringappend"); + + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "0"); + internal_iter->AddPut("b", "0"); + internal_iter->AddSingleDeletion("b"); + internal_iter->AddMerge("a", "1"); + internal_iter->AddMerge("b", "2"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ImmutableOptions(options), MutableCFOptions(options), + BytewiseComparator(), internal_iter, nullptr /* version */, + 1 /* sequence */, options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + db_iter->SeekToFirst(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "0"); + db_iter->Next(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + db_iter->Next(); + ASSERT_FALSE(db_iter->Valid()); +} + +TEST_F(DBIteratorTest, DBIterator12) { + ReadOptions ro; + Options options; + options.merge_operator = nullptr; + + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "1"); + internal_iter->AddPut("b", "2"); + internal_iter->AddPut("c", "3"); + internal_iter->AddSingleDeletion("b"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ImmutableOptions(options), MutableCFOptions(options), + BytewiseComparator(), internal_iter, nullptr /* version */, + 10 /* sequence */, 0 /* force seek */, nullptr /* read_callback */)); + db_iter->SeekToLast(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "c"); + ASSERT_EQ(db_iter->value().ToString(), "3"); + db_iter->Prev(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "1"); + db_iter->Prev(); + ASSERT_FALSE(db_iter->Valid()); +} + +TEST_F(DBIteratorTest, DBIterator13) { + ReadOptions ro; + Options options; + options.merge_operator = nullptr; + + std::string key; + key.resize(9); + key.assign(9, static_cast(0)); + key[0] = 'b'; + + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut(key, "0"); + internal_iter->AddPut(key, "1"); + internal_iter->AddPut(key, "2"); + internal_iter->AddPut(key, "3"); + internal_iter->AddPut(key, "4"); + internal_iter->AddPut(key, "5"); + internal_iter->AddPut(key, "6"); + internal_iter->AddPut(key, "7"); + internal_iter->AddPut(key, "8"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ImmutableOptions(options), MutableCFOptions(options), + BytewiseComparator(), internal_iter, nullptr /* version */, + 2 /* sequence */, 3 /* max_sequential_skip_in_iterations */, + nullptr /* read_callback */)); + db_iter->Seek("b"); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), key); + ASSERT_EQ(db_iter->value().ToString(), "2"); +} + +TEST_F(DBIteratorTest, DBIterator14) { + ReadOptions ro; + Options options; + options.merge_operator = nullptr; + + std::string key("b"); + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("b", "0"); + internal_iter->AddPut("b", "1"); + internal_iter->AddPut("b", "2"); + internal_iter->AddPut("b", "3"); + internal_iter->AddPut("a", "4"); + internal_iter->AddPut("a", "5"); + internal_iter->AddPut("a", "6"); + internal_iter->AddPut("c", "7"); + internal_iter->AddPut("c", "8"); + internal_iter->AddPut("c", "9"); + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ImmutableOptions(options), MutableCFOptions(options), + BytewiseComparator(), internal_iter, nullptr /* version */, + 4 /* sequence */, 1 /* max_sequential_skip_in_iterations */, + nullptr /* read_callback */)); + db_iter->Seek("b"); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(db_iter->key().ToString(), "b"); + ASSERT_EQ(db_iter->value().ToString(), "3"); + db_iter->SeekToFirst(); + ASSERT_EQ(db_iter->key().ToString(), "a"); + ASSERT_EQ(db_iter->value().ToString(), "4"); +} + +class DBIterWithMergeIterTest : public testing::Test { + public: + DBIterWithMergeIterTest() + : env_(Env::Default()), icomp_(BytewiseComparator()) { + options_.merge_operator = nullptr; + + internal_iter1_ = new TestIterator(BytewiseComparator()); + internal_iter1_->Add("a", kTypeValue, "1", 3u); + internal_iter1_->Add("f", kTypeValue, "2", 5u); + internal_iter1_->Add("g", kTypeValue, "3", 7u); + internal_iter1_->Finish(); + + internal_iter2_ = new TestIterator(BytewiseComparator()); + internal_iter2_->Add("a", kTypeValue, "4", 6u); + internal_iter2_->Add("b", kTypeValue, "5", 1u); + internal_iter2_->Add("c", kTypeValue, "6", 2u); + internal_iter2_->Add("d", kTypeValue, "7", 3u); + internal_iter2_->Finish(); + + std::vector child_iters; + child_iters.push_back(internal_iter1_); + child_iters.push_back(internal_iter2_); + InternalKeyComparator icomp(BytewiseComparator()); + InternalIterator* merge_iter = + NewMergingIterator(&icomp_, &child_iters[0], 2u); + + db_iter_.reset(NewDBIterator( + env_, ro_, ImmutableOptions(options_), MutableCFOptions(options_), + BytewiseComparator(), merge_iter, nullptr /* version */, + 8 /* read data earlier than seqId 8 */, + 3 /* max iterators before reseek */, nullptr /* read_callback */)); + } + + Env* env_; + ReadOptions ro_; + Options options_; + TestIterator* internal_iter1_; + TestIterator* internal_iter2_; + InternalKeyComparator icomp_; + Iterator* merge_iter_; + std::unique_ptr db_iter_; +}; + +TEST_F(DBIterWithMergeIterTest, InnerMergeIterator1) { + db_iter_->SeekToFirst(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "a"); + ASSERT_EQ(db_iter_->value().ToString(), "4"); + db_iter_->Next(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "b"); + ASSERT_EQ(db_iter_->value().ToString(), "5"); + db_iter_->Next(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "c"); + ASSERT_EQ(db_iter_->value().ToString(), "6"); + db_iter_->Next(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "d"); + ASSERT_EQ(db_iter_->value().ToString(), "7"); + db_iter_->Next(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "f"); + ASSERT_EQ(db_iter_->value().ToString(), "2"); + db_iter_->Next(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "g"); + ASSERT_EQ(db_iter_->value().ToString(), "3"); + db_iter_->Next(); + ASSERT_FALSE(db_iter_->Valid()); +} + +TEST_F(DBIterWithMergeIterTest, InnerMergeIterator2) { + // Test Prev() when one child iterator is at its end. + db_iter_->SeekForPrev("g"); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "g"); + ASSERT_EQ(db_iter_->value().ToString(), "3"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "f"); + ASSERT_EQ(db_iter_->value().ToString(), "2"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "d"); + ASSERT_EQ(db_iter_->value().ToString(), "7"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "c"); + ASSERT_EQ(db_iter_->value().ToString(), "6"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "b"); + ASSERT_EQ(db_iter_->value().ToString(), "5"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "a"); + ASSERT_EQ(db_iter_->value().ToString(), "4"); +} + +TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace1) { + // Test Prev() when one child iterator is at its end but more rows + // are added. + db_iter_->Seek("f"); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "f"); + ASSERT_EQ(db_iter_->value().ToString(), "2"); + + // Test call back inserts a key in the end of the mem table after + // MergeIterator::Prev() realized the mem table iterator is at its end + // and before an SeekToLast() is called. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "MergeIterator::Prev:BeforePrev", + [&](void* /*arg*/) { internal_iter2_->Add("z", kTypeValue, "7", 12u); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "d"); + ASSERT_EQ(db_iter_->value().ToString(), "7"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "c"); + ASSERT_EQ(db_iter_->value().ToString(), "6"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "b"); + ASSERT_EQ(db_iter_->value().ToString(), "5"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "a"); + ASSERT_EQ(db_iter_->value().ToString(), "4"); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace2) { + // Test Prev() when one child iterator is at its end but more rows + // are added. + db_iter_->Seek("f"); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "f"); + ASSERT_EQ(db_iter_->value().ToString(), "2"); + + // Test call back inserts entries for update a key in the end of the + // mem table after MergeIterator::Prev() realized the mem tableiterator is at + // its end and before an SeekToLast() is called. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "MergeIterator::Prev:BeforePrev", [&](void* /*arg*/) { + internal_iter2_->Add("z", kTypeValue, "7", 12u); + internal_iter2_->Add("z", kTypeValue, "7", 11u); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "d"); + ASSERT_EQ(db_iter_->value().ToString(), "7"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "c"); + ASSERT_EQ(db_iter_->value().ToString(), "6"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "b"); + ASSERT_EQ(db_iter_->value().ToString(), "5"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "a"); + ASSERT_EQ(db_iter_->value().ToString(), "4"); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace3) { + // Test Prev() when one child iterator is at its end but more rows + // are added and max_skipped is triggered. + db_iter_->Seek("f"); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "f"); + ASSERT_EQ(db_iter_->value().ToString(), "2"); + + // Test call back inserts entries for update a key in the end of the + // mem table after MergeIterator::Prev() realized the mem table iterator is at + // its end and before an SeekToLast() is called. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "MergeIterator::Prev:BeforePrev", [&](void* /*arg*/) { + internal_iter2_->Add("z", kTypeValue, "7", 16u, true); + internal_iter2_->Add("z", kTypeValue, "7", 15u, true); + internal_iter2_->Add("z", kTypeValue, "7", 14u, true); + internal_iter2_->Add("z", kTypeValue, "7", 13u, true); + internal_iter2_->Add("z", kTypeValue, "7", 12u, true); + internal_iter2_->Add("z", kTypeValue, "7", 11u, true); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "d"); + ASSERT_EQ(db_iter_->value().ToString(), "7"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "c"); + ASSERT_EQ(db_iter_->value().ToString(), "6"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "b"); + ASSERT_EQ(db_iter_->value().ToString(), "5"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "a"); + ASSERT_EQ(db_iter_->value().ToString(), "4"); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace4) { + // Test Prev() when one child iterator has more rows inserted + // between Seek() and Prev() when changing directions. + internal_iter2_->Add("z", kTypeValue, "9", 4u); + + db_iter_->Seek("g"); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "g"); + ASSERT_EQ(db_iter_->value().ToString(), "3"); + + // Test call back inserts entries for update a key before "z" in + // mem table after MergeIterator::Prev() calls mem table iterator's + // Seek() and before calling Prev() + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "MergeIterator::Prev:BeforePrev", [&](void* arg) { + IteratorWrapper* it = reinterpret_cast(arg); + if (it->key().starts_with("z")) { + internal_iter2_->Add("x", kTypeValue, "7", 16u, true); + internal_iter2_->Add("x", kTypeValue, "7", 15u, true); + internal_iter2_->Add("x", kTypeValue, "7", 14u, true); + internal_iter2_->Add("x", kTypeValue, "7", 13u, true); + internal_iter2_->Add("x", kTypeValue, "7", 12u, true); + internal_iter2_->Add("x", kTypeValue, "7", 11u, true); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "f"); + ASSERT_EQ(db_iter_->value().ToString(), "2"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "d"); + ASSERT_EQ(db_iter_->value().ToString(), "7"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "c"); + ASSERT_EQ(db_iter_->value().ToString(), "6"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "b"); + ASSERT_EQ(db_iter_->value().ToString(), "5"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "a"); + ASSERT_EQ(db_iter_->value().ToString(), "4"); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace5) { + internal_iter2_->Add("z", kTypeValue, "9", 4u); + + // Test Prev() when one child iterator has more rows inserted + // between Seek() and Prev() when changing directions. + db_iter_->Seek("g"); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "g"); + ASSERT_EQ(db_iter_->value().ToString(), "3"); + + // Test call back inserts entries for update a key before "z" in + // mem table after MergeIterator::Prev() calls mem table iterator's + // Seek() and before calling Prev() + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "MergeIterator::Prev:BeforePrev", [&](void* arg) { + IteratorWrapper* it = reinterpret_cast(arg); + if (it->key().starts_with("z")) { + internal_iter2_->Add("x", kTypeValue, "7", 16u, true); + internal_iter2_->Add("x", kTypeValue, "7", 15u, true); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "f"); + ASSERT_EQ(db_iter_->value().ToString(), "2"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "d"); + ASSERT_EQ(db_iter_->value().ToString(), "7"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "c"); + ASSERT_EQ(db_iter_->value().ToString(), "6"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "b"); + ASSERT_EQ(db_iter_->value().ToString(), "5"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "a"); + ASSERT_EQ(db_iter_->value().ToString(), "4"); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace6) { + internal_iter2_->Add("z", kTypeValue, "9", 4u); + + // Test Prev() when one child iterator has more rows inserted + // between Seek() and Prev() when changing directions. + db_iter_->Seek("g"); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "g"); + ASSERT_EQ(db_iter_->value().ToString(), "3"); + + // Test call back inserts an entry for update a key before "z" in + // mem table after MergeIterator::Prev() calls mem table iterator's + // Seek() and before calling Prev() + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "MergeIterator::Prev:BeforePrev", [&](void* arg) { + IteratorWrapper* it = reinterpret_cast(arg); + if (it->key().starts_with("z")) { + internal_iter2_->Add("x", kTypeValue, "7", 16u, true); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "f"); + ASSERT_EQ(db_iter_->value().ToString(), "2"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "d"); + ASSERT_EQ(db_iter_->value().ToString(), "7"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "c"); + ASSERT_EQ(db_iter_->value().ToString(), "6"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "b"); + ASSERT_EQ(db_iter_->value().ToString(), "5"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "a"); + ASSERT_EQ(db_iter_->value().ToString(), "4"); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace7) { + internal_iter1_->Add("u", kTypeValue, "10", 4u); + internal_iter1_->Add("v", kTypeValue, "11", 4u); + internal_iter1_->Add("w", kTypeValue, "12", 4u); + internal_iter2_->Add("z", kTypeValue, "9", 4u); + + // Test Prev() when one child iterator has more rows inserted + // between Seek() and Prev() when changing directions. + db_iter_->Seek("g"); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "g"); + ASSERT_EQ(db_iter_->value().ToString(), "3"); + + // Test call back inserts entries for update a key before "z" in + // mem table after MergeIterator::Prev() calls mem table iterator's + // Seek() and before calling Prev() + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "MergeIterator::Prev:BeforePrev", [&](void* arg) { + IteratorWrapper* it = reinterpret_cast(arg); + if (it->key().starts_with("z")) { + internal_iter2_->Add("x", kTypeValue, "7", 16u, true); + internal_iter2_->Add("x", kTypeValue, "7", 15u, true); + internal_iter2_->Add("x", kTypeValue, "7", 14u, true); + internal_iter2_->Add("x", kTypeValue, "7", 13u, true); + internal_iter2_->Add("x", kTypeValue, "7", 12u, true); + internal_iter2_->Add("x", kTypeValue, "7", 11u, true); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "f"); + ASSERT_EQ(db_iter_->value().ToString(), "2"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "d"); + ASSERT_EQ(db_iter_->value().ToString(), "7"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "c"); + ASSERT_EQ(db_iter_->value().ToString(), "6"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "b"); + ASSERT_EQ(db_iter_->value().ToString(), "5"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "a"); + ASSERT_EQ(db_iter_->value().ToString(), "4"); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace8) { + // internal_iter1_: a, f, g + // internal_iter2_: a, b, c, d, adding (z) + internal_iter2_->Add("z", kTypeValue, "9", 4u); + + // Test Prev() when one child iterator has more rows inserted + // between Seek() and Prev() when changing directions. + db_iter_->Seek("g"); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "g"); + ASSERT_EQ(db_iter_->value().ToString(), "3"); + + // Test call back inserts two keys before "z" in mem table after + // MergeIterator::Prev() calls mem table iterator's Seek() and + // before calling Prev() + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "MergeIterator::Prev:BeforePrev", [&](void* arg) { + IteratorWrapper* it = reinterpret_cast(arg); + if (it->key().starts_with("z")) { + internal_iter2_->Add("x", kTypeValue, "7", 16u, true); + internal_iter2_->Add("y", kTypeValue, "7", 17u, true); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "f"); + ASSERT_EQ(db_iter_->value().ToString(), "2"); + db_iter_->Prev(); + ASSERT_TRUE(db_iter_->Valid()); + ASSERT_EQ(db_iter_->key().ToString(), "d"); + ASSERT_EQ(db_iter_->value().ToString(), "7"); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBIteratorTest, SeekPrefixTombstones) { + ReadOptions ro; + Options options; + options.prefix_extractor.reset(NewNoopTransform()); + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddDeletion("b"); + internal_iter->AddDeletion("c"); + internal_iter->AddDeletion("d"); + internal_iter->AddDeletion("e"); + internal_iter->AddDeletion("f"); + internal_iter->AddDeletion("g"); + internal_iter->Finish(); + + ro.prefix_same_as_start = true; + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ImmutableOptions(options), MutableCFOptions(options), + BytewiseComparator(), internal_iter, nullptr /* version */, + 10 /* sequence */, options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + int skipped_keys = 0; + + get_perf_context()->Reset(); + db_iter->SeekForPrev("z"); + skipped_keys = + static_cast(get_perf_context()->internal_key_skipped_count); + ASSERT_EQ(skipped_keys, 0); + + get_perf_context()->Reset(); + db_iter->Seek("a"); + skipped_keys = + static_cast(get_perf_context()->internal_key_skipped_count); + ASSERT_EQ(skipped_keys, 0); +} + +TEST_F(DBIteratorTest, SeekToFirstLowerBound) { + const int kNumKeys = 3; + for (int i = 0; i < kNumKeys + 2; ++i) { + // + 2 for two special cases: lower bound before and lower bound after the + // internal iterator's keys + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + for (int j = 1; j <= kNumKeys; ++j) { + internal_iter->AddPut(std::to_string(j), "val"); + } + internal_iter->Finish(); + + ReadOptions ro; + auto lower_bound_str = std::to_string(i); + Slice lower_bound(lower_bound_str); + ro.iterate_lower_bound = &lower_bound; + Options options; + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ImmutableOptions(options), MutableCFOptions(options), + BytewiseComparator(), internal_iter, nullptr /* version */, + 10 /* sequence */, options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->SeekToFirst(); + if (i == kNumKeys + 1) { + // lower bound was beyond the last key + ASSERT_FALSE(db_iter->Valid()); + ASSERT_OK(db_iter->status()); + } else { + ASSERT_TRUE(db_iter->Valid()); + int expected; + if (i == 0) { + // lower bound was before the first key + expected = 1; + } else { + // lower bound was at the ith key + expected = i; + } + ASSERT_EQ(std::to_string(expected), db_iter->key().ToString()); + } + } +} + +TEST_F(DBIteratorTest, PrevLowerBound) { + const int kNumKeys = 3; + const int kLowerBound = 2; + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + for (int j = 1; j <= kNumKeys; ++j) { + internal_iter->AddPut(std::to_string(j), "val"); + } + internal_iter->Finish(); + + ReadOptions ro; + auto lower_bound_str = std::to_string(kLowerBound); + Slice lower_bound(lower_bound_str); + ro.iterate_lower_bound = &lower_bound; + Options options; + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ImmutableOptions(options), MutableCFOptions(options), + BytewiseComparator(), internal_iter, nullptr /* version */, + 10 /* sequence */, options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->SeekToLast(); + for (int i = kNumKeys; i >= kLowerBound; --i) { + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(std::to_string(i), db_iter->key().ToString()); + db_iter->Prev(); + } + ASSERT_FALSE(db_iter->Valid()); +} + +TEST_F(DBIteratorTest, SeekLessLowerBound) { + const int kNumKeys = 3; + const int kLowerBound = 2; + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + for (int j = 1; j <= kNumKeys; ++j) { + internal_iter->AddPut(std::to_string(j), "val"); + } + internal_iter->Finish(); + + ReadOptions ro; + auto lower_bound_str = std::to_string(kLowerBound); + Slice lower_bound(lower_bound_str); + ro.iterate_lower_bound = &lower_bound; + Options options; + std::unique_ptr db_iter(NewDBIterator( + env_, ro, ImmutableOptions(options), MutableCFOptions(options), + BytewiseComparator(), internal_iter, nullptr /* version */, + 10 /* sequence */, options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + auto before_lower_bound_str = std::to_string(kLowerBound - 1); + Slice before_lower_bound(lower_bound_str); + + db_iter->Seek(before_lower_bound); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_EQ(lower_bound_str, db_iter->key().ToString()); +} + +TEST_F(DBIteratorTest, ReverseToForwardWithDisappearingKeys) { + Options options; + options.prefix_extractor.reset(NewCappedPrefixTransform(0)); + + TestIterator* internal_iter = new TestIterator(BytewiseComparator()); + internal_iter->AddPut("a", "A"); + internal_iter->AddPut("b", "B"); + for (int i = 0; i < 100; ++i) { + internal_iter->AddPut("c" + std::to_string(i), ""); + } + internal_iter->Finish(); + + std::unique_ptr db_iter(NewDBIterator( + env_, ReadOptions(), ImmutableOptions(options), MutableCFOptions(options), + BytewiseComparator(), internal_iter, nullptr /* version */, + 10 /* sequence */, options.max_sequential_skip_in_iterations, + nullptr /* read_callback */)); + + db_iter->SeekForPrev("a"); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_OK(db_iter->status()); + ASSERT_EQ("a", db_iter->key().ToString()); + + internal_iter->Vanish("a"); + db_iter->Next(); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_OK(db_iter->status()); + ASSERT_EQ("b", db_iter->key().ToString()); + + // A (sort of) bug used to cause DBIter to pointlessly drag the internal + // iterator all the way to the end. But this doesn't really matter at the time + // of writing because the only iterator that can see disappearing keys is + // ForwardIterator, which doesn't support SeekForPrev(). + EXPECT_LT(internal_iter->steps(), 20); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_iterator_test.cc b/librocksdb-sys/rocksdb/db/db_iterator_test.cc new file mode 100644 index 0000000..862377b --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_iterator_test.cc @@ -0,0 +1,3304 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include + +#include "db/arena_wrapped_db_iter.h" +#include "db/db_iter.h" +#include "db/db_test_util.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/iostats_context.h" +#include "rocksdb/perf_context.h" +#include "table/block_based/flush_block_policy_impl.h" +#include "util/random.h" +#include "utilities/merge_operators/string_append/stringappend2.h" + +namespace ROCKSDB_NAMESPACE { + +// A dumb ReadCallback which saying every key is committed. +class DummyReadCallback : public ReadCallback { + public: + DummyReadCallback() : ReadCallback(kMaxSequenceNumber) {} + bool IsVisibleFullCheck(SequenceNumber /*seq*/) override { return true; } + void SetSnapshot(SequenceNumber seq) { max_visible_seq_ = seq; } +}; + +class DBIteratorBaseTest : public DBTestBase { + public: + DBIteratorBaseTest() + : DBTestBase("db_iterator_test", /*env_do_fsync=*/true) {} +}; + +TEST_F(DBIteratorBaseTest, APICallsWithPerfContext) { + // Set up the DB + Options options = CurrentOptions(); + DestroyAndReopen(options); + Random rnd(301); + for (int i = 1; i <= 3; i++) { + ASSERT_OK(Put(std::to_string(i), std::to_string(i))); + } + + // Setup iterator and PerfContext + Iterator* iter = db_->NewIterator(ReadOptions()); + std::string key_str = std::to_string(2); + Slice key(key_str); + SetPerfLevel(kEnableCount); + get_perf_context()->Reset(); + + // Initial PerfContext counters + ASSERT_EQ(0, get_perf_context()->iter_seek_count); + ASSERT_EQ(0, get_perf_context()->iter_next_count); + ASSERT_EQ(0, get_perf_context()->iter_prev_count); + + // Test Seek-related API calls PerfContext counter + iter->Seek(key); + iter->SeekToFirst(); + iter->SeekToLast(); + iter->SeekForPrev(key); + ASSERT_EQ(4, get_perf_context()->iter_seek_count); + ASSERT_EQ(0, get_perf_context()->iter_next_count); + ASSERT_EQ(0, get_perf_context()->iter_prev_count); + + // Test Next() calls PerfContext counter + iter->Next(); + ASSERT_EQ(4, get_perf_context()->iter_seek_count); + ASSERT_EQ(1, get_perf_context()->iter_next_count); + ASSERT_EQ(0, get_perf_context()->iter_prev_count); + + // Test Prev() calls PerfContext counter + iter->Prev(); + ASSERT_EQ(4, get_perf_context()->iter_seek_count); + ASSERT_EQ(1, get_perf_context()->iter_next_count); + ASSERT_EQ(1, get_perf_context()->iter_prev_count); + + delete iter; +} + +// Test param: +// bool: whether to pass read_callback to NewIterator(). +class DBIteratorTest : public DBIteratorBaseTest, + public testing::WithParamInterface { + public: + DBIteratorTest() {} + + Iterator* NewIterator(const ReadOptions& read_options, + ColumnFamilyHandle* column_family = nullptr) { + if (column_family == nullptr) { + column_family = db_->DefaultColumnFamily(); + } + auto* cfd = + static_cast_with_check(column_family)->cfd(); + SequenceNumber seq = read_options.snapshot != nullptr + ? read_options.snapshot->GetSequenceNumber() + : db_->GetLatestSequenceNumber(); + bool use_read_callback = GetParam(); + DummyReadCallback* read_callback = nullptr; + if (use_read_callback) { + read_callback = new DummyReadCallback(); + read_callback->SetSnapshot(seq); + InstrumentedMutexLock lock(&mutex_); + read_callbacks_.push_back( + std::unique_ptr(read_callback)); + } + return dbfull()->NewIteratorImpl(read_options, cfd, seq, read_callback); + } + + private: + InstrumentedMutex mutex_; + std::vector> read_callbacks_; +}; + +TEST_P(DBIteratorTest, IteratorProperty) { + // The test needs to be changed if kPersistedTier is supported in iterator. + Options options = CurrentOptions(); + CreateAndReopenWithCF({"pikachu"}, options); + ASSERT_OK(Put(1, "1", "2")); + ASSERT_OK(Delete(1, "2")); + ReadOptions ropt; + ropt.pin_data = false; + { + std::unique_ptr iter(NewIterator(ropt, handles_[1])); + iter->SeekToFirst(); + std::string prop_value; + ASSERT_NOK(iter->GetProperty("non_existing.value", &prop_value)); + ASSERT_OK(iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value)); + ASSERT_EQ("0", prop_value); + ASSERT_OK(iter->GetProperty("rocksdb.iterator.internal-key", &prop_value)); + ASSERT_EQ("1", prop_value); + iter->Next(); + ASSERT_OK(iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value)); + ASSERT_EQ("Iterator is not valid.", prop_value); + + // Get internal key at which the iteration stopped (tombstone in this case). + ASSERT_OK(iter->GetProperty("rocksdb.iterator.internal-key", &prop_value)); + ASSERT_EQ("2", prop_value); + } + Close(); +} + +TEST_P(DBIteratorTest, PersistedTierOnIterator) { + // The test needs to be changed if kPersistedTier is supported in iterator. + Options options = CurrentOptions(); + CreateAndReopenWithCF({"pikachu"}, options); + ReadOptions ropt; + ropt.read_tier = kPersistedTier; + + auto* iter = db_->NewIterator(ropt, handles_[1]); + ASSERT_TRUE(iter->status().IsNotSupported()); + delete iter; + + std::vector iters; + ASSERT_TRUE(db_->NewIterators(ropt, {handles_[1]}, &iters).IsNotSupported()); + Close(); +} + +TEST_P(DBIteratorTest, NonBlockingIteration) { + do { + ReadOptions non_blocking_opts, regular_opts; + anon::OptionsOverride options_override; + options_override.full_block_cache = true; + Options options = CurrentOptions(options_override); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + non_blocking_opts.read_tier = kBlockCacheTier; + + CreateAndReopenWithCF({"pikachu"}, options); + // write one kv to the database. + ASSERT_OK(Put(1, "a", "b")); + + // scan using non-blocking iterator. We should find it because + // it is in memtable. + Iterator* iter = NewIterator(non_blocking_opts, handles_[1]); + int count = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_OK(iter->status()); + count++; + } + ASSERT_EQ(count, 1); + delete iter; + + // flush memtable to storage. Now, the key should not be in the + // memtable neither in the block cache. + ASSERT_OK(Flush(1)); + + // verify that a non-blocking iterator does not find any + // kvs. Neither does it do any IOs to storage. + uint64_t numopen = TestGetTickerCount(options, NO_FILE_OPENS); + uint64_t cache_added = TestGetTickerCount(options, BLOCK_CACHE_ADD); + iter = NewIterator(non_blocking_opts, handles_[1]); + count = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + count++; + } + ASSERT_EQ(count, 0); + ASSERT_TRUE(iter->status().IsIncomplete()); + ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS)); + ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD)); + delete iter; + + // read in the specified block via a regular get + ASSERT_EQ(Get(1, "a"), "b"); + + // verify that we can find it via a non-blocking scan + numopen = TestGetTickerCount(options, NO_FILE_OPENS); + cache_added = TestGetTickerCount(options, BLOCK_CACHE_ADD); + iter = NewIterator(non_blocking_opts, handles_[1]); + count = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_OK(iter->status()); + count++; + } + ASSERT_EQ(count, 1); + ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS)); + ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD)); + delete iter; + + // This test verifies block cache behaviors, which is not used by plain + // table format. + } while (ChangeOptions(kSkipPlainTable | kSkipNoSeekToLast | kSkipMmapReads)); +} + +TEST_P(DBIteratorTest, IterSeekBeforePrev) { + ASSERT_OK(Put("a", "b")); + ASSERT_OK(Put("c", "d")); + EXPECT_OK(dbfull()->Flush(FlushOptions())); + ASSERT_OK(Put("0", "f")); + ASSERT_OK(Put("1", "h")); + EXPECT_OK(dbfull()->Flush(FlushOptions())); + ASSERT_OK(Put("2", "j")); + auto iter = NewIterator(ReadOptions()); + iter->Seek(Slice("c")); + iter->Prev(); + iter->Seek(Slice("a")); + iter->Prev(); + delete iter; +} + +TEST_P(DBIteratorTest, IterReseekNewUpperBound) { + Random rnd(301); + Options options = CurrentOptions(); + BlockBasedTableOptions table_options; + table_options.block_size = 1024; + table_options.block_size_deviation = 50; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.compression = kNoCompression; + Reopen(options); + + ASSERT_OK(Put("a", rnd.RandomString(400))); + ASSERT_OK(Put("aabb", rnd.RandomString(400))); + ASSERT_OK(Put("aaef", rnd.RandomString(400))); + ASSERT_OK(Put("b", rnd.RandomString(400))); + EXPECT_OK(dbfull()->Flush(FlushOptions())); + ReadOptions opts; + Slice ub = Slice("aa"); + opts.iterate_upper_bound = &ub; + auto iter = NewIterator(opts); + iter->Seek(Slice("a")); + ub = Slice("b"); + iter->Seek(Slice("aabc")); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().ToString(), "aaef"); + delete iter; +} + +TEST_P(DBIteratorTest, IterSeekForPrevBeforeNext) { + ASSERT_OK(Put("a", "b")); + ASSERT_OK(Put("c", "d")); + EXPECT_OK(dbfull()->Flush(FlushOptions())); + ASSERT_OK(Put("0", "f")); + ASSERT_OK(Put("1", "h")); + EXPECT_OK(dbfull()->Flush(FlushOptions())); + ASSERT_OK(Put("2", "j")); + auto iter = NewIterator(ReadOptions()); + iter->SeekForPrev(Slice("0")); + iter->Next(); + iter->SeekForPrev(Slice("1")); + iter->Next(); + delete iter; +} + +namespace { +std::string MakeLongKey(size_t length, char c) { + return std::string(length, c); +} +} // anonymous namespace + +TEST_P(DBIteratorTest, IterLongKeys) { + ASSERT_OK(Put(MakeLongKey(20, 0), "0")); + ASSERT_OK(Put(MakeLongKey(32, 2), "2")); + ASSERT_OK(Put("a", "b")); + EXPECT_OK(dbfull()->Flush(FlushOptions())); + ASSERT_OK(Put(MakeLongKey(50, 1), "1")); + ASSERT_OK(Put(MakeLongKey(127, 3), "3")); + ASSERT_OK(Put(MakeLongKey(64, 4), "4")); + auto iter = NewIterator(ReadOptions()); + + // Create a key that needs to be skipped for Seq too new + iter->Seek(MakeLongKey(20, 0)); + ASSERT_EQ(IterStatus(iter), MakeLongKey(20, 0) + "->0"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), MakeLongKey(50, 1) + "->1"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), MakeLongKey(32, 2) + "->2"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), MakeLongKey(127, 3) + "->3"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), MakeLongKey(64, 4) + "->4"); + + iter->SeekForPrev(MakeLongKey(127, 3)); + ASSERT_EQ(IterStatus(iter), MakeLongKey(127, 3) + "->3"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), MakeLongKey(32, 2) + "->2"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), MakeLongKey(50, 1) + "->1"); + delete iter; + + iter = NewIterator(ReadOptions()); + iter->Seek(MakeLongKey(50, 1)); + ASSERT_EQ(IterStatus(iter), MakeLongKey(50, 1) + "->1"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), MakeLongKey(32, 2) + "->2"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), MakeLongKey(127, 3) + "->3"); + delete iter; +} + +TEST_P(DBIteratorTest, IterNextWithNewerSeq) { + ASSERT_OK(Put("0", "0")); + EXPECT_OK(dbfull()->Flush(FlushOptions())); + ASSERT_OK(Put("a", "b")); + ASSERT_OK(Put("c", "d")); + ASSERT_OK(Put("d", "e")); + auto iter = NewIterator(ReadOptions()); + + // Create a key that needs to be skipped for Seq too new + for (uint64_t i = 0; i < last_options_.max_sequential_skip_in_iterations + 1; + i++) { + ASSERT_OK(Put("b", "f")); + } + + iter->Seek(Slice("a")); + ASSERT_EQ(IterStatus(iter), "a->b"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "c->d"); + iter->SeekForPrev(Slice("b")); + ASSERT_EQ(IterStatus(iter), "a->b"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "c->d"); + + delete iter; +} + +TEST_P(DBIteratorTest, IterPrevWithNewerSeq) { + ASSERT_OK(Put("0", "0")); + EXPECT_OK(dbfull()->Flush(FlushOptions())); + ASSERT_OK(Put("a", "b")); + ASSERT_OK(Put("c", "d")); + ASSERT_OK(Put("d", "e")); + auto iter = NewIterator(ReadOptions()); + + // Create a key that needs to be skipped for Seq too new + for (uint64_t i = 0; i < last_options_.max_sequential_skip_in_iterations + 1; + i++) { + ASSERT_OK(Put("b", "f")); + } + + iter->Seek(Slice("d")); + ASSERT_EQ(IterStatus(iter), "d->e"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "c->d"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "a->b"); + iter->Prev(); + iter->SeekForPrev(Slice("d")); + ASSERT_EQ(IterStatus(iter), "d->e"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "c->d"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "a->b"); + iter->Prev(); + delete iter; +} + +TEST_P(DBIteratorTest, IterPrevWithNewerSeq2) { + ASSERT_OK(Put("0", "0")); + EXPECT_OK(dbfull()->Flush(FlushOptions())); + ASSERT_OK(Put("a", "b")); + ASSERT_OK(Put("c", "d")); + ASSERT_OK(Put("e", "f")); + auto iter = NewIterator(ReadOptions()); + auto iter2 = NewIterator(ReadOptions()); + iter->Seek(Slice("c")); + iter2->SeekForPrev(Slice("d")); + ASSERT_EQ(IterStatus(iter), "c->d"); + ASSERT_EQ(IterStatus(iter2), "c->d"); + + // Create a key that needs to be skipped for Seq too new + for (uint64_t i = 0; i < last_options_.max_sequential_skip_in_iterations + 1; + i++) { + ASSERT_OK(Put("b", "f")); + } + + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "a->b"); + iter->Prev(); + iter2->Prev(); + ASSERT_EQ(IterStatus(iter2), "a->b"); + iter2->Prev(); + delete iter; + delete iter2; +} + +TEST_P(DBIteratorTest, IterEmpty) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + Iterator* iter = NewIterator(ReadOptions(), handles_[1]); + + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->Seek("foo"); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->SeekForPrev("foo"); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + ASSERT_OK(iter->status()); + + delete iter; + } while (ChangeCompactOptions()); +} + +TEST_P(DBIteratorTest, IterSingle) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "a", "va")); + Iterator* iter = NewIterator(ReadOptions(), handles_[1]); + + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->Seek(""); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + iter->SeekForPrev(""); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->Seek("a"); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + iter->SeekForPrev("a"); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->Seek("b"); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + iter->SeekForPrev("b"); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + delete iter; + } while (ChangeCompactOptions()); +} + +TEST_P(DBIteratorTest, IterMulti) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "a", "va")); + ASSERT_OK(Put(1, "b", "vb")); + ASSERT_OK(Put(1, "c", "vc")); + Iterator* iter = NewIterator(ReadOptions(), handles_[1]); + + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "b->vb"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "b->vb"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->Seek(""); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Seek("a"); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Seek("ax"); + ASSERT_EQ(IterStatus(iter), "b->vb"); + iter->SeekForPrev("d"); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->SeekForPrev("c"); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->SeekForPrev("bx"); + ASSERT_EQ(IterStatus(iter), "b->vb"); + + iter->Seek("b"); + ASSERT_EQ(IterStatus(iter), "b->vb"); + iter->Seek("z"); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + iter->SeekForPrev("b"); + ASSERT_EQ(IterStatus(iter), "b->vb"); + iter->SeekForPrev(""); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + // Switch from reverse to forward + iter->SeekToLast(); + iter->Prev(); + iter->Prev(); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "b->vb"); + + // Switch from forward to reverse + iter->SeekToFirst(); + iter->Next(); + iter->Next(); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "b->vb"); + + // Make sure iter stays at snapshot + ASSERT_OK(Put(1, "a", "va2")); + ASSERT_OK(Put(1, "a2", "va3")); + ASSERT_OK(Put(1, "b", "vb2")); + ASSERT_OK(Put(1, "c", "vc2")); + ASSERT_OK(Delete(1, "b")); + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "b->vb"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "b->vb"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + delete iter; + } while (ChangeCompactOptions()); +} + +// Check that we can skip over a run of user keys +// by using reseek rather than sequential scan +TEST_P(DBIteratorTest, IterReseek) { + anon::OptionsOverride options_override; + options_override.skip_policy = kSkipNoSnapshot; + Options options = CurrentOptions(options_override); + options.max_sequential_skip_in_iterations = 3; + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // insert three keys with same userkey and verify that + // reseek is not invoked. For each of these test cases, + // verify that we can find the next key "b". + ASSERT_OK(Put(1, "a", "zero")); + ASSERT_OK(Put(1, "a", "one")); + ASSERT_OK(Put(1, "a", "two")); + ASSERT_OK(Put(1, "b", "bone")); + Iterator* iter = NewIterator(ReadOptions(), handles_[1]); + iter->SeekToFirst(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 0); + ASSERT_EQ(IterStatus(iter), "a->two"); + iter->Next(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 0); + ASSERT_EQ(IterStatus(iter), "b->bone"); + delete iter; + + // insert a total of three keys with same userkey and verify + // that reseek is still not invoked. + ASSERT_OK(Put(1, "a", "three")); + iter = NewIterator(ReadOptions(), handles_[1]); + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "a->three"); + iter->Next(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 0); + ASSERT_EQ(IterStatus(iter), "b->bone"); + delete iter; + + // insert a total of four keys with same userkey and verify + // that reseek is invoked. + ASSERT_OK(Put(1, "a", "four")); + iter = NewIterator(ReadOptions(), handles_[1]); + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "a->four"); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 0); + iter->Next(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 1); + ASSERT_EQ(IterStatus(iter), "b->bone"); + delete iter; + + // Testing reverse iterator + // At this point, we have three versions of "a" and one version of "b". + // The reseek statistics is already at 1. + int num_reseeks = static_cast( + TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION)); + + // Insert another version of b and assert that reseek is not invoked + ASSERT_OK(Put(1, "b", "btwo")); + iter = NewIterator(ReadOptions(), handles_[1]); + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "b->btwo"); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), + num_reseeks); + iter->Prev(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), + num_reseeks + 1); + ASSERT_EQ(IterStatus(iter), "a->four"); + delete iter; + + // insert two more versions of b. This makes a total of 4 versions + // of b and 4 versions of a. + ASSERT_OK(Put(1, "b", "bthree")); + ASSERT_OK(Put(1, "b", "bfour")); + iter = NewIterator(ReadOptions(), handles_[1]); + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "b->bfour"); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), + num_reseeks + 2); + iter->Prev(); + + // the previous Prev call should have invoked reseek + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), + num_reseeks + 3); + ASSERT_EQ(IterStatus(iter), "a->four"); + delete iter; +} + +TEST_F(DBIteratorTest, ReseekUponDirectionChange) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.prefix_extractor.reset(NewFixedPrefixTransform(1)); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.merge_operator.reset( + new StringAppendTESTOperator(/*delim_char=*/' ')); + DestroyAndReopen(options); + ASSERT_OK(Put("foo", "value")); + ASSERT_OK(Put("bar", "value")); + { + std::unique_ptr it(db_->NewIterator(ReadOptions())); + it->SeekToLast(); + it->Prev(); + it->Next(); + } + ASSERT_EQ(1, + options.statistics->getTickerCount(NUMBER_OF_RESEEKS_IN_ITERATION)); + + const std::string merge_key("good"); + ASSERT_OK(Put(merge_key, "orig")); + ASSERT_OK(Merge(merge_key, "suffix")); + { + std::unique_ptr it(db_->NewIterator(ReadOptions())); + it->Seek(merge_key); + ASSERT_TRUE(it->Valid()); + const uint64_t prev_reseek_count = + options.statistics->getTickerCount(NUMBER_OF_RESEEKS_IN_ITERATION); + it->Prev(); + ASSERT_EQ(prev_reseek_count + 1, options.statistics->getTickerCount( + NUMBER_OF_RESEEKS_IN_ITERATION)); + } +} + +TEST_P(DBIteratorTest, IterSmallAndLargeMix) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "a", "va")); + ASSERT_OK(Put(1, "b", std::string(100000, 'b'))); + ASSERT_OK(Put(1, "c", "vc")); + ASSERT_OK(Put(1, "d", std::string(100000, 'd'))); + ASSERT_OK(Put(1, "e", std::string(100000, 'e'))); + + Iterator* iter = NewIterator(ReadOptions(), handles_[1]); + + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "b->" + std::string(100000, 'b')); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "d->" + std::string(100000, 'd')); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "e->" + std::string(100000, 'e')); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), "e->" + std::string(100000, 'e')); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "d->" + std::string(100000, 'd')); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "b->" + std::string(100000, 'b')); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + delete iter; + } while (ChangeCompactOptions()); +} + +TEST_P(DBIteratorTest, IterMultiWithDelete) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "ka", "va")); + ASSERT_OK(Put(1, "kb", "vb")); + ASSERT_OK(Put(1, "kc", "vc")); + ASSERT_OK(Delete(1, "kb")); + ASSERT_EQ("NOT_FOUND", Get(1, "kb")); + + Iterator* iter = NewIterator(ReadOptions(), handles_[1]); + iter->Seek("kc"); + ASSERT_EQ(IterStatus(iter), "kc->vc"); + if (!CurrentOptions().merge_operator) { + // TODO: merge operator does not support backward iteration yet + if (kPlainTableAllBytesPrefix != option_config_ && + kBlockBasedTableWithWholeKeyHashIndex != option_config_ && + kHashLinkList != option_config_ && + kHashSkipList != option_config_) { // doesn't support SeekToLast + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "ka->va"); + } + } + delete iter; + } while (ChangeOptions()); +} + +TEST_P(DBIteratorTest, IterPrevMaxSkip) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + for (int i = 0; i < 2; i++) { + ASSERT_OK(Put(1, "key1", "v1")); + ASSERT_OK(Put(1, "key2", "v2")); + ASSERT_OK(Put(1, "key3", "v3")); + ASSERT_OK(Put(1, "key4", "v4")); + ASSERT_OK(Put(1, "key5", "v5")); + } + + VerifyIterLast("key5->v5", 1); + + ASSERT_OK(Delete(1, "key5")); + VerifyIterLast("key4->v4", 1); + + ASSERT_OK(Delete(1, "key4")); + VerifyIterLast("key3->v3", 1); + + ASSERT_OK(Delete(1, "key3")); + VerifyIterLast("key2->v2", 1); + + ASSERT_OK(Delete(1, "key2")); + VerifyIterLast("key1->v1", 1); + + ASSERT_OK(Delete(1, "key1")); + VerifyIterLast("(invalid)", 1); + } while (ChangeOptions(kSkipMergePut | kSkipNoSeekToLast)); +} + +TEST_P(DBIteratorTest, IterWithSnapshot) { + anon::OptionsOverride options_override; + options_override.skip_policy = kSkipNoSnapshot; + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions(options_override)); + ASSERT_OK(Put(1, "key1", "val1")); + ASSERT_OK(Put(1, "key2", "val2")); + ASSERT_OK(Put(1, "key3", "val3")); + ASSERT_OK(Put(1, "key4", "val4")); + ASSERT_OK(Put(1, "key5", "val5")); + + const Snapshot* snapshot = db_->GetSnapshot(); + ReadOptions options; + options.snapshot = snapshot; + Iterator* iter = NewIterator(options, handles_[1]); + + ASSERT_OK(Put(1, "key0", "val0")); + // Put more values after the snapshot + ASSERT_OK(Put(1, "key100", "val100")); + ASSERT_OK(Put(1, "key101", "val101")); + + iter->Seek("key5"); + ASSERT_EQ(IterStatus(iter), "key5->val5"); + if (!CurrentOptions().merge_operator) { + // TODO: merge operator does not support backward iteration yet + if (kPlainTableAllBytesPrefix != option_config_ && + kBlockBasedTableWithWholeKeyHashIndex != option_config_ && + kHashLinkList != option_config_ && kHashSkipList != option_config_) { + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "key4->val4"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "key3->val3"); + + iter->Next(); + ASSERT_EQ(IterStatus(iter), "key4->val4"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "key5->val5"); + } + iter->Next(); + ASSERT_TRUE(!iter->Valid()); + } + + if (!CurrentOptions().merge_operator) { + // TODO(gzh): merge operator does not support backward iteration yet + if (kPlainTableAllBytesPrefix != option_config_ && + kBlockBasedTableWithWholeKeyHashIndex != option_config_ && + kHashLinkList != option_config_ && kHashSkipList != option_config_) { + iter->SeekForPrev("key1"); + ASSERT_EQ(IterStatus(iter), "key1->val1"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "key2->val2"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "key3->val3"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "key2->val2"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "key1->val1"); + iter->Prev(); + ASSERT_TRUE(!iter->Valid()); + } + } + db_->ReleaseSnapshot(snapshot); + delete iter; + } while (ChangeOptions()); +} + +TEST_P(DBIteratorTest, IteratorPinsRef) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "foo", "hello")); + + // Get iterator that will yield the current contents of the DB. + Iterator* iter = NewIterator(ReadOptions(), handles_[1]); + + // Write to force compactions + ASSERT_OK(Put(1, "foo", "newvalue1")); + for (int i = 0; i < 100; i++) { + // 100K values + ASSERT_OK(Put(1, Key(i), Key(i) + std::string(100000, 'v'))); + } + ASSERT_OK(Put(1, "foo", "newvalue2")); + + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("foo", iter->key().ToString()); + ASSERT_EQ("hello", iter->value().ToString()); + iter->Next(); + ASSERT_TRUE(!iter->Valid()); + delete iter; + } while (ChangeCompactOptions()); +} + +TEST_P(DBIteratorTest, IteratorDeleteAfterCfDelete) { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + + ASSERT_OK(Put(1, "foo", "delete-cf-then-delete-iter")); + ASSERT_OK(Put(1, "hello", "value2")); + + ColumnFamilyHandle* cf = handles_[1]; + ReadOptions ro; + + auto* iter = db_->NewIterator(ro, cf); + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "foo->delete-cf-then-delete-iter"); + + // delete CF handle + EXPECT_OK(db_->DestroyColumnFamilyHandle(cf)); + handles_.erase(std::begin(handles_) + 1); + + // delete Iterator after CF handle is deleted + iter->Next(); + ASSERT_EQ(IterStatus(iter), "hello->value2"); + delete iter; +} + +TEST_P(DBIteratorTest, IteratorDeleteAfterCfDrop) { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + + ASSERT_OK(Put(1, "foo", "drop-cf-then-delete-iter")); + + ReadOptions ro; + ColumnFamilyHandle* cf = handles_[1]; + + auto* iter = db_->NewIterator(ro, cf); + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "foo->drop-cf-then-delete-iter"); + + // drop and delete CF + EXPECT_OK(db_->DropColumnFamily(cf)); + EXPECT_OK(db_->DestroyColumnFamilyHandle(cf)); + handles_.erase(std::begin(handles_) + 1); + + // delete Iterator after CF handle is dropped + delete iter; +} + +// SetOptions not defined in ROCKSDB LITE +TEST_P(DBIteratorTest, DBIteratorBoundTest) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + + options.prefix_extractor = nullptr; + DestroyAndReopen(options); + ASSERT_OK(Put("a", "0")); + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Put("foo1", "bar1")); + ASSERT_OK(Put("g1", "0")); + + // testing basic case with no iterate_upper_bound and no prefix_extractor + { + ReadOptions ro; + ro.iterate_upper_bound = nullptr; + + std::unique_ptr iter(NewIterator(ro)); + + iter->Seek("foo"); + + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("foo")), 0); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("foo1")), 0); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("g1")), 0); + + iter->SeekForPrev("g1"); + + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("g1")), 0); + + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("foo1")), 0); + + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("foo")), 0); + } + + // testing iterate_upper_bound and forward iterator + // to make sure it stops at bound + { + ReadOptions ro; + // iterate_upper_bound points beyond the last expected entry + Slice prefix("foo2"); + ro.iterate_upper_bound = &prefix; + + std::unique_ptr iter(NewIterator(ro)); + + iter->Seek("foo"); + + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("foo")), 0); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(("foo1")), 0); + + iter->Next(); + // should stop here... + ASSERT_TRUE(!iter->Valid()); + } + // Testing SeekToLast with iterate_upper_bound set + { + ReadOptions ro; + + Slice prefix("foo"); + ro.iterate_upper_bound = &prefix; + + std::unique_ptr iter(NewIterator(ro)); + + iter->SeekToLast(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("a")), 0); + } + + // prefix is the first letter of the key + ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "fixed:1"}})); + ASSERT_OK(Put("a", "0")); + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Put("foo1", "bar1")); + ASSERT_OK(Put("g1", "0")); + + // testing with iterate_upper_bound and prefix_extractor + // Seek target and iterate_upper_bound are not is same prefix + // This should be an error + { + ReadOptions ro; + Slice upper_bound("g"); + ro.iterate_upper_bound = &upper_bound; + + std::unique_ptr iter(NewIterator(ro)); + + iter->Seek("foo"); + + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("foo", iter->key().ToString()); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("foo1", iter->key().ToString()); + + iter->Next(); + ASSERT_TRUE(!iter->Valid()); + } + + // testing that iterate_upper_bound prevents iterating over deleted items + // if the bound has already reached + { + options.prefix_extractor = nullptr; + DestroyAndReopen(options); + ASSERT_OK(Put("a", "0")); + ASSERT_OK(Put("b", "0")); + ASSERT_OK(Put("b1", "0")); + ASSERT_OK(Put("c", "0")); + ASSERT_OK(Put("d", "0")); + ASSERT_OK(Put("e", "0")); + ASSERT_OK(Delete("c")); + ASSERT_OK(Delete("d")); + + // base case with no bound + ReadOptions ro; + ro.iterate_upper_bound = nullptr; + + std::unique_ptr iter(NewIterator(ro)); + + iter->Seek("b"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("b")), 0); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(("b1")), 0); + + get_perf_context()->Reset(); + iter->Next(); + + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ( + static_cast(get_perf_context()->internal_delete_skipped_count), 2); + + // now testing with iterate_bound + Slice prefix("c"); + ro.iterate_upper_bound = &prefix; + + iter.reset(NewIterator(ro)); + + get_perf_context()->Reset(); + + iter->Seek("b"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("b")), 0); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(("b1")), 0); + + iter->Next(); + // the iteration should stop as soon as the bound key is reached + // even though the key is deleted + // hence internal_delete_skipped_count should be 0 + ASSERT_TRUE(!iter->Valid()); + ASSERT_EQ( + static_cast(get_perf_context()->internal_delete_skipped_count), 0); + } +} + +TEST_P(DBIteratorTest, DBIteratorBoundMultiSeek) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.prefix_extractor = nullptr; + DestroyAndReopen(options); + ASSERT_OK(Put("a", "0")); + ASSERT_OK(Put("z", "0")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("foo1", "bar1")); + ASSERT_OK(Put("foo2", "bar2")); + ASSERT_OK(Put("foo3", "bar3")); + ASSERT_OK(Put("foo4", "bar4")); + + { + std::string up_str = "foo5"; + Slice up(up_str); + ReadOptions ro; + ro.iterate_upper_bound = &up; + std::unique_ptr iter(NewIterator(ro)); + + iter->Seek("foo1"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("foo1")), 0); + + uint64_t prev_block_cache_hit = + TestGetTickerCount(options, BLOCK_CACHE_HIT); + uint64_t prev_block_cache_miss = + TestGetTickerCount(options, BLOCK_CACHE_MISS); + + ASSERT_GT(prev_block_cache_hit + prev_block_cache_miss, 0); + + iter->Seek("foo4"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("foo4")), 0); + ASSERT_EQ(prev_block_cache_hit, + TestGetTickerCount(options, BLOCK_CACHE_HIT)); + ASSERT_EQ(prev_block_cache_miss, + TestGetTickerCount(options, BLOCK_CACHE_MISS)); + + iter->Seek("foo2"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("foo2")), 0); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("foo3")), 0); + ASSERT_EQ(prev_block_cache_hit, + TestGetTickerCount(options, BLOCK_CACHE_HIT)); + ASSERT_EQ(prev_block_cache_miss, + TestGetTickerCount(options, BLOCK_CACHE_MISS)); + } +} + +TEST_P(DBIteratorTest, DBIteratorBoundOptimizationTest) { + for (auto format_version : {2, 3, 4}) { + int upper_bound_hits = 0; + Options options = CurrentOptions(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTableIterator:out_of_bound", + [&upper_bound_hits](void*) { upper_bound_hits++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + options.env = env_; + options.create_if_missing = true; + options.prefix_extractor = nullptr; + BlockBasedTableOptions table_options; + table_options.format_version = format_version; + table_options.flush_block_policy_factory = + std::make_shared(); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + DestroyAndReopen(options); + ASSERT_OK(Put("foo1", "bar1")); + ASSERT_OK(Put("foo2", "bar2")); + ASSERT_OK(Put("foo4", "bar4")); + ASSERT_OK(Flush()); + + Slice ub("foo3"); + ReadOptions ro; + ro.iterate_upper_bound = &ub; + + std::unique_ptr iter(NewIterator(ro)); + + iter->Seek("foo"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("foo1")), 0); + ASSERT_EQ(upper_bound_hits, 0); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("foo2")), 0); + ASSERT_EQ(upper_bound_hits, 0); + + iter->Next(); + ASSERT_FALSE(iter->Valid()); + ASSERT_EQ(upper_bound_hits, 1); + } +} + +// Enable kBinarySearchWithFirstKey, do some iterator operations and check that +// they don't do unnecessary block reads. +TEST_P(DBIteratorTest, IndexWithFirstKey) { + for (int tailing = 0; tailing < 2; ++tailing) { + SCOPED_TRACE("tailing = " + std::to_string(tailing)); + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.prefix_extractor = nullptr; + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + Statistics* stats = options.statistics.get(); + BlockBasedTableOptions table_options; + table_options.index_type = + BlockBasedTableOptions::IndexType::kBinarySearchWithFirstKey; + table_options.index_shortening = + BlockBasedTableOptions::IndexShorteningMode::kNoShortening; + table_options.flush_block_policy_factory = + std::make_shared(); + table_options.block_cache = + NewLRUCache(8000); // fits all blocks and their cache metadata overhead + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + DestroyAndReopen(options); + ASSERT_OK(Merge("a1", "x1")); + ASSERT_OK(Merge("b1", "y1")); + ASSERT_OK(Merge("c0", "z1")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("a2", "x2")); + ASSERT_OK(Merge("b2", "y2")); + ASSERT_OK(Merge("c0", "z2")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("a3", "x3")); + ASSERT_OK(Merge("b3", "y3")); + ASSERT_OK(Merge("c3", "z3")); + ASSERT_OK(Flush()); + + // Block cache is not important for this test. + // We use BLOCK_CACHE_DATA_* counters just because they're the most readily + // available way of counting block accesses. + + ReadOptions ropt; + ropt.tailing = tailing; + std::unique_ptr iter(NewIterator(ropt)); + + ropt.read_tier = ReadTier::kBlockCacheTier; + std::unique_ptr nonblocking_iter(NewIterator(ropt)); + + iter->Seek("b10"); + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ("b2", iter->key().ToString()); + EXPECT_EQ("y2", iter->value().ToString()); + EXPECT_EQ(1, stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + + // The cache-only iterator should succeed too, using the blocks pulled into + // the cache by the previous iterator. + nonblocking_iter->Seek("b10"); + ASSERT_TRUE(nonblocking_iter->Valid()); + EXPECT_EQ("b2", nonblocking_iter->key().ToString()); + EXPECT_EQ("y2", nonblocking_iter->value().ToString()); + EXPECT_EQ(1, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + + // ... but it shouldn't be able to step forward since the next block is + // not in cache yet. + nonblocking_iter->Next(); + ASSERT_FALSE(nonblocking_iter->Valid()); + ASSERT_TRUE(nonblocking_iter->status().IsIncomplete()); + + // ... nor should a seek to the next key succeed. + nonblocking_iter->Seek("b20"); + ASSERT_FALSE(nonblocking_iter->Valid()); + ASSERT_TRUE(nonblocking_iter->status().IsIncomplete()); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ("b3", iter->key().ToString()); + EXPECT_EQ("y3", iter->value().ToString()); + EXPECT_EQ(4, stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + EXPECT_EQ(1, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + + // After the blocking iterator loaded the next block, the nonblocking + // iterator's seek should succeed. + nonblocking_iter->Seek("b20"); + ASSERT_TRUE(nonblocking_iter->Valid()); + EXPECT_EQ("b3", nonblocking_iter->key().ToString()); + EXPECT_EQ("y3", nonblocking_iter->value().ToString()); + EXPECT_EQ(2, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + + iter->Seek("c0"); + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ("c0", iter->key().ToString()); + EXPECT_EQ("z1,z2", iter->value().ToString()); + EXPECT_EQ(2, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + EXPECT_EQ(6, stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ("c3", iter->key().ToString()); + EXPECT_EQ("z3", iter->value().ToString()); + EXPECT_EQ(2, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + EXPECT_EQ(7, stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + + iter.reset(); + + // Enable iterate_upper_bound and check that iterator is not trying to read + // blocks that are fully above upper bound. + std::string ub = "b3"; + Slice ub_slice(ub); + ropt.iterate_upper_bound = &ub_slice; + iter.reset(NewIterator(ropt)); + + iter->Seek("b2"); + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ("b2", iter->key().ToString()); + EXPECT_EQ("y2", iter->value().ToString()); + EXPECT_EQ(3, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + EXPECT_EQ(7, stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + + iter->Next(); + ASSERT_FALSE(iter->Valid()); + EXPECT_EQ(3, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + EXPECT_EQ(7, stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + } +} + +TEST_P(DBIteratorTest, IndexWithFirstKeyGet) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.prefix_extractor = nullptr; + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + Statistics* stats = options.statistics.get(); + BlockBasedTableOptions table_options; + table_options.index_type = + BlockBasedTableOptions::IndexType::kBinarySearchWithFirstKey; + table_options.index_shortening = + BlockBasedTableOptions::IndexShorteningMode::kNoShortening; + table_options.flush_block_policy_factory = + std::make_shared(); + table_options.block_cache = NewLRUCache(1000); // fits all blocks + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + DestroyAndReopen(options); + ASSERT_OK(Merge("a", "x1")); + ASSERT_OK(Merge("c", "y1")); + ASSERT_OK(Merge("e", "z1")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("c", "y2")); + ASSERT_OK(Merge("e", "z2")); + ASSERT_OK(Flush()); + + // Get() between blocks shouldn't read any blocks. + ASSERT_EQ("NOT_FOUND", Get("b")); + EXPECT_EQ(0, stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + EXPECT_EQ(0, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + + // Get() of an existing key shouldn't read any unnecessary blocks when there's + // only one key per block. + + ASSERT_EQ("y1,y2", Get("c")); + EXPECT_EQ(2, stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + EXPECT_EQ(0, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + + ASSERT_EQ("x1", Get("a")); + EXPECT_EQ(3, stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + EXPECT_EQ(0, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + + EXPECT_EQ(std::vector({"NOT_FOUND", "z1,z2"}), + MultiGet({"b", "e"})); +} + +// TODO(3.13): fix the issue of Seek() + Prev() which might not necessary +// return the biggest key which is smaller than the seek key. +TEST_P(DBIteratorTest, PrevAfterAndNextAfterMerge) { + Options options; + options.create_if_missing = true; + options.merge_operator = MergeOperators::CreatePutOperator(); + options.env = env_; + DestroyAndReopen(options); + + // write three entries with different keys using Merge() + WriteOptions wopts; + ASSERT_OK(db_->Merge(wopts, "1", "data1")); + ASSERT_OK(db_->Merge(wopts, "2", "data2")); + ASSERT_OK(db_->Merge(wopts, "3", "data3")); + + std::unique_ptr it(NewIterator(ReadOptions())); + + it->Seek("2"); + ASSERT_TRUE(it->Valid()); + ASSERT_EQ("2", it->key().ToString()); + + it->Prev(); + ASSERT_TRUE(it->Valid()); + ASSERT_EQ("1", it->key().ToString()); + + it->SeekForPrev("1"); + ASSERT_TRUE(it->Valid()); + ASSERT_EQ("1", it->key().ToString()); + + it->Next(); + ASSERT_TRUE(it->Valid()); + ASSERT_EQ("2", it->key().ToString()); +} + +class DBIteratorTestForPinnedData : public DBIteratorTest { + public: + enum TestConfig { + NORMAL, + CLOSE_AND_OPEN, + COMPACT_BEFORE_READ, + FLUSH_EVERY_1000, + MAX + }; + DBIteratorTestForPinnedData() : DBIteratorTest() {} + void PinnedDataIteratorRandomized(TestConfig run_config) { + // Generate Random data + Random rnd(301); + + int puts = 100000; + int key_pool = static_cast(puts * 0.7); + int key_size = 100; + int val_size = 1000; + int seeks_percentage = 20; // 20% of keys will be used to test seek() + int delete_percentage = 20; // 20% of keys will be deleted + int merge_percentage = 20; // 20% of keys will be added using Merge() + + Options options = CurrentOptions(); + BlockBasedTableOptions table_options; + table_options.use_delta_encoding = false; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.merge_operator = MergeOperators::CreatePutOperator(); + DestroyAndReopen(options); + + std::vector generated_keys(key_pool); + for (int i = 0; i < key_pool; i++) { + generated_keys[i] = rnd.RandomString(key_size); + } + + std::map true_data; + std::vector random_keys; + std::vector deleted_keys; + for (int i = 0; i < puts; i++) { + auto& k = generated_keys[rnd.Next() % key_pool]; + auto v = rnd.RandomString(val_size); + + // Insert data to true_data map and to DB + true_data[k] = v; + if (rnd.PercentTrue(merge_percentage)) { + ASSERT_OK(db_->Merge(WriteOptions(), k, v)); + } else { + ASSERT_OK(Put(k, v)); + } + + // Pick random keys to be used to test Seek() + if (rnd.PercentTrue(seeks_percentage)) { + random_keys.push_back(k); + } + + // Delete some random keys + if (rnd.PercentTrue(delete_percentage)) { + deleted_keys.push_back(k); + true_data.erase(k); + ASSERT_OK(Delete(k)); + } + + if (run_config == TestConfig::FLUSH_EVERY_1000) { + if (i && i % 1000 == 0) { + ASSERT_OK(Flush()); + } + } + } + + if (run_config == TestConfig::CLOSE_AND_OPEN) { + Close(); + Reopen(options); + } else if (run_config == TestConfig::COMPACT_BEFORE_READ) { + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + } + + ReadOptions ro; + ro.pin_data = true; + auto iter = NewIterator(ro); + + { + // Test Seek to random keys + std::vector keys_slices; + std::vector true_keys; + for (auto& k : random_keys) { + iter->Seek(k); + if (!iter->Valid()) { + ASSERT_EQ(true_data.lower_bound(k), true_data.end()); + continue; + } + std::string prop_value; + ASSERT_OK( + iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value)); + ASSERT_EQ("1", prop_value); + keys_slices.push_back(iter->key()); + true_keys.push_back(true_data.lower_bound(k)->first); + } + + for (size_t i = 0; i < keys_slices.size(); i++) { + ASSERT_EQ(keys_slices[i].ToString(), true_keys[i]); + } + } + + { + // Test SeekForPrev to random keys + std::vector keys_slices; + std::vector true_keys; + for (auto& k : random_keys) { + iter->SeekForPrev(k); + if (!iter->Valid()) { + ASSERT_EQ(true_data.upper_bound(k), true_data.begin()); + continue; + } + std::string prop_value; + ASSERT_OK( + iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value)); + ASSERT_EQ("1", prop_value); + keys_slices.push_back(iter->key()); + true_keys.push_back((--true_data.upper_bound(k))->first); + } + + for (size_t i = 0; i < keys_slices.size(); i++) { + ASSERT_EQ(keys_slices[i].ToString(), true_keys[i]); + } + } + + { + // Test iterating all data forward + std::vector all_keys; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + std::string prop_value; + ASSERT_OK( + iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value)); + ASSERT_EQ("1", prop_value); + all_keys.push_back(iter->key()); + } + ASSERT_EQ(all_keys.size(), true_data.size()); + + // Verify that all keys slices are valid + auto data_iter = true_data.begin(); + for (size_t i = 0; i < all_keys.size(); i++) { + ASSERT_EQ(all_keys[i].ToString(), data_iter->first); + data_iter++; + } + } + + { + // Test iterating all data backward + std::vector all_keys; + for (iter->SeekToLast(); iter->Valid(); iter->Prev()) { + std::string prop_value; + ASSERT_OK( + iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value)); + ASSERT_EQ("1", prop_value); + all_keys.push_back(iter->key()); + } + ASSERT_EQ(all_keys.size(), true_data.size()); + + // Verify that all keys slices are valid (backward) + auto data_iter = true_data.rbegin(); + for (size_t i = 0; i < all_keys.size(); i++) { + ASSERT_EQ(all_keys[i].ToString(), data_iter->first); + data_iter++; + } + } + + delete iter; + } +}; + +#if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) +TEST_P(DBIteratorTestForPinnedData, PinnedDataIteratorRandomizedNormal) { + PinnedDataIteratorRandomized(TestConfig::NORMAL); +} +#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) + +TEST_P(DBIteratorTestForPinnedData, PinnedDataIteratorRandomizedCLoseAndOpen) { + PinnedDataIteratorRandomized(TestConfig::CLOSE_AND_OPEN); +} + +TEST_P(DBIteratorTestForPinnedData, + PinnedDataIteratorRandomizedCompactBeforeRead) { + PinnedDataIteratorRandomized(TestConfig::COMPACT_BEFORE_READ); +} + +TEST_P(DBIteratorTestForPinnedData, PinnedDataIteratorRandomizedFlush) { + PinnedDataIteratorRandomized(TestConfig::FLUSH_EVERY_1000); +} + +INSTANTIATE_TEST_CASE_P(DBIteratorTestForPinnedDataInstance, + DBIteratorTestForPinnedData, + testing::Values(true, false)); + +TEST_P(DBIteratorTest, PinnedDataIteratorMultipleFiles) { + Options options = CurrentOptions(); + BlockBasedTableOptions table_options; + table_options.use_delta_encoding = false; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.disable_auto_compactions = true; + options.write_buffer_size = 1024 * 1024 * 10; // 10 Mb + DestroyAndReopen(options); + + std::map true_data; + + // Generate 4 sst files in L2 + Random rnd(301); + for (int i = 1; i <= 1000; i++) { + std::string k = Key(i * 3); + std::string v = rnd.RandomString(100); + ASSERT_OK(Put(k, v)); + true_data[k] = v; + if (i % 250 == 0) { + ASSERT_OK(Flush()); + } + } + ASSERT_EQ(FilesPerLevel(0), "4"); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(FilesPerLevel(0), "0,4"); + + // Generate 4 sst files in L0 + for (int i = 1; i <= 1000; i++) { + std::string k = Key(i * 2); + std::string v = rnd.RandomString(100); + ASSERT_OK(Put(k, v)); + true_data[k] = v; + if (i % 250 == 0) { + ASSERT_OK(Flush()); + } + } + ASSERT_EQ(FilesPerLevel(0), "4,4"); + + // Add some keys/values in memtables + for (int i = 1; i <= 1000; i++) { + std::string k = Key(i); + std::string v = rnd.RandomString(100); + ASSERT_OK(Put(k, v)); + true_data[k] = v; + } + ASSERT_EQ(FilesPerLevel(0), "4,4"); + + ReadOptions ro; + ro.pin_data = true; + auto iter = NewIterator(ro); + + std::vector> results; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + std::string prop_value; + ASSERT_OK(iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value)); + ASSERT_EQ("1", prop_value); + results.emplace_back(iter->key(), iter->value().ToString()); + } + + ASSERT_EQ(results.size(), true_data.size()); + auto data_iter = true_data.begin(); + for (size_t i = 0; i < results.size(); i++, data_iter++) { + auto& kv = results[i]; + ASSERT_EQ(kv.first, data_iter->first); + ASSERT_EQ(kv.second, data_iter->second); + } + + delete iter; +} + +TEST_P(DBIteratorTest, PinnedDataIteratorMergeOperator) { + Options options = CurrentOptions(); + BlockBasedTableOptions table_options; + table_options.use_delta_encoding = false; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.merge_operator = MergeOperators::CreateUInt64AddOperator(); + DestroyAndReopen(options); + + std::string numbers[7]; + for (int val = 0; val <= 6; val++) { + PutFixed64(numbers + val, val); + } + + // +1 all keys in range [ 0 => 999] + for (int i = 0; i < 1000; i++) { + WriteOptions wo; + ASSERT_OK(db_->Merge(wo, Key(i), numbers[1])); + } + + // +2 all keys divisible by 2 in range [ 0 => 999] + for (int i = 0; i < 1000; i += 2) { + WriteOptions wo; + ASSERT_OK(db_->Merge(wo, Key(i), numbers[2])); + } + + // +3 all keys divisible by 5 in range [ 0 => 999] + for (int i = 0; i < 1000; i += 5) { + WriteOptions wo; + ASSERT_OK(db_->Merge(wo, Key(i), numbers[3])); + } + + ReadOptions ro; + ro.pin_data = true; + auto iter = NewIterator(ro); + + std::vector> results; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + std::string prop_value; + ASSERT_OK(iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value)); + ASSERT_EQ("1", prop_value); + results.emplace_back(iter->key(), iter->value().ToString()); + } + + ASSERT_EQ(results.size(), 1000); + for (size_t i = 0; i < results.size(); i++) { + auto& kv = results[i]; + ASSERT_EQ(kv.first, Key(static_cast(i))); + int expected_val = 1; + if (i % 2 == 0) { + expected_val += 2; + } + if (i % 5 == 0) { + expected_val += 3; + } + ASSERT_EQ(kv.second, numbers[expected_val]); + } + + delete iter; +} + +TEST_P(DBIteratorTest, PinnedDataIteratorReadAfterUpdate) { + Options options = CurrentOptions(); + BlockBasedTableOptions table_options; + table_options.use_delta_encoding = false; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.write_buffer_size = 100000; + DestroyAndReopen(options); + + Random rnd(301); + + std::map true_data; + for (int i = 0; i < 1000; i++) { + std::string k = rnd.RandomString(10); + std::string v = rnd.RandomString(1000); + ASSERT_OK(Put(k, v)); + true_data[k] = v; + } + + ReadOptions ro; + ro.pin_data = true; + auto iter = NewIterator(ro); + + // Delete 50% of the keys and update the other 50% + for (auto& kv : true_data) { + if (rnd.OneIn(2)) { + ASSERT_OK(Delete(kv.first)); + } else { + std::string new_val = rnd.RandomString(1000); + ASSERT_OK(Put(kv.first, new_val)); + } + } + + std::vector> results; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + std::string prop_value; + ASSERT_OK(iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value)); + ASSERT_EQ("1", prop_value); + results.emplace_back(iter->key(), iter->value().ToString()); + } + + auto data_iter = true_data.begin(); + for (size_t i = 0; i < results.size(); i++, data_iter++) { + auto& kv = results[i]; + ASSERT_EQ(kv.first, data_iter->first); + ASSERT_EQ(kv.second, data_iter->second); + } + + delete iter; +} + +class SliceTransformLimitedDomainGeneric : public SliceTransform { + const char* Name() const override { + return "SliceTransformLimitedDomainGeneric"; + } + + Slice Transform(const Slice& src) const override { + return Slice(src.data(), 1); + } + + bool InDomain(const Slice& src) const override { + // prefix will be x???? + return src.size() >= 1; + } + + bool InRange(const Slice& dst) const override { + // prefix will be x???? + return dst.size() == 1; + } +}; + +TEST_P(DBIteratorTest, IterSeekForPrevCrossingFiles) { + Options options = CurrentOptions(); + options.prefix_extractor.reset(NewFixedPrefixTransform(1)); + options.disable_auto_compactions = true; + // Enable prefix bloom for SST files + BlockBasedTableOptions table_options; + table_options.filter_policy.reset(NewBloomFilterPolicy(10, true)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DestroyAndReopen(options); + + ASSERT_OK(Put("a1", "va1")); + ASSERT_OK(Put("a2", "va2")); + ASSERT_OK(Put("a3", "va3")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("b1", "vb1")); + ASSERT_OK(Put("b2", "vb2")); + ASSERT_OK(Put("b3", "vb3")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("b4", "vb4")); + ASSERT_OK(Put("d1", "vd1")); + ASSERT_OK(Put("d2", "vd2")); + ASSERT_OK(Put("d4", "vd4")); + ASSERT_OK(Flush()); + + MoveFilesToLevel(1); + { + ReadOptions ro; + Iterator* iter = NewIterator(ro); + + iter->SeekForPrev("a4"); + ASSERT_EQ(iter->key().ToString(), "a3"); + ASSERT_EQ(iter->value().ToString(), "va3"); + + iter->SeekForPrev("c2"); + ASSERT_EQ(iter->key().ToString(), "b3"); + iter->SeekForPrev("d3"); + ASSERT_EQ(iter->key().ToString(), "d2"); + iter->SeekForPrev("b5"); + ASSERT_EQ(iter->key().ToString(), "b4"); + delete iter; + } + + { + ReadOptions ro; + ro.prefix_same_as_start = true; + Iterator* iter = NewIterator(ro); + iter->SeekForPrev("c2"); + ASSERT_TRUE(!iter->Valid()); + ASSERT_OK(iter->status()); + delete iter; + } +} + +TEST_P(DBIteratorTest, IterSeekForPrevCrossingFilesCustomPrefixExtractor) { + Options options = CurrentOptions(); + options.prefix_extractor = + std::make_shared(); + options.disable_auto_compactions = true; + // Enable prefix bloom for SST files + BlockBasedTableOptions table_options; + table_options.filter_policy.reset(NewBloomFilterPolicy(10, true)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DestroyAndReopen(options); + + ASSERT_OK(Put("a1", "va1")); + ASSERT_OK(Put("a2", "va2")); + ASSERT_OK(Put("a3", "va3")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("b1", "vb1")); + ASSERT_OK(Put("b2", "vb2")); + ASSERT_OK(Put("b3", "vb3")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("b4", "vb4")); + ASSERT_OK(Put("d1", "vd1")); + ASSERT_OK(Put("d2", "vd2")); + ASSERT_OK(Put("d4", "vd4")); + ASSERT_OK(Flush()); + + MoveFilesToLevel(1); + { + ReadOptions ro; + Iterator* iter = NewIterator(ro); + + iter->SeekForPrev("a4"); + ASSERT_EQ(iter->key().ToString(), "a3"); + ASSERT_EQ(iter->value().ToString(), "va3"); + + iter->SeekForPrev("c2"); + ASSERT_EQ(iter->key().ToString(), "b3"); + iter->SeekForPrev("d3"); + ASSERT_EQ(iter->key().ToString(), "d2"); + iter->SeekForPrev("b5"); + ASSERT_EQ(iter->key().ToString(), "b4"); + delete iter; + } + + { + ReadOptions ro; + ro.prefix_same_as_start = true; + Iterator* iter = NewIterator(ro); + iter->SeekForPrev("c2"); + ASSERT_TRUE(!iter->Valid()); + ASSERT_OK(iter->status()); + delete iter; + } +} + +TEST_P(DBIteratorTest, IterPrevKeyCrossingBlocks) { + Options options = CurrentOptions(); + BlockBasedTableOptions table_options; + table_options.block_size = 1; // every block will contain one entry + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.merge_operator = MergeOperators::CreateStringAppendTESTOperator(); + options.disable_auto_compactions = true; + options.max_sequential_skip_in_iterations = 8; + + DestroyAndReopen(options); + + // Putting such deletes will force DBIter::Prev() to fallback to a Seek + for (int file_num = 0; file_num < 10; file_num++) { + ASSERT_OK(Delete("key4")); + ASSERT_OK(Flush()); + } + + // First File containing 5 blocks of puts + ASSERT_OK(Put("key1", "val1.0")); + ASSERT_OK(Put("key2", "val2.0")); + ASSERT_OK(Put("key3", "val3.0")); + ASSERT_OK(Put("key4", "val4.0")); + ASSERT_OK(Put("key5", "val5.0")); + ASSERT_OK(Flush()); + + // Second file containing 9 blocks of merge operands + ASSERT_OK(db_->Merge(WriteOptions(), "key1", "val1.1")); + ASSERT_OK(db_->Merge(WriteOptions(), "key1", "val1.2")); + + ASSERT_OK(db_->Merge(WriteOptions(), "key2", "val2.1")); + ASSERT_OK(db_->Merge(WriteOptions(), "key2", "val2.2")); + ASSERT_OK(db_->Merge(WriteOptions(), "key2", "val2.3")); + + ASSERT_OK(db_->Merge(WriteOptions(), "key3", "val3.1")); + ASSERT_OK(db_->Merge(WriteOptions(), "key3", "val3.2")); + ASSERT_OK(db_->Merge(WriteOptions(), "key3", "val3.3")); + ASSERT_OK(db_->Merge(WriteOptions(), "key3", "val3.4")); + ASSERT_OK(Flush()); + + { + ReadOptions ro; + ro.fill_cache = false; + Iterator* iter = NewIterator(ro); + + iter->SeekToLast(); + ASSERT_EQ(iter->key().ToString(), "key5"); + ASSERT_EQ(iter->value().ToString(), "val5.0"); + + iter->Prev(); + ASSERT_EQ(iter->key().ToString(), "key4"); + ASSERT_EQ(iter->value().ToString(), "val4.0"); + + iter->Prev(); + ASSERT_EQ(iter->key().ToString(), "key3"); + ASSERT_EQ(iter->value().ToString(), "val3.0,val3.1,val3.2,val3.3,val3.4"); + + iter->Prev(); + ASSERT_EQ(iter->key().ToString(), "key2"); + ASSERT_EQ(iter->value().ToString(), "val2.0,val2.1,val2.2,val2.3"); + + iter->Prev(); + ASSERT_EQ(iter->key().ToString(), "key1"); + ASSERT_EQ(iter->value().ToString(), "val1.0,val1.1,val1.2"); + + delete iter; + } +} + +TEST_P(DBIteratorTest, IterPrevKeyCrossingBlocksRandomized) { + Options options = CurrentOptions(); + options.merge_operator = MergeOperators::CreateStringAppendTESTOperator(); + options.disable_auto_compactions = true; + options.level0_slowdown_writes_trigger = (1 << 30); + options.level0_stop_writes_trigger = (1 << 30); + options.max_sequential_skip_in_iterations = 8; + DestroyAndReopen(options); + + const int kNumKeys = 500; + // Small number of merge operands to make sure that DBIter::Prev() don't + // fall back to Seek() + const int kNumMergeOperands = 3; + // Use value size that will make sure that every block contain 1 key + const int kValSize = + static_cast(BlockBasedTableOptions().block_size) * 4; + // Percentage of keys that wont get merge operations + const int kNoMergeOpPercentage = 20; + // Percentage of keys that will be deleted + const int kDeletePercentage = 10; + + // For half of the key range we will write multiple deletes first to + // force DBIter::Prev() to fall back to Seek() + for (int file_num = 0; file_num < 10; file_num++) { + for (int i = 0; i < kNumKeys; i += 2) { + ASSERT_OK(Delete(Key(i))); + } + ASSERT_OK(Flush()); + } + + Random rnd(301); + std::map true_data; + std::string gen_key; + std::string gen_val; + + for (int i = 0; i < kNumKeys; i++) { + gen_key = Key(i); + gen_val = rnd.RandomString(kValSize); + + ASSERT_OK(Put(gen_key, gen_val)); + true_data[gen_key] = gen_val; + } + ASSERT_OK(Flush()); + + // Separate values and merge operands in different file so that we + // make sure that we don't merge them while flushing but actually + // merge them in the read path + for (int i = 0; i < kNumKeys; i++) { + if (rnd.PercentTrue(kNoMergeOpPercentage)) { + // Dont give merge operations for some keys + continue; + } + + for (int j = 0; j < kNumMergeOperands; j++) { + gen_key = Key(i); + gen_val = rnd.RandomString(kValSize); + + ASSERT_OK(db_->Merge(WriteOptions(), gen_key, gen_val)); + true_data[gen_key] += "," + gen_val; + } + } + ASSERT_OK(Flush()); + + for (int i = 0; i < kNumKeys; i++) { + if (rnd.PercentTrue(kDeletePercentage)) { + gen_key = Key(i); + + ASSERT_OK(Delete(gen_key)); + true_data.erase(gen_key); + } + } + ASSERT_OK(Flush()); + + { + ReadOptions ro; + ro.fill_cache = false; + Iterator* iter = NewIterator(ro); + auto data_iter = true_data.rbegin(); + + for (iter->SeekToLast(); iter->Valid(); iter->Prev()) { + ASSERT_EQ(iter->key().ToString(), data_iter->first); + ASSERT_EQ(iter->value().ToString(), data_iter->second); + data_iter++; + } + ASSERT_EQ(data_iter, true_data.rend()); + + delete iter; + } + + { + ReadOptions ro; + ro.fill_cache = false; + Iterator* iter = NewIterator(ro); + auto data_iter = true_data.rbegin(); + + int entries_right = 0; + std::string seek_key; + for (iter->SeekToLast(); iter->Valid(); iter->Prev()) { + // Verify key/value of current position + ASSERT_EQ(iter->key().ToString(), data_iter->first); + ASSERT_EQ(iter->value().ToString(), data_iter->second); + + bool restore_position_with_seek = rnd.Uniform(2); + if (restore_position_with_seek) { + seek_key = iter->key().ToString(); + } + + // Do some Next() operations the restore the iterator to orignal position + int next_count = + entries_right > 0 ? rnd.Uniform(std::min(entries_right, 10)) : 0; + for (int i = 0; i < next_count; i++) { + iter->Next(); + data_iter--; + + ASSERT_EQ(iter->key().ToString(), data_iter->first); + ASSERT_EQ(iter->value().ToString(), data_iter->second); + } + + if (restore_position_with_seek) { + // Restore orignal position using Seek() + iter->Seek(seek_key); + for (int i = 0; i < next_count; i++) { + data_iter++; + } + + ASSERT_EQ(iter->key().ToString(), data_iter->first); + ASSERT_EQ(iter->value().ToString(), data_iter->second); + } else { + // Restore original position using Prev() + for (int i = 0; i < next_count; i++) { + iter->Prev(); + data_iter++; + + ASSERT_EQ(iter->key().ToString(), data_iter->first); + ASSERT_EQ(iter->value().ToString(), data_iter->second); + } + } + + entries_right++; + data_iter++; + } + ASSERT_EQ(data_iter, true_data.rend()); + + delete iter; + } +} + +TEST_P(DBIteratorTest, IteratorWithLocalStatistics) { + Options options = CurrentOptions(); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + DestroyAndReopen(options); + + Random rnd(301); + for (int i = 0; i < 1000; i++) { + // Key 10 bytes / Value 10 bytes + ASSERT_OK(Put(rnd.RandomString(10), rnd.RandomString(10))); + } + + std::atomic total_next(0); + std::atomic total_next_found(0); + std::atomic total_prev(0); + std::atomic total_prev_found(0); + std::atomic total_bytes(0); + + std::vector threads; + std::function reader_func_next = [&]() { + SetPerfLevel(kEnableCount); + get_perf_context()->Reset(); + Iterator* iter = NewIterator(ReadOptions()); + + iter->SeekToFirst(); + // Seek will bump ITER_BYTES_READ + uint64_t bytes = 0; + bytes += iter->key().size(); + bytes += iter->value().size(); + while (true) { + iter->Next(); + total_next++; + + if (!iter->Valid()) { + break; + } + total_next_found++; + bytes += iter->key().size(); + bytes += iter->value().size(); + } + + delete iter; + ASSERT_EQ(bytes, get_perf_context()->iter_read_bytes); + SetPerfLevel(kDisable); + total_bytes += bytes; + }; + + std::function reader_func_prev = [&]() { + SetPerfLevel(kEnableCount); + Iterator* iter = NewIterator(ReadOptions()); + + iter->SeekToLast(); + // Seek will bump ITER_BYTES_READ + uint64_t bytes = 0; + bytes += iter->key().size(); + bytes += iter->value().size(); + while (true) { + iter->Prev(); + total_prev++; + + if (!iter->Valid()) { + break; + } + total_prev_found++; + bytes += iter->key().size(); + bytes += iter->value().size(); + } + + delete iter; + ASSERT_EQ(bytes, get_perf_context()->iter_read_bytes); + SetPerfLevel(kDisable); + total_bytes += bytes; + }; + + for (int i = 0; i < 10; i++) { + threads.emplace_back(reader_func_next); + } + for (int i = 0; i < 15; i++) { + threads.emplace_back(reader_func_prev); + } + + for (auto& t : threads) { + t.join(); + } + + ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_NEXT), (uint64_t)total_next); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_NEXT_FOUND), + (uint64_t)total_next_found); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_PREV), (uint64_t)total_prev); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_PREV_FOUND), + (uint64_t)total_prev_found); + ASSERT_EQ(TestGetTickerCount(options, ITER_BYTES_READ), + (uint64_t)total_bytes); +} + +TEST_P(DBIteratorTest, ReadAhead) { + Options options; + env_->count_random_reads_ = true; + options.env = env_; + options.disable_auto_compactions = true; + options.write_buffer_size = 4 << 20; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + BlockBasedTableOptions table_options; + table_options.block_size = 1024; + table_options.no_block_cache = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + + std::string value(1024, 'a'); + for (int i = 0; i < 100; i++) { + ASSERT_OK(Put(Key(i), value)); + } + ASSERT_OK(Flush()); + MoveFilesToLevel(2); + + for (int i = 0; i < 100; i++) { + ASSERT_OK(Put(Key(i), value)); + } + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + + for (int i = 0; i < 100; i++) { + ASSERT_OK(Put(Key(i), value)); + } + ASSERT_OK(Flush()); + ASSERT_EQ("1,1,1", FilesPerLevel()); + + env_->random_read_bytes_counter_ = 0; + options.statistics->setTickerCount(NO_FILE_OPENS, 0); + ReadOptions read_options; + auto* iter = NewIterator(read_options); + iter->SeekToFirst(); + int64_t num_file_opens = TestGetTickerCount(options, NO_FILE_OPENS); + size_t bytes_read = env_->random_read_bytes_counter_; + delete iter; + + env_->random_read_bytes_counter_ = 0; + options.statistics->setTickerCount(NO_FILE_OPENS, 0); + read_options.readahead_size = 1024 * 10; + iter = NewIterator(read_options); + iter->SeekToFirst(); + int64_t num_file_opens_readahead = TestGetTickerCount(options, NO_FILE_OPENS); + size_t bytes_read_readahead = env_->random_read_bytes_counter_; + delete iter; + ASSERT_EQ(num_file_opens, num_file_opens_readahead); + ASSERT_GT(bytes_read_readahead, bytes_read); + ASSERT_GT(bytes_read_readahead, read_options.readahead_size * 3); + + // Verify correctness. + iter = NewIterator(read_options); + int count = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_EQ(value, iter->value()); + count++; + } + ASSERT_EQ(100, count); + for (int i = 0; i < 100; i++) { + iter->Seek(Key(i)); + ASSERT_EQ(value, iter->value()); + } + delete iter; +} + +// Insert a key, create a snapshot iterator, overwrite key lots of times, +// seek to a smaller key. Expect DBIter to fall back to a seek instead of +// going through all the overwrites linearly. +TEST_P(DBIteratorTest, DBIteratorSkipRecentDuplicatesTest) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.max_sequential_skip_in_iterations = 3; + options.prefix_extractor = nullptr; + options.write_buffer_size = 1 << 27; // big enough to avoid flush + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + DestroyAndReopen(options); + + // Insert. + ASSERT_OK(Put("b", "0")); + + // Create iterator. + ReadOptions ro; + std::unique_ptr iter(NewIterator(ro)); + + // Insert a lot. + for (int i = 0; i < 100; ++i) { + ASSERT_OK(Put("b", std::to_string(i + 1).c_str())); + } + + // Check that memtable wasn't flushed. + std::string val; + ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level0", &val)); + EXPECT_EQ("0", val); + + // Seek iterator to a smaller key. + get_perf_context()->Reset(); + iter->Seek("a"); + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ("b", iter->key().ToString()); + EXPECT_EQ("0", iter->value().ToString()); + + // Check that the seek didn't do too much work. + // Checks are not tight, just make sure that everything is well below 100. + EXPECT_LT(get_perf_context()->internal_key_skipped_count, 4); + EXPECT_LT(get_perf_context()->internal_recent_skipped_count, 8); + EXPECT_LT(get_perf_context()->seek_on_memtable_count, 10); + EXPECT_LT(get_perf_context()->next_on_memtable_count, 10); + EXPECT_LT(get_perf_context()->prev_on_memtable_count, 10); + + // Check that iterator did something like what we expect. + EXPECT_EQ(get_perf_context()->internal_delete_skipped_count, 0); + EXPECT_EQ(get_perf_context()->internal_merge_count, 0); + EXPECT_GE(get_perf_context()->internal_recent_skipped_count, 2); + EXPECT_GE(get_perf_context()->seek_on_memtable_count, 2); + EXPECT_EQ(1, + options.statistics->getTickerCount(NUMBER_OF_RESEEKS_IN_ITERATION)); +} + +TEST_P(DBIteratorTest, Refresh) { + ASSERT_OK(Put("x", "y")); + + std::unique_ptr iter(NewIterator(ReadOptions())); + ASSERT_OK(iter->status()); + iter->Seek(Slice("a")); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("x")), 0); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + + ASSERT_OK(Put("c", "d")); + + iter->Seek(Slice("a")); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("x")), 0); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + + ASSERT_OK(iter->status()); + ASSERT_OK(iter->Refresh()); + + iter->Seek(Slice("a")); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("c")), 0); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("x")), 0); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + + EXPECT_OK(dbfull()->Flush(FlushOptions())); + + ASSERT_OK(Put("m", "n")); + + iter->Seek(Slice("a")); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("c")), 0); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("x")), 0); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + + ASSERT_OK(iter->status()); + ASSERT_OK(iter->Refresh()); + + iter->Seek(Slice("a")); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("c")), 0); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("m")), 0); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("x")), 0); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + + iter.reset(); +} + +TEST_P(DBIteratorTest, RefreshWithSnapshot) { + ASSERT_OK(Put("x", "y")); + const Snapshot* snapshot = db_->GetSnapshot(); + ReadOptions options; + options.snapshot = snapshot; + Iterator* iter = NewIterator(options); + ASSERT_OK(iter->status()); + + iter->Seek(Slice("a")); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("x")), 0); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + + ASSERT_OK(Put("c", "d")); + + iter->Seek(Slice("a")); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Slice("x")), 0); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + + ASSERT_OK(iter->status()); + Status s = iter->Refresh(); + ASSERT_TRUE(s.IsNotSupported()); + db_->ReleaseSnapshot(snapshot); + delete iter; +} + +TEST_P(DBIteratorTest, CreationFailure) { + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::NewInternalIterator:StatusCallback", [](void* arg) { + *(reinterpret_cast(arg)) = Status::Corruption("test status"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + Iterator* iter = NewIterator(ReadOptions()); + ASSERT_FALSE(iter->Valid()); + ASSERT_TRUE(iter->status().IsCorruption()); + delete iter; +} + +TEST_P(DBIteratorTest, UpperBoundWithChangeDirection) { + Options options = CurrentOptions(); + options.max_sequential_skip_in_iterations = 3; + DestroyAndReopen(options); + + // write a bunch of kvs to the database. + ASSERT_OK(Put("a", "1")); + ASSERT_OK(Put("y", "1")); + ASSERT_OK(Put("y1", "1")); + ASSERT_OK(Put("y2", "1")); + ASSERT_OK(Put("y3", "1")); + ASSERT_OK(Put("z", "1")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("a", "1")); + ASSERT_OK(Put("z", "1")); + ASSERT_OK(Put("bar", "1")); + ASSERT_OK(Put("foo", "1")); + + std::string upper_bound = "x"; + Slice ub_slice(upper_bound); + ReadOptions ro; + ro.iterate_upper_bound = &ub_slice; + ro.max_skippable_internal_keys = 1000; + + Iterator* iter = NewIterator(ro); + iter->Seek("foo"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("foo", iter->key().ToString()); + + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ("bar", iter->key().ToString()); + + delete iter; +} + +TEST_P(DBIteratorTest, TableFilter) { + ASSERT_OK(Put("a", "1")); + EXPECT_OK(dbfull()->Flush(FlushOptions())); + ASSERT_OK(Put("b", "2")); + ASSERT_OK(Put("c", "3")); + EXPECT_OK(dbfull()->Flush(FlushOptions())); + ASSERT_OK(Put("d", "4")); + ASSERT_OK(Put("e", "5")); + ASSERT_OK(Put("f", "6")); + EXPECT_OK(dbfull()->Flush(FlushOptions())); + + // Ensure the table_filter callback is called once for each table. + { + std::set unseen{1, 2, 3}; + ReadOptions opts; + opts.table_filter = [&](const TableProperties& props) { + auto it = unseen.find(props.num_entries); + if (it == unseen.end()) { + ADD_FAILURE() << "saw table properties with an unexpected " + << props.num_entries << " entries"; + } else { + unseen.erase(it); + } + return true; + }; + auto iter = NewIterator(opts); + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "a->1"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "b->2"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "c->3"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "d->4"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "e->5"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "f->6"); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + ASSERT_TRUE(unseen.empty()); + delete iter; + } + + // Ensure returning false in the table_filter hides the keys from that table + // during iteration. + { + ReadOptions opts; + opts.table_filter = [](const TableProperties& props) { + return props.num_entries != 2; + }; + auto iter = NewIterator(opts); + iter->SeekToFirst(); + ASSERT_EQ(IterStatus(iter), "a->1"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "d->4"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "e->5"); + iter->Next(); + ASSERT_EQ(IterStatus(iter), "f->6"); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + delete iter; + } +} + +TEST_P(DBIteratorTest, UpperBoundWithPrevReseek) { + Options options = CurrentOptions(); + options.max_sequential_skip_in_iterations = 3; + DestroyAndReopen(options); + + // write a bunch of kvs to the database. + ASSERT_OK(Put("a", "1")); + ASSERT_OK(Put("y", "1")); + ASSERT_OK(Put("z", "1")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("a", "1")); + ASSERT_OK(Put("z", "1")); + ASSERT_OK(Put("bar", "1")); + ASSERT_OK(Put("foo", "1")); + ASSERT_OK(Put("foo", "2")); + + ASSERT_OK(Put("foo", "3")); + ASSERT_OK(Put("foo", "4")); + ASSERT_OK(Put("foo", "5")); + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK(Put("foo", "6")); + + std::string upper_bound = "x"; + Slice ub_slice(upper_bound); + ReadOptions ro; + ro.snapshot = snapshot; + ro.iterate_upper_bound = &ub_slice; + + Iterator* iter = NewIterator(ro); + iter->SeekForPrev("goo"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("foo", iter->key().ToString()); + iter->Prev(); + + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("bar", iter->key().ToString()); + + delete iter; + db_->ReleaseSnapshot(snapshot); +} + +TEST_P(DBIteratorTest, SkipStatistics) { + Options options = CurrentOptions(); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + DestroyAndReopen(options); + + int skip_count = 0; + + // write a bunch of kvs to the database. + ASSERT_OK(Put("a", "1")); + ASSERT_OK(Put("b", "1")); + ASSERT_OK(Put("c", "1")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("d", "1")); + ASSERT_OK(Put("e", "1")); + ASSERT_OK(Put("f", "1")); + ASSERT_OK(Put("a", "2")); + ASSERT_OK(Put("b", "2")); + ASSERT_OK(Flush()); + ASSERT_OK(Delete("d")); + ASSERT_OK(Delete("e")); + ASSERT_OK(Delete("f")); + + Iterator* iter = NewIterator(ReadOptions()); + int count = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_OK(iter->status()); + count++; + } + ASSERT_EQ(count, 3); + delete iter; + skip_count += 8; // 3 deletes + 3 original keys + 2 lower in sequence + ASSERT_EQ(skip_count, TestGetTickerCount(options, NUMBER_ITER_SKIP)); + + iter = NewIterator(ReadOptions()); + count = 0; + for (iter->SeekToLast(); iter->Valid(); iter->Prev()) { + ASSERT_OK(iter->status()); + count++; + } + ASSERT_EQ(count, 3); + delete iter; + skip_count += 8; // Same as above, but in reverse order + ASSERT_EQ(skip_count, TestGetTickerCount(options, NUMBER_ITER_SKIP)); + + ASSERT_OK(Put("aa", "1")); + ASSERT_OK(Put("ab", "1")); + ASSERT_OK(Put("ac", "1")); + ASSERT_OK(Put("ad", "1")); + ASSERT_OK(Flush()); + ASSERT_OK(Delete("ab")); + ASSERT_OK(Delete("ac")); + ASSERT_OK(Delete("ad")); + + ReadOptions ro; + Slice prefix("b"); + ro.iterate_upper_bound = &prefix; + + iter = NewIterator(ro); + count = 0; + for (iter->Seek("aa"); iter->Valid(); iter->Next()) { + ASSERT_OK(iter->status()); + count++; + } + ASSERT_EQ(count, 1); + delete iter; + skip_count += 6; // 3 deletes + 3 original keys + ASSERT_EQ(skip_count, TestGetTickerCount(options, NUMBER_ITER_SKIP)); + + iter = NewIterator(ro); + count = 0; + for (iter->SeekToLast(); iter->Valid(); iter->Prev()) { + ASSERT_OK(iter->status()); + count++; + } + ASSERT_EQ(count, 2); + delete iter; + // 3 deletes + 3 original keys + lower sequence of "a" + skip_count += 7; + ASSERT_EQ(skip_count, TestGetTickerCount(options, NUMBER_ITER_SKIP)); +} + +TEST_P(DBIteratorTest, SeekAfterHittingManyInternalKeys) { + Options options = CurrentOptions(); + DestroyAndReopen(options); + ReadOptions ropts; + ropts.max_skippable_internal_keys = 2; + + ASSERT_OK(Put("1", "val_1")); + // Add more tombstones than max_skippable_internal_keys so that Next() fails. + ASSERT_OK(Delete("2")); + ASSERT_OK(Delete("3")); + ASSERT_OK(Delete("4")); + ASSERT_OK(Delete("5")); + ASSERT_OK(Put("6", "val_6")); + + std::unique_ptr iter(NewIterator(ropts)); + iter->SeekToFirst(); + + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().ToString(), "1"); + ASSERT_EQ(iter->value().ToString(), "val_1"); + + // This should fail as incomplete due to too many non-visible internal keys on + // the way to the next valid user key. + iter->Next(); + ASSERT_TRUE(!iter->Valid()); + ASSERT_TRUE(iter->status().IsIncomplete()); + + // Get the internal key at which Next() failed. + std::string prop_value; + ASSERT_OK(iter->GetProperty("rocksdb.iterator.internal-key", &prop_value)); + ASSERT_EQ("4", prop_value); + + // Create a new iterator to seek to the internal key. + std::unique_ptr iter2(NewIterator(ropts)); + iter2->Seek(prop_value); + ASSERT_TRUE(iter2->Valid()); + ASSERT_OK(iter2->status()); + + ASSERT_EQ(iter2->key().ToString(), "6"); + ASSERT_EQ(iter2->value().ToString(), "val_6"); +} + +// Reproduces a former bug where iterator would skip some records when DBIter +// re-seeks subiterator with Incomplete status. +TEST_P(DBIteratorTest, NonBlockingIterationBugRepro) { + Options options = CurrentOptions(); + BlockBasedTableOptions table_options; + // Make sure the sst file has more than one block. + table_options.flush_block_policy_factory = + std::make_shared(); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DestroyAndReopen(options); + + // Two records in sst file, each in its own block. + ASSERT_OK(Put("b", "")); + ASSERT_OK(Put("d", "")); + ASSERT_OK(Flush()); + + // Create a nonblocking iterator before writing to memtable. + ReadOptions ropt; + ropt.read_tier = kBlockCacheTier; + std::unique_ptr iter(NewIterator(ropt)); + + // Overwrite a key in memtable many times to hit + // max_sequential_skip_in_iterations (which is 8 by default). + for (int i = 0; i < 20; ++i) { + ASSERT_OK(Put("c", "")); + } + + // Load the second block in sst file into the block cache. + { + std::unique_ptr iter2(NewIterator(ReadOptions())); + iter2->Seek("d"); + } + + // Finally seek the nonblocking iterator. + iter->Seek("a"); + // With the bug, the status used to be OK, and the iterator used to point to + // "d". + EXPECT_TRUE(iter->status().IsIncomplete()); +} + +TEST_P(DBIteratorTest, SeekBackwardAfterOutOfUpperBound) { + ASSERT_OK(Put("a", "")); + ASSERT_OK(Put("b", "")); + ASSERT_OK(Flush()); + + ReadOptions ropt; + Slice ub = "b"; + ropt.iterate_upper_bound = &ub; + + std::unique_ptr it(dbfull()->NewIterator(ropt)); + it->SeekForPrev("a"); + ASSERT_TRUE(it->Valid()); + ASSERT_OK(it->status()); + ASSERT_EQ("a", it->key().ToString()); + it->Next(); + ASSERT_FALSE(it->Valid()); + ASSERT_OK(it->status()); + it->SeekForPrev("a"); + ASSERT_OK(it->status()); + + ASSERT_TRUE(it->Valid()); + ASSERT_EQ("a", it->key().ToString()); +} + +TEST_P(DBIteratorTest, AvoidReseekLevelIterator) { + Options options = CurrentOptions(); + options.compression = CompressionType::kNoCompression; + BlockBasedTableOptions table_options; + table_options.block_size = 800; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + + Random rnd(301); + std::string random_str = rnd.RandomString(180); + + ASSERT_OK(Put("1", random_str)); + ASSERT_OK(Put("2", random_str)); + ASSERT_OK(Put("3", random_str)); + ASSERT_OK(Put("4", random_str)); + // A new block + ASSERT_OK(Put("5", random_str)); + ASSERT_OK(Put("6", random_str)); + ASSERT_OK(Put("7", random_str)); + ASSERT_OK(Flush()); + ASSERT_OK(Put("8", random_str)); + ASSERT_OK(Put("9", random_str)); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + int num_find_file_in_level = 0; + int num_idx_blk_seek = 0; + SyncPoint::GetInstance()->SetCallBack( + "LevelIterator::Seek:BeforeFindFile", + [&](void* /*arg*/) { num_find_file_in_level++; }); + SyncPoint::GetInstance()->SetCallBack( + "IndexBlockIter::Seek:0", [&](void* /*arg*/) { num_idx_blk_seek++; }); + SyncPoint::GetInstance()->EnableProcessing(); + + { + std::unique_ptr iter(NewIterator(ReadOptions())); + iter->Seek("1"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(1, num_find_file_in_level); + ASSERT_EQ(1, num_idx_blk_seek); + + iter->Seek("2"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(1, num_find_file_in_level); + ASSERT_EQ(1, num_idx_blk_seek); + + iter->Seek("3"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(1, num_find_file_in_level); + ASSERT_EQ(1, num_idx_blk_seek); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(1, num_find_file_in_level); + ASSERT_EQ(1, num_idx_blk_seek); + + iter->Seek("5"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(1, num_find_file_in_level); + ASSERT_EQ(2, num_idx_blk_seek); + + iter->Seek("6"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(1, num_find_file_in_level); + ASSERT_EQ(2, num_idx_blk_seek); + + iter->Seek("7"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(1, num_find_file_in_level); + ASSERT_EQ(3, num_idx_blk_seek); + + iter->Seek("8"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(2, num_find_file_in_level); + // Still re-seek because "8" is the boundary key, which has + // the same user key as the seek key. + ASSERT_EQ(4, num_idx_blk_seek); + + iter->Seek("5"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(3, num_find_file_in_level); + ASSERT_EQ(5, num_idx_blk_seek); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(3, num_find_file_in_level); + ASSERT_EQ(5, num_idx_blk_seek); + + // Seek backward never triggers the index block seek to be skipped + iter->Seek("5"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(3, num_find_file_in_level); + ASSERT_EQ(6, num_idx_blk_seek); + } + + SyncPoint::GetInstance()->DisableProcessing(); +} + +// MyRocks may change iterate bounds before seek. Simply test to make sure such +// usage doesn't break iterator. +TEST_P(DBIteratorTest, IterateBoundChangedBeforeSeek) { + Options options = CurrentOptions(); + options.compression = CompressionType::kNoCompression; + BlockBasedTableOptions table_options; + table_options.block_size = 100; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + std::string value(50, 'v'); + Reopen(options); + ASSERT_OK(Put("aaa", value)); + ASSERT_OK(Flush()); + ASSERT_OK(Put("bbb", "v")); + ASSERT_OK(Put("ccc", "v")); + ASSERT_OK(Put("ddd", "v")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("eee", "v")); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + std::string ub1 = "e"; + std::string ub2 = "c"; + Slice ub(ub1); + ReadOptions read_opts1; + read_opts1.iterate_upper_bound = &ub; + Iterator* iter = NewIterator(read_opts1); + // Seek and iterate accross block boundary. + iter->Seek("b"); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ("bbb", iter->key()); + ub = Slice(ub2); + iter->Seek("b"); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ("bbb", iter->key()); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + delete iter; + + std::string lb1 = "a"; + std::string lb2 = "c"; + Slice lb(lb1); + ReadOptions read_opts2; + read_opts2.iterate_lower_bound = &lb; + iter = NewIterator(read_opts2); + iter->SeekForPrev("d"); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ("ccc", iter->key()); + lb = Slice(lb2); + iter->SeekForPrev("d"); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ("ccc", iter->key()); + iter->Prev(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + delete iter; +} + +TEST_P(DBIteratorTest, IterateWithLowerBoundAcrossFileBoundary) { + ASSERT_OK(Put("aaa", "v")); + ASSERT_OK(Put("bbb", "v")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("ccc", "v")); + ASSERT_OK(Put("ddd", "v")); + ASSERT_OK(Flush()); + // Move both files to bottom level. + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + Slice lower_bound("b"); + ReadOptions read_opts; + read_opts.iterate_lower_bound = &lower_bound; + std::unique_ptr iter(NewIterator(read_opts)); + iter->SeekForPrev("d"); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ("ccc", iter->key()); + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ("bbb", iter->key()); + iter->Prev(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); +} + +TEST_P(DBIteratorTest, Blob) { + Options options = CurrentOptions(); + options.enable_blob_files = true; + options.max_sequential_skip_in_iterations = 2; + options.statistics = CreateDBStatistics(); + + Reopen(options); + + // Note: we have 4 KVs (3 of which are hidden) for key "b" and + // max_sequential_skip_in_iterations is set to 2. Thus, we need to do a reseek + // anytime we move from "b" to "c" or vice versa. + ASSERT_OK(Put("a", "va")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("b", "vb0")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("b", "vb1")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("b", "vb2")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("b", "vb3")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("c", "vc")); + ASSERT_OK(Flush()); + + std::unique_ptr iter_guard(NewIterator(ReadOptions())); + Iterator* const iter = iter_guard.get(); + + iter->SeekToFirst(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 0); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Next(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 0); + ASSERT_EQ(IterStatus(iter), "b->vb3"); + iter->Next(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 1); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Next(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 1); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + iter->SeekToFirst(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 1); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Prev(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 1); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->SeekToLast(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 1); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Prev(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 2); + ASSERT_EQ(IterStatus(iter), "b->vb3"); + iter->Prev(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 2); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Prev(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 2); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + iter->SeekToLast(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 2); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Next(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 2); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + iter->Seek(""); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 2); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Seek("a"); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 2); + ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Seek("ax"); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 2); + ASSERT_EQ(IterStatus(iter), "b->vb3"); + + iter->SeekForPrev("d"); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 2); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->SeekForPrev("c"); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 2); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->SeekForPrev("bx"); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 3); + ASSERT_EQ(IterStatus(iter), "b->vb3"); + + iter->Seek("b"); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 3); + ASSERT_EQ(IterStatus(iter), "b->vb3"); + iter->Seek("z"); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 3); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + iter->SeekForPrev("b"); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 4); + ASSERT_EQ(IterStatus(iter), "b->vb3"); + iter->SeekForPrev(""); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 4); + ASSERT_EQ(IterStatus(iter), "(invalid)"); + + // Switch from reverse to forward + iter->SeekToLast(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 4); + iter->Prev(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 5); + iter->Prev(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 5); + iter->Next(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 6); + ASSERT_EQ(IterStatus(iter), "b->vb3"); + + // Switch from forward to reverse + iter->SeekToFirst(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 6); + iter->Next(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 6); + iter->Next(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 7); + iter->Prev(); + ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 8); + ASSERT_EQ(IterStatus(iter), "b->vb3"); +} + +INSTANTIATE_TEST_CASE_P(DBIteratorTestInstance, DBIteratorTest, + testing::Values(true, false)); + +// Tests how DBIter work with ReadCallback +class DBIteratorWithReadCallbackTest : public DBIteratorTest {}; + +TEST_F(DBIteratorWithReadCallbackTest, ReadCallback) { + class TestReadCallback : public ReadCallback { + public: + explicit TestReadCallback(SequenceNumber _max_visible_seq) + : ReadCallback(_max_visible_seq) {} + + bool IsVisibleFullCheck(SequenceNumber seq) override { + return seq <= max_visible_seq_; + } + }; + + ASSERT_OK(Put("foo", "v1")); + ASSERT_OK(Put("foo", "v2")); + ASSERT_OK(Put("foo", "v3")); + ASSERT_OK(Put("a", "va")); + ASSERT_OK(Put("z", "vz")); + SequenceNumber seq1 = db_->GetLatestSequenceNumber(); + TestReadCallback callback1(seq1); + ASSERT_OK(Put("foo", "v4")); + ASSERT_OK(Put("foo", "v5")); + ASSERT_OK(Put("bar", "v7")); + + SequenceNumber seq2 = db_->GetLatestSequenceNumber(); + auto* cfd = + static_cast_with_check(db_->DefaultColumnFamily()) + ->cfd(); + // The iterator are suppose to see data before seq1. + Iterator* iter = + dbfull()->NewIteratorImpl(ReadOptions(), cfd, seq2, &callback1); + + // Seek + // The latest value of "foo" before seq1 is "v3" + iter->Seek("foo"); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ("foo", iter->key()); + ASSERT_EQ("v3", iter->value()); + // "bar" is not visible to the iterator. It will move on to the next key + // "foo". + iter->Seek("bar"); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ("foo", iter->key()); + ASSERT_EQ("v3", iter->value()); + + // Next + // Seek to "a" + iter->Seek("a"); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ("va", iter->value()); + // "bar" is not visible to the iterator. It will move on to the next key + // "foo". + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ("foo", iter->key()); + ASSERT_EQ("v3", iter->value()); + + // Prev + // Seek to "z" + iter->Seek("z"); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ("vz", iter->value()); + // The previous key is "foo", which is visible to the iterator. + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ("foo", iter->key()); + ASSERT_EQ("v3", iter->value()); + // "bar" is not visible to the iterator. It will move on to the next key "a". + iter->Prev(); // skipping "bar" + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ("a", iter->key()); + ASSERT_EQ("va", iter->value()); + + // SeekForPrev + // The previous key is "foo", which is visible to the iterator. + iter->SeekForPrev("y"); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ("foo", iter->key()); + ASSERT_EQ("v3", iter->value()); + // "bar" is not visible to the iterator. It will move on to the next key "a". + iter->SeekForPrev("bar"); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ("a", iter->key()); + ASSERT_EQ("va", iter->value()); + + delete iter; + + // Prev beyond max_sequential_skip_in_iterations + uint64_t num_versions = + CurrentOptions().max_sequential_skip_in_iterations + 10; + for (uint64_t i = 0; i < num_versions; i++) { + ASSERT_OK(Put("bar", std::to_string(i))); + } + SequenceNumber seq3 = db_->GetLatestSequenceNumber(); + TestReadCallback callback2(seq3); + ASSERT_OK(Put("bar", "v8")); + SequenceNumber seq4 = db_->GetLatestSequenceNumber(); + + // The iterator is suppose to see data before seq3. + iter = dbfull()->NewIteratorImpl(ReadOptions(), cfd, seq4, &callback2); + // Seek to "z", which is visible. + iter->Seek("z"); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ("vz", iter->value()); + // Previous key is "foo" and the last value "v5" is visible. + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ("foo", iter->key()); + ASSERT_EQ("v5", iter->value()); + // Since the number of values of "bar" is more than + // max_sequential_skip_in_iterations, Prev() will ultimately fallback to + // seek in forward direction. Here we test the fallback seek is correct. + // The last visible value should be (num_versions - 1), as "v8" is not + // visible. + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ("bar", iter->key()); + ASSERT_EQ(std::to_string(num_versions - 1), iter->value()); + + delete iter; +} + +TEST_F(DBIteratorTest, BackwardIterationOnInplaceUpdateMemtable) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.inplace_update_support = false; + options.env = env_; + DestroyAndReopen(options); + constexpr int kNumKeys = 10; + + // Write kNumKeys to WAL. + for (int i = 0; i < kNumKeys; ++i) { + ASSERT_OK(Put(Key(i), "val")); + } + ReadOptions read_opts; + read_opts.total_order_seek = true; + { + std::unique_ptr iter(db_->NewIterator(read_opts)); + int count = 0; + for (iter->SeekToLast(); iter->Valid(); iter->Prev()) { + ++count; + } + ASSERT_EQ(kNumKeys, count); + } + + // Reopen and rebuild the memtable from WAL. + options.create_if_missing = false; + options.avoid_flush_during_recovery = true; + options.inplace_update_support = true; + options.allow_concurrent_memtable_write = false; + Reopen(options); + { + std::unique_ptr iter(db_->NewIterator(read_opts)); + iter->SeekToLast(); + // Backward iteration not supported due to inplace_update_support = true. + ASSERT_TRUE(iter->status().IsNotSupported()); + ASSERT_FALSE(iter->Valid()); + } +} + +TEST_F(DBIteratorTest, IteratorRefreshReturnSV) { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + DestroyAndReopen(options); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "z")); + std::unique_ptr iter{db_->NewIterator(ReadOptions())}; + SyncPoint::GetInstance()->SetCallBack( + "ArenaWrappedDBIter::Refresh:SV", [&](void*) { + ASSERT_OK(db_->Put(WriteOptions(), "dummy", "new SV")); + // This makes the local SV obselete. + ASSERT_OK(Flush()); + SyncPoint::GetInstance()->DisableProcessing(); + }); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(iter->Refresh()); + iter.reset(); + // iter used to not cleanup SV, so the Close() below would hit an assertion + // error. + Close(); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_kv_checksum_test.cc b/librocksdb-sys/rocksdb/db/db_kv_checksum_test.cc new file mode 100644 index 0000000..6143992 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_kv_checksum_test.cc @@ -0,0 +1,885 @@ +// Copyright (c) 2020-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_index.h" +#include "db/db_test_util.h" +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +enum class WriteBatchOpType { + kPut = 0, + kDelete, + kSingleDelete, + kMerge, + kPutEntity, + kDeleteRange, + kNum, +}; + +// Integer addition is needed for `::testing::Range()` to take the enum type. +WriteBatchOpType operator+(WriteBatchOpType lhs, const int rhs) { + using T = std::underlying_type::type; + return static_cast(static_cast(lhs) + rhs); +} + +enum class WriteMode { + // `Write()` a `WriteBatch` constructed with `protection_bytes_per_key = 0` + // and `WriteOptions::protection_bytes_per_key = 0` + kWriteUnprotectedBatch = 0, + // `Write()` a `WriteBatch` constructed with `protection_bytes_per_key > 0`. + kWriteProtectedBatch, + // `Write()` a `WriteBatch` constructed with `protection_bytes_per_key == 0`. + // Protection is enabled via `WriteOptions::protection_bytes_per_key > 0`. + kWriteOptionProtectedBatch, + // TODO(ajkr): add a mode that uses `Write()` wrappers, e.g., `Put()`. + kNum, +}; + +// Integer addition is needed for `::testing::Range()` to take the enum type. +WriteMode operator+(WriteMode lhs, const int rhs) { + using T = std::underlying_type::type; + return static_cast(static_cast(lhs) + rhs); +} + +std::pair GetWriteBatch(ColumnFamilyHandle* cf_handle, + size_t protection_bytes_per_key, + WriteBatchOpType op_type) { + Status s; + WriteBatch wb(0 /* reserved_bytes */, 0 /* max_bytes */, + protection_bytes_per_key, 0 /* default_cf_ts_sz */); + switch (op_type) { + case WriteBatchOpType::kPut: + s = wb.Put(cf_handle, "key", "val"); + break; + case WriteBatchOpType::kDelete: + s = wb.Delete(cf_handle, "key"); + break; + case WriteBatchOpType::kSingleDelete: + s = wb.SingleDelete(cf_handle, "key"); + break; + case WriteBatchOpType::kDeleteRange: + s = wb.DeleteRange(cf_handle, "begin", "end"); + break; + case WriteBatchOpType::kMerge: + s = wb.Merge(cf_handle, "key", "val"); + break; + case WriteBatchOpType::kPutEntity: + s = wb.PutEntity(cf_handle, "key", + {{"attr_name1", "foo"}, {"attr_name2", "bar"}}); + break; + case WriteBatchOpType::kNum: + assert(false); + } + return {std::move(wb), std::move(s)}; +} + +class DbKvChecksumTestBase : public DBTestBase { + public: + DbKvChecksumTestBase(const std::string& path, bool env_do_fsync) + : DBTestBase(path, env_do_fsync) {} + + ColumnFamilyHandle* GetCFHandleToUse(ColumnFamilyHandle* column_family, + WriteBatchOpType op_type) const { + // Note: PutEntity cannot be called without column family + if (op_type == WriteBatchOpType::kPutEntity && !column_family) { + return db_->DefaultColumnFamily(); + } + + return column_family; + } +}; + +class DbKvChecksumTest + : public DbKvChecksumTestBase, + public ::testing::WithParamInterface< + std::tuple> { + public: + DbKvChecksumTest() + : DbKvChecksumTestBase("db_kv_checksum_test", /*env_do_fsync=*/false) { + op_type_ = std::get<0>(GetParam()); + corrupt_byte_addend_ = std::get<1>(GetParam()); + write_mode_ = std::get<2>(GetParam()); + memtable_protection_bytes_per_key_ = std::get<3>(GetParam()); + } + + Status ExecuteWrite(ColumnFamilyHandle* cf_handle) { + switch (write_mode_) { + case WriteMode::kWriteUnprotectedBatch: { + auto batch_and_status = + GetWriteBatch(GetCFHandleToUse(cf_handle, op_type_), + 0 /* protection_bytes_per_key */, op_type_); + assert(batch_and_status.second.ok()); + // Default write option has protection_bytes_per_key = 0 + return db_->Write(WriteOptions(), &batch_and_status.first); + } + case WriteMode::kWriteProtectedBatch: { + auto batch_and_status = + GetWriteBatch(GetCFHandleToUse(cf_handle, op_type_), + 8 /* protection_bytes_per_key */, op_type_); + assert(batch_and_status.second.ok()); + return db_->Write(WriteOptions(), &batch_and_status.first); + } + case WriteMode::kWriteOptionProtectedBatch: { + auto batch_and_status = + GetWriteBatch(GetCFHandleToUse(cf_handle, op_type_), + 0 /* protection_bytes_per_key */, op_type_); + assert(batch_and_status.second.ok()); + WriteOptions write_opts; + write_opts.protection_bytes_per_key = 8; + return db_->Write(write_opts, &batch_and_status.first); + } + case WriteMode::kNum: + assert(false); + } + return Status::NotSupported("WriteMode " + + std::to_string(static_cast(write_mode_))); + } + + void CorruptNextByteCallBack(void* arg) { + Slice encoded = *static_cast(arg); + if (entry_len_ == std::numeric_limits::max()) { + // We learn the entry size on the first attempt + entry_len_ = encoded.size(); + } + char* buf = const_cast(encoded.data()); + buf[corrupt_byte_offset_] += corrupt_byte_addend_; + ++corrupt_byte_offset_; + } + + bool MoreBytesToCorrupt() { return corrupt_byte_offset_ < entry_len_; } + + protected: + WriteBatchOpType op_type_; + char corrupt_byte_addend_; + WriteMode write_mode_; + uint32_t memtable_protection_bytes_per_key_; + size_t corrupt_byte_offset_ = 0; + size_t entry_len_ = std::numeric_limits::max(); +}; + +std::string GetOpTypeString(const WriteBatchOpType& op_type) { + switch (op_type) { + case WriteBatchOpType::kPut: + return "Put"; + case WriteBatchOpType::kDelete: + return "Delete"; + case WriteBatchOpType::kSingleDelete: + return "SingleDelete"; + case WriteBatchOpType::kDeleteRange: + return "DeleteRange"; + case WriteBatchOpType::kMerge: + return "Merge"; + case WriteBatchOpType::kPutEntity: + return "PutEntity"; + case WriteBatchOpType::kNum: + assert(false); + } + assert(false); + return ""; +} + +std::string GetWriteModeString(const WriteMode& mode) { + switch (mode) { + case WriteMode::kWriteUnprotectedBatch: + return "WriteUnprotectedBatch"; + case WriteMode::kWriteProtectedBatch: + return "WriteProtectedBatch"; + case WriteMode::kWriteOptionProtectedBatch: + return "kWriteOptionProtectedBatch"; + case WriteMode::kNum: + assert(false); + } + return ""; +} + +INSTANTIATE_TEST_CASE_P( + DbKvChecksumTest, DbKvChecksumTest, + ::testing::Combine(::testing::Range(static_cast(0), + WriteBatchOpType::kNum), + ::testing::Values(2, 103, 251), + ::testing::Range(WriteMode::kWriteProtectedBatch, + WriteMode::kNum), + ::testing::Values(0)), + [](const testing::TestParamInfo< + std::tuple>& args) { + std::ostringstream oss; + oss << GetOpTypeString(std::get<0>(args.param)) << "Add" + << static_cast( + static_cast(std::get<1>(args.param))) + << GetWriteModeString(std::get<2>(args.param)) + << static_cast(std::get<3>(args.param)); + return oss.str(); + }); + +// TODO(ajkr): add a test that corrupts the `WriteBatch` contents. Such +// corruptions should only be detectable in `WriteMode::kWriteProtectedBatch`. + +TEST_P(DbKvChecksumTest, MemTableAddCorrupted) { + // This test repeatedly attempts to write `WriteBatch`es containing a single + // entry of type `op_type_`. Each attempt has one byte corrupted in its + // memtable entry by adding `corrupt_byte_addend_` to its original value. The + // test repeats until an attempt has been made on each byte in the encoded + // memtable entry. All attempts are expected to fail with `Status::Corruption` + SyncPoint::GetInstance()->SetCallBack( + "MemTable::Add:Encoded", + std::bind(&DbKvChecksumTest::CorruptNextByteCallBack, this, + std::placeholders::_1)); + + while (MoreBytesToCorrupt()) { + // Failed memtable insert always leads to read-only mode, so we have to + // reopen for every attempt. + Options options = CurrentOptions(); + if (op_type_ == WriteBatchOpType::kMerge) { + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + } + Reopen(options); + + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_TRUE(ExecuteWrite(nullptr /* cf_handle */).IsCorruption()); + SyncPoint::GetInstance()->DisableProcessing(); + + // In case the above callback is not invoked, this test will run + // numeric_limits::max() times until it reports an error (or will + // exhaust disk space). Added this assert to report error early. + ASSERT_TRUE(entry_len_ < std::numeric_limits::max()); + } +} + +TEST_P(DbKvChecksumTest, MemTableAddWithColumnFamilyCorrupted) { + // This test repeatedly attempts to write `WriteBatch`es containing a single + // entry of type `op_type_` to a non-default column family. Each attempt has + // one byte corrupted in its memtable entry by adding `corrupt_byte_addend_` + // to its original value. The test repeats until an attempt has been made on + // each byte in the encoded memtable entry. All attempts are expected to fail + // with `Status::Corruption`. + Options options = CurrentOptions(); + if (op_type_ == WriteBatchOpType::kMerge) { + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + } + CreateAndReopenWithCF({"pikachu"}, options); + SyncPoint::GetInstance()->SetCallBack( + "MemTable::Add:Encoded", + std::bind(&DbKvChecksumTest::CorruptNextByteCallBack, this, + std::placeholders::_1)); + + while (MoreBytesToCorrupt()) { + // Failed memtable insert always leads to read-only mode, so we have to + // reopen for every attempt. + ReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu"}, options); + + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_TRUE(ExecuteWrite(handles_[1]).IsCorruption()); + SyncPoint::GetInstance()->DisableProcessing(); + + // In case the above callback is not invoked, this test will run + // numeric_limits::max() times until it reports an error (or will + // exhaust disk space). Added this assert to report error early. + ASSERT_TRUE(entry_len_ < std::numeric_limits::max()); + } +} + +TEST_P(DbKvChecksumTest, NoCorruptionCase) { + // If this test fails, we may have found a piece of malfunctioned hardware + auto batch_and_status = + GetWriteBatch(GetCFHandleToUse(nullptr, op_type_), + 8 /* protection_bytes_per_key */, op_type_); + ASSERT_OK(batch_and_status.second); + ASSERT_OK(batch_and_status.first.VerifyChecksum()); +} + +TEST_P(DbKvChecksumTest, WriteToWALCorrupted) { + // This test repeatedly attempts to write `WriteBatch`es containing a single + // entry of type `op_type_`. Each attempt has one byte corrupted by adding + // `corrupt_byte_addend_` to its original value. The test repeats until an + // attempt has been made on each byte in the encoded write batch. All attempts + // are expected to fail with `Status::Corruption` + Options options = CurrentOptions(); + if (op_type_ == WriteBatchOpType::kMerge) { + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + } + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::WriteToWAL:log_entry", + std::bind(&DbKvChecksumTest::CorruptNextByteCallBack, this, + std::placeholders::_1)); + // First 8 bytes are for sequence number which is not protected in write batch + corrupt_byte_offset_ = 8; + + while (MoreBytesToCorrupt()) { + // Corrupted write batch leads to read-only mode, so we have to + // reopen for every attempt. + Reopen(options); + auto log_size_pre_write = dbfull()->TEST_total_log_size(); + + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_TRUE(ExecuteWrite(nullptr /* cf_handle */).IsCorruption()); + // Confirm that nothing was written to WAL + ASSERT_EQ(log_size_pre_write, dbfull()->TEST_total_log_size()); + ASSERT_TRUE(dbfull()->TEST_GetBGError().IsCorruption()); + SyncPoint::GetInstance()->DisableProcessing(); + + // In case the above callback is not invoked, this test will run + // numeric_limits::max() times until it reports an error (or will + // exhaust disk space). Added this assert to report error early. + ASSERT_TRUE(entry_len_ < std::numeric_limits::max()); + } +} + +TEST_P(DbKvChecksumTest, WriteToWALWithColumnFamilyCorrupted) { + // This test repeatedly attempts to write `WriteBatch`es containing a single + // entry of type `op_type_`. Each attempt has one byte corrupted by adding + // `corrupt_byte_addend_` to its original value. The test repeats until an + // attempt has been made on each byte in the encoded write batch. All attempts + // are expected to fail with `Status::Corruption` + Options options = CurrentOptions(); + if (op_type_ == WriteBatchOpType::kMerge) { + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + } + CreateAndReopenWithCF({"pikachu"}, options); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::WriteToWAL:log_entry", + std::bind(&DbKvChecksumTest::CorruptNextByteCallBack, this, + std::placeholders::_1)); + // First 8 bytes are for sequence number which is not protected in write batch + corrupt_byte_offset_ = 8; + + while (MoreBytesToCorrupt()) { + // Corrupted write batch leads to read-only mode, so we have to + // reopen for every attempt. + ReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu"}, options); + auto log_size_pre_write = dbfull()->TEST_total_log_size(); + + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_TRUE(ExecuteWrite(nullptr /* cf_handle */).IsCorruption()); + // Confirm that nothing was written to WAL + ASSERT_EQ(log_size_pre_write, dbfull()->TEST_total_log_size()); + ASSERT_TRUE(dbfull()->TEST_GetBGError().IsCorruption()); + SyncPoint::GetInstance()->DisableProcessing(); + + // In case the above callback is not invoked, this test will run + // numeric_limits::max() times until it reports an error (or will + // exhaust disk space). Added this assert to report error early. + ASSERT_TRUE(entry_len_ < std::numeric_limits::max()); + } +} + +class DbKvChecksumTestMergedBatch + : public DbKvChecksumTestBase, + public ::testing::WithParamInterface< + std::tuple> { + public: + DbKvChecksumTestMergedBatch() + : DbKvChecksumTestBase("db_kv_checksum_test", /*env_do_fsync=*/false) { + op_type1_ = std::get<0>(GetParam()); + op_type2_ = std::get<1>(GetParam()); + corrupt_byte_addend_ = std::get<2>(GetParam()); + } + + protected: + WriteBatchOpType op_type1_; + WriteBatchOpType op_type2_; + char corrupt_byte_addend_; +}; + +void CorruptWriteBatch(Slice* content, size_t offset, + char corrupt_byte_addend) { + ASSERT_TRUE(offset < content->size()); + char* buf = const_cast(content->data()); + buf[offset] += corrupt_byte_addend; +} + +TEST_P(DbKvChecksumTestMergedBatch, NoCorruptionCase) { + // Veirfy write batch checksum after write batch append + auto batch1 = GetWriteBatch(GetCFHandleToUse(nullptr, op_type1_), + 8 /* protection_bytes_per_key */, op_type1_); + ASSERT_OK(batch1.second); + auto batch2 = GetWriteBatch(GetCFHandleToUse(nullptr, op_type2_), + 8 /* protection_bytes_per_key */, op_type2_); + ASSERT_OK(batch2.second); + ASSERT_OK(WriteBatchInternal::Append(&batch1.first, &batch2.first)); + ASSERT_OK(batch1.first.VerifyChecksum()); +} + +TEST_P(DbKvChecksumTestMergedBatch, WriteToWALCorrupted) { + // This test has two writers repeatedly attempt to write `WriteBatch`es + // containing a single entry of type op_type1_ and op_type2_ respectively. The + // leader of the write group writes the batch containinng the entry of type + // op_type1_. One byte of the pre-merged write batches is corrupted by adding + // `corrupt_byte_addend_` to the batch's original value during each attempt. + // The test repeats until an attempt has been made on each byte in both + // pre-merged write batches. All attempts are expected to fail with + // `Status::Corruption`. + Options options = CurrentOptions(); + if (op_type1_ == WriteBatchOpType::kMerge || + op_type2_ == WriteBatchOpType::kMerge) { + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + } + + auto leader_batch_and_status = + GetWriteBatch(GetCFHandleToUse(nullptr, op_type1_), + 8 /* protection_bytes_per_key */, op_type1_); + ASSERT_OK(leader_batch_and_status.second); + auto follower_batch_and_status = + GetWriteBatch(GetCFHandleToUse(nullptr, op_type2_), + 8 /* protection_bytes_per_key */, op_type2_); + size_t leader_batch_size = leader_batch_and_status.first.GetDataSize(); + size_t total_bytes = + leader_batch_size + follower_batch_and_status.first.GetDataSize(); + // First 8 bytes are for sequence number which is not protected in write batch + size_t corrupt_byte_offset = 8; + + std::atomic follower_joined{false}; + std::atomic leader_count{0}; + port::Thread follower_thread; + // This callback should only be called by the leader thread + SyncPoint::GetInstance()->SetCallBack( + "WriteThread::JoinBatchGroup:Wait2", [&](void* arg_leader) { + auto* leader = reinterpret_cast(arg_leader); + ASSERT_EQ(leader->state, WriteThread::STATE_GROUP_LEADER); + + // This callback should only be called by the follower thread + SyncPoint::GetInstance()->SetCallBack( + "WriteThread::JoinBatchGroup:Wait", [&](void* arg_follower) { + auto* follower = + reinterpret_cast(arg_follower); + // The leader thread will wait on this bool and hence wait until + // this writer joins the write group + ASSERT_NE(follower->state, WriteThread::STATE_GROUP_LEADER); + if (corrupt_byte_offset >= leader_batch_size) { + Slice batch_content = follower->batch->Data(); + CorruptWriteBatch(&batch_content, + corrupt_byte_offset - leader_batch_size, + corrupt_byte_addend_); + } + // Leader busy waits on this flag + follower_joined = true; + // So the follower does not enter the outer callback at + // WriteThread::JoinBatchGroup:Wait2 + SyncPoint::GetInstance()->DisableProcessing(); + }); + + // Start the other writer thread which will join the write group as + // follower + follower_thread = port::Thread([&]() { + follower_batch_and_status = + GetWriteBatch(GetCFHandleToUse(nullptr, op_type2_), + 8 /* protection_bytes_per_key */, op_type2_); + ASSERT_OK(follower_batch_and_status.second); + ASSERT_TRUE( + db_->Write(WriteOptions(), &follower_batch_and_status.first) + .IsCorruption()); + }); + + ASSERT_EQ(leader->batch->GetDataSize(), leader_batch_size); + if (corrupt_byte_offset < leader_batch_size) { + Slice batch_content = leader->batch->Data(); + CorruptWriteBatch(&batch_content, corrupt_byte_offset, + corrupt_byte_addend_); + } + leader_count++; + while (!follower_joined) { + // busy waiting + } + }); + while (corrupt_byte_offset < total_bytes) { + // Reopen DB since it failed WAL write which lead to read-only mode + Reopen(options); + SyncPoint::GetInstance()->EnableProcessing(); + auto log_size_pre_write = dbfull()->TEST_total_log_size(); + leader_batch_and_status = + GetWriteBatch(GetCFHandleToUse(nullptr, op_type1_), + 8 /* protection_bytes_per_key */, op_type1_); + ASSERT_OK(leader_batch_and_status.second); + ASSERT_TRUE(db_->Write(WriteOptions(), &leader_batch_and_status.first) + .IsCorruption()); + follower_thread.join(); + // Prevent leader thread from entering this callback + SyncPoint::GetInstance()->ClearCallBack("WriteThread::JoinBatchGroup:Wait"); + ASSERT_EQ(1, leader_count); + // Nothing should have been written to WAL + ASSERT_EQ(log_size_pre_write, dbfull()->TEST_total_log_size()); + ASSERT_TRUE(dbfull()->TEST_GetBGError().IsCorruption()); + + corrupt_byte_offset++; + if (corrupt_byte_offset == leader_batch_size) { + // skip over the sequence number part of follower's write batch + corrupt_byte_offset += 8; + } + follower_joined = false; + leader_count = 0; + } + SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_P(DbKvChecksumTestMergedBatch, WriteToWALWithColumnFamilyCorrupted) { + // This test has two writers repeatedly attempt to write `WriteBatch`es + // containing a single entry of type op_type1_ and op_type2_ respectively. The + // leader of the write group writes the batch containinng the entry of type + // op_type1_. One byte of the pre-merged write batches is corrupted by adding + // `corrupt_byte_addend_` to the batch's original value during each attempt. + // The test repeats until an attempt has been made on each byte in both + // pre-merged write batches. All attempts are expected to fail with + // `Status::Corruption`. + Options options = CurrentOptions(); + if (op_type1_ == WriteBatchOpType::kMerge || + op_type2_ == WriteBatchOpType::kMerge) { + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + } + CreateAndReopenWithCF({"ramen"}, options); + + auto leader_batch_and_status = + GetWriteBatch(GetCFHandleToUse(handles_[1], op_type1_), + 8 /* protection_bytes_per_key */, op_type1_); + ASSERT_OK(leader_batch_and_status.second); + auto follower_batch_and_status = + GetWriteBatch(GetCFHandleToUse(handles_[1], op_type2_), + 8 /* protection_bytes_per_key */, op_type2_); + size_t leader_batch_size = leader_batch_and_status.first.GetDataSize(); + size_t total_bytes = + leader_batch_size + follower_batch_and_status.first.GetDataSize(); + // First 8 bytes are for sequence number which is not protected in write batch + size_t corrupt_byte_offset = 8; + + std::atomic follower_joined{false}; + std::atomic leader_count{0}; + port::Thread follower_thread; + // This callback should only be called by the leader thread + SyncPoint::GetInstance()->SetCallBack( + "WriteThread::JoinBatchGroup:Wait2", [&](void* arg_leader) { + auto* leader = reinterpret_cast(arg_leader); + ASSERT_EQ(leader->state, WriteThread::STATE_GROUP_LEADER); + + // This callback should only be called by the follower thread + SyncPoint::GetInstance()->SetCallBack( + "WriteThread::JoinBatchGroup:Wait", [&](void* arg_follower) { + auto* follower = + reinterpret_cast(arg_follower); + // The leader thread will wait on this bool and hence wait until + // this writer joins the write group + ASSERT_NE(follower->state, WriteThread::STATE_GROUP_LEADER); + if (corrupt_byte_offset >= leader_batch_size) { + Slice batch_content = + WriteBatchInternal::Contents(follower->batch); + CorruptWriteBatch(&batch_content, + corrupt_byte_offset - leader_batch_size, + corrupt_byte_addend_); + } + follower_joined = true; + // So the follower does not enter the outer callback at + // WriteThread::JoinBatchGroup:Wait2 + SyncPoint::GetInstance()->DisableProcessing(); + }); + + // Start the other writer thread which will join the write group as + // follower + follower_thread = port::Thread([&]() { + follower_batch_and_status = + GetWriteBatch(GetCFHandleToUse(handles_[1], op_type2_), + 8 /* protection_bytes_per_key */, op_type2_); + ASSERT_OK(follower_batch_and_status.second); + ASSERT_TRUE( + db_->Write(WriteOptions(), &follower_batch_and_status.first) + .IsCorruption()); + }); + + ASSERT_EQ(leader->batch->GetDataSize(), leader_batch_size); + if (corrupt_byte_offset < leader_batch_size) { + Slice batch_content = WriteBatchInternal::Contents(leader->batch); + CorruptWriteBatch(&batch_content, corrupt_byte_offset, + corrupt_byte_addend_); + } + leader_count++; + while (!follower_joined) { + // busy waiting + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + while (corrupt_byte_offset < total_bytes) { + // Reopen DB since it failed WAL write which lead to read-only mode + ReopenWithColumnFamilies({kDefaultColumnFamilyName, "ramen"}, options); + SyncPoint::GetInstance()->EnableProcessing(); + auto log_size_pre_write = dbfull()->TEST_total_log_size(); + leader_batch_and_status = + GetWriteBatch(GetCFHandleToUse(handles_[1], op_type1_), + 8 /* protection_bytes_per_key */, op_type1_); + ASSERT_OK(leader_batch_and_status.second); + ASSERT_TRUE(db_->Write(WriteOptions(), &leader_batch_and_status.first) + .IsCorruption()); + follower_thread.join(); + // Prevent leader thread from entering this callback + SyncPoint::GetInstance()->ClearCallBack("WriteThread::JoinBatchGroup:Wait"); + + ASSERT_EQ(1, leader_count); + // Nothing should have been written to WAL + ASSERT_EQ(log_size_pre_write, dbfull()->TEST_total_log_size()); + ASSERT_TRUE(dbfull()->TEST_GetBGError().IsCorruption()); + + corrupt_byte_offset++; + if (corrupt_byte_offset == leader_batch_size) { + // skip over the sequence number part of follower's write batch + corrupt_byte_offset += 8; + } + follower_joined = false; + leader_count = 0; + } + SyncPoint::GetInstance()->DisableProcessing(); +} + +INSTANTIATE_TEST_CASE_P( + DbKvChecksumTestMergedBatch, DbKvChecksumTestMergedBatch, + ::testing::Combine(::testing::Range(static_cast(0), + WriteBatchOpType::kNum), + ::testing::Range(static_cast(0), + WriteBatchOpType::kNum), + ::testing::Values(2, 103, 251)), + [](const testing::TestParamInfo< + std::tuple>& args) { + std::ostringstream oss; + oss << GetOpTypeString(std::get<0>(args.param)) + << GetOpTypeString(std::get<1>(args.param)) << "Add" + << static_cast( + static_cast(std::get<2>(args.param))); + return oss.str(); + }); + +// TODO: add test for transactions +// TODO: add test for corrupted write batch with WAL disabled + +class DbKVChecksumWALToWriteBatchTest : public DBTestBase { + public: + DbKVChecksumWALToWriteBatchTest() + : DBTestBase("db_kv_checksum_test", /*env_do_fsync=*/false) {} +}; + +TEST_F(DbKVChecksumWALToWriteBatchTest, WriteBatchChecksumHandoff) { + Options options = CurrentOptions(); + Reopen(options); + ASSERT_OK(db_->Put(WriteOptions(), "key", "val")); + std::string content = ""; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::RecoverLogFiles:BeforeUpdateProtectionInfo:batch", + [&](void* batch_ptr) { + WriteBatch* batch = reinterpret_cast(batch_ptr); + content.assign(batch->Data().data(), batch->GetDataSize()); + Slice batch_content = batch->Data(); + // Corrupt first bit + CorruptWriteBatch(&batch_content, 0, 1); + }); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::RecoverLogFiles:BeforeUpdateProtectionInfo:checksum", + [&](void* checksum_ptr) { + // Verify that checksum is produced on the batch content + uint64_t checksum = *reinterpret_cast(checksum_ptr); + ASSERT_EQ(checksum, XXH3_64bits(content.data(), content.size())); + }); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_TRUE(TryReopen(options).IsCorruption()); + SyncPoint::GetInstance()->DisableProcessing(); +}; + +// TODO (cbi): add DeleteRange coverage once it is implemented +class DbMemtableKVChecksumTest : public DbKvChecksumTest { + public: + DbMemtableKVChecksumTest() : DbKvChecksumTest() {} + + protected: + // Indices in the memtable entry that we will not corrupt. + // For memtable entry format, see comments in MemTable::Add(). + // We do not corrupt key length and value length fields in this test + // case since it causes segfault and ASAN will complain. + // For this test case, key and value are all of length 3, so + // key length field is at index 0 and value length field is at index 12. + const std::set index_not_to_corrupt{0, 12}; + + void SkipNotToCorruptEntry() { + if (index_not_to_corrupt.find(corrupt_byte_offset_) != + index_not_to_corrupt.end()) { + corrupt_byte_offset_++; + } + } +}; + +INSTANTIATE_TEST_CASE_P( + DbMemtableKVChecksumTest, DbMemtableKVChecksumTest, + ::testing::Combine(::testing::Range(static_cast(0), + WriteBatchOpType::kDeleteRange), + ::testing::Values(2, 103, 251), + ::testing::Range(static_cast(0), + WriteMode::kWriteOptionProtectedBatch), + // skip 1 byte checksum as it makes test flaky + ::testing::Values(2, 4, 8)), + [](const testing::TestParamInfo< + std::tuple>& args) { + std::ostringstream oss; + oss << GetOpTypeString(std::get<0>(args.param)) << "Add" + << static_cast( + static_cast(std::get<1>(args.param))) + << GetWriteModeString(std::get<2>(args.param)) + << static_cast(std::get<3>(args.param)); + return oss.str(); + }); + +TEST_P(DbMemtableKVChecksumTest, GetWithCorruptAfterMemtableInsert) { + // Record memtable entry size. + // Not corrupting memtable entry here since it will segfault + // or fail some asserts inside memtablerep implementation + // e.g., when key_len is corrupted. + SyncPoint::GetInstance()->SetCallBack( + "MemTable::Add:BeforeReturn:Encoded", [&](void* arg) { + Slice encoded = *static_cast(arg); + entry_len_ = encoded.size(); + }); + + SyncPoint::GetInstance()->SetCallBack( + "Memtable::SaveValue:Begin:entry", [&](void* entry) { + char* buf = *static_cast(entry); + buf[corrupt_byte_offset_] += corrupt_byte_addend_; + ++corrupt_byte_offset_; + }); + SyncPoint::GetInstance()->EnableProcessing(); + Options options = CurrentOptions(); + options.memtable_protection_bytes_per_key = + memtable_protection_bytes_per_key_; + if (op_type_ == WriteBatchOpType::kMerge) { + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + } + + SkipNotToCorruptEntry(); + while (MoreBytesToCorrupt()) { + Reopen(options); + ASSERT_OK(ExecuteWrite(nullptr)); + std::string val; + ASSERT_TRUE(db_->Get(ReadOptions(), "key", &val).IsCorruption()); + Destroy(options); + SkipNotToCorruptEntry(); + } +} + +TEST_P(DbMemtableKVChecksumTest, + GetWithColumnFamilyCorruptAfterMemtableInsert) { + // Record memtable entry size. + // Not corrupting memtable entry here since it will segfault + // or fail some asserts inside memtablerep implementation + // e.g., when key_len is corrupted. + SyncPoint::GetInstance()->SetCallBack( + "MemTable::Add:BeforeReturn:Encoded", [&](void* arg) { + Slice encoded = *static_cast(arg); + entry_len_ = encoded.size(); + }); + + SyncPoint::GetInstance()->SetCallBack( + "Memtable::SaveValue:Begin:entry", [&](void* entry) { + char* buf = *static_cast(entry); + buf[corrupt_byte_offset_] += corrupt_byte_addend_; + ++corrupt_byte_offset_; + }); + SyncPoint::GetInstance()->EnableProcessing(); + Options options = CurrentOptions(); + options.memtable_protection_bytes_per_key = + memtable_protection_bytes_per_key_; + if (op_type_ == WriteBatchOpType::kMerge) { + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + } + + SkipNotToCorruptEntry(); + while (MoreBytesToCorrupt()) { + Reopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + ASSERT_OK(ExecuteWrite(handles_[1])); + std::string val; + ASSERT_TRUE( + db_->Get(ReadOptions(), handles_[1], "key", &val).IsCorruption()); + Destroy(options); + SkipNotToCorruptEntry(); + } +} + +TEST_P(DbMemtableKVChecksumTest, IteratorWithCorruptAfterMemtableInsert) { + SyncPoint::GetInstance()->SetCallBack( + "MemTable::Add:BeforeReturn:Encoded", + std::bind(&DbKvChecksumTest::CorruptNextByteCallBack, this, + std::placeholders::_1)); + SyncPoint::GetInstance()->EnableProcessing(); + Options options = CurrentOptions(); + options.memtable_protection_bytes_per_key = + memtable_protection_bytes_per_key_; + if (op_type_ == WriteBatchOpType::kMerge) { + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + } + + SkipNotToCorruptEntry(); + while (MoreBytesToCorrupt()) { + Reopen(options); + ASSERT_OK(ExecuteWrite(nullptr)); + Iterator* it = db_->NewIterator(ReadOptions()); + it->SeekToFirst(); + ASSERT_FALSE(it->Valid()); + ASSERT_TRUE(it->status().IsCorruption()); + delete it; + Destroy(options); + SkipNotToCorruptEntry(); + } +} + +TEST_P(DbMemtableKVChecksumTest, + IteratorWithColumnFamilyCorruptAfterMemtableInsert) { + SyncPoint::GetInstance()->SetCallBack( + "MemTable::Add:BeforeReturn:Encoded", + std::bind(&DbKvChecksumTest::CorruptNextByteCallBack, this, + std::placeholders::_1)); + SyncPoint::GetInstance()->EnableProcessing(); + Options options = CurrentOptions(); + options.memtable_protection_bytes_per_key = + memtable_protection_bytes_per_key_; + if (op_type_ == WriteBatchOpType::kMerge) { + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + } + + SkipNotToCorruptEntry(); + while (MoreBytesToCorrupt()) { + Reopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + ASSERT_OK(ExecuteWrite(handles_[1])); + Iterator* it = db_->NewIterator(ReadOptions(), handles_[1]); + it->SeekToFirst(); + ASSERT_FALSE(it->Valid()); + ASSERT_TRUE(it->status().IsCorruption()); + delete it; + Destroy(options); + SkipNotToCorruptEntry(); + } +} + +TEST_P(DbMemtableKVChecksumTest, FlushWithCorruptAfterMemtableInsert) { + SyncPoint::GetInstance()->SetCallBack( + "MemTable::Add:BeforeReturn:Encoded", + std::bind(&DbKvChecksumTest::CorruptNextByteCallBack, this, + std::placeholders::_1)); + SyncPoint::GetInstance()->EnableProcessing(); + Options options = CurrentOptions(); + options.memtable_protection_bytes_per_key = + memtable_protection_bytes_per_key_; + if (op_type_ == WriteBatchOpType::kMerge) { + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + } + + SkipNotToCorruptEntry(); + // Not corruping each byte like other tests since Flush() is relatively slow. + Reopen(options); + ASSERT_OK(ExecuteWrite(nullptr)); + ASSERT_TRUE(Flush().IsCorruption()); + // DB enters read-only state when flush reads corrupted data + ASSERT_TRUE(dbfull()->TEST_GetBGError().IsCorruption()); + Destroy(options); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_log_iter_test.cc b/librocksdb-sys/rocksdb/db/db_log_iter_test.cc new file mode 100644 index 0000000..4c94345 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_log_iter_test.cc @@ -0,0 +1,297 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +// Introduction of SyncPoint effectively disabled building and running this test +// in Release build. +// which is a pity, it is a good test + +#include "db/db_test_util.h" +#include "env/mock_env.h" +#include "port/stack_trace.h" + +namespace ROCKSDB_NAMESPACE { + +class DBTestXactLogIterator : public DBTestBase { + public: + DBTestXactLogIterator() + : DBTestBase("db_log_iter_test", /*env_do_fsync=*/true) {} + + std::unique_ptr OpenTransactionLogIter( + const SequenceNumber seq) { + std::unique_ptr iter; + Status status = dbfull()->GetUpdatesSince(seq, &iter); + EXPECT_OK(status); + EXPECT_TRUE(iter->Valid()); + return iter; + } +}; + +namespace { +SequenceNumber ReadRecords(std::unique_ptr& iter, + int& count, bool expect_ok = true) { + count = 0; + SequenceNumber lastSequence = 0; + BatchResult res; + while (iter->Valid()) { + res = iter->GetBatch(); + EXPECT_TRUE(res.sequence > lastSequence); + ++count; + lastSequence = res.sequence; + EXPECT_OK(iter->status()); + iter->Next(); + } + if (expect_ok) { + EXPECT_OK(iter->status()); + } else { + EXPECT_NOK(iter->status()); + } + return res.sequence; +} + +void ExpectRecords(const int expected_no_records, + std::unique_ptr& iter) { + int num_records; + ReadRecords(iter, num_records); + ASSERT_EQ(num_records, expected_no_records); +} +} // anonymous namespace + +TEST_F(DBTestXactLogIterator, TransactionLogIterator) { + do { + Options options = OptionsForLogIterTest(); + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + ASSERT_OK(Put(0, "key1", DummyString(1024))); + ASSERT_OK(Put(1, "key2", DummyString(1024))); + ASSERT_OK(Put(1, "key2", DummyString(1024))); + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 3U); + { + auto iter = OpenTransactionLogIter(0); + ExpectRecords(3, iter); + } + ReopenWithColumnFamilies({"default", "pikachu"}, options); + env_->SleepForMicroseconds(2 * 1000 * 1000); + { + ASSERT_OK(Put(0, "key4", DummyString(1024))); + ASSERT_OK(Put(1, "key5", DummyString(1024))); + ASSERT_OK(Put(0, "key6", DummyString(1024))); + } + { + auto iter = OpenTransactionLogIter(0); + ExpectRecords(6, iter); + } + } while (ChangeCompactOptions()); +} + +#ifndef NDEBUG // sync point is not included with DNDEBUG build +TEST_F(DBTestXactLogIterator, TransactionLogIteratorRace) { + static const int LOG_ITERATOR_RACE_TEST_COUNT = 2; + static const char* sync_points[LOG_ITERATOR_RACE_TEST_COUNT][4] = { + {"WalManager::GetSortedWalFiles:1", "WalManager::PurgeObsoleteFiles:1", + "WalManager::PurgeObsoleteFiles:2", "WalManager::GetSortedWalFiles:2"}, + {"WalManager::GetSortedWalsOfType:1", "WalManager::PurgeObsoleteFiles:1", + "WalManager::PurgeObsoleteFiles:2", + "WalManager::GetSortedWalsOfType:2"}}; + for (int test = 0; test < LOG_ITERATOR_RACE_TEST_COUNT; ++test) { + // Setup sync point dependency to reproduce the race condition of + // a log file moved to archived dir, in the middle of GetSortedWalFiles + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {sync_points[test][0], sync_points[test][1]}, + {sync_points[test][2], sync_points[test][3]}, + }); + + do { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + Options options = OptionsForLogIterTest(); + DestroyAndReopen(options); + ASSERT_OK(Put("key1", DummyString(1024))); + ASSERT_OK(dbfull()->Flush(FlushOptions())); + ASSERT_OK(Put("key2", DummyString(1024))); + ASSERT_OK(dbfull()->Flush(FlushOptions())); + ASSERT_OK(Put("key3", DummyString(1024))); + ASSERT_OK(dbfull()->Flush(FlushOptions())); + ASSERT_OK(Put("key4", DummyString(1024))); + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 4U); + ASSERT_OK(dbfull()->FlushWAL(false)); + + { + auto iter = OpenTransactionLogIter(0); + ExpectRecords(4, iter); + } + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + // trigger async flush, and log move. Well, log move will + // wait until the GetSortedWalFiles:1 to reproduce the race + // condition + FlushOptions flush_options; + flush_options.wait = false; + ASSERT_OK(dbfull()->Flush(flush_options)); + + // "key5" would be written in a new memtable and log + ASSERT_OK(Put("key5", DummyString(1024))); + ASSERT_OK(dbfull()->FlushWAL(false)); + { + // this iter would miss "key4" if not fixed + auto iter = OpenTransactionLogIter(0); + ExpectRecords(5, iter); + } + } while (ChangeCompactOptions()); + } +} +#endif + +TEST_F(DBTestXactLogIterator, TransactionLogIteratorStallAtLastRecord) { + do { + Options options = OptionsForLogIterTest(); + DestroyAndReopen(options); + ASSERT_OK(Put("key1", DummyString(1024))); + auto iter = OpenTransactionLogIter(0); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + iter->Next(); + ASSERT_TRUE(!iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_OK(Put("key2", DummyString(1024))); + iter->Next(); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + } while (ChangeCompactOptions()); +} + +TEST_F(DBTestXactLogIterator, TransactionLogIteratorCheckAfterRestart) { + do { + Options options = OptionsForLogIterTest(); + DestroyAndReopen(options); + ASSERT_OK(Put("key1", DummyString(1024))); + ASSERT_OK(Put("key2", DummyString(1023))); + ASSERT_OK(dbfull()->Flush(FlushOptions())); + Reopen(options); + auto iter = OpenTransactionLogIter(0); + ExpectRecords(2, iter); + } while (ChangeCompactOptions()); +} + +TEST_F(DBTestXactLogIterator, TransactionLogIteratorCorruptedLog) { + do { + Options options = OptionsForLogIterTest(); + DestroyAndReopen(options); + + for (int i = 0; i < 1024; i++) { + ASSERT_OK(Put("key" + std::to_string(i), DummyString(10))); + } + + ASSERT_OK(Flush()); + ASSERT_OK(db_->FlushWAL(false)); + + // Corrupt this log to create a gap + ASSERT_OK(db_->DisableFileDeletions()); + + VectorLogPtr wal_files; + ASSERT_OK(db_->GetSortedWalFiles(wal_files)); + ASSERT_FALSE(wal_files.empty()); + + const auto logfile_path = dbname_ + "/" + wal_files.front()->PathName(); + ASSERT_OK(test::TruncateFile(env_, logfile_path, + wal_files.front()->SizeFileBytes() / 2)); + + ASSERT_OK(db_->EnableFileDeletions()); + + // Insert a new entry to a new log file + ASSERT_OK(Put("key1025", DummyString(10))); + ASSERT_OK(db_->FlushWAL(false)); + + // Try to read from the beginning. Should stop before the gap and read less + // than 1025 entries + auto iter = OpenTransactionLogIter(0); + int count = 0; + SequenceNumber last_sequence_read = ReadRecords(iter, count, false); + ASSERT_LT(last_sequence_read, 1025U); + + // Try to read past the gap, should be able to seek to key1025 + auto iter2 = OpenTransactionLogIter(last_sequence_read + 1); + ExpectRecords(1, iter2); + } while (ChangeCompactOptions()); +} + +TEST_F(DBTestXactLogIterator, TransactionLogIteratorBatchOperations) { + do { + Options options = OptionsForLogIterTest(); + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + WriteBatch batch; + ASSERT_OK(batch.Put(handles_[1], "key1", DummyString(1024))); + ASSERT_OK(batch.Put(handles_[0], "key2", DummyString(1024))); + ASSERT_OK(batch.Put(handles_[1], "key3", DummyString(1024))); + ASSERT_OK(batch.Delete(handles_[0], "key2")); + ASSERT_OK(dbfull()->Write(WriteOptions(), &batch)); + ASSERT_OK(Flush(1)); + ASSERT_OK(Flush(0)); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_OK(Put(1, "key4", DummyString(1024))); + auto iter = OpenTransactionLogIter(3); + ExpectRecords(2, iter); + } while (ChangeCompactOptions()); +} + +TEST_F(DBTestXactLogIterator, TransactionLogIteratorBlobs) { + Options options = OptionsForLogIterTest(); + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + { + WriteBatch batch; + ASSERT_OK(batch.Put(handles_[1], "key1", DummyString(1024))); + ASSERT_OK(batch.Put(handles_[0], "key2", DummyString(1024))); + ASSERT_OK(batch.PutLogData(Slice("blob1"))); + ASSERT_OK(batch.Put(handles_[1], "key3", DummyString(1024))); + ASSERT_OK(batch.PutLogData(Slice("blob2"))); + ASSERT_OK(batch.Delete(handles_[0], "key2")); + ASSERT_OK(dbfull()->Write(WriteOptions(), &batch)); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + } + + auto res = OpenTransactionLogIter(0)->GetBatch(); + struct Handler : public WriteBatch::Handler { + std::string seen; + Status PutCF(uint32_t cf, const Slice& key, const Slice& value) override { + seen += "Put(" + std::to_string(cf) + ", " + key.ToString() + ", " + + std::to_string(value.size()) + ")"; + return Status::OK(); + } + Status MergeCF(uint32_t cf, const Slice& key, const Slice& value) override { + seen += "Merge(" + std::to_string(cf) + ", " + key.ToString() + ", " + + std::to_string(value.size()) + ")"; + return Status::OK(); + } + void LogData(const Slice& blob) override { + seen += "LogData(" + blob.ToString() + ")"; + } + Status DeleteCF(uint32_t cf, const Slice& key) override { + seen += "Delete(" + std::to_string(cf) + ", " + key.ToString() + ")"; + return Status::OK(); + } + } handler; + ASSERT_OK(res.writeBatchPtr->Iterate(&handler)); + ASSERT_EQ( + "Put(1, key1, 1024)" + "Put(0, key2, 1024)" + "LogData(blob1)" + "Put(1, key3, 1024)" + "LogData(blob2)" + "Delete(0, key2)", + handler.seen); +} +} // namespace ROCKSDB_NAMESPACE + + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_logical_block_size_cache_test.cc b/librocksdb-sys/rocksdb/db/db_logical_block_size_cache_test.cc new file mode 100644 index 0000000..ff56d56 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_logical_block_size_cache_test.cc @@ -0,0 +1,505 @@ +// Copyright (c) 2020-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "test_util/testharness.h" + +#ifdef OS_LINUX +#include "env/io_posix.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" + +namespace ROCKSDB_NAMESPACE { +class EnvWithCustomLogicalBlockSizeCache : public EnvWrapper { + public: + EnvWithCustomLogicalBlockSizeCache(Env* env, LogicalBlockSizeCache* cache) + : EnvWrapper(env), cache_(cache) {} + + Status RegisterDbPaths(const std::vector& paths) override { + return cache_->RefAndCacheLogicalBlockSize(paths); + } + + Status UnregisterDbPaths(const std::vector& paths) override { + cache_->UnrefAndTryRemoveCachedLogicalBlockSize(paths); + return Status::OK(); + } + + private: + LogicalBlockSizeCache* cache_; +}; + +class DBLogicalBlockSizeCacheTest : public testing::Test { + public: + DBLogicalBlockSizeCacheTest() + : dbname_(test::PerThreadDBPath("logical_block_size_cache_test")), + data_path_0_(dbname_ + "/data_path_0"), + data_path_1_(dbname_ + "/data_path_1"), + cf_path_0_(dbname_ + "/cf_path_0"), + cf_path_1_(dbname_ + "/cf_path_1") { + auto get_fd_block_size = [&](int fd) { return fd; }; + auto get_dir_block_size = [&](const std::string& /*dir*/, size_t* size) { + *size = 1024; + return Status::OK(); + }; + cache_.reset( + new LogicalBlockSizeCache(get_fd_block_size, get_dir_block_size)); + env_.reset( + new EnvWithCustomLogicalBlockSizeCache(Env::Default(), cache_.get())); + } + + protected: + std::string dbname_; + std::string data_path_0_; + std::string data_path_1_; + std::string cf_path_0_; + std::string cf_path_1_; + std::unique_ptr cache_; + std::unique_ptr env_; +}; + +TEST_F(DBLogicalBlockSizeCacheTest, OpenClose) { + // Tests that Open will cache the logical block size for data paths, + // and Close will remove the cached sizes. + Options options; + options.create_if_missing = true; + options.env = env_.get(); + options.db_paths = {{data_path_0_, 2048}, {data_path_1_, 2048}}; + + for (int i = 0; i < 2; i++) { + DB* db; + if (!i) { + printf("Open\n"); + ASSERT_OK(DB::Open(options, dbname_, &db)); + } else { + printf("OpenForReadOnly\n"); + ASSERT_OK(DB::OpenForReadOnly(options, dbname_, &db)); + } + ASSERT_EQ(2, cache_->Size()); + ASSERT_TRUE(cache_->Contains(data_path_0_)); + ASSERT_EQ(1, cache_->GetRefCount(data_path_0_)); + ASSERT_TRUE(cache_->Contains(data_path_1_)); + ASSERT_EQ(1, cache_->GetRefCount(data_path_1_)); + ASSERT_OK(db->Close()); + ASSERT_EQ(0, cache_->Size()); + delete db; + } + ASSERT_OK(DestroyDB(dbname_, options, {})); +} + +TEST_F(DBLogicalBlockSizeCacheTest, OpenDelete) { + // Tests that Open will cache the logical block size for data paths, + // and delete the db pointer will remove the cached sizes. + Options options; + options.create_if_missing = true; + options.env = env_.get(); + + for (int i = 0; i < 2; i++) { + DB* db; + if (!i) { + printf("Open\n"); + ASSERT_OK(DB::Open(options, dbname_, &db)); + } else { + printf("OpenForReadOnly\n"); + ASSERT_OK(DB::OpenForReadOnly(options, dbname_, &db)); + } + ASSERT_EQ(1, cache_->Size()); + ASSERT_TRUE(cache_->Contains(dbname_)); + ASSERT_EQ(1, cache_->GetRefCount(dbname_)); + delete db; + ASSERT_EQ(0, cache_->Size()); + } + ASSERT_OK(DestroyDB(dbname_, options, {})); +} + +TEST_F(DBLogicalBlockSizeCacheTest, CreateColumnFamily) { + // Tests that CreateColumnFamily will cache the cf_paths, + // drop the column family handle won't drop the cache, + // drop and then delete the column family handle will drop the cache. + Options options; + options.create_if_missing = true; + options.env = env_.get(); + ColumnFamilyOptions cf_options; + cf_options.cf_paths = {{cf_path_0_, 1024}, {cf_path_1_, 2048}}; + + DB* db; + ASSERT_OK(DB::Open(options, dbname_, &db)); + ASSERT_EQ(1, cache_->Size()); + ASSERT_TRUE(cache_->Contains(dbname_)); + ASSERT_EQ(1, cache_->GetRefCount(dbname_)); + + ColumnFamilyHandle* cf = nullptr; + ASSERT_OK(db->CreateColumnFamily(cf_options, "cf", &cf)); + ASSERT_EQ(3, cache_->Size()); + ASSERT_TRUE(cache_->Contains(dbname_)); + ASSERT_EQ(1, cache_->GetRefCount(dbname_)); + ASSERT_TRUE(cache_->Contains(cf_path_0_)); + ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_)); + ASSERT_TRUE(cache_->Contains(cf_path_1_)); + ASSERT_EQ(1, cache_->GetRefCount(cf_path_1_)); + + // Drop column family does not drop cache. + ASSERT_OK(db->DropColumnFamily(cf)); + ASSERT_EQ(3, cache_->Size()); + ASSERT_TRUE(cache_->Contains(dbname_)); + ASSERT_EQ(1, cache_->GetRefCount(dbname_)); + ASSERT_TRUE(cache_->Contains(cf_path_0_)); + ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_)); + ASSERT_TRUE(cache_->Contains(cf_path_1_)); + ASSERT_EQ(1, cache_->GetRefCount(cf_path_1_)); + + // Delete handle will drop cache. + ASSERT_OK(db->DestroyColumnFamilyHandle(cf)); + ASSERT_TRUE(cache_->Contains(dbname_)); + ASSERT_EQ(1, cache_->GetRefCount(dbname_)); + + delete db; + ASSERT_EQ(0, cache_->Size()); + ASSERT_OK(DestroyDB(dbname_, options, {{"cf", cf_options}})); +} + +TEST_F(DBLogicalBlockSizeCacheTest, CreateColumnFamilies) { + // To test: + // (1) CreateColumnFamilies will cache the cf_paths in + // DBLogicalBlockSizeCache + // (2) Dropping column family handles associated with + // that cf_paths won't drop the cached cf_paths + // (3) Deleting all the column family handles associated + // with that cf_paths will drop the cached cf_paths + + Options options; + options.create_if_missing = true; + options.env = env_.get(); + ColumnFamilyOptions cf_options; + cf_options.cf_paths = {{cf_path_0_, 1024}}; + + DB* db; + ASSERT_OK(DB::Open(options, dbname_, &db)); + ASSERT_EQ(1, cache_->Size()); + ASSERT_TRUE(cache_->Contains(dbname_)); + ASSERT_EQ(1, cache_->GetRefCount(dbname_)); + + std::vector cfs; + ASSERT_OK(db->CreateColumnFamilies(cf_options, {"cf1", "cf2"}, &cfs)); + ASSERT_EQ(2, cache_->Size()); + ASSERT_TRUE(cache_->Contains(dbname_)); + ASSERT_EQ(1, cache_->GetRefCount(dbname_)); + ASSERT_TRUE(cache_->Contains(cf_path_0_)); + ASSERT_EQ(2, cache_->GetRefCount(cf_path_0_)); + + // Drop column family does not drop cf_path_0_'s entry from cache + for (ColumnFamilyHandle* cf : cfs) { + ASSERT_OK(db->DropColumnFamily(cf)); + ASSERT_EQ(2, cache_->Size()); + ASSERT_TRUE(cache_->Contains(dbname_)); + ASSERT_EQ(1, cache_->GetRefCount(dbname_)); + ASSERT_TRUE(cache_->Contains(cf_path_0_)); + ASSERT_EQ(2, cache_->GetRefCount(cf_path_0_)); + } + + // Delete one cf handle will not drop cf_path_0_'s entry from cache because + // another handle is still referencing cf_path_0_. + ASSERT_OK(db->DestroyColumnFamilyHandle(cfs[0])); + ASSERT_EQ(2, cache_->Size()); + ASSERT_TRUE(cache_->Contains(dbname_)); + ASSERT_EQ(1, cache_->GetRefCount(dbname_)); + ASSERT_TRUE(cache_->Contains(cf_path_0_)); + + // Delete all cf handles and ensure the ref count of cf_path_0_ in cache_ + // can be properly decreased by releasing any background reference to the + // ColumnFamilyData during db deletion + ASSERT_OK(db->DestroyColumnFamilyHandle(cfs[1])); + ASSERT_TRUE(cache_->Contains(dbname_)); + ASSERT_EQ(1, cache_->GetRefCount(dbname_)); + delete db; + + // Now cf_path_0_ in cache_ has been properly decreased and cf_path_0_'s entry + // is dropped from cache + ASSERT_EQ(0, cache_->Size()); + ASSERT_OK( + DestroyDB(dbname_, options, {{"cf1", cf_options}, {"cf2", cf_options}})); +} + +TEST_F(DBLogicalBlockSizeCacheTest, OpenWithColumnFamilies) { + // Tests that Open two column families with the same cf_path will cache the + // cf_path and have 2 references to the cached size, + // drop the column family handle won't drop the cache, + // drop and then delete the column family handle will drop the cache. + Options options; + options.create_if_missing = true; + options.env = env_.get(); + + ColumnFamilyOptions cf_options; + cf_options.cf_paths = {{cf_path_0_, 1024}}; + + for (int i = 0; i < 2; i++) { + DB* db; + ColumnFamilyHandle* cf1 = nullptr; + ColumnFamilyHandle* cf2 = nullptr; + ASSERT_OK(DB::Open(options, dbname_, &db)); + ASSERT_OK(db->CreateColumnFamily(cf_options, "cf1", &cf1)); + ASSERT_OK(db->CreateColumnFamily(cf_options, "cf2", &cf2)); + ASSERT_OK(db->DestroyColumnFamilyHandle(cf1)); + ASSERT_OK(db->DestroyColumnFamilyHandle(cf2)); + delete db; + ASSERT_EQ(0, cache_->Size()); + + std::vector cfs; + if (!i) { + printf("Open\n"); + ASSERT_OK(DB::Open(options, dbname_, + {{"cf1", cf_options}, + {"cf2", cf_options}, + {"default", ColumnFamilyOptions()}}, + &cfs, &db)); + } else { + printf("OpenForReadOnly\n"); + ASSERT_OK(DB::OpenForReadOnly(options, dbname_, + {{"cf1", cf_options}, + {"cf2", cf_options}, + {"default", ColumnFamilyOptions()}}, + &cfs, &db)); + } + + // Logical block sizes of dbname_ and cf_path_0_ are cached during Open. + ASSERT_EQ(2, cache_->Size()); + ASSERT_TRUE(cache_->Contains(dbname_)); + ASSERT_EQ(1, cache_->GetRefCount(dbname_)); + ASSERT_TRUE(cache_->Contains(cf_path_0_)); + ASSERT_EQ(2, cache_->GetRefCount(cf_path_0_)); + + // Drop handles won't drop the cache. + ASSERT_OK(db->DropColumnFamily(cfs[0])); + ASSERT_OK(db->DropColumnFamily(cfs[1])); + ASSERT_EQ(2, cache_->Size()); + ASSERT_TRUE(cache_->Contains(dbname_)); + ASSERT_EQ(1, cache_->GetRefCount(dbname_)); + ASSERT_TRUE(cache_->Contains(cf_path_0_)); + ASSERT_EQ(2, cache_->GetRefCount(cf_path_0_)); + + // Delete 1st handle won't drop the cache for cf_path_0_. + ASSERT_OK(db->DestroyColumnFamilyHandle(cfs[0])); + ASSERT_EQ(2, cache_->Size()); + ASSERT_TRUE(cache_->Contains(dbname_)); + ASSERT_EQ(1, cache_->GetRefCount(dbname_)); + ASSERT_TRUE(cache_->Contains(cf_path_0_)); + ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_)); + + // Delete 2nd handle will drop the cache for cf_path_0_. + ASSERT_OK(db->DestroyColumnFamilyHandle(cfs[1])); + ASSERT_EQ(1, cache_->Size()); + ASSERT_TRUE(cache_->Contains(dbname_)); + ASSERT_EQ(1, cache_->GetRefCount(dbname_)); + + // Delete the default handle won't affect the cache because db still refers + // to the default CF. + ASSERT_OK(db->DestroyColumnFamilyHandle(cfs[2])); + ASSERT_EQ(1, cache_->Size()); + ASSERT_TRUE(cache_->Contains(dbname_)); + ASSERT_EQ(1, cache_->GetRefCount(dbname_)); + + delete db; + ASSERT_EQ(0, cache_->Size()); + } + ASSERT_OK( + DestroyDB(dbname_, options, {{"cf1", cf_options}, {"cf2", cf_options}})); +} + +TEST_F(DBLogicalBlockSizeCacheTest, DestroyColumnFamilyHandle) { + // Tests that destroy column family without dropping won't drop the cache, + // because compaction and flush might still need to get logical block size + // when opening new files. + Options options; + options.create_if_missing = true; + options.env = env_.get(); + ColumnFamilyOptions cf_options; + cf_options.cf_paths = {{cf_path_0_, 1024}}; + + DB* db; + ASSERT_OK(DB::Open(options, dbname_, &db)); + ASSERT_EQ(1, cache_->Size()); + ASSERT_TRUE(cache_->Contains(dbname_)); + ASSERT_EQ(1, cache_->GetRefCount(dbname_)); + ColumnFamilyHandle* cf = nullptr; + ASSERT_OK(db->CreateColumnFamily(cf_options, "cf", &cf)); + ASSERT_EQ(2, cache_->Size()); + ASSERT_TRUE(cache_->Contains(dbname_)); + ASSERT_EQ(1, cache_->GetRefCount(dbname_)); + ASSERT_TRUE(cache_->Contains(cf_path_0_)); + ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_)); + + // Delete handle won't drop cache. + ASSERT_OK(db->DestroyColumnFamilyHandle(cf)); + ASSERT_EQ(2, cache_->Size()); + ASSERT_TRUE(cache_->Contains(dbname_)); + ASSERT_EQ(1, cache_->GetRefCount(dbname_)); + ASSERT_TRUE(cache_->Contains(cf_path_0_)); + ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_)); + + delete db; + ASSERT_EQ(0, cache_->Size()); + + // Open with column families. + std::vector cfs; + for (int i = 0; i < 2; i++) { + if (!i) { + printf("Open\n"); + ASSERT_OK(DB::Open( + options, dbname_, + {{"cf", cf_options}, {"default", ColumnFamilyOptions()}}, &cfs, &db)); + } else { + printf("OpenForReadOnly\n"); + ASSERT_OK(DB::OpenForReadOnly( + options, dbname_, + {{"cf", cf_options}, {"default", ColumnFamilyOptions()}}, &cfs, &db)); + } + // cf_path_0_ and dbname_ are cached. + ASSERT_EQ(2, cache_->Size()); + ASSERT_TRUE(cache_->Contains(dbname_)); + ASSERT_EQ(1, cache_->GetRefCount(dbname_)); + ASSERT_TRUE(cache_->Contains(cf_path_0_)); + ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_)); + + // Deleting handle won't drop cache. + ASSERT_OK(db->DestroyColumnFamilyHandle(cfs[0])); + ASSERT_OK(db->DestroyColumnFamilyHandle(cfs[1])); + ASSERT_EQ(2, cache_->Size()); + ASSERT_TRUE(cache_->Contains(dbname_)); + ASSERT_EQ(1, cache_->GetRefCount(dbname_)); + ASSERT_TRUE(cache_->Contains(cf_path_0_)); + ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_)); + + delete db; + ASSERT_EQ(0, cache_->Size()); + } + ASSERT_OK(DestroyDB(dbname_, options, {{"cf", cf_options}})); +} + +TEST_F(DBLogicalBlockSizeCacheTest, MultiDBWithDifferentPaths) { + // Tests the cache behavior when there are multiple DBs sharing the same env + // with different db_paths and cf_paths. + Options options; + options.create_if_missing = true; + options.env = env_.get(); + + ASSERT_OK(env_->CreateDirIfMissing(dbname_)); + + DB* db0; + ASSERT_OK(DB::Open(options, data_path_0_, &db0)); + ASSERT_EQ(1, cache_->Size()); + ASSERT_TRUE(cache_->Contains(data_path_0_)); + + ColumnFamilyOptions cf_options0; + cf_options0.cf_paths = {{cf_path_0_, 1024}}; + ColumnFamilyHandle* cf0; + ASSERT_OK(db0->CreateColumnFamily(cf_options0, "cf", &cf0)); + ASSERT_EQ(2, cache_->Size()); + ASSERT_TRUE(cache_->Contains(data_path_0_)); + ASSERT_EQ(1, cache_->GetRefCount(data_path_0_)); + ASSERT_TRUE(cache_->Contains(cf_path_0_)); + ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_)); + + DB* db1; + ASSERT_OK(DB::Open(options, data_path_1_, &db1)); + ASSERT_EQ(3, cache_->Size()); + ASSERT_TRUE(cache_->Contains(data_path_0_)); + ASSERT_EQ(1, cache_->GetRefCount(data_path_0_)); + ASSERT_TRUE(cache_->Contains(cf_path_0_)); + ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_)); + ASSERT_TRUE(cache_->Contains(data_path_1_)); + ASSERT_EQ(1, cache_->GetRefCount(data_path_1_)); + + ColumnFamilyOptions cf_options1; + cf_options1.cf_paths = {{cf_path_1_, 1024}}; + ColumnFamilyHandle* cf1; + ASSERT_OK(db1->CreateColumnFamily(cf_options1, "cf", &cf1)); + ASSERT_EQ(4, cache_->Size()); + ASSERT_TRUE(cache_->Contains(data_path_0_)); + ASSERT_EQ(1, cache_->GetRefCount(data_path_0_)); + ASSERT_TRUE(cache_->Contains(cf_path_0_)); + ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_)); + ASSERT_TRUE(cache_->Contains(data_path_1_)); + ASSERT_EQ(1, cache_->GetRefCount(data_path_1_)); + ASSERT_TRUE(cache_->Contains(cf_path_1_)); + ASSERT_EQ(1, cache_->GetRefCount(cf_path_1_)); + + ASSERT_OK(db0->DestroyColumnFamilyHandle(cf0)); + delete db0; + ASSERT_EQ(2, cache_->Size()); + ASSERT_TRUE(cache_->Contains(data_path_1_)); + ASSERT_EQ(1, cache_->GetRefCount(data_path_1_)); + ASSERT_TRUE(cache_->Contains(cf_path_1_)); + ASSERT_EQ(1, cache_->GetRefCount(cf_path_1_)); + ASSERT_OK(DestroyDB(data_path_0_, options, {{"cf", cf_options0}})); + + ASSERT_OK(db1->DestroyColumnFamilyHandle(cf1)); + delete db1; + ASSERT_EQ(0, cache_->Size()); + ASSERT_OK(DestroyDB(data_path_1_, options, {{"cf", cf_options1}})); +} + +TEST_F(DBLogicalBlockSizeCacheTest, MultiDBWithSamePaths) { + // Tests the cache behavior when there are multiple DBs sharing the same env + // with the same db_paths and cf_paths. + Options options; + options.create_if_missing = true; + options.env = env_.get(); + options.db_paths = {{data_path_0_, 1024}}; + ColumnFamilyOptions cf_options; + cf_options.cf_paths = {{cf_path_0_, 1024}}; + + ASSERT_OK(env_->CreateDirIfMissing(dbname_)); + + DB* db0; + ASSERT_OK(DB::Open(options, dbname_ + "/db0", &db0)); + ASSERT_EQ(1, cache_->Size()); + ASSERT_TRUE(cache_->Contains(data_path_0_)); + ASSERT_EQ(1, cache_->GetRefCount(data_path_0_)); + + ColumnFamilyHandle* cf0; + ASSERT_OK(db0->CreateColumnFamily(cf_options, "cf", &cf0)); + ASSERT_EQ(2, cache_->Size()); + ASSERT_TRUE(cache_->Contains(data_path_0_)); + ASSERT_EQ(1, cache_->GetRefCount(data_path_0_)); + ASSERT_TRUE(cache_->Contains(cf_path_0_)); + ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_)); + + DB* db1; + ASSERT_OK(DB::Open(options, dbname_ + "/db1", &db1)); + ASSERT_EQ(2, cache_->Size()); + ASSERT_TRUE(cache_->Contains(data_path_0_)); + ASSERT_EQ(2, cache_->GetRefCount(data_path_0_)); + ASSERT_TRUE(cache_->Contains(cf_path_0_)); + ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_)); + + ColumnFamilyHandle* cf1; + ASSERT_OK(db1->CreateColumnFamily(cf_options, "cf", &cf1)); + ASSERT_EQ(2, cache_->Size()); + ASSERT_TRUE(cache_->Contains(data_path_0_)); + ASSERT_EQ(2, cache_->GetRefCount(data_path_0_)); + ASSERT_TRUE(cache_->Contains(cf_path_0_)); + ASSERT_EQ(2, cache_->GetRefCount(cf_path_0_)); + + ASSERT_OK(db0->DestroyColumnFamilyHandle(cf0)); + delete db0; + ASSERT_EQ(2, cache_->Size()); + ASSERT_TRUE(cache_->Contains(data_path_0_)); + ASSERT_EQ(1, cache_->GetRefCount(data_path_0_)); + ASSERT_TRUE(cache_->Contains(cf_path_0_)); + ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_)); + ASSERT_OK(DestroyDB(dbname_ + "/db0", options, {{"cf", cf_options}})); + + ASSERT_OK(db1->DestroyColumnFamilyHandle(cf1)); + delete db1; + ASSERT_EQ(0, cache_->Size()); + ASSERT_OK(DestroyDB(dbname_ + "/db1", options, {{"cf", cf_options}})); +} + +} // namespace ROCKSDB_NAMESPACE +#endif // OS_LINUX + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_memtable_test.cc b/librocksdb-sys/rocksdb/db/db_memtable_test.cc new file mode 100644 index 0000000..cae592d --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_memtable_test.cc @@ -0,0 +1,344 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include +#include + +#include "db/db_test_util.h" +#include "db/memtable.h" +#include "db/range_del_aggregator.h" +#include "port/stack_trace.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/slice_transform.h" + +namespace ROCKSDB_NAMESPACE { + +class DBMemTableTest : public DBTestBase { + public: + DBMemTableTest() : DBTestBase("db_memtable_test", /*env_do_fsync=*/true) {} +}; + +class MockMemTableRep : public MemTableRep { + public: + explicit MockMemTableRep(Allocator* allocator, MemTableRep* rep) + : MemTableRep(allocator), rep_(rep), num_insert_with_hint_(0) {} + + KeyHandle Allocate(const size_t len, char** buf) override { + return rep_->Allocate(len, buf); + } + + void Insert(KeyHandle handle) override { rep_->Insert(handle); } + + void InsertWithHint(KeyHandle handle, void** hint) override { + num_insert_with_hint_++; + EXPECT_NE(nullptr, hint); + last_hint_in_ = *hint; + rep_->InsertWithHint(handle, hint); + last_hint_out_ = *hint; + } + + bool Contains(const char* key) const override { return rep_->Contains(key); } + + void Get(const LookupKey& k, void* callback_args, + bool (*callback_func)(void* arg, const char* entry)) override { + rep_->Get(k, callback_args, callback_func); + } + + size_t ApproximateMemoryUsage() override { + return rep_->ApproximateMemoryUsage(); + } + + Iterator* GetIterator(Arena* arena) override { + return rep_->GetIterator(arena); + } + + void* last_hint_in() { return last_hint_in_; } + void* last_hint_out() { return last_hint_out_; } + int num_insert_with_hint() { return num_insert_with_hint_; } + + private: + std::unique_ptr rep_; + void* last_hint_in_; + void* last_hint_out_; + int num_insert_with_hint_; +}; + +class MockMemTableRepFactory : public MemTableRepFactory { + public: + MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator& cmp, + Allocator* allocator, + const SliceTransform* transform, + Logger* logger) override { + SkipListFactory factory; + MemTableRep* skiplist_rep = + factory.CreateMemTableRep(cmp, allocator, transform, logger); + mock_rep_ = new MockMemTableRep(allocator, skiplist_rep); + return mock_rep_; + } + + MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator& cmp, + Allocator* allocator, + const SliceTransform* transform, + Logger* logger, + uint32_t column_family_id) override { + last_column_family_id_ = column_family_id; + return CreateMemTableRep(cmp, allocator, transform, logger); + } + + const char* Name() const override { return "MockMemTableRepFactory"; } + + MockMemTableRep* rep() { return mock_rep_; } + + bool IsInsertConcurrentlySupported() const override { return false; } + + uint32_t GetLastColumnFamilyId() { return last_column_family_id_; } + + private: + MockMemTableRep* mock_rep_; + // workaround since there's no std::numeric_limits::max() yet. + uint32_t last_column_family_id_ = static_cast(-1); +}; + +class TestPrefixExtractor : public SliceTransform { + public: + const char* Name() const override { return "TestPrefixExtractor"; } + + Slice Transform(const Slice& key) const override { + const char* p = separator(key); + if (p == nullptr) { + return Slice(); + } + return Slice(key.data(), p - key.data() + 1); + } + + bool InDomain(const Slice& key) const override { + return separator(key) != nullptr; + } + + bool InRange(const Slice& /*key*/) const override { return false; } + + private: + const char* separator(const Slice& key) const { + return reinterpret_cast(memchr(key.data(), '_', key.size())); + } +}; + +// Test that ::Add properly returns false when inserting duplicate keys +TEST_F(DBMemTableTest, DuplicateSeq) { + SequenceNumber seq = 123; + std::string value; + MergeContext merge_context; + Options options; + InternalKeyComparator ikey_cmp(options.comparator); + ReadRangeDelAggregator range_del_agg(&ikey_cmp, + kMaxSequenceNumber /* upper_bound */); + + // Create a MemTable + InternalKeyComparator cmp(BytewiseComparator()); + auto factory = std::make_shared(); + options.memtable_factory = factory; + ImmutableOptions ioptions(options); + WriteBufferManager wb(options.db_write_buffer_size); + MemTable* mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb, + kMaxSequenceNumber, 0 /* column_family_id */); + + // Write some keys and make sure it returns false on duplicates + ASSERT_OK( + mem->Add(seq, kTypeValue, "key", "value2", nullptr /* kv_prot_info */)); + ASSERT_TRUE( + mem->Add(seq, kTypeValue, "key", "value2", nullptr /* kv_prot_info */) + .IsTryAgain()); + // Changing the type should still cause the duplicatae key + ASSERT_TRUE( + mem->Add(seq, kTypeMerge, "key", "value2", nullptr /* kv_prot_info */) + .IsTryAgain()); + // Changing the seq number will make the key fresh + ASSERT_OK(mem->Add(seq + 1, kTypeMerge, "key", "value2", + nullptr /* kv_prot_info */)); + // Test with different types for duplicate keys + ASSERT_TRUE( + mem->Add(seq, kTypeDeletion, "key", "", nullptr /* kv_prot_info */) + .IsTryAgain()); + ASSERT_TRUE( + mem->Add(seq, kTypeSingleDeletion, "key", "", nullptr /* kv_prot_info */) + .IsTryAgain()); + + // Test the duplicate keys under stress + for (int i = 0; i < 10000; i++) { + bool insert_dup = i % 10 == 1; + if (!insert_dup) { + seq++; + } + Status s = mem->Add(seq, kTypeValue, "foo", "value" + std::to_string(seq), + nullptr /* kv_prot_info */); + if (insert_dup) { + ASSERT_TRUE(s.IsTryAgain()); + } else { + ASSERT_OK(s); + } + } + delete mem; + + // Test with InsertWithHint + options.memtable_insert_with_hint_prefix_extractor.reset( + new TestPrefixExtractor()); // which uses _ to extract the prefix + ioptions = ImmutableOptions(options); + mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb, + kMaxSequenceNumber, 0 /* column_family_id */); + // Insert a duplicate key with _ in it + ASSERT_OK( + mem->Add(seq, kTypeValue, "key_1", "value", nullptr /* kv_prot_info */)); + ASSERT_TRUE( + mem->Add(seq, kTypeValue, "key_1", "value", nullptr /* kv_prot_info */) + .IsTryAgain()); + delete mem; + + // Test when InsertConcurrently will be invoked + options.allow_concurrent_memtable_write = true; + ioptions = ImmutableOptions(options); + mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb, + kMaxSequenceNumber, 0 /* column_family_id */); + MemTablePostProcessInfo post_process_info; + ASSERT_OK(mem->Add(seq, kTypeValue, "key", "value", + nullptr /* kv_prot_info */, true, &post_process_info)); + ASSERT_TRUE(mem->Add(seq, kTypeValue, "key", "value", + nullptr /* kv_prot_info */, true, &post_process_info) + .IsTryAgain()); + delete mem; +} + +// A simple test to verify that the concurrent merge writes is functional +TEST_F(DBMemTableTest, ConcurrentMergeWrite) { + int num_ops = 1000; + std::string value; + MergeContext merge_context; + Options options; + // A merge operator that is not sensitive to concurrent writes since in this + // test we don't order the writes. + options.merge_operator = MergeOperators::CreateUInt64AddOperator(); + + // Create a MemTable + InternalKeyComparator cmp(BytewiseComparator()); + auto factory = std::make_shared(); + options.memtable_factory = factory; + options.allow_concurrent_memtable_write = true; + ImmutableOptions ioptions(options); + WriteBufferManager wb(options.db_write_buffer_size); + MemTable* mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb, + kMaxSequenceNumber, 0 /* column_family_id */); + + // Put 0 as the base + PutFixed64(&value, static_cast(0)); + ASSERT_OK(mem->Add(0, kTypeValue, "key", value, nullptr /* kv_prot_info */)); + value.clear(); + + // Write Merge concurrently + ROCKSDB_NAMESPACE::port::Thread write_thread1([&]() { + MemTablePostProcessInfo post_process_info1; + std::string v1; + for (int seq = 1; seq < num_ops / 2; seq++) { + PutFixed64(&v1, seq); + ASSERT_OK(mem->Add(seq, kTypeMerge, "key", v1, nullptr /* kv_prot_info */, + true, &post_process_info1)); + v1.clear(); + } + }); + ROCKSDB_NAMESPACE::port::Thread write_thread2([&]() { + MemTablePostProcessInfo post_process_info2; + std::string v2; + for (int seq = num_ops / 2; seq < num_ops; seq++) { + PutFixed64(&v2, seq); + ASSERT_OK(mem->Add(seq, kTypeMerge, "key", v2, nullptr /* kv_prot_info */, + true, &post_process_info2)); + v2.clear(); + } + }); + write_thread1.join(); + write_thread2.join(); + + Status status; + ReadOptions roptions; + SequenceNumber max_covering_tombstone_seq = 0; + LookupKey lkey("key", kMaxSequenceNumber); + bool res = mem->Get(lkey, &value, /*columns=*/nullptr, /*timestamp=*/nullptr, + &status, &merge_context, &max_covering_tombstone_seq, + roptions, false /* immutable_memtable */); + ASSERT_OK(status); + ASSERT_TRUE(res); + uint64_t ivalue = DecodeFixed64(Slice(value).data()); + uint64_t sum = 0; + for (int seq = 0; seq < num_ops; seq++) { + sum += seq; + } + ASSERT_EQ(ivalue, sum); + + delete mem; +} + +TEST_F(DBMemTableTest, InsertWithHint) { + Options options; + options.allow_concurrent_memtable_write = false; + options.create_if_missing = true; + options.memtable_factory.reset(new MockMemTableRepFactory()); + options.memtable_insert_with_hint_prefix_extractor.reset( + new TestPrefixExtractor()); + options.env = env_; + Reopen(options); + MockMemTableRep* rep = + reinterpret_cast(options.memtable_factory.get()) + ->rep(); + ASSERT_OK(Put("foo_k1", "foo_v1")); + ASSERT_EQ(nullptr, rep->last_hint_in()); + void* hint_foo = rep->last_hint_out(); + ASSERT_OK(Put("foo_k2", "foo_v2")); + ASSERT_EQ(hint_foo, rep->last_hint_in()); + ASSERT_EQ(hint_foo, rep->last_hint_out()); + ASSERT_OK(Put("foo_k3", "foo_v3")); + ASSERT_EQ(hint_foo, rep->last_hint_in()); + ASSERT_EQ(hint_foo, rep->last_hint_out()); + ASSERT_OK(Put("bar_k1", "bar_v1")); + ASSERT_EQ(nullptr, rep->last_hint_in()); + void* hint_bar = rep->last_hint_out(); + ASSERT_NE(hint_foo, hint_bar); + ASSERT_OK(Put("bar_k2", "bar_v2")); + ASSERT_EQ(hint_bar, rep->last_hint_in()); + ASSERT_EQ(hint_bar, rep->last_hint_out()); + ASSERT_EQ(5, rep->num_insert_with_hint()); + ASSERT_OK(Put("NotInPrefixDomain", "vvv")); + ASSERT_EQ(5, rep->num_insert_with_hint()); + ASSERT_EQ("foo_v1", Get("foo_k1")); + ASSERT_EQ("foo_v2", Get("foo_k2")); + ASSERT_EQ("foo_v3", Get("foo_k3")); + ASSERT_EQ("bar_v1", Get("bar_k1")); + ASSERT_EQ("bar_v2", Get("bar_k2")); + ASSERT_EQ("vvv", Get("NotInPrefixDomain")); +} + +TEST_F(DBMemTableTest, ColumnFamilyId) { + // Verifies MemTableRepFactory is told the right column family id. + Options options; + options.env = CurrentOptions().env; + options.allow_concurrent_memtable_write = false; + options.create_if_missing = true; + options.memtable_factory.reset(new MockMemTableRepFactory()); + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + for (uint32_t cf = 0; cf < 2; ++cf) { + ASSERT_OK(Put(cf, "key", "val")); + ASSERT_OK(Flush(cf)); + ASSERT_EQ( + cf, static_cast(options.memtable_factory.get()) + ->GetLastColumnFamilyId()); + } +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_merge_operand_test.cc b/librocksdb-sys/rocksdb/db/db_merge_operand_test.cc new file mode 100644 index 0000000..b6b9ff2 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_merge_operand_test.cc @@ -0,0 +1,481 @@ +// Copyright (c) 2018-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/db_test_util.h" +#include "port/stack_trace.h" +#include "rocksdb/perf_context.h" +#include "rocksdb/utilities/debug.h" +#include "table/block_based/block_builder.h" +#include "test_util/sync_point.h" +#include "rocksdb/merge_operator.h" +#include "utilities/fault_injection_env.h" +#include "utilities/merge_operators.h" +#include "utilities/merge_operators/sortlist.h" +#include "utilities/merge_operators/string_append/stringappend2.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { +class LimitedStringAppendMergeOp : public StringAppendTESTOperator { + public: + LimitedStringAppendMergeOp(int limit, char delim) + : StringAppendTESTOperator(delim), limit_(limit) {} + + const char* Name() const override { + return "DBMergeOperatorTest::LimitedStringAppendMergeOp"; + } + + bool ShouldMerge(const std::vector& operands) const override { + if (operands.size() > 0 && limit_ > 0 && operands.size() >= limit_) { + return true; + } + return false; + } + + private: + size_t limit_ = 0; +}; +} // anonymous namespace + +class DBMergeOperandTest : public DBTestBase { + public: + DBMergeOperandTest() + : DBTestBase("db_merge_operand_test", /*env_do_fsync=*/true) {} +}; + +TEST_F(DBMergeOperandTest, CacheEvictedMergeOperandReadAfterFreeBug) { + // There was a bug of reading merge operands after they are mistakely freed + // in DB::GetMergeOperands, which is surfaced by cache full. + // See PR#9507 for more. + Options options = CurrentOptions(); + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + BlockBasedTableOptions table_options; + + // Small cache to simulate cache full + table_options.block_cache = NewLRUCache(1); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + Reopen(options); + int num_records = 4; + int number_of_operands = 0; + std::vector values(num_records); + GetMergeOperandsOptions merge_operands_info; + merge_operands_info.expected_max_number_of_operands = num_records; + + ASSERT_OK(Merge("k1", "v1")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("k1", "v2")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("k1", "v3")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("k1", "v4")); + + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + "k1", values.data(), &merge_operands_info, + &number_of_operands)); + ASSERT_EQ(number_of_operands, 4); + ASSERT_EQ(values[0].ToString(), "v1"); + ASSERT_EQ(values[1].ToString(), "v2"); + ASSERT_EQ(values[2].ToString(), "v3"); + ASSERT_EQ(values[3].ToString(), "v4"); +} + +TEST_F(DBMergeOperandTest, FlushedMergeOperandReadAfterFreeBug) { + // Repro for a bug where a memtable containing a merge operand could be + // deleted before the merge operand was saved to the result. + auto options = CurrentOptions(); + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + Reopen(options); + + ASSERT_OK(Merge("key", "value")); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::GetImpl:PostMemTableGet:0", + "DBMergeOperandTest::FlushedMergeOperandReadAfterFreeBug:PreFlush"}, + {"DBMergeOperandTest::FlushedMergeOperandReadAfterFreeBug:PostFlush", + "DBImpl::GetImpl:PostMemTableGet:1"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + auto flush_thread = port::Thread([&]() { + TEST_SYNC_POINT( + "DBMergeOperandTest::FlushedMergeOperandReadAfterFreeBug:PreFlush"); + ASSERT_OK(Flush()); + TEST_SYNC_POINT( + "DBMergeOperandTest::FlushedMergeOperandReadAfterFreeBug:PostFlush"); + }); + + PinnableSlice value; + GetMergeOperandsOptions merge_operands_info; + merge_operands_info.expected_max_number_of_operands = 1; + int number_of_operands; + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + "key", &value, &merge_operands_info, + &number_of_operands)); + ASSERT_EQ(1, number_of_operands); + + flush_thread.join(); +} + +TEST_F(DBMergeOperandTest, GetMergeOperandsBasic) { + Options options = CurrentOptions(); + // Use only the latest two merge operands. + options.merge_operator = std::make_shared(2, ','); + Reopen(options); + int num_records = 4; + int number_of_operands = 0; + std::vector values(num_records); + GetMergeOperandsOptions merge_operands_info; + merge_operands_info.expected_max_number_of_operands = num_records; + + // k0 value in memtable + ASSERT_OK(Put("k0", "PutARock")); + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + "k0", values.data(), &merge_operands_info, + &number_of_operands)); + ASSERT_EQ(values[0], "PutARock"); + + // k0.1 value in SST + ASSERT_OK(Put("k0.1", "RockInSST")); + ASSERT_OK(Flush()); + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + "k0.1", values.data(), &merge_operands_info, + &number_of_operands)); + ASSERT_EQ(values[0], "RockInSST"); + + // All k1 values are in memtable. + ASSERT_OK(Merge("k1", "a")); + ASSERT_OK(Put("k1", "x")); + ASSERT_OK(Merge("k1", "b")); + ASSERT_OK(Merge("k1", "c")); + ASSERT_OK(Merge("k1", "d")); + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + "k1", values.data(), &merge_operands_info, + &number_of_operands)); + ASSERT_EQ(values[0], "x"); + ASSERT_EQ(values[1], "b"); + ASSERT_EQ(values[2], "c"); + ASSERT_EQ(values[3], "d"); + + // expected_max_number_of_operands is less than number of merge operands so + // status should be Incomplete. + merge_operands_info.expected_max_number_of_operands = num_records - 1; + Status status = db_->GetMergeOperands( + ReadOptions(), db_->DefaultColumnFamily(), "k1", values.data(), + &merge_operands_info, &number_of_operands); + ASSERT_EQ(status.IsIncomplete(), true); + merge_operands_info.expected_max_number_of_operands = num_records; + + // All k1.1 values are in memtable. + ASSERT_OK(Merge("k1.1", "r")); + ASSERT_OK(Delete("k1.1")); + ASSERT_OK(Merge("k1.1", "c")); + ASSERT_OK(Merge("k1.1", "k")); + ASSERT_OK(Merge("k1.1", "s")); + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + "k1.1", values.data(), &merge_operands_info, + &number_of_operands)); + ASSERT_EQ(values[0], "c"); + ASSERT_EQ(values[1], "k"); + ASSERT_EQ(values[2], "s"); + + // All k2 values are flushed to L0 into a single file. + ASSERT_OK(Merge("k2", "q")); + ASSERT_OK(Merge("k2", "w")); + ASSERT_OK(Merge("k2", "e")); + ASSERT_OK(Merge("k2", "r")); + ASSERT_OK(Flush()); + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + "k2", values.data(), &merge_operands_info, + &number_of_operands)); + ASSERT_EQ(values[0], "q"); + ASSERT_EQ(values[1], "w"); + ASSERT_EQ(values[2], "e"); + ASSERT_EQ(values[3], "r"); + + // All k2.1 values are flushed to L0 into a single file. + ASSERT_OK(Merge("k2.1", "m")); + ASSERT_OK(Put("k2.1", "l")); + ASSERT_OK(Merge("k2.1", "n")); + ASSERT_OK(Merge("k2.1", "o")); + ASSERT_OK(Flush()); + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + "k2.1", values.data(), &merge_operands_info, + &number_of_operands)); + ASSERT_EQ(values[0], "l,n,o"); + + // All k2.2 values are flushed to L0 into a single file. + ASSERT_OK(Merge("k2.2", "g")); + ASSERT_OK(Delete("k2.2")); + ASSERT_OK(Merge("k2.2", "o")); + ASSERT_OK(Merge("k2.2", "t")); + ASSERT_OK(Flush()); + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + "k2.2", values.data(), &merge_operands_info, + &number_of_operands)); + ASSERT_EQ(values[0], "o,t"); + + // Do some compaction that will make the following tests more predictable + // Slice start("PutARock"); + // Slice end("t"); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + // All k3 values are flushed and are in different files. + ASSERT_OK(Merge("k3", "ab")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("k3", "bc")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("k3", "cd")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("k3", "de")); + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + "k3", values.data(), &merge_operands_info, + &number_of_operands)); + ASSERT_EQ(values[0], "ab"); + ASSERT_EQ(values[1], "bc"); + ASSERT_EQ(values[2], "cd"); + ASSERT_EQ(values[3], "de"); + + // All k3.1 values are flushed and are in different files. + ASSERT_OK(Merge("k3.1", "ab")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("k3.1", "bc")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("k3.1", "cd")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("k3.1", "de")); + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + "k3.1", values.data(), &merge_operands_info, + &number_of_operands)); + ASSERT_EQ(values[0], "bc"); + ASSERT_EQ(values[1], "cd"); + ASSERT_EQ(values[2], "de"); + + // All k3.2 values are flushed and are in different files. + ASSERT_OK(Merge("k3.2", "ab")); + ASSERT_OK(Flush()); + ASSERT_OK(Delete("k3.2")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("k3.2", "cd")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("k3.2", "de")); + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + "k3.2", values.data(), &merge_operands_info, + &number_of_operands)); + ASSERT_EQ(values[0], "cd"); + ASSERT_EQ(values[1], "de"); + + // All K4 values are in different levels + ASSERT_OK(Merge("k4", "ba")); + ASSERT_OK(Flush()); + MoveFilesToLevel(4); + ASSERT_OK(Merge("k4", "cb")); + ASSERT_OK(Flush()); + MoveFilesToLevel(3); + ASSERT_OK(Merge("k4", "dc")); + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + ASSERT_OK(Merge("k4", "ed")); + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + "k4", values.data(), &merge_operands_info, + &number_of_operands)); + ASSERT_EQ(values[0], "ba"); + ASSERT_EQ(values[1], "cb"); + ASSERT_EQ(values[2], "dc"); + ASSERT_EQ(values[3], "ed"); + + // First 3 k5 values are in SST and next 4 k5 values are in Immutable + // Memtable + ASSERT_OK(Merge("k5", "who")); + ASSERT_OK(Merge("k5", "am")); + ASSERT_OK(Merge("k5", "i")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("k5", "remember")); + ASSERT_OK(Merge("k5", "i")); + ASSERT_OK(Merge("k5", "am")); + ASSERT_OK(Merge("k5", "rocks")); + ASSERT_OK(dbfull()->TEST_SwitchMemtable()); + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + "k5", values.data(), &merge_operands_info, + &number_of_operands)); + ASSERT_EQ(values[0], "remember"); + ASSERT_EQ(values[1], "i"); + ASSERT_EQ(values[2], "am"); +} + +TEST_F(DBMergeOperandTest, BlobDBGetMergeOperandsBasic) { + Options options = CurrentOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + // Use only the latest two merge operands. + options.merge_operator = std::make_shared(2, ','); + Reopen(options); + int num_records = 4; + int number_of_operands = 0; + std::vector values(num_records); + GetMergeOperandsOptions merge_operands_info; + merge_operands_info.expected_max_number_of_operands = num_records; + + // All k1 values are in memtable. + ASSERT_OK(Put("k1", "x")); + ASSERT_OK(Merge("k1", "b")); + ASSERT_OK(Merge("k1", "c")); + ASSERT_OK(Merge("k1", "d")); + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + "k1", values.data(), &merge_operands_info, + &number_of_operands)); + ASSERT_EQ(values[0], "x"); + ASSERT_EQ(values[1], "b"); + ASSERT_EQ(values[2], "c"); + ASSERT_EQ(values[3], "d"); + + // expected_max_number_of_operands is less than number of merge operands so + // status should be Incomplete. + merge_operands_info.expected_max_number_of_operands = num_records - 1; + Status status = db_->GetMergeOperands( + ReadOptions(), db_->DefaultColumnFamily(), "k1", values.data(), + &merge_operands_info, &number_of_operands); + ASSERT_EQ(status.IsIncomplete(), true); + merge_operands_info.expected_max_number_of_operands = num_records; + + // All k2 values are flushed to L0 into a single file. + ASSERT_OK(Put("k2", "q")); + ASSERT_OK(Merge("k2", "w")); + ASSERT_OK(Merge("k2", "e")); + ASSERT_OK(Merge("k2", "r")); + ASSERT_OK(Flush()); + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + "k2", values.data(), &merge_operands_info, + &number_of_operands)); + ASSERT_EQ(values[0], "q,w,e,r"); + + // Do some compaction that will make the following tests more predictable + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + // All k3 values are flushed and are in different files. + ASSERT_OK(Put("k3", "ab")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("k3", "bc")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("k3", "cd")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("k3", "de")); + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + "k3", values.data(), &merge_operands_info, + &number_of_operands)); + ASSERT_EQ(values[0], "ab"); + ASSERT_EQ(values[1], "bc"); + ASSERT_EQ(values[2], "cd"); + ASSERT_EQ(values[3], "de"); + + // All K4 values are in different levels + ASSERT_OK(Put("k4", "ba")); + ASSERT_OK(Flush()); + MoveFilesToLevel(4); + ASSERT_OK(Merge("k4", "cb")); + ASSERT_OK(Flush()); + MoveFilesToLevel(3); + ASSERT_OK(Merge("k4", "dc")); + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + ASSERT_OK(Merge("k4", "ed")); + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + "k4", values.data(), &merge_operands_info, + &number_of_operands)); + ASSERT_EQ(values[0], "ba"); + ASSERT_EQ(values[1], "cb"); + ASSERT_EQ(values[2], "dc"); + ASSERT_EQ(values[3], "ed"); +} + +TEST_F(DBMergeOperandTest, GetMergeOperandsLargeResultOptimization) { + // These constants are chosen to trigger the large result optimization + // (pinning a bundle of `DBImpl` resources). + const int kNumOperands = 1024; + const int kOperandLen = 1024; + + Options options = CurrentOptions(); + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + DestroyAndReopen(options); + + Random rnd(301); + std::vector expected_merge_operands; + expected_merge_operands.reserve(kNumOperands); + for (int i = 0; i < kNumOperands; ++i) { + expected_merge_operands.emplace_back(rnd.RandomString(kOperandLen)); + ASSERT_OK(Merge("key", expected_merge_operands.back())); + } + + std::vector merge_operands(kNumOperands); + GetMergeOperandsOptions merge_operands_info; + merge_operands_info.expected_max_number_of_operands = kNumOperands; + int num_merge_operands = 0; + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + "key", merge_operands.data(), + &merge_operands_info, &num_merge_operands)); + ASSERT_EQ(num_merge_operands, kNumOperands); + + // Ensures the large result optimization was used. + for (int i = 0; i < kNumOperands; ++i) { + ASSERT_TRUE(merge_operands[i].IsPinned()); + } + + // Add a Flush() to change the `SuperVersion` to challenge the resource + // pinning. + ASSERT_OK(Flush()); + + for (int i = 0; i < kNumOperands; ++i) { + ASSERT_EQ(expected_merge_operands[i], merge_operands[i]); + } +} + +TEST_F(DBMergeOperandTest, GetMergeOperandsBaseDeletionInImmMem) { + // In this test, "k1" has a MERGE in a mutable memtable on top of a base + // DELETE in an immutable memtable. + Options opts = CurrentOptions(); + opts.max_write_buffer_number = 10; + opts.min_write_buffer_number_to_merge = 10; + opts.merge_operator = MergeOperators::CreateDeprecatedPutOperator(); + Reopen(opts); + + ASSERT_OK(Put("k1", "val")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("k0", "val")); + ASSERT_OK(Delete("k1")); + ASSERT_OK(Put("k2", "val")); + ASSERT_OK(dbfull()->TEST_SwitchMemtable()); + ASSERT_OK(Merge("k1", "val")); + + { + std::vector values(2); + + GetMergeOperandsOptions merge_operands_info; + merge_operands_info.expected_max_number_of_operands = + static_cast(values.size()); + + std::string key = "k1", from_db; + int number_of_operands = 0; + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + key, values.data(), &merge_operands_info, + &number_of_operands)); + ASSERT_EQ(1, number_of_operands); + from_db = values[0].ToString(); + ASSERT_EQ("val", from_db); + } + + { + std::string val; + ASSERT_OK(db_->Get(ReadOptions(), "k1", &val)); + ASSERT_EQ("val", val); + } +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_merge_operator_test.cc b/librocksdb-sys/rocksdb/db/db_merge_operator_test.cc new file mode 100644 index 0000000..aa1253a --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_merge_operator_test.cc @@ -0,0 +1,866 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#include +#include + +#include "db/db_test_util.h" +#include "db/forward_iterator.h" +#include "port/stack_trace.h" +#include "rocksdb/merge_operator.h" +#include "util/random.h" +#include "utilities/merge_operators.h" +#include "utilities/merge_operators/string_append/stringappend2.h" + +namespace ROCKSDB_NAMESPACE { + +class TestReadCallback : public ReadCallback { + public: + TestReadCallback(SnapshotChecker* snapshot_checker, + SequenceNumber snapshot_seq) + : ReadCallback(snapshot_seq), + snapshot_checker_(snapshot_checker), + snapshot_seq_(snapshot_seq) {} + + bool IsVisibleFullCheck(SequenceNumber seq) override { + return snapshot_checker_->CheckInSnapshot(seq, snapshot_seq_) == + SnapshotCheckerResult::kInSnapshot; + } + + private: + SnapshotChecker* snapshot_checker_; + SequenceNumber snapshot_seq_; +}; + +// Test merge operator functionality. +class DBMergeOperatorTest : public DBTestBase { + public: + DBMergeOperatorTest() + : DBTestBase("db_merge_operator_test", /*env_do_fsync=*/false) {} + + std::string GetWithReadCallback(SnapshotChecker* snapshot_checker, + const Slice& key, + const Snapshot* snapshot = nullptr) { + SequenceNumber seq = snapshot == nullptr ? db_->GetLatestSequenceNumber() + : snapshot->GetSequenceNumber(); + TestReadCallback read_callback(snapshot_checker, seq); + ReadOptions read_opt; + read_opt.snapshot = snapshot; + PinnableSlice value; + DBImpl::GetImplOptions get_impl_options; + get_impl_options.column_family = db_->DefaultColumnFamily(); + get_impl_options.value = &value; + get_impl_options.callback = &read_callback; + Status s = dbfull()->GetImpl(read_opt, key, get_impl_options); + if (!s.ok()) { + return s.ToString(); + } + return value.ToString(); + } +}; + +TEST_F(DBMergeOperatorTest, LimitMergeOperands) { + class LimitedStringAppendMergeOp : public StringAppendTESTOperator { + public: + LimitedStringAppendMergeOp(int limit, char delim) + : StringAppendTESTOperator(delim), limit_(limit) {} + + const char* Name() const override { + return "DBMergeOperatorTest::LimitedStringAppendMergeOp"; + } + + bool ShouldMerge(const std::vector& operands) const override { + if (operands.size() > 0 && limit_ > 0 && operands.size() >= limit_) { + return true; + } + return false; + } + + private: + size_t limit_ = 0; + }; + + Options options = CurrentOptions(); + options.create_if_missing = true; + // Use only the latest two merge operands. + options.merge_operator = std::make_shared(2, ','); + options.env = env_; + Reopen(options); + // All K1 values are in memtable. + ASSERT_OK(Merge("k1", "a")); + ASSERT_OK(Merge("k1", "b")); + ASSERT_OK(Merge("k1", "c")); + ASSERT_OK(Merge("k1", "d")); + std::string value; + ASSERT_OK(db_->Get(ReadOptions(), "k1", &value)); + // Make sure that only the latest two merge operands are used. If this was + // not the case the value would be "a,b,c,d". + ASSERT_EQ(value, "c,d"); + + // All K2 values are flushed to L0 into a single file. + ASSERT_OK(Merge("k2", "a")); + ASSERT_OK(Merge("k2", "b")); + ASSERT_OK(Merge("k2", "c")); + ASSERT_OK(Merge("k2", "d")); + ASSERT_OK(Flush()); + ASSERT_OK(db_->Get(ReadOptions(), "k2", &value)); + ASSERT_EQ(value, "c,d"); + + // All K3 values are flushed and are in different files. + ASSERT_OK(Merge("k3", "ab")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("k3", "bc")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("k3", "cd")); + ASSERT_OK(Flush()); + ASSERT_OK(Merge("k3", "de")); + ASSERT_OK(db_->Get(ReadOptions(), "k3", &value)); + ASSERT_EQ(value, "cd,de"); + + // All K4 values are in different levels + ASSERT_OK(Merge("k4", "ab")); + ASSERT_OK(Flush()); + MoveFilesToLevel(4); + ASSERT_OK(Merge("k4", "bc")); + ASSERT_OK(Flush()); + MoveFilesToLevel(3); + ASSERT_OK(Merge("k4", "cd")); + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + ASSERT_OK(Merge("k4", "de")); + ASSERT_OK(db_->Get(ReadOptions(), "k4", &value)); + ASSERT_EQ(value, "cd,de"); +} + +TEST_F(DBMergeOperatorTest, MergeErrorOnRead) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.merge_operator.reset(new TestPutOperator()); + options.env = env_; + Reopen(options); + ASSERT_OK(Merge("k1", "v1")); + ASSERT_OK(Merge("k1", "corrupted")); + std::string value; + ASSERT_TRUE(db_->Get(ReadOptions(), "k1", &value).IsCorruption()); + VerifyDBInternal({{"k1", "corrupted"}, {"k1", "v1"}}); +} + +TEST_F(DBMergeOperatorTest, MergeErrorOnWrite) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.merge_operator.reset(new TestPutOperator()); + options.max_successive_merges = 3; + options.env = env_; + Reopen(options); + ASSERT_OK(Merge("k1", "v1")); + ASSERT_OK(Merge("k1", "v2")); + // Will trigger a merge when hitting max_successive_merges and the merge + // will fail. The delta will be inserted nevertheless. + ASSERT_OK(Merge("k1", "corrupted")); + // Data should stay unmerged after the error. + VerifyDBInternal({{"k1", "corrupted"}, {"k1", "v2"}, {"k1", "v1"}}); +} + +TEST_F(DBMergeOperatorTest, MergeErrorOnIteration) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.merge_operator.reset(new TestPutOperator()); + options.env = env_; + + DestroyAndReopen(options); + ASSERT_OK(Merge("k1", "v1")); + ASSERT_OK(Merge("k1", "corrupted")); + ASSERT_OK(Put("k2", "v2")); + auto* iter = db_->NewIterator(ReadOptions()); + iter->Seek("k1"); + ASSERT_FALSE(iter->Valid()); + ASSERT_TRUE(iter->status().IsCorruption()); + delete iter; + iter = db_->NewIterator(ReadOptions()); + iter->Seek("k2"); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + iter->Prev(); + ASSERT_FALSE(iter->Valid()); + ASSERT_TRUE(iter->status().IsCorruption()); + delete iter; + VerifyDBInternal({{"k1", "corrupted"}, {"k1", "v1"}, {"k2", "v2"}}); + + DestroyAndReopen(options); + ASSERT_OK(Merge("k1", "v1")); + ASSERT_OK(Put("k2", "v2")); + ASSERT_OK(Merge("k2", "corrupted")); + iter = db_->NewIterator(ReadOptions()); + iter->Seek("k1"); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + ASSERT_TRUE(iter->status().IsCorruption()); + delete iter; + VerifyDBInternal({{"k1", "v1"}, {"k2", "corrupted"}, {"k2", "v2"}}); +} + + +TEST_F(DBMergeOperatorTest, MergeOperatorFailsWithMustMerge) { + // This is like a mini-stress test dedicated to `OpFailureScope::kMustMerge`. + // Some or most of it might be deleted upon adding that option to the actual + // stress test. + // + // "k0" and "k2" are stable (uncorrupted) keys before and after a corrupted + // key ("k1"). The outer loop (`i`) varies which write (`j`) to "k1" triggers + // the corruption. Inside that loop there are three cases: + // + // - Case 1: pure `Merge()`s + // - Case 2: `Merge()`s on top of a `Put()` + // - Case 3: `Merge()`s on top of a `Delete()` + // + // For each case we test query results before flush, after flush, and after + // compaction, as well as cleanup after deletion+compaction. The queries + // expect "k0" and "k2" to always be readable. "k1" is expected to be readable + // only by APIs that do not require merging, such as `GetMergeOperands()`. + const int kNumOperands = 3; + Options options = CurrentOptions(); + options.merge_operator.reset(new TestPutOperator()); + options.env = env_; + Reopen(options); + + for (int i = 0; i < kNumOperands; ++i) { + auto check_query = [&]() { + { + std::string value; + ASSERT_OK(db_->Get(ReadOptions(), "k0", &value)); + Status s = db_->Get(ReadOptions(), "k1", &value); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_EQ(Status::SubCode::kMergeOperatorFailed, s.subcode()); + ASSERT_OK(db_->Get(ReadOptions(), "k2", &value)); + } + + { + std::unique_ptr iter; + iter.reset(db_->NewIterator(ReadOptions())); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("k0", iter->key()); + iter->Next(); + ASSERT_TRUE(iter->status().IsCorruption()); + ASSERT_EQ(Status::SubCode::kMergeOperatorFailed, + iter->status().subcode()); + + iter->SeekToLast(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("k2", iter->key()); + iter->Prev(); + ASSERT_TRUE(iter->status().IsCorruption()); + + iter->Seek("k2"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("k2", iter->key()); + } + + std::vector values(kNumOperands); + GetMergeOperandsOptions merge_operands_info; + merge_operands_info.expected_max_number_of_operands = kNumOperands; + int num_operands_found = 0; + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + "k1", values.data(), &merge_operands_info, + &num_operands_found)); + ASSERT_EQ(kNumOperands, num_operands_found); + for (int j = 0; j < num_operands_found; ++j) { + if (i == j) { + ASSERT_EQ(values[j], "corrupted_must_merge"); + } else { + ASSERT_EQ(values[j], "ok"); + } + } + }; + + ASSERT_OK(Put("k0", "val")); + ASSERT_OK(Put("k2", "val")); + + // Case 1 + for (int j = 0; j < kNumOperands; ++j) { + if (j == i) { + ASSERT_OK(Merge("k1", "corrupted_must_merge")); + } else { + ASSERT_OK(Merge("k1", "ok")); + } + } + check_query(); + ASSERT_OK(Flush()); + check_query(); + { + CompactRangeOptions cro; + cro.bottommost_level_compaction = + BottommostLevelCompaction::kForceOptimized; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + } + check_query(); + + // Case 2 + for (int j = 0; j < kNumOperands; ++j) { + Slice val; + if (j == i) { + val = "corrupted_must_merge"; + } else { + val = "ok"; + } + if (j == 0) { + ASSERT_OK(Put("k1", val)); + } else { + ASSERT_OK(Merge("k1", val)); + } + } + check_query(); + ASSERT_OK(Flush()); + check_query(); + { + CompactRangeOptions cro; + cro.bottommost_level_compaction = + BottommostLevelCompaction::kForceOptimized; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + } + check_query(); + + // Case 3 + ASSERT_OK(Delete("k1")); + for (int j = 0; j < kNumOperands; ++j) { + if (i == j) { + ASSERT_OK(Merge("k1", "corrupted_must_merge")); + } else { + ASSERT_OK(Merge("k1", "ok")); + } + } + check_query(); + ASSERT_OK(Flush()); + check_query(); + { + CompactRangeOptions cro; + cro.bottommost_level_compaction = + BottommostLevelCompaction::kForceOptimized; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + } + check_query(); + + // Verify obsolete data removal still happens + ASSERT_OK(Delete("k0")); + ASSERT_OK(Delete("k1")); + ASSERT_OK(Delete("k2")); + ASSERT_EQ("NOT_FOUND", Get("k0")); + ASSERT_EQ("NOT_FOUND", Get("k1")); + ASSERT_EQ("NOT_FOUND", Get("k2")); + CompactRangeOptions cro; + cro.bottommost_level_compaction = + BottommostLevelCompaction::kForceOptimized; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("", FilesPerLevel()); + } +} + +TEST_F(DBMergeOperatorTest, DataBlockBinaryAndHash) { + // Basic test to check that merge operator works with data block index type + // DataBlockBinaryAndHash. + Options options = CurrentOptions(); + options.create_if_missing = true; + options.merge_operator.reset(new TestPutOperator()); + options.env = env_; + BlockBasedTableOptions table_options; + table_options.block_restart_interval = 16; + table_options.data_block_index_type = + BlockBasedTableOptions::DataBlockIndexType::kDataBlockBinaryAndHash; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DestroyAndReopen(options); + + const int kNumKeys = 100; + for (int i = 0; i < kNumKeys; ++i) { + ASSERT_OK(db_->Merge(WriteOptions(), Key(i), std::to_string(i))); + } + ASSERT_OK(Flush()); + std::string value; + for (int i = 0; i < kNumKeys; ++i) { + ASSERT_OK(db_->Get(ReadOptions(), Key(i), &value)); + ASSERT_EQ(std::to_string(i), value); + } + + std::vector snapshots; + for (int i = 0; i < kNumKeys; ++i) { + ASSERT_OK(db_->Delete(WriteOptions(), Key(i))); + for (int j = 0; j < 3; ++j) { + ASSERT_OK(db_->Merge(WriteOptions(), Key(i), std::to_string(i * 3 + j))); + snapshots.push_back(db_->GetSnapshot()); + } + } + ASSERT_OK(Flush()); + for (int i = 0; i < kNumKeys; ++i) { + ASSERT_OK(db_->Get(ReadOptions(), Key(i), &value)); + ASSERT_EQ(std::to_string(i * 3 + 2), value); + } + for (auto snapshot : snapshots) { + db_->ReleaseSnapshot(snapshot); + } +} + +class MergeOperatorPinningTest : public DBMergeOperatorTest, + public testing::WithParamInterface { + public: + MergeOperatorPinningTest() { disable_block_cache_ = GetParam(); } + + bool disable_block_cache_; +}; + +INSTANTIATE_TEST_CASE_P(MergeOperatorPinningTest, MergeOperatorPinningTest, + ::testing::Bool()); + +TEST_P(MergeOperatorPinningTest, OperandsMultiBlocks) { + Options options = CurrentOptions(); + BlockBasedTableOptions table_options; + table_options.block_size = 1; // every block will contain one entry + table_options.no_block_cache = disable_block_cache_; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.merge_operator = MergeOperators::CreateStringAppendTESTOperator(); + options.level0_slowdown_writes_trigger = (1 << 30); + options.level0_stop_writes_trigger = (1 << 30); + options.disable_auto_compactions = true; + DestroyAndReopen(options); + + const int kKeysPerFile = 10; + const int kOperandsPerKeyPerFile = 7; + const int kOperandSize = 100; + // Filse to write in L0 before compacting to lower level + const int kFilesPerLevel = 3; + + Random rnd(301); + std::map true_data; + int batch_num = 1; + int lvl_to_fill = 4; + int key_id = 0; + while (true) { + for (int j = 0; j < kKeysPerFile; j++) { + std::string key = Key(key_id % 35); + key_id++; + for (int k = 0; k < kOperandsPerKeyPerFile; k++) { + std::string val = rnd.RandomString(kOperandSize); + ASSERT_OK(db_->Merge(WriteOptions(), key, val)); + if (true_data[key].size() == 0) { + true_data[key] = val; + } else { + true_data[key] += "," + val; + } + } + } + + if (lvl_to_fill == -1) { + // Keep last batch in memtable and stop + break; + } + + ASSERT_OK(Flush()); + if (batch_num % kFilesPerLevel == 0) { + if (lvl_to_fill != 0) { + MoveFilesToLevel(lvl_to_fill); + } + lvl_to_fill--; + } + batch_num++; + } + + // 3 L0 files + // 1 L1 file + // 3 L2 files + // 1 L3 file + // 3 L4 Files + ASSERT_EQ(FilesPerLevel(), "3,1,3,1,3"); + + VerifyDBFromMap(true_data); +} + +class MergeOperatorHook : public MergeOperator { + public: + explicit MergeOperatorHook(std::shared_ptr _merge_op) + : merge_op_(_merge_op) {} + + bool FullMergeV2(const MergeOperationInput& merge_in, + MergeOperationOutput* merge_out) const override { + before_merge_(); + bool res = merge_op_->FullMergeV2(merge_in, merge_out); + after_merge_(); + return res; + } + + const char* Name() const override { return merge_op_->Name(); } + + std::shared_ptr merge_op_; + std::function before_merge_ = []() {}; + std::function after_merge_ = []() {}; +}; + +TEST_P(MergeOperatorPinningTest, EvictCacheBeforeMerge) { + Options options = CurrentOptions(); + + auto merge_hook = + std::make_shared(MergeOperators::CreateMaxOperator()); + options.merge_operator = merge_hook; + options.disable_auto_compactions = true; + options.level0_slowdown_writes_trigger = (1 << 30); + options.level0_stop_writes_trigger = (1 << 30); + options.max_open_files = 20; + BlockBasedTableOptions bbto; + bbto.no_block_cache = disable_block_cache_; + if (bbto.no_block_cache == false) { + bbto.block_cache = NewLRUCache(64 * 1024 * 1024); + } else { + bbto.block_cache = nullptr; + } + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + DestroyAndReopen(options); + + const int kNumOperands = 30; + const int kNumKeys = 1000; + const int kOperandSize = 100; + Random rnd(301); + + // 1000 keys every key have 30 operands, every operand is in a different file + std::map true_data; + for (int i = 0; i < kNumOperands; i++) { + for (int j = 0; j < kNumKeys; j++) { + std::string k = Key(j); + std::string v = rnd.RandomString(kOperandSize); + ASSERT_OK(db_->Merge(WriteOptions(), k, v)); + + true_data[k] = std::max(true_data[k], v); + } + ASSERT_OK(Flush()); + } + + std::vector file_numbers = ListTableFiles(env_, dbname_); + ASSERT_EQ(file_numbers.size(), kNumOperands); + int merge_cnt = 0; + + // Code executed before merge operation + merge_hook->before_merge_ = [&]() { + // Evict all tables from cache before every merge operation + auto* table_cache = dbfull()->TEST_table_cache(); + for (uint64_t num : file_numbers) { + TableCache::Evict(table_cache, num); + } + // Decrease cache capacity to force all unrefed blocks to be evicted + if (bbto.block_cache) { + bbto.block_cache->SetCapacity(1); + } + merge_cnt++; + }; + + // Code executed after merge operation + merge_hook->after_merge_ = [&]() { + // Increase capacity again after doing the merge + if (bbto.block_cache) { + bbto.block_cache->SetCapacity(64 * 1024 * 1024); + } + }; + + size_t total_reads; + VerifyDBFromMap(true_data, &total_reads); + ASSERT_EQ(merge_cnt, total_reads); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + VerifyDBFromMap(true_data, &total_reads); +} + +TEST_P(MergeOperatorPinningTest, TailingIterator) { + Options options = CurrentOptions(); + options.merge_operator = MergeOperators::CreateMaxOperator(); + BlockBasedTableOptions bbto; + bbto.no_block_cache = disable_block_cache_; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + DestroyAndReopen(options); + + const int kNumOperands = 100; + const int kNumWrites = 100000; + + std::function writer_func = [&]() { + int k = 0; + for (int i = 0; i < kNumWrites; i++) { + ASSERT_OK(db_->Merge(WriteOptions(), Key(k), Key(k))); + + if (i && i % kNumOperands == 0) { + k++; + } + if (i && i % 127 == 0) { + ASSERT_OK(Flush()); + } + if (i && i % 317 == 0) { + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + } + } + }; + + std::function reader_func = [&]() { + ReadOptions ro; + ro.tailing = true; + Iterator* iter = db_->NewIterator(ro); + ASSERT_OK(iter->status()); + iter->SeekToFirst(); + for (int i = 0; i < (kNumWrites / kNumOperands); i++) { + while (!iter->Valid()) { + // wait for the key to be written + env_->SleepForMicroseconds(100); + iter->Seek(Key(i)); + } + ASSERT_EQ(iter->key(), Key(i)); + ASSERT_EQ(iter->value(), Key(i)); + + iter->Next(); + } + ASSERT_OK(iter->status()); + + delete iter; + }; + + ROCKSDB_NAMESPACE::port::Thread writer_thread(writer_func); + ROCKSDB_NAMESPACE::port::Thread reader_thread(reader_func); + + writer_thread.join(); + reader_thread.join(); +} + +TEST_F(DBMergeOperatorTest, TailingIteratorMemtableUnrefedBySomeoneElse) { + Options options = CurrentOptions(); + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + DestroyAndReopen(options); + + // Overview of the test: + // * There are two merge operands for the same key: one in an sst file, + // another in a memtable. + // * Seek a tailing iterator to this key. + // * As part of the seek, the iterator will: + // (a) first visit the operand in the memtable and tell ForwardIterator + // to pin this operand, then + // (b) move on to the operand in the sst file, then pass both operands + // to merge operator. + // * The memtable may get flushed and unreferenced by another thread between + // (a) and (b). The test simulates it by flushing the memtable inside a + // SyncPoint callback located between (a) and (b). + // * In this case it's ForwardIterator's responsibility to keep the memtable + // pinned until (b) is complete. There used to be a bug causing + // ForwardIterator to not pin it in some circumstances. This test + // reproduces it. + + ASSERT_OK(db_->Merge(WriteOptions(), "key", "sst")); + ASSERT_OK(db_->Flush(FlushOptions())); // Switch to SuperVersion A + ASSERT_OK(db_->Merge(WriteOptions(), "key", "memtable")); + + // Pin SuperVersion A + std::unique_ptr someone_else(db_->NewIterator(ReadOptions())); + ASSERT_OK(someone_else->status()); + + bool pushed_first_operand = false; + bool stepped_to_next_operand = false; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBIter::MergeValuesNewToOld:PushedFirstOperand", [&](void*) { + EXPECT_FALSE(pushed_first_operand); + pushed_first_operand = true; + EXPECT_OK(db_->Flush(FlushOptions())); // Switch to SuperVersion B + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBIter::MergeValuesNewToOld:SteppedToNextOperand", [&](void*) { + EXPECT_FALSE(stepped_to_next_operand); + stepped_to_next_operand = true; + someone_else.reset(); // Unpin SuperVersion A + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ReadOptions ro; + ro.tailing = true; + std::unique_ptr iter(db_->NewIterator(ro)); + iter->Seek("key"); + + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ(std::string("sst,memtable"), iter->value().ToString()); + EXPECT_TRUE(pushed_first_operand); + EXPECT_TRUE(stepped_to_next_operand); +} + +TEST_F(DBMergeOperatorTest, SnapshotCheckerAndReadCallback) { + Options options = CurrentOptions(); + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + DestroyAndReopen(options); + + class TestSnapshotChecker : public SnapshotChecker { + public: + SnapshotCheckerResult CheckInSnapshot( + SequenceNumber seq, SequenceNumber snapshot_seq) const override { + return IsInSnapshot(seq, snapshot_seq) + ? SnapshotCheckerResult::kInSnapshot + : SnapshotCheckerResult::kNotInSnapshot; + } + + bool IsInSnapshot(SequenceNumber seq, SequenceNumber snapshot_seq) const { + switch (snapshot_seq) { + case 0: + return seq == 0; + case 1: + return seq <= 1; + case 2: + // seq = 2 not visible to snapshot with seq = 2 + return seq <= 1; + case 3: + return seq <= 3; + case 4: + // seq = 4 not visible to snpahost with seq = 4 + return seq <= 3; + default: + // seq >=4 is uncommitted + return seq <= 4; + }; + } + }; + TestSnapshotChecker* snapshot_checker = new TestSnapshotChecker(); + dbfull()->SetSnapshotChecker(snapshot_checker); + + std::string value; + ASSERT_OK(Merge("foo", "v1")); + ASSERT_EQ(1, db_->GetLatestSequenceNumber()); + ASSERT_EQ("v1", GetWithReadCallback(snapshot_checker, "foo")); + ASSERT_OK(Merge("foo", "v2")); + ASSERT_EQ(2, db_->GetLatestSequenceNumber()); + // v2 is not visible to latest snapshot, which has seq = 2. + ASSERT_EQ("v1", GetWithReadCallback(snapshot_checker, "foo")); + // Take a snapshot with seq = 2. + const Snapshot* snapshot1 = db_->GetSnapshot(); + ASSERT_EQ(2, snapshot1->GetSequenceNumber()); + // v2 is not visible to snapshot1, which has seq = 2 + ASSERT_EQ("v1", GetWithReadCallback(snapshot_checker, "foo", snapshot1)); + + // Verify flush doesn't alter the result. + ASSERT_OK(Flush()); + ASSERT_EQ("v1", GetWithReadCallback(snapshot_checker, "foo", snapshot1)); + ASSERT_EQ("v1", GetWithReadCallback(snapshot_checker, "foo")); + + ASSERT_OK(Merge("foo", "v3")); + ASSERT_EQ(3, db_->GetLatestSequenceNumber()); + ASSERT_EQ("v1,v2,v3", GetWithReadCallback(snapshot_checker, "foo")); + ASSERT_OK(Merge("foo", "v4")); + ASSERT_EQ(4, db_->GetLatestSequenceNumber()); + // v4 is not visible to latest snapshot, which has seq = 4. + ASSERT_EQ("v1,v2,v3", GetWithReadCallback(snapshot_checker, "foo")); + const Snapshot* snapshot2 = db_->GetSnapshot(); + ASSERT_EQ(4, snapshot2->GetSequenceNumber()); + // v4 is not visible to snapshot2, which has seq = 4. + ASSERT_EQ("v1,v2,v3", + GetWithReadCallback(snapshot_checker, "foo", snapshot2)); + + // Verify flush doesn't alter the result. + ASSERT_OK(Flush()); + ASSERT_EQ("v1", GetWithReadCallback(snapshot_checker, "foo", snapshot1)); + ASSERT_EQ("v1,v2,v3", + GetWithReadCallback(snapshot_checker, "foo", snapshot2)); + ASSERT_EQ("v1,v2,v3", GetWithReadCallback(snapshot_checker, "foo")); + + ASSERT_OK(Merge("foo", "v5")); + ASSERT_EQ(5, db_->GetLatestSequenceNumber()); + // v5 is uncommitted + ASSERT_EQ("v1,v2,v3,v4", GetWithReadCallback(snapshot_checker, "foo")); + + // full manual compaction. + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + // Verify compaction doesn't alter the result. + ASSERT_EQ("v1", GetWithReadCallback(snapshot_checker, "foo", snapshot1)); + ASSERT_EQ("v1,v2,v3", + GetWithReadCallback(snapshot_checker, "foo", snapshot2)); + ASSERT_EQ("v1,v2,v3,v4", GetWithReadCallback(snapshot_checker, "foo")); + + db_->ReleaseSnapshot(snapshot1); + db_->ReleaseSnapshot(snapshot2); +} + +class PerConfigMergeOperatorPinningTest + : public DBMergeOperatorTest, + public testing::WithParamInterface> { + public: + PerConfigMergeOperatorPinningTest() { + std::tie(disable_block_cache_, option_config_) = GetParam(); + } + + bool disable_block_cache_; +}; + +INSTANTIATE_TEST_CASE_P( + MergeOperatorPinningTest, PerConfigMergeOperatorPinningTest, + ::testing::Combine(::testing::Bool(), + ::testing::Range(static_cast(DBTestBase::kDefault), + static_cast(DBTestBase::kEnd)))); + +TEST_P(PerConfigMergeOperatorPinningTest, Randomized) { + if (ShouldSkipOptions(option_config_, kSkipMergePut)) { + return; + } + + Options options = CurrentOptions(); + options.merge_operator = MergeOperators::CreateMaxOperator(); + BlockBasedTableOptions table_options; + table_options.no_block_cache = disable_block_cache_; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DestroyAndReopen(options); + + Random rnd(301); + std::map true_data; + + const int kTotalMerges = 5000; + // Every key gets ~10 operands + const int kKeyRange = kTotalMerges / 10; + const int kOperandSize = 20; + const int kNumPutBefore = kKeyRange / 10; // 10% value + const int kNumPutAfter = kKeyRange / 10; // 10% overwrite + const int kNumDelete = kKeyRange / 10; // 10% delete + + // kNumPutBefore keys will have base values + for (int i = 0; i < kNumPutBefore; i++) { + std::string key = Key(rnd.Next() % kKeyRange); + std::string value = rnd.RandomString(kOperandSize); + ASSERT_OK(db_->Put(WriteOptions(), key, value)); + + true_data[key] = value; + } + + // Do kTotalMerges merges + for (int i = 0; i < kTotalMerges; i++) { + std::string key = Key(rnd.Next() % kKeyRange); + std::string value = rnd.RandomString(kOperandSize); + ASSERT_OK(db_->Merge(WriteOptions(), key, value)); + + if (true_data[key] < value) { + true_data[key] = value; + } + } + + // Overwrite random kNumPutAfter keys + for (int i = 0; i < kNumPutAfter; i++) { + std::string key = Key(rnd.Next() % kKeyRange); + std::string value = rnd.RandomString(kOperandSize); + ASSERT_OK(db_->Put(WriteOptions(), key, value)); + + true_data[key] = value; + } + + // Delete random kNumDelete keys + for (int i = 0; i < kNumDelete; i++) { + std::string key = Key(rnd.Next() % kKeyRange); + ASSERT_OK(db_->Delete(WriteOptions(), key)); + + true_data.erase(key); + } + + VerifyDBFromMap(true_data); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_options_test.cc b/librocksdb-sys/rocksdb/db/db_options_test.cc new file mode 100644 index 0000000..df6e108 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_options_test.cc @@ -0,0 +1,1333 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include +#include +#include + +#include "db/column_family.h" +#include "db/db_impl/db_impl.h" +#include "db/db_test_util.h" +#include "options/options_helper.h" +#include "port/stack_trace.h" +#include "rocksdb/cache.h" +#include "rocksdb/convenience.h" +#include "rocksdb/rate_limiter.h" +#include "rocksdb/stats_history.h" +#include "test_util/sync_point.h" +#include "test_util/testutil.h" +#include "util/random.h" +#include "utilities/fault_injection_fs.h" + +namespace ROCKSDB_NAMESPACE { + +class DBOptionsTest : public DBTestBase { + public: + DBOptionsTest() : DBTestBase("db_options_test", /*env_do_fsync=*/true) {} + + std::unordered_map GetMutableDBOptionsMap( + const DBOptions& options) { + std::string options_str; + std::unordered_map mutable_map; + ConfigOptions config_options(options); + config_options.delimiter = "; "; + + EXPECT_OK(GetStringFromMutableDBOptions( + config_options, MutableDBOptions(options), &options_str)); + EXPECT_OK(StringToMap(options_str, &mutable_map)); + + return mutable_map; + } + + std::unordered_map GetMutableCFOptionsMap( + const ColumnFamilyOptions& options) { + std::string options_str; + ConfigOptions config_options; + config_options.delimiter = "; "; + + std::unordered_map mutable_map; + EXPECT_OK(GetStringFromMutableCFOptions( + config_options, MutableCFOptions(options), &options_str)); + EXPECT_OK(StringToMap(options_str, &mutable_map)); + return mutable_map; + } + + std::unordered_map GetRandomizedMutableCFOptionsMap( + Random* rnd) { + Options options = CurrentOptions(); + options.env = env_; + ImmutableDBOptions db_options(options); + test::RandomInitCFOptions(&options, options, rnd); + auto sanitized_options = SanitizeOptions(db_options, options); + auto opt_map = GetMutableCFOptionsMap(sanitized_options); + delete options.compaction_filter; + return opt_map; + } + + std::unordered_map GetRandomizedMutableDBOptionsMap( + Random* rnd) { + DBOptions db_options; + test::RandomInitDBOptions(&db_options, rnd); + auto sanitized_options = SanitizeOptions(dbname_, db_options); + return GetMutableDBOptionsMap(sanitized_options); + } +}; + +TEST_F(DBOptionsTest, ImmutableTrackAndVerifyWalsInManifest) { + Options options; + options.env = env_; + options.track_and_verify_wals_in_manifest = true; + + ImmutableDBOptions db_options(options); + ASSERT_TRUE(db_options.track_and_verify_wals_in_manifest); + + Reopen(options); + ASSERT_TRUE(dbfull()->GetDBOptions().track_and_verify_wals_in_manifest); + + Status s = + dbfull()->SetDBOptions({{"track_and_verify_wals_in_manifest", "false"}}); + ASSERT_FALSE(s.ok()); +} + +TEST_F(DBOptionsTest, ImmutableVerifySstUniqueIdInManifest) { + Options options; + options.env = env_; + options.verify_sst_unique_id_in_manifest = true; + + ImmutableDBOptions db_options(options); + ASSERT_TRUE(db_options.verify_sst_unique_id_in_manifest); + + Reopen(options); + ASSERT_TRUE(dbfull()->GetDBOptions().verify_sst_unique_id_in_manifest); + + Status s = + dbfull()->SetDBOptions({{"verify_sst_unique_id_in_manifest", "false"}}); + ASSERT_FALSE(s.ok()); +} + +// RocksDB lite don't support dynamic options. + +TEST_F(DBOptionsTest, AvoidUpdatingOptions) { + Options options; + options.env = env_; + options.max_background_jobs = 4; + options.delayed_write_rate = 1024; + + Reopen(options); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + bool is_changed_stats = false; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::WriteOptionsFile:PersistOptions", [&](void* /*arg*/) { + ASSERT_FALSE(is_changed_stats); // should only save options file once + is_changed_stats = true; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + // helper function to check the status and reset after each check + auto is_changed = [&] { + bool ret = is_changed_stats; + is_changed_stats = false; + return ret; + }; + + // without changing the value, but it's sanitized to a different value + ASSERT_OK(dbfull()->SetDBOptions({{"bytes_per_sync", "0"}})); + ASSERT_TRUE(is_changed()); + + // without changing the value + ASSERT_OK(dbfull()->SetDBOptions({{"max_background_jobs", "4"}})); + ASSERT_FALSE(is_changed()); + + // changing the value + ASSERT_OK(dbfull()->SetDBOptions({{"bytes_per_sync", "123"}})); + ASSERT_TRUE(is_changed()); + + // update again + ASSERT_OK(dbfull()->SetDBOptions({{"bytes_per_sync", "123"}})); + ASSERT_FALSE(is_changed()); + + // without changing a default value + ASSERT_OK(dbfull()->SetDBOptions({{"strict_bytes_per_sync", "false"}})); + ASSERT_FALSE(is_changed()); + + // now change + ASSERT_OK(dbfull()->SetDBOptions({{"strict_bytes_per_sync", "true"}})); + ASSERT_TRUE(is_changed()); + + // multiple values without change + ASSERT_OK(dbfull()->SetDBOptions( + {{"max_total_wal_size", "0"}, {"stats_dump_period_sec", "600"}})); + ASSERT_FALSE(is_changed()); + + // multiple values with change + ASSERT_OK(dbfull()->SetDBOptions( + {{"max_open_files", "100"}, {"stats_dump_period_sec", "600"}})); + ASSERT_TRUE(is_changed()); +} + +TEST_F(DBOptionsTest, GetLatestDBOptions) { + // GetOptions should be able to get latest option changed by SetOptions. + Options options; + options.create_if_missing = true; + options.env = env_; + Random rnd(228); + Reopen(options); + auto new_options = GetRandomizedMutableDBOptionsMap(&rnd); + ASSERT_OK(dbfull()->SetDBOptions(new_options)); + ASSERT_EQ(new_options, GetMutableDBOptionsMap(dbfull()->GetDBOptions())); +} + +TEST_F(DBOptionsTest, GetLatestCFOptions) { + // GetOptions should be able to get latest option changed by SetOptions. + Options options; + options.create_if_missing = true; + options.env = env_; + Random rnd(228); + Reopen(options); + CreateColumnFamilies({"foo"}, options); + ReopenWithColumnFamilies({"default", "foo"}, options); + auto options_default = GetRandomizedMutableCFOptionsMap(&rnd); + auto options_foo = GetRandomizedMutableCFOptionsMap(&rnd); + ASSERT_OK(dbfull()->SetOptions(handles_[0], options_default)); + ASSERT_OK(dbfull()->SetOptions(handles_[1], options_foo)); + ASSERT_EQ(options_default, + GetMutableCFOptionsMap(dbfull()->GetOptions(handles_[0]))); + ASSERT_EQ(options_foo, + GetMutableCFOptionsMap(dbfull()->GetOptions(handles_[1]))); +} + +TEST_F(DBOptionsTest, SetMutableTableOptions) { + Options options; + options.create_if_missing = true; + options.env = env_; + options.blob_file_size = 16384; + BlockBasedTableOptions bbto; + bbto.no_block_cache = true; + bbto.block_size = 8192; + bbto.block_restart_interval = 7; + + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + Reopen(options); + + ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily(); + Options c_opts = dbfull()->GetOptions(cfh); + + const auto* c_bbto = + c_opts.table_factory->GetOptions(); + ASSERT_NE(c_bbto, nullptr); + ASSERT_EQ(c_opts.blob_file_size, 16384); + ASSERT_EQ(c_bbto->no_block_cache, true); + ASSERT_EQ(c_bbto->block_size, 8192); + ASSERT_EQ(c_bbto->block_restart_interval, 7); + ASSERT_OK(dbfull()->SetOptions( + cfh, {{"table_factory.block_size", "16384"}, + {"table_factory.block_restart_interval", "11"}})); + ASSERT_EQ(c_bbto->block_size, 16384); + ASSERT_EQ(c_bbto->block_restart_interval, 11); + + // Now set an option that is not mutable - options should not change + ASSERT_NOK( + dbfull()->SetOptions(cfh, {{"table_factory.no_block_cache", "false"}})); + ASSERT_EQ(c_bbto->no_block_cache, true); + ASSERT_EQ(c_bbto->block_size, 16384); + ASSERT_EQ(c_bbto->block_restart_interval, 11); + + // Set some that are mutable and some that are not - options should not change + ASSERT_NOK(dbfull()->SetOptions( + cfh, {{"table_factory.no_block_cache", "false"}, + {"table_factory.block_size", "8192"}, + {"table_factory.block_restart_interval", "7"}})); + ASSERT_EQ(c_bbto->no_block_cache, true); + ASSERT_EQ(c_bbto->block_size, 16384); + ASSERT_EQ(c_bbto->block_restart_interval, 11); + + // Set some that are mutable and some that do not exist - options should not + // change + ASSERT_NOK(dbfull()->SetOptions( + cfh, {{"table_factory.block_size", "8192"}, + {"table_factory.does_not_exist", "true"}, + {"table_factory.block_restart_interval", "7"}})); + ASSERT_EQ(c_bbto->no_block_cache, true); + ASSERT_EQ(c_bbto->block_size, 16384); + ASSERT_EQ(c_bbto->block_restart_interval, 11); + + // Trying to change the table factory fails + ASSERT_NOK(dbfull()->SetOptions( + cfh, {{"table_factory", TableFactory::kPlainTableName()}})); + + // Set some on the table and some on the Column Family + ASSERT_OK(dbfull()->SetOptions( + cfh, {{"table_factory.block_size", "16384"}, + {"blob_file_size", "32768"}, + {"table_factory.block_restart_interval", "13"}})); + c_opts = dbfull()->GetOptions(cfh); + ASSERT_EQ(c_opts.blob_file_size, 32768); + ASSERT_EQ(c_bbto->block_size, 16384); + ASSERT_EQ(c_bbto->block_restart_interval, 13); + // Set some on the table and a bad one on the ColumnFamily - options should + // not change + ASSERT_NOK(dbfull()->SetOptions( + cfh, {{"table_factory.block_size", "1024"}, + {"no_such_option", "32768"}, + {"table_factory.block_restart_interval", "7"}})); + ASSERT_EQ(c_bbto->block_size, 16384); + ASSERT_EQ(c_bbto->block_restart_interval, 13); +} + +TEST_F(DBOptionsTest, SetWithCustomMemTableFactory) { + class DummySkipListFactory : public SkipListFactory { + public: + static const char* kClassName() { return "DummySkipListFactory"; } + const char* Name() const override { return kClassName(); } + explicit DummySkipListFactory() : SkipListFactory(2) {} + }; + { + // Verify the DummySkipList cannot be created + ConfigOptions config_options; + config_options.ignore_unsupported_options = false; + std::unique_ptr factory; + ASSERT_NOK(MemTableRepFactory::CreateFromString( + config_options, DummySkipListFactory::kClassName(), &factory)); + } + Options options; + options.create_if_missing = true; + // Try with fail_if_options_file_error=false/true to update the options + for (bool on_error : {false, true}) { + options.fail_if_options_file_error = on_error; + options.env = env_; + options.disable_auto_compactions = false; + + options.memtable_factory.reset(new DummySkipListFactory()); + Reopen(options); + + ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily(); + ASSERT_OK( + dbfull()->SetOptions(cfh, {{"disable_auto_compactions", "true"}})); + ColumnFamilyDescriptor cfd; + ASSERT_OK(cfh->GetDescriptor(&cfd)); + ASSERT_STREQ(cfd.options.memtable_factory->Name(), + DummySkipListFactory::kClassName()); + ColumnFamilyHandle* test = nullptr; + ASSERT_OK(dbfull()->CreateColumnFamily(options, "test", &test)); + ASSERT_OK(test->GetDescriptor(&cfd)); + ASSERT_STREQ(cfd.options.memtable_factory->Name(), + DummySkipListFactory::kClassName()); + + ASSERT_OK(dbfull()->DropColumnFamily(test)); + delete test; + } +} + +TEST_F(DBOptionsTest, SetBytesPerSync) { + const size_t kValueSize = 1024 * 1024; // 1MB + Options options; + options.create_if_missing = true; + options.bytes_per_sync = 1024 * 1024; + options.use_direct_reads = false; + options.write_buffer_size = 400 * kValueSize; + options.disable_auto_compactions = true; + options.compression = kNoCompression; + options.env = env_; + Reopen(options); + int counter = 0; + int low_bytes_per_sync = 0; + int i = 0; + const std::string kValue(kValueSize, 'v'); + ASSERT_EQ(options.bytes_per_sync, dbfull()->GetDBOptions().bytes_per_sync); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WritableFileWriter::RangeSync:0", [&](void* /*arg*/) { counter++; }); + + WriteOptions write_opts; + // should sync approximately 40MB/1MB ~= 40 times. + for (i = 0; i < 40; i++) { + ASSERT_OK(Put(Key(i), kValue, write_opts)); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + low_bytes_per_sync = counter; + ASSERT_GT(low_bytes_per_sync, 35); + ASSERT_LT(low_bytes_per_sync, 45); + + counter = 0; + // 8388608 = 8 * 1024 * 1024 + ASSERT_OK(dbfull()->SetDBOptions({{"bytes_per_sync", "8388608"}})); + ASSERT_EQ(8388608, dbfull()->GetDBOptions().bytes_per_sync); + // should sync approximately 40MB*2/8MB ~= 10 times. + // data will be 40*2MB because of previous Puts too. + for (i = 0; i < 40; i++) { + ASSERT_OK(Put(Key(i), kValue, write_opts)); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_GT(counter, 5); + ASSERT_LT(counter, 15); + + // Redundant assert. But leaving it here just to get the point across that + // low_bytes_per_sync > counter. + ASSERT_GT(low_bytes_per_sync, counter); +} + +TEST_F(DBOptionsTest, SetWalBytesPerSync) { + const size_t kValueSize = 1024 * 1024 * 3; + Options options; + options.create_if_missing = true; + options.wal_bytes_per_sync = 512; + options.write_buffer_size = 100 * kValueSize; + options.disable_auto_compactions = true; + options.compression = kNoCompression; + options.env = env_; + Reopen(options); + ASSERT_EQ(512, dbfull()->GetDBOptions().wal_bytes_per_sync); + std::atomic_int counter{0}; + int low_bytes_per_sync = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WritableFileWriter::RangeSync:0", + [&](void* /*arg*/) { counter.fetch_add(1); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + const std::string kValue(kValueSize, 'v'); + int i = 0; + for (; i < 10; i++) { + ASSERT_OK(Put(Key(i), kValue)); + } + // Do not flush. If we flush here, SwitchWAL will reuse old WAL file since its + // empty and will not get the new wal_bytes_per_sync value. + low_bytes_per_sync = counter; + // 5242880 = 1024 * 1024 * 5 + ASSERT_OK(dbfull()->SetDBOptions({{"wal_bytes_per_sync", "5242880"}})); + ASSERT_EQ(5242880, dbfull()->GetDBOptions().wal_bytes_per_sync); + counter = 0; + i = 0; + for (; i < 10; i++) { + ASSERT_OK(Put(Key(i), kValue)); + } + ASSERT_GT(counter, 0); + ASSERT_GT(low_bytes_per_sync, 0); + ASSERT_GT(low_bytes_per_sync, counter); +} + +TEST_F(DBOptionsTest, WritableFileMaxBufferSize) { + Options options; + options.create_if_missing = true; + options.writable_file_max_buffer_size = 1024 * 1024; + options.level0_file_num_compaction_trigger = 3; + options.max_manifest_file_size = 1; + options.env = env_; + int buffer_size = 1024 * 1024; + Reopen(options); + ASSERT_EQ(buffer_size, + dbfull()->GetDBOptions().writable_file_max_buffer_size); + + std::atomic match_cnt(0); + std::atomic unmatch_cnt(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WritableFileWriter::WritableFileWriter:0", [&](void* arg) { + int value = static_cast(reinterpret_cast(arg)); + if (value == buffer_size) { + match_cnt++; + } else { + unmatch_cnt++; + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + int i = 0; + for (; i < 3; i++) { + ASSERT_OK(Put("foo", std::to_string(i))); + ASSERT_OK(Put("bar", std::to_string(i))); + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(unmatch_cnt, 0); + ASSERT_GE(match_cnt, 11); + + ASSERT_OK( + dbfull()->SetDBOptions({{"writable_file_max_buffer_size", "524288"}})); + buffer_size = 512 * 1024; + match_cnt = 0; + unmatch_cnt = 0; // SetDBOptions() will create a WritableFileWriter + + ASSERT_EQ(buffer_size, + dbfull()->GetDBOptions().writable_file_max_buffer_size); + i = 0; + for (; i < 3; i++) { + ASSERT_OK(Put("foo", std::to_string(i))); + ASSERT_OK(Put("bar", std::to_string(i))); + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(unmatch_cnt, 0); + ASSERT_GE(match_cnt, 11); +} + +TEST_F(DBOptionsTest, SetOptionsAndReopen) { + Random rnd(1044); + auto rand_opts = GetRandomizedMutableCFOptionsMap(&rnd); + ASSERT_OK(dbfull()->SetOptions(rand_opts)); + // Verify if DB can be reopen after setting options. + Options options; + options.env = env_; + ASSERT_OK(TryReopen(options)); +} + +TEST_F(DBOptionsTest, EnableAutoCompactionAndTriggerStall) { + const std::string kValue(1024, 'v'); + for (int method_type = 0; method_type < 2; method_type++) { + for (int option_type = 0; option_type < 4; option_type++) { + Options options; + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.write_buffer_size = 1024 * 1024 * 10; + options.compression = CompressionType::kNoCompression; + options.level0_file_num_compaction_trigger = 1; + options.level0_stop_writes_trigger = std::numeric_limits::max(); + options.level0_slowdown_writes_trigger = std::numeric_limits::max(); + options.hard_pending_compaction_bytes_limit = + std::numeric_limits::max(); + options.soft_pending_compaction_bytes_limit = + std::numeric_limits::max(); + options.env = env_; + + DestroyAndReopen(options); + int i = 0; + for (; i < 1024; i++) { + ASSERT_OK(Put(Key(i), kValue)); + } + ASSERT_OK(Flush()); + for (; i < 1024 * 2; i++) { + ASSERT_OK(Put(Key(i), kValue)); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(2, NumTableFilesAtLevel(0)); + uint64_t l0_size = SizeAtLevel(0); + + switch (option_type) { + case 0: + // test with level0_stop_writes_trigger + options.level0_stop_writes_trigger = 2; + options.level0_slowdown_writes_trigger = 2; + break; + case 1: + options.level0_slowdown_writes_trigger = 2; + break; + case 2: + options.hard_pending_compaction_bytes_limit = l0_size; + options.soft_pending_compaction_bytes_limit = l0_size; + break; + case 3: + options.soft_pending_compaction_bytes_limit = l0_size; + break; + } + Reopen(options); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_FALSE(dbfull()->TEST_write_controler().IsStopped()); + ASSERT_FALSE(dbfull()->TEST_write_controler().NeedsDelay()); + + SyncPoint::GetInstance()->LoadDependency( + {{"DBOptionsTest::EnableAutoCompactionAndTriggerStall:1", + "BackgroundCallCompaction:0"}, + {"DBImpl::BackgroundCompaction():BeforePickCompaction", + "DBOptionsTest::EnableAutoCompactionAndTriggerStall:2"}, + {"DBOptionsTest::EnableAutoCompactionAndTriggerStall:3", + "DBImpl::BackgroundCompaction():AfterPickCompaction"}}); + // Block background compaction. + SyncPoint::GetInstance()->EnableProcessing(); + + switch (method_type) { + case 0: + ASSERT_OK( + dbfull()->SetOptions({{"disable_auto_compactions", "false"}})); + break; + case 1: + ASSERT_OK(dbfull()->EnableAutoCompaction( + {dbfull()->DefaultColumnFamily()})); + break; + } + TEST_SYNC_POINT("DBOptionsTest::EnableAutoCompactionAndTriggerStall:1"); + // Wait for stall condition recalculate. + TEST_SYNC_POINT("DBOptionsTest::EnableAutoCompactionAndTriggerStall:2"); + + switch (option_type) { + case 0: + ASSERT_TRUE(dbfull()->TEST_write_controler().IsStopped()); + break; + case 1: + ASSERT_FALSE(dbfull()->TEST_write_controler().IsStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + break; + case 2: + ASSERT_TRUE(dbfull()->TEST_write_controler().IsStopped()); + break; + case 3: + ASSERT_FALSE(dbfull()->TEST_write_controler().IsStopped()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + break; + } + TEST_SYNC_POINT("DBOptionsTest::EnableAutoCompactionAndTriggerStall:3"); + + // Background compaction executed. + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_FALSE(dbfull()->TEST_write_controler().IsStopped()); + ASSERT_FALSE(dbfull()->TEST_write_controler().NeedsDelay()); + } + } +} + +TEST_F(DBOptionsTest, SetOptionsMayTriggerCompaction) { + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.create_if_missing = true; + options.level0_file_num_compaction_trigger = 1000; + options.env = env_; + Reopen(options); + for (int i = 0; i < 3; i++) { + // Need to insert two keys to avoid trivial move. + ASSERT_OK(Put("foo", std::to_string(i))); + ASSERT_OK(Put("bar", std::to_string(i))); + ASSERT_OK(Flush()); + } + ASSERT_EQ("3", FilesPerLevel()); + ASSERT_OK( + dbfull()->SetOptions({{"level0_file_num_compaction_trigger", "3"}})); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,1", FilesPerLevel()); +} + +TEST_F(DBOptionsTest, SetBackgroundCompactionThreads) { + Options options; + options.create_if_missing = true; + options.max_background_compactions = 1; // default value + options.env = env_; + Reopen(options); + ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); + ASSERT_OK(dbfull()->SetDBOptions({{"max_background_compactions", "3"}})); + ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); + auto stop_token = dbfull()->TEST_write_controler().GetStopToken(); + ASSERT_EQ(3, dbfull()->TEST_BGCompactionsAllowed()); +} + +TEST_F(DBOptionsTest, SetBackgroundFlushThreads) { + Options options; + options.create_if_missing = true; + options.max_background_flushes = 1; + options.env = env_; + Reopen(options); + ASSERT_EQ(1, dbfull()->TEST_BGFlushesAllowed()); + ASSERT_EQ(1, env_->GetBackgroundThreads(Env::Priority::HIGH)); + ASSERT_OK(dbfull()->SetDBOptions({{"max_background_flushes", "3"}})); + ASSERT_EQ(3, env_->GetBackgroundThreads(Env::Priority::HIGH)); + ASSERT_EQ(3, dbfull()->TEST_BGFlushesAllowed()); +} + +TEST_F(DBOptionsTest, SetBackgroundJobs) { + Options options; + options.create_if_missing = true; + options.max_background_jobs = 8; + options.env = env_; + Reopen(options); + + for (int i = 0; i < 2; ++i) { + if (i > 0) { + options.max_background_jobs = 12; + ASSERT_OK(dbfull()->SetDBOptions( + {{"max_background_jobs", + std::to_string(options.max_background_jobs)}})); + } + + const int expected_max_flushes = options.max_background_jobs / 4; + + ASSERT_EQ(expected_max_flushes, dbfull()->TEST_BGFlushesAllowed()); + ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); + + auto stop_token = dbfull()->TEST_write_controler().GetStopToken(); + + const int expected_max_compactions = 3 * expected_max_flushes; + + ASSERT_EQ(expected_max_flushes, dbfull()->TEST_BGFlushesAllowed()); + ASSERT_EQ(expected_max_compactions, dbfull()->TEST_BGCompactionsAllowed()); + + ASSERT_EQ(expected_max_flushes, + env_->GetBackgroundThreads(Env::Priority::HIGH)); + ASSERT_EQ(expected_max_compactions, + env_->GetBackgroundThreads(Env::Priority::LOW)); + } +} + +TEST_F(DBOptionsTest, AvoidFlushDuringShutdown) { + Options options; + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.env = env_; + WriteOptions write_without_wal; + write_without_wal.disableWAL = true; + + ASSERT_FALSE(options.avoid_flush_during_shutdown); + DestroyAndReopen(options); + ASSERT_OK(Put("foo", "v1", write_without_wal)); + Reopen(options); + ASSERT_EQ("v1", Get("foo")); + ASSERT_EQ("1", FilesPerLevel()); + + DestroyAndReopen(options); + ASSERT_OK(Put("foo", "v2", write_without_wal)); + ASSERT_OK(dbfull()->SetDBOptions({{"avoid_flush_during_shutdown", "true"}})); + Reopen(options); + ASSERT_EQ("NOT_FOUND", Get("foo")); + ASSERT_EQ("", FilesPerLevel()); +} + +TEST_F(DBOptionsTest, SetDelayedWriteRateOption) { + Options options; + options.create_if_missing = true; + options.delayed_write_rate = 2 * 1024U * 1024U; + options.env = env_; + Reopen(options); + ASSERT_EQ(2 * 1024U * 1024U, + dbfull()->TEST_write_controler().max_delayed_write_rate()); + + ASSERT_OK(dbfull()->SetDBOptions({{"delayed_write_rate", "20000"}})); + ASSERT_EQ(20000, dbfull()->TEST_write_controler().max_delayed_write_rate()); +} + +TEST_F(DBOptionsTest, MaxTotalWalSizeChange) { + Random rnd(1044); + const auto value_size = size_t(1024); + std::string value = rnd.RandomString(value_size); + + Options options; + options.create_if_missing = true; + options.env = env_; + CreateColumnFamilies({"1", "2", "3"}, options); + ReopenWithColumnFamilies({"default", "1", "2", "3"}, options); + + WriteOptions write_options; + + const int key_count = 100; + for (int i = 0; i < key_count; ++i) { + for (size_t cf = 0; cf < handles_.size(); ++cf) { + ASSERT_OK(Put(static_cast(cf), Key(i), value)); + } + } + ASSERT_OK(dbfull()->SetDBOptions({{"max_total_wal_size", "10"}})); + + for (size_t cf = 0; cf < handles_.size(); ++cf) { + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[cf])); + ASSERT_EQ("1", FilesPerLevel(static_cast(cf))); + } +} + +TEST_F(DBOptionsTest, SetStatsDumpPeriodSec) { + Options options; + options.create_if_missing = true; + options.stats_dump_period_sec = 5; + options.env = env_; + Reopen(options); + ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_dump_period_sec); + + for (int i = 0; i < 20; i++) { + unsigned int num = rand() % 5000 + 1; + ASSERT_OK(dbfull()->SetDBOptions( + {{"stats_dump_period_sec", std::to_string(num)}})); + ASSERT_EQ(num, dbfull()->GetDBOptions().stats_dump_period_sec); + } + Close(); +} + +TEST_F(DBOptionsTest, SetOptionsStatsPersistPeriodSec) { + Options options; + options.create_if_missing = true; + options.stats_persist_period_sec = 5; + options.env = env_; + Reopen(options); + ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_persist_period_sec); + + ASSERT_OK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "12345"}})); + ASSERT_EQ(12345u, dbfull()->GetDBOptions().stats_persist_period_sec); + ASSERT_NOK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "abcde"}})); + ASSERT_EQ(12345u, dbfull()->GetDBOptions().stats_persist_period_sec); +} + +static void assert_candidate_files_empty(DBImpl* dbfull, const bool empty) { + dbfull->TEST_LockMutex(); + JobContext job_context(0); + dbfull->FindObsoleteFiles(&job_context, false); + ASSERT_EQ(empty, job_context.full_scan_candidate_files.empty()); + dbfull->TEST_UnlockMutex(); + if (job_context.HaveSomethingToDelete()) { + // fulfill the contract of FindObsoleteFiles by calling PurgeObsoleteFiles + // afterwards; otherwise the test may hang on shutdown + dbfull->PurgeObsoleteFiles(job_context); + } + job_context.Clean(); +} + +TEST_F(DBOptionsTest, DeleteObsoleteFilesPeriodChange) { + Options options; + options.env = env_; + SetTimeElapseOnlySleepOnReopen(&options); + options.create_if_missing = true; + ASSERT_OK(TryReopen(options)); + + // Verify that candidate files set is empty when no full scan requested. + assert_candidate_files_empty(dbfull(), true); + + ASSERT_OK( + dbfull()->SetDBOptions({{"delete_obsolete_files_period_micros", "0"}})); + + // After delete_obsolete_files_period_micros updated to 0, the next call + // to FindObsoleteFiles should make a full scan + assert_candidate_files_empty(dbfull(), false); + + ASSERT_OK( + dbfull()->SetDBOptions({{"delete_obsolete_files_period_micros", "20"}})); + + assert_candidate_files_empty(dbfull(), true); + + env_->MockSleepForMicroseconds(20); + assert_candidate_files_empty(dbfull(), true); + + env_->MockSleepForMicroseconds(1); + assert_candidate_files_empty(dbfull(), false); + + Close(); +} + +TEST_F(DBOptionsTest, MaxOpenFilesChange) { + SpecialEnv env(env_); + Options options; + options.env = CurrentOptions().env; + options.max_open_files = -1; + + Reopen(options); + + Cache* tc = dbfull()->TEST_table_cache(); + + ASSERT_EQ(-1, dbfull()->GetDBOptions().max_open_files); + ASSERT_LT(2000, tc->GetCapacity()); + ASSERT_OK(dbfull()->SetDBOptions({{"max_open_files", "1024"}})); + ASSERT_EQ(1024, dbfull()->GetDBOptions().max_open_files); + // examine the table cache (actual size should be 1014) + ASSERT_GT(1500, tc->GetCapacity()); + Close(); +} + +TEST_F(DBOptionsTest, SanitizeDelayedWriteRate) { + Options options; + options.env = CurrentOptions().env; + options.delayed_write_rate = 0; + Reopen(options); + ASSERT_EQ(16 * 1024 * 1024, dbfull()->GetDBOptions().delayed_write_rate); + + options.rate_limiter.reset(NewGenericRateLimiter(31 * 1024 * 1024)); + Reopen(options); + ASSERT_EQ(31 * 1024 * 1024, dbfull()->GetDBOptions().delayed_write_rate); +} + +TEST_F(DBOptionsTest, SanitizeUniversalTTLCompaction) { + Options options; + options.env = CurrentOptions().env; + options.compaction_style = kCompactionStyleUniversal; + + options.ttl = 0; + options.periodic_compaction_seconds = 0; + Reopen(options); + ASSERT_EQ(0, dbfull()->GetOptions().ttl); + ASSERT_EQ(0, dbfull()->GetOptions().periodic_compaction_seconds); + + options.ttl = 0; + options.periodic_compaction_seconds = 100; + Reopen(options); + ASSERT_EQ(0, dbfull()->GetOptions().ttl); + ASSERT_EQ(100, dbfull()->GetOptions().periodic_compaction_seconds); + + options.ttl = 100; + options.periodic_compaction_seconds = 0; + Reopen(options); + ASSERT_EQ(100, dbfull()->GetOptions().ttl); + ASSERT_EQ(100, dbfull()->GetOptions().periodic_compaction_seconds); + + options.ttl = 100; + options.periodic_compaction_seconds = 500; + Reopen(options); + ASSERT_EQ(100, dbfull()->GetOptions().ttl); + ASSERT_EQ(100, dbfull()->GetOptions().periodic_compaction_seconds); +} + +TEST_F(DBOptionsTest, SanitizeTtlDefault) { + Options options; + options.env = CurrentOptions().env; + Reopen(options); + ASSERT_EQ(30 * 24 * 60 * 60, dbfull()->GetOptions().ttl); + + options.compaction_style = kCompactionStyleLevel; + options.ttl = 0; + Reopen(options); + ASSERT_EQ(0, dbfull()->GetOptions().ttl); + + options.ttl = 100; + Reopen(options); + ASSERT_EQ(100, dbfull()->GetOptions().ttl); +} + +TEST_F(DBOptionsTest, SanitizeFIFOPeriodicCompaction) { + Options options; + options.compaction_style = kCompactionStyleFIFO; + options.env = CurrentOptions().env; + // Default value allows RocksDB to set ttl to 30 days. + ASSERT_EQ(30 * 24 * 60 * 60, dbfull()->GetOptions().ttl); + + // Disable + options.ttl = 0; + Reopen(options); + ASSERT_EQ(0, dbfull()->GetOptions().ttl); + + options.ttl = 100; + Reopen(options); + ASSERT_EQ(100, dbfull()->GetOptions().ttl); + + options.ttl = 100 * 24 * 60 * 60; + Reopen(options); + ASSERT_EQ(100 * 24 * 60 * 60, dbfull()->GetOptions().ttl); + + // periodic_compaction_seconds should have no effect + // on FIFO compaction. + options.ttl = 500; + options.periodic_compaction_seconds = 300; + Reopen(options); + ASSERT_EQ(500, dbfull()->GetOptions().ttl); +} + +TEST_F(DBOptionsTest, SetFIFOCompactionOptions) { + Options options; + options.env = CurrentOptions().env; + options.compaction_style = kCompactionStyleFIFO; + options.write_buffer_size = 10 << 10; // 10KB + options.arena_block_size = 4096; + options.compression = kNoCompression; + options.create_if_missing = true; + options.compaction_options_fifo.allow_compaction = false; + options.num_levels = 1; + env_->SetMockSleep(); + options.env = env_; + + // NOTE: Presumed unnecessary and removed: resetting mock time in env + + // Test dynamically changing ttl. + options.ttl = 1 * 60 * 60; // 1 hour + ASSERT_OK(TryReopen(options)); + + Random rnd(301); + for (int i = 0; i < 10; i++) { + // Generate and flush a file about 10KB. + for (int j = 0; j < 10; j++) { + ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumTableFilesAtLevel(0), 10); + + env_->MockSleepForSeconds(61); + + // No files should be compacted as ttl is set to 1 hour. + ASSERT_EQ(dbfull()->GetOptions().ttl, 3600); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(NumTableFilesAtLevel(0), 10); + + // Set ttl to 1 minute. So all files should get deleted. + ASSERT_OK(dbfull()->SetOptions({{"ttl", "60"}})); + ASSERT_EQ(dbfull()->GetOptions().ttl, 60); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + + // NOTE: Presumed unnecessary and removed: resetting mock time in env + + // Test dynamically changing compaction_options_fifo.max_table_files_size + options.compaction_options_fifo.max_table_files_size = 500 << 10; // 00KB + options.ttl = 0; + DestroyAndReopen(options); + + for (int i = 0; i < 10; i++) { + // Generate and flush a file about 10KB. + for (int j = 0; j < 10; j++) { + ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumTableFilesAtLevel(0), 10); + + // No files should be compacted as max_table_files_size is set to 500 KB. + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, + 500 << 10); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(NumTableFilesAtLevel(0), 10); + + // Set max_table_files_size to 12 KB. So only 1 file should remain now. + ASSERT_OK(dbfull()->SetOptions( + {{"compaction_options_fifo", "{max_table_files_size=12288;}"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, + 12 << 10); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumTableFilesAtLevel(0), 1); + + // Test dynamically changing compaction_options_fifo.allow_compaction + options.compaction_options_fifo.max_table_files_size = 500 << 10; // 500KB + options.ttl = 0; + options.compaction_options_fifo.allow_compaction = false; + options.level0_file_num_compaction_trigger = 6; + DestroyAndReopen(options); + + for (int i = 0; i < 10; i++) { + // Generate and flush a file about 10KB. + for (int j = 0; j < 10; j++) { + ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumTableFilesAtLevel(0), 10); + + // No files should be compacted as max_table_files_size is set to 500 KB and + // allow_compaction is false + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, + false); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(NumTableFilesAtLevel(0), 10); + + // Set allow_compaction to true. So number of files should be between 1 and 5. + ASSERT_OK(dbfull()->SetOptions( + {{"compaction_options_fifo", "{allow_compaction=true;}"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, + true); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_GE(NumTableFilesAtLevel(0), 1); + ASSERT_LE(NumTableFilesAtLevel(0), 5); + + // Test dynamically setting `file_temperature_age_thresholds` + ASSERT_TRUE( + dbfull() + ->GetOptions() + .compaction_options_fifo.file_temperature_age_thresholds.empty()); + ASSERT_OK(dbfull()->SetOptions({{"compaction_options_fifo", + "{file_temperature_age_thresholds={{age=10;" + "temperature=kWarm}:{age=30000;" + "temperature=kCold}}}"}})); + auto opts = dbfull()->GetOptions(); + const auto& fifo_temp_opt = + opts.compaction_options_fifo.file_temperature_age_thresholds; + ASSERT_EQ(fifo_temp_opt.size(), 2); + ASSERT_EQ(fifo_temp_opt[0].temperature, Temperature::kWarm); + ASSERT_EQ(fifo_temp_opt[0].age, 10); + ASSERT_EQ(fifo_temp_opt[1].temperature, Temperature::kCold); + ASSERT_EQ(fifo_temp_opt[1].age, 30000); +} + +TEST_F(DBOptionsTest, CompactionReadaheadSizeChange) { + SpecialEnv env(env_); + Options options; + options.env = &env; + + options.compaction_readahead_size = 0; + options.level0_file_num_compaction_trigger = 2; + const std::string kValue(1024, 'v'); + Reopen(options); + + ASSERT_EQ(1024 * 1024 * 2, + dbfull()->GetDBOptions().compaction_readahead_size); + ASSERT_OK(dbfull()->SetDBOptions({{"compaction_readahead_size", "256"}})); + ASSERT_EQ(256, dbfull()->GetDBOptions().compaction_readahead_size); + for (int i = 0; i < 1024; i++) { + ASSERT_OK(Put(Key(i), kValue)); + } + ASSERT_OK(Flush()); + for (int i = 0; i < 1024 * 2; i++) { + ASSERT_OK(Put(Key(i), kValue)); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(256, env_->compaction_readahead_size_); + Close(); +} + +TEST_F(DBOptionsTest, FIFOTtlBackwardCompatible) { + Options options; + options.compaction_style = kCompactionStyleFIFO; + options.write_buffer_size = 10 << 10; // 10KB + options.create_if_missing = true; + options.env = CurrentOptions().env; + options.num_levels = 1; + + ASSERT_OK(TryReopen(options)); + + Random rnd(301); + for (int i = 0; i < 10; i++) { + // Generate and flush a file about 10KB. + for (int j = 0; j < 10; j++) { + ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumTableFilesAtLevel(0), 10); + + // In release 6.0, ttl was promoted from a secondary level option under + // compaction_options_fifo to a top level option under ColumnFamilyOptions. + // We still need to handle old SetOptions calls but should ignore + // ttl under compaction_options_fifo. + ASSERT_OK(dbfull()->SetOptions( + {{"compaction_options_fifo", + "{allow_compaction=true;max_table_files_size=1024;ttl=731;file_" + "temperature_age_thresholds={temperature=kCold;age=12345}}"}, + {"ttl", "60"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, + true); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, + 1024); + auto opts = dbfull()->GetOptions(); + const auto& file_temp_age = + opts.compaction_options_fifo.file_temperature_age_thresholds; + ASSERT_EQ(file_temp_age.size(), 1); + ASSERT_EQ(file_temp_age[0].temperature, Temperature::kCold); + ASSERT_EQ(file_temp_age[0].age, 12345); + ASSERT_EQ(dbfull()->GetOptions().ttl, 60); + + // Put ttl as the first option inside compaction_options_fifo. That works as + // it doesn't overwrite any other option. + ASSERT_OK(dbfull()->SetOptions( + {{"compaction_options_fifo", + "{ttl=985;allow_compaction=true;max_table_files_size=1024;}"}, + {"ttl", "191"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, + true); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, + 1024); + ASSERT_EQ(file_temp_age.size(), 1); + ASSERT_EQ(file_temp_age[0].temperature, Temperature::kCold); + ASSERT_EQ(file_temp_age[0].age, 12345); + ASSERT_EQ(dbfull()->GetOptions().ttl, 191); +} + +TEST_F(DBOptionsTest, ChangeCompression) { + if (!Snappy_Supported() || !LZ4_Supported()) { + return; + } + Options options; + options.write_buffer_size = 10 << 10; // 10KB + options.level0_file_num_compaction_trigger = 2; + options.create_if_missing = true; + options.compression = CompressionType::kLZ4Compression; + options.bottommost_compression = CompressionType::kNoCompression; + options.bottommost_compression_opts.level = 2; + options.bottommost_compression_opts.parallel_threads = 1; + options.env = CurrentOptions().env; + + ASSERT_OK(TryReopen(options)); + + CompressionType compression_used = CompressionType::kLZ4Compression; + CompressionOptions compression_opt_used; + bool compacted = false; + SyncPoint::GetInstance()->SetCallBack( + "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) { + Compaction* c = reinterpret_cast(arg); + compression_used = c->output_compression(); + compression_opt_used = c->output_compression_opts(); + compacted = true; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put("foo", "foofoofoo")); + ASSERT_OK(Put("bar", "foofoofoo")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("foo", "foofoofoo")); + ASSERT_OK(Put("bar", "foofoofoo")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_TRUE(compacted); + ASSERT_EQ(CompressionType::kNoCompression, compression_used); + ASSERT_EQ(options.compression_opts.level, compression_opt_used.level); + ASSERT_EQ(options.compression_opts.parallel_threads, + compression_opt_used.parallel_threads); + + compression_used = CompressionType::kLZ4Compression; + compacted = false; + ASSERT_OK(dbfull()->SetOptions( + {{"bottommost_compression", "kSnappyCompression"}, + {"bottommost_compression_opts", "0:6:0:0:4:true"}})); + ASSERT_OK(Put("foo", "foofoofoo")); + ASSERT_OK(Put("bar", "foofoofoo")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("foo", "foofoofoo")); + ASSERT_OK(Put("bar", "foofoofoo")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_TRUE(compacted); + ASSERT_EQ(CompressionType::kSnappyCompression, compression_used); + ASSERT_EQ(6, compression_opt_used.level); + // Right now parallel_level is not yet allowed to be changed. + + SyncPoint::GetInstance()->DisableProcessing(); +} + + +TEST_F(DBOptionsTest, BottommostCompressionOptsWithFallbackType) { + // Verify the bottommost compression options still take effect even when the + // bottommost compression type is left at its default value. Verify for both + // automatic and manual compaction. + if (!Snappy_Supported() || !LZ4_Supported()) { + return; + } + + constexpr int kUpperCompressionLevel = 1; + constexpr int kBottommostCompressionLevel = 2; + constexpr int kNumL0Files = 2; + + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = kNumL0Files; + options.compression = CompressionType::kLZ4Compression; + options.compression_opts.level = kUpperCompressionLevel; + options.bottommost_compression_opts.level = kBottommostCompressionLevel; + options.bottommost_compression_opts.enabled = true; + Reopen(options); + + CompressionType compression_used = CompressionType::kDisableCompressionOption; + CompressionOptions compression_opt_used; + bool compacted = false; + SyncPoint::GetInstance()->SetCallBack( + "CompactionPicker::RegisterCompaction:Registered", [&](void* arg) { + Compaction* c = static_cast(arg); + compression_used = c->output_compression(); + compression_opt_used = c->output_compression_opts(); + compacted = true; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + // First, verify for automatic compaction. + for (int i = 0; i < kNumL0Files; ++i) { + ASSERT_OK(Put("foo", "foofoofoo")); + ASSERT_OK(Put("bar", "foofoofoo")); + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_TRUE(compacted); + ASSERT_EQ(CompressionType::kLZ4Compression, compression_used); + ASSERT_EQ(kBottommostCompressionLevel, compression_opt_used.level); + + // Second, verify for manual compaction. + compacted = false; + compression_used = CompressionType::kDisableCompressionOption; + compression_opt_used = CompressionOptions(); + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForceOptimized; + ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr)); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + + ASSERT_TRUE(compacted); + ASSERT_EQ(CompressionType::kLZ4Compression, compression_used); + ASSERT_EQ(kBottommostCompressionLevel, compression_opt_used.level); +} + +TEST_F(DBOptionsTest, FIFOTemperatureAgeThresholdValidation) { + Options options = CurrentOptions(); + Destroy(options); + + options.num_levels = 1; + options.compaction_style = kCompactionStyleFIFO; + options.max_open_files = -1; + // elements are not sorted + // During DB open + options.compaction_options_fifo.file_temperature_age_thresholds.push_back( + {Temperature::kCold, 1000}); + options.compaction_options_fifo.file_temperature_age_thresholds.push_back( + {Temperature::kWarm, 500}); + Status s = TryReopen(options); + ASSERT_TRUE(s.IsNotSupported()); + ASSERT_TRUE(std::strstr( + s.getState(), + "Option file_temperature_age_thresholds requires elements to be sorted " + "in increasing order with respect to `age` field.")); + // Dynamically set option + options.compaction_options_fifo.file_temperature_age_thresholds.pop_back(); + ASSERT_OK(TryReopen(options)); + s = db_->SetOptions({{"compaction_options_fifo", + "{file_temperature_age_thresholds={{temperature=kCold;" + "age=1000000}:{temperature=kWarm;age=1}}}"}}); + ASSERT_TRUE(s.IsNotSupported()); + ASSERT_TRUE(std::strstr( + s.getState(), + "Option file_temperature_age_thresholds requires elements to be sorted " + "in increasing order with respect to `age` field.")); + + // not single level + // During DB open + options.num_levels = 2; + s = TryReopen(options); + ASSERT_TRUE(s.IsNotSupported()); + ASSERT_TRUE(std::strstr(s.getState(), + "Option file_temperature_age_thresholds is only " + "supported when num_levels = 1.")); + // Dynamically set option + options.compaction_options_fifo.file_temperature_age_thresholds.clear(); + DestroyAndReopen(options); + s = db_->SetOptions( + {{"compaction_options_fifo", + "{file_temperature_age_thresholds={temperature=kCold;age=1000}}"}}); + ASSERT_TRUE(s.IsNotSupported()); + ASSERT_TRUE(std::strstr(s.getState(), + "Option file_temperature_age_thresholds is only " + "supported when num_levels = 1.")); +} + +TEST_F(DBOptionsTest, TempOptionsFailTest) { + std::shared_ptr fs; + std::unique_ptr env; + + fs.reset(new FaultInjectionTestFS(env_->GetFileSystem())); + env = NewCompositeEnv(fs); + Options options = CurrentOptions(); + options.env = env.get(); + + SyncPoint::GetInstance()->SetCallBack( + "PersistRocksDBOptions:create", + [&](void* /*arg*/) { fs->SetFilesystemActive(false); }); + SyncPoint::GetInstance()->SetCallBack( + "PersistRocksDBOptions:written", + [&](void* /*arg*/) { fs->SetFilesystemActive(true); }); + + SyncPoint::GetInstance()->EnableProcessing(); + TryReopen(options); + SyncPoint::GetInstance()->DisableProcessing(); + + std::vector filenames; + ASSERT_OK(env_->GetChildren(dbname_, &filenames)); + uint64_t number; + FileType type; + bool found_temp_file = false; + for (size_t i = 0; i < filenames.size(); i++) { + if (ParseFileName(filenames[i], &number, &type) && type == kTempFile) { + found_temp_file = true; + } + } + ASSERT_FALSE(found_temp_file); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_properties_test.cc b/librocksdb-sys/rocksdb/db/db_properties_test.cc new file mode 100644 index 0000000..a7faa64 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_properties_test.cc @@ -0,0 +1,2367 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include + +#include +#include + +#include "db/db_test_util.h" +#include "db/write_stall_stats.h" +#include "options/cf_options.h" +#include "port/stack_trace.h" +#include "rocksdb/listener.h" +#include "rocksdb/options.h" +#include "rocksdb/perf_context.h" +#include "rocksdb/perf_level.h" +#include "rocksdb/table.h" +#include "table/block_based/block.h" +#include "table/format.h" +#include "table/meta_blocks.h" +#include "table/table_builder.h" +#include "test_util/mock_time_env.h" +#include "util/random.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +class DBPropertiesTest : public DBTestBase { + public: + DBPropertiesTest() + : DBTestBase("db_properties_test", /*env_do_fsync=*/false) {} + + void AssertDbStats(const std::map& db_stats, + double expected_uptime, int expected_user_bytes_written, + int expected_wal_bytes_written, + int expected_user_writes_by_self, + int expected_user_writes_with_wal) { + ASSERT_EQ(std::to_string(expected_uptime), db_stats.at("db.uptime")); + ASSERT_EQ(std::to_string(expected_wal_bytes_written), + db_stats.at("db.wal_bytes_written")); + ASSERT_EQ("0", db_stats.at("db.wal_syncs")); + ASSERT_EQ(std::to_string(expected_user_bytes_written), + db_stats.at("db.user_bytes_written")); + ASSERT_EQ("0", db_stats.at("db.user_writes_by_other")); + ASSERT_EQ(std::to_string(expected_user_writes_by_self), + db_stats.at("db.user_writes_by_self")); + ASSERT_EQ(std::to_string(expected_user_writes_with_wal), + db_stats.at("db.user_writes_with_wal")); + ASSERT_EQ("0", db_stats.at("db.user_write_stall_micros")); + } +}; + +TEST_F(DBPropertiesTest, Empty) { + do { + Options options; + options.env = env_; + options.write_buffer_size = 100000; // Small write buffer + options.allow_concurrent_memtable_write = false; + options = CurrentOptions(options); + CreateAndReopenWithCF({"pikachu"}, options); + + std::string num; + ASSERT_TRUE(dbfull()->GetProperty( + handles_[1], "rocksdb.num-entries-active-mem-table", &num)); + ASSERT_EQ("0", num); + + ASSERT_OK(Put(1, "foo", "v1")); + ASSERT_EQ("v1", Get(1, "foo")); + ASSERT_TRUE(dbfull()->GetProperty( + handles_[1], "rocksdb.num-entries-active-mem-table", &num)); + ASSERT_EQ("1", num); + + // Block sync calls + env_->delay_sstable_sync_.store(true, std::memory_order_release); + ASSERT_OK(Put(1, "k1", std::string(100000, 'x'))); // Fill memtable + ASSERT_TRUE(dbfull()->GetProperty( + handles_[1], "rocksdb.num-entries-active-mem-table", &num)); + ASSERT_EQ("2", num); + + ASSERT_OK(Put(1, "k2", std::string(100000, 'y'))); // Trigger compaction + ASSERT_TRUE(dbfull()->GetProperty( + handles_[1], "rocksdb.num-entries-active-mem-table", &num)); + ASSERT_EQ("1", num); + + ASSERT_EQ("v1", Get(1, "foo")); + // Release sync calls + env_->delay_sstable_sync_.store(false, std::memory_order_release); + + ASSERT_OK(db_->DisableFileDeletions()); + ASSERT_TRUE( + dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num)); + ASSERT_EQ("0", num); + + ASSERT_OK(db_->DisableFileDeletions()); + ASSERT_TRUE( + dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num)); + ASSERT_EQ("0", num); + + ASSERT_OK(db_->DisableFileDeletions()); + ASSERT_TRUE( + dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num)); + ASSERT_EQ("0", num); + + ASSERT_OK(db_->EnableFileDeletions(false)); + ASSERT_TRUE( + dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num)); + ASSERT_EQ("0", num); + + ASSERT_OK(db_->EnableFileDeletions()); + ASSERT_TRUE( + dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num)); + ASSERT_EQ("1", num); + } while (ChangeOptions()); +} + +TEST_F(DBPropertiesTest, CurrentVersionNumber) { + uint64_t v1, v2, v3; + ASSERT_TRUE( + dbfull()->GetIntProperty("rocksdb.current-super-version-number", &v1)); + ASSERT_OK(Put("12345678", "")); + ASSERT_TRUE( + dbfull()->GetIntProperty("rocksdb.current-super-version-number", &v2)); + ASSERT_OK(Flush()); + ASSERT_TRUE( + dbfull()->GetIntProperty("rocksdb.current-super-version-number", &v3)); + + ASSERT_EQ(v1, v2); + ASSERT_GT(v3, v2); +} + +TEST_F(DBPropertiesTest, GetAggregatedIntPropertyTest) { + const int kKeySize = 100; + const int kValueSize = 500; + const int kKeyNum = 100; + + Options options; + options.env = env_; + options.create_if_missing = true; + options.write_buffer_size = (kKeySize + kValueSize) * kKeyNum / 10; + // Make them never flush + options.min_write_buffer_number_to_merge = 1000; + options.max_write_buffer_number = 1000; + options = CurrentOptions(options); + CreateAndReopenWithCF({"one", "two", "three", "four"}, options); + + Random rnd(301); + for (auto* handle : handles_) { + for (int i = 0; i < kKeyNum; ++i) { + ASSERT_OK(db_->Put(WriteOptions(), handle, rnd.RandomString(kKeySize), + rnd.RandomString(kValueSize))); + } + } + + uint64_t manual_sum = 0; + uint64_t api_sum = 0; + uint64_t value = 0; + for (auto* handle : handles_) { + ASSERT_TRUE( + db_->GetIntProperty(handle, DB::Properties::kSizeAllMemTables, &value)); + manual_sum += value; + } + ASSERT_TRUE(db_->GetAggregatedIntProperty(DB::Properties::kSizeAllMemTables, + &api_sum)); + ASSERT_GT(manual_sum, 0); + ASSERT_EQ(manual_sum, api_sum); + + ASSERT_FALSE(db_->GetAggregatedIntProperty(DB::Properties::kDBStats, &value)); + + uint64_t before_flush_trm; + uint64_t after_flush_trm; + for (auto* handle : handles_) { + ASSERT_TRUE(db_->GetAggregatedIntProperty( + DB::Properties::kEstimateTableReadersMem, &before_flush_trm)); + + // Issue flush and expect larger memory usage of table readers. + ASSERT_OK(db_->Flush(FlushOptions(), handle)); + + ASSERT_TRUE(db_->GetAggregatedIntProperty( + DB::Properties::kEstimateTableReadersMem, &after_flush_trm)); + ASSERT_GT(after_flush_trm, before_flush_trm); + } +} + +namespace { +void VerifySimilar(uint64_t a, uint64_t b, double bias) { + ASSERT_EQ(a == 0U, b == 0U); + if (a == 0) { + return; + } + double dbl_a = static_cast(a); + double dbl_b = static_cast(b); + if (dbl_a > dbl_b) { + ASSERT_LT(static_cast(dbl_a - dbl_b) / (dbl_a + dbl_b), bias); + } else { + ASSERT_LT(static_cast(dbl_b - dbl_a) / (dbl_a + dbl_b), bias); + } +} + +void VerifyTableProperties( + const TableProperties& base_tp, const TableProperties& new_tp, + double filter_size_bias = CACHE_LINE_SIZE >= 256 ? 0.18 : 0.1, + double index_size_bias = 0.1, double data_size_bias = 0.1, + double num_data_blocks_bias = 0.05) { + VerifySimilar(base_tp.data_size, new_tp.data_size, data_size_bias); + VerifySimilar(base_tp.index_size, new_tp.index_size, index_size_bias); + VerifySimilar(base_tp.filter_size, new_tp.filter_size, filter_size_bias); + VerifySimilar(base_tp.num_data_blocks, new_tp.num_data_blocks, + num_data_blocks_bias); + + ASSERT_EQ(base_tp.raw_key_size, new_tp.raw_key_size); + ASSERT_EQ(base_tp.raw_value_size, new_tp.raw_value_size); + ASSERT_EQ(base_tp.num_entries, new_tp.num_entries); + ASSERT_EQ(base_tp.num_deletions, new_tp.num_deletions); + ASSERT_EQ(base_tp.num_range_deletions, new_tp.num_range_deletions); + + // Merge operands may become Puts, so we only have an upper bound the exact + // number of merge operands. + ASSERT_GE(base_tp.num_merge_operands, new_tp.num_merge_operands); +} + +void GetExpectedTableProperties( + TableProperties* expected_tp, const int kKeySize, const int kValueSize, + const int kPutsPerTable, const int kDeletionsPerTable, + const int kMergeOperandsPerTable, const int kRangeDeletionsPerTable, + const int kTableCount, const int kBloomBitsPerKey, const size_t kBlockSize, + const bool index_key_is_user_key, const bool value_delta_encoding) { + const int kKeysPerTable = + kPutsPerTable + kDeletionsPerTable + kMergeOperandsPerTable; + const int kPutCount = kTableCount * kPutsPerTable; + const int kDeletionCount = kTableCount * kDeletionsPerTable; + const int kMergeCount = kTableCount * kMergeOperandsPerTable; + const int kRangeDeletionCount = kTableCount * kRangeDeletionsPerTable; + const int kKeyCount = + kPutCount + kDeletionCount + kMergeCount + kRangeDeletionCount; + const int kAvgSuccessorSize = kKeySize / 5; + const int kEncodingSavePerKey = kKeySize / 4; + expected_tp->raw_key_size = kKeyCount * (kKeySize + 8); + expected_tp->raw_value_size = + (kPutCount + kMergeCount + kRangeDeletionCount) * kValueSize; + expected_tp->num_entries = kKeyCount; + expected_tp->num_deletions = kDeletionCount + kRangeDeletionCount; + expected_tp->num_merge_operands = kMergeCount; + expected_tp->num_range_deletions = kRangeDeletionCount; + expected_tp->num_data_blocks = + kTableCount * + (kKeysPerTable * (kKeySize - kEncodingSavePerKey + kValueSize)) / + kBlockSize; + expected_tp->data_size = + kTableCount * (kKeysPerTable * (kKeySize + 8 + kValueSize)); + expected_tp->index_size = + expected_tp->num_data_blocks * + (kAvgSuccessorSize + (index_key_is_user_key ? 0 : 8) - + // discount 1 byte as value size is not encoded in value delta encoding + (value_delta_encoding ? 1 : 0)); + expected_tp->filter_size = + kTableCount * ((kKeysPerTable * kBloomBitsPerKey + 7) / 8 + + /*average-ish overhead*/ CACHE_LINE_SIZE / 2); +} +} // anonymous namespace + +TEST_F(DBPropertiesTest, ValidatePropertyInfo) { + for (const auto& ppt_name_and_info : InternalStats::ppt_name_to_info) { + // If C++ gets a std::string_literal, this would be better to check at + // compile-time using static_assert. + ASSERT_TRUE(ppt_name_and_info.first.empty() || + !isdigit(ppt_name_and_info.first.back())); + + int count = 0; + count += (ppt_name_and_info.second.handle_string == nullptr) ? 0 : 1; + count += (ppt_name_and_info.second.handle_int == nullptr) ? 0 : 1; + count += (ppt_name_and_info.second.handle_string_dbimpl == nullptr) ? 0 : 1; + ASSERT_TRUE(count == 1); + } +} + +TEST_F(DBPropertiesTest, ValidateSampleNumber) { + // When "max_open_files" is -1, we read all the files for + // "rocksdb.estimate-num-keys" computation, which is the ground truth. + // Otherwise, we sample 20 newest files to make an estimation. + // Formula: lastest_20_files_active_key_ratio * total_files + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.level0_stop_writes_trigger = 1000; + DestroyAndReopen(options); + int key = 0; + for (int files = 20; files >= 10; files -= 10) { + for (int i = 0; i < files; i++) { + int rows = files / 10; + for (int j = 0; j < rows; j++) { + ASSERT_OK(db_->Put(WriteOptions(), std::to_string(++key), "foo")); + } + ASSERT_OK(db_->Flush(FlushOptions())); + } + } + std::string num; + Reopen(options); + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num)); + ASSERT_EQ("45", num); + options.max_open_files = -1; + Reopen(options); + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num)); + ASSERT_EQ("50", num); +} + +TEST_F(DBPropertiesTest, AggregatedTableProperties) { + for (int kTableCount = 40; kTableCount <= 100; kTableCount += 30) { + const int kDeletionsPerTable = 0; + const int kMergeOperandsPerTable = 15; + const int kRangeDeletionsPerTable = 5; + const int kPutsPerTable = 100; + const int kKeySize = 80; + const int kValueSize = 200; + const int kBloomBitsPerKey = 20; + + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = 8; + options.compression = kNoCompression; + options.create_if_missing = true; + options.merge_operator.reset(new TestPutOperator()); + + BlockBasedTableOptions table_options; + table_options.filter_policy.reset( + NewBloomFilterPolicy(kBloomBitsPerKey, false)); + table_options.block_size = 1024; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + DestroyAndReopen(options); + + // Hold open a snapshot to prevent range tombstones from being compacted + // away. + ManagedSnapshot snapshot(db_); + + Random rnd(5632); + for (int table = 1; table <= kTableCount; ++table) { + for (int i = 0; i < kPutsPerTable; ++i) { + ASSERT_OK(db_->Put(WriteOptions(), rnd.RandomString(kKeySize), + rnd.RandomString(kValueSize))); + } + for (int i = 0; i < kDeletionsPerTable; i++) { + ASSERT_OK(db_->Delete(WriteOptions(), rnd.RandomString(kKeySize))); + } + for (int i = 0; i < kMergeOperandsPerTable; i++) { + ASSERT_OK(db_->Merge(WriteOptions(), rnd.RandomString(kKeySize), + rnd.RandomString(kValueSize))); + } + for (int i = 0; i < kRangeDeletionsPerTable; i++) { + std::string start = rnd.RandomString(kKeySize); + std::string end = start; + end.resize(kValueSize); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + start, end)); + } + ASSERT_OK(db_->Flush(FlushOptions())); + } + std::string property; + db_->GetProperty(DB::Properties::kAggregatedTableProperties, &property); + TableProperties output_tp; + ParseTablePropertiesString(property, &output_tp); + bool index_key_is_user_key = output_tp.index_key_is_user_key > 0; + bool value_is_delta_encoded = output_tp.index_value_is_delta_encoded > 0; + + TableProperties expected_tp; + GetExpectedTableProperties( + &expected_tp, kKeySize, kValueSize, kPutsPerTable, kDeletionsPerTable, + kMergeOperandsPerTable, kRangeDeletionsPerTable, kTableCount, + kBloomBitsPerKey, table_options.block_size, index_key_is_user_key, + value_is_delta_encoded); + + VerifyTableProperties(expected_tp, output_tp); + } +} + +TEST_F(DBPropertiesTest, ReadLatencyHistogramByLevel) { + Options options = CurrentOptions(); + options.write_buffer_size = 110 << 10; + options.level0_file_num_compaction_trigger = 6; + options.num_levels = 4; + options.compression = kNoCompression; + options.max_bytes_for_level_base = 4500 << 10; + options.target_file_size_base = 98 << 10; + options.max_write_buffer_number = 2; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.max_open_files = 11; // Make sure no proloading of table readers + + // RocksDB sanitize max open files to at least 20. Modify it back. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SanitizeOptions::AfterChangeMaxOpenFiles", [&](void* arg) { + int* max_open_files = static_cast(arg); + *max_open_files = 11; + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + BlockBasedTableOptions table_options; + table_options.no_block_cache = true; + + CreateAndReopenWithCF({"pikachu"}, options); + int key_index = 0; + Random rnd(301); + for (int num = 0; num < 8; num++) { + ASSERT_OK(Put("foo", "bar")); + GenerateNewFile(&rnd, &key_index); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + std::string prop; + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop)); + + // Get() after flushes, See latency histogram tracked. + for (int key = 0; key < key_index; key++) { + Get(Key(key)); + } + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cfstats", &prop)); + ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram")); + ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram")); + ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram")); + + // Reopen and issue Get(). See thee latency tracked + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + for (int key = 0; key < key_index; key++) { + Get(Key(key)); + } + + // Test for getting immutable_db_options_.statistics + ASSERT_TRUE(dbfull()->GetProperty(dbfull()->DefaultColumnFamily(), + "rocksdb.options-statistics", &prop)); + ASSERT_NE(std::string::npos, prop.find("rocksdb.block.cache.miss")); + ASSERT_EQ(std::string::npos, prop.find("rocksdb.db.f.micros")); + + ASSERT_TRUE(dbfull()->GetProperty(dbfull()->DefaultColumnFamily(), + "rocksdb.cf-file-histogram", &prop)); + ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram")); + ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram")); + ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram")); + + // Reopen and issue iterating. See thee latency tracked + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cf-file-histogram", &prop)); + ASSERT_EQ(std::string::npos, prop.find("** Level 0 read latency histogram")); + ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram")); + ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram")); + { + std::unique_ptr iter(db_->NewIterator(ReadOptions())); + for (iter->Seek(Key(0)); iter->Valid(); iter->Next()) { + } + ASSERT_OK(iter->status()); + } + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cf-file-histogram", &prop)); + ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram")); + ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram")); + ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram")); + + // CF 1 should show no histogram. + ASSERT_TRUE( + dbfull()->GetProperty(handles_[1], "rocksdb.cf-file-histogram", &prop)); + ASSERT_EQ(std::string::npos, prop.find("** Level 0 read latency histogram")); + ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram")); + ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram")); + // put something and read it back , CF 1 should show histogram. + ASSERT_OK(Put(1, "foo", "bar")); + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("bar", Get(1, "foo")); + + ASSERT_TRUE( + dbfull()->GetProperty(handles_[1], "rocksdb.cf-file-histogram", &prop)); + ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram")); + ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram")); + ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram")); + + // options.max_open_files preloads table readers. + options.max_open_files = -1; + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_TRUE(dbfull()->GetProperty(dbfull()->DefaultColumnFamily(), + "rocksdb.cf-file-histogram", &prop)); + ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram")); + ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram")); + ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram")); + for (int key = 0; key < key_index; key++) { + Get(Key(key)); + } + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cfstats", &prop)); + ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram")); + ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram")); + ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram")); + + // Clear internal stats + ASSERT_OK(dbfull()->ResetStats()); + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cfstats", &prop)); + ASSERT_EQ(std::string::npos, prop.find("** Level 0 read latency histogram")); + ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram")); + ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram")); +} + +TEST_F(DBPropertiesTest, AggregatedTablePropertiesAtLevel) { + const int kTableCount = 100; + const int kDeletionsPerTable = 0; + const int kMergeOperandsPerTable = 2; + const int kRangeDeletionsPerTable = 2; + const int kPutsPerTable = 10; + const int kKeySize = 50; + const int kValueSize = 400; + const int kMaxLevel = 7; + const int kBloomBitsPerKey = 20; + Random rnd(301); + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = 8; + options.compression = kNoCompression; + options.create_if_missing = true; + options.level0_file_num_compaction_trigger = 2; + options.target_file_size_base = 8192; + options.max_bytes_for_level_base = 10000; + options.max_bytes_for_level_multiplier = 2; + // This ensures there no compaction happening when we call GetProperty(). + options.disable_auto_compactions = true; + options.merge_operator.reset(new TestPutOperator()); + + BlockBasedTableOptions table_options; + table_options.filter_policy.reset( + NewBloomFilterPolicy(kBloomBitsPerKey, false)); + table_options.block_size = 1024; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + DestroyAndReopen(options); + + // Hold open a snapshot to prevent range tombstones from being compacted away. + ManagedSnapshot snapshot(db_); + + std::string level_tp_strings[kMaxLevel]; + std::string tp_string; + TableProperties level_tps[kMaxLevel]; + TableProperties tp, sum_tp, expected_tp; + for (int table = 1; table <= kTableCount; ++table) { + for (int i = 0; i < kPutsPerTable; ++i) { + ASSERT_OK(db_->Put(WriteOptions(), rnd.RandomString(kKeySize), + rnd.RandomString(kValueSize))); + } + for (int i = 0; i < kDeletionsPerTable; i++) { + ASSERT_OK(db_->Delete(WriteOptions(), rnd.RandomString(kKeySize))); + } + for (int i = 0; i < kMergeOperandsPerTable; i++) { + ASSERT_OK(db_->Merge(WriteOptions(), rnd.RandomString(kKeySize), + rnd.RandomString(kValueSize))); + } + for (int i = 0; i < kRangeDeletionsPerTable; i++) { + std::string start = rnd.RandomString(kKeySize); + std::string end = start; + end.resize(kValueSize); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + start, end)); + } + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ResetTableProperties(&sum_tp); + for (int level = 0; level < kMaxLevel; ++level) { + db_->GetProperty(DB::Properties::kAggregatedTablePropertiesAtLevel + + std::to_string(level), + &level_tp_strings[level]); + ParseTablePropertiesString(level_tp_strings[level], &level_tps[level]); + sum_tp.data_size += level_tps[level].data_size; + sum_tp.index_size += level_tps[level].index_size; + sum_tp.filter_size += level_tps[level].filter_size; + sum_tp.raw_key_size += level_tps[level].raw_key_size; + sum_tp.raw_value_size += level_tps[level].raw_value_size; + sum_tp.num_data_blocks += level_tps[level].num_data_blocks; + sum_tp.num_entries += level_tps[level].num_entries; + sum_tp.num_deletions += level_tps[level].num_deletions; + sum_tp.num_merge_operands += level_tps[level].num_merge_operands; + sum_tp.num_range_deletions += level_tps[level].num_range_deletions; + } + db_->GetProperty(DB::Properties::kAggregatedTableProperties, &tp_string); + ParseTablePropertiesString(tp_string, &tp); + bool index_key_is_user_key = tp.index_key_is_user_key > 0; + bool value_is_delta_encoded = tp.index_value_is_delta_encoded > 0; + ASSERT_EQ(sum_tp.data_size, tp.data_size); + ASSERT_EQ(sum_tp.index_size, tp.index_size); + ASSERT_EQ(sum_tp.filter_size, tp.filter_size); + ASSERT_EQ(sum_tp.raw_key_size, tp.raw_key_size); + ASSERT_EQ(sum_tp.raw_value_size, tp.raw_value_size); + ASSERT_EQ(sum_tp.num_data_blocks, tp.num_data_blocks); + ASSERT_EQ(sum_tp.num_entries, tp.num_entries); + ASSERT_EQ(sum_tp.num_deletions, tp.num_deletions); + ASSERT_EQ(sum_tp.num_merge_operands, tp.num_merge_operands); + ASSERT_EQ(sum_tp.num_range_deletions, tp.num_range_deletions); + if (table > 3) { + GetExpectedTableProperties( + &expected_tp, kKeySize, kValueSize, kPutsPerTable, kDeletionsPerTable, + kMergeOperandsPerTable, kRangeDeletionsPerTable, table, + kBloomBitsPerKey, table_options.block_size, index_key_is_user_key, + value_is_delta_encoded); + // Gives larger bias here as index block size, filter block size, + // and data block size become much harder to estimate in this test. + VerifyTableProperties(expected_tp, tp, CACHE_LINE_SIZE >= 256 ? 0.6 : 0.5, + 0.5, 0.5, 0.25); + } + } +} + +TEST_F(DBPropertiesTest, NumImmutableMemTable) { + do { + Options options = CurrentOptions(); + WriteOptions writeOpt = WriteOptions(); + writeOpt.disableWAL = true; + options.max_write_buffer_number = 4; + options.min_write_buffer_number_to_merge = 3; + options.write_buffer_size = 1000000; + options.max_write_buffer_size_to_maintain = + 5 * static_cast(options.write_buffer_size); + CreateAndReopenWithCF({"pikachu"}, options); + + std::string big_value(1000000 * 2, 'x'); + std::string num; + uint64_t value; + SetPerfLevel(kEnableTime); + ASSERT_TRUE(GetPerfLevel() == kEnableTime); + + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k1", big_value)); + ASSERT_TRUE(dbfull()->GetProperty(handles_[1], + "rocksdb.num-immutable-mem-table", &num)); + ASSERT_EQ(num, "0"); + ASSERT_TRUE(dbfull()->GetProperty( + handles_[1], DB::Properties::kNumImmutableMemTableFlushed, &num)); + ASSERT_EQ(num, "0"); + ASSERT_TRUE(dbfull()->GetProperty( + handles_[1], "rocksdb.num-entries-active-mem-table", &num)); + ASSERT_EQ(num, "1"); + get_perf_context()->Reset(); + Get(1, "k1"); + ASSERT_EQ(1, static_cast(get_perf_context()->get_from_memtable_count)); + + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k2", big_value)); + ASSERT_TRUE(dbfull()->GetProperty(handles_[1], + "rocksdb.num-immutable-mem-table", &num)); + ASSERT_EQ(num, "1"); + ASSERT_TRUE(dbfull()->GetProperty( + handles_[1], "rocksdb.num-entries-active-mem-table", &num)); + ASSERT_EQ(num, "1"); + ASSERT_TRUE(dbfull()->GetProperty( + handles_[1], "rocksdb.num-entries-imm-mem-tables", &num)); + ASSERT_EQ(num, "1"); + + get_perf_context()->Reset(); + Get(1, "k1"); + ASSERT_EQ(2, static_cast(get_perf_context()->get_from_memtable_count)); + get_perf_context()->Reset(); + Get(1, "k2"); + ASSERT_EQ(1, static_cast(get_perf_context()->get_from_memtable_count)); + + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k3", big_value)); + ASSERT_TRUE(dbfull()->GetProperty( + handles_[1], "rocksdb.cur-size-active-mem-table", &num)); + ASSERT_TRUE(dbfull()->GetProperty(handles_[1], + "rocksdb.num-immutable-mem-table", &num)); + ASSERT_EQ(num, "2"); + ASSERT_TRUE(dbfull()->GetProperty( + handles_[1], "rocksdb.num-entries-active-mem-table", &num)); + ASSERT_EQ(num, "1"); + ASSERT_TRUE(dbfull()->GetProperty( + handles_[1], "rocksdb.num-entries-imm-mem-tables", &num)); + ASSERT_EQ(num, "2"); + get_perf_context()->Reset(); + Get(1, "k2"); + ASSERT_EQ(2, static_cast(get_perf_context()->get_from_memtable_count)); + get_perf_context()->Reset(); + Get(1, "k3"); + ASSERT_EQ(1, static_cast(get_perf_context()->get_from_memtable_count)); + get_perf_context()->Reset(); + Get(1, "k1"); + ASSERT_EQ(3, static_cast(get_perf_context()->get_from_memtable_count)); + + ASSERT_OK(Flush(1)); + ASSERT_TRUE(dbfull()->GetProperty(handles_[1], + "rocksdb.num-immutable-mem-table", &num)); + ASSERT_EQ(num, "0"); + ASSERT_TRUE(dbfull()->GetProperty( + handles_[1], DB::Properties::kNumImmutableMemTableFlushed, &num)); + ASSERT_EQ(num, "3"); + ASSERT_TRUE(dbfull()->GetIntProperty( + handles_[1], "rocksdb.cur-size-active-mem-table", &value)); + // "192" is the size of the metadata of two empty skiplists, this would + // break if we change the default skiplist implementation + ASSERT_GE(value, 192); + + uint64_t int_num; + uint64_t base_total_size; + ASSERT_TRUE(dbfull()->GetIntProperty( + handles_[1], "rocksdb.estimate-num-keys", &base_total_size)); + + ASSERT_OK(dbfull()->Delete(writeOpt, handles_[1], "k2")); + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k3", "")); + ASSERT_OK(dbfull()->Delete(writeOpt, handles_[1], "k3")); + ASSERT_TRUE(dbfull()->GetIntProperty( + handles_[1], "rocksdb.num-deletes-active-mem-table", &int_num)); + ASSERT_EQ(int_num, 2U); + ASSERT_TRUE(dbfull()->GetIntProperty( + handles_[1], "rocksdb.num-entries-active-mem-table", &int_num)); + ASSERT_EQ(int_num, 3U); + + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k2", big_value)); + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k2", big_value)); + ASSERT_TRUE(dbfull()->GetIntProperty( + handles_[1], "rocksdb.num-entries-imm-mem-tables", &int_num)); + ASSERT_EQ(int_num, 4U); + ASSERT_TRUE(dbfull()->GetIntProperty( + handles_[1], "rocksdb.num-deletes-imm-mem-tables", &int_num)); + ASSERT_EQ(int_num, 2U); + + ASSERT_TRUE(dbfull()->GetIntProperty( + handles_[1], "rocksdb.estimate-num-keys", &int_num)); + ASSERT_EQ(int_num, base_total_size + 1); + + SetPerfLevel(kDisable); + ASSERT_TRUE(GetPerfLevel() == kDisable); + } while (ChangeCompactOptions()); +} + +// TODO(techdept) : Disabled flaky test #12863555 +TEST_F(DBPropertiesTest, DISABLED_GetProperty) { + // Set sizes to both background thread pool to be 1 and block them. + env_->SetBackgroundThreads(1, Env::HIGH); + env_->SetBackgroundThreads(1, Env::LOW); + test::SleepingBackgroundTask sleeping_task_low; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::LOW); + test::SleepingBackgroundTask sleeping_task_high; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, + &sleeping_task_high, Env::Priority::HIGH); + + Options options = CurrentOptions(); + WriteOptions writeOpt = WriteOptions(); + writeOpt.disableWAL = true; + options.compaction_style = kCompactionStyleUniversal; + options.level0_file_num_compaction_trigger = 1; + options.compaction_options_universal.size_ratio = 50; + options.max_background_compactions = 1; + options.max_background_flushes = 1; + options.max_write_buffer_number = 10; + options.min_write_buffer_number_to_merge = 1; + options.max_write_buffer_size_to_maintain = 0; + options.write_buffer_size = 1000000; + Reopen(options); + + std::string big_value(1000000 * 2, 'x'); + std::string num; + uint64_t int_num; + SetPerfLevel(kEnableTime); + + ASSERT_TRUE( + dbfull()->GetIntProperty("rocksdb.estimate-table-readers-mem", &int_num)); + ASSERT_EQ(int_num, 0U); + ASSERT_TRUE( + dbfull()->GetIntProperty("rocksdb.estimate-live-data-size", &int_num)); + ASSERT_EQ(int_num, 0U); + + ASSERT_OK(dbfull()->Put(writeOpt, "k1", big_value)); + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num)); + ASSERT_EQ(num, "0"); + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.mem-table-flush-pending", &num)); + ASSERT_EQ(num, "0"); + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.compaction-pending", &num)); + ASSERT_EQ(num, "0"); + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num)); + ASSERT_EQ(num, "1"); + get_perf_context()->Reset(); + + ASSERT_OK(dbfull()->Put(writeOpt, "k2", big_value)); + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num)); + ASSERT_EQ(num, "1"); + ASSERT_OK(dbfull()->Delete(writeOpt, "k-non-existing")); + ASSERT_OK(dbfull()->Put(writeOpt, "k3", big_value)); + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num)); + ASSERT_EQ(num, "2"); + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.mem-table-flush-pending", &num)); + ASSERT_EQ(num, "1"); + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.compaction-pending", &num)); + ASSERT_EQ(num, "0"); + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num)); + ASSERT_EQ(num, "2"); + // Verify the same set of properties through GetIntProperty + ASSERT_TRUE( + dbfull()->GetIntProperty("rocksdb.num-immutable-mem-table", &int_num)); + ASSERT_EQ(int_num, 2U); + ASSERT_TRUE( + dbfull()->GetIntProperty("rocksdb.mem-table-flush-pending", &int_num)); + ASSERT_EQ(int_num, 1U); + ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.compaction-pending", &int_num)); + ASSERT_EQ(int_num, 0U); + ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.estimate-num-keys", &int_num)); + ASSERT_EQ(int_num, 2U); + + ASSERT_TRUE( + dbfull()->GetIntProperty("rocksdb.estimate-table-readers-mem", &int_num)); + ASSERT_EQ(int_num, 0U); + + sleeping_task_high.WakeUp(); + sleeping_task_high.WaitUntilDone(); + dbfull()->TEST_WaitForFlushMemTable(); + + ASSERT_OK(dbfull()->Put(writeOpt, "k4", big_value)); + ASSERT_OK(dbfull()->Put(writeOpt, "k5", big_value)); + dbfull()->TEST_WaitForFlushMemTable(); + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.mem-table-flush-pending", &num)); + ASSERT_EQ(num, "0"); + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.compaction-pending", &num)); + ASSERT_EQ(num, "1"); + ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num)); + ASSERT_EQ(num, "4"); + + ASSERT_TRUE( + dbfull()->GetIntProperty("rocksdb.estimate-table-readers-mem", &int_num)); + ASSERT_GT(int_num, 0U); + + sleeping_task_low.WakeUp(); + sleeping_task_low.WaitUntilDone(); + + // Wait for compaction to be done. This is important because otherwise RocksDB + // might schedule a compaction when reopening the database, failing assertion + // (A) as a result. + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + options.max_open_files = 10; + Reopen(options); + // After reopening, no table reader is loaded, so no memory for table readers + ASSERT_TRUE( + dbfull()->GetIntProperty("rocksdb.estimate-table-readers-mem", &int_num)); + ASSERT_EQ(int_num, 0U); // (A) + ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.estimate-num-keys", &int_num)); + ASSERT_GT(int_num, 0U); + + // After reading a key, at least one table reader is loaded. + Get("k5"); + ASSERT_TRUE( + dbfull()->GetIntProperty("rocksdb.estimate-table-readers-mem", &int_num)); + ASSERT_GT(int_num, 0U); + + // Test rocksdb.num-live-versions + { + options.level0_file_num_compaction_trigger = 20; + Reopen(options); + ASSERT_TRUE( + dbfull()->GetIntProperty("rocksdb.num-live-versions", &int_num)); + ASSERT_EQ(int_num, 1U); + + // Use an iterator to hold current version + std::unique_ptr iter1(dbfull()->NewIterator(ReadOptions())); + + ASSERT_OK(dbfull()->Put(writeOpt, "k6", big_value)); + ASSERT_OK(Flush()); + ASSERT_TRUE( + dbfull()->GetIntProperty("rocksdb.num-live-versions", &int_num)); + ASSERT_EQ(int_num, 2U); + + // Use an iterator to hold current version + std::unique_ptr iter2(dbfull()->NewIterator(ReadOptions())); + + ASSERT_OK(dbfull()->Put(writeOpt, "k7", big_value)); + ASSERT_OK(Flush()); + ASSERT_TRUE( + dbfull()->GetIntProperty("rocksdb.num-live-versions", &int_num)); + ASSERT_EQ(int_num, 3U); + + iter2.reset(); + ASSERT_TRUE( + dbfull()->GetIntProperty("rocksdb.num-live-versions", &int_num)); + ASSERT_EQ(int_num, 2U); + + iter1.reset(); + ASSERT_TRUE( + dbfull()->GetIntProperty("rocksdb.num-live-versions", &int_num)); + ASSERT_EQ(int_num, 1U); + } +} + +TEST_F(DBPropertiesTest, ApproximateMemoryUsage) { + const int kNumRounds = 10; + // TODO(noetzli) kFlushesPerRound does not really correlate with how many + // flushes happen. + const int kFlushesPerRound = 10; + const int kWritesPerFlush = 10; + const int kKeySize = 100; + const int kValueSize = 1000; + Options options; + options.write_buffer_size = 1000; // small write buffer + options.min_write_buffer_number_to_merge = 4; + options.compression = kNoCompression; + options.create_if_missing = true; + options = CurrentOptions(options); + DestroyAndReopen(options); + + Random rnd(301); + + std::vector iters; + + uint64_t active_mem; + uint64_t unflushed_mem; + uint64_t all_mem; + uint64_t prev_all_mem; + + // Phase 0. The verify the initial value of all these properties are the same + // as we have no mem-tables. + dbfull()->GetIntProperty("rocksdb.cur-size-active-mem-table", &active_mem); + dbfull()->GetIntProperty("rocksdb.cur-size-all-mem-tables", &unflushed_mem); + dbfull()->GetIntProperty("rocksdb.size-all-mem-tables", &all_mem); + ASSERT_EQ(all_mem, active_mem); + ASSERT_EQ(all_mem, unflushed_mem); + + // Phase 1. Simply issue Put() and expect "cur-size-all-mem-tables" equals to + // "size-all-mem-tables" + for (int r = 0; r < kNumRounds; ++r) { + for (int f = 0; f < kFlushesPerRound; ++f) { + for (int w = 0; w < kWritesPerFlush; ++w) { + ASSERT_OK( + Put(rnd.RandomString(kKeySize), rnd.RandomString(kValueSize))); + } + } + // Make sure that there is no flush between getting the two properties. + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + dbfull()->GetIntProperty("rocksdb.cur-size-all-mem-tables", &unflushed_mem); + dbfull()->GetIntProperty("rocksdb.size-all-mem-tables", &all_mem); + // in no iterator case, these two number should be the same. + ASSERT_EQ(unflushed_mem, all_mem); + } + prev_all_mem = all_mem; + + // Phase 2. Keep issuing Put() but also create new iterators. This time we + // expect "size-all-mem-tables" > "cur-size-all-mem-tables". + for (int r = 0; r < kNumRounds; ++r) { + iters.push_back(db_->NewIterator(ReadOptions())); + for (int f = 0; f < kFlushesPerRound; ++f) { + for (int w = 0; w < kWritesPerFlush; ++w) { + ASSERT_OK( + Put(rnd.RandomString(kKeySize), rnd.RandomString(kValueSize))); + } + } + // Force flush to prevent flush from happening between getting the + // properties or after getting the properties and before the new round. + ASSERT_OK(Flush()); + + // In the second round, add iterators. + dbfull()->GetIntProperty("rocksdb.cur-size-active-mem-table", &active_mem); + dbfull()->GetIntProperty("rocksdb.cur-size-all-mem-tables", &unflushed_mem); + dbfull()->GetIntProperty("rocksdb.size-all-mem-tables", &all_mem); + ASSERT_GT(all_mem, active_mem); + ASSERT_GT(all_mem, unflushed_mem); + ASSERT_GT(all_mem, prev_all_mem); + prev_all_mem = all_mem; + } + + // Phase 3. Delete iterators and expect "size-all-mem-tables" shrinks + // whenever we release an iterator. + for (auto* iter : iters) { + ASSERT_OK(iter->status()); + delete iter; + dbfull()->GetIntProperty("rocksdb.size-all-mem-tables", &all_mem); + // Expect the size shrinking + ASSERT_LT(all_mem, prev_all_mem); + prev_all_mem = all_mem; + } + + // Expect all these three counters to be the same. + dbfull()->GetIntProperty("rocksdb.cur-size-active-mem-table", &active_mem); + dbfull()->GetIntProperty("rocksdb.cur-size-all-mem-tables", &unflushed_mem); + dbfull()->GetIntProperty("rocksdb.size-all-mem-tables", &all_mem); + ASSERT_EQ(active_mem, unflushed_mem); + ASSERT_EQ(unflushed_mem, all_mem); + + // Phase 5. Reopen, and expect all these three counters to be the same again. + Reopen(options); + dbfull()->GetIntProperty("rocksdb.cur-size-active-mem-table", &active_mem); + dbfull()->GetIntProperty("rocksdb.cur-size-all-mem-tables", &unflushed_mem); + dbfull()->GetIntProperty("rocksdb.size-all-mem-tables", &all_mem); + ASSERT_EQ(active_mem, unflushed_mem); + ASSERT_EQ(unflushed_mem, all_mem); +} + +TEST_F(DBPropertiesTest, EstimatePendingCompBytes) { + // Set sizes to both background thread pool to be 1 and block them. + env_->SetBackgroundThreads(1, Env::HIGH); + env_->SetBackgroundThreads(1, Env::LOW); + test::SleepingBackgroundTask sleeping_task_low; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::LOW); + + Options options = CurrentOptions(); + WriteOptions writeOpt = WriteOptions(); + writeOpt.disableWAL = true; + options.compaction_style = kCompactionStyleLevel; + options.level0_file_num_compaction_trigger = 2; + options.max_background_compactions = 1; + options.max_background_flushes = 1; + options.max_write_buffer_number = 10; + options.min_write_buffer_number_to_merge = 1; + options.max_write_buffer_size_to_maintain = 0; + options.write_buffer_size = 1000000; + Reopen(options); + + std::string big_value(1000000 * 2, 'x'); + std::string num; + uint64_t int_num; + + ASSERT_OK(dbfull()->Put(writeOpt, "k1", big_value)); + ASSERT_OK(Flush()); + ASSERT_TRUE(dbfull()->GetIntProperty( + "rocksdb.estimate-pending-compaction-bytes", &int_num)); + ASSERT_EQ(int_num, 0U); + + ASSERT_OK(dbfull()->Put(writeOpt, "k2", big_value)); + ASSERT_OK(Flush()); + ASSERT_TRUE(dbfull()->GetIntProperty( + "rocksdb.estimate-pending-compaction-bytes", &int_num)); + ASSERT_GT(int_num, 0U); + + ASSERT_OK(dbfull()->Put(writeOpt, "k3", big_value)); + ASSERT_OK(Flush()); + ASSERT_TRUE(dbfull()->GetIntProperty( + "rocksdb.estimate-pending-compaction-bytes", &int_num)); + ASSERT_GT(int_num, 0U); + + sleeping_task_low.WakeUp(); + sleeping_task_low.WaitUntilDone(); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_TRUE(dbfull()->GetIntProperty( + "rocksdb.estimate-pending-compaction-bytes", &int_num)); + ASSERT_EQ(int_num, 0U); +} + +TEST_F(DBPropertiesTest, EstimateCompressionRatio) { + if (!Snappy_Supported()) { + return; + } + const int kNumL0Files = 3; + const int kNumEntriesPerFile = 1000; + + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.num_levels = 3; + Reopen(options); + + ASSERT_OK(db_->SetOptions( + {{"compression_per_level", "kNoCompression:kSnappyCompression"}})); + auto opts = db_->GetOptions(); + ASSERT_EQ(opts.compression_per_level.size(), 2); + ASSERT_EQ(opts.compression_per_level[0], kNoCompression); + ASSERT_EQ(opts.compression_per_level[1], kSnappyCompression); + + // compression ratio is -1.0 when no open files at level + ASSERT_EQ(CompressionRatioAtLevel(0), -1.0); + + const std::string kVal(100, 'a'); + for (int i = 0; i < kNumL0Files; ++i) { + for (int j = 0; j < kNumEntriesPerFile; ++j) { + // Put common data ("key") at end to prevent delta encoding from + // compressing the key effectively + std::string key = std::to_string(i) + std::to_string(j) + "key"; + ASSERT_OK(dbfull()->Put(WriteOptions(), key, kVal)); + } + ASSERT_OK(Flush()); + } + + // no compression at L0, so ratio is less than one + ASSERT_LT(CompressionRatioAtLevel(0), 1.0); + ASSERT_GT(CompressionRatioAtLevel(0), 0.0); + ASSERT_EQ(CompressionRatioAtLevel(1), -1.0); + + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr)); + + ASSERT_EQ(CompressionRatioAtLevel(0), -1.0); + // Data at L1 should be highly compressed thanks to Snappy and redundant data + // in values (ratio is 12.846 as of 4/19/2016). + ASSERT_GT(CompressionRatioAtLevel(1), 10.0); +} + + +class CountingUserTblPropCollector : public TablePropertiesCollector { + public: + const char* Name() const override { return "CountingUserTblPropCollector"; } + + Status Finish(UserCollectedProperties* properties) override { + std::string encoded; + PutVarint32(&encoded, count_); + *properties = UserCollectedProperties{ + {"CountingUserTblPropCollector", message_}, + {"Count", encoded}, + }; + return Status::OK(); + } + + Status AddUserKey(const Slice& /*user_key*/, const Slice& /*value*/, + EntryType /*type*/, SequenceNumber /*seq*/, + uint64_t /*file_size*/) override { + ++count_; + return Status::OK(); + } + + UserCollectedProperties GetReadableProperties() const override { + return UserCollectedProperties{}; + } + + private: + std::string message_ = "Rocksdb"; + uint32_t count_ = 0; +}; + +class CountingUserTblPropCollectorFactory + : public TablePropertiesCollectorFactory { + public: + explicit CountingUserTblPropCollectorFactory( + uint32_t expected_column_family_id) + : expected_column_family_id_(expected_column_family_id), + num_created_(0) {} + TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context context) override { + EXPECT_EQ(expected_column_family_id_, context.column_family_id); + num_created_++; + return new CountingUserTblPropCollector(); + } + const char* Name() const override { + return "CountingUserTblPropCollectorFactory"; + } + void set_expected_column_family_id(uint32_t v) { + expected_column_family_id_ = v; + } + uint32_t expected_column_family_id_; + uint32_t num_created_; +}; + +class CountingDeleteTabPropCollector : public TablePropertiesCollector { + public: + const char* Name() const override { return "CountingDeleteTabPropCollector"; } + + Status AddUserKey(const Slice& /*user_key*/, const Slice& /*value*/, + EntryType type, SequenceNumber /*seq*/, + uint64_t /*file_size*/) override { + if (type == kEntryDelete) { + num_deletes_++; + } + return Status::OK(); + } + + bool NeedCompact() const override { return num_deletes_ > 10; } + + UserCollectedProperties GetReadableProperties() const override { + return UserCollectedProperties{}; + } + + Status Finish(UserCollectedProperties* properties) override { + *properties = + UserCollectedProperties{{"num_delete", std::to_string(num_deletes_)}}; + return Status::OK(); + } + + private: + uint32_t num_deletes_ = 0; +}; + +class CountingDeleteTabPropCollectorFactory + : public TablePropertiesCollectorFactory { + public: + TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context /*context*/) override { + return new CountingDeleteTabPropCollector(); + } + const char* Name() const override { + return "CountingDeleteTabPropCollectorFactory"; + } +}; + +class BlockCountingTablePropertiesCollector : public TablePropertiesCollector { + public: + static const std::string kNumSampledBlocksPropertyName; + + const char* Name() const override { + return "BlockCountingTablePropertiesCollector"; + } + + Status Finish(UserCollectedProperties* properties) override { + (*properties)[kNumSampledBlocksPropertyName] = + std::to_string(num_sampled_blocks_); + return Status::OK(); + } + + Status AddUserKey(const Slice& /*user_key*/, const Slice& /*value*/, + EntryType /*type*/, SequenceNumber /*seq*/, + uint64_t /*file_size*/) override { + return Status::OK(); + } + + void BlockAdd(uint64_t /* block_uncomp_bytes */, + uint64_t block_compressed_bytes_fast, + uint64_t block_compressed_bytes_slow) override { + if (block_compressed_bytes_fast > 0 || block_compressed_bytes_slow > 0) { + num_sampled_blocks_++; + } + } + + UserCollectedProperties GetReadableProperties() const override { + return UserCollectedProperties{ + {kNumSampledBlocksPropertyName, std::to_string(num_sampled_blocks_)}, + }; + } + + private: + uint32_t num_sampled_blocks_ = 0; +}; + +const std::string + BlockCountingTablePropertiesCollector::kNumSampledBlocksPropertyName = + "NumSampledBlocks"; + +class BlockCountingTablePropertiesCollectorFactory + : public TablePropertiesCollectorFactory { + public: + const char* Name() const override { + return "BlockCountingTablePropertiesCollectorFactory"; + } + + TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context /* context */) override { + return new BlockCountingTablePropertiesCollector(); + } +}; + +TEST_F(DBPropertiesTest, GetUserDefinedTableProperties) { + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = (1 << 30); + options.table_properties_collector_factories.resize(1); + std::shared_ptr collector_factory = + std::make_shared(0); + options.table_properties_collector_factories[0] = collector_factory; + Reopen(options); + // Create 4 tables + for (int table = 0; table < 4; ++table) { + for (int i = 0; i < 10 + table; ++i) { + ASSERT_OK( + db_->Put(WriteOptions(), std::to_string(table * 100 + i), "val")); + } + ASSERT_OK(db_->Flush(FlushOptions())); + } + + TablePropertiesCollection props; + ASSERT_OK(db_->GetPropertiesOfAllTables(&props)); + ASSERT_EQ(4U, props.size()); + uint32_t sum = 0; + for (const auto& item : props) { + auto& user_collected = item.second->user_collected_properties; + ASSERT_TRUE(user_collected.find("CountingUserTblPropCollector") != + user_collected.end()); + ASSERT_EQ(user_collected.at("CountingUserTblPropCollector"), "Rocksdb"); + ASSERT_TRUE(user_collected.find("Count") != user_collected.end()); + Slice key(user_collected.at("Count")); + uint32_t count; + ASSERT_TRUE(GetVarint32(&key, &count)); + sum += count; + } + ASSERT_EQ(10u + 11u + 12u + 13u, sum); + + ASSERT_GT(collector_factory->num_created_, 0U); + collector_factory->num_created_ = 0; + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr)); + ASSERT_GT(collector_factory->num_created_, 0U); +} + +TEST_F(DBPropertiesTest, UserDefinedTablePropertiesContext) { + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = 3; + options.table_properties_collector_factories.resize(1); + std::shared_ptr collector_factory = + std::make_shared(1); + options.table_properties_collector_factories[0] = collector_factory, + CreateAndReopenWithCF({"pikachu"}, options); + // Create 2 files + for (int table = 0; table < 2; ++table) { + for (int i = 0; i < 10 + table; ++i) { + ASSERT_OK(Put(1, std::to_string(table * 100 + i), "val")); + } + ASSERT_OK(Flush(1)); + } + ASSERT_GT(collector_factory->num_created_, 0U); + + collector_factory->num_created_ = 0; + // Trigger automatic compactions. + for (int table = 0; table < 3; ++table) { + for (int i = 0; i < 10 + table; ++i) { + ASSERT_OK(Put(1, std::to_string(table * 100 + i), "val")); + } + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + ASSERT_GT(collector_factory->num_created_, 0U); + + collector_factory->num_created_ = 0; + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1])); + ASSERT_GT(collector_factory->num_created_, 0U); + + // Come back to write to default column family + collector_factory->num_created_ = 0; + collector_factory->set_expected_column_family_id(0); // default CF + // Create 4 tables in default column family + for (int table = 0; table < 2; ++table) { + for (int i = 0; i < 10 + table; ++i) { + ASSERT_OK(Put(std::to_string(table * 100 + i), "val")); + } + ASSERT_OK(Flush()); + } + ASSERT_GT(collector_factory->num_created_, 0U); + + collector_factory->num_created_ = 0; + // Trigger automatic compactions. + for (int table = 0; table < 3; ++table) { + for (int i = 0; i < 10 + table; ++i) { + ASSERT_OK(Put(std::to_string(table * 100 + i), "val")); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + ASSERT_GT(collector_factory->num_created_, 0U); + + collector_factory->num_created_ = 0; + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr)); + ASSERT_GT(collector_factory->num_created_, 0U); +} + +TEST_F(DBPropertiesTest, TablePropertiesNeedCompactTest) { + Random rnd(301); + + Options options; + options.create_if_missing = true; + options.write_buffer_size = 4096; + options.max_write_buffer_number = 8; + options.level0_file_num_compaction_trigger = 2; + options.level0_slowdown_writes_trigger = 2; + options.level0_stop_writes_trigger = 4; + options.target_file_size_base = 2048; + options.max_bytes_for_level_base = 10240; + options.max_bytes_for_level_multiplier = 4; + options.soft_pending_compaction_bytes_limit = 1024 * 1024; + options.num_levels = 8; + options.env = env_; + + std::shared_ptr collector_factory = + std::make_shared(); + options.table_properties_collector_factories.resize(1); + options.table_properties_collector_factories[0] = collector_factory; + + DestroyAndReopen(options); + + const int kMaxKey = 1000; + for (int i = 0; i < kMaxKey; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(102))); + ASSERT_OK(Put(Key(kMaxKey + i), rnd.RandomString(102))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + if (NumTableFilesAtLevel(0) == 1) { + // Clear Level 0 so that when later flush a file with deletions, + // we don't trigger an organic compaction. + ASSERT_OK(Put(Key(0), "")); + ASSERT_OK(Put(Key(kMaxKey * 2), "")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + + { + int c = 0; + std::unique_ptr iter(db_->NewIterator(ReadOptions())); + iter->Seek(Key(kMaxKey - 100)); + while (iter->Valid() && iter->key().compare(Key(kMaxKey + 100)) < 0) { + iter->Next(); + ++c; + } + ASSERT_OK(iter->status()); + ASSERT_EQ(c, 200); + } + + ASSERT_OK(Delete(Key(0))); + for (int i = kMaxKey - 100; i < kMaxKey + 100; i++) { + ASSERT_OK(Delete(Key(i))); + } + ASSERT_OK(Delete(Key(kMaxKey * 2))); + + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + { + SetPerfLevel(kEnableCount); + get_perf_context()->Reset(); + int c = 0; + std::unique_ptr iter(db_->NewIterator(ReadOptions())); + iter->Seek(Key(kMaxKey - 100)); + while (iter->Valid() && iter->key().compare(Key(kMaxKey + 100)) < 0) { + iter->Next(); + } + ASSERT_OK(iter->status()); + ASSERT_EQ(c, 0); + ASSERT_LT(get_perf_context()->internal_delete_skipped_count, 30u); + ASSERT_LT(get_perf_context()->internal_key_skipped_count, 30u); + SetPerfLevel(kDisable); + } +} + +TEST_F(DBPropertiesTest, NeedCompactHintPersistentTest) { + Random rnd(301); + + Options options; + options.create_if_missing = true; + options.max_write_buffer_number = 8; + options.level0_file_num_compaction_trigger = 10; + options.level0_slowdown_writes_trigger = 10; + options.level0_stop_writes_trigger = 10; + options.disable_auto_compactions = true; + options.env = env_; + + std::shared_ptr collector_factory = + std::make_shared(); + options.table_properties_collector_factories.resize(1); + options.table_properties_collector_factories[0] = collector_factory; + + DestroyAndReopen(options); + + const int kMaxKey = 100; + for (int i = 0; i < kMaxKey; i++) { + ASSERT_OK(Put(Key(i), "")); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + + for (int i = 1; i < kMaxKey - 1; i++) { + ASSERT_OK(Delete(Key(i))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(NumTableFilesAtLevel(0), 2); + + // Restart the DB. Although number of files didn't reach + // options.level0_file_num_compaction_trigger, compaction should + // still be triggered because of the need-compaction hint. + options.disable_auto_compactions = false; + Reopen(options); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + { + SetPerfLevel(kEnableCount); + get_perf_context()->Reset(); + int c = 0; + std::unique_ptr iter(db_->NewIterator(ReadOptions())); + for (iter->Seek(Key(0)); iter->Valid(); iter->Next()) { + c++; + } + ASSERT_OK(iter->status()); + ASSERT_EQ(c, 2); + ASSERT_EQ(get_perf_context()->internal_delete_skipped_count, 0); + // We iterate every key twice. Is it a bug? + ASSERT_LE(get_perf_context()->internal_key_skipped_count, 2); + SetPerfLevel(kDisable); + } +} + +// Excluded from RocksDB lite tests due to `GetPropertiesOfAllTables()` usage. +TEST_F(DBPropertiesTest, BlockAddForCompressionSampling) { + // Sampled compression requires at least one of the following four types. + if (!Snappy_Supported() && !Zlib_Supported() && !LZ4_Supported() && + !ZSTD_Supported()) { + return; + } + + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.table_properties_collector_factories.emplace_back( + std::make_shared()); + + for (bool sample_for_compression : {false, true}) { + // For simplicity/determinism, sample 100% when enabled, or 0% when disabled + options.sample_for_compression = sample_for_compression ? 1 : 0; + + DestroyAndReopen(options); + + // Setup the following LSM: + // + // L0_0 ["a", "b"] + // L1_0 ["a", "b"] + // + // L0_0 was created by flush. L1_0 was created by compaction. Each file + // contains one data block. + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Put("a", "val")); + ASSERT_OK(Put("b", "val")); + ASSERT_OK(Flush()); + if (i == 1) { + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + } + } + + // A `BlockAdd()` should have been seen for files generated by flush or + // compaction when `sample_for_compression` is enabled. + TablePropertiesCollection file_to_props; + ASSERT_OK(db_->GetPropertiesOfAllTables(&file_to_props)); + ASSERT_EQ(2, file_to_props.size()); + for (const auto& file_and_props : file_to_props) { + auto& user_props = file_and_props.second->user_collected_properties; + ASSERT_TRUE(user_props.find(BlockCountingTablePropertiesCollector:: + kNumSampledBlocksPropertyName) != + user_props.end()); + ASSERT_EQ(user_props.at(BlockCountingTablePropertiesCollector:: + kNumSampledBlocksPropertyName), + std::to_string(sample_for_compression ? 1 : 0)); + } + } +} + +class CompressionSamplingDBPropertiesTest + : public DBPropertiesTest, + public ::testing::WithParamInterface { + public: + CompressionSamplingDBPropertiesTest() : fast_(GetParam()) {} + + protected: + const bool fast_; +}; + +INSTANTIATE_TEST_CASE_P(CompressionSamplingDBPropertiesTest, + CompressionSamplingDBPropertiesTest, ::testing::Bool()); + +// Excluded from RocksDB lite tests due to `GetPropertiesOfAllTables()` usage. +TEST_P(CompressionSamplingDBPropertiesTest, + EstimateDataSizeWithCompressionSampling) { + Options options = CurrentOptions(); + if (fast_) { + // One of the following light compression libraries must be present. + if (LZ4_Supported()) { + options.compression = kLZ4Compression; + } else if (Snappy_Supported()) { + options.compression = kSnappyCompression; + } else { + return; + } + } else { + // One of the following heavy compression libraries must be present. + if (ZSTD_Supported()) { + options.compression = kZSTD; + } else if (Zlib_Supported()) { + options.compression = kZlibCompression; + } else { + return; + } + } + options.disable_auto_compactions = true; + // For simplicity/determinism, sample 100%. + options.sample_for_compression = 1; + Reopen(options); + + // Setup the following LSM: + // + // L0_0 ["a", "b"] + // L1_0 ["a", "b"] + // + // L0_0 was created by flush. L1_0 was created by compaction. Each file + // contains one data block. The value consists of compressible data so the + // data block should be stored compressed. + std::string val(1024, 'a'); + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Put("a", val)); + ASSERT_OK(Put("b", val)); + ASSERT_OK(Flush()); + if (i == 1) { + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + } + } + + TablePropertiesCollection file_to_props; + ASSERT_OK(db_->GetPropertiesOfAllTables(&file_to_props)); + ASSERT_EQ(2, file_to_props.size()); + for (const auto& file_and_props : file_to_props) { + ASSERT_GT(file_and_props.second->data_size, 0); + if (fast_) { + ASSERT_EQ(file_and_props.second->data_size, + file_and_props.second->fast_compression_estimated_data_size); + } else { + ASSERT_EQ(file_and_props.second->data_size, + file_and_props.second->slow_compression_estimated_data_size); + } + } +} + +TEST_F(DBPropertiesTest, EstimateNumKeysUnderflow) { + Options options = CurrentOptions(); + Reopen(options); + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Delete("foo")); + ASSERT_OK(Delete("foo")); + uint64_t num_keys = 0; + ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.estimate-num-keys", &num_keys)); + ASSERT_EQ(0, num_keys); +} + +TEST_F(DBPropertiesTest, EstimateOldestKeyTime) { + uint64_t oldest_key_time = 0; + Options options = CurrentOptions(); + SetTimeElapseOnlySleepOnReopen(&options); + + // "rocksdb.estimate-oldest-key-time" only available to fifo compaction. + for (auto compaction : {kCompactionStyleLevel, kCompactionStyleUniversal, + kCompactionStyleNone}) { + options.compaction_style = compaction; + options.create_if_missing = true; + DestroyAndReopen(options); + ASSERT_OK(Put("foo", "bar")); + ASSERT_FALSE(dbfull()->GetIntProperty( + DB::Properties::kEstimateOldestKeyTime, &oldest_key_time)); + } + + int64_t mock_start_time; + ASSERT_OK(env_->GetCurrentTime(&mock_start_time)); + + options.compaction_style = kCompactionStyleFIFO; + options.ttl = 300; + options.max_open_files = -1; + options.compaction_options_fifo.allow_compaction = false; + DestroyAndReopen(options); + + env_->MockSleepForSeconds(100); + ASSERT_OK(Put("k1", "v1")); + ASSERT_TRUE(dbfull()->GetIntProperty(DB::Properties::kEstimateOldestKeyTime, + &oldest_key_time)); + ASSERT_EQ(100, oldest_key_time - mock_start_time); + ASSERT_OK(Flush()); + ASSERT_EQ("1", FilesPerLevel()); + ASSERT_TRUE(dbfull()->GetIntProperty(DB::Properties::kEstimateOldestKeyTime, + &oldest_key_time)); + ASSERT_EQ(100, oldest_key_time - mock_start_time); + + env_->MockSleepForSeconds(100); // -> 200 + ASSERT_OK(Put("k2", "v2")); + ASSERT_OK(Flush()); + ASSERT_EQ("2", FilesPerLevel()); + ASSERT_TRUE(dbfull()->GetIntProperty(DB::Properties::kEstimateOldestKeyTime, + &oldest_key_time)); + ASSERT_EQ(100, oldest_key_time - mock_start_time); + + env_->MockSleepForSeconds(100); // -> 300 + ASSERT_OK(Put("k3", "v3")); + ASSERT_OK(Flush()); + ASSERT_EQ("3", FilesPerLevel()); + ASSERT_TRUE(dbfull()->GetIntProperty(DB::Properties::kEstimateOldestKeyTime, + &oldest_key_time)); + ASSERT_EQ(100, oldest_key_time - mock_start_time); + + env_->MockSleepForSeconds(150); // -> 450 + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("2", FilesPerLevel()); + ASSERT_TRUE(dbfull()->GetIntProperty(DB::Properties::kEstimateOldestKeyTime, + &oldest_key_time)); + ASSERT_EQ(200, oldest_key_time - mock_start_time); + + env_->MockSleepForSeconds(100); // -> 550 + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("1", FilesPerLevel()); + ASSERT_TRUE(dbfull()->GetIntProperty(DB::Properties::kEstimateOldestKeyTime, + &oldest_key_time)); + ASSERT_EQ(300, oldest_key_time - mock_start_time); + + env_->MockSleepForSeconds(100); // -> 650 + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("", FilesPerLevel()); + ASSERT_FALSE(dbfull()->GetIntProperty(DB::Properties::kEstimateOldestKeyTime, + &oldest_key_time)); +} + +TEST_F(DBPropertiesTest, SstFilesSize) { + struct TestListener : public EventListener { + void OnCompactionCompleted(DB* db, + const CompactionJobInfo& /*info*/) override { + assert(callback_triggered == false); + assert(size_before_compaction > 0); + callback_triggered = true; + uint64_t total_sst_size = 0; + uint64_t live_sst_size = 0; + bool ok = db->GetIntProperty(DB::Properties::kTotalSstFilesSize, + &total_sst_size); + ASSERT_TRUE(ok); + // total_sst_size include files before and after compaction. + ASSERT_GT(total_sst_size, size_before_compaction); + ok = + db->GetIntProperty(DB::Properties::kLiveSstFilesSize, &live_sst_size); + ASSERT_TRUE(ok); + // live_sst_size only include files after compaction. + ASSERT_GT(live_sst_size, 0); + ASSERT_LT(live_sst_size, size_before_compaction); + } + + uint64_t size_before_compaction = 0; + bool callback_triggered = false; + }; + std::shared_ptr listener = std::make_shared(); + + Options options; + options.env = CurrentOptions().env; + options.disable_auto_compactions = true; + options.listeners.push_back(listener); + Reopen(options); + + for (int i = 0; i < 10; i++) { + ASSERT_OK(Put("key" + std::to_string(i), std::string(1000, 'v'))); + } + ASSERT_OK(Flush()); + for (int i = 0; i < 5; i++) { + ASSERT_OK(Delete("key" + std::to_string(i))); + } + ASSERT_OK(Flush()); + + uint64_t sst_size; + ASSERT_TRUE( + db_->GetIntProperty(DB::Properties::kTotalSstFilesSize, &sst_size)); + ASSERT_GT(sst_size, 0); + listener->size_before_compaction = sst_size; + + uint64_t obsolete_sst_size; + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kObsoleteSstFilesSize, + &obsolete_sst_size)); + ASSERT_EQ(obsolete_sst_size, 0); + + // Hold files from being deleted so we can test property for size of obsolete + // SST files. + ASSERT_OK(db_->DisableFileDeletions()); + + // Compact to clean all keys and trigger listener. + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_TRUE(listener->callback_triggered); + + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kObsoleteSstFilesSize, + &obsolete_sst_size)); + ASSERT_EQ(obsolete_sst_size, sst_size); + + // Let the obsolete files be deleted. + ASSERT_OK(db_->EnableFileDeletions()); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kObsoleteSstFilesSize, + &obsolete_sst_size)); + ASSERT_EQ(obsolete_sst_size, 0); +} + +TEST_F(DBPropertiesTest, MinObsoleteSstNumberToKeep) { + class TestListener : public EventListener { + public: + void OnTableFileCreated(const TableFileCreationInfo& info) override { + if (info.reason == TableFileCreationReason::kCompaction) { + // Verify the property indicates that SSTs created by a running + // compaction cannot be deleted. + uint64_t created_file_num; + FileType created_file_type; + std::string filename = + info.file_path.substr(info.file_path.rfind('/') + 1); + ASSERT_TRUE( + ParseFileName(filename, &created_file_num, &created_file_type)); + ASSERT_EQ(kTableFile, created_file_type); + + uint64_t keep_sst_lower_bound; + ASSERT_TRUE( + db_->GetIntProperty(DB::Properties::kMinObsoleteSstNumberToKeep, + &keep_sst_lower_bound)); + + ASSERT_LE(keep_sst_lower_bound, created_file_num); + validated_ = true; + } + } + + void SetDB(DB* db) { db_ = db; } + + int GetNumCompactions() { return num_compactions_; } + + // True if we've verified the property for at least one output file + bool Validated() { return validated_; } + + private: + int num_compactions_ = 0; + bool validated_ = false; + DB* db_ = nullptr; + }; + + const int kNumL0Files = 4; + + std::shared_ptr listener = std::make_shared(); + + Options options = CurrentOptions(); + options.listeners.push_back(listener); + options.level0_file_num_compaction_trigger = kNumL0Files; + DestroyAndReopen(options); + listener->SetDB(db_); + + for (int i = 0; i < kNumL0Files; ++i) { + // Make sure they overlap in keyspace to prevent trivial move + ASSERT_OK(Put("key1", "val")); + ASSERT_OK(Put("key2", "val")); + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_TRUE(listener->Validated()); +} + +TEST_F(DBPropertiesTest, BlobCacheProperties) { + Options options; + uint64_t value; + + options.env = CurrentOptions().env; + + // Test with empty blob cache. + constexpr size_t kCapacity = 100; + LRUCacheOptions co; + co.capacity = kCapacity; + co.num_shard_bits = 0; + co.metadata_charge_policy = kDontChargeCacheMetadata; + auto blob_cache = NewLRUCache(co); + options.blob_cache = blob_cache; + + Reopen(options); + + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlobCacheCapacity, &value)); + ASSERT_EQ(kCapacity, value); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlobCacheUsage, &value)); + ASSERT_EQ(0, value); + ASSERT_TRUE( + db_->GetIntProperty(DB::Properties::kBlobCachePinnedUsage, &value)); + ASSERT_EQ(0, value); + + // Insert unpinned blob to the cache and check size. + constexpr size_t kSize1 = 70; + ASSERT_OK(blob_cache->Insert("blob1", nullptr /*value*/, + &kNoopCacheItemHelper, kSize1)); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlobCacheCapacity, &value)); + ASSERT_EQ(kCapacity, value); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlobCacheUsage, &value)); + ASSERT_EQ(kSize1, value); + ASSERT_TRUE( + db_->GetIntProperty(DB::Properties::kBlobCachePinnedUsage, &value)); + ASSERT_EQ(0, value); + + // Insert pinned blob to the cache and check size. + constexpr size_t kSize2 = 60; + Cache::Handle* blob2 = nullptr; + ASSERT_OK(blob_cache->Insert("blob2", nullptr /*value*/, + &kNoopCacheItemHelper, kSize2, &blob2)); + ASSERT_NE(nullptr, blob2); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlobCacheCapacity, &value)); + ASSERT_EQ(kCapacity, value); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlobCacheUsage, &value)); + // blob1 is evicted. + ASSERT_EQ(kSize2, value); + ASSERT_TRUE( + db_->GetIntProperty(DB::Properties::kBlobCachePinnedUsage, &value)); + ASSERT_EQ(kSize2, value); + + // Insert another pinned blob to make the cache over-sized. + constexpr size_t kSize3 = 80; + Cache::Handle* blob3 = nullptr; + ASSERT_OK(blob_cache->Insert("blob3", nullptr /*value*/, + &kNoopCacheItemHelper, kSize3, &blob3)); + ASSERT_NE(nullptr, blob3); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlobCacheCapacity, &value)); + ASSERT_EQ(kCapacity, value); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlobCacheUsage, &value)); + ASSERT_EQ(kSize2 + kSize3, value); + ASSERT_TRUE( + db_->GetIntProperty(DB::Properties::kBlobCachePinnedUsage, &value)); + ASSERT_EQ(kSize2 + kSize3, value); + + // Check size after release. + blob_cache->Release(blob2); + blob_cache->Release(blob3); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlobCacheCapacity, &value)); + ASSERT_EQ(kCapacity, value); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlobCacheUsage, &value)); + // blob2 will be evicted, while blob3 remain in cache after release. + ASSERT_EQ(kSize3, value); + ASSERT_TRUE( + db_->GetIntProperty(DB::Properties::kBlobCachePinnedUsage, &value)); + ASSERT_EQ(0, value); +} + +TEST_F(DBPropertiesTest, BlockCacheProperties) { + Options options; + uint64_t value; + + options.env = CurrentOptions().env; + + // Block cache properties are not available for tables other than + // block-based table. + options.table_factory.reset(NewPlainTableFactory()); + Reopen(options); + ASSERT_FALSE( + db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value)); + ASSERT_FALSE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value)); + ASSERT_FALSE( + db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value)); + + options.table_factory.reset(NewCuckooTableFactory()); + Reopen(options); + ASSERT_FALSE( + db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value)); + ASSERT_FALSE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value)); + ASSERT_FALSE( + db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value)); + + // Block cache properties are not available if block cache is not used. + BlockBasedTableOptions table_options; + table_options.no_block_cache = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + ASSERT_FALSE( + db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value)); + ASSERT_FALSE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value)); + ASSERT_FALSE( + db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value)); + + // Test with empty block cache. + constexpr size_t kCapacity = 100; + LRUCacheOptions co; + co.capacity = kCapacity; + co.num_shard_bits = 0; + co.metadata_charge_policy = kDontChargeCacheMetadata; + auto block_cache = NewLRUCache(co); + table_options.block_cache = block_cache; + table_options.no_block_cache = false; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value)); + ASSERT_EQ(kCapacity, value); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value)); + ASSERT_EQ(0, value); + ASSERT_TRUE( + db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value)); + ASSERT_EQ(0, value); + + // Insert unpinned item to the cache and check size. + constexpr size_t kSize1 = 50; + ASSERT_OK(block_cache->Insert("item1", nullptr /*value*/, + &kNoopCacheItemHelper, kSize1)); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value)); + ASSERT_EQ(kCapacity, value); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value)); + ASSERT_EQ(kSize1, value); + ASSERT_TRUE( + db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value)); + ASSERT_EQ(0, value); + + // Insert pinned item to the cache and check size. + constexpr size_t kSize2 = 30; + Cache::Handle* item2 = nullptr; + ASSERT_OK(block_cache->Insert("item2", nullptr /*value*/, + &kNoopCacheItemHelper, kSize2, &item2)); + ASSERT_NE(nullptr, item2); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value)); + ASSERT_EQ(kCapacity, value); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value)); + ASSERT_EQ(kSize1 + kSize2, value); + ASSERT_TRUE( + db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value)); + ASSERT_EQ(kSize2, value); + + // Insert another pinned item to make the cache over-sized. + constexpr size_t kSize3 = 80; + Cache::Handle* item3 = nullptr; + ASSERT_OK(block_cache->Insert("item3", nullptr /*value*/, + &kNoopCacheItemHelper, kSize3, &item3)); + ASSERT_NE(nullptr, item2); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value)); + ASSERT_EQ(kCapacity, value); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value)); + // Item 1 is evicted. + ASSERT_EQ(kSize2 + kSize3, value); + ASSERT_TRUE( + db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value)); + ASSERT_EQ(kSize2 + kSize3, value); + + // Check size after release. + block_cache->Release(item2); + block_cache->Release(item3); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value)); + ASSERT_EQ(kCapacity, value); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value)); + // item2 will be evicted, while item3 remain in cache after release. + ASSERT_EQ(kSize3, value); + ASSERT_TRUE( + db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value)); + ASSERT_EQ(0, value); +} + +TEST_F(DBPropertiesTest, GetMapPropertyDbStats) { + auto mock_clock = std::make_shared(env_->GetSystemClock()); + CompositeEnvWrapper env(env_, mock_clock); + + Options opts = CurrentOptions(); + opts.env = &env; + Reopen(opts); + + { + std::map db_stats; + ASSERT_TRUE(db_->GetMapProperty(DB::Properties::kDBStats, &db_stats)); + AssertDbStats(db_stats, 0.0 /* expected_uptime */, + 0 /* expected_user_bytes_written */, + 0 /* expected_wal_bytes_written */, + 0 /* expected_user_writes_by_self */, + 0 /* expected_user_writes_with_wal */); + } + + { + mock_clock->SleepForMicroseconds(1500000); + + std::map db_stats; + ASSERT_TRUE(db_->GetMapProperty(DB::Properties::kDBStats, &db_stats)); + AssertDbStats(db_stats, 1.5 /* expected_uptime */, + 0 /* expected_user_bytes_written */, + 0 /* expected_wal_bytes_written */, + 0 /* expected_user_writes_by_self */, + 0 /* expected_user_writes_with_wal */); + } + + int expected_user_bytes_written = 0; + { + // Write with WAL disabled. + WriteOptions write_opts; + write_opts.disableWAL = true; + + WriteBatch batch; + ASSERT_OK(batch.Put("key", "val")); + expected_user_bytes_written += static_cast(batch.GetDataSize()); + + ASSERT_OK(db_->Write(write_opts, &batch)); + + std::map db_stats; + ASSERT_TRUE(db_->GetMapProperty(DB::Properties::kDBStats, &db_stats)); + AssertDbStats(db_stats, 1.5 /* expected_uptime */, + expected_user_bytes_written, + 0 /* expected_wal_bytes_written */, + 1 /* expected_user_writes_by_self */, + 0 /* expected_user_writes_with_wal */); + } + + int expected_wal_bytes_written = 0; + { + // Write with WAL enabled. + WriteBatch batch; + ASSERT_OK(batch.Delete("key")); + expected_user_bytes_written += static_cast(batch.GetDataSize()); + expected_wal_bytes_written += static_cast(batch.GetDataSize()); + + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + + std::map db_stats; + ASSERT_TRUE(db_->GetMapProperty(DB::Properties::kDBStats, &db_stats)); + AssertDbStats(db_stats, 1.5 /* expected_uptime */, + expected_user_bytes_written, expected_wal_bytes_written, + 2 /* expected_user_writes_by_self */, + 1 /* expected_user_writes_with_wal */); + } + + Close(); +} + +TEST_F(DBPropertiesTest, GetMapPropertyBlockCacheEntryStats) { + // Currently only verifies the expected properties are present + std::map values; + ASSERT_TRUE( + db_->GetMapProperty(DB::Properties::kBlockCacheEntryStats, &values)); + + ASSERT_TRUE(values.find(BlockCacheEntryStatsMapKeys::CacheId()) != + values.end()); + ASSERT_TRUE(values.find(BlockCacheEntryStatsMapKeys::CacheCapacityBytes()) != + values.end()); + ASSERT_TRUE( + values.find( + BlockCacheEntryStatsMapKeys::LastCollectionDurationSeconds()) != + values.end()); + ASSERT_TRUE( + values.find(BlockCacheEntryStatsMapKeys::LastCollectionAgeSeconds()) != + values.end()); + for (size_t i = 0; i < kNumCacheEntryRoles; ++i) { + CacheEntryRole role = static_cast(i); + ASSERT_TRUE(values.find(BlockCacheEntryStatsMapKeys::EntryCount(role)) != + values.end()); + ASSERT_TRUE(values.find(BlockCacheEntryStatsMapKeys::UsedBytes(role)) != + values.end()); + ASSERT_TRUE(values.find(BlockCacheEntryStatsMapKeys::UsedPercent(role)) != + values.end()); + } + + // There should be no extra values in the map. + ASSERT_EQ(3 * kNumCacheEntryRoles + 4, values.size()); +} + +TEST_F(DBPropertiesTest, WriteStallStatsSanityCheck) { + for (uint32_t i = 0; i < static_cast(WriteStallCause::kNone); ++i) { + WriteStallCause cause = static_cast(i); + const std::string& str = WriteStallCauseToHyphenString(cause); + ASSERT_TRUE(!str.empty()) + << "Please ensure mapping from `WriteStallCause` to " + "`WriteStallCauseToHyphenString` is complete"; + if (cause == WriteStallCause::kCFScopeWriteStallCauseEnumMax || + cause == WriteStallCause::kDBScopeWriteStallCauseEnumMax) { + ASSERT_EQ(str, InvalidWriteStallHyphenString()) + << "Please ensure order in `WriteStallCauseToHyphenString` is " + "consistent with `WriteStallCause`"; + } + } + + for (uint32_t i = 0; i < static_cast(WriteStallCondition::kNormal); + ++i) { + WriteStallCondition condition = static_cast(i); + const std::string& str = WriteStallConditionToHyphenString(condition); + ASSERT_TRUE(!str.empty()) + << "Please ensure mapping from `WriteStallCondition` to " + "`WriteStallConditionToHyphenString` is complete"; + } + + for (uint32_t i = 0; i < static_cast(WriteStallCause::kNone); ++i) { + for (uint32_t j = 0; + j < static_cast(WriteStallCondition::kNormal); ++j) { + WriteStallCause cause = static_cast(i); + WriteStallCondition condition = static_cast(j); + + if (isCFScopeWriteStallCause(cause)) { + ASSERT_TRUE(InternalCFStat(cause, condition) != + InternalStats::INTERNAL_CF_STATS_ENUM_MAX) + << "Please ensure the combination of WriteStallCause(" + + std::to_string(static_cast(cause)) + + ") + WriteStallCondition(" + + std::to_string(static_cast(condition)) + + ") is correctly mapped to a valid `InternalStats` or bypass " + "its check in this test"; + } else if (isDBScopeWriteStallCause(cause)) { + InternalStats::InternalDBStatsType internal_db_stat = + InternalDBStat(cause, condition); + if (internal_db_stat == InternalStats::kIntStatsNumMax) { + ASSERT_TRUE(cause == WriteStallCause::kWriteBufferManagerLimit && + condition == WriteStallCondition::kDelayed) + << "Please ensure the combination of WriteStallCause(" + + std::to_string(static_cast(cause)) + + ") + WriteStallCondition(" + + std::to_string(static_cast(condition)) + + ") is correctly mapped to a valid `InternalStats` or " + "bypass its check in this test"; + } + } else if (cause != WriteStallCause::kCFScopeWriteStallCauseEnumMax && + cause != WriteStallCause::kDBScopeWriteStallCauseEnumMax) { + ASSERT_TRUE(false) << "Please ensure the WriteStallCause(" + + std::to_string(static_cast(cause)) + + ") is either CF-scope or DB-scope write " + "stall cause in enum `WriteStallCause`"; + } + } + } +} +TEST_F(DBPropertiesTest, GetMapPropertyWriteStallStats) { + Options options = CurrentOptions(); + CreateAndReopenWithCF({"heavy_write_cf"}, options); + + for (auto test_cause : {WriteStallCause::kWriteBufferManagerLimit, + WriteStallCause::kMemtableLimit}) { + if (test_cause == WriteStallCause::kWriteBufferManagerLimit) { + options.write_buffer_manager.reset( + new WriteBufferManager(100000, nullptr, true)); + } else if (test_cause == WriteStallCause::kMemtableLimit) { + options.max_write_buffer_number = 2; + options.disable_auto_compactions = true; + } + ReopenWithColumnFamilies({"default", "heavy_write_cf"}, options); + + // Assert initial write stall stats are all 0 + std::map db_values; + ASSERT_TRUE(dbfull()->GetMapProperty(DB::Properties::kDBWriteStallStats, + &db_values)); + ASSERT_EQ(std::stoi(db_values[WriteStallStatsMapKeys::CauseConditionCount( + WriteStallCause::kWriteBufferManagerLimit, + WriteStallCondition::kStopped)]), + 0); + + for (int cf = 0; cf <= 1; ++cf) { + std::map cf_values; + ASSERT_TRUE(dbfull()->GetMapProperty( + handles_[cf], DB::Properties::kCFWriteStallStats, &cf_values)); + ASSERT_EQ(std::stoi(cf_values[WriteStallStatsMapKeys::TotalStops()]), 0); + ASSERT_EQ(std::stoi(cf_values[WriteStallStatsMapKeys::TotalDelays()]), 0); + } + + // Pause flush thread to help coerce write stall + std::unique_ptr sleeping_task( + new test::SleepingBackgroundTask()); + env_->SetBackgroundThreads(1, Env::HIGH); + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, + sleeping_task.get(), Env::Priority::HIGH); + sleeping_task->WaitUntilSleeping(); + + // Coerce write stall + if (test_cause == WriteStallCause::kWriteBufferManagerLimit) { + ASSERT_OK(dbfull()->Put( + WriteOptions(), handles_[1], Key(1), + DummyString(options.write_buffer_manager->buffer_size()))); + + WriteOptions wo; + wo.no_slowdown = true; + Status s = dbfull()->Put( + wo, handles_[1], Key(2), + DummyString(options.write_buffer_manager->buffer_size())); + ASSERT_TRUE(s.IsIncomplete()); + ASSERT_TRUE(s.ToString().find("Write stall") != std::string::npos); + } else if (test_cause == WriteStallCause::kMemtableLimit) { + FlushOptions fo; + fo.allow_write_stall = true; + fo.wait = false; + + ASSERT_OK( + dbfull()->Put(WriteOptions(), handles_[1], Key(1), DummyString(1))); + ASSERT_OK(dbfull()->Flush(fo, handles_[1])); + + ASSERT_OK( + dbfull()->Put(WriteOptions(), handles_[1], Key(2), DummyString(1))); + ASSERT_OK(dbfull()->Flush(fo, handles_[1])); + } + + if (test_cause == WriteStallCause::kWriteBufferManagerLimit) { + db_values.clear(); + EXPECT_TRUE(dbfull()->GetMapProperty(DB::Properties::kDBWriteStallStats, + &db_values)); + EXPECT_EQ(std::stoi(db_values[WriteStallStatsMapKeys::CauseConditionCount( + WriteStallCause::kWriteBufferManagerLimit, + WriteStallCondition::kStopped)]), + 1); + // `WriteStallCause::kWriteBufferManagerLimit` should not result in any + // CF-scope write stall stats changes + for (int cf = 0; cf <= 1; ++cf) { + std::map cf_values; + EXPECT_TRUE(dbfull()->GetMapProperty( + handles_[cf], DB::Properties::kCFWriteStallStats, &cf_values)); + EXPECT_EQ(std::stoi(cf_values[WriteStallStatsMapKeys::TotalStops()]), + 0); + EXPECT_EQ(std::stoi(cf_values[WriteStallStatsMapKeys::TotalDelays()]), + 0); + } + } else if (test_cause == WriteStallCause::kMemtableLimit) { + for (int cf = 0; cf <= 1; ++cf) { + std::map cf_values; + EXPECT_TRUE(dbfull()->GetMapProperty( + handles_[cf], DB::Properties::kCFWriteStallStats, &cf_values)); + EXPECT_EQ(std::stoi(cf_values[WriteStallStatsMapKeys::TotalStops()]), + cf == 1 ? 1 : 0); + EXPECT_EQ( + std::stoi(cf_values[WriteStallStatsMapKeys::CauseConditionCount( + WriteStallCause::kMemtableLimit, + WriteStallCondition::kStopped)]), + cf == 1 ? 1 : 0); + EXPECT_EQ(std::stoi(cf_values[WriteStallStatsMapKeys::TotalDelays()]), + 0); + EXPECT_EQ( + std::stoi(cf_values[WriteStallStatsMapKeys::CauseConditionCount( + WriteStallCause::kMemtableLimit, + WriteStallCondition::kDelayed)]), + 0); + } + } + + sleeping_task->WakeUp(); + sleeping_task->WaitUntilDone(); + } +} + +namespace { +std::string PopMetaIndexKey(InternalIterator* meta_iter) { + Status s = meta_iter->status(); + if (!s.ok()) { + return s.ToString(); + } else if (meta_iter->Valid()) { + std::string rv = meta_iter->key().ToString(); + meta_iter->Next(); + return rv; + } else { + return "NOT_FOUND"; + } +} + +} // anonymous namespace + +TEST_F(DBPropertiesTest, TableMetaIndexKeys) { + // This is to detect unexpected churn in metaindex block keys. This is more + // of a "table test" but table_test.cc doesn't depend on db_test_util.h and + // we need ChangeOptions() for broad coverage. + constexpr int kKeyCount = 100; + do { + Options options; + options = CurrentOptions(options); + DestroyAndReopen(options); + + // Create an SST file + for (int key = 0; key < kKeyCount; key++) { + ASSERT_OK(Put(Key(key), "val")); + } + ASSERT_OK(Flush()); + + // Find its file number + std::vector files; + db_->GetLiveFilesMetaData(&files); + // 1 SST file + ASSERT_EQ(1, files.size()); + + // Open it for inspection + std::string sst_file = + files[0].directory + "/" + files[0].relative_filename; + std::unique_ptr f; + ASSERT_OK(env_->GetFileSystem()->NewRandomAccessFile( + sst_file, FileOptions(), &f, nullptr)); + std::unique_ptr r; + r.reset(new RandomAccessFileReader(std::move(f), sst_file)); + uint64_t file_size = 0; + ASSERT_OK(env_->GetFileSize(sst_file, &file_size)); + + // Read metaindex + BlockContents bc; + const ReadOptions read_options; + ASSERT_OK(ReadMetaIndexBlockInFile( + r.get(), file_size, 0U, ImmutableOptions(options), read_options, &bc)); + Block metaindex_block(std::move(bc)); + std::unique_ptr meta_iter; + meta_iter.reset(metaindex_block.NewMetaIterator()); + meta_iter->SeekToFirst(); + + if (strcmp(options.table_factory->Name(), + TableFactory::kBlockBasedTableName()) == 0) { + auto bbto = options.table_factory->GetOptions(); + if (bbto->filter_policy) { + if (bbto->partition_filters) { + // The key names are intentionally hard-coded here to detect + // accidental regression on compatibility. + EXPECT_EQ("partitionedfilter.rocksdb.BuiltinBloomFilter", + PopMetaIndexKey(meta_iter.get())); + } else { + EXPECT_EQ("fullfilter.rocksdb.BuiltinBloomFilter", + PopMetaIndexKey(meta_iter.get())); + } + } + if (bbto->index_type == BlockBasedTableOptions::kHashSearch) { + EXPECT_EQ("rocksdb.hashindex.metadata", + PopMetaIndexKey(meta_iter.get())); + EXPECT_EQ("rocksdb.hashindex.prefixes", + PopMetaIndexKey(meta_iter.get())); + } + if (bbto->format_version >= 6) { + EXPECT_EQ("rocksdb.index", PopMetaIndexKey(meta_iter.get())); + } + } + EXPECT_EQ("rocksdb.properties", PopMetaIndexKey(meta_iter.get())); + EXPECT_EQ("NOT_FOUND", PopMetaIndexKey(meta_iter.get())); + } while (ChangeOptions()); +} + + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_range_del_test.cc b/librocksdb-sys/rocksdb/db/db_range_del_test.cc new file mode 100644 index 0000000..bb75592 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_range_del_test.cc @@ -0,0 +1,3520 @@ +// Copyright (c) 2016-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/db_test_util.h" +#include "db/version_set.h" +#include "port/stack_trace.h" +#include "rocksdb/utilities/write_batch_with_index.h" +#include "test_util/testutil.h" +#include "util/random.h" +#include "utilities/merge_operators.h" + +namespace ROCKSDB_NAMESPACE { + +// TODO(cbi): parameterize the test to cover user-defined timestamp cases +class DBRangeDelTest : public DBTestBase { + public: + DBRangeDelTest() : DBTestBase("db_range_del_test", /*env_do_fsync=*/false) {} + + std::string GetNumericStr(int key) { + uint64_t uint64_key = static_cast(key); + std::string str; + str.resize(8); + memcpy(&str[0], static_cast(&uint64_key), 8); + return str; + } +}; + +TEST_F(DBRangeDelTest, NonBlockBasedTableNotSupported) { + // TODO: figure out why MmapReads trips the iterator pinning assertion in + // RangeDelAggregator. Ideally it would be supported; otherwise it should at + // least be explicitly unsupported. + for (auto config : {kPlainTableAllBytesPrefix, /* kWalDirAndMmapReads */}) { + option_config_ = config; + DestroyAndReopen(CurrentOptions()); + ASSERT_TRUE(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + "dr1", "dr1") + .IsNotSupported()); + } +} + +TEST_F(DBRangeDelTest, WriteBatchWithIndexNotSupported) { + WriteBatchWithIndex indexedBatch{}; + ASSERT_TRUE(indexedBatch.DeleteRange(db_->DefaultColumnFamily(), "dr1", "dr1") + .IsNotSupported()); + ASSERT_TRUE(indexedBatch.DeleteRange("dr1", "dr1").IsNotSupported()); +} + +TEST_F(DBRangeDelTest, EndSameAsStartCoversNothing) { + ASSERT_OK(db_->Put(WriteOptions(), "b", "val")); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "b", "b")); + ASSERT_EQ("val", Get("b")); +} + +TEST_F(DBRangeDelTest, EndComesBeforeStartInvalidArgument) { + ASSERT_OK(db_->Put(WriteOptions(), "b", "val")); + ASSERT_TRUE( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "b", "a") + .IsInvalidArgument()); + ASSERT_EQ("val", Get("b")); +} + +TEST_F(DBRangeDelTest, FlushOutputHasOnlyRangeTombstones) { + do { + DestroyAndReopen(CurrentOptions()); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + "dr1", "dr2")); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + } while (ChangeOptions(kRangeDelSkipConfigs)); +} + +TEST_F(DBRangeDelTest, DictionaryCompressionWithOnlyRangeTombstones) { + Options opts = CurrentOptions(); + opts.compression_opts.max_dict_bytes = 16384; + Reopen(opts); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "dr1", + "dr2")); + ASSERT_OK(db_->Flush(FlushOptions())); +} + +TEST_F(DBRangeDelTest, CompactionOutputHasOnlyRangeTombstone) { + do { + Options opts = CurrentOptions(); + opts.disable_auto_compactions = true; + opts.statistics = CreateDBStatistics(); + DestroyAndReopen(opts); + + // snapshot protects range tombstone from dropping due to becoming obsolete. + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "z")); + ASSERT_OK(db_->Flush(FlushOptions())); + + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + ASSERT_EQ(0, NumTableFilesAtLevel(1)); + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr, + true /* disallow_trivial_move */)); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + ASSERT_EQ(0, TestGetTickerCount(opts, COMPACTION_RANGE_DEL_DROP_OBSOLETE)); + db_->ReleaseSnapshot(snapshot); + // Skip cuckoo memtables, which do not support snapshots. Skip non-leveled + // compactions as the above assertions about the number of files in a level + // do not hold true. + } while (ChangeOptions(kRangeDelSkipConfigs | kSkipUniversalCompaction | + kSkipFIFOCompaction)); +} + +TEST_F(DBRangeDelTest, CompactionOutputFilesExactlyFilled) { + // regression test for exactly filled compaction output files. Previously + // another file would be generated containing all range deletions, which + // could invalidate the non-overlapping file boundary invariant. + const int kNumPerFile = 4, kNumFiles = 2, kFileBytes = 9 << 10; + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.level0_file_num_compaction_trigger = kNumFiles; + options.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile)); + options.num_levels = 2; + options.target_file_size_base = kFileBytes; + BlockBasedTableOptions table_options; + table_options.block_size_deviation = 50; // each block holds two keys + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + + // snapshot protects range tombstone from dropping due to becoming obsolete. + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0), + Key(1))); + + Random rnd(301); + for (int i = 0; i < kNumFiles; ++i) { + std::vector values; + // Write 12K (4 values, each 3K) + for (int j = 0; j < kNumPerFile; j++) { + values.push_back(rnd.RandomString(3 << 10)); + ASSERT_OK(Put(Key(i * kNumPerFile + j), values[j])); + if (j == 0 && i > 0) { + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + } + } + // put extra key to trigger final flush + ASSERT_OK(Put("", "")); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(kNumFiles, NumTableFilesAtLevel(0)); + ASSERT_EQ(0, NumTableFilesAtLevel(1)); + + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr, + true /* disallow_trivial_move */)); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_EQ(2, NumTableFilesAtLevel(1)); + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(DBRangeDelTest, MaxCompactionBytesCutsOutputFiles) { + // Ensures range deletion spanning multiple compaction output files that are + // cut by max_compaction_bytes will have non-overlapping key-ranges. + // https://github.com/facebook/rocksdb/issues/1778 + const int kNumFiles = 2, kNumPerFile = 1 << 8, kBytesPerVal = 1 << 12; + Options opts = CurrentOptions(); + opts.comparator = test::Uint64Comparator(); + opts.disable_auto_compactions = true; + opts.level0_file_num_compaction_trigger = kNumFiles; + opts.max_compaction_bytes = kNumPerFile * kBytesPerVal; + opts.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile)); + // Want max_compaction_bytes to trigger the end of compaction output file, not + // target_file_size_base, so make the latter much bigger + // opts.target_file_size_base = 100 * opts.max_compaction_bytes; + opts.target_file_size_base = 1; + DestroyAndReopen(opts); + + // snapshot protects range tombstone from dropping due to becoming obsolete. + const Snapshot* snapshot = db_->GetSnapshot(); + + Random rnd(301); + + ASSERT_OK(Put(GetNumericStr(0), rnd.RandomString(kBytesPerVal))); + ASSERT_OK( + Put(GetNumericStr(kNumPerFile - 1), rnd.RandomString(kBytesPerVal))); + ASSERT_OK(Flush()); + ASSERT_OK(Put(GetNumericStr(kNumPerFile), rnd.RandomString(kBytesPerVal))); + ASSERT_OK( + Put(GetNumericStr(kNumPerFile * 2 - 1), rnd.RandomString(kBytesPerVal))); + ASSERT_OK(Flush()); + MoveFilesToLevel(2); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_EQ(NumTableFilesAtLevel(2), 2); + + ASSERT_OK( + db_->SetOptions(db_->DefaultColumnFamily(), + {{"target_file_size_base", + std::to_string(100 * opts.max_compaction_bytes)}})); + + // It spans the whole key-range, thus will be included in all output files + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + GetNumericStr(0), + GetNumericStr(kNumFiles * kNumPerFile - 1))); + + for (int i = 0; i < kNumFiles; ++i) { + std::vector values; + // Write 1MB (256 values, each 4K) + for (int j = 0; j < kNumPerFile; j++) { + values.push_back(rnd.RandomString(kBytesPerVal)); + ASSERT_OK(Put(GetNumericStr(kNumPerFile * i + j), values[j])); + } + // extra entry to trigger SpecialSkipListFactory's flush + ASSERT_OK(Put(GetNumericStr(kNumPerFile), "")); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(i + 1, NumTableFilesAtLevel(0)); + } + + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, + /*column_family=*/nullptr, + /*disallow_trivial_move=*/true)); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_GE(NumTableFilesAtLevel(1), 2); + std::vector> files; + dbfull()->TEST_GetFilesMetaData(db_->DefaultColumnFamily(), &files); + + for (size_t i = 0; i + 1 < files[1].size(); ++i) { + ASSERT_TRUE(InternalKeyComparator(opts.comparator) + .Compare(files[1][i].largest, files[1][i + 1].smallest) < + 0); + } + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(DBRangeDelTest, SentinelsOmittedFromOutputFile) { + // Regression test for bug where sentinel range deletions (i.e., ones with + // sequence number of zero) were included in output files. + // snapshot protects range tombstone from dropping due to becoming obsolete. + const Snapshot* snapshot = db_->GetSnapshot(); + + // gaps between ranges creates sentinels in our internal representation + std::vector> range_dels = { + {"a", "b"}, {"c", "d"}, {"e", "f"}}; + for (const auto& range_del : range_dels) { + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + range_del.first, range_del.second)); + } + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + + std::vector> files; + dbfull()->TEST_GetFilesMetaData(db_->DefaultColumnFamily(), &files); + ASSERT_GT(files[0][0].fd.smallest_seqno, 0); + + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(DBRangeDelTest, FlushRangeDelsSameStartKey) { + ASSERT_OK(db_->Put(WriteOptions(), "b1", "val")); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "c")); + ASSERT_OK(db_->Put(WriteOptions(), "b2", "val")); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "b")); + // first iteration verifies query correctness in memtable, second verifies + // query correctness for a single SST file + for (int i = 0; i < 2; ++i) { + if (i > 0) { + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + } + std::string value; + ASSERT_TRUE(db_->Get(ReadOptions(), "b1", &value).IsNotFound()); + ASSERT_OK(db_->Get(ReadOptions(), "b2", &value)); + } +} + +TEST_F(DBRangeDelTest, CompactRangeDelsSameStartKey) { + ASSERT_OK(db_->Put(WriteOptions(), "unused", + "val")); // prevents empty after compaction + ASSERT_OK(db_->Put(WriteOptions(), "b1", "val")); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "c")); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "b")); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(3, NumTableFilesAtLevel(0)); + + for (int i = 0; i < 2; ++i) { + if (i > 0) { + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr, + true /* disallow_trivial_move */)); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + } + std::string value; + ASSERT_TRUE(db_->Get(ReadOptions(), "b1", &value).IsNotFound()); + } +} + +TEST_F(DBRangeDelTest, FlushRemovesCoveredKeys) { + const int kNum = 300, kRangeBegin = 50, kRangeEnd = 250; + Options opts = CurrentOptions(); + opts.comparator = test::Uint64Comparator(); + DestroyAndReopen(opts); + + // Write a third before snapshot, a third between snapshot and tombstone, and + // a third after the tombstone. Keys older than snapshot or newer than the + // tombstone should be preserved. + const Snapshot* snapshot = nullptr; + for (int i = 0; i < kNum; ++i) { + if (i == kNum / 3) { + snapshot = db_->GetSnapshot(); + } else if (i == 2 * kNum / 3) { + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + GetNumericStr(kRangeBegin), + GetNumericStr(kRangeEnd))); + } + ASSERT_OK(db_->Put(WriteOptions(), GetNumericStr(i), "val")); + } + ASSERT_OK(db_->Flush(FlushOptions())); + + for (int i = 0; i < kNum; ++i) { + ReadOptions read_opts; + read_opts.ignore_range_deletions = true; + std::string value; + if (i < kRangeBegin || i > kRangeEnd || i < kNum / 3 || i >= 2 * kNum / 3) { + ASSERT_OK(db_->Get(read_opts, GetNumericStr(i), &value)); + } else { + ASSERT_TRUE(db_->Get(read_opts, GetNumericStr(i), &value).IsNotFound()); + } + } + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(DBRangeDelTest, CompactionRemovesCoveredKeys) { + const int kNumPerFile = 100, kNumFiles = 4; + Options opts = CurrentOptions(); + opts.comparator = test::Uint64Comparator(); + opts.disable_auto_compactions = true; + opts.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile)); + opts.num_levels = 2; + opts.statistics = CreateDBStatistics(); + DestroyAndReopen(opts); + + for (int i = 0; i < kNumFiles; ++i) { + if (i > 0) { + // range tombstone covers first half of the previous file + ASSERT_OK(db_->DeleteRange( + WriteOptions(), db_->DefaultColumnFamily(), + GetNumericStr((i - 1) * kNumPerFile), + GetNumericStr((i - 1) * kNumPerFile + kNumPerFile / 2))); + } + // Make sure a given key appears in each file so compaction won't be able to + // use trivial move, which would happen if the ranges were non-overlapping. + // Also, we need an extra element since flush is only triggered when the + // number of keys is one greater than SpecialSkipListFactory's limit. + // We choose a key outside the key-range used by the test to avoid conflict. + ASSERT_OK(db_->Put(WriteOptions(), GetNumericStr(kNumPerFile * kNumFiles), + "val")); + + for (int j = 0; j < kNumPerFile; ++j) { + ASSERT_OK( + db_->Put(WriteOptions(), GetNumericStr(i * kNumPerFile + j), "val")); + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(i + 1, NumTableFilesAtLevel(0)); + } + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_GT(NumTableFilesAtLevel(1), 0); + ASSERT_EQ((kNumFiles - 1) * kNumPerFile / 2, + TestGetTickerCount(opts, COMPACTION_KEY_DROP_RANGE_DEL)); + + for (int i = 0; i < kNumFiles; ++i) { + for (int j = 0; j < kNumPerFile; ++j) { + ReadOptions read_opts; + read_opts.ignore_range_deletions = true; + std::string value; + if (i == kNumFiles - 1 || j >= kNumPerFile / 2) { + ASSERT_OK( + db_->Get(read_opts, GetNumericStr(i * kNumPerFile + j), &value)); + } else { + ASSERT_TRUE( + db_->Get(read_opts, GetNumericStr(i * kNumPerFile + j), &value) + .IsNotFound()); + } + } + } +} + +TEST_F(DBRangeDelTest, ValidLevelSubcompactionBoundaries) { + const int kNumPerFile = 100, kNumFiles = 4, kFileBytes = 100 << 10; + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.level0_file_num_compaction_trigger = kNumFiles; + options.max_bytes_for_level_base = 2 * kFileBytes; + options.max_subcompactions = 4; + options.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile)); + options.num_levels = 3; + options.target_file_size_base = kFileBytes; + options.target_file_size_multiplier = 1; + options.max_compaction_bytes = 1500; + Reopen(options); + + Random rnd(301); + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < kNumFiles; ++j) { + if (i > 0) { + // delete [95,105) in two files, [295,305) in next two + int mid = (j + (1 - j % 2)) * kNumPerFile; + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(mid - 5), Key(mid + 5))); + } + std::vector values; + // Write 100KB (100 values, each 1K) + for (int k = 0; k < kNumPerFile; k++) { + values.push_back(rnd.RandomString(990)); + ASSERT_OK(Put(Key(j * kNumPerFile + k), values[k])); + } + // put extra key to trigger flush + ASSERT_OK(Put("", "")); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + if (j < kNumFiles - 1) { + // background compaction may happen early for kNumFiles'th file + ASSERT_EQ(NumTableFilesAtLevel(0), j + 1); + } + if (j == options.level0_file_num_compaction_trigger - 1) { + // When i == 1, compaction will output some files to L1, at which point + // L1 is not bottommost so range deletions cannot be compacted away. The + // new L1 files must be generated with non-overlapping key ranges even + // though multiple subcompactions see the same ranges deleted, else an + // assertion will fail. + // + // Only enable auto-compactions when we're ready; otherwise, the + // oversized L0 (relative to base_level) causes the compaction to run + // earlier. + ASSERT_OK(db_->EnableAutoCompaction({db_->DefaultColumnFamily()})); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "true"}})); + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + ASSERT_GT(NumTableFilesAtLevel(1), 0); + ASSERT_GT(NumTableFilesAtLevel(2), 0); + } + } + } +} + +TEST_F(DBRangeDelTest, ValidUniversalSubcompactionBoundaries) { + const int kNumPerFile = 100, kFilesPerLevel = 4, kNumLevels = 4; + Options options = CurrentOptions(); + options.compaction_options_universal.min_merge_width = kFilesPerLevel; + options.compaction_options_universal.max_merge_width = kFilesPerLevel; + options.compaction_options_universal.size_ratio = 10; + options.compaction_style = kCompactionStyleUniversal; + options.level0_file_num_compaction_trigger = kFilesPerLevel; + options.max_subcompactions = 4; + options.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile)); + options.num_levels = kNumLevels; + options.target_file_size_base = kNumPerFile << 10; + options.target_file_size_multiplier = 1; + Reopen(options); + + Random rnd(301); + for (int i = 0; i < kNumLevels - 1; ++i) { + for (int j = 0; j < kFilesPerLevel; ++j) { + if (i == kNumLevels - 2) { + // insert range deletions [95,105) in two files, [295,305) in next two + // to prepare L1 for later manual compaction. + int mid = (j + (1 - j % 2)) * kNumPerFile; + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(mid - 5), Key(mid + 5))); + } + std::vector values; + // Write 100KB (100 values, each 1K) + for (int k = 0; k < kNumPerFile; k++) { + // For the highest level, use smaller value size such that it does not + // prematurely cause auto compaction due to range tombstone adding + // additional compensated file size + values.push_back(rnd.RandomString((i == kNumLevels - 2) ? 600 : 990)); + ASSERT_OK(Put(Key(j * kNumPerFile + k), values[k])); + } + // put extra key to trigger flush + ASSERT_OK(Put("", "")); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + if (j < kFilesPerLevel - 1) { + // background compaction may happen early for kFilesPerLevel'th file + ASSERT_EQ(NumTableFilesAtLevel(0), j + 1); + } + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + if (i == kNumLevels - 2) { + // For the highest level, value size is smaller (see Put() above), + // so output file number is smaller. + ASSERT_GT(NumTableFilesAtLevel(kNumLevels - 1 - i), kFilesPerLevel - 2); + } else { + ASSERT_GT(NumTableFilesAtLevel(kNumLevels - 1 - i), kFilesPerLevel - 1); + } + } + // Now L1-L3 are full, when we compact L1->L2 we should see (1) subcompactions + // happen since input level > 0; (2) range deletions are not dropped since + // output level is not bottommost. If no file boundary assertion fails, that + // probably means universal compaction + subcompaction + range deletion are + // compatible. + ASSERT_OK(dbfull()->RunManualCompaction( + static_cast_with_check(db_->DefaultColumnFamily()) + ->cfd(), + 1 /* input_level */, 2 /* output_level */, CompactRangeOptions(), + nullptr /* begin */, nullptr /* end */, true /* exclusive */, + true /* disallow_trivial_move */, + std::numeric_limits::max() /* max_file_num_to_ignore */, + "" /*trim_ts*/)); +} + +TEST_F(DBRangeDelTest, CompactionRemovesCoveredMergeOperands) { + const int kNumPerFile = 3, kNumFiles = 3; + Options opts = CurrentOptions(); + opts.disable_auto_compactions = true; + opts.memtable_factory.reset(test::NewSpecialSkipListFactory(2 * kNumPerFile)); + opts.merge_operator = MergeOperators::CreateUInt64AddOperator(); + opts.num_levels = 2; + Reopen(opts); + + // Iterates kNumFiles * kNumPerFile + 1 times since flushing the last file + // requires an extra entry. + for (int i = 0; i <= kNumFiles * kNumPerFile; ++i) { + if (i % kNumPerFile == 0 && i / kNumPerFile == kNumFiles - 1) { + // Delete merge operands from all but the last file + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + "key", "key_")); + } + std::string val; + PutFixed64(&val, i); + ASSERT_OK(db_->Merge(WriteOptions(), "key", val)); + // we need to prevent trivial move using Puts so compaction will actually + // process the merge operands. + ASSERT_OK(db_->Put(WriteOptions(), "prevent_trivial_move", "")); + if (i > 0 && i % kNumPerFile == 0) { + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + } + + ReadOptions read_opts; + read_opts.ignore_range_deletions = true; + std::string expected, actual; + ASSERT_OK(db_->Get(read_opts, "key", &actual)); + PutFixed64(&expected, 45); // 1+2+...+9 + ASSERT_EQ(expected, actual); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + expected.clear(); + ASSERT_OK(db_->Get(read_opts, "key", &actual)); + uint64_t tmp; + Slice tmp2(actual); + GetFixed64(&tmp2, &tmp); + PutFixed64(&expected, 30); // 6+7+8+9 (earlier operands covered by tombstone) + ASSERT_EQ(expected, actual); +} + +TEST_F(DBRangeDelTest, PutDeleteRangeMergeFlush) { + // Test the sequence of operations: (1) Put, (2) DeleteRange, (3) Merge, (4) + // Flush. The `CompactionIterator` previously had a bug where we forgot to + // check for covering range tombstones when processing the (1) Put, causing + // it to reappear after the flush. + Options opts = CurrentOptions(); + opts.merge_operator = MergeOperators::CreateUInt64AddOperator(); + Reopen(opts); + + std::string val; + PutFixed64(&val, 1); + ASSERT_OK(db_->Put(WriteOptions(), "key", val)); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "key", + "key_")); + ASSERT_OK(db_->Merge(WriteOptions(), "key", val)); + ASSERT_OK(db_->Flush(FlushOptions())); + + ReadOptions read_opts; + std::string expected, actual; + ASSERT_OK(db_->Get(read_opts, "key", &actual)); + PutFixed64(&expected, 1); + ASSERT_EQ(expected, actual); +} + +TEST_F(DBRangeDelTest, ObsoleteTombstoneCleanup) { + // During compaction to bottommost level, verify range tombstones older than + // the oldest snapshot are removed, while others are preserved. + Options opts = CurrentOptions(); + opts.disable_auto_compactions = true; + opts.num_levels = 2; + opts.statistics = CreateDBStatistics(); + Reopen(opts); + + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "dr1", + "dr10")); // obsolete after compaction + ASSERT_OK(db_->Put(WriteOptions(), "key", "val")); + ASSERT_OK(db_->Flush(FlushOptions())); + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "dr2", + "dr20")); // protected by snapshot + ASSERT_OK(db_->Put(WriteOptions(), "key", "val")); + ASSERT_OK(db_->Flush(FlushOptions())); + + ASSERT_EQ(2, NumTableFilesAtLevel(0)); + ASSERT_EQ(0, NumTableFilesAtLevel(1)); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + ASSERT_EQ(1, TestGetTickerCount(opts, COMPACTION_RANGE_DEL_DROP_OBSOLETE)); + + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(DBRangeDelTest, TableEvictedDuringScan) { + // The RangeDelAggregator holds pointers into range deletion blocks created by + // table readers. This test ensures the aggregator can still access those + // blocks even if it outlives the table readers that created them. + // + // DBIter always keeps readers open for L0 files. So, in order to test + // aggregator outliving reader, we need to have deletions in L1 files, which + // are opened/closed on-demand during the scan. This is accomplished by + // setting kNumRanges > level0_stop_writes_trigger, which prevents deletions + // from all lingering in L0 (there is at most one range deletion per L0 file). + // + // The first L1 file will contain a range deletion since its begin key is 0. + // SeekToFirst() references that table's reader and adds its range tombstone + // to the aggregator. Upon advancing beyond that table's key-range via Next(), + // the table reader will be unreferenced by the iterator. Since we manually + // call Evict() on all readers before the full scan, this unreference causes + // the reader's refcount to drop to zero and thus be destroyed. + // + // When it is destroyed, we do not remove its range deletions from the + // aggregator. So, subsequent calls to Next() must be able to use these + // deletions to decide whether a key is covered. This will work as long as + // the aggregator properly references the range deletion block. + const int kNum = 25, kRangeBegin = 0, kRangeEnd = 7, kNumRanges = 5; + Options opts = CurrentOptions(); + opts.comparator = test::Uint64Comparator(); + opts.level0_file_num_compaction_trigger = 4; + opts.level0_stop_writes_trigger = 4; + opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1)); + opts.num_levels = 2; + BlockBasedTableOptions bbto; + bbto.cache_index_and_filter_blocks = true; + bbto.block_cache = NewLRUCache(8 << 20); + opts.table_factory.reset(NewBlockBasedTableFactory(bbto)); + DestroyAndReopen(opts); + + // Hold a snapshot so range deletions can't become obsolete during compaction + // to bottommost level (i.e., L1). + const Snapshot* snapshot = db_->GetSnapshot(); + for (int i = 0; i < kNum; ++i) { + ASSERT_OK(db_->Put(WriteOptions(), GetNumericStr(i), "val")); + if (i > 0) { + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + if (i >= kNum / 2 && i < kNum / 2 + kNumRanges) { + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + GetNumericStr(kRangeBegin), + GetNumericStr(kRangeEnd))); + } + } + // Must be > 1 so the first L1 file can be closed before scan finishes + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_GT(NumTableFilesAtLevel(1), 1); + std::vector file_numbers = ListTableFiles(env_, dbname_); + + ReadOptions read_opts; + auto* iter = db_->NewIterator(read_opts); + ASSERT_OK(iter->status()); + int expected = kRangeEnd; + iter->SeekToFirst(); + for (auto file_number : file_numbers) { + // This puts table caches in the state of being externally referenced only + // so they are destroyed immediately upon iterator unreferencing. + TableCache::Evict(dbfull()->TEST_table_cache(), file_number); + } + for (; iter->Valid(); iter->Next()) { + ASSERT_EQ(GetNumericStr(expected), iter->key()); + ++expected; + // Keep clearing block cache's LRU so range deletion block can be freed as + // soon as its refcount drops to zero. + bbto.block_cache->EraseUnRefEntries(); + } + ASSERT_EQ(kNum, expected); + delete iter; + db_->ReleaseSnapshot(snapshot); + + // Also test proper cache handling in GetRangeTombstoneIterator, + // via TablesRangeTombstoneSummary. (This once triggered memory leak + // report with ASAN.) + opts.max_open_files = 1; + Reopen(opts); + + std::string str; + ASSERT_OK(dbfull()->TablesRangeTombstoneSummary(db_->DefaultColumnFamily(), + 100, &str)); +} + +TEST_F(DBRangeDelTest, GetCoveredKeyFromMutableMemtable) { + do { + DestroyAndReopen(CurrentOptions()); + ASSERT_OK(db_->Put(WriteOptions(), "key", "val")); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "z")); + + ReadOptions read_opts; + std::string value; + ASSERT_TRUE(db_->Get(read_opts, "key", &value).IsNotFound()); + } while (ChangeOptions(kRangeDelSkipConfigs)); +} + +TEST_F(DBRangeDelTest, GetCoveredKeyFromImmutableMemtable) { + do { + Options opts = CurrentOptions(); + opts.max_write_buffer_number = 3; + opts.min_write_buffer_number_to_merge = 2; + // SpecialSkipListFactory lets us specify maximum number of elements the + // memtable can hold. It switches the active memtable to immutable (flush is + // prevented by the above options) upon inserting an element that would + // overflow the memtable. + opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1)); + DestroyAndReopen(opts); + + ASSERT_OK(db_->Put(WriteOptions(), "key", "val")); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "z")); + ASSERT_OK(db_->Put(WriteOptions(), "blah", "val")); + + ReadOptions read_opts; + std::string value; + ASSERT_TRUE(db_->Get(read_opts, "key", &value).IsNotFound()); + } while (ChangeOptions(kRangeDelSkipConfigs)); +} + +TEST_F(DBRangeDelTest, GetCoveredKeyFromSst) { + do { + DestroyAndReopen(CurrentOptions()); + ASSERT_OK(db_->Put(WriteOptions(), "key", "val")); + // snapshot prevents key from being deleted during flush + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "z")); + ASSERT_OK(db_->Flush(FlushOptions())); + + ReadOptions read_opts; + std::string value; + ASSERT_TRUE(db_->Get(read_opts, "key", &value).IsNotFound()); + db_->ReleaseSnapshot(snapshot); + } while (ChangeOptions(kRangeDelSkipConfigs)); +} + +TEST_F(DBRangeDelTest, GetCoveredMergeOperandFromMemtable) { + const int kNumMergeOps = 10; + Options opts = CurrentOptions(); + opts.merge_operator = MergeOperators::CreateUInt64AddOperator(); + Reopen(opts); + + for (int i = 0; i < kNumMergeOps; ++i) { + std::string val; + PutFixed64(&val, i); + ASSERT_OK(db_->Merge(WriteOptions(), "key", val)); + if (i == kNumMergeOps / 2) { + // deletes [0, 5] + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + "key", "key_")); + } + } + + ReadOptions read_opts; + std::string expected, actual; + ASSERT_OK(db_->Get(read_opts, "key", &actual)); + PutFixed64(&expected, 30); // 6+7+8+9 + ASSERT_EQ(expected, actual); + + expected.clear(); + read_opts.ignore_range_deletions = true; + ASSERT_OK(db_->Get(read_opts, "key", &actual)); + PutFixed64(&expected, 45); // 0+1+2+...+9 + ASSERT_EQ(expected, actual); +} + +TEST_F(DBRangeDelTest, GetIgnoresRangeDeletions) { + Options opts = CurrentOptions(); + opts.max_write_buffer_number = 4; + opts.min_write_buffer_number_to_merge = 3; + opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1)); + Reopen(opts); + + ASSERT_OK(db_->Put(WriteOptions(), "sst_key", "val")); + // snapshot prevents key from being deleted during flush + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "z")); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_OK(db_->Put(WriteOptions(), "imm_key", "val")); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "z")); + ASSERT_OK(db_->Put(WriteOptions(), "mem_key", "val")); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "z")); + + ReadOptions read_opts; + read_opts.ignore_range_deletions = true; + for (std::string key : {"sst_key", "imm_key", "mem_key"}) { + std::string value; + ASSERT_OK(db_->Get(read_opts, key, &value)); + } + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(DBRangeDelTest, IteratorRemovesCoveredKeys) { + const int kNum = 200, kRangeBegin = 50, kRangeEnd = 150, kNumPerFile = 25; + Options opts = CurrentOptions(); + opts.comparator = test::Uint64Comparator(); + opts.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile)); + DestroyAndReopen(opts); + + // Write half of the keys before the tombstone and half after the tombstone. + // Only covered keys (i.e., within the range and older than the tombstone) + // should be deleted. + for (int i = 0; i < kNum; ++i) { + if (i == kNum / 2) { + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + GetNumericStr(kRangeBegin), + GetNumericStr(kRangeEnd))); + } + ASSERT_OK(db_->Put(WriteOptions(), GetNumericStr(i), "val")); + } + ReadOptions read_opts; + auto* iter = db_->NewIterator(read_opts); + ASSERT_OK(iter->status()); + + int expected = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_EQ(GetNumericStr(expected), iter->key()); + if (expected == kRangeBegin - 1) { + expected = kNum / 2; + } else { + ++expected; + } + } + ASSERT_EQ(kNum, expected); + delete iter; +} + +TEST_F(DBRangeDelTest, IteratorOverUserSnapshot) { + const int kNum = 200, kRangeBegin = 50, kRangeEnd = 150, kNumPerFile = 25; + Options opts = CurrentOptions(); + opts.comparator = test::Uint64Comparator(); + opts.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile)); + DestroyAndReopen(opts); + + const Snapshot* snapshot = nullptr; + // Put a snapshot before the range tombstone, verify an iterator using that + // snapshot sees all inserted keys. + for (int i = 0; i < kNum; ++i) { + if (i == kNum / 2) { + snapshot = db_->GetSnapshot(); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + GetNumericStr(kRangeBegin), + GetNumericStr(kRangeEnd))); + } + ASSERT_OK(db_->Put(WriteOptions(), GetNumericStr(i), "val")); + } + ReadOptions read_opts; + read_opts.snapshot = snapshot; + auto* iter = db_->NewIterator(read_opts); + ASSERT_OK(iter->status()); + + int expected = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_EQ(GetNumericStr(expected), iter->key()); + ++expected; + } + ASSERT_EQ(kNum / 2, expected); + delete iter; + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(DBRangeDelTest, IteratorIgnoresRangeDeletions) { + Options opts = CurrentOptions(); + opts.max_write_buffer_number = 4; + opts.min_write_buffer_number_to_merge = 3; + opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1)); + Reopen(opts); + + ASSERT_OK(db_->Put(WriteOptions(), "sst_key", "val")); + // snapshot prevents key from being deleted during flush + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "z")); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_OK(db_->Put(WriteOptions(), "imm_key", "val")); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "z")); + ASSERT_OK(db_->Put(WriteOptions(), "mem_key", "val")); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "z")); + + ReadOptions read_opts; + read_opts.ignore_range_deletions = true; + auto* iter = db_->NewIterator(read_opts); + ASSERT_OK(iter->status()); + int i = 0; + std::string expected[] = {"imm_key", "mem_key", "sst_key"}; + for (iter->SeekToFirst(); iter->Valid(); iter->Next(), ++i) { + std::string key; + ASSERT_EQ(expected[i], iter->key()); + } + ASSERT_EQ(3, i); + delete iter; + db_->ReleaseSnapshot(snapshot); +} + +#ifndef ROCKSDB_UBSAN_RUN +TEST_F(DBRangeDelTest, TailingIteratorRangeTombstoneUnsupported) { + ASSERT_OK(db_->Put(WriteOptions(), "key", "val")); + // snapshot prevents key from being deleted during flush + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "z")); + + // iterations check unsupported in memtable, l0, and then l1 + for (int i = 0; i < 3; ++i) { + ReadOptions read_opts; + read_opts.tailing = true; + auto* iter = db_->NewIterator(read_opts); + if (i == 2) { + // For L1+, iterators over files are created on-demand, so need seek + iter->SeekToFirst(); + } + ASSERT_TRUE(iter->status().IsNotSupported()); + + delete iter; + if (i == 0) { + ASSERT_OK(db_->Flush(FlushOptions())); + } else if (i == 1) { + MoveFilesToLevel(1); + } + } + db_->ReleaseSnapshot(snapshot); +} +#endif // !ROCKSDB_UBSAN_RUN + +TEST_F(DBRangeDelTest, SubcompactionHasEmptyDedicatedRangeDelFile) { + const int kNumFiles = 2, kNumKeysPerFile = 4; + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + options.level0_file_num_compaction_trigger = kNumFiles; + options.max_subcompactions = 2; + options.num_levels = 2; + options.target_file_size_base = 4096; + Reopen(options); + + // need a L1 file for subcompaction to be triggered + ASSERT_OK( + db_->Put(WriteOptions(), db_->DefaultColumnFamily(), Key(0), "val")); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(1); + + // put enough keys to fill up the first subcompaction, and later range-delete + // them so that the first subcompaction outputs no key-values. In that case + // it'll consider making an SST file dedicated to range deletions. + for (int i = 0; i < kNumKeysPerFile; ++i) { + ASSERT_OK(db_->Put(WriteOptions(), db_->DefaultColumnFamily(), Key(i), + std::string(1024, 'a'))); + } + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0), + Key(kNumKeysPerFile))); + + // the above range tombstone can be dropped, so that one alone won't cause a + // dedicated file to be opened. We can make one protected by snapshot that + // must be considered. Make its range outside the first subcompaction's range + // to exercise the tricky part of the code. + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(kNumKeysPerFile + 1), + Key(kNumKeysPerFile + 2))); + ASSERT_OK(db_->Flush(FlushOptions())); + + ASSERT_EQ(kNumFiles, NumTableFilesAtLevel(0)); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + + ASSERT_OK(db_->EnableAutoCompaction({db_->DefaultColumnFamily()})); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(DBRangeDelTest, MemtableBloomFilter) { + // regression test for #2743. the range delete tombstones in memtable should + // be added even when Get() skips searching due to its prefix bloom filter + const int kMemtableSize = 1 << 20; // 1MB + const int kMemtablePrefixFilterSize = 1 << 13; // 8KB + const int kNumKeys = 1000; + const int kPrefixLen = 8; + Options options = CurrentOptions(); + options.memtable_prefix_bloom_size_ratio = + static_cast(kMemtablePrefixFilterSize) / kMemtableSize; + options.prefix_extractor.reset( + ROCKSDB_NAMESPACE::NewFixedPrefixTransform(kPrefixLen)); + options.write_buffer_size = kMemtableSize; + Reopen(options); + + for (int i = 0; i < kNumKeys; ++i) { + ASSERT_OK(Put(Key(i), "val")); + } + ASSERT_OK(Flush()); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0), + Key(kNumKeys))); + for (int i = 0; i < kNumKeys; ++i) { + std::string value; + ASSERT_TRUE(db_->Get(ReadOptions(), Key(i), &value).IsNotFound()); + } +} + +TEST_F(DBRangeDelTest, CompactionTreatsSplitInputLevelDeletionAtomically) { + // This test originally verified that compaction treated files containing a + // split range deletion in the input level as an atomic unit. I.e., + // compacting any input-level file(s) containing a portion of the range + // deletion causes all other input-level files containing portions of that + // same range deletion to be included in the compaction. Range deletion + // tombstones are now truncated to sstable boundaries which removed the need + // for that behavior (which could lead to excessively large + // compactions). + const int kNumFilesPerLevel = 4, kValueBytes = 4 << 10; + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.level0_file_num_compaction_trigger = kNumFilesPerLevel; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(2 /* num_entries_flush */)); + // max file size could be 2x of target file size, so set it to half of that + options.target_file_size_base = kValueBytes / 2; + // disable dynamic_file_size, as it will cut L1 files into more files (than + // kNumFilesPerLevel). + options.level_compaction_dynamic_file_size = false; + options.max_compaction_bytes = 1500; + // i == 0: CompactFiles + // i == 1: CompactRange + // i == 2: automatic compaction + for (int i = 0; i < 3; ++i) { + DestroyAndReopen(options); + + ASSERT_OK(Put(Key(0), "")); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(2); + ASSERT_EQ(1, NumTableFilesAtLevel(2)); + + // snapshot protects range tombstone from dropping due to becoming obsolete. + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(0), Key(2 * kNumFilesPerLevel))); + + Random rnd(301); + std::string value = rnd.RandomString(kValueBytes); + for (int j = 0; j < kNumFilesPerLevel; ++j) { + // give files overlapping key-ranges to prevent trivial move + ASSERT_OK(Put(Key(j), value)); + ASSERT_OK(Put(Key(2 * kNumFilesPerLevel - 1 - j), value)); + if (j > 0) { + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(j, NumTableFilesAtLevel(0)); + } + } + // put extra key to trigger final flush + ASSERT_OK(Put("", "")); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_EQ(kNumFilesPerLevel, NumTableFilesAtLevel(1)); + + ColumnFamilyMetaData meta; + db_->GetColumnFamilyMetaData(&meta); + if (i == 0) { + ASSERT_OK(db_->CompactFiles( + CompactionOptions(), {meta.levels[1].files[0].name}, 2 /* level */)); + ASSERT_EQ(0, NumTableFilesAtLevel(1)); + } else if (i == 1) { + auto begin_str = Key(0), end_str = Key(1); + Slice begin = begin_str, end = end_str; + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &begin, &end)); + ASSERT_EQ(3, NumTableFilesAtLevel(1)); + } else if (i == 2) { + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"max_bytes_for_level_base", "10000"}})); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + } + ASSERT_GT(NumTableFilesAtLevel(2), 0); + + db_->ReleaseSnapshot(snapshot); + } +} + +TEST_F(DBRangeDelTest, RangeTombstoneEndKeyAsSstableUpperBound) { + // Test the handling of the range-tombstone end-key as the + // upper-bound for an sstable. + + const int kNumFilesPerLevel = 2, kValueBytes = 4 << 10; + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.level0_file_num_compaction_trigger = kNumFilesPerLevel; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(2 /* num_entries_flush */)); + options.target_file_size_base = kValueBytes; + options.disable_auto_compactions = true; + // disable it for now, otherwise the L1 files are going be cut before data 1: + // L1: [0] [1,4] + // L2: [0,0] + // because the grandparent file is between [0]->[1] and it's size is more than + // 1/8 of target size (4k). + options.level_compaction_dynamic_file_size = false; + + DestroyAndReopen(options); + + // Create an initial sstable at L2: + // [key000000#1,1, key000000#1,1] + ASSERT_OK(Put(Key(0), "")); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(2); + ASSERT_EQ(1, NumTableFilesAtLevel(2)); + + // A snapshot protects the range tombstone from dropping due to + // becoming obsolete. + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0), + Key(2 * kNumFilesPerLevel))); + + // Create 2 additional sstables in L0. Note that the first sstable + // contains the range tombstone. + // [key000000#3,1, key000004#72057594037927935,15] + // [key000001#5,1, key000002#6,1] + Random rnd(301); + std::string value = rnd.RandomString(kValueBytes); + for (int j = 0; j < kNumFilesPerLevel; ++j) { + // Give files overlapping key-ranges to prevent a trivial move when we + // compact from L0 to L1. + ASSERT_OK(Put(Key(j), value)); + ASSERT_OK(Put(Key(2 * kNumFilesPerLevel - 1 - j), value)); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(j + 1, NumTableFilesAtLevel(0)); + } + // Compact the 2 L0 sstables to L1, resulting in the following LSM. There + // are 2 sstables generated in L1 due to the target_file_size_base setting. + // L1: + // [key000000#3,1, key000002#72057594037927935,15] + // [key000002#6,1, key000004#72057594037927935,15] + // L2: + // [key000000#1,1, key000000#1,1] + MoveFilesToLevel(1); + ASSERT_EQ(2, NumTableFilesAtLevel(1)); + + { + // Compact the second sstable in L1: + // L1: + // [key000000#3,1, key000002#72057594037927935,15] + // L2: + // [key000000#1,1, key000000#1,1] + // [key000002#6,1, key000004#72057594037927935,15] + // + // At the same time, verify the compaction does not cause the key at the + // endpoint (key000002#6,1) to disappear. + ASSERT_EQ(value, Get(Key(2))); + auto begin_str = Key(3); + const ROCKSDB_NAMESPACE::Slice begin = begin_str; + ASSERT_OK(dbfull()->TEST_CompactRange(1, &begin, nullptr)); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + ASSERT_EQ(2, NumTableFilesAtLevel(2)); + ASSERT_EQ(value, Get(Key(2))); + } + + { + // Compact the first sstable in L1. This should be copacetic, but + // was previously resulting in overlapping sstables in L2 due to + // mishandling of the range tombstone end-key when used as the + // largest key for an sstable. The resulting LSM structure should + // be: + // + // L2: + // [key000000#1,1, key000001#72057594037927935,15] + // [key000001#5,1, key000002#72057594037927935,15] + // [key000002#6,1, key000004#72057594037927935,15] + auto begin_str = Key(0); + const ROCKSDB_NAMESPACE::Slice begin = begin_str; + ASSERT_OK(dbfull()->TEST_CompactRange(1, &begin, &begin)); + ASSERT_EQ(0, NumTableFilesAtLevel(1)); + ASSERT_EQ(3, NumTableFilesAtLevel(2)); + } + + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(DBRangeDelTest, UnorderedTombstones) { + // Regression test for #2752. Range delete tombstones between + // different snapshot stripes are not stored in order, so the first + // tombstone of each snapshot stripe should be checked as a smallest + // candidate. + Options options = CurrentOptions(); + DestroyAndReopen(options); + + auto cf = db_->DefaultColumnFamily(); + + ASSERT_OK(db_->Put(WriteOptions(), cf, "a", "a")); + ASSERT_OK(db_->Flush(FlushOptions(), cf)); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr)); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + + ASSERT_OK(db_->DeleteRange(WriteOptions(), cf, "b", "c")); + // Hold a snapshot to separate these two delete ranges. + auto snapshot = db_->GetSnapshot(); + ASSERT_OK(db_->DeleteRange(WriteOptions(), cf, "a", "b")); + ASSERT_OK(db_->Flush(FlushOptions(), cf)); + db_->ReleaseSnapshot(snapshot); + + std::vector> files; + dbfull()->TEST_GetFilesMetaData(cf, &files); + ASSERT_EQ(1, files[0].size()); + ASSERT_EQ("a", files[0][0].smallest.user_key()); + ASSERT_EQ("c", files[0][0].largest.user_key()); + + std::string v; + auto s = db_->Get(ReadOptions(), "a", &v); + ASSERT_TRUE(s.IsNotFound()); +} + +class MockMergeOperator : public MergeOperator { + // Mock non-associative operator. Non-associativity is expressed by lack of + // implementation for any `PartialMerge*` functions. + public: + bool FullMergeV2(const MergeOperationInput& merge_in, + MergeOperationOutput* merge_out) const override { + assert(merge_out != nullptr); + merge_out->new_value = merge_in.operand_list.back().ToString(); + return true; + } + + const char* Name() const override { return "MockMergeOperator"; } +}; + +TEST_F(DBRangeDelTest, KeyAtOverlappingEndpointReappears) { + // This test uses a non-associative merge operator since that is a convenient + // way to get compaction to write out files with overlapping user-keys at the + // endpoints. Note, however, overlapping endpoints can also occur with other + // value types (Put, etc.), assuming the right snapshots are present. + const int kFileBytes = 1 << 20; + const int kValueBytes = 1 << 10; + const int kNumFiles = 4; + + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + options.merge_operator.reset(new MockMergeOperator()); + options.target_file_size_base = kFileBytes; + Reopen(options); + + // Push dummy data to L3 so that our actual test files on L0-L2 + // will not be considered "bottommost" level, otherwise compaction + // may prevent us from creating overlapping user keys + // as on the bottommost layer MergeHelper + ASSERT_OK(db_->Merge(WriteOptions(), "key", "dummy")); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(3); + + Random rnd(301); + const Snapshot* snapshot = nullptr; + for (int i = 0; i < kNumFiles; ++i) { + for (int j = 0; j < kFileBytes / kValueBytes; ++j) { + auto value = rnd.RandomString(kValueBytes); + ASSERT_OK(db_->Merge(WriteOptions(), "key", value)); + } + if (i == kNumFiles - 1) { + // Take snapshot to prevent covered merge operands from being dropped by + // compaction. + snapshot = db_->GetSnapshot(); + // The DeleteRange is the last write so all merge operands are covered. + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + "key", "key_")); + } + ASSERT_OK(db_->Flush(FlushOptions())); + } + ASSERT_EQ(kNumFiles, NumTableFilesAtLevel(0)); + std::string value; + ASSERT_TRUE(db_->Get(ReadOptions(), "key", &value).IsNotFound()); + + ASSERT_OK(dbfull()->TEST_CompactRange( + 0 /* level */, nullptr /* begin */, nullptr /* end */, + nullptr /* column_family */, true /* disallow_trivial_move */)); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + // Now we have multiple files at L1 all containing a single user key, thus + // guaranteeing overlap in the file endpoints. + ASSERT_GT(NumTableFilesAtLevel(1), 1); + + // Verify no merge operands reappeared after the compaction. + ASSERT_TRUE(db_->Get(ReadOptions(), "key", &value).IsNotFound()); + + // Compact and verify again. It's worthwhile because now the files have + // tighter endpoints, so we can verify that doesn't mess anything up. + ASSERT_OK(dbfull()->TEST_CompactRange( + 1 /* level */, nullptr /* begin */, nullptr /* end */, + nullptr /* column_family */, true /* disallow_trivial_move */)); + ASSERT_GT(NumTableFilesAtLevel(2), 1); + ASSERT_TRUE(db_->Get(ReadOptions(), "key", &value).IsNotFound()); + + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(DBRangeDelTest, UntruncatedTombstoneDoesNotDeleteNewerKey) { + // Verify a key newer than a range tombstone cannot be deleted by being + // compacted to the bottom level (and thus having its seqnum zeroed) before + // the range tombstone. This used to happen when range tombstones were + // untruncated on reads such that they extended past their file boundaries. + // + // Test summary: + // + // - L1 is bottommost. + // - A couple snapshots are strategically taken to prevent seqnums from being + // zeroed, range tombstone from being dropped, merge operands from being + // dropped, and merge operands from being combined. + // - Left half of files in L1 all have same user key, ensuring their file + // boundaries overlap. In the past this would cause range tombstones to be + // untruncated. + // - Right half of L1 files all have different keys, ensuring no overlap. + // - A range tombstone spans all L1 keys, so it is stored in every L1 file. + // - Keys in the right side of the key-range are overwritten. These are + // compacted down to L1 after releasing snapshots such that their seqnums + // will be zeroed. + // - A full range scan is performed. If the tombstone in the left L1 files + // were untruncated, it would now cover keys newer than it (but with zeroed + // seqnums) in the right L1 files. + const int kFileBytes = 1 << 20; + const int kValueBytes = 1 << 10; + const int kNumFiles = 4; + const int kMaxKey = kNumFiles * kFileBytes / kValueBytes; + const int kKeysOverwritten = 10; + + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + options.merge_operator.reset(new MockMergeOperator()); + options.num_levels = 2; + options.target_file_size_base = kFileBytes; + Reopen(options); + + Random rnd(301); + // - snapshots[0] prevents merge operands from being combined during + // compaction. + // - snapshots[1] prevents merge operands from being dropped due to the + // covering range tombstone. + const Snapshot* snapshots[] = {nullptr, nullptr}; + for (int i = 0; i < kNumFiles; ++i) { + for (int j = 0; j < kFileBytes / kValueBytes; ++j) { + auto value = rnd.RandomString(kValueBytes); + std::string key; + if (i < kNumFiles / 2) { + key = Key(0); + } else { + key = Key(1 + i * kFileBytes / kValueBytes + j); + } + ASSERT_OK(db_->Merge(WriteOptions(), key, value)); + } + if (i == 0) { + snapshots[0] = db_->GetSnapshot(); + } + if (i == kNumFiles - 1) { + snapshots[1] = db_->GetSnapshot(); + // The DeleteRange is the last write so all merge operands are covered. + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(0), Key(kMaxKey + 1))); + } + ASSERT_OK(db_->Flush(FlushOptions())); + } + ASSERT_EQ(kNumFiles, NumTableFilesAtLevel(0)); + + auto get_key_count = [this]() -> int { + auto* iter = db_->NewIterator(ReadOptions()); + assert(iter->status().ok()); + iter->SeekToFirst(); + int keys_found = 0; + for (; iter->Valid(); iter->Next()) { + ++keys_found; + } + delete iter; + return keys_found; + }; + + // All keys should be covered + ASSERT_EQ(0, get_key_count()); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr /* begin_key */, + nullptr /* end_key */)); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + // Roughly the left half of L1 files should have overlapping boundary keys, + // while the right half should not. + ASSERT_GE(NumTableFilesAtLevel(1), kNumFiles); + + // Now overwrite a few keys that are in L1 files that definitely don't have + // overlapping boundary keys. + for (int i = kMaxKey; i > kMaxKey - kKeysOverwritten; --i) { + auto value = rnd.RandomString(kValueBytes); + ASSERT_OK(db_->Merge(WriteOptions(), Key(i), value)); + } + ASSERT_OK(db_->Flush(FlushOptions())); + + // The overwritten keys are in L0 now, so clearly aren't covered by the range + // tombstone in L1. + ASSERT_EQ(kKeysOverwritten, get_key_count()); + + // Release snapshots so seqnums can be zeroed when L0->L1 happens. + db_->ReleaseSnapshot(snapshots[0]); + db_->ReleaseSnapshot(snapshots[1]); + + auto begin_key_storage = Key(kMaxKey - kKeysOverwritten + 1); + auto end_key_storage = Key(kMaxKey); + Slice begin_key(begin_key_storage); + Slice end_key(end_key_storage); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &begin_key, &end_key)); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_GE(NumTableFilesAtLevel(1), kNumFiles); + + ASSERT_EQ(kKeysOverwritten, get_key_count()); +} + +TEST_F(DBRangeDelTest, DeletedMergeOperandReappearsIterPrev) { + // Exposes a bug where we were using + // `RangeDelPositioningMode::kBackwardTraversal` while scanning merge operands + // in the forward direction. Confusingly, this case happened during + // `DBIter::Prev`. It could cause assertion failure, or reappearing keys. + const int kFileBytes = 1 << 20; + const int kValueBytes = 1 << 10; + // Need multiple keys so we can get results when calling `Prev()` after + // `SeekToLast()`. + const int kNumKeys = 3; + const int kNumFiles = 4; + + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + options.merge_operator.reset(new MockMergeOperator()); + options.target_file_size_base = kFileBytes; + Reopen(options); + + Random rnd(301); + const Snapshot* snapshot = nullptr; + for (int i = 0; i < kNumFiles; ++i) { + for (int j = 0; j < kFileBytes / kValueBytes; ++j) { + auto value = rnd.RandomString(kValueBytes); + ASSERT_OK(db_->Merge(WriteOptions(), Key(j % kNumKeys), value)); + if (i == 0 && j == kNumKeys) { + // Take snapshot to prevent covered merge operands from being dropped or + // merged by compaction. + snapshot = db_->GetSnapshot(); + // Do a DeleteRange near the beginning so only the oldest merge operand + // for each key is covered. This ensures the sequence of events: + // + // - `DBIter::Prev()` is called + // - After several same versions of the same user key are encountered, + // it decides to seek using `DBIter::FindValueForCurrentKeyUsingSeek`. + // - Binary searches to the newest version of the key, which is in the + // leftmost file containing the user key. + // - Scans forwards to collect all merge operands. Eventually reaches + // the rightmost file containing the oldest merge operand, which + // should be covered by the `DeleteRange`. If `RangeDelAggregator` + // were not properly using `kForwardTraversal` here, that operand + // would reappear. + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(0), Key(kNumKeys + 1))); + } + } + ASSERT_OK(db_->Flush(FlushOptions())); + } + ASSERT_EQ(kNumFiles, NumTableFilesAtLevel(0)); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr /* begin_key */, + nullptr /* end_key */)); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_GT(NumTableFilesAtLevel(1), 1); + + auto* iter = db_->NewIterator(ReadOptions()); + ASSERT_OK(iter->status()); + iter->SeekToLast(); + int keys_found = 0; + for (; iter->Valid(); iter->Prev()) { + ++keys_found; + } + delete iter; + ASSERT_EQ(kNumKeys, keys_found); + + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(DBRangeDelTest, SnapshotPreventsDroppedKeys) { + const int kFileBytes = 1 << 20; + + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + options.target_file_size_base = kFileBytes; + Reopen(options); + + ASSERT_OK(Put(Key(0), "a")); + const Snapshot* snapshot = db_->GetSnapshot(); + + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0), + Key(10))); + + ASSERT_OK(db_->Flush(FlushOptions())); + + ReadOptions read_opts; + read_opts.snapshot = snapshot; + auto* iter = db_->NewIterator(read_opts); + ASSERT_OK(iter->status()); + + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(Key(0), iter->key()); + + iter->Next(); + ASSERT_FALSE(iter->Valid()); + + delete iter; + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(DBRangeDelTest, SnapshotPreventsDroppedKeysInImmMemTables) { + const int kFileBytes = 1 << 20; + + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + options.target_file_size_base = kFileBytes; + Reopen(options); + + // block flush thread -> pin immtables in memory + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->LoadDependency({ + {"SnapshotPreventsDroppedKeysInImmMemTables:AfterNewIterator", + "DBImpl::BGWorkFlush"}, + }); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put(Key(0), "a")); + std::unique_ptr> + snapshot(db_->GetSnapshot(), + [this](const Snapshot* s) { db_->ReleaseSnapshot(s); }); + + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0), + Key(10))); + + ASSERT_OK(dbfull()->TEST_SwitchMemtable()); + + ReadOptions read_opts; + read_opts.snapshot = snapshot.get(); + std::unique_ptr iter(db_->NewIterator(read_opts)); + ASSERT_OK(iter->status()); + + TEST_SYNC_POINT("SnapshotPreventsDroppedKeysInImmMemTables:AfterNewIterator"); + + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(Key(0), iter->key()); + + iter->Next(); + ASSERT_FALSE(iter->Valid()); +} + +TEST_F(DBRangeDelTest, RangeTombstoneWrittenToMinimalSsts) { + // Adapted from + // https://github.com/cockroachdb/cockroach/blob/de8b3ea603dd1592d9dc26443c2cc92c356fbc2f/pkg/storage/engine/rocksdb_test.go#L1267-L1398. + // Regression test for issue where range tombstone was written to more files + // than necessary when it began exactly at the begin key in the next + // compaction output file. + const int kFileBytes = 1 << 20; + const int kValueBytes = 4 << 10; + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + // Have a bit of slack in the size limits but we enforce them more strictly + // when manually flushing/compacting. + options.max_compaction_bytes = 2 * kFileBytes; + options.target_file_size_base = 2 * kFileBytes; + options.write_buffer_size = 2 * kFileBytes; + Reopen(options); + + Random rnd(301); + for (char first_char : {'a', 'b', 'c'}) { + for (int i = 0; i < kFileBytes / kValueBytes; ++i) { + std::string key(1, first_char); + key.append(Key(i)); + std::string value = rnd.RandomString(kValueBytes); + ASSERT_OK(Put(key, value)); + } + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(2); + } + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_EQ(3, NumTableFilesAtLevel(2)); + + // Populate the memtable lightly while spanning the whole key-space. The + // setting of `max_compaction_bytes` will cause the L0->L1 to output multiple + // files to prevent a large L1->L2 compaction later. + ASSERT_OK(Put("a", "val")); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + "c" + Key(1), "d")); + // Our compaction output file cutting logic currently only considers point + // keys. So, in order for the range tombstone to have a chance at landing at + // the start of a new file, we need a point key at the range tombstone's + // start. + // TODO(ajkr): remove this `Put` after file cutting accounts for range + // tombstones (#3977). + ASSERT_OK(Put("c" + Key(1), "value")); + ASSERT_OK(db_->Flush(FlushOptions())); + + // Ensure manual L0->L1 compaction cuts the outputs before the range tombstone + // and the range tombstone is only placed in the second SST. + std::string begin_key_storage("c" + Key(1)); + Slice begin_key(begin_key_storage); + std::string end_key_storage("d"); + Slice end_key(end_key_storage); + ASSERT_OK(dbfull()->TEST_CompactRange( + 0 /* level */, &begin_key /* begin */, &end_key /* end */, + nullptr /* column_family */, true /* disallow_trivial_move */)); + ASSERT_EQ(2, NumTableFilesAtLevel(1)); + + std::vector all_metadata; + std::vector l1_metadata; + db_->GetLiveFilesMetaData(&all_metadata); + for (const auto& metadata : all_metadata) { + if (metadata.level == 1) { + l1_metadata.push_back(metadata); + } + } + std::sort(l1_metadata.begin(), l1_metadata.end(), + [&](const LiveFileMetaData& a, const LiveFileMetaData& b) { + return options.comparator->Compare(a.smallestkey, b.smallestkey) < + 0; + }); + ASSERT_EQ("a", l1_metadata[0].smallestkey); + ASSERT_EQ("a", l1_metadata[0].largestkey); + ASSERT_EQ("c" + Key(1), l1_metadata[1].smallestkey); + ASSERT_EQ("d", l1_metadata[1].largestkey); + + TablePropertiesCollection all_table_props; + ASSERT_OK(db_->GetPropertiesOfAllTables(&all_table_props)); + int64_t num_range_deletions = 0; + for (const auto& name_and_table_props : all_table_props) { + const auto& name = name_and_table_props.first; + const auto& table_props = name_and_table_props.second; + // The range tombstone should only be output to the second L1 SST. + if (name.size() >= l1_metadata[1].name.size() && + name.substr(name.size() - l1_metadata[1].name.size()) + .compare(l1_metadata[1].name) == 0) { + ASSERT_EQ(1, table_props->num_range_deletions); + ++num_range_deletions; + } else { + ASSERT_EQ(0, table_props->num_range_deletions); + } + } + ASSERT_EQ(1, num_range_deletions); +} + +TEST_F(DBRangeDelTest, LevelCompactOutputCutAtRangeTombstoneForTtlFiles) { + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.compaction_pri = kMinOverlappingRatio; + options.disable_auto_compactions = true; + options.ttl = 24 * 60 * 60; // 24 hours + options.target_file_size_base = 8 << 10; + env_->SetMockSleep(); + options.env = env_; + DestroyAndReopen(options); + + Random rnd(301); + // Fill some data so that future compactions are not bottommost level + // compaction, and hence they would try cut around files for ttl + for (int i = 5; i < 10; ++i) { + ASSERT_OK(Put(Key(i), rnd.RandomString(1 << 10))); + } + ASSERT_OK(Flush()); + MoveFilesToLevel(3); + ASSERT_EQ("0,0,0,1", FilesPerLevel()); + + for (int i = 5; i < 10; ++i) { + ASSERT_OK(Put(Key(i), rnd.RandomString(1 << 10))); + } + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + ASSERT_EQ("0,1,0,1", FilesPerLevel()); + + env_->MockSleepForSeconds(20 * 60 * 60); + // Prevent range tombstone from being dropped during compaction. + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(11), Key(12))); + ASSERT_OK(Put(Key(0), rnd.RandomString(1 << 10))); + ASSERT_OK(Flush()); + ASSERT_EQ("1,1,0,1", FilesPerLevel()); + // L0 file is new, L1 and L3 file are old and qualified for TTL + env_->MockSleepForSeconds(10 * 60 * 60); + MoveFilesToLevel(1); + // L1 output should be cut into 3 files: + // File 0: Key(0) + // File 1: (qualified for TTL): Key(5) - Key(10) + // File 1: DeleteRange [11, 12) + ASSERT_EQ("0,3,0,1", FilesPerLevel()); + db_->ReleaseSnapshot(snapshot); +} + +// Test SST partitioner cut after every single key +class SingleKeySstPartitioner : public SstPartitioner { + public: + const char* Name() const override { return "SingleKeySstPartitioner"; } + + PartitionerResult ShouldPartition( + const PartitionerRequest& /*request*/) override { + return kRequired; + } + + bool CanDoTrivialMove(const Slice& /*smallest_user_key*/, + const Slice& /*largest_user_key*/) override { + return false; + } +}; + +class SingleKeySstPartitionerFactory : public SstPartitionerFactory { + public: + static const char* kClassName() { return "SingleKeySstPartitionerFactory"; } + const char* Name() const override { return kClassName(); } + + std::unique_ptr CreatePartitioner( + const SstPartitioner::Context& /* context */) const override { + return std::unique_ptr(new SingleKeySstPartitioner()); + } +}; + +TEST_F(DBRangeDelTest, CompactionEmitRangeTombstoneToSSTPartitioner) { + Options options = CurrentOptions(); + auto factory = std::make_shared(); + options.sst_partitioner_factory = factory; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + + Random rnd(301); + // range deletion keys are not processed when compacting to bottommost level, + // so creating a file at older level to make the next compaction not + // bottommost level + ASSERT_OK(db_->Put(WriteOptions(), Key(4), rnd.RandomString(10))); + ASSERT_OK(Flush()); + MoveFilesToLevel(5); + + ASSERT_OK(db_->Put(WriteOptions(), Key(1), rnd.RandomString(10))); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(2), + Key(5))); + ASSERT_OK(Flush()); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + MoveFilesToLevel(1); + // SSTPartitioner decides to cut when range tombstone start key is passed to + // it. Note that the range tombstone [2, 5) itself span multiple keys, but we + // are not able to partition within its range yet. + ASSERT_EQ(2, NumTableFilesAtLevel(1)); +} + +TEST_F(DBRangeDelTest, OversizeCompactionGapBetweenPointKeyAndTombstone) { + // L2 has 2 files + // L2_0: 0, 1, 2, 3, 4 + // L2_1: 5, 6, 7 + // L0 has 1 file + // L0: 0, [5, 6), 8 + // max_compaction_bytes is less than the size of L2_0 and L2_1. + // When compacting L0 into L1, it should split into 3 files: + // compaction output should cut before key 5 and key 8 to + // limit future compaction size. + const int kNumPerFile = 4, kNumFiles = 2; + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.target_file_size_base = 9 * 1024; + options.max_compaction_bytes = 9 * 1024; + DestroyAndReopen(options); + Random rnd(301); + for (int i = 0; i < kNumFiles; ++i) { + std::vector values; + for (int j = 0; j < kNumPerFile; j++) { + values.push_back(rnd.RandomString(3 << 10)); + ASSERT_OK(Put(Key(i * kNumPerFile + j), values[j])); + } + } + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + MoveFilesToLevel(2); + ASSERT_EQ(2, NumTableFilesAtLevel(2)); + ASSERT_OK(Put(Key(0), rnd.RandomString(1 << 10))); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(5), + Key(6))); + ASSERT_OK(Put(Key(8), rnd.RandomString(1 << 10))); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr, + true /* disallow_trivial_move */)); + ASSERT_EQ(3, NumTableFilesAtLevel(1)); +} + +TEST_F(DBRangeDelTest, OversizeCompactionGapBetweenTombstone) { + // L2 has two files + // L2_0: 0, 1, 2, 3, 4. L2_1: 5, 6, 7 + // L0 has two range tombstones [0, 1), [7, 8). + // max_compaction_bytes is less than the size of L2_0. + // When compacting L0 into L1, the two range tombstones should be + // split into two files. + const int kNumPerFile = 4, kNumFiles = 2; + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.target_file_size_base = 9 * 1024; + options.max_compaction_bytes = 9 * 1024; + DestroyAndReopen(options); + Random rnd(301); + for (int i = 0; i < kNumFiles; ++i) { + std::vector values; + // Write 12K (4 values, each 3K) + for (int j = 0; j < kNumPerFile; j++) { + values.push_back(rnd.RandomString(3 << 10)); + ASSERT_OK(Put(Key(i * kNumPerFile + j), values[j])); + } + } + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + MoveFilesToLevel(2); + ASSERT_EQ(2, NumTableFilesAtLevel(2)); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0), + Key(1))); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(7), + Key(8))); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr, + true /* disallow_trivial_move */)); + // This is L0 -> L1 compaction + // The two range tombstones are broken up into two output files + // to limit compaction size. + ASSERT_EQ(2, NumTableFilesAtLevel(1)); +} + +TEST_F(DBRangeDelTest, OversizeCompactionPointKeyWithinRangetombstone) { + // L2 has two files + // L2_0: 0, 1, 2, 3, 4. L2_1: 6, 7, 8 + // L0 has [0, 9) and point key 5 + // max_compaction_bytes is less than the size of L2_0. + // When compacting L0 into L1, the compaction should cut at point key 5. + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.target_file_size_base = 9 * 1024; + options.max_compaction_bytes = 9 * 1024; + DestroyAndReopen(options); + Random rnd(301); + for (int i = 0; i < 9; ++i) { + if (i == 5) { + ++i; + } + ASSERT_OK(Put(Key(i), rnd.RandomString(3 << 10))); + } + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + MoveFilesToLevel(2); + ASSERT_EQ(2, NumTableFilesAtLevel(2)); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0), + Key(9))); + ASSERT_OK(Put(Key(5), rnd.RandomString(1 << 10))); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr, + true /* disallow_trivial_move */)); + ASSERT_EQ(2, NumTableFilesAtLevel(1)); +} + +TEST_F(DBRangeDelTest, OverlappedTombstones) { + const int kNumPerFile = 4, kNumFiles = 2; + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.target_file_size_base = 9 * 1024; + options.max_compaction_bytes = 9 * 1024; + DestroyAndReopen(options); + Random rnd(301); + for (int i = 0; i < kNumFiles; ++i) { + std::vector values; + // Write 12K (4 values, each 3K) + for (int j = 0; j < kNumPerFile; j++) { + values.push_back(rnd.RandomString(3 << 10)); + ASSERT_OK(Put(Key(i * kNumPerFile + j), values[j])); + } + } + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + MoveFilesToLevel(2); + ASSERT_EQ(2, NumTableFilesAtLevel(2)); + + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(1), + Key((kNumFiles)*kNumPerFile + 1))); + ASSERT_OK(db_->Flush(FlushOptions())); + + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr, + true /* disallow_trivial_move */)); + + // The tombstone range is not broken up into multiple SSTs which may incur a + // large compaction with L2. + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + std::vector> files; + ASSERT_OK(dbfull()->TEST_CompactRange(1, nullptr, nullptr, nullptr, + true /* disallow_trivial_move */)); + ASSERT_EQ(1, NumTableFilesAtLevel(2)); + ASSERT_EQ(0, NumTableFilesAtLevel(1)); +} + +TEST_F(DBRangeDelTest, OverlappedKeys) { + const int kNumPerFile = 4, kNumFiles = 2; + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.target_file_size_base = 9 * 1024; + options.max_compaction_bytes = 9 * 1024; + DestroyAndReopen(options); + Random rnd(301); + for (int i = 0; i < kNumFiles; ++i) { + std::vector values; + // Write 12K (4 values, each 3K) + for (int j = 0; j < kNumPerFile; j++) { + values.push_back(rnd.RandomString(3 << 10)); + ASSERT_OK(Put(Key(i * kNumPerFile + j), values[j])); + } + } + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + MoveFilesToLevel(2); + ASSERT_EQ(2, NumTableFilesAtLevel(2)); + + for (int i = 1; i < kNumFiles * kNumPerFile + 1; i++) { + ASSERT_OK(Put(Key(i), "0x123")); + } + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + + // The key range is broken up into three SSTs to avoid a future big compaction + // with the grandparent + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr, + true /* disallow_trivial_move */)); + ASSERT_EQ(3, NumTableFilesAtLevel(1)); + + ASSERT_OK(dbfull()->TEST_CompactRange(1, nullptr, nullptr, nullptr, + true /* disallow_trivial_move */)); + // L1->L2 compaction size is limited to max_compaction_bytes + ASSERT_EQ(3, NumTableFilesAtLevel(2)); + ASSERT_EQ(0, NumTableFilesAtLevel(1)); +} + +TEST_F(DBRangeDelTest, IteratorRefresh) { + // Refreshing an iterator after a range tombstone is added should cause the + // deleted range of keys to disappear. + for (bool sv_changed : {false, true}) { + ASSERT_OK(db_->Put(WriteOptions(), "key1", "value1")); + ASSERT_OK(db_->Put(WriteOptions(), "key2", "value2")); + + auto* iter = db_->NewIterator(ReadOptions()); + ASSERT_OK(iter->status()); + + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + "key2", "key3")); + + if (sv_changed) { + ASSERT_OK(db_->Flush(FlushOptions())); + } + + ASSERT_OK(iter->Refresh()); + ASSERT_OK(iter->status()); + iter->SeekToFirst(); + ASSERT_EQ("key1", iter->key()); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + + delete iter; + } +} + +void VerifyIteratorReachesEnd(InternalIterator* iter) { + ASSERT_TRUE(!iter->Valid() && iter->status().ok()); +} + +void VerifyIteratorReachesEnd(Iterator* iter) { + ASSERT_TRUE(!iter->Valid() && iter->status().ok()); +} + +TEST_F(DBRangeDelTest, IteratorReseek) { + // Range tombstone triggers reseek (seeking to a range tombstone end key) in + // merging iterator. Test set up: + // one memtable: range tombstone [0, 1) + // one immutable memtable: range tombstone [1, 2) + // one L0 file with range tombstone [2, 3) + // one L1 file with range tombstone [3, 4) + // Seek(0) should trigger cascading reseeks at all levels below memtable. + // Seek(1) should trigger cascading reseeks at all levels below immutable + // memtable. SeekToFirst and SeekToLast trigger no reseek. + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + + DestroyAndReopen(options); + // L1 + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(3), + Key(4))); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(1); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + // L0 + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(2), + Key(3))); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + // Immutable memtable + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(1), + Key(2))); + ASSERT_OK(static_cast_with_check(db_)->TEST_SwitchMemtable()); + std::string value; + ASSERT_TRUE(dbfull()->GetProperty(db_->DefaultColumnFamily(), + "rocksdb.num-immutable-mem-table", &value)); + ASSERT_EQ(1, std::stoi(value)); + // live memtable + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0), + Key(1))); + // this memtable is still active + ASSERT_TRUE(dbfull()->GetProperty(db_->DefaultColumnFamily(), + "rocksdb.num-immutable-mem-table", &value)); + ASSERT_EQ(1, std::stoi(value)); + + auto iter = db_->NewIterator(ReadOptions()); + get_perf_context()->Reset(); + iter->Seek(Key(0)); + // Reseeked immutable memtable, L0 and L1 + ASSERT_EQ(get_perf_context()->internal_range_del_reseek_count, 3); + VerifyIteratorReachesEnd(iter); + get_perf_context()->Reset(); + iter->SeekForPrev(Key(1)); + // Reseeked L0 and L1 + ASSERT_EQ(get_perf_context()->internal_range_del_reseek_count, 2); + VerifyIteratorReachesEnd(iter); + get_perf_context()->Reset(); + iter->SeekToFirst(); + ASSERT_EQ(get_perf_context()->internal_range_del_reseek_count, 0); + VerifyIteratorReachesEnd(iter); + iter->SeekToLast(); + ASSERT_EQ(get_perf_context()->internal_range_del_reseek_count, 0); + VerifyIteratorReachesEnd(iter); + delete iter; +} + +TEST_F(DBRangeDelTest, ReseekDuringNextAndPrev) { + // Range tombstone triggers reseek during Next()/Prev() in merging iterator. + // Test set up: + // memtable has: [0, 1) [2, 3) + // L0 has: 2 + // L1 has: 1, 2, 3 + // Seek(0) will reseek to 1 for L0 and L1. Seek(1) will not trigger any + // reseek. Then Next() determines 2 is covered by [2, 3), it will try to + // reseek to 3 for L0 and L1. Similar story for Prev() and SeekForPrev() is + // tested. + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + + DestroyAndReopen(options); + // L1 + ASSERT_OK(db_->Put(WriteOptions(), Key(1), "foo")); + ASSERT_OK(db_->Put(WriteOptions(), Key(2), "foo")); + ASSERT_OK(db_->Put(WriteOptions(), Key(3), "foo")); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(1); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + + // L0 + ASSERT_OK(db_->Put(WriteOptions(), Key(2), "foo")); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + + // Memtable + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0), + Key(1))); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(2), + Key(3))); + + auto iter = db_->NewIterator(ReadOptions()); + auto iter_test_forward = [&] { + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().ToString(), Key(1)); + + get_perf_context()->Reset(); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().ToString(), Key(3)); + // Reseeked L0 and L1 + ASSERT_EQ(get_perf_context()->internal_range_del_reseek_count, 2); + + // Next to Prev + get_perf_context()->Reset(); + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().ToString(), Key(1)); + // Reseeked L0 and L1 + ASSERT_EQ(get_perf_context()->internal_range_del_reseek_count, 2); + + // Prev to Next + get_perf_context()->Reset(); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().ToString(), Key(3)); + // Reseeked L0 and L1 + ASSERT_EQ(get_perf_context()->internal_range_del_reseek_count, 2); + + iter->Next(); + VerifyIteratorReachesEnd(iter); + }; + + get_perf_context()->Reset(); + iter->Seek(Key(0)); + // Reseeked L0 and L1 + ASSERT_EQ(get_perf_context()->internal_range_del_reseek_count, 2); + iter_test_forward(); + get_perf_context()->Reset(); + iter->Seek(Key(1)); + ASSERT_EQ(get_perf_context()->internal_range_del_reseek_count, 0); + iter_test_forward(); + + get_perf_context()->Reset(); + iter->SeekForPrev(Key(2)); + // Reseeked L0 and L1 + ASSERT_EQ(get_perf_context()->internal_range_del_reseek_count, 2); + iter_test_forward(); + get_perf_context()->Reset(); + iter->SeekForPrev(Key(1)); + ASSERT_EQ(get_perf_context()->internal_range_del_reseek_count, 0); + iter_test_forward(); + + get_perf_context()->Reset(); + iter->SeekToFirst(); + ASSERT_EQ(get_perf_context()->internal_range_del_reseek_count, 0); + iter_test_forward(); + + iter->SeekToLast(); + iter->Prev(); + iter_test_forward(); + delete iter; +} + +TEST_F(DBRangeDelTest, TombstoneFromCurrentLevel) { + // Range tombstone triggers reseek when covering key from the same level. + // in merging iterator. Test set up: + // memtable has: [0, 1) + // L0 has: [2, 3), 2 + // L1 has: 1, 2, 3 + // Seek(0) will reseek to 1 for L0 and L1. + // Then Next() will reseek to 3 for L1 since 2 in L0 is covered by [2, 3) in + // L0. + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + + DestroyAndReopen(options); + // L1 + ASSERT_OK(db_->Put(WriteOptions(), Key(1), "foo")); + ASSERT_OK(db_->Put(WriteOptions(), Key(2), "foo")); + ASSERT_OK(db_->Put(WriteOptions(), Key(3), "foo")); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(1); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + + // L0 + ASSERT_OK(db_->Put(WriteOptions(), Key(2), "foo")); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(2), + Key(3))); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + + // Memtable + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0), + Key(1))); + + auto iter = db_->NewIterator(ReadOptions()); + get_perf_context()->Reset(); + iter->Seek(Key(0)); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().ToString(), Key(1)); + ASSERT_EQ(get_perf_context()->internal_range_del_reseek_count, 2); + + get_perf_context()->Reset(); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().ToString(), Key(3)); + ASSERT_EQ(get_perf_context()->internal_range_del_reseek_count, 1); + + delete iter; +} + +class TombstoneTestSstPartitioner : public SstPartitioner { + public: + const char* Name() const override { return "SingleKeySstPartitioner"; } + + PartitionerResult ShouldPartition( + const PartitionerRequest& request) override { + if (cmp->Compare(*request.current_user_key, DBTestBase::Key(5)) == 0) { + return kRequired; + } else { + return kNotRequired; + } + } + + bool CanDoTrivialMove(const Slice& /*smallest_user_key*/, + const Slice& /*largest_user_key*/) override { + return false; + } + + const Comparator* cmp = BytewiseComparator(); +}; + +class TombstoneTestSstPartitionerFactory : public SstPartitionerFactory { + public: + static const char* kClassName() { + return "TombstoneTestSstPartitionerFactory"; + } + const char* Name() const override { return kClassName(); } + + std::unique_ptr CreatePartitioner( + const SstPartitioner::Context& /* context */) const override { + return std::unique_ptr(new TombstoneTestSstPartitioner()); + } +}; + +TEST_F(DBRangeDelTest, TombstoneAcrossFileBoundary) { + // Verify that a range tombstone across file boundary covers keys from older + // levels. Test set up: + // L1_0: 1, 3, [2, 6) L1_1: 5, 7, [2, 6) ([2, 6) is from compaction with + // L1_0) L2 has: 5 + // Seek(1) and then Next() should move the L1 level iterator to + // L1_1. Check if 5 is returned after Next(). + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + options.target_file_size_base = 2 * 1024; + options.max_compaction_bytes = 2 * 1024; + + // Make sure L1 files are split before "5" + auto factory = std::make_shared(); + options.sst_partitioner_factory = factory; + + DestroyAndReopen(options); + + Random rnd(301); + // L2 + // the file should be smaller than max_compaction_bytes, otherwise the file + // will be cut before 7. + ASSERT_OK(db_->Put(WriteOptions(), Key(5), rnd.RandomString(1 << 9))); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(2); + ASSERT_EQ(1, NumTableFilesAtLevel(2)); + + // L1_1 + ASSERT_OK(db_->Put(WriteOptions(), Key(5), rnd.RandomString(1 << 10))); + ASSERT_OK(db_->Put(WriteOptions(), Key(7), rnd.RandomString(1 << 10))); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + + // L1_0 + ASSERT_OK(db_->Put(WriteOptions(), Key(1), rnd.RandomString(1 << 10))); + ASSERT_OK(db_->Put(WriteOptions(), Key(3), rnd.RandomString(1 << 10))); + // Prevent keys being compacted away + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(2), + Key(6))); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(2, NumTableFilesAtLevel(0)); + MoveFilesToLevel(1); + ASSERT_EQ(2, NumTableFilesAtLevel(1)); + + auto iter = db_->NewIterator(ReadOptions()); + get_perf_context()->Reset(); + iter->Seek(Key(1)); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().ToString(), Key(1)); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().ToString(), Key(7)); + // 1 reseek into L2 when key 5 in L2 is covered by [2, 6) from L1 + ASSERT_EQ(get_perf_context()->internal_range_del_reseek_count, 1); + + delete iter; + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(DBRangeDelTest, NonOverlappingTombstonAtBoundary) { + // Verify that a range tombstone across file boundary covers keys from older + // levels. + // Test set up: + // L1_0: 1, 3, [4, 7) L1_1: 6, 8, [4, 7) + // L2: 5 + // Note that [4, 7) is at end of L1_0 and not overlapping with any point key + // in L1_0. [4, 7) from L1_0 should cover 5 is sentinel works + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + options.target_file_size_base = 2 * 1024; + options.level_compaction_dynamic_file_size = false; + DestroyAndReopen(options); + + Random rnd(301); + // L2 + ASSERT_OK(db_->Put(WriteOptions(), Key(5), rnd.RandomString(4 << 10))); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(2); + ASSERT_EQ(1, NumTableFilesAtLevel(2)); + + // L1_1 + ASSERT_OK(db_->Put(WriteOptions(), Key(6), rnd.RandomString(4 << 10))); + ASSERT_OK(db_->Put(WriteOptions(), Key(8), rnd.RandomString(4 << 10))); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + + // L1_0 + ASSERT_OK(db_->Put(WriteOptions(), Key(1), rnd.RandomString(4 << 10))); + ASSERT_OK(db_->Put(WriteOptions(), Key(3), rnd.RandomString(4 << 10))); + // Prevent keys being compacted away + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(4), + Key(7))); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(2, NumTableFilesAtLevel(0)); + MoveFilesToLevel(1); + ASSERT_EQ(2, NumTableFilesAtLevel(1)); + + auto iter = db_->NewIterator(ReadOptions()); + iter->Seek(Key(3)); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(3)); + get_perf_context()->Reset(); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().ToString(), Key(8)); + // 1 reseek into L1 since 5 from L2 is covered by [4, 7) from L1 + ASSERT_EQ(get_perf_context()->internal_range_del_reseek_count, 1); + for (auto& k : {4, 5, 6}) { + get_perf_context()->Reset(); + iter->Seek(Key(k)); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().ToString(), Key(8)); + // 1 reseek into L1 + ASSERT_EQ(get_perf_context()->internal_range_del_reseek_count, 1); + } + delete iter; +} + +TEST_F(DBRangeDelTest, OlderLevelHasNewerData) { + // L1_0: 1, 3, [2, 7) L1_1: 5, 6 at a newer sequence number than [2, 7) + // Compact L1_1 to L2. Seek(3) should not skip 5 or 6. + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + options.target_file_size_base = 3 * 1024; + DestroyAndReopen(options); + + Random rnd(301); + // L1_0 + ASSERT_OK(db_->Put(WriteOptions(), Key(1), rnd.RandomString(4 << 10))); + ASSERT_OK(db_->Put(WriteOptions(), Key(3), rnd.RandomString(4 << 10))); + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(2), + Key(7))); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(1); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + + // L1_1 + ASSERT_OK(db_->Put(WriteOptions(), Key(5), rnd.RandomString(4 << 10))); + ASSERT_OK(db_->Put(WriteOptions(), Key(6), rnd.RandomString(4 << 10))); + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + MoveFilesToLevel(1); + ASSERT_EQ(2, NumTableFilesAtLevel(1)); + + auto key = Key(6); + Slice begin(key); + EXPECT_OK(dbfull()->TEST_CompactRange(1, &begin, nullptr)); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + ASSERT_EQ(1, NumTableFilesAtLevel(2)); + + auto iter = db_->NewIterator(ReadOptions()); + iter->Seek(Key(3)); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().ToString(), Key(5)); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().ToString(), Key(6)); + delete iter; + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(DBRangeDelTest, LevelBoundaryDefinedByTombstone) { + // L1 has: 1, 2, [4, 5) + // L2 has: 4 + // Seek(3), which is over all points keys in L1, check whether + // sentinel key from L1 works in this case. + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + options.target_file_size_base = 3 * 1024; + DestroyAndReopen(options); + Random rnd(301); + // L2 + ASSERT_OK(db_->Put(WriteOptions(), Key(4), "foo")); + ASSERT_OK(db_->Flush(FlushOptions())); + const Snapshot* snapshot = db_->GetSnapshot(); + MoveFilesToLevel(2); + ASSERT_EQ(1, NumTableFilesAtLevel(2)); + + // L1_0 + ASSERT_OK(db_->Put(WriteOptions(), Key(1), rnd.RandomString(4 << 10))); + ASSERT_OK(db_->Put(WriteOptions(), Key(2), rnd.RandomString(4 << 10))); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(4), + Key(5))); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(1); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + ASSERT_EQ(1, NumTableFilesAtLevel(2)); + + auto iter = db_->NewIterator(ReadOptions()); + iter->Seek(Key(3)); + ASSERT_TRUE(!iter->Valid()); + ASSERT_OK(iter->status()); + + get_perf_context()->Reset(); + iter->SeekForPrev(Key(5)); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(2)); + db_->ReleaseSnapshot(snapshot); + delete iter; +} + +TEST_F(DBRangeDelTest, TombstoneOnlyFile) { + // L1_0: 1, 2, L1_1: [3, 5) + // L2: 3 + // Seek(2) then Next() should advance L1 iterator into L1_1. + // If sentinel works with tombstone only file, it should cover the key in L2. + // Similar story for SeekForPrev(4). + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + options.target_file_size_base = 3 * 1024; + + DestroyAndReopen(options); + Random rnd(301); + // L2 + ASSERT_OK(db_->Put(WriteOptions(), Key(3), "foo")); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(2); + ASSERT_EQ(1, NumTableFilesAtLevel(2)); + + // L1_0 + ASSERT_OK(db_->Put(WriteOptions(), Key(1), rnd.RandomString(4 << 10))); + ASSERT_OK(db_->Put(WriteOptions(), Key(2), rnd.RandomString(4 << 10))); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(1); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + ASSERT_EQ(1, NumTableFilesAtLevel(2)); + + // L1_1 + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(3), + Key(5))); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(1); + ASSERT_EQ(2, NumTableFilesAtLevel(1)); + ASSERT_EQ(1, NumTableFilesAtLevel(2)); + + auto iter = db_->NewIterator(ReadOptions()); + iter->Seek(Key(2)); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(2)); + iter->Next(); + VerifyIteratorReachesEnd(iter); + iter->SeekForPrev(Key(4)); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(2)); + iter->Next(); + VerifyIteratorReachesEnd(iter); + delete iter; +} + +void VerifyIteratorKey(InternalIterator* iter, + const std::vector& expected_keys, + bool forward = true) { + for (auto& key : expected_keys) { + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->user_key(), key); + if (forward) { + iter->Next(); + } else { + iter->Prev(); + } + } +} + +TEST_F(DBRangeDelTest, TombstoneOnlyLevel) { + // L1 [3, 5) + // L2 has: 3, 4 + // Any kind of iterator seek should skip 3 and 4 in L2. + // L1 level iterator should produce sentinel key. + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + options.target_file_size_base = 3 * 1024; + + DestroyAndReopen(options); + // L2 + ASSERT_OK(db_->Put(WriteOptions(), Key(3), "foo")); + ASSERT_OK(db_->Put(WriteOptions(), Key(4), "bar")); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(2); + ASSERT_EQ(1, NumTableFilesAtLevel(2)); + + // L1 + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(3), + Key(5))); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(1); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + + auto iter = db_->NewIterator(ReadOptions()); + get_perf_context()->Reset(); + uint64_t expected_reseek = 0; + for (auto i = 0; i < 7; ++i) { + iter->Seek(Key(i)); + VerifyIteratorReachesEnd(iter); + if (i < 5) { + ++expected_reseek; + } + ASSERT_EQ(get_perf_context()->internal_range_del_reseek_count, + expected_reseek); + iter->SeekForPrev(Key(i)); + VerifyIteratorReachesEnd(iter); + if (i > 2) { + ++expected_reseek; + } + ASSERT_EQ(get_perf_context()->internal_range_del_reseek_count, + expected_reseek); + iter->SeekToFirst(); + VerifyIteratorReachesEnd(iter); + ASSERT_EQ(get_perf_context()->internal_range_del_reseek_count, + ++expected_reseek); + iter->SeekToLast(); + VerifyIteratorReachesEnd(iter); + ASSERT_EQ(get_perf_context()->internal_range_del_reseek_count, + ++expected_reseek); + } + delete iter; + + // Check L1 LevelIterator behavior + ColumnFamilyData* cfd = + static_cast_with_check(db_->DefaultColumnFamily()) + ->cfd(); + SuperVersion* sv = cfd->GetSuperVersion(); + Arena arena; + ReadOptions read_options; + MergeIteratorBuilder merge_iter_builder(&cfd->internal_comparator(), &arena, + false /* prefix seek */); + InternalIterator* level_iter = sv->current->TEST_GetLevelIterator( + read_options, &merge_iter_builder, 1 /* level */, true); + // This is needed to make LevelIterator range tombstone aware + auto miter = merge_iter_builder.Finish(); + auto k = Key(3); + IterKey target; + target.SetInternalKey(k, kMaxSequenceNumber, kValueTypeForSeek); + level_iter->Seek(target.GetInternalKey()); + // sentinel key (file boundary as a fake key) + VerifyIteratorKey(level_iter, {Key(5)}); + VerifyIteratorReachesEnd(level_iter); + + k = Key(5); + target.SetInternalKey(k, 0, kValueTypeForSeekForPrev); + level_iter->SeekForPrev(target.GetInternalKey()); + VerifyIteratorKey(level_iter, {Key(3)}, false); + VerifyIteratorReachesEnd(level_iter); + + level_iter->SeekToFirst(); + VerifyIteratorKey(level_iter, {Key(5)}); + VerifyIteratorReachesEnd(level_iter); + + level_iter->SeekToLast(); + VerifyIteratorKey(level_iter, {Key(3)}, false); + VerifyIteratorReachesEnd(level_iter); + + miter->~InternalIterator(); +} + +TEST_F(DBRangeDelTest, TombstoneOnlyWithOlderVisibleKey) { + // L1: [3, 5) + // L2: 2, 4, 5 + // 2 and 5 should be visible + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + options.target_file_size_base = 3 * 1024; + + DestroyAndReopen(options); + // L2 + ASSERT_OK(db_->Put(WriteOptions(), Key(2), "foo")); + ASSERT_OK(db_->Put(WriteOptions(), Key(4), "bar")); + ASSERT_OK(db_->Put(WriteOptions(), Key(5), "foobar")); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(2); + ASSERT_EQ(1, NumTableFilesAtLevel(2)); + + // l1 + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(3), + Key(5))); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(1); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + + auto iter = db_->NewIterator(ReadOptions()); + auto iter_test_backward = [&] { + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(5)); + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(2)); + iter->Prev(); + VerifyIteratorReachesEnd(iter); + }; + auto iter_test_forward = [&] { + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(2)); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(5)); + iter->Next(); + VerifyIteratorReachesEnd(iter); + }; + iter->Seek(Key(4)); + iter_test_backward(); + iter->SeekForPrev(Key(4)); + iter->Next(); + iter_test_backward(); + + iter->Seek(Key(4)); + iter->Prev(); + iter_test_forward(); + iter->SeekForPrev(Key(4)); + iter_test_forward(); + + iter->SeekToFirst(); + iter_test_forward(); + iter->SeekToLast(); + iter_test_backward(); + + delete iter; +} + +TEST_F(DBRangeDelTest, TombstoneSentinelDirectionChange) { + // L1: 7 + // L2: [4, 6) + // L3: 4 + // Seek(5) will have sentinel key 6 at the top of minHeap in merging iterator. + // then do a prev, how would sentinel work? + // Redo the test after Put(5) into L1 so that there is a visible key in range + // [4, 6). + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + options.target_file_size_base = 3 * 1024; + + DestroyAndReopen(options); + // L3 + ASSERT_OK(db_->Put(WriteOptions(), Key(4), "bar")); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(3); + ASSERT_EQ(1, NumTableFilesAtLevel(3)); + // L2 + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(4), + Key(6))); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(2); + ASSERT_EQ(1, NumTableFilesAtLevel(2)); + + // L1 + ASSERT_OK(db_->Put(WriteOptions(), Key(7), "foobar")); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(1); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + + auto iter = db_->NewIterator(ReadOptions()); + iter->Seek(Key(5)); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(7)); + iter->Prev(); + ASSERT_TRUE(!iter->Valid() && iter->status().ok()); + delete iter; + + ASSERT_OK(db_->Put(WriteOptions(), Key(5), "foobar")); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(1); + ASSERT_EQ(2, NumTableFilesAtLevel(1)); + + iter = db_->NewIterator(ReadOptions()); + iter->Seek(Key(5)); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(5)); + iter->Prev(); + ASSERT_TRUE(!iter->Valid() && iter->status().ok()); + delete iter; +} + +// Right sentinel tested in many test cases above +TEST_F(DBRangeDelTest, LeftSentinelKeyTest) { + // L1_0: 0, 1 L1_1: [2, 3), 5 + // L2: 2 + // SeekForPrev(4) should give 1 due to sentinel key keeping [2, 3) alive. + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + options.target_file_size_base = 3 * 1024; + options.max_compaction_bytes = 2048; + + DestroyAndReopen(options); + // L2 + ASSERT_OK(db_->Put(WriteOptions(), Key(2), "foo")); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(2); + ASSERT_EQ(1, NumTableFilesAtLevel(2)); + + // L1_0 + Random rnd(301); + ASSERT_OK(db_->Put(WriteOptions(), Key(0), rnd.RandomString(4 << 10))); + ASSERT_OK(db_->Put(WriteOptions(), Key(1), rnd.RandomString(4 << 10))); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(1); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + + // L1_1 + ASSERT_OK(db_->Put(WriteOptions(), Key(5), "bar")); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(2), + Key(3))); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(1); + ASSERT_EQ(2, NumTableFilesAtLevel(1)); + + auto iter = db_->NewIterator(ReadOptions()); + iter->SeekForPrev(Key(4)); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(1)); + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(0)); + iter->Prev(); + ASSERT_TRUE(!iter->Valid()); + ASSERT_OK(iter->status()); + delete iter; +} + +TEST_F(DBRangeDelTest, LeftSentinelKeyTestWithNewerKey) { + // L1_0: 1, 2 newer than L1_1, L1_1: [2, 4), 5 + // L2: 3 + // SeekForPrev(4) then Prev() should give 2 and then 1. + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + options.target_file_size_base = 3 * 1024; + options.max_compaction_bytes = 3 * 1024; + + DestroyAndReopen(options); + // L2 + ASSERT_OK(db_->Put(WriteOptions(), Key(3), "foo")); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(2); + ASSERT_EQ(1, NumTableFilesAtLevel(2)); + + // L1_1 + ASSERT_OK(db_->Put(WriteOptions(), Key(5), "bar")); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(2), + Key(4))); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(1); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + + // L1_0 + Random rnd(301); + ASSERT_OK(db_->Put(WriteOptions(), Key(1), rnd.RandomString(4 << 10))); + ASSERT_OK(db_->Put(WriteOptions(), Key(2), rnd.RandomString(4 << 10))); + // Used to verify sequence number of iterator key later. + auto seq = dbfull()->TEST_GetLastVisibleSequence(); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(1); + ASSERT_EQ(2, NumTableFilesAtLevel(1)); + + Arena arena; + InternalKeyComparator icmp(options.comparator); + ReadOptions read_options; + ScopedArenaIterator iter; + iter.set( + dbfull()->NewInternalIterator(read_options, &arena, kMaxSequenceNumber)); + + auto k = Key(4); + IterKey target; + target.SetInternalKey(k, 0 /* sequence_number */, kValueTypeForSeekForPrev); + iter->SeekForPrev(target.GetInternalKey()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->user_key(), Key(2)); + SequenceNumber actual_seq; + ValueType type; + UnPackSequenceAndType(ExtractInternalKeyFooter(iter->key()), &actual_seq, + &type); + ASSERT_EQ(seq, actual_seq); + // might as well check type + ASSERT_EQ(type, kTypeValue); + + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->user_key(), Key(1)); + iter->Prev(); + ASSERT_TRUE(!iter->Valid()); + ASSERT_OK(iter->status()); +} + +TEST_F(DBRangeDelTest, SentinelKeyCommonCaseTest) { + // L1 has 3 files + // L1_0: 1, 2 L1_1: [3, 4) 5, 6, [7, 8) L1_2: 9 + // Check iterator operations on LevelIterator. + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + options.target_file_size_base = 3 * 1024; + + DestroyAndReopen(options); + Random rnd(301); + // L1_0 + ASSERT_OK(db_->Put(WriteOptions(), Key(1), rnd.RandomString(4 << 10))); + ASSERT_OK(db_->Put(WriteOptions(), Key(2), rnd.RandomString(4 << 10))); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(1); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + + // L1_1 + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(3), + Key(4))); + ASSERT_OK(db_->Put(WriteOptions(), Key(5), rnd.RandomString(4 << 10))); + ASSERT_OK(db_->Put(WriteOptions(), Key(6), rnd.RandomString(4 << 10))); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(7), + Key(8))); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(1); + ASSERT_EQ(2, NumTableFilesAtLevel(1)); + + // L1_2 + ASSERT_OK(db_->Put(WriteOptions(), Key(9), rnd.RandomString(4 << 10))); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(1); + ASSERT_EQ(3, NumTableFilesAtLevel(1)); + + ColumnFamilyData* cfd = + static_cast_with_check(db_->DefaultColumnFamily()) + ->cfd(); + SuperVersion* sv = cfd->GetSuperVersion(); + Arena arena; + ReadOptions read_options; + MergeIteratorBuilder merge_iter_builder(&cfd->internal_comparator(), &arena, + false /* prefix seek */); + InternalIterator* level_iter = sv->current->TEST_GetLevelIterator( + read_options, &merge_iter_builder, 1 /* level */, true); + // This is needed to make LevelIterator range tombstone aware + auto miter = merge_iter_builder.Finish(); + auto k = Key(7); + IterKey target; + target.SetInternalKey(k, kMaxSequenceNumber, kValueTypeForSeek); + level_iter->Seek(target.GetInternalKey()); + // The last Key(9) is a sentinel key. + VerifyIteratorKey(level_iter, {Key(8), Key(9), Key(9)}); + ASSERT_TRUE(!level_iter->Valid() && level_iter->status().ok()); + + k = Key(6); + target.SetInternalKey(k, kMaxSequenceNumber, kValueTypeForSeek); + level_iter->Seek(target.GetInternalKey()); + VerifyIteratorKey(level_iter, {Key(6), Key(8), Key(9), Key(9)}); + ASSERT_TRUE(!level_iter->Valid() && level_iter->status().ok()); + + k = Key(4); + target.SetInternalKey(k, 0, kValueTypeForSeekForPrev); + level_iter->SeekForPrev(target.GetInternalKey()); + VerifyIteratorKey(level_iter, {Key(3), Key(2), Key(1), Key(1)}, false); + ASSERT_TRUE(!level_iter->Valid() && level_iter->status().ok()); + + k = Key(5); + target.SetInternalKey(k, 0, kValueTypeForSeekForPrev); + level_iter->SeekForPrev(target.GetInternalKey()); + VerifyIteratorKey(level_iter, {Key(5), Key(3), Key(2), Key(1), Key(1)}, + false); + + level_iter->SeekToFirst(); + VerifyIteratorKey(level_iter, {Key(1), Key(2), Key(2), Key(5), Key(6), Key(8), + Key(9), Key(9)}); + ASSERT_TRUE(!level_iter->Valid() && level_iter->status().ok()); + + level_iter->SeekToLast(); + VerifyIteratorKey( + level_iter, + {Key(9), Key(9), Key(6), Key(5), Key(3), Key(2), Key(1), Key(1)}, false); + ASSERT_TRUE(!level_iter->Valid() && level_iter->status().ok()); + + miter->~InternalIterator(); +} + +TEST_F(DBRangeDelTest, PrefixSentinelKey) { + // L1: ['aaaa', 'aaad'), 'bbbb' + // L2: 'aaac', 'aaae' + // Prefix extracts first 3 chars + // Seek('aaab') should give 'aaae' as first key. + // This is to test a previous bug where prefix seek sees there is no prefix in + // the SST file, and will just set file iter to null in LevelIterator and may + // just skip to the next SST file. But in this case, we should keep the file's + // tombstone alive. + Options options = CurrentOptions(); + options.compression = kNoCompression; + options.disable_auto_compactions = true; + options.prefix_extractor.reset(NewFixedPrefixTransform(3)); + BlockBasedTableOptions table_options; + table_options.filter_policy.reset(NewBloomFilterPolicy(10, false)); + table_options.whole_key_filtering = false; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DestroyAndReopen(options); + Random rnd(301); + + // L2: + ASSERT_OK(db_->Put(WriteOptions(), "aaac", rnd.RandomString(10))); + ASSERT_OK(db_->Put(WriteOptions(), "aaae", rnd.RandomString(10))); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(2); + ASSERT_EQ(1, NumTableFilesAtLevel(2)); + + // L1 + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "aaaa", + "aaad")); + ASSERT_OK(db_->Put(WriteOptions(), "bbbb", rnd.RandomString(10))); + ASSERT_OK(db_->Flush(FlushOptions())); + MoveFilesToLevel(1); + ASSERT_EQ(1, NumTableFilesAtLevel(1)); + + auto iter = db_->NewIterator(ReadOptions()); + iter->Seek("aaab"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), "aaae"); + delete iter; +} + +TEST_F(DBRangeDelTest, RefreshMemtableIter) { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + DestroyAndReopen(options); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "z")); + ReadOptions ro; + ro.read_tier = kMemtableTier; + std::unique_ptr iter{db_->NewIterator(ro)}; + ASSERT_OK(Flush()); + // First refresh reinits iter, which had a bug where + // iter.memtable_range_tombstone_iter_ was not set to nullptr, and caused + // subsequent refresh to double free. + ASSERT_OK(iter->Refresh()); + ASSERT_OK(iter->Refresh()); +} + +TEST_F(DBRangeDelTest, RangeTombstoneRespectIterateUpperBound) { + // Memtable: a, [b, bz) + // Do a Seek on `a` with iterate_upper_bound being az + // range tombstone [b, bz) should not be processed (added to and + // popped from the min_heap in MergingIterator). + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + DestroyAndReopen(options); + + ASSERT_OK(Put("a", "bar")); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "b", "bz")); + + // I could not find a cleaner way to test this without relying on + // implementation detail. Tried to test the value of + // `internal_range_del_reseek_count` but that did not work + // since BlockBasedTable iterator becomes !Valid() when point key + // is out of bound and that reseek only happens when a point key + // is covered by some range tombstone. + SyncPoint::GetInstance()->SetCallBack("MergeIterator::PopDeleteRangeStart", + [](void*) { + // there should not be any range + // tombstone in the heap. + FAIL(); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + ReadOptions read_opts; + std::string upper_bound = "az"; + Slice upper_bound_slice = upper_bound; + read_opts.iterate_upper_bound = &upper_bound_slice; + std::unique_ptr iter{db_->NewIterator(read_opts)}; + iter->Seek("a"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), "a"); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); +} + +TEST_F(DBRangeDelTest, RangetombesoneCompensateFilesize) { + Options opts = CurrentOptions(); + opts.disable_auto_compactions = true; + DestroyAndReopen(opts); + + std::vector values; + Random rnd(301); + // file in L2 + values.push_back(rnd.RandomString(1 << 10)); + ASSERT_OK(Put("a", values.back())); + values.push_back(rnd.RandomString(1 << 10)); + ASSERT_OK(Put("b", values.back())); + ASSERT_OK(Flush()); + MoveFilesToLevel(2); + uint64_t l2_size = 0; + ASSERT_OK(Size("a", "c", 0 /* cf */, &l2_size)); + ASSERT_GT(l2_size, 0); + // file in L1 + values.push_back(rnd.RandomString(1 << 10)); + ASSERT_OK(Put("d", values.back())); + values.push_back(rnd.RandomString(1 << 10)); + ASSERT_OK(Put("e", values.back())); + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + uint64_t l1_size = 0; + ASSERT_OK(Size("d", "f", 0 /* cf */, &l1_size)); + ASSERT_GT(l1_size, 0); + + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "f")); + ASSERT_OK(Flush()); + // Range deletion compensated size computed during flush time + std::vector> level_to_files; + dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(), + &level_to_files); + ASSERT_EQ(level_to_files[0].size(), 1); + ASSERT_EQ(level_to_files[0][0].compensated_range_deletion_size, + l1_size + l2_size); + ASSERT_EQ(level_to_files[1].size(), 1); + ASSERT_EQ(level_to_files[1][0].compensated_range_deletion_size, 0); + ASSERT_EQ(level_to_files[2].size(), 1); + ASSERT_EQ(level_to_files[2][0].compensated_range_deletion_size, 0); + + // Range deletion compensated size computed during compaction time + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr, + true /* disallow_trivial_move */)); + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + ASSERT_EQ(NumTableFilesAtLevel(1), 1); + ASSERT_EQ(NumTableFilesAtLevel(2), 1); + dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(), + &level_to_files); + ASSERT_EQ(level_to_files[1].size(), 1); + ASSERT_EQ(level_to_files[1][0].compensated_range_deletion_size, l2_size); + ASSERT_EQ(level_to_files[2].size(), 1); + ASSERT_EQ(level_to_files[2][0].compensated_range_deletion_size, 0); +} + +TEST_F(DBRangeDelTest, RangetombesoneCompensateFilesizePersistDuringReopen) { + Options opts = CurrentOptions(); + opts.disable_auto_compactions = true; + DestroyAndReopen(opts); + + std::vector values; + Random rnd(301); + values.push_back(rnd.RandomString(1 << 10)); + ASSERT_OK(Put("a", values.back())); + values.push_back(rnd.RandomString(1 << 10)); + ASSERT_OK(Put("b", values.back())); + ASSERT_OK(Flush()); + MoveFilesToLevel(2); + + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "c")); + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "z")); + ASSERT_OK(Flush()); + + std::vector> level_to_files; + dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(), + &level_to_files); + ASSERT_EQ(level_to_files[0].size(), 1); + ASSERT_EQ(level_to_files[1].size(), 1); + ASSERT_EQ(level_to_files[2].size(), 1); + uint64_t l2_size = level_to_files[2][0].fd.GetFileSize(); + uint64_t l1_size = level_to_files[1][0].fd.GetFileSize(); + ASSERT_GT(l2_size, 0); + ASSERT_GT(l1_size, 0); + ASSERT_EQ(level_to_files[0][0].compensated_range_deletion_size, + l1_size + l2_size); + ASSERT_EQ(level_to_files[1][0].compensated_range_deletion_size, l2_size); + + Reopen(opts); + dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(), + &level_to_files); + ASSERT_EQ(level_to_files[0].size(), 1); + ASSERT_EQ(level_to_files[0][0].compensated_range_deletion_size, + l1_size + l2_size); + ASSERT_EQ(level_to_files[1].size(), 1); + ASSERT_EQ(level_to_files[1][0].compensated_range_deletion_size, l2_size); +} + +TEST_F(DBRangeDelTest, SingleKeyFile) { + // Test for a bug fix where a range tombstone could be added + // to an SST file while is not within the file's key range. + // Create 3 files in L0 and then L1 where all keys have the same user key + // `Key(2)`. The middle file will contain Key(2)@6 and Key(2)@5. Before fix, + // the range tombstone [Key(2), Key(5))@2 would be added to this file during + // compaction, but it is not in this file's key range. + Options opts = CurrentOptions(); + opts.disable_auto_compactions = true; + opts.target_file_size_base = 1 << 10; + opts.level_compaction_dynamic_file_size = false; + DestroyAndReopen(opts); + + // prevent range tombstone drop + std::vector snapshots; + snapshots.push_back(db_->GetSnapshot()); + + // write a key to bottommost file so the compactions below + // are not bottommost compactions and will calculate + // compensated range tombstone size. Before bug fix, an assert would fail + // during this process. + Random rnd(301); + ASSERT_OK(Put(Key(2), rnd.RandomString(8 << 10))); + ASSERT_OK(Flush()); + MoveFilesToLevel(6); + + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(2), + Key(5))); + snapshots.push_back(db_->GetSnapshot()); + std::vector values; + + values.push_back(rnd.RandomString(8 << 10)); + ASSERT_OK(Put(Key(2), rnd.RandomString(8 << 10))); + snapshots.push_back(db_->GetSnapshot()); + ASSERT_OK(Flush()); + + ASSERT_OK(Put(Key(2), rnd.RandomString(8 << 10))); + snapshots.push_back(db_->GetSnapshot()); + ASSERT_OK(Flush()); + + ASSERT_OK(Put(Key(2), rnd.RandomString(8 << 10))); + snapshots.push_back(db_->GetSnapshot()); + ASSERT_OK(Flush()); + + ASSERT_EQ(NumTableFilesAtLevel(0), 3); + CompactRangeOptions co; + co.bottommost_level_compaction = BottommostLevelCompaction::kForce; + + ASSERT_OK(dbfull()->RunManualCompaction( + static_cast_with_check(db_->DefaultColumnFamily()) + ->cfd(), + 0, 1, co, nullptr, nullptr, true, true, + std::numeric_limits::max() /*max_file_num_to_ignore*/, + "" /*trim_ts*/)); + + for (const auto s : snapshots) { + db_->ReleaseSnapshot(s); + } +} + +TEST_F(DBRangeDelTest, DoubleCountRangeTombstoneCompensatedSize) { + // Test for a bug fix if a file has multiple range tombstones + // with same start and end key but with different sequence numbers, + // we should only calculate compensated range tombstone size + // for one of them. + Options opts = CurrentOptions(); + opts.disable_auto_compactions = true; + DestroyAndReopen(opts); + + std::vector values; + Random rnd(301); + // file in L2 + ASSERT_OK(Put(Key(1), rnd.RandomString(1 << 10))); + ASSERT_OK(Put(Key(2), rnd.RandomString(1 << 10))); + ASSERT_OK(Flush()); + MoveFilesToLevel(2); + uint64_t l2_size = 0; + ASSERT_OK(Size(Key(1), Key(3), 0 /* cf */, &l2_size)); + ASSERT_GT(l2_size, 0); + + // file in L1 + ASSERT_OK(Put(Key(3), rnd.RandomString(1 << 10))); + ASSERT_OK(Put(Key(4), rnd.RandomString(1 << 10))); + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + uint64_t l1_size = 0; + ASSERT_OK(Size(Key(3), Key(5), 0 /* cf */, &l1_size)); + ASSERT_GT(l1_size, 0); + + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(1), + Key(5))); + // so that the range tombstone above is not dropped + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(1), + Key(5))); + ASSERT_OK(Flush()); + // Range deletion compensated size computed during flush time + std::vector> level_to_files; + dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(), + &level_to_files); + ASSERT_EQ(level_to_files[0].size(), 1); + // instead of 2 * (l1_size + l2_size) + ASSERT_EQ(level_to_files[0][0].compensated_range_deletion_size, + l1_size + l2_size); + + // Range deletion compensated size computed during compaction time + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr, + true /* disallow_trivial_move */)); + dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(), + &level_to_files); + ASSERT_EQ(level_to_files[1].size(), 1); + ASSERT_EQ(level_to_files[1][0].compensated_range_deletion_size, l2_size); + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(DBRangeDelTest, AddRangeDelsSameLowerAndUpperBound) { + // Test for an edge case where CompactionOutputs::AddRangeDels() + // is called with an empty range: `range_tombstone_lower_bound_` is not empty + // and have the same user_key and sequence number as `next_table_min_key. + // This used to cause file's smallest and largest key to be incorrectly set + // such that smallest > largest, and fail some assertions in iterator and/or + // assertion in VersionSet::ApproximateSize(). + Options opts = CurrentOptions(); + opts.disable_auto_compactions = true; + opts.target_file_size_base = 1 << 10; + opts.level_compaction_dynamic_file_size = false; + DestroyAndReopen(opts); + + Random rnd(301); + // Create file at bottommost level so the manual compaction below is + // non-bottommost level and goes through code path like compensate range + // tombstone size. + ASSERT_OK(Put(Key(1), "v1")); + ASSERT_OK(Put(Key(4), "v2")); + ASSERT_OK(Flush()); + MoveFilesToLevel(6); + + ASSERT_OK(Put(Key(1), rnd.RandomString(4 << 10))); + ASSERT_OK(Put(Key(3), rnd.RandomString(4 << 10))); + // So Key(3) does not get dropped. + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(2), + Key(4))); + ASSERT_OK(Flush()); + + ASSERT_OK(Put(Key(3), rnd.RandomString(4 << 10))); + ASSERT_OK(Put(Key(4), rnd.RandomString(4 << 10))); + ASSERT_OK(Flush()); + + MoveFilesToLevel(1); + // Each file will have two keys, with Key(3) straddle between two files. + // File 1: Key(1)@1, Key(3)@6, DeleteRange ends at Key(3)@6 + // File 2: Key(3)@4, Key(4)@7, DeleteRange start from Key(3)@4 + ASSERT_EQ(NumTableFilesAtLevel(1), 2); + + // Manually update compaction output file cutting decisions + // to cut before range tombstone sentinel Key(3)@4 + // and the point key Key(3)@4 itself + SyncPoint::GetInstance()->SetCallBack( + "CompactionOutputs::ShouldStopBefore::manual_decision", [opts](void* p) { + auto* pair = (std::pair*)p; + if ((opts.comparator->Compare(ExtractUserKey(pair->second), Key(3)) == + 0) && + (GetInternalKeySeqno(pair->second) <= 4)) { + *(pair->first) = true; + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + std::string begin_key = Key(0); + std::string end_key = Key(5); + Slice begin_slice{begin_key}; + Slice end_slice{end_key}; + ASSERT_OK(dbfull()->RunManualCompaction( + static_cast_with_check(db_->DefaultColumnFamily()) + ->cfd(), + 1, 2, CompactRangeOptions(), &begin_slice, &end_slice, true, + true /* disallow_trivial_move */, + std::numeric_limits::max() /*max_file_num_to_ignore*/, + "" /*trim_ts*/)); + // iterate through to check if any assertion breaks + std::unique_ptr iter{db_->NewIterator(ReadOptions())}; + iter->SeekToFirst(); + std::vector expected{1, 3, 4}; + for (auto i : expected) { + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(i)); + iter->Next(); + } + ASSERT_TRUE(iter->status().ok() && !iter->Valid()); + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(DBRangeDelTest, AddRangeDelsSingleUserKeyTombstoneOnlyFile) { + // Test for an edge case where CompactionOutputs::AddRangeDels() + // is called with an SST file that has no point keys, and that + // the lower bound and upper bound have the same user key. + // This could cause a file's smallest and largest key to be incorrectly set + // such that smallest > largest, and fail some assertions in iterator and/or + // assertion in VersionSet::ApproximateSize(). + Options opts = CurrentOptions(); + opts.disable_auto_compactions = true; + opts.target_file_size_base = 1 << 10; + opts.level_compaction_dynamic_file_size = false; + DestroyAndReopen(opts); + + Random rnd(301); + // Create file at bottommost level so the manual compaction below is + // non-bottommost level and goes through code path like compensate range + // tombstone size. + ASSERT_OK(Put(Key(1), "v1")); + ASSERT_OK(Put(Key(4), "v2")); + ASSERT_OK(Flush()); + MoveFilesToLevel(6); + + ASSERT_OK(Put(Key(1), rnd.RandomString(10))); + // Key(3)@4 + ASSERT_OK(Put(Key(3), rnd.RandomString(10))); + const Snapshot* snapshot1 = db_->GetSnapshot(); + // Key(3)@5 + ASSERT_OK(Put(Key(3), rnd.RandomString(10))); + const Snapshot* snapshot2 = db_->GetSnapshot(); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(2), + Key(4))); + // Key(3)@7 + ASSERT_OK(Put(Key(3), rnd.RandomString(10))); + ASSERT_OK(Flush()); + + // L0 -> L1 compaction: cut output into two files: + // File 1: Key(1), Key(3)@7, Range tombstone ends at Key(3)@7 + // File 2: Key(3)@5, Key(3)@4, Range tombstone starts from Key(3)@5 + SyncPoint::GetInstance()->SetCallBack( + "CompactionOutputs::ShouldStopBefore::manual_decision", [opts](void* p) { + auto* pair = (std::pair*)p; + if ((opts.comparator->Compare(ExtractUserKey(pair->second), Key(3)) == + 0) && + (GetInternalKeySeqno(pair->second) <= 6)) { + *(pair->first) = true; + SyncPoint::GetInstance()->DisableProcessing(); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + std::string begin_key = Key(0); + std::string end_key = Key(5); + Slice begin_slice{begin_key}; + Slice end_slice{end_key}; + ASSERT_OK(dbfull()->RunManualCompaction( + static_cast_with_check(db_->DefaultColumnFamily()) + ->cfd(), + 0, 1, CompactRangeOptions(), &begin_slice, &end_slice, true, + true /* disallow_trivial_move */, + std::numeric_limits::max() /*max_file_num_to_ignore*/, + "" /*trim_ts*/)); + ASSERT_EQ(NumTableFilesAtLevel(1), 2); + + // L1 -> L2 compaction, drop the snapshot protecting Key(3)@5. + // Let ShouldStopBefore() return true for Key(3)@5 (delete range sentinel) + // and Key(3)@4. + // Output should have two files: + // File 1: Key(1), Key(3)@7, range tombstone ends at Key(3)@7 + // File dropped: range tombstone only file (from Key(3)@5 to Key(3)@4) + // File 2: Range tombstone starting from Key(3)@4, Key(3)@4 + db_->ReleaseSnapshot(snapshot2); + SyncPoint::GetInstance()->SetCallBack( + "CompactionOutputs::ShouldStopBefore::manual_decision", [opts](void* p) { + auto* pair = (std::pair*)p; + if ((opts.comparator->Compare(ExtractUserKey(pair->second), Key(3)) == + 0) && + (GetInternalKeySeqno(pair->second) <= 6)) { + *(pair->first) = true; + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(dbfull()->RunManualCompaction( + static_cast_with_check(db_->DefaultColumnFamily()) + ->cfd(), + 1, 2, CompactRangeOptions(), &begin_slice, &end_slice, true, + true /* disallow_trivial_move */, + std::numeric_limits::max() /*max_file_num_to_ignore*/, + "" /*trim_ts*/)); + ASSERT_EQ(NumTableFilesAtLevel(2), 2); + // iterate through to check if any assertion breaks + std::unique_ptr iter{db_->NewIterator(ReadOptions())}; + iter->SeekToFirst(); + std::vector expected{1, 3, 4}; + for (auto i : expected) { + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key(i)); + iter->Next(); + } + ASSERT_TRUE(iter->status().ok() && !iter->Valid()); + db_->ReleaseSnapshot(snapshot1); +} + +TEST_F(DBRangeDelTest, NonBottommostCompactionDropRangetombstone) { + // L0: file 1: [DeleteRange[4, 5)], file 2: [3, 6, DeleteRange[8, 9)] + // L6 file 1: [2, 3], file 2: [7, 8] + // When compacting the two L0 files to L1, the compaction is non-bottommost + // since the compaction key range overlaps with L6 file 1. The range tombstone + // [4, 5) should be dropped since it does not overlap with any file in lower + // levels. The range tombstone [8, 9) should not be dropped. + Options opts = CurrentOptions(); + opts.level_compaction_dynamic_level_bytes = false; + opts.num_levels = 7; + opts.level0_file_num_compaction_trigger = 3; + DestroyAndReopen(opts); + + Random rnd(301); + // L6 file 1 + ASSERT_OK(Put(Key(2), rnd.RandomString(100))); + ASSERT_OK(Put(Key(3), rnd.RandomString(100))); + ASSERT_OK(Flush()); + // L6 file 2 + ASSERT_OK(Put(Key(7), rnd.RandomString(100))); + ASSERT_OK(Put(Key(8), rnd.RandomString(100))); + ASSERT_OK(Flush()); + MoveFilesToLevel(6); + ASSERT_EQ(NumTableFilesAtLevel(6), 2); + // L0 file 1 + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(4), + Key(5))); + ASSERT_OK(Flush()); + // L0 file 2 + ASSERT_OK(Put(Key(3), rnd.RandomString(100))); + ASSERT_OK(Put(Key(6), rnd.RandomString(100))); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(8), + Key(9))); + ASSERT_OK(Flush()); + // nothing is dropped during flush + std::string property; + db_->GetProperty(DB::Properties::kAggregatedTableProperties, &property); + TableProperties output_tp; + ParseTablePropertiesString(property, &output_tp); + ASSERT_EQ(output_tp.num_range_deletions, 2); + // Add one more L0 file to trigger L0->L1 compaction + ASSERT_OK(Put(Key(1), rnd.RandomString(100))); + ASSERT_OK(Put(Key(9), rnd.RandomString(100))); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumTableFilesAtLevel(1), 1); + db_->GetProperty(DB::Properties::kAggregatedTableProperties, &property); + ParseTablePropertiesString(property, &output_tp); + ASSERT_EQ(output_tp.num_range_deletions, 1); + + // Now create a snapshot protected range tombstone [4, 5), it should not + // be dropped. + ASSERT_OK(Put(Key(4), rnd.RandomString(100))); + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(4), + Key(5))); + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForceOptimized; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + // All compacted to L6 + ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel(0)); + db_->GetProperty(DB::Properties::kAggregatedTableProperties, &property); + ParseTablePropertiesString(property, &output_tp); + ASSERT_EQ(output_tp.num_range_deletions, 1); + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(DBRangeDelTest, MemtableMaxRangeDeletions) { + // Tests option `memtable_max_range_deletions`. + Options options = CurrentOptions(); + options.level_compaction_dynamic_file_size = false; + options.memtable_max_range_deletions = 50; + options.level0_file_num_compaction_trigger = 5; + DestroyAndReopen(options); + + for (int i = 0; i < 50; ++i) { + // Intentionally delete overlapping ranges to see if the option + // checks number of range tombstone fragments instead. + ASSERT_OK(Put(Key(i), "val1")); + ASSERT_OK(Put(Key(i + 1), "val2")); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(i), Key(i + 2))); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + } + // One more write to trigger flush. + ASSERT_OK(Put(Key(50), "val")); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + + // This should take effect for the next new memtable. + ASSERT_OK(db_->SetOptions({{"memtable_max_range_deletions", "1"}})); + ASSERT_OK(Flush()); + ASSERT_EQ(2, NumTableFilesAtLevel(0)); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(50), Key(100))); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(2, NumTableFilesAtLevel(0)); + // One more write to trigger flush. + ASSERT_OK(Put(Key(50), "new val")); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(3, NumTableFilesAtLevel(0)); +} +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_rate_limiter_test.cc b/librocksdb-sys/rocksdb/db/db_rate_limiter_test.cc new file mode 100644 index 0000000..84c2df2 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_rate_limiter_test.cc @@ -0,0 +1,437 @@ +// Copyright (c) 2022-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include + +#include +#include + +#include "db/db_test_util.h" +#include "port/stack_trace.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "test_util/testharness.h" +#include "util/file_checksum_helper.h" + +namespace ROCKSDB_NAMESPACE { + +class DBRateLimiterOnReadTest + : public DBTestBase, + public ::testing::WithParamInterface> { + public: + explicit DBRateLimiterOnReadTest() + : DBTestBase("db_rate_limiter_on_read_test", /*env_do_fsync=*/false), + use_direct_io_(std::get<0>(GetParam())), + use_block_cache_(std::get<1>(GetParam())), + use_readahead_(std::get<2>(GetParam())) {} + + void Init() { + options_ = GetOptions(); + Reopen(options_); + for (int i = 0; i < kNumFiles; ++i) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK(Put(Key(i * kNumKeysPerFile + j), "val")); + } + ASSERT_OK(Flush()); + } + MoveFilesToLevel(1); + } + + BlockBasedTableOptions GetTableOptions() { + BlockBasedTableOptions table_options; + table_options.no_block_cache = !use_block_cache_; + return table_options; + } + + ReadOptions GetReadOptions() { + ReadOptions read_options; + read_options.rate_limiter_priority = Env::IO_USER; + read_options.readahead_size = use_readahead_ ? kReadaheadBytes : 0; + return read_options; + } + + Options GetOptions() { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.file_checksum_gen_factory.reset(new FileChecksumGenCrc32cFactory()); + options.rate_limiter.reset(NewGenericRateLimiter( + 1 << 20 /* rate_bytes_per_sec */, 100 * 1000 /* refill_period_us */, + 10 /* fairness */, RateLimiter::Mode::kAllIo)); + options.table_factory.reset(NewBlockBasedTableFactory(GetTableOptions())); + options.use_direct_reads = use_direct_io_; + return options; + } + + protected: + const static int kNumKeysPerFile = 1; + const static int kNumFiles = 3; + const static int kReadaheadBytes = 32 << 10; // 32KB + + Options options_; + const bool use_direct_io_; + const bool use_block_cache_; + const bool use_readahead_; +}; + +std::string GetTestNameSuffix( + ::testing::TestParamInfo> info) { + std::ostringstream oss; + if (std::get<0>(info.param)) { + oss << "DirectIO"; + } else { + oss << "BufferedIO"; + } + if (std::get<1>(info.param)) { + oss << "_BlockCache"; + } else { + oss << "_NoBlockCache"; + } + if (std::get<2>(info.param)) { + oss << "_Readahead"; + } else { + oss << "_NoReadahead"; + } + return oss.str(); +} + +INSTANTIATE_TEST_CASE_P(DBRateLimiterOnReadTest, DBRateLimiterOnReadTest, + ::testing::Combine(::testing::Bool(), ::testing::Bool(), + ::testing::Bool()), + GetTestNameSuffix); + +TEST_P(DBRateLimiterOnReadTest, Get) { + if (use_direct_io_ && !IsDirectIOSupported()) { + return; + } + Init(); + + ASSERT_EQ(0, options_.rate_limiter->GetTotalRequests(Env::IO_USER)); + + int expected = 0; + for (int i = 0; i < kNumFiles; ++i) { + { + std::string value; + ASSERT_OK(db_->Get(GetReadOptions(), Key(i * kNumKeysPerFile), &value)); + ++expected; + } + ASSERT_EQ(expected, options_.rate_limiter->GetTotalRequests(Env::IO_USER)); + + { + std::string value; + ASSERT_OK(db_->Get(GetReadOptions(), Key(i * kNumKeysPerFile), &value)); + if (!use_block_cache_) { + ++expected; + } + } + ASSERT_EQ(expected, options_.rate_limiter->GetTotalRequests(Env::IO_USER)); + } +} + +TEST_P(DBRateLimiterOnReadTest, NewMultiGet) { + if (use_direct_io_ && !IsDirectIOSupported()) { + return; + } + Init(); + + ASSERT_EQ(0, options_.rate_limiter->GetTotalRequests(Env::IO_USER)); + + const int kNumKeys = kNumFiles * kNumKeysPerFile; + int64_t expected = 0; + { + std::vector key_bufs; + key_bufs.reserve(kNumKeys); + std::vector keys; + keys.reserve(kNumKeys); + for (int i = 0; i < kNumKeys; ++i) { + key_bufs.emplace_back(Key(i)); + keys.emplace_back(key_bufs[i]); + } + std::vector statuses(kNumKeys); + std::vector values(kNumKeys); + const int64_t prev_total_rl_req = options_.rate_limiter->GetTotalRequests(); + db_->MultiGet(GetReadOptions(), dbfull()->DefaultColumnFamily(), kNumKeys, + keys.data(), values.data(), statuses.data()); + const int64_t cur_total_rl_req = options_.rate_limiter->GetTotalRequests(); + for (int i = 0; i < kNumKeys; ++i) { + ASSERT_TRUE(statuses[i].ok()); + } + ASSERT_GT(cur_total_rl_req, prev_total_rl_req); + ASSERT_EQ(cur_total_rl_req - prev_total_rl_req, + options_.rate_limiter->GetTotalRequests(Env::IO_USER)); + } + expected += kNumKeys; + ASSERT_EQ(expected, options_.rate_limiter->GetTotalRequests(Env::IO_USER)); +} + +TEST_P(DBRateLimiterOnReadTest, OldMultiGet) { + // The old `vector`-returning `MultiGet()` APIs use `Read()`, which + // supports rate limiting. + if (use_direct_io_ && !IsDirectIOSupported()) { + return; + } + Init(); + + ASSERT_EQ(0, options_.rate_limiter->GetTotalRequests(Env::IO_USER)); + + const int kNumKeys = kNumFiles * kNumKeysPerFile; + int expected = 0; + { + std::vector key_bufs; + key_bufs.reserve(kNumKeys); + std::vector keys; + keys.reserve(kNumKeys); + for (int i = 0; i < kNumKeys; ++i) { + key_bufs.emplace_back(Key(i)); + keys.emplace_back(key_bufs[i]); + } + std::vector values; + std::vector statuses = + db_->MultiGet(GetReadOptions(), keys, &values); + for (int i = 0; i < kNumKeys; ++i) { + ASSERT_OK(statuses[i]); + } + } + expected += kNumKeys; + ASSERT_EQ(expected, options_.rate_limiter->GetTotalRequests(Env::IO_USER)); +} + +TEST_P(DBRateLimiterOnReadTest, Iterator) { + if (use_direct_io_ && !IsDirectIOSupported()) { + return; + } + Init(); + + std::unique_ptr iter(db_->NewIterator(GetReadOptions())); + ASSERT_EQ(0, options_.rate_limiter->GetTotalRequests(Env::IO_USER)); + + int expected = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ++expected; + ASSERT_EQ(expected, options_.rate_limiter->GetTotalRequests(Env::IO_USER)); + } + + for (iter->SeekToLast(); iter->Valid(); iter->Prev()) { + // When `use_block_cache_ == true`, the reverse scan will access the blocks + // loaded to cache during the above forward scan, in which case no further + // file reads are expected. + if (!use_block_cache_) { + ++expected; + } + } + // Reverse scan does not read evenly (one block per iteration) due to + // descending seqno ordering, so wait until after the loop to check total. + ASSERT_EQ(expected, options_.rate_limiter->GetTotalRequests(Env::IO_USER)); +} + + +TEST_P(DBRateLimiterOnReadTest, VerifyChecksum) { + if (use_direct_io_ && !IsDirectIOSupported()) { + return; + } + Init(); + + ASSERT_EQ(0, options_.rate_limiter->GetTotalRequests(Env::IO_USER)); + + ASSERT_OK(db_->VerifyChecksum(GetReadOptions())); + // There are 3 reads per file: ReadMetaIndexBlock, + // VerifyChecksumInMetaBlocks, VerifyChecksumInBlocks + int expected = kNumFiles * 3; + ASSERT_EQ(expected, options_.rate_limiter->GetTotalRequests(Env::IO_USER)); +} + +TEST_P(DBRateLimiterOnReadTest, VerifyFileChecksums) { + if (use_direct_io_ && !IsDirectIOSupported()) { + return; + } + Init(); + + ASSERT_EQ(0, options_.rate_limiter->GetTotalRequests(Env::IO_USER)); + + ASSERT_OK(db_->VerifyFileChecksums(GetReadOptions())); + // The files are tiny so there should have just been one read per file. + int expected = kNumFiles; + ASSERT_EQ(expected, options_.rate_limiter->GetTotalRequests(Env::IO_USER)); +} + + +class DBRateLimiterOnWriteTest : public DBTestBase { + public: + explicit DBRateLimiterOnWriteTest() + : DBTestBase("db_rate_limiter_on_write_test", /*env_do_fsync=*/false) {} + + void Init() { + options_ = GetOptions(); + ASSERT_OK(TryReopenWithColumnFamilies({"default"}, options_)); + Random rnd(301); + for (int i = 0; i < kNumFiles; i++) { + ASSERT_OK(Put(0, kStartKey, rnd.RandomString(2))); + ASSERT_OK(Put(0, kEndKey, rnd.RandomString(2))); + ASSERT_OK(Flush(0)); + } + } + + Options GetOptions() { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.rate_limiter.reset(NewGenericRateLimiter( + 1 << 20 /* rate_bytes_per_sec */, 100 * 1000 /* refill_period_us */, + 10 /* fairness */, RateLimiter::Mode::kWritesOnly)); + options.table_factory.reset( + NewBlockBasedTableFactory(BlockBasedTableOptions())); + return options; + } + + protected: + inline const static int64_t kNumFiles = 3; + inline const static std::string kStartKey = "a"; + inline const static std::string kEndKey = "b"; + Options options_; +}; + +TEST_F(DBRateLimiterOnWriteTest, Flush) { + std::int64_t prev_total_request = 0; + + Init(); + + std::int64_t actual_flush_request = + options_.rate_limiter->GetTotalRequests(Env::IO_TOTAL) - + prev_total_request; + std::int64_t exepcted_flush_request = kNumFiles; + EXPECT_EQ(actual_flush_request, exepcted_flush_request); + EXPECT_EQ(actual_flush_request, + options_.rate_limiter->GetTotalRequests(Env::IO_HIGH)); +} + +TEST_F(DBRateLimiterOnWriteTest, Compact) { + Init(); + + // Pre-comaction: + // level-0 : `kNumFiles` SST files overlapping on [kStartKey, kEndKey] + std::string files_per_level_pre_compaction = std::to_string(kNumFiles); + ASSERT_EQ(files_per_level_pre_compaction, FilesPerLevel(0 /* cf */)); + + std::int64_t prev_total_request = + options_.rate_limiter->GetTotalRequests(Env::IO_TOTAL); + ASSERT_EQ(0, options_.rate_limiter->GetTotalRequests(Env::IO_LOW)); + + Compact(kStartKey, kEndKey); + + std::int64_t actual_compaction_request = + options_.rate_limiter->GetTotalRequests(Env::IO_TOTAL) - + prev_total_request; + + // Post-comaction: + // level-0 : 0 SST file + // level-1 : 1 SST file + std::string files_per_level_post_compaction = "0,1"; + ASSERT_EQ(files_per_level_post_compaction, FilesPerLevel(0 /* cf */)); + + std::int64_t exepcted_compaction_request = 1; + EXPECT_EQ(actual_compaction_request, exepcted_compaction_request); + EXPECT_EQ(actual_compaction_request, + options_.rate_limiter->GetTotalRequests(Env::IO_LOW)); +} + +class DBRateLimiterOnWriteWALTest + : public DBRateLimiterOnWriteTest, + public ::testing::WithParamInterface> { + public: + static std::string GetTestNameSuffix( + ::testing::TestParamInfo> info) { + std::ostringstream oss; + if (std::get<0>(info.param)) { + oss << "DisableWAL"; + } else { + oss << "EnableWAL"; + } + if (std::get<1>(info.param)) { + oss << "_ManualWALFlush"; + } else { + oss << "_AutoWALFlush"; + } + if (std::get<2>(info.param) == Env::IO_USER) { + oss << "_RateLimitAutoWALFlush"; + } else if (std::get<2>(info.param) == Env::IO_TOTAL) { + oss << "_NoRateLimitAutoWALFlush"; + } else { + oss << "_RateLimitAutoWALFlushWithIncorrectPriority"; + } + return oss.str(); + } + + explicit DBRateLimiterOnWriteWALTest() + : disable_wal_(std::get<0>(GetParam())), + manual_wal_flush_(std::get<1>(GetParam())), + rate_limiter_priority_(std::get<2>(GetParam())) {} + + void Init() { + options_ = GetOptions(); + options_.manual_wal_flush = manual_wal_flush_; + Reopen(options_); + } + + WriteOptions GetWriteOptions() { + WriteOptions write_options; + write_options.disableWAL = disable_wal_; + write_options.rate_limiter_priority = rate_limiter_priority_; + return write_options; + } + + protected: + bool disable_wal_; + bool manual_wal_flush_; + Env::IOPriority rate_limiter_priority_; +}; + +INSTANTIATE_TEST_CASE_P( + DBRateLimiterOnWriteWALTest, DBRateLimiterOnWriteWALTest, + ::testing::Values(std::make_tuple(false, false, Env::IO_TOTAL), + std::make_tuple(false, false, Env::IO_USER), + std::make_tuple(false, false, Env::IO_HIGH), + std::make_tuple(false, true, Env::IO_USER), + std::make_tuple(true, false, Env::IO_USER)), + DBRateLimiterOnWriteWALTest::GetTestNameSuffix); + +TEST_P(DBRateLimiterOnWriteWALTest, AutoWalFlush) { + Init(); + + const bool no_rate_limit_auto_wal_flush = + (rate_limiter_priority_ == Env::IO_TOTAL); + const bool valid_arg = (rate_limiter_priority_ == Env::IO_USER && + !disable_wal_ && !manual_wal_flush_); + + std::int64_t prev_total_request = + options_.rate_limiter->GetTotalRequests(Env::IO_TOTAL); + ASSERT_EQ(0, options_.rate_limiter->GetTotalRequests(Env::IO_USER)); + + Status s = Put("foo", "v1", GetWriteOptions()); + + if (no_rate_limit_auto_wal_flush || valid_arg) { + EXPECT_TRUE(s.ok()); + } else { + EXPECT_TRUE(s.IsInvalidArgument()); + EXPECT_TRUE(s.ToString().find("WriteOptions::rate_limiter_priority") != + std::string::npos); + } + + std::int64_t actual_auto_wal_flush_request = + options_.rate_limiter->GetTotalRequests(Env::IO_TOTAL) - + prev_total_request; + std::int64_t expected_auto_wal_flush_request = valid_arg ? 1 : 0; + + EXPECT_EQ(actual_auto_wal_flush_request, expected_auto_wal_flush_request); + EXPECT_EQ(actual_auto_wal_flush_request, + options_.rate_limiter->GetTotalRequests(Env::IO_USER)); +} +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_readonly_with_timestamp_test.cc b/librocksdb-sys/rocksdb/db/db_readonly_with_timestamp_test.cc new file mode 100644 index 0000000..675e494 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_readonly_with_timestamp_test.cc @@ -0,0 +1,956 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/db_with_timestamp_test_util.h" +#include "test_util/testutil.h" + +namespace ROCKSDB_NAMESPACE { +class DBReadOnlyTestWithTimestamp : public DBBasicTestWithTimestampBase { + public: + DBReadOnlyTestWithTimestamp() + : DBBasicTestWithTimestampBase("db_readonly_test_with_timestamp") {} + + protected: + void CheckDBOpenedAsCompactedDBWithOneLevel0File() { + VersionSet* const versions = dbfull()->GetVersionSet(); + ASSERT_NE(versions, nullptr); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(cfd, nullptr); + + Version* const current = cfd->current(); + ASSERT_NE(current, nullptr); + + const VersionStorageInfo* const storage_info = current->storage_info(); + ASSERT_NE(storage_info, nullptr); + + // Only 1 L0 file. + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + // L0 is the max level. + ASSERT_EQ(storage_info->num_non_empty_levels(), 1); + } + + void CheckDBOpenedAsCompactedDBWithOnlyHighestNonEmptyLevelFiles() { + VersionSet* const versions = dbfull()->GetVersionSet(); + ASSERT_NE(versions, nullptr); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(cfd, nullptr); + + Version* const current = cfd->current(); + ASSERT_NE(current, nullptr); + + const VersionStorageInfo* const storage_info = current->storage_info(); + ASSERT_NE(storage_info, nullptr); + + // L0 has no files. + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + + // All other levels have no files except the highest level with files. + for (int i = 1; i < storage_info->num_non_empty_levels() - 1; ++i) { + ASSERT_FALSE(storage_info->LevelFilesBrief(i).num_files > 0); + } + + // The highest level with files have some files. + int highest_non_empty_level = storage_info->num_non_empty_levels() - 1; + ASSERT_TRUE( + storage_info->LevelFilesBrief(highest_non_empty_level).num_files > 0); + } +}; + +TEST_F(DBReadOnlyTestWithTimestamp, IteratorAndGetReadTimestampSizeMismatch) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::string write_timestamp = Timestamp(1, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(key)); + ASSERT_OK(s); + } + + // Reopen the database in read only mode to test its timestamp support. + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + ReadOptions read_opts; + std::string different_size_read_timestamp; + PutFixed32(&different_size_read_timestamp, 2); + Slice different_size_read_ts = different_size_read_timestamp; + read_opts.timestamp = &different_size_read_ts; + { + std::unique_ptr iter(db_->NewIterator(read_opts)); + ASSERT_FALSE(iter->Valid()); + ASSERT_TRUE(iter->status().IsInvalidArgument()); + } + + for (uint64_t key = 0; key <= kMaxKey; ++key) { + std::string value_from_get; + std::string timestamp; + ASSERT_TRUE(db_->Get(read_opts, Key1(key), &value_from_get, ×tamp) + .IsInvalidArgument()); + } + + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, + IteratorAndGetReadTimestampSpecifiedWithoutWriteTimestamp) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), "value" + std::to_string(key)); + ASSERT_OK(s); + } + + // Reopen the database in read only mode to test its timestamp support. + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + ReadOptions read_opts; + const std::string read_timestamp = Timestamp(2, 0); + Slice read_ts = read_timestamp; + read_opts.timestamp = &read_ts; + { + std::unique_ptr iter(db_->NewIterator(read_opts)); + ASSERT_FALSE(iter->Valid()); + ASSERT_TRUE(iter->status().IsInvalidArgument()); + } + + for (uint64_t key = 0; key <= kMaxKey; ++key) { + std::string value_from_get; + std::string timestamp; + ASSERT_TRUE(db_->Get(read_opts, Key1(key), &value_from_get, ×tamp) + .IsInvalidArgument()); + } + + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, + IteratorAndGetWriteWithTimestampReadWithoutTimestamp) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::string write_timestamp = Timestamp(1, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(key)); + ASSERT_OK(s); + } + + // Reopen the database in read only mode to test its timestamp support. + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + ReadOptions read_opts; + { + std::unique_ptr iter(db_->NewIterator(read_opts)); + ASSERT_FALSE(iter->Valid()); + ASSERT_TRUE(iter->status().IsInvalidArgument()); + } + + for (uint64_t key = 0; key <= kMaxKey; ++key) { + std::string value_from_get; + ASSERT_TRUE( + db_->Get(read_opts, Key1(key), &value_from_get).IsInvalidArgument()); + } + + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, IteratorAndGet) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::vector start_keys = {1, 0}; + const std::vector write_timestamps = {Timestamp(1, 0), + Timestamp(3, 0)}; + const std::vector read_timestamps = {Timestamp(2, 0), + Timestamp(4, 0)}; + for (size_t i = 0; i < write_timestamps.size(); ++i) { + WriteOptions write_opts; + for (uint64_t key = start_keys[i]; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamps[i], + "value" + std::to_string(i)); + ASSERT_OK(s); + } + } + + // Reopen the database in read only mode to test its timestamp support. + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + + auto get_value_and_check = [](DB* db, ReadOptions read_opts, Slice key, + Slice expected_value, std::string expected_ts) { + std::string value_from_get; + std::string timestamp; + ASSERT_OK(db->Get(read_opts, key.ToString(), &value_from_get, ×tamp)); + ASSERT_EQ(expected_value, value_from_get); + ASSERT_EQ(expected_ts, timestamp); + }; + for (size_t i = 0; i < read_timestamps.size(); ++i) { + ReadOptions read_opts; + Slice read_ts = read_timestamps[i]; + read_opts.timestamp = &read_ts; + std::unique_ptr it(db_->NewIterator(read_opts)); + int count = 0; + uint64_t key = 0; + // Forward iterate. + for (it->Seek(Key1(0)), key = start_keys[i]; it->Valid(); + it->Next(), ++count, ++key) { + CheckIterUserEntry(it.get(), Key1(key), kTypeValue, + "value" + std::to_string(i), write_timestamps[i]); + get_value_and_check(db_, read_opts, it->key(), it->value(), + write_timestamps[i]); + } + size_t expected_count = kMaxKey - start_keys[i] + 1; + ASSERT_EQ(expected_count, count); + + // Backward iterate. + count = 0; + for (it->SeekForPrev(Key1(kMaxKey)), key = kMaxKey; it->Valid(); + it->Prev(), ++count, --key) { + CheckIterUserEntry(it.get(), Key1(key), kTypeValue, + "value" + std::to_string(i), write_timestamps[i]); + get_value_and_check(db_, read_opts, it->key(), it->value(), + write_timestamps[i]); + } + ASSERT_EQ(static_cast(kMaxKey) - start_keys[i] + 1, count); + + // SeekToFirst()/SeekToLast() with lower/upper bounds. + // Then iter with lower and upper bounds. + uint64_t l = 0; + uint64_t r = kMaxKey + 1; + while (l < r) { + std::string lb_str = Key1(l); + Slice lb = lb_str; + std::string ub_str = Key1(r); + Slice ub = ub_str; + read_opts.iterate_lower_bound = &lb; + read_opts.iterate_upper_bound = &ub; + it.reset(db_->NewIterator(read_opts)); + for (it->SeekToFirst(), key = std::max(l, start_keys[i]), count = 0; + it->Valid(); it->Next(), ++key, ++count) { + CheckIterUserEntry(it.get(), Key1(key), kTypeValue, + "value" + std::to_string(i), write_timestamps[i]); + get_value_and_check(db_, read_opts, it->key(), it->value(), + write_timestamps[i]); + } + ASSERT_EQ(r - std::max(l, start_keys[i]), count); + + for (it->SeekToLast(), key = std::min(r, kMaxKey + 1), count = 0; + it->Valid(); it->Prev(), --key, ++count) { + CheckIterUserEntry(it.get(), Key1(key - 1), kTypeValue, + "value" + std::to_string(i), write_timestamps[i]); + get_value_and_check(db_, read_opts, it->key(), it->value(), + write_timestamps[i]); + } + l += (kMaxKey / 100); + r -= (kMaxKey / 100); + } + } + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, Iterators) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::string write_timestamp = Timestamp(1, 0); + const std::string read_timestamp = Timestamp(2, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(key)); + ASSERT_OK(s); + } + + // Reopen the database in read only mode to test its timestamp support. + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + ReadOptions read_opts; + Slice read_ts = read_timestamp; + read_opts.timestamp = &read_ts; + std::vector iters; + ASSERT_OK(db_->NewIterators(read_opts, {db_->DefaultColumnFamily()}, &iters)); + ASSERT_EQ(static_cast(1), iters.size()); + + int count = 0; + uint64_t key = 0; + // Forward iterate. + for (iters[0]->Seek(Key1(0)), key = 0; iters[0]->Valid(); + iters[0]->Next(), ++count, ++key) { + CheckIterUserEntry(iters[0], Key1(key), kTypeValue, + "value" + std::to_string(key), write_timestamp); + } + + size_t expected_count = kMaxKey - 0 + 1; + ASSERT_EQ(expected_count, count); + delete iters[0]; + + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, IteratorsReadTimestampSizeMismatch) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::string write_timestamp = Timestamp(1, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(key)); + ASSERT_OK(s); + } + + // Reopen the database in read only mode to test its timestamp support. + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + ReadOptions read_opts; + std::string different_size_read_timestamp; + PutFixed32(&different_size_read_timestamp, 2); + Slice different_size_read_ts = different_size_read_timestamp; + read_opts.timestamp = &different_size_read_ts; + { + std::vector iters; + ASSERT_TRUE( + db_->NewIterators(read_opts, {db_->DefaultColumnFamily()}, &iters) + .IsInvalidArgument()); + } + + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, + IteratorsReadTimestampSpecifiedWithoutWriteTimestamp) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), "value" + std::to_string(key)); + ASSERT_OK(s); + } + + // Reopen the database in read only mode to test its timestamp support. + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + ReadOptions read_opts; + const std::string read_timestamp = Timestamp(2, 0); + Slice read_ts = read_timestamp; + read_opts.timestamp = &read_ts; + { + std::vector iters; + ASSERT_TRUE( + db_->NewIterators(read_opts, {db_->DefaultColumnFamily()}, &iters) + .IsInvalidArgument()); + } + + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, + IteratorsWriteWithTimestampReadWithoutTimestamp) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::string write_timestamp = Timestamp(1, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(key)); + ASSERT_OK(s); + } + + // Reopen the database in read only mode to test its timestamp support. + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + ReadOptions read_opts; + { + std::vector iters; + ASSERT_TRUE( + db_->NewIterators(read_opts, {db_->DefaultColumnFamily()}, &iters) + .IsInvalidArgument()); + } + + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGetReadTimestampSizeMismatch) { + const int kNumKeysPerFile = 1026; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.disable_auto_compactions = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + std::string write_timestamp = Timestamp(1, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(0)); + ASSERT_OK(s); + } + ASSERT_OK(db_->Flush(FlushOptions())); + Close(); + + // Reopen the database in read only mode as a Compacted DB to test its + // timestamp support. + options.max_open_files = -1; + ASSERT_OK(ReadOnlyReopen(options)); + CheckDBOpenedAsCompactedDBWithOneLevel0File(); + + ReadOptions read_opts; + std::string different_size_read_timestamp; + PutFixed32(&different_size_read_timestamp, 2); + Slice different_size_read_ts = different_size_read_timestamp; + read_opts.timestamp = &different_size_read_ts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + std::string value_from_get; + std::string timestamp; + ASSERT_TRUE(db_->Get(read_opts, Key1(key), &value_from_get, ×tamp) + .IsInvalidArgument()); + } + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, + CompactedDBGetReadTimestampSpecifiedWithoutWriteTimestamp) { + const int kNumKeysPerFile = 1026; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), "value" + std::to_string(0)); + ASSERT_OK(s); + } + ASSERT_OK(db_->Flush(FlushOptions())); + Close(); + + // Reopen the database in read only mode as a Compacted DB to test its + // timestamp support. + options.max_open_files = -1; + ASSERT_OK(ReadOnlyReopen(options)); + CheckDBOpenedAsCompactedDBWithOneLevel0File(); + + ReadOptions read_opts; + const std::string read_timestamp = Timestamp(2, 0); + Slice read_ts = read_timestamp; + read_opts.timestamp = &read_ts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + std::string value_from_get; + std::string timestamp; + ASSERT_TRUE(db_->Get(read_opts, Key1(key), &value_from_get, ×tamp) + .IsInvalidArgument()); + } + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, + CompactedDBGetWriteWithTimestampReadWithoutTimestamp) { + const int kNumKeysPerFile = 1026; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.disable_auto_compactions = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + std::string write_timestamp = Timestamp(1, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(0)); + ASSERT_OK(s); + } + ASSERT_OK(db_->Flush(FlushOptions())); + Close(); + + // Reopen the database in read only mode as a Compacted DB to test its + // timestamp support. + options.max_open_files = -1; + ASSERT_OK(ReadOnlyReopen(options)); + CheckDBOpenedAsCompactedDBWithOneLevel0File(); + + ReadOptions read_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + std::string value_from_get; + ASSERT_TRUE( + db_->Get(read_opts, Key1(key), &value_from_get).IsInvalidArgument()); + } + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGetWithOnlyOneL0File) { + const int kNumKeysPerFile = 1026 * 2; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.disable_auto_compactions = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::vector start_keys = {1, 0}; + const std::vector write_timestamps = {Timestamp(1, 0), + Timestamp(3, 0)}; + const std::vector read_timestamps = {Timestamp(2, 0), + Timestamp(4, 0)}; + for (size_t i = 0; i < write_timestamps.size(); ++i) { + WriteOptions write_opts; + for (uint64_t key = start_keys[i]; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamps[i], + "value" + std::to_string(i)); + ASSERT_OK(s); + } + } + ASSERT_OK(db_->Flush(FlushOptions())); + Close(); + + // Reopen the database in read only mode as a Compacted DB to test its + // timestamp support. + options.max_open_files = -1; + ASSERT_OK(ReadOnlyReopen(options)); + CheckDBOpenedAsCompactedDBWithOneLevel0File(); + + for (size_t i = 0; i < read_timestamps.size(); ++i) { + ReadOptions read_opts; + Slice read_ts = read_timestamps[i]; + read_opts.timestamp = &read_ts; + int count = 0; + for (uint64_t key = start_keys[i]; key <= kMaxKey; ++key, ++count) { + std::string value_from_get; + std::string timestamp; + ASSERT_OK(db_->Get(read_opts, Key1(key), &value_from_get, ×tamp)); + ASSERT_EQ("value" + std::to_string(i), value_from_get); + ASSERT_EQ(write_timestamps[i], timestamp); + } + size_t expected_count = kMaxKey - start_keys[i] + 1; + ASSERT_EQ(expected_count, count); + } + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, + CompactedDBGetWithOnlyHighestNonEmptyLevelFiles) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.disable_auto_compactions = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::vector start_keys = {1, 0}; + const std::vector write_timestamps = {Timestamp(1, 0), + Timestamp(3, 0)}; + const std::vector read_timestamps = {Timestamp(2, 0), + Timestamp(4, 0)}; + for (size_t i = 0; i < write_timestamps.size(); ++i) { + WriteOptions write_opts; + for (uint64_t key = start_keys[i]; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamps[i], + "value" + std::to_string(i)); + ASSERT_OK(s); + } + } + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + Close(); + + // Reopen the database in read only mode as a Compacted DB to test its + // timestamp support. + options.max_open_files = -1; + ASSERT_OK(ReadOnlyReopen(options)); + CheckDBOpenedAsCompactedDBWithOnlyHighestNonEmptyLevelFiles(); + + for (size_t i = 0; i < read_timestamps.size(); ++i) { + ReadOptions read_opts; + Slice read_ts = read_timestamps[i]; + read_opts.timestamp = &read_ts; + int count = 0; + for (uint64_t key = start_keys[i]; key <= kMaxKey; ++key, ++count) { + std::string value_from_get; + std::string timestamp; + ASSERT_OK(db_->Get(read_opts, Key1(key), &value_from_get, ×tamp)); + ASSERT_EQ("value" + std::to_string(i), value_from_get); + ASSERT_EQ(write_timestamps[i], timestamp); + } + size_t expected_count = kMaxKey - start_keys[i] + 1; + ASSERT_EQ(expected_count, count); + } + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, + CompactedDBMultiGetReadTimestampSizeMismatch) { + const int kNumKeysPerFile = 1026; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.disable_auto_compactions = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + std::string write_timestamp = Timestamp(1, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(0)); + ASSERT_OK(s); + } + ASSERT_OK(db_->Flush(FlushOptions())); + Close(); + + // Reopen the database in read only mode as a Compacted DB to test its + // timestamp support. + options.max_open_files = -1; + ASSERT_OK(ReadOnlyReopen(options)); + CheckDBOpenedAsCompactedDBWithOneLevel0File(); + + ReadOptions read_opts; + std::string different_size_read_timestamp; + PutFixed32(&different_size_read_timestamp, 2); + Slice different_size_read_ts = different_size_read_timestamp; + read_opts.timestamp = &different_size_read_ts; + std::vector key_strs; + std::vector keys; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + key_strs.push_back(Key1(key)); + } + for (const auto& key_str : key_strs) { + keys.emplace_back(key_str); + } + std::vector values; + std::vector timestamps; + std::vector status_list = + db_->MultiGet(read_opts, keys, &values, ×tamps); + for (const auto& status : status_list) { + ASSERT_TRUE(status.IsInvalidArgument()); + } + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, + CompactedDBMultiGetReadTimestampSpecifiedWithoutWriteTimestamp) { + const int kNumKeysPerFile = 1026; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), "value" + std::to_string(0)); + ASSERT_OK(s); + } + ASSERT_OK(db_->Flush(FlushOptions())); + Close(); + + // Reopen the database in read only mode as a Compacted DB to test its + // timestamp support. + options.max_open_files = -1; + ASSERT_OK(ReadOnlyReopen(options)); + CheckDBOpenedAsCompactedDBWithOneLevel0File(); + + ReadOptions read_opts; + std::string read_timestamp = Timestamp(2, 0); + Slice read_ts = read_timestamp; + read_opts.timestamp = &read_ts; + std::vector key_strs; + std::vector keys; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + key_strs.push_back(Key1(key)); + } + for (const auto& key_str : key_strs) { + keys.emplace_back(key_str); + } + std::vector values; + std::vector timestamps; + std::vector status_list = + db_->MultiGet(read_opts, keys, &values, ×tamps); + for (const auto& status : status_list) { + ASSERT_TRUE(status.IsInvalidArgument()); + } + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, + CompactedDBMultiGetWriteWithTimestampReadWithoutTimestamp) { + const int kNumKeysPerFile = 1026; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.disable_auto_compactions = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + std::string write_timestamp = Timestamp(1, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(0)); + ASSERT_OK(s); + } + ASSERT_OK(db_->Flush(FlushOptions())); + Close(); + + // Reopen the database in read only mode as a Compacted DB to test its + // timestamp support. + options.max_open_files = -1; + ASSERT_OK(ReadOnlyReopen(options)); + CheckDBOpenedAsCompactedDBWithOneLevel0File(); + + ReadOptions read_opts; + std::vector key_strs; + std::vector keys; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + key_strs.push_back(Key1(key)); + } + for (const auto& key_str : key_strs) { + keys.emplace_back(key_str); + } + std::vector values; + std::vector status_list = db_->MultiGet(read_opts, keys, &values); + for (const auto& status : status_list) { + ASSERT_TRUE(status.IsInvalidArgument()); + } + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBMultiGetWithOnlyOneL0File) { + const int kNumKeysPerFile = 1026 * 2; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.disable_auto_compactions = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::vector start_keys = {1, 0}; + const std::vector write_timestamps = {Timestamp(1, 0), + Timestamp(3, 0)}; + const std::vector read_timestamps = {Timestamp(2, 0), + Timestamp(4, 0)}; + for (size_t i = 0; i < write_timestamps.size(); ++i) { + WriteOptions write_opts; + for (uint64_t key = start_keys[i]; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamps[i], + "value" + std::to_string(i)); + ASSERT_OK(s); + } + } + ASSERT_OK(db_->Flush(FlushOptions())); + Close(); + + // Reopen the database in read only mode as a Compacted DB to test its + // timestamp support. + options.max_open_files = -1; + ASSERT_OK(ReadOnlyReopen(options)); + CheckDBOpenedAsCompactedDBWithOneLevel0File(); + + for (size_t i = 0; i < write_timestamps.size(); ++i) { + ReadOptions read_opts; + Slice read_ts = read_timestamps[i]; + read_opts.timestamp = &read_ts; + std::vector key_strs; + std::vector keys; + for (uint64_t key = start_keys[i]; key <= kMaxKey; ++key) { + key_strs.push_back(Key1(key)); + } + for (const auto& key_str : key_strs) { + keys.emplace_back(key_str); + } + size_t batch_size = kMaxKey - start_keys[i] + 1; + std::vector values; + std::vector timestamps; + std::vector status_list = + db_->MultiGet(read_opts, keys, &values, ×tamps); + ASSERT_EQ(batch_size, values.size()); + ASSERT_EQ(batch_size, timestamps.size()); + for (uint64_t idx = 0; idx < values.size(); ++idx) { + ASSERT_EQ("value" + std::to_string(i), values[idx]); + ASSERT_EQ(write_timestamps[i], timestamps[idx]); + ASSERT_OK(status_list[idx]); + } + } + + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, + CompactedDBMultiGetWithOnlyHighestNonEmptyLevelFiles) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.disable_auto_compactions = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::vector start_keys = {1, 0}; + const std::vector write_timestamps = {Timestamp(1, 0), + Timestamp(3, 0)}; + const std::vector read_timestamps = {Timestamp(2, 0), + Timestamp(4, 0)}; + for (size_t i = 0; i < write_timestamps.size(); ++i) { + WriteOptions write_opts; + for (uint64_t key = start_keys[i]; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamps[i], + "value" + std::to_string(i)); + ASSERT_OK(s); + } + } + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + Close(); + + // Reopen the database in read only mode as a Compacted DB to test its + // timestamp support. + options.max_open_files = -1; + ASSERT_OK(ReadOnlyReopen(options)); + CheckDBOpenedAsCompactedDBWithOnlyHighestNonEmptyLevelFiles(); + + for (size_t i = 0; i < write_timestamps.size(); ++i) { + ReadOptions read_opts; + Slice read_ts = read_timestamps[i]; + read_opts.timestamp = &read_ts; + std::vector key_strs; + std::vector keys; + for (uint64_t key = start_keys[i]; key <= kMaxKey; ++key) { + key_strs.push_back(Key1(key)); + } + for (const auto& key_str : key_strs) { + keys.emplace_back(key_str); + } + size_t batch_size = kMaxKey - start_keys[i] + 1; + std::vector values; + std::vector timestamps; + std::vector status_list = + db_->MultiGet(read_opts, keys, &values, ×tamps); + ASSERT_EQ(batch_size, values.size()); + ASSERT_EQ(batch_size, timestamps.size()); + for (uint64_t idx = 0; idx < values.size(); ++idx) { + ASSERT_EQ("value" + std::to_string(i), values[idx]); + ASSERT_EQ(write_timestamps[i], timestamps[idx]); + ASSERT_OK(status_list[idx]); + } + } + + Close(); +} +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_secondary_test.cc b/librocksdb-sys/rocksdb/db/db_secondary_test.cc new file mode 100644 index 0000000..f3f0a8d --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_secondary_test.cc @@ -0,0 +1,1691 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/db_impl/db_impl_secondary.h" +#include "db/db_test_util.h" +#include "db/db_with_timestamp_test_util.h" +#include "port/stack_trace.h" +#include "rocksdb/utilities/transaction_db.h" +#include "test_util/sync_point.h" +#include "test_util/testutil.h" +#include "utilities/fault_injection_env.h" + +namespace ROCKSDB_NAMESPACE { + +class DBSecondaryTestBase : public DBBasicTestWithTimestampBase { + public: + explicit DBSecondaryTestBase(const std::string& dbname) + : DBBasicTestWithTimestampBase(dbname), + secondary_path_(), + handles_secondary_(), + db_secondary_(nullptr) { + secondary_path_ = + test::PerThreadDBPath(env_, "/db_secondary_test_secondary"); + } + + ~DBSecondaryTestBase() override { + CloseSecondary(); + if (getenv("KEEP_DB") != nullptr) { + fprintf(stdout, "Secondary DB is still at %s\n", secondary_path_.c_str()); + } else { + Options options; + options.env = env_; + EXPECT_OK(DestroyDB(secondary_path_, options)); + } + } + + protected: + Status ReopenAsSecondary(const Options& options) { + return DB::OpenAsSecondary(options, dbname_, secondary_path_, &db_); + } + + void OpenSecondary(const Options& options); + + Status TryOpenSecondary(const Options& options); + + void OpenSecondaryWithColumnFamilies( + const std::vector& column_families, const Options& options); + + void CloseSecondary() { + for (auto h : handles_secondary_) { + ASSERT_OK(db_secondary_->DestroyColumnFamilyHandle(h)); + } + handles_secondary_.clear(); + delete db_secondary_; + db_secondary_ = nullptr; + } + + DBImplSecondary* db_secondary_full() { + return static_cast(db_secondary_); + } + + void CheckFileTypeCounts(const std::string& dir, int expected_log, + int expected_sst, int expected_manifest) const; + + std::string secondary_path_; + std::vector handles_secondary_; + DB* db_secondary_; +}; + +void DBSecondaryTestBase::OpenSecondary(const Options& options) { + ASSERT_OK(TryOpenSecondary(options)); +} + +Status DBSecondaryTestBase::TryOpenSecondary(const Options& options) { + Status s = + DB::OpenAsSecondary(options, dbname_, secondary_path_, &db_secondary_); + return s; +} + +void DBSecondaryTestBase::OpenSecondaryWithColumnFamilies( + const std::vector& column_families, const Options& options) { + std::vector cf_descs; + cf_descs.emplace_back(kDefaultColumnFamilyName, options); + for (const auto& cf_name : column_families) { + cf_descs.emplace_back(cf_name, options); + } + Status s = DB::OpenAsSecondary(options, dbname_, secondary_path_, cf_descs, + &handles_secondary_, &db_secondary_); + ASSERT_OK(s); +} + +void DBSecondaryTestBase::CheckFileTypeCounts(const std::string& dir, + int expected_log, + int expected_sst, + int expected_manifest) const { + std::vector filenames; + ASSERT_OK(env_->GetChildren(dir, &filenames)); + + int log_cnt = 0, sst_cnt = 0, manifest_cnt = 0; + for (auto file : filenames) { + uint64_t number; + FileType type; + if (ParseFileName(file, &number, &type)) { + log_cnt += (type == kWalFile); + sst_cnt += (type == kTableFile); + manifest_cnt += (type == kDescriptorFile); + } + } + ASSERT_EQ(expected_log, log_cnt); + ASSERT_EQ(expected_sst, sst_cnt); + ASSERT_EQ(expected_manifest, manifest_cnt); +} + +class DBSecondaryTest : public DBSecondaryTestBase { + public: + explicit DBSecondaryTest() : DBSecondaryTestBase("db_secondary_test") {} +}; + +TEST_F(DBSecondaryTest, FailOpenIfLoggerCreationFail) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + Reopen(options); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "rocksdb::CreateLoggerFromOptions:AfterGetPath", [&](void* arg) { + auto* s = reinterpret_cast(arg); + assert(s); + *s = Status::IOError("Injected"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + options.max_open_files = -1; + Status s = TryOpenSecondary(options); + ASSERT_EQ(nullptr, options.info_log); + ASSERT_TRUE(s.IsIOError()); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(DBSecondaryTest, NonExistingDb) { + Destroy(last_options_); + + Options options = GetDefaultOptions(); + options.env = env_; + options.max_open_files = -1; + const std::string dbname = "/doesnt/exist"; + Status s = + DB::OpenAsSecondary(options, dbname, secondary_path_, &db_secondary_); + ASSERT_TRUE(s.IsIOError()); +} + +TEST_F(DBSecondaryTest, ReopenAsSecondary) { + Options options; + options.env = env_; + Reopen(options); + ASSERT_OK(Put("foo", "foo_value")); + ASSERT_OK(Put("bar", "bar_value")); + ASSERT_OK(dbfull()->Flush(FlushOptions())); + Close(); + + ASSERT_OK(ReopenAsSecondary(options)); + ASSERT_EQ("foo_value", Get("foo")); + ASSERT_EQ("bar_value", Get("bar")); + ReadOptions ropts; + ropts.verify_checksums = true; + auto db1 = static_cast(db_); + ASSERT_NE(nullptr, db1); + Iterator* iter = db1->NewIterator(ropts); + ASSERT_NE(nullptr, iter); + size_t count = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + if (0 == count) { + ASSERT_EQ("bar", iter->key().ToString()); + ASSERT_EQ("bar_value", iter->value().ToString()); + } else if (1 == count) { + ASSERT_EQ("foo", iter->key().ToString()); + ASSERT_EQ("foo_value", iter->value().ToString()); + } + ++count; + } + delete iter; + ASSERT_EQ(2, count); +} + +TEST_F(DBSecondaryTest, SimpleInternalCompaction) { + Options options; + options.env = env_; + Reopen(options); + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Put("foo", "foo_value" + std::to_string(i))); + ASSERT_OK(Put("bar", "bar_value" + std::to_string(i))); + ASSERT_OK(Flush()); + } + CompactionServiceInput input; + + ColumnFamilyMetaData meta; + db_->GetColumnFamilyMetaData(&meta); + for (auto& file : meta.levels[0].files) { + ASSERT_EQ(0, meta.levels[0].level); + input.input_files.push_back(file.name); + } + ASSERT_EQ(input.input_files.size(), 3); + + input.output_level = 1; + ASSERT_OK(db_->GetDbIdentity(input.db_id)); + Close(); + + options.max_open_files = -1; + OpenSecondary(options); + auto cfh = db_secondary_->DefaultColumnFamily(); + + CompactionServiceResult result; + ASSERT_OK(db_secondary_full()->TEST_CompactWithoutInstallation( + OpenAndCompactOptions(), cfh, input, &result)); + + ASSERT_EQ(result.output_files.size(), 1); + InternalKey smallest, largest; + smallest.DecodeFrom(result.output_files[0].smallest_internal_key); + largest.DecodeFrom(result.output_files[0].largest_internal_key); + ASSERT_EQ(smallest.user_key().ToString(), "bar"); + ASSERT_EQ(largest.user_key().ToString(), "foo"); + ASSERT_EQ(result.output_level, 1); + ASSERT_EQ(result.output_path, this->secondary_path_); + ASSERT_EQ(result.num_output_records, 2); + ASSERT_GT(result.bytes_written, 0); + ASSERT_OK(result.status); +} + +TEST_F(DBSecondaryTest, InternalCompactionMultiLevels) { + Options options; + options.env = env_; + options.disable_auto_compactions = true; + Reopen(options); + const int kRangeL2 = 10; + const int kRangeL1 = 30; + for (int i = 0; i < 10; i++) { + ASSERT_OK(Put(Key(i * kRangeL2), "value" + std::to_string(i))); + ASSERT_OK(Put(Key((i + 1) * kRangeL2 - 1), "value" + std::to_string(i))); + ASSERT_OK(Flush()); + } + MoveFilesToLevel(2); + for (int i = 0; i < 5; i++) { + ASSERT_OK(Put(Key(i * kRangeL1), "value" + std::to_string(i))); + ASSERT_OK(Put(Key((i + 1) * kRangeL1 - 1), "value" + std::to_string(i))); + ASSERT_OK(Flush()); + } + MoveFilesToLevel(1); + for (int i = 0; i < 4; i++) { + ASSERT_OK(Put(Key(i * 30), "value" + std::to_string(i))); + ASSERT_OK(Put(Key(i * 30 + 50), "value" + std::to_string(i))); + ASSERT_OK(Flush()); + } + + ColumnFamilyMetaData meta; + db_->GetColumnFamilyMetaData(&meta); + + // pick 2 files on level 0 for compaction, which has 3 overlap files on L1 + CompactionServiceInput input1; + input1.input_files.push_back(meta.levels[0].files[2].name); + input1.input_files.push_back(meta.levels[0].files[3].name); + input1.input_files.push_back(meta.levels[1].files[0].name); + input1.input_files.push_back(meta.levels[1].files[1].name); + input1.input_files.push_back(meta.levels[1].files[2].name); + + input1.output_level = 1; + ASSERT_OK(db_->GetDbIdentity(input1.db_id)); + + options.max_open_files = -1; + Close(); + + OpenSecondary(options); + auto cfh = db_secondary_->DefaultColumnFamily(); + CompactionServiceResult result; + ASSERT_OK(db_secondary_full()->TEST_CompactWithoutInstallation( + OpenAndCompactOptions(), cfh, input1, &result)); + ASSERT_OK(result.status); + + // pick 2 files on level 1 for compaction, which has 6 overlap files on L2 + CompactionServiceInput input2; + input2.input_files.push_back(meta.levels[1].files[1].name); + input2.input_files.push_back(meta.levels[1].files[2].name); + for (int i = 3; i < 9; i++) { + input2.input_files.push_back(meta.levels[2].files[i].name); + } + + input2.output_level = 2; + input2.db_id = input1.db_id; + ASSERT_OK(db_secondary_full()->TEST_CompactWithoutInstallation( + OpenAndCompactOptions(), cfh, input2, &result)); + ASSERT_OK(result.status); + + CloseSecondary(); + + // delete all l2 files, without update manifest + for (auto& file : meta.levels[2].files) { + ASSERT_OK(env_->DeleteFile(dbname_ + file.name)); + } + OpenSecondary(options); + cfh = db_secondary_->DefaultColumnFamily(); + Status s = db_secondary_full()->TEST_CompactWithoutInstallation( + OpenAndCompactOptions(), cfh, input2, &result); + ASSERT_TRUE(s.IsInvalidArgument()); + ASSERT_OK(result.status); + + // TODO: L0 -> L1 compaction should success, currently version is not built + // if files is missing. + // ASSERT_OK(db_secondary_full()->TEST_CompactWithoutInstallation(OpenAndCompactOptions(), + // cfh, input1, &result)); +} + +TEST_F(DBSecondaryTest, InternalCompactionCompactedFiles) { + Options options; + options.env = env_; + options.level0_file_num_compaction_trigger = 4; + Reopen(options); + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Put("foo", "foo_value" + std::to_string(i))); + ASSERT_OK(Put("bar", "bar_value" + std::to_string(i))); + ASSERT_OK(Flush()); + } + CompactionServiceInput input; + + ColumnFamilyMetaData meta; + db_->GetColumnFamilyMetaData(&meta); + for (auto& file : meta.levels[0].files) { + ASSERT_EQ(0, meta.levels[0].level); + input.input_files.push_back(file.name); + } + ASSERT_EQ(input.input_files.size(), 3); + + input.output_level = 1; + ASSERT_OK(db_->GetDbIdentity(input.db_id)); + + // trigger compaction to delete the files for secondary instance compaction + ASSERT_OK(Put("foo", "foo_value" + std::to_string(3))); + ASSERT_OK(Put("bar", "bar_value" + std::to_string(3))); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + Close(); + + options.max_open_files = -1; + OpenSecondary(options); + auto cfh = db_secondary_->DefaultColumnFamily(); + + CompactionServiceResult result; + Status s = db_secondary_full()->TEST_CompactWithoutInstallation( + OpenAndCompactOptions(), cfh, input, &result); + ASSERT_TRUE(s.IsInvalidArgument()); + ASSERT_OK(result.status); +} + +TEST_F(DBSecondaryTest, InternalCompactionMissingFiles) { + Options options; + options.env = env_; + options.level0_file_num_compaction_trigger = 4; + Reopen(options); + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Put("foo", "foo_value" + std::to_string(i))); + ASSERT_OK(Put("bar", "bar_value" + std::to_string(i))); + ASSERT_OK(Flush()); + } + CompactionServiceInput input; + + ColumnFamilyMetaData meta; + db_->GetColumnFamilyMetaData(&meta); + for (auto& file : meta.levels[0].files) { + ASSERT_EQ(0, meta.levels[0].level); + input.input_files.push_back(file.name); + } + ASSERT_EQ(input.input_files.size(), 3); + + input.output_level = 1; + ASSERT_OK(db_->GetDbIdentity(input.db_id)); + + Close(); + + ASSERT_OK(env_->DeleteFile(dbname_ + input.input_files[0])); + + options.max_open_files = -1; + OpenSecondary(options); + auto cfh = db_secondary_->DefaultColumnFamily(); + + CompactionServiceResult result; + Status s = db_secondary_full()->TEST_CompactWithoutInstallation( + OpenAndCompactOptions(), cfh, input, &result); + ASSERT_TRUE(s.IsInvalidArgument()); + ASSERT_OK(result.status); + + input.input_files.erase(input.input_files.begin()); + + ASSERT_OK(db_secondary_full()->TEST_CompactWithoutInstallation( + OpenAndCompactOptions(), cfh, input, &result)); + ASSERT_OK(result.status); +} + +TEST_F(DBSecondaryTest, OpenAsSecondary) { + Options options; + options.env = env_; + options.level0_file_num_compaction_trigger = 4; + Reopen(options); + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Put("foo", "foo_value" + std::to_string(i))); + ASSERT_OK(Put("bar", "bar_value" + std::to_string(i))); + ASSERT_OK(Flush()); + } + Options options1; + options1.env = env_; + options1.max_open_files = -1; + OpenSecondary(options1); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ReadOptions ropts; + ropts.verify_checksums = true; + const auto verify_db_func = [&](const std::string& foo_val, + const std::string& bar_val) { + std::string value; + ASSERT_OK(db_secondary_->Get(ropts, "foo", &value)); + ASSERT_EQ(foo_val, value); + ASSERT_OK(db_secondary_->Get(ropts, "bar", &value)); + ASSERT_EQ(bar_val, value); + Iterator* iter = db_secondary_->NewIterator(ropts); + ASSERT_NE(nullptr, iter); + iter->Seek("foo"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("foo", iter->key().ToString()); + ASSERT_EQ(foo_val, iter->value().ToString()); + iter->Seek("bar"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("bar", iter->key().ToString()); + ASSERT_EQ(bar_val, iter->value().ToString()); + size_t count = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ++count; + } + ASSERT_EQ(2, count); + delete iter; + }; + + verify_db_func("foo_value2", "bar_value2"); + + ASSERT_OK(Put("foo", "new_foo_value")); + ASSERT_OK(Put("bar", "new_bar_value")); + ASSERT_OK(Flush()); + + ASSERT_OK(db_secondary_->TryCatchUpWithPrimary()); + verify_db_func("new_foo_value", "new_bar_value"); +} + +namespace { +class TraceFileEnv : public EnvWrapper { + public: + explicit TraceFileEnv(Env* _target) : EnvWrapper(_target) {} + static const char* kClassName() { return "TraceFileEnv"; } + const char* Name() const override { return kClassName(); } + + Status NewRandomAccessFile(const std::string& f, + std::unique_ptr* r, + const EnvOptions& env_options) override { + class TracedRandomAccessFile : public RandomAccessFile { + public: + TracedRandomAccessFile(std::unique_ptr&& target, + std::atomic& counter) + : target_(std::move(target)), files_closed_(counter) {} + ~TracedRandomAccessFile() override { + files_closed_.fetch_add(1, std::memory_order_relaxed); + } + Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const override { + return target_->Read(offset, n, result, scratch); + } + + private: + std::unique_ptr target_; + std::atomic& files_closed_; + }; + Status s = target()->NewRandomAccessFile(f, r, env_options); + if (s.ok()) { + r->reset(new TracedRandomAccessFile(std::move(*r), files_closed_)); + } + return s; + } + + int files_closed() const { + return files_closed_.load(std::memory_order_relaxed); + } + + private: + std::atomic files_closed_{0}; +}; +} // anonymous namespace + +TEST_F(DBSecondaryTest, SecondaryCloseFiles) { + Options options; + options.env = env_; + options.max_open_files = 1; + options.disable_auto_compactions = true; + Reopen(options); + Options options1; + std::unique_ptr traced_env(new TraceFileEnv(env_)); + options1.env = traced_env.get(); + OpenSecondary(options1); + + static const auto verify_db = [&]() { + std::unique_ptr iter1(dbfull()->NewIterator(ReadOptions())); + std::unique_ptr iter2(db_secondary_->NewIterator(ReadOptions())); + for (iter1->SeekToFirst(), iter2->SeekToFirst(); + iter1->Valid() && iter2->Valid(); iter1->Next(), iter2->Next()) { + ASSERT_EQ(iter1->key(), iter2->key()); + ASSERT_EQ(iter1->value(), iter2->value()); + } + ASSERT_FALSE(iter1->Valid()); + ASSERT_FALSE(iter2->Valid()); + }; + + ASSERT_OK(Put("a", "value")); + ASSERT_OK(Put("c", "value")); + ASSERT_OK(Flush()); + ASSERT_OK(db_secondary_->TryCatchUpWithPrimary()); + verify_db(); + + ASSERT_OK(Put("b", "value")); + ASSERT_OK(Put("d", "value")); + ASSERT_OK(Flush()); + ASSERT_OK(db_secondary_->TryCatchUpWithPrimary()); + verify_db(); + + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_OK(db_secondary_->TryCatchUpWithPrimary()); + ASSERT_EQ(2, static_cast(traced_env.get())->files_closed()); + + Status s = db_secondary_->SetDBOptions({{"max_open_files", "-1"}}); + ASSERT_TRUE(s.IsNotSupported()); + CloseSecondary(); +} + +TEST_F(DBSecondaryTest, OpenAsSecondaryWALTailing) { + Options options; + options.env = env_; + options.level0_file_num_compaction_trigger = 4; + Reopen(options); + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Put("foo", "foo_value" + std::to_string(i))); + ASSERT_OK(Put("bar", "bar_value" + std::to_string(i))); + } + Options options1; + options1.env = env_; + options1.max_open_files = -1; + OpenSecondary(options1); + + ReadOptions ropts; + ropts.verify_checksums = true; + const auto verify_db_func = [&](const std::string& foo_val, + const std::string& bar_val) { + std::string value; + ASSERT_OK(db_secondary_->Get(ropts, "foo", &value)); + ASSERT_EQ(foo_val, value); + ASSERT_OK(db_secondary_->Get(ropts, "bar", &value)); + ASSERT_EQ(bar_val, value); + Iterator* iter = db_secondary_->NewIterator(ropts); + ASSERT_NE(nullptr, iter); + iter->Seek("foo"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("foo", iter->key().ToString()); + ASSERT_EQ(foo_val, iter->value().ToString()); + iter->Seek("bar"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("bar", iter->key().ToString()); + ASSERT_EQ(bar_val, iter->value().ToString()); + size_t count = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ++count; + } + ASSERT_EQ(2, count); + delete iter; + }; + + verify_db_func("foo_value2", "bar_value2"); + + ASSERT_OK(Put("foo", "new_foo_value")); + ASSERT_OK(Put("bar", "new_bar_value")); + + ASSERT_OK(db_secondary_->TryCatchUpWithPrimary()); + verify_db_func("new_foo_value", "new_bar_value"); + + ASSERT_OK(Flush()); + ASSERT_OK(Put("foo", "new_foo_value_1")); + ASSERT_OK(db_secondary_->TryCatchUpWithPrimary()); + verify_db_func("new_foo_value_1", "new_bar_value"); +} + +TEST_F(DBSecondaryTest, SecondaryTailingBug_ISSUE_8467) { + Options options; + options.env = env_; + Reopen(options); + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Put("foo", "foo_value" + std::to_string(i))); + ASSERT_OK(Put("bar", "bar_value" + std::to_string(i))); + } + + Options options1; + options1.env = env_; + options1.max_open_files = -1; + OpenSecondary(options1); + + const auto verify_db = [&](const std::string& foo_val, + const std::string& bar_val) { + std::string value; + ReadOptions ropts; + Status s = db_secondary_->Get(ropts, "foo", &value); + ASSERT_OK(s); + ASSERT_EQ(foo_val, value); + + s = db_secondary_->Get(ropts, "bar", &value); + ASSERT_OK(s); + ASSERT_EQ(bar_val, value); + }; + + for (int i = 0; i < 2; ++i) { + ASSERT_OK(db_secondary_->TryCatchUpWithPrimary()); + verify_db("foo_value2", "bar_value2"); + } +} + +TEST_F(DBSecondaryTest, RefreshIterator) { + Options options; + options.env = env_; + Reopen(options); + + Options options1; + options1.env = env_; + options1.max_open_files = -1; + OpenSecondary(options1); + + std::unique_ptr it(db_secondary_->NewIterator(ReadOptions())); + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Put("foo", "foo_value" + std::to_string(i))); + + ASSERT_OK(db_secondary_->TryCatchUpWithPrimary()); + if (0 == i) { + it->Seek("foo"); + ASSERT_FALSE(it->Valid()); + ASSERT_OK(it->status()); + + ASSERT_OK(it->Refresh()); + + it->Seek("foo"); + ASSERT_OK(it->status()); + ASSERT_TRUE(it->Valid()); + ASSERT_EQ("foo", it->key()); + ASSERT_EQ("foo_value0", it->value()); + } else { + it->Seek("foo"); + ASSERT_TRUE(it->Valid()); + ASSERT_EQ("foo", it->key()); + ASSERT_EQ("foo_value" + std::to_string(i - 1), it->value()); + ASSERT_OK(it->status()); + + ASSERT_OK(it->Refresh()); + + it->Seek("foo"); + ASSERT_OK(it->status()); + ASSERT_TRUE(it->Valid()); + ASSERT_EQ("foo", it->key()); + ASSERT_EQ("foo_value" + std::to_string(i), it->value()); + } + } +} + +TEST_F(DBSecondaryTest, OpenWithNonExistColumnFamily) { + Options options; + options.env = env_; + CreateAndReopenWithCF({"pikachu"}, options); + + Options options1; + options1.env = env_; + options1.max_open_files = -1; + std::vector cf_descs; + cf_descs.emplace_back(kDefaultColumnFamilyName, options1); + cf_descs.emplace_back("pikachu", options1); + cf_descs.emplace_back("eevee", options1); + Status s = DB::OpenAsSecondary(options1, dbname_, secondary_path_, cf_descs, + &handles_secondary_, &db_secondary_); + ASSERT_NOK(s); +} + +TEST_F(DBSecondaryTest, OpenWithSubsetOfColumnFamilies) { + Options options; + options.env = env_; + CreateAndReopenWithCF({"pikachu"}, options); + Options options1; + options1.env = env_; + options1.max_open_files = -1; + OpenSecondary(options1); + ASSERT_EQ(0, handles_secondary_.size()); + ASSERT_NE(nullptr, db_secondary_); + + ASSERT_OK(Put(0 /*cf*/, "foo", "foo_value")); + ASSERT_OK(Put(1 /*cf*/, "foo", "foo_value")); + ASSERT_OK(Flush(0 /*cf*/)); + ASSERT_OK(Flush(1 /*cf*/)); + ASSERT_OK(db_secondary_->TryCatchUpWithPrimary()); + ReadOptions ropts; + ropts.verify_checksums = true; + std::string value; + ASSERT_OK(db_secondary_->Get(ropts, "foo", &value)); + ASSERT_EQ("foo_value", value); +} + +TEST_F(DBSecondaryTest, SwitchToNewManifestDuringOpen) { + Options options; + options.env = env_; + Reopen(options); + Close(); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->LoadDependency( + {{"ReactiveVersionSet::MaybeSwitchManifest:AfterGetCurrentManifestPath:0", + "VersionSet::ProcessManifestWrites:BeforeNewManifest"}, + {"DBImpl::Open:AfterDeleteFiles", + "ReactiveVersionSet::MaybeSwitchManifest:AfterGetCurrentManifestPath:" + "1"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + port::Thread ro_db_thread([&]() { + Options options1; + options1.env = env_; + options1.max_open_files = -1; + Status s = TryOpenSecondary(options1); + ASSERT_TRUE(s.IsTryAgain()); + + // Try again + OpenSecondary(options1); + CloseSecondary(); + }); + Reopen(options); + ro_db_thread.join(); +} + +TEST_F(DBSecondaryTest, MissingTableFileDuringOpen) { + Options options; + options.env = env_; + options.level0_file_num_compaction_trigger = 4; + Reopen(options); + for (int i = 0; i != options.level0_file_num_compaction_trigger; ++i) { + ASSERT_OK(Put("foo", "foo_value" + std::to_string(i))); + ASSERT_OK(Put("bar", "bar_value" + std::to_string(i))); + ASSERT_OK(dbfull()->Flush(FlushOptions())); + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + Options options1; + options1.env = env_; + options1.max_open_files = -1; + OpenSecondary(options1); + ReadOptions ropts; + ropts.verify_checksums = true; + std::string value; + ASSERT_OK(db_secondary_->Get(ropts, "foo", &value)); + ASSERT_EQ("foo_value" + + std::to_string(options.level0_file_num_compaction_trigger - 1), + value); + ASSERT_OK(db_secondary_->Get(ropts, "bar", &value)); + ASSERT_EQ("bar_value" + + std::to_string(options.level0_file_num_compaction_trigger - 1), + value); + Iterator* iter = db_secondary_->NewIterator(ropts); + ASSERT_NE(nullptr, iter); + iter->Seek("bar"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("bar", iter->key().ToString()); + ASSERT_EQ("bar_value" + + std::to_string(options.level0_file_num_compaction_trigger - 1), + iter->value().ToString()); + iter->Seek("foo"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("foo", iter->key().ToString()); + ASSERT_EQ("foo_value" + + std::to_string(options.level0_file_num_compaction_trigger - 1), + iter->value().ToString()); + size_t count = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ++count; + } + ASSERT_EQ(2, count); + delete iter; +} + +TEST_F(DBSecondaryTest, MissingTableFile) { + Options options; + options.env = env_; + options.level0_file_num_compaction_trigger = 4; + Reopen(options); + + Options options1; + options1.env = env_; + options1.max_open_files = -1; + OpenSecondary(options1); + + for (int i = 0; i != options.level0_file_num_compaction_trigger; ++i) { + ASSERT_OK(Put("foo", "foo_value" + std::to_string(i))); + ASSERT_OK(Put("bar", "bar_value" + std::to_string(i))); + ASSERT_OK(dbfull()->Flush(FlushOptions())); + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_NE(nullptr, db_secondary_full()); + ReadOptions ropts; + ropts.verify_checksums = true; + std::string value; + ASSERT_NOK(db_secondary_->Get(ropts, "foo", &value)); + ASSERT_NOK(db_secondary_->Get(ropts, "bar", &value)); + + ASSERT_OK(db_secondary_->TryCatchUpWithPrimary()); + ASSERT_OK(db_secondary_->Get(ropts, "foo", &value)); + ASSERT_EQ("foo_value" + + std::to_string(options.level0_file_num_compaction_trigger - 1), + value); + ASSERT_OK(db_secondary_->Get(ropts, "bar", &value)); + ASSERT_EQ("bar_value" + + std::to_string(options.level0_file_num_compaction_trigger - 1), + value); + Iterator* iter = db_secondary_->NewIterator(ropts); + ASSERT_NE(nullptr, iter); + iter->Seek("bar"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("bar", iter->key().ToString()); + ASSERT_EQ("bar_value" + + std::to_string(options.level0_file_num_compaction_trigger - 1), + iter->value().ToString()); + iter->Seek("foo"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("foo", iter->key().ToString()); + ASSERT_EQ("foo_value" + + std::to_string(options.level0_file_num_compaction_trigger - 1), + iter->value().ToString()); + size_t count = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ++count; + } + ASSERT_EQ(2, count); + delete iter; +} + +TEST_F(DBSecondaryTest, PrimaryDropColumnFamily) { + Options options; + options.env = env_; + const std::string kCfName1 = "pikachu"; + CreateAndReopenWithCF({kCfName1}, options); + + Options options1; + options1.env = env_; + options1.max_open_files = -1; + OpenSecondaryWithColumnFamilies({kCfName1}, options1); + ASSERT_EQ(2, handles_secondary_.size()); + + ASSERT_OK(Put(1 /*cf*/, "foo", "foo_val_1")); + ASSERT_OK(Flush(1 /*cf*/)); + + ASSERT_OK(db_secondary_->TryCatchUpWithPrimary()); + ReadOptions ropts; + ropts.verify_checksums = true; + std::string value; + ASSERT_OK(db_secondary_->Get(ropts, handles_secondary_[1], "foo", &value)); + ASSERT_EQ("foo_val_1", value); + + ASSERT_OK(dbfull()->DropColumnFamily(handles_[1])); + Close(); + CheckFileTypeCounts(dbname_, 1, 0, 1); + ASSERT_OK(db_secondary_->TryCatchUpWithPrimary()); + value.clear(); + ASSERT_OK(db_secondary_->Get(ropts, handles_secondary_[1], "foo", &value)); + ASSERT_EQ("foo_val_1", value); +} + +TEST_F(DBSecondaryTest, SwitchManifest) { + Options options; + options.env = env_; + options.level0_file_num_compaction_trigger = 4; + const std::string cf1_name("test_cf"); + CreateAndReopenWithCF({cf1_name}, options); + + Options options1; + options1.env = env_; + options1.max_open_files = -1; + OpenSecondaryWithColumnFamilies({kDefaultColumnFamilyName, cf1_name}, + options1); + + const int kNumFiles = options.level0_file_num_compaction_trigger - 1; + // Keep it smaller than 10 so that key0, key1, ..., key9 are sorted as 0, 1, + // ..., 9. + const int kNumKeys = 10; + // Create two sst + for (int i = 0; i != kNumFiles; ++i) { + for (int j = 0; j != kNumKeys; ++j) { + ASSERT_OK(Put("key" + std::to_string(j), "value_" + std::to_string(i))); + } + ASSERT_OK(Flush()); + } + + ASSERT_OK(db_secondary_->TryCatchUpWithPrimary()); + const auto& range_scan_db = [&]() { + ReadOptions tmp_ropts; + tmp_ropts.total_order_seek = true; + tmp_ropts.verify_checksums = true; + std::unique_ptr iter(db_secondary_->NewIterator(tmp_ropts)); + int cnt = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next(), ++cnt) { + ASSERT_EQ("key" + std::to_string(cnt), iter->key().ToString()); + ASSERT_EQ("value_" + std::to_string(kNumFiles - 1), + iter->value().ToString()); + } + }; + + range_scan_db(); + + // While secondary instance still keeps old MANIFEST open, we close primary, + // restart primary, performs full compaction, close again, restart again so + // that next time secondary tries to catch up with primary, the secondary + // will skip the MANIFEST in middle. + ReopenWithColumnFamilies({kDefaultColumnFamilyName, cf1_name}, options); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ReopenWithColumnFamilies({kDefaultColumnFamilyName, cf1_name}, options); + ASSERT_OK(dbfull()->SetOptions({{"disable_auto_compactions", "false"}})); + + ASSERT_OK(db_secondary_->TryCatchUpWithPrimary()); + range_scan_db(); +} + +TEST_F(DBSecondaryTest, SwitchManifestTwice) { + Options options; + options.env = env_; + options.disable_auto_compactions = true; + const std::string cf1_name("test_cf"); + CreateAndReopenWithCF({cf1_name}, options); + + Options options1; + options1.env = env_; + options1.max_open_files = -1; + OpenSecondaryWithColumnFamilies({kDefaultColumnFamilyName, cf1_name}, + options1); + + ASSERT_OK(Put("0", "value0")); + ASSERT_OK(Flush()); + ASSERT_OK(db_secondary_->TryCatchUpWithPrimary()); + std::string value; + ReadOptions ropts; + ropts.verify_checksums = true; + ASSERT_OK(db_secondary_->Get(ropts, "0", &value)); + ASSERT_EQ("value0", value); + + ReopenWithColumnFamilies({kDefaultColumnFamilyName, cf1_name}, options); + ASSERT_OK(dbfull()->SetOptions({{"disable_auto_compactions", "false"}})); + ReopenWithColumnFamilies({kDefaultColumnFamilyName, cf1_name}, options); + ASSERT_OK(Put("0", "value1")); + ASSERT_OK(db_secondary_->TryCatchUpWithPrimary()); + + ASSERT_OK(db_secondary_->Get(ropts, "0", &value)); + ASSERT_EQ("value1", value); +} + +TEST_F(DBSecondaryTest, DISABLED_SwitchWAL) { + const int kNumKeysPerMemtable = 1; + Options options; + options.env = env_; + options.max_write_buffer_number = 4; + options.min_write_buffer_number_to_merge = 2; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerMemtable)); + Reopen(options); + + Options options1; + options1.env = env_; + options1.max_open_files = -1; + OpenSecondary(options1); + + const auto& verify_db = [](DB* db1, DB* db2) { + ASSERT_NE(nullptr, db1); + ASSERT_NE(nullptr, db2); + ReadOptions read_opts; + read_opts.verify_checksums = true; + std::unique_ptr it1(db1->NewIterator(read_opts)); + std::unique_ptr it2(db2->NewIterator(read_opts)); + it1->SeekToFirst(); + it2->SeekToFirst(); + for (; it1->Valid() && it2->Valid(); it1->Next(), it2->Next()) { + ASSERT_EQ(it1->key(), it2->key()); + ASSERT_EQ(it1->value(), it2->value()); + } + ASSERT_FALSE(it1->Valid()); + ASSERT_FALSE(it2->Valid()); + + for (it1->SeekToFirst(); it1->Valid(); it1->Next()) { + std::string value; + ASSERT_OK(db2->Get(read_opts, it1->key(), &value)); + ASSERT_EQ(it1->value(), value); + } + for (it2->SeekToFirst(); it2->Valid(); it2->Next()) { + std::string value; + ASSERT_OK(db1->Get(read_opts, it2->key(), &value)); + ASSERT_EQ(it2->value(), value); + } + }; + for (int k = 0; k != 16; ++k) { + ASSERT_OK(Put("key" + std::to_string(k), "value" + std::to_string(k))); + ASSERT_OK(db_secondary_->TryCatchUpWithPrimary()); + verify_db(dbfull(), db_secondary_); + } +} + +TEST_F(DBSecondaryTest, DISABLED_SwitchWALMultiColumnFamilies) { + const int kNumKeysPerMemtable = 1; + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BackgroundCallFlush:ContextCleanedUp", + "DBSecondaryTest::SwitchWALMultipleColumnFamilies:BeforeCatchUp"}}); + SyncPoint::GetInstance()->EnableProcessing(); + const std::string kCFName1 = "pikachu"; + Options options; + options.env = env_; + options.max_write_buffer_number = 4; + options.min_write_buffer_number_to_merge = 2; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerMemtable)); + CreateAndReopenWithCF({kCFName1}, options); + + Options options1; + options1.env = env_; + options1.max_open_files = -1; + OpenSecondaryWithColumnFamilies({kCFName1}, options1); + ASSERT_EQ(2, handles_secondary_.size()); + + const auto& verify_db = [](DB* db1, + const std::vector& handles1, + DB* db2, + const std::vector& handles2) { + ASSERT_NE(nullptr, db1); + ASSERT_NE(nullptr, db2); + ReadOptions read_opts; + read_opts.verify_checksums = true; + ASSERT_EQ(handles1.size(), handles2.size()); + for (size_t i = 0; i != handles1.size(); ++i) { + std::unique_ptr it1(db1->NewIterator(read_opts, handles1[i])); + std::unique_ptr it2(db2->NewIterator(read_opts, handles2[i])); + it1->SeekToFirst(); + it2->SeekToFirst(); + for (; it1->Valid() && it2->Valid(); it1->Next(), it2->Next()) { + ASSERT_EQ(it1->key(), it2->key()); + ASSERT_EQ(it1->value(), it2->value()); + } + ASSERT_FALSE(it1->Valid()); + ASSERT_FALSE(it2->Valid()); + + for (it1->SeekToFirst(); it1->Valid(); it1->Next()) { + std::string value; + ASSERT_OK(db2->Get(read_opts, handles2[i], it1->key(), &value)); + ASSERT_EQ(it1->value(), value); + } + for (it2->SeekToFirst(); it2->Valid(); it2->Next()) { + std::string value; + ASSERT_OK(db1->Get(read_opts, handles1[i], it2->key(), &value)); + ASSERT_EQ(it2->value(), value); + } + } + }; + for (int k = 0; k != 8; ++k) { + for (int j = 0; j < 2; ++j) { + ASSERT_OK(Put(0 /*cf*/, "key" + std::to_string(k), + "value" + std::to_string(k))); + ASSERT_OK(Put(1 /*cf*/, "key" + std::to_string(k), + "value" + std::to_string(k))); + } + TEST_SYNC_POINT( + "DBSecondaryTest::SwitchWALMultipleColumnFamilies:BeforeCatchUp"); + ASSERT_OK(db_secondary_->TryCatchUpWithPrimary()); + verify_db(dbfull(), handles_, db_secondary_, handles_secondary_); + SyncPoint::GetInstance()->ClearTrace(); + } +} + +TEST_F(DBSecondaryTest, CatchUpAfterFlush) { + const int kNumKeysPerMemtable = 16; + Options options; + options.env = env_; + options.max_write_buffer_number = 4; + options.min_write_buffer_number_to_merge = 2; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerMemtable)); + Reopen(options); + + Options options1; + options1.env = env_; + options1.max_open_files = -1; + OpenSecondary(options1); + + WriteOptions write_opts; + WriteBatch wb; + ASSERT_OK(wb.Put("key0", "value0")); + ASSERT_OK(wb.Put("key1", "value1")); + ASSERT_OK(dbfull()->Write(write_opts, &wb)); + ReadOptions read_opts; + std::unique_ptr iter1(db_secondary_->NewIterator(read_opts)); + iter1->Seek("key0"); + ASSERT_FALSE(iter1->Valid()); + iter1->Seek("key1"); + ASSERT_FALSE(iter1->Valid()); + ASSERT_OK(db_secondary_->TryCatchUpWithPrimary()); + iter1->Seek("key0"); + ASSERT_FALSE(iter1->Valid()); + iter1->Seek("key1"); + ASSERT_FALSE(iter1->Valid()); + ASSERT_OK(iter1->status()); + std::unique_ptr iter2(db_secondary_->NewIterator(read_opts)); + iter2->Seek("key0"); + ASSERT_TRUE(iter2->Valid()); + ASSERT_EQ("value0", iter2->value()); + iter2->Seek("key1"); + ASSERT_TRUE(iter2->Valid()); + ASSERT_OK(iter2->status()); + ASSERT_EQ("value1", iter2->value()); + + { + WriteBatch wb1; + ASSERT_OK(wb1.Put("key0", "value01")); + ASSERT_OK(wb1.Put("key1", "value11")); + ASSERT_OK(dbfull()->Write(write_opts, &wb1)); + } + + { + WriteBatch wb2; + ASSERT_OK(wb2.Put("key0", "new_value0")); + ASSERT_OK(wb2.Delete("key1")); + ASSERT_OK(dbfull()->Write(write_opts, &wb2)); + } + + ASSERT_OK(Flush()); + + ASSERT_OK(db_secondary_->TryCatchUpWithPrimary()); + std::unique_ptr iter3(db_secondary_->NewIterator(read_opts)); + // iter3 should not see value01 and value11 at all. + iter3->Seek("key0"); + ASSERT_TRUE(iter3->Valid()); + ASSERT_EQ("new_value0", iter3->value()); + iter3->Seek("key1"); + ASSERT_FALSE(iter3->Valid()); + ASSERT_OK(iter3->status()); +} + +TEST_F(DBSecondaryTest, CheckConsistencyWhenOpen) { + bool called = false; + Options options; + options.env = env_; + options.disable_auto_compactions = true; + Reopen(options); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "DBImplSecondary::CheckConsistency:AfterFirstAttempt", [&](void* arg) { + ASSERT_NE(nullptr, arg); + called = true; + auto* s = reinterpret_cast(arg); + ASSERT_NOK(*s); + }); + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::CheckConsistency:AfterGetLiveFilesMetaData", + "BackgroundCallCompaction:0"}, + {"DBImpl::BackgroundCallCompaction:PurgedObsoleteFiles", + "DBImpl::CheckConsistency:BeforeGetFileSize"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put("a", "value0")); + ASSERT_OK(Put("c", "value0")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("b", "value1")); + ASSERT_OK(Put("d", "value1")); + ASSERT_OK(Flush()); + port::Thread thread([this]() { + Options opts; + opts.env = env_; + opts.max_open_files = -1; + OpenSecondary(opts); + }); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + thread.join(); + ASSERT_TRUE(called); +} + +TEST_F(DBSecondaryTest, StartFromInconsistent) { + Options options = CurrentOptions(); + DestroyAndReopen(options); + ASSERT_OK(Put("foo", "value")); + ASSERT_OK(Flush()); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "VersionBuilder::CheckConsistencyBeforeReturn", [&](void* arg) { + ASSERT_NE(nullptr, arg); + *(reinterpret_cast(arg)) = + Status::Corruption("Inject corruption"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + Options options1; + options1.env = env_; + Status s = TryOpenSecondary(options1); + ASSERT_TRUE(s.IsCorruption()); +} + +TEST_F(DBSecondaryTest, InconsistencyDuringCatchUp) { + Options options = CurrentOptions(); + DestroyAndReopen(options); + ASSERT_OK(Put("foo", "value")); + ASSERT_OK(Flush()); + + Options options1; + options1.env = env_; + OpenSecondary(options1); + + { + std::string value; + ASSERT_OK(db_secondary_->Get(ReadOptions(), "foo", &value)); + ASSERT_EQ("value", value); + } + + ASSERT_OK(Put("bar", "value1")); + ASSERT_OK(Flush()); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "VersionBuilder::CheckConsistencyBeforeReturn", [&](void* arg) { + ASSERT_NE(nullptr, arg); + *(reinterpret_cast(arg)) = + Status::Corruption("Inject corruption"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + Status s = db_secondary_->TryCatchUpWithPrimary(); + ASSERT_TRUE(s.IsCorruption()); +} + +TEST_F(DBSecondaryTest, OpenWithTransactionDB) { + Options options = CurrentOptions(); + options.create_if_missing = true; + + // Destroy the DB to recreate as a TransactionDB. + Close(); + Destroy(options, true); + + // Create a TransactionDB. + TransactionDB* txn_db = nullptr; + TransactionDBOptions txn_db_opts; + ASSERT_OK(TransactionDB::Open(options, txn_db_opts, dbname_, &txn_db)); + ASSERT_NE(txn_db, nullptr); + db_ = txn_db; + + std::vector cfs = {"new_CF"}; + CreateColumnFamilies(cfs, options); + ASSERT_EQ(handles_.size(), 1); + + WriteOptions wopts; + TransactionOptions txn_opts; + Transaction* txn1 = txn_db->BeginTransaction(wopts, txn_opts, nullptr); + ASSERT_NE(txn1, nullptr); + ASSERT_OK(txn1->Put(handles_[0], "k1", "v1")); + ASSERT_OK(txn1->Commit()); + delete txn1; + + options = CurrentOptions(); + options.max_open_files = -1; + ASSERT_OK(TryOpenSecondary(options)); +} + +class DBSecondaryTestWithTimestamp : public DBSecondaryTestBase { + public: + explicit DBSecondaryTestWithTimestamp() + : DBSecondaryTestBase("db_secondary_test_with_timestamp") {} +}; +TEST_F(DBSecondaryTestWithTimestamp, IteratorAndGetReadTimestampSizeMismatch) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::string write_timestamp = Timestamp(1, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(key)); + ASSERT_OK(s); + } + + // Reopen the database as secondary instance to test its timestamp support. + Close(); + options.max_open_files = -1; + ASSERT_OK(ReopenAsSecondary(options)); + + ReadOptions read_opts; + std::string different_size_read_timestamp; + PutFixed32(&different_size_read_timestamp, 2); + Slice different_size_read_ts = different_size_read_timestamp; + read_opts.timestamp = &different_size_read_ts; + { + std::unique_ptr iter(db_->NewIterator(read_opts)); + ASSERT_FALSE(iter->Valid()); + ASSERT_TRUE(iter->status().IsInvalidArgument()); + } + + for (uint64_t key = 0; key <= kMaxKey; ++key) { + std::string value_from_get; + std::string timestamp; + ASSERT_TRUE(db_->Get(read_opts, Key1(key), &value_from_get, ×tamp) + .IsInvalidArgument()); + } + + Close(); +} + +TEST_F(DBSecondaryTestWithTimestamp, + IteratorAndGetReadTimestampSpecifiedWithoutWriteTimestamp) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), "value" + std::to_string(key)); + ASSERT_OK(s); + } + + // Reopen the database as secondary instance to test its timestamp support. + Close(); + options.max_open_files = -1; + ASSERT_OK(ReopenAsSecondary(options)); + + ReadOptions read_opts; + const std::string read_timestamp = Timestamp(2, 0); + Slice read_ts = read_timestamp; + read_opts.timestamp = &read_ts; + { + std::unique_ptr iter(db_->NewIterator(read_opts)); + ASSERT_FALSE(iter->Valid()); + ASSERT_TRUE(iter->status().IsInvalidArgument()); + } + + for (uint64_t key = 0; key <= kMaxKey; ++key) { + std::string value_from_get; + std::string timestamp; + ASSERT_TRUE(db_->Get(read_opts, Key1(key), &value_from_get, ×tamp) + .IsInvalidArgument()); + } + + Close(); +} + +TEST_F(DBSecondaryTestWithTimestamp, + IteratorAndGetWriteWithTimestampReadWithoutTimestamp) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::string write_timestamp = Timestamp(1, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(key)); + ASSERT_OK(s); + } + + // Reopen the database as secondary instance to test its timestamp support. + Close(); + options.max_open_files = -1; + ASSERT_OK(ReopenAsSecondary(options)); + + ReadOptions read_opts; + { + std::unique_ptr iter(db_->NewIterator(read_opts)); + ASSERT_FALSE(iter->Valid()); + ASSERT_TRUE(iter->status().IsInvalidArgument()); + } + + for (uint64_t key = 0; key <= kMaxKey; ++key) { + std::string value_from_get; + ASSERT_TRUE( + db_->Get(read_opts, Key1(key), &value_from_get).IsInvalidArgument()); + } + + Close(); +} + +TEST_F(DBSecondaryTestWithTimestamp, IteratorAndGet) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::vector start_keys = {1, 0}; + const std::vector write_timestamps = {Timestamp(1, 0), + Timestamp(3, 0)}; + const std::vector read_timestamps = {Timestamp(2, 0), + Timestamp(4, 0)}; + for (size_t i = 0; i < write_timestamps.size(); ++i) { + WriteOptions write_opts; + for (uint64_t key = start_keys[i]; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamps[i], + "value" + std::to_string(i)); + ASSERT_OK(s); + } + } + + // Reopen the database as secondary instance to test its timestamp support. + Close(); + options.max_open_files = -1; + ASSERT_OK(ReopenAsSecondary(options)); + + auto get_value_and_check = [](DB* db, ReadOptions read_opts, Slice key, + Slice expected_value, std::string expected_ts) { + std::string value_from_get; + std::string timestamp; + ASSERT_OK(db->Get(read_opts, key.ToString(), &value_from_get, ×tamp)); + ASSERT_EQ(expected_value, value_from_get); + ASSERT_EQ(expected_ts, timestamp); + }; + for (size_t i = 0; i < read_timestamps.size(); ++i) { + ReadOptions read_opts; + Slice read_ts = read_timestamps[i]; + read_opts.timestamp = &read_ts; + std::unique_ptr it(db_->NewIterator(read_opts)); + int count = 0; + uint64_t key = 0; + // Forward iterate. + for (it->Seek(Key1(0)), key = start_keys[i]; it->Valid(); + it->Next(), ++count, ++key) { + CheckIterUserEntry(it.get(), Key1(key), kTypeValue, + "value" + std::to_string(i), write_timestamps[i]); + get_value_and_check(db_, read_opts, it->key(), it->value(), + write_timestamps[i]); + } + size_t expected_count = kMaxKey - start_keys[i] + 1; + ASSERT_EQ(expected_count, count); + + // Backward iterate. + count = 0; + for (it->SeekForPrev(Key1(kMaxKey)), key = kMaxKey; it->Valid(); + it->Prev(), ++count, --key) { + CheckIterUserEntry(it.get(), Key1(key), kTypeValue, + "value" + std::to_string(i), write_timestamps[i]); + get_value_and_check(db_, read_opts, it->key(), it->value(), + write_timestamps[i]); + } + ASSERT_EQ(static_cast(kMaxKey) - start_keys[i] + 1, count); + + // SeekToFirst()/SeekToLast() with lower/upper bounds. + // Then iter with lower and upper bounds. + uint64_t l = 0; + uint64_t r = kMaxKey + 1; + while (l < r) { + std::string lb_str = Key1(l); + Slice lb = lb_str; + std::string ub_str = Key1(r); + Slice ub = ub_str; + read_opts.iterate_lower_bound = &lb; + read_opts.iterate_upper_bound = &ub; + it.reset(db_->NewIterator(read_opts)); + for (it->SeekToFirst(), key = std::max(l, start_keys[i]), count = 0; + it->Valid(); it->Next(), ++key, ++count) { + CheckIterUserEntry(it.get(), Key1(key), kTypeValue, + "value" + std::to_string(i), write_timestamps[i]); + get_value_and_check(db_, read_opts, it->key(), it->value(), + write_timestamps[i]); + } + ASSERT_EQ(r - std::max(l, start_keys[i]), count); + + for (it->SeekToLast(), key = std::min(r, kMaxKey + 1), count = 0; + it->Valid(); it->Prev(), --key, ++count) { + CheckIterUserEntry(it.get(), Key1(key - 1), kTypeValue, + "value" + std::to_string(i), write_timestamps[i]); + get_value_and_check(db_, read_opts, it->key(), it->value(), + write_timestamps[i]); + } + l += (kMaxKey / 100); + r -= (kMaxKey / 100); + } + } + Close(); +} + +TEST_F(DBSecondaryTestWithTimestamp, IteratorsReadTimestampSizeMismatch) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::string write_timestamp = Timestamp(1, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(key)); + ASSERT_OK(s); + } + + // Reopen the database as secondary instance to test its timestamp support. + Close(); + options.max_open_files = -1; + ASSERT_OK(ReopenAsSecondary(options)); + + ReadOptions read_opts; + std::string different_size_read_timestamp; + PutFixed32(&different_size_read_timestamp, 2); + Slice different_size_read_ts = different_size_read_timestamp; + read_opts.timestamp = &different_size_read_ts; + { + std::vector iters; + ASSERT_TRUE( + db_->NewIterators(read_opts, {db_->DefaultColumnFamily()}, &iters) + .IsInvalidArgument()); + } + + Close(); +} + +TEST_F(DBSecondaryTestWithTimestamp, + IteratorsReadTimestampSpecifiedWithoutWriteTimestamp) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), "value" + std::to_string(key)); + ASSERT_OK(s); + } + + // Reopen the database as secondary instance to test its timestamp support. + Close(); + options.max_open_files = -1; + ASSERT_OK(ReopenAsSecondary(options)); + + ReadOptions read_opts; + const std::string read_timestamp = Timestamp(2, 0); + Slice read_ts = read_timestamp; + read_opts.timestamp = &read_ts; + { + std::vector iters; + ASSERT_TRUE( + db_->NewIterators(read_opts, {db_->DefaultColumnFamily()}, &iters) + .IsInvalidArgument()); + } + + Close(); +} + +TEST_F(DBSecondaryTestWithTimestamp, + IteratorsWriteWithTimestampReadWithoutTimestamp) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::string write_timestamp = Timestamp(1, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(key)); + ASSERT_OK(s); + } + + // Reopen the database as secondary instance to test its timestamp support. + Close(); + options.max_open_files = -1; + ASSERT_OK(ReopenAsSecondary(options)); + + ReadOptions read_opts; + { + std::vector iters; + ASSERT_TRUE( + db_->NewIterators(read_opts, {db_->DefaultColumnFamily()}, &iters) + .IsInvalidArgument()); + } + + Close(); +} + +TEST_F(DBSecondaryTestWithTimestamp, Iterators) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::string write_timestamp = Timestamp(1, 0); + const std::string read_timestamp = Timestamp(2, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(key)); + ASSERT_OK(s); + } + + // Reopen the database as secondary instance to test its timestamp support. + Close(); + options.max_open_files = -1; + ASSERT_OK(ReopenAsSecondary(options)); + + ReadOptions read_opts; + Slice read_ts = read_timestamp; + read_opts.timestamp = &read_ts; + std::vector iters; + ASSERT_OK(db_->NewIterators(read_opts, {db_->DefaultColumnFamily()}, &iters)); + ASSERT_EQ(static_cast(1), iters.size()); + + int count = 0; + uint64_t key = 0; + // Forward iterate. + for (iters[0]->Seek(Key1(0)), key = 0; iters[0]->Valid(); + iters[0]->Next(), ++count, ++key) { + CheckIterUserEntry(iters[0], Key1(key), kTypeValue, + "value" + std::to_string(key), write_timestamp); + } + + size_t expected_count = kMaxKey - 0 + 1; + ASSERT_EQ(expected_count, count); + delete iters[0]; + + Close(); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_sst_test.cc b/librocksdb-sys/rocksdb/db/db_sst_test.cc new file mode 100644 index 0000000..4293e77 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_sst_test.cc @@ -0,0 +1,1865 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/db_test_util.h" +#include "env/mock_env.h" +#include "file/sst_file_manager_impl.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/cache.h" +#include "rocksdb/sst_file_manager.h" +#include "rocksdb/table.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +class DBSSTTest : public DBTestBase { + public: + DBSSTTest() : DBTestBase("db_sst_test", /*env_do_fsync=*/true) {} +}; + +// A class which remembers the name of each flushed file. +class FlushedFileCollector : public EventListener { + public: + FlushedFileCollector() {} + ~FlushedFileCollector() override {} + + void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& info) override { + std::lock_guard lock(mutex_); + flushed_files_.push_back(info.file_path); + } + + std::vector GetFlushedFiles() { + std::lock_guard lock(mutex_); + std::vector result; + for (auto fname : flushed_files_) { + result.push_back(fname); + } + return result; + } + void ClearFlushedFiles() { + std::lock_guard lock(mutex_); + flushed_files_.clear(); + } + + private: + std::vector flushed_files_; + std::mutex mutex_; +}; + +TEST_F(DBSSTTest, DontDeletePendingOutputs) { + Options options; + options.env = env_; + options.create_if_missing = true; + DestroyAndReopen(options); + + // Every time we write to a table file, call FOF/POF with full DB scan. This + // will make sure our pending_outputs_ protection work correctly + std::function purge_obsolete_files_function = [&]() { + JobContext job_context(0); + dbfull()->TEST_LockMutex(); + dbfull()->FindObsoleteFiles(&job_context, true /*force*/); + dbfull()->TEST_UnlockMutex(); + dbfull()->PurgeObsoleteFiles(job_context); + job_context.Clean(); + }; + + env_->table_write_callback_ = &purge_obsolete_files_function; + + for (int i = 0; i < 2; ++i) { + ASSERT_OK(Put("a", "begin")); + ASSERT_OK(Put("z", "end")); + ASSERT_OK(Flush()); + } + + // If pending output guard does not work correctly, PurgeObsoleteFiles() will + // delete the file that Compaction is trying to create, causing this: error + // db/db_test.cc:975: IO error: + // /tmp/rocksdbtest-1552237650/db_test/000009.sst: No such file or directory + Compact("a", "b"); +} + +// 1 Create some SST files by inserting K-V pairs into DB +// 2 Close DB and change suffix from ".sst" to ".ldb" for every other SST file +// 3 Open DB and check if all key can be read +TEST_F(DBSSTTest, SSTsWithLdbSuffixHandling) { + Options options = CurrentOptions(); + options.write_buffer_size = 110 << 10; // 110KB + options.num_levels = 4; + DestroyAndReopen(options); + + Random rnd(301); + int key_id = 0; + for (int i = 0; i < 10; ++i) { + GenerateNewFile(&rnd, &key_id, false); + } + ASSERT_OK(Flush()); + Close(); + int const num_files = GetSstFileCount(dbname_); + ASSERT_GT(num_files, 0); + + Reopen(options); + std::vector values; + values.reserve(key_id); + for (int k = 0; k < key_id; ++k) { + values.push_back(Get(Key(k))); + } + Close(); + + std::vector filenames; + GetSstFiles(env_, dbname_, &filenames); + int num_ldb_files = 0; + for (size_t i = 0; i < filenames.size(); ++i) { + if (i & 1) { + continue; + } + std::string const rdb_name = dbname_ + "/" + filenames[i]; + std::string const ldb_name = Rocks2LevelTableFileName(rdb_name); + ASSERT_TRUE(env_->RenameFile(rdb_name, ldb_name).ok()); + ++num_ldb_files; + } + ASSERT_GT(num_ldb_files, 0); + ASSERT_EQ(num_files, GetSstFileCount(dbname_)); + + Reopen(options); + for (int k = 0; k < key_id; ++k) { + ASSERT_EQ(values[k], Get(Key(k))); + } + Destroy(options); +} + +// Check that we don't crash when opening DB with +// DBOptions::skip_checking_sst_file_sizes_on_db_open = true. +TEST_F(DBSSTTest, SkipCheckingSSTFileSizesOnDBOpen) { + ASSERT_OK(Put("pika", "choo")); + ASSERT_OK(Flush()); + + // Just open the DB with the option set to true and check that we don't crash. + Options options; + options.env = env_; + options.skip_checking_sst_file_sizes_on_db_open = true; + Reopen(options); + + ASSERT_EQ("choo", Get("pika")); +} + +TEST_F(DBSSTTest, DontDeleteMovedFile) { + // This test triggers move compaction and verifies that the file is not + // deleted when it's part of move compaction + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.max_bytes_for_level_base = 1024 * 1024; // 1 MB + options.level0_file_num_compaction_trigger = + 2; // trigger compaction when we have 2 files + DestroyAndReopen(options); + + Random rnd(301); + // Create two 1MB sst files + for (int i = 0; i < 2; ++i) { + // Create 1MB sst file + for (int j = 0; j < 100; ++j) { + ASSERT_OK(Put(Key(i * 50 + j), rnd.RandomString(10 * 1024))); + } + ASSERT_OK(Flush()); + } + // this should execute both L0->L1 and L1->(move)->L2 compactions + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,0,1", FilesPerLevel(0)); + + // If the moved file is actually deleted (the move-safeguard in + // ~Version::Version() is not there), we get this failure: + // Corruption: Can't access /000009.sst + Reopen(options); +} + +// This reproduces a bug where we don't delete a file because when it was +// supposed to be deleted, it was blocked by pending_outputs +// Consider: +// 1. current file_number is 13 +// 2. compaction (1) starts, blocks deletion of all files starting with 13 +// (pending outputs) +// 3. file 13 is created by compaction (2) +// 4. file 13 is consumed by compaction (3) and file 15 was created. Since file +// 13 has no references, it is put into VersionSet::obsolete_files_ +// 5. FindObsoleteFiles() gets file 13 from VersionSet::obsolete_files_. File 13 +// is deleted from obsolete_files_ set. +// 6. PurgeObsoleteFiles() tries to delete file 13, but this file is blocked by +// pending outputs since compaction (1) is still running. It is not deleted and +// it is not present in obsolete_files_ anymore. Therefore, we never delete it. +TEST_F(DBSSTTest, DeleteObsoleteFilesPendingOutputs) { + Options options = CurrentOptions(); + options.env = env_; + options.write_buffer_size = 2 * 1024 * 1024; // 2 MB + options.max_bytes_for_level_base = 1024 * 1024; // 1 MB + options.level0_file_num_compaction_trigger = + 2; // trigger compaction when we have 2 files + options.max_background_flushes = 2; + options.max_background_compactions = 2; + + OnFileDeletionListener* listener = new OnFileDeletionListener(); + options.listeners.emplace_back(listener); + + Reopen(options); + + Random rnd(301); + // Create two 1MB sst files + for (int i = 0; i < 2; ++i) { + // Create 1MB sst file + for (int j = 0; j < 100; ++j) { + ASSERT_OK(Put(Key(i * 50 + j), rnd.RandomString(10 * 1024))); + } + ASSERT_OK(Flush()); + } + // this should execute both L0->L1 and L1->(move)->L2 compactions + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,0,1", FilesPerLevel(0)); + + test::SleepingBackgroundTask blocking_thread; + port::Mutex mutex_; + bool already_blocked(false); + + // block the flush + std::function block_first_time = [&]() { + bool blocking = false; + { + MutexLock l(&mutex_); + if (!already_blocked) { + blocking = true; + already_blocked = true; + } + } + if (blocking) { + blocking_thread.DoSleep(); + } + }; + env_->table_write_callback_ = &block_first_time; + // Insert 2.5MB data, which should trigger a flush because we exceed + // write_buffer_size. The flush will be blocked with block_first_time + // pending_file is protecting all the files created after + for (int j = 0; j < 256; ++j) { + ASSERT_OK(Put(Key(j), rnd.RandomString(10 * 1024))); + } + blocking_thread.WaitUntilSleeping(); + + ASSERT_OK(dbfull()->TEST_CompactRange(2, nullptr, nullptr)); + + ASSERT_EQ("0,0,0,1", FilesPerLevel(0)); + std::vector metadata; + db_->GetLiveFilesMetaData(&metadata); + ASSERT_EQ(metadata.size(), 1U); + auto file_on_L2 = metadata[0].name; + listener->SetExpectedFileName(dbname_ + file_on_L2); + + ASSERT_OK(dbfull()->TEST_CompactRange(3, nullptr, nullptr, nullptr, + true /* disallow trivial move */)); + ASSERT_EQ("0,0,0,0,1", FilesPerLevel(0)); + + // finish the flush! + blocking_thread.WakeUp(); + blocking_thread.WaitUntilDone(); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + // File just flushed is too big for L0 and L1 so gets moved to L2. + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,0,1,0,1", FilesPerLevel(0)); + + metadata.clear(); + db_->GetLiveFilesMetaData(&metadata); + ASSERT_EQ(metadata.size(), 2U); + + // This file should have been deleted during last compaction + ASSERT_EQ(Status::NotFound(), env_->FileExists(dbname_ + file_on_L2)); + listener->VerifyMatchedCount(1); +} + +// Test that producing an empty .sst file does not write it out to +// disk, and that the DeleteFile() env method is not called for +// removing the non-existing file later. +TEST_F(DBSSTTest, DeleteFileNotCalledForNotCreatedSSTFile) { + Options options = CurrentOptions(); + options.env = env_; + + OnFileDeletionListener* listener = new OnFileDeletionListener(); + options.listeners.emplace_back(listener); + + Reopen(options); + + // Flush the empty database. + ASSERT_OK(Flush()); + ASSERT_EQ("", FilesPerLevel(0)); + + // We expect no .sst files. + std::vector metadata; + db_->GetLiveFilesMetaData(&metadata); + ASSERT_EQ(metadata.size(), 0U); + + // We expect no file deletions. + listener->VerifyMatchedCount(0); +} + +// Test that producing a non-empty .sst file does write it out to +// disk, and that the DeleteFile() env method is not called for removing +// the file later. +TEST_F(DBSSTTest, DeleteFileNotCalledForCreatedSSTFile) { + Options options = CurrentOptions(); + options.env = env_; + + OnFileDeletionListener* listener = new OnFileDeletionListener(); + options.listeners.emplace_back(listener); + + Reopen(options); + + ASSERT_OK(Put("pika", "choo")); + + // Flush the non-empty database. + ASSERT_OK(Flush()); + ASSERT_EQ("1", FilesPerLevel(0)); + + // We expect 1 .sst files. + std::vector metadata; + db_->GetLiveFilesMetaData(&metadata); + ASSERT_EQ(metadata.size(), 1U); + + // We expect no file deletions. + listener->VerifyMatchedCount(0); +} + +TEST_F(DBSSTTest, DBWithSstFileManager) { + std::shared_ptr sst_file_manager(NewSstFileManager(env_)); + auto sfm = static_cast(sst_file_manager.get()); + + int files_added = 0; + int files_deleted = 0; + int files_moved = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SstFileManagerImpl::OnAddFile", [&](void* /*arg*/) { files_added++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SstFileManagerImpl::OnDeleteFile", + [&](void* /*arg*/) { files_deleted++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SstFileManagerImpl::OnMoveFile", [&](void* /*arg*/) { files_moved++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Options options = CurrentOptions(); + options.sst_file_manager = sst_file_manager; + DestroyAndReopen(options); + + Random rnd(301); + for (int i = 0; i < 25; i++) { + GenerateNewRandomFile(&rnd); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // Verify that we are tracking all sst files in dbname_ + std::unordered_map files_in_db; + ASSERT_OK(GetAllDataFiles(kTableFile, &files_in_db)); + ASSERT_EQ(sfm->GetTrackedFiles(), files_in_db); + } + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + std::unordered_map files_in_db; + ASSERT_OK(GetAllDataFiles(kTableFile, &files_in_db)); + // Verify that we are tracking all sst files in dbname_ + ASSERT_EQ(sfm->GetTrackedFiles(), files_in_db); + // Verify the total files size + uint64_t total_files_size = 0; + for (auto& file_to_size : files_in_db) { + total_files_size += file_to_size.second; + } + ASSERT_EQ(sfm->GetTotalSize(), total_files_size); + // We flushed at least 25 files + ASSERT_GE(files_added, 25); + // Compaction must have deleted some files + ASSERT_GT(files_deleted, 0); + // No files were moved + ASSERT_EQ(files_moved, 0); + + Close(); + Reopen(options); + ASSERT_EQ(sfm->GetTrackedFiles(), files_in_db); + ASSERT_EQ(sfm->GetTotalSize(), total_files_size); + + // Verify that we track all the files again after the DB is closed and opened + Close(); + sst_file_manager.reset(NewSstFileManager(env_)); + options.sst_file_manager = sst_file_manager; + sfm = static_cast(sst_file_manager.get()); + + Reopen(options); + ASSERT_EQ(sfm->GetTrackedFiles(), files_in_db); + ASSERT_EQ(sfm->GetTotalSize(), total_files_size); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBSSTTest, DBWithSstFileManagerForBlobFiles) { + std::shared_ptr sst_file_manager(NewSstFileManager(env_)); + auto sfm = static_cast(sst_file_manager.get()); + + int files_added = 0; + int files_deleted = 0; + int files_moved = 0; + int files_scheduled_to_delete = 0; + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SstFileManagerImpl::OnAddFile", [&](void* arg) { + const std::string* const file_path = + static_cast(arg); + if (file_path->find(".blob") != std::string::npos) { + files_added++; + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SstFileManagerImpl::OnDeleteFile", [&](void* arg) { + const std::string* const file_path = + static_cast(arg); + if (file_path->find(".blob") != std::string::npos) { + files_deleted++; + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SstFileManagerImpl::ScheduleFileDeletion", [&](void* arg) { + assert(arg); + const std::string* const file_path = + static_cast(arg); + if (file_path->find(".blob") != std::string::npos) { + ++files_scheduled_to_delete; + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SstFileManagerImpl::OnMoveFile", [&](void* /*arg*/) { files_moved++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Options options = CurrentOptions(); + options.sst_file_manager = sst_file_manager; + options.enable_blob_files = true; + options.blob_file_size = 32; // create one blob per file + DestroyAndReopen(options); + Random rnd(301); + + for (int i = 0; i < 10; i++) { + ASSERT_OK(Put("Key_" + std::to_string(i), "Value_" + std::to_string(i))); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // Verify that we are tracking all sst and blob files in dbname_ + std::unordered_map files_in_db; + ASSERT_OK(GetAllDataFiles(kTableFile, &files_in_db)); + ASSERT_OK(GetAllDataFiles(kBlobFile, &files_in_db)); + ASSERT_EQ(sfm->GetTrackedFiles(), files_in_db); + } + + std::vector blob_files = GetBlobFileNumbers(); + ASSERT_EQ(files_added, blob_files.size()); + // No blob file is obsoleted. + ASSERT_EQ(files_deleted, 0); + ASSERT_EQ(files_scheduled_to_delete, 0); + // No files were moved. + ASSERT_EQ(files_moved, 0); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + std::unordered_map files_in_db; + ASSERT_OK(GetAllDataFiles(kTableFile, &files_in_db)); + ASSERT_OK(GetAllDataFiles(kBlobFile, &files_in_db)); + + // Verify that we are tracking all sst and blob files in dbname_ + ASSERT_EQ(sfm->GetTrackedFiles(), files_in_db); + // Verify the total files size + uint64_t total_files_size = 0; + for (auto& file_to_size : files_in_db) { + total_files_size += file_to_size.second; + } + ASSERT_EQ(sfm->GetTotalSize(), total_files_size); + Close(); + + Reopen(options); + ASSERT_EQ(sfm->GetTrackedFiles(), files_in_db); + ASSERT_EQ(sfm->GetTotalSize(), total_files_size); + + // Verify that we track all the files again after the DB is closed and opened. + Close(); + + sst_file_manager.reset(NewSstFileManager(env_)); + options.sst_file_manager = sst_file_manager; + sfm = static_cast(sst_file_manager.get()); + + Reopen(options); + + ASSERT_EQ(sfm->GetTrackedFiles(), files_in_db); + ASSERT_EQ(sfm->GetTotalSize(), total_files_size); + + // Destroy DB and it will remove all the blob files from sst file manager and + // blob files deletion will go through ScheduleFileDeletion. + ASSERT_EQ(files_deleted, 0); + ASSERT_EQ(files_scheduled_to_delete, 0); + Close(); + ASSERT_OK(DestroyDB(dbname_, options)); + ASSERT_EQ(files_deleted, blob_files.size()); + ASSERT_EQ(files_scheduled_to_delete, blob_files.size()); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(DBSSTTest, DBWithSstFileManagerForBlobFilesWithGC) { + std::shared_ptr sst_file_manager(NewSstFileManager(env_)); + auto sfm = static_cast(sst_file_manager.get()); + Options options = CurrentOptions(); + options.sst_file_manager = sst_file_manager; + options.enable_blob_files = true; + options.blob_file_size = 32; // create one blob per file + options.disable_auto_compactions = true; + options.enable_blob_garbage_collection = true; + options.blob_garbage_collection_age_cutoff = 0.5; + + int files_added = 0; + int files_deleted = 0; + int files_moved = 0; + int files_scheduled_to_delete = 0; + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SstFileManagerImpl::OnAddFile", [&](void* arg) { + const std::string* const file_path = + static_cast(arg); + if (file_path->find(".blob") != std::string::npos) { + files_added++; + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SstFileManagerImpl::OnDeleteFile", [&](void* arg) { + const std::string* const file_path = + static_cast(arg); + if (file_path->find(".blob") != std::string::npos) { + files_deleted++; + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SstFileManagerImpl::ScheduleFileDeletion", [&](void* arg) { + assert(arg); + const std::string* const file_path = + static_cast(arg); + if (file_path->find(".blob") != std::string::npos) { + ++files_scheduled_to_delete; + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SstFileManagerImpl::OnMoveFile", [&](void* /*arg*/) { files_moved++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + DestroyAndReopen(options); + Random rnd(301); + + constexpr char first_key[] = "first_key"; + constexpr char first_value[] = "first_value"; + constexpr char second_key[] = "second_key"; + constexpr char second_value[] = "second_value"; + + ASSERT_OK(Put(first_key, first_value)); + ASSERT_OK(Put(second_key, second_value)); + ASSERT_OK(Flush()); + + constexpr char third_key[] = "third_key"; + constexpr char third_value[] = "third_value"; + constexpr char fourth_key[] = "fourth_key"; + constexpr char fourth_value[] = "fourth_value"; + constexpr char fifth_key[] = "fifth_key"; + constexpr char fifth_value[] = "fifth_value"; + + ASSERT_OK(Put(third_key, third_value)); + ASSERT_OK(Put(fourth_key, fourth_value)); + ASSERT_OK(Put(fifth_key, fifth_value)); + ASSERT_OK(Flush()); + + const std::vector original_blob_files = GetBlobFileNumbers(); + + ASSERT_EQ(original_blob_files.size(), 5); + ASSERT_EQ(files_added, 5); + ASSERT_EQ(files_deleted, 0); + ASSERT_EQ(files_scheduled_to_delete, 0); + ASSERT_EQ(files_moved, 0); + { + // Verify that we are tracking all sst and blob files in dbname_ + std::unordered_map files_in_db; + ASSERT_OK(GetAllDataFiles(kTableFile, &files_in_db)); + ASSERT_OK(GetAllDataFiles(kBlobFile, &files_in_db)); + ASSERT_EQ(sfm->GetTrackedFiles(), files_in_db); + } + + const size_t cutoff_index = static_cast( + options.blob_garbage_collection_age_cutoff * original_blob_files.size()); + + size_t expected_number_of_files = original_blob_files.size(); + // Note: turning off enable_blob_files before the compaction results in + // garbage collected values getting inlined. + ASSERT_OK(db_->SetOptions({{"enable_blob_files", "false"}})); + expected_number_of_files -= cutoff_index; + files_added = 0; + + constexpr Slice* begin = nullptr; + constexpr Slice* end = nullptr; + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), begin, end)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + sfm->WaitForEmptyTrash(); + + ASSERT_EQ(Get(first_key), first_value); + ASSERT_EQ(Get(second_key), second_value); + ASSERT_EQ(Get(third_key), third_value); + ASSERT_EQ(Get(fourth_key), fourth_value); + ASSERT_EQ(Get(fifth_key), fifth_value); + + const std::vector new_blob_files = GetBlobFileNumbers(); + + ASSERT_EQ(new_blob_files.size(), expected_number_of_files); + // No new file is added. + ASSERT_EQ(files_added, 0); + ASSERT_EQ(files_deleted, cutoff_index); + ASSERT_EQ(files_scheduled_to_delete, cutoff_index); + ASSERT_EQ(files_moved, 0); + + // Original blob files below the cutoff should be gone, original blob files at + // or above the cutoff should be still there + for (size_t i = cutoff_index; i < original_blob_files.size(); ++i) { + ASSERT_EQ(new_blob_files[i - cutoff_index], original_blob_files[i]); + } + + { + // Verify that we are tracking all sst and blob files in dbname_ + std::unordered_map files_in_db; + ASSERT_OK(GetAllDataFiles(kTableFile, &files_in_db)); + ASSERT_OK(GetAllDataFiles(kBlobFile, &files_in_db)); + ASSERT_EQ(sfm->GetTrackedFiles(), files_in_db); + } + + Close(); + ASSERT_OK(DestroyDB(dbname_, options)); + sfm->WaitForEmptyTrash(); + ASSERT_EQ(files_deleted, 5); + ASSERT_EQ(files_scheduled_to_delete, 5); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +class DBSSTTestRateLimit : public DBSSTTest, + public ::testing::WithParamInterface { + public: + DBSSTTestRateLimit() : DBSSTTest() {} + ~DBSSTTestRateLimit() override {} +}; + +TEST_P(DBSSTTestRateLimit, RateLimitedDelete) { + Destroy(last_options_); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"DBSSTTest::RateLimitedDelete:1", + "DeleteScheduler::BackgroundEmptyTrash"}, + }); + + std::vector penalties; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DeleteScheduler::BackgroundEmptyTrash:Wait", + [&](void* arg) { penalties.push_back(*(static_cast(arg))); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "InstrumentedCondVar::TimedWaitInternal", [&](void* arg) { + // Turn timed wait into a simulated sleep + uint64_t* abs_time_us = static_cast(arg); + uint64_t cur_time = env_->NowMicros(); + if (*abs_time_us > cur_time) { + env_->MockSleepForMicroseconds(*abs_time_us - cur_time); + } + + // Plus an additional short, random amount + env_->MockSleepForMicroseconds(Random::GetTLSInstance()->Uniform(10)); + + // Set wait until time to before (actual) current time to force not + // to sleep + *abs_time_us = Env::Default()->NowMicros(); + }); + + // Disable PeriodicTaskScheduler as it also has TimedWait, which could update + // the simulated sleep time + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::StartPeriodicTaskScheduler:DisableScheduler", [&](void* arg) { + bool* disable_scheduler = static_cast(arg); + *disable_scheduler = true; + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + bool different_wal_dir = GetParam(); + Options options = CurrentOptions(); + SetTimeElapseOnlySleepOnReopen(&options); + options.disable_auto_compactions = true; + options.env = env_; + options.statistics = CreateDBStatistics(); + if (different_wal_dir) { + options.wal_dir = alternative_wal_dir_; + } + + int64_t rate_bytes_per_sec = 1024 * 10; // 10 Kbs / Sec + Status s; + options.sst_file_manager.reset( + NewSstFileManager(env_, nullptr, "", 0, false, &s, 0)); + ASSERT_OK(s); + options.sst_file_manager->SetDeleteRateBytesPerSecond(rate_bytes_per_sec); + auto sfm = static_cast(options.sst_file_manager.get()); + sfm->delete_scheduler()->SetMaxTrashDBRatio(1.1); + + WriteOptions wo; + if (!different_wal_dir) { + wo.disableWAL = true; + } + Reopen(options); + // Create 4 files in L0 + for (char v = 'a'; v <= 'd'; v++) { + ASSERT_OK(Put("Key2", DummyString(1024, v), wo)); + ASSERT_OK(Put("Key3", DummyString(1024, v), wo)); + ASSERT_OK(Put("Key4", DummyString(1024, v), wo)); + ASSERT_OK(Put("Key1", DummyString(1024, v), wo)); + ASSERT_OK(Put("Key4", DummyString(1024, v), wo)); + ASSERT_OK(Flush()); + } + // We created 4 sst files in L0 + ASSERT_EQ("4", FilesPerLevel(0)); + + std::vector metadata; + db_->GetLiveFilesMetaData(&metadata); + + // Compaction will move the 4 files in L0 to trash and create 1 L1 file + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,1", FilesPerLevel(0)); + + uint64_t delete_start_time = env_->NowMicros(); + // Hold BackgroundEmptyTrash + TEST_SYNC_POINT("DBSSTTest::RateLimitedDelete:1"); + sfm->WaitForEmptyTrash(); + uint64_t time_spent_deleting = env_->NowMicros() - delete_start_time; + + uint64_t total_files_size = 0; + uint64_t expected_penlty = 0; + ASSERT_EQ(penalties.size(), metadata.size()); + for (size_t i = 0; i < metadata.size(); i++) { + total_files_size += metadata[i].size; + expected_penlty = ((total_files_size * 1000000) / rate_bytes_per_sec); + ASSERT_EQ(expected_penlty, penalties[i]); + } + ASSERT_GT(time_spent_deleting, expected_penlty * 0.9); + ASSERT_LT(time_spent_deleting, expected_penlty * 1.1); + ASSERT_EQ(4, options.statistics->getAndResetTickerCount(FILES_MARKED_TRASH)); + ASSERT_EQ( + 0, options.statistics->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY)); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +INSTANTIATE_TEST_CASE_P(RateLimitedDelete, DBSSTTestRateLimit, + ::testing::Bool()); + +TEST_F(DBSSTTest, RateLimitedWALDelete) { + Destroy(last_options_); + + std::vector penalties; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DeleteScheduler::BackgroundEmptyTrash:Wait", + [&](void* arg) { penalties.push_back(*(static_cast(arg))); }); + + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.compression = kNoCompression; + options.env = env_; + + int64_t rate_bytes_per_sec = 1024 * 10; // 10 Kbs / Sec + Status s; + options.sst_file_manager.reset( + NewSstFileManager(env_, nullptr, "", 0, false, &s, 0)); + ASSERT_OK(s); + options.sst_file_manager->SetDeleteRateBytesPerSecond(rate_bytes_per_sec); + auto sfm = static_cast(options.sst_file_manager.get()); + sfm->delete_scheduler()->SetMaxTrashDBRatio(3.1); + SetTimeElapseOnlySleepOnReopen(&options); + + ASSERT_OK(TryReopen(options)); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Create 4 files in L0 + for (char v = 'a'; v <= 'd'; v++) { + ASSERT_OK(Put("Key2", DummyString(1024, v))); + ASSERT_OK(Put("Key3", DummyString(1024, v))); + ASSERT_OK(Put("Key4", DummyString(1024, v))); + ASSERT_OK(Put("Key1", DummyString(1024, v))); + ASSERT_OK(Put("Key4", DummyString(1024, v))); + ASSERT_OK(Flush()); + } + // We created 4 sst files in L0 + ASSERT_EQ("4", FilesPerLevel(0)); + + // Compaction will move the 4 files in L0 to trash and create 1 L1 file. + // Use kForceOptimized to not rewrite the new L1 file. + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForceOptimized; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,1", FilesPerLevel(0)); + + sfm->WaitForEmptyTrash(); + ASSERT_EQ(penalties.size(), 8); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +class DBWALTestWithParam + : public DBTestBase, + public testing::WithParamInterface> { + public: + explicit DBWALTestWithParam() + : DBTestBase("db_wal_test_with_params", /*env_do_fsync=*/true) { + wal_dir_ = std::get<0>(GetParam()); + wal_dir_same_as_dbname_ = std::get<1>(GetParam()); + } + + std::string wal_dir_; + bool wal_dir_same_as_dbname_; +}; + +TEST_P(DBWALTestWithParam, WALTrashCleanupOnOpen) { + class MyEnv : public EnvWrapper { + public: + MyEnv(Env* t) : EnvWrapper(t), fake_log_delete(false) {} + const char* Name() const override { return "MyEnv"; } + Status DeleteFile(const std::string& fname) override { + if (fname.find(".log.trash") != std::string::npos && fake_log_delete) { + return Status::OK(); + } + + return target()->DeleteFile(fname); + } + + void set_fake_log_delete(bool fake) { fake_log_delete = fake; } + + private: + bool fake_log_delete; + }; + + std::unique_ptr env(new MyEnv(env_)); + Destroy(last_options_); + + env->set_fake_log_delete(true); + + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.compression = kNoCompression; + options.env = env.get(); + options.wal_dir = dbname_ + wal_dir_; + + int64_t rate_bytes_per_sec = 1024 * 10; // 10 Kbs / Sec + Status s; + options.sst_file_manager.reset( + NewSstFileManager(env_, nullptr, "", 0, false, &s, 0)); + ASSERT_OK(s); + options.sst_file_manager->SetDeleteRateBytesPerSecond(rate_bytes_per_sec); + auto sfm = static_cast(options.sst_file_manager.get()); + sfm->delete_scheduler()->SetMaxTrashDBRatio(3.1); + + Reopen(options); + + // Create 4 files in L0 + for (char v = 'a'; v <= 'd'; v++) { + if (v == 'c') { + // Maximize the change that the last log file will be preserved in trash + // before restarting the DB. + // We have to set this on the 2nd to last file for it to delay deletion + // on the last file. (Quirk of DeleteScheduler::BackgroundEmptyTrash()) + options.sst_file_manager->SetDeleteRateBytesPerSecond(1); + } + ASSERT_OK(Put("Key2", DummyString(1024, v))); + ASSERT_OK(Put("Key3", DummyString(1024, v))); + ASSERT_OK(Put("Key4", DummyString(1024, v))); + ASSERT_OK(Put("Key1", DummyString(1024, v))); + ASSERT_OK(Put("Key4", DummyString(1024, v))); + ASSERT_OK(Flush()); + } + // We created 4 sst files in L0 + ASSERT_EQ("4", FilesPerLevel(0)); + + Close(); + + options.sst_file_manager.reset(); + std::vector filenames; + int trash_log_count = 0; + if (!wal_dir_same_as_dbname_) { + // Forcibly create some trash log files + std::unique_ptr result; + ASSERT_OK(env->NewWritableFile(options.wal_dir + "/1000.log.trash", &result, + EnvOptions())); + result.reset(); + } + ASSERT_OK(env->GetChildren(options.wal_dir, &filenames)); + for (const std::string& fname : filenames) { + if (fname.find(".log.trash") != std::string::npos) { + trash_log_count++; + } + } + ASSERT_GE(trash_log_count, 1); + + env->set_fake_log_delete(false); + Reopen(options); + + filenames.clear(); + trash_log_count = 0; + ASSERT_OK(env->GetChildren(options.wal_dir, &filenames)); + for (const std::string& fname : filenames) { + if (fname.find(".log.trash") != std::string::npos) { + trash_log_count++; + } + } + ASSERT_EQ(trash_log_count, 0); + Close(); +} + +INSTANTIATE_TEST_CASE_P(DBWALTestWithParam, DBWALTestWithParam, + ::testing::Values(std::make_tuple("", true), + std::make_tuple("_wal_dir", false))); + +TEST_F(DBSSTTest, OpenDBWithExistingTrash) { + Options options = CurrentOptions(); + + options.sst_file_manager.reset( + NewSstFileManager(env_, nullptr, "", 1024 * 1024 /* 1 MB/sec */)); + auto sfm = static_cast(options.sst_file_manager.get()); + + Destroy(last_options_); + + // Add some trash files to the db directory so the DB can clean them up + ASSERT_OK(env_->CreateDirIfMissing(dbname_)); + ASSERT_OK(WriteStringToFile(env_, "abc", dbname_ + "/" + "001.sst.trash")); + ASSERT_OK(WriteStringToFile(env_, "abc", dbname_ + "/" + "002.sst.trash")); + ASSERT_OK(WriteStringToFile(env_, "abc", dbname_ + "/" + "003.sst.trash")); + + // Reopen the DB and verify that it deletes existing trash files + Reopen(options); + sfm->WaitForEmptyTrash(); + ASSERT_NOK(env_->FileExists(dbname_ + "/" + "001.sst.trash")); + ASSERT_NOK(env_->FileExists(dbname_ + "/" + "002.sst.trash")); + ASSERT_NOK(env_->FileExists(dbname_ + "/" + "003.sst.trash")); +} + +// Create a DB with 2 db_paths, and generate multiple files in the 2 +// db_paths using CompactRangeOptions, make sure that files that were +// deleted from first db_path were deleted using DeleteScheduler and +// files in the second path were not. +TEST_F(DBSSTTest, DeleteSchedulerMultipleDBPaths) { + std::atomic bg_delete_file(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DeleteScheduler::DeleteTrashFile:DeleteFile", + [&](void* /*arg*/) { bg_delete_file++; }); + // The deletion scheduler sometimes skips marking file as trash according to + // a heuristic. In that case the deletion will go through the below SyncPoint. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DeleteScheduler::DeleteFile", [&](void* /*arg*/) { bg_delete_file++; }); + + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.db_paths.emplace_back(dbname_, 1024 * 100); + options.db_paths.emplace_back(dbname_ + "_2", 1024 * 100); + options.env = env_; + + int64_t rate_bytes_per_sec = 1024 * 1024; // 1 Mb / Sec + Status s; + options.sst_file_manager.reset( + NewSstFileManager(env_, nullptr, "", rate_bytes_per_sec, false, &s, + /* max_trash_db_ratio= */ 1.1)); + + ASSERT_OK(s); + auto sfm = static_cast(options.sst_file_manager.get()); + + DestroyAndReopen(options); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + WriteOptions wo; + wo.disableWAL = true; + + // Create 4 files in L0 + for (int i = 0; i < 4; i++) { + ASSERT_OK(Put("Key" + std::to_string(i), DummyString(1024, 'A'), wo)); + ASSERT_OK(Flush()); + } + // We created 4 sst files in L0 + ASSERT_EQ("4", FilesPerLevel(0)); + // Compaction will delete files from L0 in first db path and generate a new + // file in L1 in second db path + CompactRangeOptions compact_options; + compact_options.target_path_id = 1; + Slice begin("Key0"); + Slice end("Key3"); + ASSERT_OK(db_->CompactRange(compact_options, &begin, &end)); + ASSERT_EQ("0,1", FilesPerLevel(0)); + + // Create 4 files in L0 + for (int i = 4; i < 8; i++) { + ASSERT_OK(Put("Key" + std::to_string(i), DummyString(1024, 'B'), wo)); + ASSERT_OK(Flush()); + } + ASSERT_EQ("4,1", FilesPerLevel(0)); + + // Compaction will delete files from L0 in first db path and generate a new + // file in L1 in second db path + begin = "Key4"; + end = "Key7"; + ASSERT_OK(db_->CompactRange(compact_options, &begin, &end)); + ASSERT_EQ("0,2", FilesPerLevel(0)); + + sfm->WaitForEmptyTrash(); + ASSERT_EQ(bg_delete_file, 8); + + // Compaction will delete both files and regenerate a file in L1 in second + // db path. The deleted files should still be cleaned up via delete scheduler. + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForceOptimized; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + ASSERT_EQ("0,1", FilesPerLevel(0)); + + sfm->WaitForEmptyTrash(); + ASSERT_EQ(bg_delete_file, 10); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBSSTTest, DestroyDBWithRateLimitedDelete) { + int bg_delete_file = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DeleteScheduler::DeleteTrashFile:DeleteFile", + [&](void* /*arg*/) { bg_delete_file++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Status s; + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.env = env_; + options.sst_file_manager.reset( + NewSstFileManager(env_, nullptr, "", 0, false, &s, 0)); + ASSERT_OK(s); + DestroyAndReopen(options); + + // Create 4 files in L0 + for (int i = 0; i < 4; i++) { + ASSERT_OK(Put("Key" + std::to_string(i), DummyString(1024, 'A'))); + ASSERT_OK(Flush()); + } + // We created 4 sst files in L0 + ASSERT_EQ("4", FilesPerLevel(0)); + + // Close DB and destroy it using DeleteScheduler + Close(); + + int num_sst_files = 0; + int num_wal_files = 0; + std::vector db_files; + ASSERT_OK(env_->GetChildren(dbname_, &db_files)); + for (std::string f : db_files) { + if (f.substr(f.find_last_of(".") + 1) == "sst") { + num_sst_files++; + } else if (f.substr(f.find_last_of(".") + 1) == "log") { + num_wal_files++; + } + } + ASSERT_GT(num_sst_files, 0); + ASSERT_GT(num_wal_files, 0); + + auto sfm = static_cast(options.sst_file_manager.get()); + + sfm->SetDeleteRateBytesPerSecond(1024 * 1024); + // Set an extra high trash ratio to prevent immediate/non-rate limited + // deletions + sfm->delete_scheduler()->SetMaxTrashDBRatio(1000.0); + ASSERT_OK(DestroyDB(dbname_, options)); + sfm->WaitForEmptyTrash(); + ASSERT_EQ(bg_delete_file, num_sst_files + num_wal_files); +} + +TEST_F(DBSSTTest, DBWithMaxSpaceAllowed) { + std::shared_ptr sst_file_manager(NewSstFileManager(env_)); + auto sfm = static_cast(sst_file_manager.get()); + + Options options = CurrentOptions(); + options.sst_file_manager = sst_file_manager; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + + Random rnd(301); + + // Generate a file containing 100 keys. + for (int i = 0; i < 100; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(50))); + } + ASSERT_OK(Flush()); + + uint64_t first_file_size = 0; + std::unordered_map files_in_db; + ASSERT_OK(GetAllDataFiles(kTableFile, &files_in_db, &first_file_size)); + ASSERT_EQ(sfm->GetTotalSize(), first_file_size); + + // Set the maximum allowed space usage to the current total size + sfm->SetMaxAllowedSpaceUsage(first_file_size + 1); + + ASSERT_OK(Put("key1", "val1")); + // This flush will cause bg_error_ and will fail + ASSERT_NOK(Flush()); +} + +TEST_F(DBSSTTest, DBWithMaxSpaceAllowedWithBlobFiles) { + std::shared_ptr sst_file_manager(NewSstFileManager(env_)); + auto sfm = static_cast(sst_file_manager.get()); + + Options options = CurrentOptions(); + options.sst_file_manager = sst_file_manager; + options.disable_auto_compactions = true; + options.enable_blob_files = true; + DestroyAndReopen(options); + + Random rnd(301); + + // Generate a file containing keys. + for (int i = 0; i < 10; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(50))); + } + ASSERT_OK(Flush()); + + uint64_t files_size = 0; + uint64_t total_files_size = 0; + std::unordered_map files_in_db; + + ASSERT_OK(GetAllDataFiles(kBlobFile, &files_in_db, &files_size)); + // Make sure blob files are considered by SSTFileManage in size limits. + ASSERT_GT(files_size, 0); + total_files_size = files_size; + ASSERT_OK(GetAllDataFiles(kTableFile, &files_in_db, &files_size)); + total_files_size += files_size; + ASSERT_EQ(sfm->GetTotalSize(), total_files_size); + + // Set the maximum allowed space usage to the current total size. + sfm->SetMaxAllowedSpaceUsage(total_files_size + 1); + + bool max_allowed_space_reached = false; + bool delete_blob_file = false; + // Sync point called after blob file is closed and max allowed space is + // checked. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "BlobFileCompletionCallback::CallBack::MaxAllowedSpaceReached", + [&](void* /*arg*/) { max_allowed_space_reached = true; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "BuildTable::AfterDeleteFile", + [&](void* /*arg*/) { delete_blob_file = true; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + { + "BuildTable::AfterDeleteFile", + "DBSSTTest::DBWithMaxSpaceAllowedWithBlobFiles:1", + }, + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put("key1", "val1")); + // This flush will fail + ASSERT_NOK(Flush()); + ASSERT_TRUE(max_allowed_space_reached); + + TEST_SYNC_POINT("DBSSTTest::DBWithMaxSpaceAllowedWithBlobFiles:1"); + ASSERT_TRUE(delete_blob_file); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBSSTTest, CancellingCompactionsWorks) { + std::shared_ptr sst_file_manager(NewSstFileManager(env_)); + auto sfm = static_cast(sst_file_manager.get()); + + Options options = CurrentOptions(); + options.sst_file_manager = sst_file_manager; + options.level0_file_num_compaction_trigger = 2; + options.statistics = CreateDBStatistics(); + DestroyAndReopen(options); + + int completed_compactions = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction():CancelledCompaction", [&](void* /*arg*/) { + sfm->SetMaxAllowedSpaceUsage(0); + ASSERT_EQ(sfm->GetCompactionsReservedSize(), 0); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", + [&](void* /*arg*/) { completed_compactions++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Random rnd(301); + + // Generate a file containing 10 keys. + for (int i = 0; i < 10; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(50))); + } + ASSERT_OK(Flush()); + uint64_t total_file_size = 0; + std::unordered_map files_in_db; + ASSERT_OK(GetAllDataFiles(kTableFile, &files_in_db, &total_file_size)); + // Set the maximum allowed space usage to the current total size + sfm->SetMaxAllowedSpaceUsage(2 * total_file_size + 1); + + // Generate another file to trigger compaction. + for (int i = 0; i < 10; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(50))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // Because we set a callback in CancelledCompaction, we actually + // let the compaction run + ASSERT_GT(completed_compactions, 0); + ASSERT_EQ(sfm->GetCompactionsReservedSize(), 0); + // Make sure the stat is bumped + ASSERT_GT(dbfull()->immutable_db_options().statistics.get()->getTickerCount( + COMPACTION_CANCELLED), + 0); + ASSERT_EQ(0, + dbfull()->immutable_db_options().statistics.get()->getTickerCount( + FILES_MARKED_TRASH)); + ASSERT_EQ(4, + dbfull()->immutable_db_options().statistics.get()->getTickerCount( + FILES_DELETED_IMMEDIATELY)); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBSSTTest, CancellingManualCompactionsWorks) { + std::shared_ptr sst_file_manager(NewSstFileManager(env_)); + auto sfm = static_cast(sst_file_manager.get()); + + Options options = CurrentOptions(); + options.sst_file_manager = sst_file_manager; + options.statistics = CreateDBStatistics(); + + FlushedFileCollector* collector = new FlushedFileCollector(); + options.listeners.emplace_back(collector); + + DestroyAndReopen(options); + + Random rnd(301); + + // Generate a file containing 10 keys. + for (int i = 0; i < 10; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(50))); + } + ASSERT_OK(Flush()); + uint64_t total_file_size = 0; + std::unordered_map files_in_db; + ASSERT_OK(GetAllDataFiles(kTableFile, &files_in_db, &total_file_size)); + // Set the maximum allowed space usage to the current total size + sfm->SetMaxAllowedSpaceUsage(2 * total_file_size + 1); + + // Generate another file to trigger compaction. + for (int i = 0; i < 10; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(50))); + } + ASSERT_OK(Flush()); + + // OK, now trigger a manual compaction + ASSERT_TRUE(dbfull() + ->CompactRange(CompactRangeOptions(), nullptr, nullptr) + .IsCompactionTooLarge()); + + // Wait for manual compaction to get scheduled and finish + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ(sfm->GetCompactionsReservedSize(), 0); + // Make sure the stat is bumped + ASSERT_EQ(dbfull()->immutable_db_options().statistics.get()->getTickerCount( + COMPACTION_CANCELLED), + 1); + + // Now make sure CompactFiles also gets cancelled + auto l0_files = collector->GetFlushedFiles(); + ASSERT_TRUE( + dbfull() + ->CompactFiles(ROCKSDB_NAMESPACE::CompactionOptions(), l0_files, 0) + .IsCompactionTooLarge()); + + // Wait for manual compaction to get scheduled and finish + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ(dbfull()->immutable_db_options().statistics.get()->getTickerCount( + COMPACTION_CANCELLED), + 2); + ASSERT_EQ(sfm->GetCompactionsReservedSize(), 0); + + // Now let the flush through and make sure GetCompactionsReservedSize + // returns to normal + sfm->SetMaxAllowedSpaceUsage(0); + int completed_compactions = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactFilesImpl:End", [&](void* /*arg*/) { completed_compactions++; }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(dbfull()->CompactFiles(ROCKSDB_NAMESPACE::CompactionOptions(), + l0_files, 0)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ(sfm->GetCompactionsReservedSize(), 0); + ASSERT_GT(completed_compactions, 0); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBSSTTest, DBWithMaxSpaceAllowedRandomized) { + // This test will set a maximum allowed space for the DB, then it will + // keep filling the DB until the limit is reached and bg_error_ is set. + // When bg_error_ is set we will verify that the DB size is greater + // than the limit. + + std::vector max_space_limits_mbs = {1, 10}; + std::atomic bg_error_set(false); + + std::atomic reached_max_space_on_flush(0); + std::atomic reached_max_space_on_compaction(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::FlushMemTableToOutputFile:MaxAllowedSpaceReached", + [&](void* arg) { + Status* bg_error = static_cast(arg); + bg_error_set = true; + reached_max_space_on_flush++; + // clear error to ensure compaction callback is called + *bg_error = Status::OK(); + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction():CancelledCompaction", [&](void* arg) { + bool* enough_room = static_cast(arg); + *enough_room = true; + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::FinishCompactionOutputFile:MaxAllowedSpaceReached", + [&](void* /*arg*/) { + bg_error_set = true; + reached_max_space_on_compaction++; + }); + + for (auto limit_mb : max_space_limits_mbs) { + bg_error_set = false; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + std::shared_ptr sst_file_manager(NewSstFileManager(env_)); + auto sfm = static_cast(sst_file_manager.get()); + + Options options = CurrentOptions(); + options.sst_file_manager = sst_file_manager; + options.write_buffer_size = 1024 * 512; // 512 Kb + DestroyAndReopen(options); + Random rnd(301); + + sfm->SetMaxAllowedSpaceUsage(limit_mb * 1024 * 1024); + + // It is easy to detect if the test is stuck in a loop. No need for + // complex termination logic. + while (true) { + auto s = Put(rnd.RandomString(10), rnd.RandomString(50)); + if (!s.ok()) { + break; + } + } + ASSERT_TRUE(bg_error_set); + uint64_t total_sst_files_size = 0; + std::unordered_map files_in_db; + ASSERT_OK(GetAllDataFiles(kTableFile, &files_in_db, &total_sst_files_size)); + ASSERT_GE(total_sst_files_size, limit_mb * 1024 * 1024); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + } + + ASSERT_GT(reached_max_space_on_flush, 0); + ASSERT_GT(reached_max_space_on_compaction, 0); +} + +TEST_F(DBSSTTest, OpenDBWithInfiniteMaxOpenFiles) { + // Open DB with infinite max open files + // - First iteration use 1 thread to open files + // - Second iteration use 5 threads to open files + for (int iter = 0; iter < 2; iter++) { + Options options; + options.create_if_missing = true; + options.write_buffer_size = 100000; + options.disable_auto_compactions = true; + options.max_open_files = -1; + if (iter == 0) { + options.max_file_opening_threads = 1; + } else { + options.max_file_opening_threads = 5; + } + options = CurrentOptions(options); + DestroyAndReopen(options); + + // Create 12 Files in L0 (then move then to L2) + for (int i = 0; i < 12; i++) { + std::string k = "L2_" + Key(i); + ASSERT_OK(Put(k, k + std::string(1000, 'a'))); + ASSERT_OK(Flush()); + } + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 2; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + + // Create 12 Files in L0 + for (int i = 0; i < 12; i++) { + std::string k = "L0_" + Key(i); + ASSERT_OK(Put(k, k + std::string(1000, 'a'))); + ASSERT_OK(Flush()); + } + Close(); + + // Reopening the DB will load all existing files + Reopen(options); + ASSERT_EQ("12,0,12", FilesPerLevel(0)); + std::vector> files; + dbfull()->TEST_GetFilesMetaData(db_->DefaultColumnFamily(), &files); + + for (const auto& level : files) { + for (const auto& file : level) { + ASSERT_TRUE(file.table_reader_handle != nullptr); + } + } + + for (int i = 0; i < 12; i++) { + ASSERT_EQ(Get("L0_" + Key(i)), "L0_" + Key(i) + std::string(1000, 'a')); + ASSERT_EQ(Get("L2_" + Key(i)), "L2_" + Key(i) + std::string(1000, 'a')); + } + } +} + +TEST_F(DBSSTTest, OpenDBWithInfiniteMaxOpenFilesSubjectToMemoryLimit) { + for (CacheEntryRoleOptions::Decision charge_table_reader : + {CacheEntryRoleOptions::Decision::kEnabled, + CacheEntryRoleOptions::Decision::kDisabled}) { + // Open DB with infinite max open files + // - First iteration use 1 thread to open files + // - Second iteration use 5 threads to open files + for (int iter = 0; iter < 2; iter++) { + Options options; + options.create_if_missing = true; + options.write_buffer_size = 100000; + options.disable_auto_compactions = true; + options.max_open_files = -1; + + BlockBasedTableOptions table_options; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + if (iter == 0) { + options.max_file_opening_threads = 1; + } else { + options.max_file_opening_threads = 5; + } + + DestroyAndReopen(options); + + // Create 5 Files in L0 (then move then to L2) + for (int i = 0; i < 5; i++) { + std::string k = "L2_" + Key(i); + ASSERT_OK(Put(k, k + std::string(1000, 'a'))); + ASSERT_OK(Flush()) << i; + } + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 2; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + + // Create 5 Files in L0 + for (int i = 0; i < 5; i++) { + std::string k = "L0_" + Key(i); + ASSERT_OK(Put(k, k + std::string(1000, 'a'))); + ASSERT_OK(Flush()); + } + Close(); + + table_options.cache_usage_options.options_overrides.insert( + {CacheEntryRole::kBlockBasedTableReader, + {/*.charged = */ charge_table_reader}}); + table_options.block_cache = + NewLRUCache(1024 /* capacity */, 0 /* num_shard_bits */, + true /* strict_capacity_limit */); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + // Reopening the DB will try to load all existing files, conditionally + // subject to memory limit + Status s = TryReopen(options); + + if (charge_table_reader == CacheEntryRoleOptions::Decision::kEnabled) { + EXPECT_TRUE(s.IsMemoryLimit()); + EXPECT_TRUE(s.ToString().find( + kCacheEntryRoleToCamelString[static_cast( + CacheEntryRole::kBlockBasedTableReader)]) != + std::string::npos); + EXPECT_TRUE(s.ToString().find("memory limit based on cache capacity") != + std::string::npos); + + } else { + EXPECT_TRUE(s.ok()); + ASSERT_EQ("5,0,5", FilesPerLevel(0)); + } + } + } +} + +TEST_F(DBSSTTest, GetTotalSstFilesSize) { + // We don't propagate oldest-key-time table property on compaction and + // just write 0 as default value. This affect the exact table size, since + // we encode table properties as varint64. Force time to be 0 to work around + // it. Should remove the workaround after we propagate the property on + // compaction. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "FlushJob::WriteLevel0Table:oldest_ancester_time", [&](void* arg) { + uint64_t* current_time = static_cast(arg); + *current_time = 0; + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.compression = kNoCompression; + DestroyAndReopen(options); + // Generate 5 files in L0 + for (int i = 0; i < 5; i++) { + for (int j = 0; j < 10; j++) { + std::string val = "val_file_" + std::to_string(i); + ASSERT_OK(Put(Key(j), val)); + } + ASSERT_OK(Flush()); + } + ASSERT_EQ("5", FilesPerLevel(0)); + + std::vector live_files_meta; + dbfull()->GetLiveFilesMetaData(&live_files_meta); + ASSERT_EQ(live_files_meta.size(), 5); + uint64_t single_file_size = live_files_meta[0].size; + + uint64_t live_sst_files_size = 0; + uint64_t total_sst_files_size = 0; + for (const auto& file_meta : live_files_meta) { + live_sst_files_size += file_meta.size; + } + + ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size", + &total_sst_files_size)); + // Live SST files = 5 + // Total SST files = 5 + ASSERT_EQ(live_sst_files_size, 5 * single_file_size); + ASSERT_EQ(total_sst_files_size, 5 * single_file_size); + + // hold current version + std::unique_ptr iter1(dbfull()->NewIterator(ReadOptions())); + ASSERT_OK(iter1->status()); + + // Compact 5 files into 1 file in L0 + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("0,1", FilesPerLevel(0)); + + live_files_meta.clear(); + dbfull()->GetLiveFilesMetaData(&live_files_meta); + ASSERT_EQ(live_files_meta.size(), 1); + + live_sst_files_size = 0; + total_sst_files_size = 0; + for (const auto& file_meta : live_files_meta) { + live_sst_files_size += file_meta.size; + } + ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size", + &total_sst_files_size)); + // Live SST files = 1 (compacted file) + // Total SST files = 6 (5 original files + compacted file) + ASSERT_EQ(live_sst_files_size, 1 * single_file_size); + ASSERT_EQ(total_sst_files_size, 6 * single_file_size); + + // hold current version + std::unique_ptr iter2(dbfull()->NewIterator(ReadOptions())); + ASSERT_OK(iter2->status()); + + // Delete all keys and compact, this will delete all live files + for (int i = 0; i < 10; i++) { + ASSERT_OK(Delete(Key(i))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("", FilesPerLevel(0)); + + live_files_meta.clear(); + dbfull()->GetLiveFilesMetaData(&live_files_meta); + ASSERT_EQ(live_files_meta.size(), 0); + + ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size", + &total_sst_files_size)); + // Live SST files = 0 + // Total SST files = 6 (5 original files + compacted file) + ASSERT_EQ(total_sst_files_size, 6 * single_file_size); + + ASSERT_OK(iter1->status()); + iter1.reset(); + ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size", + &total_sst_files_size)); + // Live SST files = 0 + // Total SST files = 1 (compacted file) + ASSERT_EQ(total_sst_files_size, 1 * single_file_size); + + ASSERT_OK(iter2->status()); + iter2.reset(); + ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size", + &total_sst_files_size)); + // Live SST files = 0 + // Total SST files = 0 + ASSERT_EQ(total_sst_files_size, 0); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBSSTTest, OpenDBWithoutGetFileSizeInvocations) { + Options options = CurrentOptions(); + std::unique_ptr env{MockEnv::Create(Env::Default())}; + options.env = env.get(); + options.disable_auto_compactions = true; + options.compression = kNoCompression; + options.enable_blob_files = true; + options.blob_file_size = 32; // create one blob per file + options.skip_checking_sst_file_sizes_on_db_open = true; + + DestroyAndReopen(options); + // Generate 5 files in L0 + for (int i = 0; i < 5; i++) { + for (int j = 0; j < 10; j++) { + std::string val = "val_file_" + std::to_string(i); + ASSERT_OK(Put(Key(j), val)); + } + ASSERT_OK(Flush()); + } + Close(); + + bool is_get_file_size_called = false; + SyncPoint::GetInstance()->SetCallBack( + "MockFileSystem::GetFileSize:CheckFileType", [&](void* arg) { + std::string* filename = reinterpret_cast(arg); + if (filename->find(".blob") != std::string::npos) { + is_get_file_size_called = true; + } + }); + + SyncPoint::GetInstance()->EnableProcessing(); + Reopen(options); + ASSERT_FALSE(is_get_file_size_called); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + Destroy(options); +} + +TEST_F(DBSSTTest, GetTotalSstFilesSizeVersionsFilesShared) { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.compression = kNoCompression; + DestroyAndReopen(options); + // Generate 5 files in L0 + for (int i = 0; i < 5; i++) { + ASSERT_OK(Put(Key(i), "val")); + ASSERT_OK(Flush()); + } + ASSERT_EQ("5", FilesPerLevel(0)); + + std::vector live_files_meta; + dbfull()->GetLiveFilesMetaData(&live_files_meta); + ASSERT_EQ(live_files_meta.size(), 5); + uint64_t single_file_size = live_files_meta[0].size; + + uint64_t live_sst_files_size = 0; + uint64_t total_sst_files_size = 0; + for (const auto& file_meta : live_files_meta) { + live_sst_files_size += file_meta.size; + } + + ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size", + &total_sst_files_size)); + + // Live SST files = 5 + // Total SST files = 5 + ASSERT_EQ(live_sst_files_size, 5 * single_file_size); + ASSERT_EQ(total_sst_files_size, 5 * single_file_size); + + // hold current version + std::unique_ptr iter1(dbfull()->NewIterator(ReadOptions())); + ASSERT_OK(iter1->status()); + + // Compaction will do trivial move from L0 to L1 + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("0,5", FilesPerLevel(0)); + + live_files_meta.clear(); + dbfull()->GetLiveFilesMetaData(&live_files_meta); + ASSERT_EQ(live_files_meta.size(), 5); + + live_sst_files_size = 0; + total_sst_files_size = 0; + for (const auto& file_meta : live_files_meta) { + live_sst_files_size += file_meta.size; + } + ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size", + &total_sst_files_size)); + // Live SST files = 5 + // Total SST files = 5 (used in 2 version) + ASSERT_EQ(live_sst_files_size, 5 * single_file_size); + ASSERT_EQ(total_sst_files_size, 5 * single_file_size); + + // hold current version + std::unique_ptr iter2(dbfull()->NewIterator(ReadOptions())); + ASSERT_OK(iter2->status()); + + // Delete all keys and compact, this will delete all live files + for (int i = 0; i < 5; i++) { + ASSERT_OK(Delete(Key(i))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("", FilesPerLevel(0)); + + live_files_meta.clear(); + dbfull()->GetLiveFilesMetaData(&live_files_meta); + ASSERT_EQ(live_files_meta.size(), 0); + + ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size", + &total_sst_files_size)); + // Live SST files = 0 + // Total SST files = 5 (used in 2 version) + ASSERT_EQ(total_sst_files_size, 5 * single_file_size); + + ASSERT_OK(iter1->status()); + iter1.reset(); + ASSERT_OK(iter2->status()); + iter2.reset(); + + ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size", + &total_sst_files_size)); + // Live SST files = 0 + // Total SST files = 0 + ASSERT_EQ(total_sst_files_size, 0); +} + +// This test if blob files are recorded by SST File Manager when Compaction job +// creates/delete them and in case of AtomicFlush. +TEST_F(DBSSTTest, DBWithSFMForBlobFilesAtomicFlush) { + std::shared_ptr sst_file_manager(NewSstFileManager(env_)); + auto sfm = static_cast(sst_file_manager.get()); + Options options = CurrentOptions(); + options.sst_file_manager = sst_file_manager; + options.enable_blob_files = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.enable_blob_garbage_collection = true; + options.blob_garbage_collection_age_cutoff = 0.5; + options.atomic_flush = true; + + int files_added = 0; + int files_deleted = 0; + int files_scheduled_to_delete = 0; + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SstFileManagerImpl::OnAddFile", [&](void* arg) { + const std::string* const file_path = + static_cast(arg); + if (EndsWith(*file_path, ".blob")) { + files_added++; + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SstFileManagerImpl::OnDeleteFile", [&](void* arg) { + const std::string* const file_path = + static_cast(arg); + if (EndsWith(*file_path, ".blob")) { + files_deleted++; + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SstFileManagerImpl::ScheduleFileDeletion", [&](void* arg) { + assert(arg); + const std::string* const file_path = + static_cast(arg); + if (EndsWith(*file_path, ".blob")) { + ++files_scheduled_to_delete; + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + DestroyAndReopen(options); + Random rnd(301); + + ASSERT_OK(Put("key_1", "value_1")); + ASSERT_OK(Put("key_2", "value_2")); + ASSERT_OK(Put("key_3", "value_3")); + ASSERT_OK(Put("key_4", "value_4")); + ASSERT_OK(Flush()); + + // Overwrite will create the garbage data. + ASSERT_OK(Put("key_3", "new_value_3")); + ASSERT_OK(Put("key_4", "new_value_4")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("Key5", "blob_value5")); + ASSERT_OK(Put("Key6", "blob_value6")); + ASSERT_OK(Flush()); + + ASSERT_EQ(files_added, 3); + ASSERT_EQ(files_deleted, 0); + ASSERT_EQ(files_scheduled_to_delete, 0); + files_added = 0; + + constexpr Slice* begin = nullptr; + constexpr Slice* end = nullptr; + // Compaction job will create a new file and delete the older files. + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), begin, end)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ(files_added, 1); + ASSERT_EQ(files_scheduled_to_delete, 1); + + sfm->WaitForEmptyTrash(); + + ASSERT_EQ(files_deleted, 1); + + Close(); + ASSERT_OK(DestroyDB(dbname_, options)); + + ASSERT_EQ(files_scheduled_to_delete, 4); + + sfm->WaitForEmptyTrash(); + + ASSERT_EQ(files_deleted, 4); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_statistics_test.cc b/librocksdb-sys/rocksdb/db/db_statistics_test.cc new file mode 100644 index 0000000..8b291ac --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_statistics_test.cc @@ -0,0 +1,292 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include + +#include "db/db_test_util.h" +#include "monitoring/thread_status_util.h" +#include "port/stack_trace.h" +#include "rocksdb/statistics.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +class DBStatisticsTest : public DBTestBase { + public: + DBStatisticsTest() + : DBTestBase("db_statistics_test", /*env_do_fsync=*/true) {} +}; + +TEST_F(DBStatisticsTest, CompressionStatsTest) { + for (CompressionType type : GetSupportedCompressions()) { + if (type == kNoCompression) { + continue; + } + if (type == kBZip2Compression) { + // Weird behavior in this test + continue; + } + SCOPED_TRACE("Compression type: " + std::to_string(type)); + + Options options = CurrentOptions(); + options.compression = type; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.statistics->set_stats_level(StatsLevel::kExceptTimeForMutex); + BlockBasedTableOptions bbto; + bbto.enable_index_compression = false; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + DestroyAndReopen(options); + + auto PopStat = [&](Tickers t) -> uint64_t { + return options.statistics->getAndResetTickerCount(t); + }; + + int kNumKeysWritten = 100; + double compress_to = 0.5; + // About three KVs per block + int len = static_cast(BlockBasedTableOptions().block_size / 3); + int uncomp_est = kNumKeysWritten * (len + 20); + + Random rnd(301); + std::string buf; + + // Check that compressions occur and are counted when compression is turned + // on + for (int i = 0; i < kNumKeysWritten; ++i) { + ASSERT_OK( + Put(Key(i), test::CompressibleString(&rnd, compress_to, len, &buf))); + } + ASSERT_OK(Flush()); + EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSED)); + EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSED_FROM), uncomp_est / 10); + EXPECT_NEAR2(uncomp_est * compress_to, PopStat(BYTES_COMPRESSED_TO), + uncomp_est / 10); + + EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED)); + EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM)); + EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO)); + + // And decompressions + for (int i = 0; i < kNumKeysWritten; ++i) { + auto r = Get(Key(i)); + } + EXPECT_EQ(34, PopStat(NUMBER_BLOCK_DECOMPRESSED)); + EXPECT_NEAR2(uncomp_est, PopStat(BYTES_DECOMPRESSED_TO), uncomp_est / 10); + EXPECT_NEAR2(uncomp_est * compress_to, PopStat(BYTES_DECOMPRESSED_FROM), + uncomp_est / 10); + + EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_BYPASSED)); + EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_REJECTED)); + EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED)); + EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED)); + + // Check when compression is rejected. + DestroyAndReopen(options); + + for (int i = 0; i < kNumKeysWritten; ++i) { + ASSERT_OK(Put(Key(i), rnd.RandomBinaryString(len))); + } + ASSERT_OK(Flush()); + for (int i = 0; i < kNumKeysWritten; ++i) { + auto r = Get(Key(i)); + } + EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED)); + EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSION_REJECTED), + uncomp_est / 10); + + EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSED)); + EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED)); + EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED)); + EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_FROM)); + EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_TO)); + EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_BYPASSED)); + EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM)); + EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO)); + + // Check when compression is disabled. + options.compression = kNoCompression; + DestroyAndReopen(options); + + for (int i = 0; i < kNumKeysWritten; ++i) { + ASSERT_OK(Put(Key(i), rnd.RandomBinaryString(len))); + } + ASSERT_OK(Flush()); + for (int i = 0; i < kNumKeysWritten; ++i) { + auto r = Get(Key(i)); + } + EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED)); + EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSION_BYPASSED), + uncomp_est / 10); + + EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSED)); + EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED)); + EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED)); + EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_FROM)); + EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_TO)); + EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_REJECTED)); + EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM)); + EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO)); + } +} + +TEST_F(DBStatisticsTest, MutexWaitStatsDisabledByDefault) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + CreateAndReopenWithCF({"pikachu"}, options); + const uint64_t kMutexWaitDelay = 100; + ThreadStatusUtil::TEST_SetStateDelay(ThreadStatus::STATE_MUTEX_WAIT, + kMutexWaitDelay); + ASSERT_OK(Put("hello", "rocksdb")); + ASSERT_EQ(TestGetTickerCount(options, DB_MUTEX_WAIT_MICROS), 0); + ThreadStatusUtil::TEST_SetStateDelay(ThreadStatus::STATE_MUTEX_WAIT, 0); +} + +TEST_F(DBStatisticsTest, MutexWaitStats) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.statistics->set_stats_level(StatsLevel::kAll); + CreateAndReopenWithCF({"pikachu"}, options); + const uint64_t kMutexWaitDelay = 100; + ThreadStatusUtil::TEST_SetStateDelay(ThreadStatus::STATE_MUTEX_WAIT, + kMutexWaitDelay); + ASSERT_OK(Put("hello", "rocksdb")); + ASSERT_GE(TestGetTickerCount(options, DB_MUTEX_WAIT_MICROS), kMutexWaitDelay); + ThreadStatusUtil::TEST_SetStateDelay(ThreadStatus::STATE_MUTEX_WAIT, 0); +} + +TEST_F(DBStatisticsTest, ResetStats) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + DestroyAndReopen(options); + for (int i = 0; i < 2; ++i) { + // pick arbitrary ticker and histogram. On first iteration they're zero + // because db is unused. On second iteration they're zero due to Reset(). + ASSERT_EQ(0, TestGetTickerCount(options, NUMBER_KEYS_WRITTEN)); + HistogramData histogram_data; + options.statistics->histogramData(DB_WRITE, &histogram_data); + ASSERT_EQ(0.0, histogram_data.max); + + if (i == 0) { + // The Put() makes some of the ticker/histogram stats nonzero until we + // Reset(). + ASSERT_OK(Put("hello", "rocksdb")); + ASSERT_EQ(1, TestGetTickerCount(options, NUMBER_KEYS_WRITTEN)); + options.statistics->histogramData(DB_WRITE, &histogram_data); + ASSERT_GT(histogram_data.max, 0.0); + ASSERT_OK(options.statistics->Reset()); + } + } +} + +TEST_F(DBStatisticsTest, ExcludeTickers) { + Options options = CurrentOptions(); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + DestroyAndReopen(options); + options.statistics->set_stats_level(StatsLevel::kExceptTickers); + ASSERT_OK(Put("foo", "value")); + ASSERT_EQ(0, options.statistics->getTickerCount(BYTES_WRITTEN)); + options.statistics->set_stats_level(StatsLevel::kExceptHistogramOrTimers); + Reopen(options); + ASSERT_EQ("value", Get("foo")); + ASSERT_GT(options.statistics->getTickerCount(BYTES_READ), 0); +} + + +TEST_F(DBStatisticsTest, VerifyChecksumReadStat) { + Options options = CurrentOptions(); + options.file_checksum_gen_factory = GetFileChecksumGenCrc32cFactory(); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + Reopen(options); + + // Expected to be populated regardless of `PerfLevel` in user thread + SetPerfLevel(kDisable); + + { + // Scenario 0: only WAL data. Not verified so require ticker to be zero. + ASSERT_OK(Put("foo", "value")); + ASSERT_OK(db_->VerifyFileChecksums(ReadOptions())); + ASSERT_OK(db_->VerifyChecksum()); + ASSERT_EQ(0, + options.statistics->getTickerCount(VERIFY_CHECKSUM_READ_BYTES)); + } + + // Create one SST. + ASSERT_OK(Flush()); + std::unordered_map table_files; + uint64_t table_files_size = 0; + GetAllDataFiles(kTableFile, &table_files, &table_files_size); + + { + // Scenario 1: Table verified in `VerifyFileChecksums()`. This should read + // the whole file so we require the ticker stat exactly matches the file + // size. + ASSERT_OK(options.statistics->Reset()); + ASSERT_OK(db_->VerifyFileChecksums(ReadOptions())); + ASSERT_EQ(table_files_size, + options.statistics->getTickerCount(VERIFY_CHECKSUM_READ_BYTES)); + } + + { + // Scenario 2: Table verified in `VerifyChecksum()`. This opens a + // `TableReader` to verify each block. It can involve duplicate reads of the + // same data so we set a lower-bound only. + ASSERT_OK(options.statistics->Reset()); + ASSERT_OK(db_->VerifyChecksum()); + ASSERT_GE(options.statistics->getTickerCount(VERIFY_CHECKSUM_READ_BYTES), + table_files_size); + } +} + +TEST_F(DBStatisticsTest, BlockChecksumStats) { + Options options = CurrentOptions(); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + Reopen(options); + + // Scenario 0: only WAL data. Not verified so require ticker to be zero. + ASSERT_OK(Put("foo", "value")); + ASSERT_OK(db_->VerifyChecksum()); + ASSERT_EQ(0, + options.statistics->getTickerCount(BLOCK_CHECKSUM_COMPUTE_COUNT)); + ASSERT_EQ(0, + options.statistics->getTickerCount(BLOCK_CHECKSUM_MISMATCH_COUNT)); + + // Scenario 1: Flushed table verified in `VerifyChecksum()`. This opens a + // `TableReader` to verify each of the four blocks (meta-index, table + // properties, index, and data block). + ASSERT_OK(Flush()); + ASSERT_OK(options.statistics->Reset()); + ASSERT_OK(db_->VerifyChecksum()); + ASSERT_EQ(4, + options.statistics->getTickerCount(BLOCK_CHECKSUM_COMPUTE_COUNT)); + ASSERT_EQ(0, + options.statistics->getTickerCount(BLOCK_CHECKSUM_MISMATCH_COUNT)); + + // Scenario 2: Corrupted table verified in `VerifyChecksum()`. The corruption + // is in the fourth and final verified block, i.e., the data block. + std::unordered_map table_files; + ASSERT_OK(GetAllDataFiles(kTableFile, &table_files)); + ASSERT_EQ(1, table_files.size()); + std::string table_name = table_files.begin()->first; + // Assumes the data block starts at offset zero. + ASSERT_OK(test::CorruptFile(options.env, table_name, 0 /* offset */, + 3 /* bytes_to_corrupt */)); + ASSERT_OK(options.statistics->Reset()); + ASSERT_NOK(db_->VerifyChecksum()); + ASSERT_EQ(4, + options.statistics->getTickerCount(BLOCK_CHECKSUM_COMPUTE_COUNT)); + ASSERT_EQ(1, + options.statistics->getTickerCount(BLOCK_CHECKSUM_MISMATCH_COUNT)); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_table_properties_test.cc b/librocksdb-sys/rocksdb/db/db_table_properties_test.cc new file mode 100644 index 0000000..61dcf3c --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_table_properties_test.cc @@ -0,0 +1,624 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include +#include + +#include "db/db_test_util.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/db.h" +#include "rocksdb/types.h" +#include "rocksdb/utilities/table_properties_collectors.h" +#include "table/format.h" +#include "table/meta_blocks.h" +#include "table/table_properties_internal.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/random.h" + + +namespace ROCKSDB_NAMESPACE { + +// A helper function that ensures the table properties returned in +// `GetPropertiesOfAllTablesTest` is correct. +// This test assumes entries size is different for each of the tables. +namespace { + +void VerifyTableProperties(DB* db, uint64_t expected_entries_size) { + TablePropertiesCollection props; + ASSERT_OK(db->GetPropertiesOfAllTables(&props)); + + ASSERT_EQ(4U, props.size()); + std::unordered_set unique_entries; + + // Indirect test + uint64_t sum = 0; + for (const auto& item : props) { + unique_entries.insert(item.second->num_entries); + sum += item.second->num_entries; + } + + ASSERT_EQ(props.size(), unique_entries.size()); + ASSERT_EQ(expected_entries_size, sum); + + VerifySstUniqueIds(props); +} +} // anonymous namespace + +class DBTablePropertiesTest : public DBTestBase, + public testing::WithParamInterface { + public: + DBTablePropertiesTest() + : DBTestBase("db_table_properties_test", /*env_do_fsync=*/false) {} + TablePropertiesCollection TestGetPropertiesOfTablesInRange( + std::vector ranges, std::size_t* num_properties = nullptr, + std::size_t* num_files = nullptr); +}; + +TEST_F(DBTablePropertiesTest, GetPropertiesOfAllTablesTest) { + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = 8; + // Part of strategy to prevent pinning table files + options.max_open_files = 42; + Reopen(options); + + // Create 4 tables + for (int table = 0; table < 4; ++table) { + // Use old meta name for table properties for one file + if (table == 3) { + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTableBuilder::WritePropertiesBlock:Meta", [&](void* meta) { + *reinterpret_cast(meta) = + &kPropertiesBlockOldName; + }); + SyncPoint::GetInstance()->EnableProcessing(); + } + // Build file + for (int i = 0; i < 10 + table; ++i) { + ASSERT_OK( + db_->Put(WriteOptions(), std::to_string(table * 100 + i), "val")); + } + ASSERT_OK(db_->Flush(FlushOptions())); + } + SyncPoint::GetInstance()->DisableProcessing(); + std::string original_session_id; + ASSERT_OK(db_->GetDbSessionId(original_session_id)); + + // Part of strategy to prevent pinning table files + SyncPoint::GetInstance()->SetCallBack( + "VersionEditHandler::LoadTables:skip_load_table_files", + [&](void* skip_load) { *reinterpret_cast(skip_load) = true; }); + SyncPoint::GetInstance()->EnableProcessing(); + + // 1. Read table properties directly from file + Reopen(options); + // Clear out auto-opened files + dbfull()->TEST_table_cache()->EraseUnRefEntries(); + ASSERT_EQ(dbfull()->TEST_table_cache()->GetUsage(), 0U); + VerifyTableProperties(db_, 10 + 11 + 12 + 13); + + // 2. Put two tables to table cache and + Reopen(options); + // Clear out auto-opened files + dbfull()->TEST_table_cache()->EraseUnRefEntries(); + ASSERT_EQ(dbfull()->TEST_table_cache()->GetUsage(), 0U); + // fetch key from 1st and 2nd table, which will internally place that table to + // the table cache. + for (int i = 0; i < 2; ++i) { + Get(std::to_string(i * 100 + 0)); + } + + VerifyTableProperties(db_, 10 + 11 + 12 + 13); + + // 3. Put all tables to table cache + Reopen(options); + // fetch key from all tables, which will place them in table cache. + for (int i = 0; i < 4; ++i) { + Get(std::to_string(i * 100 + 0)); + } + VerifyTableProperties(db_, 10 + 11 + 12 + 13); + + // 4. Try to read CORRUPT properties (a) directly from file, and (b) + // through reader on Get + + // It's not practical to prevent table file read on Open, so we + // corrupt after open and after purging table cache. + for (bool direct : {true, false}) { + Reopen(options); + // Clear out auto-opened files + dbfull()->TEST_table_cache()->EraseUnRefEntries(); + ASSERT_EQ(dbfull()->TEST_table_cache()->GetUsage(), 0U); + + TablePropertiesCollection props; + ASSERT_OK(db_->GetPropertiesOfAllTables(&props)); + std::string sst_file = props.begin()->first; + + // Corrupt the file's TableProperties using session id + std::string contents; + ASSERT_OK( + ReadFileToString(env_->GetFileSystem().get(), sst_file, &contents)); + size_t pos = contents.find(original_session_id); + ASSERT_NE(pos, std::string::npos); + ASSERT_OK(test::CorruptFile(env_, sst_file, static_cast(pos), 1, + /*verify checksum fails*/ false)); + + // Try to read CORRUPT properties + if (direct) { + ASSERT_TRUE(db_->GetPropertiesOfAllTables(&props).IsCorruption()); + } else { + bool found_corruption = false; + for (int i = 0; i < 4; ++i) { + std::string result = Get(std::to_string(i * 100 + 0)); + if (result.find_first_of("Corruption: block checksum mismatch") != + std::string::npos) { + found_corruption = true; + } + } + ASSERT_TRUE(found_corruption); + } + + // UN-corrupt file for next iteration + ASSERT_OK(test::CorruptFile(env_, sst_file, static_cast(pos), 1, + /*verify checksum fails*/ false)); + } + + SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBTablePropertiesTest, InvalidIgnored) { + // RocksDB versions 2.5 - 2.7 generate some properties that Block considers + // invalid in some way. This approximates that. + + // Inject properties block data that Block considers invalid + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTableBuilder::WritePropertiesBlock:BlockData", + [&](void* block_data) { + *reinterpret_cast(block_data) = Slice("X"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + // Corrupting the table properties corrupts the unique id. + // Ignore the unique id recorded in the manifest. + auto options = CurrentOptions(); + options.verify_sst_unique_id_in_manifest = false; + Reopen(options); + + // Build file + for (int i = 0; i < 10; ++i) { + ASSERT_OK(db_->Put(WriteOptions(), std::to_string(i), "val")); + } + ASSERT_OK(db_->Flush(FlushOptions())); + + SyncPoint::GetInstance()->DisableProcessing(); + + // Not crashing is good enough + TablePropertiesCollection props; + ASSERT_OK(db_->GetPropertiesOfAllTables(&props)); +} + +TEST_F(DBTablePropertiesTest, CreateOnDeletionCollectorFactory) { + ConfigOptions options; + options.ignore_unsupported_options = false; + + std::shared_ptr factory; + std::string id = CompactOnDeletionCollectorFactory::kClassName(); + ASSERT_OK( + TablePropertiesCollectorFactory::CreateFromString(options, id, &factory)); + auto del_factory = factory->CheckedCast(); + ASSERT_NE(del_factory, nullptr); + ASSERT_EQ(0U, del_factory->GetWindowSize()); + ASSERT_EQ(0U, del_factory->GetDeletionTrigger()); + ASSERT_EQ(0.0, del_factory->GetDeletionRatio()); + ASSERT_OK(TablePropertiesCollectorFactory::CreateFromString( + options, "window_size=100; deletion_trigger=90; id=" + id, &factory)); + del_factory = factory->CheckedCast(); + ASSERT_NE(del_factory, nullptr); + ASSERT_EQ(100U, del_factory->GetWindowSize()); + ASSERT_EQ(90U, del_factory->GetDeletionTrigger()); + ASSERT_EQ(0.0, del_factory->GetDeletionRatio()); + ASSERT_OK(TablePropertiesCollectorFactory::CreateFromString( + options, + "window_size=100; deletion_trigger=90; deletion_ratio=0.5; id=" + id, + &factory)); + del_factory = factory->CheckedCast(); + ASSERT_NE(del_factory, nullptr); + ASSERT_EQ(100U, del_factory->GetWindowSize()); + ASSERT_EQ(90U, del_factory->GetDeletionTrigger()); + ASSERT_EQ(0.5, del_factory->GetDeletionRatio()); +} + +TablePropertiesCollection +DBTablePropertiesTest::TestGetPropertiesOfTablesInRange( + std::vector ranges, std::size_t* num_properties, + std::size_t* num_files) { + // Since we deref zero element in the vector it can not be empty + // otherwise we pass an address to some random memory + EXPECT_GT(ranges.size(), 0U); + // run the query + TablePropertiesCollection props; + EXPECT_OK(db_->GetPropertiesOfTablesInRange( + db_->DefaultColumnFamily(), &ranges[0], ranges.size(), &props)); + + // Make sure that we've received properties for those and for those files + // only which fall within requested ranges + std::vector vmd; + db_->GetLiveFilesMetaData(&vmd); + for (auto& md : vmd) { + std::string fn = md.db_path + md.name; + bool in_range = false; + for (auto& r : ranges) { + // smallestkey < limit && largestkey >= start + if (r.limit.compare(md.smallestkey) >= 0 && + r.start.compare(md.largestkey) <= 0) { + in_range = true; + EXPECT_GT(props.count(fn), 0); + } + } + if (!in_range) { + EXPECT_EQ(props.count(fn), 0); + } + } + + if (num_properties) { + *num_properties = props.size(); + } + + if (num_files) { + *num_files = vmd.size(); + } + return props; +} + +TEST_F(DBTablePropertiesTest, GetPropertiesOfTablesInRange) { + // Fixed random sead + Random rnd(301); + + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.create_if_missing = true; + options.write_buffer_size = 4096; + options.max_write_buffer_number = 2; + options.level0_file_num_compaction_trigger = 2; + options.level0_slowdown_writes_trigger = 2; + options.level0_stop_writes_trigger = 2; + options.target_file_size_base = 2048; + options.max_bytes_for_level_base = 40960; + options.max_bytes_for_level_multiplier = 4; + options.hard_pending_compaction_bytes_limit = 16 * 1024; + options.num_levels = 8; + options.env = env_; + + DestroyAndReopen(options); + + // build a decent LSM + for (int i = 0; i < 10000; i++) { + ASSERT_OK(Put(test::RandomKey(&rnd, 5), rnd.RandomString(102))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + if (NumTableFilesAtLevel(0) == 0) { + ASSERT_OK(Put(test::RandomKey(&rnd, 5), rnd.RandomString(102))); + ASSERT_OK(Flush()); + } + + ASSERT_OK(db_->PauseBackgroundWork()); + + // Ensure that we have at least L0, L1 and L2 + ASSERT_GT(NumTableFilesAtLevel(0), 0); + ASSERT_GT(NumTableFilesAtLevel(1), 0); + ASSERT_GT(NumTableFilesAtLevel(2), 0); + + // Query the largest range + std::size_t num_properties, num_files; + TestGetPropertiesOfTablesInRange( + {Range(test::RandomKey(&rnd, 5, test::RandomKeyType::SMALLEST), + test::RandomKey(&rnd, 5, test::RandomKeyType::LARGEST))}, + &num_properties, &num_files); + ASSERT_EQ(num_properties, num_files); + + // Query the empty range + TestGetPropertiesOfTablesInRange( + {Range(test::RandomKey(&rnd, 5, test::RandomKeyType::LARGEST), + test::RandomKey(&rnd, 5, test::RandomKeyType::SMALLEST))}, + &num_properties, &num_files); + ASSERT_GT(num_files, 0); + ASSERT_EQ(num_properties, 0); + + // Query the middle rangee + TestGetPropertiesOfTablesInRange( + {Range(test::RandomKey(&rnd, 5, test::RandomKeyType::MIDDLE), + test::RandomKey(&rnd, 5, test::RandomKeyType::LARGEST))}, + &num_properties, &num_files); + ASSERT_GT(num_files, 0); + ASSERT_GT(num_files, num_properties); + ASSERT_GT(num_properties, 0); + + // Query a bunch of random ranges + for (int j = 0; j < 100; j++) { + // create a bunch of ranges + std::vector random_keys; + // Random returns numbers with zero included + // when we pass empty ranges TestGetPropertiesOfTablesInRange() + // derefs random memory in the empty ranges[0] + // so want to be greater than zero and even since + // the below loop requires that random_keys.size() to be even. + auto n = 2 * (rnd.Uniform(50) + 1); + + for (uint32_t i = 0; i < n; ++i) { + random_keys.push_back(test::RandomKey(&rnd, 5)); + } + + ASSERT_GT(random_keys.size(), 0U); + ASSERT_EQ((random_keys.size() % 2), 0U); + + std::vector ranges; + auto it = random_keys.begin(); + while (it != random_keys.end()) { + ranges.push_back(Range(*it, *(it + 1))); + it += 2; + } + + TestGetPropertiesOfTablesInRange(std::move(ranges)); + } +} + +TEST_F(DBTablePropertiesTest, GetColumnFamilyNameProperty) { + std::string kExtraCfName = "pikachu"; + CreateAndReopenWithCF({kExtraCfName}, CurrentOptions()); + + // Create one table per CF, then verify it was created with the column family + // name property. + for (uint32_t cf = 0; cf < 2; ++cf) { + ASSERT_OK(Put(cf, "key", "val")); + ASSERT_OK(Flush(cf)); + + TablePropertiesCollection fname_to_props; + ASSERT_OK(db_->GetPropertiesOfAllTables(handles_[cf], &fname_to_props)); + ASSERT_EQ(1U, fname_to_props.size()); + + std::string expected_cf_name; + if (cf > 0) { + expected_cf_name = kExtraCfName; + } else { + expected_cf_name = kDefaultColumnFamilyName; + } + ASSERT_EQ(expected_cf_name, + fname_to_props.begin()->second->column_family_name); + ASSERT_EQ(cf, static_cast( + fname_to_props.begin()->second->column_family_id)); + } +} + +TEST_F(DBTablePropertiesTest, GetDbIdentifiersProperty) { + CreateAndReopenWithCF({"goku"}, CurrentOptions()); + + for (uint32_t cf = 0; cf < 2; ++cf) { + ASSERT_OK(Put(cf, "key", "val")); + ASSERT_OK(Put(cf, "foo", "bar")); + ASSERT_OK(Flush(cf)); + + TablePropertiesCollection fname_to_props; + ASSERT_OK(db_->GetPropertiesOfAllTables(handles_[cf], &fname_to_props)); + ASSERT_EQ(1U, fname_to_props.size()); + + std::string id, sid; + ASSERT_OK(db_->GetDbIdentity(id)); + ASSERT_OK(db_->GetDbSessionId(sid)); + ASSERT_EQ(id, fname_to_props.begin()->second->db_id); + ASSERT_EQ(sid, fname_to_props.begin()->second->db_session_id); + } +} + +class DBTableHostnamePropertyTest + : public DBTestBase, + public ::testing::WithParamInterface> { + public: + DBTableHostnamePropertyTest() + : DBTestBase("db_table_hostname_property_test", + /*env_do_fsync=*/false) {} +}; + +TEST_P(DBTableHostnamePropertyTest, DbHostLocationProperty) { + option_config_ = std::get<0>(GetParam()); + Options opts = CurrentOptions(); + std::string expected_host_id = std::get<1>(GetParam()); + ; + if (expected_host_id == kHostnameForDbHostId) { + ASSERT_OK(env_->GetHostNameString(&expected_host_id)); + } else { + opts.db_host_id = expected_host_id; + } + CreateAndReopenWithCF({"goku"}, opts); + + for (uint32_t cf = 0; cf < 2; ++cf) { + ASSERT_OK(Put(cf, "key", "val")); + ASSERT_OK(Put(cf, "foo", "bar")); + ASSERT_OK(Flush(cf)); + + TablePropertiesCollection fname_to_props; + ASSERT_OK(db_->GetPropertiesOfAllTables(handles_[cf], &fname_to_props)); + ASSERT_EQ(1U, fname_to_props.size()); + + ASSERT_EQ(fname_to_props.begin()->second->db_host_id, expected_host_id); + } +} + +INSTANTIATE_TEST_CASE_P( + DBTableHostnamePropertyTest, DBTableHostnamePropertyTest, + ::testing::Values( + // OptionConfig, override db_host_location + std::make_tuple(DBTestBase::OptionConfig::kDefault, + kHostnameForDbHostId), + std::make_tuple(DBTestBase::OptionConfig::kDefault, "foobar"), + std::make_tuple(DBTestBase::OptionConfig::kDefault, ""), + std::make_tuple(DBTestBase::OptionConfig::kPlainTableFirstBytePrefix, + kHostnameForDbHostId), + std::make_tuple(DBTestBase::OptionConfig::kPlainTableFirstBytePrefix, + "foobar"), + std::make_tuple(DBTestBase::OptionConfig::kPlainTableFirstBytePrefix, + ""))); + +class DeletionTriggeredCompactionTestListener : public EventListener { + public: + void OnCompactionBegin(DB*, const CompactionJobInfo& ci) override { + ASSERT_EQ(ci.compaction_reason, + CompactionReason::kFilesMarkedForCompaction); + } + + void OnCompactionCompleted(DB*, const CompactionJobInfo& ci) override { + ASSERT_EQ(ci.compaction_reason, + CompactionReason::kFilesMarkedForCompaction); + } +}; + +TEST_P(DBTablePropertiesTest, DeletionTriggeredCompactionMarking) { + int kNumKeys = 1000; + int kWindowSize = 100; + int kNumDelsTrigger = 90; + std::shared_ptr compact_on_del = + NewCompactOnDeletionCollectorFactory(kWindowSize, kNumDelsTrigger); + + Options opts = CurrentOptions(); + opts.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + opts.table_properties_collector_factories.emplace_back(compact_on_del); + + if (GetParam() == "kCompactionStyleUniversal") { + opts.compaction_style = kCompactionStyleUniversal; + } + Reopen(opts); + + // add an L1 file to prevent tombstones from dropping due to obsolescence + // during flush + ASSERT_OK(Put(Key(0), "val")); + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + + DeletionTriggeredCompactionTestListener* listener = + new DeletionTriggeredCompactionTestListener(); + opts.listeners.emplace_back(listener); + Reopen(opts); + + for (int i = 0; i < kNumKeys; ++i) { + if (i >= kNumKeys - kWindowSize && + i < kNumKeys - kWindowSize + kNumDelsTrigger) { + ASSERT_OK(Delete(Key(i))); + } else { + ASSERT_OK(Put(Key(i), "val")); + } + } + ASSERT_OK(Flush()); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + + // Change the window size and deletion trigger and ensure new values take + // effect + kWindowSize = 50; + kNumDelsTrigger = 40; + static_cast(compact_on_del.get()) + ->SetWindowSize(kWindowSize); + static_cast(compact_on_del.get()) + ->SetDeletionTrigger(kNumDelsTrigger); + for (int i = 0; i < kNumKeys; ++i) { + if (i >= kNumKeys - kWindowSize && + i < kNumKeys - kWindowSize + kNumDelsTrigger) { + ASSERT_OK(Delete(Key(i))); + } else { + ASSERT_OK(Put(Key(i), "val")); + } + } + ASSERT_OK(Flush()); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + + // Change the window size to disable delete triggered compaction + kWindowSize = 0; + static_cast(compact_on_del.get()) + ->SetWindowSize(kWindowSize); + static_cast(compact_on_del.get()) + ->SetDeletionTrigger(kNumDelsTrigger); + for (int i = 0; i < kNumKeys; ++i) { + if (i >= kNumKeys - kWindowSize && + i < kNumKeys - kWindowSize + kNumDelsTrigger) { + ASSERT_OK(Delete(Key(i))); + } else { + ASSERT_OK(Put(Key(i), "val")); + } + } + ASSERT_OK(Flush()); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + ASSERT_LT(0, opts.statistics->getTickerCount(COMPACT_WRITE_BYTES_MARKED)); + ASSERT_LT(0, opts.statistics->getTickerCount(COMPACT_READ_BYTES_MARKED)); +} + +TEST_P(DBTablePropertiesTest, RatioBasedDeletionTriggeredCompactionMarking) { + constexpr int kNumKeys = 1000; + constexpr int kWindowSize = 0; + constexpr int kNumDelsTrigger = 0; + constexpr double kDeletionRatio = 0.1; + std::shared_ptr compact_on_del = + NewCompactOnDeletionCollectorFactory(kWindowSize, kNumDelsTrigger, + kDeletionRatio); + + Options opts = CurrentOptions(); + opts.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + opts.table_properties_collector_factories.emplace_back(compact_on_del); + + Reopen(opts); + + // Add an L2 file to prevent tombstones from dropping due to obsolescence + // during flush + ASSERT_OK(Put(Key(0), "val")); + ASSERT_OK(Flush()); + MoveFilesToLevel(2); + + auto* listener = new DeletionTriggeredCompactionTestListener(); + opts.listeners.emplace_back(listener); + Reopen(opts); + + // Generate one L0 with kNumKeys Put. + for (int i = 0; i < kNumKeys; ++i) { + ASSERT_OK(Put(Key(i), "not important")); + } + ASSERT_OK(Flush()); + + // Generate another L0 with kNumKeys Delete. + // This file, due to deletion ratio, will trigger compaction: 2@0 files to L1. + // The resulting L1 file has only one tombstone for user key 'Key(0)'. + // Again, due to deletion ratio, a compaction will be triggered: 1@1 + 1@2 + // files to L2. However, the resulting file is empty because the tombstone + // and value are both dropped. + for (int i = 0; i < kNumKeys; ++i) { + ASSERT_OK(Delete(Key(i))); + } + ASSERT_OK(Flush()); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + for (int i = 0; i < 3; ++i) { + ASSERT_EQ(0, NumTableFilesAtLevel(i)); + } +} + +INSTANTIATE_TEST_CASE_P(DBTablePropertiesTest, DBTablePropertiesTest, + ::testing::Values("kCompactionStyleLevel", + "kCompactionStyleUniversal")); + +} // namespace ROCKSDB_NAMESPACE + + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_tailing_iter_test.cc b/librocksdb-sys/rocksdb/db/db_tailing_iter_test.cc new file mode 100644 index 0000000..d3debed --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_tailing_iter_test.cc @@ -0,0 +1,708 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +// Introduction of SyncPoint effectively disabled building and running this test +// in Release build. +// which is a pity, it is a good test + +#include "db/db_test_util.h" +#include "db/forward_iterator.h" +#include "port/stack_trace.h" + +namespace { +static bool enable_io_uring = true; +extern "C" bool RocksDbIOUringEnable() { return enable_io_uring; } +} // namespace + +namespace ROCKSDB_NAMESPACE { + +class DBTestTailingIterator : public DBTestBase, + public ::testing::WithParamInterface { + public: + DBTestTailingIterator() + : DBTestBase("db_tailing_iterator_test", /*env_do_fsync=*/true) {} +}; + +INSTANTIATE_TEST_CASE_P(DBTestTailingIterator, DBTestTailingIterator, + ::testing::Bool()); + +TEST_P(DBTestTailingIterator, TailingIteratorSingle) { + ReadOptions read_options; + read_options.tailing = true; + if (GetParam()) { + read_options.async_io = true; + } + + std::unique_ptr iter(db_->NewIterator(read_options)); + iter->SeekToFirst(); + ASSERT_TRUE(!iter->Valid()); + ASSERT_OK(iter->status()); + + // add a record and check that iter can see it + ASSERT_OK(db_->Put(WriteOptions(), "mirko", "fodor")); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().ToString(), "mirko"); + + iter->Next(); + ASSERT_TRUE(!iter->Valid()); +} + +TEST_P(DBTestTailingIterator, TailingIteratorKeepAdding) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_BYPASS("Test requires non-mem or non-encrypted environment"); + return; + } + std::unique_ptr env( + new CompositeEnvWrapper(env_, FileSystem::Default())); + Options options = CurrentOptions(); + options.env = env.get(); + CreateAndReopenWithCF({"pikachu"}, options); + ReadOptions read_options; + read_options.tailing = true; + if (GetParam()) { + read_options.async_io = true; + } + + { + std::unique_ptr iter(db_->NewIterator(read_options, handles_[1])); + ASSERT_OK(iter->status()); + std::string value(1024, 'a'); + + const int num_records = 10000; + for (int i = 0; i < num_records; ++i) { + char buf[32]; + snprintf(buf, sizeof(buf), "%016d", i); + + Slice key(buf, 16); + ASSERT_OK(Put(1, key, value)); + + iter->Seek(key); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(key), 0); + } + } + Close(); +} + +TEST_P(DBTestTailingIterator, TailingIteratorSeekToNext) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_BYPASS("Test requires non-mem or non-encrypted environment"); + return; + } + std::unique_ptr env( + new CompositeEnvWrapper(env_, FileSystem::Default())); + Options options = CurrentOptions(); + options.env = env.get(); + CreateAndReopenWithCF({"pikachu"}, options); + ReadOptions read_options; + read_options.tailing = true; + if (GetParam()) { + read_options.async_io = true; + } + { + std::unique_ptr iter(db_->NewIterator(read_options, handles_[1])); + ASSERT_OK(iter->status()); + std::unique_ptr itern( + db_->NewIterator(read_options, handles_[1])); + ASSERT_OK(itern->status()); + std::string value(1024, 'a'); + + const int num_records = 1000; + for (int i = 1; i < num_records; ++i) { + char buf1[32]; + char buf2[32]; + snprintf(buf1, sizeof(buf1), "00a0%016d", i * 5); + + Slice key(buf1, 20); + ASSERT_OK(Put(1, key, value)); + + if (i % 100 == 99) { + ASSERT_OK(Flush(1)); + } + + snprintf(buf2, sizeof(buf2), "00a0%016d", i * 5 - 2); + Slice target(buf2, 20); + iter->Seek(target); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(key), 0); + if (i == 1) { + itern->SeekToFirst(); + } else { + itern->Next(); + } + ASSERT_TRUE(itern->Valid()); + ASSERT_EQ(itern->key().compare(key), 0); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + for (int i = 2 * num_records; i > 0; --i) { + char buf1[32]; + char buf2[32]; + snprintf(buf1, sizeof(buf1), "00a0%016d", i * 5); + + Slice key(buf1, 20); + ASSERT_OK(Put(1, key, value)); + + if (i % 100 == 99) { + ASSERT_OK(Flush(1)); + } + + snprintf(buf2, sizeof(buf2), "00a0%016d", i * 5 - 2); + Slice target(buf2, 20); + iter->Seek(target); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(key), 0); + } + } + Close(); +} + +TEST_P(DBTestTailingIterator, TailingIteratorTrimSeekToNext) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_BYPASS("Test requires non-mem or non-encrypted environment"); + return; + } + const uint64_t k150KB = 150 * 1024; + std::unique_ptr env( + new CompositeEnvWrapper(env_, FileSystem::Default())); + Options options; + options.env = env.get(); + options.write_buffer_size = k150KB; + options.max_write_buffer_number = 3; + options.min_write_buffer_number_to_merge = 2; + options.env = env_; + CreateAndReopenWithCF({"pikachu"}, options); + ReadOptions read_options; + read_options.tailing = true; + if (GetParam()) { + read_options.async_io = true; + } + int num_iters, deleted_iters; + + char bufe[32]; + snprintf(bufe, sizeof(bufe), "00b0%016d", 0); + Slice keyu(bufe, 20); + read_options.iterate_upper_bound = &keyu; + std::unique_ptr iter(db_->NewIterator(read_options, handles_[1])); + ASSERT_OK(iter->status()); + std::unique_ptr itern(db_->NewIterator(read_options, handles_[1])); + ASSERT_OK(itern->status()); + std::unique_ptr iterh(db_->NewIterator(read_options, handles_[1])); + ASSERT_OK(iterh->status()); + std::string value(1024, 'a'); + bool file_iters_deleted = false; + bool file_iters_renewed_null = false; + bool file_iters_renewed_copy = false; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "ForwardIterator::SeekInternal:Return", [&](void* arg) { + ForwardIterator* fiter = reinterpret_cast(arg); + ASSERT_TRUE(!file_iters_deleted || + fiter->TEST_CheckDeletedIters(&deleted_iters, &num_iters)); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "ForwardIterator::Next:Return", [&](void* arg) { + ForwardIterator* fiter = reinterpret_cast(arg); + ASSERT_TRUE(!file_iters_deleted || + fiter->TEST_CheckDeletedIters(&deleted_iters, &num_iters)); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "ForwardIterator::RenewIterators:Null", + [&](void* /*arg*/) { file_iters_renewed_null = true; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "ForwardIterator::RenewIterators:Copy", + [&](void* /*arg*/) { file_iters_renewed_copy = true; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + const int num_records = 1000; + for (int i = 1; i < num_records; ++i) { + char buf1[32]; + char buf2[32]; + char buf3[32]; + char buf4[32]; + snprintf(buf1, sizeof(buf1), "00a0%016d", i * 5); + snprintf(buf3, sizeof(buf3), "00b0%016d", i * 5); + + Slice key(buf1, 20); + ASSERT_OK(Put(1, key, value)); + Slice keyn(buf3, 20); + ASSERT_OK(Put(1, keyn, value)); + + if (i % 100 == 99) { + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + if (i == 299) { + file_iters_deleted = true; + } + snprintf(buf4, sizeof(buf4), "00a0%016d", i * 5 / 2); + Slice target(buf4, 20); + iterh->Seek(target); + ASSERT_TRUE(iter->Valid()); + for (int j = (i + 1) * 5 / 2; j < i * 5; j += 5) { + iterh->Next(); + ASSERT_TRUE(iterh->Valid()); + } + if (i == 299) { + file_iters_deleted = false; + } + } + + file_iters_deleted = true; + snprintf(buf2, sizeof(buf2), "00a0%016d", i * 5 - 2); + Slice target(buf2, 20); + iter->Seek(target); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(key), 0); + ASSERT_LE(num_iters, 1); + if (i == 1) { + itern->SeekToFirst(); + } else { + itern->Next(); + } + ASSERT_TRUE(itern->Valid()); + ASSERT_EQ(itern->key().compare(key), 0); + ASSERT_LE(num_iters, 1); + file_iters_deleted = false; + } + ASSERT_TRUE(file_iters_renewed_null); + ASSERT_TRUE(file_iters_renewed_copy); + iter = nullptr; + itern = nullptr; + iterh = nullptr; + BlockBasedTableOptions table_options; + table_options.no_block_cache = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + read_options.read_tier = kBlockCacheTier; + std::unique_ptr iteri(db_->NewIterator(read_options, handles_[1])); + ASSERT_OK(iteri->status()); + char buf5[32]; + snprintf(buf5, sizeof(buf5), "00a0%016d", (num_records / 2) * 5 - 2); + Slice target1(buf5, 20); + iteri->Seek(target1); + ASSERT_TRUE(iteri->status().IsIncomplete()); + iteri = nullptr; + + read_options.read_tier = kReadAllTier; + options.table_factory.reset(NewBlockBasedTableFactory()); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + iter.reset(db_->NewIterator(read_options, handles_[1])); + ASSERT_OK(iter->status()); + for (int i = 2 * num_records; i > 0; --i) { + char buf1[32]; + char buf2[32]; + snprintf(buf1, sizeof(buf1), "00a0%016d", i * 5); + + Slice key(buf1, 20); + ASSERT_OK(Put(1, key, value)); + + if (i % 100 == 99) { + ASSERT_OK(Flush(1)); + } + + snprintf(buf2, sizeof(buf2), "00a0%016d", i * 5 - 2); + Slice target(buf2, 20); + iter->Seek(target); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(key), 0); + } +} + +TEST_P(DBTestTailingIterator, TailingIteratorDeletes) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_BYPASS("Test requires non-mem or non-encrypted environment"); + return; + } + std::unique_ptr env( + new CompositeEnvWrapper(env_, FileSystem::Default())); + Options options = CurrentOptions(); + options.env = env.get(); + CreateAndReopenWithCF({"pikachu"}, options); + ReadOptions read_options; + read_options.tailing = true; + if (GetParam()) { + read_options.async_io = true; + } + { + std::unique_ptr iter(db_->NewIterator(read_options, handles_[1])); + ASSERT_OK(iter->status()); + + // write a single record, read it using the iterator, then delete it + ASSERT_OK(Put(1, "0test", "test")); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().ToString(), "0test"); + ASSERT_OK(Delete(1, "0test")); + + // write many more records + const int num_records = 10000; + std::string value(1024, 'A'); + + for (int i = 0; i < num_records; ++i) { + char buf[32]; + snprintf(buf, sizeof(buf), "1%015d", i); + + Slice key(buf, 16); + ASSERT_OK(Put(1, key, value)); + } + + // force a flush to make sure that no records are read from memtable + ASSERT_OK(Flush(1)); + + // skip "0test" + iter->Next(); + + // make sure we can read all new records using the existing iterator + int count = 0; + for (; iter->Valid(); iter->Next(), ++count) + ; + + ASSERT_EQ(count, num_records); + } + Close(); +} + +TEST_P(DBTestTailingIterator, TailingIteratorPrefixSeek) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_BYPASS("Test requires non-mem or non-encrypted environment"); + return; + } + ReadOptions read_options; + read_options.tailing = true; + if (GetParam()) { + read_options.async_io = true; + } + std::unique_ptr env( + new CompositeEnvWrapper(env_, FileSystem::Default())); + Options options = CurrentOptions(); + options.env = env.get(); + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.prefix_extractor.reset(NewFixedPrefixTransform(2)); + options.memtable_factory.reset(NewHashSkipListRepFactory(16)); + options.allow_concurrent_memtable_write = false; + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + { + std::unique_ptr iter(db_->NewIterator(read_options, handles_[1])); + ASSERT_OK(iter->status()); + ASSERT_OK(Put(1, "0101", "test")); + + ASSERT_OK(Flush(1)); + + ASSERT_OK(Put(1, "0202", "test")); + + // Seek(0102) shouldn't find any records since 0202 has a different prefix + iter->Seek("0102"); + ASSERT_TRUE(!iter->Valid()); + + iter->Seek("0202"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().ToString(), "0202"); + + iter->Next(); + ASSERT_TRUE(!iter->Valid()); + } + Close(); +} + +TEST_P(DBTestTailingIterator, TailingIteratorIncomplete) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_BYPASS("Test requires non-mem or non-encrypted environment"); + return; + } + std::unique_ptr env( + new CompositeEnvWrapper(env_, FileSystem::Default())); + Options options = CurrentOptions(); + options.env = env.get(); + CreateAndReopenWithCF({"pikachu"}, options); + ReadOptions read_options; + read_options.tailing = true; + if (GetParam()) { + read_options.async_io = true; + } + read_options.read_tier = kBlockCacheTier; + + std::string key("key"); + std::string value("value"); + + ASSERT_OK(db_->Put(WriteOptions(), key, value)); + + { + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_OK(iter->status()); + iter->SeekToFirst(); + // we either see the entry or it's not in cache + ASSERT_TRUE(iter->Valid() || iter->status().IsIncomplete()); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + iter->SeekToFirst(); + // should still be true after compaction + ASSERT_TRUE(iter->Valid() || iter->status().IsIncomplete()); + } + Close(); +} + +TEST_P(DBTestTailingIterator, TailingIteratorSeekToSame) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_BYPASS("Test requires non-mem or non-encrypted environment"); + return; + } + std::unique_ptr env( + new CompositeEnvWrapper(env_, FileSystem::Default())); + Options options = CurrentOptions(); + options.env = env.get(); + options.compaction_style = kCompactionStyleUniversal; + options.write_buffer_size = 1000; + CreateAndReopenWithCF({"pikachu"}, options); + + ReadOptions read_options; + read_options.tailing = true; + if (GetParam()) { + read_options.async_io = true; + } + const int NROWS = 10000; + // Write rows with keys 00000, 00002, 00004 etc. + for (int i = 0; i < NROWS; ++i) { + char buf[100]; + snprintf(buf, sizeof(buf), "%05d", 2 * i); + std::string key(buf); + std::string value("value"); + ASSERT_OK(db_->Put(WriteOptions(), key, value)); + } + + { + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_OK(iter->status()); + // Seek to 00001. We expect to find 00002. + std::string start_key = "00001"; + iter->Seek(start_key); + ASSERT_TRUE(iter->Valid()); + + std::string found = iter->key().ToString(); + ASSERT_EQ("00002", found); + + // Now seek to the same key. The iterator should remain in the same + // position. + iter->Seek(found); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(found, iter->key().ToString()); + } + Close(); +} + +// Sets iterate_upper_bound and verifies that ForwardIterator doesn't call +// Seek() on immutable iterators when target key is >= prev_key and all +// iterators, including the memtable iterator, are over the upper bound. +TEST_P(DBTestTailingIterator, TailingIteratorUpperBound) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_BYPASS("Test requires non-mem or non-encrypted environment"); + return; + } + std::unique_ptr env( + new CompositeEnvWrapper(env_, FileSystem::Default())); + Options options = CurrentOptions(); + options.env = env.get(); + CreateAndReopenWithCF({"pikachu"}, options); + + const Slice upper_bound("20", 3); + ReadOptions read_options; + read_options.tailing = true; + read_options.iterate_upper_bound = &upper_bound; + if (GetParam()) { + read_options.async_io = true; + } + ASSERT_OK(Put(1, "11", "11")); + ASSERT_OK(Put(1, "12", "12")); + ASSERT_OK(Put(1, "22", "22")); + ASSERT_OK(Flush(1)); // flush all those keys to an immutable SST file + + // Add another key to the memtable. + ASSERT_OK(Put(1, "21", "21")); + + { + bool read_async_called = false; + + SyncPoint::GetInstance()->SetCallBack( + "UpdateResults::io_uring_result", + [&](void* /*arg*/) { read_async_called = true; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + auto it = + std::unique_ptr(db_->NewIterator(read_options, handles_[1])); + ASSERT_OK(it->status()); + it->Seek("12"); + ASSERT_TRUE(it->Valid()); + ASSERT_EQ("12", it->key().ToString()); + + it->Next(); + // Not valid since "21" is over the upper bound. + ASSERT_FALSE(it->Valid()); + ASSERT_OK(it->status()); + // This keeps track of the number of times NeedToSeekImmutable() was true. + int immutable_seeks = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "ForwardIterator::SeekInternal:Immutable", + [&](void* /*arg*/) { ++immutable_seeks; }); + + // Seek to 13. This should not require any immutable seeks. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + it->Seek("13"); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + + SyncPoint::GetInstance()->SetCallBack( + "UpdateResults::io_uring_result", + [&](void* /*arg*/) { read_async_called = true; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_FALSE(it->Valid()); + ASSERT_OK(it->status()); + if (GetParam() && read_async_called) { + ASSERT_EQ(1, immutable_seeks); + } else { + ASSERT_EQ(0, immutable_seeks); + } + } + Close(); +} + +TEST_P(DBTestTailingIterator, TailingIteratorGap) { + // level 1: [20, 25] [35, 40] + // level 2: [10 - 15] [45 - 50] + // level 3: [20, 30, 40] + // Previously there is a bug in tailing_iterator that if there is a gap in + // lower level, the key will be skipped if it is within the range between + // the largest key of index n file and the smallest key of index n+1 file + // if both file fit in that gap. In this example, 25 < key < 35 + // https://github.com/facebook/rocksdb/issues/1372 + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_BYPASS("Test requires non-mem or non-encrypted environment"); + return; + } + std::unique_ptr env( + new CompositeEnvWrapper(env_, FileSystem::Default())); + Options options = CurrentOptions(); + options.env = env.get(); + CreateAndReopenWithCF({"pikachu"}, options); + + ReadOptions read_options; + read_options.tailing = true; + if (GetParam()) { + read_options.async_io = true; + } + ASSERT_OK(Put(1, "20", "20")); + ASSERT_OK(Put(1, "30", "30")); + ASSERT_OK(Put(1, "40", "40")); + ASSERT_OK(Flush(1)); + MoveFilesToLevel(3, 1); + + ASSERT_OK(Put(1, "10", "10")); + ASSERT_OK(Put(1, "15", "15")); + ASSERT_OK(Flush(1)); + ASSERT_OK(Put(1, "45", "45")); + ASSERT_OK(Put(1, "50", "50")); + ASSERT_OK(Flush(1)); + MoveFilesToLevel(2, 1); + + ASSERT_OK(Put(1, "20", "20")); + ASSERT_OK(Put(1, "25", "25")); + ASSERT_OK(Flush(1)); + ASSERT_OK(Put(1, "35", "35")); + ASSERT_OK(Put(1, "40", "40")); + ASSERT_OK(Flush(1)); + MoveFilesToLevel(1, 1); + + ColumnFamilyMetaData meta; + db_->GetColumnFamilyMetaData(handles_[1], &meta); + + { + std::unique_ptr it(db_->NewIterator(read_options, handles_[1])); + it->Seek("30"); + ASSERT_TRUE(it->Valid()); + ASSERT_EQ("30", it->key().ToString()); + + it->Next(); + ASSERT_TRUE(it->Valid()); + ASSERT_EQ("35", it->key().ToString()); + + it->Next(); + ASSERT_TRUE(it->Valid()); + ASSERT_EQ("40", it->key().ToString()); + + ASSERT_OK(it->status()); + } + Close(); +} + +TEST_P(DBTestTailingIterator, SeekWithUpperBoundBug) { + ReadOptions read_options; + read_options.tailing = true; + if (GetParam()) { + read_options.async_io = true; + } + const Slice upper_bound("cc", 3); + read_options.iterate_upper_bound = &upper_bound; + + // 1st L0 file + ASSERT_OK(db_->Put(WriteOptions(), "aa", "SEEN")); + ASSERT_OK(Flush()); + + // 2nd L0 file + ASSERT_OK(db_->Put(WriteOptions(), "zz", "NOT-SEEN")); + ASSERT_OK(Flush()); + + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_OK(iter->status()); + + iter->Seek("aa"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().ToString(), "aa"); +} + +TEST_P(DBTestTailingIterator, SeekToFirstWithUpperBoundBug) { + ReadOptions read_options; + read_options.tailing = true; + if (GetParam()) { + read_options.async_io = true; + } + const Slice upper_bound("cc", 3); + read_options.iterate_upper_bound = &upper_bound; + + // 1st L0 file + ASSERT_OK(db_->Put(WriteOptions(), "aa", "SEEN")); + ASSERT_OK(Flush()); + + // 2nd L0 file + ASSERT_OK(db_->Put(WriteOptions(), "zz", "NOT-SEEN")); + ASSERT_OK(Flush()); + + std::unique_ptr iter(db_->NewIterator(read_options)); + ASSERT_OK(iter->status()); + + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().ToString(), "aa"); + + iter->Next(); + ASSERT_FALSE(iter->Valid()); + + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().ToString(), "aa"); +} + +} // namespace ROCKSDB_NAMESPACE + + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_test.cc b/librocksdb-sys/rocksdb/db/db_test.cc new file mode 100644 index 0000000..609f96e --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_test.cc @@ -0,0 +1,7365 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +// Introduction of SyncPoint effectively disabled building and running this test +// in Release build. +// which is a pity, it is a good test +#include + +#include +#include +#include +#include +#include + +#ifndef OS_WIN +#include +#endif +#ifdef OS_SOLARIS +#include +#endif + +#include "cache/lru_cache.h" +#include "db/blob/blob_index.h" +#include "db/blob/blob_log_format.h" +#include "db/db_impl/db_impl.h" +#include "db/db_test_util.h" +#include "db/dbformat.h" +#include "db/job_context.h" +#include "db/version_set.h" +#include "db/write_batch_internal.h" +#include "env/mock_env.h" +#include "file/filename.h" +#include "monitoring/thread_status_util.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/cache.h" +#include "rocksdb/compaction_filter.h" +#include "rocksdb/convenience.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/experimental.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/options.h" +#include "rocksdb/perf_context.h" +#include "rocksdb/slice.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/snapshot.h" +#include "rocksdb/table.h" +#include "rocksdb/table_properties.h" +#include "rocksdb/thread_status.h" +#include "rocksdb/types.h" +#include "rocksdb/utilities/checkpoint.h" +#include "rocksdb/utilities/optimistic_transaction_db.h" +#include "rocksdb/utilities/write_batch_with_index.h" +#include "table/mock_table.h" +#include "table/scoped_arena_iterator.h" +#include "test_util/sync_point.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/compression.h" +#include "util/mutexlock.h" +#include "util/random.h" +#include "util/rate_limiter_impl.h" +#include "util/string_util.h" +#include "utilities/merge_operators.h" + +namespace ROCKSDB_NAMESPACE { + +// Note that whole DBTest and its child classes disable fsync on files +// and directories for speed. +// If fsync needs to be covered in a test, put it in other places. +class DBTest : public DBTestBase { + public: + DBTest() : DBTestBase("db_test", /*env_do_fsync=*/false) {} +}; + +class DBTestWithParam + : public DBTest, + public testing::WithParamInterface> { + public: + DBTestWithParam() { + max_subcompactions_ = std::get<0>(GetParam()); + exclusive_manual_compaction_ = std::get<1>(GetParam()); + } + + // Required if inheriting from testing::WithParamInterface<> + static void SetUpTestCase() {} + static void TearDownTestCase() {} + + uint32_t max_subcompactions_; + bool exclusive_manual_compaction_; +}; + +TEST_F(DBTest, MockEnvTest) { + std::unique_ptr env{MockEnv::Create(Env::Default())}; + Options options; + options.create_if_missing = true; + options.env = env.get(); + DB* db; + + const Slice keys[] = {Slice("aaa"), Slice("bbb"), Slice("ccc")}; + const Slice vals[] = {Slice("foo"), Slice("bar"), Slice("baz")}; + + ASSERT_OK(DB::Open(options, "/dir/db", &db)); + for (size_t i = 0; i < 3; ++i) { + ASSERT_OK(db->Put(WriteOptions(), keys[i], vals[i])); + } + + for (size_t i = 0; i < 3; ++i) { + std::string res; + ASSERT_OK(db->Get(ReadOptions(), keys[i], &res)); + ASSERT_TRUE(res == vals[i]); + } + + Iterator* iterator = db->NewIterator(ReadOptions()); + iterator->SeekToFirst(); + for (size_t i = 0; i < 3; ++i) { + ASSERT_TRUE(iterator->Valid()); + ASSERT_TRUE(keys[i] == iterator->key()); + ASSERT_TRUE(vals[i] == iterator->value()); + iterator->Next(); + } + ASSERT_TRUE(!iterator->Valid()); + delete iterator; + + DBImpl* dbi = static_cast_with_check(db); + ASSERT_OK(dbi->TEST_FlushMemTable()); + + for (size_t i = 0; i < 3; ++i) { + std::string res; + ASSERT_OK(db->Get(ReadOptions(), keys[i], &res)); + ASSERT_TRUE(res == vals[i]); + } + + delete db; +} + +TEST_F(DBTest, MemEnvTest) { + std::unique_ptr env{NewMemEnv(Env::Default())}; + Options options; + options.create_if_missing = true; + options.env = env.get(); + DB* db; + + const Slice keys[] = {Slice("aaa"), Slice("bbb"), Slice("ccc")}; + const Slice vals[] = {Slice("foo"), Slice("bar"), Slice("baz")}; + + ASSERT_OK(DB::Open(options, "/dir/db", &db)); + for (size_t i = 0; i < 3; ++i) { + ASSERT_OK(db->Put(WriteOptions(), keys[i], vals[i])); + } + + for (size_t i = 0; i < 3; ++i) { + std::string res; + ASSERT_OK(db->Get(ReadOptions(), keys[i], &res)); + ASSERT_TRUE(res == vals[i]); + } + + Iterator* iterator = db->NewIterator(ReadOptions()); + iterator->SeekToFirst(); + for (size_t i = 0; i < 3; ++i) { + ASSERT_TRUE(iterator->Valid()); + ASSERT_TRUE(keys[i] == iterator->key()); + ASSERT_TRUE(vals[i] == iterator->value()); + iterator->Next(); + } + ASSERT_TRUE(!iterator->Valid()); + delete iterator; + + DBImpl* dbi = static_cast_with_check(db); + ASSERT_OK(dbi->TEST_FlushMemTable()); + + for (size_t i = 0; i < 3; ++i) { + std::string res; + ASSERT_OK(db->Get(ReadOptions(), keys[i], &res)); + ASSERT_TRUE(res == vals[i]); + } + + delete db; + + options.create_if_missing = false; + ASSERT_OK(DB::Open(options, "/dir/db", &db)); + for (size_t i = 0; i < 3; ++i) { + std::string res; + ASSERT_OK(db->Get(ReadOptions(), keys[i], &res)); + ASSERT_TRUE(res == vals[i]); + } + delete db; +} + +TEST_F(DBTest, WriteEmptyBatch) { + Options options = CurrentOptions(); + options.env = env_; + options.write_buffer_size = 100000; + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(1, "foo", "bar")); + WriteOptions wo; + wo.sync = true; + wo.disableWAL = false; + WriteBatch empty_batch; + ASSERT_OK(dbfull()->Write(wo, &empty_batch)); + + // make sure we can re-open it. + ASSERT_OK(TryReopenWithColumnFamilies({"default", "pikachu"}, options)); + ASSERT_EQ("bar", Get(1, "foo")); +} + +TEST_F(DBTest, SkipDelay) { + Options options = CurrentOptions(); + options.env = env_; + options.write_buffer_size = 100000; + CreateAndReopenWithCF({"pikachu"}, options); + + for (bool sync : {true, false}) { + for (bool disableWAL : {true, false}) { + if (sync && disableWAL) { + // sync and disableWAL is incompatible. + continue; + } + // Use a small number to ensure a large delay that is still effective + // when we do Put + // TODO(myabandeh): this is time dependent and could potentially make + // the test flaky + auto token = dbfull()->TEST_write_controler().GetDelayToken(1); + std::atomic sleep_count(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::DelayWrite:Sleep", + [&](void* /*arg*/) { sleep_count.fetch_add(1); }); + std::atomic wait_count(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::DelayWrite:Wait", + [&](void* /*arg*/) { wait_count.fetch_add(1); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + WriteOptions wo; + wo.sync = sync; + wo.disableWAL = disableWAL; + wo.no_slowdown = true; + // Large enough to exceed allowance for one time interval + std::string large_value(1024, 'x'); + // Perhaps ideally this first write would fail because of delay, but + // the current implementation does not guarantee that. + dbfull()->Put(wo, "foo", large_value).PermitUncheckedError(); + // We need the 2nd write to trigger delay. This is because delay is + // estimated based on the last write size which is 0 for the first write. + ASSERT_NOK(dbfull()->Put(wo, "foo2", large_value)); + ASSERT_GE(sleep_count.load(), 0); + ASSERT_GE(wait_count.load(), 0); + token.reset(); + + token = dbfull()->TEST_write_controler().GetDelayToken(1000000); + wo.no_slowdown = false; + ASSERT_OK(dbfull()->Put(wo, "foo3", large_value)); + ASSERT_GE(sleep_count.load(), 1); + token.reset(); + } + } +} + +TEST_F(DBTest, MixedSlowdownOptions) { + Options options = CurrentOptions(); + options.env = env_; + options.write_buffer_size = 100000; + CreateAndReopenWithCF({"pikachu"}, options); + std::vector threads; + std::atomic thread_num(0); + + std::function write_slowdown_func = [&]() { + int a = thread_num.fetch_add(1); + std::string key = "foo" + std::to_string(a); + WriteOptions wo; + wo.no_slowdown = false; + ASSERT_OK(dbfull()->Put(wo, key, "bar")); + }; + std::function write_no_slowdown_func = [&]() { + int a = thread_num.fetch_add(1); + std::string key = "foo" + std::to_string(a); + WriteOptions wo; + wo.no_slowdown = true; + ASSERT_NOK(dbfull()->Put(wo, key, "bar")); + }; + // Use a small number to ensure a large delay that is still effective + // when we do Put + // TODO(myabandeh): this is time dependent and could potentially make + // the test flaky + auto token = dbfull()->TEST_write_controler().GetDelayToken(1); + std::atomic sleep_count(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::DelayWrite:BeginWriteStallDone", [&](void* /*arg*/) { + sleep_count.fetch_add(1); + if (threads.empty()) { + for (int i = 0; i < 2; ++i) { + threads.emplace_back(write_slowdown_func); + } + for (int i = 0; i < 2; ++i) { + threads.emplace_back(write_no_slowdown_func); + } + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + WriteOptions wo; + wo.sync = false; + wo.disableWAL = false; + wo.no_slowdown = false; + ASSERT_OK(dbfull()->Put(wo, "foo", "bar")); + // We need the 2nd write to trigger delay. This is because delay is + // estimated based on the last write size which is 0 for the first write. + ASSERT_OK(dbfull()->Put(wo, "foo2", "bar2")); + token.reset(); + + for (auto& t : threads) { + t.join(); + } + ASSERT_GE(sleep_count.load(), 1); + + wo.no_slowdown = true; + ASSERT_OK(dbfull()->Put(wo, "foo3", "bar")); +} + +TEST_F(DBTest, MixedSlowdownOptionsInQueue) { + Options options = CurrentOptions(); + options.env = env_; + options.write_buffer_size = 100000; + CreateAndReopenWithCF({"pikachu"}, options); + std::vector threads; + std::atomic thread_num(0); + + std::function write_no_slowdown_func = [&]() { + int a = thread_num.fetch_add(1); + std::string key = "foo" + std::to_string(a); + WriteOptions wo; + wo.no_slowdown = true; + ASSERT_NOK(dbfull()->Put(wo, key, "bar")); + }; + // Use a small number to ensure a large delay that is still effective + // when we do Put + // TODO(myabandeh): this is time dependent and could potentially make + // the test flaky + auto token = dbfull()->TEST_write_controler().GetDelayToken(1); + std::atomic sleep_count(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::DelayWrite:Sleep", [&](void* /*arg*/) { + sleep_count.fetch_add(1); + if (threads.empty()) { + for (int i = 0; i < 2; ++i) { + threads.emplace_back(write_no_slowdown_func); + } + // Sleep for 3s to allow the threads to insert themselves into the + // write queue + env_->SleepForMicroseconds(3000000ULL); + } + }); + std::atomic wait_count(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::DelayWrite:Wait", + [&](void* /*arg*/) { wait_count.fetch_add(1); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + WriteOptions wo; + wo.sync = false; + wo.disableWAL = false; + wo.no_slowdown = false; + ASSERT_OK(dbfull()->Put(wo, "foo", "bar")); + // We need the 2nd write to trigger delay. This is because delay is + // estimated based on the last write size which is 0 for the first write. + ASSERT_OK(dbfull()->Put(wo, "foo2", "bar2")); + token.reset(); + + for (auto& t : threads) { + t.join(); + } + ASSERT_EQ(sleep_count.load(), 1); + ASSERT_GE(wait_count.load(), 0); +} + +TEST_F(DBTest, MixedSlowdownOptionsStop) { + Options options = CurrentOptions(); + options.env = env_; + options.write_buffer_size = 100000; + CreateAndReopenWithCF({"pikachu"}, options); + std::vector threads; + std::atomic thread_num(0); + + std::function write_slowdown_func = [&]() { + int a = thread_num.fetch_add(1); + std::string key = "foo" + std::to_string(a); + WriteOptions wo; + wo.no_slowdown = false; + ASSERT_OK(dbfull()->Put(wo, key, "bar")); + }; + std::function write_no_slowdown_func = [&]() { + int a = thread_num.fetch_add(1); + std::string key = "foo" + std::to_string(a); + WriteOptions wo; + wo.no_slowdown = true; + ASSERT_NOK(dbfull()->Put(wo, key, "bar")); + }; + std::function wakeup_writer = [&]() { + dbfull()->mutex_.Lock(); + dbfull()->bg_cv_.SignalAll(); + dbfull()->mutex_.Unlock(); + }; + // Use a small number to ensure a large delay that is still effective + // when we do Put + // TODO(myabandeh): this is time dependent and could potentially make + // the test flaky + auto token = dbfull()->TEST_write_controler().GetStopToken(); + std::atomic wait_count(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::DelayWrite:Wait", [&](void* /*arg*/) { + wait_count.fetch_add(1); + if (threads.empty()) { + for (int i = 0; i < 2; ++i) { + threads.emplace_back(write_slowdown_func); + } + for (int i = 0; i < 2; ++i) { + threads.emplace_back(write_no_slowdown_func); + } + // Sleep for 3s to allow the threads to insert themselves into the + // write queue + env_->SleepForMicroseconds(3000000ULL); + } + token.reset(); + threads.emplace_back(wakeup_writer); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + WriteOptions wo; + wo.sync = false; + wo.disableWAL = false; + wo.no_slowdown = false; + ASSERT_OK(dbfull()->Put(wo, "foo", "bar")); + // We need the 2nd write to trigger delay. This is because delay is + // estimated based on the last write size which is 0 for the first write. + ASSERT_OK(dbfull()->Put(wo, "foo2", "bar2")); + token.reset(); + + for (auto& t : threads) { + t.join(); + } + ASSERT_GE(wait_count.load(), 1); + + wo.no_slowdown = true; + ASSERT_OK(dbfull()->Put(wo, "foo3", "bar")); +} + +TEST_F(DBTest, LevelLimitReopen) { + Options options = CurrentOptions(); + CreateAndReopenWithCF({"pikachu"}, options); + + const std::string value(1024 * 1024, ' '); + int i = 0; + while (NumTableFilesAtLevel(2, 1) == 0) { + ASSERT_OK(Put(1, Key(i++), value)); + } + + options.num_levels = 1; + options.max_bytes_for_level_multiplier_additional.resize(1, 1); + Status s = TryReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_EQ(s.IsInvalidArgument(), true); + ASSERT_EQ(s.ToString(), + "Invalid argument: db has more levels than options.num_levels"); + + options.num_levels = 10; + options.max_bytes_for_level_multiplier_additional.resize(10, 1); + ASSERT_OK(TryReopenWithColumnFamilies({"default", "pikachu"}, options)); +} + +TEST_F(DBTest, LevelReopenWithFIFO) { + const int kLevelCount = 4; + const int kKeyCount = 5; + const int kTotalSstFileCount = kLevelCount * kKeyCount; + const int kCF = 1; + + Options options = CurrentOptions(); + // Config level0_file_num_compaction_trigger to prevent L0 files being + // automatically compacted while we are constructing a LSM tree structure + // to test multi-level FIFO compaction. + options.level0_file_num_compaction_trigger = kKeyCount + 1; + CreateAndReopenWithCF({"pikachu"}, options); + + // The expected number of files per level after each file creation. + const std::string expected_files_per_level[kLevelCount][kKeyCount] = { + {"0,0,0,1", "0,0,0,2", "0,0,0,3", "0,0,0,4", "0,0,0,5"}, + {"0,0,1,5", "0,0,2,5", "0,0,3,5", "0,0,4,5", "0,0,5,5"}, + {"0,1,5,5", "0,2,5,5", "0,3,5,5", "0,4,5,5", "0,5,5,5"}, + {"1,5,5,5", "2,5,5,5", "3,5,5,5", "4,5,5,5", "5,5,5,5"}, + }; + + const std::string expected_entries[kKeyCount][kLevelCount + 1] = { + {"[ ]", "[ a3 ]", "[ a2, a3 ]", "[ a1, a2, a3 ]", "[ a0, a1, a2, a3 ]"}, + {"[ ]", "[ b3 ]", "[ b2, b3 ]", "[ b1, b2, b3 ]", "[ b0, b1, b2, b3 ]"}, + {"[ ]", "[ c3 ]", "[ c2, c3 ]", "[ c1, c2, c3 ]", "[ c0, c1, c2, c3 ]"}, + {"[ ]", "[ d3 ]", "[ d2, d3 ]", "[ d1, d2, d3 ]", "[ d0, d1, d2, d3 ]"}, + {"[ ]", "[ e3 ]", "[ e2, e3 ]", "[ e1, e2, e3 ]", "[ e0, e1, e2, e3 ]"}, + }; + + // The loop below creates the following LSM tree where each (k, v) pair + // represents a file that contains that entry. When a file is created, + // the db is reopend with FIFO compaction and verified the LSM tree + // structure is still the same. + // + // The resulting LSM tree will contain 5 different keys. Each key as + // 4 different versions, located in different level. + // + // L0: (e, e0) (d, d0) (c, c0) (b, b0) (a, a0) + // L1: (a, a1) (b, b1) (c, c1) (d, d1) (e, e1) + // L2: (a, a2) (b, b2) (c, c2) (d, d2) (e, e2) + // L3: (a, a3) (b, b3) (c, c3) (d, d3) (e, e3) + for (int l = 0; l < kLevelCount; ++l) { + int level = kLevelCount - 1 - l; + for (int p = 0; p < kKeyCount; ++p) { + std::string put_key = std::string(1, char('a' + p)); + ASSERT_OK(Put(kCF, put_key, put_key + std::to_string(level))); + ASSERT_OK(Flush(kCF)); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + for (int g = 0; g < kKeyCount; ++g) { + int entry_count = (p >= g) ? l + 1 : l; + std::string get_key = std::string(1, char('a' + g)); + CheckAllEntriesWithFifoReopen(expected_entries[g][entry_count], get_key, + kCF, {"pikachu"}, options); + } + if (level != 0) { + MoveFilesToLevel(level, kCF); + for (int g = 0; g < kKeyCount; ++g) { + int entry_count = (p >= g) ? l + 1 : l; + std::string get_key = std::string(1, char('a' + g)); + CheckAllEntriesWithFifoReopen(expected_entries[g][entry_count], + get_key, kCF, {"pikachu"}, options); + } + } + ASSERT_EQ(expected_files_per_level[l][p], FilesPerLevel(kCF)); + } + } + + // The expected number of sst files in each level after each FIFO compaction + // that deletes the oldest sst file. + const std::string expected_files_per_level_after_fifo[] = { + "5,5,5,4", "5,5,5,3", "5,5,5,2", "5,5,5,1", "5,5,5", "5,5,4", "5,5,3", + "5,5,2", "5,5,1", "5,5", "5,4", "5,3", "5,2", "5,1", + "5", "4", "3", "2", "1", "", + }; + + // The expected value entries of each key after each FIFO compaction. + // This verifies whether FIFO removes the file with the smallest key in non-L0 + // files first then the oldest files in L0. + const std::string expected_entries_after_fifo[kKeyCount][kLevelCount + 1] = { + {"[ a0, a1, a2, a3 ]", "[ a0, a1, a2 ]", "[ a0, a1 ]", "[ a0 ]", "[ ]"}, + {"[ b0, b1, b2, b3 ]", "[ b0, b1, b2 ]", "[ b0, b1 ]", "[ b0 ]", "[ ]"}, + {"[ c0, c1, c2, c3 ]", "[ c0, c1, c2 ]", "[ c0, c1 ]", "[ c0 ]", "[ ]"}, + {"[ d0, d1, d2, d3 ]", "[ d0, d1, d2 ]", "[ d0, d1 ]", "[ d0 ]", "[ ]"}, + {"[ e0, e1, e2, e3 ]", "[ e0, e1, e2 ]", "[ e0, e1 ]", "[ e0 ]", "[ ]"}, + }; + + // In the 2nd phase, we reopen the DB with FIFO compaction. In each reopen, + // we config max_table_files_size so that FIFO will remove exactly one file + // at a time upon compaction, and we will use it to verify whether the sst + // files are deleted in the correct order. + for (int i = 0; i < kTotalSstFileCount; ++i) { + uint64_t total_sst_files_size = 0; + ASSERT_TRUE(dbfull()->GetIntProperty( + handles_[1], "rocksdb.total-sst-files-size", &total_sst_files_size)); + ASSERT_TRUE(total_sst_files_size > 0); + + Options fifo_options(options); + fifo_options.compaction_style = kCompactionStyleFIFO; + options.create_if_missing = false; + fifo_options.max_open_files = -1; + fifo_options.disable_auto_compactions = false; + // Config max_table_files_size to be total_sst_files_size - 1 so that + // FIFO will delete one file. + fifo_options.compaction_options_fifo.max_table_files_size = + total_sst_files_size - 1; + ASSERT_OK( + TryReopenWithColumnFamilies({"default", "pikachu"}, fifo_options)); + // For FIFO to pick a compaction + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1])); + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + for (int g = 0; g < kKeyCount; ++g) { + std::string get_key = std::string(1, char('a' + g)); + int status_index = i / kKeyCount; + if ((i % kKeyCount) >= g) { + // If true, then it means the sst file containing the get_key in the + // current level has already been deleted, so we need to move the + // status_index for checking the expected value. + status_index++; + } + CheckAllEntriesWithFifoReopen( + expected_entries_after_fifo[g][status_index], get_key, kCF, + {"pikachu"}, options); + } + ASSERT_EQ(expected_files_per_level_after_fifo[i], FilesPerLevel(kCF)); + } +} + +TEST_F(DBTest, PutSingleDeleteGet) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "foo", "v1")); + ASSERT_EQ("v1", Get(1, "foo")); + ASSERT_OK(Put(1, "foo2", "v2")); + ASSERT_EQ("v2", Get(1, "foo2")); + ASSERT_OK(SingleDelete(1, "foo")); + ASSERT_EQ("NOT_FOUND", Get(1, "foo")); + // Skip FIFO and universal compaction because they do not apply to the test + // case. Skip MergePut because single delete does not get removed when it + // encounters a merge. + } while (ChangeOptions(kSkipFIFOCompaction | kSkipUniversalCompaction | + kSkipMergePut)); +} + +TEST_F(DBTest, ReadFromPersistedTier) { + do { + Random rnd(301); + Options options = CurrentOptions(); + for (int disableWAL = 0; disableWAL <= 1; ++disableWAL) { + CreateAndReopenWithCF({"pikachu"}, options); + WriteOptions wopt; + wopt.disableWAL = (disableWAL == 1); + // 1st round: put but not flush + ASSERT_OK(db_->Put(wopt, handles_[1], "foo", "first")); + ASSERT_OK(db_->Put(wopt, handles_[1], "bar", "one")); + ASSERT_EQ("first", Get(1, "foo")); + ASSERT_EQ("one", Get(1, "bar")); + + // Read directly from persited data. + ReadOptions ropt; + ropt.read_tier = kPersistedTier; + std::string value; + if (wopt.disableWAL) { + // as data has not yet being flushed, we expect not found. + ASSERT_TRUE(db_->Get(ropt, handles_[1], "foo", &value).IsNotFound()); + ASSERT_TRUE(db_->Get(ropt, handles_[1], "bar", &value).IsNotFound()); + } else { + ASSERT_OK(db_->Get(ropt, handles_[1], "foo", &value)); + ASSERT_OK(db_->Get(ropt, handles_[1], "bar", &value)); + } + + // Multiget + std::vector multiget_cfs; + multiget_cfs.push_back(handles_[1]); + multiget_cfs.push_back(handles_[1]); + std::vector multiget_keys; + multiget_keys.push_back("foo"); + multiget_keys.push_back("bar"); + std::vector multiget_values; + auto statuses = + db_->MultiGet(ropt, multiget_cfs, multiget_keys, &multiget_values); + if (wopt.disableWAL) { + ASSERT_TRUE(statuses[0].IsNotFound()); + ASSERT_TRUE(statuses[1].IsNotFound()); + } else { + ASSERT_OK(statuses[0]); + ASSERT_OK(statuses[1]); + } + + // 2nd round: flush and put a new value in memtable. + ASSERT_OK(Flush(1)); + ASSERT_OK(db_->Put(wopt, handles_[1], "rocksdb", "hello")); + + // once the data has been flushed, we are able to get the + // data when kPersistedTier is used. + ASSERT_TRUE(db_->Get(ropt, handles_[1], "foo", &value).ok()); + ASSERT_EQ(value, "first"); + ASSERT_TRUE(db_->Get(ropt, handles_[1], "bar", &value).ok()); + ASSERT_EQ(value, "one"); + if (wopt.disableWAL) { + ASSERT_TRUE( + db_->Get(ropt, handles_[1], "rocksdb", &value).IsNotFound()); + } else { + ASSERT_OK(db_->Get(ropt, handles_[1], "rocksdb", &value)); + ASSERT_EQ(value, "hello"); + } + + // Expect same result in multiget + multiget_cfs.push_back(handles_[1]); + multiget_keys.push_back("rocksdb"); + statuses = + db_->MultiGet(ropt, multiget_cfs, multiget_keys, &multiget_values); + ASSERT_TRUE(statuses[0].ok()); + ASSERT_EQ("first", multiget_values[0]); + ASSERT_TRUE(statuses[1].ok()); + ASSERT_EQ("one", multiget_values[1]); + if (wopt.disableWAL) { + ASSERT_TRUE(statuses[2].IsNotFound()); + } else { + ASSERT_OK(statuses[2]); + } + + // 3rd round: delete and flush + ASSERT_OK(db_->Delete(wopt, handles_[1], "foo")); + Flush(1); + ASSERT_OK(db_->Delete(wopt, handles_[1], "bar")); + + ASSERT_TRUE(db_->Get(ropt, handles_[1], "foo", &value).IsNotFound()); + if (wopt.disableWAL) { + // Still expect finding the value as its delete has not yet being + // flushed. + ASSERT_TRUE(db_->Get(ropt, handles_[1], "bar", &value).ok()); + ASSERT_EQ(value, "one"); + } else { + ASSERT_TRUE(db_->Get(ropt, handles_[1], "bar", &value).IsNotFound()); + } + ASSERT_TRUE(db_->Get(ropt, handles_[1], "rocksdb", &value).ok()); + ASSERT_EQ(value, "hello"); + + statuses = + db_->MultiGet(ropt, multiget_cfs, multiget_keys, &multiget_values); + ASSERT_TRUE(statuses[0].IsNotFound()); + if (wopt.disableWAL) { + ASSERT_TRUE(statuses[1].ok()); + ASSERT_EQ("one", multiget_values[1]); + } else { + ASSERT_TRUE(statuses[1].IsNotFound()); + } + ASSERT_TRUE(statuses[2].ok()); + ASSERT_EQ("hello", multiget_values[2]); + if (wopt.disableWAL == 0) { + DestroyAndReopen(options); + } + } + } while (ChangeOptions()); +} + +TEST_F(DBTest, SingleDeleteFlush) { + // Test to check whether flushing preserves a single delete hidden + // behind a put. + do { + Random rnd(301); + + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + CreateAndReopenWithCF({"pikachu"}, options); + + // Put values on second level (so that they will not be in the same + // compaction as the other operations. + ASSERT_OK(Put(1, "foo", "first")); + ASSERT_OK(Put(1, "bar", "one")); + ASSERT_OK(Flush(1)); + MoveFilesToLevel(2, 1); + + // (Single) delete hidden by a put + ASSERT_OK(SingleDelete(1, "foo")); + ASSERT_OK(Put(1, "foo", "second")); + ASSERT_OK(Delete(1, "bar")); + ASSERT_OK(Put(1, "bar", "two")); + ASSERT_OK(Flush(1)); + + ASSERT_OK(SingleDelete(1, "foo")); + ASSERT_OK(Delete(1, "bar")); + ASSERT_OK(Flush(1)); + + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), handles_[1], + nullptr, nullptr)); + + ASSERT_EQ("NOT_FOUND", Get(1, "bar")); + ASSERT_EQ("NOT_FOUND", Get(1, "foo")); + // Skip FIFO and universal compaction beccaus they do not apply to the test + // case. Skip MergePut because single delete does not get removed when it + // encounters a merge. + } while (ChangeOptions(kSkipFIFOCompaction | kSkipUniversalCompaction | + kSkipMergePut)); +} + +TEST_F(DBTest, SingleDeletePutFlush) { + // Single deletes that encounter the matching put in a flush should get + // removed. + do { + Random rnd(301); + + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(1, "foo", Slice())); + ASSERT_OK(Put(1, "a", Slice())); + ASSERT_OK(SingleDelete(1, "a")); + ASSERT_OK(Flush(1)); + + ASSERT_EQ("[ ]", AllEntriesFor("a", 1)); + // Skip FIFO and universal compaction because they do not apply to the test + // case. Skip MergePut because single delete does not get removed when it + // encounters a merge. + } while (ChangeOptions(kSkipFIFOCompaction | kSkipUniversalCompaction | + kSkipMergePut)); +} + +// Disable because not all platform can run it. +// It requires more than 9GB memory to run it, With single allocation +// of more than 3GB. +TEST_F(DBTest, DISABLED_SanitizeVeryVeryLargeValue) { + const size_t kValueSize = 4 * size_t{1024 * 1024 * 1024}; // 4GB value + std::string raw(kValueSize, 'v'); + Options options = CurrentOptions(); + options.env = env_; + options.merge_operator = MergeOperators::CreatePutOperator(); + options.write_buffer_size = 100000; // Small write buffer + options.paranoid_checks = true; + DestroyAndReopen(options); + + ASSERT_OK(Put("boo", "v1")); + ASSERT_TRUE(Put("foo", raw).IsInvalidArgument()); + ASSERT_TRUE(Merge("foo", raw).IsInvalidArgument()); + + WriteBatch wb; + ASSERT_TRUE(wb.Put("foo", raw).IsInvalidArgument()); + ASSERT_TRUE(wb.Merge("foo", raw).IsInvalidArgument()); + + Slice value_slice = raw; + Slice key_slice = "foo"; + SliceParts sp_key(&key_slice, 1); + SliceParts sp_value(&value_slice, 1); + + ASSERT_TRUE(wb.Put(sp_key, sp_value).IsInvalidArgument()); + ASSERT_TRUE(wb.Merge(sp_key, sp_value).IsInvalidArgument()); +} + +// Disable because not all platform can run it. +// It requires more than 9GB memory to run it, With single allocation +// of more than 3GB. +TEST_F(DBTest, DISABLED_VeryLargeValue) { + const size_t kValueSize = 3221225472u; // 3GB value + const size_t kKeySize = 8388608u; // 8MB key + std::string raw(kValueSize, 'v'); + std::string key1(kKeySize, 'c'); + std::string key2(kKeySize, 'd'); + + Options options = CurrentOptions(); + options.env = env_; + options.write_buffer_size = 100000; // Small write buffer + options.paranoid_checks = true; + DestroyAndReopen(options); + + ASSERT_OK(Put("boo", "v1")); + ASSERT_OK(Put("foo", "v1")); + ASSERT_OK(Put(key1, raw)); + raw[0] = 'w'; + ASSERT_OK(Put(key2, raw)); + dbfull()->TEST_WaitForFlushMemTable(); + + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + + std::string value; + Status s = db_->Get(ReadOptions(), key1, &value); + ASSERT_OK(s); + ASSERT_EQ(kValueSize, value.size()); + ASSERT_EQ('v', value[0]); + + s = db_->Get(ReadOptions(), key2, &value); + ASSERT_OK(s); + ASSERT_EQ(kValueSize, value.size()); + ASSERT_EQ('w', value[0]); + + // Compact all files. + Flush(); + db_->CompactRange(CompactRangeOptions(), nullptr, nullptr); + + // Check DB is not in read-only state. + ASSERT_OK(Put("boo", "v1")); + + s = db_->Get(ReadOptions(), key1, &value); + ASSERT_OK(s); + ASSERT_EQ(kValueSize, value.size()); + ASSERT_EQ('v', value[0]); + + s = db_->Get(ReadOptions(), key2, &value); + ASSERT_OK(s); + ASSERT_EQ(kValueSize, value.size()); + ASSERT_EQ('w', value[0]); +} + +TEST_F(DBTest, GetFromImmutableLayer) { + do { + Options options = CurrentOptions(); + options.env = env_; + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(1, "foo", "v1")); + ASSERT_EQ("v1", Get(1, "foo")); + + // Block sync calls + env_->delay_sstable_sync_.store(true, std::memory_order_release); + ASSERT_OK(Put(1, "k1", std::string(100000, 'x'))); // Fill memtable + ASSERT_OK(Put(1, "k2", std::string(100000, 'y'))); // Trigger flush + ASSERT_EQ("v1", Get(1, "foo")); + ASSERT_EQ("NOT_FOUND", Get(0, "foo")); + // Release sync calls + env_->delay_sstable_sync_.store(false, std::memory_order_release); + } while (ChangeOptions()); +} + +TEST_F(DBTest, GetLevel0Ordering) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + // Check that we process level-0 files in correct order. The code + // below generates two level-0 files where the earlier one comes + // before the later one in the level-0 file list since the earlier + // one has a smaller "smallest" key. + ASSERT_OK(Put(1, "bar", "b")); + ASSERT_OK(Put(1, "foo", "v1")); + ASSERT_OK(Flush(1)); + ASSERT_OK(Put(1, "foo", "v2")); + ASSERT_OK(Flush(1)); + ASSERT_EQ("v2", Get(1, "foo")); + } while (ChangeOptions()); +} + +TEST_F(DBTest, WrongLevel0Config) { + Options options = CurrentOptions(); + Close(); + ASSERT_OK(DestroyDB(dbname_, options)); + options.level0_stop_writes_trigger = 1; + options.level0_slowdown_writes_trigger = 2; + options.level0_file_num_compaction_trigger = 3; + ASSERT_OK(DB::Open(options, dbname_, &db_)); +} + +TEST_F(DBTest, GetOrderedByLevels) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "foo", "v1")); + Compact(1, "a", "z"); + ASSERT_EQ("v1", Get(1, "foo")); + ASSERT_OK(Put(1, "foo", "v2")); + ASSERT_EQ("v2", Get(1, "foo")); + ASSERT_OK(Flush(1)); + ASSERT_EQ("v2", Get(1, "foo")); + } while (ChangeOptions()); +} + +TEST_F(DBTest, GetPicksCorrectFile) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + // Arrange to have multiple files in a non-level-0 level. + ASSERT_OK(Put(1, "a", "va")); + Compact(1, "a", "b"); + ASSERT_OK(Put(1, "x", "vx")); + Compact(1, "x", "y"); + ASSERT_OK(Put(1, "f", "vf")); + Compact(1, "f", "g"); + ASSERT_EQ("va", Get(1, "a")); + ASSERT_EQ("vf", Get(1, "f")); + ASSERT_EQ("vx", Get(1, "x")); + } while (ChangeOptions()); +} + +TEST_F(DBTest, GetEncountersEmptyLevel) { + do { + Options options = CurrentOptions(); + CreateAndReopenWithCF({"pikachu"}, options); + // Arrange for the following to happen: + // * sstable A in level 0 + // * nothing in level 1 + // * sstable B in level 2 + // Then do enough Get() calls to arrange for an automatic compaction + // of sstable A. A bug would cause the compaction to be marked as + // occurring at level 1 (instead of the correct level 0). + + // Step 1: First place sstables in levels 0 and 2 + ASSERT_OK(Put(1, "a", "begin")); + ASSERT_OK(Put(1, "z", "end")); + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1])); + ASSERT_OK(dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1])); + ASSERT_OK(Put(1, "a", "begin")); + ASSERT_OK(Put(1, "z", "end")); + ASSERT_OK(Flush(1)); + ASSERT_GT(NumTableFilesAtLevel(0, 1), 0); + ASSERT_GT(NumTableFilesAtLevel(2, 1), 0); + + // Step 2: clear level 1 if necessary. + ASSERT_OK(dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1])); + ASSERT_EQ(NumTableFilesAtLevel(0, 1), 1); + ASSERT_EQ(NumTableFilesAtLevel(1, 1), 0); + ASSERT_EQ(NumTableFilesAtLevel(2, 1), 1); + + // Step 3: read a bunch of times + for (int i = 0; i < 1000; i++) { + ASSERT_EQ("NOT_FOUND", Get(1, "missing")); + } + + // Step 4: Wait for compaction to finish + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ(NumTableFilesAtLevel(0, 1), 1); // XXX + } while (ChangeOptions(kSkipUniversalCompaction | kSkipFIFOCompaction)); +} + +TEST_F(DBTest, FlushMultipleMemtable) { + do { + Options options = CurrentOptions(); + WriteOptions writeOpt = WriteOptions(); + writeOpt.disableWAL = true; + options.max_write_buffer_number = 4; + options.min_write_buffer_number_to_merge = 3; + options.max_write_buffer_size_to_maintain = -1; + CreateAndReopenWithCF({"pikachu"}, options); + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v1")); + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1")); + + ASSERT_EQ("v1", Get(1, "foo")); + ASSERT_EQ("v1", Get(1, "bar")); + ASSERT_OK(Flush(1)); + } while (ChangeCompactOptions()); +} +TEST_F(DBTest, FlushSchedule) { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.level0_stop_writes_trigger = 1 << 10; + options.level0_slowdown_writes_trigger = 1 << 10; + options.min_write_buffer_number_to_merge = 1; + options.max_write_buffer_size_to_maintain = + static_cast(options.write_buffer_size); + options.max_write_buffer_number = 2; + options.write_buffer_size = 120 * 1024; + auto flush_listener = std::make_shared(); + flush_listener->expected_flush_reason = FlushReason::kWriteBufferFull; + options.listeners.push_back(flush_listener); + CreateAndReopenWithCF({"pikachu"}, options); + std::vector threads; + + std::atomic thread_num(0); + // each column family will have 5 thread, each thread generating 2 memtables. + // each column family should end up with 10 table files + std::function fill_memtable_func = [&]() { + int a = thread_num.fetch_add(1); + Random rnd(a); + WriteOptions wo; + // this should fill up 2 memtables + for (int k = 0; k < 5000; ++k) { + ASSERT_OK(db_->Put(wo, handles_[a & 1], rnd.RandomString(13), "")); + } + }; + + for (int i = 0; i < 10; ++i) { + threads.emplace_back(fill_memtable_func); + } + + for (auto& t : threads) { + t.join(); + } + + auto default_tables = GetNumberOfSstFilesForColumnFamily(db_, "default"); + auto pikachu_tables = GetNumberOfSstFilesForColumnFamily(db_, "pikachu"); + ASSERT_LE(default_tables, static_cast(10)); + ASSERT_GT(default_tables, static_cast(0)); + ASSERT_LE(pikachu_tables, static_cast(10)); + ASSERT_GT(pikachu_tables, static_cast(0)); +} + +namespace { +class KeepFilter : public CompactionFilter { + public: + bool Filter(int /*level*/, const Slice& /*key*/, const Slice& /*value*/, + std::string* /*new_value*/, + bool* /*value_changed*/) const override { + return false; + } + + const char* Name() const override { return "KeepFilter"; } +}; + +class KeepFilterFactory : public CompactionFilterFactory { + public: + explicit KeepFilterFactory(bool check_context = false) + : check_context_(check_context) {} + + std::unique_ptr CreateCompactionFilter( + const CompactionFilter::Context& context) override { + if (check_context_) { + EXPECT_EQ(expect_full_compaction_.load(), context.is_full_compaction); + EXPECT_EQ(expect_manual_compaction_.load(), context.is_manual_compaction); + } + return std::unique_ptr(new KeepFilter()); + } + + const char* Name() const override { return "KeepFilterFactory"; } + bool check_context_; + std::atomic_bool expect_full_compaction_; + std::atomic_bool expect_manual_compaction_; +}; + +class DelayFilter : public CompactionFilter { + public: + explicit DelayFilter(DBTestBase* d) : db_test(d) {} + bool Filter(int /*level*/, const Slice& /*key*/, const Slice& /*value*/, + std::string* /*new_value*/, + bool* /*value_changed*/) const override { + db_test->env_->MockSleepForMicroseconds(1000); + return true; + } + + const char* Name() const override { return "DelayFilter"; } + + private: + DBTestBase* db_test; +}; + +class DelayFilterFactory : public CompactionFilterFactory { + public: + explicit DelayFilterFactory(DBTestBase* d) : db_test(d) {} + std::unique_ptr CreateCompactionFilter( + const CompactionFilter::Context& /*context*/) override { + return std::unique_ptr(new DelayFilter(db_test)); + } + + const char* Name() const override { return "DelayFilterFactory"; } + + private: + DBTestBase* db_test; +}; +} // anonymous namespace + + +static std::string CompressibleString(Random* rnd, int len) { + std::string r; + test::CompressibleString(rnd, 0.8, len, &r); + return r; +} + +TEST_F(DBTest, FailMoreDbPaths) { + Options options = CurrentOptions(); + options.db_paths.emplace_back(dbname_, 10000000); + options.db_paths.emplace_back(dbname_ + "_2", 1000000); + options.db_paths.emplace_back(dbname_ + "_3", 1000000); + options.db_paths.emplace_back(dbname_ + "_4", 1000000); + options.db_paths.emplace_back(dbname_ + "_5", 1000000); + ASSERT_TRUE(TryReopen(options).IsNotSupported()); +} + +void CheckColumnFamilyMeta( + const ColumnFamilyMetaData& cf_meta, const std::string& cf_name, + const std::vector>& files_by_level, + uint64_t start_time, uint64_t end_time) { + ASSERT_EQ(cf_meta.name, cf_name); + ASSERT_EQ(cf_meta.levels.size(), files_by_level.size()); + + uint64_t cf_size = 0; + size_t file_count = 0; + + for (size_t i = 0; i < cf_meta.levels.size(); ++i) { + const auto& level_meta_from_cf = cf_meta.levels[i]; + const auto& level_meta_from_files = files_by_level[i]; + + ASSERT_EQ(level_meta_from_cf.level, i); + ASSERT_EQ(level_meta_from_cf.files.size(), level_meta_from_files.size()); + + file_count += level_meta_from_cf.files.size(); + + uint64_t level_size = 0; + for (size_t j = 0; j < level_meta_from_cf.files.size(); ++j) { + const auto& file_meta_from_cf = level_meta_from_cf.files[j]; + const auto& file_meta_from_files = level_meta_from_files[j]; + + level_size += file_meta_from_cf.size; + + ASSERT_EQ(file_meta_from_cf.file_number, + file_meta_from_files.fd.GetNumber()); + ASSERT_EQ(file_meta_from_cf.file_number, + TableFileNameToNumber(file_meta_from_cf.name)); + ASSERT_EQ(file_meta_from_cf.size, file_meta_from_files.fd.file_size); + ASSERT_EQ(file_meta_from_cf.smallest_seqno, + file_meta_from_files.fd.smallest_seqno); + ASSERT_EQ(file_meta_from_cf.largest_seqno, + file_meta_from_files.fd.largest_seqno); + ASSERT_EQ(file_meta_from_cf.smallestkey, + file_meta_from_files.smallest.user_key().ToString()); + ASSERT_EQ(file_meta_from_cf.largestkey, + file_meta_from_files.largest.user_key().ToString()); + ASSERT_EQ(file_meta_from_cf.oldest_blob_file_number, + file_meta_from_files.oldest_blob_file_number); + ASSERT_EQ(file_meta_from_cf.oldest_ancester_time, + file_meta_from_files.oldest_ancester_time); + ASSERT_EQ(file_meta_from_cf.file_creation_time, + file_meta_from_files.file_creation_time); + ASSERT_GE(file_meta_from_cf.file_creation_time, start_time); + ASSERT_LE(file_meta_from_cf.file_creation_time, end_time); + ASSERT_EQ(file_meta_from_cf.epoch_number, + file_meta_from_files.epoch_number); + ASSERT_GE(file_meta_from_cf.oldest_ancester_time, start_time); + ASSERT_LE(file_meta_from_cf.oldest_ancester_time, end_time); + // More from FileStorageInfo + ASSERT_EQ(file_meta_from_cf.file_type, kTableFile); + ASSERT_EQ(file_meta_from_cf.name, + "/" + file_meta_from_cf.relative_filename); + ASSERT_EQ(file_meta_from_cf.directory, file_meta_from_cf.db_path); + } + + ASSERT_EQ(level_meta_from_cf.size, level_size); + cf_size += level_size; + } + + ASSERT_EQ(cf_meta.file_count, file_count); + ASSERT_EQ(cf_meta.size, cf_size); +} + +void CheckLiveFilesMeta( + const std::vector& live_file_meta, + const std::vector>& files_by_level) { + size_t total_file_count = 0; + for (const auto& f : files_by_level) { + total_file_count += f.size(); + } + + ASSERT_EQ(live_file_meta.size(), total_file_count); + + int level = 0; + int i = 0; + + for (const auto& meta : live_file_meta) { + if (level != meta.level) { + level = meta.level; + i = 0; + } + + ASSERT_LT(i, files_by_level[level].size()); + + const auto& expected_meta = files_by_level[level][i]; + + ASSERT_EQ(meta.column_family_name, kDefaultColumnFamilyName); + ASSERT_EQ(meta.file_number, expected_meta.fd.GetNumber()); + ASSERT_EQ(meta.file_number, TableFileNameToNumber(meta.name)); + ASSERT_EQ(meta.size, expected_meta.fd.file_size); + ASSERT_EQ(meta.smallest_seqno, expected_meta.fd.smallest_seqno); + ASSERT_EQ(meta.largest_seqno, expected_meta.fd.largest_seqno); + ASSERT_EQ(meta.smallestkey, expected_meta.smallest.user_key().ToString()); + ASSERT_EQ(meta.largestkey, expected_meta.largest.user_key().ToString()); + ASSERT_EQ(meta.oldest_blob_file_number, + expected_meta.oldest_blob_file_number); + ASSERT_EQ(meta.epoch_number, expected_meta.epoch_number); + + // More from FileStorageInfo + ASSERT_EQ(meta.file_type, kTableFile); + ASSERT_EQ(meta.name, "/" + meta.relative_filename); + ASSERT_EQ(meta.directory, meta.db_path); + + ++i; + } +} + +void AddBlobFile(const ColumnFamilyHandle* cfh, uint64_t blob_file_number, + uint64_t total_blob_count, uint64_t total_blob_bytes, + const std::string& checksum_method, + const std::string& checksum_value, + uint64_t garbage_blob_count = 0, + uint64_t garbage_blob_bytes = 0) { + ColumnFamilyData* cfd = + (static_cast(cfh))->cfd(); + assert(cfd); + + Version* const version = cfd->current(); + assert(version); + + VersionStorageInfo* const storage_info = version->storage_info(); + assert(storage_info); + + // Add a live blob file. + + auto shared_meta = SharedBlobFileMetaData::Create( + blob_file_number, total_blob_count, total_blob_bytes, checksum_method, + checksum_value); + + auto meta = BlobFileMetaData::Create(std::move(shared_meta), + BlobFileMetaData::LinkedSsts(), + garbage_blob_count, garbage_blob_bytes); + + storage_info->AddBlobFile(std::move(meta)); +} + +static void CheckBlobMetaData( + const BlobMetaData& bmd, uint64_t blob_file_number, + uint64_t total_blob_count, uint64_t total_blob_bytes, + const std::string& checksum_method, const std::string& checksum_value, + uint64_t garbage_blob_count = 0, uint64_t garbage_blob_bytes = 0) { + ASSERT_EQ(bmd.blob_file_number, blob_file_number); + ASSERT_EQ(bmd.blob_file_name, BlobFileName("", blob_file_number)); + ASSERT_EQ(bmd.blob_file_size, + total_blob_bytes + BlobLogHeader::kSize + BlobLogFooter::kSize); + + ASSERT_EQ(bmd.total_blob_count, total_blob_count); + ASSERT_EQ(bmd.total_blob_bytes, total_blob_bytes); + ASSERT_EQ(bmd.garbage_blob_count, garbage_blob_count); + ASSERT_EQ(bmd.garbage_blob_bytes, garbage_blob_bytes); + ASSERT_EQ(bmd.checksum_method, checksum_method); + ASSERT_EQ(bmd.checksum_value, checksum_value); +} + +TEST_F(DBTest, MetaDataTest) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.disable_auto_compactions = true; + + int64_t temp_time = 0; + options.env->GetCurrentTime(&temp_time); + uint64_t start_time = static_cast(temp_time); + + DestroyAndReopen(options); + + Random rnd(301); + int key_index = 0; + for (int i = 0; i < 100; ++i) { + // Add a single blob reference to each file + std::string blob_index; + BlobIndex::EncodeBlob(&blob_index, /* blob_file_number */ i + 1000, + /* offset */ 1234, /* size */ 5678, kNoCompression); + + WriteBatch batch; + ASSERT_OK(WriteBatchInternal::PutBlobIndex(&batch, 0, Key(key_index), + blob_index)); + ASSERT_OK(dbfull()->Write(WriteOptions(), &batch)); + + ++key_index; + + // Fill up the rest of the file with random values. + GenerateNewFile(&rnd, &key_index, /* nowait */ true); + + ASSERT_OK(Flush()); + } + + std::vector> files_by_level; + dbfull()->TEST_GetFilesMetaData(db_->DefaultColumnFamily(), &files_by_level); + + options.env->GetCurrentTime(&temp_time); + uint64_t end_time = static_cast(temp_time); + + ColumnFamilyMetaData cf_meta; + db_->GetColumnFamilyMetaData(&cf_meta); + CheckColumnFamilyMeta(cf_meta, kDefaultColumnFamilyName, files_by_level, + start_time, end_time); + std::vector live_file_meta; + db_->GetLiveFilesMetaData(&live_file_meta); + CheckLiveFilesMeta(live_file_meta, files_by_level); +} + +TEST_F(DBTest, AllMetaDataTest) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + constexpr uint64_t blob_file_number = 234; + constexpr uint64_t total_blob_count = 555; + constexpr uint64_t total_blob_bytes = 66666; + constexpr char checksum_method[] = "CRC32"; + constexpr char checksum_value[] = "\x3d\x87\xff\x57"; + + int64_t temp_time = 0; + options.env->GetCurrentTime(&temp_time).PermitUncheckedError(); + uint64_t start_time = static_cast(temp_time); + + Random rnd(301); + dbfull()->TEST_LockMutex(); + for (int cf = 0; cf < 2; cf++) { + AddBlobFile(handles_[cf], blob_file_number * (cf + 1), + total_blob_count * (cf + 1), total_blob_bytes * (cf + 1), + checksum_method, checksum_value); + } + dbfull()->TEST_UnlockMutex(); + + std::vector all_meta; + db_->GetAllColumnFamilyMetaData(&all_meta); + + std::vector> default_files_by_level; + std::vector> pikachu_files_by_level; + dbfull()->TEST_GetFilesMetaData(handles_[0], &default_files_by_level); + dbfull()->TEST_GetFilesMetaData(handles_[1], &pikachu_files_by_level); + + options.env->GetCurrentTime(&temp_time).PermitUncheckedError(); + uint64_t end_time = static_cast(temp_time); + + ASSERT_EQ(all_meta.size(), 2); + for (int cf = 0; cf < 2; cf++) { + const auto& cfmd = all_meta[cf]; + if (cf == 0) { + CheckColumnFamilyMeta(cfmd, "default", default_files_by_level, start_time, + end_time); + } else { + CheckColumnFamilyMeta(cfmd, "pikachu", pikachu_files_by_level, start_time, + end_time); + } + ASSERT_EQ(cfmd.blob_files.size(), 1U); + const auto& bmd = cfmd.blob_files[0]; + ASSERT_EQ(cfmd.blob_file_count, 1U); + ASSERT_EQ(cfmd.blob_file_size, bmd.blob_file_size); + ASSERT_EQ(NormalizePath(bmd.blob_file_path), NormalizePath(dbname_)); + CheckBlobMetaData(bmd, blob_file_number * (cf + 1), + total_blob_count * (cf + 1), total_blob_bytes * (cf + 1), + checksum_method, checksum_value); + } +} + +namespace { +void MinLevelHelper(DBTest* self, Options& options) { + Random rnd(301); + + for (int num = 0; num < options.level0_file_num_compaction_trigger - 1; + num++) { + std::vector values; + // Write 120KB (12 values, each 10K) + for (int i = 0; i < 12; i++) { + values.push_back(rnd.RandomString(10000)); + ASSERT_OK(self->Put(DBTestBase::Key(i), values[i])); + } + ASSERT_OK(self->dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(self->NumTableFilesAtLevel(0), num + 1); + } + + // generate one more file in level-0, and should trigger level-0 compaction + std::vector values; + for (int i = 0; i < 12; i++) { + values.push_back(rnd.RandomString(10000)); + ASSERT_OK(self->Put(DBTestBase::Key(i), values[i])); + } + ASSERT_OK(self->dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ(self->NumTableFilesAtLevel(0), 0); + ASSERT_EQ(self->NumTableFilesAtLevel(1), 1); +} + +// returns false if the calling-Test should be skipped +bool MinLevelToCompress(CompressionType& type, Options& options, int wbits, + int lev, int strategy) { + fprintf(stderr, + "Test with compression options : window_bits = %d, level = %d, " + "strategy = %d}\n", + wbits, lev, strategy); + options.write_buffer_size = 100 << 10; // 100KB + options.arena_block_size = 4096; + options.num_levels = 3; + options.level0_file_num_compaction_trigger = 3; + options.create_if_missing = true; + + if (Snappy_Supported()) { + type = kSnappyCompression; + fprintf(stderr, "using snappy\n"); + } else if (Zlib_Supported()) { + type = kZlibCompression; + fprintf(stderr, "using zlib\n"); + } else if (BZip2_Supported()) { + type = kBZip2Compression; + fprintf(stderr, "using bzip2\n"); + } else if (LZ4_Supported()) { + type = kLZ4Compression; + fprintf(stderr, "using lz4\n"); + } else if (XPRESS_Supported()) { + type = kXpressCompression; + fprintf(stderr, "using xpress\n"); + } else if (ZSTD_Supported()) { + type = kZSTD; + fprintf(stderr, "using ZSTD\n"); + } else { + fprintf(stderr, "skipping test, compression disabled\n"); + return false; + } + options.compression_per_level.resize(options.num_levels); + + // do not compress L0 + for (int i = 0; i < 1; i++) { + options.compression_per_level[i] = kNoCompression; + } + for (int i = 1; i < options.num_levels; i++) { + options.compression_per_level[i] = type; + } + return true; +} +} // anonymous namespace + +TEST_F(DBTest, MinLevelToCompress1) { + Options options = CurrentOptions(); + CompressionType type = kSnappyCompression; + if (!MinLevelToCompress(type, options, -14, -1, 0)) { + return; + } + Reopen(options); + MinLevelHelper(this, options); + + // do not compress L0 and L1 + for (int i = 0; i < 2; i++) { + options.compression_per_level[i] = kNoCompression; + } + for (int i = 2; i < options.num_levels; i++) { + options.compression_per_level[i] = type; + } + DestroyAndReopen(options); + MinLevelHelper(this, options); +} + +TEST_F(DBTest, MinLevelToCompress2) { + Options options = CurrentOptions(); + CompressionType type = kSnappyCompression; + if (!MinLevelToCompress(type, options, 15, -1, 0)) { + return; + } + Reopen(options); + MinLevelHelper(this, options); + + // do not compress L0 and L1 + for (int i = 0; i < 2; i++) { + options.compression_per_level[i] = kNoCompression; + } + for (int i = 2; i < options.num_levels; i++) { + options.compression_per_level[i] = type; + } + DestroyAndReopen(options); + MinLevelHelper(this, options); +} + +// This test may fail because of a legit case that multiple L0 files +// are trivial moved to L1. +TEST_F(DBTest, DISABLED_RepeatedWritesToSameKey) { + do { + Options options = CurrentOptions(); + options.env = env_; + options.write_buffer_size = 100000; // Small write buffer + CreateAndReopenWithCF({"pikachu"}, options); + + // We must have at most one file per level except for level-0, + // which may have up to kL0_StopWritesTrigger files. + const int kMaxFiles = + options.num_levels + options.level0_stop_writes_trigger; + + Random rnd(301); + std::string value = + rnd.RandomString(static_cast(2 * options.write_buffer_size)); + for (int i = 0; i < 5 * kMaxFiles; i++) { + ASSERT_OK(Put(1, "key", value)); + ASSERT_LE(TotalTableFiles(1), kMaxFiles); + } + } while (ChangeCompactOptions()); +} + +static bool Between(uint64_t val, uint64_t low, uint64_t high) { + bool result = (val >= low) && (val <= high); + if (!result) { + fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n", + (unsigned long long)(val), (unsigned long long)(low), + (unsigned long long)(high)); + } + return result; +} + +TEST_F(DBTest, ApproximateSizesMemTable) { + Options options = CurrentOptions(); + options.write_buffer_size = 100000000; // Large write buffer + options.compression = kNoCompression; + options.create_if_missing = true; + DestroyAndReopen(options); + auto default_cf = db_->DefaultColumnFamily(); + + const int N = 128; + Random rnd(301); + for (int i = 0; i < N; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(1024))); + } + + uint64_t size; + std::string start = Key(50); + std::string end = Key(60); + Range r(start, end); + SizeApproximationOptions size_approx_options; + size_approx_options.include_memtables = true; + size_approx_options.include_files = true; + ASSERT_OK( + db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, &size)); + ASSERT_GT(size, 6000); + ASSERT_LT(size, 204800); + // Zero if not including mem table + ASSERT_OK(db_->GetApproximateSizes(&r, 1, &size)); + ASSERT_EQ(size, 0); + + start = Key(500); + end = Key(600); + r = Range(start, end); + ASSERT_OK( + db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, &size)); + ASSERT_EQ(size, 0); + + for (int i = 0; i < N; i++) { + ASSERT_OK(Put(Key(1000 + i), rnd.RandomString(1024))); + } + + start = Key(500); + end = Key(600); + r = Range(start, end); + ASSERT_OK( + db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, &size)); + ASSERT_EQ(size, 0); + + start = Key(100); + end = Key(1020); + r = Range(start, end); + ASSERT_OK( + db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, &size)); + ASSERT_GT(size, 6000); + + options.max_write_buffer_number = 8; + options.min_write_buffer_number_to_merge = 5; + options.write_buffer_size = 1024 * N; // Not very large + DestroyAndReopen(options); + default_cf = db_->DefaultColumnFamily(); + + int keys[N * 3]; + for (int i = 0; i < N; i++) { + keys[i * 3] = i * 5; + keys[i * 3 + 1] = i * 5 + 1; + keys[i * 3 + 2] = i * 5 + 2; + } + // MemTable entry counting is estimated and can vary greatly depending on + // layout. Thus, using deterministic seed for test stability. + RandomShuffle(std::begin(keys), std::end(keys), rnd.Next()); + + for (int i = 0; i < N * 3; i++) { + ASSERT_OK(Put(Key(keys[i] + 1000), rnd.RandomString(1024))); + } + + start = Key(100); + end = Key(300); + r = Range(start, end); + ASSERT_OK( + db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, &size)); + ASSERT_EQ(size, 0); + + start = Key(1050); + end = Key(1080); + r = Range(start, end); + ASSERT_OK( + db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, &size)); + ASSERT_GT(size, 6000); + + start = Key(2100); + end = Key(2300); + r = Range(start, end); + ASSERT_OK( + db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, &size)); + ASSERT_EQ(size, 0); + + start = Key(1050); + end = Key(1080); + r = Range(start, end); + uint64_t size_with_mt, size_without_mt; + ASSERT_OK(db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, + &size_with_mt)); + ASSERT_GT(size_with_mt, 6000); + ASSERT_OK(db_->GetApproximateSizes(&r, 1, &size_without_mt)); + ASSERT_EQ(size_without_mt, 0); + + ASSERT_OK(Flush()); + + for (int i = 0; i < N; i++) { + ASSERT_OK(Put(Key(i + 1000), rnd.RandomString(1024))); + } + + start = Key(1050); + end = Key(1080); + r = Range(start, end); + ASSERT_OK(db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, + &size_with_mt)); + ASSERT_OK(db_->GetApproximateSizes(&r, 1, &size_without_mt)); + ASSERT_GT(size_with_mt, size_without_mt); + ASSERT_GT(size_without_mt, 6000); + + // Check that include_memtables flag works as expected + size_approx_options.include_memtables = false; + ASSERT_OK( + db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, &size)); + ASSERT_EQ(size, size_without_mt); + + // Check that files_size_error_margin works as expected, when the heuristic + // conditions are not met + start = Key(1); + end = Key(1000 + N - 2); + r = Range(start, end); + size_approx_options.files_size_error_margin = -1.0; // disabled + ASSERT_OK( + db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, &size)); + uint64_t size2; + size_approx_options.files_size_error_margin = 0.5; // enabled, but not used + ASSERT_OK( + db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, &size2)); + ASSERT_EQ(size, size2); +} + +TEST_F(DBTest, ApproximateSizesFilesWithErrorMargin) { + // Roughly 4 keys per data block, 1000 keys per file, + // with filter substantially larger than a data block + BlockBasedTableOptions table_options; + table_options.filter_policy.reset(NewBloomFilterPolicy(16)); + table_options.block_size = 100; + Options options = CurrentOptions(); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.write_buffer_size = 24 * 1024; + options.compression = kNoCompression; + options.create_if_missing = true; + options.target_file_size_base = 24 * 1024; + DestroyAndReopen(options); + const auto default_cf = db_->DefaultColumnFamily(); + + const int N = 64000; + Random rnd(301); + for (int i = 0; i < N; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(24))); + } + // Flush everything to files + ASSERT_OK(Flush()); + // Compact the entire key space into the next level + ASSERT_OK( + db_->CompactRange(CompactRangeOptions(), default_cf, nullptr, nullptr)); + + // Write more keys + for (int i = N; i < (N + N / 4); i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(24))); + } + // Flush everything to files again + ASSERT_OK(Flush()); + + // Wait for compaction to finish + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + { + const std::string start = Key(0); + const std::string end = Key(2 * N); + const Range r(start, end); + + SizeApproximationOptions size_approx_options; + size_approx_options.include_memtables = false; + size_approx_options.include_files = true; + size_approx_options.files_size_error_margin = -1.0; // disabled + + // Get the precise size without any approximation heuristic + uint64_t size; + ASSERT_OK(db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, + &size)); + ASSERT_NE(size, 0); + + // Get the size with an approximation heuristic + uint64_t size2; + const double error_margin = 0.2; + size_approx_options.files_size_error_margin = error_margin; + ASSERT_OK(db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, + &size2)); + ASSERT_LT(size2, size * (1 + error_margin)); + ASSERT_GT(size2, size * (1 - error_margin)); + } + + { + // Ensure that metadata is not falsely attributed only to the last data in + // the file. (In some applications, filters can be large portion of data + // size.) + // Perform many queries over small range, enough to ensure crossing file + // boundary, and make sure we never see a spike for large filter. + for (int i = 0; i < 3000; i += 10) { + const std::string start = Key(i); + const std::string end = Key(i + 11); // overlap by 1 key + const Range r(start, end); + uint64_t size; + ASSERT_OK(db_->GetApproximateSizes(&r, 1, &size)); + ASSERT_LE(size, 11 * 100); + } + } +} + +TEST_F(DBTest, GetApproximateMemTableStats) { + Options options = CurrentOptions(); + options.write_buffer_size = 100000000; + options.compression = kNoCompression; + options.create_if_missing = true; + DestroyAndReopen(options); + + const int N = 128; + Random rnd(301); + for (int i = 0; i < N; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(1024))); + } + + uint64_t count; + uint64_t size; + + std::string start = Key(50); + std::string end = Key(60); + Range r(start, end); + db_->GetApproximateMemTableStats(r, &count, &size); + ASSERT_GT(count, 0); + ASSERT_LE(count, N); + ASSERT_GT(size, 6000); + ASSERT_LT(size, 204800); + + start = Key(500); + end = Key(600); + r = Range(start, end); + db_->GetApproximateMemTableStats(r, &count, &size); + ASSERT_EQ(count, 0); + ASSERT_EQ(size, 0); + + ASSERT_OK(Flush()); + + start = Key(50); + end = Key(60); + r = Range(start, end); + db_->GetApproximateMemTableStats(r, &count, &size); + ASSERT_EQ(count, 0); + ASSERT_EQ(size, 0); + + for (int i = 0; i < N; i++) { + ASSERT_OK(Put(Key(1000 + i), rnd.RandomString(1024))); + } + + start = Key(100); + end = Key(1020); + r = Range(start, end); + db_->GetApproximateMemTableStats(r, &count, &size); + ASSERT_GT(count, 20); + ASSERT_GT(size, 6000); +} + +TEST_F(DBTest, ApproximateSizes) { + do { + Options options = CurrentOptions(); + options.write_buffer_size = 100000000; // Large write buffer + options.compression = kNoCompression; + options.create_if_missing = true; + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + uint64_t size; + ASSERT_OK(Size("", "xyz", 1, &size)); + ASSERT_TRUE(Between(size, 0, 0)); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_OK(Size("", "xyz", 1, &size)); + ASSERT_TRUE(Between(size, 0, 0)); + + // Write 8MB (80 values, each 100K) + ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0); + const int N = 80; + static const int S1 = 100000; + static const int S2 = 105000; // Allow some expansion from metadata + Random rnd(301); + for (int i = 0; i < N; i++) { + ASSERT_OK(Put(1, Key(i), rnd.RandomString(S1))); + } + + // 0 because GetApproximateSizes() does not account for memtable space + ASSERT_OK(Size("", Key(50), 1, &size)); + ASSERT_TRUE(Between(size, 0, 0)); + + // Check sizes across recovery by reopening a few times + for (int run = 0; run < 3; run++) { + ReopenWithColumnFamilies({"default", "pikachu"}, options); + + for (int compact_start = 0; compact_start < N; compact_start += 10) { + for (int i = 0; i < N; i += 10) { + ASSERT_OK(Size("", Key(i), 1, &size)); + ASSERT_TRUE(Between(size, S1 * i, S2 * i)); + ASSERT_OK(Size("", Key(i) + ".suffix", 1, &size)); + ASSERT_TRUE(Between(size, S1 * (i + 1), S2 * (i + 1))); + ASSERT_OK(Size(Key(i), Key(i + 10), 1, &size)); + ASSERT_TRUE(Between(size, S1 * 10, S2 * 10)); + } + ASSERT_OK(Size("", Key(50), 1, &size)); + ASSERT_TRUE(Between(size, S1 * 50, S2 * 50)); + ASSERT_OK(Size("", Key(50) + ".suffix", 1, &size)); + ASSERT_TRUE(Between(size, S1 * 50, S2 * 50)); + + std::string cstart_str = Key(compact_start); + std::string cend_str = Key(compact_start + 9); + Slice cstart = cstart_str; + Slice cend = cend_str; + ASSERT_OK(dbfull()->TEST_CompactRange(0, &cstart, &cend, handles_[1])); + } + + ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0); + ASSERT_GT(NumTableFilesAtLevel(1, 1), 0); + } + // ApproximateOffsetOf() is not yet implemented in plain table format. + } while (ChangeOptions(kSkipUniversalCompaction | kSkipFIFOCompaction | + kSkipPlainTable | kSkipHashIndex)); +} + +TEST_F(DBTest, ApproximateSizes_MixOfSmallAndLarge) { + do { + Options options = CurrentOptions(); + options.compression = kNoCompression; + CreateAndReopenWithCF({"pikachu"}, options); + + Random rnd(301); + std::string big1 = rnd.RandomString(100000); + ASSERT_OK(Put(1, Key(0), rnd.RandomString(10000))); + ASSERT_OK(Put(1, Key(1), rnd.RandomString(10000))); + ASSERT_OK(Put(1, Key(2), big1)); + ASSERT_OK(Put(1, Key(3), rnd.RandomString(10000))); + ASSERT_OK(Put(1, Key(4), big1)); + ASSERT_OK(Put(1, Key(5), rnd.RandomString(10000))); + ASSERT_OK(Put(1, Key(6), rnd.RandomString(300000))); + ASSERT_OK(Put(1, Key(7), rnd.RandomString(10000))); + + // Check sizes across recovery by reopening a few times + uint64_t size; + for (int run = 0; run < 3; run++) { + ReopenWithColumnFamilies({"default", "pikachu"}, options); + + ASSERT_OK(Size("", Key(0), 1, &size)); + ASSERT_TRUE(Between(size, 0, 0)); + ASSERT_OK(Size("", Key(1), 1, &size)); + ASSERT_TRUE(Between(size, 10000, 11000)); + ASSERT_OK(Size("", Key(2), 1, &size)); + ASSERT_TRUE(Between(size, 20000, 21000)); + ASSERT_OK(Size("", Key(3), 1, &size)); + ASSERT_TRUE(Between(size, 120000, 121000)); + ASSERT_OK(Size("", Key(4), 1, &size)); + ASSERT_TRUE(Between(size, 130000, 131000)); + ASSERT_OK(Size("", Key(5), 1, &size)); + ASSERT_TRUE(Between(size, 230000, 232000)); + ASSERT_OK(Size("", Key(6), 1, &size)); + ASSERT_TRUE(Between(size, 240000, 242000)); + // Ensure some overhead is accounted for, even without including all + ASSERT_OK(Size("", Key(7), 1, &size)); + ASSERT_TRUE(Between(size, 540500, 545000)); + ASSERT_OK(Size("", Key(8), 1, &size)); + ASSERT_TRUE(Between(size, 550500, 555000)); + + ASSERT_OK(Size(Key(3), Key(5), 1, &size)); + ASSERT_TRUE(Between(size, 110100, 111000)); + + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1])); + } + // ApproximateOffsetOf() is not yet implemented in plain table format. + } while (ChangeOptions(kSkipPlainTable)); +} + +TEST_F(DBTest, Snapshot) { + env_->SetMockSleep(); + anon::OptionsOverride options_override; + options_override.skip_policy = kSkipNoSnapshot; + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions(options_override)); + ASSERT_OK(Put(0, "foo", "0v1")); + ASSERT_OK(Put(1, "foo", "1v1")); + + const Snapshot* s1 = db_->GetSnapshot(); + ASSERT_EQ(1U, GetNumSnapshots()); + uint64_t time_snap1 = GetTimeOldestSnapshots(); + ASSERT_GT(time_snap1, 0U); + ASSERT_EQ(GetSequenceOldestSnapshots(), s1->GetSequenceNumber()); + ASSERT_EQ(GetTimeOldestSnapshots(), + static_cast(s1->GetUnixTime())); + ASSERT_OK(Put(0, "foo", "0v2")); + ASSERT_OK(Put(1, "foo", "1v2")); + + env_->MockSleepForSeconds(1); + + const Snapshot* s2 = db_->GetSnapshot(); + ASSERT_EQ(2U, GetNumSnapshots()); + ASSERT_EQ(time_snap1, GetTimeOldestSnapshots()); + ASSERT_EQ(GetSequenceOldestSnapshots(), s1->GetSequenceNumber()); + ASSERT_EQ(GetTimeOldestSnapshots(), + static_cast(s1->GetUnixTime())); + ASSERT_OK(Put(0, "foo", "0v3")); + ASSERT_OK(Put(1, "foo", "1v3")); + + { + ManagedSnapshot s3(db_); + ASSERT_EQ(3U, GetNumSnapshots()); + ASSERT_EQ(time_snap1, GetTimeOldestSnapshots()); + ASSERT_EQ(GetSequenceOldestSnapshots(), s1->GetSequenceNumber()); + ASSERT_EQ(GetTimeOldestSnapshots(), + static_cast(s1->GetUnixTime())); + + ASSERT_OK(Put(0, "foo", "0v4")); + ASSERT_OK(Put(1, "foo", "1v4")); + ASSERT_EQ("0v1", Get(0, "foo", s1)); + ASSERT_EQ("1v1", Get(1, "foo", s1)); + ASSERT_EQ("0v2", Get(0, "foo", s2)); + ASSERT_EQ("1v2", Get(1, "foo", s2)); + ASSERT_EQ("0v3", Get(0, "foo", s3.snapshot())); + ASSERT_EQ("1v3", Get(1, "foo", s3.snapshot())); + ASSERT_EQ("0v4", Get(0, "foo")); + ASSERT_EQ("1v4", Get(1, "foo")); + } + + ASSERT_EQ(2U, GetNumSnapshots()); + ASSERT_EQ(time_snap1, GetTimeOldestSnapshots()); + ASSERT_EQ(GetSequenceOldestSnapshots(), s1->GetSequenceNumber()); + ASSERT_EQ(GetTimeOldestSnapshots(), + static_cast(s1->GetUnixTime())); + ASSERT_EQ("0v1", Get(0, "foo", s1)); + ASSERT_EQ("1v1", Get(1, "foo", s1)); + ASSERT_EQ("0v2", Get(0, "foo", s2)); + ASSERT_EQ("1v2", Get(1, "foo", s2)); + ASSERT_EQ("0v4", Get(0, "foo")); + ASSERT_EQ("1v4", Get(1, "foo")); + + db_->ReleaseSnapshot(s1); + ASSERT_EQ("0v2", Get(0, "foo", s2)); + ASSERT_EQ("1v2", Get(1, "foo", s2)); + ASSERT_EQ("0v4", Get(0, "foo")); + ASSERT_EQ("1v4", Get(1, "foo")); + ASSERT_EQ(1U, GetNumSnapshots()); + ASSERT_LT(time_snap1, GetTimeOldestSnapshots()); + ASSERT_EQ(GetSequenceOldestSnapshots(), s2->GetSequenceNumber()); + ASSERT_EQ(GetTimeOldestSnapshots(), + static_cast(s2->GetUnixTime())); + + db_->ReleaseSnapshot(s2); + ASSERT_EQ(0U, GetNumSnapshots()); + ASSERT_EQ(GetSequenceOldestSnapshots(), 0); + ASSERT_EQ("0v4", Get(0, "foo")); + ASSERT_EQ("1v4", Get(1, "foo")); + } while (ChangeOptions()); +} + +TEST_F(DBTest, HiddenValuesAreRemoved) { + anon::OptionsOverride options_override; + options_override.skip_policy = kSkipNoSnapshot; + uint64_t size; + do { + Options options = CurrentOptions(options_override); + CreateAndReopenWithCF({"pikachu"}, options); + Random rnd(301); + FillLevels("a", "z", 1); + + std::string big = rnd.RandomString(50000); + ASSERT_OK(Put(1, "foo", big)); + ASSERT_OK(Put(1, "pastfoo", "v")); + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK(Put(1, "foo", "tiny")); + ASSERT_OK(Put(1, "pastfoo2", "v2")); // Advance sequence number one more + + ASSERT_OK(Flush(1)); + ASSERT_GT(NumTableFilesAtLevel(0, 1), 0); + + ASSERT_EQ(big, Get(1, "foo", snapshot)); + ASSERT_OK(Size("", "pastfoo", 1, &size)); + ASSERT_TRUE(Between(size, 50000, 60000)); + db_->ReleaseSnapshot(snapshot); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ tiny, " + big + " ]"); + Slice x("x"); + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, &x, handles_[1])); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ tiny ]"); + ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0); + ASSERT_GE(NumTableFilesAtLevel(1, 1), 1); + ASSERT_OK(dbfull()->TEST_CompactRange(1, nullptr, &x, handles_[1])); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ tiny ]"); + + ASSERT_OK(Size("", "pastfoo", 1, &size)); + ASSERT_TRUE(Between(size, 0, 1000)); + // ApproximateOffsetOf() is not yet implemented in plain table format, + // which is used by Size(). + } while (ChangeOptions(kSkipUniversalCompaction | kSkipFIFOCompaction | + kSkipPlainTable)); +} + +TEST_F(DBTest, UnremovableSingleDelete) { + // If we compact: + // + // Put(A, v1) Snapshot SingleDelete(A) Put(A, v2) + // + // We do not want to end up with: + // + // Put(A, v1) Snapshot Put(A, v2) + // + // Because a subsequent SingleDelete(A) would delete the Put(A, v2) + // but not Put(A, v1), so Get(A) would return v1. + anon::OptionsOverride options_override; + options_override.skip_policy = kSkipNoSnapshot; + do { + Options options = CurrentOptions(options_override); + options.disable_auto_compactions = true; + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(1, "foo", "first")); + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK(SingleDelete(1, "foo")); + ASSERT_OK(Put(1, "foo", "second")); + ASSERT_OK(Flush(1)); + + ASSERT_EQ("first", Get(1, "foo", snapshot)); + ASSERT_EQ("second", Get(1, "foo")); + + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), handles_[1], + nullptr, nullptr)); + ASSERT_EQ("[ second, SDEL, first ]", AllEntriesFor("foo", 1)); + + ASSERT_OK(SingleDelete(1, "foo")); + + ASSERT_EQ("first", Get(1, "foo", snapshot)); + ASSERT_EQ("NOT_FOUND", Get(1, "foo")); + + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), handles_[1], + nullptr, nullptr)); + + ASSERT_EQ("first", Get(1, "foo", snapshot)); + ASSERT_EQ("NOT_FOUND", Get(1, "foo")); + db_->ReleaseSnapshot(snapshot); + // Skip FIFO and universal compaction because they do not apply to the test + // case. Skip MergePut because single delete does not get removed when it + // encounters a merge. + } while (ChangeOptions(kSkipFIFOCompaction | kSkipUniversalCompaction | + kSkipMergePut)); +} + +TEST_F(DBTest, DeletionMarkers1) { + Options options = CurrentOptions(); + CreateAndReopenWithCF({"pikachu"}, options); + ASSERT_OK(Put(1, "foo", "v1")); + ASSERT_OK(Flush(1)); + const int last = 2; + MoveFilesToLevel(last, 1); + // foo => v1 is now in last level + ASSERT_EQ(NumTableFilesAtLevel(last, 1), 1); + + // Place a table at level last-1 to prevent merging with preceding mutation + ASSERT_OK(Put(1, "a", "begin")); + ASSERT_OK(Put(1, "z", "end")); + ASSERT_OK(Flush(1)); + MoveFilesToLevel(last - 1, 1); + ASSERT_EQ(NumTableFilesAtLevel(last, 1), 1); + ASSERT_EQ(NumTableFilesAtLevel(last - 1, 1), 1); + + ASSERT_OK(Delete(1, "foo")); + ASSERT_OK(Put(1, "foo", "v2")); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, DEL, v1 ]"); + ASSERT_OK(Flush(1)); // Moves to level last-2 + ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, v1 ]"); + Slice z("z"); + ASSERT_OK(dbfull()->TEST_CompactRange(last - 2, nullptr, &z, handles_[1])); + // DEL eliminated, but v1 remains because we aren't compacting that level + // (DEL can be eliminated because v2 hides v1). + ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, v1 ]"); + ASSERT_OK( + dbfull()->TEST_CompactRange(last - 1, nullptr, nullptr, handles_[1])); + // Merging last-1 w/ last, so we are the base level for "foo", so + // DEL is removed. (as is v1). + ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2 ]"); +} + +TEST_F(DBTest, DeletionMarkers2) { + Options options = CurrentOptions(); + CreateAndReopenWithCF({"pikachu"}, options); + ASSERT_OK(Put(1, "foo", "v1")); + ASSERT_OK(Flush(1)); + const int last = 2; + MoveFilesToLevel(last, 1); + // foo => v1 is now in last level + ASSERT_EQ(NumTableFilesAtLevel(last, 1), 1); + + // Place a table at level last-1 to prevent merging with preceding mutation + ASSERT_OK(Put(1, "a", "begin")); + ASSERT_OK(Put(1, "z", "end")); + ASSERT_OK(Flush(1)); + MoveFilesToLevel(last - 1, 1); + ASSERT_EQ(NumTableFilesAtLevel(last, 1), 1); + ASSERT_EQ(NumTableFilesAtLevel(last - 1, 1), 1); + + ASSERT_OK(Delete(1, "foo")); + ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v1 ]"); + ASSERT_OK(Flush(1)); // Moves to level last-2 + ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v1 ]"); + ASSERT_OK( + dbfull()->TEST_CompactRange(last - 2, nullptr, nullptr, handles_[1])); + // DEL kept: "last" file overlaps + ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v1 ]"); + ASSERT_OK( + dbfull()->TEST_CompactRange(last - 1, nullptr, nullptr, handles_[1])); + // Merging last-1 w/ last, so we are the base level for "foo", so + // DEL is removed. (as is v1). + ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]"); +} + +TEST_F(DBTest, OverlapInLevel0) { + do { + Options options = CurrentOptions(); + CreateAndReopenWithCF({"pikachu"}, options); + + // Fill levels 1 and 2 to disable the pushing of new memtables to levels > + // 0. + ASSERT_OK(Put(1, "100", "v100")); + ASSERT_OK(Put(1, "999", "v999")); + ASSERT_OK(Flush(1)); + MoveFilesToLevel(2, 1); + ASSERT_OK(Delete(1, "100")); + ASSERT_OK(Delete(1, "999")); + ASSERT_OK(Flush(1)); + MoveFilesToLevel(1, 1); + ASSERT_EQ("0,1,1", FilesPerLevel(1)); + + // Make files spanning the following ranges in level-0: + // files[0] 200 .. 900 + // files[1] 300 .. 500 + // Note that files are sorted by smallest key. + ASSERT_OK(Put(1, "300", "v300")); + ASSERT_OK(Put(1, "500", "v500")); + ASSERT_OK(Flush(1)); + ASSERT_OK(Put(1, "200", "v200")); + ASSERT_OK(Put(1, "600", "v600")); + ASSERT_OK(Put(1, "900", "v900")); + ASSERT_OK(Flush(1)); + ASSERT_EQ("2,1,1", FilesPerLevel(1)); + + // BEGIN addition to existing test + // Take this opportunity to verify SST unique ids (including Plain table) + TablePropertiesCollection tbc; + ASSERT_OK(db_->GetPropertiesOfAllTables(handles_[1], &tbc)); + VerifySstUniqueIds(tbc); + // END addition to existing test + + // Compact away the placeholder files we created initially + ASSERT_OK(dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1])); + ASSERT_OK(dbfull()->TEST_CompactRange(2, nullptr, nullptr, handles_[1])); + ASSERT_EQ("2", FilesPerLevel(1)); + + // Do a memtable compaction. Before bug-fix, the compaction would + // not detect the overlap with level-0 files and would incorrectly place + // the deletion in a deeper level. + ASSERT_OK(Delete(1, "600")); + ASSERT_OK(Flush(1)); + ASSERT_EQ("3", FilesPerLevel(1)); + ASSERT_EQ("NOT_FOUND", Get(1, "600")); + } while (ChangeOptions(kSkipUniversalCompaction | kSkipFIFOCompaction)); +} + +TEST_F(DBTest, ComparatorCheck) { + class NewComparator : public Comparator { + public: + const char* Name() const override { return "rocksdb.NewComparator"; } + int Compare(const Slice& a, const Slice& b) const override { + return BytewiseComparator()->Compare(a, b); + } + void FindShortestSeparator(std::string* s, const Slice& l) const override { + BytewiseComparator()->FindShortestSeparator(s, l); + } + void FindShortSuccessor(std::string* key) const override { + BytewiseComparator()->FindShortSuccessor(key); + } + }; + Options new_options, options; + NewComparator cmp; + do { + options = CurrentOptions(); + CreateAndReopenWithCF({"pikachu"}, options); + new_options = CurrentOptions(); + new_options.comparator = &cmp; + // only the non-default column family has non-matching comparator + Status s = TryReopenWithColumnFamilies( + {"default", "pikachu"}, std::vector({options, new_options})); + ASSERT_TRUE(!s.ok()); + ASSERT_TRUE(s.ToString().find("comparator") != std::string::npos) + << s.ToString(); + } while (ChangeCompactOptions()); +} + +TEST_F(DBTest, CustomComparator) { + class NumberComparator : public Comparator { + public: + const char* Name() const override { return "test.NumberComparator"; } + int Compare(const Slice& a, const Slice& b) const override { + return ToNumber(a) - ToNumber(b); + } + void FindShortestSeparator(std::string* s, const Slice& l) const override { + ToNumber(*s); // Check format + ToNumber(l); // Check format + } + void FindShortSuccessor(std::string* key) const override { + ToNumber(*key); // Check format + } + + private: + static int ToNumber(const Slice& x) { + // Check that there are no extra characters. + EXPECT_TRUE(x.size() >= 2 && x[0] == '[' && x[x.size() - 1] == ']') + << EscapeString(x); + int val; + char ignored; + EXPECT_TRUE(sscanf(x.ToString().c_str(), "[%i]%c", &val, &ignored) == 1) + << EscapeString(x); + return val; + } + }; + Options new_options; + NumberComparator cmp; + do { + new_options = CurrentOptions(); + new_options.create_if_missing = true; + new_options.comparator = &cmp; + new_options.write_buffer_size = 4096; // Compact more often + new_options.arena_block_size = 4096; + new_options = CurrentOptions(new_options); + DestroyAndReopen(new_options); + CreateAndReopenWithCF({"pikachu"}, new_options); + ASSERT_OK(Put(1, "[10]", "ten")); + ASSERT_OK(Put(1, "[0x14]", "twenty")); + for (int i = 0; i < 2; i++) { + ASSERT_EQ("ten", Get(1, "[10]")); + ASSERT_EQ("ten", Get(1, "[0xa]")); + ASSERT_EQ("twenty", Get(1, "[20]")); + ASSERT_EQ("twenty", Get(1, "[0x14]")); + ASSERT_EQ("NOT_FOUND", Get(1, "[15]")); + ASSERT_EQ("NOT_FOUND", Get(1, "[0xf]")); + Compact(1, "[0]", "[9999]"); + } + + for (int run = 0; run < 2; run++) { + for (int i = 0; i < 1000; i++) { + char buf[100]; + snprintf(buf, sizeof(buf), "[%d]", i * 10); + ASSERT_OK(Put(1, buf, buf)); + } + Compact(1, "[0]", "[1000000]"); + } + } while (ChangeCompactOptions()); +} + +TEST_F(DBTest, DBOpen_Options) { + Options options = CurrentOptions(); + std::string dbname = test::PerThreadDBPath("db_options_test"); + ASSERT_OK(DestroyDB(dbname, options)); + + // Does not exist, and create_if_missing == false: error + DB* db = nullptr; + options.create_if_missing = false; + Status s = DB::Open(options, dbname, &db); + ASSERT_TRUE(strstr(s.ToString().c_str(), "does not exist") != nullptr); + ASSERT_TRUE(db == nullptr); + + // Does not exist, and create_if_missing == true: OK + options.create_if_missing = true; + s = DB::Open(options, dbname, &db); + ASSERT_OK(s); + ASSERT_TRUE(db != nullptr); + + delete db; + db = nullptr; + + // Does exist, and error_if_exists == true: error + options.create_if_missing = false; + options.error_if_exists = true; + s = DB::Open(options, dbname, &db); + ASSERT_TRUE(strstr(s.ToString().c_str(), "exists") != nullptr); + ASSERT_TRUE(db == nullptr); + + // Does exist, and error_if_exists == false: OK + options.create_if_missing = true; + options.error_if_exists = false; + s = DB::Open(options, dbname, &db); + ASSERT_OK(s); + ASSERT_TRUE(db != nullptr); + + delete db; + db = nullptr; +} + +TEST_F(DBTest, DBOpen_Change_NumLevels) { + Options options = CurrentOptions(); + options.create_if_missing = true; + DestroyAndReopen(options); + ASSERT_TRUE(db_ != nullptr); + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(1, "a", "123")); + ASSERT_OK(Put(1, "b", "234")); + ASSERT_OK(Flush(1)); + MoveFilesToLevel(3, 1); + Close(); + + options.create_if_missing = false; + options.num_levels = 2; + Status s = TryReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_TRUE(strstr(s.ToString().c_str(), "Invalid argument") != nullptr); + ASSERT_TRUE(db_ == nullptr); +} + +TEST_F(DBTest, DestroyDBMetaDatabase) { + std::string dbname = test::PerThreadDBPath("db_meta"); + ASSERT_OK(env_->CreateDirIfMissing(dbname)); + std::string metadbname = MetaDatabaseName(dbname, 0); + ASSERT_OK(env_->CreateDirIfMissing(metadbname)); + std::string metametadbname = MetaDatabaseName(metadbname, 0); + ASSERT_OK(env_->CreateDirIfMissing(metametadbname)); + + // Destroy previous versions if they exist. Using the long way. + Options options = CurrentOptions(); + ASSERT_OK(DestroyDB(metametadbname, options)); + ASSERT_OK(DestroyDB(metadbname, options)); + ASSERT_OK(DestroyDB(dbname, options)); + + // Setup databases + DB* db = nullptr; + ASSERT_OK(DB::Open(options, dbname, &db)); + delete db; + db = nullptr; + ASSERT_OK(DB::Open(options, metadbname, &db)); + delete db; + db = nullptr; + ASSERT_OK(DB::Open(options, metametadbname, &db)); + delete db; + db = nullptr; + + // Delete databases + ASSERT_OK(DestroyDB(dbname, options)); + + // Check if deletion worked. + options.create_if_missing = false; + ASSERT_TRUE(!(DB::Open(options, dbname, &db)).ok()); + ASSERT_TRUE(!(DB::Open(options, metadbname, &db)).ok()); + ASSERT_TRUE(!(DB::Open(options, metametadbname, &db)).ok()); +} + +TEST_F(DBTest, SnapshotFiles) { + do { + Options options = CurrentOptions(); + options.write_buffer_size = 100000000; // Large write buffer + CreateAndReopenWithCF({"pikachu"}, options); + + Random rnd(301); + + // Write 8MB (80 values, each 100K) + ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0); + std::vector values; + for (int i = 0; i < 80; i++) { + values.push_back(rnd.RandomString(100000)); + ASSERT_OK(Put((i < 40), Key(i), values[i])); + } + + // assert that nothing makes it to disk yet. + ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0); + + // get a file snapshot + uint64_t manifest_number = 0; + uint64_t manifest_size = 0; + std::vector files; + ASSERT_OK(dbfull()->DisableFileDeletions()); + ASSERT_OK(dbfull()->GetLiveFiles(files, &manifest_size)); + + // CURRENT, MANIFEST, OPTIONS, *.sst files (one for each CF) + ASSERT_EQ(files.size(), 5U); + + uint64_t number = 0; + FileType type; + + // copy these files to a new snapshot directory + std::string snapdir = dbname_ + ".snapdir/"; + if (env_->FileExists(snapdir).ok()) { + ASSERT_OK(DestroyDir(env_, snapdir)); + } + ASSERT_OK(env_->CreateDir(snapdir)); + + for (size_t i = 0; i < files.size(); i++) { + // our clients require that GetLiveFiles returns + // files with "/" as first character! + ASSERT_EQ(files[i][0], '/'); + std::string src = dbname_ + files[i]; + std::string dest = snapdir + files[i]; + + uint64_t size; + ASSERT_OK(env_->GetFileSize(src, &size)); + + // record the number and the size of the + // latest manifest file + if (ParseFileName(files[i].substr(1), &number, &type)) { + if (type == kDescriptorFile) { + ASSERT_EQ(manifest_number, 0); + manifest_number = number; + ASSERT_GE(size, manifest_size); + size = manifest_size; // copy only valid MANIFEST data + } + } + CopyFile(src, dest, size); + } + + // release file snapshot + ASSERT_OK(dbfull()->EnableFileDeletions(/*force*/ false)); + // overwrite one key, this key should not appear in the snapshot + std::vector extras; + for (unsigned int i = 0; i < 1; i++) { + extras.push_back(rnd.RandomString(100000)); + ASSERT_OK(Put(0, Key(i), extras[i])); + } + + // verify that data in the snapshot are correct + std::vector column_families; + column_families.emplace_back("default", ColumnFamilyOptions()); + column_families.emplace_back("pikachu", ColumnFamilyOptions()); + std::vector cf_handles; + DB* snapdb; + DBOptions opts; + opts.env = env_; + opts.create_if_missing = false; + Status stat = + DB::Open(opts, snapdir, column_families, &cf_handles, &snapdb); + ASSERT_OK(stat); + + ReadOptions roptions; + std::string val; + for (unsigned int i = 0; i < 80; i++) { + ASSERT_OK(snapdb->Get(roptions, cf_handles[i < 40], Key(i), &val)); + ASSERT_EQ(values[i].compare(val), 0); + } + for (auto cfh : cf_handles) { + delete cfh; + } + delete snapdb; + + // look at the new live files after we added an 'extra' key + // and after we took the first snapshot. + uint64_t new_manifest_number = 0; + uint64_t new_manifest_size = 0; + std::vector newfiles; + ASSERT_OK(dbfull()->DisableFileDeletions()); + ASSERT_OK(dbfull()->GetLiveFiles(newfiles, &new_manifest_size)); + + // find the new manifest file. assert that this manifest file is + // the same one as in the previous snapshot. But its size should be + // larger because we added an extra key after taking the + // previous shapshot. + for (size_t i = 0; i < newfiles.size(); i++) { + std::string src = dbname_ + "/" + newfiles[i]; + // record the lognumber and the size of the + // latest manifest file + if (ParseFileName(newfiles[i].substr(1), &number, &type)) { + if (type == kDescriptorFile) { + ASSERT_EQ(new_manifest_number, 0); + uint64_t size; + new_manifest_number = number; + ASSERT_OK(env_->GetFileSize(src, &size)); + ASSERT_GE(size, new_manifest_size); + } + } + } + ASSERT_EQ(manifest_number, new_manifest_number); + ASSERT_GT(new_manifest_size, manifest_size); + + // Also test GetLiveFilesStorageInfo + std::vector new_infos; + ASSERT_OK(db_->GetLiveFilesStorageInfo(LiveFilesStorageInfoOptions(), + &new_infos)); + + // Close DB (while deletions disabled) + Close(); + + // Validate + for (auto& info : new_infos) { + std::string path = info.directory + "/" + info.relative_filename; + uint64_t size; + ASSERT_OK(env_->GetFileSize(path, &size)); + if (info.trim_to_size) { + ASSERT_LE(info.size, size); + } else if (!info.replacement_contents.empty()) { + ASSERT_EQ(info.size, info.replacement_contents.size()); + } else { + ASSERT_EQ(info.size, size); + } + if (info.file_type == kDescriptorFile) { + ASSERT_EQ(info.file_number, manifest_number); + } + } + } while (ChangeCompactOptions()); +} + +TEST_F(DBTest, ReadonlyDBGetLiveManifestSize) { + do { + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = 2; + DestroyAndReopen(options); + + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + + uint64_t manifest_size = 0; + std::vector files; + ASSERT_OK(dbfull()->GetLiveFiles(files, &manifest_size)); + + for (const std::string& f : files) { + uint64_t number = 0; + FileType type; + if (ParseFileName(f.substr(1), &number, &type)) { + if (type == kDescriptorFile) { + uint64_t size_on_disk; + ASSERT_OK(env_->GetFileSize(dbname_ + "/" + f, &size_on_disk)); + ASSERT_EQ(manifest_size, size_on_disk); + break; + } + } + } + Close(); + } while (ChangeCompactOptions()); +} + +TEST_F(DBTest, GetLiveBlobFiles) { + // Note: the following prevents an otherwise harmless data race between the + // test setup code (AddBlobFile) below and the periodic stat dumping thread. + Options options = CurrentOptions(); + options.stats_dump_period_sec = 0; + + constexpr uint64_t blob_file_number = 234; + constexpr uint64_t total_blob_count = 555; + constexpr uint64_t total_blob_bytes = 66666; + constexpr char checksum_method[] = "CRC32"; + constexpr char checksum_value[] = "\x3d\x87\xff\x57"; + constexpr uint64_t garbage_blob_count = 0; + constexpr uint64_t garbage_blob_bytes = 0; + + Reopen(options); + + AddBlobFile(db_->DefaultColumnFamily(), blob_file_number, total_blob_count, + total_blob_bytes, checksum_method, checksum_value, + garbage_blob_count, garbage_blob_bytes); + // Make sure it appears in the results returned by GetLiveFiles. + uint64_t manifest_size = 0; + std::vector files; + ASSERT_OK(dbfull()->GetLiveFiles(files, &manifest_size)); + + ASSERT_FALSE(files.empty()); + ASSERT_EQ(files[0], BlobFileName("", blob_file_number)); + + ColumnFamilyMetaData cfmd; + + db_->GetColumnFamilyMetaData(&cfmd); + ASSERT_EQ(cfmd.blob_files.size(), 1); + const BlobMetaData& bmd = cfmd.blob_files[0]; + + CheckBlobMetaData(bmd, blob_file_number, total_blob_count, total_blob_bytes, + checksum_method, checksum_value, garbage_blob_count, + garbage_blob_bytes); + ASSERT_EQ(NormalizePath(bmd.blob_file_path), NormalizePath(dbname_)); + ASSERT_EQ(cfmd.blob_file_count, 1U); + ASSERT_EQ(cfmd.blob_file_size, bmd.blob_file_size); +} + +TEST_F(DBTest, PurgeInfoLogs) { + Options options = CurrentOptions(); + options.keep_log_file_num = 5; + options.create_if_missing = true; + options.env = env_; + for (int mode = 0; mode <= 1; mode++) { + if (mode == 1) { + options.db_log_dir = dbname_ + "_logs"; + ASSERT_OK(env_->CreateDirIfMissing(options.db_log_dir)); + } else { + options.db_log_dir = ""; + } + for (int i = 0; i < 8; i++) { + Reopen(options); + } + + std::vector files; + ASSERT_OK(env_->GetChildren( + options.db_log_dir.empty() ? dbname_ : options.db_log_dir, &files)); + int info_log_count = 0; + for (std::string file : files) { + if (file.find("LOG") != std::string::npos) { + info_log_count++; + } + } + ASSERT_EQ(5, info_log_count); + + Destroy(options); + // For mode (1), test DestroyDB() to delete all the logs under DB dir. + // For mode (2), no info log file should have been put under DB dir. + // Since dbname_ has no children, there is no need to loop db_files + std::vector db_files; + ASSERT_TRUE(env_->GetChildren(dbname_, &db_files).IsNotFound()); + ASSERT_TRUE(db_files.empty()); + + if (mode == 1) { + // Cleaning up + ASSERT_OK(env_->GetChildren(options.db_log_dir, &files)); + for (std::string file : files) { + ASSERT_OK(env_->DeleteFile(options.db_log_dir + "/" + file)); + } + ASSERT_OK(env_->DeleteDir(options.db_log_dir)); + } + } +} + +// Multi-threaded test: +namespace { + +static const int kColumnFamilies = 10; +static const int kNumThreads = 10; +static const int kTestSeconds = 10; +static const int kNumKeys = 1000; + +struct MTState { + DBTest* test; + std::atomic counter[kNumThreads]; +}; + +struct MTThread { + MTState* state; + int id; + bool multiget_batched; +}; + +static void MTThreadBody(void* arg) { + MTThread* t = reinterpret_cast(arg); + int id = t->id; + DB* db = t->state->test->db_; + int counter = 0; + std::shared_ptr clock = SystemClock::Default(); + auto end_micros = clock->NowMicros() + kTestSeconds * 1000000U; + + fprintf(stderr, "... starting thread %d\n", id); + Random rnd(1000 + id); + char valbuf[1500]; + while (clock->NowMicros() < end_micros) { + t->state->counter[id].store(counter, std::memory_order_release); + + int key = rnd.Uniform(kNumKeys); + char keybuf[20]; + snprintf(keybuf, sizeof(keybuf), "%016d", key); + + if (rnd.OneIn(2)) { + // Write values of the form . + // into each of the CFs + // We add some padding for force compactions. + int unique_id = rnd.Uniform(1000000); + + // Half of the time directly use WriteBatch. Half of the time use + // WriteBatchWithIndex. + if (rnd.OneIn(2)) { + WriteBatch batch; + for (int cf = 0; cf < kColumnFamilies; ++cf) { + snprintf(valbuf, sizeof(valbuf), "%d.%d.%d.%d.%-1000d", key, id, + static_cast(counter), cf, unique_id); + ASSERT_OK(batch.Put(t->state->test->handles_[cf], Slice(keybuf), + Slice(valbuf))); + } + ASSERT_OK(db->Write(WriteOptions(), &batch)); + } else { + WriteBatchWithIndex batch(db->GetOptions().comparator); + for (int cf = 0; cf < kColumnFamilies; ++cf) { + snprintf(valbuf, sizeof(valbuf), "%d.%d.%d.%d.%-1000d", key, id, + static_cast(counter), cf, unique_id); + ASSERT_OK(batch.Put(t->state->test->handles_[cf], Slice(keybuf), + Slice(valbuf))); + } + ASSERT_OK(db->Write(WriteOptions(), batch.GetWriteBatch())); + } + } else { + // Read a value and verify that it matches the pattern written above + // and that writes to all column families were atomic (unique_id is the + // same) + std::vector keys(kColumnFamilies, Slice(keybuf)); + std::vector values; + std::vector statuses; + if (!t->multiget_batched) { + statuses = db->MultiGet(ReadOptions(), t->state->test->handles_, keys, + &values); + } else { + std::vector pin_values(keys.size()); + statuses.resize(keys.size()); + const Snapshot* snapshot = db->GetSnapshot(); + ReadOptions ro; + ro.snapshot = snapshot; + for (int cf = 0; cf < kColumnFamilies; ++cf) { + db->MultiGet(ro, t->state->test->handles_[cf], 1, &keys[cf], + &pin_values[cf], &statuses[cf]); + } + db->ReleaseSnapshot(snapshot); + values.resize(keys.size()); + for (int cf = 0; cf < kColumnFamilies; ++cf) { + if (statuses[cf].ok()) { + values[cf].assign(pin_values[cf].data(), pin_values[cf].size()); + } + } + } + Status s = statuses[0]; + // all statuses have to be the same + for (size_t i = 1; i < statuses.size(); ++i) { + // they are either both ok or both not-found + ASSERT_TRUE((s.ok() && statuses[i].ok()) || + (s.IsNotFound() && statuses[i].IsNotFound())); + } + if (s.IsNotFound()) { + // Key has not yet been written + } else { + // Check that the writer thread counter is >= the counter in the value + ASSERT_OK(s); + int unique_id = -1; + for (int i = 0; i < kColumnFamilies; ++i) { + int k, w, c, cf, u; + ASSERT_EQ(5, sscanf(values[i].c_str(), "%d.%d.%d.%d.%d", &k, &w, &c, + &cf, &u)) + << values[i]; + ASSERT_EQ(k, key); + ASSERT_GE(w, 0); + ASSERT_LT(w, kNumThreads); + ASSERT_LE(c, t->state->counter[w].load(std::memory_order_acquire)); + ASSERT_EQ(cf, i); + if (i == 0) { + unique_id = u; + } else { + // this checks that updates across column families happened + // atomically -- all unique ids are the same + ASSERT_EQ(u, unique_id); + } + } + } + } + counter++; + } + fprintf(stderr, "... stopping thread %d after %d ops\n", id, int(counter)); +} + +} // anonymous namespace + +class MultiThreadedDBTest + : public DBTest, + public ::testing::WithParamInterface> { + public: + void SetUp() override { + std::tie(option_config_, multiget_batched_) = GetParam(); + } + + static std::vector GenerateOptionConfigs() { + std::vector optionConfigs; + for (int optionConfig = kDefault; optionConfig < kEnd; ++optionConfig) { + optionConfigs.push_back(optionConfig); + } + return optionConfigs; + } + + bool multiget_batched_; +}; + +TEST_P(MultiThreadedDBTest, MultiThreaded) { + if (option_config_ == kPipelinedWrite) return; + anon::OptionsOverride options_override; + options_override.skip_policy = kSkipNoSnapshot; + Options options = CurrentOptions(options_override); + std::vector cfs; + for (int i = 1; i < kColumnFamilies; ++i) { + cfs.push_back(std::to_string(i)); + } + Reopen(options); + CreateAndReopenWithCF(cfs, options); + // Initialize state + MTState mt; + mt.test = this; + for (int id = 0; id < kNumThreads; id++) { + mt.counter[id].store(0, std::memory_order_release); + } + + // Start threads + MTThread thread[kNumThreads]; + for (int id = 0; id < kNumThreads; id++) { + thread[id].state = &mt; + thread[id].id = id; + thread[id].multiget_batched = multiget_batched_; + env_->StartThread(MTThreadBody, &thread[id]); + } + + env_->WaitForJoin(); +} + +INSTANTIATE_TEST_CASE_P( + MultiThreaded, MultiThreadedDBTest, + ::testing::Combine( + ::testing::ValuesIn(MultiThreadedDBTest::GenerateOptionConfigs()), + ::testing::Bool())); + +// Group commit test: +#if !defined(OS_WIN) +// Disable this test temporarily on Travis and appveyor as it fails +// intermittently. Github issue: #4151 +namespace { + +static const int kGCNumThreads = 4; +static const int kGCNumKeys = 1000; + +struct GCThread { + DB* db; + int id; + std::atomic done; +}; + +static void GCThreadBody(void* arg) { + GCThread* t = reinterpret_cast(arg); + int id = t->id; + DB* db = t->db; + WriteOptions wo; + + for (int i = 0; i < kGCNumKeys; ++i) { + std::string kv(std::to_string(i + id * kGCNumKeys)); + ASSERT_OK(db->Put(wo, kv, kv)); + } + t->done = true; +} + +} // anonymous namespace + +TEST_F(DBTest, GroupCommitTest) { + do { + Options options = CurrentOptions(); + options.env = env_; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + Reopen(options); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"WriteThread::JoinBatchGroup:BeganWaiting", + "DBImpl::WriteImpl:BeforeLeaderEnters"}, + {"WriteThread::AwaitState:BlockingWaiting", + "WriteThread::EnterAsBatchGroupLeader:End"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Start threads + GCThread thread[kGCNumThreads]; + for (int id = 0; id < kGCNumThreads; id++) { + thread[id].id = id; + thread[id].db = db_; + thread[id].done = false; + env_->StartThread(GCThreadBody, &thread[id]); + } + env_->WaitForJoin(); + + ASSERT_GT(TestGetTickerCount(options, WRITE_DONE_BY_OTHER), 0); + + std::vector expected_db; + for (int i = 0; i < kGCNumThreads * kGCNumKeys; ++i) { + expected_db.push_back(std::to_string(i)); + } + std::sort(expected_db.begin(), expected_db.end()); + + Iterator* itr = db_->NewIterator(ReadOptions()); + itr->SeekToFirst(); + for (auto x : expected_db) { + ASSERT_TRUE(itr->Valid()); + ASSERT_EQ(itr->key().ToString(), x); + ASSERT_EQ(itr->value().ToString(), x); + itr->Next(); + } + ASSERT_TRUE(!itr->Valid()); + delete itr; + + HistogramData hist_data; + options.statistics->histogramData(DB_WRITE, &hist_data); + ASSERT_GT(hist_data.average, 0.0); + } while (ChangeOptions(kSkipNoSeekToLast)); +} +#endif // OS_WIN + +namespace { +using KVMap = std::map; +} + +class ModelDB : public DB { + public: + class ModelSnapshot : public Snapshot { + public: + KVMap map_; + + SequenceNumber GetSequenceNumber() const override { + // no need to call this + assert(false); + return 0; + } + + int64_t GetUnixTime() const override { + // no need to call this + assert(false); + return 0; + } + + uint64_t GetTimestamp() const override { + // no need to call this + assert(false); + return 0; + } + }; + + explicit ModelDB(const Options& options) : options_(options) {} + using DB::Put; + Status Put(const WriteOptions& o, ColumnFamilyHandle* cf, const Slice& k, + const Slice& v) override { + WriteBatch batch; + Status s = batch.Put(cf, k, v); + if (!s.ok()) { + return s; + } + return Write(o, &batch); + } + Status Put(const WriteOptions& /*o*/, ColumnFamilyHandle* /*cf*/, + const Slice& /*k*/, const Slice& /*ts*/, + const Slice& /*v*/) override { + return Status::NotSupported(); + } + + using DB::PutEntity; + Status PutEntity(const WriteOptions& /* options */, + ColumnFamilyHandle* /* column_family */, + const Slice& /* key */, + const WideColumns& /* columns */) override { + return Status::NotSupported(); + } + + using DB::Close; + Status Close() override { return Status::OK(); } + using DB::Delete; + Status Delete(const WriteOptions& o, ColumnFamilyHandle* cf, + const Slice& key) override { + WriteBatch batch; + Status s = batch.Delete(cf, key); + if (!s.ok()) { + return s; + } + return Write(o, &batch); + } + Status Delete(const WriteOptions& /*o*/, ColumnFamilyHandle* /*cf*/, + const Slice& /*key*/, const Slice& /*ts*/) override { + return Status::NotSupported(); + } + using DB::SingleDelete; + Status SingleDelete(const WriteOptions& o, ColumnFamilyHandle* cf, + const Slice& key) override { + WriteBatch batch; + Status s = batch.SingleDelete(cf, key); + if (!s.ok()) { + return s; + } + return Write(o, &batch); + } + Status SingleDelete(const WriteOptions& /*o*/, ColumnFamilyHandle* /*cf*/, + const Slice& /*key*/, const Slice& /*ts*/) override { + return Status::NotSupported(); + } + using DB::Merge; + Status Merge(const WriteOptions& o, ColumnFamilyHandle* cf, const Slice& k, + const Slice& v) override { + WriteBatch batch; + Status s = batch.Merge(cf, k, v); + if (!s.ok()) { + return s; + } + return Write(o, &batch); + } + Status Merge(const WriteOptions& /*o*/, ColumnFamilyHandle* /*cf*/, + const Slice& /*k*/, const Slice& /*ts*/, + const Slice& /*value*/) override { + return Status::NotSupported(); + } + using DB::Get; + Status Get(const ReadOptions& /*options*/, ColumnFamilyHandle* /*cf*/, + const Slice& key, PinnableSlice* /*value*/) override { + return Status::NotSupported(key); + } + + using DB::GetMergeOperands; + virtual Status GetMergeOperands( + const ReadOptions& /*options*/, ColumnFamilyHandle* /*column_family*/, + const Slice& key, PinnableSlice* /*slice*/, + GetMergeOperandsOptions* /*merge_operands_options*/, + int* /*number_of_operands*/) override { + return Status::NotSupported(key); + } + + using DB::MultiGet; + std::vector MultiGet( + const ReadOptions& /*options*/, + const std::vector& /*column_family*/, + const std::vector& keys, + std::vector* /*values*/) override { + std::vector s(keys.size(), + Status::NotSupported("Not implemented.")); + return s; + } + + using DB::IngestExternalFile; + Status IngestExternalFile( + ColumnFamilyHandle* /*column_family*/, + const std::vector& /*external_files*/, + const IngestExternalFileOptions& /*options*/) override { + return Status::NotSupported("Not implemented."); + } + + using DB::IngestExternalFiles; + Status IngestExternalFiles( + const std::vector& /*args*/) override { + return Status::NotSupported("Not implemented"); + } + + using DB::CreateColumnFamilyWithImport; + virtual Status CreateColumnFamilyWithImport( + const ColumnFamilyOptions& /*options*/, + const std::string& /*column_family_name*/, + const ImportColumnFamilyOptions& /*import_options*/, + const std::vector& /*metadatas*/, + ColumnFamilyHandle** /*handle*/) override { + return Status::NotSupported("Not implemented."); + } + + using DB::VerifyChecksum; + Status VerifyChecksum(const ReadOptions&) override { + return Status::NotSupported("Not implemented."); + } + + using DB::ClipColumnFamily; + virtual Status ClipColumnFamily(ColumnFamilyHandle* /*column_family*/, + const Slice& /*begin*/, + const Slice& /*end*/) override { + return Status::NotSupported("Not implemented."); + } + + using DB::GetPropertiesOfAllTables; + Status GetPropertiesOfAllTables( + ColumnFamilyHandle* /*column_family*/, + TablePropertiesCollection* /*props*/) override { + return Status(); + } + + Status GetPropertiesOfTablesInRange( + ColumnFamilyHandle* /*column_family*/, const Range* /*range*/, + std::size_t /*n*/, TablePropertiesCollection* /*props*/) override { + return Status(); + } + + using DB::KeyMayExist; + bool KeyMayExist(const ReadOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/, const Slice& /*key*/, + std::string* /*value*/, + bool* value_found = nullptr) override { + if (value_found != nullptr) { + *value_found = false; + } + return true; // Not Supported directly + } + using DB::NewIterator; + Iterator* NewIterator(const ReadOptions& options, + ColumnFamilyHandle* /*column_family*/) override { + if (options.snapshot == nullptr) { + KVMap* saved = new KVMap; + *saved = map_; + return new ModelIter(saved, true); + } else { + const KVMap* snapshot_state = + &(reinterpret_cast(options.snapshot)->map_); + return new ModelIter(snapshot_state, false); + } + } + Status NewIterators(const ReadOptions& /*options*/, + const std::vector& /*column_family*/, + std::vector* /*iterators*/) override { + return Status::NotSupported("Not supported yet"); + } + const Snapshot* GetSnapshot() override { + ModelSnapshot* snapshot = new ModelSnapshot; + snapshot->map_ = map_; + return snapshot; + } + + void ReleaseSnapshot(const Snapshot* snapshot) override { + delete reinterpret_cast(snapshot); + } + + Status Write(const WriteOptions& /*options*/, WriteBatch* batch) override { + class Handler : public WriteBatch::Handler { + public: + KVMap* map_; + void Put(const Slice& key, const Slice& value) override { + (*map_)[key.ToString()] = value.ToString(); + } + void Merge(const Slice& /*key*/, const Slice& /*value*/) override { + // ignore merge for now + // (*map_)[key.ToString()] = value.ToString(); + } + void Delete(const Slice& key) override { map_->erase(key.ToString()); } + }; + Handler handler; + handler.map_ = &map_; + return batch->Iterate(&handler); + } + + using DB::GetProperty; + bool GetProperty(ColumnFamilyHandle* /*column_family*/, + const Slice& /*property*/, std::string* /*value*/) override { + return false; + } + using DB::GetIntProperty; + bool GetIntProperty(ColumnFamilyHandle* /*column_family*/, + const Slice& /*property*/, uint64_t* /*value*/) override { + return false; + } + using DB::GetMapProperty; + bool GetMapProperty(ColumnFamilyHandle* /*column_family*/, + const Slice& /*property*/, + std::map* /*value*/) override { + return false; + } + using DB::GetAggregatedIntProperty; + bool GetAggregatedIntProperty(const Slice& /*property*/, + uint64_t* /*value*/) override { + return false; + } + using DB::GetApproximateSizes; + Status GetApproximateSizes(const SizeApproximationOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/, + const Range* /*range*/, int n, + uint64_t* sizes) override { + for (int i = 0; i < n; i++) { + sizes[i] = 0; + } + return Status::OK(); + } + using DB::GetApproximateMemTableStats; + void GetApproximateMemTableStats(ColumnFamilyHandle* /*column_family*/, + const Range& /*range*/, + uint64_t* const count, + uint64_t* const size) override { + *count = 0; + *size = 0; + } + using DB::CompactRange; + Status CompactRange(const CompactRangeOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/, + const Slice* /*start*/, const Slice* /*end*/) override { + return Status::NotSupported("Not supported operation."); + } + + Status SetDBOptions( + const std::unordered_map& /*new_options*/) + override { + return Status::NotSupported("Not supported operation."); + } + + using DB::CompactFiles; + Status CompactFiles( + const CompactionOptions& /*compact_options*/, + ColumnFamilyHandle* /*column_family*/, + const std::vector& /*input_file_names*/, + const int /*output_level*/, const int /*output_path_id*/ = -1, + std::vector* const /*output_file_names*/ = nullptr, + CompactionJobInfo* /*compaction_job_info*/ = nullptr) override { + return Status::NotSupported("Not supported operation."); + } + + Status PauseBackgroundWork() override { + return Status::NotSupported("Not supported operation."); + } + + Status ContinueBackgroundWork() override { + return Status::NotSupported("Not supported operation."); + } + + Status EnableAutoCompaction( + const std::vector& /*column_family_handles*/) + override { + return Status::NotSupported("Not supported operation."); + } + + void EnableManualCompaction() override { return; } + + void DisableManualCompaction() override { return; } + + virtual Status WaitForCompact( + const WaitForCompactOptions& /* wait_for_compact_options */) override { + return Status::OK(); + } + + using DB::NumberLevels; + int NumberLevels(ColumnFamilyHandle* /*column_family*/) override { return 1; } + + using DB::MaxMemCompactionLevel; + int MaxMemCompactionLevel(ColumnFamilyHandle* /*column_family*/) override { + return 1; + } + + using DB::Level0StopWriteTrigger; + int Level0StopWriteTrigger(ColumnFamilyHandle* /*column_family*/) override { + return -1; + } + + const std::string& GetName() const override { return name_; } + + Env* GetEnv() const override { return nullptr; } + + using DB::GetOptions; + Options GetOptions(ColumnFamilyHandle* /*column_family*/) const override { + return options_; + } + + using DB::GetDBOptions; + DBOptions GetDBOptions() const override { return options_; } + + using DB::Flush; + Status Flush(const ROCKSDB_NAMESPACE::FlushOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/) override { + Status ret; + return ret; + } + Status Flush( + const ROCKSDB_NAMESPACE::FlushOptions& /*options*/, + const std::vector& /*column_families*/) override { + return Status::OK(); + } + + Status SyncWAL() override { return Status::OK(); } + + Status DisableFileDeletions() override { return Status::OK(); } + + Status EnableFileDeletions(bool /*force*/) override { return Status::OK(); } + + Status GetLiveFiles(std::vector&, uint64_t* /*size*/, + bool /*flush_memtable*/ = true) override { + return Status::OK(); + } + + Status GetLiveFilesChecksumInfo( + FileChecksumList* /*checksum_list*/) override { + return Status::OK(); + } + + Status GetLiveFilesStorageInfo( + const LiveFilesStorageInfoOptions& /*opts*/, + std::vector* /*files*/) override { + return Status::OK(); + } + + Status GetSortedWalFiles(VectorLogPtr& /*files*/) override { + return Status::OK(); + } + + Status GetCurrentWalFile( + std::unique_ptr* /*current_log_file*/) override { + return Status::OK(); + } + + virtual Status GetCreationTimeOfOldestFile( + uint64_t* /*creation_time*/) override { + return Status::NotSupported(); + } + + Status DeleteFile(std::string /*name*/) override { return Status::OK(); } + + Status GetUpdatesSince( + ROCKSDB_NAMESPACE::SequenceNumber, + std::unique_ptr*, + const TransactionLogIterator::ReadOptions& /*read_options*/ = + TransactionLogIterator::ReadOptions()) override { + return Status::NotSupported("Not supported in Model DB"); + } + + void GetColumnFamilyMetaData(ColumnFamilyHandle* /*column_family*/, + ColumnFamilyMetaData* /*metadata*/) override {} + + Status GetDbIdentity(std::string& /*identity*/) const override { + return Status::OK(); + } + + Status GetDbSessionId(std::string& /*session_id*/) const override { + return Status::OK(); + } + + SequenceNumber GetLatestSequenceNumber() const override { return 0; } + + Status IncreaseFullHistoryTsLow(ColumnFamilyHandle* /*cf*/, + std::string /*ts_low*/) override { + return Status::OK(); + } + + Status GetFullHistoryTsLow(ColumnFamilyHandle* /*cf*/, + std::string* /*ts_low*/) override { + return Status::OK(); + } + + ColumnFamilyHandle* DefaultColumnFamily() const override { return nullptr; } + + private: + class ModelIter : public Iterator { + public: + ModelIter(const KVMap* map, bool owned) + : map_(map), owned_(owned), iter_(map_->end()) {} + ~ModelIter() override { + if (owned_) delete map_; + } + bool Valid() const override { return iter_ != map_->end(); } + void SeekToFirst() override { iter_ = map_->begin(); } + void SeekToLast() override { + if (map_->empty()) { + iter_ = map_->end(); + } else { + iter_ = map_->find(map_->rbegin()->first); + } + } + void Seek(const Slice& k) override { + iter_ = map_->lower_bound(k.ToString()); + } + void SeekForPrev(const Slice& k) override { + iter_ = map_->upper_bound(k.ToString()); + Prev(); + } + void Next() override { ++iter_; } + void Prev() override { + if (iter_ == map_->begin()) { + iter_ = map_->end(); + return; + } + --iter_; + } + + Slice key() const override { return iter_->first; } + Slice value() const override { return iter_->second; } + Status status() const override { return Status::OK(); } + + private: + const KVMap* const map_; + const bool owned_; // Do we own map_ + KVMap::const_iterator iter_; + }; + const Options options_; + KVMap map_; + std::string name_ = ""; +}; + +#if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) +static std::string RandomKey(Random* rnd, int minimum = 0) { + int len; + do { + len = (rnd->OneIn(3) + ? 1 // Short sometimes to encourage collisions + : (rnd->OneIn(100) ? rnd->Skewed(10) : rnd->Uniform(10))); + } while (len < minimum); + return test::RandomKey(rnd, len); +} + +static bool CompareIterators(int step, DB* model, DB* db, + const Snapshot* model_snap, + const Snapshot* db_snap) { + ReadOptions options; + options.snapshot = model_snap; + Iterator* miter = model->NewIterator(options); + options.snapshot = db_snap; + Iterator* dbiter = db->NewIterator(options); + bool ok = true; + int count = 0; + for (miter->SeekToFirst(), dbiter->SeekToFirst(); + ok && miter->Valid() && dbiter->Valid(); miter->Next(), dbiter->Next()) { + count++; + if (miter->key().compare(dbiter->key()) != 0) { + fprintf(stderr, "step %d: Key mismatch: '%s' vs. '%s'\n", step, + EscapeString(miter->key()).c_str(), + EscapeString(dbiter->key()).c_str()); + ok = false; + break; + } + + if (miter->value().compare(dbiter->value()) != 0) { + fprintf(stderr, "step %d: Value mismatch for key '%s': '%s' vs. '%s'\n", + step, EscapeString(miter->key()).c_str(), + EscapeString(miter->value()).c_str(), + EscapeString(dbiter->value()).c_str()); + ok = false; + } + } + + if (ok) { + if (miter->Valid() != dbiter->Valid()) { + fprintf(stderr, "step %d: Mismatch at end of iterators: %d vs. %d\n", + step, miter->Valid(), dbiter->Valid()); + ok = false; + } + } + (void)count; + delete miter; + delete dbiter; + return ok; +} + +class DBTestRandomized : public DBTest, + public ::testing::WithParamInterface { + public: + void SetUp() override { option_config_ = GetParam(); } + + static std::vector GenerateOptionConfigs() { + std::vector option_configs; + // skip cuckoo hash as it does not support snapshot. + for (int option_config = kDefault; option_config < kEnd; ++option_config) { + if (!ShouldSkipOptions(option_config, + kSkipDeletesFilterFirst | kSkipNoSeekToLast)) { + option_configs.push_back(option_config); + } + } + option_configs.push_back(kBlockBasedTableWithIndexRestartInterval); + return option_configs; + } +}; + +INSTANTIATE_TEST_CASE_P( + DBTestRandomized, DBTestRandomized, + ::testing::ValuesIn(DBTestRandomized::GenerateOptionConfigs())); + +TEST_P(DBTestRandomized, Randomized) { + anon::OptionsOverride options_override; + options_override.skip_policy = kSkipNoSnapshot; + Options options = CurrentOptions(options_override); + DestroyAndReopen(options); + + Random rnd(test::RandomSeed() + GetParam()); + ModelDB model(options); + const int N = 10000; + const Snapshot* model_snap = nullptr; + const Snapshot* db_snap = nullptr; + std::string k, v; + for (int step = 0; step < N; step++) { + // TODO(sanjay): Test Get() works + int p = rnd.Uniform(100); + int minimum = 0; + if (option_config_ == kHashSkipList || option_config_ == kHashLinkList || + option_config_ == kPlainTableFirstBytePrefix || + option_config_ == kBlockBasedTableWithWholeKeyHashIndex || + option_config_ == kBlockBasedTableWithPrefixHashIndex) { + minimum = 1; + } + if (p < 45) { // Put + k = RandomKey(&rnd, minimum); + v = rnd.RandomString(rnd.OneIn(20) ? 100 + rnd.Uniform(100) + : rnd.Uniform(8)); + ASSERT_OK(model.Put(WriteOptions(), k, v)); + ASSERT_OK(db_->Put(WriteOptions(), k, v)); + } else if (p < 90) { // Delete + k = RandomKey(&rnd, minimum); + ASSERT_OK(model.Delete(WriteOptions(), k)); + ASSERT_OK(db_->Delete(WriteOptions(), k)); + } else { // Multi-element batch + WriteBatch b; + const int num = rnd.Uniform(8); + for (int i = 0; i < num; i++) { + if (i == 0 || !rnd.OneIn(10)) { + k = RandomKey(&rnd, minimum); + } else { + // Periodically re-use the same key from the previous iter, so + // we have multiple entries in the write batch for the same key + } + if (rnd.OneIn(2)) { + v = rnd.RandomString(rnd.Uniform(10)); + ASSERT_OK(b.Put(k, v)); + } else { + ASSERT_OK(b.Delete(k)); + } + } + ASSERT_OK(model.Write(WriteOptions(), &b)); + ASSERT_OK(db_->Write(WriteOptions(), &b)); + } + + if ((step % 100) == 0) { + // For DB instances that use the hash index + block-based table, the + // iterator will be invalid right when seeking a non-existent key, right + // than return a key that is close to it. + if (option_config_ != kBlockBasedTableWithWholeKeyHashIndex && + option_config_ != kBlockBasedTableWithPrefixHashIndex) { + ASSERT_TRUE(CompareIterators(step, &model, db_, nullptr, nullptr)); + ASSERT_TRUE(CompareIterators(step, &model, db_, model_snap, db_snap)); + } + + // Save a snapshot from each DB this time that we'll use next + // time we compare things, to make sure the current state is + // preserved with the snapshot + if (model_snap != nullptr) model.ReleaseSnapshot(model_snap); + if (db_snap != nullptr) db_->ReleaseSnapshot(db_snap); + + Reopen(options); + ASSERT_TRUE(CompareIterators(step, &model, db_, nullptr, nullptr)); + + model_snap = model.GetSnapshot(); + db_snap = db_->GetSnapshot(); + } + } + if (model_snap != nullptr) model.ReleaseSnapshot(model_snap); + if (db_snap != nullptr) db_->ReleaseSnapshot(db_snap); +} +#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) + +TEST_F(DBTest, BlockBasedTablePrefixIndexTest) { + // create a DB with block prefix index + BlockBasedTableOptions table_options; + Options options = CurrentOptions(); + table_options.index_type = BlockBasedTableOptions::kHashSearch; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.prefix_extractor.reset(NewFixedPrefixTransform(1)); + + Reopen(options); + ASSERT_OK(Put("k1", "v1")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("k2", "v2")); + + // Reopen with different prefix extractor, make sure everything still works. + // RocksDB should just fall back to the binary index. + options.prefix_extractor.reset(NewFixedPrefixTransform(2)); + + Reopen(options); + ASSERT_EQ("v1", Get("k1")); + ASSERT_EQ("v2", Get("k2")); + + // Back to original + ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "fixed:1"}})); + ASSERT_EQ("v1", Get("k1")); + ASSERT_EQ("v2", Get("k2")); + + // Same if there's a problem initally loading prefix transform + options.prefix_extractor.reset(NewFixedPrefixTransform(1)); + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTable::Open::ForceNullTablePrefixExtractor", + [&](void* arg) { *static_cast(arg) = true; }); + SyncPoint::GetInstance()->EnableProcessing(); + Reopen(options); + ASSERT_EQ("v1", Get("k1")); + ASSERT_EQ("v2", Get("k2")); + + // Change again + ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "fixed:2"}})); + ASSERT_EQ("v1", Get("k1")); + ASSERT_EQ("v2", Get("k2")); + SyncPoint::GetInstance()->DisableProcessing(); + + // Reopen with no prefix extractor, make sure everything still works. + // RocksDB should just fall back to the binary index. + table_options.index_type = BlockBasedTableOptions::kBinarySearch; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.prefix_extractor.reset(); + + Reopen(options); + ASSERT_EQ("v1", Get("k1")); + ASSERT_EQ("v2", Get("k2")); +} + +TEST_F(DBTest, BlockBasedTablePrefixHashIndexTest) { + // create a DB with block prefix index + BlockBasedTableOptions table_options; + Options options = CurrentOptions(); + table_options.index_type = BlockBasedTableOptions::kHashSearch; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.prefix_extractor.reset(NewCappedPrefixTransform(2)); + + Reopen(options); + ASSERT_OK(Put("kk1", "v1")); + ASSERT_OK(Put("kk2", "v2")); + ASSERT_OK(Put("kk", "v3")); + ASSERT_OK(Put("k", "v4")); + Flush(); + + ASSERT_EQ("v1", Get("kk1")); + ASSERT_EQ("v2", Get("kk2")); + + ASSERT_EQ("v3", Get("kk")); + ASSERT_EQ("v4", Get("k")); +} + +TEST_F(DBTest, BlockBasedTablePrefixIndexTotalOrderSeek) { + // create a DB with block prefix index + BlockBasedTableOptions table_options; + Options options = CurrentOptions(); + options.max_open_files = 10; + table_options.index_type = BlockBasedTableOptions::kHashSearch; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.prefix_extractor.reset(NewFixedPrefixTransform(1)); + + // RocksDB sanitize max open files to at least 20. Modify it back. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SanitizeOptions::AfterChangeMaxOpenFiles", [&](void* arg) { + int* max_open_files = static_cast(arg); + *max_open_files = 11; + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Reopen(options); + ASSERT_OK(Put("k1", "v1")); + ASSERT_OK(Flush()); + + CompactRangeOptions cro; + cro.change_level = true; + cro.target_level = 1; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + // Force evict tables + dbfull()->TEST_table_cache()->SetCapacity(0); + // Make table cache to keep one entry. + dbfull()->TEST_table_cache()->SetCapacity(1); + + ReadOptions read_options; + read_options.total_order_seek = true; + { + std::unique_ptr iter(db_->NewIterator(read_options)); + iter->Seek("k1"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("k1", iter->key().ToString()); + } + + // After total order seek, prefix index should still be used. + read_options.total_order_seek = false; + { + std::unique_ptr iter(db_->NewIterator(read_options)); + iter->Seek("k1"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("k1", iter->key().ToString()); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBTest, ChecksumTest) { + BlockBasedTableOptions table_options; + Options options = CurrentOptions(); + + table_options.checksum = kCRC32c; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + ASSERT_OK(Put("a", "b")); + ASSERT_OK(Put("c", "d")); + ASSERT_OK(Flush()); // table with crc checksum + + table_options.checksum = kxxHash; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + ASSERT_OK(Put("e", "f")); + ASSERT_OK(Put("g", "h")); + ASSERT_OK(Flush()); // table with xxhash checksum + + table_options.checksum = kCRC32c; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + ASSERT_EQ("b", Get("a")); + ASSERT_EQ("d", Get("c")); + ASSERT_EQ("f", Get("e")); + ASSERT_EQ("h", Get("g")); + + table_options.checksum = kCRC32c; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + ASSERT_EQ("b", Get("a")); + ASSERT_EQ("d", Get("c")); + ASSERT_EQ("f", Get("e")); + ASSERT_EQ("h", Get("g")); +} + +TEST_P(DBTestWithParam, FIFOCompactionTest) { + for (int iter = 0; iter < 2; ++iter) { + // first iteration -- auto compaction + // second iteration -- manual compaction + Options options; + options.compaction_style = kCompactionStyleFIFO; + options.write_buffer_size = 100 << 10; // 100KB + options.arena_block_size = 4096; + options.compaction_options_fifo.max_table_files_size = 500 << 10; // 500KB + options.compression = kNoCompression; + options.create_if_missing = true; + options.max_subcompactions = max_subcompactions_; + if (iter == 1) { + options.disable_auto_compactions = true; + } + options = CurrentOptions(options); + DestroyAndReopen(options); + + Random rnd(301); + for (int i = 0; i < 6; ++i) { + for (int j = 0; j < 110; ++j) { + ASSERT_OK(Put(std::to_string(i * 100 + j), rnd.RandomString(980))); + } + // flush should happen here + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + if (iter == 0) { + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } else { + CompactRangeOptions cro; + cro.exclusive_manual_compaction = exclusive_manual_compaction_; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + } + // only 5 files should survive + ASSERT_EQ(NumTableFilesAtLevel(0), 5); + for (int i = 0; i < 50; ++i) { + // these keys should be deleted in previous compaction + ASSERT_EQ("NOT_FOUND", Get(std::to_string(i))); + } + } +} + +TEST_F(DBTest, FIFOCompactionTestWithCompaction) { + Options options; + options.compaction_style = kCompactionStyleFIFO; + options.write_buffer_size = 20 << 10; // 20K + options.arena_block_size = 4096; + options.compaction_options_fifo.max_table_files_size = 1500 << 10; // 1MB + options.compaction_options_fifo.allow_compaction = true; + options.level0_file_num_compaction_trigger = 6; + options.compression = kNoCompression; + options.create_if_missing = true; + options = CurrentOptions(options); + DestroyAndReopen(options); + + Random rnd(301); + for (int i = 0; i < 60; i++) { + // Generate and flush a file about 20KB. + for (int j = 0; j < 20; j++) { + ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + // It should be compacted to 10 files. + ASSERT_EQ(NumTableFilesAtLevel(0), 10); + + for (int i = 0; i < 60; i++) { + // Generate and flush a file about 20KB. + for (int j = 0; j < 20; j++) { + ASSERT_OK(Put(std::to_string(i * 20 + j + 2000), rnd.RandomString(980))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + + // It should be compacted to no more than 20 files. + ASSERT_GT(NumTableFilesAtLevel(0), 10); + ASSERT_LT(NumTableFilesAtLevel(0), 18); + // Size limit is still guaranteed. + ASSERT_LE(SizeAtLevel(0), + options.compaction_options_fifo.max_table_files_size); +} + +TEST_F(DBTest, FIFOCompactionStyleWithCompactionAndDelete) { + Options options; + options.compaction_style = kCompactionStyleFIFO; + options.write_buffer_size = 20 << 10; // 20K + options.arena_block_size = 4096; + options.compaction_options_fifo.max_table_files_size = 1500 << 10; // 1MB + options.compaction_options_fifo.allow_compaction = true; + options.level0_file_num_compaction_trigger = 3; + options.compression = kNoCompression; + options.create_if_missing = true; + options = CurrentOptions(options); + DestroyAndReopen(options); + + Random rnd(301); + for (int i = 0; i < 3; i++) { + // Each file contains a different key which will be dropped later. + ASSERT_OK(Put("a" + std::to_string(i), rnd.RandomString(500))); + ASSERT_OK(Put("key" + std::to_string(i), "")); + ASSERT_OK(Put("z" + std::to_string(i), rnd.RandomString(500))); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + ASSERT_EQ(NumTableFilesAtLevel(0), 1); + for (int i = 0; i < 3; i++) { + ASSERT_EQ("", Get("key" + std::to_string(i))); + } + for (int i = 0; i < 3; i++) { + // Each file contains a different key which will be dropped later. + ASSERT_OK(Put("a" + std::to_string(i), rnd.RandomString(500))); + ASSERT_OK(Delete("key" + std::to_string(i))); + ASSERT_OK(Put("z" + std::to_string(i), rnd.RandomString(500))); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + ASSERT_EQ(NumTableFilesAtLevel(0), 2); + for (int i = 0; i < 3; i++) { + ASSERT_EQ("NOT_FOUND", Get("key" + std::to_string(i))); + } +} + +// Check that FIFO-with-TTL is not supported with max_open_files != -1. +// Github issue #8014 +TEST_F(DBTest, FIFOCompactionWithTTLAndMaxOpenFilesTest) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleFIFO; + options.create_if_missing = true; + options.ttl = 600; // seconds + + // TTL is not supported with max_open_files != -1. + options.max_open_files = 0; + ASSERT_TRUE(TryReopen(options).IsNotSupported()); + + options.max_open_files = 100; + ASSERT_TRUE(TryReopen(options).IsNotSupported()); + + // TTL is supported with unlimited max_open_files + options.max_open_files = -1; + ASSERT_OK(TryReopen(options)); +} + +// Check that FIFO-with-TTL is supported only with BlockBasedTableFactory. +TEST_F(DBTest, FIFOCompactionWithTTLAndVariousTableFormatsTest) { + Options options; + options.compaction_style = kCompactionStyleFIFO; + options.create_if_missing = true; + options.ttl = 600; // seconds + + options = CurrentOptions(options); + options.table_factory.reset(NewBlockBasedTableFactory()); + ASSERT_OK(TryReopen(options)); + + Destroy(options); + options.table_factory.reset(NewPlainTableFactory()); + ASSERT_TRUE(TryReopen(options).IsNotSupported()); + + Destroy(options); + options.table_factory.reset(NewAdaptiveTableFactory()); + ASSERT_TRUE(TryReopen(options).IsNotSupported()); +} + +TEST_F(DBTest, FIFOCompactionWithTTLTest) { + Options options; + options.compaction_style = kCompactionStyleFIFO; + options.write_buffer_size = 10 << 10; // 10KB + options.arena_block_size = 4096; + options.compression = kNoCompression; + options.create_if_missing = true; + env_->SetMockSleep(); + options.env = env_; + + // Test to make sure that all files with expired ttl are deleted on next + // manual compaction. + { + // NOTE: Presumed unnecessary and removed: resetting mock time in env + + options.compaction_options_fifo.max_table_files_size = 150 << 10; // 150KB + options.compaction_options_fifo.allow_compaction = false; + options.ttl = 1 * 60 * 60; // 1 hour + options = CurrentOptions(options); + DestroyAndReopen(options); + + Random rnd(301); + for (int i = 0; i < 10; i++) { + // Generate and flush a file about 10KB. + for (int j = 0; j < 10; j++) { + ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + ASSERT_EQ(NumTableFilesAtLevel(0), 10); + + // Sleep for 2 hours -- which is much greater than TTL. + env_->MockSleepForSeconds(2 * 60 * 60); + + // Since no flushes and compactions have run, the db should still be in + // the same state even after considerable time has passed. + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumTableFilesAtLevel(0), 10); + + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + } + + // Test to make sure that all files with expired ttl are deleted on next + // automatic compaction. + { + options.compaction_options_fifo.max_table_files_size = 150 << 10; // 150KB + options.compaction_options_fifo.allow_compaction = false; + options.ttl = 1 * 60 * 60; // 1 hour + options = CurrentOptions(options); + DestroyAndReopen(options); + + Random rnd(301); + for (int i = 0; i < 10; i++) { + // Generate and flush a file about 10KB. + for (int j = 0; j < 10; j++) { + ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + ASSERT_EQ(NumTableFilesAtLevel(0), 10); + + // Sleep for 2 hours -- which is much greater than TTL. + env_->MockSleepForSeconds(2 * 60 * 60); + // Just to make sure that we are in the same state even after sleeping. + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumTableFilesAtLevel(0), 10); + + // Create 1 more file to trigger TTL compaction. The old files are dropped. + for (int i = 0; i < 1; i++) { + for (int j = 0; j < 10; j++) { + ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980))); + } + ASSERT_OK(Flush()); + } + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // Only the new 10 files remain. + ASSERT_EQ(NumTableFilesAtLevel(0), 1); + ASSERT_LE(SizeAtLevel(0), + options.compaction_options_fifo.max_table_files_size); + } + + // Test that shows the fall back to size-based FIFO compaction if TTL-based + // deletion doesn't move the total size to be less than max_table_files_size. + { + options.write_buffer_size = 10 << 10; // 10KB + options.compaction_options_fifo.max_table_files_size = 150 << 10; // 150KB + options.compaction_options_fifo.allow_compaction = false; + options.ttl = 1 * 60 * 60; // 1 hour + options = CurrentOptions(options); + DestroyAndReopen(options); + + Random rnd(301); + for (int i = 0; i < 3; i++) { + // Generate and flush a file about 10KB. + for (int j = 0; j < 10; j++) { + ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + ASSERT_EQ(NumTableFilesAtLevel(0), 3); + + // Sleep for 2 hours -- which is much greater than TTL. + env_->MockSleepForSeconds(2 * 60 * 60); + // Just to make sure that we are in the same state even after sleeping. + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumTableFilesAtLevel(0), 3); + + for (int i = 0; i < 5; i++) { + for (int j = 0; j < 140; j++) { + ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + // Size limit is still guaranteed. + ASSERT_LE(SizeAtLevel(0), + options.compaction_options_fifo.max_table_files_size); + } + + // Test with TTL + Intra-L0 compactions. + { + options.compaction_options_fifo.max_table_files_size = 150 << 10; // 150KB + options.compaction_options_fifo.allow_compaction = true; + options.ttl = 1 * 60 * 60; // 1 hour + options.level0_file_num_compaction_trigger = 6; + options = CurrentOptions(options); + DestroyAndReopen(options); + + Random rnd(301); + for (int i = 0; i < 10; i++) { + // Generate and flush a file about 10KB. + for (int j = 0; j < 10; j++) { + ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + // With Intra-L0 compaction, out of 10 files, 6 files will be compacted to 1 + // (due to level0_file_num_compaction_trigger = 6). + // So total files = 1 + remaining 4 = 5. + ASSERT_EQ(NumTableFilesAtLevel(0), 5); + + // Sleep for 2 hours -- which is much greater than TTL. + env_->MockSleepForSeconds(2 * 60 * 60); + // Just to make sure that we are in the same state even after sleeping. + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumTableFilesAtLevel(0), 5); + + // Create 10 more files. The old 5 files are dropped as their ttl expired. + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 10; j++) { + ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + ASSERT_EQ(NumTableFilesAtLevel(0), 5); + ASSERT_LE(SizeAtLevel(0), + options.compaction_options_fifo.max_table_files_size); + } + + // Test with large TTL + Intra-L0 compactions. + // Files dropped based on size, as ttl doesn't kick in. + { + options.write_buffer_size = 20 << 10; // 20K + options.compaction_options_fifo.max_table_files_size = 1500 << 10; // 1.5MB + options.compaction_options_fifo.allow_compaction = true; + options.ttl = 1 * 60 * 60; // 1 hour + options.level0_file_num_compaction_trigger = 6; + options = CurrentOptions(options); + DestroyAndReopen(options); + + Random rnd(301); + for (int i = 0; i < 60; i++) { + // Generate and flush a file about 20KB. + for (int j = 0; j < 20; j++) { + ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + // It should be compacted to 10 files. + ASSERT_EQ(NumTableFilesAtLevel(0), 10); + + for (int i = 0; i < 60; i++) { + // Generate and flush a file about 20KB. + for (int j = 0; j < 20; j++) { + ASSERT_OK( + Put(std::to_string(i * 20 + j + 2000), rnd.RandomString(980))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + + // It should be compacted to no more than 20 files. + ASSERT_GT(NumTableFilesAtLevel(0), 10); + ASSERT_LT(NumTableFilesAtLevel(0), 18); + // Size limit is still guaranteed. + ASSERT_LE(SizeAtLevel(0), + options.compaction_options_fifo.max_table_files_size); + } +} + +/* + * This test is not reliable enough as it heavily depends on disk behavior. + * Disable as it is flaky. + */ +TEST_F(DBTest, DISABLED_RateLimitingTest) { + Options options = CurrentOptions(); + options.write_buffer_size = 1 << 20; // 1MB + options.level0_file_num_compaction_trigger = 2; + options.target_file_size_base = 1 << 20; // 1MB + options.max_bytes_for_level_base = 4 << 20; // 4MB + options.max_bytes_for_level_multiplier = 4; + options.compression = kNoCompression; + options.create_if_missing = true; + options.env = env_; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.IncreaseParallelism(4); + DestroyAndReopen(options); + + WriteOptions wo; + wo.disableWAL = true; + + // # no rate limiting + Random rnd(301); + uint64_t start = env_->NowMicros(); + // Write ~96M data + for (int64_t i = 0; i < (96 << 10); ++i) { + ASSERT_OK(Put(rnd.RandomString(32), rnd.RandomString((1 << 10) + 1), wo)); + } + uint64_t elapsed = env_->NowMicros() - start; + double raw_rate = env_->bytes_written_ * 1000000.0 / elapsed; + uint64_t rate_limiter_drains = + TestGetTickerCount(options, NUMBER_RATE_LIMITER_DRAINS); + ASSERT_EQ(0, rate_limiter_drains); + Close(); + + // # rate limiting with 0.7 x threshold + options.rate_limiter.reset( + NewGenericRateLimiter(static_cast(0.7 * raw_rate))); + env_->bytes_written_ = 0; + DestroyAndReopen(options); + + start = env_->NowMicros(); + // Write ~96M data + for (int64_t i = 0; i < (96 << 10); ++i) { + ASSERT_OK(Put(rnd.RandomString(32), rnd.RandomString((1 << 10) + 1), wo)); + } + rate_limiter_drains = + TestGetTickerCount(options, NUMBER_RATE_LIMITER_DRAINS) - + rate_limiter_drains; + elapsed = env_->NowMicros() - start; + Close(); + ASSERT_EQ(options.rate_limiter->GetTotalBytesThrough(), env_->bytes_written_); + // Most intervals should've been drained (interval time is 100ms, elapsed is + // micros) + ASSERT_GT(rate_limiter_drains, 0); + ASSERT_LE(rate_limiter_drains, elapsed / 100000 + 1); + double ratio = env_->bytes_written_ * 1000000 / elapsed / raw_rate; + fprintf(stderr, "write rate ratio = %.2lf, expected 0.7\n", ratio); + ASSERT_TRUE(ratio < 0.8); + + // # rate limiting with half of the raw_rate + options.rate_limiter.reset( + NewGenericRateLimiter(static_cast(raw_rate / 2))); + env_->bytes_written_ = 0; + DestroyAndReopen(options); + + start = env_->NowMicros(); + // Write ~96M data + for (int64_t i = 0; i < (96 << 10); ++i) { + ASSERT_OK(Put(rnd.RandomString(32), rnd.RandomString((1 << 10) + 1), wo)); + } + elapsed = env_->NowMicros() - start; + rate_limiter_drains = + TestGetTickerCount(options, NUMBER_RATE_LIMITER_DRAINS) - + rate_limiter_drains; + Close(); + ASSERT_EQ(options.rate_limiter->GetTotalBytesThrough(), env_->bytes_written_); + // Most intervals should've been drained (interval time is 100ms, elapsed is + // micros) + ASSERT_GT(rate_limiter_drains, elapsed / 100000 / 2); + ASSERT_LE(rate_limiter_drains, elapsed / 100000 + 1); + ratio = env_->bytes_written_ * 1000000 / elapsed / raw_rate; + fprintf(stderr, "write rate ratio = %.2lf, expected 0.5\n", ratio); + ASSERT_LT(ratio, 0.6); +} + +// This is a mocked customed rate limiter without implementing optional APIs +// (e.g, RateLimiter::GetTotalPendingRequests()) +class MockedRateLimiterWithNoOptionalAPIImpl : public RateLimiter { + public: + MockedRateLimiterWithNoOptionalAPIImpl() {} + + ~MockedRateLimiterWithNoOptionalAPIImpl() override {} + + void SetBytesPerSecond(int64_t bytes_per_second) override { + (void)bytes_per_second; + } + + using RateLimiter::Request; + void Request(const int64_t bytes, const Env::IOPriority pri, + Statistics* stats) override { + (void)bytes; + (void)pri; + (void)stats; + } + + int64_t GetSingleBurstBytes() const override { return 200; } + + int64_t GetTotalBytesThrough( + const Env::IOPriority pri = Env::IO_TOTAL) const override { + (void)pri; + return 0; + } + + int64_t GetTotalRequests( + const Env::IOPriority pri = Env::IO_TOTAL) const override { + (void)pri; + return 0; + } + + int64_t GetBytesPerSecond() const override { return 0; } +}; + +// To test that customed rate limiter not implementing optional APIs (e.g, +// RateLimiter::GetTotalPendingRequests()) works fine with RocksDB basic +// operations (e.g, Put, Get, Flush) +TEST_F(DBTest, CustomedRateLimiterWithNoOptionalAPIImplTest) { + Options options = CurrentOptions(); + options.rate_limiter.reset(new MockedRateLimiterWithNoOptionalAPIImpl()); + DestroyAndReopen(options); + ASSERT_OK(Put("abc", "def")); + ASSERT_EQ(Get("abc"), "def"); + ASSERT_OK(Flush()); + ASSERT_EQ(Get("abc"), "def"); +} + +TEST_F(DBTest, TableOptionsSanitizeTest) { + Options options = CurrentOptions(); + options.create_if_missing = true; + DestroyAndReopen(options); + ASSERT_EQ(db_->GetOptions().allow_mmap_reads, false); + + options.table_factory.reset(NewPlainTableFactory()); + options.prefix_extractor.reset(NewNoopTransform()); + Destroy(options); + ASSERT_TRUE(!TryReopen(options).IsNotSupported()); + + // Test for check of prefix_extractor when hash index is used for + // block-based table + BlockBasedTableOptions to; + to.index_type = BlockBasedTableOptions::kHashSearch; + options = CurrentOptions(); + options.create_if_missing = true; + options.table_factory.reset(NewBlockBasedTableFactory(to)); + ASSERT_TRUE(TryReopen(options).IsInvalidArgument()); + options.prefix_extractor.reset(NewFixedPrefixTransform(1)); + ASSERT_OK(TryReopen(options)); +} + +TEST_F(DBTest, ConcurrentMemtableNotSupported) { + Options options = CurrentOptions(); + options.allow_concurrent_memtable_write = true; + options.soft_pending_compaction_bytes_limit = 0; + options.hard_pending_compaction_bytes_limit = 100; + options.create_if_missing = true; + + DestroyDB(dbname_, options); + options.memtable_factory.reset(NewHashLinkListRepFactory(4, 0, 3, true, 4)); + ASSERT_NOK(TryReopen(options)); + + options.memtable_factory.reset(new SkipListFactory); + ASSERT_OK(TryReopen(options)); + + ColumnFamilyOptions cf_options(options); + cf_options.memtable_factory.reset( + NewHashLinkListRepFactory(4, 0, 3, true, 4)); + ColumnFamilyHandle* handle; + ASSERT_NOK(db_->CreateColumnFamily(cf_options, "name", &handle)); +} + + +TEST_F(DBTest, SanitizeNumThreads) { + for (int attempt = 0; attempt < 2; attempt++) { + const size_t kTotalTasks = 8; + test::SleepingBackgroundTask sleeping_tasks[kTotalTasks]; + + Options options = CurrentOptions(); + if (attempt == 0) { + options.max_background_compactions = 3; + options.max_background_flushes = 2; + } + options.create_if_missing = true; + DestroyAndReopen(options); + + for (size_t i = 0; i < kTotalTasks; i++) { + // Insert 5 tasks to low priority queue and 5 tasks to high priority queue + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, + &sleeping_tasks[i], + (i < 4) ? Env::Priority::LOW : Env::Priority::HIGH); + } + + // Wait until 10s for they are scheduled. + for (int i = 0; i < 10000; i++) { + if (options.env->GetThreadPoolQueueLen(Env::Priority::LOW) <= 1 && + options.env->GetThreadPoolQueueLen(Env::Priority::HIGH) <= 2) { + break; + } + env_->SleepForMicroseconds(1000); + } + + // pool size 3, total task 4. Queue size should be 1. + ASSERT_EQ(1U, options.env->GetThreadPoolQueueLen(Env::Priority::LOW)); + // pool size 2, total task 4. Queue size should be 2. + ASSERT_EQ(2U, options.env->GetThreadPoolQueueLen(Env::Priority::HIGH)); + + for (size_t i = 0; i < kTotalTasks; i++) { + sleeping_tasks[i].WakeUp(); + sleeping_tasks[i].WaitUntilDone(); + } + + ASSERT_OK(Put("abc", "def")); + ASSERT_EQ("def", Get("abc")); + ASSERT_OK(Flush()); + ASSERT_EQ("def", Get("abc")); + } +} + +TEST_F(DBTest, WriteSingleThreadEntry) { + std::vector threads; + dbfull()->TEST_LockMutex(); + auto w = dbfull()->TEST_BeginWrite(); + threads.emplace_back([&] { ASSERT_OK(Put("a", "b")); }); + env_->SleepForMicroseconds(10000); + threads.emplace_back([&] { ASSERT_OK(Flush()); }); + env_->SleepForMicroseconds(10000); + dbfull()->TEST_UnlockMutex(); + dbfull()->TEST_LockMutex(); + dbfull()->TEST_EndWrite(w); + dbfull()->TEST_UnlockMutex(); + + for (auto& t : threads) { + t.join(); + } +} + +TEST_F(DBTest, ConcurrentFlushWAL) { + const size_t cnt = 100; + Options options; + options.env = env_; + WriteOptions wopt; + ReadOptions ropt; + for (bool two_write_queues : {false, true}) { + for (bool manual_wal_flush : {false, true}) { + options.two_write_queues = two_write_queues; + options.manual_wal_flush = manual_wal_flush; + options.create_if_missing = true; + DestroyAndReopen(options); + std::vector threads; + threads.emplace_back([&] { + for (size_t i = 0; i < cnt; i++) { + auto istr = std::to_string(i); + ASSERT_OK(db_->Put(wopt, db_->DefaultColumnFamily(), "a" + istr, + "b" + istr)); + } + }); + if (two_write_queues) { + threads.emplace_back([&] { + for (size_t i = cnt; i < 2 * cnt; i++) { + auto istr = std::to_string(i); + WriteBatch batch(0 /* reserved_bytes */, 0 /* max_bytes */, + wopt.protection_bytes_per_key, + 0 /* default_cf_ts_sz */); + ASSERT_OK(batch.Put("a" + istr, "b" + istr)); + ASSERT_OK( + dbfull()->WriteImpl(wopt, &batch, nullptr, nullptr, 0, true)); + } + }); + } + threads.emplace_back([&] { + for (size_t i = 0; i < cnt * 100; i++) { // FlushWAL is faster than Put + ASSERT_OK(db_->FlushWAL(false)); + } + }); + for (auto& t : threads) { + t.join(); + } + options.create_if_missing = false; + // Recover from the wal and make sure that it is not corrupted + Reopen(options); + for (size_t i = 0; i < cnt; i++) { + PinnableSlice pval; + auto istr = std::to_string(i); + ASSERT_OK( + db_->Get(ropt, db_->DefaultColumnFamily(), "a" + istr, &pval)); + ASSERT_TRUE(pval == ("b" + istr)); + } + } + } +} + +// This test failure will be caught with a probability +TEST_F(DBTest, ManualFlushWalAndWriteRace) { + Options options; + options.env = env_; + options.manual_wal_flush = true; + options.create_if_missing = true; + + DestroyAndReopen(options); + + WriteOptions wopts; + wopts.sync = true; + + port::Thread writeThread([&]() { + for (int i = 0; i < 100; i++) { + auto istr = std::to_string(i); + ASSERT_OK(dbfull()->Put(wopts, "key_" + istr, "value_" + istr)); + } + }); + port::Thread flushThread([&]() { + for (int i = 0; i < 100; i++) { + ASSERT_OK(dbfull()->FlushWAL(false)); + } + }); + + writeThread.join(); + flushThread.join(); + ASSERT_OK(dbfull()->Put(wopts, "foo1", "value1")); + ASSERT_OK(dbfull()->Put(wopts, "foo2", "value2")); + Reopen(options); + ASSERT_EQ("value1", Get("foo1")); + ASSERT_EQ("value2", Get("foo2")); +} + +TEST_F(DBTest, DynamicMemtableOptions) { + const uint64_t k64KB = 1 << 16; + const uint64_t k128KB = 1 << 17; + const uint64_t k5KB = 5 * 1024; + Options options; + options.env = env_; + options.create_if_missing = true; + options.compression = kNoCompression; + options.max_background_compactions = 1; + options.write_buffer_size = k64KB; + options.arena_block_size = 16 * 1024; + options.max_write_buffer_number = 2; + // Don't trigger compact/slowdown/stop + options.level0_file_num_compaction_trigger = 1024; + options.level0_slowdown_writes_trigger = 1024; + options.level0_stop_writes_trigger = 1024; + DestroyAndReopen(options); + + auto gen_l0_kb = [this](int size) { + const int kNumPutsBeforeWaitForFlush = 64; + Random rnd(301); + for (int i = 0; i < size; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(1024))); + + // The following condition prevents a race condition between flush jobs + // acquiring work and this thread filling up multiple memtables. Without + // this, the flush might produce less files than expected because + // multiple memtables are flushed into a single L0 file. This race + // condition affects assertion (A). + if (i % kNumPutsBeforeWaitForFlush == kNumPutsBeforeWaitForFlush - 1) { + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + }; + + // Test write_buffer_size + gen_l0_kb(64); + ASSERT_EQ(NumTableFilesAtLevel(0), 1); + ASSERT_LT(SizeAtLevel(0), k64KB + k5KB); + ASSERT_GT(SizeAtLevel(0), k64KB - k5KB * 2); + + // Clean up L0 + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + + // Increase buffer size + ASSERT_OK(dbfull()->SetOptions({ + {"write_buffer_size", "131072"}, + })); + + // The existing memtable inflated 64KB->128KB when we invoked SetOptions(). + // Write 192KB, we should have a 128KB L0 file and a memtable with 64KB data. + gen_l0_kb(192); + ASSERT_EQ(NumTableFilesAtLevel(0), 1); // (A) + ASSERT_LT(SizeAtLevel(0), k128KB + 2 * k5KB); + ASSERT_GT(SizeAtLevel(0), k128KB - 4 * k5KB); + + // Decrease buffer size below current usage + ASSERT_OK(dbfull()->SetOptions({ + {"write_buffer_size", "65536"}, + })); + // The existing memtable became eligible for flush when we reduced its + // capacity to 64KB. Two keys need to be added to trigger flush: first causes + // memtable to be marked full, second schedules the flush. Then we should have + // a 128KB L0 file, a 64KB L0 file, and a memtable with just one key. + gen_l0_kb(2); + ASSERT_EQ(NumTableFilesAtLevel(0), 2); + ASSERT_LT(SizeAtLevel(0), k128KB + k64KB + 2 * k5KB); + ASSERT_GT(SizeAtLevel(0), k128KB + k64KB - 4 * k5KB); + + // Test max_write_buffer_number + // Block compaction thread, which will also block the flushes because + // max_background_flushes == 0, so flushes are getting executed by the + // compaction thread + env_->SetBackgroundThreads(1, Env::LOW); + test::SleepingBackgroundTask sleeping_task_low; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::LOW); + // Start from scratch and disable compaction/flush. Flush can only happen + // during compaction but trigger is pretty high + options.disable_auto_compactions = true; + DestroyAndReopen(options); + env_->SetBackgroundThreads(0, Env::HIGH); + + // Put until writes are stopped, bounded by 256 puts. We should see stop at + // ~128KB + int count = 0; + Random rnd(301); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::DelayWrite:Wait", + [&](void* /*arg*/) { sleeping_task_low.WakeUp(); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + while (!sleeping_task_low.WokenUp() && count < 256) { + ASSERT_OK(Put(Key(count), rnd.RandomString(1024), WriteOptions())); + count++; + } + ASSERT_GT(static_cast(count), 128 * 0.8); + ASSERT_LT(static_cast(count), 128 * 1.2); + + sleeping_task_low.WaitUntilDone(); + + // Increase + ASSERT_OK(dbfull()->SetOptions({ + {"max_write_buffer_number", "8"}, + })); + // Clean up memtable and L0 + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + sleeping_task_low.Reset(); + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::LOW); + count = 0; + while (!sleeping_task_low.WokenUp() && count < 1024) { + ASSERT_OK(Put(Key(count), rnd.RandomString(1024), WriteOptions())); + count++; + } +// Windows fails this test. Will tune in the future and figure out +// approp number +#ifndef OS_WIN + ASSERT_GT(static_cast(count), 512 * 0.8); + ASSERT_LT(static_cast(count), 512 * 1.2); +#endif + sleeping_task_low.WaitUntilDone(); + + // Decrease + ASSERT_OK(dbfull()->SetOptions({ + {"max_write_buffer_number", "4"}, + })); + // Clean up memtable and L0 + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + sleeping_task_low.Reset(); + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::LOW); + + count = 0; + while (!sleeping_task_low.WokenUp() && count < 1024) { + ASSERT_OK(Put(Key(count), rnd.RandomString(1024), WriteOptions())); + count++; + } +// Windows fails this test. Will tune in the future and figure out +// approp number +#ifndef OS_WIN + ASSERT_GT(static_cast(count), 256 * 0.8); + ASSERT_LT(static_cast(count), 266 * 1.2); +#endif + sleeping_task_low.WaitUntilDone(); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +#ifdef ROCKSDB_USING_THREAD_STATUS +namespace { +void VerifyOperationCount(Env* env, ThreadStatus::OperationType op_type, + int expected_count) { + int op_count = 0; + std::vector thread_list; + ASSERT_OK(env->GetThreadList(&thread_list)); + for (auto thread : thread_list) { + if (thread.operation_type == op_type) { + op_count++; + } + } + ASSERT_EQ(op_count, expected_count); +} +} // anonymous namespace + +TEST_F(DBTest, GetThreadStatus) { + Options options; + options.env = env_; + options.enable_thread_tracking = true; + TryReopen(options); + + std::vector thread_list; + Status s = env_->GetThreadList(&thread_list); + + for (int i = 0; i < 2; ++i) { + // repeat the test with differet number of high / low priority threads + const int kTestCount = 3; + const unsigned int kHighPriCounts[kTestCount] = {3, 2, 5}; + const unsigned int kLowPriCounts[kTestCount] = {10, 15, 3}; + const unsigned int kBottomPriCounts[kTestCount] = {2, 1, 4}; + for (int test = 0; test < kTestCount; ++test) { + // Change the number of threads in high / low priority pool. + env_->SetBackgroundThreads(kHighPriCounts[test], Env::HIGH); + env_->SetBackgroundThreads(kLowPriCounts[test], Env::LOW); + env_->SetBackgroundThreads(kBottomPriCounts[test], Env::BOTTOM); + // Wait to ensure the all threads has been registered + unsigned int thread_type_counts[ThreadStatus::NUM_THREAD_TYPES]; + // TODO(ajkr): it'd be better if SetBackgroundThreads returned only after + // all threads have been registered. + // Try up to 60 seconds. + for (int num_try = 0; num_try < 60000; num_try++) { + env_->SleepForMicroseconds(1000); + thread_list.clear(); + s = env_->GetThreadList(&thread_list); + ASSERT_OK(s); + memset(thread_type_counts, 0, sizeof(thread_type_counts)); + for (auto thread : thread_list) { + ASSERT_LT(thread.thread_type, ThreadStatus::NUM_THREAD_TYPES); + thread_type_counts[thread.thread_type]++; + } + if (thread_type_counts[ThreadStatus::HIGH_PRIORITY] == + kHighPriCounts[test] && + thread_type_counts[ThreadStatus::LOW_PRIORITY] == + kLowPriCounts[test] && + thread_type_counts[ThreadStatus::BOTTOM_PRIORITY] == + kBottomPriCounts[test]) { + break; + } + } + // Verify the number of high-priority threads + ASSERT_EQ(thread_type_counts[ThreadStatus::HIGH_PRIORITY], + kHighPriCounts[test]); + // Verify the number of low-priority threads + ASSERT_EQ(thread_type_counts[ThreadStatus::LOW_PRIORITY], + kLowPriCounts[test]); + // Verify the number of bottom-priority threads + ASSERT_EQ(thread_type_counts[ThreadStatus::BOTTOM_PRIORITY], + kBottomPriCounts[test]); + } + if (i == 0) { + // repeat the test with multiple column families + CreateAndReopenWithCF({"pikachu", "about-to-remove"}, options); + env_->GetThreadStatusUpdater()->TEST_VerifyColumnFamilyInfoMap(handles_, + true); + } + } + ASSERT_OK(db_->DropColumnFamily(handles_[2])); + delete handles_[2]; + handles_.erase(handles_.begin() + 2); + env_->GetThreadStatusUpdater()->TEST_VerifyColumnFamilyInfoMap(handles_, + true); + Close(); + env_->GetThreadStatusUpdater()->TEST_VerifyColumnFamilyInfoMap(handles_, + true); +} + +TEST_F(DBTest, DisableThreadStatus) { + Options options; + options.env = env_; + options.enable_thread_tracking = false; + TryReopen(options); + CreateAndReopenWithCF({"pikachu", "about-to-remove"}, options); + // Verify non of the column family info exists + env_->GetThreadStatusUpdater()->TEST_VerifyColumnFamilyInfoMap(handles_, + false); +} + +TEST_F(DBTest, ThreadStatusFlush) { + Options options; + options.env = env_; + options.write_buffer_size = 100000; // Small write buffer + options.enable_thread_tracking = true; + options = CurrentOptions(options); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"FlushJob::FlushJob()", "DBTest::ThreadStatusFlush:1"}, + {"DBTest::ThreadStatusFlush:2", "FlushJob::WriteLevel0Table"}, + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + CreateAndReopenWithCF({"pikachu"}, options); + VerifyOperationCount(env_, ThreadStatus::OP_FLUSH, 0); + + ASSERT_OK(Put(1, "foo", "v1")); + ASSERT_EQ("v1", Get(1, "foo")); + VerifyOperationCount(env_, ThreadStatus::OP_FLUSH, 0); + + uint64_t num_running_flushes = 0; + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kNumRunningFlushes, + &num_running_flushes)); + ASSERT_EQ(num_running_flushes, 0); + + ASSERT_OK(Put(1, "k1", std::string(100000, 'x'))); // Fill memtable + ASSERT_OK(Put(1, "k2", std::string(100000, 'y'))); // Trigger flush + + // The first sync point is to make sure there's one flush job + // running when we perform VerifyOperationCount(). + TEST_SYNC_POINT("DBTest::ThreadStatusFlush:1"); + VerifyOperationCount(env_, ThreadStatus::OP_FLUSH, 1); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kNumRunningFlushes, + &num_running_flushes)); + ASSERT_EQ(num_running_flushes, 1); + // This second sync point is to ensure the flush job will not + // be completed until we already perform VerifyOperationCount(). + TEST_SYNC_POINT("DBTest::ThreadStatusFlush:2"); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_P(DBTestWithParam, ThreadStatusSingleCompaction) { + const int kTestKeySize = 16; + const int kTestValueSize = 984; + const int kEntrySize = kTestKeySize + kTestValueSize; + const int kEntriesPerBuffer = 100; + Options options; + options.create_if_missing = true; + options.write_buffer_size = kEntrySize * kEntriesPerBuffer; + options.compaction_style = kCompactionStyleLevel; + options.target_file_size_base = options.write_buffer_size; + options.max_bytes_for_level_base = options.target_file_size_base * 2; + options.max_bytes_for_level_multiplier = 2; + options.compression = kNoCompression; + options = CurrentOptions(options); + options.env = env_; + options.enable_thread_tracking = true; + const int kNumL0Files = 4; + options.level0_file_num_compaction_trigger = kNumL0Files; + options.max_subcompactions = max_subcompactions_; + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"DBTest::ThreadStatusSingleCompaction:0", "DBImpl::BGWorkCompaction"}, + {"CompactionJob::Run():Start", "DBTest::ThreadStatusSingleCompaction:1"}, + {"DBTest::ThreadStatusSingleCompaction:2", "CompactionJob::Run():End"}, + }); + for (int tests = 0; tests < 2; ++tests) { + DestroyAndReopen(options); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Random rnd(301); + // The Put Phase. + for (int file = 0; file < kNumL0Files; ++file) { + for (int key = 0; key < kEntriesPerBuffer; ++key) { + ASSERT_OK(Put(std::to_string(key + file * kEntriesPerBuffer), + rnd.RandomString(kTestValueSize))); + } + ASSERT_OK(Flush()); + } + // This makes sure a compaction won't be scheduled until + // we have done with the above Put Phase. + uint64_t num_running_compactions = 0; + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kNumRunningCompactions, + &num_running_compactions)); + ASSERT_EQ(num_running_compactions, 0); + TEST_SYNC_POINT("DBTest::ThreadStatusSingleCompaction:0"); + ASSERT_GE(NumTableFilesAtLevel(0), + options.level0_file_num_compaction_trigger); + + // This makes sure at least one compaction is running. + TEST_SYNC_POINT("DBTest::ThreadStatusSingleCompaction:1"); + + if (options.enable_thread_tracking) { + // expecting one single L0 to L1 compaction + VerifyOperationCount(env_, ThreadStatus::OP_COMPACTION, 1); + } else { + // If thread tracking is not enabled, compaction count should be 0. + VerifyOperationCount(env_, ThreadStatus::OP_COMPACTION, 0); + } + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kNumRunningCompactions, + &num_running_compactions)); + ASSERT_EQ(num_running_compactions, 1); + // TODO(yhchiang): adding assert to verify each compaction stage. + TEST_SYNC_POINT("DBTest::ThreadStatusSingleCompaction:2"); + + // repeat the test with disabling thread tracking. + options.enable_thread_tracking = false; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + } +} + +TEST_P(DBTestWithParam, PreShutdownManualCompaction) { + Options options = CurrentOptions(); + options.max_subcompactions = max_subcompactions_; + CreateAndReopenWithCF({"pikachu"}, options); + + // iter - 0 with 7 levels + // iter - 1 with 3 levels + for (int iter = 0; iter < 2; ++iter) { + MakeTables(3, "p", "q", 1); + ASSERT_EQ("1,1,1", FilesPerLevel(1)); + + // Compaction range falls before files + Compact(1, "", "c"); + ASSERT_EQ("1,1,1", FilesPerLevel(1)); + + // Compaction range falls after files + Compact(1, "r", "z"); + ASSERT_EQ("1,1,1", FilesPerLevel(1)); + + // Compaction range overlaps files + Compact(1, "p", "q"); + ASSERT_EQ("0,0,1", FilesPerLevel(1)); + + // Populate a different range + MakeTables(3, "c", "e", 1); + ASSERT_EQ("1,1,2", FilesPerLevel(1)); + + // Compact just the new range + Compact(1, "b", "f"); + ASSERT_EQ("0,0,2", FilesPerLevel(1)); + + // Compact all + MakeTables(1, "a", "z", 1); + ASSERT_EQ("1,0,2", FilesPerLevel(1)); + CancelAllBackgroundWork(db_); + ASSERT_TRUE( + db_->CompactRange(CompactRangeOptions(), handles_[1], nullptr, nullptr) + .IsShutdownInProgress()); + ASSERT_EQ("1,0,2", FilesPerLevel(1)); + + if (iter == 0) { + options = CurrentOptions(); + options.num_levels = 3; + options.create_if_missing = true; + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + } + } +} + +TEST_F(DBTest, PreShutdownFlush) { + Options options = CurrentOptions(); + CreateAndReopenWithCF({"pikachu"}, options); + ASSERT_OK(Put(1, "key", "value")); + CancelAllBackgroundWork(db_); + Status s = + db_->CompactRange(CompactRangeOptions(), handles_[1], nullptr, nullptr); + ASSERT_TRUE(s.IsShutdownInProgress()); +} + +TEST_P(DBTestWithParam, PreShutdownMultipleCompaction) { + const int kTestKeySize = 16; + const int kTestValueSize = 984; + const int kEntrySize = kTestKeySize + kTestValueSize; + const int kEntriesPerBuffer = 40; + const int kNumL0Files = 4; + + const int kHighPriCount = 3; + const int kLowPriCount = 5; + env_->SetBackgroundThreads(kHighPriCount, Env::HIGH); + env_->SetBackgroundThreads(kLowPriCount, Env::LOW); + + Options options; + options.create_if_missing = true; + options.write_buffer_size = kEntrySize * kEntriesPerBuffer; + options.compaction_style = kCompactionStyleLevel; + options.target_file_size_base = options.write_buffer_size; + options.max_bytes_for_level_base = + options.target_file_size_base * kNumL0Files; + options.compression = kNoCompression; + options = CurrentOptions(options); + options.env = env_; + options.enable_thread_tracking = true; + options.level0_file_num_compaction_trigger = kNumL0Files; + options.max_bytes_for_level_multiplier = 2; + options.max_background_compactions = kLowPriCount; + options.level0_stop_writes_trigger = 1 << 10; + options.level0_slowdown_writes_trigger = 1 << 10; + options.max_subcompactions = max_subcompactions_; + + TryReopen(options); + Random rnd(301); + + std::vector thread_list; + // Delay both flush and compaction + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"FlushJob::FlushJob()", "CompactionJob::Run():Start"}, + {"CompactionJob::Run():Start", + "DBTest::PreShutdownMultipleCompaction:Preshutdown"}, + {"CompactionJob::Run():Start", + "DBTest::PreShutdownMultipleCompaction:VerifyCompaction"}, + {"DBTest::PreShutdownMultipleCompaction:Preshutdown", + "CompactionJob::Run():End"}, + {"CompactionJob::Run():End", + "DBTest::PreShutdownMultipleCompaction:VerifyPreshutdown"}}); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Make rocksdb busy + int key = 0; + // check how many threads are doing compaction using GetThreadList + int operation_count[ThreadStatus::NUM_OP_TYPES] = {0}; + for (int file = 0; file < 16 * kNumL0Files; ++file) { + for (int k = 0; k < kEntriesPerBuffer; ++k) { + ASSERT_OK(Put(std::to_string(key++), rnd.RandomString(kTestValueSize))); + } + + ASSERT_OK(env_->GetThreadList(&thread_list)); + for (auto thread : thread_list) { + operation_count[thread.operation_type]++; + } + + // Speed up the test + if (operation_count[ThreadStatus::OP_FLUSH] > 1 && + operation_count[ThreadStatus::OP_COMPACTION] > + 0.6 * options.max_background_compactions) { + break; + } + if (file == 15 * kNumL0Files) { + TEST_SYNC_POINT("DBTest::PreShutdownMultipleCompaction:Preshutdown"); + } + } + + TEST_SYNC_POINT("DBTest::PreShutdownMultipleCompaction:Preshutdown"); + ASSERT_GE(operation_count[ThreadStatus::OP_COMPACTION], 1); + CancelAllBackgroundWork(db_); + TEST_SYNC_POINT("DBTest::PreShutdownMultipleCompaction:VerifyPreshutdown"); + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + // Record the number of compactions at a time. + for (int i = 0; i < ThreadStatus::NUM_OP_TYPES; ++i) { + operation_count[i] = 0; + } + ASSERT_OK(env_->GetThreadList(&thread_list)); + for (auto thread : thread_list) { + operation_count[thread.operation_type]++; + } + ASSERT_EQ(operation_count[ThreadStatus::OP_COMPACTION], 0); +} + +TEST_P(DBTestWithParam, PreShutdownCompactionMiddle) { + const int kTestKeySize = 16; + const int kTestValueSize = 984; + const int kEntrySize = kTestKeySize + kTestValueSize; + const int kEntriesPerBuffer = 40; + const int kNumL0Files = 4; + + const int kHighPriCount = 3; + const int kLowPriCount = 5; + env_->SetBackgroundThreads(kHighPriCount, Env::HIGH); + env_->SetBackgroundThreads(kLowPriCount, Env::LOW); + + Options options; + options.create_if_missing = true; + options.write_buffer_size = kEntrySize * kEntriesPerBuffer; + options.compaction_style = kCompactionStyleLevel; + options.target_file_size_base = options.write_buffer_size; + options.max_bytes_for_level_base = + options.target_file_size_base * kNumL0Files; + options.compression = kNoCompression; + options = CurrentOptions(options); + options.env = env_; + options.enable_thread_tracking = true; + options.level0_file_num_compaction_trigger = kNumL0Files; + options.max_bytes_for_level_multiplier = 2; + options.max_background_compactions = kLowPriCount; + options.level0_stop_writes_trigger = 1 << 10; + options.level0_slowdown_writes_trigger = 1 << 10; + options.max_subcompactions = max_subcompactions_; + + TryReopen(options); + Random rnd(301); + + std::vector thread_list; + // Delay both flush and compaction + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBTest::PreShutdownCompactionMiddle:Preshutdown", + "CompactionJob::Run():Inprogress"}, + {"CompactionJob::Run():Start", + "DBTest::PreShutdownCompactionMiddle:VerifyCompaction"}, + {"CompactionJob::Run():Inprogress", "CompactionJob::Run():End"}, + {"CompactionJob::Run():End", + "DBTest::PreShutdownCompactionMiddle:VerifyPreshutdown"}}); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Make rocksdb busy + int key = 0; + // check how many threads are doing compaction using GetThreadList + int operation_count[ThreadStatus::NUM_OP_TYPES] = {0}; + for (int file = 0; file < 16 * kNumL0Files; ++file) { + for (int k = 0; k < kEntriesPerBuffer; ++k) { + ASSERT_OK(Put(std::to_string(key++), rnd.RandomString(kTestValueSize))); + } + + ASSERT_OK(env_->GetThreadList(&thread_list)); + for (auto thread : thread_list) { + operation_count[thread.operation_type]++; + } + + // Speed up the test + if (operation_count[ThreadStatus::OP_FLUSH] > 1 && + operation_count[ThreadStatus::OP_COMPACTION] > + 0.6 * options.max_background_compactions) { + break; + } + if (file == 15 * kNumL0Files) { + TEST_SYNC_POINT("DBTest::PreShutdownCompactionMiddle:VerifyCompaction"); + } + } + + ASSERT_GE(operation_count[ThreadStatus::OP_COMPACTION], 1); + CancelAllBackgroundWork(db_); + TEST_SYNC_POINT("DBTest::PreShutdownCompactionMiddle:Preshutdown"); + TEST_SYNC_POINT("DBTest::PreShutdownCompactionMiddle:VerifyPreshutdown"); + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + // Record the number of compactions at a time. + for (int i = 0; i < ThreadStatus::NUM_OP_TYPES; ++i) { + operation_count[i] = 0; + } + ASSERT_OK(env_->GetThreadList(&thread_list)); + for (auto thread : thread_list) { + operation_count[thread.operation_type]++; + } + ASSERT_EQ(operation_count[ThreadStatus::OP_COMPACTION], 0); +} + +#endif // ROCKSDB_USING_THREAD_STATUS + +TEST_F(DBTest, FlushOnDestroy) { + WriteOptions wo; + wo.disableWAL = true; + ASSERT_OK(Put("foo", "v1", wo)); + CancelAllBackgroundWork(db_); +} + +TEST_F(DBTest, DynamicLevelCompressionPerLevel) { + if (!Snappy_Supported()) { + return; + } + const int kNKeys = 120; + int keys[kNKeys]; + for (int i = 0; i < kNKeys; i++) { + keys[i] = i; + } + RandomShuffle(std::begin(keys), std::end(keys)); + + Random rnd(301); + Options options; + options.env = env_; + options.create_if_missing = true; + options.db_write_buffer_size = 20480; + options.write_buffer_size = 20480; + options.max_write_buffer_number = 2; + options.level0_file_num_compaction_trigger = 2; + options.level0_slowdown_writes_trigger = 2; + options.level0_stop_writes_trigger = 2; + options.target_file_size_base = 20480; + options.level_compaction_dynamic_level_bytes = true; + options.max_bytes_for_level_base = 102400; + options.max_bytes_for_level_multiplier = 4; + options.max_background_compactions = 1; + options.num_levels = 5; + + options.compression_per_level.resize(3); + options.compression_per_level[0] = kNoCompression; + options.compression_per_level[1] = kNoCompression; + options.compression_per_level[2] = kSnappyCompression; + + OnFileDeletionListener* listener = new OnFileDeletionListener(); + options.listeners.emplace_back(listener); + + DestroyAndReopen(options); + + // Insert more than 80K. L4 should be base level. Neither L0 nor L4 should + // be compressed, so total data size should be more than 80K. + for (int i = 0; i < 20; i++) { + ASSERT_OK(Put(Key(keys[i]), CompressibleString(&rnd, 4000))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ(NumTableFilesAtLevel(1), 0); + ASSERT_EQ(NumTableFilesAtLevel(2), 0); + ASSERT_EQ(NumTableFilesAtLevel(3), 0); + // Assuming each files' metadata is at least 50 bytes/ + ASSERT_GT(SizeAtLevel(0) + SizeAtLevel(4), 20U * 4000U + 50U * 4); + + // Insert 400KB. Some data will be compressed + for (int i = 21; i < 120; i++) { + ASSERT_OK(Put(Key(keys[i]), CompressibleString(&rnd, 4000))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumTableFilesAtLevel(1), 0); + ASSERT_EQ(NumTableFilesAtLevel(2), 0); + + ASSERT_LT(SizeAtLevel(0) + SizeAtLevel(3) + SizeAtLevel(4), + 120U * 4000U + 50U * 24); + // Make sure data in files in L3 is not compacted by removing all files + // in L4 and calculate number of rows + ASSERT_OK(dbfull()->SetOptions({ + {"disable_auto_compactions", "true"}, + })); + ColumnFamilyMetaData cf_meta; + db_->GetColumnFamilyMetaData(&cf_meta); + for (auto file : cf_meta.levels[4].files) { + listener->SetExpectedFileName(dbname_ + file.name); + ASSERT_OK(dbfull()->DeleteFile(file.name)); + } + listener->VerifyMatchedCount(cf_meta.levels[4].files.size()); + + int num_keys = 0; + std::unique_ptr iter(db_->NewIterator(ReadOptions())); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + num_keys++; + } + ASSERT_OK(iter->status()); + ASSERT_GT(SizeAtLevel(0) + SizeAtLevel(3), num_keys * 4000U + num_keys * 10U); +} + +TEST_F(DBTest, DynamicLevelCompressionPerLevel2) { + if (!Snappy_Supported() || !LZ4_Supported() || !Zlib_Supported()) { + return; + } + const int kNKeys = 500; + int keys[kNKeys]; + for (int i = 0; i < kNKeys; i++) { + keys[i] = i; + } + RandomShuffle(std::begin(keys), std::end(keys)); + + Random rnd(301); + Options options; + options.create_if_missing = true; + options.db_write_buffer_size = 6000000; + options.write_buffer_size = 600000; + options.max_write_buffer_number = 2; + options.level0_file_num_compaction_trigger = 2; + options.level0_slowdown_writes_trigger = 2; + options.level0_stop_writes_trigger = 2; + options.soft_pending_compaction_bytes_limit = 1024 * 1024; + options.target_file_size_base = 20; + options.env = env_; + options.level_compaction_dynamic_level_bytes = true; + options.max_bytes_for_level_base = 200; + options.max_bytes_for_level_multiplier = 8; + options.max_background_compactions = 1; + options.num_levels = 5; + std::shared_ptr mtf(new mock::MockTableFactory); + options.table_factory = mtf; + + options.compression_per_level.resize(3); + options.compression_per_level[0] = kNoCompression; + options.compression_per_level[1] = kLZ4Compression; + options.compression_per_level[2] = kZlibCompression; + + DestroyAndReopen(options); + // When base level is L4, L4 is LZ4. + std::atomic num_zlib(0); + std::atomic num_lz4(0); + std::atomic num_no(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) { + Compaction* compaction = reinterpret_cast(arg); + if (compaction->output_level() == 4) { + ASSERT_TRUE(compaction->output_compression() == kLZ4Compression); + num_lz4.fetch_add(1); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "FlushJob::WriteLevel0Table:output_compression", [&](void* arg) { + auto* compression = reinterpret_cast(arg); + ASSERT_TRUE(*compression == kNoCompression); + num_no.fetch_add(1); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + for (int i = 0; i < 100; i++) { + std::string value = rnd.RandomString(200); + ASSERT_OK(Put(Key(keys[i]), value)); + if (i % 25 == 24) { + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + } + + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + + ASSERT_EQ(NumTableFilesAtLevel(1), 0); + ASSERT_EQ(NumTableFilesAtLevel(2), 0); + ASSERT_EQ(NumTableFilesAtLevel(3), 0); + ASSERT_GT(NumTableFilesAtLevel(4), 0); + ASSERT_GT(num_no.load(), 2); + ASSERT_GT(num_lz4.load(), 0); + int prev_num_files_l4 = NumTableFilesAtLevel(4); + + // After base level turn L4->L3, L3 becomes LZ4 and L4 becomes Zlib + num_lz4.store(0); + num_no.store(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) { + Compaction* compaction = reinterpret_cast(arg); + if (compaction->output_level() == 4 && compaction->start_level() == 3) { + ASSERT_TRUE(compaction->output_compression() == kZlibCompression); + num_zlib.fetch_add(1); + } else { + ASSERT_TRUE(compaction->output_compression() == kLZ4Compression); + num_lz4.fetch_add(1); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "FlushJob::WriteLevel0Table:output_compression", [&](void* arg) { + auto* compression = reinterpret_cast(arg); + ASSERT_TRUE(*compression == kNoCompression); + num_no.fetch_add(1); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + for (int i = 101; i < 500; i++) { + std::string value = rnd.RandomString(200); + ASSERT_OK(Put(Key(keys[i]), value)); + if (i % 100 == 99) { + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + } + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ASSERT_EQ(NumTableFilesAtLevel(1), 0); + ASSERT_EQ(NumTableFilesAtLevel(2), 0); + ASSERT_GT(NumTableFilesAtLevel(3), 0); + ASSERT_GT(NumTableFilesAtLevel(4), prev_num_files_l4); + ASSERT_GT(num_no.load(), 2); + ASSERT_GT(num_lz4.load(), 0); + ASSERT_GT(num_zlib.load(), 0); +} + +TEST_F(DBTest, DynamicCompactionOptions) { + // minimum write buffer size is enforced at 64KB + const uint64_t k32KB = 1 << 15; + const uint64_t k64KB = 1 << 16; + const uint64_t k128KB = 1 << 17; + const uint64_t k1MB = 1 << 20; + const uint64_t k4KB = 1 << 12; + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.env = env_; + options.create_if_missing = true; + options.compression = kNoCompression; + options.soft_pending_compaction_bytes_limit = 1024 * 1024; + options.write_buffer_size = k64KB; + options.arena_block_size = 4 * k4KB; + options.max_write_buffer_number = 2; + // Compaction related options + options.level0_file_num_compaction_trigger = 3; + options.level0_slowdown_writes_trigger = 4; + options.level0_stop_writes_trigger = 8; + options.target_file_size_base = k64KB; + options.max_compaction_bytes = options.target_file_size_base * 10; + options.target_file_size_multiplier = 1; + options.max_bytes_for_level_base = k128KB; + options.max_bytes_for_level_multiplier = 4; + + // Block flush thread and disable compaction thread + env_->SetBackgroundThreads(1, Env::LOW); + env_->SetBackgroundThreads(1, Env::HIGH); + DestroyAndReopen(options); + + auto gen_l0_kb = [this](int start, int size, int stride) { + Random rnd(301); + for (int i = 0; i < size; i++) { + ASSERT_OK(Put(Key(start + stride * i), rnd.RandomString(1024))); + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + }; + + // Write 3 files that have the same key range. + // Since level0_file_num_compaction_trigger is 3, compaction should be + // triggered. The compaction should result in one L1 file + gen_l0_kb(0, 64, 1); + ASSERT_EQ(NumTableFilesAtLevel(0), 1); + gen_l0_kb(0, 64, 1); + ASSERT_EQ(NumTableFilesAtLevel(0), 2); + gen_l0_kb(0, 64, 1); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,1", FilesPerLevel()); + std::vector metadata; + db_->GetLiveFilesMetaData(&metadata); + ASSERT_EQ(1U, metadata.size()); + ASSERT_LE(metadata[0].size, k64KB + k4KB); + ASSERT_GE(metadata[0].size, k64KB - k4KB); + + // Test compaction trigger and target_file_size_base + // Reduce compaction trigger to 2, and reduce L1 file size to 32KB. + // Writing to 64KB L0 files should trigger a compaction. Since these + // 2 L0 files have the same key range, compaction merge them and should + // result in 2 32KB L1 files. + ASSERT_OK( + dbfull()->SetOptions({{"level0_file_num_compaction_trigger", "2"}, + {"target_file_size_base", std::to_string(k32KB)}})); + + gen_l0_kb(0, 64, 1); + ASSERT_EQ("1,1", FilesPerLevel()); + gen_l0_kb(0, 64, 1); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,2", FilesPerLevel()); + metadata.clear(); + db_->GetLiveFilesMetaData(&metadata); + ASSERT_EQ(2U, metadata.size()); + ASSERT_LE(metadata[0].size, k32KB + k4KB); + ASSERT_GE(metadata[0].size, k32KB - k4KB); + ASSERT_LE(metadata[1].size, k32KB + k4KB); + ASSERT_GE(metadata[1].size, k32KB - k4KB); + + // Test max_bytes_for_level_base + // Increase level base size to 256KB and write enough data that will + // fill L1 and L2. L1 size should be around 256KB while L2 size should be + // around 256KB x 4. + ASSERT_OK(dbfull()->SetOptions( + {{"max_bytes_for_level_base", std::to_string(k1MB)}})); + + // writing 96 x 64KB => 6 * 1024KB + // (L1 + L2) = (1 + 4) * 1024KB + for (int i = 0; i < 96; ++i) { + gen_l0_kb(i, 64, 96); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_GT(SizeAtLevel(1), k1MB / 2); + ASSERT_LT(SizeAtLevel(1), k1MB + k1MB / 2); + + // Within (0.5, 1.5) of 4MB. + ASSERT_GT(SizeAtLevel(2), 2 * k1MB); + ASSERT_LT(SizeAtLevel(2), 6 * k1MB); + + // Test max_bytes_for_level_multiplier and + // max_bytes_for_level_base. Now, reduce both mulitplier and level base, + // After filling enough data that can fit in L1 - L3, we should see L1 size + // reduces to 128KB from 256KB which was asserted previously. Same for L2. + ASSERT_OK(dbfull()->SetOptions( + {{"max_bytes_for_level_multiplier", "2"}, + {"max_bytes_for_level_base", std::to_string(k128KB)}})); + + // writing 20 x 64KB = 10 x 128KB + // (L1 + L2 + L3) = (1 + 2 + 4) * 128KB + for (int i = 0; i < 20; ++i) { + gen_l0_kb(i, 64, 32); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + uint64_t total_size = SizeAtLevel(1) + SizeAtLevel(2) + SizeAtLevel(3); + ASSERT_TRUE(total_size < k128KB * 7 * 1.5); + + // Test level0_stop_writes_trigger. + // Clean up memtable and L0. Block compaction threads. If continue to write + // and flush memtables. We should see put stop after 8 memtable flushes + // since level0_stop_writes_trigger = 8 + ASSERT_OK(dbfull()->TEST_FlushMemTable(true, true)); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + // Block compaction + test::SleepingBackgroundTask sleeping_task_low; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::LOW); + sleeping_task_low.WaitUntilSleeping(); + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + int count = 0; + Random rnd(301); + WriteOptions wo; + while (count < 64) { + ASSERT_OK(Put(Key(count), rnd.RandomString(1024), wo)); + ASSERT_OK(dbfull()->TEST_FlushMemTable(true, true)); + count++; + if (dbfull()->TEST_write_controler().IsStopped()) { + sleeping_task_low.WakeUp(); + break; + } + } + // Stop trigger = 8 + ASSERT_EQ(count, 8); + // Unblock + sleeping_task_low.WaitUntilDone(); + + // Now reduce level0_stop_writes_trigger to 6. Clear up memtables and L0. + // Block compaction thread again. Perform the put and memtable flushes + // until we see the stop after 6 memtable flushes. + ASSERT_OK(dbfull()->SetOptions({{"level0_stop_writes_trigger", "6"}})); + ASSERT_OK(dbfull()->TEST_FlushMemTable(true)); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + + // Block compaction again + sleeping_task_low.Reset(); + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::LOW); + sleeping_task_low.WaitUntilSleeping(); + count = 0; + while (count < 64) { + ASSERT_OK(Put(Key(count), rnd.RandomString(1024), wo)); + ASSERT_OK(dbfull()->TEST_FlushMemTable(true, true)); + count++; + if (dbfull()->TEST_write_controler().IsStopped()) { + sleeping_task_low.WakeUp(); + break; + } + } + ASSERT_EQ(count, 6); + // Unblock + sleeping_task_low.WaitUntilDone(); + + // Test disable_auto_compactions + // Compaction thread is unblocked but auto compaction is disabled. Write + // 4 L0 files and compaction should be triggered. If auto compaction is + // disabled, then TEST_WaitForCompact will be waiting for nothing. Number of + // L0 files do not change after the call. + ASSERT_OK(dbfull()->SetOptions({{"disable_auto_compactions", "true"}})); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + + for (int i = 0; i < 4; ++i) { + ASSERT_OK(Put(Key(i), rnd.RandomString(1024))); + // Wait for compaction so that put won't stop + ASSERT_OK(dbfull()->TEST_FlushMemTable(true)); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumTableFilesAtLevel(0), 4); + + // Enable auto compaction and perform the same test, # of L0 files should be + // reduced after compaction. + ASSERT_OK(dbfull()->SetOptions({{"disable_auto_compactions", "false"}})); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + + for (int i = 0; i < 4; ++i) { + ASSERT_OK(Put(Key(i), rnd.RandomString(1024))); + // Wait for compaction so that put won't stop + ASSERT_OK(dbfull()->TEST_FlushMemTable(true)); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_LT(NumTableFilesAtLevel(0), 4); +} + +// Test dynamic FIFO compaction options. +// This test covers just option parsing and makes sure that the options are +// correctly assigned. Also look at DBOptionsTest.SetFIFOCompactionOptions +// test which makes sure that the FIFO compaction funcionality is working +// as expected on dynamically changing the options. +// Even more FIFOCompactionTests are at DBTest.FIFOCompaction* . +TEST_F(DBTest, DynamicFIFOCompactionOptions) { + Options options; + options.ttl = 0; + options.create_if_missing = true; + options.env = env_; + DestroyAndReopen(options); + + // Initial defaults + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, + 1024 * 1024 * 1024); + ASSERT_EQ(dbfull()->GetOptions().ttl, 0); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, + false); + + ASSERT_OK(dbfull()->SetOptions( + {{"compaction_options_fifo", "{max_table_files_size=23;}"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, + 23); + ASSERT_EQ(dbfull()->GetOptions().ttl, 0); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, + false); + + ASSERT_OK(dbfull()->SetOptions({{"ttl", "97"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, + 23); + ASSERT_EQ(dbfull()->GetOptions().ttl, 97); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, + false); + + ASSERT_OK(dbfull()->SetOptions({{"ttl", "203"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, + 23); + ASSERT_EQ(dbfull()->GetOptions().ttl, 203); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, + false); + + ASSERT_OK(dbfull()->SetOptions( + {{"compaction_options_fifo", "{allow_compaction=true;}"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, + 23); + ASSERT_EQ(dbfull()->GetOptions().ttl, 203); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, + true); + + ASSERT_OK(dbfull()->SetOptions( + {{"compaction_options_fifo", "{max_table_files_size=31;}"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, + 31); + ASSERT_EQ(dbfull()->GetOptions().ttl, 203); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, + true); + + ASSERT_OK(dbfull()->SetOptions( + {{"compaction_options_fifo", + "{max_table_files_size=51;allow_compaction=true;}"}})); + ASSERT_OK(dbfull()->SetOptions({{"ttl", "49"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, + 51); + ASSERT_EQ(dbfull()->GetOptions().ttl, 49); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, + true); +} + +TEST_F(DBTest, DynamicUniversalCompactionOptions) { + Options options; + options.create_if_missing = true; + options.env = env_; + DestroyAndReopen(options); + + // Initial defaults + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.size_ratio, 1U); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.min_merge_width, + 2u); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.max_merge_width, + UINT_MAX); + ASSERT_EQ(dbfull() + ->GetOptions() + .compaction_options_universal.max_size_amplification_percent, + 200u); + ASSERT_EQ(dbfull() + ->GetOptions() + .compaction_options_universal.compression_size_percent, + -1); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.stop_style, + kCompactionStopStyleTotalSize); + ASSERT_EQ( + dbfull()->GetOptions().compaction_options_universal.allow_trivial_move, + false); + + ASSERT_OK(dbfull()->SetOptions( + {{"compaction_options_universal", "{size_ratio=7;}"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.size_ratio, 7u); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.min_merge_width, + 2u); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.max_merge_width, + UINT_MAX); + ASSERT_EQ(dbfull() + ->GetOptions() + .compaction_options_universal.max_size_amplification_percent, + 200u); + ASSERT_EQ(dbfull() + ->GetOptions() + .compaction_options_universal.compression_size_percent, + -1); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.stop_style, + kCompactionStopStyleTotalSize); + ASSERT_EQ( + dbfull()->GetOptions().compaction_options_universal.allow_trivial_move, + false); + + ASSERT_OK(dbfull()->SetOptions( + {{"compaction_options_universal", "{min_merge_width=11;}"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.size_ratio, 7u); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.min_merge_width, + 11u); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.max_merge_width, + UINT_MAX); + ASSERT_EQ(dbfull() + ->GetOptions() + .compaction_options_universal.max_size_amplification_percent, + 200u); + ASSERT_EQ(dbfull() + ->GetOptions() + .compaction_options_universal.compression_size_percent, + -1); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.stop_style, + kCompactionStopStyleTotalSize); + ASSERT_EQ( + dbfull()->GetOptions().compaction_options_universal.allow_trivial_move, + false); +} + +TEST_F(DBTest, FileCreationRandomFailure) { + Options options; + options.env = env_; + options.create_if_missing = true; + options.write_buffer_size = 100000; // Small write buffer + options.target_file_size_base = 200000; + options.max_bytes_for_level_base = 1000000; + options.max_bytes_for_level_multiplier = 2; + + DestroyAndReopen(options); + Random rnd(301); + + constexpr int kCDTKeysPerBuffer = 4; + constexpr int kTestSize = kCDTKeysPerBuffer * 4096; + constexpr int kTotalIteration = 20; + // the second half of the test involves in random failure + // of file creation. + constexpr int kRandomFailureTest = kTotalIteration / 2; + + std::vector values; + for (int i = 0; i < kTestSize; ++i) { + values.push_back("NOT_FOUND"); + } + for (int j = 0; j < kTotalIteration; ++j) { + if (j == kRandomFailureTest) { + env_->non_writeable_rate_.store(90); + } + for (int k = 0; k < kTestSize; ++k) { + // here we expect some of the Put fails. + std::string value = rnd.RandomString(100); + Status s = Put(Key(k), Slice(value)); + if (s.ok()) { + // update the latest successful put + values[k] = value; + } + // But everything before we simulate the failure-test should succeed. + if (j < kRandomFailureTest) { + ASSERT_OK(s); + } + } + } + + // If rocksdb does not do the correct job, internal assert will fail here. + ASSERT_TRUE(dbfull()->TEST_WaitForFlushMemTable().IsIOError()); + ASSERT_TRUE(dbfull()->TEST_WaitForCompact().IsIOError()); + + // verify we have the latest successful update + for (int k = 0; k < kTestSize; ++k) { + auto v = Get(Key(k)); + ASSERT_EQ(v, values[k]); + } + + // reopen and reverify we have the latest successful update + env_->non_writeable_rate_.store(0); + Reopen(options); + for (int k = 0; k < kTestSize; ++k) { + auto v = Get(Key(k)); + ASSERT_EQ(v, values[k]); + } +} + + +TEST_F(DBTest, DynamicMiscOptions) { + // Test max_sequential_skip_in_iterations + Options options; + options.env = env_; + options.create_if_missing = true; + options.max_sequential_skip_in_iterations = 16; + options.compression = kNoCompression; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + DestroyAndReopen(options); + + auto assert_reseek_count = [this, &options](int key_start, int num_reseek) { + int key0 = key_start; + int key1 = key_start + 1; + int key2 = key_start + 2; + Random rnd(301); + ASSERT_OK(Put(Key(key0), rnd.RandomString(8))); + for (int i = 0; i < 10; ++i) { + ASSERT_OK(Put(Key(key1), rnd.RandomString(8))); + } + ASSERT_OK(Put(Key(key2), rnd.RandomString(8))); + std::unique_ptr iter(db_->NewIterator(ReadOptions())); + iter->Seek(Key(key1)); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Key(key1)), 0); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(Key(key2)), 0); + ASSERT_EQ(num_reseek, + TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION)); + }; + // No reseek + assert_reseek_count(100, 0); + + ASSERT_OK(dbfull()->SetOptions({{"max_sequential_skip_in_iterations", "4"}})); + // Clear memtable and make new option effective + ASSERT_OK(dbfull()->TEST_FlushMemTable(true)); + // Trigger reseek + assert_reseek_count(200, 1); + + ASSERT_OK( + dbfull()->SetOptions({{"max_sequential_skip_in_iterations", "16"}})); + // Clear memtable and make new option effective + ASSERT_OK(dbfull()->TEST_FlushMemTable(true)); + // No reseek + assert_reseek_count(300, 1); + + MutableCFOptions mutable_cf_options; + CreateAndReopenWithCF({"pikachu"}, options); + // Test soft_pending_compaction_bytes_limit, + // hard_pending_compaction_bytes_limit + ASSERT_OK(dbfull()->SetOptions( + handles_[1], {{"soft_pending_compaction_bytes_limit", "200"}, + {"hard_pending_compaction_bytes_limit", "300"}})); + ASSERT_OK(dbfull()->TEST_GetLatestMutableCFOptions(handles_[1], + &mutable_cf_options)); + ASSERT_EQ(200, mutable_cf_options.soft_pending_compaction_bytes_limit); + ASSERT_EQ(300, mutable_cf_options.hard_pending_compaction_bytes_limit); + // Test report_bg_io_stats + ASSERT_OK( + dbfull()->SetOptions(handles_[1], {{"report_bg_io_stats", "true"}})); + // sanity check + ASSERT_OK(dbfull()->TEST_GetLatestMutableCFOptions(handles_[1], + &mutable_cf_options)); + ASSERT_TRUE(mutable_cf_options.report_bg_io_stats); + // Test compression + // sanity check + ASSERT_OK(dbfull()->SetOptions({{"compression", "kNoCompression"}})); + ASSERT_OK(dbfull()->TEST_GetLatestMutableCFOptions(handles_[0], + &mutable_cf_options)); + ASSERT_EQ(CompressionType::kNoCompression, mutable_cf_options.compression); + + if (Snappy_Supported()) { + ASSERT_OK(dbfull()->SetOptions({{"compression", "kSnappyCompression"}})); + ASSERT_OK(dbfull()->TEST_GetLatestMutableCFOptions(handles_[0], + &mutable_cf_options)); + ASSERT_EQ(CompressionType::kSnappyCompression, + mutable_cf_options.compression); + } + + // Test paranoid_file_checks already done in db_block_cache_test + ASSERT_OK( + dbfull()->SetOptions(handles_[1], {{"paranoid_file_checks", "true"}})); + ASSERT_OK(dbfull()->TEST_GetLatestMutableCFOptions(handles_[1], + &mutable_cf_options)); + ASSERT_TRUE(mutable_cf_options.report_bg_io_stats); + ASSERT_TRUE(mutable_cf_options.check_flush_compaction_key_order); + + ASSERT_OK(dbfull()->SetOptions( + handles_[1], {{"check_flush_compaction_key_order", "false"}})); + ASSERT_OK(dbfull()->TEST_GetLatestMutableCFOptions(handles_[1], + &mutable_cf_options)); + ASSERT_FALSE(mutable_cf_options.check_flush_compaction_key_order); +} + +TEST_F(DBTest, L0L1L2AndUpHitCounter) { + const int kNumLevels = 3; + const int kNumKeysPerLevel = 10000; + const int kNumKeysPerDb = kNumLevels * kNumKeysPerLevel; + + Options options = CurrentOptions(); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + Reopen(options); + + // After the below loop there will be one file on each of L0, L1, and L2. + int key = 0; + for (int output_level = kNumLevels - 1; output_level >= 0; --output_level) { + for (int i = 0; i < kNumKeysPerLevel; ++i) { + ASSERT_OK(Put(Key(key), "val")); + key++; + } + ASSERT_OK(Flush()); + for (int input_level = 0; input_level < output_level; ++input_level) { + // `TEST_CompactRange(input_level, ...)` compacts from `input_level` to + // `input_level + 1`. + ASSERT_OK(dbfull()->TEST_CompactRange(input_level, nullptr, nullptr)); + } + } + assert(key == kNumKeysPerDb); + + ASSERT_EQ(0, TestGetTickerCount(options, GET_HIT_L0)); + ASSERT_EQ(0, TestGetTickerCount(options, GET_HIT_L1)); + ASSERT_EQ(0, TestGetTickerCount(options, GET_HIT_L2_AND_UP)); + + for (int i = 0; i < kNumKeysPerDb; i++) { + ASSERT_EQ(Get(Key(i)), "val"); + } + + ASSERT_EQ(kNumKeysPerLevel, TestGetTickerCount(options, GET_HIT_L0)); + ASSERT_EQ(kNumKeysPerLevel, TestGetTickerCount(options, GET_HIT_L1)); + ASSERT_EQ(kNumKeysPerLevel, TestGetTickerCount(options, GET_HIT_L2_AND_UP)); + + ASSERT_EQ(kNumKeysPerDb, TestGetTickerCount(options, GET_HIT_L0) + + TestGetTickerCount(options, GET_HIT_L1) + + TestGetTickerCount(options, GET_HIT_L2_AND_UP)); +} + +TEST_F(DBTest, EncodeDecompressedBlockSizeTest) { + // iter 0 -- zlib + // iter 1 -- bzip2 + // iter 2 -- lz4 + // iter 3 -- lz4HC + // iter 4 -- xpress + CompressionType compressions[] = {kZlibCompression, kBZip2Compression, + kLZ4Compression, kLZ4HCCompression, + kXpressCompression}; + for (auto comp : compressions) { + if (!CompressionTypeSupported(comp)) { + continue; + } + // first_table_version 1 -- generate with table_version == 1, read with + // table_version == 2 + // first_table_version 2 -- generate with table_version == 2, read with + // table_version == 1 + for (int first_table_version = 1; first_table_version <= 2; + ++first_table_version) { + BlockBasedTableOptions table_options; + table_options.format_version = first_table_version; + table_options.filter_policy.reset(NewBloomFilterPolicy(10)); + Options options = CurrentOptions(); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.create_if_missing = true; + options.compression = comp; + DestroyAndReopen(options); + + int kNumKeysWritten = 1000; + + Random rnd(301); + for (int i = 0; i < kNumKeysWritten; ++i) { + // compressible string + ASSERT_OK(Put(Key(i), rnd.RandomString(128) + std::string(128, 'a'))); + } + + table_options.format_version = first_table_version == 1 ? 2 : 1; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + for (int i = 0; i < kNumKeysWritten; ++i) { + auto r = Get(Key(i)); + ASSERT_EQ(r.substr(128), std::string(128, 'a')); + } + } + } +} + +TEST_F(DBTest, CloseSpeedup) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleLevel; + options.write_buffer_size = 110 << 10; // 110KB + options.arena_block_size = 4 << 10; + options.level0_file_num_compaction_trigger = 2; + options.num_levels = 4; + options.max_bytes_for_level_base = 400 * 1024; + options.max_write_buffer_number = 16; + + // Block background threads + env_->SetBackgroundThreads(1, Env::LOW); + env_->SetBackgroundThreads(1, Env::HIGH); + test::SleepingBackgroundTask sleeping_task_low; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::LOW); + test::SleepingBackgroundTask sleeping_task_high; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, + &sleeping_task_high, Env::Priority::HIGH); + + std::vector filenames; + ASSERT_OK(env_->GetChildren(dbname_, &filenames)); + // In Windows, LOCK file cannot be deleted because it is locked by db_test + // After closing db_test, the LOCK file is unlocked and can be deleted + // Delete archival files. + bool deleteDir = true; + for (size_t i = 0; i < filenames.size(); ++i) { + Status s = env_->DeleteFile(dbname_ + "/" + filenames[i]); + if (!s.ok()) { + deleteDir = false; + } + } + if (deleteDir) { + ASSERT_OK(env_->DeleteDir(dbname_)); + } + DestroyAndReopen(options); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + env_->SetBackgroundThreads(1, Env::LOW); + env_->SetBackgroundThreads(1, Env::HIGH); + Random rnd(301); + int key_idx = 0; + + // First three 110KB files are not going to level 2 + // After that, (100K, 200K) + for (int num = 0; num < 5; num++) { + GenerateNewFile(&rnd, &key_idx, true); + } + + ASSERT_EQ(0, GetSstFileCount(dbname_)); + + Close(); + ASSERT_EQ(0, GetSstFileCount(dbname_)); + + // Unblock background threads + sleeping_task_high.WakeUp(); + sleeping_task_high.WaitUntilDone(); + sleeping_task_low.WakeUp(); + sleeping_task_low.WaitUntilDone(); + + Destroy(options); +} + +class DelayedMergeOperator : public MergeOperator { + private: + DBTest* db_test_; + + public: + explicit DelayedMergeOperator(DBTest* d) : db_test_(d) {} + + bool FullMergeV2(const MergeOperationInput& merge_in, + MergeOperationOutput* merge_out) const override { + db_test_->env_->MockSleepForMicroseconds(1000 * + merge_in.operand_list.size()); + merge_out->new_value = ""; + return true; + } + + const char* Name() const override { return "DelayedMergeOperator"; } +}; + +TEST_F(DBTest, MergeTestTime) { + std::string one, two, three; + PutFixed64(&one, 1); + PutFixed64(&two, 2); + PutFixed64(&three, 3); + + // Enable time profiling + SetPerfLevel(kEnableTime); + Options options = CurrentOptions(); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.merge_operator.reset(new DelayedMergeOperator(this)); + SetTimeElapseOnlySleepOnReopen(&options); + DestroyAndReopen(options); + + // NOTE: Presumed unnecessary and removed: resetting mock time in env + + ASSERT_EQ(TestGetTickerCount(options, MERGE_OPERATION_TOTAL_TIME), 0); + ASSERT_OK(db_->Put(WriteOptions(), "foo", one)); + ASSERT_OK(Flush()); + ASSERT_OK(db_->Merge(WriteOptions(), "foo", two)); + ASSERT_OK(Flush()); + ASSERT_OK(db_->Merge(WriteOptions(), "foo", three)); + ASSERT_OK(Flush()); + + ReadOptions opt; + opt.verify_checksums = true; + opt.snapshot = nullptr; + std::string result; + ASSERT_OK(db_->Get(opt, "foo", &result)); + + ASSERT_EQ(2000000, TestGetTickerCount(options, MERGE_OPERATION_TOTAL_TIME)); + + ReadOptions read_options; + std::unique_ptr iter(db_->NewIterator(read_options)); + int count = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_OK(iter->status()); + ++count; + } + + ASSERT_EQ(1, count); + ASSERT_EQ(4000000, TestGetTickerCount(options, MERGE_OPERATION_TOTAL_TIME)); +#ifdef ROCKSDB_USING_THREAD_STATUS + ASSERT_GT(TestGetTickerCount(options, FLUSH_WRITE_BYTES), 0); +#endif // ROCKSDB_USING_THREAD_STATUS +} + +TEST_P(DBTestWithParam, MergeCompactionTimeTest) { + SetPerfLevel(kEnableTime); + Options options = CurrentOptions(); + options.compaction_filter_factory = std::make_shared(); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.merge_operator.reset(new DelayedMergeOperator(this)); + options.disable_auto_compactions = true; + options.max_subcompactions = max_subcompactions_; + SetTimeElapseOnlySleepOnReopen(&options); + DestroyAndReopen(options); + + constexpr unsigned n = 1000; + for (unsigned i = 0; i < n; i++) { + ASSERT_OK(db_->Merge(WriteOptions(), "foo", "TEST")); + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + + CompactRangeOptions cro; + cro.exclusive_manual_compaction = exclusive_manual_compaction_; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + ASSERT_EQ(uint64_t{n} * 1000000U, + TestGetTickerCount(options, MERGE_OPERATION_TOTAL_TIME)); +} + +TEST_P(DBTestWithParam, FilterCompactionTimeTest) { + Options options = CurrentOptions(); + options.compaction_filter_factory = + std::make_shared(this); + options.disable_auto_compactions = true; + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.statistics->set_stats_level(kExceptTimeForMutex); + options.max_subcompactions = max_subcompactions_; + SetTimeElapseOnlySleepOnReopen(&options); + DestroyAndReopen(options); + + unsigned n = 0; + // put some data + for (int table = 0; table < 4; ++table) { + for (int i = 0; i < 10 + table; ++i) { + ASSERT_OK(Put(std::to_string(table * 100 + i), "val")); + ++n; + } + ASSERT_OK(Flush()); + } + + CompactRangeOptions cro; + cro.exclusive_manual_compaction = exclusive_manual_compaction_; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ(0U, CountLiveFiles()); + + Reopen(options); + + Iterator* itr = db_->NewIterator(ReadOptions()); + itr->SeekToFirst(); + ASSERT_OK(itr->status()); + ASSERT_EQ(uint64_t{n} * 1000000U, + TestGetTickerCount(options, FILTER_OPERATION_TOTAL_TIME)); + delete itr; +} + +TEST_F(DBTest, TestLogCleanup) { + Options options = CurrentOptions(); + options.write_buffer_size = 64 * 1024; // very small + // only two memtables allowed ==> only two log files + options.max_write_buffer_number = 2; + Reopen(options); + + for (int i = 0; i < 100000; ++i) { + ASSERT_OK(Put(Key(i), "val")); + // only 2 memtables will be alive, so logs_to_free needs to always be below + // 2 + ASSERT_LT(dbfull()->TEST_LogsToFreeSize(), static_cast(3)); + } +} + +TEST_F(DBTest, EmptyCompactedDB) { + Options options = CurrentOptions(); + options.max_open_files = -1; + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + Status s = Put("new", "value"); + ASSERT_TRUE(s.IsNotSupported()); + Close(); +} + +TEST_F(DBTest, SuggestCompactRangeTest) { + class CompactionFilterFactoryGetContext : public CompactionFilterFactory { + public: + std::unique_ptr CreateCompactionFilter( + const CompactionFilter::Context& context) override { + saved_context = context; + std::unique_ptr empty_filter; + return empty_filter; + } + const char* Name() const override { + return "CompactionFilterFactoryGetContext"; + } + static bool IsManual(CompactionFilterFactory* compaction_filter_factory) { + return reinterpret_cast( + compaction_filter_factory) + ->saved_context.is_manual_compaction; + } + CompactionFilter::Context saved_context; + }; + + Options options = CurrentOptions(); + options.memtable_factory.reset(test::NewSpecialSkipListFactory( + DBTestBase::kNumKeysByGenerateNewRandomFile)); + options.compaction_style = kCompactionStyleLevel; + options.compaction_filter_factory.reset( + new CompactionFilterFactoryGetContext()); + options.write_buffer_size = 200 << 10; + options.arena_block_size = 4 << 10; + options.level0_file_num_compaction_trigger = 4; + options.num_levels = 4; + options.compression = kNoCompression; + options.max_bytes_for_level_base = 450 << 10; + options.target_file_size_base = 98 << 10; + options.max_compaction_bytes = static_cast(1) << 60; // inf + + Reopen(options); + + Random rnd(301); + + for (int num = 0; num < 10; num++) { + GenerateNewRandomFile(&rnd); + } + + ASSERT_TRUE(!CompactionFilterFactoryGetContext::IsManual( + options.compaction_filter_factory.get())); + + // make sure either L0 or L1 has file + while (NumTableFilesAtLevel(0) == 0 && NumTableFilesAtLevel(1) == 0) { + GenerateNewRandomFile(&rnd); + } + + // compact it three times + for (int i = 0; i < 3; ++i) { + ASSERT_OK(experimental::SuggestCompactRange(db_, nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + + // All files are compacted + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_EQ(0, NumTableFilesAtLevel(1)); + + GenerateNewRandomFile(&rnd); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + + // nonoverlapping with the file on level 0 + Slice start("a"), end("b"); + ASSERT_OK(experimental::SuggestCompactRange(db_, &start, &end)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // should not compact the level 0 file + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + + start = Slice("j"); + end = Slice("m"); + ASSERT_OK(experimental::SuggestCompactRange(db_, &start, &end)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // SuggestCompactRange() is not going to be reported as manual compaction + ASSERT_TRUE(!CompactionFilterFactoryGetContext::IsManual( + options.compaction_filter_factory.get())); + + // now it should compact the level 0 file + // as it's a trivial move to L1, it triggers another one to compact to L2 + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_EQ(0, NumTableFilesAtLevel(1)); +} + +TEST_F(DBTest, SuggestCompactRangeUniversal) { + Options options = CurrentOptions(); + options.memtable_factory.reset(test::NewSpecialSkipListFactory( + DBTestBase::kNumKeysByGenerateNewRandomFile)); + options.compaction_style = kCompactionStyleUniversal; + options.write_buffer_size = 200 << 10; + options.arena_block_size = 4 << 10; + options.level0_file_num_compaction_trigger = 4; + options.num_levels = 4; + options.compression = kNoCompression; + options.max_bytes_for_level_base = 450 << 10; + options.target_file_size_base = 98 << 10; + options.max_compaction_bytes = static_cast(1) << 60; // inf + + Reopen(options); + + Random rnd(301); + + for (int num = 0; num < 10; num++) { + GenerateNewRandomFile(&rnd); + } + + ASSERT_EQ("1,2,3,4", FilesPerLevel()); + for (int i = 0; i < 3; i++) { + ASSERT_OK( + db_->SuggestCompactRange(db_->DefaultColumnFamily(), nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + + // All files are compacted + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_EQ(0, NumTableFilesAtLevel(1)); + ASSERT_EQ(0, NumTableFilesAtLevel(2)); + + GenerateNewRandomFile(&rnd); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + + // nonoverlapping with the file on level 0 + Slice start("a"), end("b"); + ASSERT_OK(experimental::SuggestCompactRange(db_, &start, &end)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // should not compact the level 0 file + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + + start = Slice("j"); + end = Slice("m"); + ASSERT_OK(experimental::SuggestCompactRange(db_, &start, &end)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // now it should compact the level 0 file to the last level + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_EQ(0, NumTableFilesAtLevel(1)); +} + +TEST_F(DBTest, PromoteL0) { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.write_buffer_size = 10 * 1024 * 1024; + DestroyAndReopen(options); + + // non overlapping ranges + std::vector> ranges = { + {81, 160}, {0, 80}, {161, 240}, {241, 320}}; + + int32_t value_size = 10 * 1024; // 10 KB + + Random rnd(301); + std::map values; + for (const auto& range : ranges) { + for (int32_t j = range.first; j < range.second; j++) { + values[j] = rnd.RandomString(value_size); + ASSERT_OK(Put(Key(j), values[j])); + } + ASSERT_OK(Flush()); + } + + int32_t level0_files = NumTableFilesAtLevel(0, 0); + ASSERT_EQ(level0_files, ranges.size()); + ASSERT_EQ(NumTableFilesAtLevel(1, 0), 0); // No files in L1 + + // Promote L0 level to L2. + ASSERT_OK(experimental::PromoteL0(db_, db_->DefaultColumnFamily(), 2)); + // We expect that all the files were trivially moved from L0 to L2 + ASSERT_EQ(NumTableFilesAtLevel(0, 0), 0); + ASSERT_EQ(NumTableFilesAtLevel(2, 0), level0_files); + + for (const auto& kv : values) { + ASSERT_EQ(Get(Key(kv.first)), kv.second); + } +} + +TEST_F(DBTest, PromoteL0Failure) { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.write_buffer_size = 10 * 1024 * 1024; + DestroyAndReopen(options); + + // Produce two L0 files with overlapping ranges. + ASSERT_OK(Put(Key(0), "")); + ASSERT_OK(Put(Key(3), "")); + ASSERT_OK(Flush()); + ASSERT_OK(Put(Key(1), "")); + ASSERT_OK(Flush()); + + Status status; + // Fails because L0 has overlapping files. + status = experimental::PromoteL0(db_, db_->DefaultColumnFamily()); + ASSERT_TRUE(status.IsInvalidArgument()); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + // Now there is a file in L1. + ASSERT_GE(NumTableFilesAtLevel(1, 0), 1); + + ASSERT_OK(Put(Key(5), "")); + ASSERT_OK(Flush()); + // Fails because L1 is non-empty. + status = experimental::PromoteL0(db_, db_->DefaultColumnFamily()); + ASSERT_TRUE(status.IsInvalidArgument()); +} + +// Github issue #596 +TEST_F(DBTest, CompactRangeWithEmptyBottomLevel) { + const int kNumLevels = 2; + const int kNumL0Files = 2; + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.num_levels = kNumLevels; + DestroyAndReopen(options); + + Random rnd(301); + for (int i = 0; i < kNumL0Files; ++i) { + ASSERT_OK(Put(Key(0), rnd.RandomString(1024))); + ASSERT_OK(Flush()); + } + ASSERT_EQ(NumTableFilesAtLevel(0), kNumL0Files); + ASSERT_EQ(NumTableFilesAtLevel(1), 0); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + ASSERT_EQ(NumTableFilesAtLevel(1), kNumL0Files); +} + +TEST_F(DBTest, AutomaticConflictsWithManualCompaction) { + const int kNumL0Files = 50; + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = 4; + // never slowdown / stop + options.level0_slowdown_writes_trigger = 999999; + options.level0_stop_writes_trigger = 999999; + options.max_background_compactions = 10; + DestroyAndReopen(options); + + // schedule automatic compactions after the manual one starts, but before it + // finishes to ensure conflict. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BackgroundCompaction:Start", + "DBTest::AutomaticConflictsWithManualCompaction:PrePuts"}, + {"DBTest::AutomaticConflictsWithManualCompaction:PostPuts", + "DBImpl::BackgroundCompaction:NonTrivial:AfterRun"}}); + std::atomic callback_count(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::MaybeScheduleFlushOrCompaction:Conflict", + [&](void* /*arg*/) { callback_count.fetch_add(1); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Random rnd(301); + for (int i = 0; i < 2; ++i) { + // put two keys to ensure no trivial move + for (int j = 0; j < 2; ++j) { + ASSERT_OK(Put(Key(j), rnd.RandomString(1024))); + } + ASSERT_OK(Flush()); + } + port::Thread manual_compaction_thread([this]() { + CompactRangeOptions croptions; + croptions.exclusive_manual_compaction = true; + ASSERT_OK(db_->CompactRange(croptions, nullptr, nullptr)); + }); + + TEST_SYNC_POINT("DBTest::AutomaticConflictsWithManualCompaction:PrePuts"); + for (int i = 0; i < kNumL0Files; ++i) { + // put two keys to ensure no trivial move + for (int j = 0; j < 2; ++j) { + ASSERT_OK(Put(Key(j), rnd.RandomString(1024))); + } + ASSERT_OK(Flush()); + } + TEST_SYNC_POINT("DBTest::AutomaticConflictsWithManualCompaction:PostPuts"); + + ASSERT_GE(callback_count.load(), 1); + for (int i = 0; i < 2; ++i) { + ASSERT_NE("NOT_FOUND", Get(Key(i))); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + manual_compaction_thread.join(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); +} + +TEST_F(DBTest, CompactFilesShouldTriggerAutoCompaction) { + Options options = CurrentOptions(); + options.max_background_compactions = 1; + options.level0_file_num_compaction_trigger = 4; + options.level0_slowdown_writes_trigger = 36; + options.level0_stop_writes_trigger = 36; + DestroyAndReopen(options); + + // generate files for manual compaction + Random rnd(301); + for (int i = 0; i < 2; ++i) { + // put two keys to ensure no trivial move + for (int j = 0; j < 2; ++j) { + ASSERT_OK(Put(Key(j), rnd.RandomString(1024))); + } + ASSERT_OK(Flush()); + } + + ROCKSDB_NAMESPACE::ColumnFamilyMetaData cf_meta_data; + db_->GetColumnFamilyMetaData(db_->DefaultColumnFamily(), &cf_meta_data); + + std::vector input_files; + input_files.push_back(cf_meta_data.levels[0].files[0].name); + + SyncPoint::GetInstance()->LoadDependency({ + {"CompactFilesImpl:0", + "DBTest::CompactFilesShouldTriggerAutoCompaction:Begin"}, + {"DBTest::CompactFilesShouldTriggerAutoCompaction:End", + "CompactFilesImpl:1"}, + }); + + SyncPoint::GetInstance()->EnableProcessing(); + + port::Thread manual_compaction_thread([&]() { + auto s = db_->CompactFiles(CompactionOptions(), db_->DefaultColumnFamily(), + input_files, 0); + ASSERT_OK(s); + }); + + TEST_SYNC_POINT("DBTest::CompactFilesShouldTriggerAutoCompaction:Begin"); + // generate enough files to trigger compaction + for (int i = 0; i < 20; ++i) { + for (int j = 0; j < 2; ++j) { + ASSERT_OK(Put(Key(j), rnd.RandomString(1024))); + } + ASSERT_OK(Flush()); + } + db_->GetColumnFamilyMetaData(db_->DefaultColumnFamily(), &cf_meta_data); + ASSERT_GT(cf_meta_data.levels[0].files.size(), + options.level0_file_num_compaction_trigger); + TEST_SYNC_POINT("DBTest::CompactFilesShouldTriggerAutoCompaction:End"); + + manual_compaction_thread.join(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + db_->GetColumnFamilyMetaData(db_->DefaultColumnFamily(), &cf_meta_data); + ASSERT_LE(cf_meta_data.levels[0].files.size(), + options.level0_file_num_compaction_trigger); +} + +// Github issue #595 +// Large write batch with column families +TEST_F(DBTest, LargeBatchWithColumnFamilies) { + Options options = CurrentOptions(); + options.env = env_; + options.write_buffer_size = 100000; // Small write buffer + CreateAndReopenWithCF({"pikachu"}, options); + int64_t j = 0; + for (int i = 0; i < 5; i++) { + for (int pass = 1; pass <= 3; pass++) { + WriteBatch batch; + size_t write_size = 1024 * 1024 * (5 + i); + fprintf(stderr, "prepare: %" ROCKSDB_PRIszt " MB, pass:%d\n", + (write_size / 1024 / 1024), pass); + for (;;) { + std::string data(3000, j++ % 127 + 20); + data += std::to_string(j); + ASSERT_OK(batch.Put(handles_[0], Slice(data), Slice(data))); + if (batch.GetDataSize() > write_size) { + break; + } + } + fprintf(stderr, "write: %" ROCKSDB_PRIszt " MB\n", + (batch.GetDataSize() / 1024 / 1024)); + ASSERT_OK(dbfull()->Write(WriteOptions(), &batch)); + fprintf(stderr, "done\n"); + } + } + // make sure we can re-open it. + ASSERT_OK(TryReopenWithColumnFamilies({"default", "pikachu"}, options)); +} + +// Make sure that Flushes can proceed in parallel with CompactRange() +TEST_F(DBTest, FlushesInParallelWithCompactRange) { + // iter == 0 -- leveled + // iter == 1 -- leveled, but throw in a flush between two levels compacting + // iter == 2 -- universal + for (int iter = 0; iter < 3; ++iter) { + Options options = CurrentOptions(); + if (iter < 2) { + options.compaction_style = kCompactionStyleLevel; + } else { + options.compaction_style = kCompactionStyleUniversal; + } + options.write_buffer_size = 110 << 10; + options.level0_file_num_compaction_trigger = 4; + options.num_levels = 4; + options.compression = kNoCompression; + options.max_bytes_for_level_base = 450 << 10; + options.target_file_size_base = 98 << 10; + options.max_write_buffer_number = 2; + + DestroyAndReopen(options); + + Random rnd(301); + for (int num = 0; num < 14; num++) { + GenerateNewRandomFile(&rnd); + } + + if (iter == 1) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::RunManualCompaction()::1", + "DBTest::FlushesInParallelWithCompactRange:1"}, + {"DBTest::FlushesInParallelWithCompactRange:2", + "DBImpl::RunManualCompaction()::2"}}); + } else { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"CompactionJob::Run():Start", + "DBTest::FlushesInParallelWithCompactRange:1"}, + {"DBTest::FlushesInParallelWithCompactRange:2", + "CompactionJob::Run():End"}}); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + std::vector threads; + threads.emplace_back([&]() { Compact("a", "z"); }); + + TEST_SYNC_POINT("DBTest::FlushesInParallelWithCompactRange:1"); + + // this has to start a flush. if flushes are blocked, this will try to + // create + // 3 memtables, and that will fail because max_write_buffer_number is 2 + for (int num = 0; num < 3; num++) { + GenerateNewRandomFile(&rnd, /* nowait */ true); + } + + TEST_SYNC_POINT("DBTest::FlushesInParallelWithCompactRange:2"); + + for (auto& t : threads) { + t.join(); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + } +} + +TEST_F(DBTest, DelayedWriteRate) { + const int kEntriesPerMemTable = 100; + const int kTotalFlushes = 12; + + Options options = CurrentOptions(); + env_->SetBackgroundThreads(1, Env::LOW); + options.env = env_; + options.write_buffer_size = 100000000; + options.max_write_buffer_number = 256; + options.max_background_compactions = 1; + options.level0_file_num_compaction_trigger = 3; + options.level0_slowdown_writes_trigger = 3; + options.level0_stop_writes_trigger = 999999; + options.delayed_write_rate = 20000000; // Start with 200MB/s + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kEntriesPerMemTable)); + + SetTimeElapseOnlySleepOnReopen(&options); + CreateAndReopenWithCF({"pikachu"}, options); + + // Block compactions + test::SleepingBackgroundTask sleeping_task_low; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::LOW); + + for (int i = 0; i < 3; i++) { + ASSERT_OK(Put(Key(i), std::string(10000, 'x'))); + ASSERT_OK(Flush()); + } + + // These writes will be slowed down to 1KB/s + uint64_t estimated_sleep_time = 0; + Random rnd(301); + ASSERT_OK(Put("", "")); + uint64_t cur_rate = options.delayed_write_rate; + for (int i = 0; i < kTotalFlushes; i++) { + uint64_t size_memtable = 0; + for (int j = 0; j < kEntriesPerMemTable; j++) { + auto rand_num = rnd.Uniform(20); + // Spread the size range to more. + size_t entry_size = rand_num * rand_num * rand_num; + WriteOptions wo; + ASSERT_OK(Put(Key(i), std::string(entry_size, 'x'), wo)); + size_memtable += entry_size + 18; + // Occasionally sleep a while + if (rnd.Uniform(20) == 6) { + env_->SleepForMicroseconds(2666); + } + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + estimated_sleep_time += size_memtable * 1000000u / cur_rate; + // Slow down twice. One for memtable switch and one for flush finishes. + cur_rate = static_cast(static_cast(cur_rate) * + kIncSlowdownRatio * kIncSlowdownRatio); + } + // Estimate the total sleep time fall into the rough range. + ASSERT_GT(env_->NowMicros(), estimated_sleep_time / 2); + ASSERT_LT(env_->NowMicros(), estimated_sleep_time * 2); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + sleeping_task_low.WakeUp(); + sleeping_task_low.WaitUntilDone(); +} + +TEST_F(DBTest, HardLimit) { + Options options = CurrentOptions(); + options.env = env_; + env_->SetBackgroundThreads(1, Env::LOW); + options.max_write_buffer_number = 256; + options.write_buffer_size = 110 << 10; // 110KB + options.arena_block_size = 4 * 1024; + options.level0_file_num_compaction_trigger = 4; + options.level0_slowdown_writes_trigger = 999999; + options.level0_stop_writes_trigger = 999999; + options.hard_pending_compaction_bytes_limit = 800 << 10; + options.max_bytes_for_level_base = 10000000000u; + options.max_background_compactions = 1; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + + env_->SetBackgroundThreads(1, Env::LOW); + test::SleepingBackgroundTask sleeping_task_low; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::LOW); + + CreateAndReopenWithCF({"pikachu"}, options); + + std::atomic callback_count(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::DelayWrite:Wait", [&](void* /*arg*/) { + callback_count.fetch_add(1); + sleeping_task_low.WakeUp(); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Random rnd(301); + int key_idx = 0; + for (int num = 0; num < 5; num++) { + GenerateNewFile(&rnd, &key_idx, true); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + + ASSERT_EQ(0, callback_count.load()); + + for (int num = 0; num < 5; num++) { + GenerateNewFile(&rnd, &key_idx, true); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + ASSERT_GE(callback_count.load(), 1); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + sleeping_task_low.WaitUntilDone(); +} + +#if !defined(ROCKSDB_DISABLE_STALL_NOTIFICATION) +class WriteStallListener : public EventListener { + public: + WriteStallListener() : condition_(WriteStallCondition::kNormal) {} + void OnStallConditionsChanged(const WriteStallInfo& info) override { + MutexLock l(&mutex_); + condition_ = info.condition.cur; + } + bool CheckCondition(WriteStallCondition expected) { + MutexLock l(&mutex_); + return expected == condition_; + } + + private: + port::Mutex mutex_; + WriteStallCondition condition_; +}; + +TEST_F(DBTest, SoftLimit) { + Options options = CurrentOptions(); + options.env = env_; + options.write_buffer_size = 100000; // Small write buffer + options.max_write_buffer_number = 256; + options.level0_file_num_compaction_trigger = 1; + options.level0_slowdown_writes_trigger = 3; + options.level0_stop_writes_trigger = 999999; + options.delayed_write_rate = 20000; // About 200KB/s limited rate + options.soft_pending_compaction_bytes_limit = 160000; + options.target_file_size_base = 99999999; // All into one file + options.max_bytes_for_level_base = 50000; + options.max_bytes_for_level_multiplier = 10; + options.max_background_compactions = 1; + options.compression = kNoCompression; + WriteStallListener* listener = new WriteStallListener(); + options.listeners.emplace_back(listener); + + // FlushMemtable with opt.wait=true does not wait for + // `OnStallConditionsChanged` being called. The event listener is triggered + // on `JobContext::Clean`, which happens after flush result is installed. + // We use sync point to create a custom WaitForFlush that waits for + // context cleanup. + port::Mutex flush_mutex; + port::CondVar flush_cv(&flush_mutex); + bool flush_finished = false; + auto InstallFlushCallback = [&]() { + { + MutexLock l(&flush_mutex); + flush_finished = false; + } + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCallFlush:ContextCleanedUp", [&](void*) { + { + MutexLock l(&flush_mutex); + flush_finished = true; + } + flush_cv.SignalAll(); + }); + }; + auto WaitForFlush = [&]() { + { + MutexLock l(&flush_mutex); + while (!flush_finished) { + flush_cv.Wait(); + } + } + SyncPoint::GetInstance()->ClearCallBack( + "DBImpl::BackgroundCallFlush:ContextCleanedUp"); + }; + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Reopen(options); + + // Generating 360KB in Level 3 + for (int i = 0; i < 72; i++) { + ASSERT_OK(Put(Key(i), std::string(5000, 'x'))); + if (i % 10 == 0) { + ASSERT_OK(dbfull()->TEST_FlushMemTable(true, true)); + } + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + MoveFilesToLevel(3); + + // Generating 360KB in Level 2 + for (int i = 0; i < 72; i++) { + ASSERT_OK(Put(Key(i), std::string(5000, 'x'))); + if (i % 10 == 0) { + ASSERT_OK(dbfull()->TEST_FlushMemTable(true, true)); + } + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + MoveFilesToLevel(2); + + ASSERT_OK(Put(Key(0), "")); + + test::SleepingBackgroundTask sleeping_task_low; + // Block compactions + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::LOW); + sleeping_task_low.WaitUntilSleeping(); + + // Create 3 L0 files, making score of L0 to be 3. + for (int i = 0; i < 3; i++) { + ASSERT_OK(Put(Key(i), std::string(5000, 'x'))); + ASSERT_OK(Put(Key(100 - i), std::string(5000, 'x'))); + // Flush the file. File size is around 30KB. + InstallFlushCallback(); + ASSERT_OK(dbfull()->TEST_FlushMemTable(true, true)); + WaitForFlush(); + } + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_TRUE(listener->CheckCondition(WriteStallCondition::kDelayed)); + + sleeping_task_low.WakeUp(); + sleeping_task_low.WaitUntilDone(); + sleeping_task_low.Reset(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // Now there is one L1 file but doesn't trigger soft_rate_limit + // + // TODO: soft_rate_limit is depreciated. If this test + // relies on soft_rate_limit, then we need to change the test. + // + // The L1 file size is around 30KB. + ASSERT_EQ(NumTableFilesAtLevel(1), 1); + ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_TRUE(listener->CheckCondition(WriteStallCondition::kNormal)); + + // Only allow one compactin going through. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "BackgroundCallCompaction:0", [&](void* /*arg*/) { + // Schedule a sleeping task. + sleeping_task_low.Reset(); + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, + &sleeping_task_low, Env::Priority::LOW); + }); + + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::LOW); + sleeping_task_low.WaitUntilSleeping(); + // Create 3 L0 files, making score of L0 to be 3 + for (int i = 0; i < 3; i++) { + ASSERT_OK(Put(Key(10 + i), std::string(5000, 'x'))); + ASSERT_OK(Put(Key(90 - i), std::string(5000, 'x'))); + // Flush the file. File size is around 30KB. + InstallFlushCallback(); + ASSERT_OK(dbfull()->TEST_FlushMemTable(true, true)); + WaitForFlush(); + } + + // Wake up sleep task to enable compaction to run and waits + // for it to go to sleep state again to make sure one compaction + // goes through. + sleeping_task_low.WakeUp(); + sleeping_task_low.WaitUntilSleeping(); + + // Now there is one L1 file (around 60KB) which exceeds 50KB base by 10KB + // Given level multiplier 10, estimated pending compaction is around 100KB + // doesn't trigger soft_pending_compaction_bytes_limit + ASSERT_EQ(NumTableFilesAtLevel(1), 1); + ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_TRUE(listener->CheckCondition(WriteStallCondition::kNormal)); + + // Create 3 L0 files, making score of L0 to be 3, higher than L0. + for (int i = 0; i < 3; i++) { + ASSERT_OK(Put(Key(20 + i), std::string(5000, 'x'))); + ASSERT_OK(Put(Key(80 - i), std::string(5000, 'x'))); + // Flush the file. File size is around 30KB. + InstallFlushCallback(); + ASSERT_OK(dbfull()->TEST_FlushMemTable(true, true)); + WaitForFlush(); + } + // Wake up sleep task to enable compaction to run and waits + // for it to go to sleep state again to make sure one compaction + // goes through. + sleeping_task_low.WakeUp(); + sleeping_task_low.WaitUntilSleeping(); + + // Now there is one L1 file (around 90KB) which exceeds 50KB base by 40KB + // L2 size is 360KB, so the estimated level fanout 4, estimated pending + // compaction is around 200KB + // triggerring soft_pending_compaction_bytes_limit + ASSERT_EQ(NumTableFilesAtLevel(1), 1); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_TRUE(listener->CheckCondition(WriteStallCondition::kDelayed)); + + sleeping_task_low.WakeUp(); + sleeping_task_low.WaitUntilSleeping(); + + ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_TRUE(listener->CheckCondition(WriteStallCondition::kNormal)); + + // shrink level base so L2 will hit soft limit easier. + ASSERT_OK(dbfull()->SetOptions({ + {"max_bytes_for_level_base", "5000"}, + })); + + ASSERT_OK(Put("", "")); + ASSERT_OK(Flush()); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + ASSERT_TRUE(listener->CheckCondition(WriteStallCondition::kDelayed)); + + sleeping_task_low.WaitUntilSleeping(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + sleeping_task_low.WakeUp(); + sleeping_task_low.WaitUntilDone(); +} + +TEST_F(DBTest, LastWriteBufferDelay) { + Options options = CurrentOptions(); + options.env = env_; + options.write_buffer_size = 100000; + options.max_write_buffer_number = 4; + options.delayed_write_rate = 20000; + options.compression = kNoCompression; + options.disable_auto_compactions = true; + int kNumKeysPerMemtable = 3; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerMemtable)); + + Reopen(options); + test::SleepingBackgroundTask sleeping_task; + // Block flushes + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task, + Env::Priority::HIGH); + sleeping_task.WaitUntilSleeping(); + + // Create 3 L0 files, making score of L0 to be 3. + for (int i = 0; i < 3; i++) { + // Fill one mem table + for (int j = 0; j < kNumKeysPerMemtable; j++) { + ASSERT_OK(Put(Key(j), "")); + } + ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); + } + // Inserting a new entry would create a new mem table, triggering slow down. + ASSERT_OK(Put(Key(0), "")); + ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); + + sleeping_task.WakeUp(); + sleeping_task.WaitUntilDone(); +} +#endif // !defined(ROCKSDB_DISABLE_STALL_NOTIFICATION) + +TEST_F(DBTest, FailWhenCompressionNotSupportedTest) { + CompressionType compressions[] = {kZlibCompression, kBZip2Compression, + kLZ4Compression, kLZ4HCCompression, + kXpressCompression}; + for (auto comp : compressions) { + if (!CompressionTypeSupported(comp)) { + // not supported, we should fail the Open() + Options options = CurrentOptions(); + options.compression = comp; + ASSERT_TRUE(!TryReopen(options).ok()); + // Try if CreateColumnFamily also fails + options.compression = kNoCompression; + ASSERT_OK(TryReopen(options)); + ColumnFamilyOptions cf_options(options); + cf_options.compression = comp; + ColumnFamilyHandle* handle; + ASSERT_TRUE(!db_->CreateColumnFamily(cf_options, "name", &handle).ok()); + } + } +} + +TEST_F(DBTest, CreateColumnFamilyShouldFailOnIncompatibleOptions) { + Options options = CurrentOptions(); + options.max_open_files = 100; + Reopen(options); + + ColumnFamilyOptions cf_options(options); + // ttl is now supported when max_open_files is -1. + cf_options.ttl = 3600; + ColumnFamilyHandle* handle; + ASSERT_OK(db_->CreateColumnFamily(cf_options, "pikachu", &handle)); + delete handle; +} + +TEST_F(DBTest, RowCache) { + Options options = CurrentOptions(); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + LRUCacheOptions cache_options; + cache_options.capacity = 8192; + options.row_cache = cache_options.MakeSharedRowCache(); + // BEGIN check that Cache classes as aliases of each other. + // Currently, RowCache and BlockCache are aliases for Cache. + // This is expected to change (carefully, intentionally) + std::shared_ptr row_cache = options.row_cache; + std::shared_ptr cache = row_cache; + std::shared_ptr block_cache = row_cache; + row_cache = cache; + block_cache = cache; + row_cache = block_cache; + cache = block_cache; + // END check that Cache classes as aliases of each other. + DestroyAndReopen(options); + + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Flush()); + + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), 0); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), 0); + ASSERT_EQ(Get("foo"), "bar"); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), 0); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), 1); + ASSERT_EQ(Get("foo"), "bar"); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), 1); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), 1); +} + +TEST_F(DBTest, PinnableSliceAndRowCache) { + Options options = CurrentOptions(); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.row_cache = NewLRUCache(8192); + DestroyAndReopen(options); + + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Flush()); + + ASSERT_EQ(Get("foo"), "bar"); + ASSERT_EQ( + reinterpret_cast(options.row_cache.get())->TEST_GetLRUSize(), + 1); + + { + PinnableSlice pin_slice; + ASSERT_EQ(Get("foo", &pin_slice), Status::OK()); + ASSERT_EQ(pin_slice.ToString(), "bar"); + // Entry is already in cache, lookup will remove the element from lru + ASSERT_EQ( + reinterpret_cast(options.row_cache.get())->TEST_GetLRUSize(), + 0); + } + // After PinnableSlice destruction element is added back in LRU + ASSERT_EQ( + reinterpret_cast(options.row_cache.get())->TEST_GetLRUSize(), + 1); +} + +TEST_F(DBTest, ReusePinnableSlice) { + Options options = CurrentOptions(); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.row_cache = NewLRUCache(8192); + DestroyAndReopen(options); + + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Flush()); + + ASSERT_EQ(Get("foo"), "bar"); + ASSERT_EQ( + reinterpret_cast(options.row_cache.get())->TEST_GetLRUSize(), + 1); + + { + PinnableSlice pin_slice; + ASSERT_EQ(Get("foo", &pin_slice), Status::OK()); + ASSERT_EQ(Get("foo", &pin_slice), Status::OK()); + ASSERT_EQ(pin_slice.ToString(), "bar"); + + // Entry is already in cache, lookup will remove the element from lru + ASSERT_EQ( + reinterpret_cast(options.row_cache.get())->TEST_GetLRUSize(), + 0); + } + // After PinnableSlice destruction element is added back in LRU + ASSERT_EQ( + reinterpret_cast(options.row_cache.get())->TEST_GetLRUSize(), + 1); + + { + std::vector multiget_keys; + multiget_keys.push_back("foo"); + std::vector multiget_values(1); + std::vector statuses({Status::NotFound()}); + ReadOptions ropt; + dbfull()->MultiGet(ropt, dbfull()->DefaultColumnFamily(), + multiget_keys.size(), multiget_keys.data(), + multiget_values.data(), statuses.data()); + ASSERT_EQ(Status::OK(), statuses[0]); + dbfull()->MultiGet(ropt, dbfull()->DefaultColumnFamily(), + multiget_keys.size(), multiget_keys.data(), + multiget_values.data(), statuses.data()); + ASSERT_EQ(Status::OK(), statuses[0]); + + // Entry is already in cache, lookup will remove the element from lru + ASSERT_EQ( + reinterpret_cast(options.row_cache.get())->TEST_GetLRUSize(), + 0); + } + // After PinnableSlice destruction element is added back in LRU + ASSERT_EQ( + reinterpret_cast(options.row_cache.get())->TEST_GetLRUSize(), + 1); + + { + std::vector multiget_cfs; + multiget_cfs.push_back(dbfull()->DefaultColumnFamily()); + std::vector multiget_keys; + multiget_keys.push_back("foo"); + std::vector multiget_values(1); + std::vector statuses({Status::NotFound()}); + ReadOptions ropt; + dbfull()->MultiGet(ropt, multiget_keys.size(), multiget_cfs.data(), + multiget_keys.data(), multiget_values.data(), + statuses.data()); + ASSERT_EQ(Status::OK(), statuses[0]); + dbfull()->MultiGet(ropt, multiget_keys.size(), multiget_cfs.data(), + multiget_keys.data(), multiget_values.data(), + statuses.data()); + ASSERT_EQ(Status::OK(), statuses[0]); + + // Entry is already in cache, lookup will remove the element from lru + ASSERT_EQ( + reinterpret_cast(options.row_cache.get())->TEST_GetLRUSize(), + 0); + } + // After PinnableSlice destruction element is added back in LRU + ASSERT_EQ( + reinterpret_cast(options.row_cache.get())->TEST_GetLRUSize(), + 1); +} + + +TEST_F(DBTest, DeletingOldWalAfterDrop) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"Test:AllowFlushes", "DBImpl::BGWorkFlush"}, + {"DBImpl::BGWorkFlush:done", "Test:WaitForFlush"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace(); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + Options options = CurrentOptions(); + options.max_total_wal_size = 8192; + options.compression = kNoCompression; + options.write_buffer_size = 1 << 20; + options.level0_file_num_compaction_trigger = (1 << 30); + options.level0_slowdown_writes_trigger = (1 << 30); + options.level0_stop_writes_trigger = (1 << 30); + options.disable_auto_compactions = true; + DestroyAndReopen(options); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + CreateColumnFamilies({"cf1", "cf2"}, options); + ASSERT_OK(Put(0, "key1", DummyString(8192))); + ASSERT_OK(Put(0, "key2", DummyString(8192))); + // the oldest wal should now be getting_flushed + ASSERT_OK(db_->DropColumnFamily(handles_[0])); + // all flushes should now do nothing because their CF is dropped + TEST_SYNC_POINT("Test:AllowFlushes"); + TEST_SYNC_POINT("Test:WaitForFlush"); + uint64_t lognum1 = dbfull()->TEST_LogfileNumber(); + ASSERT_OK(Put(1, "key3", DummyString(8192))); + ASSERT_OK(Put(1, "key4", DummyString(8192))); + // new wal should have been created + uint64_t lognum2 = dbfull()->TEST_LogfileNumber(); + EXPECT_GT(lognum2, lognum1); +} + +TEST_F(DBTest, UnsupportedManualSync) { + DestroyAndReopen(CurrentOptions()); + env_->is_wal_sync_thread_safe_.store(false); + Status s = db_->SyncWAL(); + ASSERT_TRUE(s.IsNotSupported()); +} + +INSTANTIATE_TEST_CASE_P(DBTestWithParam, DBTestWithParam, + ::testing::Combine(::testing::Values(1, 4), + ::testing::Bool())); + +TEST_F(DBTest, PauseBackgroundWorkTest) { + Options options = CurrentOptions(); + options.write_buffer_size = 100000; // Small write buffer + Reopen(options); + + std::vector threads; + std::atomic done(false); + ASSERT_OK(db_->PauseBackgroundWork()); + threads.emplace_back([&]() { + Random rnd(301); + for (int i = 0; i < 10000; ++i) { + ASSERT_OK(Put(rnd.RandomString(10), rnd.RandomString(10))); + } + done.store(true); + }); + env_->SleepForMicroseconds(200000); + // make sure the thread is not done + ASSERT_FALSE(done.load()); + ASSERT_OK(db_->ContinueBackgroundWork()); + for (auto& t : threads) { + t.join(); + } + // now it's done + ASSERT_TRUE(done.load()); +} + +// Keep spawning short-living threads that create an iterator and quit. +// Meanwhile in another thread keep flushing memtables. +// This used to cause a deadlock. +TEST_F(DBTest, ThreadLocalPtrDeadlock) { + std::atomic flushes_done{0}; + std::atomic threads_destroyed{0}; + auto done = [&] { return flushes_done.load() > 10; }; + + port::Thread flushing_thread([&] { + for (int i = 0; !done(); ++i) { + ASSERT_OK(db_->Put(WriteOptions(), Slice("hi"), + Slice(std::to_string(i).c_str()))); + ASSERT_OK(db_->Flush(FlushOptions())); + int cnt = ++flushes_done; + fprintf(stderr, "Flushed %d times\n", cnt); + } + }); + + std::vector thread_spawning_threads(10); + for (auto& t : thread_spawning_threads) { + t = port::Thread([&] { + while (!done()) { + { + port::Thread tmp_thread([&] { + auto it = db_->NewIterator(ReadOptions()); + ASSERT_OK(it->status()); + delete it; + }); + tmp_thread.join(); + } + ++threads_destroyed; + } + }); + } + + for (auto& t : thread_spawning_threads) { + t.join(); + } + flushing_thread.join(); + fprintf(stderr, "Done. Flushed %d times, destroyed %d threads\n", + flushes_done.load(), threads_destroyed.load()); +} + +TEST_F(DBTest, LargeBlockSizeTest) { + Options options = CurrentOptions(); + CreateAndReopenWithCF({"pikachu"}, options); + ASSERT_OK(Put(0, "foo", "bar")); + BlockBasedTableOptions table_options; + table_options.block_size = 8LL * 1024 * 1024 * 1024LL; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + ASSERT_NOK(TryReopenWithColumnFamilies({"default", "pikachu"}, options)); +} + + +TEST_F(DBTest, CreationTimeOfOldestFile) { + const int kNumKeysPerFile = 32; + const int kNumLevelFiles = 2; + const int kValueSize = 100; + + Options options = CurrentOptions(); + options.max_open_files = -1; + env_->SetMockSleep(); + options.env = env_; + + // NOTE: Presumed unnecessary and removed: resetting mock time in env + + DestroyAndReopen(options); + + bool set_file_creation_time_to_zero = true; + int idx = 0; + + int64_t time_1 = 0; + env_->GetCurrentTime(&time_1); + const uint64_t uint_time_1 = static_cast(time_1); + + // Add 50 hours + env_->MockSleepForSeconds(50 * 60 * 60); + + int64_t time_2 = 0; + env_->GetCurrentTime(&time_2); + const uint64_t uint_time_2 = static_cast(time_2); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "PropertyBlockBuilder::AddTableProperty:Start", [&](void* arg) { + TableProperties* props = reinterpret_cast(arg); + if (set_file_creation_time_to_zero) { + if (idx == 0) { + props->file_creation_time = 0; + idx++; + } else if (idx == 1) { + props->file_creation_time = uint_time_1; + idx = 0; + } + } else { + if (idx == 0) { + props->file_creation_time = uint_time_1; + idx++; + } else if (idx == 1) { + props->file_creation_time = uint_time_2; + } + } + }); + // Set file creation time in manifest all to 0. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "FileMetaData::FileMetaData", [&](void* arg) { + FileMetaData* meta = static_cast(arg); + meta->file_creation_time = 0; + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Random rnd(301); + for (int i = 0; i < kNumLevelFiles; ++i) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK( + Put(Key(i * kNumKeysPerFile + j), rnd.RandomString(kValueSize))); + } + ASSERT_OK(Flush()); + } + + // At this point there should be 2 files, one with file_creation_time = 0 and + // the other non-zero. GetCreationTimeOfOldestFile API should return 0. + uint64_t creation_time; + Status s1 = dbfull()->GetCreationTimeOfOldestFile(&creation_time); + ASSERT_EQ(0, creation_time); + ASSERT_EQ(s1, Status::OK()); + + // Testing with non-zero file creation time. + set_file_creation_time_to_zero = false; + options = CurrentOptions(); + options.max_open_files = -1; + options.env = env_; + + // NOTE: Presumed unnecessary and removed: resetting mock time in env + + DestroyAndReopen(options); + + for (int i = 0; i < kNumLevelFiles; ++i) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK( + Put(Key(i * kNumKeysPerFile + j), rnd.RandomString(kValueSize))); + } + ASSERT_OK(Flush()); + } + + // At this point there should be 2 files with non-zero file creation time. + // GetCreationTimeOfOldestFile API should return non-zero value. + uint64_t ctime; + Status s2 = dbfull()->GetCreationTimeOfOldestFile(&ctime); + ASSERT_EQ(uint_time_1, ctime); + ASSERT_EQ(s2, Status::OK()); + + // Testing with max_open_files != -1 + options = CurrentOptions(); + options.max_open_files = 10; + DestroyAndReopen(options); + Status s3 = dbfull()->GetCreationTimeOfOldestFile(&ctime); + ASSERT_EQ(s3, Status::NotSupported()); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBTest, MemoryUsageWithMaxWriteBufferSizeToMaintain) { + Options options = CurrentOptions(); + options.max_write_buffer_size_to_maintain = 10000; + options.write_buffer_size = 160000; + Reopen(options); + Random rnd(301); + bool memory_limit_exceeded = false; + + ColumnFamilyData* cfd = + static_cast(db_->DefaultColumnFamily())->cfd(); + + for (int i = 0; i < 1000; i++) { + std::string value = rnd.RandomString(1000); + ASSERT_OK(Put("keykey_" + std::to_string(i), value)); + + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + + const uint64_t cur_active_mem = cfd->mem()->ApproximateMemoryUsage(); + const uint64_t size_all_mem_table = + cur_active_mem + cfd->imm()->ApproximateMemoryUsage(); + + // Errors out if memory usage keeps on increasing beyond the limit. + // Once memory limit exceeds, memory_limit_exceeded is set and if + // size_all_mem_table doesn't drop out in the next write then it errors out + // (not expected behaviour). If memory usage drops then + // memory_limit_exceeded is set to false. + if ((size_all_mem_table > cur_active_mem) && + (cur_active_mem >= + static_cast(options.max_write_buffer_size_to_maintain)) && + (size_all_mem_table > + static_cast(options.max_write_buffer_size_to_maintain) + + options.write_buffer_size)) { + ASSERT_FALSE(memory_limit_exceeded); + memory_limit_exceeded = true; + } else { + memory_limit_exceeded = false; + } + } +} + +TEST_F(DBTest, ShuttingDownNotBlockStalledWrites) { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + Reopen(options); + Random rnd(403); + + for (int i = 0; i < 20; i++) { + ASSERT_OK(Put("key_" + std::to_string(i), rnd.RandomString(10))); + ASSERT_OK(Flush()); + } + ASSERT_EQ(GetSstFileCount(dbname_), 20); + + // We need !disable_auto_compactions for writes to stall but also want to + // delay compaction so stalled writes unblocked due to kShutdownInProgress. BG + // compaction will first wait for the sync point + // DBTest::ShuttingDownNotBlockStalledWrites. Then it waits extra 2 sec to + // allow CancelAllBackgroundWork() to set shutting_down_. + SyncPoint::GetInstance()->SetCallBack( + "BackgroundCallCompaction:0", + [&](void* /* arg */) { env_->SleepForMicroseconds(2 * 1000 * 1000); }); + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::DelayWrite:Wait", "DBTest::ShuttingDownNotBlockStalledWrites"}, + {"DBTest::ShuttingDownNotBlockStalledWrites", + "BackgroundCallCompaction:0"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + options.level0_stop_writes_trigger = 20; + options.disable_auto_compactions = false; + Reopen(options); + + std::thread thd([&]() { + Status s = Put("key_" + std::to_string(101), "101"); + ASSERT_EQ(s.code(), Status::kShutdownInProgress); + }); + + TEST_SYNC_POINT("DBTest::ShuttingDownNotBlockStalledWrites"); + CancelAllBackgroundWork(db_, true); + + thd.join(); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_test2.cc b/librocksdb-sys/rocksdb/db/db_test2.cc new file mode 100644 index 0000000..52d64a9 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_test2.cc @@ -0,0 +1,7680 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include +#include +#include + +#include "db/db_test_util.h" +#include "db/read_callback.h" +#include "db/version_edit.h" +#include "options/options_helper.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/experimental.h" +#include "rocksdb/iostats_context.h" +#include "rocksdb/persistent_cache.h" +#include "rocksdb/trace_record.h" +#include "rocksdb/trace_record_result.h" +#include "rocksdb/utilities/replayer.h" +#include "rocksdb/wal_filter.h" +#include "test_util/testutil.h" +#include "util/random.h" +#include "utilities/fault_injection_env.h" + +namespace ROCKSDB_NAMESPACE { + +class DBTest2 : public DBTestBase { + public: + DBTest2() : DBTestBase("db_test2", /*env_do_fsync=*/true) {} + std::vector GetLevelFileMetadatas(int level, int cf = 0) { + VersionSet* const versions = dbfull()->GetVersionSet(); + assert(versions); + ColumnFamilyData* const cfd = + versions->GetColumnFamilySet()->GetColumnFamily(cf); + assert(cfd); + Version* const current = cfd->current(); + assert(current); + VersionStorageInfo* const storage_info = current->storage_info(); + assert(storage_info); + return storage_info->LevelFiles(level); + } +}; + +TEST_F(DBTest2, OpenForReadOnly) { + DB* db_ptr = nullptr; + std::string dbname = test::PerThreadDBPath("db_readonly"); + Options options = CurrentOptions(); + options.create_if_missing = true; + // OpenForReadOnly should fail but will create in the file system + ASSERT_NOK(DB::OpenForReadOnly(options, dbname, &db_ptr)); + // Since is created, we should be able to delete the dir + // We first get the list files under + // There should not be any subdirectories -- this is not checked here + std::vector files; + ASSERT_OK(env_->GetChildren(dbname, &files)); + for (auto& f : files) { + ASSERT_OK(env_->DeleteFile(dbname + "/" + f)); + } + // should be empty now and we should be able to delete it + ASSERT_OK(env_->DeleteDir(dbname)); + options.create_if_missing = false; + // OpenForReadOnly should fail since was successfully deleted + ASSERT_NOK(DB::OpenForReadOnly(options, dbname, &db_ptr)); + // With create_if_missing false, there should not be a dir in the file system + ASSERT_NOK(env_->FileExists(dbname)); +} + +TEST_F(DBTest2, OpenForReadOnlyWithColumnFamilies) { + DB* db_ptr = nullptr; + std::string dbname = test::PerThreadDBPath("db_readonly"); + Options options = CurrentOptions(); + options.create_if_missing = true; + + ColumnFamilyOptions cf_options(options); + std::vector column_families; + column_families.push_back( + ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options)); + column_families.push_back(ColumnFamilyDescriptor("goku", cf_options)); + std::vector handles; + // OpenForReadOnly should fail but will create in the file system + ASSERT_NOK( + DB::OpenForReadOnly(options, dbname, column_families, &handles, &db_ptr)); + // Since is created, we should be able to delete the dir + // We first get the list files under + // There should not be any subdirectories -- this is not checked here + std::vector files; + ASSERT_OK(env_->GetChildren(dbname, &files)); + for (auto& f : files) { + ASSERT_OK(env_->DeleteFile(dbname + "/" + f)); + } + // should be empty now and we should be able to delete it + ASSERT_OK(env_->DeleteDir(dbname)); + options.create_if_missing = false; + // OpenForReadOnly should fail since was successfully deleted + ASSERT_NOK( + DB::OpenForReadOnly(options, dbname, column_families, &handles, &db_ptr)); + // With create_if_missing false, there should not be a dir in the file system + ASSERT_NOK(env_->FileExists(dbname)); +} + +class PartitionedIndexTestListener : public EventListener { + public: + void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& info) override { + ASSERT_GT(info.table_properties.index_partitions, 1); + ASSERT_EQ(info.table_properties.index_key_is_user_key, 0); + } +}; + +TEST_F(DBTest2, PartitionedIndexUserToInternalKey) { + const int kValueSize = 10500; + const int kNumEntriesPerFile = 1000; + const int kNumFiles = 3; + const int kNumDistinctKeys = 30; + + BlockBasedTableOptions table_options; + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + table_options.index_type = BlockBasedTableOptions::kTwoLevelIndexSearch; + PartitionedIndexTestListener* listener = new PartitionedIndexTestListener(); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.listeners.emplace_back(listener); + std::vector snapshots; + Reopen(options); + Random rnd(301); + + for (int i = 0; i < kNumFiles; i++) { + for (int j = 0; j < kNumEntriesPerFile; j++) { + int key_id = (i * kNumEntriesPerFile + j) % kNumDistinctKeys; + std::string value = rnd.RandomString(kValueSize); + ASSERT_OK(Put("keykey_" + std::to_string(key_id), value)); + snapshots.push_back(db_->GetSnapshot()); + } + ASSERT_OK(Flush()); + } + + for (auto s : snapshots) { + db_->ReleaseSnapshot(s); + } +} + + +class PrefixFullBloomWithReverseComparator + : public DBTestBase, + public ::testing::WithParamInterface { + public: + PrefixFullBloomWithReverseComparator() + : DBTestBase("prefix_bloom_reverse", /*env_do_fsync=*/true) {} + void SetUp() override { if_cache_filter_ = GetParam(); } + bool if_cache_filter_; +}; + +TEST_P(PrefixFullBloomWithReverseComparator, + PrefixFullBloomWithReverseComparator) { + Options options = last_options_; + options.comparator = ReverseBytewiseComparator(); + options.prefix_extractor.reset(NewCappedPrefixTransform(3)); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + BlockBasedTableOptions bbto; + if (if_cache_filter_) { + bbto.no_block_cache = false; + bbto.cache_index_and_filter_blocks = true; + bbto.block_cache = NewLRUCache(1); + } + bbto.filter_policy.reset(NewBloomFilterPolicy(10, false)); + bbto.whole_key_filtering = false; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + DestroyAndReopen(options); + + ASSERT_OK(dbfull()->Put(WriteOptions(), "bar123", "foo")); + ASSERT_OK(dbfull()->Put(WriteOptions(), "bar234", "foo2")); + ASSERT_OK(dbfull()->Put(WriteOptions(), "foo123", "foo3")); + + ASSERT_OK(dbfull()->Flush(FlushOptions())); + + if (bbto.block_cache) { + bbto.block_cache->EraseUnRefEntries(); + } + + std::unique_ptr iter(db_->NewIterator(ReadOptions())); + iter->Seek("bar345"); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("bar234", iter->key().ToString()); + ASSERT_EQ("foo2", iter->value().ToString()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("bar123", iter->key().ToString()); + ASSERT_EQ("foo", iter->value().ToString()); + + iter->Seek("foo234"); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("foo123", iter->key().ToString()); + ASSERT_EQ("foo3", iter->value().ToString()); + + iter->Seek("bar"); + ASSERT_OK(iter->status()); + ASSERT_TRUE(!iter->Valid()); +} + +INSTANTIATE_TEST_CASE_P(PrefixFullBloomWithReverseComparator, + PrefixFullBloomWithReverseComparator, testing::Bool()); + +TEST_F(DBTest2, IteratorPropertyVersionNumber) { + ASSERT_OK(Put("", "")); + Iterator* iter1 = db_->NewIterator(ReadOptions()); + ASSERT_OK(iter1->status()); + std::string prop_value; + ASSERT_OK( + iter1->GetProperty("rocksdb.iterator.super-version-number", &prop_value)); + uint64_t version_number1 = + static_cast(std::atoi(prop_value.c_str())); + + ASSERT_OK(Put("", "")); + ASSERT_OK(Flush()); + + Iterator* iter2 = db_->NewIterator(ReadOptions()); + ASSERT_OK(iter2->status()); + ASSERT_OK( + iter2->GetProperty("rocksdb.iterator.super-version-number", &prop_value)); + uint64_t version_number2 = + static_cast(std::atoi(prop_value.c_str())); + + ASSERT_GT(version_number2, version_number1); + + ASSERT_OK(Put("", "")); + + Iterator* iter3 = db_->NewIterator(ReadOptions()); + ASSERT_OK(iter3->status()); + ASSERT_OK( + iter3->GetProperty("rocksdb.iterator.super-version-number", &prop_value)); + uint64_t version_number3 = + static_cast(std::atoi(prop_value.c_str())); + + ASSERT_EQ(version_number2, version_number3); + + iter1->SeekToFirst(); + ASSERT_OK( + iter1->GetProperty("rocksdb.iterator.super-version-number", &prop_value)); + uint64_t version_number1_new = + static_cast(std::atoi(prop_value.c_str())); + ASSERT_EQ(version_number1, version_number1_new); + + delete iter1; + delete iter2; + delete iter3; +} + +TEST_F(DBTest2, CacheIndexAndFilterWithDBRestart) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + BlockBasedTableOptions table_options; + table_options.cache_index_and_filter_blocks = true; + table_options.filter_policy.reset(NewBloomFilterPolicy(20)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(1, "a", "begin")); + ASSERT_OK(Put(1, "z", "end")); + ASSERT_OK(Flush(1)); + TryReopenWithColumnFamilies({"default", "pikachu"}, options); + + std::string value; + value = Get(1, "a"); +} + +TEST_F(DBTest2, MaxSuccessiveMergesChangeWithDBRecovery) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.max_successive_merges = 3; + options.merge_operator = MergeOperators::CreatePutOperator(); + options.disable_auto_compactions = true; + DestroyAndReopen(options); + ASSERT_OK(Put("poi", "Finch")); + ASSERT_OK(db_->Merge(WriteOptions(), "poi", "Reese")); + ASSERT_OK(db_->Merge(WriteOptions(), "poi", "Shaw")); + ASSERT_OK(db_->Merge(WriteOptions(), "poi", "Root")); + options.max_successive_merges = 2; + Reopen(options); +} + +class DBTestSharedWriteBufferAcrossCFs + : public DBTestBase, + public testing::WithParamInterface> { + public: + DBTestSharedWriteBufferAcrossCFs() + : DBTestBase("db_test_shared_write_buffer", /*env_do_fsync=*/true) {} + void SetUp() override { + use_old_interface_ = std::get<0>(GetParam()); + cost_cache_ = std::get<1>(GetParam()); + } + bool use_old_interface_; + bool cost_cache_; +}; + +TEST_P(DBTestSharedWriteBufferAcrossCFs, SharedWriteBufferAcrossCFs) { + Options options = CurrentOptions(); + options.arena_block_size = 4096; + auto flush_listener = std::make_shared(); + options.listeners.push_back(flush_listener); + // Don't trip the listener at shutdown. + options.avoid_flush_during_shutdown = true; + + // Avoid undeterministic value by malloc_usable_size(); + // Force arena block size to 1 + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "Arena::Arena:0", [&](void* arg) { + size_t* block_size = static_cast(arg); + *block_size = 1; + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "Arena::AllocateNewBlock:0", [&](void* arg) { + std::pair* pair = + static_cast*>(arg); + *std::get<0>(*pair) = *std::get<1>(*pair); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // The total soft write buffer size is about 105000 + std::shared_ptr cache = NewLRUCache(4 * 1024 * 1024, 2); + ASSERT_LT(cache->GetUsage(), 256 * 1024); + + if (use_old_interface_) { + options.db_write_buffer_size = 120000; // this is the real limit + } else if (!cost_cache_) { + options.write_buffer_manager.reset(new WriteBufferManager(114285)); + } else { + options.write_buffer_manager.reset(new WriteBufferManager(114285, cache)); + } + options.write_buffer_size = 500000; // this is never hit + CreateAndReopenWithCF({"pikachu", "dobrynia", "nikitich"}, options); + + WriteOptions wo; + wo.disableWAL = true; + + std::function wait_flush = [&]() { + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[0])); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[2])); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[3])); + // Ensure background work is fully finished including listener callbacks + // before accessing listener state. + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + }; + + // Create some data and flush "default" and "nikitich" so that they + // are newer CFs created. + flush_listener->expected_flush_reason = FlushReason::kManualFlush; + ASSERT_OK(Put(3, Key(1), DummyString(1), wo)); + Flush(3); + ASSERT_OK(Put(3, Key(1), DummyString(1), wo)); + ASSERT_OK(Put(0, Key(1), DummyString(1), wo)); + Flush(0); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"), + static_cast(1)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"), + static_cast(1)); + + flush_listener->expected_flush_reason = FlushReason::kWriteBufferManager; + ASSERT_OK(Put(3, Key(1), DummyString(30000), wo)); + if (cost_cache_) { + ASSERT_GE(cache->GetUsage(), 256 * 1024); + ASSERT_LE(cache->GetUsage(), 2 * 256 * 1024); + } + wait_flush(); + ASSERT_OK(Put(0, Key(1), DummyString(60000), wo)); + if (cost_cache_) { + ASSERT_GE(cache->GetUsage(), 256 * 1024); + ASSERT_LE(cache->GetUsage(), 2 * 256 * 1024); + } + wait_flush(); + ASSERT_OK(Put(2, Key(1), DummyString(1), wo)); + // No flush should trigger + wait_flush(); + { + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"), + static_cast(1)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"), + static_cast(0)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "dobrynia"), + static_cast(0)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"), + static_cast(1)); + } + + // Trigger a flush. Flushing "nikitich". + ASSERT_OK(Put(3, Key(2), DummyString(30000), wo)); + wait_flush(); + ASSERT_OK(Put(0, Key(1), DummyString(1), wo)); + wait_flush(); + { + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"), + static_cast(1)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"), + static_cast(0)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "dobrynia"), + static_cast(0)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"), + static_cast(2)); + } + + // Without hitting the threshold, no flush should trigger. + ASSERT_OK(Put(2, Key(1), DummyString(30000), wo)); + wait_flush(); + ASSERT_OK(Put(2, Key(1), DummyString(1), wo)); + wait_flush(); + ASSERT_OK(Put(2, Key(1), DummyString(1), wo)); + wait_flush(); + { + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"), + static_cast(1)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"), + static_cast(0)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "dobrynia"), + static_cast(0)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"), + static_cast(2)); + } + + // Hit the write buffer limit again. "default" + // will have been flushed. + ASSERT_OK(Put(2, Key(2), DummyString(10000), wo)); + wait_flush(); + ASSERT_OK(Put(3, Key(1), DummyString(1), wo)); + wait_flush(); + ASSERT_OK(Put(0, Key(1), DummyString(1), wo)); + wait_flush(); + ASSERT_OK(Put(0, Key(1), DummyString(1), wo)); + wait_flush(); + ASSERT_OK(Put(0, Key(1), DummyString(1), wo)); + wait_flush(); + { + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"), + static_cast(2)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"), + static_cast(0)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "dobrynia"), + static_cast(0)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"), + static_cast(2)); + } + + // Trigger another flush. This time "dobrynia". "pikachu" should not + // be flushed, althrough it was never flushed. + ASSERT_OK(Put(1, Key(1), DummyString(1), wo)); + wait_flush(); + ASSERT_OK(Put(2, Key(1), DummyString(80000), wo)); + wait_flush(); + ASSERT_OK(Put(1, Key(1), DummyString(1), wo)); + wait_flush(); + ASSERT_OK(Put(2, Key(1), DummyString(1), wo)); + wait_flush(); + + { + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"), + static_cast(2)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"), + static_cast(0)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "dobrynia"), + static_cast(1)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"), + static_cast(2)); + } + if (cost_cache_) { + ASSERT_GE(cache->GetUsage(), 256 * 1024); + Close(); + options.write_buffer_manager.reset(); + last_options_.write_buffer_manager.reset(); + ASSERT_LT(cache->GetUsage(), 256 * 1024); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +INSTANTIATE_TEST_CASE_P(DBTestSharedWriteBufferAcrossCFs, + DBTestSharedWriteBufferAcrossCFs, + ::testing::Values(std::make_tuple(true, false), + std::make_tuple(false, false), + std::make_tuple(false, true))); + +TEST_F(DBTest2, SharedWriteBufferLimitAcrossDB) { + std::string dbname2 = test::PerThreadDBPath("db_shared_wb_db2"); + Options options = CurrentOptions(); + options.arena_block_size = 4096; + auto flush_listener = std::make_shared(); + options.listeners.push_back(flush_listener); + // Don't trip the listener at shutdown. + options.avoid_flush_during_shutdown = true; + // Avoid undeterministic value by malloc_usable_size(); + // Force arena block size to 1 + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "Arena::Arena:0", [&](void* arg) { + size_t* block_size = static_cast(arg); + *block_size = 1; + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "Arena::AllocateNewBlock:0", [&](void* arg) { + std::pair* pair = + static_cast*>(arg); + *std::get<0>(*pair) = *std::get<1>(*pair); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + options.write_buffer_size = 500000; // this is never hit + // Use a write buffer total size so that the soft limit is about + // 105000. + options.write_buffer_manager.reset(new WriteBufferManager(120000)); + CreateAndReopenWithCF({"cf1", "cf2"}, options); + + ASSERT_OK(DestroyDB(dbname2, options)); + DB* db2 = nullptr; + ASSERT_OK(DB::Open(options, dbname2, &db2)); + + WriteOptions wo; + wo.disableWAL = true; + + std::function wait_flush = [&]() { + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[0])); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[2])); + ASSERT_OK(static_cast(db2)->TEST_WaitForFlushMemTable()); + // Ensure background work is fully finished including listener callbacks + // before accessing listener state. + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + ASSERT_OK( + static_cast_with_check(db2)->TEST_WaitForBackgroundWork()); + }; + + // Trigger a flush on cf2 + flush_listener->expected_flush_reason = FlushReason::kWriteBufferManager; + ASSERT_OK(Put(2, Key(1), DummyString(70000), wo)); + wait_flush(); + ASSERT_OK(Put(0, Key(1), DummyString(20000), wo)); + wait_flush(); + + // Insert to DB2 + ASSERT_OK(db2->Put(wo, Key(2), DummyString(20000))); + wait_flush(); + + ASSERT_OK(Put(2, Key(1), DummyString(1), wo)); + wait_flush(); + ASSERT_OK(static_cast(db2)->TEST_WaitForFlushMemTable()); + { + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default") + + GetNumberOfSstFilesForColumnFamily(db_, "cf1") + + GetNumberOfSstFilesForColumnFamily(db_, "cf2"), + static_cast(1)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db2, "default"), + static_cast(0)); + } + + // Triggering to flush another CF in DB1 + ASSERT_OK(db2->Put(wo, Key(2), DummyString(70000))); + wait_flush(); + ASSERT_OK(Put(2, Key(1), DummyString(1), wo)); + wait_flush(); + { + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"), + static_cast(1)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "cf1"), + static_cast(0)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "cf2"), + static_cast(1)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db2, "default"), + static_cast(0)); + } + + // Triggering flush in DB2. + ASSERT_OK(db2->Put(wo, Key(3), DummyString(40000))); + wait_flush(); + ASSERT_OK(db2->Put(wo, Key(1), DummyString(1))); + wait_flush(); + ASSERT_OK(static_cast(db2)->TEST_WaitForFlushMemTable()); + { + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"), + static_cast(1)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "cf1"), + static_cast(0)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "cf2"), + static_cast(1)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db2, "default"), + static_cast(1)); + } + + delete db2; + ASSERT_OK(DestroyDB(dbname2, options)); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBTest2, TestWriteBufferNoLimitWithCache) { + Options options = CurrentOptions(); + options.arena_block_size = 4096; + std::shared_ptr cache = NewLRUCache(LRUCacheOptions( + 10000000 /* capacity */, 1 /* num_shard_bits */, + false /* strict_capacity_limit */, 0.0 /* high_pri_pool_ratio */, + nullptr /* memory_allocator */, kDefaultToAdaptiveMutex, + kDontChargeCacheMetadata)); + + options.write_buffer_size = 50000; // this is never hit + // Use a write buffer total size so that the soft limit is about + // 105000. + options.write_buffer_manager.reset(new WriteBufferManager(0, cache)); + Reopen(options); + + ASSERT_OK(Put("foo", "bar")); + // One dummy entry is 256KB. + ASSERT_GT(cache->GetUsage(), 128000); +} + +namespace { +void ValidateKeyExistence(DB* db, const std::vector& keys_must_exist, + const std::vector& keys_must_not_exist) { + // Ensure that expected keys exist + std::vector values; + if (keys_must_exist.size() > 0) { + std::vector status_list = + db->MultiGet(ReadOptions(), keys_must_exist, &values); + for (size_t i = 0; i < keys_must_exist.size(); i++) { + ASSERT_OK(status_list[i]); + } + } + + // Ensure that given keys don't exist + if (keys_must_not_exist.size() > 0) { + std::vector status_list = + db->MultiGet(ReadOptions(), keys_must_not_exist, &values); + for (size_t i = 0; i < keys_must_not_exist.size(); i++) { + ASSERT_TRUE(status_list[i].IsNotFound()); + } + } +} + +} // anonymous namespace + +TEST_F(DBTest2, WalFilterTest) { + class TestWalFilter : public WalFilter { + private: + // Processing option that is requested to be applied at the given index + WalFilter::WalProcessingOption wal_processing_option_; + // Index at which to apply wal_processing_option_ + // At other indexes default wal_processing_option::kContinueProcessing is + // returned. + size_t apply_option_at_record_index_; + // Current record index, incremented with each record encountered. + size_t current_record_index_; + + public: + TestWalFilter(WalFilter::WalProcessingOption wal_processing_option, + size_t apply_option_for_record_index) + : wal_processing_option_(wal_processing_option), + apply_option_at_record_index_(apply_option_for_record_index), + current_record_index_(0) {} + + WalProcessingOption LogRecord(const WriteBatch& /*batch*/, + WriteBatch* /*new_batch*/, + bool* /*batch_changed*/) const override { + WalFilter::WalProcessingOption option_to_return; + + if (current_record_index_ == apply_option_at_record_index_) { + option_to_return = wal_processing_option_; + } else { + option_to_return = WalProcessingOption::kContinueProcessing; + } + + // Filter is passed as a const object for RocksDB to not modify the + // object, however we modify it for our own purpose here and hence + // cast the constness away. + (const_cast(this)->current_record_index_)++; + + return option_to_return; + } + + const char* Name() const override { return "TestWalFilter"; } + }; + + // Create 3 batches with two keys each + std::vector> batch_keys(3); + + batch_keys[0].push_back("key1"); + batch_keys[0].push_back("key2"); + batch_keys[1].push_back("key3"); + batch_keys[1].push_back("key4"); + batch_keys[2].push_back("key5"); + batch_keys[2].push_back("key6"); + + // Test with all WAL processing options + for (int option = 0; + option < static_cast( + WalFilter::WalProcessingOption::kWalProcessingOptionMax); + option++) { + Options options = OptionsForLogIterTest(); + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // Write given keys in given batches + for (size_t i = 0; i < batch_keys.size(); i++) { + WriteBatch batch; + for (size_t j = 0; j < batch_keys[i].size(); j++) { + ASSERT_OK(batch.Put(handles_[0], batch_keys[i][j], DummyString(1024))); + } + ASSERT_OK(dbfull()->Write(WriteOptions(), &batch)); + } + + WalFilter::WalProcessingOption wal_processing_option = + static_cast(option); + + // Create a test filter that would apply wal_processing_option at the first + // record + size_t apply_option_for_record_index = 1; + TestWalFilter test_wal_filter(wal_processing_option, + apply_option_for_record_index); + + // Reopen database with option to use WAL filter + options = OptionsForLogIterTest(); + options.wal_filter = &test_wal_filter; + Status status = + TryReopenWithColumnFamilies({"default", "pikachu"}, options); + if (wal_processing_option == + WalFilter::WalProcessingOption::kCorruptedRecord) { + ASSERT_NOK(status); + // In case of corruption we can turn off paranoid_checks to reopen + // databse + options.paranoid_checks = false; + ReopenWithColumnFamilies({"default", "pikachu"}, options); + } else { + ASSERT_OK(status); + } + + // Compute which keys we expect to be found + // and which we expect not to be found after recovery. + std::vector keys_must_exist; + std::vector keys_must_not_exist; + switch (wal_processing_option) { + case WalFilter::WalProcessingOption::kCorruptedRecord: + case WalFilter::WalProcessingOption::kContinueProcessing: { + fprintf(stderr, "Testing with complete WAL processing\n"); + // we expect all records to be processed + for (size_t i = 0; i < batch_keys.size(); i++) { + for (size_t j = 0; j < batch_keys[i].size(); j++) { + keys_must_exist.push_back(Slice(batch_keys[i][j])); + } + } + break; + } + case WalFilter::WalProcessingOption::kIgnoreCurrentRecord: { + fprintf(stderr, + "Testing with ignoring record %" ROCKSDB_PRIszt " only\n", + apply_option_for_record_index); + // We expect the record with apply_option_for_record_index to be not + // found. + for (size_t i = 0; i < batch_keys.size(); i++) { + for (size_t j = 0; j < batch_keys[i].size(); j++) { + if (i == apply_option_for_record_index) { + keys_must_not_exist.push_back(Slice(batch_keys[i][j])); + } else { + keys_must_exist.push_back(Slice(batch_keys[i][j])); + } + } + } + break; + } + case WalFilter::WalProcessingOption::kStopReplay: { + fprintf(stderr, + "Testing with stopping replay from record %" ROCKSDB_PRIszt + "\n", + apply_option_for_record_index); + // We expect records beyond apply_option_for_record_index to be not + // found. + for (size_t i = 0; i < batch_keys.size(); i++) { + for (size_t j = 0; j < batch_keys[i].size(); j++) { + if (i >= apply_option_for_record_index) { + keys_must_not_exist.push_back(Slice(batch_keys[i][j])); + } else { + keys_must_exist.push_back(Slice(batch_keys[i][j])); + } + } + } + break; + } + default: + FAIL(); // unhandled case + } + + bool checked_after_reopen = false; + + while (true) { + // Ensure that expected keys exists + // and not expected keys don't exist after recovery + ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist); + + if (checked_after_reopen) { + break; + } + + // reopen database again to make sure previous log(s) are not used + //(even if they were skipped) + // reopn database with option to use WAL filter + options = OptionsForLogIterTest(); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + + checked_after_reopen = true; + } + } +} + +TEST_F(DBTest2, WalFilterTestWithChangeBatch) { + class ChangeBatchHandler : public WriteBatch::Handler { + private: + // Batch to insert keys in + WriteBatch* new_write_batch_; + // Number of keys to add in the new batch + size_t num_keys_to_add_in_new_batch_; + // Number of keys added to new batch + size_t num_keys_added_; + + public: + ChangeBatchHandler(WriteBatch* new_write_batch, + size_t num_keys_to_add_in_new_batch) + : new_write_batch_(new_write_batch), + num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch), + num_keys_added_(0) {} + void Put(const Slice& key, const Slice& value) override { + if (num_keys_added_ < num_keys_to_add_in_new_batch_) { + ASSERT_OK(new_write_batch_->Put(key, value)); + ++num_keys_added_; + } + } + }; + + class TestWalFilterWithChangeBatch : public WalFilter { + private: + // Index at which to start changing records + size_t change_records_from_index_; + // Number of keys to add in the new batch + size_t num_keys_to_add_in_new_batch_; + // Current record index, incremented with each record encountered. + size_t current_record_index_; + + public: + TestWalFilterWithChangeBatch(size_t change_records_from_index, + size_t num_keys_to_add_in_new_batch) + : change_records_from_index_(change_records_from_index), + num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch), + current_record_index_(0) {} + + WalProcessingOption LogRecord(const WriteBatch& batch, + WriteBatch* new_batch, + bool* batch_changed) const override { + if (current_record_index_ >= change_records_from_index_) { + ChangeBatchHandler handler(new_batch, num_keys_to_add_in_new_batch_); + Status s = batch.Iterate(&handler); + if (s.ok()) { + *batch_changed = true; + } else { + assert(false); + } + } + + // Filter is passed as a const object for RocksDB to not modify the + // object, however we modify it for our own purpose here and hence + // cast the constness away. + (const_cast(this) + ->current_record_index_)++; + + return WalProcessingOption::kContinueProcessing; + } + + const char* Name() const override { return "TestWalFilterWithChangeBatch"; } + }; + + std::vector> batch_keys(3); + + batch_keys[0].push_back("key1"); + batch_keys[0].push_back("key2"); + batch_keys[1].push_back("key3"); + batch_keys[1].push_back("key4"); + batch_keys[2].push_back("key5"); + batch_keys[2].push_back("key6"); + + Options options = OptionsForLogIterTest(); + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // Write given keys in given batches + for (size_t i = 0; i < batch_keys.size(); i++) { + WriteBatch batch; + for (size_t j = 0; j < batch_keys[i].size(); j++) { + ASSERT_OK(batch.Put(handles_[0], batch_keys[i][j], DummyString(1024))); + } + ASSERT_OK(dbfull()->Write(WriteOptions(), &batch)); + } + + // Create a test filter that would apply wal_processing_option at the first + // record + size_t change_records_from_index = 1; + size_t num_keys_to_add_in_new_batch = 1; + TestWalFilterWithChangeBatch test_wal_filter_with_change_batch( + change_records_from_index, num_keys_to_add_in_new_batch); + + // Reopen database with option to use WAL filter + options = OptionsForLogIterTest(); + options.wal_filter = &test_wal_filter_with_change_batch; + ReopenWithColumnFamilies({"default", "pikachu"}, options); + + // Ensure that all keys exist before change_records_from_index_ + // And after that index only single key exists + // as our filter adds only single key for each batch + std::vector keys_must_exist; + std::vector keys_must_not_exist; + + for (size_t i = 0; i < batch_keys.size(); i++) { + for (size_t j = 0; j < batch_keys[i].size(); j++) { + if (i >= change_records_from_index && j >= num_keys_to_add_in_new_batch) { + keys_must_not_exist.push_back(Slice(batch_keys[i][j])); + } else { + keys_must_exist.push_back(Slice(batch_keys[i][j])); + } + } + } + + bool checked_after_reopen = false; + + while (true) { + // Ensure that expected keys exists + // and not expected keys don't exist after recovery + ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist); + + if (checked_after_reopen) { + break; + } + + // reopen database again to make sure previous log(s) are not used + //(even if they were skipped) + // reopn database with option to use WAL filter + options = OptionsForLogIterTest(); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + + checked_after_reopen = true; + } +} + +TEST_F(DBTest2, WalFilterTestWithChangeBatchExtraKeys) { + class TestWalFilterWithChangeBatchAddExtraKeys : public WalFilter { + public: + WalProcessingOption LogRecord(const WriteBatch& batch, + WriteBatch* new_batch, + bool* batch_changed) const override { + *new_batch = batch; + Status s = new_batch->Put("key_extra", "value_extra"); + if (s.ok()) { + *batch_changed = true; + } else { + assert(false); + } + return WalProcessingOption::kContinueProcessing; + } + + const char* Name() const override { + return "WalFilterTestWithChangeBatchExtraKeys"; + } + }; + + std::vector> batch_keys(3); + + batch_keys[0].push_back("key1"); + batch_keys[0].push_back("key2"); + batch_keys[1].push_back("key3"); + batch_keys[1].push_back("key4"); + batch_keys[2].push_back("key5"); + batch_keys[2].push_back("key6"); + + Options options = OptionsForLogIterTest(); + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // Write given keys in given batches + for (size_t i = 0; i < batch_keys.size(); i++) { + WriteBatch batch; + for (size_t j = 0; j < batch_keys[i].size(); j++) { + ASSERT_OK(batch.Put(handles_[0], batch_keys[i][j], DummyString(1024))); + } + ASSERT_OK(dbfull()->Write(WriteOptions(), &batch)); + } + + // Create a test filter that would add extra keys + TestWalFilterWithChangeBatchAddExtraKeys test_wal_filter_extra_keys; + + // Reopen database with option to use WAL filter + options = OptionsForLogIterTest(); + options.wal_filter = &test_wal_filter_extra_keys; + Status status = TryReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_TRUE(status.IsNotSupported()); + + // Reopen without filter, now reopen should succeed - previous + // attempt to open must not have altered the db. + options = OptionsForLogIterTest(); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + + std::vector keys_must_exist; + std::vector keys_must_not_exist; // empty vector + + for (size_t i = 0; i < batch_keys.size(); i++) { + for (size_t j = 0; j < batch_keys[i].size(); j++) { + keys_must_exist.push_back(Slice(batch_keys[i][j])); + } + } + + ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist); +} + +TEST_F(DBTest2, WalFilterTestWithColumnFamilies) { + class TestWalFilterWithColumnFamilies : public WalFilter { + private: + // column_family_id -> log_number map (provided to WALFilter) + std::map cf_log_number_map_; + // column_family_name -> column_family_id map (provided to WALFilter) + std::map cf_name_id_map_; + // column_family_name -> keys_found_in_wal map + // We store keys that are applicable to the column_family + // during recovery (i.e. aren't already flushed to SST file(s)) + // for verification against the keys we expect. + std::map> cf_wal_keys_; + + public: + void ColumnFamilyLogNumberMap( + const std::map& cf_lognumber_map, + const std::map& cf_name_id_map) override { + cf_log_number_map_ = cf_lognumber_map; + cf_name_id_map_ = cf_name_id_map; + } + + WalProcessingOption LogRecordFound(unsigned long long log_number, + const std::string& /*log_file_name*/, + const WriteBatch& batch, + WriteBatch* /*new_batch*/, + bool* /*batch_changed*/) override { + class LogRecordBatchHandler : public WriteBatch::Handler { + private: + const std::map& cf_log_number_map_; + std::map>& cf_wal_keys_; + unsigned long long log_number_; + + public: + LogRecordBatchHandler( + unsigned long long current_log_number, + const std::map& cf_log_number_map, + std::map>& cf_wal_keys) + : cf_log_number_map_(cf_log_number_map), + cf_wal_keys_(cf_wal_keys), + log_number_(current_log_number) {} + + Status PutCF(uint32_t column_family_id, const Slice& key, + const Slice& /*value*/) override { + auto it = cf_log_number_map_.find(column_family_id); + assert(it != cf_log_number_map_.end()); + unsigned long long log_number_for_cf = it->second; + // If the current record is applicable for column_family_id + // (i.e. isn't flushed to SST file(s) for column_family_id) + // add it to the cf_wal_keys_ map for verification. + if (log_number_ >= log_number_for_cf) { + cf_wal_keys_[column_family_id].push_back( + std::string(key.data(), key.size())); + } + return Status::OK(); + } + } handler(log_number, cf_log_number_map_, cf_wal_keys_); + + Status s = batch.Iterate(&handler); + if (!s.ok()) { + // TODO(AR) is this ok? + return WalProcessingOption::kCorruptedRecord; + } + + return WalProcessingOption::kContinueProcessing; + } + + const char* Name() const override { + return "WalFilterTestWithColumnFamilies"; + } + + const std::map>& GetColumnFamilyKeys() { + return cf_wal_keys_; + } + + const std::map& GetColumnFamilyNameIdMap() { + return cf_name_id_map_; + } + }; + + std::vector> batch_keys_pre_flush(3); + + batch_keys_pre_flush[0].push_back("key1"); + batch_keys_pre_flush[0].push_back("key2"); + batch_keys_pre_flush[1].push_back("key3"); + batch_keys_pre_flush[1].push_back("key4"); + batch_keys_pre_flush[2].push_back("key5"); + batch_keys_pre_flush[2].push_back("key6"); + + Options options = OptionsForLogIterTest(); + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // Write given keys in given batches + for (size_t i = 0; i < batch_keys_pre_flush.size(); i++) { + WriteBatch batch; + for (size_t j = 0; j < batch_keys_pre_flush[i].size(); j++) { + ASSERT_OK(batch.Put(handles_[0], batch_keys_pre_flush[i][j], + DummyString(1024))); + ASSERT_OK(batch.Put(handles_[1], batch_keys_pre_flush[i][j], + DummyString(1024))); + } + ASSERT_OK(dbfull()->Write(WriteOptions(), &batch)); + } + + // Flush default column-family + ASSERT_OK(db_->Flush(FlushOptions(), handles_[0])); + + // Do some more writes + std::vector> batch_keys_post_flush(3); + + batch_keys_post_flush[0].push_back("key7"); + batch_keys_post_flush[0].push_back("key8"); + batch_keys_post_flush[1].push_back("key9"); + batch_keys_post_flush[1].push_back("key10"); + batch_keys_post_flush[2].push_back("key11"); + batch_keys_post_flush[2].push_back("key12"); + + // Write given keys in given batches + for (size_t i = 0; i < batch_keys_post_flush.size(); i++) { + WriteBatch batch; + for (size_t j = 0; j < batch_keys_post_flush[i].size(); j++) { + ASSERT_OK(batch.Put(handles_[0], batch_keys_post_flush[i][j], + DummyString(1024))); + ASSERT_OK(batch.Put(handles_[1], batch_keys_post_flush[i][j], + DummyString(1024))); + } + ASSERT_OK(dbfull()->Write(WriteOptions(), &batch)); + } + + // On Recovery we should only find the second batch applicable to default CF + // But both batches applicable to pikachu CF + + // Create a test filter that would add extra keys + TestWalFilterWithColumnFamilies test_wal_filter_column_families; + + // Reopen database with option to use WAL filter + options = OptionsForLogIterTest(); + options.wal_filter = &test_wal_filter_column_families; + Status status = TryReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_TRUE(status.ok()); + + // verify that handles_[0] only has post_flush keys + // while handles_[1] has pre and post flush keys + auto cf_wal_keys = test_wal_filter_column_families.GetColumnFamilyKeys(); + auto name_id_map = test_wal_filter_column_families.GetColumnFamilyNameIdMap(); + size_t index = 0; + auto keys_cf = cf_wal_keys[name_id_map[kDefaultColumnFamilyName]]; + // default column-family, only post_flush keys are expected + for (size_t i = 0; i < batch_keys_post_flush.size(); i++) { + for (size_t j = 0; j < batch_keys_post_flush[i].size(); j++) { + Slice key_from_the_log(keys_cf[index++]); + Slice batch_key(batch_keys_post_flush[i][j]); + ASSERT_EQ(key_from_the_log.compare(batch_key), 0); + } + } + ASSERT_EQ(index, keys_cf.size()); + + index = 0; + keys_cf = cf_wal_keys[name_id_map["pikachu"]]; + // pikachu column-family, all keys are expected + for (size_t i = 0; i < batch_keys_pre_flush.size(); i++) { + for (size_t j = 0; j < batch_keys_pre_flush[i].size(); j++) { + Slice key_from_the_log(keys_cf[index++]); + Slice batch_key(batch_keys_pre_flush[i][j]); + ASSERT_EQ(key_from_the_log.compare(batch_key), 0); + } + } + + for (size_t i = 0; i < batch_keys_post_flush.size(); i++) { + for (size_t j = 0; j < batch_keys_post_flush[i].size(); j++) { + Slice key_from_the_log(keys_cf[index++]); + Slice batch_key(batch_keys_post_flush[i][j]); + ASSERT_EQ(key_from_the_log.compare(batch_key), 0); + } + } + ASSERT_EQ(index, keys_cf.size()); +} + +TEST_F(DBTest2, PresetCompressionDict) { + // Verifies that compression ratio improves when dictionary is enabled, and + // improves even further when the dictionary is trained by ZSTD. + const size_t kBlockSizeBytes = 4 << 10; + const size_t kL0FileBytes = 128 << 10; + const size_t kApproxPerBlockOverheadBytes = 50; + const int kNumL0Files = 5; + + Options options; + // Make sure to use any custom env that the test is configured with. + options.env = CurrentOptions().env; + options.allow_concurrent_memtable_write = false; + options.arena_block_size = kBlockSizeBytes; + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.level0_file_num_compaction_trigger = kNumL0Files; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kL0FileBytes / kBlockSizeBytes)); + options.num_levels = 2; + options.target_file_size_base = kL0FileBytes; + options.target_file_size_multiplier = 2; + options.write_buffer_size = kL0FileBytes; + BlockBasedTableOptions table_options; + table_options.block_size = kBlockSizeBytes; + std::vector compression_types; + if (Zlib_Supported()) { + compression_types.push_back(kZlibCompression); + } +#if LZ4_VERSION_NUMBER >= 10400 // r124+ + compression_types.push_back(kLZ4Compression); + compression_types.push_back(kLZ4HCCompression); +#endif // LZ4_VERSION_NUMBER >= 10400 + if (ZSTD_Supported()) { + compression_types.push_back(kZSTD); + } + + enum DictionaryTypes : int { + kWithoutDict, + kWithDict, + kWithZSTDfinalizeDict, + kWithZSTDTrainedDict, + kDictEnd, + }; + + for (auto compression_type : compression_types) { + options.compression = compression_type; + size_t bytes_without_dict = 0; + size_t bytes_with_dict = 0; + size_t bytes_with_zstd_finalize_dict = 0; + size_t bytes_with_zstd_trained_dict = 0; + for (int i = kWithoutDict; i < kDictEnd; i++) { + // First iteration: compress without preset dictionary + // Second iteration: compress with preset dictionary + // Third iteration (zstd only): compress with zstd-trained dictionary + // + // To make sure the compression dictionary has the intended effect, we + // verify the compressed size is smaller in successive iterations. Also in + // the non-first iterations, verify the data we get out is the same data + // we put in. + switch (i) { + case kWithoutDict: + options.compression_opts.max_dict_bytes = 0; + options.compression_opts.zstd_max_train_bytes = 0; + break; + case kWithDict: + options.compression_opts.max_dict_bytes = kBlockSizeBytes; + options.compression_opts.zstd_max_train_bytes = 0; + break; + case kWithZSTDfinalizeDict: + if (compression_type != kZSTD || + !ZSTD_FinalizeDictionarySupported()) { + continue; + } + options.compression_opts.max_dict_bytes = kBlockSizeBytes; + options.compression_opts.zstd_max_train_bytes = kL0FileBytes; + options.compression_opts.use_zstd_dict_trainer = false; + break; + case kWithZSTDTrainedDict: + if (compression_type != kZSTD || !ZSTD_TrainDictionarySupported()) { + continue; + } + options.compression_opts.max_dict_bytes = kBlockSizeBytes; + options.compression_opts.zstd_max_train_bytes = kL0FileBytes; + options.compression_opts.use_zstd_dict_trainer = true; + break; + default: + assert(false); + } + + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + CreateAndReopenWithCF({"pikachu"}, options); + Random rnd(301); + std::string seq_datas[10]; + for (int j = 0; j < 10; ++j) { + seq_datas[j] = + rnd.RandomString(kBlockSizeBytes - kApproxPerBlockOverheadBytes); + } + + ASSERT_EQ(0, NumTableFilesAtLevel(0, 1)); + for (int j = 0; j < kNumL0Files; ++j) { + for (size_t k = 0; k < kL0FileBytes / kBlockSizeBytes + 1; ++k) { + auto key_num = j * (kL0FileBytes / kBlockSizeBytes) + k; + ASSERT_OK(Put(1, Key(static_cast(key_num)), + seq_datas[(key_num / 10) % 10])); + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); + ASSERT_EQ(j + 1, NumTableFilesAtLevel(0, 1)); + } + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1], + true /* disallow_trivial_move */)); + ASSERT_EQ(0, NumTableFilesAtLevel(0, 1)); + ASSERT_GT(NumTableFilesAtLevel(1, 1), 0); + + // Get the live sst files size + size_t total_sst_bytes = TotalSize(1); + if (i == kWithoutDict) { + bytes_without_dict = total_sst_bytes; + } else if (i == kWithDict) { + bytes_with_dict = total_sst_bytes; + } else if (i == kWithZSTDfinalizeDict) { + bytes_with_zstd_finalize_dict = total_sst_bytes; + } else if (i == kWithZSTDTrainedDict) { + bytes_with_zstd_trained_dict = total_sst_bytes; + } + + for (size_t j = 0; j < kNumL0Files * (kL0FileBytes / kBlockSizeBytes); + j++) { + ASSERT_EQ(seq_datas[(j / 10) % 10], Get(1, Key(static_cast(j)))); + } + if (i == kWithDict) { + ASSERT_GT(bytes_without_dict, bytes_with_dict); + } else if (i == kWithZSTDTrainedDict) { + // In zstd compression, it is sometimes possible that using a finalized + // dictionary does not get as good a compression ratio as raw content + // dictionary. But using a dictionary should always get better + // compression ratio than not using one. + ASSERT_TRUE(bytes_with_dict > bytes_with_zstd_finalize_dict || + bytes_without_dict > bytes_with_zstd_finalize_dict); + } else if (i == kWithZSTDTrainedDict) { + // In zstd compression, it is sometimes possible that using a trained + // dictionary does not get as good a compression ratio as without + // training. + // But using a dictionary (with or without training) should always get + // better compression ratio than not using one. + ASSERT_TRUE(bytes_with_dict > bytes_with_zstd_trained_dict || + bytes_without_dict > bytes_with_zstd_trained_dict); + } + + DestroyAndReopen(options); + } + } +} + +TEST_F(DBTest2, PresetCompressionDictLocality) { + if (!ZSTD_Supported()) { + return; + } + // Verifies that compression dictionary is generated from local data. The + // verification simply checks all output SSTs have different compression + // dictionaries. We do not verify effectiveness as that'd likely be flaky in + // the future. + const int kNumEntriesPerFile = 1 << 10; // 1KB + const int kNumBytesPerEntry = 1 << 10; // 1KB + const int kNumFiles = 4; + Options options = CurrentOptions(); + options.compression = kZSTD; + options.compression_opts.max_dict_bytes = 1 << 14; // 16KB + options.compression_opts.zstd_max_train_bytes = 1 << 18; // 256KB + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.target_file_size_base = kNumEntriesPerFile * kNumBytesPerEntry; + BlockBasedTableOptions table_options; + table_options.cache_index_and_filter_blocks = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + + Random rnd(301); + for (int i = 0; i < kNumFiles; ++i) { + for (int j = 0; j < kNumEntriesPerFile; ++j) { + ASSERT_OK(Put(Key(i * kNumEntriesPerFile + j), + rnd.RandomString(kNumBytesPerEntry))); + } + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + ASSERT_EQ(NumTableFilesAtLevel(1), i + 1); + } + + // Store all the dictionaries generated during a full compaction. + std::vector compression_dicts; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTableBuilder::WriteCompressionDictBlock:RawDict", + [&](void* arg) { + compression_dicts.emplace_back(static_cast(arg)->ToString()); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + CompactRangeOptions compact_range_opts; + compact_range_opts.bottommost_level_compaction = + BottommostLevelCompaction::kForceOptimized; + ASSERT_OK(db_->CompactRange(compact_range_opts, nullptr, nullptr)); + + // Dictionary compression should not be so good as to compress four totally + // random files into one. If it does then there's probably something wrong + // with the test. + ASSERT_GT(NumTableFilesAtLevel(1), 1); + + // Furthermore, there should be one compression dictionary generated per file. + // And they should all be different from each other. + ASSERT_EQ(NumTableFilesAtLevel(1), + static_cast(compression_dicts.size())); + for (size_t i = 1; i < compression_dicts.size(); ++i) { + std::string& a = compression_dicts[i - 1]; + std::string& b = compression_dicts[i]; + size_t alen = a.size(); + size_t blen = b.size(); + ASSERT_TRUE(alen != blen || memcmp(a.data(), b.data(), alen) != 0); + } +} + +class PresetCompressionDictTest + : public DBTestBase, + public testing::WithParamInterface> { + public: + PresetCompressionDictTest() + : DBTestBase("db_test2", false /* env_do_fsync */), + compression_type_(std::get<0>(GetParam())), + bottommost_(std::get<1>(GetParam())) {} + + protected: + const CompressionType compression_type_; + const bool bottommost_; +}; + +INSTANTIATE_TEST_CASE_P( + DBTest2, PresetCompressionDictTest, + ::testing::Combine(::testing::ValuesIn(GetSupportedDictCompressions()), + ::testing::Bool())); + +TEST_P(PresetCompressionDictTest, Flush) { + // Verifies that dictionary is generated and written during flush only when + // `ColumnFamilyOptions::compression` enables dictionary. Also verifies the + // size of the dictionary is within expectations according to the limit on + // buffering set by `CompressionOptions::max_dict_buffer_bytes`. + const size_t kValueLen = 256; + const size_t kKeysPerFile = 1 << 10; + const size_t kDictLen = 16 << 10; + const size_t kBlockLen = 4 << 10; + + Options options = CurrentOptions(); + if (bottommost_) { + options.bottommost_compression = compression_type_; + options.bottommost_compression_opts.enabled = true; + options.bottommost_compression_opts.max_dict_bytes = kDictLen; + options.bottommost_compression_opts.max_dict_buffer_bytes = kBlockLen; + } else { + options.compression = compression_type_; + options.compression_opts.max_dict_bytes = kDictLen; + options.compression_opts.max_dict_buffer_bytes = kBlockLen; + } + options.memtable_factory.reset(test::NewSpecialSkipListFactory(kKeysPerFile)); + options.statistics = CreateDBStatistics(); + BlockBasedTableOptions bbto; + bbto.block_size = kBlockLen; + bbto.cache_index_and_filter_blocks = true; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + Reopen(options); + + Random rnd(301); + for (size_t i = 0; i <= kKeysPerFile; ++i) { + ASSERT_OK(Put(Key(static_cast(i)), rnd.RandomString(kValueLen))); + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + + // We can use `BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT` to detect whether a + // compression dictionary exists since dictionaries would be preloaded when + // the flush finishes. + if (bottommost_) { + // Flush is never considered bottommost. This should change in the future + // since flushed files may have nothing underneath them, like the one in + // this test case. + ASSERT_EQ( + TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT), + 0); + } else { + ASSERT_GT( + TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT), + 0); + // TODO(ajkr): fix the below assertion to work with ZSTD. The expectation on + // number of bytes needs to be adjusted in case the cached block is in + // ZSTD's digested dictionary format. + if (compression_type_ != kZSTD && + compression_type_ != kZSTDNotFinalCompression) { + // Although we limited buffering to `kBlockLen`, there may be up to two + // blocks of data included in the dictionary since we only check limit + // after each block is built. + ASSERT_LE(TestGetTickerCount(options, + BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT), + 2 * kBlockLen); + } + } +} + +TEST_P(PresetCompressionDictTest, CompactNonBottommost) { + // Verifies that dictionary is generated and written during compaction to + // non-bottommost level only when `ColumnFamilyOptions::compression` enables + // dictionary. Also verifies the size of the dictionary is within expectations + // according to the limit on buffering set by + // `CompressionOptions::max_dict_buffer_bytes`. + const size_t kValueLen = 256; + const size_t kKeysPerFile = 1 << 10; + const size_t kDictLen = 16 << 10; + const size_t kBlockLen = 4 << 10; + + Options options = CurrentOptions(); + if (bottommost_) { + options.bottommost_compression = compression_type_; + options.bottommost_compression_opts.enabled = true; + options.bottommost_compression_opts.max_dict_bytes = kDictLen; + options.bottommost_compression_opts.max_dict_buffer_bytes = kBlockLen; + } else { + options.compression = compression_type_; + options.compression_opts.max_dict_bytes = kDictLen; + options.compression_opts.max_dict_buffer_bytes = kBlockLen; + } + options.disable_auto_compactions = true; + options.statistics = CreateDBStatistics(); + BlockBasedTableOptions bbto; + bbto.block_size = kBlockLen; + bbto.cache_index_and_filter_blocks = true; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + Reopen(options); + + Random rnd(301); + for (size_t j = 0; j <= kKeysPerFile; ++j) { + ASSERT_OK(Put(Key(static_cast(j)), rnd.RandomString(kValueLen))); + } + ASSERT_OK(Flush()); + MoveFilesToLevel(2); + + for (int i = 0; i < 2; ++i) { + for (size_t j = 0; j <= kKeysPerFile; ++j) { + ASSERT_OK(Put(Key(static_cast(j)), rnd.RandomString(kValueLen))); + } + ASSERT_OK(Flush()); + } + ASSERT_EQ("2,0,1", FilesPerLevel(0)); + + uint64_t prev_compression_dict_bytes_inserted = + TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT); + // This L0->L1 compaction merges the two L0 files into L1. The produced L1 + // file is not bottommost due to the existing L2 file covering the same key- + // range. + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr)); + ASSERT_EQ("0,1,1", FilesPerLevel(0)); + // We can use `BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT` to detect whether a + // compression dictionary exists since dictionaries would be preloaded when + // the compaction finishes. + if (bottommost_) { + ASSERT_EQ( + TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT), + prev_compression_dict_bytes_inserted); + } else { + ASSERT_GT( + TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT), + prev_compression_dict_bytes_inserted); + // TODO(ajkr): fix the below assertion to work with ZSTD. The expectation on + // number of bytes needs to be adjusted in case the cached block is in + // ZSTD's digested dictionary format. + if (compression_type_ != kZSTD && + compression_type_ != kZSTDNotFinalCompression) { + // Although we limited buffering to `kBlockLen`, there may be up to two + // blocks of data included in the dictionary since we only check limit + // after each block is built. + ASSERT_LE(TestGetTickerCount(options, + BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT), + prev_compression_dict_bytes_inserted + 2 * kBlockLen); + } + } +} + +TEST_P(PresetCompressionDictTest, CompactBottommost) { + // Verifies that dictionary is generated and written during compaction to + // non-bottommost level only when either `ColumnFamilyOptions::compression` or + // `ColumnFamilyOptions::bottommost_compression` enables dictionary. Also + // verifies the size of the dictionary is within expectations according to the + // limit on buffering set by `CompressionOptions::max_dict_buffer_bytes`. + const size_t kValueLen = 256; + const size_t kKeysPerFile = 1 << 10; + const size_t kDictLen = 16 << 10; + const size_t kBlockLen = 4 << 10; + + Options options = CurrentOptions(); + if (bottommost_) { + options.bottommost_compression = compression_type_; + options.bottommost_compression_opts.enabled = true; + options.bottommost_compression_opts.max_dict_bytes = kDictLen; + options.bottommost_compression_opts.max_dict_buffer_bytes = kBlockLen; + } else { + options.compression = compression_type_; + options.compression_opts.max_dict_bytes = kDictLen; + options.compression_opts.max_dict_buffer_bytes = kBlockLen; + } + options.disable_auto_compactions = true; + options.statistics = CreateDBStatistics(); + BlockBasedTableOptions bbto; + bbto.block_size = kBlockLen; + bbto.cache_index_and_filter_blocks = true; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + Reopen(options); + + Random rnd(301); + for (int i = 0; i < 2; ++i) { + for (size_t j = 0; j <= kKeysPerFile; ++j) { + ASSERT_OK(Put(Key(static_cast(j)), rnd.RandomString(kValueLen))); + } + ASSERT_OK(Flush()); + } + ASSERT_EQ("2", FilesPerLevel(0)); + + uint64_t prev_compression_dict_bytes_inserted = + TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT); + CompactRangeOptions cro; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,1", FilesPerLevel(0)); + ASSERT_GT( + TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT), + prev_compression_dict_bytes_inserted); + // TODO(ajkr): fix the below assertion to work with ZSTD. The expectation on + // number of bytes needs to be adjusted in case the cached block is in ZSTD's + // digested dictionary format. + if (compression_type_ != kZSTD && + compression_type_ != kZSTDNotFinalCompression) { + // Although we limited buffering to `kBlockLen`, there may be up to two + // blocks of data included in the dictionary since we only check limit after + // each block is built. + ASSERT_LE( + TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT), + prev_compression_dict_bytes_inserted + 2 * kBlockLen); + } +} + +class CompactionCompressionListener : public EventListener { + public: + explicit CompactionCompressionListener(Options* db_options) + : db_options_(db_options) {} + + void OnCompactionCompleted(DB* db, const CompactionJobInfo& ci) override { + // Figure out last level with files + int bottommost_level = 0; + for (int level = 0; level < db->NumberLevels(); level++) { + std::string files_at_level; + ASSERT_TRUE( + db->GetProperty("rocksdb.num-files-at-level" + std::to_string(level), + &files_at_level)); + if (files_at_level != "0") { + bottommost_level = level; + } + } + + if (db_options_->bottommost_compression != kDisableCompressionOption && + ci.output_level == bottommost_level) { + ASSERT_EQ(ci.compression, db_options_->bottommost_compression); + } else if (db_options_->compression_per_level.size() != 0) { + ASSERT_EQ(ci.compression, + db_options_->compression_per_level[ci.output_level]); + } else { + ASSERT_EQ(ci.compression, db_options_->compression); + } + max_level_checked = std::max(max_level_checked, ci.output_level); + } + + int max_level_checked = 0; + const Options* db_options_; +}; + +enum CompressionFailureType { + kTestCompressionFail, + kTestDecompressionFail, + kTestDecompressionCorruption +}; + +class CompressionFailuresTest + : public DBTest2, + public testing::WithParamInterface> { + public: + CompressionFailuresTest() { + std::tie(compression_failure_type_, compression_type_, + compression_max_dict_bytes_, compression_parallel_threads_) = + GetParam(); + } + + CompressionFailureType compression_failure_type_ = kTestCompressionFail; + CompressionType compression_type_ = kNoCompression; + uint32_t compression_max_dict_bytes_ = 0; + uint32_t compression_parallel_threads_ = 0; +}; + +INSTANTIATE_TEST_CASE_P( + DBTest2, CompressionFailuresTest, + ::testing::Combine(::testing::Values(kTestCompressionFail, + kTestDecompressionFail, + kTestDecompressionCorruption), + ::testing::ValuesIn(GetSupportedCompressions()), + ::testing::Values(0, 10), ::testing::Values(1, 4))); + +TEST_P(CompressionFailuresTest, CompressionFailures) { + if (compression_type_ == kNoCompression) { + return; + } + + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = 2; + options.max_bytes_for_level_base = 1024; + options.max_bytes_for_level_multiplier = 2; + options.num_levels = 7; + options.max_background_compactions = 1; + options.target_file_size_base = 512; + + BlockBasedTableOptions table_options; + table_options.block_size = 512; + table_options.verify_compression = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + options.compression = compression_type_; + options.compression_opts.parallel_threads = compression_parallel_threads_; + options.compression_opts.max_dict_bytes = compression_max_dict_bytes_; + options.bottommost_compression_opts.parallel_threads = + compression_parallel_threads_; + options.bottommost_compression_opts.max_dict_bytes = + compression_max_dict_bytes_; + + if (compression_failure_type_ == kTestCompressionFail) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompressData:TamperWithReturnValue", [](void* arg) { + bool* ret = static_cast(arg); + *ret = false; + }); + } else if (compression_failure_type_ == kTestDecompressionFail) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "UncompressBlockData:TamperWithReturnValue", [](void* arg) { + Status* ret = static_cast(arg); + ASSERT_OK(*ret); + *ret = Status::Corruption("kTestDecompressionFail"); + }); + } else if (compression_failure_type_ == kTestDecompressionCorruption) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "UncompressBlockData:" + "TamperWithDecompressionOutput", + [](void* arg) { + BlockContents* contents = static_cast(arg); + // Ensure uncompressed data != original data + const size_t len = contents->data.size() + 1; + std::unique_ptr fake_data(new char[len]()); + *contents = BlockContents(std::move(fake_data), len); + }); + } + + std::map key_value_written; + + const int kKeySize = 5; + const int kValUnitSize = 16; + const int kValSize = 256; + Random rnd(405); + + Status s = Status::OK(); + + DestroyAndReopen(options); + // Write 10 random files + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 5; j++) { + std::string key = rnd.RandomString(kKeySize); + // Ensure good compression ratio + std::string valueUnit = rnd.RandomString(kValUnitSize); + std::string value; + for (int k = 0; k < kValSize; k += kValUnitSize) { + value += valueUnit; + } + s = Put(key, value); + if (compression_failure_type_ == kTestCompressionFail) { + key_value_written[key] = value; + ASSERT_OK(s); + } + } + s = Flush(); + if (compression_failure_type_ == kTestCompressionFail) { + ASSERT_OK(s); + } + s = dbfull()->TEST_WaitForCompact(); + if (compression_failure_type_ == kTestCompressionFail) { + ASSERT_OK(s); + } + if (i == 4) { + // Make compression fail at the mid of table building + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + } + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + if (compression_failure_type_ == kTestCompressionFail) { + // Should be kNoCompression, check content consistency + std::unique_ptr db_iter(db_->NewIterator(ReadOptions())); + for (db_iter->SeekToFirst(); db_iter->Valid(); db_iter->Next()) { + std::string key = db_iter->key().ToString(); + std::string value = db_iter->value().ToString(); + ASSERT_NE(key_value_written.find(key), key_value_written.end()); + ASSERT_EQ(key_value_written[key], value); + key_value_written.erase(key); + } + ASSERT_EQ(0, key_value_written.size()); + } else if (compression_failure_type_ == kTestDecompressionFail) { + ASSERT_EQ(std::string(s.getState()), + "Could not decompress: kTestDecompressionFail"); + } else if (compression_failure_type_ == kTestDecompressionCorruption) { + ASSERT_EQ(std::string(s.getState()), + "Decompressed block did not match pre-compression block"); + } +} + +TEST_F(DBTest2, CompressionOptions) { + if (!Zlib_Supported() || !Snappy_Supported()) { + return; + } + + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = 2; + options.max_bytes_for_level_base = 100; + options.max_bytes_for_level_multiplier = 2; + options.num_levels = 7; + options.max_background_compactions = 1; + + CompactionCompressionListener* listener = + new CompactionCompressionListener(&options); + options.listeners.emplace_back(listener); + + const int kKeySize = 5; + const int kValSize = 20; + Random rnd(301); + + std::vector compression_parallel_threads = {1, 4}; + + std::map key_value_written; + + for (int iter = 0; iter <= 2; iter++) { + listener->max_level_checked = 0; + + if (iter == 0) { + // Use different compression algorithms for different levels but + // always use Zlib for bottommost level + options.compression_per_level = {kNoCompression, kNoCompression, + kNoCompression, kSnappyCompression, + kSnappyCompression, kSnappyCompression, + kZlibCompression}; + options.compression = kNoCompression; + options.bottommost_compression = kZlibCompression; + } else if (iter == 1) { + // Use Snappy except for bottommost level use ZLib + options.compression_per_level = {}; + options.compression = kSnappyCompression; + options.bottommost_compression = kZlibCompression; + } else if (iter == 2) { + // Use Snappy everywhere + options.compression_per_level = {}; + options.compression = kSnappyCompression; + options.bottommost_compression = kDisableCompressionOption; + } + + for (auto num_threads : compression_parallel_threads) { + options.compression_opts.parallel_threads = num_threads; + options.bottommost_compression_opts.parallel_threads = num_threads; + + DestroyAndReopen(options); + // Write 10 random files + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 5; j++) { + std::string key = rnd.RandomString(kKeySize); + std::string value = rnd.RandomString(kValSize); + key_value_written[key] = value; + ASSERT_OK(Put(key, value)); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + + // Make sure that we wrote enough to check all 7 levels + ASSERT_EQ(listener->max_level_checked, 6); + + // Make sure database content is the same as key_value_written + std::unique_ptr db_iter(db_->NewIterator(ReadOptions())); + for (db_iter->SeekToFirst(); db_iter->Valid(); db_iter->Next()) { + std::string key = db_iter->key().ToString(); + std::string value = db_iter->value().ToString(); + ASSERT_NE(key_value_written.find(key), key_value_written.end()); + ASSERT_EQ(key_value_written[key], value); + key_value_written.erase(key); + } + ASSERT_OK(db_iter->status()); + ASSERT_EQ(0, key_value_written.size()); + } + } +} + +class CompactionStallTestListener : public EventListener { + public: + CompactionStallTestListener() + : compacting_files_cnt_(0), compacted_files_cnt_(0) {} + + void OnCompactionBegin(DB* /*db*/, const CompactionJobInfo& ci) override { + ASSERT_EQ(ci.cf_name, "default"); + ASSERT_EQ(ci.base_input_level, 0); + ASSERT_EQ(ci.compaction_reason, CompactionReason::kLevelL0FilesNum); + compacting_files_cnt_ += ci.input_files.size(); + } + + void OnCompactionCompleted(DB* /*db*/, const CompactionJobInfo& ci) override { + ASSERT_EQ(ci.cf_name, "default"); + ASSERT_EQ(ci.base_input_level, 0); + ASSERT_EQ(ci.compaction_reason, CompactionReason::kLevelL0FilesNum); + compacted_files_cnt_ += ci.input_files.size(); + } + + std::atomic compacting_files_cnt_; + std::atomic compacted_files_cnt_; +}; + +TEST_F(DBTest2, CompactionStall) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BGWorkCompaction", "DBTest2::CompactionStall:0"}, + {"DBImpl::BGWorkCompaction", "DBTest2::CompactionStall:1"}, + {"DBTest2::CompactionStall:2", + "DBImpl::NotifyOnCompactionBegin::UnlockMutex"}, + {"DBTest2::CompactionStall:3", + "DBImpl::NotifyOnCompactionCompleted::UnlockMutex"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = 4; + options.max_background_compactions = 40; + CompactionStallTestListener* listener = new CompactionStallTestListener(); + options.listeners.emplace_back(listener); + DestroyAndReopen(options); + // make sure all background compaction jobs can be scheduled + auto stop_token = + dbfull()->TEST_write_controler().GetCompactionPressureToken(); + + Random rnd(301); + + // 4 Files in L0 + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 10; j++) { + ASSERT_OK(Put(rnd.RandomString(10), rnd.RandomString(10))); + } + ASSERT_OK(Flush()); + } + + // Wait for compaction to be triggered + TEST_SYNC_POINT("DBTest2::CompactionStall:0"); + + // Clear "DBImpl::BGWorkCompaction" SYNC_POINT since we want to hold it again + // at DBTest2::CompactionStall::1 + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace(); + + // Another 6 L0 files to trigger compaction again + for (int i = 0; i < 6; i++) { + for (int j = 0; j < 10; j++) { + ASSERT_OK(Put(rnd.RandomString(10), rnd.RandomString(10))); + } + ASSERT_OK(Flush()); + } + + // Wait for another compaction to be triggered + TEST_SYNC_POINT("DBTest2::CompactionStall:1"); + + // Hold NotifyOnCompactionBegin in the unlock mutex section + TEST_SYNC_POINT("DBTest2::CompactionStall:2"); + + // Hold NotifyOnCompactionCompleted in the unlock mutex section + TEST_SYNC_POINT("DBTest2::CompactionStall:3"); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_LT(NumTableFilesAtLevel(0), + options.level0_file_num_compaction_trigger); + ASSERT_GT(listener->compacted_files_cnt_.load(), + 10 - options.level0_file_num_compaction_trigger); + ASSERT_EQ(listener->compacting_files_cnt_.load(), + listener->compacted_files_cnt_.load()); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + + +TEST_F(DBTest2, FirstSnapshotTest) { + Options options; + options.write_buffer_size = 100000; // Small write buffer + options = CurrentOptions(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // This snapshot will have sequence number 0 what is expected behaviour. + const Snapshot* s1 = db_->GetSnapshot(); + + ASSERT_OK(Put(1, "k1", std::string(100000, 'x'))); // Fill memtable + ASSERT_OK(Put(1, "k2", std::string(100000, 'y'))); // Trigger flush + + db_->ReleaseSnapshot(s1); +} + +TEST_F(DBTest2, DuplicateSnapshot) { + Options options; + options = CurrentOptions(options); + std::vector snapshots; + DBImpl* dbi = static_cast_with_check(db_); + SequenceNumber oldest_ww_snap, first_ww_snap; + + ASSERT_OK(Put("k", "v")); // inc seq + snapshots.push_back(db_->GetSnapshot()); + snapshots.push_back(db_->GetSnapshot()); + ASSERT_OK(Put("k", "v")); // inc seq + snapshots.push_back(db_->GetSnapshot()); + snapshots.push_back(dbi->GetSnapshotForWriteConflictBoundary()); + first_ww_snap = snapshots.back()->GetSequenceNumber(); + ASSERT_OK(Put("k", "v")); // inc seq + snapshots.push_back(dbi->GetSnapshotForWriteConflictBoundary()); + snapshots.push_back(db_->GetSnapshot()); + ASSERT_OK(Put("k", "v")); // inc seq + snapshots.push_back(db_->GetSnapshot()); + + { + InstrumentedMutexLock l(dbi->mutex()); + auto seqs = dbi->snapshots().GetAll(&oldest_ww_snap); + ASSERT_EQ(seqs.size(), 4); // duplicates are not counted + ASSERT_EQ(oldest_ww_snap, first_ww_snap); + } + + for (auto s : snapshots) { + db_->ReleaseSnapshot(s); + } +} + +class PinL0IndexAndFilterBlocksTest + : public DBTestBase, + public testing::WithParamInterface> { + public: + PinL0IndexAndFilterBlocksTest() + : DBTestBase("db_pin_l0_index_bloom_test", /*env_do_fsync=*/true) {} + void SetUp() override { + infinite_max_files_ = std::get<0>(GetParam()); + disallow_preload_ = std::get<1>(GetParam()); + } + + void CreateTwoLevels(Options* options, bool close_afterwards) { + if (infinite_max_files_) { + options->max_open_files = -1; + } + options->create_if_missing = true; + options->statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + BlockBasedTableOptions table_options; + table_options.cache_index_and_filter_blocks = true; + table_options.pin_l0_filter_and_index_blocks_in_cache = true; + table_options.filter_policy.reset(NewBloomFilterPolicy(20)); + options->table_factory.reset(NewBlockBasedTableFactory(table_options)); + CreateAndReopenWithCF({"pikachu"}, *options); + + ASSERT_OK(Put(1, "a", "begin")); + ASSERT_OK(Put(1, "z", "end")); + ASSERT_OK(Flush(1)); + // move this table to L1 + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1])); + ASSERT_EQ(1, NumTableFilesAtLevel(1, 1)); + + // reset block cache + table_options.block_cache = NewLRUCache(64 * 1024); + options->table_factory.reset(NewBlockBasedTableFactory(table_options)); + TryReopenWithColumnFamilies({"default", "pikachu"}, *options); + // create new table at L0 + ASSERT_OK(Put(1, "a2", "begin2")); + ASSERT_OK(Put(1, "z2", "end2")); + ASSERT_OK(Flush(1)); + + if (close_afterwards) { + Close(); // This ensures that there is no ref to block cache entries + } + table_options.block_cache->EraseUnRefEntries(); + } + + bool infinite_max_files_; + bool disallow_preload_; +}; + +TEST_P(PinL0IndexAndFilterBlocksTest, + IndexAndFilterBlocksOfNewTableAddedToCacheWithPinning) { + Options options = CurrentOptions(); + if (infinite_max_files_) { + options.max_open_files = -1; + } + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + BlockBasedTableOptions table_options; + table_options.cache_index_and_filter_blocks = true; + table_options.pin_l0_filter_and_index_blocks_in_cache = true; + table_options.filter_policy.reset(NewBloomFilterPolicy(20)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(1, "key", "val")); + // Create a new table. + ASSERT_OK(Flush(1)); + + // index/filter blocks added to block cache right after table creation. + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT)); + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); + ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT)); + + // only index/filter were added + ASSERT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_ADD)); + ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_MISS)); + + std::string value; + // Miss and hit count should remain the same, they're all pinned. + ASSERT_TRUE(db_->KeyMayExist(ReadOptions(), handles_[1], "key", &value)); + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT)); + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); + ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT)); + + // Miss and hit count should remain the same, they're all pinned. + value = Get(1, "key"); + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT)); + ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); + ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT)); +} + +TEST_P(PinL0IndexAndFilterBlocksTest, + MultiLevelIndexAndFilterBlocksCachedWithPinning) { + Options options = CurrentOptions(); + PinL0IndexAndFilterBlocksTest::CreateTwoLevels(&options, false); + // get base cache values + uint64_t fm = TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS); + uint64_t fh = TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT); + uint64_t im = TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS); + uint64_t ih = TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT); + + std::string value; + // this should be read from L0 + // so cache values don't change + value = Get(1, "a2"); + ASSERT_EQ(fm, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(fh, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT)); + ASSERT_EQ(im, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); + ASSERT_EQ(ih, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT)); + + // this should be read from L1 + // the file is opened, prefetching results in a cache filter miss + // the block is loaded and added to the cache, + // then the get results in a cache hit for L1 + // When we have inifinite max_files, there is still cache miss because we have + // reset the block cache + value = Get(1, "a"); + ASSERT_EQ(fm + 1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(im + 1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); +} + +TEST_P(PinL0IndexAndFilterBlocksTest, DisablePrefetchingNonL0IndexAndFilter) { + Options options = CurrentOptions(); + // This ensures that db does not ref anything in the block cache, so + // EraseUnRefEntries could clear them up. + bool close_afterwards = true; + PinL0IndexAndFilterBlocksTest::CreateTwoLevels(&options, close_afterwards); + + // Get base cache values + uint64_t fm = TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS); + uint64_t fh = TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT); + uint64_t im = TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS); + uint64_t ih = TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT); + + if (disallow_preload_) { + // Now we have two files. We narrow the max open files to allow 3 entries + // so that preloading SST files won't happen. + options.max_open_files = 13; + // RocksDB sanitize max open files to at least 20. Modify it back. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SanitizeOptions::AfterChangeMaxOpenFiles", [&](void* arg) { + int* max_open_files = static_cast(arg); + *max_open_files = 13; + }); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Reopen database. If max_open_files is set as -1, table readers will be + // preloaded. This will trigger a BlockBasedTable::Open() and prefetch + // L0 index and filter. Level 1's prefetching is disabled in DB::Open() + TryReopenWithColumnFamilies({"default", "pikachu"}, options); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + if (!disallow_preload_) { + // After reopen, cache miss are increased by one because we read (and only + // read) filter and index on L0 + ASSERT_EQ(fm + 1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(fh, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT)); + ASSERT_EQ(im + 1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); + ASSERT_EQ(ih, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT)); + } else { + // If max_open_files is not -1, we do not preload table readers, so there is + // no change. + ASSERT_EQ(fm, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(fh, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT)); + ASSERT_EQ(im, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); + ASSERT_EQ(ih, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT)); + } + std::string value; + // this should be read from L0 + value = Get(1, "a2"); + // If max_open_files is -1, we have pinned index and filter in Rep, so there + // will not be changes in index and filter misses or hits. If max_open_files + // is not -1, Get() will open a TableReader and prefetch index and filter. + ASSERT_EQ(fm + 1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(fh, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT)); + ASSERT_EQ(im + 1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); + ASSERT_EQ(ih, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT)); + + // this should be read from L1 + value = Get(1, "a"); + if (!disallow_preload_) { + // In infinite max files case, there's a cache miss in executing Get() + // because index and filter are not prefetched before. + ASSERT_EQ(fm + 2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(fh, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT)); + ASSERT_EQ(im + 2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); + ASSERT_EQ(ih, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT)); + } else { + // In this case, cache miss will be increased by one in + // BlockBasedTable::Open() because this is not in DB::Open() code path so we + // will prefetch L1's index and filter. Cache hit will also be increased by + // one because Get() will read index and filter from the block cache + // prefetched in previous Open() call. + ASSERT_EQ(fm + 2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(fh + 1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT)); + ASSERT_EQ(im + 2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); + ASSERT_EQ(ih + 1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT)); + } + + // Force a full compaction to one single file. There will be a block + // cache read for both of index and filter. If prefetch doesn't explicitly + // happen, it will happen when verifying the file. + Compact(1, "a", "zzzzz"); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + if (!disallow_preload_) { + ASSERT_EQ(fm + 3, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(fh, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT)); + ASSERT_EQ(im + 3, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); + ASSERT_EQ(ih + 2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT)); + } else { + ASSERT_EQ(fm + 3, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(fh + 1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT)); + ASSERT_EQ(im + 3, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); + ASSERT_EQ(ih + 3, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT)); + } + + // Bloom and index hit will happen when a Get() happens. + value = Get(1, "a"); + if (!disallow_preload_) { + ASSERT_EQ(fm + 3, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(fh + 1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT)); + ASSERT_EQ(im + 3, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); + ASSERT_EQ(ih + 3, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT)); + } else { + ASSERT_EQ(fm + 3, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS)); + ASSERT_EQ(fh + 2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT)); + ASSERT_EQ(im + 3, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS)); + ASSERT_EQ(ih + 4, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT)); + } +} + +INSTANTIATE_TEST_CASE_P(PinL0IndexAndFilterBlocksTest, + PinL0IndexAndFilterBlocksTest, + ::testing::Values(std::make_tuple(true, false), + std::make_tuple(false, false), + std::make_tuple(false, true))); + +TEST_F(DBTest2, MaxCompactionBytesTest) { + Options options = CurrentOptions(); + options.memtable_factory.reset(test::NewSpecialSkipListFactory( + DBTestBase::kNumKeysByGenerateNewRandomFile)); + options.compaction_style = kCompactionStyleLevel; + options.write_buffer_size = 200 << 10; + options.arena_block_size = 4 << 10; + options.level0_file_num_compaction_trigger = 4; + options.num_levels = 4; + options.compression = kNoCompression; + options.max_bytes_for_level_base = 450 << 10; + options.target_file_size_base = 100 << 10; + // Infinite for full compaction. + options.max_compaction_bytes = options.target_file_size_base * 100; + + Reopen(options); + + Random rnd(301); + + for (int num = 0; num < 8; num++) { + GenerateNewRandomFile(&rnd); + } + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_EQ("0,0,8", FilesPerLevel(0)); + + // When compact from Ln -> Ln+1, cut a file if the file overlaps with + // more than three files in Ln+1. + options.max_compaction_bytes = options.target_file_size_base * 3; + Reopen(options); + + GenerateNewRandomFile(&rnd); + // Add three more small files that overlap with the previous file + for (int i = 0; i < 3; i++) { + ASSERT_OK(Put("a", "z")); + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // Output files to L1 are cut to 4 pieces, according to + // options.max_compaction_bytes (300K) + // There are 8 files on L2 (grandparents level), each one is 100K. The first + // file overlaps with a, b which max_compaction_bytes is less than 300K, the + // second one overlaps with d, e, which is also less than 300K. Including any + // extra grandparent file will make the future compaction larger than 300K. + // L1: [ 1 ] [ 2 ] [ 3 ] [ 4 ] + // L2: [a] [b] [c] [d] [e] [f] [g] [h] + ASSERT_EQ("0,4,8", FilesPerLevel(0)); +} + +static void UniqueIdCallback(void* arg) { + int* result = reinterpret_cast(arg); + if (*result == -1) { + *result = 0; + } + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "GetUniqueIdFromFile:FS_IOC_GETVERSION", UniqueIdCallback); +} + +class MockPersistentCache : public PersistentCache { + public: + explicit MockPersistentCache(const bool is_compressed, const size_t max_size) + : is_compressed_(is_compressed), max_size_(max_size) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "GetUniqueIdFromFile:FS_IOC_GETVERSION", UniqueIdCallback); + } + + ~MockPersistentCache() override {} + + PersistentCache::StatsType Stats() override { + return PersistentCache::StatsType(); + } + + uint64_t NewId() override { + return last_id_.fetch_add(1, std::memory_order_relaxed); + } + + Status Insert(const Slice& page_key, const char* data, + const size_t size) override { + MutexLock _(&lock_); + + if (size_ > max_size_) { + size_ -= data_.begin()->second.size(); + data_.erase(data_.begin()); + } + + data_.insert(std::make_pair(page_key.ToString(), std::string(data, size))); + size_ += size; + return Status::OK(); + } + + Status Lookup(const Slice& page_key, std::unique_ptr* data, + size_t* size) override { + MutexLock _(&lock_); + auto it = data_.find(page_key.ToString()); + if (it == data_.end()) { + return Status::NotFound(); + } + + assert(page_key.ToString() == it->first); + data->reset(new char[it->second.size()]); + memcpy(data->get(), it->second.c_str(), it->second.size()); + *size = it->second.size(); + return Status::OK(); + } + + bool IsCompressed() override { return is_compressed_; } + + std::string GetPrintableOptions() const override { + return "MockPersistentCache"; + } + + port::Mutex lock_; + std::map data_; + const bool is_compressed_ = true; + size_t size_ = 0; + const size_t max_size_ = 10 * 1024; // 10KiB + std::atomic last_id_{1}; +}; + +#ifdef OS_LINUX +// Make sure that in CPU time perf context counters, Env::NowCPUNanos() +// is used, rather than Env::CPUNanos(); +TEST_F(DBTest2, TestPerfContextGetCpuTime) { + // force resizing table cache so table handle is not preloaded so that + // we can measure find_table_nanos during Get(). + dbfull()->TEST_table_cache()->SetCapacity(0); + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Flush()); + env_->now_cpu_count_.store(0); + env_->SetMockSleep(); + + // NOTE: Presumed unnecessary and removed: resetting mock time in env + + // CPU timing is not enabled with kEnableTimeExceptForMutex + SetPerfLevel(PerfLevel::kEnableTimeExceptForMutex); + ASSERT_EQ("bar", Get("foo")); + ASSERT_EQ(0, get_perf_context()->get_cpu_nanos); + ASSERT_EQ(0, env_->now_cpu_count_.load()); + + constexpr uint64_t kDummyAddonSeconds = uint64_t{1000000}; + constexpr uint64_t kDummyAddonNanos = 1000000000U * kDummyAddonSeconds; + + // Add time to NowNanos() reading. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "TableCache::FindTable:0", + [&](void* /*arg*/) { env_->MockSleepForSeconds(kDummyAddonSeconds); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + SetPerfLevel(PerfLevel::kEnableTimeAndCPUTimeExceptForMutex); + ASSERT_EQ("bar", Get("foo")); + ASSERT_GT(env_->now_cpu_count_.load(), 2); + ASSERT_LT(get_perf_context()->get_cpu_nanos, kDummyAddonNanos); + ASSERT_GT(get_perf_context()->find_table_nanos, kDummyAddonNanos); + + SetPerfLevel(PerfLevel::kDisable); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBTest2, TestPerfContextIterCpuTime) { + DestroyAndReopen(CurrentOptions()); + // force resizing table cache so table handle is not preloaded so that + // we can measure find_table_nanos during iteration + dbfull()->TEST_table_cache()->SetCapacity(0); + + const size_t kNumEntries = 10; + for (size_t i = 0; i < kNumEntries; ++i) { + ASSERT_OK(Put("k" + std::to_string(i), "v" + std::to_string(i))); + } + ASSERT_OK(Flush()); + for (size_t i = 0; i < kNumEntries; ++i) { + ASSERT_EQ("v" + std::to_string(i), Get("k" + std::to_string(i))); + } + std::string last_key = "k" + std::to_string(kNumEntries - 1); + std::string last_value = "v" + std::to_string(kNumEntries - 1); + env_->now_cpu_count_.store(0); + env_->SetMockSleep(); + + // NOTE: Presumed unnecessary and removed: resetting mock time in env + + // CPU timing is not enabled with kEnableTimeExceptForMutex + SetPerfLevel(PerfLevel::kEnableTimeExceptForMutex); + Iterator* iter = db_->NewIterator(ReadOptions()); + iter->Seek("k0"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("v0", iter->value().ToString()); + iter->SeekForPrev(last_key); + ASSERT_TRUE(iter->Valid()); + iter->SeekToLast(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(last_value, iter->value().ToString()); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("v0", iter->value().ToString()); + ASSERT_EQ(0, get_perf_context()->iter_seek_cpu_nanos); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("v1", iter->value().ToString()); + ASSERT_EQ(0, get_perf_context()->iter_next_cpu_nanos); + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ("v0", iter->value().ToString()); + ASSERT_EQ(0, get_perf_context()->iter_prev_cpu_nanos); + ASSERT_EQ(0, env_->now_cpu_count_.load()); + delete iter; + + constexpr uint64_t kDummyAddonSeconds = uint64_t{1000000}; + constexpr uint64_t kDummyAddonNanos = 1000000000U * kDummyAddonSeconds; + + // Add time to NowNanos() reading. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "TableCache::FindTable:0", + [&](void* /*arg*/) { env_->MockSleepForSeconds(kDummyAddonSeconds); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + SetPerfLevel(PerfLevel::kEnableTimeAndCPUTimeExceptForMutex); + iter = db_->NewIterator(ReadOptions()); + iter->Seek("k0"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("v0", iter->value().ToString()); + iter->SeekForPrev(last_key); + ASSERT_TRUE(iter->Valid()); + iter->SeekToLast(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(last_value, iter->value().ToString()); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("v0", iter->value().ToString()); + ASSERT_GT(get_perf_context()->iter_seek_cpu_nanos, 0); + ASSERT_LT(get_perf_context()->iter_seek_cpu_nanos, kDummyAddonNanos); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("v1", iter->value().ToString()); + ASSERT_GT(get_perf_context()->iter_next_cpu_nanos, 0); + ASSERT_LT(get_perf_context()->iter_next_cpu_nanos, kDummyAddonNanos); + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ("v0", iter->value().ToString()); + ASSERT_GT(get_perf_context()->iter_prev_cpu_nanos, 0); + ASSERT_LT(get_perf_context()->iter_prev_cpu_nanos, kDummyAddonNanos); + ASSERT_GE(env_->now_cpu_count_.load(), 12); + ASSERT_GT(get_perf_context()->find_table_nanos, kDummyAddonNanos); + + SetPerfLevel(PerfLevel::kDisable); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + delete iter; +} +#endif // OS_LINUX + +#if !defined OS_SOLARIS +TEST_F(DBTest2, PersistentCache) { + int num_iter = 80; + + Options options; + options.write_buffer_size = 64 * 1024; // small write buffer + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options = CurrentOptions(options); + + auto bsizes = {/*no block cache*/ 0, /*1M*/ 1 * 1024 * 1024}; + auto types = {/*compressed*/ 1, /*uncompressed*/ 0}; + for (auto bsize : bsizes) { + for (auto type : types) { + BlockBasedTableOptions table_options; + table_options.persistent_cache.reset( + new MockPersistentCache(type, 10 * 1024)); + table_options.no_block_cache = true; + table_options.block_cache = bsize ? NewLRUCache(bsize) : nullptr; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + // default column family doesn't have block cache + Options no_block_cache_opts; + no_block_cache_opts.statistics = options.statistics; + no_block_cache_opts = CurrentOptions(no_block_cache_opts); + BlockBasedTableOptions table_options_no_bc; + table_options_no_bc.no_block_cache = true; + no_block_cache_opts.table_factory.reset( + NewBlockBasedTableFactory(table_options_no_bc)); + ReopenWithColumnFamilies( + {"default", "pikachu"}, + std::vector({no_block_cache_opts, options})); + + Random rnd(301); + + // Write 8MB (80 values, each 100K) + ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0); + std::vector values; + std::string str; + for (int i = 0; i < num_iter; i++) { + if (i % 4 == 0) { // high compression ratio + str = rnd.RandomString(1000); + } + values.push_back(str); + ASSERT_OK(Put(1, Key(i), values[i])); + } + + // flush all data from memtable so that reads are from block cache + ASSERT_OK(Flush(1)); + + for (int i = 0; i < num_iter; i++) { + ASSERT_EQ(Get(1, Key(i)), values[i]); + } + + auto hit = options.statistics->getTickerCount(PERSISTENT_CACHE_HIT); + auto miss = options.statistics->getTickerCount(PERSISTENT_CACHE_MISS); + + ASSERT_GT(hit, 0); + ASSERT_GT(miss, 0); + } + } +} +#endif // !defined OS_SOLARIS + +namespace { +void CountSyncPoint() { + TEST_SYNC_POINT_CALLBACK("DBTest2::MarkedPoint", nullptr /* arg */); +} +} // anonymous namespace + +TEST_F(DBTest2, SyncPointMarker) { + std::atomic sync_point_called(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBTest2::MarkedPoint", + [&](void* /*arg*/) { sync_point_called.fetch_add(1); }); + + // The first dependency enforces Marker can be loaded before MarkedPoint. + // The second checks that thread 1's MarkedPoint should be disabled here. + // Execution order: + // | Thread 1 | Thread 2 | + // | | Marker | + // | MarkedPoint | | + // | Thread1First | | + // | | MarkedPoint | + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependencyAndMarkers( + {{"DBTest2::SyncPointMarker:Thread1First", "DBTest2::MarkedPoint"}}, + {{"DBTest2::SyncPointMarker:Marker", "DBTest2::MarkedPoint"}}); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + std::function func1 = [&]() { + CountSyncPoint(); + TEST_SYNC_POINT("DBTest2::SyncPointMarker:Thread1First"); + }; + + std::function func2 = [&]() { + TEST_SYNC_POINT("DBTest2::SyncPointMarker:Marker"); + CountSyncPoint(); + }; + + auto thread1 = port::Thread(func1); + auto thread2 = port::Thread(func2); + thread1.join(); + thread2.join(); + + // Callback is only executed once + ASSERT_EQ(sync_point_called.load(), 1); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +size_t GetEncodedEntrySize(size_t key_size, size_t value_size) { + std::string buffer; + + PutVarint32(&buffer, static_cast(0)); + PutVarint32(&buffer, static_cast(key_size)); + PutVarint32(&buffer, static_cast(value_size)); + + return buffer.size() + key_size + value_size; +} + +TEST_F(DBTest2, ReadAmpBitmap) { + Options options = CurrentOptions(); + BlockBasedTableOptions bbto; + uint32_t bytes_per_bit[2] = {1, 16}; + for (size_t k = 0; k < 2; k++) { + // Disable delta encoding to make it easier to calculate read amplification + bbto.use_delta_encoding = false; + // Huge block cache to make it easier to calculate read amplification + bbto.block_cache = NewLRUCache(1024 * 1024 * 1024); + bbto.read_amp_bytes_per_bit = bytes_per_bit[k]; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + DestroyAndReopen(options); + + const size_t kNumEntries = 10000; + + Random rnd(301); + for (size_t i = 0; i < kNumEntries; i++) { + ASSERT_OK(Put(Key(static_cast(i)), rnd.RandomString(100))); + } + ASSERT_OK(Flush()); + + Close(); + Reopen(options); + + // Read keys/values randomly and verify that reported read amp error + // is less than 2% + uint64_t total_useful_bytes = 0; + std::set read_keys; + std::string value; + for (size_t i = 0; i < kNumEntries * 5; i++) { + int key_idx = rnd.Next() % kNumEntries; + std::string key = Key(key_idx); + ASSERT_OK(db_->Get(ReadOptions(), key, &value)); + + if (read_keys.find(key_idx) == read_keys.end()) { + auto internal_key = InternalKey(key, 0, ValueType::kTypeValue); + total_useful_bytes += + GetEncodedEntrySize(internal_key.size(), value.size()); + read_keys.insert(key_idx); + } + + double expected_read_amp = + static_cast(total_useful_bytes) / + options.statistics->getTickerCount(READ_AMP_TOTAL_READ_BYTES); + + double read_amp = + static_cast(options.statistics->getTickerCount( + READ_AMP_ESTIMATE_USEFUL_BYTES)) / + options.statistics->getTickerCount(READ_AMP_TOTAL_READ_BYTES); + + double error_pct = fabs(expected_read_amp - read_amp) * 100; + // Error between reported read amp and real read amp should be less than + // 2% + EXPECT_LE(error_pct, 2); + } + + // Make sure we read every thing in the DB (which is smaller than our cache) + Iterator* iter = db_->NewIterator(ReadOptions()); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_EQ(iter->value().ToString(), Get(iter->key().ToString())); + } + ASSERT_OK(iter->status()); + delete iter; + + // Read amp is on average 100% since we read all what we loaded in memory + if (k == 0) { + ASSERT_EQ( + options.statistics->getTickerCount(READ_AMP_ESTIMATE_USEFUL_BYTES), + options.statistics->getTickerCount(READ_AMP_TOTAL_READ_BYTES)); + } else { + ASSERT_NEAR( + options.statistics->getTickerCount(READ_AMP_ESTIMATE_USEFUL_BYTES) * + 1.0f / + options.statistics->getTickerCount(READ_AMP_TOTAL_READ_BYTES), + 1, .01); + } + } +} + +#ifndef OS_SOLARIS // GetUniqueIdFromFile is not implemented +TEST_F(DBTest2, ReadAmpBitmapLiveInCacheAfterDBClose) { + { + const int kIdBufLen = 100; + char id_buf[kIdBufLen]; + Status s = Status::NotSupported(); +#ifndef OS_WIN + // You can't open a directory on windows using random access file + std::unique_ptr file; + s = env_->NewRandomAccessFile(dbname_, &file, EnvOptions()); + if (s.ok()) { + if (file->GetUniqueId(id_buf, kIdBufLen) == 0) { + // fs holding db directory doesn't support getting a unique file id, + // this means that running this test will fail because lru_cache will + // load the blocks again regardless of them being already in the cache + return; + } + } +#endif + if (!s.ok()) { + std::unique_ptr dir; + ASSERT_OK(env_->NewDirectory(dbname_, &dir)); + if (dir->GetUniqueId(id_buf, kIdBufLen) == 0) { + // fs holding db directory doesn't support getting a unique file id, + // this means that running this test will fail because lru_cache will + // load the blocks again regardless of them being already in the cache + return; + } + } + } + uint32_t bytes_per_bit[2] = {1, 16}; + for (size_t k = 0; k < 2; k++) { + std::shared_ptr lru_cache = NewLRUCache(1024 * 1024 * 1024); + std::shared_ptr stats = ROCKSDB_NAMESPACE::CreateDBStatistics(); + + Options options = CurrentOptions(); + BlockBasedTableOptions bbto; + // Disable delta encoding to make it easier to calculate read amplification + bbto.use_delta_encoding = false; + // Huge block cache to make it easier to calculate read amplification + bbto.block_cache = lru_cache; + bbto.read_amp_bytes_per_bit = bytes_per_bit[k]; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + options.statistics = stats; + DestroyAndReopen(options); + + const int kNumEntries = 10000; + + Random rnd(301); + for (int i = 0; i < kNumEntries; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(100))); + } + ASSERT_OK(Flush()); + + Close(); + Reopen(options); + + std::set read_keys; + std::string value; + // Iter1: Read half the DB, Read even keys + // Key(0), Key(2), Key(4), Key(6), Key(8), ... + for (int i = 0; i < kNumEntries; i += 2) { + std::string key = Key(i); + ASSERT_OK(db_->Get(ReadOptions(), key, &value)); + + if (read_keys.find(i) == read_keys.end()) { + auto internal_key = InternalKey(key, 0, ValueType::kTypeValue); + read_keys.insert(i); + } + } + + size_t total_useful_bytes_iter1 = + options.statistics->getTickerCount(READ_AMP_ESTIMATE_USEFUL_BYTES); + size_t total_loaded_bytes_iter1 = + options.statistics->getTickerCount(READ_AMP_TOTAL_READ_BYTES); + + Close(); + std::shared_ptr new_statistics = + ROCKSDB_NAMESPACE::CreateDBStatistics(); + // Destroy old statistics obj that the blocks in lru_cache are pointing to + options.statistics.reset(); + // Use the statistics object that we just created + options.statistics = new_statistics; + Reopen(options); + + // Iter2: Read half the DB, Read odd keys + // Key(1), Key(3), Key(5), Key(7), Key(9), ... + for (int i = 1; i < kNumEntries; i += 2) { + std::string key = Key(i); + ASSERT_OK(db_->Get(ReadOptions(), key, &value)); + + if (read_keys.find(i) == read_keys.end()) { + auto internal_key = InternalKey(key, 0, ValueType::kTypeValue); + read_keys.insert(i); + } + } + + size_t total_useful_bytes_iter2 = + options.statistics->getTickerCount(READ_AMP_ESTIMATE_USEFUL_BYTES); + size_t total_loaded_bytes_iter2 = + options.statistics->getTickerCount(READ_AMP_TOTAL_READ_BYTES); + + // Read amp is on average 100% since we read all what we loaded in memory + if (k == 0) { + ASSERT_EQ(total_useful_bytes_iter1 + total_useful_bytes_iter2, + total_loaded_bytes_iter1 + total_loaded_bytes_iter2); + } else { + ASSERT_NEAR((total_useful_bytes_iter1 + total_useful_bytes_iter2) * 1.0f / + (total_loaded_bytes_iter1 + total_loaded_bytes_iter2), + 1, .01); + } + } +} +#endif // !OS_SOLARIS + +TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) { + Options options = CurrentOptions(); + options.num_levels = 3; + options.IncreaseParallelism(20); + DestroyAndReopen(options); + + ASSERT_OK(Put(Key(0), "a")); + ASSERT_OK(Put(Key(5), "a")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put(Key(10), "a")); + ASSERT_OK(Put(Key(15), "a")); + ASSERT_OK(Flush()); + + CompactRangeOptions cro; + cro.change_level = true; + cro.target_level = 2; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + auto get_stat = [](std::string level_str, LevelStatType type, + std::map props) { + auto prop_str = + "compaction." + level_str + "." + + InternalStats::compaction_level_stats.at(type).property_name.c_str(); + auto prop_item = props.find(prop_str); + return prop_item == props.end() ? 0 : std::stod(prop_item->second); + }; + + // Trivial move 2 files to L2 + ASSERT_EQ("0,0,2", FilesPerLevel()); + // Also test that the stats GetMapProperty API reporting the same result + { + std::map prop; + ASSERT_TRUE(dbfull()->GetMapProperty("rocksdb.cfstats", &prop)); + ASSERT_EQ(0, get_stat("L0", LevelStatType::NUM_FILES, prop)); + ASSERT_EQ(0, get_stat("L1", LevelStatType::NUM_FILES, prop)); + ASSERT_EQ(2, get_stat("L2", LevelStatType::NUM_FILES, prop)); + ASSERT_EQ(2, get_stat("Sum", LevelStatType::NUM_FILES, prop)); + } + + // While the compaction is running, we will create 2 new files that + // can fit in L2, these 2 files will be moved to L2 and overlap with + // the running compaction and break the LSM consistency. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::Run():Start", [&](void* /*arg*/) { + ASSERT_OK( + dbfull()->SetOptions({{"level0_file_num_compaction_trigger", "2"}, + {"max_bytes_for_level_base", "1"}})); + ASSERT_OK(Put(Key(6), "a")); + ASSERT_OK(Put(Key(7), "a")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put(Key(8), "a")); + ASSERT_OK(Put(Key(9), "a")); + ASSERT_OK(Flush()); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Run a manual compaction that will compact the 2 files in L2 + // into 1 file in L2 + cro.exclusive_manual_compaction = false; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForceOptimized; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + // Test that the stats GetMapProperty API reporting 1 file in L2 + { + std::map prop; + ASSERT_TRUE(dbfull()->GetMapProperty("rocksdb.cfstats", &prop)); + ASSERT_EQ(1, get_stat("L2", LevelStatType::NUM_FILES, prop)); + } +} + +TEST_F(DBTest2, ManualCompactionOverlapManualCompaction) { + Options options = CurrentOptions(); + options.num_levels = 2; + options.IncreaseParallelism(20); + options.disable_auto_compactions = true; + DestroyAndReopen(options); + + ASSERT_OK(Put(Key(0), "a")); + ASSERT_OK(Put(Key(5), "a")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put(Key(10), "a")); + ASSERT_OK(Put(Key(15), "a")); + ASSERT_OK(Flush()); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + // Trivial move 2 files to L1 + ASSERT_EQ("0,2", FilesPerLevel()); + + std::function bg_manual_compact = [&]() { + std::string k1 = Key(6); + std::string k2 = Key(9); + Slice k1s(k1); + Slice k2s(k2); + CompactRangeOptions cro; + cro.exclusive_manual_compaction = false; + ASSERT_OK(db_->CompactRange(cro, &k1s, &k2s)); + }; + ROCKSDB_NAMESPACE::port::Thread bg_thread; + + // While the compaction is running, we will create 2 new files that + // can fit in L1, these 2 files will be moved to L1 and overlap with + // the running compaction and break the LSM consistency. + std::atomic flag(false); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::Run():Start", [&](void* /*arg*/) { + if (flag.exchange(true)) { + // We want to make sure to call this callback only once + return; + } + ASSERT_OK(Put(Key(6), "a")); + ASSERT_OK(Put(Key(7), "a")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put(Key(8), "a")); + ASSERT_OK(Put(Key(9), "a")); + ASSERT_OK(Flush()); + + // Start a non-exclusive manual compaction in a bg thread + bg_thread = port::Thread(bg_manual_compact); + // This manual compaction conflict with the other manual compaction + // so it should wait until the first compaction finish + env_->SleepForMicroseconds(1000000); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Run a manual compaction that will compact the 2 files in L1 + // into 1 file in L1 + CompactRangeOptions cro; + cro.exclusive_manual_compaction = false; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForceOptimized; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + bg_thread.join(); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBTest2, PausingManualCompaction1) { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.num_levels = 7; + + DestroyAndReopen(options); + Random rnd(301); + // Generate a file containing 10 keys. + for (int i = 0; i < 10; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(50))); + } + ASSERT_OK(Flush()); + + // Generate another file containing same keys + for (int i = 0; i < 10; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(50))); + } + ASSERT_OK(Flush()); + + int manual_compactions_paused = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::Run():PausingManualCompaction:1", [&](void* arg) { + auto canceled = static_cast*>(arg); + // CompactRange triggers manual compaction and cancel the compaction + // by set *canceled as true + if (canceled != nullptr) { + canceled->store(true, std::memory_order_release); + } + manual_compactions_paused += 1; + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "TestCompactFiles:PausingManualCompaction:3", [&](void* arg) { + auto paused = static_cast*>(arg); + // CompactFiles() relies on manual_compactions_paused to + // determine if thie compaction should be paused or not + ASSERT_EQ(0, paused->load(std::memory_order_acquire)); + paused->fetch_add(1, std::memory_order_release); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + std::vector files_before_compact, files_after_compact; + // Remember file name before compaction is triggered + std::vector files_meta; + dbfull()->GetLiveFilesMetaData(&files_meta); + for (auto file : files_meta) { + files_before_compact.push_back(file.name); + } + + // OK, now trigger a manual compaction + ASSERT_TRUE(dbfull() + ->CompactRange(CompactRangeOptions(), nullptr, nullptr) + .IsManualCompactionPaused()); + + // Wait for compactions to get scheduled and stopped + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // Get file names after compaction is stopped + files_meta.clear(); + dbfull()->GetLiveFilesMetaData(&files_meta); + for (auto file : files_meta) { + files_after_compact.push_back(file.name); + } + + // Like nothing happened + ASSERT_EQ(files_before_compact, files_after_compact); + ASSERT_EQ(manual_compactions_paused, 1); + + manual_compactions_paused = 0; + // Now make sure CompactFiles also not run + ASSERT_TRUE(dbfull() + ->CompactFiles(ROCKSDB_NAMESPACE::CompactionOptions(), + files_before_compact, 0) + .IsManualCompactionPaused()); + // Wait for manual compaction to get scheduled and finish + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + files_meta.clear(); + files_after_compact.clear(); + dbfull()->GetLiveFilesMetaData(&files_meta); + for (auto file : files_meta) { + files_after_compact.push_back(file.name); + } + + ASSERT_EQ(files_before_compact, files_after_compact); + // CompactFiles returns at entry point + ASSERT_EQ(manual_compactions_paused, 0); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +// PausingManualCompaction does not affect auto compaction +TEST_F(DBTest2, PausingManualCompaction2) { + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = 2; + options.disable_auto_compactions = false; + + DestroyAndReopen(options); + dbfull()->DisableManualCompaction(); + + Random rnd(301); + for (int i = 0; i < 2; i++) { + // Generate a file containing 100 keys. + for (int j = 0; j < 100; j++) { + ASSERT_OK(Put(Key(j), rnd.RandomString(50))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + std::vector files_meta; + dbfull()->GetLiveFilesMetaData(&files_meta); + ASSERT_EQ(files_meta.size(), 1); +} + +TEST_F(DBTest2, PausingManualCompaction3) { + CompactRangeOptions compact_options; + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.num_levels = 7; + + Random rnd(301); + auto generate_files = [&]() { + for (int i = 0; i < options.num_levels; i++) { + for (int j = 0; j < options.num_levels - i + 1; j++) { + for (int k = 0; k < 1000; k++) { + ASSERT_OK(Put(Key(k + j * 1000), rnd.RandomString(50))); + } + ASSERT_OK(Flush()); + } + + for (int l = 1; l < options.num_levels - i; l++) { + MoveFilesToLevel(l); + } + } + }; + + DestroyAndReopen(options); + generate_files(); + ASSERT_EQ("2,3,4,5,6,7,8", FilesPerLevel()); + int run_manual_compactions = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::Run():PausingManualCompaction:1", + [&](void* /*arg*/) { run_manual_compactions++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + dbfull()->DisableManualCompaction(); + ASSERT_TRUE(dbfull() + ->CompactRange(compact_options, nullptr, nullptr) + .IsManualCompactionPaused()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // As manual compaction disabled, not even reach sync point + ASSERT_EQ(run_manual_compactions, 0); + ASSERT_EQ("2,3,4,5,6,7,8", FilesPerLevel()); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearCallBack( + "CompactionJob::Run():PausingManualCompaction:1"); + dbfull()->EnableManualCompaction(); + ASSERT_OK(dbfull()->CompactRange(compact_options, nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,0,0,0,0,0,2", FilesPerLevel()); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBTest2, PausingManualCompaction4) { + CompactRangeOptions compact_options; + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.num_levels = 7; + + Random rnd(301); + auto generate_files = [&]() { + for (int i = 0; i < options.num_levels; i++) { + for (int j = 0; j < options.num_levels - i + 1; j++) { + for (int k = 0; k < 1000; k++) { + ASSERT_OK(Put(Key(k + j * 1000), rnd.RandomString(50))); + } + ASSERT_OK(Flush()); + } + + for (int l = 1; l < options.num_levels - i; l++) { + MoveFilesToLevel(l); + } + } + }; + + DestroyAndReopen(options); + generate_files(); + ASSERT_EQ("2,3,4,5,6,7,8", FilesPerLevel()); + int run_manual_compactions = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::Run():PausingManualCompaction:2", [&](void* arg) { + auto canceled = static_cast*>(arg); + // CompactRange triggers manual compaction and cancel the compaction + // by set *canceled as true + if (canceled != nullptr) { + canceled->store(true, std::memory_order_release); + } + run_manual_compactions++; + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "TestCompactFiles:PausingManualCompaction:3", [&](void* arg) { + auto paused = static_cast*>(arg); + // CompactFiles() relies on manual_compactions_paused to + // determine if thie compaction should be paused or not + ASSERT_EQ(0, paused->load(std::memory_order_acquire)); + paused->fetch_add(1, std::memory_order_release); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_TRUE(dbfull() + ->CompactRange(compact_options, nullptr, nullptr) + .IsManualCompactionPaused()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(run_manual_compactions, 1); + ASSERT_EQ("2,3,4,5,6,7,8", FilesPerLevel()); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearCallBack( + "CompactionJob::Run():PausingManualCompaction:2"); + ASSERT_OK(dbfull()->CompactRange(compact_options, nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,0,0,0,0,0,2", FilesPerLevel()); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBTest2, CancelManualCompaction1) { + CompactRangeOptions compact_options; + auto canceledPtr = + std::unique_ptr>(new std::atomic{true}); + compact_options.canceled = canceledPtr.get(); + + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.num_levels = 7; + + Random rnd(301); + auto generate_files = [&]() { + for (int i = 0; i < options.num_levels; i++) { + for (int j = 0; j < options.num_levels - i + 1; j++) { + for (int k = 0; k < 1000; k++) { + ASSERT_OK(Put(Key(k + j * 1000), rnd.RandomString(50))); + } + ASSERT_OK(Flush()); + } + + for (int l = 1; l < options.num_levels - i; l++) { + MoveFilesToLevel(l); + } + } + }; + + DestroyAndReopen(options); + generate_files(); + ASSERT_EQ("2,3,4,5,6,7,8", FilesPerLevel()); + + int run_manual_compactions = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::Run():PausingManualCompaction:1", + [&](void* /*arg*/) { run_manual_compactions++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Setup a callback to disable compactions after a couple of levels are + // compacted + int compactions_run = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::RunManualCompaction()::1", + [&](void* /*arg*/) { ++compactions_run; }); + + ASSERT_TRUE(dbfull() + ->CompactRange(compact_options, nullptr, nullptr) + .IsManualCompactionPaused()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // Since compactions are disabled, we shouldn't start compacting. + // E.g. we should call the compaction function exactly one time. + ASSERT_EQ(compactions_run, 0); + ASSERT_EQ(run_manual_compactions, 0); + ASSERT_EQ("2,3,4,5,6,7,8", FilesPerLevel()); + + compactions_run = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearCallBack( + "DBImpl::RunManualCompaction()::1"); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::RunManualCompaction()::1", [&](void* /*arg*/) { + ++compactions_run; + // After 3 compactions disable + if (compactions_run == 3) { + compact_options.canceled->store(true, std::memory_order_release); + } + }); + + compact_options.canceled->store(false, std::memory_order_release); + ASSERT_TRUE(dbfull() + ->CompactRange(compact_options, nullptr, nullptr) + .IsManualCompactionPaused()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ(compactions_run, 3); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearCallBack( + "DBImpl::RunManualCompaction()::1"); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearCallBack( + "CompactionJob::Run():PausingManualCompaction:1"); + + // Compactions should work again if we re-enable them.. + compact_options.canceled->store(false, std::memory_order_relaxed); + ASSERT_OK(dbfull()->CompactRange(compact_options, nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,0,0,0,0,0,2", FilesPerLevel()); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBTest2, CancelManualCompaction2) { + CompactRangeOptions compact_options; + auto canceledPtr = + std::unique_ptr>(new std::atomic{true}); + compact_options.canceled = canceledPtr.get(); + compact_options.max_subcompactions = 1; + + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.num_levels = 7; + + Random rnd(301); + auto generate_files = [&]() { + for (int i = 0; i < options.num_levels; i++) { + for (int j = 0; j < options.num_levels - i + 1; j++) { + for (int k = 0; k < 1000; k++) { + ASSERT_OK(Put(Key(k + j * 1000), rnd.RandomString(50))); + } + ASSERT_OK(Flush()); + } + + for (int l = 1; l < options.num_levels - i; l++) { + MoveFilesToLevel(l); + } + } + }; + + DestroyAndReopen(options); + generate_files(); + ASSERT_EQ("2,3,4,5,6,7,8", FilesPerLevel()); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + int compactions_run = 0; + std::atomic kv_compactions{0}; + int compactions_stopped_at = 0; + int kv_compactions_stopped_at = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::RunManualCompaction()::1", [&](void* /*arg*/) { + ++compactions_run; + // After 3 compactions disable + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionIterator:ProcessKV", [&](void* /*arg*/) { + int kv_compactions_run = + kv_compactions.fetch_add(1, std::memory_order_release); + if (kv_compactions_run == 5) { + compact_options.canceled->store(true, std::memory_order_release); + kv_compactions_stopped_at = kv_compactions_run; + compactions_stopped_at = compactions_run; + } + }); + + compact_options.canceled->store(false, std::memory_order_release); + ASSERT_TRUE(dbfull() + ->CompactRange(compact_options, nullptr, nullptr) + .IsManualCompactionPaused()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // NOTE: as we set compact_options.max_subcompacitons = 1, and store true to + // the canceled variable from the single compacting thread (via callback), + // this value is deterministically kv_compactions_stopped_at + 1. + ASSERT_EQ(kv_compactions, kv_compactions_stopped_at + 1); + ASSERT_EQ(compactions_run, compactions_stopped_at); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearCallBack( + "CompactionIterator::ProcessKV"); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearCallBack( + "DBImpl::RunManualCompaction()::1"); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearCallBack( + "CompactionJob::Run():PausingManualCompaction:1"); + + // Compactions should work again if we re-enable them.. + compact_options.canceled->store(false, std::memory_order_relaxed); + ASSERT_OK(dbfull()->CompactRange(compact_options, nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,0,0,0,0,0,2", FilesPerLevel()); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +class CancelCompactionListener : public EventListener { + public: + CancelCompactionListener() + : num_compaction_started_(0), num_compaction_ended_(0) {} + + void OnCompactionBegin(DB* /*db*/, const CompactionJobInfo& ci) override { + ASSERT_EQ(ci.cf_name, "default"); + ASSERT_EQ(ci.base_input_level, 0); + num_compaction_started_++; + } + + void OnCompactionCompleted(DB* /*db*/, const CompactionJobInfo& ci) override { + ASSERT_EQ(ci.cf_name, "default"); + ASSERT_EQ(ci.base_input_level, 0); + ASSERT_EQ(ci.status.code(), code_); + ASSERT_EQ(ci.status.subcode(), subcode_); + num_compaction_ended_++; + } + + std::atomic num_compaction_started_; + std::atomic num_compaction_ended_; + Status::Code code_; + Status::SubCode subcode_; +}; + +TEST_F(DBTest2, CancelManualCompactionWithListener) { + CompactRangeOptions compact_options; + auto canceledPtr = + std::unique_ptr>(new std::atomic{true}); + compact_options.canceled = canceledPtr.get(); + compact_options.max_subcompactions = 1; + + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + CancelCompactionListener* listener = new CancelCompactionListener(); + options.listeners.emplace_back(listener); + + DestroyAndReopen(options); + + Random rnd(301); + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 10; j++) { + ASSERT_OK(Put(Key(i + j * 10), rnd.RandomString(50))); + } + ASSERT_OK(Flush()); + } + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionIterator:ProcessKV", [&](void* /*arg*/) { + compact_options.canceled->store(true, std::memory_order_release); + }); + + int running_compaction = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::FinishCompactionOutputFile1", + [&](void* /*arg*/) { running_compaction++; }); + + // Case I: 1 Notify begin compaction, 2 Set *canceled as true to disable + // manual compaction in the callback function, 3 Compaction not run, + // 4 Notify compaction end. + listener->code_ = Status::kIncomplete; + listener->subcode_ = Status::SubCode::kManualCompactionPaused; + + compact_options.canceled->store(false, std::memory_order_release); + ASSERT_TRUE(dbfull() + ->CompactRange(compact_options, nullptr, nullptr) + .IsManualCompactionPaused()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_GT(listener->num_compaction_started_, 0); + ASSERT_EQ(listener->num_compaction_started_, listener->num_compaction_ended_); + ASSERT_EQ(running_compaction, 0); + + listener->num_compaction_started_ = 0; + listener->num_compaction_ended_ = 0; + + // Case II: 1 Set *canceled as true in the callback function to disable manual + // compaction, 2 Notify begin compaction (return without notifying), 3 Notify + // compaction end (return without notifying). + ASSERT_TRUE(dbfull() + ->CompactRange(compact_options, nullptr, nullptr) + .IsManualCompactionPaused()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ(listener->num_compaction_started_, 0); + ASSERT_EQ(listener->num_compaction_started_, listener->num_compaction_ended_); + ASSERT_EQ(running_compaction, 0); + + // Case III: 1 Notify begin compaction, 2 Compaction in between + // 3. Set *canceled as true in the callback function to disable manual + // compaction, 4 Notify compaction end. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearCallBack( + "CompactionIterator:ProcessKV"); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::Run:BeforeVerify", [&](void* /*arg*/) { + compact_options.canceled->store(true, std::memory_order_release); + }); + + listener->code_ = Status::kOk; + listener->subcode_ = Status::SubCode::kNone; + + compact_options.canceled->store(false, std::memory_order_release); + ASSERT_OK(dbfull()->CompactRange(compact_options, nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_GT(listener->num_compaction_started_, 0); + ASSERT_EQ(listener->num_compaction_started_, listener->num_compaction_ended_); + + // Compaction job will succeed. + ASSERT_GT(running_compaction, 0); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBTest2, CompactionOnBottomPriorityWithListener) { + int num_levels = 3; + const int kNumFilesTrigger = 4; + + Options options = CurrentOptions(); + env_->SetBackgroundThreads(0, Env::Priority::HIGH); + env_->SetBackgroundThreads(0, Env::Priority::LOW); + env_->SetBackgroundThreads(1, Env::Priority::BOTTOM); + options.env = env_; + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = num_levels; + options.write_buffer_size = 100 << 10; // 100KB + options.target_file_size_base = 32 << 10; // 32KB + options.level0_file_num_compaction_trigger = kNumFilesTrigger; + // Trigger compaction if size amplification exceeds 110% + options.compaction_options_universal.max_size_amplification_percent = 110; + + CancelCompactionListener* listener = new CancelCompactionListener(); + options.listeners.emplace_back(listener); + + DestroyAndReopen(options); + + int num_bottom_thread_compaction_scheduled = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:ForwardToBottomPriPool", + [&](void* /*arg*/) { num_bottom_thread_compaction_scheduled++; }); + + int num_compaction_jobs = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::Run():End", + [&](void* /*arg*/) { num_compaction_jobs++; }); + + listener->code_ = Status::kOk; + listener->subcode_ = Status::SubCode::kNone; + + Random rnd(301); + for (int i = 0; i < 1; ++i) { + for (int num = 0; num < kNumFilesTrigger; num++) { + int key_idx = 0; + GenerateNewFile(&rnd, &key_idx, true /* no_wait */); + // use no_wait above because that one waits for flush and compaction. We + // don't want to wait for compaction because the full compaction is + // intentionally blocked while more files are flushed. + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_GT(num_bottom_thread_compaction_scheduled, 0); + ASSERT_EQ(num_compaction_jobs, 1); + ASSERT_GT(listener->num_compaction_started_, 0); + ASSERT_EQ(listener->num_compaction_started_, listener->num_compaction_ended_); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBTest2, OptimizeForPointLookup) { + Options options = CurrentOptions(); + Close(); + options.OptimizeForPointLookup(2); + ASSERT_OK(DB::Open(options, dbname_, &db_)); + + ASSERT_OK(Put("foo", "v1")); + ASSERT_EQ("v1", Get("foo")); + ASSERT_OK(Flush()); + ASSERT_EQ("v1", Get("foo")); +} + +TEST_F(DBTest2, OptimizeForSmallDB) { + Options options = CurrentOptions(); + Close(); + options.OptimizeForSmallDb(); + + // Find the cache object + ASSERT_TRUE(options.table_factory->IsInstanceOf( + TableFactory::kBlockBasedTableName())); + auto table_options = + options.table_factory->GetOptions(); + + ASSERT_TRUE(table_options != nullptr); + std::shared_ptr cache = table_options->block_cache; + + ASSERT_EQ(0, cache->GetUsage()); + ASSERT_OK(DB::Open(options, dbname_, &db_)); + ASSERT_OK(Put("foo", "v1")); + + // memtable size is costed to the block cache + ASSERT_NE(0, cache->GetUsage()); + + ASSERT_EQ("v1", Get("foo")); + ASSERT_OK(Flush()); + + size_t prev_size = cache->GetUsage(); + // Remember block cache size, so that we can find that + // it is filled after Get(). + // Use pinnable slice so that it can ping the block so that + // when we check the size it is not evicted. + PinnableSlice value; + ASSERT_OK(db_->Get(ReadOptions(), db_->DefaultColumnFamily(), "foo", &value)); + ASSERT_GT(cache->GetUsage(), prev_size); + value.Reset(); +} + + +TEST_F(DBTest2, IterRaceFlush1) { + ASSERT_OK(Put("foo", "v1")); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::NewIterator:1", "DBTest2::IterRaceFlush:1"}, + {"DBTest2::IterRaceFlush:2", "DBImpl::NewIterator:2"}}); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ROCKSDB_NAMESPACE::port::Thread t1([&] { + TEST_SYNC_POINT("DBTest2::IterRaceFlush:1"); + ASSERT_OK(Put("foo", "v2")); + ASSERT_OK(Flush()); + TEST_SYNC_POINT("DBTest2::IterRaceFlush:2"); + }); + + // iterator is created after the first Put(), and its snapshot sequence is + // assigned after second Put(), so it must see v2. + { + std::unique_ptr it(db_->NewIterator(ReadOptions())); + it->Seek("foo"); + ASSERT_TRUE(it->Valid()); + ASSERT_OK(it->status()); + ASSERT_EQ("foo", it->key().ToString()); + ASSERT_EQ("v2", it->value().ToString()); + } + + t1.join(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBTest2, IterRaceFlush2) { + ASSERT_OK(Put("foo", "v1")); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::NewIterator:3", "DBTest2::IterRaceFlush2:1"}, + {"DBTest2::IterRaceFlush2:2", "DBImpl::NewIterator:4"}}); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ROCKSDB_NAMESPACE::port::Thread t1([&] { + TEST_SYNC_POINT("DBTest2::IterRaceFlush2:1"); + ASSERT_OK(Put("foo", "v2")); + ASSERT_OK(Flush()); + TEST_SYNC_POINT("DBTest2::IterRaceFlush2:2"); + }); + + // iterator is created after the first Put(), and its snapshot sequence is + // assigned before second Put(), thus it must see v1. + { + std::unique_ptr it(db_->NewIterator(ReadOptions())); + it->Seek("foo"); + ASSERT_TRUE(it->Valid()); + ASSERT_OK(it->status()); + ASSERT_EQ("foo", it->key().ToString()); + ASSERT_EQ("v1", it->value().ToString()); + } + + t1.join(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBTest2, IterRefreshRaceFlush) { + ASSERT_OK(Put("foo", "v1")); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"ArenaWrappedDBIter::Refresh:1", "DBTest2::IterRefreshRaceFlush:1"}, + {"DBTest2::IterRefreshRaceFlush:2", "ArenaWrappedDBIter::Refresh:2"}}); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ROCKSDB_NAMESPACE::port::Thread t1([&] { + TEST_SYNC_POINT("DBTest2::IterRefreshRaceFlush:1"); + ASSERT_OK(Put("foo", "v2")); + ASSERT_OK(Flush()); + TEST_SYNC_POINT("DBTest2::IterRefreshRaceFlush:2"); + }); + + // iterator is refreshed after the first Put(), and its sequence number is + // assigned after second Put(), thus it must see v2. + { + std::unique_ptr it(db_->NewIterator(ReadOptions())); + ASSERT_OK(it->status()); + ASSERT_OK(it->Refresh()); + it->Seek("foo"); + ASSERT_TRUE(it->Valid()); + ASSERT_OK(it->status()); + ASSERT_EQ("foo", it->key().ToString()); + ASSERT_EQ("v2", it->value().ToString()); + } + + t1.join(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBTest2, GetRaceFlush1) { + ASSERT_OK(Put("foo", "v1")); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::GetImpl:1", "DBTest2::GetRaceFlush:1"}, + {"DBTest2::GetRaceFlush:2", "DBImpl::GetImpl:2"}}); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ROCKSDB_NAMESPACE::port::Thread t1([&] { + TEST_SYNC_POINT("DBTest2::GetRaceFlush:1"); + ASSERT_OK(Put("foo", "v2")); + ASSERT_OK(Flush()); + TEST_SYNC_POINT("DBTest2::GetRaceFlush:2"); + }); + + // Get() is issued after the first Put(), so it should see either + // "v1" or "v2". + ASSERT_NE("NOT_FOUND", Get("foo")); + t1.join(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBTest2, GetRaceFlush2) { + ASSERT_OK(Put("foo", "v1")); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::GetImpl:3", "DBTest2::GetRaceFlush:1"}, + {"DBTest2::GetRaceFlush:2", "DBImpl::GetImpl:4"}}); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + port::Thread t1([&] { + TEST_SYNC_POINT("DBTest2::GetRaceFlush:1"); + ASSERT_OK(Put("foo", "v2")); + ASSERT_OK(Flush()); + TEST_SYNC_POINT("DBTest2::GetRaceFlush:2"); + }); + + // Get() is issued after the first Put(), so it should see either + // "v1" or "v2". + ASSERT_NE("NOT_FOUND", Get("foo")); + t1.join(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBTest2, DirectIO) { + if (!IsDirectIOSupported()) { + return; + } + Options options = CurrentOptions(); + options.use_direct_reads = options.use_direct_io_for_flush_and_compaction = + true; + options.allow_mmap_reads = options.allow_mmap_writes = false; + DestroyAndReopen(options); + + ASSERT_OK(Put(Key(0), "a")); + ASSERT_OK(Put(Key(5), "a")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put(Key(10), "a")); + ASSERT_OK(Put(Key(15), "a")); + ASSERT_OK(Flush()); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + Reopen(options); +} + +TEST_F(DBTest2, MemtableOnlyIterator) { + Options options = CurrentOptions(); + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(1, "foo", "first")); + ASSERT_OK(Put(1, "bar", "second")); + + ReadOptions ropt; + ropt.read_tier = kMemtableTier; + std::string value; + Iterator* it = nullptr; + + // Before flushing + // point lookups + ASSERT_OK(db_->Get(ropt, handles_[1], "foo", &value)); + ASSERT_EQ("first", value); + ASSERT_OK(db_->Get(ropt, handles_[1], "bar", &value)); + ASSERT_EQ("second", value); + + // Memtable-only iterator (read_tier=kMemtableTier); data not flushed yet. + it = db_->NewIterator(ropt, handles_[1]); + int count = 0; + for (it->SeekToFirst(); it->Valid(); it->Next()) { + ASSERT_TRUE(it->Valid()); + count++; + } + ASSERT_TRUE(!it->Valid()); + ASSERT_EQ(2, count); + delete it; + + Flush(1); + + // After flushing + // point lookups + ASSERT_OK(db_->Get(ropt, handles_[1], "foo", &value)); + ASSERT_EQ("first", value); + ASSERT_OK(db_->Get(ropt, handles_[1], "bar", &value)); + ASSERT_EQ("second", value); + // nothing should be returned using memtable-only iterator after flushing. + it = db_->NewIterator(ropt, handles_[1]); + ASSERT_OK(it->status()); + count = 0; + for (it->SeekToFirst(); it->Valid(); it->Next()) { + ASSERT_TRUE(it->Valid()); + count++; + } + ASSERT_TRUE(!it->Valid()); + ASSERT_EQ(0, count); + ASSERT_OK(it->status()); + delete it; + + // Add a key to memtable + ASSERT_OK(Put(1, "foobar", "third")); + it = db_->NewIterator(ropt, handles_[1]); + ASSERT_OK(it->status()); + count = 0; + for (it->SeekToFirst(); it->Valid(); it->Next()) { + ASSERT_TRUE(it->Valid()); + ASSERT_EQ("foobar", it->key().ToString()); + ASSERT_EQ("third", it->value().ToString()); + count++; + } + ASSERT_TRUE(!it->Valid()); + ASSERT_EQ(1, count); + ASSERT_OK(it->status()); + delete it; +} + +TEST_F(DBTest2, LowPriWrite) { + Options options = CurrentOptions(); + // Compaction pressure should trigger since 6 files + options.level0_file_num_compaction_trigger = 4; + options.level0_slowdown_writes_trigger = 12; + options.level0_stop_writes_trigger = 30; + options.delayed_write_rate = 8 * 1024 * 1024; + Reopen(options); + + std::atomic rate_limit_count(0); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "GenericRateLimiter::Request:1", [&](void* arg) { + rate_limit_count.fetch_add(1); + int64_t* rate_bytes_per_sec = static_cast(arg); + ASSERT_EQ(1024 * 1024, *rate_bytes_per_sec); + }); + // Block compaction + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"DBTest.LowPriWrite:0", "DBImpl::BGWorkCompaction"}, + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + WriteOptions wo; + for (int i = 0; i < 6; i++) { + wo.low_pri = false; + ASSERT_OK(Put("", "", wo)); + wo.low_pri = true; + ASSERT_OK(Put("", "", wo)); + ASSERT_OK(Flush()); + } + ASSERT_EQ(0, rate_limit_count.load()); + wo.low_pri = true; + ASSERT_OK(Put("", "", wo)); + ASSERT_EQ(1, rate_limit_count.load()); + wo.low_pri = false; + ASSERT_OK(Put("", "", wo)); + ASSERT_EQ(1, rate_limit_count.load()); + + TEST_SYNC_POINT("DBTest.LowPriWrite:0"); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + wo.low_pri = true; + ASSERT_OK(Put("", "", wo)); + ASSERT_EQ(1, rate_limit_count.load()); + wo.low_pri = false; + ASSERT_OK(Put("", "", wo)); + ASSERT_EQ(1, rate_limit_count.load()); +} + +TEST_F(DBTest2, RateLimitedCompactionReads) { + // compaction input has 512KB data + const int kNumKeysPerFile = 128; + const int kBytesPerKey = 1024; + const int kNumL0Files = 4; + + for (int compaction_readahead_size : {0, 32 << 10}) { + for (auto use_direct_io : {false, true}) { + if (use_direct_io && !IsDirectIOSupported()) { + continue; + } + Options options = CurrentOptions(); + options.compaction_readahead_size = compaction_readahead_size; + options.compression = kNoCompression; + options.level0_file_num_compaction_trigger = kNumL0Files; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + // takes roughly one second, split into 100 x 10ms intervals. Each + // interval permits 5.12KB, which is smaller than the block size, so this + // test exercises the code for chunking reads. + options.rate_limiter.reset(NewGenericRateLimiter( + static_cast(kNumL0Files * kNumKeysPerFile * + kBytesPerKey) /* rate_bytes_per_sec */, + 10 * 1000 /* refill_period_us */, 10 /* fairness */, + RateLimiter::Mode::kReadsOnly)); + options.use_direct_reads = + options.use_direct_io_for_flush_and_compaction = use_direct_io; + BlockBasedTableOptions bbto; + bbto.block_size = 16384; + bbto.no_block_cache = true; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + DestroyAndReopen(options); + + for (int i = 0; i < kNumL0Files; ++i) { + for (int j = 0; j <= kNumKeysPerFile; ++j) { + ASSERT_OK(Put(Key(j), DummyString(kBytesPerKey))); + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + if (i + 1 < kNumL0Files) { + ASSERT_EQ(i + 1, NumTableFilesAtLevel(0)); + } + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + + // should be slightly above 512KB due to non-data blocks read. Arbitrarily + // chose 1MB as the upper bound on the total bytes read. + size_t rate_limited_bytes = static_cast( + options.rate_limiter->GetTotalBytesThrough(Env::IO_TOTAL)); + // The charges can exist for `IO_LOW` and `IO_USER` priorities. + size_t rate_limited_bytes_by_pri = + options.rate_limiter->GetTotalBytesThrough(Env::IO_LOW) + + options.rate_limiter->GetTotalBytesThrough(Env::IO_USER); + ASSERT_EQ(rate_limited_bytes, + static_cast(rate_limited_bytes_by_pri)); + // Include the explicit prefetch of the footer in direct I/O case. + size_t direct_io_extra = use_direct_io ? 512 * 1024 : 0; + ASSERT_GE( + rate_limited_bytes, + static_cast(kNumKeysPerFile * kBytesPerKey * kNumL0Files)); + ASSERT_LT( + rate_limited_bytes, + static_cast(2 * kNumKeysPerFile * kBytesPerKey * kNumL0Files + + direct_io_extra)); + + Iterator* iter = db_->NewIterator(ReadOptions()); + ASSERT_OK(iter->status()); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_EQ(iter->value().ToString(), DummyString(kBytesPerKey)); + } + delete iter; + // bytes read for user iterator shouldn't count against the rate limit. + rate_limited_bytes_by_pri = + options.rate_limiter->GetTotalBytesThrough(Env::IO_LOW) + + options.rate_limiter->GetTotalBytesThrough(Env::IO_USER); + ASSERT_EQ(rate_limited_bytes, + static_cast(rate_limited_bytes_by_pri)); + } + } +} + +// Make sure DB can be reopen with reduced number of levels, given no file +// is on levels higher than the new num_levels. +TEST_F(DBTest2, ReduceLevel) { + Options options; + options.env = env_; + options.disable_auto_compactions = true; + options.num_levels = 7; + Reopen(options); + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Flush()); + MoveFilesToLevel(6); + ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel()); + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 1; + ASSERT_OK(dbfull()->CompactRange(compact_options, nullptr, nullptr)); + ASSERT_EQ("0,1", FilesPerLevel()); + options.num_levels = 3; + Reopen(options); + ASSERT_EQ("0,1", FilesPerLevel()); +} + +// Test that ReadCallback is actually used in both memtbale and sst tables +TEST_F(DBTest2, ReadCallbackTest) { + Options options; + options.disable_auto_compactions = true; + options.num_levels = 7; + options.env = env_; + Reopen(options); + std::vector snapshots; + // Try to create a db with multiple layers and a memtable + const std::string key = "foo"; + const std::string value = "bar"; + // This test assumes that the seq start with 1 and increased by 1 after each + // write batch of size 1. If that behavior changes, the test needs to be + // updated as well. + // TODO(myabandeh): update this test to use the seq number that is returned by + // the DB instead of assuming what seq the DB used. + int i = 1; + for (; i < 10; i++) { + ASSERT_OK(Put(key, value + std::to_string(i))); + // Take a snapshot to avoid the value being removed during compaction + auto snapshot = dbfull()->GetSnapshot(); + snapshots.push_back(snapshot); + } + ASSERT_OK(Flush()); + for (; i < 20; i++) { + ASSERT_OK(Put(key, value + std::to_string(i))); + // Take a snapshot to avoid the value being removed during compaction + auto snapshot = dbfull()->GetSnapshot(); + snapshots.push_back(snapshot); + } + ASSERT_OK(Flush()); + MoveFilesToLevel(6); + ASSERT_EQ("0,0,0,0,0,0,2", FilesPerLevel()); + for (; i < 30; i++) { + ASSERT_OK(Put(key, value + std::to_string(i))); + auto snapshot = dbfull()->GetSnapshot(); + snapshots.push_back(snapshot); + } + ASSERT_OK(Flush()); + ASSERT_EQ("1,0,0,0,0,0,2", FilesPerLevel()); + // And also add some values to the memtable + for (; i < 40; i++) { + ASSERT_OK(Put(key, value + std::to_string(i))); + auto snapshot = dbfull()->GetSnapshot(); + snapshots.push_back(snapshot); + } + + class TestReadCallback : public ReadCallback { + public: + explicit TestReadCallback(SequenceNumber snapshot) + : ReadCallback(snapshot), snapshot_(snapshot) {} + bool IsVisibleFullCheck(SequenceNumber seq) override { + return seq <= snapshot_; + } + + private: + SequenceNumber snapshot_; + }; + + for (int seq = 1; seq < i; seq++) { + PinnableSlice pinnable_val; + ReadOptions roptions; + TestReadCallback callback(seq); + bool dont_care = true; + DBImpl::GetImplOptions get_impl_options; + get_impl_options.column_family = dbfull()->DefaultColumnFamily(); + get_impl_options.value = &pinnable_val; + get_impl_options.value_found = &dont_care; + get_impl_options.callback = &callback; + Status s = dbfull()->GetImpl(roptions, key, get_impl_options); + ASSERT_TRUE(s.ok()); + // Assuming that after each Put the DB increased seq by one, the value and + // seq number must be equal since we also inc value by 1 after each Put. + ASSERT_EQ(value + std::to_string(seq), pinnable_val.ToString()); + } + + for (auto snapshot : snapshots) { + dbfull()->ReleaseSnapshot(snapshot); + } +} + + +TEST_F(DBTest2, LiveFilesOmitObsoleteFiles) { + // Regression test for race condition where an obsolete file is returned to + // user as a "live file" but then deleted, all while file deletions are + // disabled. + // + // It happened like this: + // + // 1. [flush thread] Log file "x.log" found by FindObsoleteFiles + // 2. [user thread] DisableFileDeletions, GetSortedWalFiles are called and the + // latter returned "x.log" + // 3. [flush thread] PurgeObsoleteFiles deleted "x.log" + // 4. [user thread] Reading "x.log" failed + // + // Unfortunately the only regression test I can come up with involves sleep. + // We cannot set SyncPoints to repro since, once the fix is applied, the + // SyncPoints would cause a deadlock as the repro's sequence of events is now + // prohibited. + // + // Instead, if we sleep for a second between Find and Purge, and ensure the + // read attempt happens after purge, then the sequence of events will almost + // certainly happen on the old code. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"DBImpl::BackgroundCallFlush:FilesFound", + "DBTest2::LiveFilesOmitObsoleteFiles:FlushTriggered"}, + {"DBImpl::PurgeObsoleteFiles:End", + "DBTest2::LiveFilesOmitObsoleteFiles:LiveFilesCaptured"}, + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::PurgeObsoleteFiles:Begin", + [&](void* /*arg*/) { env_->SleepForMicroseconds(1000000); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put("key", "val")); + FlushOptions flush_opts; + flush_opts.wait = false; + db_->Flush(flush_opts); + TEST_SYNC_POINT("DBTest2::LiveFilesOmitObsoleteFiles:FlushTriggered"); + + ASSERT_OK(db_->DisableFileDeletions()); + VectorLogPtr log_files; + ASSERT_OK(db_->GetSortedWalFiles(log_files)); + TEST_SYNC_POINT("DBTest2::LiveFilesOmitObsoleteFiles:LiveFilesCaptured"); + for (const auto& log_file : log_files) { + ASSERT_OK(env_->FileExists(LogFileName(dbname_, log_file->LogNumber()))); + } + + ASSERT_OK(db_->EnableFileDeletions()); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBTest2, TestNumPread) { + Options options = CurrentOptions(); + bool prefetch_supported = + test::IsPrefetchSupported(env_->GetFileSystem(), dbname_); + // disable block cache + BlockBasedTableOptions table_options; + table_options.no_block_cache = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + env_->count_random_reads_ = true; + env_->random_file_open_counter_.store(0); + ASSERT_OK(Put("bar", "foo")); + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Flush()); + if (prefetch_supported) { + // After flush, we'll open the file and read footer, meta block, + // property block and index block. + ASSERT_EQ(4, env_->random_read_counter_.Read()); + } else { + // With prefetch not supported, we will do a single read into a buffer + ASSERT_EQ(1, env_->random_read_counter_.Read()); + } + ASSERT_EQ(1, env_->random_file_open_counter_.load()); + + // One pread per a normal data block read + env_->random_file_open_counter_.store(0); + env_->random_read_counter_.Reset(); + ASSERT_EQ("bar", Get("foo")); + ASSERT_EQ(1, env_->random_read_counter_.Read()); + // All files are already opened. + ASSERT_EQ(0, env_->random_file_open_counter_.load()); + + env_->random_file_open_counter_.store(0); + env_->random_read_counter_.Reset(); + ASSERT_OK(Put("bar2", "foo2")); + ASSERT_OK(Put("foo2", "bar2")); + ASSERT_OK(Flush()); + if (prefetch_supported) { + // After flush, we'll open the file and read footer, meta block, + // property block and index block. + ASSERT_EQ(4, env_->random_read_counter_.Read()); + } else { + // With prefetch not supported, we will do a single read into a buffer + ASSERT_EQ(1, env_->random_read_counter_.Read()); + } + ASSERT_EQ(1, env_->random_file_open_counter_.load()); + + env_->random_file_open_counter_.store(0); + env_->random_read_counter_.Reset(); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + if (prefetch_supported) { + // Compaction needs two input blocks, which requires 2 preads, and + // generate a new SST file which needs 4 preads (footer, meta block, + // property block and index block). In total 6. + ASSERT_EQ(6, env_->random_read_counter_.Read()); + } else { + // With prefetch off, compaction needs two input blocks, + // followed by a single buffered read. In total 3. + ASSERT_EQ(3, env_->random_read_counter_.Read()); + } + // All compaction input files should have already been opened. + ASSERT_EQ(1, env_->random_file_open_counter_.load()); + + // One pread per a normal data block read + env_->random_file_open_counter_.store(0); + env_->random_read_counter_.Reset(); + ASSERT_EQ("foo2", Get("bar2")); + ASSERT_EQ(1, env_->random_read_counter_.Read()); + // SST files are already opened. + ASSERT_EQ(0, env_->random_file_open_counter_.load()); +} + +class TraceExecutionResultHandler : public TraceRecordResult::Handler { + public: + TraceExecutionResultHandler() {} + ~TraceExecutionResultHandler() override {} + + virtual Status Handle(const StatusOnlyTraceExecutionResult& result) override { + if (result.GetStartTimestamp() > result.GetEndTimestamp()) { + return Status::InvalidArgument("Invalid timestamps."); + } + result.GetStatus().PermitUncheckedError(); + switch (result.GetTraceType()) { + case kTraceWrite: { + total_latency_ += result.GetLatency(); + cnt_++; + writes_++; + break; + } + default: + return Status::Corruption("Type mismatch."); + } + return Status::OK(); + } + + virtual Status Handle( + const SingleValueTraceExecutionResult& result) override { + if (result.GetStartTimestamp() > result.GetEndTimestamp()) { + return Status::InvalidArgument("Invalid timestamps."); + } + result.GetStatus().PermitUncheckedError(); + switch (result.GetTraceType()) { + case kTraceGet: { + total_latency_ += result.GetLatency(); + cnt_++; + gets_++; + break; + } + default: + return Status::Corruption("Type mismatch."); + } + return Status::OK(); + } + + virtual Status Handle( + const MultiValuesTraceExecutionResult& result) override { + if (result.GetStartTimestamp() > result.GetEndTimestamp()) { + return Status::InvalidArgument("Invalid timestamps."); + } + for (const Status& s : result.GetMultiStatus()) { + s.PermitUncheckedError(); + } + switch (result.GetTraceType()) { + case kTraceMultiGet: { + total_latency_ += result.GetLatency(); + cnt_++; + multigets_++; + break; + } + default: + return Status::Corruption("Type mismatch."); + } + return Status::OK(); + } + + virtual Status Handle(const IteratorTraceExecutionResult& result) override { + if (result.GetStartTimestamp() > result.GetEndTimestamp()) { + return Status::InvalidArgument("Invalid timestamps."); + } + result.GetStatus().PermitUncheckedError(); + switch (result.GetTraceType()) { + case kTraceIteratorSeek: + case kTraceIteratorSeekForPrev: { + total_latency_ += result.GetLatency(); + cnt_++; + seeks_++; + break; + } + default: + return Status::Corruption("Type mismatch."); + } + return Status::OK(); + } + + void Reset() { + total_latency_ = 0; + cnt_ = 0; + writes_ = 0; + gets_ = 0; + seeks_ = 0; + multigets_ = 0; + } + + double GetAvgLatency() const { + return cnt_ == 0 ? 0.0 : 1.0 * total_latency_ / cnt_; + } + + int GetNumWrites() const { return writes_; } + + int GetNumGets() const { return gets_; } + + int GetNumIterSeeks() const { return seeks_; } + + int GetNumMultiGets() const { return multigets_; } + + private: + std::atomic total_latency_{0}; + std::atomic cnt_{0}; + std::atomic writes_{0}; + std::atomic gets_{0}; + std::atomic seeks_{0}; + std::atomic multigets_{0}; +}; + +TEST_F(DBTest2, TraceAndReplay) { + Options options = CurrentOptions(); + options.merge_operator = MergeOperators::CreatePutOperator(); + ReadOptions ro; + WriteOptions wo; + TraceOptions trace_opts; + EnvOptions env_opts; + CreateAndReopenWithCF({"pikachu"}, options); + Random rnd(301); + Iterator* single_iter = nullptr; + + ASSERT_TRUE(db_->EndTrace().IsIOError()); + + std::string trace_filename = dbname_ + "/rocksdb.trace"; + std::unique_ptr trace_writer; + ASSERT_OK(NewFileTraceWriter(env_, env_opts, trace_filename, &trace_writer)); + ASSERT_OK(db_->StartTrace(trace_opts, std::move(trace_writer))); + + // 5 Writes + ASSERT_OK(Put(0, "a", "1")); + ASSERT_OK(Merge(0, "b", "2")); + ASSERT_OK(Delete(0, "c")); + ASSERT_OK(SingleDelete(0, "d")); + ASSERT_OK(db_->DeleteRange(wo, dbfull()->DefaultColumnFamily(), "e", "f")); + + // 6th Write + WriteBatch batch; + ASSERT_OK(batch.Put("f", "11")); + ASSERT_OK(batch.Merge("g", "12")); + ASSERT_OK(batch.Delete("h")); + ASSERT_OK(batch.SingleDelete("i")); + ASSERT_OK(batch.DeleteRange("j", "k")); + ASSERT_OK(db_->Write(wo, &batch)); + + // 2 Seek(ForPrev)s + single_iter = db_->NewIterator(ro); + single_iter->Seek("f"); // Seek 1 + single_iter->SeekForPrev("g"); + ASSERT_OK(single_iter->status()); + delete single_iter; + + // 2 Gets + ASSERT_EQ("1", Get(0, "a")); + ASSERT_EQ("12", Get(0, "g")); + + // 7th and 8th Write, 3rd Get + ASSERT_OK(Put(1, "foo", "bar")); + ASSERT_OK(Put(1, "rocksdb", "rocks")); + ASSERT_EQ("NOT_FOUND", Get(1, "leveldb")); + + // Total Write x 8, Get x 3, Seek x 2. + ASSERT_OK(db_->EndTrace()); + // These should not get into the trace file as it is after EndTrace. + ASSERT_OK(Put("hello", "world")); + ASSERT_OK(Merge("foo", "bar")); + + // Open another db, replay, and verify the data + std::string value; + std::string dbname2 = test::PerThreadDBPath(env_, "/db_replay"); + ASSERT_OK(DestroyDB(dbname2, options)); + + // Using a different name than db2, to pacify infer's use-after-lifetime + // warnings (http://fbinfer.com). + DB* db2_init = nullptr; + options.create_if_missing = true; + ASSERT_OK(DB::Open(options, dbname2, &db2_init)); + ColumnFamilyHandle* cf; + ASSERT_OK( + db2_init->CreateColumnFamily(ColumnFamilyOptions(), "pikachu", &cf)); + delete cf; + delete db2_init; + + DB* db2 = nullptr; + std::vector column_families; + ColumnFamilyOptions cf_options; + cf_options.merge_operator = MergeOperators::CreatePutOperator(); + column_families.push_back(ColumnFamilyDescriptor("default", cf_options)); + column_families.push_back( + ColumnFamilyDescriptor("pikachu", ColumnFamilyOptions())); + std::vector handles; + DBOptions db_opts; + db_opts.env = env_; + ASSERT_OK(DB::Open(db_opts, dbname2, column_families, &handles, &db2)); + + env_->SleepForMicroseconds(100); + // Verify that the keys don't already exist + ASSERT_TRUE(db2->Get(ro, handles[0], "a", &value).IsNotFound()); + ASSERT_TRUE(db2->Get(ro, handles[0], "g", &value).IsNotFound()); + + std::unique_ptr trace_reader; + ASSERT_OK(NewFileTraceReader(env_, env_opts, trace_filename, &trace_reader)); + std::unique_ptr replayer; + ASSERT_OK( + db2->NewDefaultReplayer(handles, std::move(trace_reader), &replayer)); + + TraceExecutionResultHandler res_handler; + std::function &&)> res_cb = + [&res_handler](Status exec_s, std::unique_ptr&& res) { + ASSERT_TRUE(exec_s.ok() || exec_s.IsNotSupported()); + if (res != nullptr) { + ASSERT_OK(res->Accept(&res_handler)); + res.reset(); + } + }; + + // Unprepared replay should fail with Status::Incomplete() + ASSERT_TRUE(replayer->Replay(ReplayOptions(), nullptr).IsIncomplete()); + ASSERT_OK(replayer->Prepare()); + // Ok to repeatedly Prepare(). + ASSERT_OK(replayer->Prepare()); + // Replay using 1 thread, 1x speed. + ASSERT_OK(replayer->Replay(ReplayOptions(1, 1.0), res_cb)); + ASSERT_GE(res_handler.GetAvgLatency(), 0.0); + ASSERT_EQ(res_handler.GetNumWrites(), 8); + ASSERT_EQ(res_handler.GetNumGets(), 3); + ASSERT_EQ(res_handler.GetNumIterSeeks(), 2); + ASSERT_EQ(res_handler.GetNumMultiGets(), 0); + res_handler.Reset(); + + ASSERT_OK(db2->Get(ro, handles[0], "a", &value)); + ASSERT_EQ("1", value); + ASSERT_OK(db2->Get(ro, handles[0], "g", &value)); + ASSERT_EQ("12", value); + ASSERT_TRUE(db2->Get(ro, handles[0], "hello", &value).IsNotFound()); + ASSERT_TRUE(db2->Get(ro, handles[0], "world", &value).IsNotFound()); + + ASSERT_OK(db2->Get(ro, handles[1], "foo", &value)); + ASSERT_EQ("bar", value); + ASSERT_OK(db2->Get(ro, handles[1], "rocksdb", &value)); + ASSERT_EQ("rocks", value); + + // Re-replay should fail with Status::Incomplete() if Prepare() was not + // called. Currently we don't distinguish between unprepared and trace end. + ASSERT_TRUE(replayer->Replay(ReplayOptions(), nullptr).IsIncomplete()); + + // Re-replay using 2 threads, 2x speed. + ASSERT_OK(replayer->Prepare()); + ASSERT_OK(replayer->Replay(ReplayOptions(2, 2.0), res_cb)); + ASSERT_GE(res_handler.GetAvgLatency(), 0.0); + ASSERT_EQ(res_handler.GetNumWrites(), 8); + ASSERT_EQ(res_handler.GetNumGets(), 3); + ASSERT_EQ(res_handler.GetNumIterSeeks(), 2); + ASSERT_EQ(res_handler.GetNumMultiGets(), 0); + res_handler.Reset(); + + // Re-replay using 2 threads, 1/2 speed. + ASSERT_OK(replayer->Prepare()); + ASSERT_OK(replayer->Replay(ReplayOptions(2, 0.5), res_cb)); + ASSERT_GE(res_handler.GetAvgLatency(), 0.0); + ASSERT_EQ(res_handler.GetNumWrites(), 8); + ASSERT_EQ(res_handler.GetNumGets(), 3); + ASSERT_EQ(res_handler.GetNumIterSeeks(), 2); + ASSERT_EQ(res_handler.GetNumMultiGets(), 0); + res_handler.Reset(); + + replayer.reset(); + + for (auto handle : handles) { + delete handle; + } + delete db2; + ASSERT_OK(DestroyDB(dbname2, options)); +} + +TEST_F(DBTest2, TraceAndManualReplay) { + Options options = CurrentOptions(); + options.merge_operator = MergeOperators::CreatePutOperator(); + ReadOptions ro; + WriteOptions wo; + TraceOptions trace_opts; + EnvOptions env_opts; + CreateAndReopenWithCF({"pikachu"}, options); + Random rnd(301); + Iterator* single_iter = nullptr; + + ASSERT_TRUE(db_->EndTrace().IsIOError()); + + std::string trace_filename = dbname_ + "/rocksdb.trace"; + std::unique_ptr trace_writer; + ASSERT_OK(NewFileTraceWriter(env_, env_opts, trace_filename, &trace_writer)); + ASSERT_OK(db_->StartTrace(trace_opts, std::move(trace_writer))); + + ASSERT_OK(Put(0, "a", "1")); + ASSERT_OK(Merge(0, "b", "2")); + ASSERT_OK(Delete(0, "c")); + ASSERT_OK(SingleDelete(0, "d")); + ASSERT_OK(db_->DeleteRange(wo, dbfull()->DefaultColumnFamily(), "e", "f")); + + WriteBatch batch; + ASSERT_OK(batch.Put("f", "11")); + ASSERT_OK(batch.Merge("g", "12")); + ASSERT_OK(batch.Delete("h")); + ASSERT_OK(batch.SingleDelete("i")); + ASSERT_OK(batch.DeleteRange("j", "k")); + ASSERT_OK(db_->Write(wo, &batch)); + + single_iter = db_->NewIterator(ro); + single_iter->Seek("f"); + single_iter->SeekForPrev("g"); + ASSERT_OK(single_iter->status()); + delete single_iter; + + // Write some sequenced keys for testing lower/upper bounds of iterator. + batch.Clear(); + ASSERT_OK(batch.Put("iter-0", "iter-0")); + ASSERT_OK(batch.Put("iter-1", "iter-1")); + ASSERT_OK(batch.Put("iter-2", "iter-2")); + ASSERT_OK(batch.Put("iter-3", "iter-3")); + ASSERT_OK(batch.Put("iter-4", "iter-4")); + ASSERT_OK(db_->Write(wo, &batch)); + + ReadOptions bounded_ro = ro; + Slice lower_bound("iter-1"); + Slice upper_bound("iter-3"); + bounded_ro.iterate_lower_bound = &lower_bound; + bounded_ro.iterate_upper_bound = &upper_bound; + single_iter = db_->NewIterator(bounded_ro); + single_iter->Seek("iter-0"); + ASSERT_EQ(single_iter->key().ToString(), "iter-1"); + single_iter->Seek("iter-2"); + ASSERT_EQ(single_iter->key().ToString(), "iter-2"); + single_iter->Seek("iter-4"); + ASSERT_FALSE(single_iter->Valid()); + single_iter->SeekForPrev("iter-0"); + ASSERT_FALSE(single_iter->Valid()); + single_iter->SeekForPrev("iter-2"); + ASSERT_EQ(single_iter->key().ToString(), "iter-2"); + single_iter->SeekForPrev("iter-4"); + ASSERT_EQ(single_iter->key().ToString(), "iter-2"); + ASSERT_OK(single_iter->status()); + delete single_iter; + + ASSERT_EQ("1", Get(0, "a")); + ASSERT_EQ("12", Get(0, "g")); + + ASSERT_OK(Put(1, "foo", "bar")); + ASSERT_OK(Put(1, "rocksdb", "rocks")); + ASSERT_EQ("NOT_FOUND", Get(1, "leveldb")); + + // Same as TraceAndReplay, Write x 8, Get x 3, Seek x 2. + // Plus 1 WriteBatch for iterator with lower/upper bounds, and 6 + // Seek(ForPrev)s. + // Total Write x 9, Get x 3, Seek x 8 + ASSERT_OK(db_->EndTrace()); + // These should not get into the trace file as it is after EndTrace. + ASSERT_OK(Put("hello", "world")); + ASSERT_OK(Merge("foo", "bar")); + + // Open another db, replay, and verify the data + std::string value; + std::string dbname2 = test::PerThreadDBPath(env_, "/db_replay"); + ASSERT_OK(DestroyDB(dbname2, options)); + + // Using a different name than db2, to pacify infer's use-after-lifetime + // warnings (http://fbinfer.com). + DB* db2_init = nullptr; + options.create_if_missing = true; + ASSERT_OK(DB::Open(options, dbname2, &db2_init)); + ColumnFamilyHandle* cf; + ASSERT_OK( + db2_init->CreateColumnFamily(ColumnFamilyOptions(), "pikachu", &cf)); + delete cf; + delete db2_init; + + DB* db2 = nullptr; + std::vector column_families; + ColumnFamilyOptions cf_options; + cf_options.merge_operator = MergeOperators::CreatePutOperator(); + column_families.push_back(ColumnFamilyDescriptor("default", cf_options)); + column_families.push_back( + ColumnFamilyDescriptor("pikachu", ColumnFamilyOptions())); + std::vector handles; + DBOptions db_opts; + db_opts.env = env_; + ASSERT_OK(DB::Open(db_opts, dbname2, column_families, &handles, &db2)); + + env_->SleepForMicroseconds(100); + // Verify that the keys don't already exist + ASSERT_TRUE(db2->Get(ro, handles[0], "a", &value).IsNotFound()); + ASSERT_TRUE(db2->Get(ro, handles[0], "g", &value).IsNotFound()); + + std::unique_ptr trace_reader; + ASSERT_OK(NewFileTraceReader(env_, env_opts, trace_filename, &trace_reader)); + std::unique_ptr replayer; + ASSERT_OK( + db2->NewDefaultReplayer(handles, std::move(trace_reader), &replayer)); + + TraceExecutionResultHandler res_handler; + + // Manual replay for 2 times. The 2nd checks if the replay can restart. + std::unique_ptr record; + std::unique_ptr result; + for (int i = 0; i < 2; i++) { + // Next should fail if unprepared. + ASSERT_TRUE(replayer->Next(nullptr).IsIncomplete()); + ASSERT_OK(replayer->Prepare()); + Status s = Status::OK(); + // Looping until trace end. + while (s.ok()) { + s = replayer->Next(&record); + // Skip unsupported operations. + if (s.IsNotSupported()) { + continue; + } + if (s.ok()) { + ASSERT_OK(replayer->Execute(record, &result)); + if (result != nullptr) { + ASSERT_OK(result->Accept(&res_handler)); + if (record->GetTraceType() == kTraceIteratorSeek || + record->GetTraceType() == kTraceIteratorSeekForPrev) { + IteratorSeekQueryTraceRecord* iter_rec = + dynamic_cast(record.get()); + IteratorTraceExecutionResult* iter_res = + dynamic_cast(result.get()); + // Check if lower/upper bounds are correctly saved and decoded. + std::string lower_str = iter_rec->GetLowerBound().ToString(); + std::string upper_str = iter_rec->GetUpperBound().ToString(); + std::string iter_key = iter_res->GetKey().ToString(); + std::string iter_value = iter_res->GetValue().ToString(); + if (!lower_str.empty() && !upper_str.empty()) { + ASSERT_EQ(lower_str, "iter-1"); + ASSERT_EQ(upper_str, "iter-3"); + if (iter_res->GetValid()) { + // If iterator is valid, then lower_bound <= key < upper_bound. + ASSERT_GE(iter_key, lower_str); + ASSERT_LT(iter_key, upper_str); + } else { + // If iterator is invalid, then + // key < lower_bound or key >= upper_bound. + ASSERT_TRUE(iter_key < lower_str || iter_key >= upper_str); + } + } + // If iterator is invalid, the key and value should be empty. + if (!iter_res->GetValid()) { + ASSERT_TRUE(iter_key.empty()); + ASSERT_TRUE(iter_value.empty()); + } + } + result.reset(); + } + } + } + // Status::Incomplete() will be returned when manually reading the trace + // end, or Prepare() was not called. + ASSERT_TRUE(s.IsIncomplete()); + ASSERT_TRUE(replayer->Next(nullptr).IsIncomplete()); + ASSERT_GE(res_handler.GetAvgLatency(), 0.0); + ASSERT_EQ(res_handler.GetNumWrites(), 9); + ASSERT_EQ(res_handler.GetNumGets(), 3); + ASSERT_EQ(res_handler.GetNumIterSeeks(), 8); + ASSERT_EQ(res_handler.GetNumMultiGets(), 0); + res_handler.Reset(); + } + + ASSERT_OK(db2->Get(ro, handles[0], "a", &value)); + ASSERT_EQ("1", value); + ASSERT_OK(db2->Get(ro, handles[0], "g", &value)); + ASSERT_EQ("12", value); + ASSERT_TRUE(db2->Get(ro, handles[0], "hello", &value).IsNotFound()); + ASSERT_TRUE(db2->Get(ro, handles[0], "world", &value).IsNotFound()); + + ASSERT_OK(db2->Get(ro, handles[1], "foo", &value)); + ASSERT_EQ("bar", value); + ASSERT_OK(db2->Get(ro, handles[1], "rocksdb", &value)); + ASSERT_EQ("rocks", value); + + // Test execution of artificially created TraceRecords. + uint64_t fake_ts = 1U; + // Write + batch.Clear(); + ASSERT_OK(batch.Put("trace-record-write1", "write1")); + ASSERT_OK(batch.Put("trace-record-write2", "write2")); + record.reset(new WriteQueryTraceRecord(batch.Data(), fake_ts++)); + ASSERT_OK(replayer->Execute(record, &result)); + ASSERT_TRUE(result != nullptr); + ASSERT_OK(result->Accept(&res_handler)); // Write x 1 + ASSERT_OK(db2->Get(ro, handles[0], "trace-record-write1", &value)); + ASSERT_EQ("write1", value); + ASSERT_OK(db2->Get(ro, handles[0], "trace-record-write2", &value)); + ASSERT_EQ("write2", value); + ASSERT_GE(res_handler.GetAvgLatency(), 0.0); + ASSERT_EQ(res_handler.GetNumWrites(), 1); + ASSERT_EQ(res_handler.GetNumGets(), 0); + ASSERT_EQ(res_handler.GetNumIterSeeks(), 0); + ASSERT_EQ(res_handler.GetNumMultiGets(), 0); + res_handler.Reset(); + + // Get related + // Get an existing key. + record.reset(new GetQueryTraceRecord(handles[0]->GetID(), + "trace-record-write1", fake_ts++)); + ASSERT_OK(replayer->Execute(record, &result)); + ASSERT_TRUE(result != nullptr); + ASSERT_OK(result->Accept(&res_handler)); // Get x 1 + // Get an non-existing key, should still return Status::OK(). + record.reset(new GetQueryTraceRecord(handles[0]->GetID(), "trace-record-get", + fake_ts++)); + ASSERT_OK(replayer->Execute(record, &result)); + ASSERT_TRUE(result != nullptr); + ASSERT_OK(result->Accept(&res_handler)); // Get x 2 + // Get from an invalid (non-existing) cf_id. + uint32_t invalid_cf_id = handles[1]->GetID() + 1; + record.reset(new GetQueryTraceRecord(invalid_cf_id, "whatever", fake_ts++)); + ASSERT_TRUE(replayer->Execute(record, &result).IsCorruption()); + ASSERT_TRUE(result == nullptr); + ASSERT_GE(res_handler.GetAvgLatency(), 0.0); + ASSERT_EQ(res_handler.GetNumWrites(), 0); + ASSERT_EQ(res_handler.GetNumGets(), 2); + ASSERT_EQ(res_handler.GetNumIterSeeks(), 0); + ASSERT_EQ(res_handler.GetNumMultiGets(), 0); + res_handler.Reset(); + + // Iteration related + for (IteratorSeekQueryTraceRecord::SeekType seekType : + {IteratorSeekQueryTraceRecord::kSeek, + IteratorSeekQueryTraceRecord::kSeekForPrev}) { + // Seek to an existing key. + record.reset(new IteratorSeekQueryTraceRecord( + seekType, handles[0]->GetID(), "trace-record-write1", fake_ts++)); + ASSERT_OK(replayer->Execute(record, &result)); + ASSERT_TRUE(result != nullptr); + ASSERT_OK(result->Accept(&res_handler)); // Seek x 1 in one iteration + // Seek to an non-existing key, should still return Status::OK(). + record.reset(new IteratorSeekQueryTraceRecord( + seekType, handles[0]->GetID(), "trace-record-get", fake_ts++)); + ASSERT_OK(replayer->Execute(record, &result)); + ASSERT_TRUE(result != nullptr); + ASSERT_OK(result->Accept(&res_handler)); // Seek x 2 in one iteration + // Seek from an invalid cf_id. + record.reset(new IteratorSeekQueryTraceRecord(seekType, invalid_cf_id, + "whatever", fake_ts++)); + ASSERT_TRUE(replayer->Execute(record, &result).IsCorruption()); + ASSERT_TRUE(result == nullptr); + } + ASSERT_GE(res_handler.GetAvgLatency(), 0.0); + ASSERT_EQ(res_handler.GetNumWrites(), 0); + ASSERT_EQ(res_handler.GetNumGets(), 0); + ASSERT_EQ(res_handler.GetNumIterSeeks(), 4); // Seek x 2 in two iterations + ASSERT_EQ(res_handler.GetNumMultiGets(), 0); + res_handler.Reset(); + + // MultiGet related + // Get existing keys. + record.reset(new MultiGetQueryTraceRecord( + std::vector({handles[0]->GetID(), handles[1]->GetID()}), + std::vector({"a", "foo"}), fake_ts++)); + ASSERT_OK(replayer->Execute(record, &result)); + ASSERT_TRUE(result != nullptr); + ASSERT_OK(result->Accept(&res_handler)); // MultiGet x 1 + // Get all non-existing keys, should still return Status::OK(). + record.reset(new MultiGetQueryTraceRecord( + std::vector({handles[0]->GetID(), handles[1]->GetID()}), + std::vector({"no1", "no2"}), fake_ts++)); + ASSERT_OK(replayer->Execute(record, &result)); + ASSERT_TRUE(result != nullptr); + ASSERT_OK(result->Accept(&res_handler)); // MultiGet x 2 + // Get mixed of existing and non-existing keys, should still return + // Status::OK(). + record.reset(new MultiGetQueryTraceRecord( + std::vector({handles[0]->GetID(), handles[1]->GetID()}), + std::vector({"a", "no2"}), fake_ts++)); + ASSERT_OK(replayer->Execute(record, &result)); + ASSERT_TRUE(result != nullptr); + MultiValuesTraceExecutionResult* mvr = + dynamic_cast(result.get()); + ASSERT_TRUE(mvr != nullptr); + ASSERT_OK(mvr->GetMultiStatus()[0]); + ASSERT_TRUE(mvr->GetMultiStatus()[1].IsNotFound()); + ASSERT_EQ(mvr->GetValues()[0], "1"); + ASSERT_EQ(mvr->GetValues()[1], ""); + ASSERT_OK(result->Accept(&res_handler)); // MultiGet x 3 + // Get from an invalid (non-existing) cf_id. + record.reset(new MultiGetQueryTraceRecord( + std::vector( + {handles[0]->GetID(), handles[1]->GetID(), invalid_cf_id}), + std::vector({"a", "foo", "whatever"}), fake_ts++)); + ASSERT_TRUE(replayer->Execute(record, &result).IsCorruption()); + ASSERT_TRUE(result == nullptr); + // Empty MultiGet + record.reset(new MultiGetQueryTraceRecord( + std::vector(), std::vector(), fake_ts++)); + ASSERT_TRUE(replayer->Execute(record, &result).IsInvalidArgument()); + ASSERT_TRUE(result == nullptr); + // MultiGet size mismatch + record.reset(new MultiGetQueryTraceRecord( + std::vector({handles[0]->GetID(), handles[1]->GetID()}), + std::vector({"a"}), fake_ts++)); + ASSERT_TRUE(replayer->Execute(record, &result).IsInvalidArgument()); + ASSERT_TRUE(result == nullptr); + ASSERT_GE(res_handler.GetAvgLatency(), 0.0); + ASSERT_EQ(res_handler.GetNumWrites(), 0); + ASSERT_EQ(res_handler.GetNumGets(), 0); + ASSERT_EQ(res_handler.GetNumIterSeeks(), 0); + ASSERT_EQ(res_handler.GetNumMultiGets(), 3); + res_handler.Reset(); + + replayer.reset(); + + for (auto handle : handles) { + delete handle; + } + delete db2; + ASSERT_OK(DestroyDB(dbname2, options)); +} + +TEST_F(DBTest2, TraceWithLimit) { + Options options = CurrentOptions(); + options.merge_operator = MergeOperators::CreatePutOperator(); + ReadOptions ro; + WriteOptions wo; + TraceOptions trace_opts; + EnvOptions env_opts; + CreateAndReopenWithCF({"pikachu"}, options); + Random rnd(301); + + // test the max trace file size options + trace_opts.max_trace_file_size = 5; + std::string trace_filename = dbname_ + "/rocksdb.trace1"; + std::unique_ptr trace_writer; + ASSERT_OK(NewFileTraceWriter(env_, env_opts, trace_filename, &trace_writer)); + ASSERT_OK(db_->StartTrace(trace_opts, std::move(trace_writer))); + ASSERT_OK(Put(0, "a", "1")); + ASSERT_OK(Put(0, "b", "1")); + ASSERT_OK(Put(0, "c", "1")); + ASSERT_OK(db_->EndTrace()); + + std::string dbname2 = test::PerThreadDBPath(env_, "/db_replay2"); + std::string value; + ASSERT_OK(DestroyDB(dbname2, options)); + + // Using a different name than db2, to pacify infer's use-after-lifetime + // warnings (http://fbinfer.com). + DB* db2_init = nullptr; + options.create_if_missing = true; + ASSERT_OK(DB::Open(options, dbname2, &db2_init)); + ColumnFamilyHandle* cf; + ASSERT_OK( + db2_init->CreateColumnFamily(ColumnFamilyOptions(), "pikachu", &cf)); + delete cf; + delete db2_init; + + DB* db2 = nullptr; + std::vector column_families; + ColumnFamilyOptions cf_options; + cf_options.merge_operator = MergeOperators::CreatePutOperator(); + column_families.push_back(ColumnFamilyDescriptor("default", cf_options)); + column_families.push_back( + ColumnFamilyDescriptor("pikachu", ColumnFamilyOptions())); + std::vector handles; + DBOptions db_opts; + db_opts.env = env_; + ASSERT_OK(DB::Open(db_opts, dbname2, column_families, &handles, &db2)); + + env_->SleepForMicroseconds(100); + // Verify that the keys don't already exist + ASSERT_TRUE(db2->Get(ro, handles[0], "a", &value).IsNotFound()); + ASSERT_TRUE(db2->Get(ro, handles[0], "b", &value).IsNotFound()); + ASSERT_TRUE(db2->Get(ro, handles[0], "c", &value).IsNotFound()); + + std::unique_ptr trace_reader; + ASSERT_OK(NewFileTraceReader(env_, env_opts, trace_filename, &trace_reader)); + std::unique_ptr replayer; + ASSERT_OK( + db2->NewDefaultReplayer(handles, std::move(trace_reader), &replayer)); + ASSERT_OK(replayer->Prepare()); + ASSERT_OK(replayer->Replay(ReplayOptions(), nullptr)); + replayer.reset(); + + ASSERT_TRUE(db2->Get(ro, handles[0], "a", &value).IsNotFound()); + ASSERT_TRUE(db2->Get(ro, handles[0], "b", &value).IsNotFound()); + ASSERT_TRUE(db2->Get(ro, handles[0], "c", &value).IsNotFound()); + + for (auto handle : handles) { + delete handle; + } + delete db2; + ASSERT_OK(DestroyDB(dbname2, options)); +} + +TEST_F(DBTest2, TraceWithSampling) { + Options options = CurrentOptions(); + ReadOptions ro; + WriteOptions wo; + TraceOptions trace_opts; + EnvOptions env_opts; + CreateAndReopenWithCF({"pikachu"}, options); + Random rnd(301); + + // test the trace file sampling options + trace_opts.sampling_frequency = 2; + std::string trace_filename = dbname_ + "/rocksdb.trace_sampling"; + std::unique_ptr trace_writer; + ASSERT_OK(NewFileTraceWriter(env_, env_opts, trace_filename, &trace_writer)); + ASSERT_OK(db_->StartTrace(trace_opts, std::move(trace_writer))); + ASSERT_OK(Put(0, "a", "1")); + ASSERT_OK(Put(0, "b", "2")); + ASSERT_OK(Put(0, "c", "3")); + ASSERT_OK(Put(0, "d", "4")); + ASSERT_OK(Put(0, "e", "5")); + ASSERT_OK(db_->EndTrace()); + + std::string dbname2 = test::PerThreadDBPath(env_, "/db_replay_sampling"); + std::string value; + ASSERT_OK(DestroyDB(dbname2, options)); + + // Using a different name than db2, to pacify infer's use-after-lifetime + // warnings (http://fbinfer.com). + DB* db2_init = nullptr; + options.create_if_missing = true; + ASSERT_OK(DB::Open(options, dbname2, &db2_init)); + ColumnFamilyHandle* cf; + ASSERT_OK( + db2_init->CreateColumnFamily(ColumnFamilyOptions(), "pikachu", &cf)); + delete cf; + delete db2_init; + + DB* db2 = nullptr; + std::vector column_families; + ColumnFamilyOptions cf_options; + column_families.push_back(ColumnFamilyDescriptor("default", cf_options)); + column_families.push_back( + ColumnFamilyDescriptor("pikachu", ColumnFamilyOptions())); + std::vector handles; + DBOptions db_opts; + db_opts.env = env_; + ASSERT_OK(DB::Open(db_opts, dbname2, column_families, &handles, &db2)); + + env_->SleepForMicroseconds(100); + ASSERT_TRUE(db2->Get(ro, handles[0], "a", &value).IsNotFound()); + ASSERT_TRUE(db2->Get(ro, handles[0], "b", &value).IsNotFound()); + ASSERT_TRUE(db2->Get(ro, handles[0], "c", &value).IsNotFound()); + ASSERT_TRUE(db2->Get(ro, handles[0], "d", &value).IsNotFound()); + ASSERT_TRUE(db2->Get(ro, handles[0], "e", &value).IsNotFound()); + + std::unique_ptr trace_reader; + ASSERT_OK(NewFileTraceReader(env_, env_opts, trace_filename, &trace_reader)); + std::unique_ptr replayer; + ASSERT_OK( + db2->NewDefaultReplayer(handles, std::move(trace_reader), &replayer)); + ASSERT_OK(replayer->Prepare()); + ASSERT_OK(replayer->Replay(ReplayOptions(), nullptr)); + replayer.reset(); + + ASSERT_TRUE(db2->Get(ro, handles[0], "a", &value).IsNotFound()); + ASSERT_FALSE(db2->Get(ro, handles[0], "b", &value).IsNotFound()); + ASSERT_TRUE(db2->Get(ro, handles[0], "c", &value).IsNotFound()); + ASSERT_FALSE(db2->Get(ro, handles[0], "d", &value).IsNotFound()); + ASSERT_TRUE(db2->Get(ro, handles[0], "e", &value).IsNotFound()); + + for (auto handle : handles) { + delete handle; + } + delete db2; + ASSERT_OK(DestroyDB(dbname2, options)); +} + +TEST_F(DBTest2, TraceWithFilter) { + Options options = CurrentOptions(); + options.merge_operator = MergeOperators::CreatePutOperator(); + ReadOptions ro; + WriteOptions wo; + TraceOptions trace_opts; + EnvOptions env_opts; + CreateAndReopenWithCF({"pikachu"}, options); + Random rnd(301); + Iterator* single_iter = nullptr; + + trace_opts.filter = TraceFilterType::kTraceFilterWrite; + + std::string trace_filename = dbname_ + "/rocksdb.trace"; + std::unique_ptr trace_writer; + ASSERT_OK(NewFileTraceWriter(env_, env_opts, trace_filename, &trace_writer)); + ASSERT_OK(db_->StartTrace(trace_opts, std::move(trace_writer))); + + ASSERT_OK(Put(0, "a", "1")); + ASSERT_OK(Merge(0, "b", "2")); + ASSERT_OK(Delete(0, "c")); + ASSERT_OK(SingleDelete(0, "d")); + ASSERT_OK(db_->DeleteRange(wo, dbfull()->DefaultColumnFamily(), "e", "f")); + + WriteBatch batch; + ASSERT_OK(batch.Put("f", "11")); + ASSERT_OK(batch.Merge("g", "12")); + ASSERT_OK(batch.Delete("h")); + ASSERT_OK(batch.SingleDelete("i")); + ASSERT_OK(batch.DeleteRange("j", "k")); + ASSERT_OK(db_->Write(wo, &batch)); + + single_iter = db_->NewIterator(ro); + single_iter->Seek("f"); + single_iter->SeekForPrev("g"); + delete single_iter; + + ASSERT_EQ("1", Get(0, "a")); + ASSERT_EQ("12", Get(0, "g")); + + ASSERT_OK(Put(1, "foo", "bar")); + ASSERT_OK(Put(1, "rocksdb", "rocks")); + ASSERT_EQ("NOT_FOUND", Get(1, "leveldb")); + + ASSERT_OK(db_->EndTrace()); + // These should not get into the trace file as it is after EndTrace. + ASSERT_OK(Put("hello", "world")); + ASSERT_OK(Merge("foo", "bar")); + + // Open another db, replay, and verify the data + std::string value; + std::string dbname2 = test::PerThreadDBPath(env_, "db_replay"); + ASSERT_OK(DestroyDB(dbname2, options)); + + // Using a different name than db2, to pacify infer's use-after-lifetime + // warnings (http://fbinfer.com). + DB* db2_init = nullptr; + options.create_if_missing = true; + ASSERT_OK(DB::Open(options, dbname2, &db2_init)); + ColumnFamilyHandle* cf; + ASSERT_OK( + db2_init->CreateColumnFamily(ColumnFamilyOptions(), "pikachu", &cf)); + delete cf; + delete db2_init; + + DB* db2 = nullptr; + std::vector column_families; + ColumnFamilyOptions cf_options; + cf_options.merge_operator = MergeOperators::CreatePutOperator(); + column_families.push_back(ColumnFamilyDescriptor("default", cf_options)); + column_families.push_back( + ColumnFamilyDescriptor("pikachu", ColumnFamilyOptions())); + std::vector handles; + DBOptions db_opts; + db_opts.env = env_; + ASSERT_OK(DB::Open(db_opts, dbname2, column_families, &handles, &db2)); + + env_->SleepForMicroseconds(100); + // Verify that the keys don't already exist + ASSERT_TRUE(db2->Get(ro, handles[0], "a", &value).IsNotFound()); + ASSERT_TRUE(db2->Get(ro, handles[0], "g", &value).IsNotFound()); + + std::unique_ptr trace_reader; + ASSERT_OK(NewFileTraceReader(env_, env_opts, trace_filename, &trace_reader)); + std::unique_ptr replayer; + ASSERT_OK( + db2->NewDefaultReplayer(handles, std::move(trace_reader), &replayer)); + ASSERT_OK(replayer->Prepare()); + ASSERT_OK(replayer->Replay(ReplayOptions(), nullptr)); + replayer.reset(); + + // All the key-values should not present since we filter out the WRITE ops. + ASSERT_TRUE(db2->Get(ro, handles[0], "a", &value).IsNotFound()); + ASSERT_TRUE(db2->Get(ro, handles[0], "g", &value).IsNotFound()); + ASSERT_TRUE(db2->Get(ro, handles[0], "hello", &value).IsNotFound()); + ASSERT_TRUE(db2->Get(ro, handles[0], "world", &value).IsNotFound()); + ASSERT_TRUE(db2->Get(ro, handles[0], "foo", &value).IsNotFound()); + ASSERT_TRUE(db2->Get(ro, handles[0], "rocksdb", &value).IsNotFound()); + + for (auto handle : handles) { + delete handle; + } + delete db2; + ASSERT_OK(DestroyDB(dbname2, options)); + + // Set up a new db. + std::string dbname3 = test::PerThreadDBPath(env_, "db_not_trace_read"); + ASSERT_OK(DestroyDB(dbname3, options)); + + DB* db3_init = nullptr; + options.create_if_missing = true; + ColumnFamilyHandle* cf3; + ASSERT_OK(DB::Open(options, dbname3, &db3_init)); + ASSERT_OK( + db3_init->CreateColumnFamily(ColumnFamilyOptions(), "pikachu", &cf3)); + delete cf3; + delete db3_init; + + column_families.clear(); + column_families.push_back(ColumnFamilyDescriptor("default", cf_options)); + column_families.push_back( + ColumnFamilyDescriptor("pikachu", ColumnFamilyOptions())); + handles.clear(); + + DB* db3 = nullptr; + ASSERT_OK(DB::Open(db_opts, dbname3, column_families, &handles, &db3)); + + env_->SleepForMicroseconds(100); + // Verify that the keys don't already exist + ASSERT_TRUE(db3->Get(ro, handles[0], "a", &value).IsNotFound()); + ASSERT_TRUE(db3->Get(ro, handles[0], "g", &value).IsNotFound()); + + // The tracer will not record the READ ops. + trace_opts.filter = TraceFilterType::kTraceFilterGet; + std::string trace_filename3 = dbname_ + "/rocksdb.trace_3"; + std::unique_ptr trace_writer3; + ASSERT_OK( + NewFileTraceWriter(env_, env_opts, trace_filename3, &trace_writer3)); + ASSERT_OK(db3->StartTrace(trace_opts, std::move(trace_writer3))); + + ASSERT_OK(db3->Put(wo, handles[0], "a", "1")); + ASSERT_OK(db3->Merge(wo, handles[0], "b", "2")); + ASSERT_OK(db3->Delete(wo, handles[0], "c")); + ASSERT_OK(db3->SingleDelete(wo, handles[0], "d")); + + ASSERT_OK(db3->Get(ro, handles[0], "a", &value)); + ASSERT_EQ(value, "1"); + ASSERT_TRUE(db3->Get(ro, handles[0], "c", &value).IsNotFound()); + + ASSERT_OK(db3->EndTrace()); + + for (auto handle : handles) { + delete handle; + } + delete db3; + ASSERT_OK(DestroyDB(dbname3, options)); + + std::unique_ptr trace_reader3; + ASSERT_OK( + NewFileTraceReader(env_, env_opts, trace_filename3, &trace_reader3)); + + // Count the number of records in the trace file; + int count = 0; + std::string data; + Status s; + while (true) { + s = trace_reader3->Read(&data); + if (!s.ok()) { + break; + } + count += 1; + } + // We also need to count the header and footer + // 4 WRITE + HEADER + FOOTER = 6 + ASSERT_EQ(count, 6); +} + + +TEST_F(DBTest2, PinnableSliceAndMmapReads) { + Options options = CurrentOptions(); + options.env = env_; + if (!IsMemoryMappedAccessSupported()) { + ROCKSDB_GTEST_SKIP("Test requires default environment"); + return; + } + options.allow_mmap_reads = true; + options.max_open_files = 100; + options.compression = kNoCompression; + Reopen(options); + + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Flush()); + + PinnableSlice pinned_value; + ASSERT_EQ(Get("foo", &pinned_value), Status::OK()); + // It is not safe to pin mmap files as they might disappear by compaction + ASSERT_FALSE(pinned_value.IsPinned()); + ASSERT_EQ(pinned_value.ToString(), "bar"); + + ASSERT_OK(dbfull()->TEST_CompactRange( + 0 /* level */, nullptr /* begin */, nullptr /* end */, + nullptr /* column_family */, true /* disallow_trivial_move */)); + + // Ensure pinned_value doesn't rely on memory munmap'd by the above + // compaction. It crashes if it does. + ASSERT_EQ(pinned_value.ToString(), "bar"); + + pinned_value.Reset(); + // Unsafe to pin mmap files when they could be kicked out of table cache + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + ASSERT_EQ(Get("foo", &pinned_value), Status::OK()); + ASSERT_FALSE(pinned_value.IsPinned()); + ASSERT_EQ(pinned_value.ToString(), "bar"); + + pinned_value.Reset(); + // In read-only mode with infinite capacity on table cache it should pin the + // value and avoid the memcpy + Close(); + options.max_open_files = -1; + ASSERT_OK(ReadOnlyReopen(options)); + ASSERT_EQ(Get("foo", &pinned_value), Status::OK()); + ASSERT_TRUE(pinned_value.IsPinned()); + ASSERT_EQ(pinned_value.ToString(), "bar"); +} + +TEST_F(DBTest2, DISABLED_IteratorPinnedMemory) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + BlockBasedTableOptions bbto; + bbto.no_block_cache = false; + bbto.cache_index_and_filter_blocks = false; + bbto.block_cache = NewLRUCache(100000); + bbto.block_size = 400; // small block size + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + Reopen(options); + + Random rnd(301); + std::string v = rnd.RandomString(400); + + // Since v is the size of a block, each key should take a block + // of 400+ bytes. + ASSERT_OK(Put("1", v)); + ASSERT_OK(Put("3", v)); + ASSERT_OK(Put("5", v)); + ASSERT_OK(Put("7", v)); + ASSERT_OK(Flush()); + + ASSERT_EQ(0, bbto.block_cache->GetPinnedUsage()); + + // Verify that iterators don't pin more than one data block in block cache + // at each time. + { + std::unique_ptr iter(db_->NewIterator(ReadOptions())); + iter->SeekToFirst(); + + for (int i = 0; i < 4; i++) { + ASSERT_TRUE(iter->Valid()); + // Block cache should contain exactly one block. + ASSERT_GT(bbto.block_cache->GetPinnedUsage(), 0); + ASSERT_LT(bbto.block_cache->GetPinnedUsage(), 800); + iter->Next(); + } + ASSERT_FALSE(iter->Valid()); + + iter->Seek("4"); + ASSERT_TRUE(iter->Valid()); + + ASSERT_GT(bbto.block_cache->GetPinnedUsage(), 0); + ASSERT_LT(bbto.block_cache->GetPinnedUsage(), 800); + + iter->Seek("3"); + ASSERT_TRUE(iter->Valid()); + + ASSERT_OK(iter->status()); + + ASSERT_GT(bbto.block_cache->GetPinnedUsage(), 0); + ASSERT_LT(bbto.block_cache->GetPinnedUsage(), 800); + } + ASSERT_EQ(0, bbto.block_cache->GetPinnedUsage()); + + // Test compaction case + ASSERT_OK(Put("2", v)); + ASSERT_OK(Put("5", v)); + ASSERT_OK(Put("6", v)); + ASSERT_OK(Put("8", v)); + ASSERT_OK(Flush()); + + // Clear existing data in block cache + bbto.block_cache->SetCapacity(0); + bbto.block_cache->SetCapacity(100000); + + // Verify compaction input iterators don't hold more than one data blocks at + // one time. + std::atomic finished(false); + std::atomic block_newed(0); + std::atomic block_destroyed(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "Block::Block:0", [&](void* /*arg*/) { + if (finished) { + return; + } + // Two iterators. At most 2 outstanding blocks. + EXPECT_GE(block_newed.load(), block_destroyed.load()); + EXPECT_LE(block_newed.load(), block_destroyed.load() + 1); + block_newed.fetch_add(1); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "Block::~Block", [&](void* /*arg*/) { + if (finished) { + return; + } + // Two iterators. At most 2 outstanding blocks. + EXPECT_GE(block_newed.load(), block_destroyed.load() + 1); + EXPECT_LE(block_newed.load(), block_destroyed.load() + 2); + block_destroyed.fetch_add(1); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::Run:BeforeVerify", + [&](void* /*arg*/) { finished = true; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + // Two input files. Each of them has 4 data blocks. + ASSERT_EQ(8, block_newed.load()); + ASSERT_EQ(8, block_destroyed.load()); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBTest2, TestGetColumnFamilyHandleUnlocked) { + // Setup sync point dependency to reproduce the race condition of + // DBImpl::GetColumnFamilyHandleUnlocked + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"TestGetColumnFamilyHandleUnlocked::GetColumnFamilyHandleUnlocked1", + "TestGetColumnFamilyHandleUnlocked::PreGetColumnFamilyHandleUnlocked2"}, + {"TestGetColumnFamilyHandleUnlocked::GetColumnFamilyHandleUnlocked2", + "TestGetColumnFamilyHandleUnlocked::ReadColumnFamilyHandle1"}, + }); + SyncPoint::GetInstance()->EnableProcessing(); + + CreateColumnFamilies({"test1", "test2"}, Options()); + ASSERT_EQ(handles_.size(), 2); + + DBImpl* dbi = static_cast_with_check(db_); + port::Thread user_thread1([&]() { + auto cfh = dbi->GetColumnFamilyHandleUnlocked(handles_[0]->GetID()); + ASSERT_EQ(cfh->GetID(), handles_[0]->GetID()); + TEST_SYNC_POINT( + "TestGetColumnFamilyHandleUnlocked::GetColumnFamilyHandleUnlocked1"); + TEST_SYNC_POINT( + "TestGetColumnFamilyHandleUnlocked::ReadColumnFamilyHandle1"); + ASSERT_EQ(cfh->GetID(), handles_[0]->GetID()); + }); + + port::Thread user_thread2([&]() { + TEST_SYNC_POINT( + "TestGetColumnFamilyHandleUnlocked::PreGetColumnFamilyHandleUnlocked2"); + auto cfh = dbi->GetColumnFamilyHandleUnlocked(handles_[1]->GetID()); + ASSERT_EQ(cfh->GetID(), handles_[1]->GetID()); + TEST_SYNC_POINT( + "TestGetColumnFamilyHandleUnlocked::GetColumnFamilyHandleUnlocked2"); + ASSERT_EQ(cfh->GetID(), handles_[1]->GetID()); + }); + + user_thread1.join(); + user_thread2.join(); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(DBTest2, TestCompactFiles) { + // Setup sync point dependency to reproduce the race condition of + // DBImpl::GetColumnFamilyHandleUnlocked + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"TestCompactFiles::IngestExternalFile1", + "TestCompactFiles::IngestExternalFile2"}, + }); + SyncPoint::GetInstance()->EnableProcessing(); + + Options options; + options.env = env_; + options.num_levels = 2; + options.disable_auto_compactions = true; + Reopen(options); + auto* handle = db_->DefaultColumnFamily(); + ASSERT_EQ(db_->NumberLevels(handle), 2); + + ROCKSDB_NAMESPACE::SstFileWriter sst_file_writer{ + ROCKSDB_NAMESPACE::EnvOptions(), options}; + std::string external_file1 = dbname_ + "/test_compact_files1.sst_t"; + std::string external_file2 = dbname_ + "/test_compact_files2.sst_t"; + std::string external_file3 = dbname_ + "/test_compact_files3.sst_t"; + + ASSERT_OK(sst_file_writer.Open(external_file1)); + ASSERT_OK(sst_file_writer.Put("1", "1")); + ASSERT_OK(sst_file_writer.Put("2", "2")); + ASSERT_OK(sst_file_writer.Finish()); + + ASSERT_OK(sst_file_writer.Open(external_file2)); + ASSERT_OK(sst_file_writer.Put("3", "3")); + ASSERT_OK(sst_file_writer.Put("4", "4")); + ASSERT_OK(sst_file_writer.Finish()); + + ASSERT_OK(sst_file_writer.Open(external_file3)); + ASSERT_OK(sst_file_writer.Put("5", "5")); + ASSERT_OK(sst_file_writer.Put("6", "6")); + ASSERT_OK(sst_file_writer.Finish()); + + ASSERT_OK(db_->IngestExternalFile(handle, {external_file1, external_file3}, + IngestExternalFileOptions())); + ASSERT_EQ(NumTableFilesAtLevel(1, 0), 2); + std::vector files; + GetSstFiles(env_, dbname_, &files); + ASSERT_EQ(files.size(), 2); + + Status user_thread1_status; + port::Thread user_thread1([&]() { + user_thread1_status = + db_->CompactFiles(CompactionOptions(), handle, files, 1); + }); + + Status user_thread2_status; + port::Thread user_thread2([&]() { + user_thread2_status = db_->IngestExternalFile(handle, {external_file2}, + IngestExternalFileOptions()); + TEST_SYNC_POINT("TestCompactFiles::IngestExternalFile1"); + }); + + user_thread1.join(); + user_thread2.join(); + + ASSERT_OK(user_thread1_status); + ASSERT_OK(user_thread2_status); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(DBTest2, MultiDBParallelOpenTest) { + const int kNumDbs = 2; + Options options = CurrentOptions(); + std::vector dbnames; + for (int i = 0; i < kNumDbs; ++i) { + dbnames.emplace_back(test::PerThreadDBPath(env_, "db" + std::to_string(i))); + ASSERT_OK(DestroyDB(dbnames.back(), options)); + } + + // Verify empty DBs can be created in parallel + std::vector open_threads; + std::vector dbs{static_cast(kNumDbs), nullptr}; + options.create_if_missing = true; + for (int i = 0; i < kNumDbs; ++i) { + open_threads.emplace_back( + [&](int dbnum) { + ASSERT_OK(DB::Open(options, dbnames[dbnum], &dbs[dbnum])); + }, + i); + } + + // Now add some data and close, so next we can verify non-empty DBs can be + // recovered in parallel + for (int i = 0; i < kNumDbs; ++i) { + open_threads[i].join(); + ASSERT_OK(dbs[i]->Put(WriteOptions(), "xi", "gua")); + delete dbs[i]; + } + + // Verify non-empty DBs can be recovered in parallel + open_threads.clear(); + for (int i = 0; i < kNumDbs; ++i) { + open_threads.emplace_back( + [&](int dbnum) { + ASSERT_OK(DB::Open(options, dbnames[dbnum], &dbs[dbnum])); + }, + i); + } + + // Wait and cleanup + for (int i = 0; i < kNumDbs; ++i) { + open_threads[i].join(); + delete dbs[i]; + ASSERT_OK(DestroyDB(dbnames[i], options)); + } +} + +namespace { +class DummyOldStats : public Statistics { + public: + const char* Name() const override { return "DummyOldStats"; } + uint64_t getTickerCount(uint32_t /*ticker_type*/) const override { return 0; } + void recordTick(uint32_t /* ticker_type */, uint64_t /* count */) override { + num_rt++; + } + void setTickerCount(uint32_t /*ticker_type*/, uint64_t /*count*/) override {} + uint64_t getAndResetTickerCount(uint32_t /*ticker_type*/) override { + return 0; + } + void measureTime(uint32_t /*histogram_type*/, uint64_t /*count*/) override { + num_mt++; + } + void histogramData( + uint32_t /*histogram_type*/, + ROCKSDB_NAMESPACE::HistogramData* const /*data*/) const override {} + std::string getHistogramString(uint32_t /*type*/) const override { + return ""; + } + bool HistEnabledForType(uint32_t /*type*/) const override { return false; } + std::string ToString() const override { return ""; } + std::atomic num_rt{0}; + std::atomic num_mt{0}; +}; +} // anonymous namespace + +TEST_F(DBTest2, OldStatsInterface) { + DummyOldStats* dos = new DummyOldStats(); + std::shared_ptr stats(dos); + Options options = CurrentOptions(); + options.create_if_missing = true; + options.statistics = stats; + Reopen(options); + + ASSERT_OK(Put("foo", "bar")); + ASSERT_EQ("bar", Get("foo")); + ASSERT_OK(Flush()); + ASSERT_EQ("bar", Get("foo")); + + ASSERT_GT(dos->num_rt, 0); + ASSERT_GT(dos->num_mt, 0); +} + +TEST_F(DBTest2, CloseWithUnreleasedSnapshot) { + const Snapshot* ss = db_->GetSnapshot(); + + for (auto h : handles_) { + db_->DestroyColumnFamilyHandle(h); + } + handles_.clear(); + + ASSERT_NOK(db_->Close()); + db_->ReleaseSnapshot(ss); + ASSERT_OK(db_->Close()); + delete db_; + db_ = nullptr; +} + +TEST_F(DBTest2, PrefixBloomReseek) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.prefix_extractor.reset(NewCappedPrefixTransform(3)); + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(NewBloomFilterPolicy(10, false)); + bbto.whole_key_filtering = false; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + DestroyAndReopen(options); + + // Construct two L1 files with keys: + // f1:[aaa1 ccc1] f2:[ddd0] + ASSERT_OK(Put("aaa1", "")); + ASSERT_OK(Put("ccc1", "")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("ddd0", "")); + ASSERT_OK(Flush()); + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kSkip; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + ASSERT_OK(Put("bbb1", "")); + + Iterator* iter = db_->NewIterator(ReadOptions()); + ASSERT_OK(iter->status()); + + // Seeking into f1, the iterator will check bloom filter which returns the + // file iterator ot be invalidate, and the cursor will put into f2, with + // the next key to be "ddd0". + iter->Seek("bbb1"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("bbb1", iter->key().ToString()); + + // Reseek ccc1, the L1 iterator needs to go back to f1 and reseek. + iter->Seek("ccc1"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("ccc1", iter->key().ToString()); + + delete iter; +} + +TEST_F(DBTest2, PrefixBloomFilteredOut) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.prefix_extractor.reset(NewCappedPrefixTransform(3)); + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(NewBloomFilterPolicy(10, false)); + bbto.whole_key_filtering = false; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + DestroyAndReopen(options); + + // Construct two L1 files with keys: + // f1:[aaa1 ccc1] f2:[ddd0] + ASSERT_OK(Put("aaa1", "")); + ASSERT_OK(Put("ccc1", "")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("ddd0", "")); + ASSERT_OK(Flush()); + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kSkip; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + Iterator* iter = db_->NewIterator(ReadOptions()); + ASSERT_OK(iter->status()); + + // Bloom filter is filterd out by f1. + // This is just one of several valid position following the contract. + // Postioning to ccc1 or ddd0 is also valid. This is just to validate + // the behavior of the current implementation. If underlying implementation + // changes, the test might fail here. + iter->Seek("bbb1"); + ASSERT_OK(iter->status()); + ASSERT_FALSE(iter->Valid()); + + delete iter; +} + +TEST_F(DBTest2, RowCacheSnapshot) { + Options options = CurrentOptions(); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.row_cache = NewLRUCache(8 * 8192); + DestroyAndReopen(options); + + ASSERT_OK(Put("foo", "bar1")); + + const Snapshot* s1 = db_->GetSnapshot(); + + ASSERT_OK(Put("foo", "bar2")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("foo2", "bar")); + const Snapshot* s2 = db_->GetSnapshot(); + ASSERT_OK(Put("foo3", "bar")); + const Snapshot* s3 = db_->GetSnapshot(); + + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), 0); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), 0); + ASSERT_EQ(Get("foo"), "bar2"); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), 0); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), 1); + ASSERT_EQ(Get("foo"), "bar2"); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), 1); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), 1); + ASSERT_EQ(Get("foo", s1), "bar1"); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), 1); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), 2); + ASSERT_EQ(Get("foo", s2), "bar2"); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), 2); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), 2); + ASSERT_EQ(Get("foo", s1), "bar1"); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), 3); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), 2); + ASSERT_EQ(Get("foo", s3), "bar2"); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), 4); + ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), 2); + + db_->ReleaseSnapshot(s1); + db_->ReleaseSnapshot(s2); + db_->ReleaseSnapshot(s3); +} + +// When DB is reopened with multiple column families, the manifest file +// is written after the first CF is flushed, and it is written again +// after each flush. If DB crashes between the flushes, the flushed CF +// flushed will pass the latest log file, and now we require it not +// to be corrupted, and triggering a corruption report. +// We need to fix the bug and enable the test. +TEST_F(DBTest2, CrashInRecoveryMultipleCF) { + const std::vector sync_points = { + "DBImpl::RecoverLogFiles:BeforeFlushFinalMemtable", + "VersionSet::ProcessManifestWrites:BeforeWriteLastVersionEdit:0"}; + for (const auto& test_sync_point : sync_points) { + Options options = CurrentOptions(); + // First destroy original db to ensure a clean start. + DestroyAndReopen(options); + options.create_if_missing = true; + options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery; + CreateAndReopenWithCF({"pikachu"}, options); + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Flush()); + ASSERT_OK(Put(1, "foo", "bar")); + ASSERT_OK(Flush(1)); + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Put(1, "foo", "bar")); + // The value is large enough to be divided to two blocks. + std::string large_value(400, ' '); + ASSERT_OK(Put("foo1", large_value)); + ASSERT_OK(Put("foo2", large_value)); + Close(); + + // Corrupt the log file in the middle, so that it is not corrupted + // in the tail. + std::vector filenames; + ASSERT_OK(env_->GetChildren(dbname_, &filenames)); + for (const auto& f : filenames) { + uint64_t number; + FileType type; + if (ParseFileName(f, &number, &type) && type == FileType::kWalFile) { + std::string fname = dbname_ + "/" + f; + std::string file_content; + ASSERT_OK(ReadFileToString(env_, fname, &file_content)); + file_content[400] = 'h'; + file_content[401] = 'a'; + ASSERT_OK(WriteStringToFile(env_, file_content, fname)); + break; + } + } + + // Reopen and freeze the file system after the first manifest write. + FaultInjectionTestEnv fit_env(options.env); + options.env = &fit_env; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + test_sync_point, + [&](void* /*arg*/) { fit_env.SetFilesystemActive(false); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_NOK(TryReopenWithColumnFamilies( + {kDefaultColumnFamilyName, "pikachu"}, options)); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + fit_env.SetFilesystemActive(true); + // If we continue using failure ingestion Env, it will conplain something + // when renaming current file, which is not expected. Need to investigate + // why. + options.env = env_; + ASSERT_OK(TryReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu"}, + options)); + } +} + +TEST_F(DBTest2, SeekFileRangeDeleteTail) { + Options options = CurrentOptions(); + options.prefix_extractor.reset(NewCappedPrefixTransform(1)); + options.num_levels = 3; + DestroyAndReopen(options); + + ASSERT_OK(Put("a", "a")); + const Snapshot* s1 = db_->GetSnapshot(); + ASSERT_OK( + db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "f")); + ASSERT_OK(Put("b", "a")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("x", "a")); + ASSERT_OK(Put("z", "a")); + ASSERT_OK(Flush()); + + CompactRangeOptions cro; + cro.change_level = true; + cro.target_level = 2; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + { + ReadOptions ro; + ro.total_order_seek = true; + std::unique_ptr iter(db_->NewIterator(ro)); + ASSERT_OK(iter->status()); + iter->Seek("e"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("x", iter->key().ToString()); + } + db_->ReleaseSnapshot(s1); +} + +TEST_F(DBTest2, BackgroundPurgeTest) { + Options options = CurrentOptions(); + options.write_buffer_manager = + std::make_shared(1 << 20); + options.avoid_unnecessary_blocking_io = true; + DestroyAndReopen(options); + size_t base_value = options.write_buffer_manager->memory_usage(); + + ASSERT_OK(Put("a", "a")); + Iterator* iter = db_->NewIterator(ReadOptions()); + ASSERT_OK(iter->status()); + ASSERT_OK(Flush()); + size_t value = options.write_buffer_manager->memory_usage(); + ASSERT_GT(value, base_value); + + db_->GetEnv()->SetBackgroundThreads(1, Env::Priority::HIGH); + test::SleepingBackgroundTask sleeping_task_after; + db_->GetEnv()->Schedule(&test::SleepingBackgroundTask::DoSleepTask, + &sleeping_task_after, Env::Priority::HIGH); + delete iter; + + Env::Default()->SleepForMicroseconds(100000); + value = options.write_buffer_manager->memory_usage(); + ASSERT_GT(value, base_value); + + sleeping_task_after.WakeUp(); + sleeping_task_after.WaitUntilDone(); + + test::SleepingBackgroundTask sleeping_task_after2; + db_->GetEnv()->Schedule(&test::SleepingBackgroundTask::DoSleepTask, + &sleeping_task_after2, Env::Priority::HIGH); + sleeping_task_after2.WakeUp(); + sleeping_task_after2.WaitUntilDone(); + + value = options.write_buffer_manager->memory_usage(); + ASSERT_EQ(base_value, value); +} + +TEST_F(DBTest2, SwitchMemtableRaceWithNewManifest) { + Options options = CurrentOptions(); + DestroyAndReopen(options); + options.max_manifest_file_size = 10; + options.create_if_missing = true; + CreateAndReopenWithCF({"pikachu"}, options); + ASSERT_EQ(2, handles_.size()); + + ASSERT_OK(Put("foo", "value")); + const int kL0Files = options.level0_file_num_compaction_trigger; + for (int i = 0; i < kL0Files; ++i) { + ASSERT_OK(Put(/*cf=*/1, "a", std::to_string(i))); + ASSERT_OK(Flush(/*cf=*/1)); + } + + port::Thread thread([&]() { ASSERT_OK(Flush()); }); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + thread.join(); +} + +TEST_F(DBTest2, SameSmallestInSameLevel) { + // This test validates fractional casacading logic when several files at one + // one level only contains the same user key. + Options options = CurrentOptions(); + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + DestroyAndReopen(options); + + ASSERT_OK(Put("key", "1")); + ASSERT_OK(Put("key", "2")); + ASSERT_OK(db_->Merge(WriteOptions(), "key", "3")); + ASSERT_OK(db_->Merge(WriteOptions(), "key", "4")); + ASSERT_OK(Flush()); + CompactRangeOptions cro; + cro.change_level = true; + cro.target_level = 2; + ASSERT_OK(dbfull()->CompactRange(cro, db_->DefaultColumnFamily(), nullptr, + nullptr)); + + ASSERT_OK(db_->Merge(WriteOptions(), "key", "5")); + ASSERT_OK(Flush()); + ASSERT_OK(db_->Merge(WriteOptions(), "key", "6")); + ASSERT_OK(Flush()); + ASSERT_OK(db_->Merge(WriteOptions(), "key", "7")); + ASSERT_OK(Flush()); + ASSERT_OK(db_->Merge(WriteOptions(), "key", "8")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("0,4,1", FilesPerLevel()); + + ASSERT_EQ("2,3,4,5,6,7,8", Get("key")); +} + +TEST_F(DBTest2, FileConsistencyCheckInOpen) { + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Flush()); + + SyncPoint::GetInstance()->SetCallBack( + "VersionBuilder::CheckConsistencyBeforeReturn", [&](void* arg) { + Status* ret_s = static_cast(arg); + *ret_s = Status::Corruption("fcc"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + Options options = CurrentOptions(); + options.force_consistency_checks = true; + ASSERT_NOK(TryReopen(options)); + + SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBTest2, BlockBasedTablePrefixIndexSeekForPrev) { + // create a DB with block prefix index + BlockBasedTableOptions table_options; + Options options = CurrentOptions(); + table_options.block_size = 300; + table_options.index_type = BlockBasedTableOptions::kHashSearch; + table_options.index_shortening = + BlockBasedTableOptions::IndexShorteningMode::kNoShortening; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.prefix_extractor.reset(NewFixedPrefixTransform(1)); + + Reopen(options); + + Random rnd(301); + std::string large_value = rnd.RandomString(500); + + ASSERT_OK(Put("a1", large_value)); + ASSERT_OK(Put("x1", large_value)); + ASSERT_OK(Put("y1", large_value)); + ASSERT_OK(Flush()); + + { + std::unique_ptr iterator(db_->NewIterator(ReadOptions())); + ASSERT_OK(iterator->status()); + iterator->SeekForPrev("x3"); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("x1", iterator->key().ToString()); + + iterator->SeekForPrev("a3"); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("a1", iterator->key().ToString()); + + iterator->SeekForPrev("y3"); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("y1", iterator->key().ToString()); + + // Query more than one non-existing prefix to cover the case both + // of empty hash bucket and hash bucket conflict. + iterator->SeekForPrev("b1"); + // Result should be not valid or "a1". + if (iterator->Valid()) { + ASSERT_EQ("a1", iterator->key().ToString()); + } + + iterator->SeekForPrev("c1"); + // Result should be not valid or "a1". + if (iterator->Valid()) { + ASSERT_EQ("a1", iterator->key().ToString()); + } + + iterator->SeekForPrev("d1"); + // Result should be not valid or "a1". + if (iterator->Valid()) { + ASSERT_EQ("a1", iterator->key().ToString()); + } + + iterator->SeekForPrev("y3"); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("y1", iterator->key().ToString()); + } +} + +TEST_F(DBTest2, PartitionedIndexPrefetchFailure) { + Options options = last_options_; + options.env = env_; + options.max_open_files = 20; + BlockBasedTableOptions bbto; + bbto.index_type = BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + bbto.metadata_block_size = 128; + bbto.block_size = 128; + bbto.block_cache = NewLRUCache(16777216); + bbto.cache_index_and_filter_blocks = true; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + DestroyAndReopen(options); + + // Force no table cache so every read will preload the SST file. + dbfull()->TEST_table_cache()->SetCapacity(0); + bbto.block_cache->SetCapacity(0); + + Random rnd(301); + for (int i = 0; i < 4096; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(32))); + } + ASSERT_OK(Flush()); + + // Try different random failures in table open for 300 times. + for (int i = 0; i < 300; i++) { + env_->num_reads_fails_ = 0; + env_->rand_reads_fail_odd_ = 8; + + std::string value; + Status s = dbfull()->Get(ReadOptions(), Key(1), &value); + if (env_->num_reads_fails_ > 0) { + ASSERT_NOK(s); + } else { + ASSERT_OK(s); + } + } + + env_->rand_reads_fail_odd_ = 0; +} + +TEST_F(DBTest2, ChangePrefixExtractor) { + for (bool use_partitioned_filter : {true, false}) { + // create a DB with block prefix index + BlockBasedTableOptions table_options; + Options options = CurrentOptions(); + + // Sometimes filter is checked based on upper bound. Assert counters + // for that case. Otherwise, only check data correctness. + bool expect_filter_check = !use_partitioned_filter; + table_options.partition_filters = use_partitioned_filter; + if (use_partitioned_filter) { + table_options.index_type = + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + } + table_options.filter_policy.reset(NewBloomFilterPolicy(10, false)); + + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.statistics = CreateDBStatistics(); + + options.prefix_extractor.reset(NewFixedPrefixTransform(2)); + DestroyAndReopen(options); + + Random rnd(301); + + ASSERT_OK(Put("aa", "")); + ASSERT_OK(Put("xb", "")); + ASSERT_OK(Put("xx1", "")); + ASSERT_OK(Put("xz1", "")); + ASSERT_OK(Put("zz", "")); + ASSERT_OK(Flush()); + + // After reopening DB with prefix size 2 => 1, prefix extractor + // won't take effective unless it won't change results based + // on upper bound and seek key. + options.prefix_extractor.reset(NewFixedPrefixTransform(1)); + Reopen(options); + + { + std::unique_ptr iterator(db_->NewIterator(ReadOptions())); + ASSERT_OK(iterator->status()); + iterator->Seek("xa"); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("xb", iterator->key().ToString()); + if (expect_filter_check) { + EXPECT_EQ(0, PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH)); + } + + iterator->Seek("xz"); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("xz1", iterator->key().ToString()); + if (expect_filter_check) { + EXPECT_EQ(0, PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH)); + } + } + + std::string ub_str = "xg9"; + Slice ub(ub_str); + ReadOptions ro; + ro.iterate_upper_bound = &ub; + + { + std::unique_ptr iterator(db_->NewIterator(ro)); + ASSERT_OK(iterator->status()); + + // SeekForPrev() never uses prefix bloom if it is changed. + iterator->SeekForPrev("xg0"); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("xb", iterator->key().ToString()); + if (expect_filter_check) { + EXPECT_EQ(0, PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH)); + } + } + + ub_str = "xx9"; + ub = Slice(ub_str); + { + std::unique_ptr iterator(db_->NewIterator(ro)); + ASSERT_OK(iterator->status()); + + iterator->Seek("x"); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("xb", iterator->key().ToString()); + if (expect_filter_check) { + EXPECT_EQ(0, PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH)); + } + + iterator->Seek("xx0"); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("xx1", iterator->key().ToString()); + if (expect_filter_check) { + EXPECT_EQ(1, PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH)); + } + } + + CompactRangeOptions compact_range_opts; + compact_range_opts.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(compact_range_opts, nullptr, nullptr)); + ASSERT_OK(db_->CompactRange(compact_range_opts, nullptr, nullptr)); + + // Re-execute similar queries after a full compaction + { + std::unique_ptr iterator(db_->NewIterator(ReadOptions())); + + iterator->Seek("x"); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("xb", iterator->key().ToString()); + if (expect_filter_check) { + EXPECT_EQ(1, PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH)); + } + + iterator->Seek("xg"); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("xx1", iterator->key().ToString()); + if (expect_filter_check) { + EXPECT_EQ(1, PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH)); + } + + iterator->Seek("xz"); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("xz1", iterator->key().ToString()); + if (expect_filter_check) { + EXPECT_EQ(1, PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH)); + } + + ASSERT_OK(iterator->status()); + } + { + std::unique_ptr iterator(db_->NewIterator(ro)); + + iterator->SeekForPrev("xx0"); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("xb", iterator->key().ToString()); + if (expect_filter_check) { + EXPECT_EQ(1, PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH)); + } + + iterator->Seek("xx0"); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("xx1", iterator->key().ToString()); + if (expect_filter_check) { + EXPECT_EQ(1, PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH)); + } + + ASSERT_OK(iterator->status()); + } + + ub_str = "xg9"; + ub = Slice(ub_str); + { + std::unique_ptr iterator(db_->NewIterator(ro)); + iterator->SeekForPrev("xg0"); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("xb", iterator->key().ToString()); + if (expect_filter_check) { + EXPECT_EQ(1, PopTicker(options, NON_LAST_LEVEL_SEEK_FILTER_MATCH)); + } + ASSERT_OK(iterator->status()); + } + } +} + +TEST_F(DBTest2, BlockBasedTablePrefixGetIndexNotFound) { + // create a DB with block prefix index + BlockBasedTableOptions table_options; + Options options = CurrentOptions(); + table_options.block_size = 300; + table_options.index_type = BlockBasedTableOptions::kHashSearch; + table_options.index_shortening = + BlockBasedTableOptions::IndexShorteningMode::kNoShortening; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.prefix_extractor.reset(NewFixedPrefixTransform(1)); + options.level0_file_num_compaction_trigger = 8; + + Reopen(options); + + ASSERT_OK(Put("b1", "ok")); + ASSERT_OK(Flush()); + + // Flushing several files so that the chance that hash bucket + // is empty fo "b" in at least one of the files is high. + ASSERT_OK(Put("a1", "")); + ASSERT_OK(Put("c1", "")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("a2", "")); + ASSERT_OK(Put("c2", "")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("a3", "")); + ASSERT_OK(Put("c3", "")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("a4", "")); + ASSERT_OK(Put("c4", "")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("a5", "")); + ASSERT_OK(Put("c5", "")); + ASSERT_OK(Flush()); + + ASSERT_EQ("ok", Get("b1")); +} + +TEST_F(DBTest2, AutoPrefixMode1) { + do { + // create a DB with block prefix index + Options options = CurrentOptions(); + BlockBasedTableOptions table_options = + *options.table_factory->GetOptions(); + table_options.filter_policy.reset(NewBloomFilterPolicy(10, false)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.prefix_extractor.reset(NewFixedPrefixTransform(1)); + options.statistics = CreateDBStatistics(); + + Reopen(options); + + Random rnd(301); + std::string large_value = rnd.RandomString(500); + + ASSERT_OK(Put("a1", large_value)); + ASSERT_OK(Put("x1", large_value)); + ASSERT_OK(Put("y1", large_value)); + ASSERT_OK(Flush()); + + ReadOptions ro; + ro.total_order_seek = false; + ro.auto_prefix_mode = true; + + const auto hit_stat = options.num_levels == 1 + ? LAST_LEVEL_SEEK_FILTER_MATCH + : NON_LAST_LEVEL_SEEK_FILTER_MATCH; + const auto miss_stat = options.num_levels == 1 + ? LAST_LEVEL_SEEK_FILTERED + : NON_LAST_LEVEL_SEEK_FILTERED; + { + std::unique_ptr iterator(db_->NewIterator(ro)); + iterator->Seek("b1"); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("x1", iterator->key().ToString()); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, hit_stat)); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, miss_stat)); + ASSERT_OK(iterator->status()); + } + + Slice ub; + ro.iterate_upper_bound = &ub; + + ub = "b9"; + { + std::unique_ptr iterator(db_->NewIterator(ro)); + iterator->Seek("b1"); + ASSERT_FALSE(iterator->Valid()); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, hit_stat)); + EXPECT_EQ(1, TestGetAndResetTickerCount(options, miss_stat)); + ASSERT_OK(iterator->status()); + } + + ub = "z"; + { + std::unique_ptr iterator(db_->NewIterator(ro)); + iterator->Seek("b1"); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("x1", iterator->key().ToString()); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, hit_stat)); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, miss_stat)); + ASSERT_OK(iterator->status()); + } + + ub = "c"; + { + std::unique_ptr iterator(db_->NewIterator(ro)); + iterator->Seek("b1"); + ASSERT_FALSE(iterator->Valid()); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, hit_stat)); + EXPECT_EQ(1, TestGetAndResetTickerCount(options, miss_stat)); + ASSERT_OK(iterator->status()); + } + + ub = "c1"; + { + std::unique_ptr iterator(db_->NewIterator(ro)); + iterator->Seek("b1"); + ASSERT_FALSE(iterator->Valid()); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, hit_stat)); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, miss_stat)); + ASSERT_OK(iterator->status()); + } + + // The same queries without recreating iterator + { + std::unique_ptr iterator(db_->NewIterator(ro)); + + ub = "b9"; + iterator->Seek("b1"); + ASSERT_FALSE(iterator->Valid()); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, hit_stat)); + EXPECT_EQ(1, TestGetAndResetTickerCount(options, miss_stat)); + ASSERT_OK(iterator->status()); + + ub = "z"; + iterator->Seek("b1"); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("x1", iterator->key().ToString()); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, hit_stat)); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, miss_stat)); + + ub = "c"; + iterator->Seek("b1"); + ASSERT_FALSE(iterator->Valid()); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, hit_stat)); + EXPECT_EQ(1, TestGetAndResetTickerCount(options, miss_stat)); + + ub = "b9"; + iterator->SeekForPrev("b1"); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("a1", iterator->key().ToString()); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, hit_stat)); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, miss_stat)); + + ub = "zz"; + iterator->SeekToLast(); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("y1", iterator->key().ToString()); + + iterator->SeekToFirst(); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("a1", iterator->key().ToString()); + } + + // Similar, now with reverse comparator + // Technically, we are violating axiom 2 of prefix_extractors, but + // it should be revised because of major use-cases using + // ReverseBytewiseComparator with capped/fixed prefix Seek. (FIXME) + options.comparator = ReverseBytewiseComparator(); + options.prefix_extractor.reset(NewFixedPrefixTransform(1)); + + DestroyAndReopen(options); + + ASSERT_OK(Put("a1", large_value)); + ASSERT_OK(Put("x1", large_value)); + ASSERT_OK(Put("y1", large_value)); + ASSERT_OK(Flush()); + + { + std::unique_ptr iterator(db_->NewIterator(ro)); + + ub = "b1"; + iterator->Seek("b9"); + ASSERT_FALSE(iterator->Valid()); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, hit_stat)); + EXPECT_EQ(1, TestGetAndResetTickerCount(options, miss_stat)); + ASSERT_OK(iterator->status()); + + ub = "b1"; + iterator->Seek("z"); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("y1", iterator->key().ToString()); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, hit_stat)); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, miss_stat)); + + ub = "b1"; + iterator->Seek("c"); + ASSERT_FALSE(iterator->Valid()); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, hit_stat)); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, miss_stat)); + + ub = "b"; + iterator->Seek("c9"); + ASSERT_FALSE(iterator->Valid()); + // Fails if ReverseBytewiseComparator::IsSameLengthImmediateSuccessor + // is "correctly" implemented. + EXPECT_EQ(0, TestGetAndResetTickerCount(options, hit_stat)); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, miss_stat)); + + ub = "a"; + iterator->Seek("b9"); + // Fails if ReverseBytewiseComparator::IsSameLengthImmediateSuccessor + // is "correctly" implemented. + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("a1", iterator->key().ToString()); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, hit_stat)); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, miss_stat)); + + ub = "b"; + iterator->Seek("a"); + ASSERT_FALSE(iterator->Valid()); + // Fails if ReverseBytewiseComparator::IsSameLengthImmediateSuccessor + // matches BytewiseComparator::IsSameLengthImmediateSuccessor. Upper + // comparing before seek key prevents a real bug from surfacing. + EXPECT_EQ(0, TestGetAndResetTickerCount(options, hit_stat)); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, miss_stat)); + + ub = "b1"; + iterator->SeekForPrev("b9"); + ASSERT_TRUE(iterator->Valid()); + // Fails if ReverseBytewiseComparator::IsSameLengthImmediateSuccessor + // is "correctly" implemented. + ASSERT_EQ("x1", iterator->key().ToString()); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, hit_stat)); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, miss_stat)); + + ub = "a"; + iterator->SeekToLast(); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("a1", iterator->key().ToString()); + + iterator->SeekToFirst(); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("y1", iterator->key().ToString()); + } + + // Now something a bit different, related to "short" keys that + // auto_prefix_mode can omit. See "BUG" section of auto_prefix_mode. + options.comparator = BytewiseComparator(); + for (const auto config : {"fixed:2", "capped:2"}) { + ASSERT_OK(SliceTransform::CreateFromString(ConfigOptions(), config, + &options.prefix_extractor)); + + // FIXME: kHashSearch, etc. requires all keys be InDomain + if (StartsWith(config, "fixed") && + (table_options.index_type == BlockBasedTableOptions::kHashSearch || + StartsWith(options.memtable_factory->Name(), "Hash"))) { + continue; + } + DestroyAndReopen(options); + + const char* a_end_stuff = "a\xffXYZ"; + const char* b_begin_stuff = "b\x00XYZ"; + ASSERT_OK(Put("a", large_value)); + ASSERT_OK(Put("b", large_value)); + ASSERT_OK(Put(Slice(b_begin_stuff, 3), large_value)); + ASSERT_OK(Put("c", large_value)); + ASSERT_OK(Flush()); + + // control showing valid optimization with auto_prefix mode + ub = Slice(a_end_stuff, 4); + ro.iterate_upper_bound = &ub; + + std::unique_ptr iterator(db_->NewIterator(ro)); + iterator->Seek(Slice(a_end_stuff, 2)); + ASSERT_FALSE(iterator->Valid()); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, hit_stat)); + EXPECT_EQ(1, TestGetAndResetTickerCount(options, miss_stat)); + ASSERT_OK(iterator->status()); + + // test, cannot be validly optimized with auto_prefix_mode + ub = Slice(b_begin_stuff, 2); + ro.iterate_upper_bound = &ub; + + iterator->Seek(Slice(a_end_stuff, 2)); + // !!! BUG !!! See "BUG" section of auto_prefix_mode. + ASSERT_FALSE(iterator->Valid()); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, hit_stat)); + EXPECT_EQ(1, TestGetAndResetTickerCount(options, miss_stat)); + ASSERT_OK(iterator->status()); + + // To prove that is the wrong result, now use total order seek + ReadOptions tos_ro = ro; + tos_ro.total_order_seek = true; + tos_ro.auto_prefix_mode = false; + iterator.reset(db_->NewIterator(tos_ro)); + iterator->Seek(Slice(a_end_stuff, 2)); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ("b", iterator->key().ToString()); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, hit_stat)); + EXPECT_EQ(0, TestGetAndResetTickerCount(options, miss_stat)); + ASSERT_OK(iterator->status()); + } + } while (ChangeOptions(kSkipPlainTable)); +} + +class RenameCurrentTest : public DBTestBase, + public testing::WithParamInterface { + public: + RenameCurrentTest() + : DBTestBase("rename_current_test", /*env_do_fsync=*/true), + sync_point_(GetParam()) {} + + ~RenameCurrentTest() override {} + + void SetUp() override { + env_->no_file_overwrite_.store(true, std::memory_order_release); + } + + void TearDown() override { + env_->no_file_overwrite_.store(false, std::memory_order_release); + } + + void SetupSyncPoints() { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->SetCallBack(sync_point_, [&](void* arg) { + Status* s = reinterpret_cast(arg); + assert(s); + *s = Status::IOError("Injected IO error."); + }); + } + + const std::string sync_point_; +}; + +INSTANTIATE_TEST_CASE_P(DistributedFS, RenameCurrentTest, + ::testing::Values("SetCurrentFile:BeforeRename", + "SetCurrentFile:AfterRename")); + +TEST_P(RenameCurrentTest, Open) { + Destroy(last_options_); + Options options = GetDefaultOptions(); + options.create_if_missing = true; + SetupSyncPoints(); + SyncPoint::GetInstance()->EnableProcessing(); + Status s = TryReopen(options); + ASSERT_NOK(s); + + SyncPoint::GetInstance()->DisableProcessing(); + Reopen(options); +} + +TEST_P(RenameCurrentTest, Flush) { + Destroy(last_options_); + Options options = GetDefaultOptions(); + options.max_manifest_file_size = 1; + options.create_if_missing = true; + Reopen(options); + ASSERT_OK(Put("key", "value")); + SetupSyncPoints(); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_NOK(Flush()); + + ASSERT_NOK(Put("foo", "value")); + + SyncPoint::GetInstance()->DisableProcessing(); + Reopen(options); + ASSERT_EQ("value", Get("key")); + ASSERT_EQ("NOT_FOUND", Get("foo")); +} + +TEST_P(RenameCurrentTest, Compaction) { + Destroy(last_options_); + Options options = GetDefaultOptions(); + options.max_manifest_file_size = 1; + options.create_if_missing = true; + Reopen(options); + ASSERT_OK(Put("a", "a_value")); + ASSERT_OK(Put("c", "c_value")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("b", "b_value")); + ASSERT_OK(Put("d", "d_value")); + ASSERT_OK(Flush()); + + SetupSyncPoints(); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_NOK(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr, + /*end=*/nullptr)); + + ASSERT_NOK(Put("foo", "value")); + + SyncPoint::GetInstance()->DisableProcessing(); + Reopen(options); + ASSERT_EQ("NOT_FOUND", Get("foo")); + ASSERT_EQ("d_value", Get("d")); +} + +TEST_F(DBTest2, LastLevelTemperature) { + class TestListener : public EventListener { + public: + void OnFileReadFinish(const FileOperationInfo& info) override { + UpdateFileTemperature(info); + } + + void OnFileWriteFinish(const FileOperationInfo& info) override { + UpdateFileTemperature(info); + } + + void OnFileFlushFinish(const FileOperationInfo& info) override { + UpdateFileTemperature(info); + } + + void OnFileSyncFinish(const FileOperationInfo& info) override { + UpdateFileTemperature(info); + } + + void OnFileCloseFinish(const FileOperationInfo& info) override { + UpdateFileTemperature(info); + } + + bool ShouldBeNotifiedOnFileIO() override { return true; } + + std::unordered_map file_temperatures; + + private: + void UpdateFileTemperature(const FileOperationInfo& info) { + auto filename = GetFileName(info.path); + uint64_t number; + FileType type; + ASSERT_TRUE(ParseFileName(filename, &number, &type)); + if (type == kTableFile) { + MutexLock l(&mutex_); + auto ret = file_temperatures.insert({number, info.temperature}); + if (!ret.second) { + // the same file temperature should always be the same for all events + ASSERT_TRUE(ret.first->second == info.temperature); + } + } + } + + std::string GetFileName(const std::string& fname) { + auto filename = fname.substr(fname.find_last_of(kFilePathSeparator) + 1); + // workaround only for Windows that the file path could contain both + // Windows FilePathSeparator and '/' + filename = filename.substr(filename.find_last_of('/') + 1); + return filename; + } + + port::Mutex mutex_; + }; + + const int kNumLevels = 7; + const int kLastLevel = kNumLevels - 1; + + auto* listener = new TestListener(); + + Options options = CurrentOptions(); + options.bottommost_temperature = Temperature::kWarm; + options.level0_file_num_compaction_trigger = 2; + options.level_compaction_dynamic_level_bytes = true; + options.num_levels = kNumLevels; + options.statistics = CreateDBStatistics(); + options.listeners.emplace_back(listener); + Reopen(options); + + auto size = GetSstSizeHelper(Temperature::kUnknown); + ASSERT_EQ(size, 0); + size = GetSstSizeHelper(Temperature::kWarm); + ASSERT_EQ(size, 0); + size = GetSstSizeHelper(Temperature::kHot); + ASSERT_EQ(size, 0); + + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Put("bar", "bar")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Put("bar", "bar")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + get_iostats_context()->Reset(); + IOStatsContext* iostats = get_iostats_context(); + + ColumnFamilyMetaData metadata; + db_->GetColumnFamilyMetaData(&metadata); + ASSERT_EQ(1, metadata.file_count); + SstFileMetaData meta = metadata.levels[kLastLevel].files[0]; + ASSERT_EQ(Temperature::kWarm, meta.temperature); + uint64_t number; + FileType type; + ASSERT_TRUE(ParseFileName(meta.name, &number, &type)); + ASSERT_EQ(listener->file_temperatures.at(number), meta.temperature); + + size = GetSstSizeHelper(Temperature::kUnknown); + ASSERT_EQ(size, 0); + size = GetSstSizeHelper(Temperature::kWarm); + ASSERT_GT(size, 0); + ASSERT_EQ(iostats->file_io_stats_by_temperature.hot_file_read_count, 0); + ASSERT_EQ(iostats->file_io_stats_by_temperature.warm_file_read_count, 0); + ASSERT_EQ(iostats->file_io_stats_by_temperature.hot_file_read_count, 0); + ASSERT_EQ(options.statistics->getTickerCount(HOT_FILE_READ_BYTES), 0); + ASSERT_GT(options.statistics->getTickerCount(WARM_FILE_READ_BYTES), 0); + ASSERT_EQ(options.statistics->getTickerCount(COLD_FILE_READ_BYTES), 0); + + ASSERT_EQ("bar", Get("foo")); + + ASSERT_EQ(iostats->file_io_stats_by_temperature.hot_file_read_count, 0); + ASSERT_EQ(iostats->file_io_stats_by_temperature.warm_file_read_count, 1); + ASSERT_EQ(iostats->file_io_stats_by_temperature.hot_file_read_count, 0); + ASSERT_EQ(iostats->file_io_stats_by_temperature.hot_file_bytes_read, 0); + ASSERT_GT(iostats->file_io_stats_by_temperature.warm_file_bytes_read, 0); + ASSERT_EQ(iostats->file_io_stats_by_temperature.cold_file_bytes_read, 0); + ASSERT_EQ(options.statistics->getTickerCount(HOT_FILE_READ_BYTES), 0); + ASSERT_GT(options.statistics->getTickerCount(WARM_FILE_READ_BYTES), 0); + ASSERT_EQ(options.statistics->getTickerCount(COLD_FILE_READ_BYTES), 0); + ASSERT_EQ(options.statistics->getTickerCount(HOT_FILE_READ_COUNT), 0); + ASSERT_GT(options.statistics->getTickerCount(WARM_FILE_READ_COUNT), 0); + ASSERT_EQ(options.statistics->getTickerCount(COLD_FILE_READ_COUNT), 0); + + // non-bottommost file still has unknown temperature + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Put("bar", "bar")); + ASSERT_OK(Flush()); + ASSERT_EQ("bar", Get("bar")); + ASSERT_EQ(iostats->file_io_stats_by_temperature.hot_file_read_count, 0); + ASSERT_EQ(iostats->file_io_stats_by_temperature.warm_file_read_count, 1); + ASSERT_EQ(iostats->file_io_stats_by_temperature.hot_file_read_count, 0); + ASSERT_EQ(iostats->file_io_stats_by_temperature.hot_file_bytes_read, 0); + ASSERT_GT(iostats->file_io_stats_by_temperature.warm_file_bytes_read, 0); + ASSERT_EQ(iostats->file_io_stats_by_temperature.cold_file_bytes_read, 0); + ASSERT_EQ(options.statistics->getTickerCount(HOT_FILE_READ_BYTES), 0); + ASSERT_GT(options.statistics->getTickerCount(WARM_FILE_READ_BYTES), 0); + ASSERT_EQ(options.statistics->getTickerCount(COLD_FILE_READ_BYTES), 0); + ASSERT_EQ(options.statistics->getTickerCount(HOT_FILE_READ_COUNT), 0); + ASSERT_GT(options.statistics->getTickerCount(WARM_FILE_READ_COUNT), 0); + ASSERT_EQ(options.statistics->getTickerCount(COLD_FILE_READ_COUNT), 0); + + db_->GetColumnFamilyMetaData(&metadata); + ASSERT_EQ(2, metadata.file_count); + meta = metadata.levels[0].files[0]; + ASSERT_EQ(Temperature::kUnknown, meta.temperature); + ASSERT_TRUE(ParseFileName(meta.name, &number, &type)); + ASSERT_EQ(listener->file_temperatures.at(number), meta.temperature); + + meta = metadata.levels[kLastLevel].files[0]; + ASSERT_EQ(Temperature::kWarm, meta.temperature); + ASSERT_TRUE(ParseFileName(meta.name, &number, &type)); + ASSERT_EQ(listener->file_temperatures.at(number), meta.temperature); + + size = GetSstSizeHelper(Temperature::kUnknown); + ASSERT_GT(size, 0); + size = GetSstSizeHelper(Temperature::kWarm); + ASSERT_GT(size, 0); + + // reopen and check the information is persisted + Reopen(options); + db_->GetColumnFamilyMetaData(&metadata); + ASSERT_EQ(2, metadata.file_count); + meta = metadata.levels[0].files[0]; + ASSERT_EQ(Temperature::kUnknown, meta.temperature); + ASSERT_TRUE(ParseFileName(meta.name, &number, &type)); + ASSERT_EQ(listener->file_temperatures.at(number), meta.temperature); + + meta = metadata.levels[kLastLevel].files[0]; + ASSERT_EQ(Temperature::kWarm, meta.temperature); + ASSERT_TRUE(ParseFileName(meta.name, &number, &type)); + ASSERT_EQ(listener->file_temperatures.at(number), meta.temperature); + size = GetSstSizeHelper(Temperature::kUnknown); + ASSERT_GT(size, 0); + size = GetSstSizeHelper(Temperature::kWarm); + ASSERT_GT(size, 0); + + // check other non-exist temperatures + size = GetSstSizeHelper(Temperature::kHot); + ASSERT_EQ(size, 0); + size = GetSstSizeHelper(Temperature::kCold); + ASSERT_EQ(size, 0); + std::string prop; + ASSERT_TRUE(dbfull()->GetProperty( + DB::Properties::kLiveSstFilesSizeAtTemperature + std::to_string(22), + &prop)); + ASSERT_EQ(std::atoi(prop.c_str()), 0); + + Reopen(options); + db_->GetColumnFamilyMetaData(&metadata); + ASSERT_EQ(2, metadata.file_count); + meta = metadata.levels[0].files[0]; + ASSERT_EQ(Temperature::kUnknown, meta.temperature); + ASSERT_TRUE(ParseFileName(meta.name, &number, &type)); + ASSERT_EQ(listener->file_temperatures.at(number), meta.temperature); + + meta = metadata.levels[kLastLevel].files[0]; + ASSERT_EQ(Temperature::kWarm, meta.temperature); + ASSERT_TRUE(ParseFileName(meta.name, &number, &type)); + ASSERT_EQ(listener->file_temperatures.at(number), meta.temperature); +} + +TEST_F(DBTest2, LastLevelTemperatureUniversal) { + const int kTriggerNum = 3; + const int kNumLevels = 5; + const int kBottommostLevel = kNumLevels - 1; + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.level0_file_num_compaction_trigger = kTriggerNum; + options.num_levels = kNumLevels; + options.statistics = CreateDBStatistics(); + DestroyAndReopen(options); + + auto size = GetSstSizeHelper(Temperature::kUnknown); + ASSERT_EQ(size, 0); + size = GetSstSizeHelper(Temperature::kWarm); + ASSERT_EQ(size, 0); + size = GetSstSizeHelper(Temperature::kHot); + ASSERT_EQ(size, 0); + get_iostats_context()->Reset(); + IOStatsContext* iostats = get_iostats_context(); + + for (int i = 0; i < kTriggerNum; i++) { + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Put("bar", "bar")); + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ColumnFamilyMetaData metadata; + db_->GetColumnFamilyMetaData(&metadata); + ASSERT_EQ(1, metadata.file_count); + ASSERT_EQ(Temperature::kUnknown, + metadata.levels[kBottommostLevel].files[0].temperature); + size = GetSstSizeHelper(Temperature::kUnknown); + ASSERT_GT(size, 0); + size = GetSstSizeHelper(Temperature::kWarm); + ASSERT_EQ(size, 0); + ASSERT_EQ(iostats->file_io_stats_by_temperature.hot_file_read_count, 0); + ASSERT_EQ(iostats->file_io_stats_by_temperature.warm_file_read_count, 0); + ASSERT_EQ(iostats->file_io_stats_by_temperature.cold_file_read_count, 0); + ASSERT_EQ(options.statistics->getTickerCount(HOT_FILE_READ_BYTES), 0); + ASSERT_EQ(options.statistics->getTickerCount(WARM_FILE_READ_BYTES), 0); + ASSERT_EQ(options.statistics->getTickerCount(COLD_FILE_READ_BYTES), 0); + ASSERT_EQ(options.statistics->getTickerCount(HOT_FILE_READ_COUNT), 0); + ASSERT_EQ(options.statistics->getTickerCount(WARM_FILE_READ_COUNT), 0); + ASSERT_EQ(options.statistics->getTickerCount(COLD_FILE_READ_COUNT), 0); + ASSERT_EQ("bar", Get("foo")); + + ASSERT_EQ(iostats->file_io_stats_by_temperature.hot_file_read_count, 0); + ASSERT_EQ(iostats->file_io_stats_by_temperature.warm_file_read_count, 0); + ASSERT_EQ(iostats->file_io_stats_by_temperature.hot_file_read_count, 0); + ASSERT_EQ(iostats->file_io_stats_by_temperature.hot_file_bytes_read, 0); + ASSERT_EQ(iostats->file_io_stats_by_temperature.warm_file_bytes_read, 0); + ASSERT_EQ(iostats->file_io_stats_by_temperature.cold_file_bytes_read, 0); + ASSERT_EQ(options.statistics->getTickerCount(HOT_FILE_READ_BYTES), 0); + ASSERT_EQ(options.statistics->getTickerCount(WARM_FILE_READ_BYTES), 0); + ASSERT_EQ(options.statistics->getTickerCount(COLD_FILE_READ_BYTES), 0); + ASSERT_EQ(options.statistics->getTickerCount(HOT_FILE_READ_COUNT), 0); + ASSERT_EQ(options.statistics->getTickerCount(WARM_FILE_READ_COUNT), 0); + ASSERT_EQ(options.statistics->getTickerCount(COLD_FILE_READ_COUNT), 0); + + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Put("bar", "bar")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + db_->GetColumnFamilyMetaData(&metadata); + ASSERT_EQ(2, metadata.file_count); + ASSERT_EQ(Temperature::kUnknown, metadata.levels[0].files[0].temperature); + size = GetSstSizeHelper(Temperature::kUnknown); + ASSERT_GT(size, 0); + size = GetSstSizeHelper(Temperature::kWarm); + ASSERT_EQ(size, 0); + + // Update bottommost temperature + options.bottommost_temperature = Temperature::kWarm; + Reopen(options); + db_->GetColumnFamilyMetaData(&metadata); + // Should not impact existing ones + ASSERT_EQ(Temperature::kUnknown, + metadata.levels[kBottommostLevel].files[0].temperature); + size = GetSstSizeHelper(Temperature::kUnknown); + ASSERT_GT(size, 0); + size = GetSstSizeHelper(Temperature::kWarm); + ASSERT_EQ(size, 0); + + // new generated file should have the new settings + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + db_->GetColumnFamilyMetaData(&metadata); + ASSERT_EQ(1, metadata.file_count); + ASSERT_EQ(Temperature::kWarm, + metadata.levels[kBottommostLevel].files[0].temperature); + size = GetSstSizeHelper(Temperature::kUnknown); + ASSERT_EQ(size, 0); + size = GetSstSizeHelper(Temperature::kWarm); + ASSERT_GT(size, 0); + ASSERT_EQ(options.statistics->getTickerCount(HOT_FILE_READ_BYTES), 0); + ASSERT_GT(options.statistics->getTickerCount(WARM_FILE_READ_BYTES), 0); + ASSERT_EQ(options.statistics->getTickerCount(COLD_FILE_READ_BYTES), 0); + ASSERT_EQ(options.statistics->getTickerCount(HOT_FILE_READ_COUNT), 0); + ASSERT_GT(options.statistics->getTickerCount(WARM_FILE_READ_COUNT), 0); + ASSERT_EQ(options.statistics->getTickerCount(COLD_FILE_READ_COUNT), 0); + + // non-bottommost file still has unknown temperature + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Put("bar", "bar")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + db_->GetColumnFamilyMetaData(&metadata); + ASSERT_EQ(2, metadata.file_count); + ASSERT_EQ(Temperature::kUnknown, metadata.levels[0].files[0].temperature); + size = GetSstSizeHelper(Temperature::kUnknown); + ASSERT_GT(size, 0); + size = GetSstSizeHelper(Temperature::kWarm); + ASSERT_GT(size, 0); + + // check other non-exist temperatures + size = GetSstSizeHelper(Temperature::kHot); + ASSERT_EQ(size, 0); + size = GetSstSizeHelper(Temperature::kCold); + ASSERT_EQ(size, 0); + std::string prop; + ASSERT_TRUE(dbfull()->GetProperty( + DB::Properties::kLiveSstFilesSizeAtTemperature + std::to_string(22), + &prop)); + ASSERT_EQ(std::atoi(prop.c_str()), 0); + + // Update bottommost temperature dynamically with SetOptions + auto s = db_->SetOptions({{"last_level_temperature", "kCold"}}); + ASSERT_OK(s); + ASSERT_EQ(db_->GetOptions().bottommost_temperature, Temperature::kCold); + db_->GetColumnFamilyMetaData(&metadata); + // Should not impact the existing files + ASSERT_EQ(Temperature::kWarm, + metadata.levels[kBottommostLevel].files[0].temperature); + size = GetSstSizeHelper(Temperature::kUnknown); + ASSERT_GT(size, 0); + size = GetSstSizeHelper(Temperature::kWarm); + ASSERT_GT(size, 0); + size = GetSstSizeHelper(Temperature::kCold); + ASSERT_EQ(size, 0); + + // new generated files should have the new settings + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + db_->GetColumnFamilyMetaData(&metadata); + ASSERT_EQ(1, metadata.file_count); + ASSERT_EQ(Temperature::kCold, + metadata.levels[kBottommostLevel].files[0].temperature); + size = GetSstSizeHelper(Temperature::kUnknown); + ASSERT_EQ(size, 0); + size = GetSstSizeHelper(Temperature::kWarm); + ASSERT_EQ(size, 0); + size = GetSstSizeHelper(Temperature::kCold); + ASSERT_GT(size, 0); + + // kLastTemperature is an invalid temperature + options.bottommost_temperature = Temperature::kLastTemperature; + s = TryReopen(options); + ASSERT_TRUE(s.IsIOError()); +} + +TEST_F(DBTest2, LastLevelStatistics) { + Options options = CurrentOptions(); + options.bottommost_temperature = Temperature::kWarm; + options.level0_file_num_compaction_trigger = 2; + options.level_compaction_dynamic_level_bytes = true; + options.statistics = CreateDBStatistics(); + Reopen(options); + + // generate 1 sst on level 0 + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Put("bar", "bar")); + ASSERT_OK(Flush()); + ASSERT_EQ("bar", Get("bar")); + + ASSERT_GT(options.statistics->getTickerCount(NON_LAST_LEVEL_READ_BYTES), 0); + ASSERT_GT(options.statistics->getTickerCount(NON_LAST_LEVEL_READ_COUNT), 0); + ASSERT_EQ(options.statistics->getTickerCount(LAST_LEVEL_READ_BYTES), 0); + ASSERT_EQ(options.statistics->getTickerCount(LAST_LEVEL_READ_COUNT), 0); + + // 2nd flush to trigger compaction + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Put("bar", "bar")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ("bar", Get("bar")); + + ASSERT_EQ(options.statistics->getTickerCount(LAST_LEVEL_READ_BYTES), + options.statistics->getTickerCount(WARM_FILE_READ_BYTES)); + ASSERT_EQ(options.statistics->getTickerCount(LAST_LEVEL_READ_COUNT), + options.statistics->getTickerCount(WARM_FILE_READ_COUNT)); + + auto pre_bytes = + options.statistics->getTickerCount(NON_LAST_LEVEL_READ_BYTES); + auto pre_count = + options.statistics->getTickerCount(NON_LAST_LEVEL_READ_COUNT); + + // 3rd flush to generate 1 sst on level 0 + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Put("bar", "bar")); + ASSERT_OK(Flush()); + ASSERT_EQ("bar", Get("bar")); + + ASSERT_GT(options.statistics->getTickerCount(NON_LAST_LEVEL_READ_BYTES), + pre_bytes); + ASSERT_GT(options.statistics->getTickerCount(NON_LAST_LEVEL_READ_COUNT), + pre_count); + ASSERT_EQ(options.statistics->getTickerCount(LAST_LEVEL_READ_BYTES), + options.statistics->getTickerCount(WARM_FILE_READ_BYTES)); + ASSERT_EQ(options.statistics->getTickerCount(LAST_LEVEL_READ_COUNT), + options.statistics->getTickerCount(WARM_FILE_READ_COUNT)); +} + +TEST_F(DBTest2, CheckpointFileTemperature) { + class NoLinkTestFS : public FileTemperatureTestFS { + using FileTemperatureTestFS::FileTemperatureTestFS; + + IOStatus LinkFile(const std::string&, const std::string&, const IOOptions&, + IODebugContext*) override { + // return not supported to force checkpoint copy the file instead of just + // link + return IOStatus::NotSupported(); + } + }; + auto test_fs = std::make_shared(env_->GetFileSystem()); + std::unique_ptr env(new CompositeEnvWrapper(env_, test_fs)); + Options options = CurrentOptions(); + options.bottommost_temperature = Temperature::kWarm; + // set dynamic_level to true so the compaction would compact the data to the + // last level directly which will have the last_level_temperature + options.level_compaction_dynamic_level_bytes = true; + options.level0_file_num_compaction_trigger = 2; + options.env = env.get(); + Reopen(options); + + // generate a bottommost file and a non-bottommost file + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Put("bar", "bar")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Put("bar", "bar")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Put("bar", "bar")); + ASSERT_OK(Flush()); + auto size = GetSstSizeHelper(Temperature::kWarm); + ASSERT_GT(size, 0); + + std::map temperatures; + std::vector infos; + ASSERT_OK( + dbfull()->GetLiveFilesStorageInfo(LiveFilesStorageInfoOptions(), &infos)); + for (auto info : infos) { + temperatures.emplace(info.file_number, info.temperature); + } + + test_fs->PopRequestedSstFileTemperatures(); + Checkpoint* checkpoint; + ASSERT_OK(Checkpoint::Create(db_, &checkpoint)); + ASSERT_OK( + checkpoint->CreateCheckpoint(dbname_ + kFilePathSeparator + "tempcp")); + + // checking src file src_temperature hints: 2 sst files: 1 sst is kWarm, + // another is kUnknown + std::vector> requested_temps; + test_fs->PopRequestedSstFileTemperatures(&requested_temps); + // Two requests + ASSERT_EQ(requested_temps.size(), 2); + std::set distinct_requests; + for (const auto& requested_temp : requested_temps) { + // Matching manifest temperatures + ASSERT_EQ(temperatures.at(requested_temp.first), requested_temp.second); + distinct_requests.insert(requested_temp.first); + } + // Each request to distinct file + ASSERT_EQ(distinct_requests.size(), requested_temps.size()); + + delete checkpoint; + Close(); +} + +TEST_F(DBTest2, FileTemperatureManifestFixup) { + auto test_fs = std::make_shared(env_->GetFileSystem()); + std::unique_ptr env(new CompositeEnvWrapper(env_, test_fs)); + Options options = CurrentOptions(); + options.bottommost_temperature = Temperature::kWarm; + // set dynamic_level to true so the compaction would compact the data to the + // last level directly which will have the last_level_temperature + options.level_compaction_dynamic_level_bytes = true; + options.level0_file_num_compaction_trigger = 2; + options.env = env.get(); + std::vector cfs = {/*"default",*/ "test1", "test2"}; + CreateAndReopenWithCF(cfs, options); + // Needed for later re-opens (weird) + cfs.insert(cfs.begin(), kDefaultColumnFamilyName); + + // Generate a bottommost file in all CFs + for (int cf = 0; cf < 3; ++cf) { + ASSERT_OK(Put(cf, "a", "val")); + ASSERT_OK(Put(cf, "c", "val")); + ASSERT_OK(Flush(cf)); + ASSERT_OK(Put(cf, "b", "val")); + ASSERT_OK(Put(cf, "d", "val")); + ASSERT_OK(Flush(cf)); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // verify + ASSERT_GT(GetSstSizeHelper(Temperature::kWarm), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kHot), 0); + + // Generate a non-bottommost file in all CFs + for (int cf = 0; cf < 3; ++cf) { + ASSERT_OK(Put(cf, "e", "val")); + ASSERT_OK(Flush(cf)); + } + + // re-verify + ASSERT_GT(GetSstSizeHelper(Temperature::kWarm), 0); + // Not supported: ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kHot), 0); + + // Now change FS temperature on bottommost file(s) to kCold + std::map current_temps; + test_fs->CopyCurrentSstFileTemperatures(¤t_temps); + for (auto e : current_temps) { + if (e.second == Temperature::kWarm) { + test_fs->OverrideSstFileTemperature(e.first, Temperature::kCold); + } + } + // Metadata not yet updated + ASSERT_EQ(Get("a"), "val"); + ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0); + + // Update with Close and UpdateManifestForFilesState, but first save cf + // descriptors + std::vector column_families; + for (size_t i = 0; i < handles_.size(); ++i) { + ColumnFamilyDescriptor cfdescriptor; + handles_[i]->GetDescriptor(&cfdescriptor).PermitUncheckedError(); + column_families.push_back(cfdescriptor); + } + Close(); + experimental::UpdateManifestForFilesStateOptions update_opts; + update_opts.update_temperatures = true; + + ASSERT_OK(experimental::UpdateManifestForFilesState( + options, dbname_, column_families, update_opts)); + + // Re-open and re-verify after update + ReopenWithColumnFamilies(cfs, options); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + // Not supported: ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kWarm), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kHot), 0); + + // Change kUnknown to kHot + test_fs->CopyCurrentSstFileTemperatures(¤t_temps); + for (auto e : current_temps) { + if (e.second == Temperature::kUnknown) { + test_fs->OverrideSstFileTemperature(e.first, Temperature::kHot); + } + } + + // Update with Close and UpdateManifestForFilesState + Close(); + ASSERT_OK(experimental::UpdateManifestForFilesState( + options, dbname_, column_families, update_opts)); + + // Re-open and re-verify after update + ReopenWithColumnFamilies(cfs, options); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kWarm), 0); + ASSERT_GT(GetSstSizeHelper(Temperature::kHot), 0); + + Close(); +} + +// WAL recovery mode is WALRecoveryMode::kPointInTimeRecovery. +TEST_F(DBTest2, PointInTimeRecoveryWithIOErrorWhileReadingWal) { + Options options = CurrentOptions(); + DestroyAndReopen(options); + ASSERT_OK(Put("foo", "value0")); + Close(); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + bool should_inject_error = false; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::RecoverLogFiles:BeforeReadWal", + [&](void* /*arg*/) { should_inject_error = true; }); + SyncPoint::GetInstance()->SetCallBack( + "LogReader::ReadMore:AfterReadFile", [&](void* arg) { + if (should_inject_error) { + ASSERT_NE(nullptr, arg); + *reinterpret_cast(arg) = Status::IOError("Injected IOError"); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + options.avoid_flush_during_recovery = true; + options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery; + Status s = TryReopen(options); + ASSERT_TRUE(s.IsIOError()); +} + +TEST_F(DBTest2, PointInTimeRecoveryWithSyncFailureInCFCreation) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BackgroundCallFlush:Start:1", + "PointInTimeRecoveryWithSyncFailureInCFCreation:1"}, + {"PointInTimeRecoveryWithSyncFailureInCFCreation:2", + "DBImpl::BackgroundCallFlush:Start:2"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + CreateColumnFamilies({"test1"}, Options()); + ASSERT_OK(Put("foo", "bar")); + + // Creating a CF when a flush is going on, log is synced but the + // closed log file is not synced and corrupted. + port::Thread flush_thread([&]() { ASSERT_NOK(Flush()); }); + TEST_SYNC_POINT("PointInTimeRecoveryWithSyncFailureInCFCreation:1"); + CreateColumnFamilies({"test2"}, Options()); + env_->corrupt_in_sync_ = true; + TEST_SYNC_POINT("PointInTimeRecoveryWithSyncFailureInCFCreation:2"); + flush_thread.join(); + env_->corrupt_in_sync_ = false; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + // Reopening the DB should not corrupt anything + Options options = CurrentOptions(); + options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery; + ReopenWithColumnFamilies({"default", "test1", "test2"}, options); +} + +TEST_F(DBTest2, SortL0FilesByEpochNumber) { + Options options = CurrentOptions(); + options.num_levels = 1; + options.compaction_style = kCompactionStyleUniversal; + DestroyAndReopen(options); + + // Set up L0 files to be sorted by their epoch_number + ASSERT_OK(Put("key1", "seq1")); + + SstFileWriter sst_file_writer{EnvOptions(), options}; + std::string external_file1 = dbname_ + "/test_files1.sst"; + std::string external_file2 = dbname_ + "/test_files2.sst"; + ASSERT_OK(sst_file_writer.Open(external_file1)); + ASSERT_OK(sst_file_writer.Put("key2", "seq0")); + ASSERT_OK(sst_file_writer.Finish()); + ASSERT_OK(sst_file_writer.Open(external_file2)); + ASSERT_OK(sst_file_writer.Put("key3", "seq0")); + ASSERT_OK(sst_file_writer.Finish()); + + ASSERT_OK(Put("key4", "seq2")); + ASSERT_OK(Flush()); + + auto* handle = db_->DefaultColumnFamily(); + ASSERT_OK(db_->IngestExternalFile(handle, {external_file1, external_file2}, + IngestExternalFileOptions())); + + // To verify L0 files are sorted by epoch_number in descending order + // instead of largest_seqno + std::vector level0_files = GetLevelFileMetadatas(0 /* level*/); + ASSERT_EQ(level0_files.size(), 3); + + EXPECT_EQ(level0_files[0]->epoch_number, 3); + EXPECT_EQ(level0_files[0]->fd.largest_seqno, 0); + ASSERT_EQ(level0_files[0]->num_entries, 1); + ASSERT_TRUE(level0_files[0]->largest.user_key() == Slice("key3")); + + EXPECT_EQ(level0_files[1]->epoch_number, 2); + EXPECT_EQ(level0_files[1]->fd.largest_seqno, 0); + ASSERT_EQ(level0_files[1]->num_entries, 1); + ASSERT_TRUE(level0_files[1]->largest.user_key() == Slice("key2")); + + EXPECT_EQ(level0_files[2]->epoch_number, 1); + EXPECT_EQ(level0_files[2]->fd.largest_seqno, 2); + ASSERT_EQ(level0_files[2]->num_entries, 2); + ASSERT_TRUE(level0_files[2]->largest.user_key() == Slice("key4")); + ASSERT_TRUE(level0_files[2]->smallest.user_key() == Slice("key1")); + + // To verify compacted file is assigned with the minimum epoch_number + // among input files' + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + level0_files = GetLevelFileMetadatas(0 /* level*/); + ASSERT_EQ(level0_files.size(), 1); + EXPECT_EQ(level0_files[0]->epoch_number, 1); + ASSERT_EQ(level0_files[0]->num_entries, 4); + ASSERT_TRUE(level0_files[0]->largest.user_key() == Slice("key4")); + ASSERT_TRUE(level0_files[0]->smallest.user_key() == Slice("key1")); +} + +TEST_F(DBTest2, SameEpochNumberAfterCompactRangeChangeLevel) { + Options options = CurrentOptions(); + options.num_levels = 7; + options.compaction_style = CompactionStyle::kCompactionStyleLevel; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + + // Set up the file in L1 to be moved to L0 in later step of CompactRange() + ASSERT_OK(Put("key1", "seq1")); + ASSERT_OK(Flush()); + MoveFilesToLevel(1, 0); + std::vector level0_files = GetLevelFileMetadatas(0 /* level*/); + ASSERT_EQ(level0_files.size(), 0); + std::vector level1_files = GetLevelFileMetadatas(1 /* level*/); + ASSERT_EQ(level1_files.size(), 1); + std::vector level2_files = GetLevelFileMetadatas(2 /* level*/); + ASSERT_EQ(level2_files.size(), 0); + + ASSERT_EQ(level1_files[0]->epoch_number, 1); + + // To verify CompactRange() moving file to L0 still keeps the file's + // epoch_number + CompactRangeOptions croptions; + croptions.change_level = true; + croptions.target_level = 0; + ASSERT_OK(db_->CompactRange(croptions, nullptr, nullptr)); + level0_files = GetLevelFileMetadatas(0 /* level*/); + level1_files = GetLevelFileMetadatas(1 /* level*/); + ASSERT_EQ(level0_files.size(), 1); + ASSERT_EQ(level1_files.size(), 0); + + EXPECT_EQ(level0_files[0]->epoch_number, 1); + + ASSERT_EQ(level0_files[0]->num_entries, 1); + ASSERT_TRUE(level0_files[0]->largest.user_key() == Slice("key1")); +} + +TEST_F(DBTest2, RecoverEpochNumber) { + for (bool allow_ingest_behind : {true, false}) { + Options options = CurrentOptions(); + options.allow_ingest_behind = allow_ingest_behind; + options.num_levels = 7; + options.compaction_style = kCompactionStyleLevel; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + CreateAndReopenWithCF({"cf1"}, options); + VersionSet* versions = dbfull()->GetVersionSet(); + assert(versions); + const ColumnFamilyData* default_cf = + versions->GetColumnFamilySet()->GetDefault(); + const ColumnFamilyData* cf1 = + versions->GetColumnFamilySet()->GetColumnFamily("cf1"); + + // Set up files in default CF to recover in later step + ASSERT_OK(Put("key1", "epoch1")); + ASSERT_OK(Flush()); + MoveFilesToLevel(1 /* level*/, 0 /* cf*/); + ASSERT_OK(Put("key2", "epoch2")); + ASSERT_OK(Flush()); + + std::vector level0_files = + GetLevelFileMetadatas(0 /* level*/); + ASSERT_EQ(level0_files.size(), 1); + ASSERT_EQ(level0_files[0]->epoch_number, + allow_ingest_behind + ? 2 + kReservedEpochNumberForFileIngestedBehind + : 2); + ASSERT_EQ(level0_files[0]->num_entries, 1); + ASSERT_TRUE(level0_files[0]->largest.user_key() == Slice("key2")); + + std::vector level1_files = + GetLevelFileMetadatas(1 /* level*/); + ASSERT_EQ(level1_files.size(), 1); + ASSERT_EQ(level1_files[0]->epoch_number, + allow_ingest_behind + ? 1 + kReservedEpochNumberForFileIngestedBehind + : 1); + ASSERT_EQ(level1_files[0]->num_entries, 1); + ASSERT_TRUE(level1_files[0]->largest.user_key() == Slice("key1")); + + // Set up files in cf1 to recover in later step + ASSERT_OK(Put(1 /* cf */, "cf1_key1", "epoch1")); + ASSERT_OK(Flush(1 /* cf */)); + + std::vector level0_files_cf1 = + GetLevelFileMetadatas(0 /* level*/, 1 /* cf*/); + ASSERT_EQ(level0_files_cf1.size(), 1); + ASSERT_EQ(level0_files_cf1[0]->epoch_number, + allow_ingest_behind + ? 1 + kReservedEpochNumberForFileIngestedBehind + : 1); + ASSERT_EQ(level0_files_cf1[0]->num_entries, 1); + ASSERT_TRUE(level0_files_cf1[0]->largest.user_key() == Slice("cf1_key1")); + + ASSERT_EQ(default_cf->GetNextEpochNumber(), + allow_ingest_behind + ? 3 + kReservedEpochNumberForFileIngestedBehind + : 3); + ASSERT_EQ(cf1->GetNextEpochNumber(), + allow_ingest_behind + ? 2 + kReservedEpochNumberForFileIngestedBehind + : 2); + + // To verify epoch_number of files of different levels/CFs are + // persisted and recovered correctly + ReopenWithColumnFamilies({"default", "cf1"}, options); + versions = dbfull()->GetVersionSet(); + assert(versions); + default_cf = versions->GetColumnFamilySet()->GetDefault(); + cf1 = versions->GetColumnFamilySet()->GetColumnFamily("cf1"); + + level0_files = GetLevelFileMetadatas(0 /* level*/); + ASSERT_EQ(level0_files.size(), 1); + EXPECT_EQ(level0_files[0]->epoch_number, + allow_ingest_behind + ? 2 + kReservedEpochNumberForFileIngestedBehind + : 2); + ASSERT_EQ(level0_files[0]->num_entries, 1); + ASSERT_TRUE(level0_files[0]->largest.user_key() == Slice("key2")); + + level1_files = GetLevelFileMetadatas(1 /* level*/); + ASSERT_EQ(level1_files.size(), 1); + EXPECT_EQ(level1_files[0]->epoch_number, + allow_ingest_behind + ? 1 + kReservedEpochNumberForFileIngestedBehind + : 1); + ASSERT_EQ(level1_files[0]->num_entries, 1); + ASSERT_TRUE(level1_files[0]->largest.user_key() == Slice("key1")); + + level0_files_cf1 = GetLevelFileMetadatas(0 /* level*/, 1 /* cf*/); + ASSERT_EQ(level0_files_cf1.size(), 1); + EXPECT_EQ(level0_files_cf1[0]->epoch_number, + allow_ingest_behind + ? 1 + kReservedEpochNumberForFileIngestedBehind + : 1); + ASSERT_EQ(level0_files_cf1[0]->num_entries, 1); + ASSERT_TRUE(level0_files_cf1[0]->largest.user_key() == Slice("cf1_key1")); + + // To verify next epoch number is recovered correctly + EXPECT_EQ(default_cf->GetNextEpochNumber(), + allow_ingest_behind + ? 3 + kReservedEpochNumberForFileIngestedBehind + : 3); + EXPECT_EQ(cf1->GetNextEpochNumber(), + allow_ingest_behind + ? 2 + kReservedEpochNumberForFileIngestedBehind + : 2); + } +} + + +TEST_F(DBTest2, RenameDirectory) { + Options options = CurrentOptions(); + DestroyAndReopen(options); + ASSERT_OK(Put("foo", "value0")); + Close(); + auto old_dbname = dbname_; + auto new_dbname = dbname_ + "_2"; + EXPECT_OK(env_->RenameFile(dbname_, new_dbname)); + options.create_if_missing = false; + dbname_ = new_dbname; + ASSERT_OK(TryReopen(options)); + ASSERT_EQ("value0", Get("foo")); + Destroy(options); + dbname_ = old_dbname; +} + +TEST_F(DBTest2, SstUniqueIdVerifyBackwardCompatible) { + const int kNumSst = 3; + const int kLevel0Trigger = 4; + auto options = CurrentOptions(); + options.level0_file_num_compaction_trigger = kLevel0Trigger; + options.statistics = CreateDBStatistics(); + // Skip for now + options.verify_sst_unique_id_in_manifest = false; + Reopen(options); + + std::atomic_int skipped = 0; + std::atomic_int passed = 0; + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTable::Open::SkippedVerifyUniqueId", + [&](void* /*arg*/) { skipped++; }); + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTable::Open::PassedVerifyUniqueId", + [&](void* /*arg*/) { passed++; }); + SyncPoint::GetInstance()->EnableProcessing(); + + // generate a few SSTs + for (int i = 0; i < kNumSst; i++) { + for (int j = 0; j < 100; j++) { + ASSERT_OK(Put(Key(i * 10 + j), "value")); + } + ASSERT_OK(Flush()); + } + + // Verification has been skipped on files so far + EXPECT_EQ(skipped, kNumSst); + EXPECT_EQ(passed, 0); + + // Reopen with verification + options.verify_sst_unique_id_in_manifest = true; + skipped = 0; + passed = 0; + Reopen(options); + EXPECT_EQ(skipped, 0); + EXPECT_EQ(passed, kNumSst); + + // Now simulate no unique id in manifest for next file + // NOTE: this only works for loading manifest from disk, + // not in-memory manifest, so we need to re-open below. + SyncPoint::GetInstance()->SetCallBack( + "VersionEdit::EncodeTo:UniqueId", [&](void* arg) { + auto unique_id = static_cast(arg); + // remove id before writing it to manifest + (*unique_id)[0] = 0; + (*unique_id)[1] = 0; + }); + + // test compaction generated Sst + for (int i = kNumSst; i < kLevel0Trigger; i++) { + for (int j = 0; j < 100; j++) { + ASSERT_OK(Put(Key(i * 10 + j), "value")); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ("0,1", FilesPerLevel(0)); + + // Reopen (with verification) + ASSERT_TRUE(options.verify_sst_unique_id_in_manifest); + skipped = 0; + passed = 0; + Reopen(options); + EXPECT_EQ(skipped, 1); + EXPECT_EQ(passed, 0); +} + +TEST_F(DBTest2, SstUniqueIdVerify) { + const int kNumSst = 3; + const int kLevel0Trigger = 4; + auto options = CurrentOptions(); + options.level0_file_num_compaction_trigger = kLevel0Trigger; + // Allow mismatch for now + options.verify_sst_unique_id_in_manifest = false; + Reopen(options); + + SyncPoint::GetInstance()->SetCallBack( + "PropertyBlockBuilder::AddTableProperty:Start", [&](void* props_vs) { + auto props = static_cast(props_vs); + // update table property session_id to a different one, which + // changes unique ID + props->db_session_id = DBImpl::GenerateDbSessionId(nullptr); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + // generate a few SSTs + for (int i = 0; i < kNumSst; i++) { + for (int j = 0; j < 100; j++) { + ASSERT_OK(Put(Key(i * 10 + j), "value")); + } + ASSERT_OK(Flush()); + } + + // Reopen with verification should report corruption + options.verify_sst_unique_id_in_manifest = true; + auto s = TryReopen(options); + ASSERT_TRUE(s.IsCorruption()); + + // Reopen without verification should be fine + options.verify_sst_unique_id_in_manifest = false; + Reopen(options); + + // test compaction generated Sst + for (int i = kNumSst; i < kLevel0Trigger; i++) { + for (int j = 0; j < 100; j++) { + ASSERT_OK(Put(Key(i * 10 + j), "value")); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ("0,1", FilesPerLevel(0)); + + // Reopen with verification should fail + options.verify_sst_unique_id_in_manifest = true; + s = TryReopen(options); + ASSERT_TRUE(s.IsCorruption()); +} + +TEST_F(DBTest2, SstUniqueIdVerifyMultiCFs) { + const int kNumSst = 3; + const int kLevel0Trigger = 4; + auto options = CurrentOptions(); + options.level0_file_num_compaction_trigger = kLevel0Trigger; + // Allow mismatch for now + options.verify_sst_unique_id_in_manifest = false; + + CreateAndReopenWithCF({"one", "two"}, options); + + // generate good SSTs + for (int cf_num : {0, 2}) { + for (int i = 0; i < kNumSst; i++) { + for (int j = 0; j < 100; j++) { + ASSERT_OK(Put(cf_num, Key(i * 10 + j), "value")); + } + ASSERT_OK(Flush(cf_num)); + } + } + + // generate SSTs with bad unique id + SyncPoint::GetInstance()->SetCallBack( + "PropertyBlockBuilder::AddTableProperty:Start", [&](void* props_vs) { + auto props = static_cast(props_vs); + // update table property session_id to a different one + props->db_session_id = DBImpl::GenerateDbSessionId(nullptr); + }); + SyncPoint::GetInstance()->EnableProcessing(); + for (int i = 0; i < kNumSst; i++) { + for (int j = 0; j < 100; j++) { + ASSERT_OK(Put(1, Key(i * 10 + j), "value")); + } + ASSERT_OK(Flush(1)); + } + + // Reopen with verification should report corruption + options.verify_sst_unique_id_in_manifest = true; + auto s = TryReopenWithColumnFamilies({"default", "one", "two"}, options); + ASSERT_TRUE(s.IsCorruption()); +} + +TEST_F(DBTest2, BestEffortsRecoveryWithSstUniqueIdVerification) { + const auto tamper_with_uniq_id = [&](void* arg) { + auto props = static_cast(arg); + assert(props); + // update table property session_id to a different one + props->db_session_id = DBImpl::GenerateDbSessionId(nullptr); + }; + + const auto assert_db = [&](size_t expected_count, + const std::string& expected_v) { + std::unique_ptr it(db_->NewIterator(ReadOptions())); + size_t cnt = 0; + for (it->SeekToFirst(); it->Valid(); it->Next(), ++cnt) { + ASSERT_EQ(std::to_string(cnt), it->key()); + ASSERT_EQ(expected_v, it->value()); + } + ASSERT_EQ(expected_count, cnt); + }; + + const int num_l0_compaction_trigger = 8; + const int num_l0 = num_l0_compaction_trigger - 1; + Options options = CurrentOptions(); + options.level0_file_num_compaction_trigger = num_l0_compaction_trigger; + + for (int k = 0; k < num_l0; ++k) { + // Allow mismatch for now + options.verify_sst_unique_id_in_manifest = false; + + DestroyAndReopen(options); + + constexpr size_t num_keys_per_file = 10; + for (int i = 0; i < num_l0; ++i) { + for (size_t j = 0; j < num_keys_per_file; ++j) { + ASSERT_OK(Put(std::to_string(j), "v" + std::to_string(i))); + } + if (i == k) { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->SetCallBack( + "PropertyBlockBuilder::AddTableProperty:Start", + tamper_with_uniq_id); + SyncPoint::GetInstance()->EnableProcessing(); + } + ASSERT_OK(Flush()); + } + + options.verify_sst_unique_id_in_manifest = true; + Status s = TryReopen(options); + ASSERT_TRUE(s.IsCorruption()); + + options.best_efforts_recovery = true; + Reopen(options); + assert_db(k == 0 ? 0 : num_keys_per_file, "v" + std::to_string(k - 1)); + + // Reopen with regular recovery + options.best_efforts_recovery = false; + Reopen(options); + assert_db(k == 0 ? 0 : num_keys_per_file, "v" + std::to_string(k - 1)); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + for (size_t i = 0; i < num_keys_per_file; ++i) { + ASSERT_OK(Put(std::to_string(i), "v")); + } + ASSERT_OK(Flush()); + Reopen(options); + { + for (size_t i = 0; i < num_keys_per_file; ++i) { + ASSERT_EQ("v", Get(std::to_string(i))); + } + } + } +} + +TEST_F(DBTest2, GetLatestSeqAndTsForKey) { + Destroy(last_options_); + + Options options = CurrentOptions(); + options.max_write_buffer_size_to_maintain = 64 << 10; + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + options.statistics = CreateDBStatistics(); + + Reopen(options); + + constexpr uint64_t kTsU64Value = 12; + + for (uint64_t key = 0; key < 100; ++key) { + std::string ts; + PutFixed64(&ts, kTsU64Value); + + std::string key_str; + PutFixed64(&key_str, key); + std::reverse(key_str.begin(), key_str.end()); + ASSERT_OK(db_->Put(WriteOptions(), key_str, ts, "value")); + } + + ASSERT_OK(Flush()); + + constexpr bool cache_only = true; + constexpr SequenceNumber lower_bound_seq = 0; + auto* cfhi = static_cast_with_check( + dbfull()->DefaultColumnFamily()); + assert(cfhi); + assert(cfhi->cfd()); + SuperVersion* sv = cfhi->cfd()->GetSuperVersion(); + for (uint64_t key = 0; key < 100; ++key) { + std::string key_str; + PutFixed64(&key_str, key); + std::reverse(key_str.begin(), key_str.end()); + std::string ts; + SequenceNumber seq = kMaxSequenceNumber; + bool found_record_for_key = false; + bool is_blob_index = false; + + const Status s = dbfull()->GetLatestSequenceForKey( + sv, key_str, cache_only, lower_bound_seq, &seq, &ts, + &found_record_for_key, &is_blob_index); + ASSERT_OK(s); + std::string expected_ts; + PutFixed64(&expected_ts, kTsU64Value); + ASSERT_EQ(expected_ts, ts); + ASSERT_TRUE(found_record_for_key); + ASSERT_FALSE(is_blob_index); + } + + // Verify that no read to SST files. + ASSERT_EQ(0, options.statistics->getTickerCount(GET_HIT_L0)); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_test_util.cc b/librocksdb-sys/rocksdb/db/db_test_util.cc new file mode 100644 index 0000000..5a64b2f --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_test_util.cc @@ -0,0 +1,1767 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/db_test_util.h" + +#include "cache/cache_reservation_manager.h" +#include "db/forward_iterator.h" +#include "env/mock_env.h" +#include "port/lang.h" +#include "rocksdb/cache.h" +#include "rocksdb/convenience.h" +#include "rocksdb/env_encryption.h" +#include "rocksdb/unique_id.h" +#include "rocksdb/utilities/object_registry.h" +#include "table/format.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { +int64_t MaybeCurrentTime(Env* env) { + int64_t time = 1337346000; // arbitrary fallback default + env->GetCurrentTime(&time).PermitUncheckedError(); + return time; +} +} // anonymous namespace + +// Special Env used to delay background operations + +SpecialEnv::SpecialEnv(Env* base, bool time_elapse_only_sleep) + : EnvWrapper(base), + maybe_starting_time_(MaybeCurrentTime(base)), + rnd_(301), + sleep_counter_(this), + time_elapse_only_sleep_(time_elapse_only_sleep), + no_slowdown_(time_elapse_only_sleep) { + delay_sstable_sync_.store(false, std::memory_order_release); + drop_writes_.store(false, std::memory_order_release); + no_space_.store(false, std::memory_order_release); + non_writable_.store(false, std::memory_order_release); + count_random_reads_ = false; + count_sequential_reads_ = false; + manifest_sync_error_.store(false, std::memory_order_release); + manifest_write_error_.store(false, std::memory_order_release); + log_write_error_.store(false, std::memory_order_release); + no_file_overwrite_.store(false, std::memory_order_release); + random_file_open_counter_.store(0, std::memory_order_relaxed); + delete_count_.store(0, std::memory_order_relaxed); + num_open_wal_file_.store(0); + log_write_slowdown_ = 0; + bytes_written_ = 0; + sync_counter_ = 0; + non_writeable_rate_ = 0; + new_writable_count_ = 0; + non_writable_count_ = 0; + table_write_callback_ = nullptr; +} +DBTestBase::DBTestBase(const std::string path, bool env_do_fsync) + : mem_env_(nullptr), encrypted_env_(nullptr), option_config_(kDefault) { + Env* base_env = Env::Default(); + ConfigOptions config_options; + EXPECT_OK(test::CreateEnvFromSystem(config_options, &base_env, &env_guard_)); + EXPECT_NE(nullptr, base_env); + if (getenv("MEM_ENV")) { + mem_env_ = MockEnv::Create(base_env, base_env->GetSystemClock()); + } + if (getenv("ENCRYPTED_ENV")) { + std::shared_ptr provider; + std::string provider_id = getenv("ENCRYPTED_ENV"); + if (provider_id.find("=") == std::string::npos && + !EndsWith(provider_id, "://test")) { + provider_id = provider_id + "://test"; + } + EXPECT_OK(EncryptionProvider::CreateFromString(ConfigOptions(), provider_id, + &provider)); + encrypted_env_ = NewEncryptedEnv(mem_env_ ? mem_env_ : base_env, provider); + } + env_ = new SpecialEnv(encrypted_env_ ? encrypted_env_ + : (mem_env_ ? mem_env_ : base_env)); + env_->SetBackgroundThreads(1, Env::LOW); + env_->SetBackgroundThreads(1, Env::HIGH); + env_->skip_fsync_ = !env_do_fsync; + dbname_ = test::PerThreadDBPath(env_, path); + alternative_wal_dir_ = dbname_ + "/wal"; + alternative_db_log_dir_ = dbname_ + "/db_log_dir"; + auto options = CurrentOptions(); + options.env = env_; + auto delete_options = options; + delete_options.wal_dir = alternative_wal_dir_; + EXPECT_OK(DestroyDB(dbname_, delete_options)); + // Destroy it for not alternative WAL dir is used. + EXPECT_OK(DestroyDB(dbname_, options)); + db_ = nullptr; + Reopen(options); + Random::GetTLSInstance()->Reset(0xdeadbeef); +} + +DBTestBase::~DBTestBase() { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + Close(); + Options options; + options.db_paths.emplace_back(dbname_, 0); + options.db_paths.emplace_back(dbname_ + "_2", 0); + options.db_paths.emplace_back(dbname_ + "_3", 0); + options.db_paths.emplace_back(dbname_ + "_4", 0); + options.env = env_; + + if (getenv("KEEP_DB")) { + printf("DB is still at %s\n", dbname_.c_str()); + } else { + EXPECT_OK(DestroyDB(dbname_, options)); + } + delete env_; +} + +bool DBTestBase::ShouldSkipOptions(int option_config, int skip_mask) { + + if ((skip_mask & kSkipUniversalCompaction) && + (option_config == kUniversalCompaction || + option_config == kUniversalCompactionMultiLevel || + option_config == kUniversalSubcompactions)) { + return true; + } + if ((skip_mask & kSkipMergePut) && option_config == kMergePut) { + return true; + } + if ((skip_mask & kSkipNoSeekToLast) && + (option_config == kHashLinkList || option_config == kHashSkipList)) { + return true; + } + if ((skip_mask & kSkipPlainTable) && + (option_config == kPlainTableAllBytesPrefix || + option_config == kPlainTableFirstBytePrefix || + option_config == kPlainTableCappedPrefix || + option_config == kPlainTableCappedPrefixNonMmap)) { + return true; + } + if ((skip_mask & kSkipHashIndex) && + (option_config == kBlockBasedTableWithPrefixHashIndex || + option_config == kBlockBasedTableWithWholeKeyHashIndex)) { + return true; + } + if ((skip_mask & kSkipFIFOCompaction) && option_config == kFIFOCompaction) { + return true; + } + if ((skip_mask & kSkipMmapReads) && option_config == kWalDirAndMmapReads) { + return true; + } + return false; +} + +// Switch to a fresh database with the next option configuration to +// test. Return false if there are no more configurations to test. +bool DBTestBase::ChangeOptions(int skip_mask) { + for (option_config_++; option_config_ < kEnd; option_config_++) { + if (ShouldSkipOptions(option_config_, skip_mask)) { + continue; + } + break; + } + + if (option_config_ >= kEnd) { + Destroy(last_options_); + return false; + } else { + auto options = CurrentOptions(); + options.create_if_missing = true; + DestroyAndReopen(options); + return true; + } +} + +// Switch between different compaction styles. +bool DBTestBase::ChangeCompactOptions() { + if (option_config_ == kDefault) { + option_config_ = kUniversalCompaction; + Destroy(last_options_); + auto options = CurrentOptions(); + options.create_if_missing = true; + Reopen(options); + return true; + } else if (option_config_ == kUniversalCompaction) { + option_config_ = kUniversalCompactionMultiLevel; + Destroy(last_options_); + auto options = CurrentOptions(); + options.create_if_missing = true; + Reopen(options); + return true; + } else if (option_config_ == kUniversalCompactionMultiLevel) { + option_config_ = kLevelSubcompactions; + Destroy(last_options_); + auto options = CurrentOptions(); + assert(options.max_subcompactions > 1); + Reopen(options); + return true; + } else if (option_config_ == kLevelSubcompactions) { + option_config_ = kUniversalSubcompactions; + Destroy(last_options_); + auto options = CurrentOptions(); + assert(options.max_subcompactions > 1); + Reopen(options); + return true; + } else { + return false; + } +} + +// Switch between different WAL settings +bool DBTestBase::ChangeWalOptions() { + if (option_config_ == kDefault) { + option_config_ = kDBLogDir; + Destroy(last_options_); + auto options = CurrentOptions(); + Destroy(options); + options.create_if_missing = true; + Reopen(options); + return true; + } else if (option_config_ == kDBLogDir) { + option_config_ = kWalDirAndMmapReads; + Destroy(last_options_); + auto options = CurrentOptions(); + Destroy(options); + options.create_if_missing = true; + Reopen(options); + return true; + } else if (option_config_ == kWalDirAndMmapReads) { + option_config_ = kRecycleLogFiles; + Destroy(last_options_); + auto options = CurrentOptions(); + Destroy(options); + Reopen(options); + return true; + } else { + return false; + } +} + +// Switch between different filter policy +// Jump from kDefault to kFilter to kFullFilter +bool DBTestBase::ChangeFilterOptions() { + if (option_config_ == kDefault) { + option_config_ = kFilter; + } else if (option_config_ == kFilter) { + option_config_ = kFullFilterWithNewTableReaderForCompactions; + } else if (option_config_ == kFullFilterWithNewTableReaderForCompactions) { + option_config_ = kPartitionedFilterWithNewTableReaderForCompactions; + } else { + return false; + } + Destroy(last_options_); + + auto options = CurrentOptions(); + options.create_if_missing = true; + TryReopen(options); + return true; +} + +// Switch between different DB options for file ingestion tests. +bool DBTestBase::ChangeOptionsForFileIngestionTest() { + if (option_config_ == kDefault) { + option_config_ = kUniversalCompaction; + Destroy(last_options_); + auto options = CurrentOptions(); + options.create_if_missing = true; + TryReopen(options); + return true; + } else if (option_config_ == kUniversalCompaction) { + option_config_ = kUniversalCompactionMultiLevel; + Destroy(last_options_); + auto options = CurrentOptions(); + options.create_if_missing = true; + TryReopen(options); + return true; + } else if (option_config_ == kUniversalCompactionMultiLevel) { + option_config_ = kLevelSubcompactions; + Destroy(last_options_); + auto options = CurrentOptions(); + assert(options.max_subcompactions > 1); + TryReopen(options); + return true; + } else if (option_config_ == kLevelSubcompactions) { + option_config_ = kUniversalSubcompactions; + Destroy(last_options_); + auto options = CurrentOptions(); + assert(options.max_subcompactions > 1); + TryReopen(options); + return true; + } else if (option_config_ == kUniversalSubcompactions) { + option_config_ = kDirectIO; + Destroy(last_options_); + auto options = CurrentOptions(); + TryReopen(options); + return true; + } else { + return false; + } +} + +// Return the current option configuration. +Options DBTestBase::CurrentOptions( + const anon::OptionsOverride& options_override) const { + return GetOptions(option_config_, GetDefaultOptions(), options_override); +} + +Options DBTestBase::CurrentOptions( + const Options& default_options, + const anon::OptionsOverride& options_override) const { + return GetOptions(option_config_, default_options, options_override); +} + +Options DBTestBase::GetDefaultOptions() const { + Options options; + options.write_buffer_size = 4090 * 4096; + options.target_file_size_base = 2 * 1024 * 1024; + options.max_bytes_for_level_base = 10 * 1024 * 1024; + options.max_open_files = 5000; + options.wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords; + options.compaction_pri = CompactionPri::kByCompensatedSize; + // The original default value for this option is false, + // and many unit tests assume this value. It also makes + // it easier to create desired LSM shape in unit tests. + // Unit tests for this option sets level_compaction_dynamic_level_bytes=true + // explicitly. + options.level_compaction_dynamic_level_bytes = false; + options.env = env_; + if (!env_->skip_fsync_) { + options.track_and_verify_wals_in_manifest = true; + } + return options; +} + +Options DBTestBase::GetOptions( + int option_config, const Options& default_options, + const anon::OptionsOverride& options_override) const { + // this redundant copy is to minimize code change w/o having lint error. + Options options = default_options; + BlockBasedTableOptions table_options; + bool set_block_based_table_factory = true; +#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && \ + !defined(OS_AIX) + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearCallBack( + "NewRandomAccessFile:O_DIRECT"); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearCallBack( + "NewWritableFile:O_DIRECT"); +#endif + // kMustFreeHeapAllocations -> indicates ASAN build + if (kMustFreeHeapAllocations && !options_override.full_block_cache) { + // Detecting block cache use-after-free is normally difficult in unit + // tests, because as a cache, it tends to keep unreferenced entries in + // memory, and we normally want unit tests to take advantage of block + // cache for speed. However, we also want a strong chance of detecting + // block cache use-after-free in unit tests in ASAN builds, so for ASAN + // builds we use a trivially small block cache to which entries can be + // added but are immediately freed on no more references. + table_options.block_cache = NewLRUCache(/* too small */ 1); + } + + bool can_allow_mmap = IsMemoryMappedAccessSupported(); + switch (option_config) { + case kHashSkipList: + options.prefix_extractor.reset(NewFixedPrefixTransform(1)); + options.memtable_factory.reset(NewHashSkipListRepFactory(16)); + options.allow_concurrent_memtable_write = false; + options.unordered_write = false; + break; + case kPlainTableFirstBytePrefix: + options.table_factory.reset(NewPlainTableFactory()); + options.prefix_extractor.reset(NewFixedPrefixTransform(1)); + options.allow_mmap_reads = can_allow_mmap; + options.max_sequential_skip_in_iterations = 999999; + set_block_based_table_factory = false; + break; + case kPlainTableCappedPrefix: + options.table_factory.reset(NewPlainTableFactory()); + options.prefix_extractor.reset(NewCappedPrefixTransform(8)); + options.allow_mmap_reads = can_allow_mmap; + options.max_sequential_skip_in_iterations = 999999; + set_block_based_table_factory = false; + break; + case kPlainTableCappedPrefixNonMmap: + options.table_factory.reset(NewPlainTableFactory()); + options.prefix_extractor.reset(NewCappedPrefixTransform(8)); + options.allow_mmap_reads = false; + options.max_sequential_skip_in_iterations = 999999; + set_block_based_table_factory = false; + break; + case kPlainTableAllBytesPrefix: + options.table_factory.reset(NewPlainTableFactory()); + options.prefix_extractor.reset(NewNoopTransform()); + options.allow_mmap_reads = can_allow_mmap; + options.max_sequential_skip_in_iterations = 999999; + set_block_based_table_factory = false; + break; + case kVectorRep: + options.memtable_factory.reset(new VectorRepFactory(100)); + options.allow_concurrent_memtable_write = false; + options.unordered_write = false; + break; + case kHashLinkList: + options.prefix_extractor.reset(NewFixedPrefixTransform(1)); + options.memtable_factory.reset( + NewHashLinkListRepFactory(4, 0, 3, true, 4)); + options.allow_concurrent_memtable_write = false; + options.unordered_write = false; + break; + case kDirectIO: { + options.use_direct_reads = true; + options.use_direct_io_for_flush_and_compaction = true; + options.compaction_readahead_size = 2 * 1024 * 1024; + SetupSyncPointsToMockDirectIO(); + break; + } + case kMergePut: + options.merge_operator = MergeOperators::CreatePutOperator(); + break; + case kFilter: + table_options.filter_policy.reset(NewBloomFilterPolicy(10, true)); + break; + case kFullFilterWithNewTableReaderForCompactions: + table_options.filter_policy.reset(NewBloomFilterPolicy(10, false)); + options.compaction_readahead_size = 10 * 1024 * 1024; + break; + case kPartitionedFilterWithNewTableReaderForCompactions: + table_options.filter_policy.reset(NewBloomFilterPolicy(10, false)); + table_options.partition_filters = true; + table_options.index_type = + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + options.compaction_readahead_size = 10 * 1024 * 1024; + break; + case kUncompressed: + options.compression = kNoCompression; + break; + case kNumLevel_3: + options.num_levels = 3; + break; + case kDBLogDir: + options.db_log_dir = alternative_db_log_dir_; + break; + case kWalDirAndMmapReads: + options.wal_dir = alternative_wal_dir_; + // mmap reads should be orthogonal to WalDir setting, so we piggyback to + // this option config to test mmap reads as well + options.allow_mmap_reads = can_allow_mmap; + break; + case kManifestFileSize: + options.max_manifest_file_size = 50; // 50 bytes + break; + case kPerfOptions: + options.delayed_write_rate = 8 * 1024 * 1024; + options.report_bg_io_stats = true; + // TODO(3.13) -- test more options + break; + case kUniversalCompaction: + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = 1; + break; + case kUniversalCompactionMultiLevel: + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = 8; + break; + case kInfiniteMaxOpenFiles: + options.max_open_files = -1; + break; + case kCRC32cChecksum: { + // Old default was CRC32c, but XXH3 (new default) is faster on common + // hardware + table_options.checksum = kCRC32c; + // Thrown in here for basic coverage: + options.DisableExtraChecks(); + break; + } + case kFIFOCompaction: { + options.compaction_style = kCompactionStyleFIFO; + options.max_open_files = -1; + break; + } + case kBlockBasedTableWithPrefixHashIndex: { + table_options.index_type = BlockBasedTableOptions::kHashSearch; + options.prefix_extractor.reset(NewFixedPrefixTransform(1)); + break; + } + case kBlockBasedTableWithWholeKeyHashIndex: { + table_options.index_type = BlockBasedTableOptions::kHashSearch; + options.prefix_extractor.reset(NewNoopTransform()); + break; + } + case kBlockBasedTableWithPartitionedIndex: { + table_options.format_version = 3; + table_options.index_type = BlockBasedTableOptions::kTwoLevelIndexSearch; + options.prefix_extractor.reset(NewNoopTransform()); + break; + } + case kBlockBasedTableWithPartitionedIndexFormat4: { + table_options.format_version = 4; + // Format 4 changes the binary index format. Since partitioned index is a + // super-set of simple indexes, we are also using kTwoLevelIndexSearch to + // test this format. + table_options.index_type = BlockBasedTableOptions::kTwoLevelIndexSearch; + // The top-level index in partition filters are also affected by format 4. + table_options.filter_policy.reset(NewBloomFilterPolicy(10, false)); + table_options.partition_filters = true; + table_options.index_block_restart_interval = 8; + break; + } + case kBlockBasedTableWithIndexRestartInterval: { + table_options.index_block_restart_interval = 8; + break; + } + case kBlockBasedTableWithLatestFormat: { + // In case different from default + table_options.format_version = kLatestFormatVersion; + break; + } + case kOptimizeFiltersForHits: { + options.optimize_filters_for_hits = true; + set_block_based_table_factory = true; + break; + } + case kRowCache: { + options.row_cache = NewLRUCache(1024 * 1024); + break; + } + case kRecycleLogFiles: { + options.recycle_log_file_num = 2; + break; + } + case kLevelSubcompactions: { + options.max_subcompactions = 4; + break; + } + case kUniversalSubcompactions: { + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = 8; + options.max_subcompactions = 4; + break; + } + case kConcurrentSkipList: { + options.allow_concurrent_memtable_write = true; + options.enable_write_thread_adaptive_yield = true; + break; + } + case kPipelinedWrite: { + options.enable_pipelined_write = true; + break; + } + case kConcurrentWALWrites: { + // This options optimize 2PC commit path + options.two_write_queues = true; + options.manual_wal_flush = true; + break; + } + case kUnorderedWrite: { + options.allow_concurrent_memtable_write = false; + options.unordered_write = false; + break; + } + + default: + break; + } + + if (options_override.filter_policy) { + table_options.filter_policy = options_override.filter_policy; + table_options.partition_filters = options_override.partition_filters; + table_options.metadata_block_size = options_override.metadata_block_size; + } + if (set_block_based_table_factory) { + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + } + options.level_compaction_dynamic_level_bytes = + options_override.level_compaction_dynamic_level_bytes; + options.env = env_; + options.create_if_missing = true; + options.fail_if_options_file_error = true; + return options; +} + +void DBTestBase::CreateColumnFamilies(const std::vector& cfs, + const Options& options) { + ColumnFamilyOptions cf_opts(options); + size_t cfi = handles_.size(); + handles_.resize(cfi + cfs.size()); + for (auto cf : cfs) { + Status s = db_->CreateColumnFamily(cf_opts, cf, &handles_[cfi++]); + ASSERT_OK(s); + } +} + +void DBTestBase::CreateAndReopenWithCF(const std::vector& cfs, + const Options& options) { + CreateColumnFamilies(cfs, options); + std::vector cfs_plus_default = cfs; + cfs_plus_default.insert(cfs_plus_default.begin(), kDefaultColumnFamilyName); + ReopenWithColumnFamilies(cfs_plus_default, options); +} + +void DBTestBase::ReopenWithColumnFamilies(const std::vector& cfs, + const std::vector& options) { + ASSERT_OK(TryReopenWithColumnFamilies(cfs, options)); +} + +void DBTestBase::ReopenWithColumnFamilies(const std::vector& cfs, + const Options& options) { + ASSERT_OK(TryReopenWithColumnFamilies(cfs, options)); +} + +void DBTestBase::SetTimeElapseOnlySleepOnReopen(DBOptions* options) { + time_elapse_only_sleep_on_reopen_ = true; + + // Need to disable stats dumping and persisting which also use + // RepeatableThread, which uses InstrumentedCondVar::TimedWaitInternal. + // With time_elapse_only_sleep_, this can hang on some platforms (MacOS) + // because (a) on some platforms, pthread_cond_timedwait does not appear + // to release the lock for other threads to operate if the deadline time + // is already passed, and (b) TimedWait calls are currently a bad abstraction + // because the deadline parameter is usually computed from Env time, + // but is interpreted in real clock time. + options->stats_dump_period_sec = 0; + options->stats_persist_period_sec = 0; +} + +void DBTestBase::MaybeInstallTimeElapseOnlySleep(const DBOptions& options) { + if (time_elapse_only_sleep_on_reopen_) { + assert(options.env == env_ || + static_cast_with_check(options.env) + ->env_target() == env_); + assert(options.stats_dump_period_sec == 0); + assert(options.stats_persist_period_sec == 0); + // We cannot set these before destroying the last DB because they might + // cause a deadlock or similar without the appropriate options set in + // the DB. + env_->time_elapse_only_sleep_ = true; + env_->no_slowdown_ = true; + } else { + // Going back in same test run is not yet supported, so no + // reset in this case. + } +} + +Status DBTestBase::TryReopenWithColumnFamilies( + const std::vector& cfs, const std::vector& options) { + Close(); + EXPECT_EQ(cfs.size(), options.size()); + std::vector column_families; + for (size_t i = 0; i < cfs.size(); ++i) { + column_families.push_back(ColumnFamilyDescriptor(cfs[i], options[i])); + } + DBOptions db_opts = DBOptions(options[0]); + last_options_ = options[0]; + MaybeInstallTimeElapseOnlySleep(db_opts); + return DB::Open(db_opts, dbname_, column_families, &handles_, &db_); +} + +Status DBTestBase::TryReopenWithColumnFamilies( + const std::vector& cfs, const Options& options) { + Close(); + std::vector v_opts(cfs.size(), options); + return TryReopenWithColumnFamilies(cfs, v_opts); +} + +void DBTestBase::Reopen(const Options& options) { + ASSERT_OK(TryReopen(options)); +} + +void DBTestBase::Close() { + for (auto h : handles_) { + EXPECT_OK(db_->DestroyColumnFamilyHandle(h)); + } + handles_.clear(); + delete db_; + db_ = nullptr; +} + +void DBTestBase::DestroyAndReopen(const Options& options) { + // Destroy using last options + Destroy(last_options_); + Reopen(options); +} + +void DBTestBase::Destroy(const Options& options, bool delete_cf_paths) { + std::vector column_families; + if (delete_cf_paths) { + for (size_t i = 0; i < handles_.size(); ++i) { + ColumnFamilyDescriptor cfdescriptor; + handles_[i]->GetDescriptor(&cfdescriptor).PermitUncheckedError(); + column_families.push_back(cfdescriptor); + } + } + Close(); + ASSERT_OK(DestroyDB(dbname_, options, column_families)); +} + +Status DBTestBase::ReadOnlyReopen(const Options& options) { + MaybeInstallTimeElapseOnlySleep(options); + return DB::OpenForReadOnly(options, dbname_, &db_); +} + +Status DBTestBase::TryReopen(const Options& options) { + Close(); + last_options_.table_factory.reset(); + // Note: operator= is an unsafe approach here since it destructs + // std::shared_ptr in the same order of their creation, in contrast to + // destructors which destructs them in the opposite order of creation. One + // particular problem is that the cache destructor might invoke callback + // functions that use Option members such as statistics. To work around this + // problem, we manually call destructor of table_factory which eventually + // clears the block cache. + last_options_ = options; + MaybeInstallTimeElapseOnlySleep(options); + return DB::Open(options, dbname_, &db_); +} + +bool DBTestBase::IsDirectIOSupported() { + return test::IsDirectIOSupported(env_, dbname_); +} + +bool DBTestBase::IsMemoryMappedAccessSupported() const { + return (!encrypted_env_); +} + +Status DBTestBase::Flush(int cf) { + if (cf == 0) { + return db_->Flush(FlushOptions()); + } else { + return db_->Flush(FlushOptions(), handles_[cf]); + } +} + +Status DBTestBase::Flush(const std::vector& cf_ids) { + std::vector cfhs; + std::for_each(cf_ids.begin(), cf_ids.end(), + [&cfhs, this](int id) { cfhs.emplace_back(handles_[id]); }); + return db_->Flush(FlushOptions(), cfhs); +} + +Status DBTestBase::Put(const Slice& k, const Slice& v, WriteOptions wo) { + if (kMergePut == option_config_) { + return db_->Merge(wo, k, v); + } else { + return db_->Put(wo, k, v); + } +} + +Status DBTestBase::Put(int cf, const Slice& k, const Slice& v, + WriteOptions wo) { + if (kMergePut == option_config_) { + return db_->Merge(wo, handles_[cf], k, v); + } else { + return db_->Put(wo, handles_[cf], k, v); + } +} + +Status DBTestBase::Merge(const Slice& k, const Slice& v, WriteOptions wo) { + return db_->Merge(wo, k, v); +} + +Status DBTestBase::Merge(int cf, const Slice& k, const Slice& v, + WriteOptions wo) { + return db_->Merge(wo, handles_[cf], k, v); +} + +Status DBTestBase::Delete(const std::string& k) { + return db_->Delete(WriteOptions(), k); +} + +Status DBTestBase::Delete(int cf, const std::string& k) { + return db_->Delete(WriteOptions(), handles_[cf], k); +} + +Status DBTestBase::SingleDelete(const std::string& k) { + return db_->SingleDelete(WriteOptions(), k); +} + +Status DBTestBase::SingleDelete(int cf, const std::string& k) { + return db_->SingleDelete(WriteOptions(), handles_[cf], k); +} + +std::string DBTestBase::Get(const std::string& k, const Snapshot* snapshot) { + ReadOptions options; + options.verify_checksums = true; + options.snapshot = snapshot; + std::string result; + Status s = db_->Get(options, k, &result); + if (s.IsNotFound()) { + result = "NOT_FOUND"; + } else if (!s.ok()) { + result = s.ToString(); + } + return result; +} + +std::string DBTestBase::Get(int cf, const std::string& k, + const Snapshot* snapshot) { + ReadOptions options; + options.verify_checksums = true; + options.snapshot = snapshot; + std::string result; + Status s = db_->Get(options, handles_[cf], k, &result); + if (s.IsNotFound()) { + result = "NOT_FOUND"; + } else if (!s.ok()) { + result = s.ToString(); + } + return result; +} + +std::vector DBTestBase::MultiGet(std::vector cfs, + const std::vector& k, + const Snapshot* snapshot, + const bool batched, + const bool async) { + ReadOptions options; + options.verify_checksums = true; + options.snapshot = snapshot; + options.async_io = async; + std::vector handles; + std::vector keys; + std::vector result; + + for (unsigned int i = 0; i < cfs.size(); ++i) { + handles.push_back(handles_[cfs[i]]); + keys.push_back(k[i]); + } + std::vector s; + if (!batched) { + s = db_->MultiGet(options, handles, keys, &result); + for (size_t i = 0; i < s.size(); ++i) { + if (s[i].IsNotFound()) { + result[i] = "NOT_FOUND"; + } else if (!s[i].ok()) { + result[i] = s[i].ToString(); + } + } + } else { + std::vector pin_values(cfs.size()); + result.resize(cfs.size()); + s.resize(cfs.size()); + db_->MultiGet(options, cfs.size(), handles.data(), keys.data(), + pin_values.data(), s.data()); + for (size_t i = 0; i < s.size(); ++i) { + if (s[i].IsNotFound()) { + result[i] = "NOT_FOUND"; + } else if (!s[i].ok()) { + result[i] = s[i].ToString(); + } else { + result[i].assign(pin_values[i].data(), pin_values[i].size()); + // Increase likelihood of detecting potential use-after-free bugs with + // PinnableSlices tracking the same resource + pin_values[i].Reset(); + } + } + } + return result; +} + +std::vector DBTestBase::MultiGet(const std::vector& k, + const Snapshot* snapshot, + const bool async) { + ReadOptions options; + options.verify_checksums = true; + options.snapshot = snapshot; + options.async_io = async; + std::vector keys; + std::vector result(k.size()); + std::vector statuses(k.size()); + std::vector pin_values(k.size()); + + for (size_t i = 0; i < k.size(); ++i) { + keys.push_back(k[i]); + } + db_->MultiGet(options, dbfull()->DefaultColumnFamily(), keys.size(), + keys.data(), pin_values.data(), statuses.data()); + for (size_t i = 0; i < statuses.size(); ++i) { + if (statuses[i].IsNotFound()) { + result[i] = "NOT_FOUND"; + } else if (!statuses[i].ok()) { + result[i] = statuses[i].ToString(); + } else { + result[i].assign(pin_values[i].data(), pin_values[i].size()); + // Increase likelihood of detecting potential use-after-free bugs with + // PinnableSlices tracking the same resource + pin_values[i].Reset(); + } + } + return result; +} + +Status DBTestBase::Get(const std::string& k, PinnableSlice* v) { + ReadOptions options; + options.verify_checksums = true; + Status s = dbfull()->Get(options, dbfull()->DefaultColumnFamily(), k, v); + return s; +} + +uint64_t DBTestBase::GetNumSnapshots() { + uint64_t int_num; + EXPECT_TRUE(dbfull()->GetIntProperty("rocksdb.num-snapshots", &int_num)); + return int_num; +} + +uint64_t DBTestBase::GetTimeOldestSnapshots() { + uint64_t int_num; + EXPECT_TRUE( + dbfull()->GetIntProperty("rocksdb.oldest-snapshot-time", &int_num)); + return int_num; +} + +uint64_t DBTestBase::GetSequenceOldestSnapshots() { + uint64_t int_num; + EXPECT_TRUE( + dbfull()->GetIntProperty("rocksdb.oldest-snapshot-sequence", &int_num)); + return int_num; +} + +// Return a string that contains all key,value pairs in order, +// formatted like "(k1->v1)(k2->v2)". +std::string DBTestBase::Contents(int cf) { + std::vector forward; + std::string result; + Iterator* iter = (cf == 0) ? db_->NewIterator(ReadOptions()) + : db_->NewIterator(ReadOptions(), handles_[cf]); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + std::string s = IterStatus(iter); + result.push_back('('); + result.append(s); + result.push_back(')'); + forward.push_back(s); + } + + // Check reverse iteration results are the reverse of forward results + unsigned int matched = 0; + for (iter->SeekToLast(); iter->Valid(); iter->Prev()) { + EXPECT_LT(matched, forward.size()); + EXPECT_EQ(IterStatus(iter), forward[forward.size() - matched - 1]); + matched++; + } + EXPECT_EQ(matched, forward.size()); + + delete iter; + return result; +} + +void DBTestBase::CheckAllEntriesWithFifoReopen( + const std::string& expected_value, const Slice& user_key, int cf, + const std::vector& cfs, const Options& options) { + ASSERT_EQ(AllEntriesFor(user_key, cf), expected_value); + + std::vector cfs_plus_default = cfs; + cfs_plus_default.insert(cfs_plus_default.begin(), kDefaultColumnFamilyName); + + Options fifo_options(options); + fifo_options.compaction_style = kCompactionStyleFIFO; + fifo_options.max_open_files = -1; + fifo_options.disable_auto_compactions = true; + ASSERT_OK(TryReopenWithColumnFamilies(cfs_plus_default, fifo_options)); + ASSERT_EQ(AllEntriesFor(user_key, cf), expected_value); + + ASSERT_OK(TryReopenWithColumnFamilies(cfs_plus_default, options)); + ASSERT_EQ(AllEntriesFor(user_key, cf), expected_value); +} + +std::string DBTestBase::AllEntriesFor(const Slice& user_key, int cf) { + Arena arena; + auto options = CurrentOptions(); + InternalKeyComparator icmp(options.comparator); + ReadOptions read_options; + ScopedArenaIterator iter; + if (cf == 0) { + iter.set(dbfull()->NewInternalIterator(read_options, &arena, + kMaxSequenceNumber)); + } else { + iter.set(dbfull()->NewInternalIterator(read_options, &arena, + kMaxSequenceNumber, handles_[cf])); + } + InternalKey target(user_key, kMaxSequenceNumber, kTypeValue); + iter->Seek(target.Encode()); + std::string result; + if (!iter->status().ok()) { + result = iter->status().ToString(); + } else { + result = "[ "; + bool first = true; + while (iter->Valid()) { + ParsedInternalKey ikey(Slice(), 0, kTypeValue); + if (ParseInternalKey(iter->key(), &ikey, true /* log_err_key */) != + Status::OK()) { + result += "CORRUPTED"; + } else { + if (!last_options_.comparator->Equal(ikey.user_key, user_key)) { + break; + } + if (!first) { + result += ", "; + } + first = false; + switch (ikey.type) { + case kTypeValue: + result += iter->value().ToString(); + break; + case kTypeMerge: + // keep it the same as kTypeValue for testing kMergePut + result += iter->value().ToString(); + break; + case kTypeDeletion: + result += "DEL"; + break; + case kTypeSingleDeletion: + result += "SDEL"; + break; + default: + assert(false); + break; + } + } + iter->Next(); + } + if (!first) { + result += " "; + } + result += "]"; + } + return result; +} + +int DBTestBase::NumSortedRuns(int cf) { + ColumnFamilyMetaData cf_meta; + if (cf == 0) { + db_->GetColumnFamilyMetaData(&cf_meta); + } else { + db_->GetColumnFamilyMetaData(handles_[cf], &cf_meta); + } + int num_sr = static_cast(cf_meta.levels[0].files.size()); + for (size_t i = 1U; i < cf_meta.levels.size(); i++) { + if (cf_meta.levels[i].files.size() > 0) { + num_sr++; + } + } + return num_sr; +} + +uint64_t DBTestBase::TotalSize(int cf) { + ColumnFamilyMetaData cf_meta; + if (cf == 0) { + db_->GetColumnFamilyMetaData(&cf_meta); + } else { + db_->GetColumnFamilyMetaData(handles_[cf], &cf_meta); + } + return cf_meta.size; +} + +uint64_t DBTestBase::SizeAtLevel(int level) { + std::vector metadata; + db_->GetLiveFilesMetaData(&metadata); + uint64_t sum = 0; + for (const auto& m : metadata) { + if (m.level == level) { + sum += m.size; + } + } + return sum; +} + +size_t DBTestBase::TotalLiveFiles(int cf) { + ColumnFamilyMetaData cf_meta; + if (cf == 0) { + db_->GetColumnFamilyMetaData(&cf_meta); + } else { + db_->GetColumnFamilyMetaData(handles_[cf], &cf_meta); + } + size_t num_files = 0; + for (auto& level : cf_meta.levels) { + num_files += level.files.size(); + } + return num_files; +} + +size_t DBTestBase::TotalLiveFilesAtPath(int cf, const std::string& path) { + ColumnFamilyMetaData cf_meta; + if (cf == 0) { + db_->GetColumnFamilyMetaData(&cf_meta); + } else { + db_->GetColumnFamilyMetaData(handles_[cf], &cf_meta); + } + size_t num_files = 0; + for (auto& level : cf_meta.levels) { + for (auto& f : level.files) { + if (f.directory == path) { + num_files++; + } + } + } + return num_files; +} + +size_t DBTestBase::CountLiveFiles() { + std::vector metadata; + db_->GetLiveFilesMetaData(&metadata); + return metadata.size(); +} + +int DBTestBase::NumTableFilesAtLevel(int level, int cf) { + std::string property; + if (cf == 0) { + // default cfd + EXPECT_TRUE(db_->GetProperty( + "rocksdb.num-files-at-level" + std::to_string(level), &property)); + } else { + EXPECT_TRUE(db_->GetProperty( + handles_[cf], "rocksdb.num-files-at-level" + std::to_string(level), + &property)); + } + return atoi(property.c_str()); +} + +double DBTestBase::CompressionRatioAtLevel(int level, int cf) { + std::string property; + if (cf == 0) { + // default cfd + EXPECT_TRUE(db_->GetProperty( + "rocksdb.compression-ratio-at-level" + std::to_string(level), + &property)); + } else { + EXPECT_TRUE(db_->GetProperty( + handles_[cf], + "rocksdb.compression-ratio-at-level" + std::to_string(level), + &property)); + } + return std::stod(property); +} + +int DBTestBase::TotalTableFiles(int cf, int levels) { + if (levels == -1) { + levels = (cf == 0) ? db_->NumberLevels() : db_->NumberLevels(handles_[1]); + } + int result = 0; + for (int level = 0; level < levels; level++) { + result += NumTableFilesAtLevel(level, cf); + } + return result; +} + +// Return spread of files per level +std::string DBTestBase::FilesPerLevel(int cf) { + int num_levels = + (cf == 0) ? db_->NumberLevels() : db_->NumberLevels(handles_[1]); + std::string result; + size_t last_non_zero_offset = 0; + for (int level = 0; level < num_levels; level++) { + int f = NumTableFilesAtLevel(level, cf); + char buf[100]; + snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f); + result += buf; + if (f > 0) { + last_non_zero_offset = result.size(); + } + } + result.resize(last_non_zero_offset); + return result; +} + + +std::vector DBTestBase::GetBlobFileNumbers() { + VersionSet* const versions = dbfull()->GetVersionSet(); + assert(versions); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + assert(cfd); + + Version* const current = cfd->current(); + assert(current); + + const VersionStorageInfo* const storage_info = current->storage_info(); + assert(storage_info); + + const auto& blob_files = storage_info->GetBlobFiles(); + + std::vector result; + result.reserve(blob_files.size()); + + for (const auto& blob_file : blob_files) { + assert(blob_file); + result.emplace_back(blob_file->GetBlobFileNumber()); + } + + return result; +} + +size_t DBTestBase::CountFiles() { + size_t count = 0; + std::vector files; + if (env_->GetChildren(dbname_, &files).ok()) { + count += files.size(); + } + + if (dbname_ != last_options_.wal_dir) { + if (env_->GetChildren(last_options_.wal_dir, &files).ok()) { + count += files.size(); + } + } + + return count; +}; + +Status DBTestBase::CountFiles(size_t* count) { + std::vector files; + Status s = env_->GetChildren(dbname_, &files); + if (!s.ok()) { + return s; + } + size_t files_count = files.size(); + + if (dbname_ != last_options_.wal_dir) { + s = env_->GetChildren(last_options_.wal_dir, &files); + if (!s.ok()) { + return s; + } + *count = files_count + files.size(); + } + + return Status::OK(); +} + +Status DBTestBase::Size(const Slice& start, const Slice& limit, int cf, + uint64_t* size) { + Range r(start, limit); + if (cf == 0) { + return db_->GetApproximateSizes(&r, 1, size); + } else { + return db_->GetApproximateSizes(handles_[1], &r, 1, size); + } +} + +void DBTestBase::Compact(int cf, const Slice& start, const Slice& limit, + uint32_t target_path_id) { + CompactRangeOptions compact_options; + compact_options.target_path_id = target_path_id; + ASSERT_OK(db_->CompactRange(compact_options, handles_[cf], &start, &limit)); +} + +void DBTestBase::Compact(int cf, const Slice& start, const Slice& limit) { + ASSERT_OK( + db_->CompactRange(CompactRangeOptions(), handles_[cf], &start, &limit)); +} + +void DBTestBase::Compact(const Slice& start, const Slice& limit) { + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &start, &limit)); +} + +// Do n memtable compactions, each of which produces an sstable +// covering the range [small,large]. +void DBTestBase::MakeTables(int n, const std::string& small, + const std::string& large, int cf) { + for (int i = 0; i < n; i++) { + ASSERT_OK(Put(cf, small, "begin")); + ASSERT_OK(Put(cf, large, "end")); + ASSERT_OK(Flush(cf)); + MoveFilesToLevel(n - i - 1, cf); + } +} + +// Prevent pushing of new sstables into deeper levels by adding +// tables that cover a specified range to all levels. +void DBTestBase::FillLevels(const std::string& smallest, + const std::string& largest, int cf) { + MakeTables(db_->NumberLevels(handles_[cf]), smallest, largest, cf); +} + +void DBTestBase::MoveFilesToLevel(int level, int cf) { + for (int l = 0; l < level; ++l) { + if (cf > 0) { + EXPECT_OK(dbfull()->TEST_CompactRange(l, nullptr, nullptr, handles_[cf])); + } else { + EXPECT_OK(dbfull()->TEST_CompactRange(l, nullptr, nullptr)); + } + } +} + +void DBTestBase::DumpFileCounts(const char* label) { + fprintf(stderr, "---\n%s:\n", label); + fprintf(stderr, "maxoverlap: %" PRIu64 "\n", + dbfull()->TEST_MaxNextLevelOverlappingBytes()); + for (int level = 0; level < db_->NumberLevels(); level++) { + int num = NumTableFilesAtLevel(level); + if (num > 0) { + fprintf(stderr, " level %3d : %d files\n", level, num); + } + } +} + +std::string DBTestBase::DumpSSTableList() { + std::string property; + db_->GetProperty("rocksdb.sstables", &property); + return property; +} + +void DBTestBase::GetSstFiles(Env* env, std::string path, + std::vector* files) { + EXPECT_OK(env->GetChildren(path, files)); + + files->erase(std::remove_if(files->begin(), files->end(), + [](std::string name) { + uint64_t number; + FileType type; + return !(ParseFileName(name, &number, &type) && + type == kTableFile); + }), + files->end()); +} + +int DBTestBase::GetSstFileCount(std::string path) { + std::vector files; + DBTestBase::GetSstFiles(env_, path, &files); + return static_cast(files.size()); +} + +// this will generate non-overlapping files since it keeps increasing key_idx +void DBTestBase::GenerateNewFile(int cf, Random* rnd, int* key_idx, + bool nowait) { + for (int i = 0; i < KNumKeysByGenerateNewFile; i++) { + ASSERT_OK(Put(cf, Key(*key_idx), rnd->RandomString((i == 99) ? 1 : 990))); + (*key_idx)++; + } + if (!nowait) { + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } +} + +// this will generate non-overlapping files since it keeps increasing key_idx +void DBTestBase::GenerateNewFile(Random* rnd, int* key_idx, bool nowait) { + for (int i = 0; i < KNumKeysByGenerateNewFile; i++) { + ASSERT_OK(Put(Key(*key_idx), rnd->RandomString((i == 99) ? 1 : 990))); + (*key_idx)++; + } + if (!nowait) { + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } +} + +const int DBTestBase::kNumKeysByGenerateNewRandomFile = 51; + +void DBTestBase::GenerateNewRandomFile(Random* rnd, bool nowait) { + for (int i = 0; i < kNumKeysByGenerateNewRandomFile; i++) { + ASSERT_OK(Put("key" + rnd->RandomString(7), rnd->RandomString(2000))); + } + ASSERT_OK(Put("key" + rnd->RandomString(7), rnd->RandomString(200))); + if (!nowait) { + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } +} + +std::string DBTestBase::IterStatus(Iterator* iter) { + std::string result; + if (iter->Valid()) { + result = iter->key().ToString() + "->" + iter->value().ToString(); + } else { + result = "(invalid)"; + } + return result; +} + +Options DBTestBase::OptionsForLogIterTest() { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.WAL_ttl_seconds = 1000; + return options; +} + +std::string DBTestBase::DummyString(size_t len, char c) { + return std::string(len, c); +} + +void DBTestBase::VerifyIterLast(std::string expected_key, int cf) { + Iterator* iter; + ReadOptions ro; + if (cf == 0) { + iter = db_->NewIterator(ro); + } else { + iter = db_->NewIterator(ro, handles_[cf]); + } + iter->SeekToLast(); + ASSERT_EQ(IterStatus(iter), expected_key); + delete iter; +} + +// Used to test InplaceUpdate + +// If previous value is nullptr or delta is > than previous value, +// sets newValue with delta +// If previous value is not empty, +// updates previous value with 'b' string of previous value size - 1. +UpdateStatus DBTestBase::updateInPlaceSmallerSize(char* prevValue, + uint32_t* prevSize, + Slice delta, + std::string* newValue) { + if (prevValue == nullptr) { + *newValue = std::string(delta.size(), 'c'); + return UpdateStatus::UPDATED; + } else { + *prevSize = *prevSize - 1; + std::string str_b = std::string(*prevSize, 'b'); + memcpy(prevValue, str_b.c_str(), str_b.size()); + return UpdateStatus::UPDATED_INPLACE; + } +} + +UpdateStatus DBTestBase::updateInPlaceSmallerVarintSize(char* prevValue, + uint32_t* prevSize, + Slice delta, + std::string* newValue) { + if (prevValue == nullptr) { + *newValue = std::string(delta.size(), 'c'); + return UpdateStatus::UPDATED; + } else { + *prevSize = 1; + std::string str_b = std::string(*prevSize, 'b'); + memcpy(prevValue, str_b.c_str(), str_b.size()); + return UpdateStatus::UPDATED_INPLACE; + } +} + +UpdateStatus DBTestBase::updateInPlaceLargerSize(char* /*prevValue*/, + uint32_t* /*prevSize*/, + Slice delta, + std::string* newValue) { + *newValue = std::string(delta.size(), 'c'); + return UpdateStatus::UPDATED; +} + +UpdateStatus DBTestBase::updateInPlaceNoAction(char* /*prevValue*/, + uint32_t* /*prevSize*/, + Slice /*delta*/, + std::string* /*newValue*/) { + return UpdateStatus::UPDATE_FAILED; +} + +// Utility method to test InplaceUpdate +void DBTestBase::validateNumberOfEntries(int numValues, int cf) { + Arena arena; + auto options = CurrentOptions(); + InternalKeyComparator icmp(options.comparator); + ReadOptions read_options; + ScopedArenaIterator iter; + if (cf != 0) { + iter.set(dbfull()->NewInternalIterator(read_options, &arena, + kMaxSequenceNumber, handles_[cf])); + } else { + iter.set(dbfull()->NewInternalIterator(read_options, &arena, + kMaxSequenceNumber)); + } + iter->SeekToFirst(); + ASSERT_OK(iter->status()); + int seq = numValues; + while (iter->Valid()) { + ParsedInternalKey ikey; + ikey.clear(); + ASSERT_OK(ParseInternalKey(iter->key(), &ikey, true /* log_err_key */)); + + // checks sequence number for updates + ASSERT_EQ(ikey.sequence, (unsigned)seq--); + iter->Next(); + } + ASSERT_EQ(0, seq); +} + +void DBTestBase::CopyFile(const std::string& source, + const std::string& destination, uint64_t size) { + const EnvOptions soptions; + std::unique_ptr srcfile; + ASSERT_OK(env_->NewSequentialFile(source, &srcfile, soptions)); + std::unique_ptr destfile; + ASSERT_OK(env_->NewWritableFile(destination, &destfile, soptions)); + + if (size == 0) { + // default argument means copy everything + ASSERT_OK(env_->GetFileSize(source, &size)); + } + + char buffer[4096]; + Slice slice; + while (size > 0) { + uint64_t one = std::min(uint64_t(sizeof(buffer)), size); + ASSERT_OK(srcfile->Read(one, &slice, buffer)); + ASSERT_OK(destfile->Append(slice)); + size -= slice.size(); + } + ASSERT_OK(destfile->Close()); +} + +Status DBTestBase::GetAllDataFiles( + const FileType file_type, std::unordered_map* files, + uint64_t* total_size /* = nullptr */) { + if (total_size) { + *total_size = 0; + } + std::vector children; + Status s = env_->GetChildren(dbname_, &children); + if (s.ok()) { + for (auto& file_name : children) { + uint64_t number; + FileType type; + if (ParseFileName(file_name, &number, &type) && type == file_type) { + std::string file_path = dbname_ + "/" + file_name; + uint64_t file_size = 0; + s = env_->GetFileSize(file_path, &file_size); + if (!s.ok()) { + break; + } + (*files)[file_path] = file_size; + if (total_size) { + *total_size += file_size; + } + } + } + } + return s; +} + +std::vector DBTestBase::ListTableFiles(Env* env, + const std::string& path) { + std::vector files; + std::vector file_numbers; + EXPECT_OK(env->GetChildren(path, &files)); + uint64_t number; + FileType type; + for (size_t i = 0; i < files.size(); ++i) { + if (ParseFileName(files[i], &number, &type)) { + if (type == kTableFile) { + file_numbers.push_back(number); + } + } + } + return file_numbers; +} + +void DBTestBase::VerifyDBFromMap(std::map true_data, + size_t* total_reads_res, bool tailing_iter, + std::map status) { + size_t total_reads = 0; + + for (auto& kv : true_data) { + Status s = status[kv.first]; + if (s.ok()) { + ASSERT_EQ(Get(kv.first), kv.second); + } else { + std::string value; + ASSERT_EQ(s, db_->Get(ReadOptions(), kv.first, &value)); + } + total_reads++; + } + + // Normal Iterator + { + int iter_cnt = 0; + ReadOptions ro; + ro.total_order_seek = true; + Iterator* iter = db_->NewIterator(ro); + // Verify Iterator::Next() + iter_cnt = 0; + auto data_iter = true_data.begin(); + Status s; + for (iter->SeekToFirst(); iter->Valid(); iter->Next(), data_iter++) { + ASSERT_EQ(iter->key().ToString(), data_iter->first); + Status current_status = status[data_iter->first]; + if (!current_status.ok()) { + s = current_status; + } + ASSERT_EQ(iter->status(), s); + if (current_status.ok()) { + ASSERT_EQ(iter->value().ToString(), data_iter->second); + } + iter_cnt++; + total_reads++; + } + ASSERT_EQ(data_iter, true_data.end()) + << iter_cnt << " / " << true_data.size(); + delete iter; + + // Verify Iterator::Prev() + // Use a new iterator to make sure its status is clean. + iter = db_->NewIterator(ro); + iter_cnt = 0; + s = Status::OK(); + auto data_rev = true_data.rbegin(); + for (iter->SeekToLast(); iter->Valid(); iter->Prev(), data_rev++) { + ASSERT_EQ(iter->key().ToString(), data_rev->first); + Status current_status = status[data_rev->first]; + if (!current_status.ok()) { + s = current_status; + } + ASSERT_EQ(iter->status(), s); + if (current_status.ok()) { + ASSERT_EQ(iter->value().ToString(), data_rev->second); + } + iter_cnt++; + total_reads++; + } + ASSERT_EQ(data_rev, true_data.rend()) + << iter_cnt << " / " << true_data.size(); + + // Verify Iterator::Seek() + for (auto kv : true_data) { + iter->Seek(kv.first); + ASSERT_EQ(kv.first, iter->key().ToString()); + ASSERT_EQ(kv.second, iter->value().ToString()); + total_reads++; + } + delete iter; + } + + if (tailing_iter) { + // Tailing iterator + int iter_cnt = 0; + ReadOptions ro; + ro.tailing = true; + ro.total_order_seek = true; + Iterator* iter = db_->NewIterator(ro); + + // Verify ForwardIterator::Next() + iter_cnt = 0; + auto data_iter = true_data.begin(); + for (iter->SeekToFirst(); iter->Valid(); iter->Next(), data_iter++) { + ASSERT_EQ(iter->key().ToString(), data_iter->first); + ASSERT_EQ(iter->value().ToString(), data_iter->second); + iter_cnt++; + total_reads++; + } + ASSERT_EQ(data_iter, true_data.end()) + << iter_cnt << " / " << true_data.size(); + + // Verify ForwardIterator::Seek() + for (auto kv : true_data) { + iter->Seek(kv.first); + ASSERT_EQ(kv.first, iter->key().ToString()); + ASSERT_EQ(kv.second, iter->value().ToString()); + total_reads++; + } + + delete iter; + } + + if (total_reads_res) { + *total_reads_res = total_reads; + } +} + +void DBTestBase::VerifyDBInternal( + std::vector> true_data) { + Arena arena; + InternalKeyComparator icmp(last_options_.comparator); + ReadOptions read_options; + auto iter = + dbfull()->NewInternalIterator(read_options, &arena, kMaxSequenceNumber); + iter->SeekToFirst(); + for (auto p : true_data) { + ASSERT_TRUE(iter->Valid()); + ParsedInternalKey ikey; + ASSERT_OK(ParseInternalKey(iter->key(), &ikey, true /* log_err_key */)); + ASSERT_EQ(p.first, ikey.user_key); + ASSERT_EQ(p.second, iter->value()); + iter->Next(); + }; + ASSERT_FALSE(iter->Valid()); + iter->~InternalIterator(); +} + + +uint64_t DBTestBase::GetNumberOfSstFilesForColumnFamily( + DB* db, std::string column_family_name) { + std::vector metadata; + db->GetLiveFilesMetaData(&metadata); + uint64_t result = 0; + for (auto& fileMetadata : metadata) { + result += (fileMetadata.column_family_name == column_family_name); + } + return result; +} + +uint64_t DBTestBase::GetSstSizeHelper(Temperature temperature) { + std::string prop; + EXPECT_TRUE(dbfull()->GetProperty( + DB::Properties::kLiveSstFilesSizeAtTemperature + + std::to_string(static_cast(temperature)), + &prop)); + return static_cast(std::atoi(prop.c_str())); +} + +void VerifySstUniqueIds(const TablePropertiesCollection& props) { + ASSERT_FALSE(props.empty()); // suspicious test if empty + std::unordered_set seen; + for (auto& pair : props) { + std::string id; + ASSERT_OK(GetUniqueIdFromTableProperties(*pair.second, &id)); + ASSERT_TRUE(seen.insert(id).second); + } +} + +template +TargetCacheChargeTrackingCache::TargetCacheChargeTrackingCache( + std::shared_ptr target) + : CacheWrapper(std::move(target)), + cur_cache_charge_(0), + cache_charge_peak_(0), + cache_charge_increment_(0), + last_peak_tracked_(false), + cache_charge_increments_sum_(0) {} + +template +Status TargetCacheChargeTrackingCache::Insert(const Slice& key, + ObjectPtr value, + const CacheItemHelper* helper, + size_t charge, Handle** handle, + Priority priority) { + Status s = target_->Insert(key, value, helper, charge, handle, priority); + if (helper == kCrmHelper) { + if (last_peak_tracked_) { + cache_charge_peak_ = 0; + cache_charge_increment_ = 0; + last_peak_tracked_ = false; + } + if (s.ok()) { + cur_cache_charge_ += charge; + } + cache_charge_peak_ = std::max(cache_charge_peak_, cur_cache_charge_); + cache_charge_increment_ += charge; + } + + return s; +} + +template +bool TargetCacheChargeTrackingCache::Release(Handle* handle, + bool erase_if_last_ref) { + auto helper = GetCacheItemHelper(handle); + if (helper == kCrmHelper) { + if (!last_peak_tracked_) { + cache_charge_peaks_.push_back(cache_charge_peak_); + cache_charge_increments_sum_ += cache_charge_increment_; + last_peak_tracked_ = true; + } + cur_cache_charge_ -= GetCharge(handle); + } + bool is_successful = target_->Release(handle, erase_if_last_ref); + return is_successful; +} + +template +const Cache::CacheItemHelper* TargetCacheChargeTrackingCache::kCrmHelper = + CacheReservationManagerImpl::TEST_GetCacheItemHelperForRole(); + +template class TargetCacheChargeTrackingCache< + CacheEntryRole::kFilterConstruction>; +template class TargetCacheChargeTrackingCache< + CacheEntryRole::kBlockBasedTableReader>; +template class TargetCacheChargeTrackingCache; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/db_test_util.h b/librocksdb-sys/rocksdb/db/db_test_util.h new file mode 100644 index 0000000..52e856c --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_test_util.h @@ -0,0 +1,1380 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "db/db_impl/db_impl.h" +#include "file/filename.h" +#include "rocksdb/advanced_options.h" +#include "rocksdb/cache.h" +#include "rocksdb/compaction_filter.h" +#include "rocksdb/convenience.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/file_system.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/io_status.h" +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "rocksdb/sst_file_writer.h" +#include "rocksdb/statistics.h" +#include "rocksdb/table.h" +#include "rocksdb/utilities/checkpoint.h" +#include "table/mock_table.h" +#include "table/scoped_arena_iterator.h" +#include "test_util/sync_point.h" +#include "test_util/testharness.h" +#include "util/cast_util.h" +#include "util/compression.h" +#include "util/mutexlock.h" +#include "util/string_util.h" +#include "utilities/merge_operators.h" + +// In case defined by Windows headers +#undef small + +namespace ROCKSDB_NAMESPACE { +class MockEnv; + +namespace anon { +class AtomicCounter { + public: + explicit AtomicCounter(Env* env = NULL) + : env_(env), cond_count_(&mu_), count_(0) {} + + void Increment() { + MutexLock l(&mu_); + count_++; + cond_count_.SignalAll(); + } + + int Read() { + MutexLock l(&mu_); + return count_; + } + + bool WaitFor(int count) { + MutexLock l(&mu_); + + uint64_t start = env_->NowMicros(); + while (count_ < count) { + uint64_t now = env_->NowMicros(); + cond_count_.TimedWait(now + /*1s*/ 1 * 1000 * 1000); + if (env_->NowMicros() - start > /*10s*/ 10 * 1000 * 1000) { + return false; + } + if (count_ < count) { + GTEST_LOG_(WARNING) << "WaitFor is taking more time than usual"; + } + } + + return true; + } + + void Reset() { + MutexLock l(&mu_); + count_ = 0; + cond_count_.SignalAll(); + } + + private: + Env* env_; + port::Mutex mu_; + port::CondVar cond_count_; + int count_; +}; + +struct OptionsOverride { + std::shared_ptr filter_policy = nullptr; + // These will be used only if filter_policy is set + bool partition_filters = false; + // Force using a default block cache. (Setting to false allows ASAN build + // use a trivially small block cache for better UAF error detection.) + bool full_block_cache = false; + uint64_t metadata_block_size = 1024; + + // Used as a bit mask of individual enums in which to skip an XF test point + int skip_policy = 0; + + // The default value for this option is changed from false to true. + // Keeping the default to false for unit tests as old unit tests assume + // this behavior. Tests for level_compaction_dynamic_level_bytes + // will set the option to true explicitly. + bool level_compaction_dynamic_level_bytes = false; +}; + +} // namespace anon + +enum SkipPolicy { kSkipNone = 0, kSkipNoSnapshot = 1, kSkipNoPrefix = 2 }; + +// Special Env used to delay background operations +class SpecialEnv : public EnvWrapper { + public: + explicit SpecialEnv(Env* base, bool time_elapse_only_sleep = false); + + static const char* kClassName() { return "SpecialEnv"; } + const char* Name() const override { return kClassName(); } + + Status NewWritableFile(const std::string& f, std::unique_ptr* r, + const EnvOptions& soptions) override { + class SSTableFile : public WritableFile { + private: + SpecialEnv* env_; + std::unique_ptr base_; + + public: + SSTableFile(SpecialEnv* env, std::unique_ptr&& base) + : env_(env), base_(std::move(base)) {} + Status Append(const Slice& data) override { + if (env_->table_write_callback_) { + (*env_->table_write_callback_)(); + } + if (env_->drop_writes_.load(std::memory_order_acquire)) { + // Drop writes on the floor + return Status::OK(); + } else if (env_->no_space_.load(std::memory_order_acquire)) { + return Status::NoSpace("No space left on device"); + } else { + env_->bytes_written_ += data.size(); + return base_->Append(data); + } + } + Status Append( + const Slice& data, + const DataVerificationInfo& /* verification_info */) override { + return Append(data); + } + Status PositionedAppend(const Slice& data, uint64_t offset) override { + if (env_->table_write_callback_) { + (*env_->table_write_callback_)(); + } + if (env_->drop_writes_.load(std::memory_order_acquire)) { + // Drop writes on the floor + return Status::OK(); + } else if (env_->no_space_.load(std::memory_order_acquire)) { + return Status::NoSpace("No space left on device"); + } else { + env_->bytes_written_ += data.size(); + return base_->PositionedAppend(data, offset); + } + } + Status PositionedAppend( + const Slice& data, uint64_t offset, + const DataVerificationInfo& /* verification_info */) override { + return PositionedAppend(data, offset); + } + Status Truncate(uint64_t size) override { return base_->Truncate(size); } + Status RangeSync(uint64_t offset, uint64_t nbytes) override { + Status s = base_->RangeSync(offset, nbytes); +#if !(defined NDEBUG) || !defined(OS_WIN) + TEST_SYNC_POINT_CALLBACK("SpecialEnv::SStableFile::RangeSync", &s); +#endif // !(defined NDEBUG) || !defined(OS_WIN) + return s; + } + Status Close() override { +// SyncPoint is not supported in Released Windows Mode. +#if !(defined NDEBUG) || !defined(OS_WIN) + // Check preallocation size + // preallocation size is never passed to base file. + size_t preallocation_size = preallocation_block_size(); + TEST_SYNC_POINT_CALLBACK("DBTestWritableFile.GetPreallocationStatus", + &preallocation_size); +#endif // !(defined NDEBUG) || !defined(OS_WIN) + Status s = base_->Close(); +#if !(defined NDEBUG) || !defined(OS_WIN) + TEST_SYNC_POINT_CALLBACK("SpecialEnv::SStableFile::Close", &s); +#endif // !(defined NDEBUG) || !defined(OS_WIN) + return s; + } + Status Flush() override { return base_->Flush(); } + Status Sync() override { + ++env_->sync_counter_; + while (env_->delay_sstable_sync_.load(std::memory_order_acquire)) { + env_->SleepForMicroseconds(100000); + } + Status s; + if (!env_->skip_fsync_) { + s = base_->Sync(); + } +#if !(defined NDEBUG) || !defined(OS_WIN) + TEST_SYNC_POINT_CALLBACK("SpecialEnv::SStableFile::Sync", &s); +#endif // !(defined NDEBUG) || !defined(OS_WIN) + return s; + } + void SetIOPriority(Env::IOPriority pri) override { + base_->SetIOPriority(pri); + } + Env::IOPriority GetIOPriority() override { + return base_->GetIOPriority(); + } + bool use_direct_io() const override { return base_->use_direct_io(); } + Status Allocate(uint64_t offset, uint64_t len) override { + return base_->Allocate(offset, len); + } + size_t GetUniqueId(char* id, size_t max_size) const override { + return base_->GetUniqueId(id, max_size); + } + }; + class ManifestFile : public WritableFile { + public: + ManifestFile(SpecialEnv* env, std::unique_ptr&& b) + : env_(env), base_(std::move(b)) {} + Status Append(const Slice& data) override { + if (env_->manifest_write_error_.load(std::memory_order_acquire)) { + return Status::IOError("simulated writer error"); + } else { + return base_->Append(data); + } + } + Status Append( + const Slice& data, + const DataVerificationInfo& /*verification_info*/) override { + return Append(data); + } + + Status Truncate(uint64_t size) override { return base_->Truncate(size); } + Status Close() override { return base_->Close(); } + Status Flush() override { return base_->Flush(); } + Status Sync() override { + ++env_->sync_counter_; + if (env_->manifest_sync_error_.load(std::memory_order_acquire)) { + return Status::IOError("simulated sync error"); + } else { + if (env_->skip_fsync_) { + return Status::OK(); + } else { + return base_->Sync(); + } + } + } + uint64_t GetFileSize() override { return base_->GetFileSize(); } + Status Allocate(uint64_t offset, uint64_t len) override { + return base_->Allocate(offset, len); + } + + private: + SpecialEnv* env_; + std::unique_ptr base_; + }; + class WalFile : public WritableFile { + public: + WalFile(SpecialEnv* env, std::unique_ptr&& b) + : env_(env), base_(std::move(b)) { + env_->num_open_wal_file_.fetch_add(1); + } + virtual ~WalFile() { env_->num_open_wal_file_.fetch_add(-1); } + Status Append(const Slice& data) override { +#if !(defined NDEBUG) || !defined(OS_WIN) + TEST_SYNC_POINT("SpecialEnv::WalFile::Append:1"); +#endif + Status s; + if (env_->log_write_error_.load(std::memory_order_acquire)) { + s = Status::IOError("simulated writer error"); + } else { + int slowdown = + env_->log_write_slowdown_.load(std::memory_order_acquire); + if (slowdown > 0) { + env_->SleepForMicroseconds(slowdown); + } + s = base_->Append(data); + } +#if !(defined NDEBUG) || !defined(OS_WIN) + TEST_SYNC_POINT("SpecialEnv::WalFile::Append:2"); +#endif + return s; + } + Status Append( + const Slice& data, + const DataVerificationInfo& /* verification_info */) override { + return Append(data); + } + Status Truncate(uint64_t size) override { return base_->Truncate(size); } + void PrepareWrite(size_t offset, size_t len) override { + base_->PrepareWrite(offset, len); + } + void SetPreallocationBlockSize(size_t size) override { + base_->SetPreallocationBlockSize(size); + } + Status Close() override { +// SyncPoint is not supported in Released Windows Mode. +#if !(defined NDEBUG) || !defined(OS_WIN) + // Check preallocation size + size_t block_size, last_allocated_block; + base_->GetPreallocationStatus(&block_size, &last_allocated_block); + TEST_SYNC_POINT_CALLBACK("DBTestWalFile.GetPreallocationStatus", + &block_size); +#endif // !(defined NDEBUG) || !defined(OS_WIN) + + return base_->Close(); + } + Status Flush() override { return base_->Flush(); } + Status Sync() override { + ++env_->sync_counter_; + if (env_->corrupt_in_sync_) { + EXPECT_OK(Append(std::string(33000, ' '))); + return Status::IOError("Ingested Sync Failure"); + } + if (env_->skip_fsync_) { + return Status::OK(); + } else { + return base_->Sync(); + } + } + bool IsSyncThreadSafe() const override { + return env_->is_wal_sync_thread_safe_.load(); + } + Status Allocate(uint64_t offset, uint64_t len) override { + return base_->Allocate(offset, len); + } + + private: + SpecialEnv* env_; + std::unique_ptr base_; + }; + class OtherFile : public WritableFile { + public: + OtherFile(SpecialEnv* env, std::unique_ptr&& b) + : env_(env), base_(std::move(b)) {} + Status Append(const Slice& data) override { return base_->Append(data); } + Status Append( + const Slice& data, + const DataVerificationInfo& /*verification_info*/) override { + return Append(data); + } + Status Truncate(uint64_t size) override { return base_->Truncate(size); } + Status Close() override { return base_->Close(); } + Status Flush() override { return base_->Flush(); } + Status Sync() override { + if (env_->skip_fsync_) { + return Status::OK(); + } else { + return base_->Sync(); + } + } + uint64_t GetFileSize() override { return base_->GetFileSize(); } + Status Allocate(uint64_t offset, uint64_t len) override { + return base_->Allocate(offset, len); + } + + private: + SpecialEnv* env_; + std::unique_ptr base_; + }; + + if (no_file_overwrite_.load(std::memory_order_acquire) && + target()->FileExists(f).ok()) { + return Status::NotSupported("SpecialEnv::no_file_overwrite_ is true."); + } + + if (non_writeable_rate_.load(std::memory_order_acquire) > 0) { + uint32_t random_number; + { + MutexLock l(&rnd_mutex_); + random_number = rnd_.Uniform(100); + } + if (random_number < non_writeable_rate_.load()) { + return Status::IOError("simulated random write error"); + } + } + + new_writable_count_++; + + if (non_writable_count_.load() > 0) { + non_writable_count_--; + return Status::IOError("simulated write error"); + } + + EnvOptions optimized = soptions; + if (strstr(f.c_str(), "MANIFEST") != nullptr || + strstr(f.c_str(), "log") != nullptr) { + optimized.use_mmap_writes = false; + optimized.use_direct_writes = false; + } + + Status s = target()->NewWritableFile(f, r, optimized); + if (s.ok()) { + if (strstr(f.c_str(), ".sst") != nullptr) { + r->reset(new SSTableFile(this, std::move(*r))); + } else if (strstr(f.c_str(), "MANIFEST") != nullptr) { + r->reset(new ManifestFile(this, std::move(*r))); + } else if (strstr(f.c_str(), "log") != nullptr) { + r->reset(new WalFile(this, std::move(*r))); + } else { + r->reset(new OtherFile(this, std::move(*r))); + } + } + return s; + } + + Status NewRandomAccessFile(const std::string& f, + std::unique_ptr* r, + const EnvOptions& soptions) override { + class CountingFile : public RandomAccessFile { + public: + CountingFile(std::unique_ptr&& target, + anon::AtomicCounter* counter, + std::atomic* bytes_read) + : target_(std::move(target)), + counter_(counter), + bytes_read_(bytes_read) {} + virtual Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const override { + counter_->Increment(); + Status s = target_->Read(offset, n, result, scratch); + *bytes_read_ += result->size(); + return s; + } + + virtual Status Prefetch(uint64_t offset, size_t n) override { + Status s = target_->Prefetch(offset, n); + *bytes_read_ += n; + return s; + } + + private: + std::unique_ptr target_; + anon::AtomicCounter* counter_; + std::atomic* bytes_read_; + }; + + class RandomFailureFile : public RandomAccessFile { + public: + RandomFailureFile(std::unique_ptr&& target, + std::atomic* failure_cnt, uint32_t fail_odd) + : target_(std::move(target)), + fail_cnt_(failure_cnt), + fail_odd_(fail_odd) {} + virtual Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const override { + if (Random::GetTLSInstance()->OneIn(fail_odd_)) { + fail_cnt_->fetch_add(1); + return Status::IOError("random error"); + } + return target_->Read(offset, n, result, scratch); + } + + virtual Status Prefetch(uint64_t offset, size_t n) override { + return target_->Prefetch(offset, n); + } + + private: + std::unique_ptr target_; + std::atomic* fail_cnt_; + uint32_t fail_odd_; + }; + + Status s = target()->NewRandomAccessFile(f, r, soptions); + random_file_open_counter_++; + if (s.ok()) { + if (count_random_reads_) { + r->reset(new CountingFile(std::move(*r), &random_read_counter_, + &random_read_bytes_counter_)); + } else if (rand_reads_fail_odd_ > 0) { + r->reset(new RandomFailureFile(std::move(*r), &num_reads_fails_, + rand_reads_fail_odd_)); + } + } + + if (s.ok() && soptions.compaction_readahead_size > 0) { + compaction_readahead_size_ = soptions.compaction_readahead_size; + } + return s; + } + + virtual Status NewSequentialFile(const std::string& f, + std::unique_ptr* r, + const EnvOptions& soptions) override { + class CountingFile : public SequentialFile { + public: + CountingFile(std::unique_ptr&& target, + anon::AtomicCounter* counter) + : target_(std::move(target)), counter_(counter) {} + virtual Status Read(size_t n, Slice* result, char* scratch) override { + counter_->Increment(); + return target_->Read(n, result, scratch); + } + virtual Status Skip(uint64_t n) override { return target_->Skip(n); } + + private: + std::unique_ptr target_; + anon::AtomicCounter* counter_; + }; + + Status s = target()->NewSequentialFile(f, r, soptions); + if (s.ok() && count_sequential_reads_) { + r->reset(new CountingFile(std::move(*r), &sequential_read_counter_)); + } + return s; + } + + virtual void SleepForMicroseconds(int micros) override { + sleep_counter_.Increment(); + if (no_slowdown_ || time_elapse_only_sleep_) { + addon_microseconds_.fetch_add(micros); + } + if (!no_slowdown_) { + target()->SleepForMicroseconds(micros); + } + } + + void MockSleepForMicroseconds(int64_t micros) { + sleep_counter_.Increment(); + assert(no_slowdown_); + addon_microseconds_.fetch_add(micros); + } + + void MockSleepForSeconds(int64_t seconds) { + sleep_counter_.Increment(); + assert(no_slowdown_); + addon_microseconds_.fetch_add(seconds * 1000000); + } + + virtual Status GetCurrentTime(int64_t* unix_time) override { + Status s; + if (time_elapse_only_sleep_) { + *unix_time = maybe_starting_time_; + } else { + s = target()->GetCurrentTime(unix_time); + } + if (s.ok()) { + // mock microseconds elapsed to seconds of time + *unix_time += addon_microseconds_.load() / 1000000; + } + return s; + } + + virtual uint64_t NowCPUNanos() override { + now_cpu_count_.fetch_add(1); + return target()->NowCPUNanos(); + } + + virtual uint64_t NowNanos() override { + return (time_elapse_only_sleep_ ? 0 : target()->NowNanos()) + + addon_microseconds_.load() * 1000; + } + + virtual uint64_t NowMicros() override { + return (time_elapse_only_sleep_ ? 0 : target()->NowMicros()) + + addon_microseconds_.load(); + } + + virtual Status DeleteFile(const std::string& fname) override { + delete_count_.fetch_add(1); + return target()->DeleteFile(fname); + } + + void SetMockSleep(bool enabled = true) { no_slowdown_ = enabled; } + + Status NewDirectory(const std::string& name, + std::unique_ptr* result) override { + if (!skip_fsync_) { + return target()->NewDirectory(name, result); + } else { + class NoopDirectory : public Directory { + public: + NoopDirectory() {} + ~NoopDirectory() {} + + Status Fsync() override { return Status::OK(); } + Status Close() override { return Status::OK(); } + }; + + result->reset(new NoopDirectory()); + return Status::OK(); + } + } + + Status RenameFile(const std::string& src, const std::string& dest) override { + rename_count_.fetch_add(1); + if (rename_error_.load(std::memory_order_acquire)) { + return Status::NotSupported("Simulated `RenameFile()` error."); + } + return target()->RenameFile(src, dest); + } + + // Something to return when mocking current time + const int64_t maybe_starting_time_; + + Random rnd_; + port::Mutex rnd_mutex_; // Lock to pretect rnd_ + + // sstable Sync() calls are blocked while this pointer is non-nullptr. + std::atomic delay_sstable_sync_; + + // Drop writes on the floor while this pointer is non-nullptr. + std::atomic drop_writes_; + + // Simulate no-space errors while this pointer is non-nullptr. + std::atomic no_space_; + + // Simulate non-writable file system while this pointer is non-nullptr + std::atomic non_writable_; + + // Force sync of manifest files to fail while this pointer is non-nullptr + std::atomic manifest_sync_error_; + + // Force write to manifest files to fail while this pointer is non-nullptr + std::atomic manifest_write_error_; + + // Force write to log files to fail while this pointer is non-nullptr + std::atomic log_write_error_; + + // Force `RenameFile()` to fail while this pointer is non-nullptr + std::atomic rename_error_{false}; + + // Slow down every log write, in micro-seconds. + std::atomic log_write_slowdown_; + + // If true, returns Status::NotSupported for file overwrite. + std::atomic no_file_overwrite_; + + // Number of WAL files that are still open for write. + std::atomic num_open_wal_file_; + + bool count_random_reads_; + uint32_t rand_reads_fail_odd_ = 0; + std::atomic num_reads_fails_; + anon::AtomicCounter random_read_counter_; + std::atomic random_read_bytes_counter_; + std::atomic random_file_open_counter_; + + bool count_sequential_reads_; + anon::AtomicCounter sequential_read_counter_; + + anon::AtomicCounter sleep_counter_; + + std::atomic bytes_written_; + + std::atomic sync_counter_; + + // If true, all fsync to files and directories are skipped. + bool skip_fsync_ = false; + + // If true, ingest the corruption to file during sync. + bool corrupt_in_sync_ = false; + + std::atomic non_writeable_rate_; + + std::atomic new_writable_count_; + + std::atomic non_writable_count_; + + std::function* table_write_callback_; + + std::atomic now_cpu_count_; + + std::atomic delete_count_; + + std::atomic rename_count_{0}; + + std::atomic is_wal_sync_thread_safe_{true}; + + std::atomic compaction_readahead_size_{}; + + private: // accessing these directly is prone to error + friend class DBTestBase; + + std::atomic addon_microseconds_{0}; + + // Do not modify in the env of a running DB (could cause deadlock) + std::atomic time_elapse_only_sleep_; + + bool no_slowdown_; +}; + +class FileTemperatureTestFS : public FileSystemWrapper { + public: + explicit FileTemperatureTestFS(const std::shared_ptr& fs) + : FileSystemWrapper(fs) {} + + static const char* kClassName() { return "FileTemperatureTestFS"; } + const char* Name() const override { return kClassName(); } + + IOStatus NewSequentialFile(const std::string& fname, const FileOptions& opts, + std::unique_ptr* result, + IODebugContext* dbg) override { + IOStatus s = target()->NewSequentialFile(fname, opts, result, dbg); + uint64_t number; + FileType type; + if (ParseFileName(GetFileName(fname), &number, &type) && + type == kTableFile) { + MutexLock lock(&mu_); + requested_sst_file_temperatures_.emplace_back(number, opts.temperature); + if (s.ok()) { + if (opts.temperature != Temperature::kUnknown) { + // Be extra picky and don't open if a wrong non-unknown temperature is + // provided + auto e = current_sst_file_temperatures_.find(number); + if (e != current_sst_file_temperatures_.end() && + e->second != opts.temperature) { + result->reset(); + return IOStatus::PathNotFound("Temperature mismatch on " + fname); + } + } + *result = WrapWithTemperature( + number, std::move(*result)); + } + } + return s; + } + + IOStatus NewRandomAccessFile(const std::string& fname, + const FileOptions& opts, + std::unique_ptr* result, + IODebugContext* dbg) override { + IOStatus s = target()->NewRandomAccessFile(fname, opts, result, dbg); + uint64_t number; + FileType type; + if (ParseFileName(GetFileName(fname), &number, &type) && + type == kTableFile) { + MutexLock lock(&mu_); + requested_sst_file_temperatures_.emplace_back(number, opts.temperature); + if (s.ok()) { + if (opts.temperature != Temperature::kUnknown) { + // Be extra picky and don't open if a wrong non-unknown temperature is + // provided + auto e = current_sst_file_temperatures_.find(number); + if (e != current_sst_file_temperatures_.end() && + e->second != opts.temperature) { + result->reset(); + return IOStatus::PathNotFound("Temperature mismatch on " + fname); + } + } + *result = WrapWithTemperature( + number, std::move(*result)); + } + } + return s; + } + + void PopRequestedSstFileTemperatures( + std::vector>* out = nullptr) { + MutexLock lock(&mu_); + if (out) { + *out = std::move(requested_sst_file_temperatures_); + assert(requested_sst_file_temperatures_.empty()); + } else { + requested_sst_file_temperatures_.clear(); + } + } + + IOStatus NewWritableFile(const std::string& fname, const FileOptions& opts, + std::unique_ptr* result, + IODebugContext* dbg) override { + uint64_t number; + FileType type; + if (ParseFileName(GetFileName(fname), &number, &type) && + type == kTableFile) { + MutexLock lock(&mu_); + current_sst_file_temperatures_[number] = opts.temperature; + } + return target()->NewWritableFile(fname, opts, result, dbg); + } + + void CopyCurrentSstFileTemperatures(std::map* out) { + MutexLock lock(&mu_); + *out = current_sst_file_temperatures_; + } + + void OverrideSstFileTemperature(uint64_t number, Temperature temp) { + MutexLock lock(&mu_); + current_sst_file_temperatures_[number] = temp; + } + + protected: + port::Mutex mu_; + std::vector> + requested_sst_file_temperatures_; + std::map current_sst_file_temperatures_; + + std::string GetFileName(const std::string& fname) { + auto filename = fname.substr(fname.find_last_of(kFilePathSeparator) + 1); + // workaround only for Windows that the file path could contain both Windows + // FilePathSeparator and '/' + filename = filename.substr(filename.find_last_of('/') + 1); + return filename; + } + + template + std::unique_ptr WrapWithTemperature(uint64_t number, + std::unique_ptr&& t) { + class FileWithTemp : public FileOwnerWrapperT { + public: + FileWithTemp(FileTemperatureTestFS* fs, uint64_t number, + std::unique_ptr&& t) + : FileOwnerWrapperT(std::move(t)), fs_(fs), number_(number) {} + + Temperature GetTemperature() const override { + MutexLock lock(&fs_->mu_); + return fs_->current_sst_file_temperatures_[number_]; + } + + private: + FileTemperatureTestFS* fs_; + uint64_t number_; + }; + return std::make_unique(this, number, std::move(t)); + } +}; + +class OnFileDeletionListener : public EventListener { + public: + OnFileDeletionListener() : matched_count_(0), expected_file_name_("") {} + const char* Name() const override { return kClassName(); } + static const char* kClassName() { return "OnFileDeletionListener"; } + + void SetExpectedFileName(const std::string file_name) { + expected_file_name_ = file_name; + } + + void VerifyMatchedCount(size_t expected_value) { + ASSERT_EQ(matched_count_, expected_value); + } + + void OnTableFileDeleted(const TableFileDeletionInfo& info) override { + if (expected_file_name_ != "") { + ASSERT_EQ(expected_file_name_, info.file_path); + expected_file_name_ = ""; + matched_count_++; + } + } + + private: + size_t matched_count_; + std::string expected_file_name_; +}; + +class FlushCounterListener : public EventListener { + public: + const char* Name() const override { return kClassName(); } + static const char* kClassName() { return "FlushCounterListener"; } + std::atomic count{0}; + std::atomic expected_flush_reason{FlushReason::kOthers}; + + void OnFlushBegin(DB* /*db*/, const FlushJobInfo& flush_job_info) override { + count++; + ASSERT_EQ(expected_flush_reason.load(), flush_job_info.flush_reason); + } +}; + +// A test merge operator mimics put but also fails if one of merge operands is +// "corrupted", "corrupted_try_merge", or "corrupted_must_merge". +class TestPutOperator : public MergeOperator { + public: + virtual bool FullMergeV2(const MergeOperationInput& merge_in, + MergeOperationOutput* merge_out) const override { + static const std::map + bad_operand_to_op_failure_scope = { + {"corrupted", MergeOperator::OpFailureScope::kDefault}, + {"corrupted_try_merge", MergeOperator::OpFailureScope::kTryMerge}, + {"corrupted_must_merge", + MergeOperator::OpFailureScope::kMustMerge}}; + auto check_operand = + [](Slice operand_val, + MergeOperator::OpFailureScope* op_failure_scope) -> bool { + auto iter = bad_operand_to_op_failure_scope.find(operand_val.ToString()); + if (iter != bad_operand_to_op_failure_scope.end()) { + *op_failure_scope = iter->second; + return false; + } + return true; + }; + if (merge_in.existing_value != nullptr && + !check_operand(*merge_in.existing_value, + &merge_out->op_failure_scope)) { + return false; + } + for (auto value : merge_in.operand_list) { + if (!check_operand(value, &merge_out->op_failure_scope)) { + return false; + } + } + merge_out->existing_operand = merge_in.operand_list.back(); + return true; + } + + virtual const char* Name() const override { return "TestPutOperator"; } +}; + +/* + * A cache wrapper that tracks certain CacheEntryRole's cache charge, its + * peaks and increments + * + * p0 + * / \ p1 + * / \ /\ + * / \/ \ + * a / b \ + * peaks = {p0, p1} + * increments = {p1-a, p2-b} + */ +template +class TargetCacheChargeTrackingCache : public CacheWrapper { + public: + explicit TargetCacheChargeTrackingCache(std::shared_ptr target); + + const char* Name() const override { return "TargetCacheChargeTrackingCache"; } + + Status Insert(const Slice& key, ObjectPtr value, + const CacheItemHelper* helper, size_t charge, + Handle** handle = nullptr, + Priority priority = Priority::LOW) override; + + using Cache::Release; + bool Release(Handle* handle, bool erase_if_last_ref = false) override; + + std::size_t GetCacheCharge() { return cur_cache_charge_; } + + std::deque GetChargedCachePeaks() { return cache_charge_peaks_; } + + std::size_t GetChargedCacheIncrementSum() { + return cache_charge_increments_sum_; + } + + private: + static const Cache::CacheItemHelper* kCrmHelper; + + std::size_t cur_cache_charge_; + std::size_t cache_charge_peak_; + std::size_t cache_charge_increment_; + bool last_peak_tracked_; + std::deque cache_charge_peaks_; + std::size_t cache_charge_increments_sum_; +}; + +class DBTestBase : public testing::Test { + public: + // Sequence of option configurations to try + enum OptionConfig : int { + kDefault = 0, + kBlockBasedTableWithPrefixHashIndex = 1, + kBlockBasedTableWithWholeKeyHashIndex = 2, + kPlainTableFirstBytePrefix = 3, + kPlainTableCappedPrefix = 4, + kPlainTableCappedPrefixNonMmap = 5, + kPlainTableAllBytesPrefix = 6, + kVectorRep = 7, + kHashLinkList = 8, + kMergePut = 9, + kFilter = 10, + kFullFilterWithNewTableReaderForCompactions = 11, + kUncompressed = 12, + kNumLevel_3 = 13, + kDBLogDir = 14, + kWalDirAndMmapReads = 15, + kManifestFileSize = 16, + kPerfOptions = 17, + kHashSkipList = 18, + kUniversalCompaction = 19, + kUniversalCompactionMultiLevel = 20, + kInfiniteMaxOpenFiles = 21, + kCRC32cChecksum = 22, + kFIFOCompaction = 23, + kOptimizeFiltersForHits = 24, + kRowCache = 25, + kRecycleLogFiles = 26, + kConcurrentSkipList = 27, + kPipelinedWrite = 28, + kConcurrentWALWrites = 29, + kDirectIO, + kLevelSubcompactions, + kBlockBasedTableWithIndexRestartInterval, + kBlockBasedTableWithPartitionedIndex, + kBlockBasedTableWithPartitionedIndexFormat4, + kBlockBasedTableWithLatestFormat, + kPartitionedFilterWithNewTableReaderForCompactions, + kUniversalSubcompactions, + kUnorderedWrite, + // This must be the last line + kEnd, + }; + + public: + std::string dbname_; + std::string alternative_wal_dir_; + std::string alternative_db_log_dir_; + MockEnv* mem_env_; + Env* encrypted_env_; + SpecialEnv* env_; + std::shared_ptr env_guard_; + DB* db_; + std::vector handles_; + + int option_config_; + Options last_options_; + + // Skip some options, as they may not be applicable to a specific test. + // To add more skip constants, use values 4, 8, 16, etc. + enum OptionSkip { + kNoSkip = 0, + kSkipDeletesFilterFirst = 1, + kSkipUniversalCompaction = 2, + kSkipMergePut = 4, + kSkipPlainTable = 8, + kSkipHashIndex = 16, + kSkipNoSeekToLast = 32, + kSkipFIFOCompaction = 128, + kSkipMmapReads = 256, + }; + + const int kRangeDelSkipConfigs = + // Plain tables do not support range deletions. + kSkipPlainTable | + // MmapReads disables the iterator pinning that RangeDelAggregator + // requires. + kSkipMmapReads; + + // `env_do_fsync` decides whether the special Env would do real + // fsync for files and directories. Skipping fsync can speed up + // tests, but won't cover the exact fsync logic. + DBTestBase(const std::string path, bool env_do_fsync); + + ~DBTestBase(); + + static std::string Key(int i) { + char buf[100]; + snprintf(buf, sizeof(buf), "key%06d", i); + return std::string(buf); + } + + static bool ShouldSkipOptions(int option_config, int skip_mask = kNoSkip); + + // Switch to a fresh database with the next option configuration to + // test. Return false if there are no more configurations to test. + bool ChangeOptions(int skip_mask = kNoSkip); + + // Switch between different compaction styles. + bool ChangeCompactOptions(); + + // Switch between different WAL-realted options. + bool ChangeWalOptions(); + + // Switch between different filter policy + // Jump from kDefault to kFilter to kFullFilter + bool ChangeFilterOptions(); + + // Switch between different DB options for file ingestion tests. + bool ChangeOptionsForFileIngestionTest(); + + // Return the current option configuration. + Options CurrentOptions(const anon::OptionsOverride& options_override = + anon::OptionsOverride()) const; + + Options CurrentOptions(const Options& default_options, + const anon::OptionsOverride& options_override = + anon::OptionsOverride()) const; + + Options GetDefaultOptions() const; + + Options GetOptions(int option_config) const { + return GetOptions(option_config, GetDefaultOptions()); + } + + Options GetOptions(int option_config, const Options& default_options, + const anon::OptionsOverride& options_override = + anon::OptionsOverride()) const; + + DBImpl* dbfull() { return static_cast_with_check(db_); } + + void CreateColumnFamilies(const std::vector& cfs, + const Options& options); + + void CreateAndReopenWithCF(const std::vector& cfs, + const Options& options); + + void ReopenWithColumnFamilies(const std::vector& cfs, + const std::vector& options); + + void ReopenWithColumnFamilies(const std::vector& cfs, + const Options& options); + + Status TryReopenWithColumnFamilies(const std::vector& cfs, + const std::vector& options); + + Status TryReopenWithColumnFamilies(const std::vector& cfs, + const Options& options); + + void Reopen(const Options& options); + + void Close(); + + void DestroyAndReopen(const Options& options); + + void Destroy(const Options& options, bool delete_cf_paths = false); + + Status ReadOnlyReopen(const Options& options); + + Status TryReopen(const Options& options); + + bool IsDirectIOSupported(); + + bool IsMemoryMappedAccessSupported() const; + + Status Flush(int cf = 0); + + Status Flush(const std::vector& cf_ids); + + Status Put(const Slice& k, const Slice& v, WriteOptions wo = WriteOptions()); + + Status Put(int cf, const Slice& k, const Slice& v, + WriteOptions wo = WriteOptions()); + + Status Merge(const Slice& k, const Slice& v, + WriteOptions wo = WriteOptions()); + + Status Merge(int cf, const Slice& k, const Slice& v, + WriteOptions wo = WriteOptions()); + + Status Delete(const std::string& k); + + Status Delete(int cf, const std::string& k); + + Status SingleDelete(const std::string& k); + + Status SingleDelete(int cf, const std::string& k); + + std::string Get(const std::string& k, const Snapshot* snapshot = nullptr); + + std::string Get(int cf, const std::string& k, + const Snapshot* snapshot = nullptr); + + Status Get(const std::string& k, PinnableSlice* v); + + std::vector MultiGet(std::vector cfs, + const std::vector& k, + const Snapshot* snapshot, + const bool batched, + const bool async = false); + + std::vector MultiGet(const std::vector& k, + const Snapshot* snapshot = nullptr, + const bool async = false); + + uint64_t GetNumSnapshots(); + + uint64_t GetTimeOldestSnapshots(); + + uint64_t GetSequenceOldestSnapshots(); + + // Return a string that contains all key,value pairs in order, + // formatted like "(k1->v1)(k2->v2)". + std::string Contents(int cf = 0); + + std::string AllEntriesFor(const Slice& user_key, int cf = 0); + + // Similar to AllEntriesFor but this function also covers reopen with fifo. + // Note that test cases with snapshots or entries in memtable should simply + // use AllEntriesFor instead as snapshots and entries in memtable will + // survive after db reopen. + void CheckAllEntriesWithFifoReopen(const std::string& expected_value, + const Slice& user_key, int cf, + const std::vector& cfs, + const Options& options); + + int NumSortedRuns(int cf = 0); + + uint64_t TotalSize(int cf = 0); + + uint64_t SizeAtLevel(int level); + + size_t TotalLiveFiles(int cf = 0); + + size_t TotalLiveFilesAtPath(int cf, const std::string& path); + + size_t CountLiveFiles(); + + int NumTableFilesAtLevel(int level, int cf = 0); + + double CompressionRatioAtLevel(int level, int cf = 0); + + int TotalTableFiles(int cf = 0, int levels = -1); + + std::vector GetBlobFileNumbers(); + + // Return spread of files per level + std::string FilesPerLevel(int cf = 0); + + size_t CountFiles(); + + Status CountFiles(size_t* count); + + Status Size(const Slice& start, const Slice& limit, uint64_t* size) { + return Size(start, limit, 0, size); + } + + Status Size(const Slice& start, const Slice& limit, int cf, uint64_t* size); + + void Compact(int cf, const Slice& start, const Slice& limit, + uint32_t target_path_id); + + void Compact(int cf, const Slice& start, const Slice& limit); + + void Compact(const Slice& start, const Slice& limit); + + // Do n memtable compactions, each of which produces an sstable + // covering the range [small,large]. + void MakeTables(int n, const std::string& small, const std::string& large, + int cf = 0); + + // Prevent pushing of new sstables into deeper levels by adding + // tables that cover a specified range to all levels. + void FillLevels(const std::string& smallest, const std::string& largest, + int cf); + + void MoveFilesToLevel(int level, int cf = 0); + + void DumpFileCounts(const char* label); + + std::string DumpSSTableList(); + + static void GetSstFiles(Env* env, std::string path, + std::vector* files); + + int GetSstFileCount(std::string path); + + // this will generate non-overlapping files since it keeps increasing key_idx + void GenerateNewFile(Random* rnd, int* key_idx, bool nowait = false); + + void GenerateNewFile(int fd, Random* rnd, int* key_idx, bool nowait = false); + + static const int kNumKeysByGenerateNewRandomFile; + static const int KNumKeysByGenerateNewFile = 100; + + void GenerateNewRandomFile(Random* rnd, bool nowait = false); + + std::string IterStatus(Iterator* iter); + + Options OptionsForLogIterTest(); + + std::string DummyString(size_t len, char c = 'a'); + + void VerifyIterLast(std::string expected_key, int cf = 0); + + // Used to test InplaceUpdate + + // If previous value is nullptr or delta is > than previous value, + // sets newValue with delta + // If previous value is not empty, + // updates previous value with 'b' string of previous value size - 1. + static UpdateStatus updateInPlaceSmallerSize(char* prevValue, + uint32_t* prevSize, Slice delta, + std::string* newValue); + + static UpdateStatus updateInPlaceSmallerVarintSize(char* prevValue, + uint32_t* prevSize, + Slice delta, + std::string* newValue); + + static UpdateStatus updateInPlaceLargerSize(char* prevValue, + uint32_t* prevSize, Slice delta, + std::string* newValue); + + static UpdateStatus updateInPlaceNoAction(char* prevValue, uint32_t* prevSize, + Slice delta, std::string* newValue); + + // Utility method to test InplaceUpdate + void validateNumberOfEntries(int numValues, int cf = 0); + + void CopyFile(const std::string& source, const std::string& destination, + uint64_t size = 0); + + Status GetAllDataFiles(const FileType file_type, + std::unordered_map* sst_files, + uint64_t* total_size = nullptr); + + std::vector ListTableFiles(Env* env, const std::string& path); + + void VerifyDBFromMap( + std::map true_data, + size_t* total_reads_res = nullptr, bool tailing_iter = false, + std::map status = std::map()); + + void VerifyDBInternal( + std::vector> true_data); + + uint64_t GetNumberOfSstFilesForColumnFamily(DB* db, + std::string column_family_name); + + uint64_t GetSstSizeHelper(Temperature temperature); + + uint64_t TestGetTickerCount(const Options& options, Tickers ticker_type) { + return options.statistics->getTickerCount(ticker_type); + } + + uint64_t TestGetAndResetTickerCount(const Options& options, + Tickers ticker_type) { + return options.statistics->getAndResetTickerCount(ticker_type); + } + // Short name for TestGetAndResetTickerCount + uint64_t PopTicker(const Options& options, Tickers ticker_type) { + return options.statistics->getAndResetTickerCount(ticker_type); + } + + // Note: reverting this setting within the same test run is not yet + // supported + void SetTimeElapseOnlySleepOnReopen(DBOptions* options); + + void ResetTableProperties(TableProperties* tp) { + tp->data_size = 0; + tp->index_size = 0; + tp->filter_size = 0; + tp->raw_key_size = 0; + tp->raw_value_size = 0; + tp->num_data_blocks = 0; + tp->num_entries = 0; + tp->num_deletions = 0; + tp->num_merge_operands = 0; + tp->num_range_deletions = 0; + } + + void ParseTablePropertiesString(std::string tp_string, TableProperties* tp) { + double dummy_double; + std::replace(tp_string.begin(), tp_string.end(), ';', ' '); + std::replace(tp_string.begin(), tp_string.end(), '=', ' '); + ResetTableProperties(tp); + sscanf(tp_string.c_str(), + "# data blocks %" SCNu64 " # entries %" SCNu64 + " # deletions %" SCNu64 " # merge operands %" SCNu64 + " # range deletions %" SCNu64 " raw key size %" SCNu64 + " raw average key size %lf " + " raw value size %" SCNu64 + " raw average value size %lf " + " data block size %" SCNu64 " index block size (user-key? %" SCNu64 + ", delta-value? %" SCNu64 ") %" SCNu64 " filter block size %" SCNu64, + &tp->num_data_blocks, &tp->num_entries, &tp->num_deletions, + &tp->num_merge_operands, &tp->num_range_deletions, &tp->raw_key_size, + &dummy_double, &tp->raw_value_size, &dummy_double, &tp->data_size, + &tp->index_key_is_user_key, &tp->index_value_is_delta_encoded, + &tp->index_size, &tp->filter_size); + } + + private: // Prone to error on direct use + void MaybeInstallTimeElapseOnlySleep(const DBOptions& options); + + bool time_elapse_only_sleep_on_reopen_ = false; +}; + +// For verifying that all files generated by current version have SST +// unique ids. +void VerifySstUniqueIds(const TablePropertiesCollection& props); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/db_universal_compaction_test.cc b/librocksdb-sys/rocksdb/db/db_universal_compaction_test.cc new file mode 100644 index 0000000..282a578 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_universal_compaction_test.cc @@ -0,0 +1,2239 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/db_test_util.h" +#include "port/stack_trace.h" +#include "rocksdb/utilities/table_properties_collectors.h" +#include "test_util/sync_point.h" +#include "test_util/testutil.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +static std::string CompressibleString(Random* rnd, int len) { + std::string r; + test::CompressibleString(rnd, 0.8, len, &r); + return r; +} + +class DBTestUniversalCompactionBase + : public DBTestBase, + public ::testing::WithParamInterface> { + public: + explicit DBTestUniversalCompactionBase(const std::string& path) + : DBTestBase(path, /*env_do_fsync=*/false) {} + void SetUp() override { + num_levels_ = std::get<0>(GetParam()); + exclusive_manual_compaction_ = std::get<1>(GetParam()); + } + int num_levels_; + bool exclusive_manual_compaction_; +}; + +class DBTestUniversalCompaction : public DBTestUniversalCompactionBase { + public: + DBTestUniversalCompaction() + : DBTestUniversalCompactionBase("/db_universal_compaction_test") {} +}; + +class DBTestUniversalCompaction2 : public DBTestBase { + public: + DBTestUniversalCompaction2() + : DBTestBase("db_universal_compaction_test2", /*env_do_fsync=*/false) {} +}; + +namespace { +void VerifyCompactionResult( + const ColumnFamilyMetaData& cf_meta, + const std::set& overlapping_file_numbers) { +#ifndef NDEBUG + for (auto& level : cf_meta.levels) { + for (auto& file : level.files) { + assert(overlapping_file_numbers.find(file.name) == + overlapping_file_numbers.end()); + } + } +#endif +} + +class KeepFilter : public CompactionFilter { + public: + bool Filter(int /*level*/, const Slice& /*key*/, const Slice& /*value*/, + std::string* /*new_value*/, + bool* /*value_changed*/) const override { + return false; + } + + const char* Name() const override { return "KeepFilter"; } +}; + +class KeepFilterFactory : public CompactionFilterFactory { + public: + explicit KeepFilterFactory(bool check_context = false) + : check_context_(check_context) {} + + std::unique_ptr CreateCompactionFilter( + const CompactionFilter::Context& context) override { + if (check_context_) { + EXPECT_EQ(expect_full_compaction_.load(), context.is_full_compaction); + EXPECT_EQ(expect_manual_compaction_.load(), context.is_manual_compaction); + } + return std::unique_ptr(new KeepFilter()); + } + + const char* Name() const override { return "KeepFilterFactory"; } + bool check_context_; + std::atomic_bool expect_full_compaction_; + std::atomic_bool expect_manual_compaction_; +}; +} // anonymous namespace + +// Make sure we don't trigger a problem if the trigger condtion is given +// to be 0, which is invalid. +TEST_P(DBTestUniversalCompaction, UniversalCompactionSingleSortedRun) { + Options options = CurrentOptions(); + + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = num_levels_; + // Config universal compaction to always compact to one single sorted run. + options.level0_file_num_compaction_trigger = 0; + options.compaction_options_universal.size_ratio = 10; + options.compaction_options_universal.min_merge_width = 2; + options.compaction_options_universal.max_size_amplification_percent = 0; + + options.write_buffer_size = 105 << 10; // 105KB + options.arena_block_size = 4 << 10; + options.target_file_size_base = 32 << 10; // 32KB + // trigger compaction if there are >= 4 files + KeepFilterFactory* filter = new KeepFilterFactory(true); + filter->expect_manual_compaction_.store(false); + options.compaction_filter_factory.reset(filter); + + DestroyAndReopen(options); + ASSERT_EQ(1, db_->GetOptions().level0_file_num_compaction_trigger); + + Random rnd(301); + int key_idx = 0; + + filter->expect_full_compaction_.store(true); + + for (int num = 0; num < 16; num++) { + // Write 100KB file. And immediately it should be compacted to one file. + GenerateNewFile(&rnd, &key_idx); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumSortedRuns(0), 1); + } + ASSERT_OK(Put(Key(key_idx), "")); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumSortedRuns(0), 1); +} + +TEST_P(DBTestUniversalCompaction, OptimizeFiltersForHits) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.compaction_options_universal.size_ratio = 5; + options.num_levels = num_levels_; + options.write_buffer_size = 105 << 10; // 105KB + options.arena_block_size = 4 << 10; + options.target_file_size_base = 32 << 10; // 32KB + // trigger compaction if there are >= 4 files + options.level0_file_num_compaction_trigger = 4; + BlockBasedTableOptions bbto; + bbto.cache_index_and_filter_blocks = true; + bbto.filter_policy.reset(NewBloomFilterPolicy(10, false)); + bbto.whole_key_filtering = true; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + options.optimize_filters_for_hits = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.memtable_factory.reset(test::NewSpecialSkipListFactory(3)); + + DestroyAndReopen(options); + + // block compaction from happening + env_->SetBackgroundThreads(1, Env::LOW); + test::SleepingBackgroundTask sleeping_task_low; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::LOW); + + for (int num = 0; num < options.level0_file_num_compaction_trigger; num++) { + ASSERT_OK(Put(Key(num * 10), "val")); + if (num) { + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + ASSERT_OK(Put(Key(30 + num * 10), "val")); + ASSERT_OK(Put(Key(60 + num * 10), "val")); + } + ASSERT_OK(Put("", "")); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + + // Query set of non existing keys + for (int i = 5; i < 90; i += 10) { + ASSERT_EQ(Get(Key(i)), "NOT_FOUND"); + } + + // Make sure bloom filter is used at least once. + ASSERT_GT(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0); + auto prev_counter = TestGetTickerCount(options, BLOOM_FILTER_USEFUL); + + // Make sure bloom filter is used for all but the last L0 file when looking + // up a non-existent key that's in the range of all L0 files. + ASSERT_EQ(Get(Key(35)), "NOT_FOUND"); + ASSERT_EQ(prev_counter + NumTableFilesAtLevel(0) - 1, + TestGetTickerCount(options, BLOOM_FILTER_USEFUL)); + prev_counter = TestGetTickerCount(options, BLOOM_FILTER_USEFUL); + + // Unblock compaction and wait it for happening. + sleeping_task_low.WakeUp(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // The same queries will not trigger bloom filter + for (int i = 5; i < 90; i += 10) { + ASSERT_EQ(Get(Key(i)), "NOT_FOUND"); + } + ASSERT_EQ(prev_counter, TestGetTickerCount(options, BLOOM_FILTER_USEFUL)); +} + +// TODO(kailiu) The tests on UniversalCompaction has some issues: +// 1. A lot of magic numbers ("11" or "12"). +// 2. Made assumption on the memtable flush conditions, which may change from +// time to time. +TEST_P(DBTestUniversalCompaction, UniversalCompactionTrigger) { + Options options; + options.compaction_style = kCompactionStyleUniversal; + options.compaction_options_universal.size_ratio = 5; + options.num_levels = num_levels_; + options.write_buffer_size = 105 << 10; // 105KB + options.arena_block_size = 4 << 10; + options.target_file_size_base = 32 << 10; // 32KB + // trigger compaction if there are >= 4 files + options.level0_file_num_compaction_trigger = 4; + KeepFilterFactory* filter = new KeepFilterFactory(true); + filter->expect_manual_compaction_.store(false); + options.compaction_filter_factory.reset(filter); + + options = CurrentOptions(options); + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBTestWritableFile.GetPreallocationStatus", [&](void* arg) { + ASSERT_TRUE(arg != nullptr); + size_t preallocation_size = *(static_cast(arg)); + if (num_levels_ > 3) { + ASSERT_LE(preallocation_size, options.target_file_size_base * 1.1); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Random rnd(301); + int key_idx = 0; + + filter->expect_full_compaction_.store(true); + // Stage 1: + // Generate a set of files at level 0, but don't trigger level-0 + // compaction. + for (int num = 0; num < options.level0_file_num_compaction_trigger - 1; + num++) { + // Write 100KB + GenerateNewFile(1, &rnd, &key_idx); + } + + // Generate one more file at level-0, which should trigger level-0 + // compaction. + GenerateNewFile(1, &rnd, &key_idx); + // Suppose each file flushed from mem table has size 1. Now we compact + // (level0_file_num_compaction_trigger+1)=4 files and should have a big + // file of size 4. + ASSERT_EQ(NumSortedRuns(1), 1); + + // Stage 2: + // Now we have one file at level 0, with size 4. We also have some data in + // mem table. Let's continue generating new files at level 0, but don't + // trigger level-0 compaction. + // First, clean up memtable before inserting new data. This will generate + // a level-0 file, with size around 0.4 (according to previously written + // data amount). + filter->expect_full_compaction_.store(false); + ASSERT_OK(Flush(1)); + for (int num = 0; num < options.level0_file_num_compaction_trigger - 3; + num++) { + GenerateNewFile(1, &rnd, &key_idx); + ASSERT_EQ(NumSortedRuns(1), num + 3); + } + + // Generate one more file at level-0, which should trigger level-0 + // compaction. + GenerateNewFile(1, &rnd, &key_idx); + // Before compaction, we have 4 files at level 0, with size 4, 0.4, 1, 1. + // After compaction, we should have 2 files, with size 4, 2.4. + ASSERT_EQ(NumSortedRuns(1), 2); + + // Stage 3: + // Now we have 2 files at level 0, with size 4 and 2.4. Continue + // generating new files at level 0. + for (int num = 0; num < options.level0_file_num_compaction_trigger - 3; + num++) { + GenerateNewFile(1, &rnd, &key_idx); + ASSERT_EQ(NumSortedRuns(1), num + 3); + } + + // Generate one more file at level-0, which should trigger level-0 + // compaction. + GenerateNewFile(1, &rnd, &key_idx); + // Before compaction, we have 4 files at level 0, with size 4, 2.4, 1, 1. + // After compaction, we should have 3 files, with size 4, 2.4, 2. + ASSERT_EQ(NumSortedRuns(1), 3); + + // Stage 4: + // Now we have 3 files at level 0, with size 4, 2.4, 2. Let's generate a + // new file of size 1. + GenerateNewFile(1, &rnd, &key_idx); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // Level-0 compaction is triggered, but no file will be picked up. + ASSERT_EQ(NumSortedRuns(1), 4); + + // Stage 5: + // Now we have 4 files at level 0, with size 4, 2.4, 2, 1. Let's generate + // a new file of size 1. + filter->expect_full_compaction_.store(true); + GenerateNewFile(1, &rnd, &key_idx); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // All files at level 0 will be compacted into a single one. + ASSERT_EQ(NumSortedRuns(1), 1); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_P(DBTestUniversalCompaction, UniversalCompactionSizeAmplification) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = num_levels_; + options.write_buffer_size = 100 << 10; // 100KB + options.target_file_size_base = 32 << 10; // 32KB + options.level0_file_num_compaction_trigger = 3; + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // Trigger compaction if size amplification exceeds 110% + options.compaction_options_universal.max_size_amplification_percent = 110; + options = CurrentOptions(options); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + + Random rnd(301); + int key_idx = 0; + + // Generate two files in Level 0. Both files are approx the same size. + for (int num = 0; num < options.level0_file_num_compaction_trigger - 1; + num++) { + // Write 110KB (11 values, each 10K) + for (int i = 0; i < 11; i++) { + ASSERT_OK(Put(1, Key(key_idx), rnd.RandomString(10000))); + key_idx++; + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); + ASSERT_EQ(NumSortedRuns(1), num + 1); + } + ASSERT_EQ(NumSortedRuns(1), 2); + + // Flush whatever is remaining in memtable. This is typically + // small, which should not trigger size ratio based compaction + // but will instead trigger size amplification. + ASSERT_OK(Flush(1)); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // Verify that size amplification did occur + ASSERT_EQ(NumSortedRuns(1), 1); +} + +TEST_P(DBTestUniversalCompaction, DynamicUniversalCompactionSizeAmplification) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = 1; + options.write_buffer_size = 100 << 10; // 100KB + options.target_file_size_base = 32 << 10; // 32KB + options.level0_file_num_compaction_trigger = 3; + // Initial setup of compaction_options_universal will prevent universal + // compaction from happening + options.compaction_options_universal.size_ratio = 100; + options.compaction_options_universal.min_merge_width = 100; + DestroyAndReopen(options); + + int total_picked_compactions = 0; + int total_size_amp_compactions = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "UniversalCompactionBuilder::PickCompaction:Return", [&](void* arg) { + if (arg) { + total_picked_compactions++; + Compaction* c = static_cast(arg); + if (c->compaction_reason() == + CompactionReason::kUniversalSizeAmplification) { + total_size_amp_compactions++; + } + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + MutableCFOptions mutable_cf_options; + CreateAndReopenWithCF({"pikachu"}, options); + + Random rnd(301); + int key_idx = 0; + + // Generate two files in Level 0. Both files are approx the same size. + for (int num = 0; num < options.level0_file_num_compaction_trigger - 1; + num++) { + // Write 110KB (11 values, each 10K) + for (int i = 0; i < 11; i++) { + ASSERT_OK(Put(1, Key(key_idx), rnd.RandomString(10000))); + key_idx++; + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); + ASSERT_EQ(NumSortedRuns(1), num + 1); + } + ASSERT_EQ(NumSortedRuns(1), 2); + + // Flush whatever is remaining in memtable. This is typically + // small, which should not trigger size ratio based compaction + // but could instead trigger size amplification if it's set + // to 110. + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // Verify compaction did not happen + ASSERT_EQ(NumSortedRuns(1), 3); + + // Trigger compaction if size amplification exceeds 110% without reopening DB + ASSERT_EQ(dbfull() + ->GetOptions(handles_[1]) + .compaction_options_universal.max_size_amplification_percent, + 200U); + ASSERT_OK(dbfull()->SetOptions(handles_[1], + {{"compaction_options_universal", + "{max_size_amplification_percent=110;}"}})); + ASSERT_EQ(dbfull() + ->GetOptions(handles_[1]) + .compaction_options_universal.max_size_amplification_percent, + 110u); + ASSERT_OK(dbfull()->TEST_GetLatestMutableCFOptions(handles_[1], + &mutable_cf_options)); + ASSERT_EQ(110u, mutable_cf_options.compaction_options_universal + .max_size_amplification_percent); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // Verify that size amplification did happen + ASSERT_EQ(NumSortedRuns(1), 1); + ASSERT_EQ(total_picked_compactions, 1); + ASSERT_EQ(total_size_amp_compactions, 1); +} + +TEST_P(DBTestUniversalCompaction, DynamicUniversalCompactionReadAmplification) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = 1; + options.write_buffer_size = 100 << 10; // 100KB + options.target_file_size_base = 32 << 10; // 32KB + options.level0_file_num_compaction_trigger = 3; + // Initial setup of compaction_options_universal will prevent universal + // compaction from happening + options.compaction_options_universal.max_size_amplification_percent = 2000; + options.compaction_options_universal.size_ratio = 0; + options.compaction_options_universal.min_merge_width = 100; + DestroyAndReopen(options); + + int total_picked_compactions = 0; + int total_size_ratio_compactions = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "UniversalCompactionBuilder::PickCompaction:Return", [&](void* arg) { + if (arg) { + total_picked_compactions++; + Compaction* c = static_cast(arg); + if (c->compaction_reason() == CompactionReason::kUniversalSizeRatio) { + total_size_ratio_compactions++; + } + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + MutableCFOptions mutable_cf_options; + CreateAndReopenWithCF({"pikachu"}, options); + + Random rnd(301); + int key_idx = 0; + + // Generate three files in Level 0. All files are approx the same size. + for (int num = 0; num < options.level0_file_num_compaction_trigger; num++) { + // Write 110KB (11 values, each 10K) + for (int i = 0; i < 11; i++) { + ASSERT_OK(Put(1, Key(key_idx), rnd.RandomString(10000))); + key_idx++; + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); + ASSERT_EQ(NumSortedRuns(1), num + 1); + } + ASSERT_EQ(NumSortedRuns(1), options.level0_file_num_compaction_trigger); + + // Flush whatever is remaining in memtable. This is typically small, about + // 30KB. + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // Verify compaction did not happen + ASSERT_EQ(NumSortedRuns(1), options.level0_file_num_compaction_trigger + 1); + ASSERT_EQ(total_picked_compactions, 0); + + ASSERT_OK(dbfull()->SetOptions( + handles_[1], + {{"compaction_options_universal", + "{min_merge_width=2;max_merge_width=2;size_ratio=100;}"}})); + ASSERT_EQ(dbfull() + ->GetOptions(handles_[1]) + .compaction_options_universal.min_merge_width, + 2u); + ASSERT_EQ(dbfull() + ->GetOptions(handles_[1]) + .compaction_options_universal.max_merge_width, + 2u); + ASSERT_EQ( + dbfull()->GetOptions(handles_[1]).compaction_options_universal.size_ratio, + 100u); + + ASSERT_OK(dbfull()->TEST_GetLatestMutableCFOptions(handles_[1], + &mutable_cf_options)); + ASSERT_EQ(mutable_cf_options.compaction_options_universal.size_ratio, 100u); + ASSERT_EQ(mutable_cf_options.compaction_options_universal.min_merge_width, + 2u); + ASSERT_EQ(mutable_cf_options.compaction_options_universal.max_merge_width, + 2u); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // Files in L0 are approx: 0.3 (30KB), 1, 1, 1. + // On compaction: the files are below the size amp threshold, so we + // fallthrough to checking read amp conditions. The configured size ratio is + // not big enough to take 0.3 into consideration. So the next files 1 and 1 + // are compacted together first as they satisfy size ratio condition and + // (min_merge_width, max_merge_width) condition, to give out a file size of 2. + // Next, the newly generated 2 and the last file 1 are compacted together. So + // at the end: #sortedRuns = 2, #picked_compactions = 2, and all the picked + // ones are size ratio based compactions. + ASSERT_EQ(NumSortedRuns(1), 2); + // If max_merge_width had not been changed dynamically above, and if it + // continued to be the default value of UINIT_MAX, total_picked_compactions + // would have been 1. + ASSERT_EQ(total_picked_compactions, 2); + ASSERT_EQ(total_size_ratio_compactions, 2); +} + +TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) { + const int kTestKeySize = 16; + const int kTestValueSize = 984; + const int kEntrySize = kTestKeySize + kTestValueSize; + const int kEntriesPerBuffer = 10; + + ChangeCompactOptions(); + Options options; + options.create_if_missing = true; + options.compaction_style = kCompactionStyleLevel; + options.num_levels = 1; + options.target_file_size_base = options.write_buffer_size; + options.compression = kNoCompression; + options = CurrentOptions(options); + options.write_buffer_size = kEntrySize * kEntriesPerBuffer; + CreateAndReopenWithCF({"pikachu"}, options); + ASSERT_EQ(options.compaction_style, kCompactionStyleUniversal); + Random rnd(301); + for (int key = 1024 * kEntriesPerBuffer; key >= 0; --key) { + ASSERT_OK(Put(1, std::to_string(key), rnd.RandomString(kTestValueSize))); + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ColumnFamilyMetaData cf_meta; + dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta); + std::vector compaction_input_file_names; + for (auto file : cf_meta.levels[0].files) { + if (rnd.OneIn(2)) { + compaction_input_file_names.push_back(file.name); + } + } + + if (compaction_input_file_names.size() == 0) { + compaction_input_file_names.push_back(cf_meta.levels[0].files[0].name); + } + + // expect fail since universal compaction only allow L0 output + ASSERT_FALSE(dbfull() + ->CompactFiles(CompactionOptions(), handles_[1], + compaction_input_file_names, 1) + .ok()); + + // expect ok and verify the compacted files no longer exist. + ASSERT_OK(dbfull()->CompactFiles(CompactionOptions(), handles_[1], + compaction_input_file_names, 0)); + + dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta); + VerifyCompactionResult( + cf_meta, std::set(compaction_input_file_names.begin(), + compaction_input_file_names.end())); + + compaction_input_file_names.clear(); + + // Pick the first and the last file, expect everything is + // compacted into one single file. + compaction_input_file_names.push_back(cf_meta.levels[0].files[0].name); + compaction_input_file_names.push_back( + cf_meta.levels[0].files[cf_meta.levels[0].files.size() - 1].name); + ASSERT_OK(dbfull()->CompactFiles(CompactionOptions(), handles_[1], + compaction_input_file_names, 0)); + + dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta); + ASSERT_EQ(cf_meta.levels[0].files.size(), 1U); +} + +TEST_P(DBTestUniversalCompaction, UniversalCompactionTargetLevel) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.write_buffer_size = 100 << 10; // 100KB + options.num_levels = 7; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + + // Generate 3 overlapping files + Random rnd(301); + for (int i = 0; i < 210; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(100))); + } + ASSERT_OK(Flush()); + + for (int i = 200; i < 300; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(100))); + } + ASSERT_OK(Flush()); + + for (int i = 250; i < 260; i++) { + ASSERT_OK(Put(Key(i), rnd.RandomString(100))); + } + ASSERT_OK(Flush()); + + ASSERT_EQ("3", FilesPerLevel(0)); + // Compact all files into 1 file and put it in L4 + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 4; + compact_options.exclusive_manual_compaction = exclusive_manual_compaction_; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + ASSERT_EQ("0,0,0,0,1", FilesPerLevel(0)); +} + +#if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) +class DBTestUniversalCompactionMultiLevels + : public DBTestUniversalCompactionBase { + public: + DBTestUniversalCompactionMultiLevels() + : DBTestUniversalCompactionBase( + "/db_universal_compaction_multi_levels_test") {} +}; + +TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionMultiLevels) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = num_levels_; + options.write_buffer_size = 100 << 10; // 100KB + options.level0_file_num_compaction_trigger = 8; + options.max_background_compactions = 3; + options.target_file_size_base = 32 * 1024; + CreateAndReopenWithCF({"pikachu"}, options); + + // Trigger compaction if size amplification exceeds 110% + options.compaction_options_universal.max_size_amplification_percent = 110; + options = CurrentOptions(options); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + + Random rnd(301); + int num_keys = 100000; + for (int i = 0; i < num_keys * 2; i++) { + ASSERT_OK(Put(1, Key(i % num_keys), Key(i))); + } + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + for (int i = num_keys; i < num_keys * 2; i++) { + ASSERT_EQ(Get(1, Key(i % num_keys)), Key(i)); + } +} + +// Tests universal compaction with trivial move enabled +TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionTrivialMove) { + int32_t trivial_move = 0; + int32_t non_trivial_move = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:TrivialMove", + [&](void* /*arg*/) { trivial_move++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial", [&](void* arg) { + non_trivial_move++; + ASSERT_TRUE(arg != nullptr); + int output_level = *(static_cast(arg)); + ASSERT_EQ(output_level, 0); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.compaction_options_universal.allow_trivial_move = true; + options.num_levels = 3; + options.write_buffer_size = 100 << 10; // 100KB + options.level0_file_num_compaction_trigger = 3; + options.max_background_compactions = 2; + options.target_file_size_base = 32 * 1024; + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // Trigger compaction if size amplification exceeds 110% + options.compaction_options_universal.max_size_amplification_percent = 110; + options = CurrentOptions(options); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + + Random rnd(301); + int num_keys = 150000; + for (int i = 0; i < num_keys; i++) { + ASSERT_OK(Put(1, Key(i), Key(i))); + } + std::vector values; + + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_GT(trivial_move, 0); + ASSERT_GT(non_trivial_move, 0); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +INSTANTIATE_TEST_CASE_P(MultiLevels, DBTestUniversalCompactionMultiLevels, + ::testing::Combine(::testing::Values(3, 20), + ::testing::Bool())); + +class DBTestUniversalCompactionParallel : public DBTestUniversalCompactionBase { + public: + DBTestUniversalCompactionParallel() + : DBTestUniversalCompactionBase("/db_universal_compaction_prallel_test") { + } +}; + +TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = num_levels_; + options.env = env_; + options.write_buffer_size = 1 << 10; // 1KB + options.level0_file_num_compaction_trigger = 3; + options.max_background_compactions = 3; + options.max_background_flushes = 3; + options.target_file_size_base = 1 * 1024; + options.compaction_options_universal.max_size_amplification_percent = 110; + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // Delay every compaction so multiple compactions will happen. + std::atomic num_compactions_running(0); + std::atomic has_parallel(false); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::Run():Start", [&](void* /*arg*/) { + if (num_compactions_running.fetch_add(1) > 0) { + has_parallel.store(true); + return; + } + for (int nwait = 0; nwait < 20000; nwait++) { + if (has_parallel.load() || num_compactions_running.load() > 1) { + has_parallel.store(true); + break; + } + env_->SleepForMicroseconds(1000); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::Run():End", + [&](void* /*arg*/) { num_compactions_running.fetch_add(-1); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + options = CurrentOptions(options); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + + Random rnd(301); + int num_keys = 30000; + for (int i = 0; i < num_keys * 2; i++) { + ASSERT_OK(Put(1, Key(i % num_keys), Key(i))); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ASSERT_EQ(num_compactions_running.load(), 0); + ASSERT_TRUE(has_parallel.load()); + + for (int i = num_keys; i < num_keys * 2; i++) { + ASSERT_EQ(Get(1, Key(i % num_keys)), Key(i)); + } + + // Reopen and check. + ReopenWithColumnFamilies({"default", "pikachu"}, options); + for (int i = num_keys; i < num_keys * 2; i++) { + ASSERT_EQ(Get(1, Key(i % num_keys)), Key(i)); + } +} + +TEST_P(DBTestUniversalCompactionParallel, PickByFileNumberBug) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = num_levels_; + options.write_buffer_size = 1 * 1024; // 1KB + options.level0_file_num_compaction_trigger = 7; + options.max_background_compactions = 2; + options.target_file_size_base = 1024 * 1024; // 1MB + + // Disable size amplifiction compaction + options.compaction_options_universal.max_size_amplification_percent = + UINT_MAX; + DestroyAndReopen(options); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBTestUniversalCompactionParallel::PickByFileNumberBug:0", + "BackgroundCallCompaction:0"}, + {"UniversalCompactionBuilder::PickCompaction:Return", + "DBTestUniversalCompactionParallel::PickByFileNumberBug:1"}, + {"DBTestUniversalCompactionParallel::PickByFileNumberBug:2", + "CompactionJob::Run():Start"}}); + + int total_picked_compactions = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "UniversalCompactionBuilder::PickCompaction:Return", [&](void* arg) { + if (arg) { + total_picked_compactions++; + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Write 7 files to trigger compaction + int key_idx = 1; + for (int i = 1; i <= 70; i++) { + std::string k = Key(key_idx++); + ASSERT_OK(Put(k, k)); + if (i % 10 == 0) { + ASSERT_OK(Flush()); + } + } + + // Wait for the 1st background compaction process to start + TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:0"); + TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:1"); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace(); + + // Write 3 files while 1st compaction is held + // These 3 files have different sizes to avoid compacting based on size_ratio + int num_keys = 1000; + for (int i = 0; i < 3; i++) { + for (int j = 1; j <= num_keys; j++) { + std::string k = Key(key_idx++); + ASSERT_OK(Put(k, k)); + } + ASSERT_OK(Flush()); + num_keys -= 100; + } + + // Hold the 1st compaction from finishing + TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:2"); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // There should only be one picked compaction as the score drops below one + // after the first one is picked. + EXPECT_EQ(total_picked_compactions, 1); + EXPECT_EQ(TotalTableFiles(), 4); + + // Stop SyncPoint and destroy the DB and reopen it again + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + key_idx = 1; + total_picked_compactions = 0; + DestroyAndReopen(options); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Write 7 files to trigger compaction + for (int i = 1; i <= 70; i++) { + std::string k = Key(key_idx++); + ASSERT_OK(Put(k, k)); + if (i % 10 == 0) { + ASSERT_OK(Flush()); + } + } + + // Wait for the 1st background compaction process to start + TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:0"); + TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:1"); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace(); + + // Write 8 files while 1st compaction is held + // These 8 files have different sizes to avoid compacting based on size_ratio + num_keys = 1000; + for (int i = 0; i < 8; i++) { + for (int j = 1; j <= num_keys; j++) { + std::string k = Key(key_idx++); + ASSERT_OK(Put(k, k)); + } + ASSERT_OK(Flush()); + num_keys -= 100; + } + + // Wait for the 2nd background compaction process to start + TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:0"); + TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:1"); + + // Hold the 1st and 2nd compaction from finishing + TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:2"); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // This time we will trigger a compaction because of size ratio and + // another compaction because of number of files that are not compacted + // greater than 7 + EXPECT_GE(total_picked_compactions, 2); +} + +INSTANTIATE_TEST_CASE_P(Parallel, DBTestUniversalCompactionParallel, + ::testing::Combine(::testing::Values(1, 10), + ::testing::Values(false))); +#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) + +TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.write_buffer_size = 105 << 10; // 105KB + options.arena_block_size = 4 << 10; // 4KB + options.target_file_size_base = 32 << 10; // 32KB + options.level0_file_num_compaction_trigger = 4; + options.num_levels = num_levels_; + options.compaction_options_universal.compression_size_percent = -1; + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + Random rnd(301); + int key_idx = 0; + + for (int num = 0; num < options.level0_file_num_compaction_trigger; num++) { + // Write 100KB (100 values, each 1K) + for (int i = 0; i < 100; i++) { + ASSERT_OK(Put(1, Key(key_idx), rnd.RandomString(990))); + key_idx++; + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); + + if (num < options.level0_file_num_compaction_trigger - 1) { + ASSERT_EQ(NumSortedRuns(1), num + 1); + } + } + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumSortedRuns(1), 1); +} + +TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.write_buffer_size = 105 << 10; // 105KB + options.arena_block_size = 4 << 10; // 4KB + options.target_file_size_base = 32 << 10; // 32KB + // trigger compaction if there are >= 4 files + options.level0_file_num_compaction_trigger = 4; + options.compaction_options_universal.size_ratio = 10; + options.compaction_options_universal.stop_style = + kCompactionStopStyleSimilarSize; + options.num_levels = num_levels_; + DestroyAndReopen(options); + + Random rnd(301); + int key_idx = 0; + + // Stage 1: + // Generate a set of files at level 0, but don't trigger level-0 + // compaction. + for (int num = 0; num < options.level0_file_num_compaction_trigger - 1; + num++) { + // Write 100KB (100 values, each 1K) + for (int i = 0; i < 100; i++) { + ASSERT_OK(Put(Key(key_idx), rnd.RandomString(990))); + key_idx++; + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(NumSortedRuns(), num + 1); + } + + // Generate one more file at level-0, which should trigger level-0 + // compaction. + for (int i = 0; i < 100; i++) { + ASSERT_OK(Put(Key(key_idx), rnd.RandomString(990))); + key_idx++; + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // Suppose each file flushed from mem table has size 1. Now we compact + // (level0_file_num_compaction_trigger+1)=4 files and should have a big + // file of size 4. + ASSERT_EQ(NumSortedRuns(), 1); + + // Stage 2: + // Now we have one file at level 0, with size 4. We also have some data in + // mem table. Let's continue generating new files at level 0, but don't + // trigger level-0 compaction. + // First, clean up memtable before inserting new data. This will generate + // a level-0 file, with size around 0.4 (according to previously written + // data amount). + ASSERT_OK(dbfull()->Flush(FlushOptions())); + for (int num = 0; num < options.level0_file_num_compaction_trigger - 3; + num++) { + // Write 110KB (11 values, each 10K) + for (int i = 0; i < 100; i++) { + ASSERT_OK(Put(Key(key_idx), rnd.RandomString(990))); + key_idx++; + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(NumSortedRuns(), num + 3); + } + + // Generate one more file at level-0, which should trigger level-0 + // compaction. + for (int i = 0; i < 100; i++) { + ASSERT_OK(Put(Key(key_idx), rnd.RandomString(990))); + key_idx++; + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // Before compaction, we have 4 files at level 0, with size 4, 0.4, 1, 1. + // After compaction, we should have 3 files, with size 4, 0.4, 2. + ASSERT_EQ(NumSortedRuns(), 3); + // Stage 3: + // Now we have 3 files at level 0, with size 4, 0.4, 2. Generate one + // more file at level-0, which should trigger level-0 compaction. + for (int i = 0; i < 100; i++) { + ASSERT_OK(Put(Key(key_idx), rnd.RandomString(990))); + key_idx++; + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // Level-0 compaction is triggered, but no file will be picked up. + ASSERT_EQ(NumSortedRuns(), 4); +} + +TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) { + if (!Snappy_Supported()) { + return; + } + + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.write_buffer_size = 100 << 10; // 100KB + options.target_file_size_base = 32 << 10; // 32KB + options.level0_file_num_compaction_trigger = 2; + options.num_levels = num_levels_; + options.compaction_options_universal.compression_size_percent = 70; + DestroyAndReopen(options); + + Random rnd(301); + int key_idx = 0; + + // The first compaction (2) is compressed. + for (int num = 0; num < 2; num++) { + // Write 110KB (11 values, each 10K) + for (int i = 0; i < 11; i++) { + ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000))); + key_idx++; + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + ASSERT_LT(TotalSize(), 110000U * 2 * 0.9); + + // The second compaction (4) is compressed + for (int num = 0; num < 2; num++) { + // Write 110KB (11 values, each 10K) + for (int i = 0; i < 11; i++) { + ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000))); + key_idx++; + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + ASSERT_LT(TotalSize(), 110000 * 4 * 0.9); + + // The third compaction (2 4) is compressed since this time it is + // (1 1 3.2) and 3.2/5.2 doesn't reach ratio. + for (int num = 0; num < 2; num++) { + // Write 110KB (11 values, each 10K) + for (int i = 0; i < 11; i++) { + ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000))); + key_idx++; + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + ASSERT_LT(TotalSize(), 110000 * 6 * 0.9); + + // When we start for the compaction up to (2 4 8), the latest + // compressed is not compressed. + for (int num = 0; num < 8; num++) { + // Write 110KB (11 values, each 10K) + for (int i = 0; i < 11; i++) { + ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000))); + key_idx++; + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + ASSERT_GT(TotalSize(), 110000 * 11 * 0.8 + 110000 * 2); +} + +TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio2) { + if (!Snappy_Supported()) { + return; + } + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.write_buffer_size = 100 << 10; // 100KB + options.target_file_size_base = 32 << 10; // 32KB + options.level0_file_num_compaction_trigger = 2; + options.num_levels = num_levels_; + options.compaction_options_universal.compression_size_percent = 95; + DestroyAndReopen(options); + + Random rnd(301); + int key_idx = 0; + + // When we start for the compaction up to (2 4 8), the latest + // compressed is compressed given the size ratio to compress. + for (int num = 0; num < 14; num++) { + // Write 120KB (12 values, each 10K) + for (int i = 0; i < 12; i++) { + ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000))); + key_idx++; + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + ASSERT_LT(TotalSize(), 120000U * 12 * 0.82 + 120000 * 2); +} + +#if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) +// Test that checks trivial move in universal compaction +TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest1) { + int32_t trivial_move = 0; + int32_t non_trivial_move = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:TrivialMove", + [&](void* /*arg*/) { trivial_move++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial", [&](void* arg) { + non_trivial_move++; + ASSERT_TRUE(arg != nullptr); + int output_level = *(static_cast(arg)); + ASSERT_EQ(output_level, 0); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.compaction_options_universal.allow_trivial_move = true; + options.num_levels = 2; + options.write_buffer_size = 100 << 10; // 100KB + options.level0_file_num_compaction_trigger = 3; + options.max_background_compactions = 1; + options.target_file_size_base = 32 * 1024; + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // Trigger compaction if size amplification exceeds 110% + options.compaction_options_universal.max_size_amplification_percent = 110; + options = CurrentOptions(options); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + + Random rnd(301); + int num_keys = 250000; + for (int i = 0; i < num_keys; i++) { + ASSERT_OK(Put(1, Key(i), Key(i))); + } + std::vector values; + + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_GT(trivial_move, 0); + ASSERT_GT(non_trivial_move, 0); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} +// Test that checks trivial move in universal compaction +TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) { + int32_t trivial_move = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:TrivialMove", + [&](void* /*arg*/) { trivial_move++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:NonTrivial", [&](void* arg) { + ASSERT_TRUE(arg != nullptr); + int output_level = *(static_cast(arg)); + ASSERT_EQ(output_level, 0); + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.compaction_options_universal.allow_trivial_move = true; + options.num_levels = 15; + options.write_buffer_size = 100 << 10; // 100KB + options.level0_file_num_compaction_trigger = 8; + options.max_background_compactions = 2; + options.target_file_size_base = 64 * 1024; + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + // Trigger compaction if size amplification exceeds 110% + options.compaction_options_universal.max_size_amplification_percent = 110; + options = CurrentOptions(options); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + + Random rnd(301); + int num_keys = 500000; + for (int i = 0; i < num_keys; i++) { + ASSERT_OK(Put(1, Key(i), Key(i))); + } + std::vector values; + + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_GT(trivial_move, 0); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} +#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) + +TEST_P(DBTestUniversalCompaction, UniversalCompactionFourPaths) { + Options options = CurrentOptions(); + options.db_paths.emplace_back(dbname_, 300 * 1024); + options.db_paths.emplace_back(dbname_ + "_2", 300 * 1024); + options.db_paths.emplace_back(dbname_ + "_3", 500 * 1024); + options.db_paths.emplace_back(dbname_ + "_4", 1024 * 1024 * 1024); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + options.compaction_style = kCompactionStyleUniversal; + options.compaction_options_universal.size_ratio = 5; + options.write_buffer_size = 111 << 10; // 114KB + options.arena_block_size = 4 << 10; + options.level0_file_num_compaction_trigger = 2; + options.num_levels = 1; + + std::vector filenames; + if (env_->GetChildren(options.db_paths[1].path, &filenames).ok()) { + // Delete archival files. + for (size_t i = 0; i < filenames.size(); ++i) { + ASSERT_OK( + env_->DeleteFile(options.db_paths[1].path + "/" + filenames[i])); + } + ASSERT_OK(env_->DeleteDir(options.db_paths[1].path)); + } + Reopen(options); + + Random rnd(301); + int key_idx = 0; + + // First three 110KB files are not going to second path. + // After that, (100K, 200K) + for (int num = 0; num < 3; num++) { + GenerateNewFile(&rnd, &key_idx); + } + + // Another 110KB triggers a compaction to 400K file to second path + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[2].path)); + + // (1, 4) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[2].path)); + ASSERT_EQ(1, GetSstFileCount(dbname_)); + + // (1,1,4) -> (2, 4) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[2].path)); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(0, GetSstFileCount(dbname_)); + + // (1, 2, 4) -> (3, 4) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[2].path)); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(0, GetSstFileCount(dbname_)); + + // (1, 3, 4) -> (8) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[3].path)); + + // (1, 8) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[3].path)); + ASSERT_EQ(1, GetSstFileCount(dbname_)); + + // (1, 1, 8) -> (2, 8) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[3].path)); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); + + // (1, 2, 8) -> (3, 8) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[3].path)); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(0, GetSstFileCount(dbname_)); + + // (1, 3, 8) -> (4, 8) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[2].path)); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[3].path)); + + // (1, 4, 8) -> (5, 8) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[3].path)); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[2].path)); + ASSERT_EQ(0, GetSstFileCount(dbname_)); + + for (int i = 0; i < key_idx; i++) { + auto v = Get(Key(i)); + ASSERT_NE(v, "NOT_FOUND"); + ASSERT_TRUE(v.size() == 1 || v.size() == 990); + } + + Reopen(options); + + for (int i = 0; i < key_idx; i++) { + auto v = Get(Key(i)); + ASSERT_NE(v, "NOT_FOUND"); + ASSERT_TRUE(v.size() == 1 || v.size() == 990); + } + + Destroy(options); +} + +TEST_P(DBTestUniversalCompaction, UniversalCompactionCFPathUse) { + Options options = CurrentOptions(); + options.db_paths.emplace_back(dbname_, 300 * 1024); + options.db_paths.emplace_back(dbname_ + "_2", 300 * 1024); + options.db_paths.emplace_back(dbname_ + "_3", 500 * 1024); + options.db_paths.emplace_back(dbname_ + "_4", 1024 * 1024 * 1024); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + options.compaction_style = kCompactionStyleUniversal; + options.compaction_options_universal.size_ratio = 10; + options.write_buffer_size = 111 << 10; // 114KB + options.arena_block_size = 4 << 10; + options.level0_file_num_compaction_trigger = 2; + options.num_levels = 1; + + std::vector option_vector; + option_vector.emplace_back(options); + ColumnFamilyOptions cf_opt1(options), cf_opt2(options); + // Configure CF1 specific paths. + cf_opt1.cf_paths.emplace_back(dbname_ + "cf1", 300 * 1024); + cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_2", 300 * 1024); + cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_3", 500 * 1024); + cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_4", 1024 * 1024 * 1024); + option_vector.emplace_back(DBOptions(options), cf_opt1); + CreateColumnFamilies({"one"}, option_vector[1]); + + // Configura CF2 specific paths. + cf_opt2.cf_paths.emplace_back(dbname_ + "cf2", 300 * 1024); + cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_2", 300 * 1024); + cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_3", 500 * 1024); + cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_4", 1024 * 1024 * 1024); + option_vector.emplace_back(DBOptions(options), cf_opt2); + CreateColumnFamilies({"two"}, option_vector[2]); + + ReopenWithColumnFamilies({"default", "one", "two"}, option_vector); + + Random rnd(301); + int key_idx = 0; + int key_idx1 = 0; + int key_idx2 = 0; + + auto generate_file = [&]() { + GenerateNewFile(0, &rnd, &key_idx); + GenerateNewFile(1, &rnd, &key_idx1); + GenerateNewFile(2, &rnd, &key_idx2); + }; + + auto check_sstfilecount = [&](int path_id, int expected) { + ASSERT_EQ(expected, GetSstFileCount(options.db_paths[path_id].path)); + ASSERT_EQ(expected, GetSstFileCount(cf_opt1.cf_paths[path_id].path)); + ASSERT_EQ(expected, GetSstFileCount(cf_opt2.cf_paths[path_id].path)); + }; + + auto check_getvalues = [&]() { + for (int i = 0; i < key_idx; i++) { + auto v = Get(0, Key(i)); + ASSERT_NE(v, "NOT_FOUND"); + ASSERT_TRUE(v.size() == 1 || v.size() == 990); + } + + for (int i = 0; i < key_idx1; i++) { + auto v = Get(1, Key(i)); + ASSERT_NE(v, "NOT_FOUND"); + ASSERT_TRUE(v.size() == 1 || v.size() == 990); + } + + for (int i = 0; i < key_idx2; i++) { + auto v = Get(2, Key(i)); + ASSERT_NE(v, "NOT_FOUND"); + ASSERT_TRUE(v.size() == 1 || v.size() == 990); + } + }; + + // First three 110KB files are not going to second path. + // After that, (100K, 200K) + for (int num = 0; num < 3; num++) { + generate_file(); + } + + // Another 110KB triggers a compaction to 400K file to second path + generate_file(); + check_sstfilecount(2, 1); + + // (1, 4) + generate_file(); + check_sstfilecount(2, 1); + check_sstfilecount(0, 1); + + // (1,1,4) -> (2, 4) + generate_file(); + check_sstfilecount(2, 1); + check_sstfilecount(1, 1); + check_sstfilecount(0, 0); + + // (1, 2, 4) -> (3, 4) + generate_file(); + check_sstfilecount(2, 1); + check_sstfilecount(1, 1); + check_sstfilecount(0, 0); + + // (1, 3, 4) -> (8) + generate_file(); + check_sstfilecount(3, 1); + + // (1, 8) + generate_file(); + check_sstfilecount(3, 1); + check_sstfilecount(0, 1); + + // (1, 1, 8) -> (2, 8) + generate_file(); + check_sstfilecount(3, 1); + check_sstfilecount(1, 1); + + // (1, 2, 8) -> (3, 8) + generate_file(); + check_sstfilecount(3, 1); + check_sstfilecount(1, 1); + check_sstfilecount(0, 0); + + // (1, 3, 8) -> (4, 8) + generate_file(); + check_sstfilecount(2, 1); + check_sstfilecount(3, 1); + + // (1, 4, 8) -> (5, 8) + generate_file(); + check_sstfilecount(3, 1); + check_sstfilecount(2, 1); + check_sstfilecount(0, 0); + + check_getvalues(); + + ReopenWithColumnFamilies({"default", "one", "two"}, option_vector); + + check_getvalues(); + + Destroy(options, true); +} + +TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) { + std::function verify_func = [&](int num_keys_in_db) { + std::string keys_in_db; + Iterator* iter = dbfull()->NewIterator(ReadOptions(), handles_[1]); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + keys_in_db.append(iter->key().ToString()); + keys_in_db.push_back(','); + } + delete iter; + + std::string expected_keys; + for (int i = 0; i <= num_keys_in_db; i++) { + expected_keys.append(Key(i)); + expected_keys.push_back(','); + } + + ASSERT_EQ(keys_in_db, expected_keys); + }; + + Random rnd(301); + int max_key1 = 200; + int max_key2 = 600; + int max_key3 = 800; + const int KNumKeysPerFile = 10; + + // Stage 1: open a DB with universal compaction, num_levels=1 + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = 1; + options.write_buffer_size = 200 << 10; // 200KB + options.level0_file_num_compaction_trigger = 3; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(KNumKeysPerFile)); + options = CurrentOptions(options); + CreateAndReopenWithCF({"pikachu"}, options); + + for (int i = 0; i <= max_key1; i++) { + // each value is 10K + ASSERT_OK(Put(1, Key(i), rnd.RandomString(10000))); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // Stage 2: reopen with universal compaction, num_levels=4 + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = 4; + options = CurrentOptions(options); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + + verify_func(max_key1); + + // Insert more keys + for (int i = max_key1 + 1; i <= max_key2; i++) { + // each value is 10K + ASSERT_OK(Put(1, Key(i), rnd.RandomString(10000))); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + verify_func(max_key2); + // Compaction to non-L0 has happened. + ASSERT_GT(NumTableFilesAtLevel(options.num_levels - 1, 1), 0); + + // Stage 3: Revert it back to one level and revert to num_levels=1. + options.num_levels = 4; + options.target_file_size_base = INT_MAX; + ReopenWithColumnFamilies({"default", "pikachu"}, options); + // Compact all to level 0 + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 0; + compact_options.exclusive_manual_compaction = exclusive_manual_compaction_; + ASSERT_OK( + dbfull()->CompactRange(compact_options, handles_[1], nullptr, nullptr)); + // Need to restart it once to remove higher level records in manifest. + ReopenWithColumnFamilies({"default", "pikachu"}, options); + // Final reopen + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = 1; + options = CurrentOptions(options); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + + // Insert more keys + for (int i = max_key2 + 1; i <= max_key3; i++) { + // each value is 10K + ASSERT_OK(Put(1, Key(i), rnd.RandomString(10000))); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + ASSERT_OK(Flush(1)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + verify_func(max_key3); +} + +TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) { + if (!Snappy_Supported()) { + return; + } + Options options = CurrentOptions(); + options.db_paths.emplace_back(dbname_, 500 * 1024); + options.db_paths.emplace_back(dbname_ + "_2", 1024 * 1024 * 1024); + options.compaction_style = kCompactionStyleUniversal; + options.compaction_options_universal.size_ratio = 5; + options.write_buffer_size = 111 << 10; // 114KB + options.arena_block_size = 4 << 10; + options.level0_file_num_compaction_trigger = 2; + options.num_levels = 1; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + + std::vector filenames; + if (env_->GetChildren(options.db_paths[1].path, &filenames).ok()) { + // Delete archival files. + for (size_t i = 0; i < filenames.size(); ++i) { + ASSERT_OK( + env_->DeleteFile(options.db_paths[1].path + "/" + filenames[i])); + } + ASSERT_OK(env_->DeleteDir(options.db_paths[1].path)); + } + Reopen(options); + + Random rnd(301); + int key_idx = 0; + + // First three 110KB files are not going to second path. + // After that, (100K, 200K) + for (int num = 0; num < 3; num++) { + GenerateNewFile(&rnd, &key_idx); + } + + // Another 110KB triggers a compaction to 400K file to second path + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); + + // (1, 4) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(1, GetSstFileCount(dbname_)); + + // (1,1,4) -> (2, 4) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(1, GetSstFileCount(dbname_)); + + // (1, 2, 4) -> (3, 4) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(2, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(0, GetSstFileCount(dbname_)); + + // (1, 3, 4) -> (8) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(0, GetSstFileCount(dbname_)); + + // (1, 8) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(1, GetSstFileCount(dbname_)); + + // (1, 1, 8) -> (2, 8) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(1, GetSstFileCount(dbname_)); + + // (1, 2, 8) -> (3, 8) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(2, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(0, GetSstFileCount(dbname_)); + + // (1, 3, 8) -> (4, 8) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(2, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(0, GetSstFileCount(dbname_)); + + // (1, 4, 8) -> (5, 8) + GenerateNewFile(&rnd, &key_idx); + ASSERT_EQ(2, GetSstFileCount(options.db_paths[1].path)); + ASSERT_EQ(0, GetSstFileCount(dbname_)); + + for (int i = 0; i < key_idx; i++) { + auto v = Get(Key(i)); + ASSERT_NE(v, "NOT_FOUND"); + ASSERT_TRUE(v.size() == 1 || v.size() == 990); + } + + Reopen(options); + + for (int i = 0; i < key_idx; i++) { + auto v = Get(Key(i)); + ASSERT_NE(v, "NOT_FOUND"); + ASSERT_TRUE(v.size() == 1 || v.size() == 990); + } + + Destroy(options); +} + +TEST_P(DBTestUniversalCompaction, ConcurrentBottomPriLowPriCompactions) { + if (num_levels_ == 1) { + // for single-level universal, everything's bottom level so nothing should + // be executed in bottom-pri thread pool. + return; + } + const int kNumFilesTrigger = 3; + Env::Default()->SetBackgroundThreads(1, Env::Priority::BOTTOM); + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.max_background_compactions = 2; + options.num_levels = num_levels_; + options.write_buffer_size = 100 << 10; // 100KB + options.target_file_size_base = 32 << 10; // 32KB + options.level0_file_num_compaction_trigger = kNumFilesTrigger; + // Trigger compaction if size amplification exceeds 110% + options.compaction_options_universal.max_size_amplification_percent = 110; + DestroyAndReopen(options); + + // Need to get a token to enable compaction parallelism up to + // `max_background_compactions` jobs. + auto pressure_token = + dbfull()->TEST_write_controler().GetCompactionPressureToken(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {// wait for the full compaction to be picked before adding files intended + // for the second one. + {"DBImpl::BackgroundCompaction:ForwardToBottomPriPool", + "DBTestUniversalCompaction:ConcurrentBottomPriLowPriCompactions:0"}, + // the full (bottom-pri) compaction waits until a partial (low-pri) + // compaction has started to verify they can run in parallel. + {"DBImpl::BackgroundCompaction:NonTrivial", + "DBImpl::BGWorkBottomCompaction"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + Random rnd(301); + for (int i = 0; i < 2; ++i) { + for (int num = 0; num < kNumFilesTrigger; num++) { + int key_idx = 0; + GenerateNewFile(&rnd, &key_idx, true /* no_wait */); + // use no_wait above because that one waits for flush and compaction. We + // don't want to wait for compaction because the full compaction is + // intentionally blocked while more files are flushed. + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + if (i == 0) { + TEST_SYNC_POINT( + "DBTestUniversalCompaction:ConcurrentBottomPriLowPriCompactions:0"); + } + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // First compaction should output to bottom level. Second should output to L0 + // since older L0 files pending compaction prevent it from being placed lower. + ASSERT_EQ(NumSortedRuns(), 2); + ASSERT_GT(NumTableFilesAtLevel(0), 0); + ASSERT_GT(NumTableFilesAtLevel(num_levels_ - 1), 0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + Env::Default()->SetBackgroundThreads(0, Env::Priority::BOTTOM); +} + +TEST_P(DBTestUniversalCompaction, RecalculateScoreAfterPicking) { + // Regression test for extra compactions scheduled. Once enough compactions + // have been scheduled to bring the score below one, we should stop + // scheduling more; otherwise, other CFs/DBs may be delayed unnecessarily. + const int kNumFilesTrigger = 8; + Options options = CurrentOptions(); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); + options.compaction_options_universal.max_merge_width = kNumFilesTrigger / 2; + options.compaction_options_universal.max_size_amplification_percent = + static_cast(-1); + options.compaction_style = kCompactionStyleUniversal; + options.level0_file_num_compaction_trigger = kNumFilesTrigger; + options.num_levels = num_levels_; + Reopen(options); + + std::atomic num_compactions_attempted(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:Start", + [&](void* /*arg*/) { ++num_compactions_attempted; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Random rnd(301); + for (int num = 0; num < kNumFilesTrigger; num++) { + ASSERT_EQ(NumSortedRuns(), num); + int key_idx = 0; + GenerateNewFile(&rnd, &key_idx); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // Compacting the first four files was enough to bring the score below one so + // there's no need to schedule any more compactions. + ASSERT_EQ(1, num_compactions_attempted); + ASSERT_EQ(NumSortedRuns(), 5); +} + +TEST_P(DBTestUniversalCompaction, FinalSortedRunCompactFilesConflict) { + // Regression test for conflict between: + // (1) Running CompactFiles including file in the final sorted run; and + // (2) Picking universal size-amp-triggered compaction, which always includes + // the final sorted run. + if (exclusive_manual_compaction_) { + return; + } + + Options opts = CurrentOptions(); + opts.compaction_style = kCompactionStyleUniversal; + opts.compaction_options_universal.max_size_amplification_percent = 50; + opts.compaction_options_universal.min_merge_width = 2; + opts.compression = kNoCompression; + opts.level0_file_num_compaction_trigger = 2; + opts.max_background_compactions = 2; + opts.num_levels = num_levels_; + Reopen(opts); + + // make sure compaction jobs can be parallelized + auto stop_token = + dbfull()->TEST_write_controler().GetCompactionPressureToken(); + + ASSERT_OK(Put("key", "val")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(NumTableFilesAtLevel(num_levels_ - 1), 1); + ColumnFamilyMetaData cf_meta; + ColumnFamilyHandle* default_cfh = db_->DefaultColumnFamily(); + dbfull()->GetColumnFamilyMetaData(default_cfh, &cf_meta); + ASSERT_EQ(1, cf_meta.levels[num_levels_ - 1].files.size()); + std::string first_sst_filename = + cf_meta.levels[num_levels_ - 1].files[0].name; + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"CompactFilesImpl:0", + "DBTestUniversalCompaction:FinalSortedRunCompactFilesConflict:0"}, + {"DBImpl::BackgroundCompaction():AfterPickCompaction", + "CompactFilesImpl:1"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + port::Thread compact_files_thread([&]() { + ASSERT_OK(dbfull()->CompactFiles(CompactionOptions(), default_cfh, + {first_sst_filename}, num_levels_ - 1)); + }); + + TEST_SYNC_POINT( + "DBTestUniversalCompaction:FinalSortedRunCompactFilesConflict:0"); + for (int i = 0; i < 2; ++i) { + ASSERT_OK(Put("key", "val")); + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + compact_files_thread.join(); +} + +INSTANTIATE_TEST_CASE_P(NumLevels, DBTestUniversalCompaction, + ::testing::Combine(::testing::Values(1, 3, 5), + ::testing::Bool())); + +class DBTestUniversalManualCompactionOutputPathId + : public DBTestUniversalCompactionBase { + public: + DBTestUniversalManualCompactionOutputPathId() + : DBTestUniversalCompactionBase( + "/db_universal_compaction_manual_pid_test") {} +}; + +TEST_P(DBTestUniversalManualCompactionOutputPathId, + ManualCompactionOutputPathId) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.db_paths.emplace_back(dbname_, 1000000000); + options.db_paths.emplace_back(dbname_ + "_2", 1000000000); + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = num_levels_; + options.target_file_size_base = 1 << 30; // Big size + options.level0_file_num_compaction_trigger = 10; + Destroy(options); + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + MakeTables(3, "p", "q", 1); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(2, TotalLiveFiles(1)); + ASSERT_EQ(2, GetSstFileCount(options.db_paths[0].path)); + ASSERT_EQ(0, GetSstFileCount(options.db_paths[1].path)); + + // Full compaction to DB path 0 + CompactRangeOptions compact_options; + compact_options.target_path_id = 1; + compact_options.exclusive_manual_compaction = exclusive_manual_compaction_; + ASSERT_OK(db_->CompactRange(compact_options, handles_[1], nullptr, nullptr)); + ASSERT_EQ(1, TotalLiveFiles(1)); + ASSERT_EQ(0, TotalLiveFilesAtPath(1, options.db_paths[0].path)); + ASSERT_EQ(1, TotalLiveFilesAtPath(1, options.db_paths[1].path)); + + ReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu"}, options); + ASSERT_EQ(1, TotalLiveFiles(1)); + ASSERT_EQ(0, TotalLiveFilesAtPath(1, options.db_paths[0].path)); + ASSERT_EQ(1, TotalLiveFilesAtPath(1, options.db_paths[1].path)); + + MakeTables(1, "p", "q", 1); + ASSERT_EQ(2, TotalLiveFiles(1)); + ASSERT_EQ(1, TotalLiveFilesAtPath(1, options.db_paths[0].path)); + ASSERT_EQ(1, TotalLiveFilesAtPath(1, options.db_paths[1].path)); + + ReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu"}, options); + ASSERT_EQ(2, TotalLiveFiles(1)); + ASSERT_EQ(1, TotalLiveFilesAtPath(1, options.db_paths[0].path)); + ASSERT_EQ(1, TotalLiveFilesAtPath(1, options.db_paths[1].path)); + + // Full compaction to DB path 0 + compact_options.target_path_id = 0; + compact_options.exclusive_manual_compaction = exclusive_manual_compaction_; + ASSERT_OK(db_->CompactRange(compact_options, handles_[1], nullptr, nullptr)); + ASSERT_EQ(1, TotalLiveFiles(1)); + ASSERT_EQ(1, TotalLiveFilesAtPath(1, options.db_paths[0].path)); + ASSERT_EQ(0, TotalLiveFilesAtPath(1, options.db_paths[1].path)); + + // Fail when compacting to an invalid path ID + compact_options.target_path_id = 2; + compact_options.exclusive_manual_compaction = exclusive_manual_compaction_; + ASSERT_TRUE(db_->CompactRange(compact_options, handles_[1], nullptr, nullptr) + .IsInvalidArgument()); +} + +INSTANTIATE_TEST_CASE_P(OutputPathId, + DBTestUniversalManualCompactionOutputPathId, + ::testing::Combine(::testing::Values(1, 8), + ::testing::Bool())); + +TEST_F(DBTestUniversalCompaction2, BasicL0toL1) { + const int kNumKeys = 3000; + const int kWindowSize = 100; + const int kNumDelsTrigger = 90; + + Options opts = CurrentOptions(); + opts.table_properties_collector_factories.emplace_back( + NewCompactOnDeletionCollectorFactory(kWindowSize, kNumDelsTrigger)); + opts.compaction_style = kCompactionStyleUniversal; + opts.level0_file_num_compaction_trigger = 2; + opts.compression = kNoCompression; + opts.compaction_options_universal.size_ratio = 10; + opts.compaction_options_universal.min_merge_width = 2; + opts.compaction_options_universal.max_size_amplification_percent = 200; + Reopen(opts); + + // add an L1 file to prevent tombstones from dropping due to obsolescence + // during flush + int i; + for (i = 0; i < 2000; ++i) { + ASSERT_OK(Put(Key(i), "val")); + } + ASSERT_OK(Flush()); + // MoveFilesToLevel(6); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + for (i = 1999; i < kNumKeys; ++i) { + if (i >= kNumKeys - kWindowSize && + i < kNumKeys - kWindowSize + kNumDelsTrigger) { + ASSERT_OK(Delete(Key(i))); + } else { + ASSERT_OK(Put(Key(i), "val")); + } + } + ASSERT_OK(Flush()); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_GT(NumTableFilesAtLevel(6), 0); +} + +#if defined(ENABLE_SINGLE_LEVEL_DTC) +TEST_F(DBTestUniversalCompaction2, SingleLevel) { + const int kNumKeys = 3000; + const int kWindowSize = 100; + const int kNumDelsTrigger = 90; + + Options opts = CurrentOptions(); + opts.table_properties_collector_factories.emplace_back( + NewCompactOnDeletionCollectorFactory(kWindowSize, kNumDelsTrigger)); + opts.compaction_style = kCompactionStyleUniversal; + opts.level0_file_num_compaction_trigger = 2; + opts.compression = kNoCompression; + opts.num_levels = 1; + opts.compaction_options_universal.size_ratio = 10; + opts.compaction_options_universal.min_merge_width = 2; + opts.compaction_options_universal.max_size_amplification_percent = 200; + Reopen(opts); + + // add an L1 file to prevent tombstones from dropping due to obsolescence + // during flush + int i; + for (i = 0; i < 2000; ++i) { + ASSERT_OK(Put(Key(i), "val")); + } + ASSERT_OK(Flush()); + + for (i = 1999; i < kNumKeys; ++i) { + if (i >= kNumKeys - kWindowSize && + i < kNumKeys - kWindowSize + kNumDelsTrigger) { + ASSERT_OK(Delete(Key(i))); + } else { + ASSERT_OK(Put(Key(i), "val")); + } + } + ASSERT_OK(Flush()(; + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); +} +#endif // ENABLE_SINGLE_LEVEL_DTC + +TEST_F(DBTestUniversalCompaction2, MultipleLevels) { + const int kWindowSize = 100; + const int kNumDelsTrigger = 90; + + Options opts = CurrentOptions(); + opts.table_properties_collector_factories.emplace_back( + NewCompactOnDeletionCollectorFactory(kWindowSize, kNumDelsTrigger)); + opts.compaction_style = kCompactionStyleUniversal; + opts.level0_file_num_compaction_trigger = 4; + opts.compression = kNoCompression; + opts.compaction_options_universal.size_ratio = 10; + opts.compaction_options_universal.min_merge_width = 2; + opts.compaction_options_universal.max_size_amplification_percent = 200; + Reopen(opts); + + // add an L1 file to prevent tombstones from dropping due to obsolescence + // during flush + int i; + for (i = 0; i < 500; ++i) { + ASSERT_OK(Put(Key(i), "val")); + } + ASSERT_OK(Flush()); + for (i = 500; i < 1000; ++i) { + ASSERT_OK(Put(Key(i), "val")); + } + ASSERT_OK(Flush()); + for (i = 1000; i < 1500; ++i) { + ASSERT_OK(Put(Key(i), "val")); + } + ASSERT_OK(Flush()); + for (i = 1500; i < 2000; ++i) { + ASSERT_OK(Put(Key(i), "val")); + } + ASSERT_OK(Flush()); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_GT(NumTableFilesAtLevel(6), 0); + + for (i = 1999; i < 2333; ++i) { + ASSERT_OK(Put(Key(i), "val")); + } + ASSERT_OK(Flush()); + for (i = 2333; i < 2666; ++i) { + ASSERT_OK(Put(Key(i), "val")); + } + ASSERT_OK(Flush()); + for (i = 2666; i < 2999; ++i) { + ASSERT_OK(Put(Key(i), "val")); + } + ASSERT_OK(Flush()); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_GT(NumTableFilesAtLevel(6), 0); + ASSERT_GT(NumTableFilesAtLevel(5), 0); + + for (i = 1900; i < 2100; ++i) { + ASSERT_OK(Delete(Key(i))); + } + ASSERT_OK(Flush()); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_EQ(0, NumTableFilesAtLevel(1)); + ASSERT_EQ(0, NumTableFilesAtLevel(2)); + ASSERT_EQ(0, NumTableFilesAtLevel(3)); + ASSERT_EQ(0, NumTableFilesAtLevel(4)); + ASSERT_EQ(0, NumTableFilesAtLevel(5)); + ASSERT_GT(NumTableFilesAtLevel(6), 0); +} + +TEST_F(DBTestUniversalCompaction2, OverlappingL0) { + const int kWindowSize = 100; + const int kNumDelsTrigger = 90; + + Options opts = CurrentOptions(); + opts.table_properties_collector_factories.emplace_back( + NewCompactOnDeletionCollectorFactory(kWindowSize, kNumDelsTrigger)); + opts.compaction_style = kCompactionStyleUniversal; + opts.level0_file_num_compaction_trigger = 5; + opts.compression = kNoCompression; + opts.compaction_options_universal.size_ratio = 10; + opts.compaction_options_universal.min_merge_width = 2; + opts.compaction_options_universal.max_size_amplification_percent = 200; + Reopen(opts); + + // add an L1 file to prevent tombstones from dropping due to obsolescence + // during flush + int i; + for (i = 0; i < 2000; ++i) { + ASSERT_OK(Put(Key(i), "val")); + } + ASSERT_OK(Flush()); + for (i = 2000; i < 3000; ++i) { + ASSERT_OK(Put(Key(i), "val")); + } + ASSERT_OK(Flush()); + for (i = 3500; i < 4000; ++i) { + ASSERT_OK(Put(Key(i), "val")); + } + ASSERT_OK(Flush()); + for (i = 2900; i < 3100; ++i) { + ASSERT_OK(Delete(Key(i))); + } + ASSERT_OK(Flush()); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(2, NumTableFilesAtLevel(0)); + ASSERT_GT(NumTableFilesAtLevel(6), 0); +} + +TEST_F(DBTestUniversalCompaction2, IngestBehind) { + const int kNumKeys = 3000; + const int kWindowSize = 100; + const int kNumDelsTrigger = 90; + + Options opts = CurrentOptions(); + opts.table_properties_collector_factories.emplace_back( + NewCompactOnDeletionCollectorFactory(kWindowSize, kNumDelsTrigger)); + opts.compaction_style = kCompactionStyleUniversal; + opts.level0_file_num_compaction_trigger = 2; + opts.compression = kNoCompression; + opts.allow_ingest_behind = true; + opts.compaction_options_universal.size_ratio = 10; + opts.compaction_options_universal.min_merge_width = 2; + opts.compaction_options_universal.max_size_amplification_percent = 200; + Reopen(opts); + + // add an L1 file to prevent tombstones from dropping due to obsolescence + // during flush + int i; + for (i = 0; i < 2000; ++i) { + ASSERT_OK(Put(Key(i), "val")); + } + ASSERT_OK(Flush()); + // MoveFilesToLevel(6); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + for (i = 1999; i < kNumKeys; ++i) { + if (i >= kNumKeys - kWindowSize && + i < kNumKeys - kWindowSize + kNumDelsTrigger) { + ASSERT_OK(Delete(Key(i))); + } else { + ASSERT_OK(Put(Key(i), "val")); + } + } + ASSERT_OK(Flush()); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_EQ(0, NumTableFilesAtLevel(6)); + ASSERT_GT(NumTableFilesAtLevel(5), 0); +} + +TEST_F(DBTestUniversalCompaction2, PeriodicCompactionDefault) { + Options options; + options.compaction_style = kCompactionStyleUniversal; + options.env = env_; + KeepFilterFactory* filter = new KeepFilterFactory(true); + options.compaction_filter_factory.reset(filter); + Reopen(options); + ASSERT_EQ(30 * 24 * 60 * 60, + dbfull()->GetOptions().periodic_compaction_seconds); + + KeepFilter df; + options.compaction_filter_factory.reset(); + options.compaction_filter = &df; + Reopen(options); + ASSERT_EQ(30 * 24 * 60 * 60, + dbfull()->GetOptions().periodic_compaction_seconds); + + options.ttl = 60 * 24 * 60 * 60; + options.compaction_filter = nullptr; + Reopen(options); + ASSERT_EQ(30 * 24 * 60 * 60, + dbfull()->GetOptions().periodic_compaction_seconds); + + options.periodic_compaction_seconds = 45 * 24 * 60 * 60; + options.ttl = 50 * 24 * 60 * 60; + Reopen(options); + ASSERT_EQ(45 * 24 * 60 * 60, + dbfull()->GetOptions().periodic_compaction_seconds); + + options.periodic_compaction_seconds = 0; + options.ttl = 50 * 24 * 60 * 60; + Reopen(options); + ASSERT_EQ(50 * 24 * 60 * 60, + dbfull()->GetOptions().periodic_compaction_seconds); +} + +TEST_F(DBTestUniversalCompaction2, PeriodicCompaction) { + Options opts = CurrentOptions(); + opts.env = env_; + opts.compaction_style = kCompactionStyleUniversal; + opts.level0_file_num_compaction_trigger = 10; + opts.max_open_files = -1; + opts.compaction_options_universal.size_ratio = 10; + opts.compaction_options_universal.min_merge_width = 2; + opts.compaction_options_universal.max_size_amplification_percent = 200; + opts.periodic_compaction_seconds = 48 * 60 * 60; // 2 days + opts.num_levels = 5; + env_->SetMockSleep(); + Reopen(opts); + + // NOTE: Presumed unnecessary and removed: resetting mock time in env + + int periodic_compactions = 0; + int start_level = -1; + int output_level = -1; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "UniversalCompactionPicker::PickPeriodicCompaction:Return", + [&](void* arg) { + Compaction* compaction = reinterpret_cast(arg); + ASSERT_TRUE(arg != nullptr); + ASSERT_TRUE(compaction->compaction_reason() == + CompactionReason::kPeriodicCompaction); + start_level = compaction->start_level(); + output_level = compaction->output_level(); + periodic_compactions++; + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Case 1: Oldest flushed file excceeds periodic compaction threshold. + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Flush()); + ASSERT_EQ(0, periodic_compactions); + // Move clock forward so that the flushed file would qualify periodic + // compaction. + env_->MockSleepForSeconds(48 * 60 * 60 + 100); + + // Another flush would trigger compaction the oldest file. + ASSERT_OK(Put("foo", "bar2")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ(1, periodic_compactions); + ASSERT_EQ(0, start_level); + ASSERT_EQ(4, output_level); + + // Case 2: Oldest compacted file excceeds periodic compaction threshold + periodic_compactions = 0; + // A flush doesn't trigger a periodic compaction when threshold not hit + ASSERT_OK(Put("foo", "bar2")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(0, periodic_compactions); + + // After periodic compaction threshold hits, a flush will trigger + // a compaction + ASSERT_OK(Put("foo", "bar2")); + env_->MockSleepForSeconds(48 * 60 * 60 + 100); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(1, periodic_compactions); + ASSERT_EQ(0, start_level); + ASSERT_EQ(4, output_level); +} + +} // namespace ROCKSDB_NAMESPACE + + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_wal_test.cc b/librocksdb-sys/rocksdb/db/db_wal_test.cc new file mode 100644 index 0000000..72b6f7c --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_wal_test.cc @@ -0,0 +1,2666 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/db_test_util.h" +#include "db/db_with_timestamp_test_util.h" +#include "options/options_helper.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/file_system.h" +#include "test_util/sync_point.h" +#include "utilities/fault_injection_env.h" +#include "utilities/fault_injection_fs.h" + +namespace ROCKSDB_NAMESPACE { +class DBWALTestBase : public DBTestBase { + protected: + explicit DBWALTestBase(const std::string& dir_name) + : DBTestBase(dir_name, /*env_do_fsync=*/true) {} + +#if defined(ROCKSDB_PLATFORM_POSIX) + public: +#if defined(ROCKSDB_FALLOCATE_PRESENT) + bool IsFallocateSupported() { + // Test fallocate support of running file system. + // Skip this test if fallocate is not supported. + std::string fname_test_fallocate = dbname_ + "/preallocate_testfile"; + int fd = -1; + do { + fd = open(fname_test_fallocate.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644); + } while (fd < 0 && errno == EINTR); + assert(fd > 0); + int alloc_status = fallocate(fd, 0, 0, 1); + int err_number = errno; + close(fd); + assert(env_->DeleteFile(fname_test_fallocate) == Status::OK()); + if (err_number == ENOSYS || err_number == EOPNOTSUPP) { + fprintf(stderr, "Skipped preallocated space check: %s\n", + errnoStr(err_number).c_str()); + return false; + } + assert(alloc_status == 0); + return true; + } +#endif // ROCKSDB_FALLOCATE_PRESENT + + uint64_t GetAllocatedFileSize(std::string file_name) { + struct stat sbuf; + int err = stat(file_name.c_str(), &sbuf); + assert(err == 0); + return sbuf.st_blocks * 512; + } +#endif // ROCKSDB_PLATFORM_POSIX +}; + +class DBWALTest : public DBWALTestBase { + public: + DBWALTest() : DBWALTestBase("/db_wal_test") {} +}; + +// A SpecialEnv enriched to give more insight about deleted files +class EnrichedSpecialEnv : public SpecialEnv { + public: + explicit EnrichedSpecialEnv(Env* base) : SpecialEnv(base) {} + Status NewSequentialFile(const std::string& f, + std::unique_ptr* r, + const EnvOptions& soptions) override { + InstrumentedMutexLock l(&env_mutex_); + if (f == skipped_wal) { + deleted_wal_reopened = true; + if (IsWAL(f) && largest_deleted_wal.size() != 0 && + f.compare(largest_deleted_wal) <= 0) { + gap_in_wals = true; + } + } + return SpecialEnv::NewSequentialFile(f, r, soptions); + } + Status DeleteFile(const std::string& fname) override { + if (IsWAL(fname)) { + deleted_wal_cnt++; + InstrumentedMutexLock l(&env_mutex_); + // If this is the first WAL, remember its name and skip deleting it. We + // remember its name partly because the application might attempt to + // delete the file again. + if (skipped_wal.size() != 0 && skipped_wal != fname) { + if (largest_deleted_wal.size() == 0 || + largest_deleted_wal.compare(fname) < 0) { + largest_deleted_wal = fname; + } + } else { + skipped_wal = fname; + return Status::OK(); + } + } + return SpecialEnv::DeleteFile(fname); + } + bool IsWAL(const std::string& fname) { + // printf("iswal %s\n", fname.c_str()); + return fname.compare(fname.size() - 3, 3, "log") == 0; + } + + InstrumentedMutex env_mutex_; + // the wal whose actual delete was skipped by the env + std::string skipped_wal = ""; + // the largest WAL that was requested to be deleted + std::string largest_deleted_wal = ""; + // number of WALs that were successfully deleted + std::atomic deleted_wal_cnt = {0}; + // the WAL whose delete from fs was skipped is reopened during recovery + std::atomic deleted_wal_reopened = {false}; + // whether a gap in the WALs was detected during recovery + std::atomic gap_in_wals = {false}; +}; + +class DBWALTestWithEnrichedEnv : public DBTestBase { + public: + DBWALTestWithEnrichedEnv() + : DBTestBase("db_wal_test", /*env_do_fsync=*/true) { + enriched_env_ = new EnrichedSpecialEnv(env_->target()); + auto options = CurrentOptions(); + options.env = enriched_env_; + options.allow_2pc = true; + Reopen(options); + delete env_; + // to be deleted by the parent class + env_ = enriched_env_; + } + + protected: + EnrichedSpecialEnv* enriched_env_; +}; + +// Test that the recovery would successfully avoid the gaps between the logs. +// One known scenario that could cause this is that the application issue the +// WAL deletion out of order. For the sake of simplicity in the test, here we +// create the gap by manipulating the env to skip deletion of the first WAL but +// not the ones after it. +TEST_F(DBWALTestWithEnrichedEnv, SkipDeletedWALs) { + auto options = last_options_; + // To cause frequent WAL deletion + options.write_buffer_size = 128; + Reopen(options); + + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::PurgeObsoleteFiles:End", + "DBWALTestWithEnrichedEnv.SkipDeletedWALs:AfterFlush"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + WriteOptions writeOpt = WriteOptions(); + for (int i = 0; i < 128 * 5; i++) { + ASSERT_OK(dbfull()->Put(writeOpt, "foo", "v1")); + } + FlushOptions fo; + fo.wait = true; + ASSERT_OK(db_->Flush(fo)); + + TEST_SYNC_POINT("DBWALTestWithEnrichedEnv.SkipDeletedWALs:AfterFlush"); + + // some wals are deleted + ASSERT_NE(0, enriched_env_->deleted_wal_cnt); + // but not the first one + ASSERT_NE(0, enriched_env_->skipped_wal.size()); + + // Test that the WAL that was not deleted will be skipped during recovery + options = last_options_; + Reopen(options); + ASSERT_FALSE(enriched_env_->deleted_wal_reopened); + ASSERT_FALSE(enriched_env_->gap_in_wals); + + SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBWALTest, WAL) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + WriteOptions writeOpt = WriteOptions(); + writeOpt.disableWAL = true; + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v1")); + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1")); + + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + ASSERT_EQ("v1", Get(1, "foo")); + ASSERT_EQ("v1", Get(1, "bar")); + + writeOpt.disableWAL = false; + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v2")); + writeOpt.disableWAL = true; + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v2")); + + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + // Both value's should be present. + ASSERT_EQ("v2", Get(1, "bar")); + ASSERT_EQ("v2", Get(1, "foo")); + + writeOpt.disableWAL = true; + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v3")); + writeOpt.disableWAL = false; + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v3")); + + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + // again both values should be present. + ASSERT_EQ("v3", Get(1, "foo")); + ASSERT_EQ("v3", Get(1, "bar")); + } while (ChangeWalOptions()); +} + +TEST_F(DBWALTest, RollLog) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "foo", "v1")); + ASSERT_OK(Put(1, "baz", "v5")); + + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + for (int i = 0; i < 10; i++) { + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + } + ASSERT_OK(Put(1, "foo", "v4")); + for (int i = 0; i < 10; i++) { + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + } + } while (ChangeWalOptions()); +} + +TEST_F(DBWALTest, SyncWALNotBlockWrite) { + Options options = CurrentOptions(); + options.max_write_buffer_number = 4; + DestroyAndReopen(options); + + ASSERT_OK(Put("foo1", "bar1")); + ASSERT_OK(Put("foo5", "bar5")); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"WritableFileWriter::SyncWithoutFlush:1", + "DBWALTest::SyncWALNotBlockWrite:1"}, + {"DBWALTest::SyncWALNotBlockWrite:2", + "WritableFileWriter::SyncWithoutFlush:2"}, + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ROCKSDB_NAMESPACE::port::Thread thread([&]() { ASSERT_OK(db_->SyncWAL()); }); + + TEST_SYNC_POINT("DBWALTest::SyncWALNotBlockWrite:1"); + ASSERT_OK(Put("foo2", "bar2")); + ASSERT_OK(Put("foo3", "bar3")); + FlushOptions fo; + fo.wait = false; + ASSERT_OK(db_->Flush(fo)); + ASSERT_OK(Put("foo4", "bar4")); + + TEST_SYNC_POINT("DBWALTest::SyncWALNotBlockWrite:2"); + + thread.join(); + + ASSERT_EQ(Get("foo1"), "bar1"); + ASSERT_EQ(Get("foo2"), "bar2"); + ASSERT_EQ(Get("foo3"), "bar3"); + ASSERT_EQ(Get("foo4"), "bar4"); + ASSERT_EQ(Get("foo5"), "bar5"); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBWALTest, SyncWALNotWaitWrite) { + ASSERT_OK(Put("foo1", "bar1")); + ASSERT_OK(Put("foo3", "bar3")); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"SpecialEnv::WalFile::Append:1", "DBWALTest::SyncWALNotWaitWrite:1"}, + {"DBWALTest::SyncWALNotWaitWrite:2", "SpecialEnv::WalFile::Append:2"}, + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ROCKSDB_NAMESPACE::port::Thread thread( + [&]() { ASSERT_OK(Put("foo2", "bar2")); }); + // Moving this to SyncWAL before the actual fsync + // TEST_SYNC_POINT("DBWALTest::SyncWALNotWaitWrite:1"); + ASSERT_OK(db_->SyncWAL()); + // Moving this to SyncWAL after actual fsync + // TEST_SYNC_POINT("DBWALTest::SyncWALNotWaitWrite:2"); + + thread.join(); + + ASSERT_EQ(Get("foo1"), "bar1"); + ASSERT_EQ(Get("foo2"), "bar2"); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBWALTest, Recover) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "foo", "v1")); + ASSERT_OK(Put(1, "baz", "v5")); + + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + ASSERT_EQ("v1", Get(1, "foo")); + ASSERT_EQ("v1", Get(1, "foo")); + ASSERT_EQ("v5", Get(1, "baz")); + ASSERT_OK(Put(1, "bar", "v2")); + ASSERT_OK(Put(1, "foo", "v3")); + + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + ASSERT_EQ("v3", Get(1, "foo")); + ASSERT_OK(Put(1, "foo", "v4")); + ASSERT_EQ("v4", Get(1, "foo")); + ASSERT_EQ("v2", Get(1, "bar")); + ASSERT_EQ("v5", Get(1, "baz")); + } while (ChangeWalOptions()); +} + +class DBWALTestWithTimestamp + : public DBBasicTestWithTimestampBase, + public testing::WithParamInterface { + public: + DBWALTestWithTimestamp() + : DBBasicTestWithTimestampBase("db_wal_test_with_timestamp") {} + + Status CreateAndReopenWithTs(const std::vector& cfs, + const Options& ts_options, bool persist_udt, + bool avoid_flush_during_recovery = false) { + Options default_options = CurrentOptions(); + default_options.allow_concurrent_memtable_write = + persist_udt ? true : false; + DestroyAndReopen(default_options); + CreateColumnFamilies(cfs, ts_options); + return ReopenColumnFamiliesWithTs(cfs, ts_options, persist_udt, + avoid_flush_during_recovery); + } + + Status ReopenColumnFamiliesWithTs(const std::vector& cfs, + Options ts_options, bool persist_udt, + bool avoid_flush_during_recovery = false) { + Options default_options = CurrentOptions(); + default_options.create_if_missing = false; + default_options.allow_concurrent_memtable_write = + persist_udt ? true : false; + default_options.avoid_flush_during_recovery = avoid_flush_during_recovery; + ts_options.create_if_missing = false; + + std::vector cf_options(cfs.size(), ts_options); + std::vector cfs_plus_default = cfs; + cfs_plus_default.insert(cfs_plus_default.begin(), kDefaultColumnFamilyName); + cf_options.insert(cf_options.begin(), default_options); + Close(); + return TryReopenWithColumnFamilies(cfs_plus_default, cf_options); + } + + Status Put(uint32_t cf, const Slice& key, const Slice& ts, + const Slice& value) { + WriteOptions write_opts; + return db_->Put(write_opts, handles_[cf], key, ts, value); + } + + void CheckGet(const ReadOptions& read_opts, uint32_t cf, const Slice& key, + const std::string& expected_value, + const std::string& expected_ts) { + std::string actual_value; + std::string actual_ts; + ASSERT_OK( + db_->Get(read_opts, handles_[cf], key, &actual_value, &actual_ts)); + ASSERT_EQ(expected_value, actual_value); + ASSERT_EQ(expected_ts, actual_ts); + } +}; + +TEST_P(DBWALTestWithTimestamp, RecoverAndNoFlush) { + // Set up the option that enables user defined timestmp size. + std::string ts1; + PutFixed64(&ts1, 1); + Options ts_options; + ts_options.create_if_missing = true; + ts_options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + // Test that user-defined timestamps are recovered from WAL regardless of + // the value of this flag because UDTs are saved in WAL nonetheless. + // We however need to explicitly disable flush during recovery by setting + // `avoid_flush_during_recovery=true` so that we can avoid timestamps getting + // stripped when the `persist_user_defined_timestamps` flag is false, so that + // all written timestamps are available for testing user-defined time travel + // read. + bool persist_udt = test::ShouldPersistUDT(GetParam()); + ts_options.persist_user_defined_timestamps = persist_udt; + bool avoid_flush_during_recovery = true; + + ReadOptions read_opts; + do { + Slice ts_slice = ts1; + read_opts.timestamp = &ts_slice; + ASSERT_OK(CreateAndReopenWithTs({"pikachu"}, ts_options, persist_udt, + avoid_flush_during_recovery)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"), 0U); + ASSERT_OK(Put(1, "foo", ts1, "v1")); + ASSERT_OK(Put(1, "baz", ts1, "v5")); + + ASSERT_OK(ReopenColumnFamiliesWithTs({"pikachu"}, ts_options, persist_udt, + avoid_flush_during_recovery)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"), 0U); + // Do a timestamped read with ts1 after second reopen. + CheckGet(read_opts, 1, "foo", "v1", ts1); + CheckGet(read_opts, 1, "baz", "v5", ts1); + + // Write more value versions for key "foo" and "bar" before and after second + // reopen. + std::string ts2; + PutFixed64(&ts2, 2); + ASSERT_OK(Put(1, "bar", ts2, "v2")); + ASSERT_OK(Put(1, "foo", ts2, "v3")); + + ASSERT_OK(ReopenColumnFamiliesWithTs({"pikachu"}, ts_options, persist_udt, + avoid_flush_during_recovery)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"), 0U); + std::string ts3; + PutFixed64(&ts3, 3); + ASSERT_OK(Put(1, "foo", ts3, "v4")); + + // All the key value pairs available for read: + // "foo" -> [(ts1, "v1"), (ts2, "v3"), (ts3, "v4")] + // "bar" -> [(ts2, "v2")] + // "baz" -> [(ts1, "v5")] + // Do a timestamped read with ts1 after third reopen. + // read_opts.timestamp is set to ts1 for below reads + CheckGet(read_opts, 1, "foo", "v1", ts1); + std::string value; + ASSERT_TRUE(db_->Get(read_opts, handles_[1], "bar", &value).IsNotFound()); + CheckGet(read_opts, 1, "baz", "v5", ts1); + + // Do a timestamped read with ts2 after third reopen. + ts_slice = ts2; + // read_opts.timestamp is set to ts2 for below reads. + CheckGet(read_opts, 1, "foo", "v3", ts2); + CheckGet(read_opts, 1, "bar", "v2", ts2); + CheckGet(read_opts, 1, "baz", "v5", ts1); + + // Do a timestamped read with ts3 after third reopen. + ts_slice = ts3; + // read_opts.timestamp is set to ts3 for below reads. + CheckGet(read_opts, 1, "foo", "v4", ts3); + CheckGet(read_opts, 1, "bar", "v2", ts2); + CheckGet(read_opts, 1, "baz", "v5", ts1); + } while (ChangeWalOptions()); +} + +TEST_P(DBWALTestWithTimestamp, RecoverAndFlush) { + // Set up the option that enables user defined timestamp size. + std::string min_ts; + std::string write_ts; + PutFixed64(&min_ts, 0); + PutFixed64(&write_ts, 1); + Options ts_options; + ts_options.create_if_missing = true; + ts_options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + bool persist_udt = test::ShouldPersistUDT(GetParam()); + ts_options.persist_user_defined_timestamps = persist_udt; + + std::string smallest_ukey_without_ts = "baz"; + std::string largest_ukey_without_ts = "foo"; + + ASSERT_OK(CreateAndReopenWithTs({"pikachu"}, ts_options, persist_udt)); + // No flush, no sst files, because of no data. + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"), 0U); + ASSERT_OK(Put(1, largest_ukey_without_ts, write_ts, "v1")); + ASSERT_OK(Put(1, smallest_ukey_without_ts, write_ts, "v5")); + + ASSERT_OK(ReopenColumnFamiliesWithTs({"pikachu"}, ts_options, persist_udt)); + // Memtable recovered from WAL flushed because `avoid_flush_during_recovery` + // defaults to false, created one L0 file. + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"), 1U); + + std::vector> level_to_files; + dbfull()->TEST_GetFilesMetaData(handles_[1], &level_to_files); + ASSERT_GT(level_to_files.size(), 1); + // L0 only has one SST file. + ASSERT_EQ(level_to_files[0].size(), 1); + auto meta = level_to_files[0][0]; + if (persist_udt) { + ASSERT_EQ(smallest_ukey_without_ts + write_ts, meta.smallest.user_key()); + ASSERT_EQ(largest_ukey_without_ts + write_ts, meta.largest.user_key()); + } else { + ASSERT_EQ(smallest_ukey_without_ts + min_ts, meta.smallest.user_key()); + ASSERT_EQ(largest_ukey_without_ts + min_ts, meta.largest.user_key()); + } +} + +// Param 0: test mode for the user-defined timestamp feature +INSTANTIATE_TEST_CASE_P( + P, DBWALTestWithTimestamp, + ::testing::Values( + test::UserDefinedTimestampTestMode::kStripUserDefinedTimestamp, + test::UserDefinedTimestampTestMode::kNormal)); + +TEST_F(DBWALTestWithTimestamp, EnableDisableUDT) { + Options options; + options.create_if_missing = true; + options.comparator = BytewiseComparator(); + bool avoid_flush_during_recovery = true; + ASSERT_OK(CreateAndReopenWithTs({"pikachu"}, options, true /* persist_udt */, + avoid_flush_during_recovery)); + + ASSERT_OK(db_->Put(WriteOptions(), handles_[1], "foo", "v1")); + ASSERT_OK(db_->Put(WriteOptions(), handles_[1], "baz", "v5")); + + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + options.persist_user_defined_timestamps = false; + // Test handle timestamp size inconsistency in WAL when enabling user-defined + // timestamps. + ASSERT_OK(ReopenColumnFamiliesWithTs({"pikachu"}, options, + false /* persist_udt */, + avoid_flush_during_recovery)); + + std::string ts; + PutFixed64(&ts, 0); + Slice ts_slice = ts; + ReadOptions read_opts; + read_opts.timestamp = &ts_slice; + // Pre-existing entries are treated as if they have the min timestamp. + CheckGet(read_opts, 1, "foo", "v1", ts); + CheckGet(read_opts, 1, "baz", "v5", ts); + ts.clear(); + PutFixed64(&ts, 1); + ASSERT_OK(db_->Put(WriteOptions(), handles_[1], "foo", ts, "v2")); + ASSERT_OK(db_->Put(WriteOptions(), handles_[1], "baz", ts, "v6")); + CheckGet(read_opts, 1, "foo", "v2", ts); + CheckGet(read_opts, 1, "baz", "v6", ts); + + options.comparator = BytewiseComparator(); + // Open the column family again with the UDT feature disabled. Test handle + // timestamp size inconsistency in WAL when disabling user-defined timestamps + ASSERT_OK(ReopenColumnFamiliesWithTs({"pikachu"}, options, + true /* persist_udt */, + avoid_flush_during_recovery)); + ASSERT_EQ("v2", Get(1, "foo")); + ASSERT_EQ("v6", Get(1, "baz")); +} + +TEST_F(DBWALTest, RecoverWithTableHandle) { + do { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.avoid_flush_during_recovery = false; + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(1, "foo", "v1")); + ASSERT_OK(Put(1, "bar", "v2")); + ASSERT_OK(Flush(1)); + ASSERT_OK(Put(1, "foo", "v3")); + ASSERT_OK(Put(1, "bar", "v4")); + ASSERT_OK(Flush(1)); + ASSERT_OK(Put(1, "big", std::string(100, 'a'))); + + options = CurrentOptions(); + const int kSmallMaxOpenFiles = 13; + if (option_config_ == kDBLogDir) { + // Use this option to check not preloading files + // Set the max open files to be small enough so no preload will + // happen. + options.max_open_files = kSmallMaxOpenFiles; + // RocksDB sanitize max open files to at least 20. Modify it back. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SanitizeOptions::AfterChangeMaxOpenFiles", [&](void* arg) { + int* max_open_files = static_cast(arg); + *max_open_files = kSmallMaxOpenFiles; + }); + + } else if (option_config_ == kWalDirAndMmapReads) { + // Use this option to check always loading all files. + options.max_open_files = 100; + } else { + options.max_open_files = -1; + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + + std::vector> files; + dbfull()->TEST_GetFilesMetaData(handles_[1], &files); + size_t total_files = 0; + for (const auto& level : files) { + total_files += level.size(); + } + ASSERT_EQ(total_files, 3); + for (const auto& level : files) { + for (const auto& file : level) { + if (options.max_open_files == kSmallMaxOpenFiles) { + ASSERT_TRUE(file.table_reader_handle == nullptr); + } else { + ASSERT_TRUE(file.table_reader_handle != nullptr); + } + } + } + } while (ChangeWalOptions()); +} + +TEST_F(DBWALTest, RecoverWithBlob) { + // Write a value that's below the prospective size limit for blobs and another + // one that's above. Note that blob files are not actually enabled at this + // point. + constexpr uint64_t min_blob_size = 10; + + constexpr char short_value[] = "short"; + static_assert(sizeof(short_value) - 1 < min_blob_size, + "short_value too long"); + + constexpr char long_value[] = "long_value"; + static_assert(sizeof(long_value) - 1 >= min_blob_size, + "long_value too short"); + + ASSERT_OK(Put("key1", short_value)); + ASSERT_OK(Put("key2", long_value)); + + // There should be no files just yet since we haven't flushed. + { + VersionSet* const versions = dbfull()->GetVersionSet(); + ASSERT_NE(versions, nullptr); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(cfd, nullptr); + + Version* const current = cfd->current(); + ASSERT_NE(current, nullptr); + + const VersionStorageInfo* const storage_info = current->storage_info(); + ASSERT_NE(storage_info, nullptr); + + ASSERT_EQ(storage_info->num_non_empty_levels(), 0); + ASSERT_TRUE(storage_info->GetBlobFiles().empty()); + } + + // Reopen the database with blob files enabled. A new table file/blob file + // pair should be written during recovery. + Options options; + options.enable_blob_files = true; + options.min_blob_size = min_blob_size; + options.avoid_flush_during_recovery = false; + options.disable_auto_compactions = true; + options.env = env_; + + Reopen(options); + + ASSERT_EQ(Get("key1"), short_value); + ASSERT_EQ(Get("key2"), long_value); + + VersionSet* const versions = dbfull()->GetVersionSet(); + ASSERT_NE(versions, nullptr); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(cfd, nullptr); + + Version* const current = cfd->current(); + ASSERT_NE(current, nullptr); + + const VersionStorageInfo* const storage_info = current->storage_info(); + ASSERT_NE(storage_info, nullptr); + + const auto& l0_files = storage_info->LevelFiles(0); + ASSERT_EQ(l0_files.size(), 1); + + const FileMetaData* const table_file = l0_files[0]; + ASSERT_NE(table_file, nullptr); + + const auto& blob_files = storage_info->GetBlobFiles(); + ASSERT_EQ(blob_files.size(), 1); + + const auto& blob_file = blob_files.front(); + ASSERT_NE(blob_file, nullptr); + + ASSERT_EQ(table_file->smallest.user_key(), "key1"); + ASSERT_EQ(table_file->largest.user_key(), "key2"); + ASSERT_EQ(table_file->fd.smallest_seqno, 1); + ASSERT_EQ(table_file->fd.largest_seqno, 2); + ASSERT_EQ(table_file->oldest_blob_file_number, + blob_file->GetBlobFileNumber()); + + ASSERT_EQ(blob_file->GetTotalBlobCount(), 1); + + const InternalStats* const internal_stats = cfd->internal_stats(); + ASSERT_NE(internal_stats, nullptr); + + const auto& compaction_stats = internal_stats->TEST_GetCompactionStats(); + ASSERT_FALSE(compaction_stats.empty()); + ASSERT_EQ(compaction_stats[0].bytes_written, table_file->fd.GetFileSize()); + ASSERT_EQ(compaction_stats[0].bytes_written_blob, + blob_file->GetTotalBlobBytes()); + ASSERT_EQ(compaction_stats[0].num_output_files, 1); + ASSERT_EQ(compaction_stats[0].num_output_files_blob, 1); + + const uint64_t* const cf_stats_value = internal_stats->TEST_GetCFStatsValue(); + ASSERT_EQ(cf_stats_value[InternalStats::BYTES_FLUSHED], + compaction_stats[0].bytes_written + + compaction_stats[0].bytes_written_blob); +} + +TEST_F(DBWALTest, RecoverWithBlobMultiSST) { + // Write several large (4 KB) values without flushing. Note that blob files + // are not actually enabled at this point. + std::string large_value(1 << 12, 'a'); + + constexpr int num_keys = 64; + + for (int i = 0; i < num_keys; ++i) { + ASSERT_OK(Put(Key(i), large_value)); + } + + // There should be no files just yet since we haven't flushed. + { + VersionSet* const versions = dbfull()->GetVersionSet(); + ASSERT_NE(versions, nullptr); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(cfd, nullptr); + + Version* const current = cfd->current(); + ASSERT_NE(current, nullptr); + + const VersionStorageInfo* const storage_info = current->storage_info(); + ASSERT_NE(storage_info, nullptr); + + ASSERT_EQ(storage_info->num_non_empty_levels(), 0); + ASSERT_TRUE(storage_info->GetBlobFiles().empty()); + } + + // Reopen the database with blob files enabled and write buffer size set to a + // smaller value. Multiple table files+blob files should be written and added + // to the Version during recovery. + Options options; + options.write_buffer_size = 1 << 16; // 64 KB + options.enable_blob_files = true; + options.avoid_flush_during_recovery = false; + options.disable_auto_compactions = true; + options.env = env_; + + Reopen(options); + + for (int i = 0; i < num_keys; ++i) { + ASSERT_EQ(Get(Key(i)), large_value); + } + + VersionSet* const versions = dbfull()->GetVersionSet(); + ASSERT_NE(versions, nullptr); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(cfd, nullptr); + + Version* const current = cfd->current(); + ASSERT_NE(current, nullptr); + + const VersionStorageInfo* const storage_info = current->storage_info(); + ASSERT_NE(storage_info, nullptr); + + const auto& l0_files = storage_info->LevelFiles(0); + ASSERT_GT(l0_files.size(), 1); + + const auto& blob_files = storage_info->GetBlobFiles(); + ASSERT_GT(blob_files.size(), 1); + + ASSERT_EQ(l0_files.size(), blob_files.size()); +} + +TEST_F(DBWALTest, WALWithChecksumHandoff) { +#ifndef ROCKSDB_ASSERT_STATUS_CHECKED + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_SKIP("Test requires non-mem or non-encrypted environment"); + return; + } + std::shared_ptr fault_fs( + new FaultInjectionTestFS(FileSystem::Default())); + std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); + do { + Options options = CurrentOptions(); + + options.checksum_handoff_file_types.Add(FileType::kWalFile); + options.env = fault_fs_env.get(); + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kCRC32c); + + CreateAndReopenWithCF({"pikachu"}, options); + WriteOptions writeOpt = WriteOptions(); + writeOpt.disableWAL = true; + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v1")); + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1")); + + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_EQ("v1", Get(1, "foo")); + ASSERT_EQ("v1", Get(1, "bar")); + + writeOpt.disableWAL = false; + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v2")); + writeOpt.disableWAL = true; + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v2")); + + ReopenWithColumnFamilies({"default", "pikachu"}, options); + // Both value's should be present. + ASSERT_EQ("v2", Get(1, "bar")); + ASSERT_EQ("v2", Get(1, "foo")); + + writeOpt.disableWAL = true; + // This put, data is persisted by Flush + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v3")); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + writeOpt.disableWAL = false; + // Data is persisted in the WAL + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "zoo", "v3")); + // The hash does not match, write fails + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kxxHash); + writeOpt.disableWAL = false; + ASSERT_NOK(dbfull()->Put(writeOpt, handles_[1], "foo", "v3")); + + ReopenWithColumnFamilies({"default", "pikachu"}, options); + // Due to the write failure, Get should not find + ASSERT_NE("v3", Get(1, "foo")); + ASSERT_EQ("v3", Get(1, "zoo")); + ASSERT_EQ("v3", Get(1, "bar")); + + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kCRC32c); + // Each write will be similated as corrupted. + fault_fs->IngestDataCorruptionBeforeWrite(); + writeOpt.disableWAL = true; + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v4")); + writeOpt.disableWAL = false; + ASSERT_NOK(dbfull()->Put(writeOpt, handles_[1], "foo", "v4")); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_NE("v4", Get(1, "foo")); + ASSERT_NE("v4", Get(1, "bar")); + fault_fs->NoDataCorruptionBeforeWrite(); + + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kNoChecksum); + // The file system does not provide checksum method and verification. + writeOpt.disableWAL = true; + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v5")); + writeOpt.disableWAL = false; + ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v5")); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_EQ("v5", Get(1, "foo")); + ASSERT_EQ("v5", Get(1, "bar")); + + Destroy(options); + } while (ChangeWalOptions()); +#endif // ROCKSDB_ASSERT_STATUS_CHECKED +} + +TEST_F(DBWALTest, LockWal) { + do { + Options options = CurrentOptions(); + options.create_if_missing = true; + DestroyAndReopen(options); + + ASSERT_OK(Put("foo", "v")); + ASSERT_OK(Put("bar", "v")); + + ASSERT_OK(db_->LockWAL()); + // Verify writes are stopped + WriteOptions wopts; + wopts.no_slowdown = true; + Status s = db_->Put(wopts, "foo", "dontcare"); + ASSERT_TRUE(s.IsIncomplete()); + { + VectorLogPtr wals; + ASSERT_OK(db_->GetSortedWalFiles(wals)); + ASSERT_FALSE(wals.empty()); + } + port::Thread worker([&]() { + Status tmp_s = db_->Flush(FlushOptions()); + ASSERT_OK(tmp_s); + }); + FlushOptions flush_opts; + flush_opts.wait = false; + s = db_->Flush(flush_opts); + ASSERT_TRUE(s.IsTryAgain()); + ASSERT_OK(db_->UnlockWAL()); + ASSERT_OK(db_->Put(WriteOptions(), "foo", "dontcare")); + + worker.join(); + } while (ChangeWalOptions()); +} + +class DBRecoveryTestBlobError + : public DBWALTest, + public testing::WithParamInterface { + public: + DBRecoveryTestBlobError() : sync_point_(GetParam()) {} + + std::string sync_point_; +}; + +INSTANTIATE_TEST_CASE_P(DBRecoveryTestBlobError, DBRecoveryTestBlobError, + ::testing::ValuesIn(std::vector{ + "BlobFileBuilder::WriteBlobToFile:AddRecord", + "BlobFileBuilder::WriteBlobToFile:AppendFooter"})); + +TEST_P(DBRecoveryTestBlobError, RecoverWithBlobError) { + // Write a value. Note that blob files are not actually enabled at this point. + ASSERT_OK(Put("key", "blob")); + + // Reopen with blob files enabled but make blob file writing fail during + // recovery. + SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* arg) { + Status* const s = static_cast(arg); + assert(s); + + (*s) = Status::IOError(sync_point_); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + Options options; + options.enable_blob_files = true; + options.avoid_flush_during_recovery = false; + options.disable_auto_compactions = true; + options.env = env_; + + ASSERT_NOK(TryReopen(options)); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + // Make sure the files generated by the failed recovery have been deleted. + std::vector files; + ASSERT_OK(env_->GetChildren(dbname_, &files)); + for (const auto& file : files) { + uint64_t number = 0; + FileType type = kTableFile; + + if (!ParseFileName(file, &number, &type)) { + continue; + } + + ASSERT_NE(type, kTableFile); + ASSERT_NE(type, kBlobFile); + } +} + +TEST_F(DBWALTest, IgnoreRecoveredLog) { + std::string backup_logs = dbname_ + "/backup_logs"; + + do { + // delete old files in backup_logs directory + ASSERT_OK(env_->CreateDirIfMissing(backup_logs)); + std::vector old_files; + ASSERT_OK(env_->GetChildren(backup_logs, &old_files)); + for (auto& file : old_files) { + ASSERT_OK(env_->DeleteFile(backup_logs + "/" + file)); + } + Options options = CurrentOptions(); + options.create_if_missing = true; + options.merge_operator = MergeOperators::CreateUInt64AddOperator(); + options.wal_dir = dbname_ + "/logs"; + DestroyAndReopen(options); + + // fill up the DB + std::string one, two; + PutFixed64(&one, 1); + PutFixed64(&two, 2); + ASSERT_OK(db_->Merge(WriteOptions(), Slice("foo"), Slice(one))); + ASSERT_OK(db_->Merge(WriteOptions(), Slice("foo"), Slice(one))); + ASSERT_OK(db_->Merge(WriteOptions(), Slice("bar"), Slice(one))); + + // copy the logs to backup + std::vector logs; + ASSERT_OK(env_->GetChildren(options.wal_dir, &logs)); + for (auto& log : logs) { + CopyFile(options.wal_dir + "/" + log, backup_logs + "/" + log); + } + + // recover the DB + Reopen(options); + ASSERT_EQ(two, Get("foo")); + ASSERT_EQ(one, Get("bar")); + Close(); + + // copy the logs from backup back to wal dir + for (auto& log : logs) { + CopyFile(backup_logs + "/" + log, options.wal_dir + "/" + log); + } + // this should ignore the log files, recovery should not happen again + // if the recovery happens, the same merge operator would be called twice, + // leading to incorrect results + Reopen(options); + ASSERT_EQ(two, Get("foo")); + ASSERT_EQ(one, Get("bar")); + Close(); + Destroy(options); + Reopen(options); + Close(); + + // copy the logs from backup back to wal dir + ASSERT_OK(env_->CreateDirIfMissing(options.wal_dir)); + for (auto& log : logs) { + CopyFile(backup_logs + "/" + log, options.wal_dir + "/" + log); + } + // assert that we successfully recovered only from logs, even though we + // destroyed the DB + Reopen(options); + ASSERT_EQ(two, Get("foo")); + ASSERT_EQ(one, Get("bar")); + + // Recovery will fail if DB directory doesn't exist. + Destroy(options); + // copy the logs from backup back to wal dir + ASSERT_OK(env_->CreateDirIfMissing(options.wal_dir)); + for (auto& log : logs) { + CopyFile(backup_logs + "/" + log, options.wal_dir + "/" + log); + // we won't be needing this file no more + ASSERT_OK(env_->DeleteFile(backup_logs + "/" + log)); + } + Status s = TryReopen(options); + ASSERT_NOK(s); + Destroy(options); + } while (ChangeWalOptions()); +} + +TEST_F(DBWALTest, RecoveryWithEmptyLog) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "foo", "v1")); + ASSERT_OK(Put(1, "foo", "v2")); + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "foo", "v3")); + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + ASSERT_EQ("v3", Get(1, "foo")); + } while (ChangeWalOptions()); +} + +#if !(defined NDEBUG) || !defined(OS_WIN) +TEST_F(DBWALTest, PreallocateBlock) { + Options options = CurrentOptions(); + options.write_buffer_size = 10 * 1000 * 1000; + options.max_total_wal_size = 0; + + size_t expected_preallocation_size = static_cast( + options.write_buffer_size + options.write_buffer_size / 10); + + DestroyAndReopen(options); + + std::atomic called(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBTestWalFile.GetPreallocationStatus", [&](void* arg) { + ASSERT_TRUE(arg != nullptr); + size_t preallocation_size = *(static_cast(arg)); + ASSERT_EQ(expected_preallocation_size, preallocation_size); + called.fetch_add(1); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(Put("", "")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("", "")); + Close(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ASSERT_EQ(2, called.load()); + + options.max_total_wal_size = 1000 * 1000; + expected_preallocation_size = static_cast(options.max_total_wal_size); + Reopen(options); + called.store(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBTestWalFile.GetPreallocationStatus", [&](void* arg) { + ASSERT_TRUE(arg != nullptr); + size_t preallocation_size = *(static_cast(arg)); + ASSERT_EQ(expected_preallocation_size, preallocation_size); + called.fetch_add(1); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(Put("", "")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("", "")); + Close(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ASSERT_EQ(2, called.load()); + + options.db_write_buffer_size = 800 * 1000; + expected_preallocation_size = + static_cast(options.db_write_buffer_size); + Reopen(options); + called.store(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBTestWalFile.GetPreallocationStatus", [&](void* arg) { + ASSERT_TRUE(arg != nullptr); + size_t preallocation_size = *(static_cast(arg)); + ASSERT_EQ(expected_preallocation_size, preallocation_size); + called.fetch_add(1); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(Put("", "")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("", "")); + Close(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ASSERT_EQ(2, called.load()); + + expected_preallocation_size = 700 * 1000; + std::shared_ptr write_buffer_manager = + std::make_shared(static_cast(700 * 1000)); + options.write_buffer_manager = write_buffer_manager; + Reopen(options); + called.store(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBTestWalFile.GetPreallocationStatus", [&](void* arg) { + ASSERT_TRUE(arg != nullptr); + size_t preallocation_size = *(static_cast(arg)); + ASSERT_EQ(expected_preallocation_size, preallocation_size); + called.fetch_add(1); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(Put("", "")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("", "")); + Close(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ASSERT_EQ(2, called.load()); +} +#endif // !(defined NDEBUG) || !defined(OS_WIN) + +TEST_F(DBWALTest, DISABLED_FullPurgePreservesRecycledLog) { + // TODO(ajkr): Disabled until WAL recycling is fixed for + // `kPointInTimeRecovery`. + + // For github issue #1303 + for (int i = 0; i < 2; ++i) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.recycle_log_file_num = 2; + if (i != 0) { + options.wal_dir = alternative_wal_dir_; + } + + DestroyAndReopen(options); + ASSERT_OK(Put("foo", "v1")); + VectorLogPtr log_files; + ASSERT_OK(dbfull()->GetSortedWalFiles(log_files)); + ASSERT_GT(log_files.size(), 0); + ASSERT_OK(Flush()); + + // Now the original WAL is in log_files[0] and should be marked for + // recycling. + // Verify full purge cannot remove this file. + JobContext job_context(0); + dbfull()->TEST_LockMutex(); + dbfull()->FindObsoleteFiles(&job_context, true /* force */); + dbfull()->TEST_UnlockMutex(); + dbfull()->PurgeObsoleteFiles(job_context); + + if (i == 0) { + ASSERT_OK( + env_->FileExists(LogFileName(dbname_, log_files[0]->LogNumber()))); + } else { + ASSERT_OK(env_->FileExists( + LogFileName(alternative_wal_dir_, log_files[0]->LogNumber()))); + } + } +} + +TEST_F(DBWALTest, DISABLED_FullPurgePreservesLogPendingReuse) { + // TODO(ajkr): Disabled until WAL recycling is fixed for + // `kPointInTimeRecovery`. + + // Ensures full purge cannot delete a WAL while it's in the process of being + // recycled. In particular, we force the full purge after a file has been + // chosen for reuse, but before it has been renamed. + for (int i = 0; i < 2; ++i) { + Options options = CurrentOptions(); + options.recycle_log_file_num = 1; + if (i != 0) { + options.wal_dir = alternative_wal_dir_; + } + DestroyAndReopen(options); + + // The first flush creates a second log so writes can continue before the + // flush finishes. + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Flush()); + + // The second flush can recycle the first log. Sync points enforce the + // full purge happens after choosing the log to recycle and before it is + // renamed. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"DBImpl::CreateWAL:BeforeReuseWritableFile1", + "DBWALTest::FullPurgePreservesLogPendingReuse:PreFullPurge"}, + {"DBWALTest::FullPurgePreservesLogPendingReuse:PostFullPurge", + "DBImpl::CreateWAL:BeforeReuseWritableFile2"}, + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ROCKSDB_NAMESPACE::port::Thread thread([&]() { + TEST_SYNC_POINT( + "DBWALTest::FullPurgePreservesLogPendingReuse:PreFullPurge"); + ASSERT_OK(db_->EnableFileDeletions(true)); + TEST_SYNC_POINT( + "DBWALTest::FullPurgePreservesLogPendingReuse:PostFullPurge"); + }); + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Flush()); + thread.join(); + } +} + +TEST_F(DBWALTest, GetSortedWalFiles) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + VectorLogPtr log_files; + ASSERT_OK(dbfull()->GetSortedWalFiles(log_files)); + ASSERT_EQ(0, log_files.size()); + + ASSERT_OK(Put(1, "foo", "v1")); + ASSERT_OK(dbfull()->GetSortedWalFiles(log_files)); + ASSERT_EQ(1, log_files.size()); + } while (ChangeWalOptions()); +} + +TEST_F(DBWALTest, GetCurrentWalFile) { + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + + std::unique_ptr* bad_log_file = nullptr; + ASSERT_NOK(dbfull()->GetCurrentWalFile(bad_log_file)); + + std::unique_ptr log_file; + ASSERT_OK(dbfull()->GetCurrentWalFile(&log_file)); + + // nothing has been written to the log yet + ASSERT_EQ(log_file->StartSequence(), 0); + ASSERT_EQ(log_file->SizeFileBytes(), 0); + ASSERT_EQ(log_file->Type(), kAliveLogFile); + ASSERT_GT(log_file->LogNumber(), 0); + + // add some data and verify that the file size actually moves foward + ASSERT_OK(Put(0, "foo", "v1")); + ASSERT_OK(Put(0, "foo2", "v2")); + ASSERT_OK(Put(0, "foo3", "v3")); + + ASSERT_OK(dbfull()->GetCurrentWalFile(&log_file)); + + ASSERT_EQ(log_file->StartSequence(), 0); + ASSERT_GT(log_file->SizeFileBytes(), 0); + ASSERT_EQ(log_file->Type(), kAliveLogFile); + ASSERT_GT(log_file->LogNumber(), 0); + + // force log files to cycle and add some more data, then check if + // log number moves forward + + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + for (int i = 0; i < 10; i++) { + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + } + + ASSERT_OK(Put(0, "foo4", "v4")); + ASSERT_OK(Put(0, "foo5", "v5")); + ASSERT_OK(Put(0, "foo6", "v6")); + + ASSERT_OK(dbfull()->GetCurrentWalFile(&log_file)); + + ASSERT_EQ(log_file->StartSequence(), 0); + ASSERT_GT(log_file->SizeFileBytes(), 0); + ASSERT_EQ(log_file->Type(), kAliveLogFile); + ASSERT_GT(log_file->LogNumber(), 0); + + } while (ChangeWalOptions()); +} + +TEST_F(DBWALTest, RecoveryWithLogDataForSomeCFs) { + // Test for regression of WAL cleanup missing files that don't contain data + // for every column family. + do { + CreateAndReopenWithCF({"pikachu"}, CurrentOptions()); + ASSERT_OK(Put(1, "foo", "v1")); + ASSERT_OK(Put(1, "foo", "v2")); + uint64_t earliest_log_nums[2]; + for (int i = 0; i < 2; ++i) { + if (i > 0) { + ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); + } + VectorLogPtr log_files; + ASSERT_OK(dbfull()->GetSortedWalFiles(log_files)); + if (log_files.size() > 0) { + earliest_log_nums[i] = log_files[0]->LogNumber(); + } else { + earliest_log_nums[i] = std::numeric_limits::max(); + } + } + // Check at least the first WAL was cleaned up during the recovery. + ASSERT_LT(earliest_log_nums[0], earliest_log_nums[1]); + } while (ChangeWalOptions()); +} + +TEST_F(DBWALTest, RecoverWithLargeLog) { + do { + { + Options options = CurrentOptions(); + CreateAndReopenWithCF({"pikachu"}, options); + ASSERT_OK(Put(1, "big1", std::string(200000, '1'))); + ASSERT_OK(Put(1, "big2", std::string(200000, '2'))); + ASSERT_OK(Put(1, "small3", std::string(10, '3'))); + ASSERT_OK(Put(1, "small4", std::string(10, '4'))); + ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0); + } + + // Make sure that if we re-open with a small write buffer size that + // we flush table files in the middle of a large log file. + Options options; + options.write_buffer_size = 100000; + options = CurrentOptions(options); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_EQ(NumTableFilesAtLevel(0, 1), 3); + ASSERT_EQ(std::string(200000, '1'), Get(1, "big1")); + ASSERT_EQ(std::string(200000, '2'), Get(1, "big2")); + ASSERT_EQ(std::string(10, '3'), Get(1, "small3")); + ASSERT_EQ(std::string(10, '4'), Get(1, "small4")); + ASSERT_GT(NumTableFilesAtLevel(0, 1), 1); + } while (ChangeWalOptions()); +} + +// In https://reviews.facebook.net/D20661 we change +// recovery behavior: previously for each log file each column family +// memtable was flushed, even it was empty. Now it's changed: +// we try to create the smallest number of table files by merging +// updates from multiple logs +TEST_F(DBWALTest, RecoverCheckFileAmountWithSmallWriteBuffer) { + Options options = CurrentOptions(); + options.write_buffer_size = 5000000; + CreateAndReopenWithCF({"pikachu", "dobrynia", "nikitich"}, options); + + // Since we will reopen DB with smaller write_buffer_size, + // each key will go to new SST file + ASSERT_OK(Put(1, Key(10), DummyString(1000000))); + ASSERT_OK(Put(1, Key(10), DummyString(1000000))); + ASSERT_OK(Put(1, Key(10), DummyString(1000000))); + ASSERT_OK(Put(1, Key(10), DummyString(1000000))); + + ASSERT_OK(Put(3, Key(10), DummyString(1))); + // Make 'dobrynia' to be flushed and new WAL file to be created + ASSERT_OK(Put(2, Key(10), DummyString(7500000))); + ASSERT_OK(Put(2, Key(1), DummyString(1))); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[2])); + { + auto tables = ListTableFiles(env_, dbname_); + ASSERT_EQ(tables.size(), static_cast(1)); + // Make sure 'dobrynia' was flushed: check sst files amount + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "dobrynia"), + static_cast(1)); + } + // New WAL file + ASSERT_OK(Put(1, Key(1), DummyString(1))); + ASSERT_OK(Put(1, Key(1), DummyString(1))); + ASSERT_OK(Put(3, Key(10), DummyString(1))); + ASSERT_OK(Put(3, Key(10), DummyString(1))); + ASSERT_OK(Put(3, Key(10), DummyString(1))); + + options.write_buffer_size = 4096; + options.arena_block_size = 4096; + ReopenWithColumnFamilies({"default", "pikachu", "dobrynia", "nikitich"}, + options); + { + // No inserts => default is empty + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"), + static_cast(0)); + // First 4 keys goes to separate SSTs + 1 more SST for 2 smaller keys + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"), + static_cast(5)); + // 1 SST for big key + 1 SST for small one + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "dobrynia"), + static_cast(2)); + // 1 SST for all keys + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"), + static_cast(1)); + } +} + +// In https://reviews.facebook.net/D20661 we change +// recovery behavior: previously for each log file each column family +// memtable was flushed, even it wasn't empty. Now it's changed: +// we try to create the smallest number of table files by merging +// updates from multiple logs +TEST_F(DBWALTest, RecoverCheckFileAmount) { + Options options = CurrentOptions(); + options.write_buffer_size = 100000; + options.arena_block_size = 4 * 1024; + options.avoid_flush_during_recovery = false; + CreateAndReopenWithCF({"pikachu", "dobrynia", "nikitich"}, options); + + ASSERT_OK(Put(0, Key(1), DummyString(1))); + ASSERT_OK(Put(1, Key(1), DummyString(1))); + ASSERT_OK(Put(2, Key(1), DummyString(1))); + + // Make 'nikitich' memtable to be flushed + ASSERT_OK(Put(3, Key(10), DummyString(1002400))); + ASSERT_OK(Put(3, Key(1), DummyString(1))); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[3])); + // 4 memtable are not flushed, 1 sst file + { + auto tables = ListTableFiles(env_, dbname_); + ASSERT_EQ(tables.size(), static_cast(1)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"), + static_cast(1)); + } + // Memtable for 'nikitich' has flushed, new WAL file has opened + // 4 memtable still not flushed + + // Write to new WAL file + ASSERT_OK(Put(0, Key(1), DummyString(1))); + ASSERT_OK(Put(1, Key(1), DummyString(1))); + ASSERT_OK(Put(2, Key(1), DummyString(1))); + + // Fill up 'nikitich' one more time + ASSERT_OK(Put(3, Key(10), DummyString(1002400))); + // make it flush + ASSERT_OK(Put(3, Key(1), DummyString(1))); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[3])); + // There are still 4 memtable not flushed, and 2 sst tables + ASSERT_OK(Put(0, Key(1), DummyString(1))); + ASSERT_OK(Put(1, Key(1), DummyString(1))); + ASSERT_OK(Put(2, Key(1), DummyString(1))); + + { + auto tables = ListTableFiles(env_, dbname_); + ASSERT_EQ(tables.size(), static_cast(2)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"), + static_cast(2)); + } + + ReopenWithColumnFamilies({"default", "pikachu", "dobrynia", "nikitich"}, + options); + { + std::vector table_files = ListTableFiles(env_, dbname_); + // Check, that records for 'default', 'dobrynia' and 'pikachu' from + // first, second and third WALs went to the same SST. + // So, there is 6 SSTs: three for 'nikitich', one for 'default', one for + // 'dobrynia', one for 'pikachu' + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"), + static_cast(1)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"), + static_cast(3)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "dobrynia"), + static_cast(1)); + ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"), + static_cast(1)); + } +} + +TEST_F(DBWALTest, SyncMultipleLogs) { + const uint64_t kNumBatches = 2; + const int kBatchSize = 1000; + + Options options = CurrentOptions(); + options.create_if_missing = true; + options.write_buffer_size = 4096; + Reopen(options); + + WriteBatch batch; + WriteOptions wo; + wo.sync = true; + + for (uint64_t b = 0; b < kNumBatches; b++) { + batch.Clear(); + for (int i = 0; i < kBatchSize; i++) { + ASSERT_OK(batch.Put(Key(i), DummyString(128))); + } + + ASSERT_OK(dbfull()->Write(wo, &batch)); + } + + ASSERT_OK(dbfull()->SyncWAL()); +} + +// Github issue 1339. Prior the fix we read sequence id from the first log to +// a local variable, then keep increase the variable as we replay logs, +// ignoring actual sequence id of the records. This is incorrect if some writes +// come with WAL disabled. +TEST_F(DBWALTest, PartOfWritesWithWALDisabled) { + std::unique_ptr fault_env( + new FaultInjectionTestEnv(env_)); + Options options = CurrentOptions(); + options.env = fault_env.get(); + options.disable_auto_compactions = true; + WriteOptions wal_on, wal_off; + wal_on.sync = true; + wal_on.disableWAL = false; + wal_off.disableWAL = true; + CreateAndReopenWithCF({"dummy"}, options); + ASSERT_OK(Put(1, "dummy", "d1", wal_on)); // seq id 1 + ASSERT_OK(Put(1, "dummy", "d2", wal_off)); + ASSERT_OK(Put(1, "dummy", "d3", wal_off)); + ASSERT_OK(Put(0, "key", "v4", wal_on)); // seq id 4 + ASSERT_OK(Flush(0)); + ASSERT_OK(Put(0, "key", "v5", wal_on)); // seq id 5 + ASSERT_EQ("v5", Get(0, "key")); + ASSERT_OK(dbfull()->FlushWAL(false)); + // Simulate a crash. + fault_env->SetFilesystemActive(false); + Close(); + fault_env->ResetState(); + ReopenWithColumnFamilies({"default", "dummy"}, options); + // Prior to the fix, we may incorrectly recover "v5" with sequence id = 3. + ASSERT_EQ("v5", Get(0, "key")); + // Destroy DB before destruct fault_env. + Destroy(options); +} + +// +// Test WAL recovery for the various modes available +// +class RecoveryTestHelper { + public: + // Number of WAL files to generate + static constexpr int kWALFilesCount = 10; + // Starting number for the WAL file name like 00010.log + static constexpr int kWALFileOffset = 10; + // Keys to be written per WAL file + static constexpr int kKeysPerWALFile = 133; + // Size of the value + static constexpr int kValueSize = 96; + + // Create WAL files with values filled in + static void FillData(DBWALTestBase* test, const Options& options, + const size_t wal_count, size_t* count) { + // Calling internal functions requires sanitized options. + Options sanitized_options = SanitizeOptions(test->dbname_, options); + const ImmutableDBOptions db_options(sanitized_options); + + *count = 0; + + std::shared_ptr table_cache = NewLRUCache(50, 0); + FileOptions file_options; + WriteBufferManager write_buffer_manager(db_options.db_write_buffer_size); + + std::unique_ptr versions; + std::unique_ptr wal_manager; + WriteController write_controller; + + versions.reset(new VersionSet( + test->dbname_, &db_options, file_options, table_cache.get(), + &write_buffer_manager, &write_controller, + /*block_cache_tracer=*/nullptr, + /*io_tracer=*/nullptr, /*db_id*/ "", /*db_session_id*/ "")); + + wal_manager.reset( + new WalManager(db_options, file_options, /*io_tracer=*/nullptr)); + + std::unique_ptr current_log_writer; + + for (size_t j = kWALFileOffset; j < wal_count + kWALFileOffset; j++) { + uint64_t current_log_number = j; + std::string fname = LogFileName(test->dbname_, current_log_number); + std::unique_ptr file_writer; + ASSERT_OK(WritableFileWriter::Create(db_options.env->GetFileSystem(), + fname, file_options, &file_writer, + nullptr)); + log::Writer* log_writer = + new log::Writer(std::move(file_writer), current_log_number, + db_options.recycle_log_file_num > 0, false, + db_options.wal_compression); + ASSERT_OK(log_writer->AddCompressionTypeRecord()); + current_log_writer.reset(log_writer); + + WriteBatch batch; + for (int i = 0; i < kKeysPerWALFile; i++) { + std::string key = "key" + std::to_string((*count)++); + std::string value = test->DummyString(kValueSize); + ASSERT_NE(current_log_writer.get(), nullptr); + uint64_t seq = versions->LastSequence() + 1; + batch.Clear(); + ASSERT_OK(batch.Put(key, value)); + WriteBatchInternal::SetSequence(&batch, seq); + ASSERT_OK(current_log_writer->AddRecord( + WriteBatchInternal::Contents(&batch))); + versions->SetLastAllocatedSequence(seq); + versions->SetLastPublishedSequence(seq); + versions->SetLastSequence(seq); + } + } + } + + // Recreate and fill the store with some data + static size_t FillData(DBWALTestBase* test, Options* options) { + options->create_if_missing = true; + test->DestroyAndReopen(*options); + test->Close(); + + size_t count = 0; + FillData(test, *options, kWALFilesCount, &count); + return count; + } + + // Read back all the keys we wrote and return the number of keys found + static size_t GetData(DBWALTestBase* test) { + size_t count = 0; + for (size_t i = 0; i < kWALFilesCount * kKeysPerWALFile; i++) { + if (test->Get("key" + std::to_string(i)) != "NOT_FOUND") { + ++count; + } + } + return count; + } + + // Manuall corrupt the specified WAL + static void CorruptWAL(DBWALTestBase* test, const Options& options, + const double off, const double len, + const int wal_file_id, const bool trunc = false) { + Env* env = options.env; + std::string fname = LogFileName(test->dbname_, wal_file_id); + uint64_t size; + ASSERT_OK(env->GetFileSize(fname, &size)); + ASSERT_GT(size, 0); +#ifdef OS_WIN + // Windows disk cache behaves differently. When we truncate + // the original content is still in the cache due to the original + // handle is still open. Generally, in Windows, one prohibits + // shared access to files and it is not needed for WAL but we allow + // it to induce corruption at various tests. + test->Close(); +#endif + if (trunc) { + ASSERT_OK( + test::TruncateFile(env, fname, static_cast(size * off))); + } else { + ASSERT_OK(test::CorruptFile(env, fname, static_cast(size * off + 8), + static_cast(size * len), false)); + } + } +}; + +class DBWALTestWithParams : public DBWALTestBase, + public ::testing::WithParamInterface< + std::tuple> { + public: + DBWALTestWithParams() : DBWALTestBase("/db_wal_test_with_params") {} +}; + +INSTANTIATE_TEST_CASE_P( + Wal, DBWALTestWithParams, + ::testing::Combine(::testing::Bool(), ::testing::Range(0, 4, 1), + ::testing::Range(RecoveryTestHelper::kWALFileOffset, + RecoveryTestHelper::kWALFileOffset + + RecoveryTestHelper::kWALFilesCount, + 1), + ::testing::Values(CompressionType::kNoCompression, + CompressionType::kZSTD))); + +class DBWALTestWithParamsVaryingRecoveryMode + : public DBWALTestBase, + public ::testing::WithParamInterface< + std::tuple> { + public: + DBWALTestWithParamsVaryingRecoveryMode() + : DBWALTestBase("/db_wal_test_with_params_mode") {} +}; + +INSTANTIATE_TEST_CASE_P( + Wal, DBWALTestWithParamsVaryingRecoveryMode, + ::testing::Combine( + ::testing::Bool(), ::testing::Range(0, 4, 1), + ::testing::Range(RecoveryTestHelper::kWALFileOffset, + RecoveryTestHelper::kWALFileOffset + + RecoveryTestHelper::kWALFilesCount, + 1), + ::testing::Values(WALRecoveryMode::kTolerateCorruptedTailRecords, + WALRecoveryMode::kAbsoluteConsistency, + WALRecoveryMode::kPointInTimeRecovery, + WALRecoveryMode::kSkipAnyCorruptedRecords), + ::testing::Values(CompressionType::kNoCompression, + CompressionType::kZSTD))); + +// Test scope: +// - We expect to open the data store when there is incomplete trailing writes +// at the end of any of the logs +// - We do not expect to open the data store for corruption +TEST_P(DBWALTestWithParams, kTolerateCorruptedTailRecords) { + bool trunc = std::get<0>(GetParam()); // Corruption style + // Corruption offset position + int corrupt_offset = std::get<1>(GetParam()); + int wal_file_id = std::get<2>(GetParam()); // WAL file + + // Fill data for testing + Options options = CurrentOptions(); + const size_t row_count = RecoveryTestHelper::FillData(this, &options); + // test checksum failure or parsing + RecoveryTestHelper::CorruptWAL(this, options, corrupt_offset * .3, + /*len%=*/.1, wal_file_id, trunc); + + options.wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords; + if (trunc) { + options.create_if_missing = false; + ASSERT_OK(TryReopen(options)); + const size_t recovered_row_count = RecoveryTestHelper::GetData(this); + ASSERT_TRUE(corrupt_offset == 0 || recovered_row_count > 0); + ASSERT_LT(recovered_row_count, row_count); + } else { + ASSERT_NOK(TryReopen(options)); + } +} + +// Test scope: +// We don't expect the data store to be opened if there is any corruption +// (leading, middle or trailing -- incomplete writes or corruption) +TEST_P(DBWALTestWithParams, kAbsoluteConsistency) { + // Verify clean slate behavior + Options options = CurrentOptions(); + const size_t row_count = RecoveryTestHelper::FillData(this, &options); + options.create_if_missing = false; + ASSERT_OK(TryReopen(options)); + ASSERT_EQ(RecoveryTestHelper::GetData(this), row_count); + + bool trunc = std::get<0>(GetParam()); // Corruption style + // Corruption offset position + int corrupt_offset = std::get<1>(GetParam()); + int wal_file_id = std::get<2>(GetParam()); // WAL file + // WAL compression type + CompressionType compression_type = std::get<3>(GetParam()); + options.wal_compression = compression_type; + + if (trunc && corrupt_offset == 0) { + return; + } + + // fill with new date + RecoveryTestHelper::FillData(this, &options); + // corrupt the wal + RecoveryTestHelper::CorruptWAL(this, options, corrupt_offset * .33, + /*len%=*/.1, wal_file_id, trunc); + // verify + options.wal_recovery_mode = WALRecoveryMode::kAbsoluteConsistency; + options.create_if_missing = false; + ASSERT_NOK(TryReopen(options)); +} + +// Test scope: +// We don't expect the data store to be opened if there is any inconsistency +// between WAL and SST files +TEST_F(DBWALTest, kPointInTimeRecoveryCFConsistency) { + Options options = CurrentOptions(); + options.avoid_flush_during_recovery = true; + + // Create DB with multiple column families. + CreateAndReopenWithCF({"one", "two"}, options); + ASSERT_OK(Put(1, "key1", "val1")); + ASSERT_OK(Put(2, "key2", "val2")); + + // Record the offset at this point + Env* env = options.env; + uint64_t wal_file_id = dbfull()->TEST_LogfileNumber(); + std::string fname = LogFileName(dbname_, wal_file_id); + uint64_t offset_to_corrupt; + ASSERT_OK(env->GetFileSize(fname, &offset_to_corrupt)); + ASSERT_GT(offset_to_corrupt, 0); + + ASSERT_OK(Put(1, "key3", "val3")); + // Corrupt WAL at location of key3 + ASSERT_OK(test::CorruptFile(env, fname, static_cast(offset_to_corrupt), + 4, false)); + ASSERT_OK(Put(2, "key4", "val4")); + ASSERT_OK(Put(1, "key5", "val5")); + ASSERT_OK(Flush(2)); + + // PIT recovery & verify + options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery; + ASSERT_NOK(TryReopenWithColumnFamilies({"default", "one", "two"}, options)); +} + +TEST_F(DBWALTest, RaceInstallFlushResultsWithWalObsoletion) { + Options options = CurrentOptions(); + options.env = env_; + options.track_and_verify_wals_in_manifest = true; + // The following make sure there are two bg flush threads. + options.max_background_jobs = 8; + + DestroyAndReopen(options); + + const std::string cf1_name("cf1"); + CreateAndReopenWithCF({cf1_name}, options); + assert(handles_.size() == 2); + + { + dbfull()->TEST_LockMutex(); + ASSERT_LE(2, dbfull()->GetBGJobLimits().max_flushes); + dbfull()->TEST_UnlockMutex(); + } + + ASSERT_OK(dbfull()->PauseBackgroundWork()); + + ASSERT_OK(db_->Put(WriteOptions(), handles_[1], "foo", "value")); + ASSERT_OK(db_->Put(WriteOptions(), "foo", "value")); + + ASSERT_OK(dbfull()->TEST_FlushMemTable( + /*wait=*/false, /*allow_write_stall=*/true, handles_[1])); + + ASSERT_OK(db_->Put(WriteOptions(), "foo", "value")); + + ASSERT_OK(dbfull()->TEST_FlushMemTable( + /*wait=*/false, /*allow_write_stall=*/true, handles_[0])); + + bool called = false; + std::atomic bg_flush_threads{0}; + std::atomic wal_synced{false}; + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCallFlush:start", [&](void* /*arg*/) { + int cur = bg_flush_threads.load(); + int desired = cur + 1; + if (cur > 0 || + !bg_flush_threads.compare_exchange_strong(cur, desired)) { + while (!wal_synced.load()) { + // Wait until the other bg flush thread finishes committing WAL sync + // operation to the MANIFEST. + } + } + }); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::FlushMemTableToOutputFile:CommitWal:1", + [&](void* /*arg*/) { wal_synced.store(true); }); + // This callback will be called when the first bg flush thread reaches the + // point before entering the MANIFEST write queue after flushing the SST + // file. + // The purpose of the sync points here is to ensure both bg flush threads + // finish computing `min_wal_number_to_keep` before any of them updates the + // `log_number` for the column family that's being flushed. + SyncPoint::GetInstance()->SetCallBack( + "MemTableList::TryInstallMemtableFlushResults:AfterComputeMinWalToKeep", + [&](void* /*arg*/) { + dbfull()->mutex()->AssertHeld(); + if (!called) { + // We are the first bg flush thread in the MANIFEST write queue. + // We set up the dependency between sync points for two threads that + // will be executing the same code. + // For the interleaving of events, see + // https://github.com/facebook/rocksdb/pull/9715. + // bg flush thread1 will release the db mutex while in the MANIFEST + // write queue. In the meantime, bg flush thread2 locks db mutex and + // computes the min_wal_number_to_keep (before thread1 writes to + // MANIFEST thus before cf1->log_number is updated). Bg thread2 joins + // the MANIFEST write queue afterwards and bg flush thread1 proceeds + // with writing to MANIFEST. + called = true; + SyncPoint::GetInstance()->LoadDependency({ + {"VersionSet::LogAndApply:WriteManifestStart", + "DBWALTest::RaceInstallFlushResultsWithWalObsoletion:BgFlush2"}, + {"DBWALTest::RaceInstallFlushResultsWithWalObsoletion:BgFlush2", + "VersionSet::LogAndApply:WriteManifest"}, + }); + } else { + // The other bg flush thread has already been in the MANIFEST write + // queue, and we are after. + TEST_SYNC_POINT( + "DBWALTest::RaceInstallFlushResultsWithWalObsoletion:BgFlush2"); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(dbfull()->ContinueBackgroundWork()); + + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[0])); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); + + ASSERT_TRUE(called); + + Close(); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + DB* db1 = nullptr; + Status s = DB::OpenForReadOnly(options, dbname_, &db1); + ASSERT_OK(s); + assert(db1); + delete db1; +} + +TEST_F(DBWALTest, FixSyncWalOnObseletedWalWithNewManifestCausingMissingWAL) { + Options options = CurrentOptions(); + // Small size to force manifest creation + options.max_manifest_file_size = 1; + options.track_and_verify_wals_in_manifest = true; + DestroyAndReopen(options); + + // Accumulate memtable m1 and create the 1st wal (i.e, 4.log) + ASSERT_OK(Put(Key(1), "")); + ASSERT_OK(Put(Key(2), "")); + ASSERT_OK(Put(Key(3), "")); + + const std::string wal_file_path = db_->GetName() + "/000004.log"; + + // Coerce the following sequence of events: + // (1) Flush() marks 4.log to be obsoleted, 8.log to be the latest (i.e, + // active) log and release the lock + // (2) SyncWAL() proceeds with the lock. It + // creates a new manifest and syncs all the inactive wals before the latest + // (i.e, active log), which is 4.log. Note that SyncWAL() is not aware of the + // fact that 4.log has marked as to be obseleted. Such wal + // sync will then add a WAL addition record of 4.log to the new manifest + // without any special treatment. Prior to the fix, there is no WAL deletion + // record to offset it. (3) BackgroundFlush() will eventually purge 4.log. + + bool wal_synced = false; + SyncPoint::GetInstance()->SetCallBack( + "FindObsoleteFiles::PostMutexUnlock", [&](void*) { + ASSERT_OK(env_->FileExists(wal_file_path)); + uint64_t pre_sync_wal_manifest_no = + dbfull()->TEST_Current_Manifest_FileNo(); + ASSERT_OK(db_->SyncWAL()); + uint64_t post_sync_wal_manifest_no = + dbfull()->TEST_Current_Manifest_FileNo(); + bool new_manifest_created = + post_sync_wal_manifest_no == pre_sync_wal_manifest_no + 1; + ASSERT_TRUE(new_manifest_created); + wal_synced = true; + }); + + + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + + ASSERT_TRUE(wal_synced); + // BackgroundFlush() purged 4.log + // because the memtable associated with the WAL was flushed and new WAL was + // created (i.e, 8.log) + ASSERT_TRUE(env_->FileExists(wal_file_path).IsNotFound()); + + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); + + // To verify the corruption of "Missing WAL with log number: 4" under + // `options.track_and_verify_wals_in_manifest = true` is fixed. + // + // Before the fix, `db_->SyncWAL()` will sync and record WAL addtion of the + // obseleted WAL 4.log in a new manifest without any special treament. + // This will result in missing-wal corruption in DB::Reopen(). + Status s = TryReopen(options); + EXPECT_OK(s); +} + +// Test scope: +// - We expect to open data store under all circumstances +// - We expect only data upto the point where the first error was encountered +TEST_P(DBWALTestWithParams, kPointInTimeRecovery) { + const int maxkeys = + RecoveryTestHelper::kWALFilesCount * RecoveryTestHelper::kKeysPerWALFile; + + bool trunc = std::get<0>(GetParam()); // Corruption style + // Corruption offset position + int corrupt_offset = std::get<1>(GetParam()); + int wal_file_id = std::get<2>(GetParam()); // WAL file + // WAL compression type + CompressionType compression_type = std::get<3>(GetParam()); + + // Fill data for testing + Options options = CurrentOptions(); + options.wal_compression = compression_type; + const size_t row_count = RecoveryTestHelper::FillData(this, &options); + + // Corrupt the wal + // The offset here was 0.3 which cuts off right at the end of a + // valid fragment after wal zstd compression checksum is enabled, + // so changed the value to 0.33. + RecoveryTestHelper::CorruptWAL(this, options, corrupt_offset * .33, + /*len%=*/.1, wal_file_id, trunc); + + // Verify + options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery; + options.create_if_missing = false; + ASSERT_OK(TryReopen(options)); + + // Probe data for invariants + size_t recovered_row_count = RecoveryTestHelper::GetData(this); + ASSERT_LT(recovered_row_count, row_count); + + // Verify a prefix of keys were recovered. But not in the case of full WAL + // truncation, because we have no way to know there was a corruption when + // truncation happened on record boundaries (preventing recovery holes in + // that case requires using `track_and_verify_wals_in_manifest`). + if (!trunc || corrupt_offset != 0) { + bool expect_data = true; + for (size_t k = 0; k < maxkeys; ++k) { + bool found = Get("key" + std::to_string(k)) != "NOT_FOUND"; + if (expect_data && !found) { + expect_data = false; + } + ASSERT_EQ(found, expect_data); + } + } + + const size_t min = RecoveryTestHelper::kKeysPerWALFile * + (wal_file_id - RecoveryTestHelper::kWALFileOffset); + ASSERT_GE(recovered_row_count, min); + if (!trunc && corrupt_offset != 0) { + const size_t max = RecoveryTestHelper::kKeysPerWALFile * + (wal_file_id - RecoveryTestHelper::kWALFileOffset + 1); + ASSERT_LE(recovered_row_count, max); + } +} + +// Test scope: +// - We expect to open the data store under all scenarios +// - We expect to have recovered records past the corruption zone +TEST_P(DBWALTestWithParams, kSkipAnyCorruptedRecords) { + bool trunc = std::get<0>(GetParam()); // Corruption style + // Corruption offset position + int corrupt_offset = std::get<1>(GetParam()); + int wal_file_id = std::get<2>(GetParam()); // WAL file + // WAL compression type + CompressionType compression_type = std::get<3>(GetParam()); + + // Fill data for testing + Options options = CurrentOptions(); + options.wal_compression = compression_type; + const size_t row_count = RecoveryTestHelper::FillData(this, &options); + + // Corrupt the WAL + RecoveryTestHelper::CorruptWAL(this, options, corrupt_offset * .3, + /*len%=*/.1, wal_file_id, trunc); + + // Verify behavior + options.wal_recovery_mode = WALRecoveryMode::kSkipAnyCorruptedRecords; + options.create_if_missing = false; + ASSERT_OK(TryReopen(options)); + + // Probe data for invariants + size_t recovered_row_count = RecoveryTestHelper::GetData(this); + ASSERT_LT(recovered_row_count, row_count); + + if (!trunc) { + ASSERT_TRUE(corrupt_offset != 0 || recovered_row_count > 0); + } +} + +TEST_F(DBWALTest, AvoidFlushDuringRecovery) { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.avoid_flush_during_recovery = false; + + // Test with flush after recovery. + Reopen(options); + ASSERT_OK(Put("foo", "v1")); + ASSERT_OK(Put("bar", "v2")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("foo", "v3")); + ASSERT_OK(Put("bar", "v4")); + ASSERT_EQ(1, TotalTableFiles()); + // Reopen DB. Check if WAL logs flushed. + Reopen(options); + ASSERT_EQ("v3", Get("foo")); + ASSERT_EQ("v4", Get("bar")); + ASSERT_EQ(2, TotalTableFiles()); + + // Test without flush after recovery. + options.avoid_flush_during_recovery = true; + DestroyAndReopen(options); + ASSERT_OK(Put("foo", "v5")); + ASSERT_OK(Put("bar", "v6")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("foo", "v7")); + ASSERT_OK(Put("bar", "v8")); + ASSERT_EQ(1, TotalTableFiles()); + // Reopen DB. WAL logs should not be flushed this time. + Reopen(options); + ASSERT_EQ("v7", Get("foo")); + ASSERT_EQ("v8", Get("bar")); + ASSERT_EQ(1, TotalTableFiles()); + + // Force flush with allow_2pc. + options.avoid_flush_during_recovery = true; + options.allow_2pc = true; + ASSERT_OK(Put("foo", "v9")); + ASSERT_OK(Put("bar", "v10")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("foo", "v11")); + ASSERT_OK(Put("bar", "v12")); + Reopen(options); + ASSERT_EQ("v11", Get("foo")); + ASSERT_EQ("v12", Get("bar")); + ASSERT_EQ(3, TotalTableFiles()); +} + +TEST_F(DBWALTest, WalCleanupAfterAvoidFlushDuringRecovery) { + // Verifies WAL files that were present during recovery, but not flushed due + // to avoid_flush_during_recovery, will be considered for deletion at a later + // stage. We check at least one such file is deleted during Flush(). + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.avoid_flush_during_recovery = true; + Reopen(options); + + ASSERT_OK(Put("foo", "v1")); + Reopen(options); + for (int i = 0; i < 2; ++i) { + if (i > 0) { + // Flush() triggers deletion of obsolete tracked files + ASSERT_OK(Flush()); + } + VectorLogPtr log_files; + ASSERT_OK(dbfull()->GetSortedWalFiles(log_files)); + if (i == 0) { + ASSERT_GT(log_files.size(), 0); + } else { + ASSERT_EQ(0, log_files.size()); + } + } +} + +TEST_F(DBWALTest, RecoverWithoutFlush) { + Options options = CurrentOptions(); + options.avoid_flush_during_recovery = true; + options.create_if_missing = false; + options.disable_auto_compactions = true; + options.write_buffer_size = 64 * 1024 * 1024; + + size_t count = RecoveryTestHelper::FillData(this, &options); + auto validateData = [this, count]() { + for (size_t i = 0; i < count; i++) { + ASSERT_NE(Get("key" + std::to_string(i)), "NOT_FOUND"); + } + }; + Reopen(options); + validateData(); + // Insert some data without flush + ASSERT_OK(Put("foo", "foo_v1")); + ASSERT_OK(Put("bar", "bar_v1")); + Reopen(options); + validateData(); + ASSERT_EQ(Get("foo"), "foo_v1"); + ASSERT_EQ(Get("bar"), "bar_v1"); + // Insert again and reopen + ASSERT_OK(Put("foo", "foo_v2")); + ASSERT_OK(Put("bar", "bar_v2")); + Reopen(options); + validateData(); + ASSERT_EQ(Get("foo"), "foo_v2"); + ASSERT_EQ(Get("bar"), "bar_v2"); + // manual flush and insert again + ASSERT_OK(Flush()); + ASSERT_EQ(Get("foo"), "foo_v2"); + ASSERT_EQ(Get("bar"), "bar_v2"); + ASSERT_OK(Put("foo", "foo_v3")); + ASSERT_OK(Put("bar", "bar_v3")); + Reopen(options); + validateData(); + ASSERT_EQ(Get("foo"), "foo_v3"); + ASSERT_EQ(Get("bar"), "bar_v3"); +} + +TEST_F(DBWALTest, RecoverWithoutFlushMultipleCF) { + const std::string kSmallValue = "v"; + const std::string kLargeValue = DummyString(1024); + Options options = CurrentOptions(); + options.avoid_flush_during_recovery = true; + options.create_if_missing = false; + options.disable_auto_compactions = true; + + auto countWalFiles = [this]() { + VectorLogPtr log_files; + if (!dbfull()->GetSortedWalFiles(log_files).ok()) { + return size_t{0}; + } + return log_files.size(); + }; + + // Create DB with multiple column families and multiple log files. + CreateAndReopenWithCF({"one", "two"}, options); + ASSERT_OK(Put(0, "key1", kSmallValue)); + ASSERT_OK(Put(1, "key2", kLargeValue)); + ASSERT_OK(Flush(1)); + ASSERT_EQ(1, countWalFiles()); + ASSERT_OK(Put(0, "key3", kSmallValue)); + ASSERT_OK(Put(2, "key4", kLargeValue)); + ASSERT_OK(Flush(2)); + ASSERT_EQ(2, countWalFiles()); + + // Reopen, insert and flush. + options.db_write_buffer_size = 64 * 1024 * 1024; + ReopenWithColumnFamilies({"default", "one", "two"}, options); + ASSERT_EQ(Get(0, "key1"), kSmallValue); + ASSERT_EQ(Get(1, "key2"), kLargeValue); + ASSERT_EQ(Get(0, "key3"), kSmallValue); + ASSERT_EQ(Get(2, "key4"), kLargeValue); + // Insert more data. + ASSERT_OK(Put(0, "key5", kLargeValue)); + ASSERT_OK(Put(1, "key6", kLargeValue)); + ASSERT_EQ(3, countWalFiles()); + ASSERT_OK(Flush(1)); + ASSERT_OK(Put(2, "key7", kLargeValue)); + ASSERT_OK(dbfull()->FlushWAL(false)); + ASSERT_EQ(4, countWalFiles()); + + // Reopen twice and validate. + for (int i = 0; i < 2; i++) { + ReopenWithColumnFamilies({"default", "one", "two"}, options); + ASSERT_EQ(Get(0, "key1"), kSmallValue); + ASSERT_EQ(Get(1, "key2"), kLargeValue); + ASSERT_EQ(Get(0, "key3"), kSmallValue); + ASSERT_EQ(Get(2, "key4"), kLargeValue); + ASSERT_EQ(Get(0, "key5"), kLargeValue); + ASSERT_EQ(Get(1, "key6"), kLargeValue); + ASSERT_EQ(Get(2, "key7"), kLargeValue); + ASSERT_EQ(4, countWalFiles()); + } +} + +// In this test we are trying to do the following: +// 1. Create a DB with corrupted WAL log; +// 2. Open with avoid_flush_during_recovery = true; +// 3. Append more data without flushing, which creates new WAL log. +// 4. Open again. See if it can correctly handle previous corruption. +TEST_P(DBWALTestWithParamsVaryingRecoveryMode, + RecoverFromCorruptedWALWithoutFlush) { + const int kAppendKeys = 100; + Options options = CurrentOptions(); + options.avoid_flush_during_recovery = true; + options.create_if_missing = false; + options.disable_auto_compactions = true; + options.write_buffer_size = 64 * 1024 * 1024; + + auto getAll = [this]() { + std::vector> data; + ReadOptions ropt; + Iterator* iter = dbfull()->NewIterator(ropt); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + data.push_back( + std::make_pair(iter->key().ToString(), iter->value().ToString())); + } + delete iter; + return data; + }; + + bool trunc = std::get<0>(GetParam()); // Corruption style + // Corruption offset position + int corrupt_offset = std::get<1>(GetParam()); + int wal_file_id = std::get<2>(GetParam()); // WAL file + WALRecoveryMode recovery_mode = std::get<3>(GetParam()); + // WAL compression type + CompressionType compression_type = std::get<4>(GetParam()); + + options.wal_recovery_mode = recovery_mode; + options.wal_compression = compression_type; + // Create corrupted WAL + RecoveryTestHelper::FillData(this, &options); + RecoveryTestHelper::CorruptWAL(this, options, corrupt_offset * .3, + /*len%=*/.1, wal_file_id, trunc); + // Skip the test if DB won't open. + if (!TryReopen(options).ok()) { + ASSERT_TRUE(options.wal_recovery_mode == + WALRecoveryMode::kAbsoluteConsistency || + (!trunc && options.wal_recovery_mode == + WALRecoveryMode::kTolerateCorruptedTailRecords)); + return; + } + ASSERT_OK(TryReopen(options)); + // Append some more data. + for (int k = 0; k < kAppendKeys; k++) { + std::string key = "extra_key" + std::to_string(k); + std::string value = DummyString(RecoveryTestHelper::kValueSize); + ASSERT_OK(Put(key, value)); + } + // Save data for comparison. + auto data = getAll(); + // Reopen. Verify data. + ASSERT_OK(TryReopen(options)); + auto actual_data = getAll(); + ASSERT_EQ(data, actual_data); +} + +// Tests that total log size is recovered if we set +// avoid_flush_during_recovery=true. +// Flush should trigger if max_total_wal_size is reached. +TEST_F(DBWALTest, RestoreTotalLogSizeAfterRecoverWithoutFlush) { + auto test_listener = std::make_shared(); + test_listener->expected_flush_reason = FlushReason::kWalFull; + + constexpr size_t kKB = 1024; + constexpr size_t kMB = 1024 * 1024; + Options options = CurrentOptions(); + options.avoid_flush_during_recovery = true; + options.max_total_wal_size = 1 * kMB; + options.listeners.push_back(test_listener); + // Have to open DB in multi-CF mode to trigger flush when + // max_total_wal_size is reached. + CreateAndReopenWithCF({"one"}, options); + // Write some keys and we will end up with one log file which is slightly + // smaller than 1MB. + std::string value_100k(100 * kKB, 'v'); + std::string value_300k(300 * kKB, 'v'); + ASSERT_OK(Put(0, "foo", "v1")); + for (int i = 0; i < 9; i++) { + ASSERT_OK(Put(1, "key" + std::to_string(i), value_100k)); + } + // Get log files before reopen. + VectorLogPtr log_files_before; + ASSERT_OK(dbfull()->GetSortedWalFiles(log_files_before)); + ASSERT_EQ(1, log_files_before.size()); + uint64_t log_size_before = log_files_before[0]->SizeFileBytes(); + ASSERT_GT(log_size_before, 900 * kKB); + ASSERT_LT(log_size_before, 1 * kMB); + ReopenWithColumnFamilies({"default", "one"}, options); + // Write one more value to make log larger than 1MB. + ASSERT_OK(Put(1, "bar", value_300k)); + // Get log files again. A new log file will be opened. + VectorLogPtr log_files_after_reopen; + ASSERT_OK(dbfull()->GetSortedWalFiles(log_files_after_reopen)); + ASSERT_EQ(2, log_files_after_reopen.size()); + ASSERT_EQ(log_files_before[0]->LogNumber(), + log_files_after_reopen[0]->LogNumber()); + ASSERT_GT(log_files_after_reopen[0]->SizeFileBytes() + + log_files_after_reopen[1]->SizeFileBytes(), + 1 * kMB); + // Write one more key to trigger flush. + ASSERT_OK(Put(0, "foo", "v2")); + for (auto* h : handles_) { + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(h)); + } + // Flushed two column families. + ASSERT_EQ(2, test_listener->count.load()); +} + +#if defined(ROCKSDB_PLATFORM_POSIX) +#if defined(ROCKSDB_FALLOCATE_PRESENT) +// Tests that we will truncate the preallocated space of the last log from +// previous. +TEST_F(DBWALTest, TruncateLastLogAfterRecoverWithoutFlush) { + constexpr size_t kKB = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.avoid_flush_during_recovery = true; + if (mem_env_) { + ROCKSDB_GTEST_SKIP("Test requires non-mem environment"); + return; + } + if (!IsFallocateSupported()) { + return; + } + + DestroyAndReopen(options); + size_t preallocated_size = + dbfull()->TEST_GetWalPreallocateBlockSize(options.write_buffer_size); + ASSERT_OK(Put("foo", "v1")); + VectorLogPtr log_files_before; + ASSERT_OK(dbfull()->GetSortedWalFiles(log_files_before)); + ASSERT_EQ(1, log_files_before.size()); + auto& file_before = log_files_before[0]; + ASSERT_LT(file_before->SizeFileBytes(), 1 * kKB); + // The log file has preallocated space. + ASSERT_GE(GetAllocatedFileSize(dbname_ + file_before->PathName()), + preallocated_size); + Reopen(options); + VectorLogPtr log_files_after; + ASSERT_OK(dbfull()->GetSortedWalFiles(log_files_after)); + ASSERT_EQ(1, log_files_after.size()); + ASSERT_LT(log_files_after[0]->SizeFileBytes(), 1 * kKB); + // The preallocated space should be truncated. + ASSERT_LT(GetAllocatedFileSize(dbname_ + file_before->PathName()), + preallocated_size); +} +// Tests that we will truncate the preallocated space of the last log from +// previous. +TEST_F(DBWALTest, TruncateLastLogAfterRecoverWithFlush) { + constexpr size_t kKB = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.avoid_flush_during_recovery = false; + options.avoid_flush_during_shutdown = true; + if (mem_env_) { + ROCKSDB_GTEST_SKIP("Test requires non-mem environment"); + return; + } + if (!IsFallocateSupported()) { + return; + } + + DestroyAndReopen(options); + size_t preallocated_size = + dbfull()->TEST_GetWalPreallocateBlockSize(options.write_buffer_size); + ASSERT_OK(Put("foo", "v1")); + VectorLogPtr log_files_before; + ASSERT_OK(dbfull()->GetSortedWalFiles(log_files_before)); + ASSERT_EQ(1, log_files_before.size()); + auto& file_before = log_files_before[0]; + ASSERT_LT(file_before->SizeFileBytes(), 1 * kKB); + ASSERT_GE(GetAllocatedFileSize(dbname_ + file_before->PathName()), + preallocated_size); + // The log file has preallocated space. + Close(); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::PurgeObsoleteFiles:Begin", + "DBWALTest::TruncateLastLogAfterRecoverWithFlush:AfterRecover"}, + {"DBWALTest::TruncateLastLogAfterRecoverWithFlush:AfterTruncate", + "DBImpl::DeleteObsoleteFileImpl::BeforeDeletion"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + port::Thread reopen_thread([&]() { Reopen(options); }); + + TEST_SYNC_POINT( + "DBWALTest::TruncateLastLogAfterRecoverWithFlush:AfterRecover"); + // After the flush during Open, the log file should get deleted. However, + // if the process is in a crash loop, the log file may not get + // deleted and thte preallocated space will keep accumulating. So we need + // to ensure it gets trtuncated. + EXPECT_LT(GetAllocatedFileSize(dbname_ + file_before->PathName()), + preallocated_size); + TEST_SYNC_POINT( + "DBWALTest::TruncateLastLogAfterRecoverWithFlush:AfterTruncate"); + reopen_thread.join(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DBWALTest, TruncateLastLogAfterRecoverWALEmpty) { + Options options = CurrentOptions(); + options.env = env_; + options.avoid_flush_during_recovery = false; + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_SKIP("Test requires non-mem/non-encrypted environment"); + return; + } + if (!IsFallocateSupported()) { + return; + } + + DestroyAndReopen(options); + size_t preallocated_size = + dbfull()->TEST_GetWalPreallocateBlockSize(options.write_buffer_size); + Close(); + std::vector filenames; + std::string last_log; + uint64_t last_log_num = 0; + ASSERT_OK(env_->GetChildren(dbname_, &filenames)); + for (auto fname : filenames) { + uint64_t number; + FileType type; + if (ParseFileName(fname, &number, &type, nullptr)) { + if (type == kWalFile && number > last_log_num) { + last_log = fname; + } + } + } + ASSERT_NE(last_log, ""); + last_log = dbname_ + '/' + last_log; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::PurgeObsoleteFiles:Begin", + "DBWALTest::TruncateLastLogAfterRecoverWithFlush:AfterRecover"}, + {"DBWALTest::TruncateLastLogAfterRecoverWithFlush:AfterTruncate", + "DBImpl::DeleteObsoleteFileImpl::BeforeDeletion"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "PosixWritableFile::Close", + [](void* arg) { *(reinterpret_cast(arg)) = 0; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + // Preallocate space for the empty log file. This could happen if WAL data + // was buffered in memory and the process crashed. + std::unique_ptr log_file; + ASSERT_OK(env_->ReopenWritableFile(last_log, &log_file, EnvOptions())); + log_file->SetPreallocationBlockSize(preallocated_size); + log_file->PrepareWrite(0, 4096); + log_file.reset(); + + ASSERT_GE(GetAllocatedFileSize(last_log), preallocated_size); + + port::Thread reopen_thread([&]() { Reopen(options); }); + + TEST_SYNC_POINT( + "DBWALTest::TruncateLastLogAfterRecoverWithFlush:AfterRecover"); + // The preallocated space should be truncated. + EXPECT_LT(GetAllocatedFileSize(last_log), preallocated_size); + TEST_SYNC_POINT( + "DBWALTest::TruncateLastLogAfterRecoverWithFlush:AfterTruncate"); + reopen_thread.join(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(DBWALTest, ReadOnlyRecoveryNoTruncate) { + constexpr size_t kKB = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.avoid_flush_during_recovery = true; + if (mem_env_) { + ROCKSDB_GTEST_SKIP("Test requires non-mem environment"); + return; + } + if (!IsFallocateSupported()) { + return; + } + + // create DB and close with file truncate disabled + std::atomic_bool enable_truncate{false}; + + SyncPoint::GetInstance()->SetCallBack( + "PosixWritableFile::Close", [&](void* arg) { + if (!enable_truncate) { + *(reinterpret_cast(arg)) = 0; + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + + DestroyAndReopen(options); + size_t preallocated_size = + dbfull()->TEST_GetWalPreallocateBlockSize(options.write_buffer_size); + ASSERT_OK(Put("foo", "v1")); + VectorLogPtr log_files_before; + ASSERT_OK(dbfull()->GetSortedWalFiles(log_files_before)); + ASSERT_EQ(1, log_files_before.size()); + auto& file_before = log_files_before[0]; + ASSERT_LT(file_before->SizeFileBytes(), 1 * kKB); + // The log file has preallocated space. + auto db_size = GetAllocatedFileSize(dbname_ + file_before->PathName()); + ASSERT_GE(db_size, preallocated_size); + Close(); + + // enable truncate and open DB as readonly, the file should not be truncated + // and DB size is not changed. + enable_truncate = true; + ASSERT_OK(ReadOnlyReopen(options)); + VectorLogPtr log_files_after; + ASSERT_OK(dbfull()->GetSortedWalFiles(log_files_after)); + ASSERT_EQ(1, log_files_after.size()); + ASSERT_LT(log_files_after[0]->SizeFileBytes(), 1 * kKB); + ASSERT_EQ(log_files_after[0]->PathName(), file_before->PathName()); + // The preallocated space should NOT be truncated. + // the DB size is almost the same. + ASSERT_NEAR(GetAllocatedFileSize(dbname_ + file_before->PathName()), db_size, + db_size / 100); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} +#endif // ROCKSDB_FALLOCATE_PRESENT +#endif // ROCKSDB_PLATFORM_POSIX + +TEST_F(DBWALTest, WalInManifestButNotInSortedWals) { + Options options = CurrentOptions(); + options.track_and_verify_wals_in_manifest = true; + options.wal_recovery_mode = WALRecoveryMode::kAbsoluteConsistency; + + // Build a way to make wal files selectively go missing + bool wals_go_missing = false; + struct MissingWalFs : public FileSystemWrapper { + MissingWalFs(const std::shared_ptr& t, + bool* _wals_go_missing_flag) + : FileSystemWrapper(t), wals_go_missing_flag(_wals_go_missing_flag) {} + bool* wals_go_missing_flag; + IOStatus GetChildren(const std::string& dir, const IOOptions& io_opts, + std::vector* r, + IODebugContext* dbg) override { + IOStatus s = target_->GetChildren(dir, io_opts, r, dbg); + if (s.ok() && *wals_go_missing_flag) { + for (size_t i = 0; i < r->size();) { + if (EndsWith(r->at(i), ".log")) { + r->erase(r->begin() + i); + } else { + ++i; + } + } + } + return s; + } + const char* Name() const override { return "MissingWalFs"; } + }; + auto my_fs = + std::make_shared(env_->GetFileSystem(), &wals_go_missing); + std::unique_ptr my_env(NewCompositeEnv(my_fs)); + options.env = my_env.get(); + + CreateAndReopenWithCF({"blah"}, options); + + // Currently necessary to get a WAL tracked in manifest; see + // https://github.com/facebook/rocksdb/issues/10080 + ASSERT_OK(Put(0, "x", "y")); + ASSERT_OK(db_->SyncWAL()); + ASSERT_OK(Put(1, "x", "y")); + ASSERT_OK(db_->SyncWAL()); + ASSERT_OK(Flush(1)); + + ASSERT_FALSE(dbfull()->GetVersionSet()->GetWalSet().GetWals().empty()); + std::vector> wals; + ASSERT_OK(db_->GetSortedWalFiles(wals)); + wals_go_missing = true; + ASSERT_NOK(db_->GetSortedWalFiles(wals)); + wals_go_missing = false; + Close(); +} + + +TEST_F(DBWALTest, WalTermTest) { + Options options = CurrentOptions(); + options.env = env_; + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(1, "foo", "bar")); + + WriteOptions wo; + wo.sync = true; + wo.disableWAL = false; + + WriteBatch batch; + ASSERT_OK(batch.Put("foo", "bar")); + batch.MarkWalTerminationPoint(); + ASSERT_OK(batch.Put("foo2", "bar2")); + + ASSERT_OK(dbfull()->Write(wo, &batch)); + + // make sure we can re-open it. + ASSERT_OK(TryReopenWithColumnFamilies({"default", "pikachu"}, options)); + ASSERT_EQ("bar", Get(1, "foo")); + ASSERT_EQ("NOT_FOUND", Get(1, "foo2")); +} + +TEST_F(DBWALTest, GetCompressedWalsAfterSync) { + if (db_->GetOptions().wal_compression == kNoCompression) { + ROCKSDB_GTEST_BYPASS("stream compression not present"); + return; + } + Options options = GetDefaultOptions(); + options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery; + options.create_if_missing = true; + options.env = env_; + options.avoid_flush_during_recovery = true; + options.track_and_verify_wals_in_manifest = true; + // Enable WAL compression so that the newly-created WAL will be non-empty + // after DB open, even if point-in-time WAL recovery encounters no + // corruption. + options.wal_compression = kZSTD; + DestroyAndReopen(options); + + // Write something to memtable and WAL so that log_empty_ will be false after + // next DB::Open(). + ASSERT_OK(Put("a", "v")); + + Reopen(options); + + // New WAL is created, thanks to !log_empty_. + ASSERT_OK(dbfull()->TEST_SwitchWAL()); + + ASSERT_OK(Put("b", "v")); + + ASSERT_OK(db_->SyncWAL()); + + VectorLogPtr wals; + Status s = dbfull()->GetSortedWalFiles(wals); + ASSERT_OK(s); +} + +TEST_F(DBWALTest, EmptyWalReopenTest) { + Options options = CurrentOptions(); + options.env = env_; + CreateAndReopenWithCF({"pikachu"}, options); + + // make sure we can re-open it. + ASSERT_OK(TryReopenWithColumnFamilies({"default", "pikachu"}, options)); + + { + std::vector files; + int num_wal_files = 0; + ASSERT_OK(env_->GetChildren(dbname_, &files)); + for (const auto& file : files) { + uint64_t number = 0; + FileType type = kWalFile; + if (ParseFileName(file, &number, &type) && type == kWalFile) { + num_wal_files++; + } + } + + ASSERT_EQ(num_wal_files, 1); + } +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_with_timestamp_basic_test.cc b/librocksdb-sys/rocksdb/db/db_with_timestamp_basic_test.cc new file mode 100644 index 0000000..202c4c3 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_with_timestamp_basic_test.cc @@ -0,0 +1,4152 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/db_with_timestamp_test_util.h" +#include "port/stack_trace.h" +#include "rocksdb/perf_context.h" +#include "rocksdb/utilities/debug.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/block_based/block_builder.h" +#include "test_util/sync_point.h" +#include "test_util/testutil.h" +#include "utilities/fault_injection_env.h" +#include "utilities/merge_operators/string_append/stringappend2.h" + +namespace ROCKSDB_NAMESPACE { +class DBBasicTestWithTimestamp : public DBBasicTestWithTimestampBase { + public: + DBBasicTestWithTimestamp() + : DBBasicTestWithTimestampBase("db_basic_test_with_timestamp") {} +}; + +TEST_F(DBBasicTestWithTimestamp, SanityChecks) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.avoid_flush_during_shutdown = true; + options.merge_operator = MergeOperators::CreateStringAppendTESTOperator(); + DestroyAndReopen(options); + + Options options1 = CurrentOptions(); + options1.env = env_; + options1.comparator = test::BytewiseComparatorWithU64TsWrapper(); + options1.merge_operator = MergeOperators::CreateStringAppendTESTOperator(); + assert(options1.comparator && + options1.comparator->timestamp_size() == sizeof(uint64_t)); + ColumnFamilyHandle* handle = nullptr; + Status s = db_->CreateColumnFamily(options1, "data", &handle); + ASSERT_OK(s); + + std::string dummy_ts(sizeof(uint64_t), '\0'); + // Perform timestamp operations on default cf. + ASSERT_TRUE( + db_->Put(WriteOptions(), "key", dummy_ts, "value").IsInvalidArgument()); + ASSERT_TRUE(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), "key", + dummy_ts, "value") + .IsInvalidArgument()); + ASSERT_TRUE(db_->Delete(WriteOptions(), "key", dummy_ts).IsInvalidArgument()); + ASSERT_TRUE( + db_->SingleDelete(WriteOptions(), "key", dummy_ts).IsInvalidArgument()); + ASSERT_TRUE(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + "begin_key", "end_key", dummy_ts) + .IsInvalidArgument()); + + // Perform non-timestamp operations on "data" cf. + ASSERT_TRUE( + db_->Put(WriteOptions(), handle, "key", "value").IsInvalidArgument()); + ASSERT_TRUE(db_->Delete(WriteOptions(), handle, "key").IsInvalidArgument()); + ASSERT_TRUE( + db_->SingleDelete(WriteOptions(), handle, "key").IsInvalidArgument()); + + ASSERT_TRUE( + db_->Merge(WriteOptions(), handle, "key", "value").IsInvalidArgument()); + ASSERT_TRUE(db_->DeleteRange(WriteOptions(), handle, "begin_key", "end_key") + .IsInvalidArgument()); + + { + WriteBatch wb; + ASSERT_OK(wb.Put(handle, "key", "value")); + ASSERT_TRUE(db_->Write(WriteOptions(), &wb).IsInvalidArgument()); + } + { + WriteBatch wb; + ASSERT_OK(wb.Delete(handle, "key")); + ASSERT_TRUE(db_->Write(WriteOptions(), &wb).IsInvalidArgument()); + } + { + WriteBatch wb; + ASSERT_OK(wb.SingleDelete(handle, "key")); + ASSERT_TRUE(db_->Write(WriteOptions(), &wb).IsInvalidArgument()); + } + { + WriteBatch wb; + ASSERT_OK(wb.DeleteRange(handle, "begin_key", "end_key")); + ASSERT_TRUE(db_->Write(WriteOptions(), &wb).IsInvalidArgument()); + } + + // Perform timestamp operations with timestamps of incorrect size. + const std::string wrong_ts(sizeof(uint32_t), '\0'); + ASSERT_TRUE(db_->Put(WriteOptions(), handle, "key", wrong_ts, "value") + .IsInvalidArgument()); + ASSERT_TRUE(db_->Merge(WriteOptions(), handle, "key", wrong_ts, "value") + .IsInvalidArgument()); + ASSERT_TRUE( + db_->Delete(WriteOptions(), handle, "key", wrong_ts).IsInvalidArgument()); + ASSERT_TRUE(db_->SingleDelete(WriteOptions(), handle, "key", wrong_ts) + .IsInvalidArgument()); + ASSERT_TRUE( + db_->DeleteRange(WriteOptions(), handle, "begin_key", "end_key", wrong_ts) + .IsInvalidArgument()); + + delete handle; +} + +TEST_F(DBBasicTestWithTimestamp, MixedCfs) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.avoid_flush_during_shutdown = true; + DestroyAndReopen(options); + + Options options1 = CurrentOptions(); + options1.env = env_; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options1.comparator = &test_cmp; + ColumnFamilyHandle* handle = nullptr; + Status s = db_->CreateColumnFamily(options1, "data", &handle); + ASSERT_OK(s); + + WriteBatch wb; + ASSERT_OK(wb.Put("a", "value")); + ASSERT_OK(wb.Put(handle, "a", "value")); + { + std::string ts = Timestamp(1, 0); + const auto ts_sz_func = [kTimestampSize, handle](uint32_t cf_id) { + assert(handle); + if (cf_id == 0) { + return static_cast(0); + } else if (cf_id == handle->GetID()) { + return kTimestampSize; + } else { + assert(false); + return std::numeric_limits::max(); + } + }; + ASSERT_OK(wb.UpdateTimestamps(ts, ts_sz_func)); + ASSERT_OK(db_->Write(WriteOptions(), &wb)); + } + + const auto verify_db = [this](ColumnFamilyHandle* h, const std::string& key, + const std::string& ts, + const std::string& expected_value) { + ASSERT_EQ(expected_value, Get(key)); + Slice read_ts_slice(ts); + ReadOptions read_opts; + read_opts.timestamp = &read_ts_slice; + std::string value; + ASSERT_OK(db_->Get(read_opts, h, key, &value)); + ASSERT_EQ(expected_value, value); + }; + + verify_db(handle, "a", Timestamp(1, 0), "value"); + + delete handle; + Close(); + + std::vector cf_descs; + cf_descs.emplace_back(kDefaultColumnFamilyName, options); + cf_descs.emplace_back("data", options1); + options.create_if_missing = false; + s = DB::Open(options, dbname_, cf_descs, &handles_, &db_); + ASSERT_OK(s); + + verify_db(handles_[1], "a", Timestamp(1, 0), "value"); + + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, CompactRangeWithSpecifiedRange) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + + WriteOptions write_opts; + std::string ts = Timestamp(1, 0); + + ASSERT_OK(db_->Put(write_opts, "foo1", ts, "bar")); + ASSERT_OK(Flush()); + + ASSERT_OK(db_->Put(write_opts, "foo2", ts, "bar")); + ASSERT_OK(Flush()); + + std::string start_str = "foo"; + std::string end_str = "foo2"; + Slice start(start_str), end(end_str); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &start, &end)); + + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, GcPreserveLatestVersionBelowFullHistoryLow) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + + std::string ts_str = Timestamp(1, 0); + WriteOptions wopts; + ASSERT_OK(db_->Put(wopts, "k1", ts_str, "v1")); + ASSERT_OK(db_->Put(wopts, "k2", ts_str, "v2")); + ASSERT_OK(db_->Put(wopts, "k3", ts_str, "v3")); + + ts_str = Timestamp(2, 0); + ASSERT_OK(db_->Delete(wopts, "k3", ts_str)); + + ts_str = Timestamp(4, 0); + ASSERT_OK(db_->Put(wopts, "k1", ts_str, "v5")); + + ts_str = Timestamp(5, 0); + ASSERT_OK( + db_->DeleteRange(wopts, db_->DefaultColumnFamily(), "k0", "k9", ts_str)); + + ts_str = Timestamp(3, 0); + Slice ts = ts_str; + CompactRangeOptions cro; + cro.full_history_ts_low = &ts; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + ASSERT_OK(Flush()); + + ReadOptions ropts; + ropts.timestamp = &ts; + std::string value; + Status s = db_->Get(ropts, "k1", &value); + ASSERT_OK(s); + ASSERT_EQ("v1", value); + + std::string key_ts; + ASSERT_TRUE(db_->Get(ropts, "k3", &value, &key_ts).IsNotFound()); + ASSERT_EQ(Timestamp(2, 0), key_ts); + + ts_str = Timestamp(5, 0); + ts = ts_str; + ropts.timestamp = &ts; + ASSERT_TRUE(db_->Get(ropts, "k2", &value, &key_ts).IsNotFound()); + ASSERT_EQ(Timestamp(5, 0), key_ts); + ASSERT_TRUE(db_->Get(ropts, "k2", &value).IsNotFound()); + + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, UpdateFullHistoryTsLow) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + + const std::string kKey = "test kKey"; + + // Test set ts_low first and flush() + int current_ts_low = 5; + std::string ts_low_str = Timestamp(current_ts_low, 0); + Slice ts_low = ts_low_str; + CompactRangeOptions comp_opts; + comp_opts.full_history_ts_low = &ts_low; + comp_opts.bottommost_level_compaction = BottommostLevelCompaction::kForce; + + ASSERT_OK(db_->CompactRange(comp_opts, nullptr, nullptr)); + + auto* cfd = + static_cast_with_check(db_->DefaultColumnFamily()) + ->cfd(); + auto result_ts_low = cfd->GetFullHistoryTsLow(); + + ASSERT_TRUE(test_cmp.CompareTimestamp(ts_low, result_ts_low) == 0); + + for (int i = 0; i < 10; i++) { + WriteOptions write_opts; + std::string ts = Timestamp(i, 0); + ASSERT_OK(db_->Put(write_opts, kKey, ts, Key(i))); + } + ASSERT_OK(Flush()); + + for (int i = 0; i < 10; i++) { + ReadOptions read_opts; + std::string ts_str = Timestamp(i, 0); + Slice ts = ts_str; + read_opts.timestamp = &ts; + std::string value; + Status status = db_->Get(read_opts, kKey, &value); + if (i < current_ts_low) { + ASSERT_TRUE(status.IsInvalidArgument()); + } else { + ASSERT_OK(status); + ASSERT_TRUE(value.compare(Key(i)) == 0); + } + } + + // Test set ts_low and then trigger compaction + for (int i = 10; i < 20; i++) { + WriteOptions write_opts; + std::string ts = Timestamp(i, 0); + ASSERT_OK(db_->Put(write_opts, kKey, ts, Key(i))); + } + + ASSERT_OK(Flush()); + + current_ts_low = 15; + ts_low_str = Timestamp(current_ts_low, 0); + ts_low = ts_low_str; + comp_opts.full_history_ts_low = &ts_low; + ASSERT_OK(db_->CompactRange(comp_opts, nullptr, nullptr)); + result_ts_low = cfd->GetFullHistoryTsLow(); + ASSERT_TRUE(test_cmp.CompareTimestamp(ts_low, result_ts_low) == 0); + + for (int i = current_ts_low; i < 20; i++) { + ReadOptions read_opts; + std::string ts_str = Timestamp(i, 0); + Slice ts = ts_str; + read_opts.timestamp = &ts; + std::string value; + Status status = db_->Get(read_opts, kKey, &value); + ASSERT_OK(status); + ASSERT_TRUE(value.compare(Key(i)) == 0); + } + + // Test invalid compaction with range + Slice start(kKey), end(kKey); + Status s = db_->CompactRange(comp_opts, &start, &end); + ASSERT_TRUE(s.IsInvalidArgument()); + s = db_->CompactRange(comp_opts, &start, nullptr); + ASSERT_TRUE(s.IsInvalidArgument()); + s = db_->CompactRange(comp_opts, nullptr, &end); + ASSERT_TRUE(s.IsInvalidArgument()); + + // Test invalid compaction with the decreasing ts_low + ts_low_str = Timestamp(current_ts_low - 1, 0); + ts_low = ts_low_str; + comp_opts.full_history_ts_low = &ts_low; + s = db_->CompactRange(comp_opts, nullptr, nullptr); + ASSERT_TRUE(s.IsInvalidArgument()); + + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, UpdateFullHistoryTsLowWithPublicAPI) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + std::string ts_low_str = Timestamp(9, 0); + ASSERT_OK( + db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(), ts_low_str)); + std::string result_ts_low; + ASSERT_OK(db_->GetFullHistoryTsLow(nullptr, &result_ts_low)); + ASSERT_TRUE(test_cmp.CompareTimestamp(ts_low_str, result_ts_low) == 0); + // test increase full_history_low backward + std::string ts_low_str_back = Timestamp(8, 0); + auto s = db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(), + ts_low_str_back); + ASSERT_EQ(s, Status::InvalidArgument()); + // test IncreaseFullHistoryTsLow with a timestamp whose length is longger + // than the cf's timestamp size + std::string ts_low_str_long(Timestamp(0, 0).size() + 1, 'a'); + s = db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(), + ts_low_str_long); + ASSERT_EQ(s, Status::InvalidArgument()); + // test IncreaseFullHistoryTsLow with a timestamp which is null + std::string ts_low_str_null = ""; + s = db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(), + ts_low_str_null); + ASSERT_EQ(s, Status::InvalidArgument()); + // test IncreaseFullHistoryTsLow for a column family that does not enable + // timestamp + options.comparator = BytewiseComparator(); + DestroyAndReopen(options); + ts_low_str = Timestamp(10, 0); + s = db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(), ts_low_str); + ASSERT_EQ(s, Status::InvalidArgument()); + // test GetFullHistoryTsLow for a column family that does not enable + // timestamp + std::string current_ts_low; + s = db_->GetFullHistoryTsLow(db_->DefaultColumnFamily(), ¤t_ts_low); + ASSERT_EQ(s, Status::InvalidArgument()); + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, GetApproximateSizes) { + Options options = CurrentOptions(); + options.write_buffer_size = 100000000; // Large write buffer + options.compression = kNoCompression; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + auto default_cf = db_->DefaultColumnFamily(); + + WriteOptions write_opts; + std::string ts = Timestamp(1, 0); + + const int N = 128; + Random rnd(301); + for (int i = 0; i < N; i++) { + ASSERT_OK(db_->Put(write_opts, Key(i), ts, rnd.RandomString(1024))); + } + + uint64_t size; + std::string start = Key(50); + std::string end = Key(60); + Range r(start, end); + SizeApproximationOptions size_approx_options; + size_approx_options.include_memtables = true; + size_approx_options.include_files = true; + ASSERT_OK( + db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, &size)); + ASSERT_GT(size, 6000); + ASSERT_LT(size, 204800); + + // test multiple ranges + std::vector ranges; + std::string start_tmp = Key(10); + std::string end_tmp = Key(20); + ranges.emplace_back(Range(start_tmp, end_tmp)); + ranges.emplace_back(Range(start, end)); + uint64_t range_sizes[2]; + ASSERT_OK(db_->GetApproximateSizes(size_approx_options, default_cf, + ranges.data(), 2, range_sizes)); + + ASSERT_EQ(range_sizes[1], size); + + // Zero if not including mem table + ASSERT_OK(db_->GetApproximateSizes(&r, 1, &size)); + ASSERT_EQ(size, 0); + + start = Key(500); + end = Key(600); + r = Range(start, end); + ASSERT_OK( + db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, &size)); + ASSERT_EQ(size, 0); + + // Test range boundaries + ASSERT_OK(db_->Put(write_opts, Key(1000), ts, rnd.RandomString(1024))); + // Should include start key + start = Key(1000); + end = Key(1100); + r = Range(start, end); + ASSERT_OK( + db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, &size)); + ASSERT_GT(size, 0); + + // Should exclude end key + start = Key(900); + end = Key(1000); + r = Range(start, end); + ASSERT_OK( + db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, &size)); + ASSERT_EQ(size, 0); + + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, SimpleIterate) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::vector start_keys = {1, 0}; + const std::vector write_timestamps = {Timestamp(1, 0), + Timestamp(3, 0)}; + const std::vector read_timestamps = {Timestamp(2, 0), + Timestamp(4, 0)}; + for (size_t i = 0; i < write_timestamps.size(); ++i) { + WriteOptions write_opts; + for (uint64_t key = start_keys[i]; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamps[i], + "value" + std::to_string(i)); + ASSERT_OK(s); + } + } + for (size_t i = 0; i < read_timestamps.size(); ++i) { + ReadOptions read_opts; + Slice read_ts = read_timestamps[i]; + read_opts.timestamp = &read_ts; + std::unique_ptr it(db_->NewIterator(read_opts)); + int count = 0; + uint64_t key = 0; + // Forward iterate. + for (it->Seek(Key1(0)), key = start_keys[i]; it->Valid(); + it->Next(), ++count, ++key) { + CheckIterUserEntry(it.get(), Key1(key), kTypeValue, + "value" + std::to_string(i), write_timestamps[i]); + } + size_t expected_count = kMaxKey - start_keys[i] + 1; + ASSERT_EQ(expected_count, count); + + // Backward iterate. + count = 0; + for (it->SeekForPrev(Key1(kMaxKey)), key = kMaxKey; it->Valid(); + it->Prev(), ++count, --key) { + CheckIterUserEntry(it.get(), Key1(key), kTypeValue, + "value" + std::to_string(i), write_timestamps[i]); + } + ASSERT_EQ(static_cast(kMaxKey) - start_keys[i] + 1, count); + + // SeekToFirst()/SeekToLast() with lower/upper bounds. + // Then iter with lower and upper bounds. + uint64_t l = 0; + uint64_t r = kMaxKey + 1; + while (l < r) { + std::string lb_str = Key1(l); + Slice lb = lb_str; + std::string ub_str = Key1(r); + Slice ub = ub_str; + read_opts.iterate_lower_bound = &lb; + read_opts.iterate_upper_bound = &ub; + it.reset(db_->NewIterator(read_opts)); + for (it->SeekToFirst(), key = std::max(l, start_keys[i]), count = 0; + it->Valid(); it->Next(), ++key, ++count) { + CheckIterUserEntry(it.get(), Key1(key), kTypeValue, + "value" + std::to_string(i), write_timestamps[i]); + } + ASSERT_EQ(r - std::max(l, start_keys[i]), count); + + for (it->SeekToLast(), key = std::min(r, kMaxKey + 1), count = 0; + it->Valid(); it->Prev(), --key, ++count) { + CheckIterUserEntry(it.get(), Key1(key - 1), kTypeValue, + "value" + std::to_string(i), write_timestamps[i]); + } + l += (kMaxKey / 100); + r -= (kMaxKey / 100); + } + } + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, TrimHistoryTest) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + auto check_value_by_ts = [](DB* db, Slice key, std::string readTs, + Status status, std::string checkValue, + std::string expected_ts) { + ReadOptions ropts; + Slice ts = readTs; + ropts.timestamp = &ts; + std::string value; + std::string key_ts; + Status s = db->Get(ropts, key, &value, &key_ts); + ASSERT_TRUE(s == status); + if (s.ok()) { + ASSERT_EQ(checkValue, value); + } + if (s.ok() || s.IsNotFound()) { + ASSERT_EQ(expected_ts, key_ts); + } + }; + // Construct data of different versions with different ts + ASSERT_OK(db_->Put(WriteOptions(), "k1", Timestamp(2, 0), "v1")); + ASSERT_OK(db_->Put(WriteOptions(), "k1", Timestamp(4, 0), "v2")); + ASSERT_OK(db_->Delete(WriteOptions(), "k1", Timestamp(5, 0))); + ASSERT_OK(db_->Put(WriteOptions(), "k1", Timestamp(6, 0), "v3")); + check_value_by_ts(db_, "k1", Timestamp(7, 0), Status::OK(), "v3", + Timestamp(6, 0)); + ASSERT_OK(Flush()); + Close(); + + ColumnFamilyOptions cf_options(options); + std::vector column_families; + column_families.push_back( + ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options)); + DBOptions db_options(options); + + // Trim data whose version > Timestamp(5, 0), read(k1, ts(7)) <- NOT_FOUND. + ASSERT_OK(DB::OpenAndTrimHistory(db_options, dbname_, column_families, + &handles_, &db_, Timestamp(5, 0))); + check_value_by_ts(db_, "k1", Timestamp(7, 0), Status::NotFound(), "", + Timestamp(5, 0)); + Close(); + + // Trim data whose timestamp > Timestamp(4, 0), read(k1, ts(7)) <- v2 + ASSERT_OK(DB::OpenAndTrimHistory(db_options, dbname_, column_families, + &handles_, &db_, Timestamp(4, 0))); + check_value_by_ts(db_, "k1", Timestamp(7, 0), Status::OK(), "v2", + Timestamp(4, 0)); + Close(); + + Reopen(options); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "k1", + "k3", Timestamp(7, 0))); + check_value_by_ts(db_, "k1", Timestamp(8, 0), Status::NotFound(), "", + Timestamp(7, 0)); + Close(); + // Trim data whose timestamp > Timestamp(6, 0), read(k1, ts(8)) <- v2 + ASSERT_OK(DB::OpenAndTrimHistory(db_options, dbname_, column_families, + &handles_, &db_, Timestamp(6, 0))); + check_value_by_ts(db_, "k1", Timestamp(8, 0), Status::OK(), "v2", + Timestamp(4, 0)); + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, OpenAndTrimHistoryInvalidOptionTest) { + Destroy(last_options_); + + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + + ColumnFamilyOptions cf_options(options); + std::vector column_families; + column_families.push_back( + ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options)); + DBOptions db_options(options); + + // OpenAndTrimHistory should not work with avoid_flush_during_recovery + db_options.avoid_flush_during_recovery = true; + ASSERT_TRUE(DB::OpenAndTrimHistory(db_options, dbname_, column_families, + &handles_, &db_, Timestamp(0, 0)) + .IsInvalidArgument()); +} + +TEST_F(DBBasicTestWithTimestamp, GetTimestampTableProperties) { + Options options = CurrentOptions(); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + // Create 2 tables + for (int table = 0; table < 2; ++table) { + for (int i = 0; i < 10; i++) { + std::string ts = Timestamp(i, 0); + ASSERT_OK(db_->Put(WriteOptions(), "key", ts, Key(i))); + } + ASSERT_OK(Flush()); + } + + TablePropertiesCollection props; + ASSERT_OK(db_->GetPropertiesOfAllTables(&props)); + ASSERT_EQ(2U, props.size()); + for (const auto& item : props) { + auto& user_collected = item.second->user_collected_properties; + ASSERT_TRUE(user_collected.find("rocksdb.timestamp_min") != + user_collected.end()); + ASSERT_TRUE(user_collected.find("rocksdb.timestamp_max") != + user_collected.end()); + ASSERT_EQ(user_collected.at("rocksdb.timestamp_min"), Timestamp(0, 0)); + ASSERT_EQ(user_collected.at("rocksdb.timestamp_max"), Timestamp(9, 0)); + } + Close(); +} + +class DBBasicTestWithTimestampTableOptions + : public DBBasicTestWithTimestampBase, + public testing::WithParamInterface { + public: + explicit DBBasicTestWithTimestampTableOptions() + : DBBasicTestWithTimestampBase( + "db_basic_test_with_timestamp_table_options") {} +}; + +INSTANTIATE_TEST_CASE_P( + Timestamp, DBBasicTestWithTimestampTableOptions, + testing::Values( + BlockBasedTableOptions::IndexType::kBinarySearch, + BlockBasedTableOptions::IndexType::kHashSearch, + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch, + BlockBasedTableOptions::IndexType::kBinarySearchWithFirstKey)); + +TEST_P(DBBasicTestWithTimestampTableOptions, GetAndMultiGet) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.prefix_extractor.reset(NewFixedPrefixTransform(3)); + options.compression = kNoCompression; + BlockBasedTableOptions bbto; + bbto.index_type = GetParam(); + bbto.block_size = 100; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator cmp(kTimestampSize); + options.comparator = &cmp; + DestroyAndReopen(options); + constexpr uint64_t kNumKeys = 1024; + for (uint64_t k = 0; k < kNumKeys; ++k) { + WriteOptions write_opts; + ASSERT_OK(db_->Put(write_opts, Key1(k), Timestamp(1, 0), + "value" + std::to_string(k))); + } + ASSERT_OK(Flush()); + { + ReadOptions read_opts; + read_opts.total_order_seek = true; + std::string ts_str = Timestamp(2, 0); + Slice ts = ts_str; + read_opts.timestamp = &ts; + std::unique_ptr it(db_->NewIterator(read_opts)); + // verify Get() + for (it->SeekToFirst(); it->Valid(); it->Next()) { + std::string value_from_get; + std::string key_str(it->key().data(), it->key().size()); + std::string timestamp; + ASSERT_OK(db_->Get(read_opts, key_str, &value_from_get, ×tamp)); + ASSERT_EQ(it->value(), value_from_get); + ASSERT_EQ(Timestamp(1, 0), timestamp); + } + + // verify MultiGet() + constexpr uint64_t step = 2; + static_assert(0 == (kNumKeys % step), + "kNumKeys must be a multiple of step"); + for (uint64_t k = 0; k < kNumKeys; k += 2) { + std::vector key_strs; + std::vector keys; + for (size_t i = 0; i < step; ++i) { + key_strs.push_back(Key1(k + i)); + } + for (size_t i = 0; i < step; ++i) { + keys.emplace_back(key_strs[i]); + } + std::vector values; + std::vector timestamps; + std::vector statuses = + db_->MultiGet(read_opts, keys, &values, ×tamps); + ASSERT_EQ(step, statuses.size()); + ASSERT_EQ(step, values.size()); + ASSERT_EQ(step, timestamps.size()); + for (uint64_t i = 0; i < step; ++i) { + ASSERT_OK(statuses[i]); + ASSERT_EQ("value" + std::to_string(k + i), values[i]); + ASSERT_EQ(Timestamp(1, 0), timestamps[i]); + } + } + } + Close(); +} + +TEST_P(DBBasicTestWithTimestampTableOptions, SeekWithPrefixLessThanKey) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.prefix_extractor.reset(NewFixedPrefixTransform(3)); + options.memtable_whole_key_filtering = true; + options.memtable_prefix_bloom_size_ratio = 0.1; + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(NewBloomFilterPolicy(10, false)); + bbto.cache_index_and_filter_blocks = true; + bbto.whole_key_filtering = true; + bbto.index_type = GetParam(); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + + WriteOptions write_opts; + std::string ts = Timestamp(1, 0); + + ASSERT_OK(db_->Put(write_opts, "foo1", ts, "bar")); + ASSERT_OK(Flush()); + + ASSERT_OK(db_->Put(write_opts, "foo2", ts, "bar")); + ASSERT_OK(Flush()); + + // Move sst file to next level + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + ASSERT_OK(db_->Put(write_opts, "foo3", ts, "bar")); + ASSERT_OK(Flush()); + + ReadOptions read_opts; + Slice read_ts = ts; + read_opts.timestamp = &read_ts; + { + std::unique_ptr iter(db_->NewIterator(read_opts)); + iter->Seek("foo"); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + + iter->Seek("bbb"); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + } + + Close(); +} + +TEST_P(DBBasicTestWithTimestampTableOptions, SeekWithCappedPrefix) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + // All of the keys or this test must be longer than 3 characters + constexpr int kMinKeyLen = 3; + options.prefix_extractor.reset(NewCappedPrefixTransform(kMinKeyLen)); + options.memtable_whole_key_filtering = true; + options.memtable_prefix_bloom_size_ratio = 0.1; + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(NewBloomFilterPolicy(10, false)); + bbto.cache_index_and_filter_blocks = true; + bbto.whole_key_filtering = true; + bbto.index_type = GetParam(); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + + WriteOptions write_opts; + std::string ts = Timestamp(1, 0); + + ASSERT_OK(db_->Put(write_opts, "foo1", ts, "bar")); + ASSERT_OK(Flush()); + + ASSERT_OK(db_->Put(write_opts, "foo2", ts, "bar")); + ASSERT_OK(Flush()); + + // Move sst file to next level + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + ASSERT_OK(db_->Put(write_opts, "foo3", ts, "bar")); + ASSERT_OK(Flush()); + + ReadOptions read_opts; + ts = Timestamp(2, 0); + Slice read_ts = ts; + read_opts.timestamp = &read_ts; + { + std::unique_ptr iter(db_->NewIterator(read_opts)); + // Make sure the prefix extractor doesn't include timestamp, otherwise it + // may return invalid result. + iter->Seek("foo"); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + } + + Close(); +} + +TEST_P(DBBasicTestWithTimestampTableOptions, SeekWithBound) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.prefix_extractor.reset(NewFixedPrefixTransform(2)); + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(NewBloomFilterPolicy(10, false)); + bbto.cache_index_and_filter_blocks = true; + bbto.whole_key_filtering = true; + bbto.index_type = GetParam(); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + + WriteOptions write_opts; + std::string ts = Timestamp(1, 0); + + ASSERT_OK(db_->Put(write_opts, "foo1", ts, "bar1")); + ASSERT_OK(Flush()); + + ASSERT_OK(db_->Put(write_opts, "foo2", ts, "bar2")); + ASSERT_OK(Flush()); + + // Move sst file to next level + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + for (int i = 3; i < 9; ++i) { + ASSERT_OK(db_->Put(write_opts, "foo" + std::to_string(i), ts, + "bar" + std::to_string(i))); + } + ASSERT_OK(Flush()); + + ReadOptions read_opts; + ts = Timestamp(2, 0); + Slice read_ts = ts; + read_opts.timestamp = &read_ts; + std::string up_bound = "foo5"; // exclusive + Slice up_bound_slice = up_bound; + std::string lo_bound = "foo2"; // inclusive + Slice lo_bound_slice = lo_bound; + read_opts.iterate_upper_bound = &up_bound_slice; + read_opts.iterate_lower_bound = &lo_bound_slice; + read_opts.auto_prefix_mode = true; + { + std::unique_ptr iter(db_->NewIterator(read_opts)); + // Make sure the prefix extractor doesn't include timestamp, otherwise it + // may return invalid result. + iter->Seek("foo"); + CheckIterUserEntry(iter.get(), lo_bound, kTypeValue, "bar2", + Timestamp(1, 0)); + iter->SeekToFirst(); + CheckIterUserEntry(iter.get(), lo_bound, kTypeValue, "bar2", + Timestamp(1, 0)); + iter->SeekForPrev("g"); + CheckIterUserEntry(iter.get(), "foo4", kTypeValue, "bar4", Timestamp(1, 0)); + iter->SeekToLast(); + CheckIterUserEntry(iter.get(), "foo4", kTypeValue, "bar4", Timestamp(1, 0)); + } + + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, ChangeIterationDirection) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.env = env_; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.prefix_extractor.reset(NewFixedPrefixTransform(1)); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + DestroyAndReopen(options); + const std::vector timestamps = {Timestamp(1, 1), Timestamp(0, 2), + Timestamp(4, 3)}; + const std::vector> kvs = { + std::make_tuple("aa", "value1"), std::make_tuple("ab", "value2")}; + for (const auto& ts : timestamps) { + WriteBatch wb(0, 0, 0, kTimestampSize); + for (const auto& kv : kvs) { + const std::string& key = std::get<0>(kv); + const std::string& value = std::get<1>(kv); + ASSERT_OK(wb.Put(key, value)); + } + + ASSERT_OK(wb.UpdateTimestamps( + ts, [kTimestampSize](uint32_t) { return kTimestampSize; })); + ASSERT_OK(db_->Write(WriteOptions(), &wb)); + } + std::string read_ts_str = Timestamp(5, 3); + Slice read_ts = read_ts_str; + ReadOptions read_opts; + read_opts.timestamp = &read_ts; + std::unique_ptr it(db_->NewIterator(read_opts)); + + it->SeekToFirst(); + ASSERT_TRUE(it->Valid()); + it->Prev(); + ASSERT_FALSE(it->Valid()); + + it->SeekToLast(); + ASSERT_TRUE(it->Valid()); + uint64_t prev_reseek_count = + options.statistics->getTickerCount(NUMBER_OF_RESEEKS_IN_ITERATION); + ASSERT_EQ(0, prev_reseek_count); + it->Next(); + ASSERT_FALSE(it->Valid()); + ASSERT_EQ(1 + prev_reseek_count, + options.statistics->getTickerCount(NUMBER_OF_RESEEKS_IN_ITERATION)); + + it->Seek(std::get<0>(kvs[0])); + CheckIterUserEntry(it.get(), std::get<0>(kvs[0]), kTypeValue, + std::get<1>(kvs[0]), Timestamp(4, 3)); + it->Next(); + CheckIterUserEntry(it.get(), std::get<0>(kvs[1]), kTypeValue, + std::get<1>(kvs[1]), Timestamp(4, 3)); + it->Prev(); + CheckIterUserEntry(it.get(), std::get<0>(kvs[0]), kTypeValue, + std::get<1>(kvs[0]), Timestamp(4, 3)); + + prev_reseek_count = + options.statistics->getTickerCount(NUMBER_OF_RESEEKS_IN_ITERATION); + ASSERT_EQ(1, prev_reseek_count); + it->Next(); + CheckIterUserEntry(it.get(), std::get<0>(kvs[1]), kTypeValue, + std::get<1>(kvs[1]), Timestamp(4, 3)); + ASSERT_EQ(1 + prev_reseek_count, + options.statistics->getTickerCount(NUMBER_OF_RESEEKS_IN_ITERATION)); + + it->SeekForPrev(std::get<0>(kvs[1])); + CheckIterUserEntry(it.get(), std::get<0>(kvs[1]), kTypeValue, + std::get<1>(kvs[1]), Timestamp(4, 3)); + it->Prev(); + CheckIterUserEntry(it.get(), std::get<0>(kvs[0]), kTypeValue, + std::get<1>(kvs[0]), Timestamp(4, 3)); + + prev_reseek_count = + options.statistics->getTickerCount(NUMBER_OF_RESEEKS_IN_ITERATION); + it->Next(); + CheckIterUserEntry(it.get(), std::get<0>(kvs[1]), kTypeValue, + std::get<1>(kvs[1]), Timestamp(4, 3)); + ASSERT_EQ(1 + prev_reseek_count, + options.statistics->getTickerCount(NUMBER_OF_RESEEKS_IN_ITERATION)); + + it.reset(); + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, SimpleForwardIterateLowerTsBound) { + constexpr int kNumKeysPerFile = 128; + constexpr uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::vector write_timestamps = {Timestamp(1, 0), + Timestamp(3, 0)}; + const std::vector read_timestamps = {Timestamp(2, 0), + Timestamp(4, 0)}; + const std::vector read_timestamps_lb = {Timestamp(1, 0), + Timestamp(1, 0)}; + for (size_t i = 0; i < write_timestamps.size(); ++i) { + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamps[i], + "value" + std::to_string(i)); + ASSERT_OK(s); + } + } + for (size_t i = 0; i < read_timestamps.size(); ++i) { + ReadOptions read_opts; + Slice read_ts = read_timestamps[i]; + Slice read_ts_lb = read_timestamps_lb[i]; + read_opts.timestamp = &read_ts; + read_opts.iter_start_ts = &read_ts_lb; + std::unique_ptr it(db_->NewIterator(read_opts)); + int count = 0; + uint64_t key = 0; + for (it->Seek(Key1(0)), key = 0; it->Valid(); it->Next(), ++count, ++key) { + CheckIterEntry(it.get(), Key1(key), kTypeValue, + "value" + std::to_string(i), write_timestamps[i]); + if (i > 0) { + it->Next(); + CheckIterEntry(it.get(), Key1(key), kTypeValue, + "value" + std::to_string(i - 1), + write_timestamps[i - 1]); + } + } + size_t expected_count = kMaxKey + 1; + ASSERT_EQ(expected_count, count); + } + // Delete all keys@ts=5 and check iteration result with start ts set + { + std::string write_timestamp = Timestamp(5, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key < kMaxKey + 1; ++key) { + Status s = db_->Delete(write_opts, Key1(key), write_timestamp); + ASSERT_OK(s); + } + + std::string read_timestamp = Timestamp(6, 0); + ReadOptions read_opts; + Slice read_ts = read_timestamp; + read_opts.timestamp = &read_ts; + std::string read_timestamp_lb = Timestamp(2, 0); + Slice read_ts_lb = read_timestamp_lb; + read_opts.iter_start_ts = &read_ts_lb; + std::unique_ptr it(db_->NewIterator(read_opts)); + int count = 0; + uint64_t key = 0; + for (it->Seek(Key1(0)), key = 0; it->Valid(); it->Next(), ++count, ++key) { + CheckIterEntry(it.get(), Key1(key), kTypeDeletionWithTimestamp, Slice(), + write_timestamp); + // Skip key@ts=3 and land on tombstone key@ts=5 + it->Next(); + } + ASSERT_EQ(kMaxKey + 1, count); + } + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, BackwardIterateLowerTsBound) { + constexpr int kNumKeysPerFile = 128; + constexpr uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::vector write_timestamps = {Timestamp(1, 0), + Timestamp(3, 0)}; + const std::vector read_timestamps = {Timestamp(2, 0), + Timestamp(4, 0)}; + const std::vector read_timestamps_lb = {Timestamp(1, 0), + Timestamp(1, 0)}; + for (size_t i = 0; i < write_timestamps.size(); ++i) { + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamps[i], + "value" + std::to_string(i)); + ASSERT_OK(s); + } + } + for (size_t i = 0; i < read_timestamps.size(); ++i) { + ReadOptions read_opts; + Slice read_ts = read_timestamps[i]; + Slice read_ts_lb = read_timestamps_lb[i]; + read_opts.timestamp = &read_ts; + read_opts.iter_start_ts = &read_ts_lb; + std::unique_ptr it(db_->NewIterator(read_opts)); + int count = 0; + uint64_t key = 0; + for (it->SeekForPrev(Key1(kMaxKey)), key = kMaxKey; it->Valid(); + it->Prev(), ++count, --key) { + CheckIterEntry(it.get(), Key1(key), kTypeValue, "value0", + write_timestamps[0]); + if (i > 0) { + it->Prev(); + CheckIterEntry(it.get(), Key1(key), kTypeValue, "value1", + write_timestamps[1]); + } + } + size_t expected_count = kMaxKey + 1; + ASSERT_EQ(expected_count, count); + } + // Delete all keys@ts=5 and check iteration result with start ts set + { + std::string write_timestamp = Timestamp(5, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key < kMaxKey + 1; ++key) { + Status s = db_->Delete(write_opts, Key1(key), write_timestamp); + ASSERT_OK(s); + } + + std::string read_timestamp = Timestamp(6, 0); + ReadOptions read_opts; + Slice read_ts = read_timestamp; + read_opts.timestamp = &read_ts; + std::string read_timestamp_lb = Timestamp(2, 0); + Slice read_ts_lb = read_timestamp_lb; + read_opts.iter_start_ts = &read_ts_lb; + std::unique_ptr it(db_->NewIterator(read_opts)); + int count = 0; + uint64_t key = kMaxKey; + for (it->SeekForPrev(Key1(key)), key = kMaxKey; it->Valid(); + it->Prev(), ++count, --key) { + CheckIterEntry(it.get(), Key1(key), kTypeValue, "value1", + Timestamp(3, 0)); + it->Prev(); + CheckIterEntry(it.get(), Key1(key), kTypeDeletionWithTimestamp, Slice(), + write_timestamp); + } + ASSERT_EQ(kMaxKey + 1, count); + } + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, SimpleBackwardIterateLowerTsBound) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + + std::string ts_ub_buf = Timestamp(5, 0); + Slice ts_ub = ts_ub_buf; + std::string ts_lb_buf = Timestamp(1, 0); + Slice ts_lb = ts_lb_buf; + + { + ReadOptions read_opts; + read_opts.timestamp = &ts_ub; + read_opts.iter_start_ts = &ts_lb; + std::unique_ptr it(db_->NewIterator(read_opts)); + it->SeekToLast(); + ASSERT_FALSE(it->Valid()); + ASSERT_OK(it->status()); + + it->SeekForPrev("foo"); + ASSERT_FALSE(it->Valid()); + ASSERT_OK(it->status()); + } + + // Test iterate_upper_bound + ASSERT_OK(db_->Put(WriteOptions(), "a", Timestamp(0, 0), "v0")); + ASSERT_OK(db_->SingleDelete(WriteOptions(), "a", Timestamp(1, 0))); + + for (int i = 0; i < 5; ++i) { + ASSERT_OK(db_->Put(WriteOptions(), "b", Timestamp(i, 0), + "v" + std::to_string(i))); + } + + { + ReadOptions read_opts; + read_opts.timestamp = &ts_ub; + read_opts.iter_start_ts = &ts_lb; + std::string key_ub_str = "b"; // exclusive + Slice key_ub = key_ub_str; + read_opts.iterate_upper_bound = &key_ub; + std::unique_ptr it(db_->NewIterator(read_opts)); + it->SeekToLast(); + CheckIterEntry(it.get(), "a", kTypeSingleDeletion, Slice(), + Timestamp(1, 0)); + + key_ub_str = "a"; // exclusive + key_ub = key_ub_str; + read_opts.iterate_upper_bound = &key_ub; + it.reset(db_->NewIterator(read_opts)); + it->SeekToLast(); + ASSERT_FALSE(it->Valid()); + ASSERT_OK(it->status()); + } + + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, BackwardIterateLowerTsBound_Reseek) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.max_sequential_skip_in_iterations = 2; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + + for (int i = 0; i < 10; ++i) { + ASSERT_OK(db_->Put(WriteOptions(), "a", Timestamp(i, 0), + "v" + std::to_string(i))); + } + + for (int i = 0; i < 10; ++i) { + ASSERT_OK(db_->Put(WriteOptions(), "b", Timestamp(i, 0), + "v" + std::to_string(i))); + } + + { + std::string ts_ub_buf = Timestamp(6, 0); + Slice ts_ub = ts_ub_buf; + std::string ts_lb_buf = Timestamp(4, 0); + Slice ts_lb = ts_lb_buf; + + ReadOptions read_opts; + read_opts.timestamp = &ts_ub; + read_opts.iter_start_ts = &ts_lb; + std::unique_ptr it(db_->NewIterator(read_opts)); + it->SeekToLast(); + for (int i = 0; i < 3 && it->Valid(); it->Prev(), ++i) { + CheckIterEntry(it.get(), "b", kTypeValue, "v" + std::to_string(4 + i), + Timestamp(4 + i, 0)); + } + for (int i = 0; i < 3 && it->Valid(); it->Prev(), ++i) { + CheckIterEntry(it.get(), "a", kTypeValue, "v" + std::to_string(4 + i), + Timestamp(4 + i, 0)); + } + } + + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, ReseekToTargetTimestamp) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + constexpr size_t kNumKeys = 16; + options.max_sequential_skip_in_iterations = kNumKeys / 2; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + // Insert kNumKeys + WriteOptions write_opts; + Status s; + for (size_t i = 0; i != kNumKeys; ++i) { + std::string ts = Timestamp(static_cast(i + 1), 0); + s = db_->Put(write_opts, "foo", ts, "value" + std::to_string(i)); + ASSERT_OK(s); + } + { + ReadOptions read_opts; + std::string ts_str = Timestamp(1, 0); + Slice ts = ts_str; + read_opts.timestamp = &ts; + std::unique_ptr iter(db_->NewIterator(read_opts)); + iter->SeekToFirst(); + CheckIterUserEntry(iter.get(), "foo", kTypeValue, "value0", ts_str); + ASSERT_EQ( + 1, options.statistics->getTickerCount(NUMBER_OF_RESEEKS_IN_ITERATION)); + + ts_str = Timestamp(kNumKeys, 0); + ts = ts_str; + read_opts.timestamp = &ts; + iter.reset(db_->NewIterator(read_opts)); + iter->SeekToLast(); + CheckIterUserEntry(iter.get(), "foo", kTypeValue, + "value" + std::to_string(kNumKeys - 1), ts_str); + ASSERT_EQ( + 2, options.statistics->getTickerCount(NUMBER_OF_RESEEKS_IN_ITERATION)); + } + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, ReseekToNextUserKey) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + constexpr size_t kNumKeys = 16; + options.max_sequential_skip_in_iterations = kNumKeys / 2; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + // Write kNumKeys + 1 keys + WriteOptions write_opts; + Status s; + for (size_t i = 0; i != kNumKeys; ++i) { + std::string ts = Timestamp(static_cast(i + 1), 0); + s = db_->Put(write_opts, "a", ts, "value" + std::to_string(i)); + ASSERT_OK(s); + } + { + std::string ts_str = Timestamp(static_cast(kNumKeys + 1), 0); + WriteBatch batch(0, 0, 0, kTimestampSize); + { ASSERT_OK(batch.Put("a", "new_value")); } + { ASSERT_OK(batch.Put("b", "new_value")); } + s = batch.UpdateTimestamps( + ts_str, [kTimestampSize](uint32_t) { return kTimestampSize; }); + ASSERT_OK(s); + s = db_->Write(write_opts, &batch); + ASSERT_OK(s); + } + { + ReadOptions read_opts; + std::string ts_str = Timestamp(static_cast(kNumKeys + 1), 0); + Slice ts = ts_str; + read_opts.timestamp = &ts; + std::unique_ptr iter(db_->NewIterator(read_opts)); + iter->Seek("a"); + iter->Next(); + CheckIterUserEntry(iter.get(), "b", kTypeValue, "new_value", ts_str); + ASSERT_EQ( + 1, options.statistics->getTickerCount(NUMBER_OF_RESEEKS_IN_ITERATION)); + } + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, ReseekToUserKeyBeforeSavedKey) { + Options options = GetDefaultOptions(); + options.env = env_; + options.create_if_missing = true; + constexpr size_t kNumKeys = 16; + options.max_sequential_skip_in_iterations = kNumKeys / 2; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + for (size_t i = 0; i < kNumKeys; ++i) { + std::string ts = Timestamp(static_cast(i + 1), 0); + WriteOptions write_opts; + Status s = db_->Put(write_opts, "b", ts, "value" + std::to_string(i)); + ASSERT_OK(s); + } + { + std::string ts = Timestamp(1, 0); + WriteOptions write_opts; + ASSERT_OK(db_->Put(write_opts, "a", ts, "value")); + } + { + ReadOptions read_opts; + std::string ts_str = Timestamp(1, 0); + Slice ts = ts_str; + read_opts.timestamp = &ts; + std::unique_ptr iter(db_->NewIterator(read_opts)); + iter->SeekToLast(); + iter->Prev(); + CheckIterUserEntry(iter.get(), "a", kTypeValue, "value", ts_str); + ASSERT_EQ( + 1, options.statistics->getTickerCount(NUMBER_OF_RESEEKS_IN_ITERATION)); + } + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, MultiGetWithFastLocalBloom) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(NewBloomFilterPolicy(10, false)); + bbto.cache_index_and_filter_blocks = true; + bbto.whole_key_filtering = true; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + + // Write any value + WriteOptions write_opts; + std::string ts = Timestamp(1, 0); + + ASSERT_OK(db_->Put(write_opts, "foo", ts, "bar")); + + ASSERT_OK(Flush()); + + // Read with MultiGet + ReadOptions read_opts; + Slice read_ts = ts; + read_opts.timestamp = &read_ts; + size_t batch_size = 1; + std::vector keys(batch_size); + std::vector values(batch_size); + std::vector statuses(batch_size); + std::vector timestamps(batch_size); + keys[0] = "foo"; + ColumnFamilyHandle* cfh = db_->DefaultColumnFamily(); + db_->MultiGet(read_opts, cfh, batch_size, keys.data(), values.data(), + timestamps.data(), statuses.data(), true); + + ASSERT_OK(statuses[0]); + ASSERT_EQ(Timestamp(1, 0), timestamps[0]); + for (auto& elem : values) { + elem.Reset(); + } + + ASSERT_OK(db_->SingleDelete(WriteOptions(), "foo", Timestamp(2, 0))); + ts = Timestamp(3, 0); + read_ts = ts; + read_opts.timestamp = &read_ts; + db_->MultiGet(read_opts, cfh, batch_size, keys.data(), values.data(), + timestamps.data(), statuses.data(), true); + ASSERT_TRUE(statuses[0].IsNotFound()); + ASSERT_EQ(Timestamp(2, 0), timestamps[0]); + + Close(); +} + +TEST_P(DBBasicTestWithTimestampTableOptions, MultiGetWithPrefix) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.prefix_extractor.reset(NewCappedPrefixTransform(5)); + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(NewBloomFilterPolicy(10, false)); + bbto.cache_index_and_filter_blocks = true; + bbto.whole_key_filtering = false; + bbto.index_type = GetParam(); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + + // Write any value + WriteOptions write_opts; + std::string ts = Timestamp(1, 0); + + ASSERT_OK(db_->Put(write_opts, "foo", ts, "bar")); + + ASSERT_OK(Flush()); + + // Read with MultiGet + ReadOptions read_opts; + Slice read_ts = ts; + read_opts.timestamp = &read_ts; + size_t batch_size = 1; + std::vector keys(batch_size); + std::vector values(batch_size); + std::vector statuses(batch_size); + std::vector timestamps(batch_size); + keys[0] = "foo"; + ColumnFamilyHandle* cfh = db_->DefaultColumnFamily(); + db_->MultiGet(read_opts, cfh, batch_size, keys.data(), values.data(), + timestamps.data(), statuses.data(), true); + + ASSERT_OK(statuses[0]); + ASSERT_EQ(Timestamp(1, 0), timestamps[0]); + for (auto& elem : values) { + elem.Reset(); + } + + ASSERT_OK(db_->SingleDelete(WriteOptions(), "foo", Timestamp(2, 0))); + // TODO re-enable after fixing a bug of kHashSearch + if (GetParam() != BlockBasedTableOptions::IndexType::kHashSearch) { + ASSERT_OK(Flush()); + } + + ts = Timestamp(3, 0); + read_ts = ts; + db_->MultiGet(read_opts, cfh, batch_size, keys.data(), values.data(), + timestamps.data(), statuses.data(), true); + ASSERT_TRUE(statuses[0].IsNotFound()); + ASSERT_EQ(Timestamp(2, 0), timestamps[0]); + + Close(); +} + +TEST_P(DBBasicTestWithTimestampTableOptions, MultiGetWithMemBloomFilter) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.prefix_extractor.reset(NewCappedPrefixTransform(5)); + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(NewBloomFilterPolicy(10, false)); + bbto.cache_index_and_filter_blocks = true; + bbto.whole_key_filtering = false; + bbto.index_type = GetParam(); + options.memtable_prefix_bloom_size_ratio = 0.1; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + + // Write any value + WriteOptions write_opts; + std::string ts = Timestamp(1, 0); + + ASSERT_OK(db_->Put(write_opts, "foo", ts, "bar")); + + // Read with MultiGet + ts = Timestamp(2, 0); + Slice read_ts = ts; + ReadOptions read_opts; + read_opts.timestamp = &read_ts; + size_t batch_size = 1; + std::vector keys(batch_size); + std::vector values(batch_size); + std::vector statuses(batch_size); + keys[0] = "foo"; + ColumnFamilyHandle* cfh = db_->DefaultColumnFamily(); + db_->MultiGet(read_opts, cfh, batch_size, keys.data(), values.data(), + statuses.data()); + + ASSERT_OK(statuses[0]); + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, MultiGetRangeFiltering) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(NewBloomFilterPolicy(10, false)); + bbto.cache_index_and_filter_blocks = true; + bbto.whole_key_filtering = false; + options.memtable_prefix_bloom_size_ratio = 0.1; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + + // Write any value + WriteOptions write_opts; + std::string ts = Timestamp(1, 0); + + // random data + for (int i = 0; i < 3; i++) { + auto key = std::to_string(i * 10); + auto value = std::to_string(i * 10); + Slice key_slice = key; + Slice value_slice = value; + ASSERT_OK(db_->Put(write_opts, key_slice, ts, value_slice)); + ASSERT_OK(Flush()); + } + + // Make num_levels to 2 to do key range filtering of sst files + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + ASSERT_OK(db_->Put(write_opts, "foo", ts, "bar")); + + ASSERT_OK(Flush()); + + // Read with MultiGet + ts = Timestamp(2, 0); + Slice read_ts = ts; + ReadOptions read_opts; + read_opts.timestamp = &read_ts; + size_t batch_size = 1; + std::vector keys(batch_size); + std::vector values(batch_size); + std::vector statuses(batch_size); + keys[0] = "foo"; + ColumnFamilyHandle* cfh = db_->DefaultColumnFamily(); + db_->MultiGet(read_opts, cfh, batch_size, keys.data(), values.data(), + statuses.data()); + + ASSERT_OK(statuses[0]); + Close(); +} + +TEST_P(DBBasicTestWithTimestampTableOptions, MultiGetPrefixFilter) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.prefix_extractor.reset(NewCappedPrefixTransform(3)); + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(NewBloomFilterPolicy(10, false)); + bbto.cache_index_and_filter_blocks = true; + bbto.whole_key_filtering = false; + bbto.index_type = GetParam(); + options.memtable_prefix_bloom_size_ratio = 0.1; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + + WriteOptions write_opts; + std::string ts = Timestamp(1, 0); + + ASSERT_OK(db_->Put(write_opts, "foo", ts, "bar")); + + ASSERT_OK(Flush()); + // Read with MultiGet + ts = Timestamp(2, 0); + Slice read_ts = ts; + ReadOptions read_opts; + read_opts.timestamp = &read_ts; + size_t batch_size = 1; + std::vector keys(batch_size); + std::vector values(batch_size); + std::vector timestamps(batch_size); + keys[0] = "foo"; + ColumnFamilyHandle* cfh = db_->DefaultColumnFamily(); + std::vector cfhs(keys.size(), cfh); + std::vector statuses = + db_->MultiGet(read_opts, cfhs, keys, &values, ×tamps); + + ASSERT_OK(statuses[0]); + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, MaxKeysSkippedDuringNext) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + constexpr size_t max_skippable_internal_keys = 2; + const size_t kNumKeys = max_skippable_internal_keys + 2; + WriteOptions write_opts; + Status s; + { + std::string ts = Timestamp(1, 0); + ASSERT_OK(db_->Put(write_opts, "a", ts, "value")); + } + for (size_t i = 0; i < kNumKeys; ++i) { + std::string ts = Timestamp(static_cast(i + 1), 0); + s = db_->Put(write_opts, "b", ts, "value" + std::to_string(i)); + ASSERT_OK(s); + } + { + ReadOptions read_opts; + read_opts.max_skippable_internal_keys = max_skippable_internal_keys; + std::string ts_str = Timestamp(1, 0); + Slice ts = ts_str; + read_opts.timestamp = &ts; + std::unique_ptr iter(db_->NewIterator(read_opts)); + iter->SeekToFirst(); + iter->Next(); + ASSERT_TRUE(iter->status().IsIncomplete()); + } + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, MaxKeysSkippedDuringPrev) { + Options options = GetDefaultOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + constexpr size_t max_skippable_internal_keys = 2; + const size_t kNumKeys = max_skippable_internal_keys + 2; + WriteOptions write_opts; + Status s; + { + std::string ts = Timestamp(1, 0); + ASSERT_OK(db_->Put(write_opts, "b", ts, "value")); + } + for (size_t i = 0; i < kNumKeys; ++i) { + std::string ts = Timestamp(static_cast(i + 1), 0); + s = db_->Put(write_opts, "a", ts, "value" + std::to_string(i)); + ASSERT_OK(s); + } + { + ReadOptions read_opts; + read_opts.max_skippable_internal_keys = max_skippable_internal_keys; + std::string ts_str = Timestamp(1, 0); + Slice ts = ts_str; + read_opts.timestamp = &ts; + std::unique_ptr iter(db_->NewIterator(read_opts)); + iter->SeekToLast(); + iter->Prev(); + ASSERT_TRUE(iter->status().IsIncomplete()); + } + Close(); +} + +// Create two L0, and compact them to a new L1. In this test, L1 is L_bottom. +// Two L0s: +// f1 f2 +// ... +// Since f2.smallest < f1.largest < f2.largest +// f1 and f2 will be the inputs of a real compaction instead of trivial move. +TEST_F(DBBasicTestWithTimestamp, CompactDeletionWithTimestampMarkerToBottom) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.num_levels = 2; + options.level0_file_num_compaction_trigger = 2; + DestroyAndReopen(options); + WriteOptions write_opts; + std::string ts = Timestamp(1, 0); + ASSERT_OK(db_->Put(write_opts, "a", ts, "value0")); + ASSERT_OK(Flush()); + + ts = Timestamp(2, 0); + ASSERT_OK(db_->Put(write_opts, "b", ts, "value0")); + ts = Timestamp(3, 0); + ASSERT_OK(db_->Delete(write_opts, "a", ts)); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ReadOptions read_opts; + ts = Timestamp(1, 0); + Slice read_ts = ts; + read_opts.timestamp = &read_ts; + std::string value; + Status s = db_->Get(read_opts, "a", &value); + ASSERT_OK(s); + ASSERT_EQ("value0", value); + + ts = Timestamp(3, 0); + read_ts = ts; + read_opts.timestamp = &read_ts; + std::string key_ts; + s = db_->Get(read_opts, "a", &value, &key_ts); + ASSERT_TRUE(s.IsNotFound()); + ASSERT_EQ(Timestamp(3, 0), key_ts); + + // Time-travel to the past before deletion + ts = Timestamp(2, 0); + read_ts = ts; + read_opts.timestamp = &read_ts; + s = db_->Get(read_opts, "a", &value); + ASSERT_OK(s); + ASSERT_EQ("value0", value); + Close(); +} + +#if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) +class DBBasicTestWithTimestampFilterPrefixSettings + : public DBBasicTestWithTimestampBase, + public testing::WithParamInterface< + std::tuple, bool, bool, + std::shared_ptr, bool, double, + BlockBasedTableOptions::IndexType>> { + public: + DBBasicTestWithTimestampFilterPrefixSettings() + : DBBasicTestWithTimestampBase( + "db_basic_test_with_timestamp_filter_prefix") {} +}; + +TEST_P(DBBasicTestWithTimestampFilterPrefixSettings, GetAndMultiGet) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + BlockBasedTableOptions bbto; + bbto.filter_policy = std::get<0>(GetParam()); + bbto.whole_key_filtering = std::get<1>(GetParam()); + bbto.cache_index_and_filter_blocks = std::get<2>(GetParam()); + bbto.index_type = std::get<6>(GetParam()); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + options.prefix_extractor = std::get<3>(GetParam()); + options.memtable_whole_key_filtering = std::get<4>(GetParam()); + options.memtable_prefix_bloom_size_ratio = std::get<5>(GetParam()); + + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + const int kMaxKey = 1000; + + // Write any value + WriteOptions write_opts; + std::string ts = Timestamp(1, 0); + + int idx = 0; + for (; idx < kMaxKey / 4; idx++) { + ASSERT_OK(db_->Put(write_opts, Key1(idx), ts, "bar")); + ASSERT_OK(db_->Put(write_opts, KeyWithPrefix("foo", idx), ts, "bar")); + } + + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + for (; idx < kMaxKey / 2; idx++) { + ASSERT_OK(db_->Put(write_opts, Key1(idx), ts, "bar")); + ASSERT_OK(db_->Put(write_opts, KeyWithPrefix("foo", idx), ts, "bar")); + } + + ASSERT_OK(Flush()); + + for (; idx < kMaxKey; idx++) { + ASSERT_OK(db_->Put(write_opts, Key1(idx), ts, "bar")); + ASSERT_OK(db_->Put(write_opts, KeyWithPrefix("foo", idx), ts, "bar")); + } + + // Read with MultiGet + ReadOptions read_opts; + Slice read_ts = ts; + read_opts.timestamp = &read_ts; + + for (idx = 0; idx < kMaxKey; idx++) { + size_t batch_size = 4; + std::vector keys_str(batch_size); + std::vector values(batch_size); + std::vector statuses(batch_size); + ColumnFamilyHandle* cfh = db_->DefaultColumnFamily(); + + keys_str[0] = Key1(idx); + keys_str[1] = KeyWithPrefix("foo", idx); + keys_str[2] = Key1(kMaxKey + idx); + keys_str[3] = KeyWithPrefix("foo", kMaxKey + idx); + + auto keys = ConvertStrToSlice(keys_str); + + db_->MultiGet(read_opts, cfh, batch_size, keys.data(), values.data(), + statuses.data()); + + for (int i = 0; i < 2; i++) { + ASSERT_OK(statuses[i]); + } + for (int i = 2; i < 4; i++) { + ASSERT_TRUE(statuses[i].IsNotFound()); + } + + for (int i = 0; i < 2; i++) { + std::string value; + ASSERT_OK(db_->Get(read_opts, keys[i], &value)); + std::unique_ptr it1(db_->NewIterator(read_opts)); + ASSERT_NE(nullptr, it1); + ASSERT_OK(it1->status()); + it1->Seek(keys[i]); + ASSERT_TRUE(it1->Valid()); + } + + for (int i = 2; i < 4; i++) { + std::string value; + Status s = db_->Get(read_opts, keys[i], &value); + ASSERT_TRUE(s.IsNotFound()); + } + } + Close(); +} + +INSTANTIATE_TEST_CASE_P( + Timestamp, DBBasicTestWithTimestampFilterPrefixSettings, + ::testing::Combine( + ::testing::Values( + std::shared_ptr(nullptr), + std::shared_ptr(NewBloomFilterPolicy(10, true)), + std::shared_ptr(NewBloomFilterPolicy(10, + false))), + ::testing::Bool(), ::testing::Bool(), + ::testing::Values( + std::shared_ptr(NewFixedPrefixTransform(1)), + std::shared_ptr(NewFixedPrefixTransform(4)), + std::shared_ptr(NewFixedPrefixTransform(7)), + std::shared_ptr(NewFixedPrefixTransform(8))), + ::testing::Bool(), ::testing::Values(0, 0.1), + ::testing::Values( + BlockBasedTableOptions::IndexType::kBinarySearch, + BlockBasedTableOptions::IndexType::kHashSearch, + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch, + BlockBasedTableOptions::IndexType::kBinarySearchWithFirstKey))); +#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) + +class DataVisibilityTest : public DBBasicTestWithTimestampBase { + public: + DataVisibilityTest() : DBBasicTestWithTimestampBase("data_visibility_test") { + // Initialize test data + for (int i = 0; i < kTestDataSize; i++) { + test_data_[i].key = "key" + std::to_string(i); + test_data_[i].value = "value" + std::to_string(i); + test_data_[i].timestamp = Timestamp(i, 0); + test_data_[i].ts = i; + test_data_[i].seq_num = kMaxSequenceNumber; + } + } + + protected: + struct TestData { + std::string key; + std::string value; + int ts; + std::string timestamp; + SequenceNumber seq_num; + }; + + constexpr static int kTestDataSize = 3; + TestData test_data_[kTestDataSize]; + + void PutTestData(int index, ColumnFamilyHandle* cfh = nullptr) { + ASSERT_LE(index, kTestDataSize); + WriteOptions write_opts; + + if (cfh == nullptr) { + ASSERT_OK(db_->Put(write_opts, test_data_[index].key, + test_data_[index].timestamp, test_data_[index].value)); + const Snapshot* snap = db_->GetSnapshot(); + test_data_[index].seq_num = snap->GetSequenceNumber(); + if (index > 0) { + ASSERT_GT(test_data_[index].seq_num, test_data_[index - 1].seq_num); + } + db_->ReleaseSnapshot(snap); + } else { + ASSERT_OK(db_->Put(write_opts, cfh, test_data_[index].key, + test_data_[index].timestamp, test_data_[index].value)); + } + } + + void AssertVisibility(int ts, SequenceNumber seq, + std::vector statuses) { + ASSERT_EQ(kTestDataSize, statuses.size()); + for (int i = 0; i < kTestDataSize; i++) { + if (test_data_[i].seq_num <= seq && test_data_[i].ts <= ts) { + ASSERT_OK(statuses[i]); + } else { + ASSERT_TRUE(statuses[i].IsNotFound()); + } + } + } + + std::vector GetKeys() { + std::vector ret(kTestDataSize); + for (int i = 0; i < kTestDataSize; i++) { + ret[i] = test_data_[i].key; + } + return ret; + } + + void VerifyDefaultCF(int ts, const Snapshot* snap = nullptr) { + ReadOptions read_opts; + std::string read_ts = Timestamp(ts, 0); + Slice read_ts_slice = read_ts; + read_opts.timestamp = &read_ts_slice; + read_opts.snapshot = snap; + + ColumnFamilyHandle* cfh = db_->DefaultColumnFamily(); + std::vector cfs(kTestDataSize, cfh); + SequenceNumber seq = + snap ? snap->GetSequenceNumber() : kMaxSequenceNumber - 1; + + // There're several MultiGet interfaces with not exactly the same + // implementations, query data with all of them. + auto keys = GetKeys(); + std::vector values; + auto s1 = db_->MultiGet(read_opts, cfs, keys, &values); + AssertVisibility(ts, seq, s1); + + auto s2 = db_->MultiGet(read_opts, keys, &values); + AssertVisibility(ts, seq, s2); + + std::vector timestamps; + auto s3 = db_->MultiGet(read_opts, cfs, keys, &values, ×tamps); + AssertVisibility(ts, seq, s3); + + auto s4 = db_->MultiGet(read_opts, keys, &values, ×tamps); + AssertVisibility(ts, seq, s4); + + std::vector values_ps5(kTestDataSize); + std::vector s5(kTestDataSize); + db_->MultiGet(read_opts, cfh, kTestDataSize, keys.data(), values_ps5.data(), + s5.data()); + AssertVisibility(ts, seq, s5); + + std::vector values_ps6(kTestDataSize); + std::vector s6(kTestDataSize); + std::vector timestamps_array(kTestDataSize); + db_->MultiGet(read_opts, cfh, kTestDataSize, keys.data(), values_ps6.data(), + timestamps_array.data(), s6.data()); + AssertVisibility(ts, seq, s6); + + std::vector values_ps7(kTestDataSize); + std::vector s7(kTestDataSize); + db_->MultiGet(read_opts, kTestDataSize, cfs.data(), keys.data(), + values_ps7.data(), s7.data()); + AssertVisibility(ts, seq, s7); + + std::vector values_ps8(kTestDataSize); + std::vector s8(kTestDataSize); + db_->MultiGet(read_opts, kTestDataSize, cfs.data(), keys.data(), + values_ps8.data(), timestamps_array.data(), s8.data()); + AssertVisibility(ts, seq, s8); + } + + void VerifyDefaultCF(const Snapshot* snap = nullptr) { + for (int i = 0; i <= kTestDataSize; i++) { + VerifyDefaultCF(i, snap); + } + } +}; +constexpr int DataVisibilityTest::kTestDataSize; + +// Application specifies timestamp but not snapshot. +// reader writer +// ts'=90 +// ts=100 +// seq=10 +// seq'=11 +// write finishes +// GetImpl(ts,seq) +// It is OK to return if ts>=t1 AND seq>=s1. If ts>=t1 but seqDisableProcessing(); + SyncPoint::GetInstance()->LoadDependency({ + {"DBImpl::GetImpl:3", + "DataVisibilityTest::PointLookupWithoutSnapshot1:BeforePut"}, + {"DataVisibilityTest::PointLookupWithoutSnapshot1:AfterPut", + "DBImpl::GetImpl:4"}, + }); + SyncPoint::GetInstance()->EnableProcessing(); + port::Thread writer_thread([this]() { + std::string write_ts = Timestamp(1, 0); + WriteOptions write_opts; + TEST_SYNC_POINT( + "DataVisibilityTest::PointLookupWithoutSnapshot1:BeforePut"); + Status s = db_->Put(write_opts, "foo", write_ts, "value"); + ASSERT_OK(s); + TEST_SYNC_POINT("DataVisibilityTest::PointLookupWithoutSnapshot1:AfterPut"); + }); + ReadOptions read_opts; + std::string read_ts_str = Timestamp(3, 0); + Slice read_ts = read_ts_str; + read_opts.timestamp = &read_ts; + std::string value; + Status s = db_->Get(read_opts, "foo", &value); + + writer_thread.join(); + ASSERT_TRUE(s.IsNotFound()); + Close(); +} + +// Application specifies timestamp but not snapshot. +// reader writer +// ts'=90 +// ts=100 +// seq=10 +// seq'=11 +// write finishes +// Flush +// GetImpl(ts,seq) +// It is OK to return if ts>=t1 AND seq>=s1. If ts>=t1 but seqDisableProcessing(); + SyncPoint::GetInstance()->LoadDependency({ + {"DBImpl::GetImpl:3", + "DataVisibilityTest::PointLookupWithoutSnapshot2:BeforePut"}, + {"DataVisibilityTest::PointLookupWithoutSnapshot2:AfterPut", + "DBImpl::GetImpl:4"}, + }); + SyncPoint::GetInstance()->EnableProcessing(); + port::Thread writer_thread([this]() { + std::string write_ts = Timestamp(1, 0); + WriteOptions write_opts; + TEST_SYNC_POINT( + "DataVisibilityTest::PointLookupWithoutSnapshot2:BeforePut"); + Status s = db_->Put(write_opts, "foo", write_ts, "value"); + ASSERT_OK(s); + ASSERT_OK(Flush()); + + write_ts = Timestamp(2, 0); + s = db_->Put(write_opts, "bar", write_ts, "value"); + ASSERT_OK(s); + TEST_SYNC_POINT("DataVisibilityTest::PointLookupWithoutSnapshot2:AfterPut"); + }); + ReadOptions read_opts; + std::string read_ts_str = Timestamp(3, 0); + Slice read_ts = read_ts_str; + read_opts.timestamp = &read_ts; + std::string value; + Status s = db_->Get(read_opts, "foo", &value); + writer_thread.join(); + ASSERT_TRUE(s.IsNotFound()); + Close(); +} + +// Application specifies both timestamp and snapshot. +// reader writer +// seq=10 +// ts'=90 +// ts=100 +// seq'=11 +// write finishes +// GetImpl(ts,seq) +// Since application specifies both timestamp and snapshot, application expects +// to see data that visible in BOTH timestamp and sequence number. Therefore, +// can be returned only if t1<=ts AND s1<=seq. +TEST_F(DataVisibilityTest, PointLookupWithSnapshot1) { + Options options = CurrentOptions(); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->LoadDependency({ + {"DataVisibilityTest::PointLookupWithSnapshot1:AfterTakingSnap", + "DataVisibilityTest::PointLookupWithSnapshot1:BeforePut"}, + {"DataVisibilityTest::PointLookupWithSnapshot1:AfterPut", + "DBImpl::GetImpl:1"}, + }); + SyncPoint::GetInstance()->EnableProcessing(); + port::Thread writer_thread([this]() { + std::string write_ts = Timestamp(1, 0); + WriteOptions write_opts; + TEST_SYNC_POINT("DataVisibilityTest::PointLookupWithSnapshot1:BeforePut"); + Status s = db_->Put(write_opts, "foo", write_ts, "value"); + TEST_SYNC_POINT("DataVisibilityTest::PointLookupWithSnapshot1:AfterPut"); + ASSERT_OK(s); + }); + ReadOptions read_opts; + const Snapshot* snap = db_->GetSnapshot(); + TEST_SYNC_POINT( + "DataVisibilityTest::PointLookupWithSnapshot1:AfterTakingSnap"); + read_opts.snapshot = snap; + std::string read_ts_str = Timestamp(3, 0); + Slice read_ts = read_ts_str; + read_opts.timestamp = &read_ts; + std::string value; + Status s = db_->Get(read_opts, "foo", &value); + writer_thread.join(); + + ASSERT_TRUE(s.IsNotFound()); + + db_->ReleaseSnapshot(snap); + Close(); +} + +// Application specifies both timestamp and snapshot. +// reader writer +// seq=10 +// ts'=90 +// ts=100 +// seq'=11 +// write finishes +// Flush +// GetImpl(ts,seq) +// Since application specifies both timestamp and snapshot, application expects +// to see data that visible in BOTH timestamp and sequence number. Therefore, +// can be returned only if t1<=ts AND s1<=seq. +TEST_F(DataVisibilityTest, PointLookupWithSnapshot2) { + Options options = CurrentOptions(); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->LoadDependency({ + {"DataVisibilityTest::PointLookupWithSnapshot2:AfterTakingSnap", + "DataVisibilityTest::PointLookupWithSnapshot2:BeforePut"}, + }); + SyncPoint::GetInstance()->EnableProcessing(); + port::Thread writer_thread([this]() { + std::string write_ts = Timestamp(1, 0); + WriteOptions write_opts; + TEST_SYNC_POINT("DataVisibilityTest::PointLookupWithSnapshot2:BeforePut"); + Status s = db_->Put(write_opts, "foo", write_ts, "value1"); + ASSERT_OK(s); + ASSERT_OK(Flush()); + + write_ts = Timestamp(2, 0); + s = db_->Put(write_opts, "bar", write_ts, "value2"); + ASSERT_OK(s); + }); + const Snapshot* snap = db_->GetSnapshot(); + TEST_SYNC_POINT( + "DataVisibilityTest::PointLookupWithSnapshot2:AfterTakingSnap"); + writer_thread.join(); + std::string read_ts_str = Timestamp(3, 0); + Slice read_ts = read_ts_str; + ReadOptions read_opts; + read_opts.snapshot = snap; + read_opts.timestamp = &read_ts; + std::string value; + Status s = db_->Get(read_opts, "foo", &value); + ASSERT_TRUE(s.IsNotFound()); + db_->ReleaseSnapshot(snap); + Close(); +} + +// Application specifies timestamp but not snapshot. +// reader writer +// ts'=90 +// ts=100 +// seq=10 +// seq'=11 +// write finishes +// scan(ts,seq) +// can be seen in scan as long as ts>=t1 AND seq>=s1. If ts>=t1 but +// seqDisableProcessing(); + SyncPoint::GetInstance()->LoadDependency({ + {"DBImpl::NewIterator:3", + "DataVisibilityTest::RangeScanWithoutSnapshot:BeforePut"}, + }); + SyncPoint::GetInstance()->EnableProcessing(); + port::Thread writer_thread([this]() { + WriteOptions write_opts; + TEST_SYNC_POINT("DataVisibilityTest::RangeScanWithoutSnapshot:BeforePut"); + for (int i = 0; i < 3; ++i) { + std::string write_ts = Timestamp(i + 1, 0); + Status s = db_->Put(write_opts, "key" + std::to_string(i), write_ts, + "value" + std::to_string(i)); + ASSERT_OK(s); + } + }); + std::string read_ts_str = Timestamp(10, 0); + Slice read_ts = read_ts_str; + ReadOptions read_opts; + read_opts.total_order_seek = true; + read_opts.timestamp = &read_ts; + Iterator* it = db_->NewIterator(read_opts); + ASSERT_NE(nullptr, it); + writer_thread.join(); + it->SeekToFirst(); + ASSERT_FALSE(it->Valid()); + delete it; + Close(); +} + +// Application specifies both timestamp and snapshot. +// reader writer +// seq=10 +// ts'=90 +// ts=100 seq'=11 +// write finishes +// scan(ts,seq) +// can be seen by the scan only if t1<=ts AND s1<=seq. If t1<=ts +// but s1>seq, then the key should not be returned. +TEST_F(DataVisibilityTest, RangeScanWithSnapshot) { + Options options = CurrentOptions(); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->LoadDependency({ + {"DataVisibilityTest::RangeScanWithSnapshot:AfterTakingSnapshot", + "DataVisibilityTest::RangeScanWithSnapshot:BeforePut"}, + }); + SyncPoint::GetInstance()->EnableProcessing(); + port::Thread writer_thread([this]() { + WriteOptions write_opts; + TEST_SYNC_POINT("DataVisibilityTest::RangeScanWithSnapshot:BeforePut"); + for (int i = 0; i < 3; ++i) { + std::string write_ts = Timestamp(i + 1, 0); + Status s = db_->Put(write_opts, "key" + std::to_string(i), write_ts, + "value" + std::to_string(i)); + ASSERT_OK(s); + } + }); + const Snapshot* snap = db_->GetSnapshot(); + TEST_SYNC_POINT( + "DataVisibilityTest::RangeScanWithSnapshot:AfterTakingSnapshot"); + + writer_thread.join(); + + std::string read_ts_str = Timestamp(10, 0); + Slice read_ts = read_ts_str; + ReadOptions read_opts; + read_opts.snapshot = snap; + read_opts.total_order_seek = true; + read_opts.timestamp = &read_ts; + Iterator* it = db_->NewIterator(read_opts); + ASSERT_NE(nullptr, it); + it->Seek("key0"); + ASSERT_FALSE(it->Valid()); + + delete it; + db_->ReleaseSnapshot(snap); + Close(); +} + +// Application specifies both timestamp and snapshot. +// Query each combination and make sure for MultiGet key , only +// return keys that ts>=t1 AND seq>=s1. +TEST_F(DataVisibilityTest, MultiGetWithTimestamp) { + Options options = CurrentOptions(); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + + const Snapshot* snap0 = db_->GetSnapshot(); + PutTestData(0); + VerifyDefaultCF(); + VerifyDefaultCF(snap0); + + const Snapshot* snap1 = db_->GetSnapshot(); + PutTestData(1); + VerifyDefaultCF(); + VerifyDefaultCF(snap0); + VerifyDefaultCF(snap1); + + ASSERT_OK(Flush()); + + const Snapshot* snap2 = db_->GetSnapshot(); + PutTestData(2); + VerifyDefaultCF(); + VerifyDefaultCF(snap0); + VerifyDefaultCF(snap1); + VerifyDefaultCF(snap2); + + db_->ReleaseSnapshot(snap0); + db_->ReleaseSnapshot(snap1); + db_->ReleaseSnapshot(snap2); + + Close(); +} + +// Application specifies timestamp but not snapshot. +// reader writer +// ts'=0, 1 +// ts=3 +// seq=10 +// seq'=11, 12 +// write finishes +// MultiGet(ts,seq) +// For MultiGet , only return keys that ts>=t1 AND seq>=s1. +TEST_F(DataVisibilityTest, MultiGetWithoutSnapshot) { + Options options = CurrentOptions(); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->LoadDependency({ + {"DBImpl::MultiGet:AfterGetSeqNum1", + "DataVisibilityTest::MultiGetWithoutSnapshot:BeforePut"}, + {"DataVisibilityTest::MultiGetWithoutSnapshot:AfterPut", + "DBImpl::MultiGet:AfterGetSeqNum2"}, + }); + SyncPoint::GetInstance()->EnableProcessing(); + port::Thread writer_thread([this]() { + TEST_SYNC_POINT("DataVisibilityTest::MultiGetWithoutSnapshot:BeforePut"); + PutTestData(0); + PutTestData(1); + TEST_SYNC_POINT("DataVisibilityTest::MultiGetWithoutSnapshot:AfterPut"); + }); + + ReadOptions read_opts; + std::string read_ts = Timestamp(kTestDataSize, 0); + Slice read_ts_slice = read_ts; + read_opts.timestamp = &read_ts_slice; + auto keys = GetKeys(); + std::vector values; + auto ss = db_->MultiGet(read_opts, keys, &values); + + writer_thread.join(); + for (auto s : ss) { + ASSERT_TRUE(s.IsNotFound()); + } + VerifyDefaultCF(); + Close(); +} + +TEST_F(DataVisibilityTest, MultiGetCrossCF) { + Options options = CurrentOptions(); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + + CreateAndReopenWithCF({"second"}, options); + ColumnFamilyHandle* second_cf = handles_[1]; + + const Snapshot* snap0 = db_->GetSnapshot(); + PutTestData(0); + PutTestData(0, second_cf); + VerifyDefaultCF(); + VerifyDefaultCF(snap0); + + const Snapshot* snap1 = db_->GetSnapshot(); + PutTestData(1); + PutTestData(1, second_cf); + VerifyDefaultCF(); + VerifyDefaultCF(snap0); + VerifyDefaultCF(snap1); + + ASSERT_OK(Flush()); + + const Snapshot* snap2 = db_->GetSnapshot(); + PutTestData(2); + PutTestData(2, second_cf); + VerifyDefaultCF(); + VerifyDefaultCF(snap0); + VerifyDefaultCF(snap1); + VerifyDefaultCF(snap2); + + ReadOptions read_opts; + std::string read_ts = Timestamp(kTestDataSize, 0); + Slice read_ts_slice = read_ts; + read_opts.timestamp = &read_ts_slice; + read_opts.snapshot = snap1; + auto keys = GetKeys(); + auto keys2 = GetKeys(); + keys.insert(keys.end(), keys2.begin(), keys2.end()); + std::vector cfs(kTestDataSize, + db_->DefaultColumnFamily()); + std::vector cfs2(kTestDataSize, second_cf); + cfs.insert(cfs.end(), cfs2.begin(), cfs2.end()); + + std::vector values; + auto ss = db_->MultiGet(read_opts, cfs, keys, &values); + for (int i = 0; i < 2 * kTestDataSize; i++) { + if (i % 3 == 0) { + // only the first key for each column family should be returned + ASSERT_OK(ss[i]); + } else { + ASSERT_TRUE(ss[i].IsNotFound()); + } + } + + db_->ReleaseSnapshot(snap0); + db_->ReleaseSnapshot(snap1); + db_->ReleaseSnapshot(snap2); + Close(); +} + +#if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) +class DBBasicTestWithTimestampCompressionSettings + : public DBBasicTestWithTimestampBase, + public testing::WithParamInterface< + std::tuple, CompressionType, + uint32_t, uint32_t>> { + public: + DBBasicTestWithTimestampCompressionSettings() + : DBBasicTestWithTimestampBase( + "db_basic_test_with_timestamp_compression") {} +}; + +TEST_P(DBBasicTestWithTimestampCompressionSettings, PutAndGet) { + const int kNumKeysPerFile = 1024; + const size_t kNumTimestamps = 4; + Options options = CurrentOptions(); + options.create_if_missing = true; + options.env = env_; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + size_t ts_sz = Timestamp(0, 0).size(); + TestComparator test_cmp(ts_sz); + options.comparator = &test_cmp; + BlockBasedTableOptions bbto; + bbto.filter_policy = std::get<0>(GetParam()); + bbto.whole_key_filtering = true; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + + const CompressionType comp_type = std::get<1>(GetParam()); +#if LZ4_VERSION_NUMBER < 10400 // r124+ + if (comp_type == kLZ4Compression || comp_type == kLZ4HCCompression) { + return; + } +#endif // LZ4_VERSION_NUMBER >= 10400 + if (!ZSTD_Supported() && comp_type == kZSTD) { + return; + } + if (!Zlib_Supported() && comp_type == kZlibCompression) { + return; + } + + options.compression = comp_type; + options.compression_opts.max_dict_bytes = std::get<2>(GetParam()); + if (comp_type == kZSTD) { + options.compression_opts.zstd_max_train_bytes = std::get<2>(GetParam()); + } + options.compression_opts.parallel_threads = std::get<3>(GetParam()); + options.target_file_size_base = 1 << 26; // 64MB + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + size_t num_cfs = handles_.size(); + ASSERT_EQ(2, num_cfs); + std::vector write_ts_list; + std::vector read_ts_list; + + for (size_t i = 0; i != kNumTimestamps; ++i) { + write_ts_list.push_back(Timestamp(i * 2, 0)); + read_ts_list.push_back(Timestamp(1 + i * 2, 0)); + const Slice write_ts = write_ts_list.back(); + WriteOptions wopts; + for (int cf = 0; cf != static_cast(num_cfs); ++cf) { + for (size_t j = 0; j != (kNumKeysPerFile - 1) / kNumTimestamps; ++j) { + ASSERT_OK( + db_->Put(wopts, handles_[cf], Key1(j), write_ts, + "value_" + std::to_string(j) + "_" + std::to_string(i))); + } + } + } + const auto& verify_db_func = [&]() { + for (size_t i = 0; i != kNumTimestamps; ++i) { + ReadOptions ropts; + const Slice read_ts = read_ts_list[i]; + ropts.timestamp = &read_ts; + for (int cf = 0; cf != static_cast(num_cfs); ++cf) { + ColumnFamilyHandle* cfh = handles_[cf]; + for (size_t j = 0; j != (kNumKeysPerFile - 1) / kNumTimestamps; ++j) { + std::string value; + ASSERT_OK(db_->Get(ropts, cfh, Key1(j), &value)); + ASSERT_EQ("value_" + std::to_string(j) + "_" + std::to_string(i), + value); + } + } + } + }; + verify_db_func(); + Close(); +} + +TEST_P(DBBasicTestWithTimestampCompressionSettings, PutDeleteGet) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + const int kNumKeysPerFile = 1024; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + BlockBasedTableOptions bbto; + bbto.filter_policy = std::get<0>(GetParam()); + bbto.whole_key_filtering = true; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + + const CompressionType comp_type = std::get<1>(GetParam()); +#if LZ4_VERSION_NUMBER < 10400 // r124+ + if (comp_type == kLZ4Compression || comp_type == kLZ4HCCompression) { + return; + } +#endif // LZ4_VERSION_NUMBER >= 10400 + if (!ZSTD_Supported() && comp_type == kZSTD) { + return; + } + if (!Zlib_Supported() && comp_type == kZlibCompression) { + return; + } + + options.compression = comp_type; + options.compression_opts.max_dict_bytes = std::get<2>(GetParam()); + if (comp_type == kZSTD) { + options.compression_opts.zstd_max_train_bytes = std::get<2>(GetParam()); + } + options.compression_opts.parallel_threads = std::get<3>(GetParam()); + options.target_file_size_base = 1 << 26; // 64MB + + DestroyAndReopen(options); + + const size_t kNumL0Files = + static_cast(Options().level0_file_num_compaction_trigger); + { + // Half of the keys will go through Deletion and remaining half with + // SingleDeletion. Generate enough L0 files with ts=1 to trigger compaction + // to L1 + std::string ts = Timestamp(1, 0); + WriteOptions wopts; + for (size_t i = 0; i < kNumL0Files; ++i) { + for (int j = 0; j < kNumKeysPerFile; ++j) { + ASSERT_OK(db_->Put(wopts, Key1(j), ts, "value" + std::to_string(i))); + } + ASSERT_OK(db_->Flush(FlushOptions())); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // Generate another L0 at ts=3 + ts = Timestamp(3, 0); + for (int i = 0; i < kNumKeysPerFile; ++i) { + std::string key_str = Key1(i); + Slice key(key_str); + if ((i % 3) == 0) { + if (i < kNumKeysPerFile / 2) { + ASSERT_OK(db_->Delete(wopts, key, ts)); + } else { + ASSERT_OK(db_->SingleDelete(wopts, key, ts)); + } + } else { + ASSERT_OK(db_->Put(wopts, key, ts, "new_value")); + } + } + ASSERT_OK(db_->Flush(FlushOptions())); + // Populate memtable at ts=5 + ts = Timestamp(5, 0); + for (int i = 0; i != kNumKeysPerFile; ++i) { + std::string key_str = Key1(i); + Slice key(key_str); + if ((i % 3) == 1) { + if (i < kNumKeysPerFile / 2) { + ASSERT_OK(db_->Delete(wopts, key, ts)); + } else { + ASSERT_OK(db_->SingleDelete(wopts, key, ts)); + } + } else if ((i % 3) == 2) { + ASSERT_OK(db_->Put(wopts, key, ts, "new_value_2")); + } + } + } + { + std::string ts_str = Timestamp(6, 0); + Slice ts = ts_str; + ReadOptions ropts; + ropts.timestamp = &ts; + for (uint64_t i = 0; i != static_cast(kNumKeysPerFile); ++i) { + std::string value; + std::string key_ts; + Status s = db_->Get(ropts, Key1(i), &value, &key_ts); + if ((i % 3) == 2) { + ASSERT_OK(s); + ASSERT_EQ("new_value_2", value); + ASSERT_EQ(Timestamp(5, 0), key_ts); + } else if ((i % 3) == 1) { + ASSERT_TRUE(s.IsNotFound()); + ASSERT_EQ(Timestamp(5, 0), key_ts); + } else { + ASSERT_TRUE(s.IsNotFound()); + ASSERT_EQ(Timestamp(3, 0), key_ts); + } + } + } +} + +// A class which remembers the name of each flushed file. +class FlushedFileCollector : public EventListener { + public: + FlushedFileCollector() {} + ~FlushedFileCollector() override {} + + void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& info) override { + InstrumentedMutexLock lock(&mutex_); + flushed_files_.push_back(info.file_path); + } + + std::vector GetFlushedFiles() { + std::vector result; + { + InstrumentedMutexLock lock(&mutex_); + result = flushed_files_; + } + return result; + } + + void ClearFlushedFiles() { + InstrumentedMutexLock lock(&mutex_); + flushed_files_.clear(); + } + + private: + std::vector flushed_files_; + InstrumentedMutex mutex_; +}; + +TEST_P(DBBasicTestWithTimestampCompressionSettings, PutAndGetWithCompaction) { + const int kNumKeysPerFile = 1024; + const size_t kNumTimestamps = 2; + const size_t kNumKeysPerTimestamp = (kNumKeysPerFile - 1) / kNumTimestamps; + const size_t kSplitPosBase = kNumKeysPerTimestamp / 2; + Options options = CurrentOptions(); + options.create_if_missing = true; + options.env = env_; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + + FlushedFileCollector* collector = new FlushedFileCollector(); + options.listeners.emplace_back(collector); + + size_t ts_sz = Timestamp(0, 0).size(); + TestComparator test_cmp(ts_sz); + options.comparator = &test_cmp; + BlockBasedTableOptions bbto; + bbto.filter_policy = std::get<0>(GetParam()); + bbto.whole_key_filtering = true; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + + const CompressionType comp_type = std::get<1>(GetParam()); +#if LZ4_VERSION_NUMBER < 10400 // r124+ + if (comp_type == kLZ4Compression || comp_type == kLZ4HCCompression) { + return; + } +#endif // LZ4_VERSION_NUMBER >= 10400 + if (!ZSTD_Supported() && comp_type == kZSTD) { + return; + } + if (!Zlib_Supported() && comp_type == kZlibCompression) { + return; + } + + options.compression = comp_type; + options.compression_opts.max_dict_bytes = std::get<2>(GetParam()); + if (comp_type == kZSTD) { + options.compression_opts.zstd_max_train_bytes = std::get<2>(GetParam()); + } + options.compression_opts.parallel_threads = std::get<3>(GetParam()); + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + + size_t num_cfs = handles_.size(); + ASSERT_EQ(2, num_cfs); + std::vector write_ts_list; + std::vector read_ts_list; + + const auto& verify_records_func = [&](size_t i, size_t begin, size_t end, + ColumnFamilyHandle* cfh) { + std::string value; + std::string timestamp; + + ReadOptions ropts; + const Slice read_ts = read_ts_list[i]; + ropts.timestamp = &read_ts; + std::string expected_timestamp = + std::string(write_ts_list[i].data(), write_ts_list[i].size()); + + for (size_t j = begin; j <= end; ++j) { + ASSERT_OK(db_->Get(ropts, cfh, Key1(j), &value, ×tamp)); + ASSERT_EQ("value_" + std::to_string(j) + "_" + std::to_string(i), value); + ASSERT_EQ(expected_timestamp, timestamp); + } + }; + + for (size_t i = 0; i != kNumTimestamps; ++i) { + write_ts_list.push_back(Timestamp(i * 2, 0)); + read_ts_list.push_back(Timestamp(1 + i * 2, 0)); + const Slice write_ts = write_ts_list.back(); + WriteOptions wopts; + for (int cf = 0; cf != static_cast(num_cfs); ++cf) { + size_t memtable_get_start = 0; + for (size_t j = 0; j != kNumKeysPerTimestamp; ++j) { + ASSERT_OK( + db_->Put(wopts, handles_[cf], Key1(j), write_ts, + "value_" + std::to_string(j) + "_" + std::to_string(i))); + if (j == kSplitPosBase + i || j == kNumKeysPerTimestamp - 1) { + verify_records_func(i, memtable_get_start, j, handles_[cf]); + memtable_get_start = j + 1; + + // flush all keys with the same timestamp to two sst files, split at + // incremental positions such that lowerlevel[1].smallest.userkey == + // higherlevel[0].largest.userkey + ASSERT_OK(Flush(cf)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); // wait for flush (which + // is also a compaction) + + // compact files (2 at each level) to a lower level such that all + // keys with the same timestamp is at one level, with newer versions + // at higher levels. + CompactionOptions compact_opt; + compact_opt.compression = kNoCompression; + ASSERT_OK(db_->CompactFiles(compact_opt, handles_[cf], + collector->GetFlushedFiles(), + static_cast(kNumTimestamps - i))); + collector->ClearFlushedFiles(); + } + } + } + } + const auto& verify_db_func = [&]() { + for (size_t i = 0; i != kNumTimestamps; ++i) { + ReadOptions ropts; + const Slice read_ts = read_ts_list[i]; + ropts.timestamp = &read_ts; + std::string expected_timestamp(write_ts_list[i].data(), + write_ts_list[i].size()); + for (int cf = 0; cf != static_cast(num_cfs); ++cf) { + ColumnFamilyHandle* cfh = handles_[cf]; + verify_records_func(i, 0, kNumKeysPerTimestamp - 1, cfh); + } + } + }; + verify_db_func(); + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, BatchWriteAndMultiGet) { + const int kNumKeysPerFile = 8192; + const size_t kNumTimestamps = 2; + const size_t kNumKeysPerTimestamp = (kNumKeysPerFile - 1) / kNumTimestamps; + Options options = CurrentOptions(); + options.create_if_missing = true; + options.env = env_; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + options.memtable_prefix_bloom_size_ratio = 0.1; + options.memtable_whole_key_filtering = true; + + size_t ts_sz = Timestamp(0, 0).size(); + TestComparator test_cmp(ts_sz); + options.comparator = &test_cmp; + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(NewBloomFilterPolicy( + 10 /*bits_per_key*/, false /*use_block_based_builder*/)); + bbto.whole_key_filtering = true; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + DestroyAndReopen(options); + CreateAndReopenWithCF({"pikachu"}, options); + size_t num_cfs = handles_.size(); + ASSERT_EQ(2, num_cfs); + std::vector write_ts_list; + std::vector read_ts_list; + + const auto& verify_records_func = [&](size_t i, ColumnFamilyHandle* cfh) { + std::vector keys; + std::vector key_vals; + std::vector values; + std::vector timestamps; + + for (size_t j = 0; j != kNumKeysPerTimestamp; ++j) { + key_vals.push_back(Key1(j)); + } + for (size_t j = 0; j != kNumKeysPerTimestamp; ++j) { + keys.push_back(key_vals[j]); + } + + ReadOptions ropts; + const Slice read_ts = read_ts_list[i]; + ropts.timestamp = &read_ts; + std::string expected_timestamp(write_ts_list[i].data(), + write_ts_list[i].size()); + + std::vector cfhs(keys.size(), cfh); + std::vector statuses = + db_->MultiGet(ropts, cfhs, keys, &values, ×tamps); + for (size_t j = 0; j != kNumKeysPerTimestamp; ++j) { + ASSERT_OK(statuses[j]); + ASSERT_EQ("value_" + std::to_string(j) + "_" + std::to_string(i), + values[j]); + ASSERT_EQ(expected_timestamp, timestamps[j]); + } + }; + + const std::string dummy_ts(ts_sz, '\0'); + for (size_t i = 0; i != kNumTimestamps; ++i) { + write_ts_list.push_back(Timestamp(i * 2, 0)); + read_ts_list.push_back(Timestamp(1 + i * 2, 0)); + const Slice& write_ts = write_ts_list.back(); + for (int cf = 0; cf != static_cast(num_cfs); ++cf) { + WriteOptions wopts; + WriteBatch batch(0, 0, 0, ts_sz); + for (size_t j = 0; j != kNumKeysPerTimestamp; ++j) { + const std::string key = Key1(j); + const std::string value = + "value_" + std::to_string(j) + "_" + std::to_string(i); + ASSERT_OK(batch.Put(handles_[cf], key, value)); + } + ASSERT_OK(batch.UpdateTimestamps(write_ts, + [ts_sz](uint32_t) { return ts_sz; })); + ASSERT_OK(db_->Write(wopts, &batch)); + + verify_records_func(i, handles_[cf]); + + ASSERT_OK(Flush(cf)); + } + } + + const auto& verify_db_func = [&]() { + for (size_t i = 0; i != kNumTimestamps; ++i) { + ReadOptions ropts; + const Slice read_ts = read_ts_list[i]; + ropts.timestamp = &read_ts; + for (int cf = 0; cf != static_cast(num_cfs); ++cf) { + ColumnFamilyHandle* cfh = handles_[cf]; + verify_records_func(i, cfh); + } + } + }; + verify_db_func(); + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, MultiGetNoReturnTs) { + Options options = CurrentOptions(); + options.env = env_; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + WriteOptions write_opts; + std::string ts = Timestamp(1, 0); + ASSERT_OK(db_->Put(write_opts, "foo", ts, "value")); + ASSERT_OK(db_->Put(write_opts, "bar", ts, "value")); + ASSERT_OK(db_->Put(write_opts, "fooxxxxxxxxxxxxxxxx", ts, "value")); + ASSERT_OK(db_->Put(write_opts, "barxxxxxxxxxxxxxxxx", ts, "value")); + ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily(); + ts = Timestamp(2, 0); + Slice read_ts = ts; + ReadOptions read_opts; + read_opts.timestamp = &read_ts; + { + ColumnFamilyHandle* column_families[] = {cfh, cfh}; + Slice keys[] = {"foo", "bar"}; + PinnableSlice values[] = {PinnableSlice(), PinnableSlice()}; + Status statuses[] = {Status::OK(), Status::OK()}; + dbfull()->MultiGet(read_opts, /*num_keys=*/2, &column_families[0], &keys[0], + &values[0], &statuses[0], /*sorted_input=*/false); + for (const auto& s : statuses) { + ASSERT_OK(s); + } + } + { + ColumnFamilyHandle* column_families[] = {cfh, cfh, cfh, cfh}; + // Make user keys longer than configured timestamp size (16 bytes) to + // verify RocksDB does not use the trailing bytes 'x' as timestamp. + Slice keys[] = {"fooxxxxxxxxxxxxxxxx", "barxxxxxxxxxxxxxxxx", "foo", "bar"}; + PinnableSlice values[] = {PinnableSlice(), PinnableSlice(), PinnableSlice(), + PinnableSlice()}; + Status statuses[] = {Status::OK(), Status::OK(), Status::OK(), + Status::OK()}; + dbfull()->MultiGet(read_opts, /*num_keys=*/4, &column_families[0], &keys[0], + &values[0], &statuses[0], /*sorted_input=*/false); + for (const auto& s : statuses) { + ASSERT_OK(s); + } + } + Close(); +} + + +INSTANTIATE_TEST_CASE_P( + Timestamp, DBBasicTestWithTimestampCompressionSettings, + ::testing::Combine( + ::testing::Values(std::shared_ptr(nullptr), + std::shared_ptr( + NewBloomFilterPolicy(10, false))), + ::testing::Values(kNoCompression, kZlibCompression, kLZ4Compression, + kLZ4HCCompression, kZSTD), + ::testing::Values(0, 1 << 14), ::testing::Values(1, 4))); + +class DBBasicTestWithTimestampPrefixSeek + : public DBBasicTestWithTimestampBase, + public testing::WithParamInterface< + std::tuple, + std::shared_ptr, bool, + BlockBasedTableOptions::IndexType>> { + public: + DBBasicTestWithTimestampPrefixSeek() + : DBBasicTestWithTimestampBase( + "/db_basic_test_with_timestamp_prefix_seek") {} +}; + +TEST_P(DBBasicTestWithTimestampPrefixSeek, IterateWithPrefix) { + const size_t kNumKeysPerFile = 128; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.prefix_extractor = std::get<0>(GetParam()); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + BlockBasedTableOptions bbto; + bbto.filter_policy = std::get<1>(GetParam()); + bbto.index_type = std::get<3>(GetParam()); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + DestroyAndReopen(options); + + const uint64_t kMaxKey = 0xffffffffffffffff; + const uint64_t kMinKey = 0xfffffffffffff000; + const std::vector write_ts_list = {Timestamp(3, 0xffffffff), + Timestamp(6, 0xffffffff)}; + WriteOptions write_opts; + { + for (size_t i = 0; i != write_ts_list.size(); ++i) { + for (uint64_t key = kMaxKey; key >= kMinKey; --key) { + Status s = db_->Put(write_opts, Key1(key), write_ts_list[i], + "value" + std::to_string(i)); + ASSERT_OK(s); + } + } + } + const std::vector read_ts_list = {Timestamp(5, 0xffffffff), + Timestamp(9, 0xffffffff)}; + { + ReadOptions read_opts; + read_opts.total_order_seek = false; + read_opts.prefix_same_as_start = std::get<2>(GetParam()); + fprintf(stdout, "%s %s %d\n", options.prefix_extractor->Name(), + bbto.filter_policy ? bbto.filter_policy->Name() : "null", + static_cast(read_opts.prefix_same_as_start)); + for (size_t i = 0; i != read_ts_list.size(); ++i) { + Slice read_ts = read_ts_list[i]; + read_opts.timestamp = &read_ts; + std::unique_ptr iter(db_->NewIterator(read_opts)); + + // Seek to kMaxKey + iter->Seek(Key1(kMaxKey)); + CheckIterUserEntry(iter.get(), Key1(kMaxKey), kTypeValue, + "value" + std::to_string(i), write_ts_list[i]); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + + // Seek to kMinKey + iter->Seek(Key1(kMinKey)); + CheckIterUserEntry(iter.get(), Key1(kMinKey), kTypeValue, + "value" + std::to_string(i), write_ts_list[i]); + iter->Prev(); + ASSERT_FALSE(iter->Valid()); + } + const std::vector targets = {kMinKey, kMinKey + 0x10, + kMinKey + 0x100, kMaxKey}; + const SliceTransform* const pe = options.prefix_extractor.get(); + ASSERT_NE(nullptr, pe); + const size_t kPrefixShift = + 8 * (Key1(0).size() - pe->Transform(Key1(0)).size()); + const uint64_t kPrefixMask = + ~((static_cast(1) << kPrefixShift) - 1); + const uint64_t kNumKeysWithinPrefix = + (static_cast(1) << kPrefixShift); + for (size_t i = 0; i != read_ts_list.size(); ++i) { + Slice read_ts = read_ts_list[i]; + read_opts.timestamp = &read_ts; + std::unique_ptr it(db_->NewIterator(read_opts)); + // Forward and backward iterate. + for (size_t j = 0; j != targets.size(); ++j) { + std::string start_key = Key1(targets[j]); + uint64_t expected_ub = + (targets[j] & kPrefixMask) - 1 + kNumKeysWithinPrefix; + uint64_t expected_key = targets[j]; + size_t count = 0; + it->Seek(Key1(targets[j])); + while (it->Valid()) { + std::string saved_prev_key; + saved_prev_key.assign(it->key().data(), it->key().size()); + + // Out of prefix + if (!read_opts.prefix_same_as_start && + pe->Transform(saved_prev_key) != pe->Transform(start_key)) { + break; + } + CheckIterUserEntry(it.get(), Key1(expected_key), kTypeValue, + "value" + std::to_string(i), write_ts_list[i]); + ++count; + ++expected_key; + it->Next(); + } + ASSERT_EQ(expected_ub - targets[j] + 1, count); + + count = 0; + expected_key = targets[j]; + it->SeekForPrev(start_key); + uint64_t expected_lb = (targets[j] & kPrefixMask); + while (it->Valid()) { + // Out of prefix + if (!read_opts.prefix_same_as_start && + pe->Transform(it->key()) != pe->Transform(start_key)) { + break; + } + CheckIterUserEntry(it.get(), Key1(expected_key), kTypeValue, + "value" + std::to_string(i), write_ts_list[i]); + ++count; + --expected_key; + it->Prev(); + } + ASSERT_EQ(targets[j] - std::max(expected_lb, kMinKey) + 1, count); + } + } + } + Close(); +} + +// TODO(yanqin): consider handling non-fixed-length prefix extractors, e.g. +// NoopTransform. +INSTANTIATE_TEST_CASE_P( + Timestamp, DBBasicTestWithTimestampPrefixSeek, + ::testing::Combine( + ::testing::Values( + std::shared_ptr(NewFixedPrefixTransform(1)), + std::shared_ptr(NewFixedPrefixTransform(4)), + std::shared_ptr(NewFixedPrefixTransform(7)), + std::shared_ptr(NewFixedPrefixTransform(8))), + ::testing::Values(std::shared_ptr(nullptr), + std::shared_ptr( + NewBloomFilterPolicy(10 /*bits_per_key*/, false)), + std::shared_ptr( + NewBloomFilterPolicy(20 /*bits_per_key*/, + false))), + ::testing::Bool(), + ::testing::Values( + BlockBasedTableOptions::IndexType::kBinarySearch, + BlockBasedTableOptions::IndexType::kHashSearch, + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch, + BlockBasedTableOptions::IndexType::kBinarySearchWithFirstKey))); + +class DBBasicTestWithTsIterTombstones + : public DBBasicTestWithTimestampBase, + public testing::WithParamInterface< + std::tuple, + std::shared_ptr, int, + BlockBasedTableOptions::IndexType>> { + public: + DBBasicTestWithTsIterTombstones() + : DBBasicTestWithTimestampBase("/db_basic_ts_iter_tombstones") {} +}; + +TEST_P(DBBasicTestWithTsIterTombstones, IterWithDelete) { + constexpr size_t kNumKeysPerFile = 128; + Options options = CurrentOptions(); + options.env = env_; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.prefix_extractor = std::get<0>(GetParam()); + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + BlockBasedTableOptions bbto; + bbto.filter_policy = std::get<1>(GetParam()); + bbto.index_type = std::get<3>(GetParam()); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + options.num_levels = std::get<2>(GetParam()); + DestroyAndReopen(options); + std::vector write_ts_strs = {Timestamp(2, 0), Timestamp(4, 0)}; + constexpr uint64_t kMaxKey = 0xffffffffffffffff; + constexpr uint64_t kMinKey = 0xfffffffffffff000; + // Insert kMinKey...kMaxKey + uint64_t key = kMinKey; + WriteOptions write_opts; + Slice ts = write_ts_strs[0]; + do { + Status s = db_->Put(write_opts, Key1(key), write_ts_strs[0], + "value" + std::to_string(key)); + ASSERT_OK(s); + if (kMaxKey == key) { + break; + } + ++key; + } while (true); + + for (key = kMaxKey; key >= kMinKey; --key) { + Status s; + if (0 != (key % 2)) { + s = db_->Put(write_opts, Key1(key), write_ts_strs[1], + "value1" + std::to_string(key)); + } else { + s = db_->Delete(write_opts, Key1(key), write_ts_strs[1]); + } + ASSERT_OK(s); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + { + std::string read_ts = Timestamp(4, 0); + ts = read_ts; + ReadOptions read_opts; + read_opts.total_order_seek = true; + read_opts.timestamp = &ts; + std::unique_ptr iter(db_->NewIterator(read_opts)); + size_t count = 0; + key = kMinKey + 1; + for (iter->SeekToFirst(); iter->Valid(); iter->Next(), ++count, key += 2) { + ASSERT_EQ(Key1(key), iter->key()); + ASSERT_EQ("value1" + std::to_string(key), iter->value()); + } + ASSERT_EQ((kMaxKey - kMinKey + 1) / 2, count); + + for (iter->SeekToLast(), count = 0, key = kMaxKey; iter->Valid(); + key -= 2, ++count, iter->Prev()) { + ASSERT_EQ(Key1(key), iter->key()); + ASSERT_EQ("value1" + std::to_string(key), iter->value()); + } + ASSERT_EQ((kMaxKey - kMinKey + 1) / 2, count); + } + Close(); +} + +INSTANTIATE_TEST_CASE_P( + Timestamp, DBBasicTestWithTsIterTombstones, + ::testing::Combine( + ::testing::Values( + std::shared_ptr(NewFixedPrefixTransform(7)), + std::shared_ptr(NewFixedPrefixTransform(8))), + ::testing::Values(std::shared_ptr(nullptr), + std::shared_ptr( + NewBloomFilterPolicy(10, false)), + std::shared_ptr( + NewBloomFilterPolicy(20, false))), + ::testing::Values(2, 6), + ::testing::Values( + BlockBasedTableOptions::IndexType::kBinarySearch, + BlockBasedTableOptions::IndexType::kHashSearch, + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch, + BlockBasedTableOptions::IndexType::kBinarySearchWithFirstKey))); +#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) + +class UpdateFullHistoryTsLowTest : public DBBasicTestWithTimestampBase { + public: + UpdateFullHistoryTsLowTest() + : DBBasicTestWithTimestampBase("/update_full_history_ts_low_test") {} +}; + +TEST_F(UpdateFullHistoryTsLowTest, ConcurrentUpdate) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + std::string lower_ts_low = Timestamp(10, 0); + std::string higher_ts_low = Timestamp(25, 0); + const size_t kTimestampSize = lower_ts_low.size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + + DestroyAndReopen(options); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + // This workaround swaps `lower_ts_low` originally used for update by the + // caller to `higher_ts_low` after its writer is queued to make sure + // the caller will always get a TryAgain error. + // It mimics cases where two threads update full_history_ts_low concurrently + // with one thread writing a higher ts_low and one thread writing a lower + // ts_low. + VersionEdit* version_edit; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::IncreaseFullHistoryTsLowImpl:BeforeEdit", + [&](void* arg) { version_edit = reinterpret_cast(arg); }); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:BeforeWriterWaiting", + [&](void* /*arg*/) { version_edit->SetFullHistoryTsLow(higher_ts_low); }); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_TRUE( + db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(), lower_ts_low) + .IsTryAgain()); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + Close(); +} + +// Tests the effect of flag `persist_user_defined_timestamps` on the file +// boundaries contained in the Manifest, a.k.a FileMetaData.smallest, +// FileMetaData.largest. +class HandleFileBoundariesTest + : public DBBasicTestWithTimestampBase, + public testing::WithParamInterface { + public: + HandleFileBoundariesTest() + : DBBasicTestWithTimestampBase("/handle_file_boundaries") {} +}; + +TEST_P(HandleFileBoundariesTest, ConfigurePersistUdt) { + Options options = CurrentOptions(); + options.env = env_; + // Write a timestamp that is not the min timestamp to help test the behavior + // of flag `persist_user_defined_timestamps`. + std::string write_ts; + std::string min_ts; + PutFixed64(&write_ts, 1); + PutFixed64(&min_ts, 0); + std::string smallest_ukey_without_ts = "bar"; + std::string largest_ukey_without_ts = "foo"; + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + bool persist_udt = test::ShouldPersistUDT(GetParam()); + options.persist_user_defined_timestamps = persist_udt; + if (!persist_udt) { + options.allow_concurrent_memtable_write = false; + } + DestroyAndReopen(options); + + ASSERT_OK( + db_->Put(WriteOptions(), smallest_ukey_without_ts, write_ts, "val1")); + ASSERT_OK( + db_->Put(WriteOptions(), largest_ukey_without_ts, write_ts, "val2")); + + // Create a L0 SST file and its record is added to the Manfiest. + ASSERT_OK(Flush()); + Close(); + + options.create_if_missing = false; + // Reopen the DB and process manifest file. + Reopen(options); + + std::vector> level_to_files; + dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(), + &level_to_files); + ASSERT_GT(level_to_files.size(), 1); + // L0 only has one SST file. + ASSERT_EQ(level_to_files[0].size(), 1); + auto file_meta = level_to_files[0][0]; + if (persist_udt) { + ASSERT_EQ(smallest_ukey_without_ts + write_ts, + file_meta.smallest.user_key()); + ASSERT_EQ(largest_ukey_without_ts + write_ts, file_meta.largest.user_key()); + } else { + // If `persist_user_defined_timestamps` is false, the file boundaries should + // have the min timestamp. Behind the scenes, when file boundaries in + // FileMetaData is persisted to Manifest, the original user-defined + // timestamps in user key are stripped. When manifest is read and processed + // during DB open, a min timestamp is padded to the file boundaries. This + // test's writes contain non min timestamp to verify this logic end-to-end. + ASSERT_EQ(smallest_ukey_without_ts + min_ts, file_meta.smallest.user_key()); + ASSERT_EQ(largest_ukey_without_ts + min_ts, file_meta.largest.user_key()); + } + Close(); +} + +INSTANTIATE_TEST_CASE_P( + ConfigurePersistUdt, HandleFileBoundariesTest, + ::testing::Values( + test::UserDefinedTimestampTestMode::kStripUserDefinedTimestamp, + test::UserDefinedTimestampTestMode::kNormal)); + +TEST_F(DBBasicTestWithTimestamp, EnableDisableUDT) { + Options options = CurrentOptions(); + options.env = env_; + // Create a column family without user-defined timestamps. + options.comparator = BytewiseComparator(); + options.persist_user_defined_timestamps = true; + DestroyAndReopen(options); + + // Create one SST file, its user keys have no user-defined timestamps. + ASSERT_OK(db_->Put(WriteOptions(), "foo", "val1")); + ASSERT_OK(Flush(0)); + Close(); + + // Reopen the existing column family and enable user-defined timestamps + // feature for it. + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + options.persist_user_defined_timestamps = false; + options.allow_concurrent_memtable_write = false; + Reopen(options); + + std::string value; + ASSERT_TRUE(db_->Get(ReadOptions(), "foo", &value).IsInvalidArgument()); + std::string read_ts; + PutFixed64(&read_ts, 0); + ReadOptions ropts; + Slice read_ts_slice = read_ts; + ropts.timestamp = &read_ts_slice; + std::string key_ts; + // Entries in pre-existing SST files are treated as if they have minimum + // user-defined timestamps. + ASSERT_OK(db_->Get(ropts, "foo", &value, &key_ts)); + ASSERT_EQ("val1", value); + ASSERT_EQ(read_ts, key_ts); + + // Do timestamped read / write. + std::string write_ts; + PutFixed64(&write_ts, 1); + ASSERT_OK(db_->Put(WriteOptions(), "foo", write_ts, "val2")); + read_ts.clear(); + PutFixed64(&read_ts, 1); + ASSERT_OK(db_->Get(ropts, "foo", &value, &key_ts)); + ASSERT_EQ("val2", value); + ASSERT_EQ(write_ts, key_ts); + // The user keys in this SST file don't have user-defined timestamps either, + // because `persist_user_defined_timestamps` flag is set to false. + ASSERT_OK(Flush(0)); + Close(); + + // Reopen the existing column family while disabling user-defined timestamps. + options.comparator = BytewiseComparator(); + Reopen(options); + + ASSERT_TRUE(db_->Get(ropts, "foo", &value).IsInvalidArgument()); + ASSERT_OK(db_->Get(ReadOptions(), "foo", &value)); + ASSERT_EQ("val2", value); + + // Continue to write / read the column family without user-defined timestamps. + ASSERT_OK(db_->Put(WriteOptions(), "foo", "val3")); + ASSERT_OK(db_->Get(ReadOptions(), "foo", &value)); + ASSERT_EQ("val3", value); + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, + GCPreserveRangeTombstoneWhenNoOrSmallFullHistoryLow) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + + std::string ts_str = Timestamp(1, 0); + WriteOptions wopts; + ASSERT_OK(db_->Put(wopts, "k1", ts_str, "v1")); + ASSERT_OK(db_->Put(wopts, "k2", ts_str, "v2")); + ASSERT_OK(db_->Put(wopts, "k3", ts_str, "v3")); + ts_str = Timestamp(2, 0); + ASSERT_OK( + db_->DeleteRange(wopts, db_->DefaultColumnFamily(), "k1", "k3", ts_str)); + + ts_str = Timestamp(3, 0); + Slice ts = ts_str; + ReadOptions ropts; + ropts.timestamp = &ts; + CompactRangeOptions cro; + cro.full_history_ts_low = nullptr; + std::string value, key_ts; + Status s; + auto verify = [&] { + s = db_->Get(ropts, "k1", &value); + ASSERT_TRUE(s.IsNotFound()); + + s = db_->Get(ropts, "k2", &value, &key_ts); + ASSERT_TRUE(s.IsNotFound()); + ASSERT_EQ(key_ts, Timestamp(2, 0)); + + ASSERT_OK(db_->Get(ropts, "k3", &value, &key_ts)); + ASSERT_EQ(value, "v3"); + ASSERT_EQ(Timestamp(1, 0), key_ts); + + size_t batch_size = 3; + std::vector key_strs = {"k1", "k2", "k3"}; + std::vector keys{key_strs.begin(), key_strs.end()}; + std::vector values(batch_size); + std::vector statuses(batch_size); + db_->MultiGet(ropts, db_->DefaultColumnFamily(), batch_size, keys.data(), + values.data(), statuses.data(), true /* sorted_input */); + ASSERT_TRUE(statuses[0].IsNotFound()); + ASSERT_TRUE(statuses[1].IsNotFound()); + ASSERT_OK(statuses[2]); + ; + ASSERT_EQ(values[2], "v3"); + }; + verify(); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + verify(); + std::string lb = Timestamp(0, 0); + Slice lb_slice = lb; + cro.full_history_ts_low = &lb_slice; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + verify(); + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, + GCRangeTombstonesAndCoveredKeysRespectingTslow) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(NewBloomFilterPolicy(10, false)); + bbto.cache_index_and_filter_blocks = true; + bbto.whole_key_filtering = true; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.num_levels = 2; + DestroyAndReopen(options); + + WriteOptions wopts; + ASSERT_OK(db_->Put(wopts, "k1", Timestamp(1, 0), "v1")); + ASSERT_OK(db_->Delete(wopts, "k2", Timestamp(2, 0))); + ASSERT_OK(db_->DeleteRange(wopts, db_->DefaultColumnFamily(), "k1", "k3", + Timestamp(3, 0))); + ASSERT_OK(db_->Put(wopts, "k3", Timestamp(4, 0), "v3")); + + ReadOptions ropts; + std::string read_ts = Timestamp(5, 0); + Slice read_ts_slice = read_ts; + ropts.timestamp = &read_ts_slice; + size_t batch_size = 3; + std::vector key_strs = {"k1", "k2", "k3"}; + std::vector keys = {key_strs.begin(), key_strs.end()}; + std::vector values(batch_size); + std::vector statuses(batch_size); + std::vector timestamps(batch_size); + db_->MultiGet(ropts, db_->DefaultColumnFamily(), batch_size, keys.data(), + values.data(), timestamps.data(), statuses.data(), + true /* sorted_input */); + ASSERT_TRUE(statuses[0].IsNotFound()); + ASSERT_EQ(timestamps[0], Timestamp(3, 0)); + ASSERT_TRUE(statuses[1].IsNotFound()); + // DeleteRange has a higher timestamp than Delete for "k2" + ASSERT_EQ(timestamps[1], Timestamp(3, 0)); + ASSERT_OK(statuses[2]); + ASSERT_EQ(values[2], "v3"); + ASSERT_EQ(timestamps[2], Timestamp(4, 0)); + + CompactRangeOptions cro; + // Range tombstone has timestamp >= full_history_ts_low, covered keys + // are not dropped. + std::string compaction_ts_str = Timestamp(2, 0); + Slice compaction_ts = compaction_ts_str; + cro.full_history_ts_low = &compaction_ts; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ropts.timestamp = &compaction_ts; + std::string value, ts; + ASSERT_OK(db_->Get(ropts, "k1", &value, &ts)); + ASSERT_EQ(value, "v1"); + // timestamp is below full_history_ts_low, zeroed out as the key goes into + // bottommost level + ASSERT_EQ(ts, Timestamp(0, 0)); + ASSERT_TRUE(db_->Get(ropts, "k2", &value, &ts).IsNotFound()); + ASSERT_EQ(ts, Timestamp(2, 0)); + + compaction_ts_str = Timestamp(4, 0); + compaction_ts = compaction_ts_str; + cro.full_history_ts_low = &compaction_ts; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ropts.timestamp = &read_ts_slice; + // k1, k2 and the range tombstone should be dropped + // k3 should still exist + db_->MultiGet(ropts, db_->DefaultColumnFamily(), batch_size, keys.data(), + values.data(), timestamps.data(), statuses.data(), + true /* sorted_input */); + ASSERT_TRUE(statuses[0].IsNotFound()); + ASSERT_TRUE(timestamps[0].empty()); + ASSERT_TRUE(statuses[1].IsNotFound()); + ASSERT_TRUE(timestamps[1].empty()); + ASSERT_OK(statuses[2]); + ASSERT_EQ(values[2], "v3"); + ASSERT_EQ(timestamps[2], Timestamp(4, 0)); + + Close(); +} + +TEST_P(DBBasicTestWithTimestampTableOptions, DeleteRangeBaiscReadAndIterate) { + const int kNum = 200, kRangeBegin = 50, kRangeEnd = 150, kNumPerFile = 25; + Options options = CurrentOptions(); + options.prefix_extractor.reset(NewFixedPrefixTransform(3)); + options.compression = kNoCompression; + BlockBasedTableOptions bbto; + bbto.index_type = GetParam(); + bbto.block_size = 100; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile)); + DestroyAndReopen(options); + + // Write half of the keys before the tombstone and half after the tombstone. + // Only covered keys (i.e., within the range and older than the tombstone) + // should be deleted. + for (int i = 0; i < kNum; ++i) { + if (i == kNum / 2) { + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key1(kRangeBegin), Key1(kRangeEnd), + Timestamp(i, 0))); + } + ASSERT_OK(db_->Put(WriteOptions(), Key1(i), Timestamp(i, 0), + "val" + std::to_string(i))); + if (i == kNum - kNumPerFile) { + ASSERT_OK(Flush()); + } + } + + ReadOptions read_opts; + read_opts.total_order_seek = true; + std::string read_ts = Timestamp(kNum, 0); + Slice read_ts_slice = read_ts; + read_opts.timestamp = &read_ts_slice; + { + std::unique_ptr iter(db_->NewIterator(read_opts)); + ASSERT_OK(iter->status()); + + int expected = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_EQ(Key1(expected), iter->key()); + if (expected == kRangeBegin - 1) { + expected = kNum / 2; + } else { + ++expected; + } + } + ASSERT_EQ(kNum, expected); + + expected = kNum / 2; + for (iter->Seek(Key1(kNum / 2)); iter->Valid(); iter->Next()) { + ASSERT_EQ(Key1(expected), iter->key()); + ++expected; + } + ASSERT_EQ(kNum, expected); + + expected = kRangeBegin - 1; + for (iter->SeekForPrev(Key1(kNum / 2 - 1)); iter->Valid(); iter->Prev()) { + ASSERT_EQ(Key1(expected), iter->key()); + --expected; + } + ASSERT_EQ(-1, expected); + + read_ts = Timestamp(0, 0); + read_ts_slice = read_ts; + read_opts.timestamp = &read_ts_slice; + iter.reset(db_->NewIterator(read_opts)); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), Key1(0)); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + } + + read_ts = Timestamp(kNum, 0); + read_ts_slice = read_ts; + read_opts.timestamp = &read_ts_slice; + std::string value, timestamp; + Status s; + for (int i = 0; i < kNum; ++i) { + s = db_->Get(read_opts, Key1(i), &value, ×tamp); + if (i >= kRangeBegin && i < kNum / 2) { + ASSERT_TRUE(s.IsNotFound()); + ASSERT_EQ(timestamp, Timestamp(kNum / 2, 0)); + } else { + ASSERT_OK(s); + ASSERT_EQ(value, "val" + std::to_string(i)); + ASSERT_EQ(timestamp, Timestamp(i, 0)); + } + } + + size_t batch_size = kNum; + std::vector key_strs(batch_size); + std::vector keys(batch_size); + std::vector values(batch_size); + std::vector statuses(batch_size); + std::vector timestamps(batch_size); + for (int i = 0; i < kNum; ++i) { + key_strs[i] = Key1(i); + keys[i] = key_strs[i]; + } + db_->MultiGet(read_opts, db_->DefaultColumnFamily(), batch_size, keys.data(), + values.data(), timestamps.data(), statuses.data(), + true /* sorted_input */); + for (int i = 0; i < kNum; ++i) { + if (i >= kRangeBegin && i < kNum / 2) { + ASSERT_TRUE(statuses[i].IsNotFound()); + ASSERT_EQ(timestamps[i], Timestamp(kNum / 2, 0)); + } else { + ASSERT_OK(statuses[i]); + ASSERT_EQ(values[i], "val" + std::to_string(i)); + ASSERT_EQ(timestamps[i], Timestamp(i, 0)); + } + } + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, DeleteRangeGetIteratorWithSnapshot) { + // 4 keys 0, 1, 2, 3 at timestamps 0, 1, 2, 3 respectively. + // A range tombstone [1, 3) at timestamp 1 and has a sequence number between + // key 1 and 2. + Options options = CurrentOptions(); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + WriteOptions write_opts; + std::string put_ts = Timestamp(0, 0); + const int kNum = 4, kNumPerFile = 1, kRangeBegin = 1, kRangeEnd = 3; + options.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile)); + const Snapshot* before_tombstone = nullptr; + const Snapshot* after_tombstone = nullptr; + for (int i = 0; i < kNum; ++i) { + ASSERT_OK(db_->Put(WriteOptions(), Key1(i), Timestamp(i, 0), + "val" + std::to_string(i))); + if (i == kRangeBegin) { + before_tombstone = db_->GetSnapshot(); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key1(kRangeBegin), Key1(kRangeEnd), + Timestamp(kRangeBegin, 0))); + } + if (i == kNum / 2) { + ASSERT_OK(Flush()); + } + } + assert(before_tombstone); + after_tombstone = db_->GetSnapshot(); + // snapshot and ts before tombstone + std::string read_ts_str = Timestamp(kRangeBegin - 1, 0); // (0, 0) + Slice read_ts = read_ts_str; + ReadOptions read_opts; + read_opts.timestamp = &read_ts; + read_opts.snapshot = before_tombstone; + std::vector expected_status = { + Status::OK(), Status::NotFound(), Status::NotFound(), Status::NotFound()}; + std::vector expected_values(kNum); + expected_values[0] = "val" + std::to_string(0); + std::vector expected_timestamps(kNum); + expected_timestamps[0] = Timestamp(0, 0); + + size_t batch_size = kNum; + std::vector key_strs(batch_size); + std::vector keys(batch_size); + std::vector values(batch_size); + std::vector statuses(batch_size); + std::vector timestamps(batch_size); + for (int i = 0; i < kNum; ++i) { + key_strs[i] = Key1(i); + keys[i] = key_strs[i]; + } + + auto verify = [&] { + db_->MultiGet(read_opts, db_->DefaultColumnFamily(), batch_size, + keys.data(), values.data(), timestamps.data(), + statuses.data(), true /* sorted_input */); + std::string value, timestamp; + Status s; + for (int i = 0; i < kNum; ++i) { + s = db_->Get(read_opts, Key1(i), &value, ×tamp); + ASSERT_EQ(s, expected_status[i]); + ASSERT_EQ(statuses[i], expected_status[i]); + if (s.ok()) { + ASSERT_EQ(value, expected_values[i]); + ASSERT_EQ(values[i], expected_values[i]); + } + if (!timestamp.empty()) { + ASSERT_EQ(timestamp, expected_timestamps[i]); + ASSERT_EQ(timestamps[i], expected_timestamps[i]); + } else { + ASSERT_TRUE(timestamps[i].empty()); + } + } + std::unique_ptr iter(db_->NewIterator(read_opts)); + std::unique_ptr iter_for_seek(db_->NewIterator(read_opts)); + iter->SeekToFirst(); + for (int i = 0; i < kNum; ++i) { + if (expected_status[i].ok()) { + auto verify_iter = [&](Iterator* iter_ptr) { + ASSERT_TRUE(iter_ptr->Valid()); + ASSERT_EQ(iter_ptr->key(), keys[i]); + ASSERT_EQ(iter_ptr->value(), expected_values[i]); + ASSERT_EQ(iter_ptr->timestamp(), expected_timestamps[i]); + }; + verify_iter(iter.get()); + iter->Next(); + + iter_for_seek->Seek(keys[i]); + verify_iter(iter_for_seek.get()); + + iter_for_seek->SeekForPrev(keys[i]); + verify_iter(iter_for_seek.get()); + } + } + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + }; + + verify(); + + // snapshot before tombstone and ts after tombstone + read_ts_str = Timestamp(kNum, 0); // (4, 0) + read_ts = read_ts_str; + read_opts.timestamp = &read_ts; + read_opts.snapshot = before_tombstone; + expected_status[1] = Status::OK(); + expected_timestamps[1] = Timestamp(1, 0); + expected_values[1] = "val" + std::to_string(1); + verify(); + + // snapshot after tombstone and ts before tombstone + read_ts_str = Timestamp(kRangeBegin - 1, 0); // (0, 0) + read_ts = read_ts_str; + read_opts.timestamp = &read_ts; + read_opts.snapshot = after_tombstone; + expected_status[1] = Status::NotFound(); + expected_timestamps[1].clear(); + expected_values[1].clear(); + verify(); + + // snapshot and ts after tombstone + read_ts_str = Timestamp(kNum, 0); // (4, 0) + read_ts = read_ts_str; + read_opts.timestamp = &read_ts; + read_opts.snapshot = after_tombstone; + for (int i = 0; i < kNum; ++i) { + if (i == kRangeBegin) { + expected_status[i] = Status::NotFound(); + expected_values[i].clear(); + } else { + expected_status[i] = Status::OK(); + expected_values[i] = "val" + std::to_string(i); + } + expected_timestamps[i] = Timestamp(i, 0); + } + verify(); + + db_->ReleaseSnapshot(before_tombstone); + db_->ReleaseSnapshot(after_tombstone); + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, MergeBasic) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.merge_operator = std::make_shared('.'); + DestroyAndReopen(options); + + const std::array write_ts_strs = { + Timestamp(100, 0), Timestamp(200, 0), Timestamp(300, 0)}; + constexpr size_t kNumOfUniqKeys = 100; + ColumnFamilyHandle* default_cf = db_->DefaultColumnFamily(); + + for (size_t i = 0; i < write_ts_strs.size(); ++i) { + for (size_t j = 0; j < kNumOfUniqKeys; ++j) { + Status s; + if (i == 0) { + const std::string val = "v" + std::to_string(j) + "_0"; + s = db_->Put(WriteOptions(), Key1(j), write_ts_strs[i], val); + } else { + const std::string merge_op = std::to_string(i); + s = db_->Merge(WriteOptions(), default_cf, Key1(j), write_ts_strs[i], + merge_op); + } + ASSERT_OK(s); + } + } + + std::array read_ts_strs = { + Timestamp(150, 0), Timestamp(250, 0), Timestamp(350, 0)}; + + const auto verify_db_with_get = [&]() { + for (size_t i = 0; i < kNumOfUniqKeys; ++i) { + const std::string base_val = "v" + std::to_string(i) + "_0"; + const std::array expected_values = { + base_val, base_val + ".1", base_val + ".1.2"}; + const std::array& expected_ts = write_ts_strs; + ReadOptions read_opts; + for (size_t j = 0; j < read_ts_strs.size(); ++j) { + Slice read_ts = read_ts_strs[j]; + read_opts.timestamp = &read_ts; + std::string value; + std::string ts; + const Status s = db_->Get(read_opts, Key1(i), &value, &ts); + ASSERT_OK(s); + ASSERT_EQ(expected_values[j], value); + ASSERT_EQ(expected_ts[j], ts); + + // Do Seek/SeekForPrev + std::unique_ptr it(db_->NewIterator(read_opts)); + it->Seek(Key1(i)); + ASSERT_TRUE(it->Valid()); + ASSERT_EQ(expected_values[j], it->value()); + ASSERT_EQ(expected_ts[j], it->timestamp()); + + it->SeekForPrev(Key1(i)); + ASSERT_TRUE(it->Valid()); + ASSERT_EQ(expected_values[j], it->value()); + ASSERT_EQ(expected_ts[j], it->timestamp()); + } + } + }; + + const auto verify_db_with_iterator = [&]() { + std::string value_suffix; + for (size_t i = 0; i < read_ts_strs.size(); ++i) { + ReadOptions read_opts; + Slice read_ts = read_ts_strs[i]; + read_opts.timestamp = &read_ts; + std::unique_ptr it(db_->NewIterator(read_opts)); + size_t key_int_val = 0; + for (it->SeekToFirst(); it->Valid(); it->Next(), ++key_int_val) { + const std::string key = Key1(key_int_val); + const std::string value = + "v" + std::to_string(key_int_val) + "_0" + value_suffix; + ASSERT_EQ(key, it->key()); + ASSERT_EQ(value, it->value()); + ASSERT_EQ(write_ts_strs[i], it->timestamp()); + } + ASSERT_EQ(kNumOfUniqKeys, key_int_val); + + key_int_val = kNumOfUniqKeys - 1; + for (it->SeekToLast(); it->Valid(); it->Prev(), --key_int_val) { + const std::string key = Key1(key_int_val); + const std::string value = + "v" + std::to_string(key_int_val) + "_0" + value_suffix; + ASSERT_EQ(key, it->key()); + ASSERT_EQ(value, it->value()); + ASSERT_EQ(write_ts_strs[i], it->timestamp()); + } + ASSERT_EQ(std::numeric_limits::max(), key_int_val); + + value_suffix = value_suffix + "." + std::to_string(i + 1); + } + }; + + verify_db_with_get(); + verify_db_with_iterator(); + + ASSERT_OK(db_->Flush(FlushOptions())); + + verify_db_with_get(); + verify_db_with_iterator(); + + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, MergeAfterDeletion) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.merge_operator = std::make_shared('.'); + DestroyAndReopen(options); + + ColumnFamilyHandle* const column_family = db_->DefaultColumnFamily(); + + const size_t num_keys_per_file = 10; + const size_t num_merges_per_key = 2; + for (size_t i = 0; i < num_keys_per_file; ++i) { + std::string ts = Timestamp(i + 10000, 0); + Status s = db_->Delete(WriteOptions(), Key1(i), ts); + ASSERT_OK(s); + for (size_t j = 1; j <= num_merges_per_key; ++j) { + ts = Timestamp(i + 10000 + j, 0); + s = db_->Merge(WriteOptions(), column_family, Key1(i), ts, + std::to_string(j)); + ASSERT_OK(s); + } + } + + const auto verify_db = [&]() { + ReadOptions read_opts; + std::string read_ts_str = Timestamp(20000, 0); + Slice ts = read_ts_str; + read_opts.timestamp = &ts; + std::unique_ptr it(db_->NewIterator(read_opts)); + size_t count = 0; + for (it->SeekToFirst(); it->Valid(); it->Next(), ++count) { + std::string key = Key1(count); + ASSERT_EQ(key, it->key()); + std::string value; + for (size_t j = 1; j <= num_merges_per_key; ++j) { + value.append(std::to_string(j)); + if (j < num_merges_per_key) { + value.push_back('.'); + } + } + ASSERT_EQ(value, it->value()); + std::string ts1 = Timestamp(count + 10000 + num_merges_per_key, 0); + ASSERT_EQ(ts1, it->timestamp()); + } + ASSERT_OK(it->status()); + ASSERT_EQ(num_keys_per_file, count); + for (it->SeekToLast(); it->Valid(); it->Prev(), --count) { + std::string key = Key1(count - 1); + ASSERT_EQ(key, it->key()); + std::string value; + for (size_t j = 1; j <= num_merges_per_key; ++j) { + value.append(std::to_string(j)); + if (j < num_merges_per_key) { + value.push_back('.'); + } + } + ASSERT_EQ(value, it->value()); + std::string ts1 = Timestamp(count - 1 + 10000 + num_merges_per_key, 0); + ASSERT_EQ(ts1, it->timestamp()); + } + ASSERT_OK(it->status()); + ASSERT_EQ(0, count); + }; + + verify_db(); + + Close(); +} + +TEST_F(DBBasicTestWithTimestamp, RangeTombstoneApproximateSize) { + // Test code path for calculating range tombstone compensated size + // during flush and compaction. + Options options = CurrentOptions(); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + // So that the compaction below is non-bottommost and will calcualte + // compensated range tombstone size. + ASSERT_OK(db_->Put(WriteOptions(), Key(1), Timestamp(1, 0), "val")); + ASSERT_OK(Flush()); + MoveFilesToLevel(5); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0), + Key(1), Timestamp(1, 0))); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(1), + Key(2), Timestamp(2, 0))); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->RunManualCompaction( + static_cast_with_check(db_->DefaultColumnFamily()) + ->cfd(), + 0 /* input_level */, 1 /* output_level */, CompactRangeOptions(), + nullptr /* begin */, nullptr /* end */, true /* exclusive */, + true /* disallow_trivial_move */, + std::numeric_limits::max() /* max_file_num_to_ignore */, + "" /*trim_ts*/)); +} + +TEST_F(DBBasicTestWithTimestamp, IterSeekToLastWithIterateUpperbound) { + // Test for a bug fix where DBIter::SeekToLast() could fail when + // iterate_upper_bound and iter_start_ts are both set. + Options options = CurrentOptions(); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + DestroyAndReopen(options); + + ASSERT_OK(db_->Put(WriteOptions(), Key(1), Timestamp(2, 0), "val")); + ReadOptions ro; + std::string k = Key(1); + Slice k_slice = k; + ro.iterate_upper_bound = &k_slice; + std::string ts = Timestamp(3, 0); + Slice read_ts = ts; + ro.timestamp = &read_ts; + std::string start_ts = Timestamp(0, 0); + Slice start_ts_slice = start_ts; + ro.iter_start_ts = &start_ts_slice; + std::unique_ptr iter{db_->NewIterator(ro)}; + iter->SeekToLast(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); +} + +TEST_F(DBBasicTestWithTimestamp, TimestampFilterTableReadOnGet) { + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + BlockBasedTableOptions bbto; + bbto.block_size = 100; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + DestroyAndReopen(options); + + // Put + // Create two SST files + // file1: key => [1, 3], timestamp => [10, 20] + // file2, key => [2, 4], timestamp => [30, 40] + { + WriteOptions write_opts; + std::string write_ts = Timestamp(10, 0); + ASSERT_OK(db_->Put(write_opts, Key1(1), write_ts, "value1")); + write_ts = Timestamp(20, 0); + ASSERT_OK(db_->Put(write_opts, Key1(3), write_ts, "value3")); + ASSERT_OK(Flush()); + + write_ts = Timestamp(30, 0); + ASSERT_OK(db_->Put(write_opts, Key1(2), write_ts, "value2")); + write_ts = Timestamp(40, 0); + ASSERT_OK(db_->Put(write_opts, Key1(4), write_ts, "value4")); + ASSERT_OK(Flush()); + } + + // Get with timestamp + { + auto prev_checked_events = options.statistics->getTickerCount( + Tickers::TIMESTAMP_FILTER_TABLE_CHECKED); + auto prev_filtered_events = options.statistics->getTickerCount( + Tickers::TIMESTAMP_FILTER_TABLE_FILTERED); + + // key=3 (ts=20) does not exist at timestamp=1 + std::string read_ts_str = Timestamp(1, 0); + Slice read_ts_slice = Slice(read_ts_str); + ReadOptions read_opts; + read_opts.timestamp = &read_ts_slice; + std::string value_from_get = ""; + std::string timestamp_from_get = ""; + auto status = + db_->Get(read_opts, Key1(3), &value_from_get, ×tamp_from_get); + ASSERT_TRUE(status.IsNotFound()); + ASSERT_EQ(value_from_get, std::string("")); + ASSERT_EQ(timestamp_from_get, std::string("")); + + // key=3 is in the key ranges for both files, so both files will be queried. + // The table read was skipped because the timestamp is out of the table + // range, i.e.., 1 < [10,20], [30,40]. + // The tickers increase by 2 due to 2 files. + ASSERT_EQ(prev_checked_events + 2, + options.statistics->getTickerCount( + Tickers::TIMESTAMP_FILTER_TABLE_CHECKED)); + ASSERT_EQ(prev_filtered_events + 2, + options.statistics->getTickerCount( + Tickers::TIMESTAMP_FILTER_TABLE_FILTERED)); + + // key=3 (ts=20) exists at timestamp = 25 + read_ts_str = Timestamp(25, 0); + read_ts_slice = Slice(read_ts_str); + read_opts.timestamp = &read_ts_slice; + ASSERT_OK( + db_->Get(read_opts, Key1(3), &value_from_get, ×tamp_from_get)); + ASSERT_EQ("value3", value_from_get); + ASSERT_EQ(Timestamp(20, 0), timestamp_from_get); + + // file1 was not skipped, because the timestamp is in range, [10,20] < 25. + // file2 was skipped, because the timestamp is not in range, 25 < [30,40]. + // So the checked ticker increase by 2 due to 2 files; + // filtered ticker increase by 1 because file2 was skipped + ASSERT_EQ(prev_checked_events + 4, + options.statistics->getTickerCount( + Tickers::TIMESTAMP_FILTER_TABLE_CHECKED)); + ASSERT_EQ(prev_filtered_events + 3, + options.statistics->getTickerCount( + Tickers::TIMESTAMP_FILTER_TABLE_FILTERED)); + } + + Close(); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_with_timestamp_compaction_test.cc b/librocksdb-sys/rocksdb/db/db_with_timestamp_compaction_test.cc new file mode 100644 index 0000000..7d80c85 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_with_timestamp_compaction_test.cc @@ -0,0 +1,353 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/compaction/compaction.h" +#include "db/db_test_util.h" +#include "port/stack_trace.h" +#include "test_util/testutil.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { +std::string Key1(uint64_t key) { + std::string ret; + PutFixed64(&ret, key); + std::reverse(ret.begin(), ret.end()); + return ret; +} + +std::string Timestamp(uint64_t ts) { + std::string ret; + PutFixed64(&ret, ts); + return ret; +} +} // anonymous namespace + +class TimestampCompatibleCompactionTest : public DBTestBase { + public: + TimestampCompatibleCompactionTest() + : DBTestBase("ts_compatible_compaction_test", /*env_do_fsync=*/true) {} + + std::string Get(const std::string& key, uint64_t ts) { + ReadOptions read_opts; + std::string ts_str = Timestamp(ts); + Slice ts_slice = ts_str; + read_opts.timestamp = &ts_slice; + std::string value; + Status s = db_->Get(read_opts, key, &value); + if (s.IsNotFound()) { + value.assign("NOT_FOUND"); + } else if (!s.ok()) { + value.assign(s.ToString()); + } + return value; + } +}; + +TEST_F(TimestampCompatibleCompactionTest, UserKeyCrossFileBoundary) { + Options options = CurrentOptions(); + options.env = env_; + options.compaction_style = kCompactionStyleLevel; + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + options.level0_file_num_compaction_trigger = 3; + constexpr size_t kNumKeysPerFile = 101; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) { + const auto* compaction = reinterpret_cast(arg); + ASSERT_NE(nullptr, compaction); + ASSERT_EQ(0, compaction->start_level()); + ASSERT_EQ(1, compaction->num_input_levels()); + // Check that all 3 L0 ssts are picked for level compaction. + ASSERT_EQ(3, compaction->num_input_files(0)); + }); + SyncPoint::GetInstance()->EnableProcessing(); + // Write a L0 with keys 0, 1, ..., 99 with ts from 100 to 199. + uint64_t ts = 100; + uint64_t key = 0; + WriteOptions write_opts; + for (; key < kNumKeysPerFile - 1; ++key, ++ts) { + std::string ts_str = Timestamp(ts); + ASSERT_OK( + db_->Put(write_opts, Key1(key), ts_str, "foo_" + std::to_string(key))); + } + // Write another L0 with keys 99 with newer ts. + ASSERT_OK(Flush()); + uint64_t saved_read_ts1 = ts++; + key = 99; + for (int i = 0; i < 4; ++i, ++ts) { + std::string ts_str = Timestamp(ts); + ASSERT_OK( + db_->Put(write_opts, Key1(key), ts_str, "bar_" + std::to_string(key))); + } + ASSERT_OK(Flush()); + uint64_t saved_read_ts2 = ts++; + // Write another L0 with keys 99, 100, 101, ..., 150 + for (; key <= 150; ++key, ++ts) { + std::string ts_str = Timestamp(ts); + ASSERT_OK( + db_->Put(write_opts, Key1(key), ts_str, "foo1_" + std::to_string(key))); + } + ASSERT_OK(Flush()); + // Wait for compaction to finish + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + uint64_t read_ts = ts; + ASSERT_EQ("foo_99", Get(Key1(99), saved_read_ts1)); + ASSERT_EQ("bar_99", Get(Key1(99), saved_read_ts2)); + ASSERT_EQ("foo1_99", Get(Key1(99), read_ts)); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(TimestampCompatibleCompactionTest, MultipleSubCompactions) { + Options options = CurrentOptions(); + options.env = env_; + options.compaction_style = kCompactionStyleUniversal; + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + options.level0_file_num_compaction_trigger = 3; + options.max_subcompactions = 3; + options.target_file_size_base = 1024; + options.statistics = CreateDBStatistics(); + DestroyAndReopen(options); + + uint64_t ts = 100; + uint64_t key = 0; + WriteOptions write_opts; + + // Write keys 0, 1, ..., 499 with ts from 100 to 599. + { + for (; key <= 499; ++key, ++ts) { + std::string ts_str = Timestamp(ts); + ASSERT_OK(db_->Put(write_opts, Key1(key), ts_str, + "foo_" + std::to_string(key))); + } + } + + // Write keys 500, ..., 999 with ts from 600 to 1099. + { + for (; key <= 999; ++key, ++ts) { + std::string ts_str = Timestamp(ts); + ASSERT_OK(db_->Put(write_opts, Key1(key), ts_str, + "foo_" + std::to_string(key))); + } + ASSERT_OK(Flush()); + } + + // Wait for compaction to finish + { + ASSERT_OK(dbfull()->RunManualCompaction( + static_cast_with_check( + db_->DefaultColumnFamily()) + ->cfd(), + 0 /* input_level */, 1 /* output_level */, CompactRangeOptions(), + nullptr /* begin */, nullptr /* end */, true /* exclusive */, + true /* disallow_trivial_move */, + std::numeric_limits::max() /* max_file_num_to_ignore */, + "" /*trim_ts*/)); + } + + // Check stats to make sure multiple subcompactions were scheduled for + // boundaries not to be nullptr. + { + HistogramData num_sub_compactions; + options.statistics->histogramData(NUM_SUBCOMPACTIONS_SCHEDULED, + &num_sub_compactions); + ASSERT_GT(num_sub_compactions.sum, 1); + } + + for (key = 0; key <= 999; ++key) { + ASSERT_EQ("foo_" + std::to_string(key), Get(Key1(key), ts)); + } +} + +class TestFilePartitioner : public SstPartitioner { + public: + explicit TestFilePartitioner() {} + ~TestFilePartitioner() override {} + + const char* Name() const override { return "TestFilePartitioner"; } + PartitionerResult ShouldPartition( + const PartitionerRequest& /*request*/) override { + return PartitionerResult::kRequired; + } + bool CanDoTrivialMove(const Slice& /*smallest_user_key*/, + const Slice& /*largest_user_key*/) override { + return false; + } +}; + +class TestFilePartitionerFactory : public SstPartitionerFactory { + public: + explicit TestFilePartitionerFactory() {} + std::unique_ptr CreatePartitioner( + const SstPartitioner::Context& /*context*/) const override { + std::unique_ptr ret = + std::make_unique(); + return ret; + } + const char* Name() const override { return "TestFilePartitionerFactory"; } +}; + +TEST_F(TimestampCompatibleCompactionTest, CompactFilesRangeCheckL0) { + Options options = CurrentOptions(); + options.env = env_; + options.sst_partitioner_factory = + std::make_shared(); + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + options.disable_auto_compactions = true; + DestroyAndReopen(options); + + constexpr int kNumFiles = 10; + constexpr int kKeysPerFile = 2; + const std::string user_key = "foo"; + constexpr uint64_t start_ts = 10000; + + uint64_t cur_ts = start_ts; + for (int k = 0; k < kNumFiles; ++k) { + for (int i = 0; i < kKeysPerFile; ++i) { + ASSERT_OK(db_->Put(WriteOptions(), user_key, Timestamp(cur_ts), + "v" + std::to_string(i))); + ++cur_ts; + } + ASSERT_OK(db_->Flush(FlushOptions())); + } + + std::vector input_files{}; + { + std::vector files; + ASSERT_OK(env_->GetChildren(dbname_, &files)); + for (const auto& f : files) { + uint64_t file_num = 0; + FileType file_type = FileType::kWalFile; + if (!ParseFileName(f, &file_num, &file_type) || + file_type != FileType::kTableFile) { + continue; + } + input_files.emplace_back(f); + } + // sorting here by name, which also happens to sort by generation date. + std::sort(input_files.begin(), input_files.end()); + assert(kNumFiles == input_files.size()); + std::vector tmp; + tmp.emplace_back(input_files[input_files.size() / 2]); + input_files.swap(tmp); + } + + { + std::vector output_file_names; + CompactionJobInfo compaction_job_info; + ASSERT_OK(db_->CompactFiles(CompactionOptions(), input_files, + /*output_level=*/1, /*output_path_id=*/-1, + &output_file_names, &compaction_job_info)); + // We expect the L0 files older than the original provided input were all + // included in the compaction. + ASSERT_EQ(static_cast(kNumFiles / 2 + 1), + compaction_job_info.input_files.size()); + } +} + +TEST_F(TimestampCompatibleCompactionTest, CompactFilesRangeCheckL1) { + Options options = CurrentOptions(); + options.env = env_; + options.sst_partitioner_factory = + std::make_shared(); + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + + constexpr int kNumFiles = 4; + options.level0_file_num_compaction_trigger = kNumFiles; + + DestroyAndReopen(options); + + constexpr int kKeysPerFile = 2; + const std::string user_key = "foo"; + constexpr uint64_t start_ts = 10000; + + uint64_t cur_ts = start_ts; + // Generate some initial files in both L0 and L1. + for (int k = 0; k < kNumFiles; ++k) { + for (int i = 0; i < kKeysPerFile; ++i) { + ASSERT_OK(db_->Put(WriteOptions(), user_key, Timestamp(cur_ts), + "v" + std::to_string(i))); + ++cur_ts; + } + ASSERT_OK(db_->Flush(FlushOptions())); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ(0, NumTableFilesAtLevel(/*level=*/0, /*cf=*/0)); + ASSERT_EQ(kNumFiles * kKeysPerFile, + NumTableFilesAtLevel(/*level=*/1, /*cf=*/0)); + + constexpr int additional_l0s = 2; + for (int i = 0; i < additional_l0s; ++i, ++cur_ts) { + ASSERT_OK(db_->Put(WriteOptions(), user_key, Timestamp(cur_ts), "v")); + ASSERT_OK(db_->Flush(FlushOptions())); + } + ASSERT_EQ(additional_l0s, NumTableFilesAtLevel(/*level=*/0, /*cf=*/0)); + + std::vector inputs; + { + std::vector fmetas; + db_->GetLiveFilesMetaData(&fmetas); + bool included_one_l1 = false; + for (const auto& meta : fmetas) { + if (meta.level == 0) { + inputs.emplace_back(meta.relative_filename); + } else if (!included_one_l1) { + inputs.emplace_back(meta.relative_filename); + included_one_l1 = true; + } + } + } + ASSERT_EQ(static_cast(3), inputs.size()); + { + std::vector output_file_names; + CompactionJobInfo compaction_job_info; + + ASSERT_OK(db_->CompactFiles(CompactionOptions(), inputs, /*output_level=*/1, + /*output_path_id=*/-1, &output_file_names, + &compaction_job_info)); + ASSERT_EQ(kNumFiles * kKeysPerFile + 2, output_file_names.size()); + ASSERT_EQ(kNumFiles * kKeysPerFile + 2, + static_cast(compaction_job_info.input_files.size())); + } +} + +TEST_F(TimestampCompatibleCompactionTest, EmptyCompactionOutput) { + Options options = CurrentOptions(); + options.env = env_; + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + DestroyAndReopen(options); + + std::string ts_str = Timestamp(1); + WriteOptions wopts; + ASSERT_OK( + db_->DeleteRange(wopts, db_->DefaultColumnFamily(), "k1", "k3", ts_str)); + ASSERT_OK(Flush()); + + ts_str = Timestamp(3); + Slice ts = ts_str; + CompactRangeOptions cro; + // range tombstone will be dropped during compaction + cro.full_history_ts_low = &ts; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_with_timestamp_test_util.cc b/librocksdb-sys/rocksdb/db/db_with_timestamp_test_util.cc new file mode 100644 index 0000000..f562bcb --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_with_timestamp_test_util.cc @@ -0,0 +1,96 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/db_with_timestamp_test_util.h" + +namespace ROCKSDB_NAMESPACE { +std::string DBBasicTestWithTimestampBase::Key1(uint64_t k) { + std::string ret; + PutFixed64(&ret, k); + std::reverse(ret.begin(), ret.end()); + return ret; +} + +std::string DBBasicTestWithTimestampBase::KeyWithPrefix(std::string prefix, + uint64_t k) { + std::string ret; + PutFixed64(&ret, k); + std::reverse(ret.begin(), ret.end()); + return prefix + ret; +} + +std::vector DBBasicTestWithTimestampBase::ConvertStrToSlice( + std::vector& strings) { + std::vector ret; + for (const auto& s : strings) { + ret.emplace_back(s); + } + return ret; +} + +std::string DBBasicTestWithTimestampBase::Timestamp(uint64_t low, + uint64_t high) { + std::string ts; + PutFixed64(&ts, low); + PutFixed64(&ts, high); + return ts; +} + +void DBBasicTestWithTimestampBase::CheckIterUserEntry( + const Iterator* it, const Slice& expected_key, + ValueType expected_value_type, const Slice& expected_value, + const Slice& expected_ts) const { + ASSERT_TRUE(it->Valid()); + ASSERT_OK(it->status()); + ASSERT_EQ(expected_key, it->key()); + if (kTypeValue == expected_value_type) { + ASSERT_EQ(expected_value, it->value()); + } + ASSERT_EQ(expected_ts, it->timestamp()); +} + +void DBBasicTestWithTimestampBase::CheckIterEntry( + const Iterator* it, const Slice& expected_ukey, SequenceNumber expected_seq, + ValueType expected_val_type, const Slice& expected_value, + const Slice& expected_ts) const { + ASSERT_TRUE(it->Valid()); + ASSERT_OK(it->status()); + std::string ukey_and_ts; + ukey_and_ts.assign(expected_ukey.data(), expected_ukey.size()); + ukey_and_ts.append(expected_ts.data(), expected_ts.size()); + ParsedInternalKey parsed_ikey; + ASSERT_OK(ParseInternalKey(it->key(), &parsed_ikey, true /* log_err_key */)); + ASSERT_EQ(ukey_and_ts, parsed_ikey.user_key); + ASSERT_EQ(expected_val_type, parsed_ikey.type); + ASSERT_EQ(expected_seq, parsed_ikey.sequence); + if (expected_val_type == kTypeValue) { + ASSERT_EQ(expected_value, it->value()); + } + ASSERT_EQ(expected_ts, it->timestamp()); +} + +void DBBasicTestWithTimestampBase::CheckIterEntry( + const Iterator* it, const Slice& expected_ukey, ValueType expected_val_type, + const Slice& expected_value, const Slice& expected_ts) const { + ASSERT_TRUE(it->Valid()); + ASSERT_OK(it->status()); + std::string ukey_and_ts; + ukey_and_ts.assign(expected_ukey.data(), expected_ukey.size()); + ukey_and_ts.append(expected_ts.data(), expected_ts.size()); + + ParsedInternalKey parsed_ikey; + ASSERT_OK(ParseInternalKey(it->key(), &parsed_ikey, true /* log_err_key */)); + ASSERT_EQ(expected_val_type, parsed_ikey.type); + ASSERT_EQ(Slice(ukey_and_ts), parsed_ikey.user_key); + if (expected_val_type == kTypeValue) { + ASSERT_EQ(expected_value, it->value()); + } + ASSERT_EQ(expected_ts, it->timestamp()); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/db_with_timestamp_test_util.h b/librocksdb-sys/rocksdb/db/db_with_timestamp_test_util.h new file mode 100644 index 0000000..8a0d8e4 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_with_timestamp_test_util.h @@ -0,0 +1,126 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include "db/db_test_util.h" +#include "port/stack_trace.h" +#include "test_util/testutil.h" + +namespace ROCKSDB_NAMESPACE { +class DBBasicTestWithTimestampBase : public DBTestBase { + public: + explicit DBBasicTestWithTimestampBase(const std::string& dbname) + : DBTestBase(dbname, /*env_do_fsync=*/true) {} + + protected: + static std::string Key1(uint64_t k); + + static std::string KeyWithPrefix(std::string prefix, uint64_t k); + + static std::vector ConvertStrToSlice( + std::vector& strings); + + class TestComparator : public Comparator { + private: + const Comparator* cmp_without_ts_; + + public: + explicit TestComparator(size_t ts_sz) + : Comparator(ts_sz), cmp_without_ts_(nullptr) { + cmp_without_ts_ = BytewiseComparator(); + } + + const char* Name() const override { return "TestComparator"; } + + void FindShortSuccessor(std::string*) const override {} + + void FindShortestSeparator(std::string*, const Slice&) const override {} + + int Compare(const Slice& a, const Slice& b) const override { + int r = CompareWithoutTimestamp(a, b); + if (r != 0 || 0 == timestamp_size()) { + return r; + } + return -CompareTimestamp( + Slice(a.data() + a.size() - timestamp_size(), timestamp_size()), + Slice(b.data() + b.size() - timestamp_size(), timestamp_size())); + } + + using Comparator::CompareWithoutTimestamp; + int CompareWithoutTimestamp(const Slice& a, bool a_has_ts, const Slice& b, + bool b_has_ts) const override { + if (a_has_ts) { + assert(a.size() >= timestamp_size()); + } + if (b_has_ts) { + assert(b.size() >= timestamp_size()); + } + Slice lhs = a_has_ts ? StripTimestampFromUserKey(a, timestamp_size()) : a; + Slice rhs = b_has_ts ? StripTimestampFromUserKey(b, timestamp_size()) : b; + return cmp_without_ts_->Compare(lhs, rhs); + } + + int CompareTimestamp(const Slice& ts1, const Slice& ts2) const override { + if (!ts1.data() && !ts2.data()) { + return 0; + } else if (ts1.data() && !ts2.data()) { + return 1; + } else if (!ts1.data() && ts2.data()) { + return -1; + } + assert(ts1.size() == ts2.size()); + uint64_t low1 = 0; + uint64_t low2 = 0; + uint64_t high1 = 0; + uint64_t high2 = 0; + const size_t kSize = ts1.size(); + std::unique_ptr ts1_buf(new char[kSize]); + memcpy(ts1_buf.get(), ts1.data(), ts1.size()); + std::unique_ptr ts2_buf(new char[kSize]); + memcpy(ts2_buf.get(), ts2.data(), ts2.size()); + Slice ts1_copy = Slice(ts1_buf.get(), kSize); + Slice ts2_copy = Slice(ts2_buf.get(), kSize); + auto* ptr1 = const_cast(&ts1_copy); + auto* ptr2 = const_cast(&ts2_copy); + if (!GetFixed64(ptr1, &low1) || !GetFixed64(ptr1, &high1) || + !GetFixed64(ptr2, &low2) || !GetFixed64(ptr2, &high2)) { + assert(false); + } + if (high1 < high2) { + return -1; + } else if (high1 > high2) { + return 1; + } + if (low1 < low2) { + return -1; + } else if (low1 > low2) { + return 1; + } + return 0; + } + }; + + std::string Timestamp(uint64_t low, uint64_t high); + + void CheckIterUserEntry(const Iterator* it, const Slice& expected_key, + ValueType expected_value_type, + const Slice& expected_value, + const Slice& expected_ts) const; + + void CheckIterEntry(const Iterator* it, const Slice& expected_ukey, + SequenceNumber expected_seq, ValueType expected_val_type, + const Slice& expected_value, + const Slice& expected_ts) const; + + void CheckIterEntry(const Iterator* it, const Slice& expected_ukey, + ValueType expected_val_type, const Slice& expected_value, + const Slice& expected_ts) const; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/db_write_buffer_manager_test.cc b/librocksdb-sys/rocksdb/db/db_write_buffer_manager_test.cc new file mode 100644 index 0000000..82704e1 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_write_buffer_manager_test.cc @@ -0,0 +1,927 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/db_test_util.h" +#include "db/write_thread.h" +#include "port/stack_trace.h" + +namespace ROCKSDB_NAMESPACE { + +class DBWriteBufferManagerTest : public DBTestBase, + public testing::WithParamInterface { + public: + DBWriteBufferManagerTest() + : DBTestBase("db_write_buffer_manager_test", /*env_do_fsync=*/false) {} + bool cost_cache_; +}; + +TEST_P(DBWriteBufferManagerTest, SharedBufferAcrossCFs1) { + Options options = CurrentOptions(); + options.arena_block_size = 4096; + options.write_buffer_size = 500000; // this is never hit + std::shared_ptr cache = NewLRUCache(4 * 1024 * 1024, 2); + ASSERT_LT(cache->GetUsage(), 256 * 1024); + cost_cache_ = GetParam(); + + if (cost_cache_) { + options.write_buffer_manager.reset( + new WriteBufferManager(100000, cache, true)); + } else { + options.write_buffer_manager.reset( + new WriteBufferManager(100000, nullptr, true)); + } + + WriteOptions wo; + wo.disableWAL = true; + + CreateAndReopenWithCF({"cf1", "cf2", "cf3"}, options); + ASSERT_OK(Put(3, Key(1), DummyString(1), wo)); + Flush(3); + ASSERT_OK(Put(3, Key(1), DummyString(1), wo)); + ASSERT_OK(Put(0, Key(1), DummyString(1), wo)); + Flush(0); + + // Write to "Default", "cf2" and "cf3". + ASSERT_OK(Put(3, Key(1), DummyString(30000), wo)); + ASSERT_OK(Put(0, Key(1), DummyString(40000), wo)); + ASSERT_OK(Put(2, Key(1), DummyString(1), wo)); + + ASSERT_OK(Put(3, Key(2), DummyString(40000), wo)); + // WriteBufferManager::buffer_size_ has exceeded after the previous write is + // completed. + + // This make sures write will go through and if stall was in effect, it will + // end. + ASSERT_OK(Put(0, Key(2), DummyString(1), wo)); +} + +// Test Single DB with multiple writer threads get blocked when +// WriteBufferManager execeeds buffer_size_ and flush is waiting to be +// finished. +TEST_P(DBWriteBufferManagerTest, SharedWriteBufferAcrossCFs2) { + Options options = CurrentOptions(); + options.arena_block_size = 4096; + options.write_buffer_size = 500000; // this is never hit + std::shared_ptr cache = NewLRUCache(4 * 1024 * 1024, 2); + ASSERT_LT(cache->GetUsage(), 256 * 1024); + cost_cache_ = GetParam(); + + if (cost_cache_) { + options.write_buffer_manager.reset( + new WriteBufferManager(100000, cache, true)); + } else { + options.write_buffer_manager.reset( + new WriteBufferManager(100000, nullptr, true)); + } + WriteOptions wo; + wo.disableWAL = true; + + CreateAndReopenWithCF({"cf1", "cf2", "cf3"}, options); + ASSERT_OK(Put(3, Key(1), DummyString(1), wo)); + Flush(3); + ASSERT_OK(Put(3, Key(1), DummyString(1), wo)); + ASSERT_OK(Put(0, Key(1), DummyString(1), wo)); + Flush(0); + + // Write to "Default", "cf2" and "cf3". No flush will be triggered. + ASSERT_OK(Put(3, Key(1), DummyString(30000), wo)); + ASSERT_OK(Put(0, Key(1), DummyString(40000), wo)); + ASSERT_OK(Put(2, Key(1), DummyString(1), wo)); + + ASSERT_OK(Put(3, Key(2), DummyString(40000), wo)); + // WriteBufferManager::buffer_size_ has exceeded after the previous write is + // completed. + + std::unordered_set w_set; + std::vector threads; + int wait_count_db = 0; + int num_writers = 4; + InstrumentedMutex mutex; + InstrumentedCondVar cv(&mutex); + std::atomic thread_num(0); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBWriteBufferManagerTest::SharedWriteBufferAcrossCFs:0", + "DBImpl::BackgroundCallFlush:start"}}); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WBMStallInterface::BlockDB", [&](void*) { + InstrumentedMutexLock lock(&mutex); + wait_count_db++; + cv.SignalAll(); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WriteThread::WriteStall::Wait", [&](void* arg) { + InstrumentedMutexLock lock(&mutex); + WriteThread::Writer* w = reinterpret_cast(arg); + w_set.insert(w); + // Allow the flush to continue if all writer threads are blocked. + if (w_set.size() == (unsigned long)num_writers) { + TEST_SYNC_POINT( + "DBWriteBufferManagerTest::SharedWriteBufferAcrossCFs:0"); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + bool s = true; + + std::function writer = [&](int cf) { + int a = thread_num.fetch_add(1); + std::string key = "foo" + std::to_string(a); + Status tmp = Put(cf, Slice(key), DummyString(1), wo); + InstrumentedMutexLock lock(&mutex); + s = s && tmp.ok(); + }; + + // Flow: + // main_writer thread will write but will be blocked (as Flush will on hold, + // buffer_size_ has exceeded, thus will create stall in effect). + // | + // | + // multiple writer threads will be created to write across multiple columns + // and they will be blocked. + // | + // | + // Last writer thread will write and when its blocked it will signal Flush to + // continue to clear the stall. + + threads.emplace_back(writer, 1); + // Wait untill first thread (main_writer) writing to DB is blocked and then + // create the multiple writers which will be blocked from getting added to the + // queue because stall is in effect. + { + InstrumentedMutexLock lock(&mutex); + while (wait_count_db != 1) { + cv.Wait(); + } + } + for (int i = 0; i < num_writers; i++) { + threads.emplace_back(writer, i % 4); + } + for (auto& t : threads) { + t.join(); + } + + ASSERT_TRUE(s); + + // Number of DBs blocked. + ASSERT_EQ(wait_count_db, 1); + // Number of Writer threads blocked. + ASSERT_EQ(w_set.size(), num_writers); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +// Test multiple DBs get blocked when WriteBufferManager limit exceeds and flush +// is waiting to be finished but DBs tries to write meanwhile. +TEST_P(DBWriteBufferManagerTest, SharedWriteBufferLimitAcrossDB) { + std::vector dbnames; + std::vector dbs; + int num_dbs = 3; + + for (int i = 0; i < num_dbs; i++) { + dbs.push_back(nullptr); + dbnames.push_back( + test::PerThreadDBPath("db_shared_wb_db" + std::to_string(i))); + } + + Options options = CurrentOptions(); + options.arena_block_size = 4096; + options.write_buffer_size = 500000; // this is never hit + std::shared_ptr cache = NewLRUCache(4 * 1024 * 1024, 2); + ASSERT_LT(cache->GetUsage(), 256 * 1024); + cost_cache_ = GetParam(); + + if (cost_cache_) { + options.write_buffer_manager.reset( + new WriteBufferManager(100000, cache, true)); + } else { + options.write_buffer_manager.reset( + new WriteBufferManager(100000, nullptr, true)); + } + CreateAndReopenWithCF({"cf1", "cf2"}, options); + + for (int i = 0; i < num_dbs; i++) { + ASSERT_OK(DestroyDB(dbnames[i], options)); + ASSERT_OK(DB::Open(options, dbnames[i], &(dbs[i]))); + } + WriteOptions wo; + wo.disableWAL = true; + + for (int i = 0; i < num_dbs; i++) { + ASSERT_OK(dbs[i]->Put(wo, Key(1), DummyString(20000))); + } + // Insert to db_. + ASSERT_OK(Put(0, Key(1), DummyString(30000), wo)); + + // WriteBufferManager Limit exceeded. + std::vector threads; + int wait_count_db = 0; + InstrumentedMutex mutex; + InstrumentedCondVar cv(&mutex); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBWriteBufferManagerTest::SharedWriteBufferAcrossCFs:0", + "DBImpl::BackgroundCallFlush:start"}}); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WBMStallInterface::BlockDB", [&](void*) { + { + InstrumentedMutexLock lock(&mutex); + wait_count_db++; + cv.Signal(); + // Since this is the last DB, signal Flush to continue. + if (wait_count_db == num_dbs + 1) { + TEST_SYNC_POINT( + "DBWriteBufferManagerTest::SharedWriteBufferAcrossCFs:0"); + } + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + bool s = true; + + // Write to DB. + std::function write_db = [&](DB* db) { + Status tmp = db->Put(wo, Key(3), DummyString(1)); + InstrumentedMutexLock lock(&mutex); + s = s && tmp.ok(); + }; + + // Flow: + // db_ will write and will be blocked (as Flush will on hold and will create + // stall in effect). + // | + // multiple dbs writers will be created to write to that db and they will be + // blocked. + // | + // | + // Last writer will write and when its blocked it will signal Flush to + // continue to clear the stall. + + threads.emplace_back(write_db, db_); + // Wait untill first DB is blocked and then create the multiple writers for + // different DBs which will be blocked from getting added to the queue because + // stall is in effect. + { + InstrumentedMutexLock lock(&mutex); + while (wait_count_db != 1) { + cv.Wait(); + } + } + for (int i = 0; i < num_dbs; i++) { + threads.emplace_back(write_db, dbs[i]); + } + for (auto& t : threads) { + t.join(); + } + + ASSERT_TRUE(s); + ASSERT_EQ(num_dbs + 1, wait_count_db); + // Clean up DBs. + for (int i = 0; i < num_dbs; i++) { + ASSERT_OK(dbs[i]->Close()); + ASSERT_OK(DestroyDB(dbnames[i], options)); + delete dbs[i]; + } + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +// Test multiple threads writing across multiple DBs and multiple columns get +// blocked when stall by WriteBufferManager is in effect. +TEST_P(DBWriteBufferManagerTest, SharedWriteBufferLimitAcrossDB1) { + std::vector dbnames; + std::vector dbs; + int num_dbs = 3; + + for (int i = 0; i < num_dbs; i++) { + dbs.push_back(nullptr); + dbnames.push_back( + test::PerThreadDBPath("db_shared_wb_db" + std::to_string(i))); + } + + Options options = CurrentOptions(); + options.arena_block_size = 4096; + options.write_buffer_size = 500000; // this is never hit + std::shared_ptr cache = NewLRUCache(4 * 1024 * 1024, 2); + ASSERT_LT(cache->GetUsage(), 256 * 1024); + cost_cache_ = GetParam(); + + if (cost_cache_) { + options.write_buffer_manager.reset( + new WriteBufferManager(100000, cache, true)); + } else { + options.write_buffer_manager.reset( + new WriteBufferManager(100000, nullptr, true)); + } + CreateAndReopenWithCF({"cf1", "cf2"}, options); + + for (int i = 0; i < num_dbs; i++) { + ASSERT_OK(DestroyDB(dbnames[i], options)); + ASSERT_OK(DB::Open(options, dbnames[i], &(dbs[i]))); + } + WriteOptions wo; + wo.disableWAL = true; + + for (int i = 0; i < num_dbs; i++) { + ASSERT_OK(dbs[i]->Put(wo, Key(1), DummyString(20000))); + } + // Insert to db_. + ASSERT_OK(Put(0, Key(1), DummyString(30000), wo)); + + // WriteBufferManager::buffer_size_ has exceeded after the previous write to + // dbs[0] is completed. + std::vector threads; + int wait_count_db = 0; + InstrumentedMutex mutex; + InstrumentedCondVar cv(&mutex); + std::unordered_set w_set; + std::vector writer_threads; + std::atomic thread_num(0); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBWriteBufferManagerTest::SharedWriteBufferAcrossCFs:0", + "DBImpl::BackgroundCallFlush:start"}}); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WBMStallInterface::BlockDB", [&](void*) { + { + InstrumentedMutexLock lock(&mutex); + wait_count_db++; + thread_num.fetch_add(1); + cv.Signal(); + // Allow the flush to continue if all writer threads are blocked. + if (thread_num.load(std::memory_order_relaxed) == 2 * num_dbs + 1) { + TEST_SYNC_POINT( + "DBWriteBufferManagerTest::SharedWriteBufferAcrossCFs:0"); + } + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WriteThread::WriteStall::Wait", [&](void* arg) { + WriteThread::Writer* w = reinterpret_cast(arg); + { + InstrumentedMutexLock lock(&mutex); + w_set.insert(w); + thread_num.fetch_add(1); + // Allow the flush continue if all writer threads are blocked. + if (thread_num.load(std::memory_order_relaxed) == 2 * num_dbs + 1) { + TEST_SYNC_POINT( + "DBWriteBufferManagerTest::SharedWriteBufferAcrossCFs:0"); + } + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + bool s1 = true, s2 = true; + // Write to multiple columns of db_. + std::function write_cf = [&](int cf) { + Status tmp = Put(cf, Key(3), DummyString(1), wo); + InstrumentedMutexLock lock(&mutex); + s1 = s1 && tmp.ok(); + }; + // Write to multiple DBs. + std::function write_db = [&](DB* db) { + Status tmp = db->Put(wo, Key(3), DummyString(1)); + InstrumentedMutexLock lock(&mutex); + s2 = s2 && tmp.ok(); + }; + + // Flow: + // thread will write to db_ will be blocked (as Flush will on hold, + // buffer_size_ has exceeded and will create stall in effect). + // | + // | + // multiple writers threads writing to different DBs and to db_ across + // multiple columns will be created and they will be blocked due to stall. + // | + // | + // Last writer thread will write and when its blocked it will signal Flush to + // continue to clear the stall. + threads.emplace_back(write_db, db_); + // Wait untill first thread is blocked and then create the multiple writer + // threads. + { + InstrumentedMutexLock lock(&mutex); + while (wait_count_db != 1) { + cv.Wait(); + } + } + + for (int i = 0; i < num_dbs; i++) { + // Write to multiple columns of db_. + writer_threads.emplace_back(write_cf, i % 3); + // Write to different dbs. + threads.emplace_back(write_db, dbs[i]); + } + for (auto& t : threads) { + t.join(); + } + for (auto& t : writer_threads) { + t.join(); + } + + ASSERT_TRUE(s1); + ASSERT_TRUE(s2); + + // Number of DBs blocked. + ASSERT_EQ(num_dbs + 1, wait_count_db); + // Number of Writer threads blocked. + ASSERT_EQ(w_set.size(), num_dbs); + // Clean up DBs. + for (int i = 0; i < num_dbs; i++) { + ASSERT_OK(dbs[i]->Close()); + ASSERT_OK(DestroyDB(dbnames[i], options)); + delete dbs[i]; + } + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +// Test multiple threads writing across multiple columns of db_ by passing +// different values to WriteOption.no_slown_down. +TEST_P(DBWriteBufferManagerTest, MixedSlowDownOptionsSingleDB) { + Options options = CurrentOptions(); + options.arena_block_size = 4096; + options.write_buffer_size = 500000; // this is never hit + std::shared_ptr cache = NewLRUCache(4 * 1024 * 1024, 2); + ASSERT_LT(cache->GetUsage(), 256 * 1024); + cost_cache_ = GetParam(); + + if (cost_cache_) { + options.write_buffer_manager.reset( + new WriteBufferManager(100000, cache, true)); + } else { + options.write_buffer_manager.reset( + new WriteBufferManager(100000, nullptr, true)); + } + WriteOptions wo; + wo.disableWAL = true; + + CreateAndReopenWithCF({"cf1", "cf2", "cf3"}, options); + + ASSERT_OK(Put(3, Key(1), DummyString(1), wo)); + Flush(3); + ASSERT_OK(Put(3, Key(1), DummyString(1), wo)); + ASSERT_OK(Put(0, Key(1), DummyString(1), wo)); + Flush(0); + + // Write to "Default", "cf2" and "cf3". No flush will be triggered. + ASSERT_OK(Put(3, Key(1), DummyString(30000), wo)); + ASSERT_OK(Put(0, Key(1), DummyString(40000), wo)); + ASSERT_OK(Put(2, Key(1), DummyString(1), wo)); + ASSERT_OK(Put(3, Key(2), DummyString(40000), wo)); + + // WriteBufferManager::buffer_size_ has exceeded after the previous write to + // db_ is completed. + + std::unordered_set w_slowdown_set; + std::vector threads; + int wait_count_db = 0; + int num_writers = 4; + InstrumentedMutex mutex; + InstrumentedCondVar cv(&mutex); + std::atomic thread_num(0); + std::atomic w_no_slowdown(0); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBWriteBufferManagerTest::SharedWriteBufferAcrossCFs:0", + "DBImpl::BackgroundCallFlush:start"}}); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WBMStallInterface::BlockDB", [&](void*) { + { + InstrumentedMutexLock lock(&mutex); + wait_count_db++; + cv.SignalAll(); + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WriteThread::WriteStall::Wait", [&](void* arg) { + { + InstrumentedMutexLock lock(&mutex); + WriteThread::Writer* w = reinterpret_cast(arg); + w_slowdown_set.insert(w); + // Allow the flush continue if all writer threads are blocked. + if (w_slowdown_set.size() + (unsigned long)w_no_slowdown.load( + std::memory_order_relaxed) == + (unsigned long)num_writers) { + TEST_SYNC_POINT( + "DBWriteBufferManagerTest::SharedWriteBufferAcrossCFs:0"); + } + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + bool s1 = true, s2 = true; + + std::function write_slow_down = [&](int cf) { + int a = thread_num.fetch_add(1); + std::string key = "foo" + std::to_string(a); + WriteOptions write_op; + write_op.no_slowdown = false; + Status tmp = Put(cf, Slice(key), DummyString(1), write_op); + InstrumentedMutexLock lock(&mutex); + s1 = s1 && tmp.ok(); + }; + + std::function write_no_slow_down = [&](int cf) { + int a = thread_num.fetch_add(1); + std::string key = "foo" + std::to_string(a); + WriteOptions write_op; + write_op.no_slowdown = true; + Status tmp = Put(cf, Slice(key), DummyString(1), write_op); + { + InstrumentedMutexLock lock(&mutex); + s2 = s2 && !tmp.ok(); + w_no_slowdown.fetch_add(1); + // Allow the flush continue if all writer threads are blocked. + if (w_slowdown_set.size() + + (unsigned long)w_no_slowdown.load(std::memory_order_relaxed) == + (unsigned long)num_writers) { + TEST_SYNC_POINT( + "DBWriteBufferManagerTest::SharedWriteBufferAcrossCFs:0"); + } + } + }; + + // Flow: + // main_writer thread will write but will be blocked (as Flush will on hold, + // buffer_size_ has exceeded, thus will create stall in effect). + // | + // | + // multiple writer threads will be created to write across multiple columns + // with different values of WriteOptions.no_slowdown. Some of them will + // be blocked and some of them will return with Incomplete status. + // | + // | + // Last writer thread will write and when its blocked/return it will signal + // Flush to continue to clear the stall. + threads.emplace_back(write_slow_down, 1); + // Wait untill first thread (main_writer) writing to DB is blocked and then + // create the multiple writers which will be blocked from getting added to the + // queue because stall is in effect. + { + InstrumentedMutexLock lock(&mutex); + while (wait_count_db != 1) { + cv.Wait(); + } + } + + for (int i = 0; i < num_writers; i += 2) { + threads.emplace_back(write_no_slow_down, (i) % 4); + threads.emplace_back(write_slow_down, (i + 1) % 4); + } + for (auto& t : threads) { + t.join(); + } + + ASSERT_TRUE(s1); + ASSERT_TRUE(s2); + // Number of DBs blocked. + ASSERT_EQ(wait_count_db, 1); + // Number of Writer threads blocked. + ASSERT_EQ(w_slowdown_set.size(), num_writers / 2); + // Number of Writer threads with WriteOptions.no_slowdown = true. + ASSERT_EQ(w_no_slowdown.load(std::memory_order_relaxed), num_writers / 2); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +// Test multiple threads writing across multiple columns of db_ and different +// dbs by passing different values to WriteOption.no_slown_down. +TEST_P(DBWriteBufferManagerTest, MixedSlowDownOptionsMultipleDB) { + std::vector dbnames; + std::vector dbs; + int num_dbs = 4; + + for (int i = 0; i < num_dbs; i++) { + dbs.push_back(nullptr); + dbnames.push_back( + test::PerThreadDBPath("db_shared_wb_db" + std::to_string(i))); + } + + Options options = CurrentOptions(); + options.arena_block_size = 4096; + options.write_buffer_size = 500000; // this is never hit + std::shared_ptr cache = NewLRUCache(4 * 1024 * 1024, 2); + ASSERT_LT(cache->GetUsage(), 256 * 1024); + cost_cache_ = GetParam(); + + if (cost_cache_) { + options.write_buffer_manager.reset( + new WriteBufferManager(100000, cache, true)); + } else { + options.write_buffer_manager.reset( + new WriteBufferManager(100000, nullptr, true)); + } + CreateAndReopenWithCF({"cf1", "cf2"}, options); + + for (int i = 0; i < num_dbs; i++) { + ASSERT_OK(DestroyDB(dbnames[i], options)); + ASSERT_OK(DB::Open(options, dbnames[i], &(dbs[i]))); + } + WriteOptions wo; + wo.disableWAL = true; + + for (int i = 0; i < num_dbs; i++) { + ASSERT_OK(dbs[i]->Put(wo, Key(1), DummyString(20000))); + } + // Insert to db_. + ASSERT_OK(Put(0, Key(1), DummyString(30000), wo)); + + // WriteBufferManager::buffer_size_ has exceeded after the previous write to + // dbs[0] is completed. + std::vector threads; + int wait_count_db = 0; + InstrumentedMutex mutex; + InstrumentedCondVar cv(&mutex); + std::unordered_set w_slowdown_set; + std::vector writer_threads; + std::atomic thread_num(0); + std::atomic w_no_slowdown(0); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBWriteBufferManagerTest::SharedWriteBufferAcrossCFs:0", + "DBImpl::BackgroundCallFlush:start"}}); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WBMStallInterface::BlockDB", [&](void*) { + InstrumentedMutexLock lock(&mutex); + wait_count_db++; + cv.Signal(); + // Allow the flush continue if all writer threads are blocked. + if (w_slowdown_set.size() + + (unsigned long)(w_no_slowdown.load(std::memory_order_relaxed) + + wait_count_db) == + (unsigned long)(2 * num_dbs + 1)) { + TEST_SYNC_POINT( + "DBWriteBufferManagerTest::SharedWriteBufferAcrossCFs:0"); + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WriteThread::WriteStall::Wait", [&](void* arg) { + WriteThread::Writer* w = reinterpret_cast(arg); + InstrumentedMutexLock lock(&mutex); + w_slowdown_set.insert(w); + // Allow the flush continue if all writer threads are blocked. + if (w_slowdown_set.size() + + (unsigned long)(w_no_slowdown.load(std::memory_order_relaxed) + + wait_count_db) == + (unsigned long)(2 * num_dbs + 1)) { + TEST_SYNC_POINT( + "DBWriteBufferManagerTest::SharedWriteBufferAcrossCFs:0"); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + bool s1 = true, s2 = true; + std::function write_slow_down = [&](DB* db) { + int a = thread_num.fetch_add(1); + std::string key = "foo" + std::to_string(a); + WriteOptions write_op; + write_op.no_slowdown = false; + Status tmp = db->Put(write_op, Slice(key), DummyString(1)); + InstrumentedMutexLock lock(&mutex); + s1 = s1 && tmp.ok(); + }; + + std::function write_no_slow_down = [&](DB* db) { + int a = thread_num.fetch_add(1); + std::string key = "foo" + std::to_string(a); + WriteOptions write_op; + write_op.no_slowdown = true; + Status tmp = db->Put(write_op, Slice(key), DummyString(1)); + { + InstrumentedMutexLock lock(&mutex); + s2 = s2 && !tmp.ok(); + w_no_slowdown.fetch_add(1); + if (w_slowdown_set.size() + + (unsigned long)(w_no_slowdown.load(std::memory_order_relaxed) + + wait_count_db) == + (unsigned long)(2 * num_dbs + 1)) { + TEST_SYNC_POINT( + "DBWriteBufferManagerTest::SharedWriteBufferAcrossCFs:0"); + } + } + }; + + // Flow: + // first thread will write but will be blocked (as Flush will on hold, + // buffer_size_ has exceeded, thus will create stall in effect). + // | + // | + // multiple writer threads will be created to write across multiple columns + // of db_ and different DBs with different values of + // WriteOptions.no_slowdown. Some of them will be blocked and some of them + // will return with Incomplete status. + // | + // | + // Last writer thread will write and when its blocked/return it will signal + // Flush to continue to clear the stall. + threads.emplace_back(write_slow_down, db_); + // Wait untill first thread writing to DB is blocked and then + // create the multiple writers. + { + InstrumentedMutexLock lock(&mutex); + while (wait_count_db != 1) { + cv.Wait(); + } + } + + for (int i = 0; i < num_dbs; i += 2) { + // Write to multiple columns of db_. + writer_threads.emplace_back(write_slow_down, db_); + writer_threads.emplace_back(write_no_slow_down, db_); + // Write to different DBs. + threads.emplace_back(write_slow_down, dbs[i]); + threads.emplace_back(write_no_slow_down, dbs[i + 1]); + } + + for (auto& t : threads) { + t.join(); + } + + for (auto& t : writer_threads) { + t.join(); + } + + ASSERT_TRUE(s1); + ASSERT_TRUE(s2); + // Number of DBs blocked. + ASSERT_EQ((num_dbs / 2) + 1, wait_count_db); + // Number of writer threads writing to db_ blocked from getting added to the + // queue. + ASSERT_EQ(w_slowdown_set.size(), num_dbs / 2); + // Number of threads with WriteOptions.no_slowdown = true. + ASSERT_EQ(w_no_slowdown.load(std::memory_order_relaxed), num_dbs); + + // Clean up DBs. + for (int i = 0; i < num_dbs; i++) { + ASSERT_OK(dbs[i]->Close()); + ASSERT_OK(DestroyDB(dbnames[i], options)); + delete dbs[i]; + } + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + + +// Tests a `WriteBufferManager` constructed with `allow_stall == false` does not +// thrash memtable switching when full and a CF receives multiple writes. +// Instead, we expect to switch a CF's memtable for flush only when that CF does +// not have any pending or running flush. +// +// This test uses multiple DBs each with a single CF instead of a single DB +// with multiple CFs. That way we can control which CF is considered for switch +// by writing to that CF's DB. +// +// Not supported in LITE mode due to `GetProperty()` unavailable. +TEST_P(DBWriteBufferManagerTest, StopSwitchingMemTablesOnceFlushing) { + Options options = CurrentOptions(); + options.arena_block_size = 4 << 10; // 4KB + options.write_buffer_size = 1 << 20; // 1MB + std::shared_ptr cache = + NewLRUCache(4 << 20 /* capacity (4MB) */, 2 /* num_shard_bits */); + ASSERT_LT(cache->GetUsage(), 256 << 10 /* 256KB */); + cost_cache_ = GetParam(); + if (cost_cache_) { + options.write_buffer_manager.reset(new WriteBufferManager( + 512 << 10 /* buffer_size (512KB) */, cache, false /* allow_stall */)); + } else { + options.write_buffer_manager.reset( + new WriteBufferManager(512 << 10 /* buffer_size (512KB) */, + nullptr /* cache */, false /* allow_stall */)); + } + + Reopen(options); + std::string dbname = test::PerThreadDBPath("db_shared_wbm_db"); + DB* shared_wbm_db = nullptr; + + ASSERT_OK(DestroyDB(dbname, options)); + ASSERT_OK(DB::Open(options, dbname, &shared_wbm_db)); + + // The last write will make WBM need flush, but it won't flush yet. + ASSERT_OK(Put(Key(1), DummyString(256 << 10 /* 256KB */), WriteOptions())); + ASSERT_FALSE(options.write_buffer_manager->ShouldFlush()); + ASSERT_OK(Put(Key(1), DummyString(256 << 10 /* 256KB */), WriteOptions())); + ASSERT_TRUE(options.write_buffer_manager->ShouldFlush()); + + // Flushes will be pending, not running because flush threads are blocked. + test::SleepingBackgroundTask sleeping_task_high; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, + &sleeping_task_high, Env::Priority::HIGH); + + for (int i = 0; i < 3; ++i) { + ASSERT_OK( + shared_wbm_db->Put(WriteOptions(), Key(1), DummyString(1 /* len */))); + std::string prop; + ASSERT_TRUE( + shared_wbm_db->GetProperty("rocksdb.num-immutable-mem-table", &prop)); + ASSERT_EQ(std::to_string(i > 0 ? 1 : 0), prop); + ASSERT_TRUE( + shared_wbm_db->GetProperty("rocksdb.mem-table-flush-pending", &prop)); + ASSERT_EQ(std::to_string(i > 0 ? 1 : 0), prop); + } + + // Clean up DBs. + sleeping_task_high.WakeUp(); + sleeping_task_high.WaitUntilDone(); + ASSERT_OK(shared_wbm_db->Close()); + ASSERT_OK(DestroyDB(dbname, options)); + delete shared_wbm_db; +} + +TEST_F(DBWriteBufferManagerTest, RuntimeChangeableAllowStall) { + constexpr int kBigValue = 10000; + + Options options = CurrentOptions(); + options.write_buffer_manager.reset( + new WriteBufferManager(1, nullptr /* cache */, true /* allow_stall */)); + DestroyAndReopen(options); + + // Pause flush thread so that + // (a) the only way to exist write stall below is to change the `allow_stall` + // (b) the write stall is "stable" without being interfered by flushes so that + // we can check it without flakiness + std::unique_ptr sleeping_task( + new test::SleepingBackgroundTask()); + env_->SetBackgroundThreads(1, Env::HIGH); + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, + sleeping_task.get(), Env::Priority::HIGH); + sleeping_task->WaitUntilSleeping(); + + // Test 1: test setting `allow_stall` from true to false + // + // Assert existence of a write stall + WriteOptions wo_no_slowdown; + wo_no_slowdown.no_slowdown = true; + Status s = Put(Key(0), DummyString(kBigValue), wo_no_slowdown); + ASSERT_TRUE(s.IsIncomplete()); + ASSERT_TRUE(s.ToString().find("Write stall") != std::string::npos); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"WBMStallInterface::BlockDB", + "DBWriteBufferManagerTest::RuntimeChangeableThreadSafeParameters::" + "ChangeParameter"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Test `SetAllowStall()` + port::Thread thread1([&] { ASSERT_OK(Put(Key(0), DummyString(kBigValue))); }); + port::Thread thread2([&] { + TEST_SYNC_POINT( + "DBWriteBufferManagerTest::RuntimeChangeableThreadSafeParameters::" + "ChangeParameter"); + options.write_buffer_manager->SetAllowStall(false); + }); + + // Verify `allow_stall` is successfully set to false in thread2. + // Othwerwise, thread1's write will be stalled and this test will hang + // forever. + thread1.join(); + thread2.join(); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + // Test 2: test setting `allow_stall` from false to true + // + // Assert no write stall + ASSERT_OK(Put(Key(0), DummyString(kBigValue), wo_no_slowdown)); + + // Test `SetAllowStall()` + options.write_buffer_manager->SetAllowStall(true); + + // Verify `allow_stall` is successfully set to true. + // Otherwise the following write will not be stalled and therefore succeed. + s = Put(Key(0), DummyString(kBigValue), wo_no_slowdown); + ASSERT_TRUE(s.IsIncomplete()); + ASSERT_TRUE(s.ToString().find("Write stall") != std::string::npos); + sleeping_task->WakeUp(); +} + +INSTANTIATE_TEST_CASE_P(DBWriteBufferManagerTest, DBWriteBufferManagerTest, + testing::Bool()); + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/db_write_test.cc b/librocksdb-sys/rocksdb/db/db_write_test.cc new file mode 100644 index 0000000..d1e3d53 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/db_write_test.cc @@ -0,0 +1,795 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include +#include +#include +#include +#include +#include + +#include "db/db_test_util.h" +#include "db/write_batch_internal.h" +#include "db/write_thread.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "test_util/sync_point.h" +#include "util/random.h" +#include "util/string_util.h" +#include "utilities/fault_injection_env.h" +#include "utilities/fault_injection_fs.h" + +namespace ROCKSDB_NAMESPACE { + +// Test variations of WriteImpl. +class DBWriteTest : public DBTestBase, public testing::WithParamInterface { + public: + DBWriteTest() : DBTestBase("db_write_test", /*env_do_fsync=*/true) {} + + Options GetOptions() { return DBTestBase::GetOptions(GetParam()); } + + void Open() { DBTestBase::Reopen(GetOptions()); } +}; + +class DBWriteTestUnparameterized : public DBTestBase { + public: + explicit DBWriteTestUnparameterized() + : DBTestBase("pipelined_write_test", /*env_do_fsync=*/false) {} +}; + +// It is invalid to do sync write while disabling WAL. +TEST_P(DBWriteTest, SyncAndDisableWAL) { + WriteOptions write_options; + write_options.sync = true; + write_options.disableWAL = true; + ASSERT_TRUE(dbfull()->Put(write_options, "foo", "bar").IsInvalidArgument()); + WriteBatch batch; + ASSERT_OK(batch.Put("foo", "bar")); + ASSERT_TRUE(dbfull()->Write(write_options, &batch).IsInvalidArgument()); +} + +TEST_P(DBWriteTest, WriteStallRemoveNoSlowdownWrite) { + Options options = GetOptions(); + options.level0_stop_writes_trigger = options.level0_slowdown_writes_trigger = + 4; + std::vector threads; + std::atomic thread_num(0); + port::Mutex mutex; + port::CondVar cv(&mutex); + // Guarded by mutex + int writers = 0; + + Reopen(options); + + std::function write_slowdown_func = [&]() { + int a = thread_num.fetch_add(1); + std::string key = "foo" + std::to_string(a); + WriteOptions wo; + wo.no_slowdown = false; + ASSERT_OK(dbfull()->Put(wo, key, "bar")); + }; + std::function write_no_slowdown_func = [&]() { + int a = thread_num.fetch_add(1); + std::string key = "foo" + std::to_string(a); + WriteOptions wo; + wo.no_slowdown = true; + Status s = dbfull()->Put(wo, key, "bar"); + ASSERT_TRUE(s.ok() || s.IsIncomplete()); + }; + std::function unblock_main_thread_func = [&](void*) { + mutex.Lock(); + ++writers; + cv.SignalAll(); + mutex.Unlock(); + }; + + // Create 3 L0 files and schedule 4th without waiting + ASSERT_OK(Put("foo" + std::to_string(thread_num.fetch_add(1)), "bar")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("foo" + std::to_string(thread_num.fetch_add(1)), "bar")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("foo" + std::to_string(thread_num.fetch_add(1)), "bar")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("foo" + std::to_string(thread_num.fetch_add(1)), "bar")); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WriteThread::JoinBatchGroup:Start", unblock_main_thread_func); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBWriteTest::WriteStallRemoveNoSlowdownWrite:1", + "DBImpl::BackgroundCallFlush:start"}, + {"DBWriteTest::WriteStallRemoveNoSlowdownWrite:2", + "DBImplWrite::PipelinedWriteImpl:AfterJoinBatchGroup"}, + // Make compaction start wait for the write stall to be detected and + // implemented by a write group leader + {"DBWriteTest::WriteStallRemoveNoSlowdownWrite:3", + "BackgroundCallCompaction:0"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Schedule creation of 4th L0 file without waiting. This will seal the + // memtable and then wait for a sync point before writing the file. We need + // to do it this way because SwitchMemtable() needs to enter the + // write_thread + FlushOptions fopt; + fopt.wait = false; + ASSERT_OK(dbfull()->Flush(fopt)); + + // Create a mix of slowdown/no_slowdown write threads + mutex.Lock(); + // First leader + threads.emplace_back(write_slowdown_func); + while (writers != 1) { + cv.Wait(); + } + + // Second leader. Will stall writes + // Build a writers list with no slowdown in the middle: + // +-------------+ + // | slowdown +<----+ newest + // +--+----------+ + // | + // v + // +--+----------+ + // | no slowdown | + // +--+----------+ + // | + // v + // +--+----------+ + // | slowdown + + // +-------------+ + threads.emplace_back(write_slowdown_func); + while (writers != 2) { + cv.Wait(); + } + threads.emplace_back(write_no_slowdown_func); + while (writers != 3) { + cv.Wait(); + } + threads.emplace_back(write_slowdown_func); + while (writers != 4) { + cv.Wait(); + } + + mutex.Unlock(); + + TEST_SYNC_POINT("DBWriteTest::WriteStallRemoveNoSlowdownWrite:1"); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(nullptr)); + // This would have triggered a write stall. Unblock the write group leader + TEST_SYNC_POINT("DBWriteTest::WriteStallRemoveNoSlowdownWrite:2"); + // The leader is going to create missing newer links. When the leader + // finishes, the next leader is going to delay writes and fail writers with + // no_slowdown + + TEST_SYNC_POINT("DBWriteTest::WriteStallRemoveNoSlowdownWrite:3"); + for (auto& t : threads) { + t.join(); + } + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_P(DBWriteTest, WriteThreadHangOnWriteStall) { + Options options = GetOptions(); + options.level0_stop_writes_trigger = options.level0_slowdown_writes_trigger = + 4; + std::vector threads; + std::atomic thread_num(0); + port::Mutex mutex; + port::CondVar cv(&mutex); + // Guarded by mutex + int writers = 0; + + Reopen(options); + + std::function write_slowdown_func = [&]() { + int a = thread_num.fetch_add(1); + std::string key = "foo" + std::to_string(a); + WriteOptions wo; + wo.no_slowdown = false; + ASSERT_OK(dbfull()->Put(wo, key, "bar")); + }; + std::function write_no_slowdown_func = [&]() { + int a = thread_num.fetch_add(1); + std::string key = "foo" + std::to_string(a); + WriteOptions wo; + wo.no_slowdown = true; + Status s = dbfull()->Put(wo, key, "bar"); + ASSERT_TRUE(s.ok() || s.IsIncomplete()); + }; + std::function unblock_main_thread_func = [&](void*) { + mutex.Lock(); + ++writers; + cv.SignalAll(); + mutex.Unlock(); + }; + + // Create 3 L0 files and schedule 4th without waiting + ASSERT_OK(Put("foo" + std::to_string(thread_num.fetch_add(1)), "bar")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("foo" + std::to_string(thread_num.fetch_add(1)), "bar")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("foo" + std::to_string(thread_num.fetch_add(1)), "bar")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("foo" + std::to_string(thread_num.fetch_add(1)), "bar")); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WriteThread::JoinBatchGroup:Start", unblock_main_thread_func); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBWriteTest::WriteThreadHangOnWriteStall:1", + "DBImpl::BackgroundCallFlush:start"}, + {"DBWriteTest::WriteThreadHangOnWriteStall:2", + "DBImpl::WriteImpl:BeforeLeaderEnters"}, + // Make compaction start wait for the write stall to be detected and + // implemented by a write group leader + {"DBWriteTest::WriteThreadHangOnWriteStall:3", + "BackgroundCallCompaction:0"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Schedule creation of 4th L0 file without waiting. This will seal the + // memtable and then wait for a sync point before writing the file. We need + // to do it this way because SwitchMemtable() needs to enter the + // write_thread + FlushOptions fopt; + fopt.wait = false; + ASSERT_OK(dbfull()->Flush(fopt)); + + // Create a mix of slowdown/no_slowdown write threads + mutex.Lock(); + // First leader + threads.emplace_back(write_slowdown_func); + while (writers != 1) { + cv.Wait(); + } + // Second leader. Will stall writes + threads.emplace_back(write_slowdown_func); + threads.emplace_back(write_no_slowdown_func); + threads.emplace_back(write_slowdown_func); + threads.emplace_back(write_no_slowdown_func); + threads.emplace_back(write_slowdown_func); + while (writers != 6) { + cv.Wait(); + } + mutex.Unlock(); + + TEST_SYNC_POINT("DBWriteTest::WriteThreadHangOnWriteStall:1"); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(nullptr)); + // This would have triggered a write stall. Unblock the write group leader + TEST_SYNC_POINT("DBWriteTest::WriteThreadHangOnWriteStall:2"); + // The leader is going to create missing newer links. When the leader + // finishes, the next leader is going to delay writes and fail writers with + // no_slowdown + + TEST_SYNC_POINT("DBWriteTest::WriteThreadHangOnWriteStall:3"); + for (auto& t : threads) { + t.join(); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_P(DBWriteTest, IOErrorOnWALWritePropagateToWriteThreadFollower) { + constexpr int kNumThreads = 5; + std::unique_ptr mock_env( + new FaultInjectionTestEnv(env_)); + Options options = GetOptions(); + options.env = mock_env.get(); + Reopen(options); + std::atomic ready_count{0}; + std::atomic leader_count{0}; + std::vector threads; + mock_env->SetFilesystemActive(false); + + // Wait until all threads linked to write threads, to make sure + // all threads join the same batch group. + SyncPoint::GetInstance()->SetCallBack( + "WriteThread::JoinBatchGroup:Wait", [&](void* arg) { + ready_count++; + auto* w = reinterpret_cast(arg); + if (w->state == WriteThread::STATE_GROUP_LEADER) { + leader_count++; + while (ready_count < kNumThreads) { + // busy waiting + } + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + for (int i = 0; i < kNumThreads; i++) { + threads.push_back(port::Thread( + [&](int index) { + // All threads should fail. + auto res = Put("key" + std::to_string(index), "value"); + if (options.manual_wal_flush) { + ASSERT_TRUE(res.ok()); + // we should see fs error when we do the flush + + // TSAN reports a false alarm for lock-order-inversion but Open and + // FlushWAL are not run concurrently. Disabling this until TSAN is + // fixed. + // res = dbfull()->FlushWAL(false); + // ASSERT_FALSE(res.ok()); + } else { + ASSERT_FALSE(res.ok()); + } + }, + i)); + } + for (int i = 0; i < kNumThreads; i++) { + threads[i].join(); + } + ASSERT_EQ(1, leader_count); + + // The Failed PUT operations can cause a BG error to be set. + // Mark it as Checked for the ASSERT_STATUS_CHECKED + dbfull()->Resume().PermitUncheckedError(); + + // Close before mock_env destruct. + Close(); +} + +TEST_F(DBWriteTestUnparameterized, PipelinedWriteRace) { + // This test was written to trigger a race in ExitAsBatchGroupLeader in case + // enable_pipelined_write_ was true. + // Writers for which ShouldWriteToMemtable() evaluates to false are removed + // from the write_group via CompleteFollower/ CompleteLeader. Writers in the + // middle of the group are fully unlinked, but if that writers is the + // last_writer, then we did not update the predecessor's link_older, i.e., + // this writer was still reachable via newest_writer_. + // + // But the problem was, that CompleteFollower already wakes up the thread + // owning that writer before the writer has been removed. This resulted in a + // race - if the leader thread was fast enough, then everything was fine. + // However, if the woken up thread finished the current write operation and + // then performed yet another write, then a new writer instance was added + // to newest_writer_. It is possible that the new writer is located on the + // same address on stack, and if this happened, then we had a problem, + // because the old code tried to find the last_writer in the list to unlink + // it, which in this case produced a cycle in the list. + // Whether two invocations of PipelinedWriteImpl() by the same thread actually + // allocate the writer on the same address depends on the OS and/or compiler, + // so it is rather hard to create a deterministic test for this. + + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_pipelined_write = true; + std::vector threads; + + std::atomic write_counter{0}; + std::atomic active_writers{0}; + std::atomic second_write_starting{false}; + std::atomic second_write_in_progress{false}; + std::atomic leader{nullptr}; + std::atomic finished_WAL_write{false}; + + DestroyAndReopen(options); + + auto write_one_doc = [&]() { + int a = write_counter.fetch_add(1); + std::string key = "foo" + std::to_string(a); + WriteOptions wo; + ASSERT_OK(dbfull()->Put(wo, key, "bar")); + --active_writers; + }; + + auto write_two_docs = [&]() { + write_one_doc(); + second_write_starting = true; + write_one_doc(); + }; + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WriteThread::JoinBatchGroup:Wait", [&](void* arg) { + if (second_write_starting.load()) { + second_write_in_progress = true; + return; + } + auto* w = reinterpret_cast(arg); + if (w->state == WriteThread::STATE_GROUP_LEADER) { + active_writers++; + if (leader.load() == nullptr) { + leader.store(w); + while (active_writers.load() < 2) { + // wait for another thread to join the write_group + } + } + } else { + // we disable the memtable for all followers so that they they are + // removed from the write_group before enqueuing it for the memtable + // write + w->disable_memtable = true; + active_writers++; + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WriteThread::ExitAsBatchGroupLeader:Start", [&](void* arg) { + auto* wg = reinterpret_cast(arg); + if (wg->leader == leader && !finished_WAL_write) { + finished_WAL_write = true; + while (active_writers.load() < 3) { + // wait for the new writer to be enqueued + } + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WriteThread::ExitAsBatchGroupLeader:AfterCompleteWriters", + [&](void* arg) { + auto* wg = reinterpret_cast(arg); + if (wg->leader == leader) { + while (!second_write_in_progress.load()) { + // wait for the old follower thread to start the next write + } + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // start leader + one follower + threads.emplace_back(write_one_doc); + while (leader.load() == nullptr) { + // wait for leader + } + + // we perform two writes in the follower, so that for the second write + // the thread reinserts a Writer with the same address + threads.emplace_back(write_two_docs); + + // wait for the leader to enter ExitAsBatchGroupLeader + while (!finished_WAL_write.load()) { + // wait for write_group to have finished the WAL writes + } + + // start another writer thread to be enqueued before the leader can + // complete the writers from its write_group + threads.emplace_back(write_one_doc); + + for (auto& t : threads) { + t.join(); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_P(DBWriteTest, ManualWalFlushInEffect) { + Options options = GetOptions(); + Reopen(options); + // try the 1st WAL created during open + ASSERT_TRUE(Put("key" + std::to_string(0), "value").ok()); + ASSERT_TRUE(options.manual_wal_flush != dbfull()->WALBufferIsEmpty()); + ASSERT_TRUE(dbfull()->FlushWAL(false).ok()); + ASSERT_TRUE(dbfull()->WALBufferIsEmpty()); + // try the 2nd wal created during SwitchWAL + ASSERT_OK(dbfull()->TEST_SwitchWAL()); + ASSERT_TRUE(Put("key" + std::to_string(0), "value").ok()); + ASSERT_TRUE(options.manual_wal_flush != dbfull()->WALBufferIsEmpty()); + ASSERT_TRUE(dbfull()->FlushWAL(false).ok()); + ASSERT_TRUE(dbfull()->WALBufferIsEmpty()); +} + +TEST_P(DBWriteTest, UnflushedPutRaceWithTrackedWalSync) { + // Repro race condition bug where unflushed WAL data extended the synced size + // recorded to MANIFEST despite being unrecoverable. + Options options = GetOptions(); + std::unique_ptr fault_env( + new FaultInjectionTestEnv(env_)); + options.env = fault_env.get(); + options.manual_wal_flush = true; + options.track_and_verify_wals_in_manifest = true; + Reopen(options); + + ASSERT_OK(Put("key1", "val1")); + + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::SyncWAL:Begin", + [this](void* /* arg */) { ASSERT_OK(Put("key2", "val2")); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(db_->FlushWAL(true /* sync */)); + + // Ensure callback ran. + ASSERT_EQ("val2", Get("key2")); + + Close(); + + // Simulate full loss of unsynced data. This drops "key2" -> "val2" from the + // DB WAL. + fault_env->DropUnsyncedFileData(); + + Reopen(options); + + // Need to close before `fault_env` goes out of scope. + Close(); +} + +TEST_P(DBWriteTest, InactiveWalFullySyncedBeforeUntracked) { + // Repro bug where a WAL is appended and switched after + // `FlushWAL(true /* sync */)`'s sync finishes and before it untracks fully + // synced inactive logs. Previously such a WAL would be wrongly untracked + // so the final append would never be synced. + Options options = GetOptions(); + std::unique_ptr fault_env( + new FaultInjectionTestEnv(env_)); + options.env = fault_env.get(); + Reopen(options); + + ASSERT_OK(Put("key1", "val1")); + + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::SyncWAL:BeforeMarkLogsSynced:1", [this](void* /* arg */) { + ASSERT_OK(Put("key2", "val2")); + ASSERT_OK(dbfull()->TEST_SwitchMemtable()); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(db_->FlushWAL(true /* sync */)); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + + ASSERT_OK(Put("key3", "val3")); + + ASSERT_OK(db_->FlushWAL(true /* sync */)); + + Close(); + + // Simulate full loss of unsynced data. This should drop nothing since we did + // `FlushWAL(true /* sync */)` before `Close()`. + fault_env->DropUnsyncedFileData(); + + Reopen(options); + + ASSERT_EQ("val1", Get("key1")); + ASSERT_EQ("val2", Get("key2")); + ASSERT_EQ("val3", Get("key3")); + + // Need to close before `fault_env` goes out of scope. + Close(); +} + +TEST_P(DBWriteTest, IOErrorOnWALWriteTriggersReadOnlyMode) { + std::unique_ptr mock_env( + new FaultInjectionTestEnv(env_)); + Options options = GetOptions(); + options.env = mock_env.get(); + Reopen(options); + for (int i = 0; i < 2; i++) { + // Forcibly fail WAL write for the first Put only. Subsequent Puts should + // fail due to read-only mode + mock_env->SetFilesystemActive(i != 0); + auto res = Put("key" + std::to_string(i), "value"); + // TSAN reports a false alarm for lock-order-inversion but Open and + // FlushWAL are not run concurrently. Disabling this until TSAN is + // fixed. + /* + if (options.manual_wal_flush && i == 0) { + // even with manual_wal_flush the 2nd Put should return error because of + // the read-only mode + ASSERT_TRUE(res.ok()); + // we should see fs error when we do the flush + res = dbfull()->FlushWAL(false); + } + */ + if (!options.manual_wal_flush) { + ASSERT_NOK(res); + } else { + ASSERT_OK(res); + } + } + // Close before mock_env destruct. + Close(); +} + +TEST_P(DBWriteTest, IOErrorOnSwitchMemtable) { + Random rnd(301); + std::unique_ptr mock_env( + new FaultInjectionTestEnv(env_)); + Options options = GetOptions(); + options.env = mock_env.get(); + options.writable_file_max_buffer_size = 4 * 1024 * 1024; + options.write_buffer_size = 3 * 512 * 1024; + options.wal_bytes_per_sync = 256 * 1024; + options.manual_wal_flush = true; + Reopen(options); + mock_env->SetFilesystemActive(false, Status::IOError("Not active")); + Status s; + for (int i = 0; i < 4 * 512; ++i) { + s = Put(Key(i), rnd.RandomString(1024)); + if (!s.ok()) { + break; + } + } + ASSERT_EQ(s.severity(), Status::Severity::kFatalError); + + mock_env->SetFilesystemActive(true); + // Close before mock_env destruct. + Close(); +} + +// Test that db->LockWAL() flushes the WAL after locking, which can fail +TEST_P(DBWriteTest, LockWALInEffect) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_SKIP("Test requires non-mem or non-encrypted environment"); + return; + } + Options options = GetOptions(); + std::shared_ptr fault_fs( + new FaultInjectionTestFS(FileSystem::Default())); + std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); + options.env = fault_fs_env.get(); + options.disable_auto_compactions = true; + options.paranoid_checks = false; + options.max_bgerror_resume_count = 0; // manual Resume() + Reopen(options); + // try the 1st WAL created during open + ASSERT_OK(Put("key0", "value")); + ASSERT_NE(options.manual_wal_flush, dbfull()->WALBufferIsEmpty()); + ASSERT_OK(db_->LockWAL()); + ASSERT_TRUE(dbfull()->WALBufferIsEmpty()); + ASSERT_OK(db_->UnlockWAL()); + // try the 2nd wal created during SwitchWAL + ASSERT_OK(dbfull()->TEST_SwitchWAL()); + ASSERT_OK(Put("key1", "value")); + ASSERT_NE(options.manual_wal_flush, dbfull()->WALBufferIsEmpty()); + ASSERT_OK(db_->LockWAL()); + ASSERT_TRUE(dbfull()->WALBufferIsEmpty()); + ASSERT_OK(db_->UnlockWAL()); + + // The above `TEST_SwitchWAL()` triggered a flush. That flush needs to finish + // before we make the filesystem inactive, otherwise the flush might hit an + // unrecoverable error (e.g., failed MANIFEST update). + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(nullptr)); + + // Fail the WAL flush if applicable + fault_fs->SetFilesystemActive(false); + Status s = Put("key2", "value"); + if (options.manual_wal_flush) { + ASSERT_OK(s); + // I/O failure + ASSERT_NOK(db_->LockWAL()); + // Should not need UnlockWAL after LockWAL fails + } else { + ASSERT_NOK(s); + ASSERT_OK(db_->LockWAL()); + ASSERT_OK(db_->UnlockWAL()); + } + fault_fs->SetFilesystemActive(true); + ASSERT_OK(db_->Resume()); + // Writes should work again + ASSERT_OK(Put("key3", "value")); + ASSERT_EQ(Get("key3"), "value"); + + // Should be extraneous, but allowed + ASSERT_NOK(db_->UnlockWAL()); + + // Close before mock_env destruct. + Close(); +} + +TEST_P(DBWriteTest, LockWALConcurrentRecursive) { + Options options = GetOptions(); + Reopen(options); + ASSERT_OK(Put("k1", "val")); + ASSERT_OK(db_->LockWAL()); // 0 -> 1 + auto frozen_seqno = db_->GetLatestSequenceNumber(); + std::atomic t1_completed{false}; + port::Thread t1{[&]() { + // Won't finish until WAL unlocked + ASSERT_OK(Put("k1", "val2")); + t1_completed = true; + }}; + + ASSERT_OK(db_->LockWAL()); // 1 -> 2 + // Read-only ops are OK + ASSERT_EQ(Get("k1"), "val"); + { + std::vector files; + LiveFilesStorageInfoOptions lf_opts; + // A DB flush could deadlock + lf_opts.wal_size_for_flush = UINT64_MAX; + ASSERT_OK(db_->GetLiveFilesStorageInfo({lf_opts}, &files)); + } + + port::Thread t2{[&]() { + ASSERT_OK(db_->LockWAL()); // 2 -> 3 or 1 -> 2 + }}; + + ASSERT_OK(db_->UnlockWAL()); // 2 -> 1 or 3 -> 2 + // Give t1 an extra chance to jump in case of bug + std::this_thread::yield(); + t2.join(); + ASSERT_FALSE(t1_completed.load()); + + // Should now have 2 outstanding LockWAL + ASSERT_EQ(Get("k1"), "val"); + + ASSERT_OK(db_->UnlockWAL()); // 2 -> 1 + + ASSERT_FALSE(t1_completed.load()); + ASSERT_EQ(Get("k1"), "val"); + ASSERT_EQ(frozen_seqno, db_->GetLatestSequenceNumber()); + + // Ensure final Unlock is concurrency safe and extra Unlock is safe but + // non-OK + std::atomic unlock_ok{0}; + port::Thread t3{[&]() { + if (db_->UnlockWAL().ok()) { + unlock_ok++; + } + ASSERT_OK(db_->LockWAL()); + if (db_->UnlockWAL().ok()) { + unlock_ok++; + } + }}; + + if (db_->UnlockWAL().ok()) { + unlock_ok++; + } + t3.join(); + + // There was one extra unlock, so just one non-ok + ASSERT_EQ(unlock_ok.load(), 2); + + // Write can proceed + t1.join(); + ASSERT_TRUE(t1_completed.load()); + ASSERT_EQ(Get("k1"), "val2"); + // And new writes + ASSERT_OK(Put("k2", "val")); + ASSERT_EQ(Get("k2"), "val"); +} + +TEST_P(DBWriteTest, ConcurrentlyDisabledWAL) { + Options options = GetOptions(); + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + options.statistics->set_stats_level(StatsLevel::kAll); + Reopen(options); + std::string wal_key_prefix = "WAL_KEY_"; + std::string no_wal_key_prefix = "K_"; + // 100 KB value each for NO-WAL operation + std::string no_wal_value(1024 * 100, 'X'); + // 1B value each for WAL operation + std::string wal_value = "0"; + std::thread threads[10]; + for (int t = 0; t < 10; t++) { + threads[t] = std::thread([t, wal_key_prefix, wal_value, no_wal_key_prefix, + no_wal_value, this] { + for (int i = 0; i < 10; i++) { + ROCKSDB_NAMESPACE::WriteOptions write_option_disable; + write_option_disable.disableWAL = true; + ROCKSDB_NAMESPACE::WriteOptions write_option_default; + std::string no_wal_key = + no_wal_key_prefix + std::to_string(t) + "_" + std::to_string(i); + ASSERT_OK(this->Put(no_wal_key, no_wal_value, write_option_disable)); + std::string wal_key = + wal_key_prefix + std::to_string(i) + "_" + std::to_string(i); + ASSERT_OK(this->Put(wal_key, wal_value, write_option_default)); + ASSERT_OK(dbfull()->SyncWAL()); + } + return; + }); + } + for (auto& t : threads) { + t.join(); + } + uint64_t bytes_num = options.statistics->getTickerCount( + ROCKSDB_NAMESPACE::Tickers::WAL_FILE_BYTES); + // written WAL size should less than 100KB (even included HEADER & FOOTER + // overhead) + ASSERT_LE(bytes_num, 1024 * 100); +} + +INSTANTIATE_TEST_CASE_P(DBWriteTestInstance, DBWriteTest, + testing::Values(DBTestBase::kDefault, + DBTestBase::kConcurrentWALWrites, + DBTestBase::kPipelinedWrite)); + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/dbformat.cc b/librocksdb-sys/rocksdb/db/dbformat.cc new file mode 100644 index 0000000..2d24c89 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/dbformat.cc @@ -0,0 +1,242 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "db/dbformat.h" + +#include + +#include + +#include "db/lookup_key.h" +#include "monitoring/perf_context_imp.h" +#include "port/port.h" +#include "util/coding.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +// kValueTypeForSeek defines the ValueType that should be passed when +// constructing a ParsedInternalKey object for seeking to a particular +// sequence number (since we sort sequence numbers in decreasing order +// and the value type is embedded as the low 8 bits in the sequence +// number in internal keys, we need to use the highest-numbered +// ValueType, not the lowest). +const ValueType kValueTypeForSeek = kTypeWideColumnEntity; +const ValueType kValueTypeForSeekForPrev = kTypeDeletion; +const std::string kDisableUserTimestamp(""); + +EntryType GetEntryType(ValueType value_type) { + switch (value_type) { + case kTypeValue: + return kEntryPut; + case kTypeDeletion: + return kEntryDelete; + case kTypeDeletionWithTimestamp: + return kEntryDeleteWithTimestamp; + case kTypeSingleDeletion: + return kEntrySingleDelete; + case kTypeMerge: + return kEntryMerge; + case kTypeRangeDeletion: + return kEntryRangeDeletion; + case kTypeBlobIndex: + return kEntryBlobIndex; + case kTypeWideColumnEntity: + return kEntryWideColumnEntity; + default: + return kEntryOther; + } +} + +void AppendInternalKey(std::string* result, const ParsedInternalKey& key) { + result->append(key.user_key.data(), key.user_key.size()); + PutFixed64(result, PackSequenceAndType(key.sequence, key.type)); +} + +void AppendInternalKeyWithDifferentTimestamp(std::string* result, + const ParsedInternalKey& key, + const Slice& ts) { + assert(key.user_key.size() >= ts.size()); + result->append(key.user_key.data(), key.user_key.size() - ts.size()); + result->append(ts.data(), ts.size()); + PutFixed64(result, PackSequenceAndType(key.sequence, key.type)); +} + +void AppendInternalKeyFooter(std::string* result, SequenceNumber s, + ValueType t) { + PutFixed64(result, PackSequenceAndType(s, t)); +} + +void AppendKeyWithMinTimestamp(std::string* result, const Slice& key, + size_t ts_sz) { + assert(ts_sz > 0); + const std::string kTsMin(ts_sz, static_cast(0)); + result->append(key.data(), key.size()); + result->append(kTsMin.data(), ts_sz); +} + +void AppendKeyWithMaxTimestamp(std::string* result, const Slice& key, + size_t ts_sz) { + assert(ts_sz > 0); + const std::string kTsMax(ts_sz, static_cast(0xff)); + result->append(key.data(), key.size()); + result->append(kTsMax.data(), ts_sz); +} + +void AppendUserKeyWithMaxTimestamp(std::string* result, const Slice& key, + size_t ts_sz) { + assert(ts_sz > 0); + result->append(key.data(), key.size() - ts_sz); + + static constexpr char kTsMax[] = "\xff\xff\xff\xff\xff\xff\xff\xff\xff"; + if (ts_sz < strlen(kTsMax)) { + result->append(kTsMax, ts_sz); + } else { + result->append(std::string(ts_sz, '\xff')); + } +} + +void PadInternalKeyWithMinTimestamp(std::string* result, const Slice& key, + size_t ts_sz) { + assert(ts_sz > 0); + size_t user_key_size = key.size() - kNumInternalBytes; + result->reserve(key.size() + ts_sz); + result->append(key.data(), user_key_size); + result->append(ts_sz, static_cast(0)); + result->append(key.data() + user_key_size, kNumInternalBytes); +} + +void StripTimestampFromInternalKey(std::string* result, const Slice& key, + size_t ts_sz) { + assert(key.size() >= ts_sz + kNumInternalBytes); + result->reserve(key.size() - ts_sz); + result->append(key.data(), key.size() - kNumInternalBytes - ts_sz); + result->append(key.data() + key.size() - kNumInternalBytes, + kNumInternalBytes); +} + +void ReplaceInternalKeyWithMinTimestamp(std::string* result, const Slice& key, + size_t ts_sz) { + const size_t key_sz = key.size(); + assert(key_sz >= ts_sz + kNumInternalBytes); + result->reserve(key_sz); + result->append(key.data(), key_sz - kNumInternalBytes - ts_sz); + result->append(ts_sz, static_cast(0)); + result->append(key.data() + key_sz - kNumInternalBytes, kNumInternalBytes); +} + +std::string ParsedInternalKey::DebugString(bool log_err_key, bool hex) const { + std::string result = "'"; + if (log_err_key) { + result += user_key.ToString(hex); + } else { + result += ""; + } + + char buf[50]; + snprintf(buf, sizeof(buf), "' seq:%" PRIu64 ", type:%d", sequence, + static_cast(type)); + + result += buf; + return result; +} + +std::string InternalKey::DebugString(bool hex) const { + std::string result; + ParsedInternalKey parsed; + if (ParseInternalKey(rep_, &parsed, false /* log_err_key */).ok()) { + result = parsed.DebugString(true /* log_err_key */, hex); // TODO + } else { + result = "(bad)"; + result.append(EscapeString(rep_)); + } + return result; +} + +int InternalKeyComparator::Compare(const ParsedInternalKey& a, + const ParsedInternalKey& b) const { + // Order by: + // increasing user key (according to user-supplied comparator) + // decreasing sequence number + // decreasing type (though sequence# should be enough to disambiguate) + int r = user_comparator_.Compare(a.user_key, b.user_key); + if (r == 0) { + if (a.sequence > b.sequence) { + r = -1; + } else if (a.sequence < b.sequence) { + r = +1; + } else if (a.type > b.type) { + r = -1; + } else if (a.type < b.type) { + r = +1; + } + } + return r; +} + +int InternalKeyComparator::Compare(const Slice& a, + const ParsedInternalKey& b) const { + // Order by: + // increasing user key (according to user-supplied comparator) + // decreasing sequence number + // decreasing type (though sequence# should be enough to disambiguate) + int r = user_comparator_.Compare(ExtractUserKey(a), b.user_key); + if (r == 0) { + const uint64_t anum = + DecodeFixed64(a.data() + a.size() - kNumInternalBytes); + const uint64_t bnum = (b.sequence << 8) | b.type; + if (anum > bnum) { + r = -1; + } else if (anum < bnum) { + r = +1; + } + } + return r; +} + +int InternalKeyComparator::Compare(const ParsedInternalKey& a, + const Slice& b) const { + return -Compare(b, a); +} + +LookupKey::LookupKey(const Slice& _user_key, SequenceNumber s, + const Slice* ts) { + size_t usize = _user_key.size(); + size_t ts_sz = (nullptr == ts) ? 0 : ts->size(); + size_t needed = usize + ts_sz + 13; // A conservative estimate + char* dst; + if (needed <= sizeof(space_)) { + dst = space_; + } else { + dst = new char[needed]; + } + start_ = dst; + // NOTE: We don't support users keys of more than 2GB :) + dst = EncodeVarint32(dst, static_cast(usize + ts_sz + 8)); + kstart_ = dst; + memcpy(dst, _user_key.data(), usize); + dst += usize; + if (nullptr != ts) { + memcpy(dst, ts->data(), ts_sz); + dst += ts_sz; + } + EncodeFixed64(dst, PackSequenceAndType(s, kValueTypeForSeek)); + dst += 8; + end_ = dst; +} + +void IterKey::EnlargeBuffer(size_t key_size) { + // If size is smaller than buffer size, continue using current buffer, + // or the static allocated one, as default + assert(key_size > buf_size_); + // Need to enlarge the buffer. + ResetBuffer(); + buf_ = new char[key_size]; + buf_size_ = key_size; +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/dbformat.h b/librocksdb-sys/rocksdb/db/dbformat.h new file mode 100644 index 0000000..d0b3bba --- /dev/null +++ b/librocksdb-sys/rocksdb/db/dbformat.h @@ -0,0 +1,959 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include + +#include +#include +#include + +#include "rocksdb/comparator.h" +#include "rocksdb/slice.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/types.h" +#include "util/coding.h" +#include "util/user_comparator_wrapper.h" + +namespace ROCKSDB_NAMESPACE { + +// The file declares data structures and functions that deal with internal +// keys. +// Each internal key contains a user key, a sequence number (SequenceNumber) +// and a type (ValueType), and they are usually encoded together. +// There are some related helper classes here. + +class InternalKey; + +// Value types encoded as the last component of internal keys. +// DO NOT CHANGE THESE ENUM VALUES: they are embedded in the on-disk +// data structures. +// The highest bit of the value type needs to be reserved to SST tables +// for them to do more flexible encoding. +enum ValueType : unsigned char { + kTypeDeletion = 0x0, + kTypeValue = 0x1, + kTypeMerge = 0x2, + kTypeLogData = 0x3, // WAL only. + kTypeColumnFamilyDeletion = 0x4, // WAL only. + kTypeColumnFamilyValue = 0x5, // WAL only. + kTypeColumnFamilyMerge = 0x6, // WAL only. + kTypeSingleDeletion = 0x7, + kTypeColumnFamilySingleDeletion = 0x8, // WAL only. + kTypeBeginPrepareXID = 0x9, // WAL only. + kTypeEndPrepareXID = 0xA, // WAL only. + kTypeCommitXID = 0xB, // WAL only. + kTypeRollbackXID = 0xC, // WAL only. + kTypeNoop = 0xD, // WAL only. + kTypeColumnFamilyRangeDeletion = 0xE, // WAL only. + kTypeRangeDeletion = 0xF, // meta block + kTypeColumnFamilyBlobIndex = 0x10, // Blob DB only + kTypeBlobIndex = 0x11, // Blob DB only + // When the prepared record is also persisted in db, we use a different + // record. This is to ensure that the WAL that is generated by a WritePolicy + // is not mistakenly read by another, which would result into data + // inconsistency. + kTypeBeginPersistedPrepareXID = 0x12, // WAL only. + // Similar to kTypeBeginPersistedPrepareXID, this is to ensure that WAL + // generated by WriteUnprepared write policy is not mistakenly read by + // another. + kTypeBeginUnprepareXID = 0x13, // WAL only. + kTypeDeletionWithTimestamp = 0x14, + kTypeCommitXIDAndTimestamp = 0x15, // WAL only + kTypeWideColumnEntity = 0x16, + kTypeColumnFamilyWideColumnEntity = 0x17, // WAL only + kTypeMaxValid, // Should be after the last valid type, only used for + // validation + kMaxValue = 0x7F // Not used for storing records. +}; + +// Defined in dbformat.cc +extern const ValueType kValueTypeForSeek; +extern const ValueType kValueTypeForSeekForPrev; + +// Checks whether a type is an inline value type +// (i.e. a type used in memtable skiplist and sst file datablock). +inline bool IsValueType(ValueType t) { + return t <= kTypeMerge || kTypeSingleDeletion == t || kTypeBlobIndex == t || + kTypeDeletionWithTimestamp == t || kTypeWideColumnEntity == t; +} + +// Checks whether a type is from user operation +// kTypeRangeDeletion is in meta block so this API is separated from above +// kTypeMaxValid can be from keys generated by +// TruncatedRangeDelIterator::start_key() +inline bool IsExtendedValueType(ValueType t) { + return IsValueType(t) || t == kTypeRangeDeletion || t == kTypeMaxValid; +} + +// We leave eight bits empty at the bottom so a type and sequence# +// can be packed together into 64-bits. +static const SequenceNumber kMaxSequenceNumber = ((0x1ull << 56) - 1); + +static const SequenceNumber kDisableGlobalSequenceNumber = + std::numeric_limits::max(); + +constexpr uint64_t kNumInternalBytes = 8; + +// Defined in dbformat.cc +extern const std::string kDisableUserTimestamp; + +// The data structure that represents an internal key in the way that user_key, +// sequence number and type are stored in separated forms. +struct ParsedInternalKey { + Slice user_key; + SequenceNumber sequence; + ValueType type; + + ParsedInternalKey() + : sequence(kMaxSequenceNumber), + type(kTypeDeletion) // Make code analyzer happy + {} // Intentionally left uninitialized (for speed) + // u contains timestamp if user timestamp feature is enabled. + ParsedInternalKey(const Slice& u, const SequenceNumber& seq, ValueType t) + : user_key(u), sequence(seq), type(t) {} + std::string DebugString(bool log_err_key, bool hex) const; + + void clear() { + user_key.clear(); + sequence = 0; + type = kTypeDeletion; + } + + void SetTimestamp(const Slice& ts) { + assert(ts.size() <= user_key.size()); + const char* addr = user_key.data() + user_key.size() - ts.size(); + memcpy(const_cast(addr), ts.data(), ts.size()); + } + + Slice GetTimestamp(size_t ts_sz) { + assert(ts_sz <= user_key.size()); + const char* addr = user_key.data() + user_key.size() - ts_sz; + return Slice(const_cast(addr), ts_sz); + } +}; + +// Return the length of the encoding of "key". +inline size_t InternalKeyEncodingLength(const ParsedInternalKey& key) { + return key.user_key.size() + kNumInternalBytes; +} + +// Pack a sequence number and a ValueType into a uint64_t +inline uint64_t PackSequenceAndType(uint64_t seq, ValueType t) { + assert(seq <= kMaxSequenceNumber); + // kTypeMaxValid is used in TruncatedRangeDelIterator, see its constructor. + assert(IsExtendedValueType(t) || t == kTypeMaxValid); + return (seq << 8) | t; +} + +// Given the result of PackSequenceAndType, store the sequence number in *seq +// and the ValueType in *t. +inline void UnPackSequenceAndType(uint64_t packed, uint64_t* seq, + ValueType* t) { + *seq = packed >> 8; + *t = static_cast(packed & 0xff); + + // Commented the following two assertions in order to test key-value checksum + // on corrupted keys without crashing ("DbKvChecksumTest"). + // assert(*seq <= kMaxSequenceNumber); + // assert(IsExtendedValueType(*t)); +} + +EntryType GetEntryType(ValueType value_type); + +// Append the serialization of "key" to *result. +void AppendInternalKey(std::string* result, const ParsedInternalKey& key); + +// Append the serialization of "key" to *result, replacing the original +// timestamp with argument ts. +void AppendInternalKeyWithDifferentTimestamp(std::string* result, + const ParsedInternalKey& key, + const Slice& ts); + +// Serialized internal key consists of user key followed by footer. +// This function appends the footer to *result, assuming that *result already +// contains the user key at the end. +void AppendInternalKeyFooter(std::string* result, SequenceNumber s, + ValueType t); + +// Append the key and a minimal timestamp to *result +void AppendKeyWithMinTimestamp(std::string* result, const Slice& key, + size_t ts_sz); + +// Append the key and a maximal timestamp to *result +void AppendKeyWithMaxTimestamp(std::string* result, const Slice& key, + size_t ts_sz); + +// `key` is a user key with timestamp. Append the user key without timestamp +// and the maximal timestamp to *result. +void AppendUserKeyWithMaxTimestamp(std::string* result, const Slice& key, + size_t ts_sz); + +// `key` is an internal key containing a user key without timestamp. Create a +// new key in *result by padding a min timestamp of size `ts_sz` to the user key +// and copying the remaining internal key bytes. +void PadInternalKeyWithMinTimestamp(std::string* result, const Slice& key, + size_t ts_sz); + +// `key` is an internal key containing a user key with timestamp of size +// `ts_sz`. Create a new internal key in *result by stripping the timestamp from +// the user key and copying the remaining internal key bytes. +void StripTimestampFromInternalKey(std::string* result, const Slice& key, + size_t ts_sz); + +// `key` is an internal key containing a user key with timestamp of size +// `ts_sz`. Create a new internal key in *result while replace the original +// timestamp with min timestamp. +void ReplaceInternalKeyWithMinTimestamp(std::string* result, const Slice& key, + size_t ts_sz); + +// Attempt to parse an internal key from "internal_key". On success, +// stores the parsed data in "*result", and returns true. +// +// On error, returns false, leaves "*result" in an undefined state. +Status ParseInternalKey(const Slice& internal_key, ParsedInternalKey* result, + bool log_err_key); + +// Returns the user key portion of an internal key. +inline Slice ExtractUserKey(const Slice& internal_key) { + assert(internal_key.size() >= kNumInternalBytes); + return Slice(internal_key.data(), internal_key.size() - kNumInternalBytes); +} + +inline Slice ExtractUserKeyAndStripTimestamp(const Slice& internal_key, + size_t ts_sz) { + Slice ret = internal_key; + ret.remove_suffix(kNumInternalBytes + ts_sz); + return ret; +} + +inline Slice StripTimestampFromUserKey(const Slice& user_key, size_t ts_sz) { + Slice ret = user_key; + ret.remove_suffix(ts_sz); + return ret; +} + +inline Slice ExtractTimestampFromUserKey(const Slice& user_key, size_t ts_sz) { + assert(user_key.size() >= ts_sz); + return Slice(user_key.data() + user_key.size() - ts_sz, ts_sz); +} + +inline Slice ExtractTimestampFromKey(const Slice& internal_key, size_t ts_sz) { + const size_t key_size = internal_key.size(); + assert(key_size >= kNumInternalBytes + ts_sz); + return Slice(internal_key.data() + key_size - ts_sz - kNumInternalBytes, + ts_sz); +} + +inline uint64_t ExtractInternalKeyFooter(const Slice& internal_key) { + assert(internal_key.size() >= kNumInternalBytes); + const size_t n = internal_key.size(); + return DecodeFixed64(internal_key.data() + n - kNumInternalBytes); +} + +inline ValueType ExtractValueType(const Slice& internal_key) { + uint64_t num = ExtractInternalKeyFooter(internal_key); + unsigned char c = num & 0xff; + return static_cast(c); +} + +// A comparator for internal keys that uses a specified comparator for +// the user key portion and breaks ties by decreasing sequence number. +class InternalKeyComparator +#ifdef NDEBUG + final +#endif + : public CompareInterface { + private: + UserComparatorWrapper user_comparator_; + + public: + // `InternalKeyComparator`s constructed with the default constructor are not + // usable and will segfault on any attempt to use them for comparisons. + InternalKeyComparator() = default; + + // @param named If true, assign a name to this comparator based on the + // underlying comparator's name. This involves an allocation and copy in + // this constructor to precompute the result of `Name()`. To avoid this + // overhead, set `named` to false. In that case, `Name()` will return a + // generic name that is non-specific to the underlying comparator. + explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c) {} + virtual ~InternalKeyComparator() {} + + int Compare(const Slice& a, const Slice& b) const override; + + bool Equal(const Slice& a, const Slice& b) const { + // TODO Use user_comparator_.Equal(). Perhaps compare seqno before + // comparing the user key too. + return Compare(a, b) == 0; + } + + // Same as Compare except that it excludes the value type from comparison + int CompareKeySeq(const Slice& a, const Slice& b) const; + + const Comparator* user_comparator() const { + return user_comparator_.user_comparator(); + } + + int Compare(const InternalKey& a, const InternalKey& b) const; + int Compare(const ParsedInternalKey& a, const ParsedInternalKey& b) const; + int Compare(const Slice& a, const ParsedInternalKey& b) const; + int Compare(const ParsedInternalKey& a, const Slice& b) const; + // In this `Compare()` overload, the sequence numbers provided in + // `a_global_seqno` and `b_global_seqno` override the sequence numbers in `a` + // and `b`, respectively. To disable sequence number override(s), provide the + // value `kDisableGlobalSequenceNumber`. + int Compare(const Slice& a, SequenceNumber a_global_seqno, const Slice& b, + SequenceNumber b_global_seqno) const; +}; + +// The class represent the internal key in encoded form. +class InternalKey { + private: + std::string rep_; + + public: + InternalKey() {} // Leave rep_ as empty to indicate it is invalid + InternalKey(const Slice& _user_key, SequenceNumber s, ValueType t) { + AppendInternalKey(&rep_, ParsedInternalKey(_user_key, s, t)); + } + InternalKey(const Slice& _user_key, SequenceNumber s, ValueType t, Slice ts) { + AppendInternalKeyWithDifferentTimestamp( + &rep_, ParsedInternalKey(_user_key, s, t), ts); + } + + // sets the internal key to be bigger or equal to all internal keys with this + // user key + void SetMaxPossibleForUserKey(const Slice& _user_key) { + AppendInternalKey( + &rep_, ParsedInternalKey(_user_key, 0, static_cast(0))); + } + + // sets the internal key to be smaller or equal to all internal keys with this + // user key + void SetMinPossibleForUserKey(const Slice& _user_key) { + AppendInternalKey(&rep_, ParsedInternalKey(_user_key, kMaxSequenceNumber, + kValueTypeForSeek)); + } + + bool Valid() const { + ParsedInternalKey parsed; + return (ParseInternalKey(Slice(rep_), &parsed, false /* log_err_key */) + .ok()); // TODO + } + + void DecodeFrom(const Slice& s) { rep_.assign(s.data(), s.size()); } + Slice Encode() const { + assert(!rep_.empty()); + return rep_; + } + + Slice user_key() const { return ExtractUserKey(rep_); } + size_t size() const { return rep_.size(); } + + void Set(const Slice& _user_key, SequenceNumber s, ValueType t) { + SetFrom(ParsedInternalKey(_user_key, s, t)); + } + + void Set(const Slice& _user_key_with_ts, SequenceNumber s, ValueType t, + const Slice& ts) { + ParsedInternalKey pik = ParsedInternalKey(_user_key_with_ts, s, t); + // Should not call pik.SetTimestamp() directly as it overwrites the buffer + // containing _user_key. + SetFrom(pik, ts); + } + + void SetFrom(const ParsedInternalKey& p) { + rep_.clear(); + AppendInternalKey(&rep_, p); + } + + void SetFrom(const ParsedInternalKey& p, const Slice& ts) { + rep_.clear(); + AppendInternalKeyWithDifferentTimestamp(&rep_, p, ts); + } + + void Clear() { rep_.clear(); } + + // The underlying representation. + // Intended only to be used together with ConvertFromUserKey(). + std::string* rep() { return &rep_; } + + // Assuming that *rep() contains a user key, this method makes internal key + // out of it in-place. This saves a memcpy compared to Set()/SetFrom(). + void ConvertFromUserKey(SequenceNumber s, ValueType t) { + AppendInternalKeyFooter(&rep_, s, t); + } + + std::string DebugString(bool hex) const; +}; + +inline int InternalKeyComparator::Compare(const InternalKey& a, + const InternalKey& b) const { + return Compare(a.Encode(), b.Encode()); +} + +inline Status ParseInternalKey(const Slice& internal_key, + ParsedInternalKey* result, bool log_err_key) { + const size_t n = internal_key.size(); + + if (n < kNumInternalBytes) { + return Status::Corruption("Corrupted Key: Internal Key too small. Size=" + + std::to_string(n) + ". "); + } + + uint64_t num = DecodeFixed64(internal_key.data() + n - kNumInternalBytes); + unsigned char c = num & 0xff; + result->sequence = num >> 8; + result->type = static_cast(c); + assert(result->type <= ValueType::kMaxValue); + result->user_key = Slice(internal_key.data(), n - kNumInternalBytes); + + if (IsExtendedValueType(result->type)) { + return Status::OK(); + } else { + return Status::Corruption("Corrupted Key", + result->DebugString(log_err_key, true)); + } +} + +// Update the sequence number in the internal key. +// Guarantees not to invalidate ikey.data(). +inline void UpdateInternalKey(std::string* ikey, uint64_t seq, ValueType t) { + size_t ikey_sz = ikey->size(); + assert(ikey_sz >= kNumInternalBytes); + uint64_t newval = (seq << 8) | t; + + // Note: Since C++11, strings are guaranteed to be stored contiguously and + // string::operator[]() is guaranteed not to change ikey.data(). + EncodeFixed64(&(*ikey)[ikey_sz - kNumInternalBytes], newval); +} + +// Get the sequence number from the internal key +inline uint64_t GetInternalKeySeqno(const Slice& internal_key) { + const size_t n = internal_key.size(); + assert(n >= kNumInternalBytes); + uint64_t num = DecodeFixed64(internal_key.data() + n - kNumInternalBytes); + return num >> 8; +} + +// The class to store keys in an efficient way. It allows: +// 1. Users can either copy the key into it, or have it point to an unowned +// address. +// 2. For copied key, a short inline buffer is kept to reduce memory +// allocation for smaller keys. +// 3. It tracks user key or internal key, and allow conversion between them. +class IterKey { + public: + IterKey() + : buf_(space_), + key_(buf_), + key_size_(0), + buf_size_(sizeof(space_)), + is_user_key_(true) {} + // No copying allowed + IterKey(const IterKey&) = delete; + void operator=(const IterKey&) = delete; + + ~IterKey() { ResetBuffer(); } + + // The bool will be picked up by the next calls to SetKey + void SetIsUserKey(bool is_user_key) { is_user_key_ = is_user_key; } + + // Returns the key in whichever format that was provided to KeyIter + // If user-defined timestamp is enabled, then timestamp is included in the + // return result. + Slice GetKey() const { return Slice(key_, key_size_); } + + Slice GetInternalKey() const { + assert(!IsUserKey()); + return Slice(key_, key_size_); + } + + // If user-defined timestamp is enabled, then timestamp is included in the + // return result of GetUserKey(); + Slice GetUserKey() const { + if (IsUserKey()) { + return Slice(key_, key_size_); + } else { + assert(key_size_ >= kNumInternalBytes); + return Slice(key_, key_size_ - kNumInternalBytes); + } + } + + size_t Size() const { return key_size_; } + + void Clear() { key_size_ = 0; } + + // Append "non_shared_data" to its back, from "shared_len" + // This function is used in Block::Iter::ParseNextKey + // shared_len: bytes in [0, shard_len-1] would be remained + // non_shared_data: data to be append, its length must be >= non_shared_len + void TrimAppend(const size_t shared_len, const char* non_shared_data, + const size_t non_shared_len) { + assert(shared_len <= key_size_); + size_t total_size = shared_len + non_shared_len; + + if (IsKeyPinned() /* key is not in buf_ */) { + // Copy the key from external memory to buf_ (copy shared_len bytes) + EnlargeBufferIfNeeded(total_size); + memcpy(buf_, key_, shared_len); + } else if (total_size > buf_size_) { + // Need to allocate space, delete previous space + char* p = new char[total_size]; + memcpy(p, key_, shared_len); + + if (buf_ != space_) { + delete[] buf_; + } + + buf_ = p; + buf_size_ = total_size; + } + + memcpy(buf_ + shared_len, non_shared_data, non_shared_len); + key_ = buf_; + key_size_ = total_size; + } + + // A version of `TrimAppend` assuming the last bytes of length `ts_sz` in the + // user key part of `key_` is not counted towards shared bytes. And the + // decoded key needed a min timestamp of length `ts_sz` pad to the user key. + void TrimAppendWithTimestamp(const size_t shared_len, + const char* non_shared_data, + const size_t non_shared_len, + const size_t ts_sz) { + std::string kTsMin(ts_sz, static_cast(0)); + std::string key_with_ts; + std::vector key_parts_with_ts; + if (IsUserKey()) { + key_parts_with_ts = {Slice(key_, shared_len), + Slice(non_shared_data, non_shared_len), + Slice(kTsMin)}; + } else { + assert(shared_len + non_shared_len >= kNumInternalBytes); + // Invaraint: shared_user_key_len + shared_internal_bytes_len = shared_len + // In naming below `*_len` variables, keyword `user_key` refers to the + // user key part of the existing key in `key_` as apposed to the new key. + // Similary, `internal_bytes` refers to the footer part of the existing + // key. These bytes potentially will move between user key part and the + // footer part in the new key. + const size_t user_key_len = key_size_ - kNumInternalBytes; + const size_t sharable_user_key_len = user_key_len - ts_sz; + const size_t shared_user_key_len = + std::min(shared_len, sharable_user_key_len); + const size_t shared_internal_bytes_len = shared_len - shared_user_key_len; + + // One Slice among the three Slices will get split into two Slices, plus + // a timestamp slice. + key_parts_with_ts.reserve(5); + bool ts_added = false; + // Add slice parts and find the right location to add the min timestamp. + MaybeAddKeyPartsWithTimestamp( + key_, shared_user_key_len, + shared_internal_bytes_len + non_shared_len < kNumInternalBytes, + shared_len + non_shared_len - kNumInternalBytes, kTsMin, + key_parts_with_ts, &ts_added); + MaybeAddKeyPartsWithTimestamp( + key_ + user_key_len, shared_internal_bytes_len, + non_shared_len < kNumInternalBytes, + shared_internal_bytes_len + non_shared_len - kNumInternalBytes, + kTsMin, key_parts_with_ts, &ts_added); + MaybeAddKeyPartsWithTimestamp(non_shared_data, non_shared_len, + non_shared_len >= kNumInternalBytes, + non_shared_len - kNumInternalBytes, kTsMin, + key_parts_with_ts, &ts_added); + assert(ts_added); + } + + Slice new_key(SliceParts(&key_parts_with_ts.front(), + static_cast(key_parts_with_ts.size())), + &key_with_ts); + SetKey(new_key); + } + + Slice SetKey(const Slice& key, bool copy = true) { + // is_user_key_ expected to be set already via SetIsUserKey + return SetKeyImpl(key, copy); + } + + // If user-defined timestamp is enabled, then `key` includes timestamp. + // TODO(yanqin) this is also used to set prefix, which do not include + // timestamp. Should be handled. + Slice SetUserKey(const Slice& key, bool copy = true) { + is_user_key_ = true; + return SetKeyImpl(key, copy); + } + + Slice SetInternalKey(const Slice& key, bool copy = true) { + is_user_key_ = false; + return SetKeyImpl(key, copy); + } + + // Copies the content of key, updates the reference to the user key in ikey + // and returns a Slice referencing the new copy. + Slice SetInternalKey(const Slice& key, ParsedInternalKey* ikey) { + size_t key_n = key.size(); + assert(key_n >= kNumInternalBytes); + SetInternalKey(key); + ikey->user_key = Slice(key_, key_n - kNumInternalBytes); + return Slice(key_, key_n); + } + + // Copy the key into IterKey own buf_ + void OwnKey() { + assert(IsKeyPinned() == true); + + Reserve(key_size_); + memcpy(buf_, key_, key_size_); + key_ = buf_; + } + + // Update the sequence number in the internal key. Guarantees not to + // invalidate slices to the key (and the user key). + void UpdateInternalKey(uint64_t seq, ValueType t, const Slice* ts = nullptr) { + assert(!IsKeyPinned()); + assert(key_size_ >= kNumInternalBytes); + if (ts) { + assert(key_size_ >= kNumInternalBytes + ts->size()); + memcpy(&buf_[key_size_ - kNumInternalBytes - ts->size()], ts->data(), + ts->size()); + } + uint64_t newval = (seq << 8) | t; + EncodeFixed64(&buf_[key_size_ - kNumInternalBytes], newval); + } + + bool IsKeyPinned() const { return (key_ != buf_); } + + // If `ts` is provided, user_key should not contain timestamp, + // and `ts` is appended after user_key. + // TODO: more efficient storage for timestamp. + void SetInternalKey(const Slice& key_prefix, const Slice& user_key, + SequenceNumber s, + ValueType value_type = kValueTypeForSeek, + const Slice* ts = nullptr) { + size_t psize = key_prefix.size(); + size_t usize = user_key.size(); + size_t ts_sz = (ts != nullptr ? ts->size() : 0); + EnlargeBufferIfNeeded(psize + usize + sizeof(uint64_t) + ts_sz); + if (psize > 0) { + memcpy(buf_, key_prefix.data(), psize); + } + memcpy(buf_ + psize, user_key.data(), usize); + if (ts) { + memcpy(buf_ + psize + usize, ts->data(), ts_sz); + } + EncodeFixed64(buf_ + usize + psize + ts_sz, + PackSequenceAndType(s, value_type)); + + key_ = buf_; + key_size_ = psize + usize + sizeof(uint64_t) + ts_sz; + is_user_key_ = false; + } + + void SetInternalKey(const Slice& user_key, SequenceNumber s, + ValueType value_type = kValueTypeForSeek, + const Slice* ts = nullptr) { + SetInternalKey(Slice(), user_key, s, value_type, ts); + } + + void Reserve(size_t size) { + EnlargeBufferIfNeeded(size); + key_size_ = size; + } + + void SetInternalKey(const ParsedInternalKey& parsed_key) { + SetInternalKey(Slice(), parsed_key); + } + + void SetInternalKey(const Slice& key_prefix, + const ParsedInternalKey& parsed_key_suffix) { + SetInternalKey(key_prefix, parsed_key_suffix.user_key, + parsed_key_suffix.sequence, parsed_key_suffix.type); + } + + void EncodeLengthPrefixedKey(const Slice& key) { + auto size = key.size(); + EnlargeBufferIfNeeded(size + static_cast(VarintLength(size))); + char* ptr = EncodeVarint32(buf_, static_cast(size)); + memcpy(ptr, key.data(), size); + key_ = buf_; + is_user_key_ = true; + } + + bool IsUserKey() const { return is_user_key_; } + + private: + char* buf_; + const char* key_; + size_t key_size_; + size_t buf_size_; + char space_[39]; // Avoid allocation for short keys + bool is_user_key_; + + Slice SetKeyImpl(const Slice& key, bool copy) { + size_t size = key.size(); + if (copy) { + // Copy key to buf_ + EnlargeBufferIfNeeded(size); + memcpy(buf_, key.data(), size); + key_ = buf_; + } else { + // Update key_ to point to external memory + key_ = key.data(); + } + key_size_ = size; + return Slice(key_, key_size_); + } + + void ResetBuffer() { + if (buf_ != space_) { + delete[] buf_; + buf_ = space_; + } + buf_size_ = sizeof(space_); + key_size_ = 0; + } + + // Enlarge the buffer size if needed based on key_size. + // By default, static allocated buffer is used. Once there is a key + // larger than the static allocated buffer, another buffer is dynamically + // allocated, until a larger key buffer is requested. In that case, we + // reallocate buffer and delete the old one. + void EnlargeBufferIfNeeded(size_t key_size) { + // If size is smaller than buffer size, continue using current buffer, + // or the static allocated one, as default + if (key_size > buf_size_) { + EnlargeBuffer(key_size); + } + } + + void EnlargeBuffer(size_t key_size); + + void MaybeAddKeyPartsWithTimestamp(const char* slice_data, + const size_t slice_sz, bool add_timestamp, + const size_t left_sz, + const std::string& min_timestamp, + std::vector& key_parts, + bool* ts_added) { + if (add_timestamp && !*ts_added) { + assert(slice_sz >= left_sz); + key_parts.emplace_back(slice_data, left_sz); + key_parts.emplace_back(min_timestamp); + key_parts.emplace_back(slice_data + left_sz, slice_sz - left_sz); + *ts_added = true; + } else { + key_parts.emplace_back(slice_data, slice_sz); + } + } +}; + +// Convert from a SliceTransform of user keys, to a SliceTransform of +// internal keys. +class InternalKeySliceTransform : public SliceTransform { + public: + explicit InternalKeySliceTransform(const SliceTransform* transform) + : transform_(transform) {} + + virtual const char* Name() const override { return transform_->Name(); } + + virtual Slice Transform(const Slice& src) const override { + auto user_key = ExtractUserKey(src); + return transform_->Transform(user_key); + } + + virtual bool InDomain(const Slice& src) const override { + auto user_key = ExtractUserKey(src); + return transform_->InDomain(user_key); + } + + virtual bool InRange(const Slice& dst) const override { + auto user_key = ExtractUserKey(dst); + return transform_->InRange(user_key); + } + + const SliceTransform* user_prefix_extractor() const { return transform_; } + + private: + // Like comparator, InternalKeySliceTransform will not take care of the + // deletion of transform_ + const SliceTransform* const transform_; +}; + +// Read the key of a record from a write batch. +// if this record represent the default column family then cf_record +// must be passed as false, otherwise it must be passed as true. +bool ReadKeyFromWriteBatchEntry(Slice* input, Slice* key, bool cf_record); + +// Read record from a write batch piece from input. +// tag, column_family, key, value and blob are return values. Callers own the +// slice they point to. +// Tag is defined as ValueType. +// input will be advanced to after the record. +// If user-defined timestamp is enabled for a column family, then the `key` +// resulting from this call will include timestamp. +Status ReadRecordFromWriteBatch(Slice* input, char* tag, + uint32_t* column_family, Slice* key, + Slice* value, Slice* blob, Slice* xid); + +// When user call DeleteRange() to delete a range of keys, +// we will store a serialized RangeTombstone in MemTable and SST. +// the struct here is an easy-understood form +// start/end_key_ is the start/end user key of the range to be deleted +struct RangeTombstone { + Slice start_key_; + Slice end_key_; + SequenceNumber seq_; + // TODO: we should optimize the storage here when user-defined timestamp + // is NOT enabled: they currently take up (16 + 32 + 32) bytes per tombstone. + Slice ts_; + std::string pinned_start_key_; + std::string pinned_end_key_; + + RangeTombstone() = default; + RangeTombstone(Slice sk, Slice ek, SequenceNumber sn) + : start_key_(sk), end_key_(ek), seq_(sn) {} + + // User-defined timestamp is enabled, `sk` and `ek` should be user key + // with timestamp, `ts` will replace the timestamps in `sk` and + // `ek`. + RangeTombstone(Slice sk, Slice ek, SequenceNumber sn, Slice ts) + : seq_(sn), ts_(ts) { + assert(!ts.empty()); + pinned_start_key_.reserve(sk.size()); + pinned_start_key_.append(sk.data(), sk.size() - ts.size()); + pinned_start_key_.append(ts.data(), ts.size()); + pinned_end_key_.reserve(ek.size()); + pinned_end_key_.append(ek.data(), ek.size() - ts.size()); + pinned_end_key_.append(ts.data(), ts.size()); + start_key_ = pinned_start_key_; + end_key_ = pinned_end_key_; + } + + RangeTombstone(ParsedInternalKey parsed_key, Slice value) { + start_key_ = parsed_key.user_key; + seq_ = parsed_key.sequence; + end_key_ = value; + } + + // be careful to use Serialize(), allocates new memory + std::pair Serialize() const { + auto key = InternalKey(start_key_, seq_, kTypeRangeDeletion); + return std::make_pair(std::move(key), end_key_); + } + + // be careful to use SerializeKey(), allocates new memory + InternalKey SerializeKey() const { + return InternalKey(start_key_, seq_, kTypeRangeDeletion); + } + + // The tombstone end-key is exclusive, so we generate an internal-key here + // which has a similar property. Using kMaxSequenceNumber guarantees that + // the returned internal-key will compare less than any other internal-key + // with the same user-key. This in turn guarantees that the serialized + // end-key for a tombstone such as [a-b] will compare less than the key "b". + // + // be careful to use SerializeEndKey(), allocates new memory + InternalKey SerializeEndKey() const { + if (!ts_.empty()) { + static constexpr char kTsMax[] = "\xff\xff\xff\xff\xff\xff\xff\xff\xff"; + if (ts_.size() <= strlen(kTsMax)) { + return InternalKey(end_key_, kMaxSequenceNumber, kTypeRangeDeletion, + Slice(kTsMax, ts_.size())); + } else { + return InternalKey(end_key_, kMaxSequenceNumber, kTypeRangeDeletion, + std::string(ts_.size(), '\xff')); + } + } + return InternalKey(end_key_, kMaxSequenceNumber, kTypeRangeDeletion); + } +}; + +inline int InternalKeyComparator::Compare(const Slice& akey, + const Slice& bkey) const { + // Order by: + // increasing user key (according to user-supplied comparator) + // decreasing sequence number + // decreasing type (though sequence# should be enough to disambiguate) + int r = user_comparator_.Compare(ExtractUserKey(akey), ExtractUserKey(bkey)); + if (r == 0) { + const uint64_t anum = + DecodeFixed64(akey.data() + akey.size() - kNumInternalBytes); + const uint64_t bnum = + DecodeFixed64(bkey.data() + bkey.size() - kNumInternalBytes); + if (anum > bnum) { + r = -1; + } else if (anum < bnum) { + r = +1; + } + } + return r; +} + +inline int InternalKeyComparator::CompareKeySeq(const Slice& akey, + const Slice& bkey) const { + // Order by: + // increasing user key (according to user-supplied comparator) + // decreasing sequence number + int r = user_comparator_.Compare(ExtractUserKey(akey), ExtractUserKey(bkey)); + if (r == 0) { + // Shift the number to exclude the last byte which contains the value type + const uint64_t anum = + DecodeFixed64(akey.data() + akey.size() - kNumInternalBytes) >> 8; + const uint64_t bnum = + DecodeFixed64(bkey.data() + bkey.size() - kNumInternalBytes) >> 8; + if (anum > bnum) { + r = -1; + } else if (anum < bnum) { + r = +1; + } + } + return r; +} + +inline int InternalKeyComparator::Compare(const Slice& a, + SequenceNumber a_global_seqno, + const Slice& b, + SequenceNumber b_global_seqno) const { + int r = user_comparator_.Compare(ExtractUserKey(a), ExtractUserKey(b)); + if (r == 0) { + uint64_t a_footer, b_footer; + if (a_global_seqno == kDisableGlobalSequenceNumber) { + a_footer = ExtractInternalKeyFooter(a); + } else { + a_footer = PackSequenceAndType(a_global_seqno, ExtractValueType(a)); + } + if (b_global_seqno == kDisableGlobalSequenceNumber) { + b_footer = ExtractInternalKeyFooter(b); + } else { + b_footer = PackSequenceAndType(b_global_seqno, ExtractValueType(b)); + } + if (a_footer > b_footer) { + r = -1; + } else if (a_footer < b_footer) { + r = +1; + } + } + return r; +} + +// Wrap InternalKeyComparator as a comparator class for ParsedInternalKey. +struct ParsedInternalKeyComparator { + explicit ParsedInternalKeyComparator(const InternalKeyComparator* c) + : cmp(c) {} + + bool operator()(const ParsedInternalKey& a, + const ParsedInternalKey& b) const { + return cmp->Compare(a, b) < 0; + } + + const InternalKeyComparator* cmp; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/dbformat_test.cc b/librocksdb-sys/rocksdb/db/dbformat_test.cc new file mode 100644 index 0000000..3b6190d --- /dev/null +++ b/librocksdb-sys/rocksdb/db/dbformat_test.cc @@ -0,0 +1,343 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/dbformat.h" + +#include "table/block_based/index_builder.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" + +namespace ROCKSDB_NAMESPACE { + +static std::string IKey(const std::string& user_key, uint64_t seq, + ValueType vt) { + std::string encoded; + AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt)); + return encoded; +} + +static std::string Shorten(const std::string& s, const std::string& l) { + std::string result = s; + ShortenedIndexBuilder::FindShortestInternalKeySeparator(*BytewiseComparator(), + &result, l); + return result; +} + +static std::string ShortSuccessor(const std::string& s) { + std::string result = s; + ShortenedIndexBuilder::FindShortInternalKeySuccessor(*BytewiseComparator(), + &result); + return result; +} + +static void TestKey(const std::string& key, uint64_t seq, ValueType vt) { + std::string encoded = IKey(key, seq, vt); + + Slice in(encoded); + ParsedInternalKey decoded("", 0, kTypeValue); + + ASSERT_OK(ParseInternalKey(in, &decoded, true /* log_err_key */)); + ASSERT_EQ(key, decoded.user_key.ToString()); + ASSERT_EQ(seq, decoded.sequence); + ASSERT_EQ(vt, decoded.type); + + ASSERT_NOK(ParseInternalKey(Slice("bar"), &decoded, true /* log_err_key */)); +} + +class FormatTest : public testing::Test {}; + +TEST_F(FormatTest, InternalKey_EncodeDecode) { + const char* keys[] = {"", "k", "hello", "longggggggggggggggggggggg"}; + const uint64_t seq[] = {1, + 2, + 3, + (1ull << 8) - 1, + 1ull << 8, + (1ull << 8) + 1, + (1ull << 16) - 1, + 1ull << 16, + (1ull << 16) + 1, + (1ull << 32) - 1, + 1ull << 32, + (1ull << 32) + 1}; + for (unsigned int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) { + for (unsigned int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) { + TestKey(keys[k], seq[s], kTypeValue); + TestKey("hello", 1, kTypeDeletion); + } + } +} + +TEST_F(FormatTest, InternalKeyShortSeparator) { + // When user keys are same + ASSERT_EQ(IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 99, kTypeValue))); + ASSERT_EQ( + IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 101, kTypeValue))); + ASSERT_EQ( + IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeValue))); + ASSERT_EQ( + IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeDeletion))); + + // When user keys are misordered + ASSERT_EQ(IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), IKey("bar", 99, kTypeValue))); + + // When user keys are different, but correctly ordered + ASSERT_EQ( + IKey("g", kMaxSequenceNumber, kValueTypeForSeek), + Shorten(IKey("foo", 100, kTypeValue), IKey("hello", 200, kTypeValue))); + + ASSERT_EQ(IKey("ABC2", kMaxSequenceNumber, kValueTypeForSeek), + Shorten(IKey("ABC1AAAAA", 100, kTypeValue), + IKey("ABC2ABB", 200, kTypeValue))); + + ASSERT_EQ(IKey("AAA2", kMaxSequenceNumber, kValueTypeForSeek), + Shorten(IKey("AAA1AAA", 100, kTypeValue), + IKey("AAA2AA", 200, kTypeValue))); + + ASSERT_EQ( + IKey("AAA2", kMaxSequenceNumber, kValueTypeForSeek), + Shorten(IKey("AAA1AAA", 100, kTypeValue), IKey("AAA4", 200, kTypeValue))); + + ASSERT_EQ( + IKey("AAA1B", kMaxSequenceNumber, kValueTypeForSeek), + Shorten(IKey("AAA1AAA", 100, kTypeValue), IKey("AAA2", 200, kTypeValue))); + + ASSERT_EQ(IKey("AAA2", kMaxSequenceNumber, kValueTypeForSeek), + Shorten(IKey("AAA1AAA", 100, kTypeValue), + IKey("AAA2A", 200, kTypeValue))); + + ASSERT_EQ( + IKey("AAA1", 100, kTypeValue), + Shorten(IKey("AAA1", 100, kTypeValue), IKey("AAA2", 200, kTypeValue))); + + // When start user key is prefix of limit user key + ASSERT_EQ( + IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), IKey("foobar", 200, kTypeValue))); + + // When limit user key is prefix of start user key + ASSERT_EQ( + IKey("foobar", 100, kTypeValue), + Shorten(IKey("foobar", 100, kTypeValue), IKey("foo", 200, kTypeValue))); +} + +TEST_F(FormatTest, InternalKeyShortestSuccessor) { + ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek), + ShortSuccessor(IKey("foo", 100, kTypeValue))); + ASSERT_EQ(IKey("\xff\xff", 100, kTypeValue), + ShortSuccessor(IKey("\xff\xff", 100, kTypeValue))); +} + +TEST_F(FormatTest, IterKeyOperation) { + IterKey k; + const char p[] = "abcdefghijklmnopqrstuvwxyz"; + const char q[] = "0123456789"; + + ASSERT_EQ(std::string(k.GetUserKey().data(), k.GetUserKey().size()), + std::string("")); + + k.TrimAppend(0, p, 3); + ASSERT_EQ(std::string(k.GetUserKey().data(), k.GetUserKey().size()), + std::string("abc")); + + k.TrimAppend(1, p, 3); + ASSERT_EQ(std::string(k.GetUserKey().data(), k.GetUserKey().size()), + std::string("aabc")); + + k.TrimAppend(0, p, 26); + ASSERT_EQ(std::string(k.GetUserKey().data(), k.GetUserKey().size()), + std::string("abcdefghijklmnopqrstuvwxyz")); + + k.TrimAppend(26, q, 10); + ASSERT_EQ(std::string(k.GetUserKey().data(), k.GetUserKey().size()), + std::string("abcdefghijklmnopqrstuvwxyz0123456789")); + + k.TrimAppend(36, q, 1); + ASSERT_EQ(std::string(k.GetUserKey().data(), k.GetUserKey().size()), + std::string("abcdefghijklmnopqrstuvwxyz01234567890")); + + k.TrimAppend(26, q, 1); + ASSERT_EQ(std::string(k.GetUserKey().data(), k.GetUserKey().size()), + std::string("abcdefghijklmnopqrstuvwxyz0")); + + // Size going up, memory allocation is triggered + k.TrimAppend(27, p, 26); + ASSERT_EQ(std::string(k.GetUserKey().data(), k.GetUserKey().size()), + std::string("abcdefghijklmnopqrstuvwxyz0" + "abcdefghijklmnopqrstuvwxyz")); +} + +TEST_F(FormatTest, IterKeyWithTimestampOperation) { + IterKey k; + k.SetUserKey(""); + const char p[] = "abcdefghijklmnopqrstuvwxyz"; + const char q[] = "0123456789"; + + ASSERT_EQ(std::string(k.GetUserKey().data(), k.GetUserKey().size()), + std::string("")); + + size_t ts_sz = 8; + std::string min_timestamp(ts_sz, static_cast(0)); + k.TrimAppendWithTimestamp(0, p, 3, ts_sz); + ASSERT_EQ(std::string(k.GetUserKey().data(), k.GetUserKey().size()), + "abc" + min_timestamp); + + k.TrimAppendWithTimestamp(1, p, 3, ts_sz); + ASSERT_EQ(std::string(k.GetUserKey().data(), k.GetUserKey().size()), + "aabc" + min_timestamp); + + k.TrimAppendWithTimestamp(0, p, 26, ts_sz); + ASSERT_EQ(std::string(k.GetUserKey().data(), k.GetUserKey().size()), + "abcdefghijklmnopqrstuvwxyz" + min_timestamp); + + k.TrimAppendWithTimestamp(26, q, 10, ts_sz); + ASSERT_EQ(std::string(k.GetUserKey().data(), k.GetUserKey().size()), + "abcdefghijklmnopqrstuvwxyz0123456789" + min_timestamp); + + k.TrimAppendWithTimestamp(36, q, 1, ts_sz); + ASSERT_EQ(std::string(k.GetUserKey().data(), k.GetUserKey().size()), + "abcdefghijklmnopqrstuvwxyz01234567890" + min_timestamp); + + k.TrimAppendWithTimestamp(26, q, 1, ts_sz); + ASSERT_EQ(std::string(k.GetUserKey().data(), k.GetUserKey().size()), + "abcdefghijklmnopqrstuvwxyz0" + min_timestamp); + + k.TrimAppendWithTimestamp(27, p, 26, ts_sz); + ASSERT_EQ(std::string(k.GetUserKey().data(), k.GetUserKey().size()), + "abcdefghijklmnopqrstuvwxyz0" + "abcdefghijklmnopqrstuvwxyz" + + min_timestamp); + // IterKey holds an internal key, the last 8 bytes hold the key footer, the + // timestamp is expected to be added before the key footer. + std::string key_without_ts = "keywithoutts"; + k.SetInternalKey(key_without_ts + min_timestamp + "internal"); + + ASSERT_EQ(std::string(k.GetInternalKey().data(), k.GetInternalKey().size()), + key_without_ts + min_timestamp + "internal"); + k.TrimAppendWithTimestamp(0, p, 10, ts_sz); + ASSERT_EQ(std::string(k.GetInternalKey().data(), k.GetInternalKey().size()), + "ab" + min_timestamp + "cdefghij"); + + k.TrimAppendWithTimestamp(1, p, 8, ts_sz); + ASSERT_EQ(std::string(k.GetInternalKey().data(), k.GetInternalKey().size()), + "a" + min_timestamp + "abcdefgh"); + + k.TrimAppendWithTimestamp(9, p, 3, ts_sz); + ASSERT_EQ(std::string(k.GetInternalKey().data(), k.GetInternalKey().size()), + "aabc" + min_timestamp + "defghabc"); + + k.TrimAppendWithTimestamp(10, q, 10, ts_sz); + ASSERT_EQ(std::string(k.GetInternalKey().data(), k.GetInternalKey().size()), + "aabcdefgha01" + min_timestamp + "23456789"); + + k.TrimAppendWithTimestamp(20, q, 1, ts_sz); + ASSERT_EQ(std::string(k.GetInternalKey().data(), k.GetInternalKey().size()), + "aabcdefgha012" + min_timestamp + "34567890"); + + k.TrimAppendWithTimestamp(21, p, 26, ts_sz); + ASSERT_EQ( + std::string(k.GetInternalKey().data(), k.GetInternalKey().size()), + "aabcdefgha01234567890abcdefghijklmnopqr" + min_timestamp + "stuvwxyz"); +} + +TEST_F(FormatTest, UpdateInternalKey) { + std::string user_key("abcdefghijklmnopqrstuvwxyz"); + uint64_t new_seq = 0x123456; + ValueType new_val_type = kTypeDeletion; + + std::string ikey; + AppendInternalKey(&ikey, ParsedInternalKey(user_key, 100U, kTypeValue)); + size_t ikey_size = ikey.size(); + UpdateInternalKey(&ikey, new_seq, new_val_type); + ASSERT_EQ(ikey_size, ikey.size()); + + Slice in(ikey); + ParsedInternalKey decoded; + ASSERT_OK(ParseInternalKey(in, &decoded, true /* log_err_key */)); + ASSERT_EQ(user_key, decoded.user_key.ToString()); + ASSERT_EQ(new_seq, decoded.sequence); + ASSERT_EQ(new_val_type, decoded.type); +} + +TEST_F(FormatTest, RangeTombstoneSerializeEndKey) { + RangeTombstone t("a", "b", 2); + InternalKey k("b", 3, kTypeValue); + const InternalKeyComparator cmp(BytewiseComparator()); + ASSERT_LT(cmp.Compare(t.SerializeEndKey(), k), 0); +} + +TEST_F(FormatTest, PadInternalKeyWithMinTimestamp) { + std::string orig_user_key = "foo"; + std::string orig_internal_key = IKey(orig_user_key, 100, kTypeValue); + size_t ts_sz = 8; + + std::string key_buf; + PadInternalKeyWithMinTimestamp(&key_buf, orig_internal_key, ts_sz); + ParsedInternalKey key_with_timestamp; + Slice in(key_buf); + ASSERT_OK(ParseInternalKey(in, &key_with_timestamp, true /*log_err_key*/)); + + std::string min_timestamp(ts_sz, static_cast(0)); + ASSERT_EQ(orig_user_key + min_timestamp, key_with_timestamp.user_key); + ASSERT_EQ(100, key_with_timestamp.sequence); + ASSERT_EQ(kTypeValue, key_with_timestamp.type); +} + +TEST_F(FormatTest, StripTimestampFromInternalKey) { + std::string orig_user_key = "foo"; + size_t ts_sz = 8; + std::string timestamp(ts_sz, static_cast(0)); + orig_user_key.append(timestamp.data(), timestamp.size()); + std::string orig_internal_key = IKey(orig_user_key, 100, kTypeValue); + + std::string key_buf; + StripTimestampFromInternalKey(&key_buf, orig_internal_key, ts_sz); + ParsedInternalKey key_without_timestamp; + Slice in(key_buf); + ASSERT_OK(ParseInternalKey(in, &key_without_timestamp, true /*log_err_key*/)); + + ASSERT_EQ("foo", key_without_timestamp.user_key); + ASSERT_EQ(100, key_without_timestamp.sequence); + ASSERT_EQ(kTypeValue, key_without_timestamp.type); +} + +TEST_F(FormatTest, ReplaceInternalKeyWithMinTimestamp) { + std::string orig_user_key = "foo"; + size_t ts_sz = 8; + orig_user_key.append(ts_sz, static_cast(1)); + std::string orig_internal_key = IKey(orig_user_key, 100, kTypeValue); + + std::string key_buf; + ReplaceInternalKeyWithMinTimestamp(&key_buf, orig_internal_key, ts_sz); + ParsedInternalKey new_key; + Slice in(key_buf); + ASSERT_OK(ParseInternalKey(in, &new_key, true /*log_err_key*/)); + + std::string min_timestamp(ts_sz, static_cast(0)); + size_t ukey_diff_offset = new_key.user_key.difference_offset(orig_user_key); + ASSERT_EQ(min_timestamp, + Slice(new_key.user_key.data() + ukey_diff_offset, ts_sz)); + ASSERT_EQ(orig_user_key.size(), new_key.user_key.size()); + ASSERT_EQ(100, new_key.sequence); + ASSERT_EQ(kTypeValue, new_key.type); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/deletefile_test.cc b/librocksdb-sys/rocksdb/db/deletefile_test.cc new file mode 100644 index 0000000..481eda7 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/deletefile_test.cc @@ -0,0 +1,603 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + + +#include + +#include +#include +#include + +#include "db/db_impl/db_impl.h" +#include "db/db_test_util.h" +#include "db/version_set.h" +#include "db/write_batch_internal.h" +#include "file/filename.h" +#include "port/stack_trace.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/transaction_log.h" +#include "test_util/sync_point.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +class DeleteFileTest : public DBTestBase { + public: + const int numlevels_; + const std::string wal_dir_; + + DeleteFileTest() + : DBTestBase("deletefile_test", /*env_do_fsync=*/true), + numlevels_(7), + wal_dir_(dbname_ + "/wal_files") {} + + void SetOptions(Options* options) { + ASSERT_NE(options, nullptr); + options->delete_obsolete_files_period_micros = 0; // always do full purge + options->enable_thread_tracking = true; + options->write_buffer_size = 1024 * 1024 * 1000; + options->target_file_size_base = 1024 * 1024 * 1000; + options->max_bytes_for_level_base = 1024 * 1024 * 1000; + options->WAL_ttl_seconds = 300; // Used to test log files + options->WAL_size_limit_MB = 1024; // Used to test log files + options->wal_dir = wal_dir_; + } + + void AddKeys(int numkeys, int startkey = 0) { + WriteOptions options; + options.sync = false; + ReadOptions roptions; + for (int i = startkey; i < (numkeys + startkey); i++) { + std::string temp = std::to_string(i); + Slice key(temp); + Slice value(temp); + ASSERT_OK(db_->Put(options, key, value)); + } + } + + int numKeysInLevels(std::vector& metadata, + std::vector* keysperlevel = nullptr) { + if (keysperlevel != nullptr) { + keysperlevel->resize(numlevels_); + } + + int numKeys = 0; + for (size_t i = 0; i < metadata.size(); i++) { + int startkey = atoi(metadata[i].smallestkey.c_str()); + int endkey = atoi(metadata[i].largestkey.c_str()); + int numkeysinfile = (endkey - startkey + 1); + numKeys += numkeysinfile; + if (keysperlevel != nullptr) { + (*keysperlevel)[(int)metadata[i].level] += numkeysinfile; + } + fprintf(stderr, "level %d name %s smallest %s largest %s\n", + metadata[i].level, metadata[i].name.c_str(), + metadata[i].smallestkey.c_str(), metadata[i].largestkey.c_str()); + } + return numKeys; + } + + void CreateTwoLevels() { + AddKeys(50000, 10000); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + for (int i = 0; i < 2; ++i) { + ASSERT_OK(dbfull()->TEST_CompactRange(i, nullptr, nullptr)); + } + + AddKeys(50000, 10000); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr)); + } + + void CheckFileTypeCounts(const std::string& dir, int required_log, + int required_sst, int required_manifest) { + std::vector filenames; + ASSERT_OK(env_->GetChildren(dir, &filenames)); + + int log_cnt = 0, sst_cnt = 0, manifest_cnt = 0; + for (auto file : filenames) { + uint64_t number; + FileType type; + if (ParseFileName(file, &number, &type)) { + log_cnt += (type == kWalFile); + sst_cnt += (type == kTableFile); + manifest_cnt += (type == kDescriptorFile); + } + } + if (required_log >= 0) { + ASSERT_EQ(required_log, log_cnt); + } + if (required_sst >= 0) { + ASSERT_EQ(required_sst, sst_cnt); + } + if (required_manifest >= 0) { + ASSERT_EQ(required_manifest, manifest_cnt); + } + } + + static void DoSleep(void* arg) { + auto test = reinterpret_cast(arg); + test->env_->SleepForMicroseconds(2 * 1000 * 1000); + } + + // An empty job to guard all jobs are processed + static void GuardFinish(void* /*arg*/) { + TEST_SYNC_POINT("DeleteFileTest::GuardFinish"); + } +}; + +TEST_F(DeleteFileTest, AddKeysAndQueryLevels) { + Options options = CurrentOptions(); + SetOptions(&options); + Destroy(options); + options.create_if_missing = true; + Reopen(options); + + CreateTwoLevels(); + std::vector metadata; + db_->GetLiveFilesMetaData(&metadata); + + std::string level1file = ""; + int level1keycount = 0; + std::string level2file = ""; + int level2keycount = 0; + int level1index = 0; + int level2index = 1; + + ASSERT_EQ((int)metadata.size(), 2); + if (metadata[0].level == 2) { + level1index = 1; + level2index = 0; + } + + level1file = metadata[level1index].name; + int startkey = atoi(metadata[level1index].smallestkey.c_str()); + int endkey = atoi(metadata[level1index].largestkey.c_str()); + level1keycount = (endkey - startkey + 1); + level2file = metadata[level2index].name; + startkey = atoi(metadata[level2index].smallestkey.c_str()); + endkey = atoi(metadata[level2index].largestkey.c_str()); + level2keycount = (endkey - startkey + 1); + + // COntrolled setup. Levels 1 and 2 should both have 50K files. + // This is a little fragile as it depends on the current + // compaction heuristics. + ASSERT_EQ(level1keycount, 50000); + ASSERT_EQ(level2keycount, 50000); + + Status status = db_->DeleteFile("0.sst"); + ASSERT_TRUE(status.IsInvalidArgument()); + + // intermediate level files cannot be deleted. + status = db_->DeleteFile(level1file); + ASSERT_TRUE(status.IsInvalidArgument()); + + // Lowest level file deletion should succeed. + status = db_->DeleteFile(level2file); + ASSERT_OK(status); +} + +TEST_F(DeleteFileTest, PurgeObsoleteFilesTest) { + Options options = CurrentOptions(); + SetOptions(&options); + Destroy(options); + options.create_if_missing = true; + Reopen(options); + + CreateTwoLevels(); + // there should be only one (empty) log file because CreateTwoLevels() + // flushes the memtables to disk + CheckFileTypeCounts(wal_dir_, 1, 0, 0); + // 2 ssts, 1 manifest + CheckFileTypeCounts(dbname_, 0, 2, 1); + std::string first("0"), last("999999"); + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 2; + Slice first_slice(first), last_slice(last); + ASSERT_OK(db_->CompactRange(compact_options, &first_slice, &last_slice)); + // 1 sst after compaction + CheckFileTypeCounts(dbname_, 0, 1, 1); + + // this time, we keep an iterator alive + Reopen(options); + Iterator* itr = nullptr; + CreateTwoLevels(); + itr = db_->NewIterator(ReadOptions()); + ASSERT_OK(itr->status()); + ASSERT_OK(db_->CompactRange(compact_options, &first_slice, &last_slice)); + ASSERT_OK(itr->status()); + // 3 sst after compaction with live iterator + CheckFileTypeCounts(dbname_, 0, 3, 1); + delete itr; + // 1 sst after iterator deletion + CheckFileTypeCounts(dbname_, 0, 1, 1); +} + +TEST_F(DeleteFileTest, BackgroundPurgeIteratorTest) { + Options options = CurrentOptions(); + SetOptions(&options); + Destroy(options); + options.create_if_missing = true; + Reopen(options); + + std::string first("0"), last("999999"); + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 2; + Slice first_slice(first), last_slice(last); + + // We keep an iterator alive + Iterator* itr = nullptr; + CreateTwoLevels(); + ReadOptions read_options; + read_options.background_purge_on_iterator_cleanup = true; + itr = db_->NewIterator(read_options); + ASSERT_OK(itr->status()); + ASSERT_OK(db_->CompactRange(compact_options, &first_slice, &last_slice)); + // 3 sst after compaction with live iterator + CheckFileTypeCounts(dbname_, 0, 3, 1); + test::SleepingBackgroundTask sleeping_task_before; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, + &sleeping_task_before, Env::Priority::HIGH); + delete itr; + test::SleepingBackgroundTask sleeping_task_after; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, + &sleeping_task_after, Env::Priority::HIGH); + + // Make sure no purges are executed foreground + CheckFileTypeCounts(dbname_, 0, 3, 1); + sleeping_task_before.WakeUp(); + sleeping_task_before.WaitUntilDone(); + + // Make sure all background purges are executed + sleeping_task_after.WakeUp(); + sleeping_task_after.WaitUntilDone(); + // 1 sst after iterator deletion + CheckFileTypeCounts(dbname_, 0, 1, 1); +} + +TEST_F(DeleteFileTest, PurgeDuringOpen) { + Options options = CurrentOptions(); + CheckFileTypeCounts(dbname_, -1, 0, -1); + Close(); + std::unique_ptr file; + ASSERT_OK(options.env->NewWritableFile(dbname_ + "/000002.sst", &file, + EnvOptions())); + ASSERT_OK(file->Close()); + CheckFileTypeCounts(dbname_, -1, 1, -1); + options.avoid_unnecessary_blocking_io = false; + options.create_if_missing = false; + Reopen(options); + CheckFileTypeCounts(dbname_, -1, 0, -1); + Close(); + + // test background purge + options.avoid_unnecessary_blocking_io = true; + options.create_if_missing = false; + ASSERT_OK(options.env->NewWritableFile(dbname_ + "/000002.sst", &file, + EnvOptions())); + ASSERT_OK(file->Close()); + CheckFileTypeCounts(dbname_, -1, 1, -1); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->LoadDependency( + {{"DeleteFileTest::PurgeDuringOpen:1", "DBImpl::BGWorkPurge:start"}}); + SyncPoint::GetInstance()->EnableProcessing(); + Reopen(options); + // the obsolete file is not deleted until the background purge job is ran + CheckFileTypeCounts(dbname_, -1, 1, -1); + TEST_SYNC_POINT("DeleteFileTest::PurgeDuringOpen:1"); + ASSERT_OK(dbfull()->TEST_WaitForPurge()); + CheckFileTypeCounts(dbname_, -1, 0, -1); +} + +TEST_F(DeleteFileTest, BackgroundPurgeCFDropTest) { + Options options = CurrentOptions(); + SetOptions(&options); + Destroy(options); + options.create_if_missing = true; + Reopen(options); + + auto do_test = [&](bool bg_purge) { + ColumnFamilyOptions co; + co.max_write_buffer_size_to_maintain = + static_cast(co.write_buffer_size); + WriteOptions wo; + FlushOptions fo; + ColumnFamilyHandle* cfh = nullptr; + + ASSERT_OK(db_->CreateColumnFamily(co, "dropme", &cfh)); + + ASSERT_OK(db_->Put(wo, cfh, "pika", "chu")); + ASSERT_OK(db_->Flush(fo, cfh)); + // Expect 1 sst file. + CheckFileTypeCounts(dbname_, 0, 1, 1); + + ASSERT_OK(db_->DropColumnFamily(cfh)); + // Still 1 file, it won't be deleted while ColumnFamilyHandle is alive. + CheckFileTypeCounts(dbname_, 0, 1, 1); + + delete cfh; + test::SleepingBackgroundTask sleeping_task_after; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, + &sleeping_task_after, Env::Priority::HIGH); + // If background purge is enabled, the file should still be there. + CheckFileTypeCounts(dbname_, 0, bg_purge ? 1 : 0, 1); + TEST_SYNC_POINT("DeleteFileTest::BackgroundPurgeCFDropTest:1"); + + // Execute background purges. + sleeping_task_after.WakeUp(); + sleeping_task_after.WaitUntilDone(); + // The file should have been deleted. + CheckFileTypeCounts(dbname_, 0, 0, 1); + }; + + { + SCOPED_TRACE("avoid_unnecessary_blocking_io = false"); + do_test(false); + } + + options.avoid_unnecessary_blocking_io = true; + options.create_if_missing = false; + Reopen(options); + ASSERT_OK(dbfull()->TEST_WaitForPurge()); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->LoadDependency( + {{"DeleteFileTest::BackgroundPurgeCFDropTest:1", + "DBImpl::BGWorkPurge:start"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + { + SCOPED_TRACE("avoid_unnecessary_blocking_io = true"); + do_test(true); + } +} + +// This test is to reproduce a bug that read invalid ReadOption in iterator +// cleanup function +TEST_F(DeleteFileTest, BackgroundPurgeCopyOptions) { + Options options = CurrentOptions(); + SetOptions(&options); + Destroy(options); + options.create_if_missing = true; + Reopen(options); + + std::string first("0"), last("999999"); + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 2; + Slice first_slice(first), last_slice(last); + + // We keep an iterator alive + Iterator* itr = nullptr; + CreateTwoLevels(); + { + ReadOptions read_options; + read_options.background_purge_on_iterator_cleanup = true; + itr = db_->NewIterator(read_options); + ASSERT_OK(itr->status()); + // ReadOptions is deleted, but iterator cleanup function should not be + // affected + } + + ASSERT_OK(db_->CompactRange(compact_options, &first_slice, &last_slice)); + // 3 sst after compaction with live iterator + CheckFileTypeCounts(dbname_, 0, 3, 1); + delete itr; + + test::SleepingBackgroundTask sleeping_task_after; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, + &sleeping_task_after, Env::Priority::HIGH); + + // Make sure all background purges are executed + sleeping_task_after.WakeUp(); + sleeping_task_after.WaitUntilDone(); + // 1 sst after iterator deletion + CheckFileTypeCounts(dbname_, 0, 1, 1); +} + +TEST_F(DeleteFileTest, BackgroundPurgeTestMultipleJobs) { + Options options = CurrentOptions(); + SetOptions(&options); + Destroy(options); + options.create_if_missing = true; + Reopen(options); + + std::string first("0"), last("999999"); + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 2; + Slice first_slice(first), last_slice(last); + + // We keep an iterator alive + CreateTwoLevels(); + ReadOptions read_options; + read_options.background_purge_on_iterator_cleanup = true; + Iterator* itr1 = db_->NewIterator(read_options); + ASSERT_OK(itr1->status()); + CreateTwoLevels(); + Iterator* itr2 = db_->NewIterator(read_options); + ASSERT_OK(itr2->status()); + ASSERT_OK(db_->CompactRange(compact_options, &first_slice, &last_slice)); + // 5 sst files after 2 compactions with 2 live iterators + CheckFileTypeCounts(dbname_, 0, 5, 1); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + // ~DBImpl should wait until all BGWorkPurge are finished + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::~DBImpl:WaitJob", "DBImpl::BGWorkPurge"}, + {"DeleteFileTest::GuardFinish", + "DeleteFileTest::BackgroundPurgeTestMultipleJobs:DBClose"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + delete itr1; + env_->Schedule(&DeleteFileTest::DoSleep, this, Env::Priority::HIGH); + delete itr2; + env_->Schedule(&DeleteFileTest::GuardFinish, nullptr, Env::Priority::HIGH); + Close(); + + TEST_SYNC_POINT("DeleteFileTest::BackgroundPurgeTestMultipleJobs:DBClose"); + // 1 sst after iterator deletion + CheckFileTypeCounts(dbname_, 0, 1, 1); +} + +TEST_F(DeleteFileTest, DeleteFileWithIterator) { + Options options = CurrentOptions(); + SetOptions(&options); + Destroy(options); + options.create_if_missing = true; + Reopen(options); + + CreateTwoLevels(); + ReadOptions read_options; + Iterator* it = db_->NewIterator(read_options); + ASSERT_OK(it->status()); + std::vector metadata; + db_->GetLiveFilesMetaData(&metadata); + + std::string level2file; + + ASSERT_EQ(metadata.size(), static_cast(2)); + if (metadata[0].level == 1) { + level2file = metadata[1].name; + } else { + level2file = metadata[0].name; + } + + Status status = db_->DeleteFile(level2file); + fprintf(stdout, "Deletion status %s: %s\n", level2file.c_str(), + status.ToString().c_str()); + ASSERT_OK(status); + it->SeekToFirst(); + int numKeysIterated = 0; + while (it->Valid()) { + numKeysIterated++; + it->Next(); + } + ASSERT_EQ(numKeysIterated, 50000); + delete it; +} + +TEST_F(DeleteFileTest, DeleteLogFiles) { + Options options = CurrentOptions(); + SetOptions(&options); + Destroy(options); + options.create_if_missing = true; + Reopen(options); + + AddKeys(10, 0); + VectorLogPtr logfiles; + ASSERT_OK(db_->GetSortedWalFiles(logfiles)); + ASSERT_GT(logfiles.size(), 0UL); + // Take the last log file which is expected to be alive and try to delete it + // Should not succeed because live logs are not allowed to be deleted + std::unique_ptr alive_log = std::move(logfiles.back()); + ASSERT_EQ(alive_log->Type(), kAliveLogFile); + ASSERT_OK(env_->FileExists(wal_dir_ + "/" + alive_log->PathName())); + fprintf(stdout, "Deleting alive log file %s\n", + alive_log->PathName().c_str()); + ASSERT_NOK(db_->DeleteFile(alive_log->PathName())); + ASSERT_OK(env_->FileExists(wal_dir_ + "/" + alive_log->PathName())); + logfiles.clear(); + + // Call Flush to bring about a new working log file and add more keys + // Call Flush again to flush out memtable and move alive log to archived log + // and try to delete the archived log file + FlushOptions fopts; + ASSERT_OK(db_->Flush(fopts)); + AddKeys(10, 0); + ASSERT_OK(db_->Flush(fopts)); + ASSERT_OK(db_->GetSortedWalFiles(logfiles)); + ASSERT_GT(logfiles.size(), 0UL); + std::unique_ptr archived_log = std::move(logfiles.front()); + ASSERT_EQ(archived_log->Type(), kArchivedLogFile); + ASSERT_OK(env_->FileExists(wal_dir_ + "/" + archived_log->PathName())); + fprintf(stdout, "Deleting archived log file %s\n", + archived_log->PathName().c_str()); + ASSERT_OK(db_->DeleteFile(archived_log->PathName())); + ASSERT_TRUE( + env_->FileExists(wal_dir_ + "/" + archived_log->PathName()).IsNotFound()); +} + +TEST_F(DeleteFileTest, DeleteNonDefaultColumnFamily) { + Options options = CurrentOptions(); + SetOptions(&options); + Destroy(options); + options.create_if_missing = true; + Reopen(options); + CreateAndReopenWithCF({"new_cf"}, options); + + Random rnd(5); + for (int i = 0; i < 1000; ++i) { + ASSERT_OK(db_->Put(WriteOptions(), handles_[1], test::RandomKey(&rnd, 10), + test::RandomKey(&rnd, 10))); + } + ASSERT_OK(db_->Flush(FlushOptions(), handles_[1])); + for (int i = 0; i < 1000; ++i) { + ASSERT_OK(db_->Put(WriteOptions(), handles_[1], test::RandomKey(&rnd, 10), + test::RandomKey(&rnd, 10))); + } + ASSERT_OK(db_->Flush(FlushOptions(), handles_[1])); + + std::vector metadata; + db_->GetLiveFilesMetaData(&metadata); + ASSERT_EQ(2U, metadata.size()); + ASSERT_EQ("new_cf", metadata[0].column_family_name); + ASSERT_EQ("new_cf", metadata[1].column_family_name); + auto old_file = metadata[0].smallest_seqno < metadata[1].smallest_seqno + ? metadata[0].name + : metadata[1].name; + auto new_file = metadata[0].smallest_seqno > metadata[1].smallest_seqno + ? metadata[0].name + : metadata[1].name; + ASSERT_TRUE(db_->DeleteFile(new_file).IsInvalidArgument()); + ASSERT_OK(db_->DeleteFile(old_file)); + + { + std::unique_ptr itr(db_->NewIterator(ReadOptions(), handles_[1])); + ASSERT_OK(itr->status()); + int count = 0; + for (itr->SeekToFirst(); itr->Valid(); itr->Next()) { + ASSERT_OK(itr->status()); + ++count; + } + ASSERT_EQ(count, 1000); + } + + Close(); + ReopenWithColumnFamilies({kDefaultColumnFamilyName, "new_cf"}, options); + + { + std::unique_ptr itr(db_->NewIterator(ReadOptions(), handles_[1])); + int count = 0; + for (itr->SeekToFirst(); itr->Valid(); itr->Next()) { + ASSERT_OK(itr->status()); + ++count; + } + ASSERT_EQ(count, 1000); + } +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/librocksdb-sys/rocksdb/db/error_handler.cc b/librocksdb-sys/rocksdb/db/error_handler.cc new file mode 100644 index 0000000..5582195 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/error_handler.cc @@ -0,0 +1,805 @@ +// Copyright (c) 2018-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#include "db/error_handler.h" + +#include "db/db_impl/db_impl.h" +#include "db/event_helpers.h" +#include "file/sst_file_manager_impl.h" +#include "logging/logging.h" +#include "port/lang.h" + +namespace ROCKSDB_NAMESPACE { + +// Maps to help decide the severity of an error based on the +// BackgroundErrorReason, Code, SubCode and whether db_options.paranoid_checks +// is set or not. There are 3 maps, going from most specific to least specific +// (i.e from all 4 fields in a tuple to only the BackgroundErrorReason and +// paranoid_checks). The less specific map serves as a catch all in case we miss +// a specific error code or subcode. +std::map, + Status::Severity> + ErrorSeverityMap = { + // Errors during BG compaction + {std::make_tuple(BackgroundErrorReason::kCompaction, + Status::Code::kIOError, Status::SubCode::kNoSpace, + true), + Status::Severity::kSoftError}, + {std::make_tuple(BackgroundErrorReason::kCompaction, + Status::Code::kIOError, Status::SubCode::kNoSpace, + false), + Status::Severity::kNoError}, + {std::make_tuple(BackgroundErrorReason::kCompaction, + Status::Code::kIOError, Status::SubCode::kSpaceLimit, + true), + Status::Severity::kHardError}, + {std::make_tuple(BackgroundErrorReason::kCompaction, + Status::Code::kIOError, Status::SubCode::kIOFenced, + true), + Status::Severity::kFatalError}, + {std::make_tuple(BackgroundErrorReason::kCompaction, + Status::Code::kIOError, Status::SubCode::kIOFenced, + false), + Status::Severity::kFatalError}, + // Errors during BG flush + {std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError, + Status::SubCode::kNoSpace, true), + Status::Severity::kHardError}, + {std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError, + Status::SubCode::kNoSpace, false), + Status::Severity::kNoError}, + {std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError, + Status::SubCode::kSpaceLimit, true), + Status::Severity::kHardError}, + {std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError, + Status::SubCode::kIOFenced, true), + Status::Severity::kFatalError}, + {std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError, + Status::SubCode::kIOFenced, false), + Status::Severity::kFatalError}, + // Errors during Write + {std::make_tuple(BackgroundErrorReason::kWriteCallback, + Status::Code::kIOError, Status::SubCode::kNoSpace, + true), + Status::Severity::kHardError}, + {std::make_tuple(BackgroundErrorReason::kWriteCallback, + Status::Code::kIOError, Status::SubCode::kNoSpace, + false), + Status::Severity::kHardError}, + {std::make_tuple(BackgroundErrorReason::kWriteCallback, + Status::Code::kIOError, Status::SubCode::kIOFenced, + true), + Status::Severity::kFatalError}, + {std::make_tuple(BackgroundErrorReason::kWriteCallback, + Status::Code::kIOError, Status::SubCode::kIOFenced, + false), + Status::Severity::kFatalError}, + // Errors during MANIFEST write + {std::make_tuple(BackgroundErrorReason::kManifestWrite, + Status::Code::kIOError, Status::SubCode::kNoSpace, + true), + Status::Severity::kHardError}, + {std::make_tuple(BackgroundErrorReason::kManifestWrite, + Status::Code::kIOError, Status::SubCode::kNoSpace, + false), + Status::Severity::kHardError}, + {std::make_tuple(BackgroundErrorReason::kManifestWrite, + Status::Code::kIOError, Status::SubCode::kIOFenced, + true), + Status::Severity::kFatalError}, + {std::make_tuple(BackgroundErrorReason::kManifestWrite, + Status::Code::kIOError, Status::SubCode::kIOFenced, + false), + Status::Severity::kFatalError}, + // Errors during BG flush with WAL disabled + {std::make_tuple(BackgroundErrorReason::kFlushNoWAL, + Status::Code::kIOError, Status::SubCode::kNoSpace, + true), + Status::Severity::kHardError}, + {std::make_tuple(BackgroundErrorReason::kFlushNoWAL, + Status::Code::kIOError, Status::SubCode::kNoSpace, + false), + Status::Severity::kNoError}, + {std::make_tuple(BackgroundErrorReason::kFlushNoWAL, + Status::Code::kIOError, Status::SubCode::kSpaceLimit, + true), + Status::Severity::kHardError}, + {std::make_tuple(BackgroundErrorReason::kFlushNoWAL, + Status::Code::kIOError, Status::SubCode::kIOFenced, + true), + Status::Severity::kFatalError}, + {std::make_tuple(BackgroundErrorReason::kFlushNoWAL, + Status::Code::kIOError, Status::SubCode::kIOFenced, + false), + Status::Severity::kFatalError}, + // Errors during MANIFEST write when WAL is disabled + {std::make_tuple(BackgroundErrorReason::kManifestWriteNoWAL, + Status::Code::kIOError, Status::SubCode::kNoSpace, + true), + Status::Severity::kHardError}, + {std::make_tuple(BackgroundErrorReason::kManifestWriteNoWAL, + Status::Code::kIOError, Status::SubCode::kNoSpace, + false), + Status::Severity::kHardError}, + {std::make_tuple(BackgroundErrorReason::kManifestWriteNoWAL, + Status::Code::kIOError, Status::SubCode::kIOFenced, + true), + Status::Severity::kFatalError}, + {std::make_tuple(BackgroundErrorReason::kManifestWriteNoWAL, + Status::Code::kIOError, Status::SubCode::kIOFenced, + false), + Status::Severity::kFatalError}, + +}; + +std::map, + Status::Severity> + DefaultErrorSeverityMap = { + // Errors during BG compaction + {std::make_tuple(BackgroundErrorReason::kCompaction, + Status::Code::kCorruption, true), + Status::Severity::kUnrecoverableError}, + {std::make_tuple(BackgroundErrorReason::kCompaction, + Status::Code::kCorruption, false), + Status::Severity::kNoError}, + {std::make_tuple(BackgroundErrorReason::kCompaction, + Status::Code::kIOError, true), + Status::Severity::kFatalError}, + {std::make_tuple(BackgroundErrorReason::kCompaction, + Status::Code::kIOError, false), + Status::Severity::kNoError}, + // Errors during BG flush + {std::make_tuple(BackgroundErrorReason::kFlush, + Status::Code::kCorruption, true), + Status::Severity::kUnrecoverableError}, + {std::make_tuple(BackgroundErrorReason::kFlush, + Status::Code::kCorruption, false), + Status::Severity::kNoError}, + {std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError, + true), + Status::Severity::kFatalError}, + {std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError, + false), + Status::Severity::kNoError}, + // Errors during Write + {std::make_tuple(BackgroundErrorReason::kWriteCallback, + Status::Code::kCorruption, true), + Status::Severity::kUnrecoverableError}, + {std::make_tuple(BackgroundErrorReason::kWriteCallback, + Status::Code::kCorruption, false), + Status::Severity::kNoError}, + {std::make_tuple(BackgroundErrorReason::kWriteCallback, + Status::Code::kIOError, true), + Status::Severity::kFatalError}, + {std::make_tuple(BackgroundErrorReason::kWriteCallback, + Status::Code::kIOError, false), + Status::Severity::kNoError}, + {std::make_tuple(BackgroundErrorReason::kManifestWrite, + Status::Code::kIOError, true), + Status::Severity::kFatalError}, + {std::make_tuple(BackgroundErrorReason::kManifestWrite, + Status::Code::kIOError, false), + Status::Severity::kFatalError}, + // Errors during BG flush with WAL disabled + {std::make_tuple(BackgroundErrorReason::kFlushNoWAL, + Status::Code::kCorruption, true), + Status::Severity::kUnrecoverableError}, + {std::make_tuple(BackgroundErrorReason::kFlushNoWAL, + Status::Code::kCorruption, false), + Status::Severity::kNoError}, + {std::make_tuple(BackgroundErrorReason::kFlushNoWAL, + Status::Code::kIOError, true), + Status::Severity::kFatalError}, + {std::make_tuple(BackgroundErrorReason::kFlushNoWAL, + Status::Code::kIOError, false), + Status::Severity::kNoError}, + {std::make_tuple(BackgroundErrorReason::kManifestWriteNoWAL, + Status::Code::kIOError, true), + Status::Severity::kFatalError}, + {std::make_tuple(BackgroundErrorReason::kManifestWriteNoWAL, + Status::Code::kIOError, false), + Status::Severity::kFatalError}, +}; + +std::map, Status::Severity> + DefaultReasonMap = { + // Errors during BG compaction + {std::make_tuple(BackgroundErrorReason::kCompaction, true), + Status::Severity::kFatalError}, + {std::make_tuple(BackgroundErrorReason::kCompaction, false), + Status::Severity::kNoError}, + // Errors during BG flush + {std::make_tuple(BackgroundErrorReason::kFlush, true), + Status::Severity::kFatalError}, + {std::make_tuple(BackgroundErrorReason::kFlush, false), + Status::Severity::kNoError}, + // Errors during Write + {std::make_tuple(BackgroundErrorReason::kWriteCallback, true), + Status::Severity::kFatalError}, + {std::make_tuple(BackgroundErrorReason::kWriteCallback, false), + Status::Severity::kFatalError}, + // Errors during Memtable update + {std::make_tuple(BackgroundErrorReason::kMemTable, true), + Status::Severity::kFatalError}, + {std::make_tuple(BackgroundErrorReason::kMemTable, false), + Status::Severity::kFatalError}, +}; + +void ErrorHandler::CancelErrorRecovery() { + db_mutex_->AssertHeld(); + + // We'll release the lock before calling sfm, so make sure no new + // recovery gets scheduled at that point + auto_recovery_ = false; + SstFileManagerImpl* sfm = + reinterpret_cast(db_options_.sst_file_manager.get()); + if (sfm) { + // This may or may not cancel a pending recovery + db_mutex_->Unlock(); + bool cancelled = sfm->CancelErrorRecovery(this); + db_mutex_->Lock(); + if (cancelled) { + recovery_in_prog_ = false; + } + } + + // If auto recovery is also runing to resume from the retryable error, + // we should wait and end the auto recovery. + EndAutoRecovery(); +} + +STATIC_AVOID_DESTRUCTION(const Status, kOkStatus){Status::OK()}; + +// This is the main function for looking at an error during a background +// operation and deciding the severity, and error recovery strategy. The high +// level algorithm is as follows - +// 1. Classify the severity of the error based on the ErrorSeverityMap, +// DefaultErrorSeverityMap and DefaultReasonMap defined earlier +// 2. Call a Status code specific override function to adjust the severity +// if needed. The reason for this is our ability to recover may depend on +// the exact options enabled in DBOptions +// 3. Determine if auto recovery is possible. A listener notification callback +// is called, which can disable the auto recovery even if we decide its +// feasible +// 4. For Status::NoSpace() errors, rely on SstFileManagerImpl to control +// the actual recovery. If no sst file manager is specified in DBOptions, +// a default one is allocated during DB::Open(), so there will always be +// one. +// This can also get called as part of a recovery operation. In that case, we +// also track the error separately in recovery_error_ so we can tell in the +// end whether recovery succeeded or not +const Status& ErrorHandler::HandleKnownErrors(const Status& bg_err, + BackgroundErrorReason reason) { + db_mutex_->AssertHeld(); + if (bg_err.ok()) { + return kOkStatus; + } + + if (bg_error_stats_ != nullptr) { + RecordTick(bg_error_stats_.get(), ERROR_HANDLER_BG_ERROR_COUNT); + RecordTick(bg_error_stats_.get(), ERROR_HANDLER_BG_ERROR_COUNT_MISSPELLED); + } + ROCKS_LOG_INFO(db_options_.info_log, + "ErrorHandler: Set regular background error\n"); + + bool paranoid = db_options_.paranoid_checks; + Status::Severity sev = Status::Severity::kFatalError; + Status new_bg_err; + DBRecoverContext context; + bool found = false; + + { + auto entry = ErrorSeverityMap.find( + std::make_tuple(reason, bg_err.code(), bg_err.subcode(), paranoid)); + if (entry != ErrorSeverityMap.end()) { + sev = entry->second; + found = true; + } + } + + if (!found) { + auto entry = DefaultErrorSeverityMap.find( + std::make_tuple(reason, bg_err.code(), paranoid)); + if (entry != DefaultErrorSeverityMap.end()) { + sev = entry->second; + found = true; + } + } + + if (!found) { + auto entry = DefaultReasonMap.find(std::make_tuple(reason, paranoid)); + if (entry != DefaultReasonMap.end()) { + sev = entry->second; + } + } + + new_bg_err = Status(bg_err, sev); + + // Check if recovery is currently in progress. If it is, we will save this + // error so we can check it at the end to see if recovery succeeded or not + if (recovery_in_prog_ && recovery_error_.ok()) { + recovery_error_ = new_bg_err; + } + + bool auto_recovery = auto_recovery_; + if (new_bg_err.severity() >= Status::Severity::kFatalError && auto_recovery) { + auto_recovery = false; + } + + // Allow some error specific overrides + if (new_bg_err.subcode() == IOStatus::SubCode::kNoSpace || + new_bg_err.subcode() == IOStatus::SubCode::kSpaceLimit) { + new_bg_err = OverrideNoSpaceError(new_bg_err, &auto_recovery); + } + + if (!new_bg_err.ok()) { + Status s = new_bg_err; + EventHelpers::NotifyOnBackgroundError(db_options_.listeners, reason, &s, + db_mutex_, &auto_recovery); + if (!s.ok() && (s.severity() > bg_error_.severity())) { + bg_error_ = s; + } else { + // This error is less severe than previously encountered error. Don't + // take any further action + return bg_error_; + } + } + + recover_context_ = context; + if (auto_recovery) { + recovery_in_prog_ = true; + + // Kick-off error specific recovery + if (new_bg_err.subcode() == IOStatus::SubCode::kNoSpace || + new_bg_err.subcode() == IOStatus::SubCode::kSpaceLimit) { + RecoverFromNoSpace(); + } + } + if (bg_error_.severity() >= Status::Severity::kHardError) { + is_db_stopped_.store(true, std::memory_order_release); + } + return bg_error_; +} + +// This is the main function for looking at IO related error during the +// background operations. The main logic is: +// 1) File scope IO error is treated as retryable IO error in the write +// path. In RocksDB, If a file has write IO error and it is at file scope, +// RocksDB never write to the same file again. RocksDB will create a new +// file and rewrite the whole content. Thus, it is retryable. +// 1) if the error is caused by data loss, the error is mapped to +// unrecoverable error. Application/user must take action to handle +// this situation (File scope case is excluded). +// 2) if the error is a Retryable IO error (i.e., it is a file scope IO error, +// or its retryable flag is set and not a data loss error), auto resume +// will be called and the auto resume can be controlled by resume count +// and resume interval options. There are three sub-cases: +// a) if the error happens during compaction, it is mapped to a soft error. +// the compaction thread will reschedule a new compaction. +// b) if the error happens during flush and also WAL is empty, it is mapped +// to a soft error. Note that, it includes the case that IO error happens +// in SST or manifest write during flush. +// c) all other errors are mapped to hard error. +// 3) for other cases, SetBGError(const Status& bg_err, BackgroundErrorReason +// reason) will be called to handle other error cases. +const Status& ErrorHandler::SetBGError(const Status& bg_status, + BackgroundErrorReason reason) { + db_mutex_->AssertHeld(); + Status tmp_status = bg_status; + IOStatus bg_io_err = status_to_io_status(std::move(tmp_status)); + + if (bg_io_err.ok()) { + return kOkStatus; + } + ROCKS_LOG_WARN(db_options_.info_log, "Background IO error %s", + bg_io_err.ToString().c_str()); + + if (recovery_in_prog_ && recovery_io_error_.ok()) { + recovery_io_error_ = bg_io_err; + } + if (BackgroundErrorReason::kManifestWrite == reason || + BackgroundErrorReason::kManifestWriteNoWAL == reason) { + // Always returns ok + ROCKS_LOG_INFO(db_options_.info_log, "Disabling File Deletions"); + db_->DisableFileDeletionsWithLock().PermitUncheckedError(); + } + + Status new_bg_io_err = bg_io_err; + DBRecoverContext context; + if (bg_io_err.GetScope() != IOStatus::IOErrorScope::kIOErrorScopeFile && + bg_io_err.GetDataLoss()) { + // First, data loss (non file scope) is treated as unrecoverable error. So + // it can directly overwrite any existing bg_error_. + bool auto_recovery = false; + Status bg_err(new_bg_io_err, Status::Severity::kUnrecoverableError); + CheckAndSetRecoveryAndBGError(bg_err); + if (bg_error_stats_ != nullptr) { + RecordTick(bg_error_stats_.get(), ERROR_HANDLER_BG_ERROR_COUNT); + RecordTick(bg_error_stats_.get(), + ERROR_HANDLER_BG_ERROR_COUNT_MISSPELLED); + RecordTick(bg_error_stats_.get(), ERROR_HANDLER_BG_IO_ERROR_COUNT); + RecordTick(bg_error_stats_.get(), + ERROR_HANDLER_BG_IO_ERROR_COUNT_MISSPELLED); + } + ROCKS_LOG_INFO( + db_options_.info_log, + "ErrorHandler: Set background IO error as unrecoverable error\n"); + EventHelpers::NotifyOnBackgroundError(db_options_.listeners, reason, + &bg_err, db_mutex_, &auto_recovery); + recover_context_ = context; + return bg_error_; + } else if (bg_io_err.subcode() != IOStatus::SubCode::kNoSpace && + (bg_io_err.GetScope() == + IOStatus::IOErrorScope::kIOErrorScopeFile || + bg_io_err.GetRetryable())) { + // Second, check if the error is a retryable IO error (file scope IO error + // is also treated as retryable IO error in RocksDB write path). if it is + // retryable error and its severity is higher than bg_error_, overwrite the + // bg_error_ with new error. In current stage, for retryable IO error of + // compaction, treat it as soft error. In other cases, treat the retryable + // IO error as hard error. Note that, all the NoSpace error should be + // handled by the SstFileManager::StartErrorRecovery(). Therefore, no matter + // it is retryable or file scope, this logic will be bypassed. + bool auto_recovery = false; + EventHelpers::NotifyOnBackgroundError(db_options_.listeners, reason, + &new_bg_io_err, db_mutex_, + &auto_recovery); + if (bg_error_stats_ != nullptr) { + RecordTick(bg_error_stats_.get(), ERROR_HANDLER_BG_ERROR_COUNT); + RecordTick(bg_error_stats_.get(), + ERROR_HANDLER_BG_ERROR_COUNT_MISSPELLED); + RecordTick(bg_error_stats_.get(), ERROR_HANDLER_BG_IO_ERROR_COUNT); + RecordTick(bg_error_stats_.get(), + ERROR_HANDLER_BG_IO_ERROR_COUNT_MISSPELLED); + RecordTick(bg_error_stats_.get(), + ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT); + RecordTick(bg_error_stats_.get(), + ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT_MISSPELLED); + } + ROCKS_LOG_INFO(db_options_.info_log, + "ErrorHandler: Set background retryable IO error\n"); + if (BackgroundErrorReason::kCompaction == reason) { + // We map the retryable IO error during compaction to soft error. Since + // compaction can reschedule by itself. We will not set the BG error in + // this case + // TODO: a better way to set or clean the retryable IO error which + // happens during compaction SST file write. + if (bg_error_stats_ != nullptr) { + RecordTick(bg_error_stats_.get(), ERROR_HANDLER_AUTORESUME_COUNT); + } + ROCKS_LOG_INFO( + db_options_.info_log, + "ErrorHandler: Compaction will schedule by itself to resume\n"); + return bg_error_; + } else if (BackgroundErrorReason::kFlushNoWAL == reason || + BackgroundErrorReason::kManifestWriteNoWAL == reason) { + // When the BG Retryable IO error reason is flush without WAL, + // We map it to a soft error. At the same time, all the background work + // should be stopped except the BG work from recovery. Therefore, we + // set the soft_error_no_bg_work_ to true. At the same time, since DB + // continues to receive writes when BG error is soft error, to avoid + // to many small memtable being generated during auto resume, the flush + // reason is set to kErrorRecoveryRetryFlush. + Status bg_err(new_bg_io_err, Status::Severity::kSoftError); + CheckAndSetRecoveryAndBGError(bg_err); + soft_error_no_bg_work_ = true; + context.flush_reason = FlushReason::kErrorRecoveryRetryFlush; + recover_context_ = context; + return StartRecoverFromRetryableBGIOError(bg_io_err); + } else { + Status bg_err(new_bg_io_err, Status::Severity::kHardError); + CheckAndSetRecoveryAndBGError(bg_err); + recover_context_ = context; + return StartRecoverFromRetryableBGIOError(bg_io_err); + } + } else { + if (bg_error_stats_ != nullptr) { + RecordTick(bg_error_stats_.get(), ERROR_HANDLER_BG_IO_ERROR_COUNT); + RecordTick(bg_error_stats_.get(), + ERROR_HANDLER_BG_IO_ERROR_COUNT_MISSPELLED); + } + // HandleKnownErrors() will use recovery_error_, so ignore + // recovery_io_error_. + // TODO: Do some refactoring and use only one recovery_error_ + recovery_io_error_.PermitUncheckedError(); + return HandleKnownErrors(new_bg_io_err, reason); + } +} + +Status ErrorHandler::OverrideNoSpaceError(const Status& bg_error, + bool* auto_recovery) { + if (bg_error.severity() >= Status::Severity::kFatalError) { + return bg_error; + } + + if (db_options_.sst_file_manager.get() == nullptr) { + // We rely on SFM to poll for enough disk space and recover + *auto_recovery = false; + return bg_error; + } + + if (db_options_.allow_2pc && + (bg_error.severity() <= Status::Severity::kSoftError)) { + // Don't know how to recover, as the contents of the current WAL file may + // be inconsistent, and it may be needed for 2PC. If 2PC is not enabled, + // we can just flush the memtable and discard the log + *auto_recovery = false; + return Status(bg_error, Status::Severity::kFatalError); + } + + { + uint64_t free_space; + if (db_options_.env->GetFreeSpace(db_options_.db_paths[0].path, + &free_space) == Status::NotSupported()) { + *auto_recovery = false; + } + } + + return bg_error; +} + +void ErrorHandler::RecoverFromNoSpace() { + SstFileManagerImpl* sfm = + reinterpret_cast(db_options_.sst_file_manager.get()); + + // Inform SFM of the error, so it can kick-off the recovery + if (sfm) { + sfm->StartErrorRecovery(this, bg_error_); + } +} + +Status ErrorHandler::ClearBGError() { + db_mutex_->AssertHeld(); + + // Signal that recovery succeeded + if (recovery_error_.ok()) { + Status old_bg_error = bg_error_; + // old_bg_error is only for notifying listeners, so may not be checked + old_bg_error.PermitUncheckedError(); + // Clear and check the recovery IO and BG error + bg_error_ = Status::OK(); + recovery_io_error_ = IOStatus::OK(); + bg_error_.PermitUncheckedError(); + recovery_io_error_.PermitUncheckedError(); + recovery_in_prog_ = false; + soft_error_no_bg_work_ = false; + EventHelpers::NotifyOnErrorRecoveryEnd(db_options_.listeners, old_bg_error, + bg_error_, db_mutex_); + } + return recovery_error_; +} + +Status ErrorHandler::RecoverFromBGError(bool is_manual) { + InstrumentedMutexLock l(db_mutex_); + bool no_bg_work_original_flag = soft_error_no_bg_work_; + if (is_manual) { + // If its a manual recovery and there's a background recovery in progress + // return busy status + if (recovery_in_prog_) { + return Status::Busy(); + } + recovery_in_prog_ = true; + + // In manual resume, we allow the bg work to run. If it is a auto resume, + // the bg work should follow this tag. + soft_error_no_bg_work_ = false; + + // In manual resume, if the bg error is a soft error and also requires + // no bg work, the error must be recovered by call the flush with + // flush reason: kErrorRecoveryRetryFlush. In other case, the flush + // reason is set to kErrorRecovery. + if (no_bg_work_original_flag) { + recover_context_.flush_reason = FlushReason::kErrorRecoveryRetryFlush; + } else { + recover_context_.flush_reason = FlushReason::kErrorRecovery; + } + } + + if (bg_error_.severity() == Status::Severity::kSoftError && + recover_context_.flush_reason == FlushReason::kErrorRecovery) { + // Simply clear the background error and return + recovery_error_ = Status::OK(); + return ClearBGError(); + } + + // Reset recovery_error_. We will use this to record any errors that happen + // during the recovery process. While recovering, the only operations that + // can generate background errors should be the flush operations + recovery_error_ = Status::OK(); + recovery_error_.PermitUncheckedError(); + Status s = db_->ResumeImpl(recover_context_); + if (s.ok()) { + soft_error_no_bg_work_ = false; + } else { + soft_error_no_bg_work_ = no_bg_work_original_flag; + } + + // For manual recover, shutdown, and fatal error cases, set + // recovery_in_prog_ to false. For automatic background recovery, leave it + // as is regardless of success or failure as it will be retried + if (is_manual || s.IsShutdownInProgress() || + bg_error_.severity() >= Status::Severity::kFatalError) { + recovery_in_prog_ = false; + } + return s; +} + +const Status& ErrorHandler::StartRecoverFromRetryableBGIOError( + const IOStatus& io_error) { + db_mutex_->AssertHeld(); + if (bg_error_.ok()) { + return bg_error_; + } else if (io_error.ok()) { + return kOkStatus; + } else if (db_options_.max_bgerror_resume_count <= 0 || recovery_in_prog_) { + // Auto resume BG error is not enabled, directly return bg_error_. + return bg_error_; + } + if (bg_error_stats_ != nullptr) { + RecordTick(bg_error_stats_.get(), ERROR_HANDLER_AUTORESUME_COUNT); + } + ROCKS_LOG_INFO( + db_options_.info_log, + "ErrorHandler: Call StartRecoverFromRetryableBGIOError to resume\n"); + if (recovery_thread_) { + // In this case, if recovery_in_prog_ is false, current thread should + // wait the previous recover thread to finish and create a new thread + // to recover from the bg error. + db_mutex_->Unlock(); + recovery_thread_->join(); + db_mutex_->Lock(); + } + + recovery_in_prog_ = true; + recovery_thread_.reset( + new port::Thread(&ErrorHandler::RecoverFromRetryableBGIOError, this)); + + if (recovery_io_error_.ok() && recovery_error_.ok()) { + return recovery_error_; + } else { + return bg_error_; + } +} + +// Automatic recover from Retryable BG IO error. Must be called after db +// mutex is released. +void ErrorHandler::RecoverFromRetryableBGIOError() { + TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeStart"); + InstrumentedMutexLock l(db_mutex_); + if (end_recovery_) { + EventHelpers::NotifyOnErrorRecoveryEnd(db_options_.listeners, bg_error_, + Status::ShutdownInProgress(), + db_mutex_); + return; + } + DBRecoverContext context = recover_context_; + int resume_count = db_options_.max_bgerror_resume_count; + uint64_t wait_interval = db_options_.bgerror_resume_retry_interval; + uint64_t retry_count = 0; + // Recover from the retryable error. Create a separate thread to do it. + while (resume_count > 0) { + if (end_recovery_) { + EventHelpers::NotifyOnErrorRecoveryEnd(db_options_.listeners, bg_error_, + Status::ShutdownInProgress(), + db_mutex_); + return; + } + TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeResume0"); + TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeResume1"); + recovery_io_error_ = IOStatus::OK(); + recovery_error_ = Status::OK(); + retry_count++; + Status s = db_->ResumeImpl(context); + if (bg_error_stats_ != nullptr) { + RecordTick(bg_error_stats_.get(), + ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT); + } + if (s.IsShutdownInProgress() || + bg_error_.severity() >= Status::Severity::kFatalError) { + // If DB shutdown in progress or the error severity is higher than + // Hard Error, stop auto resume and returns. + recovery_in_prog_ = false; + if (bg_error_stats_ != nullptr) { + RecordInHistogram(bg_error_stats_.get(), + ERROR_HANDLER_AUTORESUME_RETRY_COUNT, retry_count); + } + EventHelpers::NotifyOnErrorRecoveryEnd(db_options_.listeners, bg_error_, + bg_error_, db_mutex_); + return; + } + if (!recovery_io_error_.ok() && + recovery_error_.severity() <= Status::Severity::kHardError && + recovery_io_error_.GetRetryable()) { + // If new BG IO error happens during auto recovery and it is retryable + // and its severity is Hard Error or lower, the auto resmue sleep for + // a period of time and redo auto resume if it is allowed. + TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeWait0"); + TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeWait1"); + int64_t wait_until = db_options_.clock->NowMicros() + wait_interval; + cv_.TimedWait(wait_until); + } else { + // There are three possibility: 1) recover_io_error is set during resume + // and the error is not retryable, 2) recover is successful, 3) other + // error happens during resume and cannot be resumed here. + if (recovery_io_error_.ok() && recovery_error_.ok() && s.ok()) { + // recover from the retryable IO error and no other BG errors. Clean + // the bg_error and notify user. + TEST_SYNC_POINT("RecoverFromRetryableBGIOError:RecoverSuccess"); + Status old_bg_error = bg_error_; + is_db_stopped_.store(false, std::memory_order_release); + bg_error_ = Status::OK(); + bg_error_.PermitUncheckedError(); + EventHelpers::NotifyOnErrorRecoveryEnd( + db_options_.listeners, old_bg_error, bg_error_, db_mutex_); + if (bg_error_stats_ != nullptr) { + RecordTick(bg_error_stats_.get(), + ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT); + RecordInHistogram(bg_error_stats_.get(), + ERROR_HANDLER_AUTORESUME_RETRY_COUNT, retry_count); + } + recovery_in_prog_ = false; + if (soft_error_no_bg_work_) { + soft_error_no_bg_work_ = false; + } + return; + } else { + // In this case: 1) recovery_io_error is more serious or not retryable + // 2) other Non IO recovery_error happens. The auto recovery stops. + recovery_in_prog_ = false; + if (bg_error_stats_ != nullptr) { + RecordInHistogram(bg_error_stats_.get(), + ERROR_HANDLER_AUTORESUME_RETRY_COUNT, retry_count); + } + EventHelpers::NotifyOnErrorRecoveryEnd( + db_options_.listeners, bg_error_, + !recovery_io_error_.ok() + ? recovery_io_error_ + : (!recovery_error_.ok() ? recovery_error_ : s), + db_mutex_); + return; + } + } + resume_count--; + } + recovery_in_prog_ = false; + EventHelpers::NotifyOnErrorRecoveryEnd( + db_options_.listeners, bg_error_, + Status::Aborted("Exceeded resume retry count"), db_mutex_); + TEST_SYNC_POINT("RecoverFromRetryableBGIOError:LoopOut"); + if (bg_error_stats_ != nullptr) { + RecordInHistogram(bg_error_stats_.get(), + ERROR_HANDLER_AUTORESUME_RETRY_COUNT, retry_count); + } + return; +} + +void ErrorHandler::CheckAndSetRecoveryAndBGError(const Status& bg_err) { + if (recovery_in_prog_ && recovery_error_.ok()) { + recovery_error_ = bg_err; + } + if (bg_err.severity() > bg_error_.severity()) { + bg_error_ = bg_err; + } + if (bg_error_.severity() >= Status::Severity::kHardError) { + is_db_stopped_.store(true, std::memory_order_release); + } + return; +} + +void ErrorHandler::EndAutoRecovery() { + db_mutex_->AssertHeld(); + if (!end_recovery_) { + end_recovery_ = true; + } + cv_.SignalAll(); + db_mutex_->Unlock(); + if (recovery_thread_) { + recovery_thread_->join(); + } + db_mutex_->Lock(); + return; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/error_handler.h b/librocksdb-sys/rocksdb/db/error_handler.h new file mode 100644 index 0000000..34e08a5 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/error_handler.h @@ -0,0 +1,124 @@ +// Copyright (c) 2018-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include "monitoring/instrumented_mutex.h" +#include "options/db_options.h" +#include "rocksdb/io_status.h" +#include "rocksdb/listener.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +class DBImpl; + +// This structure is used to store the DB recovery context. The context is +// the information that related to the recover actions. For example, it contains +// FlushReason, which tells the flush job why this flush is called. +struct DBRecoverContext { + FlushReason flush_reason; + + DBRecoverContext() : flush_reason(FlushReason::kErrorRecovery) {} + + DBRecoverContext(FlushReason reason) : flush_reason(reason) {} +}; + +class ErrorHandler { + public: + ErrorHandler(DBImpl* db, const ImmutableDBOptions& db_options, + InstrumentedMutex* db_mutex) + : db_(db), + db_options_(db_options), + cv_(db_mutex), + end_recovery_(false), + recovery_thread_(nullptr), + db_mutex_(db_mutex), + auto_recovery_(false), + recovery_in_prog_(false), + soft_error_no_bg_work_(false), + is_db_stopped_(false), + bg_error_stats_(db_options.statistics) { + // Clear the checked flag for uninitialized errors + bg_error_.PermitUncheckedError(); + recovery_error_.PermitUncheckedError(); + recovery_io_error_.PermitUncheckedError(); + } + + void EnableAutoRecovery() { auto_recovery_ = true; } + + Status::Severity GetErrorSeverity(BackgroundErrorReason reason, + Status::Code code, Status::SubCode subcode); + + const Status& SetBGError(const Status& bg_err, BackgroundErrorReason reason); + + Status GetBGError() const { return bg_error_; } + + Status GetRecoveryError() const { return recovery_error_; } + + Status ClearBGError(); + + bool IsDBStopped() { return is_db_stopped_.load(std::memory_order_acquire); } + + bool IsBGWorkStopped() { + assert(db_mutex_); + db_mutex_->AssertHeld(); + return !bg_error_.ok() && + (bg_error_.severity() >= Status::Severity::kHardError || + !auto_recovery_ || soft_error_no_bg_work_); + } + + bool IsSoftErrorNoBGWork() { return soft_error_no_bg_work_; } + + bool IsRecoveryInProgress() { return recovery_in_prog_; } + + Status RecoverFromBGError(bool is_manual = false); + void CancelErrorRecovery(); + + void EndAutoRecovery(); + + private: + DBImpl* db_; + const ImmutableDBOptions& db_options_; + Status bg_error_; + // A separate Status variable used to record any errors during the + // recovery process from hard errors + Status recovery_error_; + // A separate IO Status variable used to record any IO errors during + // the recovery process. At the same time, recovery_error_ is also set. + IOStatus recovery_io_error_; + // The condition variable used with db_mutex during auto resume for time + // wait. + InstrumentedCondVar cv_; + bool end_recovery_; + std::unique_ptr recovery_thread_; + + InstrumentedMutex* db_mutex_; + // A flag indicating whether automatic recovery from errors is enabled + bool auto_recovery_; + bool recovery_in_prog_; + // A flag to indicate that for the soft error, we should not allow any + // background work except the work is from recovery. + bool soft_error_no_bg_work_; + + // Used to store the context for recover, such as flush reason. + DBRecoverContext recover_context_; + std::atomic is_db_stopped_; + + // The pointer of DB statistics. + std::shared_ptr bg_error_stats_; + + const Status& HandleKnownErrors(const Status& bg_err, + BackgroundErrorReason reason); + Status OverrideNoSpaceError(const Status& bg_error, bool* auto_recovery); + void RecoverFromNoSpace(); + const Status& StartRecoverFromRetryableBGIOError(const IOStatus& io_error); + void RecoverFromRetryableBGIOError(); + // First, if it is in recovery and the recovery_error is ok. Set the + // recovery_error_ to bg_err. Second, if the severity is higher than the + // current bg_error_, overwrite it. + void CheckAndSetRecoveryAndBGError(const Status& bg_err); +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/error_handler_fs_test.cc b/librocksdb-sys/rocksdb/db/error_handler_fs_test.cc new file mode 100644 index 0000000..442631d --- /dev/null +++ b/librocksdb-sys/rocksdb/db/error_handler_fs_test.cc @@ -0,0 +1,2862 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/db_test_util.h" +#include "file/sst_file_manager_impl.h" +#include "port/stack_trace.h" +#include "rocksdb/io_status.h" +#include "rocksdb/sst_file_manager.h" +#include "test_util/sync_point.h" +#include "util/random.h" +#include "utilities/fault_injection_env.h" +#include "utilities/fault_injection_fs.h" + +namespace ROCKSDB_NAMESPACE { + +class DBErrorHandlingFSTest : public DBTestBase { + public: + DBErrorHandlingFSTest() + : DBTestBase("db_error_handling_fs_test", /*env_do_fsync=*/true) { + fault_fs_.reset(new FaultInjectionTestFS(env_->GetFileSystem())); + fault_env_.reset(new CompositeEnvWrapper(env_, fault_fs_)); + } + + std::string GetManifestNameFromLiveFiles() { + std::vector live_files; + uint64_t manifest_size; + + Status s = dbfull()->GetLiveFiles(live_files, &manifest_size, false); + if (!s.ok()) { + return ""; + } + for (auto& file : live_files) { + uint64_t num = 0; + FileType type; + if (ParseFileName(file, &num, &type) && type == kDescriptorFile) { + return file; + } + } + return ""; + } + + std::shared_ptr fault_fs_; + std::unique_ptr fault_env_; +}; + +class ErrorHandlerFSListener : public EventListener { + public: + ErrorHandlerFSListener() + : mutex_(), + cv_(&mutex_), + no_auto_recovery_(false), + recovery_complete_(false), + file_creation_started_(false), + override_bg_error_(false), + file_count_(0), + fault_fs_(nullptr) {} + ~ErrorHandlerFSListener() { + file_creation_error_.PermitUncheckedError(); + bg_error_.PermitUncheckedError(); + new_bg_error_.PermitUncheckedError(); + } + + void OnTableFileCreationStarted( + const TableFileCreationBriefInfo& /*ti*/) override { + InstrumentedMutexLock l(&mutex_); + file_creation_started_ = true; + if (file_count_ > 0) { + if (--file_count_ == 0) { + fault_fs_->SetFilesystemActive(false, file_creation_error_); + file_creation_error_ = IOStatus::OK(); + } + } + cv_.SignalAll(); + } + + void OnErrorRecoveryBegin(BackgroundErrorReason /*reason*/, Status bg_error, + bool* auto_recovery) override { + bg_error.PermitUncheckedError(); + if (*auto_recovery && no_auto_recovery_) { + *auto_recovery = false; + } + } + + void OnErrorRecoveryEnd(const BackgroundErrorRecoveryInfo& info) override { + InstrumentedMutexLock l(&mutex_); + recovery_complete_ = true; + cv_.SignalAll(); + new_bg_error_ = info.new_bg_error; + } + + bool WaitForRecovery(uint64_t /*abs_time_us*/) { + InstrumentedMutexLock l(&mutex_); + while (!recovery_complete_) { + cv_.Wait(/*abs_time_us*/); + } + if (recovery_complete_) { + recovery_complete_ = false; + return true; + } + return false; + } + + void WaitForTableFileCreationStarted(uint64_t /*abs_time_us*/) { + InstrumentedMutexLock l(&mutex_); + while (!file_creation_started_) { + cv_.Wait(/*abs_time_us*/); + } + file_creation_started_ = false; + } + + void OnBackgroundError(BackgroundErrorReason /*reason*/, + Status* bg_error) override { + if (override_bg_error_) { + *bg_error = bg_error_; + override_bg_error_ = false; + } + } + + void EnableAutoRecovery(bool enable = true) { no_auto_recovery_ = !enable; } + + void OverrideBGError(Status bg_err) { + bg_error_ = bg_err; + override_bg_error_ = true; + } + + void InjectFileCreationError(FaultInjectionTestFS* fs, int file_count, + IOStatus io_s) { + fault_fs_ = fs; + file_count_ = file_count; + file_creation_error_ = io_s; + } + + Status new_bg_error() { return new_bg_error_; } + + private: + InstrumentedMutex mutex_; + InstrumentedCondVar cv_; + bool no_auto_recovery_; + bool recovery_complete_; + bool file_creation_started_; + bool override_bg_error_; + int file_count_; + IOStatus file_creation_error_; + Status bg_error_; + Status new_bg_error_; + FaultInjectionTestFS* fault_fs_; +}; + +TEST_F(DBErrorHandlingFSTest, FLushWriteError) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.statistics = CreateDBStatistics(); + Status s; + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + + ASSERT_OK(Put(Key(0), "val")); + SyncPoint::GetInstance()->SetCallBack("FlushJob::Start", [&](void*) { + fault_fs_->SetFilesystemActive(false, IOStatus::NoSpace("Out of space")); + }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + s = dbfull()->Resume(); + ASSERT_OK(s); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_ERROR_COUNT)); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_IO_ERROR_COUNT)); + ASSERT_EQ(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT)); + ASSERT_EQ(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_COUNT)); + ASSERT_EQ(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT)); + ASSERT_EQ(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT)); + + Reopen(options); + ASSERT_EQ("val", Get(Key(0))); + Destroy(options); +} + +// All the NoSpace IOError will be handled as the regular BG Error no matter the +// retryable flag is set of not. So the auto resume for retryable IO Error will +// not be triggered. Also, it is mapped as hard error. +TEST_F(DBErrorHandlingFSTest, FLushWriteNoSpaceError) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 2; + options.bgerror_resume_retry_interval = 100000; // 0.1 second + options.statistics = CreateDBStatistics(); + Status s; + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + + IOStatus error_msg = IOStatus::NoSpace("Retryable IO Error"); + error_msg.SetRetryable(true); + + ASSERT_OK(Put(Key(1), "val1")); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeFinishBuildTable", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + s = dbfull()->Resume(); + ASSERT_OK(s); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_ERROR_COUNT)); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_IO_ERROR_COUNT)); + ASSERT_EQ(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT)); + ASSERT_EQ(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_COUNT)); + ASSERT_EQ(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT)); + ASSERT_EQ(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT)); + Destroy(options); +} + +TEST_F(DBErrorHandlingFSTest, FLushWriteRetryableError) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 0; + options.statistics = CreateDBStatistics(); + Status s; + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + ASSERT_OK(Put(Key(1), "val1")); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeFinishBuildTable", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + s = dbfull()->Resume(); + ASSERT_OK(s); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_ERROR_COUNT)); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_IO_ERROR_COUNT)); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT)); + ASSERT_EQ(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_COUNT)); + ASSERT_EQ(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT)); + ASSERT_EQ(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT)); + Reopen(options); + ASSERT_EQ("val1", Get(Key(1))); + + ASSERT_OK(Put(Key(2), "val2")); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeSyncTable", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + s = dbfull()->Resume(); + ASSERT_OK(s); + Reopen(options); + ASSERT_EQ("val2", Get(Key(2))); + + ASSERT_OK(Put(Key(3), "val3")); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeCloseTableFile", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + s = dbfull()->Resume(); + ASSERT_OK(s); + Reopen(options); + ASSERT_EQ("val3", Get(Key(3))); + + Destroy(options); +} + +TEST_F(DBErrorHandlingFSTest, FLushWriteFileScopeError) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 0; + Status s; + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + + IOStatus error_msg = IOStatus::IOError("File Scope Data Loss Error"); + error_msg.SetDataLoss(true); + error_msg.SetScope( + ROCKSDB_NAMESPACE::IOStatus::IOErrorScope::kIOErrorScopeFile); + error_msg.SetRetryable(false); + + ASSERT_OK(Put(Key(1), "val1")); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeFinishBuildTable", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + s = dbfull()->Resume(); + ASSERT_OK(s); + Reopen(options); + ASSERT_EQ("val1", Get(Key(1))); + + ASSERT_OK(Put(Key(2), "val2")); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeSyncTable", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + s = dbfull()->Resume(); + ASSERT_OK(s); + Reopen(options); + ASSERT_EQ("val2", Get(Key(2))); + + ASSERT_OK(Put(Key(3), "val3")); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeCloseTableFile", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + s = dbfull()->Resume(); + ASSERT_OK(s); + Reopen(options); + ASSERT_EQ("val3", Get(Key(3))); + + // not file scope, but retyrable set + error_msg.SetDataLoss(false); + error_msg.SetScope( + ROCKSDB_NAMESPACE::IOStatus::IOErrorScope::kIOErrorScopeFileSystem); + error_msg.SetRetryable(true); + + ASSERT_OK(Put(Key(3), "val3")); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeCloseTableFile", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + s = dbfull()->Resume(); + ASSERT_OK(s); + Reopen(options); + ASSERT_EQ("val3", Get(Key(3))); + + Destroy(options); +} + +TEST_F(DBErrorHandlingFSTest, FLushWALWriteRetryableError) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 0; + Status s; + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + listener->EnableAutoRecovery(false); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::SyncClosedLogs:Start", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + SyncPoint::GetInstance()->EnableProcessing(); + + CreateAndReopenWithCF({"pikachu, sdfsdfsdf"}, options); + + WriteOptions wo = WriteOptions(); + wo.disableWAL = false; + ASSERT_OK(Put(Key(1), "val1", wo)); + + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + auto cfh = dbfull()->GetColumnFamilyHandle(1); + s = dbfull()->DropColumnFamily(cfh); + + s = dbfull()->Resume(); + ASSERT_OK(s); + ASSERT_EQ("val1", Get(Key(1))); + ASSERT_OK(Put(Key(3), "val3", wo)); + ASSERT_EQ("val3", Get(Key(3))); + s = Flush(); + ASSERT_OK(s); + ASSERT_EQ("val3", Get(Key(3))); + + Destroy(options); +} + +TEST_F(DBErrorHandlingFSTest, FLushWALAtomicWriteRetryableError) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 0; + options.atomic_flush = true; + Status s; + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + listener->EnableAutoRecovery(false); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::SyncClosedLogs:Start", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + SyncPoint::GetInstance()->EnableProcessing(); + + CreateAndReopenWithCF({"pikachu, sdfsdfsdf"}, options); + + WriteOptions wo = WriteOptions(); + wo.disableWAL = false; + ASSERT_OK(Put(Key(1), "val1", wo)); + + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + auto cfh = dbfull()->GetColumnFamilyHandle(1); + s = dbfull()->DropColumnFamily(cfh); + + s = dbfull()->Resume(); + ASSERT_OK(s); + ASSERT_EQ("val1", Get(Key(1))); + ASSERT_OK(Put(Key(3), "val3", wo)); + ASSERT_EQ("val3", Get(Key(3))); + s = Flush(); + ASSERT_OK(s); + ASSERT_EQ("val3", Get(Key(3))); + + Destroy(options); +} + +// The flush error is injected before we finish the table build +TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError1) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 0; + options.statistics = CreateDBStatistics(); + Status s; + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + WriteOptions wo = WriteOptions(); + wo.disableWAL = true; + ASSERT_OK(Put(Key(1), "val1", wo)); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeFinishBuildTable", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_OK(Put(Key(2), "val2", wo)); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + ASSERT_EQ("val2", Get(Key(2))); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + s = dbfull()->Resume(); + ASSERT_OK(s); + ASSERT_EQ("val1", Get(Key(1))); + ASSERT_EQ("val2", Get(Key(2))); + ASSERT_OK(Put(Key(3), "val3", wo)); + ASSERT_EQ("val3", Get(Key(3))); + s = Flush(); + ASSERT_OK(s); + ASSERT_EQ("val3", Get(Key(3))); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_ERROR_COUNT)); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_IO_ERROR_COUNT)); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT)); + ASSERT_EQ(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_COUNT)); + ASSERT_EQ(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT)); + ASSERT_EQ(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT)); + + Destroy(options); +} + +// The retryable IO error is injected before we sync table +TEST_F(DBErrorHandlingFSTest, FLushWriteNoWALRetryableError2) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 0; + Status s; + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + WriteOptions wo = WriteOptions(); + wo.disableWAL = true; + + ASSERT_OK(Put(Key(1), "val1", wo)); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeSyncTable", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_OK(Put(Key(2), "val2", wo)); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + ASSERT_EQ("val2", Get(Key(2))); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + s = dbfull()->Resume(); + ASSERT_OK(s); + ASSERT_EQ("val1", Get(Key(1))); + ASSERT_EQ("val2", Get(Key(2))); + ASSERT_OK(Put(Key(3), "val3", wo)); + ASSERT_EQ("val3", Get(Key(3))); + s = Flush(); + ASSERT_OK(s); + ASSERT_EQ("val3", Get(Key(3))); + + Destroy(options); +} + +// The retryable IO error is injected before we close the table file +TEST_F(DBErrorHandlingFSTest, FLushWriteNoWALRetryableError3) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 0; + Status s; + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + WriteOptions wo = WriteOptions(); + wo.disableWAL = true; + + ASSERT_OK(Put(Key(1), "val1", wo)); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeCloseTableFile", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_OK(Put(Key(2), "val2", wo)); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + ASSERT_EQ("val2", Get(Key(2))); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + s = dbfull()->Resume(); + ASSERT_OK(s); + ASSERT_EQ("val1", Get(Key(1))); + ASSERT_EQ("val2", Get(Key(2))); + ASSERT_OK(Put(Key(3), "val3", wo)); + ASSERT_EQ("val3", Get(Key(3))); + s = Flush(); + ASSERT_OK(s); + ASSERT_EQ("val3", Get(Key(3))); + + Destroy(options); +} + +TEST_F(DBErrorHandlingFSTest, ManifestWriteError) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + Status s; + std::string old_manifest; + std::string new_manifest; + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + old_manifest = GetManifestNameFromLiveFiles(); + + ASSERT_OK(Put(Key(0), "val")); + ASSERT_OK(Flush()); + ASSERT_OK(Put(Key(1), "val")); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:WriteManifest", [&](void*) { + fault_fs_->SetFilesystemActive(false, + IOStatus::NoSpace("Out of space")); + }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + s = dbfull()->Resume(); + ASSERT_OK(s); + + new_manifest = GetManifestNameFromLiveFiles(); + ASSERT_NE(new_manifest, old_manifest); + + Reopen(options); + ASSERT_EQ("val", Get(Key(0))); + ASSERT_EQ("val", Get(Key(1))); + Close(); +} + +TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableError) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 0; + Status s; + std::string old_manifest; + std::string new_manifest; + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + old_manifest = GetManifestNameFromLiveFiles(); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + ASSERT_OK(Put(Key(0), "val")); + ASSERT_OK(Flush()); + ASSERT_OK(Put(Key(1), "val")); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:WriteManifest", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + s = dbfull()->Resume(); + ASSERT_OK(s); + + new_manifest = GetManifestNameFromLiveFiles(); + ASSERT_NE(new_manifest, old_manifest); + + Reopen(options); + ASSERT_EQ("val", Get(Key(0))); + ASSERT_EQ("val", Get(Key(1))); + Close(); +} + +TEST_F(DBErrorHandlingFSTest, ManifestWriteFileScopeError) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 0; + Status s; + std::string old_manifest; + std::string new_manifest; + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + old_manifest = GetManifestNameFromLiveFiles(); + + IOStatus error_msg = IOStatus::IOError("File Scope Data Loss Error"); + error_msg.SetDataLoss(true); + error_msg.SetScope( + ROCKSDB_NAMESPACE::IOStatus::IOErrorScope::kIOErrorScopeFile); + error_msg.SetRetryable(false); + + ASSERT_OK(Put(Key(0), "val")); + ASSERT_OK(Flush()); + ASSERT_OK(Put(Key(1), "val")); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:WriteManifest", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + s = dbfull()->Resume(); + ASSERT_OK(s); + + new_manifest = GetManifestNameFromLiveFiles(); + ASSERT_NE(new_manifest, old_manifest); + + Reopen(options); + ASSERT_EQ("val", Get(Key(0))); + ASSERT_EQ("val", Get(Key(1))); + Close(); +} + +TEST_F(DBErrorHandlingFSTest, ManifestWriteNoWALRetryableError) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 0; + Status s; + std::string old_manifest; + std::string new_manifest; + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + old_manifest = GetManifestNameFromLiveFiles(); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + WriteOptions wo = WriteOptions(); + wo.disableWAL = true; + ASSERT_OK(Put(Key(0), "val", wo)); + ASSERT_OK(Flush()); + ASSERT_OK(Put(Key(1), "val", wo)); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:WriteManifest", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + s = dbfull()->Resume(); + ASSERT_OK(s); + + new_manifest = GetManifestNameFromLiveFiles(); + ASSERT_NE(new_manifest, old_manifest); + + Reopen(options); + ASSERT_EQ("val", Get(Key(0))); + ASSERT_EQ("val", Get(Key(1))); + Close(); +} + +TEST_F(DBErrorHandlingFSTest, DoubleManifestWriteError) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + Status s; + std::string old_manifest; + std::string new_manifest; + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + old_manifest = GetManifestNameFromLiveFiles(); + + ASSERT_OK(Put(Key(0), "val")); + ASSERT_OK(Flush()); + ASSERT_OK(Put(Key(1), "val")); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:WriteManifest", [&](void*) { + fault_fs_->SetFilesystemActive(false, + IOStatus::NoSpace("Out of space")); + }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); + fault_fs_->SetFilesystemActive(true); + + // This Resume() will attempt to create a new manifest file and fail again + s = dbfull()->Resume(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); + fault_fs_->SetFilesystemActive(true); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); + + // A successful Resume() will create a new manifest file + s = dbfull()->Resume(); + ASSERT_OK(s); + + new_manifest = GetManifestNameFromLiveFiles(); + ASSERT_NE(new_manifest, old_manifest); + + Reopen(options); + ASSERT_EQ("val", Get(Key(0))); + ASSERT_EQ("val", Get(Key(1))); + Close(); +} + +TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteError) { + if (mem_env_ != nullptr) { + ROCKSDB_GTEST_SKIP("Test requires non-mock environment"); + return; + } + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.level0_file_num_compaction_trigger = 2; + options.listeners.emplace_back(listener); + Status s; + std::string old_manifest; + std::string new_manifest; + std::atomic fail_manifest(false); + DestroyAndReopen(options); + old_manifest = GetManifestNameFromLiveFiles(); + + ASSERT_OK(Put(Key(0), "val")); + ASSERT_OK(Put(Key(2), "val")); + s = Flush(); + ASSERT_OK(s); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + // Wait for flush of 2nd L0 file before starting compaction + {{"DBImpl::FlushMemTable:FlushMemTableFinished", + "BackgroundCallCompaction:0"}, + // Wait for compaction to detect manifest write error + {"BackgroundCallCompaction:1", "CompactionManifestWriteError:0"}, + // Make compaction thread wait for error to be cleared + {"CompactionManifestWriteError:1", + "DBImpl::BackgroundCallCompaction:FoundObsoleteFiles"}, + // Wait for DB instance to clear bg_error before calling + // TEST_WaitForCompact + {"SstFileManagerImpl::ErrorCleared", "CompactionManifestWriteError:2"}}); + // trigger manifest write failure in compaction thread + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "BackgroundCallCompaction:0", [&](void*) { fail_manifest.store(true); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:WriteManifest", [&](void*) { + if (fail_manifest.load()) { + fault_fs_->SetFilesystemActive(false, + IOStatus::NoSpace("Out of space")); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put(Key(1), "val")); + // This Flush will trigger a compaction, which will fail when appending to + // the manifest + s = Flush(); + ASSERT_OK(s); + + TEST_SYNC_POINT("CompactionManifestWriteError:0"); + // Clear all errors so when the compaction is retried, it will succeed + fault_fs_->SetFilesystemActive(true); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + TEST_SYNC_POINT("CompactionManifestWriteError:1"); + TEST_SYNC_POINT("CompactionManifestWriteError:2"); + + s = dbfull()->TEST_WaitForCompact(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ASSERT_OK(s); + + new_manifest = GetManifestNameFromLiveFiles(); + ASSERT_NE(new_manifest, old_manifest); + Reopen(options); + ASSERT_EQ("val", Get(Key(0))); + ASSERT_EQ("val", Get(Key(1))); + ASSERT_EQ("val", Get(Key(2))); + Close(); +} + +TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteRetryableError) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.level0_file_num_compaction_trigger = 2; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 0; + Status s; + std::string old_manifest; + std::string new_manifest; + std::atomic fail_manifest(false); + DestroyAndReopen(options); + old_manifest = GetManifestNameFromLiveFiles(); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + ASSERT_OK(Put(Key(0), "val")); + ASSERT_OK(Put(Key(2), "val")); + s = Flush(); + ASSERT_OK(s); + + listener->OverrideBGError(Status(error_msg, Status::Severity::kHardError)); + listener->EnableAutoRecovery(false); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + // Wait for flush of 2nd L0 file before starting compaction + {{"DBImpl::FlushMemTable:FlushMemTableFinished", + "BackgroundCallCompaction:0"}, + // Wait for compaction to detect manifest write error + {"BackgroundCallCompaction:1", "CompactionManifestWriteError:0"}, + // Make compaction thread wait for error to be cleared + {"CompactionManifestWriteError:1", + "DBImpl::BackgroundCallCompaction:FoundObsoleteFiles"}}); + // trigger manifest write failure in compaction thread + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "BackgroundCallCompaction:0", [&](void*) { fail_manifest.store(true); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:WriteManifest", [&](void*) { + if (fail_manifest.load()) { + fault_fs_->SetFilesystemActive(false, error_msg); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put(Key(1), "val")); + s = Flush(); + ASSERT_OK(s); + + TEST_SYNC_POINT("CompactionManifestWriteError:0"); + TEST_SYNC_POINT("CompactionManifestWriteError:1"); + + s = dbfull()->TEST_WaitForCompact(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); + + fault_fs_->SetFilesystemActive(true); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); + s = dbfull()->Resume(); + ASSERT_OK(s); + + new_manifest = GetManifestNameFromLiveFiles(); + ASSERT_NE(new_manifest, old_manifest); + + Reopen(options); + ASSERT_EQ("val", Get(Key(0))); + ASSERT_EQ("val", Get(Key(1))); + ASSERT_EQ("val", Get(Key(2))); + Close(); +} + +TEST_F(DBErrorHandlingFSTest, CompactionWriteError) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.level0_file_num_compaction_trigger = 2; + options.listeners.emplace_back(listener); + Status s; + DestroyAndReopen(options); + + ASSERT_OK(Put(Key(0), "va;")); + ASSERT_OK(Put(Key(2), "va;")); + s = Flush(); + ASSERT_OK(s); + + listener->OverrideBGError( + Status(Status::NoSpace(), Status::Severity::kHardError)); + listener->EnableAutoRecovery(false); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::FlushMemTable:FlushMemTableFinished", + "BackgroundCallCompaction:0"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "BackgroundCallCompaction:0", [&](void*) { + fault_fs_->SetFilesystemActive(false, + IOStatus::NoSpace("Out of space")); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put(Key(1), "val")); + s = Flush(); + ASSERT_OK(s); + + s = dbfull()->TEST_WaitForCompact(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); + + fault_fs_->SetFilesystemActive(true); + s = dbfull()->Resume(); + ASSERT_OK(s); + Destroy(options); +} + +TEST_F(DBErrorHandlingFSTest, DISABLED_CompactionWriteRetryableError) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.level0_file_num_compaction_trigger = 2; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 0; + Status s; + DestroyAndReopen(options); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + ASSERT_OK(Put(Key(0), "va;")); + ASSERT_OK(Put(Key(2), "va;")); + s = Flush(); + ASSERT_OK(s); + + listener->OverrideBGError(Status(error_msg, Status::Severity::kHardError)); + listener->EnableAutoRecovery(false); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::FlushMemTable:FlushMemTableFinished", + "BackgroundCallCompaction:0"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::OpenCompactionOutputFile", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:Finish", + [&](void*) { CancelAllBackgroundWork(dbfull()); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put(Key(1), "val")); + s = Flush(); + ASSERT_OK(s); + + s = dbfull()->TEST_GetBGError(); + ASSERT_OK(s); + fault_fs_->SetFilesystemActive(true); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); + s = dbfull()->Resume(); + ASSERT_OK(s); + Destroy(options); +} + +TEST_F(DBErrorHandlingFSTest, DISABLED_CompactionWriteFileScopeError) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.level0_file_num_compaction_trigger = 2; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 0; + Status s; + DestroyAndReopen(options); + + IOStatus error_msg = IOStatus::IOError("File Scope Data Loss Error"); + error_msg.SetDataLoss(true); + error_msg.SetScope( + ROCKSDB_NAMESPACE::IOStatus::IOErrorScope::kIOErrorScopeFile); + error_msg.SetRetryable(false); + + ASSERT_OK(Put(Key(0), "va;")); + ASSERT_OK(Put(Key(2), "va;")); + s = Flush(); + ASSERT_OK(s); + + listener->OverrideBGError(Status(error_msg, Status::Severity::kHardError)); + listener->EnableAutoRecovery(false); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::FlushMemTable:FlushMemTableFinished", + "BackgroundCallCompaction:0"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::OpenCompactionOutputFile", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:Finish", + [&](void*) { CancelAllBackgroundWork(dbfull()); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put(Key(1), "val")); + s = Flush(); + ASSERT_OK(s); + + s = dbfull()->TEST_GetBGError(); + ASSERT_OK(s); + + fault_fs_->SetFilesystemActive(true); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); + s = dbfull()->Resume(); + ASSERT_OK(s); + Destroy(options); +} + +TEST_F(DBErrorHandlingFSTest, CorruptionError) { + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.level0_file_num_compaction_trigger = 2; + Status s; + DestroyAndReopen(options); + + ASSERT_OK(Put(Key(0), "va;")); + ASSERT_OK(Put(Key(2), "va;")); + s = Flush(); + ASSERT_OK(s); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::FlushMemTable:FlushMemTableFinished", + "BackgroundCallCompaction:0"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "BackgroundCallCompaction:0", [&](void*) { + fault_fs_->SetFilesystemActive(false, + IOStatus::Corruption("Corruption")); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put(Key(1), "val")); + s = Flush(); + ASSERT_OK(s); + + s = dbfull()->TEST_WaitForCompact(); + ASSERT_EQ(s.severity(), + ROCKSDB_NAMESPACE::Status::Severity::kUnrecoverableError); + + fault_fs_->SetFilesystemActive(true); + s = dbfull()->Resume(); + ASSERT_NOK(s); + Destroy(options); +} + +TEST_F(DBErrorHandlingFSTest, AutoRecoverFlushError) { + if (mem_env_ != nullptr) { + ROCKSDB_GTEST_SKIP("Test requires non-mock environment"); + return; + } + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.statistics = CreateDBStatistics(); + Status s; + + listener->EnableAutoRecovery(); + DestroyAndReopen(options); + + ASSERT_OK(Put(Key(0), "val")); + SyncPoint::GetInstance()->SetCallBack("FlushJob::Start", [&](void*) { + fault_fs_->SetFilesystemActive(false, IOStatus::NoSpace("Out of space")); + }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + ASSERT_EQ(listener->WaitForRecovery(5000000), true); + + s = Put(Key(1), "val"); + ASSERT_OK(s); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_ERROR_COUNT)); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_IO_ERROR_COUNT)); + ASSERT_EQ(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT)); + ASSERT_EQ(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_COUNT)); + ASSERT_EQ(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT)); + ASSERT_EQ(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT)); + + Reopen(options); + ASSERT_EQ("val", Get(Key(0))); + ASSERT_EQ("val", Get(Key(1))); + Destroy(options); +} + +TEST_F(DBErrorHandlingFSTest, FailRecoverFlushError) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + Status s; + + listener->EnableAutoRecovery(); + DestroyAndReopen(options); + + ASSERT_OK(Put(Key(0), "val")); + SyncPoint::GetInstance()->SetCallBack("FlushJob::Start", [&](void*) { + fault_fs_->SetFilesystemActive(false, IOStatus::NoSpace("Out of space")); + }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); + // We should be able to shutdown the database while auto recovery is going + // on in the background + Close(); + DestroyDB(dbname_, options).PermitUncheckedError(); +} + +TEST_F(DBErrorHandlingFSTest, WALWriteError) { + if (mem_env_ != nullptr) { + ROCKSDB_GTEST_SKIP("Test requires non-mock environment"); + return; + } + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.writable_file_max_buffer_size = 32768; + options.listeners.emplace_back(listener); + Status s; + Random rnd(301); + + listener->EnableAutoRecovery(); + DestroyAndReopen(options); + + { + WriteBatch batch; + + for (auto i = 0; i < 100; ++i) { + ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024))); + } + + WriteOptions wopts; + wopts.sync = true; + ASSERT_OK(dbfull()->Write(wopts, &batch)); + }; + + { + WriteBatch batch; + int write_error = 0; + + for (auto i = 100; i < 199; ++i) { + ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024))); + } + + SyncPoint::GetInstance()->SetCallBack( + "WritableFileWriter::Append:BeforePrepareWrite", [&](void*) { + write_error++; + if (write_error > 2) { + fault_fs_->SetFilesystemActive(false, + IOStatus::NoSpace("Out of space")); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + WriteOptions wopts; + wopts.sync = true; + s = dbfull()->Write(wopts, &batch); + ASSERT_EQ(s, s.NoSpace()); + } + SyncPoint::GetInstance()->DisableProcessing(); + // `ClearAllCallBacks()` is needed in addition to `DisableProcessing()` to + // drain all callbacks. Otherwise, a pending callback in the background + // could re-disable `fault_fs_` after we enable it below. + SyncPoint::GetInstance()->ClearAllCallBacks(); + fault_fs_->SetFilesystemActive(true); + ASSERT_EQ(listener->WaitForRecovery(5000000), true); + for (auto i = 0; i < 199; ++i) { + if (i < 100) { + ASSERT_NE(Get(Key(i)), "NOT_FOUND"); + } else { + ASSERT_EQ(Get(Key(i)), "NOT_FOUND"); + } + } + Reopen(options); + for (auto i = 0; i < 199; ++i) { + if (i < 100) { + ASSERT_NE(Get(Key(i)), "NOT_FOUND"); + } else { + ASSERT_EQ(Get(Key(i)), "NOT_FOUND"); + } + } + Close(); +} + +TEST_F(DBErrorHandlingFSTest, WALWriteRetryableError) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.writable_file_max_buffer_size = 32768; + options.listeners.emplace_back(listener); + options.paranoid_checks = true; + options.max_bgerror_resume_count = 0; + Random rnd(301); + + DestroyAndReopen(options); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + // For the first batch, write is successful, require sync + { + WriteBatch batch; + + for (auto i = 0; i < 100; ++i) { + ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024))); + } + + WriteOptions wopts; + wopts.sync = true; + ASSERT_OK(dbfull()->Write(wopts, &batch)); + }; + + // For the second batch, the first 2 file Append are successful, then the + // following Append fails due to file system retryable IOError. + { + WriteBatch batch; + int write_error = 0; + + for (auto i = 100; i < 200; ++i) { + ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024))); + } + + SyncPoint::GetInstance()->SetCallBack( + "WritableFileWriter::Append:BeforePrepareWrite", [&](void*) { + write_error++; + if (write_error > 2) { + fault_fs_->SetFilesystemActive(false, error_msg); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + WriteOptions wopts; + wopts.sync = true; + Status s = dbfull()->Write(wopts, &batch); + ASSERT_TRUE(s.IsIOError()); + } + fault_fs_->SetFilesystemActive(true); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); + + // Data in corrupted WAL are not stored + for (auto i = 0; i < 199; ++i) { + if (i < 100) { + ASSERT_NE(Get(Key(i)), "NOT_FOUND"); + } else { + ASSERT_EQ(Get(Key(i)), "NOT_FOUND"); + } + } + + // Resume and write a new batch, should be in the WAL + ASSERT_OK(dbfull()->Resume()); + { + WriteBatch batch; + + for (auto i = 200; i < 300; ++i) { + ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024))); + } + + WriteOptions wopts; + wopts.sync = true; + ASSERT_OK(dbfull()->Write(wopts, &batch)); + }; + + Reopen(options); + for (auto i = 0; i < 300; ++i) { + if (i < 100 || i >= 200) { + ASSERT_NE(Get(Key(i)), "NOT_FOUND"); + } else { + ASSERT_EQ(Get(Key(i)), "NOT_FOUND"); + } + } + Close(); +} + +TEST_F(DBErrorHandlingFSTest, MultiCFWALWriteError) { + if (mem_env_ != nullptr) { + ROCKSDB_GTEST_SKIP("Test requires non-mock environment"); + return; + } + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.writable_file_max_buffer_size = 32768; + options.listeners.emplace_back(listener); + Random rnd(301); + + listener->EnableAutoRecovery(); + CreateAndReopenWithCF({"one", "two", "three"}, options); + + { + WriteBatch batch; + + for (auto i = 1; i < 4; ++i) { + for (auto j = 0; j < 100; ++j) { + ASSERT_OK(batch.Put(handles_[i], Key(j), rnd.RandomString(1024))); + } + } + + WriteOptions wopts; + wopts.sync = true; + ASSERT_OK(dbfull()->Write(wopts, &batch)); + }; + + { + WriteBatch batch; + int write_error = 0; + + // Write to one CF + for (auto i = 100; i < 199; ++i) { + ASSERT_OK(batch.Put(handles_[2], Key(i), rnd.RandomString(1024))); + } + + SyncPoint::GetInstance()->SetCallBack( + "WritableFileWriter::Append:BeforePrepareWrite", [&](void*) { + write_error++; + if (write_error > 2) { + fault_fs_->SetFilesystemActive(false, + IOStatus::NoSpace("Out of space")); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + WriteOptions wopts; + wopts.sync = true; + Status s = dbfull()->Write(wopts, &batch); + ASSERT_TRUE(s.IsNoSpace()); + } + SyncPoint::GetInstance()->DisableProcessing(); + // `ClearAllCallBacks()` is needed in addition to `DisableProcessing()` to + // drain all callbacks. Otherwise, a pending callback in the background + // could re-disable `fault_fs_` after we enable it below. + SyncPoint::GetInstance()->ClearAllCallBacks(); + fault_fs_->SetFilesystemActive(true); + ASSERT_EQ(listener->WaitForRecovery(5000000), true); + + for (auto i = 1; i < 4; ++i) { + // Every CF should have been flushed + ASSERT_EQ(NumTableFilesAtLevel(0, i), 1); + } + + for (auto i = 1; i < 4; ++i) { + for (auto j = 0; j < 199; ++j) { + if (j < 100) { + ASSERT_NE(Get(i, Key(j)), "NOT_FOUND"); + } else { + ASSERT_EQ(Get(i, Key(j)), "NOT_FOUND"); + } + } + } + ReopenWithColumnFamilies({"default", "one", "two", "three"}, options); + for (auto i = 1; i < 4; ++i) { + for (auto j = 0; j < 199; ++j) { + if (j < 100) { + ASSERT_NE(Get(i, Key(j)), "NOT_FOUND"); + } else { + ASSERT_EQ(Get(i, Key(j)), "NOT_FOUND"); + } + } + } + Close(); +} + +TEST_F(DBErrorHandlingFSTest, MultiDBCompactionError) { + if (mem_env_ != nullptr) { + ROCKSDB_GTEST_SKIP("Test requires non-mock environment"); + return; + } + FaultInjectionTestEnv* def_env = new FaultInjectionTestEnv(env_); + std::vector> fault_envs; + std::vector fault_fs; + std::vector options; + std::vector> listener; + std::vector db; + std::shared_ptr sfm(NewSstFileManager(def_env)); + int kNumDbInstances = 3; + Random rnd(301); + + for (auto i = 0; i < kNumDbInstances; ++i) { + listener.emplace_back(new ErrorHandlerFSListener()); + options.emplace_back(GetDefaultOptions()); + fault_fs.emplace_back(new FaultInjectionTestFS(env_->GetFileSystem())); + std::shared_ptr fs(fault_fs.back()); + fault_envs.emplace_back(new CompositeEnvWrapper(def_env, fs)); + options[i].env = fault_envs.back().get(); + options[i].create_if_missing = true; + options[i].level0_file_num_compaction_trigger = 2; + options[i].writable_file_max_buffer_size = 32768; + options[i].listeners.emplace_back(listener[i]); + options[i].sst_file_manager = sfm; + DB* dbptr; + char buf[16]; + + listener[i]->EnableAutoRecovery(); + // Setup for returning error for the 3rd SST, which would be level 1 + listener[i]->InjectFileCreationError(fault_fs[i], 3, + IOStatus::NoSpace("Out of space")); + snprintf(buf, sizeof(buf), "_%d", i); + ASSERT_OK(DestroyDB(dbname_ + std::string(buf), options[i])); + ASSERT_OK(DB::Open(options[i], dbname_ + std::string(buf), &dbptr)); + db.emplace_back(dbptr); + } + + for (auto i = 0; i < kNumDbInstances; ++i) { + WriteBatch batch; + + for (auto j = 0; j <= 100; ++j) { + ASSERT_OK(batch.Put(Key(j), rnd.RandomString(1024))); + } + + WriteOptions wopts; + wopts.sync = true; + ASSERT_OK(db[i]->Write(wopts, &batch)); + ASSERT_OK(db[i]->Flush(FlushOptions())); + } + + def_env->SetFilesystemActive(false, Status::NoSpace("Out of space")); + for (auto i = 0; i < kNumDbInstances; ++i) { + WriteBatch batch; + + // Write to one CF + for (auto j = 100; j < 199; ++j) { + ASSERT_OK(batch.Put(Key(j), rnd.RandomString(1024))); + } + + WriteOptions wopts; + wopts.sync = true; + ASSERT_OK(db[i]->Write(wopts, &batch)); + ASSERT_OK(db[i]->Flush(FlushOptions())); + } + + for (auto i = 0; i < kNumDbInstances; ++i) { + Status s = static_cast(db[i])->TEST_WaitForCompact(); + ASSERT_EQ(s.severity(), Status::Severity::kSoftError); + fault_fs[i]->SetFilesystemActive(true); + } + + def_env->SetFilesystemActive(true); + for (auto i = 0; i < kNumDbInstances; ++i) { + std::string prop; + ASSERT_EQ(listener[i]->WaitForRecovery(5000000), true); + ASSERT_OK(static_cast(db[i])->TEST_WaitForCompact()); + EXPECT_TRUE(db[i]->GetProperty( + "rocksdb.num-files-at-level" + std::to_string(0), &prop)); + EXPECT_EQ(atoi(prop.c_str()), 0); + EXPECT_TRUE(db[i]->GetProperty( + "rocksdb.num-files-at-level" + std::to_string(1), &prop)); + EXPECT_EQ(atoi(prop.c_str()), 1); + } + + SstFileManagerImpl* sfmImpl = + static_cast_with_check(sfm.get()); + sfmImpl->Close(); + + for (auto i = 0; i < kNumDbInstances; ++i) { + char buf[16]; + snprintf(buf, sizeof(buf), "_%d", i); + delete db[i]; + fault_fs[i]->SetFilesystemActive(true); + if (getenv("KEEP_DB")) { + printf("DB is still at %s%s\n", dbname_.c_str(), buf); + } else { + ASSERT_OK(DestroyDB(dbname_ + std::string(buf), options[i])); + } + } + options.clear(); + sfm.reset(); + delete def_env; +} + +TEST_F(DBErrorHandlingFSTest, MultiDBVariousErrors) { + if (mem_env_ != nullptr) { + ROCKSDB_GTEST_SKIP("Test requires non-mock environment"); + return; + } + FaultInjectionTestEnv* def_env = new FaultInjectionTestEnv(env_); + std::vector> fault_envs; + std::vector fault_fs; + std::vector options; + std::vector> listener; + std::vector db; + std::shared_ptr sfm(NewSstFileManager(def_env)); + int kNumDbInstances = 3; + Random rnd(301); + + for (auto i = 0; i < kNumDbInstances; ++i) { + listener.emplace_back(new ErrorHandlerFSListener()); + options.emplace_back(GetDefaultOptions()); + fault_fs.emplace_back(new FaultInjectionTestFS(env_->GetFileSystem())); + std::shared_ptr fs(fault_fs.back()); + fault_envs.emplace_back(new CompositeEnvWrapper(def_env, fs)); + options[i].env = fault_envs.back().get(); + options[i].create_if_missing = true; + options[i].level0_file_num_compaction_trigger = 2; + options[i].writable_file_max_buffer_size = 32768; + options[i].listeners.emplace_back(listener[i]); + options[i].sst_file_manager = sfm; + DB* dbptr; + char buf[16]; + + listener[i]->EnableAutoRecovery(); + switch (i) { + case 0: + // Setup for returning error for the 3rd SST, which would be level 1 + listener[i]->InjectFileCreationError(fault_fs[i], 3, + IOStatus::NoSpace("Out of space")); + break; + case 1: + // Setup for returning error after the 1st SST, which would result + // in a hard error + listener[i]->InjectFileCreationError(fault_fs[i], 2, + IOStatus::NoSpace("Out of space")); + break; + default: + break; + } + snprintf(buf, sizeof(buf), "_%d", i); + ASSERT_OK(DestroyDB(dbname_ + std::string(buf), options[i])); + ASSERT_OK(DB::Open(options[i], dbname_ + std::string(buf), &dbptr)); + db.emplace_back(dbptr); + } + + for (auto i = 0; i < kNumDbInstances; ++i) { + WriteBatch batch; + + for (auto j = 0; j <= 100; ++j) { + ASSERT_OK(batch.Put(Key(j), rnd.RandomString(1024))); + } + + WriteOptions wopts; + wopts.sync = true; + ASSERT_OK(db[i]->Write(wopts, &batch)); + ASSERT_OK(db[i]->Flush(FlushOptions())); + } + + def_env->SetFilesystemActive(false, Status::NoSpace("Out of space")); + for (auto i = 0; i < kNumDbInstances; ++i) { + WriteBatch batch; + + // Write to one CF + for (auto j = 100; j < 199; ++j) { + ASSERT_OK(batch.Put(Key(j), rnd.RandomString(1024))); + } + + WriteOptions wopts; + wopts.sync = true; + ASSERT_OK(db[i]->Write(wopts, &batch)); + if (i != 1) { + ASSERT_OK(db[i]->Flush(FlushOptions())); + } else { + ASSERT_TRUE(db[i]->Flush(FlushOptions()).IsNoSpace()); + } + } + + for (auto i = 0; i < kNumDbInstances; ++i) { + Status s = static_cast(db[i])->TEST_WaitForCompact(); + switch (i) { + case 0: + ASSERT_EQ(s.severity(), Status::Severity::kSoftError); + break; + case 1: + ASSERT_EQ(s.severity(), Status::Severity::kHardError); + break; + case 2: + ASSERT_OK(s); + break; + } + fault_fs[i]->SetFilesystemActive(true); + } + + def_env->SetFilesystemActive(true); + for (auto i = 0; i < kNumDbInstances; ++i) { + std::string prop; + if (i < 2) { + ASSERT_EQ(listener[i]->WaitForRecovery(5000000), true); + } + if (i == 1) { + ASSERT_OK(static_cast(db[i])->TEST_WaitForCompact()); + } + EXPECT_TRUE(db[i]->GetProperty( + "rocksdb.num-files-at-level" + std::to_string(0), &prop)); + EXPECT_EQ(atoi(prop.c_str()), 0); + EXPECT_TRUE(db[i]->GetProperty( + "rocksdb.num-files-at-level" + std::to_string(1), &prop)); + EXPECT_EQ(atoi(prop.c_str()), 1); + } + + SstFileManagerImpl* sfmImpl = + static_cast_with_check(sfm.get()); + sfmImpl->Close(); + + for (auto i = 0; i < kNumDbInstances; ++i) { + char buf[16]; + snprintf(buf, sizeof(buf), "_%d", i); + fault_fs[i]->SetFilesystemActive(true); + delete db[i]; + if (getenv("KEEP_DB")) { + printf("DB is still at %s%s\n", dbname_.c_str(), buf); + } else { + EXPECT_OK(DestroyDB(dbname_ + std::string(buf), options[i])); + } + } + options.clear(); + delete def_env; +} + +// When Put the KV-pair, the write option is set to disable WAL. +// If retryable error happens in this condition, map the bg error +// to soft error and trigger auto resume. During auto resume, SwitchMemtable +// is disabled to avoid small SST tables. Write can still be applied before +// the bg error is cleaned unless the memtable is full. +TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover1) { + // Activate the FS before the first resume + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 2; + options.bgerror_resume_retry_interval = 100000; // 0.1 second + options.statistics = CreateDBStatistics(); + Status s; + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + WriteOptions wo = WriteOptions(); + wo.disableWAL = true; + ASSERT_OK(Put(Key(1), "val1", wo)); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"RecoverFromRetryableBGIOError:LoopOut", + "FLushWritNoWALRetryableeErrorAutoRecover1:1"}}); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeFinishBuildTable", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ("val1", Get(Key(1))); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + TEST_SYNC_POINT("FLushWritNoWALRetryableeErrorAutoRecover1:1"); + ASSERT_EQ("val1", Get(Key(1))); + ASSERT_EQ("val1", Get(Key(1))); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + ASSERT_EQ(3, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_ERROR_COUNT)); + ASSERT_EQ(3, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_IO_ERROR_COUNT)); + ASSERT_EQ(3, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT)); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_COUNT)); + ASSERT_LE(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT)); + ASSERT_LE(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT)); + HistogramData autoresume_retry; + options.statistics->histogramData(ERROR_HANDLER_AUTORESUME_RETRY_COUNT, + &autoresume_retry); + ASSERT_GE(autoresume_retry.max, 0); + ASSERT_OK(Put(Key(2), "val2", wo)); + s = Flush(); + // Since auto resume fails, the bg error is not cleand, flush will + // return the bg_error set before. + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + ASSERT_EQ("val2", Get(Key(2))); + + // call auto resume + ASSERT_OK(dbfull()->Resume()); + ASSERT_OK(Put(Key(3), "val3", wo)); + // After resume is successful, the flush should be ok. + ASSERT_OK(Flush()); + ASSERT_EQ("val3", Get(Key(3))); + Destroy(options); +} + +TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover2) { + // Activate the FS before the first resume + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 2; + options.bgerror_resume_retry_interval = 100000; // 0.1 second + options.statistics = CreateDBStatistics(); + Status s; + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + WriteOptions wo = WriteOptions(); + wo.disableWAL = true; + ASSERT_OK(Put(Key(1), "val1", wo)); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeFinishBuildTable", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ("val1", Get(Key(1))); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + ASSERT_EQ(listener->WaitForRecovery(5000000), true); + ASSERT_EQ("val1", Get(Key(1))); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_ERROR_COUNT)); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_IO_ERROR_COUNT)); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT)); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_COUNT)); + ASSERT_LE(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT)); + ASSERT_LE(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT)); + HistogramData autoresume_retry; + options.statistics->histogramData(ERROR_HANDLER_AUTORESUME_RETRY_COUNT, + &autoresume_retry); + ASSERT_GE(autoresume_retry.max, 0); + ASSERT_OK(Put(Key(2), "val2", wo)); + s = Flush(); + // Since auto resume is successful, the bg error is cleaned, flush will + // be successful. + ASSERT_OK(s); + ASSERT_EQ("val2", Get(Key(2))); + Destroy(options); +} + +// Auto resume fromt the flush retryable IO error. Activate the FS before the +// first resume. Resume is successful +TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover1) { + // Activate the FS before the first resume + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 2; + options.bgerror_resume_retry_interval = 100000; // 0.1 second + Status s; + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + ASSERT_OK(Put(Key(1), "val1")); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeFinishBuildTable", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + ASSERT_EQ(listener->WaitForRecovery(5000000), true); + + ASSERT_EQ("val1", Get(Key(1))); + Reopen(options); + ASSERT_EQ("val1", Get(Key(1))); + ASSERT_OK(Put(Key(2), "val2")); + ASSERT_OK(Flush()); + ASSERT_EQ("val2", Get(Key(2))); + + Destroy(options); +} + +// Auto resume fromt the flush retryable IO error and set the retry limit count. +// Never activate the FS and auto resume should fail at the end +TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover2) { + // Fail all the resume and let user to resume + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 2; + options.bgerror_resume_retry_interval = 100000; // 0.1 second + Status s; + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + ASSERT_OK(Put(Key(1), "val1")); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"FLushWritRetryableeErrorAutoRecover2:0", + "RecoverFromRetryableBGIOError:BeforeStart"}, + {"RecoverFromRetryableBGIOError:LoopOut", + "FLushWritRetryableeErrorAutoRecover2:1"}}); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeFinishBuildTable", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + TEST_SYNC_POINT("FLushWritRetryableeErrorAutoRecover2:0"); + TEST_SYNC_POINT("FLushWritRetryableeErrorAutoRecover2:1"); + fault_fs_->SetFilesystemActive(true); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); + + ASSERT_EQ("val1", Get(Key(1))); + // Auto resume fails due to FS does not recover during resume. User call + // resume manually here. + s = dbfull()->Resume(); + ASSERT_EQ("val1", Get(Key(1))); + ASSERT_OK(s); + ASSERT_OK(Put(Key(2), "val2")); + ASSERT_OK(Flush()); + ASSERT_EQ("val2", Get(Key(2))); + + Destroy(options); +} + +// Auto resume fromt the flush retryable IO error and set the retry limit count. +// Fail the first resume and let the second resume be successful. +TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableErrorAutoRecover) { + // Fail the first resume and let the second resume be successful + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 2; + options.bgerror_resume_retry_interval = 100000; // 0.1 second + Status s; + std::string old_manifest; + std::string new_manifest; + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + old_manifest = GetManifestNameFromLiveFiles(); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + ASSERT_OK(Put(Key(0), "val")); + ASSERT_OK(Flush()); + ASSERT_OK(Put(Key(1), "val")); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"RecoverFromRetryableBGIOError:BeforeStart", + "ManifestWriteRetryableErrorAutoRecover:0"}, + {"ManifestWriteRetryableErrorAutoRecover:1", + "RecoverFromRetryableBGIOError:BeforeWait1"}, + {"RecoverFromRetryableBGIOError:RecoverSuccess", + "ManifestWriteRetryableErrorAutoRecover:2"}}); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:WriteManifest", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + TEST_SYNC_POINT("ManifestWriteRetryableErrorAutoRecover:0"); + fault_fs_->SetFilesystemActive(true); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + TEST_SYNC_POINT("ManifestWriteRetryableErrorAutoRecover:1"); + TEST_SYNC_POINT("ManifestWriteRetryableErrorAutoRecover:2"); + SyncPoint::GetInstance()->DisableProcessing(); + + new_manifest = GetManifestNameFromLiveFiles(); + ASSERT_NE(new_manifest, old_manifest); + + Reopen(options); + ASSERT_EQ("val", Get(Key(0))); + ASSERT_EQ("val", Get(Key(1))); + Close(); +} + +TEST_F(DBErrorHandlingFSTest, ManifestWriteNoWALRetryableErrorAutoRecover) { + // Fail the first resume and let the second resume be successful + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 2; + options.bgerror_resume_retry_interval = 100000; // 0.1 second + Status s; + std::string old_manifest; + std::string new_manifest; + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + old_manifest = GetManifestNameFromLiveFiles(); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + WriteOptions wo = WriteOptions(); + wo.disableWAL = true; + ASSERT_OK(Put(Key(0), "val", wo)); + ASSERT_OK(Flush()); + ASSERT_OK(Put(Key(1), "val", wo)); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"RecoverFromRetryableBGIOError:BeforeStart", + "ManifestWriteNoWALRetryableErrorAutoRecover:0"}, + {"ManifestWriteNoWALRetryableErrorAutoRecover:1", + "RecoverFromRetryableBGIOError:BeforeWait1"}, + {"RecoverFromRetryableBGIOError:RecoverSuccess", + "ManifestWriteNoWALRetryableErrorAutoRecover:2"}}); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:WriteManifest", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + TEST_SYNC_POINT("ManifestWriteNoWALRetryableErrorAutoRecover:0"); + fault_fs_->SetFilesystemActive(true); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + TEST_SYNC_POINT("ManifestWriteNoWALRetryableErrorAutoRecover:1"); + TEST_SYNC_POINT("ManifestWriteNoWALRetryableErrorAutoRecover:2"); + SyncPoint::GetInstance()->DisableProcessing(); + + new_manifest = GetManifestNameFromLiveFiles(); + ASSERT_NE(new_manifest, old_manifest); + + Reopen(options); + ASSERT_EQ("val", Get(Key(0))); + ASSERT_EQ("val", Get(Key(1))); + Close(); +} + +TEST_F(DBErrorHandlingFSTest, + CompactionManifestWriteRetryableErrorAutoRecover) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.level0_file_num_compaction_trigger = 2; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 2; + options.bgerror_resume_retry_interval = 100000; // 0.1 second + Status s; + std::string old_manifest; + std::string new_manifest; + std::atomic fail_manifest(false); + DestroyAndReopen(options); + old_manifest = GetManifestNameFromLiveFiles(); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + ASSERT_OK(Put(Key(0), "val")); + ASSERT_OK(Put(Key(2), "val")); + ASSERT_OK(Flush()); + + listener->OverrideBGError(Status(error_msg, Status::Severity::kHardError)); + listener->EnableAutoRecovery(false); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + // Wait for flush of 2nd L0 file before starting compaction + {{"DBImpl::FlushMemTable:FlushMemTableFinished", + "BackgroundCallCompaction:0"}, + // Wait for compaction to detect manifest write error + {"BackgroundCallCompaction:1", "CompactionManifestWriteErrorAR:0"}, + // Make compaction thread wait for error to be cleared + {"CompactionManifestWriteErrorAR:1", + "DBImpl::BackgroundCallCompaction:FoundObsoleteFiles"}, + {"CompactionManifestWriteErrorAR:2", + "RecoverFromRetryableBGIOError:BeforeStart"}, + // Fail the first resume, before the wait in resume + {"RecoverFromRetryableBGIOError:BeforeResume0", + "CompactionManifestWriteErrorAR:3"}, + // Activate the FS before the second resume + {"CompactionManifestWriteErrorAR:4", + "RecoverFromRetryableBGIOError:BeforeResume1"}, + // Wait the auto resume be sucessful + {"RecoverFromRetryableBGIOError:RecoverSuccess", + "CompactionManifestWriteErrorAR:5"}}); + // trigger manifest write failure in compaction thread + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "BackgroundCallCompaction:0", [&](void*) { fail_manifest.store(true); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:WriteManifest", [&](void*) { + if (fail_manifest.load()) { + fault_fs_->SetFilesystemActive(false, error_msg); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put(Key(1), "val")); + s = Flush(); + ASSERT_OK(s); + + TEST_SYNC_POINT("CompactionManifestWriteErrorAR:0"); + TEST_SYNC_POINT("CompactionManifestWriteErrorAR:1"); + + s = dbfull()->TEST_WaitForCompact(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); + TEST_SYNC_POINT("CompactionManifestWriteErrorAR:2"); + TEST_SYNC_POINT("CompactionManifestWriteErrorAR:3"); + fault_fs_->SetFilesystemActive(true); + SyncPoint::GetInstance()->ClearAllCallBacks(); + TEST_SYNC_POINT("CompactionManifestWriteErrorAR:4"); + TEST_SYNC_POINT("CompactionManifestWriteErrorAR:5"); + SyncPoint::GetInstance()->DisableProcessing(); + + new_manifest = GetManifestNameFromLiveFiles(); + ASSERT_NE(new_manifest, old_manifest); + + Reopen(options); + ASSERT_EQ("val", Get(Key(0))); + ASSERT_EQ("val", Get(Key(1))); + ASSERT_EQ("val", Get(Key(2))); + Close(); +} + +TEST_F(DBErrorHandlingFSTest, CompactionWriteRetryableErrorAutoRecover) { + // In this test, in the first round of compaction, the FS is set to error. + // So the first compaction fails due to retryable IO error and it is mapped + // to soft error. Then, compaction is rescheduled, in the second round of + // compaction, the FS is set to active and compaction is successful, so + // the test will hit the CompactionJob::FinishCompactionOutputFile1 sync + // point. + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.level0_file_num_compaction_trigger = 2; + options.listeners.emplace_back(listener); + Status s; + std::atomic fail_first(false); + std::atomic fail_second(true); + DestroyAndReopen(options); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + ASSERT_OK(Put(Key(0), "va;")); + ASSERT_OK(Put(Key(2), "va;")); + s = Flush(); + ASSERT_OK(s); + + listener->OverrideBGError(Status(error_msg, Status::Severity::kHardError)); + listener->EnableAutoRecovery(false); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::FlushMemTable:FlushMemTableFinished", + "BackgroundCallCompaction:0"}, + {"CompactionJob::FinishCompactionOutputFile1", + "CompactionWriteRetryableErrorAutoRecover0"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:Start", + [&](void*) { fault_fs_->SetFilesystemActive(true); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "BackgroundCallCompaction:0", [&](void*) { fail_first.store(true); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::OpenCompactionOutputFile", [&](void*) { + if (fail_first.load() && fail_second.load()) { + fault_fs_->SetFilesystemActive(false, error_msg); + fail_second.store(false); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put(Key(1), "val")); + s = Flush(); + ASSERT_OK(s); + + s = dbfull()->TEST_WaitForCompact(); + ASSERT_OK(s); + TEST_SYNC_POINT("CompactionWriteRetryableErrorAutoRecover0"); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); + Destroy(options); +} + +TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover1) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.writable_file_max_buffer_size = 32768; + options.listeners.emplace_back(listener); + options.paranoid_checks = true; + options.max_bgerror_resume_count = 2; + options.bgerror_resume_retry_interval = 100000; // 0.1 second + Status s; + Random rnd(301); + + DestroyAndReopen(options); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + // For the first batch, write is successful, require sync + { + WriteBatch batch; + + for (auto i = 0; i < 100; ++i) { + ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024))); + } + + WriteOptions wopts; + wopts.sync = true; + ASSERT_OK(dbfull()->Write(wopts, &batch)); + }; + + // For the second batch, the first 2 file Append are successful, then the + // following Append fails due to file system retryable IOError. + { + WriteBatch batch; + int write_error = 0; + + for (auto i = 100; i < 200; ++i) { + ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024))); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"WALWriteErrorDone", "RecoverFromRetryableBGIOError:BeforeStart"}, + {"RecoverFromRetryableBGIOError:BeforeResume0", "WALWriteError1:0"}, + {"WALWriteError1:1", "RecoverFromRetryableBGIOError:BeforeResume1"}, + {"RecoverFromRetryableBGIOError:RecoverSuccess", "WALWriteError1:2"}}); + + SyncPoint::GetInstance()->SetCallBack( + "WritableFileWriter::Append:BeforePrepareWrite", [&](void*) { + write_error++; + if (write_error > 2) { + fault_fs_->SetFilesystemActive(false, error_msg); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + WriteOptions wopts; + wopts.sync = true; + s = dbfull()->Write(wopts, &batch); + ASSERT_EQ(true, s.IsIOError()); + TEST_SYNC_POINT("WALWriteErrorDone"); + + TEST_SYNC_POINT("WALWriteError1:0"); + fault_fs_->SetFilesystemActive(true); + SyncPoint::GetInstance()->ClearAllCallBacks(); + TEST_SYNC_POINT("WALWriteError1:1"); + TEST_SYNC_POINT("WALWriteError1:2"); + } + SyncPoint::GetInstance()->DisableProcessing(); + + // Data in corrupted WAL are not stored + for (auto i = 0; i < 199; ++i) { + if (i < 100) { + ASSERT_NE(Get(Key(i)), "NOT_FOUND"); + } else { + ASSERT_EQ(Get(Key(i)), "NOT_FOUND"); + } + } + + // Resume and write a new batch, should be in the WAL + { + WriteBatch batch; + + for (auto i = 200; i < 300; ++i) { + ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024))); + } + + WriteOptions wopts; + wopts.sync = true; + ASSERT_OK(dbfull()->Write(wopts, &batch)); + }; + + Reopen(options); + for (auto i = 0; i < 300; ++i) { + if (i < 100 || i >= 200) { + ASSERT_NE(Get(Key(i)), "NOT_FOUND"); + } else { + ASSERT_EQ(Get(Key(i)), "NOT_FOUND"); + } + } + Close(); +} + +TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover2) { + // Fail the first recover and try second time. + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.writable_file_max_buffer_size = 32768; + options.listeners.emplace_back(listener); + options.paranoid_checks = true; + options.max_bgerror_resume_count = 2; + options.bgerror_resume_retry_interval = 100000; // 0.1 second + Status s; + Random rnd(301); + + DestroyAndReopen(options); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + // For the first batch, write is successful, require sync + { + WriteBatch batch; + + for (auto i = 0; i < 100; ++i) { + ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024))); + } + + WriteOptions wopts; + wopts.sync = true; + ASSERT_OK(dbfull()->Write(wopts, &batch)); + }; + + // For the second batch, the first 2 file Append are successful, then the + // following Append fails due to file system retryable IOError. + { + WriteBatch batch; + int write_error = 0; + + for (auto i = 100; i < 200; ++i) { + ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024))); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"RecoverFromRetryableBGIOError:BeforeWait0", "WALWriteError2:0"}, + {"WALWriteError2:1", "RecoverFromRetryableBGIOError:BeforeWait1"}, + {"RecoverFromRetryableBGIOError:RecoverSuccess", "WALWriteError2:2"}}); + + SyncPoint::GetInstance()->SetCallBack( + "WritableFileWriter::Append:BeforePrepareWrite", [&](void*) { + write_error++; + if (write_error > 2) { + fault_fs_->SetFilesystemActive(false, error_msg); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + WriteOptions wopts; + wopts.sync = true; + s = dbfull()->Write(wopts, &batch); + ASSERT_EQ(true, s.IsIOError()); + + TEST_SYNC_POINT("WALWriteError2:0"); + fault_fs_->SetFilesystemActive(true); + SyncPoint::GetInstance()->ClearAllCallBacks(); + TEST_SYNC_POINT("WALWriteError2:1"); + TEST_SYNC_POINT("WALWriteError2:2"); + } + SyncPoint::GetInstance()->DisableProcessing(); + + // Data in corrupted WAL are not stored + for (auto i = 0; i < 199; ++i) { + if (i < 100) { + ASSERT_NE(Get(Key(i)), "NOT_FOUND"); + } else { + ASSERT_EQ(Get(Key(i)), "NOT_FOUND"); + } + } + + // Resume and write a new batch, should be in the WAL + { + WriteBatch batch; + + for (auto i = 200; i < 300; ++i) { + ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024))); + } + + WriteOptions wopts; + wopts.sync = true; + ASSERT_OK(dbfull()->Write(wopts, &batch)); + }; + + Reopen(options); + for (auto i = 0; i < 300; ++i) { + if (i < 100 || i >= 200) { + ASSERT_NE(Get(Key(i)), "NOT_FOUND"); + } else { + ASSERT_EQ(Get(Key(i)), "NOT_FOUND"); + } + } + Close(); +} + +// Fail auto resume from a flush retryable error and verify that +// OnErrorRecoveryEnd listener callback is called +TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAbortRecovery) { + // Activate the FS before the first resume + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 2; + options.bgerror_resume_retry_interval = 100000; // 0.1 second + Status s; + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + ASSERT_OK(Put(Key(1), "val1")); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeFinishBuildTable", + [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); + + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + ASSERT_EQ(listener->WaitForRecovery(5000000), true); + ASSERT_EQ(listener->new_bg_error(), Status::Aborted()); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + + Destroy(options); +} + +TEST_F(DBErrorHandlingFSTest, FlushReadError) { + std::shared_ptr listener = + std::make_shared(); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.statistics = CreateDBStatistics(); + Status s; + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + + ASSERT_OK(Put(Key(0), "val")); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeOutputValidation", [&](void*) { + IOStatus st = IOStatus::IOError(); + st.SetRetryable(true); + st.SetScope(IOStatus::IOErrorScope::kIOErrorScopeFile); + fault_fs_->SetFilesystemActive(false, st); + }); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeDeleteFile", + [&](void*) { fault_fs_->SetFilesystemActive(true, IOStatus::OK()); }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + ASSERT_EQ(listener->WaitForRecovery(5000000), true); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_ERROR_COUNT)); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_IO_ERROR_COUNT)); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT)); + ASSERT_LE(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_COUNT)); + ASSERT_LE(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT)); + s = dbfull()->TEST_GetBGError(); + ASSERT_OK(s); + + Reopen(GetDefaultOptions()); + ASSERT_EQ("val", Get(Key(0))); +} + +TEST_F(DBErrorHandlingFSTest, AtomicFlushReadError) { + std::shared_ptr listener = + std::make_shared(); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.statistics = CreateDBStatistics(); + Status s; + + listener->EnableAutoRecovery(false); + options.atomic_flush = true; + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(0, Key(0), "val")); + ASSERT_OK(Put(1, Key(0), "val")); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeOutputValidation", [&](void*) { + IOStatus st = IOStatus::IOError(); + st.SetRetryable(true); + st.SetScope(IOStatus::IOErrorScope::kIOErrorScopeFile); + fault_fs_->SetFilesystemActive(false, st); + }); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeDeleteFile", + [&](void*) { fault_fs_->SetFilesystemActive(true, IOStatus::OK()); }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush({0, 1}); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + ASSERT_EQ(listener->WaitForRecovery(5000000), true); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_ERROR_COUNT)); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_IO_ERROR_COUNT)); + ASSERT_EQ(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT)); + ASSERT_LE(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_COUNT)); + ASSERT_LE(0, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT)); + s = dbfull()->TEST_GetBGError(); + ASSERT_OK(s); + + TryReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu"}, + GetDefaultOptions()); + ASSERT_EQ("val", Get(Key(0))); +} + +TEST_F(DBErrorHandlingFSTest, AtomicFlushNoSpaceError) { + std::shared_ptr listener = + std::make_shared(); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.statistics = CreateDBStatistics(); + Status s; + + listener->EnableAutoRecovery(true); + options.atomic_flush = true; + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(0, Key(0), "val")); + ASSERT_OK(Put(1, Key(0), "val")); + SyncPoint::GetInstance()->SetCallBack("BuildTable:create_file", [&](void*) { + IOStatus st = IOStatus::NoSpace(); + fault_fs_->SetFilesystemActive(false, st); + }); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BeforeDeleteFile", + [&](void*) { fault_fs_->SetFilesystemActive(true, IOStatus::OK()); }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush({0, 1}); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + ASSERT_EQ(listener->WaitForRecovery(5000000), true); + ASSERT_LE(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_ERROR_COUNT)); + ASSERT_LE(1, options.statistics->getAndResetTickerCount( + ERROR_HANDLER_BG_IO_ERROR_COUNT)); + s = dbfull()->TEST_GetBGError(); + ASSERT_OK(s); + + TryReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu"}, + GetDefaultOptions()); + ASSERT_EQ("val", Get(Key(0))); +} + +TEST_F(DBErrorHandlingFSTest, CompactionReadRetryableErrorAutoRecover) { + // In this test, in the first round of compaction, the FS is set to error. + // So the first compaction fails due to retryable IO error and it is mapped + // to soft error. Then, compaction is rescheduled, in the second round of + // compaction, the FS is set to active and compaction is successful, so + // the test will hit the CompactionJob::FinishCompactionOutputFile1 sync + // point. + std::shared_ptr listener = + std::make_shared(); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.level0_file_num_compaction_trigger = 2; + options.listeners.emplace_back(listener); + BlockBasedTableOptions table_options; + table_options.no_block_cache = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Status s; + std::atomic fail_first(false); + std::atomic fail_second(true); + Random rnd(301); + DestroyAndReopen(options); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + for (int i = 0; i < 100; ++i) { + ASSERT_OK(Put(Key(i), rnd.RandomString(1024))); + } + s = Flush(); + ASSERT_OK(s); + + listener->OverrideBGError(Status(error_msg, Status::Severity::kHardError)); + listener->EnableAutoRecovery(false); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::FlushMemTable:FlushMemTableFinished", + "BackgroundCallCompaction:0"}, + {"CompactionJob::FinishCompactionOutputFile1", + "CompactionWriteRetryableErrorAutoRecover0"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:Start", + [&](void*) { fault_fs_->SetFilesystemActive(true); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "BackgroundCallCompaction:0", [&](void*) { fail_first.store(true); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::Run():PausingManualCompaction:2", [&](void*) { + if (fail_first.load() && fail_second.load()) { + fault_fs_->SetFilesystemActive(false, error_msg); + fail_second.store(false); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put(Key(1), "val")); + s = Flush(); + ASSERT_OK(s); + + s = dbfull()->TEST_WaitForCompact(); + ASSERT_OK(s); + TEST_SYNC_POINT("CompactionWriteRetryableErrorAutoRecover0"); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); + + Reopen(GetDefaultOptions()); +} + +class DBErrorHandlingFencingTest : public DBErrorHandlingFSTest, + public testing::WithParamInterface {}; + +TEST_P(DBErrorHandlingFencingTest, FLushWriteFenced) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.paranoid_checks = GetParam(); + Status s; + + listener->EnableAutoRecovery(true); + DestroyAndReopen(options); + + ASSERT_OK(Put(Key(0), "val")); + SyncPoint::GetInstance()->SetCallBack("FlushJob::Start", [&](void*) { + fault_fs_->SetFilesystemActive(false, IOStatus::IOFenced("IO fenced")); + }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kFatalError); + ASSERT_TRUE(s.IsIOFenced()); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + s = dbfull()->Resume(); + ASSERT_TRUE(s.IsIOFenced()); + Destroy(options); +} + +TEST_P(DBErrorHandlingFencingTest, ManifestWriteFenced) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.paranoid_checks = GetParam(); + Status s; + std::string old_manifest; + std::string new_manifest; + + listener->EnableAutoRecovery(true); + DestroyAndReopen(options); + old_manifest = GetManifestNameFromLiveFiles(); + + ASSERT_OK(Put(Key(0), "val")); + ASSERT_OK(Flush()); + ASSERT_OK(Put(Key(1), "val")); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:WriteManifest", [&](void*) { + fault_fs_->SetFilesystemActive(false, IOStatus::IOFenced("IO fenced")); + }); + SyncPoint::GetInstance()->EnableProcessing(); + s = Flush(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kFatalError); + ASSERT_TRUE(s.IsIOFenced()); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + s = dbfull()->Resume(); + ASSERT_TRUE(s.IsIOFenced()); + Close(); +} + +TEST_P(DBErrorHandlingFencingTest, CompactionWriteFenced) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.level0_file_num_compaction_trigger = 2; + options.listeners.emplace_back(listener); + options.paranoid_checks = GetParam(); + Status s; + DestroyAndReopen(options); + + ASSERT_OK(Put(Key(0), "va;")); + ASSERT_OK(Put(Key(2), "va;")); + s = Flush(); + ASSERT_OK(s); + + listener->EnableAutoRecovery(true); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::FlushMemTable:FlushMemTableFinished", + "BackgroundCallCompaction:0"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "BackgroundCallCompaction:0", [&](void*) { + fault_fs_->SetFilesystemActive(false, IOStatus::IOFenced("IO fenced")); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(Put(Key(1), "val")); + s = Flush(); + ASSERT_OK(s); + + s = dbfull()->TEST_WaitForCompact(); + ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kFatalError); + ASSERT_TRUE(s.IsIOFenced()); + + fault_fs_->SetFilesystemActive(true); + s = dbfull()->Resume(); + ASSERT_TRUE(s.IsIOFenced()); + Destroy(options); +} + +TEST_P(DBErrorHandlingFencingTest, WALWriteFenced) { + std::shared_ptr listener( + new ErrorHandlerFSListener()); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.writable_file_max_buffer_size = 32768; + options.listeners.emplace_back(listener); + options.paranoid_checks = GetParam(); + Status s; + Random rnd(301); + + listener->EnableAutoRecovery(true); + DestroyAndReopen(options); + + { + WriteBatch batch; + + for (auto i = 0; i < 100; ++i) { + ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024))); + } + + WriteOptions wopts; + wopts.sync = true; + ASSERT_OK(dbfull()->Write(wopts, &batch)); + }; + + { + WriteBatch batch; + int write_error = 0; + + for (auto i = 100; i < 199; ++i) { + ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024))); + } + + SyncPoint::GetInstance()->SetCallBack( + "WritableFileWriter::Append:BeforePrepareWrite", [&](void*) { + write_error++; + if (write_error > 2) { + fault_fs_->SetFilesystemActive(false, + IOStatus::IOFenced("IO fenced")); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + WriteOptions wopts; + wopts.sync = true; + s = dbfull()->Write(wopts, &batch); + ASSERT_TRUE(s.IsIOFenced()); + } + SyncPoint::GetInstance()->DisableProcessing(); + fault_fs_->SetFilesystemActive(true); + { + WriteBatch batch; + + for (auto i = 0; i < 100; ++i) { + ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024))); + } + + WriteOptions wopts; + wopts.sync = true; + s = dbfull()->Write(wopts, &batch); + ASSERT_TRUE(s.IsIOFenced()); + } + Close(); +} + +INSTANTIATE_TEST_CASE_P(DBErrorHandlingFSTest, DBErrorHandlingFencingTest, + ::testing::Bool()); + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/event_helpers.cc b/librocksdb-sys/rocksdb/db/event_helpers.cc new file mode 100644 index 0000000..4360144 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/event_helpers.cc @@ -0,0 +1,323 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/event_helpers.h" + +#include "rocksdb/convenience.h" +#include "rocksdb/listener.h" +#include "rocksdb/utilities/customizable_util.h" + +namespace ROCKSDB_NAMESPACE { +Status EventListener::CreateFromString(const ConfigOptions& config_options, + const std::string& id, + std::shared_ptr* result) { + return LoadSharedObject(config_options, id, result); +} + +namespace { +template +inline T SafeDivide(T a, T b) { + return b == 0 ? 0 : a / b; +} +} // anonymous namespace + +void EventHelpers::AppendCurrentTime(JSONWriter* jwriter) { + *jwriter << "time_micros" + << std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); +} + +void EventHelpers::NotifyTableFileCreationStarted( + const std::vector>& listeners, + const std::string& db_name, const std::string& cf_name, + const std::string& file_path, int job_id, TableFileCreationReason reason) { + if (listeners.empty()) { + return; + } + TableFileCreationBriefInfo info; + info.db_name = db_name; + info.cf_name = cf_name; + info.file_path = file_path; + info.job_id = job_id; + info.reason = reason; + for (auto& listener : listeners) { + listener->OnTableFileCreationStarted(info); + } +} + +void EventHelpers::NotifyOnBackgroundError( + const std::vector>& listeners, + BackgroundErrorReason reason, Status* bg_error, InstrumentedMutex* db_mutex, + bool* auto_recovery) { + if (listeners.empty()) { + return; + } + db_mutex->AssertHeld(); + // release lock while notifying events + db_mutex->Unlock(); + for (auto& listener : listeners) { + listener->OnBackgroundError(reason, bg_error); + bg_error->PermitUncheckedError(); + if (*auto_recovery) { + listener->OnErrorRecoveryBegin(reason, *bg_error, auto_recovery); + } + } + db_mutex->Lock(); +} + +void EventHelpers::LogAndNotifyTableFileCreationFinished( + EventLogger* event_logger, + const std::vector>& listeners, + const std::string& db_name, const std::string& cf_name, + const std::string& file_path, int job_id, const FileDescriptor& fd, + uint64_t oldest_blob_file_number, const TableProperties& table_properties, + TableFileCreationReason reason, const Status& s, + const std::string& file_checksum, + const std::string& file_checksum_func_name) { + if (s.ok() && event_logger) { + JSONWriter jwriter; + AppendCurrentTime(&jwriter); + jwriter << "cf_name" << cf_name << "job" << job_id << "event" + << "table_file_creation" + << "file_number" << fd.GetNumber() << "file_size" + << fd.GetFileSize() << "file_checksum" + << Slice(file_checksum).ToString(true) << "file_checksum_func_name" + << file_checksum_func_name << "smallest_seqno" << fd.smallest_seqno + << "largest_seqno" << fd.largest_seqno; + + // table_properties + { + jwriter << "table_properties"; + jwriter.StartObject(); + + // basic properties: + jwriter << "data_size" << table_properties.data_size << "index_size" + << table_properties.index_size << "index_partitions" + << table_properties.index_partitions << "top_level_index_size" + << table_properties.top_level_index_size + << "index_key_is_user_key" + << table_properties.index_key_is_user_key + << "index_value_is_delta_encoded" + << table_properties.index_value_is_delta_encoded << "filter_size" + << table_properties.filter_size << "raw_key_size" + << table_properties.raw_key_size << "raw_average_key_size" + << SafeDivide(table_properties.raw_key_size, + table_properties.num_entries) + << "raw_value_size" << table_properties.raw_value_size + << "raw_average_value_size" + << SafeDivide(table_properties.raw_value_size, + table_properties.num_entries) + << "num_data_blocks" << table_properties.num_data_blocks + << "num_entries" << table_properties.num_entries + << "num_filter_entries" << table_properties.num_filter_entries + << "num_deletions" << table_properties.num_deletions + << "num_merge_operands" << table_properties.num_merge_operands + << "num_range_deletions" << table_properties.num_range_deletions + << "format_version" << table_properties.format_version + << "fixed_key_len" << table_properties.fixed_key_len + << "filter_policy" << table_properties.filter_policy_name + << "column_family_name" << table_properties.column_family_name + << "column_family_id" << table_properties.column_family_id + << "comparator" << table_properties.comparator_name + << "merge_operator" << table_properties.merge_operator_name + << "prefix_extractor_name" + << table_properties.prefix_extractor_name << "property_collectors" + << table_properties.property_collectors_names << "compression" + << table_properties.compression_name << "compression_options" + << table_properties.compression_options << "creation_time" + << table_properties.creation_time << "oldest_key_time" + << table_properties.oldest_key_time << "file_creation_time" + << table_properties.file_creation_time + << "slow_compression_estimated_data_size" + << table_properties.slow_compression_estimated_data_size + << "fast_compression_estimated_data_size" + << table_properties.fast_compression_estimated_data_size + << "db_id" << table_properties.db_id << "db_session_id" + << table_properties.db_session_id << "orig_file_number" + << table_properties.orig_file_number << "seqno_to_time_mapping"; + + if (table_properties.seqno_to_time_mapping.empty()) { + jwriter << "N/A"; + } else { + SeqnoToTimeMapping tmp; + Status status = tmp.Add(table_properties.seqno_to_time_mapping); + if (status.ok()) { + jwriter << tmp.ToHumanString(); + } else { + jwriter << "Invalid"; + } + } + + // user collected properties + for (const auto& prop : table_properties.readable_properties) { + jwriter << prop.first << prop.second; + } + jwriter.EndObject(); + } + + if (oldest_blob_file_number != kInvalidBlobFileNumber) { + jwriter << "oldest_blob_file_number" << oldest_blob_file_number; + } + + jwriter.EndObject(); + + event_logger->Log(jwriter); + } + + if (listeners.empty()) { + return; + } + TableFileCreationInfo info; + info.db_name = db_name; + info.cf_name = cf_name; + info.file_path = file_path; + info.file_size = fd.file_size; + info.job_id = job_id; + info.table_properties = table_properties; + info.reason = reason; + info.status = s; + info.file_checksum = file_checksum; + info.file_checksum_func_name = file_checksum_func_name; + for (auto& listener : listeners) { + listener->OnTableFileCreated(info); + } + info.status.PermitUncheckedError(); +} + +void EventHelpers::LogAndNotifyTableFileDeletion( + EventLogger* event_logger, int job_id, uint64_t file_number, + const std::string& file_path, const Status& status, + const std::string& dbname, + const std::vector>& listeners) { + JSONWriter jwriter; + AppendCurrentTime(&jwriter); + + jwriter << "job" << job_id << "event" + << "table_file_deletion" + << "file_number" << file_number; + if (!status.ok()) { + jwriter << "status" << status.ToString(); + } + + jwriter.EndObject(); + + event_logger->Log(jwriter); + + if (listeners.empty()) { + return; + } + TableFileDeletionInfo info; + info.db_name = dbname; + info.job_id = job_id; + info.file_path = file_path; + info.status = status; + for (auto& listener : listeners) { + listener->OnTableFileDeleted(info); + } + info.status.PermitUncheckedError(); +} + +void EventHelpers::NotifyOnErrorRecoveryEnd( + const std::vector>& listeners, + const Status& old_bg_error, const Status& new_bg_error, + InstrumentedMutex* db_mutex) { + if (!listeners.empty()) { + db_mutex->AssertHeld(); + // release lock while notifying events + db_mutex->Unlock(); + for (auto& listener : listeners) { + BackgroundErrorRecoveryInfo info; + info.old_bg_error = old_bg_error; + info.new_bg_error = new_bg_error; + listener->OnErrorRecoveryCompleted(old_bg_error); + listener->OnErrorRecoveryEnd(info); + info.old_bg_error.PermitUncheckedError(); + info.new_bg_error.PermitUncheckedError(); + } + db_mutex->Lock(); + } +} + +void EventHelpers::NotifyBlobFileCreationStarted( + const std::vector>& listeners, + const std::string& db_name, const std::string& cf_name, + const std::string& file_path, int job_id, + BlobFileCreationReason creation_reason) { + if (listeners.empty()) { + return; + } + BlobFileCreationBriefInfo info(db_name, cf_name, file_path, job_id, + creation_reason); + for (const auto& listener : listeners) { + listener->OnBlobFileCreationStarted(info); + } +} + +void EventHelpers::LogAndNotifyBlobFileCreationFinished( + EventLogger* event_logger, + const std::vector>& listeners, + const std::string& db_name, const std::string& cf_name, + const std::string& file_path, int job_id, uint64_t file_number, + BlobFileCreationReason creation_reason, const Status& s, + const std::string& file_checksum, + const std::string& file_checksum_func_name, uint64_t total_blob_count, + uint64_t total_blob_bytes) { + if (s.ok() && event_logger) { + JSONWriter jwriter; + AppendCurrentTime(&jwriter); + jwriter << "cf_name" << cf_name << "job" << job_id << "event" + << "blob_file_creation" + << "file_number" << file_number << "total_blob_count" + << total_blob_count << "total_blob_bytes" << total_blob_bytes + << "file_checksum" << file_checksum << "file_checksum_func_name" + << file_checksum_func_name << "status" << s.ToString(); + + jwriter.EndObject(); + event_logger->Log(jwriter); + } + + if (listeners.empty()) { + return; + } + BlobFileCreationInfo info(db_name, cf_name, file_path, job_id, + creation_reason, total_blob_count, total_blob_bytes, + s, file_checksum, file_checksum_func_name); + for (const auto& listener : listeners) { + listener->OnBlobFileCreated(info); + } + info.status.PermitUncheckedError(); +} + +void EventHelpers::LogAndNotifyBlobFileDeletion( + EventLogger* event_logger, + const std::vector>& listeners, int job_id, + uint64_t file_number, const std::string& file_path, const Status& status, + const std::string& dbname) { + if (event_logger) { + JSONWriter jwriter; + AppendCurrentTime(&jwriter); + + jwriter << "job" << job_id << "event" + << "blob_file_deletion" + << "file_number" << file_number; + if (!status.ok()) { + jwriter << "status" << status.ToString(); + } + + jwriter.EndObject(); + event_logger->Log(jwriter); + } + if (listeners.empty()) { + return; + } + BlobFileDeletionInfo info(dbname, file_path, job_id, status); + for (const auto& listener : listeners) { + listener->OnBlobFileDeleted(info); + } + info.status.PermitUncheckedError(); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/event_helpers.h b/librocksdb-sys/rocksdb/db/event_helpers.h new file mode 100644 index 0000000..a1331d8 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/event_helpers.h @@ -0,0 +1,78 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include +#include +#include + +#include "db/column_family.h" +#include "db/version_edit.h" +#include "logging/event_logger.h" +#include "rocksdb/listener.h" +#include "rocksdb/table_properties.h" + +namespace ROCKSDB_NAMESPACE { + +class EventHelpers { + public: + static void AppendCurrentTime(JSONWriter* json_writer); + static void NotifyTableFileCreationStarted( + const std::vector>& listeners, + const std::string& db_name, const std::string& cf_name, + const std::string& file_path, int job_id, TableFileCreationReason reason); + static void NotifyOnBackgroundError( + const std::vector>& listeners, + BackgroundErrorReason reason, Status* bg_error, + InstrumentedMutex* db_mutex, bool* auto_recovery); + static void LogAndNotifyTableFileCreationFinished( + EventLogger* event_logger, + const std::vector>& listeners, + const std::string& db_name, const std::string& cf_name, + const std::string& file_path, int job_id, const FileDescriptor& fd, + uint64_t oldest_blob_file_number, const TableProperties& table_properties, + TableFileCreationReason reason, const Status& s, + const std::string& file_checksum, + const std::string& file_checksum_func_name); + static void LogAndNotifyTableFileDeletion( + EventLogger* event_logger, int job_id, uint64_t file_number, + const std::string& file_path, const Status& status, + const std::string& db_name, + const std::vector>& listeners); + static void NotifyOnErrorRecoveryEnd( + const std::vector>& listeners, + const Status& old_bg_error, const Status& new_bg_error, + InstrumentedMutex* db_mutex); + + static void NotifyBlobFileCreationStarted( + const std::vector>& listeners, + const std::string& db_name, const std::string& cf_name, + const std::string& file_path, int job_id, + BlobFileCreationReason creation_reason); + + static void LogAndNotifyBlobFileCreationFinished( + EventLogger* event_logger, + const std::vector>& listeners, + const std::string& db_name, const std::string& cf_name, + const std::string& file_path, int job_id, uint64_t file_number, + BlobFileCreationReason creation_reason, const Status& s, + const std::string& file_checksum, + const std::string& file_checksum_func_name, uint64_t total_blob_count, + uint64_t total_blob_bytes); + + static void LogAndNotifyBlobFileDeletion( + EventLogger* event_logger, + const std::vector>& listeners, int job_id, + uint64_t file_number, const std::string& file_path, const Status& status, + const std::string& db_name); + + private: + static void LogAndNotifyTableFileCreation( + EventLogger* event_logger, + const std::vector>& listeners, + const FileDescriptor& fd, const TableFileCreationInfo& info); +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/experimental.cc b/librocksdb-sys/rocksdb/db/experimental.cc new file mode 100644 index 0000000..f6f920b --- /dev/null +++ b/librocksdb-sys/rocksdb/db/experimental.cc @@ -0,0 +1,145 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "rocksdb/experimental.h" + +#include "db/db_impl/db_impl.h" +#include "db/version_util.h" +#include "logging/logging.h" + +namespace ROCKSDB_NAMESPACE { +namespace experimental { + + +Status SuggestCompactRange(DB* db, ColumnFamilyHandle* column_family, + const Slice* begin, const Slice* end) { + if (db == nullptr) { + return Status::InvalidArgument("DB is empty"); + } + + return db->SuggestCompactRange(column_family, begin, end); +} + +Status PromoteL0(DB* db, ColumnFamilyHandle* column_family, int target_level) { + if (db == nullptr) { + return Status::InvalidArgument("Didn't recognize DB object"); + } + return db->PromoteL0(column_family, target_level); +} + + +Status SuggestCompactRange(DB* db, const Slice* begin, const Slice* end) { + return SuggestCompactRange(db, db->DefaultColumnFamily(), begin, end); +} + +Status UpdateManifestForFilesState( + const DBOptions& db_opts, const std::string& db_name, + const std::vector& column_families, + const UpdateManifestForFilesStateOptions& opts) { + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + OfflineManifestWriter w(db_opts, db_name); + Status s = w.Recover(column_families); + + size_t files_updated = 0; + size_t cfs_updated = 0; + auto fs = db_opts.env->GetFileSystem(); + + for (auto cfd : *w.Versions().GetColumnFamilySet()) { + if (!s.ok()) { + break; + } + assert(cfd); + + if (cfd->IsDropped() || !cfd->initialized()) { + continue; + } + + const auto* current = cfd->current(); + assert(current); + + const auto* vstorage = current->storage_info(); + assert(vstorage); + + VersionEdit edit; + edit.SetColumnFamily(cfd->GetID()); + + /* SST files */ + for (int level = 0; level < cfd->NumberLevels(); level++) { + if (!s.ok()) { + break; + } + const auto& level_files = vstorage->LevelFiles(level); + + for (const auto& lf : level_files) { + assert(lf); + + uint64_t number = lf->fd.GetNumber(); + std::string fname = + TableFileName(w.IOptions().db_paths, number, lf->fd.GetPathId()); + + std::unique_ptr f; + FileOptions fopts; + // Use kUnknown to signal the FileSystem to search all tiers for the + // file. + fopts.temperature = Temperature::kUnknown; + + IOStatus file_ios = + fs->NewSequentialFile(fname, fopts, &f, /*dbg*/ nullptr); + if (file_ios.ok()) { + if (opts.update_temperatures) { + Temperature temp = f->GetTemperature(); + if (temp != Temperature::kUnknown && temp != lf->temperature) { + // Current state inconsistent with manifest + ++files_updated; + edit.DeleteFile(level, number); + edit.AddFile( + level, number, lf->fd.GetPathId(), lf->fd.GetFileSize(), + lf->smallest, lf->largest, lf->fd.smallest_seqno, + lf->fd.largest_seqno, lf->marked_for_compaction, temp, + lf->oldest_blob_file_number, lf->oldest_ancester_time, + lf->file_creation_time, lf->epoch_number, lf->file_checksum, + lf->file_checksum_func_name, lf->unique_id, + lf->compensated_range_deletion_size, lf->tail_size, + lf->user_defined_timestamps_persisted); + } + } + } else { + s = file_ios; + break; + } + } + } + + if (s.ok() && edit.NumEntries() > 0) { + std::unique_ptr db_dir; + s = fs->NewDirectory(db_name, IOOptions(), &db_dir, nullptr); + if (s.ok()) { + s = w.LogAndApply(read_options, cfd, &edit, db_dir.get()); + } + if (s.ok()) { + ++cfs_updated; + } + } + } + + if (cfs_updated > 0) { + ROCKS_LOG_INFO(db_opts.info_log, + "UpdateManifestForFilesState: updated %zu files in %zu CFs", + files_updated, cfs_updated); + } else if (s.ok()) { + ROCKS_LOG_INFO(db_opts.info_log, + "UpdateManifestForFilesState: no updates needed"); + } + if (!s.ok()) { + ROCKS_LOG_ERROR(db_opts.info_log, "UpdateManifestForFilesState failed: %s", + s.ToString().c_str()); + } + + return s; +} + +} // namespace experimental +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/external_sst_file_basic_test.cc b/librocksdb-sys/rocksdb/db/external_sst_file_basic_test.cc new file mode 100644 index 0000000..3933499 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/external_sst_file_basic_test.cc @@ -0,0 +1,2084 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include + +#include "db/db_test_util.h" +#include "db/version_edit.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/sst_file_writer.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/random.h" +#include "utilities/fault_injection_env.h" + +namespace ROCKSDB_NAMESPACE { + +class ExternalSSTFileBasicTest + : public DBTestBase, + public ::testing::WithParamInterface> { + public: + ExternalSSTFileBasicTest() + : DBTestBase("external_sst_file_basic_test", /*env_do_fsync=*/true) { + sst_files_dir_ = dbname_ + "_sst_files/"; + fault_injection_test_env_.reset(new FaultInjectionTestEnv(env_)); + DestroyAndRecreateExternalSSTFilesDir(); + + // Check if the Env supports RandomRWFile + std::string file_path = sst_files_dir_ + "test_random_rw_file"; + std::unique_ptr wfile; + assert(env_->NewWritableFile(file_path, &wfile, EnvOptions()).ok()); + wfile.reset(); + std::unique_ptr rwfile; + Status s = env_->NewRandomRWFile(file_path, &rwfile, EnvOptions()); + if (s.IsNotSupported()) { + random_rwfile_supported_ = false; + } else { + EXPECT_OK(s); + random_rwfile_supported_ = true; + } + rwfile.reset(); + EXPECT_OK(env_->DeleteFile(file_path)); + } + + void DestroyAndRecreateExternalSSTFilesDir() { + ASSERT_OK(DestroyDir(env_, sst_files_dir_)); + ASSERT_OK(env_->CreateDir(sst_files_dir_)); + } + + Status DeprecatedAddFile(const std::vector& files, + bool move_files = false, + bool skip_snapshot_check = false) { + IngestExternalFileOptions opts; + opts.move_files = move_files; + opts.snapshot_consistency = !skip_snapshot_check; + opts.allow_global_seqno = false; + opts.allow_blocking_flush = false; + return db_->IngestExternalFile(files, opts); + } + + Status AddFileWithFileChecksum( + const std::vector& files, + const std::vector& files_checksums, + const std::vector& files_checksum_func_names, + bool verify_file_checksum = true, bool move_files = false, + bool skip_snapshot_check = false, bool write_global_seqno = true) { + IngestExternalFileOptions opts; + opts.move_files = move_files; + opts.snapshot_consistency = !skip_snapshot_check; + opts.allow_global_seqno = false; + opts.allow_blocking_flush = false; + opts.write_global_seqno = write_global_seqno; + opts.verify_file_checksum = verify_file_checksum; + + IngestExternalFileArg arg; + arg.column_family = db_->DefaultColumnFamily(); + arg.external_files = files; + arg.options = opts; + arg.files_checksums = files_checksums; + arg.files_checksum_func_names = files_checksum_func_names; + return db_->IngestExternalFiles({arg}); + } + + Status GenerateAndAddExternalFile( + const Options options, std::vector keys, + const std::vector& value_types, + std::vector> range_deletions, int file_id, + bool write_global_seqno, bool verify_checksums_before_ingest, + std::map* true_data) { + assert(value_types.size() == 1 || keys.size() == value_types.size()); + std::string file_path = sst_files_dir_ + std::to_string(file_id); + SstFileWriter sst_file_writer(EnvOptions(), options); + + Status s = sst_file_writer.Open(file_path); + if (!s.ok()) { + return s; + } + for (size_t i = 0; i < range_deletions.size(); i++) { + // Account for the effect of range deletions on true_data before + // all point operators, even though sst_file_writer.DeleteRange + // must be called before other sst_file_writer methods. This is + // because point writes take precedence over range deletions + // in the same ingested sst. This precedence is part of + // `SstFileWriter::DeleteRange()`'s API contract. + std::string start_key = Key(range_deletions[i].first); + std::string end_key = Key(range_deletions[i].second); + s = sst_file_writer.DeleteRange(start_key, end_key); + if (!s.ok()) { + sst_file_writer.Finish(); + return s; + } + auto start_key_it = true_data->find(start_key); + if (start_key_it == true_data->end()) { + start_key_it = true_data->upper_bound(start_key); + } + auto end_key_it = true_data->find(end_key); + if (end_key_it == true_data->end()) { + end_key_it = true_data->upper_bound(end_key); + } + true_data->erase(start_key_it, end_key_it); + } + for (size_t i = 0; i < keys.size(); i++) { + std::string key = Key(keys[i]); + std::string value = Key(keys[i]) + std::to_string(file_id); + ValueType value_type = + (value_types.size() == 1 ? value_types[0] : value_types[i]); + switch (value_type) { + case ValueType::kTypeValue: + s = sst_file_writer.Put(key, value); + (*true_data)[key] = value; + break; + case ValueType::kTypeMerge: + s = sst_file_writer.Merge(key, value); + // we only use TestPutOperator in this test + (*true_data)[key] = value; + break; + case ValueType::kTypeDeletion: + s = sst_file_writer.Delete(key); + true_data->erase(key); + break; + default: + return Status::InvalidArgument("Value type is not supported"); + } + if (!s.ok()) { + sst_file_writer.Finish(); + return s; + } + } + s = sst_file_writer.Finish(); + + if (s.ok()) { + IngestExternalFileOptions ifo; + ifo.allow_global_seqno = true; + ifo.write_global_seqno = write_global_seqno; + ifo.verify_checksums_before_ingest = verify_checksums_before_ingest; + s = db_->IngestExternalFile({file_path}, ifo); + } + return s; + } + + Status GenerateAndAddExternalFile( + const Options options, std::vector keys, + const std::vector& value_types, int file_id, + bool write_global_seqno, bool verify_checksums_before_ingest, + std::map* true_data) { + return GenerateAndAddExternalFile( + options, keys, value_types, {}, file_id, write_global_seqno, + verify_checksums_before_ingest, true_data); + } + + Status GenerateAndAddExternalFile( + const Options options, std::vector keys, const ValueType value_type, + int file_id, bool write_global_seqno, bool verify_checksums_before_ingest, + std::map* true_data) { + return GenerateAndAddExternalFile( + options, keys, std::vector(1, value_type), file_id, + write_global_seqno, verify_checksums_before_ingest, true_data); + } + + ~ExternalSSTFileBasicTest() override { + DestroyDir(env_, sst_files_dir_).PermitUncheckedError(); + } + + protected: + std::string sst_files_dir_; + std::unique_ptr fault_injection_test_env_; + bool random_rwfile_supported_; +}; + +TEST_F(ExternalSSTFileBasicTest, Basic) { + Options options = CurrentOptions(); + + SstFileWriter sst_file_writer(EnvOptions(), options); + + // Current file size should be 0 after sst_file_writer init and before open a + // file. + ASSERT_EQ(sst_file_writer.FileSize(), 0); + + // file1.sst (0 => 99) + std::string file1 = sst_files_dir_ + "file1.sst"; + ASSERT_OK(sst_file_writer.Open(file1)); + for (int k = 0; k < 100; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); + } + ExternalSstFileInfo file1_info; + Status s = sst_file_writer.Finish(&file1_info); + ASSERT_OK(s) << s.ToString(); + + // Current file size should be non-zero after success write. + ASSERT_GT(sst_file_writer.FileSize(), 0); + + ASSERT_EQ(file1_info.file_path, file1); + ASSERT_EQ(file1_info.num_entries, 100); + ASSERT_EQ(file1_info.smallest_key, Key(0)); + ASSERT_EQ(file1_info.largest_key, Key(99)); + ASSERT_EQ(file1_info.num_range_del_entries, 0); + ASSERT_EQ(file1_info.smallest_range_del_key, ""); + ASSERT_EQ(file1_info.largest_range_del_key, ""); + ASSERT_EQ(file1_info.file_checksum, kUnknownFileChecksum); + ASSERT_EQ(file1_info.file_checksum_func_name, kUnknownFileChecksumFuncName); + // sst_file_writer already finished, cannot add this value + s = sst_file_writer.Put(Key(100), "bad_val"); + ASSERT_NOK(s) << s.ToString(); + s = sst_file_writer.DeleteRange(Key(100), Key(200)); + ASSERT_NOK(s) << s.ToString(); + + DestroyAndReopen(options); + // Add file using file path + s = DeprecatedAddFile({file1}); + ASSERT_OK(s) << s.ToString(); + ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U); + for (int k = 0; k < 100; k++) { + ASSERT_EQ(Get(Key(k)), Key(k) + "_val"); + } + + DestroyAndRecreateExternalSSTFilesDir(); +} + +class ChecksumVerifyHelper { + private: + Options options_; + + public: + ChecksumVerifyHelper(Options& options) : options_(options) {} + ~ChecksumVerifyHelper() {} + + Status GetSingleFileChecksumAndFuncName( + const std::string& file_path, std::string* file_checksum, + std::string* file_checksum_func_name) { + Status s; + EnvOptions soptions; + std::unique_ptr file_reader; + s = options_.env->NewSequentialFile(file_path, &file_reader, soptions); + if (!s.ok()) { + return s; + } + std::unique_ptr scratch(new char[2048]); + Slice result; + FileChecksumGenFactory* file_checksum_gen_factory = + options_.file_checksum_gen_factory.get(); + if (file_checksum_gen_factory == nullptr) { + *file_checksum = kUnknownFileChecksum; + *file_checksum_func_name = kUnknownFileChecksumFuncName; + return Status::OK(); + } else { + FileChecksumGenContext gen_context; + std::unique_ptr file_checksum_gen = + file_checksum_gen_factory->CreateFileChecksumGenerator(gen_context); + *file_checksum_func_name = file_checksum_gen->Name(); + s = file_reader->Read(2048, &result, scratch.get()); + if (!s.ok()) { + return s; + } + while (result.size() != 0) { + file_checksum_gen->Update(scratch.get(), result.size()); + s = file_reader->Read(2048, &result, scratch.get()); + if (!s.ok()) { + return s; + } + } + file_checksum_gen->Finalize(); + *file_checksum = file_checksum_gen->GetChecksum(); + } + return Status::OK(); + } +}; + +TEST_F(ExternalSSTFileBasicTest, BasicWithFileChecksumCrc32c) { + Options options = CurrentOptions(); + options.file_checksum_gen_factory = GetFileChecksumGenCrc32cFactory(); + ChecksumVerifyHelper checksum_helper(options); + + SstFileWriter sst_file_writer(EnvOptions(), options); + + // Current file size should be 0 after sst_file_writer init and before open a + // file. + ASSERT_EQ(sst_file_writer.FileSize(), 0); + + // file1.sst (0 => 99) + std::string file1 = sst_files_dir_ + "file1.sst"; + ASSERT_OK(sst_file_writer.Open(file1)); + for (int k = 0; k < 100; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); + } + ExternalSstFileInfo file1_info; + Status s = sst_file_writer.Finish(&file1_info); + ASSERT_OK(s) << s.ToString(); + std::string file_checksum, file_checksum_func_name; + ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName( + file1, &file_checksum, &file_checksum_func_name)); + + // Current file size should be non-zero after success write. + ASSERT_GT(sst_file_writer.FileSize(), 0); + + ASSERT_EQ(file1_info.file_path, file1); + ASSERT_EQ(file1_info.num_entries, 100); + ASSERT_EQ(file1_info.smallest_key, Key(0)); + ASSERT_EQ(file1_info.largest_key, Key(99)); + ASSERT_EQ(file1_info.num_range_del_entries, 0); + ASSERT_EQ(file1_info.smallest_range_del_key, ""); + ASSERT_EQ(file1_info.largest_range_del_key, ""); + ASSERT_EQ(file1_info.file_checksum, file_checksum); + ASSERT_EQ(file1_info.file_checksum_func_name, file_checksum_func_name); + // sst_file_writer already finished, cannot add this value + s = sst_file_writer.Put(Key(100), "bad_val"); + ASSERT_NOK(s) << s.ToString(); + s = sst_file_writer.DeleteRange(Key(100), Key(200)); + ASSERT_NOK(s) << s.ToString(); + + DestroyAndReopen(options); + // Add file using file path + s = DeprecatedAddFile({file1}); + ASSERT_OK(s) << s.ToString(); + ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U); + for (int k = 0; k < 100; k++) { + ASSERT_EQ(Get(Key(k)), Key(k) + "_val"); + } + + DestroyAndRecreateExternalSSTFilesDir(); +} + +TEST_F(ExternalSSTFileBasicTest, IngestFileWithFileChecksum) { + Options old_options = CurrentOptions(); + Options options = CurrentOptions(); + options.file_checksum_gen_factory = GetFileChecksumGenCrc32cFactory(); + const ImmutableCFOptions ioptions(options); + ChecksumVerifyHelper checksum_helper(options); + + SstFileWriter sst_file_writer(EnvOptions(), options); + + // file01.sst (1000 => 1099) + std::string file1 = sst_files_dir_ + "file01.sst"; + ASSERT_OK(sst_file_writer.Open(file1)); + for (int k = 1000; k < 1100; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); + } + ExternalSstFileInfo file1_info; + Status s = sst_file_writer.Finish(&file1_info); + ASSERT_OK(s) << s.ToString(); + ASSERT_EQ(file1_info.file_path, file1); + ASSERT_EQ(file1_info.num_entries, 100); + ASSERT_EQ(file1_info.smallest_key, Key(1000)); + ASSERT_EQ(file1_info.largest_key, Key(1099)); + std::string file_checksum1, file_checksum_func_name1; + ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName( + file1, &file_checksum1, &file_checksum_func_name1)); + ASSERT_EQ(file1_info.file_checksum, file_checksum1); + ASSERT_EQ(file1_info.file_checksum_func_name, file_checksum_func_name1); + + // file02.sst (1100 => 1299) + std::string file2 = sst_files_dir_ + "file02.sst"; + ASSERT_OK(sst_file_writer.Open(file2)); + for (int k = 1100; k < 1300; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); + } + ExternalSstFileInfo file2_info; + s = sst_file_writer.Finish(&file2_info); + ASSERT_OK(s) << s.ToString(); + ASSERT_EQ(file2_info.file_path, file2); + ASSERT_EQ(file2_info.num_entries, 200); + ASSERT_EQ(file2_info.smallest_key, Key(1100)); + ASSERT_EQ(file2_info.largest_key, Key(1299)); + std::string file_checksum2, file_checksum_func_name2; + ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName( + file2, &file_checksum2, &file_checksum_func_name2)); + ASSERT_EQ(file2_info.file_checksum, file_checksum2); + ASSERT_EQ(file2_info.file_checksum_func_name, file_checksum_func_name2); + + // file03.sst (1300 => 1499) + std::string file3 = sst_files_dir_ + "file03.sst"; + ASSERT_OK(sst_file_writer.Open(file3)); + for (int k = 1300; k < 1500; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap")); + } + ExternalSstFileInfo file3_info; + s = sst_file_writer.Finish(&file3_info); + ASSERT_OK(s) << s.ToString(); + ASSERT_EQ(file3_info.file_path, file3); + ASSERT_EQ(file3_info.num_entries, 200); + ASSERT_EQ(file3_info.smallest_key, Key(1300)); + ASSERT_EQ(file3_info.largest_key, Key(1499)); + std::string file_checksum3, file_checksum_func_name3; + ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName( + file3, &file_checksum3, &file_checksum_func_name3)); + ASSERT_EQ(file3_info.file_checksum, file_checksum3); + ASSERT_EQ(file3_info.file_checksum_func_name, file_checksum_func_name3); + + // file04.sst (1500 => 1799) + std::string file4 = sst_files_dir_ + "file04.sst"; + ASSERT_OK(sst_file_writer.Open(file4)); + for (int k = 1500; k < 1800; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap")); + } + ExternalSstFileInfo file4_info; + s = sst_file_writer.Finish(&file4_info); + ASSERT_OK(s) << s.ToString(); + ASSERT_EQ(file4_info.file_path, file4); + ASSERT_EQ(file4_info.num_entries, 300); + ASSERT_EQ(file4_info.smallest_key, Key(1500)); + ASSERT_EQ(file4_info.largest_key, Key(1799)); + std::string file_checksum4, file_checksum_func_name4; + ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName( + file4, &file_checksum4, &file_checksum_func_name4)); + ASSERT_EQ(file4_info.file_checksum, file_checksum4); + ASSERT_EQ(file4_info.file_checksum_func_name, file_checksum_func_name4); + + // file05.sst (1800 => 1899) + std::string file5 = sst_files_dir_ + "file05.sst"; + ASSERT_OK(sst_file_writer.Open(file5)); + for (int k = 1800; k < 2000; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap")); + } + ExternalSstFileInfo file5_info; + s = sst_file_writer.Finish(&file5_info); + ASSERT_OK(s) << s.ToString(); + ASSERT_EQ(file5_info.file_path, file5); + ASSERT_EQ(file5_info.num_entries, 200); + ASSERT_EQ(file5_info.smallest_key, Key(1800)); + ASSERT_EQ(file5_info.largest_key, Key(1999)); + std::string file_checksum5, file_checksum_func_name5; + ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName( + file5, &file_checksum5, &file_checksum_func_name5)); + ASSERT_EQ(file5_info.file_checksum, file_checksum5); + ASSERT_EQ(file5_info.file_checksum_func_name, file_checksum_func_name5); + + // file06.sst (2000 => 2199) + std::string file6 = sst_files_dir_ + "file06.sst"; + ASSERT_OK(sst_file_writer.Open(file6)); + for (int k = 2000; k < 2200; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap")); + } + ExternalSstFileInfo file6_info; + s = sst_file_writer.Finish(&file6_info); + ASSERT_OK(s) << s.ToString(); + ASSERT_EQ(file6_info.file_path, file6); + ASSERT_EQ(file6_info.num_entries, 200); + ASSERT_EQ(file6_info.smallest_key, Key(2000)); + ASSERT_EQ(file6_info.largest_key, Key(2199)); + std::string file_checksum6, file_checksum_func_name6; + ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName( + file6, &file_checksum6, &file_checksum_func_name6)); + ASSERT_EQ(file6_info.file_checksum, file_checksum6); + ASSERT_EQ(file6_info.file_checksum_func_name, file_checksum_func_name6); + + s = AddFileWithFileChecksum({file1}, {file_checksum1, "xyz"}, + {file_checksum1}, true, false, false, false); + // does not care the checksum input since db does not enable file checksum + ASSERT_OK(s) << s.ToString(); + ASSERT_OK(env_->FileExists(file1)); + std::vector live_files; + dbfull()->GetLiveFilesMetaData(&live_files); + std::set set1; + for (auto f : live_files) { + set1.insert(f.name); + ASSERT_EQ(f.file_checksum, kUnknownFileChecksum); + ASSERT_EQ(f.file_checksum_func_name, kUnknownFileChecksumFuncName); + } + + // check the temperature of the file being ingested + ColumnFamilyMetaData metadata; + db_->GetColumnFamilyMetaData(&metadata); + ASSERT_EQ(1, metadata.file_count); + ASSERT_EQ(Temperature::kUnknown, metadata.levels[6].files[0].temperature); + auto size = GetSstSizeHelper(Temperature::kUnknown); + ASSERT_GT(size, 0); + size = GetSstSizeHelper(Temperature::kWarm); + ASSERT_EQ(size, 0); + size = GetSstSizeHelper(Temperature::kHot); + ASSERT_EQ(size, 0); + size = GetSstSizeHelper(Temperature::kCold); + ASSERT_EQ(size, 0); + + // Reopen Db with checksum enabled + Reopen(options); + // Enable verify_file_checksum option + // The checksum vector does not match, fail the ingestion + s = AddFileWithFileChecksum({file2}, {file_checksum2, "xyz"}, + {file_checksum_func_name2}, true, false, false, + false); + ASSERT_NOK(s) << s.ToString(); + + // Enable verify_file_checksum option + // The checksum name does not match, fail the ingestion + s = AddFileWithFileChecksum({file2}, {file_checksum2}, {"xyz"}, true, false, + false, false); + ASSERT_NOK(s) << s.ToString(); + + // Enable verify_file_checksum option + // The checksum itself does not match, fail the ingestion + s = AddFileWithFileChecksum({file2}, {"xyz"}, {file_checksum_func_name2}, + true, false, false, false); + ASSERT_NOK(s) << s.ToString(); + + // Enable verify_file_checksum option + // All matches, ingestion is successful + s = AddFileWithFileChecksum({file2}, {file_checksum2}, + {file_checksum_func_name2}, true, false, false, + false); + ASSERT_OK(s) << s.ToString(); + std::vector live_files1; + dbfull()->GetLiveFilesMetaData(&live_files1); + for (auto f : live_files1) { + if (set1.find(f.name) == set1.end()) { + ASSERT_EQ(f.file_checksum, file_checksum2); + ASSERT_EQ(f.file_checksum_func_name, file_checksum_func_name2); + set1.insert(f.name); + } + } + ASSERT_OK(env_->FileExists(file2)); + + // Enable verify_file_checksum option + // No checksum information is provided, generate it when ingesting + std::vector checksum, checksum_func; + s = AddFileWithFileChecksum({file3}, checksum, checksum_func, true, false, + false, false); + ASSERT_OK(s) << s.ToString(); + std::vector live_files2; + dbfull()->GetLiveFilesMetaData(&live_files2); + for (auto f : live_files2) { + if (set1.find(f.name) == set1.end()) { + ASSERT_EQ(f.file_checksum, file_checksum3); + ASSERT_EQ(f.file_checksum_func_name, file_checksum_func_name3); + set1.insert(f.name); + } + } + ASSERT_OK(s) << s.ToString(); + ASSERT_OK(env_->FileExists(file3)); + + // Does not enable verify_file_checksum options + // The checksum name does not match, fail the ingestion + s = AddFileWithFileChecksum({file4}, {file_checksum4}, {"xyz"}, false, false, + false, false); + ASSERT_NOK(s) << s.ToString(); + + // Does not enable verify_file_checksum options + // Checksum function name matches, store the checksum being ingested. + s = AddFileWithFileChecksum({file4}, {"asd"}, {file_checksum_func_name4}, + false, false, false, false); + ASSERT_OK(s) << s.ToString(); + std::vector live_files3; + dbfull()->GetLiveFilesMetaData(&live_files3); + for (auto f : live_files3) { + if (set1.find(f.name) == set1.end()) { + ASSERT_FALSE(f.file_checksum == file_checksum4); + ASSERT_EQ(f.file_checksum, "asd"); + ASSERT_EQ(f.file_checksum_func_name, file_checksum_func_name4); + set1.insert(f.name); + } + } + ASSERT_OK(s) << s.ToString(); + ASSERT_OK(env_->FileExists(file4)); + + // enable verify_file_checksum options, DB enable checksum, and enable + // write_global_seq. So the checksum stored is different from the one + // ingested due to the sequence number changes. + s = AddFileWithFileChecksum({file5}, {file_checksum5}, + {file_checksum_func_name5}, true, false, false, + true); + ASSERT_OK(s) << s.ToString(); + std::vector live_files4; + dbfull()->GetLiveFilesMetaData(&live_files4); + for (auto f : live_files4) { + if (set1.find(f.name) == set1.end()) { + std::string cur_checksum5, cur_checksum_func_name5; + ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName( + dbname_ + f.name, &cur_checksum5, &cur_checksum_func_name5)); + ASSERT_EQ(f.file_checksum, cur_checksum5); + ASSERT_EQ(f.file_checksum_func_name, file_checksum_func_name5); + set1.insert(f.name); + } + } + ASSERT_OK(s) << s.ToString(); + ASSERT_OK(env_->FileExists(file5)); + + // Does not enable verify_file_checksum options and also the ingested file + // checksum information is empty. DB will generate and store the checksum + // in Manifest. + std::vector files_c6, files_name6; + s = AddFileWithFileChecksum({file6}, files_c6, files_name6, false, false, + false, false); + ASSERT_OK(s) << s.ToString(); + std::vector live_files6; + dbfull()->GetLiveFilesMetaData(&live_files6); + for (auto f : live_files6) { + if (set1.find(f.name) == set1.end()) { + ASSERT_EQ(f.file_checksum, file_checksum6); + ASSERT_EQ(f.file_checksum_func_name, file_checksum_func_name6); + set1.insert(f.name); + } + } + ASSERT_OK(s) << s.ToString(); + ASSERT_OK(env_->FileExists(file6)); + db_->GetColumnFamilyMetaData(&metadata); + size = GetSstSizeHelper(Temperature::kUnknown); + ASSERT_GT(size, 0); + size = GetSstSizeHelper(Temperature::kWarm); + ASSERT_EQ(size, 0); + size = GetSstSizeHelper(Temperature::kHot); + ASSERT_EQ(size, 0); + size = GetSstSizeHelper(Temperature::kCold); + ASSERT_EQ(size, 0); +} + +TEST_F(ExternalSSTFileBasicTest, NoCopy) { + Options options = CurrentOptions(); + const ImmutableCFOptions ioptions(options); + + SstFileWriter sst_file_writer(EnvOptions(), options); + + // file1.sst (0 => 99) + std::string file1 = sst_files_dir_ + "file1.sst"; + ASSERT_OK(sst_file_writer.Open(file1)); + for (int k = 0; k < 100; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); + } + ExternalSstFileInfo file1_info; + Status s = sst_file_writer.Finish(&file1_info); + ASSERT_OK(s) << s.ToString(); + ASSERT_EQ(file1_info.file_path, file1); + ASSERT_EQ(file1_info.num_entries, 100); + ASSERT_EQ(file1_info.smallest_key, Key(0)); + ASSERT_EQ(file1_info.largest_key, Key(99)); + + // file2.sst (100 => 299) + std::string file2 = sst_files_dir_ + "file2.sst"; + ASSERT_OK(sst_file_writer.Open(file2)); + for (int k = 100; k < 300; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); + } + ExternalSstFileInfo file2_info; + s = sst_file_writer.Finish(&file2_info); + ASSERT_OK(s) << s.ToString(); + ASSERT_EQ(file2_info.file_path, file2); + ASSERT_EQ(file2_info.num_entries, 200); + ASSERT_EQ(file2_info.smallest_key, Key(100)); + ASSERT_EQ(file2_info.largest_key, Key(299)); + + // file3.sst (110 => 124) .. overlap with file2.sst + std::string file3 = sst_files_dir_ + "file3.sst"; + ASSERT_OK(sst_file_writer.Open(file3)); + for (int k = 110; k < 125; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap")); + } + ExternalSstFileInfo file3_info; + s = sst_file_writer.Finish(&file3_info); + ASSERT_OK(s) << s.ToString(); + ASSERT_EQ(file3_info.file_path, file3); + ASSERT_EQ(file3_info.num_entries, 15); + ASSERT_EQ(file3_info.smallest_key, Key(110)); + ASSERT_EQ(file3_info.largest_key, Key(124)); + + s = DeprecatedAddFile({file1}, true /* move file */); + ASSERT_OK(s) << s.ToString(); + ASSERT_EQ(Status::NotFound(), env_->FileExists(file1)); + + s = DeprecatedAddFile({file2}, false /* copy file */); + ASSERT_OK(s) << s.ToString(); + ASSERT_OK(env_->FileExists(file2)); + + // This file has overlapping values with the existing data + s = DeprecatedAddFile({file3}, true /* move file */); + ASSERT_NOK(s) << s.ToString(); + ASSERT_OK(env_->FileExists(file3)); + + for (int k = 0; k < 300; k++) { + ASSERT_EQ(Get(Key(k)), Key(k) + "_val"); + } +} + +TEST_P(ExternalSSTFileBasicTest, IngestFileWithGlobalSeqnoPickedSeqno) { + bool write_global_seqno = std::get<0>(GetParam()); + bool verify_checksums_before_ingest = std::get<1>(GetParam()); + do { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + DestroyAndReopen(options); + std::map true_data; + + int file_id = 1; + + ASSERT_OK(GenerateAndAddExternalFile( + options, {1, 2, 3, 4, 5, 6}, ValueType::kTypeValue, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // File doesn't overwrite any keys, no seqno needed + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {10, 11, 12, 13}, ValueType::kTypeValue, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // File doesn't overwrite any keys, no seqno needed + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {1, 4, 6}, ValueType::kTypeValue, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // File overwrites some keys, a seqno will be assigned + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 1); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {11, 15, 19}, ValueType::kTypeValue, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // File overwrites some keys, a seqno will be assigned + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {120, 130}, ValueType::kTypeValue, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // File doesn't overwrite any keys, no seqno needed + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {1, 130}, ValueType::kTypeValue, file_id++, write_global_seqno, + verify_checksums_before_ingest, &true_data)); + // File overwrites some keys, a seqno will be assigned + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 3); + + // Write some keys through normal write path + for (int i = 0; i < 50; i++) { + ASSERT_OK(Put(Key(i), "memtable")); + true_data[Key(i)] = "memtable"; + } + SequenceNumber last_seqno = dbfull()->GetLatestSequenceNumber(); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {60, 61, 62}, ValueType::kTypeValue, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // File doesn't overwrite any keys, no seqno needed + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {40, 41, 42}, ValueType::kTypeValue, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // File overwrites some keys, a seqno will be assigned + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 1); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {20, 30, 40}, ValueType::kTypeValue, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // File overwrites some keys, a seqno will be assigned + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 2); + + const Snapshot* snapshot = db_->GetSnapshot(); + + // We will need a seqno for the file regardless if the file overwrite + // keys in the DB or not because we have a snapshot + ASSERT_OK(GenerateAndAddExternalFile( + options, {1000, 1002}, ValueType::kTypeValue, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // A global seqno will be assigned anyway because of the snapshot + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 3); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {2000, 3002}, ValueType::kTypeValue, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // A global seqno will be assigned anyway because of the snapshot + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 4); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {1, 20, 40, 100, 150}, ValueType::kTypeValue, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // A global seqno will be assigned anyway because of the snapshot + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5); + + db_->ReleaseSnapshot(snapshot); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {5000, 5001}, ValueType::kTypeValue, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // No snapshot anymore, no need to assign a seqno + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5); + + size_t kcnt = 0; + VerifyDBFromMap(true_data, &kcnt, false); + } while (ChangeOptionsForFileIngestionTest()); +} + +TEST_P(ExternalSSTFileBasicTest, IngestFileWithMultipleValueType) { + bool write_global_seqno = std::get<0>(GetParam()); + bool verify_checksums_before_ingest = std::get<1>(GetParam()); + do { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.merge_operator.reset(new TestPutOperator()); + DestroyAndReopen(options); + std::map true_data; + + int file_id = 1; + + ASSERT_OK(GenerateAndAddExternalFile( + options, {1, 2, 3, 4, 5, 6}, ValueType::kTypeValue, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // File doesn't overwrite any keys, no seqno needed + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {10, 11, 12, 13}, ValueType::kTypeValue, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // File doesn't overwrite any keys, no seqno needed + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {1, 4, 6}, ValueType::kTypeMerge, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // File overwrites some keys, a seqno will be assigned + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 1); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {11, 15, 19}, ValueType::kTypeDeletion, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // File overwrites some keys, a seqno will be assigned + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {120, 130}, ValueType::kTypeMerge, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // File doesn't overwrite any keys, no seqno needed + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {1, 130}, ValueType::kTypeDeletion, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // File overwrites some keys, a seqno will be assigned + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 3); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {120}, {ValueType::kTypeValue}, {{120, 135}}, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // File overwrites some keys, a seqno will be assigned + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 4); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {}, {}, {{110, 120}}, file_id++, write_global_seqno, + verify_checksums_before_ingest, &true_data)); + // The range deletion ends on a key, but it doesn't actually delete + // this key because the largest key in the range is exclusive. Still, + // it counts as an overlap so a new seqno will be assigned. + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 5); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {}, {}, {{100, 109}}, file_id++, write_global_seqno, + verify_checksums_before_ingest, &true_data)); + // File doesn't overwrite any keys, no seqno needed + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 5); + + // Write some keys through normal write path + for (int i = 0; i < 50; i++) { + ASSERT_OK(Put(Key(i), "memtable")); + true_data[Key(i)] = "memtable"; + } + SequenceNumber last_seqno = dbfull()->GetLatestSequenceNumber(); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {60, 61, 62}, ValueType::kTypeValue, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // File doesn't overwrite any keys, no seqno needed + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {40, 41, 42}, ValueType::kTypeMerge, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // File overwrites some keys, a seqno will be assigned + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 1); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {20, 30, 40}, ValueType::kTypeDeletion, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // File overwrites some keys, a seqno will be assigned + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 2); + + const Snapshot* snapshot = db_->GetSnapshot(); + + // We will need a seqno for the file regardless if the file overwrite + // keys in the DB or not because we have a snapshot + ASSERT_OK(GenerateAndAddExternalFile( + options, {1000, 1002}, ValueType::kTypeMerge, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // A global seqno will be assigned anyway because of the snapshot + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 3); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {2000, 3002}, ValueType::kTypeMerge, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // A global seqno will be assigned anyway because of the snapshot + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 4); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {1, 20, 40, 100, 150}, ValueType::kTypeMerge, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // A global seqno will be assigned anyway because of the snapshot + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5); + + db_->ReleaseSnapshot(snapshot); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {5000, 5001}, ValueType::kTypeValue, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data)); + // No snapshot anymore, no need to assign a seqno + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5); + + size_t kcnt = 0; + VerifyDBFromMap(true_data, &kcnt, false); + } while (ChangeOptionsForFileIngestionTest()); +} + +TEST_P(ExternalSSTFileBasicTest, IngestFileWithMixedValueType) { + bool write_global_seqno = std::get<0>(GetParam()); + bool verify_checksums_before_ingest = std::get<1>(GetParam()); + do { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.merge_operator.reset(new TestPutOperator()); + DestroyAndReopen(options); + std::map true_data; + + int file_id = 1; + + ASSERT_OK(GenerateAndAddExternalFile( + options, {1, 2, 3, 4, 5, 6}, + {ValueType::kTypeValue, ValueType::kTypeMerge, ValueType::kTypeValue, + ValueType::kTypeMerge, ValueType::kTypeValue, ValueType::kTypeMerge}, + file_id++, write_global_seqno, verify_checksums_before_ingest, + &true_data)); + // File doesn't overwrite any keys, no seqno needed + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {10, 11, 12, 13}, + {ValueType::kTypeValue, ValueType::kTypeMerge, ValueType::kTypeValue, + ValueType::kTypeMerge}, + file_id++, write_global_seqno, verify_checksums_before_ingest, + &true_data)); + // File doesn't overwrite any keys, no seqno needed + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {1, 4, 6}, + {ValueType::kTypeDeletion, ValueType::kTypeValue, + ValueType::kTypeMerge}, + file_id++, write_global_seqno, verify_checksums_before_ingest, + &true_data)); + // File overwrites some keys, a seqno will be assigned + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 1); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {11, 15, 19}, + {ValueType::kTypeDeletion, ValueType::kTypeMerge, + ValueType::kTypeValue}, + file_id++, write_global_seqno, verify_checksums_before_ingest, + &true_data)); + // File overwrites some keys, a seqno will be assigned + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {120, 130}, {ValueType::kTypeValue, ValueType::kTypeMerge}, + file_id++, write_global_seqno, verify_checksums_before_ingest, + &true_data)); + // File doesn't overwrite any keys, no seqno needed + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {1, 130}, {ValueType::kTypeMerge, ValueType::kTypeDeletion}, + file_id++, write_global_seqno, verify_checksums_before_ingest, + &true_data)); + // File overwrites some keys, a seqno will be assigned + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 3); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {150, 151, 152}, + {ValueType::kTypeValue, ValueType::kTypeMerge, + ValueType::kTypeDeletion}, + {{150, 160}, {180, 190}}, file_id++, write_global_seqno, + verify_checksums_before_ingest, &true_data)); + // File doesn't overwrite any keys, no seqno needed + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 3); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {150, 151, 152}, + {ValueType::kTypeValue, ValueType::kTypeMerge, ValueType::kTypeValue}, + {{200, 250}}, file_id++, write_global_seqno, + verify_checksums_before_ingest, &true_data)); + // File overwrites some keys, a seqno will be assigned + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 4); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {300, 301, 302}, + {ValueType::kTypeValue, ValueType::kTypeMerge, + ValueType::kTypeDeletion}, + {{1, 2}, {152, 154}}, file_id++, write_global_seqno, + verify_checksums_before_ingest, &true_data)); + // File overwrites some keys, a seqno will be assigned + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 5); + + // Write some keys through normal write path + for (int i = 0; i < 50; i++) { + ASSERT_OK(Put(Key(i), "memtable")); + true_data[Key(i)] = "memtable"; + } + SequenceNumber last_seqno = dbfull()->GetLatestSequenceNumber(); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {60, 61, 62}, + {ValueType::kTypeValue, ValueType::kTypeMerge, ValueType::kTypeValue}, + file_id++, write_global_seqno, verify_checksums_before_ingest, + &true_data)); + // File doesn't overwrite any keys, no seqno needed + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {40, 41, 42}, + {ValueType::kTypeValue, ValueType::kTypeDeletion, + ValueType::kTypeDeletion}, + file_id++, write_global_seqno, verify_checksums_before_ingest, + &true_data)); + // File overwrites some keys, a seqno will be assigned + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 1); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {20, 30, 40}, + {ValueType::kTypeDeletion, ValueType::kTypeDeletion, + ValueType::kTypeDeletion}, + file_id++, write_global_seqno, verify_checksums_before_ingest, + &true_data)); + // File overwrites some keys, a seqno will be assigned + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 2); + + const Snapshot* snapshot = db_->GetSnapshot(); + + // We will need a seqno for the file regardless if the file overwrite + // keys in the DB or not because we have a snapshot + ASSERT_OK(GenerateAndAddExternalFile( + options, {1000, 1002}, {ValueType::kTypeValue, ValueType::kTypeMerge}, + file_id++, write_global_seqno, verify_checksums_before_ingest, + &true_data)); + // A global seqno will be assigned anyway because of the snapshot + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 3); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {2000, 3002}, {ValueType::kTypeValue, ValueType::kTypeMerge}, + file_id++, write_global_seqno, verify_checksums_before_ingest, + &true_data)); + // A global seqno will be assigned anyway because of the snapshot + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 4); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {1, 20, 40, 100, 150}, + {ValueType::kTypeDeletion, ValueType::kTypeDeletion, + ValueType::kTypeValue, ValueType::kTypeMerge, ValueType::kTypeMerge}, + file_id++, write_global_seqno, verify_checksums_before_ingest, + &true_data)); + // A global seqno will be assigned anyway because of the snapshot + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5); + + db_->ReleaseSnapshot(snapshot); + + ASSERT_OK(GenerateAndAddExternalFile( + options, {5000, 5001}, {ValueType::kTypeValue, ValueType::kTypeMerge}, + file_id++, write_global_seqno, verify_checksums_before_ingest, + &true_data)); + // No snapshot anymore, no need to assign a seqno + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5); + + size_t kcnt = 0; + VerifyDBFromMap(true_data, &kcnt, false); + } while (ChangeOptionsForFileIngestionTest()); +} + +TEST_F(ExternalSSTFileBasicTest, FadviseTrigger) { + Options options = CurrentOptions(); + const int kNumKeys = 10000; + + size_t total_fadvised_bytes = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "SstFileWriter::Rep::InvalidatePageCache", [&](void* arg) { + size_t fadvise_size = *(reinterpret_cast(arg)); + total_fadvised_bytes += fadvise_size; + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + std::unique_ptr sst_file_writer; + + std::string sst_file_path = sst_files_dir_ + "file_fadvise_disable.sst"; + sst_file_writer.reset( + new SstFileWriter(EnvOptions(), options, nullptr, false)); + ASSERT_OK(sst_file_writer->Open(sst_file_path)); + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(sst_file_writer->Put(Key(i), Key(i))); + } + ASSERT_OK(sst_file_writer->Finish()); + // fadvise disabled + ASSERT_EQ(total_fadvised_bytes, 0); + + sst_file_path = sst_files_dir_ + "file_fadvise_enable.sst"; + sst_file_writer.reset( + new SstFileWriter(EnvOptions(), options, nullptr, true)); + ASSERT_OK(sst_file_writer->Open(sst_file_path)); + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(sst_file_writer->Put(Key(i), Key(i))); + } + ASSERT_OK(sst_file_writer->Finish()); + // fadvise enabled + ASSERT_EQ(total_fadvised_bytes, sst_file_writer->FileSize()); + ASSERT_GT(total_fadvised_bytes, 0); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(ExternalSSTFileBasicTest, SyncFailure) { + Options options; + options.create_if_missing = true; + options.env = fault_injection_test_env_.get(); + + std::vector> test_cases = { + {"ExternalSstFileIngestionJob::BeforeSyncIngestedFile", + "ExternalSstFileIngestionJob::AfterSyncIngestedFile"}, + {"ExternalSstFileIngestionJob::BeforeSyncDir", + "ExternalSstFileIngestionJob::AfterSyncDir"}, + {"ExternalSstFileIngestionJob::BeforeSyncGlobalSeqno", + "ExternalSstFileIngestionJob::AfterSyncGlobalSeqno"}}; + + for (size_t i = 0; i < test_cases.size(); i++) { + bool no_sync = false; + SyncPoint::GetInstance()->SetCallBack(test_cases[i].first, [&](void*) { + fault_injection_test_env_->SetFilesystemActive(false); + }); + SyncPoint::GetInstance()->SetCallBack(test_cases[i].second, [&](void*) { + fault_injection_test_env_->SetFilesystemActive(true); + }); + if (i == 0) { + SyncPoint::GetInstance()->SetCallBack( + "ExternalSstFileIngestionJob::Prepare:Reopen", [&](void* s) { + Status* status = static_cast(s); + if (status->IsNotSupported()) { + no_sync = true; + } + }); + } + if (i == 2) { + SyncPoint::GetInstance()->SetCallBack( + "ExternalSstFileIngestionJob::NewRandomRWFile", [&](void* s) { + Status* status = static_cast(s); + if (status->IsNotSupported()) { + no_sync = true; + } + }); + } + SyncPoint::GetInstance()->EnableProcessing(); + + DestroyAndReopen(options); + if (i == 2) { + ASSERT_OK(Put("foo", "v1")); + } + + Options sst_file_writer_options; + sst_file_writer_options.env = fault_injection_test_env_.get(); + std::unique_ptr sst_file_writer( + new SstFileWriter(EnvOptions(), sst_file_writer_options)); + std::string file_name = + sst_files_dir_ + "sync_failure_test_" + std::to_string(i) + ".sst"; + ASSERT_OK(sst_file_writer->Open(file_name)); + ASSERT_OK(sst_file_writer->Put("bar", "v2")); + ASSERT_OK(sst_file_writer->Finish()); + + IngestExternalFileOptions ingest_opt; + ASSERT_FALSE(ingest_opt.write_global_seqno); // new default + if (i == 0) { + ingest_opt.move_files = true; + } + const Snapshot* snapshot = db_->GetSnapshot(); + if (i == 2) { + ingest_opt.write_global_seqno = true; + } + Status s = db_->IngestExternalFile({file_name}, ingest_opt); + if (no_sync) { + ASSERT_OK(s); + } else { + ASSERT_NOK(s); + } + db_->ReleaseSnapshot(snapshot); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + Destroy(options); + } +} + +TEST_F(ExternalSSTFileBasicTest, ReopenNotSupported) { + Options options; + options.create_if_missing = true; + options.env = env_; + + SyncPoint::GetInstance()->SetCallBack( + "ExternalSstFileIngestionJob::Prepare:Reopen", [&](void* arg) { + Status* s = static_cast(arg); + *s = Status::NotSupported(); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + DestroyAndReopen(options); + + Options sst_file_writer_options; + sst_file_writer_options.env = env_; + std::unique_ptr sst_file_writer( + new SstFileWriter(EnvOptions(), sst_file_writer_options)); + std::string file_name = + sst_files_dir_ + "reopen_not_supported_test_" + ".sst"; + ASSERT_OK(sst_file_writer->Open(file_name)); + ASSERT_OK(sst_file_writer->Put("bar", "v2")); + ASSERT_OK(sst_file_writer->Finish()); + + IngestExternalFileOptions ingest_opt; + ingest_opt.move_files = true; + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK(db_->IngestExternalFile({file_name}, ingest_opt)); + db_->ReleaseSnapshot(snapshot); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + Destroy(options); +} + +TEST_F(ExternalSSTFileBasicTest, VerifyChecksumReadahead) { + Options options; + options.create_if_missing = true; + SpecialEnv senv(env_); + options.env = &senv; + DestroyAndReopen(options); + + Options sst_file_writer_options; + sst_file_writer_options.env = env_; + std::unique_ptr sst_file_writer( + new SstFileWriter(EnvOptions(), sst_file_writer_options)); + std::string file_name = sst_files_dir_ + "verify_checksum_readahead_test.sst"; + ASSERT_OK(sst_file_writer->Open(file_name)); + Random rnd(301); + std::string value = rnd.RandomString(4000); + for (int i = 0; i < 5000; i++) { + ASSERT_OK(sst_file_writer->Put(DBTestBase::Key(i), value)); + } + ASSERT_OK(sst_file_writer->Finish()); + + // Ingest it once without verifying checksums to see the baseline + // preads. + IngestExternalFileOptions ingest_opt; + ingest_opt.move_files = false; + senv.count_random_reads_ = true; + senv.random_read_bytes_counter_ = 0; + ASSERT_OK(db_->IngestExternalFile({file_name}, ingest_opt)); + + auto base_num_reads = senv.random_read_counter_.Read(); + // Make sure the counter is enabled. + ASSERT_GT(base_num_reads, 0); + + // Ingest again and observe the reads made for for readahead. + ingest_opt.move_files = false; + ingest_opt.verify_checksums_before_ingest = true; + ingest_opt.verify_checksums_readahead_size = size_t{2 * 1024 * 1024}; + + senv.count_random_reads_ = true; + senv.random_read_bytes_counter_ = 0; + ASSERT_OK(db_->IngestExternalFile({file_name}, ingest_opt)); + + // Make sure the counter is enabled. + ASSERT_GT(senv.random_read_counter_.Read() - base_num_reads, 0); + + // The SST file is about 20MB. Readahead size is 2MB. + // Give a conservative 15 reads for metadata blocks, the number + // of random reads should be within 20 MB / 2MB + 15 = 25. + ASSERT_LE(senv.random_read_counter_.Read() - base_num_reads, 40); + + Destroy(options); +} + +TEST_F(ExternalSSTFileBasicTest, IngestRangeDeletionTombstoneWithGlobalSeqno) { + for (int i = 5; i < 25; i++) { + ASSERT_OK(db_->Put(WriteOptions(), db_->DefaultColumnFamily(), Key(i), + Key(i) + "_val")); + } + + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + Reopen(options); + SstFileWriter sst_file_writer(EnvOptions(), options); + + // file.sst (delete 0 => 30) + std::string file = sst_files_dir_ + "file.sst"; + ASSERT_OK(sst_file_writer.Open(file)); + ASSERT_OK(sst_file_writer.DeleteRange(Key(0), Key(30))); + ExternalSstFileInfo file_info; + ASSERT_OK(sst_file_writer.Finish(&file_info)); + ASSERT_EQ(file_info.file_path, file); + ASSERT_EQ(file_info.num_entries, 0); + ASSERT_EQ(file_info.smallest_key, ""); + ASSERT_EQ(file_info.largest_key, ""); + ASSERT_EQ(file_info.num_range_del_entries, 1); + ASSERT_EQ(file_info.smallest_range_del_key, Key(0)); + ASSERT_EQ(file_info.largest_range_del_key, Key(30)); + + IngestExternalFileOptions ifo; + ifo.move_files = true; + ifo.snapshot_consistency = true; + ifo.allow_global_seqno = true; + ifo.write_global_seqno = true; + ifo.verify_checksums_before_ingest = false; + ASSERT_OK(db_->IngestExternalFile({file}, ifo)); + + for (int i = 5; i < 25; i++) { + std::string res; + ASSERT_TRUE(db_->Get(ReadOptions(), Key(i), &res).IsNotFound()); + } +} + +TEST_P(ExternalSSTFileBasicTest, IngestionWithRangeDeletions) { + int kNumLevels = 7; + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.num_levels = kNumLevels; + Reopen(options); + + std::map true_data; + int file_id = 1; + // prevent range deletions from being dropped due to becoming obsolete. + const Snapshot* snapshot = db_->GetSnapshot(); + + // range del [0, 50) in L6 file, [50, 100) in L0 file, [100, 150) in memtable + for (int i = 0; i < 3; i++) { + if (i != 0) { + db_->Flush(FlushOptions()); + if (i == 1) { + MoveFilesToLevel(kNumLevels - 1); + } + } + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(50 * i), Key(50 * (i + 1)))); + } + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + ASSERT_EQ(0, NumTableFilesAtLevel(kNumLevels - 2)); + ASSERT_EQ(1, NumTableFilesAtLevel(kNumLevels - 1)); + + bool write_global_seqno = std::get<0>(GetParam()); + bool verify_checksums_before_ingest = std::get<1>(GetParam()); + // overlaps with L0 file but not memtable, so flush is skipped and file is + // ingested into L0 + SequenceNumber last_seqno = dbfull()->GetLatestSequenceNumber(); + ASSERT_OK(GenerateAndAddExternalFile( + options, {60, 90}, {ValueType::kTypeValue, ValueType::kTypeValue}, + {{65, 70}, {70, 85}}, file_id++, write_global_seqno, + verify_checksums_before_ingest, &true_data)); + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), ++last_seqno); + ASSERT_EQ(2, NumTableFilesAtLevel(0)); + ASSERT_EQ(0, NumTableFilesAtLevel(kNumLevels - 2)); + ASSERT_EQ(1, NumTableFilesAtLevel(options.num_levels - 1)); + + // overlaps with L6 file but not memtable or L0 file, so flush is skipped and + // file is ingested into L5 + ASSERT_OK(GenerateAndAddExternalFile( + options, {10, 40}, {ValueType::kTypeValue, ValueType::kTypeValue}, + file_id++, write_global_seqno, verify_checksums_before_ingest, + &true_data)); + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), ++last_seqno); + ASSERT_EQ(2, NumTableFilesAtLevel(0)); + ASSERT_EQ(1, NumTableFilesAtLevel(kNumLevels - 2)); + ASSERT_EQ(1, NumTableFilesAtLevel(options.num_levels - 1)); + + // overlaps with L5 file but not memtable or L0 file, so flush is skipped and + // file is ingested into L4 + ASSERT_OK(GenerateAndAddExternalFile( + options, {}, {}, {{5, 15}}, file_id++, write_global_seqno, + verify_checksums_before_ingest, &true_data)); + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), ++last_seqno); + ASSERT_EQ(2, NumTableFilesAtLevel(0)); + ASSERT_EQ(1, NumTableFilesAtLevel(kNumLevels - 2)); + ASSERT_EQ(1, NumTableFilesAtLevel(options.num_levels - 2)); + ASSERT_EQ(1, NumTableFilesAtLevel(options.num_levels - 1)); + + // ingested file overlaps with memtable, so flush is triggered before the file + // is ingested such that the ingested data is considered newest. So L0 file + // count increases by two. + ASSERT_OK(GenerateAndAddExternalFile( + options, {100, 140}, {ValueType::kTypeValue, ValueType::kTypeValue}, + file_id++, write_global_seqno, verify_checksums_before_ingest, + &true_data)); + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), ++last_seqno); + ASSERT_EQ(4, NumTableFilesAtLevel(0)); + ASSERT_EQ(1, NumTableFilesAtLevel(kNumLevels - 2)); + ASSERT_EQ(1, NumTableFilesAtLevel(options.num_levels - 1)); + + // snapshot unneeded now that all range deletions are persisted + db_->ReleaseSnapshot(snapshot); + + // overlaps with nothing, so places at bottom level and skips incrementing + // seqnum. + ASSERT_OK(GenerateAndAddExternalFile( + options, {151, 175}, {ValueType::kTypeValue, ValueType::kTypeValue}, + {{160, 200}}, file_id++, write_global_seqno, + verify_checksums_before_ingest, &true_data)); + ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno); + ASSERT_EQ(4, NumTableFilesAtLevel(0)); + ASSERT_EQ(1, NumTableFilesAtLevel(kNumLevels - 2)); + ASSERT_EQ(2, NumTableFilesAtLevel(options.num_levels - 1)); + VerifyDBFromMap(true_data); +} + +TEST_F(ExternalSSTFileBasicTest, AdjacentRangeDeletionTombstones) { + Options options = CurrentOptions(); + SstFileWriter sst_file_writer(EnvOptions(), options); + + // file8.sst (delete 300 => 400) + std::string file8 = sst_files_dir_ + "file8.sst"; + ASSERT_OK(sst_file_writer.Open(file8)); + ASSERT_OK(sst_file_writer.DeleteRange(Key(300), Key(400))); + ExternalSstFileInfo file8_info; + Status s = sst_file_writer.Finish(&file8_info); + ASSERT_OK(s) << s.ToString(); + ASSERT_EQ(file8_info.file_path, file8); + ASSERT_EQ(file8_info.num_entries, 0); + ASSERT_EQ(file8_info.smallest_key, ""); + ASSERT_EQ(file8_info.largest_key, ""); + ASSERT_EQ(file8_info.num_range_del_entries, 1); + ASSERT_EQ(file8_info.smallest_range_del_key, Key(300)); + ASSERT_EQ(file8_info.largest_range_del_key, Key(400)); + + // file9.sst (delete 400 => 500) + std::string file9 = sst_files_dir_ + "file9.sst"; + ASSERT_OK(sst_file_writer.Open(file9)); + ASSERT_OK(sst_file_writer.DeleteRange(Key(400), Key(500))); + ExternalSstFileInfo file9_info; + s = sst_file_writer.Finish(&file9_info); + ASSERT_OK(s) << s.ToString(); + ASSERT_EQ(file9_info.file_path, file9); + ASSERT_EQ(file9_info.num_entries, 0); + ASSERT_EQ(file9_info.smallest_key, ""); + ASSERT_EQ(file9_info.largest_key, ""); + ASSERT_EQ(file9_info.num_range_del_entries, 1); + ASSERT_EQ(file9_info.smallest_range_del_key, Key(400)); + ASSERT_EQ(file9_info.largest_range_del_key, Key(500)); + + // Range deletion tombstones are exclusive on their end key, so these SSTs + // should not be considered as overlapping. + s = DeprecatedAddFile({file8, file9}); + ASSERT_OK(s) << s.ToString(); + ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U); + DestroyAndRecreateExternalSSTFilesDir(); +} + +TEST_F(ExternalSSTFileBasicTest, UnorderedRangeDeletions) { + int kNumLevels = 7; + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.num_levels = kNumLevels; + Reopen(options); + + std::map true_data; + int file_id = 1; + + // prevent range deletions from being dropped due to becoming obsolete. + const Snapshot* snapshot = db_->GetSnapshot(); + + // Range del [0, 50) in memtable + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0), + Key(50))); + + // Out of order range del overlaps memtable, so flush is required before file + // is ingested into L0 + ASSERT_OK(GenerateAndAddExternalFile( + options, {60, 90}, {ValueType::kTypeValue, ValueType::kTypeValue}, + {{65, 70}, {45, 50}}, file_id++, true /* write_global_seqno */, + true /* verify_checksums_before_ingest */, &true_data)); + ASSERT_EQ(2, true_data.size()); + ASSERT_EQ(2, NumTableFilesAtLevel(0)); + ASSERT_EQ(0, NumTableFilesAtLevel(kNumLevels - 1)); + VerifyDBFromMap(true_data); + + // Compact to L6 + MoveFilesToLevel(kNumLevels - 1); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_EQ(1, NumTableFilesAtLevel(kNumLevels - 1)); + VerifyDBFromMap(true_data); + + // Ingest a file containing out of order range dels that cover nothing + ASSERT_OK(GenerateAndAddExternalFile( + options, {151, 175}, {ValueType::kTypeValue, ValueType::kTypeValue}, + {{160, 200}, {120, 180}}, file_id++, true /* write_global_seqno */, + true /* verify_checksums_before_ingest */, &true_data)); + ASSERT_EQ(4, true_data.size()); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + ASSERT_EQ(2, NumTableFilesAtLevel(kNumLevels - 1)); + VerifyDBFromMap(true_data); + + // Ingest a file containing out of order range dels that cover keys in L6 + ASSERT_OK(GenerateAndAddExternalFile( + options, {}, {}, {{190, 200}, {170, 180}, {55, 65}}, file_id++, + true /* write_global_seqno */, true /* verify_checksums_before_ingest */, + &true_data)); + ASSERT_EQ(2, true_data.size()); + ASSERT_EQ(1, NumTableFilesAtLevel(kNumLevels - 2)); + ASSERT_EQ(2, NumTableFilesAtLevel(kNumLevels - 1)); + VerifyDBFromMap(true_data); + + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(ExternalSSTFileBasicTest, RangeDeletionEndComesBeforeStart) { + Options options = CurrentOptions(); + SstFileWriter sst_file_writer(EnvOptions(), options); + + // "file.sst" + // Verify attempt to delete 300 => 200 fails. + // Then, verify attempt to delete 300 => 300 succeeds but writes nothing. + // Afterwards, verify attempt to delete 300 => 400 works normally. + std::string file = sst_files_dir_ + "file.sst"; + ASSERT_OK(sst_file_writer.Open(file)); + ASSERT_TRUE( + sst_file_writer.DeleteRange(Key(300), Key(200)).IsInvalidArgument()); + ASSERT_OK(sst_file_writer.DeleteRange(Key(300), Key(300))); + ASSERT_OK(sst_file_writer.DeleteRange(Key(300), Key(400))); + ExternalSstFileInfo file_info; + Status s = sst_file_writer.Finish(&file_info); + ASSERT_OK(s) << s.ToString(); + ASSERT_EQ(file_info.file_path, file); + ASSERT_EQ(file_info.num_entries, 0); + ASSERT_EQ(file_info.smallest_key, ""); + ASSERT_EQ(file_info.largest_key, ""); + ASSERT_EQ(file_info.num_range_del_entries, 1); + ASSERT_EQ(file_info.smallest_range_del_key, Key(300)); + ASSERT_EQ(file_info.largest_range_del_key, Key(400)); +} + +TEST_P(ExternalSSTFileBasicTest, IngestFileWithBadBlockChecksum) { + bool change_checksum_called = false; + const auto& change_checksum = [&](void* arg) { + if (!change_checksum_called) { + char* buf = reinterpret_cast(arg); + assert(nullptr != buf); + buf[0] ^= 0x1; + change_checksum_called = true; + } + }; + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTableBuilder::WriteMaybeCompressedBlock:TamperWithChecksum", + change_checksum); + SyncPoint::GetInstance()->EnableProcessing(); + int file_id = 0; + bool write_global_seqno = std::get<0>(GetParam()); + bool verify_checksums_before_ingest = std::get<1>(GetParam()); + do { + Options options = CurrentOptions(); + DestroyAndReopen(options); + std::map true_data; + Status s = GenerateAndAddExternalFile( + options, {1, 2, 3, 4, 5, 6}, ValueType::kTypeValue, file_id++, + write_global_seqno, verify_checksums_before_ingest, &true_data); + if (verify_checksums_before_ingest) { + ASSERT_NOK(s); + } else { + ASSERT_OK(s); + } + change_checksum_called = false; + } while (ChangeOptionsForFileIngestionTest()); +} + +TEST_P(ExternalSSTFileBasicTest, IngestFileWithFirstByteTampered) { + if (!random_rwfile_supported_) { + ROCKSDB_GTEST_SKIP("Test requires NewRandomRWFile support"); + return; + } + SyncPoint::GetInstance()->DisableProcessing(); + int file_id = 0; + EnvOptions env_options; + do { + Options options = CurrentOptions(); + std::string file_path = sst_files_dir_ + std::to_string(file_id++); + SstFileWriter sst_file_writer(env_options, options); + Status s = sst_file_writer.Open(file_path); + ASSERT_OK(s); + for (int i = 0; i != 100; ++i) { + std::string key = Key(i); + std::string value = Key(i) + std::to_string(0); + ASSERT_OK(sst_file_writer.Put(key, value)); + } + ASSERT_OK(sst_file_writer.Finish()); + { + // Get file size + uint64_t file_size = 0; + ASSERT_OK(env_->GetFileSize(file_path, &file_size)); + ASSERT_GT(file_size, 8); + std::unique_ptr rwfile; + ASSERT_OK(env_->NewRandomRWFile(file_path, &rwfile, EnvOptions())); + // Manually corrupt the file + // We deterministically corrupt the first byte because we currently + // cannot choose a random offset. The reason for this limitation is that + // we do not checksum property block at present. + const uint64_t offset = 0; + char scratch[8] = {0}; + Slice buf; + ASSERT_OK(rwfile->Read(offset, sizeof(scratch), &buf, scratch)); + scratch[0] ^= 0xff; // flip one bit + ASSERT_OK(rwfile->Write(offset, buf)); + } + // Ingest file. + IngestExternalFileOptions ifo; + ifo.write_global_seqno = std::get<0>(GetParam()); + ifo.verify_checksums_before_ingest = std::get<1>(GetParam()); + s = db_->IngestExternalFile({file_path}, ifo); + if (ifo.verify_checksums_before_ingest) { + ASSERT_NOK(s); + } else { + ASSERT_OK(s); + } + } while (ChangeOptionsForFileIngestionTest()); +} + +TEST_P(ExternalSSTFileBasicTest, IngestExternalFileWithCorruptedPropsBlock) { + bool verify_checksums_before_ingest = std::get<1>(GetParam()); + if (!verify_checksums_before_ingest) { + ROCKSDB_GTEST_BYPASS("Bypassing test when !verify_checksums_before_ingest"); + return; + } + if (!random_rwfile_supported_) { + ROCKSDB_GTEST_SKIP("Test requires NewRandomRWFile support"); + return; + } + uint64_t props_block_offset = 0; + size_t props_block_size = 0; + const auto& get_props_block_offset = [&](void* arg) { + props_block_offset = *reinterpret_cast(arg); + }; + const auto& get_props_block_size = [&](void* arg) { + props_block_size = *reinterpret_cast(arg); + }; + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTableBuilder::WritePropertiesBlock:GetPropsBlockOffset", + get_props_block_offset); + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTableBuilder::WritePropertiesBlock:GetPropsBlockSize", + get_props_block_size); + SyncPoint::GetInstance()->EnableProcessing(); + int file_id = 0; + Random64 rand(time(nullptr)); + do { + std::string file_path = sst_files_dir_ + std::to_string(file_id++); + Options options = CurrentOptions(); + SstFileWriter sst_file_writer(EnvOptions(), options); + Status s = sst_file_writer.Open(file_path); + ASSERT_OK(s); + for (int i = 0; i != 100; ++i) { + std::string key = Key(i); + std::string value = Key(i) + std::to_string(0); + ASSERT_OK(sst_file_writer.Put(key, value)); + } + ASSERT_OK(sst_file_writer.Finish()); + + { + std::unique_ptr rwfile; + ASSERT_OK(env_->NewRandomRWFile(file_path, &rwfile, EnvOptions())); + // Manually corrupt the file + ASSERT_GT(props_block_size, 8); + uint64_t offset = + props_block_offset + rand.Next() % (props_block_size - 8); + char scratch[8] = {0}; + Slice buf; + ASSERT_OK(rwfile->Read(offset, sizeof(scratch), &buf, scratch)); + scratch[0] ^= 0xff; // flip one bit + ASSERT_OK(rwfile->Write(offset, buf)); + } + + // Ingest file. + IngestExternalFileOptions ifo; + ifo.write_global_seqno = std::get<0>(GetParam()); + ifo.verify_checksums_before_ingest = true; + s = db_->IngestExternalFile({file_path}, ifo); + ASSERT_NOK(s); + } while (ChangeOptionsForFileIngestionTest()); +} + +TEST_F(ExternalSSTFileBasicTest, OverlappingFiles) { + Options options = CurrentOptions(); + + std::vector files; + { + SstFileWriter sst_file_writer(EnvOptions(), options); + std::string file1 = sst_files_dir_ + "file1.sst"; + ASSERT_OK(sst_file_writer.Open(file1)); + ASSERT_OK(sst_file_writer.Put("a", "z")); + ASSERT_OK(sst_file_writer.Put("i", "m")); + ExternalSstFileInfo file1_info; + ASSERT_OK(sst_file_writer.Finish(&file1_info)); + files.push_back(std::move(file1)); + } + { + SstFileWriter sst_file_writer(EnvOptions(), options); + std::string file2 = sst_files_dir_ + "file2.sst"; + ASSERT_OK(sst_file_writer.Open(file2)); + ASSERT_OK(sst_file_writer.Put("i", "k")); + ExternalSstFileInfo file2_info; + ASSERT_OK(sst_file_writer.Finish(&file2_info)); + files.push_back(std::move(file2)); + } + + IngestExternalFileOptions ifo; + ASSERT_OK(db_->IngestExternalFile(files, ifo)); + ASSERT_EQ(Get("a"), "z"); + ASSERT_EQ(Get("i"), "k"); + + int total_keys = 0; + Iterator* iter = db_->NewIterator(ReadOptions()); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_OK(iter->status()); + total_keys++; + } + delete iter; + ASSERT_EQ(total_keys, 2); + + ASSERT_EQ(2, NumTableFilesAtLevel(0)); +} + +TEST_F(ExternalSSTFileBasicTest, IngestFileAfterDBPut) { + // Repro https://github.com/facebook/rocksdb/issues/6245. + // Flush three files to L0. Ingest one more file to trigger L0->L1 compaction + // via trivial move. The bug happened when L1 files were incorrectly sorted + // resulting in an old value for "k" returned by `Get()`. + Options options = CurrentOptions(); + + ASSERT_OK(Put("k", "a")); + Flush(); + ASSERT_OK(Put("k", "a")); + Flush(); + ASSERT_OK(Put("k", "a")); + Flush(); + SstFileWriter sst_file_writer(EnvOptions(), options); + + // Current file size should be 0 after sst_file_writer init and before open a + // file. + ASSERT_EQ(sst_file_writer.FileSize(), 0); + + std::string file1 = sst_files_dir_ + "file1.sst"; + ASSERT_OK(sst_file_writer.Open(file1)); + ASSERT_OK(sst_file_writer.Put("k", "b")); + + ExternalSstFileInfo file1_info; + Status s = sst_file_writer.Finish(&file1_info); + ASSERT_OK(s) << s.ToString(); + + // Current file size should be non-zero after success write. + ASSERT_GT(sst_file_writer.FileSize(), 0); + + IngestExternalFileOptions ifo; + s = db_->IngestExternalFile({file1}, ifo); + ASSERT_OK(s); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ(Get("k"), "b"); +} + +TEST_F(ExternalSSTFileBasicTest, IngestWithTemperature) { + Options options = CurrentOptions(); + const ImmutableCFOptions ioptions(options); + options.bottommost_temperature = Temperature::kWarm; + SstFileWriter sst_file_writer(EnvOptions(), options); + options.level0_file_num_compaction_trigger = 2; + Reopen(options); + + auto size = GetSstSizeHelper(Temperature::kUnknown); + ASSERT_EQ(size, 0); + size = GetSstSizeHelper(Temperature::kWarm); + ASSERT_EQ(size, 0); + size = GetSstSizeHelper(Temperature::kHot); + ASSERT_EQ(size, 0); + + // create file01.sst (1000 => 1099) and ingest it + std::string file1 = sst_files_dir_ + "file01.sst"; + ASSERT_OK(sst_file_writer.Open(file1)); + for (int k = 1000; k < 1100; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); + } + ExternalSstFileInfo file1_info; + Status s = sst_file_writer.Finish(&file1_info); + ASSERT_OK(s); + ASSERT_EQ(file1_info.file_path, file1); + ASSERT_EQ(file1_info.num_entries, 100); + ASSERT_EQ(file1_info.smallest_key, Key(1000)); + ASSERT_EQ(file1_info.largest_key, Key(1099)); + + std::vector files; + std::vector files_checksums; + std::vector files_checksum_func_names; + Temperature file_temperature = Temperature::kWarm; + + files.push_back(file1); + IngestExternalFileOptions in_opts; + in_opts.move_files = false; + in_opts.snapshot_consistency = true; + in_opts.allow_global_seqno = false; + in_opts.allow_blocking_flush = false; + in_opts.write_global_seqno = true; + in_opts.verify_file_checksum = false; + IngestExternalFileArg arg; + arg.column_family = db_->DefaultColumnFamily(); + arg.external_files = files; + arg.options = in_opts; + arg.files_checksums = files_checksums; + arg.files_checksum_func_names = files_checksum_func_names; + arg.file_temperature = file_temperature; + s = db_->IngestExternalFiles({arg}); + ASSERT_OK(s); + + // check the temperature of the file being ingested + ColumnFamilyMetaData metadata; + db_->GetColumnFamilyMetaData(&metadata); + ASSERT_EQ(1, metadata.file_count); + ASSERT_EQ(Temperature::kWarm, metadata.levels[6].files[0].temperature); + size = GetSstSizeHelper(Temperature::kUnknown); + ASSERT_EQ(size, 0); + size = GetSstSizeHelper(Temperature::kWarm); + ASSERT_GT(size, 1); + + // non-bottommost file still has unknown temperature + ASSERT_OK(Put("foo", "bar")); + ASSERT_OK(Put("bar", "bar")); + ASSERT_OK(Flush()); + db_->GetColumnFamilyMetaData(&metadata); + ASSERT_EQ(2, metadata.file_count); + ASSERT_EQ(Temperature::kUnknown, metadata.levels[0].files[0].temperature); + size = GetSstSizeHelper(Temperature::kUnknown); + ASSERT_GT(size, 0); + size = GetSstSizeHelper(Temperature::kWarm); + ASSERT_GT(size, 0); + + // reopen and check the information is persisted + Reopen(options); + db_->GetColumnFamilyMetaData(&metadata); + ASSERT_EQ(2, metadata.file_count); + ASSERT_EQ(Temperature::kUnknown, metadata.levels[0].files[0].temperature); + ASSERT_EQ(Temperature::kWarm, metadata.levels[6].files[0].temperature); + size = GetSstSizeHelper(Temperature::kUnknown); + ASSERT_GT(size, 0); + size = GetSstSizeHelper(Temperature::kWarm); + ASSERT_GT(size, 0); + + // check other non-exist temperatures + size = GetSstSizeHelper(Temperature::kHot); + ASSERT_EQ(size, 0); + size = GetSstSizeHelper(Temperature::kCold); + ASSERT_EQ(size, 0); + std::string prop; + ASSERT_TRUE(dbfull()->GetProperty( + DB::Properties::kLiveSstFilesSizeAtTemperature + std::to_string(22), + &prop)); + ASSERT_EQ(std::atoi(prop.c_str()), 0); +} + +TEST_F(ExternalSSTFileBasicTest, FailIfNotBottommostLevel) { + Options options = GetDefaultOptions(); + + std::string file_path = sst_files_dir_ + std::to_string(1); + SstFileWriter sfw(EnvOptions(), options); + + ASSERT_OK(sfw.Open(file_path)); + ASSERT_OK(sfw.Put("b", "dontcare")); + ASSERT_OK(sfw.Finish()); + + // Test universal compaction + ingest with snapshot consistency + options.create_if_missing = true; + options.compaction_style = CompactionStyle::kCompactionStyleUniversal; + DestroyAndReopen(options); + { + const Snapshot* snapshot = db_->GetSnapshot(); + ManagedSnapshot snapshot_guard(db_, snapshot); + IngestExternalFileOptions ifo; + ifo.fail_if_not_bottommost_level = true; + ifo.snapshot_consistency = true; + const Status s = db_->IngestExternalFile({file_path}, ifo); + ASSERT_TRUE(s.IsTryAgain()); + } + + // Test level compaction + options.compaction_style = CompactionStyle::kCompactionStyleLevel; + options.num_levels = 2; + DestroyAndReopen(options); + ASSERT_OK(db_->Put(WriteOptions(), "a", "dontcare")); + ASSERT_OK(db_->Put(WriteOptions(), "c", "dontcare")); + ASSERT_OK(db_->Flush(FlushOptions())); + + ASSERT_OK(db_->Put(WriteOptions(), "b", "dontcare")); + ASSERT_OK(db_->Put(WriteOptions(), "d", "dontcare")); + ASSERT_OK(db_->Flush(FlushOptions())); + + { + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + IngestExternalFileOptions ifo; + ifo.fail_if_not_bottommost_level = true; + const Status s = db_->IngestExternalFile({file_path}, ifo); + ASSERT_TRUE(s.IsTryAgain()); + } +} + +TEST_F(ExternalSSTFileBasicTest, VerifyChecksum) { + const std::string kPutVal = "put_val"; + const std::string kIngestedVal = "ingested_val"; + + ASSERT_OK(Put("k", kPutVal, WriteOptions())); + ASSERT_OK(Flush()); + + std::string external_file = sst_files_dir_ + "/file_to_ingest.sst"; + { + SstFileWriter sst_file_writer{EnvOptions(), CurrentOptions()}; + + ASSERT_OK(sst_file_writer.Open(external_file)); + ASSERT_OK(sst_file_writer.Put("k", kIngestedVal)); + ASSERT_OK(sst_file_writer.Finish()); + } + + ASSERT_OK(db_->IngestExternalFile(db_->DefaultColumnFamily(), {external_file}, + IngestExternalFileOptions())); + + ASSERT_OK(db_->VerifyChecksum()); +} + +TEST_F(ExternalSSTFileBasicTest, VerifySstUniqueId) { + const std::string kPutVal = "put_val"; + const std::string kIngestedVal = "ingested_val"; + + ASSERT_OK(Put("k", kPutVal, WriteOptions())); + ASSERT_OK(Flush()); + + std::string external_file = sst_files_dir_ + "/file_to_ingest.sst"; + { + SstFileWriter sst_file_writer{EnvOptions(), CurrentOptions()}; + + ASSERT_OK(sst_file_writer.Open(external_file)); + ASSERT_OK(sst_file_writer.Put("k", kIngestedVal)); + ASSERT_OK(sst_file_writer.Finish()); + } + + ASSERT_OK(db_->IngestExternalFile(db_->DefaultColumnFamily(), {external_file}, + IngestExternalFileOptions())); + + // Test ingest file without session_id and db_id (for example generated by an + // older version of sst_writer) + SyncPoint::GetInstance()->SetCallBack( + "PropertyBlockBuilder::AddTableProperty:Start", [&](void* props_vs) { + auto props = static_cast(props_vs); + // update table property session_id to a different one + props->db_session_id = ""; + props->db_id = ""; + }); + std::atomic_int skipped = 0, passed = 0; + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTable::Open::SkippedVerifyUniqueId", + [&](void* /*arg*/) { skipped++; }); + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTable::Open::PassedVerifyUniqueId", + [&](void* /*arg*/) { passed++; }); + SyncPoint::GetInstance()->EnableProcessing(); + + auto options = CurrentOptions(); + ASSERT_TRUE(options.verify_sst_unique_id_in_manifest); + Reopen(options); + ASSERT_EQ(skipped, 0); + ASSERT_EQ(passed, 2); // one flushed + one ingested + + external_file = sst_files_dir_ + "/file_to_ingest2.sst"; + { + SstFileWriter sst_file_writer{EnvOptions(), CurrentOptions()}; + + ASSERT_OK(sst_file_writer.Open(external_file)); + ASSERT_OK(sst_file_writer.Put("k", kIngestedVal)); + ASSERT_OK(sst_file_writer.Finish()); + } + + ASSERT_OK(db_->IngestExternalFile(db_->DefaultColumnFamily(), {external_file}, + IngestExternalFileOptions())); + + // Two table file opens skipping verification: + // * ExternalSstFileIngestionJob::GetIngestedFileInfo + // * TableCache::GetTableReader + ASSERT_EQ(skipped, 2); + ASSERT_EQ(passed, 2); + + // Check same after re-open (except no GetIngestedFileInfo) + skipped = 0; + passed = 0; + Reopen(options); + ASSERT_EQ(skipped, 1); + ASSERT_EQ(passed, 2); +} + +TEST_F(ExternalSSTFileBasicTest, StableSnapshotWhileLoggingToManifest) { + const std::string kPutVal = "put_val"; + const std::string kIngestedVal = "ingested_val"; + + ASSERT_OK(Put("k", kPutVal, WriteOptions())); + ASSERT_OK(Flush()); + + std::string external_file = sst_files_dir_ + "/file_to_ingest.sst"; + { + SstFileWriter sst_file_writer{EnvOptions(), CurrentOptions()}; + ASSERT_OK(sst_file_writer.Open(external_file)); + ASSERT_OK(sst_file_writer.Put("k", kIngestedVal)); + ASSERT_OK(sst_file_writer.Finish()); + } + + const Snapshot* snapshot = nullptr; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:WriteManifest", [&](void* /* arg */) { + // prevent background compaction job to call this callback + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + snapshot = db_->GetSnapshot(); + ReadOptions read_opts; + read_opts.snapshot = snapshot; + std::string value; + ASSERT_OK(db_->Get(read_opts, "k", &value)); + ASSERT_EQ(kPutVal, value); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(db_->IngestExternalFile(db_->DefaultColumnFamily(), {external_file}, + IngestExternalFileOptions())); + auto ingested_file_seqno = db_->GetLatestSequenceNumber(); + ASSERT_NE(nullptr, snapshot); + // snapshot is taken before SST ingestion is done + ASSERT_EQ(ingested_file_seqno, snapshot->GetSequenceNumber() + 1); + + ReadOptions read_opts; + read_opts.snapshot = snapshot; + std::string value; + ASSERT_OK(db_->Get(read_opts, "k", &value)); + ASSERT_EQ(kPutVal, value); + db_->ReleaseSnapshot(snapshot); + + // After reopen, sequence number should be up current such that + // ingested value is read + Reopen(CurrentOptions()); + ASSERT_OK(db_->Get(ReadOptions(), "k", &value)); + ASSERT_EQ(kIngestedVal, value); + + // New write should get higher seqno compared to ingested file + ASSERT_OK(Put("k", kPutVal, WriteOptions())); + ASSERT_EQ(db_->GetLatestSequenceNumber(), ingested_file_seqno + 1); +} + +INSTANTIATE_TEST_CASE_P(ExternalSSTFileBasicTest, ExternalSSTFileBasicTest, + testing::Values(std::make_tuple(true, true), + std::make_tuple(true, false), + std::make_tuple(false, true), + std::make_tuple(false, false))); + + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/external_sst_file_ingestion_job.cc b/librocksdb-sys/rocksdb/db/external_sst_file_ingestion_job.cc new file mode 100644 index 0000000..c665ed8 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/external_sst_file_ingestion_job.cc @@ -0,0 +1,1109 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + + +#include "db/external_sst_file_ingestion_job.h" + +#include +#include +#include +#include +#include + +#include "db/db_impl/db_impl.h" +#include "db/version_edit.h" +#include "file/file_util.h" +#include "file/random_access_file_reader.h" +#include "logging/logging.h" +#include "table/merging_iterator.h" +#include "table/scoped_arena_iterator.h" +#include "table/sst_file_writer_collectors.h" +#include "table/table_builder.h" +#include "table/unique_id_impl.h" +#include "test_util/sync_point.h" +#include "util/stop_watch.h" + +namespace ROCKSDB_NAMESPACE { + +Status ExternalSstFileIngestionJob::Prepare( + const std::vector& external_files_paths, + const std::vector& files_checksums, + const std::vector& files_checksum_func_names, + const Temperature& file_temperature, uint64_t next_file_number, + SuperVersion* sv) { + Status status; + + // Read the information of files we are ingesting + for (const std::string& file_path : external_files_paths) { + IngestedFileInfo file_to_ingest; + status = + GetIngestedFileInfo(file_path, next_file_number++, &file_to_ingest, sv); + if (!status.ok()) { + return status; + } + + if (file_to_ingest.cf_id != + TablePropertiesCollectorFactory::Context::kUnknownColumnFamily && + file_to_ingest.cf_id != cfd_->GetID()) { + return Status::InvalidArgument( + "External file column family id don't match"); + } + + if (file_to_ingest.num_entries == 0 && + file_to_ingest.num_range_deletions == 0) { + return Status::InvalidArgument("File contain no entries"); + } + + if (!file_to_ingest.smallest_internal_key.Valid() || + !file_to_ingest.largest_internal_key.Valid()) { + return Status::Corruption("Generated table have corrupted keys"); + } + + files_to_ingest_.emplace_back(std::move(file_to_ingest)); + } + + const Comparator* ucmp = cfd_->internal_comparator().user_comparator(); + auto num_files = files_to_ingest_.size(); + if (num_files == 0) { + return Status::InvalidArgument("The list of files is empty"); + } else if (num_files > 1) { + // Verify that passed files don't have overlapping ranges + autovector sorted_files; + for (size_t i = 0; i < num_files; i++) { + sorted_files.push_back(&files_to_ingest_[i]); + } + + std::sort( + sorted_files.begin(), sorted_files.end(), + [&ucmp](const IngestedFileInfo* info1, const IngestedFileInfo* info2) { + return sstableKeyCompare(ucmp, info1->smallest_internal_key, + info2->smallest_internal_key) < 0; + }); + + for (size_t i = 0; i + 1 < num_files; i++) { + if (sstableKeyCompare(ucmp, sorted_files[i]->largest_internal_key, + sorted_files[i + 1]->smallest_internal_key) >= 0) { + files_overlap_ = true; + break; + } + } + } + + // Hanlde the file temperature + for (size_t i = 0; i < num_files; i++) { + files_to_ingest_[i].file_temperature = file_temperature; + } + + if (ingestion_options_.ingest_behind && files_overlap_) { + return Status::NotSupported("Files have overlapping ranges"); + } + + // Copy/Move external files into DB + std::unordered_set ingestion_path_ids; + for (IngestedFileInfo& f : files_to_ingest_) { + f.copy_file = false; + const std::string path_outside_db = f.external_file_path; + const std::string path_inside_db = TableFileName( + cfd_->ioptions()->cf_paths, f.fd.GetNumber(), f.fd.GetPathId()); + if (ingestion_options_.move_files) { + status = + fs_->LinkFile(path_outside_db, path_inside_db, IOOptions(), nullptr); + if (status.ok()) { + // It is unsafe to assume application had sync the file and file + // directory before ingest the file. For integrity of RocksDB we need + // to sync the file. + std::unique_ptr file_to_sync; + Status s = fs_->ReopenWritableFile(path_inside_db, env_options_, + &file_to_sync, nullptr); + TEST_SYNC_POINT_CALLBACK("ExternalSstFileIngestionJob::Prepare:Reopen", + &s); + // Some file systems (especially remote/distributed) don't support + // reopening a file for writing and don't require reopening and + // syncing the file. Ignore the NotSupported error in that case. + if (!s.IsNotSupported()) { + status = s; + if (status.ok()) { + TEST_SYNC_POINT( + "ExternalSstFileIngestionJob::BeforeSyncIngestedFile"); + status = SyncIngestedFile(file_to_sync.get()); + TEST_SYNC_POINT( + "ExternalSstFileIngestionJob::AfterSyncIngestedFile"); + if (!status.ok()) { + ROCKS_LOG_WARN(db_options_.info_log, + "Failed to sync ingested file %s: %s", + path_inside_db.c_str(), status.ToString().c_str()); + } + } + } + } else if (status.IsNotSupported() && + ingestion_options_.failed_move_fall_back_to_copy) { + // Original file is on a different FS, use copy instead of hard linking. + f.copy_file = true; + ROCKS_LOG_INFO(db_options_.info_log, + "Triy to link file %s but it's not supported : %s", + path_outside_db.c_str(), status.ToString().c_str()); + } + } else { + f.copy_file = true; + } + + if (f.copy_file) { + TEST_SYNC_POINT_CALLBACK("ExternalSstFileIngestionJob::Prepare:CopyFile", + nullptr); + // CopyFile also sync the new file. + status = + CopyFile(fs_.get(), path_outside_db, path_inside_db, 0, + db_options_.use_fsync, io_tracer_, Temperature::kUnknown); + } + TEST_SYNC_POINT("ExternalSstFileIngestionJob::Prepare:FileAdded"); + if (!status.ok()) { + break; + } + f.internal_file_path = path_inside_db; + // Initialize the checksum information of ingested files. + f.file_checksum = kUnknownFileChecksum; + f.file_checksum_func_name = kUnknownFileChecksumFuncName; + ingestion_path_ids.insert(f.fd.GetPathId()); + } + + TEST_SYNC_POINT("ExternalSstFileIngestionJob::BeforeSyncDir"); + if (status.ok()) { + for (auto path_id : ingestion_path_ids) { + status = directories_->GetDataDir(path_id)->FsyncWithDirOptions( + IOOptions(), nullptr, + DirFsyncOptions(DirFsyncOptions::FsyncReason::kNewFileSynced)); + if (!status.ok()) { + ROCKS_LOG_WARN(db_options_.info_log, + "Failed to sync directory %" ROCKSDB_PRIszt + " while ingest file: %s", + path_id, status.ToString().c_str()); + break; + } + } + } + TEST_SYNC_POINT("ExternalSstFileIngestionJob::AfterSyncDir"); + + // Generate and check the sst file checksum. Note that, if + // IngestExternalFileOptions::write_global_seqno is true, we will not update + // the checksum information in the files_to_ingests_ here, since the file is + // upadted with the new global_seqno. After global_seqno is updated, DB will + // generate the new checksum and store it in the Manifest. In all other cases + // if ingestion_options_.write_global_seqno == true and + // verify_file_checksum is false, we only check the checksum function name. + if (status.ok() && db_options_.file_checksum_gen_factory != nullptr) { + if (ingestion_options_.verify_file_checksum == false && + files_checksums.size() == files_to_ingest_.size() && + files_checksum_func_names.size() == files_to_ingest_.size()) { + // Only when verify_file_checksum == false and the checksum for ingested + // files are provided, DB will use the provided checksum and does not + // generate the checksum for ingested files. + need_generate_file_checksum_ = false; + } else { + need_generate_file_checksum_ = true; + } + FileChecksumGenContext gen_context; + std::unique_ptr file_checksum_gen = + db_options_.file_checksum_gen_factory->CreateFileChecksumGenerator( + gen_context); + std::vector generated_checksums; + std::vector generated_checksum_func_names; + // Step 1: generate the checksum for ingested sst file. + if (need_generate_file_checksum_) { + for (size_t i = 0; i < files_to_ingest_.size(); i++) { + std::string generated_checksum; + std::string generated_checksum_func_name; + std::string requested_checksum_func_name; + // TODO: rate limit file reads for checksum calculation during file + // ingestion. + IOStatus io_s = GenerateOneFileChecksum( + fs_.get(), files_to_ingest_[i].internal_file_path, + db_options_.file_checksum_gen_factory.get(), + requested_checksum_func_name, &generated_checksum, + &generated_checksum_func_name, + ingestion_options_.verify_checksums_readahead_size, + db_options_.allow_mmap_reads, io_tracer_, + db_options_.rate_limiter.get(), + Env::IO_TOTAL /* rate_limiter_priority */); + if (!io_s.ok()) { + status = io_s; + ROCKS_LOG_WARN(db_options_.info_log, + "Sst file checksum generation of file: %s failed: %s", + files_to_ingest_[i].internal_file_path.c_str(), + status.ToString().c_str()); + break; + } + if (ingestion_options_.write_global_seqno == false) { + files_to_ingest_[i].file_checksum = generated_checksum; + files_to_ingest_[i].file_checksum_func_name = + generated_checksum_func_name; + } + generated_checksums.push_back(generated_checksum); + generated_checksum_func_names.push_back(generated_checksum_func_name); + } + } + + // Step 2: based on the verify_file_checksum and ingested checksum + // information, do the verification. + if (status.ok()) { + if (files_checksums.size() == files_to_ingest_.size() && + files_checksum_func_names.size() == files_to_ingest_.size()) { + // Verify the checksum and checksum function name. + if (ingestion_options_.verify_file_checksum) { + for (size_t i = 0; i < files_to_ingest_.size(); i++) { + if (files_checksum_func_names[i] != + generated_checksum_func_names[i]) { + status = Status::InvalidArgument( + "Checksum function name does not match with the checksum " + "function name of this DB"); + ROCKS_LOG_WARN( + db_options_.info_log, + "Sst file checksum verification of file: %s failed: %s", + external_files_paths[i].c_str(), status.ToString().c_str()); + break; + } + if (files_checksums[i] != generated_checksums[i]) { + status = Status::Corruption( + "Ingested checksum does not match with the generated " + "checksum"); + ROCKS_LOG_WARN( + db_options_.info_log, + "Sst file checksum verification of file: %s failed: %s", + files_to_ingest_[i].internal_file_path.c_str(), + status.ToString().c_str()); + break; + } + } + } else { + // If verify_file_checksum is not enabled, we only verify the + // checksum function name. If it does not match, fail the ingestion. + // If matches, we trust the ingested checksum information and store + // in the Manifest. + for (size_t i = 0; i < files_to_ingest_.size(); i++) { + if (files_checksum_func_names[i] != file_checksum_gen->Name()) { + status = Status::InvalidArgument( + "Checksum function name does not match with the checksum " + "function name of this DB"); + ROCKS_LOG_WARN( + db_options_.info_log, + "Sst file checksum verification of file: %s failed: %s", + external_files_paths[i].c_str(), status.ToString().c_str()); + break; + } + files_to_ingest_[i].file_checksum = files_checksums[i]; + files_to_ingest_[i].file_checksum_func_name = + files_checksum_func_names[i]; + } + } + } else if (files_checksums.size() != files_checksum_func_names.size() || + (files_checksums.size() == files_checksum_func_names.size() && + files_checksums.size() != 0)) { + // The checksum or checksum function name vector are not both empty + // and they are incomplete. + status = Status::InvalidArgument( + "The checksum information of ingested sst files are nonempty and " + "the size of checksums or the size of the checksum function " + "names " + "does not match with the number of ingested sst files"); + ROCKS_LOG_WARN( + db_options_.info_log, + "The ingested sst files checksum information is incomplete: %s", + status.ToString().c_str()); + } + } + } + + // TODO: The following is duplicated with Cleanup(). + if (!status.ok()) { + IOOptions io_opts; + // We failed, remove all files that we copied into the db + for (IngestedFileInfo& f : files_to_ingest_) { + if (f.internal_file_path.empty()) { + continue; + } + Status s = fs_->DeleteFile(f.internal_file_path, io_opts, nullptr); + if (!s.ok()) { + ROCKS_LOG_WARN(db_options_.info_log, + "AddFile() clean up for file %s failed : %s", + f.internal_file_path.c_str(), s.ToString().c_str()); + } + } + } + + return status; +} + +Status ExternalSstFileIngestionJob::NeedsFlush(bool* flush_needed, + SuperVersion* super_version) { + autovector ranges; + autovector keys; + size_t ts_sz = cfd_->user_comparator()->timestamp_size(); + if (ts_sz) { + // Check all ranges [begin, end] inclusively. Add maximum + // timestamp to include all `begin` keys, and add minimal timestamp to + // include all `end` keys. + for (const IngestedFileInfo& file_to_ingest : files_to_ingest_) { + std::string begin_str; + std::string end_str; + AppendUserKeyWithMaxTimestamp( + &begin_str, file_to_ingest.smallest_internal_key.user_key(), ts_sz); + AppendKeyWithMinTimestamp( + &end_str, file_to_ingest.largest_internal_key.user_key(), ts_sz); + keys.emplace_back(std::move(begin_str)); + keys.emplace_back(std::move(end_str)); + } + for (size_t i = 0; i < files_to_ingest_.size(); ++i) { + ranges.emplace_back(keys[2 * i], keys[2 * i + 1]); + } + } else { + for (const IngestedFileInfo& file_to_ingest : files_to_ingest_) { + ranges.emplace_back(file_to_ingest.smallest_internal_key.user_key(), + file_to_ingest.largest_internal_key.user_key()); + } + } + Status status = cfd_->RangesOverlapWithMemtables( + ranges, super_version, db_options_.allow_data_in_errors, flush_needed); + if (status.ok() && *flush_needed && + !ingestion_options_.allow_blocking_flush) { + status = Status::InvalidArgument("External file requires flush"); + } + return status; +} + +// REQUIRES: we have become the only writer by entering both write_thread_ and +// nonmem_write_thread_ +Status ExternalSstFileIngestionJob::Run() { + Status status; + SuperVersion* super_version = cfd_->GetSuperVersion(); +#ifndef NDEBUG + // We should never run the job with a memtable that is overlapping + // with the files we are ingesting + bool need_flush = false; + status = NeedsFlush(&need_flush, super_version); + if (!status.ok()) { + return status; + } + if (need_flush) { + return Status::TryAgain(); + } + assert(status.ok() && need_flush == false); +#endif + + bool force_global_seqno = false; + + if (ingestion_options_.snapshot_consistency && !db_snapshots_->empty()) { + // We need to assign a global sequence number to all the files even + // if the don't overlap with any ranges since we have snapshots + force_global_seqno = true; + } + // It is safe to use this instead of LastAllocatedSequence since we are + // the only active writer, and hence they are equal + SequenceNumber last_seqno = versions_->LastSequence(); + edit_.SetColumnFamily(cfd_->GetID()); + // The levels that the files will be ingested into + + for (IngestedFileInfo& f : files_to_ingest_) { + SequenceNumber assigned_seqno = 0; + if (ingestion_options_.ingest_behind) { + status = CheckLevelForIngestedBehindFile(&f); + } else { + status = AssignLevelAndSeqnoForIngestedFile( + super_version, force_global_seqno, cfd_->ioptions()->compaction_style, + last_seqno, &f, &assigned_seqno); + } + + // Modify the smallest/largest internal key to include the sequence number + // that we just learned. Only overwrite sequence number zero. There could + // be a nonzero sequence number already to indicate a range tombstone's + // exclusive endpoint. + ParsedInternalKey smallest_parsed, largest_parsed; + if (status.ok()) { + status = ParseInternalKey(*f.smallest_internal_key.rep(), + &smallest_parsed, false /* log_err_key */); + } + if (status.ok()) { + status = ParseInternalKey(*f.largest_internal_key.rep(), &largest_parsed, + false /* log_err_key */); + } + if (!status.ok()) { + return status; + } + if (smallest_parsed.sequence == 0) { + UpdateInternalKey(f.smallest_internal_key.rep(), assigned_seqno, + smallest_parsed.type); + } + if (largest_parsed.sequence == 0) { + UpdateInternalKey(f.largest_internal_key.rep(), assigned_seqno, + largest_parsed.type); + } + + status = AssignGlobalSeqnoForIngestedFile(&f, assigned_seqno); + TEST_SYNC_POINT_CALLBACK("ExternalSstFileIngestionJob::Run", + &assigned_seqno); + if (assigned_seqno > last_seqno) { + assert(assigned_seqno == last_seqno + 1); + last_seqno = assigned_seqno; + ++consumed_seqno_count_; + } + if (!status.ok()) { + return status; + } + + status = GenerateChecksumForIngestedFile(&f); + if (!status.ok()) { + return status; + } + + // We use the import time as the ancester time. This is the time the data + // is written to the database. + int64_t temp_current_time = 0; + uint64_t current_time = kUnknownFileCreationTime; + uint64_t oldest_ancester_time = kUnknownOldestAncesterTime; + if (clock_->GetCurrentTime(&temp_current_time).ok()) { + current_time = oldest_ancester_time = + static_cast(temp_current_time); + } + uint64_t tail_size = 0; + bool contain_no_data_blocks = f.table_properties.num_entries > 0 && + (f.table_properties.num_entries == + f.table_properties.num_range_deletions); + if (f.table_properties.tail_start_offset > 0 || contain_no_data_blocks) { + uint64_t file_size = f.fd.GetFileSize(); + assert(f.table_properties.tail_start_offset <= file_size); + tail_size = file_size - f.table_properties.tail_start_offset; + } + + FileMetaData f_metadata( + f.fd.GetNumber(), f.fd.GetPathId(), f.fd.GetFileSize(), + f.smallest_internal_key, f.largest_internal_key, f.assigned_seqno, + f.assigned_seqno, false, f.file_temperature, kInvalidBlobFileNumber, + oldest_ancester_time, current_time, + ingestion_options_.ingest_behind + ? kReservedEpochNumberForFileIngestedBehind + : cfd_->NewEpochNumber(), + f.file_checksum, f.file_checksum_func_name, f.unique_id, 0, tail_size, + static_cast( + f.table_properties.user_defined_timestamps_persisted)); + f_metadata.temperature = f.file_temperature; + edit_.AddFile(f.picked_level, f_metadata); + } + + CreateEquivalentFileIngestingCompactions(); + return status; +} + +void ExternalSstFileIngestionJob::CreateEquivalentFileIngestingCompactions() { + // A map from output level to input of compactions equivalent to this + // ingestion job. + // TODO: simplify below logic to creating compaction per ingested file + // instead of per output level, once we figure out how to treat ingested files + // with adjacent range deletion tombstones to same output level in the same + // job as non-overlapping compactions. + std::map + output_level_to_file_ingesting_compaction_input; + + for (const auto& pair : edit_.GetNewFiles()) { + int output_level = pair.first; + const FileMetaData& f_metadata = pair.second; + + CompactionInputFiles& input = + output_level_to_file_ingesting_compaction_input[output_level]; + if (input.files.empty()) { + // Treat the source level of ingested files to be level 0 + input.level = 0; + } + + compaction_input_metdatas_.push_back(new FileMetaData(f_metadata)); + input.files.push_back(compaction_input_metdatas_.back()); + } + + for (const auto& pair : output_level_to_file_ingesting_compaction_input) { + int output_level = pair.first; + const CompactionInputFiles& input = pair.second; + + const auto& mutable_cf_options = *(cfd_->GetLatestMutableCFOptions()); + file_ingesting_compactions_.push_back(new Compaction( + cfd_->current()->storage_info(), *cfd_->ioptions(), mutable_cf_options, + mutable_db_options_, {input}, output_level, + MaxFileSizeForLevel( + mutable_cf_options, output_level, + cfd_->ioptions()->compaction_style) /* output file size + limit, + * not applicable + */ + , + LLONG_MAX /* max compaction bytes, not applicable */, + 0 /* output path ID, not applicable */, mutable_cf_options.compression, + mutable_cf_options.compression_opts, Temperature::kUnknown, + 0 /* max_subcompaction, not applicable */, + {} /* grandparents, not applicable */, false /* is manual */, + "" /* trim_ts */, -1 /* score, not applicable */, + false /* is deletion compaction, not applicable */, + files_overlap_ /* l0_files_might_overlap, not applicable */, + CompactionReason::kExternalSstIngestion)); + } +} + +void ExternalSstFileIngestionJob::RegisterRange() { + for (const auto& c : file_ingesting_compactions_) { + cfd_->compaction_picker()->RegisterCompaction(c); + } +} + +void ExternalSstFileIngestionJob::UnregisterRange() { + for (const auto& c : file_ingesting_compactions_) { + cfd_->compaction_picker()->UnregisterCompaction(c); + delete c; + } + file_ingesting_compactions_.clear(); + + for (const auto& f : compaction_input_metdatas_) { + delete f; + } + compaction_input_metdatas_.clear(); +} + +void ExternalSstFileIngestionJob::UpdateStats() { + // Update internal stats for new ingested files + uint64_t total_keys = 0; + uint64_t total_l0_files = 0; + uint64_t total_time = clock_->NowMicros() - job_start_time_; + + EventLoggerStream stream = event_logger_->Log(); + stream << "event" + << "ingest_finished"; + stream << "files_ingested"; + stream.StartArray(); + + for (IngestedFileInfo& f : files_to_ingest_) { + InternalStats::CompactionStats stats( + CompactionReason::kExternalSstIngestion, 1); + stats.micros = total_time; + // If actual copy occurred for this file, then we need to count the file + // size as the actual bytes written. If the file was linked, then we ignore + // the bytes written for file metadata. + // TODO (yanqin) maybe account for file metadata bytes for exact accuracy? + if (f.copy_file) { + stats.bytes_written = f.fd.GetFileSize(); + } else { + stats.bytes_moved = f.fd.GetFileSize(); + } + stats.num_output_files = 1; + cfd_->internal_stats()->AddCompactionStats(f.picked_level, + Env::Priority::USER, stats); + cfd_->internal_stats()->AddCFStats(InternalStats::BYTES_INGESTED_ADD_FILE, + f.fd.GetFileSize()); + total_keys += f.num_entries; + if (f.picked_level == 0) { + total_l0_files += 1; + } + ROCKS_LOG_INFO( + db_options_.info_log, + "[AddFile] External SST file %s was ingested in L%d with path %s " + "(global_seqno=%" PRIu64 ")\n", + f.external_file_path.c_str(), f.picked_level, + f.internal_file_path.c_str(), f.assigned_seqno); + stream << "file" << f.internal_file_path << "level" << f.picked_level; + } + stream.EndArray(); + + stream << "lsm_state"; + stream.StartArray(); + auto vstorage = cfd_->current()->storage_info(); + for (int level = 0; level < vstorage->num_levels(); ++level) { + stream << vstorage->NumLevelFiles(level); + } + stream.EndArray(); + + cfd_->internal_stats()->AddCFStats(InternalStats::INGESTED_NUM_KEYS_TOTAL, + total_keys); + cfd_->internal_stats()->AddCFStats(InternalStats::INGESTED_NUM_FILES_TOTAL, + files_to_ingest_.size()); + cfd_->internal_stats()->AddCFStats( + InternalStats::INGESTED_LEVEL0_NUM_FILES_TOTAL, total_l0_files); +} + +void ExternalSstFileIngestionJob::Cleanup(const Status& status) { + IOOptions io_opts; + if (!status.ok()) { + // We failed to add the files to the database + // remove all the files we copied + for (IngestedFileInfo& f : files_to_ingest_) { + if (f.internal_file_path.empty()) { + continue; + } + Status s = fs_->DeleteFile(f.internal_file_path, io_opts, nullptr); + if (!s.ok()) { + ROCKS_LOG_WARN(db_options_.info_log, + "AddFile() clean up for file %s failed : %s", + f.internal_file_path.c_str(), s.ToString().c_str()); + } + } + consumed_seqno_count_ = 0; + files_overlap_ = false; + } else if (status.ok() && ingestion_options_.move_files) { + // The files were moved and added successfully, remove original file links + for (IngestedFileInfo& f : files_to_ingest_) { + Status s = fs_->DeleteFile(f.external_file_path, io_opts, nullptr); + if (!s.ok()) { + ROCKS_LOG_WARN( + db_options_.info_log, + "%s was added to DB successfully but failed to remove original " + "file link : %s", + f.external_file_path.c_str(), s.ToString().c_str()); + } + } + } +} + +Status ExternalSstFileIngestionJob::GetIngestedFileInfo( + const std::string& external_file, uint64_t new_file_number, + IngestedFileInfo* file_to_ingest, SuperVersion* sv) { + file_to_ingest->external_file_path = external_file; + + // Get external file size + Status status = fs_->GetFileSize(external_file, IOOptions(), + &file_to_ingest->file_size, nullptr); + if (!status.ok()) { + return status; + } + + // Assign FD with number + file_to_ingest->fd = + FileDescriptor(new_file_number, 0, file_to_ingest->file_size); + + // Create TableReader for external file + std::unique_ptr table_reader; + std::unique_ptr sst_file; + std::unique_ptr sst_file_reader; + + status = + fs_->NewRandomAccessFile(external_file, env_options_, &sst_file, nullptr); + if (!status.ok()) { + return status; + } + sst_file_reader.reset(new RandomAccessFileReader( + std::move(sst_file), external_file, nullptr /*Env*/, io_tracer_)); + + // TODO(yuzhangyu): User-defined timestamps doesn't support external sst file + // ingestion. Pass in the correct `user_defined_timestamps_persisted` flag + // for creating `TableReaderOptions` when the support is there. + status = cfd_->ioptions()->table_factory->NewTableReader( + TableReaderOptions( + *cfd_->ioptions(), sv->mutable_cf_options.prefix_extractor, + env_options_, cfd_->internal_comparator(), + sv->mutable_cf_options.block_protection_bytes_per_key, + /*skip_filters*/ false, /*immortal*/ false, + /*force_direct_prefetch*/ false, /*level*/ -1, + /*block_cache_tracer*/ nullptr, + /*max_file_size_for_l0_meta_pin*/ 0, versions_->DbSessionId(), + /*cur_file_num*/ new_file_number), + std::move(sst_file_reader), file_to_ingest->file_size, &table_reader); + if (!status.ok()) { + return status; + } + + if (ingestion_options_.verify_checksums_before_ingest) { + // If customized readahead size is needed, we can pass a user option + // all the way to here. Right now we just rely on the default readahead + // to keep things simple. + // TODO: plumb Env::IOActivity + ReadOptions ro; + ro.readahead_size = ingestion_options_.verify_checksums_readahead_size; + status = table_reader->VerifyChecksum( + ro, TableReaderCaller::kExternalSSTIngestion); + if (!status.ok()) { + return status; + } + } + + // Get the external file properties + auto props = table_reader->GetTableProperties(); + const auto& uprops = props->user_collected_properties; + + // Get table version + auto version_iter = uprops.find(ExternalSstFilePropertyNames::kVersion); + if (version_iter == uprops.end()) { + return Status::Corruption("External file version not found"); + } + file_to_ingest->version = DecodeFixed32(version_iter->second.c_str()); + + auto seqno_iter = uprops.find(ExternalSstFilePropertyNames::kGlobalSeqno); + if (file_to_ingest->version == 2) { + // version 2 imply that we have global sequence number + if (seqno_iter == uprops.end()) { + return Status::Corruption( + "External file global sequence number not found"); + } + + // Set the global sequence number + file_to_ingest->original_seqno = DecodeFixed64(seqno_iter->second.c_str()); + if (props->external_sst_file_global_seqno_offset == 0) { + file_to_ingest->global_seqno_offset = 0; + return Status::Corruption("Was not able to find file global seqno field"); + } + file_to_ingest->global_seqno_offset = + static_cast(props->external_sst_file_global_seqno_offset); + } else if (file_to_ingest->version == 1) { + // SST file V1 should not have global seqno field + assert(seqno_iter == uprops.end()); + file_to_ingest->original_seqno = 0; + if (ingestion_options_.allow_blocking_flush || + ingestion_options_.allow_global_seqno) { + return Status::InvalidArgument( + "External SST file V1 does not support global seqno"); + } + } else { + return Status::InvalidArgument("External file version is not supported"); + } + // Get number of entries in table + file_to_ingest->num_entries = props->num_entries; + file_to_ingest->num_range_deletions = props->num_range_deletions; + + ParsedInternalKey key; + // TODO: plumb Env::IOActivity + ReadOptions ro; + std::unique_ptr iter(table_reader->NewIterator( + ro, sv->mutable_cf_options.prefix_extractor.get(), /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kExternalSSTIngestion)); + std::unique_ptr range_del_iter( + table_reader->NewRangeTombstoneIterator(ro)); + + // Get first (smallest) and last (largest) key from file. + file_to_ingest->smallest_internal_key = + InternalKey("", 0, ValueType::kTypeValue); + file_to_ingest->largest_internal_key = + InternalKey("", 0, ValueType::kTypeValue); + bool bounds_set = false; + bool allow_data_in_errors = db_options_.allow_data_in_errors; + iter->SeekToFirst(); + if (iter->Valid()) { + Status pik_status = + ParseInternalKey(iter->key(), &key, allow_data_in_errors); + if (!pik_status.ok()) { + return Status::Corruption("Corrupted key in external file. ", + pik_status.getState()); + } + if (key.sequence != 0) { + return Status::Corruption("External file has non zero sequence number"); + } + file_to_ingest->smallest_internal_key.SetFrom(key); + + iter->SeekToLast(); + pik_status = ParseInternalKey(iter->key(), &key, allow_data_in_errors); + if (!pik_status.ok()) { + return Status::Corruption("Corrupted key in external file. ", + pik_status.getState()); + } + if (key.sequence != 0) { + return Status::Corruption("External file has non zero sequence number"); + } + file_to_ingest->largest_internal_key.SetFrom(key); + + bounds_set = true; + } + + // We may need to adjust these key bounds, depending on whether any range + // deletion tombstones extend past them. + const Comparator* ucmp = cfd_->internal_comparator().user_comparator(); + if (range_del_iter != nullptr) { + for (range_del_iter->SeekToFirst(); range_del_iter->Valid(); + range_del_iter->Next()) { + Status pik_status = + ParseInternalKey(range_del_iter->key(), &key, allow_data_in_errors); + if (!pik_status.ok()) { + return Status::Corruption("Corrupted key in external file. ", + pik_status.getState()); + } + RangeTombstone tombstone(key, range_del_iter->value()); + + InternalKey start_key = tombstone.SerializeKey(); + if (!bounds_set || + sstableKeyCompare(ucmp, start_key, + file_to_ingest->smallest_internal_key) < 0) { + file_to_ingest->smallest_internal_key = start_key; + } + InternalKey end_key = tombstone.SerializeEndKey(); + if (!bounds_set || + sstableKeyCompare(ucmp, end_key, + file_to_ingest->largest_internal_key) > 0) { + file_to_ingest->largest_internal_key = end_key; + } + bounds_set = true; + } + } + + file_to_ingest->cf_id = static_cast(props->column_family_id); + + file_to_ingest->table_properties = *props; + + auto s = GetSstInternalUniqueId(props->db_id, props->db_session_id, + props->orig_file_number, + &(file_to_ingest->unique_id)); + if (!s.ok()) { + ROCKS_LOG_WARN(db_options_.info_log, + "Failed to get SST unique id for file %s", + file_to_ingest->internal_file_path.c_str()); + file_to_ingest->unique_id = kNullUniqueId64x2; + } + + return status; +} + +Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile( + SuperVersion* sv, bool force_global_seqno, CompactionStyle compaction_style, + SequenceNumber last_seqno, IngestedFileInfo* file_to_ingest, + SequenceNumber* assigned_seqno) { + Status status; + *assigned_seqno = 0; + if (force_global_seqno) { + *assigned_seqno = last_seqno + 1; + if (compaction_style == kCompactionStyleUniversal || files_overlap_) { + if (ingestion_options_.fail_if_not_bottommost_level) { + status = Status::TryAgain( + "Files cannot be ingested to Lmax. Please make sure key range of " + "Lmax does not overlap with files to ingest."); + return status; + } + file_to_ingest->picked_level = 0; + return status; + } + } + + bool overlap_with_db = false; + Arena arena; + // TODO: plumb Env::IOActivity + ReadOptions ro; + ro.total_order_seek = true; + int target_level = 0; + auto* vstorage = cfd_->current()->storage_info(); + + for (int lvl = 0; lvl < cfd_->NumberLevels(); lvl++) { + if (lvl > 0 && lvl < vstorage->base_level()) { + continue; + } + if (cfd_->RangeOverlapWithCompaction( + file_to_ingest->smallest_internal_key.user_key(), + file_to_ingest->largest_internal_key.user_key(), lvl)) { + // We must use L0 or any level higher than `lvl` to be able to overwrite + // the compaction output keys that we overlap with in this level, We also + // need to assign this file a seqno to overwrite the compaction output + // keys in level `lvl` + overlap_with_db = true; + break; + } else if (vstorage->NumLevelFiles(lvl) > 0) { + bool overlap_with_level = false; + status = sv->current->OverlapWithLevelIterator( + ro, env_options_, file_to_ingest->smallest_internal_key.user_key(), + file_to_ingest->largest_internal_key.user_key(), lvl, + &overlap_with_level); + if (!status.ok()) { + return status; + } + if (overlap_with_level) { + // We must use L0 or any level higher than `lvl` to be able to overwrite + // the keys that we overlap with in this level, We also need to assign + // this file a seqno to overwrite the existing keys in level `lvl` + overlap_with_db = true; + break; + } + + if (compaction_style == kCompactionStyleUniversal && lvl != 0) { + const std::vector& level_files = + vstorage->LevelFiles(lvl); + const SequenceNumber level_largest_seqno = + (*std::max_element(level_files.begin(), level_files.end(), + [](FileMetaData* f1, FileMetaData* f2) { + return f1->fd.largest_seqno < + f2->fd.largest_seqno; + })) + ->fd.largest_seqno; + // should only assign seqno to current level's largest seqno when + // the file fits + if (level_largest_seqno != 0 && + IngestedFileFitInLevel(file_to_ingest, lvl)) { + *assigned_seqno = level_largest_seqno; + } else { + continue; + } + } + } else if (compaction_style == kCompactionStyleUniversal) { + continue; + } + + // We don't overlap with any keys in this level, but we still need to check + // if our file can fit in it + if (IngestedFileFitInLevel(file_to_ingest, lvl)) { + target_level = lvl; + } + } + // If files overlap, we have to ingest them at level 0 and assign the newest + // sequence number + if (files_overlap_) { + target_level = 0; + *assigned_seqno = last_seqno + 1; + } + + if (ingestion_options_.fail_if_not_bottommost_level && + target_level < cfd_->NumberLevels() - 1) { + status = Status::TryAgain( + "Files cannot be ingested to Lmax. Please make sure key range of Lmax " + "and ongoing compaction's output to Lmax" + "does not overlap with files to ingest."); + return status; + } + + TEST_SYNC_POINT_CALLBACK( + "ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile", + &overlap_with_db); + file_to_ingest->picked_level = target_level; + if (overlap_with_db && *assigned_seqno == 0) { + *assigned_seqno = last_seqno + 1; + } + return status; +} + +Status ExternalSstFileIngestionJob::CheckLevelForIngestedBehindFile( + IngestedFileInfo* file_to_ingest) { + auto* vstorage = cfd_->current()->storage_info(); + // First, check if new files fit in the bottommost level + int bottom_lvl = cfd_->NumberLevels() - 1; + if (!IngestedFileFitInLevel(file_to_ingest, bottom_lvl)) { + return Status::InvalidArgument( + "Can't ingest_behind file as it doesn't fit " + "at the bottommost level!"); + } + + // Second, check if despite allow_ingest_behind=true we still have 0 seqnums + // at some upper level + for (int lvl = 0; lvl < cfd_->NumberLevels() - 1; lvl++) { + for (auto file : vstorage->LevelFiles(lvl)) { + if (file->fd.smallest_seqno == 0) { + return Status::InvalidArgument( + "Can't ingest_behind file as despite allow_ingest_behind=true " + "there are files with 0 seqno in database at upper levels!"); + } + } + } + + file_to_ingest->picked_level = bottom_lvl; + return Status::OK(); +} + +Status ExternalSstFileIngestionJob::AssignGlobalSeqnoForIngestedFile( + IngestedFileInfo* file_to_ingest, SequenceNumber seqno) { + if (file_to_ingest->original_seqno == seqno) { + // This file already have the correct global seqno + return Status::OK(); + } else if (!ingestion_options_.allow_global_seqno) { + return Status::InvalidArgument("Global seqno is required, but disabled"); + } else if (file_to_ingest->global_seqno_offset == 0) { + return Status::InvalidArgument( + "Trying to set global seqno for a file that don't have a global seqno " + "field"); + } + + if (ingestion_options_.write_global_seqno) { + // Determine if we can write global_seqno to a given offset of file. + // If the file system does not support random write, then we should not. + // Otherwise we should. + std::unique_ptr rwfile; + Status status = fs_->NewRandomRWFile(file_to_ingest->internal_file_path, + env_options_, &rwfile, nullptr); + TEST_SYNC_POINT_CALLBACK("ExternalSstFileIngestionJob::NewRandomRWFile", + &status); + if (status.ok()) { + FSRandomRWFilePtr fsptr(std::move(rwfile), io_tracer_, + file_to_ingest->internal_file_path); + std::string seqno_val; + PutFixed64(&seqno_val, seqno); + status = fsptr->Write(file_to_ingest->global_seqno_offset, seqno_val, + IOOptions(), nullptr); + if (status.ok()) { + TEST_SYNC_POINT("ExternalSstFileIngestionJob::BeforeSyncGlobalSeqno"); + status = SyncIngestedFile(fsptr.get()); + TEST_SYNC_POINT("ExternalSstFileIngestionJob::AfterSyncGlobalSeqno"); + if (!status.ok()) { + ROCKS_LOG_WARN(db_options_.info_log, + "Failed to sync ingested file %s after writing global " + "sequence number: %s", + file_to_ingest->internal_file_path.c_str(), + status.ToString().c_str()); + } + } + if (!status.ok()) { + return status; + } + } else if (!status.IsNotSupported()) { + return status; + } + } + + file_to_ingest->assigned_seqno = seqno; + return Status::OK(); +} + +IOStatus ExternalSstFileIngestionJob::GenerateChecksumForIngestedFile( + IngestedFileInfo* file_to_ingest) { + if (db_options_.file_checksum_gen_factory == nullptr || + need_generate_file_checksum_ == false || + ingestion_options_.write_global_seqno == false) { + // If file_checksum_gen_factory is not set, we are not able to generate + // the checksum. if write_global_seqno is false, it means we will use + // file checksum generated during Prepare(). This step will be skipped. + return IOStatus::OK(); + } + std::string file_checksum; + std::string file_checksum_func_name; + std::string requested_checksum_func_name; + // TODO: rate limit file reads for checksum calculation during file ingestion. + IOStatus io_s = GenerateOneFileChecksum( + fs_.get(), file_to_ingest->internal_file_path, + db_options_.file_checksum_gen_factory.get(), requested_checksum_func_name, + &file_checksum, &file_checksum_func_name, + ingestion_options_.verify_checksums_readahead_size, + db_options_.allow_mmap_reads, io_tracer_, db_options_.rate_limiter.get(), + Env::IO_TOTAL /* rate_limiter_priority */); + if (!io_s.ok()) { + return io_s; + } + file_to_ingest->file_checksum = file_checksum; + file_to_ingest->file_checksum_func_name = file_checksum_func_name; + return IOStatus::OK(); +} + +bool ExternalSstFileIngestionJob::IngestedFileFitInLevel( + const IngestedFileInfo* file_to_ingest, int level) { + if (level == 0) { + // Files can always fit in L0 + return true; + } + + auto* vstorage = cfd_->current()->storage_info(); + Slice file_smallest_user_key( + file_to_ingest->smallest_internal_key.user_key()); + Slice file_largest_user_key(file_to_ingest->largest_internal_key.user_key()); + + if (vstorage->OverlapInLevel(level, &file_smallest_user_key, + &file_largest_user_key)) { + // File overlap with another files in this level, we cannot + // add it to this level + return false; + } + + // File did not overlap with level files, nor compaction output + return true; +} + +template +Status ExternalSstFileIngestionJob::SyncIngestedFile(TWritableFile* file) { + assert(file != nullptr); + if (db_options_.use_fsync) { + return file->Fsync(IOOptions(), nullptr); + } else { + return file->Sync(IOOptions(), nullptr); + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/external_sst_file_ingestion_job.h b/librocksdb-sys/rocksdb/db/external_sst_file_ingestion_job.h new file mode 100644 index 0000000..49bb1e3 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/external_sst_file_ingestion_job.h @@ -0,0 +1,238 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once +#include +#include +#include + +#include "db/column_family.h" +#include "db/internal_stats.h" +#include "db/snapshot_impl.h" +#include "db/version_edit.h" +#include "env/file_system_tracer.h" +#include "logging/event_logger.h" +#include "options/db_options.h" +#include "rocksdb/db.h" +#include "rocksdb/file_system.h" +#include "rocksdb/sst_file_writer.h" +#include "util/autovector.h" + +namespace ROCKSDB_NAMESPACE { + +class Directories; +class SystemClock; + +struct IngestedFileInfo { + // External file path + std::string external_file_path; + // Smallest internal key in external file + InternalKey smallest_internal_key; + // Largest internal key in external file + InternalKey largest_internal_key; + // Sequence number for keys in external file + SequenceNumber original_seqno; + // Offset of the global sequence number field in the file, will + // be zero if version is 1 (global seqno is not supported) + size_t global_seqno_offset; + // External file size + uint64_t file_size; + // total number of keys in external file + uint64_t num_entries; + // total number of range deletions in external file + uint64_t num_range_deletions; + // Id of column family this file shoule be ingested into + uint32_t cf_id; + // TableProperties read from external file + TableProperties table_properties; + // Version of external file + int version; + + // FileDescriptor for the file inside the DB + FileDescriptor fd; + // file path that we picked for file inside the DB + std::string internal_file_path; + // Global sequence number that we picked for the file inside the DB + SequenceNumber assigned_seqno = 0; + // Level inside the DB we picked for the external file. + int picked_level = 0; + // Whether to copy or link the external sst file. copy_file will be set to + // false if ingestion_options.move_files is true and underlying FS + // supports link operation. Need to provide a default value to make the + // undefined-behavior sanity check of llvm happy. Since + // ingestion_options.move_files is false by default, thus copy_file is true + // by default. + bool copy_file = true; + // The checksum of ingested file + std::string file_checksum; + // The name of checksum function that generate the checksum + std::string file_checksum_func_name; + // The temperature of the file to be ingested + Temperature file_temperature = Temperature::kUnknown; + // Unique id of the file to be ingested + UniqueId64x2 unique_id{}; +}; + +class ExternalSstFileIngestionJob { + public: + ExternalSstFileIngestionJob( + VersionSet* versions, ColumnFamilyData* cfd, + const ImmutableDBOptions& db_options, + const MutableDBOptions& mutable_db_options, const EnvOptions& env_options, + SnapshotList* db_snapshots, + const IngestExternalFileOptions& ingestion_options, + Directories* directories, EventLogger* event_logger, + const std::shared_ptr& io_tracer) + : clock_(db_options.clock), + fs_(db_options.fs, io_tracer), + versions_(versions), + cfd_(cfd), + db_options_(db_options), + mutable_db_options_(mutable_db_options), + env_options_(env_options), + db_snapshots_(db_snapshots), + ingestion_options_(ingestion_options), + directories_(directories), + event_logger_(event_logger), + job_start_time_(clock_->NowMicros()), + consumed_seqno_count_(0), + io_tracer_(io_tracer) { + assert(directories != nullptr); + } + + ~ExternalSstFileIngestionJob() { + for (const auto& c : file_ingesting_compactions_) { + cfd_->compaction_picker()->UnregisterCompaction(c); + delete c; + } + + for (const auto& f : compaction_input_metdatas_) { + delete f; + } + } + + // Prepare the job by copying external files into the DB. + Status Prepare(const std::vector& external_files_paths, + const std::vector& files_checksums, + const std::vector& files_checksum_func_names, + const Temperature& file_temperature, uint64_t next_file_number, + SuperVersion* sv); + + // Check if we need to flush the memtable before running the ingestion job + // This will be true if the files we are ingesting are overlapping with any + // key range in the memtable. + // + // @param super_version A referenced SuperVersion that will be held for the + // duration of this function. + // + // Thread-safe + Status NeedsFlush(bool* flush_needed, SuperVersion* super_version); + + // Will execute the ingestion job and prepare edit() to be applied. + // REQUIRES: Mutex held + Status Run(); + + // Register key range involved in this ingestion job + // to prevent key range conflict with other ongoing compaction/file ingestion + // REQUIRES: Mutex held + void RegisterRange(); + + // Unregister key range registered for this ingestion job + // REQUIRES: Mutex held + void UnregisterRange(); + + // Update column family stats. + // REQUIRES: Mutex held + void UpdateStats(); + + // Cleanup after successful/failed job + void Cleanup(const Status& status); + + VersionEdit* edit() { return &edit_; } + + const autovector& files_to_ingest() const { + return files_to_ingest_; + } + + // How many sequence numbers did we consume as part of the ingest job? + int ConsumedSequenceNumbersCount() const { return consumed_seqno_count_; } + + private: + // Open the external file and populate `file_to_ingest` with all the + // external information we need to ingest this file. + Status GetIngestedFileInfo(const std::string& external_file, + uint64_t new_file_number, + IngestedFileInfo* file_to_ingest, + SuperVersion* sv); + + // Assign `file_to_ingest` the appropriate sequence number and the lowest + // possible level that it can be ingested to according to compaction_style. + // REQUIRES: Mutex held + Status AssignLevelAndSeqnoForIngestedFile(SuperVersion* sv, + bool force_global_seqno, + CompactionStyle compaction_style, + SequenceNumber last_seqno, + IngestedFileInfo* file_to_ingest, + SequenceNumber* assigned_seqno); + + // File that we want to ingest behind always goes to the lowest level; + // we just check that it fits in the level, that DB allows ingest_behind, + // and that we don't have 0 seqnums at the upper levels. + // REQUIRES: Mutex held + Status CheckLevelForIngestedBehindFile(IngestedFileInfo* file_to_ingest); + + // Set the file global sequence number to `seqno` + Status AssignGlobalSeqnoForIngestedFile(IngestedFileInfo* file_to_ingest, + SequenceNumber seqno); + // Generate the file checksum and store in the IngestedFileInfo + IOStatus GenerateChecksumForIngestedFile(IngestedFileInfo* file_to_ingest); + + // Check if `file_to_ingest` can fit in level `level` + // REQUIRES: Mutex held + bool IngestedFileFitInLevel(const IngestedFileInfo* file_to_ingest, + int level); + + // Helper method to sync given file. + template + Status SyncIngestedFile(TWritableFile* file); + + // Create equivalent `Compaction` objects to this file ingestion job + // , which will be used to check range conflict with other ongoing + // compactions. + void CreateEquivalentFileIngestingCompactions(); + + SystemClock* clock_; + FileSystemPtr fs_; + VersionSet* versions_; + ColumnFamilyData* cfd_; + const ImmutableDBOptions& db_options_; + const MutableDBOptions& mutable_db_options_; + const EnvOptions& env_options_; + SnapshotList* db_snapshots_; + autovector files_to_ingest_; + const IngestExternalFileOptions& ingestion_options_; + Directories* directories_; + EventLogger* event_logger_; + VersionEdit edit_; + uint64_t job_start_time_; + int consumed_seqno_count_; + // Set in ExternalSstFileIngestionJob::Prepare(), if true all files are + // ingested in L0 + bool files_overlap_{false}; + // Set in ExternalSstFileIngestionJob::Prepare(), if true and DB + // file_checksum_gen_factory is set, DB will generate checksum each file. + bool need_generate_file_checksum_{true}; + std::shared_ptr io_tracer_; + + // Below are variables used in (un)registering range for this ingestion job + // + // FileMetaData used in inputs of compactions equivalent to this ingestion + // job + std::vector compaction_input_metdatas_; + // Compactions equivalent to this ingestion job + std::vector file_ingesting_compactions_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/external_sst_file_test.cc b/librocksdb-sys/rocksdb/db/external_sst_file_test.cc new file mode 100644 index 0000000..4249b2f --- /dev/null +++ b/librocksdb-sys/rocksdb/db/external_sst_file_test.cc @@ -0,0 +1,2891 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include +#include + +#include "db/db_test_util.h" +#include "db/dbformat.h" +#include "file/filename.h" +#include "options/options_helper.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/sst_file_reader.h" +#include "rocksdb/sst_file_writer.h" +#include "test_util/testutil.h" +#include "util/random.h" +#include "util/thread_guard.h" +#include "utilities/fault_injection_env.h" + +namespace ROCKSDB_NAMESPACE { + +// A test environment that can be configured to fail the Link operation. +class ExternalSSTTestFS : public FileSystemWrapper { + public: + ExternalSSTTestFS(const std::shared_ptr& t, bool fail_link) + : FileSystemWrapper(t), fail_link_(fail_link) {} + static const char* kClassName() { return "ExternalSSTTestFS"; } + const char* Name() const override { return kClassName(); } + + IOStatus LinkFile(const std::string& s, const std::string& t, + const IOOptions& options, IODebugContext* dbg) override { + if (fail_link_) { + return IOStatus::NotSupported("Link failed"); + } + return target()->LinkFile(s, t, options, dbg); + } + + void set_fail_link(bool fail_link) { fail_link_ = fail_link; } + + private: + bool fail_link_; +}; + +class ExternalSSTFileTestBase : public DBTestBase { + public: + ExternalSSTFileTestBase() + : DBTestBase("external_sst_file_test", /*env_do_fsync=*/true) { + sst_files_dir_ = dbname_ + "/sst_files/"; + DestroyAndRecreateExternalSSTFilesDir(); + } + + void DestroyAndRecreateExternalSSTFilesDir() { + ASSERT_OK(DestroyDir(env_, sst_files_dir_)); + ASSERT_OK(env_->CreateDir(sst_files_dir_)); + } + + ~ExternalSSTFileTestBase() override { + DestroyDir(env_, sst_files_dir_).PermitUncheckedError(); + } + + protected: + std::string sst_files_dir_; +}; + +class ExternSSTFileLinkFailFallbackTest + : public ExternalSSTFileTestBase, + public ::testing::WithParamInterface> { + public: + ExternSSTFileLinkFailFallbackTest() { + fs_ = std::make_shared(env_->GetFileSystem(), true); + test_env_.reset(new CompositeEnvWrapper(env_, fs_)); + options_ = CurrentOptions(); + options_.disable_auto_compactions = true; + options_.env = test_env_.get(); + } + + void TearDown() override { + delete db_; + db_ = nullptr; + ASSERT_OK(DestroyDB(dbname_, options_)); + } + + protected: + Options options_; + std::shared_ptr fs_; + std::unique_ptr test_env_; +}; + +class ExternalSSTFileTest + : public ExternalSSTFileTestBase, + public ::testing::WithParamInterface> { + public: + ExternalSSTFileTest() {} + + Status GenerateOneExternalFile( + const Options& options, ColumnFamilyHandle* cfh, + std::vector>& data, int file_id, + bool sort_data, std::string* external_file_path, + std::map* true_data) { + // Generate a file id if not provided + if (-1 == file_id) { + file_id = (++last_file_id_); + } + // Sort data if asked to do so + if (sort_data) { + std::sort(data.begin(), data.end(), + [&](const std::pair& e1, + const std::pair& e2) { + return options.comparator->Compare(e1.first, e2.first) < 0; + }); + auto uniq_iter = std::unique( + data.begin(), data.end(), + [&](const std::pair& e1, + const std::pair& e2) { + return options.comparator->Compare(e1.first, e2.first) == 0; + }); + data.resize(uniq_iter - data.begin()); + } + std::string file_path = sst_files_dir_ + std::to_string(file_id); + SstFileWriter sst_file_writer(EnvOptions(), options, cfh); + Status s = sst_file_writer.Open(file_path); + if (!s.ok()) { + return s; + } + for (const auto& entry : data) { + s = sst_file_writer.Put(entry.first, entry.second); + if (!s.ok()) { + sst_file_writer.Finish().PermitUncheckedError(); + return s; + } + } + s = sst_file_writer.Finish(); + if (s.ok() && external_file_path != nullptr) { + *external_file_path = file_path; + } + if (s.ok() && nullptr != true_data) { + for (const auto& entry : data) { + true_data->insert({entry.first, entry.second}); + } + } + return s; + } + + Status GenerateAndAddExternalFile( + const Options options, + std::vector> data, int file_id = -1, + bool allow_global_seqno = false, bool write_global_seqno = false, + bool verify_checksums_before_ingest = true, bool ingest_behind = false, + bool sort_data = false, + std::map* true_data = nullptr, + ColumnFamilyHandle* cfh = nullptr) { + // Generate a file id if not provided + if (file_id == -1) { + file_id = last_file_id_ + 1; + last_file_id_++; + } + + // Sort data if asked to do so + if (sort_data) { + std::sort(data.begin(), data.end(), + [&](const std::pair& e1, + const std::pair& e2) { + return options.comparator->Compare(e1.first, e2.first) < 0; + }); + auto uniq_iter = std::unique( + data.begin(), data.end(), + [&](const std::pair& e1, + const std::pair& e2) { + return options.comparator->Compare(e1.first, e2.first) == 0; + }); + data.resize(uniq_iter - data.begin()); + } + std::string file_path = sst_files_dir_ + std::to_string(file_id); + SstFileWriter sst_file_writer(EnvOptions(), options, cfh); + + Status s = sst_file_writer.Open(file_path); + if (!s.ok()) { + return s; + } + for (auto& entry : data) { + s = sst_file_writer.Put(entry.first, entry.second); + if (!s.ok()) { + sst_file_writer.Finish().PermitUncheckedError(); + return s; + } + } + s = sst_file_writer.Finish(); + + if (s.ok()) { + IngestExternalFileOptions ifo; + ifo.allow_global_seqno = allow_global_seqno; + ifo.write_global_seqno = allow_global_seqno ? write_global_seqno : false; + ifo.verify_checksums_before_ingest = verify_checksums_before_ingest; + ifo.ingest_behind = ingest_behind; + if (cfh) { + s = db_->IngestExternalFile(cfh, {file_path}, ifo); + } else { + s = db_->IngestExternalFile({file_path}, ifo); + } + } + + if (s.ok() && true_data) { + for (auto& entry : data) { + (*true_data)[entry.first] = entry.second; + } + } + + return s; + } + + Status GenerateAndAddExternalFiles( + const Options& options, + const std::vector& column_families, + const std::vector& ifos, + std::vector>>& data, + int file_id, bool sort_data, + std::vector>& true_data) { + if (-1 == file_id) { + file_id = (++last_file_id_); + } + // Generate external SST files, one for each column family + size_t num_cfs = column_families.size(); + assert(ifos.size() == num_cfs); + assert(data.size() == num_cfs); + std::vector args(num_cfs); + for (size_t i = 0; i != num_cfs; ++i) { + std::string external_file_path; + Status s = GenerateOneExternalFile( + options, column_families[i], data[i], file_id, sort_data, + &external_file_path, + true_data.size() == num_cfs ? &true_data[i] : nullptr); + if (!s.ok()) { + return s; + } + ++file_id; + + args[i].column_family = column_families[i]; + args[i].external_files.push_back(external_file_path); + args[i].options = ifos[i]; + } + return db_->IngestExternalFiles(args); + } + + Status GenerateAndAddExternalFile( + const Options options, std::vector> data, + int file_id = -1, bool allow_global_seqno = false, + bool write_global_seqno = false, + bool verify_checksums_before_ingest = true, bool ingest_behind = false, + bool sort_data = false, + std::map* true_data = nullptr, + ColumnFamilyHandle* cfh = nullptr) { + std::vector> file_data; + for (auto& entry : data) { + file_data.emplace_back(Key(entry.first), entry.second); + } + return GenerateAndAddExternalFile(options, file_data, file_id, + allow_global_seqno, write_global_seqno, + verify_checksums_before_ingest, + ingest_behind, sort_data, true_data, cfh); + } + + Status GenerateAndAddExternalFile( + const Options options, std::vector keys, int file_id = -1, + bool allow_global_seqno = false, bool write_global_seqno = false, + bool verify_checksums_before_ingest = true, bool ingest_behind = false, + bool sort_data = false, + std::map* true_data = nullptr, + ColumnFamilyHandle* cfh = nullptr) { + std::vector> file_data; + for (auto& k : keys) { + file_data.emplace_back(Key(k), Key(k) + std::to_string(file_id)); + } + return GenerateAndAddExternalFile(options, file_data, file_id, + allow_global_seqno, write_global_seqno, + verify_checksums_before_ingest, + ingest_behind, sort_data, true_data, cfh); + } + + Status DeprecatedAddFile(const std::vector& files, + bool move_files = false, + bool skip_snapshot_check = false, + bool skip_write_global_seqno = false) { + IngestExternalFileOptions opts; + opts.move_files = move_files; + opts.snapshot_consistency = !skip_snapshot_check; + opts.allow_global_seqno = false; + opts.allow_blocking_flush = false; + opts.write_global_seqno = !skip_write_global_seqno; + return db_->IngestExternalFile(files, opts); + } + + protected: + int last_file_id_ = 0; +}; + +TEST_F(ExternalSSTFileTest, Basic) { + do { + Options options = CurrentOptions(); + + SstFileWriter sst_file_writer(EnvOptions(), options); + + // Current file size should be 0 after sst_file_writer init and before open + // a file. + ASSERT_EQ(sst_file_writer.FileSize(), 0); + + // file1.sst (0 => 99) + std::string file1 = sst_files_dir_ + "file1.sst"; + ASSERT_OK(sst_file_writer.Open(file1)); + for (int k = 0; k < 100; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); + } + ExternalSstFileInfo file1_info; + ASSERT_OK(sst_file_writer.Finish(&file1_info)); + + // Current file size should be non-zero after success write. + ASSERT_GT(sst_file_writer.FileSize(), 0); + + ASSERT_EQ(file1_info.file_path, file1); + ASSERT_EQ(file1_info.num_entries, 100); + ASSERT_EQ(file1_info.smallest_key, Key(0)); + ASSERT_EQ(file1_info.largest_key, Key(99)); + ASSERT_EQ(file1_info.num_range_del_entries, 0); + ASSERT_EQ(file1_info.smallest_range_del_key, ""); + ASSERT_EQ(file1_info.largest_range_del_key, ""); + // sst_file_writer already finished, cannot add this value + ASSERT_NOK(sst_file_writer.Put(Key(100), "bad_val")); + + // file2.sst (100 => 199) + std::string file2 = sst_files_dir_ + "file2.sst"; + ASSERT_OK(sst_file_writer.Open(file2)); + for (int k = 100; k < 200; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); + } + // Cannot add this key because it's not after last added key + ASSERT_NOK(sst_file_writer.Put(Key(99), "bad_val")); + ExternalSstFileInfo file2_info; + ASSERT_OK(sst_file_writer.Finish(&file2_info)); + ASSERT_EQ(file2_info.file_path, file2); + ASSERT_EQ(file2_info.num_entries, 100); + ASSERT_EQ(file2_info.smallest_key, Key(100)); + ASSERT_EQ(file2_info.largest_key, Key(199)); + + // file3.sst (195 => 299) + // This file values overlap with file2 values + std::string file3 = sst_files_dir_ + "file3.sst"; + ASSERT_OK(sst_file_writer.Open(file3)); + for (int k = 195; k < 300; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap")); + } + ExternalSstFileInfo file3_info; + ASSERT_OK(sst_file_writer.Finish(&file3_info)); + + // Current file size should be non-zero after success finish. + ASSERT_GT(sst_file_writer.FileSize(), 0); + ASSERT_EQ(file3_info.file_path, file3); + ASSERT_EQ(file3_info.num_entries, 105); + ASSERT_EQ(file3_info.smallest_key, Key(195)); + ASSERT_EQ(file3_info.largest_key, Key(299)); + + // file4.sst (30 => 39) + // This file values overlap with file1 values + std::string file4 = sst_files_dir_ + "file4.sst"; + ASSERT_OK(sst_file_writer.Open(file4)); + for (int k = 30; k < 40; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap")); + } + ExternalSstFileInfo file4_info; + ASSERT_OK(sst_file_writer.Finish(&file4_info)); + ASSERT_EQ(file4_info.file_path, file4); + ASSERT_EQ(file4_info.num_entries, 10); + ASSERT_EQ(file4_info.smallest_key, Key(30)); + ASSERT_EQ(file4_info.largest_key, Key(39)); + + // file5.sst (400 => 499) + std::string file5 = sst_files_dir_ + "file5.sst"; + ASSERT_OK(sst_file_writer.Open(file5)); + for (int k = 400; k < 500; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); + } + ExternalSstFileInfo file5_info; + ASSERT_OK(sst_file_writer.Finish(&file5_info)); + ASSERT_EQ(file5_info.file_path, file5); + ASSERT_EQ(file5_info.num_entries, 100); + ASSERT_EQ(file5_info.smallest_key, Key(400)); + ASSERT_EQ(file5_info.largest_key, Key(499)); + + // file6.sst (delete 400 => 500) + std::string file6 = sst_files_dir_ + "file6.sst"; + ASSERT_OK(sst_file_writer.Open(file6)); + ASSERT_OK(sst_file_writer.DeleteRange(Key(400), Key(500))); + ExternalSstFileInfo file6_info; + ASSERT_OK(sst_file_writer.Finish(&file6_info)); + ASSERT_EQ(file6_info.file_path, file6); + ASSERT_EQ(file6_info.num_entries, 0); + ASSERT_EQ(file6_info.smallest_key, ""); + ASSERT_EQ(file6_info.largest_key, ""); + ASSERT_EQ(file6_info.num_range_del_entries, 1); + ASSERT_EQ(file6_info.smallest_range_del_key, Key(400)); + ASSERT_EQ(file6_info.largest_range_del_key, Key(500)); + + // file7.sst (delete 500 => 570, put 520 => 599 divisible by 2) + std::string file7 = sst_files_dir_ + "file7.sst"; + ASSERT_OK(sst_file_writer.Open(file7)); + ASSERT_OK(sst_file_writer.DeleteRange(Key(500), Key(550))); + for (int k = 520; k < 560; k += 2) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); + } + ASSERT_OK(sst_file_writer.DeleteRange(Key(525), Key(575))); + for (int k = 560; k < 600; k += 2) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); + } + ExternalSstFileInfo file7_info; + ASSERT_OK(sst_file_writer.Finish(&file7_info)); + ASSERT_EQ(file7_info.file_path, file7); + ASSERT_EQ(file7_info.num_entries, 40); + ASSERT_EQ(file7_info.smallest_key, Key(520)); + ASSERT_EQ(file7_info.largest_key, Key(598)); + ASSERT_EQ(file7_info.num_range_del_entries, 2); + ASSERT_EQ(file7_info.smallest_range_del_key, Key(500)); + ASSERT_EQ(file7_info.largest_range_del_key, Key(575)); + + // file8.sst (delete 600 => 700) + std::string file8 = sst_files_dir_ + "file8.sst"; + ASSERT_OK(sst_file_writer.Open(file8)); + ASSERT_OK(sst_file_writer.DeleteRange(Key(600), Key(700))); + ExternalSstFileInfo file8_info; + ASSERT_OK(sst_file_writer.Finish(&file8_info)); + ASSERT_EQ(file8_info.file_path, file8); + ASSERT_EQ(file8_info.num_entries, 0); + ASSERT_EQ(file8_info.smallest_key, ""); + ASSERT_EQ(file8_info.largest_key, ""); + ASSERT_EQ(file8_info.num_range_del_entries, 1); + ASSERT_EQ(file8_info.smallest_range_del_key, Key(600)); + ASSERT_EQ(file8_info.largest_range_del_key, Key(700)); + + // Cannot create an empty sst file + std::string file_empty = sst_files_dir_ + "file_empty.sst"; + ExternalSstFileInfo file_empty_info; + ASSERT_NOK(sst_file_writer.Finish(&file_empty_info)); + + DestroyAndReopen(options); + // Add file using file path + ASSERT_OK(DeprecatedAddFile({file1})); + ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U); + for (int k = 0; k < 100; k++) { + ASSERT_EQ(Get(Key(k)), Key(k) + "_val"); + } + + // Add file while holding a snapshot will fail + const Snapshot* s1 = db_->GetSnapshot(); + if (s1 != nullptr) { + ASSERT_NOK(DeprecatedAddFile({file2})); + db_->ReleaseSnapshot(s1); + } + // We can add the file after releaseing the snapshot + ASSERT_OK(DeprecatedAddFile({file2})); + + ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U); + for (int k = 0; k < 200; k++) { + ASSERT_EQ(Get(Key(k)), Key(k) + "_val"); + } + + // This file has overlapping values with the existing data + ASSERT_NOK(DeprecatedAddFile({file3})); + + // This file has overlapping values with the existing data + ASSERT_NOK(DeprecatedAddFile({file4})); + + // Overwrite values of keys divisible by 5 + for (int k = 0; k < 200; k += 5) { + ASSERT_OK(Put(Key(k), Key(k) + "_val_new")); + } + ASSERT_NE(db_->GetLatestSequenceNumber(), 0U); + + // Key range of file5 (400 => 499) don't overlap with any keys in DB + ASSERT_OK(DeprecatedAddFile({file5})); + + // This file has overlapping values with the existing data + ASSERT_NOK(DeprecatedAddFile({file6})); + + // Key range of file7 (500 => 598) don't overlap with any keys in DB + ASSERT_OK(DeprecatedAddFile({file7})); + + // Key range of file7 (600 => 700) don't overlap with any keys in DB + ASSERT_OK(DeprecatedAddFile({file8})); + + // Make sure values are correct before and after flush/compaction + for (int i = 0; i < 2; i++) { + for (int k = 0; k < 200; k++) { + std::string value = Key(k) + "_val"; + if (k % 5 == 0) { + value += "_new"; + } + ASSERT_EQ(Get(Key(k)), value); + } + for (int k = 400; k < 500; k++) { + std::string value = Key(k) + "_val"; + ASSERT_EQ(Get(Key(k)), value); + } + for (int k = 500; k < 600; k++) { + std::string value = Key(k) + "_val"; + if (k < 520 || k % 2 == 1) { + value = "NOT_FOUND"; + } + ASSERT_EQ(Get(Key(k)), value); + } + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + } + + Close(); + options.disable_auto_compactions = true; + Reopen(options); + + // Delete keys in range (400 => 499) + for (int k = 400; k < 500; k++) { + ASSERT_OK(Delete(Key(k))); + } + // We deleted range (400 => 499) but cannot add file5 because + // of the range tombstones + ASSERT_NOK(DeprecatedAddFile({file5})); + + // Compacting the DB will remove the tombstones + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + // Now we can add the file + ASSERT_OK(DeprecatedAddFile({file5})); + + // Verify values of file5 in DB + for (int k = 400; k < 500; k++) { + std::string value = Key(k) + "_val"; + ASSERT_EQ(Get(Key(k)), value); + } + DestroyAndRecreateExternalSSTFilesDir(); + } while (ChangeOptions(kSkipPlainTable | kSkipFIFOCompaction | + kRangeDelSkipConfigs)); +} + +class SstFileWriterCollector : public TablePropertiesCollector { + public: + explicit SstFileWriterCollector(const std::string prefix) : prefix_(prefix) { + name_ = prefix_ + "_SstFileWriterCollector"; + } + + const char* Name() const override { return name_.c_str(); } + + Status Finish(UserCollectedProperties* properties) override { + std::string count = std::to_string(count_); + *properties = UserCollectedProperties{ + {prefix_ + "_SstFileWriterCollector", "YES"}, + {prefix_ + "_Count", count}, + }; + return Status::OK(); + } + + Status AddUserKey(const Slice& /*user_key*/, const Slice& /*value*/, + EntryType /*type*/, SequenceNumber /*seq*/, + uint64_t /*file_size*/) override { + ++count_; + return Status::OK(); + } + + UserCollectedProperties GetReadableProperties() const override { + return UserCollectedProperties{}; + } + + private: + uint32_t count_ = 0; + std::string prefix_; + std::string name_; +}; + +class SstFileWriterCollectorFactory : public TablePropertiesCollectorFactory { + public: + explicit SstFileWriterCollectorFactory(std::string prefix) + : prefix_(prefix), num_created_(0) {} + TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context /*context*/) override { + num_created_++; + return new SstFileWriterCollector(prefix_); + } + const char* Name() const override { return "SstFileWriterCollectorFactory"; } + + std::string prefix_; + uint32_t num_created_; +}; + +TEST_F(ExternalSSTFileTest, AddList) { + do { + Options options = CurrentOptions(); + + auto abc_collector = std::make_shared("abc"); + auto xyz_collector = std::make_shared("xyz"); + + options.table_properties_collector_factories.emplace_back(abc_collector); + options.table_properties_collector_factories.emplace_back(xyz_collector); + + SstFileWriter sst_file_writer(EnvOptions(), options); + + // file1.sst (0 => 99) + std::string file1 = sst_files_dir_ + "file1.sst"; + ASSERT_OK(sst_file_writer.Open(file1)); + for (int k = 0; k < 100; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); + } + ExternalSstFileInfo file1_info; + ASSERT_OK(sst_file_writer.Finish(&file1_info)); + ASSERT_EQ(file1_info.file_path, file1); + ASSERT_EQ(file1_info.num_entries, 100); + ASSERT_EQ(file1_info.smallest_key, Key(0)); + ASSERT_EQ(file1_info.largest_key, Key(99)); + // sst_file_writer already finished, cannot add this value + ASSERT_NOK(sst_file_writer.Put(Key(100), "bad_val")); + + // file2.sst (100 => 199) + std::string file2 = sst_files_dir_ + "file2.sst"; + ASSERT_OK(sst_file_writer.Open(file2)); + for (int k = 100; k < 200; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); + } + // Cannot add this key because it's not after last added key + ASSERT_NOK(sst_file_writer.Put(Key(99), "bad_val")); + ExternalSstFileInfo file2_info; + ASSERT_OK(sst_file_writer.Finish(&file2_info)); + ASSERT_EQ(file2_info.file_path, file2); + ASSERT_EQ(file2_info.num_entries, 100); + ASSERT_EQ(file2_info.smallest_key, Key(100)); + ASSERT_EQ(file2_info.largest_key, Key(199)); + + // file3.sst (195 => 199) + // This file values overlap with file2 values + std::string file3 = sst_files_dir_ + "file3.sst"; + ASSERT_OK(sst_file_writer.Open(file3)); + for (int k = 195; k < 200; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap")); + } + ExternalSstFileInfo file3_info; + ASSERT_OK(sst_file_writer.Finish(&file3_info)); + ASSERT_EQ(file3_info.file_path, file3); + ASSERT_EQ(file3_info.num_entries, 5); + ASSERT_EQ(file3_info.smallest_key, Key(195)); + ASSERT_EQ(file3_info.largest_key, Key(199)); + + // file4.sst (30 => 39) + // This file values overlap with file1 values + std::string file4 = sst_files_dir_ + "file4.sst"; + ASSERT_OK(sst_file_writer.Open(file4)); + for (int k = 30; k < 40; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap")); + } + ExternalSstFileInfo file4_info; + ASSERT_OK(sst_file_writer.Finish(&file4_info)); + ASSERT_EQ(file4_info.file_path, file4); + ASSERT_EQ(file4_info.num_entries, 10); + ASSERT_EQ(file4_info.smallest_key, Key(30)); + ASSERT_EQ(file4_info.largest_key, Key(39)); + + // file5.sst (200 => 299) + std::string file5 = sst_files_dir_ + "file5.sst"; + ASSERT_OK(sst_file_writer.Open(file5)); + for (int k = 200; k < 300; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); + } + ExternalSstFileInfo file5_info; + ASSERT_OK(sst_file_writer.Finish(&file5_info)); + ASSERT_EQ(file5_info.file_path, file5); + ASSERT_EQ(file5_info.num_entries, 100); + ASSERT_EQ(file5_info.smallest_key, Key(200)); + ASSERT_EQ(file5_info.largest_key, Key(299)); + + // file6.sst (delete 0 => 100) + std::string file6 = sst_files_dir_ + "file6.sst"; + ASSERT_OK(sst_file_writer.Open(file6)); + ASSERT_OK(sst_file_writer.DeleteRange(Key(0), Key(75))); + ASSERT_OK(sst_file_writer.DeleteRange(Key(25), Key(100))); + ExternalSstFileInfo file6_info; + ASSERT_OK(sst_file_writer.Finish(&file6_info)); + ASSERT_EQ(file6_info.file_path, file6); + ASSERT_EQ(file6_info.num_entries, 0); + ASSERT_EQ(file6_info.smallest_key, ""); + ASSERT_EQ(file6_info.largest_key, ""); + ASSERT_EQ(file6_info.num_range_del_entries, 2); + ASSERT_EQ(file6_info.smallest_range_del_key, Key(0)); + ASSERT_EQ(file6_info.largest_range_del_key, Key(100)); + + // file7.sst (delete 99 => 201) + std::string file7 = sst_files_dir_ + "file7.sst"; + ASSERT_OK(sst_file_writer.Open(file7)); + ASSERT_OK(sst_file_writer.DeleteRange(Key(99), Key(201))); + ExternalSstFileInfo file7_info; + ASSERT_OK(sst_file_writer.Finish(&file7_info)); + ASSERT_EQ(file7_info.file_path, file7); + ASSERT_EQ(file7_info.num_entries, 0); + ASSERT_EQ(file7_info.smallest_key, ""); + ASSERT_EQ(file7_info.largest_key, ""); + ASSERT_EQ(file7_info.num_range_del_entries, 1); + ASSERT_EQ(file7_info.smallest_range_del_key, Key(99)); + ASSERT_EQ(file7_info.largest_range_del_key, Key(201)); + + // list 1 has internal key range conflict + std::vector file_list0({file1, file2}); + std::vector file_list1({file3, file2, file1}); + std::vector file_list2({file5}); + std::vector file_list3({file3, file4}); + std::vector file_list4({file5, file7}); + std::vector file_list5({file6, file7}); + + DestroyAndReopen(options); + + // These lists of files have key ranges that overlap with each other + ASSERT_NOK(DeprecatedAddFile(file_list1)); + // Both of the following overlap on the range deletion tombstone. + ASSERT_NOK(DeprecatedAddFile(file_list4)); + ASSERT_NOK(DeprecatedAddFile(file_list5)); + + // Add files using file path list + ASSERT_OK(DeprecatedAddFile(file_list0)); + ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U); + for (int k = 0; k < 200; k++) { + ASSERT_EQ(Get(Key(k)), Key(k) + "_val"); + } + + TablePropertiesCollection props; + ASSERT_OK(db_->GetPropertiesOfAllTables(&props)); + ASSERT_EQ(props.size(), 2); + for (auto file_props : props) { + auto user_props = file_props.second->user_collected_properties; + ASSERT_EQ(user_props["abc_SstFileWriterCollector"], "YES"); + ASSERT_EQ(user_props["xyz_SstFileWriterCollector"], "YES"); + ASSERT_EQ(user_props["abc_Count"], "100"); + ASSERT_EQ(user_props["xyz_Count"], "100"); + } + + // Add file while holding a snapshot will fail + const Snapshot* s1 = db_->GetSnapshot(); + if (s1 != nullptr) { + ASSERT_NOK(DeprecatedAddFile(file_list2)); + db_->ReleaseSnapshot(s1); + } + // We can add the file after releaseing the snapshot + ASSERT_OK(DeprecatedAddFile(file_list2)); + ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U); + for (int k = 0; k < 300; k++) { + ASSERT_EQ(Get(Key(k)), Key(k) + "_val"); + } + + ASSERT_OK(db_->GetPropertiesOfAllTables(&props)); + ASSERT_EQ(props.size(), 3); + for (auto file_props : props) { + auto user_props = file_props.second->user_collected_properties; + ASSERT_EQ(user_props["abc_SstFileWriterCollector"], "YES"); + ASSERT_EQ(user_props["xyz_SstFileWriterCollector"], "YES"); + ASSERT_EQ(user_props["abc_Count"], "100"); + ASSERT_EQ(user_props["xyz_Count"], "100"); + } + + // This file list has overlapping values with the existing data + ASSERT_NOK(DeprecatedAddFile(file_list3)); + + // Overwrite values of keys divisible by 5 + for (int k = 0; k < 200; k += 5) { + ASSERT_OK(Put(Key(k), Key(k) + "_val_new")); + } + ASSERT_NE(db_->GetLatestSequenceNumber(), 0U); + + // Make sure values are correct before and after flush/compaction + for (int i = 0; i < 2; i++) { + for (int k = 0; k < 200; k++) { + std::string value = Key(k) + "_val"; + if (k % 5 == 0) { + value += "_new"; + } + ASSERT_EQ(Get(Key(k)), value); + } + for (int k = 200; k < 300; k++) { + std::string value = Key(k) + "_val"; + ASSERT_EQ(Get(Key(k)), value); + } + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + } + + // Delete keys in range (200 => 299) + for (int k = 200; k < 300; k++) { + ASSERT_OK(Delete(Key(k))); + } + // We deleted range (200 => 299) but cannot add file5 because + // of the range tombstones + ASSERT_NOK(DeprecatedAddFile(file_list2)); + + // Compacting the DB will remove the tombstones + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + // Now we can add the file + ASSERT_OK(DeprecatedAddFile(file_list2)); + + // Verify values of file5 in DB + for (int k = 200; k < 300; k++) { + std::string value = Key(k) + "_val"; + ASSERT_EQ(Get(Key(k)), value); + } + DestroyAndRecreateExternalSSTFilesDir(); + } while (ChangeOptions(kSkipPlainTable | kSkipFIFOCompaction | + kRangeDelSkipConfigs)); +} + +TEST_F(ExternalSSTFileTest, AddListAtomicity) { + do { + Options options = CurrentOptions(); + + SstFileWriter sst_file_writer(EnvOptions(), options); + + // files[0].sst (0 => 99) + // files[1].sst (100 => 199) + // ... + // file[8].sst (800 => 899) + int n = 9; + std::vector files(n); + std::vector files_info(n); + for (int i = 0; i < n; i++) { + files[i] = sst_files_dir_ + "file" + std::to_string(i) + ".sst"; + ASSERT_OK(sst_file_writer.Open(files[i])); + for (int k = i * 100; k < (i + 1) * 100; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); + } + ASSERT_OK(sst_file_writer.Finish(&files_info[i])); + ASSERT_EQ(files_info[i].file_path, files[i]); + ASSERT_EQ(files_info[i].num_entries, 100); + ASSERT_EQ(files_info[i].smallest_key, Key(i * 100)); + ASSERT_EQ(files_info[i].largest_key, Key((i + 1) * 100 - 1)); + } + files.push_back(sst_files_dir_ + "file" + std::to_string(n) + ".sst"); + ASSERT_NOK(DeprecatedAddFile(files)); + for (int k = 0; k < n * 100; k++) { + ASSERT_EQ("NOT_FOUND", Get(Key(k))); + } + files.pop_back(); + ASSERT_OK(DeprecatedAddFile(files)); + for (int k = 0; k < n * 100; k++) { + std::string value = Key(k) + "_val"; + ASSERT_EQ(Get(Key(k)), value); + } + DestroyAndRecreateExternalSSTFilesDir(); + } while (ChangeOptions(kSkipPlainTable | kSkipFIFOCompaction)); +} +// This test reporduce a bug that can happen in some cases if the DB started +// purging obsolete files when we are adding an external sst file. +// This situation may result in deleting the file while it's being added. +TEST_F(ExternalSSTFileTest, PurgeObsoleteFilesBug) { + Options options = CurrentOptions(); + SstFileWriter sst_file_writer(EnvOptions(), options); + + // file1.sst (0 => 500) + std::string sst_file_path = sst_files_dir_ + "file1.sst"; + ASSERT_OK(sst_file_writer.Open(sst_file_path)); + for (int i = 0; i < 500; i++) { + std::string k = Key(i); + ASSERT_OK(sst_file_writer.Put(k, k + "_val")); + } + + ExternalSstFileInfo sst_file_info; + ASSERT_OK(sst_file_writer.Finish(&sst_file_info)); + + options.delete_obsolete_files_period_micros = 0; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "ExternalSstFileIngestionJob::Prepare:FileAdded", [&](void* /* arg */) { + ASSERT_OK(Put("aaa", "bbb")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("aaa", "xxx")); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(DeprecatedAddFile({sst_file_path})); + + for (int i = 0; i < 500; i++) { + std::string k = Key(i); + std::string v = k + "_val"; + ASSERT_EQ(Get(k), v); + } + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(ExternalSSTFileTest, SkipSnapshot) { + Options options = CurrentOptions(); + + SstFileWriter sst_file_writer(EnvOptions(), options); + + // file1.sst (0 => 99) + std::string file1 = sst_files_dir_ + "file1.sst"; + ASSERT_OK(sst_file_writer.Open(file1)); + for (int k = 0; k < 100; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); + } + ExternalSstFileInfo file1_info; + ASSERT_OK(sst_file_writer.Finish(&file1_info)); + ASSERT_EQ(file1_info.file_path, file1); + ASSERT_EQ(file1_info.num_entries, 100); + ASSERT_EQ(file1_info.smallest_key, Key(0)); + ASSERT_EQ(file1_info.largest_key, Key(99)); + + // file2.sst (100 => 299) + std::string file2 = sst_files_dir_ + "file2.sst"; + ASSERT_OK(sst_file_writer.Open(file2)); + for (int k = 100; k < 300; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); + } + ExternalSstFileInfo file2_info; + ASSERT_OK(sst_file_writer.Finish(&file2_info)); + ASSERT_EQ(file2_info.file_path, file2); + ASSERT_EQ(file2_info.num_entries, 200); + ASSERT_EQ(file2_info.smallest_key, Key(100)); + ASSERT_EQ(file2_info.largest_key, Key(299)); + + ASSERT_OK(DeprecatedAddFile({file1})); + + // Add file will fail when holding snapshot and use the default + // skip_snapshot_check to false + const Snapshot* s1 = db_->GetSnapshot(); + if (s1 != nullptr) { + ASSERT_NOK(DeprecatedAddFile({file2})); + } + + // Add file will success when set skip_snapshot_check to true even db holding + // snapshot + if (s1 != nullptr) { + ASSERT_OK(DeprecatedAddFile({file2}, false, true)); + db_->ReleaseSnapshot(s1); + } + + // file3.sst (300 => 399) + std::string file3 = sst_files_dir_ + "file3.sst"; + ASSERT_OK(sst_file_writer.Open(file3)); + for (int k = 300; k < 400; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); + } + ExternalSstFileInfo file3_info; + ASSERT_OK(sst_file_writer.Finish(&file3_info)); + ASSERT_EQ(file3_info.file_path, file3); + ASSERT_EQ(file3_info.num_entries, 100); + ASSERT_EQ(file3_info.smallest_key, Key(300)); + ASSERT_EQ(file3_info.largest_key, Key(399)); + + // check that we have change the old key + ASSERT_EQ(Get(Key(300)), "NOT_FOUND"); + const Snapshot* s2 = db_->GetSnapshot(); + ASSERT_OK(DeprecatedAddFile({file3}, false, true)); + ASSERT_EQ(Get(Key(300)), Key(300) + ("_val")); + ASSERT_EQ(Get(Key(300), s2), Key(300) + ("_val")); + + db_->ReleaseSnapshot(s2); +} + +TEST_F(ExternalSSTFileTest, MultiThreaded) { + env_->skip_fsync_ = true; + // Bulk load 10 files every file contain 1000 keys + int num_files = 10; + int keys_per_file = 1000; + + // Generate file names + std::vector file_names; + for (int i = 0; i < num_files; i++) { + std::string file_name = "file_" + std::to_string(i) + ".sst"; + file_names.push_back(sst_files_dir_ + file_name); + } + + do { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + std::atomic thread_num(0); + std::function write_file_func = [&]() { + int file_idx = thread_num.fetch_add(1); + int range_start = file_idx * keys_per_file; + int range_end = range_start + keys_per_file; + + SstFileWriter sst_file_writer(EnvOptions(), options); + + ASSERT_OK(sst_file_writer.Open(file_names[file_idx])); + + for (int k = range_start; k < range_end; k++) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k))); + } + + ASSERT_OK(sst_file_writer.Finish()); + }; + // Write num_files files in parallel + std::vector sst_writer_threads; + for (int i = 0; i < num_files; ++i) { + sst_writer_threads.emplace_back(write_file_func); + } + + for (auto& t : sst_writer_threads) { + t.join(); + } + + fprintf(stderr, "Wrote %d files (%d keys)\n", num_files, + num_files * keys_per_file); + + thread_num.store(0); + std::atomic files_added(0); + // Thread 0 -> Load {f0,f1} + // Thread 1 -> Load {f0,f1} + // Thread 2 -> Load {f2,f3} + // Thread 3 -> Load {f2,f3} + // Thread 4 -> Load {f4,f5} + // Thread 5 -> Load {f4,f5} + // ... + std::function load_file_func = [&]() { + // We intentionally add every file twice, and assert that it was added + // only once and the other add failed + int thread_id = thread_num.fetch_add(1); + int file_idx = (thread_id / 2) * 2; + // sometimes we use copy, sometimes link .. the result should be the same + bool move_file = (thread_id % 3 == 0); + + std::vector files_to_add; + + files_to_add = {file_names[file_idx]}; + if (static_cast(file_idx + 1) < file_names.size()) { + files_to_add.push_back(file_names[file_idx + 1]); + } + + Status s = DeprecatedAddFile(files_to_add, move_file); + if (s.ok()) { + files_added += static_cast(files_to_add.size()); + } + }; + + // Bulk load num_files files in parallel + std::vector add_file_threads; + DestroyAndReopen(options); + for (int i = 0; i < num_files; ++i) { + add_file_threads.emplace_back(load_file_func); + } + + for (auto& t : add_file_threads) { + t.join(); + } + ASSERT_EQ(files_added.load(), num_files); + fprintf(stderr, "Loaded %d files (%d keys)\n", num_files, + num_files * keys_per_file); + + // Overwrite values of keys divisible by 100 + for (int k = 0; k < num_files * keys_per_file; k += 100) { + std::string key = Key(k); + ASSERT_OK(Put(key, key + "_new")); + } + + for (int i = 0; i < 2; i++) { + // Make sure the values are correct before and after flush/compaction + for (int k = 0; k < num_files * keys_per_file; ++k) { + std::string key = Key(k); + std::string value = (k % 100 == 0) ? (key + "_new") : key; + ASSERT_EQ(Get(key), value); + } + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + } + + fprintf(stderr, "Verified %d values\n", num_files * keys_per_file); + DestroyAndRecreateExternalSSTFilesDir(); + } while (ChangeOptions(kSkipPlainTable | kSkipFIFOCompaction)); +} + +TEST_F(ExternalSSTFileTest, OverlappingRanges) { + env_->skip_fsync_ = true; + Random rnd(301); + SequenceNumber assigned_seqno = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "ExternalSstFileIngestionJob::Run", [&assigned_seqno](void* arg) { + ASSERT_TRUE(arg != nullptr); + assigned_seqno = *(static_cast(arg)); + }); + bool need_flush = false; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::IngestExternalFile:NeedFlush", [&need_flush](void* arg) { + ASSERT_TRUE(arg != nullptr); + need_flush = *(static_cast(arg)); + }); + bool overlap_with_db = false; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile", + [&overlap_with_db](void* arg) { + ASSERT_TRUE(arg != nullptr); + overlap_with_db = *(static_cast(arg)); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + do { + Options options = CurrentOptions(); + env_->skip_fsync_ = true; + DestroyAndReopen(options); + + SstFileWriter sst_file_writer(EnvOptions(), options); + + printf("Option config = %d\n", option_config_); + std::vector> key_ranges; + for (int i = 0; i < 100; i++) { + int range_start = rnd.Uniform(20000); + int keys_per_range = 10 + rnd.Uniform(41); + + key_ranges.emplace_back(range_start, range_start + keys_per_range); + } + + int memtable_add = 0; + int success_add_file = 0; + int failed_add_file = 0; + std::map true_data; + for (size_t i = 0; i < key_ranges.size(); i++) { + int range_start = key_ranges[i].first; + int range_end = key_ranges[i].second; + + Status s; + std::string range_val = "range_" + std::to_string(i); + + // For 20% of ranges we use DB::Put, for 80% we use DB::AddFile + if (i && i % 5 == 0) { + // Use DB::Put to insert range (insert into memtable) + range_val += "_put"; + for (int k = range_start; k <= range_end; k++) { + s = Put(Key(k), range_val); + ASSERT_OK(s); + } + memtable_add++; + } else { + // Use DB::AddFile to insert range + range_val += "_add_file"; + + // Generate the file containing the range + std::string file_name = sst_files_dir_ + env_->GenerateUniqueId(); + s = sst_file_writer.Open(file_name); + ASSERT_OK(s); + for (int k = range_start; k <= range_end; k++) { + s = sst_file_writer.Put(Key(k), range_val); + ASSERT_OK(s); + } + ExternalSstFileInfo file_info; + s = sst_file_writer.Finish(&file_info); + ASSERT_OK(s); + + // Insert the generated file + s = DeprecatedAddFile({file_name}); + auto it = true_data.lower_bound(Key(range_start)); + if (option_config_ != kUniversalCompaction && + option_config_ != kUniversalCompactionMultiLevel && + option_config_ != kUniversalSubcompactions) { + if (it != true_data.end() && it->first <= Key(range_end)) { + // This range overlap with data already exist in DB + ASSERT_NOK(s); + failed_add_file++; + } else { + ASSERT_OK(s); + success_add_file++; + } + } else { + if ((it != true_data.end() && it->first <= Key(range_end)) || + need_flush || assigned_seqno > 0 || overlap_with_db) { + // This range overlap with data already exist in DB + ASSERT_NOK(s); + failed_add_file++; + } else { + ASSERT_OK(s); + success_add_file++; + } + } + } + + if (s.ok()) { + // Update true_data map to include the new inserted data + for (int k = range_start; k <= range_end; k++) { + true_data[Key(k)] = range_val; + } + } + + // Flush / Compact the DB + if (i && i % 50 == 0) { + ASSERT_OK(Flush()); + } + if (i && i % 75 == 0) { + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + } + } + + printf("Total: %" ROCKSDB_PRIszt + " ranges\n" + "AddFile()|Success: %d ranges\n" + "AddFile()|RangeConflict: %d ranges\n" + "Put(): %d ranges\n", + key_ranges.size(), success_add_file, failed_add_file, memtable_add); + + // Verify the correctness of the data + for (const auto& kv : true_data) { + ASSERT_EQ(Get(kv.first), kv.second); + } + printf("keys/values verified\n"); + DestroyAndRecreateExternalSSTFilesDir(); + } while (ChangeOptions(kSkipPlainTable | kSkipFIFOCompaction)); +} + +TEST_P(ExternalSSTFileTest, PickedLevel) { + env_->skip_fsync_ = true; + Options options = CurrentOptions(); + options.disable_auto_compactions = false; + options.level0_file_num_compaction_trigger = 4; + options.num_levels = 4; + DestroyAndReopen(options); + + std::map true_data; + + // File 0 will go to last level (L3) + ASSERT_OK(GenerateAndAddExternalFile(options, {1, 10}, -1, false, false, true, + false, false, &true_data)); + EXPECT_EQ(FilesPerLevel(), "0,0,0,1"); + + // File 1 will go to level L2 (since it overlap with file 0 in L3) + ASSERT_OK(GenerateAndAddExternalFile(options, {2, 9}, -1, false, false, true, + false, false, &true_data)); + EXPECT_EQ(FilesPerLevel(), "0,0,1,1"); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"ExternalSSTFileTest::PickedLevel:0", "BackgroundCallCompaction:0"}, + {"DBImpl::BackgroundCompaction:Start", + "ExternalSSTFileTest::PickedLevel:1"}, + {"ExternalSSTFileTest::PickedLevel:2", + "DBImpl::BackgroundCompaction:NonTrivial:AfterRun"}, + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Flush 4 files containing the same keys + for (int i = 0; i < 4; i++) { + ASSERT_OK(Put(Key(3), Key(3) + "put")); + ASSERT_OK(Put(Key(8), Key(8) + "put")); + true_data[Key(3)] = Key(3) + "put"; + true_data[Key(8)] = Key(8) + "put"; + ASSERT_OK(Flush()); + } + + // Wait for BackgroundCompaction() to be called + TEST_SYNC_POINT("ExternalSSTFileTest::PickedLevel:0"); + TEST_SYNC_POINT("ExternalSSTFileTest::PickedLevel:1"); + + EXPECT_EQ(FilesPerLevel(), "4,0,1,1"); + + // This file overlaps with file 0 (L3), file 1 (L2) and the + // output of compaction going to L1 + ASSERT_OK(GenerateAndAddExternalFile(options, {4, 7}, -1, + true /* allow_global_seqno */, false, + true, false, false, &true_data)); + EXPECT_EQ(FilesPerLevel(), "5,0,1,1"); + + // This file does not overlap with any file or with the running compaction + ASSERT_OK(GenerateAndAddExternalFile(options, {9000, 9001}, -1, false, false, + false, false, false, &true_data)); + EXPECT_EQ(FilesPerLevel(), "5,0,1,2"); + + // Hold compaction from finishing + TEST_SYNC_POINT("ExternalSSTFileTest::PickedLevel:2"); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + EXPECT_EQ(FilesPerLevel(), "1,1,1,2"); + + size_t kcnt = 0; + VerifyDBFromMap(true_data, &kcnt, false); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(ExternalSSTFileTest, IngestNonExistingFile) { + Options options = CurrentOptions(); + DestroyAndReopen(options); + + Status s = db_->IngestExternalFile({"non_existing_file"}, + IngestExternalFileOptions()); + ASSERT_NOK(s); + + // Verify file deletion is not impacted (verify a bug fix) + ASSERT_OK(Put(Key(1), Key(1))); + ASSERT_OK(Put(Key(9), Key(9))); + ASSERT_OK(Flush()); + + ASSERT_OK(Put(Key(1), Key(1))); + ASSERT_OK(Put(Key(9), Key(9))); + ASSERT_OK(Flush()); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // After full compaction, there should be only 1 file. + std::vector files; + ASSERT_OK(env_->GetChildren(dbname_, &files)); + int num_sst_files = 0; + for (auto& f : files) { + uint64_t number; + FileType type; + if (ParseFileName(f, &number, &type) && type == kTableFile) { + num_sst_files++; + } + } + ASSERT_EQ(1, num_sst_files); +} + +#if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) +TEST_F(ExternalSSTFileTest, CompactDuringAddFileRandom) { + env_->skip_fsync_ = true; + Options options = CurrentOptions(); + options.disable_auto_compactions = false; + options.level0_file_num_compaction_trigger = 2; + options.num_levels = 2; + DestroyAndReopen(options); + + std::function bg_compact = [&]() { + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + }; + + int range_id = 0; + std::vector file_keys; + std::function bg_addfile = [&]() { + ASSERT_OK(GenerateAndAddExternalFile(options, file_keys, range_id, + true /* allow_global_seqno */)); + }; + + const int num_of_ranges = 1000; + std::vector threads; + while (range_id < num_of_ranges) { + int range_start = range_id * 10; + int range_end = range_start + 10; + + file_keys.clear(); + for (int k = range_start + 1; k < range_end; k++) { + file_keys.push_back(k); + } + ASSERT_OK(Put(Key(range_start), Key(range_start))); + ASSERT_OK(Put(Key(range_end), Key(range_end))); + ASSERT_OK(Flush()); + + if (range_id % 10 == 0) { + threads.emplace_back(bg_compact); + } + threads.emplace_back(bg_addfile); + + for (auto& t : threads) { + t.join(); + } + threads.clear(); + + range_id++; + } + + for (int rid = 0; rid < num_of_ranges; rid++) { + int range_start = rid * 10; + int range_end = range_start + 10; + + ASSERT_EQ(Get(Key(range_start)), Key(range_start)) << rid; + ASSERT_EQ(Get(Key(range_end)), Key(range_end)) << rid; + for (int k = range_start + 1; k < range_end; k++) { + std::string v = Key(k) + std::to_string(rid); + ASSERT_EQ(Get(Key(k)), v) << rid; + } + } +} +#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) + +TEST_F(ExternalSSTFileTest, PickedLevelDynamic) { + env_->skip_fsync_ = true; + Options options = CurrentOptions(); + options.disable_auto_compactions = false; + options.level0_file_num_compaction_trigger = 4; + options.level_compaction_dynamic_level_bytes = true; + options.num_levels = 4; + DestroyAndReopen(options); + std::map true_data; + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"ExternalSSTFileTest::PickedLevelDynamic:0", + "BackgroundCallCompaction:0"}, + {"DBImpl::BackgroundCompaction:Start", + "ExternalSSTFileTest::PickedLevelDynamic:1"}, + {"ExternalSSTFileTest::PickedLevelDynamic:2", + "DBImpl::BackgroundCompaction:NonTrivial:AfterRun"}, + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Flush 4 files containing the same keys + for (int i = 0; i < 4; i++) { + for (int k = 20; k <= 30; k++) { + ASSERT_OK(Put(Key(k), Key(k) + "put")); + true_data[Key(k)] = Key(k) + "put"; + } + for (int k = 50; k <= 60; k++) { + ASSERT_OK(Put(Key(k), Key(k) + "put")); + true_data[Key(k)] = Key(k) + "put"; + } + ASSERT_OK(Flush()); + } + + // Wait for BackgroundCompaction() to be called + TEST_SYNC_POINT("ExternalSSTFileTest::PickedLevelDynamic:0"); + TEST_SYNC_POINT("ExternalSSTFileTest::PickedLevelDynamic:1"); + + // This file overlaps with the output of the compaction (going to L3) + // so the file will be added to L0 since L3 is the base level + ASSERT_OK(GenerateAndAddExternalFile(options, {31, 32, 33, 34}, -1, + true /* allow_global_seqno */, false, + true, false, false, &true_data)); + EXPECT_EQ(FilesPerLevel(), "5"); + + // This file does not overlap with the current running compactiong + ASSERT_OK(GenerateAndAddExternalFile(options, {9000, 9001}, -1, false, false, + true, false, false, &true_data)); + EXPECT_EQ(FilesPerLevel(), "5,0,0,1"); + + // Hold compaction from finishing + TEST_SYNC_POINT("ExternalSSTFileTest::PickedLevelDynamic:2"); + + // Output of the compaction will go to L3 + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + EXPECT_EQ(FilesPerLevel(), "1,0,0,2"); + + Close(); + options.disable_auto_compactions = true; + Reopen(options); + + ASSERT_OK(GenerateAndAddExternalFile(options, {1, 15, 19}, -1, false, false, + true, false, false, &true_data)); + ASSERT_EQ(FilesPerLevel(), "1,0,0,3"); + + ASSERT_OK(GenerateAndAddExternalFile(options, {1000, 1001, 1002}, -1, false, + false, true, false, false, &true_data)); + ASSERT_EQ(FilesPerLevel(), "1,0,0,4"); + + ASSERT_OK(GenerateAndAddExternalFile(options, {500, 600, 700}, -1, false, + false, true, false, false, &true_data)); + ASSERT_EQ(FilesPerLevel(), "1,0,0,5"); + + // File 5 overlaps with file 2 (L3 / base level) + ASSERT_OK(GenerateAndAddExternalFile(options, {2, 10}, -1, false, false, true, + false, false, &true_data)); + ASSERT_EQ(FilesPerLevel(), "2,0,0,5"); + + // File 6 overlaps with file 2 (L3 / base level) and file 5 (L0) + ASSERT_OK(GenerateAndAddExternalFile(options, {3, 9}, -1, false, false, true, + false, false, &true_data)); + ASSERT_EQ(FilesPerLevel(), "3,0,0,5"); + + // Verify data in files + size_t kcnt = 0; + VerifyDBFromMap(true_data, &kcnt, false); + + // Write range [5 => 10] to L0 + for (int i = 5; i <= 10; i++) { + std::string k = Key(i); + std::string v = k + "put"; + ASSERT_OK(Put(k, v)); + true_data[k] = v; + } + ASSERT_OK(Flush()); + ASSERT_EQ(FilesPerLevel(), "4,0,0,5"); + + // File 7 overlaps with file 4 (L3) + ASSERT_OK(GenerateAndAddExternalFile(options, {650, 651, 652}, -1, false, + false, true, false, false, &true_data)); + ASSERT_EQ(FilesPerLevel(), "5,0,0,5"); + + VerifyDBFromMap(true_data, &kcnt, false); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(ExternalSSTFileTest, AddExternalSstFileWithCustomCompartor) { + Options options = CurrentOptions(); + options.comparator = ReverseBytewiseComparator(); + DestroyAndReopen(options); + + SstFileWriter sst_file_writer(EnvOptions(), options); + + // Generate files with these key ranges + // {14 -> 0} + // {24 -> 10} + // {34 -> 20} + // {44 -> 30} + // .. + std::vector generated_files; + for (int i = 0; i < 10; i++) { + std::string file_name = sst_files_dir_ + env_->GenerateUniqueId(); + ASSERT_OK(sst_file_writer.Open(file_name)); + + int range_end = i * 10; + int range_start = range_end + 15; + for (int k = (range_start - 1); k >= range_end; k--) { + ASSERT_OK(sst_file_writer.Put(Key(k), Key(k))); + } + ExternalSstFileInfo file_info; + ASSERT_OK(sst_file_writer.Finish(&file_info)); + generated_files.push_back(file_name); + } + + std::vector in_files; + + // These 2nd and 3rd files overlap with each other + in_files = {generated_files[0], generated_files[4], generated_files[5], + generated_files[7]}; + ASSERT_NOK(DeprecatedAddFile(in_files)); + + // These 2 files don't overlap with each other + in_files = {generated_files[0], generated_files[2]}; + ASSERT_OK(DeprecatedAddFile(in_files)); + + // These 2 files don't overlap with each other but overlap with keys in DB + in_files = {generated_files[3], generated_files[7]}; + ASSERT_NOK(DeprecatedAddFile(in_files)); + + // Files don't overlap and don't overlap with DB key range + in_files = {generated_files[4], generated_files[6], generated_files[8]}; + ASSERT_OK(DeprecatedAddFile(in_files)); + + for (int i = 0; i < 100; i++) { + if (i % 20 <= 14) { + ASSERT_EQ(Get(Key(i)), Key(i)); + } else { + ASSERT_EQ(Get(Key(i)), "NOT_FOUND"); + } + } +} + +TEST_F(ExternalSSTFileTest, AddFileTrivialMoveBug) { + Options options = CurrentOptions(); + options.num_levels = 3; + options.IncreaseParallelism(20); + DestroyAndReopen(options); + + ASSERT_OK(GenerateAndAddExternalFile(options, {1, 4}, 1)); // L3 + ASSERT_OK(GenerateAndAddExternalFile(options, {2, 3}, 2)); // L2 + + ASSERT_OK(GenerateAndAddExternalFile(options, {10, 14}, 3)); // L3 + ASSERT_OK(GenerateAndAddExternalFile(options, {12, 13}, 4)); // L2 + + ASSERT_OK(GenerateAndAddExternalFile(options, {20, 24}, 5)); // L3 + ASSERT_OK(GenerateAndAddExternalFile(options, {22, 23}, 6)); // L2 + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::Run():Start", [&](void* /*arg*/) { + // Fit in L3 but will overlap with the compaction output so will be + // added to L2. Prior to the fix, a compaction will then trivially move + // this file to L3 and break LSM consistency + static std::atomic called = {false}; + if (!called) { + called = true; + ASSERT_OK(dbfull()->SetOptions({{"max_bytes_for_level_base", "1"}})); + ASSERT_OK(GenerateAndAddExternalFile(options, {15, 16}, 7, + true /* allow_global_seqno */)); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + CompactRangeOptions cro; + cro.exclusive_manual_compaction = false; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(ExternalSSTFileTest, CompactAddedFiles) { + Options options = CurrentOptions(); + options.num_levels = 3; + DestroyAndReopen(options); + + ASSERT_OK(GenerateAndAddExternalFile(options, {1, 10}, 1)); // L3 + ASSERT_OK(GenerateAndAddExternalFile(options, {2, 9}, 2)); // L2 + ASSERT_OK(GenerateAndAddExternalFile(options, {3, 8}, 3)); // L1 + ASSERT_OK(GenerateAndAddExternalFile(options, {4, 7}, 4)); // L0 + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); +} + +TEST_F(ExternalSSTFileTest, SstFileWriterNonSharedKeys) { + Options options = CurrentOptions(); + DestroyAndReopen(options); + std::string file_path = sst_files_dir_ + "/not_shared"; + SstFileWriter sst_file_writer(EnvOptions(), options); + + std::string suffix(100, 'X'); + ASSERT_OK(sst_file_writer.Open(file_path)); + ASSERT_OK(sst_file_writer.Put("A" + suffix, "VAL")); + ASSERT_OK(sst_file_writer.Put("BB" + suffix, "VAL")); + ASSERT_OK(sst_file_writer.Put("CC" + suffix, "VAL")); + ASSERT_OK(sst_file_writer.Put("CXD" + suffix, "VAL")); + ASSERT_OK(sst_file_writer.Put("CZZZ" + suffix, "VAL")); + ASSERT_OK(sst_file_writer.Put("ZAAAX" + suffix, "VAL")); + + ASSERT_OK(sst_file_writer.Finish()); + ASSERT_OK(DeprecatedAddFile({file_path})); +} + +TEST_F(ExternalSSTFileTest, WithUnorderedWrite) { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::WriteImpl:UnorderedWriteAfterWriteWAL", + "ExternalSSTFileTest::WithUnorderedWrite:WaitWriteWAL"}, + {"DBImpl::WaitForPendingWrites:BeforeBlock", + "DBImpl::WriteImpl:BeforeUnorderedWriteMemtable"}}); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::IngestExternalFile:NeedFlush", [&](void* need_flush) { + ASSERT_TRUE(*reinterpret_cast(need_flush)); + }); + + Options options = CurrentOptions(); + options.unordered_write = true; + DestroyAndReopen(options); + ASSERT_OK(Put("foo", "v1")); + SyncPoint::GetInstance()->EnableProcessing(); + port::Thread writer([&]() { ASSERT_OK(Put("bar", "v2")); }); + + TEST_SYNC_POINT("ExternalSSTFileTest::WithUnorderedWrite:WaitWriteWAL"); + ASSERT_OK(GenerateAndAddExternalFile(options, {{"bar", "v3"}}, -1, + true /* allow_global_seqno */)); + ASSERT_EQ(Get("bar"), "v3"); + + writer.join(); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +#if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) +TEST_P(ExternalSSTFileTest, IngestFileWithGlobalSeqnoRandomized) { + env_->skip_fsync_ = true; + Options options = CurrentOptions(); + options.IncreaseParallelism(20); + options.level0_slowdown_writes_trigger = 256; + options.level0_stop_writes_trigger = 256; + + bool write_global_seqno = std::get<0>(GetParam()); + bool verify_checksums_before_ingest = std::get<1>(GetParam()); + for (int iter = 0; iter < 2; iter++) { + bool write_to_memtable = (iter == 0); + DestroyAndReopen(options); + + Random rnd(301); + std::map true_data; + for (int i = 0; i < 500; i++) { + std::vector> random_data; + for (int j = 0; j < 100; j++) { + std::string k = rnd.RandomString(rnd.Next() % 20); + std::string v = rnd.RandomString(rnd.Next() % 50); + random_data.emplace_back(k, v); + } + + if (write_to_memtable && rnd.OneIn(4)) { + // 25% of writes go through memtable + for (auto& entry : random_data) { + ASSERT_OK(Put(entry.first, entry.second)); + true_data[entry.first] = entry.second; + } + } else { + ASSERT_OK(GenerateAndAddExternalFile( + options, random_data, -1, true, write_global_seqno, + verify_checksums_before_ingest, false, true, &true_data)); + } + } + size_t kcnt = 0; + VerifyDBFromMap(true_data, &kcnt, false); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + VerifyDBFromMap(true_data, &kcnt, false); + } +} +#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) + +TEST_P(ExternalSSTFileTest, IngestFileWithGlobalSeqnoAssignedLevel) { + Options options = CurrentOptions(); + options.num_levels = 5; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + std::vector> file_data; + std::map true_data; + + // Insert 100 -> 200 into the memtable + for (int i = 100; i <= 200; i++) { + ASSERT_OK(Put(Key(i), "memtable")); + true_data[Key(i)] = "memtable"; + } + + // Insert 0 -> 20 using AddFile + file_data.clear(); + for (int i = 0; i <= 20; i++) { + file_data.emplace_back(Key(i), "L4"); + } + bool write_global_seqno = std::get<0>(GetParam()); + bool verify_checksums_before_ingest = std::get<1>(GetParam()); + ASSERT_OK(GenerateAndAddExternalFile( + options, file_data, -1, true, write_global_seqno, + verify_checksums_before_ingest, false, false, &true_data)); + + // This file don't overlap with anything in the DB, will go to L4 + ASSERT_EQ("0,0,0,0,1", FilesPerLevel()); + + // Insert 80 -> 130 using AddFile + file_data.clear(); + for (int i = 80; i <= 130; i++) { + file_data.emplace_back(Key(i), "L0"); + } + ASSERT_OK(GenerateAndAddExternalFile( + options, file_data, -1, true, write_global_seqno, + verify_checksums_before_ingest, false, false, &true_data)); + + // This file overlap with the memtable, so it will flush it and add + // it self to L0 + ASSERT_EQ("2,0,0,0,1", FilesPerLevel()); + + // Insert 30 -> 50 using AddFile + file_data.clear(); + for (int i = 30; i <= 50; i++) { + file_data.emplace_back(Key(i), "L4"); + } + ASSERT_OK(GenerateAndAddExternalFile( + options, file_data, -1, true, write_global_seqno, + verify_checksums_before_ingest, false, false, &true_data)); + + // This file don't overlap with anything in the DB and fit in L4 as well + ASSERT_EQ("2,0,0,0,2", FilesPerLevel()); + + // Insert 10 -> 40 using AddFile + file_data.clear(); + for (int i = 10; i <= 40; i++) { + file_data.emplace_back(Key(i), "L3"); + } + ASSERT_OK(GenerateAndAddExternalFile( + options, file_data, -1, true, write_global_seqno, + verify_checksums_before_ingest, false, false, &true_data)); + + // This file overlap with files in L4, we will ingest it in L3 + ASSERT_EQ("2,0,0,1,2", FilesPerLevel()); + + size_t kcnt = 0; + VerifyDBFromMap(true_data, &kcnt, false); +} + +TEST_P(ExternalSSTFileTest, IngestFileWithGlobalSeqnoMemtableFlush) { + Options options = CurrentOptions(); + DestroyAndReopen(options); + uint64_t entries_in_memtable; + std::map true_data; + + for (int k : {10, 20, 40, 80}) { + ASSERT_OK(Put(Key(k), "memtable")); + true_data[Key(k)] = "memtable"; + } + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kNumEntriesActiveMemTable, + &entries_in_memtable)); + ASSERT_GE(entries_in_memtable, 1); + + bool write_global_seqno = std::get<0>(GetParam()); + bool verify_checksums_before_ingest = std::get<1>(GetParam()); + // No need for flush + ASSERT_OK(GenerateAndAddExternalFile( + options, {90, 100, 110}, -1, true, write_global_seqno, + verify_checksums_before_ingest, false, false, &true_data)); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kNumEntriesActiveMemTable, + &entries_in_memtable)); + ASSERT_GE(entries_in_memtable, 1); + + // This file will flush the memtable + ASSERT_OK(GenerateAndAddExternalFile( + options, {19, 20, 21}, -1, true, write_global_seqno, + verify_checksums_before_ingest, false, false, &true_data)); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kNumEntriesActiveMemTable, + &entries_in_memtable)); + ASSERT_EQ(entries_in_memtable, 0); + + for (int k : {200, 201, 205, 206}) { + ASSERT_OK(Put(Key(k), "memtable")); + true_data[Key(k)] = "memtable"; + } + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kNumEntriesActiveMemTable, + &entries_in_memtable)); + ASSERT_GE(entries_in_memtable, 1); + + // No need for flush, this file keys fit between the memtable keys + ASSERT_OK(GenerateAndAddExternalFile( + options, {202, 203, 204}, -1, true, write_global_seqno, + verify_checksums_before_ingest, false, false, &true_data)); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kNumEntriesActiveMemTable, + &entries_in_memtable)); + ASSERT_GE(entries_in_memtable, 1); + + // This file will flush the memtable + ASSERT_OK(GenerateAndAddExternalFile( + options, {206, 207}, -1, true, write_global_seqno, + verify_checksums_before_ingest, false, false, &true_data)); + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kNumEntriesActiveMemTable, + &entries_in_memtable)); + ASSERT_EQ(entries_in_memtable, 0); + + size_t kcnt = 0; + VerifyDBFromMap(true_data, &kcnt, false); +} + +TEST_P(ExternalSSTFileTest, L0SortingIssue) { + Options options = CurrentOptions(); + options.num_levels = 2; + DestroyAndReopen(options); + std::map true_data; + + ASSERT_OK(Put(Key(1), "memtable")); + ASSERT_OK(Put(Key(10), "memtable")); + + bool write_global_seqno = std::get<0>(GetParam()); + bool verify_checksums_before_ingest = std::get<1>(GetParam()); + // No Flush needed, No global seqno needed, Ingest in L1 + ASSERT_OK( + GenerateAndAddExternalFile(options, {7, 8}, -1, true, write_global_seqno, + verify_checksums_before_ingest, false, false)); + // No Flush needed, but need a global seqno, Ingest in L0 + ASSERT_OK( + GenerateAndAddExternalFile(options, {7, 8}, -1, true, write_global_seqno, + verify_checksums_before_ingest, false, false)); + printf("%s\n", FilesPerLevel().c_str()); + + // Overwrite what we added using external files + ASSERT_OK(Put(Key(7), "memtable")); + ASSERT_OK(Put(Key(8), "memtable")); + + // Read values from memtable + ASSERT_EQ(Get(Key(7)), "memtable"); + ASSERT_EQ(Get(Key(8)), "memtable"); + + // Flush and read from L0 + ASSERT_OK(Flush()); + printf("%s\n", FilesPerLevel().c_str()); + ASSERT_EQ(Get(Key(7)), "memtable"); + ASSERT_EQ(Get(Key(8)), "memtable"); +} + +TEST_F(ExternalSSTFileTest, CompactionDeadlock) { + Options options = CurrentOptions(); + options.num_levels = 2; + options.level0_file_num_compaction_trigger = 4; + options.level0_slowdown_writes_trigger = 4; + options.level0_stop_writes_trigger = 4; + DestroyAndReopen(options); + + // atomic conter of currently running bg threads + std::atomic running_threads(0); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"DBImpl::DelayWrite:Wait", "ExternalSSTFileTest::DeadLock:0"}, + {"ExternalSSTFileTest::DeadLock:1", "DBImpl::AddFile:Start"}, + {"DBImpl::AddFile:MutexLock", "ExternalSSTFileTest::DeadLock:2"}, + {"ExternalSSTFileTest::DeadLock:3", "BackgroundCallCompaction:0"}, + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Start ingesting and extrnal file in the background + ROCKSDB_NAMESPACE::port::Thread bg_ingest_file([&]() { + running_threads += 1; + ASSERT_OK(GenerateAndAddExternalFile(options, {5, 6})); + running_threads -= 1; + }); + + ASSERT_OK(Put(Key(1), "memtable")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put(Key(2), "memtable")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put(Key(3), "memtable")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put(Key(4), "memtable")); + ASSERT_OK(Flush()); + + // This thread will try to insert into the memtable but since we have 4 L0 + // files this thread will be blocked and hold the writer thread + ROCKSDB_NAMESPACE::port::Thread bg_block_put([&]() { + running_threads += 1; + ASSERT_OK(Put(Key(10), "memtable")); + running_threads -= 1; + }); + + // Make sure DelayWrite is called first + TEST_SYNC_POINT("ExternalSSTFileTest::DeadLock:0"); + + // `DBImpl::AddFile:Start` will wait until we be here + TEST_SYNC_POINT("ExternalSSTFileTest::DeadLock:1"); + + // Wait for IngestExternalFile() to start and aquire mutex + TEST_SYNC_POINT("ExternalSSTFileTest::DeadLock:2"); + + // Now let compaction start + TEST_SYNC_POINT("ExternalSSTFileTest::DeadLock:3"); + + // Wait for max 5 seconds, if we did not finish all bg threads + // then we hit the deadlock bug + for (int i = 0; i < 10; i++) { + if (running_threads.load() == 0) { + break; + } + // Make sure we do a "real sleep", not a mock one. + SystemClock::Default()->SleepForMicroseconds(500000); + } + + ASSERT_EQ(running_threads.load(), 0); + + bg_ingest_file.join(); + bg_block_put.join(); +} + +TEST_F(ExternalSSTFileTest, DirtyExit) { + Options options = CurrentOptions(); + DestroyAndReopen(options); + std::string file_path = sst_files_dir_ + "/dirty_exit"; + std::unique_ptr sst_file_writer; + + // Destruct SstFileWriter without calling Finish() + sst_file_writer.reset(new SstFileWriter(EnvOptions(), options)); + ASSERT_OK(sst_file_writer->Open(file_path)); + sst_file_writer.reset(); + + // Destruct SstFileWriter with a failing Finish + sst_file_writer.reset(new SstFileWriter(EnvOptions(), options)); + ASSERT_OK(sst_file_writer->Open(file_path)); + ASSERT_NOK(sst_file_writer->Finish()); +} + +TEST_F(ExternalSSTFileTest, FileWithCFInfo) { + Options options = CurrentOptions(); + CreateAndReopenWithCF({"koko", "toto"}, options); + + SstFileWriter sfw_default(EnvOptions(), options, handles_[0]); + SstFileWriter sfw_cf1(EnvOptions(), options, handles_[1]); + SstFileWriter sfw_cf2(EnvOptions(), options, handles_[2]); + SstFileWriter sfw_unknown(EnvOptions(), options); + + // default_cf.sst + const std::string cf_default_sst = sst_files_dir_ + "/default_cf.sst"; + ASSERT_OK(sfw_default.Open(cf_default_sst)); + ASSERT_OK(sfw_default.Put("K1", "V1")); + ASSERT_OK(sfw_default.Put("K2", "V2")); + ASSERT_OK(sfw_default.Finish()); + + // cf1.sst + const std::string cf1_sst = sst_files_dir_ + "/cf1.sst"; + ASSERT_OK(sfw_cf1.Open(cf1_sst)); + ASSERT_OK(sfw_cf1.Put("K3", "V1")); + ASSERT_OK(sfw_cf1.Put("K4", "V2")); + ASSERT_OK(sfw_cf1.Finish()); + + // cf_unknown.sst + const std::string unknown_sst = sst_files_dir_ + "/cf_unknown.sst"; + ASSERT_OK(sfw_unknown.Open(unknown_sst)); + ASSERT_OK(sfw_unknown.Put("K5", "V1")); + ASSERT_OK(sfw_unknown.Put("K6", "V2")); + ASSERT_OK(sfw_unknown.Finish()); + + IngestExternalFileOptions ifo; + + // SST CF don't match + ASSERT_NOK(db_->IngestExternalFile(handles_[0], {cf1_sst}, ifo)); + // SST CF don't match + ASSERT_NOK(db_->IngestExternalFile(handles_[2], {cf1_sst}, ifo)); + // SST CF match + ASSERT_OK(db_->IngestExternalFile(handles_[1], {cf1_sst}, ifo)); + + // SST CF don't match + ASSERT_NOK(db_->IngestExternalFile(handles_[1], {cf_default_sst}, ifo)); + // SST CF don't match + ASSERT_NOK(db_->IngestExternalFile(handles_[2], {cf_default_sst}, ifo)); + // SST CF match + ASSERT_OK(db_->IngestExternalFile(handles_[0], {cf_default_sst}, ifo)); + + // SST CF unknown + ASSERT_OK(db_->IngestExternalFile(handles_[1], {unknown_sst}, ifo)); + // SST CF unknown + ASSERT_OK(db_->IngestExternalFile(handles_[2], {unknown_sst}, ifo)); + // SST CF unknown + ASSERT_OK(db_->IngestExternalFile(handles_[0], {unknown_sst}, ifo)); + + // Cannot ingest a file into a dropped CF + ASSERT_OK(db_->DropColumnFamily(handles_[1])); + ASSERT_NOK(db_->IngestExternalFile(handles_[1], {unknown_sst}, ifo)); + + // CF was not dropped, ok to Ingest + ASSERT_OK(db_->IngestExternalFile(handles_[2], {unknown_sst}, ifo)); +} + +/* + * Test and verify the functionality of ingestion_options.move_files and + * ingestion_options.failed_move_fall_back_to_copy + */ +TEST_P(ExternSSTFileLinkFailFallbackTest, LinkFailFallBackExternalSst) { + const bool fail_link = std::get<0>(GetParam()); + const bool failed_move_fall_back_to_copy = std::get<1>(GetParam()); + fs_->set_fail_link(fail_link); + const EnvOptions env_options; + DestroyAndReopen(options_); + const int kNumKeys = 10000; + IngestExternalFileOptions ifo; + ifo.move_files = true; + ifo.failed_move_fall_back_to_copy = failed_move_fall_back_to_copy; + + std::string file_path = sst_files_dir_ + "file1.sst"; + // Create SstFileWriter for default column family + SstFileWriter sst_file_writer(env_options, options_); + ASSERT_OK(sst_file_writer.Open(file_path)); + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(sst_file_writer.Put(Key(i), Key(i) + "_value")); + } + ASSERT_OK(sst_file_writer.Finish()); + uint64_t file_size = 0; + ASSERT_OK(env_->GetFileSize(file_path, &file_size)); + + bool copyfile = false; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "ExternalSstFileIngestionJob::Prepare:CopyFile", + [&](void* /* arg */) { copyfile = true; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + const Status s = db_->IngestExternalFile({file_path}, ifo); + + ColumnFamilyHandleImpl* cfh = + static_cast(dbfull()->DefaultColumnFamily()); + ColumnFamilyData* cfd = cfh->cfd(); + const InternalStats* internal_stats_ptr = cfd->internal_stats(); + const std::vector& comp_stats = + internal_stats_ptr->TEST_GetCompactionStats(); + uint64_t bytes_copied = 0; + uint64_t bytes_moved = 0; + for (const auto& stats : comp_stats) { + bytes_copied += stats.bytes_written; + bytes_moved += stats.bytes_moved; + } + + if (!fail_link) { + // Link operation succeeds. External SST should be moved. + ASSERT_OK(s); + ASSERT_EQ(0, bytes_copied); + ASSERT_EQ(file_size, bytes_moved); + ASSERT_FALSE(copyfile); + } else { + // Link operation fails. + ASSERT_EQ(0, bytes_moved); + if (failed_move_fall_back_to_copy) { + ASSERT_OK(s); + // Copy file is true since a failed link falls back to copy file. + ASSERT_TRUE(copyfile); + ASSERT_EQ(file_size, bytes_copied); + } else { + ASSERT_TRUE(s.IsNotSupported()); + // Copy file is false since a failed link does not fall back to copy file. + ASSERT_FALSE(copyfile); + ASSERT_EQ(0, bytes_copied); + } + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +class TestIngestExternalFileListener : public EventListener { + public: + void OnExternalFileIngested(DB* /*db*/, + const ExternalFileIngestionInfo& info) override { + ingested_files.push_back(info); + } + + std::vector ingested_files; +}; + +TEST_P(ExternalSSTFileTest, IngestionListener) { + Options options = CurrentOptions(); + TestIngestExternalFileListener* listener = + new TestIngestExternalFileListener(); + options.listeners.emplace_back(listener); + CreateAndReopenWithCF({"koko", "toto"}, options); + + bool write_global_seqno = std::get<0>(GetParam()); + bool verify_checksums_before_ingest = std::get<1>(GetParam()); + // Ingest into default cf + ASSERT_OK(GenerateAndAddExternalFile( + options, {1, 2}, -1, true, write_global_seqno, + verify_checksums_before_ingest, false, true, nullptr, handles_[0])); + ASSERT_EQ(listener->ingested_files.size(), 1); + ASSERT_EQ(listener->ingested_files.back().cf_name, "default"); + ASSERT_EQ(listener->ingested_files.back().global_seqno, 0); + ASSERT_EQ(listener->ingested_files.back().table_properties.column_family_id, + 0); + ASSERT_EQ(listener->ingested_files.back().table_properties.column_family_name, + "default"); + + // Ingest into cf1 + ASSERT_OK(GenerateAndAddExternalFile( + options, {1, 2}, -1, true, write_global_seqno, + verify_checksums_before_ingest, false, true, nullptr, handles_[1])); + ASSERT_EQ(listener->ingested_files.size(), 2); + ASSERT_EQ(listener->ingested_files.back().cf_name, "koko"); + ASSERT_EQ(listener->ingested_files.back().global_seqno, 0); + ASSERT_EQ(listener->ingested_files.back().table_properties.column_family_id, + 1); + ASSERT_EQ(listener->ingested_files.back().table_properties.column_family_name, + "koko"); + + // Ingest into cf2 + ASSERT_OK(GenerateAndAddExternalFile( + options, {1, 2}, -1, true, write_global_seqno, + verify_checksums_before_ingest, false, true, nullptr, handles_[2])); + ASSERT_EQ(listener->ingested_files.size(), 3); + ASSERT_EQ(listener->ingested_files.back().cf_name, "toto"); + ASSERT_EQ(listener->ingested_files.back().global_seqno, 0); + ASSERT_EQ(listener->ingested_files.back().table_properties.column_family_id, + 2); + ASSERT_EQ(listener->ingested_files.back().table_properties.column_family_name, + "toto"); +} + +TEST_F(ExternalSSTFileTest, SnapshotInconsistencyBug) { + Options options = CurrentOptions(); + DestroyAndReopen(options); + const int kNumKeys = 10000; + + // Insert keys using normal path and take a snapshot + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(Put(Key(i), Key(i) + "_V1")); + } + const Snapshot* snap = db_->GetSnapshot(); + + // Overwrite all keys using IngestExternalFile + std::string sst_file_path = sst_files_dir_ + "file1.sst"; + SstFileWriter sst_file_writer(EnvOptions(), options); + ASSERT_OK(sst_file_writer.Open(sst_file_path)); + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(sst_file_writer.Put(Key(i), Key(i) + "_V2")); + } + ASSERT_OK(sst_file_writer.Finish()); + + IngestExternalFileOptions ifo; + ifo.move_files = true; + ASSERT_OK(db_->IngestExternalFile({sst_file_path}, ifo)); + + for (int i = 0; i < kNumKeys; i++) { + ASSERT_EQ(Get(Key(i), snap), Key(i) + "_V1"); + ASSERT_EQ(Get(Key(i)), Key(i) + "_V2"); + } + + db_->ReleaseSnapshot(snap); +} + +TEST_P(ExternalSSTFileTest, IngestBehind) { + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = 3; + options.disable_auto_compactions = false; + DestroyAndReopen(options); + std::vector> file_data; + std::map true_data; + + // Insert 100 -> 200 into the memtable + for (int i = 100; i <= 200; i++) { + ASSERT_OK(Put(Key(i), "memtable")); + } + + // Insert 100 -> 200 using IngestExternalFile + file_data.clear(); + for (int i = 0; i <= 20; i++) { + file_data.emplace_back(Key(i), "ingest_behind"); + true_data[Key(i)] = "ingest_behind"; + } + + bool allow_global_seqno = true; + bool ingest_behind = true; + bool write_global_seqno = std::get<0>(GetParam()); + bool verify_checksums_before_ingest = std::get<1>(GetParam()); + + // Can't ingest behind since allow_ingest_behind isn't set to true + ASSERT_NOK(GenerateAndAddExternalFile( + options, file_data, -1, allow_global_seqno, write_global_seqno, + verify_checksums_before_ingest, ingest_behind, false /*sort_data*/, + &true_data)); + + options.allow_ingest_behind = true; + // check that we still can open the DB, as num_levels should be + // sanitized to 3 + options.num_levels = 2; + DestroyAndReopen(options); + + options.num_levels = 3; + DestroyAndReopen(options); + true_data.clear(); + // Insert 100 -> 200 into the memtable + for (int i = 100; i <= 200; i++) { + ASSERT_OK(Put(Key(i), "memtable")); + true_data[Key(i)] = "memtable"; + } + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + // Universal picker should go at second from the bottom level + ASSERT_EQ("0,1", FilesPerLevel()); + ASSERT_OK(GenerateAndAddExternalFile( + options, file_data, -1, allow_global_seqno, write_global_seqno, + verify_checksums_before_ingest, true /*ingest_behind*/, + false /*sort_data*/, &true_data)); + ASSERT_EQ("0,1,1", FilesPerLevel()); + // this time ingest should fail as the file doesn't fit to the bottom level + ASSERT_NOK(GenerateAndAddExternalFile( + options, file_data, -1, allow_global_seqno, write_global_seqno, + verify_checksums_before_ingest, true /*ingest_behind*/, + false /*sort_data*/, &true_data)); + ASSERT_EQ("0,1,1", FilesPerLevel()); + std::vector> level_to_files; + dbfull()->TEST_GetFilesMetaData(db_->DefaultColumnFamily(), &level_to_files); + uint64_t ingested_file_number = level_to_files[2][0].fd.GetNumber(); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + // Last level should not be compacted + ASSERT_EQ("0,1,1", FilesPerLevel()); + dbfull()->TEST_GetFilesMetaData(db_->DefaultColumnFamily(), &level_to_files); + ASSERT_EQ(ingested_file_number, level_to_files[2][0].fd.GetNumber()); + size_t kcnt = 0; + VerifyDBFromMap(true_data, &kcnt, false); + + // Auto-compaction should not include the last level. + // Trigger compaction if size amplification exceeds 110%. + options.compaction_options_universal.max_size_amplification_percent = 110; + options.level0_file_num_compaction_trigger = 4; + TryReopen(options); + Random rnd(301); + for (int i = 0; i < 4; ++i) { + for (int j = 0; j < 10; j++) { + true_data[Key(j)] = rnd.RandomString(1000); + ASSERT_OK(Put(Key(j), true_data[Key(j)])); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + dbfull()->TEST_GetFilesMetaData(db_->DefaultColumnFamily(), &level_to_files); + ASSERT_EQ(1, level_to_files[2].size()); + ASSERT_EQ(ingested_file_number, level_to_files[2][0].fd.GetNumber()); + + // Turning off the option allows DB to compact ingested files. + options.allow_ingest_behind = false; + TryReopen(options); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + dbfull()->TEST_GetFilesMetaData(db_->DefaultColumnFamily(), &level_to_files); + ASSERT_EQ(1, level_to_files[2].size()); + ASSERT_NE(ingested_file_number, level_to_files[2][0].fd.GetNumber()); + VerifyDBFromMap(true_data, &kcnt, false); +} + +TEST_F(ExternalSSTFileTest, SkipBloomFilter) { + Options options = CurrentOptions(); + + BlockBasedTableOptions table_options; + table_options.filter_policy.reset(NewBloomFilterPolicy(10)); + table_options.cache_index_and_filter_blocks = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + // Create external SST file and include bloom filters + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + DestroyAndReopen(options); + { + std::string file_path = sst_files_dir_ + "sst_with_bloom.sst"; + SstFileWriter sst_file_writer(EnvOptions(), options); + ASSERT_OK(sst_file_writer.Open(file_path)); + ASSERT_OK(sst_file_writer.Put("Key1", "Value1")); + ASSERT_OK(sst_file_writer.Finish()); + + ASSERT_OK( + db_->IngestExternalFile({file_path}, IngestExternalFileOptions())); + + ASSERT_EQ(Get("Key1"), "Value1"); + ASSERT_GE( + options.statistics->getTickerCount(Tickers::BLOCK_CACHE_FILTER_ADD), 1); + } + + // Create external SST file but skip bloom filters + options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + DestroyAndReopen(options); + { + std::string file_path = sst_files_dir_ + "sst_with_no_bloom.sst"; + SstFileWriter sst_file_writer(EnvOptions(), options, nullptr, true, + Env::IOPriority::IO_TOTAL, + true /* skip_filters */); + ASSERT_OK(sst_file_writer.Open(file_path)); + ASSERT_OK(sst_file_writer.Put("Key1", "Value1")); + ASSERT_OK(sst_file_writer.Finish()); + + ASSERT_OK( + db_->IngestExternalFile({file_path}, IngestExternalFileOptions())); + + ASSERT_EQ(Get("Key1"), "Value1"); + ASSERT_EQ( + options.statistics->getTickerCount(Tickers::BLOCK_CACHE_FILTER_ADD), 0); + } +} + +TEST_F(ExternalSSTFileTest, IngestFileWrittenWithCompressionDictionary) { + if (!ZSTD_Supported()) { + return; + } + const int kNumEntries = 1 << 10; + const int kNumBytesPerEntry = 1 << 10; + Options options = CurrentOptions(); + options.compression = kZSTD; + options.compression_opts.max_dict_bytes = 1 << 14; // 16KB + options.compression_opts.zstd_max_train_bytes = 1 << 18; // 256KB + DestroyAndReopen(options); + + std::atomic num_compression_dicts(0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTableBuilder::WriteCompressionDictBlock:RawDict", + [&](void* /* arg */) { ++num_compression_dicts; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + Random rnd(301); + std::vector> random_data; + for (int i = 0; i < kNumEntries; i++) { + std::string val = rnd.RandomString(kNumBytesPerEntry); + random_data.emplace_back(Key(i), std::move(val)); + } + ASSERT_OK(GenerateAndAddExternalFile(options, std::move(random_data))); + ASSERT_EQ(1, num_compression_dicts); +} + +class ExternalSSTBlockChecksumTest + : public ExternalSSTFileTestBase, + public testing::WithParamInterface {}; + +INSTANTIATE_TEST_CASE_P(FormatVersions, ExternalSSTBlockChecksumTest, + testing::ValuesIn(test::kFooterFormatVersionsToTest)); + +// Very slow, not worth the cost to run regularly +TEST_P(ExternalSSTBlockChecksumTest, DISABLED_HugeBlockChecksum) { + BlockBasedTableOptions table_options; + table_options.format_version = GetParam(); + for (auto t : GetSupportedChecksums()) { + table_options.checksum = t; + Options options = CurrentOptions(); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + SstFileWriter sst_file_writer(EnvOptions(), options); + + // 2^32 - 1, will lead to data block with more than 2^32 bytes + size_t huge_size = std::numeric_limits::max(); + + std::string f = sst_files_dir_ + "f.sst"; + ASSERT_OK(sst_file_writer.Open(f)); + { + Random64 r(123); + std::string huge(huge_size, 0); + for (size_t j = 0; j + 7 < huge_size; j += 8) { + EncodeFixed64(&huge[j], r.Next()); + } + ASSERT_OK(sst_file_writer.Put("Huge", huge)); + } + + ExternalSstFileInfo f_info; + ASSERT_OK(sst_file_writer.Finish(&f_info)); + ASSERT_GT(f_info.file_size, uint64_t{huge_size} + 10); + + SstFileReader sst_file_reader(options); + ASSERT_OK(sst_file_reader.Open(f)); + ASSERT_OK(sst_file_reader.VerifyChecksum()); + } +} + +TEST_P(ExternalSSTFileTest, IngestFilesIntoMultipleColumnFamilies_Success) { + std::unique_ptr fault_injection_env( + new FaultInjectionTestEnv(env_)); + Options options = CurrentOptions(); + options.env = fault_injection_env.get(); + CreateAndReopenWithCF({"pikachu", "eevee"}, options); + + // Exercise different situations in different column families: two are empty + // (so no new sequence number is needed), but at least one overlaps with the + // DB and needs to bump the sequence number. + ASSERT_OK(db_->Put(WriteOptions(), "foo1", "oldvalue")); + + std::vector column_families; + column_families.push_back(handles_[0]); + column_families.push_back(handles_[1]); + column_families.push_back(handles_[2]); + std::vector ifos(column_families.size()); + for (auto& ifo : ifos) { + ifo.allow_global_seqno = true; // Always allow global_seqno + // May or may not write global_seqno + ifo.write_global_seqno = std::get<0>(GetParam()); + // Whether to verify checksums before ingestion + ifo.verify_checksums_before_ingest = std::get<1>(GetParam()); + } + std::vector>> data; + data.push_back( + {std::make_pair("foo1", "fv1"), std::make_pair("foo2", "fv2")}); + data.push_back( + {std::make_pair("bar1", "bv1"), std::make_pair("bar2", "bv2")}); + data.push_back( + {std::make_pair("bar3", "bv3"), std::make_pair("bar4", "bv4")}); + + // Resize the true_data vector upon construction to avoid re-alloc + std::vector> true_data( + column_families.size()); + ASSERT_OK(GenerateAndAddExternalFiles(options, column_families, ifos, data, + -1, true, true_data)); + Close(); + ReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu", "eevee"}, + options); + ASSERT_EQ(3, handles_.size()); + int cf = 0; + for (const auto& verify_map : true_data) { + for (const auto& elem : verify_map) { + const std::string& key = elem.first; + const std::string& value = elem.second; + ASSERT_EQ(value, Get(cf, key)); + } + ++cf; + } + Close(); + Destroy(options, true /* delete_cf_paths */); +} + +TEST_P(ExternalSSTFileTest, + IngestFilesIntoMultipleColumnFamilies_NoMixedStateWithSnapshot) { + std::unique_ptr fault_injection_env( + new FaultInjectionTestEnv(env_)); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->LoadDependency({ + {"DBImpl::IngestExternalFiles:InstallSVForFirstCF:0", + "ExternalSSTFileTest::IngestFilesIntoMultipleColumnFamilies_MixedState:" + "BeforeRead"}, + {"ExternalSSTFileTest::IngestFilesIntoMultipleColumnFamilies_MixedState:" + "AfterRead", + "DBImpl::IngestExternalFiles:InstallSVForFirstCF:1"}, + }); + SyncPoint::GetInstance()->EnableProcessing(); + + Options options = CurrentOptions(); + options.env = fault_injection_env.get(); + CreateAndReopenWithCF({"pikachu", "eevee"}, options); + const std::vector> data_before_ingestion = + {{{"foo1", "fv1_0"}, {"foo2", "fv2_0"}, {"foo3", "fv3_0"}}, + {{"bar1", "bv1_0"}, {"bar2", "bv2_0"}, {"bar3", "bv3_0"}}, + {{"bar4", "bv4_0"}, {"bar5", "bv5_0"}, {"bar6", "bv6_0"}}}; + for (size_t i = 0; i != handles_.size(); ++i) { + int cf = static_cast(i); + const auto& orig_data = data_before_ingestion[i]; + for (const auto& kv : orig_data) { + ASSERT_OK(Put(cf, kv.first, kv.second)); + } + ASSERT_OK(Flush(cf)); + } + + std::vector column_families; + column_families.push_back(handles_[0]); + column_families.push_back(handles_[1]); + column_families.push_back(handles_[2]); + std::vector ifos(column_families.size()); + for (auto& ifo : ifos) { + ifo.allow_global_seqno = true; // Always allow global_seqno + // May or may not write global_seqno + ifo.write_global_seqno = std::get<0>(GetParam()); + // Whether to verify checksums before ingestion + ifo.verify_checksums_before_ingest = std::get<1>(GetParam()); + } + std::vector>> data; + data.push_back( + {std::make_pair("foo1", "fv1"), std::make_pair("foo2", "fv2")}); + data.push_back( + {std::make_pair("bar1", "bv1"), std::make_pair("bar2", "bv2")}); + data.push_back( + {std::make_pair("bar3", "bv3"), std::make_pair("bar4", "bv4")}); + // Resize the true_data vector upon construction to avoid re-alloc + std::vector> true_data( + column_families.size()); + // Take snapshot before ingestion starts + ReadOptions read_opts; + read_opts.total_order_seek = true; + read_opts.snapshot = dbfull()->GetSnapshot(); + std::vector iters(handles_.size()); + + // Range scan checks first kv of each CF before ingestion starts. + for (size_t i = 0; i != handles_.size(); ++i) { + iters[i] = dbfull()->NewIterator(read_opts, handles_[i]); + iters[i]->SeekToFirst(); + ASSERT_TRUE(iters[i]->Valid()); + const std::string& key = iters[i]->key().ToString(); + const std::string& value = iters[i]->value().ToString(); + const std::map& orig_data = + data_before_ingestion[i]; + std::map::const_iterator it = orig_data.find(key); + ASSERT_NE(orig_data.end(), it); + ASSERT_EQ(it->second, value); + iters[i]->Next(); + } + port::Thread ingest_thread([&]() { + ASSERT_OK(GenerateAndAddExternalFiles(options, column_families, ifos, data, + -1, true, true_data)); + }); + TEST_SYNC_POINT( + "ExternalSSTFileTest::IngestFilesIntoMultipleColumnFamilies_MixedState:" + "BeforeRead"); + // Should see only data before ingestion + for (size_t i = 0; i != handles_.size(); ++i) { + const auto& orig_data = data_before_ingestion[i]; + for (; iters[i]->Valid(); iters[i]->Next()) { + const std::string& key = iters[i]->key().ToString(); + const std::string& value = iters[i]->value().ToString(); + std::map::const_iterator it = + orig_data.find(key); + ASSERT_NE(orig_data.end(), it); + ASSERT_EQ(it->second, value); + } + } + TEST_SYNC_POINT( + "ExternalSSTFileTest::IngestFilesIntoMultipleColumnFamilies_MixedState:" + "AfterRead"); + ingest_thread.join(); + for (auto* iter : iters) { + delete iter; + } + iters.clear(); + dbfull()->ReleaseSnapshot(read_opts.snapshot); + + Close(); + ReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu", "eevee"}, + options); + // Should see consistent state after ingestion for all column families even + // without snapshot. + ASSERT_EQ(3, handles_.size()); + int cf = 0; + for (const auto& verify_map : true_data) { + for (const auto& elem : verify_map) { + const std::string& key = elem.first; + const std::string& value = elem.second; + ASSERT_EQ(value, Get(cf, key)); + } + ++cf; + } + Close(); + Destroy(options, true /* delete_cf_paths */); +} + +TEST_P(ExternalSSTFileTest, IngestFilesIntoMultipleColumnFamilies_PrepareFail) { + std::unique_ptr fault_injection_env( + new FaultInjectionTestEnv(env_)); + Options options = CurrentOptions(); + options.env = fault_injection_env.get(); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->LoadDependency({ + {"DBImpl::IngestExternalFiles:BeforeLastJobPrepare:0", + "ExternalSSTFileTest::IngestFilesIntoMultipleColumnFamilies_PrepareFail:" + "0"}, + {"ExternalSSTFileTest::IngestFilesIntoMultipleColumnFamilies:PrepareFail:" + "1", + "DBImpl::IngestExternalFiles:BeforeLastJobPrepare:1"}, + }); + SyncPoint::GetInstance()->EnableProcessing(); + CreateAndReopenWithCF({"pikachu", "eevee"}, options); + std::vector column_families; + column_families.push_back(handles_[0]); + column_families.push_back(handles_[1]); + column_families.push_back(handles_[2]); + std::vector ifos(column_families.size()); + for (auto& ifo : ifos) { + ifo.allow_global_seqno = true; // Always allow global_seqno + // May or may not write global_seqno + ifo.write_global_seqno = std::get<0>(GetParam()); + // Whether to verify block checksums before ingest + ifo.verify_checksums_before_ingest = std::get<1>(GetParam()); + } + std::vector>> data; + data.push_back( + {std::make_pair("foo1", "fv1"), std::make_pair("foo2", "fv2")}); + data.push_back( + {std::make_pair("bar1", "bv1"), std::make_pair("bar2", "bv2")}); + data.push_back( + {std::make_pair("bar3", "bv3"), std::make_pair("bar4", "bv4")}); + + // Resize the true_data vector upon construction to avoid re-alloc + std::vector> true_data( + column_families.size()); + port::Thread ingest_thread([&]() { + ASSERT_NOK(GenerateAndAddExternalFiles(options, column_families, ifos, data, + -1, true, true_data)); + }); + TEST_SYNC_POINT( + "ExternalSSTFileTest::IngestFilesIntoMultipleColumnFamilies_PrepareFail:" + "0"); + fault_injection_env->SetFilesystemActive(false); + TEST_SYNC_POINT( + "ExternalSSTFileTest::IngestFilesIntoMultipleColumnFamilies:PrepareFail:" + "1"); + ingest_thread.join(); + + fault_injection_env->SetFilesystemActive(true); + Close(); + ReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu", "eevee"}, + options); + ASSERT_EQ(3, handles_.size()); + int cf = 0; + for (const auto& verify_map : true_data) { + for (const auto& elem : verify_map) { + const std::string& key = elem.first; + ASSERT_EQ("NOT_FOUND", Get(cf, key)); + } + ++cf; + } + Close(); + Destroy(options, true /* delete_cf_paths */); +} + +TEST_P(ExternalSSTFileTest, IngestFilesIntoMultipleColumnFamilies_CommitFail) { + std::unique_ptr fault_injection_env( + new FaultInjectionTestEnv(env_)); + Options options = CurrentOptions(); + options.env = fault_injection_env.get(); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->LoadDependency({ + {"DBImpl::IngestExternalFiles:BeforeJobsRun:0", + "ExternalSSTFileTest::IngestFilesIntoMultipleColumnFamilies_CommitFail:" + "0"}, + {"ExternalSSTFileTest::IngestFilesIntoMultipleColumnFamilies_CommitFail:" + "1", + "DBImpl::IngestExternalFiles:BeforeJobsRun:1"}, + }); + SyncPoint::GetInstance()->EnableProcessing(); + CreateAndReopenWithCF({"pikachu", "eevee"}, options); + std::vector column_families; + column_families.push_back(handles_[0]); + column_families.push_back(handles_[1]); + column_families.push_back(handles_[2]); + std::vector ifos(column_families.size()); + for (auto& ifo : ifos) { + ifo.allow_global_seqno = true; // Always allow global_seqno + // May or may not write global_seqno + ifo.write_global_seqno = std::get<0>(GetParam()); + // Whether to verify block checksums before ingestion + ifo.verify_checksums_before_ingest = std::get<1>(GetParam()); + } + std::vector>> data; + data.push_back( + {std::make_pair("foo1", "fv1"), std::make_pair("foo2", "fv2")}); + data.push_back( + {std::make_pair("bar1", "bv1"), std::make_pair("bar2", "bv2")}); + data.push_back( + {std::make_pair("bar3", "bv3"), std::make_pair("bar4", "bv4")}); + // Resize the true_data vector upon construction to avoid re-alloc + std::vector> true_data( + column_families.size()); + port::Thread ingest_thread([&]() { + ASSERT_NOK(GenerateAndAddExternalFiles(options, column_families, ifos, data, + -1, true, true_data)); + }); + TEST_SYNC_POINT( + "ExternalSSTFileTest::IngestFilesIntoMultipleColumnFamilies_CommitFail:" + "0"); + fault_injection_env->SetFilesystemActive(false); + TEST_SYNC_POINT( + "ExternalSSTFileTest::IngestFilesIntoMultipleColumnFamilies_CommitFail:" + "1"); + ingest_thread.join(); + + fault_injection_env->SetFilesystemActive(true); + Close(); + ReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu", "eevee"}, + options); + ASSERT_EQ(3, handles_.size()); + int cf = 0; + for (const auto& verify_map : true_data) { + for (const auto& elem : verify_map) { + const std::string& key = elem.first; + ASSERT_EQ("NOT_FOUND", Get(cf, key)); + } + ++cf; + } + Close(); + Destroy(options, true /* delete_cf_paths */); +} + +TEST_P(ExternalSSTFileTest, + IngestFilesIntoMultipleColumnFamilies_PartialManifestWriteFail) { + std::unique_ptr fault_injection_env( + new FaultInjectionTestEnv(env_)); + Options options = CurrentOptions(); + options.env = fault_injection_env.get(); + + CreateAndReopenWithCF({"pikachu", "eevee"}, options); + + SyncPoint::GetInstance()->ClearTrace(); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->LoadDependency({ + {"VersionSet::ProcessManifestWrites:BeforeWriteLastVersionEdit:0", + "ExternalSSTFileTest::IngestFilesIntoMultipleColumnFamilies_" + "PartialManifestWriteFail:0"}, + {"ExternalSSTFileTest::IngestFilesIntoMultipleColumnFamilies_" + "PartialManifestWriteFail:1", + "VersionSet::ProcessManifestWrites:BeforeWriteLastVersionEdit:1"}, + }); + SyncPoint::GetInstance()->EnableProcessing(); + + std::vector column_families; + column_families.push_back(handles_[0]); + column_families.push_back(handles_[1]); + column_families.push_back(handles_[2]); + std::vector ifos(column_families.size()); + for (auto& ifo : ifos) { + ifo.allow_global_seqno = true; // Always allow global_seqno + // May or may not write global_seqno + ifo.write_global_seqno = std::get<0>(GetParam()); + // Whether to verify block checksums before ingestion + ifo.verify_checksums_before_ingest = std::get<1>(GetParam()); + } + std::vector>> data; + data.push_back( + {std::make_pair("foo1", "fv1"), std::make_pair("foo2", "fv2")}); + data.push_back( + {std::make_pair("bar1", "bv1"), std::make_pair("bar2", "bv2")}); + data.push_back( + {std::make_pair("bar3", "bv3"), std::make_pair("bar4", "bv4")}); + // Resize the true_data vector upon construction to avoid re-alloc + std::vector> true_data( + column_families.size()); + port::Thread ingest_thread([&]() { + ASSERT_NOK(GenerateAndAddExternalFiles(options, column_families, ifos, data, + -1, true, true_data)); + }); + TEST_SYNC_POINT( + "ExternalSSTFileTest::IngestFilesIntoMultipleColumnFamilies_" + "PartialManifestWriteFail:0"); + fault_injection_env->SetFilesystemActive(false); + TEST_SYNC_POINT( + "ExternalSSTFileTest::IngestFilesIntoMultipleColumnFamilies_" + "PartialManifestWriteFail:1"); + ingest_thread.join(); + + ASSERT_OK(fault_injection_env->DropUnsyncedFileData()); + fault_injection_env->SetFilesystemActive(true); + Close(); + ReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu", "eevee"}, + options); + ASSERT_EQ(3, handles_.size()); + int cf = 0; + for (const auto& verify_map : true_data) { + for (const auto& elem : verify_map) { + const std::string& key = elem.first; + ASSERT_EQ("NOT_FOUND", Get(cf, key)); + } + ++cf; + } + Close(); + Destroy(options, true /* delete_cf_paths */); +} + +TEST_P(ExternalSSTFileTest, IngestFilesTriggerFlushingWithTwoWriteQueue) { + Options options = CurrentOptions(); + // Use large buffer to avoid memtable flush + options.write_buffer_size = 1024 * 1024; + options.two_write_queues = true; + DestroyAndReopen(options); + + ASSERT_OK(dbfull()->Put(WriteOptions(), "1000", "v1")); + ASSERT_OK(dbfull()->Put(WriteOptions(), "1001", "v1")); + ASSERT_OK(dbfull()->Put(WriteOptions(), "9999", "v1")); + + // Put one key which is overlap with keys in memtable. + // It will trigger flushing memtable and require this thread is + // currently at the front of the 2nd writer queue. We must make + // sure that it won't enter the 2nd writer queue for the second time. + std::vector> data; + data.push_back(std::make_pair("1001", "v2")); + ASSERT_OK(GenerateAndAddExternalFile(options, data, -1, true)); +} + +TEST_P(ExternalSSTFileTest, DeltaEncodingWhileGlobalSeqnoPresent) { + Options options = CurrentOptions(); + DestroyAndReopen(options); + constexpr size_t kValueSize = 8; + Random rnd(301); + std::string value = rnd.RandomString(kValueSize); + + // Write some key to make global seqno larger than zero + for (int i = 0; i < 10; i++) { + ASSERT_OK(Put("ab" + Key(i), value)); + } + // Get a Snapshot to make RocksDB assign global seqno to ingested sst files. + auto snap = dbfull()->GetSnapshot(); + + std::string fname = sst_files_dir_ + "test_file"; + ROCKSDB_NAMESPACE::SstFileWriter writer(EnvOptions(), options); + ASSERT_OK(writer.Open(fname)); + std::string key1 = "ab"; + std::string key2 = "ab"; + + // Make the prefix of key2 is same with key1 add zero seqno. The tail of every + // key is composed as (seqno << 8 | value_type), and here `1` represents + // ValueType::kTypeValue + + PutFixed64(&key2, PackSequenceAndType(0, kTypeValue)); + key2 += "cdefghijkl"; + + ASSERT_OK(writer.Put(key1, value)); + ASSERT_OK(writer.Put(key2, value)); + + ExternalSstFileInfo info; + ASSERT_OK(writer.Finish(&info)); + + ASSERT_OK(dbfull()->IngestExternalFile({info.file_path}, + IngestExternalFileOptions())); + dbfull()->ReleaseSnapshot(snap); + ASSERT_EQ(value, Get(key1)); + // You will get error here + ASSERT_EQ(value, Get(key2)); +} + +TEST_P(ExternalSSTFileTest, + DeltaEncodingWhileGlobalSeqnoPresentIteratorSwitch) { + // Regression test for bug where global seqno corrupted the shared bytes + // buffer when switching from reverse iteration to forward iteration. + constexpr size_t kValueSize = 8; + Options options = CurrentOptions(); + + Random rnd(301); + std::string value = rnd.RandomString(kValueSize); + + std::string key0 = "aa"; + std::string key1 = "ab"; + // Make the prefix of key2 is same with key1 add zero seqno. The tail of every + // key is composed as (seqno << 8 | value_type), and here `1` represents + // ValueType::kTypeValue + std::string key2 = "ab"; + PutFixed64(&key2, PackSequenceAndType(0, kTypeValue)); + key2 += "cdefghijkl"; + std::string key3 = key2 + "_"; + + // Write some key to make global seqno larger than zero + ASSERT_OK(Put(key0, value)); + + std::string fname = sst_files_dir_ + "test_file"; + ROCKSDB_NAMESPACE::SstFileWriter writer(EnvOptions(), options); + ASSERT_OK(writer.Open(fname)); + + // key0 is a dummy to ensure the turnaround point (key1) comes from Prev + // cache rather than block (restart keys are pinned in block). + ASSERT_OK(writer.Put(key0, value)); + ASSERT_OK(writer.Put(key1, value)); + ASSERT_OK(writer.Put(key2, value)); + ASSERT_OK(writer.Put(key3, value)); + + ExternalSstFileInfo info; + ASSERT_OK(writer.Finish(&info)); + + ASSERT_OK(dbfull()->IngestExternalFile({info.file_path}, + IngestExternalFileOptions())); + ReadOptions read_opts; + // Prevents Seek() when switching directions, which circumvents the bug. + read_opts.total_order_seek = true; + Iterator* iter = db_->NewIterator(read_opts); + // Scan backwards to key2. File iterator will then be positioned at key1. + iter->Seek(key3); + ASSERT_EQ(key3, iter->key()); + iter->Prev(); + ASSERT_EQ(key2, iter->key()); + // Scan forwards and make sure key3 is present. Previously key3 would be + // corrupted by the global seqno from key1. + iter->Next(); + ASSERT_EQ(key3, iter->key()); + delete iter; +} + +INSTANTIATE_TEST_CASE_P(ExternalSSTFileTest, ExternalSSTFileTest, + testing::Values(std::make_tuple(false, false), + std::make_tuple(false, true), + std::make_tuple(true, false), + std::make_tuple(true, true))); + +INSTANTIATE_TEST_CASE_P(ExternSSTFileLinkFailFallbackTest, + ExternSSTFileLinkFailFallbackTest, + testing::Values(std::make_tuple(true, false), + std::make_tuple(true, true), + std::make_tuple(false, false))); + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/fault_injection_test.cc b/librocksdb-sys/rocksdb/db/fault_injection_test.cc new file mode 100644 index 0000000..ddd4b47 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/fault_injection_test.cc @@ -0,0 +1,637 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright 2014 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +// This test uses a custom Env to keep track of the state of a filesystem as of +// the last "sync". It then checks for data loss errors by purposely dropping +// file data (or entire files) not protected by a "sync". + +#include "db/db_impl/db_impl.h" +#include "db/log_format.h" +#include "db/version_set.h" +#include "env/mock_env.h" +#include "file/filename.h" +#include "rocksdb/cache.h" +#include "rocksdb/convenience.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/table.h" +#include "rocksdb/write_batch.h" +#include "test_util/sync_point.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/mutexlock.h" +#include "util/random.h" +#include "utilities/fault_injection_env.h" +#ifndef NDEBUG +#include "utilities/fault_injection_fs.h" +#endif + +namespace ROCKSDB_NAMESPACE { + +static const int kValueSize = 1000; +static const int kMaxNumValues = 2000; +static const size_t kNumIterations = 3; + +enum FaultInjectionOptionConfig { + kDefault, + kDifferentDataDir, + kWalDir, + kSyncWal, + kWalDirSyncWal, + kMultiLevels, + kEnd, +}; +class FaultInjectionTest + : public testing::Test, + public testing::WithParamInterface> { + protected: + int option_config_; + int non_inclusive_end_range_; // kEnd or equivalent to that + // When need to make sure data is persistent, sync WAL + bool sync_use_wal_; + // When need to make sure data is persistent, call DB::CompactRange() + bool sync_use_compact_; + + bool sequential_order_; + + public: + enum ExpectedVerifResult { kValExpectFound, kValExpectNoError }; + enum ResetMethod { + kResetDropUnsyncedData, + kResetDropRandomUnsyncedData, + kResetDeleteUnsyncedFiles, + kResetDropAndDeleteUnsynced + }; + + std::unique_ptr base_env_; + FaultInjectionTestEnv* env_; + std::string dbname_; + std::shared_ptr tiny_cache_; + Options options_; + DB* db_; + + FaultInjectionTest() + : option_config_(std::get<1>(GetParam())), + non_inclusive_end_range_(std::get<2>(GetParam())), + sync_use_wal_(false), + sync_use_compact_(true), + base_env_(nullptr), + env_(nullptr), + db_(nullptr) { + EXPECT_OK( + test::CreateEnvFromSystem(ConfigOptions(), &system_env_, &env_guard_)); + EXPECT_NE(system_env_, nullptr); + } + + ~FaultInjectionTest() override { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + } + + bool ChangeOptions() { + option_config_++; + if (option_config_ >= non_inclusive_end_range_) { + return false; + } else { + if (option_config_ == kMultiLevels) { + base_env_.reset(MockEnv::Create(system_env_)); + } + return true; + } + } + + // Return the current option configuration. + Options CurrentOptions() { + sync_use_wal_ = false; + sync_use_compact_ = true; + Options options; + switch (option_config_) { + case kWalDir: + options.wal_dir = test::PerThreadDBPath(env_, "fault_test_wal"); + break; + case kDifferentDataDir: + options.db_paths.emplace_back( + test::PerThreadDBPath(env_, "fault_test_data"), 1000000U); + break; + case kSyncWal: + sync_use_wal_ = true; + sync_use_compact_ = false; + break; + case kWalDirSyncWal: + options.wal_dir = test::PerThreadDBPath(env_, "/fault_test_wal"); + sync_use_wal_ = true; + sync_use_compact_ = false; + break; + case kMultiLevels: + options.write_buffer_size = 64 * 1024; + options.target_file_size_base = 64 * 1024; + options.level0_file_num_compaction_trigger = 2; + options.level0_slowdown_writes_trigger = 2; + options.level0_stop_writes_trigger = 4; + options.max_bytes_for_level_base = 128 * 1024; + options.max_write_buffer_number = 2; + options.max_background_compactions = 8; + options.max_background_flushes = 8; + sync_use_wal_ = true; + sync_use_compact_ = false; + break; + default: + break; + } + return options; + } + + Status NewDB() { + assert(db_ == nullptr); + assert(tiny_cache_ == nullptr); + assert(env_ == nullptr); + + env_ = new FaultInjectionTestEnv(base_env_ ? base_env_.get() : system_env_); + + options_ = CurrentOptions(); + options_.env = env_; + options_.paranoid_checks = true; + + BlockBasedTableOptions table_options; + tiny_cache_ = NewLRUCache(100); + table_options.block_cache = tiny_cache_; + options_.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + dbname_ = test::PerThreadDBPath("fault_test"); + + EXPECT_OK(DestroyDB(dbname_, options_)); + + options_.create_if_missing = true; + Status s = OpenDB(); + options_.create_if_missing = false; + return s; + } + + void SetUp() override { + sequential_order_ = std::get<0>(GetParam()); + ASSERT_OK(NewDB()); + } + + void TearDown() override { + CloseDB(); + + Status s = DestroyDB(dbname_, options_); + + delete env_; + env_ = nullptr; + + tiny_cache_.reset(); + + ASSERT_OK(s); + } + + void Build(const WriteOptions& write_options, int start_idx, int num_vals) { + std::string key_space, value_space; + WriteBatch batch; + for (int i = start_idx; i < start_idx + num_vals; i++) { + Slice key = Key(i, &key_space); + batch.Clear(); + ASSERT_OK(batch.Put(key, Value(i, &value_space))); + ASSERT_OK(db_->Write(write_options, &batch)); + } + } + + Status ReadValue(int i, std::string* val) const { + std::string key_space, value_space; + Slice key = Key(i, &key_space); + Value(i, &value_space); + ReadOptions options; + return db_->Get(options, key, val); + } + + Status Verify(int start_idx, int num_vals, + ExpectedVerifResult expected) const { + std::string val; + std::string value_space; + Status s; + for (int i = start_idx; i < start_idx + num_vals && s.ok(); i++) { + Value(i, &value_space); + s = ReadValue(i, &val); + if (s.ok()) { + EXPECT_EQ(value_space, val); + } + if (expected == kValExpectFound) { + if (!s.ok()) { + fprintf(stderr, "Error when read %dth record (expect found): %s\n", i, + s.ToString().c_str()); + return s; + } + } else if (!s.ok() && !s.IsNotFound()) { + fprintf(stderr, "Error when read %dth record: %s\n", i, + s.ToString().c_str()); + return s; + } + } + return Status::OK(); + } + + // Return the ith key + Slice Key(int i, std::string* storage) const { + unsigned long long num = i; + if (!sequential_order_) { + // random transfer + const int m = 0x5bd1e995; + num *= m; + num ^= num << 24; + } + char buf[100]; + snprintf(buf, sizeof(buf), "%016d", static_cast(num)); + storage->assign(buf, strlen(buf)); + return Slice(*storage); + } + + // Return the value to associate with the specified key + Slice Value(int k, std::string* storage) const { + Random r(k); + *storage = r.RandomString(kValueSize); + return Slice(*storage); + } + + void CloseDB() { + delete db_; + db_ = nullptr; + } + + Status OpenDB() { + CloseDB(); + env_->ResetState(); + Status s = DB::Open(options_, dbname_, &db_); + assert(db_ != nullptr); + return s; + } + + void DeleteAllData() { + Iterator* iter = db_->NewIterator(ReadOptions()); + WriteOptions options; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_OK(db_->Delete(WriteOptions(), iter->key())); + } + ASSERT_OK(iter->status()); + delete iter; + + FlushOptions flush_options; + flush_options.wait = true; + ASSERT_OK(db_->Flush(flush_options)); + } + + // rnd cannot be null for kResetDropRandomUnsyncedData + void ResetDBState(ResetMethod reset_method, Random* rnd = nullptr) { + env_->AssertNoOpenFile(); + switch (reset_method) { + case kResetDropUnsyncedData: + ASSERT_OK(env_->DropUnsyncedFileData()); + break; + case kResetDropRandomUnsyncedData: + ASSERT_OK(env_->DropRandomUnsyncedFileData(rnd)); + break; + case kResetDeleteUnsyncedFiles: + ASSERT_OK(env_->DeleteFilesCreatedAfterLastDirSync()); + break; + case kResetDropAndDeleteUnsynced: + ASSERT_OK(env_->DropUnsyncedFileData()); + ASSERT_OK(env_->DeleteFilesCreatedAfterLastDirSync()); + break; + default: + assert(false); + } + } + + void PartialCompactTestPreFault(int num_pre_sync, int num_post_sync) { + DeleteAllData(); + + WriteOptions write_options; + write_options.sync = sync_use_wal_; + + Build(write_options, 0, num_pre_sync); + if (sync_use_compact_) { + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + } + write_options.sync = false; + Build(write_options, num_pre_sync, num_post_sync); + } + + void PartialCompactTestReopenWithFault(ResetMethod reset_method, + int num_pre_sync, int num_post_sync, + Random* rnd = nullptr) { + env_->SetFilesystemActive(false); + CloseDB(); + ResetDBState(reset_method, rnd); + ASSERT_OK(OpenDB()); + ASSERT_OK(Verify(0, num_pre_sync, FaultInjectionTest::kValExpectFound)); + ASSERT_OK(Verify(num_pre_sync, num_post_sync, + FaultInjectionTest::kValExpectNoError)); + WaitCompactionFinish(); + ASSERT_OK(Verify(0, num_pre_sync, FaultInjectionTest::kValExpectFound)); + ASSERT_OK(Verify(num_pre_sync, num_post_sync, + FaultInjectionTest::kValExpectNoError)); + } + + void NoWriteTestPreFault() {} + + void NoWriteTestReopenWithFault(ResetMethod reset_method) { + CloseDB(); + ResetDBState(reset_method); + ASSERT_OK(OpenDB()); + } + + void WaitCompactionFinish() { + ASSERT_OK(static_cast(db_->GetRootDB())->TEST_WaitForCompact()); + ASSERT_OK(db_->Put(WriteOptions(), "", "")); + } + + private: + Env* system_env_; + std::shared_ptr env_guard_; +}; + +class FaultInjectionTestSplitted : public FaultInjectionTest {}; + +TEST_P(FaultInjectionTestSplitted, FaultTest) { + do { + Random rnd(301); + + for (size_t idx = 0; idx < kNumIterations; idx++) { + int num_pre_sync = rnd.Uniform(kMaxNumValues); + int num_post_sync = rnd.Uniform(kMaxNumValues); + + PartialCompactTestPreFault(num_pre_sync, num_post_sync); + PartialCompactTestReopenWithFault(kResetDropUnsyncedData, num_pre_sync, + num_post_sync); + NoWriteTestPreFault(); + NoWriteTestReopenWithFault(kResetDropUnsyncedData); + + PartialCompactTestPreFault(num_pre_sync, num_post_sync); + PartialCompactTestReopenWithFault(kResetDropRandomUnsyncedData, + num_pre_sync, num_post_sync, &rnd); + NoWriteTestPreFault(); + NoWriteTestReopenWithFault(kResetDropUnsyncedData); + + // Setting a separate data path won't pass the test as we don't sync + // it after creating new files, + PartialCompactTestPreFault(num_pre_sync, num_post_sync); + PartialCompactTestReopenWithFault(kResetDropAndDeleteUnsynced, + num_pre_sync, num_post_sync); + NoWriteTestPreFault(); + NoWriteTestReopenWithFault(kResetDropAndDeleteUnsynced); + + PartialCompactTestPreFault(num_pre_sync, num_post_sync); + // No new files created so we expect all values since no files will be + // dropped. + PartialCompactTestReopenWithFault(kResetDeleteUnsyncedFiles, num_pre_sync, + num_post_sync); + NoWriteTestPreFault(); + NoWriteTestReopenWithFault(kResetDeleteUnsyncedFiles); + } + } while (ChangeOptions()); +} + +// Previous log file is not fsynced if sync is forced after log rolling. +TEST_P(FaultInjectionTest, WriteOptionSyncTest) { + test::SleepingBackgroundTask sleeping_task_low; + env_->SetBackgroundThreads(1, Env::HIGH); + // Block the job queue to prevent flush job from running. + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::HIGH); + sleeping_task_low.WaitUntilSleeping(); + + WriteOptions write_options; + write_options.sync = false; + + std::string key_space, value_space; + ASSERT_OK( + db_->Put(write_options, Key(1, &key_space), Value(1, &value_space))); + FlushOptions flush_options; + flush_options.wait = false; + ASSERT_OK(db_->Flush(flush_options)); + write_options.sync = true; + ASSERT_OK( + db_->Put(write_options, Key(2, &key_space), Value(2, &value_space))); + ASSERT_OK(db_->FlushWAL(false)); + + env_->SetFilesystemActive(false); + NoWriteTestReopenWithFault(kResetDropAndDeleteUnsynced); + sleeping_task_low.WakeUp(); + sleeping_task_low.WaitUntilDone(); + + ASSERT_OK(OpenDB()); + std::string val; + Value(2, &value_space); + ASSERT_OK(ReadValue(2, &val)); + ASSERT_EQ(value_space, val); + + Value(1, &value_space); + ASSERT_OK(ReadValue(1, &val)); + ASSERT_EQ(value_space, val); +} + +TEST_P(FaultInjectionTest, UninstalledCompaction) { + options_.target_file_size_base = 32 * 1024; + options_.write_buffer_size = 100 << 10; // 100KB + options_.level0_file_num_compaction_trigger = 6; + options_.level0_stop_writes_trigger = 1 << 10; + options_.level0_slowdown_writes_trigger = 1 << 10; + options_.max_background_compactions = 1; + OpenDB(); + + if (!sequential_order_) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"FaultInjectionTest::FaultTest:0", "DBImpl::BGWorkCompaction"}, + {"CompactionJob::Run():End", "FaultInjectionTest::FaultTest:1"}, + {"FaultInjectionTest::FaultTest:2", + "DBImpl::BackgroundCompaction:NonTrivial:AfterRun"}, + }); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + int kNumKeys = 1000; + Build(WriteOptions(), 0, kNumKeys); + FlushOptions flush_options; + flush_options.wait = true; + ASSERT_OK(db_->Flush(flush_options)); + ASSERT_OK(db_->Put(WriteOptions(), "", "")); + TEST_SYNC_POINT("FaultInjectionTest::FaultTest:0"); + TEST_SYNC_POINT("FaultInjectionTest::FaultTest:1"); + env_->SetFilesystemActive(false); + TEST_SYNC_POINT("FaultInjectionTest::FaultTest:2"); + CloseDB(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ResetDBState(kResetDropUnsyncedData); + + std::atomic opened(false); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::Open:Opened", [&](void* /*arg*/) { opened.store(true); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BGWorkCompaction", + [&](void* /*arg*/) { ASSERT_TRUE(opened.load()); }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(OpenDB()); + ASSERT_OK(Verify(0, kNumKeys, FaultInjectionTest::kValExpectFound)); + WaitCompactionFinish(); + ASSERT_OK(Verify(0, kNumKeys, FaultInjectionTest::kValExpectFound)); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_P(FaultInjectionTest, ManualLogSyncTest) { + test::SleepingBackgroundTask sleeping_task_low; + env_->SetBackgroundThreads(1, Env::HIGH); + // Block the job queue to prevent flush job from running. + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low, + Env::Priority::HIGH); + sleeping_task_low.WaitUntilSleeping(); + + WriteOptions write_options; + write_options.sync = false; + + std::string key_space, value_space; + ASSERT_OK( + db_->Put(write_options, Key(1, &key_space), Value(1, &value_space))); + FlushOptions flush_options; + flush_options.wait = false; + ASSERT_OK(db_->Flush(flush_options)); + ASSERT_OK( + db_->Put(write_options, Key(2, &key_space), Value(2, &value_space))); + ASSERT_OK(db_->FlushWAL(true)); + + env_->SetFilesystemActive(false); + NoWriteTestReopenWithFault(kResetDropAndDeleteUnsynced); + sleeping_task_low.WakeUp(); + sleeping_task_low.WaitUntilDone(); + + ASSERT_OK(OpenDB()); + std::string val; + Value(2, &value_space); + ASSERT_OK(ReadValue(2, &val)); + ASSERT_EQ(value_space, val); + + Value(1, &value_space); + ASSERT_OK(ReadValue(1, &val)); + ASSERT_EQ(value_space, val); +} + +TEST_P(FaultInjectionTest, WriteBatchWalTerminationTest) { + ReadOptions ro; + Options options = CurrentOptions(); + options.env = env_; + + WriteOptions wo; + wo.sync = true; + wo.disableWAL = false; + WriteBatch batch; + ASSERT_OK(batch.Put("cats", "dogs")); + batch.MarkWalTerminationPoint(); + ASSERT_OK(batch.Put("boys", "girls")); + ASSERT_OK(db_->Write(wo, &batch)); + + env_->SetFilesystemActive(false); + NoWriteTestReopenWithFault(kResetDropAndDeleteUnsynced); + ASSERT_OK(OpenDB()); + + std::string val; + ASSERT_OK(db_->Get(ro, "cats", &val)); + ASSERT_EQ("dogs", val); + ASSERT_EQ(db_->Get(ro, "boys", &val), Status::NotFound()); +} + +TEST_P(FaultInjectionTest, NoDuplicateTrailingEntries) { + auto fault_fs = std::make_shared(FileSystem::Default()); + fault_fs->EnableWriteErrorInjection(); + fault_fs->SetFilesystemDirectWritable(false); + const std::string file_name = NormalizePath(dbname_ + "/test_file"); + std::unique_ptr log_writer = nullptr; + constexpr uint64_t log_number = 0; + { + std::unique_ptr file; + const Status s = + fault_fs->NewWritableFile(file_name, FileOptions(), &file, nullptr); + ASSERT_OK(s); + std::unique_ptr fwriter( + new WritableFileWriter(std::move(file), file_name, FileOptions())); + log_writer.reset(new log::Writer(std::move(fwriter), log_number, + /*recycle_log_files=*/false)); + } + + fault_fs->SetRandomWriteError( + 0xdeadbeef, /*one_in=*/1, IOStatus::IOError("Injected IOError"), + /*inject_for_all_file_types=*/true, /*types=*/{}); + + { + VersionEdit edit; + edit.SetColumnFamily(0); + std::string buf; + assert(edit.EncodeTo(&buf)); + const Status s = log_writer->AddRecord(buf); + ASSERT_NOK(s); + } + + fault_fs->DisableWriteErrorInjection(); + + // Closing the log writer will cause WritableFileWriter::Close() and flush + // remaining data from its buffer to underlying file. + log_writer.reset(); + + { + std::unique_ptr file; + Status s = + fault_fs->NewSequentialFile(file_name, FileOptions(), &file, nullptr); + ASSERT_OK(s); + std::unique_ptr freader( + new SequentialFileReader(std::move(file), file_name)); + Status log_read_s; + class LogReporter : public log::Reader::Reporter { + public: + Status* status_; + explicit LogReporter(Status* _s) : status_(_s) {} + void Corruption(size_t /*bytes*/, const Status& _s) override { + if (status_->ok()) { + *status_ = _s; + } + } + } reporter(&log_read_s); + std::unique_ptr log_reader(new log::Reader( + nullptr, std::move(freader), &reporter, /*checksum=*/true, log_number)); + Slice record; + std::string data; + size_t count = 0; + while (log_reader->ReadRecord(&record, &data) && log_read_s.ok()) { + VersionEdit edit; + ASSERT_OK(edit.DecodeFrom(data)); + ++count; + } + // Verify that only one version edit exists in the file. + ASSERT_EQ(1, count); + } +} + +INSTANTIATE_TEST_CASE_P( + FaultTest, FaultInjectionTest, + ::testing::Values(std::make_tuple(false, kDefault, kEnd), + std::make_tuple(true, kDefault, kEnd))); + +INSTANTIATE_TEST_CASE_P( + FaultTest, FaultInjectionTestSplitted, + ::testing::Values(std::make_tuple(false, kDefault, kSyncWal), + std::make_tuple(true, kDefault, kSyncWal), + std::make_tuple(false, kSyncWal, kEnd), + std::make_tuple(true, kSyncWal, kEnd))); + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/file_indexer.cc b/librocksdb-sys/rocksdb/db/file_indexer.cc new file mode 100644 index 0000000..ee6cfdc --- /dev/null +++ b/librocksdb-sys/rocksdb/db/file_indexer.cc @@ -0,0 +1,216 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/file_indexer.h" + +#include +#include + +#include "db/version_edit.h" +#include "rocksdb/comparator.h" + +namespace ROCKSDB_NAMESPACE { + +FileIndexer::FileIndexer(const Comparator* ucmp) + : num_levels_(0), ucmp_(ucmp), level_rb_(nullptr) {} + +size_t FileIndexer::NumLevelIndex() const { return next_level_index_.size(); } + +size_t FileIndexer::LevelIndexSize(size_t level) const { + if (level >= next_level_index_.size()) { + return 0; + } + return next_level_index_[level].num_index; +} + +void FileIndexer::GetNextLevelIndex(const size_t level, const size_t file_index, + const int cmp_smallest, + const int cmp_largest, int32_t* left_bound, + int32_t* right_bound) const { + assert(level > 0); + + // Last level, no hint + if (level == num_levels_ - 1) { + *left_bound = 0; + *right_bound = -1; + return; + } + + assert(level < num_levels_ - 1); + assert(static_cast(file_index) <= level_rb_[level]); + + const IndexUnit* index_units = next_level_index_[level].index_units; + const auto& index = index_units[file_index]; + + if (cmp_smallest < 0) { + *left_bound = (level > 0 && file_index > 0) + ? index_units[file_index - 1].largest_lb + : 0; + *right_bound = index.smallest_rb; + } else if (cmp_smallest == 0) { + *left_bound = index.smallest_lb; + *right_bound = index.smallest_rb; + } else if (cmp_largest < 0) { + *left_bound = index.smallest_lb; + *right_bound = index.largest_rb; + } else if (cmp_largest == 0) { + *left_bound = index.largest_lb; + *right_bound = index.largest_rb; + } else { + *left_bound = index.largest_lb; + *right_bound = level_rb_[level + 1]; + } + + assert(*left_bound >= 0); + assert(*left_bound <= *right_bound + 1); + assert(*right_bound <= level_rb_[level + 1]); +} + +void FileIndexer::UpdateIndex(Arena* arena, const size_t num_levels, + std::vector* const files) { + if (files == nullptr) { + return; + } + if (num_levels == 0) { // uint_32 0-1 would cause bad behavior + num_levels_ = num_levels; + return; + } + assert(level_rb_ == nullptr); // level_rb_ should be init here + + num_levels_ = num_levels; + next_level_index_.resize(num_levels); + + char* mem = arena->AllocateAligned(num_levels_ * sizeof(int32_t)); + level_rb_ = new (mem) int32_t[num_levels_]; + for (size_t i = 0; i < num_levels_; i++) { + level_rb_[i] = -1; + } + + // L1 - Ln-1 + for (size_t level = 1; level < num_levels_ - 1; ++level) { + const auto& upper_files = files[level]; + const int32_t upper_size = static_cast(upper_files.size()); + const auto& lower_files = files[level + 1]; + level_rb_[level] = static_cast(upper_files.size()) - 1; + if (upper_size == 0) { + continue; + } + IndexLevel& index_level = next_level_index_[level]; + index_level.num_index = upper_size; + mem = arena->AllocateAligned(upper_size * sizeof(IndexUnit)); + index_level.index_units = new (mem) IndexUnit[upper_size]; + + CalculateLB( + upper_files, lower_files, &index_level, + [this](const FileMetaData* a, const FileMetaData* b) -> int { + return ucmp_->CompareWithoutTimestamp(a->smallest.user_key(), + b->largest.user_key()); + }, + [](IndexUnit* index, int32_t f_idx) { index->smallest_lb = f_idx; }); + CalculateLB( + upper_files, lower_files, &index_level, + [this](const FileMetaData* a, const FileMetaData* b) -> int { + return ucmp_->CompareWithoutTimestamp(a->largest.user_key(), + b->largest.user_key()); + }, + [](IndexUnit* index, int32_t f_idx) { index->largest_lb = f_idx; }); + CalculateRB( + upper_files, lower_files, &index_level, + [this](const FileMetaData* a, const FileMetaData* b) -> int { + return ucmp_->CompareWithoutTimestamp(a->smallest.user_key(), + b->smallest.user_key()); + }, + [](IndexUnit* index, int32_t f_idx) { index->smallest_rb = f_idx; }); + CalculateRB( + upper_files, lower_files, &index_level, + [this](const FileMetaData* a, const FileMetaData* b) -> int { + return ucmp_->CompareWithoutTimestamp(a->largest.user_key(), + b->smallest.user_key()); + }, + [](IndexUnit* index, int32_t f_idx) { index->largest_rb = f_idx; }); + } + + level_rb_[num_levels_ - 1] = + static_cast(files[num_levels_ - 1].size()) - 1; +} + +void FileIndexer::CalculateLB( + const std::vector& upper_files, + const std::vector& lower_files, IndexLevel* index_level, + std::function cmp_op, + std::function set_index) { + const int32_t upper_size = static_cast(upper_files.size()); + const int32_t lower_size = static_cast(lower_files.size()); + int32_t upper_idx = 0; + int32_t lower_idx = 0; + + IndexUnit* index = index_level->index_units; + while (upper_idx < upper_size && lower_idx < lower_size) { + int cmp = cmp_op(upper_files[upper_idx], lower_files[lower_idx]); + + if (cmp == 0) { + set_index(&index[upper_idx], lower_idx); + ++upper_idx; + } else if (cmp > 0) { + // Lower level's file (largest) is smaller, a key won't hit in that + // file. Move to next lower file + ++lower_idx; + } else { + // Lower level's file becomes larger, update the index, and + // move to the next upper file + set_index(&index[upper_idx], lower_idx); + ++upper_idx; + } + } + + while (upper_idx < upper_size) { + // Lower files are exhausted, that means the remaining upper files are + // greater than any lower files. Set the index to be the lower level size. + set_index(&index[upper_idx], lower_size); + ++upper_idx; + } +} + +void FileIndexer::CalculateRB( + const std::vector& upper_files, + const std::vector& lower_files, IndexLevel* index_level, + std::function cmp_op, + std::function set_index) { + const int32_t upper_size = static_cast(upper_files.size()); + const int32_t lower_size = static_cast(lower_files.size()); + int32_t upper_idx = upper_size - 1; + int32_t lower_idx = lower_size - 1; + + IndexUnit* index = index_level->index_units; + while (upper_idx >= 0 && lower_idx >= 0) { + int cmp = cmp_op(upper_files[upper_idx], lower_files[lower_idx]); + + if (cmp == 0) { + set_index(&index[upper_idx], lower_idx); + --upper_idx; + } else if (cmp < 0) { + // Lower level's file (smallest) is larger, a key won't hit in that + // file. Move to next lower file. + --lower_idx; + } else { + // Lower level's file becomes smaller, update the index, and move to + // the next the upper file + set_index(&index[upper_idx], lower_idx); + --upper_idx; + } + } + while (upper_idx >= 0) { + // Lower files are exhausted, that means the remaining upper files are + // smaller than any lower files. Set it to -1. + set_index(&index[upper_idx], -1); + --upper_idx; + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/file_indexer.h b/librocksdb-sys/rocksdb/db/file_indexer.h new file mode 100644 index 0000000..45cb136 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/file_indexer.h @@ -0,0 +1,140 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include +#include +#include +#include + +#include "memory/arena.h" +#include "port/port.h" +#include "util/autovector.h" + +namespace ROCKSDB_NAMESPACE { + +class Comparator; +struct FileMetaData; +struct FdWithKeyRange; +struct FileLevel; + +// The file tree structure in Version is prebuilt and the range of each file +// is known. On Version::Get(), it uses binary search to find a potential file +// and then check if a target key can be found in the file by comparing the key +// to each file's smallest and largest key. The results of these comparisons +// can be reused beyond checking if a key falls into a file's range. +// With some pre-calculated knowledge, each key comparison that has been done +// can serve as a hint to narrow down further searches: if a key compared to +// be smaller than a file's smallest or largest, that comparison can be used +// to find out the right bound of next binary search. Similarly, if a key +// compared to be larger than a file's smallest or largest, it can be utilized +// to find out the left bound of next binary search. +// With these hints: it can greatly reduce the range of binary search, +// especially for bottom levels, given that one file most likely overlaps with +// only N files from level below (where N is max_bytes_for_level_multiplier). +// So on level L, we will only look at ~N files instead of N^L files on the +// naive approach. +class FileIndexer { + public: + explicit FileIndexer(const Comparator* ucmp); + + size_t NumLevelIndex() const; + + size_t LevelIndexSize(size_t level) const; + + // Return a file index range in the next level to search for a key based on + // smallest and largest key comparison for the current file specified by + // level and file_index. When *left_index < *right_index, both index should + // be valid and fit in the vector size. + void GetNextLevelIndex(const size_t level, const size_t file_index, + const int cmp_smallest, const int cmp_largest, + int32_t* left_bound, int32_t* right_bound) const; + + void UpdateIndex(Arena* arena, const size_t num_levels, + std::vector* const files); + + enum { kLevelMaxIndex = std::numeric_limits::max() }; + + private: + size_t num_levels_; + const Comparator* ucmp_; + + struct IndexUnit { + IndexUnit() + : smallest_lb(0), largest_lb(0), smallest_rb(-1), largest_rb(-1) {} + // During file search, a key is compared against smallest and largest + // from a FileMetaData. It can have 3 possible outcomes: + // (1) key is smaller than smallest, implying it is also smaller than + // larger. Precalculated index based on "smallest < smallest" can + // be used to provide right bound. + // (2) key is in between smallest and largest. + // Precalculated index based on "smallest > greatest" can be used to + // provide left bound. + // Precalculated index based on "largest < smallest" can be used to + // provide right bound. + // (3) key is larger than largest, implying it is also larger than smallest. + // Precalculated index based on "largest > largest" can be used to + // provide left bound. + // + // As a result, we will need to do: + // Compare smallest (<=) and largest keys from upper level file with + // smallest key from lower level to get a right bound. + // Compare smallest (>=) and largest keys from upper level file with + // largest key from lower level to get a left bound. + // + // Example: + // level 1: [50 - 60] + // level 2: [1 - 40], [45 - 55], [58 - 80] + // A key 35, compared to be less than 50, 3rd file on level 2 can be + // skipped according to rule (1). LB = 0, RB = 1. + // A key 53, sits in the middle 50 and 60. 1st file on level 2 can be + // skipped according to rule (2)-a, but the 3rd file cannot be skipped + // because 60 is greater than 58. LB = 1, RB = 2. + // A key 70, compared to be larger than 60. 1st and 2nd file can be skipped + // according to rule (3). LB = 2, RB = 2. + // + // Point to a left most file in a lower level that may contain a key, + // which compares greater than smallest of a FileMetaData (upper level) + int32_t smallest_lb; + // Point to a left most file in a lower level that may contain a key, + // which compares greater than largest of a FileMetaData (upper level) + int32_t largest_lb; + // Point to a right most file in a lower level that may contain a key, + // which compares smaller than smallest of a FileMetaData (upper level) + int32_t smallest_rb; + // Point to a right most file in a lower level that may contain a key, + // which compares smaller than largest of a FileMetaData (upper level) + int32_t largest_rb; + }; + + // Data structure to store IndexUnits in a whole level + struct IndexLevel { + size_t num_index; + IndexUnit* index_units; + + IndexLevel() : num_index(0), index_units(nullptr) {} + }; + + void CalculateLB( + const std::vector& upper_files, + const std::vector& lower_files, IndexLevel* index_level, + std::function cmp_op, + std::function set_index); + + void CalculateRB( + const std::vector& upper_files, + const std::vector& lower_files, IndexLevel* index_level, + std::function cmp_op, + std::function set_index); + + autovector next_level_index_; + int32_t* level_rb_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/file_indexer_test.cc b/librocksdb-sys/rocksdb/db/file_indexer_test.cc new file mode 100644 index 0000000..5c82189 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/file_indexer_test.cc @@ -0,0 +1,352 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/file_indexer.h" + +#include + +#include "db/dbformat.h" +#include "db/version_edit.h" +#include "port/stack_trace.h" +#include "rocksdb/comparator.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" + +namespace ROCKSDB_NAMESPACE { + +class IntComparator : public Comparator { + public: + int Compare(const Slice& a, const Slice& b) const override { + assert(a.size() == 8); + assert(b.size() == 8); + int64_t diff = *reinterpret_cast(a.data()) - + *reinterpret_cast(b.data()); + if (diff < 0) { + return -1; + } else if (diff == 0) { + return 0; + } else { + return 1; + } + } + + const char* Name() const override { return "IntComparator"; } + + void FindShortestSeparator(std::string* /*start*/, + const Slice& /*limit*/) const override {} + + void FindShortSuccessor(std::string* /*key*/) const override {} +}; + +class FileIndexerTest : public testing::Test { + public: + FileIndexerTest() + : kNumLevels(4), files(new std::vector[kNumLevels]) {} + + ~FileIndexerTest() override { + ClearFiles(); + delete[] files; + } + + void AddFile(int level, int64_t smallest, int64_t largest) { + auto* f = new FileMetaData(); + f->smallest = IntKey(smallest); + f->largest = IntKey(largest); + files[level].push_back(f); + } + + InternalKey IntKey(int64_t v) { + return InternalKey(Slice(reinterpret_cast(&v), 8), 0, kTypeValue); + } + + void ClearFiles() { + for (uint32_t i = 0; i < kNumLevels; ++i) { + for (auto* f : files[i]) { + delete f; + } + files[i].clear(); + } + } + + void GetNextLevelIndex(const uint32_t level, const uint32_t file_index, + const int cmp_smallest, const int cmp_largest, + int32_t* left_index, int32_t* right_index) { + *left_index = 100; + *right_index = 100; + indexer->GetNextLevelIndex(level, file_index, cmp_smallest, cmp_largest, + left_index, right_index); + } + + int32_t left = 100; + int32_t right = 100; + const uint32_t kNumLevels; + IntComparator ucmp; + FileIndexer* indexer; + + std::vector* files; +}; + +// Case 0: Empty +TEST_F(FileIndexerTest, Empty) { + Arena arena; + indexer = new FileIndexer(&ucmp); + indexer->UpdateIndex(&arena, 0, files); + delete indexer; +} + +// Case 1: no overlap, files are on the left of next level files +TEST_F(FileIndexerTest, no_overlap_left) { + Arena arena; + indexer = new FileIndexer(&ucmp); + // level 1 + AddFile(1, 100, 200); + AddFile(1, 300, 400); + AddFile(1, 500, 600); + // level 2 + AddFile(2, 1500, 1600); + AddFile(2, 1601, 1699); + AddFile(2, 1700, 1800); + // level 3 + AddFile(3, 2500, 2600); + AddFile(3, 2601, 2699); + AddFile(3, 2700, 2800); + indexer->UpdateIndex(&arena, kNumLevels, files); + for (uint32_t level = 1; level < 3; ++level) { + for (uint32_t f = 0; f < 3; ++f) { + GetNextLevelIndex(level, f, -1, -1, &left, &right); + ASSERT_EQ(0, left); + ASSERT_EQ(-1, right); + GetNextLevelIndex(level, f, 0, -1, &left, &right); + ASSERT_EQ(0, left); + ASSERT_EQ(-1, right); + GetNextLevelIndex(level, f, 1, -1, &left, &right); + ASSERT_EQ(0, left); + ASSERT_EQ(-1, right); + GetNextLevelIndex(level, f, 1, 0, &left, &right); + ASSERT_EQ(0, left); + ASSERT_EQ(-1, right); + GetNextLevelIndex(level, f, 1, 1, &left, &right); + ASSERT_EQ(0, left); + ASSERT_EQ(2, right); + } + } + delete indexer; + ClearFiles(); +} + +// Case 2: no overlap, files are on the right of next level files +TEST_F(FileIndexerTest, no_overlap_right) { + Arena arena; + indexer = new FileIndexer(&ucmp); + // level 1 + AddFile(1, 2100, 2200); + AddFile(1, 2300, 2400); + AddFile(1, 2500, 2600); + // level 2 + AddFile(2, 1500, 1600); + AddFile(2, 1501, 1699); + AddFile(2, 1700, 1800); + // level 3 + AddFile(3, 500, 600); + AddFile(3, 501, 699); + AddFile(3, 700, 800); + indexer->UpdateIndex(&arena, kNumLevels, files); + for (uint32_t level = 1; level < 3; ++level) { + for (uint32_t f = 0; f < 3; ++f) { + GetNextLevelIndex(level, f, -1, -1, &left, &right); + ASSERT_EQ(f == 0 ? 0 : 3, left); + ASSERT_EQ(2, right); + GetNextLevelIndex(level, f, 0, -1, &left, &right); + ASSERT_EQ(3, left); + ASSERT_EQ(2, right); + GetNextLevelIndex(level, f, 1, -1, &left, &right); + ASSERT_EQ(3, left); + ASSERT_EQ(2, right); + GetNextLevelIndex(level, f, 1, -1, &left, &right); + ASSERT_EQ(3, left); + ASSERT_EQ(2, right); + GetNextLevelIndex(level, f, 1, 0, &left, &right); + ASSERT_EQ(3, left); + ASSERT_EQ(2, right); + GetNextLevelIndex(level, f, 1, 1, &left, &right); + ASSERT_EQ(3, left); + ASSERT_EQ(2, right); + } + } + delete indexer; +} + +// Case 3: empty L2 +TEST_F(FileIndexerTest, empty_L2) { + Arena arena; + indexer = new FileIndexer(&ucmp); + for (uint32_t i = 1; i < kNumLevels; ++i) { + ASSERT_EQ(0U, indexer->LevelIndexSize(i)); + } + // level 1 + AddFile(1, 2100, 2200); + AddFile(1, 2300, 2400); + AddFile(1, 2500, 2600); + // level 3 + AddFile(3, 500, 600); + AddFile(3, 501, 699); + AddFile(3, 700, 800); + indexer->UpdateIndex(&arena, kNumLevels, files); + for (uint32_t f = 0; f < 3; ++f) { + GetNextLevelIndex(1, f, -1, -1, &left, &right); + ASSERT_EQ(0, left); + ASSERT_EQ(-1, right); + GetNextLevelIndex(1, f, 0, -1, &left, &right); + ASSERT_EQ(0, left); + ASSERT_EQ(-1, right); + GetNextLevelIndex(1, f, 1, -1, &left, &right); + ASSERT_EQ(0, left); + ASSERT_EQ(-1, right); + GetNextLevelIndex(1, f, 1, -1, &left, &right); + ASSERT_EQ(0, left); + ASSERT_EQ(-1, right); + GetNextLevelIndex(1, f, 1, 0, &left, &right); + ASSERT_EQ(0, left); + ASSERT_EQ(-1, right); + GetNextLevelIndex(1, f, 1, 1, &left, &right); + ASSERT_EQ(0, left); + ASSERT_EQ(-1, right); + } + delete indexer; + ClearFiles(); +} + +// Case 4: mixed +TEST_F(FileIndexerTest, mixed) { + Arena arena; + indexer = new FileIndexer(&ucmp); + // level 1 + AddFile(1, 100, 200); + AddFile(1, 250, 400); + AddFile(1, 450, 500); + // level 2 + AddFile(2, 100, 150); // 0 + AddFile(2, 200, 250); // 1 + AddFile(2, 251, 300); // 2 + AddFile(2, 301, 350); // 3 + AddFile(2, 500, 600); // 4 + // level 3 + AddFile(3, 0, 50); + AddFile(3, 100, 200); + AddFile(3, 201, 250); + indexer->UpdateIndex(&arena, kNumLevels, files); + // level 1, 0 + GetNextLevelIndex(1, 0, -1, -1, &left, &right); + ASSERT_EQ(0, left); + ASSERT_EQ(0, right); + GetNextLevelIndex(1, 0, 0, -1, &left, &right); + ASSERT_EQ(0, left); + ASSERT_EQ(0, right); + GetNextLevelIndex(1, 0, 1, -1, &left, &right); + ASSERT_EQ(0, left); + ASSERT_EQ(1, right); + GetNextLevelIndex(1, 0, 1, 0, &left, &right); + ASSERT_EQ(1, left); + ASSERT_EQ(1, right); + GetNextLevelIndex(1, 0, 1, 1, &left, &right); + ASSERT_EQ(1, left); + ASSERT_EQ(4, right); + // level 1, 1 + GetNextLevelIndex(1, 1, -1, -1, &left, &right); + ASSERT_EQ(1, left); + ASSERT_EQ(1, right); + GetNextLevelIndex(1, 1, 0, -1, &left, &right); + ASSERT_EQ(1, left); + ASSERT_EQ(1, right); + GetNextLevelIndex(1, 1, 1, -1, &left, &right); + ASSERT_EQ(1, left); + ASSERT_EQ(3, right); + GetNextLevelIndex(1, 1, 1, 0, &left, &right); + ASSERT_EQ(4, left); + ASSERT_EQ(3, right); + GetNextLevelIndex(1, 1, 1, 1, &left, &right); + ASSERT_EQ(4, left); + ASSERT_EQ(4, right); + // level 1, 2 + GetNextLevelIndex(1, 2, -1, -1, &left, &right); + ASSERT_EQ(4, left); + ASSERT_EQ(3, right); + GetNextLevelIndex(1, 2, 0, -1, &left, &right); + ASSERT_EQ(4, left); + ASSERT_EQ(3, right); + GetNextLevelIndex(1, 2, 1, -1, &left, &right); + ASSERT_EQ(4, left); + ASSERT_EQ(4, right); + GetNextLevelIndex(1, 2, 1, 0, &left, &right); + ASSERT_EQ(4, left); + ASSERT_EQ(4, right); + GetNextLevelIndex(1, 2, 1, 1, &left, &right); + ASSERT_EQ(4, left); + ASSERT_EQ(4, right); + // level 2, 0 + GetNextLevelIndex(2, 0, -1, -1, &left, &right); + ASSERT_EQ(0, left); + ASSERT_EQ(1, right); + GetNextLevelIndex(2, 0, 0, -1, &left, &right); + ASSERT_EQ(1, left); + ASSERT_EQ(1, right); + GetNextLevelIndex(2, 0, 1, -1, &left, &right); + ASSERT_EQ(1, left); + ASSERT_EQ(1, right); + GetNextLevelIndex(2, 0, 1, 0, &left, &right); + ASSERT_EQ(1, left); + ASSERT_EQ(1, right); + GetNextLevelIndex(2, 0, 1, 1, &left, &right); + ASSERT_EQ(1, left); + ASSERT_EQ(2, right); + // level 2, 1 + GetNextLevelIndex(2, 1, -1, -1, &left, &right); + ASSERT_EQ(1, left); + ASSERT_EQ(1, right); + GetNextLevelIndex(2, 1, 0, -1, &left, &right); + ASSERT_EQ(1, left); + ASSERT_EQ(1, right); + GetNextLevelIndex(2, 1, 1, -1, &left, &right); + ASSERT_EQ(1, left); + ASSERT_EQ(2, right); + GetNextLevelIndex(2, 1, 1, 0, &left, &right); + ASSERT_EQ(2, left); + ASSERT_EQ(2, right); + GetNextLevelIndex(2, 1, 1, 1, &left, &right); + ASSERT_EQ(2, left); + ASSERT_EQ(2, right); + // level 2, [2 - 4], no overlap + for (uint32_t f = 2; f <= 4; ++f) { + GetNextLevelIndex(2, f, -1, -1, &left, &right); + ASSERT_EQ(f == 2 ? 2 : 3, left); + ASSERT_EQ(2, right); + GetNextLevelIndex(2, f, 0, -1, &left, &right); + ASSERT_EQ(3, left); + ASSERT_EQ(2, right); + GetNextLevelIndex(2, f, 1, -1, &left, &right); + ASSERT_EQ(3, left); + ASSERT_EQ(2, right); + GetNextLevelIndex(2, f, 1, 0, &left, &right); + ASSERT_EQ(3, left); + ASSERT_EQ(2, right); + GetNextLevelIndex(2, f, 1, 1, &left, &right); + ASSERT_EQ(3, left); + ASSERT_EQ(2, right); + } + delete indexer; + ClearFiles(); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/filename_test.cc b/librocksdb-sys/rocksdb/db/filename_test.cc new file mode 100644 index 0000000..04c81b3 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/filename_test.cc @@ -0,0 +1,241 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "file/filename.h" + +#include "db/dbformat.h" +#include "port/port.h" +#include "test_util/testharness.h" + +namespace ROCKSDB_NAMESPACE { + +class FileNameTest : public testing::Test {}; + +TEST_F(FileNameTest, Parse) { + Slice db; + FileType type; + uint64_t number; + + char kDefautInfoLogDir = 1; + char kDifferentInfoLogDir = 2; + char kNoCheckLogDir = 4; + char kAllMode = kDefautInfoLogDir | kDifferentInfoLogDir | kNoCheckLogDir; + + // Successful parses + static struct { + const char* fname; + uint64_t number; + FileType type; + char mode; + } cases[] = { + {"100.log", 100, kWalFile, kAllMode}, + {"0.log", 0, kWalFile, kAllMode}, + {"0.sst", 0, kTableFile, kAllMode}, + {"CURRENT", 0, kCurrentFile, kAllMode}, + {"LOCK", 0, kDBLockFile, kAllMode}, + {"MANIFEST-2", 2, kDescriptorFile, kAllMode}, + {"MANIFEST-7", 7, kDescriptorFile, kAllMode}, + {"METADB-2", 2, kMetaDatabase, kAllMode}, + {"METADB-7", 7, kMetaDatabase, kAllMode}, + {"LOG", 0, kInfoLogFile, kDefautInfoLogDir}, + {"LOG.old", 0, kInfoLogFile, kDefautInfoLogDir}, + {"LOG.old.6688", 6688, kInfoLogFile, kDefautInfoLogDir}, + {"rocksdb_dir_LOG", 0, kInfoLogFile, kDifferentInfoLogDir}, + {"rocksdb_dir_LOG.old", 0, kInfoLogFile, kDifferentInfoLogDir}, + {"rocksdb_dir_LOG.old.6688", 6688, kInfoLogFile, kDifferentInfoLogDir}, + {"18446744073709551615.log", 18446744073709551615ull, kWalFile, kAllMode}, + }; + for (char mode : {kDifferentInfoLogDir, kDefautInfoLogDir, kNoCheckLogDir}) { + for (unsigned int i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + InfoLogPrefix info_log_prefix(mode != kDefautInfoLogDir, "/rocksdb/dir"); + if (cases[i].mode & mode) { + std::string f = cases[i].fname; + if (mode == kNoCheckLogDir) { + ASSERT_TRUE(ParseFileName(f, &number, &type)) << f; + } else { + ASSERT_TRUE(ParseFileName(f, &number, info_log_prefix.prefix, &type)) + << f; + } + ASSERT_EQ(cases[i].type, type) << f; + ASSERT_EQ(cases[i].number, number) << f; + } + } + } + + // Errors + static const char* errors[] = {"", + "foo", + "foo-dx-100.log", + ".log", + "", + "manifest", + "CURREN", + "CURRENTX", + "MANIFES", + "MANIFEST", + "MANIFEST-", + "XMANIFEST-3", + "MANIFEST-3x", + "META", + "METADB", + "METADB-", + "XMETADB-3", + "METADB-3x", + "LOC", + "LOCKx", + "LO", + "LOGx", + "18446744073709551616.log", + "184467440737095516150.log", + "100", + "100.", + "100.lop"}; + for (unsigned int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) { + std::string f = errors[i]; + ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f; + }; +} + +TEST_F(FileNameTest, InfoLogFileName) { + std::string dbname = ("/data/rocksdb"); + std::string db_absolute_path; + ASSERT_OK(Env::Default()->GetAbsolutePath(dbname, &db_absolute_path)); + + ASSERT_EQ("/data/rocksdb/LOG", InfoLogFileName(dbname, db_absolute_path, "")); + ASSERT_EQ("/data/rocksdb/LOG.old.666", + OldInfoLogFileName(dbname, 666u, db_absolute_path, "")); + + ASSERT_EQ("/data/rocksdb_log/data_rocksdb_LOG", + InfoLogFileName(dbname, db_absolute_path, "/data/rocksdb_log")); + ASSERT_EQ( + "/data/rocksdb_log/data_rocksdb_LOG.old.666", + OldInfoLogFileName(dbname, 666u, db_absolute_path, "/data/rocksdb_log")); +} + +TEST_F(FileNameTest, Construction) { + uint64_t number; + FileType type; + std::string fname; + + fname = CurrentFileName("foo"); + ASSERT_EQ("foo/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(0U, number); + ASSERT_EQ(kCurrentFile, type); + + fname = LockFileName("foo"); + ASSERT_EQ("foo/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(0U, number); + ASSERT_EQ(kDBLockFile, type); + + fname = LogFileName("foo", 192); + ASSERT_EQ("foo/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(192U, number); + ASSERT_EQ(kWalFile, type); + + fname = TableFileName({DbPath("bar", 0)}, 200, 0); + std::string fname1 = + TableFileName({DbPath("foo", 0), DbPath("bar", 0)}, 200, 1); + ASSERT_EQ(fname, fname1); + ASSERT_EQ("bar/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(200U, number); + ASSERT_EQ(kTableFile, type); + + fname = DescriptorFileName("bar", 100); + ASSERT_EQ("bar/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(100U, number); + ASSERT_EQ(kDescriptorFile, type); + + fname = TempFileName("tmp", 999); + ASSERT_EQ("tmp/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(999U, number); + ASSERT_EQ(kTempFile, type); + + fname = MetaDatabaseName("met", 100); + ASSERT_EQ("met/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(100U, number); + ASSERT_EQ(kMetaDatabase, type); +} + +TEST_F(FileNameTest, NormalizePath) { + // No leading slash + const std::string sep = std::string(1, kFilePathSeparator); + + std::string expected = "FOLDER" + sep + "filename.ext"; + std::string given = "FOLDER" + sep + "filename.ext"; + + ASSERT_EQ(expected, NormalizePath(given)); + + // Two chars /a + + expected = sep + "a"; + given = expected; + ASSERT_EQ(expected, NormalizePath(given)); + + // Two chars a/ + expected = "a" + sep; + given = expected; + ASSERT_EQ(expected, NormalizePath(given)); + + // Server only + expected = sep + sep + "a"; + given = expected; + ASSERT_EQ(expected, NormalizePath(given)); + + // Two slashes after character + expected = "a" + sep; + given = "a" + sep + sep; + + ASSERT_EQ(expected, NormalizePath(given)); + + // slash only / + expected = sep; + given = expected; + ASSERT_EQ(expected, NormalizePath(given)); + + // UNC only // + expected = sep; + given = sep + sep; + + ASSERT_EQ(expected, NormalizePath(given)); + + // 3 slashesy // + expected = sep + sep; + given = sep + sep + sep; + ASSERT_EQ(expected, NormalizePath(given)); + + // 3 slashes // + expected = sep + sep + "a" + sep; + given = sep + sep + sep + "a" + sep; + ASSERT_EQ(expected, NormalizePath(given)); + + // 2 separators in the middle + expected = "a" + sep + "b"; + given = "a" + sep + sep + "b"; + ASSERT_EQ(expected, NormalizePath(given)); + + // UNC with duplicate slashes + expected = sep + sep + "SERVER" + sep + "a" + sep + "b" + sep + "c"; + given = sep + sep + "SERVER" + sep + "a" + sep + sep + "b" + sep + "c"; + ASSERT_EQ(expected, NormalizePath(given)); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/flush_job.cc b/librocksdb-sys/rocksdb/db/flush_job.cc new file mode 100644 index 0000000..d3a777b --- /dev/null +++ b/librocksdb-sys/rocksdb/db/flush_job.cc @@ -0,0 +1,1168 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/flush_job.h" + +#include +#include +#include + +#include "db/builder.h" +#include "db/db_iter.h" +#include "db/dbformat.h" +#include "db/event_helpers.h" +#include "db/log_reader.h" +#include "db/log_writer.h" +#include "db/memtable.h" +#include "db/memtable_list.h" +#include "db/merge_context.h" +#include "db/range_tombstone_fragmenter.h" +#include "db/version_edit.h" +#include "db/version_set.h" +#include "file/file_util.h" +#include "file/filename.h" +#include "logging/event_logger.h" +#include "logging/log_buffer.h" +#include "logging/logging.h" +#include "monitoring/iostats_context_imp.h" +#include "monitoring/perf_context_imp.h" +#include "monitoring/thread_status_util.h" +#include "port/port.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/statistics.h" +#include "rocksdb/status.h" +#include "rocksdb/table.h" +#include "table/merging_iterator.h" +#include "table/table_builder.h" +#include "table/two_level_iterator.h" +#include "test_util/sync_point.h" +#include "util/coding.h" +#include "util/mutexlock.h" +#include "util/stop_watch.h" + +namespace ROCKSDB_NAMESPACE { + +const char* GetFlushReasonString(FlushReason flush_reason) { + switch (flush_reason) { + case FlushReason::kOthers: + return "Other Reasons"; + case FlushReason::kGetLiveFiles: + return "Get Live Files"; + case FlushReason::kShutDown: + return "Shut down"; + case FlushReason::kExternalFileIngestion: + return "External File Ingestion"; + case FlushReason::kManualCompaction: + return "Manual Compaction"; + case FlushReason::kWriteBufferManager: + return "Write Buffer Manager"; + case FlushReason::kWriteBufferFull: + return "Write Buffer Full"; + case FlushReason::kTest: + return "Test"; + case FlushReason::kDeleteFiles: + return "Delete Files"; + case FlushReason::kAutoCompaction: + return "Auto Compaction"; + case FlushReason::kManualFlush: + return "Manual Flush"; + case FlushReason::kErrorRecovery: + return "Error Recovery"; + case FlushReason::kErrorRecoveryRetryFlush: + return "Error Recovery Retry Flush"; + case FlushReason::kWalFull: + return "WAL Full"; + default: + return "Invalid"; + } +} + +FlushJob::FlushJob( + const std::string& dbname, ColumnFamilyData* cfd, + const ImmutableDBOptions& db_options, + const MutableCFOptions& mutable_cf_options, uint64_t max_memtable_id, + const FileOptions& file_options, VersionSet* versions, + InstrumentedMutex* db_mutex, std::atomic* shutting_down, + std::vector existing_snapshots, + SequenceNumber earliest_write_conflict_snapshot, + SnapshotChecker* snapshot_checker, JobContext* job_context, + FlushReason flush_reason, LogBuffer* log_buffer, FSDirectory* db_directory, + FSDirectory* output_file_directory, CompressionType output_compression, + Statistics* stats, EventLogger* event_logger, bool measure_io_stats, + const bool sync_output_directory, const bool write_manifest, + Env::Priority thread_pri, const std::shared_ptr& io_tracer, + const SeqnoToTimeMapping& seqno_time_mapping, const std::string& db_id, + const std::string& db_session_id, std::string full_history_ts_low, + BlobFileCompletionCallback* blob_callback) + : dbname_(dbname), + db_id_(db_id), + db_session_id_(db_session_id), + cfd_(cfd), + db_options_(db_options), + mutable_cf_options_(mutable_cf_options), + max_memtable_id_(max_memtable_id), + file_options_(file_options), + versions_(versions), + db_mutex_(db_mutex), + shutting_down_(shutting_down), + existing_snapshots_(std::move(existing_snapshots)), + earliest_write_conflict_snapshot_(earliest_write_conflict_snapshot), + snapshot_checker_(snapshot_checker), + job_context_(job_context), + flush_reason_(flush_reason), + log_buffer_(log_buffer), + db_directory_(db_directory), + output_file_directory_(output_file_directory), + output_compression_(output_compression), + stats_(stats), + event_logger_(event_logger), + measure_io_stats_(measure_io_stats), + sync_output_directory_(sync_output_directory), + write_manifest_(write_manifest), + edit_(nullptr), + base_(nullptr), + pick_memtable_called(false), + thread_pri_(thread_pri), + io_tracer_(io_tracer), + clock_(db_options_.clock), + full_history_ts_low_(std::move(full_history_ts_low)), + blob_callback_(blob_callback), + db_impl_seqno_time_mapping_(seqno_time_mapping) { + // Update the thread status to indicate flush. + ReportStartedFlush(); + TEST_SYNC_POINT("FlushJob::FlushJob()"); +} + +FlushJob::~FlushJob() { ThreadStatusUtil::ResetThreadStatus(); } + +void FlushJob::ReportStartedFlush() { + ThreadStatusUtil::SetEnableTracking(db_options_.enable_thread_tracking); + ThreadStatusUtil::SetColumnFamily(cfd_); + ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_FLUSH); + ThreadStatusUtil::SetThreadOperationProperty(ThreadStatus::COMPACTION_JOB_ID, + job_context_->job_id); + + IOSTATS_RESET(bytes_written); +} + +void FlushJob::ReportFlushInputSize(const autovector& mems) { + uint64_t input_size = 0; + for (auto* mem : mems) { + input_size += mem->ApproximateMemoryUsage(); + } + ThreadStatusUtil::IncreaseThreadOperationProperty( + ThreadStatus::FLUSH_BYTES_MEMTABLES, input_size); +} + +void FlushJob::RecordFlushIOStats() { + RecordTick(stats_, FLUSH_WRITE_BYTES, IOSTATS(bytes_written)); + ThreadStatusUtil::IncreaseThreadOperationProperty( + ThreadStatus::FLUSH_BYTES_WRITTEN, IOSTATS(bytes_written)); + IOSTATS_RESET(bytes_written); +} +void FlushJob::PickMemTable() { + db_mutex_->AssertHeld(); + assert(!pick_memtable_called); + pick_memtable_called = true; + + // Maximum "NextLogNumber" of the memtables to flush. + // When mempurge feature is turned off, this variable is useless + // because the memtables are implicitly sorted by increasing order of creation + // time. Therefore mems_->back()->GetNextLogNumber() is already equal to + // max_next_log_number. However when Mempurge is on, the memtables are no + // longer sorted by increasing order of creation time. Therefore this variable + // becomes necessary because mems_->back()->GetNextLogNumber() is no longer + // necessarily equal to max_next_log_number. + uint64_t max_next_log_number = 0; + + // Save the contents of the earliest memtable as a new Table + cfd_->imm()->PickMemtablesToFlush(max_memtable_id_, &mems_, + &max_next_log_number); + if (mems_.empty()) { + return; + } + + // Track effective cutoff user-defined timestamp during flush if + // user-defined timestamps can be stripped. + GetEffectiveCutoffUDTForPickedMemTables(); + + ReportFlushInputSize(mems_); + + // entries mems are (implicitly) sorted in ascending order by their created + // time. We will use the first memtable's `edit` to keep the meta info for + // this flush. + MemTable* m = mems_[0]; + edit_ = m->GetEdits(); + edit_->SetPrevLogNumber(0); + // SetLogNumber(log_num) indicates logs with number smaller than log_num + // will no longer be picked up for recovery. + edit_->SetLogNumber(max_next_log_number); + edit_->SetColumnFamily(cfd_->GetID()); + + // path 0 for level 0 file. + meta_.fd = FileDescriptor(versions_->NewFileNumber(), 0, 0); + meta_.epoch_number = cfd_->NewEpochNumber(); + + base_ = cfd_->current(); + base_->Ref(); // it is likely that we do not need this reference +} + +Status FlushJob::Run(LogsWithPrepTracker* prep_tracker, FileMetaData* file_meta, + bool* switched_to_mempurge) { + TEST_SYNC_POINT("FlushJob::Start"); + db_mutex_->AssertHeld(); + assert(pick_memtable_called); + // Mempurge threshold can be dynamically changed. + // For sake of consistency, mempurge_threshold is + // saved locally to maintain consistency in each + // FlushJob::Run call. + double mempurge_threshold = + mutable_cf_options_.experimental_mempurge_threshold; + + AutoThreadOperationStageUpdater stage_run(ThreadStatus::STAGE_FLUSH_RUN); + if (mems_.empty()) { + ROCKS_LOG_BUFFER(log_buffer_, "[%s] Nothing in memtable to flush", + cfd_->GetName().c_str()); + return Status::OK(); + } + + // I/O measurement variables + PerfLevel prev_perf_level = PerfLevel::kEnableTime; + uint64_t prev_write_nanos = 0; + uint64_t prev_fsync_nanos = 0; + uint64_t prev_range_sync_nanos = 0; + uint64_t prev_prepare_write_nanos = 0; + uint64_t prev_cpu_write_nanos = 0; + uint64_t prev_cpu_read_nanos = 0; + if (measure_io_stats_) { + prev_perf_level = GetPerfLevel(); + SetPerfLevel(PerfLevel::kEnableTime); + prev_write_nanos = IOSTATS(write_nanos); + prev_fsync_nanos = IOSTATS(fsync_nanos); + prev_range_sync_nanos = IOSTATS(range_sync_nanos); + prev_prepare_write_nanos = IOSTATS(prepare_write_nanos); + prev_cpu_write_nanos = IOSTATS(cpu_write_nanos); + prev_cpu_read_nanos = IOSTATS(cpu_read_nanos); + } + Status mempurge_s = Status::NotFound("No MemPurge."); + if ((mempurge_threshold > 0.0) && + (flush_reason_ == FlushReason::kWriteBufferFull) && (!mems_.empty()) && + MemPurgeDecider(mempurge_threshold) && !(db_options_.atomic_flush)) { + cfd_->SetMempurgeUsed(); + mempurge_s = MemPurge(); + if (!mempurge_s.ok()) { + // Mempurge is typically aborted when the output + // bytes cannot be contained onto a single output memtable. + if (mempurge_s.IsAborted()) { + ROCKS_LOG_INFO(db_options_.info_log, "Mempurge process aborted: %s\n", + mempurge_s.ToString().c_str()); + } else { + // However the mempurge process can also fail for + // other reasons (eg: new_mem->Add() fails). + ROCKS_LOG_WARN(db_options_.info_log, "Mempurge process failed: %s\n", + mempurge_s.ToString().c_str()); + } + } else { + if (switched_to_mempurge) { + *switched_to_mempurge = true; + } else { + // The mempurge process was successful, but no switch_to_mempurge + // pointer provided so no way to propagate the state of flush job. + ROCKS_LOG_WARN(db_options_.info_log, + "Mempurge process succeeded" + "but no 'switched_to_mempurge' ptr provided.\n"); + } + } + } + Status s; + if (mempurge_s.ok()) { + base_->Unref(); + s = Status::OK(); + } else { + // This will release and re-acquire the mutex. + s = WriteLevel0Table(); + } + + if (s.ok() && cfd_->IsDropped()) { + s = Status::ColumnFamilyDropped("Column family dropped during compaction"); + } + if ((s.ok() || s.IsColumnFamilyDropped()) && + shutting_down_->load(std::memory_order_acquire)) { + s = Status::ShutdownInProgress("Database shutdown"); + } + + if (s.ok()) { + s = MaybeIncreaseFullHistoryTsLowToAboveCutoffUDT(); + } + + if (!s.ok()) { + cfd_->imm()->RollbackMemtableFlush(mems_, meta_.fd.GetNumber()); + } else if (write_manifest_) { + TEST_SYNC_POINT("FlushJob::InstallResults"); + // Replace immutable memtable with the generated Table + s = cfd_->imm()->TryInstallMemtableFlushResults( + cfd_, mutable_cf_options_, mems_, prep_tracker, versions_, db_mutex_, + meta_.fd.GetNumber(), &job_context_->memtables_to_free, db_directory_, + log_buffer_, &committed_flush_jobs_info_, + !(mempurge_s.ok()) /* write_edit : true if no mempurge happened (or if aborted), + but 'false' if mempurge successful: no new min log number + or new level 0 file path to write to manifest. */); + } + + if (s.ok() && file_meta != nullptr) { + *file_meta = meta_; + } + RecordFlushIOStats(); + + // When measure_io_stats_ is true, the default 512 bytes is not enough. + auto stream = event_logger_->LogToBuffer(log_buffer_, 1024); + stream << "job" << job_context_->job_id << "event" + << "flush_finished"; + stream << "output_compression" + << CompressionTypeToString(output_compression_); + stream << "lsm_state"; + stream.StartArray(); + auto vstorage = cfd_->current()->storage_info(); + for (int level = 0; level < vstorage->num_levels(); ++level) { + stream << vstorage->NumLevelFiles(level); + } + stream.EndArray(); + + const auto& blob_files = vstorage->GetBlobFiles(); + if (!blob_files.empty()) { + assert(blob_files.front()); + stream << "blob_file_head" << blob_files.front()->GetBlobFileNumber(); + + assert(blob_files.back()); + stream << "blob_file_tail" << blob_files.back()->GetBlobFileNumber(); + } + + stream << "immutable_memtables" << cfd_->imm()->NumNotFlushed(); + + if (measure_io_stats_) { + if (prev_perf_level != PerfLevel::kEnableTime) { + SetPerfLevel(prev_perf_level); + } + stream << "file_write_nanos" << (IOSTATS(write_nanos) - prev_write_nanos); + stream << "file_range_sync_nanos" + << (IOSTATS(range_sync_nanos) - prev_range_sync_nanos); + stream << "file_fsync_nanos" << (IOSTATS(fsync_nanos) - prev_fsync_nanos); + stream << "file_prepare_write_nanos" + << (IOSTATS(prepare_write_nanos) - prev_prepare_write_nanos); + stream << "file_cpu_write_nanos" + << (IOSTATS(cpu_write_nanos) - prev_cpu_write_nanos); + stream << "file_cpu_read_nanos" + << (IOSTATS(cpu_read_nanos) - prev_cpu_read_nanos); + } + + TEST_SYNC_POINT("FlushJob::End"); + return s; +} + +void FlushJob::Cancel() { + db_mutex_->AssertHeld(); + assert(base_ != nullptr); + base_->Unref(); +} + +Status FlushJob::MemPurge() { + Status s; + db_mutex_->AssertHeld(); + db_mutex_->Unlock(); + assert(!mems_.empty()); + + // Measure purging time. + const uint64_t start_micros = clock_->NowMicros(); + const uint64_t start_cpu_micros = clock_->CPUMicros(); + + MemTable* new_mem = nullptr; + // For performance/log investigation purposes: + // look at how much useful payload we harvest in the new_mem. + // This value is then printed to the DB log. + double new_mem_capacity = 0.0; + + // Create two iterators, one for the memtable data (contains + // info from puts + deletes), and one for the memtable + // Range Tombstones (from DeleteRanges). + // TODO: plumb Env::IOActivity + ReadOptions ro; + ro.total_order_seek = true; + Arena arena; + std::vector memtables; + std::vector> + range_del_iters; + for (MemTable* m : mems_) { + memtables.push_back(m->NewIterator(ro, &arena)); + auto* range_del_iter = m->NewRangeTombstoneIterator( + ro, kMaxSequenceNumber, true /* immutable_memtable */); + if (range_del_iter != nullptr) { + range_del_iters.emplace_back(range_del_iter); + } + } + + assert(!memtables.empty()); + SequenceNumber first_seqno = kMaxSequenceNumber; + SequenceNumber earliest_seqno = kMaxSequenceNumber; + // Pick first and earliest seqno as min of all first_seqno + // and earliest_seqno of the mempurged memtables. + for (const auto& mem : mems_) { + first_seqno = mem->GetFirstSequenceNumber() < first_seqno + ? mem->GetFirstSequenceNumber() + : first_seqno; + earliest_seqno = mem->GetEarliestSequenceNumber() < earliest_seqno + ? mem->GetEarliestSequenceNumber() + : earliest_seqno; + } + + ScopedArenaIterator iter( + NewMergingIterator(&(cfd_->internal_comparator()), memtables.data(), + static_cast(memtables.size()), &arena)); + + auto* ioptions = cfd_->ioptions(); + + // Place iterator at the First (meaning most recent) key node. + iter->SeekToFirst(); + + const std::string* const full_history_ts_low = &(cfd_->GetFullHistoryTsLow()); + std::unique_ptr range_del_agg( + new CompactionRangeDelAggregator(&(cfd_->internal_comparator()), + existing_snapshots_, + full_history_ts_low)); + for (auto& rd_iter : range_del_iters) { + range_del_agg->AddTombstones(std::move(rd_iter)); + } + + // If there is valid data in the memtable, + // or at least range tombstones, copy over the info + // to the new memtable. + if (iter->Valid() || !range_del_agg->IsEmpty()) { + // MaxSize is the size of a memtable. + size_t maxSize = mutable_cf_options_.write_buffer_size; + std::unique_ptr compaction_filter; + if (ioptions->compaction_filter_factory != nullptr && + ioptions->compaction_filter_factory->ShouldFilterTableFileCreation( + TableFileCreationReason::kFlush)) { + CompactionFilter::Context ctx; + ctx.is_full_compaction = false; + ctx.is_manual_compaction = false; + ctx.column_family_id = cfd_->GetID(); + ctx.reason = TableFileCreationReason::kFlush; + compaction_filter = + ioptions->compaction_filter_factory->CreateCompactionFilter(ctx); + if (compaction_filter != nullptr && + !compaction_filter->IgnoreSnapshots()) { + s = Status::NotSupported( + "CompactionFilter::IgnoreSnapshots() = false is not supported " + "anymore."); + return s; + } + } + + new_mem = new MemTable((cfd_->internal_comparator()), *(cfd_->ioptions()), + mutable_cf_options_, cfd_->write_buffer_mgr(), + earliest_seqno, cfd_->GetID()); + assert(new_mem != nullptr); + + Env* env = db_options_.env; + assert(env); + MergeHelper merge( + env, (cfd_->internal_comparator()).user_comparator(), + (ioptions->merge_operator).get(), compaction_filter.get(), + ioptions->logger, true /* internal key corruption is not ok */, + existing_snapshots_.empty() ? 0 : existing_snapshots_.back(), + snapshot_checker_); + assert(job_context_); + SequenceNumber job_snapshot_seq = job_context_->GetJobSnapshotSequence(); + const std::atomic kManualCompactionCanceledFalse{false}; + CompactionIterator c_iter( + iter.get(), (cfd_->internal_comparator()).user_comparator(), &merge, + kMaxSequenceNumber, &existing_snapshots_, + earliest_write_conflict_snapshot_, job_snapshot_seq, snapshot_checker_, + env, ShouldReportDetailedTime(env, ioptions->stats), + true /* internal key corruption is not ok */, range_del_agg.get(), + nullptr, ioptions->allow_data_in_errors, + ioptions->enforce_single_del_contracts, + /*manual_compaction_canceled=*/kManualCompactionCanceledFalse, + false /* must_count_input_entries */, + /*compaction=*/nullptr, compaction_filter.get(), + /*shutting_down=*/nullptr, ioptions->info_log, full_history_ts_low); + + // Set earliest sequence number in the new memtable + // to be equal to the earliest sequence number of the + // memtable being flushed (See later if there is a need + // to update this number!). + new_mem->SetEarliestSequenceNumber(earliest_seqno); + // Likewise for first seq number. + new_mem->SetFirstSequenceNumber(first_seqno); + SequenceNumber new_first_seqno = kMaxSequenceNumber; + + c_iter.SeekToFirst(); + + // Key transfer + for (; c_iter.Valid(); c_iter.Next()) { + const ParsedInternalKey ikey = c_iter.ikey(); + const Slice value = c_iter.value(); + new_first_seqno = + ikey.sequence < new_first_seqno ? ikey.sequence : new_first_seqno; + + // Should we update "OldestKeyTime" ???? -> timestamp appear + // to still be an "experimental" feature. + s = new_mem->Add( + ikey.sequence, ikey.type, ikey.user_key, value, + nullptr, // KV protection info set as nullptr since it + // should only be useful for the first add to + // the original memtable. + false, // : allow concurrent_memtable_writes_ + // Not seen as necessary for now. + nullptr, // get_post_process_info(m) must be nullptr + // when concurrent_memtable_writes is switched off. + nullptr); // hint, only used when concurrent_memtable_writes_ + // is switched on. + if (!s.ok()) { + break; + } + + // If new_mem has size greater than maxSize, + // then rollback to regular flush operation, + // and destroy new_mem. + if (new_mem->ApproximateMemoryUsage() > maxSize) { + s = Status::Aborted("Mempurge filled more than one memtable."); + new_mem_capacity = 1.0; + break; + } + } + + // Check status and propagate + // potential error status from c_iter + if (!s.ok()) { + c_iter.status().PermitUncheckedError(); + } else if (!c_iter.status().ok()) { + s = c_iter.status(); + } + + // Range tombstone transfer. + if (s.ok()) { + auto range_del_it = range_del_agg->NewIterator(); + for (range_del_it->SeekToFirst(); range_del_it->Valid(); + range_del_it->Next()) { + auto tombstone = range_del_it->Tombstone(); + new_first_seqno = + tombstone.seq_ < new_first_seqno ? tombstone.seq_ : new_first_seqno; + s = new_mem->Add( + tombstone.seq_, // Sequence number + kTypeRangeDeletion, // KV type + tombstone.start_key_, // Key is start key. + tombstone.end_key_, // Value is end key. + nullptr, // KV protection info set as nullptr since it + // should only be useful for the first add to + // the original memtable. + false, // : allow concurrent_memtable_writes_ + // Not seen as necessary for now. + nullptr, // get_post_process_info(m) must be nullptr + // when concurrent_memtable_writes is switched off. + nullptr); // hint, only used when concurrent_memtable_writes_ + // is switched on. + + if (!s.ok()) { + break; + } + + // If new_mem has size greater than maxSize, + // then rollback to regular flush operation, + // and destroy new_mem. + if (new_mem->ApproximateMemoryUsage() > maxSize) { + s = Status::Aborted(Slice("Mempurge filled more than one memtable.")); + new_mem_capacity = 1.0; + break; + } + } + } + + // If everything happened smoothly and new_mem contains valid data, + // decide if it is flushed to storage or kept in the imm() + // memtable list (memory). + if (s.ok() && (new_first_seqno != kMaxSequenceNumber)) { + // Rectify the first sequence number, which (unlike the earliest seq + // number) needs to be present in the new memtable. + new_mem->SetFirstSequenceNumber(new_first_seqno); + + // The new_mem is added to the list of immutable memtables + // only if it filled at less than 100% capacity and isn't flagged + // as in need of being flushed. + if (new_mem->ApproximateMemoryUsage() < maxSize && + !(new_mem->ShouldFlushNow())) { + // Construct fragmented memtable range tombstones without mutex + new_mem->ConstructFragmentedRangeTombstones(); + db_mutex_->Lock(); + uint64_t new_mem_id = mems_[0]->GetID(); + + new_mem->SetID(new_mem_id); + new_mem->SetNextLogNumber(mems_[0]->GetNextLogNumber()); + + // This addition will not trigger another flush, because + // we do not call SchedulePendingFlush(). + cfd_->imm()->Add(new_mem, &job_context_->memtables_to_free); + new_mem->Ref(); + // Piggyback FlushJobInfo on the first flushed memtable. + db_mutex_->AssertHeld(); + meta_.fd.file_size = 0; + mems_[0]->SetFlushJobInfo(GetFlushJobInfo()); + db_mutex_->Unlock(); + } else { + s = Status::Aborted(Slice("Mempurge filled more than one memtable.")); + new_mem_capacity = 1.0; + if (new_mem) { + job_context_->memtables_to_free.push_back(new_mem); + } + } + } else { + // In this case, the newly allocated new_mem is empty. + assert(new_mem != nullptr); + job_context_->memtables_to_free.push_back(new_mem); + } + } + + // Reacquire the mutex for WriteLevel0 function. + db_mutex_->Lock(); + + // If mempurge successful, don't write input tables to level0, + // but write any full output table to level0. + if (s.ok()) { + TEST_SYNC_POINT("DBImpl::FlushJob:MemPurgeSuccessful"); + } else { + TEST_SYNC_POINT("DBImpl::FlushJob:MemPurgeUnsuccessful"); + } + const uint64_t micros = clock_->NowMicros() - start_micros; + const uint64_t cpu_micros = clock_->CPUMicros() - start_cpu_micros; + ROCKS_LOG_INFO(db_options_.info_log, + "[%s] [JOB %d] Mempurge lasted %" PRIu64 + " microseconds, and %" PRIu64 + " cpu " + "microseconds. Status is %s ok. Perc capacity: %f\n", + cfd_->GetName().c_str(), job_context_->job_id, micros, + cpu_micros, s.ok() ? "" : "not", new_mem_capacity); + + return s; +} + +bool FlushJob::MemPurgeDecider(double threshold) { + // Never trigger mempurge if threshold is not a strictly positive value. + if (!(threshold > 0.0)) { + return false; + } + if (threshold > (1.0 * mems_.size())) { + return true; + } + // Payload and useful_payload (in bytes). + // The useful payload ratio of a given MemTable + // is estimated to be useful_payload/payload. + uint64_t payload = 0, useful_payload = 0, entry_size = 0; + + // Local variables used repetitively inside the for-loop + // when iterating over the sampled entries. + Slice key_slice, value_slice; + ParsedInternalKey res; + SnapshotImpl min_snapshot; + std::string vget; + Status mget_s, parse_s; + MergeContext merge_context; + SequenceNumber max_covering_tombstone_seq = 0, sqno = 0, + min_seqno_snapshot = 0; + bool get_res, can_be_useful_payload, not_in_next_mems; + + // If estimated_useful_payload is > threshold, + // then flush to storage, else MemPurge. + double estimated_useful_payload = 0.0; + // Cochran formula for determining sample size. + // 95% confidence interval, 7% precision. + // n0 = (1.96*1.96)*0.25/(0.07*0.07) = 196.0 + // TODO: plumb Env::IOActivity + double n0 = 196.0; + ReadOptions ro; + ro.total_order_seek = true; + + // Iterate over each memtable of the set. + for (auto mem_iter = std::begin(mems_); mem_iter != std::end(mems_); + mem_iter++) { + MemTable* mt = *mem_iter; + + // Else sample from the table. + uint64_t nentries = mt->num_entries(); + // Corrected Cochran formula for small populations + // (converges to n0 for large populations). + uint64_t target_sample_size = + static_cast(ceil(n0 / (1.0 + (n0 / nentries)))); + std::unordered_set sentries = {}; + // Populate sample entries set. + mt->UniqueRandomSample(target_sample_size, &sentries); + + // Estimate the garbage ratio by comparing if + // each sample corresponds to a valid entry. + for (const char* ss : sentries) { + key_slice = GetLengthPrefixedSlice(ss); + parse_s = ParseInternalKey(key_slice, &res, true /*log_err_key*/); + if (!parse_s.ok()) { + ROCKS_LOG_WARN(db_options_.info_log, + "Memtable Decider: ParseInternalKey did not parse " + "key_slice %s successfully.", + key_slice.data()); + } + + // Size of the entry is "key size (+ value size if KV entry)" + entry_size = key_slice.size(); + if (res.type == kTypeValue) { + value_slice = + GetLengthPrefixedSlice(key_slice.data() + key_slice.size()); + entry_size += value_slice.size(); + } + + // Count entry bytes as payload. + payload += entry_size; + + LookupKey lkey(res.user_key, kMaxSequenceNumber); + + // Paranoia: zero out these values just in case. + max_covering_tombstone_seq = 0; + sqno = 0; + + // Pick the oldest existing snapshot that is more recent + // than the sequence number of the sampled entry. + min_seqno_snapshot = kMaxSequenceNumber; + for (SequenceNumber seq_num : existing_snapshots_) { + if (seq_num > res.sequence && seq_num < min_seqno_snapshot) { + min_seqno_snapshot = seq_num; + } + } + min_snapshot.number_ = min_seqno_snapshot; + ro.snapshot = + min_seqno_snapshot < kMaxSequenceNumber ? &min_snapshot : nullptr; + + // Estimate if the sample entry is valid or not. + get_res = mt->Get(lkey, &vget, /*columns=*/nullptr, /*timestamp=*/nullptr, + &mget_s, &merge_context, &max_covering_tombstone_seq, + &sqno, ro, true /* immutable_memtable */); + if (!get_res) { + ROCKS_LOG_WARN( + db_options_.info_log, + "Memtable Get returned false when Get(sampled entry). " + "Yet each sample entry should exist somewhere in the memtable, " + "unrelated to whether it has been deleted or not."); + } + + // TODO(bjlemaire): evaluate typeMerge. + // This is where the sampled entry is estimated to be + // garbage or not. Note that this is a garbage *estimation* + // because we do not include certain items such as + // CompactionFitlers triggered at flush, or if the same delete + // has been inserted twice or more in the memtable. + + // Evaluate if the entry can be useful payload + // Situation #1: entry is a KV entry, was found in the memtable mt + // and the sequence numbers match. + can_be_useful_payload = (res.type == kTypeValue) && get_res && + mget_s.ok() && (sqno == res.sequence); + + // Situation #2: entry is a delete entry, was found in the memtable mt + // (because gres==true) and no valid KV entry is found. + // (note: duplicate delete entries are also taken into + // account here, because the sequence number 'sqno' + // in memtable->Get(&sqno) operation is set to be equal + // to the most recent delete entry as well). + can_be_useful_payload |= + ((res.type == kTypeDeletion) || (res.type == kTypeSingleDeletion)) && + mget_s.IsNotFound() && get_res && (sqno == res.sequence); + + // If there is a chance that the entry is useful payload + // Verify that the entry does not appear in the following memtables + // (memtables with greater memtable ID/larger sequence numbers). + if (can_be_useful_payload) { + not_in_next_mems = true; + for (auto next_mem_iter = mem_iter + 1; + next_mem_iter != std::end(mems_); next_mem_iter++) { + if ((*next_mem_iter) + ->Get(lkey, &vget, /*columns=*/nullptr, /*timestamp=*/nullptr, + &mget_s, &merge_context, &max_covering_tombstone_seq, + &sqno, ro, true /* immutable_memtable */)) { + not_in_next_mems = false; + break; + } + } + if (not_in_next_mems) { + useful_payload += entry_size; + } + } + } + if (payload > 0) { + // We use the estimated useful payload ratio to + // evaluate how many of the memtable bytes are useful bytes. + estimated_useful_payload += + (mt->ApproximateMemoryUsage()) * (useful_payload * 1.0 / payload); + + ROCKS_LOG_INFO(db_options_.info_log, + "Mempurge sampling [CF %s] - found garbage ratio from " + "sampling: %f. Threshold is %f\n", + cfd_->GetName().c_str(), + (payload - useful_payload) * 1.0 / payload, threshold); + } else { + ROCKS_LOG_WARN(db_options_.info_log, + "Mempurge sampling: null payload measured, and collected " + "sample size is %zu\n.", + sentries.size()); + } + } + // We convert the total number of useful payload bytes + // into the proportion of memtable necessary to store all these bytes. + // We compare this proportion with the threshold value. + return ((estimated_useful_payload / mutable_cf_options_.write_buffer_size) < + threshold); +} + +Status FlushJob::WriteLevel0Table() { + AutoThreadOperationStageUpdater stage_updater( + ThreadStatus::STAGE_FLUSH_WRITE_L0); + db_mutex_->AssertHeld(); + const uint64_t start_micros = clock_->NowMicros(); + const uint64_t start_cpu_micros = clock_->CPUMicros(); + Status s; + + SequenceNumber smallest_seqno = mems_.front()->GetEarliestSequenceNumber(); + if (!db_impl_seqno_time_mapping_.Empty()) { + // make a local copy, as the seqno_time_mapping from db_impl is not thread + // safe, which will be used while not holding the db_mutex. + seqno_to_time_mapping_ = db_impl_seqno_time_mapping_.Copy(smallest_seqno); + } + + std::vector blob_file_additions; + + { + auto write_hint = cfd_->CalculateSSTWriteHint(0); + Env::IOPriority io_priority = GetRateLimiterPriorityForWrite(); + db_mutex_->Unlock(); + if (log_buffer_) { + log_buffer_->FlushBufferToLog(); + } + // memtables and range_del_iters store internal iterators over each data + // memtable and its associated range deletion memtable, respectively, at + // corresponding indexes. + std::vector memtables; + std::vector> + range_del_iters; + ReadOptions ro; + ro.total_order_seek = true; + ro.io_activity = Env::IOActivity::kFlush; + Arena arena; + uint64_t total_num_entries = 0, total_num_deletes = 0; + uint64_t total_data_size = 0; + size_t total_memory_usage = 0; + uint64_t total_num_range_deletes = 0; + // Used for testing: + uint64_t mems_size = mems_.size(); + (void)mems_size; // avoids unused variable error when + // TEST_SYNC_POINT_CALLBACK not used. + TEST_SYNC_POINT_CALLBACK("FlushJob::WriteLevel0Table:num_memtables", + &mems_size); + assert(job_context_); + for (MemTable* m : mems_) { + ROCKS_LOG_INFO( + db_options_.info_log, + "[%s] [JOB %d] Flushing memtable with next log file: %" PRIu64 "\n", + cfd_->GetName().c_str(), job_context_->job_id, m->GetNextLogNumber()); + memtables.push_back(m->NewIterator(ro, &arena)); + auto* range_del_iter = m->NewRangeTombstoneIterator( + ro, kMaxSequenceNumber, true /* immutable_memtable */); + if (range_del_iter != nullptr) { + range_del_iters.emplace_back(range_del_iter); + } + total_num_entries += m->num_entries(); + total_num_deletes += m->num_deletes(); + total_data_size += m->get_data_size(); + total_memory_usage += m->ApproximateMemoryUsage(); + total_num_range_deletes += m->num_range_deletes(); + } + + // TODO(cbi): when memtable is flushed due to number of range deletions + // hitting limit memtable_max_range_deletions, flush_reason_ is still + // "Write Buffer Full", should make update flush_reason_ accordingly. + event_logger_->Log() << "job" << job_context_->job_id << "event" + << "flush_started" + << "num_memtables" << mems_.size() << "num_entries" + << total_num_entries << "num_deletes" + << total_num_deletes << "total_data_size" + << total_data_size << "memory_usage" + << total_memory_usage << "num_range_deletes" + << total_num_range_deletes << "flush_reason" + << GetFlushReasonString(flush_reason_); + + { + ScopedArenaIterator iter( + NewMergingIterator(&cfd_->internal_comparator(), memtables.data(), + static_cast(memtables.size()), &arena)); + ROCKS_LOG_INFO(db_options_.info_log, + "[%s] [JOB %d] Level-0 flush table #%" PRIu64 ": started", + cfd_->GetName().c_str(), job_context_->job_id, + meta_.fd.GetNumber()); + + TEST_SYNC_POINT_CALLBACK("FlushJob::WriteLevel0Table:output_compression", + &output_compression_); + int64_t _current_time = 0; + auto status = clock_->GetCurrentTime(&_current_time); + // Safe to proceed even if GetCurrentTime fails. So, log and proceed. + if (!status.ok()) { + ROCKS_LOG_WARN( + db_options_.info_log, + "Failed to get current time to populate creation_time property. " + "Status: %s", + status.ToString().c_str()); + } + const uint64_t current_time = static_cast(_current_time); + + uint64_t oldest_key_time = mems_.front()->ApproximateOldestKeyTime(); + + // It's not clear whether oldest_key_time is always available. In case + // it is not available, use current_time. + uint64_t oldest_ancester_time = std::min(current_time, oldest_key_time); + + TEST_SYNC_POINT_CALLBACK( + "FlushJob::WriteLevel0Table:oldest_ancester_time", + &oldest_ancester_time); + meta_.oldest_ancester_time = oldest_ancester_time; + meta_.file_creation_time = current_time; + + uint64_t num_input_entries = 0; + uint64_t memtable_payload_bytes = 0; + uint64_t memtable_garbage_bytes = 0; + IOStatus io_s; + + const std::string* const full_history_ts_low = + (full_history_ts_low_.empty()) ? nullptr : &full_history_ts_low_; + TableBuilderOptions tboptions( + *cfd_->ioptions(), mutable_cf_options_, cfd_->internal_comparator(), + cfd_->int_tbl_prop_collector_factories(), output_compression_, + mutable_cf_options_.compression_opts, cfd_->GetID(), cfd_->GetName(), + 0 /* level */, false /* is_bottommost */, + TableFileCreationReason::kFlush, oldest_key_time, current_time, + db_id_, db_session_id_, 0 /* target_file_size */, + meta_.fd.GetNumber()); + const SequenceNumber job_snapshot_seq = + job_context_->GetJobSnapshotSequence(); + const ReadOptions read_options(Env::IOActivity::kFlush); + s = BuildTable(dbname_, versions_, db_options_, tboptions, file_options_, + read_options, cfd_->table_cache(), iter.get(), + std::move(range_del_iters), &meta_, &blob_file_additions, + existing_snapshots_, earliest_write_conflict_snapshot_, + job_snapshot_seq, snapshot_checker_, + mutable_cf_options_.paranoid_file_checks, + cfd_->internal_stats(), &io_s, io_tracer_, + BlobFileCreationReason::kFlush, seqno_to_time_mapping_, + event_logger_, job_context_->job_id, io_priority, + &table_properties_, write_hint, full_history_ts_low, + blob_callback_, base_, &num_input_entries, + &memtable_payload_bytes, &memtable_garbage_bytes); + // TODO: Cleanup io_status in BuildTable and table builders + assert(!s.ok() || io_s.ok()); + io_s.PermitUncheckedError(); + if (num_input_entries != total_num_entries && s.ok()) { + std::string msg = "Expected " + std::to_string(total_num_entries) + + " entries in memtables, but read " + + std::to_string(num_input_entries); + ROCKS_LOG_WARN(db_options_.info_log, "[%s] [JOB %d] Level-0 flush %s", + cfd_->GetName().c_str(), job_context_->job_id, + msg.c_str()); + if (db_options_.flush_verify_memtable_count) { + s = Status::Corruption(msg); + } + } + if (tboptions.reason == TableFileCreationReason::kFlush) { + TEST_SYNC_POINT("DBImpl::FlushJob:Flush"); + RecordTick(stats_, MEMTABLE_PAYLOAD_BYTES_AT_FLUSH, + memtable_payload_bytes); + RecordTick(stats_, MEMTABLE_GARBAGE_BYTES_AT_FLUSH, + memtable_garbage_bytes); + } + LogFlush(db_options_.info_log); + } + ROCKS_LOG_BUFFER(log_buffer_, + "[%s] [JOB %d] Level-0 flush table #%" PRIu64 ": %" PRIu64 + " bytes %s" + "%s", + cfd_->GetName().c_str(), job_context_->job_id, + meta_.fd.GetNumber(), meta_.fd.GetFileSize(), + s.ToString().c_str(), + meta_.marked_for_compaction ? " (needs compaction)" : ""); + + if (s.ok() && output_file_directory_ != nullptr && sync_output_directory_) { + s = output_file_directory_->FsyncWithDirOptions( + IOOptions(), nullptr, + DirFsyncOptions(DirFsyncOptions::FsyncReason::kNewFileSynced)); + } + TEST_SYNC_POINT_CALLBACK("FlushJob::WriteLevel0Table", &mems_); + db_mutex_->Lock(); + } + base_->Unref(); + + // Note that if file_size is zero, the file has been deleted and + // should not be added to the manifest. + const bool has_output = meta_.fd.GetFileSize() > 0; + + if (s.ok() && has_output) { + TEST_SYNC_POINT("DBImpl::FlushJob:SSTFileCreated"); + // if we have more than 1 background thread, then we cannot + // insert files directly into higher levels because some other + // threads could be concurrently producing compacted files for + // that key range. + // Add file to L0 + edit_->AddFile(0 /* level */, meta_.fd.GetNumber(), meta_.fd.GetPathId(), + meta_.fd.GetFileSize(), meta_.smallest, meta_.largest, + meta_.fd.smallest_seqno, meta_.fd.largest_seqno, + meta_.marked_for_compaction, meta_.temperature, + meta_.oldest_blob_file_number, meta_.oldest_ancester_time, + meta_.file_creation_time, meta_.epoch_number, + meta_.file_checksum, meta_.file_checksum_func_name, + meta_.unique_id, meta_.compensated_range_deletion_size, + meta_.tail_size, meta_.user_defined_timestamps_persisted); + edit_->SetBlobFileAdditions(std::move(blob_file_additions)); + } + // Piggyback FlushJobInfo on the first first flushed memtable. + mems_[0]->SetFlushJobInfo(GetFlushJobInfo()); + + // Note that here we treat flush as level 0 compaction in internal stats + InternalStats::CompactionStats stats(CompactionReason::kFlush, 1); + const uint64_t micros = clock_->NowMicros() - start_micros; + const uint64_t cpu_micros = clock_->CPUMicros() - start_cpu_micros; + stats.micros = micros; + stats.cpu_micros = cpu_micros; + + ROCKS_LOG_INFO(db_options_.info_log, + "[%s] [JOB %d] Flush lasted %" PRIu64 + " microseconds, and %" PRIu64 " cpu microseconds.\n", + cfd_->GetName().c_str(), job_context_->job_id, micros, + cpu_micros); + + if (has_output) { + stats.bytes_written = meta_.fd.GetFileSize(); + stats.num_output_files = 1; + } + + const auto& blobs = edit_->GetBlobFileAdditions(); + for (const auto& blob : blobs) { + stats.bytes_written_blob += blob.GetTotalBlobBytes(); + } + + stats.num_output_files_blob = static_cast(blobs.size()); + + RecordTimeToHistogram(stats_, FLUSH_TIME, stats.micros); + cfd_->internal_stats()->AddCompactionStats(0 /* level */, thread_pri_, stats); + cfd_->internal_stats()->AddCFStats( + InternalStats::BYTES_FLUSHED, + stats.bytes_written + stats.bytes_written_blob); + RecordFlushIOStats(); + + return s; +} + +Env::IOPriority FlushJob::GetRateLimiterPriorityForWrite() { + if (versions_ && versions_->GetColumnFamilySet() && + versions_->GetColumnFamilySet()->write_controller()) { + WriteController* write_controller = + versions_->GetColumnFamilySet()->write_controller(); + if (write_controller->IsStopped() || write_controller->NeedsDelay()) { + return Env::IO_USER; + } + } + + return Env::IO_HIGH; +} + +std::unique_ptr FlushJob::GetFlushJobInfo() const { + db_mutex_->AssertHeld(); + std::unique_ptr info(new FlushJobInfo{}); + info->cf_id = cfd_->GetID(); + info->cf_name = cfd_->GetName(); + + const uint64_t file_number = meta_.fd.GetNumber(); + info->file_path = + MakeTableFileName(cfd_->ioptions()->cf_paths[0].path, file_number); + info->file_number = file_number; + info->oldest_blob_file_number = meta_.oldest_blob_file_number; + info->thread_id = db_options_.env->GetThreadID(); + info->job_id = job_context_->job_id; + info->smallest_seqno = meta_.fd.smallest_seqno; + info->largest_seqno = meta_.fd.largest_seqno; + info->table_properties = table_properties_; + info->flush_reason = flush_reason_; + info->blob_compression_type = mutable_cf_options_.blob_compression_type; + + // Update BlobFilesInfo. + for (const auto& blob_file : edit_->GetBlobFileAdditions()) { + BlobFileAdditionInfo blob_file_addition_info( + BlobFileName(cfd_->ioptions()->cf_paths.front().path, + blob_file.GetBlobFileNumber()) /*blob_file_path*/, + blob_file.GetBlobFileNumber(), blob_file.GetTotalBlobCount(), + blob_file.GetTotalBlobBytes()); + info->blob_file_addition_infos.emplace_back( + std::move(blob_file_addition_info)); + } + return info; +} + +void FlushJob::GetEffectiveCutoffUDTForPickedMemTables() { + db_mutex_->AssertHeld(); + assert(pick_memtable_called); + const auto* ucmp = cfd_->internal_comparator().user_comparator(); + assert(ucmp); + const size_t ts_sz = ucmp->timestamp_size(); + if (db_options_.atomic_flush || ts_sz == 0 || + cfd_->ioptions()->persist_user_defined_timestamps) { + return; + } + for (MemTable* m : mems_) { + Slice table_newest_udt = m->GetNewestUDT(); + // The picked Memtables should have ascending ID, and should have + // non-decreasing newest user-defined timestamps. + if (!cutoff_udt_.empty()) { + assert(table_newest_udt.size() == cutoff_udt_.size()); + assert(ucmp->CompareTimestamp(table_newest_udt, cutoff_udt_) >= 0); + cutoff_udt_.clear(); + } + cutoff_udt_.assign(table_newest_udt.data(), table_newest_udt.size()); + } +} + +Status FlushJob::MaybeIncreaseFullHistoryTsLowToAboveCutoffUDT() { + db_mutex_->AssertHeld(); + const auto* ucmp = cfd_->user_comparator(); + assert(ucmp); + const std::string& full_history_ts_low = cfd_->GetFullHistoryTsLow(); + // Update full_history_ts_low to right above cutoff udt only if that would + // increase it. + if (cutoff_udt_.empty() || + (!full_history_ts_low.empty() && + ucmp->CompareTimestamp(cutoff_udt_, full_history_ts_low) < 0)) { + return Status::OK(); + } + Slice cutoff_udt_slice = cutoff_udt_; + uint64_t cutoff_udt_ts = 0; + bool format_res = GetFixed64(&cutoff_udt_slice, &cutoff_udt_ts); + assert(format_res); + (void)format_res; + std::string new_full_history_ts_low; + // TODO(yuzhangyu): Add a member to AdvancedColumnFamilyOptions for an + // operation to get the next immediately larger user-defined timestamp to + // expand this feature to other user-defined timestamp formats. + PutFixed64(&new_full_history_ts_low, cutoff_udt_ts + 1); + VersionEdit edit; + edit.SetColumnFamily(cfd_->GetID()); + edit.SetFullHistoryTsLow(new_full_history_ts_low); + return versions_->LogAndApply(cfd_, *cfd_->GetLatestMutableCFOptions(), + ReadOptions(), &edit, db_mutex_, + output_file_directory_); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/flush_job.h b/librocksdb-sys/rocksdb/db/flush_job.h new file mode 100644 index 0000000..43d10ff --- /dev/null +++ b/librocksdb-sys/rocksdb/db/flush_job.h @@ -0,0 +1,218 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "db/blob/blob_file_completion_callback.h" +#include "db/column_family.h" +#include "db/flush_scheduler.h" +#include "db/internal_stats.h" +#include "db/job_context.h" +#include "db/log_writer.h" +#include "db/logs_with_prep_tracker.h" +#include "db/memtable_list.h" +#include "db/seqno_to_time_mapping.h" +#include "db/snapshot_impl.h" +#include "db/version_edit.h" +#include "db/write_controller.h" +#include "db/write_thread.h" +#include "logging/event_logger.h" +#include "monitoring/instrumented_mutex.h" +#include "options/db_options.h" +#include "port/port.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/listener.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/transaction_log.h" +#include "table/scoped_arena_iterator.h" +#include "util/autovector.h" +#include "util/stop_watch.h" +#include "util/thread_local.h" + +namespace ROCKSDB_NAMESPACE { + +class DBImpl; +class MemTable; +class SnapshotChecker; +class TableCache; +class Version; +class VersionEdit; +class VersionSet; +class Arena; + +class FlushJob { + public: + // TODO(icanadi) make effort to reduce number of parameters here + // IMPORTANT: mutable_cf_options needs to be alive while FlushJob is alive + FlushJob(const std::string& dbname, ColumnFamilyData* cfd, + const ImmutableDBOptions& db_options, + const MutableCFOptions& mutable_cf_options, uint64_t max_memtable_id, + const FileOptions& file_options, VersionSet* versions, + InstrumentedMutex* db_mutex, std::atomic* shutting_down, + std::vector existing_snapshots, + SequenceNumber earliest_write_conflict_snapshot, + SnapshotChecker* snapshot_checker, JobContext* job_context, + FlushReason flush_reason, LogBuffer* log_buffer, + FSDirectory* db_directory, FSDirectory* output_file_directory, + CompressionType output_compression, Statistics* stats, + EventLogger* event_logger, bool measure_io_stats, + const bool sync_output_directory, const bool write_manifest, + Env::Priority thread_pri, const std::shared_ptr& io_tracer, + const SeqnoToTimeMapping& seq_time_mapping, + const std::string& db_id = "", const std::string& db_session_id = "", + std::string full_history_ts_low = "", + BlobFileCompletionCallback* blob_callback = nullptr); + + ~FlushJob(); + + // Require db_mutex held. + // Once PickMemTable() is called, either Run() or Cancel() has to be called. + void PickMemTable(); + Status Run(LogsWithPrepTracker* prep_tracker = nullptr, + FileMetaData* file_meta = nullptr, + bool* switched_to_mempurge = nullptr); + void Cancel(); + const autovector& GetMemTables() const { return mems_; } + + std::list>* GetCommittedFlushJobsInfo() { + return &committed_flush_jobs_info_; + } + + private: + friend class FlushJobTest_GetRateLimiterPriorityForWrite_Test; + + void ReportStartedFlush(); + void ReportFlushInputSize(const autovector& mems); + void RecordFlushIOStats(); + Status WriteLevel0Table(); + + // Memtable Garbage Collection algorithm: a MemPurge takes the list + // of immutable memtables and filters out (or "purge") the outdated bytes + // out of it. The output (the filtered bytes, or "useful payload") is + // then transfered into a new memtable. If this memtable is filled, then + // the mempurge is aborted and rerouted to a regular flush process. Else, + // depending on the heuristics, placed onto the immutable memtable list. + // The addition to the imm list will not trigger a flush operation. The + // flush of the imm list will instead be triggered once the mutable memtable + // is added to the imm list. + // This process is typically intended for workloads with heavy overwrites + // when we want to avoid SSD writes (and reads) as much as possible. + // "MemPurge" is an experimental feature still at a very early stage + // of development. At the moment it is only compatible with the Get, Put, + // Delete operations as well as Iterators and CompactionFilters. + // For this early version, "MemPurge" is called by setting the + // options.experimental_mempurge_threshold value as >0.0. When this is + // the case, ALL automatic flush operations (kWRiteBufferManagerFull) will + // first go through the MemPurge process. Therefore, we strongly + // recommend all users not to set this flag as true given that the MemPurge + // process has not matured yet. + Status MemPurge(); + bool MemPurgeDecider(double threshold); + // The rate limiter priority (io_priority) is determined dynamically here. + Env::IOPriority GetRateLimiterPriorityForWrite(); + std::unique_ptr GetFlushJobInfo() const; + + // Require db_mutex held. + // Called only when UDT feature is enabled and + // `persist_user_defined_timestamps` flag is false. Because we will refrain + // from flushing as long as there are still UDTs in a memtable that hasn't + // expired w.r.t `full_history_ts_low`. However, flush is continued if there + // is risk of entering write stall mode. In that case, we need + // to track the effective cutoff timestamp below which all the udts are + // removed because of flush, and use it to increase `full_history_ts_low` if + // the effective cutoff timestamp is newer. See + // `MaybeIncreaseFullHistoryTsLowToAboveCutoffUDT` for details. + void GetEffectiveCutoffUDTForPickedMemTables(); + + Status MaybeIncreaseFullHistoryTsLowToAboveCutoffUDT(); + + const std::string& dbname_; + const std::string db_id_; + const std::string db_session_id_; + ColumnFamilyData* cfd_; + const ImmutableDBOptions& db_options_; + const MutableCFOptions& mutable_cf_options_; + // A variable storing the largest memtable id to flush in this + // flush job. RocksDB uses this variable to select the memtables to flush in + // this job. All memtables in this column family with an ID smaller than or + // equal to max_memtable_id_ will be selected for flush. + uint64_t max_memtable_id_; + const FileOptions file_options_; + VersionSet* versions_; + InstrumentedMutex* db_mutex_; + std::atomic* shutting_down_; + std::vector existing_snapshots_; + SequenceNumber earliest_write_conflict_snapshot_; + SnapshotChecker* snapshot_checker_; + JobContext* job_context_; + FlushReason flush_reason_; + LogBuffer* log_buffer_; + FSDirectory* db_directory_; + FSDirectory* output_file_directory_; + CompressionType output_compression_; + Statistics* stats_; + EventLogger* event_logger_; + TableProperties table_properties_; + bool measure_io_stats_; + // True if this flush job should call fsync on the output directory. False + // otherwise. + // Usually sync_output_directory_ is true. A flush job needs to call sync on + // the output directory before committing to the MANIFEST. + // However, an individual flush job does not have to call sync on the output + // directory if it is part of an atomic flush. After all flush jobs in the + // atomic flush succeed, call sync once on each distinct output directory. + const bool sync_output_directory_; + // True if this flush job should write to MANIFEST after successfully + // flushing memtables. False otherwise. + // Usually write_manifest_ is true. A flush job commits to the MANIFEST after + // flushing the memtables. + // However, an individual flush job cannot rashly write to the MANIFEST + // immediately after it finishes the flush if it is part of an atomic flush. + // In this case, only after all flush jobs succeed in flush can RocksDB + // commit to the MANIFEST. + const bool write_manifest_; + // The current flush job can commit flush result of a concurrent flush job. + // We collect FlushJobInfo of all jobs committed by current job and fire + // OnFlushCompleted for them. + std::list> committed_flush_jobs_info_; + + // Variables below are set by PickMemTable(): + FileMetaData meta_; + autovector mems_; + VersionEdit* edit_; + Version* base_; + bool pick_memtable_called; + Env::Priority thread_pri_; + + const std::shared_ptr io_tracer_; + SystemClock* clock_; + + const std::string full_history_ts_low_; + BlobFileCompletionCallback* blob_callback_; + + // reference to the seqno_time_mapping_ in db_impl.h, not safe to read without + // db mutex + const SeqnoToTimeMapping& db_impl_seqno_time_mapping_; + SeqnoToTimeMapping seqno_to_time_mapping_; + + // Keeps track of the newest user-defined timestamp for this flush job if + // `persist_user_defined_timestamps` flag is false. + std::string cutoff_udt_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/flush_job_test.cc b/librocksdb-sys/rocksdb/db/flush_job_test.cc new file mode 100644 index 0000000..9fd9c13 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/flush_job_test.cc @@ -0,0 +1,820 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/flush_job.h" + +#include +#include +#include +#include + +#include "db/blob/blob_index.h" +#include "db/column_family.h" +#include "db/db_impl/db_impl.h" +#include "db/version_set.h" +#include "file/writable_file_writer.h" +#include "rocksdb/cache.h" +#include "rocksdb/file_system.h" +#include "rocksdb/write_buffer_manager.h" +#include "table/mock_table.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/random.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +// TODO(icanadi) Mock out everything else: +// 1. VersionSet +// 2. Memtable +class FlushJobTestBase : public testing::Test { + protected: + FlushJobTestBase(std::string dbname, const Comparator* ucmp) + : env_(Env::Default()), + fs_(env_->GetFileSystem()), + dbname_(std::move(dbname)), + ucmp_(ucmp), + options_(), + db_options_(options_), + column_family_names_({kDefaultColumnFamilyName, "foo", "bar"}), + table_cache_(NewLRUCache(50000, 16)), + write_buffer_manager_(db_options_.db_write_buffer_size), + shutting_down_(false), + mock_table_factory_(new mock::MockTableFactory()) {} + + virtual ~FlushJobTestBase() { + if (getenv("KEEP_DB")) { + fprintf(stdout, "db is still in %s\n", dbname_.c_str()); + } else { + // destroy versions_ to release all file handles + versions_.reset(); + EXPECT_OK(DestroyDir(env_, dbname_)); + } + } + + void NewDB() { + ASSERT_OK(SetIdentityFile(env_, dbname_)); + VersionEdit new_db; + + new_db.SetLogNumber(0); + new_db.SetNextFile(2); + new_db.SetLastSequence(0); + + autovector new_cfs; + SequenceNumber last_seq = 1; + uint32_t cf_id = 1; + for (size_t i = 1; i != column_family_names_.size(); ++i) { + VersionEdit new_cf; + new_cf.AddColumnFamily(column_family_names_[i]); + new_cf.SetColumnFamily(cf_id++); + new_cf.SetComparatorName(ucmp_->Name()); + new_cf.SetPersistUserDefinedTimestamps(persist_udt_); + new_cf.SetLogNumber(0); + new_cf.SetNextFile(2); + new_cf.SetLastSequence(last_seq++); + new_cfs.emplace_back(new_cf); + } + + const std::string manifest = DescriptorFileName(dbname_, 1); + const auto& fs = env_->GetFileSystem(); + std::unique_ptr file_writer; + Status s = WritableFileWriter::Create( + fs, manifest, fs->OptimizeForManifestWrite(env_options_), &file_writer, + nullptr); + ASSERT_OK(s); + + { + log::Writer log(std::move(file_writer), 0, false); + std::string record; + new_db.EncodeTo(&record); + s = log.AddRecord(record); + ASSERT_OK(s); + + for (const auto& e : new_cfs) { + record.clear(); + e.EncodeTo(&record); + s = log.AddRecord(record); + ASSERT_OK(s); + } + } + ASSERT_OK(s); + // Make "CURRENT" file that points to the new manifest file. + s = SetCurrentFile(fs_.get(), dbname_, 1, nullptr); + ASSERT_OK(s); + } + + void SetUp() override { + EXPECT_OK(env_->CreateDirIfMissing(dbname_)); + + // TODO(icanadi) Remove this once we mock out VersionSet + NewDB(); + + db_options_.env = env_; + db_options_.fs = fs_; + db_options_.db_paths.emplace_back(dbname_, + std::numeric_limits::max()); + db_options_.statistics = CreateDBStatistics(); + + cf_options_.comparator = ucmp_; + cf_options_.persist_user_defined_timestamps = persist_udt_; + cf_options_.paranoid_file_checks = paranoid_file_checks_; + + std::vector column_families; + cf_options_.table_factory = mock_table_factory_; + for (const auto& cf_name : column_family_names_) { + column_families.emplace_back(cf_name, cf_options_); + } + + versions_.reset( + new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id*/ "", /*db_session_id*/ "")); + EXPECT_OK(versions_->Recover(column_families, false)); + } + + Env* env_; + std::shared_ptr fs_; + std::string dbname_; + const Comparator* const ucmp_; + EnvOptions env_options_; + Options options_; + ImmutableDBOptions db_options_; + const std::vector column_family_names_; + std::shared_ptr table_cache_; + WriteController write_controller_; + WriteBufferManager write_buffer_manager_; + ColumnFamilyOptions cf_options_; + std::unique_ptr versions_; + InstrumentedMutex mutex_; + std::atomic shutting_down_; + std::shared_ptr mock_table_factory_; + + bool persist_udt_ = true; + bool paranoid_file_checks_ = false; + + SeqnoToTimeMapping empty_seqno_to_time_mapping_; +}; + +class FlushJobTest : public FlushJobTestBase { + public: + FlushJobTest() + : FlushJobTestBase(test::PerThreadDBPath("flush_job_test"), + BytewiseComparator()) {} +}; + +TEST_F(FlushJobTest, Empty) { + JobContext job_context(0); + auto cfd = versions_->GetColumnFamilySet()->GetDefault(); + EventLogger event_logger(db_options_.info_log.get()); + SnapshotChecker* snapshot_checker = nullptr; // not relavant + FlushJob flush_job( + dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_, + *cfd->GetLatestMutableCFOptions(), + std::numeric_limits::max() /* memtable_id */, env_options_, + versions_.get(), &mutex_, &shutting_down_, {}, kMaxSequenceNumber, + snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr, + nullptr, kNoCompression, nullptr, &event_logger, false, + true /* sync_output_directory */, true /* write_manifest */, + Env::Priority::USER, nullptr /*IOTracer*/, empty_seqno_to_time_mapping_); + { + InstrumentedMutexLock l(&mutex_); + flush_job.PickMemTable(); + ASSERT_OK(flush_job.Run()); + } + job_context.Clean(); +} + +TEST_F(FlushJobTest, NonEmpty) { + JobContext job_context(0); + auto cfd = versions_->GetColumnFamilySet()->GetDefault(); + auto new_mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions(), + kMaxSequenceNumber); + new_mem->Ref(); + auto inserted_keys = mock::MakeMockFile(); + // Test data: + // seqno [ 1, 2 ... 8998, 8999, 9000, 9001, 9002 ... 9999 ] + // key [ 1001, 1002 ... 9998, 9999, 0, 1, 2 ... 999 ] + // range-delete "9995" -> "9999" at seqno 10000 + // blob references with seqnos 10001..10006 + for (int i = 1; i < 10000; ++i) { + std::string key(std::to_string((i + 1000) % 10000)); + std::string value("value" + key); + ASSERT_OK(new_mem->Add(SequenceNumber(i), kTypeValue, key, value, + nullptr /* kv_prot_info */)); + if ((i + 1000) % 10000 < 9995) { + InternalKey internal_key(key, SequenceNumber(i), kTypeValue); + inserted_keys.push_back({internal_key.Encode().ToString(), value}); + } + } + + { + ASSERT_OK(new_mem->Add(SequenceNumber(10000), kTypeRangeDeletion, "9995", + "9999a", nullptr /* kv_prot_info */)); + InternalKey internal_key("9995", SequenceNumber(10000), kTypeRangeDeletion); + inserted_keys.push_back({internal_key.Encode().ToString(), "9999a"}); + } + + // Note: the first two blob references will not be considered when resolving + // the oldest blob file referenced (the first one is inlined TTL, while the + // second one is TTL and thus points to a TTL blob file). + constexpr std::array blob_file_numbers{ + {kInvalidBlobFileNumber, 5, 103, 17, 102, 101}}; + for (size_t i = 0; i < blob_file_numbers.size(); ++i) { + std::string key(std::to_string(i + 10001)); + std::string blob_index; + if (i == 0) { + BlobIndex::EncodeInlinedTTL(&blob_index, /* expiration */ 1234567890ULL, + "foo"); + } else if (i == 1) { + BlobIndex::EncodeBlobTTL(&blob_index, /* expiration */ 1234567890ULL, + blob_file_numbers[i], /* offset */ i << 10, + /* size */ i << 20, kNoCompression); + } else { + BlobIndex::EncodeBlob(&blob_index, blob_file_numbers[i], + /* offset */ i << 10, /* size */ i << 20, + kNoCompression); + } + + const SequenceNumber seq(i + 10001); + ASSERT_OK(new_mem->Add(seq, kTypeBlobIndex, key, blob_index, + nullptr /* kv_prot_info */)); + + InternalKey internal_key(key, seq, kTypeBlobIndex); + inserted_keys.push_back({internal_key.Encode().ToString(), blob_index}); + } + mock::SortKVVector(&inserted_keys); + + autovector to_delete; + new_mem->ConstructFragmentedRangeTombstones(); + cfd->imm()->Add(new_mem, &to_delete); + for (auto& m : to_delete) { + delete m; + } + + EventLogger event_logger(db_options_.info_log.get()); + SnapshotChecker* snapshot_checker = nullptr; // not relavant + FlushJob flush_job( + dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_, + *cfd->GetLatestMutableCFOptions(), + std::numeric_limits::max() /* memtable_id */, env_options_, + versions_.get(), &mutex_, &shutting_down_, {}, kMaxSequenceNumber, + snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr, + nullptr, kNoCompression, db_options_.statistics.get(), &event_logger, + true, true /* sync_output_directory */, true /* write_manifest */, + Env::Priority::USER, nullptr /*IOTracer*/, empty_seqno_to_time_mapping_); + + HistogramData hist; + FileMetaData file_meta; + mutex_.Lock(); + flush_job.PickMemTable(); + ASSERT_OK(flush_job.Run(nullptr, &file_meta)); + mutex_.Unlock(); + db_options_.statistics->histogramData(FLUSH_TIME, &hist); + ASSERT_GT(hist.average, 0.0); + + ASSERT_EQ(std::to_string(0), file_meta.smallest.user_key().ToString()); + ASSERT_EQ("9999a", file_meta.largest.user_key().ToString()); + ASSERT_EQ(1, file_meta.fd.smallest_seqno); + ASSERT_EQ(10006, file_meta.fd.largest_seqno); + ASSERT_EQ(17, file_meta.oldest_blob_file_number); + mock_table_factory_->AssertSingleFile(inserted_keys); + job_context.Clean(); +} + +TEST_F(FlushJobTest, FlushMemTablesSingleColumnFamily) { + const size_t num_mems = 2; + const size_t num_mems_to_flush = 1; + const size_t num_keys_per_table = 100; + JobContext job_context(0); + ColumnFamilyData* cfd = versions_->GetColumnFamilySet()->GetDefault(); + std::vector memtable_ids; + std::vector new_mems; + for (size_t i = 0; i != num_mems; ++i) { + MemTable* mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions(), + kMaxSequenceNumber); + mem->SetID(i); + mem->Ref(); + new_mems.emplace_back(mem); + memtable_ids.push_back(mem->GetID()); + + for (size_t j = 0; j < num_keys_per_table; ++j) { + std::string key(std::to_string(j + i * num_keys_per_table)); + std::string value("value" + key); + ASSERT_OK(mem->Add(SequenceNumber(j + i * num_keys_per_table), kTypeValue, + key, value, nullptr /* kv_prot_info */)); + } + } + + autovector to_delete; + for (auto mem : new_mems) { + mem->ConstructFragmentedRangeTombstones(); + cfd->imm()->Add(mem, &to_delete); + } + + EventLogger event_logger(db_options_.info_log.get()); + SnapshotChecker* snapshot_checker = nullptr; // not relavant + + assert(memtable_ids.size() == num_mems); + uint64_t smallest_memtable_id = memtable_ids.front(); + uint64_t flush_memtable_id = smallest_memtable_id + num_mems_to_flush - 1; + FlushJob flush_job( + dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_, + *cfd->GetLatestMutableCFOptions(), flush_memtable_id, env_options_, + versions_.get(), &mutex_, &shutting_down_, {}, kMaxSequenceNumber, + snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr, + nullptr, kNoCompression, db_options_.statistics.get(), &event_logger, + true, true /* sync_output_directory */, true /* write_manifest */, + Env::Priority::USER, nullptr /*IOTracer*/, empty_seqno_to_time_mapping_); + HistogramData hist; + FileMetaData file_meta; + mutex_.Lock(); + flush_job.PickMemTable(); + ASSERT_OK(flush_job.Run(nullptr /* prep_tracker */, &file_meta)); + mutex_.Unlock(); + db_options_.statistics->histogramData(FLUSH_TIME, &hist); + ASSERT_GT(hist.average, 0.0); + + ASSERT_EQ(std::to_string(0), file_meta.smallest.user_key().ToString()); + ASSERT_EQ("99", file_meta.largest.user_key().ToString()); + ASSERT_EQ(0, file_meta.fd.smallest_seqno); + ASSERT_EQ(SequenceNumber(num_mems_to_flush * num_keys_per_table - 1), + file_meta.fd.largest_seqno); + ASSERT_EQ(kInvalidBlobFileNumber, file_meta.oldest_blob_file_number); + + for (auto m : to_delete) { + delete m; + } + to_delete.clear(); + job_context.Clean(); +} + +TEST_F(FlushJobTest, FlushMemtablesMultipleColumnFamilies) { + autovector all_cfds; + for (auto cfd : *versions_->GetColumnFamilySet()) { + all_cfds.push_back(cfd); + } + const std::vector num_memtables = {2, 1, 3}; + assert(num_memtables.size() == column_family_names_.size()); + const size_t num_keys_per_memtable = 1000; + JobContext job_context(0); + std::vector memtable_ids; + std::vector smallest_seqs; + std::vector largest_seqs; + autovector to_delete; + SequenceNumber curr_seqno = 0; + size_t k = 0; + for (auto cfd : all_cfds) { + smallest_seqs.push_back(curr_seqno); + for (size_t i = 0; i != num_memtables[k]; ++i) { + MemTable* mem = cfd->ConstructNewMemtable( + *cfd->GetLatestMutableCFOptions(), kMaxSequenceNumber); + mem->SetID(i); + mem->Ref(); + + for (size_t j = 0; j != num_keys_per_memtable; ++j) { + std::string key(std::to_string(j + i * num_keys_per_memtable)); + std::string value("value" + key); + ASSERT_OK(mem->Add(curr_seqno++, kTypeValue, key, value, + nullptr /* kv_prot_info */)); + } + mem->ConstructFragmentedRangeTombstones(); + cfd->imm()->Add(mem, &to_delete); + } + largest_seqs.push_back(curr_seqno - 1); + memtable_ids.push_back(num_memtables[k++] - 1); + } + + EventLogger event_logger(db_options_.info_log.get()); + SnapshotChecker* snapshot_checker = nullptr; // not relevant + std::vector> flush_jobs; + k = 0; + for (auto cfd : all_cfds) { + std::vector snapshot_seqs; + flush_jobs.emplace_back(new FlushJob( + dbname_, cfd, db_options_, *cfd->GetLatestMutableCFOptions(), + memtable_ids[k], env_options_, versions_.get(), &mutex_, + &shutting_down_, snapshot_seqs, kMaxSequenceNumber, snapshot_checker, + &job_context, FlushReason::kTest, nullptr, nullptr, nullptr, + kNoCompression, db_options_.statistics.get(), &event_logger, true, + false /* sync_output_directory */, false /* write_manifest */, + Env::Priority::USER, nullptr /*IOTracer*/, + empty_seqno_to_time_mapping_)); + k++; + } + HistogramData hist; + std::vector file_metas; + // Call reserve to avoid auto-resizing + file_metas.reserve(flush_jobs.size()); + mutex_.Lock(); + for (auto& job : flush_jobs) { + job->PickMemTable(); + } + for (auto& job : flush_jobs) { + FileMetaData meta; + // Run will release and re-acquire mutex + ASSERT_OK(job->Run(nullptr /**/, &meta)); + file_metas.emplace_back(meta); + } + autovector file_meta_ptrs; + for (auto& meta : file_metas) { + file_meta_ptrs.push_back(&meta); + } + autovector*> mems_list; + for (size_t i = 0; i != all_cfds.size(); ++i) { + const auto& mems = flush_jobs[i]->GetMemTables(); + mems_list.push_back(&mems); + } + autovector mutable_cf_options_list; + for (auto cfd : all_cfds) { + mutable_cf_options_list.push_back(cfd->GetLatestMutableCFOptions()); + } + autovector>*> + committed_flush_jobs_info; + for (auto& job : flush_jobs) { + committed_flush_jobs_info.push_back(job->GetCommittedFlushJobsInfo()); + } + + Status s = InstallMemtableAtomicFlushResults( + nullptr /* imm_lists */, all_cfds, mutable_cf_options_list, mems_list, + versions_.get(), nullptr /* prep_tracker */, &mutex_, file_meta_ptrs, + committed_flush_jobs_info, &job_context.memtables_to_free, + nullptr /* db_directory */, nullptr /* log_buffer */); + ASSERT_OK(s); + + mutex_.Unlock(); + db_options_.statistics->histogramData(FLUSH_TIME, &hist); + ASSERT_GT(hist.average, 0.0); + k = 0; + for (const auto& file_meta : file_metas) { + ASSERT_EQ(std::to_string(0), file_meta.smallest.user_key().ToString()); + ASSERT_EQ("999", file_meta.largest.user_key() + .ToString()); // max key by bytewise comparator + ASSERT_EQ(smallest_seqs[k], file_meta.fd.smallest_seqno); + ASSERT_EQ(largest_seqs[k], file_meta.fd.largest_seqno); + // Verify that imm is empty + ASSERT_EQ(std::numeric_limits::max(), + all_cfds[k]->imm()->GetEarliestMemTableID()); + ASSERT_EQ(0, all_cfds[k]->imm()->GetLatestMemTableID()); + ++k; + } + + for (auto m : to_delete) { + delete m; + } + to_delete.clear(); + job_context.Clean(); +} + +TEST_F(FlushJobTest, Snapshots) { + JobContext job_context(0); + auto cfd = versions_->GetColumnFamilySet()->GetDefault(); + auto new_mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions(), + kMaxSequenceNumber); + + std::set snapshots_set; + int keys = 10000; + int max_inserts_per_keys = 8; + + Random rnd(301); + for (int i = 0; i < keys / 2; ++i) { + snapshots_set.insert(rnd.Uniform(keys * (max_inserts_per_keys / 2)) + 1); + } + // set has already removed the duplicate snapshots + std::vector snapshots(snapshots_set.begin(), + snapshots_set.end()); + + new_mem->Ref(); + SequenceNumber current_seqno = 0; + auto inserted_keys = mock::MakeMockFile(); + for (int i = 1; i < keys; ++i) { + std::string key(std::to_string(i)); + int insertions = rnd.Uniform(max_inserts_per_keys); + for (int j = 0; j < insertions; ++j) { + std::string value(rnd.HumanReadableString(10)); + auto seqno = ++current_seqno; + ASSERT_OK(new_mem->Add(SequenceNumber(seqno), kTypeValue, key, value, + nullptr /* kv_prot_info */)); + // a key is visible only if: + // 1. it's the last one written (j == insertions - 1) + // 2. there's a snapshot pointing at it + bool visible = (j == insertions - 1) || + (snapshots_set.find(seqno) != snapshots_set.end()); + if (visible) { + InternalKey internal_key(key, seqno, kTypeValue); + inserted_keys.push_back({internal_key.Encode().ToString(), value}); + } + } + } + mock::SortKVVector(&inserted_keys); + + autovector to_delete; + new_mem->ConstructFragmentedRangeTombstones(); + cfd->imm()->Add(new_mem, &to_delete); + for (auto& m : to_delete) { + delete m; + } + + EventLogger event_logger(db_options_.info_log.get()); + SnapshotChecker* snapshot_checker = nullptr; // not relavant + FlushJob flush_job( + dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_, + *cfd->GetLatestMutableCFOptions(), + std::numeric_limits::max() /* memtable_id */, env_options_, + versions_.get(), &mutex_, &shutting_down_, snapshots, kMaxSequenceNumber, + snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr, + nullptr, kNoCompression, db_options_.statistics.get(), &event_logger, + true, true /* sync_output_directory */, true /* write_manifest */, + Env::Priority::USER, nullptr /*IOTracer*/, empty_seqno_to_time_mapping_); + mutex_.Lock(); + flush_job.PickMemTable(); + ASSERT_OK(flush_job.Run()); + mutex_.Unlock(); + mock_table_factory_->AssertSingleFile(inserted_keys); + HistogramData hist; + db_options_.statistics->histogramData(FLUSH_TIME, &hist); + ASSERT_GT(hist.average, 0.0); + job_context.Clean(); +} + +TEST_F(FlushJobTest, GetRateLimiterPriorityForWrite) { + // Prepare a FlushJob that flush MemTables of Single Column Family. + const size_t num_mems = 2; + const size_t num_mems_to_flush = 1; + const size_t num_keys_per_table = 100; + JobContext job_context(0); + ColumnFamilyData* cfd = versions_->GetColumnFamilySet()->GetDefault(); + std::vector memtable_ids; + std::vector new_mems; + for (size_t i = 0; i != num_mems; ++i) { + MemTable* mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions(), + kMaxSequenceNumber); + mem->SetID(i); + mem->Ref(); + new_mems.emplace_back(mem); + memtable_ids.push_back(mem->GetID()); + + for (size_t j = 0; j < num_keys_per_table; ++j) { + std::string key(std::to_string(j + i * num_keys_per_table)); + std::string value("value" + key); + ASSERT_OK(mem->Add(SequenceNumber(j + i * num_keys_per_table), kTypeValue, + key, value, nullptr /* kv_prot_info */)); + } + } + + autovector to_delete; + for (auto mem : new_mems) { + mem->ConstructFragmentedRangeTombstones(); + cfd->imm()->Add(mem, &to_delete); + } + + EventLogger event_logger(db_options_.info_log.get()); + SnapshotChecker* snapshot_checker = nullptr; // not relavant + + assert(memtable_ids.size() == num_mems); + uint64_t smallest_memtable_id = memtable_ids.front(); + uint64_t flush_memtable_id = smallest_memtable_id + num_mems_to_flush - 1; + FlushJob flush_job( + dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_, + *cfd->GetLatestMutableCFOptions(), flush_memtable_id, env_options_, + versions_.get(), &mutex_, &shutting_down_, {}, kMaxSequenceNumber, + snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr, + nullptr, kNoCompression, db_options_.statistics.get(), &event_logger, + true, true /* sync_output_directory */, true /* write_manifest */, + Env::Priority::USER, nullptr /*IOTracer*/, empty_seqno_to_time_mapping_); + + // When the state from WriteController is normal. + ASSERT_EQ(flush_job.GetRateLimiterPriorityForWrite(), Env::IO_HIGH); + + WriteController* write_controller = + flush_job.versions_->GetColumnFamilySet()->write_controller(); + + { + // When the state from WriteController is Delayed. + std::unique_ptr delay_token = + write_controller->GetDelayToken(1000000); + ASSERT_EQ(flush_job.GetRateLimiterPriorityForWrite(), Env::IO_USER); + } + + { + // When the state from WriteController is Stopped. + std::unique_ptr stop_token = + write_controller->GetStopToken(); + ASSERT_EQ(flush_job.GetRateLimiterPriorityForWrite(), Env::IO_USER); + } +} + +// Test parameters: +// param 0): paranoid file check +// param 1): user-defined timestamp test mode +class FlushJobTimestampTest + : public FlushJobTestBase, + public testing::WithParamInterface< + std::tuple> { + public: + FlushJobTimestampTest() + : FlushJobTestBase(test::PerThreadDBPath("flush_job_ts_gc_test"), + test::BytewiseComparatorWithU64TsWrapper()) {} + + void AddKeyValueToMemtable(MemTable* memtable, std::string key, uint64_t ts, + SequenceNumber seq, ValueType value_type, + Slice value) { + std::string key_str(std::move(key)); + PutFixed64(&key_str, ts); + ASSERT_OK(memtable->Add(seq, value_type, key_str, value, + nullptr /* kv_prot_info */)); + } + + protected: + void SetUp() override { + paranoid_file_checks_ = std::get<0>(GetParam()); + auto udt_test_mode = std::get<1>(GetParam()); + persist_udt_ = test::ShouldPersistUDT(udt_test_mode); + FlushJobTestBase::SetUp(); + } + static constexpr uint64_t kStartTs = 10; + static constexpr SequenceNumber kStartSeq = 0; + SequenceNumber curr_seq_{kStartSeq}; + std::atomic curr_ts_{kStartTs}; + + void CheckFileMetaData(ColumnFamilyData* cfd, + const InternalKey& expected_smallest, + const InternalKey& expected_largest, + const FileMetaData* meta_from_flush) const { + ASSERT_EQ(expected_smallest.Encode(), meta_from_flush->smallest.Encode()); + ASSERT_EQ(expected_largest.Encode(), meta_from_flush->largest.Encode()); + + const VersionStorageInfo* storage_info = cfd->current()->storage_info(); + const std::vector& l0_files = storage_info->LevelFiles(0); + + ASSERT_EQ(l0_files.size(), 1); + auto installed_file_meta = l0_files[0]; + ASSERT_EQ(expected_smallest.Encode(), + installed_file_meta->smallest.Encode()); + ASSERT_EQ(expected_largest.Encode(), installed_file_meta->largest.Encode()); + } + void CheckFullHistoryTsLow(ColumnFamilyData* cfd, + const std::string& expected_full_history_ts_low) { + ASSERT_EQ(expected_full_history_ts_low, cfd->GetFullHistoryTsLow()); + } +}; + +TEST_P(FlushJobTimestampTest, AllKeysExpired) { + ColumnFamilyData* cfd = versions_->GetColumnFamilySet()->GetDefault(); + autovector to_delete; + + { + MemTable* new_mem = cfd->ConstructNewMemtable( + *cfd->GetLatestMutableCFOptions(), kMaxSequenceNumber); + new_mem->Ref(); + for (int i = 0; i < 100; ++i) { + uint64_t ts = curr_ts_.fetch_add(1); + SequenceNumber seq = (curr_seq_++); + AddKeyValueToMemtable(new_mem, test::EncodeInt(0), ts, seq, + ValueType::kTypeValue, "0_value"); + } + uint64_t ts = curr_ts_.fetch_add(1); + SequenceNumber seq = (curr_seq_++); + AddKeyValueToMemtable(new_mem, test::EncodeInt(0), ts, seq, + ValueType::kTypeDeletionWithTimestamp, ""); + new_mem->ConstructFragmentedRangeTombstones(); + cfd->imm()->Add(new_mem, &to_delete); + } + + std::vector snapshots; + constexpr SnapshotChecker* const snapshot_checker = nullptr; + JobContext job_context(0); + EventLogger event_logger(db_options_.info_log.get()); + std::string full_history_ts_low; + PutFixed64(&full_history_ts_low, std::numeric_limits::max()); + cfd->SetFullHistoryTsLow(full_history_ts_low); + FlushJob flush_job( + dbname_, cfd, db_options_, *cfd->GetLatestMutableCFOptions(), + std::numeric_limits::max() /* memtable_id */, env_options_, + versions_.get(), &mutex_, &shutting_down_, snapshots, kMaxSequenceNumber, + snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr, + nullptr, kNoCompression, db_options_.statistics.get(), &event_logger, + true, true /* sync_output_directory */, true /* write_manifest */, + Env::Priority::USER, nullptr /*IOTracer*/, empty_seqno_to_time_mapping_, + /*db_id=*/"", + /*db_session_id=*/"", full_history_ts_low); + + FileMetaData fmeta; + mutex_.Lock(); + flush_job.PickMemTable(); + ASSERT_OK(flush_job.Run(/*prep_tracker=*/nullptr, &fmeta)); + mutex_.Unlock(); + + { + std::string key = test::EncodeInt(0); + if (!persist_udt_) { + // When `AdvancedColumnFamilyOptions.persist_user_defined_timestamps` flag + // is set to false. The user-defined timestamp is stripped from user key + // during flush, making the user key logically containing the minimum + // timestamp. + key.append(test::EncodeInt(0)); + } else { + key.append(test::EncodeInt(curr_ts_.load(std::memory_order_relaxed) - 1)); + } + InternalKey ikey(key, curr_seq_ - 1, ValueType::kTypeDeletionWithTimestamp); + CheckFileMetaData(cfd, ikey, ikey, &fmeta); + CheckFullHistoryTsLow(cfd, full_history_ts_low); + } + + job_context.Clean(); + ASSERT_TRUE(to_delete.empty()); +} + +TEST_P(FlushJobTimestampTest, NoKeyExpired) { + ColumnFamilyData* cfd = versions_->GetColumnFamilySet()->GetDefault(); + autovector to_delete; + + { + MemTable* new_mem = cfd->ConstructNewMemtable( + *cfd->GetLatestMutableCFOptions(), kMaxSequenceNumber); + new_mem->Ref(); + for (int i = 0; i < 100; ++i) { + uint64_t ts = curr_ts_.fetch_add(1); + SequenceNumber seq = (curr_seq_++); + AddKeyValueToMemtable(new_mem, test::EncodeInt(0), ts, seq, + ValueType::kTypeValue, "0_value"); + } + new_mem->ConstructFragmentedRangeTombstones(); + cfd->imm()->Add(new_mem, &to_delete); + } + + std::vector snapshots; + SnapshotChecker* const snapshot_checker = nullptr; + JobContext job_context(0); + EventLogger event_logger(db_options_.info_log.get()); + std::string full_history_ts_low; + PutFixed64(&full_history_ts_low, 0); + cfd->SetFullHistoryTsLow(full_history_ts_low); + FlushJob flush_job( + dbname_, cfd, db_options_, *cfd->GetLatestMutableCFOptions(), + std::numeric_limits::max() /* memtable_id */, env_options_, + versions_.get(), &mutex_, &shutting_down_, snapshots, kMaxSequenceNumber, + snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr, + nullptr, kNoCompression, db_options_.statistics.get(), &event_logger, + true, true /* sync_output_directory */, true /* write_manifest */, + Env::Priority::USER, nullptr /*IOTracer*/, empty_seqno_to_time_mapping_, + /*db_id=*/"", + /*db_session_id=*/"", full_history_ts_low); + + FileMetaData fmeta; + mutex_.Lock(); + flush_job.PickMemTable(); + ASSERT_OK(flush_job.Run(/*prep_tracker=*/nullptr, &fmeta)); + mutex_.Unlock(); + + { + std::string ukey = test::EncodeInt(0); + std::string smallest_key; + std::string largest_key; + std::string expected_full_history_ts_low; + if (!persist_udt_) { + // When `AdvancedColumnFamilyOptions.persist_user_defined_timestamps` flag + // is set to false. The user-defined timestamp is stripped from user key + // during flush, making the user key logically containing the minimum + // timestamp, which is hardcoded to be all zeros for now. + smallest_key = ukey + test::EncodeInt(0); + largest_key = ukey + test::EncodeInt(0); + // When not all keys have expired and `persist_user_defined_timestamps` is + // false. UDTs will be removed during flush, `full_history_ts_low` should + // be automatically increased to above the effective cutoff UDT in the + // flush. + PutFixed64(&expected_full_history_ts_low, curr_ts_.fetch_add(1)); + } else { + smallest_key = + ukey + test::EncodeInt(curr_ts_.load(std::memory_order_relaxed) - 1); + largest_key = ukey + test::EncodeInt(kStartTs); + expected_full_history_ts_low = full_history_ts_low; + } + InternalKey smallest(smallest_key, curr_seq_ - 1, ValueType::kTypeValue); + InternalKey largest(largest_key, kStartSeq, ValueType::kTypeValue); + CheckFileMetaData(cfd, smallest, largest, &fmeta); + CheckFullHistoryTsLow(cfd, expected_full_history_ts_low); + } + job_context.Clean(); + ASSERT_TRUE(to_delete.empty()); +} + +// Param 0: paranoid file check +// Param 1: test mode for the user-defined timestamp feature +INSTANTIATE_TEST_CASE_P( + FlushJobTimestampTest, FlushJobTimestampTest, + ::testing::Combine( + ::testing::Bool(), + ::testing::Values( + test::UserDefinedTimestampTestMode::kStripUserDefinedTimestamp, + test::UserDefinedTimestampTestMode::kNormal))); + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/flush_scheduler.cc b/librocksdb-sys/rocksdb/db/flush_scheduler.cc new file mode 100644 index 0000000..6f4d3e1 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/flush_scheduler.cc @@ -0,0 +1,86 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/flush_scheduler.h" + +#include + +#include "db/column_family.h" + +namespace ROCKSDB_NAMESPACE { + +void FlushScheduler::ScheduleWork(ColumnFamilyData* cfd) { +#ifndef NDEBUG + { + std::lock_guard lock(checking_mutex_); + assert(checking_set_.count(cfd) == 0); + checking_set_.insert(cfd); + } +#endif // NDEBUG + cfd->Ref(); +// Suppress false positive clang analyzer warnings. +#ifndef __clang_analyzer__ + Node* node = new Node{cfd, head_.load(std::memory_order_relaxed)}; + while (!head_.compare_exchange_strong( + node->next, node, std::memory_order_relaxed, std::memory_order_relaxed)) { + // failing CAS updates the first param, so we are already set for + // retry. TakeNextColumnFamily won't happen until after another + // inter-thread synchronization, so we don't even need release + // semantics for this CAS + } +#endif // __clang_analyzer__ +} + +ColumnFamilyData* FlushScheduler::TakeNextColumnFamily() { + while (true) { + if (head_.load(std::memory_order_relaxed) == nullptr) { + return nullptr; + } + + // dequeue the head + Node* node = head_.load(std::memory_order_relaxed); + head_.store(node->next, std::memory_order_relaxed); + ColumnFamilyData* cfd = node->column_family; + delete node; + +#ifndef NDEBUG + { + std::lock_guard lock(checking_mutex_); + auto iter = checking_set_.find(cfd); + assert(iter != checking_set_.end()); + checking_set_.erase(iter); + } +#endif // NDEBUG + + if (!cfd->IsDropped()) { + // success + return cfd; + } + + // no longer relevant, retry + cfd->UnrefAndTryDelete(); + } +} + +bool FlushScheduler::Empty() { + auto rv = head_.load(std::memory_order_relaxed) == nullptr; +#ifndef NDEBUG + std::lock_guard lock(checking_mutex_); + // Empty is allowed to be called concurrnetly with ScheduleFlush. It would + // only miss the recent schedules. + assert((rv == checking_set_.empty()) || rv); +#endif // NDEBUG + return rv; +} + +void FlushScheduler::Clear() { + ColumnFamilyData* cfd; + while ((cfd = TakeNextColumnFamily()) != nullptr) { + cfd->UnrefAndTryDelete(); + } + assert(head_.load(std::memory_order_relaxed) == nullptr); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/flush_scheduler.h b/librocksdb-sys/rocksdb/db/flush_scheduler.h new file mode 100644 index 0000000..eb03f3e --- /dev/null +++ b/librocksdb-sys/rocksdb/db/flush_scheduler.h @@ -0,0 +1,55 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include +#include + +#include "util/autovector.h" + +namespace ROCKSDB_NAMESPACE { + +class ColumnFamilyData; + +// FlushScheduler keeps track of all column families whose memtable may +// be full and require flushing. Unless otherwise noted, all methods on +// FlushScheduler should be called only with the DB mutex held or from +// a single-threaded recovery context. +class FlushScheduler { + public: + FlushScheduler() : head_(nullptr) {} + + // May be called from multiple threads at once, but not concurrent with + // any other method calls on this instance + void ScheduleWork(ColumnFamilyData* cfd); + + // Removes and returns Ref()-ed column family. Client needs to Unref(). + // Filters column families that have been dropped. + ColumnFamilyData* TakeNextColumnFamily(); + + // This can be called concurrently with ScheduleWork but it would miss all + // the scheduled flushes after the last synchronization. This would result + // into less precise enforcement of memtable sizes but should not matter much. + bool Empty(); + + void Clear(); + + private: + struct Node { + ColumnFamilyData* column_family; + Node* next; + }; + + std::atomic head_; +#ifndef NDEBUG + std::mutex checking_mutex_; + std::set checking_set_; +#endif // NDEBUG +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/forward_iterator.cc b/librocksdb-sys/rocksdb/db/forward_iterator.cc new file mode 100644 index 0000000..c769156 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/forward_iterator.cc @@ -0,0 +1,1069 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/forward_iterator.h" + +#include +#include +#include + +#include "db/column_family.h" +#include "db/db_impl/db_impl.h" +#include "db/db_iter.h" +#include "db/dbformat.h" +#include "db/job_context.h" +#include "db/range_del_aggregator.h" +#include "db/range_tombstone_fragmenter.h" +#include "rocksdb/env.h" +#include "rocksdb/slice.h" +#include "rocksdb/slice_transform.h" +#include "table/merging_iterator.h" +#include "test_util/sync_point.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +// Usage: +// ForwardLevelIterator iter; +// iter.SetFileIndex(file_index); +// iter.Seek(target); // or iter.SeekToFirst(); +// iter.Next() +class ForwardLevelIterator : public InternalIterator { + public: + ForwardLevelIterator( + const ColumnFamilyData* const cfd, const ReadOptions& read_options, + const std::vector& files, + const std::shared_ptr& prefix_extractor, + bool allow_unprepared_value, uint8_t block_protection_bytes_per_key) + : cfd_(cfd), + read_options_(read_options), + files_(files), + valid_(false), + file_index_(std::numeric_limits::max()), + file_iter_(nullptr), + pinned_iters_mgr_(nullptr), + prefix_extractor_(prefix_extractor), + allow_unprepared_value_(allow_unprepared_value), + block_protection_bytes_per_key_(block_protection_bytes_per_key) { + status_.PermitUncheckedError(); // Allow uninitialized status through + } + + ~ForwardLevelIterator() override { + // Reset current pointer + if (pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled()) { + pinned_iters_mgr_->PinIterator(file_iter_); + } else { + delete file_iter_; + } + } + + void SetFileIndex(uint32_t file_index) { + assert(file_index < files_.size()); + status_ = Status::OK(); + if (file_index != file_index_) { + file_index_ = file_index; + Reset(); + } + } + void Reset() { + assert(file_index_ < files_.size()); + + // Reset current pointer + if (pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled()) { + pinned_iters_mgr_->PinIterator(file_iter_); + } else { + delete file_iter_; + } + + ReadRangeDelAggregator range_del_agg(&cfd_->internal_comparator(), + kMaxSequenceNumber /* upper_bound */); + file_iter_ = cfd_->table_cache()->NewIterator( + read_options_, *(cfd_->soptions()), cfd_->internal_comparator(), + *files_[file_index_], + read_options_.ignore_range_deletions ? nullptr : &range_del_agg, + prefix_extractor_, /*table_reader_ptr=*/nullptr, + /*file_read_hist=*/nullptr, TableReaderCaller::kUserIterator, + /*arena=*/nullptr, /*skip_filters=*/false, /*level=*/-1, + /*max_file_size_for_l0_meta_pin=*/0, + /*smallest_compaction_key=*/nullptr, + /*largest_compaction_key=*/nullptr, allow_unprepared_value_, + block_protection_bytes_per_key_); + file_iter_->SetPinnedItersMgr(pinned_iters_mgr_); + valid_ = false; + if (!range_del_agg.IsEmpty()) { + status_ = Status::NotSupported( + "Range tombstones unsupported with ForwardIterator"); + } + } + void SeekToLast() override { + status_ = Status::NotSupported("ForwardLevelIterator::SeekToLast()"); + valid_ = false; + } + void Prev() override { + status_ = Status::NotSupported("ForwardLevelIterator::Prev()"); + valid_ = false; + } + bool Valid() const override { return valid_; } + void SeekToFirst() override { + assert(file_iter_ != nullptr); + if (!status_.ok()) { + assert(!valid_); + return; + } + file_iter_->SeekToFirst(); + valid_ = file_iter_->Valid(); + } + void Seek(const Slice& internal_key) override { + assert(file_iter_ != nullptr); + + // This deviates from the usual convention for InternalIterator::Seek() in + // that it doesn't discard pre-existing error status. That's because this + // Seek() is only supposed to be called immediately after SetFileIndex() + // (which discards pre-existing error status), and SetFileIndex() may set + // an error status, which we shouldn't discard. + if (!status_.ok()) { + assert(!valid_); + return; + } + + file_iter_->Seek(internal_key); + valid_ = file_iter_->Valid(); + } + void SeekForPrev(const Slice& /*internal_key*/) override { + status_ = Status::NotSupported("ForwardLevelIterator::SeekForPrev()"); + valid_ = false; + } + void Next() override { + assert(valid_); + file_iter_->Next(); + for (;;) { + valid_ = file_iter_->Valid(); + if (!file_iter_->status().ok()) { + assert(!valid_); + return; + } + if (valid_) { + return; + } + if (file_index_ + 1 >= files_.size()) { + valid_ = false; + return; + } + SetFileIndex(file_index_ + 1); + if (!status_.ok()) { + assert(!valid_); + return; + } + file_iter_->SeekToFirst(); + } + } + Slice key() const override { + assert(valid_); + return file_iter_->key(); + } + Slice value() const override { + assert(valid_); + return file_iter_->value(); + } + Status status() const override { + if (!status_.ok()) { + return status_; + } else if (file_iter_) { + return file_iter_->status(); + } + return Status::OK(); + } + bool PrepareValue() override { + assert(valid_); + if (file_iter_->PrepareValue()) { + return true; + } + + assert(!file_iter_->Valid()); + valid_ = false; + return false; + } + bool IsKeyPinned() const override { + return pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled() && + file_iter_->IsKeyPinned(); + } + bool IsValuePinned() const override { + return pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled() && + file_iter_->IsValuePinned(); + } + void SetPinnedItersMgr(PinnedIteratorsManager* pinned_iters_mgr) override { + pinned_iters_mgr_ = pinned_iters_mgr; + if (file_iter_) { + file_iter_->SetPinnedItersMgr(pinned_iters_mgr_); + } + } + + private: + const ColumnFamilyData* const cfd_; + const ReadOptions& read_options_; + const std::vector& files_; + + bool valid_; + uint32_t file_index_; + Status status_; + InternalIterator* file_iter_; + PinnedIteratorsManager* pinned_iters_mgr_; + // Kept alive by ForwardIterator::sv_->mutable_cf_options + const std::shared_ptr& prefix_extractor_; + const bool allow_unprepared_value_; + const uint8_t block_protection_bytes_per_key_; +}; + +ForwardIterator::ForwardIterator(DBImpl* db, const ReadOptions& read_options, + ColumnFamilyData* cfd, + SuperVersion* current_sv, + bool allow_unprepared_value) + : db_(db), + read_options_(read_options), + cfd_(cfd), + prefix_extractor_(current_sv->mutable_cf_options.prefix_extractor.get()), + user_comparator_(cfd->user_comparator()), + allow_unprepared_value_(allow_unprepared_value), + immutable_min_heap_(MinIterComparator(&cfd_->internal_comparator())), + sv_(current_sv), + mutable_iter_(nullptr), + current_(nullptr), + valid_(false), + status_(Status::OK()), + immutable_status_(Status::OK()), + has_iter_trimmed_for_upper_bound_(false), + current_over_upper_bound_(false), + is_prev_set_(false), + is_prev_inclusive_(false), + pinned_iters_mgr_(nullptr) { + if (sv_) { + RebuildIterators(false); + } + if (!CheckFSFeatureSupport(cfd_->ioptions()->env->GetFileSystem().get(), + FSSupportedOps::kAsyncIO)) { + read_options_.async_io = false; + } + // immutable_status_ is a local aggregation of the + // status of the immutable Iterators. + // We have to PermitUncheckedError in case it is never + // used, otherwise it will fail ASSERT_STATUS_CHECKED. + immutable_status_.PermitUncheckedError(); +} + +ForwardIterator::~ForwardIterator() { Cleanup(true); } + +void ForwardIterator::SVCleanup(DBImpl* db, SuperVersion* sv, + bool background_purge_on_iterator_cleanup) { + if (sv->Unref()) { + // Job id == 0 means that this is not our background process, but rather + // user thread + JobContext job_context(0); + db->mutex_.Lock(); + sv->Cleanup(); + db->FindObsoleteFiles(&job_context, false, true); + if (background_purge_on_iterator_cleanup) { + db->ScheduleBgLogWriterClose(&job_context); + db->AddSuperVersionsToFreeQueue(sv); + db->SchedulePurge(); + } + db->mutex_.Unlock(); + if (!background_purge_on_iterator_cleanup) { + delete sv; + } + if (job_context.HaveSomethingToDelete()) { + db->PurgeObsoleteFiles(job_context, background_purge_on_iterator_cleanup); + } + job_context.Clean(); + } +} + +namespace { +struct SVCleanupParams { + DBImpl* db; + SuperVersion* sv; + bool background_purge_on_iterator_cleanup; +}; +} // anonymous namespace + +// Used in PinnedIteratorsManager to release pinned SuperVersion +void ForwardIterator::DeferredSVCleanup(void* arg) { + auto d = reinterpret_cast(arg); + ForwardIterator::SVCleanup(d->db, d->sv, + d->background_purge_on_iterator_cleanup); + delete d; +} + +void ForwardIterator::SVCleanup() { + if (sv_ == nullptr) { + return; + } + bool background_purge = + read_options_.background_purge_on_iterator_cleanup || + db_->immutable_db_options().avoid_unnecessary_blocking_io; + if (pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled()) { + // pinned_iters_mgr_ tells us to make sure that all visited key-value slices + // are alive until pinned_iters_mgr_->ReleasePinnedData() is called. + // The slices may point into some memtables owned by sv_, so we need to keep + // sv_ referenced until pinned_iters_mgr_ unpins everything. + auto p = new SVCleanupParams{db_, sv_, background_purge}; + pinned_iters_mgr_->PinPtr(p, &ForwardIterator::DeferredSVCleanup); + } else { + SVCleanup(db_, sv_, background_purge); + } +} + +void ForwardIterator::Cleanup(bool release_sv) { + if (mutable_iter_ != nullptr) { + DeleteIterator(mutable_iter_, true /* is_arena */); + } + + for (auto* m : imm_iters_) { + DeleteIterator(m, true /* is_arena */); + } + imm_iters_.clear(); + + for (auto* f : l0_iters_) { + DeleteIterator(f); + } + l0_iters_.clear(); + + for (auto* l : level_iters_) { + DeleteIterator(l); + } + level_iters_.clear(); + + if (release_sv) { + SVCleanup(); + } +} + +bool ForwardIterator::Valid() const { + // See UpdateCurrent(). + return valid_ ? !current_over_upper_bound_ : false; +} + +void ForwardIterator::SeekToFirst() { + if (sv_ == nullptr) { + RebuildIterators(true); + } else if (sv_->version_number != cfd_->GetSuperVersionNumber()) { + RenewIterators(); + } else if (immutable_status_.IsIncomplete()) { + ResetIncompleteIterators(); + } + SeekInternal(Slice(), true, false); +} + +bool ForwardIterator::IsOverUpperBound(const Slice& internal_key) const { + return !(read_options_.iterate_upper_bound == nullptr || + cfd_->internal_comparator().user_comparator()->Compare( + ExtractUserKey(internal_key), + *read_options_.iterate_upper_bound) < 0); +} + +void ForwardIterator::Seek(const Slice& internal_key) { + if (sv_ == nullptr) { + RebuildIterators(true); + } else if (sv_->version_number != cfd_->GetSuperVersionNumber()) { + RenewIterators(); + } else if (immutable_status_.IsIncomplete()) { + ResetIncompleteIterators(); + } + + SeekInternal(internal_key, false, false); + if (read_options_.async_io) { + SeekInternal(internal_key, false, true); + } +} + +// In case of async_io, SeekInternal is called twice with seek_after_async_io +// enabled in second call which only does seeking part to retrieve the blocks. +void ForwardIterator::SeekInternal(const Slice& internal_key, + bool seek_to_first, + bool seek_after_async_io) { + assert(mutable_iter_); + // mutable + if (!seek_after_async_io) { + seek_to_first ? mutable_iter_->SeekToFirst() + : mutable_iter_->Seek(internal_key); + } + + // immutable + // TODO(ljin): NeedToSeekImmutable has negative impact on performance + // if it turns to need to seek immutable often. We probably want to have + // an option to turn it off. + if (seek_to_first || seek_after_async_io || + NeedToSeekImmutable(internal_key)) { + if (!seek_after_async_io) { + immutable_status_ = Status::OK(); + if (has_iter_trimmed_for_upper_bound_ && + ( + // prev_ is not set yet + is_prev_set_ == false || + // We are doing SeekToFirst() and internal_key.size() = 0 + seek_to_first || + // prev_key_ > internal_key + cfd_->internal_comparator().InternalKeyComparator::Compare( + prev_key_.GetInternalKey(), internal_key) > 0)) { + // Some iterators are trimmed. Need to rebuild. + RebuildIterators(true); + // Already seeked mutable iter, so seek again + seek_to_first ? mutable_iter_->SeekToFirst() + : mutable_iter_->Seek(internal_key); + } + { + auto tmp = MinIterHeap(MinIterComparator(&cfd_->internal_comparator())); + immutable_min_heap_.swap(tmp); + } + for (size_t i = 0; i < imm_iters_.size(); i++) { + auto* m = imm_iters_[i]; + seek_to_first ? m->SeekToFirst() : m->Seek(internal_key); + if (!m->status().ok()) { + immutable_status_ = m->status(); + } else if (m->Valid()) { + immutable_min_heap_.push(m); + } + } + } + + Slice target_user_key; + if (!seek_to_first) { + target_user_key = ExtractUserKey(internal_key); + } + const VersionStorageInfo* vstorage = sv_->current->storage_info(); + const std::vector& l0 = vstorage->LevelFiles(0); + for (size_t i = 0; i < l0.size(); ++i) { + if (!l0_iters_[i]) { + continue; + } + if (seek_after_async_io) { + if (!l0_iters_[i]->status().IsTryAgain()) { + continue; + } + } + + if (seek_to_first) { + l0_iters_[i]->SeekToFirst(); + } else { + // If the target key passes over the largest key, we are sure Next() + // won't go over this file. + if (seek_after_async_io == false && + user_comparator_->Compare(target_user_key, + l0[i]->largest.user_key()) > 0) { + if (read_options_.iterate_upper_bound != nullptr) { + has_iter_trimmed_for_upper_bound_ = true; + DeleteIterator(l0_iters_[i]); + l0_iters_[i] = nullptr; + } + continue; + } + l0_iters_[i]->Seek(internal_key); + } + + if (l0_iters_[i]->status().IsTryAgain()) { + assert(!seek_after_async_io); + continue; + } else if (!l0_iters_[i]->status().ok()) { + immutable_status_ = l0_iters_[i]->status(); + } else if (l0_iters_[i]->Valid() && + !IsOverUpperBound(l0_iters_[i]->key())) { + immutable_min_heap_.push(l0_iters_[i]); + } else { + has_iter_trimmed_for_upper_bound_ = true; + DeleteIterator(l0_iters_[i]); + l0_iters_[i] = nullptr; + } + } + + for (int32_t level = 1; level < vstorage->num_levels(); ++level) { + const std::vector& level_files = + vstorage->LevelFiles(level); + if (level_files.empty()) { + continue; + } + if (level_iters_[level - 1] == nullptr) { + continue; + } + + if (seek_after_async_io) { + if (!level_iters_[level - 1]->status().IsTryAgain()) { + continue; + } + } + uint32_t f_idx = 0; + if (!seek_to_first && !seek_after_async_io) { + f_idx = FindFileInRange(level_files, internal_key, 0, + static_cast(level_files.size())); + } + + // Seek + if (seek_after_async_io || f_idx < level_files.size()) { + if (!seek_after_async_io) { + level_iters_[level - 1]->SetFileIndex(f_idx); + } + seek_to_first ? level_iters_[level - 1]->SeekToFirst() + : level_iters_[level - 1]->Seek(internal_key); + + if (level_iters_[level - 1]->status().IsTryAgain()) { + assert(!seek_after_async_io); + continue; + } else if (!level_iters_[level - 1]->status().ok()) { + immutable_status_ = level_iters_[level - 1]->status(); + } else if (level_iters_[level - 1]->Valid() && + !IsOverUpperBound(level_iters_[level - 1]->key())) { + immutable_min_heap_.push(level_iters_[level - 1]); + } else { + // Nothing in this level is interesting. Remove. + has_iter_trimmed_for_upper_bound_ = true; + DeleteIterator(level_iters_[level - 1]); + level_iters_[level - 1] = nullptr; + } + } + } + + if (seek_to_first) { + is_prev_set_ = false; + } else { + prev_key_.SetInternalKey(internal_key); + is_prev_set_ = true; + is_prev_inclusive_ = true; + } + + TEST_SYNC_POINT_CALLBACK("ForwardIterator::SeekInternal:Immutable", this); + } else if (current_ && current_ != mutable_iter_) { + // current_ is one of immutable iterators, push it back to the heap + immutable_min_heap_.push(current_); + } + + // For async_io, it should be updated when seek_after_async_io is true (in + // second call). + if (seek_to_first || !read_options_.async_io || seek_after_async_io) { + UpdateCurrent(); + } + TEST_SYNC_POINT_CALLBACK("ForwardIterator::SeekInternal:Return", this); +} + +void ForwardIterator::Next() { + assert(valid_); + bool update_prev_key = false; + + if (sv_ == nullptr || sv_->version_number != cfd_->GetSuperVersionNumber()) { + std::string current_key = key().ToString(); + Slice old_key(current_key.data(), current_key.size()); + + if (sv_ == nullptr) { + RebuildIterators(true); + } else { + RenewIterators(); + } + + SeekInternal(old_key, false, false); + if (read_options_.async_io) { + SeekInternal(old_key, false, true); + } + + if (!valid_ || key().compare(old_key) != 0) { + return; + } + } else if (current_ != mutable_iter_) { + // It is going to advance immutable iterator + + if (is_prev_set_ && prefix_extractor_) { + // advance prev_key_ to current_ only if they share the same prefix + update_prev_key = + prefix_extractor_->Transform(prev_key_.GetUserKey()) + .compare(prefix_extractor_->Transform(current_->key())) == 0; + } else { + update_prev_key = true; + } + + if (update_prev_key) { + prev_key_.SetInternalKey(current_->key()); + is_prev_set_ = true; + is_prev_inclusive_ = false; + } + } + + current_->Next(); + if (current_ != mutable_iter_) { + if (!current_->status().ok()) { + immutable_status_ = current_->status(); + } else if ((current_->Valid()) && (!IsOverUpperBound(current_->key()))) { + immutable_min_heap_.push(current_); + } else { + if ((current_->Valid()) && (IsOverUpperBound(current_->key()))) { + // remove the current iterator + DeleteCurrentIter(); + current_ = nullptr; + } + if (update_prev_key) { + mutable_iter_->Seek(prev_key_.GetInternalKey()); + } + } + } + UpdateCurrent(); + TEST_SYNC_POINT_CALLBACK("ForwardIterator::Next:Return", this); +} + +Slice ForwardIterator::key() const { + assert(valid_); + return current_->key(); +} + +Slice ForwardIterator::value() const { + assert(valid_); + return current_->value(); +} + +Status ForwardIterator::status() const { + if (!status_.ok()) { + return status_; + } else if (!mutable_iter_->status().ok()) { + return mutable_iter_->status(); + } + + return immutable_status_; +} + +bool ForwardIterator::PrepareValue() { + assert(valid_); + if (current_->PrepareValue()) { + return true; + } + + assert(!current_->Valid()); + assert(!current_->status().ok()); + assert(current_ != mutable_iter_); // memtable iterator can't fail + assert(immutable_status_.ok()); + + valid_ = false; + immutable_status_ = current_->status(); + return false; +} + +Status ForwardIterator::GetProperty(std::string prop_name, std::string* prop) { + assert(prop != nullptr); + if (prop_name == "rocksdb.iterator.super-version-number") { + *prop = std::to_string(sv_->version_number); + return Status::OK(); + } + return Status::InvalidArgument(); +} + +void ForwardIterator::SetPinnedItersMgr( + PinnedIteratorsManager* pinned_iters_mgr) { + pinned_iters_mgr_ = pinned_iters_mgr; + UpdateChildrenPinnedItersMgr(); +} + +void ForwardIterator::UpdateChildrenPinnedItersMgr() { + // Set PinnedIteratorsManager for mutable memtable iterator. + if (mutable_iter_) { + mutable_iter_->SetPinnedItersMgr(pinned_iters_mgr_); + } + + // Set PinnedIteratorsManager for immutable memtable iterators. + for (InternalIterator* child_iter : imm_iters_) { + if (child_iter) { + child_iter->SetPinnedItersMgr(pinned_iters_mgr_); + } + } + + // Set PinnedIteratorsManager for L0 files iterators. + for (InternalIterator* child_iter : l0_iters_) { + if (child_iter) { + child_iter->SetPinnedItersMgr(pinned_iters_mgr_); + } + } + + // Set PinnedIteratorsManager for L1+ levels iterators. + for (ForwardLevelIterator* child_iter : level_iters_) { + if (child_iter) { + child_iter->SetPinnedItersMgr(pinned_iters_mgr_); + } + } +} + +bool ForwardIterator::IsKeyPinned() const { + return pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled() && + current_->IsKeyPinned(); +} + +bool ForwardIterator::IsValuePinned() const { + return pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled() && + current_->IsValuePinned(); +} + +void ForwardIterator::RebuildIterators(bool refresh_sv) { + // Clean up + Cleanup(refresh_sv); + if (refresh_sv) { + // New + sv_ = cfd_->GetReferencedSuperVersion(db_); + } + ReadRangeDelAggregator range_del_agg(&cfd_->internal_comparator(), + kMaxSequenceNumber /* upper_bound */); + mutable_iter_ = sv_->mem->NewIterator(read_options_, &arena_); + sv_->imm->AddIterators(read_options_, &imm_iters_, &arena_); + if (!read_options_.ignore_range_deletions) { + std::unique_ptr range_del_iter( + sv_->mem->NewRangeTombstoneIterator( + read_options_, sv_->current->version_set()->LastSequence(), + false /* immutable_memtable */)); + range_del_agg.AddTombstones(std::move(range_del_iter)); + // Always return Status::OK(). + Status temp_s = sv_->imm->AddRangeTombstoneIterators(read_options_, &arena_, + &range_del_agg); + assert(temp_s.ok()); + } + has_iter_trimmed_for_upper_bound_ = false; + + const auto* vstorage = sv_->current->storage_info(); + const auto& l0_files = vstorage->LevelFiles(0); + l0_iters_.reserve(l0_files.size()); + for (const auto* l0 : l0_files) { + if ((read_options_.iterate_upper_bound != nullptr) && + cfd_->internal_comparator().user_comparator()->Compare( + l0->smallest.user_key(), *read_options_.iterate_upper_bound) > 0) { + // No need to set has_iter_trimmed_for_upper_bound_: this ForwardIterator + // will never be interested in files with smallest key above + // iterate_upper_bound, since iterate_upper_bound can't be changed. + l0_iters_.push_back(nullptr); + continue; + } + l0_iters_.push_back(cfd_->table_cache()->NewIterator( + read_options_, *cfd_->soptions(), cfd_->internal_comparator(), *l0, + read_options_.ignore_range_deletions ? nullptr : &range_del_agg, + sv_->mutable_cf_options.prefix_extractor, + /*table_reader_ptr=*/nullptr, /*file_read_hist=*/nullptr, + TableReaderCaller::kUserIterator, /*arena=*/nullptr, + /*skip_filters=*/false, /*level=*/-1, + MaxFileSizeForL0MetaPin(sv_->mutable_cf_options), + /*smallest_compaction_key=*/nullptr, + /*largest_compaction_key=*/nullptr, allow_unprepared_value_, + sv_->mutable_cf_options.block_protection_bytes_per_key)); + } + BuildLevelIterators(vstorage, sv_); + current_ = nullptr; + is_prev_set_ = false; + + UpdateChildrenPinnedItersMgr(); + if (!range_del_agg.IsEmpty()) { + status_ = Status::NotSupported( + "Range tombstones unsupported with ForwardIterator"); + valid_ = false; + } +} + +void ForwardIterator::RenewIterators() { + SuperVersion* svnew; + assert(sv_); + svnew = cfd_->GetReferencedSuperVersion(db_); + + if (mutable_iter_ != nullptr) { + DeleteIterator(mutable_iter_, true /* is_arena */); + } + for (auto* m : imm_iters_) { + DeleteIterator(m, true /* is_arena */); + } + imm_iters_.clear(); + + mutable_iter_ = svnew->mem->NewIterator(read_options_, &arena_); + svnew->imm->AddIterators(read_options_, &imm_iters_, &arena_); + ReadRangeDelAggregator range_del_agg(&cfd_->internal_comparator(), + kMaxSequenceNumber /* upper_bound */); + if (!read_options_.ignore_range_deletions) { + std::unique_ptr range_del_iter( + svnew->mem->NewRangeTombstoneIterator( + read_options_, sv_->current->version_set()->LastSequence(), + false /* immutable_memtable */)); + range_del_agg.AddTombstones(std::move(range_del_iter)); + // Always return Status::OK(). + Status temp_s = svnew->imm->AddRangeTombstoneIterators( + read_options_, &arena_, &range_del_agg); + assert(temp_s.ok()); + } + + const auto* vstorage = sv_->current->storage_info(); + const auto& l0_files = vstorage->LevelFiles(0); + const auto* vstorage_new = svnew->current->storage_info(); + const auto& l0_files_new = vstorage_new->LevelFiles(0); + size_t iold, inew; + bool found; + std::vector l0_iters_new; + l0_iters_new.reserve(l0_files_new.size()); + + for (inew = 0; inew < l0_files_new.size(); inew++) { + found = false; + for (iold = 0; iold < l0_files.size(); iold++) { + if (l0_files[iold] == l0_files_new[inew]) { + found = true; + break; + } + } + if (found) { + if (l0_iters_[iold] == nullptr) { + l0_iters_new.push_back(nullptr); + TEST_SYNC_POINT_CALLBACK("ForwardIterator::RenewIterators:Null", this); + } else { + l0_iters_new.push_back(l0_iters_[iold]); + l0_iters_[iold] = nullptr; + TEST_SYNC_POINT_CALLBACK("ForwardIterator::RenewIterators:Copy", this); + } + continue; + } + l0_iters_new.push_back(cfd_->table_cache()->NewIterator( + read_options_, *cfd_->soptions(), cfd_->internal_comparator(), + *l0_files_new[inew], + read_options_.ignore_range_deletions ? nullptr : &range_del_agg, + svnew->mutable_cf_options.prefix_extractor, + /*table_reader_ptr=*/nullptr, /*file_read_hist=*/nullptr, + TableReaderCaller::kUserIterator, /*arena=*/nullptr, + /*skip_filters=*/false, /*level=*/-1, + MaxFileSizeForL0MetaPin(svnew->mutable_cf_options), + /*smallest_compaction_key=*/nullptr, + /*largest_compaction_key=*/nullptr, allow_unprepared_value_, + svnew->mutable_cf_options.block_protection_bytes_per_key)); + } + + for (auto* f : l0_iters_) { + DeleteIterator(f); + } + l0_iters_.clear(); + l0_iters_ = l0_iters_new; + + for (auto* l : level_iters_) { + DeleteIterator(l); + } + level_iters_.clear(); + BuildLevelIterators(vstorage_new, svnew); + current_ = nullptr; + is_prev_set_ = false; + SVCleanup(); + sv_ = svnew; + + UpdateChildrenPinnedItersMgr(); + if (!range_del_agg.IsEmpty()) { + status_ = Status::NotSupported( + "Range tombstones unsupported with ForwardIterator"); + valid_ = false; + } +} + +void ForwardIterator::BuildLevelIterators(const VersionStorageInfo* vstorage, + SuperVersion* sv) { + level_iters_.reserve(vstorage->num_levels() - 1); + for (int32_t level = 1; level < vstorage->num_levels(); ++level) { + const auto& level_files = vstorage->LevelFiles(level); + if ((level_files.empty()) || + ((read_options_.iterate_upper_bound != nullptr) && + (user_comparator_->Compare(*read_options_.iterate_upper_bound, + level_files[0]->smallest.user_key()) < + 0))) { + level_iters_.push_back(nullptr); + if (!level_files.empty()) { + has_iter_trimmed_for_upper_bound_ = true; + } + } else { + level_iters_.push_back(new ForwardLevelIterator( + cfd_, read_options_, level_files, + sv->mutable_cf_options.prefix_extractor, allow_unprepared_value_, + sv->mutable_cf_options.block_protection_bytes_per_key)); + } + } +} + +void ForwardIterator::ResetIncompleteIterators() { + const auto& l0_files = sv_->current->storage_info()->LevelFiles(0); + for (size_t i = 0; i < l0_iters_.size(); ++i) { + assert(i < l0_files.size()); + if (!l0_iters_[i] || !l0_iters_[i]->status().IsIncomplete()) { + continue; + } + DeleteIterator(l0_iters_[i]); + l0_iters_[i] = cfd_->table_cache()->NewIterator( + read_options_, *cfd_->soptions(), cfd_->internal_comparator(), + *l0_files[i], /*range_del_agg=*/nullptr, + sv_->mutable_cf_options.prefix_extractor, + /*table_reader_ptr=*/nullptr, /*file_read_hist=*/nullptr, + TableReaderCaller::kUserIterator, /*arena=*/nullptr, + /*skip_filters=*/false, /*level=*/-1, + MaxFileSizeForL0MetaPin(sv_->mutable_cf_options), + /*smallest_compaction_key=*/nullptr, + /*largest_compaction_key=*/nullptr, allow_unprepared_value_, + sv_->mutable_cf_options.block_protection_bytes_per_key); + l0_iters_[i]->SetPinnedItersMgr(pinned_iters_mgr_); + } + + for (auto* level_iter : level_iters_) { + if (level_iter && level_iter->status().IsIncomplete()) { + level_iter->Reset(); + } + } + + current_ = nullptr; + is_prev_set_ = false; +} + +void ForwardIterator::UpdateCurrent() { + if (immutable_min_heap_.empty() && !mutable_iter_->Valid()) { + current_ = nullptr; + } else if (immutable_min_heap_.empty()) { + current_ = mutable_iter_; + } else if (!mutable_iter_->Valid()) { + current_ = immutable_min_heap_.top(); + immutable_min_heap_.pop(); + } else { + current_ = immutable_min_heap_.top(); + assert(current_ != nullptr); + assert(current_->Valid()); + int cmp = cfd_->internal_comparator().InternalKeyComparator::Compare( + mutable_iter_->key(), current_->key()); + assert(cmp != 0); + if (cmp > 0) { + immutable_min_heap_.pop(); + } else { + current_ = mutable_iter_; + } + } + valid_ = current_ != nullptr && immutable_status_.ok(); + if (!status_.ok()) { + status_ = Status::OK(); + } + + // Upper bound doesn't apply to the memtable iterator. We want Valid() to + // return false when all iterators are over iterate_upper_bound, but can't + // just set valid_ to false, as that would effectively disable the tailing + // optimization (Seek() would be called on all immutable iterators regardless + // of whether the target key is greater than prev_key_). + current_over_upper_bound_ = valid_ && IsOverUpperBound(current_->key()); +} + +bool ForwardIterator::NeedToSeekImmutable(const Slice& target) { + // We maintain the interval (prev_key_, immutable_min_heap_.top()->key()) + // such that there are no records with keys within that range in + // immutable_min_heap_. Since immutable structures (SST files and immutable + // memtables) can't change in this version, we don't need to do a seek if + // 'target' belongs to that interval (immutable_min_heap_.top() is already + // at the correct position). + + if (!valid_ || !current_ || !is_prev_set_ || !immutable_status_.ok()) { + return true; + } + Slice prev_key = prev_key_.GetInternalKey(); + if (prefix_extractor_ && prefix_extractor_->Transform(target).compare( + prefix_extractor_->Transform(prev_key)) != 0) { + return true; + } + if (cfd_->internal_comparator().InternalKeyComparator::Compare( + prev_key, target) >= (is_prev_inclusive_ ? 1 : 0)) { + return true; + } + + if (immutable_min_heap_.empty() && current_ == mutable_iter_) { + // Nothing to seek on. + return false; + } + if (cfd_->internal_comparator().InternalKeyComparator::Compare( + target, current_ == mutable_iter_ ? immutable_min_heap_.top()->key() + : current_->key()) > 0) { + return true; + } + return false; +} + +void ForwardIterator::DeleteCurrentIter() { + const VersionStorageInfo* vstorage = sv_->current->storage_info(); + const std::vector& l0 = vstorage->LevelFiles(0); + for (size_t i = 0; i < l0.size(); ++i) { + if (!l0_iters_[i]) { + continue; + } + if (l0_iters_[i] == current_) { + has_iter_trimmed_for_upper_bound_ = true; + DeleteIterator(l0_iters_[i]); + l0_iters_[i] = nullptr; + return; + } + } + + for (int32_t level = 1; level < vstorage->num_levels(); ++level) { + if (level_iters_[level - 1] == nullptr) { + continue; + } + if (level_iters_[level - 1] == current_) { + has_iter_trimmed_for_upper_bound_ = true; + DeleteIterator(level_iters_[level - 1]); + level_iters_[level - 1] = nullptr; + } + } +} + +bool ForwardIterator::TEST_CheckDeletedIters(int* pdeleted_iters, + int* pnum_iters) { + bool retval = false; + int deleted_iters = 0; + int num_iters = 0; + + const VersionStorageInfo* vstorage = sv_->current->storage_info(); + const std::vector& l0 = vstorage->LevelFiles(0); + for (size_t i = 0; i < l0.size(); ++i) { + if (!l0_iters_[i]) { + retval = true; + deleted_iters++; + } else { + num_iters++; + } + } + + for (int32_t level = 1; level < vstorage->num_levels(); ++level) { + if ((level_iters_[level - 1] == nullptr) && + (!vstorage->LevelFiles(level).empty())) { + retval = true; + deleted_iters++; + } else if (!vstorage->LevelFiles(level).empty()) { + num_iters++; + } + } + if ((!retval) && num_iters <= 1) { + retval = true; + } + if (pdeleted_iters) { + *pdeleted_iters = deleted_iters; + } + if (pnum_iters) { + *pnum_iters = num_iters; + } + return retval; +} + +uint32_t ForwardIterator::FindFileInRange( + const std::vector& files, const Slice& internal_key, + uint32_t left, uint32_t right) { + auto cmp = [&](const FileMetaData* f, const Slice& k) -> bool { + return cfd_->internal_comparator().InternalKeyComparator::Compare( + f->largest.Encode(), k) < 0; + }; + const auto& b = files.begin(); + return static_cast( + std::lower_bound(b + left, b + right, internal_key, cmp) - b); +} + +void ForwardIterator::DeleteIterator(InternalIterator* iter, bool is_arena) { + if (iter == nullptr) { + return; + } + + if (pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled()) { + pinned_iters_mgr_->PinIterator(iter, is_arena); + } else { + if (is_arena) { + iter->~InternalIterator(); + } else { + delete iter; + } + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/forward_iterator.h b/librocksdb-sys/rocksdb/db/forward_iterator.h new file mode 100644 index 0000000..cb418ae --- /dev/null +++ b/librocksdb-sys/rocksdb/db/forward_iterator.h @@ -0,0 +1,166 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include "rocksdb/comparator.h" + +#include +#include +#include + +#include "memory/arena.h" +#include "rocksdb/db.h" +#include "rocksdb/iterator.h" +#include "rocksdb/options.h" +#include "table/internal_iterator.h" + +namespace ROCKSDB_NAMESPACE { + +class DBImpl; +class Env; +struct SuperVersion; +class ColumnFamilyData; +class ForwardLevelIterator; +class VersionStorageInfo; +struct FileMetaData; + +class MinIterComparator { + public: + explicit MinIterComparator(const CompareInterface* comparator) + : comparator_(comparator) {} + + bool operator()(InternalIterator* a, InternalIterator* b) { + return comparator_->Compare(a->key(), b->key()) > 0; + } + + private: + const CompareInterface* comparator_; +}; + +using MinIterHeap = + std::priority_queue, + MinIterComparator>; + +/** + * ForwardIterator is a special type of iterator that only supports Seek() + * and Next(). It is expected to perform better than TailingIterator by + * removing the encapsulation and making all information accessible within + * the iterator. At the current implementation, snapshot is taken at the + * time Seek() is called. The Next() followed do not see new values after. + */ +class ForwardIterator : public InternalIterator { + public: + ForwardIterator(DBImpl* db, const ReadOptions& read_options, + ColumnFamilyData* cfd, SuperVersion* current_sv = nullptr, + bool allow_unprepared_value = false); + virtual ~ForwardIterator(); + + void SeekForPrev(const Slice& /*target*/) override { + status_ = Status::NotSupported("ForwardIterator::SeekForPrev()"); + valid_ = false; + } + void SeekToLast() override { + status_ = Status::NotSupported("ForwardIterator::SeekToLast()"); + valid_ = false; + } + void Prev() override { + status_ = Status::NotSupported("ForwardIterator::Prev"); + valid_ = false; + } + + virtual bool Valid() const override; + void SeekToFirst() override; + virtual void Seek(const Slice& target) override; + virtual void Next() override; + virtual Slice key() const override; + virtual Slice value() const override; + virtual Status status() const override; + virtual bool PrepareValue() override; + virtual Status GetProperty(std::string prop_name, std::string* prop) override; + virtual void SetPinnedItersMgr( + PinnedIteratorsManager* pinned_iters_mgr) override; + virtual bool IsKeyPinned() const override; + virtual bool IsValuePinned() const override; + + bool TEST_CheckDeletedIters(int* deleted_iters, int* num_iters); + + private: + void Cleanup(bool release_sv); + // Unreference and, if needed, clean up the current SuperVersion. This is + // either done immediately or deferred until this iterator is unpinned by + // PinnedIteratorsManager. + void SVCleanup(); + static void SVCleanup(DBImpl* db, SuperVersion* sv, + bool background_purge_on_iterator_cleanup); + static void DeferredSVCleanup(void* arg); + + void RebuildIterators(bool refresh_sv); + void RenewIterators(); + void BuildLevelIterators(const VersionStorageInfo* vstorage, + SuperVersion* sv); + void ResetIncompleteIterators(); + void SeekInternal(const Slice& internal_key, bool seek_to_first, + bool seek_after_async_io); + + void UpdateCurrent(); + bool NeedToSeekImmutable(const Slice& internal_key); + void DeleteCurrentIter(); + uint32_t FindFileInRange(const std::vector& files, + const Slice& internal_key, uint32_t left, + uint32_t right); + + bool IsOverUpperBound(const Slice& internal_key) const; + + // Set PinnedIteratorsManager for all children Iterators, this function should + // be called whenever we update children Iterators or pinned_iters_mgr_. + void UpdateChildrenPinnedItersMgr(); + + // A helper function that will release iter in the proper manner, or pass it + // to pinned_iters_mgr_ to release it later if pinning is enabled. + void DeleteIterator(InternalIterator* iter, bool is_arena = false); + + DBImpl* const db_; + ReadOptions read_options_; + ColumnFamilyData* const cfd_; + const SliceTransform* const prefix_extractor_; + const Comparator* user_comparator_; + const bool allow_unprepared_value_; + MinIterHeap immutable_min_heap_; + + SuperVersion* sv_; + InternalIterator* mutable_iter_; + std::vector imm_iters_; + std::vector l0_iters_; + std::vector level_iters_; + InternalIterator* current_; + bool valid_; + + // Internal iterator status; set only by one of the unsupported methods. + Status status_; + // Status of immutable iterators, maintained here to avoid iterating over + // all of them in status(). + Status immutable_status_; + // Indicates that at least one of the immutable iterators pointed to a key + // larger than iterate_upper_bound and was therefore destroyed. Seek() may + // need to rebuild such iterators. + bool has_iter_trimmed_for_upper_bound_; + // Is current key larger than iterate_upper_bound? If so, makes Valid() + // return false. + bool current_over_upper_bound_; + + // Left endpoint of the range of keys that immutable iterators currently + // cover. When Seek() is called with a key that's within that range, immutable + // iterators don't need to be moved; see NeedToSeekImmutable(). This key is + // included in the range after a Seek(), but excluded when advancing the + // iterator using Next(). + IterKey prev_key_; + bool is_prev_set_; + bool is_prev_inclusive_; + + PinnedIteratorsManager* pinned_iters_mgr_; + Arena arena_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/forward_iterator_bench.cc b/librocksdb-sys/rocksdb/db/forward_iterator_bench.cc new file mode 100644 index 0000000..b57b119 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/forward_iterator_bench.cc @@ -0,0 +1,378 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#if !defined(GFLAGS) +#include +int main() { + fprintf(stderr, "Please install gflags to run rocksdb tools\n"); + return 1; +} +#elif defined(OS_MACOSX) || defined(OS_WIN) +// Block forward_iterator_bench under MAC and Windows +int main() { return 0; } +#else +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "port/port.h" +#include "rocksdb/cache.h" +#include "rocksdb/db.h" +#include "rocksdb/status.h" +#include "rocksdb/table.h" +#include "test_util/testharness.h" +#include "util/gflags_compat.h" + +const int MAX_SHARDS = 100000; + +DEFINE_int32(writers, 8, ""); +DEFINE_int32(readers, 8, ""); +DEFINE_int64(rate, 100000, ""); +DEFINE_int64(value_size, 300, ""); +DEFINE_int64(shards, 1000, ""); +DEFINE_int64(memtable_size, 500000000, ""); +DEFINE_int64(block_cache_size, 300000000, ""); +DEFINE_int64(block_size, 65536, ""); +DEFINE_double(runtime, 300.0, ""); +DEFINE_bool(cache_only_first, true, ""); +DEFINE_bool(iterate_upper_bound, true, ""); + +struct Stats { + char pad1[128] __attribute__((__unused__)); + std::atomic written{0}; + char pad2[128] __attribute__((__unused__)); + std::atomic read{0}; + std::atomic cache_misses{0}; + char pad3[128] __attribute__((__unused__)); +} stats; + +struct Key { + Key() {} + Key(uint64_t shard_in, uint64_t seqno_in) + : shard_be(htobe64(shard_in)), seqno_be(htobe64(seqno_in)) {} + + uint64_t shard() const { return be64toh(shard_be); } + uint64_t seqno() const { return be64toh(seqno_be); } + + private: + uint64_t shard_be; + uint64_t seqno_be; +} __attribute__((__packed__)); + +struct Reader; +struct Writer; + +struct ShardState { + char pad1[128] __attribute__((__unused__)); + std::atomic last_written{0}; + Writer* writer; + Reader* reader; + char pad2[128] __attribute__((__unused__)); + std::atomic last_read{0}; + std::unique_ptr it; + std::unique_ptr it_cacheonly; + Key upper_bound; + ROCKSDB_NAMESPACE::Slice upper_bound_slice; + char pad3[128] __attribute__((__unused__)); +}; + +struct Reader { + public: + explicit Reader(std::vector* shard_states, + ROCKSDB_NAMESPACE::DB* db) + : shard_states_(shard_states), db_(db) { + sem_init(&sem_, 0, 0); + thread_ = port::Thread(&Reader::run, this); + } + + void run() { + while (1) { + sem_wait(&sem_); + if (done_.load()) { + break; + } + + uint64_t shard; + { + std::lock_guard guard(queue_mutex_); + assert(!shards_pending_queue_.empty()); + shard = shards_pending_queue_.front(); + shards_pending_queue_.pop(); + shards_pending_set_.reset(shard); + } + readOnceFromShard(shard); + } + } + + void readOnceFromShard(uint64_t shard) { + ShardState& state = (*shard_states_)[shard]; + if (!state.it) { + // Initialize iterators + ROCKSDB_NAMESPACE::ReadOptions options; + options.tailing = true; + if (FLAGS_iterate_upper_bound) { + state.upper_bound = Key(shard, std::numeric_limits::max()); + state.upper_bound_slice = ROCKSDB_NAMESPACE::Slice( + (const char*)&state.upper_bound, sizeof(state.upper_bound)); + options.iterate_upper_bound = &state.upper_bound_slice; + } + + state.it.reset(db_->NewIterator(options)); + + if (FLAGS_cache_only_first) { + options.read_tier = ROCKSDB_NAMESPACE::ReadTier::kBlockCacheTier; + state.it_cacheonly.reset(db_->NewIterator(options)); + } + } + + const uint64_t upto = state.last_written.load(); + for (ROCKSDB_NAMESPACE::Iterator* it : + {state.it_cacheonly.get(), state.it.get()}) { + if (it == nullptr) { + continue; + } + if (state.last_read.load() >= upto) { + break; + } + bool need_seek = true; + for (uint64_t seq = state.last_read.load() + 1; seq <= upto; ++seq) { + if (need_seek) { + Key from(shard, state.last_read.load() + 1); + it->Seek(ROCKSDB_NAMESPACE::Slice((const char*)&from, sizeof(from))); + need_seek = false; + } else { + it->Next(); + } + if (it->status().IsIncomplete()) { + ++::stats.cache_misses; + break; + } + assert(it->Valid()); + assert(it->key().size() == sizeof(Key)); + Key key; + memcpy(&key, it->key().data(), it->key().size()); + // fprintf(stderr, "Expecting (%ld, %ld) read (%ld, %ld)\n", + // shard, seq, key.shard(), key.seqno()); + assert(key.shard() == shard); + assert(key.seqno() == seq); + state.last_read.store(seq); + ++::stats.read; + } + } + } + + void onWrite(uint64_t shard) { + { + std::lock_guard guard(queue_mutex_); + if (!shards_pending_set_.test(shard)) { + shards_pending_queue_.push(shard); + shards_pending_set_.set(shard); + sem_post(&sem_); + } + } + } + + ~Reader() { + done_.store(true); + sem_post(&sem_); + thread_.join(); + } + + private: + char pad1[128] __attribute__((__unused__)); + std::vector* shard_states_; + ROCKSDB_NAMESPACE::DB* db_; + ROCKSDB_NAMESPACE::port::Thread thread_; + sem_t sem_; + std::mutex queue_mutex_; + std::bitset shards_pending_set_; + std::queue shards_pending_queue_; + std::atomic done_{false}; + char pad2[128] __attribute__((__unused__)); +}; + +struct Writer { + explicit Writer(std::vector* shard_states, + ROCKSDB_NAMESPACE::DB* db) + : shard_states_(shard_states), db_(db) {} + + void start() { thread_ = port::Thread(&Writer::run, this); } + + void run() { + std::queue workq; + std::chrono::steady_clock::time_point deadline( + std::chrono::steady_clock::now() + + std::chrono::nanoseconds((uint64_t)(1000000000 * FLAGS_runtime))); + std::vector my_shards; + for (int i = 1; i <= FLAGS_shards; ++i) { + if ((*shard_states_)[i].writer == this) { + my_shards.push_back(i); + } + } + + std::mt19937 rng{std::random_device()()}; + std::uniform_int_distribution shard_dist( + 0, static_cast(my_shards.size()) - 1); + std::string value(FLAGS_value_size, '*'); + + while (1) { + auto now = std::chrono::steady_clock::now(); + if (FLAGS_runtime >= 0 && now >= deadline) { + break; + } + if (workq.empty()) { + for (int i = 0; i < FLAGS_rate; i += FLAGS_writers) { + std::chrono::nanoseconds offset(1000000000LL * i / FLAGS_rate); + workq.push(now + offset); + } + } + while (!workq.empty() && workq.front() < now) { + workq.pop(); + uint64_t shard = my_shards[shard_dist(rng)]; + ShardState& state = (*shard_states_)[shard]; + uint64_t seqno = state.last_written.load() + 1; + Key key(shard, seqno); + // fprintf(stderr, "Writing (%ld, %ld)\n", shard, seqno); + ROCKSDB_NAMESPACE::Status status = + db_->Put(ROCKSDB_NAMESPACE::WriteOptions(), + ROCKSDB_NAMESPACE::Slice((const char*)&key, sizeof(key)), + ROCKSDB_NAMESPACE::Slice(value)); + assert(status.ok()); + state.last_written.store(seqno); + state.reader->onWrite(shard); + ++::stats.written; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + // fprintf(stderr, "Writer done\n"); + } + + ~Writer() { thread_.join(); } + + private: + char pad1[128] __attribute__((__unused__)); + std::vector* shard_states_; + ROCKSDB_NAMESPACE::DB* db_; + ROCKSDB_NAMESPACE::port::Thread thread_; + char pad2[128] __attribute__((__unused__)); +}; + +struct StatsThread { + explicit StatsThread(ROCKSDB_NAMESPACE::DB* db) + : db_(db), thread_(&StatsThread::run, this) {} + + void run() { + auto tstart = std::chrono::steady_clock::now(), tlast = tstart; + uint64_t wlast = 0, rlast = 0; + while (!done_.load()) { + { + std::unique_lock lock(cvm_); + cv_.wait_for(lock, std::chrono::seconds(1)); + } + auto now = std::chrono::steady_clock::now(); + double elapsed = + std::chrono::duration_cast >(now - + tlast) + .count(); + uint64_t w = ::stats.written.load(); + uint64_t r = ::stats.read.load(); + fprintf(stderr, + "%s elapsed %4lds | written %10ld | w/s %10.0f | read %10ld | " + "r/s %10.0f | cache misses %10ld\n", + db_->GetEnv()->TimeToString(time(nullptr)).c_str(), + std::chrono::duration_cast(now - tstart) + .count(), + w, (w - wlast) / elapsed, r, (r - rlast) / elapsed, + ::stats.cache_misses.load()); + wlast = w; + rlast = r; + tlast = now; + } + } + + ~StatsThread() { + { + std::lock_guard guard(cvm_); + done_.store(true); + } + cv_.notify_all(); + thread_.join(); + } + + private: + ROCKSDB_NAMESPACE::DB* db_; + std::mutex cvm_; + std::condition_variable cv_; + ROCKSDB_NAMESPACE::port::Thread thread_; + std::atomic done_{false}; +}; + +int main(int argc, char** argv) { + GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + + std::mt19937 rng{std::random_device()()}; + ROCKSDB_NAMESPACE::Status status; + std::string path = + ROCKSDB_NAMESPACE::test::PerThreadDBPath("forward_iterator_test"); + fprintf(stderr, "db path is %s\n", path.c_str()); + ROCKSDB_NAMESPACE::Options options; + options.create_if_missing = true; + options.compression = ROCKSDB_NAMESPACE::CompressionType::kNoCompression; + options.compaction_style = + ROCKSDB_NAMESPACE::CompactionStyle::kCompactionStyleNone; + options.level0_slowdown_writes_trigger = 99999; + options.level0_stop_writes_trigger = 99999; + options.use_direct_io_for_flush_and_compaction = true; + options.write_buffer_size = FLAGS_memtable_size; + ROCKSDB_NAMESPACE::BlockBasedTableOptions table_options; + table_options.block_cache = + ROCKSDB_NAMESPACE::NewLRUCache(FLAGS_block_cache_size); + table_options.block_size = FLAGS_block_size; + options.table_factory.reset( + ROCKSDB_NAMESPACE::NewBlockBasedTableFactory(table_options)); + + status = ROCKSDB_NAMESPACE::DestroyDB(path, options); + assert(status.ok()); + ROCKSDB_NAMESPACE::DB* db_raw; + status = ROCKSDB_NAMESPACE::DB::Open(options, path, &db_raw); + assert(status.ok()); + std::unique_ptr db(db_raw); + + std::vector shard_states(FLAGS_shards + 1); + std::deque readers; + while (static_cast(readers.size()) < FLAGS_readers) { + readers.emplace_back(&shard_states, db_raw); + } + std::deque writers; + while (static_cast(writers.size()) < FLAGS_writers) { + writers.emplace_back(&shard_states, db_raw); + } + + // Each shard gets a random reader and random writer assigned to it + for (int i = 1; i <= FLAGS_shards; ++i) { + std::uniform_int_distribution reader_dist(0, FLAGS_readers - 1); + std::uniform_int_distribution writer_dist(0, FLAGS_writers - 1); + shard_states[i].reader = &readers[reader_dist(rng)]; + shard_states[i].writer = &writers[writer_dist(rng)]; + } + + StatsThread stats_thread(db_raw); + for (Writer& w : writers) { + w.start(); + } + + writers.clear(); + readers.clear(); +} +#endif // !defined(GFLAGS) diff --git a/librocksdb-sys/rocksdb/db/history_trimming_iterator.h b/librocksdb-sys/rocksdb/db/history_trimming_iterator.h new file mode 100644 index 0000000..4af5cde --- /dev/null +++ b/librocksdb-sys/rocksdb/db/history_trimming_iterator.h @@ -0,0 +1,95 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include +#include + +#include "db/dbformat.h" +#include "rocksdb/iterator.h" +#include "rocksdb/slice.h" +#include "table/internal_iterator.h" + +namespace ROCKSDB_NAMESPACE { + +class HistoryTrimmingIterator : public InternalIterator { + public: + explicit HistoryTrimmingIterator(InternalIterator* input, + const Comparator* cmp, const std::string& ts) + : input_(input), filter_ts_(ts), cmp_(cmp) { + assert(cmp_->timestamp_size() > 0 && !ts.empty()); + } + + bool filter() const { + if (!input_->Valid()) { + return true; + } + Slice current_ts = ExtractTimestampFromKey(key(), cmp_->timestamp_size()); + return cmp_->CompareTimestamp(current_ts, Slice(filter_ts_)) <= 0; + } + + bool Valid() const override { return input_->Valid(); } + + void SeekToFirst() override { + input_->SeekToFirst(); + while (!filter()) { + input_->Next(); + } + } + + void SeekToLast() override { + input_->SeekToLast(); + while (!filter()) { + input_->Prev(); + } + } + + void Seek(const Slice& target) override { + input_->Seek(target); + while (!filter()) { + input_->Next(); + } + } + + void SeekForPrev(const Slice& target) override { + input_->SeekForPrev(target); + while (!filter()) { + input_->Prev(); + } + } + + void Next() override { + do { + input_->Next(); + } while (!filter()); + } + + void Prev() override { + do { + input_->Prev(); + } while (!filter()); + } + + Slice key() const override { return input_->key(); } + + Slice value() const override { return input_->value(); } + + Status status() const override { return input_->status(); } + + bool IsKeyPinned() const override { return input_->IsKeyPinned(); } + + bool IsValuePinned() const override { return input_->IsValuePinned(); } + + bool IsDeleteRangeSentinelKey() const override { + return input_->IsDeleteRangeSentinelKey(); + } + + private: + InternalIterator* input_; + const std::string filter_ts_; + const Comparator* const cmp_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/import_column_family_job.cc b/librocksdb-sys/rocksdb/db/import_column_family_job.cc new file mode 100644 index 0000000..11c5fd4 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/import_column_family_job.cc @@ -0,0 +1,431 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/version_builder.h" + +#include "db/import_column_family_job.h" + +#include +#include +#include +#include + +#include "db/version_edit.h" +#include "file/file_util.h" +#include "file/random_access_file_reader.h" +#include "logging/logging.h" +#include "table/merging_iterator.h" +#include "table/scoped_arena_iterator.h" +#include "table/sst_file_writer_collectors.h" +#include "table/table_builder.h" +#include "table/unique_id_impl.h" +#include "util/stop_watch.h" + +namespace ROCKSDB_NAMESPACE { + +Status ImportColumnFamilyJob::Prepare(uint64_t next_file_number, + SuperVersion* sv) { + Status status; + std::vector cf_ingest_infos; + for (const auto& metadata_per_cf : metadatas_) { + // Read the information of files we are importing + ColumnFamilyIngestFileInfo cf_file_info; + InternalKey smallest, largest; + int num_files = 0; + std::vector files_to_import_per_cf; + for (size_t i = 0; i < metadata_per_cf.size(); i++) { + auto file_metadata = *metadata_per_cf[i]; + const auto file_path = file_metadata.db_path + "/" + file_metadata.name; + IngestedFileInfo file_to_import; + status = GetIngestedFileInfo(file_path, next_file_number++, sv, + file_metadata, &file_to_import); + if (!status.ok()) { + return status; + } + + if (file_to_import.num_entries == 0) { + status = Status::InvalidArgument("File contain no entries"); + return status; + } + + if (!file_to_import.smallest_internal_key.Valid() || + !file_to_import.largest_internal_key.Valid()) { + status = Status::Corruption("File has corrupted keys"); + return status; + } + + files_to_import_per_cf.push_back(file_to_import); + num_files++; + + // Calculate the smallest and largest keys of all files in this CF + if (i == 0) { + smallest = file_to_import.smallest_internal_key; + largest = file_to_import.largest_internal_key; + } else { + if (cfd_->internal_comparator().Compare( + smallest, file_to_import.smallest_internal_key) < 0) { + smallest = file_to_import.smallest_internal_key; + } + if (cfd_->internal_comparator().Compare( + largest, file_to_import.largest_internal_key) > 0) { + largest = file_to_import.largest_internal_key; + } + } + } + + if (num_files == 0) { + status = Status::InvalidArgument("The list of files is empty"); + return status; + } + files_to_import_.push_back(files_to_import_per_cf); + cf_file_info.smallest_internal_key = smallest; + cf_file_info.largest_internal_key = largest; + cf_ingest_infos.push_back(cf_file_info); + } + + std::sort(cf_ingest_infos.begin(), cf_ingest_infos.end(), + [this](const ColumnFamilyIngestFileInfo& info1, + const ColumnFamilyIngestFileInfo& info2) { + return cfd_->user_comparator()->Compare( + info1.smallest_internal_key.user_key(), + info2.smallest_internal_key.user_key()) < 0; + }); + + for (size_t i = 0; i + 1 < cf_ingest_infos.size(); i++) { + if (cfd_->user_comparator()->Compare( + cf_ingest_infos[i].largest_internal_key.user_key(), + cf_ingest_infos[i + 1].smallest_internal_key.user_key()) >= 0) { + status = Status::InvalidArgument("CFs have overlapping ranges"); + return status; + } + } + + // Copy/Move external files into DB + auto hardlink_files = import_options_.move_files; + + for (auto& files_to_import_per_cf : files_to_import_) { + for (auto& f : files_to_import_per_cf) { + const auto path_outside_db = f.external_file_path; + const auto path_inside_db = TableFileName( + cfd_->ioptions()->cf_paths, f.fd.GetNumber(), f.fd.GetPathId()); + + if (hardlink_files) { + status = fs_->LinkFile(path_outside_db, path_inside_db, IOOptions(), + nullptr); + if (status.IsNotSupported()) { + // Original file is on a different FS, use copy instead of hard + // linking + hardlink_files = false; + ROCKS_LOG_INFO(db_options_.info_log, + "Try to link file %s but it's not supported : %s", + f.internal_file_path.c_str(), + status.ToString().c_str()); + } + } + if (!hardlink_files) { + status = + CopyFile(fs_.get(), path_outside_db, path_inside_db, 0, + db_options_.use_fsync, io_tracer_, Temperature::kUnknown); + } + if (!status.ok()) { + break; + } + f.copy_file = !hardlink_files; + f.internal_file_path = path_inside_db; + } + if (!status.ok()) { + break; + } + } + + if (!status.ok()) { + // We failed, remove all files that we copied into the db + for (auto& files_to_import_per_cf : files_to_import_) { + for (auto& f : files_to_import_per_cf) { + if (f.internal_file_path.empty()) { + break; + } + const auto s = + fs_->DeleteFile(f.internal_file_path, IOOptions(), nullptr); + if (!s.ok()) { + ROCKS_LOG_WARN(db_options_.info_log, + "AddFile() clean up for file %s failed : %s", + f.internal_file_path.c_str(), s.ToString().c_str()); + } + } + } + } + + return status; +} + +// REQUIRES: we have become the only writer by entering both write_thread_ and +// nonmem_write_thread_ +Status ImportColumnFamilyJob::Run() { + // We use the import time as the ancester time. This is the time the data + // is written to the database. + int64_t temp_current_time = 0; + uint64_t oldest_ancester_time = kUnknownOldestAncesterTime; + uint64_t current_time = kUnknownOldestAncesterTime; + if (clock_->GetCurrentTime(&temp_current_time).ok()) { + current_time = oldest_ancester_time = + static_cast(temp_current_time); + } + + // Recover files' epoch number using dummy VersionStorageInfo + VersionBuilder dummy_version_builder( + cfd_->current()->version_set()->file_options(), cfd_->ioptions(), + cfd_->table_cache(), cfd_->current()->storage_info(), + cfd_->current()->version_set(), + cfd_->GetFileMetadataCacheReservationManager()); + VersionStorageInfo dummy_vstorage( + &cfd_->internal_comparator(), cfd_->user_comparator(), + cfd_->NumberLevels(), cfd_->ioptions()->compaction_style, + nullptr /* src_vstorage */, cfd_->ioptions()->force_consistency_checks, + EpochNumberRequirement::kMightMissing); + Status s; + + for (size_t i = 0; s.ok() && i < files_to_import_.size(); ++i) { + for (size_t j = 0; s.ok() && j < files_to_import_[i].size(); ++j) { + const auto& f = files_to_import_[i][j]; + const auto& file_metadata = *metadatas_[i][j]; + + uint64_t tail_size = 0; + bool contain_no_data_blocks = f.table_properties.num_entries > 0 && + (f.table_properties.num_entries == + f.table_properties.num_range_deletions); + if (f.table_properties.tail_start_offset > 0 || contain_no_data_blocks) { + uint64_t file_size = f.fd.GetFileSize(); + assert(f.table_properties.tail_start_offset <= file_size); + tail_size = file_size - f.table_properties.tail_start_offset; + } + + VersionEdit dummy_version_edit; + dummy_version_edit.AddFile( + file_metadata.level, f.fd.GetNumber(), f.fd.GetPathId(), + f.fd.GetFileSize(), f.smallest_internal_key, f.largest_internal_key, + file_metadata.smallest_seqno, file_metadata.largest_seqno, false, + file_metadata.temperature, kInvalidBlobFileNumber, + oldest_ancester_time, current_time, file_metadata.epoch_number, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, f.unique_id, 0, + tail_size, + static_cast( + f.table_properties.user_defined_timestamps_persisted)); + s = dummy_version_builder.Apply(&dummy_version_edit); + } + } + + if (s.ok()) { + s = dummy_version_builder.SaveTo(&dummy_vstorage); + } + if (s.ok()) { + dummy_vstorage.RecoverEpochNumbers(cfd_); + } + + // Record changes from this CF import in VersionEdit, including files with + // recovered epoch numbers + if (s.ok()) { + edit_.SetColumnFamily(cfd_->GetID()); + + for (int level = 0; level < dummy_vstorage.num_levels(); level++) { + for (FileMetaData* file_meta : dummy_vstorage.LevelFiles(level)) { + edit_.AddFile(level, *file_meta); + // If incoming sequence number is higher, update local sequence number. + if (file_meta->fd.largest_seqno > versions_->LastSequence()) { + versions_->SetLastAllocatedSequence(file_meta->fd.largest_seqno); + versions_->SetLastPublishedSequence(file_meta->fd.largest_seqno); + versions_->SetLastSequence(file_meta->fd.largest_seqno); + } + } + } + } + + // Release resources occupied by the dummy VersionStorageInfo + for (int level = 0; level < dummy_vstorage.num_levels(); level++) { + for (FileMetaData* file_meta : dummy_vstorage.LevelFiles(level)) { + file_meta->refs--; + if (file_meta->refs <= 0) { + delete file_meta; + } + } + } + return s; +} + +void ImportColumnFamilyJob::Cleanup(const Status& status) { + if (!status.ok()) { + // We failed to add files to the database remove all the files we copied. + for (auto& files_to_import_per_cf : files_to_import_) { + for (auto& f : files_to_import_per_cf) { + const auto s = + fs_->DeleteFile(f.internal_file_path, IOOptions(), nullptr); + if (!s.ok()) { + ROCKS_LOG_WARN(db_options_.info_log, + "AddFile() clean up for file %s failed : %s", + f.internal_file_path.c_str(), s.ToString().c_str()); + } + } + } + } else if (status.ok() && import_options_.move_files) { + // The files were moved and added successfully, remove original file links + for (auto& files_to_import_per_cf : files_to_import_) { + for (auto& f : files_to_import_per_cf) { + const auto s = + fs_->DeleteFile(f.external_file_path, IOOptions(), nullptr); + if (!s.ok()) { + ROCKS_LOG_WARN( + db_options_.info_log, + "%s was added to DB successfully but failed to remove original " + "file link : %s", + f.external_file_path.c_str(), s.ToString().c_str()); + } + } + } + } +} + +Status ImportColumnFamilyJob::GetIngestedFileInfo( + const std::string& external_file, uint64_t new_file_number, + SuperVersion* sv, const LiveFileMetaData& file_meta, + IngestedFileInfo* file_to_import) { + file_to_import->external_file_path = external_file; + Status status; + if (file_meta.size > 0) { + file_to_import->file_size = file_meta.size; + } else { + // Get external file size + status = fs_->GetFileSize(external_file, IOOptions(), + &file_to_import->file_size, nullptr); + if (!status.ok()) { + return status; + } + } + // Assign FD with number + file_to_import->fd = + FileDescriptor(new_file_number, 0, file_to_import->file_size); + + // Create TableReader for external file + std::unique_ptr table_reader; + std::unique_ptr sst_file; + std::unique_ptr sst_file_reader; + + status = + fs_->NewRandomAccessFile(external_file, env_options_, &sst_file, nullptr); + if (!status.ok()) { + return status; + } + sst_file_reader.reset(new RandomAccessFileReader( + std::move(sst_file), external_file, nullptr /*Env*/, io_tracer_)); + + // TODO(yuzhangyu): User-defined timestamps doesn't support importing column + // family. Pass in the correct `user_defined_timestamps_persisted` flag for + // creating `TableReaderOptions` when the support is there. + status = cfd_->ioptions()->table_factory->NewTableReader( + TableReaderOptions( + *cfd_->ioptions(), sv->mutable_cf_options.prefix_extractor, + env_options_, cfd_->internal_comparator(), + sv->mutable_cf_options.block_protection_bytes_per_key, + /*skip_filters*/ false, /*immortal*/ false, + /*force_direct_prefetch*/ false, /*level*/ -1, + /*block_cache_tracer*/ nullptr, + /*max_file_size_for_l0_meta_pin*/ 0, versions_->DbSessionId(), + /*cur_file_num*/ new_file_number), + std::move(sst_file_reader), file_to_import->file_size, &table_reader); + if (!status.ok()) { + return status; + } + + // Get the external file properties + auto props = table_reader->GetTableProperties(); + + // Set original_seqno to 0. + file_to_import->original_seqno = 0; + + // Get number of entries in table + file_to_import->num_entries = props->num_entries; + + // If the importing files were exported with Checkpoint::ExportColumnFamily(), + // we cannot simply recompute smallest and largest used to truncate range + // tombstones from file content, and we expect smallest and largest populated + // in file_meta. + if (file_meta.smallest.empty()) { + assert(file_meta.largest.empty()); + // TODO: plumb Env::IOActivity + ReadOptions ro; + std::unique_ptr iter(table_reader->NewIterator( + ro, sv->mutable_cf_options.prefix_extractor.get(), /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kExternalSSTIngestion)); + + // Get first (smallest) key from file + iter->SeekToFirst(); + bool bound_set = false; + if (iter->Valid()) { + file_to_import->smallest_internal_key.DecodeFrom(iter->key()); + iter->SeekToLast(); + file_to_import->largest_internal_key.DecodeFrom(iter->key()); + bound_set = true; + } + + std::unique_ptr range_del_iter{ + table_reader->NewRangeTombstoneIterator(ro)}; + if (range_del_iter != nullptr) { + range_del_iter->SeekToFirst(); + if (range_del_iter->Valid()) { + ParsedInternalKey key; + Status pik_status = ParseInternalKey(range_del_iter->key(), &key, + db_options_.allow_data_in_errors); + if (!pik_status.ok()) { + return Status::Corruption("Corrupted key in external file. ", + pik_status.getState()); + } + RangeTombstone first_tombstone(key, range_del_iter->value()); + InternalKey start_key = first_tombstone.SerializeKey(); + const InternalKeyComparator* icmp = &cfd_->internal_comparator(); + if (!bound_set || + icmp->Compare(start_key, file_to_import->smallest_internal_key) < + 0) { + file_to_import->smallest_internal_key = start_key; + } + + range_del_iter->SeekToLast(); + pik_status = ParseInternalKey(range_del_iter->key(), &key, + db_options_.allow_data_in_errors); + if (!pik_status.ok()) { + return Status::Corruption("Corrupted key in external file. ", + pik_status.getState()); + } + RangeTombstone last_tombstone(key, range_del_iter->value()); + InternalKey end_key = last_tombstone.SerializeEndKey(); + if (!bound_set || + icmp->Compare(end_key, file_to_import->largest_internal_key) > 0) { + file_to_import->largest_internal_key = end_key; + } + bound_set = true; + } + } + assert(bound_set); + } else { + assert(!file_meta.largest.empty()); + file_to_import->smallest_internal_key.DecodeFrom(file_meta.smallest); + file_to_import->largest_internal_key.DecodeFrom(file_meta.largest); + } + + file_to_import->cf_id = static_cast(props->column_family_id); + + file_to_import->table_properties = *props; + + auto s = GetSstInternalUniqueId(props->db_id, props->db_session_id, + props->orig_file_number, + &(file_to_import->unique_id)); + if (!s.ok()) { + ROCKS_LOG_WARN(db_options_.info_log, + "Failed to get SST unique id for file %s", + file_to_import->internal_file_path.c_str()); + } + + return status; +} +} // namespace ROCKSDB_NAMESPACE \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/db/import_column_family_job.h b/librocksdb-sys/rocksdb/db/import_column_family_job.h new file mode 100644 index 0000000..fb41c4b --- /dev/null +++ b/librocksdb-sys/rocksdb/db/import_column_family_job.h @@ -0,0 +1,91 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once +#include +#include +#include + +#include "db/column_family.h" +#include "db/external_sst_file_ingestion_job.h" +#include "db/snapshot_impl.h" +#include "options/db_options.h" +#include "rocksdb/db.h" +#include "rocksdb/metadata.h" +#include "rocksdb/sst_file_writer.h" +#include "util/autovector.h" + +namespace ROCKSDB_NAMESPACE { +struct EnvOptions; +class SystemClock; + +// Imports a set of sst files as is into a new column family. Logic is similar +// to ExternalSstFileIngestionJob. +class ImportColumnFamilyJob { + // All file information of an imported CF, mainly used to + // calculate whether there is overlap between CFs + struct ColumnFamilyIngestFileInfo { + // Smallest internal key in cf + InternalKey smallest_internal_key; + // Largest internal key in cf + InternalKey largest_internal_key; + }; + + public: + ImportColumnFamilyJob( + VersionSet* versions, ColumnFamilyData* cfd, + const ImmutableDBOptions& db_options, const EnvOptions& env_options, + const ImportColumnFamilyOptions& import_options, + const std::vector>& metadatas, + const std::shared_ptr& io_tracer) + : clock_(db_options.clock), + versions_(versions), + cfd_(cfd), + db_options_(db_options), + fs_(db_options_.fs, io_tracer), + env_options_(env_options), + import_options_(import_options), + metadatas_(metadatas), + io_tracer_(io_tracer) {} + + // Prepare the job by copying external files into the DB. + Status Prepare(uint64_t next_file_number, SuperVersion* sv); + + // Will execute the import job and prepare edit() to be applied. + // REQUIRES: Mutex held + Status Run(); + + // Cleanup after successful/failed job + void Cleanup(const Status& status); + + VersionEdit* edit() { return &edit_; } + + const std::vector>& files_to_import() const { + return files_to_import_; + } + + private: + // Open the external file and populate `file_to_import` with all the + // external information we need to import this file. + Status GetIngestedFileInfo(const std::string& external_file, + uint64_t new_file_number, SuperVersion* sv, + const LiveFileMetaData& file_meta, + IngestedFileInfo* file_to_import); + + SystemClock* clock_; + VersionSet* versions_; + ColumnFamilyData* cfd_; + const ImmutableDBOptions& db_options_; + const FileSystemPtr fs_; + const EnvOptions& env_options_; + std::vector> files_to_import_; + VersionEdit edit_; + const ImportColumnFamilyOptions& import_options_; + const std::vector> metadatas_; + const std::shared_ptr io_tracer_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/import_column_family_test.cc b/librocksdb-sys/rocksdb/db/import_column_family_test.cc new file mode 100644 index 0000000..f6c1a02 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/import_column_family_test.cc @@ -0,0 +1,891 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + + +#include + +#include "db/db_test_util.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/sst_file_writer.h" +#include "test_util/testutil.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +class ImportColumnFamilyTest : public DBTestBase { + public: + ImportColumnFamilyTest() + : DBTestBase("import_column_family_test", /*env_do_fsync=*/true) { + sst_files_dir_ = dbname_ + "/sst_files/"; + export_files_dir_ = test::PerThreadDBPath(env_, "export"); + export_files_dir2_ = test::PerThreadDBPath(env_, "export2"); + + DestroyAndRecreateExternalSSTFilesDir(); + import_cfh_ = nullptr; + import_cfh2_ = nullptr; + metadata_ptr_ = nullptr; + metadata_ptr2_ = nullptr; + } + + ~ImportColumnFamilyTest() { + if (import_cfh_) { + EXPECT_OK(db_->DropColumnFamily(import_cfh_)); + EXPECT_OK(db_->DestroyColumnFamilyHandle(import_cfh_)); + import_cfh_ = nullptr; + } + if (import_cfh2_) { + EXPECT_OK(db_->DropColumnFamily(import_cfh2_)); + EXPECT_OK(db_->DestroyColumnFamilyHandle(import_cfh2_)); + import_cfh2_ = nullptr; + } + if (metadata_ptr_) { + delete metadata_ptr_; + metadata_ptr_ = nullptr; + } + + if (metadata_ptr2_) { + delete metadata_ptr2_; + metadata_ptr2_ = nullptr; + } + EXPECT_OK(DestroyDir(env_, sst_files_dir_)); + EXPECT_OK(DestroyDir(env_, export_files_dir_)); + EXPECT_OK(DestroyDir(env_, export_files_dir2_)); + } + + void DestroyAndRecreateExternalSSTFilesDir() { + EXPECT_OK(DestroyDir(env_, sst_files_dir_)); + EXPECT_OK(env_->CreateDir(sst_files_dir_)); + EXPECT_OK(DestroyDir(env_, export_files_dir_)); + EXPECT_OK(DestroyDir(env_, export_files_dir2_)); + } + + LiveFileMetaData LiveFileMetaDataInit(std::string name, std::string path, + int level, + SequenceNumber smallest_seqno, + SequenceNumber largest_seqno) { + LiveFileMetaData metadata; + metadata.name = name; + metadata.db_path = path; + metadata.smallest_seqno = smallest_seqno; + metadata.largest_seqno = largest_seqno; + metadata.level = level; + return metadata; + } + + protected: + std::string sst_files_dir_; + std::string export_files_dir_; + std::string export_files_dir2_; + ColumnFamilyHandle* import_cfh_; + ColumnFamilyHandle* import_cfh2_; + ExportImportFilesMetaData* metadata_ptr_; + ExportImportFilesMetaData* metadata_ptr2_; +}; + +TEST_F(ImportColumnFamilyTest, ImportSSTFileWriterFiles) { + Options options = CurrentOptions(); + CreateAndReopenWithCF({"koko"}, options); + + SstFileWriter sfw_cf1(EnvOptions(), options, handles_[1]); + SstFileWriter sfw_unknown(EnvOptions(), options); + + // cf1.sst + const std::string cf1_sst_name = "cf1.sst"; + const std::string cf1_sst = sst_files_dir_ + cf1_sst_name; + ASSERT_OK(sfw_cf1.Open(cf1_sst)); + ASSERT_OK(sfw_cf1.Put("K1", "V1")); + ASSERT_OK(sfw_cf1.Put("K2", "V2")); + ASSERT_OK(sfw_cf1.Finish()); + + // cf_unknown.sst + const std::string unknown_sst_name = "cf_unknown.sst"; + const std::string unknown_sst = sst_files_dir_ + unknown_sst_name; + ASSERT_OK(sfw_unknown.Open(unknown_sst)); + ASSERT_OK(sfw_unknown.Put("K3", "V1")); + ASSERT_OK(sfw_unknown.Put("K4", "V2")); + ASSERT_OK(sfw_unknown.Finish()); + + { + // Import sst file corresponding to cf1 onto a new cf and verify + ExportImportFilesMetaData metadata; + metadata.files.push_back( + LiveFileMetaDataInit(cf1_sst_name, sst_files_dir_, 0, 10, 19)); + metadata.db_comparator_name = options.comparator->Name(); + + ASSERT_OK(db_->CreateColumnFamilyWithImport( + options, "toto", ImportColumnFamilyOptions(), metadata, &import_cfh_)); + ASSERT_NE(import_cfh_, nullptr); + + std::string value; + ASSERT_OK(db_->Get(ReadOptions(), import_cfh_, "K1", &value)); + ASSERT_EQ(value, "V1"); + ASSERT_OK(db_->Get(ReadOptions(), import_cfh_, "K2", &value)); + ASSERT_EQ(value, "V2"); + ASSERT_OK(db_->DropColumnFamily(import_cfh_)); + ASSERT_OK(db_->DestroyColumnFamilyHandle(import_cfh_)); + import_cfh_ = nullptr; + } + + { + // Import sst file corresponding to unknown cf onto a new cf and verify + ExportImportFilesMetaData metadata; + metadata.files.push_back( + LiveFileMetaDataInit(unknown_sst_name, sst_files_dir_, 0, 20, 29)); + metadata.db_comparator_name = options.comparator->Name(); + + ASSERT_OK(db_->CreateColumnFamilyWithImport( + options, "yoyo", ImportColumnFamilyOptions(), metadata, &import_cfh_)); + ASSERT_NE(import_cfh_, nullptr); + + std::string value; + ASSERT_OK(db_->Get(ReadOptions(), import_cfh_, "K3", &value)); + ASSERT_EQ(value, "V1"); + ASSERT_OK(db_->Get(ReadOptions(), import_cfh_, "K4", &value)); + ASSERT_EQ(value, "V2"); + } + EXPECT_OK(db_->DestroyColumnFamilyHandle(import_cfh_)); + import_cfh_ = nullptr; + + // verify sst unique id during reopen + options.verify_sst_unique_id_in_manifest = true; + ReopenWithColumnFamilies({"default", "koko", "yoyo"}, options); +} + +TEST_F(ImportColumnFamilyTest, ImportSSTFileWriterFilesWithOverlap) { + Options options = CurrentOptions(); + CreateAndReopenWithCF({"koko"}, options); + + SstFileWriter sfw_cf1(EnvOptions(), options, handles_[1]); + + // file3.sst + const std::string file3_sst_name = "file3.sst"; + const std::string file3_sst = sst_files_dir_ + file3_sst_name; + ASSERT_OK(sfw_cf1.Open(file3_sst)); + for (int i = 0; i < 100; ++i) { + ASSERT_OK(sfw_cf1.Put(Key(i), Key(i) + "_val")); + } + ASSERT_OK(sfw_cf1.Finish()); + + // file2.sst + const std::string file2_sst_name = "file2.sst"; + const std::string file2_sst = sst_files_dir_ + file2_sst_name; + ASSERT_OK(sfw_cf1.Open(file2_sst)); + for (int i = 0; i < 100; i += 2) { + ASSERT_OK(sfw_cf1.Put(Key(i), Key(i) + "_overwrite1")); + } + ASSERT_OK(sfw_cf1.Finish()); + + // file1a.sst + const std::string file1a_sst_name = "file1a.sst"; + const std::string file1a_sst = sst_files_dir_ + file1a_sst_name; + ASSERT_OK(sfw_cf1.Open(file1a_sst)); + for (int i = 0; i < 52; i += 4) { + ASSERT_OK(sfw_cf1.Put(Key(i), Key(i) + "_overwrite2")); + } + ASSERT_OK(sfw_cf1.Finish()); + + // file1b.sst + const std::string file1b_sst_name = "file1b.sst"; + const std::string file1b_sst = sst_files_dir_ + file1b_sst_name; + ASSERT_OK(sfw_cf1.Open(file1b_sst)); + for (int i = 52; i < 100; i += 4) { + ASSERT_OK(sfw_cf1.Put(Key(i), Key(i) + "_overwrite2")); + } + ASSERT_OK(sfw_cf1.Finish()); + + // file0a.sst + const std::string file0a_sst_name = "file0a.sst"; + const std::string file0a_sst = sst_files_dir_ + file0a_sst_name; + ASSERT_OK(sfw_cf1.Open(file0a_sst)); + for (int i = 0; i < 100; i += 16) { + ASSERT_OK(sfw_cf1.Put(Key(i), Key(i) + "_overwrite3")); + } + ASSERT_OK(sfw_cf1.Finish()); + + // file0b.sst + const std::string file0b_sst_name = "file0b.sst"; + const std::string file0b_sst = sst_files_dir_ + file0b_sst_name; + ASSERT_OK(sfw_cf1.Open(file0b_sst)); + for (int i = 0; i < 100; i += 16) { + ASSERT_OK(sfw_cf1.Put(Key(i), Key(i) + "_overwrite4")); + } + ASSERT_OK(sfw_cf1.Finish()); + + // Import sst files and verify + ExportImportFilesMetaData metadata; + metadata.files.push_back( + LiveFileMetaDataInit(file3_sst_name, sst_files_dir_, 3, 10, 19)); + metadata.files.push_back( + LiveFileMetaDataInit(file2_sst_name, sst_files_dir_, 2, 20, 29)); + metadata.files.push_back( + LiveFileMetaDataInit(file1a_sst_name, sst_files_dir_, 1, 30, 34)); + metadata.files.push_back( + LiveFileMetaDataInit(file1b_sst_name, sst_files_dir_, 1, 35, 39)); + metadata.files.push_back( + LiveFileMetaDataInit(file0a_sst_name, sst_files_dir_, 0, 40, 49)); + metadata.files.push_back( + LiveFileMetaDataInit(file0b_sst_name, sst_files_dir_, 0, 50, 59)); + metadata.db_comparator_name = options.comparator->Name(); + + ASSERT_OK(db_->CreateColumnFamilyWithImport( + options, "toto", ImportColumnFamilyOptions(), metadata, &import_cfh_)); + ASSERT_NE(import_cfh_, nullptr); + + for (int i = 0; i < 100; i++) { + std::string value; + ASSERT_OK(db_->Get(ReadOptions(), import_cfh_, Key(i), &value)); + if (i % 16 == 0) { + ASSERT_EQ(value, Key(i) + "_overwrite4"); + } else if (i % 4 == 0) { + ASSERT_EQ(value, Key(i) + "_overwrite2"); + } else if (i % 2 == 0) { + ASSERT_EQ(value, Key(i) + "_overwrite1"); + } else { + ASSERT_EQ(value, Key(i) + "_val"); + } + } + + for (int i = 0; i < 100; i += 5) { + ASSERT_OK( + db_->Put(WriteOptions(), import_cfh_, Key(i), Key(i) + "_overwrite5")); + } + + // Flush and check again + ASSERT_OK(db_->Flush(FlushOptions(), import_cfh_)); + for (int i = 0; i < 100; i++) { + std::string value; + ASSERT_OK(db_->Get(ReadOptions(), import_cfh_, Key(i), &value)); + if (i % 5 == 0) { + ASSERT_EQ(value, Key(i) + "_overwrite5"); + } else if (i % 16 == 0) { + ASSERT_EQ(value, Key(i) + "_overwrite4"); + } else if (i % 4 == 0) { + ASSERT_EQ(value, Key(i) + "_overwrite2"); + } else if (i % 2 == 0) { + ASSERT_EQ(value, Key(i) + "_overwrite1"); + } else { + ASSERT_EQ(value, Key(i) + "_val"); + } + } + + // Compact and check again. + ASSERT_OK( + db_->CompactRange(CompactRangeOptions(), import_cfh_, nullptr, nullptr)); + for (int i = 0; i < 100; i++) { + std::string value; + ASSERT_OK(db_->Get(ReadOptions(), import_cfh_, Key(i), &value)); + if (i % 5 == 0) { + ASSERT_EQ(value, Key(i) + "_overwrite5"); + } else if (i % 16 == 0) { + ASSERT_EQ(value, Key(i) + "_overwrite4"); + } else if (i % 4 == 0) { + ASSERT_EQ(value, Key(i) + "_overwrite2"); + } else if (i % 2 == 0) { + ASSERT_EQ(value, Key(i) + "_overwrite1"); + } else { + ASSERT_EQ(value, Key(i) + "_val"); + } + } +} + +TEST_F(ImportColumnFamilyTest, ImportSSTFileWriterFilesWithRangeTombstone) { + // Test for a bug where import file's smallest and largest key did not + // consider range tombstone. + Options options = CurrentOptions(); + CreateAndReopenWithCF({"koko"}, options); + + SstFileWriter sfw_cf1(EnvOptions(), options, handles_[1]); + // cf1.sst + const std::string cf1_sst_name = "cf1.sst"; + const std::string cf1_sst = sst_files_dir_ + cf1_sst_name; + ASSERT_OK(sfw_cf1.Open(cf1_sst)); + ASSERT_OK(sfw_cf1.Put("K1", "V1")); + ASSERT_OK(sfw_cf1.Put("K2", "V2")); + ASSERT_OK(sfw_cf1.DeleteRange("K3", "K4")); + ASSERT_OK(sfw_cf1.DeleteRange("K7", "K9")); + + ASSERT_OK(sfw_cf1.Finish()); + + // Import sst file corresponding to cf1 onto a new cf and verify + ExportImportFilesMetaData metadata; + metadata.files.push_back( + LiveFileMetaDataInit(cf1_sst_name, sst_files_dir_, 0, 0, 19)); + metadata.db_comparator_name = options.comparator->Name(); + + ASSERT_OK(db_->CreateColumnFamilyWithImport( + options, "toto", ImportColumnFamilyOptions(), metadata, &import_cfh_)); + ASSERT_NE(import_cfh_, nullptr); + + ColumnFamilyMetaData import_cf_meta; + db_->GetColumnFamilyMetaData(import_cfh_, &import_cf_meta); + ASSERT_EQ(import_cf_meta.file_count, 1); + const SstFileMetaData* file_meta = nullptr; + for (const auto& level_meta : import_cf_meta.levels) { + if (!level_meta.files.empty()) { + file_meta = &(level_meta.files[0]); + break; + } + } + ASSERT_TRUE(file_meta != nullptr); + InternalKey largest; + largest.DecodeFrom(file_meta->largest); + ASSERT_EQ(largest.user_key(), "K9"); + + std::string value; + ASSERT_OK(db_->Get(ReadOptions(), import_cfh_, "K1", &value)); + ASSERT_EQ(value, "V1"); + ASSERT_OK(db_->Get(ReadOptions(), import_cfh_, "K2", &value)); + ASSERT_EQ(value, "V2"); + ASSERT_OK(db_->DropColumnFamily(import_cfh_)); + ASSERT_OK(db_->DestroyColumnFamilyHandle(import_cfh_)); + import_cfh_ = nullptr; +} + +TEST_F(ImportColumnFamilyTest, ImportExportedSSTFromAnotherCF) { + Options options = CurrentOptions(); + CreateAndReopenWithCF({"koko"}, options); + + for (int i = 0; i < 100; ++i) { + ASSERT_OK(Put(1, Key(i), Key(i) + "_val")); + } + ASSERT_OK(Flush(1)); + + ASSERT_OK( + db_->CompactRange(CompactRangeOptions(), handles_[1], nullptr, nullptr)); + + // Overwrite the value in the same set of keys. + for (int i = 0; i < 100; ++i) { + ASSERT_OK(Put(1, Key(i), Key(i) + "_overwrite")); + } + + // Flush to create L0 file. + ASSERT_OK(Flush(1)); + for (int i = 0; i < 100; ++i) { + ASSERT_OK(Put(1, Key(i), Key(i) + "_overwrite2")); + } + + // Flush again to create another L0 file. It should have higher sequencer. + ASSERT_OK(Flush(1)); + + Checkpoint* checkpoint; + ASSERT_OK(Checkpoint::Create(db_, &checkpoint)); + ASSERT_OK(checkpoint->ExportColumnFamily(handles_[1], export_files_dir_, + &metadata_ptr_)); + ASSERT_NE(metadata_ptr_, nullptr); + delete checkpoint; + + ImportColumnFamilyOptions import_options; + import_options.move_files = false; + ASSERT_OK(db_->CreateColumnFamilyWithImport(options, "toto", import_options, + *metadata_ptr_, &import_cfh_)); + ASSERT_NE(import_cfh_, nullptr); + + import_options.move_files = true; + ASSERT_OK(db_->CreateColumnFamilyWithImport(options, "yoyo", import_options, + *metadata_ptr_, &import_cfh2_)); + ASSERT_NE(import_cfh2_, nullptr); + delete metadata_ptr_; + metadata_ptr_ = NULL; + + std::string value1, value2; + + for (int i = 0; i < 100; ++i) { + ASSERT_OK(db_->Get(ReadOptions(), import_cfh_, Key(i), &value1)); + ASSERT_EQ(Get(1, Key(i)), value1); + } + + for (int i = 0; i < 100; ++i) { + ASSERT_OK(db_->Get(ReadOptions(), import_cfh2_, Key(i), &value2)); + ASSERT_EQ(Get(1, Key(i)), value2); + } + + // Modify keys in cf1 and verify. + for (int i = 0; i < 25; i++) { + ASSERT_OK(db_->Delete(WriteOptions(), import_cfh_, Key(i))); + } + for (int i = 25; i < 50; i++) { + ASSERT_OK( + db_->Put(WriteOptions(), import_cfh_, Key(i), Key(i) + "_overwrite3")); + } + for (int i = 0; i < 25; ++i) { + ASSERT_TRUE( + db_->Get(ReadOptions(), import_cfh_, Key(i), &value1).IsNotFound()); + } + for (int i = 25; i < 50; ++i) { + ASSERT_OK(db_->Get(ReadOptions(), import_cfh_, Key(i), &value1)); + ASSERT_EQ(Key(i) + "_overwrite3", value1); + } + for (int i = 50; i < 100; ++i) { + ASSERT_OK(db_->Get(ReadOptions(), import_cfh_, Key(i), &value1)); + ASSERT_EQ(Key(i) + "_overwrite2", value1); + } + + for (int i = 0; i < 100; ++i) { + ASSERT_OK(db_->Get(ReadOptions(), import_cfh2_, Key(i), &value2)); + ASSERT_EQ(Get(1, Key(i)), value2); + } + + // Compact and check again. + ASSERT_OK(db_->Flush(FlushOptions(), import_cfh_)); + ASSERT_OK( + db_->CompactRange(CompactRangeOptions(), import_cfh_, nullptr, nullptr)); + + for (int i = 0; i < 25; ++i) { + ASSERT_TRUE( + db_->Get(ReadOptions(), import_cfh_, Key(i), &value1).IsNotFound()); + } + for (int i = 25; i < 50; ++i) { + ASSERT_OK(db_->Get(ReadOptions(), import_cfh_, Key(i), &value1)); + ASSERT_EQ(Key(i) + "_overwrite3", value1); + } + for (int i = 50; i < 100; ++i) { + ASSERT_OK(db_->Get(ReadOptions(), import_cfh_, Key(i), &value1)); + ASSERT_EQ(Key(i) + "_overwrite2", value1); + } + + for (int i = 0; i < 100; ++i) { + ASSERT_OK(db_->Get(ReadOptions(), import_cfh2_, Key(i), &value2)); + ASSERT_EQ(Get(1, Key(i)), value2); + } +} + +TEST_F(ImportColumnFamilyTest, ImportExportedSSTFromAnotherDB) { + Options options = CurrentOptions(); + CreateAndReopenWithCF({"koko"}, options); + + for (int i = 0; i < 100; ++i) { + ASSERT_OK(Put(1, Key(i), Key(i) + "_val")); + } + ASSERT_OK(Flush(1)); + + // Compact to create a L1 file. + ASSERT_OK( + db_->CompactRange(CompactRangeOptions(), handles_[1], nullptr, nullptr)); + + // Overwrite the value in the same set of keys. + for (int i = 0; i < 50; ++i) { + ASSERT_OK(Put(1, Key(i), Key(i) + "_overwrite")); + } + + // Flush to create L0 file. + ASSERT_OK(Flush(1)); + + for (int i = 0; i < 25; ++i) { + ASSERT_OK(Put(1, Key(i), Key(i) + "_overwrite2")); + } + + // Flush again to create another L0 file. It should have higher sequencer. + ASSERT_OK(Flush(1)); + + Checkpoint* checkpoint; + ASSERT_OK(Checkpoint::Create(db_, &checkpoint)); + ASSERT_OK(checkpoint->ExportColumnFamily(handles_[1], export_files_dir_, + &metadata_ptr_)); + ASSERT_NE(metadata_ptr_, nullptr); + delete checkpoint; + + // Create a new db and import the files. + DB* db_copy; + ASSERT_OK(DestroyDir(env_, dbname_ + "/db_copy")); + ASSERT_OK(DB::Open(options, dbname_ + "/db_copy", &db_copy)); + ColumnFamilyHandle* cfh = nullptr; + ASSERT_OK(db_copy->CreateColumnFamilyWithImport(ColumnFamilyOptions(), "yoyo", + ImportColumnFamilyOptions(), + *metadata_ptr_, &cfh)); + ASSERT_NE(cfh, nullptr); + + for (int i = 0; i < 100; ++i) { + std::string value; + ASSERT_OK(db_copy->Get(ReadOptions(), cfh, Key(i), &value)); + ASSERT_EQ(Get(1, Key(i)), value); + } + ASSERT_OK(db_copy->DropColumnFamily(cfh)); + ASSERT_OK(db_copy->DestroyColumnFamilyHandle(cfh)); + delete db_copy; + ASSERT_OK(DestroyDir(env_, dbname_ + "/db_copy")); +} + +TEST_F(ImportColumnFamilyTest, + ImportExportedSSTFromAnotherCFWithRangeTombstone) { + // Test for a bug where import file's smallest and largest key did not + // consider range tombstone. + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + CreateAndReopenWithCF({"koko"}, options); + + for (int i = 10; i < 20; ++i) { + ASSERT_OK(Put(1, Key(i), Key(i) + "_val")); + } + ASSERT_OK(Flush(1 /* cf */)); + MoveFilesToLevel(1 /* level */, 1 /* cf */); + const Snapshot* snapshot = db_->GetSnapshot(); + ASSERT_OK(db_->DeleteRange(WriteOptions(), handles_[1], Key(0), Key(25))); + ASSERT_OK(Put(1, Key(1), "t")); + ASSERT_OK(Flush(1)); + // Tests importing a range tombstone only file + ASSERT_OK(db_->DeleteRange(WriteOptions(), handles_[1], Key(0), Key(2))); + + Checkpoint* checkpoint; + ASSERT_OK(Checkpoint::Create(db_, &checkpoint)); + ASSERT_OK(checkpoint->ExportColumnFamily(handles_[1], export_files_dir_, + &metadata_ptr_)); + ASSERT_NE(metadata_ptr_, nullptr); + delete checkpoint; + + ImportColumnFamilyOptions import_options; + import_options.move_files = false; + ASSERT_OK(db_->CreateColumnFamilyWithImport(options, "toto", import_options, + *metadata_ptr_, &import_cfh_)); + ASSERT_NE(import_cfh_, nullptr); + + import_options.move_files = true; + ASSERT_OK(db_->CreateColumnFamilyWithImport(options, "yoyo", import_options, + *metadata_ptr_, &import_cfh2_)); + ASSERT_NE(import_cfh2_, nullptr); + delete metadata_ptr_; + metadata_ptr_ = nullptr; + + std::string value1, value2; + ReadOptions ro_latest; + ReadOptions ro_snapshot; + ro_snapshot.snapshot = snapshot; + + for (int i = 10; i < 20; ++i) { + ASSERT_TRUE(db_->Get(ro_latest, import_cfh_, Key(i), &value1).IsNotFound()); + ASSERT_OK(db_->Get(ro_snapshot, import_cfh_, Key(i), &value1)); + ASSERT_EQ(Get(1, Key(i), snapshot), value1); + } + ASSERT_TRUE(db_->Get(ro_latest, import_cfh_, Key(1), &value1).IsNotFound()); + + for (int i = 10; i < 20; ++i) { + ASSERT_TRUE( + db_->Get(ro_latest, import_cfh2_, Key(i), &value1).IsNotFound()); + + ASSERT_OK(db_->Get(ro_snapshot, import_cfh2_, Key(i), &value2)); + ASSERT_EQ(Get(1, Key(i), snapshot), value2); + } + ASSERT_TRUE(db_->Get(ro_latest, import_cfh2_, Key(1), &value1).IsNotFound()); + + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(ImportColumnFamilyTest, LevelFilesOverlappingAtEndpoints) { + // Imports a column family containing a level where two files overlap at their + // endpoints. "Overlap" means the largest user key in one file is the same as + // the smallest user key in the second file. + const int kFileBytes = 128 << 10; // 128KB + const int kValueBytes = 1 << 10; // 1KB + const int kNumFiles = 4; + + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.num_levels = 2; + CreateAndReopenWithCF({"koko"}, options); + + Random rnd(301); + // Every key is snapshot protected to ensure older versions will not be + // dropped during compaction. + std::vector snapshots; + snapshots.reserve(kFileBytes / kValueBytes * kNumFiles); + for (int i = 0; i < kNumFiles; ++i) { + for (int j = 0; j < kFileBytes / kValueBytes; ++j) { + auto value = rnd.RandomString(kValueBytes); + ASSERT_OK(Put(1, "key", value)); + snapshots.push_back(db_->GetSnapshot()); + } + ASSERT_OK(Flush(1)); + } + + // Compact to create overlapping L1 files. + ASSERT_OK( + db_->CompactRange(CompactRangeOptions(), handles_[1], nullptr, nullptr)); + ASSERT_GT(NumTableFilesAtLevel(1, 1), 1); + + Checkpoint* checkpoint; + ASSERT_OK(Checkpoint::Create(db_, &checkpoint)); + ASSERT_OK(checkpoint->ExportColumnFamily(handles_[1], export_files_dir_, + &metadata_ptr_)); + ASSERT_NE(metadata_ptr_, nullptr); + delete checkpoint; + + // Create a new db and import the files. + DB* db_copy; + ASSERT_OK(DestroyDir(env_, dbname_ + "/db_copy")); + ASSERT_OK(DB::Open(options, dbname_ + "/db_copy", &db_copy)); + ColumnFamilyHandle* cfh = nullptr; + ASSERT_OK(db_copy->CreateColumnFamilyWithImport(ColumnFamilyOptions(), "yoyo", + ImportColumnFamilyOptions(), + *metadata_ptr_, &cfh)); + ASSERT_NE(cfh, nullptr); + + { + std::string value; + ASSERT_OK(db_copy->Get(ReadOptions(), cfh, "key", &value)); + } + ASSERT_OK(db_copy->DropColumnFamily(cfh)); + ASSERT_OK(db_copy->DestroyColumnFamilyHandle(cfh)); + delete db_copy; + ASSERT_OK(DestroyDir(env_, dbname_ + "/db_copy")); + for (const Snapshot* snapshot : snapshots) { + db_->ReleaseSnapshot(snapshot); + } +} + +TEST_F(ImportColumnFamilyTest, ImportColumnFamilyNegativeTest) { + Options options = CurrentOptions(); + CreateAndReopenWithCF({"koko"}, options); + + { + // Create column family with existing cf name. + ExportImportFilesMetaData metadata; + metadata.db_comparator_name = options.comparator->Name(); + Status s = db_->CreateColumnFamilyWithImport(ColumnFamilyOptions(), "koko", + ImportColumnFamilyOptions(), + metadata, &import_cfh_); + ASSERT_TRUE(std::strstr(s.getState(), "Column family already exists")); + ASSERT_EQ(import_cfh_, nullptr); + } + + { + // Import with no files specified. + ExportImportFilesMetaData metadata; + metadata.db_comparator_name = options.comparator->Name(); + Status s = db_->CreateColumnFamilyWithImport(ColumnFamilyOptions(), "yoyo", + ImportColumnFamilyOptions(), + metadata, &import_cfh_); + ASSERT_TRUE(std::strstr(s.getState(), "The list of files is empty")); + ASSERT_EQ(import_cfh_, nullptr); + } + + { + // Import with overlapping keys in sst files. + ExportImportFilesMetaData metadata; + SstFileWriter sfw_cf1(EnvOptions(), options, handles_[1]); + const std::string file1_sst_name = "file1.sst"; + const std::string file1_sst = sst_files_dir_ + file1_sst_name; + ASSERT_OK(sfw_cf1.Open(file1_sst)); + ASSERT_OK(sfw_cf1.Put("K1", "V1")); + ASSERT_OK(sfw_cf1.Put("K2", "V2")); + ASSERT_OK(sfw_cf1.Finish()); + const std::string file2_sst_name = "file2.sst"; + const std::string file2_sst = sst_files_dir_ + file2_sst_name; + ASSERT_OK(sfw_cf1.Open(file2_sst)); + ASSERT_OK(sfw_cf1.Put("K2", "V2")); + ASSERT_OK(sfw_cf1.Put("K3", "V3")); + ASSERT_OK(sfw_cf1.Finish()); + + metadata.files.push_back( + LiveFileMetaDataInit(file1_sst_name, sst_files_dir_, 1, 10, 19)); + metadata.files.push_back( + LiveFileMetaDataInit(file2_sst_name, sst_files_dir_, 1, 10, 19)); + metadata.db_comparator_name = options.comparator->Name(); + + ASSERT_NOK(db_->CreateColumnFamilyWithImport(ColumnFamilyOptions(), "yoyo", + ImportColumnFamilyOptions(), + metadata, &import_cfh_)); + ASSERT_EQ(import_cfh_, nullptr); + } + + { + // Import with a mismatching comparator, should fail with appropriate error. + ExportImportFilesMetaData metadata; + Options mismatch_options = CurrentOptions(); + mismatch_options.comparator = ReverseBytewiseComparator(); + SstFileWriter sfw_cf1(EnvOptions(), mismatch_options, handles_[1]); + const std::string file1_sst_name = "file1.sst"; + const std::string file1_sst = sst_files_dir_ + file1_sst_name; + ASSERT_OK(sfw_cf1.Open(file1_sst)); + ASSERT_OK(sfw_cf1.Put("K2", "V2")); + ASSERT_OK(sfw_cf1.Put("K1", "V1")); + ASSERT_OK(sfw_cf1.Finish()); + + metadata.files.push_back( + LiveFileMetaDataInit(file1_sst_name, sst_files_dir_, 1, 10, 19)); + metadata.db_comparator_name = mismatch_options.comparator->Name(); + + Status s = db_->CreateColumnFamilyWithImport(ColumnFamilyOptions(), "coco", + ImportColumnFamilyOptions(), + metadata, &import_cfh_); + ASSERT_TRUE(std::strstr(s.getState(), "Comparator name mismatch")); + ASSERT_EQ(import_cfh_, nullptr); + } + + { + // Import with non existent sst file should fail with appropriate error + ExportImportFilesMetaData metadata; + SstFileWriter sfw_cf1(EnvOptions(), options, handles_[1]); + const std::string file1_sst_name = "file1.sst"; + const std::string file1_sst = sst_files_dir_ + file1_sst_name; + ASSERT_OK(sfw_cf1.Open(file1_sst)); + ASSERT_OK(sfw_cf1.Put("K1", "V1")); + ASSERT_OK(sfw_cf1.Put("K2", "V2")); + ASSERT_OK(sfw_cf1.Finish()); + const std::string file3_sst_name = "file3.sst"; + + metadata.files.push_back( + LiveFileMetaDataInit(file1_sst_name, sst_files_dir_, 1, 10, 19)); + metadata.files.push_back( + LiveFileMetaDataInit(file3_sst_name, sst_files_dir_, 1, 10, 19)); + metadata.db_comparator_name = options.comparator->Name(); + + Status s = db_->CreateColumnFamilyWithImport(ColumnFamilyOptions(), "yoyo", + ImportColumnFamilyOptions(), + metadata, &import_cfh_); + ASSERT_TRUE(std::strstr(s.getState(), "No such file or directory")); + ASSERT_EQ(import_cfh_, nullptr); + + // Test successful import after a failure with the same CF name. Ensures + // there is no side effect with CF when there is a failed import + metadata.files.pop_back(); + metadata.db_comparator_name = options.comparator->Name(); + + ASSERT_OK(db_->CreateColumnFamilyWithImport(ColumnFamilyOptions(), "yoyo", + ImportColumnFamilyOptions(), + metadata, &import_cfh_)); + ASSERT_NE(import_cfh_, nullptr); + } +} + +TEST_F(ImportColumnFamilyTest, ImportMultiColumnFamilyTest) { + Options options = CurrentOptions(); + CreateAndReopenWithCF({"koko"}, options); + + for (int i = 0; i < 100; ++i) { + ASSERT_OK(Put(1, Key(i), Key(i) + "_val")); + } + ASSERT_OK(Flush(1)); + + ASSERT_OK( + db_->CompactRange(CompactRangeOptions(), handles_[1], nullptr, nullptr)); + + // Overwrite the value in the same set of keys. + for (int i = 0; i < 100; ++i) { + ASSERT_OK(Put(1, Key(i), Key(i) + "_overwrite")); + } + + // Flush again to create another L0 file. It should have higher sequencer. + ASSERT_OK(Flush(1)); + + Checkpoint* checkpoint1; + Checkpoint* checkpoint2; + ASSERT_OK(Checkpoint::Create(db_, &checkpoint1)); + ASSERT_OK(checkpoint1->ExportColumnFamily(handles_[1], export_files_dir_, + &metadata_ptr_)); + + // Create a new db and import the files. + DB* db_copy; + ASSERT_OK(DestroyDir(env_, dbname_ + "/db_copy")); + ASSERT_OK(DB::Open(options, dbname_ + "/db_copy", &db_copy)); + ColumnFamilyHandle* copy_cfh = nullptr; + ASSERT_OK(db_copy->CreateColumnFamily(options, "koko", ©_cfh)); + WriteOptions wo; + for (int i = 100; i < 200; ++i) { + ASSERT_OK(db_copy->Put(wo, copy_cfh, Key(i), Key(i) + "_val")); + } + ASSERT_OK(db_copy->Flush(FlushOptions())); + for (int i = 100; i < 200; ++i) { + ASSERT_OK(db_copy->Put(wo, copy_cfh, Key(i), Key(i) + "_overwrite")); + } + ASSERT_OK(db_copy->Flush(FlushOptions())); + for (int i = 100; i < 200; ++i) { + ASSERT_OK(db_copy->Put(wo, copy_cfh, Key(i), Key(i) + "_overwrite2")); + } + ASSERT_OK(db_copy->Flush(FlushOptions())); + + // Flush again to create another L0 file. It should have higher sequencer. + ASSERT_OK(Checkpoint::Create(db_copy, &checkpoint2)); + ASSERT_OK(checkpoint2->ExportColumnFamily(copy_cfh, export_files_dir2_, + &metadata_ptr2_)); + + ASSERT_NE(metadata_ptr_, nullptr); + ASSERT_NE(metadata_ptr2_, nullptr); + delete checkpoint1; + delete checkpoint2; + ImportColumnFamilyOptions import_options; + import_options.move_files = false; + + std::vector metadatas = {metadata_ptr_, + metadata_ptr2_}; + ASSERT_OK(db_->CreateColumnFamilyWithImport(options, "toto", import_options, + metadatas, &import_cfh_)); + + std::string value1, value2; + for (int i = 0; i < 100; ++i) { + ASSERT_OK(db_->Get(ReadOptions(), import_cfh_, Key(i), &value1)); + ASSERT_EQ(Get(1, Key(i)), value1); + } + + for (int i = 100; i < 200; ++i) { + ASSERT_OK(db_->Get(ReadOptions(), import_cfh_, Key(i), &value1)); + ASSERT_OK(db_copy->Get(ReadOptions(), copy_cfh, Key(i), &value2)); + ASSERT_EQ(value1, value2); + } + + ASSERT_OK(db_copy->DropColumnFamily(copy_cfh)); + ASSERT_OK(db_copy->DestroyColumnFamilyHandle(copy_cfh)); + delete db_copy; + ASSERT_OK(DestroyDir(env_, dbname_ + "/db_copy")); +} + +TEST_F(ImportColumnFamilyTest, ImportMultiColumnFamilyWithOverlap) { + Options options = CurrentOptions(); + CreateAndReopenWithCF({"koko"}, options); + + for (int i = 0; i < 100; ++i) { + ASSERT_OK(Put(1, Key(i), Key(i) + "_val")); + } + + Checkpoint* checkpoint1; + Checkpoint* checkpoint2; + ASSERT_OK(Checkpoint::Create(db_, &checkpoint1)); + ASSERT_OK(checkpoint1->ExportColumnFamily(handles_[1], export_files_dir_, + &metadata_ptr_)); + + // Create a new db and import the files. + DB* db_copy; + ASSERT_OK(DestroyDir(env_, dbname_ + "/db_copy")); + ASSERT_OK(DB::Open(options, dbname_ + "/db_copy", &db_copy)); + ColumnFamilyHandle* copy_cfh = nullptr; + ASSERT_OK(db_copy->CreateColumnFamily(options, "koko", ©_cfh)); + WriteOptions wo; + for (int i = 50; i < 150; ++i) { + ASSERT_OK(db_copy->Put(wo, copy_cfh, Key(i), Key(i) + "_val")); + } + ASSERT_OK(db_copy->Flush(FlushOptions())); + + // Flush again to create another L0 file. It should have higher sequencer. + ASSERT_OK(Checkpoint::Create(db_copy, &checkpoint2)); + ASSERT_OK(checkpoint2->ExportColumnFamily(copy_cfh, export_files_dir2_, + &metadata_ptr2_)); + + ASSERT_NE(metadata_ptr_, nullptr); + ASSERT_NE(metadata_ptr2_, nullptr); + delete checkpoint1; + delete checkpoint2; + ImportColumnFamilyOptions import_options; + import_options.move_files = false; + + std::vector metadatas = {metadata_ptr_, + metadata_ptr2_}; + + ASSERT_EQ(db_->CreateColumnFamilyWithImport(options, "toto", import_options, + metadatas, &import_cfh_), + Status::InvalidArgument("CFs have overlapping ranges")); + + ASSERT_OK(db_copy->DropColumnFamily(copy_cfh)); + ASSERT_OK(db_copy->DestroyColumnFamilyHandle(copy_cfh)); + delete db_copy; + ASSERT_OK(DestroyDir(env_, dbname_ + "/db_copy")); +} +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/librocksdb-sys/rocksdb/db/internal_stats.cc b/librocksdb-sys/rocksdb/db/internal_stats.cc new file mode 100644 index 0000000..6ef4b43 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/internal_stats.cc @@ -0,0 +1,2133 @@ +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/internal_stats.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cache/cache_entry_roles.h" +#include "cache/cache_entry_stats.h" +#include "db/column_family.h" +#include "db/db_impl/db_impl.h" +#include "db/write_stall_stats.h" +#include "port/port.h" +#include "rocksdb/system_clock.h" +#include "rocksdb/table.h" +#include "table/block_based/cachable_entry.h" +#include "util/hash_containers.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + + +const std::map InternalStats::compaction_level_stats = + { + {LevelStatType::NUM_FILES, LevelStat{"NumFiles", "Files"}}, + {LevelStatType::COMPACTED_FILES, + LevelStat{"CompactedFiles", "CompactedFiles"}}, + {LevelStatType::SIZE_BYTES, LevelStat{"SizeBytes", "Size"}}, + {LevelStatType::SCORE, LevelStat{"Score", "Score"}}, + {LevelStatType::READ_GB, LevelStat{"ReadGB", "Read(GB)"}}, + {LevelStatType::RN_GB, LevelStat{"RnGB", "Rn(GB)"}}, + {LevelStatType::RNP1_GB, LevelStat{"Rnp1GB", "Rnp1(GB)"}}, + {LevelStatType::WRITE_GB, LevelStat{"WriteGB", "Write(GB)"}}, + {LevelStatType::W_NEW_GB, LevelStat{"WnewGB", "Wnew(GB)"}}, + {LevelStatType::MOVED_GB, LevelStat{"MovedGB", "Moved(GB)"}}, + {LevelStatType::WRITE_AMP, LevelStat{"WriteAmp", "W-Amp"}}, + {LevelStatType::READ_MBPS, LevelStat{"ReadMBps", "Rd(MB/s)"}}, + {LevelStatType::WRITE_MBPS, LevelStat{"WriteMBps", "Wr(MB/s)"}}, + {LevelStatType::COMP_SEC, LevelStat{"CompSec", "Comp(sec)"}}, + {LevelStatType::COMP_CPU_SEC, + LevelStat{"CompMergeCPU", "CompMergeCPU(sec)"}}, + {LevelStatType::COMP_COUNT, LevelStat{"CompCount", "Comp(cnt)"}}, + {LevelStatType::AVG_SEC, LevelStat{"AvgSec", "Avg(sec)"}}, + {LevelStatType::KEY_IN, LevelStat{"KeyIn", "KeyIn"}}, + {LevelStatType::KEY_DROP, LevelStat{"KeyDrop", "KeyDrop"}}, + {LevelStatType::R_BLOB_GB, LevelStat{"RblobGB", "Rblob(GB)"}}, + {LevelStatType::W_BLOB_GB, LevelStat{"WblobGB", "Wblob(GB)"}}, +}; + +const std::map + InternalStats::db_stats_type_to_info = { + {InternalStats::kIntStatsWalFileBytes, + DBStatInfo{"db.wal_bytes_written"}}, + {InternalStats::kIntStatsWalFileSynced, DBStatInfo{"db.wal_syncs"}}, + {InternalStats::kIntStatsBytesWritten, + DBStatInfo{"db.user_bytes_written"}}, + {InternalStats::kIntStatsNumKeysWritten, + DBStatInfo{"db.user_keys_written"}}, + {InternalStats::kIntStatsWriteDoneByOther, + DBStatInfo{"db.user_writes_by_other"}}, + {InternalStats::kIntStatsWriteDoneBySelf, + DBStatInfo{"db.user_writes_by_self"}}, + {InternalStats::kIntStatsWriteWithWal, + DBStatInfo{"db.user_writes_with_wal"}}, + {InternalStats::kIntStatsWriteStallMicros, + DBStatInfo{"db.user_write_stall_micros"}}, + {InternalStats::kIntStatsWriteBufferManagerLimitStopsCounts, + DBStatInfo{WriteStallStatsMapKeys::CauseConditionCount( + WriteStallCause::kWriteBufferManagerLimit, + WriteStallCondition::kStopped)}}, +}; + +namespace { +const double kMB = 1048576.0; +const double kGB = kMB * 1024; +const double kMicrosInSec = 1000000.0; + +void PrintLevelStatsHeader(char* buf, size_t len, const std::string& cf_name, + const std::string& group_by) { + int written_size = + snprintf(buf, len, "\n** Compaction Stats [%s] **\n", cf_name.c_str()); + written_size = std::min(written_size, static_cast(len)); + auto hdr = [](LevelStatType t) { + return InternalStats::compaction_level_stats.at(t).header_name.c_str(); + }; + int line_size = snprintf( + buf + written_size, len - written_size, + "%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s " + "%s\n", + // Note that we skip COMPACTED_FILES and merge it with Files column + group_by.c_str(), hdr(LevelStatType::NUM_FILES), + hdr(LevelStatType::SIZE_BYTES), hdr(LevelStatType::SCORE), + hdr(LevelStatType::READ_GB), hdr(LevelStatType::RN_GB), + hdr(LevelStatType::RNP1_GB), hdr(LevelStatType::WRITE_GB), + hdr(LevelStatType::W_NEW_GB), hdr(LevelStatType::MOVED_GB), + hdr(LevelStatType::WRITE_AMP), hdr(LevelStatType::READ_MBPS), + hdr(LevelStatType::WRITE_MBPS), hdr(LevelStatType::COMP_SEC), + hdr(LevelStatType::COMP_CPU_SEC), hdr(LevelStatType::COMP_COUNT), + hdr(LevelStatType::AVG_SEC), hdr(LevelStatType::KEY_IN), + hdr(LevelStatType::KEY_DROP), hdr(LevelStatType::R_BLOB_GB), + hdr(LevelStatType::W_BLOB_GB)); + + written_size += line_size; + written_size = std::min(written_size, static_cast(len)); + snprintf(buf + written_size, len - written_size, "%s\n", + std::string(line_size, '-').c_str()); +} + +void PrepareLevelStats(std::map* level_stats, + int num_files, int being_compacted, + double total_file_size, double score, double w_amp, + const InternalStats::CompactionStats& stats) { + const uint64_t bytes_read = stats.bytes_read_non_output_levels + + stats.bytes_read_output_level + + stats.bytes_read_blob; + const uint64_t bytes_written = stats.bytes_written + stats.bytes_written_blob; + const int64_t bytes_new = stats.bytes_written - stats.bytes_read_output_level; + const double elapsed = (stats.micros + 1) / kMicrosInSec; + + (*level_stats)[LevelStatType::NUM_FILES] = num_files; + (*level_stats)[LevelStatType::COMPACTED_FILES] = being_compacted; + (*level_stats)[LevelStatType::SIZE_BYTES] = total_file_size; + (*level_stats)[LevelStatType::SCORE] = score; + (*level_stats)[LevelStatType::READ_GB] = bytes_read / kGB; + (*level_stats)[LevelStatType::RN_GB] = + stats.bytes_read_non_output_levels / kGB; + (*level_stats)[LevelStatType::RNP1_GB] = stats.bytes_read_output_level / kGB; + (*level_stats)[LevelStatType::WRITE_GB] = stats.bytes_written / kGB; + (*level_stats)[LevelStatType::W_NEW_GB] = bytes_new / kGB; + (*level_stats)[LevelStatType::MOVED_GB] = stats.bytes_moved / kGB; + (*level_stats)[LevelStatType::WRITE_AMP] = w_amp; + (*level_stats)[LevelStatType::READ_MBPS] = bytes_read / kMB / elapsed; + (*level_stats)[LevelStatType::WRITE_MBPS] = bytes_written / kMB / elapsed; + (*level_stats)[LevelStatType::COMP_SEC] = stats.micros / kMicrosInSec; + (*level_stats)[LevelStatType::COMP_CPU_SEC] = stats.cpu_micros / kMicrosInSec; + (*level_stats)[LevelStatType::COMP_COUNT] = stats.count; + (*level_stats)[LevelStatType::AVG_SEC] = + stats.count == 0 ? 0 : stats.micros / kMicrosInSec / stats.count; + (*level_stats)[LevelStatType::KEY_IN] = + static_cast(stats.num_input_records); + (*level_stats)[LevelStatType::KEY_DROP] = + static_cast(stats.num_dropped_records); + (*level_stats)[LevelStatType::R_BLOB_GB] = stats.bytes_read_blob / kGB; + (*level_stats)[LevelStatType::W_BLOB_GB] = stats.bytes_written_blob / kGB; +} + +void PrintLevelStats(char* buf, size_t len, const std::string& name, + const std::map& stat_value) { + snprintf( + buf, len, + "%4s " /* Level */ + "%6d/%-3d " /* Files */ + "%8s " /* Size */ + "%5.1f " /* Score */ + "%8.1f " /* Read(GB) */ + "%7.1f " /* Rn(GB) */ + "%8.1f " /* Rnp1(GB) */ + "%9.1f " /* Write(GB) */ + "%8.1f " /* Wnew(GB) */ + "%9.1f " /* Moved(GB) */ + "%5.1f " /* W-Amp */ + "%8.1f " /* Rd(MB/s) */ + "%8.1f " /* Wr(MB/s) */ + "%9.2f " /* Comp(sec) */ + "%17.2f " /* CompMergeCPU(sec) */ + "%9d " /* Comp(cnt) */ + "%8.3f " /* Avg(sec) */ + "%7s " /* KeyIn */ + "%6s " /* KeyDrop */ + "%9.1f " /* Rblob(GB) */ + "%9.1f\n", /* Wblob(GB) */ + name.c_str(), static_cast(stat_value.at(LevelStatType::NUM_FILES)), + static_cast(stat_value.at(LevelStatType::COMPACTED_FILES)), + BytesToHumanString( + static_cast(stat_value.at(LevelStatType::SIZE_BYTES))) + .c_str(), + stat_value.at(LevelStatType::SCORE), + stat_value.at(LevelStatType::READ_GB), + stat_value.at(LevelStatType::RN_GB), + stat_value.at(LevelStatType::RNP1_GB), + stat_value.at(LevelStatType::WRITE_GB), + stat_value.at(LevelStatType::W_NEW_GB), + stat_value.at(LevelStatType::MOVED_GB), + stat_value.at(LevelStatType::WRITE_AMP), + stat_value.at(LevelStatType::READ_MBPS), + stat_value.at(LevelStatType::WRITE_MBPS), + stat_value.at(LevelStatType::COMP_SEC), + stat_value.at(LevelStatType::COMP_CPU_SEC), + static_cast(stat_value.at(LevelStatType::COMP_COUNT)), + stat_value.at(LevelStatType::AVG_SEC), + NumberToHumanString( + static_cast(stat_value.at(LevelStatType::KEY_IN))) + .c_str(), + NumberToHumanString( + static_cast(stat_value.at(LevelStatType::KEY_DROP))) + .c_str(), + stat_value.at(LevelStatType::R_BLOB_GB), + stat_value.at(LevelStatType::W_BLOB_GB)); +} + +void PrintLevelStats(char* buf, size_t len, const std::string& name, + int num_files, int being_compacted, double total_file_size, + double score, double w_amp, + const InternalStats::CompactionStats& stats) { + std::map level_stats; + PrepareLevelStats(&level_stats, num_files, being_compacted, total_file_size, + score, w_amp, stats); + PrintLevelStats(buf, len, name, level_stats); +} + +// Assumes that trailing numbers represent an optional argument. This requires +// property names to not end with numbers. +std::pair GetPropertyNameAndArg(const Slice& property) { + Slice name = property, arg = property; + size_t sfx_len = 0; + while (sfx_len < property.size() && + isdigit(property[property.size() - sfx_len - 1])) { + ++sfx_len; + } + name.remove_suffix(sfx_len); + arg.remove_prefix(property.size() - sfx_len); + return {name, arg}; +} +} // anonymous namespace + +static const std::string rocksdb_prefix = "rocksdb."; + +static const std::string num_files_at_level_prefix = "num-files-at-level"; +static const std::string compression_ratio_at_level_prefix = + "compression-ratio-at-level"; +static const std::string allstats = "stats"; +static const std::string sstables = "sstables"; +static const std::string cfstats = "cfstats"; +static const std::string cfstats_no_file_histogram = + "cfstats-no-file-histogram"; +static const std::string cf_file_histogram = "cf-file-histogram"; +static const std::string cf_write_stall_stats = "cf-write-stall-stats"; +static const std::string dbstats = "dbstats"; +static const std::string db_write_stall_stats = "db-write-stall-stats"; +static const std::string levelstats = "levelstats"; +static const std::string block_cache_entry_stats = "block-cache-entry-stats"; +static const std::string fast_block_cache_entry_stats = + "fast-block-cache-entry-stats"; +static const std::string num_immutable_mem_table = "num-immutable-mem-table"; +static const std::string num_immutable_mem_table_flushed = + "num-immutable-mem-table-flushed"; +static const std::string mem_table_flush_pending = "mem-table-flush-pending"; +static const std::string compaction_pending = "compaction-pending"; +static const std::string background_errors = "background-errors"; +static const std::string cur_size_active_mem_table = + "cur-size-active-mem-table"; +static const std::string cur_size_all_mem_tables = "cur-size-all-mem-tables"; +static const std::string size_all_mem_tables = "size-all-mem-tables"; +static const std::string num_entries_active_mem_table = + "num-entries-active-mem-table"; +static const std::string num_entries_imm_mem_tables = + "num-entries-imm-mem-tables"; +static const std::string num_deletes_active_mem_table = + "num-deletes-active-mem-table"; +static const std::string num_deletes_imm_mem_tables = + "num-deletes-imm-mem-tables"; +static const std::string estimate_num_keys = "estimate-num-keys"; +static const std::string estimate_table_readers_mem = + "estimate-table-readers-mem"; +static const std::string is_file_deletions_enabled = + "is-file-deletions-enabled"; +static const std::string num_snapshots = "num-snapshots"; +static const std::string oldest_snapshot_time = "oldest-snapshot-time"; +static const std::string oldest_snapshot_sequence = "oldest-snapshot-sequence"; +static const std::string num_live_versions = "num-live-versions"; +static const std::string current_version_number = + "current-super-version-number"; +static const std::string estimate_live_data_size = "estimate-live-data-size"; +static const std::string min_log_number_to_keep_str = "min-log-number-to-keep"; +static const std::string min_obsolete_sst_number_to_keep_str = + "min-obsolete-sst-number-to-keep"; +static const std::string base_level_str = "base-level"; +static const std::string total_sst_files_size = "total-sst-files-size"; +static const std::string live_sst_files_size = "live-sst-files-size"; +static const std::string obsolete_sst_files_size = "obsolete-sst-files-size"; +static const std::string live_sst_files_size_at_temperature = + "live-sst-files-size-at-temperature"; +static const std::string estimate_pending_comp_bytes = + "estimate-pending-compaction-bytes"; +static const std::string aggregated_table_properties = + "aggregated-table-properties"; +static const std::string aggregated_table_properties_at_level = + aggregated_table_properties + "-at-level"; +static const std::string num_running_compactions = "num-running-compactions"; +static const std::string num_running_flushes = "num-running-flushes"; +static const std::string actual_delayed_write_rate = + "actual-delayed-write-rate"; +static const std::string is_write_stopped = "is-write-stopped"; +static const std::string estimate_oldest_key_time = "estimate-oldest-key-time"; +static const std::string block_cache_capacity = "block-cache-capacity"; +static const std::string block_cache_usage = "block-cache-usage"; +static const std::string block_cache_pinned_usage = "block-cache-pinned-usage"; +static const std::string options_statistics = "options-statistics"; +static const std::string num_blob_files = "num-blob-files"; +static const std::string blob_stats = "blob-stats"; +static const std::string total_blob_file_size = "total-blob-file-size"; +static const std::string live_blob_file_size = "live-blob-file-size"; +static const std::string live_blob_file_garbage_size = + "live-blob-file-garbage-size"; +static const std::string blob_cache_capacity = "blob-cache-capacity"; +static const std::string blob_cache_usage = "blob-cache-usage"; +static const std::string blob_cache_pinned_usage = "blob-cache-pinned-usage"; + +const std::string DB::Properties::kNumFilesAtLevelPrefix = + rocksdb_prefix + num_files_at_level_prefix; +const std::string DB::Properties::kCompressionRatioAtLevelPrefix = + rocksdb_prefix + compression_ratio_at_level_prefix; +const std::string DB::Properties::kStats = rocksdb_prefix + allstats; +const std::string DB::Properties::kSSTables = rocksdb_prefix + sstables; +const std::string DB::Properties::kCFStats = rocksdb_prefix + cfstats; +const std::string DB::Properties::kCFStatsNoFileHistogram = + rocksdb_prefix + cfstats_no_file_histogram; +const std::string DB::Properties::kCFFileHistogram = + rocksdb_prefix + cf_file_histogram; +const std::string DB::Properties::kCFWriteStallStats = + rocksdb_prefix + cf_write_stall_stats; +const std::string DB::Properties::kDBWriteStallStats = + rocksdb_prefix + db_write_stall_stats; +const std::string DB::Properties::kDBStats = rocksdb_prefix + dbstats; +const std::string DB::Properties::kLevelStats = rocksdb_prefix + levelstats; +const std::string DB::Properties::kBlockCacheEntryStats = + rocksdb_prefix + block_cache_entry_stats; +const std::string DB::Properties::kFastBlockCacheEntryStats = + rocksdb_prefix + fast_block_cache_entry_stats; +const std::string DB::Properties::kNumImmutableMemTable = + rocksdb_prefix + num_immutable_mem_table; +const std::string DB::Properties::kNumImmutableMemTableFlushed = + rocksdb_prefix + num_immutable_mem_table_flushed; +const std::string DB::Properties::kMemTableFlushPending = + rocksdb_prefix + mem_table_flush_pending; +const std::string DB::Properties::kCompactionPending = + rocksdb_prefix + compaction_pending; +const std::string DB::Properties::kNumRunningCompactions = + rocksdb_prefix + num_running_compactions; +const std::string DB::Properties::kNumRunningFlushes = + rocksdb_prefix + num_running_flushes; +const std::string DB::Properties::kBackgroundErrors = + rocksdb_prefix + background_errors; +const std::string DB::Properties::kCurSizeActiveMemTable = + rocksdb_prefix + cur_size_active_mem_table; +const std::string DB::Properties::kCurSizeAllMemTables = + rocksdb_prefix + cur_size_all_mem_tables; +const std::string DB::Properties::kSizeAllMemTables = + rocksdb_prefix + size_all_mem_tables; +const std::string DB::Properties::kNumEntriesActiveMemTable = + rocksdb_prefix + num_entries_active_mem_table; +const std::string DB::Properties::kNumEntriesImmMemTables = + rocksdb_prefix + num_entries_imm_mem_tables; +const std::string DB::Properties::kNumDeletesActiveMemTable = + rocksdb_prefix + num_deletes_active_mem_table; +const std::string DB::Properties::kNumDeletesImmMemTables = + rocksdb_prefix + num_deletes_imm_mem_tables; +const std::string DB::Properties::kEstimateNumKeys = + rocksdb_prefix + estimate_num_keys; +const std::string DB::Properties::kEstimateTableReadersMem = + rocksdb_prefix + estimate_table_readers_mem; +const std::string DB::Properties::kIsFileDeletionsEnabled = + rocksdb_prefix + is_file_deletions_enabled; +const std::string DB::Properties::kNumSnapshots = + rocksdb_prefix + num_snapshots; +const std::string DB::Properties::kOldestSnapshotTime = + rocksdb_prefix + oldest_snapshot_time; +const std::string DB::Properties::kOldestSnapshotSequence = + rocksdb_prefix + oldest_snapshot_sequence; +const std::string DB::Properties::kNumLiveVersions = + rocksdb_prefix + num_live_versions; +const std::string DB::Properties::kCurrentSuperVersionNumber = + rocksdb_prefix + current_version_number; +const std::string DB::Properties::kEstimateLiveDataSize = + rocksdb_prefix + estimate_live_data_size; +const std::string DB::Properties::kMinLogNumberToKeep = + rocksdb_prefix + min_log_number_to_keep_str; +const std::string DB::Properties::kMinObsoleteSstNumberToKeep = + rocksdb_prefix + min_obsolete_sst_number_to_keep_str; +const std::string DB::Properties::kTotalSstFilesSize = + rocksdb_prefix + total_sst_files_size; +const std::string DB::Properties::kLiveSstFilesSize = + rocksdb_prefix + live_sst_files_size; +const std::string DB::Properties::kObsoleteSstFilesSize = + rocksdb_prefix + obsolete_sst_files_size; +const std::string DB::Properties::kBaseLevel = rocksdb_prefix + base_level_str; +const std::string DB::Properties::kEstimatePendingCompactionBytes = + rocksdb_prefix + estimate_pending_comp_bytes; +const std::string DB::Properties::kAggregatedTableProperties = + rocksdb_prefix + aggregated_table_properties; +const std::string DB::Properties::kAggregatedTablePropertiesAtLevel = + rocksdb_prefix + aggregated_table_properties_at_level; +const std::string DB::Properties::kActualDelayedWriteRate = + rocksdb_prefix + actual_delayed_write_rate; +const std::string DB::Properties::kIsWriteStopped = + rocksdb_prefix + is_write_stopped; +const std::string DB::Properties::kEstimateOldestKeyTime = + rocksdb_prefix + estimate_oldest_key_time; +const std::string DB::Properties::kBlockCacheCapacity = + rocksdb_prefix + block_cache_capacity; +const std::string DB::Properties::kBlockCacheUsage = + rocksdb_prefix + block_cache_usage; +const std::string DB::Properties::kBlockCachePinnedUsage = + rocksdb_prefix + block_cache_pinned_usage; +const std::string DB::Properties::kOptionsStatistics = + rocksdb_prefix + options_statistics; +const std::string DB::Properties::kLiveSstFilesSizeAtTemperature = + rocksdb_prefix + live_sst_files_size_at_temperature; +const std::string DB::Properties::kNumBlobFiles = + rocksdb_prefix + num_blob_files; +const std::string DB::Properties::kBlobStats = rocksdb_prefix + blob_stats; +const std::string DB::Properties::kTotalBlobFileSize = + rocksdb_prefix + total_blob_file_size; +const std::string DB::Properties::kLiveBlobFileSize = + rocksdb_prefix + live_blob_file_size; +const std::string DB::Properties::kLiveBlobFileGarbageSize = + rocksdb_prefix + live_blob_file_garbage_size; +const std::string DB::Properties::kBlobCacheCapacity = + rocksdb_prefix + blob_cache_capacity; +const std::string DB::Properties::kBlobCacheUsage = + rocksdb_prefix + blob_cache_usage; +const std::string DB::Properties::kBlobCachePinnedUsage = + rocksdb_prefix + blob_cache_pinned_usage; + +const std::string InternalStats::kPeriodicCFStats = + DB::Properties::kCFStats + ".periodic"; +const int InternalStats::kMaxNoChangePeriodSinceDump = 8; + +const UnorderedMap + InternalStats::ppt_name_to_info = { + {DB::Properties::kNumFilesAtLevelPrefix, + {false, &InternalStats::HandleNumFilesAtLevel, nullptr, nullptr, + nullptr}}, + {DB::Properties::kCompressionRatioAtLevelPrefix, + {false, &InternalStats::HandleCompressionRatioAtLevelPrefix, nullptr, + nullptr, nullptr}}, + {DB::Properties::kLevelStats, + {false, &InternalStats::HandleLevelStats, nullptr, nullptr, nullptr}}, + {DB::Properties::kStats, + {false, &InternalStats::HandleStats, nullptr, nullptr, nullptr}}, + {DB::Properties::kCFStats, + {false, &InternalStats::HandleCFStats, nullptr, + &InternalStats::HandleCFMapStats, nullptr}}, + {InternalStats::kPeriodicCFStats, + {false, &InternalStats::HandleCFStatsPeriodic, nullptr, nullptr, + nullptr}}, + {DB::Properties::kCFStatsNoFileHistogram, + {false, &InternalStats::HandleCFStatsNoFileHistogram, nullptr, nullptr, + nullptr}}, + {DB::Properties::kCFFileHistogram, + {false, &InternalStats::HandleCFFileHistogram, nullptr, nullptr, + nullptr}}, + {DB::Properties::kCFWriteStallStats, + {false, &InternalStats::HandleCFWriteStallStats, nullptr, + &InternalStats::HandleCFWriteStallStatsMap, nullptr}}, + {DB::Properties::kDBStats, + {false, &InternalStats::HandleDBStats, nullptr, + &InternalStats::HandleDBMapStats, nullptr}}, + {DB::Properties::kDBWriteStallStats, + {false, &InternalStats::HandleDBWriteStallStats, nullptr, + &InternalStats::HandleDBWriteStallStatsMap, nullptr}}, + {DB::Properties::kBlockCacheEntryStats, + {true, &InternalStats::HandleBlockCacheEntryStats, nullptr, + &InternalStats::HandleBlockCacheEntryStatsMap, nullptr}}, + {DB::Properties::kFastBlockCacheEntryStats, + {true, &InternalStats::HandleFastBlockCacheEntryStats, nullptr, + &InternalStats::HandleFastBlockCacheEntryStatsMap, nullptr}}, + {DB::Properties::kSSTables, + {false, &InternalStats::HandleSsTables, nullptr, nullptr, nullptr}}, + {DB::Properties::kAggregatedTableProperties, + {false, &InternalStats::HandleAggregatedTableProperties, nullptr, + &InternalStats::HandleAggregatedTablePropertiesMap, nullptr}}, + {DB::Properties::kAggregatedTablePropertiesAtLevel, + {false, &InternalStats::HandleAggregatedTablePropertiesAtLevel, + nullptr, &InternalStats::HandleAggregatedTablePropertiesAtLevelMap, + nullptr}}, + {DB::Properties::kNumImmutableMemTable, + {false, nullptr, &InternalStats::HandleNumImmutableMemTable, nullptr, + nullptr}}, + {DB::Properties::kNumImmutableMemTableFlushed, + {false, nullptr, &InternalStats::HandleNumImmutableMemTableFlushed, + nullptr, nullptr}}, + {DB::Properties::kMemTableFlushPending, + {false, nullptr, &InternalStats::HandleMemTableFlushPending, nullptr, + nullptr}}, + {DB::Properties::kCompactionPending, + {false, nullptr, &InternalStats::HandleCompactionPending, nullptr, + nullptr}}, + {DB::Properties::kBackgroundErrors, + {false, nullptr, &InternalStats::HandleBackgroundErrors, nullptr, + nullptr}}, + {DB::Properties::kCurSizeActiveMemTable, + {false, nullptr, &InternalStats::HandleCurSizeActiveMemTable, nullptr, + nullptr}}, + {DB::Properties::kCurSizeAllMemTables, + {false, nullptr, &InternalStats::HandleCurSizeAllMemTables, nullptr, + nullptr}}, + {DB::Properties::kSizeAllMemTables, + {false, nullptr, &InternalStats::HandleSizeAllMemTables, nullptr, + nullptr}}, + {DB::Properties::kNumEntriesActiveMemTable, + {false, nullptr, &InternalStats::HandleNumEntriesActiveMemTable, + nullptr, nullptr}}, + {DB::Properties::kNumEntriesImmMemTables, + {false, nullptr, &InternalStats::HandleNumEntriesImmMemTables, nullptr, + nullptr}}, + {DB::Properties::kNumDeletesActiveMemTable, + {false, nullptr, &InternalStats::HandleNumDeletesActiveMemTable, + nullptr, nullptr}}, + {DB::Properties::kNumDeletesImmMemTables, + {false, nullptr, &InternalStats::HandleNumDeletesImmMemTables, nullptr, + nullptr}}, + {DB::Properties::kEstimateNumKeys, + {false, nullptr, &InternalStats::HandleEstimateNumKeys, nullptr, + nullptr}}, + {DB::Properties::kEstimateTableReadersMem, + {true, nullptr, &InternalStats::HandleEstimateTableReadersMem, nullptr, + nullptr}}, + {DB::Properties::kIsFileDeletionsEnabled, + {false, nullptr, &InternalStats::HandleIsFileDeletionsEnabled, nullptr, + nullptr}}, + {DB::Properties::kNumSnapshots, + {false, nullptr, &InternalStats::HandleNumSnapshots, nullptr, + nullptr}}, + {DB::Properties::kOldestSnapshotTime, + {false, nullptr, &InternalStats::HandleOldestSnapshotTime, nullptr, + nullptr}}, + {DB::Properties::kOldestSnapshotSequence, + {false, nullptr, &InternalStats::HandleOldestSnapshotSequence, nullptr, + nullptr}}, + {DB::Properties::kNumLiveVersions, + {false, nullptr, &InternalStats::HandleNumLiveVersions, nullptr, + nullptr}}, + {DB::Properties::kCurrentSuperVersionNumber, + {false, nullptr, &InternalStats::HandleCurrentSuperVersionNumber, + nullptr, nullptr}}, + {DB::Properties::kEstimateLiveDataSize, + {true, nullptr, &InternalStats::HandleEstimateLiveDataSize, nullptr, + nullptr}}, + {DB::Properties::kMinLogNumberToKeep, + {false, nullptr, &InternalStats::HandleMinLogNumberToKeep, nullptr, + nullptr}}, + {DB::Properties::kMinObsoleteSstNumberToKeep, + {false, nullptr, &InternalStats::HandleMinObsoleteSstNumberToKeep, + nullptr, nullptr}}, + {DB::Properties::kBaseLevel, + {false, nullptr, &InternalStats::HandleBaseLevel, nullptr, nullptr}}, + {DB::Properties::kTotalSstFilesSize, + {false, nullptr, &InternalStats::HandleTotalSstFilesSize, nullptr, + nullptr}}, + {DB::Properties::kLiveSstFilesSize, + {false, nullptr, &InternalStats::HandleLiveSstFilesSize, nullptr, + nullptr}}, + {DB::Properties::kLiveSstFilesSizeAtTemperature, + {false, &InternalStats::HandleLiveSstFilesSizeAtTemperature, nullptr, + nullptr, nullptr}}, + {DB::Properties::kObsoleteSstFilesSize, + {false, nullptr, &InternalStats::HandleObsoleteSstFilesSize, nullptr, + nullptr}}, + {DB::Properties::kEstimatePendingCompactionBytes, + {false, nullptr, &InternalStats::HandleEstimatePendingCompactionBytes, + nullptr, nullptr}}, + {DB::Properties::kNumRunningFlushes, + {false, nullptr, &InternalStats::HandleNumRunningFlushes, nullptr, + nullptr}}, + {DB::Properties::kNumRunningCompactions, + {false, nullptr, &InternalStats::HandleNumRunningCompactions, nullptr, + nullptr}}, + {DB::Properties::kActualDelayedWriteRate, + {false, nullptr, &InternalStats::HandleActualDelayedWriteRate, nullptr, + nullptr}}, + {DB::Properties::kIsWriteStopped, + {false, nullptr, &InternalStats::HandleIsWriteStopped, nullptr, + nullptr}}, + {DB::Properties::kEstimateOldestKeyTime, + {false, nullptr, &InternalStats::HandleEstimateOldestKeyTime, nullptr, + nullptr}}, + {DB::Properties::kBlockCacheCapacity, + {false, nullptr, &InternalStats::HandleBlockCacheCapacity, nullptr, + nullptr}}, + {DB::Properties::kBlockCacheUsage, + {false, nullptr, &InternalStats::HandleBlockCacheUsage, nullptr, + nullptr}}, + {DB::Properties::kBlockCachePinnedUsage, + {false, nullptr, &InternalStats::HandleBlockCachePinnedUsage, nullptr, + nullptr}}, + {DB::Properties::kOptionsStatistics, + {true, nullptr, nullptr, nullptr, + &DBImpl::GetPropertyHandleOptionsStatistics}}, + {DB::Properties::kNumBlobFiles, + {false, nullptr, &InternalStats::HandleNumBlobFiles, nullptr, + nullptr}}, + {DB::Properties::kBlobStats, + {false, &InternalStats::HandleBlobStats, nullptr, nullptr, nullptr}}, + {DB::Properties::kTotalBlobFileSize, + {false, nullptr, &InternalStats::HandleTotalBlobFileSize, nullptr, + nullptr}}, + {DB::Properties::kLiveBlobFileSize, + {false, nullptr, &InternalStats::HandleLiveBlobFileSize, nullptr, + nullptr}}, + {DB::Properties::kLiveBlobFileGarbageSize, + {false, nullptr, &InternalStats::HandleLiveBlobFileGarbageSize, + nullptr, nullptr}}, + {DB::Properties::kBlobCacheCapacity, + {false, nullptr, &InternalStats::HandleBlobCacheCapacity, nullptr, + nullptr}}, + {DB::Properties::kBlobCacheUsage, + {false, nullptr, &InternalStats::HandleBlobCacheUsage, nullptr, + nullptr}}, + {DB::Properties::kBlobCachePinnedUsage, + {false, nullptr, &InternalStats::HandleBlobCachePinnedUsage, nullptr, + nullptr}}, +}; + +InternalStats::InternalStats(int num_levels, SystemClock* clock, + ColumnFamilyData* cfd) + : db_stats_{}, + cf_stats_value_{}, + cf_stats_count_{}, + comp_stats_(num_levels), + comp_stats_by_pri_(Env::Priority::TOTAL), + file_read_latency_(num_levels), + has_cf_change_since_dump_(true), + bg_error_count_(0), + number_levels_(num_levels), + clock_(clock), + cfd_(cfd), + started_at_(clock->NowMicros()) { + Cache* block_cache = GetBlockCacheForStats(); + if (block_cache) { + // Extract or create stats collector. Could fail in rare cases. + Status s = CacheEntryStatsCollector::GetShared( + block_cache, clock_, &cache_entry_stats_collector_); + if (s.ok()) { + assert(cache_entry_stats_collector_); + } else { + assert(!cache_entry_stats_collector_); + } + } +} + +void InternalStats::TEST_GetCacheEntryRoleStats(CacheEntryRoleStats* stats, + bool foreground) { + CollectCacheEntryStats(foreground); + if (cache_entry_stats_collector_) { + cache_entry_stats_collector_->GetStats(stats); + } +} + +void InternalStats::CollectCacheEntryStats(bool foreground) { + // This function is safe to call from any thread because + // cache_entry_stats_collector_ field is const after constructor + // and ->GetStats does its own synchronization, which also suffices for + // cache_entry_stats_. + + if (!cache_entry_stats_collector_) { + return; // nothing to do (e.g. no block cache) + } + + // For "background" collections, strictly cap the collection time by + // expanding effective cache TTL. For foreground, be more aggressive about + // getting latest data. + int min_interval_seconds = foreground ? 10 : 180; + // 1/500 = max of 0.2% of one CPU thread + int min_interval_factor = foreground ? 10 : 500; + cache_entry_stats_collector_->CollectStats(min_interval_seconds, + min_interval_factor); +} + +std::function +InternalStats::CacheEntryRoleStats::GetEntryCallback() { + return [&](const Slice& /*key*/, Cache::ObjectPtr /*value*/, size_t charge, + const Cache::CacheItemHelper* helper) -> void { + size_t role_idx = + static_cast(helper ? helper->role : CacheEntryRole::kMisc); + entry_counts[role_idx]++; + total_charges[role_idx] += charge; + }; +} + +void InternalStats::CacheEntryRoleStats::BeginCollection( + Cache* cache, SystemClock*, uint64_t start_time_micros) { + Clear(); + last_start_time_micros_ = start_time_micros; + ++collection_count; + std::ostringstream str; + str << cache->Name() << "@" << static_cast(cache) << "#" + << port::GetProcessID(); + cache_id = str.str(); + cache_capacity = cache->GetCapacity(); + cache_usage = cache->GetUsage(); + table_size = cache->GetTableAddressCount(); + occupancy = cache->GetOccupancyCount(); + hash_seed = cache->GetHashSeed(); +} + +void InternalStats::CacheEntryRoleStats::EndCollection( + Cache*, SystemClock*, uint64_t end_time_micros) { + last_end_time_micros_ = end_time_micros; +} + +void InternalStats::CacheEntryRoleStats::SkippedCollection() { + ++copies_of_last_collection; +} + +uint64_t InternalStats::CacheEntryRoleStats::GetLastDurationMicros() const { + if (last_end_time_micros_ > last_start_time_micros_) { + return last_end_time_micros_ - last_start_time_micros_; + } else { + return 0U; + } +} + +std::string InternalStats::CacheEntryRoleStats::ToString( + SystemClock* clock) const { + std::ostringstream str; + str << "Block cache " << cache_id + << " capacity: " << BytesToHumanString(cache_capacity) + << " seed: " << hash_seed << " usage: " << BytesToHumanString(cache_usage) + << " table_size: " << table_size << " occupancy: " << occupancy + << " collections: " << collection_count + << " last_copies: " << copies_of_last_collection + << " last_secs: " << (GetLastDurationMicros() / 1000000.0) + << " secs_since: " + << ((clock->NowMicros() - last_end_time_micros_) / 1000000U) << "\n"; + str << "Block cache entry stats(count,size,portion):"; + for (size_t i = 0; i < kNumCacheEntryRoles; ++i) { + if (entry_counts[i] > 0) { + str << " " << kCacheEntryRoleToCamelString[i] << "(" << entry_counts[i] + << "," << BytesToHumanString(total_charges[i]) << "," + << (100.0 * total_charges[i] / cache_capacity) << "%)"; + } + } + str << "\n"; + return str.str(); +} + +void InternalStats::CacheEntryRoleStats::ToMap( + std::map* values, SystemClock* clock) const { + values->clear(); + auto& v = *values; + v[BlockCacheEntryStatsMapKeys::CacheId()] = cache_id; + v[BlockCacheEntryStatsMapKeys::CacheCapacityBytes()] = + std::to_string(cache_capacity); + v[BlockCacheEntryStatsMapKeys::LastCollectionDurationSeconds()] = + std::to_string(GetLastDurationMicros() / 1000000.0); + v[BlockCacheEntryStatsMapKeys::LastCollectionAgeSeconds()] = + std::to_string((clock->NowMicros() - last_end_time_micros_) / 1000000U); + for (size_t i = 0; i < kNumCacheEntryRoles; ++i) { + auto role = static_cast(i); + v[BlockCacheEntryStatsMapKeys::EntryCount(role)] = + std::to_string(entry_counts[i]); + v[BlockCacheEntryStatsMapKeys::UsedBytes(role)] = + std::to_string(total_charges[i]); + v[BlockCacheEntryStatsMapKeys::UsedPercent(role)] = + std::to_string(100.0 * total_charges[i] / cache_capacity); + } +} + +bool InternalStats::HandleBlockCacheEntryStatsInternal(std::string* value, + bool fast) { + if (!cache_entry_stats_collector_) { + return false; + } + CollectCacheEntryStats(!fast /* foreground */); + CacheEntryRoleStats stats; + cache_entry_stats_collector_->GetStats(&stats); + *value = stats.ToString(clock_); + return true; +} + +bool InternalStats::HandleBlockCacheEntryStatsMapInternal( + std::map* values, bool fast) { + if (!cache_entry_stats_collector_) { + return false; + } + CollectCacheEntryStats(!fast /* foreground */); + CacheEntryRoleStats stats; + cache_entry_stats_collector_->GetStats(&stats); + stats.ToMap(values, clock_); + return true; +} + +bool InternalStats::HandleBlockCacheEntryStats(std::string* value, + Slice /*suffix*/) { + return HandleBlockCacheEntryStatsInternal(value, false /* fast */); +} + +bool InternalStats::HandleBlockCacheEntryStatsMap( + std::map* values, Slice /*suffix*/) { + return HandleBlockCacheEntryStatsMapInternal(values, false /* fast */); +} + +bool InternalStats::HandleFastBlockCacheEntryStats(std::string* value, + Slice /*suffix*/) { + return HandleBlockCacheEntryStatsInternal(value, true /* fast */); +} + +bool InternalStats::HandleFastBlockCacheEntryStatsMap( + std::map* values, Slice /*suffix*/) { + return HandleBlockCacheEntryStatsMapInternal(values, true /* fast */); +} + +bool InternalStats::HandleLiveSstFilesSizeAtTemperature(std::string* value, + Slice suffix) { + uint64_t temperature; + bool ok = ConsumeDecimalNumber(&suffix, &temperature) && suffix.empty(); + if (!ok) { + return false; + } + + uint64_t size = 0; + const auto* vstorage = cfd_->current()->storage_info(); + for (int level = 0; level < vstorage->num_levels(); level++) { + for (const auto& file_meta : vstorage->LevelFiles(level)) { + if (static_cast(file_meta->temperature) == temperature) { + size += file_meta->fd.GetFileSize(); + } + } + } + + *value = std::to_string(size); + return true; +} + +bool InternalStats::HandleNumBlobFiles(uint64_t* value, DBImpl* /*db*/, + Version* /*version*/) { + assert(value); + assert(cfd_); + + const auto* current = cfd_->current(); + assert(current); + + const auto* vstorage = current->storage_info(); + assert(vstorage); + + const auto& blob_files = vstorage->GetBlobFiles(); + + *value = blob_files.size(); + + return true; +} + +bool InternalStats::HandleBlobStats(std::string* value, Slice /*suffix*/) { + assert(value); + assert(cfd_); + + const auto* current = cfd_->current(); + assert(current); + + const auto* vstorage = current->storage_info(); + assert(vstorage); + + const auto blob_st = vstorage->GetBlobStats(); + + std::ostringstream oss; + + oss << "Number of blob files: " << vstorage->GetBlobFiles().size() + << "\nTotal size of blob files: " << blob_st.total_file_size + << "\nTotal size of garbage in blob files: " << blob_st.total_garbage_size + << "\nBlob file space amplification: " << blob_st.space_amp << '\n'; + + value->append(oss.str()); + + return true; +} + +bool InternalStats::HandleTotalBlobFileSize(uint64_t* value, DBImpl* /*db*/, + Version* /*version*/) { + assert(value); + assert(cfd_); + + *value = cfd_->GetTotalBlobFileSize(); + + return true; +} + +bool InternalStats::HandleLiveBlobFileSize(uint64_t* value, DBImpl* /*db*/, + Version* /*version*/) { + assert(value); + assert(cfd_); + + const auto* current = cfd_->current(); + assert(current); + + const auto* vstorage = current->storage_info(); + assert(vstorage); + + *value = vstorage->GetBlobStats().total_file_size; + + return true; +} + +bool InternalStats::HandleLiveBlobFileGarbageSize(uint64_t* value, + DBImpl* /*db*/, + Version* /*version*/) { + assert(value); + assert(cfd_); + + const auto* current = cfd_->current(); + assert(current); + + const auto* vstorage = current->storage_info(); + assert(vstorage); + + *value = vstorage->GetBlobStats().total_garbage_size; + + return true; +} + +Cache* InternalStats::GetBlobCacheForStats() { + return cfd_->ioptions()->blob_cache.get(); +} + +bool InternalStats::HandleBlobCacheCapacity(uint64_t* value, DBImpl* /*db*/, + Version* /*version*/) { + Cache* blob_cache = GetBlobCacheForStats(); + if (blob_cache) { + *value = static_cast(blob_cache->GetCapacity()); + return true; + } + return false; +} + +bool InternalStats::HandleBlobCacheUsage(uint64_t* value, DBImpl* /*db*/, + Version* /*version*/) { + Cache* blob_cache = GetBlobCacheForStats(); + if (blob_cache) { + *value = static_cast(blob_cache->GetUsage()); + return true; + } + return false; +} + +bool InternalStats::HandleBlobCachePinnedUsage(uint64_t* value, DBImpl* /*db*/, + Version* /*version*/) { + Cache* blob_cache = GetBlobCacheForStats(); + if (blob_cache) { + *value = static_cast(blob_cache->GetPinnedUsage()); + return true; + } + return false; +} + +const DBPropertyInfo* GetPropertyInfo(const Slice& property) { + std::string ppt_name = GetPropertyNameAndArg(property).first.ToString(); + auto ppt_info_iter = InternalStats::ppt_name_to_info.find(ppt_name); + if (ppt_info_iter == InternalStats::ppt_name_to_info.end()) { + return nullptr; + } + return &ppt_info_iter->second; +} + +bool InternalStats::GetStringProperty(const DBPropertyInfo& property_info, + const Slice& property, + std::string* value) { + assert(value != nullptr); + assert(property_info.handle_string != nullptr); + Slice arg = GetPropertyNameAndArg(property).second; + return (this->*(property_info.handle_string))(value, arg); +} + +bool InternalStats::GetMapProperty(const DBPropertyInfo& property_info, + const Slice& property, + std::map* value) { + assert(value != nullptr); + assert(property_info.handle_map != nullptr); + Slice arg = GetPropertyNameAndArg(property).second; + return (this->*(property_info.handle_map))(value, arg); +} + +bool InternalStats::GetIntProperty(const DBPropertyInfo& property_info, + uint64_t* value, DBImpl* db) { + assert(value != nullptr); + assert(property_info.handle_int != nullptr && + !property_info.need_out_of_mutex); + db->mutex_.AssertHeld(); + return (this->*(property_info.handle_int))(value, db, nullptr /* version */); +} + +bool InternalStats::GetIntPropertyOutOfMutex( + const DBPropertyInfo& property_info, Version* version, uint64_t* value) { + assert(value != nullptr); + assert(property_info.handle_int != nullptr && + property_info.need_out_of_mutex); + return (this->*(property_info.handle_int))(value, nullptr /* db */, version); +} + +bool InternalStats::HandleNumFilesAtLevel(std::string* value, Slice suffix) { + uint64_t level; + const auto* vstorage = cfd_->current()->storage_info(); + bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty(); + if (!ok || static_cast(level) >= number_levels_) { + return false; + } else { + char buf[100]; + snprintf(buf, sizeof(buf), "%d", + vstorage->NumLevelFiles(static_cast(level))); + *value = buf; + return true; + } +} + +bool InternalStats::HandleCompressionRatioAtLevelPrefix(std::string* value, + Slice suffix) { + uint64_t level; + const auto* vstorage = cfd_->current()->storage_info(); + bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty(); + if (!ok || level >= static_cast(number_levels_)) { + return false; + } + *value = std::to_string( + vstorage->GetEstimatedCompressionRatioAtLevel(static_cast(level))); + return true; +} + +bool InternalStats::HandleLevelStats(std::string* value, Slice /*suffix*/) { + char buf[1000]; + const auto* vstorage = cfd_->current()->storage_info(); + snprintf(buf, sizeof(buf), + "Level Files Size(MB)\n" + "--------------------\n"); + value->append(buf); + + for (int level = 0; level < number_levels_; level++) { + snprintf(buf, sizeof(buf), "%3d %8d %8.0f\n", level, + vstorage->NumLevelFiles(level), + vstorage->NumLevelBytes(level) / kMB); + value->append(buf); + } + return true; +} + +bool InternalStats::HandleStats(std::string* value, Slice suffix) { + if (!HandleCFStats(value, suffix)) { + return false; + } + if (!HandleDBStats(value, suffix)) { + return false; + } + return true; +} + +bool InternalStats::HandleCFMapStats( + std::map* cf_stats, Slice /*suffix*/) { + DumpCFMapStats(cf_stats); + return true; +} + +bool InternalStats::HandleCFStats(std::string* value, Slice /*suffix*/) { + DumpCFStats(value); + return true; +} + +bool InternalStats::HandleCFStatsPeriodic(std::string* value, + Slice /*suffix*/) { + bool has_change = has_cf_change_since_dump_; + if (!has_change) { + // If file histogram changes, there is activity in this period too. + uint64_t new_histogram_num = 0; + for (int level = 0; level < number_levels_; level++) { + new_histogram_num += file_read_latency_[level].num(); + } + new_histogram_num += blob_file_read_latency_.num(); + if (new_histogram_num != last_histogram_num) { + has_change = true; + last_histogram_num = new_histogram_num; + } + } + if (has_change) { + no_cf_change_period_since_dump_ = 0; + has_cf_change_since_dump_ = false; + } else if (no_cf_change_period_since_dump_++ > 0) { + // Not ready to sync + if (no_cf_change_period_since_dump_ == kMaxNoChangePeriodSinceDump) { + // Next periodic, we need to dump stats even if there is no change. + no_cf_change_period_since_dump_ = 0; + } + return true; + } + + DumpCFStatsNoFileHistogram(/*is_periodic=*/true, value); + DumpCFFileHistogram(value); + return true; +} + +bool InternalStats::HandleCFStatsNoFileHistogram(std::string* value, + Slice /*suffix*/) { + DumpCFStatsNoFileHistogram(/*is_periodic=*/false, value); + return true; +} + +bool InternalStats::HandleCFFileHistogram(std::string* value, + Slice /*suffix*/) { + DumpCFFileHistogram(value); + return true; +} + +bool InternalStats::HandleCFWriteStallStats(std::string* value, + Slice /*suffix*/) { + DumpCFStatsWriteStall(value); + return true; +} + +bool InternalStats::HandleCFWriteStallStatsMap( + std::map* value, Slice /*suffix*/) { + DumpCFMapStatsWriteStall(value); + return true; +} + +bool InternalStats::HandleDBMapStats( + std::map* db_stats, Slice /*suffix*/) { + DumpDBMapStats(db_stats); + return true; +} + +bool InternalStats::HandleDBStats(std::string* value, Slice /*suffix*/) { + DumpDBStats(value); + return true; +} + +bool InternalStats::HandleDBWriteStallStats(std::string* value, + Slice /*suffix*/) { + DumpDBStatsWriteStall(value); + return true; +} + +bool InternalStats::HandleDBWriteStallStatsMap( + std::map* value, Slice /*suffix*/) { + DumpDBMapStatsWriteStall(value); + return true; +} + +bool InternalStats::HandleSsTables(std::string* value, Slice /*suffix*/) { + auto* current = cfd_->current(); + *value = current->DebugString(true, true); + return true; +} + +bool InternalStats::HandleAggregatedTableProperties(std::string* value, + Slice /*suffix*/) { + std::shared_ptr tp; + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + auto s = cfd_->current()->GetAggregatedTableProperties(read_options, &tp); + if (!s.ok()) { + return false; + } + *value = tp->ToString(); + return true; +} + +static std::map MapUint64ValuesToString( + const std::map& from) { + std::map to; + for (const auto& e : from) { + to[e.first] = std::to_string(e.second); + } + return to; +} + +bool InternalStats::HandleAggregatedTablePropertiesMap( + std::map* values, Slice /*suffix*/) { + std::shared_ptr tp; + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + auto s = cfd_->current()->GetAggregatedTableProperties(read_options, &tp); + if (!s.ok()) { + return false; + } + *values = MapUint64ValuesToString(tp->GetAggregatablePropertiesAsMap()); + return true; +} + +bool InternalStats::HandleAggregatedTablePropertiesAtLevel(std::string* values, + Slice suffix) { + uint64_t level; + bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty(); + if (!ok || static_cast(level) >= number_levels_) { + return false; + } + std::shared_ptr tp; + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + auto s = cfd_->current()->GetAggregatedTableProperties( + read_options, &tp, static_cast(level)); + if (!s.ok()) { + return false; + } + *values = tp->ToString(); + return true; +} + +bool InternalStats::HandleAggregatedTablePropertiesAtLevelMap( + std::map* values, Slice suffix) { + uint64_t level; + bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty(); + if (!ok || static_cast(level) >= number_levels_) { + return false; + } + std::shared_ptr tp; + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + auto s = cfd_->current()->GetAggregatedTableProperties( + read_options, &tp, static_cast(level)); + if (!s.ok()) { + return false; + } + *values = MapUint64ValuesToString(tp->GetAggregatablePropertiesAsMap()); + return true; +} + +bool InternalStats::HandleNumImmutableMemTable(uint64_t* value, DBImpl* /*db*/, + Version* /*version*/) { + *value = cfd_->imm()->NumNotFlushed(); + return true; +} + +bool InternalStats::HandleNumImmutableMemTableFlushed(uint64_t* value, + DBImpl* /*db*/, + Version* /*version*/) { + *value = cfd_->imm()->NumFlushed(); + return true; +} + +bool InternalStats::HandleMemTableFlushPending(uint64_t* value, DBImpl* /*db*/, + Version* /*version*/) { + *value = (cfd_->imm()->IsFlushPending() ? 1 : 0); + return true; +} + +bool InternalStats::HandleNumRunningFlushes(uint64_t* value, DBImpl* db, + Version* /*version*/) { + *value = db->num_running_flushes(); + return true; +} + +bool InternalStats::HandleCompactionPending(uint64_t* value, DBImpl* /*db*/, + Version* /*version*/) { + // 1 if the system already determines at least one compaction is needed. + // 0 otherwise, + const auto* vstorage = cfd_->current()->storage_info(); + *value = (cfd_->compaction_picker()->NeedsCompaction(vstorage) ? 1 : 0); + return true; +} + +bool InternalStats::HandleNumRunningCompactions(uint64_t* value, DBImpl* db, + Version* /*version*/) { + *value = db->num_running_compactions_; + return true; +} + +bool InternalStats::HandleBackgroundErrors(uint64_t* value, DBImpl* /*db*/, + Version* /*version*/) { + // Accumulated number of errors in background flushes or compactions. + *value = GetBackgroundErrorCount(); + return true; +} + +bool InternalStats::HandleCurSizeActiveMemTable(uint64_t* value, DBImpl* /*db*/, + Version* /*version*/) { + // Current size of the active memtable + // Using ApproximateMemoryUsageFast to avoid the need for synchronization + *value = cfd_->mem()->ApproximateMemoryUsageFast(); + return true; +} + +bool InternalStats::HandleCurSizeAllMemTables(uint64_t* value, DBImpl* /*db*/, + Version* /*version*/) { + // Current size of the active memtable + immutable memtables + // Using ApproximateMemoryUsageFast to avoid the need for synchronization + *value = cfd_->mem()->ApproximateMemoryUsageFast() + + cfd_->imm()->ApproximateUnflushedMemTablesMemoryUsage(); + return true; +} + +bool InternalStats::HandleSizeAllMemTables(uint64_t* value, DBImpl* /*db*/, + Version* /*version*/) { + // Using ApproximateMemoryUsageFast to avoid the need for synchronization + *value = cfd_->mem()->ApproximateMemoryUsageFast() + + cfd_->imm()->ApproximateMemoryUsage(); + return true; +} + +bool InternalStats::HandleNumEntriesActiveMemTable(uint64_t* value, + DBImpl* /*db*/, + Version* /*version*/) { + // Current number of entires in the active memtable + *value = cfd_->mem()->num_entries(); + return true; +} + +bool InternalStats::HandleNumEntriesImmMemTables(uint64_t* value, + DBImpl* /*db*/, + Version* /*version*/) { + // Current number of entries in the immutable memtables + *value = cfd_->imm()->current()->GetTotalNumEntries(); + return true; +} + +bool InternalStats::HandleNumDeletesActiveMemTable(uint64_t* value, + DBImpl* /*db*/, + Version* /*version*/) { + // Current number of entires in the active memtable + *value = cfd_->mem()->num_deletes(); + return true; +} + +bool InternalStats::HandleNumDeletesImmMemTables(uint64_t* value, + DBImpl* /*db*/, + Version* /*version*/) { + // Current number of entries in the immutable memtables + *value = cfd_->imm()->current()->GetTotalNumDeletes(); + return true; +} + +bool InternalStats::HandleEstimateNumKeys(uint64_t* value, DBImpl* /*db*/, + Version* /*version*/) { + // Estimate number of entries in the column family: + // Use estimated entries in tables + total entries in memtables. + const auto* vstorage = cfd_->current()->storage_info(); + uint64_t estimate_keys = cfd_->mem()->num_entries() + + cfd_->imm()->current()->GetTotalNumEntries() + + vstorage->GetEstimatedActiveKeys(); + uint64_t estimate_deletes = + cfd_->mem()->num_deletes() + cfd_->imm()->current()->GetTotalNumDeletes(); + *value = estimate_keys > estimate_deletes * 2 + ? estimate_keys - (estimate_deletes * 2) + : 0; + return true; +} + +bool InternalStats::HandleNumSnapshots(uint64_t* value, DBImpl* db, + Version* /*version*/) { + *value = db->snapshots().count(); + return true; +} + +bool InternalStats::HandleOldestSnapshotTime(uint64_t* value, DBImpl* db, + Version* /*version*/) { + *value = static_cast(db->snapshots().GetOldestSnapshotTime()); + return true; +} + +bool InternalStats::HandleOldestSnapshotSequence(uint64_t* value, DBImpl* db, + Version* /*version*/) { + *value = static_cast(db->snapshots().GetOldestSnapshotSequence()); + return true; +} + +bool InternalStats::HandleNumLiveVersions(uint64_t* value, DBImpl* /*db*/, + Version* /*version*/) { + *value = cfd_->GetNumLiveVersions(); + return true; +} + +bool InternalStats::HandleCurrentSuperVersionNumber(uint64_t* value, + DBImpl* /*db*/, + Version* /*version*/) { + *value = cfd_->GetSuperVersionNumber(); + return true; +} + +bool InternalStats::HandleIsFileDeletionsEnabled(uint64_t* value, DBImpl* db, + Version* /*version*/) { + *value = db->IsFileDeletionsEnabled() ? 1 : 0; + return true; +} + +bool InternalStats::HandleBaseLevel(uint64_t* value, DBImpl* /*db*/, + Version* /*version*/) { + const auto* vstorage = cfd_->current()->storage_info(); + *value = vstorage->base_level(); + return true; +} + +bool InternalStats::HandleTotalSstFilesSize(uint64_t* value, DBImpl* /*db*/, + Version* /*version*/) { + *value = cfd_->GetTotalSstFilesSize(); + return true; +} + +bool InternalStats::HandleLiveSstFilesSize(uint64_t* value, DBImpl* /*db*/, + Version* /*version*/) { + *value = cfd_->GetLiveSstFilesSize(); + return true; +} + +bool InternalStats::HandleObsoleteSstFilesSize(uint64_t* value, DBImpl* db, + Version* /*version*/) { + *value = db->GetObsoleteSstFilesSize(); + return true; +} + +bool InternalStats::HandleEstimatePendingCompactionBytes(uint64_t* value, + DBImpl* /*db*/, + Version* /*version*/) { + const auto* vstorage = cfd_->current()->storage_info(); + *value = vstorage->estimated_compaction_needed_bytes(); + return true; +} + +bool InternalStats::HandleEstimateTableReadersMem(uint64_t* value, + DBImpl* /*db*/, + Version* version) { + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + *value = (version == nullptr) + ? 0 + : version->GetMemoryUsageByTableReaders(read_options); + return true; +} + +bool InternalStats::HandleEstimateLiveDataSize(uint64_t* value, DBImpl* /*db*/, + Version* version) { + const auto* vstorage = version->storage_info(); + *value = vstorage->EstimateLiveDataSize(); + return true; +} + +bool InternalStats::HandleMinLogNumberToKeep(uint64_t* value, DBImpl* db, + Version* /*version*/) { + *value = db->MinLogNumberToKeep(); + return true; +} + +bool InternalStats::HandleMinObsoleteSstNumberToKeep(uint64_t* value, + DBImpl* db, + Version* /*version*/) { + *value = db->MinObsoleteSstNumberToKeep(); + return true; +} + +bool InternalStats::HandleActualDelayedWriteRate(uint64_t* value, DBImpl* db, + Version* /*version*/) { + const WriteController& wc = db->write_controller(); + if (!wc.NeedsDelay()) { + *value = 0; + } else { + *value = wc.delayed_write_rate(); + } + return true; +} + +bool InternalStats::HandleIsWriteStopped(uint64_t* value, DBImpl* db, + Version* /*version*/) { + *value = db->write_controller().IsStopped() ? 1 : 0; + return true; +} + +bool InternalStats::HandleEstimateOldestKeyTime(uint64_t* value, DBImpl* /*db*/, + Version* /*version*/) { + // TODO(yiwu): The property is currently available for fifo compaction + // with allow_compaction = false. This is because we don't propagate + // oldest_key_time on compaction. + if (cfd_->ioptions()->compaction_style != kCompactionStyleFIFO || + cfd_->GetCurrentMutableCFOptions() + ->compaction_options_fifo.allow_compaction) { + return false; + } + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + TablePropertiesCollection collection; + auto s = cfd_->current()->GetPropertiesOfAllTables(read_options, &collection); + if (!s.ok()) { + return false; + } + *value = std::numeric_limits::max(); + for (auto& p : collection) { + *value = std::min(*value, p.second->oldest_key_time); + if (*value == 0) { + break; + } + } + if (*value > 0) { + *value = std::min({cfd_->mem()->ApproximateOldestKeyTime(), + cfd_->imm()->ApproximateOldestKeyTime(), *value}); + } + return *value > 0 && *value < std::numeric_limits::max(); +} + +Cache* InternalStats::GetBlockCacheForStats() { + auto* table_factory = cfd_->ioptions()->table_factory.get(); + assert(table_factory != nullptr); + return table_factory->GetOptions(TableFactory::kBlockCacheOpts()); +} + +bool InternalStats::HandleBlockCacheCapacity(uint64_t* value, DBImpl* /*db*/, + Version* /*version*/) { + Cache* block_cache = GetBlockCacheForStats(); + if (block_cache) { + *value = static_cast(block_cache->GetCapacity()); + return true; + } + return false; +} + +bool InternalStats::HandleBlockCacheUsage(uint64_t* value, DBImpl* /*db*/, + Version* /*version*/) { + Cache* block_cache = GetBlockCacheForStats(); + if (block_cache) { + *value = static_cast(block_cache->GetUsage()); + return true; + } + return false; +} + +bool InternalStats::HandleBlockCachePinnedUsage(uint64_t* value, DBImpl* /*db*/, + Version* /*version*/) { + Cache* block_cache = GetBlockCacheForStats(); + if (block_cache) { + *value = static_cast(block_cache->GetPinnedUsage()); + return true; + } + return false; +} + +void InternalStats::DumpDBMapStats( + std::map* db_stats) { + for (int i = 0; i < static_cast(kIntStatsNumMax); ++i) { + InternalDBStatsType type = static_cast(i); + (*db_stats)[db_stats_type_to_info.at(type).property_name] = + std::to_string(GetDBStats(type)); + } + double seconds_up = (clock_->NowMicros() - started_at_) / kMicrosInSec; + (*db_stats)["db.uptime"] = std::to_string(seconds_up); +} + +void InternalStats::DumpDBStats(std::string* value) { + char buf[1000]; + // DB-level stats, only available from default column family + double seconds_up = (clock_->NowMicros() - started_at_) / kMicrosInSec; + double interval_seconds_up = seconds_up - db_stats_snapshot_.seconds_up; + snprintf(buf, sizeof(buf), + "\n** DB Stats **\nUptime(secs): %.1f total, %.1f interval\n", + seconds_up, interval_seconds_up); + value->append(buf); + // Cumulative + uint64_t user_bytes_written = + GetDBStats(InternalStats::kIntStatsBytesWritten); + uint64_t num_keys_written = + GetDBStats(InternalStats::kIntStatsNumKeysWritten); + uint64_t write_other = GetDBStats(InternalStats::kIntStatsWriteDoneByOther); + uint64_t write_self = GetDBStats(InternalStats::kIntStatsWriteDoneBySelf); + uint64_t wal_bytes = GetDBStats(InternalStats::kIntStatsWalFileBytes); + uint64_t wal_synced = GetDBStats(InternalStats::kIntStatsWalFileSynced); + uint64_t write_with_wal = GetDBStats(InternalStats::kIntStatsWriteWithWal); + uint64_t write_stall_micros = + GetDBStats(InternalStats::kIntStatsWriteStallMicros); + + const int kHumanMicrosLen = 32; + char human_micros[kHumanMicrosLen]; + + // Data + // writes: total number of write requests. + // keys: total number of key updates issued by all the write requests + // commit groups: number of group commits issued to the DB. Each group can + // contain one or more writes. + // so writes/keys is the average number of put in multi-put or put + // writes/groups is the average group commit size. + // + // The format is the same for interval stats. + snprintf(buf, sizeof(buf), + "Cumulative writes: %s writes, %s keys, %s commit groups, " + "%.1f writes per commit group, ingest: %.2f GB, %.2f MB/s\n", + NumberToHumanString(write_other + write_self).c_str(), + NumberToHumanString(num_keys_written).c_str(), + NumberToHumanString(write_self).c_str(), + (write_other + write_self) / + std::max(1.0, static_cast(write_self)), + user_bytes_written / kGB, + user_bytes_written / kMB / std::max(seconds_up, 0.001)); + value->append(buf); + // WAL + snprintf(buf, sizeof(buf), + "Cumulative WAL: %s writes, %s syncs, " + "%.2f writes per sync, written: %.2f GB, %.2f MB/s\n", + NumberToHumanString(write_with_wal).c_str(), + NumberToHumanString(wal_synced).c_str(), + write_with_wal / std::max(1.0, static_cast(wal_synced)), + wal_bytes / kGB, wal_bytes / kMB / std::max(seconds_up, 0.001)); + value->append(buf); + // Stall + AppendHumanMicros(write_stall_micros, human_micros, kHumanMicrosLen, true); + snprintf(buf, sizeof(buf), "Cumulative stall: %s, %.1f percent\n", + human_micros, + // 10000 = divide by 1M to get secs, then multiply by 100 for pct + write_stall_micros / 10000.0 / std::max(seconds_up, 0.001)); + value->append(buf); + + // Interval + uint64_t interval_write_other = write_other - db_stats_snapshot_.write_other; + uint64_t interval_write_self = write_self - db_stats_snapshot_.write_self; + uint64_t interval_num_keys_written = + num_keys_written - db_stats_snapshot_.num_keys_written; + snprintf( + buf, sizeof(buf), + "Interval writes: %s writes, %s keys, %s commit groups, " + "%.1f writes per commit group, ingest: %.2f MB, %.2f MB/s\n", + NumberToHumanString(interval_write_other + interval_write_self).c_str(), + NumberToHumanString(interval_num_keys_written).c_str(), + NumberToHumanString(interval_write_self).c_str(), + static_cast(interval_write_other + interval_write_self) / + std::max(1.0, static_cast(interval_write_self)), + (user_bytes_written - db_stats_snapshot_.ingest_bytes) / kMB, + (user_bytes_written - db_stats_snapshot_.ingest_bytes) / kMB / + std::max(interval_seconds_up, 0.001)), + value->append(buf); + + uint64_t interval_write_with_wal = + write_with_wal - db_stats_snapshot_.write_with_wal; + uint64_t interval_wal_synced = wal_synced - db_stats_snapshot_.wal_synced; + uint64_t interval_wal_bytes = wal_bytes - db_stats_snapshot_.wal_bytes; + + snprintf(buf, sizeof(buf), + "Interval WAL: %s writes, %s syncs, " + "%.2f writes per sync, written: %.2f GB, %.2f MB/s\n", + NumberToHumanString(interval_write_with_wal).c_str(), + NumberToHumanString(interval_wal_synced).c_str(), + interval_write_with_wal / + std::max(1.0, static_cast(interval_wal_synced)), + interval_wal_bytes / kGB, + interval_wal_bytes / kMB / std::max(interval_seconds_up, 0.001)); + value->append(buf); + + // Stall + AppendHumanMicros(write_stall_micros - db_stats_snapshot_.write_stall_micros, + human_micros, kHumanMicrosLen, true); + snprintf(buf, sizeof(buf), "Interval stall: %s, %.1f percent\n", human_micros, + // 10000 = divide by 1M to get secs, then multiply by 100 for pct + (write_stall_micros - db_stats_snapshot_.write_stall_micros) / + 10000.0 / std::max(interval_seconds_up, 0.001)); + value->append(buf); + + std::string write_stall_stats; + DumpDBStatsWriteStall(&write_stall_stats); + value->append(write_stall_stats); + + db_stats_snapshot_.seconds_up = seconds_up; + db_stats_snapshot_.ingest_bytes = user_bytes_written; + db_stats_snapshot_.write_other = write_other; + db_stats_snapshot_.write_self = write_self; + db_stats_snapshot_.num_keys_written = num_keys_written; + db_stats_snapshot_.wal_bytes = wal_bytes; + db_stats_snapshot_.wal_synced = wal_synced; + db_stats_snapshot_.write_with_wal = write_with_wal; + db_stats_snapshot_.write_stall_micros = write_stall_micros; +} + +void InternalStats::DumpDBMapStatsWriteStall( + std::map* value) { + constexpr uint32_t max_db_scope_write_stall_cause = + static_cast(WriteStallCause::kDBScopeWriteStallCauseEnumMax); + + for (uint32_t i = + max_db_scope_write_stall_cause - kNumDBScopeWriteStallCauses; + i < max_db_scope_write_stall_cause; ++i) { + for (uint32_t j = 0; + j < static_cast(WriteStallCondition::kNormal); ++j) { + WriteStallCause cause = static_cast(i); + WriteStallCondition condition = static_cast(j); + InternalStats::InternalDBStatsType internal_db_stat = + InternalDBStat(cause, condition); + + if (internal_db_stat == InternalStats::kIntStatsNumMax) { + continue; + } + + std::string name = + WriteStallStatsMapKeys::CauseConditionCount(cause, condition); + uint64_t stat = + db_stats_[static_cast(internal_db_stat)].load( + std::memory_order_relaxed); + (*value)[name] = std::to_string(stat); + } + } +} + +void InternalStats::DumpDBStatsWriteStall(std::string* value) { + assert(value); + + std::map write_stall_stats_map; + DumpDBMapStatsWriteStall(&write_stall_stats_map); + + std::ostringstream str; + str << "Write Stall (count): "; + + for (auto write_stall_stats_map_iter = write_stall_stats_map.begin(); + write_stall_stats_map_iter != write_stall_stats_map.end(); + write_stall_stats_map_iter++) { + const auto& name_and_stat = *write_stall_stats_map_iter; + str << name_and_stat.first << ": " << name_and_stat.second; + if (std::next(write_stall_stats_map_iter) == write_stall_stats_map.end()) { + str << "\n"; + } else { + str << ", "; + } + } + *value = str.str(); +} + +/** + * Dump Compaction Level stats to a map of stat name with "compaction." prefix + * to value in double as string. The level in stat name is represented with + * a prefix "Lx" where "x" is the level number. A special level "Sum" + * represents the sum of a stat for all levels. + * The result also contains IO stall counters which keys start with "io_stalls." + * and values represent uint64 encoded as strings. + */ +void InternalStats::DumpCFMapStats( + std::map* cf_stats) { + const VersionStorageInfo* vstorage = cfd_->current()->storage_info(); + CompactionStats compaction_stats_sum; + std::map> levels_stats; + DumpCFMapStats(vstorage, &levels_stats, &compaction_stats_sum); + for (auto const& level_ent : levels_stats) { + auto level_str = + level_ent.first == -1 ? "Sum" : "L" + std::to_string(level_ent.first); + for (auto const& stat_ent : level_ent.second) { + auto stat_type = stat_ent.first; + auto key_str = + "compaction." + level_str + "." + + InternalStats::compaction_level_stats.at(stat_type).property_name; + (*cf_stats)[key_str] = std::to_string(stat_ent.second); + } + } + + DumpCFMapStatsWriteStall(cf_stats); +} + +void InternalStats::DumpCFMapStats( + const VersionStorageInfo* vstorage, + std::map>* levels_stats, + CompactionStats* compaction_stats_sum) { + assert(vstorage); + + int num_levels_to_check = + (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) + ? vstorage->num_levels() - 1 + : 1; + + // Compaction scores are sorted based on its value. Restore them to the + // level order + std::vector compaction_score(number_levels_, 0); + for (int i = 0; i < num_levels_to_check; ++i) { + compaction_score[vstorage->CompactionScoreLevel(i)] = + vstorage->CompactionScore(i); + } + // Count # of files being compacted for each level + std::vector files_being_compacted(number_levels_, 0); + for (int level = 0; level < number_levels_; ++level) { + for (auto* f : vstorage->LevelFiles(level)) { + if (f->being_compacted) { + ++files_being_compacted[level]; + } + } + } + + int total_files = 0; + int total_files_being_compacted = 0; + double total_file_size = 0; + uint64_t flush_ingest = cf_stats_value_[BYTES_FLUSHED]; + uint64_t add_file_ingest = cf_stats_value_[BYTES_INGESTED_ADD_FILE]; + uint64_t curr_ingest = flush_ingest + add_file_ingest; + for (int level = 0; level < number_levels_; level++) { + int files = vstorage->NumLevelFiles(level); + total_files += files; + total_files_being_compacted += files_being_compacted[level]; + if (comp_stats_[level].micros > 0 || comp_stats_[level].cpu_micros > 0 || + files > 0) { + compaction_stats_sum->Add(comp_stats_[level]); + total_file_size += vstorage->NumLevelBytes(level); + uint64_t input_bytes; + if (level == 0) { + input_bytes = curr_ingest; + } else { + input_bytes = comp_stats_[level].bytes_read_non_output_levels + + comp_stats_[level].bytes_read_blob; + } + double w_amp = + (input_bytes == 0) + ? 0.0 + : static_cast(comp_stats_[level].bytes_written + + comp_stats_[level].bytes_written_blob) / + input_bytes; + std::map level_stats; + PrepareLevelStats(&level_stats, files, files_being_compacted[level], + static_cast(vstorage->NumLevelBytes(level)), + compaction_score[level], w_amp, comp_stats_[level]); + (*levels_stats)[level] = level_stats; + } + } + // Cumulative summary + double w_amp = (0 == curr_ingest) + ? 0.0 + : (compaction_stats_sum->bytes_written + + compaction_stats_sum->bytes_written_blob) / + static_cast(curr_ingest); + // Stats summary across levels + std::map sum_stats; + PrepareLevelStats(&sum_stats, total_files, total_files_being_compacted, + total_file_size, 0, w_amp, *compaction_stats_sum); + (*levels_stats)[-1] = sum_stats; // -1 is for the Sum level +} + +void InternalStats::DumpCFMapStatsByPriority( + std::map>* priorities_stats) { + for (size_t priority = 0; priority < comp_stats_by_pri_.size(); priority++) { + if (comp_stats_by_pri_[priority].micros > 0) { + std::map priority_stats; + PrepareLevelStats(&priority_stats, 0 /* num_files */, + 0 /* being_compacted */, 0 /* total_file_size */, + 0 /* compaction_score */, 0 /* w_amp */, + comp_stats_by_pri_[priority]); + (*priorities_stats)[static_cast(priority)] = priority_stats; + } + } +} + +void InternalStats::DumpCFMapStatsWriteStall( + std::map* value) { + uint64_t total_delays = 0; + uint64_t total_stops = 0; + constexpr uint32_t max_cf_scope_write_stall_cause = + static_cast(WriteStallCause::kCFScopeWriteStallCauseEnumMax); + + for (uint32_t i = + max_cf_scope_write_stall_cause - kNumCFScopeWriteStallCauses; + i < max_cf_scope_write_stall_cause; ++i) { + for (uint32_t j = 0; + j < static_cast(WriteStallCondition::kNormal); ++j) { + WriteStallCause cause = static_cast(i); + WriteStallCondition condition = static_cast(j); + InternalStats::InternalCFStatsType internal_cf_stat = + InternalCFStat(cause, condition); + + if (internal_cf_stat == InternalStats::INTERNAL_CF_STATS_ENUM_MAX) { + continue; + } + + std::string name = + WriteStallStatsMapKeys::CauseConditionCount(cause, condition); + uint64_t stat = + cf_stats_count_[static_cast(internal_cf_stat)]; + (*value)[name] = std::to_string(stat); + + if (condition == WriteStallCondition::kDelayed) { + total_delays += stat; + } else if (condition == WriteStallCondition::kStopped) { + total_stops += stat; + } + } + } + + (*value)[WriteStallStatsMapKeys:: + CFL0FileCountLimitDelaysWithOngoingCompaction()] = + std::to_string( + cf_stats_count_[L0_FILE_COUNT_LIMIT_DELAYS_WITH_ONGOING_COMPACTION]); + (*value)[WriteStallStatsMapKeys:: + CFL0FileCountLimitStopsWithOngoingCompaction()] = + std::to_string( + cf_stats_count_[L0_FILE_COUNT_LIMIT_STOPS_WITH_ONGOING_COMPACTION]); + + (*value)[WriteStallStatsMapKeys::TotalStops()] = std::to_string(total_stops); + (*value)[WriteStallStatsMapKeys::TotalDelays()] = + std::to_string(total_delays); +} + +void InternalStats::DumpCFStatsWriteStall(std::string* value, + uint64_t* total_stall_count) { + assert(value); + + std::map write_stall_stats_map; + DumpCFMapStatsWriteStall(&write_stall_stats_map); + + std::ostringstream str; + str << "Write Stall (count): "; + + for (auto write_stall_stats_map_iter = write_stall_stats_map.begin(); + write_stall_stats_map_iter != write_stall_stats_map.end(); + write_stall_stats_map_iter++) { + const auto& name_and_stat = *write_stall_stats_map_iter; + str << name_and_stat.first << ": " << name_and_stat.second; + if (std::next(write_stall_stats_map_iter) == write_stall_stats_map.end()) { + str << "\n"; + } else { + str << ", "; + } + } + + if (total_stall_count) { + *total_stall_count = + ParseUint64( + write_stall_stats_map[WriteStallStatsMapKeys::TotalStops()]) + + ParseUint64( + write_stall_stats_map[WriteStallStatsMapKeys::TotalDelays()]); + if (*total_stall_count > 0) { + str << "interval: " << *total_stall_count - cf_stats_snapshot_.stall_count + << " total count\n"; + } + } + *value = str.str(); +} + +void InternalStats::DumpCFStats(std::string* value) { + DumpCFStatsNoFileHistogram(/*is_periodic=*/false, value); + DumpCFFileHistogram(value); +} + +void InternalStats::DumpCFStatsNoFileHistogram(bool is_periodic, + std::string* value) { + char buf[2000]; + // Per-ColumnFamily stats + PrintLevelStatsHeader(buf, sizeof(buf), cfd_->GetName(), "Level"); + value->append(buf); + + // Print stats for each level + const VersionStorageInfo* vstorage = cfd_->current()->storage_info(); + std::map> levels_stats; + CompactionStats compaction_stats_sum; + DumpCFMapStats(vstorage, &levels_stats, &compaction_stats_sum); + for (int l = 0; l < number_levels_; ++l) { + if (levels_stats.find(l) != levels_stats.end()) { + PrintLevelStats(buf, sizeof(buf), "L" + std::to_string(l), + levels_stats[l]); + value->append(buf); + } + } + + // Print sum of level stats + PrintLevelStats(buf, sizeof(buf), "Sum", levels_stats[-1]); + value->append(buf); + + uint64_t flush_ingest = cf_stats_value_[BYTES_FLUSHED]; + uint64_t add_file_ingest = cf_stats_value_[BYTES_INGESTED_ADD_FILE]; + uint64_t ingest_files_addfile = cf_stats_value_[INGESTED_NUM_FILES_TOTAL]; + uint64_t ingest_l0_files_addfile = + cf_stats_value_[INGESTED_LEVEL0_NUM_FILES_TOTAL]; + uint64_t ingest_keys_addfile = cf_stats_value_[INGESTED_NUM_KEYS_TOTAL]; + // Interval summary + uint64_t interval_flush_ingest = + flush_ingest - cf_stats_snapshot_.ingest_bytes_flush; + uint64_t interval_add_file_inget = + add_file_ingest - cf_stats_snapshot_.ingest_bytes_addfile; + uint64_t interval_ingest = + interval_flush_ingest + interval_add_file_inget + 1; + CompactionStats interval_stats(compaction_stats_sum); + interval_stats.Subtract(cf_stats_snapshot_.comp_stats); + double w_amp = + (interval_stats.bytes_written + interval_stats.bytes_written_blob) / + static_cast(interval_ingest); + PrintLevelStats(buf, sizeof(buf), "Int", 0, 0, 0, 0, w_amp, interval_stats); + value->append(buf); + + PrintLevelStatsHeader(buf, sizeof(buf), cfd_->GetName(), "Priority"); + value->append(buf); + std::map> priorities_stats; + DumpCFMapStatsByPriority(&priorities_stats); + for (size_t priority = 0; priority < comp_stats_by_pri_.size(); ++priority) { + if (priorities_stats.find(static_cast(priority)) != + priorities_stats.end()) { + PrintLevelStats( + buf, sizeof(buf), + Env::PriorityToString(static_cast(priority)), + priorities_stats[static_cast(priority)]); + value->append(buf); + } + } + + const auto blob_st = vstorage->GetBlobStats(); + + snprintf(buf, sizeof(buf), + "\nBlob file count: %" ROCKSDB_PRIszt + ", total size: %.1f GB, garbage size: %.1f GB, space amp: %.1f\n\n", + vstorage->GetBlobFiles().size(), blob_st.total_file_size / kGB, + blob_st.total_garbage_size / kGB, blob_st.space_amp); + value->append(buf); + + uint64_t now_micros = clock_->NowMicros(); + double seconds_up = (now_micros - started_at_) / kMicrosInSec; + double interval_seconds_up = seconds_up - cf_stats_snapshot_.seconds_up; + snprintf(buf, sizeof(buf), "Uptime(secs): %.1f total, %.1f interval\n", + seconds_up, interval_seconds_up); + value->append(buf); + snprintf(buf, sizeof(buf), "Flush(GB): cumulative %.3f, interval %.3f\n", + flush_ingest / kGB, interval_flush_ingest / kGB); + value->append(buf); + snprintf(buf, sizeof(buf), "AddFile(GB): cumulative %.3f, interval %.3f\n", + add_file_ingest / kGB, interval_add_file_inget / kGB); + value->append(buf); + + uint64_t interval_ingest_files_addfile = + ingest_files_addfile - cf_stats_snapshot_.ingest_files_addfile; + snprintf(buf, sizeof(buf), + "AddFile(Total Files): cumulative %" PRIu64 ", interval %" PRIu64 + "\n", + ingest_files_addfile, interval_ingest_files_addfile); + value->append(buf); + + uint64_t interval_ingest_l0_files_addfile = + ingest_l0_files_addfile - cf_stats_snapshot_.ingest_l0_files_addfile; + snprintf(buf, sizeof(buf), + "AddFile(L0 Files): cumulative %" PRIu64 ", interval %" PRIu64 "\n", + ingest_l0_files_addfile, interval_ingest_l0_files_addfile); + value->append(buf); + + uint64_t interval_ingest_keys_addfile = + ingest_keys_addfile - cf_stats_snapshot_.ingest_keys_addfile; + snprintf(buf, sizeof(buf), + "AddFile(Keys): cumulative %" PRIu64 ", interval %" PRIu64 "\n", + ingest_keys_addfile, interval_ingest_keys_addfile); + value->append(buf); + + // Compact + uint64_t compact_bytes_read = 0; + uint64_t compact_bytes_write = 0; + uint64_t compact_micros = 0; + for (int level = 0; level < number_levels_; level++) { + compact_bytes_read += comp_stats_[level].bytes_read_output_level + + comp_stats_[level].bytes_read_non_output_levels + + comp_stats_[level].bytes_read_blob; + compact_bytes_write += comp_stats_[level].bytes_written + + comp_stats_[level].bytes_written_blob; + compact_micros += comp_stats_[level].micros; + } + + snprintf(buf, sizeof(buf), + "Cumulative compaction: %.2f GB write, %.2f MB/s write, " + "%.2f GB read, %.2f MB/s read, %.1f seconds\n", + compact_bytes_write / kGB, + compact_bytes_write / kMB / std::max(seconds_up, 0.001), + compact_bytes_read / kGB, + compact_bytes_read / kMB / std::max(seconds_up, 0.001), + compact_micros / kMicrosInSec); + value->append(buf); + + // Compaction interval + uint64_t interval_compact_bytes_write = + compact_bytes_write - cf_stats_snapshot_.compact_bytes_write; + uint64_t interval_compact_bytes_read = + compact_bytes_read - cf_stats_snapshot_.compact_bytes_read; + uint64_t interval_compact_micros = + compact_micros - cf_stats_snapshot_.compact_micros; + + snprintf( + buf, sizeof(buf), + "Interval compaction: %.2f GB write, %.2f MB/s write, " + "%.2f GB read, %.2f MB/s read, %.1f seconds\n", + interval_compact_bytes_write / kGB, + interval_compact_bytes_write / kMB / std::max(interval_seconds_up, 0.001), + interval_compact_bytes_read / kGB, + interval_compact_bytes_read / kMB / std::max(interval_seconds_up, 0.001), + interval_compact_micros / kMicrosInSec); + value->append(buf); + if (is_periodic) { + cf_stats_snapshot_.compact_bytes_write = compact_bytes_write; + cf_stats_snapshot_.compact_bytes_read = compact_bytes_read; + cf_stats_snapshot_.compact_micros = compact_micros; + } + + std::string write_stall_stats; + uint64_t total_stall_count; + DumpCFStatsWriteStall(&write_stall_stats, &total_stall_count); + value->append(write_stall_stats); + + if (is_periodic) { + cf_stats_snapshot_.seconds_up = seconds_up; + cf_stats_snapshot_.ingest_bytes_flush = flush_ingest; + cf_stats_snapshot_.ingest_bytes_addfile = add_file_ingest; + cf_stats_snapshot_.ingest_files_addfile = ingest_files_addfile; + cf_stats_snapshot_.ingest_l0_files_addfile = ingest_l0_files_addfile; + cf_stats_snapshot_.ingest_keys_addfile = ingest_keys_addfile; + cf_stats_snapshot_.comp_stats = compaction_stats_sum; + cf_stats_snapshot_.stall_count = total_stall_count; + } + + // Do not gather cache entry stats during CFStats because DB + // mutex is held. Only dump last cached collection (rely on DB + // periodic stats dump to update) + if (cache_entry_stats_collector_) { + CacheEntryRoleStats stats; + // thread safe + cache_entry_stats_collector_->GetStats(&stats); + + constexpr uint64_t kDayInMicros = uint64_t{86400} * 1000000U; + + // Skip if stats are extremely old (> 1 day, incl not yet populated) + if (now_micros - stats.last_end_time_micros_ < kDayInMicros) { + value->append(stats.ToString(clock_)); + } + } +} + +void InternalStats::DumpCFFileHistogram(std::string* value) { + assert(value); + assert(cfd_); + + std::ostringstream oss; + oss << "\n** File Read Latency Histogram By Level [" << cfd_->GetName() + << "] **\n"; + + for (int level = 0; level < number_levels_; level++) { + if (!file_read_latency_[level].Empty()) { + oss << "** Level " << level << " read latency histogram (micros):\n" + << file_read_latency_[level].ToString() << '\n'; + } + } + + if (!blob_file_read_latency_.Empty()) { + oss << "** Blob file read latency histogram (micros):\n" + << blob_file_read_latency_.ToString() << '\n'; + } + + value->append(oss.str()); +} + + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/internal_stats.h b/librocksdb-sys/rocksdb/db/internal_stats.h new file mode 100644 index 0000000..85c1a6b --- /dev/null +++ b/librocksdb-sys/rocksdb/db/internal_stats.h @@ -0,0 +1,877 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// + +#pragma once + +#include +#include +#include +#include + +#include "cache/cache_entry_roles.h" +#include "db/version_set.h" +#include "rocksdb/system_clock.h" +#include "util/hash_containers.h" + +namespace ROCKSDB_NAMESPACE { + +template +class CacheEntryStatsCollector; +class DBImpl; +class MemTableList; + +// Config for retrieving a property's value. +struct DBPropertyInfo { + bool need_out_of_mutex; + + // gcc had an internal error for initializing union of pointer-to-member- + // functions. Workaround is to populate exactly one of the following function + // pointers with a non-nullptr value. + + // @param value Value-result argument for storing the property's string value + // @param suffix Argument portion of the property. For example, suffix would + // be "5" for the property "rocksdb.num-files-at-level5". So far, only + // certain string properties take an argument. + bool (InternalStats::*handle_string)(std::string* value, Slice suffix); + + // @param value Value-result argument for storing the property's uint64 value + // @param db Many of the int properties rely on DBImpl methods. + // @param version Version is needed in case the property is retrieved without + // holding db mutex, which is only supported for int properties. + bool (InternalStats::*handle_int)(uint64_t* value, DBImpl* db, + Version* version); + + // @param props Map of general properties to populate + // @param suffix Argument portion of the property. (see handle_string) + bool (InternalStats::*handle_map)(std::map* props, + Slice suffix); + + // handle the string type properties rely on DBImpl methods + // @param value Value-result argument for storing the property's string value + bool (DBImpl::*handle_string_dbimpl)(std::string* value); +}; + +extern const DBPropertyInfo* GetPropertyInfo(const Slice& property); + +#undef SCORE +enum class LevelStatType { + INVALID = 0, + NUM_FILES, + COMPACTED_FILES, + SIZE_BYTES, + SCORE, + READ_GB, + RN_GB, + RNP1_GB, + WRITE_GB, + W_NEW_GB, + MOVED_GB, + WRITE_AMP, + READ_MBPS, + WRITE_MBPS, + COMP_SEC, + COMP_CPU_SEC, + COMP_COUNT, + AVG_SEC, + KEY_IN, + KEY_DROP, + R_BLOB_GB, + W_BLOB_GB, + TOTAL // total number of types +}; + +struct LevelStat { + // This what will be L?.property_name in the flat map returned to the user + std::string property_name; + // This will be what we will print in the header in the cli + std::string header_name; +}; + +struct DBStatInfo { + // This what will be property_name in the flat map returned to the user + std::string property_name; +}; + +class InternalStats { + public: + static const std::map compaction_level_stats; + + enum InternalCFStatsType { + MEMTABLE_LIMIT_DELAYS, + MEMTABLE_LIMIT_STOPS, + L0_FILE_COUNT_LIMIT_DELAYS, + L0_FILE_COUNT_LIMIT_STOPS, + PENDING_COMPACTION_BYTES_LIMIT_DELAYS, + PENDING_COMPACTION_BYTES_LIMIT_STOPS, + // Write slowdown caused by l0 file count limit while there is ongoing L0 + // compaction + L0_FILE_COUNT_LIMIT_DELAYS_WITH_ONGOING_COMPACTION, + // Write stop caused by l0 file count limit while there is ongoing L0 + // compaction + L0_FILE_COUNT_LIMIT_STOPS_WITH_ONGOING_COMPACTION, + WRITE_STALLS_ENUM_MAX, + // End of all write stall stats + BYTES_FLUSHED, + BYTES_INGESTED_ADD_FILE, + INGESTED_NUM_FILES_TOTAL, + INGESTED_LEVEL0_NUM_FILES_TOTAL, + INGESTED_NUM_KEYS_TOTAL, + INTERNAL_CF_STATS_ENUM_MAX, + }; + + enum InternalDBStatsType { + kIntStatsWalFileBytes, + kIntStatsWalFileSynced, + kIntStatsBytesWritten, + kIntStatsNumKeysWritten, + kIntStatsWriteDoneByOther, + kIntStatsWriteDoneBySelf, + kIntStatsWriteWithWal, + // TODO(hx235): Currently `kIntStatsWriteStallMicros` only measures + // "delayed" time of CF-scope write stalls, not including the "stopped" time + // nor any DB-scope write stalls (e.g, ones triggered by + // `WriteBufferManager`). + // + // However, the word "write stall" includes both "delayed" and "stopped" + // (see `WriteStallCondition`) and DB-scope writes stalls (see + // `WriteStallCause`). + // + // So we should improve, rename or clarify it + kIntStatsWriteStallMicros, + kIntStatsWriteBufferManagerLimitStopsCounts, + kIntStatsNumMax, + }; + + static const std::map db_stats_type_to_info; + + InternalStats(int num_levels, SystemClock* clock, ColumnFamilyData* cfd); + + // Per level compaction stats + struct CompactionOutputsStats { + uint64_t num_output_records = 0; + uint64_t bytes_written = 0; + uint64_t bytes_written_blob = 0; + uint64_t num_output_files = 0; + uint64_t num_output_files_blob = 0; + + void Add(const CompactionOutputsStats& stats) { + this->num_output_records += stats.num_output_records; + this->bytes_written += stats.bytes_written; + this->bytes_written_blob += stats.bytes_written_blob; + this->num_output_files += stats.num_output_files; + this->num_output_files_blob += stats.num_output_files_blob; + } + }; + + // Per level compaction stats. comp_stats_[level] stores the stats for + // compactions that produced data for the specified "level". + struct CompactionStats { + uint64_t micros; + uint64_t cpu_micros; + + // The number of bytes read from all non-output levels (table files) + uint64_t bytes_read_non_output_levels; + + // The number of bytes read from the compaction output level (table files) + uint64_t bytes_read_output_level; + + // The number of bytes read from blob files + uint64_t bytes_read_blob; + + // Total number of bytes written to table files during compaction + uint64_t bytes_written; + + // Total number of bytes written to blob files during compaction + uint64_t bytes_written_blob; + + // Total number of bytes moved to the output level (table files) + uint64_t bytes_moved; + + // The number of compaction input files in all non-output levels (table + // files) + int num_input_files_in_non_output_levels; + + // The number of compaction input files in the output level (table files) + int num_input_files_in_output_level; + + // The number of compaction output files (table files) + int num_output_files; + + // The number of compaction output files (blob files) + int num_output_files_blob; + + // Total incoming entries during compaction between levels N and N+1 + uint64_t num_input_records; + + // Accumulated diff number of entries + // (num input entries - num output entries) for compaction levels N and N+1 + uint64_t num_dropped_records; + + // Total output entries from compaction + uint64_t num_output_records; + + // Number of compactions done + int count; + + // Number of compactions done per CompactionReason + int counts[static_cast(CompactionReason::kNumOfReasons)]{}; + + explicit CompactionStats() + : micros(0), + cpu_micros(0), + bytes_read_non_output_levels(0), + bytes_read_output_level(0), + bytes_read_blob(0), + bytes_written(0), + bytes_written_blob(0), + bytes_moved(0), + num_input_files_in_non_output_levels(0), + num_input_files_in_output_level(0), + num_output_files(0), + num_output_files_blob(0), + num_input_records(0), + num_dropped_records(0), + num_output_records(0), + count(0) { + int num_of_reasons = static_cast(CompactionReason::kNumOfReasons); + for (int i = 0; i < num_of_reasons; i++) { + counts[i] = 0; + } + } + + explicit CompactionStats(CompactionReason reason, int c) + : micros(0), + cpu_micros(0), + bytes_read_non_output_levels(0), + bytes_read_output_level(0), + bytes_read_blob(0), + bytes_written(0), + bytes_written_blob(0), + bytes_moved(0), + num_input_files_in_non_output_levels(0), + num_input_files_in_output_level(0), + num_output_files(0), + num_output_files_blob(0), + num_input_records(0), + num_dropped_records(0), + num_output_records(0), + count(c) { + int num_of_reasons = static_cast(CompactionReason::kNumOfReasons); + for (int i = 0; i < num_of_reasons; i++) { + counts[i] = 0; + } + int r = static_cast(reason); + if (r >= 0 && r < num_of_reasons) { + counts[r] = c; + } else { + count = 0; + } + } + + CompactionStats(const CompactionStats& c) + : micros(c.micros), + cpu_micros(c.cpu_micros), + bytes_read_non_output_levels(c.bytes_read_non_output_levels), + bytes_read_output_level(c.bytes_read_output_level), + bytes_read_blob(c.bytes_read_blob), + bytes_written(c.bytes_written), + bytes_written_blob(c.bytes_written_blob), + bytes_moved(c.bytes_moved), + num_input_files_in_non_output_levels( + c.num_input_files_in_non_output_levels), + num_input_files_in_output_level(c.num_input_files_in_output_level), + num_output_files(c.num_output_files), + num_output_files_blob(c.num_output_files_blob), + num_input_records(c.num_input_records), + num_dropped_records(c.num_dropped_records), + num_output_records(c.num_output_records), + count(c.count) { + int num_of_reasons = static_cast(CompactionReason::kNumOfReasons); + for (int i = 0; i < num_of_reasons; i++) { + counts[i] = c.counts[i]; + } + } + + CompactionStats& operator=(const CompactionStats& c) { + micros = c.micros; + cpu_micros = c.cpu_micros; + bytes_read_non_output_levels = c.bytes_read_non_output_levels; + bytes_read_output_level = c.bytes_read_output_level; + bytes_read_blob = c.bytes_read_blob; + bytes_written = c.bytes_written; + bytes_written_blob = c.bytes_written_blob; + bytes_moved = c.bytes_moved; + num_input_files_in_non_output_levels = + c.num_input_files_in_non_output_levels; + num_input_files_in_output_level = c.num_input_files_in_output_level; + num_output_files = c.num_output_files; + num_output_files_blob = c.num_output_files_blob; + num_input_records = c.num_input_records; + num_dropped_records = c.num_dropped_records; + num_output_records = c.num_output_records; + count = c.count; + + int num_of_reasons = static_cast(CompactionReason::kNumOfReasons); + for (int i = 0; i < num_of_reasons; i++) { + counts[i] = c.counts[i]; + } + return *this; + } + + void Clear() { + this->micros = 0; + this->cpu_micros = 0; + this->bytes_read_non_output_levels = 0; + this->bytes_read_output_level = 0; + this->bytes_read_blob = 0; + this->bytes_written = 0; + this->bytes_written_blob = 0; + this->bytes_moved = 0; + this->num_input_files_in_non_output_levels = 0; + this->num_input_files_in_output_level = 0; + this->num_output_files = 0; + this->num_output_files_blob = 0; + this->num_input_records = 0; + this->num_dropped_records = 0; + this->num_output_records = 0; + this->count = 0; + int num_of_reasons = static_cast(CompactionReason::kNumOfReasons); + for (int i = 0; i < num_of_reasons; i++) { + counts[i] = 0; + } + } + + void Add(const CompactionStats& c) { + this->micros += c.micros; + this->cpu_micros += c.cpu_micros; + this->bytes_read_non_output_levels += c.bytes_read_non_output_levels; + this->bytes_read_output_level += c.bytes_read_output_level; + this->bytes_read_blob += c.bytes_read_blob; + this->bytes_written += c.bytes_written; + this->bytes_written_blob += c.bytes_written_blob; + this->bytes_moved += c.bytes_moved; + this->num_input_files_in_non_output_levels += + c.num_input_files_in_non_output_levels; + this->num_input_files_in_output_level += + c.num_input_files_in_output_level; + this->num_output_files += c.num_output_files; + this->num_output_files_blob += c.num_output_files_blob; + this->num_input_records += c.num_input_records; + this->num_dropped_records += c.num_dropped_records; + this->num_output_records += c.num_output_records; + this->count += c.count; + int num_of_reasons = static_cast(CompactionReason::kNumOfReasons); + for (int i = 0; i < num_of_reasons; i++) { + counts[i] += c.counts[i]; + } + } + + void Add(const CompactionOutputsStats& stats) { + this->num_output_files += static_cast(stats.num_output_files); + this->num_output_records += stats.num_output_records; + this->bytes_written += stats.bytes_written; + this->bytes_written_blob += stats.bytes_written_blob; + this->num_output_files_blob += + static_cast(stats.num_output_files_blob); + } + + void Subtract(const CompactionStats& c) { + this->micros -= c.micros; + this->cpu_micros -= c.cpu_micros; + this->bytes_read_non_output_levels -= c.bytes_read_non_output_levels; + this->bytes_read_output_level -= c.bytes_read_output_level; + this->bytes_read_blob -= c.bytes_read_blob; + this->bytes_written -= c.bytes_written; + this->bytes_written_blob -= c.bytes_written_blob; + this->bytes_moved -= c.bytes_moved; + this->num_input_files_in_non_output_levels -= + c.num_input_files_in_non_output_levels; + this->num_input_files_in_output_level -= + c.num_input_files_in_output_level; + this->num_output_files -= c.num_output_files; + this->num_output_files_blob -= c.num_output_files_blob; + this->num_input_records -= c.num_input_records; + this->num_dropped_records -= c.num_dropped_records; + this->num_output_records -= c.num_output_records; + this->count -= c.count; + int num_of_reasons = static_cast(CompactionReason::kNumOfReasons); + for (int i = 0; i < num_of_reasons; i++) { + counts[i] -= c.counts[i]; + } + } + + void ResetCompactionReason(CompactionReason reason) { + int num_of_reasons = static_cast(CompactionReason::kNumOfReasons); + assert(count == 1); // only support update one compaction reason + for (int i = 0; i < num_of_reasons; i++) { + counts[i] = 0; + } + int r = static_cast(reason); + assert(r >= 0 && r < num_of_reasons); + counts[r] = 1; + } + }; + + // Compaction stats, for per_key_placement compaction, it includes 2 levels + // stats: the last level and the penultimate level. + struct CompactionStatsFull { + // the stats for the target primary output level + CompactionStats stats; + + // stats for penultimate level output if exist + bool has_penultimate_level_output = false; + CompactionStats penultimate_level_stats; + + explicit CompactionStatsFull() : stats(), penultimate_level_stats() {} + + explicit CompactionStatsFull(CompactionReason reason, int c) + : stats(reason, c), penultimate_level_stats(reason, c){}; + + uint64_t TotalBytesWritten() const { + uint64_t bytes_written = stats.bytes_written + stats.bytes_written_blob; + if (has_penultimate_level_output) { + bytes_written += penultimate_level_stats.bytes_written + + penultimate_level_stats.bytes_written_blob; + } + return bytes_written; + } + + uint64_t DroppedRecords() { + uint64_t output_records = stats.num_output_records; + if (has_penultimate_level_output) { + output_records += penultimate_level_stats.num_output_records; + } + if (stats.num_input_records > output_records) { + return stats.num_input_records - output_records; + } + return 0; + } + + void SetMicros(uint64_t val) { + stats.micros = val; + penultimate_level_stats.micros = val; + } + + void AddCpuMicros(uint64_t val) { + stats.cpu_micros += val; + penultimate_level_stats.cpu_micros += val; + } + }; + + // For use with CacheEntryStatsCollector + struct CacheEntryRoleStats { + uint64_t cache_capacity = 0; + uint64_t cache_usage = 0; + size_t table_size = 0; + size_t occupancy = 0; + std::string cache_id; + std::array total_charges; + std::array entry_counts; + uint32_t collection_count = 0; + uint32_t copies_of_last_collection = 0; + uint64_t last_start_time_micros_ = 0; + uint64_t last_end_time_micros_ = 0; + uint32_t hash_seed = 0; + + void Clear() { + // Wipe everything except collection_count + uint32_t saved_collection_count = collection_count; + *this = CacheEntryRoleStats(); + collection_count = saved_collection_count; + } + + void BeginCollection(Cache*, SystemClock*, uint64_t start_time_micros); + std::function + GetEntryCallback(); + void EndCollection(Cache*, SystemClock*, uint64_t end_time_micros); + void SkippedCollection(); + + std::string ToString(SystemClock* clock) const; + void ToMap(std::map* values, + SystemClock* clock) const; + + private: + uint64_t GetLastDurationMicros() const; + }; + + void Clear() { + for (int i = 0; i < kIntStatsNumMax; i++) { + db_stats_[i].store(0); + } + for (int i = 0; i < INTERNAL_CF_STATS_ENUM_MAX; i++) { + cf_stats_count_[i] = 0; + cf_stats_value_[i] = 0; + } + for (auto& comp_stat : comp_stats_) { + comp_stat.Clear(); + } + per_key_placement_comp_stats_.Clear(); + for (auto& h : file_read_latency_) { + h.Clear(); + } + blob_file_read_latency_.Clear(); + cf_stats_snapshot_.Clear(); + db_stats_snapshot_.Clear(); + bg_error_count_ = 0; + started_at_ = clock_->NowMicros(); + has_cf_change_since_dump_ = true; + } + + void AddCompactionStats(int level, Env::Priority thread_pri, + const CompactionStats& stats) { + comp_stats_[level].Add(stats); + comp_stats_by_pri_[thread_pri].Add(stats); + } + + void AddCompactionStats(int level, Env::Priority thread_pri, + const CompactionStatsFull& comp_stats_full) { + AddCompactionStats(level, thread_pri, comp_stats_full.stats); + if (comp_stats_full.has_penultimate_level_output) { + per_key_placement_comp_stats_.Add( + comp_stats_full.penultimate_level_stats); + } + } + + void IncBytesMoved(int level, uint64_t amount) { + comp_stats_[level].bytes_moved += amount; + } + + void AddCFStats(InternalCFStatsType type, uint64_t value) { + has_cf_change_since_dump_ = true; + cf_stats_value_[type] += value; + ++cf_stats_count_[type]; + } + + void AddDBStats(InternalDBStatsType type, uint64_t value, + bool concurrent = false) { + auto& v = db_stats_[type]; + if (concurrent) { + v.fetch_add(value, std::memory_order_relaxed); + } else { + v.store(v.load(std::memory_order_relaxed) + value, + std::memory_order_relaxed); + } + } + + uint64_t GetDBStats(InternalDBStatsType type) { + return db_stats_[type].load(std::memory_order_relaxed); + } + + HistogramImpl* GetFileReadHist(int level) { + return &file_read_latency_[level]; + } + + HistogramImpl* GetBlobFileReadHist() { return &blob_file_read_latency_; } + + uint64_t GetBackgroundErrorCount() const { return bg_error_count_; } + + uint64_t BumpAndGetBackgroundErrorCount() { return ++bg_error_count_; } + + bool GetStringProperty(const DBPropertyInfo& property_info, + const Slice& property, std::string* value); + + bool GetMapProperty(const DBPropertyInfo& property_info, + const Slice& property, + std::map* value); + + bool GetIntProperty(const DBPropertyInfo& property_info, uint64_t* value, + DBImpl* db); + + bool GetIntPropertyOutOfMutex(const DBPropertyInfo& property_info, + Version* version, uint64_t* value); + + // Unless there is a recent enough collection of the stats, collect and + // saved new cache entry stats. If `foreground`, require data to be more + // recent to skip re-collection. + // + // This should only be called while NOT holding the DB mutex. + void CollectCacheEntryStats(bool foreground); + + const uint64_t* TEST_GetCFStatsValue() const { return cf_stats_value_; } + + const std::vector& TEST_GetCompactionStats() const { + return comp_stats_; + } + + const CompactionStats& TEST_GetPerKeyPlacementCompactionStats() const { + return per_key_placement_comp_stats_; + } + + void TEST_GetCacheEntryRoleStats(CacheEntryRoleStats* stats, bool foreground); + + // Store a mapping from the user-facing DB::Properties string to our + // DBPropertyInfo struct used internally for retrieving properties. + static const UnorderedMap ppt_name_to_info; + + static const std::string kPeriodicCFStats; + + private: + void DumpDBMapStats(std::map* db_stats); + void DumpDBStats(std::string* value); + + void DumpDBMapStatsWriteStall(std::map* value); + void DumpDBStatsWriteStall(std::string* value); + + void DumpCFMapStats(std::map* cf_stats); + void DumpCFMapStats( + const VersionStorageInfo* vstorage, + std::map>* level_stats, + CompactionStats* compaction_stats_sum); + void DumpCFMapStatsByPriority( + std::map>* priorities_stats); + void DumpCFStats(std::string* value); + // if is_periodic = true, it is an internal call by RocksDB periodically to + // dump the status. + void DumpCFStatsNoFileHistogram(bool is_periodic, std::string* value); + // if is_periodic = true, it is an internal call by RocksDB periodically to + // dump the status. + void DumpCFFileHistogram(std::string* value); + + void DumpCFMapStatsWriteStall(std::map* value); + void DumpCFStatsWriteStall(std::string* value, + uint64_t* total_stall_count = nullptr); + + Cache* GetBlockCacheForStats(); + Cache* GetBlobCacheForStats(); + + // Per-DB stats + std::atomic db_stats_[kIntStatsNumMax]; + // Per-ColumnFamily stats + uint64_t cf_stats_value_[INTERNAL_CF_STATS_ENUM_MAX]; + uint64_t cf_stats_count_[INTERNAL_CF_STATS_ENUM_MAX]; + // Initialize/reference the collector in constructor so that we don't need + // additional synchronization in InternalStats, relying on synchronization + // in CacheEntryStatsCollector::GetStats. This collector is pinned in cache + // (through a shared_ptr) so that it does not get immediately ejected from + // a full cache, which would force a re-scan on the next GetStats. + std::shared_ptr> + cache_entry_stats_collector_; + // Per-ColumnFamily/level compaction stats + std::vector comp_stats_; + std::vector comp_stats_by_pri_; + CompactionStats per_key_placement_comp_stats_; + std::vector file_read_latency_; + HistogramImpl blob_file_read_latency_; + bool has_cf_change_since_dump_; + // How many periods of no change since the last time stats are dumped for + // a periodic dump. + int no_cf_change_period_since_dump_ = 0; + uint64_t last_histogram_num = std::numeric_limits::max(); + static const int kMaxNoChangePeriodSinceDump; + + // Used to compute per-interval statistics + struct CFStatsSnapshot { + // ColumnFamily-level stats + CompactionStats comp_stats; + uint64_t ingest_bytes_flush; // Bytes written to L0 (Flush) + uint64_t stall_count; // Total counts of CF-scope write stalls + // Stats from compaction jobs - bytes written, bytes read, duration. + uint64_t compact_bytes_write; + uint64_t compact_bytes_read; + uint64_t compact_micros; + double seconds_up; + + // AddFile specific stats + uint64_t ingest_bytes_addfile; // Total Bytes ingested + uint64_t ingest_files_addfile; // Total number of files ingested + uint64_t ingest_l0_files_addfile; // Total number of files ingested to L0 + uint64_t ingest_keys_addfile; // Total number of keys ingested + + CFStatsSnapshot() + : ingest_bytes_flush(0), + stall_count(0), + compact_bytes_write(0), + compact_bytes_read(0), + compact_micros(0), + seconds_up(0), + ingest_bytes_addfile(0), + ingest_files_addfile(0), + ingest_l0_files_addfile(0), + ingest_keys_addfile(0) {} + + void Clear() { + comp_stats.Clear(); + ingest_bytes_flush = 0; + stall_count = 0; + compact_bytes_write = 0; + compact_bytes_read = 0; + compact_micros = 0; + seconds_up = 0; + ingest_bytes_addfile = 0; + ingest_files_addfile = 0; + ingest_l0_files_addfile = 0; + ingest_keys_addfile = 0; + } + } cf_stats_snapshot_; + + struct DBStatsSnapshot { + // DB-level stats + uint64_t ingest_bytes; // Bytes written by user + uint64_t wal_bytes; // Bytes written to WAL + uint64_t wal_synced; // Number of times WAL is synced + uint64_t write_with_wal; // Number of writes that request WAL + // These count the number of writes processed by the calling thread or + // another thread. + uint64_t write_other; + uint64_t write_self; + // Total number of keys written. write_self and write_other measure number + // of write requests written, Each of the write request can contain updates + // to multiple keys. num_keys_written is total number of keys updated by all + // those writes. + uint64_t num_keys_written; + // Total time writes delayed by stalls. + uint64_t write_stall_micros; + double seconds_up; + + DBStatsSnapshot() + : ingest_bytes(0), + wal_bytes(0), + wal_synced(0), + write_with_wal(0), + write_other(0), + write_self(0), + num_keys_written(0), + write_stall_micros(0), + seconds_up(0) {} + + void Clear() { + ingest_bytes = 0; + wal_bytes = 0; + wal_synced = 0; + write_with_wal = 0; + write_other = 0; + write_self = 0; + num_keys_written = 0; + write_stall_micros = 0; + seconds_up = 0; + } + } db_stats_snapshot_; + + // Handler functions for getting property values. They use "value" as a value- + // result argument, and return true upon successfully setting "value". + bool HandleNumFilesAtLevel(std::string* value, Slice suffix); + bool HandleCompressionRatioAtLevelPrefix(std::string* value, Slice suffix); + bool HandleLevelStats(std::string* value, Slice suffix); + bool HandleStats(std::string* value, Slice suffix); + bool HandleCFMapStats(std::map* compaction_stats, + Slice suffix); + bool HandleCFStats(std::string* value, Slice suffix); + bool HandleCFStatsNoFileHistogram(std::string* value, Slice suffix); + bool HandleCFFileHistogram(std::string* value, Slice suffix); + bool HandleCFStatsPeriodic(std::string* value, Slice suffix); + bool HandleCFWriteStallStats(std::string* value, Slice suffix); + bool HandleCFWriteStallStatsMap(std::map* values, + Slice suffix); + bool HandleDBMapStats(std::map* compaction_stats, + Slice suffix); + bool HandleDBStats(std::string* value, Slice suffix); + bool HandleDBWriteStallStats(std::string* value, Slice suffix); + bool HandleDBWriteStallStatsMap(std::map* values, + Slice suffix); + bool HandleSsTables(std::string* value, Slice suffix); + bool HandleAggregatedTableProperties(std::string* value, Slice suffix); + bool HandleAggregatedTablePropertiesAtLevel(std::string* value, Slice suffix); + bool HandleAggregatedTablePropertiesMap( + std::map* values, Slice suffix); + bool HandleAggregatedTablePropertiesAtLevelMap( + std::map* values, Slice suffix); + bool HandleNumImmutableMemTable(uint64_t* value, DBImpl* db, + Version* version); + bool HandleNumImmutableMemTableFlushed(uint64_t* value, DBImpl* db, + Version* version); + bool HandleMemTableFlushPending(uint64_t* value, DBImpl* db, + Version* version); + bool HandleNumRunningFlushes(uint64_t* value, DBImpl* db, Version* version); + bool HandleCompactionPending(uint64_t* value, DBImpl* db, Version* version); + bool HandleNumRunningCompactions(uint64_t* value, DBImpl* db, + Version* version); + bool HandleBackgroundErrors(uint64_t* value, DBImpl* db, Version* version); + bool HandleCurSizeActiveMemTable(uint64_t* value, DBImpl* db, + Version* version); + bool HandleCurSizeAllMemTables(uint64_t* value, DBImpl* db, Version* version); + bool HandleSizeAllMemTables(uint64_t* value, DBImpl* db, Version* version); + bool HandleNumEntriesActiveMemTable(uint64_t* value, DBImpl* db, + Version* version); + bool HandleNumEntriesImmMemTables(uint64_t* value, DBImpl* db, + Version* version); + bool HandleNumDeletesActiveMemTable(uint64_t* value, DBImpl* db, + Version* version); + bool HandleNumDeletesImmMemTables(uint64_t* value, DBImpl* db, + Version* version); + bool HandleEstimateNumKeys(uint64_t* value, DBImpl* db, Version* version); + bool HandleNumSnapshots(uint64_t* value, DBImpl* db, Version* version); + bool HandleOldestSnapshotTime(uint64_t* value, DBImpl* db, Version* version); + bool HandleOldestSnapshotSequence(uint64_t* value, DBImpl* db, + Version* version); + bool HandleNumLiveVersions(uint64_t* value, DBImpl* db, Version* version); + bool HandleCurrentSuperVersionNumber(uint64_t* value, DBImpl* db, + Version* version); + bool HandleIsFileDeletionsEnabled(uint64_t* value, DBImpl* db, + Version* version); + bool HandleBaseLevel(uint64_t* value, DBImpl* db, Version* version); + bool HandleTotalSstFilesSize(uint64_t* value, DBImpl* db, Version* version); + bool HandleLiveSstFilesSize(uint64_t* value, DBImpl* db, Version* version); + bool HandleObsoleteSstFilesSize(uint64_t* value, DBImpl* db, + Version* version); + bool HandleEstimatePendingCompactionBytes(uint64_t* value, DBImpl* db, + Version* version); + bool HandleEstimateTableReadersMem(uint64_t* value, DBImpl* db, + Version* version); + bool HandleEstimateLiveDataSize(uint64_t* value, DBImpl* db, + Version* version); + bool HandleMinLogNumberToKeep(uint64_t* value, DBImpl* db, Version* version); + bool HandleMinObsoleteSstNumberToKeep(uint64_t* value, DBImpl* db, + Version* version); + bool HandleActualDelayedWriteRate(uint64_t* value, DBImpl* db, + Version* version); + bool HandleIsWriteStopped(uint64_t* value, DBImpl* db, Version* version); + bool HandleEstimateOldestKeyTime(uint64_t* value, DBImpl* db, + Version* version); + bool HandleBlockCacheCapacity(uint64_t* value, DBImpl* db, Version* version); + bool HandleBlockCacheUsage(uint64_t* value, DBImpl* db, Version* version); + bool HandleBlockCachePinnedUsage(uint64_t* value, DBImpl* db, + Version* version); + bool HandleBlockCacheEntryStatsInternal(std::string* value, bool fast); + bool HandleBlockCacheEntryStatsMapInternal( + std::map* values, bool fast); + bool HandleBlockCacheEntryStats(std::string* value, Slice suffix); + bool HandleBlockCacheEntryStatsMap(std::map* values, + Slice suffix); + bool HandleFastBlockCacheEntryStats(std::string* value, Slice suffix); + bool HandleFastBlockCacheEntryStatsMap( + std::map* values, Slice suffix); + bool HandleLiveSstFilesSizeAtTemperature(std::string* value, Slice suffix); + bool HandleNumBlobFiles(uint64_t* value, DBImpl* db, Version* version); + bool HandleBlobStats(std::string* value, Slice suffix); + bool HandleTotalBlobFileSize(uint64_t* value, DBImpl* db, Version* version); + bool HandleLiveBlobFileSize(uint64_t* value, DBImpl* db, Version* version); + bool HandleLiveBlobFileGarbageSize(uint64_t* value, DBImpl* db, + Version* version); + bool HandleBlobCacheCapacity(uint64_t* value, DBImpl* db, Version* version); + bool HandleBlobCacheUsage(uint64_t* value, DBImpl* db, Version* version); + bool HandleBlobCachePinnedUsage(uint64_t* value, DBImpl* db, + Version* version); + + // Total number of background errors encountered. Every time a flush task + // or compaction task fails, this counter is incremented. The failure can + // be caused by any possible reason, including file system errors, out of + // resources, or input file corruption. Failing when retrying the same flush + // or compaction will cause the counter to increase too. + uint64_t bg_error_count_; + + const int number_levels_; + SystemClock* clock_; + ColumnFamilyData* cfd_; + uint64_t started_at_; +}; + + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/job_context.h b/librocksdb-sys/rocksdb/db/job_context.h new file mode 100644 index 0000000..a550ba2 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/job_context.h @@ -0,0 +1,237 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include +#include + +#include "db/column_family.h" +#include "db/log_writer.h" +#include "db/version_set.h" + +namespace ROCKSDB_NAMESPACE { + +class MemTable; +struct SuperVersion; + +struct SuperVersionContext { + struct WriteStallNotification { + WriteStallInfo write_stall_info; + const ImmutableOptions* immutable_options; + }; + + autovector superversions_to_free; +#ifndef ROCKSDB_DISABLE_STALL_NOTIFICATION + autovector write_stall_notifications; +#endif + std::unique_ptr + new_superversion; // if nullptr no new superversion + + explicit SuperVersionContext(bool create_superversion = false) + : new_superversion(create_superversion ? new SuperVersion() : nullptr) {} + + explicit SuperVersionContext(SuperVersionContext&& other) noexcept + : superversions_to_free(std::move(other.superversions_to_free)), +#ifndef ROCKSDB_DISABLE_STALL_NOTIFICATION + write_stall_notifications(std::move(other.write_stall_notifications)), +#endif + new_superversion(std::move(other.new_superversion)) { + } + // No copies + SuperVersionContext(const SuperVersionContext& other) = delete; + void operator=(const SuperVersionContext& other) = delete; + + void NewSuperVersion() { + new_superversion = std::unique_ptr(new SuperVersion()); + } + + inline bool HaveSomethingToDelete() const { +#ifndef ROCKSDB_DISABLE_STALL_NOTIFICATION + return !superversions_to_free.empty() || !write_stall_notifications.empty(); +#else + return !superversions_to_free.empty(); +#endif + } + + void PushWriteStallNotification(WriteStallCondition old_cond, + WriteStallCondition new_cond, + const std::string& name, + const ImmutableOptions* ioptions) { +#if !defined(ROCKSDB_DISABLE_STALL_NOTIFICATION) + WriteStallNotification notif; + notif.write_stall_info.cf_name = name; + notif.write_stall_info.condition.prev = old_cond; + notif.write_stall_info.condition.cur = new_cond; + notif.immutable_options = ioptions; + write_stall_notifications.push_back(notif); +#else + (void)old_cond; + (void)new_cond; + (void)name; + (void)ioptions; +#endif // !defined(ROCKSDB_DISABLE_STALL_NOTIFICATION) + } + + void Clean() { +#if !defined(ROCKSDB_DISABLE_STALL_NOTIFICATION) + // notify listeners on changed write stall conditions + for (auto& notif : write_stall_notifications) { + for (auto& listener : notif.immutable_options->listeners) { + listener->OnStallConditionsChanged(notif.write_stall_info); + } + } + write_stall_notifications.clear(); +#endif + // free superversions + for (auto s : superversions_to_free) { + delete s; + } + superversions_to_free.clear(); + } + + ~SuperVersionContext() { +#ifndef ROCKSDB_DISABLE_STALL_NOTIFICATION + assert(write_stall_notifications.empty()); +#endif + assert(superversions_to_free.empty()); + } +}; + +struct JobContext { + inline bool HaveSomethingToDelete() const { + return !(full_scan_candidate_files.empty() && sst_delete_files.empty() && + blob_delete_files.empty() && log_delete_files.empty() && + manifest_delete_files.empty()); + } + + inline bool HaveSomethingToClean() const { + bool sv_have_sth = false; + for (const auto& sv_ctx : superversion_contexts) { + if (sv_ctx.HaveSomethingToDelete()) { + sv_have_sth = true; + break; + } + } + return memtables_to_free.size() > 0 || logs_to_free.size() > 0 || + job_snapshot != nullptr || sv_have_sth; + } + + SequenceNumber GetJobSnapshotSequence() const { + if (job_snapshot) { + assert(job_snapshot->snapshot()); + return job_snapshot->snapshot()->GetSequenceNumber(); + } + return kMaxSequenceNumber; + } + + // Structure to store information for candidate files to delete. + struct CandidateFileInfo { + std::string file_name; + std::string file_path; + CandidateFileInfo(std::string name, std::string path) + : file_name(std::move(name)), file_path(std::move(path)) {} + bool operator==(const CandidateFileInfo& other) const { + return file_name == other.file_name && file_path == other.file_path; + } + }; + + // Unique job id + int job_id; + + // a list of all files that we'll consider deleting + // (every once in a while this is filled up with all files + // in the DB directory) + // (filled only if we're doing full scan) + std::vector full_scan_candidate_files; + + // the list of all live sst files that cannot be deleted + std::vector sst_live; + + // the list of sst files that we need to delete + std::vector sst_delete_files; + + // the list of all live blob files that cannot be deleted + std::vector blob_live; + + // the list of blob files that we need to delete + std::vector blob_delete_files; + + // a list of log files that we need to delete + std::vector log_delete_files; + + // a list of log files that we need to preserve during full purge since they + // will be reused later + std::vector log_recycle_files; + + // a list of manifest files that we need to delete + std::vector manifest_delete_files; + + // a list of memtables to be free + autovector memtables_to_free; + + // contexts for installing superversions for multiple column families + std::vector superversion_contexts; + + autovector logs_to_free; + + // the current manifest_file_number, log_number and prev_log_number + // that corresponds to the set of files in 'live'. + uint64_t manifest_file_number; + uint64_t pending_manifest_file_number; + uint64_t log_number; + uint64_t prev_log_number; + + uint64_t min_pending_output = 0; + uint64_t prev_total_log_size = 0; + size_t num_alive_log_files = 0; + uint64_t size_log_to_delete = 0; + + // Snapshot taken before flush/compaction job. + std::unique_ptr job_snapshot; + + explicit JobContext(int _job_id, bool create_superversion = false) { + job_id = _job_id; + manifest_file_number = 0; + pending_manifest_file_number = 0; + log_number = 0; + prev_log_number = 0; + superversion_contexts.emplace_back( + SuperVersionContext(create_superversion)); + } + + // For non-empty JobContext Clean() has to be called at least once before + // before destruction (see asserts in ~JobContext()). Should be called with + // unlocked DB mutex. Destructor doesn't call Clean() to avoid accidentally + // doing potentially slow Clean() with locked DB mutex. + void Clean() { + // free superversions + for (auto& sv_context : superversion_contexts) { + sv_context.Clean(); + } + // free pending memtables + for (auto m : memtables_to_free) { + delete m; + } + for (auto l : logs_to_free) { + delete l; + } + + memtables_to_free.clear(); + logs_to_free.clear(); + job_snapshot.reset(); + } + + ~JobContext() { + assert(memtables_to_free.size() == 0); + assert(logs_to_free.size() == 0); + } +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/kv_checksum.h b/librocksdb-sys/rocksdb/db/kv_checksum.h new file mode 100644 index 0000000..53c0248 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/kv_checksum.h @@ -0,0 +1,484 @@ +// Copyright (c) 2020-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file contains classes containing fields to protect individual entries. +// The classes are named "ProtectionInfo", where indicates the +// combination of fields that are covered. Each field has a single letter +// abbreviation as follows. +// +// K = key +// V = value +// O = optype aka value type +// S = seqno +// C = CF ID +// +// Then, for example, a class that protects an entry consisting of key, value, +// optype, and CF ID (i.e., a `WriteBatch` entry) would be named +// `ProtectionInfoKVOC`. +// +// The `ProtectionInfo.*` classes are templated on the integer type used to hold +// the XOR of hashes for each field. Only unsigned integer types are supported, +// and the maximum supported integer width is 64 bits. When the integer type is +// narrower than the hash values, we lop off the most significant bits to make +// them fit. +// +// The `ProtectionInfo.*` classes are all intended to be non-persistent. We do +// not currently make the byte order consistent for integer fields before +// hashing them, so the resulting values are endianness-dependent. + +#pragma once + +#include + +#include "db/dbformat.h" +#include "rocksdb/types.h" +#include "util/hash.h" + +namespace ROCKSDB_NAMESPACE { + +template +class ProtectionInfo; +template +class ProtectionInfoKVO; +template +class ProtectionInfoKVOC; +template +class ProtectionInfoKVOS; +template +class ProtectionInfoKV; + +// Aliases for 64-bit protection infos. +using ProtectionInfo64 = ProtectionInfo; +using ProtectionInfoKVO64 = ProtectionInfoKVO; +using ProtectionInfoKVOC64 = ProtectionInfoKVOC; +using ProtectionInfoKVOS64 = ProtectionInfoKVOS; + +template +class ProtectionInfo { + public: + ProtectionInfo() = default; + + Status GetStatus() const; + ProtectionInfoKVO ProtectKVO(const Slice& key, const Slice& value, + ValueType op_type) const; + ProtectionInfoKVO ProtectKVO(const SliceParts& key, + const SliceParts& value, + ValueType op_type) const; + ProtectionInfoKV ProtectKV(const Slice& key, const Slice& value) const; + + private: + friend class ProtectionInfoKVO; + friend class ProtectionInfoKVOS; + friend class ProtectionInfoKVOC; + friend class ProtectionInfoKV; + + // Each field is hashed with an independent value so we can catch fields being + // swapped. Per the `NPHash64()` docs, using consecutive seeds is a pitfall, + // and we should instead vary our seeds by a large odd number. This value by + // which we increment (0xD28AAD72F49BD50B) was taken from + // `head -c8 /dev/urandom | hexdump`, run repeatedly until it yielded an odd + // number. The values are computed manually since the Windows C++ compiler + // complains about the overflow when adding constants. + static const uint64_t kSeedK = 0; + static const uint64_t kSeedV = 0xD28AAD72F49BD50B; + static const uint64_t kSeedO = 0xA5155AE5E937AA16; + static const uint64_t kSeedS = 0x77A00858DDD37F21; + static const uint64_t kSeedC = 0x4A2AB5CBD26F542C; + + ProtectionInfo(T val) : val_(val) { + static_assert(sizeof(ProtectionInfo) == sizeof(T), ""); + } + + T GetVal() const { return val_; } + void SetVal(T val) { val_ = val; } + + void Encode(uint8_t len, char* dst) const { + assert(sizeof(val_) >= len); + switch (len) { + case 1: + dst[0] = static_cast(val_); + break; + case 2: + EncodeFixed16(dst, static_cast(val_)); + break; + case 4: + EncodeFixed32(dst, static_cast(val_)); + break; + case 8: + EncodeFixed64(dst, static_cast(val_)); + break; + default: + assert(false); + } + } + + bool Verify(uint8_t len, const char* checksum_ptr) const { + assert(sizeof(val_) >= len); + switch (len) { + case 1: + return static_cast(checksum_ptr[0]) == + static_cast(val_); + case 2: + return DecodeFixed16(checksum_ptr) == static_cast(val_); + case 4: + return DecodeFixed32(checksum_ptr) == static_cast(val_); + case 8: + return DecodeFixed64(checksum_ptr) == static_cast(val_); + default: + assert(false); + return false; + } + } + + T val_ = 0; +}; + +template +class ProtectionInfoKVO { + public: + ProtectionInfoKVO() = default; + + ProtectionInfo StripKVO(const Slice& key, const Slice& value, + ValueType op_type) const; + ProtectionInfo StripKVO(const SliceParts& key, const SliceParts& value, + ValueType op_type) const; + + ProtectionInfoKVOC ProtectC(ColumnFamilyId column_family_id) const; + ProtectionInfoKVOS ProtectS(SequenceNumber sequence_number) const; + + void UpdateK(const Slice& old_key, const Slice& new_key); + void UpdateK(const SliceParts& old_key, const SliceParts& new_key); + void UpdateV(const Slice& old_value, const Slice& new_value); + void UpdateV(const SliceParts& old_value, const SliceParts& new_value); + void UpdateO(ValueType old_op_type, ValueType new_op_type); + + // Encode this protection info into `len` bytes and stores them in `dst`. + void Encode(uint8_t len, char* dst) const { info_.Encode(len, dst); } + // Verify this protection info against the protection info encoded by Encode() + // at the first `len` bytes of `checksum_ptr`. + // Returns true iff the verification is successful. + bool Verify(uint8_t len, const char* checksum_ptr) const { + return info_.Verify(len, checksum_ptr); + } + + private: + friend class ProtectionInfo; + friend class ProtectionInfoKVOS; + friend class ProtectionInfoKVOC; + + explicit ProtectionInfoKVO(T val) : info_(val) { + static_assert(sizeof(ProtectionInfoKVO) == sizeof(T), ""); + } + + T GetVal() const { return info_.GetVal(); } + void SetVal(T val) { info_.SetVal(val); } + + ProtectionInfo info_; +}; + +template +class ProtectionInfoKVOC { + public: + ProtectionInfoKVOC() = default; + + ProtectionInfoKVO StripC(ColumnFamilyId column_family_id) const; + + void UpdateK(const Slice& old_key, const Slice& new_key) { + kvo_.UpdateK(old_key, new_key); + } + void UpdateK(const SliceParts& old_key, const SliceParts& new_key) { + kvo_.UpdateK(old_key, new_key); + } + void UpdateV(const Slice& old_value, const Slice& new_value) { + kvo_.UpdateV(old_value, new_value); + } + void UpdateV(const SliceParts& old_value, const SliceParts& new_value) { + kvo_.UpdateV(old_value, new_value); + } + void UpdateO(ValueType old_op_type, ValueType new_op_type) { + kvo_.UpdateO(old_op_type, new_op_type); + } + void UpdateC(ColumnFamilyId old_column_family_id, + ColumnFamilyId new_column_family_id); + + void Encode(uint8_t len, char* dst) const { kvo_.Encode(len, dst); } + bool Verify(uint8_t len, const char* checksum_ptr) const { + return kvo_.Verify(len, checksum_ptr); + } + + private: + friend class ProtectionInfoKVO; + + explicit ProtectionInfoKVOC(T val) : kvo_(val) { + static_assert(sizeof(ProtectionInfoKVOC) == sizeof(T), ""); + } + + T GetVal() const { return kvo_.GetVal(); } + void SetVal(T val) { kvo_.SetVal(val); } + + ProtectionInfoKVO kvo_; +}; + +template +class ProtectionInfoKVOS { + public: + ProtectionInfoKVOS() = default; + + ProtectionInfoKVO StripS(SequenceNumber sequence_number) const; + + void UpdateK(const Slice& old_key, const Slice& new_key) { + kvo_.UpdateK(old_key, new_key); + } + void UpdateK(const SliceParts& old_key, const SliceParts& new_key) { + kvo_.UpdateK(old_key, new_key); + } + void UpdateV(const Slice& old_value, const Slice& new_value) { + kvo_.UpdateV(old_value, new_value); + } + void UpdateV(const SliceParts& old_value, const SliceParts& new_value) { + kvo_.UpdateV(old_value, new_value); + } + void UpdateO(ValueType old_op_type, ValueType new_op_type) { + kvo_.UpdateO(old_op_type, new_op_type); + } + void UpdateS(SequenceNumber old_sequence_number, + SequenceNumber new_sequence_number); + + void Encode(uint8_t len, char* dst) const { kvo_.Encode(len, dst); } + bool Verify(uint8_t len, const char* checksum_ptr) const { + return kvo_.Verify(len, checksum_ptr); + } + + private: + friend class ProtectionInfoKVO; + + explicit ProtectionInfoKVOS(T val) : kvo_(val) { + static_assert(sizeof(ProtectionInfoKVOS) == sizeof(T), ""); + } + + T GetVal() const { return kvo_.GetVal(); } + void SetVal(T val) { kvo_.SetVal(val); } + + ProtectionInfoKVO kvo_; +}; + +template +class ProtectionInfoKV { + public: + ProtectionInfoKV() = default; + + void Encode(uint8_t len, char* dst) const { info_.Encode(len, dst); } + bool Verify(uint8_t len, const char* checksum_ptr) const { + return info_.Verify(len, checksum_ptr); + } + + private: + friend class ProtectionInfo; + + explicit ProtectionInfoKV(T val) : info_(val) { + static_assert(sizeof(ProtectionInfoKV) == sizeof(T)); + } + + ProtectionInfo info_; +}; + +template +Status ProtectionInfo::GetStatus() const { + if (val_ != 0) { + return Status::Corruption("ProtectionInfo mismatch"); + } + return Status::OK(); +} + +template +ProtectionInfoKVO ProtectionInfo::ProtectKVO(const Slice& key, + const Slice& value, + ValueType op_type) const { + T val = GetVal(); + val = val ^ static_cast(GetSliceNPHash64(key, ProtectionInfo::kSeedK)); + val = + val ^ static_cast(GetSliceNPHash64(value, ProtectionInfo::kSeedV)); + val = val ^ + static_cast(NPHash64(reinterpret_cast(&op_type), + sizeof(op_type), ProtectionInfo::kSeedO)); + return ProtectionInfoKVO(val); +} + +template +ProtectionInfoKVO ProtectionInfo::ProtectKVO(const SliceParts& key, + const SliceParts& value, + ValueType op_type) const { + T val = GetVal(); + val = val ^ + static_cast(GetSlicePartsNPHash64(key, ProtectionInfo::kSeedK)); + val = val ^ + static_cast(GetSlicePartsNPHash64(value, ProtectionInfo::kSeedV)); + val = val ^ + static_cast(NPHash64(reinterpret_cast(&op_type), + sizeof(op_type), ProtectionInfo::kSeedO)); + return ProtectionInfoKVO(val); +} + +template +ProtectionInfoKV ProtectionInfo::ProtectKV(const Slice& key, + const Slice& value) const { + T val = GetVal(); + val = val ^ static_cast(GetSliceNPHash64(key, ProtectionInfo::kSeedK)); + val = + val ^ static_cast(GetSliceNPHash64(value, ProtectionInfo::kSeedV)); + return ProtectionInfoKV(val); +} + +template +void ProtectionInfoKVO::UpdateK(const Slice& old_key, const Slice& new_key) { + T val = GetVal(); + val = val ^ + static_cast(GetSliceNPHash64(old_key, ProtectionInfo::kSeedK)); + val = val ^ + static_cast(GetSliceNPHash64(new_key, ProtectionInfo::kSeedK)); + SetVal(val); +} + +template +void ProtectionInfoKVO::UpdateK(const SliceParts& old_key, + const SliceParts& new_key) { + T val = GetVal(); + val = val ^ static_cast( + GetSlicePartsNPHash64(old_key, ProtectionInfo::kSeedK)); + val = val ^ static_cast( + GetSlicePartsNPHash64(new_key, ProtectionInfo::kSeedK)); + SetVal(val); +} + +template +void ProtectionInfoKVO::UpdateV(const Slice& old_value, + const Slice& new_value) { + T val = GetVal(); + val = val ^ + static_cast(GetSliceNPHash64(old_value, ProtectionInfo::kSeedV)); + val = val ^ + static_cast(GetSliceNPHash64(new_value, ProtectionInfo::kSeedV)); + SetVal(val); +} + +template +void ProtectionInfoKVO::UpdateV(const SliceParts& old_value, + const SliceParts& new_value) { + T val = GetVal(); + val = val ^ static_cast( + GetSlicePartsNPHash64(old_value, ProtectionInfo::kSeedV)); + val = val ^ static_cast( + GetSlicePartsNPHash64(new_value, ProtectionInfo::kSeedV)); + SetVal(val); +} + +template +void ProtectionInfoKVO::UpdateO(ValueType old_op_type, + ValueType new_op_type) { + T val = GetVal(); + val = val ^ static_cast(NPHash64(reinterpret_cast(&old_op_type), + sizeof(old_op_type), + ProtectionInfo::kSeedO)); + val = val ^ static_cast(NPHash64(reinterpret_cast(&new_op_type), + sizeof(new_op_type), + ProtectionInfo::kSeedO)); + SetVal(val); +} + +template +ProtectionInfo ProtectionInfoKVO::StripKVO(const Slice& key, + const Slice& value, + ValueType op_type) const { + T val = GetVal(); + val = val ^ static_cast(GetSliceNPHash64(key, ProtectionInfo::kSeedK)); + val = + val ^ static_cast(GetSliceNPHash64(value, ProtectionInfo::kSeedV)); + val = val ^ + static_cast(NPHash64(reinterpret_cast(&op_type), + sizeof(op_type), ProtectionInfo::kSeedO)); + return ProtectionInfo(val); +} + +template +ProtectionInfo ProtectionInfoKVO::StripKVO(const SliceParts& key, + const SliceParts& value, + ValueType op_type) const { + T val = GetVal(); + val = val ^ + static_cast(GetSlicePartsNPHash64(key, ProtectionInfo::kSeedK)); + val = val ^ + static_cast(GetSlicePartsNPHash64(value, ProtectionInfo::kSeedV)); + val = val ^ + static_cast(NPHash64(reinterpret_cast(&op_type), + sizeof(op_type), ProtectionInfo::kSeedO)); + return ProtectionInfo(val); +} + +template +ProtectionInfoKVOC ProtectionInfoKVO::ProtectC( + ColumnFamilyId column_family_id) const { + T val = GetVal(); + val = val ^ static_cast(NPHash64( + reinterpret_cast(&column_family_id), + sizeof(column_family_id), ProtectionInfo::kSeedC)); + return ProtectionInfoKVOC(val); +} + +template +ProtectionInfoKVO ProtectionInfoKVOC::StripC( + ColumnFamilyId column_family_id) const { + T val = GetVal(); + val = val ^ static_cast(NPHash64( + reinterpret_cast(&column_family_id), + sizeof(column_family_id), ProtectionInfo::kSeedC)); + return ProtectionInfoKVO(val); +} + +template +void ProtectionInfoKVOC::UpdateC(ColumnFamilyId old_column_family_id, + ColumnFamilyId new_column_family_id) { + T val = GetVal(); + val = val ^ static_cast(NPHash64( + reinterpret_cast(&old_column_family_id), + sizeof(old_column_family_id), ProtectionInfo::kSeedC)); + val = val ^ static_cast(NPHash64( + reinterpret_cast(&new_column_family_id), + sizeof(new_column_family_id), ProtectionInfo::kSeedC)); + SetVal(val); +} + +template +ProtectionInfoKVOS ProtectionInfoKVO::ProtectS( + SequenceNumber sequence_number) const { + T val = GetVal(); + val = val ^ static_cast(NPHash64(reinterpret_cast(&sequence_number), + sizeof(sequence_number), + ProtectionInfo::kSeedS)); + return ProtectionInfoKVOS(val); +} + +template +ProtectionInfoKVO ProtectionInfoKVOS::StripS( + SequenceNumber sequence_number) const { + T val = GetVal(); + val = val ^ static_cast(NPHash64(reinterpret_cast(&sequence_number), + sizeof(sequence_number), + ProtectionInfo::kSeedS)); + return ProtectionInfoKVO(val); +} + +template +void ProtectionInfoKVOS::UpdateS(SequenceNumber old_sequence_number, + SequenceNumber new_sequence_number) { + T val = GetVal(); + val = val ^ static_cast(NPHash64( + reinterpret_cast(&old_sequence_number), + sizeof(old_sequence_number), ProtectionInfo::kSeedS)); + val = val ^ static_cast(NPHash64( + reinterpret_cast(&new_sequence_number), + sizeof(new_sequence_number), ProtectionInfo::kSeedS)); + SetVal(val); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/listener_test.cc b/librocksdb-sys/rocksdb/db/listener_test.cc new file mode 100644 index 0000000..206dba9 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/listener_test.cc @@ -0,0 +1,1599 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include + +#include "db/blob/blob_index.h" +#include "db/db_impl/db_impl.h" +#include "db/db_test_util.h" +#include "db/dbformat.h" +#include "db/version_set.h" +#include "db/write_batch_internal.h" +#include "file/filename.h" +#include "monitoring/statistics_impl.h" +#include "rocksdb/cache.h" +#include "rocksdb/compaction_filter.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/options.h" +#include "rocksdb/perf_context.h" +#include "rocksdb/slice.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/table.h" +#include "rocksdb/table_properties.h" +#include "test_util/sync_point.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/hash.h" +#include "util/mutexlock.h" +#include "util/rate_limiter_impl.h" +#include "util/string_util.h" +#include "utilities/merge_operators.h" + + +namespace ROCKSDB_NAMESPACE { + +class EventListenerTest : public DBTestBase { + public: + EventListenerTest() : DBTestBase("listener_test", /*env_do_fsync=*/true) {} + + static std::string BlobStr(uint64_t blob_file_number, uint64_t offset, + uint64_t size) { + std::string blob_index; + BlobIndex::EncodeBlob(&blob_index, blob_file_number, offset, size, + kNoCompression); + return blob_index; + } + + const size_t k110KB = 110 << 10; +}; + +struct TestPropertiesCollector + : public ROCKSDB_NAMESPACE::TablePropertiesCollector { + ROCKSDB_NAMESPACE::Status AddUserKey( + const ROCKSDB_NAMESPACE::Slice& /*key*/, + const ROCKSDB_NAMESPACE::Slice& /*value*/, + ROCKSDB_NAMESPACE::EntryType /*type*/, + ROCKSDB_NAMESPACE::SequenceNumber /*seq*/, + uint64_t /*file_size*/) override { + return Status::OK(); + } + ROCKSDB_NAMESPACE::Status Finish( + ROCKSDB_NAMESPACE::UserCollectedProperties* properties) override { + properties->insert({"0", "1"}); + return Status::OK(); + } + + const char* Name() const override { return "TestTablePropertiesCollector"; } + + ROCKSDB_NAMESPACE::UserCollectedProperties GetReadableProperties() + const override { + ROCKSDB_NAMESPACE::UserCollectedProperties ret; + ret["2"] = "3"; + return ret; + } +}; + +class TestPropertiesCollectorFactory : public TablePropertiesCollectorFactory { + public: + TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context /*context*/) override { + return new TestPropertiesCollector; + } + const char* Name() const override { return "TestTablePropertiesCollector"; } +}; + +class TestCompactionListener : public EventListener { + public: + explicit TestCompactionListener(EventListenerTest* test) : test_(test) {} + + void OnCompactionCompleted(DB* db, const CompactionJobInfo& ci) override { + std::lock_guard lock(mutex_); + compacted_dbs_.push_back(db); + ASSERT_GT(ci.input_files.size(), 0U); + ASSERT_EQ(ci.input_files.size(), ci.input_file_infos.size()); + + for (size_t i = 0; i < ci.input_file_infos.size(); ++i) { + ASSERT_EQ(ci.input_file_infos[i].level, ci.base_input_level); + ASSERT_EQ(ci.input_file_infos[i].file_number, + TableFileNameToNumber(ci.input_files[i])); + } + + ASSERT_GT(ci.output_files.size(), 0U); + ASSERT_EQ(ci.output_files.size(), ci.output_file_infos.size()); + + ASSERT_TRUE(test_); + ASSERT_EQ(test_->db_, db); + + std::vector> files_by_level; + test_->dbfull()->TEST_GetFilesMetaData(test_->handles_[ci.cf_id], + &files_by_level); + ASSERT_GT(files_by_level.size(), ci.output_level); + + for (size_t i = 0; i < ci.output_file_infos.size(); ++i) { + ASSERT_EQ(ci.output_file_infos[i].level, ci.output_level); + ASSERT_EQ(ci.output_file_infos[i].file_number, + TableFileNameToNumber(ci.output_files[i])); + + auto it = std::find_if( + files_by_level[ci.output_level].begin(), + files_by_level[ci.output_level].end(), [&](const FileMetaData& meta) { + return meta.fd.GetNumber() == ci.output_file_infos[i].file_number; + }); + ASSERT_NE(it, files_by_level[ci.output_level].end()); + + ASSERT_EQ(ci.output_file_infos[i].oldest_blob_file_number, + it->oldest_blob_file_number); + } + + ASSERT_EQ(db->GetEnv()->GetThreadID(), ci.thread_id); + ASSERT_GT(ci.thread_id, 0U); + + for (auto fl : {ci.input_files, ci.output_files}) { + for (auto fn : fl) { + auto it = ci.table_properties.find(fn); + ASSERT_NE(it, ci.table_properties.end()); + auto tp = it->second; + ASSERT_TRUE(tp != nullptr); + ASSERT_EQ(tp->user_collected_properties.find("0")->second, "1"); + } + } + } + + EventListenerTest* test_; + std::vector compacted_dbs_; + std::mutex mutex_; +}; + +TEST_F(EventListenerTest, OnSingleDBCompactionTest) { + const int kTestKeySize = 16; + const int kTestValueSize = 984; + const int kEntrySize = kTestKeySize + kTestValueSize; + const int kEntriesPerBuffer = 100; + const int kNumL0Files = 4; + + Options options; + options.env = CurrentOptions().env; + options.create_if_missing = true; + options.write_buffer_size = kEntrySize * kEntriesPerBuffer; + options.compaction_style = kCompactionStyleLevel; + options.target_file_size_base = options.write_buffer_size; + options.max_bytes_for_level_base = options.target_file_size_base * 2; + options.max_bytes_for_level_multiplier = 2; + options.compression = kNoCompression; +#ifdef ROCKSDB_USING_THREAD_STATUS + options.enable_thread_tracking = true; +#endif // ROCKSDB_USING_THREAD_STATUS + options.level0_file_num_compaction_trigger = kNumL0Files; + options.table_properties_collector_factories.push_back( + std::make_shared()); + + TestCompactionListener* listener = new TestCompactionListener(this); + options.listeners.emplace_back(listener); + std::vector cf_names = {"pikachu", "ilya", "muromec", + "dobrynia", "nikitich", "alyosha", + "popovich"}; + CreateAndReopenWithCF(cf_names, options); + ASSERT_OK(Put(1, "pikachu", std::string(90000, 'p'))); + + WriteBatch batch; + ASSERT_OK(WriteBatchInternal::PutBlobIndex(&batch, 1, "ditto", + BlobStr(123, 0, 1 << 10))); + ASSERT_OK(dbfull()->Write(WriteOptions(), &batch)); + + ASSERT_OK(Put(2, "ilya", std::string(90000, 'i'))); + ASSERT_OK(Put(3, "muromec", std::string(90000, 'm'))); + ASSERT_OK(Put(4, "dobrynia", std::string(90000, 'd'))); + ASSERT_OK(Put(5, "nikitich", std::string(90000, 'n'))); + ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a'))); + ASSERT_OK(Put(7, "popovich", std::string(90000, 'p'))); + for (int i = 1; i < 8; ++i) { + ASSERT_OK(Flush(i)); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), handles_[i], + nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + + ASSERT_EQ(listener->compacted_dbs_.size(), cf_names.size()); + for (size_t i = 0; i < cf_names.size(); ++i) { + ASSERT_EQ(listener->compacted_dbs_[i], db_); + } +} + +// This simple Listener can only handle one flush at a time. +class TestFlushListener : public EventListener { + public: + TestFlushListener(Env* env, EventListenerTest* test) + : slowdown_count(0), stop_count(0), db_closed(), env_(env), test_(test) { + db_closed = false; + } + + virtual ~TestFlushListener() { + prev_fc_info_.status.PermitUncheckedError(); // Ignore the status + } + void OnTableFileCreated(const TableFileCreationInfo& info) override { + // remember the info for later checking the FlushJobInfo. + prev_fc_info_ = info; + ASSERT_GT(info.db_name.size(), 0U); + ASSERT_GT(info.cf_name.size(), 0U); + ASSERT_GT(info.file_path.size(), 0U); + ASSERT_GT(info.job_id, 0); + ASSERT_GT(info.table_properties.data_size, 0U); + ASSERT_GT(info.table_properties.raw_key_size, 0U); + ASSERT_GT(info.table_properties.raw_value_size, 0U); + ASSERT_GT(info.table_properties.num_data_blocks, 0U); + ASSERT_GT(info.table_properties.num_entries, 0U); + ASSERT_EQ(info.file_checksum, kUnknownFileChecksum); + ASSERT_EQ(info.file_checksum_func_name, kUnknownFileChecksumFuncName); + +#ifdef ROCKSDB_USING_THREAD_STATUS + // Verify the id of the current thread that created this table + // file matches the id of any active flush or compaction thread. + uint64_t thread_id = env_->GetThreadID(); + std::vector thread_list; + ASSERT_OK(env_->GetThreadList(&thread_list)); + bool found_match = false; + for (auto thread_status : thread_list) { + if (thread_status.operation_type == ThreadStatus::OP_FLUSH || + thread_status.operation_type == ThreadStatus::OP_COMPACTION) { + if (thread_id == thread_status.thread_id) { + found_match = true; + break; + } + } + } + ASSERT_TRUE(found_match); +#endif // ROCKSDB_USING_THREAD_STATUS + } + + void OnFlushCompleted(DB* db, const FlushJobInfo& info) override { + flushed_dbs_.push_back(db); + flushed_column_family_names_.push_back(info.cf_name); + if (info.triggered_writes_slowdown) { + slowdown_count++; + } + if (info.triggered_writes_stop) { + stop_count++; + } + // verify whether the previously created file matches the flushed file. + ASSERT_EQ(prev_fc_info_.db_name, db->GetName()); + ASSERT_EQ(prev_fc_info_.cf_name, info.cf_name); + ASSERT_EQ(prev_fc_info_.job_id, info.job_id); + ASSERT_EQ(prev_fc_info_.file_path, info.file_path); + ASSERT_EQ(TableFileNameToNumber(info.file_path), info.file_number); + + // Note: the following chunk relies on the notification pertaining to the + // database pointed to by DBTestBase::db_, and is thus bypassed when + // that assumption does not hold (see the test case MultiDBMultiListeners + // below). + ASSERT_TRUE(test_); + if (db == test_->db_) { + std::vector> files_by_level; + ASSERT_LT(info.cf_id, test_->handles_.size()); + ASSERT_GE(info.cf_id, 0u); + ASSERT_NE(test_->handles_[info.cf_id], nullptr); + test_->dbfull()->TEST_GetFilesMetaData(test_->handles_[info.cf_id], + &files_by_level); + + ASSERT_FALSE(files_by_level.empty()); + auto it = std::find_if(files_by_level[0].begin(), files_by_level[0].end(), + [&](const FileMetaData& meta) { + return meta.fd.GetNumber() == info.file_number; + }); + ASSERT_NE(it, files_by_level[0].end()); + ASSERT_EQ(info.oldest_blob_file_number, it->oldest_blob_file_number); + } + + ASSERT_EQ(db->GetEnv()->GetThreadID(), info.thread_id); + ASSERT_GT(info.thread_id, 0U); + ASSERT_EQ(info.table_properties.user_collected_properties.find("0")->second, + "1"); + } + + std::vector flushed_column_family_names_; + std::vector flushed_dbs_; + int slowdown_count; + int stop_count; + bool db_closing; + std::atomic_bool db_closed; + TableFileCreationInfo prev_fc_info_; + + protected: + Env* env_; + EventListenerTest* test_; +}; + +TEST_F(EventListenerTest, OnSingleDBFlushTest) { + Options options; + options.env = CurrentOptions().env; + options.write_buffer_size = k110KB; +#ifdef ROCKSDB_USING_THREAD_STATUS + options.enable_thread_tracking = true; +#endif // ROCKSDB_USING_THREAD_STATUS + TestFlushListener* listener = new TestFlushListener(options.env, this); + options.listeners.emplace_back(listener); + std::vector cf_names = {"pikachu", "ilya", "muromec", + "dobrynia", "nikitich", "alyosha", + "popovich"}; + options.table_properties_collector_factories.push_back( + std::make_shared()); + CreateAndReopenWithCF(cf_names, options); + + ASSERT_OK(Put(1, "pikachu", std::string(90000, 'p'))); + + WriteBatch batch; + ASSERT_OK(WriteBatchInternal::PutBlobIndex(&batch, 1, "ditto", + BlobStr(456, 0, 1 << 10))); + ASSERT_OK(dbfull()->Write(WriteOptions(), &batch)); + + ASSERT_OK(Put(2, "ilya", std::string(90000, 'i'))); + ASSERT_OK(Put(3, "muromec", std::string(90000, 'm'))); + ASSERT_OK(Put(4, "dobrynia", std::string(90000, 'd'))); + ASSERT_OK(Put(5, "nikitich", std::string(90000, 'n'))); + ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a'))); + ASSERT_OK(Put(7, "popovich", std::string(90000, 'p'))); + for (int i = 1; i < 8; ++i) { + ASSERT_OK(Flush(i)); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + // Ensure background work is fully finished including listener callbacks + // before accessing listener state. + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + ASSERT_EQ(listener->flushed_dbs_.size(), i); + ASSERT_EQ(listener->flushed_column_family_names_.size(), i); + } + + // make sure callback functions are called in the right order + for (size_t i = 0; i < cf_names.size(); ++i) { + ASSERT_EQ(listener->flushed_dbs_[i], db_); + ASSERT_EQ(listener->flushed_column_family_names_[i], cf_names[i]); + } +} + +TEST_F(EventListenerTest, MultiCF) { + Options options; + options.env = CurrentOptions().env; + options.write_buffer_size = k110KB; +#ifdef ROCKSDB_USING_THREAD_STATUS + options.enable_thread_tracking = true; +#endif // ROCKSDB_USING_THREAD_STATUS + for (auto atomic_flush : {false, true}) { + options.atomic_flush = atomic_flush; + options.create_if_missing = true; + DestroyAndReopen(options); + TestFlushListener* listener = new TestFlushListener(options.env, this); + options.listeners.emplace_back(listener); + options.table_properties_collector_factories.push_back( + std::make_shared()); + std::vector cf_names = {"pikachu", "ilya", "muromec", + "dobrynia", "nikitich", "alyosha", + "popovich"}; + CreateAndReopenWithCF(cf_names, options); + + ASSERT_OK(Put(1, "pikachu", std::string(90000, 'p'))); + ASSERT_OK(Put(2, "ilya", std::string(90000, 'i'))); + ASSERT_OK(Put(3, "muromec", std::string(90000, 'm'))); + ASSERT_OK(Put(4, "dobrynia", std::string(90000, 'd'))); + ASSERT_OK(Put(5, "nikitich", std::string(90000, 'n'))); + ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a'))); + ASSERT_OK(Put(7, "popovich", std::string(90000, 'p'))); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + for (int i = 1; i < 8; ++i) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::NotifyOnFlushCompleted::PostAllOnFlushCompleted", + "EventListenerTest.MultiCF:PreVerifyListener"}}); + ASSERT_OK(Flush(i)); + TEST_SYNC_POINT("EventListenerTest.MultiCF:PreVerifyListener"); + ASSERT_EQ(listener->flushed_dbs_.size(), i); + ASSERT_EQ(listener->flushed_column_family_names_.size(), i); + // make sure callback functions are called in the right order + if (i == 7) { + for (size_t j = 0; j < cf_names.size(); j++) { + ASSERT_EQ(listener->flushed_dbs_[j], db_); + ASSERT_EQ(listener->flushed_column_family_names_[j], cf_names[j]); + } + } + } + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + Close(); + } +} + +TEST_F(EventListenerTest, MultiDBMultiListeners) { + Options options; + options.env = CurrentOptions().env; +#ifdef ROCKSDB_USING_THREAD_STATUS + options.enable_thread_tracking = true; +#endif // ROCKSDB_USING_THREAD_STATUS + options.table_properties_collector_factories.push_back( + std::make_shared()); + std::vector listeners; + const int kNumDBs = 5; + const int kNumListeners = 10; + for (int i = 0; i < kNumListeners; ++i) { + listeners.emplace_back(new TestFlushListener(options.env, this)); + } + + std::vector cf_names = {"pikachu", "ilya", "muromec", + "dobrynia", "nikitich", "alyosha", + "popovich"}; + + options.create_if_missing = true; + for (int i = 0; i < kNumListeners; ++i) { + options.listeners.emplace_back(listeners[i]); + } + DBOptions db_opts(options); + ColumnFamilyOptions cf_opts(options); + + std::vector dbs; + std::vector> vec_handles; + + for (int d = 0; d < kNumDBs; ++d) { + ASSERT_OK(DestroyDB(dbname_ + std::to_string(d), options)); + DB* db; + std::vector handles; + ASSERT_OK(DB::Open(options, dbname_ + std::to_string(d), &db)); + for (size_t c = 0; c < cf_names.size(); ++c) { + ColumnFamilyHandle* handle; + ASSERT_OK(db->CreateColumnFamily(cf_opts, cf_names[c], &handle)); + handles.push_back(handle); + } + + vec_handles.push_back(std::move(handles)); + dbs.push_back(db); + } + + for (int d = 0; d < kNumDBs; ++d) { + for (size_t c = 0; c < cf_names.size(); ++c) { + ASSERT_OK(dbs[d]->Put(WriteOptions(), vec_handles[d][c], cf_names[c], + cf_names[c])); + } + } + + for (size_t c = 0; c < cf_names.size(); ++c) { + for (int d = 0; d < kNumDBs; ++d) { + ASSERT_OK(dbs[d]->Flush(FlushOptions(), vec_handles[d][c])); + ASSERT_OK( + static_cast_with_check(dbs[d])->TEST_WaitForFlushMemTable()); + } + } + + for (int d = 0; d < kNumDBs; ++d) { + // Ensure background work is fully finished including listener callbacks + // before accessing listener state. + ASSERT_OK( + static_cast_with_check(dbs[d])->TEST_WaitForBackgroundWork()); + } + + for (auto* listener : listeners) { + int pos = 0; + for (size_t c = 0; c < cf_names.size(); ++c) { + for (int d = 0; d < kNumDBs; ++d) { + ASSERT_EQ(listener->flushed_dbs_[pos], dbs[d]); + ASSERT_EQ(listener->flushed_column_family_names_[pos], cf_names[c]); + pos++; + } + } + } + + for (auto handles : vec_handles) { + for (auto h : handles) { + delete h; + } + handles.clear(); + } + vec_handles.clear(); + + for (auto db : dbs) { + delete db; + } +} + +TEST_F(EventListenerTest, DisableBGCompaction) { + Options options; + options.env = CurrentOptions().env; +#ifdef ROCKSDB_USING_THREAD_STATUS + options.enable_thread_tracking = true; +#endif // ROCKSDB_USING_THREAD_STATUS + TestFlushListener* listener = new TestFlushListener(options.env, this); + const int kCompactionTrigger = 1; + const int kSlowdownTrigger = 5; + const int kStopTrigger = 100; + options.level0_file_num_compaction_trigger = kCompactionTrigger; + options.level0_slowdown_writes_trigger = kSlowdownTrigger; + options.level0_stop_writes_trigger = kStopTrigger; + options.max_write_buffer_number = 10; + options.listeners.emplace_back(listener); + // BG compaction is disabled. Number of L0 files will simply keeps + // increasing in this test. + options.compaction_style = kCompactionStyleNone; + options.compression = kNoCompression; + options.write_buffer_size = 100000; // Small write buffer + options.table_properties_collector_factories.push_back( + std::make_shared()); + + CreateAndReopenWithCF({"pikachu"}, options); + ColumnFamilyMetaData cf_meta; + db_->GetColumnFamilyMetaData(handles_[1], &cf_meta); + + // keep writing until writes are forced to stop. + for (int i = 0; static_cast(cf_meta.file_count) < kSlowdownTrigger * 10; + ++i) { + ASSERT_OK( + Put(1, std::to_string(i), std::string(10000, 'x'), WriteOptions())); + FlushOptions fo; + fo.allow_write_stall = true; + ASSERT_OK(db_->Flush(fo, handles_[1])); + db_->GetColumnFamilyMetaData(handles_[1], &cf_meta); + } + // Ensure background work is fully finished including listener callbacks + // before accessing listener state. + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + ASSERT_GE(listener->slowdown_count, kSlowdownTrigger * 9); +} + +class TestCompactionReasonListener : public EventListener { + public: + void OnCompactionCompleted(DB* /*db*/, const CompactionJobInfo& ci) override { + std::lock_guard lock(mutex_); + compaction_reasons_.push_back(ci.compaction_reason); + } + + std::vector compaction_reasons_; + std::mutex mutex_; +}; + +TEST_F(EventListenerTest, CompactionReasonLevel) { + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.env = CurrentOptions().env; + options.create_if_missing = true; + options.memtable_factory.reset(test::NewSpecialSkipListFactory( + DBTestBase::kNumKeysByGenerateNewRandomFile)); + + TestCompactionReasonListener* listener = new TestCompactionReasonListener(); + options.listeners.emplace_back(listener); + + options.level0_file_num_compaction_trigger = 4; + options.compaction_style = kCompactionStyleLevel; + + DestroyAndReopen(options); + Random rnd(301); + + // Write 4 files in L0 + for (int i = 0; i < 4; i++) { + GenerateNewRandomFile(&rnd); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ(listener->compaction_reasons_.size(), 1); + ASSERT_EQ(listener->compaction_reasons_[0], + CompactionReason::kLevelL0FilesNum); + + DestroyAndReopen(options); + + // Write 3 non-overlapping files in L0 + for (int k = 1; k <= 30; k++) { + ASSERT_OK(Put(Key(k), Key(k))); + if (k % 10 == 0) { + Flush(); + } + } + + // Do a trivial move from L0 -> L1 + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + options.max_bytes_for_level_base = 1; + Close(); + listener->compaction_reasons_.clear(); + Reopen(options); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_GT(listener->compaction_reasons_.size(), 1); + + for (auto compaction_reason : listener->compaction_reasons_) { + ASSERT_EQ(compaction_reason, CompactionReason::kLevelMaxLevelSize); + } + + options.disable_auto_compactions = true; + Close(); + listener->compaction_reasons_.clear(); + Reopen(options); + + ASSERT_OK(Put("key", "value")); + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForceOptimized; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + ASSERT_GT(listener->compaction_reasons_.size(), 0); + for (auto compaction_reason : listener->compaction_reasons_) { + ASSERT_EQ(compaction_reason, CompactionReason::kManualCompaction); + } +} + +TEST_F(EventListenerTest, CompactionReasonUniversal) { + Options options; + options.env = CurrentOptions().env; + options.create_if_missing = true; + options.memtable_factory.reset(test::NewSpecialSkipListFactory( + DBTestBase::kNumKeysByGenerateNewRandomFile)); + + TestCompactionReasonListener* listener = new TestCompactionReasonListener(); + options.listeners.emplace_back(listener); + + options.compaction_style = kCompactionStyleUniversal; + + Random rnd(301); + + options.level0_file_num_compaction_trigger = 8; + options.compaction_options_universal.max_size_amplification_percent = 100000; + options.compaction_options_universal.size_ratio = 100000; + DestroyAndReopen(options); + listener->compaction_reasons_.clear(); + + // Write 8 files in L0 + for (int i = 0; i < 8; i++) { + GenerateNewRandomFile(&rnd); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_GT(listener->compaction_reasons_.size(), 0); + for (auto compaction_reason : listener->compaction_reasons_) { + ASSERT_EQ(compaction_reason, CompactionReason::kUniversalSizeRatio); + } + + options.level0_file_num_compaction_trigger = 8; + options.compaction_options_universal.max_size_amplification_percent = 1; + options.compaction_options_universal.size_ratio = 100000; + + DestroyAndReopen(options); + listener->compaction_reasons_.clear(); + + // Write 8 files in L0 + for (int i = 0; i < 8; i++) { + GenerateNewRandomFile(&rnd); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_GT(listener->compaction_reasons_.size(), 0); + for (auto compaction_reason : listener->compaction_reasons_) { + ASSERT_EQ(compaction_reason, CompactionReason::kUniversalSizeAmplification); + } + + options.disable_auto_compactions = true; + Close(); + listener->compaction_reasons_.clear(); + Reopen(options); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + ASSERT_GT(listener->compaction_reasons_.size(), 0); + for (auto compaction_reason : listener->compaction_reasons_) { + ASSERT_EQ(compaction_reason, CompactionReason::kManualCompaction); + } +} + +TEST_F(EventListenerTest, CompactionReasonFIFO) { + Options options; + options.env = CurrentOptions().env; + options.create_if_missing = true; + options.memtable_factory.reset(test::NewSpecialSkipListFactory( + DBTestBase::kNumKeysByGenerateNewRandomFile)); + + TestCompactionReasonListener* listener = new TestCompactionReasonListener(); + options.listeners.emplace_back(listener); + + options.level0_file_num_compaction_trigger = 4; + options.compaction_style = kCompactionStyleFIFO; + options.compaction_options_fifo.max_table_files_size = 1; + + DestroyAndReopen(options); + Random rnd(301); + + // Write 4 files in L0 + for (int i = 0; i < 4; i++) { + GenerateNewRandomFile(&rnd); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_GT(listener->compaction_reasons_.size(), 0); + for (auto compaction_reason : listener->compaction_reasons_) { + ASSERT_EQ(compaction_reason, CompactionReason::kFIFOMaxSize); + } +} + +class TableFileCreationListener : public EventListener { + public: + class TestFS : public FileSystemWrapper { + public: + explicit TestFS(const std::shared_ptr& t) + : FileSystemWrapper(t) {} + static const char* kClassName() { return "TestEnv"; } + const char* Name() const override { return kClassName(); } + + void SetStatus(IOStatus s) { status_ = s; } + + IOStatus NewWritableFile(const std::string& fname, const FileOptions& opts, + std::unique_ptr* result, + IODebugContext* dbg) override { + if (fname.size() > 4 && fname.substr(fname.size() - 4) == ".sst") { + if (!status_.ok()) { + return status_; + } + } + return target()->NewWritableFile(fname, opts, result, dbg); + } + + private: + IOStatus status_; + }; + + TableFileCreationListener() { + for (int i = 0; i < 2; i++) { + started_[i] = finished_[i] = failure_[i] = 0; + } + } + + int Index(TableFileCreationReason reason) { + int idx; + switch (reason) { + case TableFileCreationReason::kFlush: + idx = 0; + break; + case TableFileCreationReason::kCompaction: + idx = 1; + break; + default: + idx = -1; + } + return idx; + } + + void CheckAndResetCounters(int flush_started, int flush_finished, + int flush_failure, int compaction_started, + int compaction_finished, int compaction_failure) { + ASSERT_EQ(started_[0], flush_started); + ASSERT_EQ(finished_[0], flush_finished); + ASSERT_EQ(failure_[0], flush_failure); + ASSERT_EQ(started_[1], compaction_started); + ASSERT_EQ(finished_[1], compaction_finished); + ASSERT_EQ(failure_[1], compaction_failure); + for (int i = 0; i < 2; i++) { + started_[i] = finished_[i] = failure_[i] = 0; + } + } + + void OnTableFileCreationStarted( + const TableFileCreationBriefInfo& info) override { + int idx = Index(info.reason); + if (idx >= 0) { + started_[idx]++; + } + ASSERT_GT(info.db_name.size(), 0U); + ASSERT_GT(info.cf_name.size(), 0U); + ASSERT_GT(info.file_path.size(), 0U); + ASSERT_GT(info.job_id, 0); + } + + void OnTableFileCreated(const TableFileCreationInfo& info) override { + int idx = Index(info.reason); + if (idx >= 0) { + finished_[idx]++; + } + ASSERT_GT(info.db_name.size(), 0U); + ASSERT_GT(info.cf_name.size(), 0U); + ASSERT_GT(info.file_path.size(), 0U); + ASSERT_GT(info.job_id, 0); + ASSERT_EQ(info.file_checksum, kUnknownFileChecksum); + ASSERT_EQ(info.file_checksum_func_name, kUnknownFileChecksumFuncName); + if (info.status.ok()) { + if (info.table_properties.num_range_deletions == 0U) { + ASSERT_GT(info.table_properties.data_size, 0U); + ASSERT_GT(info.table_properties.raw_key_size, 0U); + ASSERT_GT(info.table_properties.raw_value_size, 0U); + ASSERT_GT(info.table_properties.num_data_blocks, 0U); + ASSERT_GT(info.table_properties.num_entries, 0U); + } + } else { + if (idx >= 0) { + failure_[idx]++; + last_failure_ = info.status; + } + } + } + + int started_[2]; + int finished_[2]; + int failure_[2]; + Status last_failure_; +}; + +TEST_F(EventListenerTest, TableFileCreationListenersTest) { + auto listener = std::make_shared(); + Options options; + std::shared_ptr test_fs = + std::make_shared( + CurrentOptions().env->GetFileSystem()); + std::unique_ptr test_env = NewCompositeEnv(test_fs); + options.create_if_missing = true; + options.listeners.push_back(listener); + options.env = test_env.get(); + DestroyAndReopen(options); + + ASSERT_OK(Put("foo", "aaa")); + ASSERT_OK(Put("bar", "bbb")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + listener->CheckAndResetCounters(1, 1, 0, 0, 0, 0); + ASSERT_OK(Put("foo", "aaa1")); + ASSERT_OK(Put("bar", "bbb1")); + test_fs->SetStatus(IOStatus::NotSupported("not supported")); + ASSERT_NOK(Flush()); + listener->CheckAndResetCounters(1, 1, 1, 0, 0, 0); + ASSERT_TRUE(listener->last_failure_.IsNotSupported()); + test_fs->SetStatus(IOStatus::OK()); + + Reopen(options); + ASSERT_OK(Put("foo", "aaa2")); + ASSERT_OK(Put("bar", "bbb2")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + listener->CheckAndResetCounters(1, 1, 0, 0, 0, 0); + + const Slice kRangeStart = "a"; + const Slice kRangeEnd = "z"; + ASSERT_OK( + dbfull()->CompactRange(CompactRangeOptions(), &kRangeStart, &kRangeEnd)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + listener->CheckAndResetCounters(0, 0, 0, 1, 1, 0); + + ASSERT_OK(Put("foo", "aaa3")); + ASSERT_OK(Put("bar", "bbb3")); + ASSERT_OK(Flush()); + test_fs->SetStatus(IOStatus::NotSupported("not supported")); + ASSERT_NOK( + dbfull()->CompactRange(CompactRangeOptions(), &kRangeStart, &kRangeEnd)); + ASSERT_NOK(dbfull()->TEST_WaitForCompact()); + listener->CheckAndResetCounters(1, 1, 0, 1, 1, 1); + ASSERT_TRUE(listener->last_failure_.IsNotSupported()); + + // Reset + test_fs->SetStatus(IOStatus::OK()); + DestroyAndReopen(options); + + // Verify that an empty table file that is immediately deleted gives Aborted + // status to listener. + ASSERT_OK(Put("baz", "z")); + ASSERT_OK(SingleDelete("baz")); + ASSERT_OK(Flush()); + listener->CheckAndResetCounters(1, 1, 1, 0, 0, 0); + ASSERT_TRUE(listener->last_failure_.IsAborted()); + + // Also in compaction + ASSERT_OK(Put("baz", "z")); + ASSERT_OK(Flush()); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + kRangeStart, kRangeEnd)); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + listener->CheckAndResetCounters(2, 2, 0, 1, 1, 1); + ASSERT_TRUE(listener->last_failure_.IsAborted()); + + Close(); // Avoid UAF on listener +} + +class MemTableSealedListener : public EventListener { + private: + SequenceNumber latest_seq_number_; + + public: + MemTableSealedListener() {} + void OnMemTableSealed(const MemTableInfo& info) override { + latest_seq_number_ = info.first_seqno; + } + + void OnFlushCompleted(DB* /*db*/, + const FlushJobInfo& flush_job_info) override { + ASSERT_LE(flush_job_info.smallest_seqno, latest_seq_number_); + } +}; + +TEST_F(EventListenerTest, MemTableSealedListenerTest) { + auto listener = std::make_shared(); + Options options; + options.env = CurrentOptions().env; + options.create_if_missing = true; + options.listeners.push_back(listener); + DestroyAndReopen(options); + + for (unsigned int i = 0; i < 10; i++) { + std::string tag = std::to_string(i); + ASSERT_OK(Put("foo" + tag, "aaa")); + ASSERT_OK(Put("bar" + tag, "bbb")); + + ASSERT_OK(Flush()); + } +} + +class ColumnFamilyHandleDeletionStartedListener : public EventListener { + private: + std::vector cfs_; + int counter; + + public: + explicit ColumnFamilyHandleDeletionStartedListener( + const std::vector& cfs) + : cfs_(cfs), counter(0) { + cfs_.insert(cfs_.begin(), kDefaultColumnFamilyName); + } + void OnColumnFamilyHandleDeletionStarted( + ColumnFamilyHandle* handle) override { + ASSERT_EQ(cfs_[handle->GetID()], handle->GetName()); + counter++; + } + int getCounter() { return counter; } +}; + +TEST_F(EventListenerTest, ColumnFamilyHandleDeletionStartedListenerTest) { + std::vector cfs{"pikachu", "eevee", "Mewtwo"}; + auto listener = + std::make_shared(cfs); + Options options; + options.env = CurrentOptions().env; + options.create_if_missing = true; + options.listeners.push_back(listener); + CreateAndReopenWithCF(cfs, options); + ASSERT_EQ(handles_.size(), 4); + delete handles_[3]; + delete handles_[2]; + delete handles_[1]; + handles_.resize(1); + ASSERT_EQ(listener->getCounter(), 3); +} + +class BackgroundErrorListener : public EventListener { + private: + SpecialEnv* env_; + int counter_; + + public: + BackgroundErrorListener(SpecialEnv* env) : env_(env), counter_(0) {} + + void OnBackgroundError(BackgroundErrorReason /*reason*/, + Status* bg_error) override { + if (counter_ == 0) { + // suppress the first error and disable write-dropping such that a retry + // can succeed. + *bg_error = Status::OK(); + env_->drop_writes_.store(false, std::memory_order_release); + env_->SetMockSleep(false); + } + ++counter_; + } + + int counter() { return counter_; } +}; + +TEST_F(EventListenerTest, BackgroundErrorListenerFailedFlushTest) { + auto listener = std::make_shared(env_); + Options options; + options.create_if_missing = true; + options.env = env_; + options.listeners.push_back(listener); + options.memtable_factory.reset(test::NewSpecialSkipListFactory(1)); + options.paranoid_checks = true; + DestroyAndReopen(options); + + // the usual TEST_WaitForFlushMemTable() doesn't work for failed flushes, so + // forge a custom one for the failed flush case. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BGWorkFlush:done", + "EventListenerTest:BackgroundErrorListenerFailedFlushTest:1"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + env_->drop_writes_.store(true, std::memory_order_release); + env_->SetMockSleep(); + + ASSERT_OK(Put("key0", "val")); + ASSERT_OK(Put("key1", "val")); + TEST_SYNC_POINT("EventListenerTest:BackgroundErrorListenerFailedFlushTest:1"); + ASSERT_EQ(1, listener->counter()); + ASSERT_OK(Put("key2", "val")); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(1, NumTableFilesAtLevel(0)); +} + +TEST_F(EventListenerTest, BackgroundErrorListenerFailedCompactionTest) { + auto listener = std::make_shared(env_); + Options options; + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.env = env_; + options.level0_file_num_compaction_trigger = 2; + options.listeners.push_back(listener); + options.memtable_factory.reset(test::NewSpecialSkipListFactory(2)); + options.paranoid_checks = true; + DestroyAndReopen(options); + + // third iteration triggers the second memtable's flush + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Put("key0", "val")); + if (i > 0) { + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + } + ASSERT_OK(Put("key1", "val")); + } + ASSERT_EQ(2, NumTableFilesAtLevel(0)); + + env_->drop_writes_.store(true, std::memory_order_release); + env_->SetMockSleep(); + ASSERT_OK(dbfull()->SetOptions({{"disable_auto_compactions", "false"}})); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(1, listener->counter()); + + // trigger flush so compaction is triggered again; this time it succeeds + // The previous failed compaction may get retried automatically, so we may + // be left with 0 or 1 files in level 1, depending on when the retry gets + // scheduled + ASSERT_OK(Put("key0", "val")); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_LE(1, NumTableFilesAtLevel(0)); +} + +class TestFileOperationListener : public EventListener { + public: + TestFileOperationListener() { + file_reads_.store(0); + file_reads_success_.store(0); + file_writes_.store(0); + file_writes_success_.store(0); + file_flushes_.store(0); + file_flushes_success_.store(0); + file_closes_.store(0); + file_closes_success_.store(0); + file_syncs_.store(0); + file_syncs_success_.store(0); + file_truncates_.store(0); + file_truncates_success_.store(0); + file_seq_reads_.store(0); + blob_file_reads_.store(0); + blob_file_writes_.store(0); + blob_file_flushes_.store(0); + blob_file_closes_.store(0); + blob_file_syncs_.store(0); + blob_file_truncates_.store(0); + } + + void OnFileReadFinish(const FileOperationInfo& info) override { + ++file_reads_; + if (info.status.ok()) { + ++file_reads_success_; + } + if (info.path.find("MANIFEST") != std::string::npos) { + ++file_seq_reads_; + } + if (EndsWith(info.path, ".blob")) { + ++blob_file_reads_; + } + ReportDuration(info); + } + + void OnFileWriteFinish(const FileOperationInfo& info) override { + ++file_writes_; + if (info.status.ok()) { + ++file_writes_success_; + } + if (EndsWith(info.path, ".blob")) { + ++blob_file_writes_; + } + ReportDuration(info); + } + + void OnFileFlushFinish(const FileOperationInfo& info) override { + ++file_flushes_; + if (info.status.ok()) { + ++file_flushes_success_; + } + if (EndsWith(info.path, ".blob")) { + ++blob_file_flushes_; + } + ReportDuration(info); + } + + void OnFileCloseFinish(const FileOperationInfo& info) override { + ++file_closes_; + if (info.status.ok()) { + ++file_closes_success_; + } + if (EndsWith(info.path, ".blob")) { + ++blob_file_closes_; + } + ReportDuration(info); + } + + void OnFileSyncFinish(const FileOperationInfo& info) override { + ++file_syncs_; + if (info.status.ok()) { + ++file_syncs_success_; + } + if (EndsWith(info.path, ".blob")) { + ++blob_file_syncs_; + } + ReportDuration(info); + } + + void OnFileTruncateFinish(const FileOperationInfo& info) override { + ++file_truncates_; + if (info.status.ok()) { + ++file_truncates_success_; + } + if (EndsWith(info.path, ".blob")) { + ++blob_file_truncates_; + } + ReportDuration(info); + } + + bool ShouldBeNotifiedOnFileIO() override { return true; } + + std::atomic file_reads_; + std::atomic file_reads_success_; + std::atomic file_writes_; + std::atomic file_writes_success_; + std::atomic file_flushes_; + std::atomic file_flushes_success_; + std::atomic file_closes_; + std::atomic file_closes_success_; + std::atomic file_syncs_; + std::atomic file_syncs_success_; + std::atomic file_truncates_; + std::atomic file_truncates_success_; + std::atomic file_seq_reads_; + std::atomic blob_file_reads_; + std::atomic blob_file_writes_; + std::atomic blob_file_flushes_; + std::atomic blob_file_closes_; + std::atomic blob_file_syncs_; + std::atomic blob_file_truncates_; + + private: + void ReportDuration(const FileOperationInfo& info) const { + ASSERT_GT(info.duration.count(), 0); + } +}; + +TEST_F(EventListenerTest, OnFileOperationTest) { + Options options; + options.env = CurrentOptions().env; + options.create_if_missing = true; + + TestFileOperationListener* listener = new TestFileOperationListener(); + options.listeners.emplace_back(listener); + + options.use_direct_io_for_flush_and_compaction = false; + Status s = TryReopen(options); + if (s.IsInvalidArgument()) { + options.use_direct_io_for_flush_and_compaction = false; + } else { + ASSERT_OK(s); + } + DestroyAndReopen(options); + ASSERT_OK(Put("foo", "aaa")); + ASSERT_OK(dbfull()->Flush(FlushOptions())); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_GE(listener->file_writes_.load(), + listener->file_writes_success_.load()); + ASSERT_GT(listener->file_writes_.load(), 0); + ASSERT_GE(listener->file_flushes_.load(), + listener->file_flushes_success_.load()); + ASSERT_GT(listener->file_flushes_.load(), 0); + Close(); + + Reopen(options); + ASSERT_GE(listener->file_reads_.load(), listener->file_reads_success_.load()); + ASSERT_GT(listener->file_reads_.load(), 0); + ASSERT_GE(listener->file_closes_.load(), + listener->file_closes_success_.load()); + ASSERT_GT(listener->file_closes_.load(), 0); + ASSERT_GE(listener->file_syncs_.load(), listener->file_syncs_success_.load()); + ASSERT_GT(listener->file_syncs_.load(), 0); + if (true == options.use_direct_io_for_flush_and_compaction) { + ASSERT_GE(listener->file_truncates_.load(), + listener->file_truncates_success_.load()); + ASSERT_GT(listener->file_truncates_.load(), 0); + } +} + +TEST_F(EventListenerTest, OnBlobFileOperationTest) { + Options options; + options.env = CurrentOptions().env; + options.create_if_missing = true; + TestFileOperationListener* listener = new TestFileOperationListener(); + options.listeners.emplace_back(listener); + options.disable_auto_compactions = true; + options.enable_blob_files = true; + options.min_blob_size = 0; + options.enable_blob_garbage_collection = true; + options.blob_garbage_collection_age_cutoff = 0.5; + + DestroyAndReopen(options); + + ASSERT_OK(Put("Key1", "blob_value1")); + ASSERT_OK(Put("Key2", "blob_value2")); + ASSERT_OK(Put("Key3", "blob_value3")); + ASSERT_OK(Put("Key4", "blob_value4")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("Key3", "new_blob_value3")); + ASSERT_OK(Put("Key4", "new_blob_value4")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("Key5", "blob_value5")); + ASSERT_OK(Put("Key6", "blob_value6")); + ASSERT_OK(Flush()); + + ASSERT_GT(listener->blob_file_writes_.load(), 0U); + ASSERT_GT(listener->blob_file_flushes_.load(), 0U); + Close(); + + Reopen(options); + ASSERT_GT(listener->blob_file_closes_.load(), 0U); + ASSERT_GT(listener->blob_file_syncs_.load(), 0U); + if (true == options.use_direct_io_for_flush_and_compaction) { + ASSERT_GT(listener->blob_file_truncates_.load(), 0U); + } +} + +TEST_F(EventListenerTest, ReadManifestAndWALOnRecovery) { + Options options; + options.env = CurrentOptions().env; + options.create_if_missing = true; + + TestFileOperationListener* listener = new TestFileOperationListener(); + options.listeners.emplace_back(listener); + + options.use_direct_io_for_flush_and_compaction = false; + Status s = TryReopen(options); + if (s.IsInvalidArgument()) { + options.use_direct_io_for_flush_and_compaction = false; + } else { + ASSERT_OK(s); + } + DestroyAndReopen(options); + ASSERT_OK(Put("foo", "aaa")); + Close(); + + size_t seq_reads = listener->file_seq_reads_.load(); + Reopen(options); + ASSERT_GT(listener->file_seq_reads_.load(), seq_reads); +} + +class BlobDBJobLevelEventListenerTest : public EventListener { + public: + explicit BlobDBJobLevelEventListenerTest(EventListenerTest* test) + : test_(test), call_count_(0) {} + + const VersionStorageInfo* GetVersionStorageInfo() const { + VersionSet* const versions = test_->dbfull()->GetVersionSet(); + assert(versions); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + EXPECT_NE(cfd, nullptr); + + Version* const current = cfd->current(); + EXPECT_NE(current, nullptr); + + const VersionStorageInfo* const storage_info = current->storage_info(); + EXPECT_NE(storage_info, nullptr); + + return storage_info; + } + + void CheckBlobFileAdditions( + const std::vector& blob_file_addition_infos) const { + const auto* vstorage = GetVersionStorageInfo(); + + EXPECT_FALSE(blob_file_addition_infos.empty()); + + for (const auto& blob_file_addition_info : blob_file_addition_infos) { + const auto meta = vstorage->GetBlobFileMetaData( + blob_file_addition_info.blob_file_number); + + EXPECT_NE(meta, nullptr); + EXPECT_EQ(meta->GetBlobFileNumber(), + blob_file_addition_info.blob_file_number); + EXPECT_EQ(meta->GetTotalBlobBytes(), + blob_file_addition_info.total_blob_bytes); + EXPECT_EQ(meta->GetTotalBlobCount(), + blob_file_addition_info.total_blob_count); + EXPECT_FALSE(blob_file_addition_info.blob_file_path.empty()); + } + } + + std::vector GetFlushedFiles() { + std::lock_guard lock(mutex_); + std::vector result; + for (const auto& fname : flushed_files_) { + result.push_back(fname); + } + return result; + } + + void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& info) override { + call_count_++; + + { + std::lock_guard lock(mutex_); + flushed_files_.push_back(info.file_path); + } + + EXPECT_EQ(info.blob_compression_type, kNoCompression); + + CheckBlobFileAdditions(info.blob_file_addition_infos); + } + + void OnCompactionCompleted(DB* /*db*/, + const CompactionJobInfo& info) override { + call_count_++; + + EXPECT_EQ(info.blob_compression_type, kNoCompression); + + CheckBlobFileAdditions(info.blob_file_addition_infos); + + EXPECT_FALSE(info.blob_file_garbage_infos.empty()); + + for (const auto& blob_file_garbage_info : info.blob_file_garbage_infos) { + EXPECT_GT(blob_file_garbage_info.blob_file_number, 0U); + EXPECT_GT(blob_file_garbage_info.garbage_blob_count, 0U); + EXPECT_GT(blob_file_garbage_info.garbage_blob_bytes, 0U); + EXPECT_FALSE(blob_file_garbage_info.blob_file_path.empty()); + } + } + + EventListenerTest* test_; + uint32_t call_count_; + + private: + std::vector flushed_files_; + std::mutex mutex_; +}; + +// Test OnFlushCompleted EventListener called for blob files +TEST_F(EventListenerTest, BlobDBOnFlushCompleted) { + Options options; + options.env = CurrentOptions().env; + options.enable_blob_files = true; + options.create_if_missing = true; + options.disable_auto_compactions = true; + + options.min_blob_size = 0; + BlobDBJobLevelEventListenerTest* blob_event_listener = + new BlobDBJobLevelEventListenerTest(this); + options.listeners.emplace_back(blob_event_listener); + + DestroyAndReopen(options); + + ASSERT_OK(Put("Key1", "blob_value1")); + ASSERT_OK(Put("Key2", "blob_value2")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("Key3", "blob_value3")); + ASSERT_OK(Flush()); + + ASSERT_EQ(Get("Key1"), "blob_value1"); + ASSERT_EQ(Get("Key2"), "blob_value2"); + ASSERT_EQ(Get("Key3"), "blob_value3"); + + ASSERT_GT(blob_event_listener->call_count_, 0U); +} + +// Test OnCompactionCompleted EventListener called for blob files +TEST_F(EventListenerTest, BlobDBOnCompactionCompleted) { + Options options; + options.env = CurrentOptions().env; + options.enable_blob_files = true; + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.min_blob_size = 0; + BlobDBJobLevelEventListenerTest* blob_event_listener = + new BlobDBJobLevelEventListenerTest(this); + options.listeners.emplace_back(blob_event_listener); + + options.enable_blob_garbage_collection = true; + options.blob_garbage_collection_age_cutoff = 0.5; + + DestroyAndReopen(options); + + ASSERT_OK(Put("Key1", "blob_value1")); + ASSERT_OK(Put("Key2", "blob_value2")); + ASSERT_OK(Put("Key3", "blob_value3")); + ASSERT_OK(Put("Key4", "blob_value4")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("Key3", "new_blob_value3")); + ASSERT_OK(Put("Key4", "new_blob_value4")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("Key5", "blob_value5")); + ASSERT_OK(Put("Key6", "blob_value6")); + ASSERT_OK(Flush()); + + blob_event_listener->call_count_ = 0; + constexpr Slice* begin = nullptr; + constexpr Slice* end = nullptr; + + // On compaction, because of blob_garbage_collection_age_cutoff, it will + // delete the oldest blob file and create new blob file during compaction. + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), begin, end)); + + // Make sure, OnCompactionCompleted is called. + ASSERT_GT(blob_event_listener->call_count_, 0U); +} + +// Test CompactFiles calls OnCompactionCompleted EventListener for blob files +// and populate the blob files info. +TEST_F(EventListenerTest, BlobDBCompactFiles) { + Options options; + options.env = CurrentOptions().env; + options.enable_blob_files = true; + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.min_blob_size = 0; + options.enable_blob_garbage_collection = true; + options.blob_garbage_collection_age_cutoff = 0.5; + + BlobDBJobLevelEventListenerTest* blob_event_listener = + new BlobDBJobLevelEventListenerTest(this); + options.listeners.emplace_back(blob_event_listener); + + DestroyAndReopen(options); + + ASSERT_OK(Put("Key1", "blob_value1")); + ASSERT_OK(Put("Key2", "blob_value2")); + ASSERT_OK(Put("Key3", "blob_value3")); + ASSERT_OK(Put("Key4", "blob_value4")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("Key3", "new_blob_value3")); + ASSERT_OK(Put("Key4", "new_blob_value4")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("Key5", "blob_value5")); + ASSERT_OK(Put("Key6", "blob_value6")); + ASSERT_OK(Flush()); + + std::vector output_file_names; + CompactionJobInfo compaction_job_info; + + // On compaction, because of blob_garbage_collection_age_cutoff, it will + // delete the oldest blob file and create new blob file during compaction + // which will be populated in output_files_names. + ASSERT_OK(dbfull()->CompactFiles( + CompactionOptions(), blob_event_listener->GetFlushedFiles(), 1, -1, + &output_file_names, &compaction_job_info)); + + bool is_blob_in_output = false; + for (const auto& file : output_file_names) { + if (EndsWith(file, ".blob")) { + is_blob_in_output = true; + } + } + ASSERT_TRUE(is_blob_in_output); + + for (const auto& blob_file_addition_info : + compaction_job_info.blob_file_addition_infos) { + EXPECT_GT(blob_file_addition_info.blob_file_number, 0U); + EXPECT_GT(blob_file_addition_info.total_blob_bytes, 0U); + EXPECT_GT(blob_file_addition_info.total_blob_count, 0U); + EXPECT_FALSE(blob_file_addition_info.blob_file_path.empty()); + } + + for (const auto& blob_file_garbage_info : + compaction_job_info.blob_file_garbage_infos) { + EXPECT_GT(blob_file_garbage_info.blob_file_number, 0U); + EXPECT_GT(blob_file_garbage_info.garbage_blob_count, 0U); + EXPECT_GT(blob_file_garbage_info.garbage_blob_bytes, 0U); + EXPECT_FALSE(blob_file_garbage_info.blob_file_path.empty()); + } +} + +class BlobDBFileLevelEventListener : public EventListener { + public: + void OnBlobFileCreationStarted( + const BlobFileCreationBriefInfo& info) override { + files_started_++; + EXPECT_FALSE(info.db_name.empty()); + EXPECT_FALSE(info.cf_name.empty()); + EXPECT_FALSE(info.file_path.empty()); + EXPECT_GT(info.job_id, 0); + } + + void OnBlobFileCreated(const BlobFileCreationInfo& info) override { + files_created_++; + EXPECT_FALSE(info.db_name.empty()); + EXPECT_FALSE(info.cf_name.empty()); + EXPECT_FALSE(info.file_path.empty()); + EXPECT_GT(info.job_id, 0); + EXPECT_GT(info.total_blob_count, 0U); + EXPECT_GT(info.total_blob_bytes, 0U); + EXPECT_EQ(info.file_checksum, kUnknownFileChecksum); + EXPECT_EQ(info.file_checksum_func_name, kUnknownFileChecksumFuncName); + EXPECT_TRUE(info.status.ok()); + } + + void OnBlobFileDeleted(const BlobFileDeletionInfo& info) override { + files_deleted_++; + EXPECT_FALSE(info.db_name.empty()); + EXPECT_FALSE(info.file_path.empty()); + EXPECT_GT(info.job_id, 0); + EXPECT_TRUE(info.status.ok()); + } + + void CheckCounters() { + EXPECT_EQ(files_started_, files_created_); + EXPECT_GT(files_started_, 0U); + EXPECT_GT(files_deleted_, 0U); + EXPECT_LT(files_deleted_, files_created_); + } + + private: + std::atomic files_started_{}; + std::atomic files_created_{}; + std::atomic files_deleted_{}; +}; + +TEST_F(EventListenerTest, BlobDBFileTest) { + Options options; + options.env = CurrentOptions().env; + options.enable_blob_files = true; + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.min_blob_size = 0; + options.enable_blob_garbage_collection = true; + options.blob_garbage_collection_age_cutoff = 0.5; + + BlobDBFileLevelEventListener* blob_event_listener = + new BlobDBFileLevelEventListener(); + options.listeners.emplace_back(blob_event_listener); + + DestroyAndReopen(options); + + ASSERT_OK(Put("Key1", "blob_value1")); + ASSERT_OK(Put("Key2", "blob_value2")); + ASSERT_OK(Put("Key3", "blob_value3")); + ASSERT_OK(Put("Key4", "blob_value4")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("Key3", "new_blob_value3")); + ASSERT_OK(Put("Key4", "new_blob_value4")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("Key5", "blob_value5")); + ASSERT_OK(Put("Key6", "blob_value6")); + ASSERT_OK(Flush()); + + constexpr Slice* begin = nullptr; + constexpr Slice* end = nullptr; + + // On compaction, because of blob_garbage_collection_age_cutoff, it will + // delete the oldest blob file and create new blob file during compaction. + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), begin, end)); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + blob_event_listener->CheckCounters(); +} + +} // namespace ROCKSDB_NAMESPACE + + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/log_format.h b/librocksdb-sys/rocksdb/db/log_format.h new file mode 100644 index 0000000..a976b3f --- /dev/null +++ b/librocksdb-sys/rocksdb/db/log_format.h @@ -0,0 +1,55 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Log format information shared by reader and writer. +// See ../doc/log_format.txt for more detail. + +#pragma once + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { +namespace log { + +enum RecordType { + // Zero is reserved for preallocated files + kZeroType = 0, + kFullType = 1, + + // For fragments + kFirstType = 2, + kMiddleType = 3, + kLastType = 4, + + // For recycled log files + kRecyclableFullType = 5, + kRecyclableFirstType = 6, + kRecyclableMiddleType = 7, + kRecyclableLastType = 8, + + // Compression Type + kSetCompressionType = 9, + + // User-defined timestamp sizes + kUserDefinedTimestampSizeType = 10, + kRecyclableUserDefinedTimestampSizeType = 11, +}; +constexpr int kMaxRecordType = kRecyclableUserDefinedTimestampSizeType; + +constexpr unsigned int kBlockSize = 32768; + +// Header is checksum (4 bytes), length (2 bytes), type (1 byte) +constexpr int kHeaderSize = 4 + 2 + 1; + +// Recyclable header is checksum (4 bytes), length (2 bytes), type (1 byte), +// log number (4 bytes). +constexpr int kRecyclableHeaderSize = 4 + 2 + 1 + 4; + +} // namespace log +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/log_reader.cc b/librocksdb-sys/rocksdb/db/log_reader.cc new file mode 100644 index 0000000..5ec262d --- /dev/null +++ b/librocksdb-sys/rocksdb/db/log_reader.cc @@ -0,0 +1,934 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/log_reader.h" + +#include + +#include "file/sequence_file_reader.h" +#include "port/lang.h" +#include "rocksdb/env.h" +#include "test_util/sync_point.h" +#include "util/coding.h" +#include "util/crc32c.h" + +namespace ROCKSDB_NAMESPACE { +namespace log { + +Reader::Reporter::~Reporter() {} + +Reader::Reader(std::shared_ptr info_log, + std::unique_ptr&& _file, + Reporter* reporter, bool checksum, uint64_t log_num) + : info_log_(info_log), + file_(std::move(_file)), + reporter_(reporter), + checksum_(checksum), + backing_store_(new char[kBlockSize]), + buffer_(), + eof_(false), + read_error_(false), + eof_offset_(0), + last_record_offset_(0), + end_of_buffer_offset_(0), + log_number_(log_num), + recycled_(false), + first_record_read_(false), + compression_type_(kNoCompression), + compression_type_record_read_(false), + uncompress_(nullptr), + hash_state_(nullptr), + uncompress_hash_state_(nullptr){}; + +Reader::~Reader() { + delete[] backing_store_; + if (uncompress_) { + delete uncompress_; + } + if (hash_state_) { + XXH3_freeState(hash_state_); + } + if (uncompress_hash_state_) { + XXH3_freeState(uncompress_hash_state_); + } +} + +// For kAbsoluteConsistency, on clean shutdown we don't expect any error +// in the log files. For other modes, we can ignore only incomplete records +// in the last log file, which are presumably due to a write in progress +// during restart (or from log recycling). +// +// TODO krad: Evaluate if we need to move to a more strict mode where we +// restrict the inconsistency to only the last log +bool Reader::ReadRecord(Slice* record, std::string* scratch, + WALRecoveryMode wal_recovery_mode, + uint64_t* record_checksum) { + scratch->clear(); + record->clear(); + if (record_checksum != nullptr) { + if (hash_state_ == nullptr) { + hash_state_ = XXH3_createState(); + } + XXH3_64bits_reset(hash_state_); + } + if (uncompress_) { + uncompress_->Reset(); + } + bool in_fragmented_record = false; + // Record offset of the logical record that we're reading + // 0 is a dummy value to make compilers happy + uint64_t prospective_record_offset = 0; + + Slice fragment; + while (true) { + uint64_t physical_record_offset = end_of_buffer_offset_ - buffer_.size(); + size_t drop_size = 0; + const unsigned int record_type = + ReadPhysicalRecord(&fragment, &drop_size, record_checksum); + switch (record_type) { + case kFullType: + case kRecyclableFullType: + if (in_fragmented_record && !scratch->empty()) { + // Handle bug in earlier versions of log::Writer where + // it could emit an empty kFirstType record at the tail end + // of a block followed by a kFullType or kFirstType record + // at the beginning of the next block. + ReportCorruption(scratch->size(), "partial record without end(1)"); + } + // No need to compute record_checksum since the record + // consists of a single fragment and the checksum is computed + // in ReadPhysicalRecord() if WAL compression is enabled + if (record_checksum != nullptr && uncompress_ == nullptr) { + // No need to stream since the record is a single fragment + *record_checksum = XXH3_64bits(fragment.data(), fragment.size()); + } + prospective_record_offset = physical_record_offset; + scratch->clear(); + *record = fragment; + last_record_offset_ = prospective_record_offset; + first_record_read_ = true; + return true; + + case kFirstType: + case kRecyclableFirstType: + if (in_fragmented_record && !scratch->empty()) { + // Handle bug in earlier versions of log::Writer where + // it could emit an empty kFirstType record at the tail end + // of a block followed by a kFullType or kFirstType record + // at the beginning of the next block. + ReportCorruption(scratch->size(), "partial record without end(2)"); + XXH3_64bits_reset(hash_state_); + } + if (record_checksum != nullptr) { + XXH3_64bits_update(hash_state_, fragment.data(), fragment.size()); + } + prospective_record_offset = physical_record_offset; + scratch->assign(fragment.data(), fragment.size()); + in_fragmented_record = true; + break; + + case kMiddleType: + case kRecyclableMiddleType: + if (!in_fragmented_record) { + ReportCorruption(fragment.size(), + "missing start of fragmented record(1)"); + } else { + if (record_checksum != nullptr) { + XXH3_64bits_update(hash_state_, fragment.data(), fragment.size()); + } + scratch->append(fragment.data(), fragment.size()); + } + break; + + case kLastType: + case kRecyclableLastType: + if (!in_fragmented_record) { + ReportCorruption(fragment.size(), + "missing start of fragmented record(2)"); + } else { + if (record_checksum != nullptr) { + XXH3_64bits_update(hash_state_, fragment.data(), fragment.size()); + *record_checksum = XXH3_64bits_digest(hash_state_); + } + scratch->append(fragment.data(), fragment.size()); + *record = Slice(*scratch); + last_record_offset_ = prospective_record_offset; + first_record_read_ = true; + return true; + } + break; + + case kSetCompressionType: { + if (compression_type_record_read_) { + ReportCorruption(fragment.size(), + "read multiple SetCompressionType records"); + } + if (first_record_read_) { + ReportCorruption(fragment.size(), + "SetCompressionType not the first record"); + } + prospective_record_offset = physical_record_offset; + scratch->clear(); + last_record_offset_ = prospective_record_offset; + CompressionTypeRecord compression_record(kNoCompression); + Status s = compression_record.DecodeFrom(&fragment); + if (!s.ok()) { + ReportCorruption(fragment.size(), + "could not decode SetCompressionType record"); + } else { + InitCompression(compression_record); + } + break; + } + case kUserDefinedTimestampSizeType: + case kRecyclableUserDefinedTimestampSizeType: { + if (in_fragmented_record && !scratch->empty()) { + ReportCorruption( + scratch->size(), + "user-defined timestamp size record interspersed partial record"); + } + prospective_record_offset = physical_record_offset; + scratch->clear(); + last_record_offset_ = prospective_record_offset; + UserDefinedTimestampSizeRecord ts_record; + Status s = ts_record.DecodeFrom(&fragment); + if (!s.ok()) { + ReportCorruption( + fragment.size(), + "could not decode user-defined timestamp size record"); + } else { + s = UpdateRecordedTimestampSize( + ts_record.GetUserDefinedTimestampSize()); + if (!s.ok()) { + ReportCorruption(fragment.size(), s.getState()); + } + } + break; + } + + case kBadHeader: + if (wal_recovery_mode == WALRecoveryMode::kAbsoluteConsistency || + wal_recovery_mode == WALRecoveryMode::kPointInTimeRecovery) { + // In clean shutdown we don't expect any error in the log files. + // In point-in-time recovery an incomplete record at the end could + // produce a hole in the recovered data. Report an error here, which + // higher layers can choose to ignore when it's provable there is no + // hole. + ReportCorruption(drop_size, "truncated header"); + } + FALLTHROUGH_INTENDED; + + case kEof: + if (in_fragmented_record) { + if (wal_recovery_mode == WALRecoveryMode::kAbsoluteConsistency || + wal_recovery_mode == WALRecoveryMode::kPointInTimeRecovery) { + // In clean shutdown we don't expect any error in the log files. + // In point-in-time recovery an incomplete record at the end could + // produce a hole in the recovered data. Report an error here, which + // higher layers can choose to ignore when it's provable there is no + // hole. + ReportCorruption(scratch->size(), "error reading trailing data"); + } + // This can be caused by the writer dying immediately after + // writing a physical record but before completing the next; don't + // treat it as a corruption, just ignore the entire logical record. + scratch->clear(); + } + return false; + + case kOldRecord: + if (wal_recovery_mode != WALRecoveryMode::kSkipAnyCorruptedRecords) { + // Treat a record from a previous instance of the log as EOF. + if (in_fragmented_record) { + if (wal_recovery_mode == WALRecoveryMode::kAbsoluteConsistency || + wal_recovery_mode == WALRecoveryMode::kPointInTimeRecovery) { + // In clean shutdown we don't expect any error in the log files. + // In point-in-time recovery an incomplete record at the end could + // produce a hole in the recovered data. Report an error here, + // which higher layers can choose to ignore when it's provable + // there is no hole. + ReportCorruption(scratch->size(), "error reading trailing data"); + } + // This can be caused by the writer dying immediately after + // writing a physical record but before completing the next; don't + // treat it as a corruption, just ignore the entire logical record. + scratch->clear(); + } + return false; + } + FALLTHROUGH_INTENDED; + + case kBadRecord: + if (in_fragmented_record) { + ReportCorruption(scratch->size(), "error in middle of record"); + in_fragmented_record = false; + scratch->clear(); + } + break; + + case kBadRecordLen: + if (eof_) { + if (wal_recovery_mode == WALRecoveryMode::kAbsoluteConsistency || + wal_recovery_mode == WALRecoveryMode::kPointInTimeRecovery) { + // In clean shutdown we don't expect any error in the log files. + // In point-in-time recovery an incomplete record at the end could + // produce a hole in the recovered data. Report an error here, which + // higher layers can choose to ignore when it's provable there is no + // hole. + ReportCorruption(drop_size, "truncated record body"); + } + return false; + } + FALLTHROUGH_INTENDED; + + case kBadRecordChecksum: + if (recycled_ && wal_recovery_mode == + WALRecoveryMode::kTolerateCorruptedTailRecords) { + scratch->clear(); + return false; + } + if (record_type == kBadRecordLen) { + ReportCorruption(drop_size, "bad record length"); + } else { + ReportCorruption(drop_size, "checksum mismatch"); + } + if (in_fragmented_record) { + ReportCorruption(scratch->size(), "error in middle of record"); + in_fragmented_record = false; + scratch->clear(); + } + break; + + default: { + char buf[40]; + snprintf(buf, sizeof(buf), "unknown record type %u", record_type); + ReportCorruption( + (fragment.size() + (in_fragmented_record ? scratch->size() : 0)), + buf); + in_fragmented_record = false; + scratch->clear(); + break; + } + } + } + return false; +} + +uint64_t Reader::LastRecordOffset() { return last_record_offset_; } + +uint64_t Reader::LastRecordEnd() { + return end_of_buffer_offset_ - buffer_.size(); +} + +void Reader::UnmarkEOF() { + if (read_error_) { + return; + } + eof_ = false; + if (eof_offset_ == 0) { + return; + } + UnmarkEOFInternal(); +} + +void Reader::UnmarkEOFInternal() { + // If the EOF was in the middle of a block (a partial block was read) we have + // to read the rest of the block as ReadPhysicalRecord can only read full + // blocks and expects the file position indicator to be aligned to the start + // of a block. + // + // consumed_bytes + buffer_size() + remaining == kBlockSize + + size_t consumed_bytes = eof_offset_ - buffer_.size(); + size_t remaining = kBlockSize - eof_offset_; + + // backing_store_ is used to concatenate what is left in buffer_ and + // the remainder of the block. If buffer_ already uses backing_store_, + // we just append the new data. + if (buffer_.data() != backing_store_ + consumed_bytes) { + // Buffer_ does not use backing_store_ for storage. + // Copy what is left in buffer_ to backing_store. + memmove(backing_store_ + consumed_bytes, buffer_.data(), buffer_.size()); + } + + Slice read_buffer; + // TODO: rate limit log reader with approriate priority. + // TODO: avoid overcharging rate limiter: + // Note that the Read here might overcharge SequentialFileReader's internal + // rate limiter if priority is not IO_TOTAL, e.g., when there is not enough + // content left until EOF to read. + Status status = + file_->Read(remaining, &read_buffer, backing_store_ + eof_offset_, + Env::IO_TOTAL /* rate_limiter_priority */); + + size_t added = read_buffer.size(); + end_of_buffer_offset_ += added; + + if (!status.ok()) { + if (added > 0) { + ReportDrop(added, status); + } + + read_error_ = true; + return; + } + + if (read_buffer.data() != backing_store_ + eof_offset_) { + // Read did not write to backing_store_ + memmove(backing_store_ + eof_offset_, read_buffer.data(), + read_buffer.size()); + } + + buffer_ = Slice(backing_store_ + consumed_bytes, + eof_offset_ + added - consumed_bytes); + + if (added < remaining) { + eof_ = true; + eof_offset_ += added; + } else { + eof_offset_ = 0; + } +} + +void Reader::ReportCorruption(size_t bytes, const char* reason) { + ReportDrop(bytes, Status::Corruption(reason)); +} + +void Reader::ReportDrop(size_t bytes, const Status& reason) { + if (reporter_ != nullptr) { + reporter_->Corruption(bytes, reason); + } +} + +bool Reader::ReadMore(size_t* drop_size, int* error) { + if (!eof_ && !read_error_) { + // Last read was a full read, so this is a trailer to skip + buffer_.clear(); + // TODO: rate limit log reader with approriate priority. + // TODO: avoid overcharging rate limiter: + // Note that the Read here might overcharge SequentialFileReader's internal + // rate limiter if priority is not IO_TOTAL, e.g., when there is not enough + // content left until EOF to read. + Status status = file_->Read(kBlockSize, &buffer_, backing_store_, + Env::IO_TOTAL /* rate_limiter_priority */); + TEST_SYNC_POINT_CALLBACK("LogReader::ReadMore:AfterReadFile", &status); + end_of_buffer_offset_ += buffer_.size(); + if (!status.ok()) { + buffer_.clear(); + ReportDrop(kBlockSize, status); + read_error_ = true; + *error = kEof; + return false; + } else if (buffer_.size() < static_cast(kBlockSize)) { + eof_ = true; + eof_offset_ = buffer_.size(); + } + return true; + } else { + // Note that if buffer_ is non-empty, we have a truncated header at the + // end of the file, which can be caused by the writer crashing in the + // middle of writing the header. Unless explicitly requested we don't + // considering this an error, just report EOF. + if (buffer_.size()) { + *drop_size = buffer_.size(); + buffer_.clear(); + *error = kBadHeader; + return false; + } + buffer_.clear(); + *error = kEof; + return false; + } +} + +unsigned int Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size, + uint64_t* fragment_checksum) { + while (true) { + // We need at least the minimum header size + if (buffer_.size() < static_cast(kHeaderSize)) { + // the default value of r is meaningless because ReadMore will overwrite + // it if it returns false; in case it returns true, the return value will + // not be used anyway + int r = kEof; + if (!ReadMore(drop_size, &r)) { + return r; + } + continue; + } + + // Parse the header + const char* header = buffer_.data(); + const uint32_t a = static_cast(header[4]) & 0xff; + const uint32_t b = static_cast(header[5]) & 0xff; + const unsigned int type = header[6]; + const uint32_t length = a | (b << 8); + int header_size = kHeaderSize; + if ((type >= kRecyclableFullType && type <= kRecyclableLastType) || + type == kRecyclableUserDefinedTimestampSizeType) { + if (end_of_buffer_offset_ - buffer_.size() == 0) { + recycled_ = true; + } + header_size = kRecyclableHeaderSize; + // We need enough for the larger header + if (buffer_.size() < static_cast(kRecyclableHeaderSize)) { + int r = kEof; + if (!ReadMore(drop_size, &r)) { + return r; + } + continue; + } + const uint32_t log_num = DecodeFixed32(header + 7); + if (log_num != log_number_) { + return kOldRecord; + } + } + if (header_size + length > buffer_.size()) { + assert(buffer_.size() >= static_cast(header_size)); + *drop_size = buffer_.size(); + buffer_.clear(); + // If the end of the read has been reached without seeing + // `header_size + length` bytes of payload, report a corruption. The + // higher layers can decide how to handle it based on the recovery mode, + // whether this occurred at EOF, whether this is the final WAL, etc. + return kBadRecordLen; + } + + if (type == kZeroType && length == 0) { + // Skip zero length record without reporting any drops since + // such records are produced by the mmap based writing code in + // env_posix.cc that preallocates file regions. + // NOTE: this should never happen in DB written by new RocksDB versions, + // since we turn off mmap writes to manifest and log files + buffer_.clear(); + return kBadRecord; + } + + // Check crc + if (checksum_) { + uint32_t expected_crc = crc32c::Unmask(DecodeFixed32(header)); + uint32_t actual_crc = crc32c::Value(header + 6, length + header_size - 6); + if (actual_crc != expected_crc) { + // Drop the rest of the buffer since "length" itself may have + // been corrupted and if we trust it, we could find some + // fragment of a real log record that just happens to look + // like a valid log record. + *drop_size = buffer_.size(); + buffer_.clear(); + return kBadRecordChecksum; + } + } + + buffer_.remove_prefix(header_size + length); + + if (!uncompress_ || type == kSetCompressionType || + type == kUserDefinedTimestampSizeType || + type == kRecyclableUserDefinedTimestampSizeType) { + *result = Slice(header + header_size, length); + return type; + } else { + // Uncompress compressed records + uncompressed_record_.clear(); + if (fragment_checksum != nullptr) { + if (uncompress_hash_state_ == nullptr) { + uncompress_hash_state_ = XXH3_createState(); + } + XXH3_64bits_reset(uncompress_hash_state_); + } + + size_t uncompressed_size = 0; + int remaining = 0; + const char* input = header + header_size; + do { + remaining = uncompress_->Uncompress( + input, length, uncompressed_buffer_.get(), &uncompressed_size); + input = nullptr; + if (remaining < 0) { + buffer_.clear(); + return kBadRecord; + } + if (uncompressed_size > 0) { + if (fragment_checksum != nullptr) { + XXH3_64bits_update(uncompress_hash_state_, + uncompressed_buffer_.get(), uncompressed_size); + } + uncompressed_record_.append(uncompressed_buffer_.get(), + uncompressed_size); + } + } while (remaining > 0 || uncompressed_size == kBlockSize); + + if (fragment_checksum != nullptr) { + // We can remove this check by updating hash_state_ directly, + // but that requires resetting hash_state_ for full and first types + // for edge cases like consecutive fist type records. + // Leaving the check as is since it is cleaner and can revert to the + // above approach if it causes performance impact. + *fragment_checksum = XXH3_64bits_digest(uncompress_hash_state_); + uint64_t actual_checksum = XXH3_64bits(uncompressed_record_.data(), + uncompressed_record_.size()); + if (*fragment_checksum != actual_checksum) { + // uncompressed_record_ contains bad content that does not match + // actual decompressed content + return kBadRecord; + } + } + *result = Slice(uncompressed_record_); + return type; + } + } +} + +// Initialize uncompress related fields +void Reader::InitCompression(const CompressionTypeRecord& compression_record) { + compression_type_ = compression_record.GetCompressionType(); + compression_type_record_read_ = true; + constexpr uint32_t compression_format_version = 2; + uncompress_ = StreamingUncompress::Create( + compression_type_, compression_format_version, kBlockSize); + assert(uncompress_ != nullptr); + uncompressed_buffer_ = std::unique_ptr(new char[kBlockSize]); + assert(uncompressed_buffer_); +} + +Status Reader::UpdateRecordedTimestampSize( + const std::vector>& cf_to_ts_sz) { + for (const auto& [cf, ts_sz] : cf_to_ts_sz) { + // Zero user-defined timestamp size are not recorded. + if (ts_sz == 0) { + return Status::Corruption( + "User-defined timestamp size record contains zero timestamp size."); + } + // The user-defined timestamp size record for a column family should not be + // updated in the same log file. + if (recorded_cf_to_ts_sz_.count(cf) != 0) { + return Status::Corruption( + "User-defined timestamp size record contains update to " + "recorded column family."); + } + recorded_cf_to_ts_sz_.insert(std::make_pair(cf, ts_sz)); + } + return Status::OK(); +} + +bool FragmentBufferedReader::ReadRecord(Slice* record, std::string* scratch, + WALRecoveryMode /*unused*/, + uint64_t* /* checksum */) { + assert(record != nullptr); + assert(scratch != nullptr); + record->clear(); + scratch->clear(); + if (uncompress_) { + uncompress_->Reset(); + } + + uint64_t prospective_record_offset = 0; + uint64_t physical_record_offset = end_of_buffer_offset_ - buffer_.size(); + size_t drop_size = 0; + unsigned int fragment_type_or_err = 0; // Initialize to make compiler happy + Slice fragment; + while (TryReadFragment(&fragment, &drop_size, &fragment_type_or_err)) { + switch (fragment_type_or_err) { + case kFullType: + case kRecyclableFullType: + if (in_fragmented_record_ && !fragments_.empty()) { + ReportCorruption(fragments_.size(), "partial record without end(1)"); + } + fragments_.clear(); + *record = fragment; + prospective_record_offset = physical_record_offset; + last_record_offset_ = prospective_record_offset; + first_record_read_ = true; + in_fragmented_record_ = false; + return true; + + case kFirstType: + case kRecyclableFirstType: + if (in_fragmented_record_ || !fragments_.empty()) { + ReportCorruption(fragments_.size(), "partial record without end(2)"); + } + prospective_record_offset = physical_record_offset; + fragments_.assign(fragment.data(), fragment.size()); + in_fragmented_record_ = true; + break; + + case kMiddleType: + case kRecyclableMiddleType: + if (!in_fragmented_record_) { + ReportCorruption(fragment.size(), + "missing start of fragmented record(1)"); + } else { + fragments_.append(fragment.data(), fragment.size()); + } + break; + + case kLastType: + case kRecyclableLastType: + if (!in_fragmented_record_) { + ReportCorruption(fragment.size(), + "missing start of fragmented record(2)"); + } else { + fragments_.append(fragment.data(), fragment.size()); + scratch->assign(fragments_.data(), fragments_.size()); + fragments_.clear(); + *record = Slice(*scratch); + last_record_offset_ = prospective_record_offset; + first_record_read_ = true; + in_fragmented_record_ = false; + return true; + } + break; + + case kSetCompressionType: { + if (compression_type_record_read_) { + ReportCorruption(fragment.size(), + "read multiple SetCompressionType records"); + } + if (first_record_read_) { + ReportCorruption(fragment.size(), + "SetCompressionType not the first record"); + } + fragments_.clear(); + prospective_record_offset = physical_record_offset; + last_record_offset_ = prospective_record_offset; + in_fragmented_record_ = false; + CompressionTypeRecord compression_record(kNoCompression); + Status s = compression_record.DecodeFrom(&fragment); + if (!s.ok()) { + ReportCorruption(fragment.size(), + "could not decode SetCompressionType record"); + } else { + InitCompression(compression_record); + } + break; + } + + case kUserDefinedTimestampSizeType: + case kRecyclableUserDefinedTimestampSizeType: { + if (in_fragmented_record_ && !scratch->empty()) { + ReportCorruption( + scratch->size(), + "user-defined timestamp size record interspersed partial record"); + } + fragments_.clear(); + prospective_record_offset = physical_record_offset; + last_record_offset_ = prospective_record_offset; + in_fragmented_record_ = false; + UserDefinedTimestampSizeRecord ts_record; + Status s = ts_record.DecodeFrom(&fragment); + if (!s.ok()) { + ReportCorruption( + fragment.size(), + "could not decode user-defined timestamp size record"); + } else { + s = UpdateRecordedTimestampSize( + ts_record.GetUserDefinedTimestampSize()); + if (!s.ok()) { + ReportCorruption(fragment.size(), s.getState()); + } + } + break; + } + + case kBadHeader: + case kBadRecord: + case kEof: + case kOldRecord: + if (in_fragmented_record_) { + ReportCorruption(fragments_.size(), "error in middle of record"); + in_fragmented_record_ = false; + fragments_.clear(); + } + break; + + case kBadRecordChecksum: + if (recycled_) { + fragments_.clear(); + return false; + } + ReportCorruption(drop_size, "checksum mismatch"); + if (in_fragmented_record_) { + ReportCorruption(fragments_.size(), "error in middle of record"); + in_fragmented_record_ = false; + fragments_.clear(); + } + break; + + default: { + char buf[40]; + snprintf(buf, sizeof(buf), "unknown record type %u", + fragment_type_or_err); + ReportCorruption( + fragment.size() + (in_fragmented_record_ ? fragments_.size() : 0), + buf); + in_fragmented_record_ = false; + fragments_.clear(); + break; + } + } + } + return false; +} + +void FragmentBufferedReader::UnmarkEOF() { + if (read_error_) { + return; + } + eof_ = false; + UnmarkEOFInternal(); +} + +bool FragmentBufferedReader::TryReadMore(size_t* drop_size, int* error) { + if (!eof_ && !read_error_) { + // Last read was a full read, so this is a trailer to skip + buffer_.clear(); + // TODO: rate limit log reader with approriate priority. + // TODO: avoid overcharging rate limiter: + // Note that the Read here might overcharge SequentialFileReader's internal + // rate limiter if priority is not IO_TOTAL, e.g., when there is not enough + // content left until EOF to read. + Status status = file_->Read(kBlockSize, &buffer_, backing_store_, + Env::IO_TOTAL /* rate_limiter_priority */); + end_of_buffer_offset_ += buffer_.size(); + if (!status.ok()) { + buffer_.clear(); + ReportDrop(kBlockSize, status); + read_error_ = true; + *error = kEof; + return false; + } else if (buffer_.size() < static_cast(kBlockSize)) { + eof_ = true; + eof_offset_ = buffer_.size(); + TEST_SYNC_POINT_CALLBACK( + "FragmentBufferedLogReader::TryReadMore:FirstEOF", nullptr); + } + return true; + } else if (!read_error_) { + UnmarkEOF(); + } + if (!read_error_) { + return true; + } + *error = kEof; + *drop_size = buffer_.size(); + if (buffer_.size() > 0) { + *error = kBadHeader; + } + buffer_.clear(); + return false; +} + +// return true if the caller should process the fragment_type_or_err. +bool FragmentBufferedReader::TryReadFragment( + Slice* fragment, size_t* drop_size, unsigned int* fragment_type_or_err) { + assert(fragment != nullptr); + assert(drop_size != nullptr); + assert(fragment_type_or_err != nullptr); + + while (buffer_.size() < static_cast(kHeaderSize)) { + size_t old_size = buffer_.size(); + int error = kEof; + if (!TryReadMore(drop_size, &error)) { + *fragment_type_or_err = error; + return false; + } else if (old_size == buffer_.size()) { + return false; + } + } + const char* header = buffer_.data(); + const uint32_t a = static_cast(header[4]) & 0xff; + const uint32_t b = static_cast(header[5]) & 0xff; + const unsigned int type = header[6]; + const uint32_t length = a | (b << 8); + int header_size = kHeaderSize; + if ((type >= kRecyclableFullType && type <= kRecyclableLastType) || + type == kRecyclableUserDefinedTimestampSizeType) { + if (end_of_buffer_offset_ - buffer_.size() == 0) { + recycled_ = true; + } + header_size = kRecyclableHeaderSize; + while (buffer_.size() < static_cast(kRecyclableHeaderSize)) { + size_t old_size = buffer_.size(); + int error = kEof; + if (!TryReadMore(drop_size, &error)) { + *fragment_type_or_err = error; + return false; + } else if (old_size == buffer_.size()) { + return false; + } + } + const uint32_t log_num = DecodeFixed32(header + 7); + if (log_num != log_number_) { + *fragment_type_or_err = kOldRecord; + return true; + } + } + + while (header_size + length > buffer_.size()) { + size_t old_size = buffer_.size(); + int error = kEof; + if (!TryReadMore(drop_size, &error)) { + *fragment_type_or_err = error; + return false; + } else if (old_size == buffer_.size()) { + return false; + } + } + + if (type == kZeroType && length == 0) { + buffer_.clear(); + *fragment_type_or_err = kBadRecord; + return true; + } + + if (checksum_) { + uint32_t expected_crc = crc32c::Unmask(DecodeFixed32(header)); + uint32_t actual_crc = crc32c::Value(header + 6, length + header_size - 6); + if (actual_crc != expected_crc) { + *drop_size = buffer_.size(); + buffer_.clear(); + *fragment_type_or_err = kBadRecordChecksum; + return true; + } + } + + buffer_.remove_prefix(header_size + length); + + if (!uncompress_ || type == kSetCompressionType || + type == kUserDefinedTimestampSizeType || + type == kRecyclableUserDefinedTimestampSizeType) { + *fragment = Slice(header + header_size, length); + *fragment_type_or_err = type; + return true; + } else { + // Uncompress compressed records + uncompressed_record_.clear(); + size_t uncompressed_size = 0; + int remaining = 0; + const char* input = header + header_size; + do { + remaining = uncompress_->Uncompress( + input, length, uncompressed_buffer_.get(), &uncompressed_size); + input = nullptr; + if (remaining < 0) { + buffer_.clear(); + *fragment_type_or_err = kBadRecord; + return true; + } + if (uncompressed_size > 0) { + uncompressed_record_.append(uncompressed_buffer_.get(), + uncompressed_size); + } + } while (remaining > 0 || uncompressed_size == kBlockSize); + *fragment = Slice(std::move(uncompressed_record_)); + *fragment_type_or_err = type; + return true; + } +} + +} // namespace log +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/log_reader.h b/librocksdb-sys/rocksdb/db/log_reader.h new file mode 100644 index 0000000..697d1b5 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/log_reader.h @@ -0,0 +1,242 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include + +#include +#include +#include + +#include "db/log_format.h" +#include "file/sequence_file_reader.h" +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "util/compression.h" +#include "util/hash_containers.h" +#include "util/udt_util.h" +#include "util/xxhash.h" + +namespace ROCKSDB_NAMESPACE { +class Logger; + +namespace log { + +/** + * Reader is a general purpose log stream reader implementation. The actual job + * of reading from the device is implemented by the SequentialFile interface. + * + * Please see Writer for details on the file and record layout. + */ +class Reader { + public: + // Interface for reporting errors. + class Reporter { + public: + virtual ~Reporter(); + + // Some corruption was detected. "size" is the approximate number + // of bytes dropped due to the corruption. + virtual void Corruption(size_t bytes, const Status& status) = 0; + }; + + // Create a reader that will return log records from "*file". + // "*file" must remain live while this Reader is in use. + // + // If "reporter" is non-nullptr, it is notified whenever some data is + // dropped due to a detected corruption. "*reporter" must remain + // live while this Reader is in use. + // + // If "checksum" is true, verify checksums if available. + Reader(std::shared_ptr info_log, + std::unique_ptr&& file, Reporter* reporter, + bool checksum, uint64_t log_num); + // No copying allowed + Reader(const Reader&) = delete; + void operator=(const Reader&) = delete; + + virtual ~Reader(); + + // Read the next record into *record. Returns true if read + // successfully, false if we hit end of the input. May use + // "*scratch" as temporary storage. The contents filled in *record + // will only be valid until the next mutating operation on this + // reader or the next mutation to *scratch. + // If record_checksum is not nullptr, then this function will calculate the + // checksum of the record read and set record_checksum to it. The checksum is + // calculated from the original buffers that contain the contents of the + // record. + virtual bool ReadRecord(Slice* record, std::string* scratch, + WALRecoveryMode wal_recovery_mode = + WALRecoveryMode::kTolerateCorruptedTailRecords, + uint64_t* record_checksum = nullptr); + + // Return the recorded user-defined timestamp size that have been read so + // far. This only applies to WAL logs. + const UnorderedMap& GetRecordedTimestampSize() const { + return recorded_cf_to_ts_sz_; + } + + // Returns the physical offset of the last record returned by ReadRecord. + // + // Undefined before the first call to ReadRecord. + uint64_t LastRecordOffset(); + + // Returns the first physical offset after the last record returned by + // ReadRecord, or zero before first call to ReadRecord. This can also be + // thought of as the "current" position in processing the file bytes. + uint64_t LastRecordEnd(); + + // returns true if the reader has encountered an eof condition. + bool IsEOF() { return eof_; } + + // returns true if the reader has encountered read error. + bool hasReadError() const { return read_error_; } + + // when we know more data has been written to the file. we can use this + // function to force the reader to look again in the file. + // Also aligns the file position indicator to the start of the next block + // by reading the rest of the data from the EOF position to the end of the + // block that was partially read. + virtual void UnmarkEOF(); + + SequentialFileReader* file() { return file_.get(); } + + Reporter* GetReporter() const { return reporter_; } + + uint64_t GetLogNumber() const { return log_number_; } + + size_t GetReadOffset() const { + return static_cast(end_of_buffer_offset_); + } + + bool IsCompressedAndEmptyFile() { + return !first_record_read_ && compression_type_record_read_; + } + + protected: + std::shared_ptr info_log_; + const std::unique_ptr file_; + Reporter* const reporter_; + bool const checksum_; + char* const backing_store_; + + // Internal state variables used for reading records + Slice buffer_; + bool eof_; // Last Read() indicated EOF by returning < kBlockSize + bool read_error_; // Error occurred while reading from file + + // Offset of the file position indicator within the last block when an + // EOF was detected. + size_t eof_offset_; + + // Offset of the last record returned by ReadRecord. + uint64_t last_record_offset_; + // Offset of the first location past the end of buffer_. + uint64_t end_of_buffer_offset_; + + // which log number this is + uint64_t const log_number_; + + // Whether this is a recycled log file + bool recycled_; + + // Whether the first record has been read or not. + bool first_record_read_; + // Type of compression used + CompressionType compression_type_; + // Track whether the compression type record has been read or not. + bool compression_type_record_read_; + StreamingUncompress* uncompress_; + // Reusable uncompressed output buffer + std::unique_ptr uncompressed_buffer_; + // Reusable uncompressed record + std::string uncompressed_record_; + // Used for stream hashing fragment content in ReadRecord() + XXH3_state_t* hash_state_; + // Used for stream hashing uncompressed buffer in ReadPhysicalRecord() + XXH3_state_t* uncompress_hash_state_; + + // The recorded user-defined timestamp sizes that have been read so far. This + // is only for WAL logs. + UnorderedMap recorded_cf_to_ts_sz_; + + // Extend record types with the following special values + enum { + kEof = kMaxRecordType + 1, + // Returned whenever we find an invalid physical record. + // Currently there are three situations in which this happens: + // * The record has an invalid CRC (ReadPhysicalRecord reports a drop) + // * The record is a 0-length record (No drop is reported) + kBadRecord = kMaxRecordType + 2, + // Returned when we fail to read a valid header. + kBadHeader = kMaxRecordType + 3, + // Returned when we read an old record from a previous user of the log. + kOldRecord = kMaxRecordType + 4, + // Returned when we get a bad record length + kBadRecordLen = kMaxRecordType + 5, + // Returned when we get a bad record checksum + kBadRecordChecksum = kMaxRecordType + 6, + }; + + // Return type, or one of the preceding special values + // If WAL compressioned is enabled, fragment_checksum is the checksum of the + // fragment computed from the orginal buffer containinng uncompressed + // fragment. + unsigned int ReadPhysicalRecord(Slice* result, size_t* drop_size, + uint64_t* fragment_checksum = nullptr); + + // Read some more + bool ReadMore(size_t* drop_size, int* error); + + void UnmarkEOFInternal(); + + // Reports dropped bytes to the reporter. + // buffer_ must be updated to remove the dropped bytes prior to invocation. + void ReportCorruption(size_t bytes, const char* reason); + void ReportDrop(size_t bytes, const Status& reason); + + void InitCompression(const CompressionTypeRecord& compression_record); + + Status UpdateRecordedTimestampSize( + const std::vector>& cf_to_ts_sz); +}; + +class FragmentBufferedReader : public Reader { + public: + FragmentBufferedReader(std::shared_ptr info_log, + std::unique_ptr&& _file, + Reporter* reporter, bool checksum, uint64_t log_num) + : Reader(info_log, std::move(_file), reporter, checksum, log_num), + fragments_(), + in_fragmented_record_(false) {} + ~FragmentBufferedReader() override {} + bool ReadRecord(Slice* record, std::string* scratch, + WALRecoveryMode wal_recovery_mode = + WALRecoveryMode::kTolerateCorruptedTailRecords, + uint64_t* record_checksum = nullptr) override; + void UnmarkEOF() override; + + private: + std::string fragments_; + bool in_fragmented_record_; + + bool TryReadFragment(Slice* result, size_t* drop_size, + unsigned int* fragment_type_or_err); + + bool TryReadMore(size_t* drop_size, int* error); + + // No copy allowed + FragmentBufferedReader(const FragmentBufferedReader&); + void operator=(const FragmentBufferedReader&); +}; + +} // namespace log +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/log_test.cc b/librocksdb-sys/rocksdb/db/log_test.cc new file mode 100644 index 0000000..fa5e2aa --- /dev/null +++ b/librocksdb-sys/rocksdb/db/log_test.cc @@ -0,0 +1,1212 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/log_reader.h" +#include "db/log_writer.h" +#include "file/sequence_file_reader.h" +#include "file/writable_file_writer.h" +#include "rocksdb/env.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/coding.h" +#include "util/crc32c.h" +#include "util/random.h" +#include "utilities/memory_allocators.h" + +namespace ROCKSDB_NAMESPACE { +namespace log { + +// Construct a string of the specified length made out of the supplied +// partial string. +static std::string BigString(const std::string& partial_string, size_t n) { + std::string result; + while (result.size() < n) { + result.append(partial_string); + } + result.resize(n); + return result; +} + +// Construct a string from a number +static std::string NumberString(int n) { + char buf[50]; + snprintf(buf, sizeof(buf), "%d.", n); + return std::string(buf); +} + +// Return a skewed potentially long string +static std::string RandomSkewedString(int i, Random* rnd) { + return BigString(NumberString(i), rnd->Skewed(17)); +} + +// Param type is tuple +// get<0>(tuple): non-zero if recycling log, zero if regular log +// get<1>(tuple): true if allow retry after read EOF, false otherwise +// get<2>(tuple): type of compression used +class LogTest + : public ::testing::TestWithParam> { + private: + class StringSource : public FSSequentialFile { + public: + Slice& contents_; + bool force_error_; + size_t force_error_position_; + bool force_eof_; + size_t force_eof_position_; + bool returned_partial_; + bool fail_after_read_partial_; + explicit StringSource(Slice& contents, bool fail_after_read_partial) + : contents_(contents), + force_error_(false), + force_error_position_(0), + force_eof_(false), + force_eof_position_(0), + returned_partial_(false), + fail_after_read_partial_(fail_after_read_partial) {} + + IOStatus Read(size_t n, const IOOptions& /*opts*/, Slice* result, + char* scratch, IODebugContext* /*dbg*/) override { + if (fail_after_read_partial_) { + EXPECT_TRUE(!returned_partial_) << "must not Read() after eof/error"; + } + + if (force_error_) { + if (force_error_position_ >= n) { + force_error_position_ -= n; + } else { + *result = Slice(contents_.data(), force_error_position_); + contents_.remove_prefix(force_error_position_); + force_error_ = false; + returned_partial_ = true; + return IOStatus::Corruption("read error"); + } + } + + if (contents_.size() < n) { + n = contents_.size(); + returned_partial_ = true; + } + + if (force_eof_) { + if (force_eof_position_ >= n) { + force_eof_position_ -= n; + } else { + force_eof_ = false; + n = force_eof_position_; + returned_partial_ = true; + } + } + + // By using scratch we ensure that caller has control over the + // lifetime of result.data() + memcpy(scratch, contents_.data(), n); + *result = Slice(scratch, n); + + contents_.remove_prefix(n); + return IOStatus::OK(); + } + + IOStatus Skip(uint64_t n) override { + if (n > contents_.size()) { + contents_.clear(); + return IOStatus::NotFound("in-memory file skipepd past end"); + } + + contents_.remove_prefix(n); + + return IOStatus::OK(); + } + }; + + class ReportCollector : public Reader::Reporter { + public: + size_t dropped_bytes_; + std::string message_; + + ReportCollector() : dropped_bytes_(0) {} + void Corruption(size_t bytes, const Status& status) override { + dropped_bytes_ += bytes; + message_.append(status.ToString()); + } + }; + + std::string& dest_contents() { return sink_->contents_; } + + const std::string& dest_contents() const { return sink_->contents_; } + + void reset_source_contents() { source_->contents_ = dest_contents(); } + + Slice reader_contents_; + test::StringSink* sink_; + StringSource* source_; + ReportCollector report_; + + protected: + std::unique_ptr writer_; + std::unique_ptr reader_; + bool allow_retry_read_; + CompressionType compression_type_; + + public: + LogTest() + : reader_contents_(), + sink_(new test::StringSink(&reader_contents_)), + source_(new StringSource(reader_contents_, !std::get<1>(GetParam()))), + allow_retry_read_(std::get<1>(GetParam())), + compression_type_(std::get<2>(GetParam())) { + std::unique_ptr sink_holder(sink_); + std::unique_ptr file_writer(new WritableFileWriter( + std::move(sink_holder), "" /* don't care */, FileOptions())); + Writer* writer = + new Writer(std::move(file_writer), 123, std::get<0>(GetParam()), false, + compression_type_); + writer_.reset(writer); + std::unique_ptr source_holder(source_); + std::unique_ptr file_reader( + new SequentialFileReader(std::move(source_holder), "" /* file name */)); + if (allow_retry_read_) { + reader_.reset(new FragmentBufferedReader(nullptr, std::move(file_reader), + &report_, true /* checksum */, + 123 /* log_number */)); + } else { + reader_.reset(new Reader(nullptr, std::move(file_reader), &report_, + true /* checksum */, 123 /* log_number */)); + } + } + + Slice* get_reader_contents() { return &reader_contents_; } + + void Write(const std::string& msg, + const UnorderedMap* cf_to_ts_sz = nullptr) { + if (cf_to_ts_sz != nullptr && !cf_to_ts_sz->empty()) { + ASSERT_OK(writer_->MaybeAddUserDefinedTimestampSizeRecord(*cf_to_ts_sz)); + } + ASSERT_OK(writer_->AddRecord(Slice(msg))); + } + + size_t WrittenBytes() const { return dest_contents().size(); } + + std::string Read(const WALRecoveryMode wal_recovery_mode = + WALRecoveryMode::kTolerateCorruptedTailRecords, + UnorderedMap* cf_to_ts_sz = nullptr) { + std::string scratch; + Slice record; + bool ret = false; + uint64_t record_checksum; + ret = reader_->ReadRecord(&record, &scratch, wal_recovery_mode, + &record_checksum); + if (cf_to_ts_sz != nullptr) { + *cf_to_ts_sz = reader_->GetRecordedTimestampSize(); + } + if (ret) { + if (!allow_retry_read_) { + // allow_retry_read_ means using FragmentBufferedReader which does not + // support record checksum yet. + uint64_t actual_record_checksum = + XXH3_64bits(record.data(), record.size()); + assert(actual_record_checksum == record_checksum); + } + return record.ToString(); + } else { + return "EOF"; + } + } + + void IncrementByte(int offset, char delta) { + dest_contents()[offset] += delta; + } + + void SetByte(int offset, char new_byte) { + dest_contents()[offset] = new_byte; + } + + void ShrinkSize(int bytes) { sink_->Drop(bytes); } + + void FixChecksum(int header_offset, int len, bool recyclable) { + // Compute crc of type/len/data + int header_size = recyclable ? kRecyclableHeaderSize : kHeaderSize; + uint32_t crc = crc32c::Value(&dest_contents()[header_offset + 6], + header_size - 6 + len); + crc = crc32c::Mask(crc); + EncodeFixed32(&dest_contents()[header_offset], crc); + } + + void ForceError(size_t position = 0) { + source_->force_error_ = true; + source_->force_error_position_ = position; + } + + size_t DroppedBytes() const { return report_.dropped_bytes_; } + + std::string ReportMessage() const { return report_.message_; } + + void ForceEOF(size_t position = 0) { + source_->force_eof_ = true; + source_->force_eof_position_ = position; + } + + void UnmarkEOF() { + source_->returned_partial_ = false; + reader_->UnmarkEOF(); + } + + bool IsEOF() { return reader_->IsEOF(); } + + // Returns OK iff recorded error message contains "msg" + std::string MatchError(const std::string& msg) const { + if (report_.message_.find(msg) == std::string::npos) { + return report_.message_; + } else { + return "OK"; + } + } + + void CheckRecordAndTimestampSize( + std::string record, UnorderedMap& expected_ts_sz) { + UnorderedMap recorded_ts_sz; + ASSERT_EQ(record, + Read(WALRecoveryMode:: + kTolerateCorruptedTailRecords /* wal_recovery_mode */, + &recorded_ts_sz)); + EXPECT_EQ(expected_ts_sz, recorded_ts_sz); + } +}; + +TEST_P(LogTest, Empty) { ASSERT_EQ("EOF", Read()); } + +TEST_P(LogTest, ReadWrite) { + Write("foo"); + Write("bar"); + Write(""); + Write("xxxx"); + ASSERT_EQ("foo", Read()); + ASSERT_EQ("bar", Read()); + ASSERT_EQ("", Read()); + ASSERT_EQ("xxxx", Read()); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ("EOF", Read()); // Make sure reads at eof work +} + +TEST_P(LogTest, ReadWriteWithTimestampSize) { + UnorderedMap ts_sz_one = { + {1, sizeof(uint64_t)}, + }; + Write("foo", &ts_sz_one); + Write("bar"); + UnorderedMap ts_sz_two = {{2, sizeof(char)}}; + Write("", &ts_sz_two); + Write("xxxx"); + + CheckRecordAndTimestampSize("foo", ts_sz_one); + CheckRecordAndTimestampSize("bar", ts_sz_one); + UnorderedMap expected_ts_sz_two; + // User-defined timestamp size records are accumulated and applied to + // subsequent records. + expected_ts_sz_two.insert(ts_sz_one.begin(), ts_sz_one.end()); + expected_ts_sz_two.insert(ts_sz_two.begin(), ts_sz_two.end()); + CheckRecordAndTimestampSize("", expected_ts_sz_two); + CheckRecordAndTimestampSize("xxxx", expected_ts_sz_two); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ("EOF", Read()); // Make sure reads at eof work +} + +TEST_P(LogTest, ReadWriteWithTimestampSizeZeroTimestampIgnored) { + UnorderedMap ts_sz_one = {{1, sizeof(uint64_t)}}; + Write("foo", &ts_sz_one); + UnorderedMap ts_sz_two(ts_sz_one.begin(), ts_sz_one.end()); + ts_sz_two.insert(std::make_pair(2, 0)); + Write("bar", &ts_sz_two); + + CheckRecordAndTimestampSize("foo", ts_sz_one); + CheckRecordAndTimestampSize("bar", ts_sz_one); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ("EOF", Read()); // Make sure reads at eof work +} + +TEST_P(LogTest, ManyBlocks) { + for (int i = 0; i < 100000; i++) { + Write(NumberString(i)); + } + for (int i = 0; i < 100000; i++) { + ASSERT_EQ(NumberString(i), Read()); + } + ASSERT_EQ("EOF", Read()); +} + +TEST_P(LogTest, Fragmentation) { + Write("small"); + Write(BigString("medium", 50000)); + Write(BigString("large", 100000)); + ASSERT_EQ("small", Read()); + ASSERT_EQ(BigString("medium", 50000), Read()); + ASSERT_EQ(BigString("large", 100000), Read()); + ASSERT_EQ("EOF", Read()); +} + +TEST_P(LogTest, MarginalTrailer) { + // Make a trailer that is exactly the same length as an empty record. + int header_size = + std::get<0>(GetParam()) ? kRecyclableHeaderSize : kHeaderSize; + const int n = kBlockSize - 2 * header_size; + Write(BigString("foo", n)); + ASSERT_EQ((unsigned int)(kBlockSize - header_size), WrittenBytes()); + Write(""); + Write("bar"); + ASSERT_EQ(BigString("foo", n), Read()); + ASSERT_EQ("", Read()); + ASSERT_EQ("bar", Read()); + ASSERT_EQ("EOF", Read()); +} + +TEST_P(LogTest, MarginalTrailer2) { + // Make a trailer that is exactly the same length as an empty record. + int header_size = + std::get<0>(GetParam()) ? kRecyclableHeaderSize : kHeaderSize; + const int n = kBlockSize - 2 * header_size; + Write(BigString("foo", n)); + ASSERT_EQ((unsigned int)(kBlockSize - header_size), WrittenBytes()); + Write("bar"); + ASSERT_EQ(BigString("foo", n), Read()); + ASSERT_EQ("bar", Read()); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(0U, DroppedBytes()); + ASSERT_EQ("", ReportMessage()); +} + +TEST_P(LogTest, ShortTrailer) { + int header_size = + std::get<0>(GetParam()) ? kRecyclableHeaderSize : kHeaderSize; + const int n = kBlockSize - 2 * header_size + 4; + Write(BigString("foo", n)); + ASSERT_EQ((unsigned int)(kBlockSize - header_size + 4), WrittenBytes()); + Write(""); + Write("bar"); + ASSERT_EQ(BigString("foo", n), Read()); + ASSERT_EQ("", Read()); + ASSERT_EQ("bar", Read()); + ASSERT_EQ("EOF", Read()); +} + +TEST_P(LogTest, AlignedEof) { + int header_size = + std::get<0>(GetParam()) ? kRecyclableHeaderSize : kHeaderSize; + const int n = kBlockSize - 2 * header_size + 4; + Write(BigString("foo", n)); + ASSERT_EQ((unsigned int)(kBlockSize - header_size + 4), WrittenBytes()); + ASSERT_EQ(BigString("foo", n), Read()); + ASSERT_EQ("EOF", Read()); +} + +TEST_P(LogTest, RandomRead) { + const int N = 500; + Random write_rnd(301); + for (int i = 0; i < N; i++) { + Write(RandomSkewedString(i, &write_rnd)); + } + Random read_rnd(301); + for (int i = 0; i < N; i++) { + ASSERT_EQ(RandomSkewedString(i, &read_rnd), Read()); + } + ASSERT_EQ("EOF", Read()); +} + +// Tests of all the error paths in log_reader.cc follow: + +TEST_P(LogTest, ReadError) { + Write("foo"); + ForceError(); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ((unsigned int)kBlockSize, DroppedBytes()); + ASSERT_EQ("OK", MatchError("read error")); +} + +TEST_P(LogTest, BadRecordType) { + Write("foo"); + // Type is stored in header[6] + IncrementByte(6, 100); + FixChecksum(0, 3, false); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(3U, DroppedBytes()); + ASSERT_EQ("OK", MatchError("unknown record type")); +} + +TEST_P(LogTest, TruncatedTrailingRecordIsIgnored) { + Write("foo"); + ShrinkSize(4); // Drop all payload as well as a header byte + ASSERT_EQ("EOF", Read()); + // Truncated last record is ignored, not treated as an error + ASSERT_EQ(0U, DroppedBytes()); + ASSERT_EQ("", ReportMessage()); +} + +TEST_P(LogTest, TruncatedTrailingRecordIsNotIgnored) { + if (allow_retry_read_) { + // If read retry is allowed, then truncated trailing record should not + // raise an error. + return; + } + Write("foo"); + ShrinkSize(4); // Drop all payload as well as a header byte + ASSERT_EQ("EOF", Read(WALRecoveryMode::kAbsoluteConsistency)); + // Truncated last record is ignored, not treated as an error + ASSERT_GT(DroppedBytes(), 0U); + ASSERT_EQ("OK", MatchError("Corruption: truncated header")); +} + +TEST_P(LogTest, BadLength) { + if (allow_retry_read_) { + // If read retry is allowed, then we should not raise an error when the + // record length specified in header is longer than data currently + // available. It's possible that the body of the record is not written yet. + return; + } + bool recyclable_log = (std::get<0>(GetParam()) != 0); + int header_size = recyclable_log ? kRecyclableHeaderSize : kHeaderSize; + const int kPayloadSize = kBlockSize - header_size; + Write(BigString("bar", kPayloadSize)); + Write("foo"); + // Least significant size byte is stored in header[4]. + IncrementByte(4, 1); + if (!recyclable_log) { + ASSERT_EQ("foo", Read()); + ASSERT_EQ(kBlockSize, DroppedBytes()); + ASSERT_EQ("OK", MatchError("bad record length")); + } else { + ASSERT_EQ("EOF", Read()); + } +} + +TEST_P(LogTest, BadLengthAtEndIsIgnored) { + if (allow_retry_read_) { + // If read retry is allowed, then we should not raise an error when the + // record length specified in header is longer than data currently + // available. It's possible that the body of the record is not written yet. + return; + } + Write("foo"); + ShrinkSize(1); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(0U, DroppedBytes()); + ASSERT_EQ("", ReportMessage()); +} + +TEST_P(LogTest, BadLengthAtEndIsNotIgnored) { + if (allow_retry_read_) { + // If read retry is allowed, then we should not raise an error when the + // record length specified in header is longer than data currently + // available. It's possible that the body of the record is not written yet. + return; + } + Write("foo"); + ShrinkSize(1); + ASSERT_EQ("EOF", Read(WALRecoveryMode::kAbsoluteConsistency)); + ASSERT_GT(DroppedBytes(), 0U); + ASSERT_EQ("OK", MatchError("Corruption: truncated record body")); +} + +TEST_P(LogTest, ChecksumMismatch) { + Write("foooooo"); + IncrementByte(0, 14); + ASSERT_EQ("EOF", Read()); + bool recyclable_log = (std::get<0>(GetParam()) != 0); + if (!recyclable_log) { + ASSERT_EQ(14U, DroppedBytes()); + ASSERT_EQ("OK", MatchError("checksum mismatch")); + } else { + ASSERT_EQ(0U, DroppedBytes()); + ASSERT_EQ("", ReportMessage()); + } +} + +TEST_P(LogTest, UnexpectedMiddleType) { + Write("foo"); + bool recyclable_log = (std::get<0>(GetParam()) != 0); + SetByte(6, static_cast(recyclable_log ? kRecyclableMiddleType + : kMiddleType)); + FixChecksum(0, 3, !!recyclable_log); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(3U, DroppedBytes()); + ASSERT_EQ("OK", MatchError("missing start")); +} + +TEST_P(LogTest, UnexpectedLastType) { + Write("foo"); + bool recyclable_log = (std::get<0>(GetParam()) != 0); + SetByte(6, + static_cast(recyclable_log ? kRecyclableLastType : kLastType)); + FixChecksum(0, 3, !!recyclable_log); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(3U, DroppedBytes()); + ASSERT_EQ("OK", MatchError("missing start")); +} + +TEST_P(LogTest, UnexpectedFullType) { + Write("foo"); + Write("bar"); + bool recyclable_log = (std::get<0>(GetParam()) != 0); + SetByte( + 6, static_cast(recyclable_log ? kRecyclableFirstType : kFirstType)); + FixChecksum(0, 3, !!recyclable_log); + ASSERT_EQ("bar", Read()); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(3U, DroppedBytes()); + ASSERT_EQ("OK", MatchError("partial record without end")); +} + +TEST_P(LogTest, UnexpectedFirstType) { + Write("foo"); + Write(BigString("bar", 100000)); + bool recyclable_log = (std::get<0>(GetParam()) != 0); + SetByte( + 6, static_cast(recyclable_log ? kRecyclableFirstType : kFirstType)); + FixChecksum(0, 3, !!recyclable_log); + ASSERT_EQ(BigString("bar", 100000), Read()); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(3U, DroppedBytes()); + ASSERT_EQ("OK", MatchError("partial record without end")); +} + +TEST_P(LogTest, MissingLastIsIgnored) { + Write(BigString("bar", kBlockSize)); + // Remove the LAST block, including header. + ShrinkSize(14); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ("", ReportMessage()); + ASSERT_EQ(0U, DroppedBytes()); +} + +TEST_P(LogTest, MissingLastIsNotIgnored) { + if (allow_retry_read_) { + // If read retry is allowed, then truncated trailing record should not + // raise an error. + return; + } + Write(BigString("bar", kBlockSize)); + // Remove the LAST block, including header. + ShrinkSize(14); + ASSERT_EQ("EOF", Read(WALRecoveryMode::kAbsoluteConsistency)); + ASSERT_GT(DroppedBytes(), 0U); + ASSERT_EQ("OK", MatchError("Corruption: error reading trailing data")); +} + +TEST_P(LogTest, PartialLastIsIgnored) { + Write(BigString("bar", kBlockSize)); + // Cause a bad record length in the LAST block. + ShrinkSize(1); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ("", ReportMessage()); + ASSERT_EQ(0U, DroppedBytes()); +} + +TEST_P(LogTest, PartialLastIsNotIgnored) { + if (allow_retry_read_) { + // If read retry is allowed, then truncated trailing record should not + // raise an error. + return; + } + Write(BigString("bar", kBlockSize)); + // Cause a bad record length in the LAST block. + ShrinkSize(1); + ASSERT_EQ("EOF", Read(WALRecoveryMode::kAbsoluteConsistency)); + ASSERT_GT(DroppedBytes(), 0U); + ASSERT_EQ("OK", MatchError("Corruption: truncated record body")); +} + +TEST_P(LogTest, ErrorJoinsRecords) { + // Consider two fragmented records: + // first(R1) last(R1) first(R2) last(R2) + // where the middle two fragments disappear. We do not want + // first(R1),last(R2) to get joined and returned as a valid record. + + // Write records that span two blocks + Write(BigString("foo", kBlockSize)); + Write(BigString("bar", kBlockSize)); + Write("correct"); + + // Wipe the middle block + for (unsigned int offset = kBlockSize; offset < 2 * kBlockSize; offset++) { + SetByte(offset, 'x'); + } + + bool recyclable_log = (std::get<0>(GetParam()) != 0); + if (!recyclable_log) { + ASSERT_EQ("correct", Read()); + ASSERT_EQ("EOF", Read()); + size_t dropped = DroppedBytes(); + ASSERT_LE(dropped, 2 * kBlockSize + 100); + ASSERT_GE(dropped, 2 * kBlockSize); + } else { + ASSERT_EQ("EOF", Read()); + } +} + +TEST_P(LogTest, ClearEofSingleBlock) { + Write("foo"); + Write("bar"); + bool recyclable_log = (std::get<0>(GetParam()) != 0); + int header_size = recyclable_log ? kRecyclableHeaderSize : kHeaderSize; + ForceEOF(3 + header_size + 2); + ASSERT_EQ("foo", Read()); + UnmarkEOF(); + ASSERT_EQ("bar", Read()); + ASSERT_TRUE(IsEOF()); + ASSERT_EQ("EOF", Read()); + Write("xxx"); + UnmarkEOF(); + ASSERT_EQ("xxx", Read()); + ASSERT_TRUE(IsEOF()); +} + +TEST_P(LogTest, ClearEofMultiBlock) { + size_t num_full_blocks = 5; + bool recyclable_log = (std::get<0>(GetParam()) != 0); + int header_size = recyclable_log ? kRecyclableHeaderSize : kHeaderSize; + size_t n = (kBlockSize - header_size) * num_full_blocks + 25; + Write(BigString("foo", n)); + Write(BigString("bar", n)); + ForceEOF(n + num_full_blocks * header_size + header_size + 3); + ASSERT_EQ(BigString("foo", n), Read()); + ASSERT_TRUE(IsEOF()); + UnmarkEOF(); + ASSERT_EQ(BigString("bar", n), Read()); + ASSERT_TRUE(IsEOF()); + Write(BigString("xxx", n)); + UnmarkEOF(); + ASSERT_EQ(BigString("xxx", n), Read()); + ASSERT_TRUE(IsEOF()); +} + +TEST_P(LogTest, ClearEofError) { + // If an error occurs during Read() in UnmarkEOF(), the records contained + // in the buffer should be returned on subsequent calls of ReadRecord() + // until no more full records are left, whereafter ReadRecord() should return + // false to indicate that it cannot read any further. + + Write("foo"); + Write("bar"); + UnmarkEOF(); + ASSERT_EQ("foo", Read()); + ASSERT_TRUE(IsEOF()); + Write("xxx"); + ForceError(0); + UnmarkEOF(); + ASSERT_EQ("bar", Read()); + ASSERT_EQ("EOF", Read()); +} + +TEST_P(LogTest, ClearEofError2) { + Write("foo"); + Write("bar"); + UnmarkEOF(); + ASSERT_EQ("foo", Read()); + Write("xxx"); + ForceError(3); + UnmarkEOF(); + ASSERT_EQ("bar", Read()); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ(3U, DroppedBytes()); + ASSERT_EQ("OK", MatchError("read error")); +} + +TEST_P(LogTest, Recycle) { + bool recyclable_log = (std::get<0>(GetParam()) != 0); + if (!recyclable_log) { + return; // test is only valid for recycled logs + } + Write("foo"); + Write("bar"); + Write("baz"); + Write("bif"); + Write("blitz"); + while (get_reader_contents()->size() < log::kBlockSize * 2) { + Write("xxxxxxxxxxxxxxxx"); + } + std::unique_ptr sink( + new test::OverwritingStringSink(get_reader_contents())); + std::unique_ptr dest_holder(new WritableFileWriter( + std::move(sink), "" /* don't care */, FileOptions())); + Writer recycle_writer(std::move(dest_holder), 123, true); + ASSERT_OK(recycle_writer.AddRecord(Slice("foooo"))); + ASSERT_OK(recycle_writer.AddRecord(Slice("bar"))); + ASSERT_GE(get_reader_contents()->size(), log::kBlockSize * 2); + ASSERT_EQ("foooo", Read()); + ASSERT_EQ("bar", Read()); + ASSERT_EQ("EOF", Read()); +} + +TEST_P(LogTest, RecycleWithTimestampSize) { + bool recyclable_log = (std::get<0>(GetParam()) != 0); + if (!recyclable_log) { + return; // test is only valid for recycled logs + } + UnorderedMap ts_sz_one = { + {1, sizeof(uint32_t)}, + }; + Write("foo", &ts_sz_one); + Write("bar"); + Write("baz"); + Write("bif"); + Write("blitz"); + while (get_reader_contents()->size() < log::kBlockSize * 2) { + Write("xxxxxxxxxxxxxxxx"); + } + std::unique_ptr sink( + new test::OverwritingStringSink(get_reader_contents())); + std::unique_ptr dest_holder(new WritableFileWriter( + std::move(sink), "" /* don't care */, FileOptions())); + Writer recycle_writer(std::move(dest_holder), 123, true); + UnorderedMap ts_sz_two = { + {2, sizeof(uint64_t)}, + }; + ASSERT_OK(recycle_writer.MaybeAddUserDefinedTimestampSizeRecord(ts_sz_two)); + ASSERT_OK(recycle_writer.AddRecord(Slice("foooo"))); + ASSERT_OK(recycle_writer.AddRecord(Slice("bar"))); + ASSERT_GE(get_reader_contents()->size(), log::kBlockSize * 2); + CheckRecordAndTimestampSize("foooo", ts_sz_two); + CheckRecordAndTimestampSize("bar", ts_sz_two); + ASSERT_EQ("EOF", Read()); +} + +// Do NOT enable compression for this instantiation. +INSTANTIATE_TEST_CASE_P( + Log, LogTest, + ::testing::Combine(::testing::Values(0, 1), ::testing::Bool(), + ::testing::Values(CompressionType::kNoCompression))); + +class RetriableLogTest : public ::testing::TestWithParam { + private: + class ReportCollector : public Reader::Reporter { + public: + size_t dropped_bytes_; + std::string message_; + + ReportCollector() : dropped_bytes_(0) {} + void Corruption(size_t bytes, const Status& status) override { + dropped_bytes_ += bytes; + message_.append(status.ToString()); + } + }; + + Slice contents_; + test::StringSink* sink_; + std::unique_ptr log_writer_; + Env* env_; + const std::string test_dir_; + const std::string log_file_; + std::unique_ptr writer_; + std::unique_ptr reader_; + ReportCollector report_; + std::unique_ptr log_reader_; + + public: + RetriableLogTest() + : contents_(), + sink_(new test::StringSink(&contents_)), + log_writer_(nullptr), + env_(Env::Default()), + test_dir_(test::PerThreadDBPath("retriable_log_test")), + log_file_(test_dir_ + "/log"), + writer_(nullptr), + reader_(nullptr), + log_reader_(nullptr) { + std::unique_ptr sink_holder(sink_); + std::unique_ptr wfw(new WritableFileWriter( + std::move(sink_holder), "" /* file name */, FileOptions())); + log_writer_.reset(new Writer(std::move(wfw), 123, GetParam())); + } + + Status SetupTestEnv() { + Status s; + FileOptions fopts; + auto fs = env_->GetFileSystem(); + s = fs->CreateDirIfMissing(test_dir_, IOOptions(), nullptr); + std::unique_ptr writable_file; + if (s.ok()) { + s = fs->NewWritableFile(log_file_, fopts, &writable_file, nullptr); + } + if (s.ok()) { + writer_.reset( + new WritableFileWriter(std::move(writable_file), log_file_, fopts)); + EXPECT_NE(writer_, nullptr); + } + std::unique_ptr seq_file; + if (s.ok()) { + s = fs->NewSequentialFile(log_file_, fopts, &seq_file, nullptr); + } + if (s.ok()) { + reader_.reset(new SequentialFileReader(std::move(seq_file), log_file_)); + EXPECT_NE(reader_, nullptr); + log_reader_.reset(new FragmentBufferedReader( + nullptr, std::move(reader_), &report_, true /* checksum */, + 123 /* log_number */)); + EXPECT_NE(log_reader_, nullptr); + } + return s; + } + + std::string contents() { return sink_->contents_; } + + void Encode(const std::string& msg) { + ASSERT_OK(log_writer_->AddRecord(Slice(msg))); + } + + void Write(const Slice& data) { + ASSERT_OK(writer_->Append(data)); + ASSERT_OK(writer_->Sync(true)); + } + + bool TryRead(std::string* result) { + assert(result != nullptr); + result->clear(); + std::string scratch; + Slice record; + bool r = log_reader_->ReadRecord(&record, &scratch); + if (r) { + result->assign(record.data(), record.size()); + return true; + } else { + return false; + } + } +}; + +TEST_P(RetriableLogTest, TailLog_PartialHeader) { + ASSERT_OK(SetupTestEnv()); + std::vector remaining_bytes_in_last_record; + size_t header_size = GetParam() ? kRecyclableHeaderSize : kHeaderSize; + bool eof = false; + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->LoadDependency( + {{"RetriableLogTest::TailLog:AfterPart1", + "RetriableLogTest::TailLog:BeforeReadRecord"}, + {"FragmentBufferedLogReader::TryReadMore:FirstEOF", + "RetriableLogTest::TailLog:BeforePart2"}}); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "FragmentBufferedLogReader::TryReadMore:FirstEOF", + [&](void* /*arg*/) { eof = true; }); + SyncPoint::GetInstance()->EnableProcessing(); + + size_t delta = header_size - 1; + port::Thread log_writer_thread([&]() { + size_t old_sz = contents().size(); + Encode("foo"); + size_t new_sz = contents().size(); + std::string part1 = contents().substr(old_sz, delta); + std::string part2 = + contents().substr(old_sz + delta, new_sz - old_sz - delta); + Write(Slice(part1)); + TEST_SYNC_POINT("RetriableLogTest::TailLog:AfterPart1"); + TEST_SYNC_POINT("RetriableLogTest::TailLog:BeforePart2"); + Write(Slice(part2)); + }); + + std::string record; + port::Thread log_reader_thread([&]() { + TEST_SYNC_POINT("RetriableLogTest::TailLog:BeforeReadRecord"); + while (!TryRead(&record)) { + } + }); + log_reader_thread.join(); + log_writer_thread.join(); + ASSERT_EQ("foo", record); + ASSERT_TRUE(eof); +} + +TEST_P(RetriableLogTest, TailLog_FullHeader) { + ASSERT_OK(SetupTestEnv()); + std::vector remaining_bytes_in_last_record; + size_t header_size = GetParam() ? kRecyclableHeaderSize : kHeaderSize; + bool eof = false; + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->LoadDependency( + {{"RetriableLogTest::TailLog:AfterPart1", + "RetriableLogTest::TailLog:BeforeReadRecord"}, + {"FragmentBufferedLogReader::TryReadMore:FirstEOF", + "RetriableLogTest::TailLog:BeforePart2"}}); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "FragmentBufferedLogReader::TryReadMore:FirstEOF", + [&](void* /*arg*/) { eof = true; }); + SyncPoint::GetInstance()->EnableProcessing(); + + size_t delta = header_size + 1; + port::Thread log_writer_thread([&]() { + size_t old_sz = contents().size(); + Encode("foo"); + size_t new_sz = contents().size(); + std::string part1 = contents().substr(old_sz, delta); + std::string part2 = + contents().substr(old_sz + delta, new_sz - old_sz - delta); + Write(Slice(part1)); + TEST_SYNC_POINT("RetriableLogTest::TailLog:AfterPart1"); + TEST_SYNC_POINT("RetriableLogTest::TailLog:BeforePart2"); + Write(Slice(part2)); + ASSERT_TRUE(eof); + }); + + std::string record; + port::Thread log_reader_thread([&]() { + TEST_SYNC_POINT("RetriableLogTest::TailLog:BeforeReadRecord"); + while (!TryRead(&record)) { + } + }); + log_reader_thread.join(); + log_writer_thread.join(); + ASSERT_EQ("foo", record); +} + +TEST_P(RetriableLogTest, NonBlockingReadFullRecord) { + // Clear all sync point callbacks even if this test does not use sync point. + // It is necessary, otherwise the execute of this test may hit a sync point + // with which a callback is registered. The registered callback may access + // some dead variable, causing segfault. + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + ASSERT_OK(SetupTestEnv()); + size_t header_size = GetParam() ? kRecyclableHeaderSize : kHeaderSize; + size_t delta = header_size - 1; + size_t old_sz = contents().size(); + Encode("foo-bar"); + size_t new_sz = contents().size(); + std::string part1 = contents().substr(old_sz, delta); + std::string part2 = + contents().substr(old_sz + delta, new_sz - old_sz - delta); + Write(Slice(part1)); + std::string record; + ASSERT_FALSE(TryRead(&record)); + ASSERT_TRUE(record.empty()); + Write(Slice(part2)); + ASSERT_TRUE(TryRead(&record)); + ASSERT_EQ("foo-bar", record); +} + +INSTANTIATE_TEST_CASE_P(bool, RetriableLogTest, ::testing::Values(0, 2)); + +class CompressionLogTest : public LogTest { + public: + Status SetupTestEnv() { return writer_->AddCompressionTypeRecord(); } +}; + +TEST_P(CompressionLogTest, Empty) { + CompressionType compression_type = std::get<2>(GetParam()); + if (!StreamingCompressionTypeSupported(compression_type)) { + ROCKSDB_GTEST_SKIP("Test requires support for compression type"); + return; + } + ASSERT_OK(SetupTestEnv()); + const bool compression_enabled = + std::get<2>(GetParam()) == kNoCompression ? false : true; + // If WAL compression is enabled, a record is added for the compression type + const int compression_record_size = compression_enabled ? kHeaderSize + 4 : 0; + ASSERT_EQ(compression_record_size, WrittenBytes()); + ASSERT_EQ("EOF", Read()); +} + +TEST_P(CompressionLogTest, ReadWrite) { + CompressionType compression_type = std::get<2>(GetParam()); + if (!StreamingCompressionTypeSupported(compression_type)) { + ROCKSDB_GTEST_SKIP("Test requires support for compression type"); + return; + } + ASSERT_OK(SetupTestEnv()); + Write("foo"); + Write("bar"); + Write(""); + Write("xxxx"); + ASSERT_EQ("foo", Read()); + ASSERT_EQ("bar", Read()); + ASSERT_EQ("", Read()); + ASSERT_EQ("xxxx", Read()); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ("EOF", Read()); // Make sure reads at eof work +} + +TEST_P(CompressionLogTest, ReadWriteWithTimestampSize) { + CompressionType compression_type = std::get<2>(GetParam()); + if (!StreamingCompressionTypeSupported(compression_type)) { + ROCKSDB_GTEST_SKIP("Test requires support for compression type"); + return; + } + ASSERT_OK(SetupTestEnv()); + UnorderedMap ts_sz_one = { + {1, sizeof(uint64_t)}, + }; + Write("foo", &ts_sz_one); + Write("bar"); + UnorderedMap ts_sz_two = {{2, sizeof(char)}}; + Write("", &ts_sz_two); + Write("xxxx"); + + CheckRecordAndTimestampSize("foo", ts_sz_one); + CheckRecordAndTimestampSize("bar", ts_sz_one); + UnorderedMap expected_ts_sz_two; + // User-defined timestamp size records are accumulated and applied to + // subsequent records. + expected_ts_sz_two.insert(ts_sz_one.begin(), ts_sz_one.end()); + expected_ts_sz_two.insert(ts_sz_two.begin(), ts_sz_two.end()); + CheckRecordAndTimestampSize("", expected_ts_sz_two); + CheckRecordAndTimestampSize("xxxx", expected_ts_sz_two); + ASSERT_EQ("EOF", Read()); + ASSERT_EQ("EOF", Read()); // Make sure reads at eof work +} + +TEST_P(CompressionLogTest, ManyBlocks) { + CompressionType compression_type = std::get<2>(GetParam()); + if (!StreamingCompressionTypeSupported(compression_type)) { + ROCKSDB_GTEST_SKIP("Test requires support for compression type"); + return; + } + ASSERT_OK(SetupTestEnv()); + for (int i = 0; i < 100000; i++) { + Write(NumberString(i)); + } + for (int i = 0; i < 100000; i++) { + ASSERT_EQ(NumberString(i), Read()); + } + ASSERT_EQ("EOF", Read()); +} + +TEST_P(CompressionLogTest, Fragmentation) { + CompressionType compression_type = std::get<2>(GetParam()); + if (!StreamingCompressionTypeSupported(compression_type)) { + ROCKSDB_GTEST_SKIP("Test requires support for compression type"); + return; + } + ASSERT_OK(SetupTestEnv()); + Random rnd(301); + const std::vector wal_entries = { + "small", + rnd.RandomBinaryString(3 * kBlockSize / 2), // Spans into block 2 + rnd.RandomBinaryString(3 * kBlockSize), // Spans into block 5 + }; + for (const std::string& wal_entry : wal_entries) { + Write(wal_entry); + } + + for (const std::string& wal_entry : wal_entries) { + ASSERT_EQ(wal_entry, Read()); + } + ASSERT_EQ("EOF", Read()); +} + +TEST_P(CompressionLogTest, AlignedFragmentation) { + CompressionType compression_type = std::get<2>(GetParam()); + if (!StreamingCompressionTypeSupported(compression_type)) { + ROCKSDB_GTEST_SKIP("Test requires support for compression type"); + return; + } + ASSERT_OK(SetupTestEnv()); + Random rnd(301); + int num_filler_records = 0; + // Keep writing small records until the next record will be aligned at the + // beginning of the block. + while ((WrittenBytes() & (kBlockSize - 1)) >= kHeaderSize) { + char entry = 'a'; + ASSERT_OK(writer_->AddRecord(Slice(&entry, 1))); + num_filler_records++; + } + const std::vector wal_entries = { + rnd.RandomBinaryString(3 * kBlockSize), + }; + for (const std::string& wal_entry : wal_entries) { + Write(wal_entry); + } + + for (int i = 0; i < num_filler_records; ++i) { + ASSERT_EQ("a", Read()); + } + for (const std::string& wal_entry : wal_entries) { + ASSERT_EQ(wal_entry, Read()); + } + ASSERT_EQ("EOF", Read()); +} + +INSTANTIATE_TEST_CASE_P( + Compression, CompressionLogTest, + ::testing::Combine(::testing::Values(0, 1), ::testing::Bool(), + ::testing::Values(CompressionType::kNoCompression, + CompressionType::kZSTD))); + +class StreamingCompressionTest + : public ::testing::TestWithParam> {}; + +TEST_P(StreamingCompressionTest, Basic) { + size_t input_size = std::get<0>(GetParam()); + CompressionType compression_type = std::get<1>(GetParam()); + if (!StreamingCompressionTypeSupported(compression_type)) { + ROCKSDB_GTEST_SKIP("Test requires support for compression type"); + return; + } + CompressionOptions opts; + constexpr uint32_t compression_format_version = 2; + StreamingCompress* compress = StreamingCompress::Create( + compression_type, opts, compression_format_version, kBlockSize); + StreamingUncompress* uncompress = StreamingUncompress::Create( + compression_type, compression_format_version, kBlockSize); + MemoryAllocator* allocator = new DefaultMemoryAllocator(); + std::string input_buffer = BigString("abc", input_size); + std::vector compressed_buffers; + size_t remaining; + // Call compress till the entire input is consumed + do { + char* output_buffer = (char*)allocator->Allocate(kBlockSize); + size_t output_pos; + remaining = compress->Compress(input_buffer.c_str(), input_size, + output_buffer, &output_pos); + if (output_pos > 0) { + std::string compressed_buffer; + compressed_buffer.assign(output_buffer, output_pos); + compressed_buffers.emplace_back(std::move(compressed_buffer)); + } + allocator->Deallocate((void*)output_buffer); + } while (remaining > 0); + std::string uncompressed_buffer = ""; + int ret_val = 0; + size_t output_pos; + char* uncompressed_output_buffer = (char*)allocator->Allocate(kBlockSize); + // Uncompress the fragments and concatenate them. + for (int i = 0; i < (int)compressed_buffers.size(); i++) { + // Call uncompress till either the entire input is consumed or the output + // buffer size is equal to the allocated output buffer size. + const char* input = compressed_buffers[i].c_str(); + do { + ret_val = uncompress->Uncompress(input, compressed_buffers[i].size(), + uncompressed_output_buffer, &output_pos); + input = nullptr; + if (output_pos > 0) { + std::string uncompressed_fragment; + uncompressed_fragment.assign(uncompressed_output_buffer, output_pos); + uncompressed_buffer += uncompressed_fragment; + } + } while (ret_val > 0 || output_pos == kBlockSize); + } + allocator->Deallocate((void*)uncompressed_output_buffer); + delete allocator; + delete compress; + delete uncompress; + // The final return value from uncompress() should be 0. + ASSERT_EQ(ret_val, 0); + ASSERT_EQ(input_buffer, uncompressed_buffer); +} + +INSTANTIATE_TEST_CASE_P( + StreamingCompression, StreamingCompressionTest, + ::testing::Combine(::testing::Values(10, 100, 1000, kBlockSize, + kBlockSize * 2), + ::testing::Values(CompressionType::kZSTD))); + +} // namespace log +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/log_writer.cc b/librocksdb-sys/rocksdb/db/log_writer.cc new file mode 100644 index 0000000..86e0286 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/log_writer.cc @@ -0,0 +1,279 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/log_writer.h" + +#include + +#include "file/writable_file_writer.h" +#include "rocksdb/env.h" +#include "rocksdb/io_status.h" +#include "util/coding.h" +#include "util/crc32c.h" +#include "util/udt_util.h" + +namespace ROCKSDB_NAMESPACE { +namespace log { + +Writer::Writer(std::unique_ptr&& dest, uint64_t log_number, + bool recycle_log_files, bool manual_flush, + CompressionType compression_type) + : dest_(std::move(dest)), + block_offset_(0), + log_number_(log_number), + recycle_log_files_(recycle_log_files), + manual_flush_(manual_flush), + compression_type_(compression_type), + compress_(nullptr) { + for (int i = 0; i <= kMaxRecordType; i++) { + char t = static_cast(i); + type_crc_[i] = crc32c::Value(&t, 1); + } +} + +Writer::~Writer() { + if (dest_) { + WriteBuffer().PermitUncheckedError(); + } + if (compress_) { + delete compress_; + } +} + +IOStatus Writer::WriteBuffer() { + if (dest_->seen_error()) { + return IOStatus::IOError("Seen error. Skip writing buffer."); + } + return dest_->Flush(); +} + +IOStatus Writer::Close() { + IOStatus s; + if (dest_) { + s = dest_->Close(); + dest_.reset(); + } + return s; +} + +IOStatus Writer::AddRecord(const Slice& slice, + Env::IOPriority rate_limiter_priority) { + const char* ptr = slice.data(); + size_t left = slice.size(); + + // Header size varies depending on whether we are recycling or not. + const int header_size = + recycle_log_files_ ? kRecyclableHeaderSize : kHeaderSize; + + // Fragment the record if necessary and emit it. Note that if slice + // is empty, we still want to iterate once to emit a single + // zero-length record + bool begin = true; + int compress_remaining = 0; + bool compress_start = false; + if (compress_) { + compress_->Reset(); + compress_start = true; + } + + IOStatus s; + do { + const int64_t leftover = kBlockSize - block_offset_; + assert(leftover >= 0); + if (leftover < header_size) { + // Switch to a new block + if (leftover > 0) { + // Fill the trailer (literal below relies on kHeaderSize and + // kRecyclableHeaderSize being <= 11) + assert(header_size <= 11); + s = dest_->Append(Slice("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + static_cast(leftover)), + 0 /* crc32c_checksum */, rate_limiter_priority); + if (!s.ok()) { + break; + } + } + block_offset_ = 0; + } + + // Invariant: we never leave < header_size bytes in a block. + assert(static_cast(kBlockSize - block_offset_) >= header_size); + + const size_t avail = kBlockSize - block_offset_ - header_size; + + // Compress the record if compression is enabled. + // Compress() is called at least once (compress_start=true) and after the + // previous generated compressed chunk is written out as one or more + // physical records (left=0). + if (compress_ && (compress_start || left == 0)) { + compress_remaining = compress_->Compress(slice.data(), slice.size(), + compressed_buffer_.get(), &left); + + if (compress_remaining < 0) { + // Set failure status + s = IOStatus::IOError("Unexpected WAL compression error"); + s.SetDataLoss(true); + break; + } else if (left == 0) { + // Nothing left to compress + if (!compress_start) { + break; + } + } + compress_start = false; + ptr = compressed_buffer_.get(); + } + + const size_t fragment_length = (left < avail) ? left : avail; + + RecordType type; + const bool end = (left == fragment_length && compress_remaining == 0); + if (begin && end) { + type = recycle_log_files_ ? kRecyclableFullType : kFullType; + } else if (begin) { + type = recycle_log_files_ ? kRecyclableFirstType : kFirstType; + } else if (end) { + type = recycle_log_files_ ? kRecyclableLastType : kLastType; + } else { + type = recycle_log_files_ ? kRecyclableMiddleType : kMiddleType; + } + + s = EmitPhysicalRecord(type, ptr, fragment_length, rate_limiter_priority); + ptr += fragment_length; + left -= fragment_length; + begin = false; + } while (s.ok() && (left > 0 || compress_remaining > 0)); + + if (s.ok()) { + if (!manual_flush_) { + s = dest_->Flush(rate_limiter_priority); + } + } + + return s; +} + +IOStatus Writer::AddCompressionTypeRecord() { + // Should be the first record + assert(block_offset_ == 0); + + if (compression_type_ == kNoCompression) { + // No need to add a record + return IOStatus::OK(); + } + + CompressionTypeRecord record(compression_type_); + std::string encode; + record.EncodeTo(&encode); + IOStatus s = + EmitPhysicalRecord(kSetCompressionType, encode.data(), encode.size()); + if (s.ok()) { + if (!manual_flush_) { + s = dest_->Flush(); + } + // Initialize fields required for compression + const size_t max_output_buffer_len = + kBlockSize - (recycle_log_files_ ? kRecyclableHeaderSize : kHeaderSize); + CompressionOptions opts; + constexpr uint32_t compression_format_version = 2; + compress_ = StreamingCompress::Create(compression_type_, opts, + compression_format_version, + max_output_buffer_len); + assert(compress_ != nullptr); + compressed_buffer_ = + std::unique_ptr(new char[max_output_buffer_len]); + assert(compressed_buffer_); + } else { + // Disable compression if the record could not be added. + compression_type_ = kNoCompression; + } + return s; +} + +IOStatus Writer::MaybeAddUserDefinedTimestampSizeRecord( + const UnorderedMap& cf_to_ts_sz, + Env::IOPriority rate_limiter_priority) { + std::vector> ts_sz_to_record; + for (const auto& [cf_id, ts_sz] : cf_to_ts_sz) { + if (recorded_cf_to_ts_sz_.count(cf_id) != 0) { + // A column family's user-defined timestamp size should not be + // updated while DB is running. + assert(recorded_cf_to_ts_sz_[cf_id] == ts_sz); + } else if (ts_sz != 0) { + ts_sz_to_record.emplace_back(cf_id, ts_sz); + recorded_cf_to_ts_sz_.insert(std::make_pair(cf_id, ts_sz)); + } + } + if (ts_sz_to_record.empty()) { + return IOStatus::OK(); + } + + UserDefinedTimestampSizeRecord record(std::move(ts_sz_to_record)); + std::string encoded; + record.EncodeTo(&encoded); + RecordType type = recycle_log_files_ ? kRecyclableUserDefinedTimestampSizeType + : kUserDefinedTimestampSizeType; + return EmitPhysicalRecord(type, encoded.data(), encoded.size(), + rate_limiter_priority); +} + +bool Writer::BufferIsEmpty() { return dest_->BufferIsEmpty(); } + +IOStatus Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n, + Env::IOPriority rate_limiter_priority) { + assert(n <= 0xffff); // Must fit in two bytes + + size_t header_size; + char buf[kRecyclableHeaderSize]; + + // Format the header + buf[4] = static_cast(n & 0xff); + buf[5] = static_cast(n >> 8); + buf[6] = static_cast(t); + + uint32_t crc = type_crc_[t]; + if (t < kRecyclableFullType || t == kSetCompressionType || + t == kUserDefinedTimestampSizeType) { + // Legacy record format + assert(block_offset_ + kHeaderSize + n <= kBlockSize); + header_size = kHeaderSize; + } else { + // Recyclable record format + assert(block_offset_ + kRecyclableHeaderSize + n <= kBlockSize); + header_size = kRecyclableHeaderSize; + + // Only encode low 32-bits of the 64-bit log number. This means + // we will fail to detect an old record if we recycled a log from + // ~4 billion logs ago, but that is effectively impossible, and + // even if it were we'dbe far more likely to see a false positive + // on the 32-bit CRC. + EncodeFixed32(buf + 7, static_cast(log_number_)); + crc = crc32c::Extend(crc, buf + 7, 4); + } + + // Compute the crc of the record type and the payload. + uint32_t payload_crc = crc32c::Value(ptr, n); + crc = crc32c::Crc32cCombine(crc, payload_crc, n); + crc = crc32c::Mask(crc); // Adjust for storage + TEST_SYNC_POINT_CALLBACK("LogWriter::EmitPhysicalRecord:BeforeEncodeChecksum", + &crc); + EncodeFixed32(buf, crc); + + // Write the header and the payload + IOStatus s = dest_->Append(Slice(buf, header_size), 0 /* crc32c_checksum */, + rate_limiter_priority); + if (s.ok()) { + s = dest_->Append(Slice(ptr, n), payload_crc, rate_limiter_priority); + } + block_offset_ += header_size + n; + return s; +} + +} // namespace log +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/log_writer.h b/librocksdb-sys/rocksdb/db/log_writer.h new file mode 100644 index 0000000..7a64a85 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/log_writer.h @@ -0,0 +1,145 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once + +#include +#include +#include +#include + +#include "db/log_format.h" +#include "rocksdb/compression_type.h" +#include "rocksdb/env.h" +#include "rocksdb/io_status.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "util/compression.h" +#include "util/hash_containers.h" + +namespace ROCKSDB_NAMESPACE { + +class WritableFileWriter; + +namespace log { + +/** + * Writer is a general purpose log stream writer. It provides an append-only + * abstraction for writing data. The details of the how the data is written is + * handled by the WritableFile sub-class implementation. + * + * File format: + * + * File is broken down into variable sized records. The format of each record + * is described below. + * +-----+-------------+--+----+----------+------+-- ... ----+ + * File | r0 | r1 |P | r2 | r3 | r4 | | + * +-----+-------------+--+----+----------+------+-- ... ----+ + * <--- kBlockSize ------>|<-- kBlockSize ------>| + * rn = variable size records + * P = Padding + * + * Data is written out in kBlockSize chunks. If next record does not fit + * into the space left, the leftover space will be padded with \0. + * + * Legacy record format: + * + * +---------+-----------+-----------+--- ... ---+ + * |CRC (4B) | Size (2B) | Type (1B) | Payload | + * +---------+-----------+-----------+--- ... ---+ + * + * CRC = 32bit hash computed over the record type and payload using CRC + * Size = Length of the payload data + * Type = Type of record + * (kZeroType, kFullType, kFirstType, kLastType, kMiddleType ) + * The type is used to group a bunch of records together to represent + * blocks that are larger than kBlockSize + * Payload = Byte stream as long as specified by the payload size + * + * Recyclable record format: + * + * +---------+-----------+-----------+----------------+--- ... ---+ + * |CRC (4B) | Size (2B) | Type (1B) | Log number (4B)| Payload | + * +---------+-----------+-----------+----------------+--- ... ---+ + * + * Same as above, with the addition of + * Log number = 32bit log file number, so that we can distinguish between + * records written by the most recent log writer vs a previous one. + */ +class Writer { + public: + // Create a writer that will append data to "*dest". + // "*dest" must be initially empty. + // "*dest" must remain live while this Writer is in use. + explicit Writer(std::unique_ptr&& dest, + uint64_t log_number, bool recycle_log_files, + bool manual_flush = false, + CompressionType compressionType = kNoCompression); + // No copying allowed + Writer(const Writer&) = delete; + void operator=(const Writer&) = delete; + + ~Writer(); + + IOStatus AddRecord(const Slice& slice, + Env::IOPriority rate_limiter_priority = Env::IO_TOTAL); + IOStatus AddCompressionTypeRecord(); + + // If there are column families in `cf_to_ts_sz` not included in + // `recorded_cf_to_ts_sz_` and its user-defined timestamp size is non-zero, + // adds a record of type kUserDefinedTimestampSizeType or + // kRecyclableUserDefinedTimestampSizeType for these column families. + // This timestamp size record applies to all subsequent records. + IOStatus MaybeAddUserDefinedTimestampSizeRecord( + const UnorderedMap& cf_to_ts_sz, + Env::IOPriority rate_limiter_priority = Env::IO_TOTAL); + + WritableFileWriter* file() { return dest_.get(); } + const WritableFileWriter* file() const { return dest_.get(); } + + uint64_t get_log_number() const { return log_number_; } + + IOStatus WriteBuffer(); + + IOStatus Close(); + + bool BufferIsEmpty(); + + private: + std::unique_ptr dest_; + size_t block_offset_; // Current offset in block + uint64_t log_number_; + bool recycle_log_files_; + + // crc32c values for all supported record types. These are + // pre-computed to reduce the overhead of computing the crc of the + // record type stored in the header. + uint32_t type_crc_[kMaxRecordType + 1]; + + IOStatus EmitPhysicalRecord( + RecordType type, const char* ptr, size_t length, + Env::IOPriority rate_limiter_priority = Env::IO_TOTAL); + + // If true, it does not flush after each write. Instead it relies on the upper + // layer to manually does the flush by calling ::WriteBuffer() + bool manual_flush_; + + // Compression Type + CompressionType compression_type_; + StreamingCompress* compress_; + // Reusable compressed output buffer + std::unique_ptr compressed_buffer_; + + // The recorded user-defined timestamp size that have been written so far. + // Since the user-defined timestamp size cannot be changed while the DB is + // running, existing entry in this map cannot be updated. + UnorderedMap recorded_cf_to_ts_sz_; +}; + +} // namespace log +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/logs_with_prep_tracker.cc b/librocksdb-sys/rocksdb/db/logs_with_prep_tracker.cc new file mode 100644 index 0000000..ff98155 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/logs_with_prep_tracker.cc @@ -0,0 +1,67 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#include "db/logs_with_prep_tracker.h" + +#include "port/likely.h" + +namespace ROCKSDB_NAMESPACE { +void LogsWithPrepTracker::MarkLogAsHavingPrepSectionFlushed(uint64_t log) { + assert(log != 0); + std::lock_guard lock(prepared_section_completed_mutex_); + auto it = prepared_section_completed_.find(log); + if (UNLIKELY(it == prepared_section_completed_.end())) { + prepared_section_completed_[log] = 1; + } else { + it->second += 1; + } +} + +void LogsWithPrepTracker::MarkLogAsContainingPrepSection(uint64_t log) { + assert(log != 0); + std::lock_guard lock(logs_with_prep_mutex_); + + auto rit = logs_with_prep_.rbegin(); + bool updated = false; + // Most probably the last log is the one that is being marked for + // having a prepare section; so search from the end. + for (; rit != logs_with_prep_.rend() && rit->log >= log; ++rit) { + if (rit->log == log) { + rit->cnt++; + updated = true; + break; + } + } + if (!updated) { + // We are either at the start, or at a position with rit->log < log + logs_with_prep_.insert(rit.base(), {log, 1}); + } +} + +uint64_t LogsWithPrepTracker::FindMinLogContainingOutstandingPrep() { + std::lock_guard lock(logs_with_prep_mutex_); + auto it = logs_with_prep_.begin(); + // start with the smallest log + for (; it != logs_with_prep_.end();) { + auto min_log = it->log; + { + std::lock_guard lock2(prepared_section_completed_mutex_); + auto completed_it = prepared_section_completed_.find(min_log); + if (completed_it == prepared_section_completed_.end() || + completed_it->second < it->cnt) { + return min_log; + } + assert(completed_it != prepared_section_completed_.end() && + completed_it->second == it->cnt); + prepared_section_completed_.erase(completed_it); + } + // erase from beginning in vector is not efficient but this function is not + // on the fast path. + it = logs_with_prep_.erase(it); + } + // no such log found + return 0; +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/logs_with_prep_tracker.h b/librocksdb-sys/rocksdb/db/logs_with_prep_tracker.h new file mode 100644 index 0000000..f72f0ca --- /dev/null +++ b/librocksdb-sys/rocksdb/db/logs_with_prep_tracker.h @@ -0,0 +1,62 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +// This class is used to track the log files with outstanding prepare entries. +class LogsWithPrepTracker { + public: + // Called when a transaction prepared in `log` has been committed or aborted. + void MarkLogAsHavingPrepSectionFlushed(uint64_t log); + // Called when a transaction is prepared in `log`. + void MarkLogAsContainingPrepSection(uint64_t log); + // Return the earliest log file with outstanding prepare entries. + uint64_t FindMinLogContainingOutstandingPrep(); + size_t TEST_PreparedSectionCompletedSize() { + return prepared_section_completed_.size(); + } + size_t TEST_LogsWithPrepSize() { return logs_with_prep_.size(); } + + private: + // REQUIRES: logs_with_prep_mutex_ held + // + // sorted list of log numbers still containing prepared data. + // this is used by FindObsoleteFiles to determine which + // flushed logs we must keep around because they still + // contain prepared data which has not been committed or rolled back + struct LogCnt { + uint64_t log; // the log number + uint64_t cnt; // number of prepared sections in the log + }; + std::vector logs_with_prep_; + std::mutex logs_with_prep_mutex_; + + // REQUIRES: prepared_section_completed_mutex_ held + // + // to be used in conjunction with logs_with_prep_. + // once a transaction with data in log L is committed or rolled back + // rather than updating logs_with_prep_ directly we keep track of that + // in prepared_section_completed_ which maps LOG -> instance_count. This helps + // avoiding contention between a commit thread and the prepare threads. + // + // when trying to determine the minimum log still active we first + // consult logs_with_prep_. while that root value maps to + // an equal value in prepared_section_completed_ we erase the log from + // both logs_with_prep_ and prepared_section_completed_. + std::unordered_map prepared_section_completed_; + std::mutex prepared_section_completed_mutex_; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/lookup_key.h b/librocksdb-sys/rocksdb/db/lookup_key.h new file mode 100644 index 0000000..68851bd --- /dev/null +++ b/librocksdb-sys/rocksdb/db/lookup_key.h @@ -0,0 +1,68 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include +#include + +#include "rocksdb/slice.h" +#include "rocksdb/types.h" + +namespace ROCKSDB_NAMESPACE { + +// A helper class useful for DBImpl::Get() +class LookupKey { + public: + // Initialize *this for looking up user_key at a snapshot with + // the specified sequence number. + LookupKey(const Slice& _user_key, SequenceNumber sequence, + const Slice* ts = nullptr); + + ~LookupKey(); + + // Return a key suitable for lookup in a MemTable. + Slice memtable_key() const { + return Slice(start_, static_cast(end_ - start_)); + } + + // Return an internal key (suitable for passing to an internal iterator) + Slice internal_key() const { + return Slice(kstart_, static_cast(end_ - kstart_)); + } + + // Return the user key. + // If user-defined timestamp is enabled, then timestamp is included in the + // result. + Slice user_key() const { + return Slice(kstart_, static_cast(end_ - kstart_ - 8)); + } + + private: + // We construct a char array of the form: + // klength varint32 <-- start_ + // userkey char[klength] <-- kstart_ + // tag uint64 + // <-- end_ + // The array is a suitable MemTable key. + // The suffix starting with "userkey" can be used as an InternalKey. + const char* start_; + const char* kstart_; + const char* end_; + char space_[200]; // Avoid allocation for short keys + + // No copying allowed + LookupKey(const LookupKey&); + void operator=(const LookupKey&); +}; + +inline LookupKey::~LookupKey() { + if (start_ != space_) delete[] start_; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/malloc_stats.cc b/librocksdb-sys/rocksdb/db/malloc_stats.cc new file mode 100644 index 0000000..641e01f --- /dev/null +++ b/librocksdb-sys/rocksdb/db/malloc_stats.cc @@ -0,0 +1,53 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/malloc_stats.h" + +#include + +#include + +#include "port/jemalloc_helper.h" + +namespace ROCKSDB_NAMESPACE { + +#ifdef ROCKSDB_JEMALLOC + +struct MallocStatus { + char* cur; + char* end; +}; + +static void GetJemallocStatus(void* mstat_arg, const char* status) { + MallocStatus* mstat = reinterpret_cast(mstat_arg); + size_t status_len = status ? strlen(status) : 0; + size_t buf_size = (size_t)(mstat->end - mstat->cur); + if (!status_len || status_len > buf_size) { + return; + } + + snprintf(mstat->cur, buf_size, "%s", status); + mstat->cur += status_len; +} +void DumpMallocStats(std::string* stats) { + if (!HasJemalloc()) { + return; + } + MallocStatus mstat; + const unsigned int kMallocStatusLen = 1000000; + std::unique_ptr buf{new char[kMallocStatusLen + 1]}; + mstat.cur = buf.get(); + mstat.end = buf.get() + kMallocStatusLen; + malloc_stats_print(GetJemallocStatus, &mstat, ""); + stats->append(buf.get()); +} +#else +void DumpMallocStats(std::string*) {} +#endif // ROCKSDB_JEMALLOC +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/malloc_stats.h b/librocksdb-sys/rocksdb/db/malloc_stats.h new file mode 100644 index 0000000..1cca8a9 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/malloc_stats.h @@ -0,0 +1,22 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + + +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +void DumpMallocStats(std::string*); + +} + diff --git a/librocksdb-sys/rocksdb/db/manual_compaction_test.cc b/librocksdb-sys/rocksdb/db/manual_compaction_test.cc new file mode 100644 index 0000000..e9767ab --- /dev/null +++ b/librocksdb-sys/rocksdb/db/manual_compaction_test.cc @@ -0,0 +1,309 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Test for issue 178: a manual compaction causes deleted data to reappear. +#include + +#include "port/port.h" +#include "rocksdb/compaction_filter.h" +#include "rocksdb/db.h" +#include "rocksdb/slice.h" +#include "rocksdb/write_batch.h" +#include "test_util/testharness.h" + +using ROCKSDB_NAMESPACE::CompactionFilter; +using ROCKSDB_NAMESPACE::CompactionStyle; +using ROCKSDB_NAMESPACE::CompactRangeOptions; +using ROCKSDB_NAMESPACE::CompressionType; +using ROCKSDB_NAMESPACE::DB; +using ROCKSDB_NAMESPACE::DestroyDB; +using ROCKSDB_NAMESPACE::FlushOptions; +using ROCKSDB_NAMESPACE::Iterator; +using ROCKSDB_NAMESPACE::Options; +using ROCKSDB_NAMESPACE::ReadOptions; +using ROCKSDB_NAMESPACE::Slice; +using ROCKSDB_NAMESPACE::WriteBatch; +using ROCKSDB_NAMESPACE::WriteOptions; + +namespace { + +// Reasoning: previously the number was 1100000. Since the keys are written to +// the batch in one write each write will result into one SST file. each write +// will result into one SST file. We reduced the write_buffer_size to 1K to +// basically have the same effect with however less number of keys, which +// results into less test runtime. +const int kNumKeys = 1100; + +std::string Key1(int i) { + char buf[100]; + snprintf(buf, sizeof(buf), "my_key_%d", i); + return buf; +} + +std::string Key2(int i) { return Key1(i) + "_xxx"; } + +class ManualCompactionTest : public testing::Test { + public: + ManualCompactionTest() { + // Get rid of any state from an old run. + dbname_ = ROCKSDB_NAMESPACE::test::PerThreadDBPath( + "rocksdb_manual_compaction_test"); + EXPECT_OK(DestroyDB(dbname_, Options())); + } + + std::string dbname_; +}; + +class DestroyAllCompactionFilter : public CompactionFilter { + public: + DestroyAllCompactionFilter() {} + + bool Filter(int /*level*/, const Slice& /*key*/, const Slice& existing_value, + std::string* /*new_value*/, + bool* /*value_changed*/) const override { + return existing_value.ToString() == "destroy"; + } + + const char* Name() const override { return "DestroyAllCompactionFilter"; } +}; + +class LogCompactionFilter : public CompactionFilter { + public: + const char* Name() const override { return "LogCompactionFilter"; } + + bool Filter(int level, const Slice& key, const Slice& /*existing_value*/, + std::string* /*new_value*/, + bool* /*value_changed*/) const override { + key_level_[key.ToString()] = level; + return false; + } + + void Reset() { key_level_.clear(); } + + size_t NumKeys() const { return key_level_.size(); } + + int KeyLevel(const Slice& key) { + auto it = key_level_.find(key.ToString()); + if (it == key_level_.end()) { + return -1; + } + return it->second; + } + + private: + mutable std::map key_level_; +}; + +TEST_F(ManualCompactionTest, CompactTouchesAllKeys) { + for (int iter = 0; iter < 2; ++iter) { + DB* db; + Options options; + if (iter == 0) { // level compaction + options.num_levels = 3; + options.compaction_style = CompactionStyle::kCompactionStyleLevel; + } else { // universal compaction + options.compaction_style = CompactionStyle::kCompactionStyleUniversal; + } + options.create_if_missing = true; + options.compression = CompressionType::kNoCompression; + options.compaction_filter = new DestroyAllCompactionFilter(); + ASSERT_OK(DB::Open(options, dbname_, &db)); + + ASSERT_OK(db->Put(WriteOptions(), Slice("key1"), Slice("destroy"))); + ASSERT_OK(db->Put(WriteOptions(), Slice("key2"), Slice("destroy"))); + ASSERT_OK(db->Put(WriteOptions(), Slice("key3"), Slice("value3"))); + ASSERT_OK(db->Put(WriteOptions(), Slice("key4"), Slice("destroy"))); + + Slice key4("key4"); + ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, &key4)); + Iterator* itr = db->NewIterator(ReadOptions()); + itr->SeekToFirst(); + ASSERT_TRUE(itr->Valid()); + ASSERT_EQ("key3", itr->key().ToString()); + itr->Next(); + ASSERT_TRUE(!itr->Valid()); + delete itr; + + delete options.compaction_filter; + delete db; + ASSERT_OK(DestroyDB(dbname_, options)); + } +} + +TEST_F(ManualCompactionTest, Test) { + // Open database. Disable compression since it affects the creation + // of layers and the code below is trying to test against a very + // specific scenario. + DB* db; + Options db_options; + db_options.write_buffer_size = 1024; + db_options.create_if_missing = true; + db_options.compression = CompressionType::kNoCompression; + ASSERT_OK(DB::Open(db_options, dbname_, &db)); + + // create first key range + WriteBatch batch; + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(batch.Put(Key1(i), "value for range 1 key")); + } + ASSERT_OK(db->Write(WriteOptions(), &batch)); + + // create second key range + batch.Clear(); + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(batch.Put(Key2(i), "value for range 2 key")); + } + ASSERT_OK(db->Write(WriteOptions(), &batch)); + + // delete second key range + batch.Clear(); + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(batch.Delete(Key2(i))); + } + ASSERT_OK(db->Write(WriteOptions(), &batch)); + + // compact database + std::string start_key = Key1(0); + std::string end_key = Key1(kNumKeys - 1); + Slice least(start_key.data(), start_key.size()); + Slice greatest(end_key.data(), end_key.size()); + + // commenting out the line below causes the example to work correctly + ASSERT_OK(db->CompactRange(CompactRangeOptions(), &least, &greatest)); + + // count the keys + Iterator* iter = db->NewIterator(ReadOptions()); + int num_keys = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + num_keys++; + } + delete iter; + ASSERT_EQ(kNumKeys, num_keys) << "Bad number of keys"; + + // close database + delete db; + ASSERT_OK(DestroyDB(dbname_, Options())); +} + +TEST_F(ManualCompactionTest, SkipLevel) { + DB* db; + Options options; + options.level_compaction_dynamic_level_bytes = false; + options.num_levels = 3; + // Initially, flushed L0 files won't exceed 100. + options.level0_file_num_compaction_trigger = 100; + options.compaction_style = CompactionStyle::kCompactionStyleLevel; + options.create_if_missing = true; + options.compression = CompressionType::kNoCompression; + LogCompactionFilter* filter = new LogCompactionFilter(); + options.compaction_filter = filter; + ASSERT_OK(DB::Open(options, dbname_, &db)); + + WriteOptions wo; + FlushOptions fo; + ASSERT_OK(db->Put(wo, "1", "")); + ASSERT_OK(db->Flush(fo)); + ASSERT_OK(db->Put(wo, "2", "")); + ASSERT_OK(db->Flush(fo)); + ASSERT_OK(db->Put(wo, "4", "")); + ASSERT_OK(db->Put(wo, "8", "")); + ASSERT_OK(db->Flush(fo)); + + { + // L0: 1, 2, [4, 8] + // no file has keys in range [5, 7] + Slice start("5"); + Slice end("7"); + filter->Reset(); + ASSERT_OK(db->CompactRange(CompactRangeOptions(), &start, &end)); + ASSERT_EQ(0, filter->NumKeys()); + } + + { + // L0: 1, 2, [4, 8] + // [3, 7] overlaps with 4 in L0 + Slice start("3"); + Slice end("7"); + filter->Reset(); + ASSERT_OK(db->CompactRange(CompactRangeOptions(), &start, &end)); + ASSERT_EQ(2, filter->NumKeys()); + ASSERT_EQ(0, filter->KeyLevel("4")); + ASSERT_EQ(0, filter->KeyLevel("8")); + } + + { + // L0: 1, 2 + // L1: [4, 8] + // no file has keys in range (-inf, 0] + Slice end("0"); + filter->Reset(); + ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, &end)); + ASSERT_EQ(0, filter->NumKeys()); + } + + { + // L0: 1, 2 + // L1: [4, 8] + // no file has keys in range [9, inf) + Slice start("9"); + filter->Reset(); + ASSERT_OK(db->CompactRange(CompactRangeOptions(), &start, nullptr)); + ASSERT_EQ(0, filter->NumKeys()); + } + + { + // L0: 1, 2 + // L1: [4, 8] + // [2, 2] overlaps with 2 in L0 + Slice start("2"); + Slice end("2"); + filter->Reset(); + ASSERT_OK(db->CompactRange(CompactRangeOptions(), &start, &end)); + ASSERT_EQ(1, filter->NumKeys()); + ASSERT_EQ(0, filter->KeyLevel("2")); + } + + { + // L0: 1 + // L1: 2, [4, 8] + // [2, 5] overlaps with 2 and [4, 8) in L1, skip L0 + Slice start("2"); + Slice end("5"); + filter->Reset(); + ASSERT_OK(db->CompactRange(CompactRangeOptions(), &start, &end)); + ASSERT_EQ(3, filter->NumKeys()); + ASSERT_EQ(1, filter->KeyLevel("2")); + ASSERT_EQ(1, filter->KeyLevel("4")); + ASSERT_EQ(1, filter->KeyLevel("8")); + } + + { + // L0: 1 + // L1: [2, 4, 8] + // [0, inf) overlaps all files + Slice start("0"); + filter->Reset(); + ASSERT_OK(db->CompactRange(CompactRangeOptions(), &start, nullptr)); + ASSERT_EQ(4, filter->NumKeys()); + // 1 is first compacted from L0 to L1, and then L1 intra level compaction + // compacts [2, 4, 8] only. + ASSERT_EQ(0, filter->KeyLevel("1")); + ASSERT_EQ(1, filter->KeyLevel("2")); + ASSERT_EQ(1, filter->KeyLevel("4")); + ASSERT_EQ(1, filter->KeyLevel("8")); + } + + delete filter; + delete db; + ASSERT_OK(DestroyDB(dbname_, options)); +} + +} // anonymous namespace + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/memtable.cc b/librocksdb-sys/rocksdb/db/memtable.cc new file mode 100644 index 0000000..8a71a64 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/memtable.cc @@ -0,0 +1,1714 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/memtable.h" + +#include +#include +#include +#include + +#include "db/dbformat.h" +#include "db/kv_checksum.h" +#include "db/merge_context.h" +#include "db/merge_helper.h" +#include "db/pinned_iterators_manager.h" +#include "db/range_tombstone_fragmenter.h" +#include "db/read_callback.h" +#include "db/wide/wide_column_serialization.h" +#include "logging/logging.h" +#include "memory/arena.h" +#include "memory/memory_usage.h" +#include "monitoring/perf_context_imp.h" +#include "monitoring/statistics_impl.h" +#include "port/lang.h" +#include "port/port.h" +#include "rocksdb/comparator.h" +#include "rocksdb/env.h" +#include "rocksdb/iterator.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/types.h" +#include "rocksdb/write_buffer_manager.h" +#include "table/internal_iterator.h" +#include "table/iterator_wrapper.h" +#include "table/merging_iterator.h" +#include "util/autovector.h" +#include "util/coding.h" +#include "util/mutexlock.h" + +namespace ROCKSDB_NAMESPACE { + +ImmutableMemTableOptions::ImmutableMemTableOptions( + const ImmutableOptions& ioptions, + const MutableCFOptions& mutable_cf_options) + : arena_block_size(mutable_cf_options.arena_block_size), + memtable_prefix_bloom_bits( + static_cast( + static_cast(mutable_cf_options.write_buffer_size) * + mutable_cf_options.memtable_prefix_bloom_size_ratio) * + 8u), + memtable_huge_page_size(mutable_cf_options.memtable_huge_page_size), + memtable_whole_key_filtering( + mutable_cf_options.memtable_whole_key_filtering), + inplace_update_support(ioptions.inplace_update_support), + inplace_update_num_locks(mutable_cf_options.inplace_update_num_locks), + inplace_callback(ioptions.inplace_callback), + max_successive_merges(mutable_cf_options.max_successive_merges), + statistics(ioptions.stats), + merge_operator(ioptions.merge_operator.get()), + info_log(ioptions.logger), + allow_data_in_errors(ioptions.allow_data_in_errors), + protection_bytes_per_key( + mutable_cf_options.memtable_protection_bytes_per_key) {} + +MemTable::MemTable(const InternalKeyComparator& cmp, + const ImmutableOptions& ioptions, + const MutableCFOptions& mutable_cf_options, + WriteBufferManager* write_buffer_manager, + SequenceNumber latest_seq, uint32_t column_family_id) + : comparator_(cmp), + moptions_(ioptions, mutable_cf_options), + refs_(0), + kArenaBlockSize(Arena::OptimizeBlockSize(moptions_.arena_block_size)), + mem_tracker_(write_buffer_manager), + arena_(moptions_.arena_block_size, + (write_buffer_manager != nullptr && + (write_buffer_manager->enabled() || + write_buffer_manager->cost_to_cache())) + ? &mem_tracker_ + : nullptr, + mutable_cf_options.memtable_huge_page_size), + table_(ioptions.memtable_factory->CreateMemTableRep( + comparator_, &arena_, mutable_cf_options.prefix_extractor.get(), + ioptions.logger, column_family_id)), + range_del_table_(SkipListFactory().CreateMemTableRep( + comparator_, &arena_, nullptr /* transform */, ioptions.logger, + column_family_id)), + is_range_del_table_empty_(true), + data_size_(0), + num_entries_(0), + num_deletes_(0), + num_range_deletes_(0), + write_buffer_size_(mutable_cf_options.write_buffer_size), + flush_in_progress_(false), + flush_completed_(false), + file_number_(0), + first_seqno_(0), + earliest_seqno_(latest_seq), + creation_seq_(latest_seq), + mem_next_logfile_number_(0), + min_prep_log_referenced_(0), + locks_(moptions_.inplace_update_support + ? moptions_.inplace_update_num_locks + : 0), + prefix_extractor_(mutable_cf_options.prefix_extractor.get()), + flush_state_(FLUSH_NOT_REQUESTED), + clock_(ioptions.clock), + insert_with_hint_prefix_extractor_( + ioptions.memtable_insert_with_hint_prefix_extractor.get()), + oldest_key_time_(std::numeric_limits::max()), + atomic_flush_seqno_(kMaxSequenceNumber), + approximate_memory_usage_(0), + memtable_max_range_deletions_( + mutable_cf_options.memtable_max_range_deletions) { + UpdateFlushState(); + // something went wrong if we need to flush before inserting anything + assert(!ShouldScheduleFlush()); + + // use bloom_filter_ for both whole key and prefix bloom filter + if ((prefix_extractor_ || moptions_.memtable_whole_key_filtering) && + moptions_.memtable_prefix_bloom_bits > 0) { + bloom_filter_.reset( + new DynamicBloom(&arena_, moptions_.memtable_prefix_bloom_bits, + 6 /* hard coded 6 probes */, + moptions_.memtable_huge_page_size, ioptions.logger)); + } + // Initialize cached_range_tombstone_ here since it could + // be read before it is constructed in MemTable::Add(), which could also lead + // to a data race on the global mutex table backing atomic shared_ptr. + auto new_cache = std::make_shared(); + size_t size = cached_range_tombstone_.Size(); + for (size_t i = 0; i < size; ++i) { + std::shared_ptr* local_cache_ref_ptr = + cached_range_tombstone_.AccessAtCore(i); + auto new_local_cache_ref = std::make_shared< + const std::shared_ptr>(new_cache); + std::atomic_store_explicit( + local_cache_ref_ptr, + std::shared_ptr(new_local_cache_ref, + new_cache.get()), + std::memory_order_relaxed); + } + const Comparator* ucmp = cmp.user_comparator(); + assert(ucmp); + ts_sz_ = ucmp->timestamp_size(); + persist_user_defined_timestamps_ = ioptions.persist_user_defined_timestamps; +} + +MemTable::~MemTable() { + mem_tracker_.FreeMem(); + assert(refs_ == 0); +} + +size_t MemTable::ApproximateMemoryUsage() { + autovector usages = { + arena_.ApproximateMemoryUsage(), table_->ApproximateMemoryUsage(), + range_del_table_->ApproximateMemoryUsage(), + ROCKSDB_NAMESPACE::ApproximateMemoryUsage(insert_hints_)}; + size_t total_usage = 0; + for (size_t usage : usages) { + // If usage + total_usage >= kMaxSizet, return kMaxSizet. + // the following variation is to avoid numeric overflow. + if (usage >= std::numeric_limits::max() - total_usage) { + return std::numeric_limits::max(); + } + total_usage += usage; + } + approximate_memory_usage_.store(total_usage, std::memory_order_relaxed); + // otherwise, return the actual usage + return total_usage; +} + +bool MemTable::ShouldFlushNow() { + // This is set if memtable_max_range_deletions is > 0, + // and that many range deletions are done + if (memtable_max_range_deletions_ > 0 && + num_range_deletes_.load(std::memory_order_relaxed) >= + static_cast(memtable_max_range_deletions_)) { + return true; + } + + size_t write_buffer_size = write_buffer_size_.load(std::memory_order_relaxed); + // In a lot of times, we cannot allocate arena blocks that exactly matches the + // buffer size. Thus we have to decide if we should over-allocate or + // under-allocate. + // This constant variable can be interpreted as: if we still have more than + // "kAllowOverAllocationRatio * kArenaBlockSize" space left, we'd try to over + // allocate one more block. + const double kAllowOverAllocationRatio = 0.6; + + // If arena still have room for new block allocation, we can safely say it + // shouldn't flush. + auto allocated_memory = table_->ApproximateMemoryUsage() + + range_del_table_->ApproximateMemoryUsage() + + arena_.MemoryAllocatedBytes(); + + approximate_memory_usage_.store(allocated_memory, std::memory_order_relaxed); + + // if we can still allocate one more block without exceeding the + // over-allocation ratio, then we should not flush. + if (allocated_memory + kArenaBlockSize < + write_buffer_size + kArenaBlockSize * kAllowOverAllocationRatio) { + return false; + } + + // if user keeps adding entries that exceeds write_buffer_size, we need to + // flush earlier even though we still have much available memory left. + if (allocated_memory > + write_buffer_size + kArenaBlockSize * kAllowOverAllocationRatio) { + return true; + } + + // In this code path, Arena has already allocated its "last block", which + // means the total allocatedmemory size is either: + // (1) "moderately" over allocated the memory (no more than `0.6 * arena + // block size`. Or, + // (2) the allocated memory is less than write buffer size, but we'll stop + // here since if we allocate a new arena block, we'll over allocate too much + // more (half of the arena block size) memory. + // + // In either case, to avoid over-allocate, the last block will stop allocation + // when its usage reaches a certain ratio, which we carefully choose "0.75 + // full" as the stop condition because it addresses the following issue with + // great simplicity: What if the next inserted entry's size is + // bigger than AllocatedAndUnused()? + // + // The answer is: if the entry size is also bigger than 0.25 * + // kArenaBlockSize, a dedicated block will be allocated for it; otherwise + // arena will anyway skip the AllocatedAndUnused() and allocate a new, empty + // and regular block. In either case, we *overly* over-allocated. + // + // Therefore, setting the last block to be at most "0.75 full" avoids both + // cases. + // + // NOTE: the average percentage of waste space of this approach can be counted + // as: "arena block size * 0.25 / write buffer size". User who specify a small + // write buffer size and/or big arena block size may suffer. + return arena_.AllocatedAndUnused() < kArenaBlockSize / 4; +} + +void MemTable::UpdateFlushState() { + auto state = flush_state_.load(std::memory_order_relaxed); + if (state == FLUSH_NOT_REQUESTED && ShouldFlushNow()) { + // ignore CAS failure, because that means somebody else requested + // a flush + flush_state_.compare_exchange_strong(state, FLUSH_REQUESTED, + std::memory_order_relaxed, + std::memory_order_relaxed); + } +} + +void MemTable::UpdateOldestKeyTime() { + uint64_t oldest_key_time = oldest_key_time_.load(std::memory_order_relaxed); + if (oldest_key_time == std::numeric_limits::max()) { + int64_t current_time = 0; + auto s = clock_->GetCurrentTime(¤t_time); + if (s.ok()) { + assert(current_time >= 0); + // If fail, the timestamp is already set. + oldest_key_time_.compare_exchange_strong( + oldest_key_time, static_cast(current_time), + std::memory_order_relaxed, std::memory_order_relaxed); + } + } +} + +Status MemTable::VerifyEntryChecksum(const char* entry, + uint32_t protection_bytes_per_key, + bool allow_data_in_errors) { + if (protection_bytes_per_key == 0) { + return Status::OK(); + } + uint32_t key_length; + const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length); + if (key_ptr == nullptr) { + return Status::Corruption("Unable to parse internal key length"); + } + if (key_length < 8) { + return Status::Corruption("Memtable entry internal key length too short."); + } + Slice user_key = Slice(key_ptr, key_length - 8); + + const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8); + ValueType type; + SequenceNumber seq; + UnPackSequenceAndType(tag, &seq, &type); + + uint32_t value_length = 0; + const char* value_ptr = GetVarint32Ptr( + key_ptr + key_length, key_ptr + key_length + 5, &value_length); + if (value_ptr == nullptr) { + return Status::Corruption("Unable to parse internal key value"); + } + Slice value = Slice(value_ptr, value_length); + + const char* checksum_ptr = value_ptr + value_length; + bool match = + ProtectionInfo64() + .ProtectKVO(user_key, value, type) + .ProtectS(seq) + .Verify(static_cast(protection_bytes_per_key), checksum_ptr); + if (!match) { + std::string msg( + "Corrupted memtable entry, per key-value checksum verification " + "failed."); + if (allow_data_in_errors) { + msg.append("Unrecognized value type: " + + std::to_string(static_cast(type)) + ". "); + msg.append("User key: " + user_key.ToString(/*hex=*/true) + ". "); + msg.append("seq: " + std::to_string(seq) + "."); + } + return Status::Corruption(msg.c_str()); + } + return Status::OK(); +} + +int MemTable::KeyComparator::operator()(const char* prefix_len_key1, + const char* prefix_len_key2) const { + // Internal keys are encoded as length-prefixed strings. + Slice k1 = GetLengthPrefixedSlice(prefix_len_key1); + Slice k2 = GetLengthPrefixedSlice(prefix_len_key2); + return comparator.CompareKeySeq(k1, k2); +} + +int MemTable::KeyComparator::operator()( + const char* prefix_len_key, const KeyComparator::DecodedType& key) const { + // Internal keys are encoded as length-prefixed strings. + Slice a = GetLengthPrefixedSlice(prefix_len_key); + return comparator.CompareKeySeq(a, key); +} + +void MemTableRep::InsertConcurrently(KeyHandle /*handle*/) { + throw std::runtime_error("concurrent insert not supported"); +} + +Slice MemTableRep::UserKey(const char* key) const { + Slice slice = GetLengthPrefixedSlice(key); + return Slice(slice.data(), slice.size() - 8); +} + +KeyHandle MemTableRep::Allocate(const size_t len, char** buf) { + *buf = allocator_->Allocate(len); + return static_cast(*buf); +} + +// Encode a suitable internal key target for "target" and return it. +// Uses *scratch as scratch space, and the returned pointer will point +// into this scratch space. +const char* EncodeKey(std::string* scratch, const Slice& target) { + scratch->clear(); + PutVarint32(scratch, static_cast(target.size())); + scratch->append(target.data(), target.size()); + return scratch->data(); +} + +class MemTableIterator : public InternalIterator { + public: + MemTableIterator(const MemTable& mem, const ReadOptions& read_options, + Arena* arena, bool use_range_del_table = false) + : bloom_(nullptr), + prefix_extractor_(mem.prefix_extractor_), + comparator_(mem.comparator_), + valid_(false), + arena_mode_(arena != nullptr), + value_pinned_( + !mem.GetImmutableMemTableOptions()->inplace_update_support), + protection_bytes_per_key_(mem.moptions_.protection_bytes_per_key), + status_(Status::OK()), + logger_(mem.moptions_.info_log), + ts_sz_(mem.ts_sz_) { + if (use_range_del_table) { + iter_ = mem.range_del_table_->GetIterator(arena); + } else if (prefix_extractor_ != nullptr && !read_options.total_order_seek && + !read_options.auto_prefix_mode) { + // Auto prefix mode is not implemented in memtable yet. + bloom_ = mem.bloom_filter_.get(); + iter_ = mem.table_->GetDynamicPrefixIterator(arena); + } else { + iter_ = mem.table_->GetIterator(arena); + } + status_.PermitUncheckedError(); + } + // No copying allowed + MemTableIterator(const MemTableIterator&) = delete; + void operator=(const MemTableIterator&) = delete; + + ~MemTableIterator() override { +#ifndef NDEBUG + // Assert that the MemTableIterator is never deleted while + // Pinning is Enabled. + assert(!pinned_iters_mgr_ || !pinned_iters_mgr_->PinningEnabled()); +#endif + if (arena_mode_) { + iter_->~Iterator(); + } else { + delete iter_; + } + } + +#ifndef NDEBUG + void SetPinnedItersMgr(PinnedIteratorsManager* pinned_iters_mgr) override { + pinned_iters_mgr_ = pinned_iters_mgr; + } + PinnedIteratorsManager* pinned_iters_mgr_ = nullptr; +#endif + + bool Valid() const override { return valid_ && status_.ok(); } + void Seek(const Slice& k) override { + PERF_TIMER_GUARD(seek_on_memtable_time); + PERF_COUNTER_ADD(seek_on_memtable_count, 1); + if (bloom_) { + // iterator should only use prefix bloom filter + Slice user_k_without_ts(ExtractUserKeyAndStripTimestamp(k, ts_sz_)); + if (prefix_extractor_->InDomain(user_k_without_ts)) { + if (!bloom_->MayContain( + prefix_extractor_->Transform(user_k_without_ts))) { + PERF_COUNTER_ADD(bloom_memtable_miss_count, 1); + valid_ = false; + return; + } else { + PERF_COUNTER_ADD(bloom_memtable_hit_count, 1); + } + } + } + iter_->Seek(k, nullptr); + valid_ = iter_->Valid(); + VerifyEntryChecksum(); + } + void SeekForPrev(const Slice& k) override { + PERF_TIMER_GUARD(seek_on_memtable_time); + PERF_COUNTER_ADD(seek_on_memtable_count, 1); + if (bloom_) { + Slice user_k_without_ts(ExtractUserKeyAndStripTimestamp(k, ts_sz_)); + if (prefix_extractor_->InDomain(user_k_without_ts)) { + if (!bloom_->MayContain( + prefix_extractor_->Transform(user_k_without_ts))) { + PERF_COUNTER_ADD(bloom_memtable_miss_count, 1); + valid_ = false; + return; + } else { + PERF_COUNTER_ADD(bloom_memtable_hit_count, 1); + } + } + } + iter_->Seek(k, nullptr); + valid_ = iter_->Valid(); + VerifyEntryChecksum(); + if (!Valid() && status().ok()) { + SeekToLast(); + } + while (Valid() && comparator_.comparator.Compare(k, key()) < 0) { + Prev(); + } + } + void SeekToFirst() override { + iter_->SeekToFirst(); + valid_ = iter_->Valid(); + VerifyEntryChecksum(); + } + void SeekToLast() override { + iter_->SeekToLast(); + valid_ = iter_->Valid(); + VerifyEntryChecksum(); + } + void Next() override { + PERF_COUNTER_ADD(next_on_memtable_count, 1); + assert(Valid()); + iter_->Next(); + TEST_SYNC_POINT_CALLBACK("MemTableIterator::Next:0", iter_); + valid_ = iter_->Valid(); + VerifyEntryChecksum(); + } + bool NextAndGetResult(IterateResult* result) override { + Next(); + bool is_valid = Valid(); + if (is_valid) { + result->key = key(); + result->bound_check_result = IterBoundCheck::kUnknown; + result->value_prepared = true; + } + return is_valid; + } + void Prev() override { + PERF_COUNTER_ADD(prev_on_memtable_count, 1); + assert(Valid()); + iter_->Prev(); + valid_ = iter_->Valid(); + VerifyEntryChecksum(); + } + Slice key() const override { + assert(Valid()); + return GetLengthPrefixedSlice(iter_->key()); + } + Slice value() const override { + assert(Valid()); + Slice key_slice = GetLengthPrefixedSlice(iter_->key()); + return GetLengthPrefixedSlice(key_slice.data() + key_slice.size()); + } + + Status status() const override { return status_; } + + bool IsKeyPinned() const override { + // memtable data is always pinned + return true; + } + + bool IsValuePinned() const override { + // memtable value is always pinned, except if we allow inplace update. + return value_pinned_; + } + + private: + DynamicBloom* bloom_; + const SliceTransform* const prefix_extractor_; + const MemTable::KeyComparator comparator_; + MemTableRep::Iterator* iter_; + bool valid_; + bool arena_mode_; + bool value_pinned_; + uint32_t protection_bytes_per_key_; + Status status_; + Logger* logger_; + size_t ts_sz_; + + void VerifyEntryChecksum() { + if (protection_bytes_per_key_ > 0 && Valid()) { + status_ = MemTable::VerifyEntryChecksum(iter_->key(), + protection_bytes_per_key_); + if (!status_.ok()) { + ROCKS_LOG_ERROR(logger_, "In MemtableIterator: %s", status_.getState()); + } + } + } +}; + +InternalIterator* MemTable::NewIterator(const ReadOptions& read_options, + Arena* arena) { + assert(arena != nullptr); + auto mem = arena->AllocateAligned(sizeof(MemTableIterator)); + return new (mem) MemTableIterator(*this, read_options, arena); +} + +FragmentedRangeTombstoneIterator* MemTable::NewRangeTombstoneIterator( + const ReadOptions& read_options, SequenceNumber read_seq, + bool immutable_memtable) { + if (read_options.ignore_range_deletions || + is_range_del_table_empty_.load(std::memory_order_relaxed)) { + return nullptr; + } + return NewRangeTombstoneIteratorInternal(read_options, read_seq, + immutable_memtable); +} + +FragmentedRangeTombstoneIterator* MemTable::NewRangeTombstoneIteratorInternal( + const ReadOptions& read_options, SequenceNumber read_seq, + bool immutable_memtable) { + if (immutable_memtable) { + // Note that caller should already have verified that + // !is_range_del_table_empty_ + assert(IsFragmentedRangeTombstonesConstructed()); + return new FragmentedRangeTombstoneIterator( + fragmented_range_tombstone_list_.get(), comparator_.comparator, + read_seq, read_options.timestamp); + } + + // takes current cache + std::shared_ptr cache = + std::atomic_load_explicit(cached_range_tombstone_.Access(), + std::memory_order_relaxed); + // construct fragmented tombstone list if necessary + if (!cache->initialized.load(std::memory_order_acquire)) { + cache->reader_mutex.lock(); + if (!cache->tombstones) { + auto* unfragmented_iter = + new MemTableIterator(*this, read_options, nullptr /* arena */, + true /* use_range_del_table */); + cache->tombstones.reset(new FragmentedRangeTombstoneList( + std::unique_ptr(unfragmented_iter), + comparator_.comparator)); + cache->initialized.store(true, std::memory_order_release); + } + cache->reader_mutex.unlock(); + } + + auto* fragmented_iter = new FragmentedRangeTombstoneIterator( + cache, comparator_.comparator, read_seq, read_options.timestamp); + return fragmented_iter; +} + +void MemTable::ConstructFragmentedRangeTombstones() { + assert(!IsFragmentedRangeTombstonesConstructed(false)); + // There should be no concurrent Construction + if (!is_range_del_table_empty_.load(std::memory_order_relaxed)) { + // TODO: plumb Env::IOActivity + auto* unfragmented_iter = + new MemTableIterator(*this, ReadOptions(), nullptr /* arena */, + true /* use_range_del_table */); + + fragmented_range_tombstone_list_ = + std::make_unique( + std::unique_ptr(unfragmented_iter), + comparator_.comparator); + } +} + +port::RWMutex* MemTable::GetLock(const Slice& key) { + return &locks_[GetSliceRangedNPHash(key, locks_.size())]; +} + +MemTable::MemTableStats MemTable::ApproximateStats(const Slice& start_ikey, + const Slice& end_ikey) { + uint64_t entry_count = table_->ApproximateNumEntries(start_ikey, end_ikey); + entry_count += range_del_table_->ApproximateNumEntries(start_ikey, end_ikey); + if (entry_count == 0) { + return {0, 0}; + } + uint64_t n = num_entries_.load(std::memory_order_relaxed); + if (n == 0) { + return {0, 0}; + } + if (entry_count > n) { + // (range_del_)table_->ApproximateNumEntries() is just an estimate so it can + // be larger than actual entries we have. Cap it to entries we have to limit + // the inaccuracy. + entry_count = n; + } + uint64_t data_size = data_size_.load(std::memory_order_relaxed); + return {entry_count * (data_size / n), entry_count}; +} + +Status MemTable::VerifyEncodedEntry(Slice encoded, + const ProtectionInfoKVOS64& kv_prot_info) { + uint32_t ikey_len = 0; + if (!GetVarint32(&encoded, &ikey_len)) { + return Status::Corruption("Unable to parse internal key length"); + } + if (ikey_len < 8 + ts_sz_) { + return Status::Corruption("Internal key length too short"); + } + if (ikey_len > encoded.size()) { + return Status::Corruption("Internal key length too long"); + } + uint32_t value_len = 0; + const size_t user_key_len = ikey_len - 8; + Slice key(encoded.data(), user_key_len); + encoded.remove_prefix(user_key_len); + + uint64_t packed = DecodeFixed64(encoded.data()); + ValueType value_type = kMaxValue; + SequenceNumber sequence_number = kMaxSequenceNumber; + UnPackSequenceAndType(packed, &sequence_number, &value_type); + encoded.remove_prefix(8); + + if (!GetVarint32(&encoded, &value_len)) { + return Status::Corruption("Unable to parse value length"); + } + if (value_len < encoded.size()) { + return Status::Corruption("Value length too short"); + } + if (value_len > encoded.size()) { + return Status::Corruption("Value length too long"); + } + Slice value(encoded.data(), value_len); + + return kv_prot_info.StripS(sequence_number) + .StripKVO(key, value, value_type) + .GetStatus(); +} + +void MemTable::UpdateEntryChecksum(const ProtectionInfoKVOS64* kv_prot_info, + const Slice& key, const Slice& value, + ValueType type, SequenceNumber s, + char* checksum_ptr) { + if (moptions_.protection_bytes_per_key == 0) { + return; + } + + if (kv_prot_info == nullptr) { + ProtectionInfo64() + .ProtectKVO(key, value, type) + .ProtectS(s) + .Encode(static_cast(moptions_.protection_bytes_per_key), + checksum_ptr); + } else { + kv_prot_info->Encode( + static_cast(moptions_.protection_bytes_per_key), checksum_ptr); + } +} + +Status MemTable::Add(SequenceNumber s, ValueType type, + const Slice& key, /* user key */ + const Slice& value, + const ProtectionInfoKVOS64* kv_prot_info, + bool allow_concurrent, + MemTablePostProcessInfo* post_process_info, void** hint) { + // Format of an entry is concatenation of: + // key_size : varint32 of internal_key.size() + // key bytes : char[internal_key.size()] + // value_size : varint32 of value.size() + // value bytes : char[value.size()] + // checksum : char[moptions_.protection_bytes_per_key] + uint32_t key_size = static_cast(key.size()); + uint32_t val_size = static_cast(value.size()); + uint32_t internal_key_size = key_size + 8; + const uint32_t encoded_len = VarintLength(internal_key_size) + + internal_key_size + VarintLength(val_size) + + val_size + moptions_.protection_bytes_per_key; + char* buf = nullptr; + std::unique_ptr& table = + type == kTypeRangeDeletion ? range_del_table_ : table_; + KeyHandle handle = table->Allocate(encoded_len, &buf); + + char* p = EncodeVarint32(buf, internal_key_size); + memcpy(p, key.data(), key_size); + Slice key_slice(p, key_size); + p += key_size; + uint64_t packed = PackSequenceAndType(s, type); + EncodeFixed64(p, packed); + p += 8; + p = EncodeVarint32(p, val_size); + memcpy(p, value.data(), val_size); + assert((unsigned)(p + val_size - buf + moptions_.protection_bytes_per_key) == + (unsigned)encoded_len); + + UpdateEntryChecksum(kv_prot_info, key, value, type, s, + buf + encoded_len - moptions_.protection_bytes_per_key); + Slice encoded(buf, encoded_len - moptions_.protection_bytes_per_key); + if (kv_prot_info != nullptr) { + TEST_SYNC_POINT_CALLBACK("MemTable::Add:Encoded", &encoded); + Status status = VerifyEncodedEntry(encoded, *kv_prot_info); + if (!status.ok()) { + return status; + } + } + + Slice key_without_ts = StripTimestampFromUserKey(key, ts_sz_); + + if (!allow_concurrent) { + // Extract prefix for insert with hint. + if (insert_with_hint_prefix_extractor_ != nullptr && + insert_with_hint_prefix_extractor_->InDomain(key_slice)) { + Slice prefix = insert_with_hint_prefix_extractor_->Transform(key_slice); + bool res = table->InsertKeyWithHint(handle, &insert_hints_[prefix]); + if (UNLIKELY(!res)) { + return Status::TryAgain("key+seq exists"); + } + } else { + bool res = table->InsertKey(handle); + if (UNLIKELY(!res)) { + return Status::TryAgain("key+seq exists"); + } + } + + // this is a bit ugly, but is the way to avoid locked instructions + // when incrementing an atomic + num_entries_.store(num_entries_.load(std::memory_order_relaxed) + 1, + std::memory_order_relaxed); + data_size_.store(data_size_.load(std::memory_order_relaxed) + encoded_len, + std::memory_order_relaxed); + if (type == kTypeDeletion || type == kTypeSingleDeletion || + type == kTypeDeletionWithTimestamp) { + num_deletes_.store(num_deletes_.load(std::memory_order_relaxed) + 1, + std::memory_order_relaxed); + } else if (type == kTypeRangeDeletion) { + uint64_t val = num_range_deletes_.load(std::memory_order_relaxed) + 1; + num_range_deletes_.store(val, std::memory_order_relaxed); + } + + if (bloom_filter_ && prefix_extractor_ && + prefix_extractor_->InDomain(key_without_ts)) { + bloom_filter_->Add(prefix_extractor_->Transform(key_without_ts)); + } + if (bloom_filter_ && moptions_.memtable_whole_key_filtering) { + bloom_filter_->Add(key_without_ts); + } + + // The first sequence number inserted into the memtable + assert(first_seqno_ == 0 || s >= first_seqno_); + if (first_seqno_ == 0) { + first_seqno_.store(s, std::memory_order_relaxed); + + if (earliest_seqno_ == kMaxSequenceNumber) { + earliest_seqno_.store(GetFirstSequenceNumber(), + std::memory_order_relaxed); + } + assert(first_seqno_.load() >= earliest_seqno_.load()); + } + assert(post_process_info == nullptr); + // TODO(yuzhangyu): support updating newest UDT for when `allow_concurrent` + // is true. + MaybeUpdateNewestUDT(key_slice); + UpdateFlushState(); + } else { + bool res = (hint == nullptr) + ? table->InsertKeyConcurrently(handle) + : table->InsertKeyWithHintConcurrently(handle, hint); + if (UNLIKELY(!res)) { + return Status::TryAgain("key+seq exists"); + } + + assert(post_process_info != nullptr); + post_process_info->num_entries++; + post_process_info->data_size += encoded_len; + if (type == kTypeDeletion) { + post_process_info->num_deletes++; + } + + if (bloom_filter_ && prefix_extractor_ && + prefix_extractor_->InDomain(key_without_ts)) { + bloom_filter_->AddConcurrently( + prefix_extractor_->Transform(key_without_ts)); + } + if (bloom_filter_ && moptions_.memtable_whole_key_filtering) { + bloom_filter_->AddConcurrently(key_without_ts); + } + + // atomically update first_seqno_ and earliest_seqno_. + uint64_t cur_seq_num = first_seqno_.load(std::memory_order_relaxed); + while ((cur_seq_num == 0 || s < cur_seq_num) && + !first_seqno_.compare_exchange_weak(cur_seq_num, s)) { + } + uint64_t cur_earliest_seqno = + earliest_seqno_.load(std::memory_order_relaxed); + while ( + (cur_earliest_seqno == kMaxSequenceNumber || s < cur_earliest_seqno) && + !earliest_seqno_.compare_exchange_weak(cur_earliest_seqno, s)) { + } + } + if (type == kTypeRangeDeletion) { + auto new_cache = std::make_shared(); + size_t size = cached_range_tombstone_.Size(); + if (allow_concurrent) { + post_process_info->num_range_deletes++; + range_del_mutex_.lock(); + } + for (size_t i = 0; i < size; ++i) { + std::shared_ptr* local_cache_ref_ptr = + cached_range_tombstone_.AccessAtCore(i); + auto new_local_cache_ref = std::make_shared< + const std::shared_ptr>(new_cache); + // It is okay for some reader to load old cache during invalidation as + // the new sequence number is not published yet. + // Each core will have a shared_ptr to a shared_ptr to the cached + // fragmented range tombstones, so that ref count is maintianed locally + // per-core using the per-core shared_ptr. + std::atomic_store_explicit( + local_cache_ref_ptr, + std::shared_ptr( + new_local_cache_ref, new_cache.get()), + std::memory_order_relaxed); + } + + if (allow_concurrent) { + range_del_mutex_.unlock(); + } + is_range_del_table_empty_.store(false, std::memory_order_relaxed); + } + UpdateOldestKeyTime(); + + TEST_SYNC_POINT_CALLBACK("MemTable::Add:BeforeReturn:Encoded", &encoded); + return Status::OK(); +} + +// Callback from MemTable::Get() +namespace { + +struct Saver { + Status* status; + const LookupKey* key; + bool* found_final_value; // Is value set correctly? Used by KeyMayExist + bool* merge_in_progress; + std::string* value; + PinnableWideColumns* columns; + SequenceNumber seq; + std::string* timestamp; + const MergeOperator* merge_operator; + // the merge operations encountered; + MergeContext* merge_context; + SequenceNumber max_covering_tombstone_seq; + MemTable* mem; + Logger* logger; + Statistics* statistics; + bool inplace_update_support; + bool do_merge; + SystemClock* clock; + + ReadCallback* callback_; + bool* is_blob_index; + bool allow_data_in_errors; + uint32_t protection_bytes_per_key; + bool CheckCallback(SequenceNumber _seq) { + if (callback_) { + return callback_->IsVisible(_seq); + } + return true; + } +}; +} // anonymous namespace + +static bool SaveValue(void* arg, const char* entry) { + TEST_SYNC_POINT_CALLBACK("Memtable::SaveValue:Begin:entry", &entry); + Saver* s = reinterpret_cast(arg); + assert(s != nullptr); + assert(!s->value || !s->columns); + + if (s->protection_bytes_per_key > 0) { + *(s->status) = MemTable::VerifyEntryChecksum( + entry, s->protection_bytes_per_key, s->allow_data_in_errors); + if (!s->status->ok()) { + ROCKS_LOG_ERROR(s->logger, "In SaveValue: %s", s->status->getState()); + // Memtable entry corrupted + return false; + } + } + + MergeContext* merge_context = s->merge_context; + SequenceNumber max_covering_tombstone_seq = s->max_covering_tombstone_seq; + const MergeOperator* merge_operator = s->merge_operator; + + assert(merge_context != nullptr); + + // Refer to comments under MemTable::Add() for entry format. + // Check that it belongs to same user key. + uint32_t key_length = 0; + const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length); + assert(key_length >= 8); + Slice user_key_slice = Slice(key_ptr, key_length - 8); + const Comparator* user_comparator = + s->mem->GetInternalKeyComparator().user_comparator(); + size_t ts_sz = user_comparator->timestamp_size(); + if (ts_sz && s->timestamp && max_covering_tombstone_seq > 0) { + // timestamp should already be set to range tombstone timestamp + assert(s->timestamp->size() == ts_sz); + } + if (user_comparator->EqualWithoutTimestamp(user_key_slice, + s->key->user_key())) { + // Correct user key + const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8); + ValueType type; + SequenceNumber seq; + UnPackSequenceAndType(tag, &seq, &type); + // If the value is not in the snapshot, skip it + if (!s->CheckCallback(seq)) { + return true; // to continue to the next seq + } + + if (s->seq == kMaxSequenceNumber) { + s->seq = seq; + if (s->seq > max_covering_tombstone_seq) { + if (ts_sz && s->timestamp != nullptr) { + // `timestamp` was set to range tombstone's timestamp before + // `SaveValue` is ever called. This key has a higher sequence number + // than range tombstone, and is the key with the highest seqno across + // all keys with this user_key, so we update timestamp here. + Slice ts = ExtractTimestampFromUserKey(user_key_slice, ts_sz); + s->timestamp->assign(ts.data(), ts_sz); + } + } else { + s->seq = max_covering_tombstone_seq; + } + } + + if (ts_sz > 0 && s->timestamp != nullptr) { + if (!s->timestamp->empty()) { + assert(ts_sz == s->timestamp->size()); + } + // TODO optimize for smaller size ts + const std::string kMaxTs(ts_sz, '\xff'); + if (s->timestamp->empty() || + user_comparator->CompareTimestamp(*(s->timestamp), kMaxTs) == 0) { + Slice ts = ExtractTimestampFromUserKey(user_key_slice, ts_sz); + s->timestamp->assign(ts.data(), ts_sz); + } + } + + if ((type == kTypeValue || type == kTypeMerge || type == kTypeBlobIndex || + type == kTypeWideColumnEntity || type == kTypeDeletion || + type == kTypeSingleDeletion || type == kTypeDeletionWithTimestamp) && + max_covering_tombstone_seq > seq) { + type = kTypeRangeDeletion; + } + switch (type) { + case kTypeBlobIndex: { + if (!s->do_merge) { + *(s->status) = Status::NotSupported( + "GetMergeOperands not supported by stacked BlobDB"); + *(s->found_final_value) = true; + return false; + } + + if (*(s->merge_in_progress)) { + *(s->status) = Status::NotSupported( + "Merge operator not supported by stacked BlobDB"); + *(s->found_final_value) = true; + return false; + } + + if (s->is_blob_index == nullptr) { + ROCKS_LOG_ERROR(s->logger, "Encountered unexpected blob index."); + *(s->status) = Status::NotSupported( + "Encountered unexpected blob index. Please open DB with " + "ROCKSDB_NAMESPACE::blob_db::BlobDB."); + *(s->found_final_value) = true; + return false; + } + + if (s->inplace_update_support) { + s->mem->GetLock(s->key->user_key())->ReadLock(); + } + + Slice v = GetLengthPrefixedSlice(key_ptr + key_length); + + *(s->status) = Status::OK(); + + if (s->value) { + s->value->assign(v.data(), v.size()); + } else if (s->columns) { + s->columns->SetPlainValue(v); + } + + if (s->inplace_update_support) { + s->mem->GetLock(s->key->user_key())->ReadUnlock(); + } + + *(s->found_final_value) = true; + *(s->is_blob_index) = true; + + return false; + } + case kTypeValue: { + if (s->inplace_update_support) { + s->mem->GetLock(s->key->user_key())->ReadLock(); + } + + Slice v = GetLengthPrefixedSlice(key_ptr + key_length); + + *(s->status) = Status::OK(); + + if (!s->do_merge) { + // Preserve the value with the goal of returning it as part of + // raw merge operands to the user + // TODO(yanqin) update MergeContext so that timestamps information + // can also be retained. + + merge_context->PushOperand( + v, s->inplace_update_support == false /* operand_pinned */); + } else if (*(s->merge_in_progress)) { + assert(s->do_merge); + + if (s->value || s->columns) { + std::string result; + // `op_failure_scope` (an output parameter) is not provided (set to + // nullptr) since a failure must be propagated regardless of its + // value. + *(s->status) = MergeHelper::TimedFullMerge( + merge_operator, s->key->user_key(), &v, + merge_context->GetOperands(), &result, s->logger, s->statistics, + s->clock, /* result_operand */ nullptr, + /* update_num_ops_stats */ true, + /* op_failure_scope */ nullptr); + + if (s->status->ok()) { + if (s->value) { + *(s->value) = std::move(result); + } else { + assert(s->columns); + s->columns->SetPlainValue(std::move(result)); + } + } + } + } else if (s->value) { + s->value->assign(v.data(), v.size()); + } else if (s->columns) { + s->columns->SetPlainValue(v); + } + + if (s->inplace_update_support) { + s->mem->GetLock(s->key->user_key())->ReadUnlock(); + } + + *(s->found_final_value) = true; + + if (s->is_blob_index != nullptr) { + *(s->is_blob_index) = false; + } + + return false; + } + case kTypeWideColumnEntity: { + if (s->inplace_update_support) { + s->mem->GetLock(s->key->user_key())->ReadLock(); + } + + Slice v = GetLengthPrefixedSlice(key_ptr + key_length); + + *(s->status) = Status::OK(); + + if (!s->do_merge) { + // Preserve the value with the goal of returning it as part of + // raw merge operands to the user + + Slice value_of_default; + *(s->status) = WideColumnSerialization::GetValueOfDefaultColumn( + v, value_of_default); + + if (s->status->ok()) { + merge_context->PushOperand( + value_of_default, + s->inplace_update_support == false /* operand_pinned */); + } + } else if (*(s->merge_in_progress)) { + assert(s->do_merge); + + if (s->value) { + Slice value_of_default; + *(s->status) = WideColumnSerialization::GetValueOfDefaultColumn( + v, value_of_default); + if (s->status->ok()) { + // `op_failure_scope` (an output parameter) is not provided (set + // to nullptr) since a failure must be propagated regardless of + // its value. + *(s->status) = MergeHelper::TimedFullMerge( + merge_operator, s->key->user_key(), &value_of_default, + merge_context->GetOperands(), s->value, s->logger, + s->statistics, s->clock, /* result_operand */ nullptr, + /* update_num_ops_stats */ true, + /* op_failure_scope */ nullptr); + } + } else if (s->columns) { + std::string result; + // `op_failure_scope` (an output parameter) is not provided (set to + // nullptr) since a failure must be propagated regardless of its + // value. + *(s->status) = MergeHelper::TimedFullMergeWithEntity( + merge_operator, s->key->user_key(), v, + merge_context->GetOperands(), &result, s->logger, s->statistics, + s->clock, /* update_num_ops_stats */ true, + /* op_failure_scope */ nullptr); + + if (s->status->ok()) { + *(s->status) = s->columns->SetWideColumnValue(std::move(result)); + } + } + } else if (s->value) { + Slice value_of_default; + *(s->status) = WideColumnSerialization::GetValueOfDefaultColumn( + v, value_of_default); + if (s->status->ok()) { + s->value->assign(value_of_default.data(), value_of_default.size()); + } + } else if (s->columns) { + *(s->status) = s->columns->SetWideColumnValue(v); + } + + if (s->inplace_update_support) { + s->mem->GetLock(s->key->user_key())->ReadUnlock(); + } + + *(s->found_final_value) = true; + + if (s->is_blob_index != nullptr) { + *(s->is_blob_index) = false; + } + + return false; + } + case kTypeDeletion: + case kTypeDeletionWithTimestamp: + case kTypeSingleDeletion: + case kTypeRangeDeletion: { + if (*(s->merge_in_progress)) { + if (s->value || s->columns) { + std::string result; + // `op_failure_scope` (an output parameter) is not provided (set to + // nullptr) since a failure must be propagated regardless of its + // value. + *(s->status) = MergeHelper::TimedFullMerge( + merge_operator, s->key->user_key(), nullptr, + merge_context->GetOperands(), &result, s->logger, s->statistics, + s->clock, /* result_operand */ nullptr, + /* update_num_ops_stats */ true, + /* op_failure_scope */ nullptr); + + if (s->status->ok()) { + if (s->value) { + *(s->value) = std::move(result); + } else { + assert(s->columns); + s->columns->SetPlainValue(std::move(result)); + } + } + } else { + // We have found a final value (a base deletion) and have newer + // merge operands that we do not intend to merge. Nothing remains + // to be done so assign status to OK. + *(s->status) = Status::OK(); + } + } else { + *(s->status) = Status::NotFound(); + } + *(s->found_final_value) = true; + return false; + } + case kTypeMerge: { + if (!merge_operator) { + *(s->status) = Status::InvalidArgument( + "merge_operator is not properly initialized."); + // Normally we continue the loop (return true) when we see a merge + // operand. But in case of an error, we should stop the loop + // immediately and pretend we have found the value to stop further + // seek. Otherwise, the later call will override this error status. + *(s->found_final_value) = true; + return false; + } + Slice v = GetLengthPrefixedSlice(key_ptr + key_length); + *(s->merge_in_progress) = true; + merge_context->PushOperand( + v, s->inplace_update_support == false /* operand_pinned */); + PERF_COUNTER_ADD(internal_merge_point_lookup_count, 1); + + if (s->do_merge && merge_operator->ShouldMerge( + merge_context->GetOperandsDirectionBackward())) { + if (s->value || s->columns) { + std::string result; + // `op_failure_scope` (an output parameter) is not provided (set to + // nullptr) since a failure must be propagated regardless of its + // value. + *(s->status) = MergeHelper::TimedFullMerge( + merge_operator, s->key->user_key(), nullptr, + merge_context->GetOperands(), &result, s->logger, s->statistics, + s->clock, /* result_operand */ nullptr, + /* update_num_ops_stats */ true, + /* op_failure_scope */ nullptr); + + if (s->status->ok()) { + if (s->value) { + *(s->value) = std::move(result); + } else { + assert(s->columns); + s->columns->SetPlainValue(std::move(result)); + } + } + } + + *(s->found_final_value) = true; + return false; + } + return true; + } + default: { + std::string msg("Corrupted value not expected."); + if (s->allow_data_in_errors) { + msg.append("Unrecognized value type: " + + std::to_string(static_cast(type)) + ". "); + msg.append("User key: " + user_key_slice.ToString(/*hex=*/true) + + ". "); + msg.append("seq: " + std::to_string(seq) + "."); + } + *(s->status) = Status::Corruption(msg.c_str()); + return false; + } + } + } + + // s->state could be Corrupt, merge or notfound + return false; +} + +bool MemTable::Get(const LookupKey& key, std::string* value, + PinnableWideColumns* columns, std::string* timestamp, + Status* s, MergeContext* merge_context, + SequenceNumber* max_covering_tombstone_seq, + SequenceNumber* seq, const ReadOptions& read_opts, + bool immutable_memtable, ReadCallback* callback, + bool* is_blob_index, bool do_merge) { + // The sequence number is updated synchronously in version_set.h + if (IsEmpty()) { + // Avoiding recording stats for speed. + return false; + } + + PERF_TIMER_GUARD(get_from_memtable_time); + + std::unique_ptr range_del_iter( + NewRangeTombstoneIterator(read_opts, + GetInternalKeySeqno(key.internal_key()), + immutable_memtable)); + if (range_del_iter != nullptr) { + SequenceNumber covering_seq = + range_del_iter->MaxCoveringTombstoneSeqnum(key.user_key()); + if (covering_seq > *max_covering_tombstone_seq) { + *max_covering_tombstone_seq = covering_seq; + if (timestamp) { + // Will be overwritten in SaveValue() if there is a point key with + // a higher seqno. + timestamp->assign(range_del_iter->timestamp().data(), + range_del_iter->timestamp().size()); + } + } + } + + bool found_final_value = false; + bool merge_in_progress = s->IsMergeInProgress(); + bool may_contain = true; + Slice user_key_without_ts = StripTimestampFromUserKey(key.user_key(), ts_sz_); + bool bloom_checked = false; + if (bloom_filter_) { + // when both memtable_whole_key_filtering and prefix_extractor_ are set, + // only do whole key filtering for Get() to save CPU + if (moptions_.memtable_whole_key_filtering) { + may_contain = bloom_filter_->MayContain(user_key_without_ts); + bloom_checked = true; + } else { + assert(prefix_extractor_); + if (prefix_extractor_->InDomain(user_key_without_ts)) { + may_contain = bloom_filter_->MayContain( + prefix_extractor_->Transform(user_key_without_ts)); + bloom_checked = true; + } + } + } + + if (bloom_filter_ && !may_contain) { + // iter is null if prefix bloom says the key does not exist + PERF_COUNTER_ADD(bloom_memtable_miss_count, 1); + *seq = kMaxSequenceNumber; + } else { + if (bloom_checked) { + PERF_COUNTER_ADD(bloom_memtable_hit_count, 1); + } + GetFromTable(key, *max_covering_tombstone_seq, do_merge, callback, + is_blob_index, value, columns, timestamp, s, merge_context, + seq, &found_final_value, &merge_in_progress); + } + + // No change to value, since we have not yet found a Put/Delete + // Propagate corruption error + if (!found_final_value && merge_in_progress && !s->IsCorruption()) { + *s = Status::MergeInProgress(); + } + PERF_COUNTER_ADD(get_from_memtable_count, 1); + return found_final_value; +} + +void MemTable::GetFromTable(const LookupKey& key, + SequenceNumber max_covering_tombstone_seq, + bool do_merge, ReadCallback* callback, + bool* is_blob_index, std::string* value, + PinnableWideColumns* columns, + std::string* timestamp, Status* s, + MergeContext* merge_context, SequenceNumber* seq, + bool* found_final_value, bool* merge_in_progress) { + Saver saver; + saver.status = s; + saver.found_final_value = found_final_value; + saver.merge_in_progress = merge_in_progress; + saver.key = &key; + saver.value = value; + saver.columns = columns; + saver.timestamp = timestamp; + saver.seq = kMaxSequenceNumber; + saver.mem = this; + saver.merge_context = merge_context; + saver.max_covering_tombstone_seq = max_covering_tombstone_seq; + saver.merge_operator = moptions_.merge_operator; + saver.logger = moptions_.info_log; + saver.inplace_update_support = moptions_.inplace_update_support; + saver.statistics = moptions_.statistics; + saver.clock = clock_; + saver.callback_ = callback; + saver.is_blob_index = is_blob_index; + saver.do_merge = do_merge; + saver.allow_data_in_errors = moptions_.allow_data_in_errors; + saver.protection_bytes_per_key = moptions_.protection_bytes_per_key; + table_->Get(key, &saver, SaveValue); + *seq = saver.seq; +} + +void MemTable::MultiGet(const ReadOptions& read_options, MultiGetRange* range, + ReadCallback* callback, bool immutable_memtable) { + // The sequence number is updated synchronously in version_set.h + if (IsEmpty()) { + // Avoiding recording stats for speed. + return; + } + PERF_TIMER_GUARD(get_from_memtable_time); + + // For now, memtable Bloom filter is effectively disabled if there are any + // range tombstones. This is the simplest way to ensure range tombstones are + // handled. TODO: allow Bloom checks where max_covering_tombstone_seq==0 + bool no_range_del = read_options.ignore_range_deletions || + is_range_del_table_empty_.load(std::memory_order_relaxed); + MultiGetRange temp_range(*range, range->begin(), range->end()); + if (bloom_filter_ && no_range_del) { + bool whole_key = + !prefix_extractor_ || moptions_.memtable_whole_key_filtering; + std::array bloom_keys; + std::array may_match; + std::array range_indexes; + int num_keys = 0; + for (auto iter = temp_range.begin(); iter != temp_range.end(); ++iter) { + if (whole_key) { + bloom_keys[num_keys] = iter->ukey_without_ts; + range_indexes[num_keys++] = iter.index(); + } else if (prefix_extractor_->InDomain(iter->ukey_without_ts)) { + bloom_keys[num_keys] = + prefix_extractor_->Transform(iter->ukey_without_ts); + range_indexes[num_keys++] = iter.index(); + } + } + bloom_filter_->MayContain(num_keys, &bloom_keys[0], &may_match[0]); + for (int i = 0; i < num_keys; ++i) { + if (!may_match[i]) { + temp_range.SkipIndex(range_indexes[i]); + PERF_COUNTER_ADD(bloom_memtable_miss_count, 1); + } else { + PERF_COUNTER_ADD(bloom_memtable_hit_count, 1); + } + } + } + for (auto iter = temp_range.begin(); iter != temp_range.end(); ++iter) { + bool found_final_value{false}; + bool merge_in_progress = iter->s->IsMergeInProgress(); + if (!no_range_del) { + std::unique_ptr range_del_iter( + NewRangeTombstoneIteratorInternal( + read_options, GetInternalKeySeqno(iter->lkey->internal_key()), + immutable_memtable)); + SequenceNumber covering_seq = + range_del_iter->MaxCoveringTombstoneSeqnum(iter->lkey->user_key()); + if (covering_seq > iter->max_covering_tombstone_seq) { + iter->max_covering_tombstone_seq = covering_seq; + if (iter->timestamp) { + // Will be overwritten in SaveValue() if there is a point key with + // a higher seqno. + iter->timestamp->assign(range_del_iter->timestamp().data(), + range_del_iter->timestamp().size()); + } + } + } + SequenceNumber dummy_seq; + GetFromTable(*(iter->lkey), iter->max_covering_tombstone_seq, true, + callback, &iter->is_blob_index, + iter->value ? iter->value->GetSelf() : nullptr, iter->columns, + iter->timestamp, iter->s, &(iter->merge_context), &dummy_seq, + &found_final_value, &merge_in_progress); + + if (!found_final_value && merge_in_progress) { + *(iter->s) = Status::MergeInProgress(); + } + + if (found_final_value) { + if (iter->value) { + iter->value->PinSelf(); + range->AddValueSize(iter->value->size()); + } else { + assert(iter->columns); + range->AddValueSize(iter->columns->serialized_size()); + } + + range->MarkKeyDone(iter); + RecordTick(moptions_.statistics, MEMTABLE_HIT); + if (range->GetValueSize() > read_options.value_size_soft_limit) { + // Set all remaining keys in range to Abort + for (auto range_iter = range->begin(); range_iter != range->end(); + ++range_iter) { + range->MarkKeyDone(range_iter); + *(range_iter->s) = Status::Aborted(); + } + break; + } + } + } + PERF_COUNTER_ADD(get_from_memtable_count, 1); +} + +Status MemTable::Update(SequenceNumber seq, ValueType value_type, + const Slice& key, const Slice& value, + const ProtectionInfoKVOS64* kv_prot_info) { + LookupKey lkey(key, seq); + Slice mem_key = lkey.memtable_key(); + + std::unique_ptr iter( + table_->GetDynamicPrefixIterator()); + iter->Seek(lkey.internal_key(), mem_key.data()); + + if (iter->Valid()) { + // Refer to comments under MemTable::Add() for entry format. + // Check that it belongs to same user key. We do not check the + // sequence number since the Seek() call above should have skipped + // all entries with overly large sequence numbers. + const char* entry = iter->key(); + uint32_t key_length = 0; + const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length); + if (comparator_.comparator.user_comparator()->Equal( + Slice(key_ptr, key_length - 8), lkey.user_key())) { + // Correct user key + const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8); + ValueType type; + SequenceNumber existing_seq; + UnPackSequenceAndType(tag, &existing_seq, &type); + assert(existing_seq != seq); + if (type == value_type) { + Slice prev_value = GetLengthPrefixedSlice(key_ptr + key_length); + uint32_t prev_size = static_cast(prev_value.size()); + uint32_t new_size = static_cast(value.size()); + + // Update value, if new value size <= previous value size + if (new_size <= prev_size) { + char* p = + EncodeVarint32(const_cast(key_ptr) + key_length, new_size); + WriteLock wl(GetLock(lkey.user_key())); + memcpy(p, value.data(), value.size()); + assert((unsigned)((p + value.size()) - entry) == + (unsigned)(VarintLength(key_length) + key_length + + VarintLength(value.size()) + value.size())); + RecordTick(moptions_.statistics, NUMBER_KEYS_UPDATED); + if (kv_prot_info != nullptr) { + ProtectionInfoKVOS64 updated_kv_prot_info(*kv_prot_info); + // `seq` is swallowed and `existing_seq` prevails. + updated_kv_prot_info.UpdateS(seq, existing_seq); + UpdateEntryChecksum(&updated_kv_prot_info, key, value, type, + existing_seq, p + value.size()); + Slice encoded(entry, p + value.size() - entry); + return VerifyEncodedEntry(encoded, updated_kv_prot_info); + } else { + UpdateEntryChecksum(nullptr, key, value, type, existing_seq, + p + value.size()); + } + return Status::OK(); + } + } + } + } + + // The latest value is not value_type or key doesn't exist + return Add(seq, value_type, key, value, kv_prot_info); +} + +Status MemTable::UpdateCallback(SequenceNumber seq, const Slice& key, + const Slice& delta, + const ProtectionInfoKVOS64* kv_prot_info) { + LookupKey lkey(key, seq); + Slice memkey = lkey.memtable_key(); + + std::unique_ptr iter( + table_->GetDynamicPrefixIterator()); + iter->Seek(lkey.internal_key(), memkey.data()); + + if (iter->Valid()) { + // Refer to comments under MemTable::Add() for entry format. + // Check that it belongs to same user key. We do not check the + // sequence number since the Seek() call above should have skipped + // all entries with overly large sequence numbers. + const char* entry = iter->key(); + uint32_t key_length = 0; + const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length); + if (comparator_.comparator.user_comparator()->Equal( + Slice(key_ptr, key_length - 8), lkey.user_key())) { + // Correct user key + const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8); + ValueType type; + uint64_t existing_seq; + UnPackSequenceAndType(tag, &existing_seq, &type); + if (type == kTypeValue) { + Slice prev_value = GetLengthPrefixedSlice(key_ptr + key_length); + uint32_t prev_size = static_cast(prev_value.size()); + + char* prev_buffer = const_cast(prev_value.data()); + uint32_t new_prev_size = prev_size; + + std::string str_value; + WriteLock wl(GetLock(lkey.user_key())); + auto status = moptions_.inplace_callback(prev_buffer, &new_prev_size, + delta, &str_value); + if (status == UpdateStatus::UPDATED_INPLACE) { + // Value already updated by callback. + assert(new_prev_size <= prev_size); + if (new_prev_size < prev_size) { + // overwrite the new prev_size + char* p = EncodeVarint32(const_cast(key_ptr) + key_length, + new_prev_size); + if (VarintLength(new_prev_size) < VarintLength(prev_size)) { + // shift the value buffer as well. + memcpy(p, prev_buffer, new_prev_size); + prev_buffer = p; + } + } + RecordTick(moptions_.statistics, NUMBER_KEYS_UPDATED); + UpdateFlushState(); + Slice new_value(prev_buffer, new_prev_size); + if (kv_prot_info != nullptr) { + ProtectionInfoKVOS64 updated_kv_prot_info(*kv_prot_info); + // `seq` is swallowed and `existing_seq` prevails. + updated_kv_prot_info.UpdateS(seq, existing_seq); + updated_kv_prot_info.UpdateV(delta, new_value); + Slice encoded(entry, prev_buffer + new_prev_size - entry); + UpdateEntryChecksum(&updated_kv_prot_info, key, new_value, type, + existing_seq, prev_buffer + new_prev_size); + return VerifyEncodedEntry(encoded, updated_kv_prot_info); + } else { + UpdateEntryChecksum(nullptr, key, new_value, type, existing_seq, + prev_buffer + new_prev_size); + } + return Status::OK(); + } else if (status == UpdateStatus::UPDATED) { + Status s; + if (kv_prot_info != nullptr) { + ProtectionInfoKVOS64 updated_kv_prot_info(*kv_prot_info); + updated_kv_prot_info.UpdateV(delta, str_value); + s = Add(seq, kTypeValue, key, Slice(str_value), + &updated_kv_prot_info); + } else { + s = Add(seq, kTypeValue, key, Slice(str_value), + nullptr /* kv_prot_info */); + } + RecordTick(moptions_.statistics, NUMBER_KEYS_WRITTEN); + UpdateFlushState(); + return s; + } else if (status == UpdateStatus::UPDATE_FAILED) { + // `UPDATE_FAILED` is named incorrectly. It indicates no update + // happened. It does not indicate a failure happened. + UpdateFlushState(); + return Status::OK(); + } + } + } + } + // The latest value is not `kTypeValue` or key doesn't exist + return Status::NotFound(); +} + +size_t MemTable::CountSuccessiveMergeEntries(const LookupKey& key) { + Slice memkey = key.memtable_key(); + + // A total ordered iterator is costly for some memtablerep (prefix aware + // reps). By passing in the user key, we allow efficient iterator creation. + // The iterator only needs to be ordered within the same user key. + std::unique_ptr iter( + table_->GetDynamicPrefixIterator()); + iter->Seek(key.internal_key(), memkey.data()); + + size_t num_successive_merges = 0; + + for (; iter->Valid(); iter->Next()) { + const char* entry = iter->key(); + uint32_t key_length = 0; + const char* iter_key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length); + if (!comparator_.comparator.user_comparator()->Equal( + Slice(iter_key_ptr, key_length - 8), key.user_key())) { + break; + } + + const uint64_t tag = DecodeFixed64(iter_key_ptr + key_length - 8); + ValueType type; + uint64_t unused; + UnPackSequenceAndType(tag, &unused, &type); + if (type != kTypeMerge) { + break; + } + + ++num_successive_merges; + } + + return num_successive_merges; +} + +void MemTableRep::Get(const LookupKey& k, void* callback_args, + bool (*callback_func)(void* arg, const char* entry)) { + auto iter = GetDynamicPrefixIterator(); + for (iter->Seek(k.internal_key(), k.memtable_key().data()); + iter->Valid() && callback_func(callback_args, iter->key()); + iter->Next()) { + } +} + +void MemTable::RefLogContainingPrepSection(uint64_t log) { + assert(log > 0); + auto cur = min_prep_log_referenced_.load(); + while ((log < cur || cur == 0) && + !min_prep_log_referenced_.compare_exchange_strong(cur, log)) { + cur = min_prep_log_referenced_.load(); + } +} + +uint64_t MemTable::GetMinLogContainingPrepSection() { + return min_prep_log_referenced_.load(); +} + +void MemTable::MaybeUpdateNewestUDT(const Slice& user_key) { + if (ts_sz_ == 0 || persist_user_defined_timestamps_) { + return; + } + const Comparator* ucmp = GetInternalKeyComparator().user_comparator(); + Slice udt = ExtractTimestampFromUserKey(user_key, ts_sz_); + if (newest_udt_.empty() || ucmp->CompareTimestamp(udt, newest_udt_) > 0) { + newest_udt_ = udt; + } +} + +const Slice& MemTable::GetNewestUDT() const { + // This path should not be invoked for MemTables that does not enable the UDT + // in Memtable only feature. + assert(ts_sz_ > 0 && !persist_user_defined_timestamps_); + return newest_udt_; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/memtable.h b/librocksdb-sys/rocksdb/db/memtable.h new file mode 100644 index 0000000..c55b347 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/memtable.h @@ -0,0 +1,704 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include +#include +#include +#include +#include +#include +#include + +#include "db/dbformat.h" +#include "db/kv_checksum.h" +#include "db/range_tombstone_fragmenter.h" +#include "db/read_callback.h" +#include "db/version_edit.h" +#include "memory/allocator.h" +#include "memory/concurrent_arena.h" +#include "monitoring/instrumented_mutex.h" +#include "options/cf_options.h" +#include "rocksdb/db.h" +#include "rocksdb/memtablerep.h" +#include "table/multiget_context.h" +#include "util/dynamic_bloom.h" +#include "util/hash.h" +#include "util/hash_containers.h" + +namespace ROCKSDB_NAMESPACE { + +struct FlushJobInfo; +class Mutex; +class MemTableIterator; +class MergeContext; +class SystemClock; + +struct ImmutableMemTableOptions { + explicit ImmutableMemTableOptions(const ImmutableOptions& ioptions, + const MutableCFOptions& mutable_cf_options); + size_t arena_block_size; + uint32_t memtable_prefix_bloom_bits; + size_t memtable_huge_page_size; + bool memtable_whole_key_filtering; + bool inplace_update_support; + size_t inplace_update_num_locks; + UpdateStatus (*inplace_callback)(char* existing_value, + uint32_t* existing_value_size, + Slice delta_value, + std::string* merged_value); + size_t max_successive_merges; + Statistics* statistics; + MergeOperator* merge_operator; + Logger* info_log; + bool allow_data_in_errors; + uint32_t protection_bytes_per_key; +}; + +// Batched counters to updated when inserting keys in one write batch. +// In post process of the write batch, these can be updated together. +// Only used in concurrent memtable insert case. +struct MemTablePostProcessInfo { + uint64_t data_size = 0; + uint64_t num_entries = 0; + uint64_t num_deletes = 0; + uint64_t num_range_deletes = 0; +}; + +using MultiGetRange = MultiGetContext::Range; +// Note: Many of the methods in this class have comments indicating that +// external synchronization is required as these methods are not thread-safe. +// It is up to higher layers of code to decide how to prevent concurrent +// invocation of these methods. This is usually done by acquiring either +// the db mutex or the single writer thread. +// +// Some of these methods are documented to only require external +// synchronization if this memtable is immutable. Calling MarkImmutable() is +// not sufficient to guarantee immutability. It is up to higher layers of +// code to determine if this MemTable can still be modified by other threads. +// Eg: The Superversion stores a pointer to the current MemTable (that can +// be modified) and a separate list of the MemTables that can no longer be +// written to (aka the 'immutable memtables'). +class MemTable { + public: + struct KeyComparator : public MemTableRep::KeyComparator { + const InternalKeyComparator comparator; + explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) {} + virtual int operator()(const char* prefix_len_key1, + const char* prefix_len_key2) const override; + virtual int operator()(const char* prefix_len_key, + const DecodedType& key) const override; + }; + + // MemTables are reference counted. The initial reference count + // is zero and the caller must call Ref() at least once. + // + // earliest_seq should be the current SequenceNumber in the db such that any + // key inserted into this memtable will have an equal or larger seq number. + // (When a db is first created, the earliest sequence number will be 0). + // If the earliest sequence number is not known, kMaxSequenceNumber may be + // used, but this may prevent some transactions from succeeding until the + // first key is inserted into the memtable. + explicit MemTable(const InternalKeyComparator& comparator, + const ImmutableOptions& ioptions, + const MutableCFOptions& mutable_cf_options, + WriteBufferManager* write_buffer_manager, + SequenceNumber earliest_seq, uint32_t column_family_id); + // No copying allowed + MemTable(const MemTable&) = delete; + MemTable& operator=(const MemTable&) = delete; + + // Do not delete this MemTable unless Unref() indicates it not in use. + ~MemTable(); + + // Increase reference count. + // REQUIRES: external synchronization to prevent simultaneous + // operations on the same MemTable. + void Ref() { ++refs_; } + + // Drop reference count. + // If the refcount goes to zero return this memtable, otherwise return null. + // REQUIRES: external synchronization to prevent simultaneous + // operations on the same MemTable. + MemTable* Unref() { + --refs_; + assert(refs_ >= 0); + if (refs_ <= 0) { + return this; + } + return nullptr; + } + + // Returns an estimate of the number of bytes of data in use by this + // data structure. + // + // REQUIRES: external synchronization to prevent simultaneous + // operations on the same MemTable (unless this Memtable is immutable). + size_t ApproximateMemoryUsage(); + + // As a cheap version of `ApproximateMemoryUsage()`, this function doesn't + // require external synchronization. The value may be less accurate though + size_t ApproximateMemoryUsageFast() const { + return approximate_memory_usage_.load(std::memory_order_relaxed); + } + + // used by MemTableListVersion::MemoryAllocatedBytesExcludingLast + size_t MemoryAllocatedBytes() const { + return table_->ApproximateMemoryUsage() + + range_del_table_->ApproximateMemoryUsage() + + arena_.MemoryAllocatedBytes(); + } + + // Returns a vector of unique random memtable entries of size 'sample_size'. + // + // Note: the entries are stored in the unordered_set as length-prefixed keys, + // hence their representation in the set as "const char*". + // Note2: the size of the output set 'entries' is not enforced to be strictly + // equal to 'target_sample_size'. Its final size might be slightly + // greater or slightly less than 'target_sample_size' + // + // REQUIRES: external synchronization to prevent simultaneous + // operations on the same MemTable (unless this Memtable is immutable). + // REQUIRES: SkipList memtable representation. This function is not + // implemented for any other type of memtable representation (vectorrep, + // hashskiplist,...). + void UniqueRandomSample(const uint64_t& target_sample_size, + std::unordered_set* entries) { + // TODO(bjlemaire): at the moment, only supported by skiplistrep. + // Extend it to all other memtable representations. + table_->UniqueRandomSample(num_entries(), target_sample_size, entries); + } + + // This method heuristically determines if the memtable should continue to + // host more data. + bool ShouldScheduleFlush() const { + return flush_state_.load(std::memory_order_relaxed) == FLUSH_REQUESTED; + } + + // Returns true if a flush should be scheduled and the caller should + // be the one to schedule it + bool MarkFlushScheduled() { + auto before = FLUSH_REQUESTED; + return flush_state_.compare_exchange_strong(before, FLUSH_SCHEDULED, + std::memory_order_relaxed, + std::memory_order_relaxed); + } + + // Return an iterator that yields the contents of the memtable. + // + // The caller must ensure that the underlying MemTable remains live + // while the returned iterator is live. The keys returned by this + // iterator are internal keys encoded by AppendInternalKey in the + // db/dbformat.{h,cc} module. + // + // By default, it returns an iterator for prefix seek if prefix_extractor + // is configured in Options. + // arena: If not null, the arena needs to be used to allocate the Iterator. + // Calling ~Iterator of the iterator will destroy all the states but + // those allocated in arena. + InternalIterator* NewIterator(const ReadOptions& read_options, Arena* arena); + + // Returns an iterator that yields the range tombstones of the memtable. + // The caller must ensure that the underlying MemTable remains live + // while the returned iterator is live. + // @param immutable_memtable Whether this memtable is an immutable memtable. + // This information is not stored in memtable itself, so it needs to be + // specified by the caller. This flag is used internally to decide whether a + // cached fragmented range tombstone list can be returned. This cached version + // is constructed when a memtable becomes immutable. Setting the flag to false + // will always yield correct result, but may incur performance penalty as it + // always creates a new fragmented range tombstone list. + FragmentedRangeTombstoneIterator* NewRangeTombstoneIterator( + const ReadOptions& read_options, SequenceNumber read_seq, + bool immutable_memtable); + + Status VerifyEncodedEntry(Slice encoded, + const ProtectionInfoKVOS64& kv_prot_info); + + // Add an entry into memtable that maps key to value at the + // specified sequence number and with the specified type. + // Typically value will be empty if type==kTypeDeletion. + // + // REQUIRES: if allow_concurrent = false, external synchronization to prevent + // simultaneous operations on the same MemTable. + // + // Returns `Status::TryAgain` if the `seq`, `key` combination already exists + // in the memtable and `MemTableRepFactory::CanHandleDuplicatedKey()` is true. + // The next attempt should try a larger value for `seq`. + Status Add(SequenceNumber seq, ValueType type, const Slice& key, + const Slice& value, const ProtectionInfoKVOS64* kv_prot_info, + bool allow_concurrent = false, + MemTablePostProcessInfo* post_process_info = nullptr, + void** hint = nullptr); + + // Used to Get value associated with key or Get Merge Operands associated + // with key. + // If do_merge = true the default behavior which is Get value for key is + // executed. Expected behavior is described right below. + // If memtable contains a value for key, store it in *value and return true. + // If memtable contains a deletion for key, store a NotFound() error + // in *status and return true. + // If memtable contains Merge operation as the most recent entry for a key, + // and the merge process does not stop (not reaching a value or delete), + // prepend the current merge operand to *operands. + // store MergeInProgress in s, and return false. + // Else, return false. + // If any operation was found, its most recent sequence number + // will be stored in *seq on success (regardless of whether true/false is + // returned). Otherwise, *seq will be set to kMaxSequenceNumber. + // On success, *s may be set to OK, NotFound, or MergeInProgress. Any other + // status returned indicates a corruption or other unexpected error. + // If do_merge = false then any Merge Operands encountered for key are simply + // stored in merge_context.operands_list and never actually merged to get a + // final value. The raw Merge Operands are eventually returned to the user. + // @param immutable_memtable Whether this memtable is immutable. Used + // internally by NewRangeTombstoneIterator(). See comment above + // NewRangeTombstoneIterator() for more detail. + bool Get(const LookupKey& key, std::string* value, + PinnableWideColumns* columns, std::string* timestamp, Status* s, + MergeContext* merge_context, + SequenceNumber* max_covering_tombstone_seq, SequenceNumber* seq, + const ReadOptions& read_opts, bool immutable_memtable, + ReadCallback* callback = nullptr, bool* is_blob_index = nullptr, + bool do_merge = true); + + bool Get(const LookupKey& key, std::string* value, + PinnableWideColumns* columns, std::string* timestamp, Status* s, + MergeContext* merge_context, + SequenceNumber* max_covering_tombstone_seq, + const ReadOptions& read_opts, bool immutable_memtable, + ReadCallback* callback = nullptr, bool* is_blob_index = nullptr, + bool do_merge = true) { + SequenceNumber seq; + return Get(key, value, columns, timestamp, s, merge_context, + max_covering_tombstone_seq, &seq, read_opts, immutable_memtable, + callback, is_blob_index, do_merge); + } + + // @param immutable_memtable Whether this memtable is immutable. Used + // internally by NewRangeTombstoneIterator(). See comment above + // NewRangeTombstoneIterator() for more detail. + void MultiGet(const ReadOptions& read_options, MultiGetRange* range, + ReadCallback* callback, bool immutable_memtable); + + // If `key` exists in current memtable with type value_type and the existing + // value is at least as large as the new value, updates it in-place. Otherwise + // adds the new value to the memtable out-of-place. + // + // Returns `Status::TryAgain` if the `seq`, `key` combination already exists + // in the memtable and `MemTableRepFactory::CanHandleDuplicatedKey()` is true. + // The next attempt should try a larger value for `seq`. + // + // REQUIRES: external synchronization to prevent simultaneous + // operations on the same MemTable. + Status Update(SequenceNumber seq, ValueType value_type, const Slice& key, + const Slice& value, const ProtectionInfoKVOS64* kv_prot_info); + + // If `key` exists in current memtable with type `kTypeValue` and the existing + // value is at least as large as the new value, updates it in-place. Otherwise + // if `key` exists in current memtable with type `kTypeValue`, adds the new + // value to the memtable out-of-place. + // + // Returns `Status::NotFound` if `key` does not exist in current memtable or + // the latest version of `key` does not have `kTypeValue`. + // + // Returns `Status::TryAgain` if the `seq`, `key` combination already exists + // in the memtable and `MemTableRepFactory::CanHandleDuplicatedKey()` is true. + // The next attempt should try a larger value for `seq`. + // + // REQUIRES: external synchronization to prevent simultaneous + // operations on the same MemTable. + Status UpdateCallback(SequenceNumber seq, const Slice& key, + const Slice& delta, + const ProtectionInfoKVOS64* kv_prot_info); + + // Returns the number of successive merge entries starting from the newest + // entry for the key up to the last non-merge entry or last entry for the + // key in the memtable. + size_t CountSuccessiveMergeEntries(const LookupKey& key); + + // Update counters and flush status after inserting a whole write batch + // Used in concurrent memtable inserts. + void BatchPostProcess(const MemTablePostProcessInfo& update_counters) { + num_entries_.fetch_add(update_counters.num_entries, + std::memory_order_relaxed); + data_size_.fetch_add(update_counters.data_size, std::memory_order_relaxed); + if (update_counters.num_deletes != 0) { + num_deletes_.fetch_add(update_counters.num_deletes, + std::memory_order_relaxed); + } + if (update_counters.num_range_deletes > 0) { + num_range_deletes_.fetch_add(update_counters.num_range_deletes, + std::memory_order_relaxed); + } + UpdateFlushState(); + } + + // Get total number of entries in the mem table. + // REQUIRES: external synchronization to prevent simultaneous + // operations on the same MemTable (unless this Memtable is immutable). + uint64_t num_entries() const { + return num_entries_.load(std::memory_order_relaxed); + } + + // Get total number of deletes in the mem table. + // REQUIRES: external synchronization to prevent simultaneous + // operations on the same MemTable (unless this Memtable is immutable). + uint64_t num_deletes() const { + return num_deletes_.load(std::memory_order_relaxed); + } + + // Get total number of range deletions in the mem table. + // REQUIRES: external synchronization to prevent simultaneous + // operations on the same MemTable (unless this Memtable is immutable). + uint64_t num_range_deletes() const { + return num_range_deletes_.load(std::memory_order_relaxed); + } + + uint64_t get_data_size() const { + return data_size_.load(std::memory_order_relaxed); + } + + size_t write_buffer_size() const { + return write_buffer_size_.load(std::memory_order_relaxed); + } + + // Dynamically change the memtable's capacity. If set below the current usage, + // the next key added will trigger a flush. Can only increase size when + // memtable prefix bloom is disabled, since we can't easily allocate more + // space. + void UpdateWriteBufferSize(size_t new_write_buffer_size) { + if (bloom_filter_ == nullptr || + new_write_buffer_size < write_buffer_size_) { + write_buffer_size_.store(new_write_buffer_size, + std::memory_order_relaxed); + } + } + + // Returns the edits area that is needed for flushing the memtable + VersionEdit* GetEdits() { return &edit_; } + + // Returns if there is no entry inserted to the mem table. + // REQUIRES: external synchronization to prevent simultaneous + // operations on the same MemTable (unless this Memtable is immutable). + bool IsEmpty() const { return first_seqno_ == 0; } + + // Returns the sequence number of the first element that was inserted + // into the memtable. + // REQUIRES: external synchronization to prevent simultaneous + // operations on the same MemTable (unless this Memtable is immutable). + SequenceNumber GetFirstSequenceNumber() { + return first_seqno_.load(std::memory_order_relaxed); + } + + // Returns the sequence number of the first element that was inserted + // into the memtable. + // REQUIRES: external synchronization to prevent simultaneous + // operations on the same MemTable (unless this Memtable is immutable). + void SetFirstSequenceNumber(SequenceNumber first_seqno) { + return first_seqno_.store(first_seqno, std::memory_order_relaxed); + } + + // Returns the sequence number that is guaranteed to be smaller than or equal + // to the sequence number of any key that could be inserted into this + // memtable. It can then be assumed that any write with a larger(or equal) + // sequence number will be present in this memtable or a later memtable. + // + // If the earliest sequence number could not be determined, + // kMaxSequenceNumber will be returned. + SequenceNumber GetEarliestSequenceNumber() { + return earliest_seqno_.load(std::memory_order_relaxed); + } + + // Sets the sequence number that is guaranteed to be smaller than or equal + // to the sequence number of any key that could be inserted into this + // memtable. It can then be assumed that any write with a larger(or equal) + // sequence number will be present in this memtable or a later memtable. + // Used only for MemPurge operation + void SetEarliestSequenceNumber(SequenceNumber earliest_seqno) { + return earliest_seqno_.store(earliest_seqno, std::memory_order_relaxed); + } + + // DB's latest sequence ID when the memtable is created. This number + // may be updated to a more recent one before any key is inserted. + SequenceNumber GetCreationSeq() const { return creation_seq_; } + + void SetCreationSeq(SequenceNumber sn) { creation_seq_ = sn; } + + // Returns the next active logfile number when this memtable is about to + // be flushed to storage + // REQUIRES: external synchronization to prevent simultaneous + // operations on the same MemTable. + uint64_t GetNextLogNumber() { return mem_next_logfile_number_; } + + // Sets the next active logfile number when this memtable is about to + // be flushed to storage + // REQUIRES: external synchronization to prevent simultaneous + // operations on the same MemTable. + void SetNextLogNumber(uint64_t num) { mem_next_logfile_number_ = num; } + + // if this memtable contains data from a committed + // two phase transaction we must take note of the + // log which contains that data so we can know + // when to relese that log + void RefLogContainingPrepSection(uint64_t log); + uint64_t GetMinLogContainingPrepSection(); + + // Notify the underlying storage that no more items will be added. + // REQUIRES: external synchronization to prevent simultaneous + // operations on the same MemTable. + // After MarkImmutable() is called, you should not attempt to + // write anything to this MemTable(). (Ie. do not call Add() or Update()). + void MarkImmutable() { + table_->MarkReadOnly(); + mem_tracker_.DoneAllocating(); + } + + // Notify the underlying storage that all data it contained has been + // persisted. + // REQUIRES: external synchronization to prevent simultaneous + // operations on the same MemTable. + void MarkFlushed() { table_->MarkFlushed(); } + + // return true if the current MemTableRep supports merge operator. + bool IsMergeOperatorSupported() const { + return table_->IsMergeOperatorSupported(); + } + + // return true if the current MemTableRep supports snapshots. + // inplace update prevents snapshots, + bool IsSnapshotSupported() const { + return table_->IsSnapshotSupported() && !moptions_.inplace_update_support; + } + + struct MemTableStats { + uint64_t size; + uint64_t count; + }; + + MemTableStats ApproximateStats(const Slice& start_ikey, + const Slice& end_ikey); + + // Get the lock associated for the key + port::RWMutex* GetLock(const Slice& key); + + const InternalKeyComparator& GetInternalKeyComparator() const { + return comparator_.comparator; + } + + const ImmutableMemTableOptions* GetImmutableMemTableOptions() const { + return &moptions_; + } + + uint64_t ApproximateOldestKeyTime() const { + return oldest_key_time_.load(std::memory_order_relaxed); + } + + // REQUIRES: db_mutex held. + void SetID(uint64_t id) { id_ = id; } + + uint64_t GetID() const { return id_; } + + void SetFlushCompleted(bool completed) { flush_completed_ = completed; } + + uint64_t GetFileNumber() const { return file_number_; } + + void SetFileNumber(uint64_t file_num) { file_number_ = file_num; } + + void SetFlushInProgress(bool in_progress) { + flush_in_progress_ = in_progress; + } + + void SetFlushJobInfo(std::unique_ptr&& info) { + flush_job_info_ = std::move(info); + } + + std::unique_ptr ReleaseFlushJobInfo() { + return std::move(flush_job_info_); + } + + // Returns a heuristic flush decision + bool ShouldFlushNow(); + + void ConstructFragmentedRangeTombstones(); + + // Returns whether a fragmented range tombstone list is already constructed + // for this memtable. It should be constructed right before a memtable is + // added to an immutable memtable list. Note that if a memtable does not have + // any range tombstone, then no range tombstone list will ever be constructed. + // @param allow_empty Specifies whether a memtable with no range tombstone is + // considered to have its fragmented range tombstone list constructed. + bool IsFragmentedRangeTombstonesConstructed(bool allow_empty = true) const { + if (allow_empty) { + return fragmented_range_tombstone_list_.get() != nullptr || + is_range_del_table_empty_; + } else { + return fragmented_range_tombstone_list_.get() != nullptr; + } + } + + // Get the newest user-defined timestamp contained in this MemTable. Check + // `newest_udt_` for what newer means. This method should only be invoked for + // an MemTable that has enabled user-defined timestamp feature and set + // `persist_user_defined_timestamps` to false. The tracked newest UDT will be + // used by flush job in the background to help check the MemTable's + // eligibility for Flush. + const Slice& GetNewestUDT() const; + + // Returns Corruption status if verification fails. + static Status VerifyEntryChecksum(const char* entry, + uint32_t protection_bytes_per_key, + bool allow_data_in_errors = false); + + private: + enum FlushStateEnum { FLUSH_NOT_REQUESTED, FLUSH_REQUESTED, FLUSH_SCHEDULED }; + + friend class MemTableIterator; + friend class MemTableBackwardIterator; + friend class MemTableList; + + KeyComparator comparator_; + const ImmutableMemTableOptions moptions_; + int refs_; + const size_t kArenaBlockSize; + AllocTracker mem_tracker_; + ConcurrentArena arena_; + std::unique_ptr table_; + std::unique_ptr range_del_table_; + std::atomic_bool is_range_del_table_empty_; + + // Total data size of all data inserted + std::atomic data_size_; + std::atomic num_entries_; + std::atomic num_deletes_; + std::atomic num_range_deletes_; + + // Dynamically changeable memtable option + std::atomic write_buffer_size_; + + // These are used to manage memtable flushes to storage + bool flush_in_progress_; // started the flush + bool flush_completed_; // finished the flush + uint64_t file_number_; // filled up after flush is complete + + // The updates to be applied to the transaction log when this + // memtable is flushed to storage. + VersionEdit edit_; + + // The sequence number of the kv that was inserted first + std::atomic first_seqno_; + + // The db sequence number at the time of creation or kMaxSequenceNumber + // if not set. + std::atomic earliest_seqno_; + + SequenceNumber creation_seq_; + + // The log files earlier than this number can be deleted. + uint64_t mem_next_logfile_number_; + + // the earliest log containing a prepared section + // which has been inserted into this memtable. + std::atomic min_prep_log_referenced_; + + // rw locks for inplace updates + std::vector locks_; + + const SliceTransform* const prefix_extractor_; + std::unique_ptr bloom_filter_; + + std::atomic flush_state_; + + SystemClock* clock_; + + // Extract sequential insert prefixes. + const SliceTransform* insert_with_hint_prefix_extractor_; + + // Insert hints for each prefix. + UnorderedMapH insert_hints_; + + // Timestamp of oldest key + std::atomic oldest_key_time_; + + // Memtable id to track flush. + uint64_t id_ = 0; + + // Sequence number of the atomic flush that is responsible for this memtable. + // The sequence number of atomic flush is a seq, such that no writes with + // sequence numbers greater than or equal to seq are flushed, while all + // writes with sequence number smaller than seq are flushed. + SequenceNumber atomic_flush_seqno_; + + // keep track of memory usage in table_, arena_, and range_del_table_. + // Gets refreshed inside `ApproximateMemoryUsage()` or `ShouldFlushNow` + std::atomic approximate_memory_usage_; + + // max range deletions in a memtable, before automatic flushing, 0 for + // unlimited. + uint32_t memtable_max_range_deletions_ = 0; + + // Flush job info of the current memtable. + std::unique_ptr flush_job_info_; + + // Size in bytes for the user-defined timestamps. + size_t ts_sz_; + + // Whether to persist user-defined timestamps + bool persist_user_defined_timestamps_; + + // Newest user-defined timestamp contained in this MemTable. For ts1, and ts2 + // if Comparator::CompareTimestamp(ts1, ts2) > 0, ts1 is considered newer than + // ts2. We track this field for a MemTable if its column family has UDT + // feature enabled and the `persist_user_defined_timestamp` flag is false. + // Otherwise, this field just contains an empty Slice. + Slice newest_udt_; + + // Updates flush_state_ using ShouldFlushNow() + void UpdateFlushState(); + + void UpdateOldestKeyTime(); + + void GetFromTable(const LookupKey& key, + SequenceNumber max_covering_tombstone_seq, bool do_merge, + ReadCallback* callback, bool* is_blob_index, + std::string* value, PinnableWideColumns* columns, + std::string* timestamp, Status* s, + MergeContext* merge_context, SequenceNumber* seq, + bool* found_final_value, bool* merge_in_progress); + + // Always returns non-null and assumes certain pre-checks (e.g., + // is_range_del_table_empty_) are done. This is only valid during the lifetime + // of the underlying memtable. + // read_seq and read_options.timestamp will be used as the upper bound + // for range tombstones. + FragmentedRangeTombstoneIterator* NewRangeTombstoneIteratorInternal( + const ReadOptions& read_options, SequenceNumber read_seq, + bool immutable_memtable); + + // The fragmented range tombstones of this memtable. + // This is constructed when this memtable becomes immutable + // if !is_range_del_table_empty_. + std::unique_ptr + fragmented_range_tombstone_list_; + + // makes sure there is a single range tombstone writer to invalidate cache + std::mutex range_del_mutex_; + CoreLocalArray> + cached_range_tombstone_; + + void UpdateEntryChecksum(const ProtectionInfoKVOS64* kv_prot_info, + const Slice& key, const Slice& value, ValueType type, + SequenceNumber s, char* checksum_ptr); + + void MaybeUpdateNewestUDT(const Slice& user_key); +}; + +extern const char* EncodeKey(std::string* scratch, const Slice& target); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/memtable_list.cc b/librocksdb-sys/rocksdb/db/memtable_list.cc new file mode 100644 index 0000000..ee1563f --- /dev/null +++ b/librocksdb-sys/rocksdb/db/memtable_list.cc @@ -0,0 +1,987 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#include "db/memtable_list.h" + +#include +#include +#include +#include +#include + +#include "db/db_impl/db_impl.h" +#include "db/memtable.h" +#include "db/range_tombstone_fragmenter.h" +#include "db/version_set.h" +#include "logging/log_buffer.h" +#include "logging/logging.h" +#include "monitoring/thread_status_util.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/iterator.h" +#include "table/merging_iterator.h" +#include "test_util/sync_point.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { + +class InternalKeyComparator; +class Mutex; +class VersionSet; + +void MemTableListVersion::AddMemTable(MemTable* m) { + memlist_.push_front(m); + *parent_memtable_list_memory_usage_ += m->ApproximateMemoryUsage(); +} + +void MemTableListVersion::UnrefMemTable(autovector* to_delete, + MemTable* m) { + if (m->Unref()) { + to_delete->push_back(m); + assert(*parent_memtable_list_memory_usage_ >= m->ApproximateMemoryUsage()); + *parent_memtable_list_memory_usage_ -= m->ApproximateMemoryUsage(); + } +} + +MemTableListVersion::MemTableListVersion( + size_t* parent_memtable_list_memory_usage, const MemTableListVersion& old) + : max_write_buffer_number_to_maintain_( + old.max_write_buffer_number_to_maintain_), + max_write_buffer_size_to_maintain_( + old.max_write_buffer_size_to_maintain_), + parent_memtable_list_memory_usage_(parent_memtable_list_memory_usage) { + memlist_ = old.memlist_; + for (auto& m : memlist_) { + m->Ref(); + } + + memlist_history_ = old.memlist_history_; + for (auto& m : memlist_history_) { + m->Ref(); + } +} + +MemTableListVersion::MemTableListVersion( + size_t* parent_memtable_list_memory_usage, + int max_write_buffer_number_to_maintain, + int64_t max_write_buffer_size_to_maintain) + : max_write_buffer_number_to_maintain_(max_write_buffer_number_to_maintain), + max_write_buffer_size_to_maintain_(max_write_buffer_size_to_maintain), + parent_memtable_list_memory_usage_(parent_memtable_list_memory_usage) {} + +void MemTableListVersion::Ref() { ++refs_; } + +// called by superversion::clean() +void MemTableListVersion::Unref(autovector* to_delete) { + assert(refs_ >= 1); + --refs_; + if (refs_ == 0) { + // if to_delete is equal to nullptr it means we're confident + // that refs_ will not be zero + assert(to_delete != nullptr); + for (const auto& m : memlist_) { + UnrefMemTable(to_delete, m); + } + for (const auto& m : memlist_history_) { + UnrefMemTable(to_delete, m); + } + delete this; + } +} + +int MemTableList::NumNotFlushed() const { + int size = static_cast(current_->memlist_.size()); + assert(num_flush_not_started_ <= size); + return size; +} + +int MemTableList::NumFlushed() const { + return static_cast(current_->memlist_history_.size()); +} + +// Search all the memtables starting from the most recent one. +// Return the most recent value found, if any. +// Operands stores the list of merge operations to apply, so far. +bool MemTableListVersion::Get(const LookupKey& key, std::string* value, + PinnableWideColumns* columns, + std::string* timestamp, Status* s, + MergeContext* merge_context, + SequenceNumber* max_covering_tombstone_seq, + SequenceNumber* seq, const ReadOptions& read_opts, + ReadCallback* callback, bool* is_blob_index) { + return GetFromList(&memlist_, key, value, columns, timestamp, s, + merge_context, max_covering_tombstone_seq, seq, read_opts, + callback, is_blob_index); +} + +void MemTableListVersion::MultiGet(const ReadOptions& read_options, + MultiGetRange* range, + ReadCallback* callback) { + for (auto memtable : memlist_) { + memtable->MultiGet(read_options, range, callback, + true /* immutable_memtable */); + if (range->empty()) { + return; + } + } +} + +bool MemTableListVersion::GetMergeOperands( + const LookupKey& key, Status* s, MergeContext* merge_context, + SequenceNumber* max_covering_tombstone_seq, const ReadOptions& read_opts) { + for (MemTable* memtable : memlist_) { + bool done = memtable->Get( + key, /*value=*/nullptr, /*columns=*/nullptr, /*timestamp=*/nullptr, s, + merge_context, max_covering_tombstone_seq, read_opts, + true /* immutable_memtable */, nullptr, nullptr, false); + if (done) { + return true; + } + } + return false; +} + +bool MemTableListVersion::GetFromHistory( + const LookupKey& key, std::string* value, PinnableWideColumns* columns, + std::string* timestamp, Status* s, MergeContext* merge_context, + SequenceNumber* max_covering_tombstone_seq, SequenceNumber* seq, + const ReadOptions& read_opts, bool* is_blob_index) { + return GetFromList(&memlist_history_, key, value, columns, timestamp, s, + merge_context, max_covering_tombstone_seq, seq, read_opts, + nullptr /*read_callback*/, is_blob_index); +} + +bool MemTableListVersion::GetFromList( + std::list* list, const LookupKey& key, std::string* value, + PinnableWideColumns* columns, std::string* timestamp, Status* s, + MergeContext* merge_context, SequenceNumber* max_covering_tombstone_seq, + SequenceNumber* seq, const ReadOptions& read_opts, ReadCallback* callback, + bool* is_blob_index) { + *seq = kMaxSequenceNumber; + + for (auto& memtable : *list) { + assert(memtable->IsFragmentedRangeTombstonesConstructed()); + SequenceNumber current_seq = kMaxSequenceNumber; + + bool done = + memtable->Get(key, value, columns, timestamp, s, merge_context, + max_covering_tombstone_seq, ¤t_seq, read_opts, + true /* immutable_memtable */, callback, is_blob_index); + if (*seq == kMaxSequenceNumber) { + // Store the most recent sequence number of any operation on this key. + // Since we only care about the most recent change, we only need to + // return the first operation found when searching memtables in + // reverse-chronological order. + // current_seq would be equal to kMaxSequenceNumber if the value was to be + // skipped. This allows seq to be assigned again when the next value is + // read. + *seq = current_seq; + } + + if (done) { + assert(*seq != kMaxSequenceNumber || s->IsNotFound()); + return true; + } + if (!done && !s->ok() && !s->IsMergeInProgress() && !s->IsNotFound()) { + return false; + } + } + return false; +} + +Status MemTableListVersion::AddRangeTombstoneIterators( + const ReadOptions& read_opts, Arena* /*arena*/, + RangeDelAggregator* range_del_agg) { + assert(range_del_agg != nullptr); + // Except for snapshot read, using kMaxSequenceNumber is OK because these + // are immutable memtables. + SequenceNumber read_seq = read_opts.snapshot != nullptr + ? read_opts.snapshot->GetSequenceNumber() + : kMaxSequenceNumber; + for (auto& m : memlist_) { + assert(m->IsFragmentedRangeTombstonesConstructed()); + std::unique_ptr range_del_iter( + m->NewRangeTombstoneIterator(read_opts, read_seq, + true /* immutable_memtable */)); + range_del_agg->AddTombstones(std::move(range_del_iter)); + } + return Status::OK(); +} + +void MemTableListVersion::AddIterators( + const ReadOptions& options, std::vector* iterator_list, + Arena* arena) { + for (auto& m : memlist_) { + iterator_list->push_back(m->NewIterator(options, arena)); + } +} + +void MemTableListVersion::AddIterators(const ReadOptions& options, + MergeIteratorBuilder* merge_iter_builder, + bool add_range_tombstone_iter) { + for (auto& m : memlist_) { + auto mem_iter = m->NewIterator(options, merge_iter_builder->GetArena()); + if (!add_range_tombstone_iter || options.ignore_range_deletions) { + merge_iter_builder->AddIterator(mem_iter); + } else { + // Except for snapshot read, using kMaxSequenceNumber is OK because these + // are immutable memtables. + SequenceNumber read_seq = options.snapshot != nullptr + ? options.snapshot->GetSequenceNumber() + : kMaxSequenceNumber; + TruncatedRangeDelIterator* mem_tombstone_iter = nullptr; + auto range_del_iter = m->NewRangeTombstoneIterator( + options, read_seq, true /* immutale_memtable */); + if (range_del_iter == nullptr || range_del_iter->empty()) { + delete range_del_iter; + } else { + mem_tombstone_iter = new TruncatedRangeDelIterator( + std::unique_ptr(range_del_iter), + &m->GetInternalKeyComparator(), nullptr /* smallest */, + nullptr /* largest */); + } + merge_iter_builder->AddPointAndTombstoneIterator(mem_iter, + mem_tombstone_iter); + } + } +} + +uint64_t MemTableListVersion::GetTotalNumEntries() const { + uint64_t total_num = 0; + for (auto& m : memlist_) { + total_num += m->num_entries(); + } + return total_num; +} + +MemTable::MemTableStats MemTableListVersion::ApproximateStats( + const Slice& start_ikey, const Slice& end_ikey) { + MemTable::MemTableStats total_stats = {0, 0}; + for (auto& m : memlist_) { + auto mStats = m->ApproximateStats(start_ikey, end_ikey); + total_stats.size += mStats.size; + total_stats.count += mStats.count; + } + return total_stats; +} + +uint64_t MemTableListVersion::GetTotalNumDeletes() const { + uint64_t total_num = 0; + for (auto& m : memlist_) { + total_num += m->num_deletes(); + } + return total_num; +} + +SequenceNumber MemTableListVersion::GetEarliestSequenceNumber( + bool include_history) const { + if (include_history && !memlist_history_.empty()) { + return memlist_history_.back()->GetEarliestSequenceNumber(); + } else if (!memlist_.empty()) { + return memlist_.back()->GetEarliestSequenceNumber(); + } else { + return kMaxSequenceNumber; + } +} + +SequenceNumber MemTableListVersion::GetFirstSequenceNumber() const { + SequenceNumber min_first_seqno = kMaxSequenceNumber; + // The first memtable in the list might not be the oldest one with mempurge + for (const auto& m : memlist_) { + min_first_seqno = std::min(m->GetFirstSequenceNumber(), min_first_seqno); + } + return min_first_seqno; +} + +// caller is responsible for referencing m +void MemTableListVersion::Add(MemTable* m, autovector* to_delete) { + assert(refs_ == 1); // only when refs_ == 1 is MemTableListVersion mutable + AddMemTable(m); + // m->MemoryAllocatedBytes() is added in MemoryAllocatedBytesExcludingLast + TrimHistory(to_delete, 0); +} + +// Removes m from list of memtables not flushed. Caller should NOT Unref m. +void MemTableListVersion::Remove(MemTable* m, + autovector* to_delete) { + assert(refs_ == 1); // only when refs_ == 1 is MemTableListVersion mutable + memlist_.remove(m); + + m->MarkFlushed(); + if (max_write_buffer_size_to_maintain_ > 0 || + max_write_buffer_number_to_maintain_ > 0) { + memlist_history_.push_front(m); + // Unable to get size of mutable memtable at this point, pass 0 to + // TrimHistory as a best effort. + TrimHistory(to_delete, 0); + } else { + UnrefMemTable(to_delete, m); + } +} + +// return the total memory usage assuming the oldest flushed memtable is dropped +size_t MemTableListVersion::MemoryAllocatedBytesExcludingLast() const { + size_t total_memtable_size = 0; + for (auto& memtable : memlist_) { + total_memtable_size += memtable->MemoryAllocatedBytes(); + } + for (auto& memtable : memlist_history_) { + total_memtable_size += memtable->MemoryAllocatedBytes(); + } + if (!memlist_history_.empty()) { + total_memtable_size -= memlist_history_.back()->MemoryAllocatedBytes(); + } + return total_memtable_size; +} + +bool MemTableListVersion::MemtableLimitExceeded(size_t usage) { + if (max_write_buffer_size_to_maintain_ > 0) { + // calculate the total memory usage after dropping the oldest flushed + // memtable, compare with max_write_buffer_size_to_maintain_ to decide + // whether to trim history + return MemoryAllocatedBytesExcludingLast() + usage >= + static_cast(max_write_buffer_size_to_maintain_); + } else if (max_write_buffer_number_to_maintain_ > 0) { + return memlist_.size() + memlist_history_.size() > + static_cast(max_write_buffer_number_to_maintain_); + } else { + return false; + } +} + +// Make sure we don't use up too much space in history +bool MemTableListVersion::TrimHistory(autovector* to_delete, + size_t usage) { + bool ret = false; + while (MemtableLimitExceeded(usage) && !memlist_history_.empty()) { + MemTable* x = memlist_history_.back(); + memlist_history_.pop_back(); + + UnrefMemTable(to_delete, x); + ret = true; + } + return ret; +} + +// Returns true if there is at least one memtable on which flush has +// not yet started. +bool MemTableList::IsFlushPending() const { + if ((flush_requested_ && num_flush_not_started_ > 0) || + (num_flush_not_started_ >= min_write_buffer_number_to_merge_)) { + assert(imm_flush_needed.load(std::memory_order_relaxed)); + return true; + } + return false; +} + +bool MemTableList::IsFlushPendingOrRunning() const { + if (current_->memlist_.size() - num_flush_not_started_ > 0) { + // Flush is already running on at least one memtable + return true; + } + return IsFlushPending(); +} + +// Returns the memtables that need to be flushed. +void MemTableList::PickMemtablesToFlush(uint64_t max_memtable_id, + autovector* ret, + uint64_t* max_next_log_number) { + AutoThreadOperationStageUpdater stage_updater( + ThreadStatus::STAGE_PICK_MEMTABLES_TO_FLUSH); + const auto& memlist = current_->memlist_; + bool atomic_flush = false; + + // Note: every time MemTableList::Add(mem) is called, it adds the new mem + // at the FRONT of the memlist (memlist.push_front(mem)). Therefore, by + // iterating through the memlist starting at the end, the vector + // ret is filled with memtables already sorted in increasing MemTable ID. + // However, when the mempurge feature is activated, new memtables with older + // IDs will be added to the memlist. + for (auto it = memlist.rbegin(); it != memlist.rend(); ++it) { + MemTable* m = *it; + if (!atomic_flush && m->atomic_flush_seqno_ != kMaxSequenceNumber) { + atomic_flush = true; + } + if (m->GetID() > max_memtable_id) { + break; + } + if (!m->flush_in_progress_) { + assert(!m->flush_completed_); + num_flush_not_started_--; + if (num_flush_not_started_ == 0) { + imm_flush_needed.store(false, std::memory_order_release); + } + m->flush_in_progress_ = true; // flushing will start very soon + if (max_next_log_number) { + *max_next_log_number = + std::max(m->GetNextLogNumber(), *max_next_log_number); + } + ret->push_back(m); + } else if (!ret->empty()) { + // This `break` is necessary to prevent picking non-consecutive memtables + // in case `memlist` has one or more entries with + // `flush_in_progress_ == true` sandwiched between entries with + // `flush_in_progress_ == false`. This could happen after parallel flushes + // are picked and the one flushing older memtables is rolled back. + break; + } + } + if (!atomic_flush || num_flush_not_started_ == 0) { + flush_requested_ = false; // start-flush request is complete + } +} + +void MemTableList::RollbackMemtableFlush(const autovector& mems, + uint64_t /*file_number*/) { + AutoThreadOperationStageUpdater stage_updater( + ThreadStatus::STAGE_MEMTABLE_ROLLBACK); + assert(!mems.empty()); + + // If the flush was not successful, then just reset state. + // Maybe a succeeding attempt to flush will be successful. + for (MemTable* m : mems) { + assert(m->flush_in_progress_); + assert(m->file_number_ == 0); + + m->flush_in_progress_ = false; + m->flush_completed_ = false; + m->edit_.Clear(); + num_flush_not_started_++; + } + imm_flush_needed.store(true, std::memory_order_release); +} + +// Try record a successful flush in the manifest file. It might just return +// Status::OK letting a concurrent flush to do actual the recording.. +Status MemTableList::TryInstallMemtableFlushResults( + ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options, + const autovector& mems, LogsWithPrepTracker* prep_tracker, + VersionSet* vset, InstrumentedMutex* mu, uint64_t file_number, + autovector* to_delete, FSDirectory* db_directory, + LogBuffer* log_buffer, + std::list>* committed_flush_jobs_info, + bool write_edits) { + AutoThreadOperationStageUpdater stage_updater( + ThreadStatus::STAGE_MEMTABLE_INSTALL_FLUSH_RESULTS); + mu->AssertHeld(); + + const ReadOptions read_options(Env::IOActivity::kFlush); + + // Flush was successful + // Record the status on the memtable object. Either this call or a call by a + // concurrent flush thread will read the status and write it to manifest. + for (size_t i = 0; i < mems.size(); ++i) { + // All the edits are associated with the first memtable of this batch. + assert(i == 0 || mems[i]->GetEdits()->NumEntries() == 0); + + mems[i]->flush_completed_ = true; + mems[i]->file_number_ = file_number; + } + + // if some other thread is already committing, then return + Status s; + if (commit_in_progress_) { + TEST_SYNC_POINT("MemTableList::TryInstallMemtableFlushResults:InProgress"); + return s; + } + + // Only a single thread can be executing this piece of code + commit_in_progress_ = true; + + // Retry until all completed flushes are committed. New flushes can finish + // while the current thread is writing manifest where mutex is released. + while (s.ok()) { + auto& memlist = current_->memlist_; + // The back is the oldest; if flush_completed_ is not set to it, it means + // that we were assigned a more recent memtable. The memtables' flushes must + // be recorded in manifest in order. A concurrent flush thread, who is + // assigned to flush the oldest memtable, will later wake up and does all + // the pending writes to manifest, in order. + if (memlist.empty() || !memlist.back()->flush_completed_) { + break; + } + // scan all memtables from the earliest, and commit those + // (in that order) that have finished flushing. Memtables + // are always committed in the order that they were created. + uint64_t batch_file_number = 0; + size_t batch_count = 0; + autovector edit_list; + autovector memtables_to_flush; + // enumerate from the last (earliest) element to see how many batch finished + for (auto it = memlist.rbegin(); it != memlist.rend(); ++it) { + MemTable* m = *it; + if (!m->flush_completed_) { + break; + } + if (it == memlist.rbegin() || batch_file_number != m->file_number_) { + batch_file_number = m->file_number_; + if (m->edit_.GetBlobFileAdditions().empty()) { + ROCKS_LOG_BUFFER(log_buffer, + "[%s] Level-0 commit table #%" PRIu64 " started", + cfd->GetName().c_str(), m->file_number_); + } else { + ROCKS_LOG_BUFFER(log_buffer, + "[%s] Level-0 commit table #%" PRIu64 + " (+%zu blob files) started", + cfd->GetName().c_str(), m->file_number_, + m->edit_.GetBlobFileAdditions().size()); + } + + edit_list.push_back(&m->edit_); + memtables_to_flush.push_back(m); + std::unique_ptr info = m->ReleaseFlushJobInfo(); + if (info != nullptr) { + committed_flush_jobs_info->push_back(std::move(info)); + } + } + batch_count++; + } + + // TODO(myabandeh): Not sure how batch_count could be 0 here. + if (batch_count > 0) { + uint64_t min_wal_number_to_keep = 0; + assert(edit_list.size() > 0); + if (vset->db_options()->allow_2pc) { + // Note that if mempurge is successful, the edit_list will + // not be applicable (contains info of new min_log number to keep, + // and level 0 file path of SST file created during normal flush, + // so both pieces of information are irrelevant after a successful + // mempurge operation). + min_wal_number_to_keep = PrecomputeMinLogNumberToKeep2PC( + vset, *cfd, edit_list, memtables_to_flush, prep_tracker); + + // We piggyback the information of earliest log file to keep in the + // manifest entry for the last file flushed. + } else { + min_wal_number_to_keep = + PrecomputeMinLogNumberToKeepNon2PC(vset, *cfd, edit_list); + } + + VersionEdit wal_deletion; + wal_deletion.SetMinLogNumberToKeep(min_wal_number_to_keep); + if (vset->db_options()->track_and_verify_wals_in_manifest) { + if (min_wal_number_to_keep > + vset->GetWalSet().GetMinWalNumberToKeep()) { + wal_deletion.DeleteWalsBefore(min_wal_number_to_keep); + } + TEST_SYNC_POINT_CALLBACK( + "MemTableList::TryInstallMemtableFlushResults:" + "AfterComputeMinWalToKeep", + nullptr); + } + edit_list.push_back(&wal_deletion); + + const auto manifest_write_cb = [this, cfd, batch_count, log_buffer, + to_delete, mu](const Status& status) { + RemoveMemTablesOrRestoreFlags(status, cfd, batch_count, log_buffer, + to_delete, mu); + }; + if (write_edits) { + // this can release and reacquire the mutex. + s = vset->LogAndApply(cfd, mutable_cf_options, read_options, edit_list, + mu, db_directory, /*new_descriptor_log=*/false, + /*column_family_options=*/nullptr, + manifest_write_cb); + } else { + // If write_edit is false (e.g: successful mempurge), + // then remove old memtables, wake up manifest write queue threads, + // and don't commit anything to the manifest file. + RemoveMemTablesOrRestoreFlags(s, cfd, batch_count, log_buffer, + to_delete, mu); + // Note: cfd->SetLogNumber is only called when a VersionEdit + // is written to MANIFEST. When mempurge is succesful, we skip + // this step, therefore cfd->GetLogNumber is always is + // earliest log with data unflushed. + // Notify new head of manifest write queue. + // wake up all the waiting writers + // TODO(bjlemaire): explain full reason WakeUpWaitingManifestWriters + // needed or investigate more. + vset->WakeUpWaitingManifestWriters(); + } + } + } + commit_in_progress_ = false; + return s; +} + +// New memtables are inserted at the front of the list. +void MemTableList::Add(MemTable* m, autovector* to_delete) { + assert(static_cast(current_->memlist_.size()) >= num_flush_not_started_); + InstallNewVersion(); + // this method is used to move mutable memtable into an immutable list. + // since mutable memtable is already refcounted by the DBImpl, + // and when moving to the immutable list we don't unref it, + // we don't have to ref the memtable here. we just take over the + // reference from the DBImpl. + current_->Add(m, to_delete); + m->MarkImmutable(); + num_flush_not_started_++; + if (num_flush_not_started_ == 1) { + imm_flush_needed.store(true, std::memory_order_release); + } + UpdateCachedValuesFromMemTableListVersion(); + ResetTrimHistoryNeeded(); +} + +bool MemTableList::TrimHistory(autovector* to_delete, size_t usage) { + InstallNewVersion(); + bool ret = current_->TrimHistory(to_delete, usage); + UpdateCachedValuesFromMemTableListVersion(); + ResetTrimHistoryNeeded(); + return ret; +} + +// Returns an estimate of the number of bytes of data in use. +size_t MemTableList::ApproximateUnflushedMemTablesMemoryUsage() { + size_t total_size = 0; + for (auto& memtable : current_->memlist_) { + total_size += memtable->ApproximateMemoryUsage(); + } + return total_size; +} + +size_t MemTableList::ApproximateMemoryUsage() { return current_memory_usage_; } + +size_t MemTableList::MemoryAllocatedBytesExcludingLast() const { + const size_t usage = current_memory_allocted_bytes_excluding_last_.load( + std::memory_order_relaxed); + return usage; +} + +bool MemTableList::HasHistory() const { + const bool has_history = current_has_history_.load(std::memory_order_relaxed); + return has_history; +} + +void MemTableList::UpdateCachedValuesFromMemTableListVersion() { + const size_t total_memtable_size = + current_->MemoryAllocatedBytesExcludingLast(); + current_memory_allocted_bytes_excluding_last_.store( + total_memtable_size, std::memory_order_relaxed); + + const bool has_history = current_->HasHistory(); + current_has_history_.store(has_history, std::memory_order_relaxed); +} + +uint64_t MemTableList::ApproximateOldestKeyTime() const { + if (!current_->memlist_.empty()) { + return current_->memlist_.back()->ApproximateOldestKeyTime(); + } + return std::numeric_limits::max(); +} + +void MemTableList::InstallNewVersion() { + if (current_->refs_ == 1) { + // we're the only one using the version, just keep using it + } else { + // somebody else holds the current version, we need to create new one + MemTableListVersion* version = current_; + current_ = new MemTableListVersion(¤t_memory_usage_, *version); + current_->Ref(); + version->Unref(); + } +} + +void MemTableList::RemoveMemTablesOrRestoreFlags( + const Status& s, ColumnFamilyData* cfd, size_t batch_count, + LogBuffer* log_buffer, autovector* to_delete, + InstrumentedMutex* mu) { + assert(mu); + mu->AssertHeld(); + assert(to_delete); + // we will be changing the version in the next code path, + // so we better create a new one, since versions are immutable + InstallNewVersion(); + + // All the later memtables that have the same filenum + // are part of the same batch. They can be committed now. + uint64_t mem_id = 1; // how many memtables have been flushed. + + // commit new state only if the column family is NOT dropped. + // The reason is as follows (refer to + // ColumnFamilyTest.FlushAndDropRaceCondition). + // If the column family is dropped, then according to LogAndApply, its + // corresponding flush operation is NOT written to the MANIFEST. This + // means the DB is not aware of the L0 files generated from the flush. + // By committing the new state, we remove the memtable from the memtable + // list. Creating an iterator on this column family will not be able to + // read full data since the memtable is removed, and the DB is not aware + // of the L0 files, causing MergingIterator unable to build child + // iterators. RocksDB contract requires that the iterator can be created + // on a dropped column family, and we must be able to + // read full data as long as column family handle is not deleted, even if + // the column family is dropped. + if (s.ok() && !cfd->IsDropped()) { // commit new state + while (batch_count-- > 0) { + MemTable* m = current_->memlist_.back(); + if (m->edit_.GetBlobFileAdditions().empty()) { + ROCKS_LOG_BUFFER(log_buffer, + "[%s] Level-0 commit table #%" PRIu64 + ": memtable #%" PRIu64 " done", + cfd->GetName().c_str(), m->file_number_, mem_id); + } else { + ROCKS_LOG_BUFFER(log_buffer, + "[%s] Level-0 commit table #%" PRIu64 + " (+%zu blob files)" + ": memtable #%" PRIu64 " done", + cfd->GetName().c_str(), m->file_number_, + m->edit_.GetBlobFileAdditions().size(), mem_id); + } + + assert(m->file_number_ > 0); + current_->Remove(m, to_delete); + UpdateCachedValuesFromMemTableListVersion(); + ResetTrimHistoryNeeded(); + ++mem_id; + } + } else { + for (auto it = current_->memlist_.rbegin(); batch_count-- > 0; ++it) { + MemTable* m = *it; + // commit failed. setup state so that we can flush again. + if (m->edit_.GetBlobFileAdditions().empty()) { + ROCKS_LOG_BUFFER(log_buffer, + "Level-0 commit table #%" PRIu64 ": memtable #%" PRIu64 + " failed", + m->file_number_, mem_id); + } else { + ROCKS_LOG_BUFFER(log_buffer, + "Level-0 commit table #%" PRIu64 + " (+%zu blob files)" + ": memtable #%" PRIu64 " failed", + m->file_number_, + m->edit_.GetBlobFileAdditions().size(), mem_id); + } + + m->flush_completed_ = false; + m->flush_in_progress_ = false; + m->edit_.Clear(); + num_flush_not_started_++; + m->file_number_ = 0; + imm_flush_needed.store(true, std::memory_order_release); + ++mem_id; + } + } +} + +uint64_t MemTableList::PrecomputeMinLogContainingPrepSection( + const std::unordered_set* memtables_to_flush) { + uint64_t min_log = 0; + + for (auto& m : current_->memlist_) { + if (memtables_to_flush && memtables_to_flush->count(m)) { + continue; + } + + auto log = m->GetMinLogContainingPrepSection(); + + if (log > 0 && (min_log == 0 || log < min_log)) { + min_log = log; + } + } + + return min_log; +} + +// Commit a successful atomic flush in the manifest file. +Status InstallMemtableAtomicFlushResults( + const autovector* imm_lists, + const autovector& cfds, + const autovector& mutable_cf_options_list, + const autovector*>& mems_list, VersionSet* vset, + LogsWithPrepTracker* prep_tracker, InstrumentedMutex* mu, + const autovector& file_metas, + const autovector>*>& + committed_flush_jobs_info, + autovector* to_delete, FSDirectory* db_directory, + LogBuffer* log_buffer) { + AutoThreadOperationStageUpdater stage_updater( + ThreadStatus::STAGE_MEMTABLE_INSTALL_FLUSH_RESULTS); + mu->AssertHeld(); + + const ReadOptions read_options(Env::IOActivity::kFlush); + + size_t num = mems_list.size(); + assert(cfds.size() == num); + if (imm_lists != nullptr) { + assert(imm_lists->size() == num); + } + if (num == 0) { + return Status::OK(); + } + + for (size_t k = 0; k != num; ++k) { +#ifndef NDEBUG + const auto* imm = + (imm_lists == nullptr) ? cfds[k]->imm() : imm_lists->at(k); + if (!mems_list[k]->empty()) { + assert((*mems_list[k])[0]->GetID() == imm->GetEarliestMemTableID()); + } +#endif + assert(nullptr != file_metas[k]); + for (size_t i = 0; i != mems_list[k]->size(); ++i) { + assert(i == 0 || (*mems_list[k])[i]->GetEdits()->NumEntries() == 0); + (*mems_list[k])[i]->SetFlushCompleted(true); + (*mems_list[k])[i]->SetFileNumber(file_metas[k]->fd.GetNumber()); + } + if (committed_flush_jobs_info[k]) { + assert(!mems_list[k]->empty()); + assert((*mems_list[k])[0]); + std::unique_ptr flush_job_info = + (*mems_list[k])[0]->ReleaseFlushJobInfo(); + committed_flush_jobs_info[k]->push_back(std::move(flush_job_info)); + } + } + + Status s; + + autovector> edit_lists; + uint32_t num_entries = 0; + for (const auto mems : mems_list) { + assert(mems != nullptr); + autovector edits; + assert(!mems->empty()); + edits.emplace_back((*mems)[0]->GetEdits()); + ++num_entries; + edit_lists.emplace_back(edits); + } + + WalNumber min_wal_number_to_keep = 0; + if (vset->db_options()->allow_2pc) { + min_wal_number_to_keep = PrecomputeMinLogNumberToKeep2PC( + vset, cfds, edit_lists, mems_list, prep_tracker); + } else { + min_wal_number_to_keep = + PrecomputeMinLogNumberToKeepNon2PC(vset, cfds, edit_lists); + } + + VersionEdit wal_deletion; + wal_deletion.SetMinLogNumberToKeep(min_wal_number_to_keep); + if (vset->db_options()->track_and_verify_wals_in_manifest && + min_wal_number_to_keep > vset->GetWalSet().GetMinWalNumberToKeep()) { + wal_deletion.DeleteWalsBefore(min_wal_number_to_keep); + } + edit_lists.back().push_back(&wal_deletion); + ++num_entries; + + // Mark the version edits as an atomic group if the number of version edits + // exceeds 1. + if (cfds.size() > 1) { + for (size_t i = 0; i < edit_lists.size(); i++) { + assert((edit_lists[i].size() == 1) || + ((edit_lists[i].size() == 2) && (i == edit_lists.size() - 1))); + for (auto& e : edit_lists[i]) { + e->MarkAtomicGroup(--num_entries); + } + } + assert(0 == num_entries); + } + + // this can release and reacquire the mutex. + s = vset->LogAndApply(cfds, mutable_cf_options_list, read_options, edit_lists, + mu, db_directory); + + for (size_t k = 0; k != cfds.size(); ++k) { + auto* imm = (imm_lists == nullptr) ? cfds[k]->imm() : imm_lists->at(k); + imm->InstallNewVersion(); + } + + if (s.ok() || s.IsColumnFamilyDropped()) { + for (size_t i = 0; i != cfds.size(); ++i) { + if (cfds[i]->IsDropped()) { + continue; + } + auto* imm = (imm_lists == nullptr) ? cfds[i]->imm() : imm_lists->at(i); + for (auto m : *mems_list[i]) { + assert(m->GetFileNumber() > 0); + uint64_t mem_id = m->GetID(); + + const VersionEdit* const edit = m->GetEdits(); + assert(edit); + + if (edit->GetBlobFileAdditions().empty()) { + ROCKS_LOG_BUFFER(log_buffer, + "[%s] Level-0 commit table #%" PRIu64 + ": memtable #%" PRIu64 " done", + cfds[i]->GetName().c_str(), m->GetFileNumber(), + mem_id); + } else { + ROCKS_LOG_BUFFER(log_buffer, + "[%s] Level-0 commit table #%" PRIu64 + " (+%zu blob files)" + ": memtable #%" PRIu64 " done", + cfds[i]->GetName().c_str(), m->GetFileNumber(), + edit->GetBlobFileAdditions().size(), mem_id); + } + + imm->current_->Remove(m, to_delete); + imm->UpdateCachedValuesFromMemTableListVersion(); + imm->ResetTrimHistoryNeeded(); + } + } + } else { + for (size_t i = 0; i != cfds.size(); ++i) { + auto* imm = (imm_lists == nullptr) ? cfds[i]->imm() : imm_lists->at(i); + for (auto m : *mems_list[i]) { + uint64_t mem_id = m->GetID(); + + const VersionEdit* const edit = m->GetEdits(); + assert(edit); + + if (edit->GetBlobFileAdditions().empty()) { + ROCKS_LOG_BUFFER(log_buffer, + "[%s] Level-0 commit table #%" PRIu64 + ": memtable #%" PRIu64 " failed", + cfds[i]->GetName().c_str(), m->GetFileNumber(), + mem_id); + } else { + ROCKS_LOG_BUFFER(log_buffer, + "[%s] Level-0 commit table #%" PRIu64 + " (+%zu blob files)" + ": memtable #%" PRIu64 " failed", + cfds[i]->GetName().c_str(), m->GetFileNumber(), + edit->GetBlobFileAdditions().size(), mem_id); + } + + m->SetFlushCompleted(false); + m->SetFlushInProgress(false); + m->GetEdits()->Clear(); + m->SetFileNumber(0); + imm->num_flush_not_started_++; + } + imm->imm_flush_needed.store(true, std::memory_order_release); + } + } + + return s; +} + +void MemTableList::RemoveOldMemTables(uint64_t log_number, + autovector* to_delete) { + assert(to_delete != nullptr); + InstallNewVersion(); + auto& memlist = current_->memlist_; + autovector old_memtables; + for (auto it = memlist.rbegin(); it != memlist.rend(); ++it) { + MemTable* mem = *it; + if (mem->GetNextLogNumber() > log_number) { + break; + } + old_memtables.push_back(mem); + } + + for (auto it = old_memtables.begin(); it != old_memtables.end(); ++it) { + MemTable* mem = *it; + current_->Remove(mem, to_delete); + --num_flush_not_started_; + if (0 == num_flush_not_started_) { + imm_flush_needed.store(false, std::memory_order_release); + } + } + + UpdateCachedValuesFromMemTableListVersion(); + ResetTrimHistoryNeeded(); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/memtable_list.h b/librocksdb-sys/rocksdb/db/memtable_list.h new file mode 100644 index 0000000..e95493b --- /dev/null +++ b/librocksdb-sys/rocksdb/db/memtable_list.h @@ -0,0 +1,490 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "db/logs_with_prep_tracker.h" +#include "db/memtable.h" +#include "db/range_del_aggregator.h" +#include "file/filename.h" +#include "logging/log_buffer.h" +#include "monitoring/instrumented_mutex.h" +#include "rocksdb/db.h" +#include "rocksdb/iterator.h" +#include "rocksdb/options.h" +#include "rocksdb/types.h" +#include "util/autovector.h" + +namespace ROCKSDB_NAMESPACE { + +class ColumnFamilyData; +class InternalKeyComparator; +class InstrumentedMutex; +class MergeIteratorBuilder; +class MemTableList; + +struct FlushJobInfo; + +// keeps a list of immutable memtables in a vector. the list is immutable +// if refcount is bigger than one. It is used as a state for Get() and +// Iterator code paths +// +// This class is not thread-safe. External synchronization is required +// (such as holding the db mutex or being on the write thread). +class MemTableListVersion { + public: + explicit MemTableListVersion(size_t* parent_memtable_list_memory_usage, + const MemTableListVersion& old); + explicit MemTableListVersion(size_t* parent_memtable_list_memory_usage, + int max_write_buffer_number_to_maintain, + int64_t max_write_buffer_size_to_maintain); + + void Ref(); + void Unref(autovector* to_delete = nullptr); + + // Search all the memtables starting from the most recent one. + // Return the most recent value found, if any. + // + // If any operation was found for this key, its most recent sequence number + // will be stored in *seq on success (regardless of whether true/false is + // returned). Otherwise, *seq will be set to kMaxSequenceNumber. + bool Get(const LookupKey& key, std::string* value, + PinnableWideColumns* columns, std::string* timestamp, Status* s, + MergeContext* merge_context, + SequenceNumber* max_covering_tombstone_seq, SequenceNumber* seq, + const ReadOptions& read_opts, ReadCallback* callback = nullptr, + bool* is_blob_index = nullptr); + + bool Get(const LookupKey& key, std::string* value, + PinnableWideColumns* columns, std::string* timestamp, Status* s, + MergeContext* merge_context, + SequenceNumber* max_covering_tombstone_seq, + const ReadOptions& read_opts, ReadCallback* callback = nullptr, + bool* is_blob_index = nullptr) { + SequenceNumber seq; + return Get(key, value, columns, timestamp, s, merge_context, + max_covering_tombstone_seq, &seq, read_opts, callback, + is_blob_index); + } + + void MultiGet(const ReadOptions& read_options, MultiGetRange* range, + ReadCallback* callback); + + // Returns all the merge operands corresponding to the key by searching all + // memtables starting from the most recent one. + bool GetMergeOperands(const LookupKey& key, Status* s, + MergeContext* merge_context, + SequenceNumber* max_covering_tombstone_seq, + const ReadOptions& read_opts); + + // Similar to Get(), but searches the Memtable history of memtables that + // have already been flushed. Should only be used from in-memory only + // queries (such as Transaction validation) as the history may contain + // writes that are also present in the SST files. + bool GetFromHistory(const LookupKey& key, std::string* value, + PinnableWideColumns* columns, std::string* timestamp, + Status* s, MergeContext* merge_context, + SequenceNumber* max_covering_tombstone_seq, + SequenceNumber* seq, const ReadOptions& read_opts, + bool* is_blob_index = nullptr); + bool GetFromHistory(const LookupKey& key, std::string* value, + PinnableWideColumns* columns, std::string* timestamp, + Status* s, MergeContext* merge_context, + SequenceNumber* max_covering_tombstone_seq, + const ReadOptions& read_opts, + bool* is_blob_index = nullptr) { + SequenceNumber seq; + return GetFromHistory(key, value, columns, timestamp, s, merge_context, + max_covering_tombstone_seq, &seq, read_opts, + is_blob_index); + } + + Status AddRangeTombstoneIterators(const ReadOptions& read_opts, Arena* arena, + RangeDelAggregator* range_del_agg); + + void AddIterators(const ReadOptions& options, + std::vector* iterator_list, + Arena* arena); + + void AddIterators(const ReadOptions& options, + MergeIteratorBuilder* merge_iter_builder, + bool add_range_tombstone_iter); + + uint64_t GetTotalNumEntries() const; + + uint64_t GetTotalNumDeletes() const; + + MemTable::MemTableStats ApproximateStats(const Slice& start_ikey, + const Slice& end_ikey); + + // Returns the value of MemTable::GetEarliestSequenceNumber() on the most + // recent MemTable in this list or kMaxSequenceNumber if the list is empty. + // If include_history=true, will also search Memtables in MemTableList + // History. + SequenceNumber GetEarliestSequenceNumber(bool include_history = false) const; + + // Return the first sequence number from the memtable list, which is the + // smallest sequence number of all FirstSequenceNumber. + // Return kMaxSequenceNumber if the list is empty. + SequenceNumber GetFirstSequenceNumber() const; + + private: + friend class MemTableList; + + friend Status InstallMemtableAtomicFlushResults( + const autovector* imm_lists, + const autovector& cfds, + const autovector& mutable_cf_options_list, + const autovector*>& mems_list, + VersionSet* vset, LogsWithPrepTracker* prep_tracker, + InstrumentedMutex* mu, const autovector& file_meta, + const autovector>*>& + committed_flush_jobs_info, + autovector* to_delete, FSDirectory* db_directory, + LogBuffer* log_buffer); + + // REQUIRE: m is an immutable memtable + void Add(MemTable* m, autovector* to_delete); + // REQUIRE: m is an immutable memtable + void Remove(MemTable* m, autovector* to_delete); + + // Return true if memtable is trimmed + bool TrimHistory(autovector* to_delete, size_t usage); + + bool GetFromList(std::list* list, const LookupKey& key, + std::string* value, PinnableWideColumns* columns, + std::string* timestamp, Status* s, + MergeContext* merge_context, + SequenceNumber* max_covering_tombstone_seq, + SequenceNumber* seq, const ReadOptions& read_opts, + ReadCallback* callback = nullptr, + bool* is_blob_index = nullptr); + + void AddMemTable(MemTable* m); + + void UnrefMemTable(autovector* to_delete, MemTable* m); + + // Calculate the total amount of memory used by memlist_ and memlist_history_ + // excluding the last MemTable in memlist_history_. The reason for excluding + // the last MemTable is to see if dropping the last MemTable will keep total + // memory usage above or equal to max_write_buffer_size_to_maintain_ + size_t MemoryAllocatedBytesExcludingLast() const; + + // Whether this version contains flushed memtables that are only kept around + // for transaction conflict checking. + bool HasHistory() const { return !memlist_history_.empty(); } + + bool MemtableLimitExceeded(size_t usage); + + // Immutable MemTables that have not yet been flushed. + std::list memlist_; + + // MemTables that have already been flushed + // (used during Transaction validation) + std::list memlist_history_; + + // Maximum number of MemTables to keep in memory (including both flushed + const int max_write_buffer_number_to_maintain_; + // Maximum size of MemTables to keep in memory (including both flushed + // and not-yet-flushed tables). + const int64_t max_write_buffer_size_to_maintain_; + + int refs_ = 0; + + size_t* parent_memtable_list_memory_usage_; +}; + +// This class stores references to all the immutable memtables. +// The memtables are flushed to L0 as soon as possible and in +// any order. If there are more than one immutable memtable, their +// flushes can occur concurrently. However, they are 'committed' +// to the manifest in FIFO order to maintain correctness and +// recoverability from a crash. +// +// +// Other than imm_flush_needed and imm_trim_needed, this class is not +// thread-safe and requires external synchronization (such as holding the db +// mutex or being on the write thread.) +class MemTableList { + public: + // A list of memtables. + explicit MemTableList(int min_write_buffer_number_to_merge, + int max_write_buffer_number_to_maintain, + int64_t max_write_buffer_size_to_maintain) + : imm_flush_needed(false), + imm_trim_needed(false), + min_write_buffer_number_to_merge_(min_write_buffer_number_to_merge), + current_(new MemTableListVersion(¤t_memory_usage_, + max_write_buffer_number_to_maintain, + max_write_buffer_size_to_maintain)), + num_flush_not_started_(0), + commit_in_progress_(false), + flush_requested_(false), + current_memory_usage_(0), + current_memory_allocted_bytes_excluding_last_(0), + current_has_history_(false) { + current_->Ref(); + } + + // Should not delete MemTableList without making sure MemTableList::current() + // is Unref()'d. + ~MemTableList() {} + + MemTableListVersion* current() const { return current_; } + + // so that background threads can detect non-nullptr pointer to + // determine whether there is anything more to start flushing. + std::atomic imm_flush_needed; + + std::atomic imm_trim_needed; + + // Returns the total number of memtables in the list that haven't yet + // been flushed and logged. + int NumNotFlushed() const; + + // Returns total number of memtables in the list that have been + // completely flushed and logged. + int NumFlushed() const; + + // Returns true if there is at least one memtable on which flush has + // not yet started. + bool IsFlushPending() const; + + // Returns true if there is at least one memtable that is pending flush or + // flushing. + bool IsFlushPendingOrRunning() const; + + // Returns the earliest memtables that needs to be flushed. The returned + // memtables are guaranteed to be in the ascending order of created time. + void PickMemtablesToFlush(uint64_t max_memtable_id, + autovector* mems, + uint64_t* max_next_log_number = nullptr); + + // Reset status of the given memtable list back to pending state so that + // they can get picked up again on the next round of flush. + void RollbackMemtableFlush(const autovector& mems, + uint64_t file_number); + + // Try commit a successful flush in the manifest file. It might just return + // Status::OK letting a concurrent flush to do the actual the recording. + Status TryInstallMemtableFlushResults( + ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options, + const autovector& m, LogsWithPrepTracker* prep_tracker, + VersionSet* vset, InstrumentedMutex* mu, uint64_t file_number, + autovector* to_delete, FSDirectory* db_directory, + LogBuffer* log_buffer, + std::list>* committed_flush_jobs_info, + bool write_edits = true); + + // New memtables are inserted at the front of the list. + // Takes ownership of the referenced held on *m by the caller of Add(). + // By default, adding memtables will flag that the memtable list needs to be + // flushed, but in certain situations, like after a mempurge, we may want to + // avoid flushing the memtable list upon addition of a memtable. + void Add(MemTable* m, autovector* to_delete); + + // Returns an estimate of the number of bytes of data in use. + size_t ApproximateMemoryUsage(); + + // Returns the cached current_memory_allocted_bytes_excluding_last_ value. + size_t MemoryAllocatedBytesExcludingLast() const; + + // Returns the cached current_has_history_ value. + bool HasHistory() const; + + // Updates current_memory_allocted_bytes_excluding_last_ and + // current_has_history_ from MemTableListVersion. Must be called whenever + // InstallNewVersion is called. + void UpdateCachedValuesFromMemTableListVersion(); + + // `usage` is the current size of the mutable Memtable. When + // max_write_buffer_size_to_maintain is used, total size of mutable and + // immutable memtables is checked against it to decide whether to trim + // memtable list. + // + // Return true if memtable is trimmed + bool TrimHistory(autovector* to_delete, size_t usage); + + // Returns an estimate of the number of bytes of data used by + // the unflushed mem-tables. + size_t ApproximateUnflushedMemTablesMemoryUsage(); + + // Returns an estimate of the timestamp of the earliest key. + uint64_t ApproximateOldestKeyTime() const; + + // Request a flush of all existing memtables to storage. This will + // cause future calls to IsFlushPending() to return true if this list is + // non-empty (regardless of the min_write_buffer_number_to_merge + // parameter). This flush request will persist until the next time + // PickMemtablesToFlush() is called. + void FlushRequested() { + flush_requested_ = true; + // If there are some memtables stored in imm() that don't trigger + // flush (eg: mempurge output memtable), then update imm_flush_needed. + // Note: if race condition and imm_flush_needed is set to true + // when there is num_flush_not_started_==0, then there is no + // impact whatsoever. Imm_flush_needed is only used in an assert + // in IsFlushPending(). + if (num_flush_not_started_ > 0) { + imm_flush_needed.store(true, std::memory_order_release); + } + } + + bool HasFlushRequested() { return flush_requested_; } + + // Returns true if a trim history should be scheduled and the caller should + // be the one to schedule it + bool MarkTrimHistoryNeeded() { + auto expected = false; + return imm_trim_needed.compare_exchange_strong( + expected, true, std::memory_order_relaxed, std::memory_order_relaxed); + } + + void ResetTrimHistoryNeeded() { + auto expected = true; + imm_trim_needed.compare_exchange_strong( + expected, false, std::memory_order_relaxed, std::memory_order_relaxed); + } + + // Copying allowed + // MemTableList(const MemTableList&); + // void operator=(const MemTableList&); + + size_t* current_memory_usage() { return ¤t_memory_usage_; } + + // Returns the min log containing the prep section after memtables listsed in + // `memtables_to_flush` are flushed and their status is persisted in manifest. + uint64_t PrecomputeMinLogContainingPrepSection( + const std::unordered_set* memtables_to_flush = nullptr); + + uint64_t GetEarliestMemTableID() const { + auto& memlist = current_->memlist_; + if (memlist.empty()) { + return std::numeric_limits::max(); + } + return memlist.back()->GetID(); + } + + uint64_t GetLatestMemTableID() const { + auto& memlist = current_->memlist_; + if (memlist.empty()) { + return 0; + } + return memlist.front()->GetID(); + } + + // DB mutex held. + // Gets the newest user-defined timestamp for the Memtables in ascending ID + // order, up to the `max_memtable_id`. Used by background flush job + // to check Memtables' eligibility for flush w.r.t retaining UDTs. + std::vector GetTablesNewestUDT(uint64_t max_memtable_id) { + std::vector newest_udts; + auto& memlist = current_->memlist_; + // Iterating through the memlist starting at the end, the vector + // ret is filled with memtables already sorted in increasing MemTable ID. + for (auto it = memlist.rbegin(); it != memlist.rend(); ++it) { + MemTable* m = *it; + if (m->GetID() > max_memtable_id) { + break; + } + newest_udts.push_back(m->GetNewestUDT()); + } + return newest_udts; + } + + void AssignAtomicFlushSeq(const SequenceNumber& seq) { + const auto& memlist = current_->memlist_; + // Scan the memtable list from new to old + for (auto it = memlist.begin(); it != memlist.end(); ++it) { + MemTable* mem = *it; + if (mem->atomic_flush_seqno_ == kMaxSequenceNumber) { + mem->atomic_flush_seqno_ = seq; + } else { + // Earlier memtables must have been assigned a atomic flush seq, no + // need to continue scan. + break; + } + } + } + + // Used only by DBImplSecondary during log replay. + // Remove memtables whose data were written before the WAL with log_number + // was created, i.e. mem->GetNextLogNumber() <= log_number. The memtables are + // not freed, but put into a vector for future deref and reclamation. + void RemoveOldMemTables(uint64_t log_number, + autovector* to_delete); + + private: + friend Status InstallMemtableAtomicFlushResults( + const autovector* imm_lists, + const autovector& cfds, + const autovector& mutable_cf_options_list, + const autovector*>& mems_list, + VersionSet* vset, LogsWithPrepTracker* prep_tracker, + InstrumentedMutex* mu, const autovector& file_meta, + const autovector>*>& + committed_flush_jobs_info, + autovector* to_delete, FSDirectory* db_directory, + LogBuffer* log_buffer); + + // DB mutex held + void InstallNewVersion(); + + // DB mutex held + // Called after writing to MANIFEST + void RemoveMemTablesOrRestoreFlags(const Status& s, ColumnFamilyData* cfd, + size_t batch_count, LogBuffer* log_buffer, + autovector* to_delete, + InstrumentedMutex* mu); + + const int min_write_buffer_number_to_merge_; + + MemTableListVersion* current_; + + // the number of elements that still need flushing + int num_flush_not_started_; + + // committing in progress + bool commit_in_progress_; + + // Requested a flush of memtables to storage. It's possible to request that + // a subset of memtables be flushed. + bool flush_requested_; + + // The current memory usage. + size_t current_memory_usage_; + + // Cached value of current_->MemoryAllocatedBytesExcludingLast(). + std::atomic current_memory_allocted_bytes_excluding_last_; + + // Cached value of current_->HasHistory(). + std::atomic current_has_history_; +}; + +// Installs memtable atomic flush results. +// In most cases, imm_lists is nullptr, and the function simply uses the +// immutable memtable lists associated with the cfds. There are unit tests that +// installs flush results for external immutable memtable lists other than the +// cfds' own immutable memtable lists, e.g. MemTableLIstTest. In this case, +// imm_lists parameter is not nullptr. +extern Status InstallMemtableAtomicFlushResults( + const autovector* imm_lists, + const autovector& cfds, + const autovector& mutable_cf_options_list, + const autovector*>& mems_list, VersionSet* vset, + LogsWithPrepTracker* prep_tracker, InstrumentedMutex* mu, + const autovector& file_meta, + const autovector>*>& + committed_flush_jobs_info, + autovector* to_delete, FSDirectory* db_directory, + LogBuffer* log_buffer); +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/memtable_list_test.cc b/librocksdb-sys/rocksdb/db/memtable_list_test.cc new file mode 100644 index 0000000..dfa1dbf --- /dev/null +++ b/librocksdb-sys/rocksdb/db/memtable_list_test.cc @@ -0,0 +1,1123 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/memtable_list.h" + +#include +#include +#include + +#include "db/merge_context.h" +#include "db/version_set.h" +#include "db/write_controller.h" +#include "rocksdb/db.h" +#include "rocksdb/status.h" +#include "rocksdb/write_buffer_manager.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +class MemTableListTest : public testing::Test { + public: + std::string dbname; + DB* db; + Options options; + std::vector handles; + std::atomic file_number; + + MemTableListTest() : db(nullptr), file_number(1) { + dbname = test::PerThreadDBPath("memtable_list_test"); + options.create_if_missing = true; + EXPECT_OK(DestroyDB(dbname, options)); + } + + // Create a test db if not yet created + void CreateDB() { + if (db == nullptr) { + options.create_if_missing = true; + EXPECT_OK(DestroyDB(dbname, options)); + // Open DB only with default column family + ColumnFamilyOptions cf_options; + std::vector cf_descs; + if (udt_enabled_) { + cf_options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + } + cf_descs.emplace_back(kDefaultColumnFamilyName, cf_options); + Status s = DB::Open(options, dbname, cf_descs, &handles, &db); + EXPECT_OK(s); + + ColumnFamilyOptions cf_opt1, cf_opt2; + cf_opt1.cf_paths.emplace_back(dbname + "_one_1", + std::numeric_limits::max()); + cf_opt2.cf_paths.emplace_back(dbname + "_two_1", + std::numeric_limits::max()); + int sz = static_cast(handles.size()); + handles.resize(sz + 2); + s = db->CreateColumnFamily(cf_opt1, "one", &handles[1]); + EXPECT_OK(s); + s = db->CreateColumnFamily(cf_opt2, "two", &handles[2]); + EXPECT_OK(s); + + cf_descs.emplace_back("one", cf_options); + cf_descs.emplace_back("two", cf_options); + } + } + + ~MemTableListTest() override { + if (db) { + std::vector cf_descs(handles.size()); + for (int i = 0; i != static_cast(handles.size()); ++i) { + EXPECT_OK(handles[i]->GetDescriptor(&cf_descs[i])); + } + for (auto h : handles) { + if (h) { + EXPECT_OK(db->DestroyColumnFamilyHandle(h)); + } + } + handles.clear(); + delete db; + db = nullptr; + EXPECT_OK(DestroyDB(dbname, options, cf_descs)); + } + } + + // Calls MemTableList::TryInstallMemtableFlushResults() and sets up all + // structures needed to call this function. + Status Mock_InstallMemtableFlushResults( + MemTableList* list, const MutableCFOptions& mutable_cf_options, + const autovector& m, autovector* to_delete) { + // Create a mock Logger + test::NullLogger logger; + LogBuffer log_buffer(DEBUG_LEVEL, &logger); + + CreateDB(); + // Create a mock VersionSet + DBOptions db_options; + ImmutableDBOptions immutable_db_options(db_options); + EnvOptions env_options; + std::shared_ptr table_cache(NewLRUCache(50000, 16)); + WriteBufferManager write_buffer_manager(db_options.db_write_buffer_size); + WriteController write_controller(10000000u); + + VersionSet versions(dbname, &immutable_db_options, env_options, + table_cache.get(), &write_buffer_manager, + &write_controller, /*block_cache_tracer=*/nullptr, + /*io_tracer=*/nullptr, /*db_id*/ "", + /*db_session_id*/ ""); + std::vector cf_descs; + cf_descs.emplace_back(kDefaultColumnFamilyName, ColumnFamilyOptions()); + cf_descs.emplace_back("one", ColumnFamilyOptions()); + cf_descs.emplace_back("two", ColumnFamilyOptions()); + + EXPECT_OK(versions.Recover(cf_descs, false)); + + // Create mock default ColumnFamilyData + auto column_family_set = versions.GetColumnFamilySet(); + LogsWithPrepTracker dummy_prep_tracker; + auto cfd = column_family_set->GetDefault(); + EXPECT_TRUE(nullptr != cfd); + uint64_t file_num = file_number.fetch_add(1); + IOStatus io_s; + // Create dummy mutex. + InstrumentedMutex mutex; + InstrumentedMutexLock l(&mutex); + std::list> flush_jobs_info; + Status s = list->TryInstallMemtableFlushResults( + cfd, mutable_cf_options, m, &dummy_prep_tracker, &versions, &mutex, + file_num, to_delete, nullptr, &log_buffer, &flush_jobs_info); + EXPECT_OK(io_s); + return s; + } + + // Calls MemTableList::InstallMemtableFlushResults() and sets up all + // structures needed to call this function. + Status Mock_InstallMemtableAtomicFlushResults( + autovector& lists, const autovector& cf_ids, + const autovector& mutable_cf_options_list, + const autovector*>& mems_list, + autovector* to_delete) { + // Create a mock Logger + test::NullLogger logger; + LogBuffer log_buffer(DEBUG_LEVEL, &logger); + + CreateDB(); + // Create a mock VersionSet + DBOptions db_options; + + ImmutableDBOptions immutable_db_options(db_options); + EnvOptions env_options; + std::shared_ptr table_cache(NewLRUCache(50000, 16)); + WriteBufferManager write_buffer_manager(db_options.db_write_buffer_size); + WriteController write_controller(10000000u); + + VersionSet versions(dbname, &immutable_db_options, env_options, + table_cache.get(), &write_buffer_manager, + &write_controller, /*block_cache_tracer=*/nullptr, + /*io_tracer=*/nullptr, /*db_id*/ "", + /*db_session_id*/ ""); + std::vector cf_descs; + cf_descs.emplace_back(kDefaultColumnFamilyName, ColumnFamilyOptions()); + cf_descs.emplace_back("one", ColumnFamilyOptions()); + cf_descs.emplace_back("two", ColumnFamilyOptions()); + EXPECT_OK(versions.Recover(cf_descs, false)); + + // Create mock default ColumnFamilyData + + auto column_family_set = versions.GetColumnFamilySet(); + + LogsWithPrepTracker dummy_prep_tracker; + autovector cfds; + for (int i = 0; i != static_cast(cf_ids.size()); ++i) { + cfds.emplace_back(column_family_set->GetColumnFamily(cf_ids[i])); + EXPECT_NE(nullptr, cfds[i]); + } + std::vector file_metas; + file_metas.reserve(cf_ids.size()); + for (size_t i = 0; i != cf_ids.size(); ++i) { + FileMetaData meta; + uint64_t file_num = file_number.fetch_add(1); + meta.fd = FileDescriptor(file_num, 0, 0); + file_metas.emplace_back(meta); + } + autovector file_meta_ptrs; + for (auto& meta : file_metas) { + file_meta_ptrs.push_back(&meta); + } + std::vector>> + committed_flush_jobs_info_storage(cf_ids.size()); + autovector>*> + committed_flush_jobs_info; + for (int i = 0; i < static_cast(cf_ids.size()); ++i) { + committed_flush_jobs_info.push_back( + &committed_flush_jobs_info_storage[i]); + } + + InstrumentedMutex mutex; + InstrumentedMutexLock l(&mutex); + return InstallMemtableAtomicFlushResults( + &lists, cfds, mutable_cf_options_list, mems_list, &versions, + nullptr /* prep_tracker */, &mutex, file_meta_ptrs, + committed_flush_jobs_info, to_delete, nullptr, &log_buffer); + } + + protected: + bool udt_enabled_ = false; +}; + +TEST_F(MemTableListTest, Empty) { + // Create an empty MemTableList and validate basic functions. + MemTableList list(1, 0, 0); + + ASSERT_EQ(0, list.NumNotFlushed()); + ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire)); + ASSERT_FALSE(list.IsFlushPending()); + + autovector mems; + list.PickMemtablesToFlush( + std::numeric_limits::max() /* memtable_id */, &mems); + ASSERT_EQ(0, mems.size()); + + autovector to_delete; + list.current()->Unref(&to_delete); + ASSERT_EQ(0, to_delete.size()); +} + +TEST_F(MemTableListTest, GetTest) { + // Create MemTableList + int min_write_buffer_number_to_merge = 2; + int max_write_buffer_number_to_maintain = 0; + int64_t max_write_buffer_size_to_maintain = 0; + MemTableList list(min_write_buffer_number_to_merge, + max_write_buffer_number_to_maintain, + max_write_buffer_size_to_maintain); + + SequenceNumber seq = 1; + std::string value; + Status s; + MergeContext merge_context; + InternalKeyComparator ikey_cmp(options.comparator); + SequenceNumber max_covering_tombstone_seq = 0; + autovector to_delete; + + LookupKey lkey("key1", seq); + bool found = list.current()->Get(lkey, &value, /*columns=*/nullptr, + /*timestamp=*/nullptr, &s, &merge_context, + &max_covering_tombstone_seq, ReadOptions()); + ASSERT_FALSE(found); + + // Create a MemTable + InternalKeyComparator cmp(BytewiseComparator()); + auto factory = std::make_shared(); + options.memtable_factory = factory; + ImmutableOptions ioptions(options); + + WriteBufferManager wb(options.db_write_buffer_size); + MemTable* mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb, + kMaxSequenceNumber, 0 /* column_family_id */); + mem->Ref(); + + // Write some keys to this memtable. + ASSERT_OK( + mem->Add(++seq, kTypeDeletion, "key1", "", nullptr /* kv_prot_info */)); + ASSERT_OK(mem->Add(++seq, kTypeValue, "key2", "value2", + nullptr /* kv_prot_info */)); + ASSERT_OK(mem->Add(++seq, kTypeValue, "key1", "value1", + nullptr /* kv_prot_info */)); + ASSERT_OK(mem->Add(++seq, kTypeValue, "key2", "value2.2", + nullptr /* kv_prot_info */)); + + // Fetch the newly written keys + merge_context.Clear(); + found = mem->Get(LookupKey("key1", seq), &value, /*columns*/ nullptr, + /*timestamp*/ nullptr, &s, &merge_context, + &max_covering_tombstone_seq, ReadOptions(), + false /* immutable_memtable */); + ASSERT_TRUE(s.ok() && found); + ASSERT_EQ(value, "value1"); + + merge_context.Clear(); + found = mem->Get(LookupKey("key1", 2), &value, /*columns*/ nullptr, + /*timestamp*/ nullptr, &s, &merge_context, + &max_covering_tombstone_seq, ReadOptions(), + false /* immutable_memtable */); + // MemTable found out that this key is *not* found (at this sequence#) + ASSERT_TRUE(found && s.IsNotFound()); + + merge_context.Clear(); + found = mem->Get(LookupKey("key2", seq), &value, /*columns*/ nullptr, + /*timestamp*/ nullptr, &s, &merge_context, + &max_covering_tombstone_seq, ReadOptions(), + false /* immutable_memtable */); + ASSERT_TRUE(s.ok() && found); + ASSERT_EQ(value, "value2.2"); + + ASSERT_EQ(4, mem->num_entries()); + ASSERT_EQ(1, mem->num_deletes()); + + // Add memtable to list + // This is to make assert(memtable->IsFragmentedRangeTombstonesConstructed()) + // in MemTableListVersion::GetFromList work. + mem->ConstructFragmentedRangeTombstones(); + list.Add(mem, &to_delete); + + SequenceNumber saved_seq = seq; + + // Create another memtable and write some keys to it + WriteBufferManager wb2(options.db_write_buffer_size); + MemTable* mem2 = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb2, + kMaxSequenceNumber, 0 /* column_family_id */); + mem2->Ref(); + + ASSERT_OK( + mem2->Add(++seq, kTypeDeletion, "key1", "", nullptr /* kv_prot_info */)); + ASSERT_OK(mem2->Add(++seq, kTypeValue, "key2", "value2.3", + nullptr /* kv_prot_info */)); + + // Add second memtable to list + // This is to make assert(memtable->IsFragmentedRangeTombstonesConstructed()) + // in MemTableListVersion::GetFromList work. + mem2->ConstructFragmentedRangeTombstones(); + list.Add(mem2, &to_delete); + + // Fetch keys via MemTableList + merge_context.Clear(); + found = + list.current()->Get(LookupKey("key1", seq), &value, /*columns=*/nullptr, + /*timestamp=*/nullptr, &s, &merge_context, + &max_covering_tombstone_seq, ReadOptions()); + ASSERT_TRUE(found && s.IsNotFound()); + + merge_context.Clear(); + found = list.current()->Get(LookupKey("key1", saved_seq), &value, + /*columns=*/nullptr, /*timestamp=*/nullptr, &s, + &merge_context, &max_covering_tombstone_seq, + ReadOptions()); + ASSERT_TRUE(s.ok() && found); + ASSERT_EQ("value1", value); + + merge_context.Clear(); + found = + list.current()->Get(LookupKey("key2", seq), &value, /*columns=*/nullptr, + /*timestamp=*/nullptr, &s, &merge_context, + &max_covering_tombstone_seq, ReadOptions()); + ASSERT_TRUE(s.ok() && found); + ASSERT_EQ(value, "value2.3"); + + merge_context.Clear(); + found = list.current()->Get(LookupKey("key2", 1), &value, /*columns=*/nullptr, + /*timestamp=*/nullptr, &s, &merge_context, + &max_covering_tombstone_seq, ReadOptions()); + ASSERT_FALSE(found); + + ASSERT_EQ(2, list.NumNotFlushed()); + + list.current()->Unref(&to_delete); + for (MemTable* m : to_delete) { + delete m; + } +} + +TEST_F(MemTableListTest, GetFromHistoryTest) { + // Create MemTableList + int min_write_buffer_number_to_merge = 2; + int max_write_buffer_number_to_maintain = 2; + int64_t max_write_buffer_size_to_maintain = 2 * Arena::kInlineSize; + MemTableList list(min_write_buffer_number_to_merge, + max_write_buffer_number_to_maintain, + max_write_buffer_size_to_maintain); + + SequenceNumber seq = 1; + std::string value; + Status s; + MergeContext merge_context; + InternalKeyComparator ikey_cmp(options.comparator); + SequenceNumber max_covering_tombstone_seq = 0; + autovector to_delete; + + LookupKey lkey("key1", seq); + bool found = list.current()->Get(lkey, &value, /*columns=*/nullptr, + /*timestamp=*/nullptr, &s, &merge_context, + &max_covering_tombstone_seq, ReadOptions()); + ASSERT_FALSE(found); + + // Create a MemTable + InternalKeyComparator cmp(BytewiseComparator()); + auto factory = std::make_shared(); + options.memtable_factory = factory; + ImmutableOptions ioptions(options); + + WriteBufferManager wb(options.db_write_buffer_size); + MemTable* mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb, + kMaxSequenceNumber, 0 /* column_family_id */); + mem->Ref(); + + // Write some keys to this memtable. + ASSERT_OK( + mem->Add(++seq, kTypeDeletion, "key1", "", nullptr /* kv_prot_info */)); + ASSERT_OK(mem->Add(++seq, kTypeValue, "key2", "value2", + nullptr /* kv_prot_info */)); + ASSERT_OK(mem->Add(++seq, kTypeValue, "key2", "value2.2", + nullptr /* kv_prot_info */)); + + // Fetch the newly written keys + merge_context.Clear(); + found = mem->Get(LookupKey("key1", seq), &value, /*columns*/ nullptr, + /*timestamp*/ nullptr, &s, &merge_context, + &max_covering_tombstone_seq, ReadOptions(), + false /* immutable_memtable */); + // MemTable found out that this key is *not* found (at this sequence#) + ASSERT_TRUE(found && s.IsNotFound()); + + merge_context.Clear(); + found = mem->Get(LookupKey("key2", seq), &value, /*columns*/ nullptr, + /*timestamp*/ nullptr, &s, &merge_context, + &max_covering_tombstone_seq, ReadOptions(), + false /* immutable_memtable */); + ASSERT_TRUE(s.ok() && found); + ASSERT_EQ(value, "value2.2"); + + // Add memtable to list + // This is to make assert(memtable->IsFragmentedRangeTombstonesConstructed()) + // in MemTableListVersion::GetFromList work. + mem->ConstructFragmentedRangeTombstones(); + list.Add(mem, &to_delete); + ASSERT_EQ(0, to_delete.size()); + + // Fetch keys via MemTableList + merge_context.Clear(); + found = + list.current()->Get(LookupKey("key1", seq), &value, /*columns=*/nullptr, + /*timestamp=*/nullptr, &s, &merge_context, + &max_covering_tombstone_seq, ReadOptions()); + ASSERT_TRUE(found && s.IsNotFound()); + + merge_context.Clear(); + found = + list.current()->Get(LookupKey("key2", seq), &value, /*columns=*/nullptr, + /*timestamp=*/nullptr, &s, &merge_context, + &max_covering_tombstone_seq, ReadOptions()); + ASSERT_TRUE(s.ok() && found); + ASSERT_EQ("value2.2", value); + + // Flush this memtable from the list. + // (It will then be a part of the memtable history). + autovector to_flush; + list.PickMemtablesToFlush( + std::numeric_limits::max() /* memtable_id */, &to_flush); + ASSERT_EQ(1, to_flush.size()); + + MutableCFOptions mutable_cf_options(options); + s = Mock_InstallMemtableFlushResults(&list, mutable_cf_options, to_flush, + &to_delete); + ASSERT_OK(s); + ASSERT_EQ(0, list.NumNotFlushed()); + ASSERT_EQ(1, list.NumFlushed()); + ASSERT_EQ(0, to_delete.size()); + + // Verify keys are no longer in MemTableList + merge_context.Clear(); + found = + list.current()->Get(LookupKey("key1", seq), &value, /*columns=*/nullptr, + /*timestamp=*/nullptr, &s, &merge_context, + &max_covering_tombstone_seq, ReadOptions()); + ASSERT_FALSE(found); + + merge_context.Clear(); + found = + list.current()->Get(LookupKey("key2", seq), &value, /*columns=*/nullptr, + /*timestamp=*/nullptr, &s, &merge_context, + &max_covering_tombstone_seq, ReadOptions()); + ASSERT_FALSE(found); + + // Verify keys are present in history + merge_context.Clear(); + found = list.current()->GetFromHistory( + LookupKey("key1", seq), &value, /*columns=*/nullptr, + /*timestamp=*/nullptr, &s, &merge_context, &max_covering_tombstone_seq, + ReadOptions()); + ASSERT_TRUE(found && s.IsNotFound()); + + merge_context.Clear(); + found = list.current()->GetFromHistory( + LookupKey("key2", seq), &value, /*columns=*/nullptr, + /*timestamp=*/nullptr, &s, &merge_context, &max_covering_tombstone_seq, + ReadOptions()); + ASSERT_TRUE(found); + ASSERT_EQ("value2.2", value); + + // Create another memtable and write some keys to it + WriteBufferManager wb2(options.db_write_buffer_size); + MemTable* mem2 = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb2, + kMaxSequenceNumber, 0 /* column_family_id */); + mem2->Ref(); + + ASSERT_OK( + mem2->Add(++seq, kTypeDeletion, "key1", "", nullptr /* kv_prot_info */)); + ASSERT_OK(mem2->Add(++seq, kTypeValue, "key3", "value3", + nullptr /* kv_prot_info */)); + + // Add second memtable to list + // This is to make assert(memtable->IsFragmentedRangeTombstonesConstructed()) + // in MemTableListVersion::GetFromList work. + mem2->ConstructFragmentedRangeTombstones(); + list.Add(mem2, &to_delete); + ASSERT_EQ(0, to_delete.size()); + + to_flush.clear(); + list.PickMemtablesToFlush( + std::numeric_limits::max() /* memtable_id */, &to_flush); + ASSERT_EQ(1, to_flush.size()); + + // Flush second memtable + s = Mock_InstallMemtableFlushResults(&list, mutable_cf_options, to_flush, + &to_delete); + ASSERT_OK(s); + ASSERT_EQ(0, list.NumNotFlushed()); + ASSERT_EQ(2, list.NumFlushed()); + ASSERT_EQ(0, to_delete.size()); + + // Add a third memtable to push the first memtable out of the history + WriteBufferManager wb3(options.db_write_buffer_size); + MemTable* mem3 = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb3, + kMaxSequenceNumber, 0 /* column_family_id */); + mem3->Ref(); + // This is to make assert(memtable->IsFragmentedRangeTombstonesConstructed()) + // in MemTableListVersion::GetFromList work. + mem3->ConstructFragmentedRangeTombstones(); + list.Add(mem3, &to_delete); + ASSERT_EQ(1, list.NumNotFlushed()); + ASSERT_EQ(1, list.NumFlushed()); + ASSERT_EQ(1, to_delete.size()); + + // Verify keys are no longer in MemTableList + merge_context.Clear(); + found = + list.current()->Get(LookupKey("key1", seq), &value, /*columns=*/nullptr, + /*timestamp=*/nullptr, &s, &merge_context, + &max_covering_tombstone_seq, ReadOptions()); + ASSERT_FALSE(found); + + merge_context.Clear(); + found = + list.current()->Get(LookupKey("key2", seq), &value, /*columns=*/nullptr, + /*timestamp=*/nullptr, &s, &merge_context, + &max_covering_tombstone_seq, ReadOptions()); + ASSERT_FALSE(found); + + merge_context.Clear(); + found = + list.current()->Get(LookupKey("key3", seq), &value, /*columns=*/nullptr, + /*timestamp=*/nullptr, &s, &merge_context, + &max_covering_tombstone_seq, ReadOptions()); + ASSERT_FALSE(found); + + // Verify that the second memtable's keys are in the history + merge_context.Clear(); + found = list.current()->GetFromHistory( + LookupKey("key1", seq), &value, /*columns=*/nullptr, + /*timestamp=*/nullptr, &s, &merge_context, &max_covering_tombstone_seq, + ReadOptions()); + ASSERT_TRUE(found && s.IsNotFound()); + + merge_context.Clear(); + found = list.current()->GetFromHistory( + LookupKey("key3", seq), &value, /*columns=*/nullptr, + /*timestamp=*/nullptr, &s, &merge_context, &max_covering_tombstone_seq, + ReadOptions()); + ASSERT_TRUE(found); + ASSERT_EQ("value3", value); + + // Verify that key2 from the first memtable is no longer in the history + merge_context.Clear(); + found = + list.current()->Get(LookupKey("key2", seq), &value, /*columns=*/nullptr, + /*timestamp=*/nullptr, &s, &merge_context, + &max_covering_tombstone_seq, ReadOptions()); + ASSERT_FALSE(found); + + // Cleanup + list.current()->Unref(&to_delete); + ASSERT_EQ(3, to_delete.size()); + for (MemTable* m : to_delete) { + delete m; + } +} + +TEST_F(MemTableListTest, FlushPendingTest) { + const int num_tables = 6; + SequenceNumber seq = 1; + Status s; + + auto factory = std::make_shared(); + options.memtable_factory = factory; + ImmutableOptions ioptions(options); + InternalKeyComparator cmp(BytewiseComparator()); + WriteBufferManager wb(options.db_write_buffer_size); + autovector to_delete; + + // Create MemTableList + int min_write_buffer_number_to_merge = 3; + int max_write_buffer_number_to_maintain = 7; + int64_t max_write_buffer_size_to_maintain = + 7 * static_cast(options.write_buffer_size); + MemTableList list(min_write_buffer_number_to_merge, + max_write_buffer_number_to_maintain, + max_write_buffer_size_to_maintain); + + // Create some MemTables + uint64_t memtable_id = 0; + std::vector tables; + MutableCFOptions mutable_cf_options(options); + for (int i = 0; i < num_tables; i++) { + MemTable* mem = new MemTable(cmp, ioptions, mutable_cf_options, &wb, + kMaxSequenceNumber, 0 /* column_family_id */); + mem->SetID(memtable_id++); + mem->Ref(); + + std::string value; + MergeContext merge_context; + + ASSERT_OK(mem->Add(++seq, kTypeValue, "key1", std::to_string(i), + nullptr /* kv_prot_info */)); + ASSERT_OK(mem->Add(++seq, kTypeValue, "keyN" + std::to_string(i), "valueN", + nullptr /* kv_prot_info */)); + ASSERT_OK(mem->Add(++seq, kTypeValue, "keyX" + std::to_string(i), "value", + nullptr /* kv_prot_info */)); + ASSERT_OK(mem->Add(++seq, kTypeValue, "keyM" + std::to_string(i), "valueM", + nullptr /* kv_prot_info */)); + ASSERT_OK(mem->Add(++seq, kTypeDeletion, "keyX" + std::to_string(i), "", + nullptr /* kv_prot_info */)); + + tables.push_back(mem); + } + + // Nothing to flush + ASSERT_FALSE(list.IsFlushPending()); + ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire)); + autovector to_flush; + list.PickMemtablesToFlush( + std::numeric_limits::max() /* memtable_id */, &to_flush); + ASSERT_EQ(0, to_flush.size()); + + // Request a flush even though there is nothing to flush + list.FlushRequested(); + ASSERT_FALSE(list.IsFlushPending()); + ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire)); + + // Attempt to 'flush' to clear request for flush + list.PickMemtablesToFlush( + std::numeric_limits::max() /* memtable_id */, &to_flush); + ASSERT_EQ(0, to_flush.size()); + ASSERT_FALSE(list.IsFlushPending()); + ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire)); + + // Request a flush again + list.FlushRequested(); + // No flush pending since the list is empty. + ASSERT_FALSE(list.IsFlushPending()); + ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire)); + + // Add 2 tables + list.Add(tables[0], &to_delete); + list.Add(tables[1], &to_delete); + ASSERT_EQ(2, list.NumNotFlushed()); + ASSERT_EQ(0, to_delete.size()); + + // Even though we have less than the minimum to flush, a flush is + // pending since we had previously requested a flush and never called + // PickMemtablesToFlush() to clear the flush. + ASSERT_TRUE(list.IsFlushPending()); + ASSERT_TRUE(list.imm_flush_needed.load(std::memory_order_acquire)); + + // Pick tables to flush + list.PickMemtablesToFlush( + std::numeric_limits::max() /* memtable_id */, &to_flush); + ASSERT_EQ(2, to_flush.size()); + ASSERT_EQ(2, list.NumNotFlushed()); + ASSERT_FALSE(list.IsFlushPending()); + ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire)); + + // Revert flush + list.RollbackMemtableFlush(to_flush, 0); + ASSERT_FALSE(list.IsFlushPending()); + ASSERT_TRUE(list.imm_flush_needed.load(std::memory_order_acquire)); + to_flush.clear(); + + // Add another table + list.Add(tables[2], &to_delete); + // We now have the minimum to flush regardles of whether FlushRequested() + // was called. + ASSERT_TRUE(list.IsFlushPending()); + ASSERT_TRUE(list.imm_flush_needed.load(std::memory_order_acquire)); + ASSERT_EQ(0, to_delete.size()); + + // Pick tables to flush + list.PickMemtablesToFlush( + std::numeric_limits::max() /* memtable_id */, &to_flush); + ASSERT_EQ(3, to_flush.size()); + ASSERT_EQ(3, list.NumNotFlushed()); + ASSERT_FALSE(list.IsFlushPending()); + ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire)); + + // Pick tables to flush again + autovector to_flush2; + list.PickMemtablesToFlush( + std::numeric_limits::max() /* memtable_id */, &to_flush2); + ASSERT_EQ(0, to_flush2.size()); + ASSERT_EQ(3, list.NumNotFlushed()); + ASSERT_FALSE(list.IsFlushPending()); + ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire)); + + // Add another table + list.Add(tables[3], &to_delete); + ASSERT_FALSE(list.IsFlushPending()); + ASSERT_TRUE(list.imm_flush_needed.load(std::memory_order_acquire)); + ASSERT_EQ(0, to_delete.size()); + + // Request a flush again + list.FlushRequested(); + ASSERT_TRUE(list.IsFlushPending()); + ASSERT_TRUE(list.imm_flush_needed.load(std::memory_order_acquire)); + + // Pick tables to flush again + list.PickMemtablesToFlush( + std::numeric_limits::max() /* memtable_id */, &to_flush2); + ASSERT_EQ(1, to_flush2.size()); + ASSERT_EQ(4, list.NumNotFlushed()); + ASSERT_FALSE(list.IsFlushPending()); + ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire)); + + // Rollback first pick of tables + list.RollbackMemtableFlush(to_flush, 0); + ASSERT_TRUE(list.IsFlushPending()); + ASSERT_TRUE(list.imm_flush_needed.load(std::memory_order_acquire)); + to_flush.clear(); + + // Add another tables + list.Add(tables[4], &to_delete); + ASSERT_EQ(5, list.NumNotFlushed()); + // We now have the minimum to flush regardles of whether FlushRequested() + ASSERT_TRUE(list.IsFlushPending()); + ASSERT_TRUE(list.imm_flush_needed.load(std::memory_order_acquire)); + ASSERT_EQ(0, to_delete.size()); + + // Pick tables to flush + list.PickMemtablesToFlush( + std::numeric_limits::max() /* memtable_id */, &to_flush); + // Picks three oldest memtables. The fourth oldest is picked in `to_flush2` so + // must be excluded. The newest (fifth oldest) is non-consecutive with the + // three oldest due to omitting the fourth oldest so must not be picked. + ASSERT_EQ(3, to_flush.size()); + ASSERT_EQ(5, list.NumNotFlushed()); + ASSERT_FALSE(list.IsFlushPending()); + ASSERT_TRUE(list.imm_flush_needed.load(std::memory_order_acquire)); + + // Pick tables to flush again + autovector to_flush3; + list.PickMemtablesToFlush( + std::numeric_limits::max() /* memtable_id */, &to_flush3); + // Picks newest (fifth oldest) + ASSERT_EQ(1, to_flush3.size()); + ASSERT_EQ(5, list.NumNotFlushed()); + ASSERT_FALSE(list.IsFlushPending()); + ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire)); + + // Nothing left to flush + autovector to_flush4; + list.PickMemtablesToFlush( + std::numeric_limits::max() /* memtable_id */, &to_flush4); + ASSERT_EQ(0, to_flush4.size()); + ASSERT_EQ(5, list.NumNotFlushed()); + ASSERT_FALSE(list.IsFlushPending()); + ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire)); + + // Flush the 3 memtables that were picked in to_flush + s = Mock_InstallMemtableFlushResults(&list, mutable_cf_options, to_flush, + &to_delete); + ASSERT_OK(s); + + // Note: now to_flush contains tables[0,1,2]. to_flush2 contains + // tables[3]. to_flush3 contains tables[4]. + // Current implementation will only commit memtables in the order they were + // created. So TryInstallMemtableFlushResults will install the first 3 tables + // in to_flush and stop when it encounters a table not yet flushed. + ASSERT_EQ(2, list.NumNotFlushed()); + int num_in_history = + std::min(3, static_cast(max_write_buffer_size_to_maintain) / + static_cast(options.write_buffer_size)); + ASSERT_EQ(num_in_history, list.NumFlushed()); + ASSERT_EQ(5 - list.NumNotFlushed() - num_in_history, to_delete.size()); + + // Request a flush again. Should be nothing to flush + list.FlushRequested(); + ASSERT_FALSE(list.IsFlushPending()); + ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire)); + + // Flush the 1 memtable (tables[4]) that was picked in to_flush3 + s = MemTableListTest::Mock_InstallMemtableFlushResults( + &list, mutable_cf_options, to_flush3, &to_delete); + ASSERT_OK(s); + + // This will install 0 tables since tables[4] flushed while tables[3] has not + // yet flushed. + ASSERT_EQ(2, list.NumNotFlushed()); + ASSERT_EQ(0, to_delete.size()); + + // Flush the 1 memtable (tables[3]) that was picked in to_flush2 + s = MemTableListTest::Mock_InstallMemtableFlushResults( + &list, mutable_cf_options, to_flush2, &to_delete); + ASSERT_OK(s); + + // This will actually install 2 tables. The 1 we told it to flush, and also + // tables[4] which has been waiting for tables[3] to commit. + ASSERT_EQ(0, list.NumNotFlushed()); + num_in_history = + std::min(5, static_cast(max_write_buffer_size_to_maintain) / + static_cast(options.write_buffer_size)); + ASSERT_EQ(num_in_history, list.NumFlushed()); + ASSERT_EQ(5 - list.NumNotFlushed() - num_in_history, to_delete.size()); + + for (const auto& m : to_delete) { + // Refcount should be 0 after calling TryInstallMemtableFlushResults. + // Verify this, by Ref'ing then UnRef'ing: + m->Ref(); + ASSERT_EQ(m, m->Unref()); + delete m; + } + to_delete.clear(); + + // Add another table + list.Add(tables[5], &to_delete); + ASSERT_EQ(1, list.NumNotFlushed()); + ASSERT_EQ(5, list.GetLatestMemTableID()); + memtable_id = 4; + // Pick tables to flush. The tables to pick must have ID smaller than or + // equal to 4. Therefore, no table will be selected in this case. + autovector to_flush5; + list.FlushRequested(); + ASSERT_TRUE(list.HasFlushRequested()); + list.PickMemtablesToFlush(memtable_id, &to_flush5); + ASSERT_TRUE(to_flush5.empty()); + ASSERT_EQ(1, list.NumNotFlushed()); + ASSERT_TRUE(list.imm_flush_needed.load(std::memory_order_acquire)); + ASSERT_FALSE(list.IsFlushPending()); + ASSERT_FALSE(list.HasFlushRequested()); + + // Pick tables to flush. The tables to pick must have ID smaller than or + // equal to 5. Therefore, only tables[5] will be selected. + memtable_id = 5; + list.FlushRequested(); + list.PickMemtablesToFlush(memtable_id, &to_flush5); + ASSERT_EQ(1, static_cast(to_flush5.size())); + ASSERT_EQ(1, list.NumNotFlushed()); + ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire)); + ASSERT_FALSE(list.IsFlushPending()); + to_delete.clear(); + + list.current()->Unref(&to_delete); + int to_delete_size = + std::min(num_tables, static_cast(max_write_buffer_size_to_maintain) / + static_cast(options.write_buffer_size)); + ASSERT_EQ(to_delete_size, to_delete.size()); + + for (const auto& m : to_delete) { + // Refcount should be 0 after calling TryInstallMemtableFlushResults. + // Verify this, by Ref'ing then UnRef'ing: + m->Ref(); + ASSERT_EQ(m, m->Unref()); + delete m; + } + to_delete.clear(); +} + +TEST_F(MemTableListTest, EmptyAtomicFlushTest) { + autovector lists; + autovector cf_ids; + autovector options_list; + autovector*> to_flush; + autovector to_delete; + Status s = Mock_InstallMemtableAtomicFlushResults(lists, cf_ids, options_list, + to_flush, &to_delete); + ASSERT_OK(s); + ASSERT_TRUE(to_delete.empty()); +} + +TEST_F(MemTableListTest, AtomicFlushTest) { + const int num_cfs = 3; + const int num_tables_per_cf = 2; + SequenceNumber seq = 1; + + auto factory = std::make_shared(); + options.memtable_factory = factory; + ImmutableOptions ioptions(options); + InternalKeyComparator cmp(BytewiseComparator()); + WriteBufferManager wb(options.db_write_buffer_size); + + // Create MemTableLists + int min_write_buffer_number_to_merge = 3; + int max_write_buffer_number_to_maintain = 7; + int64_t max_write_buffer_size_to_maintain = + 7 * static_cast(options.write_buffer_size); + autovector lists; + for (int i = 0; i != num_cfs; ++i) { + lists.emplace_back(new MemTableList(min_write_buffer_number_to_merge, + max_write_buffer_number_to_maintain, + max_write_buffer_size_to_maintain)); + } + + autovector cf_ids; + std::vector> tables(num_cfs); + autovector mutable_cf_options_list; + uint32_t cf_id = 0; + for (auto& elem : tables) { + mutable_cf_options_list.emplace_back(new MutableCFOptions(options)); + uint64_t memtable_id = 0; + for (int i = 0; i != num_tables_per_cf; ++i) { + MemTable* mem = + new MemTable(cmp, ioptions, *(mutable_cf_options_list.back()), &wb, + kMaxSequenceNumber, cf_id); + mem->SetID(memtable_id++); + mem->Ref(); + + std::string value; + + ASSERT_OK(mem->Add(++seq, kTypeValue, "key1", std::to_string(i), + nullptr /* kv_prot_info */)); + ASSERT_OK(mem->Add(++seq, kTypeValue, "keyN" + std::to_string(i), + "valueN", nullptr /* kv_prot_info */)); + ASSERT_OK(mem->Add(++seq, kTypeValue, "keyX" + std::to_string(i), "value", + nullptr /* kv_prot_info */)); + ASSERT_OK(mem->Add(++seq, kTypeValue, "keyM" + std::to_string(i), + "valueM", nullptr /* kv_prot_info */)); + ASSERT_OK(mem->Add(++seq, kTypeDeletion, "keyX" + std::to_string(i), "", + nullptr /* kv_prot_info */)); + + elem.push_back(mem); + } + cf_ids.push_back(cf_id++); + } + + std::vector> flush_candidates(num_cfs); + + // Nothing to flush + for (auto i = 0; i != num_cfs; ++i) { + auto* list = lists[i]; + ASSERT_FALSE(list->IsFlushPending()); + ASSERT_FALSE(list->imm_flush_needed.load(std::memory_order_acquire)); + list->PickMemtablesToFlush( + std::numeric_limits::max() /* memtable_id */, + &flush_candidates[i]); + ASSERT_EQ(0, flush_candidates[i].size()); + } + // Request flush even though there is nothing to flush + for (auto i = 0; i != num_cfs; ++i) { + auto* list = lists[i]; + list->FlushRequested(); + ASSERT_FALSE(list->IsFlushPending()); + ASSERT_FALSE(list->imm_flush_needed.load(std::memory_order_acquire)); + } + autovector to_delete; + // Add tables to the immutable memtalbe lists associated with column families + for (auto i = 0; i != num_cfs; ++i) { + for (auto j = 0; j != num_tables_per_cf; ++j) { + lists[i]->Add(tables[i][j], &to_delete); + } + ASSERT_EQ(num_tables_per_cf, lists[i]->NumNotFlushed()); + ASSERT_TRUE(lists[i]->IsFlushPending()); + ASSERT_TRUE(lists[i]->imm_flush_needed.load(std::memory_order_acquire)); + } + std::vector flush_memtable_ids = {1, 1, 0}; + // +----+ + // list[0]: |0 1| + // list[1]: |0 1| + // | +--+ + // list[2]: |0| 1 + // +-+ + // Pick memtables to flush + for (auto i = 0; i != num_cfs; ++i) { + flush_candidates[i].clear(); + lists[i]->PickMemtablesToFlush(flush_memtable_ids[i], &flush_candidates[i]); + ASSERT_EQ(flush_memtable_ids[i] - 0 + 1, + static_cast(flush_candidates[i].size())); + } + autovector tmp_lists; + autovector tmp_cf_ids; + autovector tmp_options_list; + autovector*> to_flush; + for (auto i = 0; i != num_cfs; ++i) { + if (!flush_candidates[i].empty()) { + to_flush.push_back(&flush_candidates[i]); + tmp_lists.push_back(lists[i]); + tmp_cf_ids.push_back(i); + tmp_options_list.push_back(mutable_cf_options_list[i]); + } + } + Status s = Mock_InstallMemtableAtomicFlushResults( + tmp_lists, tmp_cf_ids, tmp_options_list, to_flush, &to_delete); + ASSERT_OK(s); + + for (auto i = 0; i != num_cfs; ++i) { + for (auto j = 0; j != num_tables_per_cf; ++j) { + if (static_cast(j) <= flush_memtable_ids[i]) { + ASSERT_LT(0, tables[i][j]->GetFileNumber()); + } + } + ASSERT_EQ( + static_cast(num_tables_per_cf) - flush_candidates[i].size(), + lists[i]->NumNotFlushed()); + } + + to_delete.clear(); + for (auto list : lists) { + list->current()->Unref(&to_delete); + delete list; + } + for (auto& mutable_cf_options : mutable_cf_options_list) { + if (mutable_cf_options != nullptr) { + delete mutable_cf_options; + mutable_cf_options = nullptr; + } + } + // All memtables in tables array must have been flushed, thus ready to be + // deleted. + ASSERT_EQ(to_delete.size(), tables.size() * tables.front().size()); + for (const auto& m : to_delete) { + // Refcount should be 0 after calling InstallMemtableFlushResults. + // Verify this by Ref'ing and then Unref'ing. + m->Ref(); + ASSERT_EQ(m, m->Unref()); + delete m; + } +} + +class MemTableListWithTimestampTest : public MemTableListTest { + public: + MemTableListWithTimestampTest() : MemTableListTest() {} + + void SetUp() override { udt_enabled_ = true; } +}; + +TEST_F(MemTableListWithTimestampTest, GetTableNewestUDT) { + const int num_tables = 3; + const int num_entries = 5; + SequenceNumber seq = 1; + + auto factory = std::make_shared(); + options.memtable_factory = factory; + options.persist_user_defined_timestamps = false; + ImmutableOptions ioptions(options); + const Comparator* ucmp = test::BytewiseComparatorWithU64TsWrapper(); + InternalKeyComparator cmp(ucmp); + WriteBufferManager wb(options.db_write_buffer_size); + + // Create MemTableList + int min_write_buffer_number_to_merge = 1; + int max_write_buffer_number_to_maintain = 4; + int64_t max_write_buffer_size_to_maintain = + 4 * static_cast(options.write_buffer_size); + MemTableList list(min_write_buffer_number_to_merge, + max_write_buffer_number_to_maintain, + max_write_buffer_size_to_maintain); + + // Create some MemTables + uint64_t memtable_id = 0; + std::vector tables; + MutableCFOptions mutable_cf_options(options); + uint64_t current_ts = 0; + autovector to_delete; + std::vector newest_udts; + + std::string key; + std::string write_ts; + for (int i = 0; i < num_tables; i++) { + MemTable* mem = new MemTable(cmp, ioptions, mutable_cf_options, &wb, + kMaxSequenceNumber, 0 /* column_family_id */); + mem->SetID(memtable_id++); + mem->Ref(); + + std::string value; + MergeContext merge_context; + + for (int j = 0; j < num_entries; j++) { + key = "key1"; + write_ts.clear(); + PutFixed64(&write_ts, current_ts); + key.append(write_ts); + ASSERT_OK(mem->Add(++seq, kTypeValue, key, std::to_string(i), + nullptr /* kv_prot_info */)); + current_ts++; + } + + tables.push_back(mem); + list.Add(tables.back(), &to_delete); + newest_udts.push_back(write_ts); + } + + ASSERT_EQ(num_tables, list.NumNotFlushed()); + ASSERT_TRUE(list.IsFlushPending()); + std::vector tables_newest_udts = list.GetTablesNewestUDT(num_tables); + ASSERT_EQ(newest_udts.size(), tables_newest_udts.size()); + for (size_t i = 0; i < tables_newest_udts.size(); i++) { + const Slice& table_newest_udt = tables_newest_udts[i]; + const Slice expected_newest_udt = newest_udts[i]; + ASSERT_EQ(expected_newest_udt, table_newest_udt); + } + + list.current()->Unref(&to_delete); + for (MemTable* m : to_delete) { + delete m; + } + to_delete.clear(); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/merge_context.h b/librocksdb-sys/rocksdb/db/merge_context.h new file mode 100644 index 0000000..8a7b072 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/merge_context.h @@ -0,0 +1,147 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#pragma once +#include +#include +#include +#include + +#include "rocksdb/slice.h" + +namespace ROCKSDB_NAMESPACE { + +const std::vector empty_operand_list; + +// The merge context for merging a user key. +// When doing a Get(), DB will create such a class and pass it when +// issuing Get() operation to memtables and version_set. The operands +// will be fetched from the context when issuing partial of full merge. +class MergeContext { + public: + // Clear all the operands + void Clear() { + if (operand_list_) { + operand_list_->clear(); + copied_operands_->clear(); + } + } + + // Push a merge operand + void PushOperand(const Slice& operand_slice, bool operand_pinned = false) { + Initialize(); + SetDirectionBackward(); + + if (operand_pinned) { + operand_list_->push_back(operand_slice); + } else { + // We need to have our own copy of the operand since it's not pinned + copied_operands_->emplace_back( + new std::string(operand_slice.data(), operand_slice.size())); + operand_list_->push_back(*copied_operands_->back()); + } + } + + // Push back a merge operand + void PushOperandBack(const Slice& operand_slice, + bool operand_pinned = false) { + Initialize(); + SetDirectionForward(); + + if (operand_pinned) { + operand_list_->push_back(operand_slice); + } else { + // We need to have our own copy of the operand since it's not pinned + copied_operands_->emplace_back( + new std::string(operand_slice.data(), operand_slice.size())); + operand_list_->push_back(*copied_operands_->back()); + } + } + + // return total number of operands in the list + size_t GetNumOperands() const { + if (!operand_list_) { + return 0; + } + return operand_list_->size(); + } + + // Get the operand at the index. + Slice GetOperand(int index) const { + assert(operand_list_); + + SetDirectionForward(); + return (*operand_list_)[index]; + } + + // Same as GetOperandsDirectionForward + // + // Note that the returned reference is only good until another call + // to this MergeContext. If the returned value is needed for longer, + // a copy must be made. + const std::vector& GetOperands() const { + return GetOperandsDirectionForward(); + } + + // Return all the operands in the order as they were merged (passed to + // FullMerge or FullMergeV2) + // + // Note that the returned reference is only good until another call + // to this MergeContext. If the returned value is needed for longer, + // a copy must be made. + const std::vector& GetOperandsDirectionForward() const { + if (!operand_list_) { + return empty_operand_list; + } + + SetDirectionForward(); + return *operand_list_; + } + + // Return all the operands in the reversed order relative to how they were + // merged (passed to FullMerge or FullMergeV2) + // + // Note that the returned reference is only good until another call + // to this MergeContext. If the returned value is needed for longer, + // a copy must be made. + const std::vector& GetOperandsDirectionBackward() const { + if (!operand_list_) { + return empty_operand_list; + } + + SetDirectionBackward(); + return *operand_list_; + } + + private: + void Initialize() { + if (!operand_list_) { + operand_list_.reset(new std::vector()); + copied_operands_.reset(new std::vector>()); + } + } + + void SetDirectionForward() const { + if (operands_reversed_ == true) { + std::reverse(operand_list_->begin(), operand_list_->end()); + operands_reversed_ = false; + } + } + + void SetDirectionBackward() const { + if (operands_reversed_ == false) { + std::reverse(operand_list_->begin(), operand_list_->end()); + operands_reversed_ = true; + } + } + + // List of operands + mutable std::unique_ptr> operand_list_; + // Copy of operands that are not pinned. + std::unique_ptr>> copied_operands_; + mutable bool operands_reversed_ = true; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/merge_helper.cc b/librocksdb-sys/rocksdb/db/merge_helper.cc new file mode 100644 index 0000000..110ac96 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/merge_helper.cc @@ -0,0 +1,606 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/merge_helper.h" + +#include + +#include "db/blob/blob_fetcher.h" +#include "db/blob/blob_index.h" +#include "db/blob/prefetch_buffer_collection.h" +#include "db/compaction/compaction_iteration_stats.h" +#include "db/dbformat.h" +#include "db/wide/wide_column_serialization.h" +#include "logging/logging.h" +#include "monitoring/perf_context_imp.h" +#include "monitoring/statistics_impl.h" +#include "port/likely.h" +#include "rocksdb/comparator.h" +#include "rocksdb/db.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/system_clock.h" +#include "table/format.h" +#include "table/internal_iterator.h" + +namespace ROCKSDB_NAMESPACE { + +MergeHelper::MergeHelper(Env* env, const Comparator* user_comparator, + const MergeOperator* user_merge_operator, + const CompactionFilter* compaction_filter, + Logger* logger, bool assert_valid_internal_key, + SequenceNumber latest_snapshot, + const SnapshotChecker* snapshot_checker, int level, + Statistics* stats, + const std::atomic* shutting_down) + : env_(env), + clock_(env->GetSystemClock().get()), + user_comparator_(user_comparator), + user_merge_operator_(user_merge_operator), + compaction_filter_(compaction_filter), + shutting_down_(shutting_down), + logger_(logger), + assert_valid_internal_key_(assert_valid_internal_key), + allow_single_operand_(false), + latest_snapshot_(latest_snapshot), + snapshot_checker_(snapshot_checker), + level_(level), + keys_(), + filter_timer_(clock_), + total_filter_time_(0U), + stats_(stats) { + assert(user_comparator_ != nullptr); + if (user_merge_operator_) { + allow_single_operand_ = user_merge_operator_->AllowSingleOperand(); + } +} + +Status MergeHelper::TimedFullMerge( + const MergeOperator* merge_operator, const Slice& key, const Slice* value, + const std::vector& operands, std::string* result, Logger* logger, + Statistics* statistics, SystemClock* clock, Slice* result_operand, + bool update_num_ops_stats, + MergeOperator::OpFailureScope* op_failure_scope) { + assert(merge_operator != nullptr); + + if (operands.empty()) { + assert(value != nullptr && result != nullptr); + result->assign(value->data(), value->size()); + return Status::OK(); + } + + if (update_num_ops_stats) { + RecordInHistogram(statistics, READ_NUM_MERGE_OPERANDS, + static_cast(operands.size())); + } + + bool success = false; + Slice tmp_result_operand(nullptr, 0); + const MergeOperator::MergeOperationInput merge_in(key, value, operands, + logger); + MergeOperator::MergeOperationOutput merge_out(*result, tmp_result_operand); + { + // Setup to time the merge + StopWatchNano timer(clock, statistics != nullptr); + PERF_TIMER_GUARD(merge_operator_time_nanos); + + // Do the merge + success = merge_operator->FullMergeV2(merge_in, &merge_out); + + if (tmp_result_operand.data()) { + // FullMergeV2 result is an existing operand + if (result_operand != nullptr) { + *result_operand = tmp_result_operand; + } else { + result->assign(tmp_result_operand.data(), tmp_result_operand.size()); + } + } else if (result_operand) { + *result_operand = Slice(nullptr, 0); + } + + RecordTick(statistics, MERGE_OPERATION_TOTAL_TIME, + statistics ? timer.ElapsedNanos() : 0); + } + + if (op_failure_scope != nullptr) { + *op_failure_scope = merge_out.op_failure_scope; + // Apply default per merge_operator.h + if (*op_failure_scope == MergeOperator::OpFailureScope::kDefault) { + *op_failure_scope = MergeOperator::OpFailureScope::kTryMerge; + } + } + + if (!success) { + RecordTick(statistics, NUMBER_MERGE_FAILURES); + return Status::Corruption(Status::SubCode::kMergeOperatorFailed); + } + + return Status::OK(); +} + +Status MergeHelper::TimedFullMergeWithEntity( + const MergeOperator* merge_operator, const Slice& key, Slice base_entity, + const std::vector& operands, std::string* result, Logger* logger, + Statistics* statistics, SystemClock* clock, bool update_num_ops_stats, + MergeOperator::OpFailureScope* op_failure_scope) { + WideColumns base_columns; + + { + const Status s = + WideColumnSerialization::Deserialize(base_entity, base_columns); + if (!s.ok()) { + return s; + } + } + + const bool has_default_column = + !base_columns.empty() && base_columns[0].name() == kDefaultWideColumnName; + + Slice value_of_default; + if (has_default_column) { + value_of_default = base_columns[0].value(); + } + + std::string merge_result; + + { + const Status s = TimedFullMerge(merge_operator, key, &value_of_default, + operands, &merge_result, logger, statistics, + clock, nullptr /* result_operand */, + update_num_ops_stats, op_failure_scope); + if (!s.ok()) { + return s; + } + } + + if (has_default_column) { + base_columns[0].value() = merge_result; + + const Status s = WideColumnSerialization::Serialize(base_columns, *result); + if (!s.ok()) { + return s; + } + } else { + const Status s = + WideColumnSerialization::Serialize(merge_result, base_columns, *result); + if (!s.ok()) { + return s; + } + } + + return Status::OK(); +} + +// PRE: iter points to the first merge type entry +// POST: iter points to the first entry beyond the merge process (or the end) +// keys_, operands_ are updated to reflect the merge result. +// keys_ stores the list of keys encountered while merging. +// operands_ stores the list of merge operands encountered while merging. +// keys_[i] corresponds to operands_[i] for each i. +// +// TODO: Avoid the snapshot stripe map lookup in CompactionRangeDelAggregator +// and just pass the StripeRep corresponding to the stripe being merged. +Status MergeHelper::MergeUntil(InternalIterator* iter, + CompactionRangeDelAggregator* range_del_agg, + const SequenceNumber stop_before, + const bool at_bottom, + const bool allow_data_in_errors, + const BlobFetcher* blob_fetcher, + const std::string* const full_history_ts_low, + PrefetchBufferCollection* prefetch_buffers, + CompactionIterationStats* c_iter_stats) { + // Get a copy of the internal key, before it's invalidated by iter->Next() + // Also maintain the list of merge operands seen. + assert(HasOperator()); + keys_.clear(); + merge_context_.Clear(); + has_compaction_filter_skip_until_ = false; + assert(user_merge_operator_); + assert(user_comparator_); + const size_t ts_sz = user_comparator_->timestamp_size(); + if (full_history_ts_low) { + assert(ts_sz > 0); + assert(ts_sz == full_history_ts_low->size()); + } + bool first_key = true; + + // We need to parse the internal key again as the parsed key is + // backed by the internal key! + // Assume no internal key corruption as it has been successfully parsed + // by the caller. + // original_key_is_iter variable is just caching the information: + // original_key_is_iter == (iter->key().ToString() == original_key) + bool original_key_is_iter = true; + std::string original_key = iter->key().ToString(); + // Important: + // orig_ikey is backed by original_key if keys_.empty() + // orig_ikey is backed by keys_.back() if !keys_.empty() + ParsedInternalKey orig_ikey; + + Status s = ParseInternalKey(original_key, &orig_ikey, allow_data_in_errors); + assert(s.ok()); + if (!s.ok()) return s; + + assert(kTypeMerge == orig_ikey.type); + + bool hit_the_next_user_key = false; + int cmp_with_full_history_ts_low = 0; + for (; iter->Valid(); iter->Next(), original_key_is_iter = false) { + if (IsShuttingDown()) { + s = Status::ShutdownInProgress(); + return s; + } + // Skip range tombstones emitted by the compaction iterator. + if (iter->IsDeleteRangeSentinelKey()) { + continue; + } + + ParsedInternalKey ikey; + assert(keys_.size() == merge_context_.GetNumOperands()); + + Status pik_status = + ParseInternalKey(iter->key(), &ikey, allow_data_in_errors); + Slice ts; + if (pik_status.ok()) { + ts = ExtractTimestampFromUserKey(ikey.user_key, ts_sz); + if (full_history_ts_low) { + cmp_with_full_history_ts_low = + user_comparator_->CompareTimestamp(ts, *full_history_ts_low); + } + } + if (!pik_status.ok()) { + // stop at corrupted key + if (assert_valid_internal_key_) { + return pik_status; + } + break; + } else if (first_key) { + // If user-defined timestamp is enabled, we expect both user key and + // timestamps are equal, as a sanity check. + assert(user_comparator_->Equal(ikey.user_key, orig_ikey.user_key)); + first_key = false; + } else if (!user_comparator_->EqualWithoutTimestamp(ikey.user_key, + orig_ikey.user_key) || + (ts_sz > 0 && + !user_comparator_->Equal(ikey.user_key, orig_ikey.user_key) && + cmp_with_full_history_ts_low >= 0)) { + // 1) hit a different user key, or + // 2) user-defined timestamp is enabled, and hit a version of user key NOT + // eligible for GC, then stop right here. + hit_the_next_user_key = true; + break; + } else if (stop_before > 0 && ikey.sequence <= stop_before && + LIKELY(snapshot_checker_ == nullptr || + snapshot_checker_->CheckInSnapshot(ikey.sequence, + stop_before) != + SnapshotCheckerResult::kNotInSnapshot)) { + // hit an entry that's possibly visible by the previous snapshot, can't + // touch that + break; + } + + // At this point we are guaranteed that we need to process this key. + + assert(IsValueType(ikey.type)); + if (ikey.type != kTypeMerge) { + // hit a put/delete/single delete + // => merge the put value or a nullptr with operands_ + // => store result in operands_.back() (and update keys_.back()) + // => change the entry type to kTypeValue for keys_.back() + // We are done! Success! + + // If there are no operands, just return the Status::OK(). That will cause + // the compaction iterator to write out the key we're currently at, which + // is the put/delete we just encountered. + if (keys_.empty()) { + return s; + } + + // TODO: if we're in compaction and it's a put, it would be nice to run + // compaction filter on it. + std::string merge_result; + MergeOperator::OpFailureScope op_failure_scope; + + if (range_del_agg && + range_del_agg->ShouldDelete( + ikey, RangeDelPositioningMode::kForwardTraversal)) { + s = TimedFullMerge(user_merge_operator_, ikey.user_key, nullptr, + merge_context_.GetOperands(), &merge_result, logger_, + stats_, clock_, + /* result_operand */ nullptr, + /* update_num_ops_stats */ false, &op_failure_scope); + } else if (ikey.type == kTypeValue) { + const Slice val = iter->value(); + + s = TimedFullMerge(user_merge_operator_, ikey.user_key, &val, + merge_context_.GetOperands(), &merge_result, logger_, + stats_, clock_, + /* result_operand */ nullptr, + /* update_num_ops_stats */ false, &op_failure_scope); + } else if (ikey.type == kTypeBlobIndex) { + BlobIndex blob_index; + + s = blob_index.DecodeFrom(iter->value()); + if (!s.ok()) { + return s; + } + + FilePrefetchBuffer* prefetch_buffer = + prefetch_buffers ? prefetch_buffers->GetOrCreatePrefetchBuffer( + blob_index.file_number()) + : nullptr; + + uint64_t bytes_read = 0; + + assert(blob_fetcher); + + PinnableSlice blob_value; + s = blob_fetcher->FetchBlob(ikey.user_key, blob_index, prefetch_buffer, + &blob_value, &bytes_read); + if (!s.ok()) { + return s; + } + + if (c_iter_stats) { + ++c_iter_stats->num_blobs_read; + c_iter_stats->total_blob_bytes_read += bytes_read; + } + + s = TimedFullMerge(user_merge_operator_, ikey.user_key, &blob_value, + merge_context_.GetOperands(), &merge_result, logger_, + stats_, clock_, + /* result_operand */ nullptr, + /* update_num_ops_stats */ false, &op_failure_scope); + } else if (ikey.type == kTypeWideColumnEntity) { + s = TimedFullMergeWithEntity( + user_merge_operator_, ikey.user_key, iter->value(), + merge_context_.GetOperands(), &merge_result, logger_, stats_, + clock_, /* update_num_ops_stats */ false, &op_failure_scope); + } else { + s = TimedFullMerge(user_merge_operator_, ikey.user_key, nullptr, + merge_context_.GetOperands(), &merge_result, logger_, + stats_, clock_, + /* result_operand */ nullptr, + /* update_num_ops_stats */ false, &op_failure_scope); + } + + // We store the result in keys_.back() and operands_.back() + // if nothing went wrong (i.e.: no operand corruption on disk) + if (s.ok()) { + // The original key encountered + original_key = std::move(keys_.back()); + orig_ikey.type = ikey.type == kTypeWideColumnEntity + ? kTypeWideColumnEntity + : kTypeValue; + UpdateInternalKey(&original_key, orig_ikey.sequence, orig_ikey.type); + keys_.clear(); + merge_context_.Clear(); + keys_.emplace_front(std::move(original_key)); + merge_context_.PushOperand(merge_result); + + // move iter to the next entry + iter->Next(); + } else if (op_failure_scope == + MergeOperator::OpFailureScope::kMustMerge) { + // Change to `Status::MergeInProgress()` to denote output consists of + // merge operands only. Leave `iter` at the non-merge entry so it will + // be output after. + s = Status::MergeInProgress(); + } + return s; + } else { + // hit a merge + // => if there is a compaction filter, apply it. + // => check for range tombstones covering the operand + // => merge the operand into the front of the operands_ list + // if not filtered + // => then continue because we haven't yet seen a Put/Delete. + // + // Keep queuing keys and operands until we either meet a put / delete + // request or later did a partial merge. + + Slice value_slice = iter->value(); + // add an operand to the list if: + // 1) it's included in one of the snapshots. in that case we *must* write + // it out, no matter what compaction filter says + // 2) it's not filtered by a compaction filter + CompactionFilter::Decision filter = + ikey.sequence <= latest_snapshot_ + ? CompactionFilter::Decision::kKeep + : FilterMerge(orig_ikey.user_key, value_slice); + if (filter != CompactionFilter::Decision::kRemoveAndSkipUntil && + range_del_agg != nullptr && + range_del_agg->ShouldDelete( + iter->key(), RangeDelPositioningMode::kForwardTraversal)) { + filter = CompactionFilter::Decision::kRemove; + } + if (filter == CompactionFilter::Decision::kKeep || + filter == CompactionFilter::Decision::kChangeValue) { + if (original_key_is_iter) { + // this is just an optimization that saves us one memcpy + keys_.emplace_front(original_key); + } else { + keys_.emplace_front(iter->key().ToString()); + } + if (keys_.size() == 1) { + // we need to re-anchor the orig_ikey because it was anchored by + // original_key before + pik_status = + ParseInternalKey(keys_.back(), &orig_ikey, allow_data_in_errors); + pik_status.PermitUncheckedError(); + assert(pik_status.ok()); + } + if (filter == CompactionFilter::Decision::kKeep) { + merge_context_.PushOperand( + value_slice, iter->IsValuePinned() /* operand_pinned */); + } else { + assert(filter == CompactionFilter::Decision::kChangeValue); + // Compaction filter asked us to change the operand from value_slice + // to compaction_filter_value_. + merge_context_.PushOperand(compaction_filter_value_, false); + } + } else if (filter == CompactionFilter::Decision::kRemoveAndSkipUntil) { + // Compaction filter asked us to remove this key altogether + // (not just this operand), along with some keys following it. + keys_.clear(); + merge_context_.Clear(); + has_compaction_filter_skip_until_ = true; + return s; + } + } + } + + if (cmp_with_full_history_ts_low >= 0) { + size_t num_merge_operands = merge_context_.GetNumOperands(); + if (ts_sz && num_merge_operands > 1) { + // We do not merge merge operands with different timestamps if they are + // not eligible for GC. + ROCKS_LOG_ERROR(logger_, "ts_sz=%d, %d merge oprands", + static_cast(ts_sz), + static_cast(num_merge_operands)); + assert(false); + } + } + + if (merge_context_.GetNumOperands() == 0) { + // we filtered out all the merge operands + return s; + } + + // We are sure we have seen this key's entire history if: + // at_bottom == true (this does not necessarily mean it is the bottommost + // layer, but rather that we are confident the key does not appear on any of + // the lower layers, at_bottom == false doesn't mean it does appear, just + // that we can't be sure, see Compaction::IsBottommostLevel for details) + // AND + // we have either encountered another key or end of key history on this + // layer. + // Note that if user-defined timestamp is enabled, we need some extra caution + // here: if full_history_ts_low is nullptr, or it's not null but the key's + // timestamp is greater than or equal to full_history_ts_low, it means this + // key cannot be dropped. We may not have seen the beginning of the key. + // + // When these conditions are true we are able to merge all the keys + // using full merge. + // + // For these cases we are not sure about, we simply miss the opportunity + // to combine the keys. Since VersionSet::SetupOtherInputs() always makes + // sure that all merge-operands on the same level get compacted together, + // this will simply lead to these merge operands moving to the next level. + bool surely_seen_the_beginning = + (hit_the_next_user_key || !iter->Valid()) && at_bottom && + (ts_sz == 0 || cmp_with_full_history_ts_low < 0); + if (surely_seen_the_beginning) { + // do a final merge with nullptr as the existing value and say + // bye to the merge type (it's now converted to a Put) + assert(kTypeMerge == orig_ikey.type); + assert(merge_context_.GetNumOperands() >= 1); + assert(merge_context_.GetNumOperands() == keys_.size()); + std::string merge_result; + MergeOperator::OpFailureScope op_failure_scope; + s = TimedFullMerge(user_merge_operator_, orig_ikey.user_key, nullptr, + merge_context_.GetOperands(), &merge_result, logger_, + stats_, clock_, + /* result_operand */ nullptr, + /* update_num_ops_stats */ false, &op_failure_scope); + if (s.ok()) { + // The original key encountered + // We are certain that keys_ is not empty here (see assertions couple of + // lines before). + original_key = std::move(keys_.back()); + orig_ikey.type = kTypeValue; + UpdateInternalKey(&original_key, orig_ikey.sequence, orig_ikey.type); + keys_.clear(); + merge_context_.Clear(); + keys_.emplace_front(std::move(original_key)); + merge_context_.PushOperand(merge_result); + } else if (op_failure_scope == MergeOperator::OpFailureScope::kMustMerge) { + // Change to `Status::MergeInProgress()` to denote output consists of + // merge operands only. + s = Status::MergeInProgress(); + } + } else { + // We haven't seen the beginning of the key nor a Put/Delete. + // Attempt to use the user's associative merge function to + // merge the stacked merge operands into a single operand. + s = Status::MergeInProgress(); + if (merge_context_.GetNumOperands() >= 2 || + (allow_single_operand_ && merge_context_.GetNumOperands() == 1)) { + bool merge_success = false; + std::string merge_result; + { + StopWatchNano timer(clock_, stats_ != nullptr); + PERF_TIMER_GUARD(merge_operator_time_nanos); + merge_success = user_merge_operator_->PartialMergeMulti( + orig_ikey.user_key, + std::deque(merge_context_.GetOperands().begin(), + merge_context_.GetOperands().end()), + &merge_result, logger_); + RecordTick(stats_, MERGE_OPERATION_TOTAL_TIME, + stats_ ? timer.ElapsedNanosSafe() : 0); + } + if (merge_success) { + // Merging of operands (associative merge) was successful. + // Replace operands with the merge result + merge_context_.Clear(); + merge_context_.PushOperand(merge_result); + keys_.erase(keys_.begin(), keys_.end() - 1); + } + } + } + + return s; +} + +MergeOutputIterator::MergeOutputIterator(const MergeHelper* merge_helper) + : merge_helper_(merge_helper) { + it_keys_ = merge_helper_->keys().rend(); + it_values_ = merge_helper_->values().rend(); +} + +void MergeOutputIterator::SeekToFirst() { + const auto& keys = merge_helper_->keys(); + const auto& values = merge_helper_->values(); + assert(keys.size() == values.size()); + it_keys_ = keys.rbegin(); + it_values_ = values.rbegin(); +} + +void MergeOutputIterator::Next() { + ++it_keys_; + ++it_values_; +} + +CompactionFilter::Decision MergeHelper::FilterMerge(const Slice& user_key, + const Slice& value_slice) { + if (compaction_filter_ == nullptr) { + return CompactionFilter::Decision::kKeep; + } + if (stats_ != nullptr && ShouldReportDetailedTime(env_, stats_)) { + filter_timer_.Start(); + } + compaction_filter_value_.clear(); + compaction_filter_skip_until_.Clear(); + auto ret = compaction_filter_->FilterV3( + level_, user_key, CompactionFilter::ValueType::kMergeOperand, + &value_slice, /* existing_columns */ nullptr, &compaction_filter_value_, + /* new_columns */ nullptr, compaction_filter_skip_until_.rep()); + if (ret == CompactionFilter::Decision::kRemoveAndSkipUntil) { + if (user_comparator_->Compare(*compaction_filter_skip_until_.rep(), + user_key) <= 0) { + // Invalid skip_until returned from compaction filter. + // Keep the key as per FilterV2/FilterV3 documentation. + ret = CompactionFilter::Decision::kKeep; + } else { + compaction_filter_skip_until_.ConvertFromUserKey(kMaxSequenceNumber, + kValueTypeForSeek); + } + } + if (stats_ != nullptr && ShouldReportDetailedTime(env_, stats_)) { + total_filter_time_ += filter_timer_.ElapsedNanosSafe(); + } + return ret; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/merge_helper.h b/librocksdb-sys/rocksdb/db/merge_helper.h new file mode 100644 index 0000000..7f624b7 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/merge_helper.h @@ -0,0 +1,224 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#pragma once + +#include +#include +#include + +#include "db/merge_context.h" +#include "db/range_del_aggregator.h" +#include "db/snapshot_checker.h" +#include "rocksdb/compaction_filter.h" +#include "rocksdb/env.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/slice.h" +#include "rocksdb/wide_columns.h" +#include "util/stop_watch.h" + +namespace ROCKSDB_NAMESPACE { + +class Comparator; +class Iterator; +class Logger; +class MergeOperator; +class Statistics; +class SystemClock; +class BlobFetcher; +class PrefetchBufferCollection; +struct CompactionIterationStats; + +class MergeHelper { + public: + MergeHelper(Env* env, const Comparator* user_comparator, + const MergeOperator* user_merge_operator, + const CompactionFilter* compaction_filter, Logger* logger, + bool assert_valid_internal_key, SequenceNumber latest_snapshot, + const SnapshotChecker* snapshot_checker = nullptr, int level = 0, + Statistics* stats = nullptr, + const std::atomic* shutting_down = nullptr); + + // Wrapper around MergeOperator::FullMergeV2() that records perf statistics. + // Result of merge will be written to result if status returned is OK. + // If operands is empty, the value will simply be copied to result. + // Set `update_num_ops_stats` to true if it is from a user read, so that + // the latency is sensitive. + // Returns one of the following statuses: + // - OK: Entries were successfully merged. + // - Corruption: Merge operator reported unsuccessful merge. The scope of the + // damage will be stored in `*op_failure_scope` when `op_failure_scope` is + // not nullptr + static Status TimedFullMerge(const MergeOperator* merge_operator, + const Slice& key, const Slice* value, + const std::vector& operands, + std::string* result, Logger* logger, + Statistics* statistics, SystemClock* clock, + Slice* result_operand, bool update_num_ops_stats, + MergeOperator::OpFailureScope* op_failure_scope); + + static Status TimedFullMergeWithEntity( + const MergeOperator* merge_operator, const Slice& key, Slice base_entity, + const std::vector& operands, std::string* result, Logger* logger, + Statistics* statistics, SystemClock* clock, bool update_num_ops_stats, + MergeOperator::OpFailureScope* op_failure_scope); + + // During compaction, merge entries until we hit + // - a corrupted key + // - a Put/Delete, + // - a different user key, + // - a specific sequence number (snapshot boundary), + // - REMOVE_AND_SKIP_UNTIL returned from compaction filter, + // or - the end of iteration + // + // The result(s) of the merge can be accessed in `MergeHelper::keys()` and + // `MergeHelper::values()`, which are invalidated the next time `MergeUntil()` + // is called. `MergeOutputIterator` is specially designed to iterate the + // results of a `MergeHelper`'s most recent `MergeUntil()`. + // + // iter: (IN) points to the first merge type entry + // (OUT) points to the first entry not included in the merge process + // range_del_agg: (IN) filters merge operands covered by range tombstones. + // stop_before: (IN) a sequence number that merge should not cross. + // 0 means no restriction + // at_bottom: (IN) true if the iterator covers the bottem level, which means + // we could reach the start of the history of this user key. + // allow_data_in_errors: (IN) if true, data details will be displayed in + // error/log messages. + // blob_fetcher: (IN) blob fetcher object for the compaction's input version. + // prefetch_buffers: (IN/OUT) a collection of blob file prefetch buffers + // used for compaction readahead. + // c_iter_stats: (OUT) compaction iteration statistics. + // + // Returns one of the following statuses: + // - OK: Entries were successfully merged. + // - MergeInProgress: Output consists of merge operands only. + // - Corruption: Merge operator reported unsuccessful merge or a corrupted + // key has been encountered and not expected (applies only when compiling + // with asserts removed). + // - ShutdownInProgress: interrupted by shutdown (*shutting_down == true). + // + // REQUIRED: The first key in the input is not corrupted. + Status MergeUntil(InternalIterator* iter, + CompactionRangeDelAggregator* range_del_agg, + const SequenceNumber stop_before, const bool at_bottom, + const bool allow_data_in_errors, + const BlobFetcher* blob_fetcher, + const std::string* const full_history_ts_low, + PrefetchBufferCollection* prefetch_buffers, + CompactionIterationStats* c_iter_stats); + + // Filters a merge operand using the compaction filter specified + // in the constructor. Returns the decision that the filter made. + // Uses compaction_filter_value_ and compaction_filter_skip_until_ for the + // optional outputs of compaction filter. + // user_key includes timestamp if user-defined timestamp is enabled. + CompactionFilter::Decision FilterMerge(const Slice& user_key, + const Slice& value_slice); + + // Query the merge result + // These are valid until the next MergeUntil call + // If the merging was successful: + // - keys() contains a single element with the latest sequence number of + // the merges. The type will be Put or Merge. See IMPORTANT 1 note, below. + // - values() contains a single element with the result of merging all the + // operands together + // + // IMPORTANT 1: the key type could change after the MergeUntil call. + // Put/Delete + Merge + ... + Merge => Put + // Merge + ... + Merge => Merge + // + // If the merge operator is not associative, and if a Put/Delete is not found + // then the merging will be unsuccessful. In this case: + // - keys() contains the list of internal keys seen in order of iteration. + // - values() contains the list of values (merges) seen in the same order. + // values() is parallel to keys() so that the first entry in + // keys() is the key associated with the first entry in values() + // and so on. These lists will be the same length. + // All of these pairs will be merges over the same user key. + // See IMPORTANT 2 note below. + // + // IMPORTANT 2: The entries were traversed in order from BACK to FRONT. + // So keys().back() was the first key seen by iterator. + // TODO: Re-style this comment to be like the first one + const std::deque& keys() const { return keys_; } + const std::vector& values() const { + return merge_context_.GetOperands(); + } + uint64_t TotalFilterTime() const { return total_filter_time_; } + bool HasOperator() const { return user_merge_operator_ != nullptr; } + + // If compaction filter returned REMOVE_AND_SKIP_UNTIL, this method will + // return true and fill *until with the key to which we should skip. + // If true, keys() and values() are empty. + bool FilteredUntil(Slice* skip_until) const { + if (!has_compaction_filter_skip_until_) { + return false; + } + assert(compaction_filter_ != nullptr); + assert(skip_until != nullptr); + assert(compaction_filter_skip_until_.Valid()); + *skip_until = compaction_filter_skip_until_.Encode(); + return true; + } + + private: + Env* env_; + SystemClock* clock_; + const Comparator* user_comparator_; + const MergeOperator* user_merge_operator_; + const CompactionFilter* compaction_filter_; + const std::atomic* shutting_down_; + Logger* logger_; + bool assert_valid_internal_key_; // enforce no internal key corruption? + bool allow_single_operand_; + SequenceNumber latest_snapshot_; + const SnapshotChecker* const snapshot_checker_; + int level_; + + // the scratch area that holds the result of MergeUntil + // valid up to the next MergeUntil call + + // Keeps track of the sequence of keys seen + std::deque keys_; + // Parallel with keys_; stores the operands + mutable MergeContext merge_context_; + + StopWatchNano filter_timer_; + uint64_t total_filter_time_; + Statistics* stats_; + + bool has_compaction_filter_skip_until_ = false; + std::string compaction_filter_value_; + InternalKey compaction_filter_skip_until_; + + bool IsShuttingDown() { + // This is a best-effort facility, so memory_order_relaxed is sufficient. + return shutting_down_ && shutting_down_->load(std::memory_order_relaxed); + } +}; + +// MergeOutputIterator can be used to iterate over the result of a merge. +class MergeOutputIterator { + public: + // The MergeOutputIterator is bound to a MergeHelper instance. + explicit MergeOutputIterator(const MergeHelper* merge_helper); + + // Seeks to the first record in the output. + void SeekToFirst(); + // Advances to the next record in the output. + void Next(); + + Slice key() { return Slice(*it_keys_); } + Slice value() { return Slice(*it_values_); } + bool Valid() { return it_keys_ != merge_helper_->keys().rend(); } + + private: + const MergeHelper* merge_helper_; + std::deque::const_reverse_iterator it_keys_; + std::vector::const_reverse_iterator it_values_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/merge_helper_test.cc b/librocksdb-sys/rocksdb/db/merge_helper_test.cc new file mode 100644 index 0000000..05408d5 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/merge_helper_test.cc @@ -0,0 +1,298 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/merge_helper.h" + +#include +#include +#include + +#include "db/dbformat.h" +#include "rocksdb/comparator.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/coding.h" +#include "util/vector_iterator.h" +#include "utilities/merge_operators.h" + +namespace ROCKSDB_NAMESPACE { + +class MergeHelperTest : public testing::Test { + public: + MergeHelperTest() : icmp_(BytewiseComparator()) { env_ = Env::Default(); } + + ~MergeHelperTest() override = default; + + Status Run(SequenceNumber stop_before, bool at_bottom, + SequenceNumber latest_snapshot = 0) { + iter_.reset(new VectorIterator(ks_, vs_, &icmp_)); + iter_->SeekToFirst(); + merge_helper_.reset(new MergeHelper(env_, icmp_.user_comparator(), + merge_op_.get(), filter_.get(), nullptr, + false, latest_snapshot)); + return merge_helper_->MergeUntil( + iter_.get(), nullptr /* range_del_agg */, stop_before, at_bottom, + false /* allow_data_in_errors */, nullptr /* blob_fetcher */, + nullptr /* full_history_ts_low */, nullptr /* prefetch_buffers */, + nullptr /* c_iter_stats */); + } + + void AddKeyVal(const std::string& user_key, const SequenceNumber& seq, + const ValueType& t, const std::string& val, + bool corrupt = false) { + InternalKey ikey(user_key, seq, t); + if (corrupt) { + test::CorruptKeyType(&ikey); + } + ks_.push_back(ikey.Encode().ToString()); + vs_.push_back(val); + } + + Env* env_; + InternalKeyComparator icmp_; + std::unique_ptr iter_; + std::shared_ptr merge_op_; + std::unique_ptr merge_helper_; + std::vector ks_; + std::vector vs_; + std::unique_ptr filter_; +}; + +// If MergeHelper encounters a new key on the last level, we know that +// the key has no more history and it can merge keys. +TEST_F(MergeHelperTest, MergeAtBottomSuccess) { + merge_op_ = MergeOperators::CreateUInt64AddOperator(); + + AddKeyVal("a", 20, kTypeMerge, test::EncodeInt(1U)); + AddKeyVal("a", 10, kTypeMerge, test::EncodeInt(3U)); + AddKeyVal("b", 10, kTypeMerge, test::EncodeInt(4U)); // <- iter_ after merge + + ASSERT_TRUE(Run(0, true).ok()); + ASSERT_EQ(ks_[2], iter_->key()); + ASSERT_EQ(test::KeyStr("a", 20, kTypeValue), merge_helper_->keys()[0]); + ASSERT_EQ(test::EncodeInt(4U), merge_helper_->values()[0]); + ASSERT_EQ(1U, merge_helper_->keys().size()); + ASSERT_EQ(1U, merge_helper_->values().size()); +} + +// Merging with a value results in a successful merge. +TEST_F(MergeHelperTest, MergeValue) { + merge_op_ = MergeOperators::CreateUInt64AddOperator(); + + AddKeyVal("a", 40, kTypeMerge, test::EncodeInt(1U)); + AddKeyVal("a", 30, kTypeMerge, test::EncodeInt(3U)); + AddKeyVal("a", 20, kTypeValue, test::EncodeInt(4U)); // <- iter_ after merge + AddKeyVal("a", 10, kTypeMerge, test::EncodeInt(1U)); + + ASSERT_TRUE(Run(0, false).ok()); + ASSERT_EQ(ks_[3], iter_->key()); + ASSERT_EQ(test::KeyStr("a", 40, kTypeValue), merge_helper_->keys()[0]); + ASSERT_EQ(test::EncodeInt(8U), merge_helper_->values()[0]); + ASSERT_EQ(1U, merge_helper_->keys().size()); + ASSERT_EQ(1U, merge_helper_->values().size()); +} + +// Merging stops before a snapshot. +TEST_F(MergeHelperTest, SnapshotBeforeValue) { + merge_op_ = MergeOperators::CreateUInt64AddOperator(); + + AddKeyVal("a", 50, kTypeMerge, test::EncodeInt(1U)); + AddKeyVal("a", 40, kTypeMerge, test::EncodeInt(3U)); // <- iter_ after merge + AddKeyVal("a", 30, kTypeMerge, test::EncodeInt(1U)); + AddKeyVal("a", 20, kTypeValue, test::EncodeInt(4U)); + AddKeyVal("a", 10, kTypeMerge, test::EncodeInt(1U)); + + ASSERT_TRUE(Run(31, true).IsMergeInProgress()); + ASSERT_EQ(ks_[2], iter_->key()); + ASSERT_EQ(test::KeyStr("a", 50, kTypeMerge), merge_helper_->keys()[0]); + ASSERT_EQ(test::EncodeInt(4U), merge_helper_->values()[0]); + ASSERT_EQ(1U, merge_helper_->keys().size()); + ASSERT_EQ(1U, merge_helper_->values().size()); +} + +// MergeHelper preserves the operand stack for merge operators that +// cannot do a partial merge. +TEST_F(MergeHelperTest, NoPartialMerge) { + merge_op_ = MergeOperators::CreateStringAppendTESTOperator(); + + AddKeyVal("a", 50, kTypeMerge, "v2"); + AddKeyVal("a", 40, kTypeMerge, "v"); // <- iter_ after merge + AddKeyVal("a", 30, kTypeMerge, "v"); + + ASSERT_TRUE(Run(31, true).IsMergeInProgress()); + ASSERT_EQ(ks_[2], iter_->key()); + ASSERT_EQ(test::KeyStr("a", 40, kTypeMerge), merge_helper_->keys()[0]); + ASSERT_EQ("v", merge_helper_->values()[0]); + ASSERT_EQ(test::KeyStr("a", 50, kTypeMerge), merge_helper_->keys()[1]); + ASSERT_EQ("v2", merge_helper_->values()[1]); + ASSERT_EQ(2U, merge_helper_->keys().size()); + ASSERT_EQ(2U, merge_helper_->values().size()); +} + +// A single operand can not be merged. +TEST_F(MergeHelperTest, SingleOperand) { + merge_op_ = MergeOperators::CreateUInt64AddOperator(); + + AddKeyVal("a", 50, kTypeMerge, test::EncodeInt(1U)); + + ASSERT_TRUE(Run(31, false).IsMergeInProgress()); + ASSERT_FALSE(iter_->Valid()); + ASSERT_EQ(test::KeyStr("a", 50, kTypeMerge), merge_helper_->keys()[0]); + ASSERT_EQ(test::EncodeInt(1U), merge_helper_->values()[0]); + ASSERT_EQ(1U, merge_helper_->keys().size()); + ASSERT_EQ(1U, merge_helper_->values().size()); +} + +// Merging with a deletion turns the deletion into a value +TEST_F(MergeHelperTest, MergeDeletion) { + merge_op_ = MergeOperators::CreateUInt64AddOperator(); + + AddKeyVal("a", 30, kTypeMerge, test::EncodeInt(3U)); + AddKeyVal("a", 20, kTypeDeletion, ""); + + ASSERT_TRUE(Run(15, false).ok()); + ASSERT_FALSE(iter_->Valid()); + ASSERT_EQ(test::KeyStr("a", 30, kTypeValue), merge_helper_->keys()[0]); + ASSERT_EQ(test::EncodeInt(3U), merge_helper_->values()[0]); + ASSERT_EQ(1U, merge_helper_->keys().size()); + ASSERT_EQ(1U, merge_helper_->values().size()); +} + +// The merge helper stops upon encountering a corrupt key +TEST_F(MergeHelperTest, CorruptKey) { + merge_op_ = MergeOperators::CreateUInt64AddOperator(); + + AddKeyVal("a", 30, kTypeMerge, test::EncodeInt(3U)); + AddKeyVal("a", 25, kTypeMerge, test::EncodeInt(1U)); + // Corrupt key + AddKeyVal("a", 20, kTypeDeletion, "", true); // <- iter_ after merge + + ASSERT_TRUE(Run(15, false).IsMergeInProgress()); + ASSERT_EQ(ks_[2], iter_->key()); + ASSERT_EQ(test::KeyStr("a", 30, kTypeMerge), merge_helper_->keys()[0]); + ASSERT_EQ(test::EncodeInt(4U), merge_helper_->values()[0]); + ASSERT_EQ(1U, merge_helper_->keys().size()); + ASSERT_EQ(1U, merge_helper_->values().size()); +} + +// The compaction filter is called on every merge operand +TEST_F(MergeHelperTest, FilterMergeOperands) { + merge_op_ = MergeOperators::CreateUInt64AddOperator(); + filter_.reset(new test::FilterNumber(5U)); + + AddKeyVal("a", 30, kTypeMerge, test::EncodeInt(3U)); + AddKeyVal("a", 29, kTypeMerge, test::EncodeInt(5U)); // Filtered + AddKeyVal("a", 28, kTypeMerge, test::EncodeInt(3U)); + AddKeyVal("a", 27, kTypeMerge, test::EncodeInt(1U)); + AddKeyVal("a", 26, kTypeMerge, test::EncodeInt(5U)); // Filtered + AddKeyVal("a", 25, kTypeValue, test::EncodeInt(1U)); + + ASSERT_TRUE(Run(15, false).ok()); + ASSERT_FALSE(iter_->Valid()); + MergeOutputIterator merge_output_iter(merge_helper_.get()); + merge_output_iter.SeekToFirst(); + ASSERT_EQ(test::KeyStr("a", 30, kTypeValue), + merge_output_iter.key().ToString()); + ASSERT_EQ(test::EncodeInt(8U), merge_output_iter.value().ToString()); + merge_output_iter.Next(); + ASSERT_FALSE(merge_output_iter.Valid()); +} + +TEST_F(MergeHelperTest, FilterAllMergeOperands) { + merge_op_ = MergeOperators::CreateUInt64AddOperator(); + filter_.reset(new test::FilterNumber(5U)); + + AddKeyVal("a", 30, kTypeMerge, test::EncodeInt(5U)); + AddKeyVal("a", 29, kTypeMerge, test::EncodeInt(5U)); + AddKeyVal("a", 28, kTypeMerge, test::EncodeInt(5U)); + AddKeyVal("a", 27, kTypeMerge, test::EncodeInt(5U)); + AddKeyVal("a", 26, kTypeMerge, test::EncodeInt(5U)); + AddKeyVal("a", 25, kTypeMerge, test::EncodeInt(5U)); + + // filtered out all + ASSERT_TRUE(Run(15, false).ok()); + ASSERT_FALSE(iter_->Valid()); + MergeOutputIterator merge_output_iter(merge_helper_.get()); + merge_output_iter.SeekToFirst(); + ASSERT_FALSE(merge_output_iter.Valid()); + + // we have one operand that will survive because it's a delete + AddKeyVal("a", 24, kTypeDeletion, test::EncodeInt(5U)); + AddKeyVal("b", 23, kTypeValue, test::EncodeInt(5U)); + ASSERT_TRUE(Run(15, true).ok()); + merge_output_iter = MergeOutputIterator(merge_helper_.get()); + ASSERT_TRUE(iter_->Valid()); + merge_output_iter.SeekToFirst(); + ASSERT_FALSE(merge_output_iter.Valid()); + + // when all merge operands are filtered out, we leave the iterator pointing to + // the Put/Delete that survived + ASSERT_EQ(test::KeyStr("a", 24, kTypeDeletion), iter_->key().ToString()); + ASSERT_EQ(test::EncodeInt(5U), iter_->value().ToString()); +} + +// Make sure that merge operands are filtered at the beginning +TEST_F(MergeHelperTest, FilterFirstMergeOperand) { + merge_op_ = MergeOperators::CreateUInt64AddOperator(); + filter_.reset(new test::FilterNumber(5U)); + + AddKeyVal("a", 31, kTypeMerge, test::EncodeInt(5U)); // Filtered + AddKeyVal("a", 30, kTypeMerge, test::EncodeInt(5U)); // Filtered + AddKeyVal("a", 29, kTypeMerge, test::EncodeInt(2U)); + AddKeyVal("a", 28, kTypeMerge, test::EncodeInt(1U)); + AddKeyVal("a", 27, kTypeMerge, test::EncodeInt(3U)); + AddKeyVal("a", 26, kTypeMerge, test::EncodeInt(5U)); // Filtered + AddKeyVal("a", 25, kTypeMerge, test::EncodeInt(5U)); // Filtered + AddKeyVal("b", 24, kTypeValue, test::EncodeInt(5U)); // next user key + + ASSERT_OK(Run(15, true)); + ASSERT_TRUE(iter_->Valid()); + MergeOutputIterator merge_output_iter(merge_helper_.get()); + merge_output_iter.SeekToFirst(); + // sequence number is 29 here, because the first merge operand got filtered + // out + ASSERT_EQ(test::KeyStr("a", 29, kTypeValue), + merge_output_iter.key().ToString()); + ASSERT_EQ(test::EncodeInt(6U), merge_output_iter.value().ToString()); + merge_output_iter.Next(); + ASSERT_FALSE(merge_output_iter.Valid()); + + // make sure that we're passing user keys into the filter + ASSERT_EQ("a", filter_->last_merge_operand_key()); +} + +// Make sure that merge operands are not filtered out if there's a snapshot +// pointing at them +TEST_F(MergeHelperTest, DontFilterMergeOperandsBeforeSnapshotTest) { + merge_op_ = MergeOperators::CreateUInt64AddOperator(); + filter_.reset(new test::FilterNumber(5U)); + + AddKeyVal("a", 31, kTypeMerge, test::EncodeInt(5U)); + AddKeyVal("a", 30, kTypeMerge, test::EncodeInt(5U)); + AddKeyVal("a", 29, kTypeMerge, test::EncodeInt(2U)); + AddKeyVal("a", 28, kTypeMerge, test::EncodeInt(1U)); + AddKeyVal("a", 27, kTypeMerge, test::EncodeInt(3U)); + AddKeyVal("a", 26, kTypeMerge, test::EncodeInt(5U)); + AddKeyVal("a", 25, kTypeMerge, test::EncodeInt(5U)); + AddKeyVal("b", 24, kTypeValue, test::EncodeInt(5U)); + + ASSERT_OK(Run(15, true, 32)); + ASSERT_TRUE(iter_->Valid()); + MergeOutputIterator merge_output_iter(merge_helper_.get()); + merge_output_iter.SeekToFirst(); + ASSERT_EQ(test::KeyStr("a", 31, kTypeValue), + merge_output_iter.key().ToString()); + ASSERT_EQ(test::EncodeInt(26U), merge_output_iter.value().ToString()); + merge_output_iter.Next(); + ASSERT_FALSE(merge_output_iter.Valid()); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/merge_operator.cc b/librocksdb-sys/rocksdb/db/merge_operator.cc new file mode 100644 index 0000000..d325856 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/merge_operator.cc @@ -0,0 +1,85 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +/** + * Back-end implementation details specific to the Merge Operator. + */ + +#include "rocksdb/merge_operator.h" + +namespace ROCKSDB_NAMESPACE { + +bool MergeOperator::FullMergeV2(const MergeOperationInput& merge_in, + MergeOperationOutput* merge_out) const { + // If FullMergeV2 is not implemented, we convert the operand_list to + // std::deque and pass it to FullMerge + std::deque operand_list_str; + for (auto& op : merge_in.operand_list) { + operand_list_str.emplace_back(op.data(), op.size()); + } + return FullMerge(merge_in.key, merge_in.existing_value, operand_list_str, + &merge_out->new_value, merge_in.logger); +} + +// The default implementation of PartialMergeMulti, which invokes +// PartialMerge multiple times internally and merges two operands at +// a time. +bool MergeOperator::PartialMergeMulti(const Slice& key, + const std::deque& operand_list, + std::string* new_value, + Logger* logger) const { + assert(operand_list.size() >= 2); + // Simply loop through the operands + Slice temp_slice(operand_list[0]); + + for (size_t i = 1; i < operand_list.size(); ++i) { + auto& operand = operand_list[i]; + std::string temp_value; + if (!PartialMerge(key, temp_slice, operand, &temp_value, logger)) { + return false; + } + swap(temp_value, *new_value); + temp_slice = Slice(*new_value); + } + + // The result will be in *new_value. All merges succeeded. + return true; +} + +// Given a "real" merge from the library, call the user's +// associative merge function one-by-one on each of the operands. +// NOTE: It is assumed that the client's merge-operator will handle any errors. +bool AssociativeMergeOperator::FullMergeV2( + const MergeOperationInput& merge_in, + MergeOperationOutput* merge_out) const { + // Simply loop through the operands + Slice temp_existing; + const Slice* existing_value = merge_in.existing_value; + for (const auto& operand : merge_in.operand_list) { + std::string temp_value; + if (!Merge(merge_in.key, existing_value, operand, &temp_value, + merge_in.logger)) { + return false; + } + swap(temp_value, merge_out->new_value); + temp_existing = Slice(merge_out->new_value); + existing_value = &temp_existing; + } + + // The result will be in *new_value. All merges succeeded. + return true; +} + +// Call the user defined simple merge on the operands; +// NOTE: It is assumed that the client's merge-operator will handle any errors. +bool AssociativeMergeOperator::PartialMerge(const Slice& key, + const Slice& left_operand, + const Slice& right_operand, + std::string* new_value, + Logger* logger) const { + return Merge(key, &left_operand, right_operand, new_value, logger); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/merge_test.cc b/librocksdb-sys/rocksdb/db/merge_test.cc new file mode 100644 index 0000000..6d1333e --- /dev/null +++ b/librocksdb-sys/rocksdb/db/merge_test.cc @@ -0,0 +1,621 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#include + +#include +#include + +#include "db/db_impl/db_impl.h" +#include "db/dbformat.h" +#include "db/write_batch_internal.h" +#include "port/stack_trace.h" +#include "rocksdb/cache.h" +#include "rocksdb/comparator.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/utilities/db_ttl.h" +#include "test_util/testharness.h" +#include "util/coding.h" +#include "utilities/merge_operators.h" + +namespace ROCKSDB_NAMESPACE { + +bool use_compression; + +class MergeTest : public testing::Test {}; + +size_t num_merge_operator_calls; +void resetNumMergeOperatorCalls() { num_merge_operator_calls = 0; } + +size_t num_partial_merge_calls; +void resetNumPartialMergeCalls() { num_partial_merge_calls = 0; } + +class CountMergeOperator : public AssociativeMergeOperator { + public: + CountMergeOperator() { + mergeOperator_ = MergeOperators::CreateUInt64AddOperator(); + } + + bool Merge(const Slice& key, const Slice* existing_value, const Slice& value, + std::string* new_value, Logger* logger) const override { + assert(new_value->empty()); + ++num_merge_operator_calls; + if (existing_value == nullptr) { + new_value->assign(value.data(), value.size()); + return true; + } + + return mergeOperator_->PartialMerge(key, *existing_value, value, new_value, + logger); + } + + bool PartialMergeMulti(const Slice& key, + const std::deque& operand_list, + std::string* new_value, + Logger* logger) const override { + assert(new_value->empty()); + ++num_partial_merge_calls; + return mergeOperator_->PartialMergeMulti(key, operand_list, new_value, + logger); + } + + const char* Name() const override { return "UInt64AddOperator"; } + + private: + std::shared_ptr mergeOperator_; +}; + +class EnvMergeTest : public EnvWrapper { + public: + EnvMergeTest() : EnvWrapper(Env::Default()) {} + static const char* kClassName() { return "MergeEnv"; } + const char* Name() const override { return kClassName(); } + // ~EnvMergeTest() override {} + + uint64_t NowNanos() override { + ++now_nanos_count_; + return target()->NowNanos(); + } + + static uint64_t now_nanos_count_; + + static std::unique_ptr singleton_; + + static EnvMergeTest* GetInstance() { + if (nullptr == singleton_) singleton_.reset(new EnvMergeTest); + return singleton_.get(); + } +}; + +uint64_t EnvMergeTest::now_nanos_count_{0}; +std::unique_ptr EnvMergeTest::singleton_; + +std::shared_ptr OpenDb(const std::string& dbname, const bool ttl = false, + const size_t max_successive_merges = 0) { + DB* db; + Options options; + options.create_if_missing = true; + options.merge_operator = std::make_shared(); + options.max_successive_merges = max_successive_merges; + options.env = EnvMergeTest::GetInstance(); + EXPECT_OK(DestroyDB(dbname, Options())); + Status s; + if (ttl) { + DBWithTTL* db_with_ttl; + s = DBWithTTL::Open(options, dbname, &db_with_ttl); + db = db_with_ttl; + } else { + s = DB::Open(options, dbname, &db); + } + EXPECT_OK(s); + assert(s.ok()); + // Allowed to call NowNanos during DB creation (in GenerateRawUniqueId() for + // session ID) + EnvMergeTest::now_nanos_count_ = 0; + return std::shared_ptr(db); +} + +// Imagine we are maintaining a set of uint64 counters. +// Each counter has a distinct name. And we would like +// to support four high level operations: +// set, add, get and remove +// This is a quick implementation without a Merge operation. +class Counters { + protected: + std::shared_ptr db_; + + WriteOptions put_option_; + ReadOptions get_option_; + WriteOptions delete_option_; + + uint64_t default_; + + public: + explicit Counters(std::shared_ptr db, uint64_t defaultCount = 0) + : db_(db), + put_option_(), + get_option_(), + delete_option_(), + default_(defaultCount) { + assert(db_); + } + + virtual ~Counters() {} + + // public interface of Counters. + // All four functions return false + // if the underlying level db operation failed. + + // mapped to a levedb Put + bool set(const std::string& key, uint64_t value) { + // just treat the internal rep of int64 as the string + char buf[sizeof(value)]; + EncodeFixed64(buf, value); + Slice slice(buf, sizeof(value)); + auto s = db_->Put(put_option_, key, slice); + + if (s.ok()) { + return true; + } else { + std::cerr << s.ToString() << std::endl; + return false; + } + } + + // mapped to a rocksdb Delete + bool remove(const std::string& key) { + auto s = db_->Delete(delete_option_, key); + + if (s.ok()) { + return true; + } else { + std::cerr << s.ToString() << std::endl; + return false; + } + } + + // mapped to a rocksdb Get + bool get(const std::string& key, uint64_t* value) { + std::string str; + auto s = db_->Get(get_option_, key, &str); + + if (s.IsNotFound()) { + // return default value if not found; + *value = default_; + return true; + } else if (s.ok()) { + // deserialization + if (str.size() != sizeof(uint64_t)) { + std::cerr << "value corruption\n"; + return false; + } + *value = DecodeFixed64(&str[0]); + return true; + } else { + std::cerr << s.ToString() << std::endl; + return false; + } + } + + // 'add' is implemented as get -> modify -> set + // An alternative is a single merge operation, see MergeBasedCounters + virtual bool add(const std::string& key, uint64_t value) { + uint64_t base = default_; + return get(key, &base) && set(key, base + value); + } + + // convenience functions for testing + void assert_set(const std::string& key, uint64_t value) { + assert(set(key, value)); + } + + void assert_remove(const std::string& key) { assert(remove(key)); } + + uint64_t assert_get(const std::string& key) { + uint64_t value = default_; + int result = get(key, &value); + assert(result); + if (result == 0) exit(1); // Disable unused variable warning. + return value; + } + + void assert_add(const std::string& key, uint64_t value) { + int result = add(key, value); + assert(result); + if (result == 0) exit(1); // Disable unused variable warning. + } +}; + +// Implement 'add' directly with the new Merge operation +class MergeBasedCounters : public Counters { + private: + WriteOptions merge_option_; // for merge + + public: + explicit MergeBasedCounters(std::shared_ptr db, uint64_t defaultCount = 0) + : Counters(db, defaultCount), merge_option_() {} + + // mapped to a rocksdb Merge operation + bool add(const std::string& key, uint64_t value) override { + char encoded[sizeof(uint64_t)]; + EncodeFixed64(encoded, value); + Slice slice(encoded, sizeof(uint64_t)); + auto s = db_->Merge(merge_option_, key, slice); + + if (s.ok()) { + return true; + } else { + std::cerr << s.ToString() << std::endl; + return false; + } + } +}; + +void dumpDb(DB* db) { + auto it = std::unique_ptr(db->NewIterator(ReadOptions())); + for (it->SeekToFirst(); it->Valid(); it->Next()) { + // uint64_t value = DecodeFixed64(it->value().data()); + // std::cout << it->key().ToString() << ": " << value << std::endl; + } + assert(it->status().ok()); // Check for any errors found during the scan +} + +void testCounters(Counters& counters, DB* db, bool test_compaction) { + FlushOptions o; + o.wait = true; + + counters.assert_set("a", 1); + + if (test_compaction) { + ASSERT_OK(db->Flush(o)); + } + + ASSERT_EQ(counters.assert_get("a"), 1); + + counters.assert_remove("b"); + + // defaut value is 0 if non-existent + ASSERT_EQ(counters.assert_get("b"), 0); + + counters.assert_add("a", 2); + + if (test_compaction) { + ASSERT_OK(db->Flush(o)); + } + + // 1+2 = 3 + ASSERT_EQ(counters.assert_get("a"), 3); + + dumpDb(db); + + // 1+...+49 = ? + uint64_t sum = 0; + for (int i = 1; i < 50; i++) { + counters.assert_add("b", i); + sum += i; + } + ASSERT_EQ(counters.assert_get("b"), sum); + + dumpDb(db); + + if (test_compaction) { + ASSERT_OK(db->Flush(o)); + + ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + dumpDb(db); + + ASSERT_EQ(counters.assert_get("a"), 3); + ASSERT_EQ(counters.assert_get("b"), sum); + } +} + +void testCountersWithFlushAndCompaction(Counters& counters, DB* db) { + ASSERT_OK(db->Put({}, "1", "1")); + ASSERT_OK(db->Flush(FlushOptions())); + + std::atomic cnt{0}; + const auto get_thread_id = [&cnt]() { + thread_local int thread_id{cnt++}; + return thread_id; + }; + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:BeforeWriterWaiting", [&](void* /*arg*/) { + int thread_id = get_thread_id(); + if (1 == thread_id) { + TEST_SYNC_POINT( + "testCountersWithFlushAndCompaction::bg_compact_thread:0"); + } else if (2 == thread_id) { + TEST_SYNC_POINT( + "testCountersWithFlushAndCompaction::bg_flush_thread:0"); + } + }); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:WriteManifest", [&](void* /*arg*/) { + int thread_id = get_thread_id(); + if (0 == thread_id) { + TEST_SYNC_POINT( + "testCountersWithFlushAndCompaction::set_options_thread:0"); + TEST_SYNC_POINT( + "testCountersWithFlushAndCompaction::set_options_thread:1"); + } + }); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:WakeUpAndDone", [&](void* arg) { + auto* mutex = reinterpret_cast(arg); + mutex->AssertHeld(); + int thread_id = get_thread_id(); + ASSERT_EQ(2, thread_id); + mutex->Unlock(); + TEST_SYNC_POINT( + "testCountersWithFlushAndCompaction::bg_flush_thread:1"); + TEST_SYNC_POINT( + "testCountersWithFlushAndCompaction::bg_flush_thread:2"); + mutex->Lock(); + }); + SyncPoint::GetInstance()->LoadDependency({ + {"testCountersWithFlushAndCompaction::set_options_thread:0", + "testCountersWithCompactionAndFlush:BeforeCompact"}, + {"testCountersWithFlushAndCompaction::bg_compact_thread:0", + "testCountersWithFlushAndCompaction:BeforeIncCounters"}, + {"testCountersWithFlushAndCompaction::bg_flush_thread:0", + "testCountersWithFlushAndCompaction::set_options_thread:1"}, + {"testCountersWithFlushAndCompaction::bg_flush_thread:1", + "testCountersWithFlushAndCompaction:BeforeVerification"}, + {"testCountersWithFlushAndCompaction:AfterGet", + "testCountersWithFlushAndCompaction::bg_flush_thread:2"}, + }); + SyncPoint::GetInstance()->EnableProcessing(); + + port::Thread set_options_thread([&]() { + ASSERT_OK(reinterpret_cast(db)->SetOptions( + {{"disable_auto_compactions", "false"}})); + }); + TEST_SYNC_POINT("testCountersWithCompactionAndFlush:BeforeCompact"); + port::Thread compact_thread([&]() { + ASSERT_OK(reinterpret_cast(db)->CompactRange( + CompactRangeOptions(), db->DefaultColumnFamily(), nullptr, nullptr)); + }); + + TEST_SYNC_POINT("testCountersWithFlushAndCompaction:BeforeIncCounters"); + counters.add("test-key", 1); + + FlushOptions flush_opts; + flush_opts.wait = false; + ASSERT_OK(db->Flush(flush_opts)); + + TEST_SYNC_POINT("testCountersWithFlushAndCompaction:BeforeVerification"); + std::string expected; + PutFixed64(&expected, 1); + std::string actual; + Status s = db->Get(ReadOptions(), "test-key", &actual); + TEST_SYNC_POINT("testCountersWithFlushAndCompaction:AfterGet"); + set_options_thread.join(); + compact_thread.join(); + ASSERT_OK(s); + ASSERT_EQ(expected, actual); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +void testSuccessiveMerge(Counters& counters, size_t max_num_merges, + size_t num_merges) { + counters.assert_remove("z"); + uint64_t sum = 0; + + for (size_t i = 1; i <= num_merges; ++i) { + resetNumMergeOperatorCalls(); + counters.assert_add("z", i); + sum += i; + + if (i % (max_num_merges + 1) == 0) { + ASSERT_EQ(num_merge_operator_calls, max_num_merges + 1); + } else { + ASSERT_EQ(num_merge_operator_calls, 0); + } + + resetNumMergeOperatorCalls(); + ASSERT_EQ(counters.assert_get("z"), sum); + ASSERT_EQ(num_merge_operator_calls, i % (max_num_merges + 1)); + } +} + +void testPartialMerge(Counters* counters, DB* db, size_t max_merge, + size_t min_merge, size_t count) { + FlushOptions o; + o.wait = true; + + // Test case 1: partial merge should be called when the number of merge + // operands exceeds the threshold. + uint64_t tmp_sum = 0; + resetNumPartialMergeCalls(); + for (size_t i = 1; i <= count; i++) { + counters->assert_add("b", i); + tmp_sum += i; + } + ASSERT_OK(db->Flush(o)); + ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(tmp_sum, counters->assert_get("b")); + if (count > max_merge) { + // in this case, FullMerge should be called instead. + ASSERT_EQ(num_partial_merge_calls, 0U); + } else { + // if count >= min_merge, then partial merge should be called once. + ASSERT_EQ((count >= min_merge), (num_partial_merge_calls == 1)); + } + + // Test case 2: partial merge should not be called when a put is found. + resetNumPartialMergeCalls(); + tmp_sum = 0; + ASSERT_OK(db->Put(ROCKSDB_NAMESPACE::WriteOptions(), "c", "10")); + for (size_t i = 1; i <= count; i++) { + counters->assert_add("c", i); + tmp_sum += i; + } + ASSERT_OK(db->Flush(o)); + ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(tmp_sum, counters->assert_get("c")); + ASSERT_EQ(num_partial_merge_calls, 0U); + // NowNanos was previously called in MergeHelper::FilterMerge(), which + // harmed performance. + ASSERT_EQ(EnvMergeTest::now_nanos_count_, 0U); +} + +void testSingleBatchSuccessiveMerge(DB* db, size_t max_num_merges, + size_t num_merges) { + ASSERT_GT(num_merges, max_num_merges); + + Slice key("BatchSuccessiveMerge"); + uint64_t merge_value = 1; + char buf[sizeof(merge_value)]; + EncodeFixed64(buf, merge_value); + Slice merge_value_slice(buf, sizeof(merge_value)); + + // Create the batch + WriteBatch batch; + for (size_t i = 0; i < num_merges; ++i) { + ASSERT_OK(batch.Merge(key, merge_value_slice)); + } + + // Apply to memtable and count the number of merges + resetNumMergeOperatorCalls(); + ASSERT_OK(db->Write(WriteOptions(), &batch)); + ASSERT_EQ( + num_merge_operator_calls, + static_cast(num_merges - (num_merges % (max_num_merges + 1)))); + + // Get the value + resetNumMergeOperatorCalls(); + std::string get_value_str; + ASSERT_OK(db->Get(ReadOptions(), key, &get_value_str)); + assert(get_value_str.size() == sizeof(uint64_t)); + uint64_t get_value = DecodeFixed64(&get_value_str[0]); + ASSERT_EQ(get_value, num_merges * merge_value); + ASSERT_EQ(num_merge_operator_calls, + static_cast((num_merges % (max_num_merges + 1)))); +} + +void runTest(const std::string& dbname, const bool use_ttl = false) { + { + auto db = OpenDb(dbname, use_ttl); + + { + Counters counters(db, 0); + testCounters(counters, db.get(), true); + } + + { + MergeBasedCounters counters(db, 0); + testCounters(counters, db.get(), use_compression); + } + } + + ASSERT_OK(DestroyDB(dbname, Options())); + + { + size_t max_merge = 5; + auto db = OpenDb(dbname, use_ttl, max_merge); + MergeBasedCounters counters(db, 0); + testCounters(counters, db.get(), use_compression); + testSuccessiveMerge(counters, max_merge, max_merge * 2); + testSingleBatchSuccessiveMerge(db.get(), 5, 7); + ASSERT_OK(db->Close()); + ASSERT_OK(DestroyDB(dbname, Options())); + } + + { + size_t max_merge = 100; + // Min merge is hard-coded to 2. + uint32_t min_merge = 2; + for (uint32_t count = min_merge - 1; count <= min_merge + 1; count++) { + auto db = OpenDb(dbname, use_ttl, max_merge); + MergeBasedCounters counters(db, 0); + testPartialMerge(&counters, db.get(), max_merge, min_merge, count); + ASSERT_OK(db->Close()); + ASSERT_OK(DestroyDB(dbname, Options())); + } + { + auto db = OpenDb(dbname, use_ttl, max_merge); + MergeBasedCounters counters(db, 0); + testPartialMerge(&counters, db.get(), max_merge, min_merge, + min_merge * 10); + ASSERT_OK(db->Close()); + ASSERT_OK(DestroyDB(dbname, Options())); + } + } + + { + { + auto db = OpenDb(dbname); + MergeBasedCounters counters(db, 0); + counters.add("test-key", 1); + counters.add("test-key", 1); + counters.add("test-key", 1); + ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + } + + DB* reopen_db; + ASSERT_OK(DB::Open(Options(), dbname, &reopen_db)); + std::string value; + ASSERT_NOK(reopen_db->Get(ReadOptions(), "test-key", &value)); + delete reopen_db; + ASSERT_OK(DestroyDB(dbname, Options())); + } + + /* Temporary remove this test + { + std::cout << "Test merge-operator not set after reopen (recovery case)\n"; + { + auto db = OpenDb(dbname); + MergeBasedCounters counters(db, 0); + counters.add("test-key", 1); + counters.add("test-key", 1); + counters.add("test-key", 1); + } + + DB* reopen_db; + ASSERT_TRUE(DB::Open(Options(), dbname, &reopen_db).IsInvalidArgument()); + } + */ +} + +TEST_F(MergeTest, MergeDbTest) { + runTest(test::PerThreadDBPath("merge_testdb")); +} + +TEST_F(MergeTest, MergeDbTtlTest) { + runTest(test::PerThreadDBPath("merge_testdbttl"), + true); // Run test on TTL database +} + +TEST_F(MergeTest, MergeWithCompactionAndFlush) { + const std::string dbname = + test::PerThreadDBPath("merge_with_compaction_and_flush"); + { + auto db = OpenDb(dbname); + { + MergeBasedCounters counters(db, 0); + testCountersWithFlushAndCompaction(counters, db.get()); + } + } + ASSERT_OK(DestroyDB(dbname, Options())); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::use_compression = false; + if (argc > 1) { + ROCKSDB_NAMESPACE::use_compression = true; + } + + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/obsolete_files_test.cc b/librocksdb-sys/rocksdb/db/obsolete_files_test.cc new file mode 100644 index 0000000..03f38c0 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/obsolete_files_test.cc @@ -0,0 +1,317 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + + +#include + +#include +#include +#include +#include + +#include "db/db_impl/db_impl.h" +#include "db/db_test_util.h" +#include "db/version_set.h" +#include "db/write_batch_internal.h" +#include "file/filename.h" +#include "port/stack_trace.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/transaction_log.h" +#include "test_util/sync_point.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +class ObsoleteFilesTest : public DBTestBase { + public: + ObsoleteFilesTest() + : DBTestBase("obsolete_files_test", /*env_do_fsync=*/true), + wal_dir_(dbname_ + "/wal_files") {} + + void AddKeys(int numkeys, int startkey) { + WriteOptions options; + options.sync = false; + for (int i = startkey; i < (numkeys + startkey); i++) { + std::string temp = std::to_string(i); + Slice key(temp); + Slice value(temp); + ASSERT_OK(db_->Put(options, key, value)); + } + } + + void createLevel0Files(int numFiles, int numKeysPerFile) { + int startKey = 0; + for (int i = 0; i < numFiles; i++) { + AddKeys(numKeysPerFile, startKey); + startKey += numKeysPerFile; + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + ASSERT_OK( + dbfull()->TEST_WaitForCompact()); // wait for background flush (flush + // is also a kind of compaction). + } + } + + void CheckFileTypeCounts(const std::string& dir, int required_log, + int required_sst, int required_manifest) { + std::vector filenames; + ASSERT_OK(env_->GetChildren(dir, &filenames)); + + int log_cnt = 0; + int sst_cnt = 0; + int manifest_cnt = 0; + for (auto file : filenames) { + uint64_t number; + FileType type; + if (ParseFileName(file, &number, &type)) { + log_cnt += (type == kWalFile); + sst_cnt += (type == kTableFile); + manifest_cnt += (type == kDescriptorFile); + } + } + ASSERT_EQ(required_log, log_cnt); + ASSERT_EQ(required_sst, sst_cnt); + ASSERT_EQ(required_manifest, manifest_cnt); + } + + void ReopenDB() { + Options options = CurrentOptions(); + // Trigger compaction when the number of level 0 files reaches 2. + options.create_if_missing = true; + options.level0_file_num_compaction_trigger = 2; + options.disable_auto_compactions = false; + options.delete_obsolete_files_period_micros = 0; // always do full purge + options.enable_thread_tracking = true; + options.write_buffer_size = 1024 * 1024 * 1000; + options.target_file_size_base = 1024 * 1024 * 1000; + options.max_bytes_for_level_base = 1024 * 1024 * 1000; + options.WAL_ttl_seconds = 300; // Used to test log files + options.WAL_size_limit_MB = 1024; // Used to test log files + options.wal_dir = wal_dir_; + + // Note: the following prevents an otherwise harmless data race between the + // test setup code (AddBlobFile) in ObsoleteFilesTest.BlobFiles and the + // periodic stat dumping thread. + options.stats_dump_period_sec = 0; + + Destroy(options); + Reopen(options); + } + + const std::string wal_dir_; +}; + +TEST_F(ObsoleteFilesTest, RaceForObsoleteFileDeletion) { + ReopenDB(); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->LoadDependency({ + {"DBImpl::BackgroundCallCompaction:FoundObsoleteFiles", + "ObsoleteFilesTest::RaceForObsoleteFileDeletion:1"}, + {"DBImpl::BackgroundCallCompaction:PurgedObsoleteFiles", + "ObsoleteFilesTest::RaceForObsoleteFileDeletion:2"}, + }); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::DeleteObsoleteFileImpl:AfterDeletion", [&](void* arg) { + Status* p_status = reinterpret_cast(arg); + ASSERT_OK(*p_status); + }); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::CloseHelper:PendingPurgeFinished", [&](void* arg) { + std::unordered_set* files_grabbed_for_purge_ptr = + reinterpret_cast*>(arg); + ASSERT_TRUE(files_grabbed_for_purge_ptr->empty()); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + createLevel0Files(2, 50000); + CheckFileTypeCounts(wal_dir_, 1, 0, 0); + + port::Thread user_thread([this]() { + JobContext jobCxt(0); + TEST_SYNC_POINT("ObsoleteFilesTest::RaceForObsoleteFileDeletion:1"); + dbfull()->TEST_LockMutex(); + dbfull()->FindObsoleteFiles(&jobCxt, true /* force=true */, + false /* no_full_scan=false */); + dbfull()->TEST_UnlockMutex(); + TEST_SYNC_POINT("ObsoleteFilesTest::RaceForObsoleteFileDeletion:2"); + dbfull()->PurgeObsoleteFiles(jobCxt); + jobCxt.Clean(); + }); + + user_thread.join(); +} + +TEST_F(ObsoleteFilesTest, DeleteObsoleteOptionsFile) { + ReopenDB(); + + createLevel0Files(2, 50000); + CheckFileTypeCounts(wal_dir_, 1, 0, 0); + + ASSERT_OK(dbfull()->DisableFileDeletions()); + for (int i = 0; i != 4; ++i) { + if (i % 2) { + ASSERT_OK(dbfull()->SetOptions(dbfull()->DefaultColumnFamily(), + {{"paranoid_file_checks", "false"}})); + } else { + ASSERT_OK(dbfull()->SetOptions(dbfull()->DefaultColumnFamily(), + {{"paranoid_file_checks", "true"}})); + } + } + ASSERT_OK(dbfull()->EnableFileDeletions(true /* force */)); + + Close(); + + std::vector files; + int opts_file_count = 0; + ASSERT_OK(env_->GetChildren(dbname_, &files)); + for (const auto& file : files) { + uint64_t file_num; + Slice dummy_info_log_name_prefix; + FileType type; + WalFileType log_type; + if (ParseFileName(file, &file_num, dummy_info_log_name_prefix, &type, + &log_type) && + type == kOptionsFile) { + opts_file_count++; + } + } + ASSERT_EQ(2, opts_file_count); +} + +TEST_F(ObsoleteFilesTest, BlobFiles) { + ReopenDB(); + + VersionSet* const versions = dbfull()->GetVersionSet(); + assert(versions); + assert(versions->GetColumnFamilySet()); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + assert(cfd); + + const ImmutableCFOptions* const ioptions = cfd->ioptions(); + assert(ioptions); + assert(!ioptions->cf_paths.empty()); + + const std::string& path = ioptions->cf_paths.front().path; + + // Add an obsolete blob file. + constexpr uint64_t first_blob_file_number = 234; + versions->AddObsoleteBlobFile(first_blob_file_number, path); + + // Add a live blob file. + Version* const version = cfd->current(); + assert(version); + + VersionStorageInfo* const storage_info = version->storage_info(); + assert(storage_info); + + constexpr uint64_t second_blob_file_number = 456; + constexpr uint64_t second_total_blob_count = 100; + constexpr uint64_t second_total_blob_bytes = 2000000; + constexpr char second_checksum_method[] = "CRC32B"; + constexpr char second_checksum_value[] = "\x6d\xbd\xf2\x3a"; + + auto shared_meta = SharedBlobFileMetaData::Create( + second_blob_file_number, second_total_blob_count, second_total_blob_bytes, + second_checksum_method, second_checksum_value); + + constexpr uint64_t second_garbage_blob_count = 0; + constexpr uint64_t second_garbage_blob_bytes = 0; + + auto meta = BlobFileMetaData::Create( + std::move(shared_meta), BlobFileMetaData::LinkedSsts(), + second_garbage_blob_count, second_garbage_blob_bytes); + + storage_info->AddBlobFile(std::move(meta)); + + // Check for obsolete files and make sure the first blob file is picked up + // and grabbed for purge. The second blob file should be on the live list. + constexpr int job_id = 0; + JobContext job_context{job_id}; + + dbfull()->TEST_LockMutex(); + constexpr bool force_full_scan = false; + dbfull()->FindObsoleteFiles(&job_context, force_full_scan); + dbfull()->TEST_UnlockMutex(); + + ASSERT_TRUE(job_context.HaveSomethingToDelete()); + ASSERT_EQ(job_context.blob_delete_files.size(), 1); + ASSERT_EQ(job_context.blob_delete_files[0].GetBlobFileNumber(), + first_blob_file_number); + + const auto& files_grabbed_for_purge = + dbfull()->TEST_GetFilesGrabbedForPurge(); + ASSERT_NE(files_grabbed_for_purge.find(first_blob_file_number), + files_grabbed_for_purge.end()); + + ASSERT_EQ(job_context.blob_live.size(), 1); + ASSERT_EQ(job_context.blob_live[0], second_blob_file_number); + + // Hack the job context a bit by adding a few files to the full scan + // list and adjusting the pending file number. We add the two files + // above as well as two additional ones, where one is old + // and should be cleaned up, and the other is still pending. + constexpr uint64_t old_blob_file_number = 123; + constexpr uint64_t pending_blob_file_number = 567; + + job_context.full_scan_candidate_files.emplace_back( + BlobFileName(old_blob_file_number), path); + job_context.full_scan_candidate_files.emplace_back( + BlobFileName(first_blob_file_number), path); + job_context.full_scan_candidate_files.emplace_back( + BlobFileName(second_blob_file_number), path); + job_context.full_scan_candidate_files.emplace_back( + BlobFileName(pending_blob_file_number), path); + + job_context.min_pending_output = pending_blob_file_number; + + // Purge obsolete files and make sure we purge the old file and the first file + // (and keep the second file and the pending file). + std::vector deleted_files; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::DeleteObsoleteFileImpl::BeforeDeletion", [&](void* arg) { + const std::string* file = static_cast(arg); + assert(file); + + constexpr char blob_extension[] = ".blob"; + + if (file->find(blob_extension) != std::string::npos) { + deleted_files.emplace_back(*file); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + + dbfull()->PurgeObsoleteFiles(job_context); + job_context.Clean(); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + ASSERT_EQ(files_grabbed_for_purge.find(first_blob_file_number), + files_grabbed_for_purge.end()); + + std::sort(deleted_files.begin(), deleted_files.end()); + const std::vector expected_deleted_files{ + BlobFileName(path, old_blob_file_number), + BlobFileName(path, first_blob_file_number)}; + + ASSERT_EQ(deleted_files, expected_deleted_files); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/librocksdb-sys/rocksdb/db/options_file_test.cc b/librocksdb-sys/rocksdb/db/options_file_test.cc new file mode 100644 index 0000000..c3adbeb --- /dev/null +++ b/librocksdb-sys/rocksdb/db/options_file_test.cc @@ -0,0 +1,110 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include + +#include "db/db_impl/db_impl.h" +#include "db/db_test_util.h" +#include "rocksdb/options.h" +#include "rocksdb/table.h" +#include "test_util/testharness.h" + +namespace ROCKSDB_NAMESPACE { +class OptionsFileTest : public testing::Test { + public: + OptionsFileTest() : dbname_(test::PerThreadDBPath("options_file_test")) {} + + std::string dbname_; +}; + +namespace { +void UpdateOptionsFiles(DB* db, + std::unordered_set* filename_history, + int* options_files_count) { + std::vector filenames; + EXPECT_OK(db->GetEnv()->GetChildren(db->GetName(), &filenames)); + uint64_t number; + FileType type; + *options_files_count = 0; + for (auto filename : filenames) { + if (ParseFileName(filename, &number, &type) && type == kOptionsFile) { + filename_history->insert(filename); + (*options_files_count)++; + } + } +} + +// Verify whether the current Options Files are the latest ones. +void VerifyOptionsFileName( + DB* db, const std::unordered_set& past_filenames) { + std::vector filenames; + std::unordered_set current_filenames; + EXPECT_OK(db->GetEnv()->GetChildren(db->GetName(), &filenames)); + uint64_t number; + FileType type; + for (auto filename : filenames) { + if (ParseFileName(filename, &number, &type) && type == kOptionsFile) { + current_filenames.insert(filename); + } + } + for (auto past_filename : past_filenames) { + if (current_filenames.find(past_filename) != current_filenames.end()) { + continue; + } + for (auto filename : current_filenames) { + ASSERT_GT(filename, past_filename); + } + } +} +} // anonymous namespace + +TEST_F(OptionsFileTest, NumberOfOptionsFiles) { + const int kReopenCount = 20; + Options opt; + opt.create_if_missing = true; + ASSERT_OK(DestroyDB(dbname_, opt)); + std::unordered_set filename_history; + DB* db; + for (int i = 0; i < kReopenCount; ++i) { + ASSERT_OK(DB::Open(opt, dbname_, &db)); + int num_options_files = 0; + UpdateOptionsFiles(db, &filename_history, &num_options_files); + ASSERT_GT(num_options_files, 0); + ASSERT_LE(num_options_files, 2); + // Make sure we always keep the latest option files. + VerifyOptionsFileName(db, filename_history); + delete db; + } +} + +TEST_F(OptionsFileTest, OptionsFileName) { + const uint64_t kOptionsFileNum = 12345; + uint64_t number; + FileType type; + + auto options_file_name = OptionsFileName("", kOptionsFileNum); + ASSERT_TRUE(ParseFileName(options_file_name, &number, &type, nullptr)); + ASSERT_EQ(type, kOptionsFile); + ASSERT_EQ(number, kOptionsFileNum); + + const uint64_t kTempOptionsFileNum = 54352; + auto temp_options_file_name = TempOptionsFileName("", kTempOptionsFileNum); + ASSERT_TRUE(ParseFileName(temp_options_file_name, &number, &type, nullptr)); + ASSERT_NE(temp_options_file_name.find(kTempFileNameSuffix), + std::string::npos); + ASSERT_EQ(type, kTempFile); + ASSERT_EQ(number, kTempOptionsFileNum); +} +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { +#if !(defined NDEBUG) || !defined(OS_WIN) + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +#else + return 0; +#endif // !(defined NDEBUG) || !defined(OS_WIN) +} diff --git a/librocksdb-sys/rocksdb/db/output_validator.cc b/librocksdb-sys/rocksdb/db/output_validator.cc new file mode 100644 index 0000000..e93e2d6 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/output_validator.cc @@ -0,0 +1,33 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#include "db/output_validator.h" + +#include "test_util/sync_point.h" +#include "util/hash.h" + +namespace ROCKSDB_NAMESPACE { +Status OutputValidator::Add(const Slice& key, const Slice& value) { + if (enable_hash_) { + // Generate a rolling 64-bit hash of the key and values + paranoid_hash_ = NPHash64(key.data(), key.size(), paranoid_hash_); + paranoid_hash_ = NPHash64(value.data(), value.size(), paranoid_hash_); + } + if (enable_order_check_) { + TEST_SYNC_POINT_CALLBACK("OutputValidator::Add:order_check", + /*arg=*/nullptr); + if (key.size() < kNumInternalBytes) { + return Status::Corruption( + "Compaction tries to write a key without internal bytes."); + } + // prev_key_ starts with empty. + if (!prev_key_.empty() && icmp_.Compare(key, prev_key_) < 0) { + return Status::Corruption("Compaction sees out-of-order keys."); + } + prev_key_.assign(key.data(), key.size()); + } + return Status::OK(); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/output_validator.h b/librocksdb-sys/rocksdb/db/output_validator.h new file mode 100644 index 0000000..40635f9 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/output_validator.h @@ -0,0 +1,48 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#pragma once +#include "db/dbformat.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { +// A class that validates key/value that is inserted to an SST file. +// Pass every key/value of the file using OutputValidator::Add() +// and the class validates key order and optionally calculate a hash +// of all the key and value. +class OutputValidator { + public: + explicit OutputValidator(const InternalKeyComparator& icmp, + bool enable_order_check, bool enable_hash, + uint64_t precalculated_hash = 0) + : icmp_(icmp), + paranoid_hash_(precalculated_hash), + enable_order_check_(enable_order_check), + enable_hash_(enable_hash) {} + + // Add a key to the KV sequence, and return whether the key follows + // criteria, e.g. key is ordered. + Status Add(const Slice& key, const Slice& value); + + // Compare result of two key orders are the same. It can be used + // to compare the keys inserted into a file, and what is read back. + // Return true if the validation passes. + bool CompareValidator(const OutputValidator& other_validator) { + return GetHash() == other_validator.GetHash(); + } + + // Not (yet) intended to be persisted, so subject to change + // without notice between releases. + uint64_t GetHash() const { return paranoid_hash_; } + + private: + const InternalKeyComparator& icmp_; + std::string prev_key_; + uint64_t paranoid_hash_ = 0; + bool enable_order_check_; + bool enable_hash_; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/perf_context_test.cc b/librocksdb-sys/rocksdb/db/perf_context_test.cc new file mode 100644 index 0000000..bb8691b --- /dev/null +++ b/librocksdb-sys/rocksdb/db/perf_context_test.cc @@ -0,0 +1,1158 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#include "rocksdb/perf_context.h" + +#include +#include +#include +#include + +#include "monitoring/histogram.h" +#include "monitoring/instrumented_mutex.h" +#include "monitoring/perf_context_imp.h" +#include "monitoring/thread_status_util.h" +#include "port/port.h" +#include "rocksdb/db.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/system_clock.h" +#include "test_util/testharness.h" +#include "util/stop_watch.h" +#include "util/string_util.h" +#include "utilities/merge_operators.h" + +bool FLAGS_random_key = false; +bool FLAGS_use_set_based_memetable = false; +int FLAGS_total_keys = 100; +int FLAGS_write_buffer_size = 1000000000; +int FLAGS_max_write_buffer_number = 8; +int FLAGS_min_write_buffer_number_to_merge = 7; +bool FLAGS_verbose = false; + +// Path to the database on file system +const std::string kDbName = + ROCKSDB_NAMESPACE::test::PerThreadDBPath("perf_context_test"); + +namespace ROCKSDB_NAMESPACE { + +std::shared_ptr OpenDb(bool read_only = false) { + DB* db; + Options options; + options.create_if_missing = true; + options.max_open_files = -1; + options.write_buffer_size = FLAGS_write_buffer_size; + options.max_write_buffer_number = FLAGS_max_write_buffer_number; + options.min_write_buffer_number_to_merge = + FLAGS_min_write_buffer_number_to_merge; + + if (FLAGS_use_set_based_memetable) { + options.prefix_extractor.reset( + ROCKSDB_NAMESPACE::NewFixedPrefixTransform(0)); + options.memtable_factory.reset(NewHashSkipListRepFactory()); + } + + Status s; + if (!read_only) { + s = DB::Open(options, kDbName, &db); + } else { + s = DB::OpenForReadOnly(options, kDbName, &db); + } + EXPECT_OK(s); + return std::shared_ptr(db); +} + +class PerfContextTest : public testing::Test {}; + +TEST_F(PerfContextTest, SeekIntoDeletion) { + ASSERT_OK(DestroyDB(kDbName, Options())); + auto db = OpenDb(); + WriteOptions write_options; + ReadOptions read_options; + + for (int i = 0; i < FLAGS_total_keys; ++i) { + std::string key = "k" + std::to_string(i); + std::string value = "v" + std::to_string(i); + + ASSERT_OK(db->Put(write_options, key, value)); + } + + for (int i = 0; i < FLAGS_total_keys - 1; ++i) { + std::string key = "k" + std::to_string(i); + ASSERT_OK(db->Delete(write_options, key)); + } + + HistogramImpl hist_get; + HistogramImpl hist_get_time; + for (int i = 0; i < FLAGS_total_keys - 1; ++i) { + std::string key = "k" + std::to_string(i); + std::string value; + + get_perf_context()->Reset(); + StopWatchNano timer(SystemClock::Default().get()); + timer.Start(); + auto status = db->Get(read_options, key, &value); + auto elapsed_nanos = timer.ElapsedNanos(); + ASSERT_TRUE(status.IsNotFound()); + hist_get.Add(get_perf_context()->user_key_comparison_count); + hist_get_time.Add(elapsed_nanos); + } + + if (FLAGS_verbose) { + std::cout << "Get user key comparison: \n" + << hist_get.ToString() << "Get time: \n" + << hist_get_time.ToString(); + } + + { + HistogramImpl hist_seek_to_first; + std::unique_ptr iter(db->NewIterator(read_options)); + + get_perf_context()->Reset(); + StopWatchNano timer(SystemClock::Default().get(), true); + iter->SeekToFirst(); + hist_seek_to_first.Add(get_perf_context()->user_key_comparison_count); + auto elapsed_nanos = timer.ElapsedNanos(); + + if (FLAGS_verbose) { + std::cout << "SeekToFirst user key comparison: \n" + << hist_seek_to_first.ToString() << "ikey skipped: " + << get_perf_context()->internal_key_skipped_count << "\n" + << "idelete skipped: " + << get_perf_context()->internal_delete_skipped_count << "\n" + << "elapsed: " << elapsed_nanos << "\n"; + } + } + + HistogramImpl hist_seek; + for (int i = 0; i < FLAGS_total_keys; ++i) { + std::unique_ptr iter(db->NewIterator(read_options)); + std::string key = "k" + std::to_string(i); + + get_perf_context()->Reset(); + StopWatchNano timer(SystemClock::Default().get(), true); + iter->Seek(key); + auto elapsed_nanos = timer.ElapsedNanos(); + hist_seek.Add(get_perf_context()->user_key_comparison_count); + if (FLAGS_verbose) { + std::cout << "seek cmp: " << get_perf_context()->user_key_comparison_count + << " ikey skipped " + << get_perf_context()->internal_key_skipped_count + << " idelete skipped " + << get_perf_context()->internal_delete_skipped_count + << " elapsed: " << elapsed_nanos << "ns\n"; + } + + get_perf_context()->Reset(); + ASSERT_TRUE(iter->Valid()); + StopWatchNano timer2(SystemClock::Default().get(), true); + iter->Next(); + auto elapsed_nanos2 = timer2.ElapsedNanos(); + if (FLAGS_verbose) { + std::cout << "next cmp: " << get_perf_context()->user_key_comparison_count + << "elapsed: " << elapsed_nanos2 << "ns\n"; + } + } + + if (FLAGS_verbose) { + std::cout << "Seek user key comparison: \n" << hist_seek.ToString(); + } +} + +TEST_F(PerfContextTest, StopWatchNanoOverhead) { + // profile the timer cost by itself! + const int kTotalIterations = 1000000; + std::vector timings(kTotalIterations); + + StopWatchNano timer(SystemClock::Default().get(), true); + for (auto& timing : timings) { + timing = timer.ElapsedNanos(true /* reset */); + } + + HistogramImpl histogram; + for (const auto timing : timings) { + histogram.Add(timing); + } + + if (FLAGS_verbose) { + std::cout << histogram.ToString(); + } +} + +TEST_F(PerfContextTest, StopWatchOverhead) { + // profile the timer cost by itself! + const int kTotalIterations = 1000000; + uint64_t elapsed = 0; + std::vector timings(kTotalIterations); + + StopWatch timer(SystemClock::Default().get(), nullptr, 0, + Histograms::HISTOGRAM_ENUM_MAX, &elapsed); + for (auto& timing : timings) { + timing = elapsed; + } + + HistogramImpl histogram; + uint64_t prev_timing = 0; + for (const auto timing : timings) { + histogram.Add(timing - prev_timing); + prev_timing = timing; + } + + if (FLAGS_verbose) { + std::cout << histogram.ToString(); + } +} + +void ProfileQueries(bool enabled_time = false) { + ASSERT_OK(DestroyDB(kDbName, Options())); // Start this test with a fresh DB + + auto db = OpenDb(); + + WriteOptions write_options; + ReadOptions read_options; + + HistogramImpl hist_put; + + HistogramImpl hist_get; + HistogramImpl hist_get_snapshot; + HistogramImpl hist_get_memtable; + HistogramImpl hist_get_files; + HistogramImpl hist_get_post_process; + HistogramImpl hist_num_memtable_checked; + + HistogramImpl hist_mget; + HistogramImpl hist_mget_snapshot; + HistogramImpl hist_mget_memtable; + HistogramImpl hist_mget_files; + HistogramImpl hist_mget_post_process; + HistogramImpl hist_mget_num_memtable_checked; + + HistogramImpl hist_write_pre_post; + HistogramImpl hist_write_wal_time; + HistogramImpl hist_write_memtable_time; + HistogramImpl hist_write_delay_time; + HistogramImpl hist_write_thread_wait_nanos; + HistogramImpl hist_write_scheduling_time; + + uint64_t total_db_mutex_nanos = 0; + + if (FLAGS_verbose) { + std::cout << "Inserting " << FLAGS_total_keys << " key/value pairs\n...\n"; + } + + std::vector keys; + const int kFlushFlag = -1; + for (int i = 0; i < FLAGS_total_keys; ++i) { + keys.push_back(i); + if (i == FLAGS_total_keys / 2) { + // Issuing a flush in the middle. + keys.push_back(kFlushFlag); + } + } + + if (FLAGS_random_key) { + RandomShuffle(std::begin(keys), std::end(keys)); + } +#ifndef NDEBUG + ThreadStatusUtil::TEST_SetStateDelay(ThreadStatus::STATE_MUTEX_WAIT, 1U); +#endif + int num_mutex_waited = 0; + for (const int i : keys) { + if (i == kFlushFlag) { + FlushOptions fo; + db->Flush(fo); + continue; + } + + std::string key = "k" + std::to_string(i); + std::string value = "v" + std::to_string(i); + + std::vector values; + + get_perf_context()->Reset(); + ASSERT_OK(db->Put(write_options, key, value)); + if (++num_mutex_waited > 3) { +#ifndef NDEBUG + ThreadStatusUtil::TEST_SetStateDelay(ThreadStatus::STATE_MUTEX_WAIT, 0U); +#endif + } + hist_write_pre_post.Add( + get_perf_context()->write_pre_and_post_process_time); + hist_write_wal_time.Add(get_perf_context()->write_wal_time); + hist_write_memtable_time.Add(get_perf_context()->write_memtable_time); + hist_write_delay_time.Add(get_perf_context()->write_delay_time); + hist_write_thread_wait_nanos.Add( + get_perf_context()->write_thread_wait_nanos); + hist_write_scheduling_time.Add( + get_perf_context()->write_scheduling_flushes_compactions_time); + hist_put.Add(get_perf_context()->user_key_comparison_count); + total_db_mutex_nanos += get_perf_context()->db_mutex_lock_nanos; + } +#ifndef NDEBUG + ThreadStatusUtil::TEST_SetStateDelay(ThreadStatus::STATE_MUTEX_WAIT, 0U); +#endif + + for (const int i : keys) { + if (i == kFlushFlag) { + continue; + } + std::string key = "k" + std::to_string(i); + std::string expected_value = "v" + std::to_string(i); + std::string value; + + std::vector multiget_keys = {Slice(key)}; + std::vector values; + + get_perf_context()->Reset(); + ASSERT_OK(db->Get(read_options, key, &value)); + ASSERT_EQ(expected_value, value); + hist_get_snapshot.Add(get_perf_context()->get_snapshot_time); + hist_get_memtable.Add(get_perf_context()->get_from_memtable_time); + hist_get_files.Add(get_perf_context()->get_from_output_files_time); + hist_num_memtable_checked.Add(get_perf_context()->get_from_memtable_count); + hist_get_post_process.Add(get_perf_context()->get_post_process_time); + hist_get.Add(get_perf_context()->user_key_comparison_count); + + get_perf_context()->Reset(); + auto statuses = db->MultiGet(read_options, multiget_keys, &values); + for (const auto& s : statuses) { + ASSERT_OK(s); + } + hist_mget_snapshot.Add(get_perf_context()->get_snapshot_time); + hist_mget_memtable.Add(get_perf_context()->get_from_memtable_time); + hist_mget_files.Add(get_perf_context()->get_from_output_files_time); + hist_mget_num_memtable_checked.Add( + get_perf_context()->get_from_memtable_count); + hist_mget_post_process.Add(get_perf_context()->get_post_process_time); + hist_mget.Add(get_perf_context()->user_key_comparison_count); + } + + if (FLAGS_verbose) { + std::cout << "Put user key comparison: \n" + << hist_put.ToString() << "Get user key comparison: \n" + << hist_get.ToString() << "MultiGet user key comparison: \n" + << hist_get.ToString(); + std::cout << "Put(): Pre and Post Process Time: \n" + << hist_write_pre_post.ToString() << " Writing WAL time: \n" + << hist_write_wal_time.ToString() << "\n" + << " Writing Mem Table time: \n" + << hist_write_memtable_time.ToString() << "\n" + << " Write Delay: \n" + << hist_write_delay_time.ToString() << "\n" + << " Waiting for Batch time: \n" + << hist_write_thread_wait_nanos.ToString() << "\n" + << " Scheduling Flushes and Compactions Time: \n" + << hist_write_scheduling_time.ToString() << "\n" + << " Total DB mutex nanos: \n" + << total_db_mutex_nanos << "\n"; + + std::cout << "Get(): Time to get snapshot: \n" + << hist_get_snapshot.ToString() + << " Time to get value from memtables: \n" + << hist_get_memtable.ToString() << "\n" + << " Time to get value from output files: \n" + << hist_get_files.ToString() << "\n" + << " Number of memtables checked: \n" + << hist_num_memtable_checked.ToString() << "\n" + << " Time to post process: \n" + << hist_get_post_process.ToString() << "\n"; + + std::cout << "MultiGet(): Time to get snapshot: \n" + << hist_mget_snapshot.ToString() + << " Time to get value from memtables: \n" + << hist_mget_memtable.ToString() << "\n" + << " Time to get value from output files: \n" + << hist_mget_files.ToString() << "\n" + << " Number of memtables checked: \n" + << hist_mget_num_memtable_checked.ToString() << "\n" + << " Time to post process: \n" + << hist_mget_post_process.ToString() << "\n"; + } + + if (enabled_time) { + ASSERT_GT(hist_get.Average(), 0); + ASSERT_GT(hist_get_snapshot.Average(), 0); + ASSERT_GT(hist_get_memtable.Average(), 0); + ASSERT_GT(hist_get_files.Average(), 0); + ASSERT_GT(hist_get_post_process.Average(), 0); + ASSERT_GT(hist_num_memtable_checked.Average(), 0); + + ASSERT_GT(hist_mget.Average(), 0); + ASSERT_GT(hist_mget_snapshot.Average(), 0); + ASSERT_GT(hist_mget_memtable.Average(), 0); + ASSERT_GT(hist_mget_files.Average(), 0); + ASSERT_GT(hist_mget_post_process.Average(), 0); + ASSERT_GT(hist_mget_num_memtable_checked.Average(), 0); + + EXPECT_GT(hist_write_pre_post.Average(), 0); + EXPECT_GT(hist_write_wal_time.Average(), 0); + EXPECT_GT(hist_write_memtable_time.Average(), 0); + EXPECT_EQ(hist_write_delay_time.Average(), 0); + EXPECT_EQ(hist_write_thread_wait_nanos.Average(), 0); + EXPECT_GT(hist_write_scheduling_time.Average(), 0); + +#ifndef NDEBUG + ASSERT_LT(total_db_mutex_nanos, 100U); +#endif + } + + db.reset(); + db = OpenDb(true); + + hist_get.Clear(); + hist_get_snapshot.Clear(); + hist_get_memtable.Clear(); + hist_get_files.Clear(); + hist_get_post_process.Clear(); + hist_num_memtable_checked.Clear(); + + hist_mget.Clear(); + hist_mget_snapshot.Clear(); + hist_mget_memtable.Clear(); + hist_mget_files.Clear(); + hist_mget_post_process.Clear(); + hist_mget_num_memtable_checked.Clear(); + + for (const int i : keys) { + if (i == kFlushFlag) { + continue; + } + std::string key = "k" + std::to_string(i); + std::string expected_value = "v" + std::to_string(i); + std::string value; + + std::vector multiget_keys = {Slice(key)}; + std::vector values; + + get_perf_context()->Reset(); + ASSERT_OK(db->Get(read_options, key, &value)); + ASSERT_EQ(expected_value, value); + hist_get_snapshot.Add(get_perf_context()->get_snapshot_time); + hist_get_memtable.Add(get_perf_context()->get_from_memtable_time); + hist_get_files.Add(get_perf_context()->get_from_output_files_time); + hist_num_memtable_checked.Add(get_perf_context()->get_from_memtable_count); + hist_get_post_process.Add(get_perf_context()->get_post_process_time); + hist_get.Add(get_perf_context()->user_key_comparison_count); + + get_perf_context()->Reset(); + auto statuses = db->MultiGet(read_options, multiget_keys, &values); + for (const auto& s : statuses) { + ASSERT_OK(s); + } + hist_mget_snapshot.Add(get_perf_context()->get_snapshot_time); + hist_mget_memtable.Add(get_perf_context()->get_from_memtable_time); + hist_mget_files.Add(get_perf_context()->get_from_output_files_time); + hist_mget_num_memtable_checked.Add( + get_perf_context()->get_from_memtable_count); + hist_mget_post_process.Add(get_perf_context()->get_post_process_time); + hist_mget.Add(get_perf_context()->user_key_comparison_count); + } + + if (FLAGS_verbose) { + std::cout << "ReadOnly Get user key comparison: \n" + << hist_get.ToString() + << "ReadOnly MultiGet user key comparison: \n" + << hist_mget.ToString(); + + std::cout << "ReadOnly Get(): Time to get snapshot: \n" + << hist_get_snapshot.ToString() + << " Time to get value from memtables: \n" + << hist_get_memtable.ToString() << "\n" + << " Time to get value from output files: \n" + << hist_get_files.ToString() << "\n" + << " Number of memtables checked: \n" + << hist_num_memtable_checked.ToString() << "\n" + << " Time to post process: \n" + << hist_get_post_process.ToString() << "\n"; + + std::cout << "ReadOnly MultiGet(): Time to get snapshot: \n" + << hist_mget_snapshot.ToString() + << " Time to get value from memtables: \n" + << hist_mget_memtable.ToString() << "\n" + << " Time to get value from output files: \n" + << hist_mget_files.ToString() << "\n" + << " Number of memtables checked: \n" + << hist_mget_num_memtable_checked.ToString() << "\n" + << " Time to post process: \n" + << hist_mget_post_process.ToString() << "\n"; + } + + if (enabled_time) { + ASSERT_GT(hist_get.Average(), 0); + ASSERT_GT(hist_get_memtable.Average(), 0); + ASSERT_GT(hist_get_files.Average(), 0); + ASSERT_GT(hist_num_memtable_checked.Average(), 0); + // In read-only mode Get(), no super version operation is needed + ASSERT_EQ(hist_get_post_process.Average(), 0); + ASSERT_GT(hist_get_snapshot.Average(), 0); + + ASSERT_GT(hist_mget.Average(), 0); + ASSERT_GT(hist_mget_snapshot.Average(), 0); + ASSERT_GT(hist_mget_memtable.Average(), 0); + ASSERT_GT(hist_mget_files.Average(), 0); + ASSERT_GT(hist_mget_post_process.Average(), 0); + ASSERT_GT(hist_mget_num_memtable_checked.Average(), 0); + } +} + +TEST_F(PerfContextTest, KeyComparisonCount) { + SetPerfLevel(kEnableCount); + ProfileQueries(); + + SetPerfLevel(kDisable); + ProfileQueries(); + + SetPerfLevel(kEnableTime); + ProfileQueries(true); +} + +// make perf_context_test +// export ROCKSDB_TESTS=PerfContextTest.SeekKeyComparison +// For one memtable: +// ./perf_context_test --write_buffer_size=500000 --total_keys=10000 +// For two memtables: +// ./perf_context_test --write_buffer_size=250000 --total_keys=10000 +// Specify --random_key=1 to shuffle the key before insertion +// Results show that, for sequential insertion, worst-case Seek Key comparison +// is close to the total number of keys (linear), when there is only one +// memtable. When there are two memtables, even the avg Seek Key comparison +// starts to become linear to the input size. + +TEST_F(PerfContextTest, SeekKeyComparison) { + ASSERT_OK(DestroyDB(kDbName, Options())); + auto db = OpenDb(); + WriteOptions write_options; + ReadOptions read_options; + + if (FLAGS_verbose) { + std::cout << "Inserting " << FLAGS_total_keys << " key/value pairs\n...\n"; + } + + std::vector keys; + for (int i = 0; i < FLAGS_total_keys; ++i) { + keys.push_back(i); + } + + if (FLAGS_random_key) { + RandomShuffle(std::begin(keys), std::end(keys)); + } + + HistogramImpl hist_put_time; + HistogramImpl hist_wal_time; + HistogramImpl hist_time_diff; + + SetPerfLevel(kEnableTime); + StopWatchNano timer(SystemClock::Default().get()); + for (const int i : keys) { + std::string key = "k" + std::to_string(i); + std::string value = "v" + std::to_string(i); + + get_perf_context()->Reset(); + timer.Start(); + ASSERT_OK(db->Put(write_options, key, value)); + auto put_time = timer.ElapsedNanos(); + hist_put_time.Add(put_time); + hist_wal_time.Add(get_perf_context()->write_wal_time); + hist_time_diff.Add(put_time - get_perf_context()->write_wal_time); + } + + if (FLAGS_verbose) { + std::cout << "Put time:\n" + << hist_put_time.ToString() << "WAL time:\n" + << hist_wal_time.ToString() << "time diff:\n" + << hist_time_diff.ToString(); + } + + HistogramImpl hist_seek; + HistogramImpl hist_next; + + for (int i = 0; i < FLAGS_total_keys; ++i) { + std::string key = "k" + std::to_string(i); + std::string value = "v" + std::to_string(i); + + std::unique_ptr iter(db->NewIterator(read_options)); + get_perf_context()->Reset(); + iter->Seek(key); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->value().ToString(), value); + hist_seek.Add(get_perf_context()->user_key_comparison_count); + } + + std::unique_ptr iter(db->NewIterator(read_options)); + for (iter->SeekToFirst(); iter->Valid();) { + get_perf_context()->Reset(); + iter->Next(); + hist_next.Add(get_perf_context()->user_key_comparison_count); + } + ASSERT_OK(iter->status()); + if (FLAGS_verbose) { + std::cout << "Seek:\n" + << hist_seek.ToString() << "Next:\n" + << hist_next.ToString(); + } +} + +TEST_F(PerfContextTest, DBMutexLockCounter) { + int stats_code[] = {0, static_cast(DB_MUTEX_WAIT_MICROS)}; + for (PerfLevel perf_level_test : + {PerfLevel::kEnableTimeExceptForMutex, PerfLevel::kEnableTime}) { + for (int c = 0; c < 2; ++c) { + InstrumentedMutex mutex(nullptr, SystemClock::Default().get(), + stats_code[c]); + mutex.Lock(); + ROCKSDB_NAMESPACE::port::Thread child_thread([&] { + SetPerfLevel(perf_level_test); + get_perf_context()->Reset(); + ASSERT_EQ(get_perf_context()->db_mutex_lock_nanos, 0); + mutex.Lock(); + mutex.Unlock(); + if (perf_level_test == PerfLevel::kEnableTimeExceptForMutex || + stats_code[c] != DB_MUTEX_WAIT_MICROS) { + ASSERT_EQ(get_perf_context()->db_mutex_lock_nanos, 0); + } else { + // increment the counter only when it's a DB Mutex + ASSERT_GT(get_perf_context()->db_mutex_lock_nanos, 0); + } + }); + SystemClock::Default()->SleepForMicroseconds(100); + mutex.Unlock(); + child_thread.join(); + } + } +} + +TEST_F(PerfContextTest, FalseDBMutexWait) { + SetPerfLevel(kEnableTime); + int stats_code[] = {0, static_cast(DB_MUTEX_WAIT_MICROS)}; + for (int c = 0; c < 2; ++c) { + InstrumentedMutex mutex(nullptr, SystemClock::Default().get(), + stats_code[c]); + InstrumentedCondVar lock(&mutex); + get_perf_context()->Reset(); + mutex.Lock(); + lock.TimedWait(100); + mutex.Unlock(); + if (stats_code[c] == static_cast(DB_MUTEX_WAIT_MICROS)) { + // increment the counter only when it's a DB Mutex + ASSERT_GT(get_perf_context()->db_condition_wait_nanos, 0); + } else { + ASSERT_EQ(get_perf_context()->db_condition_wait_nanos, 0); + } + } +} + +TEST_F(PerfContextTest, ToString) { + get_perf_context()->Reset(); + get_perf_context()->block_read_count = 12345; + + std::string zero_included = get_perf_context()->ToString(); + ASSERT_NE(std::string::npos, zero_included.find("= 0")); + ASSERT_NE(std::string::npos, zero_included.find("= 12345")); + + std::string zero_excluded = get_perf_context()->ToString(true); + ASSERT_EQ(std::string::npos, zero_excluded.find("= 0")); + ASSERT_NE(std::string::npos, zero_excluded.find("= 12345")); +} + +TEST_F(PerfContextTest, MergeOperatorTime) { + ASSERT_OK(DestroyDB(kDbName, Options())); + DB* db; + Options options; + options.create_if_missing = true; + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + Status s = DB::Open(options, kDbName, &db); + EXPECT_OK(s); + + std::string val; + ASSERT_OK(db->Merge(WriteOptions(), "k1", "val1")); + ASSERT_OK(db->Merge(WriteOptions(), "k1", "val2")); + ASSERT_OK(db->Merge(WriteOptions(), "k1", "val3")); + ASSERT_OK(db->Merge(WriteOptions(), "k1", "val4")); + + SetPerfLevel(kEnableTime); + get_perf_context()->Reset(); + ASSERT_OK(db->Get(ReadOptions(), "k1", &val)); +#ifdef OS_SOLARIS + for (int i = 0; i < 100; i++) { + ASSERT_OK(db->Get(ReadOptions(), "k1", &val)); + } +#endif + EXPECT_GT(get_perf_context()->merge_operator_time_nanos, 0); + + ASSERT_OK(db->Flush(FlushOptions())); + + get_perf_context()->Reset(); + ASSERT_OK(db->Get(ReadOptions(), "k1", &val)); +#ifdef OS_SOLARIS + for (int i = 0; i < 100; i++) { + ASSERT_OK(db->Get(ReadOptions(), "k1", &val)); + } +#endif + EXPECT_GT(get_perf_context()->merge_operator_time_nanos, 0); + + ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + get_perf_context()->Reset(); + ASSERT_OK(db->Get(ReadOptions(), "k1", &val)); +#ifdef OS_SOLARIS + for (int i = 0; i < 100; i++) { + ASSERT_OK(db->Get(ReadOptions(), "k1", &val)); + } +#endif + EXPECT_GT(get_perf_context()->merge_operator_time_nanos, 0); + + delete db; +} + +TEST_F(PerfContextTest, CopyAndMove) { + // Assignment operator + { + get_perf_context()->Reset(); + get_perf_context()->EnablePerLevelPerfContext(); + PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_useful, 1, 5); + ASSERT_EQ( + 1, + (*(get_perf_context()->level_to_perf_context))[5].bloom_filter_useful); + PerfContext perf_context_assign; + perf_context_assign = *get_perf_context(); + ASSERT_EQ( + 1, + (*(perf_context_assign.level_to_perf_context))[5].bloom_filter_useful); + get_perf_context()->ClearPerLevelPerfContext(); + get_perf_context()->Reset(); + ASSERT_EQ( + 1, + (*(perf_context_assign.level_to_perf_context))[5].bloom_filter_useful); + perf_context_assign.ClearPerLevelPerfContext(); + perf_context_assign.Reset(); + } + // Copy constructor + { + get_perf_context()->Reset(); + get_perf_context()->EnablePerLevelPerfContext(); + PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_useful, 1, 5); + ASSERT_EQ( + 1, + (*(get_perf_context()->level_to_perf_context))[5].bloom_filter_useful); + PerfContext perf_context_copy(*get_perf_context()); + ASSERT_EQ( + 1, (*(perf_context_copy.level_to_perf_context))[5].bloom_filter_useful); + get_perf_context()->ClearPerLevelPerfContext(); + get_perf_context()->Reset(); + ASSERT_EQ( + 1, (*(perf_context_copy.level_to_perf_context))[5].bloom_filter_useful); + perf_context_copy.ClearPerLevelPerfContext(); + perf_context_copy.Reset(); + } + // Move constructor + { + get_perf_context()->Reset(); + get_perf_context()->EnablePerLevelPerfContext(); + PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_useful, 1, 5); + ASSERT_EQ( + 1, + (*(get_perf_context()->level_to_perf_context))[5].bloom_filter_useful); + PerfContext perf_context_move = std::move(*get_perf_context()); + ASSERT_EQ( + 1, (*(perf_context_move.level_to_perf_context))[5].bloom_filter_useful); + get_perf_context()->ClearPerLevelPerfContext(); + get_perf_context()->Reset(); + ASSERT_EQ( + 1, (*(perf_context_move.level_to_perf_context))[5].bloom_filter_useful); + perf_context_move.ClearPerLevelPerfContext(); + perf_context_move.Reset(); + } +} + +TEST_F(PerfContextTest, PerfContextDisableEnable) { + get_perf_context()->Reset(); + get_perf_context()->EnablePerLevelPerfContext(); + PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_full_positive, 1, 0); + get_perf_context()->DisablePerLevelPerfContext(); + PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_useful, 1, 5); + get_perf_context()->EnablePerLevelPerfContext(); + PERF_COUNTER_BY_LEVEL_ADD(block_cache_hit_count, 1, 0); + get_perf_context()->DisablePerLevelPerfContext(); + PerfContext perf_context_copy(*get_perf_context()); + ASSERT_EQ(1, (*(perf_context_copy.level_to_perf_context))[0] + .bloom_filter_full_positive); + // this was set when per level perf context is disabled, should not be copied + ASSERT_NE( + 1, (*(perf_context_copy.level_to_perf_context))[5].bloom_filter_useful); + ASSERT_EQ( + 1, (*(perf_context_copy.level_to_perf_context))[0].block_cache_hit_count); + perf_context_copy.ClearPerLevelPerfContext(); + perf_context_copy.Reset(); + get_perf_context()->ClearPerLevelPerfContext(); + get_perf_context()->Reset(); +} + +TEST_F(PerfContextTest, PerfContextByLevelGetSet) { + get_perf_context()->Reset(); + get_perf_context()->EnablePerLevelPerfContext(); + PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_full_positive, 1, 0); + PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_useful, 1, 5); + PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_useful, 1, 7); + PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_useful, 1, 7); + PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_full_true_positive, 1, 2); + PERF_COUNTER_BY_LEVEL_ADD(block_cache_hit_count, 1, 0); + PERF_COUNTER_BY_LEVEL_ADD(block_cache_hit_count, 5, 2); + PERF_COUNTER_BY_LEVEL_ADD(block_cache_miss_count, 2, 3); + PERF_COUNTER_BY_LEVEL_ADD(block_cache_miss_count, 4, 1); + ASSERT_EQ( + 0, (*(get_perf_context()->level_to_perf_context))[0].bloom_filter_useful); + ASSERT_EQ( + 1, (*(get_perf_context()->level_to_perf_context))[5].bloom_filter_useful); + ASSERT_EQ( + 2, (*(get_perf_context()->level_to_perf_context))[7].bloom_filter_useful); + ASSERT_EQ(1, (*(get_perf_context()->level_to_perf_context))[0] + .bloom_filter_full_positive); + ASSERT_EQ(1, (*(get_perf_context()->level_to_perf_context))[2] + .bloom_filter_full_true_positive); + ASSERT_EQ( + 1, + (*(get_perf_context()->level_to_perf_context))[0].block_cache_hit_count); + ASSERT_EQ( + 5, + (*(get_perf_context()->level_to_perf_context))[2].block_cache_hit_count); + ASSERT_EQ( + 2, + (*(get_perf_context()->level_to_perf_context))[3].block_cache_miss_count); + ASSERT_EQ( + 4, + (*(get_perf_context()->level_to_perf_context))[1].block_cache_miss_count); + std::string zero_excluded = get_perf_context()->ToString(true); + ASSERT_NE(std::string::npos, + zero_excluded.find("bloom_filter_useful = 1@level5, 2@level7")); + ASSERT_NE(std::string::npos, + zero_excluded.find("bloom_filter_full_positive = 1@level0")); + ASSERT_NE(std::string::npos, + zero_excluded.find("bloom_filter_full_true_positive = 1@level2")); + ASSERT_NE(std::string::npos, + zero_excluded.find("block_cache_hit_count = 1@level0, 5@level2")); + ASSERT_NE(std::string::npos, + zero_excluded.find("block_cache_miss_count = 4@level1, 2@level3")); +} + +TEST_F(PerfContextTest, CPUTimer) { + if (SystemClock::Default()->CPUNanos() == 0) { + ROCKSDB_GTEST_SKIP("Target without CPUNanos support"); + return; + } + + ASSERT_OK(DestroyDB(kDbName, Options())); + auto db = OpenDb(); + WriteOptions write_options; + ReadOptions read_options; + SetPerfLevel(PerfLevel::kEnableTimeAndCPUTimeExceptForMutex); + + std::string max_str = "0"; + for (int i = 0; i < FLAGS_total_keys; ++i) { + std::string i_str = std::to_string(i); + std::string key = "k" + i_str; + std::string value = "v" + i_str; + max_str = max_str > i_str ? max_str : i_str; + + ASSERT_OK(db->Put(write_options, key, value)); + } + std::string last_key = "k" + max_str; + std::string last_value = "v" + max_str; + + { + // Get + get_perf_context()->Reset(); + std::string value; + ASSERT_OK(db->Get(read_options, "k0", &value)); + ASSERT_EQ(value, "v0"); + + if (FLAGS_verbose) { + std::cout << "Get CPU time nanos: " << get_perf_context()->get_cpu_nanos + << "ns\n"; + } + + // Iter + std::unique_ptr iter(db->NewIterator(read_options)); + + // Seek + get_perf_context()->Reset(); + iter->Seek(last_key); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(last_value, iter->value().ToString()); + + if (FLAGS_verbose) { + std::cout << "Iter Seek CPU time nanos: " + << get_perf_context()->iter_seek_cpu_nanos << "ns\n"; + } + + // SeekForPrev + get_perf_context()->Reset(); + iter->SeekForPrev(last_key); + ASSERT_TRUE(iter->Valid()); + + if (FLAGS_verbose) { + std::cout << "Iter SeekForPrev CPU time nanos: " + << get_perf_context()->iter_seek_cpu_nanos << "ns\n"; + } + + // SeekToLast + get_perf_context()->Reset(); + iter->SeekToLast(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(last_value, iter->value().ToString()); + + if (FLAGS_verbose) { + std::cout << "Iter SeekToLast CPU time nanos: " + << get_perf_context()->iter_seek_cpu_nanos << "ns\n"; + } + + // SeekToFirst + get_perf_context()->Reset(); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("v0", iter->value().ToString()); + + if (FLAGS_verbose) { + std::cout << "Iter SeekToFirst CPU time nanos: " + << get_perf_context()->iter_seek_cpu_nanos << "ns\n"; + } + + // Next + get_perf_context()->Reset(); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("v1", iter->value().ToString()); + + if (FLAGS_verbose) { + std::cout << "Iter Next CPU time nanos: " + << get_perf_context()->iter_next_cpu_nanos << "ns\n"; + } + + // Prev + get_perf_context()->Reset(); + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("v0", iter->value().ToString()); + + if (FLAGS_verbose) { + std::cout << "Iter Prev CPU time nanos: " + << get_perf_context()->iter_prev_cpu_nanos << "ns\n"; + } + + // monotonically increasing + get_perf_context()->Reset(); + auto count = get_perf_context()->iter_seek_cpu_nanos; + for (int i = 0; i < FLAGS_total_keys; ++i) { + iter->Seek("k" + std::to_string(i)); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("v" + std::to_string(i), iter->value().ToString()); + auto next_count = get_perf_context()->iter_seek_cpu_nanos; + ASSERT_GT(next_count, count); + count = next_count; + } + + // iterator creation/destruction; multiple iterators + { + std::unique_ptr iter2(db->NewIterator(read_options)); + ASSERT_EQ(count, get_perf_context()->iter_seek_cpu_nanos); + iter2->Seek(last_key); + ASSERT_TRUE(iter2->Valid()); + ASSERT_EQ(last_value, iter2->value().ToString()); + ASSERT_GT(get_perf_context()->iter_seek_cpu_nanos, count); + count = get_perf_context()->iter_seek_cpu_nanos; + } + ASSERT_EQ(count, get_perf_context()->iter_seek_cpu_nanos); + } +} + +TEST_F(PerfContextTest, MergeOperandCount) { + ASSERT_OK(DestroyDB(kDbName, Options())); + + DB* db = nullptr; + Options options; + options.create_if_missing = true; + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + + ASSERT_OK(DB::Open(options, kDbName, &db)); + std::unique_ptr db_guard(db); + + constexpr size_t num_keys = 3; + const std::string key_prefix("key"); + const std::string value_prefix("value"); + + std::vector keys; + keys.reserve(num_keys); + + for (size_t i = 0; i < num_keys; ++i) { + keys.emplace_back(key_prefix + std::to_string(i)); + } + + // Write three keys with one Put each followed by 1, 2, and 3 + // Merge operations respectively. + constexpr size_t total_merges = num_keys * (num_keys + 1) / 2; + + std::vector snapshots; + snapshots.reserve(total_merges); + + for (size_t i = 0; i < num_keys; ++i) { + const std::string suffix = std::to_string(i); + const std::string value = value_prefix + suffix; + + ASSERT_OK(db->Put(WriteOptions(), keys[i], value)); + + for (size_t j = 0; j <= i; ++j) { + // Take a snapshot before each Merge so they are preserved and not + // collapsed during flush. + snapshots.emplace_back(db); + + ASSERT_OK(db->Merge(WriteOptions(), keys[i], value + std::to_string(j))); + } + } + + auto verify = [&]() { + get_perf_context()->Reset(); + + for (size_t i = 0; i < num_keys; ++i) { + // Get + { + PinnableSlice result; + ASSERT_OK(db->Get(ReadOptions(), db->DefaultColumnFamily(), keys[i], + &result)); + ASSERT_EQ(get_perf_context()->internal_merge_point_lookup_count, i + 1); + + get_perf_context()->Reset(); + } + + // GetEntity + { + PinnableWideColumns result; + ASSERT_OK(db->GetEntity(ReadOptions(), db->DefaultColumnFamily(), + keys[i], &result)); + ASSERT_EQ(get_perf_context()->internal_merge_point_lookup_count, i + 1); + + get_perf_context()->Reset(); + } + } + + { + std::vector key_slices; + key_slices.reserve(num_keys); + + for (size_t i = 0; i < num_keys; ++i) { + key_slices.emplace_back(keys[i]); + } + + // MultiGet + { + std::vector results(num_keys); + std::vector statuses(num_keys); + + db->MultiGet(ReadOptions(), db->DefaultColumnFamily(), num_keys, + &key_slices[0], &results[0], &statuses[0]); + + for (size_t i = 0; i < num_keys; ++i) { + ASSERT_OK(statuses[i]); + } + + ASSERT_EQ(get_perf_context()->internal_merge_point_lookup_count, + total_merges); + + get_perf_context()->Reset(); + } + + // MultiGetEntity + { + std::vector results(num_keys); + std::vector statuses(num_keys); + + db->MultiGetEntity(ReadOptions(), db->DefaultColumnFamily(), num_keys, + &key_slices[0], &results[0], &statuses[0]); + + for (size_t i = 0; i < num_keys; ++i) { + ASSERT_OK(statuses[i]); + } + + ASSERT_EQ(get_perf_context()->internal_merge_point_lookup_count, + total_merges); + + get_perf_context()->Reset(); + } + } + + std::unique_ptr it(db->NewIterator(ReadOptions())); + + // Forward iteration + { + size_t i = 0; + + for (it->SeekToFirst(); it->Valid(); it->Next(), ++i) { + ASSERT_EQ(it->key(), keys[i]); + ASSERT_EQ(get_perf_context()->internal_merge_count, i + 1); + + get_perf_context()->Reset(); + } + } + + // Backward iteration + { + size_t i = num_keys - 1; + + for (it->SeekToLast(); it->Valid(); it->Prev(), --i) { + ASSERT_EQ(it->key(), keys[i]); + ASSERT_EQ(get_perf_context()->internal_merge_count, i + 1); + + get_perf_context()->Reset(); + } + } + }; + + // Verify counters when reading from memtable + verify(); + + // Verify counters when reading from table files + db->Flush(FlushOptions()); + + verify(); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + + for (int i = 1; i < argc; i++) { + int n; + char junk; + + if (sscanf(argv[i], "--write_buffer_size=%d%c", &n, &junk) == 1) { + FLAGS_write_buffer_size = n; + } + + if (sscanf(argv[i], "--total_keys=%d%c", &n, &junk) == 1) { + FLAGS_total_keys = n; + } + + if (sscanf(argv[i], "--random_key=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_random_key = n; + } + + if (sscanf(argv[i], "--use_set_based_memetable=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_use_set_based_memetable = n; + } + + if (sscanf(argv[i], "--verbose=%d%c", &n, &junk) == 1 && + (n == 0 || n == 1)) { + FLAGS_verbose = n; + } + } + + if (FLAGS_verbose) { + std::cout << kDbName << "\n"; + } + + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/periodic_task_scheduler.cc b/librocksdb-sys/rocksdb/db/periodic_task_scheduler.cc new file mode 100644 index 0000000..1306f45 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/periodic_task_scheduler.cc @@ -0,0 +1,111 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/periodic_task_scheduler.h" + +#include "rocksdb/system_clock.h" + +namespace ROCKSDB_NAMESPACE { + +// `timer_mutex` is a global mutex serves 3 purposes currently: +// (1) to ensure calls to `Start()` and `Shutdown()` are serialized, as +// they are currently not implemented in a thread-safe way; and +// (2) to ensure the `Timer::Add()`s and `Timer::Start()` run atomically, and +// the `Timer::Cancel()`s and `Timer::Shutdown()` run atomically. +// (3) protect tasks_map_ in PeriodicTaskScheduler +// Note: It's not efficient to have a static global mutex, for +// PeriodicTaskScheduler it should be okay, as the operations are called +// infrequently. +static port::Mutex timer_mutex; + +static const std::map kDefaultPeriodSeconds = { + {PeriodicTaskType::kDumpStats, kInvalidPeriodSec}, + {PeriodicTaskType::kPersistStats, kInvalidPeriodSec}, + {PeriodicTaskType::kFlushInfoLog, 10}, + {PeriodicTaskType::kRecordSeqnoTime, kInvalidPeriodSec}, +}; + +static const std::map kPeriodicTaskTypeNames = { + {PeriodicTaskType::kDumpStats, "dump_st"}, + {PeriodicTaskType::kPersistStats, "pst_st"}, + {PeriodicTaskType::kFlushInfoLog, "flush_info_log"}, + {PeriodicTaskType::kRecordSeqnoTime, "record_seq_time"}, +}; + +Status PeriodicTaskScheduler::Register(PeriodicTaskType task_type, + const PeriodicTaskFunc& fn) { + return Register(task_type, fn, kDefaultPeriodSeconds.at(task_type)); +} + +Status PeriodicTaskScheduler::Register(PeriodicTaskType task_type, + const PeriodicTaskFunc& fn, + uint64_t repeat_period_seconds) { + MutexLock l(&timer_mutex); + static std::atomic initial_delay(0); + + if (repeat_period_seconds == kInvalidPeriodSec) { + return Status::InvalidArgument("Invalid task repeat period"); + } + auto it = tasks_map_.find(task_type); + if (it != tasks_map_.end()) { + // the task already exists and it's the same, no update needed + if (it->second.repeat_every_sec == repeat_period_seconds) { + return Status::OK(); + } + // cancel the existing one before register new one + timer_->Cancel(it->second.name); + tasks_map_.erase(it); + } + + timer_->Start(); + // put task type name as prefix, for easy debug + std::string unique_id = + kPeriodicTaskTypeNames.at(task_type) + std::to_string(id_++); + + bool succeeded = timer_->Add( + fn, unique_id, + (initial_delay.fetch_add(1) % repeat_period_seconds) * kMicrosInSecond, + repeat_period_seconds * kMicrosInSecond); + if (!succeeded) { + return Status::Aborted("Failed to register periodic task"); + } + auto result = tasks_map_.try_emplace( + task_type, TaskInfo{unique_id, repeat_period_seconds}); + if (!result.second) { + return Status::Aborted("Failed to add periodic task"); + }; + return Status::OK(); +} + +Status PeriodicTaskScheduler::Unregister(PeriodicTaskType task_type) { + MutexLock l(&timer_mutex); + auto it = tasks_map_.find(task_type); + if (it != tasks_map_.end()) { + timer_->Cancel(it->second.name); + tasks_map_.erase(it); + } + if (!timer_->HasPendingTask()) { + timer_->Shutdown(); + } + return Status::OK(); +} + +Timer* PeriodicTaskScheduler::Default() { + static Timer timer(SystemClock::Default().get()); + return &timer; +} + +#ifndef NDEBUG +void PeriodicTaskScheduler::TEST_OverrideTimer(SystemClock* clock) { + static Timer test_timer(clock); + test_timer.TEST_OverrideTimer(clock); + MutexLock l(&timer_mutex); + timer_ = &test_timer; +} +#endif // NDEBUG + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/db/periodic_task_scheduler.h b/librocksdb-sys/rocksdb/db/periodic_task_scheduler.h new file mode 100644 index 0000000..4d129a6 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/periodic_task_scheduler.h @@ -0,0 +1,108 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + + +#include "util/timer.h" + +namespace ROCKSDB_NAMESPACE { +class SystemClock; + +using PeriodicTaskFunc = std::function; + +constexpr uint64_t kInvalidPeriodSec = 0; + +// List of task types +enum class PeriodicTaskType : uint8_t { + kDumpStats = 0, + kPersistStats, + kFlushInfoLog, + kRecordSeqnoTime, + kMax, +}; + +// PeriodicTaskScheduler contains the periodic task scheduled from the DB +// instance. It's used to schedule/unschedule DumpStats(), PersistStats(), +// FlushInfoLog(), etc. Each type of the task can only have one instance, +// re-register the same task type would only update the repeat period. +// +// Internally, it uses a global single threaded timer object to run the periodic +// task functions. Timer thread will always be started since the info log +// flushing cannot be disabled. +class PeriodicTaskScheduler { + public: + explicit PeriodicTaskScheduler() = default; + + PeriodicTaskScheduler(const PeriodicTaskScheduler&) = delete; + PeriodicTaskScheduler(PeriodicTaskScheduler&&) = delete; + PeriodicTaskScheduler& operator=(const PeriodicTaskScheduler&) = delete; + PeriodicTaskScheduler& operator=(PeriodicTaskScheduler&&) = delete; + + // Register a task with its default repeat period + Status Register(PeriodicTaskType task_type, const PeriodicTaskFunc& fn); + + // Register a task with specified repeat period. 0 is an invalid argument + // (kInvalidPeriodSec). To stop the task, please use Unregister() specifically + Status Register(PeriodicTaskType task_type, const PeriodicTaskFunc& fn, + uint64_t repeat_period_seconds); + + // Unregister the task + Status Unregister(PeriodicTaskType task_type); + +#ifndef NDEBUG + // Override the timer for the unittest + void TEST_OverrideTimer(SystemClock* clock); + + // Call Timer TEST_WaitForRun() which wait until Timer starting waiting. + void TEST_WaitForRun(const std::function& callback) const { + if (timer_ != nullptr) { + timer_->TEST_WaitForRun(callback); + } + } + + // Get global valid task number in the Timer + size_t TEST_GetValidTaskNum() const { + if (timer_ != nullptr) { + return timer_->TEST_GetPendingTaskNum(); + } + return 0; + } + + // If it has the specified task type registered + bool TEST_HasTask(PeriodicTaskType task_type) const { + auto it = tasks_map_.find(task_type); + return it != tasks_map_.end(); + } +#endif // NDEBUG + + private: + // default global Timer instance + static Timer* Default(); + + // Internal structure to store task information + struct TaskInfo { + TaskInfo(std::string _name, uint64_t _repeat_every_sec) + : name(std::move(_name)), repeat_every_sec(_repeat_every_sec) {} + std::string name; + uint64_t repeat_every_sec; + }; + + // Internal tasks map + std::map tasks_map_; + + // Global timer pointer, which doesn't support synchronous add/cancel tasks + // so having a global `timer_mutex` for add/cancel task. + Timer* timer_ = Default(); + + // Global task id, protected by the global `timer_mutex` + inline static uint64_t id_; + + static constexpr uint64_t kMicrosInSecond = 1000U * 1000U; +}; + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/db/periodic_task_scheduler_test.cc b/librocksdb-sys/rocksdb/db/periodic_task_scheduler_test.cc new file mode 100644 index 0000000..c1205bc --- /dev/null +++ b/librocksdb-sys/rocksdb/db/periodic_task_scheduler_test.cc @@ -0,0 +1,229 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/periodic_task_scheduler.h" + +#include "db/db_test_util.h" +#include "env/composite_env_wrapper.h" +#include "test_util/mock_time_env.h" + +namespace ROCKSDB_NAMESPACE { + +class PeriodicTaskSchedulerTest : public DBTestBase { + public: + PeriodicTaskSchedulerTest() + : DBTestBase("periodic_task_scheduler_test", /*env_do_fsync=*/true) { + mock_clock_ = std::make_shared(env_->GetSystemClock()); + mock_env_.reset(new CompositeEnvWrapper(env_, mock_clock_)); + } + + protected: + std::unique_ptr mock_env_; + std::shared_ptr mock_clock_; + + void SetUp() override { + mock_clock_->InstallTimedWaitFixCallback(); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::StartPeriodicTaskScheduler:Init", [&](void* arg) { + auto periodic_task_scheduler_ptr = + reinterpret_cast(arg); + periodic_task_scheduler_ptr->TEST_OverrideTimer(mock_clock_.get()); + }); + } +}; + +TEST_F(PeriodicTaskSchedulerTest, Basic) { + constexpr unsigned int kPeriodSec = 10; + Close(); + Options options; + options.stats_dump_period_sec = kPeriodSec; + options.stats_persist_period_sec = kPeriodSec; + options.create_if_missing = true; + options.env = mock_env_.get(); + + int dump_st_counter = 0; + SyncPoint::GetInstance()->SetCallBack("DBImpl::DumpStats:StartRunning", + [&](void*) { dump_st_counter++; }); + + int pst_st_counter = 0; + SyncPoint::GetInstance()->SetCallBack("DBImpl::PersistStats:StartRunning", + [&](void*) { pst_st_counter++; }); + + int flush_info_log_counter = 0; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::FlushInfoLog:StartRunning", + [&](void*) { flush_info_log_counter++; }); + SyncPoint::GetInstance()->EnableProcessing(); + + Reopen(options); + + ASSERT_EQ(kPeriodSec, dbfull()->GetDBOptions().stats_dump_period_sec); + ASSERT_EQ(kPeriodSec, dbfull()->GetDBOptions().stats_persist_period_sec); + + ASSERT_GT(kPeriodSec, 1u); + dbfull()->TEST_WaitForPeriodicTaskRun([&] { + mock_clock_->MockSleepForSeconds(static_cast(kPeriodSec) - 1); + }); + + const PeriodicTaskScheduler& scheduler = + dbfull()->TEST_GetPeriodicTaskScheduler(); + ASSERT_EQ(3, scheduler.TEST_GetValidTaskNum()); + + ASSERT_EQ(1, dump_st_counter); + ASSERT_EQ(1, pst_st_counter); + ASSERT_EQ(1, flush_info_log_counter); + + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(kPeriodSec)); }); + + ASSERT_EQ(2, dump_st_counter); + ASSERT_EQ(2, pst_st_counter); + ASSERT_EQ(2, flush_info_log_counter); + + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(kPeriodSec)); }); + + ASSERT_EQ(3, dump_st_counter); + ASSERT_EQ(3, pst_st_counter); + ASSERT_EQ(3, flush_info_log_counter); + + // Disable scheduler with SetOption + ASSERT_OK(dbfull()->SetDBOptions( + {{"stats_dump_period_sec", "0"}, {"stats_persist_period_sec", "0"}})); + ASSERT_EQ(0u, dbfull()->GetDBOptions().stats_dump_period_sec); + ASSERT_EQ(0u, dbfull()->GetDBOptions().stats_persist_period_sec); + + // Info log flush should still run. + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(kPeriodSec)); }); + ASSERT_EQ(3, dump_st_counter); + ASSERT_EQ(3, pst_st_counter); + ASSERT_EQ(4, flush_info_log_counter); + + ASSERT_EQ(1u, scheduler.TEST_GetValidTaskNum()); + + // Re-enable one task + ASSERT_OK(dbfull()->SetDBOptions({{"stats_dump_period_sec", "5"}})); + ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_dump_period_sec); + ASSERT_EQ(0u, dbfull()->GetDBOptions().stats_persist_period_sec); + + ASSERT_EQ(2, scheduler.TEST_GetValidTaskNum()); + + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(kPeriodSec)); }); + ASSERT_EQ(4, dump_st_counter); + ASSERT_EQ(3, pst_st_counter); + ASSERT_EQ(5, flush_info_log_counter); + + Close(); +} + +TEST_F(PeriodicTaskSchedulerTest, MultiInstances) { + constexpr int kPeriodSec = 5; + const int kInstanceNum = 10; + + Close(); + Options options; + options.stats_dump_period_sec = kPeriodSec; + options.stats_persist_period_sec = kPeriodSec; + options.create_if_missing = true; + options.env = mock_env_.get(); + + int dump_st_counter = 0; + SyncPoint::GetInstance()->SetCallBack("DBImpl::DumpStats:2", + [&](void*) { dump_st_counter++; }); + + int pst_st_counter = 0; + SyncPoint::GetInstance()->SetCallBack("DBImpl::PersistStats:StartRunning", + [&](void*) { pst_st_counter++; }); + SyncPoint::GetInstance()->EnableProcessing(); + + auto dbs = std::vector(kInstanceNum); + for (int i = 0; i < kInstanceNum; i++) { + ASSERT_OK( + DB::Open(options, test::PerThreadDBPath(std::to_string(i)), &(dbs[i]))); + } + + auto dbi = static_cast_with_check(dbs[kInstanceNum - 1]); + + const PeriodicTaskScheduler& scheduler = dbi->TEST_GetPeriodicTaskScheduler(); + ASSERT_EQ(kInstanceNum * 3, scheduler.TEST_GetValidTaskNum()); + + int expected_run = kInstanceNum; + dbi->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); + ASSERT_EQ(expected_run, dump_st_counter); + ASSERT_EQ(expected_run, pst_st_counter); + + expected_run += kInstanceNum; + dbi->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); + ASSERT_EQ(expected_run, dump_st_counter); + ASSERT_EQ(expected_run, pst_st_counter); + + expected_run += kInstanceNum; + dbi->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); + ASSERT_EQ(expected_run, dump_st_counter); + ASSERT_EQ(expected_run, pst_st_counter); + + int half = kInstanceNum / 2; + for (int i = 0; i < half; i++) { + delete dbs[i]; + } + + expected_run += (kInstanceNum - half) * 2; + + dbi->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); + dbi->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); + ASSERT_EQ(expected_run, dump_st_counter); + ASSERT_EQ(expected_run, pst_st_counter); + + for (int i = half; i < kInstanceNum; i++) { + ASSERT_OK(dbs[i]->Close()); + delete dbs[i]; + } +} + +TEST_F(PeriodicTaskSchedulerTest, MultiEnv) { + constexpr int kDumpPeriodSec = 5; + constexpr int kPersistPeriodSec = 10; + Close(); + Options options1; + options1.stats_dump_period_sec = kDumpPeriodSec; + options1.stats_persist_period_sec = kPersistPeriodSec; + options1.create_if_missing = true; + options1.env = mock_env_.get(); + + Reopen(options1); + + std::unique_ptr mock_env2( + new CompositeEnvWrapper(Env::Default(), mock_clock_)); + Options options2; + options2.stats_dump_period_sec = kDumpPeriodSec; + options2.stats_persist_period_sec = kPersistPeriodSec; + options2.create_if_missing = true; + options1.env = mock_env2.get(); + + std::string dbname = test::PerThreadDBPath("multi_env_test"); + DB* db; + ASSERT_OK(DB::Open(options2, dbname, &db)); + + ASSERT_OK(db->Close()); + delete db; + Close(); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/pinned_iterators_manager.h b/librocksdb-sys/rocksdb/db/pinned_iterators_manager.h new file mode 100644 index 0000000..0fcf231 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/pinned_iterators_manager.h @@ -0,0 +1,92 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#pragma once +#include +#include +#include +#include + +#include "table/internal_iterator.h" + +namespace ROCKSDB_NAMESPACE { + +// PinnedIteratorsManager will be notified whenever we need to pin an Iterator +// and it will be responsible for deleting pinned Iterators when they are +// not needed anymore. +class PinnedIteratorsManager : public Cleanable { + public: + PinnedIteratorsManager() : pinning_enabled(false) {} + ~PinnedIteratorsManager() { + if (pinning_enabled) { + ReleasePinnedData(); + } + } + + // Move constructor and move assignment is allowed. + PinnedIteratorsManager(PinnedIteratorsManager&& other) noexcept = default; + PinnedIteratorsManager& operator=(PinnedIteratorsManager&& other) noexcept = + default; + + // Enable Iterators pinning + void StartPinning() { + assert(pinning_enabled == false); + pinning_enabled = true; + } + + // Is pinning enabled ? + bool PinningEnabled() { return pinning_enabled; } + + // Take ownership of iter and delete it when ReleasePinnedData() is called + void PinIterator(InternalIterator* iter, bool arena = false) { + if (arena) { + PinPtr(iter, &PinnedIteratorsManager::ReleaseArenaInternalIterator); + } else { + PinPtr(iter, &PinnedIteratorsManager::ReleaseInternalIterator); + } + } + + using ReleaseFunction = void (*)(void* arg1); + void PinPtr(void* ptr, ReleaseFunction release_func) { + assert(pinning_enabled); + if (ptr == nullptr) { + return; + } + pinned_ptrs_.emplace_back(ptr, release_func); + } + + // Release pinned Iterators + inline void ReleasePinnedData() { + assert(pinning_enabled == true); + pinning_enabled = false; + + // Remove duplicate pointers + std::sort(pinned_ptrs_.begin(), pinned_ptrs_.end()); + auto unique_end = std::unique(pinned_ptrs_.begin(), pinned_ptrs_.end()); + + for (auto i = pinned_ptrs_.begin(); i != unique_end; ++i) { + void* ptr = i->first; + ReleaseFunction release_func = i->second; + release_func(ptr); + } + pinned_ptrs_.clear(); + // Also do cleanups from the base Cleanable + Cleanable::Reset(); + } + + private: + static void ReleaseInternalIterator(void* ptr) { + delete reinterpret_cast(ptr); + } + + static void ReleaseArenaInternalIterator(void* ptr) { + reinterpret_cast(ptr)->~InternalIterator(); + } + + bool pinning_enabled; + std::vector> pinned_ptrs_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/plain_table_db_test.cc b/librocksdb-sys/rocksdb/db/plain_table_db_test.cc new file mode 100644 index 0000000..d117639 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/plain_table_db_test.cc @@ -0,0 +1,1349 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + + +#include +#include + +#include "db/db_impl/db_impl.h" +#include "db/version_set.h" +#include "db/write_batch_internal.h" +#include "file/filename.h" +#include "rocksdb/cache.h" +#include "rocksdb/compaction_filter.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/table.h" +#include "table/meta_blocks.h" +#include "table/plain/plain_table_bloom.h" +#include "table/plain/plain_table_factory.h" +#include "table/plain/plain_table_key_coding.h" +#include "table/plain/plain_table_reader.h" +#include "table/table_builder.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/cast_util.h" +#include "util/hash.h" +#include "util/mutexlock.h" +#include "util/random.h" +#include "util/string_util.h" +#include "utilities/merge_operators.h" + +namespace ROCKSDB_NAMESPACE { +class PlainTableKeyDecoderTest : public testing::Test {}; + +TEST_F(PlainTableKeyDecoderTest, ReadNonMmap) { + Random rnd(301); + const uint32_t kLength = 2222; + std::string tmp = rnd.RandomString(kLength); + Slice contents(tmp); + test::StringSource* string_source = + new test::StringSource(contents, 0, false); + std::unique_ptr holder(string_source); + std::unique_ptr file_reader( + new RandomAccessFileReader(std::move(holder), "test")); + std::unique_ptr file_info( + new PlainTableReaderFileInfo(std::move(file_reader), EnvOptions(), + kLength)); + + { + PlainTableFileReader reader(file_info.get()); + + const uint32_t kReadSize = 77; + for (uint32_t pos = 0; pos < kLength; pos += kReadSize) { + uint32_t read_size = std::min(kLength - pos, kReadSize); + Slice out; + ASSERT_TRUE(reader.Read(pos, read_size, &out)); + ASSERT_EQ(0, out.compare(tmp.substr(pos, read_size))); + } + + ASSERT_LT(uint32_t(string_source->total_reads()), kLength / kReadSize / 2); + } + + std::vector>> reads = { + {{600, 30}, {590, 30}, {600, 20}, {600, 40}}, + {{800, 20}, {100, 20}, {500, 20}, {1500, 20}, {100, 20}, {80, 20}}, + {{1000, 20}, {500, 20}, {1000, 50}}, + {{1000, 20}, {500, 20}, {500, 20}}, + {{1000, 20}, {500, 20}, {200, 20}, {500, 20}}, + {{1000, 20}, {500, 20}, {200, 20}, {1000, 50}}, + {{600, 500}, {610, 20}, {100, 20}}, + {{500, 100}, {490, 100}, {550, 50}}, + }; + + std::vector num_file_reads = {2, 6, 2, 2, 4, 3, 2, 2}; + + for (size_t i = 0; i < reads.size(); i++) { + string_source->set_total_reads(0); + PlainTableFileReader reader(file_info.get()); + for (auto p : reads[i]) { + Slice out; + ASSERT_TRUE(reader.Read(p.first, p.second, &out)); + ASSERT_EQ(0, out.compare(tmp.substr(p.first, p.second))); + } + ASSERT_EQ(num_file_reads[i], string_source->total_reads()); + } +} + +class PlainTableDBTest : public testing::Test, + public testing::WithParamInterface { + protected: + private: + std::string dbname_; + Env* env_; + DB* db_; + + bool mmap_mode_; + Options last_options_; + + public: + PlainTableDBTest() : env_(Env::Default()) {} + + ~PlainTableDBTest() override { + delete db_; + EXPECT_OK(DestroyDB(dbname_, Options())); + } + + void SetUp() override { + mmap_mode_ = GetParam(); + dbname_ = test::PerThreadDBPath("plain_table_db_test"); + EXPECT_OK(DestroyDB(dbname_, Options())); + db_ = nullptr; + Reopen(); + } + + // Return the current option configuration. + Options CurrentOptions() { + Options options; + options.level_compaction_dynamic_level_bytes = false; + + PlainTableOptions plain_table_options; + plain_table_options.user_key_len = 0; + plain_table_options.bloom_bits_per_key = 2; + plain_table_options.hash_table_ratio = 0.8; + plain_table_options.index_sparseness = 3; + plain_table_options.huge_page_tlb_size = 0; + plain_table_options.encoding_type = kPrefix; + plain_table_options.full_scan_mode = false; + plain_table_options.store_index_in_file = false; + + options.table_factory.reset(NewPlainTableFactory(plain_table_options)); + options.memtable_factory.reset(NewHashLinkListRepFactory(4, 0, 3, true)); + + options.prefix_extractor.reset(NewFixedPrefixTransform(8)); + options.allow_mmap_reads = mmap_mode_; + options.allow_concurrent_memtable_write = false; + options.unordered_write = false; + return options; + } + + DBImpl* dbfull() { return static_cast_with_check(db_); } + + void Reopen(Options* options = nullptr) { ASSERT_OK(TryReopen(options)); } + + void Close() { + delete db_; + db_ = nullptr; + } + + bool mmap_mode() const { return mmap_mode_; } + + void DestroyAndReopen(Options* options = nullptr) { + // Destroy using last options + Destroy(&last_options_); + ASSERT_OK(TryReopen(options)); + } + + void Destroy(Options* options) { + delete db_; + db_ = nullptr; + ASSERT_OK(DestroyDB(dbname_, *options)); + } + + Status PureReopen(Options* options, DB** db) { + return DB::Open(*options, dbname_, db); + } + + Status ReopenForReadOnly(Options* options) { + delete db_; + db_ = nullptr; + return DB::OpenForReadOnly(*options, dbname_, &db_); + } + + Status TryReopen(Options* options = nullptr) { + delete db_; + db_ = nullptr; + Options opts; + if (options != nullptr) { + opts = *options; + } else { + opts = CurrentOptions(); + opts.create_if_missing = true; + } + last_options_ = opts; + + return DB::Open(opts, dbname_, &db_); + } + + Status Put(const Slice& k, const Slice& v) { + return db_->Put(WriteOptions(), k, v); + } + + Status Delete(const std::string& k) { return db_->Delete(WriteOptions(), k); } + + std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) { + ReadOptions options; + options.snapshot = snapshot; + std::string result; + Status s = db_->Get(options, k, &result); + if (s.IsNotFound()) { + result = "NOT_FOUND"; + } else if (!s.ok()) { + result = s.ToString(); + } + return result; + } + + int NumTableFilesAtLevel(int level) { + std::string property; + EXPECT_TRUE(db_->GetProperty( + "rocksdb.num-files-at-level" + std::to_string(level), &property)); + return atoi(property.c_str()); + } + + // Return spread of files per level + std::string FilesPerLevel() { + std::string result; + size_t last_non_zero_offset = 0; + for (int level = 0; level < db_->NumberLevels(); level++) { + int f = NumTableFilesAtLevel(level); + char buf[100]; + snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f); + result += buf; + if (f > 0) { + last_non_zero_offset = result.size(); + } + } + result.resize(last_non_zero_offset); + return result; + } + + std::string IterStatus(Iterator* iter) { + std::string result; + if (iter->Valid()) { + result = iter->key().ToString() + "->" + iter->value().ToString(); + } else { + result = "(invalid)"; + } + return result; + } +}; + +TEST_P(PlainTableDBTest, Empty) { + ASSERT_TRUE(dbfull() != nullptr); + ASSERT_EQ("NOT_FOUND", Get("0000000000000foo")); +} + +extern const uint64_t kPlainTableMagicNumber; + +class TestPlainTableReader : public PlainTableReader { + public: + TestPlainTableReader( + const EnvOptions& env_options, const InternalKeyComparator& icomparator, + EncodingType encoding_type, uint64_t file_size, int bloom_bits_per_key, + double hash_table_ratio, size_t index_sparseness, + std::unique_ptr&& props, + std::unique_ptr&& file, + const ImmutableOptions& ioptions, const SliceTransform* prefix_extractor, + bool* expect_bloom_not_match, bool store_index_in_file, + uint32_t column_family_id, const std::string& column_family_name) + : PlainTableReader(ioptions, std::move(file), env_options, icomparator, + encoding_type, file_size, props.get(), + prefix_extractor), + expect_bloom_not_match_(expect_bloom_not_match) { + Status s = MmapDataIfNeeded(); + EXPECT_TRUE(s.ok()); + + s = PopulateIndex(props.get(), bloom_bits_per_key, hash_table_ratio, + index_sparseness, 2 * 1024 * 1024); + EXPECT_TRUE(s.ok()); + + EXPECT_EQ(column_family_id, static_cast(props->column_family_id)); + EXPECT_EQ(column_family_name, props->column_family_name); + if (store_index_in_file) { + auto bloom_version_ptr = props->user_collected_properties.find( + PlainTablePropertyNames::kBloomVersion); + EXPECT_TRUE(bloom_version_ptr != props->user_collected_properties.end()); + EXPECT_EQ(bloom_version_ptr->second, std::string("1")); + if (ioptions.bloom_locality > 0) { + auto num_blocks_ptr = props->user_collected_properties.find( + PlainTablePropertyNames::kNumBloomBlocks); + EXPECT_TRUE(num_blocks_ptr != props->user_collected_properties.end()); + } + } + table_properties_ = std::move(props); + } + + ~TestPlainTableReader() override {} + + private: + bool MatchBloom(uint32_t hash) const override { + bool ret = PlainTableReader::MatchBloom(hash); + if (*expect_bloom_not_match_) { + EXPECT_TRUE(!ret); + } else { + EXPECT_TRUE(ret); + } + return ret; + } + bool* expect_bloom_not_match_; +}; + +extern const uint64_t kPlainTableMagicNumber; +class TestPlainTableFactory : public PlainTableFactory { + public: + explicit TestPlainTableFactory(bool* expect_bloom_not_match, + const PlainTableOptions& options, + uint32_t column_family_id, + std::string column_family_name) + : PlainTableFactory(options), + bloom_bits_per_key_(options.bloom_bits_per_key), + hash_table_ratio_(options.hash_table_ratio), + index_sparseness_(options.index_sparseness), + store_index_in_file_(options.store_index_in_file), + expect_bloom_not_match_(expect_bloom_not_match), + column_family_id_(column_family_id), + column_family_name_(std::move(column_family_name)) {} + + using PlainTableFactory::NewTableReader; + Status NewTableReader( + const ReadOptions& /*ro*/, const TableReaderOptions& table_reader_options, + std::unique_ptr&& file, uint64_t file_size, + std::unique_ptr* table, + bool /*prefetch_index_and_filter_in_cache*/) const override { + std::unique_ptr props; + const ReadOptions read_options; + auto s = ReadTableProperties(file.get(), file_size, kPlainTableMagicNumber, + table_reader_options.ioptions, read_options, + &props); + EXPECT_TRUE(s.ok()); + + if (store_index_in_file_) { + BlockHandle bloom_block_handle; + s = FindMetaBlockInFile(file.get(), file_size, kPlainTableMagicNumber, + table_reader_options.ioptions, read_options, + BloomBlockBuilder::kBloomBlock, + &bloom_block_handle); + EXPECT_TRUE(s.ok()); + + BlockHandle index_block_handle; + s = FindMetaBlockInFile(file.get(), file_size, kPlainTableMagicNumber, + table_reader_options.ioptions, read_options, + PlainTableIndexBuilder::kPlainTableIndexBlock, + &index_block_handle); + EXPECT_TRUE(s.ok()); + } + + auto& user_props = props->user_collected_properties; + auto encoding_type_prop = + user_props.find(PlainTablePropertyNames::kEncodingType); + assert(encoding_type_prop != user_props.end()); + EncodingType encoding_type = static_cast( + DecodeFixed32(encoding_type_prop->second.c_str())); + + std::unique_ptr new_reader(new TestPlainTableReader( + table_reader_options.env_options, + table_reader_options.internal_comparator, encoding_type, file_size, + bloom_bits_per_key_, hash_table_ratio_, index_sparseness_, + std::move(props), std::move(file), table_reader_options.ioptions, + table_reader_options.prefix_extractor.get(), expect_bloom_not_match_, + store_index_in_file_, column_family_id_, column_family_name_)); + + *table = std::move(new_reader); + return s; + } + + private: + int bloom_bits_per_key_; + double hash_table_ratio_; + size_t index_sparseness_; + bool store_index_in_file_; + bool* expect_bloom_not_match_; + const uint32_t column_family_id_; + const std::string column_family_name_; +}; + +TEST_P(PlainTableDBTest, BadOptions1) { + // Build with a prefix extractor + ASSERT_OK(Put("1000000000000foo", "v1")); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + + // Bad attempt to re-open without a prefix extractor + Options options = CurrentOptions(); + options.prefix_extractor.reset(); + ASSERT_EQ( + "Invalid argument: Prefix extractor is missing when opening a PlainTable " + "built using a prefix extractor", + TryReopen(&options).ToString()); + + // Bad attempt to re-open with different prefix extractor + options.prefix_extractor.reset(NewFixedPrefixTransform(6)); + ASSERT_EQ( + "Invalid argument: Prefix extractor given doesn't match the one used to " + "build PlainTable", + TryReopen(&options).ToString()); + + // Correct prefix extractor + options.prefix_extractor.reset(NewFixedPrefixTransform(8)); + Reopen(&options); + ASSERT_EQ("v1", Get("1000000000000foo")); +} + +TEST_P(PlainTableDBTest, BadOptions2) { + Options options = CurrentOptions(); + options.prefix_extractor.reset(); + options.create_if_missing = true; + DestroyAndReopen(&options); + // Build without a prefix extractor + // (apparently works even if hash_table_ratio > 0) + ASSERT_OK(Put("1000000000000foo", "v1")); + // Build without a prefix extractor, this call will fail and returns the + // status for this bad attempt. + ASSERT_NOK(dbfull()->TEST_FlushMemTable()); + + // Bad attempt to re-open with hash_table_ratio > 0 and no prefix extractor + Status s = TryReopen(&options); + ASSERT_EQ( + "Not implemented: PlainTable requires a prefix extractor enable prefix " + "hash mode.", + s.ToString()); + + // OK to open with hash_table_ratio == 0 and no prefix extractor + PlainTableOptions plain_table_options; + plain_table_options.hash_table_ratio = 0; + options.table_factory.reset(NewPlainTableFactory(plain_table_options)); + Reopen(&options); + ASSERT_EQ("v1", Get("1000000000000foo")); + + // OK to open newly with a prefix_extractor and hash table; builds index + // in memory. + options = CurrentOptions(); + Reopen(&options); + ASSERT_EQ("v1", Get("1000000000000foo")); +} + +TEST_P(PlainTableDBTest, Flush) { + for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024; + huge_page_tlb_size += 2 * 1024 * 1024) { + for (EncodingType encoding_type : {kPlain, kPrefix}) { + for (int bloom = -1; bloom <= 117; bloom += 117) { + const int bloom_bits = std::max(bloom, 0); + const bool full_scan_mode = bloom < 0; + for (int total_order = 0; total_order <= 1; total_order++) { + for (int store_index_in_file = 0; store_index_in_file <= 1; + ++store_index_in_file) { + Options options = CurrentOptions(); + options.create_if_missing = true; + // Set only one bucket to force bucket conflict. + // Test index interval for the same prefix to be 1, 2 and 4 + if (total_order) { + options.prefix_extractor.reset(); + + PlainTableOptions plain_table_options; + plain_table_options.user_key_len = 0; + plain_table_options.bloom_bits_per_key = bloom_bits; + plain_table_options.hash_table_ratio = 0; + plain_table_options.index_sparseness = 2; + plain_table_options.huge_page_tlb_size = huge_page_tlb_size; + plain_table_options.encoding_type = encoding_type; + plain_table_options.full_scan_mode = full_scan_mode; + plain_table_options.store_index_in_file = store_index_in_file; + + options.table_factory.reset( + NewPlainTableFactory(plain_table_options)); + } else { + PlainTableOptions plain_table_options; + plain_table_options.user_key_len = 0; + plain_table_options.bloom_bits_per_key = bloom_bits; + plain_table_options.hash_table_ratio = 0.75; + plain_table_options.index_sparseness = 16; + plain_table_options.huge_page_tlb_size = huge_page_tlb_size; + plain_table_options.encoding_type = encoding_type; + plain_table_options.full_scan_mode = full_scan_mode; + plain_table_options.store_index_in_file = store_index_in_file; + + options.table_factory.reset( + NewPlainTableFactory(plain_table_options)); + } + DestroyAndReopen(&options); + uint64_t int_num; + ASSERT_TRUE(dbfull()->GetIntProperty( + "rocksdb.estimate-table-readers-mem", &int_num)); + ASSERT_EQ(int_num, 0U); + + ASSERT_OK(Put("1000000000000foo", "v1")); + ASSERT_OK(Put("0000000000000bar", "v2")); + ASSERT_OK(Put("1000000000000foo", "v3")); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + + ASSERT_TRUE(dbfull()->GetIntProperty( + "rocksdb.estimate-table-readers-mem", &int_num)); + ASSERT_GT(int_num, 0U); + + TablePropertiesCollection ptc; + ASSERT_OK(reinterpret_cast(dbfull())->GetPropertiesOfAllTables( + &ptc)); + ASSERT_EQ(1U, ptc.size()); + auto row = ptc.begin(); + auto tp = row->second; + + if (full_scan_mode) { + // Does not support Get/Seek + std::unique_ptr iter( + dbfull()->NewIterator(ReadOptions())); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("0000000000000bar", iter->key().ToString()); + ASSERT_EQ("v2", iter->value().ToString()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("1000000000000foo", iter->key().ToString()); + ASSERT_EQ("v3", iter->value().ToString()); + iter->Next(); + ASSERT_TRUE(!iter->Valid()); + ASSERT_TRUE(iter->status().ok()); + } else { + if (!store_index_in_file) { + ASSERT_EQ(total_order ? "4" : "12", + (tp->user_collected_properties) + .at("plain_table_hash_table_size")); + ASSERT_EQ("0", (tp->user_collected_properties) + .at("plain_table_sub_index_size")); + } else { + ASSERT_EQ("0", (tp->user_collected_properties) + .at("plain_table_hash_table_size")); + ASSERT_EQ("0", (tp->user_collected_properties) + .at("plain_table_sub_index_size")); + } + ASSERT_EQ("v3", Get("1000000000000foo")); + ASSERT_EQ("v2", Get("0000000000000bar")); + } + } + } + } + } + } +} + +TEST_P(PlainTableDBTest, Flush2) { + for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024; + huge_page_tlb_size += 2 * 1024 * 1024) { + for (EncodingType encoding_type : {kPlain, kPrefix}) { + for (int bloom_bits = 0; bloom_bits <= 117; bloom_bits += 117) { + for (int total_order = 0; total_order <= 1; total_order++) { + for (int store_index_in_file = 0; store_index_in_file <= 1; + ++store_index_in_file) { + if (encoding_type == kPrefix && total_order) { + continue; + } + if (!bloom_bits && store_index_in_file) { + continue; + } + if (total_order && store_index_in_file) { + continue; + } + bool expect_bloom_not_match = false; + Options options = CurrentOptions(); + options.create_if_missing = true; + // Set only one bucket to force bucket conflict. + // Test index interval for the same prefix to be 1, 2 and 4 + PlainTableOptions plain_table_options; + if (total_order) { + options.prefix_extractor = nullptr; + plain_table_options.hash_table_ratio = 0; + plain_table_options.index_sparseness = 2; + } else { + plain_table_options.hash_table_ratio = 0.75; + plain_table_options.index_sparseness = 16; + } + plain_table_options.user_key_len = kPlainTableVariableLength; + plain_table_options.bloom_bits_per_key = bloom_bits; + plain_table_options.huge_page_tlb_size = huge_page_tlb_size; + plain_table_options.encoding_type = encoding_type; + plain_table_options.store_index_in_file = store_index_in_file; + options.table_factory.reset(new TestPlainTableFactory( + &expect_bloom_not_match, plain_table_options, + 0 /* column_family_id */, kDefaultColumnFamilyName)); + + DestroyAndReopen(&options); + ASSERT_OK(Put("0000000000000bar", "b")); + ASSERT_OK(Put("1000000000000foo", "v1")); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + + ASSERT_OK(Put("1000000000000foo", "v2")); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + ASSERT_EQ("v2", Get("1000000000000foo")); + + ASSERT_OK(Put("0000000000000eee", "v3")); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + ASSERT_EQ("v3", Get("0000000000000eee")); + + ASSERT_OK(Delete("0000000000000bar")); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + ASSERT_EQ("NOT_FOUND", Get("0000000000000bar")); + + ASSERT_OK(Put("0000000000000eee", "v5")); + ASSERT_OK(Put("9000000000000eee", "v5")); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + ASSERT_EQ("v5", Get("0000000000000eee")); + + // Test Bloom Filter + if (bloom_bits > 0) { + // Neither key nor value should exist. + expect_bloom_not_match = true; + ASSERT_EQ("NOT_FOUND", Get("5_not00000000bar")); + // Key doesn't exist any more but prefix exists. + if (total_order) { + ASSERT_EQ("NOT_FOUND", Get("1000000000000not")); + ASSERT_EQ("NOT_FOUND", Get("0000000000000not")); + } + expect_bloom_not_match = false; + } + } + } + } + } + } +} + +TEST_P(PlainTableDBTest, Immortal) { + for (EncodingType encoding_type : {kPlain, kPrefix}) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.max_open_files = -1; + // Set only one bucket to force bucket conflict. + // Test index interval for the same prefix to be 1, 2 and 4 + PlainTableOptions plain_table_options; + plain_table_options.hash_table_ratio = 0.75; + plain_table_options.index_sparseness = 16; + plain_table_options.user_key_len = kPlainTableVariableLength; + plain_table_options.bloom_bits_per_key = 10; + plain_table_options.encoding_type = encoding_type; + options.table_factory.reset(NewPlainTableFactory(plain_table_options)); + + DestroyAndReopen(&options); + ASSERT_OK(Put("0000000000000bar", "b")); + ASSERT_OK(Put("1000000000000foo", "v1")); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + + int copied = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "GetContext::SaveValue::PinSelf", [&](void* /*arg*/) { copied++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_EQ("b", Get("0000000000000bar")); + ASSERT_EQ("v1", Get("1000000000000foo")); + ASSERT_EQ(2, copied); + copied = 0; + + Close(); + ASSERT_OK(ReopenForReadOnly(&options)); + + ASSERT_EQ("b", Get("0000000000000bar")); + ASSERT_EQ("v1", Get("1000000000000foo")); + ASSERT_EQ("NOT_FOUND", Get("1000000000000bar")); + if (mmap_mode()) { + ASSERT_EQ(0, copied); + } else { + ASSERT_EQ(2, copied); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + } +} + +TEST_P(PlainTableDBTest, Iterator) { + for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024; + huge_page_tlb_size += 2 * 1024 * 1024) { + for (EncodingType encoding_type : {kPlain, kPrefix}) { + for (int bloom_bits = 0; bloom_bits <= 117; bloom_bits += 117) { + for (int total_order = 0; total_order <= 1; total_order++) { + if (encoding_type == kPrefix && total_order == 1) { + continue; + } + bool expect_bloom_not_match = false; + Options options = CurrentOptions(); + options.create_if_missing = true; + // Set only one bucket to force bucket conflict. + // Test index interval for the same prefix to be 1, 2 and 4 + if (total_order) { + options.prefix_extractor = nullptr; + + PlainTableOptions plain_table_options; + plain_table_options.user_key_len = 16; + plain_table_options.bloom_bits_per_key = bloom_bits; + plain_table_options.hash_table_ratio = 0; + plain_table_options.index_sparseness = 2; + plain_table_options.huge_page_tlb_size = huge_page_tlb_size; + plain_table_options.encoding_type = encoding_type; + + options.table_factory.reset(new TestPlainTableFactory( + &expect_bloom_not_match, plain_table_options, + 0 /* column_family_id */, kDefaultColumnFamilyName)); + } else { + PlainTableOptions plain_table_options; + plain_table_options.user_key_len = 16; + plain_table_options.bloom_bits_per_key = bloom_bits; + plain_table_options.hash_table_ratio = 0.75; + plain_table_options.index_sparseness = 16; + plain_table_options.huge_page_tlb_size = huge_page_tlb_size; + plain_table_options.encoding_type = encoding_type; + + options.table_factory.reset(new TestPlainTableFactory( + &expect_bloom_not_match, plain_table_options, + 0 /* column_family_id */, kDefaultColumnFamilyName)); + } + DestroyAndReopen(&options); + + ASSERT_OK(Put("1000000000foo002", "v_2")); + ASSERT_OK(Put("0000000000000bar", "random")); + ASSERT_OK(Put("1000000000foo001", "v1")); + ASSERT_OK(Put("3000000000000bar", "bar_v")); + ASSERT_OK(Put("1000000000foo003", "v__3")); + ASSERT_OK(Put("1000000000foo004", "v__4")); + ASSERT_OK(Put("1000000000foo005", "v__5")); + ASSERT_OK(Put("1000000000foo007", "v__7")); + ASSERT_OK(Put("1000000000foo008", "v__8")); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + ASSERT_EQ("v1", Get("1000000000foo001")); + ASSERT_EQ("v__3", Get("1000000000foo003")); + Iterator* iter = dbfull()->NewIterator(ReadOptions()); + iter->Seek("1000000000foo000"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("1000000000foo001", iter->key().ToString()); + ASSERT_EQ("v1", iter->value().ToString()); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("1000000000foo002", iter->key().ToString()); + ASSERT_EQ("v_2", iter->value().ToString()); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("1000000000foo003", iter->key().ToString()); + ASSERT_EQ("v__3", iter->value().ToString()); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("1000000000foo004", iter->key().ToString()); + ASSERT_EQ("v__4", iter->value().ToString()); + + iter->Seek("3000000000000bar"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("3000000000000bar", iter->key().ToString()); + ASSERT_EQ("bar_v", iter->value().ToString()); + + iter->Seek("1000000000foo000"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("1000000000foo001", iter->key().ToString()); + ASSERT_EQ("v1", iter->value().ToString()); + + iter->Seek("1000000000foo005"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("1000000000foo005", iter->key().ToString()); + ASSERT_EQ("v__5", iter->value().ToString()); + + iter->Seek("1000000000foo006"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("1000000000foo007", iter->key().ToString()); + ASSERT_EQ("v__7", iter->value().ToString()); + + iter->Seek("1000000000foo008"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("1000000000foo008", iter->key().ToString()); + ASSERT_EQ("v__8", iter->value().ToString()); + + if (total_order == 0) { + iter->Seek("1000000000foo009"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("3000000000000bar", iter->key().ToString()); + } + + // Test Bloom Filter + if (bloom_bits > 0) { + if (!total_order) { + // Neither key nor value should exist. + expect_bloom_not_match = true; + iter->Seek("2not000000000bar"); + ASSERT_TRUE(!iter->Valid()); + ASSERT_EQ("NOT_FOUND", Get("2not000000000bar")); + expect_bloom_not_match = false; + } else { + expect_bloom_not_match = true; + ASSERT_EQ("NOT_FOUND", Get("2not000000000bar")); + expect_bloom_not_match = false; + } + } + ASSERT_OK(iter->status()); + delete iter; + } + } + } + } +} + +namespace { +std::string NthKey(size_t n, char filler) { + std::string rv(16, filler); + rv[0] = n % 10; + rv[1] = (n / 10) % 10; + rv[2] = (n / 100) % 10; + rv[3] = (n / 1000) % 10; + return rv; +} +} // anonymous namespace + +TEST_P(PlainTableDBTest, BloomSchema) { + Options options = CurrentOptions(); + options.create_if_missing = true; + for (int bloom_locality = 0; bloom_locality <= 1; bloom_locality++) { + options.bloom_locality = bloom_locality; + PlainTableOptions plain_table_options; + plain_table_options.user_key_len = 16; + plain_table_options.bloom_bits_per_key = 3; // high FP rate for test + plain_table_options.hash_table_ratio = 0.75; + plain_table_options.index_sparseness = 16; + plain_table_options.huge_page_tlb_size = 0; + plain_table_options.encoding_type = kPlain; + + bool expect_bloom_not_match = false; + options.table_factory.reset(new TestPlainTableFactory( + &expect_bloom_not_match, plain_table_options, 0 /* column_family_id */, + kDefaultColumnFamilyName)); + DestroyAndReopen(&options); + + for (unsigned i = 0; i < 2345; ++i) { + ASSERT_OK(Put(NthKey(i, 'y'), "added")); + } + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + ASSERT_EQ("added", Get(NthKey(42, 'y'))); + + for (unsigned i = 0; i < 32; ++i) { + // Known pattern of Bloom filter false positives can detect schema change + // with high probability. Known FPs stuffed into bits: + uint32_t pattern; + if (!bloom_locality) { + pattern = 1785868347UL; + } else if (CACHE_LINE_SIZE == 64U) { + pattern = 2421694657UL; + } else if (CACHE_LINE_SIZE == 128U) { + pattern = 788710956UL; + } else { + ASSERT_EQ(CACHE_LINE_SIZE, 256U); + pattern = 163905UL; + } + bool expect_fp = pattern & (1UL << i); + // fprintf(stderr, "expect_fp@%u: %d\n", i, (int)expect_fp); + expect_bloom_not_match = !expect_fp; + ASSERT_EQ("NOT_FOUND", Get(NthKey(i, 'n'))); + } + } +} + +namespace { +std::string MakeLongKey(size_t length, char c) { + return std::string(length, c); +} +} // anonymous namespace + +TEST_P(PlainTableDBTest, IteratorLargeKeys) { + Options options = CurrentOptions(); + + PlainTableOptions plain_table_options; + plain_table_options.user_key_len = 0; + plain_table_options.bloom_bits_per_key = 0; + plain_table_options.hash_table_ratio = 0; + + options.table_factory.reset(NewPlainTableFactory(plain_table_options)); + options.create_if_missing = true; + options.prefix_extractor.reset(); + DestroyAndReopen(&options); + + std::string key_list[] = {MakeLongKey(30, '0'), MakeLongKey(16, '1'), + MakeLongKey(32, '2'), MakeLongKey(60, '3'), + MakeLongKey(90, '4'), MakeLongKey(50, '5'), + MakeLongKey(26, '6')}; + + for (size_t i = 0; i < 7; i++) { + ASSERT_OK(Put(key_list[i], std::to_string(i))); + } + + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + + Iterator* iter = dbfull()->NewIterator(ReadOptions()); + iter->Seek(key_list[0]); + + for (size_t i = 0; i < 7; i++) { + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(key_list[i], iter->key().ToString()); + ASSERT_EQ(std::to_string(i), iter->value().ToString()); + iter->Next(); + } + + ASSERT_TRUE(!iter->Valid()); + + delete iter; +} + +namespace { +std::string MakeLongKeyWithPrefix(size_t length, char c) { + return "00000000" + std::string(length - 8, c); +} +} // anonymous namespace + +TEST_P(PlainTableDBTest, IteratorLargeKeysWithPrefix) { + Options options = CurrentOptions(); + + PlainTableOptions plain_table_options; + plain_table_options.user_key_len = 16; + plain_table_options.bloom_bits_per_key = 0; + plain_table_options.hash_table_ratio = 0.8; + plain_table_options.index_sparseness = 3; + plain_table_options.huge_page_tlb_size = 0; + plain_table_options.encoding_type = kPrefix; + + options.table_factory.reset(NewPlainTableFactory(plain_table_options)); + options.create_if_missing = true; + DestroyAndReopen(&options); + + std::string key_list[] = { + MakeLongKeyWithPrefix(30, '0'), MakeLongKeyWithPrefix(16, '1'), + MakeLongKeyWithPrefix(32, '2'), MakeLongKeyWithPrefix(60, '3'), + MakeLongKeyWithPrefix(90, '4'), MakeLongKeyWithPrefix(50, '5'), + MakeLongKeyWithPrefix(26, '6')}; + + for (size_t i = 0; i < 7; i++) { + ASSERT_OK(Put(key_list[i], std::to_string(i))); + } + + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + + Iterator* iter = dbfull()->NewIterator(ReadOptions()); + iter->Seek(key_list[0]); + + for (size_t i = 0; i < 7; i++) { + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(key_list[i], iter->key().ToString()); + ASSERT_EQ(std::to_string(i), iter->value().ToString()); + iter->Next(); + } + + ASSERT_TRUE(!iter->Valid()); + + delete iter; +} + +TEST_P(PlainTableDBTest, IteratorReverseSuffixComparator) { + Options options = CurrentOptions(); + options.create_if_missing = true; + // Set only one bucket to force bucket conflict. + // Test index interval for the same prefix to be 1, 2 and 4 + test::SimpleSuffixReverseComparator comp; + options.comparator = ∁ + DestroyAndReopen(&options); + + ASSERT_OK(Put("1000000000foo002", "v_2")); + ASSERT_OK(Put("0000000000000bar", "random")); + ASSERT_OK(Put("1000000000foo001", "v1")); + ASSERT_OK(Put("3000000000000bar", "bar_v")); + ASSERT_OK(Put("1000000000foo003", "v__3")); + ASSERT_OK(Put("1000000000foo004", "v__4")); + ASSERT_OK(Put("1000000000foo005", "v__5")); + ASSERT_OK(Put("1000000000foo007", "v__7")); + ASSERT_OK(Put("1000000000foo008", "v__8")); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + ASSERT_EQ("v1", Get("1000000000foo001")); + ASSERT_EQ("v__3", Get("1000000000foo003")); + Iterator* iter = dbfull()->NewIterator(ReadOptions()); + iter->Seek("1000000000foo009"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("1000000000foo008", iter->key().ToString()); + ASSERT_EQ("v__8", iter->value().ToString()); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("1000000000foo007", iter->key().ToString()); + ASSERT_EQ("v__7", iter->value().ToString()); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("1000000000foo005", iter->key().ToString()); + ASSERT_EQ("v__5", iter->value().ToString()); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("1000000000foo004", iter->key().ToString()); + ASSERT_EQ("v__4", iter->value().ToString()); + + iter->Seek("3000000000000bar"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("3000000000000bar", iter->key().ToString()); + ASSERT_EQ("bar_v", iter->value().ToString()); + + iter->Seek("1000000000foo005"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("1000000000foo005", iter->key().ToString()); + ASSERT_EQ("v__5", iter->value().ToString()); + + iter->Seek("1000000000foo006"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("1000000000foo005", iter->key().ToString()); + ASSERT_EQ("v__5", iter->value().ToString()); + + iter->Seek("1000000000foo008"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("1000000000foo008", iter->key().ToString()); + ASSERT_EQ("v__8", iter->value().ToString()); + + iter->Seek("1000000000foo000"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("3000000000000bar", iter->key().ToString()); + + delete iter; +} + +TEST_P(PlainTableDBTest, HashBucketConflict) { + for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024; + huge_page_tlb_size += 2 * 1024 * 1024) { + for (unsigned char i = 1; i <= 3; i++) { + Options options = CurrentOptions(); + options.create_if_missing = true; + // Set only one bucket to force bucket conflict. + // Test index interval for the same prefix to be 1, 2 and 4 + + PlainTableOptions plain_table_options; + plain_table_options.user_key_len = 16; + plain_table_options.bloom_bits_per_key = 0; + plain_table_options.hash_table_ratio = 0; + plain_table_options.index_sparseness = 2 ^ i; + plain_table_options.huge_page_tlb_size = huge_page_tlb_size; + + options.table_factory.reset(NewPlainTableFactory(plain_table_options)); + + DestroyAndReopen(&options); + ASSERT_OK(Put("5000000000000fo0", "v1")); + ASSERT_OK(Put("5000000000000fo1", "v2")); + ASSERT_OK(Put("5000000000000fo2", "v")); + ASSERT_OK(Put("2000000000000fo0", "v3")); + ASSERT_OK(Put("2000000000000fo1", "v4")); + ASSERT_OK(Put("2000000000000fo2", "v")); + ASSERT_OK(Put("2000000000000fo3", "v")); + + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + + ASSERT_EQ("v1", Get("5000000000000fo0")); + ASSERT_EQ("v2", Get("5000000000000fo1")); + ASSERT_EQ("v3", Get("2000000000000fo0")); + ASSERT_EQ("v4", Get("2000000000000fo1")); + + ASSERT_EQ("NOT_FOUND", Get("5000000000000bar")); + ASSERT_EQ("NOT_FOUND", Get("2000000000000bar")); + ASSERT_EQ("NOT_FOUND", Get("5000000000000fo8")); + ASSERT_EQ("NOT_FOUND", Get("2000000000000fo8")); + + ReadOptions ro; + Iterator* iter = dbfull()->NewIterator(ro); + + iter->Seek("5000000000000fo0"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("5000000000000fo0", iter->key().ToString()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("5000000000000fo1", iter->key().ToString()); + + iter->Seek("5000000000000fo1"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("5000000000000fo1", iter->key().ToString()); + + iter->Seek("2000000000000fo0"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("2000000000000fo0", iter->key().ToString()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("2000000000000fo1", iter->key().ToString()); + + iter->Seek("2000000000000fo1"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("2000000000000fo1", iter->key().ToString()); + + iter->Seek("2000000000000bar"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("2000000000000fo0", iter->key().ToString()); + + iter->Seek("5000000000000bar"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("5000000000000fo0", iter->key().ToString()); + + iter->Seek("2000000000000fo8"); + ASSERT_TRUE(!iter->Valid() || + options.comparator->Compare(iter->key(), "20000001") > 0); + + iter->Seek("5000000000000fo8"); + ASSERT_TRUE(!iter->Valid()); + + iter->Seek("1000000000000fo2"); + ASSERT_TRUE(!iter->Valid()); + + iter->Seek("3000000000000fo2"); + ASSERT_TRUE(!iter->Valid()); + + iter->Seek("8000000000000fo2"); + ASSERT_TRUE(!iter->Valid()); + + ASSERT_OK(iter->status()); + delete iter; + } + } +} + +TEST_P(PlainTableDBTest, HashBucketConflictReverseSuffixComparator) { + for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024; + huge_page_tlb_size += 2 * 1024 * 1024) { + for (unsigned char i = 1; i <= 3; i++) { + Options options = CurrentOptions(); + options.create_if_missing = true; + test::SimpleSuffixReverseComparator comp; + options.comparator = ∁ + // Set only one bucket to force bucket conflict. + // Test index interval for the same prefix to be 1, 2 and 4 + + PlainTableOptions plain_table_options; + plain_table_options.user_key_len = 16; + plain_table_options.bloom_bits_per_key = 0; + plain_table_options.hash_table_ratio = 0; + plain_table_options.index_sparseness = 2 ^ i; + plain_table_options.huge_page_tlb_size = huge_page_tlb_size; + + options.table_factory.reset(NewPlainTableFactory(plain_table_options)); + DestroyAndReopen(&options); + ASSERT_OK(Put("5000000000000fo0", "v1")); + ASSERT_OK(Put("5000000000000fo1", "v2")); + ASSERT_OK(Put("5000000000000fo2", "v")); + ASSERT_OK(Put("2000000000000fo0", "v3")); + ASSERT_OK(Put("2000000000000fo1", "v4")); + ASSERT_OK(Put("2000000000000fo2", "v")); + ASSERT_OK(Put("2000000000000fo3", "v")); + + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + + ASSERT_EQ("v1", Get("5000000000000fo0")); + ASSERT_EQ("v2", Get("5000000000000fo1")); + ASSERT_EQ("v3", Get("2000000000000fo0")); + ASSERT_EQ("v4", Get("2000000000000fo1")); + + ASSERT_EQ("NOT_FOUND", Get("5000000000000bar")); + ASSERT_EQ("NOT_FOUND", Get("2000000000000bar")); + ASSERT_EQ("NOT_FOUND", Get("5000000000000fo8")); + ASSERT_EQ("NOT_FOUND", Get("2000000000000fo8")); + + ReadOptions ro; + Iterator* iter = dbfull()->NewIterator(ro); + + iter->Seek("5000000000000fo1"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("5000000000000fo1", iter->key().ToString()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("5000000000000fo0", iter->key().ToString()); + + iter->Seek("5000000000000fo1"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("5000000000000fo1", iter->key().ToString()); + + iter->Seek("2000000000000fo1"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("2000000000000fo1", iter->key().ToString()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("2000000000000fo0", iter->key().ToString()); + + iter->Seek("2000000000000fo1"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("2000000000000fo1", iter->key().ToString()); + + iter->Seek("2000000000000var"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("2000000000000fo3", iter->key().ToString()); + + iter->Seek("5000000000000var"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("5000000000000fo2", iter->key().ToString()); + + std::string seek_key = "2000000000000bar"; + iter->Seek(seek_key); + ASSERT_TRUE(!iter->Valid() || + options.prefix_extractor->Transform(iter->key()) != + options.prefix_extractor->Transform(seek_key)); + + iter->Seek("1000000000000fo2"); + ASSERT_TRUE(!iter->Valid()); + + iter->Seek("3000000000000fo2"); + ASSERT_TRUE(!iter->Valid()); + + iter->Seek("8000000000000fo2"); + ASSERT_TRUE(!iter->Valid()); + + ASSERT_OK(iter->status()); + delete iter; + } + } +} + +TEST_P(PlainTableDBTest, NonExistingKeyToNonEmptyBucket) { + Options options = CurrentOptions(); + options.create_if_missing = true; + // Set only one bucket to force bucket conflict. + // Test index interval for the same prefix to be 1, 2 and 4 + PlainTableOptions plain_table_options; + plain_table_options.user_key_len = 16; + plain_table_options.bloom_bits_per_key = 0; + plain_table_options.hash_table_ratio = 0; + plain_table_options.index_sparseness = 5; + + options.table_factory.reset(NewPlainTableFactory(plain_table_options)); + DestroyAndReopen(&options); + ASSERT_OK(Put("5000000000000fo0", "v1")); + ASSERT_OK(Put("5000000000000fo1", "v2")); + ASSERT_OK(Put("5000000000000fo2", "v3")); + + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + + ASSERT_EQ("v1", Get("5000000000000fo0")); + ASSERT_EQ("v2", Get("5000000000000fo1")); + ASSERT_EQ("v3", Get("5000000000000fo2")); + + ASSERT_EQ("NOT_FOUND", Get("8000000000000bar")); + ASSERT_EQ("NOT_FOUND", Get("1000000000000bar")); + + Iterator* iter = dbfull()->NewIterator(ReadOptions()); + + iter->Seek("5000000000000bar"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("5000000000000fo0", iter->key().ToString()); + + iter->Seek("5000000000000fo8"); + ASSERT_TRUE(!iter->Valid()); + + iter->Seek("1000000000000fo2"); + ASSERT_TRUE(!iter->Valid()); + + iter->Seek("8000000000000fo2"); + ASSERT_TRUE(!iter->Valid()); + + ASSERT_OK(iter->status()); + delete iter; +} + +static std::string Key(int i) { + char buf[100]; + snprintf(buf, sizeof(buf), "key_______%06d", i); + return std::string(buf); +} + +TEST_P(PlainTableDBTest, CompactionTrigger) { + Options options = CurrentOptions(); + options.write_buffer_size = 120 << 10; // 120KB + options.num_levels = 3; + options.level0_file_num_compaction_trigger = 3; + Reopen(&options); + + Random rnd(301); + + for (int num = 0; num < options.level0_file_num_compaction_trigger - 1; + num++) { + std::vector values; + // Write 120KB (10 values, each 12K) + for (int i = 0; i < 10; i++) { + values.push_back(rnd.RandomString(12 << 10)); + ASSERT_OK(Put(Key(i), values[i])); + } + ASSERT_OK(Put(Key(999), "")); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_EQ(NumTableFilesAtLevel(0), num + 1); + } + + // generate one more file in level-0, and should trigger level-0 compaction + std::vector values; + for (int i = 0; i < 12; i++) { + values.push_back(rnd.RandomString(10000)); + ASSERT_OK(Put(Key(i), values[i])); + } + ASSERT_OK(Put(Key(999), "")); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + ASSERT_EQ(NumTableFilesAtLevel(1), 1); +} + +TEST_P(PlainTableDBTest, AdaptiveTable) { + Options options = CurrentOptions(); + options.create_if_missing = true; + + options.table_factory.reset(NewPlainTableFactory()); + DestroyAndReopen(&options); + + ASSERT_OK(Put("1000000000000foo", "v1")); + ASSERT_OK(Put("0000000000000bar", "v2")); + ASSERT_OK(Put("1000000000000foo", "v3")); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + + options.create_if_missing = false; + std::shared_ptr block_based_factory( + NewBlockBasedTableFactory()); + std::shared_ptr plain_table_factory(NewPlainTableFactory()); + std::shared_ptr dummy_factory; + options.table_factory.reset(NewAdaptiveTableFactory( + block_based_factory, block_based_factory, plain_table_factory)); + Reopen(&options); + ASSERT_EQ("v3", Get("1000000000000foo")); + ASSERT_EQ("v2", Get("0000000000000bar")); + + ASSERT_OK(Put("2000000000000foo", "v4")); + ASSERT_OK(Put("3000000000000bar", "v5")); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + ASSERT_EQ("v4", Get("2000000000000foo")); + ASSERT_EQ("v5", Get("3000000000000bar")); + + Reopen(&options); + ASSERT_EQ("v3", Get("1000000000000foo")); + ASSERT_EQ("v2", Get("0000000000000bar")); + ASSERT_EQ("v4", Get("2000000000000foo")); + ASSERT_EQ("v5", Get("3000000000000bar")); + + options.paranoid_checks = false; + options.table_factory.reset(NewBlockBasedTableFactory()); + Reopen(&options); + ASSERT_NE("v3", Get("1000000000000foo")); + + options.paranoid_checks = false; + options.table_factory.reset(NewPlainTableFactory()); + Reopen(&options); + ASSERT_NE("v5", Get("3000000000000bar")); +} + +INSTANTIATE_TEST_CASE_P(PlainTableDBTest, PlainTableDBTest, ::testing::Bool()); + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/post_memtable_callback.h b/librocksdb-sys/rocksdb/db/post_memtable_callback.h new file mode 100644 index 0000000..fbf2fbe --- /dev/null +++ b/librocksdb-sys/rocksdb/db/post_memtable_callback.h @@ -0,0 +1,25 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "rocksdb/status.h" +#include "rocksdb/types.h" + +namespace ROCKSDB_NAMESPACE { + +// Callback invoked after finishing writing to the memtable but before +// publishing the sequence number to readers. +// Note that with write-prepared/write-unprepared transactions with +// two-write-queues, PreReleaseCallback is called before publishing the +// sequence numbers to readers. +class PostMemTableCallback { + public: + virtual ~PostMemTableCallback() {} + + virtual Status operator()(SequenceNumber seq, bool disable_memtable) = 0; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/pre_release_callback.h b/librocksdb-sys/rocksdb/db/pre_release_callback.h new file mode 100644 index 0000000..6b90394 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/pre_release_callback.h @@ -0,0 +1,37 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "rocksdb/status.h" +#include "rocksdb/types.h" + +namespace ROCKSDB_NAMESPACE { + +class PreReleaseCallback { + public: + virtual ~PreReleaseCallback() {} + + // Will be called while on the write thread after the write to the WAL and + // before the write to memtable. This is useful if any operation needs to be + // done before the write gets visible to the readers, or if we want to reduce + // the overhead of locking by updating something sequentially while we are on + // the write thread. If the callback fails, this function returns a non-OK + // status, the sequence number will not be released, and same status will be + // propagated to all the writers in the write group. + // seq is the sequence number that is used for this write and will be + // released. + // is_mem_disabled is currently used for debugging purposes to assert that + // the callback is done from the right write queue. + // If non-zero, log_number indicates the WAL log to which we wrote. + // index >= 0 specifies the order of callback in the same write thread. + // total > index specifies the total number of callbacks in the same write + // thread. Together with index, could be used to reduce the redundant + // operations among the callbacks. + virtual Status Callback(SequenceNumber seq, bool is_mem_disabled, + uint64_t log_number, size_t index, size_t total) = 0; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/prefix_test.cc b/librocksdb-sys/rocksdb/db/prefix_test.cc new file mode 100644 index 0000000..a8ae040 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/prefix_test.cc @@ -0,0 +1,894 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + + +#ifndef GFLAGS +#include +int main() { + fprintf(stderr, "Please install gflags to run this test... Skipping...\n"); + return 0; +} +#else + +#include +#include +#include + +#include "db/db_impl/db_impl.h" +#include "monitoring/histogram.h" +#include "rocksdb/comparator.h" +#include "rocksdb/db.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/perf_context.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/system_clock.h" +#include "rocksdb/table.h" +#include "test_util/testharness.h" +#include "util/cast_util.h" +#include "util/coding.h" +#include "util/gflags_compat.h" +#include "util/random.h" +#include "util/stop_watch.h" +#include "util/string_util.h" +#include "utilities/merge_operators.h" + +using GFLAGS_NAMESPACE::ParseCommandLineFlags; + +DEFINE_bool(trigger_deadlock, false, + "issue delete in range scan to trigger PrefixHashMap deadlock"); +DEFINE_int32(bucket_count, 100000, "number of buckets"); +DEFINE_uint64(num_locks, 10001, "number of locks"); +DEFINE_bool(random_prefix, false, "randomize prefix"); +DEFINE_uint64(total_prefixes, 100000, "total number of prefixes"); +DEFINE_uint64(items_per_prefix, 1, "total number of values per prefix"); +DEFINE_int64(write_buffer_size, 33554432, ""); +DEFINE_int32(max_write_buffer_number, 2, ""); +DEFINE_int32(min_write_buffer_number_to_merge, 1, ""); +DEFINE_int32(skiplist_height, 4, ""); +DEFINE_double(memtable_prefix_bloom_size_ratio, 0.1, ""); +DEFINE_int32(memtable_huge_page_size, 2 * 1024 * 1024, ""); +DEFINE_int32(value_size, 40, ""); +DEFINE_bool(enable_print, false, "Print options generated to console."); + +// Path to the database on file system +const std::string kDbName = + ROCKSDB_NAMESPACE::test::PerThreadDBPath("prefix_test"); + +namespace ROCKSDB_NAMESPACE { + +struct TestKey { + uint64_t prefix; + uint64_t sorted; + + TestKey(uint64_t _prefix, uint64_t _sorted) + : prefix(_prefix), sorted(_sorted) {} +}; + +// return a slice backed by test_key +inline Slice TestKeyToSlice(std::string& s, const TestKey& test_key) { + s.clear(); + PutFixed64(&s, test_key.prefix); + PutFixed64(&s, test_key.sorted); + return Slice(s.c_str(), s.size()); +} + +inline const TestKey SliceToTestKey(const Slice& slice) { + return TestKey(DecodeFixed64(slice.data()), DecodeFixed64(slice.data() + 8)); +} + +class TestKeyComparator : public Comparator { + public: + // Compare needs to be aware of the possibility of a and/or b is + // prefix only + int Compare(const Slice& a, const Slice& b) const override { + const TestKey kkey_a = SliceToTestKey(a); + const TestKey kkey_b = SliceToTestKey(b); + const TestKey* key_a = &kkey_a; + const TestKey* key_b = &kkey_b; + if (key_a->prefix != key_b->prefix) { + if (key_a->prefix < key_b->prefix) return -1; + if (key_a->prefix > key_b->prefix) return 1; + } else { + EXPECT_TRUE(key_a->prefix == key_b->prefix); + // note, both a and b could be prefix only + if (a.size() != b.size()) { + // one of them is prefix + EXPECT_TRUE( + (a.size() == sizeof(uint64_t) && b.size() == sizeof(TestKey)) || + (b.size() == sizeof(uint64_t) && a.size() == sizeof(TestKey))); + if (a.size() < b.size()) return -1; + if (a.size() > b.size()) return 1; + } else { + // both a and b are prefix + if (a.size() == sizeof(uint64_t)) { + return 0; + } + + // both a and b are whole key + EXPECT_TRUE(a.size() == sizeof(TestKey) && b.size() == sizeof(TestKey)); + if (key_a->sorted < key_b->sorted) return -1; + if (key_a->sorted > key_b->sorted) return 1; + if (key_a->sorted == key_b->sorted) return 0; + } + } + return 0; + } + + bool operator()(const TestKey& a, const TestKey& b) const { + std::string sa, sb; + return Compare(TestKeyToSlice(sa, a), TestKeyToSlice(sb, b)) < 0; + } + + const char* Name() const override { return "TestKeyComparator"; } + + void FindShortestSeparator(std::string* /*start*/, + const Slice& /*limit*/) const override {} + + void FindShortSuccessor(std::string* /*key*/) const override {} +}; + +namespace { +void PutKey(DB* db, WriteOptions write_options, uint64_t prefix, + uint64_t suffix, const Slice& value) { + TestKey test_key(prefix, suffix); + std::string s; + Slice key = TestKeyToSlice(s, test_key); + ASSERT_OK(db->Put(write_options, key, value)); +} + +void PutKey(DB* db, WriteOptions write_options, const TestKey& test_key, + const Slice& value) { + std::string s; + Slice key = TestKeyToSlice(s, test_key); + ASSERT_OK(db->Put(write_options, key, value)); +} + +void MergeKey(DB* db, WriteOptions write_options, const TestKey& test_key, + const Slice& value) { + std::string s; + Slice key = TestKeyToSlice(s, test_key); + ASSERT_OK(db->Merge(write_options, key, value)); +} + +void DeleteKey(DB* db, WriteOptions write_options, const TestKey& test_key) { + std::string s; + Slice key = TestKeyToSlice(s, test_key); + ASSERT_OK(db->Delete(write_options, key)); +} + +void SeekIterator(Iterator* iter, uint64_t prefix, uint64_t suffix) { + TestKey test_key(prefix, suffix); + std::string s; + Slice key = TestKeyToSlice(s, test_key); + iter->Seek(key); +} + +const std::string kNotFoundResult = "NOT_FOUND"; + +std::string Get(DB* db, const ReadOptions& read_options, uint64_t prefix, + uint64_t suffix) { + TestKey test_key(prefix, suffix); + std::string s2; + Slice key = TestKeyToSlice(s2, test_key); + + std::string result; + Status s = db->Get(read_options, key, &result); + if (s.IsNotFound()) { + result = kNotFoundResult; + } else if (!s.ok()) { + result = s.ToString(); + } + return result; +} + +class SamePrefixTransform : public SliceTransform { + private: + const Slice prefix_; + std::string name_; + + public: + explicit SamePrefixTransform(const Slice& prefix) + : prefix_(prefix), name_("rocksdb.SamePrefix." + prefix.ToString()) {} + + const char* Name() const override { return name_.c_str(); } + + Slice Transform(const Slice& src) const override { + assert(InDomain(src)); + return prefix_; + } + + bool InDomain(const Slice& src) const override { + if (src.size() >= prefix_.size()) { + return Slice(src.data(), prefix_.size()) == prefix_; + } + return false; + } + + bool InRange(const Slice& dst) const override { return dst == prefix_; } + + bool FullLengthEnabled(size_t* /*len*/) const override { return false; } +}; + +} // anonymous namespace + +class PrefixTest : public testing::Test { + public: + std::shared_ptr OpenDb() { + DB* db; + + options.create_if_missing = true; + options.write_buffer_size = FLAGS_write_buffer_size; + options.max_write_buffer_number = FLAGS_max_write_buffer_number; + options.min_write_buffer_number_to_merge = + FLAGS_min_write_buffer_number_to_merge; + + options.memtable_prefix_bloom_size_ratio = + FLAGS_memtable_prefix_bloom_size_ratio; + options.memtable_huge_page_size = FLAGS_memtable_huge_page_size; + + options.prefix_extractor.reset(NewFixedPrefixTransform(8)); + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(NewBloomFilterPolicy(10, false)); + bbto.whole_key_filtering = false; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + options.allow_concurrent_memtable_write = false; + + Status s = DB::Open(options, kDbName, &db); + EXPECT_OK(s); + return std::shared_ptr(db); + } + + void FirstOption() { option_config_ = kBegin; } + + bool NextOptions(int bucket_count) { + // skip some options + option_config_++; + if (option_config_ < kEnd) { + options.prefix_extractor.reset(NewFixedPrefixTransform(8)); + switch (option_config_) { + case kHashSkipList: + options.memtable_factory.reset( + NewHashSkipListRepFactory(bucket_count, FLAGS_skiplist_height)); + return true; + case kHashLinkList: + options.memtable_factory.reset( + NewHashLinkListRepFactory(bucket_count)); + return true; + case kHashLinkListHugePageTlb: + options.memtable_factory.reset( + NewHashLinkListRepFactory(bucket_count, 2 * 1024 * 1024)); + return true; + case kHashLinkListTriggerSkipList: + options.memtable_factory.reset( + NewHashLinkListRepFactory(bucket_count, 0, 3)); + return true; + default: + return false; + } + } + return false; + } + + PrefixTest() : option_config_(kBegin) { + options.comparator = new TestKeyComparator(); + } + ~PrefixTest() override { delete options.comparator; } + + protected: + enum OptionConfig { + kBegin, + kHashSkipList, + kHashLinkList, + kHashLinkListHugePageTlb, + kHashLinkListTriggerSkipList, + kEnd + }; + int option_config_; + Options options; +}; + +TEST(SamePrefixTest, InDomainTest) { + DB* db; + Options options; + options.create_if_missing = true; + options.prefix_extractor.reset(new SamePrefixTransform("HHKB")); + BlockBasedTableOptions bbto; + bbto.filter_policy.reset(NewBloomFilterPolicy(10, false)); + bbto.whole_key_filtering = false; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + WriteOptions write_options; + ReadOptions read_options; + { + ASSERT_OK(DestroyDB(kDbName, Options())); + ASSERT_OK(DB::Open(options, kDbName, &db)); + ASSERT_OK(db->Put(write_options, "HHKB pro2", "Mar 24, 2006")); + ASSERT_OK(db->Put(write_options, "HHKB pro2 Type-S", "June 29, 2011")); + ASSERT_OK(db->Put(write_options, "Realforce 87u", "idk")); + ASSERT_OK(db->Flush(FlushOptions())); + std::string result; + auto db_iter = db->NewIterator(ReadOptions()); + + db_iter->Seek("Realforce 87u"); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_OK(db_iter->status()); + ASSERT_EQ(db_iter->key(), "Realforce 87u"); + ASSERT_EQ(db_iter->value(), "idk"); + + delete db_iter; + delete db; + ASSERT_OK(DestroyDB(kDbName, Options())); + } + + { + ASSERT_OK(DB::Open(options, kDbName, &db)); + ASSERT_OK(db->Put(write_options, "pikachu", "1")); + ASSERT_OK(db->Put(write_options, "Meowth", "1")); + ASSERT_OK(db->Put(write_options, "Mewtwo", "idk")); + ASSERT_OK(db->Flush(FlushOptions())); + std::string result; + auto db_iter = db->NewIterator(ReadOptions()); + + db_iter->Seek("Mewtwo"); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_OK(db_iter->status()); + delete db_iter; + delete db; + ASSERT_OK(DestroyDB(kDbName, Options())); + } +} + +TEST_F(PrefixTest, TestResult) { + for (int num_buckets = 1; num_buckets <= 2; num_buckets++) { + FirstOption(); + while (NextOptions(num_buckets)) { + std::cout << "*** Mem table: " << options.memtable_factory->Name() + << " number of buckets: " << num_buckets << std::endl; + ASSERT_OK(DestroyDB(kDbName, Options())); + auto db = OpenDb(); + WriteOptions write_options; + ReadOptions read_options; + + // 1. Insert one row. + Slice v16("v16"); + PutKey(db.get(), write_options, 1, 6, v16); + std::unique_ptr iter(db->NewIterator(read_options)); + SeekIterator(iter.get(), 1, 6); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v16 == iter->value()); + SeekIterator(iter.get(), 1, 5); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v16 == iter->value()); + SeekIterator(iter.get(), 1, 5); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v16 == iter->value()); + iter->Next(); + ASSERT_TRUE(!iter->Valid()); + ASSERT_OK(iter->status()); + + SeekIterator(iter.get(), 2, 0); + ASSERT_TRUE(!iter->Valid()); + ASSERT_OK(iter->status()); + + ASSERT_EQ(v16.ToString(), Get(db.get(), read_options, 1, 6)); + ASSERT_EQ(kNotFoundResult, Get(db.get(), read_options, 1, 5)); + ASSERT_EQ(kNotFoundResult, Get(db.get(), read_options, 1, 7)); + ASSERT_EQ(kNotFoundResult, Get(db.get(), read_options, 0, 6)); + ASSERT_EQ(kNotFoundResult, Get(db.get(), read_options, 2, 6)); + + // 2. Insert an entry for the same prefix as the last entry in the bucket. + Slice v17("v17"); + PutKey(db.get(), write_options, 1, 7, v17); + iter.reset(db->NewIterator(read_options)); + SeekIterator(iter.get(), 1, 7); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v17 == iter->value()); + + SeekIterator(iter.get(), 1, 6); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v16 == iter->value()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v17 == iter->value()); + iter->Next(); + ASSERT_TRUE(!iter->Valid()); + ASSERT_OK(iter->status()); + + SeekIterator(iter.get(), 2, 0); + ASSERT_TRUE(!iter->Valid()); + ASSERT_OK(iter->status()); + + // 3. Insert an entry for the same prefix as the head of the bucket. + Slice v15("v15"); + PutKey(db.get(), write_options, 1, 5, v15); + iter.reset(db->NewIterator(read_options)); + + SeekIterator(iter.get(), 1, 7); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v17 == iter->value()); + + SeekIterator(iter.get(), 1, 5); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v15 == iter->value()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v16 == iter->value()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v17 == iter->value()); + + SeekIterator(iter.get(), 1, 5); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v15 == iter->value()); + + ASSERT_EQ(v15.ToString(), Get(db.get(), read_options, 1, 5)); + ASSERT_EQ(v16.ToString(), Get(db.get(), read_options, 1, 6)); + ASSERT_EQ(v17.ToString(), Get(db.get(), read_options, 1, 7)); + + // 4. Insert an entry with a larger prefix + Slice v22("v22"); + PutKey(db.get(), write_options, 2, 2, v22); + iter.reset(db->NewIterator(read_options)); + + SeekIterator(iter.get(), 2, 2); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v22 == iter->value()); + SeekIterator(iter.get(), 2, 0); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v22 == iter->value()); + + SeekIterator(iter.get(), 1, 5); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v15 == iter->value()); + + SeekIterator(iter.get(), 1, 7); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v17 == iter->value()); + + // 5. Insert an entry with a smaller prefix + Slice v02("v02"); + PutKey(db.get(), write_options, 0, 2, v02); + iter.reset(db->NewIterator(read_options)); + + SeekIterator(iter.get(), 0, 2); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v02 == iter->value()); + SeekIterator(iter.get(), 0, 0); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v02 == iter->value()); + + SeekIterator(iter.get(), 2, 0); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v22 == iter->value()); + + SeekIterator(iter.get(), 1, 5); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v15 == iter->value()); + + SeekIterator(iter.get(), 1, 7); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v17 == iter->value()); + + // 6. Insert to the beginning and the end of the first prefix + Slice v13("v13"); + Slice v18("v18"); + PutKey(db.get(), write_options, 1, 3, v13); + PutKey(db.get(), write_options, 1, 8, v18); + iter.reset(db->NewIterator(read_options)); + SeekIterator(iter.get(), 1, 7); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v17 == iter->value()); + + SeekIterator(iter.get(), 1, 3); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v13 == iter->value()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v15 == iter->value()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v16 == iter->value()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v17 == iter->value()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v18 == iter->value()); + + SeekIterator(iter.get(), 0, 0); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v02 == iter->value()); + + SeekIterator(iter.get(), 2, 0); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v22 == iter->value()); + + ASSERT_EQ(v22.ToString(), Get(db.get(), read_options, 2, 2)); + ASSERT_EQ(v02.ToString(), Get(db.get(), read_options, 0, 2)); + ASSERT_EQ(v13.ToString(), Get(db.get(), read_options, 1, 3)); + ASSERT_EQ(v15.ToString(), Get(db.get(), read_options, 1, 5)); + ASSERT_EQ(v16.ToString(), Get(db.get(), read_options, 1, 6)); + ASSERT_EQ(v17.ToString(), Get(db.get(), read_options, 1, 7)); + ASSERT_EQ(v18.ToString(), Get(db.get(), read_options, 1, 8)); + } + } +} + +// Show results in prefix +TEST_F(PrefixTest, PrefixValid) { + for (int num_buckets = 1; num_buckets <= 2; num_buckets++) { + FirstOption(); + while (NextOptions(num_buckets)) { + std::cout << "*** Mem table: " << options.memtable_factory->Name() + << " number of buckets: " << num_buckets << std::endl; + ASSERT_OK(DestroyDB(kDbName, Options())); + auto db = OpenDb(); + WriteOptions write_options; + ReadOptions read_options; + + // Insert keys with common prefix and one key with different + Slice v16("v16"); + Slice v17("v17"); + Slice v18("v18"); + Slice v19("v19"); + PutKey(db.get(), write_options, 12345, 6, v16); + PutKey(db.get(), write_options, 12345, 7, v17); + PutKey(db.get(), write_options, 12345, 8, v18); + PutKey(db.get(), write_options, 12345, 9, v19); + PutKey(db.get(), write_options, 12346, 8, v16); + ASSERT_OK(db->Flush(FlushOptions())); + TestKey test_key(12346, 8); + std::string s; + ASSERT_OK(db->Delete(write_options, TestKeyToSlice(s, test_key))); + ASSERT_OK(db->Flush(FlushOptions())); + read_options.prefix_same_as_start = true; + std::unique_ptr iter(db->NewIterator(read_options)); + SeekIterator(iter.get(), 12345, 6); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v16 == iter->value()); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v17 == iter->value()); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v18 == iter->value()); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_TRUE(v19 == iter->value()); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + ASSERT_EQ(kNotFoundResult, Get(db.get(), read_options, 12346, 8)); + + // Verify seeking past the prefix won't return a result. + SeekIterator(iter.get(), 12345, 10); + ASSERT_TRUE(!iter->Valid()); + ASSERT_OK(iter->status()); + } + } +} + +TEST_F(PrefixTest, DynamicPrefixIterator) { + while (NextOptions(FLAGS_bucket_count)) { + std::cout << "*** Mem table: " << options.memtable_factory->Name() + << std::endl; + ASSERT_OK(DestroyDB(kDbName, Options())); + auto db = OpenDb(); + WriteOptions write_options; + ReadOptions read_options; + + std::vector prefixes; + for (uint64_t i = 0; i < FLAGS_total_prefixes; ++i) { + prefixes.push_back(i); + } + + if (FLAGS_random_prefix) { + RandomShuffle(prefixes.begin(), prefixes.end()); + } + + HistogramImpl hist_put_time; + HistogramImpl hist_put_comparison; + // insert x random prefix, each with y continuous element. + for (auto prefix : prefixes) { + for (uint64_t sorted = 0; sorted < FLAGS_items_per_prefix; sorted++) { + TestKey test_key(prefix, sorted); + + std::string s; + Slice key = TestKeyToSlice(s, test_key); + std::string value(FLAGS_value_size, 0); + + get_perf_context()->Reset(); + StopWatchNano timer(SystemClock::Default().get(), true); + ASSERT_OK(db->Put(write_options, key, value)); + hist_put_time.Add(timer.ElapsedNanos()); + hist_put_comparison.Add(get_perf_context()->user_key_comparison_count); + } + } + + std::cout << "Put key comparison: \n" + << hist_put_comparison.ToString() << "Put time: \n" + << hist_put_time.ToString(); + + // test seek existing keys + HistogramImpl hist_seek_time; + HistogramImpl hist_seek_comparison; + + std::unique_ptr iter(db->NewIterator(read_options)); + + for (auto prefix : prefixes) { + TestKey test_key(prefix, FLAGS_items_per_prefix / 2); + std::string s; + Slice key = TestKeyToSlice(s, test_key); + std::string value = "v" + std::to_string(0); + + get_perf_context()->Reset(); + StopWatchNano timer(SystemClock::Default().get(), true); + auto key_prefix = options.prefix_extractor->Transform(key); + uint64_t total_keys = 0; + for (iter->Seek(key); + iter->Valid() && iter->key().starts_with(key_prefix); iter->Next()) { + if (FLAGS_trigger_deadlock) { + std::cout << "Behold the deadlock!\n"; + db->Delete(write_options, iter->key()); + } + total_keys++; + } + hist_seek_time.Add(timer.ElapsedNanos()); + hist_seek_comparison.Add(get_perf_context()->user_key_comparison_count); + ASSERT_EQ(total_keys, + FLAGS_items_per_prefix - FLAGS_items_per_prefix / 2); + } + + std::cout << "Seek key comparison: \n" + << hist_seek_comparison.ToString() << "Seek time: \n" + << hist_seek_time.ToString(); + + // test non-existing keys + HistogramImpl hist_no_seek_time; + HistogramImpl hist_no_seek_comparison; + + for (auto prefix = FLAGS_total_prefixes; + prefix < FLAGS_total_prefixes + 10000; prefix++) { + TestKey test_key(prefix, 0); + std::string s; + Slice key = TestKeyToSlice(s, test_key); + + get_perf_context()->Reset(); + StopWatchNano timer(SystemClock::Default().get(), true); + iter->Seek(key); + hist_no_seek_time.Add(timer.ElapsedNanos()); + hist_no_seek_comparison.Add( + get_perf_context()->user_key_comparison_count); + ASSERT_TRUE(!iter->Valid()); + ASSERT_OK(iter->status()); + } + + std::cout << "non-existing Seek key comparison: \n" + << hist_no_seek_comparison.ToString() + << "non-existing Seek time: \n" + << hist_no_seek_time.ToString(); + } +} + +TEST_F(PrefixTest, PrefixSeekModePrev) { + // Only for SkipListFactory + options.memtable_factory.reset(new SkipListFactory); + options.merge_operator = MergeOperators::CreatePutOperator(); + options.write_buffer_size = 1024 * 1024; + Random rnd(1); + for (size_t m = 1; m < 100; m++) { + std::cout << "[" + std::to_string(m) + "]" + "*** Mem table: " + << options.memtable_factory->Name() << std::endl; + ASSERT_OK(DestroyDB(kDbName, Options())); + auto db = OpenDb(); + WriteOptions write_options; + ReadOptions read_options; + std::map entry_maps[3], whole_map; + for (uint64_t i = 0; i < 10; i++) { + int div = i % 3 + 1; + for (uint64_t j = 0; j < 10; j++) { + whole_map[TestKey(i, j)] = entry_maps[rnd.Uniform(div)][TestKey(i, j)] = + 'v' + std::to_string(i) + std::to_string(j); + } + } + + std::map type_map; + for (size_t i = 0; i < 3; i++) { + for (auto& kv : entry_maps[i]) { + if (rnd.OneIn(3)) { + PutKey(db.get(), write_options, kv.first, kv.second); + type_map[kv.first] = "value"; + } else { + MergeKey(db.get(), write_options, kv.first, kv.second); + type_map[kv.first] = "merge"; + } + } + if (i < 2) { + ASSERT_OK(db->Flush(FlushOptions())); + } + } + + for (size_t i = 0; i < 2; i++) { + for (auto& kv : entry_maps[i]) { + if (rnd.OneIn(10)) { + whole_map.erase(kv.first); + DeleteKey(db.get(), write_options, kv.first); + entry_maps[2][kv.first] = "delete"; + } + } + } + + if (FLAGS_enable_print) { + for (size_t i = 0; i < 3; i++) { + for (auto& kv : entry_maps[i]) { + std::cout << "[" << i << "]" << kv.first.prefix << kv.first.sorted + << " " << kv.second + " " + type_map[kv.first] << std::endl; + } + } + } + + std::unique_ptr iter(db->NewIterator(read_options)); + for (uint64_t prefix = 0; prefix < 10; prefix++) { + uint64_t start_suffix = rnd.Uniform(9); + SeekIterator(iter.get(), prefix, start_suffix); + auto it = whole_map.find(TestKey(prefix, start_suffix)); + if (it == whole_map.end()) { + continue; + } + ASSERT_NE(it, whole_map.end()); + ASSERT_TRUE(iter->Valid()); + if (FLAGS_enable_print) { + std::cout << "round " << prefix + << " iter: " << SliceToTestKey(iter->key()).prefix + << SliceToTestKey(iter->key()).sorted + << " | map: " << it->first.prefix << it->first.sorted << " | " + << iter->value().ToString() << " " << it->second << std::endl; + } + ASSERT_EQ(iter->value(), it->second); + uint64_t stored_prefix = prefix; + for (size_t k = 0; k < 9; k++) { + if (rnd.OneIn(2) || it == whole_map.begin()) { + iter->Next(); + ++it; + if (FLAGS_enable_print) { + std::cout << "Next >> "; + } + } else { + iter->Prev(); + it--; + if (FLAGS_enable_print) { + std::cout << "Prev >> "; + } + } + if (!iter->Valid() || + SliceToTestKey(iter->key()).prefix != stored_prefix) { + break; + } + ASSERT_OK(iter->status()); + stored_prefix = SliceToTestKey(iter->key()).prefix; + ASSERT_TRUE(iter->Valid()); + ASSERT_NE(it, whole_map.end()); + ASSERT_EQ(iter->value(), it->second); + if (FLAGS_enable_print) { + std::cout << "iter: " << SliceToTestKey(iter->key()).prefix + << SliceToTestKey(iter->key()).sorted + << " | map: " << it->first.prefix << it->first.sorted + << " | " << iter->value().ToString() << " " << it->second + << std::endl; + } + } + } + } +} + +TEST_F(PrefixTest, PrefixSeekModePrev2) { + // Only for SkipListFactory + // test the case + // iter1 iter2 + // | prefix | suffix | | prefix | suffix | + // | 1 | 1 | | 1 | 2 | + // | 1 | 3 | | 1 | 4 | + // | 2 | 1 | | 3 | 3 | + // | 2 | 2 | | 3 | 4 | + // after seek(15), iter1 will be at 21 and iter2 will be 33. + // Then if call Prev() in prefix mode where SeekForPrev(21) gets called, + // iter2 should turn to invalid state because of bloom filter. + options.memtable_factory.reset(new SkipListFactory); + options.write_buffer_size = 1024 * 1024; + std::string v13("v13"); + ASSERT_OK(DestroyDB(kDbName, Options())); + auto db = OpenDb(); + WriteOptions write_options; + ReadOptions read_options; + PutKey(db.get(), write_options, TestKey(1, 2), "v12"); + PutKey(db.get(), write_options, TestKey(1, 4), "v14"); + PutKey(db.get(), write_options, TestKey(3, 3), "v33"); + PutKey(db.get(), write_options, TestKey(3, 4), "v34"); + ASSERT_OK(db->Flush(FlushOptions())); + ASSERT_OK( + static_cast_with_check(db.get())->TEST_WaitForFlushMemTable()); + PutKey(db.get(), write_options, TestKey(1, 1), "v11"); + PutKey(db.get(), write_options, TestKey(1, 3), "v13"); + PutKey(db.get(), write_options, TestKey(2, 1), "v21"); + PutKey(db.get(), write_options, TestKey(2, 2), "v22"); + ASSERT_OK(db->Flush(FlushOptions())); + ASSERT_OK( + static_cast_with_check(db.get())->TEST_WaitForFlushMemTable()); + std::unique_ptr iter(db->NewIterator(read_options)); + SeekIterator(iter.get(), 1, 5); + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->value(), v13); +} + +TEST_F(PrefixTest, PrefixSeekModePrev3) { + // Only for SkipListFactory + // test SeekToLast() with iterate_upper_bound_ in prefix_seek_mode + options.memtable_factory.reset(new SkipListFactory); + options.write_buffer_size = 1024 * 1024; + std::string v14("v14"); + TestKey upper_bound_key = TestKey(1, 5); + std::string s; + Slice upper_bound = TestKeyToSlice(s, upper_bound_key); + + { + ASSERT_OK(DestroyDB(kDbName, Options())); + auto db = OpenDb(); + WriteOptions write_options; + ReadOptions read_options; + read_options.iterate_upper_bound = &upper_bound; + PutKey(db.get(), write_options, TestKey(1, 2), "v12"); + PutKey(db.get(), write_options, TestKey(1, 4), "v14"); + ASSERT_OK(db->Flush(FlushOptions())); + ASSERT_OK( + static_cast_with_check(db.get())->TEST_WaitForFlushMemTable()); + PutKey(db.get(), write_options, TestKey(1, 1), "v11"); + PutKey(db.get(), write_options, TestKey(1, 3), "v13"); + PutKey(db.get(), write_options, TestKey(2, 1), "v21"); + PutKey(db.get(), write_options, TestKey(2, 2), "v22"); + ASSERT_OK(db->Flush(FlushOptions())); + ASSERT_OK( + static_cast_with_check(db.get())->TEST_WaitForFlushMemTable()); + std::unique_ptr iter(db->NewIterator(read_options)); + iter->SeekToLast(); + ASSERT_EQ(iter->value(), v14); + } + { + ASSERT_OK(DestroyDB(kDbName, Options())); + auto db = OpenDb(); + WriteOptions write_options; + ReadOptions read_options; + read_options.iterate_upper_bound = &upper_bound; + PutKey(db.get(), write_options, TestKey(1, 2), "v12"); + PutKey(db.get(), write_options, TestKey(1, 4), "v14"); + PutKey(db.get(), write_options, TestKey(3, 3), "v33"); + PutKey(db.get(), write_options, TestKey(3, 4), "v34"); + ASSERT_OK(db->Flush(FlushOptions())); + ASSERT_OK( + static_cast_with_check(db.get())->TEST_WaitForFlushMemTable()); + PutKey(db.get(), write_options, TestKey(1, 1), "v11"); + PutKey(db.get(), write_options, TestKey(1, 3), "v13"); + ASSERT_OK(db->Flush(FlushOptions())); + ASSERT_OK( + static_cast_with_check(db.get())->TEST_WaitForFlushMemTable()); + std::unique_ptr iter(db->NewIterator(read_options)); + iter->SeekToLast(); + ASSERT_EQ(iter->value(), v14); + } +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + ParseCommandLineFlags(&argc, &argv, true); + return RUN_ALL_TESTS(); +} + +#endif // GFLAGS + diff --git a/librocksdb-sys/rocksdb/db/range_del_aggregator.cc b/librocksdb-sys/rocksdb/db/range_del_aggregator.cc new file mode 100644 index 0000000..6e76f9c --- /dev/null +++ b/librocksdb-sys/rocksdb/db/range_del_aggregator.cc @@ -0,0 +1,555 @@ +// Copyright (c) 2018-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/range_del_aggregator.h" + +#include "db/compaction/compaction_iteration_stats.h" +#include "db/dbformat.h" +#include "db/pinned_iterators_manager.h" +#include "db/range_del_aggregator.h" +#include "db/range_tombstone_fragmenter.h" +#include "db/version_edit.h" +#include "rocksdb/comparator.h" +#include "rocksdb/types.h" +#include "table/internal_iterator.h" +#include "table/scoped_arena_iterator.h" +#include "table/table_builder.h" +#include "util/heap.h" +#include "util/kv_map.h" +#include "util/vector_iterator.h" + +namespace ROCKSDB_NAMESPACE { + +TruncatedRangeDelIterator::TruncatedRangeDelIterator( + std::unique_ptr iter, + const InternalKeyComparator* icmp, const InternalKey* smallest, + const InternalKey* largest) + : iter_(std::move(iter)), + icmp_(icmp), + smallest_ikey_(smallest), + largest_ikey_(largest) { + // Set up bounds such that range tombstones from this iterator are + // truncated to range [smallest_, largest_). + if (smallest != nullptr) { + pinned_bounds_.emplace_back(); + auto& parsed_smallest = pinned_bounds_.back(); + Status pik_status = ParseInternalKey(smallest->Encode(), &parsed_smallest, + false /* log_err_key */); // TODO + pik_status.PermitUncheckedError(); + parsed_smallest.type = kTypeMaxValid; + assert(pik_status.ok()); + smallest_ = &parsed_smallest; + } + if (largest != nullptr) { + pinned_bounds_.emplace_back(); + auto& parsed_largest = pinned_bounds_.back(); + + Status pik_status = ParseInternalKey(largest->Encode(), &parsed_largest, + false /* log_err_key */); // TODO + pik_status.PermitUncheckedError(); + assert(pik_status.ok()); + + if (parsed_largest.type == kTypeRangeDeletion && + parsed_largest.sequence == kMaxSequenceNumber) { + // The file boundary has been artificially extended by a range tombstone. + // We do not need to adjust largest to properly truncate range + // tombstones that extend past the boundary. + } else if (parsed_largest.sequence == 0) { + // The largest key in the sstable has a sequence number of 0. Since we + // guarantee that no internal keys with the same user key and sequence + // number can exist in a DB, we know that the largest key in this sstable + // cannot exist as the smallest key in the next sstable. This further + // implies that no range tombstone in this sstable covers largest; + // otherwise, the file boundary would have been artificially extended. + // + // Therefore, we will never truncate a range tombstone at largest, so we + // can leave it unchanged. + // TODO: maybe use kMaxValid here to ensure range tombstone having + // distinct key from point keys. + } else { + // The same user key may straddle two sstable boundaries. To ensure that + // the truncated end key can cover the largest key in this sstable, reduce + // its sequence number by 1. + parsed_largest.sequence -= 1; + // This line is not needed for correctness, but it ensures that the + // truncated end key is not covering keys from the next SST file. + parsed_largest.type = kTypeMaxValid; + } + largest_ = &parsed_largest; + } +} + +bool TruncatedRangeDelIterator::Valid() const { + assert(iter_ != nullptr); + return iter_->Valid() && + (smallest_ == nullptr || + icmp_->Compare(*smallest_, iter_->parsed_end_key()) < 0) && + (largest_ == nullptr || + icmp_->Compare(iter_->parsed_start_key(), *largest_) < 0); +} + +// NOTE: target is a user key, with timestamp if enabled. +void TruncatedRangeDelIterator::Seek(const Slice& target) { + if (largest_ != nullptr && + icmp_->Compare(*largest_, ParsedInternalKey(target, kMaxSequenceNumber, + kTypeRangeDeletion)) <= 0) { + iter_->Invalidate(); + return; + } + if (smallest_ != nullptr && + icmp_->user_comparator()->Compare(target, smallest_->user_key) < 0) { + iter_->Seek(smallest_->user_key); + return; + } + iter_->Seek(target); +} + +void TruncatedRangeDelIterator::SeekInternalKey(const Slice& target) { + if (largest_ && icmp_->Compare(*largest_, target) <= 0) { + iter_->Invalidate(); + return; + } + if (smallest_ && icmp_->Compare(target, *smallest_) < 0) { + // Since target < smallest, target < largest_. + // This seek must land on a range tombstone where end_key() > target, + // so there is no need to check again. + iter_->Seek(smallest_->user_key); + } else { + iter_->Seek(ExtractUserKey(target)); + while (Valid() && icmp_->Compare(end_key(), target) <= 0) { + Next(); + } + } +} + +// NOTE: target is a user key, with timestamp if enabled. +void TruncatedRangeDelIterator::SeekForPrev(const Slice& target) { + if (smallest_ != nullptr && + icmp_->Compare(ParsedInternalKey(target, 0, kTypeRangeDeletion), + *smallest_) < 0) { + iter_->Invalidate(); + return; + } + if (largest_ != nullptr && + icmp_->user_comparator()->Compare(largest_->user_key, target) < 0) { + iter_->SeekForPrev(largest_->user_key); + return; + } + iter_->SeekForPrev(target); +} + +void TruncatedRangeDelIterator::SeekToFirst() { + if (smallest_ != nullptr) { + iter_->Seek(smallest_->user_key); + return; + } + iter_->SeekToTopFirst(); +} + +void TruncatedRangeDelIterator::SeekToLast() { + if (largest_ != nullptr) { + iter_->SeekForPrev(largest_->user_key); + return; + } + iter_->SeekToTopLast(); +} + +std::map> +TruncatedRangeDelIterator::SplitBySnapshot( + const std::vector& snapshots) { + using FragmentedIterPair = + std::pair>; + + auto split_untruncated_iters = iter_->SplitBySnapshot(snapshots); + std::map> + split_truncated_iters; + std::for_each( + split_untruncated_iters.begin(), split_untruncated_iters.end(), + [&](FragmentedIterPair& iter_pair) { + auto truncated_iter = std::make_unique( + std::move(iter_pair.second), icmp_, smallest_ikey_, largest_ikey_); + split_truncated_iters.emplace(iter_pair.first, + std::move(truncated_iter)); + }); + return split_truncated_iters; +} + +ForwardRangeDelIterator::ForwardRangeDelIterator( + const InternalKeyComparator* icmp) + : icmp_(icmp), + unused_idx_(0), + active_seqnums_(SeqMaxComparator()), + active_iters_(EndKeyMinComparator(icmp)), + inactive_iters_(StartKeyMinComparator(icmp)) {} + +bool ForwardRangeDelIterator::ShouldDelete(const ParsedInternalKey& parsed) { + // Move active iterators that end before parsed. + while (!active_iters_.empty() && + icmp_->Compare((*active_iters_.top())->end_key(), parsed) <= 0) { + TruncatedRangeDelIterator* iter = PopActiveIter(); + do { + iter->Next(); + } while (iter->Valid() && icmp_->Compare(iter->end_key(), parsed) <= 0); + PushIter(iter, parsed); + assert(active_iters_.size() == active_seqnums_.size()); + } + + // Move inactive iterators that start before parsed. + while (!inactive_iters_.empty() && + icmp_->Compare(inactive_iters_.top()->start_key(), parsed) <= 0) { + TruncatedRangeDelIterator* iter = PopInactiveIter(); + while (iter->Valid() && icmp_->Compare(iter->end_key(), parsed) <= 0) { + iter->Next(); + } + PushIter(iter, parsed); + assert(active_iters_.size() == active_seqnums_.size()); + } + + return active_seqnums_.empty() + ? false + : (*active_seqnums_.begin())->seq() > parsed.sequence; +} + +void ForwardRangeDelIterator::Invalidate() { + unused_idx_ = 0; + active_iters_.clear(); + active_seqnums_.clear(); + inactive_iters_.clear(); +} + +ReverseRangeDelIterator::ReverseRangeDelIterator( + const InternalKeyComparator* icmp) + : icmp_(icmp), + unused_idx_(0), + active_seqnums_(SeqMaxComparator()), + active_iters_(StartKeyMaxComparator(icmp)), + inactive_iters_(EndKeyMaxComparator(icmp)) {} + +bool ReverseRangeDelIterator::ShouldDelete(const ParsedInternalKey& parsed) { + // Move active iterators that start after parsed. + while (!active_iters_.empty() && + icmp_->Compare(parsed, (*active_iters_.top())->start_key()) < 0) { + TruncatedRangeDelIterator* iter = PopActiveIter(); + do { + iter->Prev(); + } while (iter->Valid() && icmp_->Compare(parsed, iter->start_key()) < 0); + PushIter(iter, parsed); + assert(active_iters_.size() == active_seqnums_.size()); + } + + // Move inactive iterators that end after parsed. + while (!inactive_iters_.empty() && + icmp_->Compare(parsed, inactive_iters_.top()->end_key()) < 0) { + TruncatedRangeDelIterator* iter = PopInactiveIter(); + while (iter->Valid() && icmp_->Compare(parsed, iter->start_key()) < 0) { + iter->Prev(); + } + PushIter(iter, parsed); + assert(active_iters_.size() == active_seqnums_.size()); + } + + return active_seqnums_.empty() + ? false + : (*active_seqnums_.begin())->seq() > parsed.sequence; +} + +void ReverseRangeDelIterator::Invalidate() { + unused_idx_ = 0; + active_iters_.clear(); + active_seqnums_.clear(); + inactive_iters_.clear(); +} + +bool RangeDelAggregator::StripeRep::ShouldDelete( + const ParsedInternalKey& parsed, RangeDelPositioningMode mode) { + if (!InStripe(parsed.sequence) || IsEmpty()) { + return false; + } + switch (mode) { + case RangeDelPositioningMode::kForwardTraversal: + InvalidateReverseIter(); + + // Pick up previously unseen iterators. + for (auto it = std::next(iters_.begin(), forward_iter_.UnusedIdx()); + it != iters_.end(); ++it, forward_iter_.IncUnusedIdx()) { + auto& iter = *it; + forward_iter_.AddNewIter(iter.get(), parsed); + } + + return forward_iter_.ShouldDelete(parsed); + case RangeDelPositioningMode::kBackwardTraversal: + InvalidateForwardIter(); + + // Pick up previously unseen iterators. + for (auto it = std::next(iters_.begin(), reverse_iter_.UnusedIdx()); + it != iters_.end(); ++it, reverse_iter_.IncUnusedIdx()) { + auto& iter = *it; + reverse_iter_.AddNewIter(iter.get(), parsed); + } + + return reverse_iter_.ShouldDelete(parsed); + default: + assert(false); + return false; + } +} + +bool RangeDelAggregator::StripeRep::IsRangeOverlapped(const Slice& start, + const Slice& end) { + Invalidate(); + + // Set the internal start/end keys so that: + // - if start_ikey has the same user key and sequence number as the + // current end key, start_ikey will be considered greater; and + // - if end_ikey has the same user key and sequence number as the current + // start key, end_ikey will be considered greater. + ParsedInternalKey start_ikey(start, kMaxSequenceNumber, + static_cast(0)); + ParsedInternalKey end_ikey(end, 0, static_cast(0)); + for (auto& iter : iters_) { + bool checked_candidate_tombstones = false; + for (iter->SeekForPrev(start); + iter->Valid() && icmp_->Compare(iter->start_key(), end_ikey) <= 0; + iter->Next()) { + checked_candidate_tombstones = true; + if (icmp_->Compare(start_ikey, iter->end_key()) < 0 && + icmp_->Compare(iter->start_key(), end_ikey) <= 0) { + return true; + } + } + + if (!checked_candidate_tombstones) { + // Do an additional check for when the end of the range is the begin + // key of a tombstone, which we missed earlier since SeekForPrev'ing + // to the start was invalid. + iter->SeekForPrev(end); + if (iter->Valid() && icmp_->Compare(start_ikey, iter->end_key()) < 0 && + icmp_->Compare(iter->start_key(), end_ikey) <= 0) { + return true; + } + } + } + return false; +} + +void ReadRangeDelAggregator::AddTombstones( + std::unique_ptr input_iter, + const InternalKey* smallest, const InternalKey* largest) { + if (input_iter == nullptr || input_iter->empty()) { + return; + } + rep_.AddTombstones(std::make_unique( + std::move(input_iter), icmp_, smallest, largest)); +} + +bool ReadRangeDelAggregator::ShouldDeleteImpl(const ParsedInternalKey& parsed, + RangeDelPositioningMode mode) { + return rep_.ShouldDelete(parsed, mode); +} + +bool ReadRangeDelAggregator::IsRangeOverlapped(const Slice& start, + const Slice& end) { + InvalidateRangeDelMapPositions(); + return rep_.IsRangeOverlapped(start, end); +} + +void CompactionRangeDelAggregator::AddTombstones( + std::unique_ptr input_iter, + const InternalKey* smallest, const InternalKey* largest) { + if (input_iter == nullptr || input_iter->empty()) { + return; + } + // This bounds output of CompactionRangeDelAggregator::NewIterator. + if (!trim_ts_.empty()) { + assert(icmp_->user_comparator()->timestamp_size() > 0); + input_iter->SetTimestampUpperBound(&trim_ts_); + } + + assert(input_iter->lower_bound() == 0); + assert(input_iter->upper_bound() == kMaxSequenceNumber); + parent_iters_.emplace_back(new TruncatedRangeDelIterator( + std::move(input_iter), icmp_, smallest, largest)); + + Slice* ts_upper_bound = nullptr; + if (!ts_upper_bound_.empty()) { + assert(icmp_->user_comparator()->timestamp_size() > 0); + ts_upper_bound = &ts_upper_bound_; + } + auto split_iters = parent_iters_.back()->SplitBySnapshot(*snapshots_); + for (auto& split_iter : split_iters) { + auto it = reps_.find(split_iter.first); + if (it == reps_.end()) { + bool inserted; + SequenceNumber upper_bound = split_iter.second->upper_bound(); + SequenceNumber lower_bound = split_iter.second->lower_bound(); + std::tie(it, inserted) = reps_.emplace( + split_iter.first, StripeRep(icmp_, upper_bound, lower_bound)); + assert(inserted); + } + assert(it != reps_.end()); + // ts_upper_bound is used to bound ShouldDelete() to only consider + // range tombstones under full_history_ts_low_ and trim_ts_. Keys covered by + // range tombstones that are above full_history_ts_low_ should not be + // dropped prematurely: user may read with a timestamp between the range + // tombstone and the covered key. Note that we cannot set timestamp + // upperbound on the original `input_iter` since `input_iter`s are later + // used in CompactionRangeDelAggregator::NewIterator to output range + // tombstones for persistence. We do not want to only persist range + // tombstones with timestamp lower than ts_upper_bound. + split_iter.second->SetTimestampUpperBound(ts_upper_bound); + it->second.AddTombstones(std::move(split_iter.second)); + } +} + +bool CompactionRangeDelAggregator::ShouldDelete(const ParsedInternalKey& parsed, + RangeDelPositioningMode mode) { + auto it = reps_.lower_bound(parsed.sequence); + if (it == reps_.end()) { + return false; + } + return it->second.ShouldDelete(parsed, mode); +} + +namespace { + +// Produce a sorted (by start internal key) stream of range tombstones from +// `children`. lower_bound and upper_bound on internal key can be +// optionally specified. Range tombstones that ends before lower_bound or starts +// after upper_bound are excluded. +// If user-defined timestamp is enabled, lower_bound and upper_bound should +// contain timestamp. +class TruncatedRangeDelMergingIter : public InternalIterator { + public: + TruncatedRangeDelMergingIter( + const InternalKeyComparator* icmp, const Slice* lower_bound, + const Slice* upper_bound, + const std::vector>& children) + : icmp_(icmp), + lower_bound_(lower_bound), + upper_bound_(upper_bound), + heap_(StartKeyMinComparator(icmp)), + ts_sz_(icmp_->user_comparator()->timestamp_size()) { + for (auto& child : children) { + if (child != nullptr) { + assert(child->lower_bound() == 0); + assert(child->upper_bound() == kMaxSequenceNumber); + children_.push_back(child.get()); + } + } + } + + bool Valid() const override { + return !heap_.empty() && !AfterEndKey(heap_.top()); + } + Status status() const override { return Status::OK(); } + + void SeekToFirst() override { + heap_.clear(); + for (auto& child : children_) { + if (lower_bound_ != nullptr) { + child->Seek(ExtractUserKey(*lower_bound_)); + // Since the above `Seek()` operates on a user key while `lower_bound_` + // is an internal key, we may need to advance `child` farther for it to + // be in bounds. + while (child->Valid() && BeforeStartKey(child)) { + child->InternalNext(); + } + } else { + child->SeekToFirst(); + } + if (child->Valid()) { + heap_.push(child); + } + } + } + + void Next() override { + auto* top = heap_.top(); + top->InternalNext(); + if (top->Valid()) { + heap_.replace_top(top); + } else { + heap_.pop(); + } + } + + Slice key() const override { + auto* top = heap_.top(); + if (ts_sz_) { + cur_start_key_.Set(top->start_key().user_key, top->seq(), + kTypeRangeDeletion, top->timestamp()); + } else { + cur_start_key_.Set(top->start_key().user_key, top->seq(), + kTypeRangeDeletion); + } + assert(top->start_key().user_key.size() >= ts_sz_); + return cur_start_key_.Encode(); + } + + Slice value() const override { + auto* top = heap_.top(); + if (!ts_sz_) { + return top->end_key().user_key; + } + assert(top->timestamp().size() == ts_sz_); + cur_end_key_.clear(); + cur_end_key_.append(top->end_key().user_key.data(), + top->end_key().user_key.size() - ts_sz_); + cur_end_key_.append(top->timestamp().data(), ts_sz_); + return cur_end_key_; + } + + // Unused InternalIterator methods + void Prev() override { assert(false); } + void Seek(const Slice& /* target */) override { assert(false); } + void SeekForPrev(const Slice& /* target */) override { assert(false); } + void SeekToLast() override { assert(false); } + + private: + bool BeforeStartKey(const TruncatedRangeDelIterator* iter) const { + if (lower_bound_ == nullptr) { + return false; + } + return icmp_->Compare(iter->end_key(), *lower_bound_) <= 0; + } + + bool AfterEndKey(const TruncatedRangeDelIterator* iter) const { + if (upper_bound_ == nullptr) { + return false; + } + return icmp_->Compare(iter->start_key(), *upper_bound_) > 0; + } + + const InternalKeyComparator* icmp_; + const Slice* lower_bound_; + const Slice* upper_bound_; + BinaryHeap heap_; + std::vector children_; + + mutable InternalKey cur_start_key_; + mutable std::string cur_end_key_; + size_t ts_sz_; +}; + +} // anonymous namespace + +std::unique_ptr +CompactionRangeDelAggregator::NewIterator(const Slice* lower_bound, + const Slice* upper_bound) { + InvalidateRangeDelMapPositions(); + auto merging_iter = std::make_unique( + icmp_, lower_bound, upper_bound, parent_iters_); + + auto fragmented_tombstone_list = + std::make_shared( + std::move(merging_iter), *icmp_, true /* for_compaction */, + *snapshots_); + + return std::make_unique( + fragmented_tombstone_list, *icmp_, kMaxSequenceNumber /* upper_bound */); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/range_del_aggregator.h b/librocksdb-sys/rocksdb/db/range_del_aggregator.h new file mode 100644 index 0000000..dc1e730 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/range_del_aggregator.h @@ -0,0 +1,478 @@ +// Copyright (c) 2018-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "db/compaction/compaction_iteration_stats.h" +#include "db/dbformat.h" +#include "db/pinned_iterators_manager.h" +#include "db/range_del_aggregator.h" +#include "db/range_tombstone_fragmenter.h" +#include "db/version_edit.h" +#include "rocksdb/comparator.h" +#include "rocksdb/types.h" +#include "table/internal_iterator.h" +#include "table/scoped_arena_iterator.h" +#include "table/table_builder.h" +#include "util/heap.h" +#include "util/kv_map.h" + +namespace ROCKSDB_NAMESPACE { + +class TruncatedRangeDelIterator { + public: + TruncatedRangeDelIterator( + std::unique_ptr iter, + const InternalKeyComparator* icmp, const InternalKey* smallest, + const InternalKey* largest); + + bool Valid() const; + + void Next() { iter_->TopNext(); } + void Prev() { iter_->TopPrev(); } + + void InternalNext() { iter_->Next(); } + + // Seeks to the tombstone with the highest visible sequence number that covers + // target (a user key). If no such tombstone exists, the position will be at + // the earliest tombstone that ends after target. + // REQUIRES: target is a user key. + void Seek(const Slice& target); + + // Seeks to the first range tombstone with end_key() > target. + void SeekInternalKey(const Slice& target); + + // Seeks to the tombstone with the highest visible sequence number that covers + // target (a user key). If no such tombstone exists, the position will be at + // the latest tombstone that starts before target. + void SeekForPrev(const Slice& target); + + void SeekToFirst(); + void SeekToLast(); + + ParsedInternalKey start_key() const { + return (smallest_ == nullptr || + icmp_->Compare(*smallest_, iter_->parsed_start_key()) <= 0) + ? iter_->parsed_start_key() + : *smallest_; + } + + ParsedInternalKey end_key() const { + return (largest_ == nullptr || + icmp_->Compare(iter_->parsed_end_key(), *largest_) <= 0) + ? iter_->parsed_end_key() + : *largest_; + } + + SequenceNumber seq() const { return iter_->seq(); } + Slice timestamp() const { + assert(icmp_->user_comparator()->timestamp_size()); + return iter_->timestamp(); + } + void SetTimestampUpperBound(const Slice* ts_upper_bound) { + iter_->SetTimestampUpperBound(ts_upper_bound); + } + + std::map> + SplitBySnapshot(const std::vector& snapshots); + + SequenceNumber upper_bound() const { return iter_->upper_bound(); } + + SequenceNumber lower_bound() const { return iter_->lower_bound(); } + + private: + std::unique_ptr iter_; + const InternalKeyComparator* icmp_; + const ParsedInternalKey* smallest_ = nullptr; + const ParsedInternalKey* largest_ = nullptr; + std::list pinned_bounds_; + + const InternalKey* smallest_ikey_; + const InternalKey* largest_ikey_; +}; + +struct SeqMaxComparator { + bool operator()(const TruncatedRangeDelIterator* a, + const TruncatedRangeDelIterator* b) const { + return a->seq() > b->seq(); + } +}; + +struct StartKeyMinComparator { + explicit StartKeyMinComparator(const InternalKeyComparator* c) : icmp(c) {} + + bool operator()(const TruncatedRangeDelIterator* a, + const TruncatedRangeDelIterator* b) const { + return icmp->Compare(a->start_key(), b->start_key()) > 0; + } + + const InternalKeyComparator* icmp; +}; + +class ForwardRangeDelIterator { + public: + explicit ForwardRangeDelIterator(const InternalKeyComparator* icmp); + + bool ShouldDelete(const ParsedInternalKey& parsed); + void Invalidate(); + + void AddNewIter(TruncatedRangeDelIterator* iter, + const ParsedInternalKey& parsed) { + iter->Seek(parsed.user_key); + PushIter(iter, parsed); + assert(active_iters_.size() == active_seqnums_.size()); + } + + size_t UnusedIdx() const { return unused_idx_; } + void IncUnusedIdx() { unused_idx_++; } + + private: + using ActiveSeqSet = + std::multiset; + + struct EndKeyMinComparator { + explicit EndKeyMinComparator(const InternalKeyComparator* c) : icmp(c) {} + + bool operator()(const ActiveSeqSet::const_iterator& a, + const ActiveSeqSet::const_iterator& b) const { + return icmp->Compare((*a)->end_key(), (*b)->end_key()) > 0; + } + + const InternalKeyComparator* icmp; + }; + + void PushIter(TruncatedRangeDelIterator* iter, + const ParsedInternalKey& parsed) { + if (!iter->Valid()) { + // The iterator has been fully consumed, so we don't need to add it to + // either of the heaps. + return; + } + int cmp = icmp_->Compare(parsed, iter->start_key()); + if (cmp < 0) { + PushInactiveIter(iter); + } else { + PushActiveIter(iter); + } + } + + void PushActiveIter(TruncatedRangeDelIterator* iter) { + auto seq_pos = active_seqnums_.insert(iter); + active_iters_.push(seq_pos); + } + + TruncatedRangeDelIterator* PopActiveIter() { + auto active_top = active_iters_.top(); + auto iter = *active_top; + active_iters_.pop(); + active_seqnums_.erase(active_top); + return iter; + } + + void PushInactiveIter(TruncatedRangeDelIterator* iter) { + inactive_iters_.push(iter); + } + + TruncatedRangeDelIterator* PopInactiveIter() { + auto* iter = inactive_iters_.top(); + inactive_iters_.pop(); + return iter; + } + + const InternalKeyComparator* icmp_; + size_t unused_idx_; + ActiveSeqSet active_seqnums_; + BinaryHeap active_iters_; + BinaryHeap inactive_iters_; +}; + +class ReverseRangeDelIterator { + public: + explicit ReverseRangeDelIterator(const InternalKeyComparator* icmp); + + bool ShouldDelete(const ParsedInternalKey& parsed); + void Invalidate(); + + void AddNewIter(TruncatedRangeDelIterator* iter, + const ParsedInternalKey& parsed) { + iter->SeekForPrev(parsed.user_key); + PushIter(iter, parsed); + assert(active_iters_.size() == active_seqnums_.size()); + } + + size_t UnusedIdx() const { return unused_idx_; } + void IncUnusedIdx() { unused_idx_++; } + + private: + using ActiveSeqSet = + std::multiset; + + struct EndKeyMaxComparator { + explicit EndKeyMaxComparator(const InternalKeyComparator* c) : icmp(c) {} + + bool operator()(const TruncatedRangeDelIterator* a, + const TruncatedRangeDelIterator* b) const { + return icmp->Compare(a->end_key(), b->end_key()) < 0; + } + + const InternalKeyComparator* icmp; + }; + struct StartKeyMaxComparator { + explicit StartKeyMaxComparator(const InternalKeyComparator* c) : icmp(c) {} + + bool operator()(const ActiveSeqSet::const_iterator& a, + const ActiveSeqSet::const_iterator& b) const { + return icmp->Compare((*a)->start_key(), (*b)->start_key()) < 0; + } + + const InternalKeyComparator* icmp; + }; + + void PushIter(TruncatedRangeDelIterator* iter, + const ParsedInternalKey& parsed) { + if (!iter->Valid()) { + // The iterator has been fully consumed, so we don't need to add it to + // either of the heaps. + } else if (icmp_->Compare(iter->end_key(), parsed) <= 0) { + PushInactiveIter(iter); + } else { + PushActiveIter(iter); + } + } + + void PushActiveIter(TruncatedRangeDelIterator* iter) { + auto seq_pos = active_seqnums_.insert(iter); + active_iters_.push(seq_pos); + } + + TruncatedRangeDelIterator* PopActiveIter() { + auto active_top = active_iters_.top(); + auto iter = *active_top; + active_iters_.pop(); + active_seqnums_.erase(active_top); + return iter; + } + + void PushInactiveIter(TruncatedRangeDelIterator* iter) { + inactive_iters_.push(iter); + } + + TruncatedRangeDelIterator* PopInactiveIter() { + auto* iter = inactive_iters_.top(); + inactive_iters_.pop(); + return iter; + } + + const InternalKeyComparator* icmp_; + size_t unused_idx_; + ActiveSeqSet active_seqnums_; + BinaryHeap active_iters_; + BinaryHeap inactive_iters_; +}; + +enum class RangeDelPositioningMode { kForwardTraversal, kBackwardTraversal }; +class RangeDelAggregator { + public: + explicit RangeDelAggregator(const InternalKeyComparator* icmp) + : icmp_(icmp) {} + virtual ~RangeDelAggregator() {} + + virtual void AddTombstones( + std::unique_ptr input_iter, + const InternalKey* smallest = nullptr, + const InternalKey* largest = nullptr) = 0; + + bool ShouldDelete(const Slice& ikey, RangeDelPositioningMode mode) { + ParsedInternalKey parsed; + + Status pik_status = + ParseInternalKey(ikey, &parsed, false /* log_err_key */); // TODO + assert(pik_status.ok()); + if (!pik_status.ok()) { + return false; + } + + return ShouldDelete(parsed, mode); + } + virtual bool ShouldDelete(const ParsedInternalKey& parsed, + RangeDelPositioningMode mode) = 0; + + virtual void InvalidateRangeDelMapPositions() = 0; + + virtual bool IsEmpty() const = 0; + + bool AddFile(uint64_t file_number) { + return files_seen_.insert(file_number).second; + } + + protected: + class StripeRep { + public: + StripeRep(const InternalKeyComparator* icmp, SequenceNumber upper_bound, + SequenceNumber lower_bound) + : icmp_(icmp), + forward_iter_(icmp), + reverse_iter_(icmp), + upper_bound_(upper_bound), + lower_bound_(lower_bound) {} + + void AddTombstones(std::unique_ptr input_iter) { + iters_.push_back(std::move(input_iter)); + } + + bool IsEmpty() const { return iters_.empty(); } + + bool ShouldDelete(const ParsedInternalKey& parsed, + RangeDelPositioningMode mode); + + void Invalidate() { + if (!IsEmpty()) { + InvalidateForwardIter(); + InvalidateReverseIter(); + } + } + + // If user-defined timestamp is enabled, `start` and `end` are user keys + // with timestamp. + bool IsRangeOverlapped(const Slice& start, const Slice& end); + + private: + bool InStripe(SequenceNumber seq) const { + return lower_bound_ <= seq && seq <= upper_bound_; + } + + void InvalidateForwardIter() { forward_iter_.Invalidate(); } + + void InvalidateReverseIter() { reverse_iter_.Invalidate(); } + + const InternalKeyComparator* icmp_; + std::vector> iters_; + ForwardRangeDelIterator forward_iter_; + ReverseRangeDelIterator reverse_iter_; + SequenceNumber upper_bound_; + SequenceNumber lower_bound_; + }; + + const InternalKeyComparator* icmp_; + + private: + std::set files_seen_; +}; + +class ReadRangeDelAggregator final : public RangeDelAggregator { + public: + ReadRangeDelAggregator(const InternalKeyComparator* icmp, + SequenceNumber upper_bound) + : RangeDelAggregator(icmp), + rep_(icmp, upper_bound, 0 /* lower_bound */) {} + ~ReadRangeDelAggregator() override {} + + using RangeDelAggregator::ShouldDelete; + void AddTombstones( + std::unique_ptr input_iter, + const InternalKey* smallest = nullptr, + const InternalKey* largest = nullptr) override; + + bool ShouldDelete(const ParsedInternalKey& parsed, + RangeDelPositioningMode mode) final override { + if (rep_.IsEmpty()) { + return false; + } + return ShouldDeleteImpl(parsed, mode); + } + + bool IsRangeOverlapped(const Slice& start, const Slice& end); + + void InvalidateRangeDelMapPositions() override { rep_.Invalidate(); } + + bool IsEmpty() const override { return rep_.IsEmpty(); } + + private: + StripeRep rep_; + + bool ShouldDeleteImpl(const ParsedInternalKey& parsed, + RangeDelPositioningMode mode); +}; + +class CompactionRangeDelAggregator : public RangeDelAggregator { + public: + CompactionRangeDelAggregator(const InternalKeyComparator* icmp, + const std::vector& snapshots, + const std::string* full_history_ts_low = nullptr, + const std::string* trim_ts = nullptr) + : RangeDelAggregator(icmp), snapshots_(&snapshots) { + if (full_history_ts_low) { + ts_upper_bound_ = *full_history_ts_low; + } + if (trim_ts) { + trim_ts_ = *trim_ts; + // Range tombstone newer than `trim_ts` or `full_history_ts_low` should + // not be considered in ShouldDelete(). + if (ts_upper_bound_.empty()) { + ts_upper_bound_ = trim_ts_; + } else if (!trim_ts_.empty() && icmp->user_comparator()->CompareTimestamp( + trim_ts_, ts_upper_bound_) < 0) { + ts_upper_bound_ = trim_ts_; + } + } + } + ~CompactionRangeDelAggregator() override {} + + void AddTombstones( + std::unique_ptr input_iter, + const InternalKey* smallest = nullptr, + const InternalKey* largest = nullptr) override; + + using RangeDelAggregator::ShouldDelete; + bool ShouldDelete(const ParsedInternalKey& parsed, + RangeDelPositioningMode mode) override; + + bool IsRangeOverlapped(const Slice& start, const Slice& end); + + void InvalidateRangeDelMapPositions() override { + for (auto& rep : reps_) { + rep.second.Invalidate(); + } + } + + bool IsEmpty() const override { + for (const auto& rep : reps_) { + if (!rep.second.IsEmpty()) { + return false; + } + } + return true; + } + + // Creates an iterator over all the range tombstones in the aggregator, for + // use in compaction. + // + // NOTE: the internal key boundaries are used for optimization purposes to + // reduce the number of tombstones that are passed to the fragmenter; they do + // not guarantee that the resulting iterator only contains range tombstones + // that cover keys in the provided range. If required, these bounds must be + // enforced during iteration. + std::unique_ptr NewIterator( + const Slice* lower_bound = nullptr, const Slice* upper_bound = nullptr); + + private: + std::vector> parent_iters_; + std::map reps_; + + const std::vector* snapshots_; + // min over full_history_ts_low and trim_ts_ + Slice ts_upper_bound_{}; + Slice trim_ts_{}; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/range_del_aggregator_bench.cc b/librocksdb-sys/rocksdb/db/range_del_aggregator_bench.cc new file mode 100644 index 0000000..9dca707 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/range_del_aggregator_bench.cc @@ -0,0 +1,280 @@ +// Copyright (c) 2018-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#ifndef GFLAGS +#include +int main() { + fprintf(stderr, "Please install gflags to run rocksdb tools\n"); + return 1; +} +#else + +#include +#include +#include +#include +#include +#include +#include + +#include "db/dbformat.h" +#include "db/range_del_aggregator.h" +#include "db/range_tombstone_fragmenter.h" +#include "rocksdb/comparator.h" +#include "rocksdb/system_clock.h" +#include "util/coding.h" +#include "util/gflags_compat.h" +#include "util/random.h" +#include "util/stop_watch.h" +#include "util/vector_iterator.h" + +using GFLAGS_NAMESPACE::ParseCommandLineFlags; + +DEFINE_int32(num_range_tombstones, 1000, "number of range tombstones created"); + +DEFINE_int32(num_runs, 1000, "number of test runs"); + +DEFINE_int32(tombstone_start_upper_bound, 1000, + "exclusive upper bound on range tombstone start keys"); + +DEFINE_int32(should_delete_upper_bound, 1000, + "exclusive upper bound on keys passed to ShouldDelete"); + +DEFINE_double(tombstone_width_mean, 100.0, "average range tombstone width"); + +DEFINE_double(tombstone_width_stddev, 0.0, + "standard deviation of range tombstone width"); + +DEFINE_int32(seed, 0, "random number generator seed"); + +DEFINE_int32(should_deletes_per_run, 1, "number of ShouldDelete calls per run"); + +DEFINE_int32(add_tombstones_per_run, 1, + "number of AddTombstones calls per run"); + +DEFINE_bool(use_compaction_range_del_aggregator, false, + "Whether to use CompactionRangeDelAggregator. Default is to use " + "ReadRangeDelAggregator."); + +namespace { + +struct Stats { + uint64_t time_add_tombstones = 0; + uint64_t time_first_should_delete = 0; + uint64_t time_rest_should_delete = 0; + uint64_t time_fragment_tombstones = 0; +}; + +std::ostream& operator<<(std::ostream& os, const Stats& s) { + std::ios fmt_holder(nullptr); + fmt_holder.copyfmt(os); + + os << std::left; + os << std::setw(25) << "Fragment Tombstones: " + << s.time_fragment_tombstones / + (FLAGS_add_tombstones_per_run * FLAGS_num_runs * 1.0e3) + << " us\n"; + os << std::setw(25) << "AddTombstones: " + << s.time_add_tombstones / + (FLAGS_add_tombstones_per_run * FLAGS_num_runs * 1.0e3) + << " us\n"; + os << std::setw(25) << "ShouldDelete (first): " + << s.time_first_should_delete / (FLAGS_num_runs * 1.0e3) << " us\n"; + if (FLAGS_should_deletes_per_run > 1) { + os << std::setw(25) << "ShouldDelete (rest): " + << s.time_rest_should_delete / + ((FLAGS_should_deletes_per_run - 1) * FLAGS_num_runs * 1.0e3) + << " us\n"; + } + + os.copyfmt(fmt_holder); + return os; +} + +auto icmp = ROCKSDB_NAMESPACE::InternalKeyComparator( + ROCKSDB_NAMESPACE::BytewiseComparator()); + +} // anonymous namespace + +namespace ROCKSDB_NAMESPACE { + +namespace { + +// A wrapper around RangeTombstones and the underlying data of its start and end +// keys. +struct PersistentRangeTombstone { + std::string start_key; + std::string end_key; + RangeTombstone tombstone; + + PersistentRangeTombstone(std::string start, std::string end, + SequenceNumber seq) + : start_key(std::move(start)), end_key(std::move(end)) { + tombstone = RangeTombstone(start_key, end_key, seq); + } + + PersistentRangeTombstone() = default; + + PersistentRangeTombstone(const PersistentRangeTombstone& t) { *this = t; } + + PersistentRangeTombstone& operator=(const PersistentRangeTombstone& t) { + start_key = t.start_key; + end_key = t.end_key; + tombstone = RangeTombstone(start_key, end_key, t.tombstone.seq_); + + return *this; + } + + PersistentRangeTombstone(PersistentRangeTombstone&& t) noexcept { *this = t; } + + PersistentRangeTombstone& operator=(PersistentRangeTombstone&& t) { + start_key = std::move(t.start_key); + end_key = std::move(t.end_key); + tombstone = RangeTombstone(start_key, end_key, t.tombstone.seq_); + + return *this; + } +}; + +struct TombstoneStartKeyComparator { + explicit TombstoneStartKeyComparator(const Comparator* c) : cmp(c) {} + + bool operator()(const RangeTombstone& a, const RangeTombstone& b) const { + return cmp->Compare(a.start_key_, b.start_key_) < 0; + } + + const Comparator* cmp; +}; + +std::unique_ptr MakeRangeDelIterator( + const std::vector& range_dels) { + std::vector keys, values; + for (const auto& range_del : range_dels) { + auto key_and_value = range_del.tombstone.Serialize(); + keys.push_back(key_and_value.first.Encode().ToString()); + values.push_back(key_and_value.second.ToString()); + } + return std::unique_ptr( + new VectorIterator(keys, values, &icmp)); +} + +// convert long to a big-endian slice key +static std::string Key(int64_t val) { + std::string little_endian_key; + std::string big_endian_key; + PutFixed64(&little_endian_key, val); + assert(little_endian_key.size() == sizeof(val)); + big_endian_key.resize(sizeof(val)); + for (size_t i = 0; i < sizeof(val); ++i) { + big_endian_key[i] = little_endian_key[sizeof(val) - 1 - i]; + } + return big_endian_key; +} + +} // anonymous namespace + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ParseCommandLineFlags(&argc, &argv, true); + + Stats stats; + ROCKSDB_NAMESPACE::SystemClock* clock = + ROCKSDB_NAMESPACE::SystemClock::Default().get(); + ROCKSDB_NAMESPACE::Random64 rnd(FLAGS_seed); + std::default_random_engine random_gen(FLAGS_seed); + std::normal_distribution normal_dist(FLAGS_tombstone_width_mean, + FLAGS_tombstone_width_stddev); + std::vector > + all_persistent_range_tombstones(FLAGS_add_tombstones_per_run); + for (int i = 0; i < FLAGS_add_tombstones_per_run; i++) { + all_persistent_range_tombstones[i] = + std::vector( + FLAGS_num_range_tombstones); + } + auto mode = ROCKSDB_NAMESPACE::RangeDelPositioningMode::kForwardTraversal; + std::vector snapshots{0}; + for (int i = 0; i < FLAGS_num_runs; i++) { + std::unique_ptr range_del_agg = + nullptr; + if (FLAGS_use_compaction_range_del_aggregator) { + range_del_agg.reset(new ROCKSDB_NAMESPACE::CompactionRangeDelAggregator( + &icmp, snapshots)); + } else { + range_del_agg.reset(new ROCKSDB_NAMESPACE::ReadRangeDelAggregator( + &icmp, ROCKSDB_NAMESPACE::kMaxSequenceNumber /* upper_bound */)); + } + + std::vector< + std::unique_ptr > + fragmented_range_tombstone_lists(FLAGS_add_tombstones_per_run); + + for (auto& persistent_range_tombstones : all_persistent_range_tombstones) { + // TODO(abhimadan): consider whether creating the range tombstones right + // before AddTombstones is artificially warming the cache compared to + // real workloads. + for (int j = 0; j < FLAGS_num_range_tombstones; j++) { + uint64_t start = rnd.Uniform(FLAGS_tombstone_start_upper_bound); + uint64_t end = static_cast( + std::round(start + std::max(1.0, normal_dist(random_gen)))); + persistent_range_tombstones[j] = + ROCKSDB_NAMESPACE::PersistentRangeTombstone( + ROCKSDB_NAMESPACE::Key(start), ROCKSDB_NAMESPACE::Key(end), j); + } + auto iter = + ROCKSDB_NAMESPACE::MakeRangeDelIterator(persistent_range_tombstones); + ROCKSDB_NAMESPACE::StopWatchNano stop_watch_fragment_tombstones( + clock, true /* auto_start */); + fragmented_range_tombstone_lists.emplace_back( + new ROCKSDB_NAMESPACE::FragmentedRangeTombstoneList( + std::move(iter), icmp, FLAGS_use_compaction_range_del_aggregator, + snapshots)); + stats.time_fragment_tombstones += + stop_watch_fragment_tombstones.ElapsedNanos(); + std::unique_ptr + fragmented_range_del_iter( + new ROCKSDB_NAMESPACE::FragmentedRangeTombstoneIterator( + fragmented_range_tombstone_lists.back().get(), icmp, + ROCKSDB_NAMESPACE::kMaxSequenceNumber)); + + ROCKSDB_NAMESPACE::StopWatchNano stop_watch_add_tombstones( + clock, true /* auto_start */); + range_del_agg->AddTombstones(std::move(fragmented_range_del_iter)); + stats.time_add_tombstones += stop_watch_add_tombstones.ElapsedNanos(); + } + + ROCKSDB_NAMESPACE::ParsedInternalKey parsed_key; + parsed_key.sequence = FLAGS_num_range_tombstones / 2; + parsed_key.type = ROCKSDB_NAMESPACE::kTypeValue; + + uint64_t first_key = rnd.Uniform(FLAGS_should_delete_upper_bound - + FLAGS_should_deletes_per_run + 1); + + for (int j = 0; j < FLAGS_should_deletes_per_run; j++) { + std::string key_string = ROCKSDB_NAMESPACE::Key(first_key + j); + parsed_key.user_key = key_string; + + ROCKSDB_NAMESPACE::StopWatchNano stop_watch_should_delete( + clock, true /* auto_start */); + range_del_agg->ShouldDelete(parsed_key, mode); + uint64_t call_time = stop_watch_should_delete.ElapsedNanos(); + + if (j == 0) { + stats.time_first_should_delete += call_time; + } else { + stats.time_rest_should_delete += call_time; + } + } + } + + std::cout << "=========================\n" + << "Results:\n" + << "=========================\n" + << stats; + + return 0; +} + +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db/range_del_aggregator_test.cc b/librocksdb-sys/rocksdb/db/range_del_aggregator_test.cc new file mode 100644 index 0000000..89391c9 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/range_del_aggregator_test.cc @@ -0,0 +1,713 @@ +// Copyright (c) 2018-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/range_del_aggregator.h" + +#include +#include +#include + +#include "db/db_test_util.h" +#include "db/dbformat.h" +#include "db/range_tombstone_fragmenter.h" +#include "test_util/testutil.h" +#include "util/vector_iterator.h" + +namespace ROCKSDB_NAMESPACE { + +class RangeDelAggregatorTest : public testing::Test {}; + +namespace { + +static auto bytewise_icmp = InternalKeyComparator(BytewiseComparator()); + +std::unique_ptr MakeRangeDelIter( + const std::vector& range_dels) { + std::vector keys, values; + for (const auto& range_del : range_dels) { + auto key_and_value = range_del.Serialize(); + keys.push_back(key_and_value.first.Encode().ToString()); + values.push_back(key_and_value.second.ToString()); + } + return std::unique_ptr( + new VectorIterator(keys, values, &bytewise_icmp)); +} + +std::vector> +MakeFragmentedTombstoneLists( + const std::vector>& range_dels_list) { + std::vector> fragment_lists; + for (const auto& range_dels : range_dels_list) { + auto range_del_iter = MakeRangeDelIter(range_dels); + fragment_lists.emplace_back(new FragmentedRangeTombstoneList( + std::move(range_del_iter), bytewise_icmp)); + } + return fragment_lists; +} + +struct TruncatedIterScanTestCase { + ParsedInternalKey start; + ParsedInternalKey end; + SequenceNumber seq; +}; + +struct TruncatedIterSeekTestCase { + Slice target; + ParsedInternalKey start; + ParsedInternalKey end; + SequenceNumber seq; + bool invalid; +}; + +struct ShouldDeleteTestCase { + ParsedInternalKey lookup_key; + bool result; +}; + +struct IsRangeOverlappedTestCase { + Slice start; + Slice end; + bool result; +}; + +ParsedInternalKey UncutEndpoint(const Slice& s) { + return ParsedInternalKey(s, kMaxSequenceNumber, kTypeRangeDeletion); +} + +ParsedInternalKey InternalValue(const Slice& key, SequenceNumber seq, + ValueType type = kTypeValue) { + return ParsedInternalKey(key, seq, type); +} + +void VerifyIterator( + TruncatedRangeDelIterator* iter, const InternalKeyComparator& icmp, + const std::vector& expected_range_dels) { + // Test forward iteration. + iter->SeekToFirst(); + for (size_t i = 0; i < expected_range_dels.size(); i++, iter->Next()) { + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ(0, icmp.Compare(iter->start_key(), expected_range_dels[i].start)); + EXPECT_EQ(0, icmp.Compare(iter->end_key(), expected_range_dels[i].end)); + EXPECT_EQ(expected_range_dels[i].seq, iter->seq()); + } + EXPECT_FALSE(iter->Valid()); + + // Test reverse iteration. + iter->SeekToLast(); + std::vector reverse_expected_range_dels( + expected_range_dels.rbegin(), expected_range_dels.rend()); + for (size_t i = 0; i < reverse_expected_range_dels.size(); + i++, iter->Prev()) { + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ(0, icmp.Compare(iter->start_key(), + reverse_expected_range_dels[i].start)); + EXPECT_EQ( + 0, icmp.Compare(iter->end_key(), reverse_expected_range_dels[i].end)); + EXPECT_EQ(reverse_expected_range_dels[i].seq, iter->seq()); + } + EXPECT_FALSE(iter->Valid()); +} + +void VerifySeek(TruncatedRangeDelIterator* iter, + const InternalKeyComparator& icmp, + const std::vector& test_cases) { + for (const auto& test_case : test_cases) { + iter->Seek(test_case.target); + if (test_case.invalid) { + ASSERT_FALSE(iter->Valid()); + } else { + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ(0, icmp.Compare(iter->start_key(), test_case.start)); + EXPECT_EQ(0, icmp.Compare(iter->end_key(), test_case.end)); + EXPECT_EQ(test_case.seq, iter->seq()); + } + } +} + +void VerifySeekForPrev( + TruncatedRangeDelIterator* iter, const InternalKeyComparator& icmp, + const std::vector& test_cases) { + for (const auto& test_case : test_cases) { + iter->SeekForPrev(test_case.target); + if (test_case.invalid) { + ASSERT_FALSE(iter->Valid()); + } else { + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ(0, icmp.Compare(iter->start_key(), test_case.start)); + EXPECT_EQ(0, icmp.Compare(iter->end_key(), test_case.end)); + EXPECT_EQ(test_case.seq, iter->seq()); + } + } +} + +void VerifyShouldDelete(RangeDelAggregator* range_del_agg, + const std::vector& test_cases) { + for (const auto& test_case : test_cases) { + EXPECT_EQ( + test_case.result, + range_del_agg->ShouldDelete( + test_case.lookup_key, RangeDelPositioningMode::kForwardTraversal)); + } + for (auto it = test_cases.rbegin(); it != test_cases.rend(); ++it) { + const auto& test_case = *it; + EXPECT_EQ( + test_case.result, + range_del_agg->ShouldDelete( + test_case.lookup_key, RangeDelPositioningMode::kBackwardTraversal)); + } +} + +void VerifyIsRangeOverlapped( + ReadRangeDelAggregator* range_del_agg, + const std::vector& test_cases) { + for (const auto& test_case : test_cases) { + EXPECT_EQ(test_case.result, + range_del_agg->IsRangeOverlapped(test_case.start, test_case.end)); + } +} + +void CheckIterPosition(const RangeTombstone& tombstone, + const FragmentedRangeTombstoneIterator* iter) { + // Test InternalIterator interface. + EXPECT_EQ(tombstone.start_key_, ExtractUserKey(iter->key())); + EXPECT_EQ(tombstone.end_key_, iter->value()); + EXPECT_EQ(tombstone.seq_, iter->seq()); + + // Test FragmentedRangeTombstoneIterator interface. + EXPECT_EQ(tombstone.start_key_, iter->start_key()); + EXPECT_EQ(tombstone.end_key_, iter->end_key()); + EXPECT_EQ(tombstone.seq_, GetInternalKeySeqno(iter->key())); +} + +void VerifyFragmentedRangeDels( + FragmentedRangeTombstoneIterator* iter, + const std::vector& expected_tombstones) { + iter->SeekToFirst(); + for (size_t i = 0; i < expected_tombstones.size(); i++, iter->Next()) { + ASSERT_TRUE(iter->Valid()); + CheckIterPosition(expected_tombstones[i], iter); + } + EXPECT_FALSE(iter->Valid()); +} + +} // anonymous namespace + +TEST_F(RangeDelAggregatorTest, EmptyTruncatedIter) { + auto range_del_iter = MakeRangeDelIter({}); + FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), + bytewise_icmp); + std::unique_ptr input_iter( + new FragmentedRangeTombstoneIterator(&fragment_list, bytewise_icmp, + kMaxSequenceNumber)); + + TruncatedRangeDelIterator iter(std::move(input_iter), &bytewise_icmp, nullptr, + nullptr); + + iter.SeekToFirst(); + ASSERT_FALSE(iter.Valid()); + + iter.SeekToLast(); + ASSERT_FALSE(iter.Valid()); +} + +TEST_F(RangeDelAggregatorTest, UntruncatedIter) { + auto range_del_iter = + MakeRangeDelIter({{"a", "e", 10}, {"e", "g", 8}, {"j", "n", 4}}); + FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), + bytewise_icmp); + std::unique_ptr input_iter( + new FragmentedRangeTombstoneIterator(&fragment_list, bytewise_icmp, + kMaxSequenceNumber)); + + TruncatedRangeDelIterator iter(std::move(input_iter), &bytewise_icmp, nullptr, + nullptr); + + VerifyIterator( + &iter, bytewise_icmp, + {{InternalValue("a", 10, kTypeRangeDeletion), UncutEndpoint("e"), 10}, + {InternalValue("e", 8, kTypeRangeDeletion), UncutEndpoint("g"), 8}, + {InternalValue("j", 4, kTypeRangeDeletion), UncutEndpoint("n"), 4}}); + + VerifySeek( + &iter, bytewise_icmp, + {{"d", InternalValue("a", 10, kTypeRangeDeletion), UncutEndpoint("e"), + 10}, + {"e", InternalValue("e", 8, kTypeRangeDeletion), UncutEndpoint("g"), 8}, + {"ia", InternalValue("j", 4, kTypeRangeDeletion), UncutEndpoint("n"), 4}, + {"n", InternalValue("", 0, kTypeRangeDeletion), UncutEndpoint(""), 0, + true /* invalid */}, + {"", InternalValue("a", 10, kTypeRangeDeletion), UncutEndpoint("e"), + 10}}); + + VerifySeekForPrev( + &iter, bytewise_icmp, + {{"d", InternalValue("a", 10, kTypeRangeDeletion), UncutEndpoint("e"), + 10}, + {"e", InternalValue("e", 8, kTypeRangeDeletion), UncutEndpoint("g"), 8}, + {"ia", InternalValue("e", 8, kTypeRangeDeletion), UncutEndpoint("g"), 8}, + {"n", InternalValue("j", 4, kTypeRangeDeletion), UncutEndpoint("n"), 4}, + {"", InternalValue("", 0, kTypeRangeDeletion), UncutEndpoint(""), 0, + true /* invalid */}}); +} + +TEST_F(RangeDelAggregatorTest, UntruncatedIterWithSnapshot) { + auto range_del_iter = + MakeRangeDelIter({{"a", "e", 10}, {"e", "g", 8}, {"j", "n", 4}}); + FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), + bytewise_icmp); + std::unique_ptr input_iter( + new FragmentedRangeTombstoneIterator(&fragment_list, bytewise_icmp, + 9 /* snapshot */)); + + TruncatedRangeDelIterator iter(std::move(input_iter), &bytewise_icmp, nullptr, + nullptr); + + VerifyIterator( + &iter, bytewise_icmp, + {{InternalValue("e", 8, kTypeRangeDeletion), UncutEndpoint("g"), 8}, + {InternalValue("j", 4, kTypeRangeDeletion), UncutEndpoint("n"), 4}}); + + VerifySeek( + &iter, bytewise_icmp, + {{"d", InternalValue("e", 8, kTypeRangeDeletion), UncutEndpoint("g"), 8}, + {"e", InternalValue("e", 8, kTypeRangeDeletion), UncutEndpoint("g"), 8}, + {"ia", InternalValue("j", 4, kTypeRangeDeletion), UncutEndpoint("n"), 4}, + {"n", InternalValue("", 0, kTypeRangeDeletion), UncutEndpoint(""), 0, + true /* invalid */}, + {"", InternalValue("e", 8, kTypeRangeDeletion), UncutEndpoint("g"), 8}}); + + VerifySeekForPrev( + &iter, bytewise_icmp, + {{"d", InternalValue("", 0, kTypeRangeDeletion), UncutEndpoint(""), 0, + true /* invalid */}, + {"e", InternalValue("e", 8, kTypeRangeDeletion), UncutEndpoint("g"), 8}, + {"ia", InternalValue("e", 8, kTypeRangeDeletion), UncutEndpoint("g"), 8}, + {"n", InternalValue("j", 4, kTypeRangeDeletion), UncutEndpoint("n"), 4}, + {"", InternalValue("", 0, kTypeRangeDeletion), UncutEndpoint(""), 0, + true /* invalid */}}); +} + +TEST_F(RangeDelAggregatorTest, TruncatedIterPartiallyCutTombstones) { + auto range_del_iter = + MakeRangeDelIter({{"a", "e", 10}, {"e", "g", 8}, {"j", "n", 4}}); + FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), + bytewise_icmp); + std::unique_ptr input_iter( + new FragmentedRangeTombstoneIterator(&fragment_list, bytewise_icmp, + kMaxSequenceNumber)); + + InternalKey smallest("d", 7, kTypeValue); + InternalKey largest("m", 9, kTypeValue); + TruncatedRangeDelIterator iter(std::move(input_iter), &bytewise_icmp, + &smallest, &largest); + + VerifyIterator( + &iter, bytewise_icmp, + {{InternalValue("d", 7, kTypeMaxValid), UncutEndpoint("e"), 10}, + {InternalValue("e", 8, kTypeRangeDeletion), UncutEndpoint("g"), 8}, + {InternalValue("j", 4, kTypeRangeDeletion), + InternalValue("m", 8, kTypeMaxValid), 4}}); + + VerifySeek( + &iter, bytewise_icmp, + {{"d", InternalValue("d", 7, kTypeMaxValid), UncutEndpoint("e"), 10}, + {"e", InternalValue("e", 8, kTypeRangeDeletion), UncutEndpoint("g"), 8}, + {"ia", InternalValue("j", 4, kTypeRangeDeletion), + InternalValue("m", 8, kTypeMaxValid), 4, false /* invalid */}, + {"n", InternalValue("", 0, kTypeRangeDeletion), UncutEndpoint(""), 0, + true /* invalid */}, + {"", InternalValue("d", 7, kTypeMaxValid), UncutEndpoint("e"), 10}}); + + VerifySeekForPrev( + &iter, bytewise_icmp, + {{"d", InternalValue("d", 7, kTypeMaxValid), UncutEndpoint("e"), 10}, + {"e", InternalValue("e", 8, kTypeRangeDeletion), UncutEndpoint("g"), 8}, + {"ia", InternalValue("e", 8, kTypeRangeDeletion), UncutEndpoint("g"), 8}, + {"n", InternalValue("j", 4, kTypeRangeDeletion), + InternalValue("m", 8, kTypeMaxValid), 4, false /* invalid */}, + {"", InternalValue("", 0, kTypeRangeDeletion), UncutEndpoint(""), 0, + true /* invalid */}}); +} + +TEST_F(RangeDelAggregatorTest, TruncatedIterFullyCutTombstones) { + auto range_del_iter = + MakeRangeDelIter({{"a", "e", 10}, {"e", "g", 8}, {"j", "n", 4}}); + FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), + bytewise_icmp); + std::unique_ptr input_iter( + new FragmentedRangeTombstoneIterator(&fragment_list, bytewise_icmp, + kMaxSequenceNumber)); + + InternalKey smallest("f", 7, kTypeValue); + InternalKey largest("i", 9, kTypeValue); + TruncatedRangeDelIterator iter(std::move(input_iter), &bytewise_icmp, + &smallest, &largest); + + VerifyIterator( + &iter, bytewise_icmp, + {{InternalValue("f", 7, kTypeMaxValid), UncutEndpoint("g"), 8}}); + + VerifySeek( + &iter, bytewise_icmp, + {{"d", InternalValue("f", 7, kTypeMaxValid), UncutEndpoint("g"), 8}, + {"f", InternalValue("f", 7, kTypeMaxValid), UncutEndpoint("g"), 8}, + {"j", InternalValue("", 0, kTypeRangeDeletion), UncutEndpoint(""), 0, + true /* invalid */}}); + + VerifySeekForPrev( + &iter, bytewise_icmp, + {{"d", InternalValue("", 0, kTypeRangeDeletion), UncutEndpoint(""), 0, + true /* invalid */}, + {"f", InternalValue("f", 7, kTypeMaxValid), UncutEndpoint("g"), 8}, + {"j", InternalValue("f", 7, kTypeMaxValid), UncutEndpoint("g"), 8}}); +} + +TEST_F(RangeDelAggregatorTest, SingleIterInAggregator) { + auto range_del_iter = MakeRangeDelIter({{"a", "e", 10}, {"c", "g", 8}}); + FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), + bytewise_icmp); + std::unique_ptr input_iter( + new FragmentedRangeTombstoneIterator(&fragment_list, bytewise_icmp, + kMaxSequenceNumber)); + + ReadRangeDelAggregator range_del_agg(&bytewise_icmp, kMaxSequenceNumber); + range_del_agg.AddTombstones(std::move(input_iter)); + + VerifyShouldDelete(&range_del_agg, {{InternalValue("a", 19), false}, + {InternalValue("b", 9), true}, + {InternalValue("d", 9), true}, + {InternalValue("e", 7), true}, + {InternalValue("g", 7), false}}); + + VerifyIsRangeOverlapped(&range_del_agg, {{"", "_", false}, + {"_", "a", true}, + {"a", "c", true}, + {"d", "f", true}, + {"g", "l", false}}); +} + +TEST_F(RangeDelAggregatorTest, MultipleItersInAggregator) { + auto fragment_lists = MakeFragmentedTombstoneLists( + {{{"a", "e", 10}, {"c", "g", 8}}, + {{"a", "b", 20}, {"h", "i", 25}, {"ii", "j", 15}}}); + + ReadRangeDelAggregator range_del_agg(&bytewise_icmp, kMaxSequenceNumber); + for (const auto& fragment_list : fragment_lists) { + std::unique_ptr input_iter( + new FragmentedRangeTombstoneIterator(fragment_list.get(), bytewise_icmp, + kMaxSequenceNumber)); + range_del_agg.AddTombstones(std::move(input_iter)); + } + + VerifyShouldDelete(&range_del_agg, {{InternalValue("a", 19), true}, + {InternalValue("b", 19), false}, + {InternalValue("b", 9), true}, + {InternalValue("d", 9), true}, + {InternalValue("e", 7), true}, + {InternalValue("g", 7), false}, + {InternalValue("h", 24), true}, + {InternalValue("i", 24), false}, + {InternalValue("ii", 14), true}, + {InternalValue("j", 14), false}}); + + VerifyIsRangeOverlapped(&range_del_agg, {{"", "_", false}, + {"_", "a", true}, + {"a", "c", true}, + {"d", "f", true}, + {"g", "l", true}, + {"x", "y", false}}); +} + +TEST_F(RangeDelAggregatorTest, MultipleItersInAggregatorWithUpperBound) { + auto fragment_lists = MakeFragmentedTombstoneLists( + {{{"a", "e", 10}, {"c", "g", 8}}, + {{"a", "b", 20}, {"h", "i", 25}, {"ii", "j", 15}}}); + + ReadRangeDelAggregator range_del_agg(&bytewise_icmp, 19); + for (const auto& fragment_list : fragment_lists) { + std::unique_ptr input_iter( + new FragmentedRangeTombstoneIterator(fragment_list.get(), bytewise_icmp, + 19 /* snapshot */)); + range_del_agg.AddTombstones(std::move(input_iter)); + } + + VerifyShouldDelete(&range_del_agg, {{InternalValue("a", 19), false}, + {InternalValue("a", 9), true}, + {InternalValue("b", 9), true}, + {InternalValue("d", 9), true}, + {InternalValue("e", 7), true}, + {InternalValue("g", 7), false}, + {InternalValue("h", 24), false}, + {InternalValue("i", 24), false}, + {InternalValue("ii", 14), true}, + {InternalValue("j", 14), false}}); + + VerifyIsRangeOverlapped(&range_del_agg, {{"", "_", false}, + {"_", "a", true}, + {"a", "c", true}, + {"d", "f", true}, + {"g", "l", true}, + {"x", "y", false}}); +} + +TEST_F(RangeDelAggregatorTest, MultipleTruncatedItersInAggregator) { + auto fragment_lists = MakeFragmentedTombstoneLists( + {{{"a", "z", 10}}, {{"a", "z", 10}}, {{"a", "z", 10}}}); + std::vector> iter_bounds = { + {InternalKey("a", 4, kTypeValue), + InternalKey("m", kMaxSequenceNumber, kTypeRangeDeletion)}, + {InternalKey("m", 20, kTypeValue), + InternalKey("x", kMaxSequenceNumber, kTypeRangeDeletion)}, + {InternalKey("x", 5, kTypeValue), InternalKey("zz", 30, kTypeValue)}}; + + ReadRangeDelAggregator range_del_agg(&bytewise_icmp, 19); + for (size_t i = 0; i < fragment_lists.size(); i++) { + const auto& fragment_list = fragment_lists[i]; + const auto& bounds = iter_bounds[i]; + std::unique_ptr input_iter( + new FragmentedRangeTombstoneIterator(fragment_list.get(), bytewise_icmp, + 19 /* snapshot */)); + range_del_agg.AddTombstones(std::move(input_iter), &bounds.first, + &bounds.second); + } + + VerifyShouldDelete(&range_del_agg, {{InternalValue("a", 10), false}, + {InternalValue("a", 9), false}, + {InternalValue("a", 4), true}, + {InternalValue("m", 10), false}, + {InternalValue("m", 9), true}, + {InternalValue("x", 10), false}, + {InternalValue("x", 9), false}, + {InternalValue("x", 5), true}, + {InternalValue("z", 9), false}}); + + VerifyIsRangeOverlapped(&range_del_agg, {{"", "_", false}, + {"_", "a", true}, + {"a", "n", true}, + {"l", "x", true}, + {"w", "z", true}, + {"zzz", "zz", false}, + {"zz", "zzz", false}}); +} + +TEST_F(RangeDelAggregatorTest, MultipleTruncatedItersInAggregatorSameLevel) { + auto fragment_lists = MakeFragmentedTombstoneLists( + {{{"a", "z", 10}}, {{"a", "z", 10}}, {{"a", "z", 10}}}); + std::vector> iter_bounds = { + {InternalKey("a", 4, kTypeValue), + InternalKey("m", kMaxSequenceNumber, kTypeRangeDeletion)}, + {InternalKey("m", 20, kTypeValue), + InternalKey("x", kMaxSequenceNumber, kTypeRangeDeletion)}, + {InternalKey("x", 5, kTypeValue), InternalKey("zz", 30, kTypeValue)}}; + + ReadRangeDelAggregator range_del_agg(&bytewise_icmp, 19); + + auto add_iter_to_agg = [&](size_t i) { + std::unique_ptr input_iter( + new FragmentedRangeTombstoneIterator(fragment_lists[i].get(), + bytewise_icmp, 19 /* snapshot */)); + range_del_agg.AddTombstones(std::move(input_iter), &iter_bounds[i].first, + &iter_bounds[i].second); + }; + + add_iter_to_agg(0); + VerifyShouldDelete(&range_del_agg, {{InternalValue("a", 10), false}, + {InternalValue("a", 9), false}, + {InternalValue("a", 4), true}}); + + add_iter_to_agg(1); + VerifyShouldDelete(&range_del_agg, {{InternalValue("m", 10), false}, + {InternalValue("m", 9), true}}); + + add_iter_to_agg(2); + VerifyShouldDelete(&range_del_agg, {{InternalValue("x", 10), false}, + {InternalValue("x", 9), false}, + {InternalValue("x", 5), true}, + {InternalValue("z", 9), false}}); + + VerifyIsRangeOverlapped(&range_del_agg, {{"", "_", false}, + {"_", "a", true}, + {"a", "n", true}, + {"l", "x", true}, + {"w", "z", true}, + {"zzz", "zz", false}, + {"zz", "zzz", false}}); +} + +TEST_F(RangeDelAggregatorTest, CompactionAggregatorNoSnapshots) { + auto fragment_lists = MakeFragmentedTombstoneLists( + {{{"a", "e", 10}, {"c", "g", 8}}, + {{"a", "b", 20}, {"h", "i", 25}, {"ii", "j", 15}}}); + + std::vector snapshots; + CompactionRangeDelAggregator range_del_agg(&bytewise_icmp, snapshots); + for (const auto& fragment_list : fragment_lists) { + std::unique_ptr input_iter( + new FragmentedRangeTombstoneIterator(fragment_list.get(), bytewise_icmp, + kMaxSequenceNumber)); + range_del_agg.AddTombstones(std::move(input_iter)); + } + + VerifyShouldDelete(&range_del_agg, {{InternalValue("a", 19), true}, + {InternalValue("b", 19), false}, + {InternalValue("b", 9), true}, + {InternalValue("d", 9), true}, + {InternalValue("e", 7), true}, + {InternalValue("g", 7), false}, + {InternalValue("h", 24), true}, + {InternalValue("i", 24), false}, + {InternalValue("ii", 14), true}, + {InternalValue("j", 14), false}}); + + auto range_del_compaction_iter = range_del_agg.NewIterator(); + VerifyFragmentedRangeDels(range_del_compaction_iter.get(), {{"a", "b", 20}, + {"b", "c", 10}, + {"c", "e", 10}, + {"e", "g", 8}, + {"h", "i", 25}, + {"ii", "j", 15}}); +} + +TEST_F(RangeDelAggregatorTest, CompactionAggregatorWithSnapshots) { + auto fragment_lists = MakeFragmentedTombstoneLists( + {{{"a", "e", 10}, {"c", "g", 8}}, + {{"a", "b", 20}, {"h", "i", 25}, {"ii", "j", 15}}}); + + std::vector snapshots{9, 19}; + CompactionRangeDelAggregator range_del_agg(&bytewise_icmp, snapshots); + for (const auto& fragment_list : fragment_lists) { + std::unique_ptr input_iter( + new FragmentedRangeTombstoneIterator(fragment_list.get(), bytewise_icmp, + kMaxSequenceNumber)); + range_del_agg.AddTombstones(std::move(input_iter)); + } + + VerifyShouldDelete( + &range_del_agg, + { + {InternalValue("a", 19), false}, // [10, 19] + {InternalValue("a", 9), false}, // [0, 9] + {InternalValue("b", 9), false}, // [0, 9] + {InternalValue("d", 9), false}, // [0, 9] + {InternalValue("d", 7), true}, // [0, 9] + {InternalValue("e", 7), true}, // [0, 9] + {InternalValue("g", 7), false}, // [0, 9] + {InternalValue("h", 24), true}, // [20, kMaxSequenceNumber] + {InternalValue("i", 24), false}, // [20, kMaxSequenceNumber] + {InternalValue("ii", 14), true}, // [10, 19] + {InternalValue("j", 14), false} // [10, 19] + }); + + auto range_del_compaction_iter = range_del_agg.NewIterator(); + VerifyFragmentedRangeDels(range_del_compaction_iter.get(), {{"a", "b", 20}, + {"a", "b", 10}, + {"b", "c", 10}, + {"c", "e", 10}, + {"c", "e", 8}, + {"e", "g", 8}, + {"h", "i", 25}, + {"ii", "j", 15}}); +} + +TEST_F(RangeDelAggregatorTest, CompactionAggregatorEmptyIteratorLeft) { + auto fragment_lists = MakeFragmentedTombstoneLists( + {{{"a", "e", 10}, {"c", "g", 8}}, + {{"a", "b", 20}, {"h", "i", 25}, {"ii", "j", 15}}}); + + std::vector snapshots{9, 19}; + CompactionRangeDelAggregator range_del_agg(&bytewise_icmp, snapshots); + for (const auto& fragment_list : fragment_lists) { + std::unique_ptr input_iter( + new FragmentedRangeTombstoneIterator(fragment_list.get(), bytewise_icmp, + kMaxSequenceNumber)); + range_del_agg.AddTombstones(std::move(input_iter)); + } + + Slice start("_"); + Slice end("__"); +} + +TEST_F(RangeDelAggregatorTest, CompactionAggregatorEmptyIteratorRight) { + auto fragment_lists = MakeFragmentedTombstoneLists( + {{{"a", "e", 10}, {"c", "g", 8}}, + {{"a", "b", 20}, {"h", "i", 25}, {"ii", "j", 15}}}); + + std::vector snapshots{9, 19}; + CompactionRangeDelAggregator range_del_agg(&bytewise_icmp, snapshots); + for (const auto& fragment_list : fragment_lists) { + std::unique_ptr input_iter( + new FragmentedRangeTombstoneIterator(fragment_list.get(), bytewise_icmp, + kMaxSequenceNumber)); + range_del_agg.AddTombstones(std::move(input_iter)); + } + + InternalKey start_buf("p", 0, kTypeRangeDeletion); + InternalKey end_buf("q", 0, kTypeRangeDeletion); + Slice start = start_buf.Encode(); + Slice end = end_buf.Encode(); + auto range_del_compaction_iter = range_del_agg.NewIterator(&start, &end); + VerifyFragmentedRangeDels(range_del_compaction_iter.get(), {}); +} + +TEST_F(RangeDelAggregatorTest, CompactionAggregatorBoundedIterator) { + auto fragment_lists = MakeFragmentedTombstoneLists( + {{{"a", "e", 10}, {"c", "g", 8}}, + {{"a", "b", 20}, {"h", "i", 25}, {"ii", "j", 15}}}); + + std::vector snapshots{9, 19}; + CompactionRangeDelAggregator range_del_agg(&bytewise_icmp, snapshots); + for (const auto& fragment_list : fragment_lists) { + std::unique_ptr input_iter( + new FragmentedRangeTombstoneIterator(fragment_list.get(), bytewise_icmp, + kMaxSequenceNumber)); + range_del_agg.AddTombstones(std::move(input_iter)); + } + + InternalKey start_buf("bb", 0, kTypeRangeDeletion); + InternalKey end_buf("e", 9, kTypeRangeDeletion); + Slice start = start_buf.Encode(); + Slice end = end_buf.Encode(); + auto range_del_compaction_iter = range_del_agg.NewIterator(&start, &end); + VerifyFragmentedRangeDels(range_del_compaction_iter.get(), + {{"a", "c", 10}, {"c", "e", 10}, {"c", "e", 8}}); +} + +TEST_F(RangeDelAggregatorTest, + CompactionAggregatorBoundedIteratorExtraFragments) { + auto fragment_lists = MakeFragmentedTombstoneLists( + {{{"a", "d", 10}, {"c", "g", 8}}, + {{"b", "c", 20}, {"d", "f", 30}, {"h", "i", 25}, {"ii", "j", 15}}}); + + std::vector snapshots{9, 19}; + CompactionRangeDelAggregator range_del_agg(&bytewise_icmp, snapshots); + for (const auto& fragment_list : fragment_lists) { + std::unique_ptr input_iter( + new FragmentedRangeTombstoneIterator(fragment_list.get(), bytewise_icmp, + kMaxSequenceNumber)); + range_del_agg.AddTombstones(std::move(input_iter)); + } + + InternalKey start_buf("bb", 0, kTypeRangeDeletion); + InternalKey end_buf("e", 0, kTypeRangeDeletion); + Slice start = start_buf.Encode(); + Slice end = end_buf.Encode(); + auto range_del_compaction_iter = range_del_agg.NewIterator(&start, &end); + VerifyFragmentedRangeDels(range_del_compaction_iter.get(), {{"a", "b", 10}, + {"b", "c", 20}, + {"b", "c", 10}, + {"c", "d", 10}, + {"c", "d", 8}, + {"d", "f", 30}, + {"d", "f", 8}, + {"f", "g", 8}}); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/range_tombstone_fragmenter.cc b/librocksdb-sys/rocksdb/db/range_tombstone_fragmenter.cc new file mode 100644 index 0000000..7e7cede --- /dev/null +++ b/librocksdb-sys/rocksdb/db/range_tombstone_fragmenter.cc @@ -0,0 +1,502 @@ +// Copyright (c) 2018-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/range_tombstone_fragmenter.h" + +#include +#include +#include +#include +#include + +#include "util/autovector.h" +#include "util/kv_map.h" +#include "util/vector_iterator.h" + +namespace ROCKSDB_NAMESPACE { + +FragmentedRangeTombstoneList::FragmentedRangeTombstoneList( + std::unique_ptr unfragmented_tombstones, + const InternalKeyComparator& icmp, bool for_compaction, + const std::vector& snapshots) { + if (unfragmented_tombstones == nullptr) { + return; + } + bool is_sorted = true; + InternalKey pinned_last_start_key; + Slice last_start_key; + num_unfragmented_tombstones_ = 0; + total_tombstone_payload_bytes_ = 0; + for (unfragmented_tombstones->SeekToFirst(); unfragmented_tombstones->Valid(); + unfragmented_tombstones->Next(), num_unfragmented_tombstones_++) { + total_tombstone_payload_bytes_ += unfragmented_tombstones->key().size() + + unfragmented_tombstones->value().size(); + if (num_unfragmented_tombstones_ > 0 && + icmp.Compare(last_start_key, unfragmented_tombstones->key()) > 0) { + is_sorted = false; + break; + } + if (unfragmented_tombstones->IsKeyPinned()) { + last_start_key = unfragmented_tombstones->key(); + } else { + pinned_last_start_key.DecodeFrom(unfragmented_tombstones->key()); + last_start_key = pinned_last_start_key.Encode(); + } + } + if (is_sorted) { + FragmentTombstones(std::move(unfragmented_tombstones), icmp, for_compaction, + snapshots); + return; + } + + // Sort the tombstones before fragmenting them. + std::vector keys, values; + keys.reserve(num_unfragmented_tombstones_); + values.reserve(num_unfragmented_tombstones_); + // Reset the counter to zero for the next iteration over keys. + total_tombstone_payload_bytes_ = 0; + for (unfragmented_tombstones->SeekToFirst(); unfragmented_tombstones->Valid(); + unfragmented_tombstones->Next()) { + total_tombstone_payload_bytes_ += unfragmented_tombstones->key().size() + + unfragmented_tombstones->value().size(); + keys.emplace_back(unfragmented_tombstones->key().data(), + unfragmented_tombstones->key().size()); + values.emplace_back(unfragmented_tombstones->value().data(), + unfragmented_tombstones->value().size()); + } + // VectorIterator implicitly sorts by key during construction. + auto iter = std::make_unique(std::move(keys), + std::move(values), &icmp); + FragmentTombstones(std::move(iter), icmp, for_compaction, snapshots); +} + +void FragmentedRangeTombstoneList::FragmentTombstones( + std::unique_ptr unfragmented_tombstones, + const InternalKeyComparator& icmp, bool for_compaction, + const std::vector& snapshots) { + Slice cur_start_key(nullptr, 0); + auto cmp = ParsedInternalKeyComparator(&icmp); + + // Stores the end keys and sequence numbers of range tombstones with a start + // key less than or equal to cur_start_key. Provides an ordering by end key + // for use in flush_current_tombstones. + std::set cur_end_keys(cmp); + + size_t ts_sz = icmp.user_comparator()->timestamp_size(); + // Given the next start key in unfragmented_tombstones, + // flush_current_tombstones writes every tombstone fragment that starts + // and ends with a key before next_start_key, and starts with a key greater + // than or equal to cur_start_key. + auto flush_current_tombstones = [&](const Slice& next_start_key) { + auto it = cur_end_keys.begin(); + bool reached_next_start_key = false; + for (; it != cur_end_keys.end() && !reached_next_start_key; ++it) { + Slice cur_end_key = it->user_key; + if (icmp.user_comparator()->CompareWithoutTimestamp(cur_start_key, + cur_end_key) == 0) { + // Empty tombstone. + continue; + } + if (icmp.user_comparator()->CompareWithoutTimestamp(next_start_key, + cur_end_key) <= 0) { + // All the end keys in [it, cur_end_keys.end()) are after + // next_start_key, so the tombstones they represent can be used in + // fragments that start with keys greater than or equal to + // next_start_key. However, the end keys we already passed will not be + // used in any more tombstone fragments. + // + // Remove the fully fragmented tombstones and stop iteration after a + // final round of flushing to preserve the tombstones we can create more + // fragments from. + reached_next_start_key = true; + cur_end_keys.erase(cur_end_keys.begin(), it); + cur_end_key = next_start_key; + } + + // Flush a range tombstone fragment [cur_start_key, cur_end_key), which + // should not overlap with the last-flushed tombstone fragment. + assert(tombstones_.empty() || + icmp.user_comparator()->CompareWithoutTimestamp( + tombstones_.back().end_key, cur_start_key) <= 0); + + // Sort the sequence numbers of the tombstones being fragmented in + // descending order, and then flush them in that order. + autovector seqnums_to_flush; + autovector timestamps_to_flush; + for (auto flush_it = it; flush_it != cur_end_keys.end(); ++flush_it) { + seqnums_to_flush.push_back(flush_it->sequence); + if (ts_sz) { + timestamps_to_flush.push_back( + ExtractTimestampFromUserKey(flush_it->user_key, ts_sz)); + } + } + // TODO: bind the two sorting together to be more efficient + std::sort(seqnums_to_flush.begin(), seqnums_to_flush.end(), + std::greater()); + if (ts_sz) { + std::sort(timestamps_to_flush.begin(), timestamps_to_flush.end(), + [icmp](const Slice& ts1, const Slice& ts2) { + return icmp.user_comparator()->CompareTimestamp(ts1, ts2) > + 0; + }); + } + + size_t start_idx = tombstone_seqs_.size(); + size_t end_idx = start_idx + seqnums_to_flush.size(); + + // If user-defined timestamp is enabled, we should not drop tombstones + // from any snapshot stripe. Garbage collection of range tombstones + // happens in CompactionOutputs::AddRangeDels(). + if (for_compaction && ts_sz == 0) { + // Drop all tombstone seqnums that are not preserved by a snapshot. + SequenceNumber next_snapshot = kMaxSequenceNumber; + for (auto seq : seqnums_to_flush) { + if (seq <= next_snapshot) { + // This seqnum is visible by a lower snapshot. + tombstone_seqs_.push_back(seq); + auto upper_bound_it = + std::lower_bound(snapshots.begin(), snapshots.end(), seq); + if (upper_bound_it == snapshots.begin()) { + // This seqnum is the topmost one visible by the earliest + // snapshot. None of the seqnums below it will be visible, so we + // can skip them. + break; + } + next_snapshot = *std::prev(upper_bound_it); + } + } + end_idx = tombstone_seqs_.size(); + } else { + // The fragmentation is being done for reads, so preserve all seqnums. + tombstone_seqs_.insert(tombstone_seqs_.end(), seqnums_to_flush.begin(), + seqnums_to_flush.end()); + if (ts_sz) { + tombstone_timestamps_.insert(tombstone_timestamps_.end(), + timestamps_to_flush.begin(), + timestamps_to_flush.end()); + } + } + + assert(start_idx < end_idx); + if (ts_sz) { + std::string start_key_with_max_ts; + AppendUserKeyWithMaxTimestamp(&start_key_with_max_ts, cur_start_key, + ts_sz); + pinned_slices_.emplace_back(std::move(start_key_with_max_ts)); + Slice start_key = pinned_slices_.back(); + + std::string end_key_with_max_ts; + AppendUserKeyWithMaxTimestamp(&end_key_with_max_ts, cur_end_key, ts_sz); + pinned_slices_.emplace_back(std::move(end_key_with_max_ts)); + Slice end_key = pinned_slices_.back(); + + // RangeTombstoneStack expects start_key and end_key to have max + // timestamp. + tombstones_.emplace_back(start_key, end_key, start_idx, end_idx); + } else { + tombstones_.emplace_back(cur_start_key, cur_end_key, start_idx, + end_idx); + } + + cur_start_key = cur_end_key; + } + if (!reached_next_start_key) { + // There is a gap between the last flushed tombstone fragment and + // the next tombstone's start key. Remove all the end keys in + // the working set, since we have fully fragmented their corresponding + // tombstones. + cur_end_keys.clear(); + } + cur_start_key = next_start_key; + }; + + pinned_iters_mgr_.StartPinning(); + + bool no_tombstones = true; + for (unfragmented_tombstones->SeekToFirst(); unfragmented_tombstones->Valid(); + unfragmented_tombstones->Next()) { + const Slice& ikey = unfragmented_tombstones->key(); + Slice tombstone_start_key = ExtractUserKey(ikey); + SequenceNumber tombstone_seq = GetInternalKeySeqno(ikey); + if (!unfragmented_tombstones->IsKeyPinned()) { + pinned_slices_.emplace_back(tombstone_start_key.data(), + tombstone_start_key.size()); + tombstone_start_key = pinned_slices_.back(); + } + no_tombstones = false; + + Slice tombstone_end_key = unfragmented_tombstones->value(); + if (!unfragmented_tombstones->IsValuePinned()) { + pinned_slices_.emplace_back(tombstone_end_key.data(), + tombstone_end_key.size()); + tombstone_end_key = pinned_slices_.back(); + } + if (!cur_end_keys.empty() && + icmp.user_comparator()->CompareWithoutTimestamp( + cur_start_key, tombstone_start_key) != 0) { + // The start key has changed. Flush all tombstones that start before + // this new start key. + flush_current_tombstones(tombstone_start_key); + } + cur_start_key = tombstone_start_key; + + cur_end_keys.emplace(tombstone_end_key, tombstone_seq, kTypeRangeDeletion); + } + if (!cur_end_keys.empty()) { + ParsedInternalKey last_end_key = *std::prev(cur_end_keys.end()); + flush_current_tombstones(last_end_key.user_key); + } + + if (!no_tombstones) { + pinned_iters_mgr_.PinIterator(unfragmented_tombstones.release(), + false /* arena */); + } +} + +bool FragmentedRangeTombstoneList::ContainsRange(SequenceNumber lower, + SequenceNumber upper) { + std::call_once(seq_set_init_once_flag_, [this]() { + for (auto s : tombstone_seqs_) { + seq_set_.insert(s); + } + }); + auto seq_it = seq_set_.lower_bound(lower); + return seq_it != seq_set_.end() && *seq_it <= upper; +} + +FragmentedRangeTombstoneIterator::FragmentedRangeTombstoneIterator( + FragmentedRangeTombstoneList* tombstones, const InternalKeyComparator& icmp, + SequenceNumber _upper_bound, const Slice* ts_upper_bound, + SequenceNumber _lower_bound) + : tombstone_start_cmp_(icmp.user_comparator()), + tombstone_end_cmp_(icmp.user_comparator()), + icmp_(&icmp), + ucmp_(icmp.user_comparator()), + tombstones_(tombstones), + upper_bound_(_upper_bound), + lower_bound_(_lower_bound), + ts_upper_bound_(ts_upper_bound) { + assert(tombstones_ != nullptr); + Invalidate(); +} + +FragmentedRangeTombstoneIterator::FragmentedRangeTombstoneIterator( + const std::shared_ptr& tombstones, + const InternalKeyComparator& icmp, SequenceNumber _upper_bound, + const Slice* ts_upper_bound, SequenceNumber _lower_bound) + : tombstone_start_cmp_(icmp.user_comparator()), + tombstone_end_cmp_(icmp.user_comparator()), + icmp_(&icmp), + ucmp_(icmp.user_comparator()), + tombstones_ref_(tombstones), + tombstones_(tombstones_ref_.get()), + upper_bound_(_upper_bound), + lower_bound_(_lower_bound), + ts_upper_bound_(ts_upper_bound) { + assert(tombstones_ != nullptr); + Invalidate(); +} + +FragmentedRangeTombstoneIterator::FragmentedRangeTombstoneIterator( + const std::shared_ptr& tombstones_cache, + const InternalKeyComparator& icmp, SequenceNumber _upper_bound, + const Slice* ts_upper_bound, SequenceNumber _lower_bound) + : tombstone_start_cmp_(icmp.user_comparator()), + tombstone_end_cmp_(icmp.user_comparator()), + icmp_(&icmp), + ucmp_(icmp.user_comparator()), + tombstones_cache_ref_(tombstones_cache), + tombstones_(tombstones_cache_ref_->tombstones.get()), + upper_bound_(_upper_bound), + lower_bound_(_lower_bound) { + assert(tombstones_ != nullptr); + if (!ts_upper_bound || ts_upper_bound->empty()) { + ts_upper_bound_ = nullptr; + } else { + ts_upper_bound_ = ts_upper_bound; + } + Invalidate(); +} + +void FragmentedRangeTombstoneIterator::SeekToFirst() { + pos_ = tombstones_->begin(); + seq_pos_ = tombstones_->seq_begin(); +} + +void FragmentedRangeTombstoneIterator::SeekToTopFirst() { + if (tombstones_->empty()) { + Invalidate(); + return; + } + pos_ = tombstones_->begin(); + SetMaxVisibleSeqAndTimestamp(); + ScanForwardToVisibleTombstone(); +} + +void FragmentedRangeTombstoneIterator::SeekToLast() { + pos_ = std::prev(tombstones_->end()); + seq_pos_ = std::prev(tombstones_->seq_end()); +} + +void FragmentedRangeTombstoneIterator::SeekToTopLast() { + if (tombstones_->empty()) { + Invalidate(); + return; + } + pos_ = std::prev(tombstones_->end()); + SetMaxVisibleSeqAndTimestamp(); + ScanBackwardToVisibleTombstone(); +} + +// @param `target` is a user key, with timestamp if user-defined timestamp is +// enabled. +void FragmentedRangeTombstoneIterator::Seek(const Slice& target) { + if (tombstones_->empty()) { + Invalidate(); + return; + } + SeekToCoveringTombstone(target); + ScanForwardToVisibleTombstone(); +} + +void FragmentedRangeTombstoneIterator::SeekForPrev(const Slice& target) { + if (tombstones_->empty()) { + Invalidate(); + return; + } + SeekForPrevToCoveringTombstone(target); + ScanBackwardToVisibleTombstone(); +} + +void FragmentedRangeTombstoneIterator::SeekToCoveringTombstone( + const Slice& target) { + pos_ = std::upper_bound(tombstones_->begin(), tombstones_->end(), target, + tombstone_end_cmp_); + if (pos_ == tombstones_->end()) { + // All tombstones end before target. + seq_pos_ = tombstones_->seq_end(); + return; + } + SetMaxVisibleSeqAndTimestamp(); +} + +void FragmentedRangeTombstoneIterator::SeekForPrevToCoveringTombstone( + const Slice& target) { + if (tombstones_->empty()) { + Invalidate(); + return; + } + pos_ = std::upper_bound(tombstones_->begin(), tombstones_->end(), target, + tombstone_start_cmp_); + if (pos_ == tombstones_->begin()) { + // All tombstones start after target. + Invalidate(); + return; + } + --pos_; + SetMaxVisibleSeqAndTimestamp(); +} + +void FragmentedRangeTombstoneIterator::ScanForwardToVisibleTombstone() { + while (pos_ != tombstones_->end() && + (seq_pos_ == tombstones_->seq_iter(pos_->seq_end_idx) || + *seq_pos_ < lower_bound_)) { + ++pos_; + if (pos_ == tombstones_->end()) { + Invalidate(); + return; + } + SetMaxVisibleSeqAndTimestamp(); + } +} + +void FragmentedRangeTombstoneIterator::ScanBackwardToVisibleTombstone() { + while (pos_ != tombstones_->end() && + (seq_pos_ == tombstones_->seq_iter(pos_->seq_end_idx) || + *seq_pos_ < lower_bound_)) { + if (pos_ == tombstones_->begin()) { + Invalidate(); + return; + } + --pos_; + SetMaxVisibleSeqAndTimestamp(); + } +} + +void FragmentedRangeTombstoneIterator::Next() { + ++seq_pos_; + if (seq_pos_ == tombstones_->seq_iter(pos_->seq_end_idx)) { + ++pos_; + } +} + +void FragmentedRangeTombstoneIterator::TopNext() { + ++pos_; + if (pos_ == tombstones_->end()) { + return; + } + SetMaxVisibleSeqAndTimestamp(); + ScanForwardToVisibleTombstone(); +} + +void FragmentedRangeTombstoneIterator::Prev() { + if (seq_pos_ == tombstones_->seq_begin()) { + Invalidate(); + return; + } + --seq_pos_; + if (pos_ == tombstones_->end() || + seq_pos_ == tombstones_->seq_iter(pos_->seq_start_idx - 1)) { + --pos_; + } +} + +void FragmentedRangeTombstoneIterator::TopPrev() { + if (pos_ == tombstones_->begin()) { + Invalidate(); + return; + } + --pos_; + SetMaxVisibleSeqAndTimestamp(); + ScanBackwardToVisibleTombstone(); +} + +bool FragmentedRangeTombstoneIterator::Valid() const { + return tombstones_ != nullptr && pos_ != tombstones_->end(); +} + +SequenceNumber FragmentedRangeTombstoneIterator::MaxCoveringTombstoneSeqnum( + const Slice& target_user_key) { + SeekToCoveringTombstone(target_user_key); + return ValidPos() && ucmp_->CompareWithoutTimestamp(start_key(), + target_user_key) <= 0 + ? seq() + : 0; +} + +std::map> +FragmentedRangeTombstoneIterator::SplitBySnapshot( + const std::vector& snapshots) { + std::map> + splits; + SequenceNumber lower = 0; + SequenceNumber upper; + for (size_t i = 0; i <= snapshots.size(); i++) { + if (i >= snapshots.size()) { + upper = kMaxSequenceNumber; + } else { + upper = snapshots[i]; + } + if (tombstones_->ContainsRange(lower, upper)) { + splits.emplace(upper, + std::make_unique( + tombstones_, *icmp_, upper, ts_upper_bound_, lower)); + } + lower = upper + 1; + } + return splits; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/range_tombstone_fragmenter.h b/librocksdb-sys/rocksdb/db/range_tombstone_fragmenter.h new file mode 100644 index 0000000..8c7d982 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/range_tombstone_fragmenter.h @@ -0,0 +1,356 @@ +// Copyright (c) 2018-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include +#include +#include + +#include "db/dbformat.h" +#include "db/pinned_iterators_manager.h" +#include "rocksdb/status.h" +#include "table/internal_iterator.h" + +namespace ROCKSDB_NAMESPACE { +struct FragmentedRangeTombstoneList; + +struct FragmentedRangeTombstoneListCache { + // ensure only the first reader needs to initialize l + std::mutex reader_mutex; + std::unique_ptr tombstones = nullptr; + // readers will first check this bool to avoid + std::atomic initialized = false; +}; + +struct FragmentedRangeTombstoneList { + public: + // A compact representation of a "stack" of range tombstone fragments, which + // start and end at the same user keys but have different sequence numbers. + // The members seq_start_idx and seq_end_idx are intended to be parameters to + // seq_iter(). + // If user-defined timestamp is enabled, `start` and `end` should be user keys + // with timestamp, and the timestamps are set to max timestamp to be returned + // by parsed_start_key()/parsed_end_key(). seq_start_idx and seq_end_idx will + // also be used as parameters to ts_iter(). + struct RangeTombstoneStack { + RangeTombstoneStack(const Slice& start, const Slice& end, size_t start_idx, + size_t end_idx) + : start_key(start), + end_key(end), + seq_start_idx(start_idx), + seq_end_idx(end_idx) {} + Slice start_key; + Slice end_key; + size_t seq_start_idx; + size_t seq_end_idx; + }; + // Assumes unfragmented_tombstones->key() and unfragmented_tombstones->value() + // both contain timestamp if enabled. + FragmentedRangeTombstoneList( + std::unique_ptr unfragmented_tombstones, + const InternalKeyComparator& icmp, bool for_compaction = false, + const std::vector& snapshots = {}); + + std::vector::const_iterator begin() const { + return tombstones_.begin(); + } + + std::vector::const_iterator end() const { + return tombstones_.end(); + } + + std::vector::const_iterator seq_iter(size_t idx) const { + return std::next(tombstone_seqs_.begin(), idx); + } + + std::vector::const_iterator ts_iter(size_t idx) const { + return std::next(tombstone_timestamps_.begin(), idx); + } + + std::vector::const_iterator seq_begin() const { + return tombstone_seqs_.begin(); + } + + std::vector::const_iterator seq_end() const { + return tombstone_seqs_.end(); + } + + bool empty() const { return tombstones_.empty(); } + + // Returns true if the stored tombstones contain with one with a sequence + // number in [lower, upper]. + // This method is not const as it internally lazy initialize a set of + // sequence numbers (`seq_set_`). + bool ContainsRange(SequenceNumber lower, SequenceNumber upper); + + uint64_t num_unfragmented_tombstones() const { + return num_unfragmented_tombstones_; + } + + uint64_t total_tombstone_payload_bytes() const { + return total_tombstone_payload_bytes_; + } + + private: + // Given an ordered range tombstone iterator unfragmented_tombstones, + // "fragment" the tombstones into non-overlapping pieces. Each + // "non-overlapping piece" is a RangeTombstoneStack in tombstones_, which + // contains start_key, end_key, and indices that points to sequence numbers + // (in tombstone_seqs_) and timestamps (in tombstone_timestamps_). If + // for_compaction is true, then `snapshots` should be provided. Range + // tombstone fragments are dropped if they are not visible in any snapshot and + // user-defined timestamp is not enabled. That is, for each snapshot stripe + // [lower, upper], the range tombstone fragment with largest seqno in [lower, + // upper] is preserved, and all the other range tombstones are dropped. + void FragmentTombstones( + std::unique_ptr unfragmented_tombstones, + const InternalKeyComparator& icmp, bool for_compaction, + const std::vector& snapshots); + + std::vector tombstones_; + std::vector tombstone_seqs_; + std::vector tombstone_timestamps_; + std::once_flag seq_set_init_once_flag_; + std::set seq_set_; + std::list pinned_slices_; + PinnedIteratorsManager pinned_iters_mgr_; + uint64_t num_unfragmented_tombstones_; + uint64_t total_tombstone_payload_bytes_; +}; + +// FragmentedRangeTombstoneIterator converts an InternalIterator of a range-del +// meta block into an iterator over non-overlapping tombstone fragments. The +// tombstone fragmentation process should be more efficient than the range +// tombstone collapsing algorithm in RangeDelAggregator because this leverages +// the internal key ordering already provided by the input iterator, if +// applicable (when the iterator is unsorted, a new sorted iterator is created +// before proceeding). If there are few overlaps, creating a +// FragmentedRangeTombstoneIterator should be O(n), while the RangeDelAggregator +// tombstone collapsing is always O(n log n). +class FragmentedRangeTombstoneIterator : public InternalIterator { + public: + FragmentedRangeTombstoneIterator(FragmentedRangeTombstoneList* tombstones, + const InternalKeyComparator& icmp, + SequenceNumber upper_bound, + const Slice* ts_upper_bound = nullptr, + SequenceNumber lower_bound = 0); + FragmentedRangeTombstoneIterator( + const std::shared_ptr& tombstones, + const InternalKeyComparator& icmp, SequenceNumber upper_bound, + const Slice* ts_upper_bound = nullptr, SequenceNumber lower_bound = 0); + FragmentedRangeTombstoneIterator( + const std::shared_ptr& tombstones, + const InternalKeyComparator& icmp, SequenceNumber upper_bound, + const Slice* ts_upper_bound = nullptr, SequenceNumber lower_bound = 0); + + void SeekToFirst() override; + void SeekToLast() override; + + void SeekToTopFirst(); + void SeekToTopLast(); + + // NOTE: Seek and SeekForPrev do not behave in the way InternalIterator + // seeking should behave. This is OK because they are not currently used, but + // eventually FragmentedRangeTombstoneIterator should no longer implement + // InternalIterator. + // + // Seeks to the range tombstone that covers target at a seqnum in the + // snapshot. If no such tombstone exists, seek to the earliest tombstone in + // the snapshot that ends after target. + void Seek(const Slice& target) override; + // Seeks to the range tombstone that covers target at a seqnum in the + // snapshot. If no such tombstone exists, seek to the latest tombstone in the + // snapshot that starts before target. + void SeekForPrev(const Slice& target) override; + + void Next() override; + void Prev() override; + + void TopNext(); + void TopPrev(); + + bool Valid() const override; + // Note that key() and value() do not return correct timestamp. + // Caller should call timestamp() to get the current timestamp. + Slice key() const override { + MaybePinKey(); + return current_start_key_.Encode(); + } + Slice value() const override { return pos_->end_key; } + bool IsKeyPinned() const override { return false; } + bool IsValuePinned() const override { return true; } + Status status() const override { return Status::OK(); } + + bool empty() const { return tombstones_->empty(); } + void Invalidate() { + pos_ = tombstones_->end(); + seq_pos_ = tombstones_->seq_end(); + pinned_pos_ = tombstones_->end(); + pinned_seq_pos_ = tombstones_->seq_end(); + } + + RangeTombstone Tombstone() const { + assert(Valid()); + if (icmp_->user_comparator()->timestamp_size()) { + return RangeTombstone(start_key(), end_key(), seq(), timestamp()); + } + return RangeTombstone(start_key(), end_key(), seq()); + } + // Note that start_key() and end_key() are not guaranteed to have the + // correct timestamp. User can call timestamp() to get the correct + // timestamp(). + Slice start_key() const { return pos_->start_key; } + Slice end_key() const { return pos_->end_key; } + SequenceNumber seq() const { return *seq_pos_; } + Slice timestamp() const { + // seqno and timestamp are stored in the same order. + return *tombstones_->ts_iter(seq_pos_ - tombstones_->seq_begin()); + } + // Current use case is by CompactionRangeDelAggregator to set + // full_history_ts_low_. + void SetTimestampUpperBound(const Slice* ts_upper_bound) { + ts_upper_bound_ = ts_upper_bound; + } + + ParsedInternalKey parsed_start_key() const { + return ParsedInternalKey(pos_->start_key, seq(), kTypeRangeDeletion); + } + ParsedInternalKey parsed_end_key() const { + return ParsedInternalKey(pos_->end_key, kMaxSequenceNumber, + kTypeRangeDeletion); + } + + // Return the max sequence number of a range tombstone that covers + // the given user key. + // If there is no covering tombstone, then 0 is returned. + SequenceNumber MaxCoveringTombstoneSeqnum(const Slice& user_key); + + // Splits the iterator into n+1 iterators (where n is the number of + // snapshots), each providing a view over a "stripe" of sequence numbers. The + // iterators are keyed by the upper bound of their ranges (the provided + // snapshots + kMaxSequenceNumber). + // + // NOTE: the iterators in the returned map are no longer valid if their + // parent iterator is deleted, since they do not modify the refcount of the + // underlying tombstone list. Therefore, this map should be deleted before + // the parent iterator. + std::map> + SplitBySnapshot(const std::vector& snapshots); + + SequenceNumber upper_bound() const { return upper_bound_; } + SequenceNumber lower_bound() const { return lower_bound_; } + + uint64_t num_unfragmented_tombstones() const { + return tombstones_->num_unfragmented_tombstones(); + } + uint64_t total_tombstone_payload_bytes() const { + return tombstones_->total_tombstone_payload_bytes(); + } + + private: + using RangeTombstoneStack = FragmentedRangeTombstoneList::RangeTombstoneStack; + + struct RangeTombstoneStackStartComparator { + explicit RangeTombstoneStackStartComparator(const Comparator* c) : cmp(c) {} + + bool operator()(const RangeTombstoneStack& a, + const RangeTombstoneStack& b) const { + return cmp->CompareWithoutTimestamp(a.start_key, b.start_key) < 0; + } + + bool operator()(const RangeTombstoneStack& a, const Slice& b) const { + return cmp->CompareWithoutTimestamp(a.start_key, b) < 0; + } + + bool operator()(const Slice& a, const RangeTombstoneStack& b) const { + return cmp->CompareWithoutTimestamp(a, b.start_key) < 0; + } + + const Comparator* cmp; + }; + + struct RangeTombstoneStackEndComparator { + explicit RangeTombstoneStackEndComparator(const Comparator* c) : cmp(c) {} + + bool operator()(const RangeTombstoneStack& a, + const RangeTombstoneStack& b) const { + return cmp->CompareWithoutTimestamp(a.end_key, b.end_key) < 0; + } + + bool operator()(const RangeTombstoneStack& a, const Slice& b) const { + return cmp->CompareWithoutTimestamp(a.end_key, b) < 0; + } + + bool operator()(const Slice& a, const RangeTombstoneStack& b) const { + return cmp->CompareWithoutTimestamp(a, b.end_key) < 0; + } + + const Comparator* cmp; + }; + + void MaybePinKey() const { + if (pos_ != tombstones_->end() && seq_pos_ != tombstones_->seq_end() && + (pinned_pos_ != pos_ || pinned_seq_pos_ != seq_pos_)) { + current_start_key_.Set(pos_->start_key, *seq_pos_, kTypeRangeDeletion); + pinned_pos_ = pos_; + pinned_seq_pos_ = seq_pos_; + } + } + + void SeekToCoveringTombstone(const Slice& key); + void SeekForPrevToCoveringTombstone(const Slice& key); + void ScanForwardToVisibleTombstone(); + void ScanBackwardToVisibleTombstone(); + bool ValidPos() const { + return Valid() && seq_pos_ != tombstones_->seq_iter(pos_->seq_end_idx); + } + + const RangeTombstoneStackStartComparator tombstone_start_cmp_; + const RangeTombstoneStackEndComparator tombstone_end_cmp_; + const InternalKeyComparator* icmp_; + const Comparator* ucmp_; + std::shared_ptr tombstones_ref_; + std::shared_ptr tombstones_cache_ref_; + FragmentedRangeTombstoneList* tombstones_; + SequenceNumber upper_bound_; + SequenceNumber lower_bound_; + // Only consider timestamps <= ts_upper_bound_. + const Slice* ts_upper_bound_; + std::vector::const_iterator pos_; + std::vector::const_iterator seq_pos_; + mutable std::vector::const_iterator pinned_pos_; + mutable std::vector::const_iterator pinned_seq_pos_; + mutable InternalKey current_start_key_; + + // Check the current RangeTombstoneStack `pos_` against timestamp + // upper bound `ts_upper_bound_` and sequence number upper bound + // `upper_bound_`. Update the sequence number (and timestamp) pointer + // `seq_pos_` to the first valid position satisfying both bounds. + void SetMaxVisibleSeqAndTimestamp() { + seq_pos_ = std::lower_bound(tombstones_->seq_iter(pos_->seq_start_idx), + tombstones_->seq_iter(pos_->seq_end_idx), + upper_bound_, std::greater()); + if (ts_upper_bound_ && !ts_upper_bound_->empty()) { + auto ts_pos = std::lower_bound( + tombstones_->ts_iter(pos_->seq_start_idx), + tombstones_->ts_iter(pos_->seq_end_idx), *ts_upper_bound_, + [this](const Slice& s1, const Slice& s2) { + return ucmp_->CompareTimestamp(s1, s2) > 0; + }); + auto ts_idx = ts_pos - tombstones_->ts_iter(pos_->seq_start_idx); + auto seq_idx = seq_pos_ - tombstones_->seq_iter(pos_->seq_start_idx); + if (seq_idx < ts_idx) { + // seq and ts are ordered in non-increasing order. Only updates seq_pos_ + // to a larger index for smaller sequence number and timestamp. + seq_pos_ = tombstones_->seq_iter(pos_->seq_start_idx + ts_idx); + } + } + } +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/range_tombstone_fragmenter_test.cc b/librocksdb-sys/rocksdb/db/range_tombstone_fragmenter_test.cc new file mode 100644 index 0000000..eee2ca2 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/range_tombstone_fragmenter_test.cc @@ -0,0 +1,555 @@ +// Copyright (c) 2018-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/range_tombstone_fragmenter.h" + +#include "db/db_test_util.h" +#include "db/dbformat.h" +#include "rocksdb/comparator.h" +#include "test_util/testutil.h" +#include "util/vector_iterator.h" + +namespace ROCKSDB_NAMESPACE { + +class RangeTombstoneFragmenterTest : public testing::Test {}; + +namespace { + +static auto bytewise_icmp = InternalKeyComparator(BytewiseComparator()); + +std::unique_ptr MakeRangeDelIter( + const std::vector& range_dels) { + std::vector keys, values; + for (const auto& range_del : range_dels) { + auto key_and_value = range_del.Serialize(); + keys.push_back(key_and_value.first.Encode().ToString()); + values.push_back(key_and_value.second.ToString()); + } + return std::unique_ptr( + new VectorIterator(keys, values, &bytewise_icmp)); +} + +void CheckIterPosition(const RangeTombstone& tombstone, + const FragmentedRangeTombstoneIterator* iter) { + // Test InternalIterator interface. + EXPECT_EQ(tombstone.start_key_, ExtractUserKey(iter->key())); + EXPECT_EQ(tombstone.end_key_, iter->value()); + EXPECT_EQ(tombstone.seq_, iter->seq()); + + // Test FragmentedRangeTombstoneIterator interface. + EXPECT_EQ(tombstone.start_key_, iter->start_key()); + EXPECT_EQ(tombstone.end_key_, iter->end_key()); + EXPECT_EQ(tombstone.seq_, GetInternalKeySeqno(iter->key())); +} + +void VerifyFragmentedRangeDels( + FragmentedRangeTombstoneIterator* iter, + const std::vector& expected_tombstones) { + iter->SeekToFirst(); + for (size_t i = 0; i < expected_tombstones.size(); i++, iter->Next()) { + ASSERT_TRUE(iter->Valid()); + CheckIterPosition(expected_tombstones[i], iter); + } + EXPECT_FALSE(iter->Valid()); +} + +void VerifyVisibleTombstones( + FragmentedRangeTombstoneIterator* iter, + const std::vector& expected_tombstones) { + iter->SeekToTopFirst(); + for (size_t i = 0; i < expected_tombstones.size(); i++, iter->TopNext()) { + ASSERT_TRUE(iter->Valid()); + CheckIterPosition(expected_tombstones[i], iter); + } + EXPECT_FALSE(iter->Valid()); +} + +struct SeekTestCase { + Slice seek_target; + RangeTombstone expected_position; + bool out_of_range; +}; + +void VerifySeek(FragmentedRangeTombstoneIterator* iter, + const std::vector& cases) { + for (const auto& testcase : cases) { + iter->Seek(testcase.seek_target); + if (testcase.out_of_range) { + ASSERT_FALSE(iter->Valid()); + } else { + ASSERT_TRUE(iter->Valid()); + CheckIterPosition(testcase.expected_position, iter); + } + } +} + +void VerifySeekForPrev(FragmentedRangeTombstoneIterator* iter, + const std::vector& cases) { + for (const auto& testcase : cases) { + iter->SeekForPrev(testcase.seek_target); + if (testcase.out_of_range) { + ASSERT_FALSE(iter->Valid()); + } else { + ASSERT_TRUE(iter->Valid()); + CheckIterPosition(testcase.expected_position, iter); + } + } +} + +struct MaxCoveringTombstoneSeqnumTestCase { + Slice user_key; + SequenceNumber result; +}; + +void VerifyMaxCoveringTombstoneSeqnum( + FragmentedRangeTombstoneIterator* iter, + const std::vector& cases) { + for (const auto& testcase : cases) { + EXPECT_EQ(testcase.result, + iter->MaxCoveringTombstoneSeqnum(testcase.user_key)); + } +} + +} // anonymous namespace + +TEST_F(RangeTombstoneFragmenterTest, NonOverlappingTombstones) { + auto range_del_iter = MakeRangeDelIter({{"a", "b", 10}, {"c", "d", 5}}); + + FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), + bytewise_icmp); + FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp, + kMaxSequenceNumber); + ASSERT_EQ(0, iter.lower_bound()); + ASSERT_EQ(kMaxSequenceNumber, iter.upper_bound()); + VerifyFragmentedRangeDels(&iter, {{"a", "b", 10}, {"c", "d", 5}}); + VerifyMaxCoveringTombstoneSeqnum(&iter, + {{"", 0}, {"a", 10}, {"b", 0}, {"c", 5}}); +} + +TEST_F(RangeTombstoneFragmenterTest, OverlappingTombstones) { + auto range_del_iter = MakeRangeDelIter({{"a", "e", 10}, {"c", "g", 15}}); + + FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), + bytewise_icmp); + FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp, + kMaxSequenceNumber); + ASSERT_EQ(0, iter.lower_bound()); + ASSERT_EQ(kMaxSequenceNumber, iter.upper_bound()); + VerifyFragmentedRangeDels( + &iter, {{"a", "c", 10}, {"c", "e", 15}, {"c", "e", 10}, {"e", "g", 15}}); + VerifyMaxCoveringTombstoneSeqnum(&iter, + {{"a", 10}, {"c", 15}, {"e", 15}, {"g", 0}}); +} + +TEST_F(RangeTombstoneFragmenterTest, ContiguousTombstones) { + auto range_del_iter = MakeRangeDelIter( + {{"a", "c", 10}, {"c", "e", 20}, {"c", "e", 5}, {"e", "g", 15}}); + + FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), + bytewise_icmp); + FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp, + kMaxSequenceNumber); + ASSERT_EQ(0, iter.lower_bound()); + ASSERT_EQ(kMaxSequenceNumber, iter.upper_bound()); + VerifyFragmentedRangeDels( + &iter, {{"a", "c", 10}, {"c", "e", 20}, {"c", "e", 5}, {"e", "g", 15}}); + VerifyMaxCoveringTombstoneSeqnum(&iter, + {{"a", 10}, {"c", 20}, {"e", 15}, {"g", 0}}); +} + +TEST_F(RangeTombstoneFragmenterTest, RepeatedStartAndEndKey) { + auto range_del_iter = + MakeRangeDelIter({{"a", "c", 10}, {"a", "c", 7}, {"a", "c", 3}}); + + FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), + bytewise_icmp); + FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp, + kMaxSequenceNumber); + ASSERT_EQ(0, iter.lower_bound()); + ASSERT_EQ(kMaxSequenceNumber, iter.upper_bound()); + VerifyFragmentedRangeDels(&iter, + {{"a", "c", 10}, {"a", "c", 7}, {"a", "c", 3}}); + VerifyMaxCoveringTombstoneSeqnum(&iter, {{"a", 10}, {"b", 10}, {"c", 0}}); +} + +TEST_F(RangeTombstoneFragmenterTest, RepeatedStartKeyDifferentEndKeys) { + auto range_del_iter = + MakeRangeDelIter({{"a", "e", 10}, {"a", "g", 7}, {"a", "c", 3}}); + + FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), + bytewise_icmp); + FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp, + kMaxSequenceNumber); + ASSERT_EQ(0, iter.lower_bound()); + ASSERT_EQ(kMaxSequenceNumber, iter.upper_bound()); + VerifyFragmentedRangeDels(&iter, {{"a", "c", 10}, + {"a", "c", 7}, + {"a", "c", 3}, + {"c", "e", 10}, + {"c", "e", 7}, + {"e", "g", 7}}); + VerifyMaxCoveringTombstoneSeqnum(&iter, + {{"a", 10}, {"c", 10}, {"e", 7}, {"g", 0}}); +} + +TEST_F(RangeTombstoneFragmenterTest, RepeatedStartKeyMixedEndKeys) { + auto range_del_iter = MakeRangeDelIter({{"a", "c", 30}, + {"a", "g", 20}, + {"a", "e", 10}, + {"a", "g", 7}, + {"a", "c", 3}}); + + FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), + bytewise_icmp); + FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp, + kMaxSequenceNumber); + ASSERT_EQ(0, iter.lower_bound()); + ASSERT_EQ(kMaxSequenceNumber, iter.upper_bound()); + VerifyFragmentedRangeDels(&iter, {{"a", "c", 30}, + {"a", "c", 20}, + {"a", "c", 10}, + {"a", "c", 7}, + {"a", "c", 3}, + {"c", "e", 20}, + {"c", "e", 10}, + {"c", "e", 7}, + {"e", "g", 20}, + {"e", "g", 7}}); + VerifyMaxCoveringTombstoneSeqnum(&iter, + {{"a", 30}, {"c", 20}, {"e", 20}, {"g", 0}}); +} + +TEST_F(RangeTombstoneFragmenterTest, OverlapAndRepeatedStartKey) { + auto range_del_iter = MakeRangeDelIter({{"a", "e", 10}, + {"c", "g", 8}, + {"c", "i", 6}, + {"j", "n", 4}, + {"j", "l", 2}}); + + FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), + bytewise_icmp); + FragmentedRangeTombstoneIterator iter1(&fragment_list, bytewise_icmp, + kMaxSequenceNumber); + FragmentedRangeTombstoneIterator iter2(&fragment_list, bytewise_icmp, + 9 /* upper_bound */); + FragmentedRangeTombstoneIterator iter3(&fragment_list, bytewise_icmp, + 7 /* upper_bound */); + FragmentedRangeTombstoneIterator iter4(&fragment_list, bytewise_icmp, + 5 /* upper_bound */); + FragmentedRangeTombstoneIterator iter5(&fragment_list, bytewise_icmp, + 3 /* upper_bound */); + for (auto* iter : {&iter1, &iter2, &iter3, &iter4, &iter5}) { + VerifyFragmentedRangeDels(iter, {{"a", "c", 10}, + {"c", "e", 10}, + {"c", "e", 8}, + {"c", "e", 6}, + {"e", "g", 8}, + {"e", "g", 6}, + {"g", "i", 6}, + {"j", "l", 4}, + {"j", "l", 2}, + {"l", "n", 4}}); + } + + ASSERT_EQ(0, iter1.lower_bound()); + ASSERT_EQ(kMaxSequenceNumber, iter1.upper_bound()); + VerifyVisibleTombstones(&iter1, {{"a", "c", 10}, + {"c", "e", 10}, + {"e", "g", 8}, + {"g", "i", 6}, + {"j", "l", 4}, + {"l", "n", 4}}); + VerifyMaxCoveringTombstoneSeqnum( + &iter1, {{"a", 10}, {"c", 10}, {"e", 8}, {"i", 0}, {"j", 4}, {"m", 4}}); + + ASSERT_EQ(0, iter2.lower_bound()); + ASSERT_EQ(9, iter2.upper_bound()); + VerifyVisibleTombstones(&iter2, {{"c", "e", 8}, + {"e", "g", 8}, + {"g", "i", 6}, + {"j", "l", 4}, + {"l", "n", 4}}); + VerifyMaxCoveringTombstoneSeqnum( + &iter2, {{"a", 0}, {"c", 8}, {"e", 8}, {"i", 0}, {"j", 4}, {"m", 4}}); + + ASSERT_EQ(0, iter3.lower_bound()); + ASSERT_EQ(7, iter3.upper_bound()); + VerifyVisibleTombstones(&iter3, {{"c", "e", 6}, + {"e", "g", 6}, + {"g", "i", 6}, + {"j", "l", 4}, + {"l", "n", 4}}); + VerifyMaxCoveringTombstoneSeqnum( + &iter3, {{"a", 0}, {"c", 6}, {"e", 6}, {"i", 0}, {"j", 4}, {"m", 4}}); + + ASSERT_EQ(0, iter4.lower_bound()); + ASSERT_EQ(5, iter4.upper_bound()); + VerifyVisibleTombstones(&iter4, {{"j", "l", 4}, {"l", "n", 4}}); + VerifyMaxCoveringTombstoneSeqnum( + &iter4, {{"a", 0}, {"c", 0}, {"e", 0}, {"i", 0}, {"j", 4}, {"m", 4}}); + + ASSERT_EQ(0, iter5.lower_bound()); + ASSERT_EQ(3, iter5.upper_bound()); + VerifyVisibleTombstones(&iter5, {{"j", "l", 2}}); + VerifyMaxCoveringTombstoneSeqnum( + &iter5, {{"a", 0}, {"c", 0}, {"e", 0}, {"i", 0}, {"j", 2}, {"m", 0}}); +} + +TEST_F(RangeTombstoneFragmenterTest, OverlapAndRepeatedStartKeyUnordered) { + auto range_del_iter = MakeRangeDelIter({{"a", "e", 10}, + {"j", "n", 4}, + {"c", "i", 6}, + {"c", "g", 8}, + {"j", "l", 2}}); + + FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), + bytewise_icmp); + FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp, + 9 /* upper_bound */); + ASSERT_EQ(0, iter.lower_bound()); + ASSERT_EQ(9, iter.upper_bound()); + VerifyFragmentedRangeDels(&iter, {{"a", "c", 10}, + {"c", "e", 10}, + {"c", "e", 8}, + {"c", "e", 6}, + {"e", "g", 8}, + {"e", "g", 6}, + {"g", "i", 6}, + {"j", "l", 4}, + {"j", "l", 2}, + {"l", "n", 4}}); + VerifyMaxCoveringTombstoneSeqnum( + &iter, {{"a", 0}, {"c", 8}, {"e", 8}, {"i", 0}, {"j", 4}, {"m", 4}}); +} + +TEST_F(RangeTombstoneFragmenterTest, OverlapAndRepeatedStartKeyForCompaction) { + auto range_del_iter = MakeRangeDelIter({{"a", "e", 10}, + {"j", "n", 4}, + {"c", "i", 6}, + {"c", "g", 8}, + {"j", "l", 2}}); + + FragmentedRangeTombstoneList fragment_list( + std::move(range_del_iter), bytewise_icmp, true /* for_compaction */, + {} /* snapshots */); + FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp, + kMaxSequenceNumber /* upper_bound */); + VerifyFragmentedRangeDels(&iter, {{"a", "c", 10}, + {"c", "e", 10}, + {"e", "g", 8}, + {"g", "i", 6}, + {"j", "l", 4}, + {"l", "n", 4}}); +} + +TEST_F(RangeTombstoneFragmenterTest, + OverlapAndRepeatedStartKeyForCompactionWithSnapshot) { + auto range_del_iter = MakeRangeDelIter({{"a", "e", 10}, + {"j", "n", 4}, + {"c", "i", 6}, + {"c", "g", 8}, + {"j", "l", 2}}); + + FragmentedRangeTombstoneList fragment_list( + std::move(range_del_iter), bytewise_icmp, true /* for_compaction */, + {9, 20} /* snapshots */); + FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp, + kMaxSequenceNumber /* upper_bound */); + VerifyFragmentedRangeDels(&iter, {{"a", "c", 10}, + {"c", "e", 10}, + {"c", "e", 8}, + {"e", "g", 8}, + {"g", "i", 6}, + {"j", "l", 4}, + {"l", "n", 4}}); +} + +TEST_F(RangeTombstoneFragmenterTest, IteratorSplitNoSnapshots) { + auto range_del_iter = MakeRangeDelIter({{"a", "e", 10}, + {"j", "n", 4}, + {"c", "i", 6}, + {"c", "g", 8}, + {"j", "l", 2}}); + + FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), + bytewise_icmp); + FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp, + kMaxSequenceNumber /* upper_bound */); + + auto split_iters = iter.SplitBySnapshot({} /* snapshots */); + ASSERT_EQ(1, split_iters.size()); + + auto* split_iter = split_iters[kMaxSequenceNumber].get(); + ASSERT_EQ(0, split_iter->lower_bound()); + ASSERT_EQ(kMaxSequenceNumber, split_iter->upper_bound()); + VerifyVisibleTombstones(split_iter, {{"a", "c", 10}, + {"c", "e", 10}, + {"e", "g", 8}, + {"g", "i", 6}, + {"j", "l", 4}, + {"l", "n", 4}}); +} + +TEST_F(RangeTombstoneFragmenterTest, IteratorSplitWithSnapshots) { + auto range_del_iter = MakeRangeDelIter({{"a", "e", 10}, + {"j", "n", 4}, + {"c", "i", 6}, + {"c", "g", 8}, + {"j", "l", 2}}); + + FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), + bytewise_icmp); + FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp, + kMaxSequenceNumber /* upper_bound */); + + auto split_iters = iter.SplitBySnapshot({3, 5, 7, 9} /* snapshots */); + ASSERT_EQ(5, split_iters.size()); + + auto* split_iter1 = split_iters[3].get(); + ASSERT_EQ(0, split_iter1->lower_bound()); + ASSERT_EQ(3, split_iter1->upper_bound()); + VerifyVisibleTombstones(split_iter1, {{"j", "l", 2}}); + + auto* split_iter2 = split_iters[5].get(); + ASSERT_EQ(4, split_iter2->lower_bound()); + ASSERT_EQ(5, split_iter2->upper_bound()); + VerifyVisibleTombstones(split_iter2, {{"j", "l", 4}, {"l", "n", 4}}); + + auto* split_iter3 = split_iters[7].get(); + ASSERT_EQ(6, split_iter3->lower_bound()); + ASSERT_EQ(7, split_iter3->upper_bound()); + VerifyVisibleTombstones(split_iter3, + {{"c", "e", 6}, {"e", "g", 6}, {"g", "i", 6}}); + + auto* split_iter4 = split_iters[9].get(); + ASSERT_EQ(8, split_iter4->lower_bound()); + ASSERT_EQ(9, split_iter4->upper_bound()); + VerifyVisibleTombstones(split_iter4, {{"c", "e", 8}, {"e", "g", 8}}); + + auto* split_iter5 = split_iters[kMaxSequenceNumber].get(); + ASSERT_EQ(10, split_iter5->lower_bound()); + ASSERT_EQ(kMaxSequenceNumber, split_iter5->upper_bound()); + VerifyVisibleTombstones(split_iter5, {{"a", "c", 10}, {"c", "e", 10}}); +} + +TEST_F(RangeTombstoneFragmenterTest, SeekStartKey) { + // Same tombstones as OverlapAndRepeatedStartKey. + auto range_del_iter = MakeRangeDelIter({{"a", "e", 10}, + {"c", "g", 8}, + {"c", "i", 6}, + {"j", "n", 4}, + {"j", "l", 2}}); + + FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), + bytewise_icmp); + + FragmentedRangeTombstoneIterator iter1(&fragment_list, bytewise_icmp, + kMaxSequenceNumber); + VerifySeek( + &iter1, + {{"a", {"a", "c", 10}}, {"e", {"e", "g", 8}}, {"l", {"l", "n", 4}}}); + VerifySeekForPrev( + &iter1, + {{"a", {"a", "c", 10}}, {"e", {"e", "g", 8}}, {"l", {"l", "n", 4}}}); + + FragmentedRangeTombstoneIterator iter2(&fragment_list, bytewise_icmp, + 3 /* upper_bound */); + VerifySeek(&iter2, {{"a", {"j", "l", 2}}, + {"e", {"j", "l", 2}}, + {"l", {}, true /* out of range */}}); + VerifySeekForPrev(&iter2, {{"a", {}, true /* out of range */}, + {"e", {}, true /* out of range */}, + {"l", {"j", "l", 2}}}); +} + +TEST_F(RangeTombstoneFragmenterTest, SeekCovered) { + // Same tombstones as OverlapAndRepeatedStartKey. + auto range_del_iter = MakeRangeDelIter({{"a", "e", 10}, + {"c", "g", 8}, + {"c", "i", 6}, + {"j", "n", 4}, + {"j", "l", 2}}); + + FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), + bytewise_icmp); + + FragmentedRangeTombstoneIterator iter1(&fragment_list, bytewise_icmp, + kMaxSequenceNumber); + VerifySeek( + &iter1, + {{"b", {"a", "c", 10}}, {"f", {"e", "g", 8}}, {"m", {"l", "n", 4}}}); + VerifySeekForPrev( + &iter1, + {{"b", {"a", "c", 10}}, {"f", {"e", "g", 8}}, {"m", {"l", "n", 4}}}); + + FragmentedRangeTombstoneIterator iter2(&fragment_list, bytewise_icmp, + 3 /* upper_bound */); + VerifySeek(&iter2, {{"b", {"j", "l", 2}}, + {"f", {"j", "l", 2}}, + {"m", {}, true /* out of range */}}); + VerifySeekForPrev(&iter2, {{"b", {}, true /* out of range */}, + {"f", {}, true /* out of range */}, + {"m", {"j", "l", 2}}}); +} + +TEST_F(RangeTombstoneFragmenterTest, SeekEndKey) { + // Same tombstones as OverlapAndRepeatedStartKey. + auto range_del_iter = MakeRangeDelIter({{"a", "e", 10}, + {"c", "g", 8}, + {"c", "i", 6}, + {"j", "n", 4}, + {"j", "l", 2}}); + + FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), + bytewise_icmp); + + FragmentedRangeTombstoneIterator iter1(&fragment_list, bytewise_icmp, + kMaxSequenceNumber); + VerifySeek(&iter1, {{"c", {"c", "e", 10}}, + {"g", {"g", "i", 6}}, + {"i", {"j", "l", 4}}, + {"n", {}, true /* out of range */}}); + VerifySeekForPrev(&iter1, {{"c", {"c", "e", 10}}, + {"g", {"g", "i", 6}}, + {"i", {"g", "i", 6}}, + {"n", {"l", "n", 4}}}); + + FragmentedRangeTombstoneIterator iter2(&fragment_list, bytewise_icmp, + 3 /* upper_bound */); + VerifySeek(&iter2, {{"c", {"j", "l", 2}}, + {"g", {"j", "l", 2}}, + {"i", {"j", "l", 2}}, + {"n", {}, true /* out of range */}}); + VerifySeekForPrev(&iter2, {{"c", {}, true /* out of range */}, + {"g", {}, true /* out of range */}, + {"i", {}, true /* out of range */}, + {"n", {"j", "l", 2}}}); +} + +TEST_F(RangeTombstoneFragmenterTest, SeekOutOfBounds) { + // Same tombstones as OverlapAndRepeatedStartKey. + auto range_del_iter = MakeRangeDelIter({{"a", "e", 10}, + {"c", "g", 8}, + {"c", "i", 6}, + {"j", "n", 4}, + {"j", "l", 2}}); + + FragmentedRangeTombstoneList fragment_list(std::move(range_del_iter), + bytewise_icmp); + + FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp, + kMaxSequenceNumber); + VerifySeek(&iter, {{"", {"a", "c", 10}}, {"z", {}, true /* out of range */}}); + VerifySeekForPrev(&iter, + {{"", {}, true /* out of range */}, {"z", {"l", "n", 4}}}); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/read_callback.h b/librocksdb-sys/rocksdb/db/read_callback.h new file mode 100644 index 0000000..c042352 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/read_callback.h @@ -0,0 +1,54 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "db/dbformat.h" +#include "rocksdb/types.h" + +namespace ROCKSDB_NAMESPACE { + +class ReadCallback { + public: + explicit ReadCallback(SequenceNumber last_visible_seq) + : max_visible_seq_(last_visible_seq) {} + ReadCallback(SequenceNumber last_visible_seq, SequenceNumber min_uncommitted) + : max_visible_seq_(last_visible_seq), min_uncommitted_(min_uncommitted) {} + + virtual ~ReadCallback() {} + + // Will be called to see if the seq number visible; if not it moves on to + // the next seq number. + virtual bool IsVisibleFullCheck(SequenceNumber seq) = 0; + + inline bool IsVisible(SequenceNumber seq) { + assert(min_uncommitted_ > 0); + assert(min_uncommitted_ >= kMinUnCommittedSeq); + if (seq < min_uncommitted_) { // handles seq == 0 as well + assert(seq <= max_visible_seq_); + return true; + } else if (max_visible_seq_ < seq) { + assert(seq != 0); + return false; + } else { + assert(seq != 0); // already handled in the first if-then clause + return IsVisibleFullCheck(seq); + } + } + + inline SequenceNumber max_visible_seq() { return max_visible_seq_; } + + // Refresh to a more recent visible seq + virtual void Refresh(SequenceNumber seq) { max_visible_seq_ = seq; } + + protected: + // The max visible seq, it is usually the snapshot but could be larger if + // transaction has its own writes written to db. + SequenceNumber max_visible_seq_ = kMaxSequenceNumber; + // Any seq less than min_uncommitted_ is committed. + const SequenceNumber min_uncommitted_ = kMinUnCommittedSeq; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/repair.cc b/librocksdb-sys/rocksdb/db/repair.cc new file mode 100644 index 0000000..4b28ec2 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/repair.cc @@ -0,0 +1,855 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Repairer does best effort recovery to recover as much data as possible after +// a disaster without compromising consistency. It does not guarantee bringing +// the database to a time consistent state. +// +// Repair process is broken into 4 phases: +// (a) Find files +// (b) Convert logs to tables +// (c) Extract metadata +// (d) Write Descriptor +// +// (a) Find files +// +// The repairer goes through all the files in the directory, and classifies them +// based on their file name. Any file that cannot be identified by name will be +// ignored. +// +// (b) Convert logs to table +// +// Every log file that is active is replayed. All sections of the file where the +// checksum does not match is skipped over. We intentionally give preference to +// data consistency. +// +// (c) Extract metadata +// +// We scan every table to compute +// (1) smallest/largest for the table +// (2) largest sequence number in the table +// (3) oldest blob file referred to by the table (if applicable) +// +// If we are unable to scan the file, then we ignore the table. +// +// (d) Write Descriptor +// +// We generate descriptor contents: +// - log number is set to zero +// - next-file-number is set to 1 + largest file number we found +// - last-sequence-number is set to largest sequence# found across +// all tables (see 2c) +// - compaction pointers are cleared +// - every table file is added at level 0 +// +// Possible optimization 1: +// (a) Compute total size and use to pick appropriate max-level M +// (b) Sort tables by largest sequence# in the table +// (c) For each table: if it overlaps earlier table, place in level-0, +// else place in level-M. +// (d) We can provide options for time consistent recovery and unsafe recovery +// (ignore checksum failure when applicable) +// Possible optimization 2: +// Store per-table metadata (smallest, largest, largest-seq#, ...) +// in the table's meta section to speed up ScanTable. + +#include "db/version_builder.h" + +#include + +#include "db/builder.h" +#include "db/db_impl/db_impl.h" +#include "db/dbformat.h" +#include "db/log_reader.h" +#include "db/log_writer.h" +#include "db/memtable.h" +#include "db/table_cache.h" +#include "db/version_edit.h" +#include "db/write_batch_internal.h" +#include "file/filename.h" +#include "file/writable_file_writer.h" +#include "logging/logging.h" +#include "options/cf_options.h" +#include "rocksdb/comparator.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "rocksdb/write_buffer_manager.h" +#include "table/scoped_arena_iterator.h" +#include "table/unique_id_impl.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { + +class Repairer { + public: + Repairer(const std::string& dbname, const DBOptions& db_options, + const std::vector& column_families, + const ColumnFamilyOptions& default_cf_opts, + const ColumnFamilyOptions& unknown_cf_opts, bool create_unknown_cfs) + : dbname_(dbname), + db_session_id_(DBImpl::GenerateDbSessionId(db_options.env)), + env_(db_options.env), + file_options_(), + db_options_(SanitizeOptions(dbname_, db_options)), + immutable_db_options_(ImmutableDBOptions(db_options_)), + icmp_(default_cf_opts.comparator), + default_cf_opts_( + SanitizeOptions(immutable_db_options_, default_cf_opts)), + default_iopts_( + ImmutableOptions(immutable_db_options_, default_cf_opts_)), + unknown_cf_opts_( + SanitizeOptions(immutable_db_options_, unknown_cf_opts)), + create_unknown_cfs_(create_unknown_cfs), + raw_table_cache_( + // TableCache can be small since we expect each table to be opened + // once. + NewLRUCache(10, db_options_.table_cache_numshardbits)), + table_cache_(new TableCache(default_iopts_, &file_options_, + raw_table_cache_.get(), + /*block_cache_tracer=*/nullptr, + /*io_tracer=*/nullptr, db_session_id_)), + wb_(db_options_.db_write_buffer_size), + wc_(db_options_.delayed_write_rate), + vset_(dbname_, &immutable_db_options_, file_options_, + raw_table_cache_.get(), &wb_, &wc_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id=*/"", db_session_id_), + next_file_number_(1), + db_lock_(nullptr), + closed_(false) { + for (const auto& cfd : column_families) { + cf_name_to_opts_[cfd.name] = cfd.options; + } + } + + const ColumnFamilyOptions* GetColumnFamilyOptions( + const std::string& cf_name) { + if (cf_name_to_opts_.find(cf_name) == cf_name_to_opts_.end()) { + if (create_unknown_cfs_) { + return &unknown_cf_opts_; + } + return nullptr; + } + return &cf_name_to_opts_[cf_name]; + } + + // Adds a column family to the VersionSet with cf_options_ and updates + // manifest. + Status AddColumnFamily(const std::string& cf_name, uint32_t cf_id) { + // TODO: plumb Env::IOActivity; + const ReadOptions read_options; + const auto* cf_opts = GetColumnFamilyOptions(cf_name); + if (cf_opts == nullptr) { + return Status::Corruption("Encountered unknown column family with name=" + + cf_name + ", id=" + std::to_string(cf_id)); + } + Options opts(db_options_, *cf_opts); + MutableCFOptions mut_cf_opts(opts); + + VersionEdit edit; + edit.SetComparatorName(opts.comparator->Name()); + edit.SetPersistUserDefinedTimestamps(opts.persist_user_defined_timestamps); + edit.SetLogNumber(0); + edit.SetColumnFamily(cf_id); + ColumnFamilyData* cfd; + cfd = nullptr; + edit.AddColumnFamily(cf_name); + + mutex_.Lock(); + std::unique_ptr db_dir; + Status status = env_->GetFileSystem()->NewDirectory(dbname_, IOOptions(), + &db_dir, nullptr); + if (status.ok()) { + status = vset_.LogAndApply(cfd, mut_cf_opts, read_options, &edit, &mutex_, + db_dir.get(), false /* new_descriptor_log */, + cf_opts); + } + mutex_.Unlock(); + return status; + } + + Status Close() { + Status s = Status::OK(); + if (!closed_) { + if (db_lock_ != nullptr) { + s = env_->UnlockFile(db_lock_); + db_lock_ = nullptr; + } + closed_ = true; + } + return s; + } + + ~Repairer() { Close().PermitUncheckedError(); } + + Status Run() { + Status status = env_->LockFile(LockFileName(dbname_), &db_lock_); + if (!status.ok()) { + return status; + } + status = FindFiles(); + DBImpl* db_impl = nullptr; + if (status.ok()) { + // Discard older manifests and start a fresh one + for (size_t i = 0; i < manifests_.size(); i++) { + ArchiveFile(dbname_ + "/" + manifests_[i]); + } + // Just create a DBImpl temporarily so we can reuse NewDB() + db_impl = new DBImpl(db_options_, dbname_); + status = db_impl->NewDB(/*new_filenames=*/nullptr); + } + delete db_impl; + + if (status.ok()) { + // Recover using the fresh manifest created by NewDB() + status = + vset_.Recover({{kDefaultColumnFamilyName, default_cf_opts_}}, false); + } + if (status.ok()) { + // Need to scan existing SST files first so the column families are + // created before we process WAL files + ExtractMetaData(); + + // ExtractMetaData() uses table_fds_ to know which SST files' metadata to + // extract -- we need to clear it here since metadata for existing SST + // files has been extracted already + table_fds_.clear(); + ConvertLogFilesToTables(); + ExtractMetaData(); + status = AddTables(); + } + if (status.ok()) { + uint64_t bytes = 0; + for (size_t i = 0; i < tables_.size(); i++) { + bytes += tables_[i].meta.fd.GetFileSize(); + } + ROCKS_LOG_WARN(db_options_.info_log, + "**** Repaired rocksdb %s; " + "recovered %" ROCKSDB_PRIszt " files; %" PRIu64 + " bytes. " + "Some data may have been lost. " + "****", + dbname_.c_str(), tables_.size(), bytes); + } + return status; + } + + private: + struct TableInfo { + FileMetaData meta; + uint32_t column_family_id; + std::string column_family_name; + }; + + std::string const dbname_; + std::string db_session_id_; + Env* const env_; + const FileOptions file_options_; + const DBOptions db_options_; + const ImmutableDBOptions immutable_db_options_; + const InternalKeyComparator icmp_; + const ColumnFamilyOptions default_cf_opts_; + const ImmutableOptions default_iopts_; // table_cache_ holds reference + const ColumnFamilyOptions unknown_cf_opts_; + const bool create_unknown_cfs_; + std::shared_ptr raw_table_cache_; + std::unique_ptr table_cache_; + WriteBufferManager wb_; + WriteController wc_; + VersionSet vset_; + std::unordered_map cf_name_to_opts_; + InstrumentedMutex mutex_; + + std::vector manifests_; + std::vector table_fds_; + std::vector logs_; + std::vector tables_; + uint64_t next_file_number_; + // Lock over the persistent DB state. Non-nullptr iff successfully + // acquired. + FileLock* db_lock_; + bool closed_; + + Status FindFiles() { + std::vector filenames; + bool found_file = false; + std::vector to_search_paths; + + for (size_t path_id = 0; path_id < db_options_.db_paths.size(); path_id++) { + to_search_paths.push_back(db_options_.db_paths[path_id].path); + } + + // search wal_dir if user uses a customize wal_dir + bool same = immutable_db_options_.IsWalDirSameAsDBPath(dbname_); + if (!same) { + to_search_paths.push_back(immutable_db_options_.wal_dir); + } + + for (size_t path_id = 0; path_id < to_search_paths.size(); path_id++) { + ROCKS_LOG_INFO(db_options_.info_log, "Searching path %s\n", + to_search_paths[path_id].c_str()); + Status status = env_->GetChildren(to_search_paths[path_id], &filenames); + if (!status.ok()) { + return status; + } + if (!filenames.empty()) { + found_file = true; + } + + uint64_t number; + FileType type; + for (size_t i = 0; i < filenames.size(); i++) { + if (ParseFileName(filenames[i], &number, &type)) { + if (type == kDescriptorFile) { + manifests_.push_back(filenames[i]); + } else { + if (number + 1 > next_file_number_) { + next_file_number_ = number + 1; + } + if (type == kWalFile) { + logs_.push_back(number); + } else if (type == kTableFile) { + table_fds_.emplace_back(number, static_cast(path_id), + 0); + } else { + // Ignore other files + } + } + } + } + } + if (!found_file) { + return Status::Corruption(dbname_, "repair found no files"); + } + return Status::OK(); + } + + void ConvertLogFilesToTables() { + const auto& wal_dir = immutable_db_options_.GetWalDir(); + for (size_t i = 0; i < logs_.size(); i++) { + // we should use LogFileName(wal_dir, logs_[i]) here. user might uses + // wal_dir option. + std::string logname = LogFileName(wal_dir, logs_[i]); + Status status = ConvertLogToTable(wal_dir, logs_[i]); + if (!status.ok()) { + ROCKS_LOG_WARN(db_options_.info_log, + "Log #%" PRIu64 ": ignoring conversion error: %s", + logs_[i], status.ToString().c_str()); + } + ArchiveFile(logname); + } + } + + Status ConvertLogToTable(const std::string& wal_dir, uint64_t log) { + struct LogReporter : public log::Reader::Reporter { + Env* env; + std::shared_ptr info_log; + uint64_t lognum; + void Corruption(size_t bytes, const Status& s) override { + // We print error messages for corruption, but continue repairing. + ROCKS_LOG_ERROR(info_log, "Log #%" PRIu64 ": dropping %d bytes; %s", + lognum, static_cast(bytes), s.ToString().c_str()); + } + }; + + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + + // Open the log file + std::string logname = LogFileName(wal_dir, log); + const auto& fs = env_->GetFileSystem(); + std::unique_ptr lfile_reader; + Status status = SequentialFileReader::Create( + fs, logname, fs->OptimizeForLogRead(file_options_), &lfile_reader, + nullptr /* dbg */, nullptr /* rate limiter */); + if (!status.ok()) { + return status; + } + + // Create the log reader. + LogReporter reporter; + reporter.env = env_; + reporter.info_log = db_options_.info_log; + reporter.lognum = log; + // We intentionally make log::Reader do checksumming so that + // corruptions cause entire commits to be skipped instead of + // propagating bad information (like overly large sequence + // numbers). + log::Reader reader(db_options_.info_log, std::move(lfile_reader), &reporter, + true /*enable checksum*/, log); + + // Initialize per-column family memtables + for (auto* cfd : *vset_.GetColumnFamilySet()) { + cfd->CreateNewMemtable(*cfd->GetLatestMutableCFOptions(), + kMaxSequenceNumber); + } + auto cf_mems = new ColumnFamilyMemTablesImpl(vset_.GetColumnFamilySet()); + + // Read all the records and add to a memtable + const UnorderedMap& running_ts_sz = + vset_.GetRunningColumnFamiliesTimestampSize(); + std::string scratch; + Slice record; + WriteBatch batch; + + int counter = 0; + while (reader.ReadRecord(&record, &scratch)) { + if (record.size() < WriteBatchInternal::kHeader) { + reporter.Corruption(record.size(), + Status::Corruption("log record too small")); + continue; + } + Status record_status = WriteBatchInternal::SetContents(&batch, record); + if (record_status.ok()) { + const UnorderedMap& record_ts_sz = + reader.GetRecordedTimestampSize(); + record_status = HandleWriteBatchTimestampSizeDifference( + &batch, running_ts_sz, record_ts_sz, + TimestampSizeConsistencyMode::kVerifyConsistency); + if (record_status.ok()) { + record_status = + WriteBatchInternal::InsertInto(&batch, cf_mems, nullptr, nullptr); + } + } + if (record_status.ok()) { + counter += WriteBatchInternal::Count(&batch); + } else { + ROCKS_LOG_WARN(db_options_.info_log, "Log #%" PRIu64 ": ignoring %s", + log, record_status.ToString().c_str()); + } + } + + // Dump a table for each column family with entries in this log file. + for (auto* cfd : *vset_.GetColumnFamilySet()) { + // Do not record a version edit for this conversion to a Table + // since ExtractMetaData() will also generate edits. + MemTable* mem = cfd->mem(); + if (mem->IsEmpty()) { + continue; + } + + FileMetaData meta; + meta.fd = FileDescriptor(next_file_number_++, 0, 0); + // TODO: plumb Env::IOActivity + ReadOptions ro; + ro.total_order_seek = true; + Arena arena; + ScopedArenaIterator iter(mem->NewIterator(ro, &arena)); + int64_t _current_time = 0; + immutable_db_options_.clock->GetCurrentTime(&_current_time) + .PermitUncheckedError(); // ignore error + const uint64_t current_time = static_cast(_current_time); + meta.file_creation_time = current_time; + SnapshotChecker* snapshot_checker = DisableGCSnapshotChecker::Instance(); + + auto write_hint = cfd->CalculateSSTWriteHint(0); + std::vector> + range_del_iters; + auto range_del_iter = mem->NewRangeTombstoneIterator( + ro, kMaxSequenceNumber, false /* immutable_memtable */); + if (range_del_iter != nullptr) { + range_del_iters.emplace_back(range_del_iter); + } + + IOStatus io_s; + CompressionOptions default_compression; + TableBuilderOptions tboptions( + *cfd->ioptions(), *cfd->GetLatestMutableCFOptions(), + cfd->internal_comparator(), cfd->int_tbl_prop_collector_factories(), + kNoCompression, default_compression, cfd->GetID(), cfd->GetName(), + -1 /* level */, false /* is_bottommost */, + TableFileCreationReason::kRecovery, 0 /* oldest_key_time */, + 0 /* file_creation_time */, "DB Repairer" /* db_id */, db_session_id_, + 0 /*target_file_size*/, meta.fd.GetNumber()); + + SeqnoToTimeMapping empty_seqno_time_mapping; + status = BuildTable( + dbname_, /* versions */ nullptr, immutable_db_options_, tboptions, + file_options_, read_options, table_cache_.get(), iter.get(), + std::move(range_del_iters), &meta, nullptr /* blob_file_additions */, + {}, kMaxSequenceNumber, kMaxSequenceNumber, snapshot_checker, + false /* paranoid_file_checks*/, nullptr /* internal_stats */, &io_s, + nullptr /*IOTracer*/, BlobFileCreationReason::kRecovery, + empty_seqno_time_mapping, nullptr /* event_logger */, 0 /* job_id */, + Env::IO_HIGH, nullptr /* table_properties */, write_hint); + ROCKS_LOG_INFO(db_options_.info_log, + "Log #%" PRIu64 ": %d ops saved to Table #%" PRIu64 " %s", + log, counter, meta.fd.GetNumber(), + status.ToString().c_str()); + if (status.ok()) { + if (meta.fd.GetFileSize() > 0) { + table_fds_.push_back(meta.fd); + } + } else { + break; + } + } + delete cf_mems; + return status; + } + + void ExtractMetaData() { + for (size_t i = 0; i < table_fds_.size(); i++) { + TableInfo t; + t.meta.fd = table_fds_[i]; + Status status = ScanTable(&t); + if (!status.ok()) { + std::string fname = TableFileName( + db_options_.db_paths, t.meta.fd.GetNumber(), t.meta.fd.GetPathId()); + char file_num_buf[kFormatFileNumberBufSize]; + FormatFileNumber(t.meta.fd.GetNumber(), t.meta.fd.GetPathId(), + file_num_buf, sizeof(file_num_buf)); + ROCKS_LOG_WARN(db_options_.info_log, "Table #%s: ignoring %s", + file_num_buf, status.ToString().c_str()); + ArchiveFile(fname); + } else { + tables_.push_back(t); + } + } + } + + Status ScanTable(TableInfo* t) { + std::string fname = TableFileName( + db_options_.db_paths, t->meta.fd.GetNumber(), t->meta.fd.GetPathId()); + int counter = 0; + uint64_t file_size; + Status status = env_->GetFileSize(fname, &file_size); + t->meta.fd = FileDescriptor(t->meta.fd.GetNumber(), t->meta.fd.GetPathId(), + file_size); + std::shared_ptr props; + if (status.ok()) { + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + status = table_cache_->GetTableProperties( + file_options_, read_options, icmp_, t->meta, &props, + 0 /* block_protection_bytes_per_key */); + } + if (status.ok()) { + auto s = + GetSstInternalUniqueId(props->db_id, props->db_session_id, + props->orig_file_number, &t->meta.unique_id); + if (!s.ok()) { + ROCKS_LOG_WARN(db_options_.info_log, + "Table #%" PRIu64 + ": unable to get unique id, default to Unknown.", + t->meta.fd.GetNumber()); + } + t->column_family_id = static_cast(props->column_family_id); + if (t->column_family_id == + TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) { + ROCKS_LOG_WARN( + db_options_.info_log, + "Table #%" PRIu64 + ": column family unknown (probably due to legacy format); " + "adding to default column family id 0.", + t->meta.fd.GetNumber()); + t->column_family_id = 0; + } + + if (vset_.GetColumnFamilySet()->GetColumnFamily(t->column_family_id) == + nullptr) { + status = + AddColumnFamily(props->column_family_name, t->column_family_id); + } + t->meta.oldest_ancester_time = props->creation_time; + t->meta.user_defined_timestamps_persisted = + static_cast(props->user_defined_timestamps_persisted); + } + if (status.ok()) { + uint64_t tail_size = 0; + bool contain_no_data_blocks = + props->num_entries > 0 && + (props->num_entries == props->num_range_deletions); + if (props->tail_start_offset > 0 || contain_no_data_blocks) { + assert(props->tail_start_offset <= file_size); + tail_size = file_size - props->tail_start_offset; + } + t->meta.tail_size = tail_size; + } + ColumnFamilyData* cfd = nullptr; + if (status.ok()) { + cfd = vset_.GetColumnFamilySet()->GetColumnFamily(t->column_family_id); + if (cfd->GetName() != props->column_family_name) { + ROCKS_LOG_ERROR( + db_options_.info_log, + "Table #%" PRIu64 + ": inconsistent column family name '%s'; expected '%s' for column " + "family id %" PRIu32 ".", + t->meta.fd.GetNumber(), props->column_family_name.c_str(), + cfd->GetName().c_str(), t->column_family_id); + status = Status::Corruption(dbname_, "inconsistent column family name"); + } + } + if (status.ok()) { + // TODO: plumb Env::IOActivity + ReadOptions ropts; + ropts.total_order_seek = true; + InternalIterator* iter = table_cache_->NewIterator( + ropts, file_options_, cfd->internal_comparator(), t->meta, + nullptr /* range_del_agg */, + cfd->GetLatestMutableCFOptions()->prefix_extractor, + /*table_reader_ptr=*/nullptr, /*file_read_hist=*/nullptr, + TableReaderCaller::kRepair, /*arena=*/nullptr, /*skip_filters=*/false, + /*level=*/-1, /*max_file_size_for_l0_meta_pin=*/0, + /*smallest_compaction_key=*/nullptr, + /*largest_compaction_key=*/nullptr, + /*allow_unprepared_value=*/false, + cfd->GetLatestMutableCFOptions()->block_protection_bytes_per_key); + ParsedInternalKey parsed; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + Slice key = iter->key(); + Status pik_status = + ParseInternalKey(key, &parsed, db_options_.allow_data_in_errors); + if (!pik_status.ok()) { + ROCKS_LOG_ERROR(db_options_.info_log, + "Table #%" PRIu64 ": unparsable key - %s", + t->meta.fd.GetNumber(), pik_status.getState()); + continue; + } + + counter++; + + status = t->meta.UpdateBoundaries(key, iter->value(), parsed.sequence, + parsed.type); + if (!status.ok()) { + break; + } + } + if (status.ok() && !iter->status().ok()) { + status = iter->status(); + } + delete iter; + + ROCKS_LOG_INFO(db_options_.info_log, "Table #%" PRIu64 ": %d entries %s", + t->meta.fd.GetNumber(), counter, + status.ToString().c_str()); + } + if (status.ok()) { + // XXX/FIXME: This is just basic, naive handling of range tombstones, + // like call to UpdateBoundariesForRange in builder.cc where we assume + // an SST file is a full sorted run. This probably needs the extra logic + // from compaction_job.cc around call to UpdateBoundariesForRange (to + // handle range tombstones extendingg beyond range of other entries). + // TODO: plumb Env::IOActivity + ReadOptions ropts; + std::unique_ptr r_iter; + status = table_cache_->GetRangeTombstoneIterator( + ropts, cfd->internal_comparator(), t->meta, + cfd->GetLatestMutableCFOptions()->block_protection_bytes_per_key, + &r_iter); + + if (r_iter) { + r_iter->SeekToFirst(); + + while (r_iter->Valid()) { + auto tombstone = r_iter->Tombstone(); + auto kv = tombstone.Serialize(); + t->meta.UpdateBoundariesForRange( + kv.first, tombstone.SerializeEndKey(), tombstone.seq_, + cfd->internal_comparator()); + r_iter->Next(); + } + } + } + return status; + } + + Status AddTables() { + // TODO: plumb Env::IOActivity; + const ReadOptions read_options; + std::unordered_map> cf_id_to_tables; + SequenceNumber max_sequence = 0; + for (size_t i = 0; i < tables_.size(); i++) { + cf_id_to_tables[tables_[i].column_family_id].push_back(&tables_[i]); + if (max_sequence < tables_[i].meta.fd.largest_seqno) { + max_sequence = tables_[i].meta.fd.largest_seqno; + } + } + vset_.SetLastAllocatedSequence(max_sequence); + vset_.SetLastPublishedSequence(max_sequence); + vset_.SetLastSequence(max_sequence); + + for (const auto& cf_id_and_tables : cf_id_to_tables) { + auto* cfd = + vset_.GetColumnFamilySet()->GetColumnFamily(cf_id_and_tables.first); + + // Recover files' epoch number using dummy VersionStorageInfo + VersionBuilder dummy_version_builder( + cfd->current()->version_set()->file_options(), cfd->ioptions(), + cfd->table_cache(), cfd->current()->storage_info(), + cfd->current()->version_set(), + cfd->GetFileMetadataCacheReservationManager()); + VersionStorageInfo dummy_vstorage( + &cfd->internal_comparator(), cfd->user_comparator(), + cfd->NumberLevels(), cfd->ioptions()->compaction_style, + nullptr /* src_vstorage */, cfd->ioptions()->force_consistency_checks, + EpochNumberRequirement::kMightMissing); + Status s; + VersionEdit dummy_edit; + for (const auto* table : cf_id_and_tables.second) { + // TODO(opt): separate out into multiple levels + dummy_edit.AddFile( + 0, table->meta.fd.GetNumber(), table->meta.fd.GetPathId(), + table->meta.fd.GetFileSize(), table->meta.smallest, + table->meta.largest, table->meta.fd.smallest_seqno, + table->meta.fd.largest_seqno, table->meta.marked_for_compaction, + table->meta.temperature, table->meta.oldest_blob_file_number, + table->meta.oldest_ancester_time, table->meta.file_creation_time, + table->meta.epoch_number, table->meta.file_checksum, + table->meta.file_checksum_func_name, table->meta.unique_id, + table->meta.compensated_range_deletion_size, table->meta.tail_size, + table->meta.user_defined_timestamps_persisted); + } + s = dummy_version_builder.Apply(&dummy_edit); + if (s.ok()) { + s = dummy_version_builder.SaveTo(&dummy_vstorage); + } + if (s.ok()) { + dummy_vstorage.RecoverEpochNumbers(cfd); + } + if (s.ok()) { + // Record changes from this repair in VersionEdit, including files with + // recovered epoch numbers + VersionEdit edit; + edit.SetComparatorName(cfd->user_comparator()->Name()); + edit.SetPersistUserDefinedTimestamps( + cfd->ioptions()->persist_user_defined_timestamps); + edit.SetLogNumber(0); + edit.SetNextFile(next_file_number_); + edit.SetColumnFamily(cfd->GetID()); + for (int level = 0; level < dummy_vstorage.num_levels(); ++level) { + for (FileMetaData* file_meta : dummy_vstorage.LevelFiles(level)) { + edit.AddFile(level, *file_meta); + } + } + + // Release resources occupied by the dummy VersionStorageInfo + for (int level = 0; level < dummy_vstorage.num_levels(); ++level) { + for (FileMetaData* file_meta : dummy_vstorage.LevelFiles(level)) { + file_meta->refs--; + if (file_meta->refs <= 0) { + delete file_meta; + } + } + } + + // Persist record of changes + assert(next_file_number_ > 0); + vset_.MarkFileNumberUsed(next_file_number_ - 1); + mutex_.Lock(); + std::unique_ptr db_dir; + s = env_->GetFileSystem()->NewDirectory(dbname_, IOOptions(), &db_dir, + nullptr); + if (s.ok()) { + s = vset_.LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(), + read_options, &edit, &mutex_, db_dir.get(), + false /* new_descriptor_log */); + } + mutex_.Unlock(); + } + if (!s.ok()) { + return s; + } + } + return Status::OK(); + } + + void ArchiveFile(const std::string& fname) { + // Move into another directory. E.g., for + // dir/foo + // rename to + // dir/lost/foo + const char* slash = strrchr(fname.c_str(), '/'); + std::string new_dir; + if (slash != nullptr) { + new_dir.assign(fname.data(), slash - fname.data()); + } + new_dir.append("/lost"); + env_->CreateDir(new_dir).PermitUncheckedError(); // Ignore error + std::string new_file = new_dir; + new_file.append("/"); + new_file.append((slash == nullptr) ? fname.c_str() : slash + 1); + Status s = env_->RenameFile(fname, new_file); + ROCKS_LOG_INFO(db_options_.info_log, "Archiving %s: %s\n", fname.c_str(), + s.ToString().c_str()); + } +}; + +Status GetDefaultCFOptions( + const std::vector& column_families, + ColumnFamilyOptions* res) { + assert(res != nullptr); + auto iter = std::find_if(column_families.begin(), column_families.end(), + [](const ColumnFamilyDescriptor& cfd) { + return cfd.name == kDefaultColumnFamilyName; + }); + if (iter == column_families.end()) { + return Status::InvalidArgument( + "column_families", "Must contain entry for default column family"); + } + *res = iter->options; + return Status::OK(); +} +} // anonymous namespace + +Status RepairDB(const std::string& dbname, const DBOptions& db_options, + const std::vector& column_families) { + ColumnFamilyOptions default_cf_opts; + Status status = GetDefaultCFOptions(column_families, &default_cf_opts); + if (!status.ok()) { + return status; + } + + Repairer repairer(dbname, db_options, column_families, default_cf_opts, + ColumnFamilyOptions() /* unknown_cf_opts */, + false /* create_unknown_cfs */); + status = repairer.Run(); + if (status.ok()) { + status = repairer.Close(); + } + return status; +} + +Status RepairDB(const std::string& dbname, const DBOptions& db_options, + const std::vector& column_families, + const ColumnFamilyOptions& unknown_cf_opts) { + ColumnFamilyOptions default_cf_opts; + Status status = GetDefaultCFOptions(column_families, &default_cf_opts); + if (!status.ok()) { + return status; + } + + Repairer repairer(dbname, db_options, column_families, default_cf_opts, + unknown_cf_opts, true /* create_unknown_cfs */); + status = repairer.Run(); + if (status.ok()) { + status = repairer.Close(); + } + return status; +} + +Status RepairDB(const std::string& dbname, const Options& options) { + Options opts(options); + DBOptions db_options(opts); + ColumnFamilyOptions cf_options(opts); + + Repairer repairer(dbname, db_options, {}, cf_options /* default_cf_opts */, + cf_options /* unknown_cf_opts */, + true /* create_unknown_cfs */); + Status status = repairer.Run(); + if (status.ok()) { + status = repairer.Close(); + } + return status; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/repair_test.cc b/librocksdb-sys/rocksdb/db/repair_test.cc new file mode 100644 index 0000000..e8cc40a --- /dev/null +++ b/librocksdb-sys/rocksdb/db/repair_test.cc @@ -0,0 +1,625 @@ +// Copyright (c) 2016-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include +#include +#include + +#include "db/db_impl/db_impl.h" +#include "db/db_test_util.h" +#include "db/db_with_timestamp_test_util.h" +#include "file/file_util.h" +#include "rocksdb/comparator.h" +#include "rocksdb/db.h" +#include "rocksdb/options.h" +#include "rocksdb/transaction_log.h" +#include "table/unique_id_impl.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +class RepairTest : public DBTestBase { + public: + RepairTest() : DBTestBase("repair_test", /*env_do_fsync=*/true) {} + + Status GetFirstSstPath(std::string* first_sst_path) { + assert(first_sst_path != nullptr); + first_sst_path->clear(); + uint64_t manifest_size; + std::vector files; + Status s = db_->GetLiveFiles(files, &manifest_size); + if (s.ok()) { + auto sst_iter = + std::find_if(files.begin(), files.end(), [](const std::string& file) { + uint64_t number; + FileType type; + bool ok = ParseFileName(file, &number, &type); + return ok && type == kTableFile; + }); + *first_sst_path = sst_iter == files.end() ? "" : dbname_ + *sst_iter; + } + return s; + } + + void ReopenWithSstIdVerify() { + std::atomic_int verify_passed{0}; + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTable::Open::PassedVerifyUniqueId", [&](void* arg) { + // override job status + auto id = static_cast(arg); + assert(*id != kNullUniqueId64x2); + verify_passed++; + }); + SyncPoint::GetInstance()->EnableProcessing(); + auto options = CurrentOptions(); + options.verify_sst_unique_id_in_manifest = true; + Reopen(options); + + ASSERT_GT(verify_passed, 0); + SyncPoint::GetInstance()->DisableProcessing(); + } + + std::vector GetLevelFileMetadatas(int level, int cf = 0) { + VersionSet* const versions = dbfull()->GetVersionSet(); + assert(versions); + ColumnFamilyData* const cfd = + versions->GetColumnFamilySet()->GetColumnFamily(cf); + assert(cfd); + Version* const current = cfd->current(); + assert(current); + VersionStorageInfo* const storage_info = current->storage_info(); + assert(storage_info); + return storage_info->LevelFiles(level); + } +}; + +TEST_F(RepairTest, SortRepairedDBL0ByEpochNumber) { + Options options = CurrentOptions(); + DestroyAndReopen(options); + + ASSERT_OK(Put("k1", "oldest")); + ASSERT_OK(Put("k1", "older")); + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + + ASSERT_OK(Put("k1", "old")); + ASSERT_OK(Flush()); + + ASSERT_OK(Put("k1", "new")); + + std::vector level0_files = GetLevelFileMetadatas(0 /* level*/); + ASSERT_EQ(level0_files.size(), 1); + ASSERT_EQ(level0_files[0]->epoch_number, 2); + std::vector level1_files = GetLevelFileMetadatas(1 /* level*/); + ASSERT_EQ(level1_files.size(), 1); + ASSERT_EQ(level1_files[0]->epoch_number, 1); + + std::string manifest_path = + DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo()); + Close(); + ASSERT_OK(env_->FileExists(manifest_path)); + ASSERT_OK(env_->DeleteFile(manifest_path)); + + ASSERT_OK(RepairDB(dbname_, CurrentOptions())); + ReopenWithSstIdVerify(); + + EXPECT_EQ(Get("k1"), "new"); + + level0_files = GetLevelFileMetadatas(0 /* level*/); + ASSERT_EQ(level0_files.size(), 3); + EXPECT_EQ(level0_files[0]->epoch_number, 3); + EXPECT_EQ(level0_files[1]->epoch_number, 2); + EXPECT_EQ(level0_files[2]->epoch_number, 1); + level1_files = GetLevelFileMetadatas(1 /* level*/); + ASSERT_EQ(level1_files.size(), 0); +} + +TEST_F(RepairTest, LostManifest) { + // Add a couple SST files, delete the manifest, and verify RepairDB() saves + // the day. + ASSERT_OK(Put("key", "val")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("key2", "val2")); + ASSERT_OK(Flush()); + // Need to get path before Close() deletes db_, but delete it after Close() to + // ensure Close() didn't change the manifest. + std::string manifest_path = + DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo()); + + Close(); + ASSERT_OK(env_->FileExists(manifest_path)); + ASSERT_OK(env_->DeleteFile(manifest_path)); + ASSERT_OK(RepairDB(dbname_, CurrentOptions())); + ReopenWithSstIdVerify(); + + ASSERT_EQ(Get("key"), "val"); + ASSERT_EQ(Get("key2"), "val2"); +} + +TEST_F(RepairTest, LostManifestMoreDbFeatures) { + // Add a couple SST files, delete the manifest, and verify RepairDB() saves + // the day. + ASSERT_OK(Put("key", "val")); + ASSERT_OK(Put("key2", "val2")); + ASSERT_OK(Put("key3", "val3")); + ASSERT_OK(Put("key4", "val4")); + ASSERT_OK(Flush()); + // Test an SST file containing only a range tombstone + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "key2", + "key3z")); + ASSERT_OK(Flush()); + // Need to get path before Close() deletes db_, but delete it after Close() to + // ensure Close() didn't change the manifest. + std::string manifest_path = + DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo()); + + Close(); + ASSERT_OK(env_->FileExists(manifest_path)); + ASSERT_OK(env_->DeleteFile(manifest_path)); + ASSERT_OK(RepairDB(dbname_, CurrentOptions())); + + // repair from sst should work with unique_id verification + ReopenWithSstIdVerify(); + + ASSERT_EQ(Get("key"), "val"); + ASSERT_EQ(Get("key2"), "NOT_FOUND"); + ASSERT_EQ(Get("key3"), "NOT_FOUND"); + ASSERT_EQ(Get("key4"), "val4"); +} + +TEST_F(RepairTest, CorruptManifest) { + // Manifest is in an invalid format. Expect a full recovery. + ASSERT_OK(Put("key", "val")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("key2", "val2")); + ASSERT_OK(Flush()); + // Need to get path before Close() deletes db_, but overwrite it after Close() + // to ensure Close() didn't change the manifest. + std::string manifest_path = + DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo()); + + Close(); + ASSERT_OK(env_->FileExists(manifest_path)); + + ASSERT_OK(CreateFile(env_->GetFileSystem(), manifest_path, "blah", + false /* use_fsync */)); + ASSERT_OK(RepairDB(dbname_, CurrentOptions())); + + ReopenWithSstIdVerify(); + + ASSERT_EQ(Get("key"), "val"); + ASSERT_EQ(Get("key2"), "val2"); +} + +TEST_F(RepairTest, IncompleteManifest) { + // In this case, the manifest is valid but does not reference all of the SST + // files. Expect a full recovery. + ASSERT_OK(Put("key", "val")); + ASSERT_OK(Flush()); + std::string orig_manifest_path = + DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo()); + CopyFile(orig_manifest_path, orig_manifest_path + ".tmp"); + ASSERT_OK(Put("key2", "val2")); + ASSERT_OK(Flush()); + // Need to get path before Close() deletes db_, but overwrite it after Close() + // to ensure Close() didn't change the manifest. + std::string new_manifest_path = + DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo()); + + Close(); + ASSERT_OK(env_->FileExists(new_manifest_path)); + // Replace the manifest with one that is only aware of the first SST file. + CopyFile(orig_manifest_path + ".tmp", new_manifest_path); + ASSERT_OK(RepairDB(dbname_, CurrentOptions())); + + ReopenWithSstIdVerify(); + + ASSERT_EQ(Get("key"), "val"); + ASSERT_EQ(Get("key2"), "val2"); +} + +TEST_F(RepairTest, PostRepairSstFileNumbering) { + // Verify after a DB is repaired, new files will be assigned higher numbers + // than old files. + ASSERT_OK(Put("key", "val")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("key2", "val2")); + ASSERT_OK(Flush()); + uint64_t pre_repair_file_num = dbfull()->TEST_Current_Next_FileNo(); + Close(); + + ASSERT_OK(RepairDB(dbname_, CurrentOptions())); + + ReopenWithSstIdVerify(); + + uint64_t post_repair_file_num = dbfull()->TEST_Current_Next_FileNo(); + ASSERT_GE(post_repair_file_num, pre_repair_file_num); +} + +TEST_F(RepairTest, LostSst) { + // Delete one of the SST files but preserve the manifest that refers to it, + // then verify the DB is still usable for the intact SST. + ASSERT_OK(Put("key", "val")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("key2", "val2")); + ASSERT_OK(Flush()); + std::string sst_path; + ASSERT_OK(GetFirstSstPath(&sst_path)); + ASSERT_FALSE(sst_path.empty()); + ASSERT_OK(env_->DeleteFile(sst_path)); + + Close(); + ASSERT_OK(RepairDB(dbname_, CurrentOptions())); + ReopenWithSstIdVerify(); + + // Exactly one of the key-value pairs should be in the DB now. + ASSERT_TRUE((Get("key") == "val") != (Get("key2") == "val2")); +} + +TEST_F(RepairTest, CorruptSst) { + // Corrupt one of the SST files but preserve the manifest that refers to it, + // then verify the DB is still usable for the intact SST. + ASSERT_OK(Put("key", "val")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("key2", "val2")); + ASSERT_OK(Flush()); + std::string sst_path; + ASSERT_OK(GetFirstSstPath(&sst_path)); + ASSERT_FALSE(sst_path.empty()); + + ASSERT_OK(CreateFile(env_->GetFileSystem(), sst_path, "blah", + false /* use_fsync */)); + + Close(); + ASSERT_OK(RepairDB(dbname_, CurrentOptions())); + ReopenWithSstIdVerify(); + + // Exactly one of the key-value pairs should be in the DB now. + ASSERT_TRUE((Get("key") == "val") != (Get("key2") == "val2")); +} + +TEST_F(RepairTest, UnflushedSst) { + // This test case invokes repair while some data is unflushed, then verifies + // that data is in the db. + ASSERT_OK(Put("key", "val")); + VectorLogPtr wal_files; + ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files)); + ASSERT_EQ(wal_files.size(), 1); + { + uint64_t total_ssts_size; + std::unordered_map sst_files; + ASSERT_OK(GetAllDataFiles(kTableFile, &sst_files, &total_ssts_size)); + ASSERT_EQ(total_ssts_size, 0); + } + // Need to get path before Close() deletes db_, but delete it after Close() to + // ensure Close() didn't change the manifest. + std::string manifest_path = + DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo()); + + Close(); + ASSERT_OK(env_->FileExists(manifest_path)); + ASSERT_OK(env_->DeleteFile(manifest_path)); + ASSERT_OK(RepairDB(dbname_, CurrentOptions())); + ReopenWithSstIdVerify(); + + ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files)); + ASSERT_EQ(wal_files.size(), 0); + { + uint64_t total_ssts_size; + std::unordered_map sst_files; + ASSERT_OK(GetAllDataFiles(kTableFile, &sst_files, &total_ssts_size)); + ASSERT_GT(total_ssts_size, 0); + } + ASSERT_EQ(Get("key"), "val"); +} + +// Test parameters: +// param 0): paranoid file check +// param 1): user-defined timestamp test mode +class RepairTestWithTimestamp + : public DBBasicTestWithTimestampBase, + public testing::WithParamInterface< + std::tuple> { + public: + RepairTestWithTimestamp() + : DBBasicTestWithTimestampBase("repair_test_with_timestamp") {} + + Status Put(const Slice& key, const Slice& ts, const Slice& value) { + WriteOptions write_opts; + return db_->Put(write_opts, handles_[0], key, ts, value); + } + + void CheckGet(const ReadOptions& read_opts, const Slice& key, + const std::string& expected_value, + const std::string& expected_ts) { + std::string actual_value; + std::string actual_ts; + ASSERT_OK(db_->Get(read_opts, handles_[0], key, &actual_value, &actual_ts)); + ASSERT_EQ(expected_value, actual_value); + ASSERT_EQ(expected_ts, actual_ts); + } + + void CheckFileBoundaries(const Slice& smallest_user_key, + const Slice& largest_user_key) { + std::vector> level_to_files; + dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(), + &level_to_files); + ASSERT_GT(level_to_files.size(), 1); + // L0 only has one SST file. + ASSERT_EQ(level_to_files[0].size(), 1); + auto file_meta = level_to_files[0][0]; + ASSERT_EQ(smallest_user_key, file_meta.smallest.user_key()); + ASSERT_EQ(largest_user_key, file_meta.largest.user_key()); + } +}; + +TEST_P(RepairTestWithTimestamp, UnflushedSst) { + Destroy(last_options_); + + bool paranoid_file_checks = std::get<0>(GetParam()); + bool persist_udt = test::ShouldPersistUDT(std::get<1>(GetParam())); + std::string smallest_ukey_without_ts = "bar"; + std::string largest_ukey_without_ts = "foo"; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + std::string min_ts; + std::string write_ts; + PutFixed64(&min_ts, 0); + PutFixed64(&write_ts, 1); + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + options.persist_user_defined_timestamps = persist_udt; + if (!persist_udt) { + options.allow_concurrent_memtable_write = false; + } + options.paranoid_file_checks = paranoid_file_checks; + + ColumnFamilyOptions cf_options(options); + std::vector column_families; + column_families.push_back( + ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options)); + + ASSERT_OK(DB::Open(options, dbname_, column_families, &handles_, &db_)); + + ASSERT_OK(Put(smallest_ukey_without_ts, write_ts, + smallest_ukey_without_ts + ":val")); + ASSERT_OK( + Put(largest_ukey_without_ts, write_ts, largest_ukey_without_ts + ":val")); + VectorLogPtr wal_files; + ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files)); + ASSERT_EQ(wal_files.size(), 1); + { + uint64_t total_ssts_size; + std::unordered_map sst_files; + ASSERT_OK(GetAllDataFiles(kTableFile, &sst_files, &total_ssts_size)); + ASSERT_EQ(total_ssts_size, 0); + } + // Need to get path before Close() deletes db_, but delete it after Close() to + // ensure Close() didn't change the manifest. + std::string manifest_path = + DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo()); + + Close(); + ASSERT_OK(env_->FileExists(manifest_path)); + ASSERT_OK(env_->DeleteFile(manifest_path)); + ASSERT_OK(RepairDB(dbname_, options)); + ASSERT_OK(DB::Open(options, dbname_, column_families, &handles_, &db_)); + + ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files)); + ASSERT_EQ(wal_files.size(), 0); + { + uint64_t total_ssts_size; + std::unordered_map sst_files; + ASSERT_OK(GetAllDataFiles(kTableFile, &sst_files, &total_ssts_size)); + ASSERT_GT(total_ssts_size, 0); + } + + // Check file boundaries are correct for different + // `persist_user_defined_timestamps` option values. + if (persist_udt) { + CheckFileBoundaries(smallest_ukey_without_ts + write_ts, + largest_ukey_without_ts + write_ts); + } else { + CheckFileBoundaries(smallest_ukey_without_ts + min_ts, + largest_ukey_without_ts + min_ts); + } + + ReadOptions read_opts; + Slice read_ts_slice = write_ts; + read_opts.timestamp = &read_ts_slice; + if (persist_udt) { + CheckGet(read_opts, smallest_ukey_without_ts, + smallest_ukey_without_ts + ":val", write_ts); + CheckGet(read_opts, largest_ukey_without_ts, + largest_ukey_without_ts + ":val", write_ts); + } else { + // TODO(yuzhangyu): currently when `persist_user_defined_timestamps` is + // false, ts is unconditionally stripped during flush. + // When `full_history_ts_low` is set and respected during flush. + // We should prohibit reading below `full_history_ts_low` all together. + CheckGet(read_opts, smallest_ukey_without_ts, + smallest_ukey_without_ts + ":val", min_ts); + CheckGet(read_opts, largest_ukey_without_ts, + largest_ukey_without_ts + ":val", min_ts); + } +} + +// Param 0: paranoid file check +// Param 1: test mode for the user-defined timestamp feature +INSTANTIATE_TEST_CASE_P( + UnflushedSst, RepairTestWithTimestamp, + ::testing::Combine( + ::testing::Bool(), + ::testing::Values( + test::UserDefinedTimestampTestMode::kStripUserDefinedTimestamp, + test::UserDefinedTimestampTestMode::kNormal))); + +TEST_F(RepairTest, SeparateWalDir) { + do { + Options options = CurrentOptions(); + DestroyAndReopen(options); + ASSERT_OK(Put("key", "val")); + ASSERT_OK(Put("foo", "bar")); + VectorLogPtr wal_files; + ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files)); + ASSERT_EQ(wal_files.size(), 1); + { + uint64_t total_ssts_size; + std::unordered_map sst_files; + ASSERT_OK(GetAllDataFiles(kTableFile, &sst_files, &total_ssts_size)); + ASSERT_EQ(total_ssts_size, 0); + } + std::string manifest_path = + DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo()); + + Close(); + ASSERT_OK(env_->FileExists(manifest_path)); + ASSERT_OK(env_->DeleteFile(manifest_path)); + ASSERT_OK(RepairDB(dbname_, options)); + + // make sure that all WALs are converted to SSTables. + options.wal_dir = ""; + + ReopenWithSstIdVerify(); + ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files)); + ASSERT_EQ(wal_files.size(), 0); + { + uint64_t total_ssts_size; + std::unordered_map sst_files; + ASSERT_OK(GetAllDataFiles(kTableFile, &sst_files, &total_ssts_size)); + ASSERT_GT(total_ssts_size, 0); + } + ASSERT_EQ(Get("key"), "val"); + ASSERT_EQ(Get("foo"), "bar"); + + } while (ChangeWalOptions()); +} + +TEST_F(RepairTest, RepairMultipleColumnFamilies) { + // Verify repair logic associates SST files with their original column + // families. + const int kNumCfs = 3; + const int kEntriesPerCf = 2; + DestroyAndReopen(CurrentOptions()); + CreateAndReopenWithCF({"pikachu1", "pikachu2"}, CurrentOptions()); + for (int i = 0; i < kNumCfs; ++i) { + for (int j = 0; j < kEntriesPerCf; ++j) { + ASSERT_OK(Put(i, "key" + std::to_string(j), "val" + std::to_string(j))); + if (j == kEntriesPerCf - 1 && i == kNumCfs - 1) { + // Leave one unflushed so we can verify WAL entries are properly + // associated with column families. + continue; + } + ASSERT_OK(Flush(i)); + } + } + + // Need to get path before Close() deletes db_, but delete it after Close() to + // ensure Close() doesn't re-create the manifest. + std::string manifest_path = + DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo()); + Close(); + ASSERT_OK(env_->FileExists(manifest_path)); + ASSERT_OK(env_->DeleteFile(manifest_path)); + + ASSERT_OK(RepairDB(dbname_, CurrentOptions())); + + ReopenWithColumnFamilies({"default", "pikachu1", "pikachu2"}, + CurrentOptions()); + for (int i = 0; i < kNumCfs; ++i) { + for (int j = 0; j < kEntriesPerCf; ++j) { + ASSERT_EQ(Get(i, "key" + std::to_string(j)), "val" + std::to_string(j)); + } + } +} + +TEST_F(RepairTest, RepairColumnFamilyOptions) { + // Verify repair logic uses correct ColumnFamilyOptions when repairing a + // database with different options for column families. + const int kNumCfs = 2; + const int kEntriesPerCf = 2; + + Options opts(CurrentOptions()), rev_opts(CurrentOptions()); + opts.comparator = BytewiseComparator(); + rev_opts.comparator = ReverseBytewiseComparator(); + + DestroyAndReopen(opts); + CreateColumnFamilies({"reverse"}, rev_opts); + ReopenWithColumnFamilies({"default", "reverse"}, + std::vector{opts, rev_opts}); + for (int i = 0; i < kNumCfs; ++i) { + for (int j = 0; j < kEntriesPerCf; ++j) { + ASSERT_OK(Put(i, "key" + std::to_string(j), "val" + std::to_string(j))); + if (i == kNumCfs - 1 && j == kEntriesPerCf - 1) { + // Leave one unflushed so we can verify RepairDB's flush logic + continue; + } + ASSERT_OK(Flush(i)); + } + } + Close(); + + // RepairDB() records the comparator in the manifest, and DB::Open would fail + // if a different comparator were used. + ASSERT_OK(RepairDB(dbname_, opts, {{"default", opts}, {"reverse", rev_opts}}, + opts /* unknown_cf_opts */)); + ASSERT_OK(TryReopenWithColumnFamilies({"default", "reverse"}, + std::vector{opts, rev_opts})); + for (int i = 0; i < kNumCfs; ++i) { + for (int j = 0; j < kEntriesPerCf; ++j) { + ASSERT_EQ(Get(i, "key" + std::to_string(j)), "val" + std::to_string(j)); + } + } + + // Examine table properties to verify RepairDB() used the right options when + // converting WAL->SST + TablePropertiesCollection fname_to_props; + ASSERT_OK(db_->GetPropertiesOfAllTables(handles_[1], &fname_to_props)); + ASSERT_EQ(fname_to_props.size(), 2U); + for (const auto& fname_and_props : fname_to_props) { + std::string comparator_name(rev_opts.comparator->Name()); + ASSERT_EQ(comparator_name, fname_and_props.second->comparator_name); + } + Close(); + + // Also check comparator when it's provided via "unknown" CF options + ASSERT_OK(RepairDB(dbname_, opts, {{"default", opts}}, + rev_opts /* unknown_cf_opts */)); + ASSERT_OK(TryReopenWithColumnFamilies({"default", "reverse"}, + std::vector{opts, rev_opts})); + for (int i = 0; i < kNumCfs; ++i) { + for (int j = 0; j < kEntriesPerCf; ++j) { + ASSERT_EQ(Get(i, "key" + std::to_string(j)), "val" + std::to_string(j)); + } + } +} + +TEST_F(RepairTest, DbNameContainsTrailingSlash) { + { + bool tmp; + if (env_->AreFilesSame("", "", &tmp).IsNotSupported()) { + fprintf(stderr, + "skipping RepairTest.DbNameContainsTrailingSlash due to " + "unsupported Env::AreFilesSame\n"); + return; + } + } + + ASSERT_OK(Put("key", "val")); + ASSERT_OK(Flush()); + Close(); + + ASSERT_OK(RepairDB(dbname_ + "/", CurrentOptions())); + ReopenWithSstIdVerify(); + ASSERT_EQ(Get("key"), "val"); +} +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/librocksdb-sys/rocksdb/db/seqno_time_test.cc b/librocksdb-sys/rocksdb/db/seqno_time_test.cc new file mode 100644 index 0000000..b18b255 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/seqno_time_test.cc @@ -0,0 +1,995 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/db_test_util.h" +#include "db/periodic_task_scheduler.h" +#include "db/seqno_to_time_mapping.h" +#include "port/stack_trace.h" +#include "rocksdb/iostats_context.h" +#include "rocksdb/utilities/debug.h" +#include "test_util/mock_time_env.h" + + +namespace ROCKSDB_NAMESPACE { + +class SeqnoTimeTest : public DBTestBase { + public: + SeqnoTimeTest() : DBTestBase("seqno_time_test", /*env_do_fsync=*/false) { + mock_clock_ = std::make_shared(env_->GetSystemClock()); + mock_env_ = std::make_unique(env_, mock_clock_); + } + + protected: + std::unique_ptr mock_env_; + std::shared_ptr mock_clock_; + + void SetUp() override { + mock_clock_->InstallTimedWaitFixCallback(); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::StartPeriodicTaskScheduler:Init", [&](void* arg) { + auto periodic_task_scheduler_ptr = + reinterpret_cast(arg); + periodic_task_scheduler_ptr->TEST_OverrideTimer(mock_clock_.get()); + }); + } + + // make sure the file is not in cache, otherwise it won't have IO info + void AssertKeyTemperature(int key_id, Temperature expected_temperature) { + get_iostats_context()->Reset(); + IOStatsContext* iostats = get_iostats_context(); + std::string result = Get(Key(key_id)); + ASSERT_FALSE(result.empty()); + ASSERT_GT(iostats->bytes_read, 0); + switch (expected_temperature) { + case Temperature::kUnknown: + ASSERT_EQ(iostats->file_io_stats_by_temperature.cold_file_read_count, + 0); + ASSERT_EQ(iostats->file_io_stats_by_temperature.cold_file_bytes_read, + 0); + break; + case Temperature::kCold: + ASSERT_GT(iostats->file_io_stats_by_temperature.cold_file_read_count, + 0); + ASSERT_GT(iostats->file_io_stats_by_temperature.cold_file_bytes_read, + 0); + break; + default: + // the test only support kCold now for the bottommost temperature + FAIL(); + } + } +}; + +TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) { + const int kNumTrigger = 4; + const int kNumLevels = 7; + const int kNumKeys = 100; + const int kKeyPerSec = 10; + + Options options = CurrentOptions(); + options.compaction_style = kCompactionStyleUniversal; + options.preclude_last_level_data_seconds = 10000; + options.env = mock_env_.get(); + options.bottommost_temperature = Temperature::kCold; + options.num_levels = kNumLevels; + DestroyAndReopen(options); + + // pass some time first, otherwise the first a few keys write time are going + // to be zero, and internally zero has special meaning: kUnknownSeqnoTime + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); }); + + int sst_num = 0; + // Write files that are overlap and enough to trigger compaction + for (; sst_num < kNumTrigger; sst_num++) { + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun([&] { + mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); + }); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // All data is hot, only output to penultimate level + ASSERT_EQ("0,0,0,0,0,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0); + + // read a random key, which should be hot (kUnknown) + AssertKeyTemperature(20, Temperature::kUnknown); + + // Write more data, but still all hot until the 10th SST, as: + // write a key every 10 seconds, 100 keys per SST, each SST takes 1000 seconds + // The preclude_last_level_data_seconds is 10k + for (; sst_num < kNumTrigger * 2; sst_num++) { + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun([&] { + mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); + }); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0); + } + + // Now we have both hot data and cold data + for (; sst_num < kNumTrigger * 3; sst_num++) { + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun([&] { + mock_clock_->MockSleepForSeconds(static_cast(kKeyPerSec)); + }); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + uint64_t hot_data_size = GetSstSizeHelper(Temperature::kUnknown); + uint64_t cold_data_size = GetSstSizeHelper(Temperature::kCold); + ASSERT_GT(hot_data_size, 0); + ASSERT_GT(cold_data_size, 0); + // the first a few key should be cold + AssertKeyTemperature(20, Temperature::kCold); + + for (int i = 0; i < 30; i++) { + dbfull()->TEST_WaitForPeriodicTaskRun([&] { + mock_clock_->MockSleepForSeconds(static_cast(20 * kKeyPerSec)); + }); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + // the hot/cold data cut off range should be between i * 20 + 200 -> 250 + AssertKeyTemperature(i * 20 + 250, Temperature::kUnknown); + AssertKeyTemperature(i * 20 + 200, Temperature::kCold); + } + + ASSERT_LT(GetSstSizeHelper(Temperature::kUnknown), hot_data_size); + ASSERT_GT(GetSstSizeHelper(Temperature::kCold), cold_data_size); + + // Wait again, the most of the data should be cold after that + // but it may not be all cold, because if there's no new data write to SST, + // the compaction will not get the new seqno->time sampling to decide the last + // a few data's time. + for (int i = 0; i < 5; i++) { + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(1000)); }); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + } + + // any random data close to the end should be cold + AssertKeyTemperature(1000, Temperature::kCold); + + // close explicitly, because the env is local variable which will be released + // first. + Close(); +} + +TEST_F(SeqnoTimeTest, TemperatureBasicLevel) { + const int kNumLevels = 7; + const int kNumKeys = 100; + + Options options = CurrentOptions(); + options.preclude_last_level_data_seconds = 10000; + options.env = mock_env_.get(); + options.bottommost_temperature = Temperature::kCold; + options.num_levels = kNumLevels; + options.level_compaction_dynamic_level_bytes = true; + // TODO(zjay): for level compaction, auto-compaction may stuck in deadloop, if + // the penultimate level score > 1, but the hot is not cold enough to compact + // to last level, which will keep triggering compaction. + options.disable_auto_compactions = true; + DestroyAndReopen(options); + + // pass some time first, otherwise the first a few keys write time are going + // to be zero, and internally zero has special meaning: kUnknownSeqnoTime + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(10)); }); + + int sst_num = 0; + // Write files that are overlap + for (; sst_num < 4; sst_num++) { + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(10)); }); + } + ASSERT_OK(Flush()); + } + + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + // All data is hot, only output to penultimate level + ASSERT_EQ("0,0,0,0,0,1", FilesPerLevel()); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0); + + // read a random key, which should be hot (kUnknown) + AssertKeyTemperature(20, Temperature::kUnknown); + + // Adding more data to have mixed hot and cold data + for (; sst_num < 14; sst_num++) { + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(10)); }); + } + ASSERT_OK(Flush()); + } + // Second to last level + MoveFilesToLevel(5); + ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0); + ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0); + + // Compact the files to the last level which should split the hot/cold data + MoveFilesToLevel(6); + uint64_t hot_data_size = GetSstSizeHelper(Temperature::kUnknown); + uint64_t cold_data_size = GetSstSizeHelper(Temperature::kCold); + ASSERT_GT(hot_data_size, 0); + ASSERT_GT(cold_data_size, 0); + // the first a few key should be cold + AssertKeyTemperature(20, Temperature::kCold); + + // Wait some time, with each wait, the cold data is increasing and hot data is + // decreasing + for (int i = 0; i < 30; i++) { + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(200)); }); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + uint64_t pre_hot = hot_data_size; + uint64_t pre_cold = cold_data_size; + hot_data_size = GetSstSizeHelper(Temperature::kUnknown); + cold_data_size = GetSstSizeHelper(Temperature::kCold); + ASSERT_LT(hot_data_size, pre_hot); + ASSERT_GT(cold_data_size, pre_cold); + + // the hot/cold cut_off key should be around i * 20 + 400 -> 450 + AssertKeyTemperature(i * 20 + 450, Temperature::kUnknown); + AssertKeyTemperature(i * 20 + 400, Temperature::kCold); + } + + // Wait again, the most of the data should be cold after that + // hot data might not be empty, because if we don't write new data, there's + // no seqno->time sampling available to the compaction + for (int i = 0; i < 5; i++) { + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(1000)); }); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + } + + // any random data close to the end should be cold + AssertKeyTemperature(1000, Temperature::kCold); + + Close(); +} + +enum class SeqnoTimeTestType : char { + kTrackInternalTimeSeconds = 0, + kPrecludeLastLevel = 1, + kBothSetTrackSmaller = 2, +}; + +class SeqnoTimeTablePropTest + : public SeqnoTimeTest, + public ::testing::WithParamInterface { + public: + SeqnoTimeTablePropTest() : SeqnoTimeTest() {} + + void SetTrackTimeDurationOptions(uint64_t track_time_duration, + Options& options) const { + // either option set will enable the time tracking feature + switch (GetParam()) { + case SeqnoTimeTestType::kTrackInternalTimeSeconds: + options.preclude_last_level_data_seconds = 0; + options.preserve_internal_time_seconds = track_time_duration; + break; + case SeqnoTimeTestType::kPrecludeLastLevel: + options.preclude_last_level_data_seconds = track_time_duration; + options.preserve_internal_time_seconds = 0; + break; + case SeqnoTimeTestType::kBothSetTrackSmaller: + options.preclude_last_level_data_seconds = track_time_duration; + options.preserve_internal_time_seconds = track_time_duration / 10; + break; + } + } +}; + +INSTANTIATE_TEST_CASE_P( + SeqnoTimeTablePropTest, SeqnoTimeTablePropTest, + ::testing::Values(SeqnoTimeTestType::kTrackInternalTimeSeconds, + SeqnoTimeTestType::kPrecludeLastLevel, + SeqnoTimeTestType::kBothSetTrackSmaller)); + +TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) { + Options options = CurrentOptions(); + SetTrackTimeDurationOptions(10000, options); + + options.env = mock_env_.get(); + options.disable_auto_compactions = true; + DestroyAndReopen(options); + + std::set checked_file_nums; + SequenceNumber start_seq = dbfull()->GetLatestSequenceNumber(); + // Write a key every 10 seconds + for (int i = 0; i < 200; i++) { + ASSERT_OK(Put(Key(i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(10)); }); + } + ASSERT_OK(Flush()); + TablePropertiesCollection tables_props; + ASSERT_OK(dbfull()->GetPropertiesOfAllTables(&tables_props)); + ASSERT_EQ(tables_props.size(), 1); + auto it = tables_props.begin(); + SeqnoToTimeMapping tp_mapping; + ASSERT_OK(tp_mapping.Add(it->second->seqno_to_time_mapping)); + ASSERT_OK(tp_mapping.Sort()); + ASSERT_FALSE(tp_mapping.Empty()); + auto seqs = tp_mapping.TEST_GetInternalMapping(); + // about ~20 seqs->time entries, because the sample rate is 10000/100, and it + // passes 2k time. + ASSERT_GE(seqs.size(), 19); + ASSERT_LE(seqs.size(), 21); + SequenceNumber seq_end = dbfull()->GetLatestSequenceNumber(); + for (auto i = start_seq; i < start_seq + 10; i++) { + ASSERT_LE(tp_mapping.GetOldestApproximateTime(i), (i + 1) * 10); + } + start_seq += 10; + for (auto i = start_seq; i < seq_end; i++) { + // The result is within the range + ASSERT_GE(tp_mapping.GetOldestApproximateTime(i), (i - 10) * 10); + ASSERT_LE(tp_mapping.GetOldestApproximateTime(i), (i + 10) * 10); + } + checked_file_nums.insert(it->second->orig_file_number); + start_seq = seq_end; + + // Write a key every 1 seconds + for (int i = 0; i < 200; i++) { + ASSERT_OK(Put(Key(i + 190), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(1)); }); + } + seq_end = dbfull()->GetLatestSequenceNumber(); + ASSERT_OK(Flush()); + tables_props.clear(); + ASSERT_OK(dbfull()->GetPropertiesOfAllTables(&tables_props)); + ASSERT_EQ(tables_props.size(), 2); + it = tables_props.begin(); + while (it != tables_props.end()) { + if (!checked_file_nums.count(it->second->orig_file_number)) { + break; + } + it++; + } + ASSERT_TRUE(it != tables_props.end()); + + tp_mapping.Clear(); + ASSERT_OK(tp_mapping.Add(it->second->seqno_to_time_mapping)); + ASSERT_OK(tp_mapping.Sort()); + seqs = tp_mapping.TEST_GetInternalMapping(); + // There only a few time sample + ASSERT_GE(seqs.size(), 1); + ASSERT_LE(seqs.size(), 3); + for (auto i = start_seq; i < seq_end; i++) { + // The result is not very accurate, as there is more data write within small + // range of time + ASSERT_GE(tp_mapping.GetOldestApproximateTime(i), (i - start_seq) + 1000); + ASSERT_LE(tp_mapping.GetOldestApproximateTime(i), (i - start_seq) + 3000); + } + checked_file_nums.insert(it->second->orig_file_number); + start_seq = seq_end; + + // Write a key every 200 seconds + for (int i = 0; i < 200; i++) { + ASSERT_OK(Put(Key(i + 380), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(200)); }); + } + seq_end = dbfull()->GetLatestSequenceNumber(); + ASSERT_OK(Flush()); + tables_props.clear(); + ASSERT_OK(dbfull()->GetPropertiesOfAllTables(&tables_props)); + ASSERT_EQ(tables_props.size(), 3); + it = tables_props.begin(); + while (it != tables_props.end()) { + if (!checked_file_nums.count(it->second->orig_file_number)) { + break; + } + it++; + } + ASSERT_TRUE(it != tables_props.end()); + + tp_mapping.Clear(); + ASSERT_OK(tp_mapping.Add(it->second->seqno_to_time_mapping)); + ASSERT_OK(tp_mapping.Sort()); + seqs = tp_mapping.TEST_GetInternalMapping(); + // The sequence number -> time entries should be maxed + ASSERT_GE(seqs.size(), 99); + ASSERT_LE(seqs.size(), 101); + for (auto i = start_seq; i < seq_end - 99; i++) { + // likely the first 100 entries reports 0 + ASSERT_LE(tp_mapping.GetOldestApproximateTime(i), (i - start_seq) + 3000); + } + start_seq += 101; + + for (auto i = start_seq; i < seq_end; i++) { + ASSERT_GE(tp_mapping.GetOldestApproximateTime(i), + (i - start_seq) * 200 + 22200); + ASSERT_LE(tp_mapping.GetOldestApproximateTime(i), + (i - start_seq) * 200 + 22600); + } + checked_file_nums.insert(it->second->orig_file_number); + start_seq = seq_end; + + // Write a key every 100 seconds + for (int i = 0; i < 200; i++) { + ASSERT_OK(Put(Key(i + 570), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(100)); }); + } + seq_end = dbfull()->GetLatestSequenceNumber(); + ASSERT_OK(Flush()); + tables_props.clear(); + ASSERT_OK(dbfull()->GetPropertiesOfAllTables(&tables_props)); + ASSERT_EQ(tables_props.size(), 4); + it = tables_props.begin(); + while (it != tables_props.end()) { + if (!checked_file_nums.count(it->second->orig_file_number)) { + break; + } + it++; + } + ASSERT_TRUE(it != tables_props.end()); + tp_mapping.Clear(); + ASSERT_OK(tp_mapping.Add(it->second->seqno_to_time_mapping)); + ASSERT_OK(tp_mapping.Sort()); + seqs = tp_mapping.TEST_GetInternalMapping(); + ASSERT_GE(seqs.size(), 99); + ASSERT_LE(seqs.size(), 101); + + checked_file_nums.insert(it->second->orig_file_number); + + // re-enable compaction + ASSERT_OK(dbfull()->SetOptions({ + {"disable_auto_compactions", "false"}, + })); + + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + tables_props.clear(); + ASSERT_OK(dbfull()->GetPropertiesOfAllTables(&tables_props)); + ASSERT_GE(tables_props.size(), 1); + it = tables_props.begin(); + while (it != tables_props.end()) { + if (!checked_file_nums.count(it->second->orig_file_number)) { + break; + } + it++; + } + ASSERT_TRUE(it != tables_props.end()); + tp_mapping.Clear(); + ASSERT_OK(tp_mapping.Add(it->second->seqno_to_time_mapping)); + ASSERT_OK(tp_mapping.Sort()); + seqs = tp_mapping.TEST_GetInternalMapping(); + ASSERT_GE(seqs.size(), 99); + ASSERT_LE(seqs.size(), 101); + for (auto i = start_seq; i < seq_end - 99; i++) { + // likely the first 100 entries reports 0 + ASSERT_LE(tp_mapping.GetOldestApproximateTime(i), + (i - start_seq) * 100 + 50000); + } + start_seq += 101; + + for (auto i = start_seq; i < seq_end; i++) { + ASSERT_GE(tp_mapping.GetOldestApproximateTime(i), + (i - start_seq) * 100 + 52200); + ASSERT_LE(tp_mapping.GetOldestApproximateTime(i), + (i - start_seq) * 100 + 52400); + } + ASSERT_OK(db_->Close()); +} + +TEST_P(SeqnoTimeTablePropTest, MultiCFs) { + Options options = CurrentOptions(); + options.preclude_last_level_data_seconds = 0; + options.preserve_internal_time_seconds = 0; + options.env = mock_env_.get(); + options.stats_dump_period_sec = 0; + options.stats_persist_period_sec = 0; + ReopenWithColumnFamilies({"default"}, options); + + const PeriodicTaskScheduler& scheduler = + dbfull()->TEST_GetPeriodicTaskScheduler(); + ASSERT_FALSE(scheduler.TEST_HasTask(PeriodicTaskType::kRecordSeqnoTime)); + + // Write some data and increase the current time + for (int i = 0; i < 200; i++) { + ASSERT_OK(Put(Key(i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(100)); }); + } + ASSERT_OK(Flush()); + TablePropertiesCollection tables_props; + ASSERT_OK(dbfull()->GetPropertiesOfAllTables(&tables_props)); + ASSERT_EQ(tables_props.size(), 1); + auto it = tables_props.begin(); + ASSERT_TRUE(it->second->seqno_to_time_mapping.empty()); + + ASSERT_TRUE(dbfull()->TEST_GetSeqnoToTimeMapping().Empty()); + + Options options_1 = options; + SetTrackTimeDurationOptions(10000, options_1); + CreateColumnFamilies({"one"}, options_1); + ASSERT_TRUE(scheduler.TEST_HasTask(PeriodicTaskType::kRecordSeqnoTime)); + + // Write some data to the default CF (without preclude_last_level feature) + for (int i = 0; i < 200; i++) { + ASSERT_OK(Put(Key(i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(100)); }); + } + ASSERT_OK(Flush()); + + // Write some data to the CF one + for (int i = 0; i < 20; i++) { + ASSERT_OK(Put(1, Key(i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(10)); }); + } + ASSERT_OK(Flush(1)); + tables_props.clear(); + ASSERT_OK(dbfull()->GetPropertiesOfAllTables(handles_[1], &tables_props)); + ASSERT_EQ(tables_props.size(), 1); + it = tables_props.begin(); + SeqnoToTimeMapping tp_mapping; + ASSERT_OK(tp_mapping.Add(it->second->seqno_to_time_mapping)); + ASSERT_OK(tp_mapping.Sort()); + ASSERT_FALSE(tp_mapping.Empty()); + auto seqs = tp_mapping.TEST_GetInternalMapping(); + ASSERT_GE(seqs.size(), 1); + ASSERT_LE(seqs.size(), 4); + + // Create one more CF with larger preclude_last_level time + Options options_2 = options; + SetTrackTimeDurationOptions(1000000, options_2); // 1m + CreateColumnFamilies({"two"}, options_2); + + // Add more data to CF "two" to fill the in memory mapping + for (int i = 0; i < 2000; i++) { + ASSERT_OK(Put(2, Key(i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(100)); }); + } + seqs = dbfull()->TEST_GetSeqnoToTimeMapping().TEST_GetInternalMapping(); + ASSERT_GE(seqs.size(), 1000 - 1); + ASSERT_LE(seqs.size(), 1000 + 1); + + ASSERT_OK(Flush(2)); + tables_props.clear(); + ASSERT_OK(dbfull()->GetPropertiesOfAllTables(handles_[2], &tables_props)); + ASSERT_EQ(tables_props.size(), 1); + it = tables_props.begin(); + tp_mapping.Clear(); + ASSERT_OK(tp_mapping.Add(it->second->seqno_to_time_mapping)); + ASSERT_OK(tp_mapping.Sort()); + seqs = tp_mapping.TEST_GetInternalMapping(); + // the max encoded entries is 100 + ASSERT_GE(seqs.size(), 100 - 1); + ASSERT_LE(seqs.size(), 100 + 1); + + // Write some data to default CF, as all memtable with preclude_last_level + // enabled have flushed, the in-memory seqno->time mapping should be cleared + for (int i = 0; i < 10; i++) { + ASSERT_OK(Put(0, Key(i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(100)); }); + } + seqs = dbfull()->TEST_GetSeqnoToTimeMapping().TEST_GetInternalMapping(); + ASSERT_OK(Flush(0)); + + // trigger compaction for CF "two" and make sure the compaction output has + // seqno_to_time_mapping + for (int j = 0; j < 3; j++) { + for (int i = 0; i < 200; i++) { + ASSERT_OK(Put(2, Key(i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(100)); }); + } + ASSERT_OK(Flush(2)); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + tables_props.clear(); + ASSERT_OK(dbfull()->GetPropertiesOfAllTables(handles_[2], &tables_props)); + ASSERT_EQ(tables_props.size(), 1); + it = tables_props.begin(); + tp_mapping.Clear(); + ASSERT_OK(tp_mapping.Add(it->second->seqno_to_time_mapping)); + ASSERT_OK(tp_mapping.Sort()); + seqs = tp_mapping.TEST_GetInternalMapping(); + ASSERT_GE(seqs.size(), 99); + ASSERT_LE(seqs.size(), 101); + + for (int j = 0; j < 2; j++) { + for (int i = 0; i < 200; i++) { + ASSERT_OK(Put(0, Key(i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(100)); }); + } + ASSERT_OK(Flush(0)); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + tables_props.clear(); + ASSERT_OK(dbfull()->GetPropertiesOfAllTables(handles_[0], &tables_props)); + ASSERT_EQ(tables_props.size(), 1); + it = tables_props.begin(); + ASSERT_TRUE(it->second->seqno_to_time_mapping.empty()); + + // Write some data to CF "two", but don't flush to accumulate + for (int i = 0; i < 1000; i++) { + ASSERT_OK(Put(2, Key(i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(100)); }); + } + ASSERT_GE( + dbfull()->TEST_GetSeqnoToTimeMapping().TEST_GetInternalMapping().size(), + 500); + // After dropping CF "one", the in-memory mapping will be change to only + // follow CF "two" options. + ASSERT_OK(db_->DropColumnFamily(handles_[1])); + ASSERT_LE( + dbfull()->TEST_GetSeqnoToTimeMapping().TEST_GetInternalMapping().size(), + 100 + 5); + + // After dropping CF "two", the in-memory mapping is also clear. + ASSERT_OK(db_->DropColumnFamily(handles_[2])); + ASSERT_EQ( + dbfull()->TEST_GetSeqnoToTimeMapping().TEST_GetInternalMapping().size(), + 0); + + // And the timer worker is stopped + ASSERT_FALSE(scheduler.TEST_HasTask(PeriodicTaskType::kRecordSeqnoTime)); + Close(); +} + +TEST_P(SeqnoTimeTablePropTest, MultiInstancesBasic) { + const int kInstanceNum = 2; + + Options options = CurrentOptions(); + SetTrackTimeDurationOptions(10000, options); + options.env = mock_env_.get(); + options.stats_dump_period_sec = 0; + options.stats_persist_period_sec = 0; + + auto dbs = std::vector(kInstanceNum); + for (int i = 0; i < kInstanceNum; i++) { + ASSERT_OK( + DB::Open(options, test::PerThreadDBPath(std::to_string(i)), &(dbs[i]))); + } + + // Make sure the second instance has the worker enabled + auto dbi = static_cast_with_check(dbs[1]); + WriteOptions wo; + for (int i = 0; i < 200; i++) { + ASSERT_OK(dbi->Put(wo, Key(i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(100)); }); + } + SeqnoToTimeMapping seqno_to_time_mapping = dbi->TEST_GetSeqnoToTimeMapping(); + ASSERT_GT(seqno_to_time_mapping.Size(), 10); + + for (int i = 0; i < kInstanceNum; i++) { + ASSERT_OK(dbs[i]->Close()); + delete dbs[i]; + } +} + +TEST_P(SeqnoTimeTablePropTest, SeqnoToTimeMappingUniversal) { + const int kNumTrigger = 4; + const int kNumLevels = 7; + const int kNumKeys = 100; + + Options options = CurrentOptions(); + SetTrackTimeDurationOptions(10000, options); + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = kNumLevels; + options.env = mock_env_.get(); + + DestroyAndReopen(options); + + std::atomic_uint64_t num_seqno_zeroing{0}; + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "CompactionIterator::PrepareOutput:ZeroingSeq", + [&](void* /*arg*/) { num_seqno_zeroing++; }); + SyncPoint::GetInstance()->EnableProcessing(); + + int sst_num = 0; + for (; sst_num < kNumTrigger - 1; sst_num++) { + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(10)); }); + } + ASSERT_OK(Flush()); + } + TablePropertiesCollection tables_props; + ASSERT_OK(dbfull()->GetPropertiesOfAllTables(&tables_props)); + ASSERT_EQ(tables_props.size(), 3); + for (const auto& props : tables_props) { + ASSERT_FALSE(props.second->seqno_to_time_mapping.empty()); + SeqnoToTimeMapping tp_mapping; + ASSERT_OK(tp_mapping.Add(props.second->seqno_to_time_mapping)); + ASSERT_OK(tp_mapping.Sort()); + ASSERT_FALSE(tp_mapping.Empty()); + auto seqs = tp_mapping.TEST_GetInternalMapping(); + ASSERT_GE(seqs.size(), 10 - 1); + ASSERT_LE(seqs.size(), 10 + 1); + } + + // Trigger a compaction + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value")); + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(static_cast(10)); }); + } + sst_num++; + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + tables_props.clear(); + ASSERT_OK(dbfull()->GetPropertiesOfAllTables(&tables_props)); + ASSERT_EQ(tables_props.size(), 1); + + auto it = tables_props.begin(); + SeqnoToTimeMapping tp_mapping; + ASSERT_FALSE(it->second->seqno_to_time_mapping.empty()); + ASSERT_OK(tp_mapping.Add(it->second->seqno_to_time_mapping)); + + // compact to the last level + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + // make sure the data is all compacted to penultimate level if the feature is + // on, otherwise, compacted to the last level. + if (options.preclude_last_level_data_seconds > 0) { + ASSERT_GT(NumTableFilesAtLevel(5), 0); + ASSERT_EQ(NumTableFilesAtLevel(6), 0); + } else { + ASSERT_EQ(NumTableFilesAtLevel(5), 0); + ASSERT_GT(NumTableFilesAtLevel(6), 0); + } + + // regardless the file is on the last level or not, it should keep the time + // information and sequence number are not set + tables_props.clear(); + tp_mapping.Clear(); + ASSERT_OK(dbfull()->GetPropertiesOfAllTables(&tables_props)); + + ASSERT_EQ(tables_props.size(), 1); + ASSERT_EQ(num_seqno_zeroing, 0); + + it = tables_props.begin(); + ASSERT_FALSE(it->second->seqno_to_time_mapping.empty()); + ASSERT_OK(tp_mapping.Add(it->second->seqno_to_time_mapping)); + + // make half of the data expired + mock_clock_->MockSleepForSeconds(static_cast(8000)); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + tables_props.clear(); + tp_mapping.Clear(); + ASSERT_OK(dbfull()->GetPropertiesOfAllTables(&tables_props)); + + if (options.preclude_last_level_data_seconds > 0) { + ASSERT_EQ(tables_props.size(), 2); + } else { + ASSERT_EQ(tables_props.size(), 1); + } + ASSERT_GT(num_seqno_zeroing, 0); + std::vector key_versions; + ASSERT_OK(GetAllKeyVersions(db_, Slice(), Slice(), + std::numeric_limits::max(), + &key_versions)); + // make sure there're more than 300 keys and first 100 keys are having seqno + // zeroed out, the last 100 key seqno not zeroed out + ASSERT_GT(key_versions.size(), 300); + for (int i = 0; i < 100; i++) { + ASSERT_EQ(key_versions[i].sequence, 0); + } + auto rit = key_versions.rbegin(); + for (int i = 0; i < 100; i++) { + ASSERT_GT(rit->sequence, 0); + rit++; + } + + // make all data expired and compact again to push it to the last level + // regardless if the tiering feature is enabled or not + mock_clock_->MockSleepForSeconds(static_cast(20000)); + + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + + ASSERT_GT(num_seqno_zeroing, 0); + ASSERT_GT(NumTableFilesAtLevel(6), 0); + + Close(); +} + +TEST_F(SeqnoTimeTest, MappingAppend) { + SeqnoToTimeMapping test(/*max_time_duration=*/100, /*max_capacity=*/10); + + // ignore seqno == 0, as it may mean the seqno is zeroed out + ASSERT_FALSE(test.Append(0, 9)); + + ASSERT_TRUE(test.Append(3, 10)); + auto size = test.Size(); + // normal add + ASSERT_TRUE(test.Append(10, 11)); + size++; + ASSERT_EQ(size, test.Size()); + + // Append unsorted + ASSERT_FALSE(test.Append(8, 12)); + ASSERT_EQ(size, test.Size()); + + // Append with the same seqno, newer time will be accepted + ASSERT_TRUE(test.Append(10, 12)); + ASSERT_EQ(size, test.Size()); + // older time will be ignored + ASSERT_FALSE(test.Append(10, 9)); + ASSERT_EQ(size, test.Size()); + + // new seqno with old time will be ignored + ASSERT_FALSE(test.Append(12, 8)); + ASSERT_EQ(size, test.Size()); +} + +TEST_F(SeqnoTimeTest, GetOldestApproximateTime) { + SeqnoToTimeMapping test(/*max_time_duration=*/100, /*max_capacity=*/10); + + ASSERT_EQ(test.GetOldestApproximateTime(10), kUnknownSeqnoTime); + + test.Append(3, 10); + + ASSERT_EQ(test.GetOldestApproximateTime(2), kUnknownSeqnoTime); + ASSERT_EQ(test.GetOldestApproximateTime(3), 10); + ASSERT_EQ(test.GetOldestApproximateTime(10), 10); + + test.Append(10, 100); + + test.Append(100, 1000); + ASSERT_EQ(test.GetOldestApproximateTime(10), 100); + ASSERT_EQ(test.GetOldestApproximateTime(40), 100); + ASSERT_EQ(test.GetOldestApproximateTime(111), 1000); +} + +TEST_F(SeqnoTimeTest, Sort) { + SeqnoToTimeMapping test; + + // single entry + test.Add(10, 11); + ASSERT_OK(test.Sort()); + ASSERT_EQ(test.Size(), 1); + + // duplicate, should be removed by sort + test.Add(10, 11); + // same seqno, but older time, should be removed + test.Add(10, 9); + + // unuseful ones, should be removed by sort + test.Add(11, 9); + test.Add(9, 8); + + // Good ones + test.Add(1, 10); + test.Add(100, 100); + + ASSERT_OK(test.Sort()); + + auto seqs = test.TEST_GetInternalMapping(); + + std::deque expected; + expected.emplace_back(1, 10); + expected.emplace_back(10, 11); + expected.emplace_back(100, 100); + + ASSERT_EQ(expected, seqs); +} + +TEST_F(SeqnoTimeTest, EncodeDecodeBasic) { + SeqnoToTimeMapping test(0, 1000); + + std::string output; + test.Encode(output, 0, 1000, 100); + ASSERT_TRUE(output.empty()); + + for (int i = 1; i <= 1000; i++) { + ASSERT_TRUE(test.Append(i, i * 10)); + } + test.Encode(output, 0, 1000, 100); + + ASSERT_FALSE(output.empty()); + + SeqnoToTimeMapping decoded; + ASSERT_OK(decoded.Add(output)); + ASSERT_OK(decoded.Sort()); + ASSERT_EQ(decoded.Size(), SeqnoToTimeMapping::kMaxSeqnoTimePairsPerSST); + ASSERT_EQ(test.Size(), 1000); + + for (SequenceNumber seq = 0; seq <= 1000; seq++) { + // test has the more accurate time mapping, encode only pick + // kMaxSeqnoTimePairsPerSST number of entries, which is less accurate + uint64_t target_time = test.GetOldestApproximateTime(seq); + ASSERT_GE(decoded.GetOldestApproximateTime(seq), + target_time < 200 ? 0 : target_time - 200); + ASSERT_LE(decoded.GetOldestApproximateTime(seq), target_time); + } +} + +TEST_F(SeqnoTimeTest, EncodeDecodePerferNewTime) { + SeqnoToTimeMapping test(0, 10); + + test.Append(1, 10); + test.Append(5, 17); + test.Append(6, 25); + test.Append(8, 30); + + std::string output; + test.Encode(output, 1, 10, 0, 3); + + SeqnoToTimeMapping decoded; + ASSERT_OK(decoded.Add(output)); + ASSERT_OK(decoded.Sort()); + + ASSERT_EQ(decoded.Size(), 3); + + auto seqs = decoded.TEST_GetInternalMapping(); + std::deque expected; + expected.emplace_back(1, 10); + expected.emplace_back(6, 25); + expected.emplace_back(8, 30); + ASSERT_EQ(expected, seqs); + + // Add a few large time number + test.Append(10, 100); + test.Append(13, 200); + test.Append(16, 300); + + output.clear(); + test.Encode(output, 1, 20, 0, 4); + decoded.Clear(); + ASSERT_OK(decoded.Add(output)); + ASSERT_OK(decoded.Sort()); + ASSERT_EQ(decoded.Size(), 4); + + expected.clear(); + expected.emplace_back(1, 10); + // entry #6, #8 are skipped as they are too close to #1. + // entry #100 is also within skip range, but if it's skipped, there not enough + // number to fill 4 entries, so select it. + expected.emplace_back(10, 100); + expected.emplace_back(13, 200); + expected.emplace_back(16, 300); + seqs = decoded.TEST_GetInternalMapping(); + ASSERT_EQ(expected, seqs); +} + +} // namespace ROCKSDB_NAMESPACE + + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/seqno_to_time_mapping.cc b/librocksdb-sys/rocksdb/db/seqno_to_time_mapping.cc new file mode 100644 index 0000000..c692099 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/seqno_to_time_mapping.cc @@ -0,0 +1,341 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/seqno_to_time_mapping.h" + +#include "db/version_edit.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +uint64_t SeqnoToTimeMapping::GetOldestApproximateTime( + const SequenceNumber seqno) const { + assert(is_sorted_); + auto it = std::upper_bound(seqno_time_mapping_.begin(), + seqno_time_mapping_.end(), seqno); + if (it == seqno_time_mapping_.begin()) { + return 0; + } + it--; + return it->time; +} + +void SeqnoToTimeMapping::Add(SequenceNumber seqno, uint64_t time) { + if (seqno == 0) { + return; + } + is_sorted_ = false; + seqno_time_mapping_.emplace_back(seqno, time); +} + +void SeqnoToTimeMapping::TruncateOldEntries(const uint64_t now) { + assert(is_sorted_); + + if (max_time_duration_ == 0) { + return; + } + + const uint64_t cut_off_time = + now > max_time_duration_ ? now - max_time_duration_ : 0; + assert(cut_off_time <= now); // no overflow + + auto it = std::upper_bound( + seqno_time_mapping_.begin(), seqno_time_mapping_.end(), cut_off_time, + [](uint64_t target, const SeqnoTimePair& other) -> bool { + return target < other.time; + }); + if (it == seqno_time_mapping_.begin()) { + return; + } + it--; + seqno_time_mapping_.erase(seqno_time_mapping_.begin(), it); +} + +SequenceNumber SeqnoToTimeMapping::GetOldestSequenceNum(uint64_t time) { + assert(is_sorted_); + + auto it = std::upper_bound( + seqno_time_mapping_.begin(), seqno_time_mapping_.end(), time, + [](uint64_t target, const SeqnoTimePair& other) -> bool { + return target < other.time; + }); + if (it == seqno_time_mapping_.begin()) { + return 0; + } + it--; + return it->seqno; +} + +// The encoded format is: +// [num_of_entries][[seqno][time],[seqno][time],...] +// ^ ^ +// var_int delta_encoded (var_int) +void SeqnoToTimeMapping::Encode(std::string& dest, const SequenceNumber start, + const SequenceNumber end, const uint64_t now, + const uint64_t output_size) const { + assert(is_sorted_); + if (start > end) { + // It could happen when the SST file is empty, the initial value of min + // sequence number is kMaxSequenceNumber and max is 0. + // The empty output file will be removed in the final step of compaction. + return; + } + + auto start_it = std::upper_bound(seqno_time_mapping_.begin(), + seqno_time_mapping_.end(), start); + if (start_it != seqno_time_mapping_.begin()) { + start_it--; + } + + auto end_it = std::upper_bound(seqno_time_mapping_.begin(), + seqno_time_mapping_.end(), end); + if (end_it == seqno_time_mapping_.begin()) { + return; + } + if (start_it >= end_it) { + return; + } + + // truncate old entries that are not needed + if (max_time_duration_ > 0) { + const uint64_t cut_off_time = + now > max_time_duration_ ? now - max_time_duration_ : 0; + while (start_it < end_it && start_it->time < cut_off_time) { + start_it++; + } + } + // to include the first element + if (start_it != seqno_time_mapping_.begin()) { + start_it--; + } + + // If there are more data than needed, pick the entries for encoding. + // It's not the most optimized algorithm for selecting the best representative + // entries over the time. + // It starts from the beginning and makes sure the distance is larger than + // `(end - start) / size` before selecting the number. For example, for the + // following list, pick 3 entries (it will pick seqno #1, #6, #8): + // 1 -> 10 + // 5 -> 17 + // 6 -> 25 + // 8 -> 30 + // first, it always picks the first one, then there are 2 num_entries_to_fill + // and the time difference between current one vs. the last one is + // (30 - 10) = 20. 20/2 = 10. So it will skip until 10+10 = 20. => it skips + // #5 and pick #6. + // But the most optimized solution is picking #1 #5 #8, as it will be more + // evenly distributed for time. Anyway the following algorithm is simple and + // may over-select new data, which is good. We do want more accurate time + // information for recent data. + std::deque output_copy; + if (std::distance(start_it, end_it) > static_cast(output_size)) { + int64_t num_entries_to_fill = static_cast(output_size); + auto last_it = end_it; + last_it--; + uint64_t end_time = last_it->time; + uint64_t skip_until_time = 0; + for (auto it = start_it; it < end_it; it++) { + // skip if it's not reach the skip_until_time yet + if (std::distance(it, end_it) > num_entries_to_fill && + it->time < skip_until_time) { + continue; + } + output_copy.push_back(*it); + num_entries_to_fill--; + if (std::distance(it, end_it) > num_entries_to_fill && + num_entries_to_fill > 0) { + // If there are more entries than we need, re-calculate the + // skip_until_time, which means skip until that time + skip_until_time = + it->time + ((end_time - it->time) / num_entries_to_fill); + } + } + + // Make sure all entries are filled + assert(num_entries_to_fill == 0); + start_it = output_copy.begin(); + end_it = output_copy.end(); + } + + // Delta encode the data + uint64_t size = std::distance(start_it, end_it); + PutVarint64(&dest, size); + SeqnoTimePair base; + for (auto it = start_it; it < end_it; it++) { + assert(base < *it); + SeqnoTimePair val = *it - base; + base = *it; + val.Encode(dest); + } +} + +Status SeqnoToTimeMapping::Add(const std::string& seqno_time_mapping_str) { + Slice input(seqno_time_mapping_str); + if (input.empty()) { + return Status::OK(); + } + uint64_t size; + if (!GetVarint64(&input, &size)) { + return Status::Corruption("Invalid sequence number time size"); + } + is_sorted_ = false; + SeqnoTimePair base; + for (uint64_t i = 0; i < size; i++) { + SeqnoTimePair val; + Status s = val.Decode(input); + if (!s.ok()) { + return s; + } + val.Add(base); + seqno_time_mapping_.emplace_back(val); + base = val; + } + return Status::OK(); +} + +void SeqnoToTimeMapping::SeqnoTimePair::Encode(std::string& dest) const { + PutVarint64Varint64(&dest, seqno, time); +} + +Status SeqnoToTimeMapping::SeqnoTimePair::Decode(Slice& input) { + if (!GetVarint64(&input, &seqno)) { + return Status::Corruption("Invalid sequence number"); + } + if (!GetVarint64(&input, &time)) { + return Status::Corruption("Invalid time"); + } + return Status::OK(); +} + +bool SeqnoToTimeMapping::Append(SequenceNumber seqno, uint64_t time) { + assert(is_sorted_); + + // skip seq number 0, which may have special meaning, like zeroed out data + if (seqno == 0) { + return false; + } + if (!Empty()) { + if (seqno < Last().seqno || time < Last().time) { + return false; + } + if (seqno == Last().seqno) { + Last().time = time; + return true; + } + if (time == Last().time) { + // new sequence has the same time as old one, no need to add new mapping + return false; + } + } + + seqno_time_mapping_.emplace_back(seqno, time); + + if (seqno_time_mapping_.size() > max_capacity_) { + seqno_time_mapping_.pop_front(); + } + return true; +} + +bool SeqnoToTimeMapping::Resize(uint64_t min_time_duration, + uint64_t max_time_duration) { + uint64_t new_max_capacity = + CalculateMaxCapacity(min_time_duration, max_time_duration); + if (new_max_capacity == max_capacity_) { + return false; + } else if (new_max_capacity < seqno_time_mapping_.size()) { + uint64_t delta = seqno_time_mapping_.size() - new_max_capacity; + seqno_time_mapping_.erase(seqno_time_mapping_.begin(), + seqno_time_mapping_.begin() + delta); + } + max_capacity_ = new_max_capacity; + return true; +} + +Status SeqnoToTimeMapping::Sort() { + if (is_sorted_) { + return Status::OK(); + } + if (seqno_time_mapping_.empty()) { + is_sorted_ = true; + return Status::OK(); + } + + std::deque copy = std::move(seqno_time_mapping_); + + std::sort(copy.begin(), copy.end()); + + seqno_time_mapping_.clear(); + + // remove seqno = 0, which may have special meaning, like zeroed out data + while (copy.front().seqno == 0) { + copy.pop_front(); + } + + SeqnoTimePair prev = copy.front(); + for (const auto& it : copy) { + // If sequence number is the same, pick the one with larger time, which is + // more accurate than the older time. + if (it.seqno == prev.seqno) { + assert(it.time >= prev.time); + prev.time = it.time; + } else { + assert(it.seqno > prev.seqno); + // If a larger sequence number has an older time which is not useful, skip + if (it.time > prev.time) { + seqno_time_mapping_.push_back(prev); + prev = it; + } + } + } + seqno_time_mapping_.emplace_back(prev); + + is_sorted_ = true; + return Status::OK(); +} + +std::string SeqnoToTimeMapping::ToHumanString() const { + std::string ret; + for (const auto& seq_time : seqno_time_mapping_) { + AppendNumberTo(&ret, seq_time.seqno); + ret.append("->"); + AppendNumberTo(&ret, seq_time.time); + ret.append(","); + } + return ret; +} + +SeqnoToTimeMapping SeqnoToTimeMapping::Copy( + SequenceNumber smallest_seqno) const { + SeqnoToTimeMapping ret; + auto it = std::upper_bound(seqno_time_mapping_.begin(), + seqno_time_mapping_.end(), smallest_seqno); + if (it != seqno_time_mapping_.begin()) { + it--; + } + std::copy(it, seqno_time_mapping_.end(), + std::back_inserter(ret.seqno_time_mapping_)); + return ret; +} + +uint64_t SeqnoToTimeMapping::CalculateMaxCapacity(uint64_t min_time_duration, + uint64_t max_time_duration) { + if (min_time_duration == 0) { + return 0; + } + return std::min( + kMaxSeqnoToTimeEntries, + max_time_duration * kMaxSeqnoTimePairsPerCF / min_time_duration); +} + +SeqnoToTimeMapping::SeqnoTimePair SeqnoToTimeMapping::SeqnoTimePair::operator-( + const SeqnoTimePair& other) const { + SeqnoTimePair res; + res.seqno = seqno - other.seqno; + res.time = time - other.time; + return res; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/seqno_to_time_mapping.h b/librocksdb-sys/rocksdb/db/seqno_to_time_mapping.h new file mode 100644 index 0000000..4ffc9c1 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/seqno_to_time_mapping.h @@ -0,0 +1,189 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "rocksdb/status.h" +#include "rocksdb/types.h" + +namespace ROCKSDB_NAMESPACE { + +constexpr uint64_t kUnknownSeqnoTime = 0; + +// SeqnoToTimeMapping stores the sequence number to time mapping, so given a +// sequence number it can estimate the oldest possible time for that sequence +// number. For example: +// 10 -> 100 +// 50 -> 300 +// then if a key has seqno 19, the OldestApproximateTime would be 100, for 51 it +// would be 300. +// As it's a sorted list, the new entry is inserted from the back. The old data +// will be popped from the front if they're no longer used. +// +// Note: the data struct is not thread safe, both read and write need to be +// synchronized by caller. +class SeqnoToTimeMapping { + public: + // Maximum number of entries can be encoded into SST. The data is delta encode + // so the maximum data usage for each SST is < 0.3K + static constexpr uint64_t kMaxSeqnoTimePairsPerSST = 100; + + // Maximum number of entries per CF. If there's only CF with this feature on, + // the max duration divided by this number, so for example, if + // preclude_last_level_data_seconds = 100000 (~1day), then it will sample the + // seqno -> time every 1000 seconds (~17minutes). Then the maximum entry it + // needs is 100. + // When there are multiple CFs having this feature on, the sampling cadence is + // determined by the smallest setting, the capacity is determined the largest + // setting, also it's caped by kMaxSeqnoTimePairsPerCF * 10. + static constexpr uint64_t kMaxSeqnoTimePairsPerCF = 100; + + // A simple struct for sequence number to time pair + struct SeqnoTimePair { + SequenceNumber seqno = 0; + uint64_t time = 0; + + SeqnoTimePair() = default; + SeqnoTimePair(SequenceNumber _seqno, uint64_t _time) + : seqno(_seqno), time(_time) {} + + // Encode to dest string + void Encode(std::string& dest) const; + + // Decode the value from input Slice and remove it from the input + Status Decode(Slice& input); + + // subtraction of 2 SeqnoTimePair + SeqnoTimePair operator-(const SeqnoTimePair& other) const; + + // Add 2 values together + void Add(const SeqnoTimePair& obj) { + seqno += obj.seqno; + time += obj.time; + } + + // Compare SeqnoTimePair with a sequence number, used for binary search a + // sequence number in a list of SeqnoTimePair + bool operator<(const SequenceNumber& other) const { return seqno < other; } + + // Compare 2 SeqnoTimePair + bool operator<(const SeqnoTimePair& other) const { + return std::tie(seqno, time) < std::tie(other.seqno, other.time); + } + + // Check if 2 SeqnoTimePair is the same + bool operator==(const SeqnoTimePair& other) const { + return std::tie(seqno, time) == std::tie(other.seqno, other.time); + } + }; + + // constractor of SeqnoToTimeMapping + // max_time_duration is the maximum time it should track. For example, if + // preclude_last_level_data_seconds is 1 day, then if an entry is older than 1 + // day, then it can be removed. + // max_capacity is the maximum number of entry it can hold. For single CF, + // it's caped at 100 (kMaxSeqnoTimePairsPerCF), otherwise + // kMaxSeqnoTimePairsPerCF * 10. + // If it's set to 0, means it won't truncate any old data. + explicit SeqnoToTimeMapping(uint64_t max_time_duration = 0, + uint64_t max_capacity = 0) + : max_time_duration_(max_time_duration), max_capacity_(max_capacity) {} + + // Append a new entry to the list. The new entry should be newer than the + // existing ones. It maintains the internal sorted status. + bool Append(SequenceNumber seqno, uint64_t time); + + // Given a sequence number, estimate it's oldest time + uint64_t GetOldestApproximateTime(SequenceNumber seqno) const; + + // Truncate the old entries based on the current time and max_time_duration_ + void TruncateOldEntries(uint64_t now); + + // Given a time, return it's oldest possible sequence number + SequenceNumber GetOldestSequenceNum(uint64_t time); + + // Encode to a binary string + void Encode(std::string& des, SequenceNumber start, SequenceNumber end, + uint64_t now, + uint64_t output_size = kMaxSeqnoTimePairsPerSST) const; + + // Add a new random entry, unlike Append(), it can be any data, but also makes + // the list un-sorted. + void Add(SequenceNumber seqno, uint64_t time); + + // Decode and add the entries to the current obj. The list will be unsorted + Status Add(const std::string& seqno_time_mapping_str); + + // Return the number of entries + size_t Size() const { return seqno_time_mapping_.size(); } + + // Reduce the size of internal list + bool Resize(uint64_t min_time_duration, uint64_t max_time_duration); + + // Override the max_time_duration_ + void SetMaxTimeDuration(uint64_t max_time_duration) { + max_time_duration_ = max_time_duration; + } + + uint64_t GetCapacity() const { return max_capacity_; } + + // Sort the list, which also remove the redundant entries, useless entries, + // which makes sure the seqno is sorted, but also the time + Status Sort(); + + // copy the current obj from the given smallest_seqno. + SeqnoToTimeMapping Copy(SequenceNumber smallest_seqno) const; + + // If the internal list is empty + bool Empty() const { return seqno_time_mapping_.empty(); } + + // clear all entries + void Clear() { seqno_time_mapping_.clear(); } + + // return the string for user message + // Note: Not efficient, okay for print + std::string ToHumanString() const; + +#ifndef NDEBUG + const std::deque& TEST_GetInternalMapping() const { + return seqno_time_mapping_; + } +#endif + + private: + static constexpr uint64_t kMaxSeqnoToTimeEntries = + kMaxSeqnoTimePairsPerCF * 10; + + uint64_t max_time_duration_; + uint64_t max_capacity_; + + std::deque seqno_time_mapping_; + + bool is_sorted_ = true; + + static uint64_t CalculateMaxCapacity(uint64_t min_time_duration, + uint64_t max_time_duration); + + SeqnoTimePair& Last() { + assert(!Empty()); + return seqno_time_mapping_.back(); + } +}; + +// for searching the sequence number from SeqnoToTimeMapping +inline bool operator<(const SequenceNumber& seqno, + const SeqnoToTimeMapping::SeqnoTimePair& other) { + return seqno < other.seqno; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/snapshot_checker.h b/librocksdb-sys/rocksdb/db/snapshot_checker.h new file mode 100644 index 0000000..b7ff1df --- /dev/null +++ b/librocksdb-sys/rocksdb/db/snapshot_checker.h @@ -0,0 +1,58 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once +#include "rocksdb/types.h" + +namespace ROCKSDB_NAMESPACE { + +enum class SnapshotCheckerResult : int { + kInSnapshot = 0, + kNotInSnapshot = 1, + // In case snapshot is released and the checker has no clue whether + // the given sequence is visible to the snapshot. + kSnapshotReleased = 2, +}; + +// Callback class that control GC of duplicate keys in flush/compaction. +class SnapshotChecker { + public: + virtual ~SnapshotChecker() {} + virtual SnapshotCheckerResult CheckInSnapshot( + SequenceNumber sequence, SequenceNumber snapshot_sequence) const = 0; +}; + +class DisableGCSnapshotChecker : public SnapshotChecker { + public: + virtual ~DisableGCSnapshotChecker() {} + virtual SnapshotCheckerResult CheckInSnapshot( + SequenceNumber /*sequence*/, + SequenceNumber /*snapshot_sequence*/) const override { + // By returning kNotInSnapshot, we prevent all the values from being GCed + return SnapshotCheckerResult::kNotInSnapshot; + } + static DisableGCSnapshotChecker* Instance(); + + protected: + explicit DisableGCSnapshotChecker() {} +}; + +class WritePreparedTxnDB; + +// Callback class created by WritePreparedTxnDB to check if a key +// is visible by a snapshot. +class WritePreparedSnapshotChecker : public SnapshotChecker { + public: + explicit WritePreparedSnapshotChecker(WritePreparedTxnDB* txn_db); + virtual ~WritePreparedSnapshotChecker() {} + + virtual SnapshotCheckerResult CheckInSnapshot( + SequenceNumber sequence, SequenceNumber snapshot_sequence) const override; + + private: + const WritePreparedTxnDB* const txn_db_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/snapshot_impl.cc b/librocksdb-sys/rocksdb/db/snapshot_impl.cc new file mode 100644 index 0000000..98b4754 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/snapshot_impl.cc @@ -0,0 +1,25 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "rocksdb/db.h" +#include "rocksdb/snapshot.h" + +namespace ROCKSDB_NAMESPACE { + +ManagedSnapshot::ManagedSnapshot(DB* db) + : db_(db), snapshot_(db->GetSnapshot()) {} + +ManagedSnapshot::ManagedSnapshot(DB* db, const Snapshot* _snapshot) + : db_(db), snapshot_(_snapshot) {} + +ManagedSnapshot::~ManagedSnapshot() { + if (snapshot_) { + db_->ReleaseSnapshot(snapshot_); + } +} + +const Snapshot* ManagedSnapshot::snapshot() { return snapshot_; } + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/snapshot_impl.h b/librocksdb-sys/rocksdb/db/snapshot_impl.h new file mode 100644 index 0000000..23e5e98 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/snapshot_impl.h @@ -0,0 +1,239 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include + +#include "db/dbformat.h" +#include "rocksdb/db.h" +#include "util/autovector.h" + +namespace ROCKSDB_NAMESPACE { + +class SnapshotList; + +// Snapshots are kept in a doubly-linked list in the DB. +// Each SnapshotImpl corresponds to a particular sequence number. +class SnapshotImpl : public Snapshot { + public: + SequenceNumber number_; // const after creation + // It indicates the smallest uncommitted data at the time the snapshot was + // taken. This is currently used by WritePrepared transactions to limit the + // scope of queries to IsInSnapshot. + SequenceNumber min_uncommitted_ = kMinUnCommittedSeq; + + SequenceNumber GetSequenceNumber() const override { return number_; } + + int64_t GetUnixTime() const override { return unix_time_; } + + uint64_t GetTimestamp() const override { return timestamp_; } + + private: + friend class SnapshotList; + + // SnapshotImpl is kept in a doubly-linked circular list + SnapshotImpl* prev_; + SnapshotImpl* next_; + + SnapshotList* list_; // just for sanity checks + + int64_t unix_time_; + + uint64_t timestamp_; + + // Will this snapshot be used by a Transaction to do write-conflict checking? + bool is_write_conflict_boundary_; +}; + +class SnapshotList { + public: + SnapshotList() { + list_.prev_ = &list_; + list_.next_ = &list_; + list_.number_ = 0xFFFFFFFFL; // placeholder marker, for debugging + // Set all the variables to make UBSAN happy. + list_.list_ = nullptr; + list_.unix_time_ = 0; + list_.timestamp_ = 0; + list_.is_write_conflict_boundary_ = false; + count_ = 0; + } + + // No copy-construct. + SnapshotList(const SnapshotList&) = delete; + + bool empty() const { + assert(list_.next_ != &list_ || 0 == count_); + return list_.next_ == &list_; + } + SnapshotImpl* oldest() const { + assert(!empty()); + return list_.next_; + } + SnapshotImpl* newest() const { + assert(!empty()); + return list_.prev_; + } + + SnapshotImpl* New(SnapshotImpl* s, SequenceNumber seq, uint64_t unix_time, + bool is_write_conflict_boundary, + uint64_t ts = std::numeric_limits::max()) { + s->number_ = seq; + s->unix_time_ = unix_time; + s->timestamp_ = ts; + s->is_write_conflict_boundary_ = is_write_conflict_boundary; + s->list_ = this; + s->next_ = &list_; + s->prev_ = list_.prev_; + s->prev_->next_ = s; + s->next_->prev_ = s; + count_++; + return s; + } + + // Do not responsible to free the object. + void Delete(const SnapshotImpl* s) { + assert(s->list_ == this); + s->prev_->next_ = s->next_; + s->next_->prev_ = s->prev_; + count_--; + } + + // retrieve all snapshot numbers up until max_seq. They are sorted in + // ascending order (with no duplicates). + std::vector GetAll( + SequenceNumber* oldest_write_conflict_snapshot = nullptr, + const SequenceNumber& max_seq = kMaxSequenceNumber) const { + std::vector ret; + GetAll(&ret, oldest_write_conflict_snapshot, max_seq); + return ret; + } + + void GetAll(std::vector* snap_vector, + SequenceNumber* oldest_write_conflict_snapshot = nullptr, + const SequenceNumber& max_seq = kMaxSequenceNumber) const { + std::vector& ret = *snap_vector; + // So far we have no use case that would pass a non-empty vector + assert(ret.size() == 0); + + if (oldest_write_conflict_snapshot != nullptr) { + *oldest_write_conflict_snapshot = kMaxSequenceNumber; + } + + if (empty()) { + return; + } + const SnapshotImpl* s = &list_; + while (s->next_ != &list_) { + if (s->next_->number_ > max_seq) { + break; + } + // Avoid duplicates + if (ret.empty() || ret.back() != s->next_->number_) { + ret.push_back(s->next_->number_); + } + + if (oldest_write_conflict_snapshot != nullptr && + *oldest_write_conflict_snapshot == kMaxSequenceNumber && + s->next_->is_write_conflict_boundary_) { + // If this is the first write-conflict boundary snapshot in the list, + // it is the oldest + *oldest_write_conflict_snapshot = s->next_->number_; + } + + s = s->next_; + } + return; + } + + // get the sequence number of the most recent snapshot + SequenceNumber GetNewest() { + if (empty()) { + return 0; + } + return newest()->number_; + } + + int64_t GetOldestSnapshotTime() const { + if (empty()) { + return 0; + } else { + return oldest()->unix_time_; + } + } + + int64_t GetOldestSnapshotSequence() const { + if (empty()) { + return 0; + } else { + return oldest()->GetSequenceNumber(); + } + } + + uint64_t count() const { return count_; } + + private: + // Dummy head of doubly-linked list of snapshots + SnapshotImpl list_; + uint64_t count_; +}; + +// All operations on TimestampedSnapshotList must be protected by db mutex. +class TimestampedSnapshotList { + public: + explicit TimestampedSnapshotList() = default; + + std::shared_ptr GetSnapshot(uint64_t ts) const { + if (ts == std::numeric_limits::max() && !snapshots_.empty()) { + auto it = snapshots_.rbegin(); + assert(it != snapshots_.rend()); + return it->second; + } + auto it = snapshots_.find(ts); + if (it == snapshots_.end()) { + return std::shared_ptr(); + } + return it->second; + } + + void GetSnapshots( + uint64_t ts_lb, uint64_t ts_ub, + std::vector>& snapshots) const { + assert(ts_lb < ts_ub); + auto it_low = snapshots_.lower_bound(ts_lb); + auto it_high = snapshots_.lower_bound(ts_ub); + for (auto it = it_low; it != it_high; ++it) { + snapshots.emplace_back(it->second); + } + } + + void AddSnapshot(const std::shared_ptr& snapshot) { + assert(snapshot); + snapshots_.try_emplace(snapshot->GetTimestamp(), snapshot); + } + + // snapshots_to_release: the container to where the timestamped snapshots will + // be moved so that it retains the last reference to the snapshots and the + // snapshots won't be actually released which requires db mutex. The + // snapshots will be released by caller of ReleaseSnapshotsOlderThan(). + void ReleaseSnapshotsOlderThan( + uint64_t ts, + autovector>& snapshots_to_release) { + auto ub = snapshots_.lower_bound(ts); + for (auto it = snapshots_.begin(); it != ub; ++it) { + snapshots_to_release.emplace_back(it->second); + } + snapshots_.erase(snapshots_.begin(), ub); + } + + private: + std::map> snapshots_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/table_cache.cc b/librocksdb-sys/rocksdb/db/table_cache.cc new file mode 100644 index 0000000..00224fd --- /dev/null +++ b/librocksdb-sys/rocksdb/db/table_cache.cc @@ -0,0 +1,712 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/table_cache.h" + +#include "db/dbformat.h" +#include "db/range_tombstone_fragmenter.h" +#include "db/snapshot_impl.h" +#include "db/version_edit.h" +#include "file/file_util.h" +#include "file/filename.h" +#include "file/random_access_file_reader.h" +#include "monitoring/perf_context_imp.h" +#include "rocksdb/advanced_options.h" +#include "rocksdb/statistics.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/get_context.h" +#include "table/internal_iterator.h" +#include "table/iterator_wrapper.h" +#include "table/multiget_context.h" +#include "table/table_builder.h" +#include "table/table_reader.h" +#include "test_util/sync_point.h" +#include "util/cast_util.h" +#include "util/coding.h" +#include "util/stop_watch.h" + +// Generate the regular and coroutine versions of some methods by +// including table_cache_sync_and_async.h twice +// Macros in the header will expand differently based on whether +// WITH_COROUTINES or WITHOUT_COROUTINES is defined +// clang-format off +#define WITHOUT_COROUTINES +#include "db/table_cache_sync_and_async.h" +#undef WITHOUT_COROUTINES +#define WITH_COROUTINES +#include "db/table_cache_sync_and_async.h" +#undef WITH_COROUTINES +// clang-format on + +namespace ROCKSDB_NAMESPACE { + +namespace { + +static Slice GetSliceForFileNumber(const uint64_t* file_number) { + return Slice(reinterpret_cast(file_number), + sizeof(*file_number)); +} + + +void AppendVarint64(IterKey* key, uint64_t v) { + char buf[10]; + auto ptr = EncodeVarint64(buf, v); + key->TrimAppend(key->Size(), buf, ptr - buf); +} + + +} // anonymous namespace + +const int kLoadConcurency = 128; + +TableCache::TableCache(const ImmutableOptions& ioptions, + const FileOptions* file_options, Cache* const cache, + BlockCacheTracer* const block_cache_tracer, + const std::shared_ptr& io_tracer, + const std::string& db_session_id) + : ioptions_(ioptions), + file_options_(*file_options), + cache_(cache), + immortal_tables_(false), + block_cache_tracer_(block_cache_tracer), + loader_mutex_(kLoadConcurency), + io_tracer_(io_tracer), + db_session_id_(db_session_id) { + if (ioptions_.row_cache) { + // If the same cache is shared by multiple instances, we need to + // disambiguate its entries. + PutVarint64(&row_cache_id_, ioptions_.row_cache->NewId()); + } +} + +TableCache::~TableCache() {} + +Status TableCache::GetTableReader( + const ReadOptions& ro, const FileOptions& file_options, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, bool sequential_mode, + uint8_t block_protection_bytes_per_key, HistogramImpl* file_read_hist, + std::unique_ptr* table_reader, + const std::shared_ptr& prefix_extractor, + bool skip_filters, int level, bool prefetch_index_and_filter_in_cache, + size_t max_file_size_for_l0_meta_pin, Temperature file_temperature) { + std::string fname = TableFileName( + ioptions_.cf_paths, file_meta.fd.GetNumber(), file_meta.fd.GetPathId()); + std::unique_ptr file; + FileOptions fopts = file_options; + fopts.temperature = file_temperature; + Status s = PrepareIOFromReadOptions(ro, ioptions_.clock, fopts.io_options); + TEST_SYNC_POINT_CALLBACK("TableCache::GetTableReader:BeforeOpenFile", + const_cast(&s)); + if (s.ok()) { + s = ioptions_.fs->NewRandomAccessFile(fname, fopts, &file, nullptr); + } + if (s.ok()) { + RecordTick(ioptions_.stats, NO_FILE_OPENS); + } else if (s.IsPathNotFound()) { + fname = Rocks2LevelTableFileName(fname); + // If this file is also not found, we want to use the error message + // that contains the table file name which is less confusing. + Status temp_s = + PrepareIOFromReadOptions(ro, ioptions_.clock, fopts.io_options); + if (temp_s.ok()) { + temp_s = ioptions_.fs->NewRandomAccessFile(fname, file_options, &file, + nullptr); + } + if (temp_s.ok()) { + RecordTick(ioptions_.stats, NO_FILE_OPENS); + s = temp_s; + } + } + + if (s.ok()) { + if (!sequential_mode && ioptions_.advise_random_on_open) { + file->Hint(FSRandomAccessFile::kRandom); + } + StopWatch sw(ioptions_.clock, ioptions_.stats, TABLE_OPEN_IO_MICROS); + std::unique_ptr file_reader( + new RandomAccessFileReader(std::move(file), fname, ioptions_.clock, + io_tracer_, ioptions_.stats, SST_READ_MICROS, + file_read_hist, ioptions_.rate_limiter.get(), + ioptions_.listeners, file_temperature, + level == ioptions_.num_levels - 1)); + UniqueId64x2 expected_unique_id; + if (ioptions_.verify_sst_unique_id_in_manifest) { + expected_unique_id = file_meta.unique_id; + } else { + expected_unique_id = kNullUniqueId64x2; // null ID == no verification + } + s = ioptions_.table_factory->NewTableReader( + ro, + TableReaderOptions( + ioptions_, prefix_extractor, file_options, internal_comparator, + block_protection_bytes_per_key, skip_filters, immortal_tables_, + false /* force_direct_prefetch */, level, block_cache_tracer_, + max_file_size_for_l0_meta_pin, db_session_id_, + file_meta.fd.GetNumber(), expected_unique_id, + file_meta.fd.largest_seqno, file_meta.tail_size, + file_meta.user_defined_timestamps_persisted), + std::move(file_reader), file_meta.fd.GetFileSize(), table_reader, + prefetch_index_and_filter_in_cache); + TEST_SYNC_POINT("TableCache::GetTableReader:0"); + } + return s; +} + +Status TableCache::FindTable( + const ReadOptions& ro, const FileOptions& file_options, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, TypedHandle** handle, + uint8_t block_protection_bytes_per_key, + const std::shared_ptr& prefix_extractor, + const bool no_io, HistogramImpl* file_read_hist, bool skip_filters, + int level, bool prefetch_index_and_filter_in_cache, + size_t max_file_size_for_l0_meta_pin, Temperature file_temperature) { + PERF_TIMER_GUARD_WITH_CLOCK(find_table_nanos, ioptions_.clock); + uint64_t number = file_meta.fd.GetNumber(); + Slice key = GetSliceForFileNumber(&number); + *handle = cache_.Lookup(key); + TEST_SYNC_POINT_CALLBACK("TableCache::FindTable:0", + const_cast(&no_io)); + + if (*handle == nullptr) { + if (no_io) { + return Status::Incomplete("Table not found in table_cache, no_io is set"); + } + MutexLock load_lock(&loader_mutex_.Get(key)); + // We check the cache again under loading mutex + *handle = cache_.Lookup(key); + if (*handle != nullptr) { + return Status::OK(); + } + + std::unique_ptr table_reader; + Status s = GetTableReader(ro, file_options, internal_comparator, file_meta, + false /* sequential mode */, + block_protection_bytes_per_key, file_read_hist, + &table_reader, prefix_extractor, skip_filters, + level, prefetch_index_and_filter_in_cache, + max_file_size_for_l0_meta_pin, file_temperature); + if (!s.ok()) { + assert(table_reader == nullptr); + RecordTick(ioptions_.stats, NO_FILE_ERRORS); + // We do not cache error results so that if the error is transient, + // or somebody repairs the file, we recover automatically. + } else { + s = cache_.Insert(key, table_reader.get(), 1, handle); + if (s.ok()) { + // Release ownership of table reader. + table_reader.release(); + } + } + return s; + } + return Status::OK(); +} + +InternalIterator* TableCache::NewIterator( + const ReadOptions& options, const FileOptions& file_options, + const InternalKeyComparator& icomparator, const FileMetaData& file_meta, + RangeDelAggregator* range_del_agg, + const std::shared_ptr& prefix_extractor, + TableReader** table_reader_ptr, HistogramImpl* file_read_hist, + TableReaderCaller caller, Arena* arena, bool skip_filters, int level, + size_t max_file_size_for_l0_meta_pin, + const InternalKey* smallest_compaction_key, + const InternalKey* largest_compaction_key, bool allow_unprepared_value, + uint8_t block_protection_bytes_per_key, + TruncatedRangeDelIterator** range_del_iter) { + PERF_TIMER_GUARD(new_table_iterator_nanos); + + Status s; + TableReader* table_reader = nullptr; + TypedHandle* handle = nullptr; + if (table_reader_ptr != nullptr) { + *table_reader_ptr = nullptr; + } + bool for_compaction = caller == TableReaderCaller::kCompaction; + auto& fd = file_meta.fd; + table_reader = fd.table_reader; + if (table_reader == nullptr) { + s = FindTable(options, file_options, icomparator, file_meta, &handle, + block_protection_bytes_per_key, prefix_extractor, + options.read_tier == kBlockCacheTier /* no_io */, + file_read_hist, skip_filters, level, + true /* prefetch_index_and_filter_in_cache */, + max_file_size_for_l0_meta_pin, file_meta.temperature); + if (s.ok()) { + table_reader = cache_.Value(handle); + } + } + InternalIterator* result = nullptr; + if (s.ok()) { + if (options.table_filter && + !options.table_filter(*table_reader->GetTableProperties())) { + result = NewEmptyInternalIterator(arena); + } else { + result = table_reader->NewIterator( + options, prefix_extractor.get(), arena, skip_filters, caller, + file_options.compaction_readahead_size, allow_unprepared_value); + } + if (handle != nullptr) { + cache_.RegisterReleaseAsCleanup(handle, *result); + handle = nullptr; // prevent from releasing below + } + + if (for_compaction) { + table_reader->SetupForCompaction(); + } + if (table_reader_ptr != nullptr) { + *table_reader_ptr = table_reader; + } + } + if (s.ok() && !options.ignore_range_deletions) { + if (range_del_iter != nullptr) { + auto new_range_del_iter = + table_reader->NewRangeTombstoneIterator(options); + if (new_range_del_iter == nullptr || new_range_del_iter->empty()) { + delete new_range_del_iter; + *range_del_iter = nullptr; + } else { + *range_del_iter = new TruncatedRangeDelIterator( + std::unique_ptr( + new_range_del_iter), + &icomparator, &file_meta.smallest, &file_meta.largest); + } + } + if (range_del_agg != nullptr) { + if (range_del_agg->AddFile(fd.GetNumber())) { + std::unique_ptr new_range_del_iter( + static_cast( + table_reader->NewRangeTombstoneIterator(options))); + if (new_range_del_iter != nullptr) { + s = new_range_del_iter->status(); + } + if (s.ok()) { + const InternalKey* smallest = &file_meta.smallest; + const InternalKey* largest = &file_meta.largest; + if (smallest_compaction_key != nullptr) { + smallest = smallest_compaction_key; + } + if (largest_compaction_key != nullptr) { + largest = largest_compaction_key; + } + range_del_agg->AddTombstones(std::move(new_range_del_iter), smallest, + largest); + } + } + } + } + + if (handle != nullptr) { + cache_.Release(handle); + } + if (!s.ok()) { + assert(result == nullptr); + result = NewErrorInternalIterator(s, arena); + } + return result; +} + +Status TableCache::GetRangeTombstoneIterator( + const ReadOptions& options, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, uint8_t block_protection_bytes_per_key, + std::unique_ptr* out_iter) { + assert(out_iter); + const FileDescriptor& fd = file_meta.fd; + Status s; + TableReader* t = fd.table_reader; + TypedHandle* handle = nullptr; + if (t == nullptr) { + s = FindTable(options, file_options_, internal_comparator, file_meta, + &handle, block_protection_bytes_per_key); + if (s.ok()) { + t = cache_.Value(handle); + } + } + if (s.ok()) { + // Note: NewRangeTombstoneIterator could return nullptr + out_iter->reset(t->NewRangeTombstoneIterator(options)); + } + if (handle) { + if (*out_iter) { + cache_.RegisterReleaseAsCleanup(handle, **out_iter); + } else { + cache_.Release(handle); + } + } + return s; +} + +void TableCache::CreateRowCacheKeyPrefix(const ReadOptions& options, + const FileDescriptor& fd, + const Slice& internal_key, + GetContext* get_context, + IterKey& row_cache_key) { + uint64_t fd_number = fd.GetNumber(); + // We use the user key as cache key instead of the internal key, + // otherwise the whole cache would be invalidated every time the + // sequence key increases. However, to support caching snapshot + // reads, we append the sequence number (incremented by 1 to + // distinguish from 0) only in this case. + // If the snapshot is larger than the largest seqno in the file, + // all data should be exposed to the snapshot, so we treat it + // the same as there is no snapshot. The exception is that if + // a seq-checking callback is registered, some internal keys + // may still be filtered out. + uint64_t seq_no = 0; + // Maybe we can include the whole file ifsnapshot == fd.largest_seqno. + if (options.snapshot != nullptr && + (get_context->has_callback() || + static_cast_with_check(options.snapshot) + ->GetSequenceNumber() <= fd.largest_seqno)) { + // We should consider to use options.snapshot->GetSequenceNumber() + // instead of GetInternalKeySeqno(k), which will make the code + // easier to understand. + seq_no = 1 + GetInternalKeySeqno(internal_key); + } + + // Compute row cache key. + row_cache_key.TrimAppend(row_cache_key.Size(), row_cache_id_.data(), + row_cache_id_.size()); + AppendVarint64(&row_cache_key, fd_number); + AppendVarint64(&row_cache_key, seq_no); +} + +bool TableCache::GetFromRowCache(const Slice& user_key, IterKey& row_cache_key, + size_t prefix_size, GetContext* get_context) { + bool found = false; + + row_cache_key.TrimAppend(prefix_size, user_key.data(), user_key.size()); + RowCacheInterface row_cache{ioptions_.row_cache.get()}; + if (auto row_handle = row_cache.Lookup(row_cache_key.GetUserKey())) { + // Cleanable routine to release the cache entry + Cleanable value_pinner; + // If it comes here value is located on the cache. + // found_row_cache_entry points to the value on cache, + // and value_pinner has cleanup procedure for the cached entry. + // After replayGetContextLog() returns, get_context.pinnable_slice_ + // will point to cache entry buffer (or a copy based on that) and + // cleanup routine under value_pinner will be delegated to + // get_context.pinnable_slice_. Cache entry is released when + // get_context.pinnable_slice_ is reset. + row_cache.RegisterReleaseAsCleanup(row_handle, value_pinner); + replayGetContextLog(*row_cache.Value(row_handle), user_key, get_context, + &value_pinner); + RecordTick(ioptions_.stats, ROW_CACHE_HIT); + found = true; + } else { + RecordTick(ioptions_.stats, ROW_CACHE_MISS); + } + return found; +} + +Status TableCache::Get( + const ReadOptions& options, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, const Slice& k, GetContext* get_context, + uint8_t block_protection_bytes_per_key, + const std::shared_ptr& prefix_extractor, + HistogramImpl* file_read_hist, bool skip_filters, int level, + size_t max_file_size_for_l0_meta_pin) { + auto& fd = file_meta.fd; + std::string* row_cache_entry = nullptr; + bool done = false; + IterKey row_cache_key; + std::string row_cache_entry_buffer; + + // Check row cache if enabled. Since row cache does not currently store + // sequence numbers, we cannot use it if we need to fetch the sequence. + if (ioptions_.row_cache && !get_context->NeedToReadSequence()) { + auto user_key = ExtractUserKey(k); + CreateRowCacheKeyPrefix(options, fd, k, get_context, row_cache_key); + done = GetFromRowCache(user_key, row_cache_key, row_cache_key.Size(), + get_context); + if (!done) { + row_cache_entry = &row_cache_entry_buffer; + } + } + Status s; + TableReader* t = fd.table_reader; + TypedHandle* handle = nullptr; + if (!done) { + assert(s.ok()); + if (t == nullptr) { + s = FindTable(options, file_options_, internal_comparator, file_meta, + &handle, block_protection_bytes_per_key, prefix_extractor, + options.read_tier == kBlockCacheTier /* no_io */, + file_read_hist, skip_filters, level, + true /* prefetch_index_and_filter_in_cache */, + max_file_size_for_l0_meta_pin, file_meta.temperature); + if (s.ok()) { + t = cache_.Value(handle); + } + } + SequenceNumber* max_covering_tombstone_seq = + get_context->max_covering_tombstone_seq(); + if (s.ok() && max_covering_tombstone_seq != nullptr && + !options.ignore_range_deletions) { + std::unique_ptr range_del_iter( + t->NewRangeTombstoneIterator(options)); + if (range_del_iter != nullptr) { + SequenceNumber seq = + range_del_iter->MaxCoveringTombstoneSeqnum(ExtractUserKey(k)); + if (seq > *max_covering_tombstone_seq) { + *max_covering_tombstone_seq = seq; + if (get_context->NeedTimestamp()) { + get_context->SetTimestampFromRangeTombstone( + range_del_iter->timestamp()); + } + } + } + } + if (s.ok()) { + get_context->SetReplayLog(row_cache_entry); // nullptr if no cache. + s = t->Get(options, k, get_context, prefix_extractor.get(), skip_filters); + get_context->SetReplayLog(nullptr); + } else if (options.read_tier == kBlockCacheTier && s.IsIncomplete()) { + // Couldn't find Table in cache but treat as kFound if no_io set + get_context->MarkKeyMayExist(); + s = Status::OK(); + done = true; + } + } + + // Put the replay log in row cache only if something was found. + if (!done && s.ok() && row_cache_entry && !row_cache_entry->empty()) { + RowCacheInterface row_cache{ioptions_.row_cache.get()}; + size_t charge = row_cache_entry->capacity() + sizeof(std::string); + auto row_ptr = new std::string(std::move(*row_cache_entry)); + // If row cache is full, it's OK to continue. + row_cache.Insert(row_cache_key.GetUserKey(), row_ptr, charge) + .PermitUncheckedError(); + } + + if (handle != nullptr) { + cache_.Release(handle); + } + return s; +} + +void TableCache::UpdateRangeTombstoneSeqnums( + const ReadOptions& options, TableReader* t, + MultiGetContext::Range& table_range) { + std::unique_ptr range_del_iter( + t->NewRangeTombstoneIterator(options)); + if (range_del_iter != nullptr) { + for (auto iter = table_range.begin(); iter != table_range.end(); ++iter) { + SequenceNumber* max_covering_tombstone_seq = + iter->get_context->max_covering_tombstone_seq(); + SequenceNumber seq = + range_del_iter->MaxCoveringTombstoneSeqnum(iter->ukey_with_ts); + if (seq > *max_covering_tombstone_seq) { + *max_covering_tombstone_seq = seq; + if (iter->get_context->NeedTimestamp()) { + iter->get_context->SetTimestampFromRangeTombstone( + range_del_iter->timestamp()); + } + } + } + } +} + +Status TableCache::MultiGetFilter( + const ReadOptions& options, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, + const std::shared_ptr& prefix_extractor, + HistogramImpl* file_read_hist, int level, + MultiGetContext::Range* mget_range, TypedHandle** table_handle, + uint8_t block_protection_bytes_per_key) { + auto& fd = file_meta.fd; + IterKey row_cache_key; + std::string row_cache_entry_buffer; + + // Check if we need to use the row cache. If yes, then we cannot do the + // filtering here, since the filtering needs to happen after the row cache + // lookup. + KeyContext& first_key = *mget_range->begin(); + if (ioptions_.row_cache && !first_key.get_context->NeedToReadSequence()) { + return Status::NotSupported(); + } + Status s; + TableReader* t = fd.table_reader; + TypedHandle* handle = nullptr; + MultiGetContext::Range tombstone_range(*mget_range, mget_range->begin(), + mget_range->end()); + if (t == nullptr) { + s = FindTable(options, file_options_, internal_comparator, file_meta, + &handle, block_protection_bytes_per_key, prefix_extractor, + options.read_tier == kBlockCacheTier /* no_io */, + file_read_hist, + /*skip_filters=*/false, level, + true /* prefetch_index_and_filter_in_cache */, + /*max_file_size_for_l0_meta_pin=*/0, file_meta.temperature); + if (s.ok()) { + t = cache_.Value(handle); + } + *table_handle = handle; + } + if (s.ok()) { + s = t->MultiGetFilter(options, prefix_extractor.get(), mget_range); + } + if (s.ok() && !options.ignore_range_deletions) { + // Update the range tombstone sequence numbers for the keys here + // as TableCache::MultiGet may or may not be called, and even if it + // is, it may be called with fewer keys in the rangedue to filtering. + UpdateRangeTombstoneSeqnums(options, t, tombstone_range); + } + if (mget_range->empty() && handle) { + cache_.Release(handle); + *table_handle = nullptr; + } + + return s; +} + +Status TableCache::GetTableProperties( + const FileOptions& file_options, const ReadOptions& read_options, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, + std::shared_ptr* properties, + uint8_t block_protection_bytes_per_key, + const std::shared_ptr& prefix_extractor, bool no_io) { + auto table_reader = file_meta.fd.table_reader; + // table already been pre-loaded? + if (table_reader) { + *properties = table_reader->GetTableProperties(); + + return Status::OK(); + } + + TypedHandle* table_handle = nullptr; + Status s = FindTable(read_options, file_options, internal_comparator, + file_meta, &table_handle, block_protection_bytes_per_key, + prefix_extractor, no_io); + if (!s.ok()) { + return s; + } + assert(table_handle); + auto table = cache_.Value(table_handle); + *properties = table->GetTableProperties(); + cache_.Release(table_handle); + return s; +} + +Status TableCache::ApproximateKeyAnchors( + const ReadOptions& ro, const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, uint8_t block_protection_bytes_per_key, + std::vector& anchors) { + Status s; + TableReader* t = file_meta.fd.table_reader; + TypedHandle* handle = nullptr; + if (t == nullptr) { + s = FindTable(ro, file_options_, internal_comparator, file_meta, &handle, + block_protection_bytes_per_key); + if (s.ok()) { + t = cache_.Value(handle); + } + } + if (s.ok() && t != nullptr) { + s = t->ApproximateKeyAnchors(ro, anchors); + } + if (handle != nullptr) { + cache_.Release(handle); + } + return s; +} + +size_t TableCache::GetMemoryUsageByTableReader( + const FileOptions& file_options, const ReadOptions& read_options, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, uint8_t block_protection_bytes_per_key, + const std::shared_ptr& prefix_extractor) { + auto table_reader = file_meta.fd.table_reader; + // table already been pre-loaded? + if (table_reader) { + return table_reader->ApproximateMemoryUsage(); + } + + TypedHandle* table_handle = nullptr; + Status s = FindTable(read_options, file_options, internal_comparator, + file_meta, &table_handle, block_protection_bytes_per_key, + prefix_extractor, true /* no_io */); + if (!s.ok()) { + return 0; + } + assert(table_handle); + auto table = cache_.Value(table_handle); + auto ret = table->ApproximateMemoryUsage(); + cache_.Release(table_handle); + return ret; +} + +void TableCache::Evict(Cache* cache, uint64_t file_number) { + cache->Erase(GetSliceForFileNumber(&file_number)); +} + +uint64_t TableCache::ApproximateOffsetOf( + const ReadOptions& read_options, const Slice& key, + const FileMetaData& file_meta, TableReaderCaller caller, + const InternalKeyComparator& internal_comparator, + uint8_t block_protection_bytes_per_key, + const std::shared_ptr& prefix_extractor) { + uint64_t result = 0; + TableReader* table_reader = file_meta.fd.table_reader; + TypedHandle* table_handle = nullptr; + if (table_reader == nullptr) { + Status s = + FindTable(read_options, file_options_, internal_comparator, file_meta, + &table_handle, block_protection_bytes_per_key, + prefix_extractor, false /* no_io */); + if (s.ok()) { + table_reader = cache_.Value(table_handle); + } + } + + if (table_reader != nullptr) { + result = table_reader->ApproximateOffsetOf(read_options, key, caller); + } + if (table_handle != nullptr) { + cache_.Release(table_handle); + } + + return result; +} + +uint64_t TableCache::ApproximateSize( + const ReadOptions& read_options, const Slice& start, const Slice& end, + const FileMetaData& file_meta, TableReaderCaller caller, + const InternalKeyComparator& internal_comparator, + uint8_t block_protection_bytes_per_key, + const std::shared_ptr& prefix_extractor) { + uint64_t result = 0; + TableReader* table_reader = file_meta.fd.table_reader; + TypedHandle* table_handle = nullptr; + if (table_reader == nullptr) { + Status s = + FindTable(read_options, file_options_, internal_comparator, file_meta, + &table_handle, block_protection_bytes_per_key, + prefix_extractor, false /* no_io */); + if (s.ok()) { + table_reader = cache_.Value(table_handle); + } + } + + if (table_reader != nullptr) { + result = table_reader->ApproximateSize(read_options, start, end, caller); + } + if (table_handle != nullptr) { + cache_.Release(table_handle); + } + + return result; +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/table_cache.h b/librocksdb-sys/rocksdb/db/table_cache.h new file mode 100644 index 0000000..39e41cc --- /dev/null +++ b/librocksdb-sys/rocksdb/db/table_cache.h @@ -0,0 +1,283 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Thread-safe (provides internal synchronization) + +#pragma once +#include +#include +#include + +#include "cache/typed_cache.h" +#include "db/dbformat.h" +#include "db/range_del_aggregator.h" +#include "options/cf_options.h" +#include "port/port.h" +#include "rocksdb/cache.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "rocksdb/table.h" +#include "table/table_reader.h" +#include "trace_replay/block_cache_tracer.h" +#include "util/coro_utils.h" + +namespace ROCKSDB_NAMESPACE { + +class Env; +class Arena; +struct FileDescriptor; +class GetContext; +class HistogramImpl; + +// Manages caching for TableReader objects for a column family. The actual +// cache is allocated separately and passed to the constructor. TableCache +// wraps around the underlying SST file readers by providing Get(), +// MultiGet() and NewIterator() methods that hide the instantiation, +// caching and access to the TableReader. The main purpose of this is +// performance - by caching the TableReader, it avoids unnecessary file opens +// and object allocation and instantiation. One exception is compaction, where +// a new TableReader may be instantiated - see NewIterator() comments +// +// Another service provided by TableCache is managing the row cache - if the +// DB is configured with a row cache, and the lookup key is present in the row +// cache, lookup is very fast. The row cache is obtained from +// ioptions.row_cache +class TableCache { + public: + TableCache(const ImmutableOptions& ioptions, + const FileOptions* storage_options, Cache* cache, + BlockCacheTracer* const block_cache_tracer, + const std::shared_ptr& io_tracer, + const std::string& db_session_id); + ~TableCache(); + + // Cache interface for table cache + using CacheInterface = + BasicTypedCacheInterface; + using TypedHandle = CacheInterface::TypedHandle; + + // Cache interface for row cache + using RowCacheInterface = + BasicTypedCacheInterface; + using RowHandle = RowCacheInterface::TypedHandle; + + // Return an iterator for the specified file number (the corresponding + // file length must be exactly "file_size" bytes). If "table_reader_ptr" + // is non-nullptr, also sets "*table_reader_ptr" to point to the Table object + // underlying the returned iterator, or nullptr if no Table object underlies + // the returned iterator. The returned "*table_reader_ptr" object is owned + // by the cache and should not be deleted, and is valid for as long as the + // returned iterator is live. + // If !options.ignore_range_deletions, and range_del_iter is non-nullptr, + // then range_del_iter is set to a TruncatedRangeDelIterator for range + // tombstones in the SST file corresponding to the specified file number. The + // upper/lower bounds for the TruncatedRangeDelIterator are set to the SST + // file's boundary. + // @param options Must outlive the returned iterator. + // @param range_del_agg If non-nullptr, adds range deletions to the + // aggregator. If an error occurs, returns it in a NewErrorInternalIterator + // @param for_compaction If true, a new TableReader may be allocated (but + // not cached), depending on the CF options + // @param skip_filters Disables loading/accessing the filter block + // @param level The level this table is at, -1 for "not set / don't know" + InternalIterator* NewIterator( + const ReadOptions& options, const FileOptions& toptions, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, RangeDelAggregator* range_del_agg, + const std::shared_ptr& prefix_extractor, + TableReader** table_reader_ptr, HistogramImpl* file_read_hist, + TableReaderCaller caller, Arena* arena, bool skip_filters, int level, + size_t max_file_size_for_l0_meta_pin, + const InternalKey* smallest_compaction_key, + const InternalKey* largest_compaction_key, bool allow_unprepared_value, + uint8_t protection_bytes_per_key, + TruncatedRangeDelIterator** range_del_iter = nullptr); + + // If a seek to internal key "k" in specified file finds an entry, + // call get_context->SaveValue() repeatedly until + // it returns false. As a side effect, it will insert the TableReader + // into the cache and potentially evict another entry + // @param get_context Context for get operation. The result of the lookup + // can be retrieved by calling get_context->State() + // @param file_read_hist If non-nullptr, the file reader statistics are + // recorded + // @param skip_filters Disables loading/accessing the filter block + // @param level The level this table is at, -1 for "not set / don't know" + Status Get( + const ReadOptions& options, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, const Slice& k, GetContext* get_context, + uint8_t block_protection_bytes_per_key, + const std::shared_ptr& prefix_extractor = nullptr, + HistogramImpl* file_read_hist = nullptr, bool skip_filters = false, + int level = -1, size_t max_file_size_for_l0_meta_pin = 0); + + // Return the range delete tombstone iterator of the file specified by + // `file_meta`. + Status GetRangeTombstoneIterator( + const ReadOptions& options, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, uint8_t block_protection_bytes_per_key, + std::unique_ptr* out_iter); + + // Call table reader's MultiGetFilter to use the bloom filter to filter out + // keys. Returns Status::NotSupported() if row cache needs to be checked. + // If the table cache is looked up to get the table reader, the cache handle + // is returned in table_handle. This handle should be passed back to + // MultiGet() so it can be released. + Status MultiGetFilter( + const ReadOptions& options, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, + const std::shared_ptr& prefix_extractor, + HistogramImpl* file_read_hist, int level, + MultiGetContext::Range* mget_range, TypedHandle** table_handle, + uint8_t block_protection_bytes_per_key); + + // If a seek to internal key "k" in specified file finds an entry, + // call get_context->SaveValue() repeatedly until + // it returns false. As a side effect, it will insert the TableReader + // into the cache and potentially evict another entry + // @param mget_range Pointer to the structure describing a batch of keys to + // be looked up in this table file. The result is stored + // in the embedded GetContext + // @param skip_filters Disables loading/accessing the filter block + // @param level The level this table is at, -1 for "not set / don't know" + DECLARE_SYNC_AND_ASYNC( + Status, MultiGet, const ReadOptions& options, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, const MultiGetContext::Range* mget_range, + uint8_t block_protection_bytes_per_key, + const std::shared_ptr& prefix_extractor = nullptr, + HistogramImpl* file_read_hist = nullptr, bool skip_filters = false, + bool skip_range_deletions = false, int level = -1, + TypedHandle* table_handle = nullptr); + + // Evict any entry for the specified file number + static void Evict(Cache* cache, uint64_t file_number); + + // Find table reader + // @param skip_filters Disables loading/accessing the filter block + // @param level == -1 means not specified + Status FindTable( + const ReadOptions& ro, const FileOptions& toptions, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, TypedHandle**, + uint8_t block_protection_bytes_per_key, + const std::shared_ptr& prefix_extractor = nullptr, + const bool no_io = false, HistogramImpl* file_read_hist = nullptr, + bool skip_filters = false, int level = -1, + bool prefetch_index_and_filter_in_cache = true, + size_t max_file_size_for_l0_meta_pin = 0, + Temperature file_temperature = Temperature::kUnknown); + + // Get the table properties of a given table. + // @no_io: indicates if we should load table to the cache if it is not present + // in table cache yet. + // @returns: `properties` will be reset on success. Please note that we will + // return Status::Incomplete() if table is not present in cache and + // we set `no_io` to be true. + Status GetTableProperties( + const FileOptions& toptions, const ReadOptions& read_options, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, + std::shared_ptr* properties, + uint8_t block_protection_bytes_per_key, + const std::shared_ptr& prefix_extractor = nullptr, + bool no_io = false); + + Status ApproximateKeyAnchors(const ReadOptions& ro, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, + uint8_t block_protection_bytes_per_key, + std::vector& anchors); + + // Return total memory usage of the table reader of the file. + // 0 if table reader of the file is not loaded. + size_t GetMemoryUsageByTableReader( + const FileOptions& toptions, const ReadOptions& read_options, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, uint8_t block_protection_bytes_per_key, + const std::shared_ptr& prefix_extractor = nullptr); + + // Returns approximated offset of a key in a file represented by fd. + uint64_t ApproximateOffsetOf( + const ReadOptions& read_options, const Slice& key, + const FileMetaData& file_meta, TableReaderCaller caller, + const InternalKeyComparator& internal_comparator, + uint8_t block_protection_bytes_per_key, + const std::shared_ptr& prefix_extractor = nullptr); + + // Returns approximated data size between start and end keys in a file + // represented by fd (the start key must not be greater than the end key). + uint64_t ApproximateSize( + const ReadOptions& read_options, const Slice& start, const Slice& end, + const FileMetaData& file_meta, TableReaderCaller caller, + const InternalKeyComparator& internal_comparator, + uint8_t block_protection_bytes_per_key, + const std::shared_ptr& prefix_extractor = nullptr); + + CacheInterface& get_cache() { return cache_; } + + // Capacity of the backing Cache that indicates infinite TableCache capacity. + // For example when max_open_files is -1 we set the backing Cache to this. + static const int kInfiniteCapacity = 0x400000; + + // The tables opened with this TableCache will be immortal, i.e., their + // lifetime is as long as that of the DB. + void SetTablesAreImmortal() { + if (cache_.get()->GetCapacity() >= kInfiniteCapacity) { + immortal_tables_ = true; + } + } + + private: + // Build a table reader + Status GetTableReader( + const ReadOptions& ro, const FileOptions& file_options, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, bool sequential_mode, + uint8_t block_protection_bytes_per_key, HistogramImpl* file_read_hist, + std::unique_ptr* table_reader, + const std::shared_ptr& prefix_extractor = nullptr, + bool skip_filters = false, int level = -1, + bool prefetch_index_and_filter_in_cache = true, + size_t max_file_size_for_l0_meta_pin = 0, + Temperature file_temperature = Temperature::kUnknown); + + // Update the max_covering_tombstone_seq in the GetContext for each key based + // on the range deletions in the table + void UpdateRangeTombstoneSeqnums(const ReadOptions& options, TableReader* t, + MultiGetContext::Range& table_range); + + // Create a key prefix for looking up the row cache. The prefix is of the + // format row_cache_id + fd_number + seq_no. Later, the user key can be + // appended to form the full key + void CreateRowCacheKeyPrefix(const ReadOptions& options, + const FileDescriptor& fd, + const Slice& internal_key, + GetContext* get_context, IterKey& row_cache_key); + + // Helper function to lookup the row cache for a key. It appends the + // user key to row_cache_key at offset prefix_size + bool GetFromRowCache(const Slice& user_key, IterKey& row_cache_key, + size_t prefix_size, GetContext* get_context); + + const ImmutableOptions& ioptions_; + const FileOptions& file_options_; + CacheInterface cache_; + std::string row_cache_id_; + bool immortal_tables_; + BlockCacheTracer* const block_cache_tracer_; + Striped> loader_mutex_; + std::shared_ptr io_tracer_; + std::string db_session_id_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/table_cache_sync_and_async.h b/librocksdb-sys/rocksdb/db/table_cache_sync_and_async.h new file mode 100644 index 0000000..8ff03ec --- /dev/null +++ b/librocksdb-sys/rocksdb/db/table_cache_sync_and_async.h @@ -0,0 +1,130 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "util/coro_utils.h" + +#if defined(WITHOUT_COROUTINES) || \ + (defined(USE_COROUTINES) && defined(WITH_COROUTINES)) +namespace ROCKSDB_NAMESPACE { + +#if defined(WITHOUT_COROUTINES) +#endif + +// Batched version of TableCache::MultiGet. +DEFINE_SYNC_AND_ASYNC(Status, TableCache::MultiGet) +(const ReadOptions& options, const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, const MultiGetContext::Range* mget_range, + uint8_t block_protection_bytes_per_key, + const std::shared_ptr& prefix_extractor, + HistogramImpl* file_read_hist, bool skip_filters, bool skip_range_deletions, + int level, TypedHandle* handle) { + auto& fd = file_meta.fd; + Status s; + TableReader* t = fd.table_reader; + MultiGetRange table_range(*mget_range, mget_range->begin(), + mget_range->end()); + if (handle != nullptr && t == nullptr) { + t = cache_.Value(handle); + } + autovector row_cache_entries; + IterKey row_cache_key; + size_t row_cache_key_prefix_size = 0; + KeyContext& first_key = *table_range.begin(); + bool lookup_row_cache = + ioptions_.row_cache && !first_key.get_context->NeedToReadSequence(); + + // Check row cache if enabled. Since row cache does not currently store + // sequence numbers, we cannot use it if we need to fetch the sequence. + if (lookup_row_cache) { + GetContext* first_context = first_key.get_context; + CreateRowCacheKeyPrefix(options, fd, first_key.ikey, first_context, + row_cache_key); + row_cache_key_prefix_size = row_cache_key.Size(); + + for (auto miter = table_range.begin(); miter != table_range.end(); + ++miter) { + const Slice& user_key = miter->ukey_with_ts; + + GetContext* get_context = miter->get_context; + + if (GetFromRowCache(user_key, row_cache_key, row_cache_key_prefix_size, + get_context)) { + table_range.SkipKey(miter); + } else { + row_cache_entries.emplace_back(); + get_context->SetReplayLog(&(row_cache_entries.back())); + } + } + } + + // Check that table_range is not empty. Its possible all keys may have been + // found in the row cache and thus the range may now be empty + if (s.ok() && !table_range.empty()) { + if (t == nullptr) { + assert(handle == nullptr); + s = FindTable(options, file_options_, internal_comparator, file_meta, + &handle, block_protection_bytes_per_key, prefix_extractor, + options.read_tier == kBlockCacheTier /* no_io */, + file_read_hist, skip_filters, level, + true /* prefetch_index_and_filter_in_cache */, + 0 /*max_file_size_for_l0_meta_pin*/, file_meta.temperature); + TEST_SYNC_POINT_CALLBACK("TableCache::MultiGet:FindTable", &s); + if (s.ok()) { + t = cache_.Value(handle); + assert(t); + } + } + if (s.ok() && !options.ignore_range_deletions && !skip_range_deletions) { + UpdateRangeTombstoneSeqnums(options, t, table_range); + } + if (s.ok()) { + CO_AWAIT(t->MultiGet) + (options, &table_range, prefix_extractor.get(), skip_filters); + } else if (options.read_tier == kBlockCacheTier && s.IsIncomplete()) { + for (auto iter = table_range.begin(); iter != table_range.end(); ++iter) { + Status* status = iter->s; + if (status->IsIncomplete()) { + // Couldn't find Table in cache but treat as kFound if no_io set + iter->get_context->MarkKeyMayExist(); + s = Status::OK(); + } + } + } + } + + if (lookup_row_cache) { + size_t row_idx = 0; + RowCacheInterface row_cache{ioptions_.row_cache.get()}; + + for (auto miter = table_range.begin(); miter != table_range.end(); + ++miter) { + std::string& row_cache_entry = row_cache_entries[row_idx++]; + const Slice& user_key = miter->ukey_with_ts; + ; + GetContext* get_context = miter->get_context; + + get_context->SetReplayLog(nullptr); + // Compute row cache key. + row_cache_key.TrimAppend(row_cache_key_prefix_size, user_key.data(), + user_key.size()); + // Put the replay log in row cache only if something was found. + if (s.ok() && !row_cache_entry.empty()) { + size_t charge = row_cache_entry.capacity() + sizeof(std::string); + auto row_ptr = new std::string(std::move(row_cache_entry)); + // If row cache is full, it's OK. + row_cache.Insert(row_cache_key.GetUserKey(), row_ptr, charge) + .PermitUncheckedError(); + } + } + } + + if (handle != nullptr) { + cache_.Release(handle); + } + CO_RETURN s; +} +} // namespace ROCKSDB_NAMESPACE +#endif diff --git a/librocksdb-sys/rocksdb/db/table_properties_collector.cc b/librocksdb-sys/rocksdb/db/table_properties_collector.cc new file mode 100644 index 0000000..edb9a1b --- /dev/null +++ b/librocksdb-sys/rocksdb/db/table_properties_collector.cc @@ -0,0 +1,74 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/table_properties_collector.h" + +#include "db/dbformat.h" +#include "util/coding.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { + +uint64_t GetUint64Property(const UserCollectedProperties& props, + const std::string& property_name, + bool* property_present) { + auto pos = props.find(property_name); + if (pos == props.end()) { + *property_present = false; + return 0; + } + Slice raw = pos->second; + uint64_t val = 0; + *property_present = true; + return GetVarint64(&raw, &val) ? val : 0; +} + +} // anonymous namespace + +Status UserKeyTablePropertiesCollector::InternalAdd(const Slice& key, + const Slice& value, + uint64_t file_size) { + ParsedInternalKey ikey; + Status s = ParseInternalKey(key, &ikey, false /* log_err_key */); // TODO + if (!s.ok()) { + return s; + } + + return collector_->AddUserKey(ikey.user_key, value, GetEntryType(ikey.type), + ikey.sequence, file_size); +} + +void UserKeyTablePropertiesCollector::BlockAdd( + uint64_t block_uncomp_bytes, uint64_t block_compressed_bytes_fast, + uint64_t block_compressed_bytes_slow) { + return collector_->BlockAdd(block_uncomp_bytes, block_compressed_bytes_fast, + block_compressed_bytes_slow); +} + +Status UserKeyTablePropertiesCollector::Finish( + UserCollectedProperties* properties) { + return collector_->Finish(properties); +} + +UserCollectedProperties UserKeyTablePropertiesCollector::GetReadableProperties() + const { + return collector_->GetReadableProperties(); +} + +uint64_t GetDeletedKeys(const UserCollectedProperties& props) { + bool property_present_ignored; + return GetUint64Property(props, TablePropertiesNames::kDeletedKeys, + &property_present_ignored); +} + +uint64_t GetMergeOperands(const UserCollectedProperties& props, + bool* property_present) { + return GetUint64Property(props, TablePropertiesNames::kMergeOperands, + property_present); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/table_properties_collector.h b/librocksdb-sys/rocksdb/db/table_properties_collector.h new file mode 100644 index 0000000..968115c --- /dev/null +++ b/librocksdb-sys/rocksdb/db/table_properties_collector.h @@ -0,0 +1,177 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file defines a collection of statistics collectors. +#pragma once + +#include +#include +#include + +#include "db/dbformat.h" +#include "rocksdb/comparator.h" +#include "rocksdb/table_properties.h" + +namespace ROCKSDB_NAMESPACE { + +// Base class for internal table properties collector. +class IntTblPropCollector { + public: + virtual ~IntTblPropCollector() {} + virtual Status Finish(UserCollectedProperties* properties) = 0; + + virtual const char* Name() const = 0; + + // @params key the user key that is inserted into the table. + // @params value the value that is inserted into the table. + virtual Status InternalAdd(const Slice& key, const Slice& value, + uint64_t file_size) = 0; + + virtual void BlockAdd(uint64_t block_uncomp_bytes, + uint64_t block_compressed_bytes_fast, + uint64_t block_compressed_bytes_slow) = 0; + + virtual UserCollectedProperties GetReadableProperties() const = 0; + + virtual bool NeedCompact() const { return false; } +}; + +// Factory for internal table properties collector. +class IntTblPropCollectorFactory { + public: + virtual ~IntTblPropCollectorFactory() {} + // has to be thread-safe + virtual IntTblPropCollector* CreateIntTblPropCollector( + uint32_t column_family_id, int level_at_creation) = 0; + + // The name of the properties collector can be used for debugging purpose. + virtual const char* Name() const = 0; +}; + +using IntTblPropCollectorFactories = + std::vector>; + +// When rocksdb creates a new table, it will encode all "user keys" into +// "internal keys", which contains meta information of a given entry. +// +// This class extracts user key from the encoded internal key when Add() is +// invoked. +class UserKeyTablePropertiesCollector : public IntTblPropCollector { + public: + // transfer of ownership + explicit UserKeyTablePropertiesCollector(TablePropertiesCollector* collector) + : collector_(collector) {} + + virtual ~UserKeyTablePropertiesCollector() {} + + virtual Status InternalAdd(const Slice& key, const Slice& value, + uint64_t file_size) override; + + virtual void BlockAdd(uint64_t block_uncomp_bytes, + uint64_t block_compressed_bytes_fast, + uint64_t block_compressed_bytes_slow) override; + + virtual Status Finish(UserCollectedProperties* properties) override; + + virtual const char* Name() const override { return collector_->Name(); } + + UserCollectedProperties GetReadableProperties() const override; + + virtual bool NeedCompact() const override { + return collector_->NeedCompact(); + } + + protected: + std::unique_ptr collector_; +}; + +class UserKeyTablePropertiesCollectorFactory + : public IntTblPropCollectorFactory { + public: + explicit UserKeyTablePropertiesCollectorFactory( + std::shared_ptr user_collector_factory) + : user_collector_factory_(user_collector_factory) {} + virtual IntTblPropCollector* CreateIntTblPropCollector( + uint32_t column_family_id, int level_at_creation) override { + TablePropertiesCollectorFactory::Context context; + context.column_family_id = column_family_id; + context.level_at_creation = level_at_creation; + return new UserKeyTablePropertiesCollector( + user_collector_factory_->CreateTablePropertiesCollector(context)); + } + + virtual const char* Name() const override { + return user_collector_factory_->Name(); + } + + private: + std::shared_ptr user_collector_factory_; +}; + +// When rocksdb creates a newtable, it will encode all "user keys" into +// "internal keys". This class collects min/max timestamp from the encoded +// internal key when Add() is invoked. +// +// @param cmp the user comparator to compare the timestamps in internal key. +class TimestampTablePropertiesCollector : public IntTblPropCollector { + public: + explicit TimestampTablePropertiesCollector(const Comparator* cmp) + : cmp_(cmp), + timestamp_min_(kDisableUserTimestamp), + timestamp_max_(kDisableUserTimestamp) {} + + Status InternalAdd(const Slice& key, const Slice& /* value */, + uint64_t /* file_size */) override { + auto user_key = ExtractUserKey(key); + assert(cmp_ && cmp_->timestamp_size() > 0); + if (user_key.size() < cmp_->timestamp_size()) { + return Status::Corruption( + "User key size mismatch when comparing to timestamp size."); + } + auto timestamp_in_key = + ExtractTimestampFromUserKey(user_key, cmp_->timestamp_size()); + if (timestamp_max_ == kDisableUserTimestamp || + cmp_->CompareTimestamp(timestamp_in_key, timestamp_max_) > 0) { + timestamp_max_.assign(timestamp_in_key.data(), timestamp_in_key.size()); + } + if (timestamp_min_ == kDisableUserTimestamp || + cmp_->CompareTimestamp(timestamp_min_, timestamp_in_key) > 0) { + timestamp_min_.assign(timestamp_in_key.data(), timestamp_in_key.size()); + } + return Status::OK(); + } + + void BlockAdd(uint64_t /* block_uncomp_bytes */, + uint64_t /* block_compressed_bytes_fast */, + uint64_t /* block_compressed_bytes_slow */) override { + return; + } + + Status Finish(UserCollectedProperties* properties) override { + // timestamp is empty is table is empty + assert(timestamp_min_.size() == timestamp_max_.size() && + (timestamp_min_.empty() || + timestamp_max_.size() == cmp_->timestamp_size())); + properties->insert({"rocksdb.timestamp_min", timestamp_min_}); + properties->insert({"rocksdb.timestamp_max", timestamp_max_}); + return Status::OK(); + } + + const char* Name() const override { + return "TimestampTablePropertiesCollector"; + } + + UserCollectedProperties GetReadableProperties() const override { + return {{"rocksdb.timestamp_min", Slice(timestamp_min_).ToString(true)}, + {"rocksdb.timestamp_max", Slice(timestamp_max_).ToString(true)}}; + } + + protected: + const Comparator* const cmp_; + std::string timestamp_min_; + std::string timestamp_max_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/table_properties_collector_test.cc b/librocksdb-sys/rocksdb/db/table_properties_collector_test.cc new file mode 100644 index 0000000..437b7e3 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/table_properties_collector_test.cc @@ -0,0 +1,512 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/table_properties_collector.h" + +#include +#include +#include +#include +#include + +#include "db/db_impl/db_impl.h" +#include "db/dbformat.h" +#include "file/sequence_file_reader.h" +#include "file/writable_file_writer.h" +#include "options/cf_options.h" +#include "rocksdb/flush_block_policy.h" +#include "rocksdb/table.h" +#include "table/block_based/block_based_table_factory.h" +#include "table/meta_blocks.h" +#include "table/plain/plain_table_factory.h" +#include "table/table_builder.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { + +class TablePropertiesTest : public testing::Test, + public testing::WithParamInterface { + public: + void SetUp() override { backward_mode_ = GetParam(); } + + bool backward_mode_; +}; + +// Utilities test functions +namespace { +static const uint32_t kTestColumnFamilyId = 66; +static const std::string kTestColumnFamilyName = "test_column_fam"; +static const int kTestLevel = 1; + +void MakeBuilder( + const Options& options, const ImmutableOptions& ioptions, + const MutableCFOptions& moptions, + const InternalKeyComparator& internal_comparator, + const IntTblPropCollectorFactories* int_tbl_prop_collector_factories, + std::unique_ptr* writable, + std::unique_ptr* builder) { + std::unique_ptr wf(new test::StringSink); + writable->reset( + new WritableFileWriter(std::move(wf), "" /* don't care */, EnvOptions())); + TableBuilderOptions tboptions( + ioptions, moptions, internal_comparator, int_tbl_prop_collector_factories, + options.compression, options.compression_opts, kTestColumnFamilyId, + kTestColumnFamilyName, kTestLevel); + builder->reset(NewTableBuilder(tboptions, writable->get())); +} +} // namespace + +// Collects keys that starts with "A" in a table. +class RegularKeysStartWithA : public TablePropertiesCollector { + public: + const char* Name() const override { return "RegularKeysStartWithA"; } + + Status Finish(UserCollectedProperties* properties) override { + std::string encoded; + std::string encoded_num_puts; + std::string encoded_num_deletes; + std::string encoded_num_single_deletes; + std::string encoded_num_size_changes; + PutVarint32(&encoded, count_); + PutVarint32(&encoded_num_puts, num_puts_); + PutVarint32(&encoded_num_deletes, num_deletes_); + PutVarint32(&encoded_num_single_deletes, num_single_deletes_); + PutVarint32(&encoded_num_size_changes, num_size_changes_); + *properties = UserCollectedProperties{ + {"TablePropertiesTest", message_}, + {"Count", encoded}, + {"NumPuts", encoded_num_puts}, + {"NumDeletes", encoded_num_deletes}, + {"NumSingleDeletes", encoded_num_single_deletes}, + {"NumSizeChanges", encoded_num_size_changes}, + }; + return Status::OK(); + } + + Status AddUserKey(const Slice& user_key, const Slice& /*value*/, + EntryType type, SequenceNumber /*seq*/, + uint64_t file_size) override { + // simply asssume all user keys are not empty. + if (user_key.data()[0] == 'A') { + ++count_; + } + if (type == kEntryPut) { + num_puts_++; + } else if (type == kEntryDelete) { + num_deletes_++; + } else if (type == kEntrySingleDelete) { + num_single_deletes_++; + } + if (file_size < file_size_) { + message_ = "File size should not decrease."; + } else if (file_size != file_size_) { + num_size_changes_++; + } + + return Status::OK(); + } + + UserCollectedProperties GetReadableProperties() const override { + return UserCollectedProperties{}; + } + + private: + std::string message_ = "Rocksdb"; + uint32_t count_ = 0; + uint32_t num_puts_ = 0; + uint32_t num_deletes_ = 0; + uint32_t num_single_deletes_ = 0; + uint32_t num_size_changes_ = 0; + uint64_t file_size_ = 0; +}; + +// Collects keys that starts with "A" in a table. Backward compatible mode +// It is also used to test internal key table property collector +class RegularKeysStartWithABackwardCompatible + : public TablePropertiesCollector { + public: + const char* Name() const override { return "RegularKeysStartWithA"; } + + Status Finish(UserCollectedProperties* properties) override { + std::string encoded; + PutVarint32(&encoded, count_); + *properties = UserCollectedProperties{{"TablePropertiesTest", "Rocksdb"}, + {"Count", encoded}}; + return Status::OK(); + } + + Status Add(const Slice& user_key, const Slice& /*value*/) override { + // simply asssume all user keys are not empty. + if (user_key.data()[0] == 'A') { + ++count_; + } + return Status::OK(); + } + + UserCollectedProperties GetReadableProperties() const override { + return UserCollectedProperties{}; + } + + private: + uint32_t count_ = 0; +}; + +class RegularKeysStartWithAInternal : public IntTblPropCollector { + public: + const char* Name() const override { return "RegularKeysStartWithA"; } + + Status Finish(UserCollectedProperties* properties) override { + std::string encoded; + PutVarint32(&encoded, count_); + *properties = UserCollectedProperties{{"TablePropertiesTest", "Rocksdb"}, + {"Count", encoded}}; + return Status::OK(); + } + + Status InternalAdd(const Slice& user_key, const Slice& /*value*/, + uint64_t /*file_size*/) override { + // simply asssume all user keys are not empty. + if (user_key.data()[0] == 'A') { + ++count_; + } + return Status::OK(); + } + + void BlockAdd(uint64_t /* block_uncomp_bytes */, + uint64_t /* block_compressed_bytes_fast */, + uint64_t /* block_compressed_bytes_slow */) override { + // Nothing to do. + return; + } + + UserCollectedProperties GetReadableProperties() const override { + return UserCollectedProperties{}; + } + + private: + uint32_t count_ = 0; +}; + +class RegularKeysStartWithAFactory : public IntTblPropCollectorFactory, + public TablePropertiesCollectorFactory { + public: + explicit RegularKeysStartWithAFactory(bool backward_mode) + : backward_mode_(backward_mode) {} + TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context context) override { + EXPECT_EQ(kTestColumnFamilyId, context.column_family_id); + EXPECT_EQ(kTestLevel, context.level_at_creation); + if (!backward_mode_) { + return new RegularKeysStartWithA(); + } else { + return new RegularKeysStartWithABackwardCompatible(); + } + } + IntTblPropCollector* CreateIntTblPropCollector( + uint32_t /*column_family_id*/, int /* level_at_creation */) override { + return new RegularKeysStartWithAInternal(); + } + const char* Name() const override { return "RegularKeysStartWithA"; } + + bool backward_mode_; +}; + +class FlushBlockEveryThreePolicy : public FlushBlockPolicy { + public: + bool Update(const Slice& /*key*/, const Slice& /*value*/) override { + return (++count_ % 3U == 0); + } + + private: + uint64_t count_ = 0; +}; + +class FlushBlockEveryThreePolicyFactory : public FlushBlockPolicyFactory { + public: + explicit FlushBlockEveryThreePolicyFactory() {} + + const char* Name() const override { + return "FlushBlockEveryThreePolicyFactory"; + } + + FlushBlockPolicy* NewFlushBlockPolicy( + const BlockBasedTableOptions& /*table_options*/, + const BlockBuilder& /*data_block_builder*/) const override { + return new FlushBlockEveryThreePolicy; + } +}; + +extern const uint64_t kBlockBasedTableMagicNumber; +extern const uint64_t kPlainTableMagicNumber; +namespace { +void TestCustomizedTablePropertiesCollector( + bool backward_mode, uint64_t magic_number, bool test_int_tbl_prop_collector, + const Options& options, const InternalKeyComparator& internal_comparator) { + // make sure the entries will be inserted with order. + std::map, std::string> kvs = { + {{"About ", kTypeValue}, "val5"}, // starts with 'A' + {{"Abstract", kTypeValue}, "val2"}, // starts with 'A' + {{"Around ", kTypeValue}, "val7"}, // starts with 'A' + {{"Beyond ", kTypeValue}, "val3"}, + {{"Builder ", kTypeValue}, "val1"}, + {{"Love ", kTypeDeletion}, ""}, + {{"Cancel ", kTypeValue}, "val4"}, + {{"Find ", kTypeValue}, "val6"}, + {{"Rocks ", kTypeDeletion}, ""}, + {{"Foo ", kTypeSingleDeletion}, ""}, + }; + + // -- Step 1: build table + std::unique_ptr builder; + std::unique_ptr writer; + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + IntTblPropCollectorFactories int_tbl_prop_collector_factories; + if (test_int_tbl_prop_collector) { + int_tbl_prop_collector_factories.emplace_back( + new RegularKeysStartWithAFactory(backward_mode)); + } else { + GetIntTblPropCollectorFactory(ioptions, &int_tbl_prop_collector_factories); + } + MakeBuilder(options, ioptions, moptions, internal_comparator, + &int_tbl_prop_collector_factories, &writer, &builder); + + SequenceNumber seqNum = 0U; + for (const auto& kv : kvs) { + InternalKey ikey(kv.first.first, seqNum++, kv.first.second); + builder->Add(ikey.Encode(), kv.second); + } + ASSERT_OK(builder->Finish()); + ASSERT_OK(writer->Flush()); + + // -- Step 2: Read properties + test::StringSink* fwf = + static_cast(writer->writable_file()); + std::unique_ptr source( + new test::StringSource(fwf->contents())); + std::unique_ptr fake_file_reader( + new RandomAccessFileReader(std::move(source), "test")); + + std::unique_ptr props; + const ReadOptions read_options; + Status s = ReadTableProperties(fake_file_reader.get(), fwf->contents().size(), + magic_number, ioptions, read_options, &props); + ASSERT_OK(s); + + auto user_collected = props->user_collected_properties; + + ASSERT_NE(user_collected.find("TablePropertiesTest"), user_collected.end()); + ASSERT_EQ("Rocksdb", user_collected.at("TablePropertiesTest")); + + uint32_t starts_with_A = 0; + ASSERT_NE(user_collected.find("Count"), user_collected.end()); + Slice key(user_collected.at("Count")); + ASSERT_TRUE(GetVarint32(&key, &starts_with_A)); + ASSERT_EQ(3u, starts_with_A); + + if (!backward_mode && !test_int_tbl_prop_collector) { + uint32_t num_puts; + ASSERT_NE(user_collected.find("NumPuts"), user_collected.end()); + Slice key_puts(user_collected.at("NumPuts")); + ASSERT_TRUE(GetVarint32(&key_puts, &num_puts)); + ASSERT_EQ(7u, num_puts); + + uint32_t num_deletes; + ASSERT_NE(user_collected.find("NumDeletes"), user_collected.end()); + Slice key_deletes(user_collected.at("NumDeletes")); + ASSERT_TRUE(GetVarint32(&key_deletes, &num_deletes)); + ASSERT_EQ(2u, num_deletes); + + uint32_t num_single_deletes; + ASSERT_NE(user_collected.find("NumSingleDeletes"), user_collected.end()); + Slice key_single_deletes(user_collected.at("NumSingleDeletes")); + ASSERT_TRUE(GetVarint32(&key_single_deletes, &num_single_deletes)); + ASSERT_EQ(1u, num_single_deletes); + + uint32_t num_size_changes; + ASSERT_NE(user_collected.find("NumSizeChanges"), user_collected.end()); + Slice key_size_changes(user_collected.at("NumSizeChanges")); + ASSERT_TRUE(GetVarint32(&key_size_changes, &num_size_changes)); + ASSERT_GE(num_size_changes, 2u); + } +} +} // namespace + +TEST_P(TablePropertiesTest, CustomizedTablePropertiesCollector) { + // Test properties collectors with internal keys or regular keys + // for block based table + for (bool encode_as_internal : {true, false}) { + Options options; + BlockBasedTableOptions table_options; + table_options.flush_block_policy_factory = + std::make_shared(); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + test::PlainInternalKeyComparator ikc(options.comparator); + std::shared_ptr collector_factory( + new RegularKeysStartWithAFactory(backward_mode_)); + options.table_properties_collector_factories.resize(1); + options.table_properties_collector_factories[0] = collector_factory; + + TestCustomizedTablePropertiesCollector(backward_mode_, + kBlockBasedTableMagicNumber, + encode_as_internal, options, ikc); + + // test plain table + PlainTableOptions plain_table_options; + plain_table_options.user_key_len = 8; + plain_table_options.bloom_bits_per_key = 8; + plain_table_options.hash_table_ratio = 0; + + options.table_factory = + std::make_shared(plain_table_options); + TestCustomizedTablePropertiesCollector(backward_mode_, + kPlainTableMagicNumber, + encode_as_internal, options, ikc); + } +} + +namespace { +void TestInternalKeyPropertiesCollector( + bool backward_mode, uint64_t magic_number, bool sanitized, + std::shared_ptr table_factory) { + InternalKey keys[] = { + InternalKey("A ", 0, ValueType::kTypeValue), + InternalKey("B ", 1, ValueType::kTypeValue), + InternalKey("C ", 2, ValueType::kTypeValue), + InternalKey("W ", 3, ValueType::kTypeDeletion), + InternalKey("X ", 4, ValueType::kTypeDeletion), + InternalKey("Y ", 5, ValueType::kTypeDeletion), + InternalKey("Z ", 6, ValueType::kTypeDeletion), + InternalKey("a ", 7, ValueType::kTypeSingleDeletion), + InternalKey("b ", 8, ValueType::kTypeMerge), + InternalKey("c ", 9, ValueType::kTypeMerge), + }; + + std::unique_ptr builder; + std::unique_ptr writable; + Options options; + test::PlainInternalKeyComparator pikc(options.comparator); + + IntTblPropCollectorFactories int_tbl_prop_collector_factories; + options.table_factory = table_factory; + if (sanitized) { + options.table_properties_collector_factories.emplace_back( + new RegularKeysStartWithAFactory(backward_mode)); + // with sanitization, even regular properties collector will be able to + // handle internal keys. + auto comparator = options.comparator; + // HACK: Set options.info_log to avoid writing log in + // SanitizeOptions(). + options.info_log = std::make_shared(); + options = SanitizeOptions("db", // just a place holder + options); + ImmutableOptions ioptions(options); + GetIntTblPropCollectorFactory(ioptions, &int_tbl_prop_collector_factories); + options.comparator = comparator; + } + const ImmutableOptions ioptions(options); + MutableCFOptions moptions(options); + + for (int iter = 0; iter < 2; ++iter) { + MakeBuilder(options, ioptions, moptions, pikc, + &int_tbl_prop_collector_factories, &writable, &builder); + for (const auto& k : keys) { + builder->Add(k.Encode(), "val"); + } + + ASSERT_OK(builder->Finish()); + ASSERT_OK(writable->Flush()); + + test::StringSink* fwf = + static_cast(writable->writable_file()); + std::unique_ptr source( + new test::StringSource(fwf->contents())); + std::unique_ptr reader( + new RandomAccessFileReader(std::move(source), "test")); + + std::unique_ptr props; + const ReadOptions read_options; + Status s = + ReadTableProperties(reader.get(), fwf->contents().size(), magic_number, + ioptions, read_options, &props); + ASSERT_OK(s); + + auto user_collected = props->user_collected_properties; + uint64_t deleted = GetDeletedKeys(user_collected); + ASSERT_EQ(5u, deleted); // deletes + single-deletes + + bool property_present; + uint64_t merges = GetMergeOperands(user_collected, &property_present); + ASSERT_TRUE(property_present); + ASSERT_EQ(2u, merges); + + if (sanitized) { + uint32_t starts_with_A = 0; + ASSERT_NE(user_collected.find("Count"), user_collected.end()); + Slice key(user_collected.at("Count")); + ASSERT_TRUE(GetVarint32(&key, &starts_with_A)); + ASSERT_EQ(1u, starts_with_A); + + if (!backward_mode) { + uint32_t num_puts; + ASSERT_NE(user_collected.find("NumPuts"), user_collected.end()); + Slice key_puts(user_collected.at("NumPuts")); + ASSERT_TRUE(GetVarint32(&key_puts, &num_puts)); + ASSERT_EQ(3u, num_puts); + + uint32_t num_deletes; + ASSERT_NE(user_collected.find("NumDeletes"), user_collected.end()); + Slice key_deletes(user_collected.at("NumDeletes")); + ASSERT_TRUE(GetVarint32(&key_deletes, &num_deletes)); + ASSERT_EQ(4u, num_deletes); + + uint32_t num_single_deletes; + ASSERT_NE(user_collected.find("NumSingleDeletes"), + user_collected.end()); + Slice key_single_deletes(user_collected.at("NumSingleDeletes")); + ASSERT_TRUE(GetVarint32(&key_single_deletes, &num_single_deletes)); + ASSERT_EQ(1u, num_single_deletes); + } + } + } +} +} // namespace + +TEST_P(TablePropertiesTest, InternalKeyPropertiesCollector) { + TestInternalKeyPropertiesCollector( + backward_mode_, kBlockBasedTableMagicNumber, true /* sanitize */, + std::make_shared()); + if (backward_mode_) { + TestInternalKeyPropertiesCollector( + backward_mode_, kBlockBasedTableMagicNumber, false /* not sanitize */, + std::make_shared()); + } + + PlainTableOptions plain_table_options; + plain_table_options.user_key_len = 8; + plain_table_options.bloom_bits_per_key = 8; + plain_table_options.hash_table_ratio = 0; + + TestInternalKeyPropertiesCollector( + backward_mode_, kPlainTableMagicNumber, false /* not sanitize */, + std::make_shared(plain_table_options)); +} + +INSTANTIATE_TEST_CASE_P(InternalKeyPropertiesCollector, TablePropertiesTest, + ::testing::Bool()); + +INSTANTIATE_TEST_CASE_P(CustomizedTablePropertiesCollector, TablePropertiesTest, + ::testing::Bool()); + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/transaction_log_impl.cc b/librocksdb-sys/rocksdb/db/transaction_log_impl.cc new file mode 100644 index 0000000..8841b8c --- /dev/null +++ b/librocksdb-sys/rocksdb/db/transaction_log_impl.cc @@ -0,0 +1,296 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + + +#include "db/transaction_log_impl.h" + +#include + +#include "db/write_batch_internal.h" +#include "file/sequence_file_reader.h" +#include "util/defer.h" + +namespace ROCKSDB_NAMESPACE { + +TransactionLogIteratorImpl::TransactionLogIteratorImpl( + const std::string& dir, const ImmutableDBOptions* options, + const TransactionLogIterator::ReadOptions& read_options, + const EnvOptions& soptions, const SequenceNumber seq, + std::unique_ptr files, VersionSet const* const versions, + const bool seq_per_batch, const std::shared_ptr& io_tracer) + : dir_(dir), + options_(options), + read_options_(read_options), + soptions_(soptions), + starting_sequence_number_(seq), + files_(std::move(files)), + versions_(versions), + seq_per_batch_(seq_per_batch), + io_tracer_(io_tracer), + started_(false), + is_valid_(false), + current_file_index_(0), + current_batch_seq_(0), + current_last_seq_(0) { + assert(files_ != nullptr); + assert(versions_ != nullptr); + assert(!seq_per_batch_); + current_status_.PermitUncheckedError(); // Clear on start + reporter_.env = options_->env; + reporter_.info_log = options_->info_log.get(); + SeekToStartSequence(); // Seek till starting sequence +} + +Status TransactionLogIteratorImpl::OpenLogFile( + const LogFile* log_file, + std::unique_ptr* file_reader) { + FileSystemPtr fs(options_->fs, io_tracer_); + std::unique_ptr file; + std::string fname; + Status s; + EnvOptions optimized_env_options = fs->OptimizeForLogRead(soptions_); + if (log_file->Type() == kArchivedLogFile) { + fname = ArchivedLogFileName(dir_, log_file->LogNumber()); + s = fs->NewSequentialFile(fname, optimized_env_options, &file, nullptr); + } else { + fname = LogFileName(dir_, log_file->LogNumber()); + s = fs->NewSequentialFile(fname, optimized_env_options, &file, nullptr); + if (!s.ok()) { + // If cannot open file in DB directory. + // Try the archive dir, as it could have moved in the meanwhile. + fname = ArchivedLogFileName(dir_, log_file->LogNumber()); + s = fs->NewSequentialFile(fname, optimized_env_options, &file, nullptr); + } + } + if (s.ok()) { + file_reader->reset(new SequentialFileReader(std::move(file), fname, + io_tracer_, options_->listeners, + options_->rate_limiter.get())); + } + return s; +} + +BatchResult TransactionLogIteratorImpl::GetBatch() { + assert(is_valid_); // cannot call in a non valid state. + BatchResult result; + result.sequence = current_batch_seq_; + result.writeBatchPtr = std::move(current_batch_); + return result; +} + +Status TransactionLogIteratorImpl::status() { return current_status_; } + +bool TransactionLogIteratorImpl::Valid() { return started_ && is_valid_; } + +bool TransactionLogIteratorImpl::RestrictedRead(Slice* record) { + // Don't read if no more complete entries to read from logs + if (current_last_seq_ >= versions_->LastSequence()) { + return false; + } + return current_log_reader_->ReadRecord(record, &scratch_); +} + +void TransactionLogIteratorImpl::SeekToStartSequence(uint64_t start_file_index, + bool strict) { + Slice record; + started_ = false; + is_valid_ = false; + // Check invariant of TransactionLogIterator when SeekToStartSequence() + // succeeds. + const Defer defer([this]() { + if (is_valid_) { + assert(current_status_.ok()); + if (starting_sequence_number_ > current_batch_seq_) { + assert(current_batch_seq_ < current_last_seq_); + assert(current_last_seq_ >= starting_sequence_number_); + } + } + }); + if (files_->size() <= start_file_index) { + return; + } else if (!current_status_.ok()) { + return; + } + Status s = + OpenLogReader(files_->at(static_cast(start_file_index)).get()); + if (!s.ok()) { + current_status_ = s; + reporter_.Info(current_status_.ToString().c_str()); + return; + } + while (RestrictedRead(&record)) { + if (record.size() < WriteBatchInternal::kHeader) { + reporter_.Corruption(record.size(), + Status::Corruption("very small log record")); + continue; + } + UpdateCurrentWriteBatch(record); + if (current_last_seq_ >= starting_sequence_number_) { + if (strict && current_batch_seq_ != starting_sequence_number_) { + current_status_ = Status::Corruption( + "Gap in sequence number. Could not " + "seek to required sequence number"); + reporter_.Info(current_status_.ToString().c_str()); + return; + } else if (strict) { + reporter_.Info( + "Could seek required sequence number. Iterator will " + "continue."); + } + is_valid_ = true; + started_ = true; // set started_ as we could seek till starting sequence + return; + } else { + is_valid_ = false; + } + } + + // Could not find start sequence in first file. Normally this must be the + // only file. Otherwise log the error and let the iterator return next entry + // If strict is set, we want to seek exactly till the start sequence and it + // should have been present in the file we scanned above + if (strict) { + current_status_ = Status::Corruption( + "Gap in sequence number. Could not " + "seek to required sequence number"); + reporter_.Info(current_status_.ToString().c_str()); + } else if (files_->size() != 1) { + current_status_ = Status::Corruption( + "Start sequence was not found, " + "skipping to the next available"); + reporter_.Info(current_status_.ToString().c_str()); + // Let NextImpl find the next available entry. started_ remains false + // because we don't want to check for gaps while moving to start sequence + NextImpl(true); + } +} + +void TransactionLogIteratorImpl::Next() { + if (!current_status_.ok()) { + return; + } + return NextImpl(false); +} + +void TransactionLogIteratorImpl::NextImpl(bool internal) { + Slice record; + is_valid_ = false; + if (!internal && !started_) { + // Runs every time until we can seek to the start sequence + SeekToStartSequence(); + } + while (true) { + assert(current_log_reader_); + if (current_log_reader_->IsEOF()) { + current_log_reader_->UnmarkEOF(); + } + while (RestrictedRead(&record)) { + if (record.size() < WriteBatchInternal::kHeader) { + reporter_.Corruption(record.size(), + Status::Corruption("very small log record")); + continue; + } else { + // started_ should be true if called by application + assert(internal || started_); + // started_ should be false if called internally + assert(!internal || !started_); + UpdateCurrentWriteBatch(record); + if (internal && !started_) { + started_ = true; + } + return; + } + } + + // Open the next file + if (current_file_index_ < files_->size() - 1) { + ++current_file_index_; + Status s = OpenLogReader(files_->at(current_file_index_).get()); + if (!s.ok()) { + is_valid_ = false; + current_status_ = s; + return; + } + } else { + is_valid_ = false; + if (current_last_seq_ == versions_->LastSequence()) { + current_status_ = Status::OK(); + } else { + const char* msg = "Create a new iterator to fetch the new tail."; + current_status_ = Status::TryAgain(msg); + } + return; + } + } +} + +bool TransactionLogIteratorImpl::IsBatchExpected( + const WriteBatch* batch, const SequenceNumber expected_seq) { + assert(batch); + SequenceNumber batchSeq = WriteBatchInternal::Sequence(batch); + if (batchSeq != expected_seq) { + char buf[200]; + snprintf(buf, sizeof(buf), + "Discontinuity in log records. Got seq=%" PRIu64 + ", Expected seq=%" PRIu64 ", Last flushed seq=%" PRIu64 + ".Log iterator will reseek the correct batch.", + batchSeq, expected_seq, versions_->LastSequence()); + reporter_.Info(buf); + return false; + } + return true; +} + +void TransactionLogIteratorImpl::UpdateCurrentWriteBatch(const Slice& record) { + std::unique_ptr batch(new WriteBatch()); + Status s = WriteBatchInternal::SetContents(batch.get(), record); + s.PermitUncheckedError(); // TODO: What should we do with this error? + + SequenceNumber expected_seq = current_last_seq_ + 1; + // If the iterator has started, then confirm that we get continuous batches + if (started_ && !IsBatchExpected(batch.get(), expected_seq)) { + // Seek to the batch having expected sequence number + if (expected_seq < files_->at(current_file_index_)->StartSequence()) { + // Expected batch must lie in the previous log file + // Avoid underflow. + if (current_file_index_ != 0) { + current_file_index_--; + } + } + starting_sequence_number_ = expected_seq; + // currentStatus_ will be set to Ok if reseek succeeds + // Note: this is still ok in seq_pre_batch_ && two_write_queuesp_ mode + // that allows gaps in the WAL since it will still skip over the gap. + current_status_ = Status::NotFound("Gap in sequence numbers"); + // In seq_per_batch_ mode, gaps in the seq are possible so the strict mode + // should be disabled + return SeekToStartSequence(current_file_index_, !seq_per_batch_); + } + + current_batch_seq_ = WriteBatchInternal::Sequence(batch.get()); + assert(!seq_per_batch_); + current_last_seq_ = + current_batch_seq_ + WriteBatchInternal::Count(batch.get()) - 1; + // currentBatchSeq_ can only change here + assert(current_last_seq_ <= versions_->LastSequence()); + + current_batch_ = std::move(batch); + is_valid_ = true; + current_status_ = Status::OK(); +} + +Status TransactionLogIteratorImpl::OpenLogReader(const LogFile* log_file) { + std::unique_ptr file; + Status s = OpenLogFile(log_file, &file); + if (!s.ok()) { + return s; + } + assert(file); + current_log_reader_.reset( + new log::Reader(options_->info_log, std::move(file), &reporter_, + read_options_.verify_checksums_, log_file->LogNumber())); + return Status::OK(); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/transaction_log_impl.h b/librocksdb-sys/rocksdb/db/transaction_log_impl.h new file mode 100644 index 0000000..6568de2 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/transaction_log_impl.h @@ -0,0 +1,128 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include + +#include "db/log_reader.h" +#include "db/version_set.h" +#include "file/filename.h" +#include "logging/logging.h" +#include "options/db_options.h" +#include "port/port.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "rocksdb/transaction_log.h" +#include "rocksdb/types.h" + +namespace ROCKSDB_NAMESPACE { + +class LogFileImpl : public LogFile { + public: + LogFileImpl(uint64_t logNum, WalFileType logType, SequenceNumber startSeq, + uint64_t sizeBytes) + : logNumber_(logNum), + type_(logType), + startSequence_(startSeq), + sizeFileBytes_(sizeBytes) {} + + std::string PathName() const override { + if (type_ == kArchivedLogFile) { + return ArchivedLogFileName("", logNumber_); + } + return LogFileName("", logNumber_); + } + + uint64_t LogNumber() const override { return logNumber_; } + + WalFileType Type() const override { return type_; } + + SequenceNumber StartSequence() const override { return startSequence_; } + + uint64_t SizeFileBytes() const override { return sizeFileBytes_; } + + bool operator<(const LogFile& that) const { + return LogNumber() < that.LogNumber(); + } + + private: + uint64_t logNumber_; + WalFileType type_; + SequenceNumber startSequence_; + uint64_t sizeFileBytes_; +}; + +class TransactionLogIteratorImpl : public TransactionLogIterator { + public: + TransactionLogIteratorImpl( + const std::string& dir, const ImmutableDBOptions* options, + const TransactionLogIterator::ReadOptions& read_options, + const EnvOptions& soptions, const SequenceNumber seqNum, + std::unique_ptr files, VersionSet const* const versions, + const bool seq_per_batch, const std::shared_ptr& io_tracer); + + virtual bool Valid() override; + + virtual void Next() override; + + virtual Status status() override; + + virtual BatchResult GetBatch() override; + + private: + const std::string& dir_; + const ImmutableDBOptions* options_; + const TransactionLogIterator::ReadOptions read_options_; + const EnvOptions& soptions_; + SequenceNumber starting_sequence_number_; + std::unique_ptr files_; + // Used only to get latest seq. num + // TODO(icanadi) can this be just a callback? + VersionSet const* const versions_; + const bool seq_per_batch_; + std::shared_ptr io_tracer_; + + // State variables + bool started_; + bool is_valid_; // not valid when it starts of. + Status current_status_; + size_t current_file_index_; + std::unique_ptr current_batch_; + std::unique_ptr current_log_reader_; + std::string scratch_; + Status OpenLogFile(const LogFile* log_file, + std::unique_ptr* file); + + struct LogReporter : public log::Reader::Reporter { + Env* env; + Logger* info_log; + virtual void Corruption(size_t bytes, const Status& s) override { + ROCKS_LOG_ERROR(info_log, "dropping %" ROCKSDB_PRIszt " bytes; %s", bytes, + s.ToString().c_str()); + } + virtual void Info(const char* s) { ROCKS_LOG_INFO(info_log, "%s", s); } + } reporter_; + + SequenceNumber + current_batch_seq_; // sequence number at start of current batch + SequenceNumber current_last_seq_; // last sequence in the current batch + // Reads from transaction log only if the writebatch record has been written + bool RestrictedRead(Slice* record); + // Seeks to starting_sequence_number_ reading from start_file_index in files_. + // If strict is set, then must get a batch starting with + // starting_sequence_number_. + void SeekToStartSequence(uint64_t start_file_index = 0, bool strict = false); + // Implementation of Next. SeekToStartSequence calls it internally with + // internal=true to let it find next entry even if it has to jump gaps because + // the iterator may start off from the first available entry but promises to + // be continuous after that + void NextImpl(bool internal = false); + // Check if batch is expected, else return false + bool IsBatchExpected(const WriteBatch* batch, SequenceNumber expected_seq); + // Update current batch if a continuous batch is found. + void UpdateCurrentWriteBatch(const Slice& record); + Status OpenLogReader(const LogFile* file); +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/trim_history_scheduler.cc b/librocksdb-sys/rocksdb/db/trim_history_scheduler.cc new file mode 100644 index 0000000..d7ca089 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/trim_history_scheduler.cc @@ -0,0 +1,54 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/trim_history_scheduler.h" + +#include + +#include "db/column_family.h" + +namespace ROCKSDB_NAMESPACE { + +void TrimHistoryScheduler::ScheduleWork(ColumnFamilyData* cfd) { + std::lock_guard lock(checking_mutex_); + cfd->Ref(); + cfds_.push_back(cfd); + is_empty_.store(false, std::memory_order_relaxed); +} + +ColumnFamilyData* TrimHistoryScheduler::TakeNextColumnFamily() { + std::lock_guard lock(checking_mutex_); + while (true) { + if (cfds_.empty()) { + return nullptr; + } + ColumnFamilyData* cfd = cfds_.back(); + cfds_.pop_back(); + if (cfds_.empty()) { + is_empty_.store(true, std::memory_order_relaxed); + } + + if (!cfd->IsDropped()) { + // success + return cfd; + } + cfd->UnrefAndTryDelete(); + } +} + +bool TrimHistoryScheduler::Empty() { + bool is_empty = is_empty_.load(std::memory_order_relaxed); + return is_empty; +} + +void TrimHistoryScheduler::Clear() { + ColumnFamilyData* cfd; + while ((cfd = TakeNextColumnFamily()) != nullptr) { + cfd->UnrefAndTryDelete(); + } + assert(Empty()); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/trim_history_scheduler.h b/librocksdb-sys/rocksdb/db/trim_history_scheduler.h new file mode 100644 index 0000000..252802a --- /dev/null +++ b/librocksdb-sys/rocksdb/db/trim_history_scheduler.h @@ -0,0 +1,46 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include +#include + +#include "util/autovector.h" + +namespace ROCKSDB_NAMESPACE { + +class ColumnFamilyData; + +// Similar to FlushScheduler, TrimHistoryScheduler is a FIFO queue that keeps +// track of column families whose flushed immutable memtables may need to be +// removed (aka trimmed). The actual trimming may be slightly delayed. Due to +// the use of the mutex and atomic variable, ScheduleWork, +// TakeNextColumnFamily, and, Empty can be called concurrently. +class TrimHistoryScheduler { + public: + TrimHistoryScheduler() : is_empty_(true) {} + + // When a column family needs history trimming, add cfd to the FIFO queue + void ScheduleWork(ColumnFamilyData* cfd); + + // Remove the column family from the queue, the caller is responsible for + // calling `MemtableList::TrimHistory` + ColumnFamilyData* TakeNextColumnFamily(); + + bool Empty(); + + void Clear(); + + // Not on critical path, use mutex to ensure thread safety + private: + std::atomic is_empty_; + autovector cfds_; + std::mutex checking_mutex_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/version_builder.cc b/librocksdb-sys/rocksdb/db/version_builder.cc new file mode 100644 index 0000000..210b0de --- /dev/null +++ b/librocksdb-sys/rocksdb/db/version_builder.cc @@ -0,0 +1,1425 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/version_builder.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cache/cache_reservation_manager.h" +#include "db/blob/blob_file_meta.h" +#include "db/dbformat.h" +#include "db/internal_stats.h" +#include "db/table_cache.h" +#include "db/version_edit.h" +#include "db/version_set.h" +#include "port/port.h" +#include "table/table_reader.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +class VersionBuilder::Rep { + class NewestFirstByEpochNumber { + private: + inline static const NewestFirstBySeqNo seqno_cmp; + + public: + bool operator()(const FileMetaData* lhs, const FileMetaData* rhs) const { + assert(lhs); + assert(rhs); + + if (lhs->epoch_number != rhs->epoch_number) { + return lhs->epoch_number > rhs->epoch_number; + } else { + return seqno_cmp(lhs, rhs); + } + } + }; + class BySmallestKey { + public: + explicit BySmallestKey(const InternalKeyComparator* cmp) : cmp_(cmp) {} + + bool operator()(const FileMetaData* lhs, const FileMetaData* rhs) const { + assert(lhs); + assert(rhs); + assert(cmp_); + + const int r = cmp_->Compare(lhs->smallest, rhs->smallest); + if (r != 0) { + return (r < 0); + } + + // Break ties by file number + return (lhs->fd.GetNumber() < rhs->fd.GetNumber()); + } + + private: + const InternalKeyComparator* cmp_; + }; + + struct LevelState { + std::unordered_set deleted_files; + // Map from file number to file meta data. + std::unordered_map added_files; + }; + + // A class that represents the accumulated changes (like additional garbage or + // newly linked/unlinked SST files) for a given blob file after applying a + // series of VersionEdits. + class BlobFileMetaDataDelta { + public: + bool IsEmpty() const { + return !additional_garbage_count_ && !additional_garbage_bytes_ && + newly_linked_ssts_.empty() && newly_unlinked_ssts_.empty(); + } + + uint64_t GetAdditionalGarbageCount() const { + return additional_garbage_count_; + } + + uint64_t GetAdditionalGarbageBytes() const { + return additional_garbage_bytes_; + } + + const std::unordered_set& GetNewlyLinkedSsts() const { + return newly_linked_ssts_; + } + + const std::unordered_set& GetNewlyUnlinkedSsts() const { + return newly_unlinked_ssts_; + } + + void AddGarbage(uint64_t count, uint64_t bytes) { + additional_garbage_count_ += count; + additional_garbage_bytes_ += bytes; + } + + void LinkSst(uint64_t sst_file_number) { + assert(newly_linked_ssts_.find(sst_file_number) == + newly_linked_ssts_.end()); + + // Reconcile with newly unlinked SSTs on the fly. (Note: an SST can be + // linked to and unlinked from the same blob file in the case of a trivial + // move.) + auto it = newly_unlinked_ssts_.find(sst_file_number); + + if (it != newly_unlinked_ssts_.end()) { + newly_unlinked_ssts_.erase(it); + } else { + newly_linked_ssts_.emplace(sst_file_number); + } + } + + void UnlinkSst(uint64_t sst_file_number) { + assert(newly_unlinked_ssts_.find(sst_file_number) == + newly_unlinked_ssts_.end()); + + // Reconcile with newly linked SSTs on the fly. (Note: an SST can be + // linked to and unlinked from the same blob file in the case of a trivial + // move.) + auto it = newly_linked_ssts_.find(sst_file_number); + + if (it != newly_linked_ssts_.end()) { + newly_linked_ssts_.erase(it); + } else { + newly_unlinked_ssts_.emplace(sst_file_number); + } + } + + private: + uint64_t additional_garbage_count_ = 0; + uint64_t additional_garbage_bytes_ = 0; + std::unordered_set newly_linked_ssts_; + std::unordered_set newly_unlinked_ssts_; + }; + + // A class that represents the state of a blob file after applying a series of + // VersionEdits. In addition to the resulting state, it also contains the + // delta (see BlobFileMetaDataDelta above). The resulting state can be used to + // identify obsolete blob files, while the delta makes it possible to + // efficiently detect trivial moves. + class MutableBlobFileMetaData { + public: + // To be used for brand new blob files + explicit MutableBlobFileMetaData( + std::shared_ptr&& shared_meta) + : shared_meta_(std::move(shared_meta)) {} + + // To be used for pre-existing blob files + explicit MutableBlobFileMetaData( + const std::shared_ptr& meta) + : shared_meta_(meta->GetSharedMeta()), + linked_ssts_(meta->GetLinkedSsts()), + garbage_blob_count_(meta->GetGarbageBlobCount()), + garbage_blob_bytes_(meta->GetGarbageBlobBytes()) {} + + const std::shared_ptr& GetSharedMeta() const { + return shared_meta_; + } + + uint64_t GetBlobFileNumber() const { + assert(shared_meta_); + return shared_meta_->GetBlobFileNumber(); + } + + bool HasDelta() const { return !delta_.IsEmpty(); } + + const std::unordered_set& GetLinkedSsts() const { + return linked_ssts_; + } + + uint64_t GetGarbageBlobCount() const { return garbage_blob_count_; } + + uint64_t GetGarbageBlobBytes() const { return garbage_blob_bytes_; } + + bool AddGarbage(uint64_t count, uint64_t bytes) { + assert(shared_meta_); + + if (garbage_blob_count_ + count > shared_meta_->GetTotalBlobCount() || + garbage_blob_bytes_ + bytes > shared_meta_->GetTotalBlobBytes()) { + return false; + } + + delta_.AddGarbage(count, bytes); + + garbage_blob_count_ += count; + garbage_blob_bytes_ += bytes; + + return true; + } + + void LinkSst(uint64_t sst_file_number) { + delta_.LinkSst(sst_file_number); + + assert(linked_ssts_.find(sst_file_number) == linked_ssts_.end()); + linked_ssts_.emplace(sst_file_number); + } + + void UnlinkSst(uint64_t sst_file_number) { + delta_.UnlinkSst(sst_file_number); + + assert(linked_ssts_.find(sst_file_number) != linked_ssts_.end()); + linked_ssts_.erase(sst_file_number); + } + + private: + std::shared_ptr shared_meta_; + // Accumulated changes + BlobFileMetaDataDelta delta_; + // Resulting state after applying the changes + BlobFileMetaData::LinkedSsts linked_ssts_; + uint64_t garbage_blob_count_ = 0; + uint64_t garbage_blob_bytes_ = 0; + }; + + const FileOptions& file_options_; + const ImmutableCFOptions* const ioptions_; + TableCache* table_cache_; + VersionStorageInfo* base_vstorage_; + VersionSet* version_set_; + int num_levels_; + LevelState* levels_; + // Store sizes of levels larger than num_levels_. We do this instead of + // storing them in levels_ to avoid regression in case there are no files + // on invalid levels. The version is not consistent if in the end the files + // on invalid levels don't cancel out. + std::unordered_map invalid_level_sizes_; + // Whether there are invalid new files or invalid deletion on levels larger + // than num_levels_. + bool has_invalid_levels_; + // Current levels of table files affected by additions/deletions. + std::unordered_map table_file_levels_; + // Current compact cursors that should be changed after the last compaction + std::unordered_map updated_compact_cursors_; + NewestFirstByEpochNumber level_zero_cmp_by_epochno_; + NewestFirstBySeqNo level_zero_cmp_by_seqno_; + BySmallestKey level_nonzero_cmp_; + + // Mutable metadata objects for all blob files affected by the series of + // version edits. + std::map mutable_blob_file_metas_; + + std::shared_ptr file_metadata_cache_res_mgr_; + + public: + Rep(const FileOptions& file_options, const ImmutableCFOptions* ioptions, + TableCache* table_cache, VersionStorageInfo* base_vstorage, + VersionSet* version_set, + std::shared_ptr file_metadata_cache_res_mgr) + : file_options_(file_options), + ioptions_(ioptions), + table_cache_(table_cache), + base_vstorage_(base_vstorage), + version_set_(version_set), + num_levels_(base_vstorage->num_levels()), + has_invalid_levels_(false), + level_nonzero_cmp_(base_vstorage_->InternalComparator()), + file_metadata_cache_res_mgr_(file_metadata_cache_res_mgr) { + assert(ioptions_); + + levels_ = new LevelState[num_levels_]; + } + + ~Rep() { + for (int level = 0; level < num_levels_; level++) { + const auto& added = levels_[level].added_files; + for (auto& pair : added) { + UnrefFile(pair.second); + } + } + + delete[] levels_; + } + + void UnrefFile(FileMetaData* f) { + f->refs--; + if (f->refs <= 0) { + if (f->table_reader_handle) { + assert(table_cache_ != nullptr); + // NOTE: have to release in raw cache interface to avoid using a + // TypedHandle for FileMetaData::table_reader_handle + table_cache_->get_cache().get()->Release(f->table_reader_handle); + f->table_reader_handle = nullptr; + } + + if (file_metadata_cache_res_mgr_) { + Status s = file_metadata_cache_res_mgr_->UpdateCacheReservation( + f->ApproximateMemoryUsage(), false /* increase */); + s.PermitUncheckedError(); + } + delete f; + } + } + + // Mapping used for checking the consistency of links between SST files and + // blob files. It is built using the forward links (table file -> blob file), + // and is subsequently compared with the inverse mapping stored in the + // BlobFileMetaData objects. + using ExpectedLinkedSsts = + std::unordered_map; + + static void UpdateExpectedLinkedSsts( + uint64_t table_file_number, uint64_t blob_file_number, + ExpectedLinkedSsts* expected_linked_ssts) { + assert(expected_linked_ssts); + + if (blob_file_number == kInvalidBlobFileNumber) { + return; + } + + (*expected_linked_ssts)[blob_file_number].emplace(table_file_number); + } + + template + Status CheckConsistencyDetailsForLevel( + const VersionStorageInfo* vstorage, int level, Checker checker, + const std::string& sync_point, + ExpectedLinkedSsts* expected_linked_ssts) const { +#ifdef NDEBUG + (void)sync_point; +#endif + + assert(vstorage); + assert(level >= 0 && level < num_levels_); + assert(expected_linked_ssts); + + const auto& level_files = vstorage->LevelFiles(level); + + if (level_files.empty()) { + return Status::OK(); + } + + assert(level_files[0]); + UpdateExpectedLinkedSsts(level_files[0]->fd.GetNumber(), + level_files[0]->oldest_blob_file_number, + expected_linked_ssts); + + for (size_t i = 1; i < level_files.size(); ++i) { + assert(level_files[i]); + UpdateExpectedLinkedSsts(level_files[i]->fd.GetNumber(), + level_files[i]->oldest_blob_file_number, + expected_linked_ssts); + + auto lhs = level_files[i - 1]; + auto rhs = level_files[i]; + +#ifndef NDEBUG + auto pair = std::make_pair(&lhs, &rhs); + TEST_SYNC_POINT_CALLBACK(sync_point, &pair); +#endif + + const Status s = checker(lhs, rhs); + if (!s.ok()) { + return s; + } + } + + return Status::OK(); + } + + // Make sure table files are sorted correctly and that the links between + // table files and blob files are consistent. + Status CheckConsistencyDetails(const VersionStorageInfo* vstorage) const { + assert(vstorage); + + ExpectedLinkedSsts expected_linked_ssts; + + if (num_levels_ > 0) { + const InternalKeyComparator* const icmp = vstorage->InternalComparator(); + EpochNumberRequirement epoch_number_requirement = + vstorage->GetEpochNumberRequirement(); + assert(icmp); + // Check L0 + { + auto l0_checker = [this, epoch_number_requirement, icmp]( + const FileMetaData* lhs, + const FileMetaData* rhs) { + assert(lhs); + assert(rhs); + + if (epoch_number_requirement == + EpochNumberRequirement::kMightMissing) { + if (!level_zero_cmp_by_seqno_(lhs, rhs)) { + std::ostringstream oss; + oss << "L0 files are not sorted properly: files #" + << lhs->fd.GetNumber() << " with seqnos (largest, smallest) " + << lhs->fd.largest_seqno << " , " << lhs->fd.smallest_seqno + << ", #" << rhs->fd.GetNumber() + << " with seqnos (largest, smallest) " + << rhs->fd.largest_seqno << " , " << rhs->fd.smallest_seqno; + return Status::Corruption("VersionBuilder", oss.str()); + } + } else if (epoch_number_requirement == + EpochNumberRequirement::kMustPresent) { + if (lhs->epoch_number == rhs->epoch_number) { + bool range_overlapped = + icmp->Compare(lhs->smallest, rhs->largest) <= 0 && + icmp->Compare(lhs->largest, rhs->smallest) >= 0; + + if (range_overlapped) { + std::ostringstream oss; + oss << "L0 files of same epoch number but overlapping range #" + << lhs->fd.GetNumber() + << " , smallest key: " << lhs->smallest.DebugString(true) + << " , largest key: " << lhs->largest.DebugString(true) + << " , epoch number: " << lhs->epoch_number << " vs. file #" + << rhs->fd.GetNumber() + << " , smallest key: " << rhs->smallest.DebugString(true) + << " , largest key: " << rhs->largest.DebugString(true) + << " , epoch number: " << rhs->epoch_number; + return Status::Corruption("VersionBuilder", oss.str()); + } + } + + if (!level_zero_cmp_by_epochno_(lhs, rhs)) { + std::ostringstream oss; + oss << "L0 files are not sorted properly: files #" + << lhs->fd.GetNumber() << " with epoch number " + << lhs->epoch_number << ", #" << rhs->fd.GetNumber() + << " with epoch number " << rhs->epoch_number; + return Status::Corruption("VersionBuilder", oss.str()); + } + } + + return Status::OK(); + }; + + const Status s = CheckConsistencyDetailsForLevel( + vstorage, /* level */ 0, l0_checker, + "VersionBuilder::CheckConsistency0", &expected_linked_ssts); + if (!s.ok()) { + return s; + } + } + + // Check L1 and up + + for (int level = 1; level < num_levels_; ++level) { + auto checker = [this, level, icmp](const FileMetaData* lhs, + const FileMetaData* rhs) { + assert(lhs); + assert(rhs); + + if (!level_nonzero_cmp_(lhs, rhs)) { + std::ostringstream oss; + oss << 'L' << level << " files are not sorted properly: files #" + << lhs->fd.GetNumber() << ", #" << rhs->fd.GetNumber(); + + return Status::Corruption("VersionBuilder", oss.str()); + } + + // Make sure there is no overlap in level + if (icmp->Compare(lhs->largest, rhs->smallest) >= 0) { + std::ostringstream oss; + oss << 'L' << level << " has overlapping ranges: file #" + << lhs->fd.GetNumber() + << " largest key: " << lhs->largest.DebugString(true) + << " vs. file #" << rhs->fd.GetNumber() + << " smallest key: " << rhs->smallest.DebugString(true); + + return Status::Corruption("VersionBuilder", oss.str()); + } + + return Status::OK(); + }; + + const Status s = CheckConsistencyDetailsForLevel( + vstorage, level, checker, "VersionBuilder::CheckConsistency1", + &expected_linked_ssts); + if (!s.ok()) { + return s; + } + } + } + + // Make sure that all blob files in the version have non-garbage data and + // the links between them and the table files are consistent. + const auto& blob_files = vstorage->GetBlobFiles(); + for (const auto& blob_file_meta : blob_files) { + assert(blob_file_meta); + + const uint64_t blob_file_number = blob_file_meta->GetBlobFileNumber(); + + if (blob_file_meta->GetGarbageBlobCount() >= + blob_file_meta->GetTotalBlobCount()) { + std::ostringstream oss; + oss << "Blob file #" << blob_file_number + << " consists entirely of garbage"; + + return Status::Corruption("VersionBuilder", oss.str()); + } + + if (blob_file_meta->GetLinkedSsts() != + expected_linked_ssts[blob_file_number]) { + std::ostringstream oss; + oss << "Links are inconsistent between table files and blob file #" + << blob_file_number; + + return Status::Corruption("VersionBuilder", oss.str()); + } + } + + Status ret_s; + TEST_SYNC_POINT_CALLBACK("VersionBuilder::CheckConsistencyBeforeReturn", + &ret_s); + return ret_s; + } + + Status CheckConsistency(const VersionStorageInfo* vstorage) const { + assert(vstorage); + + // Always run consistency checks in debug build +#ifdef NDEBUG + if (!vstorage->force_consistency_checks()) { + return Status::OK(); + } +#endif + Status s = CheckConsistencyDetails(vstorage); + if (s.IsCorruption() && s.getState()) { + // Make it clear the error is due to force_consistency_checks = 1 or + // debug build +#ifdef NDEBUG + auto prefix = "force_consistency_checks"; +#else + auto prefix = "force_consistency_checks(DEBUG)"; +#endif + s = Status::Corruption(prefix, s.getState()); + } else { + // was only expecting corruption with message, or OK + assert(s.ok()); + } + return s; + } + + bool CheckConsistencyForNumLevels() const { + // Make sure there are no files on or beyond num_levels(). + if (has_invalid_levels_) { + return false; + } + + for (const auto& pair : invalid_level_sizes_) { + const size_t level_size = pair.second; + if (level_size != 0) { + return false; + } + } + + return true; + } + + bool IsBlobFileInVersion(uint64_t blob_file_number) const { + auto mutable_it = mutable_blob_file_metas_.find(blob_file_number); + if (mutable_it != mutable_blob_file_metas_.end()) { + return true; + } + + assert(base_vstorage_); + const auto meta = base_vstorage_->GetBlobFileMetaData(blob_file_number); + + return !!meta; + } + + MutableBlobFileMetaData* GetOrCreateMutableBlobFileMetaData( + uint64_t blob_file_number) { + auto mutable_it = mutable_blob_file_metas_.find(blob_file_number); + if (mutable_it != mutable_blob_file_metas_.end()) { + return &mutable_it->second; + } + + assert(base_vstorage_); + const auto meta = base_vstorage_->GetBlobFileMetaData(blob_file_number); + + if (meta) { + mutable_it = mutable_blob_file_metas_ + .emplace(blob_file_number, MutableBlobFileMetaData(meta)) + .first; + return &mutable_it->second; + } + + return nullptr; + } + + Status ApplyBlobFileAddition(const BlobFileAddition& blob_file_addition) { + const uint64_t blob_file_number = blob_file_addition.GetBlobFileNumber(); + + if (IsBlobFileInVersion(blob_file_number)) { + std::ostringstream oss; + oss << "Blob file #" << blob_file_number << " already added"; + + return Status::Corruption("VersionBuilder", oss.str()); + } + + // Note: we use C++11 for now but in C++14, this could be done in a more + // elegant way using generalized lambda capture. + VersionSet* const vs = version_set_; + const ImmutableCFOptions* const ioptions = ioptions_; + + auto deleter = [vs, ioptions](SharedBlobFileMetaData* shared_meta) { + if (vs) { + assert(ioptions); + assert(!ioptions->cf_paths.empty()); + assert(shared_meta); + + vs->AddObsoleteBlobFile(shared_meta->GetBlobFileNumber(), + ioptions->cf_paths.front().path); + } + + delete shared_meta; + }; + + auto shared_meta = SharedBlobFileMetaData::Create( + blob_file_number, blob_file_addition.GetTotalBlobCount(), + blob_file_addition.GetTotalBlobBytes(), + blob_file_addition.GetChecksumMethod(), + blob_file_addition.GetChecksumValue(), deleter); + + mutable_blob_file_metas_.emplace( + blob_file_number, MutableBlobFileMetaData(std::move(shared_meta))); + + return Status::OK(); + } + + Status ApplyBlobFileGarbage(const BlobFileGarbage& blob_file_garbage) { + const uint64_t blob_file_number = blob_file_garbage.GetBlobFileNumber(); + + MutableBlobFileMetaData* const mutable_meta = + GetOrCreateMutableBlobFileMetaData(blob_file_number); + + if (!mutable_meta) { + std::ostringstream oss; + oss << "Blob file #" << blob_file_number << " not found"; + + return Status::Corruption("VersionBuilder", oss.str()); + } + + if (!mutable_meta->AddGarbage(blob_file_garbage.GetGarbageBlobCount(), + blob_file_garbage.GetGarbageBlobBytes())) { + std::ostringstream oss; + oss << "Garbage overflow for blob file #" << blob_file_number; + return Status::Corruption("VersionBuilder", oss.str()); + } + + return Status::OK(); + } + + int GetCurrentLevelForTableFile(uint64_t file_number) const { + auto it = table_file_levels_.find(file_number); + if (it != table_file_levels_.end()) { + return it->second; + } + + assert(base_vstorage_); + return base_vstorage_->GetFileLocation(file_number).GetLevel(); + } + + uint64_t GetOldestBlobFileNumberForTableFile(int level, + uint64_t file_number) const { + assert(level < num_levels_); + + const auto& added_files = levels_[level].added_files; + + auto it = added_files.find(file_number); + if (it != added_files.end()) { + const FileMetaData* const meta = it->second; + assert(meta); + + return meta->oldest_blob_file_number; + } + + assert(base_vstorage_); + const FileMetaData* const meta = + base_vstorage_->GetFileMetaDataByNumber(file_number); + assert(meta); + + return meta->oldest_blob_file_number; + } + + Status ApplyFileDeletion(int level, uint64_t file_number) { + assert(level != VersionStorageInfo::FileLocation::Invalid().GetLevel()); + + const int current_level = GetCurrentLevelForTableFile(file_number); + + if (level != current_level) { + if (level >= num_levels_) { + has_invalid_levels_ = true; + } + + std::ostringstream oss; + oss << "Cannot delete table file #" << file_number << " from level " + << level << " since it is "; + if (current_level == + VersionStorageInfo::FileLocation::Invalid().GetLevel()) { + oss << "not in the LSM tree"; + } else { + oss << "on level " << current_level; + } + + return Status::Corruption("VersionBuilder", oss.str()); + } + + if (level >= num_levels_) { + assert(invalid_level_sizes_[level] > 0); + --invalid_level_sizes_[level]; + + table_file_levels_[file_number] = + VersionStorageInfo::FileLocation::Invalid().GetLevel(); + + return Status::OK(); + } + + const uint64_t blob_file_number = + GetOldestBlobFileNumberForTableFile(level, file_number); + + if (blob_file_number != kInvalidBlobFileNumber) { + MutableBlobFileMetaData* const mutable_meta = + GetOrCreateMutableBlobFileMetaData(blob_file_number); + if (mutable_meta) { + mutable_meta->UnlinkSst(file_number); + } + } + + auto& level_state = levels_[level]; + + auto& add_files = level_state.added_files; + auto add_it = add_files.find(file_number); + if (add_it != add_files.end()) { + UnrefFile(add_it->second); + add_files.erase(add_it); + } + + auto& del_files = level_state.deleted_files; + assert(del_files.find(file_number) == del_files.end()); + del_files.emplace(file_number); + + table_file_levels_[file_number] = + VersionStorageInfo::FileLocation::Invalid().GetLevel(); + + return Status::OK(); + } + + Status ApplyFileAddition(int level, const FileMetaData& meta) { + assert(level != VersionStorageInfo::FileLocation::Invalid().GetLevel()); + + const uint64_t file_number = meta.fd.GetNumber(); + + const int current_level = GetCurrentLevelForTableFile(file_number); + + if (current_level != + VersionStorageInfo::FileLocation::Invalid().GetLevel()) { + if (level >= num_levels_) { + has_invalid_levels_ = true; + } + + std::ostringstream oss; + oss << "Cannot add table file #" << file_number << " to level " << level + << " since it is already in the LSM tree on level " << current_level; + return Status::Corruption("VersionBuilder", oss.str()); + } + + if (level >= num_levels_) { + ++invalid_level_sizes_[level]; + table_file_levels_[file_number] = level; + + return Status::OK(); + } + + auto& level_state = levels_[level]; + + auto& del_files = level_state.deleted_files; + auto del_it = del_files.find(file_number); + if (del_it != del_files.end()) { + del_files.erase(del_it); + } + + FileMetaData* const f = new FileMetaData(meta); + f->refs = 1; + + if (file_metadata_cache_res_mgr_) { + Status s = file_metadata_cache_res_mgr_->UpdateCacheReservation( + f->ApproximateMemoryUsage(), true /* increase */); + if (!s.ok()) { + delete f; + s = Status::MemoryLimit( + "Can't allocate " + + kCacheEntryRoleToCamelString[static_cast( + CacheEntryRole::kFileMetadata)] + + " due to exceeding the memory limit " + "based on " + "cache capacity"); + return s; + } + } + + auto& add_files = level_state.added_files; + assert(add_files.find(file_number) == add_files.end()); + add_files.emplace(file_number, f); + + const uint64_t blob_file_number = f->oldest_blob_file_number; + + if (blob_file_number != kInvalidBlobFileNumber) { + MutableBlobFileMetaData* const mutable_meta = + GetOrCreateMutableBlobFileMetaData(blob_file_number); + if (mutable_meta) { + mutable_meta->LinkSst(file_number); + } + } + + table_file_levels_[file_number] = level; + + return Status::OK(); + } + + Status ApplyCompactCursors(int level, + const InternalKey& smallest_uncompacted_key) { + if (level < 0) { + std::ostringstream oss; + oss << "Cannot add compact cursor (" << level << "," + << smallest_uncompacted_key.Encode().ToString() + << " due to invalid level (level = " << level << ")"; + return Status::Corruption("VersionBuilder", oss.str()); + } + if (level < num_levels_) { + // Omit levels (>= num_levels_) when re-open with shrinking num_levels_ + updated_compact_cursors_[level] = smallest_uncompacted_key; + } + return Status::OK(); + } + + // Apply all of the edits in *edit to the current state. + Status Apply(const VersionEdit* edit) { + { + const Status s = CheckConsistency(base_vstorage_); + if (!s.ok()) { + return s; + } + } + + // Note: we process the blob file related changes first because the + // table file addition/deletion logic depends on the blob files + // already being there. + + // Add new blob files + for (const auto& blob_file_addition : edit->GetBlobFileAdditions()) { + const Status s = ApplyBlobFileAddition(blob_file_addition); + if (!s.ok()) { + return s; + } + } + + // Increase the amount of garbage for blob files affected by GC + for (const auto& blob_file_garbage : edit->GetBlobFileGarbages()) { + const Status s = ApplyBlobFileGarbage(blob_file_garbage); + if (!s.ok()) { + return s; + } + } + + // Delete table files + for (const auto& deleted_file : edit->GetDeletedFiles()) { + const int level = deleted_file.first; + const uint64_t file_number = deleted_file.second; + + const Status s = ApplyFileDeletion(level, file_number); + if (!s.ok()) { + return s; + } + } + + // Add new table files + for (const auto& new_file : edit->GetNewFiles()) { + const int level = new_file.first; + const FileMetaData& meta = new_file.second; + + const Status s = ApplyFileAddition(level, meta); + if (!s.ok()) { + return s; + } + } + + // Populate compact cursors for round-robin compaction, leave + // the cursor to be empty to indicate it is invalid + for (const auto& cursor : edit->GetCompactCursors()) { + const int level = cursor.first; + const InternalKey smallest_uncompacted_key = cursor.second; + const Status s = ApplyCompactCursors(level, smallest_uncompacted_key); + if (!s.ok()) { + return s; + } + } + return Status::OK(); + } + + // Helper function template for merging the blob file metadata from the base + // version with the mutable metadata representing the state after applying the + // edits. The function objects process_base and process_mutable are + // respectively called to handle a base version object when there is no + // matching mutable object, and a mutable object when there is no matching + // base version object. process_both is called to perform the merge when a + // given blob file appears both in the base version and the mutable list. The + // helper stops processing objects if a function object returns false. Blob + // files with a file number below first_blob_file are not processed. + template + void MergeBlobFileMetas(uint64_t first_blob_file, ProcessBase process_base, + ProcessMutable process_mutable, + ProcessBoth process_both) const { + assert(base_vstorage_); + + auto base_it = base_vstorage_->GetBlobFileMetaDataLB(first_blob_file); + const auto base_it_end = base_vstorage_->GetBlobFiles().end(); + + auto mutable_it = mutable_blob_file_metas_.lower_bound(first_blob_file); + const auto mutable_it_end = mutable_blob_file_metas_.end(); + + while (base_it != base_it_end && mutable_it != mutable_it_end) { + const auto& base_meta = *base_it; + assert(base_meta); + + const uint64_t base_blob_file_number = base_meta->GetBlobFileNumber(); + const uint64_t mutable_blob_file_number = mutable_it->first; + + if (base_blob_file_number < mutable_blob_file_number) { + if (!process_base(base_meta)) { + return; + } + + ++base_it; + } else if (mutable_blob_file_number < base_blob_file_number) { + const auto& mutable_meta = mutable_it->second; + + if (!process_mutable(mutable_meta)) { + return; + } + + ++mutable_it; + } else { + assert(base_blob_file_number == mutable_blob_file_number); + + const auto& mutable_meta = mutable_it->second; + + if (!process_both(base_meta, mutable_meta)) { + return; + } + + ++base_it; + ++mutable_it; + } + } + + while (base_it != base_it_end) { + const auto& base_meta = *base_it; + + if (!process_base(base_meta)) { + return; + } + + ++base_it; + } + + while (mutable_it != mutable_it_end) { + const auto& mutable_meta = mutable_it->second; + + if (!process_mutable(mutable_meta)) { + return; + } + + ++mutable_it; + } + } + + // Helper function template for finding the first blob file that has linked + // SSTs. + template + static bool CheckLinkedSsts(const Meta& meta, + uint64_t* min_oldest_blob_file_num) { + assert(min_oldest_blob_file_num); + + if (!meta.GetLinkedSsts().empty()) { + assert(*min_oldest_blob_file_num == kInvalidBlobFileNumber); + + *min_oldest_blob_file_num = meta.GetBlobFileNumber(); + + return false; + } + + return true; + } + + // Find the oldest blob file that has linked SSTs. + uint64_t GetMinOldestBlobFileNumber() const { + uint64_t min_oldest_blob_file_num = kInvalidBlobFileNumber; + + auto process_base = + [&min_oldest_blob_file_num]( + const std::shared_ptr& base_meta) { + assert(base_meta); + + return CheckLinkedSsts(*base_meta, &min_oldest_blob_file_num); + }; + + auto process_mutable = [&min_oldest_blob_file_num]( + const MutableBlobFileMetaData& mutable_meta) { + return CheckLinkedSsts(mutable_meta, &min_oldest_blob_file_num); + }; + + auto process_both = [&min_oldest_blob_file_num]( + const std::shared_ptr& base_meta, + const MutableBlobFileMetaData& mutable_meta) { +#ifndef NDEBUG + assert(base_meta); + assert(base_meta->GetSharedMeta() == mutable_meta.GetSharedMeta()); +#else + (void)base_meta; +#endif + + // Look at mutable_meta since it supersedes *base_meta + return CheckLinkedSsts(mutable_meta, &min_oldest_blob_file_num); + }; + + MergeBlobFileMetas(kInvalidBlobFileNumber, process_base, process_mutable, + process_both); + + return min_oldest_blob_file_num; + } + + static std::shared_ptr CreateBlobFileMetaData( + const MutableBlobFileMetaData& mutable_meta) { + return BlobFileMetaData::Create( + mutable_meta.GetSharedMeta(), mutable_meta.GetLinkedSsts(), + mutable_meta.GetGarbageBlobCount(), mutable_meta.GetGarbageBlobBytes()); + } + + // Add the blob file specified by meta to *vstorage if it is determined to + // contain valid data (blobs). + template + static void AddBlobFileIfNeeded(VersionStorageInfo* vstorage, Meta&& meta) { + assert(vstorage); + assert(meta); + + if (meta->GetLinkedSsts().empty() && + meta->GetGarbageBlobCount() >= meta->GetTotalBlobCount()) { + return; + } + + vstorage->AddBlobFile(std::forward(meta)); + } + + // Merge the blob file metadata from the base version with the changes (edits) + // applied, and save the result into *vstorage. + void SaveBlobFilesTo(VersionStorageInfo* vstorage) const { + assert(vstorage); + + assert(base_vstorage_); + vstorage->ReserveBlob(base_vstorage_->GetBlobFiles().size() + + mutable_blob_file_metas_.size()); + + const uint64_t oldest_blob_file_with_linked_ssts = + GetMinOldestBlobFileNumber(); + + auto process_base = + [vstorage](const std::shared_ptr& base_meta) { + assert(base_meta); + + AddBlobFileIfNeeded(vstorage, base_meta); + + return true; + }; + + auto process_mutable = + [vstorage](const MutableBlobFileMetaData& mutable_meta) { + AddBlobFileIfNeeded(vstorage, CreateBlobFileMetaData(mutable_meta)); + + return true; + }; + + auto process_both = [vstorage]( + const std::shared_ptr& base_meta, + const MutableBlobFileMetaData& mutable_meta) { + assert(base_meta); + assert(base_meta->GetSharedMeta() == mutable_meta.GetSharedMeta()); + + if (!mutable_meta.HasDelta()) { + assert(base_meta->GetGarbageBlobCount() == + mutable_meta.GetGarbageBlobCount()); + assert(base_meta->GetGarbageBlobBytes() == + mutable_meta.GetGarbageBlobBytes()); + assert(base_meta->GetLinkedSsts() == mutable_meta.GetLinkedSsts()); + + AddBlobFileIfNeeded(vstorage, base_meta); + + return true; + } + + AddBlobFileIfNeeded(vstorage, CreateBlobFileMetaData(mutable_meta)); + + return true; + }; + + MergeBlobFileMetas(oldest_blob_file_with_linked_ssts, process_base, + process_mutable, process_both); + } + + void MaybeAddFile(VersionStorageInfo* vstorage, int level, + FileMetaData* f) const { + const uint64_t file_number = f->fd.GetNumber(); + + const auto& level_state = levels_[level]; + + const auto& del_files = level_state.deleted_files; + const auto del_it = del_files.find(file_number); + + if (del_it != del_files.end()) { + // f is to-be-deleted table file + vstorage->RemoveCurrentStats(f); + } else { + const auto& add_files = level_state.added_files; + const auto add_it = add_files.find(file_number); + + // Note: if the file appears both in the base version and in the added + // list, the added FileMetaData supersedes the one in the base version. + if (add_it != add_files.end() && add_it->second != f) { + vstorage->RemoveCurrentStats(f); + } else { + vstorage->AddFile(level, f); + } + } + } + + template + void SaveSSTFilesTo(VersionStorageInfo* vstorage, int level, Cmp cmp) const { + // Merge the set of added files with the set of pre-existing files. + // Drop any deleted files. Store the result in *vstorage. + const auto& base_files = base_vstorage_->LevelFiles(level); + const auto& unordered_added_files = levels_[level].added_files; + vstorage->Reserve(level, base_files.size() + unordered_added_files.size()); + + // Sort added files for the level. + std::vector added_files; + added_files.reserve(unordered_added_files.size()); + for (const auto& pair : unordered_added_files) { + added_files.push_back(pair.second); + } + std::sort(added_files.begin(), added_files.end(), cmp); + + auto base_iter = base_files.begin(); + auto base_end = base_files.end(); + auto added_iter = added_files.begin(); + auto added_end = added_files.end(); + while (added_iter != added_end || base_iter != base_end) { + if (base_iter == base_end || + (added_iter != added_end && cmp(*added_iter, *base_iter))) { + MaybeAddFile(vstorage, level, *added_iter++); + } else { + MaybeAddFile(vstorage, level, *base_iter++); + } + } + } + + bool PromoteEpochNumberRequirementIfNeeded( + VersionStorageInfo* vstorage) const { + if (vstorage->HasMissingEpochNumber()) { + return false; + } + + for (int level = 0; level < num_levels_; ++level) { + for (const auto& pair : levels_[level].added_files) { + const FileMetaData* f = pair.second; + if (f->epoch_number == kUnknownEpochNumber) { + return false; + } + } + } + + vstorage->SetEpochNumberRequirement(EpochNumberRequirement::kMustPresent); + return true; + } + + void SaveSSTFilesTo(VersionStorageInfo* vstorage) const { + assert(vstorage); + + if (!num_levels_) { + return; + } + + EpochNumberRequirement epoch_number_requirement = + vstorage->GetEpochNumberRequirement(); + + if (epoch_number_requirement == EpochNumberRequirement::kMightMissing) { + bool promoted = PromoteEpochNumberRequirementIfNeeded(vstorage); + if (promoted) { + epoch_number_requirement = vstorage->GetEpochNumberRequirement(); + } + } + + if (epoch_number_requirement == EpochNumberRequirement::kMightMissing) { + SaveSSTFilesTo(vstorage, /* level */ 0, level_zero_cmp_by_seqno_); + } else { + SaveSSTFilesTo(vstorage, /* level */ 0, level_zero_cmp_by_epochno_); + } + + for (int level = 1; level < num_levels_; ++level) { + SaveSSTFilesTo(vstorage, level, level_nonzero_cmp_); + } + } + + void SaveCompactCursorsTo(VersionStorageInfo* vstorage) const { + for (auto iter = updated_compact_cursors_.begin(); + iter != updated_compact_cursors_.end(); iter++) { + vstorage->AddCursorForOneLevel(iter->first, iter->second); + } + } + + // Save the current state in *vstorage. + Status SaveTo(VersionStorageInfo* vstorage) const { + Status s; + +#ifndef NDEBUG + // The same check is done within Apply() so we skip it in release mode. + s = CheckConsistency(base_vstorage_); + if (!s.ok()) { + return s; + } +#endif // NDEBUG + + s = CheckConsistency(vstorage); + if (!s.ok()) { + return s; + } + + SaveSSTFilesTo(vstorage); + + SaveBlobFilesTo(vstorage); + + SaveCompactCursorsTo(vstorage); + + s = CheckConsistency(vstorage); + return s; + } + + Status LoadTableHandlers( + InternalStats* internal_stats, int max_threads, + bool prefetch_index_and_filter_in_cache, bool is_initial_load, + const std::shared_ptr& prefix_extractor, + size_t max_file_size_for_l0_meta_pin, const ReadOptions& read_options, + uint8_t block_protection_bytes_per_key) { + assert(table_cache_ != nullptr); + + size_t table_cache_capacity = + table_cache_->get_cache().get()->GetCapacity(); + bool always_load = (table_cache_capacity == TableCache::kInfiniteCapacity); + size_t max_load = std::numeric_limits::max(); + + if (!always_load) { + // If it is initial loading and not set to always loading all the + // files, we only load up to kInitialLoadLimit files, to limit the + // time reopening the DB. + const size_t kInitialLoadLimit = 16; + size_t load_limit; + // If the table cache is not 1/4 full, we pin the table handle to + // file metadata to avoid the cache read costs when reading the file. + // The downside of pinning those files is that LRU won't be followed + // for those files. This doesn't matter much because if number of files + // of the DB excceeds table cache capacity, eventually no table reader + // will be pinned and LRU will be followed. + if (is_initial_load) { + load_limit = std::min(kInitialLoadLimit, table_cache_capacity / 4); + } else { + load_limit = table_cache_capacity / 4; + } + + size_t table_cache_usage = table_cache_->get_cache().get()->GetUsage(); + if (table_cache_usage >= load_limit) { + // TODO (yanqin) find a suitable status code. + return Status::OK(); + } else { + max_load = load_limit - table_cache_usage; + } + } + + // + std::vector> files_meta; + std::vector statuses; + for (int level = 0; level < num_levels_; level++) { + for (auto& file_meta_pair : levels_[level].added_files) { + auto* file_meta = file_meta_pair.second; + // If the file has been opened before, just skip it. + if (!file_meta->table_reader_handle) { + files_meta.emplace_back(file_meta, level); + statuses.emplace_back(Status::OK()); + } + if (files_meta.size() >= max_load) { + break; + } + } + if (files_meta.size() >= max_load) { + break; + } + } + + std::atomic next_file_meta_idx(0); + std::function load_handlers_func([&]() { + while (true) { + size_t file_idx = next_file_meta_idx.fetch_add(1); + if (file_idx >= files_meta.size()) { + break; + } + + auto* file_meta = files_meta[file_idx].first; + int level = files_meta[file_idx].second; + TableCache::TypedHandle* handle = nullptr; + statuses[file_idx] = table_cache_->FindTable( + read_options, file_options_, + *(base_vstorage_->InternalComparator()), *file_meta, &handle, + block_protection_bytes_per_key, prefix_extractor, false /*no_io */, + internal_stats->GetFileReadHist(level), false, level, + prefetch_index_and_filter_in_cache, max_file_size_for_l0_meta_pin, + file_meta->temperature); + if (handle != nullptr) { + file_meta->table_reader_handle = handle; + // Load table_reader + file_meta->fd.table_reader = table_cache_->get_cache().Value(handle); + } + } + }); + + std::vector threads; + for (int i = 1; i < max_threads; i++) { + threads.emplace_back(load_handlers_func); + } + load_handlers_func(); + for (auto& t : threads) { + t.join(); + } + Status ret; + for (const auto& s : statuses) { + if (!s.ok()) { + if (ret.ok()) { + ret = s; + } + } + } + return ret; + } +}; + +VersionBuilder::VersionBuilder( + const FileOptions& file_options, const ImmutableCFOptions* ioptions, + TableCache* table_cache, VersionStorageInfo* base_vstorage, + VersionSet* version_set, + std::shared_ptr file_metadata_cache_res_mgr) + : rep_(new Rep(file_options, ioptions, table_cache, base_vstorage, + version_set, file_metadata_cache_res_mgr)) {} + +VersionBuilder::~VersionBuilder() = default; + +bool VersionBuilder::CheckConsistencyForNumLevels() { + return rep_->CheckConsistencyForNumLevels(); +} + +Status VersionBuilder::Apply(const VersionEdit* edit) { + return rep_->Apply(edit); +} + +Status VersionBuilder::SaveTo(VersionStorageInfo* vstorage) const { + return rep_->SaveTo(vstorage); +} + +Status VersionBuilder::LoadTableHandlers( + InternalStats* internal_stats, int max_threads, + bool prefetch_index_and_filter_in_cache, bool is_initial_load, + const std::shared_ptr& prefix_extractor, + size_t max_file_size_for_l0_meta_pin, const ReadOptions& read_options, + uint8_t block_protection_bytes_per_key) { + return rep_->LoadTableHandlers( + internal_stats, max_threads, prefetch_index_and_filter_in_cache, + is_initial_load, prefix_extractor, max_file_size_for_l0_meta_pin, + read_options, block_protection_bytes_per_key); +} + +uint64_t VersionBuilder::GetMinOldestBlobFileNumber() const { + return rep_->GetMinOldestBlobFileNumber(); +} + +BaseReferencedVersionBuilder::BaseReferencedVersionBuilder( + ColumnFamilyData* cfd) + : version_builder_(new VersionBuilder( + cfd->current()->version_set()->file_options(), cfd->ioptions(), + cfd->table_cache(), cfd->current()->storage_info(), + cfd->current()->version_set(), + cfd->GetFileMetadataCacheReservationManager())), + version_(cfd->current()) { + version_->Ref(); +} + +BaseReferencedVersionBuilder::BaseReferencedVersionBuilder( + ColumnFamilyData* cfd, Version* v) + : version_builder_(new VersionBuilder( + cfd->current()->version_set()->file_options(), cfd->ioptions(), + cfd->table_cache(), v->storage_info(), v->version_set(), + cfd->GetFileMetadataCacheReservationManager())), + version_(v) { + assert(version_ != cfd->current()); +} + +BaseReferencedVersionBuilder::~BaseReferencedVersionBuilder() { + version_->Unref(); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/version_builder.h b/librocksdb-sys/rocksdb/db/version_builder.h new file mode 100644 index 0000000..fb2a304 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/version_builder.h @@ -0,0 +1,93 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +#pragma once + +#include + +#include "db/version_edit.h" +#include "rocksdb/file_system.h" +#include "rocksdb/metadata.h" +#include "rocksdb/slice_transform.h" + +namespace ROCKSDB_NAMESPACE { + +struct ImmutableCFOptions; +class TableCache; +class VersionStorageInfo; +class VersionEdit; +struct FileMetaData; +class InternalStats; +class Version; +class VersionSet; +class ColumnFamilyData; +class CacheReservationManager; + +// A helper class so we can efficiently apply a whole sequence +// of edits to a particular state without creating intermediate +// Versions that contain full copies of the intermediate state. +class VersionBuilder { + public: + VersionBuilder(const FileOptions& file_options, + const ImmutableCFOptions* ioptions, TableCache* table_cache, + VersionStorageInfo* base_vstorage, VersionSet* version_set, + std::shared_ptr + file_metadata_cache_res_mgr = nullptr); + ~VersionBuilder(); + + bool CheckConsistencyForNumLevels(); + Status Apply(const VersionEdit* edit); + Status SaveTo(VersionStorageInfo* vstorage) const; + Status LoadTableHandlers( + InternalStats* internal_stats, int max_threads, + bool prefetch_index_and_filter_in_cache, bool is_initial_load, + const std::shared_ptr& prefix_extractor, + size_t max_file_size_for_l0_meta_pin, const ReadOptions& read_options, + uint8_t block_protection_bytes_per_key); + uint64_t GetMinOldestBlobFileNumber() const; + + private: + class Rep; + std::unique_ptr rep_; +}; + +// A wrapper of version builder which references the current version in +// constructor and unref it in the destructor. +// Both of the constructor and destructor need to be called inside DB Mutex. +class BaseReferencedVersionBuilder { + public: + explicit BaseReferencedVersionBuilder(ColumnFamilyData* cfd); + BaseReferencedVersionBuilder(ColumnFamilyData* cfd, Version* v); + ~BaseReferencedVersionBuilder(); + VersionBuilder* version_builder() const { return version_builder_.get(); } + + private: + std::unique_ptr version_builder_; + Version* version_; +}; + +class NewestFirstBySeqNo { + public: + bool operator()(const FileMetaData* lhs, const FileMetaData* rhs) const { + assert(lhs); + assert(rhs); + + if (lhs->fd.largest_seqno != rhs->fd.largest_seqno) { + return lhs->fd.largest_seqno > rhs->fd.largest_seqno; + } + + if (lhs->fd.smallest_seqno != rhs->fd.smallest_seqno) { + return lhs->fd.smallest_seqno > rhs->fd.smallest_seqno; + } + + // Break ties by file number + return lhs->fd.GetNumber() > rhs->fd.GetNumber(); + } +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/version_builder_test.cc b/librocksdb-sys/rocksdb/db/version_builder_test.cc new file mode 100644 index 0000000..ac80be7 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/version_builder_test.cc @@ -0,0 +1,1849 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include +#include +#include +#include +#include + +#include "db/version_edit.h" +#include "db/version_set.h" +#include "rocksdb/advanced_options.h" +#include "table/unique_id_impl.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +class VersionBuilderTest : public testing::Test { + public: + const Comparator* ucmp_; + InternalKeyComparator icmp_; + Options options_; + ImmutableOptions ioptions_; + MutableCFOptions mutable_cf_options_; + VersionStorageInfo vstorage_; + uint32_t file_num_; + CompactionOptionsFIFO fifo_options_; + std::vector size_being_compacted_; + + VersionBuilderTest() + : ucmp_(BytewiseComparator()), + icmp_(ucmp_), + ioptions_(options_), + mutable_cf_options_(options_), + vstorage_(&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, + nullptr, false), + file_num_(1) { + mutable_cf_options_.RefreshDerivedOptions(ioptions_); + size_being_compacted_.resize(options_.num_levels); + } + + ~VersionBuilderTest() override { + for (int i = 0; i < vstorage_.num_levels(); i++) { + for (auto* f : vstorage_.LevelFiles(i)) { + if (--f->refs == 0) { + delete f; + } + } + } + } + + InternalKey GetInternalKey(const char* ukey, + SequenceNumber smallest_seq = 100) { + return InternalKey(ukey, smallest_seq, kTypeValue); + } + + void Add(int level, uint64_t file_number, const char* smallest, + const char* largest, uint64_t file_size = 0, uint32_t path_id = 0, + SequenceNumber smallest_seq = 100, SequenceNumber largest_seq = 100, + uint64_t num_entries = 0, uint64_t num_deletions = 0, + bool sampled = false, SequenceNumber smallest_seqno = 0, + SequenceNumber largest_seqno = 0, + uint64_t oldest_blob_file_number = kInvalidBlobFileNumber, + uint64_t epoch_number = kUnknownEpochNumber) { + assert(level < vstorage_.num_levels()); + FileMetaData* f = new FileMetaData( + file_number, path_id, file_size, GetInternalKey(smallest, smallest_seq), + GetInternalKey(largest, largest_seq), smallest_seqno, largest_seqno, + /* marked_for_compact */ false, Temperature::kUnknown, + oldest_blob_file_number, kUnknownOldestAncesterTime, + kUnknownFileCreationTime, epoch_number, kUnknownFileChecksum, + kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0, + /* user_defined_timestamps_persisted */ true); + f->compensated_file_size = file_size; + f->num_entries = num_entries; + f->num_deletions = num_deletions; + vstorage_.AddFile(level, f); + if (sampled) { + f->init_stats_from_file = true; + vstorage_.UpdateAccumulatedStats(f); + } + } + + void AddBlob(uint64_t blob_file_number, uint64_t total_blob_count, + uint64_t total_blob_bytes, std::string checksum_method, + std::string checksum_value, + BlobFileMetaData::LinkedSsts linked_ssts, + uint64_t garbage_blob_count, uint64_t garbage_blob_bytes) { + auto shared_meta = SharedBlobFileMetaData::Create( + blob_file_number, total_blob_count, total_blob_bytes, + std::move(checksum_method), std::move(checksum_value)); + auto meta = + BlobFileMetaData::Create(std::move(shared_meta), std::move(linked_ssts), + garbage_blob_count, garbage_blob_bytes); + + vstorage_.AddBlobFile(std::move(meta)); + } + + void AddDummyFile(uint64_t table_file_number, uint64_t blob_file_number, + uint64_t epoch_number) { + constexpr int level = 0; + constexpr char smallest[] = "bar"; + constexpr char largest[] = "foo"; + constexpr uint64_t file_size = 100; + constexpr uint32_t path_id = 0; + constexpr SequenceNumber smallest_seq = 0; + constexpr SequenceNumber largest_seq = 0; + constexpr uint64_t num_entries = 0; + constexpr uint64_t num_deletions = 0; + constexpr bool sampled = false; + + Add(level, table_file_number, smallest, largest, file_size, path_id, + smallest_seq, largest_seq, num_entries, num_deletions, sampled, + smallest_seq, largest_seq, blob_file_number, epoch_number); + } + + void AddDummyFileToEdit(VersionEdit* edit, uint64_t table_file_number, + uint64_t blob_file_number, uint64_t epoch_number) { + assert(edit); + + constexpr int level = 0; + constexpr uint32_t path_id = 0; + constexpr uint64_t file_size = 100; + constexpr char smallest[] = "bar"; + constexpr char largest[] = "foo"; + constexpr SequenceNumber smallest_seqno = 100; + constexpr SequenceNumber largest_seqno = 300; + constexpr bool marked_for_compaction = false; + + edit->AddFile( + level, table_file_number, path_id, file_size, GetInternalKey(smallest), + GetInternalKey(largest), smallest_seqno, largest_seqno, + marked_for_compaction, Temperature::kUnknown, blob_file_number, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, epoch_number, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, + 0, 0, /* user_defined_timestamps_persisted */ true); + } + + void UpdateVersionStorageInfo(VersionStorageInfo* vstorage) { + assert(vstorage); + + vstorage->PrepareForVersionAppend(ioptions_, mutable_cf_options_); + vstorage->SetFinalized(); + } + + void UpdateVersionStorageInfo() { UpdateVersionStorageInfo(&vstorage_); } +}; + +void UnrefFilesInVersion(VersionStorageInfo* new_vstorage) { + for (int i = 0; i < new_vstorage->num_levels(); i++) { + for (auto* f : new_vstorage->LevelFiles(i)) { + if (--f->refs == 0) { + delete f; + } + } + } +} + +TEST_F(VersionBuilderTest, ApplyAndSaveTo) { + Add(0, 1U, "150", "200", 100U, /*path_id*/ 0, + /*smallest_seq*/ 100, /*largest_seq*/ 100, + /*num_entries*/ 0, /*num_deletions*/ 0, + /*sampled*/ false, /*smallest_seqno*/ 0, + /*largest_seqno*/ 0, + /*oldest_blob_file_number*/ kInvalidBlobFileNumber, + /*epoch_number*/ 1); + + Add(1, 66U, "150", "200", 100U); + Add(1, 88U, "201", "300", 100U); + + Add(2, 6U, "150", "179", 100U); + Add(2, 7U, "180", "220", 100U); + Add(2, 8U, "221", "300", 100U); + + Add(3, 26U, "150", "170", 100U); + Add(3, 27U, "171", "179", 100U); + Add(3, 28U, "191", "220", 100U); + Add(3, 29U, "221", "300", 100U); + + UpdateVersionStorageInfo(); + + VersionEdit version_edit; + version_edit.AddFile( + 2, 666, 0, 100U, GetInternalKey("301"), GetInternalKey("350"), 200, 200, + false, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, + 0, /* user_defined_timestamps_persisted */ true); + version_edit.DeleteFile(3, 27U); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder version_builder(env_options, &ioptions_, table_cache, + &vstorage_, version_set); + + VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, + kCompactionStyleLevel, nullptr, false); + ASSERT_OK(version_builder.Apply(&version_edit)); + ASSERT_OK(version_builder.SaveTo(&new_vstorage)); + + UpdateVersionStorageInfo(&new_vstorage); + + ASSERT_EQ(400U, new_vstorage.NumLevelBytes(2)); + ASSERT_EQ(300U, new_vstorage.NumLevelBytes(3)); + + UnrefFilesInVersion(&new_vstorage); +} + +TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic) { + ioptions_.level_compaction_dynamic_level_bytes = true; + + Add(0, 1U, "150", "200", 100U, 0, 200U, 200U, 0, 0, false, 200U, 200U, + /*oldest_blob_file_number*/ kInvalidBlobFileNumber, + /*epoch_number*/ 2); + Add(0, 88U, "201", "300", 100U, 0, 100U, 100U, 0, 0, false, 100U, 100U, + /*oldest_blob_file_number*/ kInvalidBlobFileNumber, + /*epoch_number*/ 1); + + Add(4, 6U, "150", "179", 100U); + Add(4, 7U, "180", "220", 100U); + Add(4, 8U, "221", "300", 100U); + + Add(5, 26U, "150", "170", 100U); + Add(5, 27U, "171", "179", 100U); + + UpdateVersionStorageInfo(); + + VersionEdit version_edit; + version_edit.AddFile( + 3, 666, 0, 100U, GetInternalKey("301"), GetInternalKey("350"), 200, 200, + false, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, + 0, /* user_defined_timestamps_persisted */ true); + + version_edit.DeleteFile(0, 1U); + version_edit.DeleteFile(0, 88U); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder version_builder(env_options, &ioptions_, table_cache, + &vstorage_, version_set); + + VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, + kCompactionStyleLevel, nullptr, false); + ASSERT_OK(version_builder.Apply(&version_edit)); + ASSERT_OK(version_builder.SaveTo(&new_vstorage)); + + UpdateVersionStorageInfo(&new_vstorage); + + ASSERT_EQ(0U, new_vstorage.NumLevelBytes(0)); + ASSERT_EQ(100U, new_vstorage.NumLevelBytes(3)); + ASSERT_EQ(300U, new_vstorage.NumLevelBytes(4)); + ASSERT_EQ(200U, new_vstorage.NumLevelBytes(5)); + + UnrefFilesInVersion(&new_vstorage); +} + +TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic2) { + ioptions_.level_compaction_dynamic_level_bytes = true; + + Add(0, 1U, "150", "200", 100U, 0, 200U, 200U, 0, 0, false, 200U, 200U, + /*oldest_blob_file_number*/ kInvalidBlobFileNumber, + /*epoch_number*/ 2); + Add(0, 88U, "201", "300", 100U, 0, 100U, 100U, 0, 0, false, 100U, 100U, + /*oldest_blob_file_number*/ kInvalidBlobFileNumber, + /*epoch_number*/ 1); + + Add(4, 6U, "150", "179", 100U); + Add(4, 7U, "180", "220", 100U); + Add(4, 8U, "221", "300", 100U); + + Add(5, 26U, "150", "170", 100U); + Add(5, 27U, "171", "179", 100U); + + UpdateVersionStorageInfo(); + + VersionEdit version_edit; + version_edit.AddFile( + 4, 666, 0, 100U, GetInternalKey("301"), GetInternalKey("350"), 200, 200, + false, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, + 0, /* user_defined_timestamps_persisted */ true); + version_edit.DeleteFile(0, 1U); + version_edit.DeleteFile(0, 88U); + version_edit.DeleteFile(4, 6U); + version_edit.DeleteFile(4, 7U); + version_edit.DeleteFile(4, 8U); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder version_builder(env_options, &ioptions_, table_cache, + &vstorage_, version_set); + + VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, + kCompactionStyleLevel, nullptr, false); + ASSERT_OK(version_builder.Apply(&version_edit)); + ASSERT_OK(version_builder.SaveTo(&new_vstorage)); + + UpdateVersionStorageInfo(&new_vstorage); + + ASSERT_EQ(0U, new_vstorage.NumLevelBytes(0)); + ASSERT_EQ(100U, new_vstorage.NumLevelBytes(4)); + ASSERT_EQ(200U, new_vstorage.NumLevelBytes(5)); + + UnrefFilesInVersion(&new_vstorage); +} + +TEST_F(VersionBuilderTest, ApplyMultipleAndSaveTo) { + UpdateVersionStorageInfo(); + + VersionEdit version_edit; + version_edit.AddFile( + 2, 666, 0, 100U, GetInternalKey("301"), GetInternalKey("350"), 200, 200, + false, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, + 0, /* user_defined_timestamps_persisted */ true); + version_edit.AddFile( + 2, 676, 0, 100U, GetInternalKey("401"), GetInternalKey("450"), 200, 200, + false, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, + 0, /* user_defined_timestamps_persisted */ true); + version_edit.AddFile( + 2, 636, 0, 100U, GetInternalKey("601"), GetInternalKey("650"), 200, 200, + false, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, + 0, /* user_defined_timestamps_persisted */ true); + version_edit.AddFile( + 2, 616, 0, 100U, GetInternalKey("501"), GetInternalKey("550"), 200, 200, + false, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, + 0, /* user_defined_timestamps_persisted */ true); + version_edit.AddFile( + 2, 606, 0, 100U, GetInternalKey("701"), GetInternalKey("750"), 200, 200, + false, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, + 0, /* user_defined_timestamps_persisted */ true); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder version_builder(env_options, &ioptions_, table_cache, + &vstorage_, version_set); + + VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, + kCompactionStyleLevel, nullptr, false); + ASSERT_OK(version_builder.Apply(&version_edit)); + ASSERT_OK(version_builder.SaveTo(&new_vstorage)); + + UpdateVersionStorageInfo(&new_vstorage); + + ASSERT_EQ(500U, new_vstorage.NumLevelBytes(2)); + + UnrefFilesInVersion(&new_vstorage); +} + +TEST_F(VersionBuilderTest, ApplyDeleteAndSaveTo) { + UpdateVersionStorageInfo(); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder version_builder(env_options, &ioptions_, table_cache, + &vstorage_, version_set); + + VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, + kCompactionStyleLevel, nullptr, false); + + VersionEdit version_edit; + version_edit.AddFile( + 2, 666, 0, 100U, GetInternalKey("301"), GetInternalKey("350"), 200, 200, + false, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, + 0, /* user_defined_timestamps_persisted */ true); + version_edit.AddFile( + 2, 676, 0, 100U, GetInternalKey("401"), GetInternalKey("450"), 200, 200, + false, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, + 0, /* user_defined_timestamps_persisted */ true); + version_edit.AddFile( + 2, 636, 0, 100U, GetInternalKey("601"), GetInternalKey("650"), 200, 200, + false, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, + 0, /* user_defined_timestamps_persisted */ true); + version_edit.AddFile( + 2, 616, 0, 100U, GetInternalKey("501"), GetInternalKey("550"), 200, 200, + false, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, + 0, /* user_defined_timestamps_persisted */ true); + version_edit.AddFile( + 2, 606, 0, 100U, GetInternalKey("701"), GetInternalKey("750"), 200, 200, + false, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, + 0, /* user_defined_timestamps_persisted */ true); + ASSERT_OK(version_builder.Apply(&version_edit)); + + VersionEdit version_edit2; + version_edit.AddFile( + 2, 808, 0, 100U, GetInternalKey("901"), GetInternalKey("950"), 200, 200, + false, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, + 0, /* user_defined_timestamps_persisted */ true); + version_edit2.DeleteFile(2, 616); + version_edit2.DeleteFile(2, 636); + version_edit.AddFile( + 2, 806, 0, 100U, GetInternalKey("801"), GetInternalKey("850"), 200, 200, + false, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, + 0, /* user_defined_timestamps_persisted */ true); + + ASSERT_OK(version_builder.Apply(&version_edit2)); + ASSERT_OK(version_builder.SaveTo(&new_vstorage)); + + UpdateVersionStorageInfo(&new_vstorage); + + ASSERT_EQ(300U, new_vstorage.NumLevelBytes(2)); + + UnrefFilesInVersion(&new_vstorage); +} + +TEST_F(VersionBuilderTest, ApplyFileDeletionIncorrectLevel) { + constexpr int level = 1; + constexpr uint64_t file_number = 2345; + constexpr char smallest[] = "bar"; + constexpr char largest[] = "foo"; + constexpr uint64_t file_size = 100; + + Add(level, file_number, smallest, largest, file_size); + + UpdateVersionStorageInfo(); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_, + version_set); + + VersionEdit edit; + + constexpr int incorrect_level = 3; + + edit.DeleteFile(incorrect_level, file_number); + + const Status s = builder.Apply(&edit); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), + "Cannot delete table file #2345 from level 3 since " + "it is on level 1")); +} + +TEST_F(VersionBuilderTest, ApplyFileDeletionNotInLSMTree) { + UpdateVersionStorageInfo(); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_, + version_set); + + VersionEdit edit; + + constexpr int level = 3; + constexpr uint64_t file_number = 1234; + + edit.DeleteFile(level, file_number); + + const Status s = builder.Apply(&edit); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), + "Cannot delete table file #1234 from level 3 since " + "it is not in the LSM tree")); +} + +TEST_F(VersionBuilderTest, ApplyFileDeletionAndAddition) { + constexpr int level = 1; + constexpr uint64_t file_number = 2345; + constexpr char smallest[] = "bar"; + constexpr char largest[] = "foo"; + constexpr uint64_t file_size = 10000; + constexpr uint32_t path_id = 0; + constexpr SequenceNumber smallest_seq = 100; + constexpr SequenceNumber largest_seq = 500; + constexpr uint64_t num_entries = 0; + constexpr uint64_t num_deletions = 0; + constexpr bool sampled = false; + constexpr SequenceNumber smallest_seqno = 1; + constexpr SequenceNumber largest_seqno = 1000; + + Add(level, file_number, smallest, largest, file_size, path_id, smallest_seq, + largest_seq, num_entries, num_deletions, sampled, smallest_seqno, + largest_seqno); + + UpdateVersionStorageInfo(); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_, + version_set); + + VersionEdit deletion; + + deletion.DeleteFile(level, file_number); + + ASSERT_OK(builder.Apply(&deletion)); + + VersionEdit addition; + + constexpr bool marked_for_compaction = false; + + addition.AddFile( + level, file_number, path_id, file_size, + GetInternalKey(smallest, smallest_seq), + GetInternalKey(largest, largest_seq), smallest_seqno, largest_seqno, + marked_for_compaction, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, + 0, /* user_defined_timestamps_persisted */ true); + + ASSERT_OK(builder.Apply(&addition)); + + constexpr bool force_consistency_checks = false; + VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, + kCompactionStyleLevel, &vstorage_, + force_consistency_checks); + + ASSERT_OK(builder.SaveTo(&new_vstorage)); + + UpdateVersionStorageInfo(&new_vstorage); + + ASSERT_EQ(new_vstorage.GetFileLocation(file_number).GetLevel(), level); + + UnrefFilesInVersion(&new_vstorage); +} + +TEST_F(VersionBuilderTest, ApplyFileAdditionAlreadyInBase) { + constexpr int level = 1; + constexpr uint64_t file_number = 2345; + constexpr char smallest[] = "bar"; + constexpr char largest[] = "foo"; + constexpr uint64_t file_size = 10000; + + Add(level, file_number, smallest, largest, file_size); + + UpdateVersionStorageInfo(); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_, + version_set); + + VersionEdit edit; + + constexpr int new_level = 2; + constexpr uint32_t path_id = 0; + constexpr SequenceNumber smallest_seqno = 100; + constexpr SequenceNumber largest_seqno = 1000; + constexpr bool marked_for_compaction = false; + + edit.AddFile( + new_level, file_number, path_id, file_size, GetInternalKey(smallest), + GetInternalKey(largest), smallest_seqno, largest_seqno, + marked_for_compaction, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, + 0, /* user_defined_timestamps_persisted */ true); + + const Status s = builder.Apply(&edit); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), + "Cannot add table file #2345 to level 2 since it is " + "already in the LSM tree on level 1")); +} + +TEST_F(VersionBuilderTest, ApplyFileAdditionAlreadyApplied) { + UpdateVersionStorageInfo(); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_, + version_set); + + VersionEdit edit; + + constexpr int level = 3; + constexpr uint64_t file_number = 2345; + constexpr uint32_t path_id = 0; + constexpr uint64_t file_size = 10000; + constexpr char smallest[] = "bar"; + constexpr char largest[] = "foo"; + constexpr SequenceNumber smallest_seqno = 100; + constexpr SequenceNumber largest_seqno = 1000; + constexpr bool marked_for_compaction = false; + + edit.AddFile( + level, file_number, path_id, file_size, GetInternalKey(smallest), + GetInternalKey(largest), smallest_seqno, largest_seqno, + marked_for_compaction, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, + 0, /* user_defined_timestamps_persisted */ true); + + ASSERT_OK(builder.Apply(&edit)); + + VersionEdit other_edit; + + constexpr int new_level = 2; + + other_edit.AddFile( + new_level, file_number, path_id, file_size, GetInternalKey(smallest), + GetInternalKey(largest), smallest_seqno, largest_seqno, + marked_for_compaction, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, + 0, /* user_defined_timestamps_persisted */ true); + + const Status s = builder.Apply(&other_edit); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), + "Cannot add table file #2345 to level 2 since it is " + "already in the LSM tree on level 3")); +} + +TEST_F(VersionBuilderTest, ApplyFileAdditionAndDeletion) { + UpdateVersionStorageInfo(); + + constexpr int level = 1; + constexpr uint64_t file_number = 2345; + constexpr uint32_t path_id = 0; + constexpr uint64_t file_size = 10000; + constexpr char smallest[] = "bar"; + constexpr char largest[] = "foo"; + constexpr SequenceNumber smallest_seqno = 100; + constexpr SequenceNumber largest_seqno = 1000; + constexpr bool marked_for_compaction = false; + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_, + version_set); + + VersionEdit addition; + + addition.AddFile( + level, file_number, path_id, file_size, GetInternalKey(smallest), + GetInternalKey(largest), smallest_seqno, largest_seqno, + marked_for_compaction, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, + 0, /* user_defined_timestamps_persisted */ true); + + ASSERT_OK(builder.Apply(&addition)); + + VersionEdit deletion; + + deletion.DeleteFile(level, file_number); + + ASSERT_OK(builder.Apply(&deletion)); + + constexpr bool force_consistency_checks = false; + VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, + kCompactionStyleLevel, &vstorage_, + force_consistency_checks); + + ASSERT_OK(builder.SaveTo(&new_vstorage)); + + UpdateVersionStorageInfo(&new_vstorage); + + ASSERT_FALSE(new_vstorage.GetFileLocation(file_number).IsValid()); + + UnrefFilesInVersion(&new_vstorage); +} + +TEST_F(VersionBuilderTest, ApplyBlobFileAddition) { + UpdateVersionStorageInfo(); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_, + version_set); + + VersionEdit edit; + + constexpr uint64_t blob_file_number = 1234; + constexpr uint64_t total_blob_count = 5678; + constexpr uint64_t total_blob_bytes = 999999; + constexpr char checksum_method[] = "SHA1"; + constexpr char checksum_value[] = + "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52" + "\x5c\xbd"; + + edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes, + checksum_method, checksum_value); + + // Add dummy table file to ensure the blob file is referenced. + constexpr uint64_t table_file_number = 1; + AddDummyFileToEdit(&edit, table_file_number, blob_file_number, + 1 /*epoch_number*/); + + ASSERT_OK(builder.Apply(&edit)); + + constexpr bool force_consistency_checks = false; + VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, + kCompactionStyleLevel, &vstorage_, + force_consistency_checks); + + ASSERT_OK(builder.SaveTo(&new_vstorage)); + + UpdateVersionStorageInfo(&new_vstorage); + + const auto& new_blob_files = new_vstorage.GetBlobFiles(); + ASSERT_EQ(new_blob_files.size(), 1); + + const auto new_meta = new_vstorage.GetBlobFileMetaData(blob_file_number); + + ASSERT_NE(new_meta, nullptr); + ASSERT_EQ(new_meta->GetBlobFileNumber(), blob_file_number); + ASSERT_EQ(new_meta->GetTotalBlobCount(), total_blob_count); + ASSERT_EQ(new_meta->GetTotalBlobBytes(), total_blob_bytes); + ASSERT_EQ(new_meta->GetChecksumMethod(), checksum_method); + ASSERT_EQ(new_meta->GetChecksumValue(), checksum_value); + ASSERT_EQ(new_meta->GetLinkedSsts(), + BlobFileMetaData::LinkedSsts{table_file_number}); + ASSERT_EQ(new_meta->GetGarbageBlobCount(), 0); + ASSERT_EQ(new_meta->GetGarbageBlobBytes(), 0); + + UnrefFilesInVersion(&new_vstorage); +} + +TEST_F(VersionBuilderTest, ApplyBlobFileAdditionAlreadyInBase) { + // Attempt to add a blob file that is already present in the base version. + + constexpr uint64_t blob_file_number = 1234; + constexpr uint64_t total_blob_count = 5678; + constexpr uint64_t total_blob_bytes = 999999; + constexpr char checksum_method[] = "SHA1"; + constexpr char checksum_value[] = + "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52" + "\x5c\xbd"; + constexpr uint64_t garbage_blob_count = 123; + constexpr uint64_t garbage_blob_bytes = 456789; + + AddBlob(blob_file_number, total_blob_count, total_blob_bytes, checksum_method, + checksum_value, BlobFileMetaData::LinkedSsts(), garbage_blob_count, + garbage_blob_bytes); + + UpdateVersionStorageInfo(); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_, + version_set); + + VersionEdit edit; + + edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes, + checksum_method, checksum_value); + + const Status s = builder.Apply(&edit); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "Blob file #1234 already added")); +} + +TEST_F(VersionBuilderTest, ApplyBlobFileAdditionAlreadyApplied) { + // Attempt to add the same blob file twice using version edits. + + UpdateVersionStorageInfo(); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_, + version_set); + + VersionEdit edit; + + constexpr uint64_t blob_file_number = 1234; + constexpr uint64_t total_blob_count = 5678; + constexpr uint64_t total_blob_bytes = 999999; + constexpr char checksum_method[] = "SHA1"; + constexpr char checksum_value[] = + "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52" + "\x5c\xbd"; + + edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes, + checksum_method, checksum_value); + + ASSERT_OK(builder.Apply(&edit)); + + const Status s = builder.Apply(&edit); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "Blob file #1234 already added")); +} + +TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileInBase) { + // Increase the amount of garbage for a blob file present in the base version. + + constexpr uint64_t table_file_number = 1; + constexpr uint64_t blob_file_number = 1234; + constexpr uint64_t total_blob_count = 5678; + constexpr uint64_t total_blob_bytes = 999999; + constexpr char checksum_method[] = "SHA1"; + constexpr char checksum_value[] = + "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52" + "\x5c\xbd"; + constexpr uint64_t garbage_blob_count = 123; + constexpr uint64_t garbage_blob_bytes = 456789; + + AddBlob(blob_file_number, total_blob_count, total_blob_bytes, checksum_method, + checksum_value, BlobFileMetaData::LinkedSsts{table_file_number}, + garbage_blob_count, garbage_blob_bytes); + + const auto meta = vstorage_.GetBlobFileMetaData(blob_file_number); + ASSERT_NE(meta, nullptr); + + // Add dummy table file to ensure the blob file is referenced. + AddDummyFile(table_file_number, blob_file_number, 1 /*epoch_number*/); + + UpdateVersionStorageInfo(); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_, + version_set); + + VersionEdit edit; + + constexpr uint64_t new_garbage_blob_count = 456; + constexpr uint64_t new_garbage_blob_bytes = 111111; + + edit.AddBlobFileGarbage(blob_file_number, new_garbage_blob_count, + new_garbage_blob_bytes); + + ASSERT_OK(builder.Apply(&edit)); + + constexpr bool force_consistency_checks = false; + VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, + kCompactionStyleLevel, &vstorage_, + force_consistency_checks); + + ASSERT_OK(builder.SaveTo(&new_vstorage)); + + UpdateVersionStorageInfo(&new_vstorage); + + const auto& new_blob_files = new_vstorage.GetBlobFiles(); + ASSERT_EQ(new_blob_files.size(), 1); + + const auto new_meta = new_vstorage.GetBlobFileMetaData(blob_file_number); + + ASSERT_NE(new_meta, nullptr); + ASSERT_EQ(new_meta->GetSharedMeta(), meta->GetSharedMeta()); + ASSERT_EQ(new_meta->GetBlobFileNumber(), blob_file_number); + ASSERT_EQ(new_meta->GetTotalBlobCount(), total_blob_count); + ASSERT_EQ(new_meta->GetTotalBlobBytes(), total_blob_bytes); + ASSERT_EQ(new_meta->GetChecksumMethod(), checksum_method); + ASSERT_EQ(new_meta->GetChecksumValue(), checksum_value); + ASSERT_EQ(new_meta->GetLinkedSsts(), + BlobFileMetaData::LinkedSsts{table_file_number}); + ASSERT_EQ(new_meta->GetGarbageBlobCount(), + garbage_blob_count + new_garbage_blob_count); + ASSERT_EQ(new_meta->GetGarbageBlobBytes(), + garbage_blob_bytes + new_garbage_blob_bytes); + + UnrefFilesInVersion(&new_vstorage); +} + +TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileAdditionApplied) { + // Increase the amount of garbage for a blob file added using a version edit. + + UpdateVersionStorageInfo(); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_, + version_set); + + VersionEdit addition; + + constexpr uint64_t blob_file_number = 1234; + constexpr uint64_t total_blob_count = 5678; + constexpr uint64_t total_blob_bytes = 999999; + constexpr char checksum_method[] = "SHA1"; + constexpr char checksum_value[] = + "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52" + "\x5c\xbd"; + + addition.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes, + checksum_method, checksum_value); + + // Add dummy table file to ensure the blob file is referenced. + constexpr uint64_t table_file_number = 1; + AddDummyFileToEdit(&addition, table_file_number, blob_file_number, + 1 /*epoch_number*/); + + ASSERT_OK(builder.Apply(&addition)); + + constexpr uint64_t garbage_blob_count = 123; + constexpr uint64_t garbage_blob_bytes = 456789; + + VersionEdit garbage; + + garbage.AddBlobFileGarbage(blob_file_number, garbage_blob_count, + garbage_blob_bytes); + + ASSERT_OK(builder.Apply(&garbage)); + + constexpr bool force_consistency_checks = false; + VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, + kCompactionStyleLevel, &vstorage_, + force_consistency_checks); + + ASSERT_OK(builder.SaveTo(&new_vstorage)); + + UpdateVersionStorageInfo(&new_vstorage); + + const auto& new_blob_files = new_vstorage.GetBlobFiles(); + ASSERT_EQ(new_blob_files.size(), 1); + + const auto new_meta = new_vstorage.GetBlobFileMetaData(blob_file_number); + + ASSERT_NE(new_meta, nullptr); + ASSERT_EQ(new_meta->GetBlobFileNumber(), blob_file_number); + ASSERT_EQ(new_meta->GetTotalBlobCount(), total_blob_count); + ASSERT_EQ(new_meta->GetTotalBlobBytes(), total_blob_bytes); + ASSERT_EQ(new_meta->GetChecksumMethod(), checksum_method); + ASSERT_EQ(new_meta->GetChecksumValue(), checksum_value); + ASSERT_EQ(new_meta->GetLinkedSsts(), + BlobFileMetaData::LinkedSsts{table_file_number}); + ASSERT_EQ(new_meta->GetGarbageBlobCount(), garbage_blob_count); + ASSERT_EQ(new_meta->GetGarbageBlobBytes(), garbage_blob_bytes); + + UnrefFilesInVersion(&new_vstorage); +} + +TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileNotFound) { + // Attempt to increase the amount of garbage for a blob file that is + // neither in the base version, nor was it added using a version edit. + + UpdateVersionStorageInfo(); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_, + version_set); + + VersionEdit edit; + + constexpr uint64_t blob_file_number = 1234; + constexpr uint64_t garbage_blob_count = 5678; + constexpr uint64_t garbage_blob_bytes = 999999; + + edit.AddBlobFileGarbage(blob_file_number, garbage_blob_count, + garbage_blob_bytes); + + const Status s = builder.Apply(&edit); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "Blob file #1234 not found")); +} + +TEST_F(VersionBuilderTest, BlobFileGarbageOverflow) { + // Test that VersionEdits that would result in the count/total size of garbage + // exceeding the count/total size of all blobs are rejected. + + UpdateVersionStorageInfo(); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_, + version_set); + + VersionEdit addition; + + constexpr uint64_t blob_file_number = 1234; + constexpr uint64_t total_blob_count = 5678; + constexpr uint64_t total_blob_bytes = 999999; + constexpr char checksum_method[] = "SHA1"; + constexpr char checksum_value[] = + "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52" + "\x5c\xbd"; + + addition.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes, + checksum_method, checksum_value); + + // Add dummy table file to ensure the blob file is referenced. + constexpr uint64_t table_file_number = 1; + AddDummyFileToEdit(&addition, table_file_number, blob_file_number, + 1 /*epoch_number*/); + + ASSERT_OK(builder.Apply(&addition)); + + { + // Garbage blob count overflow + constexpr uint64_t garbage_blob_count = 5679; + constexpr uint64_t garbage_blob_bytes = 999999; + + VersionEdit garbage; + + garbage.AddBlobFileGarbage(blob_file_number, garbage_blob_count, + garbage_blob_bytes); + + const Status s = builder.Apply(&garbage); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE( + std::strstr(s.getState(), "Garbage overflow for blob file #1234")); + } + + { + // Garbage blob bytes overflow + constexpr uint64_t garbage_blob_count = 5678; + constexpr uint64_t garbage_blob_bytes = 1000000; + + VersionEdit garbage; + + garbage.AddBlobFileGarbage(blob_file_number, garbage_blob_count, + garbage_blob_bytes); + + const Status s = builder.Apply(&garbage); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE( + std::strstr(s.getState(), "Garbage overflow for blob file #1234")); + } +} + +TEST_F(VersionBuilderTest, SaveBlobFilesTo) { + // Add three blob files to base version. + for (uint64_t i = 1; i <= 3; ++i) { + const uint64_t table_file_number = 2 * i; + const uint64_t blob_file_number = 2 * i + 1; + const uint64_t total_blob_count = i * 1000; + const uint64_t total_blob_bytes = i * 1000000; + const uint64_t garbage_blob_count = i * 100; + const uint64_t garbage_blob_bytes = i * 20000; + + AddBlob(blob_file_number, total_blob_count, total_blob_bytes, + /* checksum_method */ std::string(), + /* checksum_value */ std::string(), + BlobFileMetaData::LinkedSsts{table_file_number}, garbage_blob_count, + garbage_blob_bytes); + } + + // Add dummy table files to ensure the blob files are referenced. + // Note: files are added to L0, so they have to be added in reverse order + // (newest first). + for (uint64_t i = 3; i >= 1; --i) { + const uint64_t table_file_number = 2 * i; + const uint64_t blob_file_number = 2 * i + 1; + + AddDummyFile(table_file_number, blob_file_number, i /*epoch_number*/); + } + + UpdateVersionStorageInfo(); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_, + version_set); + + VersionEdit edit; + + // Add some garbage to the second and third blob files. The second blob file + // remains valid since it does not consist entirely of garbage yet. The third + // blob file is all garbage after the edit and will not be part of the new + // version. The corresponding dummy table file is also removed for + // consistency. + edit.AddBlobFileGarbage(/* blob_file_number */ 5, + /* garbage_blob_count */ 200, + /* garbage_blob_bytes */ 100000); + edit.AddBlobFileGarbage(/* blob_file_number */ 7, + /* garbage_blob_count */ 2700, + /* garbage_blob_bytes */ 2940000); + edit.DeleteFile(/* level */ 0, /* file_number */ 6); + + // Add a fourth blob file. + edit.AddBlobFile(/* blob_file_number */ 9, /* total_blob_count */ 4000, + /* total_blob_bytes */ 4000000, + /* checksum_method */ std::string(), + /* checksum_value */ std::string()); + + ASSERT_OK(builder.Apply(&edit)); + + constexpr bool force_consistency_checks = false; + VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, + kCompactionStyleLevel, &vstorage_, + force_consistency_checks); + + ASSERT_OK(builder.SaveTo(&new_vstorage)); + + UpdateVersionStorageInfo(&new_vstorage); + + const auto& new_blob_files = new_vstorage.GetBlobFiles(); + ASSERT_EQ(new_blob_files.size(), 3); + + const auto meta3 = new_vstorage.GetBlobFileMetaData(/* blob_file_number */ 3); + + ASSERT_NE(meta3, nullptr); + ASSERT_EQ(meta3->GetBlobFileNumber(), 3); + ASSERT_EQ(meta3->GetTotalBlobCount(), 1000); + ASSERT_EQ(meta3->GetTotalBlobBytes(), 1000000); + ASSERT_EQ(meta3->GetGarbageBlobCount(), 100); + ASSERT_EQ(meta3->GetGarbageBlobBytes(), 20000); + + const auto meta5 = new_vstorage.GetBlobFileMetaData(/* blob_file_number */ 5); + + ASSERT_NE(meta5, nullptr); + ASSERT_EQ(meta5->GetBlobFileNumber(), 5); + ASSERT_EQ(meta5->GetTotalBlobCount(), 2000); + ASSERT_EQ(meta5->GetTotalBlobBytes(), 2000000); + ASSERT_EQ(meta5->GetGarbageBlobCount(), 400); + ASSERT_EQ(meta5->GetGarbageBlobBytes(), 140000); + + const auto meta9 = new_vstorage.GetBlobFileMetaData(/* blob_file_number */ 9); + + ASSERT_NE(meta9, nullptr); + ASSERT_EQ(meta9->GetBlobFileNumber(), 9); + ASSERT_EQ(meta9->GetTotalBlobCount(), 4000); + ASSERT_EQ(meta9->GetTotalBlobBytes(), 4000000); + ASSERT_EQ(meta9->GetGarbageBlobCount(), 0); + ASSERT_EQ(meta9->GetGarbageBlobBytes(), 0); + + // Delete the first table file, which makes the first blob file obsolete + // since it's at the head and unreferenced. + VersionBuilder second_builder(env_options, &ioptions_, table_cache, + &new_vstorage, version_set); + + VersionEdit second_edit; + second_edit.DeleteFile(/* level */ 0, /* file_number */ 2); + + ASSERT_OK(second_builder.Apply(&second_edit)); + + VersionStorageInfo newer_vstorage(&icmp_, ucmp_, options_.num_levels, + kCompactionStyleLevel, &new_vstorage, + force_consistency_checks); + + ASSERT_OK(second_builder.SaveTo(&newer_vstorage)); + + UpdateVersionStorageInfo(&newer_vstorage); + + const auto& newer_blob_files = newer_vstorage.GetBlobFiles(); + ASSERT_EQ(newer_blob_files.size(), 2); + + const auto newer_meta3 = + newer_vstorage.GetBlobFileMetaData(/* blob_file_number */ 3); + + ASSERT_EQ(newer_meta3, nullptr); + + UnrefFilesInVersion(&newer_vstorage); + UnrefFilesInVersion(&new_vstorage); +} + +TEST_F(VersionBuilderTest, SaveBlobFilesToConcurrentJobs) { + // When multiple background jobs (flushes/compactions) are executing in + // parallel, it is possible for the VersionEdit adding blob file K to be + // applied *after* the VersionEdit adding blob file N (for N > K). This test + // case makes sure this is handled correctly. + + // Add blob file #4 (referenced by table file #3) to base version. + constexpr uint64_t base_table_file_number = 3; + constexpr uint64_t base_blob_file_number = 4; + constexpr uint64_t base_total_blob_count = 100; + constexpr uint64_t base_total_blob_bytes = 1 << 20; + + constexpr char checksum_method[] = "SHA1"; + constexpr char checksum_value[] = "\xfa\xce\xb0\x0c"; + constexpr uint64_t garbage_blob_count = 0; + constexpr uint64_t garbage_blob_bytes = 0; + + AddDummyFile(base_table_file_number, base_blob_file_number, + 1 /*epoch_number*/); + AddBlob(base_blob_file_number, base_total_blob_count, base_total_blob_bytes, + checksum_method, checksum_value, + BlobFileMetaData::LinkedSsts{base_table_file_number}, + garbage_blob_count, garbage_blob_bytes); + + UpdateVersionStorageInfo(); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_, + version_set); + + VersionEdit edit; + + // Add blob file #2 (referenced by table file #1). + constexpr int level = 0; + constexpr uint64_t table_file_number = 1; + constexpr uint32_t path_id = 0; + constexpr uint64_t file_size = 1 << 12; + constexpr char smallest[] = "key1"; + constexpr char largest[] = "key987"; + constexpr SequenceNumber smallest_seqno = 0; + constexpr SequenceNumber largest_seqno = 0; + constexpr bool marked_for_compaction = false; + + constexpr uint64_t blob_file_number = 2; + static_assert(blob_file_number < base_blob_file_number, + "Added blob file should have a smaller file number"); + + constexpr uint64_t total_blob_count = 234; + constexpr uint64_t total_blob_bytes = 1 << 22; + + edit.AddFile( + level, table_file_number, path_id, file_size, GetInternalKey(smallest), + GetInternalKey(largest), smallest_seqno, largest_seqno, + marked_for_compaction, Temperature::kUnknown, blob_file_number, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, 2 /*epoch_number*/, + checksum_value, checksum_method, kNullUniqueId64x2, 0, 0, + /* user_defined_timestamps_persisted */ true); + edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes, + checksum_method, checksum_value); + + ASSERT_OK(builder.Apply(&edit)); + + constexpr bool force_consistency_checks = true; + VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, + kCompactionStyleLevel, &vstorage_, + force_consistency_checks); + + ASSERT_OK(builder.SaveTo(&new_vstorage)); + + UpdateVersionStorageInfo(&new_vstorage); + + const auto& new_blob_files = new_vstorage.GetBlobFiles(); + ASSERT_EQ(new_blob_files.size(), 2); + + const auto base_meta = + new_vstorage.GetBlobFileMetaData(base_blob_file_number); + + ASSERT_NE(base_meta, nullptr); + ASSERT_EQ(base_meta->GetBlobFileNumber(), base_blob_file_number); + ASSERT_EQ(base_meta->GetTotalBlobCount(), base_total_blob_count); + ASSERT_EQ(base_meta->GetTotalBlobBytes(), base_total_blob_bytes); + ASSERT_EQ(base_meta->GetGarbageBlobCount(), garbage_blob_count); + ASSERT_EQ(base_meta->GetGarbageBlobBytes(), garbage_blob_bytes); + ASSERT_EQ(base_meta->GetChecksumMethod(), checksum_method); + ASSERT_EQ(base_meta->GetChecksumValue(), checksum_value); + + const auto added_meta = new_vstorage.GetBlobFileMetaData(blob_file_number); + + ASSERT_NE(added_meta, nullptr); + ASSERT_EQ(added_meta->GetBlobFileNumber(), blob_file_number); + ASSERT_EQ(added_meta->GetTotalBlobCount(), total_blob_count); + ASSERT_EQ(added_meta->GetTotalBlobBytes(), total_blob_bytes); + ASSERT_EQ(added_meta->GetGarbageBlobCount(), garbage_blob_count); + ASSERT_EQ(added_meta->GetGarbageBlobBytes(), garbage_blob_bytes); + ASSERT_EQ(added_meta->GetChecksumMethod(), checksum_method); + ASSERT_EQ(added_meta->GetChecksumValue(), checksum_value); + + UnrefFilesInVersion(&new_vstorage); +} + +TEST_F(VersionBuilderTest, CheckConsistencyForBlobFiles) { + // Initialize base version. The first table file points to a valid blob file + // in this version; the second one does not refer to any blob files. + + Add(/* level */ 1, /* file_number */ 1, /* smallest */ "150", + /* largest */ "200", /* file_size */ 100, + /* path_id */ 0, /* smallest_seq */ 100, /* largest_seq */ 100, + /* num_entries */ 0, /* num_deletions */ 0, + /* sampled */ false, /* smallest_seqno */ 100, /* largest_seqno */ 100, + /* oldest_blob_file_number */ 16); + Add(/* level */ 1, /* file_number */ 23, /* smallest */ "201", + /* largest */ "300", /* file_size */ 100, + /* path_id */ 0, /* smallest_seq */ 200, /* largest_seq */ 200, + /* num_entries */ 0, /* num_deletions */ 0, + /* sampled */ false, /* smallest_seqno */ 200, /* largest_seqno */ 200, + kInvalidBlobFileNumber); + + AddBlob(/* blob_file_number */ 16, /* total_blob_count */ 1000, + /* total_blob_bytes */ 1000000, + /* checksum_method */ std::string(), + /* checksum_value */ std::string(), BlobFileMetaData::LinkedSsts{1}, + /* garbage_blob_count */ 500, /* garbage_blob_bytes */ 300000); + + UpdateVersionStorageInfo(); + + // Add a new table file that points to the existing blob file, and add a + // new table file--blob file pair. + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_, + version_set); + + VersionEdit edit; + + edit.AddFile(/* level */ 1, /* file_number */ 606, /* path_id */ 0, + /* file_size */ 100, /* smallest */ GetInternalKey("701"), + /* largest */ GetInternalKey("750"), /* smallest_seqno */ 200, + /* largest_seqno */ 200, /* marked_for_compaction */ false, + Temperature::kUnknown, + /* oldest_blob_file_number */ 16, kUnknownOldestAncesterTime, + kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, + kNullUniqueId64x2, 0, 0, + /* user_defined_timestamps_persisted */ true); + + edit.AddFile(/* level */ 1, /* file_number */ 700, /* path_id */ 0, + /* file_size */ 100, /* smallest */ GetInternalKey("801"), + /* largest */ GetInternalKey("850"), /* smallest_seqno */ 200, + /* largest_seqno */ 200, /* marked_for_compaction */ false, + Temperature::kUnknown, + /* oldest_blob_file_number */ 1000, kUnknownOldestAncesterTime, + kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, + kNullUniqueId64x2, 0, 0, + /* user_defined_timestamps_persisted */ true); + edit.AddBlobFile(/* blob_file_number */ 1000, /* total_blob_count */ 2000, + /* total_blob_bytes */ 200000, + /* checksum_method */ std::string(), + /* checksum_value */ std::string()); + + ASSERT_OK(builder.Apply(&edit)); + + // Save to a new version in order to trigger consistency checks. + constexpr bool force_consistency_checks = true; + VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, + kCompactionStyleLevel, &vstorage_, + force_consistency_checks); + + ASSERT_OK(builder.SaveTo(&new_vstorage)); + + UpdateVersionStorageInfo(&new_vstorage); + + UnrefFilesInVersion(&new_vstorage); +} + +TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesInconsistentLinks) { + // Initialize base version. Links between the table file and the blob file + // are inconsistent. + + Add(/* level */ 1, /* file_number */ 1, /* smallest */ "150", + /* largest */ "200", /* file_size */ 100, + /* path_id */ 0, /* smallest_seq */ 100, /* largest_seq */ 100, + /* num_entries */ 0, /* num_deletions */ 0, + /* sampled */ false, /* smallest_seqno */ 100, /* largest_seqno */ 100, + /* oldest_blob_file_number */ 256); + + AddBlob(/* blob_file_number */ 16, /* total_blob_count */ 1000, + /* total_blob_bytes */ 1000000, + /* checksum_method */ std::string(), + /* checksum_value */ std::string(), BlobFileMetaData::LinkedSsts{1}, + /* garbage_blob_count */ 500, /* garbage_blob_bytes */ 300000); + + UpdateVersionStorageInfo(); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_, + version_set); + + // Save to a new version in order to trigger consistency checks. + constexpr bool force_consistency_checks = true; + VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, + kCompactionStyleLevel, &vstorage_, + force_consistency_checks); + + const Status s = builder.SaveTo(&new_vstorage); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr( + s.getState(), + "Links are inconsistent between table files and blob file #16")); + + UnrefFilesInVersion(&new_vstorage); +} + +TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesAllGarbage) { + // Initialize base version. The table file points to a blob file that is + // all garbage. + + Add(/* level */ 1, /* file_number */ 1, /* smallest */ "150", + /* largest */ "200", /* file_size */ 100, + /* path_id */ 0, /* smallest_seq */ 100, /* largest_seq */ 100, + /* num_entries */ 0, /* num_deletions */ 0, + /* sampled */ false, /* smallest_seqno */ 100, /* largest_seqno */ 100, + /* oldest_blob_file_number */ 16); + + AddBlob(/* blob_file_number */ 16, /* total_blob_count */ 1000, + /* total_blob_bytes */ 1000000, + /* checksum_method */ std::string(), + /* checksum_value */ std::string(), BlobFileMetaData::LinkedSsts{1}, + /* garbage_blob_count */ 1000, /* garbage_blob_bytes */ 1000000); + + UpdateVersionStorageInfo(); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_, + version_set); + + // Save to a new version in order to trigger consistency checks. + constexpr bool force_consistency_checks = true; + VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, + kCompactionStyleLevel, &vstorage_, + force_consistency_checks); + + const Status s = builder.SaveTo(&new_vstorage); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE( + std::strstr(s.getState(), "Blob file #16 consists entirely of garbage")); + + UnrefFilesInVersion(&new_vstorage); +} + +TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesAllGarbageLinkedSsts) { + // Initialize base version, with a table file pointing to a blob file + // that has no garbage at this point. + + Add(/* level */ 1, /* file_number */ 1, /* smallest */ "150", + /* largest */ "200", /* file_size */ 100, + /* path_id */ 0, /* smallest_seq */ 100, /* largest_seq */ 100, + /* num_entries */ 0, /* num_deletions */ 0, + /* sampled */ false, /* smallest_seqno */ 100, /* largest_seqno */ 100, + /* oldest_blob_file_number */ 16); + + AddBlob(/* blob_file_number */ 16, /* total_blob_count */ 1000, + /* total_blob_bytes */ 1000000, + /* checksum_method */ std::string(), + /* checksum_value */ std::string(), BlobFileMetaData::LinkedSsts{1}, + /* garbage_blob_count */ 0, /* garbage_blob_bytes */ 0); + + UpdateVersionStorageInfo(); + + // Mark the entire blob file garbage but do not remove the linked SST. + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_, + version_set); + + VersionEdit edit; + + edit.AddBlobFileGarbage(/* blob_file_number */ 16, + /* garbage_blob_count */ 1000, + /* garbage_blob_bytes */ 1000000); + + ASSERT_OK(builder.Apply(&edit)); + + // Save to a new version in order to trigger consistency checks. + constexpr bool force_consistency_checks = true; + VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, + kCompactionStyleLevel, &vstorage_, + force_consistency_checks); + + const Status s = builder.SaveTo(&new_vstorage); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE( + std::strstr(s.getState(), "Blob file #16 consists entirely of garbage")); + + UnrefFilesInVersion(&new_vstorage); +} + +TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) { + // Initialize base version. Table files 1..10 are linked to blob files 1..5, + // while table files 11..20 are not linked to any blob files. + + for (uint64_t i = 1; i <= 10; ++i) { + std::ostringstream oss; + oss << std::setw(2) << std::setfill('0') << i; + + const std::string key = oss.str(); + + Add(/* level */ 1, /* file_number */ i, /* smallest */ key.c_str(), + /* largest */ key.c_str(), /* file_size */ 100, + /* path_id */ 0, /* smallest_seq */ i * 100, /* largest_seq */ i * 100, + /* num_entries */ 0, /* num_deletions */ 0, + /* sampled */ false, /* smallest_seqno */ i * 100, + /* largest_seqno */ i * 100, + /* oldest_blob_file_number */ ((i - 1) % 5) + 1); + } + + for (uint64_t i = 1; i <= 5; ++i) { + AddBlob(/* blob_file_number */ i, /* total_blob_count */ 2000, + /* total_blob_bytes */ 2000000, + /* checksum_method */ std::string(), + /* checksum_value */ std::string(), + BlobFileMetaData::LinkedSsts{i, i + 5}, + /* garbage_blob_count */ 1000, /* garbage_blob_bytes */ 1000000); + } + + for (uint64_t i = 11; i <= 20; ++i) { + std::ostringstream oss; + oss << std::setw(2) << std::setfill('0') << i; + + const std::string key = oss.str(); + + Add(/* level */ 1, /* file_number */ i, /* smallest */ key.c_str(), + /* largest */ key.c_str(), /* file_size */ 100, + /* path_id */ 0, /* smallest_seq */ i * 100, /* largest_seq */ i * 100, + /* num_entries */ 0, /* num_deletions */ 0, + /* sampled */ false, /* smallest_seqno */ i * 100, + /* largest_seqno */ i * 100, kInvalidBlobFileNumber); + } + + UpdateVersionStorageInfo(); + + { + const auto& blob_files = vstorage_.GetBlobFiles(); + ASSERT_EQ(blob_files.size(), 5); + + const std::vector expected_linked_ssts{ + {1, 6}, {2, 7}, {3, 8}, {4, 9}, {5, 10}}; + + for (size_t i = 0; i < 5; ++i) { + const auto meta = + vstorage_.GetBlobFileMetaData(/* blob_file_number */ i + 1); + ASSERT_NE(meta, nullptr); + ASSERT_EQ(meta->GetLinkedSsts(), expected_linked_ssts[i]); + } + } + + VersionEdit edit; + + // Add an SST that references a blob file. + edit.AddFile( + /* level */ 1, /* file_number */ 21, /* path_id */ 0, + /* file_size */ 100, /* smallest */ GetInternalKey("21", 2100), + /* largest */ GetInternalKey("21", 2100), /* smallest_seqno */ 2100, + /* largest_seqno */ 2100, /* marked_for_compaction */ false, + Temperature::kUnknown, + /* oldest_blob_file_number */ 1, kUnknownOldestAncesterTime, + kUnknownFileCreationTime, kUnknownEpochNumber, kUnknownFileChecksum, + kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0, + /* user_defined_timestamps_persisted */ true); + + // Add an SST that does not reference any blob files. + edit.AddFile( + /* level */ 1, /* file_number */ 22, /* path_id */ 0, + /* file_size */ 100, /* smallest */ GetInternalKey("22", 2200), + /* largest */ GetInternalKey("22", 2200), /* smallest_seqno */ 2200, + /* largest_seqno */ 2200, /* marked_for_compaction */ false, + Temperature::kUnknown, kInvalidBlobFileNumber, kUnknownOldestAncesterTime, + kUnknownFileCreationTime, kUnknownEpochNumber, kUnknownFileChecksum, + kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0, + /* user_defined_timestamps_persisted */ true); + + // Delete a file that references a blob file. + edit.DeleteFile(/* level */ 1, /* file_number */ 6); + + // Delete a file that does not reference any blob files. + edit.DeleteFile(/* level */ 1, /* file_number */ 16); + + // Trivially move a file that references a blob file. Note that we save + // the original BlobFileMetaData object so we can check that no new object + // gets created. + auto meta3 = vstorage_.GetBlobFileMetaData(/* blob_file_number */ 3); + + edit.DeleteFile(/* level */ 1, /* file_number */ 3); + edit.AddFile(/* level */ 2, /* file_number */ 3, /* path_id */ 0, + /* file_size */ 100, /* smallest */ GetInternalKey("03", 300), + /* largest */ GetInternalKey("03", 300), + /* smallest_seqno */ 300, + /* largest_seqno */ 300, /* marked_for_compaction */ false, + Temperature::kUnknown, + /* oldest_blob_file_number */ 3, kUnknownOldestAncesterTime, + kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, + kNullUniqueId64x2, 0, 0, + /* user_defined_timestamps_persisted */ true); + + // Trivially move a file that does not reference any blob files. + edit.DeleteFile(/* level */ 1, /* file_number */ 13); + edit.AddFile(/* level */ 2, /* file_number */ 13, /* path_id */ 0, + /* file_size */ 100, /* smallest */ GetInternalKey("13", 1300), + /* largest */ GetInternalKey("13", 1300), + /* smallest_seqno */ 1300, + /* largest_seqno */ 1300, /* marked_for_compaction */ false, + Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, + kUnknownEpochNumber, kUnknownFileChecksum, + kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0, + /* user_defined_timestamps_persisted */ true); + + // Add one more SST file that references a blob file, then promptly + // delete it in a second version edit before the new version gets saved. + // This file should not show up as linked to the blob file in the new version. + edit.AddFile(/* level */ 1, /* file_number */ 23, /* path_id */ 0, + /* file_size */ 100, /* smallest */ GetInternalKey("23", 2300), + /* largest */ GetInternalKey("23", 2300), + /* smallest_seqno */ 2300, + /* largest_seqno */ 2300, /* marked_for_compaction */ false, + Temperature::kUnknown, + /* oldest_blob_file_number */ 5, kUnknownOldestAncesterTime, + kUnknownFileCreationTime, kUnknownEpochNumber, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, + kNullUniqueId64x2, 0, 0, + /* user_defined_timestamps_persisted */ true); + + VersionEdit edit2; + + edit2.DeleteFile(/* level */ 1, /* file_number */ 23); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_, + version_set); + + ASSERT_OK(builder.Apply(&edit)); + ASSERT_OK(builder.Apply(&edit2)); + + constexpr bool force_consistency_checks = true; + VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, + kCompactionStyleLevel, &vstorage_, + force_consistency_checks); + + ASSERT_OK(builder.SaveTo(&new_vstorage)); + + UpdateVersionStorageInfo(&new_vstorage); + + { + const auto& blob_files = new_vstorage.GetBlobFiles(); + ASSERT_EQ(blob_files.size(), 5); + + const std::vector expected_linked_ssts{ + {1, 21}, {2, 7}, {3, 8}, {4, 9}, {5, 10}}; + + for (size_t i = 0; i < 5; ++i) { + const auto meta = + new_vstorage.GetBlobFileMetaData(/* blob_file_number */ i + 1); + ASSERT_NE(meta, nullptr); + ASSERT_EQ(meta->GetLinkedSsts(), expected_linked_ssts[i]); + } + + // Make sure that no new BlobFileMetaData got created for the blob file + // affected by the trivial move. + ASSERT_EQ(new_vstorage.GetBlobFileMetaData(/* blob_file_number */ 3), + meta3); + } + + UnrefFilesInVersion(&new_vstorage); +} + +TEST_F(VersionBuilderTest, CheckConsistencyForFileDeletedTwice) { + Add(0, 1U, "150", "200", 100, /*path_id*/ 0, + /*smallest_seq*/ 100, /*largest_seq*/ 100, + /*num_entries*/ 0, /*num_deletions*/ 0, + /*sampled*/ false, /*smallest_seqno*/ 0, + /*largest_seqno*/ 0, + /*oldest_blob_file_number*/ kInvalidBlobFileNumber, + /*epoch_number*/ 1); + + UpdateVersionStorageInfo(); + + VersionEdit version_edit; + version_edit.DeleteFile(0, 1U); + + EnvOptions env_options; + constexpr TableCache* table_cache = nullptr; + constexpr VersionSet* version_set = nullptr; + + VersionBuilder version_builder(env_options, &ioptions_, table_cache, + &vstorage_, version_set); + VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, + kCompactionStyleLevel, nullptr, + true /* force_consistency_checks */); + ASSERT_OK(version_builder.Apply(&version_edit)); + ASSERT_OK(version_builder.SaveTo(&new_vstorage)); + + UpdateVersionStorageInfo(&new_vstorage); + + VersionBuilder version_builder2(env_options, &ioptions_, table_cache, + &new_vstorage, version_set); + VersionStorageInfo new_vstorage2(&icmp_, ucmp_, options_.num_levels, + kCompactionStyleLevel, nullptr, + true /* force_consistency_checks */); + ASSERT_NOK(version_builder2.Apply(&version_edit)); + + UnrefFilesInVersion(&new_vstorage); + UnrefFilesInVersion(&new_vstorage2); +} + +TEST_F(VersionBuilderTest, CheckConsistencyForL0FilesSortedByEpochNumber) { + Status s; + // To verify files of same epoch number of overlapping ranges are caught as + // corrupted + VersionEdit version_edit_1; + version_edit_1.AddFile( + /* level */ 0, /* file_number */ 1U, /* path_id */ 0, + /* file_size */ 100, /* smallest */ GetInternalKey("a", 1), + /* largest */ GetInternalKey("c", 3), /* smallest_seqno */ 1, + /* largest_seqno */ 3, /* marked_for_compaction */ false, + Temperature::kUnknown, + /* oldest_blob_file_number */ kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, + 1 /* epoch_number */, kUnknownFileChecksum, kUnknownFileChecksumFuncName, + kNullUniqueId64x2, 0, 0, /* user_defined_timestamps_persisted */ true); + version_edit_1.AddFile( + /* level */ 0, /* file_number */ 2U, /* path_id */ 0, + /* file_size */ 100, /* smallest */ GetInternalKey("b", 2), + /* largest */ GetInternalKey("d", 4), /* smallest_seqno */ 2, + /* largest_seqno */ 4, /* marked_for_compaction */ false, + Temperature::kUnknown, + /* oldest_blob_file_number */ kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, + 1 /* epoch_number */, kUnknownFileChecksum, kUnknownFileChecksumFuncName, + kNullUniqueId64x2, 0, 0, /* user_defined_timestamps_persisted */ true); + + VersionBuilder version_builder_1(EnvOptions(), &ioptions_, + nullptr /* table_cache */, &vstorage_, + nullptr /* file_metadata_cache_res_mgr */); + VersionStorageInfo new_vstorage_1( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, + nullptr /* src_vstorage */, true /* force_consistency_checks */); + + ASSERT_OK(version_builder_1.Apply(&version_edit_1)); + s = version_builder_1.SaveTo(&new_vstorage_1); + EXPECT_TRUE(s.IsCorruption()); + EXPECT_TRUE(std::strstr( + s.getState(), "L0 files of same epoch number but overlapping range")); + UnrefFilesInVersion(&new_vstorage_1); + + // To verify L0 files not sorted by epoch_number are caught as corrupted + VersionEdit version_edit_2; + version_edit_2.AddFile( + /* level */ 0, /* file_number */ 1U, /* path_id */ 0, + /* file_size */ 100, /* smallest */ GetInternalKey("a", 1), + /* largest */ GetInternalKey("a", 1), /* smallest_seqno */ 1, + /* largest_seqno */ 1, /* marked_for_compaction */ false, + Temperature::kUnknown, + /* oldest_blob_file_number */ kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, + 1 /* epoch_number */, kUnknownFileChecksum, kUnknownFileChecksumFuncName, + kNullUniqueId64x2, 0, 0, /* user_defined_timestamps_persisted */ true); + version_edit_2.AddFile( + /* level */ 0, /* file_number */ 2U, /* path_id */ 0, + /* file_size */ 100, /* smallest */ GetInternalKey("b", 2), + /* largest */ GetInternalKey("b", 2), /* smallest_seqno */ 2, + /* largest_seqno */ 2, /* marked_for_compaction */ false, + Temperature::kUnknown, + /* oldest_blob_file_number */ kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, + 2 /* epoch_number */, kUnknownFileChecksum, kUnknownFileChecksumFuncName, + kNullUniqueId64x2, 0, 0, /* user_defined_timestamps_persisted */ true); + + VersionBuilder version_builder_2(EnvOptions(), &ioptions_, + nullptr /* table_cache */, &vstorage_, + nullptr /* file_metadata_cache_res_mgr */); + VersionStorageInfo new_vstorage_2( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, + nullptr /* src_vstorage */, true /* force_consistency_checks */); + + ASSERT_OK(version_builder_2.Apply(&version_edit_2)); + s = version_builder_2.SaveTo(&new_vstorage_2); + ASSERT_TRUE(s.ok()); + + const std::vector& l0_files = new_vstorage_2.LevelFiles(0); + ASSERT_EQ(l0_files.size(), 2); + // Manually corrupt L0 files's epoch_number + l0_files[0]->epoch_number = 1; + l0_files[1]->epoch_number = 2; + + // To surface corruption error by applying dummy version edit + VersionEdit dummy_version_edit; + VersionBuilder dummy_version_builder( + EnvOptions(), &ioptions_, nullptr /* table_cache */, &vstorage_, + nullptr /* file_metadata_cache_res_mgr */); + ASSERT_OK(dummy_version_builder.Apply(&dummy_version_edit)); + s = dummy_version_builder.SaveTo(&new_vstorage_2); + EXPECT_TRUE(s.IsCorruption()); + EXPECT_TRUE(std::strstr(s.getState(), "L0 files are not sorted properly")); + + UnrefFilesInVersion(&new_vstorage_2); +} + +TEST_F(VersionBuilderTest, EstimatedActiveKeys) { + const uint32_t kTotalSamples = 20; + const uint32_t kNumLevels = 5; + const uint32_t kFilesPerLevel = 8; + const uint32_t kNumFiles = kNumLevels * kFilesPerLevel; + const uint32_t kEntriesPerFile = 1000; + const uint32_t kDeletionsPerFile = 100; + for (uint32_t i = 0; i < kNumFiles; ++i) { + Add(static_cast(i / kFilesPerLevel), i + 1, + std::to_string((i + 100) * 1000).c_str(), + std::to_string((i + 100) * 1000 + 999).c_str(), 100U, 0, 100, 100, + kEntriesPerFile, kDeletionsPerFile, (i < kTotalSamples)); + } + // minus 2X for the number of deletion entries because: + // 1x for deletion entry does not count as a data entry. + // 1x for each deletion entry will actually remove one data entry. + ASSERT_EQ(vstorage_.GetEstimatedActiveKeys(), + (kEntriesPerFile - 2 * kDeletionsPerFile) * kNumFiles); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/version_edit.cc b/librocksdb-sys/rocksdb/db/version_edit.cc new file mode 100644 index 0000000..6459c2f --- /dev/null +++ b/librocksdb-sys/rocksdb/db/version_edit.cc @@ -0,0 +1,1121 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/version_edit.h" + +#include "db/blob/blob_index.h" +#include "db/version_set.h" +#include "logging/event_logger.h" +#include "rocksdb/slice.h" +#include "table/unique_id_impl.h" +#include "test_util/sync_point.h" +#include "util/coding.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +namespace {} // anonymous namespace + +uint64_t PackFileNumberAndPathId(uint64_t number, uint64_t path_id) { + assert(number <= kFileNumberMask); + return number | (path_id * (kFileNumberMask + 1)); +} + +Status FileMetaData::UpdateBoundaries(const Slice& key, const Slice& value, + SequenceNumber seqno, + ValueType value_type) { + if (value_type == kTypeBlobIndex) { + BlobIndex blob_index; + const Status s = blob_index.DecodeFrom(value); + if (!s.ok()) { + return s; + } + + if (!blob_index.IsInlined() && !blob_index.HasTTL()) { + if (blob_index.file_number() == kInvalidBlobFileNumber) { + return Status::Corruption("Invalid blob file number"); + } + + if (oldest_blob_file_number == kInvalidBlobFileNumber || + oldest_blob_file_number > blob_index.file_number()) { + oldest_blob_file_number = blob_index.file_number(); + } + } + } + + if (smallest.size() == 0) { + smallest.DecodeFrom(key); + } + largest.DecodeFrom(key); + fd.smallest_seqno = std::min(fd.smallest_seqno, seqno); + fd.largest_seqno = std::max(fd.largest_seqno, seqno); + + return Status::OK(); +} + +void VersionEdit::Clear() { + max_level_ = 0; + db_id_.clear(); + comparator_.clear(); + log_number_ = 0; + prev_log_number_ = 0; + next_file_number_ = 0; + max_column_family_ = 0; + min_log_number_to_keep_ = 0; + last_sequence_ = 0; + has_db_id_ = false; + has_comparator_ = false; + has_log_number_ = false; + has_prev_log_number_ = false; + has_next_file_number_ = false; + has_max_column_family_ = false; + has_min_log_number_to_keep_ = false; + has_last_sequence_ = false; + compact_cursors_.clear(); + deleted_files_.clear(); + new_files_.clear(); + blob_file_additions_.clear(); + blob_file_garbages_.clear(); + wal_additions_.clear(); + wal_deletion_.Reset(); + column_family_ = 0; + is_column_family_add_ = false; + is_column_family_drop_ = false; + column_family_name_.clear(); + is_in_atomic_group_ = false; + remaining_entries_ = 0; + full_history_ts_low_.clear(); +} + +bool VersionEdit::EncodeTo(std::string* dst, + std::optional ts_sz) const { + if (has_db_id_) { + PutVarint32(dst, kDbId); + PutLengthPrefixedSlice(dst, db_id_); + } + if (has_comparator_) { + assert(has_persist_user_defined_timestamps_); + PutVarint32(dst, kComparator); + PutLengthPrefixedSlice(dst, comparator_); + } + if (has_log_number_) { + PutVarint32Varint64(dst, kLogNumber, log_number_); + } + if (has_prev_log_number_) { + PutVarint32Varint64(dst, kPrevLogNumber, prev_log_number_); + } + if (has_next_file_number_) { + PutVarint32Varint64(dst, kNextFileNumber, next_file_number_); + } + if (has_max_column_family_) { + PutVarint32Varint32(dst, kMaxColumnFamily, max_column_family_); + } + if (has_min_log_number_to_keep_) { + PutVarint32Varint64(dst, kMinLogNumberToKeep, min_log_number_to_keep_); + } + if (has_last_sequence_) { + PutVarint32Varint64(dst, kLastSequence, last_sequence_); + } + for (size_t i = 0; i < compact_cursors_.size(); i++) { + if (compact_cursors_[i].second.Valid()) { + PutVarint32(dst, kCompactCursor); + PutVarint32(dst, compact_cursors_[i].first); // level + PutLengthPrefixedSlice(dst, compact_cursors_[i].second.Encode()); + } + } + for (const auto& deleted : deleted_files_) { + PutVarint32Varint32Varint64(dst, kDeletedFile, deleted.first /* level */, + deleted.second /* file number */); + } + + bool min_log_num_written = false; + + assert(new_files_.empty() || ts_sz.has_value()); + for (size_t i = 0; i < new_files_.size(); i++) { + const FileMetaData& f = new_files_[i].second; + if (!f.smallest.Valid() || !f.largest.Valid() || + f.epoch_number == kUnknownEpochNumber) { + return false; + } + PutVarint32(dst, kNewFile4); + PutVarint32Varint64(dst, new_files_[i].first /* level */, f.fd.GetNumber()); + PutVarint64(dst, f.fd.GetFileSize()); + EncodeFileBoundaries(dst, f, ts_sz.value()); + PutVarint64Varint64(dst, f.fd.smallest_seqno, f.fd.largest_seqno); + // Customized fields' format: + // +-----------------------------+ + // | 1st field's tag (varint32) | + // +-----------------------------+ + // | 1st field's size (varint32) | + // +-----------------------------+ + // | bytes for 1st field | + // | (based on size decoded) | + // +-----------------------------+ + // | | + // | ...... | + // | | + // +-----------------------------+ + // | last field's size (varint32)| + // +-----------------------------+ + // | bytes for last field | + // | (based on size decoded) | + // +-----------------------------+ + // | terminating tag (varint32) | + // +-----------------------------+ + // + // Customized encoding for fields: + // tag kPathId: 1 byte as path_id + // tag kNeedCompaction: + // now only can take one char value 1 indicating need-compaction + // + PutVarint32(dst, NewFileCustomTag::kOldestAncesterTime); + std::string varint_oldest_ancester_time; + PutVarint64(&varint_oldest_ancester_time, f.oldest_ancester_time); + TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:VarintOldestAncesterTime", + &varint_oldest_ancester_time); + PutLengthPrefixedSlice(dst, Slice(varint_oldest_ancester_time)); + + PutVarint32(dst, NewFileCustomTag::kFileCreationTime); + std::string varint_file_creation_time; + PutVarint64(&varint_file_creation_time, f.file_creation_time); + TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:VarintFileCreationTime", + &varint_file_creation_time); + PutLengthPrefixedSlice(dst, Slice(varint_file_creation_time)); + + PutVarint32(dst, NewFileCustomTag::kEpochNumber); + std::string varint_epoch_number; + PutVarint64(&varint_epoch_number, f.epoch_number); + PutLengthPrefixedSlice(dst, Slice(varint_epoch_number)); + + PutVarint32(dst, NewFileCustomTag::kFileChecksum); + PutLengthPrefixedSlice(dst, Slice(f.file_checksum)); + + PutVarint32(dst, NewFileCustomTag::kFileChecksumFuncName); + PutLengthPrefixedSlice(dst, Slice(f.file_checksum_func_name)); + + if (f.fd.GetPathId() != 0) { + PutVarint32(dst, NewFileCustomTag::kPathId); + char p = static_cast(f.fd.GetPathId()); + PutLengthPrefixedSlice(dst, Slice(&p, 1)); + } + if (f.temperature != Temperature::kUnknown) { + PutVarint32(dst, NewFileCustomTag::kTemperature); + char p = static_cast(f.temperature); + PutLengthPrefixedSlice(dst, Slice(&p, 1)); + } + if (f.marked_for_compaction) { + PutVarint32(dst, NewFileCustomTag::kNeedCompaction); + char p = static_cast(1); + PutLengthPrefixedSlice(dst, Slice(&p, 1)); + } + if (has_min_log_number_to_keep_ && !min_log_num_written) { + PutVarint32(dst, NewFileCustomTag::kMinLogNumberToKeepHack); + std::string varint_log_number; + PutFixed64(&varint_log_number, min_log_number_to_keep_); + PutLengthPrefixedSlice(dst, Slice(varint_log_number)); + min_log_num_written = true; + } + if (f.oldest_blob_file_number != kInvalidBlobFileNumber) { + PutVarint32(dst, NewFileCustomTag::kOldestBlobFileNumber); + std::string oldest_blob_file_number; + PutVarint64(&oldest_blob_file_number, f.oldest_blob_file_number); + PutLengthPrefixedSlice(dst, Slice(oldest_blob_file_number)); + } + UniqueId64x2 unique_id = f.unique_id; + TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:UniqueId", &unique_id); + if (unique_id != kNullUniqueId64x2) { + PutVarint32(dst, NewFileCustomTag::kUniqueId); + std::string unique_id_str = EncodeUniqueIdBytes(&unique_id); + PutLengthPrefixedSlice(dst, Slice(unique_id_str)); + } + if (f.compensated_range_deletion_size) { + PutVarint32(dst, kCompensatedRangeDeletionSize); + std::string compensated_range_deletion_size; + PutVarint64(&compensated_range_deletion_size, + f.compensated_range_deletion_size); + PutLengthPrefixedSlice(dst, Slice(compensated_range_deletion_size)); + } + if (f.tail_size) { + PutVarint32(dst, NewFileCustomTag::kTailSize); + std::string varint_tail_size; + PutVarint64(&varint_tail_size, f.tail_size); + PutLengthPrefixedSlice(dst, Slice(varint_tail_size)); + } + if (!f.user_defined_timestamps_persisted) { + // The default value for the flag is true, it's only explicitly persisted + // when it's false. We are putting 0 as the value here to signal false + // (i.e. UDTS not persisted). + PutVarint32(dst, NewFileCustomTag::kUserDefinedTimestampsPersisted); + char p = static_cast(0); + PutLengthPrefixedSlice(dst, Slice(&p, 1)); + } + + TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:NewFile4:CustomizeFields", + dst); + + PutVarint32(dst, NewFileCustomTag::kTerminate); + } + + for (const auto& blob_file_addition : blob_file_additions_) { + PutVarint32(dst, kBlobFileAddition); + blob_file_addition.EncodeTo(dst); + } + + for (const auto& blob_file_garbage : blob_file_garbages_) { + PutVarint32(dst, kBlobFileGarbage); + blob_file_garbage.EncodeTo(dst); + } + + for (const auto& wal_addition : wal_additions_) { + PutVarint32(dst, kWalAddition2); + std::string encoded; + wal_addition.EncodeTo(&encoded); + PutLengthPrefixedSlice(dst, encoded); + } + + if (!wal_deletion_.IsEmpty()) { + PutVarint32(dst, kWalDeletion2); + std::string encoded; + wal_deletion_.EncodeTo(&encoded); + PutLengthPrefixedSlice(dst, encoded); + } + + // 0 is default and does not need to be explicitly written + if (column_family_ != 0) { + PutVarint32Varint32(dst, kColumnFamily, column_family_); + } + + if (is_column_family_add_) { + PutVarint32(dst, kColumnFamilyAdd); + PutLengthPrefixedSlice(dst, Slice(column_family_name_)); + } + + if (is_column_family_drop_) { + PutVarint32(dst, kColumnFamilyDrop); + } + + if (is_in_atomic_group_) { + PutVarint32(dst, kInAtomicGroup); + PutVarint32(dst, remaining_entries_); + } + + if (HasFullHistoryTsLow()) { + PutVarint32(dst, kFullHistoryTsLow); + PutLengthPrefixedSlice(dst, full_history_ts_low_); + } + + if (HasPersistUserDefinedTimestamps()) { + // persist_user_defined_timestamps flag should be logged in the same + // VersionEdit as the user comparator name. + assert(has_comparator_); + PutVarint32(dst, kPersistUserDefinedTimestamps); + char p = static_cast(persist_user_defined_timestamps_); + PutLengthPrefixedSlice(dst, Slice(&p, 1)); + } + return true; +} + +static bool GetInternalKey(Slice* input, InternalKey* dst) { + Slice str; + if (GetLengthPrefixedSlice(input, &str)) { + dst->DecodeFrom(str); + return dst->Valid(); + } else { + return false; + } +} + +bool VersionEdit::GetLevel(Slice* input, int* level, const char** /*msg*/) { + uint32_t v = 0; + if (GetVarint32(input, &v)) { + *level = v; + if (max_level_ < *level) { + max_level_ = *level; + } + return true; + } else { + return false; + } +} + +const char* VersionEdit::DecodeNewFile4From(Slice* input) { + const char* msg = nullptr; + int level = 0; + FileMetaData f; + uint64_t number = 0; + uint32_t path_id = 0; + uint64_t file_size = 0; + SequenceNumber smallest_seqno = 0; + SequenceNumber largest_seqno = kMaxSequenceNumber; + if (GetLevel(input, &level, &msg) && GetVarint64(input, &number) && + GetVarint64(input, &file_size) && GetInternalKey(input, &f.smallest) && + GetInternalKey(input, &f.largest) && + GetVarint64(input, &smallest_seqno) && + GetVarint64(input, &largest_seqno)) { + // See comments in VersionEdit::EncodeTo() for format of customized fields + while (true) { + uint32_t custom_tag = 0; + Slice field; + if (!GetVarint32(input, &custom_tag)) { + return "new-file4 custom field"; + } + if (custom_tag == kTerminate) { + break; + } + if (!GetLengthPrefixedSlice(input, &field)) { + return "new-file4 custom field length prefixed slice error"; + } + switch (custom_tag) { + case kPathId: + if (field.size() != 1) { + return "path_id field wrong size"; + } + path_id = field[0]; + if (path_id > 3) { + return "path_id wrong vaue"; + } + break; + case kOldestAncesterTime: + if (!GetVarint64(&field, &f.oldest_ancester_time)) { + return "invalid oldest ancester time"; + } + break; + case kFileCreationTime: + if (!GetVarint64(&field, &f.file_creation_time)) { + return "invalid file creation time"; + } + break; + case kEpochNumber: + if (!GetVarint64(&field, &f.epoch_number)) { + return "invalid epoch number"; + } + break; + case kFileChecksum: + f.file_checksum = field.ToString(); + break; + case kFileChecksumFuncName: + f.file_checksum_func_name = field.ToString(); + break; + case kNeedCompaction: + if (field.size() != 1) { + return "need_compaction field wrong size"; + } + f.marked_for_compaction = (field[0] == 1); + break; + case kMinLogNumberToKeepHack: + // This is a hack to encode kMinLogNumberToKeep in a + // forward-compatible fashion. + if (!GetFixed64(&field, &min_log_number_to_keep_)) { + return "deleted log number malformatted"; + } + has_min_log_number_to_keep_ = true; + break; + case kOldestBlobFileNumber: + if (!GetVarint64(&field, &f.oldest_blob_file_number)) { + return "invalid oldest blob file number"; + } + break; + case kTemperature: + if (field.size() != 1) { + return "temperature field wrong size"; + } else { + Temperature casted_field = static_cast(field[0]); + if (casted_field <= Temperature::kCold) { + f.temperature = casted_field; + } + } + break; + case kUniqueId: + if (!DecodeUniqueIdBytes(field.ToString(), &f.unique_id).ok()) { + f.unique_id = kNullUniqueId64x2; + return "invalid unique id"; + } + break; + case kCompensatedRangeDeletionSize: + if (!GetVarint64(&field, &f.compensated_range_deletion_size)) { + return "Invalid compensated range deletion size"; + } + break; + case kTailSize: + if (!GetVarint64(&field, &f.tail_size)) { + return "invalid tail start offset"; + } + break; + case kUserDefinedTimestampsPersisted: + if (field.size() != 1) { + return "user-defined timestamps persisted field wrong size"; + } + f.user_defined_timestamps_persisted = (field[0] == 1); + break; + default: + if ((custom_tag & kCustomTagNonSafeIgnoreMask) != 0) { + // Should not proceed if cannot understand it + return "new-file4 custom field not supported"; + } + break; + } + } + } else { + return "new-file4 entry"; + } + f.fd = + FileDescriptor(number, path_id, file_size, smallest_seqno, largest_seqno); + new_files_.push_back(std::make_pair(level, f)); + return nullptr; +} + +void VersionEdit::EncodeFileBoundaries(std::string* dst, + const FileMetaData& meta, + size_t ts_sz) const { + if (ts_sz == 0 || meta.user_defined_timestamps_persisted) { + PutLengthPrefixedSlice(dst, meta.smallest.Encode()); + PutLengthPrefixedSlice(dst, meta.largest.Encode()); + return; + } + std::string smallest_buf; + std::string largest_buf; + StripTimestampFromInternalKey(&smallest_buf, meta.smallest.Encode(), ts_sz); + StripTimestampFromInternalKey(&largest_buf, meta.largest.Encode(), ts_sz); + PutLengthPrefixedSlice(dst, smallest_buf); + PutLengthPrefixedSlice(dst, largest_buf); + return; +}; + +Status VersionEdit::DecodeFrom(const Slice& src) { + Clear(); +#ifndef NDEBUG + bool ignore_ignorable_tags = false; + TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:IgnoreIgnorableTags", + &ignore_ignorable_tags); +#endif + Slice input = src; + const char* msg = nullptr; + uint32_t tag = 0; + + // Temporary storage for parsing + int level = 0; + FileMetaData f; + Slice str; + InternalKey key; + while (msg == nullptr && GetVarint32(&input, &tag)) { +#ifndef NDEBUG + if (ignore_ignorable_tags && tag > kTagSafeIgnoreMask) { + tag = kTagSafeIgnoreMask; + } +#endif + switch (tag) { + case kDbId: + if (GetLengthPrefixedSlice(&input, &str)) { + db_id_ = str.ToString(); + has_db_id_ = true; + } else { + msg = "db id"; + } + break; + case kComparator: + if (GetLengthPrefixedSlice(&input, &str)) { + comparator_ = str.ToString(); + has_comparator_ = true; + } else { + msg = "comparator name"; + } + break; + + case kLogNumber: + if (GetVarint64(&input, &log_number_)) { + has_log_number_ = true; + } else { + msg = "log number"; + } + break; + + case kPrevLogNumber: + if (GetVarint64(&input, &prev_log_number_)) { + has_prev_log_number_ = true; + } else { + msg = "previous log number"; + } + break; + + case kNextFileNumber: + if (GetVarint64(&input, &next_file_number_)) { + has_next_file_number_ = true; + } else { + msg = "next file number"; + } + break; + + case kMaxColumnFamily: + if (GetVarint32(&input, &max_column_family_)) { + has_max_column_family_ = true; + } else { + msg = "max column family"; + } + break; + + case kMinLogNumberToKeep: + if (GetVarint64(&input, &min_log_number_to_keep_)) { + has_min_log_number_to_keep_ = true; + } else { + msg = "min log number to kee"; + } + break; + + case kLastSequence: + if (GetVarint64(&input, &last_sequence_)) { + has_last_sequence_ = true; + } else { + msg = "last sequence number"; + } + break; + + case kCompactCursor: + if (GetLevel(&input, &level, &msg) && GetInternalKey(&input, &key)) { + // Here we re-use the output format of compact pointer in LevelDB + // to persist compact_cursors_ + compact_cursors_.push_back(std::make_pair(level, key)); + } else { + if (!msg) { + msg = "compaction cursor"; + } + } + break; + + case kDeletedFile: { + uint64_t number = 0; + if (GetLevel(&input, &level, &msg) && GetVarint64(&input, &number)) { + deleted_files_.insert(std::make_pair(level, number)); + } else { + if (!msg) { + msg = "deleted file"; + } + } + break; + } + + case kNewFile: { + uint64_t number = 0; + uint64_t file_size = 0; + if (GetLevel(&input, &level, &msg) && GetVarint64(&input, &number) && + GetVarint64(&input, &file_size) && + GetInternalKey(&input, &f.smallest) && + GetInternalKey(&input, &f.largest)) { + f.fd = FileDescriptor(number, 0, file_size); + new_files_.push_back(std::make_pair(level, f)); + } else { + if (!msg) { + msg = "new-file entry"; + } + } + break; + } + case kNewFile2: { + uint64_t number = 0; + uint64_t file_size = 0; + SequenceNumber smallest_seqno = 0; + SequenceNumber largest_seqno = kMaxSequenceNumber; + if (GetLevel(&input, &level, &msg) && GetVarint64(&input, &number) && + GetVarint64(&input, &file_size) && + GetInternalKey(&input, &f.smallest) && + GetInternalKey(&input, &f.largest) && + GetVarint64(&input, &smallest_seqno) && + GetVarint64(&input, &largest_seqno)) { + f.fd = FileDescriptor(number, 0, file_size, smallest_seqno, + largest_seqno); + new_files_.push_back(std::make_pair(level, f)); + } else { + if (!msg) { + msg = "new-file2 entry"; + } + } + break; + } + + case kNewFile3: { + uint64_t number = 0; + uint32_t path_id = 0; + uint64_t file_size = 0; + SequenceNumber smallest_seqno = 0; + SequenceNumber largest_seqno = kMaxSequenceNumber; + if (GetLevel(&input, &level, &msg) && GetVarint64(&input, &number) && + GetVarint32(&input, &path_id) && GetVarint64(&input, &file_size) && + GetInternalKey(&input, &f.smallest) && + GetInternalKey(&input, &f.largest) && + GetVarint64(&input, &smallest_seqno) && + GetVarint64(&input, &largest_seqno)) { + f.fd = FileDescriptor(number, path_id, file_size, smallest_seqno, + largest_seqno); + new_files_.push_back(std::make_pair(level, f)); + } else { + if (!msg) { + msg = "new-file3 entry"; + } + } + break; + } + + case kNewFile4: { + msg = DecodeNewFile4From(&input); + break; + } + + case kBlobFileAddition: + case kBlobFileAddition_DEPRECATED: { + BlobFileAddition blob_file_addition; + const Status s = blob_file_addition.DecodeFrom(&input); + if (!s.ok()) { + return s; + } + + AddBlobFile(std::move(blob_file_addition)); + break; + } + + case kBlobFileGarbage: + case kBlobFileGarbage_DEPRECATED: { + BlobFileGarbage blob_file_garbage; + const Status s = blob_file_garbage.DecodeFrom(&input); + if (!s.ok()) { + return s; + } + + AddBlobFileGarbage(std::move(blob_file_garbage)); + break; + } + + case kWalAddition: { + WalAddition wal_addition; + const Status s = wal_addition.DecodeFrom(&input); + if (!s.ok()) { + return s; + } + + wal_additions_.emplace_back(std::move(wal_addition)); + break; + } + + case kWalAddition2: { + Slice encoded; + if (!GetLengthPrefixedSlice(&input, &encoded)) { + msg = "WalAddition not prefixed by length"; + break; + } + + WalAddition wal_addition; + const Status s = wal_addition.DecodeFrom(&encoded); + if (!s.ok()) { + return s; + } + + wal_additions_.emplace_back(std::move(wal_addition)); + break; + } + + case kWalDeletion: { + WalDeletion wal_deletion; + const Status s = wal_deletion.DecodeFrom(&input); + if (!s.ok()) { + return s; + } + + wal_deletion_ = std::move(wal_deletion); + break; + } + + case kWalDeletion2: { + Slice encoded; + if (!GetLengthPrefixedSlice(&input, &encoded)) { + msg = "WalDeletion not prefixed by length"; + break; + } + + WalDeletion wal_deletion; + const Status s = wal_deletion.DecodeFrom(&encoded); + if (!s.ok()) { + return s; + } + + wal_deletion_ = std::move(wal_deletion); + break; + } + + case kColumnFamily: + if (!GetVarint32(&input, &column_family_)) { + if (!msg) { + msg = "set column family id"; + } + } + break; + + case kColumnFamilyAdd: + if (GetLengthPrefixedSlice(&input, &str)) { + is_column_family_add_ = true; + column_family_name_ = str.ToString(); + } else { + if (!msg) { + msg = "column family add"; + } + } + break; + + case kColumnFamilyDrop: + is_column_family_drop_ = true; + break; + + case kInAtomicGroup: + is_in_atomic_group_ = true; + if (!GetVarint32(&input, &remaining_entries_)) { + if (!msg) { + msg = "remaining entries"; + } + } + break; + + case kFullHistoryTsLow: + if (!GetLengthPrefixedSlice(&input, &str)) { + msg = "full_history_ts_low"; + } else if (str.empty()) { + msg = "full_history_ts_low: empty"; + } else { + full_history_ts_low_.assign(str.data(), str.size()); + } + break; + + case kPersistUserDefinedTimestamps: + if (!GetLengthPrefixedSlice(&input, &str)) { + msg = "persist_user_defined_timestamps"; + } else if (str.size() != 1) { + msg = "persist_user_defined_timestamps field wrong size"; + } else { + persist_user_defined_timestamps_ = (str[0] == 1); + has_persist_user_defined_timestamps_ = true; + } + break; + + default: + if (tag & kTagSafeIgnoreMask) { + // Tag from future which can be safely ignored. + // The next field must be the length of the entry. + uint32_t field_len; + if (!GetVarint32(&input, &field_len) || + static_cast(field_len) > input.size()) { + if (!msg) { + msg = "safely ignoreable tag length error"; + } + } else { + input.remove_prefix(static_cast(field_len)); + } + } else { + msg = "unknown tag"; + } + break; + } + } + + if (msg == nullptr && !input.empty()) { + msg = "invalid tag"; + } + + Status result; + if (msg != nullptr) { + result = Status::Corruption("VersionEdit", msg); + } + return result; +} + +std::string VersionEdit::DebugString(bool hex_key) const { + std::string r; + r.append("VersionEdit {"); + if (has_db_id_) { + r.append("\n DB ID: "); + r.append(db_id_); + } + if (has_comparator_) { + r.append("\n Comparator: "); + r.append(comparator_); + } + if (has_persist_user_defined_timestamps_) { + r.append("\n PersistUserDefinedTimestamps: "); + r.append(persist_user_defined_timestamps_ ? "true" : "false"); + } + if (has_log_number_) { + r.append("\n LogNumber: "); + AppendNumberTo(&r, log_number_); + } + if (has_prev_log_number_) { + r.append("\n PrevLogNumber: "); + AppendNumberTo(&r, prev_log_number_); + } + if (has_next_file_number_) { + r.append("\n NextFileNumber: "); + AppendNumberTo(&r, next_file_number_); + } + if (has_max_column_family_) { + r.append("\n MaxColumnFamily: "); + AppendNumberTo(&r, max_column_family_); + } + if (has_min_log_number_to_keep_) { + r.append("\n MinLogNumberToKeep: "); + AppendNumberTo(&r, min_log_number_to_keep_); + } + if (has_last_sequence_) { + r.append("\n LastSeq: "); + AppendNumberTo(&r, last_sequence_); + } + for (const auto& level_and_compact_cursor : compact_cursors_) { + r.append("\n CompactCursor: "); + AppendNumberTo(&r, level_and_compact_cursor.first); + r.append(" "); + r.append(level_and_compact_cursor.second.DebugString(hex_key)); + } + for (const auto& deleted_file : deleted_files_) { + r.append("\n DeleteFile: "); + AppendNumberTo(&r, deleted_file.first); + r.append(" "); + AppendNumberTo(&r, deleted_file.second); + } + for (size_t i = 0; i < new_files_.size(); i++) { + const FileMetaData& f = new_files_[i].second; + r.append("\n AddFile: "); + AppendNumberTo(&r, new_files_[i].first); + r.append(" "); + AppendNumberTo(&r, f.fd.GetNumber()); + r.append(" "); + AppendNumberTo(&r, f.fd.GetFileSize()); + r.append(" "); + r.append(f.smallest.DebugString(hex_key)); + r.append(" .. "); + r.append(f.largest.DebugString(hex_key)); + if (f.oldest_blob_file_number != kInvalidBlobFileNumber) { + r.append(" blob_file:"); + AppendNumberTo(&r, f.oldest_blob_file_number); + } + r.append(" oldest_ancester_time:"); + AppendNumberTo(&r, f.oldest_ancester_time); + r.append(" file_creation_time:"); + AppendNumberTo(&r, f.file_creation_time); + r.append(" epoch_number:"); + AppendNumberTo(&r, f.epoch_number); + r.append(" file_checksum:"); + r.append(Slice(f.file_checksum).ToString(true)); + r.append(" file_checksum_func_name: "); + r.append(f.file_checksum_func_name); + if (f.temperature != Temperature::kUnknown) { + r.append(" temperature: "); + // Maybe change to human readable format whenthe feature becomes + // permanent + r.append(std::to_string(static_cast(f.temperature))); + } + if (f.unique_id != kNullUniqueId64x2) { + r.append(" unique_id(internal): "); + UniqueId64x2 id = f.unique_id; + r.append(InternalUniqueIdToHumanString(&id)); + r.append(" public_unique_id: "); + InternalUniqueIdToExternal(&id); + r.append(UniqueIdToHumanString(EncodeUniqueIdBytes(&id))); + } + r.append(" tail size: "); + AppendNumberTo(&r, f.tail_size); + r.append(" User-defined timestamps persisted: "); + r.append(f.user_defined_timestamps_persisted ? "true" : "false"); + } + + for (const auto& blob_file_addition : blob_file_additions_) { + r.append("\n BlobFileAddition: "); + r.append(blob_file_addition.DebugString()); + } + + for (const auto& blob_file_garbage : blob_file_garbages_) { + r.append("\n BlobFileGarbage: "); + r.append(blob_file_garbage.DebugString()); + } + + for (const auto& wal_addition : wal_additions_) { + r.append("\n WalAddition: "); + r.append(wal_addition.DebugString()); + } + + if (!wal_deletion_.IsEmpty()) { + r.append("\n WalDeletion: "); + r.append(wal_deletion_.DebugString()); + } + + r.append("\n ColumnFamily: "); + AppendNumberTo(&r, column_family_); + if (is_column_family_add_) { + r.append("\n ColumnFamilyAdd: "); + r.append(column_family_name_); + } + if (is_column_family_drop_) { + r.append("\n ColumnFamilyDrop"); + } + if (is_in_atomic_group_) { + r.append("\n AtomicGroup: "); + AppendNumberTo(&r, remaining_entries_); + r.append(" entries remains"); + } + if (HasFullHistoryTsLow()) { + r.append("\n FullHistoryTsLow: "); + r.append(Slice(full_history_ts_low_).ToString(hex_key)); + } + r.append("\n}\n"); + return r; +} + +std::string VersionEdit::DebugJSON(int edit_num, bool hex_key) const { + JSONWriter jw; + jw << "EditNumber" << edit_num; + + if (has_db_id_) { + jw << "DB ID" << db_id_; + } + if (has_comparator_) { + jw << "Comparator" << comparator_; + } + if (has_log_number_) { + jw << "LogNumber" << log_number_; + } + if (has_prev_log_number_) { + jw << "PrevLogNumber" << prev_log_number_; + } + if (has_next_file_number_) { + jw << "NextFileNumber" << next_file_number_; + } + if (has_max_column_family_) { + jw << "MaxColumnFamily" << max_column_family_; + } + if (has_min_log_number_to_keep_) { + jw << "MinLogNumberToKeep" << min_log_number_to_keep_; + } + if (has_last_sequence_) { + jw << "LastSeq" << last_sequence_; + } + + if (!deleted_files_.empty()) { + jw << "DeletedFiles"; + jw.StartArray(); + + for (const auto& deleted_file : deleted_files_) { + jw.StartArrayedObject(); + jw << "Level" << deleted_file.first; + jw << "FileNumber" << deleted_file.second; + jw.EndArrayedObject(); + } + + jw.EndArray(); + } + + if (!new_files_.empty()) { + jw << "AddedFiles"; + jw.StartArray(); + + for (size_t i = 0; i < new_files_.size(); i++) { + jw.StartArrayedObject(); + jw << "Level" << new_files_[i].first; + const FileMetaData& f = new_files_[i].second; + jw << "FileNumber" << f.fd.GetNumber(); + jw << "FileSize" << f.fd.GetFileSize(); + jw << "SmallestIKey" << f.smallest.DebugString(hex_key); + jw << "LargestIKey" << f.largest.DebugString(hex_key); + jw << "OldestAncesterTime" << f.oldest_ancester_time; + jw << "FileCreationTime" << f.file_creation_time; + jw << "EpochNumber" << f.epoch_number; + jw << "FileChecksum" << Slice(f.file_checksum).ToString(true); + jw << "FileChecksumFuncName" << f.file_checksum_func_name; + if (f.temperature != Temperature::kUnknown) { + jw << "temperature" << std::to_string(static_cast(f.temperature)); + } + if (f.oldest_blob_file_number != kInvalidBlobFileNumber) { + jw << "OldestBlobFile" << f.oldest_blob_file_number; + } + if (f.temperature != Temperature::kUnknown) { + // Maybe change to human readable format whenthe feature becomes + // permanent + jw << "Temperature" << static_cast(f.temperature); + } + jw << "TailSize" << f.tail_size; + jw << "UserDefinedTimestampsPersisted" + << f.user_defined_timestamps_persisted; + jw.EndArrayedObject(); + } + + jw.EndArray(); + } + + if (!blob_file_additions_.empty()) { + jw << "BlobFileAdditions"; + + jw.StartArray(); + + for (const auto& blob_file_addition : blob_file_additions_) { + jw.StartArrayedObject(); + jw << blob_file_addition; + jw.EndArrayedObject(); + } + + jw.EndArray(); + } + + if (!blob_file_garbages_.empty()) { + jw << "BlobFileGarbages"; + + jw.StartArray(); + + for (const auto& blob_file_garbage : blob_file_garbages_) { + jw.StartArrayedObject(); + jw << blob_file_garbage; + jw.EndArrayedObject(); + } + + jw.EndArray(); + } + + if (!wal_additions_.empty()) { + jw << "WalAdditions"; + + jw.StartArray(); + + for (const auto& wal_addition : wal_additions_) { + jw.StartArrayedObject(); + jw << wal_addition; + jw.EndArrayedObject(); + } + + jw.EndArray(); + } + + if (!wal_deletion_.IsEmpty()) { + jw << "WalDeletion"; + jw.StartObject(); + jw << wal_deletion_; + jw.EndObject(); + } + + jw << "ColumnFamily" << column_family_; + + if (is_column_family_add_) { + jw << "ColumnFamilyAdd" << column_family_name_; + } + if (is_column_family_drop_) { + jw << "ColumnFamilyDrop" << column_family_name_; + } + if (is_in_atomic_group_) { + jw << "AtomicGroup" << remaining_entries_; + } + + if (HasFullHistoryTsLow()) { + jw << "FullHistoryTsLow" << Slice(full_history_ts_low_).ToString(hex_key); + } + + jw.EndObject(); + + return jw.Get(); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/version_edit.h b/librocksdb-sys/rocksdb/db/version_edit.h new file mode 100644 index 0000000..e6d54d3 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/version_edit.h @@ -0,0 +1,744 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include +#include +#include +#include +#include +#include + +#include "db/blob/blob_file_addition.h" +#include "db/blob/blob_file_garbage.h" +#include "db/dbformat.h" +#include "db/wal_edit.h" +#include "memory/arena.h" +#include "port/malloc.h" +#include "rocksdb/advanced_cache.h" +#include "rocksdb/advanced_options.h" +#include "table/table_reader.h" +#include "table/unique_id_impl.h" +#include "util/autovector.h" + +namespace ROCKSDB_NAMESPACE { + +// Tag numbers for serialized VersionEdit. These numbers are written to +// disk and should not be changed. The number should be forward compatible so +// users can down-grade RocksDB safely. A future Tag is ignored by doing '&' +// between Tag and kTagSafeIgnoreMask field. +enum Tag : uint32_t { + kComparator = 1, + kLogNumber = 2, + kNextFileNumber = 3, + kLastSequence = 4, + kCompactCursor = 5, + kDeletedFile = 6, + kNewFile = 7, + // 8 was used for large value refs + kPrevLogNumber = 9, + kMinLogNumberToKeep = 10, + + // these are new formats divergent from open source leveldb + kNewFile2 = 100, + kNewFile3 = 102, + kNewFile4 = 103, // 4th (the latest) format version of adding files + kColumnFamily = 200, // specify column family for version edit + kColumnFamilyAdd = 201, + kColumnFamilyDrop = 202, + kMaxColumnFamily = 203, + + kInAtomicGroup = 300, + + kBlobFileAddition = 400, + kBlobFileGarbage, + + // Mask for an unidentified tag from the future which can be safely ignored. + kTagSafeIgnoreMask = 1 << 13, + + // Forward compatible (aka ignorable) records + kDbId, + kBlobFileAddition_DEPRECATED, + kBlobFileGarbage_DEPRECATED, + kWalAddition, + kWalDeletion, + kFullHistoryTsLow, + kWalAddition2, + kWalDeletion2, + kPersistUserDefinedTimestamps, +}; + +enum NewFileCustomTag : uint32_t { + kTerminate = 1, // The end of customized fields + kNeedCompaction = 2, + // Since Manifest is not entirely forward-compatible, we currently encode + // kMinLogNumberToKeep as part of NewFile as a hack. This should be removed + // when manifest becomes forward-compatible. + kMinLogNumberToKeepHack = 3, + kOldestBlobFileNumber = 4, + kOldestAncesterTime = 5, + kFileCreationTime = 6, + kFileChecksum = 7, + kFileChecksumFuncName = 8, + kTemperature = 9, + kMinTimestamp = 10, + kMaxTimestamp = 11, + kUniqueId = 12, + kEpochNumber = 13, + kCompensatedRangeDeletionSize = 14, + kTailSize = 15, + kUserDefinedTimestampsPersisted = 16, + + // If this bit for the custom tag is set, opening DB should fail if + // we don't know this field. + kCustomTagNonSafeIgnoreMask = 1 << 6, + + // Forward incompatible (aka unignorable) fields + kPathId, +}; + +class VersionSet; + +constexpr uint64_t kFileNumberMask = 0x3FFFFFFFFFFFFFFF; +constexpr uint64_t kUnknownOldestAncesterTime = 0; +constexpr uint64_t kUnknownFileCreationTime = 0; +constexpr uint64_t kUnknownEpochNumber = 0; +// If `Options::allow_ingest_behind` is true, this epoch number +// will be dedicated to files ingested behind. +constexpr uint64_t kReservedEpochNumberForFileIngestedBehind = 1; + +extern uint64_t PackFileNumberAndPathId(uint64_t number, uint64_t path_id); + +// A copyable structure contains information needed to read data from an SST +// file. It can contain a pointer to a table reader opened for the file, or +// file number and size, which can be used to create a new table reader for it. +// The behavior is undefined when a copied of the structure is used when the +// file is not in any live version any more. +struct FileDescriptor { + // Table reader in table_reader_handle + TableReader* table_reader; + uint64_t packed_number_and_path_id; + uint64_t file_size; // File size in bytes + SequenceNumber smallest_seqno; // The smallest seqno in this file + SequenceNumber largest_seqno; // The largest seqno in this file + + FileDescriptor() : FileDescriptor(0, 0, 0) {} + + FileDescriptor(uint64_t number, uint32_t path_id, uint64_t _file_size) + : FileDescriptor(number, path_id, _file_size, kMaxSequenceNumber, 0) {} + + FileDescriptor(uint64_t number, uint32_t path_id, uint64_t _file_size, + SequenceNumber _smallest_seqno, SequenceNumber _largest_seqno) + : table_reader(nullptr), + packed_number_and_path_id(PackFileNumberAndPathId(number, path_id)), + file_size(_file_size), + smallest_seqno(_smallest_seqno), + largest_seqno(_largest_seqno) {} + + FileDescriptor(const FileDescriptor& fd) { *this = fd; } + + FileDescriptor& operator=(const FileDescriptor& fd) { + table_reader = fd.table_reader; + packed_number_and_path_id = fd.packed_number_and_path_id; + file_size = fd.file_size; + smallest_seqno = fd.smallest_seqno; + largest_seqno = fd.largest_seqno; + return *this; + } + + uint64_t GetNumber() const { + return packed_number_and_path_id & kFileNumberMask; + } + uint32_t GetPathId() const { + return static_cast(packed_number_and_path_id / + (kFileNumberMask + 1)); + } + uint64_t GetFileSize() const { return file_size; } +}; + +struct FileSampledStats { + FileSampledStats() : num_reads_sampled(0) {} + FileSampledStats(const FileSampledStats& other) { *this = other; } + FileSampledStats& operator=(const FileSampledStats& other) { + num_reads_sampled = other.num_reads_sampled.load(); + return *this; + } + + // number of user reads to this file. + mutable std::atomic num_reads_sampled; +}; + +struct FileMetaData { + FileDescriptor fd; + InternalKey smallest; // Smallest internal key served by table + InternalKey largest; // Largest internal key served by table + + // Needs to be disposed when refs becomes 0. + Cache::Handle* table_reader_handle = nullptr; + + FileSampledStats stats; + + // Stats for compensating deletion entries during compaction + + // File size compensated by deletion entry. + // This is used to compute a file's compaction priority, and is updated in + // Version::ComputeCompensatedSizes() first time when the file is created or + // loaded. After it is updated (!= 0), it is immutable. + uint64_t compensated_file_size = 0; + // These values can mutate, but they can only be read or written from + // single-threaded LogAndApply thread + uint64_t num_entries = + 0; // The number of entries, including deletions and range deletions. + // The number of deletion entries, including range deletions. + uint64_t num_deletions = 0; + uint64_t raw_key_size = 0; // total uncompressed key size. + uint64_t raw_value_size = 0; // total uncompressed value size. + uint64_t num_range_deletions = 0; + // This is computed during Flush/Compaction, and is added to + // `compensated_file_size`. Currently, this estimates the size of keys in the + // next level covered by range tombstones in this file. + uint64_t compensated_range_deletion_size = 0; + + int refs = 0; // Reference count + + bool being_compacted = false; // Is this file undergoing compaction? + bool init_stats_from_file = false; // true if the data-entry stats of this + // file has initialized from file. + + bool marked_for_compaction = false; // True if client asked us nicely to + // compact this file. + Temperature temperature = Temperature::kUnknown; + + // Used only in BlobDB. The file number of the oldest blob file this SST file + // refers to. 0 is an invalid value; BlobDB numbers the files starting from 1. + uint64_t oldest_blob_file_number = kInvalidBlobFileNumber; + + // The file could be the compaction output from other SST files, which could + // in turn be outputs for compact older SST files. We track the memtable + // flush timestamp for the oldest SST file that eventually contribute data + // to this file. 0 means the information is not available. + uint64_t oldest_ancester_time = kUnknownOldestAncesterTime; + + // Unix time when the SST file is created. + uint64_t file_creation_time = kUnknownFileCreationTime; + + // The order of a file being flushed or ingested/imported. + // Compaction output file will be assigned with the minimum `epoch_number` + // among input files'. + // For L0, larger `epoch_number` indicates newer L0 file. + uint64_t epoch_number = kUnknownEpochNumber; + + // File checksum + std::string file_checksum = kUnknownFileChecksum; + + // File checksum function name + std::string file_checksum_func_name = kUnknownFileChecksumFuncName; + + // SST unique id + UniqueId64x2 unique_id{}; + + // Size of the "tail" part of a SST file + // "Tail" refers to all blocks after data blocks till the end of the SST file + uint64_t tail_size = 0; + + // Value of the `AdvancedColumnFamilyOptions.persist_user_defined_timestamps` + // flag when the file is created. Default to true, only when this flag is + // false, it's explicitly written to Manifest. + bool user_defined_timestamps_persisted = true; + + FileMetaData() = default; + + FileMetaData(uint64_t file, uint32_t file_path_id, uint64_t file_size, + const InternalKey& smallest_key, const InternalKey& largest_key, + const SequenceNumber& smallest_seq, + const SequenceNumber& largest_seq, bool marked_for_compact, + Temperature _temperature, uint64_t oldest_blob_file, + uint64_t _oldest_ancester_time, uint64_t _file_creation_time, + uint64_t _epoch_number, const std::string& _file_checksum, + const std::string& _file_checksum_func_name, + UniqueId64x2 _unique_id, + const uint64_t _compensated_range_deletion_size, + uint64_t _tail_size, bool _user_defined_timestamps_persisted) + : fd(file, file_path_id, file_size, smallest_seq, largest_seq), + smallest(smallest_key), + largest(largest_key), + compensated_range_deletion_size(_compensated_range_deletion_size), + marked_for_compaction(marked_for_compact), + temperature(_temperature), + oldest_blob_file_number(oldest_blob_file), + oldest_ancester_time(_oldest_ancester_time), + file_creation_time(_file_creation_time), + epoch_number(_epoch_number), + file_checksum(_file_checksum), + file_checksum_func_name(_file_checksum_func_name), + unique_id(std::move(_unique_id)), + tail_size(_tail_size), + user_defined_timestamps_persisted(_user_defined_timestamps_persisted) { + TEST_SYNC_POINT_CALLBACK("FileMetaData::FileMetaData", this); + } + + // REQUIRED: Keys must be given to the function in sorted order (it expects + // the last key to be the largest). + Status UpdateBoundaries(const Slice& key, const Slice& value, + SequenceNumber seqno, ValueType value_type); + + // Unlike UpdateBoundaries, ranges do not need to be presented in any + // particular order. + void UpdateBoundariesForRange(const InternalKey& start, + const InternalKey& end, SequenceNumber seqno, + const InternalKeyComparator& icmp) { + if (smallest.size() == 0 || icmp.Compare(start, smallest) < 0) { + smallest = start; + } + if (largest.size() == 0 || icmp.Compare(largest, end) < 0) { + largest = end; + } + assert(icmp.Compare(smallest, largest) <= 0); + fd.smallest_seqno = std::min(fd.smallest_seqno, seqno); + fd.largest_seqno = std::max(fd.largest_seqno, seqno); + } + + // Try to get oldest ancester time from the class itself or table properties + // if table reader is already pinned. + // 0 means the information is not available. + uint64_t TryGetOldestAncesterTime() { + if (oldest_ancester_time != kUnknownOldestAncesterTime) { + return oldest_ancester_time; + } else if (fd.table_reader != nullptr && + fd.table_reader->GetTableProperties() != nullptr) { + return fd.table_reader->GetTableProperties()->creation_time; + } + return kUnknownOldestAncesterTime; + } + + uint64_t TryGetFileCreationTime() { + if (file_creation_time != kUnknownFileCreationTime) { + return file_creation_time; + } else if (fd.table_reader != nullptr && + fd.table_reader->GetTableProperties() != nullptr) { + return fd.table_reader->GetTableProperties()->file_creation_time; + } + return kUnknownFileCreationTime; + } + + // WARNING: manual update to this function is needed + // whenever a new string property is added to FileMetaData + // to reduce approximation error. + // + // TODO: eliminate the need of manually updating this function + // for new string properties + size_t ApproximateMemoryUsage() const { + size_t usage = 0; +#ifdef ROCKSDB_MALLOC_USABLE_SIZE + usage += malloc_usable_size(const_cast(this)); +#else + usage += sizeof(*this); +#endif // ROCKSDB_MALLOC_USABLE_SIZE + usage += smallest.size() + largest.size() + file_checksum.size() + + file_checksum_func_name.size(); + return usage; + } +}; + +// A compressed copy of file meta data that just contain minimum data needed +// to serve read operations, while still keeping the pointer to full metadata +// of the file in case it is needed. +struct FdWithKeyRange { + FileDescriptor fd; + FileMetaData* file_metadata; // Point to all metadata + Slice smallest_key; // slice that contain smallest key + Slice largest_key; // slice that contain largest key + + FdWithKeyRange() + : fd(), file_metadata(nullptr), smallest_key(), largest_key() {} + + FdWithKeyRange(FileDescriptor _fd, Slice _smallest_key, Slice _largest_key, + FileMetaData* _file_metadata) + : fd(_fd), + file_metadata(_file_metadata), + smallest_key(_smallest_key), + largest_key(_largest_key) {} +}; + +// Data structure to store an array of FdWithKeyRange in one level +// Actual data is guaranteed to be stored closely +struct LevelFilesBrief { + size_t num_files; + FdWithKeyRange* files; + LevelFilesBrief() { + num_files = 0; + files = nullptr; + } +}; + +// The state of a DB at any given time is referred to as a Version. +// Any modification to the Version is considered a Version Edit. A Version is +// constructed by joining a sequence of Version Edits. Version Edits are written +// to the MANIFEST file. +class VersionEdit { + public: + void Clear(); + + void SetDBId(const std::string& db_id) { + has_db_id_ = true; + db_id_ = db_id; + } + bool HasDbId() const { return has_db_id_; } + const std::string& GetDbId() const { return db_id_; } + + void SetComparatorName(const Slice& name) { + has_comparator_ = true; + comparator_ = name.ToString(); + } + bool HasComparatorName() const { return has_comparator_; } + const std::string& GetComparatorName() const { return comparator_; } + + void SetPersistUserDefinedTimestamps(bool persist_user_defined_timestamps) { + has_persist_user_defined_timestamps_ = true; + persist_user_defined_timestamps_ = persist_user_defined_timestamps; + } + bool HasPersistUserDefinedTimestamps() const { + return has_persist_user_defined_timestamps_; + } + bool GetPersistUserDefinedTimestamps() const { + return persist_user_defined_timestamps_; + } + + void SetLogNumber(uint64_t num) { + has_log_number_ = true; + log_number_ = num; + } + bool HasLogNumber() const { return has_log_number_; } + uint64_t GetLogNumber() const { return log_number_; } + + void SetPrevLogNumber(uint64_t num) { + has_prev_log_number_ = true; + prev_log_number_ = num; + } + bool HasPrevLogNumber() const { return has_prev_log_number_; } + uint64_t GetPrevLogNumber() const { return prev_log_number_; } + + void SetNextFile(uint64_t num) { + has_next_file_number_ = true; + next_file_number_ = num; + } + bool HasNextFile() const { return has_next_file_number_; } + uint64_t GetNextFile() const { return next_file_number_; } + + void SetMaxColumnFamily(uint32_t max_column_family) { + has_max_column_family_ = true; + max_column_family_ = max_column_family; + } + bool HasMaxColumnFamily() const { return has_max_column_family_; } + uint32_t GetMaxColumnFamily() const { return max_column_family_; } + + void SetMinLogNumberToKeep(uint64_t num) { + has_min_log_number_to_keep_ = true; + min_log_number_to_keep_ = num; + } + bool HasMinLogNumberToKeep() const { return has_min_log_number_to_keep_; } + uint64_t GetMinLogNumberToKeep() const { return min_log_number_to_keep_; } + + void SetLastSequence(SequenceNumber seq) { + has_last_sequence_ = true; + last_sequence_ = seq; + } + bool HasLastSequence() const { return has_last_sequence_; } + SequenceNumber GetLastSequence() const { return last_sequence_; } + + // Delete the specified table file from the specified level. + void DeleteFile(int level, uint64_t file) { + deleted_files_.emplace(level, file); + } + + // Retrieve the table files deleted as well as their associated levels. + using DeletedFiles = std::set>; + const DeletedFiles& GetDeletedFiles() const { return deleted_files_; } + + // Add the specified table file at the specified level. + // REQUIRES: "smallest" and "largest" are smallest and largest keys in file + // REQUIRES: "oldest_blob_file_number" is the number of the oldest blob file + // referred to by this file if any, kInvalidBlobFileNumber otherwise. + void AddFile(int level, uint64_t file, uint32_t file_path_id, + uint64_t file_size, const InternalKey& smallest, + const InternalKey& largest, const SequenceNumber& smallest_seqno, + const SequenceNumber& largest_seqno, bool marked_for_compaction, + Temperature temperature, uint64_t oldest_blob_file_number, + uint64_t oldest_ancester_time, uint64_t file_creation_time, + uint64_t epoch_number, const std::string& file_checksum, + const std::string& file_checksum_func_name, + const UniqueId64x2& unique_id, + const uint64_t compensated_range_deletion_size, + uint64_t tail_size, bool user_defined_timestamps_persisted) { + assert(smallest_seqno <= largest_seqno); + new_files_.emplace_back( + level, + FileMetaData(file, file_path_id, file_size, smallest, largest, + smallest_seqno, largest_seqno, marked_for_compaction, + temperature, oldest_blob_file_number, oldest_ancester_time, + file_creation_time, epoch_number, file_checksum, + file_checksum_func_name, unique_id, + compensated_range_deletion_size, tail_size, + user_defined_timestamps_persisted)); + if (!HasLastSequence() || largest_seqno > GetLastSequence()) { + SetLastSequence(largest_seqno); + } + } + + void AddFile(int level, const FileMetaData& f) { + assert(f.fd.smallest_seqno <= f.fd.largest_seqno); + new_files_.emplace_back(level, f); + if (!HasLastSequence() || f.fd.largest_seqno > GetLastSequence()) { + SetLastSequence(f.fd.largest_seqno); + } + } + + // Retrieve the table files added as well as their associated levels. + using NewFiles = std::vector>; + const NewFiles& GetNewFiles() const { return new_files_; } + + NewFiles& GetMutableNewFiles() { return new_files_; } + + // Retrieve all the compact cursors + using CompactCursors = std::vector>; + const CompactCursors& GetCompactCursors() const { return compact_cursors_; } + void AddCompactCursor(int level, const InternalKey& cursor) { + compact_cursors_.push_back(std::make_pair(level, cursor)); + } + void SetCompactCursors( + const std::vector& compact_cursors_by_level) { + compact_cursors_.clear(); + compact_cursors_.reserve(compact_cursors_by_level.size()); + for (int i = 0; i < (int)compact_cursors_by_level.size(); i++) { + if (compact_cursors_by_level[i].Valid()) { + compact_cursors_.push_back( + std::make_pair(i, compact_cursors_by_level[i])); + } + } + } + + // Add a new blob file. + void AddBlobFile(uint64_t blob_file_number, uint64_t total_blob_count, + uint64_t total_blob_bytes, std::string checksum_method, + std::string checksum_value) { + blob_file_additions_.emplace_back( + blob_file_number, total_blob_count, total_blob_bytes, + std::move(checksum_method), std::move(checksum_value)); + } + + void AddBlobFile(BlobFileAddition blob_file_addition) { + blob_file_additions_.emplace_back(std::move(blob_file_addition)); + } + + // Retrieve all the blob files added. + using BlobFileAdditions = std::vector; + const BlobFileAdditions& GetBlobFileAdditions() const { + return blob_file_additions_; + } + + void SetBlobFileAdditions(BlobFileAdditions blob_file_additions) { + assert(blob_file_additions_.empty()); + blob_file_additions_ = std::move(blob_file_additions); + } + + // Add garbage for an existing blob file. Note: intentionally broken English + // follows. + void AddBlobFileGarbage(uint64_t blob_file_number, + uint64_t garbage_blob_count, + uint64_t garbage_blob_bytes) { + blob_file_garbages_.emplace_back(blob_file_number, garbage_blob_count, + garbage_blob_bytes); + } + + void AddBlobFileGarbage(BlobFileGarbage blob_file_garbage) { + blob_file_garbages_.emplace_back(std::move(blob_file_garbage)); + } + + // Retrieve all the blob file garbage added. + using BlobFileGarbages = std::vector; + const BlobFileGarbages& GetBlobFileGarbages() const { + return blob_file_garbages_; + } + + void SetBlobFileGarbages(BlobFileGarbages blob_file_garbages) { + assert(blob_file_garbages_.empty()); + blob_file_garbages_ = std::move(blob_file_garbages); + } + + // Add a WAL (either just created or closed). + // AddWal and DeleteWalsBefore cannot be called on the same VersionEdit. + void AddWal(WalNumber number, WalMetadata metadata = WalMetadata()) { + assert(NumEntries() == wal_additions_.size()); + wal_additions_.emplace_back(number, std::move(metadata)); + } + + // Retrieve all the added WALs. + const WalAdditions& GetWalAdditions() const { return wal_additions_; } + + bool IsWalAddition() const { return !wal_additions_.empty(); } + + // Delete a WAL (either directly deleted or archived). + // AddWal and DeleteWalsBefore cannot be called on the same VersionEdit. + void DeleteWalsBefore(WalNumber number) { + assert((NumEntries() == 1) == !wal_deletion_.IsEmpty()); + wal_deletion_ = WalDeletion(number); + } + + const WalDeletion& GetWalDeletion() const { return wal_deletion_; } + + bool IsWalDeletion() const { return !wal_deletion_.IsEmpty(); } + + bool IsWalManipulation() const { + size_t entries = NumEntries(); + return (entries > 0) && ((entries == wal_additions_.size()) || + (entries == !wal_deletion_.IsEmpty())); + } + + // Number of edits + size_t NumEntries() const { + return new_files_.size() + deleted_files_.size() + + blob_file_additions_.size() + blob_file_garbages_.size() + + wal_additions_.size() + !wal_deletion_.IsEmpty(); + } + + void SetColumnFamily(uint32_t column_family_id) { + column_family_ = column_family_id; + } + uint32_t GetColumnFamily() const { return column_family_; } + + // set column family ID by calling SetColumnFamily() + void AddColumnFamily(const std::string& name) { + assert(!is_column_family_drop_); + assert(!is_column_family_add_); + assert(NumEntries() == 0); + is_column_family_add_ = true; + column_family_name_ = name; + } + + // set column family ID by calling SetColumnFamily() + void DropColumnFamily() { + assert(!is_column_family_drop_); + assert(!is_column_family_add_); + assert(NumEntries() == 0); + is_column_family_drop_ = true; + } + + bool IsColumnFamilyManipulation() const { + return is_column_family_add_ || is_column_family_drop_; + } + + bool IsColumnFamilyAdd() const { return is_column_family_add_; } + + bool IsColumnFamilyDrop() const { return is_column_family_drop_; } + + void MarkAtomicGroup(uint32_t remaining_entries) { + is_in_atomic_group_ = true; + remaining_entries_ = remaining_entries; + } + bool IsInAtomicGroup() const { return is_in_atomic_group_; } + uint32_t GetRemainingEntries() const { return remaining_entries_; } + + bool HasFullHistoryTsLow() const { return !full_history_ts_low_.empty(); } + const std::string& GetFullHistoryTsLow() const { + assert(HasFullHistoryTsLow()); + return full_history_ts_low_; + } + void SetFullHistoryTsLow(std::string full_history_ts_low) { + assert(!full_history_ts_low.empty()); + full_history_ts_low_ = std::move(full_history_ts_low); + } + + // return true on success. + // `ts_sz` is the size in bytes for the user-defined timestamp contained in + // a user key. This argument is optional because it's only required for + // encoding a `VersionEdit` with new SST files to add. It's used to handle the + // file boundaries: `smallest`, `largest` when + // `FileMetaData.user_defined_timestamps_persisted` is false. When reading + // the Manifest file, a mirroring change needed to handle + // file boundaries are not added to the `VersionEdit.DecodeFrom` function + // because timestamp size is not available at `VersionEdit` decoding time, + // it's instead added to `VersionEditHandler::OnNonCfOperation`. + bool EncodeTo(std::string* dst, + std::optional ts_sz = std::nullopt) const; + Status DecodeFrom(const Slice& src); + + std::string DebugString(bool hex_key = false) const; + std::string DebugJSON(int edit_num, bool hex_key = false) const; + + private: + friend class ReactiveVersionSet; + friend class VersionEditHandlerBase; + friend class ListColumnFamiliesHandler; + friend class VersionEditHandler; + friend class VersionEditHandlerPointInTime; + friend class DumpManifestHandler; + friend class VersionSet; + friend class Version; + friend class AtomicGroupReadBuffer; + + bool GetLevel(Slice* input, int* level, const char** msg); + + const char* DecodeNewFile4From(Slice* input); + + // Encode file boundaries `FileMetaData.smallest` and `FileMetaData.largest`. + // User-defined timestamps in the user key will be stripped if they shouldn't + // be persisted. + void EncodeFileBoundaries(std::string* dst, const FileMetaData& meta, + size_t ts_sz) const; + + int max_level_ = 0; + std::string db_id_; + std::string comparator_; + uint64_t log_number_ = 0; + uint64_t prev_log_number_ = 0; + uint64_t next_file_number_ = 0; + uint32_t max_column_family_ = 0; + // The most recent WAL log number that is deleted + uint64_t min_log_number_to_keep_ = 0; + SequenceNumber last_sequence_ = 0; + bool has_db_id_ = false; + bool has_comparator_ = false; + bool has_log_number_ = false; + bool has_prev_log_number_ = false; + bool has_next_file_number_ = false; + bool has_max_column_family_ = false; + bool has_min_log_number_to_keep_ = false; + bool has_last_sequence_ = false; + bool has_persist_user_defined_timestamps_ = false; + + // Compaction cursors for round-robin compaction policy + CompactCursors compact_cursors_; + + DeletedFiles deleted_files_; + NewFiles new_files_; + + BlobFileAdditions blob_file_additions_; + BlobFileGarbages blob_file_garbages_; + + WalAdditions wal_additions_; + WalDeletion wal_deletion_; + + // Each version edit record should have column_family_ set + // If it's not set, it is default (0) + uint32_t column_family_ = 0; + // a version edit can be either column_family add or + // column_family drop. If it's column family add, + // it also includes column family name. + bool is_column_family_drop_ = false; + bool is_column_family_add_ = false; + std::string column_family_name_; + + bool is_in_atomic_group_ = false; + uint32_t remaining_entries_ = 0; + + std::string full_history_ts_low_; + bool persist_user_defined_timestamps_ = true; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/version_edit_handler.cc b/librocksdb-sys/rocksdb/db/version_edit_handler.cc new file mode 100644 index 0000000..7f8e303 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/version_edit_handler.cc @@ -0,0 +1,1089 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/version_edit_handler.h" + +#include +#include + +#include "db/blob/blob_file_reader.h" +#include "db/blob/blob_source.h" +#include "db/version_edit.h" +#include "logging/logging.h" +#include "monitoring/persistent_stats_history.h" +#include "util/udt_util.h" + +namespace ROCKSDB_NAMESPACE { + +void VersionEditHandlerBase::Iterate(log::Reader& reader, + Status* log_read_status) { + Slice record; + std::string scratch; + assert(log_read_status); + assert(log_read_status->ok()); + + [[maybe_unused]] size_t recovered_edits = 0; + Status s = Initialize(); + while (reader.LastRecordEnd() < max_manifest_read_size_ && s.ok() && + reader.ReadRecord(&record, &scratch) && log_read_status->ok()) { + VersionEdit edit; + s = edit.DecodeFrom(record); + if (!s.ok()) { + break; + } + + s = read_buffer_.AddEdit(&edit); + if (!s.ok()) { + break; + } + ColumnFamilyData* cfd = nullptr; + if (edit.is_in_atomic_group_) { + if (read_buffer_.IsFull()) { + for (auto& e : read_buffer_.replay_buffer()) { + s = ApplyVersionEdit(e, &cfd); + if (!s.ok()) { + break; + } + ++recovered_edits; + } + if (!s.ok()) { + break; + } + read_buffer_.Clear(); + } + } else { + s = ApplyVersionEdit(edit, &cfd); + if (s.ok()) { + ++recovered_edits; + } + } + } + if (!log_read_status->ok()) { + s = *log_read_status; + } + + CheckIterationResult(reader, &s); + + if (!s.ok()) { + if (s.IsCorruption()) { + // when we find a Corruption error, something is + // wrong with the underlying file. in this case we + // want to report the filename, so in here we append + // the filename to the Corruption message + assert(reader.file()); + + // build a new error message + std::stringstream message; + // append previous dynamic state message + const char* state = s.getState(); + if (state != nullptr) { + message << state; + message << ' '; + } + // append the filename to the corruption message + message << " The file " << reader.file()->file_name() + << " may be corrupted."; + // overwrite the status with the extended status + s = Status(s.code(), s.subcode(), s.severity(), message.str()); + } + status_ = s; + } + TEST_SYNC_POINT_CALLBACK("VersionEditHandlerBase::Iterate:Finish", + &recovered_edits); +} + +Status ListColumnFamiliesHandler::ApplyVersionEdit( + VersionEdit& edit, ColumnFamilyData** /*unused*/) { + Status s; + if (edit.is_column_family_add_) { + if (column_family_names_.find(edit.column_family_) != + column_family_names_.end()) { + s = Status::Corruption("Manifest adding the same column family twice"); + } else { + column_family_names_.insert( + {edit.column_family_, edit.column_family_name_}); + } + } else if (edit.is_column_family_drop_) { + if (column_family_names_.find(edit.column_family_) == + column_family_names_.end()) { + s = Status::Corruption("Manifest - dropping non-existing column family"); + } else { + column_family_names_.erase(edit.column_family_); + } + } + return s; +} + +Status FileChecksumRetriever::ApplyVersionEdit(VersionEdit& edit, + ColumnFamilyData** /*unused*/) { + for (const auto& deleted_file : edit.GetDeletedFiles()) { + Status s = file_checksum_list_.RemoveOneFileChecksum(deleted_file.second); + if (!s.ok()) { + return s; + } + } + for (const auto& new_file : edit.GetNewFiles()) { + Status s = file_checksum_list_.InsertOneFileChecksum( + new_file.second.fd.GetNumber(), new_file.second.file_checksum, + new_file.second.file_checksum_func_name); + if (!s.ok()) { + return s; + } + } + for (const auto& new_blob_file : edit.GetBlobFileAdditions()) { + std::string checksum_value = new_blob_file.GetChecksumValue(); + std::string checksum_method = new_blob_file.GetChecksumMethod(); + assert(checksum_value.empty() == checksum_method.empty()); + if (checksum_method.empty()) { + checksum_value = kUnknownFileChecksum; + checksum_method = kUnknownFileChecksumFuncName; + } + Status s = file_checksum_list_.InsertOneFileChecksum( + new_blob_file.GetBlobFileNumber(), checksum_value, checksum_method); + if (!s.ok()) { + return s; + } + } + return Status::OK(); +} + +VersionEditHandler::VersionEditHandler( + bool read_only, std::vector column_families, + VersionSet* version_set, bool track_missing_files, + bool no_error_if_files_missing, const std::shared_ptr& io_tracer, + const ReadOptions& read_options, bool skip_load_table_files, + EpochNumberRequirement epoch_number_requirement) + : VersionEditHandlerBase(read_options), + read_only_(read_only), + column_families_(std::move(column_families)), + version_set_(version_set), + track_missing_files_(track_missing_files), + no_error_if_files_missing_(no_error_if_files_missing), + io_tracer_(io_tracer), + skip_load_table_files_(skip_load_table_files), + initialized_(false), + epoch_number_requirement_(epoch_number_requirement) { + assert(version_set_ != nullptr); +} + +Status VersionEditHandler::Initialize() { + Status s; + if (!initialized_) { + for (const auto& cf_desc : column_families_) { + name_to_options_.emplace(cf_desc.name, cf_desc.options); + } + auto default_cf_iter = name_to_options_.find(kDefaultColumnFamilyName); + if (default_cf_iter == name_to_options_.end()) { + s = Status::InvalidArgument("Default column family not specified"); + } + if (s.ok()) { + VersionEdit default_cf_edit; + default_cf_edit.AddColumnFamily(kDefaultColumnFamilyName); + default_cf_edit.SetColumnFamily(0); + ColumnFamilyData* cfd = + CreateCfAndInit(default_cf_iter->second, default_cf_edit); + assert(cfd != nullptr); +#ifdef NDEBUG + (void)cfd; +#endif + initialized_ = true; + } + } + return s; +} + +Status VersionEditHandler::ApplyVersionEdit(VersionEdit& edit, + ColumnFamilyData** cfd) { + Status s; + if (edit.is_column_family_add_) { + s = OnColumnFamilyAdd(edit, cfd); + } else if (edit.is_column_family_drop_) { + s = OnColumnFamilyDrop(edit, cfd); + } else if (edit.IsWalAddition()) { + s = OnWalAddition(edit); + } else if (edit.IsWalDeletion()) { + s = OnWalDeletion(edit); + } else { + s = OnNonCfOperation(edit, cfd); + } + if (s.ok()) { + assert(cfd != nullptr); + s = ExtractInfoFromVersionEdit(*cfd, edit); + } + return s; +} + +Status VersionEditHandler::OnColumnFamilyAdd(VersionEdit& edit, + ColumnFamilyData** cfd) { + bool cf_in_not_found = false; + bool cf_in_builders = false; + CheckColumnFamilyId(edit, &cf_in_not_found, &cf_in_builders); + + assert(cfd != nullptr); + *cfd = nullptr; + Status s; + if (cf_in_builders || cf_in_not_found) { + s = Status::Corruption("MANIFEST adding the same column family twice: " + + edit.column_family_name_); + } + if (s.ok()) { + auto cf_options = name_to_options_.find(edit.column_family_name_); + // implicitly add persistent_stats column family without requiring user + // to specify + ColumnFamilyData* tmp_cfd = nullptr; + bool is_persistent_stats_column_family = + edit.column_family_name_.compare(kPersistentStatsColumnFamilyName) == 0; + if (cf_options == name_to_options_.end() && + !is_persistent_stats_column_family) { + column_families_not_found_.emplace(edit.column_family_, + edit.column_family_name_); + } else { + if (is_persistent_stats_column_family) { + ColumnFamilyOptions cfo; + OptimizeForPersistentStats(&cfo); + tmp_cfd = CreateCfAndInit(cfo, edit); + } else { + tmp_cfd = CreateCfAndInit(cf_options->second, edit); + } + *cfd = tmp_cfd; + } + } + return s; +} + +Status VersionEditHandler::OnColumnFamilyDrop(VersionEdit& edit, + ColumnFamilyData** cfd) { + bool cf_in_not_found = false; + bool cf_in_builders = false; + CheckColumnFamilyId(edit, &cf_in_not_found, &cf_in_builders); + + assert(cfd != nullptr); + *cfd = nullptr; + ColumnFamilyData* tmp_cfd = nullptr; + Status s; + if (cf_in_builders) { + tmp_cfd = DestroyCfAndCleanup(edit); + } else if (cf_in_not_found) { + column_families_not_found_.erase(edit.column_family_); + } else { + s = Status::Corruption("MANIFEST - dropping non-existing column family"); + } + *cfd = tmp_cfd; + return s; +} + +Status VersionEditHandler::OnWalAddition(VersionEdit& edit) { + assert(edit.IsWalAddition()); + return version_set_->wals_.AddWals(edit.GetWalAdditions()); +} + +Status VersionEditHandler::OnWalDeletion(VersionEdit& edit) { + assert(edit.IsWalDeletion()); + return version_set_->wals_.DeleteWalsBefore( + edit.GetWalDeletion().GetLogNumber()); +} + +Status VersionEditHandler::OnNonCfOperation(VersionEdit& edit, + ColumnFamilyData** cfd) { + bool cf_in_not_found = false; + bool cf_in_builders = false; + CheckColumnFamilyId(edit, &cf_in_not_found, &cf_in_builders); + + assert(cfd != nullptr); + *cfd = nullptr; + Status s; + if (!cf_in_not_found) { + if (!cf_in_builders) { + s = Status::Corruption( + "MANIFEST record referencing unknown column family"); + } + ColumnFamilyData* tmp_cfd = nullptr; + if (s.ok()) { + auto builder_iter = builders_.find(edit.column_family_); + assert(builder_iter != builders_.end()); + tmp_cfd = version_set_->GetColumnFamilySet()->GetColumnFamily( + edit.column_family_); + assert(tmp_cfd != nullptr); + // It's important to handle file boundaries before `MaybeCreateVersion` + // because `VersionEditHandlerPointInTime::MaybeCreateVersion` does + // `FileMetaData` verification that involves the file boundaries. + // All `VersionEditHandlerBase` subclasses that need to deal with + // `FileMetaData` for new files are also subclasses of + // `VersionEditHandler`, so it's sufficient to do the file boundaries + // handling in this method. + s = MaybeHandleFileBoundariesForNewFiles(edit, tmp_cfd); + if (!s.ok()) { + return s; + } + s = MaybeCreateVersion(edit, tmp_cfd, /*force_create_version=*/false); + if (s.ok()) { + s = builder_iter->second->version_builder()->Apply(&edit); + } + } + *cfd = tmp_cfd; + } + return s; +} + +// TODO maybe cache the computation result +bool VersionEditHandler::HasMissingFiles() const { + bool ret = false; + for (const auto& elem : cf_to_missing_files_) { + const auto& missing_files = elem.second; + if (!missing_files.empty()) { + ret = true; + break; + } + } + if (!ret) { + for (const auto& elem : cf_to_missing_blob_files_high_) { + if (elem.second != kInvalidBlobFileNumber) { + ret = true; + break; + } + } + } + return ret; +} + +void VersionEditHandler::CheckColumnFamilyId(const VersionEdit& edit, + bool* cf_in_not_found, + bool* cf_in_builders) const { + assert(cf_in_not_found != nullptr); + assert(cf_in_builders != nullptr); + // Not found means that user didn't supply that column + // family option AND we encountered column family add + // record. Once we encounter column family drop record, + // we will delete the column family from + // column_families_not_found. + bool in_not_found = column_families_not_found_.find(edit.column_family_) != + column_families_not_found_.end(); + // in builders means that user supplied that column family + // option AND that we encountered column family add record + bool in_builders = builders_.find(edit.column_family_) != builders_.end(); + // They cannot both be true + assert(!(in_not_found && in_builders)); + *cf_in_not_found = in_not_found; + *cf_in_builders = in_builders; +} + +void VersionEditHandler::CheckIterationResult(const log::Reader& reader, + Status* s) { + assert(s != nullptr); + if (!s->ok()) { + // Do nothing here. + } else if (!version_edit_params_.has_log_number_ || + !version_edit_params_.has_next_file_number_ || + !version_edit_params_.has_last_sequence_) { + std::string msg("no "); + if (!version_edit_params_.has_log_number_) { + msg.append("log_file_number, "); + } + if (!version_edit_params_.has_next_file_number_) { + msg.append("next_file_number, "); + } + if (!version_edit_params_.has_last_sequence_) { + msg.append("last_sequence, "); + } + msg = msg.substr(0, msg.size() - 2); + msg.append(" entry in MANIFEST"); + *s = Status::Corruption(msg); + } + // There were some column families in the MANIFEST that weren't specified + // in the argument. This is OK in read_only mode + if (s->ok() && MustOpenAllColumnFamilies() && + !column_families_not_found_.empty()) { + std::string msg; + for (const auto& cf : column_families_not_found_) { + msg.append(", "); + msg.append(cf.second); + } + msg = msg.substr(2); + *s = Status::InvalidArgument("Column families not opened: " + msg); + } + if (s->ok()) { + version_set_->GetColumnFamilySet()->UpdateMaxColumnFamily( + version_edit_params_.max_column_family_); + version_set_->MarkMinLogNumberToKeep( + version_edit_params_.min_log_number_to_keep_); + version_set_->MarkFileNumberUsed(version_edit_params_.prev_log_number_); + version_set_->MarkFileNumberUsed(version_edit_params_.log_number_); + for (auto* cfd : *(version_set_->GetColumnFamilySet())) { + if (cfd->IsDropped()) { + continue; + } + auto builder_iter = builders_.find(cfd->GetID()); + assert(builder_iter != builders_.end()); + auto* builder = builder_iter->second->version_builder(); + if (!builder->CheckConsistencyForNumLevels()) { + *s = Status::InvalidArgument( + "db has more levels than options.num_levels"); + break; + } + } + } + if (s->ok()) { + for (auto* cfd : *(version_set_->GetColumnFamilySet())) { + if (cfd->IsDropped()) { + continue; + } + if (read_only_) { + cfd->table_cache()->SetTablesAreImmortal(); + } + *s = LoadTables(cfd, /*prefetch_index_and_filter_in_cache=*/false, + /*is_initial_load=*/true); + if (!s->ok()) { + // If s is IOError::PathNotFound, then we mark the db as corrupted. + if (s->IsPathNotFound()) { + *s = Status::Corruption("Corruption: " + s->ToString()); + } + break; + } + } + } + + if (s->ok()) { + for (auto* cfd : *(version_set_->column_family_set_)) { + if (cfd->IsDropped()) { + continue; + } + assert(cfd->initialized()); + VersionEdit edit; + *s = MaybeCreateVersion(edit, cfd, /*force_create_version=*/true); + if (!s->ok()) { + break; + } + } + } + if (s->ok()) { + version_set_->manifest_file_size_ = reader.GetReadOffset(); + assert(version_set_->manifest_file_size_ > 0); + version_set_->next_file_number_.store( + version_edit_params_.next_file_number_ + 1); + SequenceNumber last_seq = version_edit_params_.last_sequence_; + assert(last_seq != kMaxSequenceNumber); + if (last_seq != kMaxSequenceNumber && + last_seq > version_set_->last_allocated_sequence_.load()) { + version_set_->last_allocated_sequence_.store(last_seq); + } + if (last_seq != kMaxSequenceNumber && + last_seq > version_set_->last_published_sequence_.load()) { + version_set_->last_published_sequence_.store(last_seq); + } + if (last_seq != kMaxSequenceNumber && + last_seq > version_set_->last_sequence_.load()) { + version_set_->last_sequence_.store(last_seq); + } + if (last_seq != kMaxSequenceNumber && + last_seq > version_set_->descriptor_last_sequence_) { + // This is the maximum last sequence of all `VersionEdit`s iterated. It + // may be greater than the maximum `largest_seqno` of all files in case + // the newest data referred to by the MANIFEST has been dropped or had its + // sequence number zeroed through compaction. + version_set_->descriptor_last_sequence_ = last_seq; + } + version_set_->prev_log_number_ = version_edit_params_.prev_log_number_; + } +} + +ColumnFamilyData* VersionEditHandler::CreateCfAndInit( + const ColumnFamilyOptions& cf_options, const VersionEdit& edit) { + ColumnFamilyData* cfd = + version_set_->CreateColumnFamily(cf_options, read_options_, &edit); + assert(cfd != nullptr); + cfd->set_initialized(); + assert(builders_.find(edit.column_family_) == builders_.end()); + builders_.emplace(edit.column_family_, + VersionBuilderUPtr(new BaseReferencedVersionBuilder(cfd))); + if (track_missing_files_) { + cf_to_missing_files_.emplace(edit.column_family_, + std::unordered_set()); + cf_to_missing_blob_files_high_.emplace(edit.column_family_, + kInvalidBlobFileNumber); + } + return cfd; +} + +ColumnFamilyData* VersionEditHandler::DestroyCfAndCleanup( + const VersionEdit& edit) { + auto builder_iter = builders_.find(edit.column_family_); + assert(builder_iter != builders_.end()); + builders_.erase(builder_iter); + if (track_missing_files_) { + auto missing_files_iter = cf_to_missing_files_.find(edit.column_family_); + assert(missing_files_iter != cf_to_missing_files_.end()); + cf_to_missing_files_.erase(missing_files_iter); + + auto missing_blob_files_high_iter = + cf_to_missing_blob_files_high_.find(edit.column_family_); + assert(missing_blob_files_high_iter != + cf_to_missing_blob_files_high_.end()); + cf_to_missing_blob_files_high_.erase(missing_blob_files_high_iter); + } + ColumnFamilyData* ret = + version_set_->GetColumnFamilySet()->GetColumnFamily(edit.column_family_); + assert(ret != nullptr); + ret->SetDropped(); + ret->UnrefAndTryDelete(); + ret = nullptr; + return ret; +} + +Status VersionEditHandler::MaybeCreateVersion(const VersionEdit& /*edit*/, + ColumnFamilyData* cfd, + bool force_create_version) { + assert(cfd->initialized()); + Status s; + if (force_create_version) { + auto builder_iter = builders_.find(cfd->GetID()); + assert(builder_iter != builders_.end()); + auto* builder = builder_iter->second->version_builder(); + auto* v = new Version(cfd, version_set_, version_set_->file_options_, + *cfd->GetLatestMutableCFOptions(), io_tracer_, + version_set_->current_version_number_++, + epoch_number_requirement_); + s = builder->SaveTo(v->storage_info()); + if (s.ok()) { + // Install new version + v->PrepareAppend( + *cfd->GetLatestMutableCFOptions(), read_options_, + !(version_set_->db_options_->skip_stats_update_on_db_open)); + version_set_->AppendVersion(cfd, v); + } else { + delete v; + } + } + return s; +} + +Status VersionEditHandler::LoadTables(ColumnFamilyData* cfd, + bool prefetch_index_and_filter_in_cache, + bool is_initial_load) { + bool skip_load_table_files = skip_load_table_files_; + TEST_SYNC_POINT_CALLBACK( + "VersionEditHandler::LoadTables:skip_load_table_files", + &skip_load_table_files); + if (skip_load_table_files) { + return Status::OK(); + } + assert(cfd != nullptr); + assert(!cfd->IsDropped()); + auto builder_iter = builders_.find(cfd->GetID()); + assert(builder_iter != builders_.end()); + assert(builder_iter->second != nullptr); + VersionBuilder* builder = builder_iter->second->version_builder(); + assert(builder); + const MutableCFOptions* moptions = cfd->GetLatestMutableCFOptions(); + Status s = builder->LoadTableHandlers( + cfd->internal_stats(), + version_set_->db_options_->max_file_opening_threads, + prefetch_index_and_filter_in_cache, is_initial_load, + moptions->prefix_extractor, MaxFileSizeForL0MetaPin(*moptions), + read_options_, moptions->block_protection_bytes_per_key); + if ((s.IsPathNotFound() || s.IsCorruption()) && no_error_if_files_missing_) { + s = Status::OK(); + } + if (!s.ok() && !version_set_->db_options_->paranoid_checks) { + s = Status::OK(); + } + return s; +} + +Status VersionEditHandler::ExtractInfoFromVersionEdit(ColumnFamilyData* cfd, + const VersionEdit& edit) { + Status s; + if (edit.has_db_id_) { + version_set_->db_id_ = edit.GetDbId(); + version_edit_params_.SetDBId(edit.db_id_); + } + if (cfd != nullptr) { + if (edit.has_log_number_) { + if (cfd->GetLogNumber() > edit.log_number_) { + ROCKS_LOG_WARN( + version_set_->db_options()->info_log, + "MANIFEST corruption detected, but ignored - Log numbers in " + "records NOT monotonically increasing"); + } else { + cfd->SetLogNumber(edit.log_number_); + version_edit_params_.SetLogNumber(edit.log_number_); + } + } + if (edit.has_comparator_) { + bool mark_sst_files_has_no_udt = false; + // If `persist_user_defined_timestamps` flag is recorded in manifest, it + // is guaranteed to be in the same VersionEdit as comparator. Otherwise, + // it's not recorded and it should have default value true. + s = ValidateUserDefinedTimestampsOptions( + cfd->user_comparator(), edit.comparator_, + cfd->ioptions()->persist_user_defined_timestamps, + edit.persist_user_defined_timestamps_, &mark_sst_files_has_no_udt); + if (!s.ok() && cf_to_cmp_names_) { + cf_to_cmp_names_->emplace(cfd->GetID(), edit.comparator_); + } + if (mark_sst_files_has_no_udt) { + cfds_to_mark_no_udt_.insert(cfd->GetID()); + } + } + if (edit.HasFullHistoryTsLow()) { + const std::string& new_ts = edit.GetFullHistoryTsLow(); + cfd->SetFullHistoryTsLow(new_ts); + } + } + + if (s.ok()) { + if (edit.has_prev_log_number_) { + version_edit_params_.SetPrevLogNumber(edit.prev_log_number_); + } + if (edit.has_next_file_number_) { + version_edit_params_.SetNextFile(edit.next_file_number_); + } + if (edit.has_max_column_family_) { + version_edit_params_.SetMaxColumnFamily(edit.max_column_family_); + } + if (edit.has_min_log_number_to_keep_) { + version_edit_params_.min_log_number_to_keep_ = + std::max(version_edit_params_.min_log_number_to_keep_, + edit.min_log_number_to_keep_); + } + if (edit.has_last_sequence_) { + // `VersionEdit::last_sequence_`s are assumed to be non-decreasing. This + // is legacy behavior that cannot change without breaking downgrade + // compatibility. + assert(!version_edit_params_.has_last_sequence_ || + version_edit_params_.last_sequence_ <= edit.last_sequence_); + version_edit_params_.SetLastSequence(edit.last_sequence_); + } + if (!version_edit_params_.has_prev_log_number_) { + version_edit_params_.SetPrevLogNumber(0); + } + } + return s; +} + +Status VersionEditHandler::MaybeHandleFileBoundariesForNewFiles( + VersionEdit& edit, const ColumnFamilyData* cfd) { + if (edit.GetNewFiles().empty()) { + return Status::OK(); + } + auto ucmp = cfd->user_comparator(); + assert(ucmp); + size_t ts_sz = ucmp->timestamp_size(); + if (ts_sz == 0) { + return Status::OK(); + } + + VersionEdit::NewFiles& new_files = edit.GetMutableNewFiles(); + assert(!new_files.empty()); + // If true, enabling user-defined timestamp is detected for this column + // family. All its existing SST files need to have the file boundaries handled + // and their `persist_user_defined_timestamps` flag set to false regardless of + // its existing value. + bool mark_existing_ssts_with_no_udt = + cfds_to_mark_no_udt_.find(cfd->GetID()) != cfds_to_mark_no_udt_.end(); + bool file_boundaries_need_handling = false; + for (auto& new_file : new_files) { + FileMetaData& meta = new_file.second; + if (meta.user_defined_timestamps_persisted && + !mark_existing_ssts_with_no_udt) { + // `FileMetaData.user_defined_timestamps_persisted` field is the value of + // the flag `AdvancedColumnFamilyOptions.persist_user_defined_timestamps` + // at the time when the SST file was created. As a result, all added SST + // files in one `VersionEdit` should have the same value for it. + if (file_boundaries_need_handling) { + return Status::Corruption( + "New files in one VersionEdit has different " + "user_defined_timestamps_persisted value."); + } + break; + } + file_boundaries_need_handling = true; + assert(!meta.user_defined_timestamps_persisted || + mark_existing_ssts_with_no_udt); + if (mark_existing_ssts_with_no_udt) { + meta.user_defined_timestamps_persisted = false; + } + std::string smallest_buf; + std::string largest_buf; + PadInternalKeyWithMinTimestamp(&smallest_buf, meta.smallest.Encode(), + ts_sz); + PadInternalKeyWithMinTimestamp(&largest_buf, meta.largest.Encode(), ts_sz); + meta.smallest.DecodeFrom(smallest_buf); + meta.largest.DecodeFrom(largest_buf); + } + return Status::OK(); +} + +VersionEditHandlerPointInTime::VersionEditHandlerPointInTime( + bool read_only, std::vector column_families, + VersionSet* version_set, const std::shared_ptr& io_tracer, + const ReadOptions& read_options, + EpochNumberRequirement epoch_number_requirement) + : VersionEditHandler(read_only, column_families, version_set, + /*track_missing_files=*/true, + /*no_error_if_files_missing=*/true, io_tracer, + read_options, epoch_number_requirement) {} + +VersionEditHandlerPointInTime::~VersionEditHandlerPointInTime() { + for (const auto& elem : versions_) { + delete elem.second; + } + versions_.clear(); +} + +void VersionEditHandlerPointInTime::CheckIterationResult( + const log::Reader& reader, Status* s) { + VersionEditHandler::CheckIterationResult(reader, s); + assert(s != nullptr); + if (s->ok()) { + for (auto* cfd : *(version_set_->column_family_set_)) { + if (cfd->IsDropped()) { + continue; + } + assert(cfd->initialized()); + auto v_iter = versions_.find(cfd->GetID()); + if (v_iter != versions_.end()) { + assert(v_iter->second != nullptr); + + version_set_->AppendVersion(cfd, v_iter->second); + versions_.erase(v_iter); + } + } + } else { + for (const auto& elem : versions_) { + delete elem.second; + } + versions_.clear(); + } +} + +ColumnFamilyData* VersionEditHandlerPointInTime::DestroyCfAndCleanup( + const VersionEdit& edit) { + ColumnFamilyData* cfd = VersionEditHandler::DestroyCfAndCleanup(edit); + auto v_iter = versions_.find(edit.column_family_); + if (v_iter != versions_.end()) { + delete v_iter->second; + versions_.erase(v_iter); + } + return cfd; +} + +Status VersionEditHandlerPointInTime::MaybeCreateVersion( + const VersionEdit& edit, ColumnFamilyData* cfd, bool force_create_version) { + assert(cfd != nullptr); + if (!force_create_version) { + assert(edit.column_family_ == cfd->GetID()); + } + auto missing_files_iter = cf_to_missing_files_.find(cfd->GetID()); + assert(missing_files_iter != cf_to_missing_files_.end()); + std::unordered_set& missing_files = missing_files_iter->second; + + auto missing_blob_files_high_iter = + cf_to_missing_blob_files_high_.find(cfd->GetID()); + assert(missing_blob_files_high_iter != cf_to_missing_blob_files_high_.end()); + const uint64_t prev_missing_blob_file_high = + missing_blob_files_high_iter->second; + + VersionBuilder* builder = nullptr; + + if (prev_missing_blob_file_high != kInvalidBlobFileNumber) { + auto builder_iter = builders_.find(cfd->GetID()); + assert(builder_iter != builders_.end()); + builder = builder_iter->second->version_builder(); + assert(builder != nullptr); + } + + // At this point, we have not yet applied the new version edits read from the + // MANIFEST. We check whether we have any missing table and blob files. + const bool prev_has_missing_files = + !missing_files.empty() || + (prev_missing_blob_file_high != kInvalidBlobFileNumber && + prev_missing_blob_file_high >= builder->GetMinOldestBlobFileNumber()); + + for (const auto& file : edit.GetDeletedFiles()) { + uint64_t file_num = file.second; + auto fiter = missing_files.find(file_num); + if (fiter != missing_files.end()) { + missing_files.erase(fiter); + } + } + + assert(!cfd->ioptions()->cf_paths.empty()); + Status s; + for (const auto& elem : edit.GetNewFiles()) { + int level = elem.first; + const FileMetaData& meta = elem.second; + const FileDescriptor& fd = meta.fd; + uint64_t file_num = fd.GetNumber(); + const std::string fpath = + MakeTableFileName(cfd->ioptions()->cf_paths[0].path, file_num); + s = VerifyFile(cfd, fpath, level, meta); + if (s.IsPathNotFound() || s.IsNotFound() || s.IsCorruption()) { + missing_files.insert(file_num); + s = Status::OK(); + } else if (!s.ok()) { + break; + } + } + + uint64_t missing_blob_file_num = prev_missing_blob_file_high; + for (const auto& elem : edit.GetBlobFileAdditions()) { + uint64_t file_num = elem.GetBlobFileNumber(); + s = VerifyBlobFile(cfd, file_num, elem); + if (s.IsPathNotFound() || s.IsNotFound() || s.IsCorruption()) { + missing_blob_file_num = std::max(missing_blob_file_num, file_num); + s = Status::OK(); + } else if (!s.ok()) { + break; + } + } + + bool has_missing_blob_files = false; + if (missing_blob_file_num != kInvalidBlobFileNumber && + missing_blob_file_num >= prev_missing_blob_file_high) { + missing_blob_files_high_iter->second = missing_blob_file_num; + has_missing_blob_files = true; + } else if (missing_blob_file_num < prev_missing_blob_file_high) { + assert(false); + } + + // We still have not applied the new version edit, but have tried to add new + // table and blob files after verifying their presence and consistency. + // Therefore, we know whether we will see new missing table and blob files + // later after actually applying the version edit. We perform the check here + // and record the result. + const bool has_missing_files = + !missing_files.empty() || has_missing_blob_files; + + bool missing_info = !version_edit_params_.has_log_number_ || + !version_edit_params_.has_next_file_number_ || + !version_edit_params_.has_last_sequence_; + + // Create version before apply edit. The version will represent the state + // before applying the version edit. + // A new version will created if: + // 1) no error has occurred so far, and + // 2) log_number_, next_file_number_ and last_sequence_ are known, and + // 3) any of the following: + // a) no missing file before, but will have missing file(s) after applying + // this version edit. + // b) no missing file after applying the version edit, and the caller + // explicitly request that a new version be created. + if (s.ok() && !missing_info && + ((has_missing_files && !prev_has_missing_files) || + (!has_missing_files && force_create_version))) { + if (!builder) { + auto builder_iter = builders_.find(cfd->GetID()); + assert(builder_iter != builders_.end()); + builder = builder_iter->second->version_builder(); + assert(builder); + } + + const MutableCFOptions* cf_opts_ptr = cfd->GetLatestMutableCFOptions(); + auto* version = new Version(cfd, version_set_, version_set_->file_options_, + *cf_opts_ptr, io_tracer_, + version_set_->current_version_number_++, + epoch_number_requirement_); + s = builder->LoadTableHandlers( + cfd->internal_stats(), + version_set_->db_options_->max_file_opening_threads, false, true, + cf_opts_ptr->prefix_extractor, MaxFileSizeForL0MetaPin(*cf_opts_ptr), + read_options_, cf_opts_ptr->block_protection_bytes_per_key); + if (!s.ok()) { + delete version; + if (s.IsCorruption()) { + s = Status::OK(); + } + return s; + } + s = builder->SaveTo(version->storage_info()); + if (s.ok()) { + version->PrepareAppend( + *cfd->GetLatestMutableCFOptions(), read_options_, + !version_set_->db_options_->skip_stats_update_on_db_open); + auto v_iter = versions_.find(cfd->GetID()); + if (v_iter != versions_.end()) { + delete v_iter->second; + v_iter->second = version; + } else { + versions_.emplace(cfd->GetID(), version); + } + } else { + delete version; + } + } + return s; +} + +Status VersionEditHandlerPointInTime::VerifyFile(ColumnFamilyData* cfd, + const std::string& fpath, + int level, + const FileMetaData& fmeta) { + return version_set_->VerifyFileMetadata(read_options_, cfd, fpath, level, + fmeta); +} + +Status VersionEditHandlerPointInTime::VerifyBlobFile( + ColumnFamilyData* cfd, uint64_t blob_file_num, + const BlobFileAddition& blob_addition) { + BlobSource* blob_source = cfd->blob_source(); + assert(blob_source); + CacheHandleGuard blob_file_reader; + + Status s = blob_source->GetBlobFileReader(read_options_, blob_file_num, + &blob_file_reader); + if (!s.ok()) { + return s; + } + // TODO: verify checksum + (void)blob_addition; + return s; +} + +Status VersionEditHandlerPointInTime::LoadTables( + ColumnFamilyData* /*cfd*/, bool /*prefetch_index_and_filter_in_cache*/, + bool /*is_initial_load*/) { + return Status::OK(); +} + +Status ManifestTailer::Initialize() { + if (Mode::kRecovery == mode_) { + return VersionEditHandler::Initialize(); + } + assert(Mode::kCatchUp == mode_); + Status s; + if (!initialized_) { + ColumnFamilySet* cfd_set = version_set_->GetColumnFamilySet(); + assert(cfd_set); + ColumnFamilyData* default_cfd = cfd_set->GetDefault(); + assert(default_cfd); + auto builder_iter = builders_.find(default_cfd->GetID()); + assert(builder_iter != builders_.end()); + + Version* dummy_version = default_cfd->dummy_versions(); + assert(dummy_version); + Version* base_version = dummy_version->Next(); + assert(base_version); + base_version->Ref(); + VersionBuilderUPtr new_builder( + new BaseReferencedVersionBuilder(default_cfd, base_version)); + builder_iter->second = std::move(new_builder); + + initialized_ = true; + } + return s; +} + +Status ManifestTailer::ApplyVersionEdit(VersionEdit& edit, + ColumnFamilyData** cfd) { + Status s = VersionEditHandler::ApplyVersionEdit(edit, cfd); + if (s.ok()) { + assert(cfd); + if (*cfd) { + cfds_changed_.insert(*cfd); + } + } + return s; +} + +Status ManifestTailer::OnColumnFamilyAdd(VersionEdit& edit, + ColumnFamilyData** cfd) { + if (Mode::kRecovery == mode_) { + return VersionEditHandler::OnColumnFamilyAdd(edit, cfd); + } + assert(Mode::kCatchUp == mode_); + ColumnFamilySet* cfd_set = version_set_->GetColumnFamilySet(); + assert(cfd_set); + ColumnFamilyData* tmp_cfd = cfd_set->GetColumnFamily(edit.GetColumnFamily()); + assert(cfd); + *cfd = tmp_cfd; + if (!tmp_cfd) { + // For now, ignore new column families created after Recover() succeeds. + return Status::OK(); + } + auto builder_iter = builders_.find(edit.GetColumnFamily()); + assert(builder_iter != builders_.end()); + + Version* dummy_version = tmp_cfd->dummy_versions(); + assert(dummy_version); + Version* base_version = dummy_version->Next(); + assert(base_version); + base_version->Ref(); + VersionBuilderUPtr new_builder( + new BaseReferencedVersionBuilder(tmp_cfd, base_version)); + builder_iter->second = std::move(new_builder); + +#ifndef NDEBUG + auto version_iter = versions_.find(edit.GetColumnFamily()); + assert(version_iter == versions_.end()); +#endif // !NDEBUG + return Status::OK(); +} + +void ManifestTailer::CheckIterationResult(const log::Reader& reader, + Status* s) { + VersionEditHandlerPointInTime::CheckIterationResult(reader, s); + assert(s); + if (s->ok()) { + if (Mode::kRecovery == mode_) { + mode_ = Mode::kCatchUp; + } else { + assert(Mode::kCatchUp == mode_); + } + } +} + +Status ManifestTailer::VerifyFile(ColumnFamilyData* cfd, + const std::string& fpath, int level, + const FileMetaData& fmeta) { + Status s = + VersionEditHandlerPointInTime::VerifyFile(cfd, fpath, level, fmeta); + // TODO: Open file or create hard link to prevent the file from being + // deleted. + return s; +} + +void DumpManifestHandler::CheckIterationResult(const log::Reader& reader, + Status* s) { + VersionEditHandler::CheckIterationResult(reader, s); + if (!s->ok()) { + fprintf(stdout, "%s\n", s->ToString().c_str()); + return; + } + assert(cf_to_cmp_names_); + for (auto* cfd : *(version_set_->column_family_set_)) { + fprintf(stdout, + "--------------- Column family \"%s\" (ID %" PRIu32 + ") --------------\n", + cfd->GetName().c_str(), cfd->GetID()); + fprintf(stdout, "log number: %" PRIu64 "\n", cfd->GetLogNumber()); + auto it = cf_to_cmp_names_->find(cfd->GetID()); + if (it != cf_to_cmp_names_->end()) { + fprintf(stdout, + "comparator: <%s>, but the comparator object is not available.\n", + it->second.c_str()); + } else { + fprintf(stdout, "comparator: %s\n", cfd->user_comparator()->Name()); + } + assert(cfd->current()); + + // Print out DebugStrings. Can include non-terminating null characters. + fwrite(cfd->current()->DebugString(hex_).data(), sizeof(char), + cfd->current()->DebugString(hex_).size(), stdout); + } + fprintf(stdout, + "next_file_number %" PRIu64 " last_sequence %" PRIu64 + " prev_log_number %" PRIu64 " max_column_family %" PRIu32 + " min_log_number_to_keep %" PRIu64 "\n", + version_set_->current_next_file_number(), + version_set_->LastSequence(), version_set_->prev_log_number(), + version_set_->column_family_set_->GetMaxColumnFamily(), + version_set_->min_log_number_to_keep()); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/version_edit_handler.h b/librocksdb-sys/rocksdb/db/version_edit_handler.h new file mode 100644 index 0000000..dd55a4d --- /dev/null +++ b/librocksdb-sys/rocksdb/db/version_edit_handler.h @@ -0,0 +1,346 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include "db/version_builder.h" +#include "db/version_edit.h" +#include "db/version_set.h" + +namespace ROCKSDB_NAMESPACE { + +struct FileMetaData; + +class VersionEditHandlerBase { + public: + explicit VersionEditHandlerBase(const ReadOptions& read_options) + : read_options_(read_options), + max_manifest_read_size_(std::numeric_limits::max()) {} + + virtual ~VersionEditHandlerBase() {} + + void Iterate(log::Reader& reader, Status* log_read_status); + + const Status& status() const { return status_; } + + AtomicGroupReadBuffer& GetReadBuffer() { return read_buffer_; } + + protected: + explicit VersionEditHandlerBase(const ReadOptions& read_options, + uint64_t max_read_size) + : read_options_(read_options), max_manifest_read_size_(max_read_size) {} + virtual Status Initialize() { return Status::OK(); } + + virtual Status ApplyVersionEdit(VersionEdit& edit, + ColumnFamilyData** cfd) = 0; + + virtual void CheckIterationResult(const log::Reader& /*reader*/, + Status* /*s*/) {} + + void ClearReadBuffer() { read_buffer_.Clear(); } + + Status status_; + + const ReadOptions& read_options_; + + private: + AtomicGroupReadBuffer read_buffer_; + const uint64_t max_manifest_read_size_; +}; + +class ListColumnFamiliesHandler : public VersionEditHandlerBase { + public: + explicit ListColumnFamiliesHandler(const ReadOptions& read_options) + : VersionEditHandlerBase(read_options) {} + + ~ListColumnFamiliesHandler() override {} + + const std::map GetColumnFamilyNames() const { + return column_family_names_; + } + + protected: + Status ApplyVersionEdit(VersionEdit& edit, + ColumnFamilyData** /*unused*/) override; + + private: + // default column family is always implicitly there + std::map column_family_names_{ + {0, kDefaultColumnFamilyName}}; +}; + +class FileChecksumRetriever : public VersionEditHandlerBase { + public: + FileChecksumRetriever(const ReadOptions& read_options, uint64_t max_read_size, + FileChecksumList& file_checksum_list) + : VersionEditHandlerBase(read_options, max_read_size), + file_checksum_list_(file_checksum_list) {} + + ~FileChecksumRetriever() override {} + + protected: + Status ApplyVersionEdit(VersionEdit& edit, + ColumnFamilyData** /*unused*/) override; + + private: + FileChecksumList& file_checksum_list_; +}; + +using VersionBuilderUPtr = std::unique_ptr; + +// A class used for scanning MANIFEST file. +// VersionEditHandler reads a MANIFEST file, parses the version edits, and +// builds the version set's in-memory state, e.g. the version storage info for +// the versions of column families. +// To use this class and its subclasses, +// 1. Create an object of VersionEditHandler or its subclasses. +// VersionEditHandler handler(read_only, column_families, version_set, +// track_missing_files, +// no_error_if_files_missing); +// 2. Status s = handler.Iterate(reader, &db_id); +// 3. Check s and handle possible errors. +// +// Not thread-safe, external synchronization is necessary if an object of +// VersionEditHandler is shared by multiple threads. +class VersionEditHandler : public VersionEditHandlerBase { + public: + explicit VersionEditHandler( + bool read_only, + const std::vector& column_families, + VersionSet* version_set, bool track_missing_files, + bool no_error_if_files_missing, + const std::shared_ptr& io_tracer, + const ReadOptions& read_options, + EpochNumberRequirement epoch_number_requirement = + EpochNumberRequirement::kMustPresent) + : VersionEditHandler( + read_only, column_families, version_set, track_missing_files, + no_error_if_files_missing, io_tracer, read_options, + /*skip_load_table_files=*/false, epoch_number_requirement) {} + + ~VersionEditHandler() override {} + + const VersionEditParams& GetVersionEditParams() const { + return version_edit_params_; + } + + bool HasMissingFiles() const; + + void GetDbId(std::string* db_id) const { + if (db_id && version_edit_params_.has_db_id_) { + *db_id = version_edit_params_.db_id_; + } + } + + protected: + explicit VersionEditHandler( + bool read_only, std::vector column_families, + VersionSet* version_set, bool track_missing_files, + bool no_error_if_files_missing, + const std::shared_ptr& io_tracer, + const ReadOptions& read_options, bool skip_load_table_files, + EpochNumberRequirement epoch_number_requirement = + EpochNumberRequirement::kMustPresent); + + Status ApplyVersionEdit(VersionEdit& edit, ColumnFamilyData** cfd) override; + + virtual Status OnColumnFamilyAdd(VersionEdit& edit, ColumnFamilyData** cfd); + + Status OnColumnFamilyDrop(VersionEdit& edit, ColumnFamilyData** cfd); + + Status OnNonCfOperation(VersionEdit& edit, ColumnFamilyData** cfd); + + Status OnWalAddition(VersionEdit& edit); + + Status OnWalDeletion(VersionEdit& edit); + + Status Initialize() override; + + void CheckColumnFamilyId(const VersionEdit& edit, bool* cf_in_not_found, + bool* cf_in_builders) const; + + void CheckIterationResult(const log::Reader& reader, Status* s) override; + + ColumnFamilyData* CreateCfAndInit(const ColumnFamilyOptions& cf_options, + const VersionEdit& edit); + + virtual ColumnFamilyData* DestroyCfAndCleanup(const VersionEdit& edit); + + virtual Status MaybeCreateVersion(const VersionEdit& edit, + ColumnFamilyData* cfd, + bool force_create_version); + + virtual Status LoadTables(ColumnFamilyData* cfd, + bool prefetch_index_and_filter_in_cache, + bool is_initial_load); + + virtual bool MustOpenAllColumnFamilies() const { return !read_only_; } + + const bool read_only_; + std::vector column_families_; + VersionSet* version_set_; + std::unordered_map builders_; + std::unordered_map name_to_options_; + // Keeps track of column families in manifest that were not found in + // column families parameters. if those column families are not dropped + // by subsequent manifest records, Recover() will return failure status. + std::unordered_map column_families_not_found_; + VersionEditParams version_edit_params_; + const bool track_missing_files_; + std::unordered_map> + cf_to_missing_files_; + std::unordered_map cf_to_missing_blob_files_high_; + bool no_error_if_files_missing_; + std::shared_ptr io_tracer_; + bool skip_load_table_files_; + bool initialized_; + std::unique_ptr> cf_to_cmp_names_; + EpochNumberRequirement epoch_number_requirement_; + std::unordered_set cfds_to_mark_no_udt_; + + private: + Status ExtractInfoFromVersionEdit(ColumnFamilyData* cfd, + const VersionEdit& edit); + + // When `FileMetaData.user_defined_timestamps_persisted` is false and + // user-defined timestamp size is non-zero. User-defined timestamps are + // stripped from file boundaries: `smallest`, `largest` in + // `VersionEdit.DecodeFrom` before they were written to Manifest. + // This is the mirroring change to handle file boundaries on the Manifest read + // path for this scenario: to pad a minimum timestamp to the user key in + // `smallest` and `largest` so their format are consistent with the running + // user comparator. + Status MaybeHandleFileBoundariesForNewFiles(VersionEdit& edit, + const ColumnFamilyData* cfd); +}; + +// A class similar to its base class, i.e. VersionEditHandler. +// VersionEditHandlerPointInTime restores the versions to the most recent point +// in time such that at this point, the version does not have missing files. +// +// Not thread-safe, external synchronization is necessary if an object of +// VersionEditHandlerPointInTime is shared by multiple threads. +class VersionEditHandlerPointInTime : public VersionEditHandler { + public: + VersionEditHandlerPointInTime( + bool read_only, std::vector column_families, + VersionSet* version_set, const std::shared_ptr& io_tracer, + const ReadOptions& read_options, + EpochNumberRequirement epoch_number_requirement = + EpochNumberRequirement::kMustPresent); + ~VersionEditHandlerPointInTime() override; + + protected: + void CheckIterationResult(const log::Reader& reader, Status* s) override; + ColumnFamilyData* DestroyCfAndCleanup(const VersionEdit& edit) override; + Status MaybeCreateVersion(const VersionEdit& edit, ColumnFamilyData* cfd, + bool force_create_version) override; + virtual Status VerifyFile(ColumnFamilyData* cfd, const std::string& fpath, + int level, const FileMetaData& fmeta); + virtual Status VerifyBlobFile(ColumnFamilyData* cfd, uint64_t blob_file_num, + const BlobFileAddition& blob_addition); + + Status LoadTables(ColumnFamilyData* cfd, + bool prefetch_index_and_filter_in_cache, + bool is_initial_load) override; + + std::unordered_map versions_; +}; + +class ManifestTailer : public VersionEditHandlerPointInTime { + public: + explicit ManifestTailer(std::vector column_families, + VersionSet* version_set, + const std::shared_ptr& io_tracer, + const ReadOptions& read_options, + EpochNumberRequirement epoch_number_requirement = + EpochNumberRequirement::kMustPresent) + : VersionEditHandlerPointInTime(/*read_only=*/false, column_families, + version_set, io_tracer, read_options, + epoch_number_requirement), + mode_(Mode::kRecovery) {} + + void PrepareToReadNewManifest() { + initialized_ = false; + ClearReadBuffer(); + } + + std::unordered_set& GetUpdatedColumnFamilies() { + return cfds_changed_; + } + + protected: + Status Initialize() override; + + bool MustOpenAllColumnFamilies() const override { return false; } + + Status ApplyVersionEdit(VersionEdit& edit, ColumnFamilyData** cfd) override; + + Status OnColumnFamilyAdd(VersionEdit& edit, ColumnFamilyData** cfd) override; + + void CheckIterationResult(const log::Reader& reader, Status* s) override; + + Status VerifyFile(ColumnFamilyData* cfd, const std::string& fpath, int level, + const FileMetaData& fmeta) override; + + enum Mode : uint8_t { + kRecovery = 0, + kCatchUp = 1, + }; + + Mode mode_; + std::unordered_set cfds_changed_; +}; + +class DumpManifestHandler : public VersionEditHandler { + public: + DumpManifestHandler(std::vector column_families, + VersionSet* version_set, + const std::shared_ptr& io_tracer, + const ReadOptions& read_options, bool verbose, bool hex, + bool json) + : VersionEditHandler( + /*read_only=*/true, column_families, version_set, + /*track_missing_files=*/false, + /*no_error_if_files_missing=*/false, io_tracer, read_options, + /*skip_load_table_files=*/true), + verbose_(verbose), + hex_(hex), + json_(json), + count_(0) { + cf_to_cmp_names_.reset(new std::unordered_map()); + } + + ~DumpManifestHandler() override {} + + Status ApplyVersionEdit(VersionEdit& edit, ColumnFamilyData** cfd) override { + // Write out each individual edit + if (verbose_ && !json_) { + // Print out DebugStrings. Can include non-terminating null characters. + fwrite(edit.DebugString(hex_).data(), sizeof(char), + edit.DebugString(hex_).size(), stdout); + } else if (json_) { + // Print out DebugStrings. Can include non-terminating null characters. + fwrite(edit.DebugString(hex_).data(), sizeof(char), + edit.DebugString(hex_).size(), stdout); + } + ++count_; + return VersionEditHandler::ApplyVersionEdit(edit, cfd); + } + + void CheckIterationResult(const log::Reader& reader, Status* s) override; + + private: + const bool verbose_; + const bool hex_; + const bool json_; + int count_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/version_edit_test.cc b/librocksdb-sys/rocksdb/db/version_edit_test.cc new file mode 100644 index 0000000..c473899 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/version_edit_test.cc @@ -0,0 +1,801 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/version_edit.h" + +#include "db/blob/blob_index.h" +#include "rocksdb/advanced_options.h" +#include "table/unique_id_impl.h" +#include "test_util/sync_point.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/coding.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +static void TestEncodeDecode(const VersionEdit& edit) { + // Encoding one `VersionEdit` and decoding it again should result in the + // exact same `VersionEdit`. However, a special handling is applied to file + // boundaries: `FileMetaData.smallest`, `FileMetaData.largest` when + // user-defined timestamps should not be persisted. In that scenario, this + // invariant does not hold. We disable this scenario in this util method to + // enable all other test cases continue to verify this invariant, while the + // special case is separately covered in test + // `EncodeDecodeNewFile4HandleFileBoundary`. + std::string encoded, encoded2; + edit.EncodeTo(&encoded, 0 /* ts_sz */); + VersionEdit parsed; + Status s = parsed.DecodeFrom(encoded); + ASSERT_TRUE(s.ok()) << s.ToString(); + parsed.EncodeTo(&encoded2, 0 /* ts_sz */); + ASSERT_EQ(encoded, encoded2); +} + +class VersionEditTest : public testing::Test {}; + +TEST_F(VersionEditTest, EncodeDecode) { + static const uint64_t kBig = 1ull << 50; + static const uint32_t kBig32Bit = 1ull << 30; + + VersionEdit edit; + for (int i = 0; i < 4; i++) { + TestEncodeDecode(edit); + edit.AddFile(3, kBig + 300 + i, kBig32Bit + 400 + i, 0, + InternalKey("foo", kBig + 500 + i, kTypeValue), + InternalKey("zoo", kBig + 600 + i, kTypeDeletion), + kBig + 500 + i, kBig + 600 + i, false, Temperature::kUnknown, + kInvalidBlobFileNumber, 888, 678, + kBig + 300 + i /* epoch_number */, "234", "crc32c", + kNullUniqueId64x2, 0, 0, true); + edit.DeleteFile(4, kBig + 700 + i); + } + + edit.SetComparatorName("foo"); + edit.SetPersistUserDefinedTimestamps(true); + edit.SetLogNumber(kBig + 100); + edit.SetNextFile(kBig + 200); + edit.SetLastSequence(kBig + 1000); + TestEncodeDecode(edit); +} + +TEST_F(VersionEditTest, EncodeDecodeNewFile4) { + static const uint64_t kBig = 1ull << 50; + + VersionEdit edit; + edit.AddFile(3, 300, 3, 100, InternalKey("foo", kBig + 500, kTypeValue), + InternalKey("zoo", kBig + 600, kTypeDeletion), kBig + 500, + kBig + 600, true, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, + 300 /* epoch_number */, kUnknownFileChecksum, + kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0, true); + edit.AddFile(4, 301, 3, 100, InternalKey("foo", kBig + 501, kTypeValue), + InternalKey("zoo", kBig + 601, kTypeDeletion), kBig + 501, + kBig + 601, false, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, + 301 /* epoch_number */, kUnknownFileChecksum, + kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0, false); + edit.AddFile(5, 302, 0, 100, InternalKey("foo", kBig + 502, kTypeValue), + InternalKey("zoo", kBig + 602, kTypeDeletion), kBig + 502, + kBig + 602, true, Temperature::kUnknown, kInvalidBlobFileNumber, + 666, 888, 302 /* epoch_number */, kUnknownFileChecksum, + kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0, true); + edit.AddFile(5, 303, 0, 100, InternalKey("foo", kBig + 503, kTypeBlobIndex), + InternalKey("zoo", kBig + 603, kTypeBlobIndex), kBig + 503, + kBig + 603, true, Temperature::kUnknown, 1001, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, + 303 /* epoch_number */, kUnknownFileChecksum, + kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0, true); + + edit.DeleteFile(4, 700); + + edit.SetComparatorName("foo"); + edit.SetPersistUserDefinedTimestamps(false); + edit.SetLogNumber(kBig + 100); + edit.SetNextFile(kBig + 200); + edit.SetLastSequence(kBig + 1000); + TestEncodeDecode(edit); + + std::string encoded, encoded2; + edit.EncodeTo(&encoded, 0 /* ts_sz */); + VersionEdit parsed; + Status s = parsed.DecodeFrom(encoded); + ASSERT_TRUE(s.ok()) << s.ToString(); + auto& new_files = parsed.GetNewFiles(); + ASSERT_TRUE(new_files[0].second.marked_for_compaction); + ASSERT_FALSE(new_files[1].second.marked_for_compaction); + ASSERT_TRUE(new_files[2].second.marked_for_compaction); + ASSERT_TRUE(new_files[3].second.marked_for_compaction); + ASSERT_EQ(3u, new_files[0].second.fd.GetPathId()); + ASSERT_EQ(3u, new_files[1].second.fd.GetPathId()); + ASSERT_EQ(0u, new_files[2].second.fd.GetPathId()); + ASSERT_EQ(0u, new_files[3].second.fd.GetPathId()); + ASSERT_EQ(kInvalidBlobFileNumber, + new_files[0].second.oldest_blob_file_number); + ASSERT_EQ(kInvalidBlobFileNumber, + new_files[1].second.oldest_blob_file_number); + ASSERT_EQ(kInvalidBlobFileNumber, + new_files[2].second.oldest_blob_file_number); + ASSERT_EQ(1001, new_files[3].second.oldest_blob_file_number); + ASSERT_TRUE(new_files[0].second.user_defined_timestamps_persisted); + ASSERT_FALSE(new_files[1].second.user_defined_timestamps_persisted); + ASSERT_TRUE(new_files[2].second.user_defined_timestamps_persisted); + ASSERT_TRUE(new_files[3].second.user_defined_timestamps_persisted); + ASSERT_FALSE(parsed.GetPersistUserDefinedTimestamps()); +} + +TEST_F(VersionEditTest, EncodeDecodeNewFile4HandleFileBoundary) { + static const uint64_t kBig = 1ull << 50; + size_t ts_sz = 16; + static std::string min_ts(ts_sz, static_cast(0)); + VersionEdit edit; + std::string smallest = "foo"; + std::string largest = "zoo"; + // In real manifest writing scenarios, one `VersionEdit` should not contain + // files with different `user_defined_timestamps_persisted` flag value. + // This is just for testing file boundaries handling w.r.t persisting user + // defined timestamps during `VersionEdit` encoding. + edit.AddFile( + 3, 300, 3, 100, InternalKey(smallest + min_ts, kBig + 500, kTypeValue), + InternalKey(largest + min_ts, kBig + 600, kTypeDeletion), kBig + 500, + kBig + 600, true, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, + 300 /* epoch_number */, kUnknownFileChecksum, + kUnknownFileChecksumFuncName, kNullUniqueId64x2, + 0 /* compensated_range_deletion_size */, 0 /* tail_size */, + false /* user_defined_timestamps_persisted */); + edit.AddFile(3, 300, 3, 100, + InternalKey(smallest + min_ts, kBig + 500, kTypeValue), + InternalKey(largest + min_ts, kBig + 600, kTypeDeletion), + kBig + 500, kBig + 600, true, Temperature::kUnknown, + kInvalidBlobFileNumber, kUnknownOldestAncesterTime, + kUnknownFileCreationTime, 300 /* epoch_number */, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, + kNullUniqueId64x2, 0 /* compensated_range_deletion_size */, + 0 /* tail_size */, true /* user_defined_timestamps_persisted */); + + std::string encoded; + edit.EncodeTo(&encoded, ts_sz); + VersionEdit parsed; + Status s = parsed.DecodeFrom(encoded); + ASSERT_TRUE(s.ok()) << s.ToString(); + auto& new_files = parsed.GetNewFiles(); + ASSERT_TRUE(new_files.size() == 2); + ASSERT_FALSE(new_files[0].second.user_defined_timestamps_persisted); + // First file's boundaries do not contain user-defined timestamps. + ASSERT_EQ(InternalKey(smallest, kBig + 500, kTypeValue).Encode(), + new_files[0].second.smallest.Encode()); + ASSERT_EQ(InternalKey(largest, kBig + 600, kTypeDeletion).Encode(), + new_files[0].second.largest.Encode()); + ASSERT_TRUE(new_files[1].second.user_defined_timestamps_persisted); + // Second file's boundaries contain user-defined timestamps. + ASSERT_EQ(InternalKey(smallest + min_ts, kBig + 500, kTypeValue).Encode(), + new_files[1].second.smallest.Encode()); + ASSERT_EQ(InternalKey(largest + min_ts, kBig + 600, kTypeDeletion).Encode(), + new_files[1].second.largest.Encode()); +} + +TEST_F(VersionEditTest, ForwardCompatibleNewFile4) { + static const uint64_t kBig = 1ull << 50; + VersionEdit edit; + edit.AddFile(3, 300, 3, 100, InternalKey("foo", kBig + 500, kTypeValue), + InternalKey("zoo", kBig + 600, kTypeDeletion), kBig + 500, + kBig + 600, true, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, + 300 /* epoch_number */, kUnknownFileChecksum, + kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0, true); + edit.AddFile(4, 301, 3, 100, InternalKey("foo", kBig + 501, kTypeValue), + InternalKey("zoo", kBig + 601, kTypeDeletion), kBig + 501, + kBig + 601, false, Temperature::kUnknown, kInvalidBlobFileNumber, + 686, 868, 301 /* epoch_number */, "234", "crc32c", + kNullUniqueId64x2, 0, 0, true); + edit.DeleteFile(4, 700); + + edit.SetComparatorName("foo"); + edit.SetPersistUserDefinedTimestamps(true); + edit.SetLogNumber(kBig + 100); + edit.SetNextFile(kBig + 200); + edit.SetLastSequence(kBig + 1000); + + std::string encoded; + + // Call back function to add extra customized builds. + bool first = true; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "VersionEdit::EncodeTo:NewFile4:CustomizeFields", [&](void* arg) { + std::string* str = reinterpret_cast(arg); + PutVarint32(str, 33); + const std::string str1 = "random_string"; + PutLengthPrefixedSlice(str, str1); + if (first) { + first = false; + PutVarint32(str, 22); + const std::string str2 = "s"; + PutLengthPrefixedSlice(str, str2); + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + edit.EncodeTo(&encoded, 0 /* ts_sz */); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + VersionEdit parsed; + Status s = parsed.DecodeFrom(encoded); + ASSERT_TRUE(s.ok()) << s.ToString(); + ASSERT_TRUE(!first); + auto& new_files = parsed.GetNewFiles(); + ASSERT_TRUE(new_files[0].second.marked_for_compaction); + ASSERT_TRUE(!new_files[1].second.marked_for_compaction); + ASSERT_EQ(3u, new_files[0].second.fd.GetPathId()); + ASSERT_EQ(3u, new_files[1].second.fd.GetPathId()); + ASSERT_EQ(1u, parsed.GetDeletedFiles().size()); + ASSERT_TRUE(parsed.GetPersistUserDefinedTimestamps()); +} + +TEST_F(VersionEditTest, NewFile4NotSupportedField) { + static const uint64_t kBig = 1ull << 50; + VersionEdit edit; + edit.AddFile(3, 300, 3, 100, InternalKey("foo", kBig + 500, kTypeValue), + InternalKey("zoo", kBig + 600, kTypeDeletion), kBig + 500, + kBig + 600, true, Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, + 300 /* epoch_number */, kUnknownFileChecksum, + kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0, false); + + edit.SetComparatorName("foo"); + edit.SetPersistUserDefinedTimestamps(false); + edit.SetLogNumber(kBig + 100); + edit.SetNextFile(kBig + 200); + edit.SetLastSequence(kBig + 1000); + + std::string encoded; + + // Call back function to add extra customized builds. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "VersionEdit::EncodeTo:NewFile4:CustomizeFields", [&](void* arg) { + std::string* str = reinterpret_cast(arg); + const std::string str1 = "s"; + PutLengthPrefixedSlice(str, str1); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + edit.EncodeTo(&encoded, 0 /* ts_sz */); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + VersionEdit parsed; + Status s = parsed.DecodeFrom(encoded); + ASSERT_NOK(s); +} + +TEST_F(VersionEditTest, EncodeEmptyFile) { + VersionEdit edit; + edit.AddFile(0, 0, 0, 0, InternalKey(), InternalKey(), 0, 0, false, + Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, + 1 /*epoch_number*/, kUnknownFileChecksum, + kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0, true); + std::string buffer; + ASSERT_TRUE(!edit.EncodeTo(&buffer, 0 /* ts_sz */)); +} + +TEST_F(VersionEditTest, ColumnFamilyTest) { + VersionEdit edit; + edit.SetColumnFamily(2); + edit.AddColumnFamily("column_family"); + edit.SetMaxColumnFamily(5); + TestEncodeDecode(edit); + + edit.Clear(); + edit.SetColumnFamily(3); + edit.DropColumnFamily(); + TestEncodeDecode(edit); +} + +TEST_F(VersionEditTest, MinLogNumberToKeep) { + VersionEdit edit; + edit.SetMinLogNumberToKeep(13); + TestEncodeDecode(edit); + + edit.Clear(); + edit.SetMinLogNumberToKeep(23); + TestEncodeDecode(edit); +} + +TEST_F(VersionEditTest, AtomicGroupTest) { + VersionEdit edit; + edit.MarkAtomicGroup(1); + TestEncodeDecode(edit); +} + +TEST_F(VersionEditTest, IgnorableField) { + VersionEdit ve; + std::string encoded; + + // Size of ignorable field is too large + PutVarint32Varint64(&encoded, 2 /* kLogNumber */, 66); + // This is a customized ignorable tag + PutVarint32Varint64(&encoded, + 0x2710 /* A field with kTagSafeIgnoreMask set */, + 5 /* fieldlength 5 */); + encoded += "abc"; // Only fills 3 bytes, + ASSERT_NOK(ve.DecodeFrom(encoded)); + + encoded.clear(); + // Error when seeing unidentified tag that is not ignorable + PutVarint32Varint64(&encoded, 2 /* kLogNumber */, 66); + // This is a customized ignorable tag + PutVarint32Varint64(&encoded, 666 /* A field with kTagSafeIgnoreMask unset */, + 3 /* fieldlength 3 */); + encoded += "abc"; // Fill 3 bytes + PutVarint32Varint64(&encoded, 3 /* next file number */, 88); + ASSERT_NOK(ve.DecodeFrom(encoded)); + + // Safely ignore an identified but safely ignorable entry + encoded.clear(); + PutVarint32Varint64(&encoded, 2 /* kLogNumber */, 66); + // This is a customized ignorable tag + PutVarint32Varint64(&encoded, + 0x2710 /* A field with kTagSafeIgnoreMask set */, + 3 /* fieldlength 3 */); + encoded += "abc"; // Fill 3 bytes + PutVarint32Varint64(&encoded, 3 /* kNextFileNumber */, 88); + + ASSERT_OK(ve.DecodeFrom(encoded)); + + ASSERT_TRUE(ve.HasLogNumber()); + ASSERT_TRUE(ve.HasNextFile()); + ASSERT_EQ(66, ve.GetLogNumber()); + ASSERT_EQ(88, ve.GetNextFile()); +} + +TEST_F(VersionEditTest, DbId) { + VersionEdit edit; + edit.SetDBId("ab34-cd12-435f-er00"); + TestEncodeDecode(edit); + + edit.Clear(); + edit.SetDBId("34ba-cd12-435f-er01"); + TestEncodeDecode(edit); +} + +TEST_F(VersionEditTest, BlobFileAdditionAndGarbage) { + VersionEdit edit; + + const std::string checksum_method_prefix = "Hash"; + const std::string checksum_value_prefix = "Value"; + + for (uint64_t blob_file_number = 1; blob_file_number <= 10; + ++blob_file_number) { + const uint64_t total_blob_count = blob_file_number << 10; + const uint64_t total_blob_bytes = blob_file_number << 20; + + std::string checksum_method(checksum_method_prefix); + AppendNumberTo(&checksum_method, blob_file_number); + + std::string checksum_value(checksum_value_prefix); + AppendNumberTo(&checksum_value, blob_file_number); + + edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes, + checksum_method, checksum_value); + + const uint64_t garbage_blob_count = total_blob_count >> 2; + const uint64_t garbage_blob_bytes = total_blob_bytes >> 1; + + edit.AddBlobFileGarbage(blob_file_number, garbage_blob_count, + garbage_blob_bytes); + } + + TestEncodeDecode(edit); +} + +TEST_F(VersionEditTest, AddWalEncodeDecode) { + VersionEdit edit; + for (uint64_t log_number = 1; log_number <= 20; log_number++) { + WalMetadata meta; + bool has_size = rand() % 2 == 0; + if (has_size) { + meta.SetSyncedSizeInBytes(rand() % 1000); + } + edit.AddWal(log_number, meta); + } + TestEncodeDecode(edit); +} + +static std::string PrefixEncodedWalAdditionWithLength( + const std::string& encoded) { + std::string ret; + PutVarint32(&ret, Tag::kWalAddition2); + PutLengthPrefixedSlice(&ret, encoded); + return ret; +} + +TEST_F(VersionEditTest, AddWalDecodeBadLogNumber) { + std::string encoded; + + { + // No log number. + std::string encoded_edit = PrefixEncodedWalAdditionWithLength(encoded); + VersionEdit edit; + Status s = edit.DecodeFrom(encoded_edit); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(s.ToString().find("Error decoding WAL log number") != + std::string::npos) + << s.ToString(); + } + + { + // log number should be varint64, + // but we only encode 128 which is not a valid representation of varint64. + char c = 0; + unsigned char* ptr = reinterpret_cast(&c); + *ptr = 128; + encoded.append(1, c); + + std::string encoded_edit = PrefixEncodedWalAdditionWithLength(encoded); + VersionEdit edit; + Status s = edit.DecodeFrom(encoded_edit); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(s.ToString().find("Error decoding WAL log number") != + std::string::npos) + << s.ToString(); + } +} + +TEST_F(VersionEditTest, AddWalDecodeBadTag) { + constexpr WalNumber kLogNumber = 100; + constexpr uint64_t kSizeInBytes = 100; + + std::string encoded; + PutVarint64(&encoded, kLogNumber); + + { + // No tag. + std::string encoded_edit = PrefixEncodedWalAdditionWithLength(encoded); + VersionEdit edit; + Status s = edit.DecodeFrom(encoded_edit); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(s.ToString().find("Error decoding tag") != std::string::npos) + << s.ToString(); + } + + { + // Only has size tag, no terminate tag. + std::string encoded_with_size = encoded; + PutVarint32(&encoded_with_size, + static_cast(WalAdditionTag::kSyncedSize)); + PutVarint64(&encoded_with_size, kSizeInBytes); + + std::string encoded_edit = + PrefixEncodedWalAdditionWithLength(encoded_with_size); + VersionEdit edit; + Status s = edit.DecodeFrom(encoded_edit); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(s.ToString().find("Error decoding tag") != std::string::npos) + << s.ToString(); + } + + { + // Only has terminate tag. + std::string encoded_with_terminate = encoded; + PutVarint32(&encoded_with_terminate, + static_cast(WalAdditionTag::kTerminate)); + + std::string encoded_edit = + PrefixEncodedWalAdditionWithLength(encoded_with_terminate); + VersionEdit edit; + ASSERT_OK(edit.DecodeFrom(encoded_edit)); + auto& wal_addition = edit.GetWalAdditions()[0]; + ASSERT_EQ(wal_addition.GetLogNumber(), kLogNumber); + ASSERT_FALSE(wal_addition.GetMetadata().HasSyncedSize()); + } +} + +TEST_F(VersionEditTest, AddWalDecodeNoSize) { + constexpr WalNumber kLogNumber = 100; + + std::string encoded; + PutVarint64(&encoded, kLogNumber); + PutVarint32(&encoded, static_cast(WalAdditionTag::kSyncedSize)); + // No real size after the size tag. + + { + // Without terminate tag. + std::string encoded_edit = PrefixEncodedWalAdditionWithLength(encoded); + VersionEdit edit; + Status s = edit.DecodeFrom(encoded_edit); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(s.ToString().find("Error decoding WAL file size") != + std::string::npos) + << s.ToString(); + } + + { + // With terminate tag. + PutVarint32(&encoded, static_cast(WalAdditionTag::kTerminate)); + + std::string encoded_edit = PrefixEncodedWalAdditionWithLength(encoded); + VersionEdit edit; + Status s = edit.DecodeFrom(encoded_edit); + ASSERT_TRUE(s.IsCorruption()); + // The terminate tag is misunderstood as the size. + ASSERT_TRUE(s.ToString().find("Error decoding tag") != std::string::npos) + << s.ToString(); + } +} + +TEST_F(VersionEditTest, AddWalDebug) { + constexpr int n = 2; + constexpr std::array kLogNumbers{{10, 20}}; + constexpr std::array kSizeInBytes{{100, 200}}; + + VersionEdit edit; + for (int i = 0; i < n; i++) { + edit.AddWal(kLogNumbers[i], WalMetadata(kSizeInBytes[i])); + } + + const WalAdditions& wals = edit.GetWalAdditions(); + + ASSERT_TRUE(edit.IsWalAddition()); + ASSERT_EQ(wals.size(), n); + for (int i = 0; i < n; i++) { + const WalAddition& wal = wals[i]; + ASSERT_EQ(wal.GetLogNumber(), kLogNumbers[i]); + ASSERT_EQ(wal.GetMetadata().GetSyncedSizeInBytes(), kSizeInBytes[i]); + } + + std::string expected_str = "VersionEdit {\n"; + for (int i = 0; i < n; i++) { + std::stringstream ss; + ss << " WalAddition: log_number: " << kLogNumbers[i] + << " synced_size_in_bytes: " << kSizeInBytes[i] << "\n"; + expected_str += ss.str(); + } + expected_str += " ColumnFamily: 0\n}\n"; + ASSERT_EQ(edit.DebugString(true), expected_str); + + std::string expected_json = "{\"EditNumber\": 4, \"WalAdditions\": ["; + for (int i = 0; i < n; i++) { + std::stringstream ss; + ss << "{\"LogNumber\": " << kLogNumbers[i] << ", " + << "\"SyncedSizeInBytes\": " << kSizeInBytes[i] << "}"; + if (i < n - 1) ss << ", "; + expected_json += ss.str(); + } + expected_json += "], \"ColumnFamily\": 0}"; + ASSERT_EQ(edit.DebugJSON(4, true), expected_json); +} + +TEST_F(VersionEditTest, DeleteWalEncodeDecode) { + VersionEdit edit; + edit.DeleteWalsBefore(rand() % 100); + TestEncodeDecode(edit); +} + +TEST_F(VersionEditTest, DeleteWalDebug) { + constexpr int n = 2; + constexpr std::array kLogNumbers{{10, 20}}; + + VersionEdit edit; + edit.DeleteWalsBefore(kLogNumbers[n - 1]); + + const WalDeletion& wal = edit.GetWalDeletion(); + + ASSERT_TRUE(edit.IsWalDeletion()); + ASSERT_EQ(wal.GetLogNumber(), kLogNumbers[n - 1]); + + std::string expected_str = "VersionEdit {\n"; + { + std::stringstream ss; + ss << " WalDeletion: log_number: " << kLogNumbers[n - 1] << "\n"; + expected_str += ss.str(); + } + expected_str += " ColumnFamily: 0\n}\n"; + ASSERT_EQ(edit.DebugString(true), expected_str); + + std::string expected_json = "{\"EditNumber\": 4, \"WalDeletion\": "; + { + std::stringstream ss; + ss << "{\"LogNumber\": " << kLogNumbers[n - 1] << "}"; + expected_json += ss.str(); + } + expected_json += ", \"ColumnFamily\": 0}"; + ASSERT_EQ(edit.DebugJSON(4, true), expected_json); +} + +TEST_F(VersionEditTest, FullHistoryTsLow) { + VersionEdit edit; + ASSERT_FALSE(edit.HasFullHistoryTsLow()); + std::string ts = test::EncodeInt(0); + edit.SetFullHistoryTsLow(ts); + TestEncodeDecode(edit); +} + +// Tests that if RocksDB is downgraded, the new types of VersionEdits +// that have a tag larger than kTagSafeIgnoreMask can be safely ignored. +TEST_F(VersionEditTest, IgnorableTags) { + SyncPoint::GetInstance()->SetCallBack( + "VersionEdit::EncodeTo:IgnoreIgnorableTags", [&](void* arg) { + bool* ignore = static_cast(arg); + *ignore = true; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + constexpr uint64_t kPrevLogNumber = 100; + constexpr uint64_t kLogNumber = 200; + constexpr uint64_t kNextFileNumber = 300; + constexpr uint64_t kColumnFamilyId = 400; + + VersionEdit edit; + // Add some ignorable entries. + for (int i = 0; i < 2; i++) { + edit.AddWal(i + 1, WalMetadata(i + 2)); + } + edit.SetDBId("db_id"); + // Add unignorable entries. + edit.SetPrevLogNumber(kPrevLogNumber); + edit.SetLogNumber(kLogNumber); + // Add more ignorable entries. + edit.DeleteWalsBefore(100); + // Add unignorable entry. + edit.SetNextFile(kNextFileNumber); + // Add more ignorable entries. + edit.SetFullHistoryTsLow("ts"); + // Add unignorable entry. + edit.SetColumnFamily(kColumnFamilyId); + + std::string encoded; + ASSERT_TRUE(edit.EncodeTo(&encoded, 0 /* ts_sz */)); + + VersionEdit decoded; + ASSERT_OK(decoded.DecodeFrom(encoded)); + + // Check that all ignorable entries are ignored. + ASSERT_FALSE(decoded.HasDbId()); + ASSERT_FALSE(decoded.HasFullHistoryTsLow()); + ASSERT_FALSE(decoded.IsWalAddition()); + ASSERT_FALSE(decoded.IsWalDeletion()); + ASSERT_TRUE(decoded.GetWalAdditions().empty()); + ASSERT_TRUE(decoded.GetWalDeletion().IsEmpty()); + + // Check that unignorable entries are still present. + ASSERT_EQ(edit.GetPrevLogNumber(), kPrevLogNumber); + ASSERT_EQ(edit.GetLogNumber(), kLogNumber); + ASSERT_EQ(edit.GetNextFile(), kNextFileNumber); + ASSERT_EQ(edit.GetColumnFamily(), kColumnFamilyId); + + SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST(FileMetaDataTest, UpdateBoundariesBlobIndex) { + FileMetaData meta; + + { + constexpr uint64_t file_number = 10; + constexpr uint32_t path_id = 0; + constexpr uint64_t file_size = 0; + + meta.fd = FileDescriptor(file_number, path_id, file_size); + } + + constexpr char key[] = "foo"; + + constexpr uint64_t expected_oldest_blob_file_number = 20; + + // Plain old value (does not affect oldest_blob_file_number) + { + constexpr char value[] = "value"; + constexpr SequenceNumber seq = 200; + + ASSERT_OK(meta.UpdateBoundaries(key, value, seq, kTypeValue)); + ASSERT_EQ(meta.oldest_blob_file_number, kInvalidBlobFileNumber); + } + + // Non-inlined, non-TTL blob index (sets oldest_blob_file_number) + { + constexpr uint64_t blob_file_number = 25; + static_assert(blob_file_number > expected_oldest_blob_file_number, + "unexpected"); + + constexpr uint64_t offset = 1000; + constexpr uint64_t size = 100; + + std::string blob_index; + BlobIndex::EncodeBlob(&blob_index, blob_file_number, offset, size, + kNoCompression); + + constexpr SequenceNumber seq = 201; + + ASSERT_OK(meta.UpdateBoundaries(key, blob_index, seq, kTypeBlobIndex)); + ASSERT_EQ(meta.oldest_blob_file_number, blob_file_number); + } + + // Another one, with the oldest blob file number (updates + // oldest_blob_file_number) + { + constexpr uint64_t offset = 2000; + constexpr uint64_t size = 300; + + std::string blob_index; + BlobIndex::EncodeBlob(&blob_index, expected_oldest_blob_file_number, offset, + size, kNoCompression); + + constexpr SequenceNumber seq = 202; + + ASSERT_OK(meta.UpdateBoundaries(key, blob_index, seq, kTypeBlobIndex)); + ASSERT_EQ(meta.oldest_blob_file_number, expected_oldest_blob_file_number); + } + + // Inlined TTL blob index (does not affect oldest_blob_file_number) + { + constexpr uint64_t expiration = 9876543210; + constexpr char value[] = "value"; + + std::string blob_index; + BlobIndex::EncodeInlinedTTL(&blob_index, expiration, value); + + constexpr SequenceNumber seq = 203; + + ASSERT_OK(meta.UpdateBoundaries(key, blob_index, seq, kTypeBlobIndex)); + ASSERT_EQ(meta.oldest_blob_file_number, expected_oldest_blob_file_number); + } + + // Non-inlined TTL blob index (does not affect oldest_blob_file_number, even + // though file number is smaller) + { + constexpr uint64_t expiration = 9876543210; + constexpr uint64_t blob_file_number = 15; + static_assert(blob_file_number < expected_oldest_blob_file_number, + "unexpected"); + + constexpr uint64_t offset = 2000; + constexpr uint64_t size = 500; + + std::string blob_index; + BlobIndex::EncodeBlobTTL(&blob_index, expiration, blob_file_number, offset, + size, kNoCompression); + + constexpr SequenceNumber seq = 204; + + ASSERT_OK(meta.UpdateBoundaries(key, blob_index, seq, kTypeBlobIndex)); + ASSERT_EQ(meta.oldest_blob_file_number, expected_oldest_blob_file_number); + } + + // Corrupt blob index + { + constexpr char corrupt_blob_index[] = "!corrupt!"; + constexpr SequenceNumber seq = 205; + + ASSERT_TRUE( + meta.UpdateBoundaries(key, corrupt_blob_index, seq, kTypeBlobIndex) + .IsCorruption()); + ASSERT_EQ(meta.oldest_blob_file_number, expected_oldest_blob_file_number); + } + + // Invalid blob file number + { + constexpr uint64_t offset = 10000; + constexpr uint64_t size = 1000; + + std::string blob_index; + BlobIndex::EncodeBlob(&blob_index, kInvalidBlobFileNumber, offset, size, + kNoCompression); + + constexpr SequenceNumber seq = 206; + + ASSERT_TRUE(meta.UpdateBoundaries(key, blob_index, seq, kTypeBlobIndex) + .IsCorruption()); + ASSERT_EQ(meta.oldest_blob_file_number, expected_oldest_blob_file_number); + } +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/version_set.cc b/librocksdb-sys/rocksdb/db/version_set.cc new file mode 100644 index 0000000..32dd4b8 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/version_set.cc @@ -0,0 +1,7334 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/version_set.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "db/blob/blob_fetcher.h" +#include "db/blob/blob_file_cache.h" +#include "db/blob/blob_file_reader.h" +#include "db/blob/blob_log_format.h" +#include "db/blob/blob_source.h" +#include "db/compaction/compaction.h" +#include "db/compaction/file_pri.h" +#include "db/dbformat.h" +#include "db/internal_stats.h" +#include "db/log_reader.h" +#include "db/log_writer.h" +#include "db/memtable.h" +#include "db/merge_context.h" +#include "db/merge_helper.h" +#include "db/pinned_iterators_manager.h" +#include "db/table_cache.h" +#include "db/version_builder.h" +#include "db/version_edit.h" +#include "db/version_edit_handler.h" +#include "file/file_util.h" +#include "table/compaction_merging_iterator.h" + +#if USE_COROUTINES +#include "folly/experimental/coro/BlockingWait.h" +#include "folly/experimental/coro/Collect.h" +#endif +#include "file/filename.h" +#include "file/random_access_file_reader.h" +#include "file/read_write_util.h" +#include "file/writable_file_writer.h" +#include "logging/logging.h" +#include "monitoring/file_read_sample.h" +#include "monitoring/perf_context_imp.h" +#include "monitoring/persistent_stats_history.h" +#include "options/options_helper.h" +#include "rocksdb/env.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/write_buffer_manager.h" +#include "table/format.h" +#include "table/get_context.h" +#include "table/internal_iterator.h" +#include "table/merging_iterator.h" +#include "table/meta_blocks.h" +#include "table/multiget_context.h" +#include "table/plain/plain_table_factory.h" +#include "table/table_reader.h" +#include "table/two_level_iterator.h" +#include "table/unique_id_impl.h" +#include "test_util/sync_point.h" +#include "util/cast_util.h" +#include "util/coding.h" +#include "util/coro_utils.h" +#include "util/stop_watch.h" +#include "util/string_util.h" +#include "util/user_comparator_wrapper.h" + +// Generate the regular and coroutine versions of some methods by +// including version_set_sync_and_async.h twice +// Macros in the header will expand differently based on whether +// WITH_COROUTINES or WITHOUT_COROUTINES is defined +// clang-format off +#define WITHOUT_COROUTINES +#include "db/version_set_sync_and_async.h" +#undef WITHOUT_COROUTINES +#define WITH_COROUTINES +#include "db/version_set_sync_and_async.h" +#undef WITH_COROUTINES +// clang-format on + +namespace ROCKSDB_NAMESPACE { + +namespace { + +// Find File in LevelFilesBrief data structure +// Within an index range defined by left and right +int FindFileInRange(const InternalKeyComparator& icmp, + const LevelFilesBrief& file_level, const Slice& key, + uint32_t left, uint32_t right) { + auto cmp = [&](const FdWithKeyRange& f, const Slice& k) -> bool { + return icmp.InternalKeyComparator::Compare(f.largest_key, k) < 0; + }; + const auto& b = file_level.files; + return static_cast(std::lower_bound(b + left, b + right, key, cmp) - b); +} + +Status OverlapWithIterator(const Comparator* ucmp, + const Slice& smallest_user_key, + const Slice& largest_user_key, + InternalIterator* iter, bool* overlap) { + InternalKey range_start(smallest_user_key, kMaxSequenceNumber, + kValueTypeForSeek); + iter->Seek(range_start.Encode()); + if (!iter->status().ok()) { + return iter->status(); + } + + *overlap = false; + if (iter->Valid()) { + ParsedInternalKey seek_result; + Status s = ParseInternalKey(iter->key(), &seek_result, + false /* log_err_key */); // TODO + if (!s.ok()) return s; + + if (ucmp->CompareWithoutTimestamp(seek_result.user_key, largest_user_key) <= + 0) { + *overlap = true; + } + } + + return iter->status(); +} + +// Class to help choose the next file to search for the particular key. +// Searches and returns files level by level. +// We can search level-by-level since entries never hop across +// levels. Therefore we are guaranteed that if we find data +// in a smaller level, later levels are irrelevant (unless we +// are MergeInProgress). +class FilePicker { + public: + FilePicker(const Slice& user_key, const Slice& ikey, + autovector* file_levels, unsigned int num_levels, + FileIndexer* file_indexer, const Comparator* user_comparator, + const InternalKeyComparator* internal_comparator) + : num_levels_(num_levels), + curr_level_(static_cast(-1)), + returned_file_level_(static_cast(-1)), + hit_file_level_(static_cast(-1)), + search_left_bound_(0), + search_right_bound_(FileIndexer::kLevelMaxIndex), + level_files_brief_(file_levels), + is_hit_file_last_in_level_(false), + curr_file_level_(nullptr), + user_key_(user_key), + ikey_(ikey), + file_indexer_(file_indexer), + user_comparator_(user_comparator), + internal_comparator_(internal_comparator) { + // Setup member variables to search first level. + search_ended_ = !PrepareNextLevel(); + if (!search_ended_) { + // Prefetch Level 0 table data to avoid cache miss if possible. + for (unsigned int i = 0; i < (*level_files_brief_)[0].num_files; ++i) { + auto* r = (*level_files_brief_)[0].files[i].fd.table_reader; + if (r) { + r->Prepare(ikey); + } + } + } + } + + int GetCurrentLevel() const { return curr_level_; } + + FdWithKeyRange* GetNextFile() { + while (!search_ended_) { // Loops over different levels. + while (curr_index_in_curr_level_ < curr_file_level_->num_files) { + // Loops over all files in current level. + FdWithKeyRange* f = &curr_file_level_->files[curr_index_in_curr_level_]; + hit_file_level_ = curr_level_; + is_hit_file_last_in_level_ = + curr_index_in_curr_level_ == curr_file_level_->num_files - 1; + int cmp_largest = -1; + + // Do key range filtering of files or/and fractional cascading if: + // (1) not all the files are in level 0, or + // (2) there are more than 3 current level files + // If there are only 3 or less current level files in the system, we + // skip the key range filtering. In this case, more likely, the system + // is highly tuned to minimize number of tables queried by each query, + // so it is unlikely that key range filtering is more efficient than + // querying the files. + if (num_levels_ > 1 || curr_file_level_->num_files > 3) { + // Check if key is within a file's range. If search left bound and + // right bound point to the same find, we are sure key falls in + // range. + assert(curr_level_ == 0 || + curr_index_in_curr_level_ == start_index_in_curr_level_ || + user_comparator_->CompareWithoutTimestamp( + user_key_, ExtractUserKey(f->smallest_key)) <= 0); + + int cmp_smallest = user_comparator_->CompareWithoutTimestamp( + user_key_, ExtractUserKey(f->smallest_key)); + if (cmp_smallest >= 0) { + cmp_largest = user_comparator_->CompareWithoutTimestamp( + user_key_, ExtractUserKey(f->largest_key)); + } + + // Setup file search bound for the next level based on the + // comparison results + if (curr_level_ > 0) { + file_indexer_->GetNextLevelIndex( + curr_level_, curr_index_in_curr_level_, cmp_smallest, + cmp_largest, &search_left_bound_, &search_right_bound_); + } + // Key falls out of current file's range + if (cmp_smallest < 0 || cmp_largest > 0) { + if (curr_level_ == 0) { + ++curr_index_in_curr_level_; + continue; + } else { + // Search next level. + break; + } + } + } + + returned_file_level_ = curr_level_; + if (curr_level_ > 0 && cmp_largest < 0) { + // No more files to search in this level. + search_ended_ = !PrepareNextLevel(); + } else { + ++curr_index_in_curr_level_; + } + return f; + } + // Start searching next level. + search_ended_ = !PrepareNextLevel(); + } + // Search ended. + return nullptr; + } + + // getter for current file level + // for GET_HIT_L0, GET_HIT_L1 & GET_HIT_L2_AND_UP counts + unsigned int GetHitFileLevel() { return hit_file_level_; } + + // Returns true if the most recent "hit file" (i.e., one returned by + // GetNextFile()) is at the last index in its level. + bool IsHitFileLastInLevel() { return is_hit_file_last_in_level_; } + + private: + unsigned int num_levels_; + unsigned int curr_level_; + unsigned int returned_file_level_; + unsigned int hit_file_level_; + int32_t search_left_bound_; + int32_t search_right_bound_; + autovector* level_files_brief_; + bool search_ended_; + bool is_hit_file_last_in_level_; + LevelFilesBrief* curr_file_level_; + unsigned int curr_index_in_curr_level_; + unsigned int start_index_in_curr_level_; + Slice user_key_; + Slice ikey_; + FileIndexer* file_indexer_; + const Comparator* user_comparator_; + const InternalKeyComparator* internal_comparator_; + + // Setup local variables to search next level. + // Returns false if there are no more levels to search. + bool PrepareNextLevel() { + curr_level_++; + while (curr_level_ < num_levels_) { + curr_file_level_ = &(*level_files_brief_)[curr_level_]; + if (curr_file_level_->num_files == 0) { + // When current level is empty, the search bound generated from upper + // level must be [0, -1] or [0, FileIndexer::kLevelMaxIndex] if it is + // also empty. + assert(search_left_bound_ == 0); + assert(search_right_bound_ == -1 || + search_right_bound_ == FileIndexer::kLevelMaxIndex); + // Since current level is empty, it will need to search all files in + // the next level + search_left_bound_ = 0; + search_right_bound_ = FileIndexer::kLevelMaxIndex; + curr_level_++; + continue; + } + + // Some files may overlap each other. We find + // all files that overlap user_key and process them in order from + // newest to oldest. In the context of merge-operator, this can occur at + // any level. Otherwise, it only occurs at Level-0 (since Put/Deletes + // are always compacted into a single entry). + int32_t start_index; + if (curr_level_ == 0) { + // On Level-0, we read through all files to check for overlap. + start_index = 0; + } else { + // On Level-n (n>=1), files are sorted. Binary search to find the + // earliest file whose largest key >= ikey. Search left bound and + // right bound are used to narrow the range. + if (search_left_bound_ <= search_right_bound_) { + if (search_right_bound_ == FileIndexer::kLevelMaxIndex) { + search_right_bound_ = + static_cast(curr_file_level_->num_files) - 1; + } + // `search_right_bound_` is an inclusive upper-bound, but since it was + // determined based on user key, it is still possible the lookup key + // falls to the right of `search_right_bound_`'s corresponding file. + // So, pass a limit one higher, which allows us to detect this case. + start_index = + FindFileInRange(*internal_comparator_, *curr_file_level_, ikey_, + static_cast(search_left_bound_), + static_cast(search_right_bound_) + 1); + if (start_index == search_right_bound_ + 1) { + // `ikey_` comes after `search_right_bound_`. The lookup key does + // not exist on this level, so let's skip this level and do a full + // binary search on the next level. + search_left_bound_ = 0; + search_right_bound_ = FileIndexer::kLevelMaxIndex; + curr_level_++; + continue; + } + } else { + // search_left_bound > search_right_bound, key does not exist in + // this level. Since no comparison is done in this level, it will + // need to search all files in the next level. + search_left_bound_ = 0; + search_right_bound_ = FileIndexer::kLevelMaxIndex; + curr_level_++; + continue; + } + } + start_index_in_curr_level_ = start_index; + curr_index_in_curr_level_ = start_index; + + return true; + } + // curr_level_ = num_levels_. So, no more levels to search. + return false; + } +}; +} // anonymous namespace + +class FilePickerMultiGet { + private: + struct FilePickerContext; + + public: + FilePickerMultiGet(MultiGetRange* range, + autovector* file_levels, + unsigned int num_levels, FileIndexer* file_indexer, + const Comparator* user_comparator, + const InternalKeyComparator* internal_comparator) + : num_levels_(num_levels), + curr_level_(static_cast(-1)), + returned_file_level_(static_cast(-1)), + hit_file_level_(static_cast(-1)), + range_(*range, range->begin(), range->end()), + maybe_repeat_key_(false), + current_level_range_(*range, range->begin(), range->end()), + current_file_range_(*range, range->begin(), range->end()), + batch_iter_(range->begin()), + batch_iter_prev_(range->begin()), + upper_key_(range->begin()), + level_files_brief_(file_levels), + is_hit_file_last_in_level_(false), + curr_file_level_(nullptr), + file_indexer_(file_indexer), + user_comparator_(user_comparator), + internal_comparator_(internal_comparator), + hit_file_(nullptr) { + for (auto iter = range_.begin(); iter != range_.end(); ++iter) { + fp_ctx_array_[iter.index()] = + FilePickerContext(0, FileIndexer::kLevelMaxIndex); + } + + // Setup member variables to search first level. + search_ended_ = !PrepareNextLevel(); + if (!search_ended_) { + // REVISIT + // Prefetch Level 0 table data to avoid cache miss if possible. + // As of now, only PlainTableReader and CuckooTableReader do any + // prefetching. This may not be necessary anymore once we implement + // batching in those table readers + for (unsigned int i = 0; i < (*level_files_brief_)[0].num_files; ++i) { + auto* r = (*level_files_brief_)[0].files[i].fd.table_reader; + if (r) { + for (auto iter = range_.begin(); iter != range_.end(); ++iter) { + r->Prepare(iter->ikey); + } + } + } + } + } + + FilePickerMultiGet(MultiGetRange* range, const FilePickerMultiGet& other) + : num_levels_(other.num_levels_), + curr_level_(other.curr_level_), + returned_file_level_(other.returned_file_level_), + hit_file_level_(other.hit_file_level_), + fp_ctx_array_(other.fp_ctx_array_), + range_(*range, range->begin(), range->end()), + maybe_repeat_key_(false), + current_level_range_(*range, range->begin(), range->end()), + current_file_range_(*range, range->begin(), range->end()), + batch_iter_(range->begin()), + batch_iter_prev_(range->begin()), + upper_key_(range->begin()), + level_files_brief_(other.level_files_brief_), + is_hit_file_last_in_level_(false), + curr_file_level_(other.curr_file_level_), + file_indexer_(other.file_indexer_), + user_comparator_(other.user_comparator_), + internal_comparator_(other.internal_comparator_), + hit_file_(nullptr) { + PrepareNextLevelForSearch(); + } + + int GetCurrentLevel() const { return curr_level_; } + + void PrepareNextLevelForSearch() { search_ended_ = !PrepareNextLevel(); } + + FdWithKeyRange* GetNextFileInLevel() { + if (batch_iter_ == current_level_range_.end() || search_ended_) { + hit_file_ = nullptr; + return nullptr; + } else { + if (maybe_repeat_key_) { + maybe_repeat_key_ = false; + // Check if we found the final value for the last key in the + // previous lookup range. If we did, then there's no need to look + // any further for that key, so advance batch_iter_. Else, keep + // batch_iter_ positioned on that key so we look it up again in + // the next file + // For L0, always advance the key because we will look in the next + // file regardless for all keys not found yet + if (current_level_range_.CheckKeyDone(batch_iter_) || + curr_level_ == 0) { + batch_iter_ = upper_key_; + } + } + // batch_iter_prev_ will become the start key for the next file + // lookup + batch_iter_prev_ = batch_iter_; + } + + MultiGetRange next_file_range(current_level_range_, batch_iter_prev_, + current_level_range_.end()); + size_t curr_file_index = + (batch_iter_ != current_level_range_.end()) + ? fp_ctx_array_[batch_iter_.index()].curr_index_in_curr_level + : curr_file_level_->num_files; + FdWithKeyRange* f; + bool is_last_key_in_file; + if (!GetNextFileInLevelWithKeys(&next_file_range, &curr_file_index, &f, + &is_last_key_in_file)) { + hit_file_ = nullptr; + return nullptr; + } else { + if (is_last_key_in_file) { + // Since cmp_largest is 0, batch_iter_ still points to the last key + // that falls in this file, instead of the next one. Increment + // the file index for all keys between batch_iter_ and upper_key_ + auto tmp_iter = batch_iter_; + while (tmp_iter != upper_key_) { + ++(fp_ctx_array_[tmp_iter.index()].curr_index_in_curr_level); + ++tmp_iter; + } + maybe_repeat_key_ = true; + } + // Set the range for this file + current_file_range_ = + MultiGetRange(next_file_range, batch_iter_prev_, upper_key_); + returned_file_level_ = curr_level_; + hit_file_level_ = curr_level_; + is_hit_file_last_in_level_ = + curr_file_index == curr_file_level_->num_files - 1; + hit_file_ = f; + return f; + } + } + + // getter for current file level + // for GET_HIT_L0, GET_HIT_L1 & GET_HIT_L2_AND_UP counts + unsigned int GetHitFileLevel() { return hit_file_level_; } + + FdWithKeyRange* GetHitFile() { return hit_file_; } + + // Returns true if the most recent "hit file" (i.e., one returned by + // GetNextFile()) is at the last index in its level. + bool IsHitFileLastInLevel() { return is_hit_file_last_in_level_; } + + bool KeyMaySpanNextFile() { return maybe_repeat_key_; } + + bool IsSearchEnded() { return search_ended_; } + + const MultiGetRange& CurrentFileRange() { return current_file_range_; } + + bool RemainingOverlapInLevel() { + return !current_level_range_.Suffix(current_file_range_).empty(); + } + + MultiGetRange& GetRange() { return range_; } + + void ReplaceRange(const MultiGetRange& other) { + assert(hit_file_ == nullptr); + range_ = other; + current_level_range_ = other; + } + + FilePickerMultiGet(FilePickerMultiGet&& other) + : num_levels_(other.num_levels_), + curr_level_(other.curr_level_), + returned_file_level_(other.returned_file_level_), + hit_file_level_(other.hit_file_level_), + fp_ctx_array_(std::move(other.fp_ctx_array_)), + range_(std::move(other.range_)), + maybe_repeat_key_(other.maybe_repeat_key_), + current_level_range_(std::move(other.current_level_range_)), + current_file_range_(std::move(other.current_file_range_)), + batch_iter_(other.batch_iter_, ¤t_level_range_), + batch_iter_prev_(other.batch_iter_prev_, ¤t_level_range_), + upper_key_(other.upper_key_, ¤t_level_range_), + level_files_brief_(other.level_files_brief_), + search_ended_(other.search_ended_), + is_hit_file_last_in_level_(other.is_hit_file_last_in_level_), + curr_file_level_(other.curr_file_level_), + file_indexer_(other.file_indexer_), + user_comparator_(other.user_comparator_), + internal_comparator_(other.internal_comparator_), + hit_file_(other.hit_file_) {} + + private: + unsigned int num_levels_; + unsigned int curr_level_; + unsigned int returned_file_level_; + unsigned int hit_file_level_; + + struct FilePickerContext { + int32_t search_left_bound; + int32_t search_right_bound; + unsigned int curr_index_in_curr_level; + unsigned int start_index_in_curr_level; + + FilePickerContext(int32_t left, int32_t right) + : search_left_bound(left), + search_right_bound(right), + curr_index_in_curr_level(0), + start_index_in_curr_level(0) {} + + FilePickerContext() = default; + }; + std::array fp_ctx_array_; + MultiGetRange range_; + bool maybe_repeat_key_; + MultiGetRange current_level_range_; + MultiGetRange current_file_range_; + // Iterator to iterate through the keys in a MultiGet batch, that gets reset + // at the beginning of each level. Each call to GetNextFile() will position + // batch_iter_ at or right after the last key that was found in the returned + // SST file + MultiGetRange::Iterator batch_iter_; + // An iterator that records the previous position of batch_iter_, i.e last + // key found in the previous SST file, in order to serve as the start of + // the batch key range for the next SST file + MultiGetRange::Iterator batch_iter_prev_; + MultiGetRange::Iterator upper_key_; + autovector* level_files_brief_; + bool search_ended_; + bool is_hit_file_last_in_level_; + LevelFilesBrief* curr_file_level_; + FileIndexer* file_indexer_; + const Comparator* user_comparator_; + const InternalKeyComparator* internal_comparator_; + FdWithKeyRange* hit_file_; + + // Iterates through files in the current level until it finds a file that + // contains at least one key from the MultiGet batch + bool GetNextFileInLevelWithKeys(MultiGetRange* next_file_range, + size_t* file_index, FdWithKeyRange** fd, + bool* is_last_key_in_file) { + size_t curr_file_index = *file_index; + FdWithKeyRange* f = nullptr; + bool file_hit = false; + int cmp_largest = -1; + if (curr_file_index >= curr_file_level_->num_files) { + // In the unlikely case the next key is a duplicate of the current key, + // and the current key is the last in the level and the internal key + // was not found, we need to skip lookup for the remaining keys and + // reset the search bounds + if (batch_iter_ != current_level_range_.end()) { + ++batch_iter_; + for (; batch_iter_ != current_level_range_.end(); ++batch_iter_) { + struct FilePickerContext& fp_ctx = fp_ctx_array_[batch_iter_.index()]; + fp_ctx.search_left_bound = 0; + fp_ctx.search_right_bound = FileIndexer::kLevelMaxIndex; + } + } + return false; + } + // Loops over keys in the MultiGet batch until it finds a file with + // atleast one of the keys. Then it keeps moving forward until the + // last key in the batch that falls in that file + while (batch_iter_ != current_level_range_.end() && + (fp_ctx_array_[batch_iter_.index()].curr_index_in_curr_level == + curr_file_index || + !file_hit)) { + struct FilePickerContext& fp_ctx = fp_ctx_array_[batch_iter_.index()]; + f = &curr_file_level_->files[fp_ctx.curr_index_in_curr_level]; + Slice& user_key = batch_iter_->ukey_without_ts; + + // Do key range filtering of files or/and fractional cascading if: + // (1) not all the files are in level 0, or + // (2) there are more than 3 current level files + // If there are only 3 or less current level files in the system, we + // skip the key range filtering. In this case, more likely, the system + // is highly tuned to minimize number of tables queried by each query, + // so it is unlikely that key range filtering is more efficient than + // querying the files. + if (num_levels_ > 1 || curr_file_level_->num_files > 3) { + // Check if key is within a file's range. If search left bound and + // right bound point to the same find, we are sure key falls in + // range. + int cmp_smallest = user_comparator_->CompareWithoutTimestamp( + user_key, false, ExtractUserKey(f->smallest_key), true); + + assert(curr_level_ == 0 || + fp_ctx.curr_index_in_curr_level == + fp_ctx.start_index_in_curr_level || + cmp_smallest <= 0); + + if (cmp_smallest >= 0) { + cmp_largest = user_comparator_->CompareWithoutTimestamp( + user_key, false, ExtractUserKey(f->largest_key), true); + } else { + cmp_largest = -1; + } + + // Setup file search bound for the next level based on the + // comparison results + if (curr_level_ > 0) { + file_indexer_->GetNextLevelIndex( + curr_level_, fp_ctx.curr_index_in_curr_level, cmp_smallest, + cmp_largest, &fp_ctx.search_left_bound, + &fp_ctx.search_right_bound); + } + // Key falls out of current file's range + if (cmp_smallest < 0 || cmp_largest > 0) { + next_file_range->SkipKey(batch_iter_); + } else { + file_hit = true; + } + } else { + file_hit = true; + } + if (cmp_largest == 0) { + // cmp_largest is 0, which means the next key will not be in this + // file, so stop looking further. However, its possible there are + // duplicates in the batch, so find the upper bound for the batch + // in this file (upper_key_) by skipping past the duplicates. We + // leave batch_iter_ as is since we may have to pick up from there + // for the next file, if this file has a merge value rather than + // final value + upper_key_ = batch_iter_; + ++upper_key_; + while (upper_key_ != current_level_range_.end() && + user_comparator_->CompareWithoutTimestamp( + batch_iter_->ukey_without_ts, false, + upper_key_->ukey_without_ts, false) == 0) { + ++upper_key_; + } + break; + } else { + if (curr_level_ == 0) { + // We need to look through all files in level 0 + ++fp_ctx.curr_index_in_curr_level; + } + ++batch_iter_; + } + if (!file_hit) { + curr_file_index = + (batch_iter_ != current_level_range_.end()) + ? fp_ctx_array_[batch_iter_.index()].curr_index_in_curr_level + : curr_file_level_->num_files; + } + } + + *fd = f; + *file_index = curr_file_index; + *is_last_key_in_file = cmp_largest == 0; + if (!*is_last_key_in_file) { + // If the largest key in the batch overlapping the file is not the + // largest key in the file, upper_ley_ would not have been updated so + // update it here + upper_key_ = batch_iter_; + } + return file_hit; + } + + // Setup local variables to search next level. + // Returns false if there are no more levels to search. + bool PrepareNextLevel() { + if (curr_level_ == 0) { + MultiGetRange::Iterator mget_iter = current_level_range_.begin(); + if (fp_ctx_array_[mget_iter.index()].curr_index_in_curr_level < + curr_file_level_->num_files) { + batch_iter_prev_ = current_level_range_.begin(); + upper_key_ = batch_iter_ = current_level_range_.begin(); + return true; + } + } + + curr_level_++; + // Reset key range to saved value + while (curr_level_ < num_levels_) { + bool level_contains_keys = false; + curr_file_level_ = &(*level_files_brief_)[curr_level_]; + if (curr_file_level_->num_files == 0) { + // When current level is empty, the search bound generated from upper + // level must be [0, -1] or [0, FileIndexer::kLevelMaxIndex] if it is + // also empty. + + for (auto mget_iter = current_level_range_.begin(); + mget_iter != current_level_range_.end(); ++mget_iter) { + struct FilePickerContext& fp_ctx = fp_ctx_array_[mget_iter.index()]; + + assert(fp_ctx.search_left_bound == 0); + assert(fp_ctx.search_right_bound == -1 || + fp_ctx.search_right_bound == FileIndexer::kLevelMaxIndex); + // Since current level is empty, it will need to search all files in + // the next level + fp_ctx.search_left_bound = 0; + fp_ctx.search_right_bound = FileIndexer::kLevelMaxIndex; + } + // Skip all subsequent empty levels + do { + ++curr_level_; + } while ((curr_level_ < num_levels_) && + (*level_files_brief_)[curr_level_].num_files == 0); + continue; + } + + // Some files may overlap each other. We find + // all files that overlap user_key and process them in order from + // newest to oldest. In the context of merge-operator, this can occur at + // any level. Otherwise, it only occurs at Level-0 (since Put/Deletes + // are always compacted into a single entry). + int32_t start_index = -1; + current_level_range_ = + MultiGetRange(range_, range_.begin(), range_.end()); + for (auto mget_iter = current_level_range_.begin(); + mget_iter != current_level_range_.end(); ++mget_iter) { + struct FilePickerContext& fp_ctx = fp_ctx_array_[mget_iter.index()]; + if (curr_level_ == 0) { + // On Level-0, we read through all files to check for overlap. + start_index = 0; + level_contains_keys = true; + } else { + // On Level-n (n>=1), files are sorted. Binary search to find the + // earliest file whose largest key >= ikey. Search left bound and + // right bound are used to narrow the range. + if (fp_ctx.search_left_bound <= fp_ctx.search_right_bound) { + if (fp_ctx.search_right_bound == FileIndexer::kLevelMaxIndex) { + fp_ctx.search_right_bound = + static_cast(curr_file_level_->num_files) - 1; + } + // `search_right_bound_` is an inclusive upper-bound, but since it + // was determined based on user key, it is still possible the lookup + // key falls to the right of `search_right_bound_`'s corresponding + // file. So, pass a limit one higher, which allows us to detect this + // case. + Slice& ikey = mget_iter->ikey; + start_index = FindFileInRange( + *internal_comparator_, *curr_file_level_, ikey, + static_cast(fp_ctx.search_left_bound), + static_cast(fp_ctx.search_right_bound) + 1); + if (start_index == fp_ctx.search_right_bound + 1) { + // `ikey_` comes after `search_right_bound_`. The lookup key does + // not exist on this level, so let's skip this level and do a full + // binary search on the next level. + fp_ctx.search_left_bound = 0; + fp_ctx.search_right_bound = FileIndexer::kLevelMaxIndex; + current_level_range_.SkipKey(mget_iter); + continue; + } else { + level_contains_keys = true; + } + } else { + // search_left_bound > search_right_bound, key does not exist in + // this level. Since no comparison is done in this level, it will + // need to search all files in the next level. + fp_ctx.search_left_bound = 0; + fp_ctx.search_right_bound = FileIndexer::kLevelMaxIndex; + current_level_range_.SkipKey(mget_iter); + continue; + } + } + fp_ctx.start_index_in_curr_level = start_index; + fp_ctx.curr_index_in_curr_level = start_index; + } + if (level_contains_keys) { + batch_iter_prev_ = current_level_range_.begin(); + upper_key_ = batch_iter_ = current_level_range_.begin(); + return true; + } + curr_level_++; + } + // curr_level_ = num_levels_. So, no more levels to search. + return false; + } +}; + +VersionStorageInfo::~VersionStorageInfo() { delete[] files_; } + +Version::~Version() { + assert(refs_ == 0); + + // Remove from linked list + prev_->next_ = next_; + next_->prev_ = prev_; + + // Drop references to files + for (int level = 0; level < storage_info_.num_levels_; level++) { + for (size_t i = 0; i < storage_info_.files_[level].size(); i++) { + FileMetaData* f = storage_info_.files_[level][i]; + assert(f->refs > 0); + f->refs--; + if (f->refs <= 0) { + assert(cfd_ != nullptr); + uint32_t path_id = f->fd.GetPathId(); + assert(path_id < cfd_->ioptions()->cf_paths.size()); + vset_->obsolete_files_.push_back( + ObsoleteFileInfo(f, cfd_->ioptions()->cf_paths[path_id].path, + cfd_->GetFileMetadataCacheReservationManager())); + } + } + } +} + +int FindFile(const InternalKeyComparator& icmp, + const LevelFilesBrief& file_level, const Slice& key) { + return FindFileInRange(icmp, file_level, key, 0, + static_cast(file_level.num_files)); +} + +void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level, + const std::vector& files, + Arena* arena) { + assert(file_level); + assert(arena); + + size_t num = files.size(); + file_level->num_files = num; + char* mem = arena->AllocateAligned(num * sizeof(FdWithKeyRange)); + file_level->files = new (mem) FdWithKeyRange[num]; + + for (size_t i = 0; i < num; i++) { + Slice smallest_key = files[i]->smallest.Encode(); + Slice largest_key = files[i]->largest.Encode(); + + // Copy key slice to sequential memory + size_t smallest_size = smallest_key.size(); + size_t largest_size = largest_key.size(); + mem = arena->AllocateAligned(smallest_size + largest_size); + memcpy(mem, smallest_key.data(), smallest_size); + memcpy(mem + smallest_size, largest_key.data(), largest_size); + + FdWithKeyRange& f = file_level->files[i]; + f.fd = files[i]->fd; + f.file_metadata = files[i]; + f.smallest_key = Slice(mem, smallest_size); + f.largest_key = Slice(mem + smallest_size, largest_size); + } +} + +static bool AfterFile(const Comparator* ucmp, const Slice* user_key, + const FdWithKeyRange* f) { + // nullptr user_key occurs before all keys and is therefore never after *f + return (user_key != nullptr && + ucmp->CompareWithoutTimestamp(*user_key, + ExtractUserKey(f->largest_key)) > 0); +} + +static bool BeforeFile(const Comparator* ucmp, const Slice* user_key, + const FdWithKeyRange* f) { + // nullptr user_key occurs after all keys and is therefore never before *f + return (user_key != nullptr && + ucmp->CompareWithoutTimestamp(*user_key, + ExtractUserKey(f->smallest_key)) < 0); +} + +bool SomeFileOverlapsRange(const InternalKeyComparator& icmp, + bool disjoint_sorted_files, + const LevelFilesBrief& file_level, + const Slice* smallest_user_key, + const Slice* largest_user_key) { + const Comparator* ucmp = icmp.user_comparator(); + if (!disjoint_sorted_files) { + // Need to check against all files + for (size_t i = 0; i < file_level.num_files; i++) { + const FdWithKeyRange* f = &(file_level.files[i]); + if (AfterFile(ucmp, smallest_user_key, f) || + BeforeFile(ucmp, largest_user_key, f)) { + // No overlap + } else { + return true; // Overlap + } + } + return false; + } + + // Binary search over file list + uint32_t index = 0; + if (smallest_user_key != nullptr) { + // Find the leftmost possible internal key for smallest_user_key + InternalKey small; + small.SetMinPossibleForUserKey(*smallest_user_key); + index = FindFile(icmp, file_level, small.Encode()); + } + + if (index >= file_level.num_files) { + // beginning of range is after all files, so no overlap. + return false; + } + + return !BeforeFile(ucmp, largest_user_key, &file_level.files[index]); +} + +namespace { + +class LevelIterator final : public InternalIterator { + public: + // @param read_options Must outlive this iterator. + LevelIterator( + TableCache* table_cache, const ReadOptions& read_options, + const FileOptions& file_options, const InternalKeyComparator& icomparator, + const LevelFilesBrief* flevel, + const std::shared_ptr& prefix_extractor, + bool should_sample, HistogramImpl* file_read_hist, + TableReaderCaller caller, bool skip_filters, int level, + uint8_t block_protection_bytes_per_key, RangeDelAggregator* range_del_agg, + const std::vector* compaction_boundaries = + nullptr, + bool allow_unprepared_value = false, + TruncatedRangeDelIterator**** range_tombstone_iter_ptr_ = nullptr) + : table_cache_(table_cache), + read_options_(read_options), + file_options_(file_options), + icomparator_(icomparator), + user_comparator_(icomparator.user_comparator()), + flevel_(flevel), + prefix_extractor_(prefix_extractor), + file_read_hist_(file_read_hist), + should_sample_(should_sample), + caller_(caller), + skip_filters_(skip_filters), + allow_unprepared_value_(allow_unprepared_value), + file_index_(flevel_->num_files), + level_(level), + range_del_agg_(range_del_agg), + pinned_iters_mgr_(nullptr), + compaction_boundaries_(compaction_boundaries), + is_next_read_sequential_(false), + block_protection_bytes_per_key_(block_protection_bytes_per_key), + range_tombstone_iter_(nullptr), + to_return_sentinel_(false) { + // Empty level is not supported. + assert(flevel_ != nullptr && flevel_->num_files > 0); + if (range_tombstone_iter_ptr_) { + *range_tombstone_iter_ptr_ = &range_tombstone_iter_; + } + } + + ~LevelIterator() override { delete file_iter_.Set(nullptr); } + + // Seek to the first file with a key >= target. + // If range_tombstone_iter_ is not nullptr, then we pretend that file + // boundaries are fake keys (sentinel keys). These keys are used to keep range + // tombstones alive even when all point keys in an SST file are exhausted. + // These sentinel keys will be skipped in merging iterator. + void Seek(const Slice& target) override; + void SeekForPrev(const Slice& target) override; + void SeekToFirst() override; + void SeekToLast() override; + void Next() final override; + bool NextAndGetResult(IterateResult* result) override; + void Prev() override; + + // In addition to valid and invalid state (!file_iter.Valid() and + // status.ok()), a third state of the iterator is when !file_iter_.Valid() and + // to_return_sentinel_. This means we are at the end of a file, and a sentinel + // key (the file boundary that we pretend as a key) is to be returned next. + // file_iter_.Valid() and to_return_sentinel_ should not both be true. + bool Valid() const override { + assert(!(file_iter_.Valid() && to_return_sentinel_)); + return file_iter_.Valid() || to_return_sentinel_; + } + Slice key() const override { + assert(Valid()); + if (to_return_sentinel_) { + // Sentinel should be returned after file_iter_ reaches the end of the + // file + assert(!file_iter_.Valid()); + return sentinel_; + } + return file_iter_.key(); + } + + Slice value() const override { + assert(Valid()); + assert(!to_return_sentinel_); + return file_iter_.value(); + } + + Status status() const override { + return file_iter_.iter() ? file_iter_.status() : Status::OK(); + } + + bool PrepareValue() override { return file_iter_.PrepareValue(); } + + inline bool MayBeOutOfLowerBound() override { + assert(Valid()); + return may_be_out_of_lower_bound_ && file_iter_.MayBeOutOfLowerBound(); + } + + inline IterBoundCheck UpperBoundCheckResult() override { + if (Valid()) { + return file_iter_.UpperBoundCheckResult(); + } else { + return IterBoundCheck::kUnknown; + } + } + + void SetPinnedItersMgr(PinnedIteratorsManager* pinned_iters_mgr) override { + pinned_iters_mgr_ = pinned_iters_mgr; + if (file_iter_.iter()) { + file_iter_.SetPinnedItersMgr(pinned_iters_mgr); + } + } + + bool IsKeyPinned() const override { + return pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled() && + file_iter_.iter() && file_iter_.IsKeyPinned(); + } + + bool IsValuePinned() const override { + return pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled() && + file_iter_.iter() && file_iter_.IsValuePinned(); + } + + bool IsDeleteRangeSentinelKey() const override { return to_return_sentinel_; } + + private: + // Return true if at least one invalid file is seen and skipped. + bool SkipEmptyFileForward(); + void SkipEmptyFileBackward(); + void SetFileIterator(InternalIterator* iter); + void InitFileIterator(size_t new_file_index); + + const Slice& file_smallest_key(size_t file_index) { + assert(file_index < flevel_->num_files); + return flevel_->files[file_index].smallest_key; + } + + const Slice& file_largest_key(size_t file_index) { + assert(file_index < flevel_->num_files); + return flevel_->files[file_index].largest_key; + } + + bool KeyReachedUpperBound(const Slice& internal_key) { + return read_options_.iterate_upper_bound != nullptr && + user_comparator_.CompareWithoutTimestamp( + ExtractUserKey(internal_key), /*a_has_ts=*/true, + *read_options_.iterate_upper_bound, /*b_has_ts=*/false) >= 0; + } + + void ClearRangeTombstoneIter() { + if (range_tombstone_iter_ && *range_tombstone_iter_) { + delete *range_tombstone_iter_; + *range_tombstone_iter_ = nullptr; + } + } + + // Move file_iter_ to the file at file_index_. + // range_tombstone_iter_ is updated with a range tombstone iterator + // into the new file. Old range tombstone iterator is cleared. + InternalIterator* NewFileIterator() { + assert(file_index_ < flevel_->num_files); + auto file_meta = flevel_->files[file_index_]; + if (should_sample_) { + sample_file_read_inc(file_meta.file_metadata); + } + + const InternalKey* smallest_compaction_key = nullptr; + const InternalKey* largest_compaction_key = nullptr; + if (compaction_boundaries_ != nullptr) { + smallest_compaction_key = (*compaction_boundaries_)[file_index_].smallest; + largest_compaction_key = (*compaction_boundaries_)[file_index_].largest; + } + CheckMayBeOutOfLowerBound(); + ClearRangeTombstoneIter(); + return table_cache_->NewIterator( + read_options_, file_options_, icomparator_, *file_meta.file_metadata, + range_del_agg_, prefix_extractor_, + nullptr /* don't need reference to table */, file_read_hist_, caller_, + /*arena=*/nullptr, skip_filters_, level_, + /*max_file_size_for_l0_meta_pin=*/0, smallest_compaction_key, + largest_compaction_key, allow_unprepared_value_, + block_protection_bytes_per_key_, range_tombstone_iter_); + } + + // Check if current file being fully within iterate_lower_bound. + // + // Note MyRocks may update iterate bounds between seek. To workaround it, + // we need to check and update may_be_out_of_lower_bound_ accordingly. + void CheckMayBeOutOfLowerBound() { + if (read_options_.iterate_lower_bound != nullptr && + file_index_ < flevel_->num_files) { + may_be_out_of_lower_bound_ = + user_comparator_.CompareWithoutTimestamp( + ExtractUserKey(file_smallest_key(file_index_)), /*a_has_ts=*/true, + *read_options_.iterate_lower_bound, /*b_has_ts=*/false) < 0; + } + } + + TableCache* table_cache_; + const ReadOptions& read_options_; + const FileOptions& file_options_; + const InternalKeyComparator& icomparator_; + const UserComparatorWrapper user_comparator_; + const LevelFilesBrief* flevel_; + mutable FileDescriptor current_value_; + // `prefix_extractor_` may be non-null even for total order seek. Checking + // this variable is not the right way to identify whether prefix iterator + // is used. + const std::shared_ptr& prefix_extractor_; + + HistogramImpl* file_read_hist_; + bool should_sample_; + TableReaderCaller caller_; + bool skip_filters_; + bool allow_unprepared_value_; + bool may_be_out_of_lower_bound_ = true; + size_t file_index_; + int level_; + RangeDelAggregator* range_del_agg_; + IteratorWrapper file_iter_; // May be nullptr + PinnedIteratorsManager* pinned_iters_mgr_; + + // To be propagated to RangeDelAggregator in order to safely truncate range + // tombstones. + const std::vector* compaction_boundaries_; + + bool is_next_read_sequential_; + + uint8_t block_protection_bytes_per_key_; + + // This is set when this level iterator is used under a merging iterator + // that processes range tombstones. range_tombstone_iter_ points to where the + // merging iterator stores the range tombstones iterator for this level. When + // this level iterator moves to a new SST file, it updates the range + // tombstones accordingly through this pointer. So the merging iterator always + // has access to the current SST file's range tombstones. + // + // The level iterator treats file boundary as fake keys (sentinel keys) to + // keep range tombstones alive if needed and make upper level, i.e. merging + // iterator, aware of file changes (when level iterator moves to a new SST + // file, there is some bookkeeping work that needs to be done at merging + // iterator end). + // + // *range_tombstone_iter_ points to range tombstones of the current SST file + TruncatedRangeDelIterator** range_tombstone_iter_; + + // Whether next/prev key is a sentinel key. + bool to_return_sentinel_ = false; + // The sentinel key to be returned + Slice sentinel_; + // Sets flags for if we should return the sentinel key next. + // The condition for returning sentinel is reaching the end of current + // file_iter_: !Valid() && status.().ok(). + void TrySetDeleteRangeSentinel(const Slice& boundary_key); + void ClearSentinel() { to_return_sentinel_ = false; } + + // Set in Seek() when a prefix seek reaches end of the current file, + // and the next file has a different prefix. SkipEmptyFileForward() + // will not move to next file when this flag is set. + bool prefix_exhausted_ = false; +}; + +void LevelIterator::TrySetDeleteRangeSentinel(const Slice& boundary_key) { + assert(range_tombstone_iter_); + if (file_iter_.iter() != nullptr && !file_iter_.Valid() && + file_iter_.status().ok()) { + to_return_sentinel_ = true; + sentinel_ = boundary_key; + } +} + +void LevelIterator::Seek(const Slice& target) { + prefix_exhausted_ = false; + ClearSentinel(); + // Check whether the seek key fall under the same file + bool need_to_reseek = true; + if (file_iter_.iter() != nullptr && file_index_ < flevel_->num_files) { + const FdWithKeyRange& cur_file = flevel_->files[file_index_]; + if (icomparator_.InternalKeyComparator::Compare( + target, cur_file.largest_key) <= 0 && + icomparator_.InternalKeyComparator::Compare( + target, cur_file.smallest_key) >= 0) { + need_to_reseek = false; + assert(static_cast(FindFile(icomparator_, *flevel_, target)) == + file_index_); + } + } + if (need_to_reseek) { + TEST_SYNC_POINT("LevelIterator::Seek:BeforeFindFile"); + size_t new_file_index = FindFile(icomparator_, *flevel_, target); + InitFileIterator(new_file_index); + } + + if (file_iter_.iter() != nullptr) { + file_iter_.Seek(target); + // Status::TryAgain indicates asynchronous request for retrieval of data + // blocks has been submitted. So it should return at this point and Seek + // should be called again to retrieve the requested block and execute the + // remaining code. + if (file_iter_.status() == Status::TryAgain()) { + return; + } + if (!file_iter_.Valid() && file_iter_.status().ok() && + prefix_extractor_ != nullptr && !read_options_.total_order_seek && + !read_options_.auto_prefix_mode && + file_index_ < flevel_->num_files - 1) { + size_t ts_sz = user_comparator_.user_comparator()->timestamp_size(); + Slice target_user_key_without_ts = + ExtractUserKeyAndStripTimestamp(target, ts_sz); + Slice next_file_first_user_key_without_ts = + ExtractUserKeyAndStripTimestamp(file_smallest_key(file_index_ + 1), + ts_sz); + if (prefix_extractor_->InDomain(target_user_key_without_ts) && + (!prefix_extractor_->InDomain(next_file_first_user_key_without_ts) || + user_comparator_.CompareWithoutTimestamp( + prefix_extractor_->Transform(target_user_key_without_ts), false, + prefix_extractor_->Transform( + next_file_first_user_key_without_ts), + false) != 0)) { + // SkipEmptyFileForward() will not advance to next file when this flag + // is set for reason detailed below. + // + // The file we initially positioned to has no keys under the target + // prefix, and the next file's smallest key has a different prefix than + // target. When doing prefix iterator seek, when keys for one prefix + // have been exhausted, it can jump to any key that is larger. Here we + // are enforcing a stricter contract than that, in order to make it + // easier for higher layers (merging and DB iterator) to reason the + // correctness: + // 1. Within the prefix, the result should be accurate. + // 2. If keys for the prefix is exhausted, it is either positioned to + // the next key after the prefix, or make the iterator invalid. + // A side benefit will be that it invalidates the iterator earlier so + // that the upper level merging iterator can merge fewer child + // iterators. + // + // The flag is cleared in Seek*() calls. There is no need to clear the + // flag in Prev() since Prev() will not be called when the flag is set + // for reasons explained below. If range_tombstone_iter_ is nullptr, + // then there is no file boundary sentinel key. Since + // !file_iter_.Valid() from the if condition above, this level iterator + // is !Valid(), so Prev() will not be called. If range_tombstone_iter_ + // is not nullptr, there are two cases depending on if this level + // iterator reaches top of the heap in merging iterator (the upper + // layer). + // If so, merging iterator will see the sentinel key, call + // NextAndGetResult() and the call to NextAndGetResult() will skip the + // sentinel key and makes this level iterator invalid. If not, then it + // could be because the upper layer is done before any method of this + // level iterator is called or another Seek*() call is invoked. Either + // way, Prev() is never called before Seek*(). + // The flag should not be cleared at the beginning of + // Next/NextAndGetResult() since it is used in SkipEmptyFileForward() + // called in Next/NextAndGetResult(). + prefix_exhausted_ = true; + } + } + + if (range_tombstone_iter_) { + TrySetDeleteRangeSentinel(file_largest_key(file_index_)); + } + } + SkipEmptyFileForward(); + CheckMayBeOutOfLowerBound(); +} + +void LevelIterator::SeekForPrev(const Slice& target) { + prefix_exhausted_ = false; + ClearSentinel(); + size_t new_file_index = FindFile(icomparator_, *flevel_, target); + // Seek beyond this level's smallest key + if (new_file_index == 0 && + icomparator_.Compare(target, file_smallest_key(0)) < 0) { + SetFileIterator(nullptr); + ClearRangeTombstoneIter(); + CheckMayBeOutOfLowerBound(); + return; + } + if (new_file_index >= flevel_->num_files) { + new_file_index = flevel_->num_files - 1; + } + + InitFileIterator(new_file_index); + if (file_iter_.iter() != nullptr) { + file_iter_.SeekForPrev(target); + if (range_tombstone_iter_ && + icomparator_.Compare(target, file_smallest_key(file_index_)) >= 0) { + // In SeekForPrev() case, it is possible that the target is less than + // file's lower boundary since largest key is used to determine file index + // (FindFile()). When target is less than file's lower boundary, sentinel + // key should not be set so that SeekForPrev() does not result in a key + // larger than target. This is correct in that there is no need to keep + // the range tombstones in this file alive as they only cover keys + // starting from the file's lower boundary, which is after `target`. + TrySetDeleteRangeSentinel(file_smallest_key(file_index_)); + } + SkipEmptyFileBackward(); + } + CheckMayBeOutOfLowerBound(); +} + +void LevelIterator::SeekToFirst() { + prefix_exhausted_ = false; + ClearSentinel(); + InitFileIterator(0); + if (file_iter_.iter() != nullptr) { + file_iter_.SeekToFirst(); + if (range_tombstone_iter_) { + // We do this in SeekToFirst() and SeekToLast() since + // we could have an empty file with only range tombstones. + TrySetDeleteRangeSentinel(file_largest_key(file_index_)); + } + } + SkipEmptyFileForward(); + CheckMayBeOutOfLowerBound(); +} + +void LevelIterator::SeekToLast() { + prefix_exhausted_ = false; + ClearSentinel(); + InitFileIterator(flevel_->num_files - 1); + if (file_iter_.iter() != nullptr) { + file_iter_.SeekToLast(); + if (range_tombstone_iter_) { + TrySetDeleteRangeSentinel(file_smallest_key(file_index_)); + } + } + SkipEmptyFileBackward(); + CheckMayBeOutOfLowerBound(); +} + +void LevelIterator::Next() { + assert(Valid()); + if (to_return_sentinel_) { + // file_iter_ is at EOF already when to_return_sentinel_ + ClearSentinel(); + } else { + file_iter_.Next(); + if (range_tombstone_iter_) { + TrySetDeleteRangeSentinel(file_largest_key(file_index_)); + } + } + SkipEmptyFileForward(); +} + +bool LevelIterator::NextAndGetResult(IterateResult* result) { + assert(Valid()); + // file_iter_ is at EOF already when to_return_sentinel_ + bool is_valid = !to_return_sentinel_ && file_iter_.NextAndGetResult(result); + if (!is_valid) { + if (to_return_sentinel_) { + ClearSentinel(); + } else if (range_tombstone_iter_) { + TrySetDeleteRangeSentinel(file_largest_key(file_index_)); + } + is_next_read_sequential_ = true; + SkipEmptyFileForward(); + is_next_read_sequential_ = false; + is_valid = Valid(); + if (is_valid) { + // This could be set in TrySetDeleteRangeSentinel() or + // SkipEmptyFileForward() above. + if (to_return_sentinel_) { + result->key = sentinel_; + result->bound_check_result = IterBoundCheck::kUnknown; + result->value_prepared = true; + } else { + result->key = key(); + result->bound_check_result = file_iter_.UpperBoundCheckResult(); + // Ideally, we should return the real file_iter_.value_prepared but the + // information is not here. It would casue an extra PrepareValue() + // for the first key of a file. + result->value_prepared = !allow_unprepared_value_; + } + } + } + return is_valid; +} + +void LevelIterator::Prev() { + assert(Valid()); + if (to_return_sentinel_) { + ClearSentinel(); + } else { + file_iter_.Prev(); + if (range_tombstone_iter_) { + TrySetDeleteRangeSentinel(file_smallest_key(file_index_)); + } + } + SkipEmptyFileBackward(); +} + +bool LevelIterator::SkipEmptyFileForward() { + bool seen_empty_file = false; + // Pause at sentinel key + while (!to_return_sentinel_ && + (file_iter_.iter() == nullptr || + (!file_iter_.Valid() && file_iter_.status().ok() && + file_iter_.iter()->UpperBoundCheckResult() != + IterBoundCheck::kOutOfBound))) { + seen_empty_file = true; + // Move to next file + if (file_index_ >= flevel_->num_files - 1 || + KeyReachedUpperBound(file_smallest_key(file_index_ + 1)) || + prefix_exhausted_) { + SetFileIterator(nullptr); + ClearRangeTombstoneIter(); + break; + } + // may init a new *range_tombstone_iter + InitFileIterator(file_index_ + 1); + // We moved to a new SST file + // Seek range_tombstone_iter_ to reset its !Valid() default state. + // We do not need to call range_tombstone_iter_.Seek* in + // LevelIterator::Seek* since when the merging iterator calls + // LevelIterator::Seek*, it should also call Seek* into the corresponding + // range tombstone iterator. + if (file_iter_.iter() != nullptr) { + file_iter_.SeekToFirst(); + if (range_tombstone_iter_) { + if (*range_tombstone_iter_) { + (*range_tombstone_iter_)->SeekToFirst(); + } + TrySetDeleteRangeSentinel(file_largest_key(file_index_)); + } + } + } + return seen_empty_file; +} + +void LevelIterator::SkipEmptyFileBackward() { + // Pause at sentinel key + while (!to_return_sentinel_ && + (file_iter_.iter() == nullptr || + (!file_iter_.Valid() && file_iter_.status().ok()))) { + // Move to previous file + if (file_index_ == 0) { + // Already the first file + SetFileIterator(nullptr); + ClearRangeTombstoneIter(); + return; + } + InitFileIterator(file_index_ - 1); + // We moved to a new SST file + // Seek range_tombstone_iter_ to reset its !Valid() default state. + if (file_iter_.iter() != nullptr) { + file_iter_.SeekToLast(); + if (range_tombstone_iter_) { + if (*range_tombstone_iter_) { + (*range_tombstone_iter_)->SeekToLast(); + } + TrySetDeleteRangeSentinel(file_smallest_key(file_index_)); + if (to_return_sentinel_) { + break; + } + } + } + } +} + +void LevelIterator::SetFileIterator(InternalIterator* iter) { + if (pinned_iters_mgr_ && iter) { + iter->SetPinnedItersMgr(pinned_iters_mgr_); + } + + InternalIterator* old_iter = file_iter_.Set(iter); + + // Update the read pattern for PrefetchBuffer. + if (is_next_read_sequential_) { + file_iter_.UpdateReadaheadState(old_iter); + } + + if (pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled()) { + pinned_iters_mgr_->PinIterator(old_iter); + } else { + delete old_iter; + } +} + +void LevelIterator::InitFileIterator(size_t new_file_index) { + if (new_file_index >= flevel_->num_files) { + file_index_ = new_file_index; + SetFileIterator(nullptr); + ClearRangeTombstoneIter(); + return; + } else { + // If the file iterator shows incomplete, we try it again if users seek + // to the same file, as this time we may go to a different data block + // which is cached in block cache. + // + if (file_iter_.iter() != nullptr && !file_iter_.status().IsIncomplete() && + new_file_index == file_index_) { + // file_iter_ is already constructed with this iterator, so + // no need to change anything + } else { + file_index_ = new_file_index; + InternalIterator* iter = NewFileIterator(); + SetFileIterator(iter); + } + } +} +} // anonymous namespace + +Status Version::GetTableProperties(const ReadOptions& read_options, + std::shared_ptr* tp, + const FileMetaData* file_meta, + const std::string* fname) const { + auto table_cache = cfd_->table_cache(); + auto ioptions = cfd_->ioptions(); + Status s = table_cache->GetTableProperties( + file_options_, read_options, cfd_->internal_comparator(), *file_meta, tp, + mutable_cf_options_.block_protection_bytes_per_key, + mutable_cf_options_.prefix_extractor, true /* no io */); + if (s.ok()) { + return s; + } + + // We only ignore error type `Incomplete` since it's by design that we + // disallow table when it's not in table cache. + if (!s.IsIncomplete()) { + return s; + } + + // 2. Table is not present in table cache, we'll read the table properties + // directly from the properties block in the file. + std::unique_ptr file; + std::string file_name; + if (fname != nullptr) { + file_name = *fname; + } else { + file_name = TableFileName(ioptions->cf_paths, file_meta->fd.GetNumber(), + file_meta->fd.GetPathId()); + } + s = ioptions->fs->NewRandomAccessFile(file_name, file_options_, &file, + nullptr); + if (!s.ok()) { + return s; + } + + // By setting the magic number to kNullTableMagicNumber, we can bypass + // the magic number check in the footer. + std::unique_ptr file_reader( + new RandomAccessFileReader( + std::move(file), file_name, ioptions->clock /* clock */, io_tracer_, + ioptions->stats /* stats */, + Histograms::SST_READ_MICROS /* hist_type */, + nullptr /* file_read_hist */, nullptr /* rate_limiter */, + ioptions->listeners)); + std::unique_ptr props; + s = ReadTableProperties( + file_reader.get(), file_meta->fd.GetFileSize(), + Footer::kNullTableMagicNumber /* table's magic number */, *ioptions, + read_options, &props); + if (!s.ok()) { + return s; + } + *tp = std::move(props); + RecordTick(ioptions->stats, NUMBER_DIRECT_LOAD_TABLE_PROPERTIES); + return s; +} + +Status Version::GetPropertiesOfAllTables(const ReadOptions& read_options, + TablePropertiesCollection* props) { + Status s; + for (int level = 0; level < storage_info_.num_levels_; level++) { + s = GetPropertiesOfAllTables(read_options, props, level); + if (!s.ok()) { + return s; + } + } + + return Status::OK(); +} + +Status Version::TablesRangeTombstoneSummary(int max_entries_to_print, + std::string* out_str) { + if (max_entries_to_print <= 0) { + return Status::OK(); + } + int num_entries_left = max_entries_to_print; + + std::stringstream ss; + + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + for (int level = 0; level < storage_info_.num_levels_; level++) { + for (const auto& file_meta : storage_info_.files_[level]) { + auto fname = + TableFileName(cfd_->ioptions()->cf_paths, file_meta->fd.GetNumber(), + file_meta->fd.GetPathId()); + + ss << "=== file : " << fname << " ===\n"; + + TableCache* table_cache = cfd_->table_cache(); + std::unique_ptr tombstone_iter; + + Status s = table_cache->GetRangeTombstoneIterator( + read_options, cfd_->internal_comparator(), *file_meta, + cfd_->GetLatestMutableCFOptions()->block_protection_bytes_per_key, + &tombstone_iter); + if (!s.ok()) { + return s; + } + if (tombstone_iter) { + tombstone_iter->SeekToFirst(); + + // TODO: print timestamp + while (tombstone_iter->Valid() && num_entries_left > 0) { + ss << "start: " << tombstone_iter->start_key().ToString(true) + << " end: " << tombstone_iter->end_key().ToString(true) + << " seq: " << tombstone_iter->seq() << '\n'; + tombstone_iter->Next(); + num_entries_left--; + } + if (num_entries_left <= 0) { + break; + } + } + } + if (num_entries_left <= 0) { + break; + } + } + assert(num_entries_left >= 0); + if (num_entries_left <= 0) { + ss << "(results may not be complete)\n"; + } + + *out_str = ss.str(); + return Status::OK(); +} + +Status Version::GetPropertiesOfAllTables(const ReadOptions& read_options, + TablePropertiesCollection* props, + int level) { + for (const auto& file_meta : storage_info_.files_[level]) { + auto fname = + TableFileName(cfd_->ioptions()->cf_paths, file_meta->fd.GetNumber(), + file_meta->fd.GetPathId()); + // 1. If the table is already present in table cache, load table + // properties from there. + std::shared_ptr table_properties; + Status s = + GetTableProperties(read_options, &table_properties, file_meta, &fname); + if (s.ok()) { + props->insert({fname, table_properties}); + } else { + return s; + } + } + + return Status::OK(); +} + +Status Version::GetPropertiesOfTablesInRange( + const ReadOptions& read_options, const Range* range, std::size_t n, + TablePropertiesCollection* props) const { + for (int level = 0; level < storage_info_.num_non_empty_levels(); level++) { + for (decltype(n) i = 0; i < n; i++) { + // Convert user_key into a corresponding internal key. + InternalKey k1(range[i].start, kMaxSequenceNumber, kValueTypeForSeek); + InternalKey k2(range[i].limit, kMaxSequenceNumber, kValueTypeForSeek); + std::vector files; + storage_info_.GetOverlappingInputs(level, &k1, &k2, &files, -1, nullptr, + false); + for (const auto& file_meta : files) { + auto fname = + TableFileName(cfd_->ioptions()->cf_paths, file_meta->fd.GetNumber(), + file_meta->fd.GetPathId()); + if (props->count(fname) == 0) { + // 1. If the table is already present in table cache, load table + // properties from there. + std::shared_ptr table_properties; + Status s = GetTableProperties(read_options, &table_properties, + file_meta, &fname); + if (s.ok()) { + props->insert({fname, table_properties}); + } else { + return s; + } + } + } + } + } + + return Status::OK(); +} + +Status Version::GetAggregatedTableProperties( + const ReadOptions& read_options, std::shared_ptr* tp, + int level) { + TablePropertiesCollection props; + Status s; + if (level < 0) { + s = GetPropertiesOfAllTables(read_options, &props); + } else { + s = GetPropertiesOfAllTables(read_options, &props, level); + } + if (!s.ok()) { + return s; + } + + auto* new_tp = new TableProperties(); + for (const auto& item : props) { + new_tp->Add(*item.second); + } + tp->reset(new_tp); + return Status::OK(); +} + +size_t Version::GetMemoryUsageByTableReaders(const ReadOptions& read_options) { + size_t total_usage = 0; + for (auto& file_level : storage_info_.level_files_brief_) { + for (size_t i = 0; i < file_level.num_files; i++) { + total_usage += cfd_->table_cache()->GetMemoryUsageByTableReader( + file_options_, read_options, cfd_->internal_comparator(), + *file_level.files[i].file_metadata, + mutable_cf_options_.block_protection_bytes_per_key, + mutable_cf_options_.prefix_extractor); + } + } + return total_usage; +} + +void Version::GetColumnFamilyMetaData(ColumnFamilyMetaData* cf_meta) { + assert(cf_meta); + assert(cfd_); + + cf_meta->name = cfd_->GetName(); + cf_meta->size = 0; + cf_meta->file_count = 0; + cf_meta->levels.clear(); + + cf_meta->blob_file_size = 0; + cf_meta->blob_file_count = 0; + cf_meta->blob_files.clear(); + + auto* ioptions = cfd_->ioptions(); + auto* vstorage = storage_info(); + + for (int level = 0; level < cfd_->NumberLevels(); level++) { + uint64_t level_size = 0; + cf_meta->file_count += vstorage->LevelFiles(level).size(); + std::vector files; + for (const auto& file : vstorage->LevelFiles(level)) { + uint32_t path_id = file->fd.GetPathId(); + std::string file_path; + if (path_id < ioptions->cf_paths.size()) { + file_path = ioptions->cf_paths[path_id].path; + } else { + assert(!ioptions->cf_paths.empty()); + file_path = ioptions->cf_paths.back().path; + } + const uint64_t file_number = file->fd.GetNumber(); + files.emplace_back( + MakeTableFileName("", file_number), file_number, file_path, + file->fd.GetFileSize(), file->fd.smallest_seqno, + file->fd.largest_seqno, file->smallest.user_key().ToString(), + file->largest.user_key().ToString(), + file->stats.num_reads_sampled.load(std::memory_order_relaxed), + file->being_compacted, file->temperature, + file->oldest_blob_file_number, file->TryGetOldestAncesterTime(), + file->TryGetFileCreationTime(), file->epoch_number, + file->file_checksum, file->file_checksum_func_name); + files.back().num_entries = file->num_entries; + files.back().num_deletions = file->num_deletions; + files.back().smallest = file->smallest.Encode().ToString(); + files.back().largest = file->largest.Encode().ToString(); + level_size += file->fd.GetFileSize(); + } + cf_meta->levels.emplace_back(level, level_size, std::move(files)); + cf_meta->size += level_size; + } + for (const auto& meta : vstorage->GetBlobFiles()) { + assert(meta); + + cf_meta->blob_files.emplace_back( + meta->GetBlobFileNumber(), BlobFileName("", meta->GetBlobFileNumber()), + ioptions->cf_paths.front().path, meta->GetBlobFileSize(), + meta->GetTotalBlobCount(), meta->GetTotalBlobBytes(), + meta->GetGarbageBlobCount(), meta->GetGarbageBlobBytes(), + meta->GetChecksumMethod(), meta->GetChecksumValue()); + ++cf_meta->blob_file_count; + cf_meta->blob_file_size += meta->GetBlobFileSize(); + } +} + +uint64_t Version::GetSstFilesSize() { + uint64_t sst_files_size = 0; + for (int level = 0; level < storage_info_.num_levels_; level++) { + for (const auto& file_meta : storage_info_.LevelFiles(level)) { + sst_files_size += file_meta->fd.GetFileSize(); + } + } + return sst_files_size; +} + +void Version::GetSstFilesBoundaryKeys(Slice* smallest_user_key, + Slice* largest_user_key) { + smallest_user_key->clear(); + largest_user_key->clear(); + bool initialized = false; + const Comparator* ucmp = storage_info_.user_comparator_; + for (int level = 0; level < cfd_->NumberLevels(); level++) { + if (storage_info_.LevelFiles(level).size() == 0) { + continue; + } + if (level == 0) { + // we need to consider all files on level 0 + for (const auto& file : storage_info_.LevelFiles(level)) { + const Slice& start_user_key = file->smallest.user_key(); + if (!initialized || + ucmp->Compare(start_user_key, *smallest_user_key) < 0) { + *smallest_user_key = start_user_key; + } + const Slice& end_user_key = file->largest.user_key(); + if (!initialized || + ucmp->Compare(end_user_key, *largest_user_key) > 0) { + *largest_user_key = end_user_key; + } + initialized = true; + } + } else { + // we only need to consider the first and last file + const Slice& start_user_key = + storage_info_.LevelFiles(level)[0]->smallest.user_key(); + if (!initialized || + ucmp->Compare(start_user_key, *smallest_user_key) < 0) { + *smallest_user_key = start_user_key; + } + const Slice& end_user_key = + storage_info_.LevelFiles(level).back()->largest.user_key(); + if (!initialized || ucmp->Compare(end_user_key, *largest_user_key) > 0) { + *largest_user_key = end_user_key; + } + initialized = true; + } + } +} + +void Version::GetCreationTimeOfOldestFile(uint64_t* creation_time) { + uint64_t oldest_time = std::numeric_limits::max(); + for (int level = 0; level < storage_info_.num_non_empty_levels_; level++) { + for (FileMetaData* meta : storage_info_.LevelFiles(level)) { + assert(meta->fd.table_reader != nullptr); + uint64_t file_creation_time = meta->TryGetFileCreationTime(); + if (file_creation_time == kUnknownFileCreationTime) { + *creation_time = 0; + return; + } + if (file_creation_time < oldest_time) { + oldest_time = file_creation_time; + } + } + } + *creation_time = oldest_time; +} + +InternalIterator* Version::TEST_GetLevelIterator( + const ReadOptions& read_options, MergeIteratorBuilder* merge_iter_builder, + int level, bool allow_unprepared_value) { + auto* arena = merge_iter_builder->GetArena(); + auto* mem = arena->AllocateAligned(sizeof(LevelIterator)); + TruncatedRangeDelIterator*** tombstone_iter_ptr = nullptr; + auto level_iter = new (mem) LevelIterator( + cfd_->table_cache(), read_options, file_options_, + cfd_->internal_comparator(), &storage_info_.LevelFilesBrief(level), + mutable_cf_options_.prefix_extractor, should_sample_file_read(), + cfd_->internal_stats()->GetFileReadHist(level), + TableReaderCaller::kUserIterator, IsFilterSkipped(level), level, + mutable_cf_options_.block_protection_bytes_per_key, + nullptr /* range_del_agg */, nullptr /* compaction_boundaries */, + allow_unprepared_value, &tombstone_iter_ptr); + if (read_options.ignore_range_deletions) { + merge_iter_builder->AddIterator(level_iter); + } else { + merge_iter_builder->AddPointAndTombstoneIterator( + level_iter, nullptr /* tombstone_iter */, tombstone_iter_ptr); + } + return level_iter; +} + +uint64_t VersionStorageInfo::GetEstimatedActiveKeys() const { + // Estimation will be inaccurate when: + // (1) there exist merge keys + // (2) keys are directly overwritten + // (3) deletion on non-existing keys + // (4) low number of samples + if (current_num_samples_ == 0) { + return 0; + } + + if (current_num_non_deletions_ <= current_num_deletions_) { + return 0; + } + + uint64_t est = current_num_non_deletions_ - current_num_deletions_; + + uint64_t file_count = 0; + for (int level = 0; level < num_levels_; ++level) { + file_count += files_[level].size(); + } + + if (current_num_samples_ < file_count) { + // casting to avoid overflowing + return static_cast( + (est * static_cast(file_count) / current_num_samples_)); + } else { + return est; + } +} + +double VersionStorageInfo::GetEstimatedCompressionRatioAtLevel( + int level) const { + assert(level < num_levels_); + uint64_t sum_file_size_bytes = 0; + uint64_t sum_data_size_bytes = 0; + for (auto* file_meta : files_[level]) { + sum_file_size_bytes += file_meta->fd.GetFileSize(); + sum_data_size_bytes += file_meta->raw_key_size + file_meta->raw_value_size; + } + if (sum_file_size_bytes == 0) { + return -1.0; + } + return static_cast(sum_data_size_bytes) / sum_file_size_bytes; +} + +void Version::AddIterators(const ReadOptions& read_options, + const FileOptions& soptions, + MergeIteratorBuilder* merge_iter_builder, + bool allow_unprepared_value) { + assert(storage_info_.finalized_); + + for (int level = 0; level < storage_info_.num_non_empty_levels(); level++) { + AddIteratorsForLevel(read_options, soptions, merge_iter_builder, level, + allow_unprepared_value); + } +} + +void Version::AddIteratorsForLevel(const ReadOptions& read_options, + const FileOptions& soptions, + MergeIteratorBuilder* merge_iter_builder, + int level, bool allow_unprepared_value) { + assert(storage_info_.finalized_); + if (level >= storage_info_.num_non_empty_levels()) { + // This is an empty level + return; + } else if (storage_info_.LevelFilesBrief(level).num_files == 0) { + // No files in this level + return; + } + + bool should_sample = should_sample_file_read(); + + auto* arena = merge_iter_builder->GetArena(); + if (level == 0) { + // Merge all level zero files together since they may overlap + TruncatedRangeDelIterator* tombstone_iter = nullptr; + for (size_t i = 0; i < storage_info_.LevelFilesBrief(0).num_files; i++) { + const auto& file = storage_info_.LevelFilesBrief(0).files[i]; + auto table_iter = cfd_->table_cache()->NewIterator( + read_options, soptions, cfd_->internal_comparator(), + *file.file_metadata, /*range_del_agg=*/nullptr, + mutable_cf_options_.prefix_extractor, nullptr, + cfd_->internal_stats()->GetFileReadHist(0), + TableReaderCaller::kUserIterator, arena, + /*skip_filters=*/false, /*level=*/0, max_file_size_for_l0_meta_pin_, + /*smallest_compaction_key=*/nullptr, + /*largest_compaction_key=*/nullptr, allow_unprepared_value, + mutable_cf_options_.block_protection_bytes_per_key, &tombstone_iter); + if (read_options.ignore_range_deletions) { + merge_iter_builder->AddIterator(table_iter); + } else { + merge_iter_builder->AddPointAndTombstoneIterator(table_iter, + tombstone_iter); + } + } + if (should_sample) { + // Count ones for every L0 files. This is done per iterator creation + // rather than Seek(), while files in other levels are recored per seek. + // If users execute one range query per iterator, there may be some + // discrepancy here. + for (FileMetaData* meta : storage_info_.LevelFiles(0)) { + sample_file_read_inc(meta); + } + } + } else if (storage_info_.LevelFilesBrief(level).num_files > 0) { + // For levels > 0, we can use a concatenating iterator that sequentially + // walks through the non-overlapping files in the level, opening them + // lazily. + auto* mem = arena->AllocateAligned(sizeof(LevelIterator)); + TruncatedRangeDelIterator*** tombstone_iter_ptr = nullptr; + auto level_iter = new (mem) LevelIterator( + cfd_->table_cache(), read_options, soptions, + cfd_->internal_comparator(), &storage_info_.LevelFilesBrief(level), + mutable_cf_options_.prefix_extractor, should_sample_file_read(), + cfd_->internal_stats()->GetFileReadHist(level), + TableReaderCaller::kUserIterator, IsFilterSkipped(level), level, + mutable_cf_options_.block_protection_bytes_per_key, + /*range_del_agg=*/nullptr, + /*compaction_boundaries=*/nullptr, allow_unprepared_value, + &tombstone_iter_ptr); + if (read_options.ignore_range_deletions) { + merge_iter_builder->AddIterator(level_iter); + } else { + merge_iter_builder->AddPointAndTombstoneIterator( + level_iter, nullptr /* tombstone_iter */, tombstone_iter_ptr); + } + } +} + +Status Version::OverlapWithLevelIterator(const ReadOptions& read_options, + const FileOptions& file_options, + const Slice& smallest_user_key, + const Slice& largest_user_key, + int level, bool* overlap) { + assert(storage_info_.finalized_); + + auto icmp = cfd_->internal_comparator(); + auto ucmp = icmp.user_comparator(); + + Arena arena; + Status status; + ReadRangeDelAggregator range_del_agg(&icmp, + kMaxSequenceNumber /* upper_bound */); + + *overlap = false; + + if (level == 0) { + for (size_t i = 0; i < storage_info_.LevelFilesBrief(0).num_files; i++) { + const auto file = &storage_info_.LevelFilesBrief(0).files[i]; + if (AfterFile(ucmp, &smallest_user_key, file) || + BeforeFile(ucmp, &largest_user_key, file)) { + continue; + } + ScopedArenaIterator iter(cfd_->table_cache()->NewIterator( + read_options, file_options, cfd_->internal_comparator(), + *file->file_metadata, &range_del_agg, + mutable_cf_options_.prefix_extractor, nullptr, + cfd_->internal_stats()->GetFileReadHist(0), + TableReaderCaller::kUserIterator, &arena, + /*skip_filters=*/false, /*level=*/0, max_file_size_for_l0_meta_pin_, + /*smallest_compaction_key=*/nullptr, + /*largest_compaction_key=*/nullptr, + /*allow_unprepared_value=*/false, + mutable_cf_options_.block_protection_bytes_per_key)); + status = OverlapWithIterator(ucmp, smallest_user_key, largest_user_key, + iter.get(), overlap); + if (!status.ok() || *overlap) { + break; + } + } + } else if (storage_info_.LevelFilesBrief(level).num_files > 0) { + auto mem = arena.AllocateAligned(sizeof(LevelIterator)); + ScopedArenaIterator iter(new (mem) LevelIterator( + cfd_->table_cache(), read_options, file_options, + cfd_->internal_comparator(), &storage_info_.LevelFilesBrief(level), + mutable_cf_options_.prefix_extractor, should_sample_file_read(), + cfd_->internal_stats()->GetFileReadHist(level), + TableReaderCaller::kUserIterator, IsFilterSkipped(level), level, + mutable_cf_options_.block_protection_bytes_per_key, &range_del_agg, + nullptr, false)); + status = OverlapWithIterator(ucmp, smallest_user_key, largest_user_key, + iter.get(), overlap); + } + + if (status.ok() && *overlap == false && + range_del_agg.IsRangeOverlapped(smallest_user_key, largest_user_key)) { + *overlap = true; + } + return status; +} + +VersionStorageInfo::VersionStorageInfo( + const InternalKeyComparator* internal_comparator, + const Comparator* user_comparator, int levels, + CompactionStyle compaction_style, VersionStorageInfo* ref_vstorage, + bool _force_consistency_checks, + EpochNumberRequirement epoch_number_requirement) + : internal_comparator_(internal_comparator), + user_comparator_(user_comparator), + // cfd is nullptr if Version is dummy + num_levels_(levels), + num_non_empty_levels_(0), + file_indexer_(user_comparator), + compaction_style_(compaction_style), + files_(new std::vector[num_levels_]), + base_level_(num_levels_ == 1 ? -1 : 1), + lowest_unnecessary_level_(-1), + level_multiplier_(0.0), + files_by_compaction_pri_(num_levels_), + level0_non_overlapping_(false), + next_file_to_compact_by_size_(num_levels_), + compaction_score_(num_levels_), + compaction_level_(num_levels_), + l0_delay_trigger_count_(0), + compact_cursor_(num_levels_), + accumulated_file_size_(0), + accumulated_raw_key_size_(0), + accumulated_raw_value_size_(0), + accumulated_num_non_deletions_(0), + accumulated_num_deletions_(0), + current_num_non_deletions_(0), + current_num_deletions_(0), + current_num_samples_(0), + estimated_compaction_needed_bytes_(0), + finalized_(false), + force_consistency_checks_(_force_consistency_checks), + epoch_number_requirement_(epoch_number_requirement) { + if (ref_vstorage != nullptr) { + accumulated_file_size_ = ref_vstorage->accumulated_file_size_; + accumulated_raw_key_size_ = ref_vstorage->accumulated_raw_key_size_; + accumulated_raw_value_size_ = ref_vstorage->accumulated_raw_value_size_; + accumulated_num_non_deletions_ = + ref_vstorage->accumulated_num_non_deletions_; + accumulated_num_deletions_ = ref_vstorage->accumulated_num_deletions_; + current_num_non_deletions_ = ref_vstorage->current_num_non_deletions_; + current_num_deletions_ = ref_vstorage->current_num_deletions_; + current_num_samples_ = ref_vstorage->current_num_samples_; + oldest_snapshot_seqnum_ = ref_vstorage->oldest_snapshot_seqnum_; + compact_cursor_ = ref_vstorage->compact_cursor_; + compact_cursor_.resize(num_levels_); + } +} + +Version::Version(ColumnFamilyData* column_family_data, VersionSet* vset, + const FileOptions& file_opt, + const MutableCFOptions mutable_cf_options, + const std::shared_ptr& io_tracer, + uint64_t version_number, + EpochNumberRequirement epoch_number_requirement) + : env_(vset->env_), + clock_(vset->clock_), + cfd_(column_family_data), + info_log_((cfd_ == nullptr) ? nullptr : cfd_->ioptions()->logger), + db_statistics_((cfd_ == nullptr) ? nullptr : cfd_->ioptions()->stats), + table_cache_((cfd_ == nullptr) ? nullptr : cfd_->table_cache()), + blob_source_(cfd_ ? cfd_->blob_source() : nullptr), + merge_operator_( + (cfd_ == nullptr) ? nullptr : cfd_->ioptions()->merge_operator.get()), + storage_info_( + (cfd_ == nullptr) ? nullptr : &cfd_->internal_comparator(), + (cfd_ == nullptr) ? nullptr : cfd_->user_comparator(), + cfd_ == nullptr ? 0 : cfd_->NumberLevels(), + cfd_ == nullptr ? kCompactionStyleLevel + : cfd_->ioptions()->compaction_style, + (cfd_ == nullptr || cfd_->current() == nullptr) + ? nullptr + : cfd_->current()->storage_info(), + cfd_ == nullptr ? false : cfd_->ioptions()->force_consistency_checks, + epoch_number_requirement), + vset_(vset), + next_(this), + prev_(this), + refs_(0), + file_options_(file_opt), + mutable_cf_options_(mutable_cf_options), + max_file_size_for_l0_meta_pin_( + MaxFileSizeForL0MetaPin(mutable_cf_options_)), + version_number_(version_number), + io_tracer_(io_tracer), + use_async_io_(false) { + if (CheckFSFeatureSupport(env_->GetFileSystem().get(), + FSSupportedOps::kAsyncIO)) { + use_async_io_ = true; + } +} + +Status Version::GetBlob(const ReadOptions& read_options, const Slice& user_key, + const Slice& blob_index_slice, + FilePrefetchBuffer* prefetch_buffer, + PinnableSlice* value, uint64_t* bytes_read) const { + BlobIndex blob_index; + + { + Status s = blob_index.DecodeFrom(blob_index_slice); + if (!s.ok()) { + return s; + } + } + + return GetBlob(read_options, user_key, blob_index, prefetch_buffer, value, + bytes_read); +} + +Status Version::GetBlob(const ReadOptions& read_options, const Slice& user_key, + const BlobIndex& blob_index, + FilePrefetchBuffer* prefetch_buffer, + PinnableSlice* value, uint64_t* bytes_read) const { + assert(value); + + if (blob_index.HasTTL() || blob_index.IsInlined()) { + return Status::Corruption("Unexpected TTL/inlined blob index"); + } + + const uint64_t blob_file_number = blob_index.file_number(); + + auto blob_file_meta = storage_info_.GetBlobFileMetaData(blob_file_number); + if (!blob_file_meta) { + return Status::Corruption("Invalid blob file number"); + } + + assert(blob_source_); + value->Reset(); + const Status s = blob_source_->GetBlob( + read_options, user_key, blob_file_number, blob_index.offset(), + blob_file_meta->GetBlobFileSize(), blob_index.size(), + blob_index.compression(), prefetch_buffer, value, bytes_read); + + return s; +} + +void Version::MultiGetBlob( + const ReadOptions& read_options, MultiGetRange& range, + std::unordered_map& blob_ctxs) { + assert(!blob_ctxs.empty()); + + autovector blob_reqs; + + for (auto& ctx : blob_ctxs) { + const auto file_number = ctx.first; + const auto blob_file_meta = storage_info_.GetBlobFileMetaData(file_number); + + autovector blob_reqs_in_file; + BlobReadContexts& blobs_in_file = ctx.second; + for (auto& blob : blobs_in_file) { + const BlobIndex& blob_index = blob.blob_index; + const KeyContext* const key_context = blob.key_context; + assert(key_context); + assert(key_context->get_context); + assert(key_context->s); + + if (key_context->value) { + key_context->value->Reset(); + } else { + assert(key_context->columns); + key_context->columns->Reset(); + } + + if (!blob_file_meta) { + *key_context->s = Status::Corruption("Invalid blob file number"); + continue; + } + + if (blob_index.HasTTL() || blob_index.IsInlined()) { + *key_context->s = + Status::Corruption("Unexpected TTL/inlined blob index"); + continue; + } + + blob_reqs_in_file.emplace_back( + key_context->get_context->ukey_to_get_blob_value(), + blob_index.offset(), blob_index.size(), blob_index.compression(), + &blob.result, key_context->s); + } + if (blob_reqs_in_file.size() > 0) { + const auto file_size = blob_file_meta->GetBlobFileSize(); + blob_reqs.emplace_back(file_number, file_size, blob_reqs_in_file); + } + } + + if (blob_reqs.size() > 0) { + blob_source_->MultiGetBlob(read_options, blob_reqs, + /*bytes_read=*/nullptr); + } + + for (auto& ctx : blob_ctxs) { + BlobReadContexts& blobs_in_file = ctx.second; + for (auto& blob : blobs_in_file) { + const KeyContext* const key_context = blob.key_context; + assert(key_context); + assert(key_context->get_context); + assert(key_context->s); + + if (key_context->s->ok()) { + if (key_context->value) { + *key_context->value = std::move(blob.result); + range.AddValueSize(key_context->value->size()); + } else { + assert(key_context->columns); + key_context->columns->SetPlainValue(std::move(blob.result)); + range.AddValueSize(key_context->columns->serialized_size()); + } + + if (range.GetValueSize() > read_options.value_size_soft_limit) { + *key_context->s = Status::Aborted(); + } + } else if (key_context->s->IsIncomplete()) { + // read_options.read_tier == kBlockCacheTier + // Cannot read blob(s): no disk I/O allowed + auto& get_context = *(key_context->get_context); + get_context.MarkKeyMayExist(); + } + } + } +} + +void Version::Get(const ReadOptions& read_options, const LookupKey& k, + PinnableSlice* value, PinnableWideColumns* columns, + std::string* timestamp, Status* status, + MergeContext* merge_context, + SequenceNumber* max_covering_tombstone_seq, + PinnedIteratorsManager* pinned_iters_mgr, bool* value_found, + bool* key_exists, SequenceNumber* seq, ReadCallback* callback, + bool* is_blob, bool do_merge) { + Slice ikey = k.internal_key(); + Slice user_key = k.user_key(); + + assert(status->ok() || status->IsMergeInProgress()); + + if (key_exists != nullptr) { + // will falsify below if not found + *key_exists = true; + } + + uint64_t tracing_get_id = BlockCacheTraceHelper::kReservedGetId; + if (vset_ && vset_->block_cache_tracer_ && + vset_->block_cache_tracer_->is_tracing_enabled()) { + tracing_get_id = vset_->block_cache_tracer_->NextGetId(); + } + + // Note: the old StackableDB-based BlobDB passes in + // GetImplOptions::is_blob_index; for the integrated BlobDB implementation, we + // need to provide it here. + bool is_blob_index = false; + bool* const is_blob_to_use = is_blob ? is_blob : &is_blob_index; + BlobFetcher blob_fetcher(this, read_options); + + assert(pinned_iters_mgr); + GetContext get_context( + user_comparator(), merge_operator_, info_log_, db_statistics_, + status->ok() ? GetContext::kNotFound : GetContext::kMerge, user_key, + do_merge ? value : nullptr, do_merge ? columns : nullptr, + do_merge ? timestamp : nullptr, value_found, merge_context, do_merge, + max_covering_tombstone_seq, clock_, seq, + merge_operator_ ? pinned_iters_mgr : nullptr, callback, is_blob_to_use, + tracing_get_id, &blob_fetcher); + + // Pin blocks that we read to hold merge operands + if (merge_operator_) { + pinned_iters_mgr->StartPinning(); + } + + FilePicker fp(user_key, ikey, &storage_info_.level_files_brief_, + storage_info_.num_non_empty_levels_, + &storage_info_.file_indexer_, user_comparator(), + internal_comparator()); + FdWithKeyRange* f = fp.GetNextFile(); + + while (f != nullptr) { + if (*max_covering_tombstone_seq > 0) { + // The remaining files we look at will only contain covered keys, so we + // stop here. + break; + } + if (get_context.sample()) { + sample_file_read_inc(f->file_metadata); + } + + bool timer_enabled = + GetPerfLevel() >= PerfLevel::kEnableTimeExceptForMutex && + get_perf_context()->per_level_perf_context_enabled; + StopWatchNano timer(clock_, timer_enabled /* auto_start */); + *status = table_cache_->Get( + read_options, *internal_comparator(), *f->file_metadata, ikey, + &get_context, mutable_cf_options_.block_protection_bytes_per_key, + mutable_cf_options_.prefix_extractor, + cfd_->internal_stats()->GetFileReadHist(fp.GetHitFileLevel()), + IsFilterSkipped(static_cast(fp.GetHitFileLevel()), + fp.IsHitFileLastInLevel()), + fp.GetHitFileLevel(), max_file_size_for_l0_meta_pin_); + // TODO: examine the behavior for corrupted key + if (timer_enabled) { + PERF_COUNTER_BY_LEVEL_ADD(get_from_table_nanos, timer.ElapsedNanos(), + fp.GetHitFileLevel()); + } + if (!status->ok()) { + if (db_statistics_ != nullptr) { + get_context.ReportCounters(); + } + return; + } + + // report the counters before returning + if (get_context.State() != GetContext::kNotFound && + get_context.State() != GetContext::kMerge && + db_statistics_ != nullptr) { + get_context.ReportCounters(); + } + switch (get_context.State()) { + case GetContext::kNotFound: + // Keep searching in other files + break; + case GetContext::kMerge: + // TODO: update per-level perfcontext user_key_return_count for kMerge + break; + case GetContext::kFound: + if (fp.GetHitFileLevel() == 0) { + RecordTick(db_statistics_, GET_HIT_L0); + } else if (fp.GetHitFileLevel() == 1) { + RecordTick(db_statistics_, GET_HIT_L1); + } else if (fp.GetHitFileLevel() >= 2) { + RecordTick(db_statistics_, GET_HIT_L2_AND_UP); + } + + PERF_COUNTER_BY_LEVEL_ADD(user_key_return_count, 1, + fp.GetHitFileLevel()); + + if (is_blob_index && do_merge && (value || columns)) { + assert(!columns || + (!columns->columns().empty() && + columns->columns().front().name() == kDefaultWideColumnName)); + + Slice blob_index = + value ? *value : columns->columns().front().value(); + + TEST_SYNC_POINT_CALLBACK("Version::Get::TamperWithBlobIndex", + &blob_index); + + constexpr FilePrefetchBuffer* prefetch_buffer = nullptr; + + PinnableSlice result; + + constexpr uint64_t* bytes_read = nullptr; + + *status = GetBlob(read_options, get_context.ukey_to_get_blob_value(), + blob_index, prefetch_buffer, &result, bytes_read); + if (!status->ok()) { + if (status->IsIncomplete()) { + get_context.MarkKeyMayExist(); + } + return; + } + + if (value) { + *value = std::move(result); + } else { + assert(columns); + columns->SetPlainValue(std::move(result)); + } + } + + return; + case GetContext::kDeleted: + // Use empty error message for speed + *status = Status::NotFound(); + return; + case GetContext::kCorrupt: + *status = Status::Corruption("corrupted key for ", user_key); + return; + case GetContext::kUnexpectedBlobIndex: + ROCKS_LOG_ERROR(info_log_, "Encounter unexpected blob index."); + *status = Status::NotSupported( + "Encounter unexpected blob index. Please open DB with " + "ROCKSDB_NAMESPACE::blob_db::BlobDB instead."); + return; + case GetContext::kMergeOperatorFailed: + *status = Status::Corruption(Status::SubCode::kMergeOperatorFailed); + return; + } + f = fp.GetNextFile(); + } + if (db_statistics_ != nullptr) { + get_context.ReportCounters(); + } + if (GetContext::kMerge == get_context.State()) { + if (!do_merge) { + *status = Status::OK(); + return; + } + if (!merge_operator_) { + *status = Status::InvalidArgument( + "merge_operator is not properly initialized."); + return; + } + // merge_operands are in saver and we hit the beginning of the key history + // do a final merge of nullptr and operands; + if (value || columns) { + std::string result; + // `op_failure_scope` (an output parameter) is not provided (set to + // nullptr) since a failure must be propagated regardless of its value. + *status = MergeHelper::TimedFullMerge( + merge_operator_, user_key, nullptr, merge_context->GetOperands(), + &result, info_log_, db_statistics_, clock_, + /* result_operand */ nullptr, /* update_num_ops_stats */ true, + /* op_failure_scope */ nullptr); + if (status->ok()) { + if (LIKELY(value != nullptr)) { + *(value->GetSelf()) = std::move(result); + value->PinSelf(); + } else { + assert(columns != nullptr); + columns->SetPlainValue(std::move(result)); + } + } + } + } else { + if (key_exists != nullptr) { + *key_exists = false; + } + *status = Status::NotFound(); // Use an empty error message for speed + } +} + +void Version::MultiGet(const ReadOptions& read_options, MultiGetRange* range, + ReadCallback* callback) { + PinnedIteratorsManager pinned_iters_mgr; + + // Pin blocks that we read to hold merge operands + if (merge_operator_) { + pinned_iters_mgr.StartPinning(); + } + uint64_t tracing_mget_id = BlockCacheTraceHelper::kReservedGetId; + + if (vset_ && vset_->block_cache_tracer_ && + vset_->block_cache_tracer_->is_tracing_enabled()) { + tracing_mget_id = vset_->block_cache_tracer_->NextGetId(); + } + // Even though we know the batch size won't be > MAX_BATCH_SIZE, + // use autovector in order to avoid unnecessary construction of GetContext + // objects, which is expensive + autovector get_ctx; + BlobFetcher blob_fetcher(this, read_options); + for (auto iter = range->begin(); iter != range->end(); ++iter) { + assert(iter->s->ok() || iter->s->IsMergeInProgress()); + get_ctx.emplace_back( + user_comparator(), merge_operator_, info_log_, db_statistics_, + iter->s->ok() ? GetContext::kNotFound : GetContext::kMerge, + iter->ukey_with_ts, iter->value, iter->columns, iter->timestamp, + nullptr, &(iter->merge_context), true, + &iter->max_covering_tombstone_seq, clock_, nullptr, + merge_operator_ ? &pinned_iters_mgr : nullptr, callback, + &iter->is_blob_index, tracing_mget_id, &blob_fetcher); + // MergeInProgress status, if set, has been transferred to the get_context + // state, so we set status to ok here. From now on, the iter status will + // be used for IO errors, and get_context state will be used for any + // key level errors + *(iter->s) = Status::OK(); + } + int get_ctx_index = 0; + for (auto iter = range->begin(); iter != range->end(); + ++iter, get_ctx_index++) { + iter->get_context = &(get_ctx[get_ctx_index]); + } + + Status s; + // blob_file => [[blob_idx, it], ...] + std::unordered_map blob_ctxs; + MultiGetRange keys_with_blobs_range(*range, range->begin(), range->end()); +#if USE_COROUTINES + if (read_options.async_io && read_options.optimize_multiget_for_io && + using_coroutines() && use_async_io_) { + s = MultiGetAsync(read_options, range, &blob_ctxs); + } else +#endif // USE_COROUTINES + { + MultiGetRange file_picker_range(*range, range->begin(), range->end()); + FilePickerMultiGet fp(&file_picker_range, &storage_info_.level_files_brief_, + storage_info_.num_non_empty_levels_, + &storage_info_.file_indexer_, user_comparator(), + internal_comparator()); + FdWithKeyRange* f = fp.GetNextFileInLevel(); + uint64_t num_index_read = 0; + uint64_t num_filter_read = 0; + uint64_t num_sst_read = 0; + uint64_t num_level_read = 0; + + int prev_level = -1; + + while (!fp.IsSearchEnded()) { + // This will be set to true later if we actually look up in a file in L0. + // For per level stats purposes, an L0 file is treated as a level + bool dump_stats_for_l0_file = false; + + // Avoid using the coroutine version if we're looking in a L0 file, since + // L0 files won't be parallelized anyway. The regular synchronous version + // is faster. + if (!read_options.async_io || !using_coroutines() || !use_async_io_ || + fp.GetHitFileLevel() == 0 || !fp.RemainingOverlapInLevel()) { + if (f) { + bool skip_filters = + IsFilterSkipped(static_cast(fp.GetHitFileLevel()), + fp.IsHitFileLastInLevel()); + // Call MultiGetFromSST for looking up a single file + s = MultiGetFromSST(read_options, fp.CurrentFileRange(), + fp.GetHitFileLevel(), skip_filters, + /*skip_range_deletions=*/false, f, blob_ctxs, + /*table_handle=*/nullptr, num_filter_read, + num_index_read, num_sst_read); + if (fp.GetHitFileLevel() == 0) { + dump_stats_for_l0_file = true; + } + } + if (s.ok()) { + f = fp.GetNextFileInLevel(); + } +#if USE_COROUTINES + } else { + std::vector> mget_tasks; + while (f != nullptr) { + MultiGetRange file_range = fp.CurrentFileRange(); + TableCache::TypedHandle* table_handle = nullptr; + bool skip_filters = + IsFilterSkipped(static_cast(fp.GetHitFileLevel()), + fp.IsHitFileLastInLevel()); + bool skip_range_deletions = false; + if (!skip_filters) { + Status status = table_cache_->MultiGetFilter( + read_options, *internal_comparator(), *f->file_metadata, + mutable_cf_options_.prefix_extractor, + cfd_->internal_stats()->GetFileReadHist(fp.GetHitFileLevel()), + fp.GetHitFileLevel(), &file_range, &table_handle, + mutable_cf_options_.block_protection_bytes_per_key); + skip_range_deletions = true; + if (status.ok()) { + skip_filters = true; + } else if (!status.IsNotSupported()) { + s = status; + } + } + + if (!s.ok()) { + break; + } + + if (!file_range.empty()) { + mget_tasks.emplace_back(MultiGetFromSSTCoroutine( + read_options, file_range, fp.GetHitFileLevel(), skip_filters, + skip_range_deletions, f, blob_ctxs, table_handle, + num_filter_read, num_index_read, num_sst_read)); + } + if (fp.KeyMaySpanNextFile()) { + break; + } + f = fp.GetNextFileInLevel(); + } + if (mget_tasks.size() > 0) { + RecordTick(db_statistics_, MULTIGET_COROUTINE_COUNT, + mget_tasks.size()); + // Collect all results so far + std::vector statuses = folly::coro::blockingWait( + folly::coro::collectAllRange(std::move(mget_tasks)) + .scheduleOn(&range->context()->executor())); + if (s.ok()) { + for (Status stat : statuses) { + if (!stat.ok()) { + s = std::move(stat); + break; + } + } + } + + if (s.ok() && fp.KeyMaySpanNextFile()) { + f = fp.GetNextFileInLevel(); + } + } +#endif // USE_COROUTINES + } + // If bad status or we found final result for all the keys + if (!s.ok() || file_picker_range.empty()) { + break; + } + if (!f) { + // Reached the end of this level. Prepare the next level + fp.PrepareNextLevelForSearch(); + if (!fp.IsSearchEnded()) { + // Its possible there is no overlap on this level and f is nullptr + f = fp.GetNextFileInLevel(); + } + if (dump_stats_for_l0_file || + (prev_level != 0 && prev_level != (int)fp.GetHitFileLevel())) { + // Dump the stats if the search has moved to the next level and + // reset for next level. + if (num_filter_read + num_index_read) { + RecordInHistogram(db_statistics_, + NUM_INDEX_AND_FILTER_BLOCKS_READ_PER_LEVEL, + num_index_read + num_filter_read); + } + if (num_sst_read) { + RecordInHistogram(db_statistics_, NUM_SST_READ_PER_LEVEL, + num_sst_read); + num_level_read++; + } + num_filter_read = 0; + num_index_read = 0; + num_sst_read = 0; + } + prev_level = fp.GetHitFileLevel(); + } + } + + // Dump stats for most recent level + if (num_filter_read + num_index_read) { + RecordInHistogram(db_statistics_, + NUM_INDEX_AND_FILTER_BLOCKS_READ_PER_LEVEL, + num_index_read + num_filter_read); + } + if (num_sst_read) { + RecordInHistogram(db_statistics_, NUM_SST_READ_PER_LEVEL, num_sst_read); + num_level_read++; + } + if (num_level_read) { + RecordInHistogram(db_statistics_, NUM_LEVEL_READ_PER_MULTIGET, + num_level_read); + } + } + + if (s.ok() && !blob_ctxs.empty()) { + MultiGetBlob(read_options, keys_with_blobs_range, blob_ctxs); + } + + // Process any left over keys + for (auto iter = range->begin(); s.ok() && iter != range->end(); ++iter) { + GetContext& get_context = *iter->get_context; + Status* status = iter->s; + Slice user_key = iter->lkey->user_key(); + + if (db_statistics_ != nullptr) { + get_context.ReportCounters(); + } + if (GetContext::kMerge == get_context.State()) { + if (!merge_operator_) { + *status = Status::InvalidArgument( + "merge_operator is not properly initialized."); + range->MarkKeyDone(iter); + continue; + } + // merge_operands are in saver and we hit the beginning of the key history + // do a final merge of nullptr and operands; + std::string result; + + // `op_failure_scope` (an output parameter) is not provided (set to + // nullptr) since a failure must be propagated regardless of its value. + *status = MergeHelper::TimedFullMerge( + merge_operator_, user_key, nullptr, iter->merge_context.GetOperands(), + &result, info_log_, db_statistics_, clock_, + /* result_operand */ nullptr, /* update_num_ops_stats */ true, + /* op_failure_scope */ nullptr); + if (LIKELY(iter->value != nullptr)) { + *iter->value->GetSelf() = std::move(result); + iter->value->PinSelf(); + range->AddValueSize(iter->value->size()); + } else { + assert(iter->columns); + iter->columns->SetPlainValue(std::move(result)); + range->AddValueSize(iter->columns->serialized_size()); + } + + range->MarkKeyDone(iter); + if (range->GetValueSize() > read_options.value_size_soft_limit) { + s = Status::Aborted(); + break; + } + } else { + range->MarkKeyDone(iter); + *status = Status::NotFound(); // Use an empty error message for speed + } + } + + for (auto iter = range->begin(); iter != range->end(); ++iter) { + range->MarkKeyDone(iter); + *(iter->s) = s; + } +} + +#ifdef USE_COROUTINES +Status Version::ProcessBatch( + const ReadOptions& read_options, FilePickerMultiGet* batch, + std::vector>& mget_tasks, + std::unordered_map* blob_ctxs, + autovector& batches, std::deque& waiting, + std::deque& to_process, unsigned int& num_tasks_queued, + std::unordered_map>& + mget_stats) { + FilePickerMultiGet& fp = *batch; + MultiGetRange range = fp.GetRange(); + // Initialize a new empty range. Any keys that are not in this level will + // eventually become part of the new range. + MultiGetRange leftover(range, range.begin(), range.begin()); + FdWithKeyRange* f = nullptr; + Status s; + + f = fp.GetNextFileInLevel(); + while (!f) { + fp.PrepareNextLevelForSearch(); + if (!fp.IsSearchEnded()) { + f = fp.GetNextFileInLevel(); + } else { + break; + } + } + while (f) { + MultiGetRange file_range = fp.CurrentFileRange(); + TableCache::TypedHandle* table_handle = nullptr; + bool skip_filters = IsFilterSkipped(static_cast(fp.GetHitFileLevel()), + fp.IsHitFileLastInLevel()); + bool skip_range_deletions = false; + if (!skip_filters) { + Status status = table_cache_->MultiGetFilter( + read_options, *internal_comparator(), *f->file_metadata, + mutable_cf_options_.prefix_extractor, + cfd_->internal_stats()->GetFileReadHist(fp.GetHitFileLevel()), + fp.GetHitFileLevel(), &file_range, &table_handle, + mutable_cf_options_.block_protection_bytes_per_key); + if (status.ok()) { + skip_filters = true; + skip_range_deletions = true; + } else if (!status.IsNotSupported()) { + s = status; + } + } + if (!s.ok()) { + break; + } + // At this point, file_range contains any keys that are likely in this + // file. It may have false positives, but that's ok since higher level + // lookups for the key are dependent on this lookup anyway. + // Add the complement of file_range to leftover. That's the set of keys + // definitely not in this level. + // Subtract the complement of file_range from range, since they will be + // processed in a separate batch in parallel. + leftover += ~file_range; + range -= ~file_range; + if (!file_range.empty()) { + int level = fp.GetHitFileLevel(); + auto stat = mget_stats.find(level); + if (stat == mget_stats.end()) { + auto entry = mget_stats.insert({level, {0, 0, 0}}); + assert(entry.second); + stat = entry.first; + } + + if (waiting.empty() && to_process.empty() && + !fp.RemainingOverlapInLevel() && leftover.empty() && + mget_tasks.empty()) { + // All keys are in one SST file, so take the fast path + s = MultiGetFromSST(read_options, file_range, fp.GetHitFileLevel(), + skip_filters, skip_range_deletions, f, *blob_ctxs, + table_handle, std::get<0>(stat->second), + std::get<1>(stat->second), + std::get<2>(stat->second)); + } else { + mget_tasks.emplace_back(MultiGetFromSSTCoroutine( + read_options, file_range, fp.GetHitFileLevel(), skip_filters, + skip_range_deletions, f, *blob_ctxs, table_handle, + std::get<0>(stat->second), std::get<1>(stat->second), + std::get<2>(stat->second))); + ++num_tasks_queued; + } + } + if (fp.KeyMaySpanNextFile() && !file_range.empty()) { + break; + } + f = fp.GetNextFileInLevel(); + } + // Split the current batch only if some keys are likely in this level and + // some are not. Only split if we're done with this level, i.e f is null. + // Otherwise, it means there are more files in this level to look at. + if (s.ok() && !f && !leftover.empty() && !range.empty()) { + fp.ReplaceRange(range); + batches.emplace_back(&leftover, fp); + to_process.emplace_back(batches.size() - 1); + } + // 1. If f is non-null, that means we might not be done with this level. + // This can happen if one of the keys is the last key in the file, i.e + // fp.KeyMaySpanNextFile() is true. + // 2. If range is empty, then we're done with this range and no need to + // prepare the next level + // 3. If some tasks were queued for this range, then the next level will be + // prepared after executing those tasks + if (!f && !range.empty() && !num_tasks_queued) { + fp.PrepareNextLevelForSearch(); + } + return s; +} + +Status Version::MultiGetAsync( + const ReadOptions& options, MultiGetRange* range, + std::unordered_map* blob_ctxs) { + autovector batches; + std::deque waiting; + std::deque to_process; + Status s; + std::vector> mget_tasks; + std::unordered_map> mget_stats; + + // Create the initial batch with the input range + batches.emplace_back(range, &storage_info_.level_files_brief_, + storage_info_.num_non_empty_levels_, + &storage_info_.file_indexer_, user_comparator(), + internal_comparator()); + to_process.emplace_back(0); + + while (!to_process.empty()) { + // As we process a batch, it may get split into two. So reserve space for + // an additional batch in the autovector in order to prevent later moves + // of elements in ProcessBatch(). + batches.reserve(batches.size() + 1); + + size_t idx = to_process.front(); + FilePickerMultiGet* batch = &batches.at(idx); + unsigned int num_tasks_queued = 0; + to_process.pop_front(); + if (batch->IsSearchEnded() || batch->GetRange().empty()) { + // If to_process is empty, i.e no more batches to look at, then we need + // schedule the enqueued coroutines and wait for them. Otherwise, we + // skip this batch and move to the next one in to_process. + if (!to_process.empty()) { + continue; + } + } else { + // Look through one level. This may split the batch and enqueue it to + // to_process + s = ProcessBatch(options, batch, mget_tasks, blob_ctxs, batches, waiting, + to_process, num_tasks_queued, mget_stats); + // If ProcessBatch didn't enqueue any coroutine tasks, it means all + // keys were filtered out. So put the batch back in to_process to + // lookup in the next level + if (!num_tasks_queued && !batch->IsSearchEnded()) { + // Put this back in the processing queue + to_process.emplace_back(idx); + } else if (num_tasks_queued) { + waiting.emplace_back(idx); + } + } + // If ProcessBatch() returned an error, then schedule the enqueued + // coroutines and wait for them, then abort the MultiGet. + if (to_process.empty() || !s.ok()) { + if (mget_tasks.size() > 0) { + assert(waiting.size()); + RecordTick(db_statistics_, MULTIGET_COROUTINE_COUNT, mget_tasks.size()); + // Collect all results so far + std::vector statuses = folly::coro::blockingWait( + folly::coro::collectAllRange(std::move(mget_tasks)) + .scheduleOn(&range->context()->executor())); + mget_tasks.clear(); + if (s.ok()) { + for (Status stat : statuses) { + if (!stat.ok()) { + s = std::move(stat); + break; + } + } + } + + if (!s.ok()) { + break; + } + + for (size_t wait_idx : waiting) { + FilePickerMultiGet& fp = batches.at(wait_idx); + // 1. If fp.GetHitFile() is non-null, then there could be more + // overlap in this level. So skip preparing next level. + // 2. If fp.GetRange() is empty, then this batch is completed + // and no need to prepare the next level. + if (!fp.GetHitFile() && !fp.GetRange().empty()) { + fp.PrepareNextLevelForSearch(); + } + } + to_process.swap(waiting); + } else { + assert(!s.ok() || waiting.size() == 0); + } + } + if (!s.ok()) { + break; + } + } + + uint64_t num_levels = 0; + for (auto& stat : mget_stats) { + if (stat.first == 0) { + num_levels += std::get<2>(stat.second); + } else { + num_levels++; + } + + uint64_t num_meta_reads = + std::get<0>(stat.second) + std::get<1>(stat.second); + uint64_t num_sst_reads = std::get<2>(stat.second); + if (num_meta_reads > 0) { + RecordInHistogram(db_statistics_, + NUM_INDEX_AND_FILTER_BLOCKS_READ_PER_LEVEL, + num_meta_reads); + } + if (num_sst_reads > 0) { + RecordInHistogram(db_statistics_, NUM_SST_READ_PER_LEVEL, num_sst_reads); + } + } + if (num_levels > 0) { + RecordInHistogram(db_statistics_, NUM_LEVEL_READ_PER_MULTIGET, num_levels); + } + + return s; +} +#endif + +bool Version::IsFilterSkipped(int level, bool is_file_last_in_level) { + // Reaching the bottom level implies misses at all upper levels, so we'll + // skip checking the filters when we predict a hit. + return cfd_->ioptions()->optimize_filters_for_hits && + (level > 0 || is_file_last_in_level) && + level == storage_info_.num_non_empty_levels() - 1; +} + +void VersionStorageInfo::GenerateLevelFilesBrief() { + level_files_brief_.resize(num_non_empty_levels_); + for (int level = 0; level < num_non_empty_levels_; level++) { + DoGenerateLevelFilesBrief(&level_files_brief_[level], files_[level], + &arena_); + } +} + +void VersionStorageInfo::PrepareForVersionAppend( + const ImmutableOptions& immutable_options, + const MutableCFOptions& mutable_cf_options) { + ComputeCompensatedSizes(); + UpdateNumNonEmptyLevels(); + CalculateBaseBytes(immutable_options, mutable_cf_options); + UpdateFilesByCompactionPri(immutable_options, mutable_cf_options); + GenerateFileIndexer(); + GenerateLevelFilesBrief(); + GenerateLevel0NonOverlapping(); + if (!immutable_options.allow_ingest_behind) { + GenerateBottommostFiles(); + } + GenerateFileLocationIndex(); +} + +void Version::PrepareAppend(const MutableCFOptions& mutable_cf_options, + const ReadOptions& read_options, + bool update_stats) { + TEST_SYNC_POINT_CALLBACK( + "Version::PrepareAppend:forced_check", + reinterpret_cast(&storage_info_.force_consistency_checks_)); + + if (update_stats) { + UpdateAccumulatedStats(read_options); + } + + storage_info_.PrepareForVersionAppend(*cfd_->ioptions(), mutable_cf_options); +} + +bool Version::MaybeInitializeFileMetaData(const ReadOptions& read_options, + FileMetaData* file_meta) { + if (file_meta->init_stats_from_file || file_meta->compensated_file_size > 0) { + return false; + } + std::shared_ptr tp; + Status s = GetTableProperties(read_options, &tp, file_meta); + file_meta->init_stats_from_file = true; + if (!s.ok()) { + ROCKS_LOG_ERROR(vset_->db_options_->info_log, + "Unable to load table properties for file %" PRIu64 + " --- %s\n", + file_meta->fd.GetNumber(), s.ToString().c_str()); + return false; + } + if (tp.get() == nullptr) return false; + file_meta->num_entries = tp->num_entries; + file_meta->num_deletions = tp->num_deletions; + file_meta->raw_value_size = tp->raw_value_size; + file_meta->raw_key_size = tp->raw_key_size; + file_meta->num_range_deletions = tp->num_range_deletions; + return true; +} + +void VersionStorageInfo::UpdateAccumulatedStats(FileMetaData* file_meta) { + TEST_SYNC_POINT_CALLBACK("VersionStorageInfo::UpdateAccumulatedStats", + nullptr); + + assert(file_meta->init_stats_from_file); + accumulated_file_size_ += file_meta->fd.GetFileSize(); + accumulated_raw_key_size_ += file_meta->raw_key_size; + accumulated_raw_value_size_ += file_meta->raw_value_size; + accumulated_num_non_deletions_ += + file_meta->num_entries - file_meta->num_deletions; + accumulated_num_deletions_ += file_meta->num_deletions; + + current_num_non_deletions_ += + file_meta->num_entries - file_meta->num_deletions; + current_num_deletions_ += file_meta->num_deletions; + current_num_samples_++; +} + +void VersionStorageInfo::RemoveCurrentStats(FileMetaData* file_meta) { + if (file_meta->init_stats_from_file) { + current_num_non_deletions_ -= + file_meta->num_entries - file_meta->num_deletions; + current_num_deletions_ -= file_meta->num_deletions; + current_num_samples_--; + } +} + +void Version::UpdateAccumulatedStats(const ReadOptions& read_options) { + // maximum number of table properties loaded from files. + const int kMaxInitCount = 20; + int init_count = 0; + // here only the first kMaxInitCount files which haven't been + // initialized from file will be updated with num_deletions. + // The motivation here is to cap the maximum I/O per Version creation. + // The reason for choosing files from lower-level instead of higher-level + // is that such design is able to propagate the initialization from + // lower-level to higher-level: When the num_deletions of lower-level + // files are updated, it will make the lower-level files have accurate + // compensated_file_size, making lower-level to higher-level compaction + // will be triggered, which creates higher-level files whose num_deletions + // will be updated here. + for (int level = 0; + level < storage_info_.num_levels_ && init_count < kMaxInitCount; + ++level) { + for (auto* file_meta : storage_info_.files_[level]) { + if (MaybeInitializeFileMetaData(read_options, file_meta)) { + // each FileMeta will be initialized only once. + storage_info_.UpdateAccumulatedStats(file_meta); + // when option "max_open_files" is -1, all the file metadata has + // already been read, so MaybeInitializeFileMetaData() won't incur + // any I/O cost. "max_open_files=-1" means that the table cache passed + // to the VersionSet and then to the ColumnFamilySet has a size of + // TableCache::kInfiniteCapacity + if (vset_->GetColumnFamilySet()->get_table_cache()->GetCapacity() == + TableCache::kInfiniteCapacity) { + continue; + } + if (++init_count >= kMaxInitCount) { + break; + } + } + } + } + // In case all sampled-files contain only deletion entries, then we + // load the table-property of a file in higher-level to initialize + // that value. + for (int level = storage_info_.num_levels_ - 1; + storage_info_.accumulated_raw_value_size_ == 0 && level >= 0; --level) { + for (int i = static_cast(storage_info_.files_[level].size()) - 1; + storage_info_.accumulated_raw_value_size_ == 0 && i >= 0; --i) { + if (MaybeInitializeFileMetaData(read_options, + storage_info_.files_[level][i])) { + storage_info_.UpdateAccumulatedStats(storage_info_.files_[level][i]); + } + } + } +} + +void VersionStorageInfo::ComputeCompensatedSizes() { + static const int kDeletionWeightOnCompaction = 2; + uint64_t average_value_size = GetAverageValueSize(); + + // compute the compensated size + for (int level = 0; level < num_levels_; level++) { + for (auto* file_meta : files_[level]) { + // Here we only compute compensated_file_size for those file_meta + // which compensated_file_size is uninitialized (== 0). This is true only + // for files that have been created right now and no other thread has + // access to them. That's why we can safely mutate compensated_file_size. + if (file_meta->compensated_file_size == 0) { + file_meta->compensated_file_size = file_meta->fd.GetFileSize(); + // Here we only boost the size of deletion entries of a file only + // when the number of deletion entries is greater than the number of + // non-deletion entries in the file. The motivation here is that in + // a stable workload, the number of deletion entries should be roughly + // equal to the number of non-deletion entries. If we compensate the + // size of deletion entries in a stable workload, the deletion + // compensation logic might introduce unwanted effet which changes the + // shape of LSM tree. + if ((file_meta->num_deletions - file_meta->num_range_deletions) * 2 >= + file_meta->num_entries) { + file_meta->compensated_file_size += + ((file_meta->num_deletions - file_meta->num_range_deletions) * 2 - + file_meta->num_entries) * + average_value_size * kDeletionWeightOnCompaction; + } + file_meta->compensated_file_size += + file_meta->compensated_range_deletion_size; + } + } + } +} + +int VersionStorageInfo::MaxInputLevel() const { + if (compaction_style_ == kCompactionStyleLevel) { + return num_levels() - 2; + } + return 0; +} + +int VersionStorageInfo::MaxOutputLevel(bool allow_ingest_behind) const { + if (allow_ingest_behind) { + assert(num_levels() > 1); + return num_levels() - 2; + } + return num_levels() - 1; +} + +void VersionStorageInfo::EstimateCompactionBytesNeeded( + const MutableCFOptions& mutable_cf_options) { + // Only implemented for level-based compaction + if (compaction_style_ != kCompactionStyleLevel) { + estimated_compaction_needed_bytes_ = 0; + return; + } + + // Start from Level 0, if level 0 qualifies compaction to level 1, + // we estimate the size of compaction. + // Then we move on to the next level and see whether it qualifies compaction + // to the next level. The size of the level is estimated as the actual size + // on the level plus the input bytes from the previous level if there is any. + // If it exceeds, take the exceeded bytes as compaction input and add the size + // of the compaction size to tatal size. + // We keep doing it to Level 2, 3, etc, until the last level and return the + // accumulated bytes. + + uint64_t bytes_compact_to_next_level = 0; + uint64_t level_size = 0; + for (auto* f : files_[0]) { + level_size += f->fd.GetFileSize(); + } + // Level 0 + bool level0_compact_triggered = false; + if (static_cast(files_[0].size()) >= + mutable_cf_options.level0_file_num_compaction_trigger || + level_size >= mutable_cf_options.max_bytes_for_level_base) { + level0_compact_triggered = true; + estimated_compaction_needed_bytes_ = level_size; + bytes_compact_to_next_level = level_size; + } else { + estimated_compaction_needed_bytes_ = 0; + } + + // Level 1 and up. + uint64_t bytes_next_level = 0; + for (int level = base_level(); level <= MaxInputLevel(); level++) { + level_size = 0; + if (bytes_next_level > 0) { +#ifndef NDEBUG + uint64_t level_size2 = 0; + for (auto* f : files_[level]) { + level_size2 += f->fd.GetFileSize(); + } + assert(level_size2 == bytes_next_level); +#endif + level_size = bytes_next_level; + bytes_next_level = 0; + } else { + for (auto* f : files_[level]) { + level_size += f->fd.GetFileSize(); + } + } + if (level == base_level() && level0_compact_triggered) { + // Add base level size to compaction if level0 compaction triggered. + estimated_compaction_needed_bytes_ += level_size; + } + // Add size added by previous compaction + level_size += bytes_compact_to_next_level; + bytes_compact_to_next_level = 0; + uint64_t level_target = MaxBytesForLevel(level); + if (level_size > level_target) { + bytes_compact_to_next_level = level_size - level_target; + // Estimate the actual compaction fan-out ratio as size ratio between + // the two levels. + + assert(bytes_next_level == 0); + if (level + 1 < num_levels_) { + for (auto* f : files_[level + 1]) { + bytes_next_level += f->fd.GetFileSize(); + } + } + if (bytes_next_level > 0) { + assert(level_size > 0); + estimated_compaction_needed_bytes_ += static_cast( + static_cast(bytes_compact_to_next_level) * + (static_cast(bytes_next_level) / + static_cast(level_size) + + 1)); + } + } + } +} + +namespace { +uint32_t GetExpiredTtlFilesCount(const ImmutableOptions& ioptions, + const MutableCFOptions& mutable_cf_options, + const std::vector& files) { + uint32_t ttl_expired_files_count = 0; + + int64_t _current_time; + auto status = ioptions.clock->GetCurrentTime(&_current_time); + if (status.ok()) { + const uint64_t current_time = static_cast(_current_time); + for (FileMetaData* f : files) { + if (!f->being_compacted) { + uint64_t oldest_ancester_time = f->TryGetOldestAncesterTime(); + if (oldest_ancester_time != 0 && + oldest_ancester_time < (current_time - mutable_cf_options.ttl)) { + ttl_expired_files_count++; + } + } + } + } + return ttl_expired_files_count; +} + +bool ShouldChangeFileTemperature(const ImmutableOptions& ioptions, + const MutableCFOptions& mutable_cf_options, + const std::vector& files) { + const std::vector& ages = + mutable_cf_options.compaction_options_fifo + .file_temperature_age_thresholds; + if (ages.empty()) { + return false; + } + if (files.empty()) { + return false; + } + int64_t _current_time; + auto status = ioptions.clock->GetCurrentTime(&_current_time); + const uint64_t current_time = static_cast(_current_time); + // We use oldest_ancestor_time of a file to be the estimate age of + // the file just older than it. This is the same logic used in + // FIFOCompactionPicker::PickTemperatureChangeCompaction(). + if (status.ok() && current_time >= ages[0].age) { + uint64_t create_time_threshold = current_time - ages[0].age; + Temperature target_temp; + assert(files.size() >= 1); + for (size_t index = files.size() - 1; index >= 1; --index) { + FileMetaData* cur_file = files[index]; + FileMetaData* prev_file = files[index - 1]; + if (!cur_file->being_compacted) { + uint64_t oldest_ancestor_time = prev_file->TryGetOldestAncesterTime(); + if (oldest_ancestor_time == kUnknownOldestAncesterTime) { + return false; + } + if (oldest_ancestor_time > create_time_threshold) { + return false; + } + target_temp = ages[0].temperature; + for (size_t i = 1; i < ages.size(); ++i) { + if (current_time >= ages[i].age && + oldest_ancestor_time <= current_time - ages[i].age) { + target_temp = ages[i].temperature; + } + } + if (cur_file->temperature != target_temp) { + return true; + } + } + } + } + return false; +} +} // anonymous namespace + +void VersionStorageInfo::ComputeCompactionScore( + const ImmutableOptions& immutable_options, + const MutableCFOptions& mutable_cf_options) { + double total_downcompact_bytes = 0.0; + // Historically, score is defined as actual bytes in a level divided by + // the level's target size, and 1.0 is the threshold for triggering + // compaction. Higher score means higher prioritization. + // Now we keep the compaction triggering condition, but consider more + // factors for prioritization, while still keeping the 1.0 threshold. + // In order to provide flexibility for reducing score while still + // maintaining it to be over 1.0, we scale the original score by 10x + // if it is larger than 1.0. + const double kScoreScale = 10.0; + int max_output_level = MaxOutputLevel(immutable_options.allow_ingest_behind); + for (int level = 0; level <= MaxInputLevel(); level++) { + double score; + if (level == 0) { + // We treat level-0 specially by bounding the number of files + // instead of number of bytes for two reasons: + // + // (1) With larger write-buffer sizes, it is nice not to do too + // many level-0 compactions. + // + // (2) The files in level-0 are merged on every read and + // therefore we wish to avoid too many files when the individual + // file size is small (perhaps because of a small write-buffer + // setting, or very high compression ratios, or lots of + // overwrites/deletions). + int num_sorted_runs = 0; + uint64_t total_size = 0; + for (auto* f : files_[level]) { + total_downcompact_bytes += static_cast(f->fd.GetFileSize()); + if (!f->being_compacted) { + total_size += f->compensated_file_size; + num_sorted_runs++; + } + } + if (compaction_style_ == kCompactionStyleUniversal) { + // For universal compaction, we use level0 score to indicate + // compaction score for the whole DB. Adding other levels as if + // they are L0 files. + for (int i = 1; i <= max_output_level; i++) { + // It's possible that a subset of the files in a level may be in a + // compaction, due to delete triggered compaction or trivial move. + // In that case, the below check may not catch a level being + // compacted as it only checks the first file. The worst that can + // happen is a scheduled compaction thread will find nothing to do. + if (!files_[i].empty() && !files_[i][0]->being_compacted) { + num_sorted_runs++; + } + } + } + + if (compaction_style_ == kCompactionStyleFIFO) { + score = static_cast(total_size) / + mutable_cf_options.compaction_options_fifo.max_table_files_size; + if (score < 1 && + mutable_cf_options.compaction_options_fifo.allow_compaction) { + score = std::max( + static_cast(num_sorted_runs) / + mutable_cf_options.level0_file_num_compaction_trigger, + score); + } + if (score < 1 && mutable_cf_options.ttl > 0) { + score = + std::max(static_cast(GetExpiredTtlFilesCount( + immutable_options, mutable_cf_options, files_[0])), + score); + } + if (score < 1 && + ShouldChangeFileTemperature(immutable_options, mutable_cf_options, + files_[0])) { + // For FIFO, just need a large enough score to trigger compaction. + const double kScoreForNeedCompaction = 1.1; + score = kScoreForNeedCompaction; + } + } else { + score = static_cast(num_sorted_runs) / + mutable_cf_options.level0_file_num_compaction_trigger; + if (compaction_style_ == kCompactionStyleLevel && num_levels() > 1) { + // Level-based involves L0->L0 compactions that can lead to oversized + // L0 files. Take into account size as well to avoid later giant + // compactions to the base level. + // If score in L0 is always too high, L0->LBase will always be + // prioritized over LBase->LBase+1 compaction and LBase will + // accumulate to too large. But if L0 score isn't high enough, L0 will + // accumulate and data is not moved to LBase fast enough. The score + // calculation below takes into account L0 size vs LBase size. + if (immutable_options.level_compaction_dynamic_level_bytes) { + if (total_size >= mutable_cf_options.max_bytes_for_level_base) { + // When calculating estimated_compaction_needed_bytes, we assume + // L0 is qualified as pending compactions. We will need to make + // sure that it qualifies for compaction. + // It might be guaranteed by logic below anyway, but we are + // explicit here to make sure we don't stop writes with no + // compaction scheduled. + score = std::max(score, 1.01); + } + if (total_size > level_max_bytes_[base_level_]) { + // In this case, we compare L0 size with actual LBase size and + // make sure score is more than 1.0 (10.0 after scaled) if L0 is + // larger than LBase. Since LBase score = LBase size / + // (target size + total_downcompact_bytes) where + // total_downcompact_bytes = total_size > LBase size, + // LBase score is lower than 10.0. So L0->LBase is prioritized + // over LBase -> LBase+1. + uint64_t base_level_size = 0; + for (auto f : files_[base_level_]) { + base_level_size += f->compensated_file_size; + } + score = std::max(score, static_cast(total_size) / + static_cast(std::max( + base_level_size, + level_max_bytes_[base_level_]))); + } + if (score > 1.0) { + score *= kScoreScale; + } + } else { + score = std::max(score, + static_cast(total_size) / + mutable_cf_options.max_bytes_for_level_base); + } + } + } + } else { // level > 0 + // Compute the ratio of current size to size limit. + uint64_t level_bytes_no_compacting = 0; + uint64_t level_total_bytes = 0; + for (auto f : files_[level]) { + level_total_bytes += f->fd.GetFileSize(); + if (!f->being_compacted) { + level_bytes_no_compacting += f->compensated_file_size; + } + } + if (!immutable_options.level_compaction_dynamic_level_bytes) { + score = static_cast(level_bytes_no_compacting) / + MaxBytesForLevel(level); + } else { + if (level_bytes_no_compacting < MaxBytesForLevel(level)) { + score = static_cast(level_bytes_no_compacting) / + MaxBytesForLevel(level); + } else { + // If there are a large mount of data being compacted down to the + // current level soon, we would de-prioritize compaction from + // a level where the incoming data would be a large ratio. We do + // it by dividing level size not by target level size, but + // the target size and the incoming compaction bytes. + score = static_cast(level_bytes_no_compacting) / + (MaxBytesForLevel(level) + total_downcompact_bytes) * + kScoreScale; + } + // Drain unnecessary levels, but with lower priority compared to + // when L0 is eligible. Only non-empty levels can be unnecessary. + // If there is no unnecessary levels, lowest_unnecessary_level_ = -1. + if (level_bytes_no_compacting > 0 && + level <= lowest_unnecessary_level_) { + score = std::max( + score, kScoreScale * + (1.001 + 0.001 * (lowest_unnecessary_level_ - level))); + } + } + if (level <= lowest_unnecessary_level_) { + total_downcompact_bytes += level_total_bytes; + } else if (level_total_bytes > MaxBytesForLevel(level)) { + total_downcompact_bytes += + static_cast(level_total_bytes - MaxBytesForLevel(level)); + } + } + compaction_level_[level] = level; + compaction_score_[level] = score; + } + + // sort all the levels based on their score. Higher scores get listed + // first. Use bubble sort because the number of entries are small. + for (int i = 0; i < num_levels() - 2; i++) { + for (int j = i + 1; j < num_levels() - 1; j++) { + if (compaction_score_[i] < compaction_score_[j]) { + double score = compaction_score_[i]; + int level = compaction_level_[i]; + compaction_score_[i] = compaction_score_[j]; + compaction_level_[i] = compaction_level_[j]; + compaction_score_[j] = score; + compaction_level_[j] = level; + } + } + } + ComputeFilesMarkedForCompaction(max_output_level); + if (!immutable_options.allow_ingest_behind) { + ComputeBottommostFilesMarkedForCompaction(); + } + if (mutable_cf_options.ttl > 0 && + compaction_style_ == kCompactionStyleLevel) { + ComputeExpiredTtlFiles(immutable_options, mutable_cf_options.ttl); + } + if (mutable_cf_options.periodic_compaction_seconds > 0) { + ComputeFilesMarkedForPeriodicCompaction( + immutable_options, mutable_cf_options.periodic_compaction_seconds, + max_output_level); + } + + if (mutable_cf_options.enable_blob_garbage_collection && + mutable_cf_options.blob_garbage_collection_age_cutoff > 0.0 && + mutable_cf_options.blob_garbage_collection_force_threshold < 1.0) { + ComputeFilesMarkedForForcedBlobGC( + mutable_cf_options.blob_garbage_collection_age_cutoff, + mutable_cf_options.blob_garbage_collection_force_threshold); + } + + EstimateCompactionBytesNeeded(mutable_cf_options); +} + +void VersionStorageInfo::ComputeFilesMarkedForCompaction(int last_level) { + files_marked_for_compaction_.clear(); + int last_qualify_level = 0; + + // Do not include files from the last level with data + // If table properties collector suggests a file on the last level, + // we should not move it to a new level. + for (int level = last_level; level >= 1; level--) { + if (!files_[level].empty()) { + last_qualify_level = level - 1; + break; + } + } + + for (int level = 0; level <= last_qualify_level; level++) { + for (auto* f : files_[level]) { + if (!f->being_compacted && f->marked_for_compaction) { + files_marked_for_compaction_.emplace_back(level, f); + } + } + } +} + +void VersionStorageInfo::ComputeExpiredTtlFiles( + const ImmutableOptions& ioptions, const uint64_t ttl) { + assert(ttl > 0); + + expired_ttl_files_.clear(); + + int64_t _current_time; + auto status = ioptions.clock->GetCurrentTime(&_current_time); + if (!status.ok()) { + return; + } + const uint64_t current_time = static_cast(_current_time); + + for (int level = 0; level < num_levels() - 1; level++) { + for (FileMetaData* f : files_[level]) { + if (!f->being_compacted) { + uint64_t oldest_ancester_time = f->TryGetOldestAncesterTime(); + if (oldest_ancester_time > 0 && + oldest_ancester_time < (current_time - ttl)) { + expired_ttl_files_.emplace_back(level, f); + } + } + } + } +} + +void VersionStorageInfo::ComputeFilesMarkedForPeriodicCompaction( + const ImmutableOptions& ioptions, + const uint64_t periodic_compaction_seconds, int last_level) { + assert(periodic_compaction_seconds > 0); + + files_marked_for_periodic_compaction_.clear(); + + int64_t temp_current_time; + auto status = ioptions.clock->GetCurrentTime(&temp_current_time); + if (!status.ok()) { + return; + } + const uint64_t current_time = static_cast(temp_current_time); + + // If periodic_compaction_seconds is larger than current time, periodic + // compaction can't possibly be triggered. + if (periodic_compaction_seconds > current_time) { + return; + } + + const uint64_t allowed_time_limit = + current_time - periodic_compaction_seconds; + + for (int level = 0; level <= last_level; level++) { + for (auto f : files_[level]) { + if (!f->being_compacted) { + // Compute a file's modification time in the following order: + // 1. Use file_creation_time table property if it is > 0. + // 2. Use creation_time table property if it is > 0. + // 3. Use file's mtime metadata if the above two table properties are 0. + // Don't consider the file at all if the modification time cannot be + // correctly determined based on the above conditions. + uint64_t file_modification_time = f->TryGetFileCreationTime(); + if (file_modification_time == kUnknownFileCreationTime) { + file_modification_time = f->TryGetOldestAncesterTime(); + } + if (file_modification_time == kUnknownOldestAncesterTime) { + auto file_path = TableFileName(ioptions.cf_paths, f->fd.GetNumber(), + f->fd.GetPathId()); + status = ioptions.env->GetFileModificationTime( + file_path, &file_modification_time); + if (!status.ok()) { + ROCKS_LOG_WARN(ioptions.logger, + "Can't get file modification time: %s: %s", + file_path.c_str(), status.ToString().c_str()); + continue; + } + } + if (file_modification_time > 0 && + file_modification_time < allowed_time_limit) { + files_marked_for_periodic_compaction_.emplace_back(level, f); + } + } + } + } +} + +void VersionStorageInfo::ComputeFilesMarkedForForcedBlobGC( + double blob_garbage_collection_age_cutoff, + double blob_garbage_collection_force_threshold) { + files_marked_for_forced_blob_gc_.clear(); + + if (blob_files_.empty()) { + return; + } + + // Number of blob files eligible for GC based on age + const size_t cutoff_count = static_cast( + blob_garbage_collection_age_cutoff * blob_files_.size()); + if (!cutoff_count) { + return; + } + + // Compute the sum of total and garbage bytes over the oldest batch of blob + // files. The oldest batch is defined as the set of blob files which are + // kept alive by the same SSTs as the very oldest one. Here is a toy example. + // Let's assume we have three SSTs 1, 2, and 3, and four blob files 10, 11, + // 12, and 13. Also, let's say SSTs 1 and 2 both rely on blob file 10 and + // potentially some higher-numbered ones, while SST 3 relies on blob file 12 + // and potentially some higher-numbered ones. Then, the SST to oldest blob + // file mapping is as follows: + // + // SST file number Oldest blob file number + // 1 10 + // 2 10 + // 3 12 + // + // This is what the same thing looks like from the blob files' POV. (Note that + // the linked SSTs simply denote the inverse mapping of the above.) + // + // Blob file number Linked SST set + // 10 {1, 2} + // 11 {} + // 12 {3} + // 13 {} + // + // Then, the oldest batch of blob files consists of blob files 10 and 11, + // and we can get rid of them by forcing the compaction of SSTs 1 and 2. + // + // Note that the overall ratio of garbage computed for the batch has to exceed + // blob_garbage_collection_force_threshold and the entire batch has to be + // eligible for GC according to blob_garbage_collection_age_cutoff in order + // for us to schedule any compactions. + const auto& oldest_meta = blob_files_.front(); + assert(oldest_meta); + + const auto& linked_ssts = oldest_meta->GetLinkedSsts(); + assert(!linked_ssts.empty()); + + size_t count = 1; + uint64_t sum_total_blob_bytes = oldest_meta->GetTotalBlobBytes(); + uint64_t sum_garbage_blob_bytes = oldest_meta->GetGarbageBlobBytes(); + + assert(cutoff_count <= blob_files_.size()); + + for (; count < cutoff_count; ++count) { + const auto& meta = blob_files_[count]; + assert(meta); + + if (!meta->GetLinkedSsts().empty()) { + // Found the beginning of the next batch of blob files + break; + } + + sum_total_blob_bytes += meta->GetTotalBlobBytes(); + sum_garbage_blob_bytes += meta->GetGarbageBlobBytes(); + } + + if (count < blob_files_.size()) { + const auto& meta = blob_files_[count]; + assert(meta); + + if (meta->GetLinkedSsts().empty()) { + // Some files in the oldest batch are not eligible for GC + return; + } + } + + if (sum_garbage_blob_bytes < + blob_garbage_collection_force_threshold * sum_total_blob_bytes) { + return; + } + + for (uint64_t sst_file_number : linked_ssts) { + const FileLocation location = GetFileLocation(sst_file_number); + assert(location.IsValid()); + + const int level = location.GetLevel(); + assert(level >= 0); + + const size_t pos = location.GetPosition(); + + FileMetaData* const sst_meta = files_[level][pos]; + assert(sst_meta); + + if (sst_meta->being_compacted) { + continue; + } + + files_marked_for_forced_blob_gc_.emplace_back(level, sst_meta); + } +} + +namespace { + +// used to sort files by size +struct Fsize { + size_t index; + FileMetaData* file; +}; + +// Comparator that is used to sort files based on their size +// In normal mode: descending size +bool CompareCompensatedSizeDescending(const Fsize& first, const Fsize& second) { + return (first.file->compensated_file_size > + second.file->compensated_file_size); +} +} // anonymous namespace + +void VersionStorageInfo::AddFile(int level, FileMetaData* f) { + auto& level_files = files_[level]; + level_files.push_back(f); + + f->refs++; +} + +void VersionStorageInfo::AddBlobFile( + std::shared_ptr blob_file_meta) { + assert(blob_file_meta); + + assert(blob_files_.empty() || + (blob_files_.back() && blob_files_.back()->GetBlobFileNumber() < + blob_file_meta->GetBlobFileNumber())); + + blob_files_.emplace_back(std::move(blob_file_meta)); +} + +VersionStorageInfo::BlobFiles::const_iterator +VersionStorageInfo::GetBlobFileMetaDataLB(uint64_t blob_file_number) const { + return std::lower_bound( + blob_files_.begin(), blob_files_.end(), blob_file_number, + [](const std::shared_ptr& lhs, uint64_t rhs) { + assert(lhs); + return lhs->GetBlobFileNumber() < rhs; + }); +} + +void VersionStorageInfo::SetFinalized() { + finalized_ = true; + +#ifndef NDEBUG + if (compaction_style_ != kCompactionStyleLevel) { + // Not level based compaction. + return; + } + assert(base_level_ < 0 || num_levels() == 1 || + (base_level_ >= 1 && base_level_ < num_levels())); + // Verify all levels newer than base_level are empty except L0 + for (int level = 1; level < base_level(); level++) { + assert(NumLevelBytes(level) == 0); + } + uint64_t max_bytes_prev_level = 0; + for (int level = base_level(); level < num_levels() - 1; level++) { + if (LevelFiles(level).size() == 0) { + continue; + } + assert(MaxBytesForLevel(level) >= max_bytes_prev_level); + max_bytes_prev_level = MaxBytesForLevel(level); + } + for (int level = 0; level < num_levels(); level++) { + assert(LevelFiles(level).size() == 0 || + LevelFiles(level).size() == LevelFilesBrief(level).num_files); + if (LevelFiles(level).size() > 0) { + assert(level < num_non_empty_levels()); + } + } + assert(compaction_level_.size() > 0); + assert(compaction_level_.size() == compaction_score_.size()); +#endif +} + +void VersionStorageInfo::UpdateNumNonEmptyLevels() { + num_non_empty_levels_ = num_levels_; + for (int i = num_levels_ - 1; i >= 0; i--) { + if (files_[i].size() != 0) { + return; + } else { + num_non_empty_levels_ = i; + } + } +} + +namespace { +// Sort `temp` based on ratio of overlapping size over file size +void SortFileByOverlappingRatio( + const InternalKeyComparator& icmp, const std::vector& files, + const std::vector& next_level_files, SystemClock* clock, + int level, int num_non_empty_levels, uint64_t ttl, + std::vector* temp) { + std::unordered_map file_to_order; + auto next_level_it = next_level_files.begin(); + + int64_t curr_time; + Status status = clock->GetCurrentTime(&curr_time); + if (!status.ok()) { + // If we can't get time, disable TTL. + ttl = 0; + } + + FileTtlBooster ttl_booster(static_cast(curr_time), ttl, + num_non_empty_levels, level); + + for (auto& file : files) { + uint64_t overlapping_bytes = 0; + // Skip files in next level that is smaller than current file + while (next_level_it != next_level_files.end() && + icmp.Compare((*next_level_it)->largest, file->smallest) < 0) { + next_level_it++; + } + + while (next_level_it != next_level_files.end() && + icmp.Compare((*next_level_it)->smallest, file->largest) < 0) { + overlapping_bytes += (*next_level_it)->fd.file_size; + + if (icmp.Compare((*next_level_it)->largest, file->largest) > 0) { + // next level file cross large boundary of current file. + break; + } + next_level_it++; + } + + uint64_t ttl_boost_score = (ttl > 0) ? ttl_booster.GetBoostScore(file) : 1; + assert(ttl_boost_score > 0); + assert(file->compensated_file_size != 0); + file_to_order[file->fd.GetNumber()] = overlapping_bytes * 1024U / + file->compensated_file_size / + ttl_boost_score; + } + + size_t num_to_sort = temp->size() > VersionStorageInfo::kNumberFilesToSort + ? VersionStorageInfo::kNumberFilesToSort + : temp->size(); + + std::partial_sort(temp->begin(), temp->begin() + num_to_sort, temp->end(), + [&](const Fsize& f1, const Fsize& f2) -> bool { + // If score is the same, pick file with smaller keys. + // This makes the algorithm more deterministic, and also + // help the trivial move case to have more files to + // extend. + if (file_to_order[f1.file->fd.GetNumber()] == + file_to_order[f2.file->fd.GetNumber()]) { + return icmp.Compare(f1.file->smallest, + f2.file->smallest) < 0; + } + return file_to_order[f1.file->fd.GetNumber()] < + file_to_order[f2.file->fd.GetNumber()]; + }); +} + +void SortFileByRoundRobin(const InternalKeyComparator& icmp, + std::vector* compact_cursor, + bool level0_non_overlapping, int level, + std::vector* temp) { + if (level == 0 && !level0_non_overlapping) { + // Using kOldestSmallestSeqFirst when level === 0, since the + // files may overlap (not fully sorted) + std::sort(temp->begin(), temp->end(), + [](const Fsize& f1, const Fsize& f2) -> bool { + return f1.file->fd.smallest_seqno < f2.file->fd.smallest_seqno; + }); + return; + } + + bool should_move_files = + compact_cursor->at(level).size() > 0 && temp->size() > 1; + + // The iterator points to the Fsize with smallest key larger than or equal to + // the given cursor + std::vector::iterator current_file_iter; + if (should_move_files) { + // Find the file of which the smallest key is larger than or equal to + // the cursor (the smallest key in the successor file of the last + // chosen file), skip this if the cursor is invalid or there is only + // one file in this level + current_file_iter = std::lower_bound( + temp->begin(), temp->end(), compact_cursor->at(level), + [&](const Fsize& f, const InternalKey& cursor) -> bool { + return icmp.Compare(cursor, f.file->smallest) > 0; + }); + + should_move_files = + current_file_iter != temp->end() && current_file_iter != temp->begin(); + } + if (should_move_files) { + // Construct a local temporary vector + std::vector local_temp; + local_temp.reserve(temp->size()); + // Move the selected File into the first position and its successors + // into the second, third, ..., positions + for (auto iter = current_file_iter; iter != temp->end(); iter++) { + local_temp.push_back(*iter); + } + // Move the origin predecessors of the selected file in a round-robin + // manner + for (auto iter = temp->begin(); iter != current_file_iter; iter++) { + local_temp.push_back(*iter); + } + // Replace all the items in temp + for (size_t i = 0; i < local_temp.size(); i++) { + temp->at(i) = local_temp[i]; + } + } +} +} // anonymous namespace + +void VersionStorageInfo::UpdateFilesByCompactionPri( + const ImmutableOptions& ioptions, const MutableCFOptions& options) { + if (compaction_style_ == kCompactionStyleNone || + compaction_style_ == kCompactionStyleFIFO || + compaction_style_ == kCompactionStyleUniversal) { + // don't need this + return; + } + // No need to sort the highest level because it is never compacted. + for (int level = 0; level < num_levels() - 1; level++) { + const std::vector& files = files_[level]; + auto& files_by_compaction_pri = files_by_compaction_pri_[level]; + assert(files_by_compaction_pri.size() == 0); + + // populate a temp vector for sorting based on size + std::vector temp(files.size()); + for (size_t i = 0; i < files.size(); i++) { + temp[i].index = i; + temp[i].file = files[i]; + } + + // sort the top number_of_files_to_sort_ based on file size + size_t num = VersionStorageInfo::kNumberFilesToSort; + if (num > temp.size()) { + num = temp.size(); + } + switch (ioptions.compaction_pri) { + case kByCompensatedSize: + std::partial_sort(temp.begin(), temp.begin() + num, temp.end(), + CompareCompensatedSizeDescending); + break; + case kOldestLargestSeqFirst: + std::sort(temp.begin(), temp.end(), + [](const Fsize& f1, const Fsize& f2) -> bool { + return f1.file->fd.largest_seqno < + f2.file->fd.largest_seqno; + }); + break; + case kOldestSmallestSeqFirst: + std::sort(temp.begin(), temp.end(), + [](const Fsize& f1, const Fsize& f2) -> bool { + return f1.file->fd.smallest_seqno < + f2.file->fd.smallest_seqno; + }); + break; + case kMinOverlappingRatio: + SortFileByOverlappingRatio(*internal_comparator_, files_[level], + files_[level + 1], ioptions.clock, level, + num_non_empty_levels_, options.ttl, &temp); + break; + case kRoundRobin: + SortFileByRoundRobin(*internal_comparator_, &compact_cursor_, + level0_non_overlapping_, level, &temp); + break; + default: + assert(false); + } + assert(temp.size() == files.size()); + + // initialize files_by_compaction_pri_ + for (size_t i = 0; i < temp.size(); i++) { + files_by_compaction_pri.push_back(static_cast(temp[i].index)); + } + next_file_to_compact_by_size_[level] = 0; + assert(files_[level].size() == files_by_compaction_pri_[level].size()); + } +} + +void VersionStorageInfo::GenerateLevel0NonOverlapping() { + assert(!finalized_); + level0_non_overlapping_ = true; + if (level_files_brief_.size() == 0) { + return; + } + + // A copy of L0 files sorted by smallest key + std::vector level0_sorted_file( + level_files_brief_[0].files, + level_files_brief_[0].files + level_files_brief_[0].num_files); + std::sort(level0_sorted_file.begin(), level0_sorted_file.end(), + [this](const FdWithKeyRange& f1, const FdWithKeyRange& f2) -> bool { + return (internal_comparator_->Compare(f1.smallest_key, + f2.smallest_key) < 0); + }); + + for (size_t i = 1; i < level0_sorted_file.size(); ++i) { + FdWithKeyRange& f = level0_sorted_file[i]; + FdWithKeyRange& prev = level0_sorted_file[i - 1]; + if (internal_comparator_->Compare(prev.largest_key, f.smallest_key) >= 0) { + level0_non_overlapping_ = false; + break; + } + } +} + +void VersionStorageInfo::GenerateBottommostFiles() { + assert(!finalized_); + assert(bottommost_files_.empty()); + for (size_t level = 0; level < level_files_brief_.size(); ++level) { + for (size_t file_idx = 0; file_idx < level_files_brief_[level].num_files; + ++file_idx) { + const FdWithKeyRange& f = level_files_brief_[level].files[file_idx]; + int l0_file_idx; + if (level == 0) { + l0_file_idx = static_cast(file_idx); + } else { + l0_file_idx = -1; + } + Slice smallest_user_key = ExtractUserKey(f.smallest_key); + Slice largest_user_key = ExtractUserKey(f.largest_key); + if (!RangeMightExistAfterSortedRun(smallest_user_key, largest_user_key, + static_cast(level), + l0_file_idx)) { + bottommost_files_.emplace_back(static_cast(level), + f.file_metadata); + } + } + } +} + +void VersionStorageInfo::GenerateFileLocationIndex() { + size_t num_files = 0; + + for (int level = 0; level < num_levels_; ++level) { + num_files += files_[level].size(); + } + + file_locations_.reserve(num_files); + + for (int level = 0; level < num_levels_; ++level) { + for (size_t pos = 0; pos < files_[level].size(); ++pos) { + const FileMetaData* const meta = files_[level][pos]; + assert(meta); + + const uint64_t file_number = meta->fd.GetNumber(); + + assert(file_locations_.find(file_number) == file_locations_.end()); + file_locations_.emplace(file_number, FileLocation(level, pos)); + } + } +} + +void VersionStorageInfo::UpdateOldestSnapshot(SequenceNumber seqnum) { + assert(seqnum >= oldest_snapshot_seqnum_); + oldest_snapshot_seqnum_ = seqnum; + if (oldest_snapshot_seqnum_ > bottommost_files_mark_threshold_) { + ComputeBottommostFilesMarkedForCompaction(); + } +} + +void VersionStorageInfo::ComputeBottommostFilesMarkedForCompaction() { + bottommost_files_marked_for_compaction_.clear(); + bottommost_files_mark_threshold_ = kMaxSequenceNumber; + for (auto& level_and_file : bottommost_files_) { + if (!level_and_file.second->being_compacted && + level_and_file.second->fd.largest_seqno != 0) { + // largest_seqno might be nonzero due to containing the final key in an + // earlier compaction, whose seqnum we didn't zero out. Multiple deletions + // ensures the file really contains deleted or overwritten keys. + if (level_and_file.second->fd.largest_seqno < oldest_snapshot_seqnum_) { + bottommost_files_marked_for_compaction_.push_back(level_and_file); + } else { + bottommost_files_mark_threshold_ = + std::min(bottommost_files_mark_threshold_, + level_and_file.second->fd.largest_seqno); + } + } + } +} + +void Version::Ref() { ++refs_; } + +bool Version::Unref() { + assert(refs_ >= 1); + --refs_; + if (refs_ == 0) { + delete this; + return true; + } + return false; +} + +bool VersionStorageInfo::OverlapInLevel(int level, + const Slice* smallest_user_key, + const Slice* largest_user_key) { + if (level >= num_non_empty_levels_) { + // empty level, no overlap + return false; + } + return SomeFileOverlapsRange(*internal_comparator_, (level > 0), + level_files_brief_[level], smallest_user_key, + largest_user_key); +} + +// Store in "*inputs" all files in "level" that overlap [begin,end] +// If hint_index is specified, then it points to a file in the +// overlapping range. +// The file_index returns a pointer to any file in an overlapping range. +void VersionStorageInfo::GetOverlappingInputs( + int level, const InternalKey* begin, const InternalKey* end, + std::vector* inputs, int hint_index, int* file_index, + bool expand_range, InternalKey** next_smallest) const { + if (level >= num_non_empty_levels_) { + // this level is empty, no overlapping inputs + return; + } + + inputs->clear(); + if (file_index) { + *file_index = -1; + } + const Comparator* user_cmp = user_comparator_; + if (level > 0) { + GetOverlappingInputsRangeBinarySearch(level, begin, end, inputs, hint_index, + file_index, false, next_smallest); + return; + } + + if (next_smallest) { + // next_smallest key only makes sense for non-level 0, where files are + // non-overlapping + *next_smallest = nullptr; + } + + Slice user_begin, user_end; + if (begin != nullptr) { + user_begin = begin->user_key(); + } + if (end != nullptr) { + user_end = end->user_key(); + } + + // index stores the file index need to check. + std::list index; + for (size_t i = 0; i < level_files_brief_[level].num_files; i++) { + index.emplace_back(i); + } + + while (!index.empty()) { + bool found_overlapping_file = false; + auto iter = index.begin(); + while (iter != index.end()) { + FdWithKeyRange* f = &(level_files_brief_[level].files[*iter]); + const Slice file_start = ExtractUserKey(f->smallest_key); + const Slice file_limit = ExtractUserKey(f->largest_key); + if (begin != nullptr && + user_cmp->CompareWithoutTimestamp(file_limit, user_begin) < 0) { + // "f" is completely before specified range; skip it + iter++; + } else if (end != nullptr && + user_cmp->CompareWithoutTimestamp(file_start, user_end) > 0) { + // "f" is completely after specified range; skip it + iter++; + } else { + // if overlap + inputs->emplace_back(files_[level][*iter]); + found_overlapping_file = true; + // record the first file index. + if (file_index && *file_index == -1) { + *file_index = static_cast(*iter); + } + // the related file is overlap, erase to avoid checking again. + iter = index.erase(iter); + if (expand_range) { + if (begin != nullptr && + user_cmp->CompareWithoutTimestamp(file_start, user_begin) < 0) { + user_begin = file_start; + } + if (end != nullptr && + user_cmp->CompareWithoutTimestamp(file_limit, user_end) > 0) { + user_end = file_limit; + } + } + } + } + // if all the files left are not overlap, break + if (!found_overlapping_file) { + break; + } + } +} + +// Store in "*inputs" files in "level" that within range [begin,end] +// Guarantee a "clean cut" boundary between the files in inputs +// and the surrounding files and the maxinum number of files. +// This will ensure that no parts of a key are lost during compaction. +// If hint_index is specified, then it points to a file in the range. +// The file_index returns a pointer to any file in an overlapping range. +void VersionStorageInfo::GetCleanInputsWithinInterval( + int level, const InternalKey* begin, const InternalKey* end, + std::vector* inputs, int hint_index, int* file_index) const { + inputs->clear(); + if (file_index) { + *file_index = -1; + } + if (level >= num_non_empty_levels_ || level == 0 || + level_files_brief_[level].num_files == 0) { + // this level is empty, no inputs within range + // also don't support clean input interval within L0 + return; + } + + GetOverlappingInputsRangeBinarySearch(level, begin, end, inputs, hint_index, + file_index, true /* within_interval */); +} + +// Store in "*inputs" all files in "level" that overlap [begin,end] +// Employ binary search to find at least one file that overlaps the +// specified range. From that file, iterate backwards and +// forwards to find all overlapping files. +// if within_range is set, then only store the maximum clean inputs +// within range [begin, end]. "clean" means there is a boundary +// between the files in "*inputs" and the surrounding files +void VersionStorageInfo::GetOverlappingInputsRangeBinarySearch( + int level, const InternalKey* begin, const InternalKey* end, + std::vector* inputs, int hint_index, int* file_index, + bool within_interval, InternalKey** next_smallest) const { + assert(level > 0); + + auto user_cmp = user_comparator_; + const FdWithKeyRange* files = level_files_brief_[level].files; + const int num_files = static_cast(level_files_brief_[level].num_files); + + // begin to use binary search to find lower bound + // and upper bound. + int start_index = 0; + int end_index = num_files; + + if (begin != nullptr) { + // if within_interval is true, with file_key would find + // not overlapping ranges in std::lower_bound. + auto cmp = [&user_cmp, &within_interval](const FdWithKeyRange& f, + const InternalKey* k) { + auto& file_key = within_interval ? f.file_metadata->smallest + : f.file_metadata->largest; + return sstableKeyCompare(user_cmp, file_key, *k) < 0; + }; + + start_index = static_cast( + std::lower_bound(files, + files + (hint_index == -1 ? num_files : hint_index), + begin, cmp) - + files); + + if (start_index > 0 && within_interval) { + bool is_overlapping = true; + while (is_overlapping && start_index < num_files) { + auto& pre_limit = files[start_index - 1].file_metadata->largest; + auto& cur_start = files[start_index].file_metadata->smallest; + is_overlapping = sstableKeyCompare(user_cmp, pre_limit, cur_start) == 0; + start_index += is_overlapping; + } + } + } + + if (end != nullptr) { + // if within_interval is true, with file_key would find + // not overlapping ranges in std::upper_bound. + auto cmp = [&user_cmp, &within_interval](const InternalKey* k, + const FdWithKeyRange& f) { + auto& file_key = within_interval ? f.file_metadata->largest + : f.file_metadata->smallest; + return sstableKeyCompare(user_cmp, *k, file_key) < 0; + }; + + end_index = static_cast( + std::upper_bound(files + start_index, files + num_files, end, cmp) - + files); + + if (end_index < num_files && within_interval) { + bool is_overlapping = true; + while (is_overlapping && end_index > start_index) { + auto& next_start = files[end_index].file_metadata->smallest; + auto& cur_limit = files[end_index - 1].file_metadata->largest; + is_overlapping = + sstableKeyCompare(user_cmp, cur_limit, next_start) == 0; + end_index -= is_overlapping; + } + } + } + + assert(start_index <= end_index); + + // If there were no overlapping files, return immediately. + if (start_index == end_index) { + if (next_smallest) { + *next_smallest = nullptr; + } + return; + } + + assert(start_index < end_index); + + // returns the index where an overlap is found + if (file_index) { + *file_index = start_index; + } + + // insert overlapping files into vector + for (int i = start_index; i < end_index; i++) { + inputs->push_back(files_[level][i]); + } + + if (next_smallest != nullptr) { + // Provide the next key outside the range covered by inputs + if (end_index < static_cast(files_[level].size())) { + **next_smallest = files_[level][end_index]->smallest; + } else { + *next_smallest = nullptr; + } + } +} + +uint64_t VersionStorageInfo::NumLevelBytes(int level) const { + assert(level >= 0); + assert(level < num_levels()); + return TotalFileSize(files_[level]); +} + +const char* VersionStorageInfo::LevelSummary( + LevelSummaryStorage* scratch) const { + int len = 0; + if (compaction_style_ == kCompactionStyleLevel && num_levels() > 1) { + assert(base_level_ < static_cast(level_max_bytes_.size())); + if (level_multiplier_ != 0.0) { + len = snprintf( + scratch->buffer, sizeof(scratch->buffer), + "base level %d level multiplier %.2f max bytes base %" PRIu64 " ", + base_level_, level_multiplier_, level_max_bytes_[base_level_]); + } + } + len += + snprintf(scratch->buffer + len, sizeof(scratch->buffer) - len, "files["); + for (int i = 0; i < num_levels(); i++) { + int sz = sizeof(scratch->buffer) - len; + int ret = snprintf(scratch->buffer + len, sz, "%d ", int(files_[i].size())); + if (ret < 0 || ret >= sz) break; + len += ret; + } + if (len > 0) { + // overwrite the last space + --len; + } + len += snprintf(scratch->buffer + len, sizeof(scratch->buffer) - len, + "] max score %.2f", compaction_score_[0]); + + if (!files_marked_for_compaction_.empty()) { + snprintf(scratch->buffer + len, sizeof(scratch->buffer) - len, + " (%" ROCKSDB_PRIszt " files need compaction)", + files_marked_for_compaction_.size()); + } + + return scratch->buffer; +} + +const char* VersionStorageInfo::LevelFileSummary(FileSummaryStorage* scratch, + int level) const { + int len = snprintf(scratch->buffer, sizeof(scratch->buffer), "files_size["); + for (const auto& f : files_[level]) { + int sz = sizeof(scratch->buffer) - len; + char sztxt[16]; + AppendHumanBytes(f->fd.GetFileSize(), sztxt, sizeof(sztxt)); + int ret = snprintf(scratch->buffer + len, sz, + "#%" PRIu64 "(seq=%" PRIu64 ",sz=%s,%d) ", + f->fd.GetNumber(), f->fd.smallest_seqno, sztxt, + static_cast(f->being_compacted)); + if (ret < 0 || ret >= sz) break; + len += ret; + } + // overwrite the last space (only if files_[level].size() is non-zero) + if (files_[level].size() && len > 0) { + --len; + } + snprintf(scratch->buffer + len, sizeof(scratch->buffer) - len, "]"); + return scratch->buffer; +} + +bool VersionStorageInfo::HasMissingEpochNumber() const { + for (int level = 0; level < num_levels_; ++level) { + for (const FileMetaData* f : files_[level]) { + if (f->epoch_number == kUnknownEpochNumber) { + return true; + } + } + } + return false; +} + +uint64_t VersionStorageInfo::GetMaxEpochNumberOfFiles() const { + uint64_t max_epoch_number = kUnknownEpochNumber; + for (int level = 0; level < num_levels_; ++level) { + for (const FileMetaData* f : files_[level]) { + max_epoch_number = std::max(max_epoch_number, f->epoch_number); + } + } + return max_epoch_number; +} + +void VersionStorageInfo::RecoverEpochNumbers(ColumnFamilyData* cfd) { + cfd->ResetNextEpochNumber(); + + bool reserve_epoch_num_for_file_ingested_behind = + cfd->ioptions()->allow_ingest_behind; + if (reserve_epoch_num_for_file_ingested_behind) { + uint64_t reserved_epoch_number = cfd->NewEpochNumber(); + assert(reserved_epoch_number == kReservedEpochNumberForFileIngestedBehind); + ROCKS_LOG_INFO(cfd->ioptions()->info_log.get(), + "[%s]CF has reserved epoch number %" PRIu64 + " for files ingested " + "behind since `Options::allow_ingest_behind` is true", + cfd->GetName().c_str(), reserved_epoch_number); + } + + if (HasMissingEpochNumber()) { + assert(epoch_number_requirement_ == EpochNumberRequirement::kMightMissing); + assert(num_levels_ >= 1); + + for (int level = num_levels_ - 1; level >= 1; --level) { + auto& files_at_level = files_[level]; + if (files_at_level.empty()) { + continue; + } + uint64_t next_epoch_number = cfd->NewEpochNumber(); + for (FileMetaData* f : files_at_level) { + f->epoch_number = next_epoch_number; + } + } + + for (auto file_meta_iter = files_[0].rbegin(); + file_meta_iter != files_[0].rend(); file_meta_iter++) { + FileMetaData* f = *file_meta_iter; + f->epoch_number = cfd->NewEpochNumber(); + } + + ROCKS_LOG_WARN(cfd->ioptions()->info_log.get(), + "[%s]CF's epoch numbers are inferred based on seqno", + cfd->GetName().c_str()); + epoch_number_requirement_ = EpochNumberRequirement::kMustPresent; + } else { + assert(epoch_number_requirement_ == EpochNumberRequirement::kMustPresent); + cfd->SetNextEpochNumber( + std::max(GetMaxEpochNumberOfFiles() + 1, cfd->GetNextEpochNumber())); + } +} + +uint64_t VersionStorageInfo::MaxNextLevelOverlappingBytes() { + uint64_t result = 0; + std::vector overlaps; + for (int level = 1; level < num_levels() - 1; level++) { + for (const auto& f : files_[level]) { + GetOverlappingInputs(level + 1, &f->smallest, &f->largest, &overlaps); + const uint64_t sum = TotalFileSize(overlaps); + if (sum > result) { + result = sum; + } + } + } + return result; +} + +uint64_t VersionStorageInfo::MaxBytesForLevel(int level) const { + // Note: the result for level zero is not really used since we set + // the level-0 compaction threshold based on number of files. + assert(level >= 0); + assert(level < static_cast(level_max_bytes_.size())); + return level_max_bytes_[level]; +} + +void VersionStorageInfo::CalculateBaseBytes(const ImmutableOptions& ioptions, + const MutableCFOptions& options) { + // Special logic to set number of sorted runs. + // It is to match the previous behavior when all files are in L0. + int num_l0_count = static_cast(files_[0].size()); + if (compaction_style_ == kCompactionStyleUniversal) { + // For universal compaction, we use level0 score to indicate + // compaction score for the whole DB. Adding other levels as if + // they are L0 files. + for (int i = 1; i < num_levels(); i++) { + if (!files_[i].empty()) { + num_l0_count++; + } + } + } + set_l0_delay_trigger_count(num_l0_count); + + level_max_bytes_.resize(ioptions.num_levels); + if (!ioptions.level_compaction_dynamic_level_bytes) { + base_level_ = (ioptions.compaction_style == kCompactionStyleLevel) ? 1 : -1; + + // Calculate for static bytes base case + for (int i = 0; i < ioptions.num_levels; ++i) { + if (i == 0 && ioptions.compaction_style == kCompactionStyleUniversal) { + level_max_bytes_[i] = options.max_bytes_for_level_base; + } else if (i > 1) { + level_max_bytes_[i] = MultiplyCheckOverflow( + MultiplyCheckOverflow(level_max_bytes_[i - 1], + options.max_bytes_for_level_multiplier), + options.MaxBytesMultiplerAdditional(i - 1)); + } else { + level_max_bytes_[i] = options.max_bytes_for_level_base; + } + } + } else { + assert(ioptions.compaction_style == kCompactionStyleLevel); + uint64_t max_level_size = 0; + + int first_non_empty_level = -1; + // Find size of non-L0 level of most data. + // Cannot use the size of the last level because it can be empty or less + // than previous levels after compaction. + for (int i = 1; i < num_levels_; i++) { + uint64_t total_size = 0; + for (const auto& f : files_[i]) { + total_size += f->fd.GetFileSize(); + } + if (total_size > 0 && first_non_empty_level == -1) { + first_non_empty_level = i; + } + if (total_size > max_level_size) { + max_level_size = total_size; + } + } + + // Prefill every level's max bytes to disallow compaction from there. + for (int i = 0; i < num_levels_; i++) { + level_max_bytes_[i] = std::numeric_limits::max(); + } + + lowest_unnecessary_level_ = -1; + if (max_level_size == 0) { + // No data for L1 and up. L0 compacts to last level directly. + // No compaction from L1+ needs to be scheduled. + base_level_ = num_levels_ - 1; + } else { + assert(first_non_empty_level >= 1); + uint64_t base_bytes_max = options.max_bytes_for_level_base; + uint64_t base_bytes_min = static_cast( + base_bytes_max / options.max_bytes_for_level_multiplier); + + // Try whether we can make last level's target size to be max_level_size + uint64_t cur_level_size = max_level_size; + for (int i = num_levels_ - 2; i >= first_non_empty_level; i--) { + // Round up after dividing + cur_level_size = static_cast( + cur_level_size / options.max_bytes_for_level_multiplier); + if (lowest_unnecessary_level_ == -1 && + cur_level_size <= base_bytes_min && + (ioptions.preclude_last_level_data_seconds == 0 || + i < num_levels_ - 2)) { + // When per_key_placement is enabled, the penultimate level is + // necessary. + lowest_unnecessary_level_ = i; + } + } + + // Calculate base level and its size. + uint64_t base_level_size; + if (cur_level_size <= base_bytes_min) { + // If per_key_placement is not enabled, + // either there is only one non-empty level after level 0, + // which can less than base_bytes_min AND necessary, + // or there is some unnecessary level. + assert(first_non_empty_level == num_levels_ - 1 || + ioptions.preclude_last_level_data_seconds > 0 || + lowest_unnecessary_level_ != -1); + // Case 1. If we make target size of last level to be max_level_size, + // target size of the first non-empty level would be smaller than + // base_bytes_min. We set it be base_bytes_min. + base_level_size = base_bytes_min + 1U; + base_level_ = first_non_empty_level; + if (base_level_ < num_levels_ - 1) { + ROCKS_LOG_INFO( + ioptions.logger, + "More existing levels in DB than needed: all non-zero " + "levels <= level %d are unnecessary. " + "max_bytes_for_level_multiplier may not be guaranteed.", + lowest_unnecessary_level_); + } + } else { + assert(lowest_unnecessary_level_ == -1); + // Find base level (where L0 data is compacted to). + base_level_ = first_non_empty_level; + while (base_level_ > 1 && cur_level_size > base_bytes_max) { + --base_level_; + cur_level_size = static_cast( + cur_level_size / options.max_bytes_for_level_multiplier); + } + if (cur_level_size > base_bytes_max) { + // Even L1 will be too large + assert(base_level_ == 1); + base_level_size = base_bytes_max; + } else { + base_level_size = std::max(static_cast(1), cur_level_size); + } + } + + level_multiplier_ = options.max_bytes_for_level_multiplier; + assert(base_level_size > 0); + + uint64_t level_size = base_level_size; + for (int i = base_level_; i < num_levels_; i++) { + if (i > base_level_) { + level_size = MultiplyCheckOverflow(level_size, level_multiplier_); + } + // Don't set any level below base_bytes_max. Otherwise, the LSM can + // assume an hourglass shape where L1+ sizes are smaller than L0. This + // causes compaction scoring, which depends on level sizes, to favor L1+ + // at the expense of L0, which may fill up and stall. + level_max_bytes_[i] = std::max(level_size, base_bytes_max); + } + } + } +} + +uint64_t VersionStorageInfo::EstimateLiveDataSize() const { + // Estimate the live data size by adding up the size of a maximal set of + // sst files with no range overlap in same or higher level. The less + // compacted, the more optimistic (smaller) this estimate is. Also, + // for multiple sorted runs within a level, file order will matter. + uint64_t size = 0; + + auto ikey_lt = [this](InternalKey* x, InternalKey* y) { + return internal_comparator_->Compare(*x, *y) < 0; + }; + // (Ordered) map of largest keys in files being included in size estimate + std::map ranges(ikey_lt); + + for (int l = num_levels_ - 1; l >= 0; l--) { + bool found_end = false; + for (auto file : files_[l]) { + // Find the first file already included with largest key is larger than + // the smallest key of `file`. If that file does not overlap with the + // current file, none of the files in the map does. If there is + // no potential overlap, we can safely insert the rest of this level + // (if the level is not 0) into the map without checking again because + // the elements in the level are sorted and non-overlapping. + auto lb = (found_end && l != 0) ? ranges.end() + : ranges.lower_bound(&file->smallest); + found_end = (lb == ranges.end()); + if (found_end || internal_comparator_->Compare( + file->largest, (*lb).second->smallest) < 0) { + ranges.emplace_hint(lb, &file->largest, file); + size += file->fd.file_size; + } + } + } + + // For BlobDB, the result also includes the exact value of live bytes in the + // blob files of the version. + for (const auto& meta : blob_files_) { + assert(meta); + + size += meta->GetTotalBlobBytes(); + size -= meta->GetGarbageBlobBytes(); + } + + return size; +} + +bool VersionStorageInfo::RangeMightExistAfterSortedRun( + const Slice& smallest_user_key, const Slice& largest_user_key, + int last_level, int last_l0_idx) { + assert((last_l0_idx != -1) == (last_level == 0)); + // TODO(ajkr): this preserves earlier behavior where we considered an L0 file + // bottommost only if it's the oldest L0 file and there are no files on older + // levels. It'd be better to consider it bottommost if there's no overlap in + // older levels/files. + if (last_level == 0 && + last_l0_idx != static_cast(LevelFiles(0).size() - 1)) { + return true; + } + + // Checks whether there are files living beyond the `last_level`. If lower + // levels have files, it checks for overlap between [`smallest_key`, + // `largest_key`] and those files. Bottomlevel optimizations can be made if + // there are no files in lower levels or if there is no overlap with the files + // in the lower levels. + for (int level = last_level + 1; level < num_levels(); level++) { + // The range is not in the bottommost level if there are files in lower + // levels when the `last_level` is 0 or if there are files in lower levels + // which overlap with [`smallest_key`, `largest_key`]. + if (files_[level].size() > 0 && + (last_level == 0 || + OverlapInLevel(level, &smallest_user_key, &largest_user_key))) { + return true; + } + } + return false; +} + +void Version::AddLiveFiles(std::vector* live_table_files, + std::vector* live_blob_files) const { + assert(live_table_files); + assert(live_blob_files); + + for (int level = 0; level < storage_info_.num_levels(); ++level) { + const auto& level_files = storage_info_.LevelFiles(level); + for (const auto& meta : level_files) { + assert(meta); + + live_table_files->emplace_back(meta->fd.GetNumber()); + } + } + + const auto& blob_files = storage_info_.GetBlobFiles(); + for (const auto& meta : blob_files) { + assert(meta); + + live_blob_files->emplace_back(meta->GetBlobFileNumber()); + } +} + +void Version::RemoveLiveFiles( + std::vector& sst_delete_candidates, + std::vector& blob_delete_candidates) const { + for (ObsoleteFileInfo& fi : sst_delete_candidates) { + if (!fi.only_delete_metadata && + storage_info()->GetFileLocation(fi.metadata->fd.GetNumber()) != + VersionStorageInfo::FileLocation::Invalid()) { + fi.only_delete_metadata = true; + } + } + + blob_delete_candidates.erase( + std::remove_if( + blob_delete_candidates.begin(), blob_delete_candidates.end(), + [this](ObsoleteBlobFileInfo& x) { + return storage_info()->GetBlobFileMetaData(x.GetBlobFileNumber()); + }), + blob_delete_candidates.end()); +} + +std::string Version::DebugString(bool hex, bool print_stats) const { + std::string r; + for (int level = 0; level < storage_info_.num_levels_; level++) { + // E.g., + // --- level 1 --- + // 17:123[1 .. 124]['a' .. 'd'] + // 20:43[124 .. 128]['e' .. 'g'] + // + // if print_stats=true: + // 17:123[1 .. 124]['a' .. 'd'](4096) + r.append("--- level "); + AppendNumberTo(&r, level); + r.append(" --- version# "); + AppendNumberTo(&r, version_number_); + if (storage_info_.compact_cursor_[level].Valid()) { + r.append(" --- compact_cursor: "); + r.append(storage_info_.compact_cursor_[level].DebugString(hex)); + } + r.append(" ---\n"); + const std::vector& files = storage_info_.files_[level]; + for (size_t i = 0; i < files.size(); i++) { + r.push_back(' '); + AppendNumberTo(&r, files[i]->fd.GetNumber()); + r.push_back(':'); + AppendNumberTo(&r, files[i]->fd.GetFileSize()); + r.append("["); + AppendNumberTo(&r, files[i]->fd.smallest_seqno); + r.append(" .. "); + AppendNumberTo(&r, files[i]->fd.largest_seqno); + r.append("]"); + r.append("["); + r.append(files[i]->smallest.DebugString(hex)); + r.append(" .. "); + r.append(files[i]->largest.DebugString(hex)); + r.append("]"); + if (files[i]->oldest_blob_file_number != kInvalidBlobFileNumber) { + r.append(" blob_file:"); + AppendNumberTo(&r, files[i]->oldest_blob_file_number); + } + if (print_stats) { + r.append("("); + r.append(std::to_string( + files[i]->stats.num_reads_sampled.load(std::memory_order_relaxed))); + r.append(")"); + } + r.append("\n"); + } + } + + const auto& blob_files = storage_info_.GetBlobFiles(); + if (!blob_files.empty()) { + r.append("--- blob files --- version# "); + AppendNumberTo(&r, version_number_); + r.append(" ---\n"); + for (const auto& blob_file_meta : blob_files) { + assert(blob_file_meta); + + r.append(blob_file_meta->DebugString()); + r.push_back('\n'); + } + } + + return r; +} + +// this is used to batch writes to the manifest file +struct VersionSet::ManifestWriter { + Status status; + bool done; + InstrumentedCondVar cv; + ColumnFamilyData* cfd; + const MutableCFOptions mutable_cf_options; + const autovector& edit_list; + const std::function manifest_write_callback; + + explicit ManifestWriter( + InstrumentedMutex* mu, ColumnFamilyData* _cfd, + const MutableCFOptions& cf_options, const autovector& e, + const std::function& manifest_wcb) + : done(false), + cv(mu), + cfd(_cfd), + mutable_cf_options(cf_options), + edit_list(e), + manifest_write_callback(manifest_wcb) {} + ~ManifestWriter() { status.PermitUncheckedError(); } + + bool IsAllWalEdits() const { + bool all_wal_edits = true; + for (const auto& e : edit_list) { + if (!e->IsWalManipulation()) { + all_wal_edits = false; + break; + } + } + return all_wal_edits; + } +}; + +Status AtomicGroupReadBuffer::AddEdit(VersionEdit* edit) { + assert(edit); + if (edit->is_in_atomic_group_) { + TEST_SYNC_POINT("AtomicGroupReadBuffer::AddEdit:AtomicGroup"); + if (replay_buffer_.empty()) { + replay_buffer_.resize(edit->remaining_entries_ + 1); + TEST_SYNC_POINT_CALLBACK( + "AtomicGroupReadBuffer::AddEdit:FirstInAtomicGroup", edit); + } + read_edits_in_atomic_group_++; + if (read_edits_in_atomic_group_ + edit->remaining_entries_ != + static_cast(replay_buffer_.size())) { + TEST_SYNC_POINT_CALLBACK( + "AtomicGroupReadBuffer::AddEdit:IncorrectAtomicGroupSize", edit); + return Status::Corruption("corrupted atomic group"); + } + replay_buffer_[read_edits_in_atomic_group_ - 1] = *edit; + if (read_edits_in_atomic_group_ == replay_buffer_.size()) { + TEST_SYNC_POINT_CALLBACK( + "AtomicGroupReadBuffer::AddEdit:LastInAtomicGroup", edit); + return Status::OK(); + } + return Status::OK(); + } + + // A normal edit. + if (!replay_buffer().empty()) { + TEST_SYNC_POINT_CALLBACK( + "AtomicGroupReadBuffer::AddEdit:AtomicGroupMixedWithNormalEdits", edit); + return Status::Corruption("corrupted atomic group"); + } + return Status::OK(); +} + +bool AtomicGroupReadBuffer::IsFull() const { + return read_edits_in_atomic_group_ == replay_buffer_.size(); +} + +bool AtomicGroupReadBuffer::IsEmpty() const { return replay_buffer_.empty(); } + +void AtomicGroupReadBuffer::Clear() { + read_edits_in_atomic_group_ = 0; + replay_buffer_.clear(); +} + +VersionSet::VersionSet(const std::string& dbname, + const ImmutableDBOptions* _db_options, + const FileOptions& storage_options, Cache* table_cache, + WriteBufferManager* write_buffer_manager, + WriteController* write_controller, + BlockCacheTracer* const block_cache_tracer, + const std::shared_ptr& io_tracer, + const std::string& db_id, + const std::string& db_session_id) + : column_family_set_(new ColumnFamilySet( + dbname, _db_options, storage_options, table_cache, + write_buffer_manager, write_controller, block_cache_tracer, io_tracer, + db_id, db_session_id)), + table_cache_(table_cache), + env_(_db_options->env), + fs_(_db_options->fs, io_tracer), + clock_(_db_options->clock), + dbname_(dbname), + db_options_(_db_options), + next_file_number_(2), + manifest_file_number_(0), // Filled by Recover() + options_file_number_(0), + options_file_size_(0), + pending_manifest_file_number_(0), + last_sequence_(0), + last_allocated_sequence_(0), + last_published_sequence_(0), + prev_log_number_(0), + current_version_number_(0), + manifest_file_size_(0), + file_options_(storage_options), + block_cache_tracer_(block_cache_tracer), + io_tracer_(io_tracer), + db_session_id_(db_session_id) {} + +VersionSet::~VersionSet() { + // we need to delete column_family_set_ because its destructor depends on + // VersionSet + column_family_set_.reset(); + for (auto& file : obsolete_files_) { + if (file.metadata->table_reader_handle) { + table_cache_->Release(file.metadata->table_reader_handle); + TableCache::Evict(table_cache_, file.metadata->fd.GetNumber()); + } + file.DeleteMetadata(); + } + obsolete_files_.clear(); + io_status_.PermitUncheckedError(); +} + +void VersionSet::Reset() { + if (column_family_set_) { + WriteBufferManager* wbm = column_family_set_->write_buffer_manager(); + WriteController* wc = column_family_set_->write_controller(); + // db_id becomes the source of truth after DBImpl::Recover(): + // https://github.com/facebook/rocksdb/blob/v7.3.1/db/db_impl/db_impl_open.cc#L527 + // Note: we may not be able to recover db_id from MANIFEST if + // options.write_dbid_to_manifest is false (default). + column_family_set_.reset(new ColumnFamilySet( + dbname_, db_options_, file_options_, table_cache_, wbm, wc, + block_cache_tracer_, io_tracer_, db_id_, db_session_id_)); + } + db_id_.clear(); + next_file_number_.store(2); + min_log_number_to_keep_.store(0); + manifest_file_number_ = 0; + options_file_number_ = 0; + pending_manifest_file_number_ = 0; + last_sequence_.store(0); + last_allocated_sequence_.store(0); + last_published_sequence_.store(0); + prev_log_number_ = 0; + descriptor_log_.reset(); + current_version_number_ = 0; + manifest_writers_.clear(); + manifest_file_size_ = 0; + obsolete_files_.clear(); + obsolete_manifests_.clear(); + wals_.Reset(); +} + +void VersionSet::AppendVersion(ColumnFamilyData* column_family_data, + Version* v) { + // compute new compaction score + v->storage_info()->ComputeCompactionScore( + *column_family_data->ioptions(), + *column_family_data->GetLatestMutableCFOptions()); + + // Mark v finalized + v->storage_info_.SetFinalized(); + + // Make "v" current + assert(v->refs_ == 0); + Version* current = column_family_data->current(); + assert(v != current); + if (current != nullptr) { + assert(current->refs_ > 0); + current->Unref(); + } + column_family_data->SetCurrent(v); + v->Ref(); + + // Append to linked list + v->prev_ = column_family_data->dummy_versions()->prev_; + v->next_ = column_family_data->dummy_versions(); + v->prev_->next_ = v; + v->next_->prev_ = v; +} + +Status VersionSet::ProcessManifestWrites( + std::deque& writers, InstrumentedMutex* mu, + FSDirectory* dir_contains_current_file, bool new_descriptor_log, + const ColumnFamilyOptions* new_cf_options, + const ReadOptions& read_options) { + mu->AssertHeld(); + assert(!writers.empty()); + ManifestWriter& first_writer = writers.front(); + ManifestWriter* last_writer = &first_writer; + + assert(!manifest_writers_.empty()); + assert(manifest_writers_.front() == &first_writer); + + autovector batch_edits; + // This vector keeps track of the corresponding user-defined timestamp size + // for `batch_edits` side by side, which is only needed for encoding a + // `VersionEdit` that adds new SST files. + // Note that anytime `batch_edits` has new element added or get existing + // element removed, `batch_edits_ts_sz` should be updated too. + autovector> batch_edits_ts_sz; + autovector versions; + autovector mutable_cf_options_ptrs; + std::vector> builder_guards; + + // Tracking `max_last_sequence` is needed to ensure we write + // `VersionEdit::last_sequence_`s in non-decreasing order according to the + // recovery code's requirement. It also allows us to defer updating + // `descriptor_last_sequence_` until the apply phase, after the log phase + // succeeds. + SequenceNumber max_last_sequence = descriptor_last_sequence_; + + if (first_writer.edit_list.front()->IsColumnFamilyManipulation()) { + // No group commits for column family add or drop + LogAndApplyCFHelper(first_writer.edit_list.front(), &max_last_sequence); + batch_edits.push_back(first_writer.edit_list.front()); + batch_edits_ts_sz.push_back(std::nullopt); + } else { + auto it = manifest_writers_.cbegin(); + size_t group_start = std::numeric_limits::max(); + while (it != manifest_writers_.cend()) { + if ((*it)->edit_list.front()->IsColumnFamilyManipulation()) { + // no group commits for column family add or drop + break; + } + last_writer = *(it++); + assert(last_writer != nullptr); + assert(last_writer->cfd != nullptr); + if (last_writer->cfd->IsDropped()) { + // If we detect a dropped CF at this point, and the corresponding + // version edits belong to an atomic group, then we need to find out + // the preceding version edits in the same atomic group, and update + // their `remaining_entries_` member variable because we are NOT going + // to write the version edits' of dropped CF to the MANIFEST. If we + // don't update, then Recover can report corrupted atomic group because + // the `remaining_entries_` do not match. + if (!batch_edits.empty()) { + if (batch_edits.back()->is_in_atomic_group_ && + batch_edits.back()->remaining_entries_ > 0) { + assert(group_start < batch_edits.size()); + const auto& edit_list = last_writer->edit_list; + size_t k = 0; + while (k < edit_list.size()) { + if (!edit_list[k]->is_in_atomic_group_) { + break; + } else if (edit_list[k]->remaining_entries_ == 0) { + ++k; + break; + } + ++k; + } + for (auto i = group_start; i < batch_edits.size(); ++i) { + assert(static_cast(k) <= + batch_edits.back()->remaining_entries_); + batch_edits[i]->remaining_entries_ -= static_cast(k); + } + } + } + continue; + } + // We do a linear search on versions because versions is small. + // TODO(yanqin) maybe consider unordered_map + Version* version = nullptr; + VersionBuilder* builder = nullptr; + for (int i = 0; i != static_cast(versions.size()); ++i) { + uint32_t cf_id = last_writer->cfd->GetID(); + if (versions[i]->cfd()->GetID() == cf_id) { + version = versions[i]; + assert(!builder_guards.empty() && + builder_guards.size() == versions.size()); + builder = builder_guards[i]->version_builder(); + TEST_SYNC_POINT_CALLBACK( + "VersionSet::ProcessManifestWrites:SameColumnFamily", &cf_id); + break; + } + } + if (version == nullptr) { + // WAL manipulations do not need to be applied to versions. + if (!last_writer->IsAllWalEdits()) { + version = new Version(last_writer->cfd, this, file_options_, + last_writer->mutable_cf_options, io_tracer_, + current_version_number_++); + versions.push_back(version); + mutable_cf_options_ptrs.push_back(&last_writer->mutable_cf_options); + builder_guards.emplace_back( + new BaseReferencedVersionBuilder(last_writer->cfd)); + builder = builder_guards.back()->version_builder(); + } + assert(last_writer->IsAllWalEdits() || builder); + assert(last_writer->IsAllWalEdits() || version); + TEST_SYNC_POINT_CALLBACK("VersionSet::ProcessManifestWrites:NewVersion", + version); + } + const Comparator* ucmp = last_writer->cfd->user_comparator(); + assert(ucmp); + std::optional edit_ts_sz = ucmp->timestamp_size(); + for (const auto& e : last_writer->edit_list) { + if (e->is_in_atomic_group_) { + if (batch_edits.empty() || !batch_edits.back()->is_in_atomic_group_ || + (batch_edits.back()->is_in_atomic_group_ && + batch_edits.back()->remaining_entries_ == 0)) { + group_start = batch_edits.size(); + } + } else if (group_start != std::numeric_limits::max()) { + group_start = std::numeric_limits::max(); + } + Status s = LogAndApplyHelper(last_writer->cfd, builder, e, + &max_last_sequence, mu); + if (!s.ok()) { + // free up the allocated memory + for (auto v : versions) { + delete v; + } + return s; + } + batch_edits.push_back(e); + batch_edits_ts_sz.push_back(edit_ts_sz); + } + } + for (int i = 0; i < static_cast(versions.size()); ++i) { + assert(!builder_guards.empty() && + builder_guards.size() == versions.size()); + auto* builder = builder_guards[i]->version_builder(); + Status s = builder->SaveTo(versions[i]->storage_info()); + if (!s.ok()) { + // free up the allocated memory + for (auto v : versions) { + delete v; + } + return s; + } + } + } + +#ifndef NDEBUG + // Verify that version edits of atomic groups have correct + // remaining_entries_. + size_t k = 0; + while (k < batch_edits.size()) { + while (k < batch_edits.size() && !batch_edits[k]->is_in_atomic_group_) { + ++k; + } + if (k == batch_edits.size()) { + break; + } + size_t i = k; + while (i < batch_edits.size()) { + if (!batch_edits[i]->is_in_atomic_group_) { + break; + } + assert(i - k + batch_edits[i]->remaining_entries_ == + batch_edits[k]->remaining_entries_); + if (batch_edits[i]->remaining_entries_ == 0) { + ++i; + break; + } + ++i; + } + assert(batch_edits[i - 1]->is_in_atomic_group_); + assert(0 == batch_edits[i - 1]->remaining_entries_); + std::vector tmp; + for (size_t j = k; j != i; ++j) { + tmp.emplace_back(batch_edits[j]); + } + TEST_SYNC_POINT_CALLBACK( + "VersionSet::ProcessManifestWrites:CheckOneAtomicGroup", &tmp); + k = i; + } +#endif // NDEBUG + + assert(pending_manifest_file_number_ == 0); + if (!descriptor_log_ || + manifest_file_size_ > db_options_->max_manifest_file_size) { + TEST_SYNC_POINT("VersionSet::ProcessManifestWrites:BeforeNewManifest"); + new_descriptor_log = true; + } else { + pending_manifest_file_number_ = manifest_file_number_; + } + + // Local cached copy of state variable(s). WriteCurrentStateToManifest() + // reads its content after releasing db mutex to avoid race with + // SwitchMemtable(). + std::unordered_map curr_state; + VersionEdit wal_additions; + if (new_descriptor_log) { + pending_manifest_file_number_ = NewFileNumber(); + batch_edits.back()->SetNextFile(next_file_number_.load()); + + // if we are writing out new snapshot make sure to persist max column + // family. + if (column_family_set_->GetMaxColumnFamily() > 0) { + first_writer.edit_list.front()->SetMaxColumnFamily( + column_family_set_->GetMaxColumnFamily()); + } + for (const auto* cfd : *column_family_set_) { + assert(curr_state.find(cfd->GetID()) == curr_state.end()); + curr_state.emplace(std::make_pair( + cfd->GetID(), + MutableCFState(cfd->GetLogNumber(), cfd->GetFullHistoryTsLow()))); + } + + for (const auto& wal : wals_.GetWals()) { + wal_additions.AddWal(wal.first, wal.second); + } + } + + uint64_t new_manifest_file_size = 0; + Status s; + IOStatus io_s; + IOStatus manifest_io_status; + { + FileOptions opt_file_opts = fs_->OptimizeForManifestWrite(file_options_); + mu->Unlock(); + TEST_SYNC_POINT("VersionSet::LogAndApply:WriteManifestStart"); + TEST_SYNC_POINT_CALLBACK("VersionSet::LogAndApply:WriteManifest", nullptr); + if (!first_writer.edit_list.front()->IsColumnFamilyManipulation()) { + for (int i = 0; i < static_cast(versions.size()); ++i) { + assert(!builder_guards.empty() && + builder_guards.size() == versions.size()); + assert(!mutable_cf_options_ptrs.empty() && + builder_guards.size() == versions.size()); + ColumnFamilyData* cfd = versions[i]->cfd_; + s = builder_guards[i]->version_builder()->LoadTableHandlers( + cfd->internal_stats(), 1 /* max_threads */, + true /* prefetch_index_and_filter_in_cache */, + false /* is_initial_load */, + mutable_cf_options_ptrs[i]->prefix_extractor, + MaxFileSizeForL0MetaPin(*mutable_cf_options_ptrs[i]), read_options, + mutable_cf_options_ptrs[i]->block_protection_bytes_per_key); + if (!s.ok()) { + if (db_options_->paranoid_checks) { + break; + } + s = Status::OK(); + } + } + } + + if (s.ok() && new_descriptor_log) { + // This is fine because everything inside of this block is serialized -- + // only one thread can be here at the same time + // create new manifest file + ROCKS_LOG_INFO(db_options_->info_log, "Creating manifest %" PRIu64 "\n", + pending_manifest_file_number_); + std::string descriptor_fname = + DescriptorFileName(dbname_, pending_manifest_file_number_); + std::unique_ptr descriptor_file; + io_s = NewWritableFile(fs_.get(), descriptor_fname, &descriptor_file, + opt_file_opts); + if (io_s.ok()) { + descriptor_file->SetPreallocationBlockSize( + db_options_->manifest_preallocation_size); + FileTypeSet tmp_set = db_options_->checksum_handoff_file_types; + std::unique_ptr file_writer(new WritableFileWriter( + std::move(descriptor_file), descriptor_fname, opt_file_opts, clock_, + io_tracer_, nullptr, db_options_->listeners, nullptr, + tmp_set.Contains(FileType::kDescriptorFile), + tmp_set.Contains(FileType::kDescriptorFile))); + descriptor_log_.reset( + new log::Writer(std::move(file_writer), 0, false)); + s = WriteCurrentStateToManifest(curr_state, wal_additions, + descriptor_log_.get(), io_s); + } else { + manifest_io_status = io_s; + s = io_s; + } + } + + if (s.ok()) { + if (!first_writer.edit_list.front()->IsColumnFamilyManipulation()) { + constexpr bool update_stats = true; + + for (int i = 0; i < static_cast(versions.size()); ++i) { + versions[i]->PrepareAppend(*mutable_cf_options_ptrs[i], read_options, + update_stats); + } + } + + // Write new records to MANIFEST log +#ifndef NDEBUG + size_t idx = 0; +#endif + assert(batch_edits.size() == batch_edits_ts_sz.size()); + for (size_t bidx = 0; bidx < batch_edits.size(); bidx++) { + auto& e = batch_edits[bidx]; + std::string record; + if (!e->EncodeTo(&record, batch_edits_ts_sz[bidx])) { + s = Status::Corruption("Unable to encode VersionEdit:" + + e->DebugString(true)); + break; + } + TEST_KILL_RANDOM_WITH_WEIGHT("VersionSet::LogAndApply:BeforeAddRecord", + REDUCE_ODDS2); +#ifndef NDEBUG + if (batch_edits.size() > 1 && batch_edits.size() - 1 == idx) { + TEST_SYNC_POINT_CALLBACK( + "VersionSet::ProcessManifestWrites:BeforeWriteLastVersionEdit:0", + nullptr); + TEST_SYNC_POINT( + "VersionSet::ProcessManifestWrites:BeforeWriteLastVersionEdit:1"); + } + ++idx; +#endif /* !NDEBUG */ + io_s = descriptor_log_->AddRecord(record); + if (!io_s.ok()) { + s = io_s; + manifest_io_status = io_s; + break; + } + } + + if (s.ok()) { + io_s = SyncManifest(db_options_, descriptor_log_->file()); + manifest_io_status = io_s; + TEST_SYNC_POINT_CALLBACK( + "VersionSet::ProcessManifestWrites:AfterSyncManifest", &io_s); + } + if (!io_s.ok()) { + s = io_s; + ROCKS_LOG_ERROR(db_options_->info_log, "MANIFEST write %s\n", + s.ToString().c_str()); + } + } + + // If we just created a new descriptor file, install it by writing a + // new CURRENT file that points to it. + if (s.ok()) { + assert(manifest_io_status.ok()); + } + if (s.ok() && new_descriptor_log) { + io_s = SetCurrentFile(fs_.get(), dbname_, pending_manifest_file_number_, + dir_contains_current_file); + if (!io_s.ok()) { + s = io_s; + } + } + + if (s.ok()) { + // find offset in manifest file where this version is stored. + new_manifest_file_size = descriptor_log_->file()->GetFileSize(); + } + + if (first_writer.edit_list.front()->is_column_family_drop_) { + TEST_SYNC_POINT("VersionSet::LogAndApply::ColumnFamilyDrop:0"); + TEST_SYNC_POINT("VersionSet::LogAndApply::ColumnFamilyDrop:1"); + TEST_SYNC_POINT("VersionSet::LogAndApply::ColumnFamilyDrop:2"); + } + + LogFlush(db_options_->info_log); + TEST_SYNC_POINT("VersionSet::LogAndApply:WriteManifestDone"); + mu->Lock(); + } + + if (s.ok()) { + // Apply WAL edits, DB mutex must be held. + for (auto& e : batch_edits) { + if (e->IsWalAddition()) { + s = wals_.AddWals(e->GetWalAdditions()); + } else if (e->IsWalDeletion()) { + s = wals_.DeleteWalsBefore(e->GetWalDeletion().GetLogNumber()); + } + if (!s.ok()) { + break; + } + } + } + + if (!io_s.ok()) { + if (io_status_.ok()) { + io_status_ = io_s; + } + } else if (!io_status_.ok()) { + io_status_ = io_s; + } + + // Append the old manifest file to the obsolete_manifest_ list to be deleted + // by PurgeObsoleteFiles later. + if (s.ok() && new_descriptor_log) { + obsolete_manifests_.emplace_back( + DescriptorFileName("", manifest_file_number_)); + } + + // Install the new versions + if (s.ok()) { + if (first_writer.edit_list.front()->is_column_family_add_) { + assert(batch_edits.size() == 1); + assert(new_cf_options != nullptr); + assert(max_last_sequence == descriptor_last_sequence_); + CreateColumnFamily(*new_cf_options, read_options, + first_writer.edit_list.front()); + } else if (first_writer.edit_list.front()->is_column_family_drop_) { + assert(batch_edits.size() == 1); + assert(max_last_sequence == descriptor_last_sequence_); + first_writer.cfd->SetDropped(); + first_writer.cfd->UnrefAndTryDelete(); + } else { + // Each version in versions corresponds to a column family. + // For each column family, update its log number indicating that logs + // with number smaller than this should be ignored. + uint64_t last_min_log_number_to_keep = 0; + for (const auto& e : batch_edits) { + ColumnFamilyData* cfd = nullptr; + if (!e->IsColumnFamilyManipulation()) { + cfd = column_family_set_->GetColumnFamily(e->column_family_); + // e would not have been added to batch_edits if its corresponding + // column family is dropped. + assert(cfd); + } + if (cfd) { + if (e->has_log_number_ && e->log_number_ > cfd->GetLogNumber()) { + cfd->SetLogNumber(e->log_number_); + } + if (e->HasFullHistoryTsLow()) { + cfd->SetFullHistoryTsLow(e->GetFullHistoryTsLow()); + } + } + if (e->has_min_log_number_to_keep_) { + last_min_log_number_to_keep = + std::max(last_min_log_number_to_keep, e->min_log_number_to_keep_); + } + } + + if (last_min_log_number_to_keep != 0) { + MarkMinLogNumberToKeep(last_min_log_number_to_keep); + } + + for (int i = 0; i < static_cast(versions.size()); ++i) { + ColumnFamilyData* cfd = versions[i]->cfd_; + AppendVersion(cfd, versions[i]); + } + } + assert(max_last_sequence >= descriptor_last_sequence_); + descriptor_last_sequence_ = max_last_sequence; + manifest_file_number_ = pending_manifest_file_number_; + manifest_file_size_ = new_manifest_file_size; + prev_log_number_ = first_writer.edit_list.front()->prev_log_number_; + } else { + std::string version_edits; + for (auto& e : batch_edits) { + version_edits += ("\n" + e->DebugString(true)); + } + ROCKS_LOG_ERROR(db_options_->info_log, + "Error in committing version edit to MANIFEST: %s", + version_edits.c_str()); + for (auto v : versions) { + delete v; + } + if (manifest_io_status.ok()) { + manifest_file_number_ = pending_manifest_file_number_; + manifest_file_size_ = new_manifest_file_size; + } + // If manifest append failed for whatever reason, the file could be + // corrupted. So we need to force the next version update to start a + // new manifest file. + descriptor_log_.reset(); + // If manifest operations failed, then we know the CURRENT file still + // points to the original MANIFEST. Therefore, we can safely delete the + // new MANIFEST. + // If manifest operations succeeded, and we are here, then it is possible + // that renaming tmp file to CURRENT failed. + // + // On local POSIX-compliant FS, the CURRENT must point to the original + // MANIFEST. We can delete the new MANIFEST for simplicity, but we can also + // keep it. Future recovery will ignore this MANIFEST. It's also ok for the + // process not to crash and continue using the db. Any future LogAndApply() + // call will switch to a new MANIFEST and update CURRENT, still ignoring + // this one. + // + // On non-local FS, it is + // possible that the rename operation succeeded on the server (remote) + // side, but the client somehow returns a non-ok status to RocksDB. Note + // that this does not violate atomicity. Should we delete the new MANIFEST + // successfully, a subsequent recovery attempt will likely see the CURRENT + // pointing to the new MANIFEST, thus fail. We will not be able to open the + // DB again. Therefore, if manifest operations succeed, we should keep the + // the new MANIFEST. If the process proceeds, any future LogAndApply() call + // will switch to a new MANIFEST and update CURRENT. If user tries to + // re-open the DB, + // a) CURRENT points to the new MANIFEST, and the new MANIFEST is present. + // b) CURRENT points to the original MANIFEST, and the original MANIFEST + // also exists. + if (new_descriptor_log && !manifest_io_status.ok()) { + ROCKS_LOG_INFO(db_options_->info_log, + "Deleting manifest %" PRIu64 " current manifest %" PRIu64 + "\n", + pending_manifest_file_number_, manifest_file_number_); + Status manifest_del_status = env_->DeleteFile( + DescriptorFileName(dbname_, pending_manifest_file_number_)); + if (!manifest_del_status.ok()) { + ROCKS_LOG_WARN(db_options_->info_log, + "Failed to delete manifest %" PRIu64 ": %s", + pending_manifest_file_number_, + manifest_del_status.ToString().c_str()); + } + } + } + + pending_manifest_file_number_ = 0; + +#ifndef NDEBUG + // This is here kind of awkwardly because there's no other consistency + // checks on `VersionSet`'s updates for the new `Version`s. We might want + // to move it to a dedicated function, or remove it if we gain enough + // confidence in `descriptor_last_sequence_`. + if (s.ok()) { + for (const auto* v : versions) { + const auto* vstorage = v->storage_info(); + for (int level = 0; level < vstorage->num_levels(); ++level) { + for (const auto& file : vstorage->LevelFiles(level)) { + assert(file->fd.largest_seqno <= descriptor_last_sequence_); + } + } + } + } +#endif // NDEBUG + + // wake up all the waiting writers + while (true) { + ManifestWriter* ready = manifest_writers_.front(); + manifest_writers_.pop_front(); + bool need_signal = true; + for (const auto& w : writers) { + if (&w == ready) { + need_signal = false; + break; + } + } + ready->status = s; + ready->done = true; + if (ready->manifest_write_callback) { + (ready->manifest_write_callback)(s); + } + if (need_signal) { + ready->cv.Signal(); + } + if (ready == last_writer) { + break; + } + } + if (!manifest_writers_.empty()) { + manifest_writers_.front()->cv.Signal(); + } + return s; +} + +void VersionSet::WakeUpWaitingManifestWriters() { + // wake up all the waiting writers + // Notify new head of manifest write queue. + if (!manifest_writers_.empty()) { + manifest_writers_.front()->cv.Signal(); + } +} + +// 'datas' is grammatically incorrect. We still use this notation to indicate +// that this variable represents a collection of column_family_data. +Status VersionSet::LogAndApply( + const autovector& column_family_datas, + const autovector& mutable_cf_options_list, + const ReadOptions& read_options, + const autovector>& edit_lists, + InstrumentedMutex* mu, FSDirectory* dir_contains_current_file, + bool new_descriptor_log, const ColumnFamilyOptions* new_cf_options, + const std::vector>& manifest_wcbs) { + mu->AssertHeld(); + int num_edits = 0; + for (const auto& elist : edit_lists) { + num_edits += static_cast(elist.size()); + } + if (num_edits == 0) { + return Status::OK(); + } else if (num_edits > 1) { +#ifndef NDEBUG + for (const auto& edit_list : edit_lists) { + for (const auto& edit : edit_list) { + assert(!edit->IsColumnFamilyManipulation()); + } + } +#endif /* ! NDEBUG */ + } + + int num_cfds = static_cast(column_family_datas.size()); + if (num_cfds == 1 && column_family_datas[0] == nullptr) { + assert(edit_lists.size() == 1 && edit_lists[0].size() == 1); + assert(edit_lists[0][0]->is_column_family_add_); + assert(new_cf_options != nullptr); + } + std::deque writers; + if (num_cfds > 0) { + assert(static_cast(num_cfds) == mutable_cf_options_list.size()); + assert(static_cast(num_cfds) == edit_lists.size()); + } + for (int i = 0; i < num_cfds; ++i) { + const auto wcb = + manifest_wcbs.empty() ? [](const Status&) {} : manifest_wcbs[i]; + writers.emplace_back(mu, column_family_datas[i], + *mutable_cf_options_list[i], edit_lists[i], wcb); + manifest_writers_.push_back(&writers[i]); + } + assert(!writers.empty()); + ManifestWriter& first_writer = writers.front(); + TEST_SYNC_POINT_CALLBACK("VersionSet::LogAndApply:BeforeWriterWaiting", + nullptr); + while (!first_writer.done && &first_writer != manifest_writers_.front()) { + first_writer.cv.Wait(); + } + if (first_writer.done) { + // All non-CF-manipulation operations can be grouped together and committed + // to MANIFEST. They should all have finished. The status code is stored in + // the first manifest writer. +#ifndef NDEBUG + for (const auto& writer : writers) { + assert(writer.done); + } + TEST_SYNC_POINT_CALLBACK("VersionSet::LogAndApply:WakeUpAndDone", mu); +#endif /* !NDEBUG */ + return first_writer.status; + } + + int num_undropped_cfds = 0; + for (auto cfd : column_family_datas) { + // if cfd == nullptr, it is a column family add. + if (cfd == nullptr || !cfd->IsDropped()) { + ++num_undropped_cfds; + } + } + if (0 == num_undropped_cfds) { + for (int i = 0; i != num_cfds; ++i) { + manifest_writers_.pop_front(); + } + // Notify new head of manifest write queue. + if (!manifest_writers_.empty()) { + manifest_writers_.front()->cv.Signal(); + } + return Status::ColumnFamilyDropped(); + } + return ProcessManifestWrites(writers, mu, dir_contains_current_file, + new_descriptor_log, new_cf_options, + read_options); +} + +void VersionSet::LogAndApplyCFHelper(VersionEdit* edit, + SequenceNumber* max_last_sequence) { + assert(max_last_sequence != nullptr); + assert(edit->IsColumnFamilyManipulation()); + edit->SetNextFile(next_file_number_.load()); + assert(!edit->HasLastSequence()); + edit->SetLastSequence(*max_last_sequence); + if (edit->is_column_family_drop_) { + // if we drop column family, we have to make sure to save max column family, + // so that we don't reuse existing ID + edit->SetMaxColumnFamily(column_family_set_->GetMaxColumnFamily()); + } +} + +Status VersionSet::LogAndApplyHelper(ColumnFamilyData* cfd, + VersionBuilder* builder, VersionEdit* edit, + SequenceNumber* max_last_sequence, + InstrumentedMutex* mu) { +#ifdef NDEBUG + (void)cfd; +#endif + mu->AssertHeld(); + assert(!edit->IsColumnFamilyManipulation()); + assert(max_last_sequence != nullptr); + + if (edit->has_log_number_) { + assert(edit->log_number_ >= cfd->GetLogNumber()); + assert(edit->log_number_ < next_file_number_.load()); + } + + if (!edit->has_prev_log_number_) { + edit->SetPrevLogNumber(prev_log_number_); + } + edit->SetNextFile(next_file_number_.load()); + if (edit->HasLastSequence() && edit->GetLastSequence() > *max_last_sequence) { + *max_last_sequence = edit->GetLastSequence(); + } else { + edit->SetLastSequence(*max_last_sequence); + } + + // The builder can be nullptr only if edit is WAL manipulation, + // because WAL edits do not need to be applied to versions, + // we return Status::OK() in this case. + assert(builder || edit->IsWalManipulation()); + return builder ? builder->Apply(edit) : Status::OK(); +} + +Status VersionSet::GetCurrentManifestPath(const std::string& dbname, + FileSystem* fs, + std::string* manifest_path, + uint64_t* manifest_file_number) { + assert(fs != nullptr); + assert(manifest_path != nullptr); + assert(manifest_file_number != nullptr); + + std::string fname; + Status s = ReadFileToString(fs, CurrentFileName(dbname), &fname); + if (!s.ok()) { + return s; + } + if (fname.empty() || fname.back() != '\n') { + return Status::Corruption("CURRENT file does not end with newline"); + } + // remove the trailing '\n' + fname.resize(fname.size() - 1); + FileType type; + bool parse_ok = ParseFileName(fname, manifest_file_number, &type); + if (!parse_ok || type != kDescriptorFile) { + return Status::Corruption("CURRENT file corrupted"); + } + *manifest_path = dbname; + if (dbname.back() != '/') { + manifest_path->push_back('/'); + } + manifest_path->append(fname); + return Status::OK(); +} + +Status VersionSet::Recover( + const std::vector& column_families, bool read_only, + std::string* db_id, bool no_error_if_files_missing) { + const ReadOptions read_options(Env::IOActivity::kDBOpen); + // Read "CURRENT" file, which contains a pointer to the current manifest + // file + std::string manifest_path; + Status s = GetCurrentManifestPath(dbname_, fs_.get(), &manifest_path, + &manifest_file_number_); + if (!s.ok()) { + return s; + } + + ROCKS_LOG_INFO(db_options_->info_log, "Recovering from manifest file: %s\n", + manifest_path.c_str()); + + std::unique_ptr manifest_file_reader; + { + std::unique_ptr manifest_file; + s = fs_->NewSequentialFile(manifest_path, + fs_->OptimizeForManifestRead(file_options_), + &manifest_file, nullptr); + if (!s.ok()) { + return s; + } + manifest_file_reader.reset(new SequentialFileReader( + std::move(manifest_file), manifest_path, + db_options_->log_readahead_size, io_tracer_, db_options_->listeners)); + } + uint64_t current_manifest_file_size = 0; + uint64_t log_number = 0; + { + VersionSet::LogReporter reporter; + Status log_read_status; + reporter.status = &log_read_status; + log::Reader reader(nullptr, std::move(manifest_file_reader), &reporter, + true /* checksum */, 0 /* log_number */); + VersionEditHandler handler( + read_only, column_families, const_cast(this), + /*track_missing_files=*/false, no_error_if_files_missing, io_tracer_, + read_options, EpochNumberRequirement::kMightMissing); + handler.Iterate(reader, &log_read_status); + s = handler.status(); + if (s.ok()) { + log_number = handler.GetVersionEditParams().log_number_; + current_manifest_file_size = reader.GetReadOffset(); + assert(current_manifest_file_size != 0); + handler.GetDbId(db_id); + } + if (s.ok()) { + RecoverEpochNumbers(); + } + } + + if (s.ok()) { + manifest_file_size_ = current_manifest_file_size; + ROCKS_LOG_INFO( + db_options_->info_log, + "Recovered from manifest file:%s succeeded," + "manifest_file_number is %" PRIu64 ", next_file_number is %" PRIu64 + ", last_sequence is %" PRIu64 ", log_number is %" PRIu64 + ",prev_log_number is %" PRIu64 ",max_column_family is %" PRIu32 + ",min_log_number_to_keep is %" PRIu64 "\n", + manifest_path.c_str(), manifest_file_number_, next_file_number_.load(), + last_sequence_.load(), log_number, prev_log_number_, + column_family_set_->GetMaxColumnFamily(), min_log_number_to_keep()); + + for (auto cfd : *column_family_set_) { + if (cfd->IsDropped()) { + continue; + } + ROCKS_LOG_INFO(db_options_->info_log, + "Column family [%s] (ID %" PRIu32 + "), log number is %" PRIu64 "\n", + cfd->GetName().c_str(), cfd->GetID(), cfd->GetLogNumber()); + } + } + + return s; +} + +namespace { +class ManifestPicker { + public: + explicit ManifestPicker(const std::string& dbname, + const std::vector& files_in_dbname); + // REQUIRES Valid() == true + std::string GetNextManifest(uint64_t* file_number, std::string* file_name); + bool Valid() const { return manifest_file_iter_ != manifest_files_.end(); } + + private: + const std::string& dbname_; + // MANIFEST file names(s) + std::vector manifest_files_; + std::vector::const_iterator manifest_file_iter_; +}; + +ManifestPicker::ManifestPicker(const std::string& dbname, + const std::vector& files_in_dbname) + : dbname_(dbname) { + // populate manifest files + assert(!files_in_dbname.empty()); + for (const auto& fname : files_in_dbname) { + uint64_t file_num = 0; + FileType file_type; + bool parse_ok = ParseFileName(fname, &file_num, &file_type); + if (parse_ok && file_type == kDescriptorFile) { + manifest_files_.push_back(fname); + } + } + // seek to first manifest + std::sort(manifest_files_.begin(), manifest_files_.end(), + [](const std::string& lhs, const std::string& rhs) { + uint64_t num1 = 0; + uint64_t num2 = 0; + FileType type1; + FileType type2; + bool parse_ok1 = ParseFileName(lhs, &num1, &type1); + bool parse_ok2 = ParseFileName(rhs, &num2, &type2); +#ifndef NDEBUG + assert(parse_ok1); + assert(parse_ok2); +#else + (void)parse_ok1; + (void)parse_ok2; +#endif + return num1 > num2; + }); + manifest_file_iter_ = manifest_files_.begin(); +} + +std::string ManifestPicker::GetNextManifest(uint64_t* number, + std::string* file_name) { + assert(Valid()); + std::string ret; + if (manifest_file_iter_ != manifest_files_.end()) { + ret.assign(dbname_); + if (ret.back() != kFilePathSeparator) { + ret.push_back(kFilePathSeparator); + } + ret.append(*manifest_file_iter_); + if (number) { + FileType type; + bool parse = ParseFileName(*manifest_file_iter_, number, &type); + assert(type == kDescriptorFile); +#ifndef NDEBUG + assert(parse); +#else + (void)parse; +#endif + } + if (file_name) { + *file_name = *manifest_file_iter_; + } + ++manifest_file_iter_; + } + return ret; +} +} // anonymous namespace + +Status VersionSet::TryRecover( + const std::vector& column_families, bool read_only, + const std::vector& files_in_dbname, std::string* db_id, + bool* has_missing_table_file) { + ManifestPicker manifest_picker(dbname_, files_in_dbname); + if (!manifest_picker.Valid()) { + return Status::Corruption("Cannot locate MANIFEST file in " + dbname_); + } + Status s; + std::string manifest_path = + manifest_picker.GetNextManifest(&manifest_file_number_, nullptr); + while (!manifest_path.empty()) { + s = TryRecoverFromOneManifest(manifest_path, column_families, read_only, + db_id, has_missing_table_file); + if (s.ok() || !manifest_picker.Valid()) { + break; + } + Reset(); + manifest_path = + manifest_picker.GetNextManifest(&manifest_file_number_, nullptr); + } + return s; +} + +Status VersionSet::TryRecoverFromOneManifest( + const std::string& manifest_path, + const std::vector& column_families, bool read_only, + std::string* db_id, bool* has_missing_table_file) { + const ReadOptions read_options(Env::IOActivity::kDBOpen); + ROCKS_LOG_INFO(db_options_->info_log, "Trying to recover from manifest: %s\n", + manifest_path.c_str()); + std::unique_ptr manifest_file_reader; + Status s; + { + std::unique_ptr manifest_file; + s = fs_->NewSequentialFile(manifest_path, + fs_->OptimizeForManifestRead(file_options_), + &manifest_file, nullptr); + if (!s.ok()) { + return s; + } + manifest_file_reader.reset(new SequentialFileReader( + std::move(manifest_file), manifest_path, + db_options_->log_readahead_size, io_tracer_, db_options_->listeners)); + } + + assert(s.ok()); + VersionSet::LogReporter reporter; + reporter.status = &s; + log::Reader reader(nullptr, std::move(manifest_file_reader), &reporter, + /*checksum=*/true, /*log_num=*/0); + VersionEditHandlerPointInTime handler_pit( + read_only, column_families, const_cast(this), io_tracer_, + read_options, EpochNumberRequirement::kMightMissing); + + handler_pit.Iterate(reader, &s); + + handler_pit.GetDbId(db_id); + + assert(nullptr != has_missing_table_file); + *has_missing_table_file = handler_pit.HasMissingFiles(); + + s = handler_pit.status(); + if (s.ok()) { + RecoverEpochNumbers(); + } + return s; +} + +void VersionSet::RecoverEpochNumbers() { + for (auto cfd : *column_family_set_) { + if (cfd->IsDropped()) { + continue; + } + assert(cfd->initialized()); + cfd->RecoverEpochNumbers(); + } +} + +Status VersionSet::ListColumnFamilies(std::vector* column_families, + const std::string& dbname, + FileSystem* fs) { + // Read "CURRENT" file, which contains a pointer to the current manifest file + std::string manifest_path; + uint64_t manifest_file_number; + Status s = + GetCurrentManifestPath(dbname, fs, &manifest_path, &manifest_file_number); + if (!s.ok()) { + return s; + } + return ListColumnFamiliesFromManifest(manifest_path, fs, column_families); +} + +Status VersionSet::ListColumnFamiliesFromManifest( + const std::string& manifest_path, FileSystem* fs, + std::vector* column_families) { + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + std::unique_ptr file_reader; + Status s; + { + std::unique_ptr file; + // these are just for performance reasons, not correctness, + // so we're fine using the defaults + s = fs->NewSequentialFile(manifest_path, FileOptions(), &file, nullptr); + if (!s.ok()) { + return s; + } + file_reader = std::make_unique( + std::move(file), manifest_path, /*io_tracer=*/nullptr); + } + + VersionSet::LogReporter reporter; + reporter.status = &s; + log::Reader reader(nullptr, std::move(file_reader), &reporter, + true /* checksum */, 0 /* log_number */); + + ListColumnFamiliesHandler handler(read_options); + handler.Iterate(reader, &s); + + assert(column_families); + column_families->clear(); + if (handler.status().ok()) { + for (const auto& iter : handler.GetColumnFamilyNames()) { + column_families->push_back(iter.second); + } + } + + return handler.status(); +} + +Status VersionSet::ReduceNumberOfLevels(const std::string& dbname, + const Options* options, + const FileOptions& file_options, + int new_levels) { + if (new_levels <= 1) { + return Status::InvalidArgument( + "Number of levels needs to be bigger than 1"); + } + + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + + ImmutableDBOptions db_options(*options); + ColumnFamilyOptions cf_options(*options); + std::shared_ptr tc(NewLRUCache(options->max_open_files - 10, + options->table_cache_numshardbits)); + WriteController wc(options->delayed_write_rate); + WriteBufferManager wb(options->db_write_buffer_size); + VersionSet versions(dbname, &db_options, file_options, tc.get(), &wb, &wc, + nullptr /*BlockCacheTracer*/, nullptr /*IOTracer*/, + /*db_id*/ "", + /*db_session_id*/ ""); + Status status; + + std::vector dummy; + ColumnFamilyDescriptor dummy_descriptor(kDefaultColumnFamilyName, + ColumnFamilyOptions(*options)); + dummy.push_back(dummy_descriptor); + status = versions.Recover(dummy); + if (!status.ok()) { + return status; + } + + Version* current_version = + versions.GetColumnFamilySet()->GetDefault()->current(); + auto* vstorage = current_version->storage_info(); + int current_levels = vstorage->num_levels(); + + if (current_levels <= new_levels) { + return Status::OK(); + } + + // Make sure there are file only on one level from + // (new_levels-1) to (current_levels-1) + int first_nonempty_level = -1; + int first_nonempty_level_filenum = 0; + for (int i = new_levels - 1; i < current_levels; i++) { + int file_num = vstorage->NumLevelFiles(i); + if (file_num != 0) { + if (first_nonempty_level < 0) { + first_nonempty_level = i; + first_nonempty_level_filenum = file_num; + } else { + char msg[255]; + snprintf(msg, sizeof(msg), + "Found at least two levels containing files: " + "[%d:%d],[%d:%d].\n", + first_nonempty_level, first_nonempty_level_filenum, i, + file_num); + return Status::InvalidArgument(msg); + } + } + } + + // we need to allocate an array with the old number of levels size to + // avoid SIGSEGV in WriteCurrentStatetoManifest() + // however, all levels bigger or equal to new_levels will be empty + std::vector* new_files_list = + new std::vector[current_levels]; + for (int i = 0; i < new_levels - 1; i++) { + new_files_list[i] = vstorage->LevelFiles(i); + } + + if (first_nonempty_level > 0) { + auto& new_last_level = new_files_list[new_levels - 1]; + + new_last_level = vstorage->LevelFiles(first_nonempty_level); + + for (size_t i = 0; i < new_last_level.size(); ++i) { + const FileMetaData* const meta = new_last_level[i]; + assert(meta); + + const uint64_t file_number = meta->fd.GetNumber(); + + vstorage->file_locations_[file_number] = + VersionStorageInfo::FileLocation(new_levels - 1, i); + } + } + + delete[] vstorage->files_; + vstorage->files_ = new_files_list; + vstorage->num_levels_ = new_levels; + vstorage->ResizeCompactCursors(new_levels); + + MutableCFOptions mutable_cf_options(*options); + VersionEdit ve; + InstrumentedMutex dummy_mutex; + InstrumentedMutexLock l(&dummy_mutex); + return versions.LogAndApply(versions.GetColumnFamilySet()->GetDefault(), + mutable_cf_options, read_options, &ve, + &dummy_mutex, nullptr, true); +} + +// Get the checksum information including the checksum and checksum function +// name of all SST and blob files in VersionSet. Store the information in +// FileChecksumList which contains a map from file number to its checksum info. +// If DB is not running, make sure call VersionSet::Recover() to load the file +// metadata from Manifest to VersionSet before calling this function. +Status VersionSet::GetLiveFilesChecksumInfo(FileChecksumList* checksum_list) { + // Clean the previously stored checksum information if any. + Status s; + if (checksum_list == nullptr) { + s = Status::InvalidArgument("checksum_list is nullptr"); + return s; + } + checksum_list->reset(); + + for (auto cfd : *column_family_set_) { + assert(cfd); + + if (cfd->IsDropped() || !cfd->initialized()) { + continue; + } + + const auto* current = cfd->current(); + assert(current); + + const auto* vstorage = current->storage_info(); + assert(vstorage); + + /* SST files */ + for (int level = 0; level < cfd->NumberLevels(); level++) { + const auto& level_files = vstorage->LevelFiles(level); + + for (const auto& file : level_files) { + assert(file); + + s = checksum_list->InsertOneFileChecksum(file->fd.GetNumber(), + file->file_checksum, + file->file_checksum_func_name); + if (!s.ok()) { + return s; + } + } + } + + /* Blob files */ + const auto& blob_files = vstorage->GetBlobFiles(); + for (const auto& meta : blob_files) { + assert(meta); + + std::string checksum_value = meta->GetChecksumValue(); + std::string checksum_method = meta->GetChecksumMethod(); + assert(checksum_value.empty() == checksum_method.empty()); + if (meta->GetChecksumMethod().empty()) { + checksum_value = kUnknownFileChecksum; + checksum_method = kUnknownFileChecksumFuncName; + } + + s = checksum_list->InsertOneFileChecksum(meta->GetBlobFileNumber(), + checksum_value, checksum_method); + if (!s.ok()) { + return s; + } + } + } + + return s; +} + +Status VersionSet::DumpManifest( + Options& options, std::string& dscname, bool verbose, bool hex, bool json, + const std::vector& cf_descs) { + assert(options.env); + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + + std::vector column_families; + Status s = ListColumnFamiliesFromManifest( + dscname, options.env->GetFileSystem().get(), &column_families); + if (!s.ok()) { + return s; + } + + // Open the specified manifest file. + std::unique_ptr file_reader; + { + std::unique_ptr file; + const std::shared_ptr& fs = options.env->GetFileSystem(); + s = fs->NewSequentialFile( + dscname, fs->OptimizeForManifestRead(file_options_), &file, nullptr); + if (!s.ok()) { + return s; + } + file_reader = std::make_unique( + std::move(file), dscname, db_options_->log_readahead_size, io_tracer_); + } + + std::map cf_name_to_desc; + for (const auto& cf_desc : cf_descs) { + cf_name_to_desc[cf_desc.name] = &cf_desc; + } + std::vector final_cf_descs; + for (const auto& cf : column_families) { + const auto iter = cf_name_to_desc.find(cf); + if (iter != cf_name_to_desc.cend()) { + final_cf_descs.push_back(*iter->second); + } else { + final_cf_descs.emplace_back(cf, options); + } + } + + DumpManifestHandler handler(final_cf_descs, this, io_tracer_, read_options, + verbose, hex, json); + { + VersionSet::LogReporter reporter; + reporter.status = &s; + log::Reader reader(nullptr, std::move(file_reader), &reporter, + true /* checksum */, 0 /* log_number */); + handler.Iterate(reader, &s); + } + + return handler.status(); +} + +void VersionSet::MarkFileNumberUsed(uint64_t number) { + // only called during recovery and repair which are single threaded, so this + // works because there can't be concurrent calls + if (next_file_number_.load(std::memory_order_relaxed) <= number) { + next_file_number_.store(number + 1, std::memory_order_relaxed); + } +} +// Called only either from ::LogAndApply which is protected by mutex or during +// recovery which is single-threaded. +void VersionSet::MarkMinLogNumberToKeep(uint64_t number) { + if (min_log_number_to_keep_.load(std::memory_order_relaxed) < number) { + min_log_number_to_keep_.store(number, std::memory_order_relaxed); + } +} + +Status VersionSet::WriteCurrentStateToManifest( + const std::unordered_map& curr_state, + const VersionEdit& wal_additions, log::Writer* log, IOStatus& io_s) { + // TODO: Break up into multiple records to reduce memory usage on recovery? + + // WARNING: This method doesn't hold a mutex!! + + // This is done without DB mutex lock held, but only within single-threaded + // LogAndApply. Column family manipulations can only happen within LogAndApply + // (the same single thread), so we're safe to iterate. + + assert(io_s.ok()); + if (db_options_->write_dbid_to_manifest) { + VersionEdit edit_for_db_id; + assert(!db_id_.empty()); + edit_for_db_id.SetDBId(db_id_); + std::string db_id_record; + if (!edit_for_db_id.EncodeTo(&db_id_record)) { + return Status::Corruption("Unable to Encode VersionEdit:" + + edit_for_db_id.DebugString(true)); + } + io_s = log->AddRecord(db_id_record); + if (!io_s.ok()) { + return io_s; + } + } + + // Save WALs. + if (!wal_additions.GetWalAdditions().empty()) { + TEST_SYNC_POINT_CALLBACK("VersionSet::WriteCurrentStateToManifest:SaveWal", + const_cast(&wal_additions)); + std::string record; + if (!wal_additions.EncodeTo(&record)) { + return Status::Corruption("Unable to Encode VersionEdit: " + + wal_additions.DebugString(true)); + } + io_s = log->AddRecord(record); + if (!io_s.ok()) { + return io_s; + } + } + + // New manifest should rollover the WAL deletion record from previous + // manifest. Otherwise, when an addition record of a deleted WAL gets added to + // this new manifest later (which can happens in e.g, SyncWAL()), this new + // manifest creates an illusion that such WAL hasn't been deleted. + VersionEdit wal_deletions; + wal_deletions.DeleteWalsBefore(min_log_number_to_keep()); + std::string wal_deletions_record; + if (!wal_deletions.EncodeTo(&wal_deletions_record)) { + return Status::Corruption("Unable to Encode VersionEdit: " + + wal_deletions.DebugString(true)); + } + io_s = log->AddRecord(wal_deletions_record); + if (!io_s.ok()) { + return io_s; + } + + for (auto cfd : *column_family_set_) { + assert(cfd); + + if (cfd->IsDropped()) { + continue; + } + assert(cfd->initialized()); + { + // Store column family info + VersionEdit edit; + if (cfd->GetID() != 0) { + // default column family is always there, + // no need to explicitly write it + edit.AddColumnFamily(cfd->GetName()); + edit.SetColumnFamily(cfd->GetID()); + } + edit.SetComparatorName( + cfd->internal_comparator().user_comparator()->Name()); + edit.SetPersistUserDefinedTimestamps( + cfd->ioptions()->persist_user_defined_timestamps); + std::string record; + if (!edit.EncodeTo(&record)) { + return Status::Corruption("Unable to Encode VersionEdit:" + + edit.DebugString(true)); + } + io_s = log->AddRecord(record); + if (!io_s.ok()) { + return io_s; + } + } + + { + // Save files + VersionEdit edit; + edit.SetColumnFamily(cfd->GetID()); + + const auto* current = cfd->current(); + assert(current); + + const auto* vstorage = current->storage_info(); + assert(vstorage); + + for (int level = 0; level < cfd->NumberLevels(); level++) { + const auto& level_files = vstorage->LevelFiles(level); + + for (const auto& f : level_files) { + assert(f); + + edit.AddFile(level, f->fd.GetNumber(), f->fd.GetPathId(), + f->fd.GetFileSize(), f->smallest, f->largest, + f->fd.smallest_seqno, f->fd.largest_seqno, + f->marked_for_compaction, f->temperature, + f->oldest_blob_file_number, f->oldest_ancester_time, + f->file_creation_time, f->epoch_number, f->file_checksum, + f->file_checksum_func_name, f->unique_id, + f->compensated_range_deletion_size, f->tail_size, + f->user_defined_timestamps_persisted); + } + } + + edit.SetCompactCursors(vstorage->GetCompactCursors()); + + const auto& blob_files = vstorage->GetBlobFiles(); + for (const auto& meta : blob_files) { + assert(meta); + + const uint64_t blob_file_number = meta->GetBlobFileNumber(); + + edit.AddBlobFile(blob_file_number, meta->GetTotalBlobCount(), + meta->GetTotalBlobBytes(), meta->GetChecksumMethod(), + meta->GetChecksumValue()); + if (meta->GetGarbageBlobCount() > 0) { + edit.AddBlobFileGarbage(blob_file_number, meta->GetGarbageBlobCount(), + meta->GetGarbageBlobBytes()); + } + } + + const auto iter = curr_state.find(cfd->GetID()); + assert(iter != curr_state.end()); + uint64_t log_number = iter->second.log_number; + edit.SetLogNumber(log_number); + + if (cfd->GetID() == 0) { + // min_log_number_to_keep is for the whole db, not for specific column + // family. So it does not need to be set for every column family, just + // need to be set once. Since default CF can never be dropped, we set + // the min_log to the default CF here. + uint64_t min_log = min_log_number_to_keep(); + if (min_log != 0) { + edit.SetMinLogNumberToKeep(min_log); + } + } + + const std::string& full_history_ts_low = iter->second.full_history_ts_low; + if (!full_history_ts_low.empty()) { + edit.SetFullHistoryTsLow(full_history_ts_low); + } + + edit.SetLastSequence(descriptor_last_sequence_); + + const Comparator* ucmp = cfd->user_comparator(); + assert(ucmp); + std::string record; + if (!edit.EncodeTo(&record, ucmp->timestamp_size())) { + return Status::Corruption("Unable to Encode VersionEdit:" + + edit.DebugString(true)); + } + io_s = log->AddRecord(record); + if (!io_s.ok()) { + return io_s; + } + } + } + return Status::OK(); +} + +// TODO(aekmekji): in CompactionJob::GenSubcompactionBoundaries(), this +// function is called repeatedly with consecutive pairs of slices. For example +// if the slice list is [a, b, c, d] this function is called with arguments +// (a,b) then (b,c) then (c,d). Knowing this, an optimization is possible where +// we avoid doing binary search for the keys b and c twice and instead somehow +// maintain state of where they first appear in the files. +uint64_t VersionSet::ApproximateSize(const SizeApproximationOptions& options, + const ReadOptions& read_options, + Version* v, const Slice& start, + const Slice& end, int start_level, + int end_level, TableReaderCaller caller) { + const auto& icmp = v->cfd_->internal_comparator(); + + // pre-condition + assert(icmp.Compare(start, end) <= 0); + + uint64_t total_full_size = 0; + const auto* vstorage = v->storage_info(); + const int num_non_empty_levels = vstorage->num_non_empty_levels(); + end_level = (end_level == -1) ? num_non_empty_levels + : std::min(end_level, num_non_empty_levels); + if (end_level <= start_level) { + return 0; + } + + // Outline of the optimization that uses options.files_size_error_margin. + // When approximating the files total size that is used to store a keys range, + // we first sum up the sizes of the files that fully fall into the range. + // Then we sum up the sizes of all the files that may intersect with the range + // (this includes all files in L0 as well). Then, if total_intersecting_size + // is smaller than total_full_size * options.files_size_error_margin - we can + // infer that the intersecting files have a sufficiently negligible + // contribution to the total size, and we can approximate the storage required + // for the keys in range as just half of the intersecting_files_size. + // E.g., if the value of files_size_error_margin is 0.1, then the error of the + // approximation is limited to only ~10% of the total size of files that fully + // fall into the keys range. In such case, this helps to avoid a costly + // process of binary searching the intersecting files that is required only + // for a more precise calculation of the total size. + + autovector first_files; + autovector last_files; + + // scan all the levels + for (int level = start_level; level < end_level; ++level) { + const LevelFilesBrief& files_brief = vstorage->LevelFilesBrief(level); + if (files_brief.num_files == 0) { + // empty level, skip exploration + continue; + } + + if (level == 0) { + // level 0 files are not in sorted order, we need to iterate through + // the list to compute the total bytes that require scanning, + // so handle the case explicitly (similarly to first_files case) + for (size_t i = 0; i < files_brief.num_files; i++) { + first_files.push_back(&files_brief.files[i]); + } + continue; + } + + assert(level > 0); + assert(files_brief.num_files > 0); + + // identify the file position for start key + const int idx_start = + FindFileInRange(icmp, files_brief, start, 0, + static_cast(files_brief.num_files - 1)); + assert(static_cast(idx_start) < files_brief.num_files); + + // identify the file position for end key + int idx_end = idx_start; + if (icmp.Compare(files_brief.files[idx_end].largest_key, end) < 0) { + idx_end = + FindFileInRange(icmp, files_brief, end, idx_start, + static_cast(files_brief.num_files - 1)); + } + assert(idx_end >= idx_start && + static_cast(idx_end) < files_brief.num_files); + + // scan all files from the starting index to the ending index + // (inferred from the sorted order) + + // first scan all the intermediate full files (excluding first and last) + for (int i = idx_start + 1; i < idx_end; ++i) { + uint64_t file_size = files_brief.files[i].fd.GetFileSize(); + // The entire file falls into the range, so we can just take its size. + assert(file_size == ApproximateSize(read_options, v, files_brief.files[i], + start, end, caller)); + total_full_size += file_size; + } + + // save the first and the last files (which may be the same file), so we + // can scan them later. + first_files.push_back(&files_brief.files[idx_start]); + if (idx_start != idx_end) { + // we need to estimate size for both files, only if they are different + last_files.push_back(&files_brief.files[idx_end]); + } + } + + // The sum of all file sizes that intersect the [start, end] keys range. + uint64_t total_intersecting_size = 0; + for (const auto* file_ptr : first_files) { + total_intersecting_size += file_ptr->fd.GetFileSize(); + } + for (const auto* file_ptr : last_files) { + total_intersecting_size += file_ptr->fd.GetFileSize(); + } + + // Now scan all the first & last files at each level, and estimate their size. + // If the total_intersecting_size is less than X% of the total_full_size - we + // want to approximate the result in order to avoid the costly binary search + // inside ApproximateSize. We use half of file size as an approximation below. + + const double margin = options.files_size_error_margin; + if (margin > 0 && total_intersecting_size < + static_cast(total_full_size * margin)) { + total_full_size += total_intersecting_size / 2; + } else { + // Estimate for all the first files (might also be last files), at each + // level + for (const auto file_ptr : first_files) { + total_full_size += + ApproximateSize(read_options, v, *file_ptr, start, end, caller); + } + + // Estimate for all the last files, at each level + for (const auto file_ptr : last_files) { + // We could use ApproximateSize here, but calling ApproximateOffsetOf + // directly is just more efficient. + total_full_size += + ApproximateOffsetOf(read_options, v, *file_ptr, end, caller); + } + } + + return total_full_size; +} + +uint64_t VersionSet::ApproximateOffsetOf(const ReadOptions& read_options, + Version* v, const FdWithKeyRange& f, + const Slice& key, + TableReaderCaller caller) { + // pre-condition + assert(v); + const auto& icmp = v->cfd_->internal_comparator(); + + uint64_t result = 0; + if (icmp.Compare(f.largest_key, key) <= 0) { + // Entire file is before "key", so just add the file size + result = f.fd.GetFileSize(); + } else if (icmp.Compare(f.smallest_key, key) > 0) { + // Entire file is after "key", so ignore + result = 0; + } else { + // "key" falls in the range for this table. Add the + // approximate offset of "key" within the table. + TableCache* table_cache = v->cfd_->table_cache(); + const MutableCFOptions& cf_opts = v->GetMutableCFOptions(); + if (table_cache != nullptr) { + result = table_cache->ApproximateOffsetOf( + read_options, key, *f.file_metadata, caller, icmp, + cf_opts.block_protection_bytes_per_key, cf_opts.prefix_extractor); + } + } + return result; +} + +uint64_t VersionSet::ApproximateSize(const ReadOptions& read_options, + Version* v, const FdWithKeyRange& f, + const Slice& start, const Slice& end, + TableReaderCaller caller) { + // pre-condition + assert(v); + const auto& icmp = v->cfd_->internal_comparator(); + assert(icmp.Compare(start, end) <= 0); + + if (icmp.Compare(f.largest_key, start) <= 0 || + icmp.Compare(f.smallest_key, end) > 0) { + // Entire file is before or after the start/end keys range + return 0; + } + + if (icmp.Compare(f.smallest_key, start) >= 0) { + // Start of the range is before the file start - approximate by end offset + return ApproximateOffsetOf(read_options, v, f, end, caller); + } + + if (icmp.Compare(f.largest_key, end) < 0) { + // End of the range is after the file end - approximate by subtracting + // start offset from the file size + uint64_t start_offset = + ApproximateOffsetOf(read_options, v, f, start, caller); + assert(f.fd.GetFileSize() >= start_offset); + return f.fd.GetFileSize() - start_offset; + } + + // The interval falls entirely in the range for this file. + TableCache* table_cache = v->cfd_->table_cache(); + if (table_cache == nullptr) { + return 0; + } + const MutableCFOptions& cf_opts = v->GetMutableCFOptions(); + return table_cache->ApproximateSize( + read_options, start, end, *f.file_metadata, caller, icmp, + cf_opts.block_protection_bytes_per_key, cf_opts.prefix_extractor); +} + +void VersionSet::RemoveLiveFiles( + std::vector& sst_delete_candidates, + std::vector& blob_delete_candidates) const { + assert(column_family_set_); + for (auto cfd : *column_family_set_) { + assert(cfd); + if (!cfd->initialized()) { + continue; + } + + auto* current = cfd->current(); + bool found_current = false; + + Version* const dummy_versions = cfd->dummy_versions(); + assert(dummy_versions); + + for (Version* v = dummy_versions->next_; v != dummy_versions; + v = v->next_) { + v->RemoveLiveFiles(sst_delete_candidates, blob_delete_candidates); + if (v == current) { + found_current = true; + } + } + + if (!found_current && current != nullptr) { + // Should never happen unless it is a bug. + assert(false); + current->RemoveLiveFiles(sst_delete_candidates, blob_delete_candidates); + } + } +} + +void VersionSet::AddLiveFiles(std::vector* live_table_files, + std::vector* live_blob_files) const { + assert(live_table_files); + assert(live_blob_files); + + // pre-calculate space requirement + size_t total_table_files = 0; + size_t total_blob_files = 0; + + assert(column_family_set_); + for (auto cfd : *column_family_set_) { + assert(cfd); + + if (!cfd->initialized()) { + continue; + } + + Version* const dummy_versions = cfd->dummy_versions(); + assert(dummy_versions); + + for (Version* v = dummy_versions->next_; v != dummy_versions; + v = v->next_) { + assert(v); + + const auto* vstorage = v->storage_info(); + assert(vstorage); + + for (int level = 0; level < vstorage->num_levels(); ++level) { + total_table_files += vstorage->LevelFiles(level).size(); + } + + total_blob_files += vstorage->GetBlobFiles().size(); + } + } + + // just one time extension to the right size + live_table_files->reserve(live_table_files->size() + total_table_files); + live_blob_files->reserve(live_blob_files->size() + total_blob_files); + + assert(column_family_set_); + for (auto cfd : *column_family_set_) { + assert(cfd); + if (!cfd->initialized()) { + continue; + } + + auto* current = cfd->current(); + bool found_current = false; + + Version* const dummy_versions = cfd->dummy_versions(); + assert(dummy_versions); + + for (Version* v = dummy_versions->next_; v != dummy_versions; + v = v->next_) { + v->AddLiveFiles(live_table_files, live_blob_files); + if (v == current) { + found_current = true; + } + } + + if (!found_current && current != nullptr) { + // Should never happen unless it is a bug. + assert(false); + current->AddLiveFiles(live_table_files, live_blob_files); + } + } +} + +InternalIterator* VersionSet::MakeInputIterator( + const ReadOptions& read_options, const Compaction* c, + RangeDelAggregator* range_del_agg, + const FileOptions& file_options_compactions, + const std::optional& start, + const std::optional& end) { + auto cfd = c->column_family_data(); + // Level-0 files have to be merged together. For other levels, + // we will make a concatenating iterator per level. + // TODO(opt): use concatenating iterator for level-0 if there is no overlap + const size_t space = (c->level() == 0 ? c->input_levels(0)->num_files + + c->num_input_levels() - 1 + : c->num_input_levels()); + InternalIterator** list = new InternalIterator*[space]; + // First item in the pair is a pointer to range tombstones. + // Second item is a pointer to a member of a LevelIterator, + // that will be initialized to where CompactionMergingIterator stores + // pointer to its range tombstones. This is used by LevelIterator + // to update pointer to range tombstones as it traverse different SST files. + std::vector< + std::pair> + range_tombstones; + size_t num = 0; + for (size_t which = 0; which < c->num_input_levels(); which++) { + if (c->input_levels(which)->num_files != 0) { + if (c->level(which) == 0) { + const LevelFilesBrief* flevel = c->input_levels(which); + for (size_t i = 0; i < flevel->num_files; i++) { + const FileMetaData& fmd = *flevel->files[i].file_metadata; + if (start.has_value() && + cfd->user_comparator()->CompareWithoutTimestamp( + *start, fmd.largest.user_key()) > 0) { + continue; + } + // We should be able to filter out the case where the end key + // equals to the end boundary, since the end key is exclusive. + // We try to be extra safe here. + if (end.has_value() && + cfd->user_comparator()->CompareWithoutTimestamp( + *end, fmd.smallest.user_key()) < 0) { + continue; + } + TruncatedRangeDelIterator* range_tombstone_iter = nullptr; + list[num++] = cfd->table_cache()->NewIterator( + read_options, file_options_compactions, + cfd->internal_comparator(), fmd, range_del_agg, + c->mutable_cf_options()->prefix_extractor, + /*table_reader_ptr=*/nullptr, + /*file_read_hist=*/nullptr, TableReaderCaller::kCompaction, + /*arena=*/nullptr, + /*skip_filters=*/false, + /*level=*/static_cast(c->level(which)), + MaxFileSizeForL0MetaPin(*c->mutable_cf_options()), + /*smallest_compaction_key=*/nullptr, + /*largest_compaction_key=*/nullptr, + /*allow_unprepared_value=*/false, + c->mutable_cf_options()->block_protection_bytes_per_key, + /*range_del_iter=*/&range_tombstone_iter); + range_tombstones.emplace_back(range_tombstone_iter, nullptr); + } + } else { + // Create concatenating iterator for the files from this level + TruncatedRangeDelIterator*** tombstone_iter_ptr = nullptr; + list[num++] = new LevelIterator( + cfd->table_cache(), read_options, file_options_compactions, + cfd->internal_comparator(), c->input_levels(which), + c->mutable_cf_options()->prefix_extractor, + /*should_sample=*/false, + /*no per level latency histogram=*/nullptr, + TableReaderCaller::kCompaction, /*skip_filters=*/false, + /*level=*/static_cast(c->level(which)), + c->mutable_cf_options()->block_protection_bytes_per_key, + range_del_agg, c->boundaries(which), false, &tombstone_iter_ptr); + range_tombstones.emplace_back(nullptr, tombstone_iter_ptr); + } + } + } + assert(num <= space); + InternalIterator* result = NewCompactionMergingIterator( + &c->column_family_data()->internal_comparator(), list, + static_cast(num), range_tombstones); + delete[] list; + return result; +} + +Status VersionSet::GetMetadataForFile(uint64_t number, int* filelevel, + FileMetaData** meta, + ColumnFamilyData** cfd) { + for (auto cfd_iter : *column_family_set_) { + if (!cfd_iter->initialized()) { + continue; + } + Version* version = cfd_iter->current(); + const auto* vstorage = version->storage_info(); + for (int level = 0; level < vstorage->num_levels(); level++) { + for (const auto& file : vstorage->LevelFiles(level)) { + if (file->fd.GetNumber() == number) { + *meta = file; + *filelevel = level; + *cfd = cfd_iter; + return Status::OK(); + } + } + } + } + return Status::NotFound("File not present in any level"); +} + +void VersionSet::GetLiveFilesMetaData(std::vector* metadata) { + for (auto cfd : *column_family_set_) { + if (cfd->IsDropped() || !cfd->initialized()) { + continue; + } + for (int level = 0; level < cfd->NumberLevels(); level++) { + for (const auto& file : + cfd->current()->storage_info()->LevelFiles(level)) { + LiveFileMetaData filemetadata; + filemetadata.column_family_name = cfd->GetName(); + uint32_t path_id = file->fd.GetPathId(); + if (path_id < cfd->ioptions()->cf_paths.size()) { + filemetadata.db_path = cfd->ioptions()->cf_paths[path_id].path; + } else { + assert(!cfd->ioptions()->cf_paths.empty()); + filemetadata.db_path = cfd->ioptions()->cf_paths.back().path; + } + filemetadata.directory = filemetadata.db_path; + const uint64_t file_number = file->fd.GetNumber(); + filemetadata.name = MakeTableFileName("", file_number); + filemetadata.relative_filename = filemetadata.name.substr(1); + filemetadata.file_number = file_number; + filemetadata.level = level; + filemetadata.size = file->fd.GetFileSize(); + filemetadata.smallestkey = file->smallest.user_key().ToString(); + filemetadata.largestkey = file->largest.user_key().ToString(); + filemetadata.smallest_seqno = file->fd.smallest_seqno; + filemetadata.largest_seqno = file->fd.largest_seqno; + filemetadata.num_reads_sampled = + file->stats.num_reads_sampled.load(std::memory_order_relaxed); + filemetadata.being_compacted = file->being_compacted; + filemetadata.num_entries = file->num_entries; + filemetadata.num_deletions = file->num_deletions; + filemetadata.oldest_blob_file_number = file->oldest_blob_file_number; + filemetadata.file_checksum = file->file_checksum; + filemetadata.file_checksum_func_name = file->file_checksum_func_name; + filemetadata.temperature = file->temperature; + filemetadata.oldest_ancester_time = file->TryGetOldestAncesterTime(); + filemetadata.file_creation_time = file->TryGetFileCreationTime(); + filemetadata.epoch_number = file->epoch_number; + metadata->push_back(filemetadata); + } + } + } +} + +void VersionSet::GetObsoleteFiles(std::vector* files, + std::vector* blob_files, + std::vector* manifest_filenames, + uint64_t min_pending_output) { + assert(files); + assert(blob_files); + assert(manifest_filenames); + assert(files->empty()); + assert(blob_files->empty()); + assert(manifest_filenames->empty()); + + std::vector pending_files; + for (auto& f : obsolete_files_) { + if (f.metadata->fd.GetNumber() < min_pending_output) { + files->emplace_back(std::move(f)); + } else { + pending_files.emplace_back(std::move(f)); + } + } + obsolete_files_.swap(pending_files); + + std::vector pending_blob_files; + for (auto& blob_file : obsolete_blob_files_) { + if (blob_file.GetBlobFileNumber() < min_pending_output) { + blob_files->emplace_back(std::move(blob_file)); + } else { + pending_blob_files.emplace_back(std::move(blob_file)); + } + } + obsolete_blob_files_.swap(pending_blob_files); + + obsolete_manifests_.swap(*manifest_filenames); +} + +uint64_t VersionSet::GetObsoleteSstFilesSize() const { + uint64_t ret = 0; + for (auto& f : obsolete_files_) { + if (f.metadata != nullptr) { + ret += f.metadata->fd.GetFileSize(); + } + } + return ret; +} + +ColumnFamilyData* VersionSet::CreateColumnFamily( + const ColumnFamilyOptions& cf_options, const ReadOptions& read_options, + const VersionEdit* edit) { + assert(edit->is_column_family_add_); + + MutableCFOptions dummy_cf_options; + Version* dummy_versions = + new Version(nullptr, this, file_options_, dummy_cf_options, io_tracer_); + // Ref() dummy version once so that later we can call Unref() to delete it + // by avoiding calling "delete" explicitly (~Version is private) + dummy_versions->Ref(); + auto new_cfd = column_family_set_->CreateColumnFamily( + edit->column_family_name_, edit->column_family_, dummy_versions, + cf_options); + + Version* v = new Version(new_cfd, this, file_options_, + *new_cfd->GetLatestMutableCFOptions(), io_tracer_, + current_version_number_++); + + constexpr bool update_stats = false; + + v->PrepareAppend(*new_cfd->GetLatestMutableCFOptions(), read_options, + update_stats); + + AppendVersion(new_cfd, v); + // GetLatestMutableCFOptions() is safe here without mutex since the + // cfd is not available to client + new_cfd->CreateNewMemtable(*new_cfd->GetLatestMutableCFOptions(), + LastSequence()); + new_cfd->SetLogNumber(edit->log_number_); + return new_cfd; +} + +uint64_t VersionSet::GetNumLiveVersions(Version* dummy_versions) { + uint64_t count = 0; + for (Version* v = dummy_versions->next_; v != dummy_versions; v = v->next_) { + count++; + } + return count; +} + +uint64_t VersionSet::GetTotalSstFilesSize(Version* dummy_versions) { + std::unordered_set unique_files; + uint64_t total_files_size = 0; + for (Version* v = dummy_versions->next_; v != dummy_versions; v = v->next_) { + VersionStorageInfo* storage_info = v->storage_info(); + for (int level = 0; level < storage_info->num_levels_; level++) { + for (const auto& file_meta : storage_info->LevelFiles(level)) { + if (unique_files.find(file_meta->fd.packed_number_and_path_id) == + unique_files.end()) { + unique_files.insert(file_meta->fd.packed_number_and_path_id); + total_files_size += file_meta->fd.GetFileSize(); + } + } + } + } + return total_files_size; +} + +uint64_t VersionSet::GetTotalBlobFileSize(Version* dummy_versions) { + std::unordered_set unique_blob_files; + + uint64_t all_versions_blob_file_size = 0; + + for (auto* v = dummy_versions->next_; v != dummy_versions; v = v->next_) { + // iterate all the versions + const auto* vstorage = v->storage_info(); + assert(vstorage); + + const auto& blob_files = vstorage->GetBlobFiles(); + + for (const auto& meta : blob_files) { + assert(meta); + + const uint64_t blob_file_number = meta->GetBlobFileNumber(); + + if (unique_blob_files.find(blob_file_number) == unique_blob_files.end()) { + // find Blob file that has not been counted + unique_blob_files.insert(blob_file_number); + all_versions_blob_file_size += meta->GetBlobFileSize(); + } + } + } + + return all_versions_blob_file_size; +} + +Status VersionSet::VerifyFileMetadata(const ReadOptions& read_options, + ColumnFamilyData* cfd, + const std::string& fpath, int level, + const FileMetaData& meta) { + uint64_t fsize = 0; + Status status = fs_->GetFileSize(fpath, IOOptions(), &fsize, nullptr); + if (status.ok()) { + if (fsize != meta.fd.GetFileSize()) { + status = Status::Corruption("File size mismatch: " + fpath); + } + } + if (status.ok() && db_options_->verify_sst_unique_id_in_manifest) { + assert(cfd); + TableCache* table_cache = cfd->table_cache(); + assert(table_cache); + + const MutableCFOptions* const cf_opts = cfd->GetLatestMutableCFOptions(); + assert(cf_opts); + std::shared_ptr pe = cf_opts->prefix_extractor; + size_t max_sz_for_l0_meta_pin = MaxFileSizeForL0MetaPin(*cf_opts); + + const FileOptions& file_opts = file_options(); + + Version* version = cfd->current(); + assert(version); + VersionStorageInfo& storage_info = version->storage_info_; + const InternalKeyComparator* icmp = storage_info.InternalComparator(); + assert(icmp); + + InternalStats* internal_stats = cfd->internal_stats(); + + TableCache::TypedHandle* handle = nullptr; + FileMetaData meta_copy = meta; + status = table_cache->FindTable( + read_options, file_opts, *icmp, meta_copy, &handle, + cf_opts->block_protection_bytes_per_key, pe, + /*no_io=*/false, internal_stats->GetFileReadHist(level), false, level, + /*prefetch_index_and_filter_in_cache*/ false, max_sz_for_l0_meta_pin, + meta_copy.temperature); + if (handle) { + table_cache->get_cache().Release(handle); + } + } + return status; +} + +ReactiveVersionSet::ReactiveVersionSet( + const std::string& dbname, const ImmutableDBOptions* _db_options, + const FileOptions& _file_options, Cache* table_cache, + WriteBufferManager* write_buffer_manager, WriteController* write_controller, + const std::shared_ptr& io_tracer) + : VersionSet(dbname, _db_options, _file_options, table_cache, + write_buffer_manager, write_controller, + /*block_cache_tracer=*/nullptr, io_tracer, /*db_id*/ "", + /*db_session_id*/ "") {} + +ReactiveVersionSet::~ReactiveVersionSet() {} + +Status ReactiveVersionSet::Recover( + const std::vector& column_families, + std::unique_ptr* manifest_reader, + std::unique_ptr* manifest_reporter, + std::unique_ptr* manifest_reader_status) { + assert(manifest_reader != nullptr); + assert(manifest_reporter != nullptr); + assert(manifest_reader_status != nullptr); + + manifest_reader_status->reset(new Status()); + manifest_reporter->reset(new LogReporter()); + static_cast_with_check(manifest_reporter->get())->status = + manifest_reader_status->get(); + Status s = MaybeSwitchManifest(manifest_reporter->get(), manifest_reader); + if (!s.ok()) { + return s; + } + log::Reader* reader = manifest_reader->get(); + assert(reader); + + manifest_tailer_.reset(new ManifestTailer( + column_families, const_cast(this), io_tracer_, + read_options_, EpochNumberRequirement::kMightMissing)); + + manifest_tailer_->Iterate(*reader, manifest_reader_status->get()); + + s = manifest_tailer_->status(); + if (s.ok()) { + RecoverEpochNumbers(); + } + return s; +} + +Status ReactiveVersionSet::ReadAndApply( + InstrumentedMutex* mu, + std::unique_ptr* manifest_reader, + Status* manifest_read_status, + std::unordered_set* cfds_changed) { + assert(manifest_reader != nullptr); + assert(cfds_changed != nullptr); + mu->AssertHeld(); + + Status s; + log::Reader* reader = manifest_reader->get(); + assert(reader); + s = MaybeSwitchManifest(reader->GetReporter(), manifest_reader); + if (!s.ok()) { + return s; + } + manifest_tailer_->Iterate(*(manifest_reader->get()), manifest_read_status); + s = manifest_tailer_->status(); + if (s.ok()) { + *cfds_changed = std::move(manifest_tailer_->GetUpdatedColumnFamilies()); + } + + return s; +} + +Status ReactiveVersionSet::MaybeSwitchManifest( + log::Reader::Reporter* reporter, + std::unique_ptr* manifest_reader) { + assert(manifest_reader != nullptr); + Status s; + std::string manifest_path; + s = GetCurrentManifestPath(dbname_, fs_.get(), &manifest_path, + &manifest_file_number_); + if (!s.ok()) { + return s; + } + std::unique_ptr manifest_file; + if (manifest_reader->get() != nullptr && + manifest_reader->get()->file()->file_name() == manifest_path) { + // CURRENT points to the same MANIFEST as before, no need to switch + // MANIFEST. + return s; + } + assert(nullptr == manifest_reader->get() || + manifest_reader->get()->file()->file_name() != manifest_path); + s = fs_->FileExists(manifest_path, IOOptions(), nullptr); + if (s.IsNotFound()) { + return Status::TryAgain( + "The primary may have switched to a new MANIFEST and deleted the old " + "one."); + } else if (!s.ok()) { + return s; + } + TEST_SYNC_POINT( + "ReactiveVersionSet::MaybeSwitchManifest:" + "AfterGetCurrentManifestPath:0"); + TEST_SYNC_POINT( + "ReactiveVersionSet::MaybeSwitchManifest:" + "AfterGetCurrentManifestPath:1"); + // The primary can also delete the MANIFEST while the secondary is reading + // it. This is OK on POSIX. For other file systems, maybe create a hard link + // to MANIFEST. The hard link should be cleaned up later by the secondary. + s = fs_->NewSequentialFile(manifest_path, + fs_->OptimizeForManifestRead(file_options_), + &manifest_file, nullptr); + std::unique_ptr manifest_file_reader; + if (s.ok()) { + manifest_file_reader.reset(new SequentialFileReader( + std::move(manifest_file), manifest_path, + db_options_->log_readahead_size, io_tracer_, db_options_->listeners)); + manifest_reader->reset(new log::FragmentBufferedReader( + nullptr, std::move(manifest_file_reader), reporter, true /* checksum */, + 0 /* log_number */)); + ROCKS_LOG_INFO(db_options_->info_log, "Switched to new manifest: %s\n", + manifest_path.c_str()); + if (manifest_tailer_) { + manifest_tailer_->PrepareToReadNewManifest(); + } + } else if (s.IsPathNotFound()) { + // This can happen if the primary switches to a new MANIFEST after the + // secondary reads the CURRENT file but before the secondary actually tries + // to open the MANIFEST. + s = Status::TryAgain( + "The primary may have switched to a new MANIFEST and deleted the old " + "one."); + } + return s; +} + +#ifndef NDEBUG +uint64_t ReactiveVersionSet::TEST_read_edits_in_atomic_group() const { + assert(manifest_tailer_); + return manifest_tailer_->GetReadBuffer().TEST_read_edits_in_atomic_group(); +} +#endif // !NDEBUG + +std::vector& ReactiveVersionSet::replay_buffer() { + assert(manifest_tailer_); + return manifest_tailer_->GetReadBuffer().replay_buffer(); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/version_set.h b/librocksdb-sys/rocksdb/db/version_set.h new file mode 100644 index 0000000..e32d0ff --- /dev/null +++ b/librocksdb-sys/rocksdb/db/version_set.h @@ -0,0 +1,1730 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// The representation of a DBImpl consists of a set of Versions. The +// newest version is called "current". Older versions may be kept +// around to provide a consistent view to live iterators. +// +// Each Version keeps track of a set of table files per level, as well as a +// set of blob files. The entire set of versions is maintained in a +// VersionSet. +// +// Version,VersionSet are thread-compatible, but require external +// synchronization on all accesses. + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cache/cache_helpers.h" +#include "db/blob/blob_file_meta.h" +#include "db/blob/blob_index.h" +#include "db/column_family.h" +#include "db/compaction/compaction.h" +#include "db/compaction/compaction_picker.h" +#include "db/dbformat.h" +#include "db/file_indexer.h" +#include "db/log_reader.h" +#include "db/range_del_aggregator.h" +#include "db/read_callback.h" +#include "db/table_cache.h" +#include "db/version_builder.h" +#include "db/version_edit.h" +#include "db/write_controller.h" +#include "env/file_system_tracer.h" +#if USE_COROUTINES +#include "folly/experimental/coro/BlockingWait.h" +#include "folly/experimental/coro/Collect.h" +#endif +#include "monitoring/instrumented_mutex.h" +#include "options/db_options.h" +#include "port/port.h" +#include "rocksdb/env.h" +#include "rocksdb/file_checksum.h" +#include "table/get_context.h" +#include "table/multiget_context.h" +#include "trace_replay/block_cache_tracer.h" +#include "util/autovector.h" +#include "util/coro_utils.h" +#include "util/hash_containers.h" + +namespace ROCKSDB_NAMESPACE { + +namespace log { +class Writer; +} + +class BlobIndex; +class Compaction; +class LogBuffer; +class LookupKey; +class MemTable; +class Version; +class VersionSet; +class WriteBufferManager; +class MergeContext; +class ColumnFamilySet; +class MergeIteratorBuilder; +class SystemClock; +class ManifestTailer; +class FilePickerMultiGet; + +// VersionEdit is always supposed to be valid and it is used to point at +// entries in Manifest. Ideally it should not be used as a container to +// carry around few of its fields as function params because it can cause +// readers to think it's a valid entry from Manifest. To avoid that confusion +// introducing VersionEditParams to simply carry around multiple VersionEdit +// params. It need not point to a valid record in Manifest. +using VersionEditParams = VersionEdit; + +// Return the smallest index i such that file_level.files[i]->largest >= key. +// Return file_level.num_files if there is no such file. +// REQUIRES: "file_level.files" contains a sorted list of +// non-overlapping files. +extern int FindFile(const InternalKeyComparator& icmp, + const LevelFilesBrief& file_level, const Slice& key); + +// Returns true iff some file in "files" overlaps the user key range +// [*smallest,*largest]. +// smallest==nullptr represents a key smaller than all keys in the DB. +// largest==nullptr represents a key largest than all keys in the DB. +// REQUIRES: If disjoint_sorted_files, file_level.files[] +// contains disjoint ranges in sorted order. +extern bool SomeFileOverlapsRange(const InternalKeyComparator& icmp, + bool disjoint_sorted_files, + const LevelFilesBrief& file_level, + const Slice* smallest_user_key, + const Slice* largest_user_key); + +// Generate LevelFilesBrief from vector +// Would copy smallest_key and largest_key data to sequential memory +// arena: Arena used to allocate the memory +extern void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level, + const std::vector& files, + Arena* arena); +enum EpochNumberRequirement { + kMightMissing, + kMustPresent, +}; + +// Information of the storage associated with each Version, including number of +// levels of LSM tree, files information at each level, files marked for +// compaction, blob files, etc. +class VersionStorageInfo { + public: + VersionStorageInfo(const InternalKeyComparator* internal_comparator, + const Comparator* user_comparator, int num_levels, + CompactionStyle compaction_style, + VersionStorageInfo* src_vstorage, + bool _force_consistency_checks, + EpochNumberRequirement epoch_number_requirement = + EpochNumberRequirement::kMustPresent); + // No copying allowed + VersionStorageInfo(const VersionStorageInfo&) = delete; + void operator=(const VersionStorageInfo&) = delete; + ~VersionStorageInfo(); + + void Reserve(int level, size_t size) { files_[level].reserve(size); } + + void AddFile(int level, FileMetaData* f); + + // Resize/Initialize the space for compact_cursor_ + void ResizeCompactCursors(int level) { + compact_cursor_.resize(level, InternalKey()); + } + + const std::vector& GetCompactCursors() const { + return compact_cursor_; + } + + // REQUIRES: ResizeCompactCursors has been called + void AddCursorForOneLevel(int level, + const InternalKey& smallest_uncompacted_key) { + compact_cursor_[level] = smallest_uncompacted_key; + } + + // REQUIRES: lock is held + // Update the compact cursor and advance the file index using increment + // so that it can point to the next cursor (increment means the number of + // input files in this level of the last compaction) + const InternalKey& GetNextCompactCursor(int level, size_t increment) { + int cmp_idx = next_file_to_compact_by_size_[level] + (int)increment; + assert(cmp_idx <= (int)files_by_compaction_pri_[level].size()); + // TODO(zichen): may need to update next_file_to_compact_by_size_ + // for parallel compaction. + InternalKey new_cursor; + if (cmp_idx >= (int)files_by_compaction_pri_[level].size()) { + cmp_idx = 0; + } + // TODO(zichen): rethink if this strategy gives us some good guarantee + return files_[level][files_by_compaction_pri_[level][cmp_idx]]->smallest; + } + + void ReserveBlob(size_t size) { blob_files_.reserve(size); } + + void AddBlobFile(std::shared_ptr blob_file_meta); + + void PrepareForVersionAppend(const ImmutableOptions& immutable_options, + const MutableCFOptions& mutable_cf_options); + + // REQUIRES: PrepareForVersionAppend has been called + void SetFinalized(); + + // Update the accumulated stats from a file-meta. + void UpdateAccumulatedStats(FileMetaData* file_meta); + + // Decrease the current stat from a to-be-deleted file-meta + void RemoveCurrentStats(FileMetaData* file_meta); + + // Updates internal structures that keep track of compaction scores + // We use compaction scores to figure out which compaction to do next + // REQUIRES: db_mutex held!! + // TODO find a better way to pass compaction_options_fifo. + void ComputeCompactionScore(const ImmutableOptions& immutable_options, + const MutableCFOptions& mutable_cf_options); + + // Estimate est_comp_needed_bytes_ + void EstimateCompactionBytesNeeded( + const MutableCFOptions& mutable_cf_options); + + // This computes files_marked_for_compaction_ and is called by + // ComputeCompactionScore() + void ComputeFilesMarkedForCompaction(int last_level); + + // This computes ttl_expired_files_ and is called by + // ComputeCompactionScore() + void ComputeExpiredTtlFiles(const ImmutableOptions& ioptions, + const uint64_t ttl); + + // This computes files_marked_for_periodic_compaction_ and is called by + // ComputeCompactionScore() + void ComputeFilesMarkedForPeriodicCompaction( + const ImmutableOptions& ioptions, + const uint64_t periodic_compaction_seconds, int last_level); + + // This computes bottommost_files_marked_for_compaction_ and is called by + // ComputeCompactionScore() or UpdateOldestSnapshot(). + // + // Among bottommost files (assumes they've already been computed), marks the + // ones that have keys that would be eliminated if recompacted, according to + // the seqnum of the oldest existing snapshot. Must be called every time + // oldest snapshot changes as that is when bottom-level files can become + // eligible for compaction. + // + // REQUIRES: DB mutex held + void ComputeBottommostFilesMarkedForCompaction(); + + // This computes files_marked_for_forced_blob_gc_ and is called by + // ComputeCompactionScore() + // + // REQUIRES: DB mutex held + void ComputeFilesMarkedForForcedBlobGC( + double blob_garbage_collection_age_cutoff, + double blob_garbage_collection_force_threshold); + + bool level0_non_overlapping() const { return level0_non_overlapping_; } + + // Updates the oldest snapshot and related internal state, like the bottommost + // files marked for compaction. + // REQUIRES: DB mutex held + void UpdateOldestSnapshot(SequenceNumber oldest_snapshot_seqnum); + + int MaxInputLevel() const; + int MaxOutputLevel(bool allow_ingest_behind) const; + + // Return level number that has idx'th highest score + int CompactionScoreLevel(int idx) const { return compaction_level_[idx]; } + + // Return idx'th highest score + double CompactionScore(int idx) const { return compaction_score_[idx]; } + + void GetOverlappingInputs( + int level, const InternalKey* begin, // nullptr means before all keys + const InternalKey* end, // nullptr means after all keys + std::vector* inputs, + int hint_index = -1, // index of overlap file + int* file_index = nullptr, // return index of overlap file + bool expand_range = true, // if set, returns files which overlap the + // range and overlap each other. If false, + // then just files intersecting the range + InternalKey** next_smallest = nullptr) // if non-null, returns the + const; // smallest key of next file not included + void GetCleanInputsWithinInterval( + int level, const InternalKey* begin, // nullptr means before all keys + const InternalKey* end, // nullptr means after all keys + std::vector* inputs, + int hint_index = -1, // index of overlap file + int* file_index = nullptr) // return index of overlap file + const; + + void GetOverlappingInputsRangeBinarySearch( + int level, // level > 0 + const InternalKey* begin, // nullptr means before all keys + const InternalKey* end, // nullptr means after all keys + std::vector* inputs, + int hint_index, // index of overlap file + int* file_index, // return index of overlap file + bool within_interval = false, // if set, force the inputs within interval + InternalKey** next_smallest = nullptr) // if non-null, returns the + const; // smallest key of next file not included + + // Returns true iff some file in the specified level overlaps + // some part of [*smallest_user_key,*largest_user_key]. + // smallest_user_key==NULL represents a key smaller than all keys in the DB. + // largest_user_key==NULL represents a key largest than all keys in the DB. + bool OverlapInLevel(int level, const Slice* smallest_user_key, + const Slice* largest_user_key); + + // Returns true iff the first or last file in inputs contains + // an overlapping user key to the file "just outside" of it (i.e. + // just after the last file, or just before the first file) + // REQUIRES: "*inputs" is a sorted list of non-overlapping files + bool HasOverlappingUserKey(const std::vector* inputs, + int level); + + int num_levels() const { return num_levels_; } + + // REQUIRES: PrepareForVersionAppend has been called + int num_non_empty_levels() const { + assert(finalized_); + return num_non_empty_levels_; + } + + // REQUIRES: PrepareForVersionAppend has been called + // This may or may not return number of level files. It is to keep backward + // compatible behavior in universal compaction. + int l0_delay_trigger_count() const { return l0_delay_trigger_count_; } + + void set_l0_delay_trigger_count(int v) { l0_delay_trigger_count_ = v; } + + // REQUIRES: This version has been saved (see VersionBuilder::SaveTo) + int NumLevelFiles(int level) const { + assert(finalized_); + return static_cast(files_[level].size()); + } + + // Return the combined file size of all files at the specified level. + uint64_t NumLevelBytes(int level) const; + + // REQUIRES: This version has been saved (see VersionBuilder::SaveTo) + const std::vector& LevelFiles(int level) const { + return files_[level]; + } + + bool HasMissingEpochNumber() const; + uint64_t GetMaxEpochNumberOfFiles() const; + EpochNumberRequirement GetEpochNumberRequirement() const { + return epoch_number_requirement_; + } + void SetEpochNumberRequirement( + EpochNumberRequirement epoch_number_requirement) { + epoch_number_requirement_ = epoch_number_requirement; + } + void RecoverEpochNumbers(ColumnFamilyData* cfd); + + class FileLocation { + public: + FileLocation() = default; + FileLocation(int level, size_t position) + : level_(level), position_(position) {} + + int GetLevel() const { return level_; } + size_t GetPosition() const { return position_; } + + bool IsValid() const { return level_ >= 0; } + + bool operator==(const FileLocation& rhs) const { + return level_ == rhs.level_ && position_ == rhs.position_; + } + + bool operator!=(const FileLocation& rhs) const { return !(*this == rhs); } + + static FileLocation Invalid() { return FileLocation(); } + + private: + int level_ = -1; + size_t position_ = 0; + }; + + // REQUIRES: PrepareForVersionAppend has been called + FileLocation GetFileLocation(uint64_t file_number) const { + const auto it = file_locations_.find(file_number); + + if (it == file_locations_.end()) { + return FileLocation::Invalid(); + } + + assert(it->second.GetLevel() < num_levels_); + assert(it->second.GetPosition() < files_[it->second.GetLevel()].size()); + assert(files_[it->second.GetLevel()][it->second.GetPosition()]); + assert(files_[it->second.GetLevel()][it->second.GetPosition()] + ->fd.GetNumber() == file_number); + + return it->second; + } + + // REQUIRES: PrepareForVersionAppend has been called + FileMetaData* GetFileMetaDataByNumber(uint64_t file_number) const { + auto location = GetFileLocation(file_number); + + if (!location.IsValid()) { + return nullptr; + } + + return files_[location.GetLevel()][location.GetPosition()]; + } + + // REQUIRES: This version has been saved (see VersionBuilder::SaveTo) + using BlobFiles = std::vector>; + const BlobFiles& GetBlobFiles() const { return blob_files_; } + + // REQUIRES: This version has been saved (see VersionBuilder::SaveTo) + BlobFiles::const_iterator GetBlobFileMetaDataLB( + uint64_t blob_file_number) const; + + // REQUIRES: This version has been saved (see VersionBuilder::SaveTo) + std::shared_ptr GetBlobFileMetaData( + uint64_t blob_file_number) const { + const auto it = GetBlobFileMetaDataLB(blob_file_number); + + assert(it == blob_files_.end() || *it); + + if (it != blob_files_.end() && + (*it)->GetBlobFileNumber() == blob_file_number) { + return *it; + } + + return std::shared_ptr(); + } + + // REQUIRES: This version has been saved (see VersionBuilder::SaveTo) + struct BlobStats { + uint64_t total_file_size = 0; + uint64_t total_garbage_size = 0; + double space_amp = 0.0; + }; + + BlobStats GetBlobStats() const { + uint64_t total_file_size = 0; + uint64_t total_garbage_size = 0; + + for (const auto& meta : blob_files_) { + assert(meta); + + total_file_size += meta->GetBlobFileSize(); + total_garbage_size += meta->GetGarbageBlobBytes(); + } + + double space_amp = 0.0; + if (total_file_size > total_garbage_size) { + space_amp = static_cast(total_file_size) / + (total_file_size - total_garbage_size); + } + + return BlobStats{total_file_size, total_garbage_size, space_amp}; + } + + const ROCKSDB_NAMESPACE::LevelFilesBrief& LevelFilesBrief(int level) const { + assert(level < static_cast(level_files_brief_.size())); + return level_files_brief_[level]; + } + + // REQUIRES: PrepareForVersionAppend has been called + const std::vector& FilesByCompactionPri(int level) const { + assert(finalized_); + return files_by_compaction_pri_[level]; + } + + // REQUIRES: ComputeCompactionScore has been called + // REQUIRES: DB mutex held during access + const autovector>& FilesMarkedForCompaction() + const { + assert(finalized_); + return files_marked_for_compaction_; + } + + void TEST_AddFileMarkedForCompaction(int level, FileMetaData* f) { + f->marked_for_compaction = true; + files_marked_for_compaction_.emplace_back(level, f); + } + + // REQUIRES: ComputeCompactionScore has been called + // REQUIRES: DB mutex held during access + // Used by Leveled Compaction only. + const autovector>& ExpiredTtlFiles() const { + assert(finalized_); + return expired_ttl_files_; + } + + // REQUIRES: ComputeCompactionScore has been called + // REQUIRES: DB mutex held during access + // Used by Leveled and Universal Compaction. + const autovector>& + FilesMarkedForPeriodicCompaction() const { + assert(finalized_); + return files_marked_for_periodic_compaction_; + } + + void TEST_AddFileMarkedForPeriodicCompaction(int level, FileMetaData* f) { + files_marked_for_periodic_compaction_.emplace_back(level, f); + } + + // REQUIRES: ComputeCompactionScore has been called + // REQUIRES: DB mutex held during access + const autovector>& + BottommostFilesMarkedForCompaction() const { + assert(finalized_); + return bottommost_files_marked_for_compaction_; + } + + // REQUIRES: ComputeCompactionScore has been called + // REQUIRES: DB mutex held during access + const autovector>& FilesMarkedForForcedBlobGC() + const { + assert(finalized_); + return files_marked_for_forced_blob_gc_; + } + + int base_level() const { return base_level_; } + double level_multiplier() const { return level_multiplier_; } + + // REQUIRES: lock is held + // Set the index that is used to offset into files_by_compaction_pri_ to find + // the next compaction candidate file. + void SetNextCompactionIndex(int level, int index) { + next_file_to_compact_by_size_[level] = index; + } + + // REQUIRES: lock is held + int NextCompactionIndex(int level) const { + return next_file_to_compact_by_size_[level]; + } + + // REQUIRES: PrepareForVersionAppend has been called + const FileIndexer& file_indexer() const { + assert(finalized_); + return file_indexer_; + } + + // Only the first few entries of files_by_compaction_pri_ are sorted. + // There is no need to sort all the files because it is likely + // that on a running system, we need to look at only the first + // few largest files because a new version is created every few + // seconds/minutes (because of concurrent compactions). + static const size_t kNumberFilesToSort = 50; + + // Return a human-readable short (single-line) summary of the number + // of files per level. Uses *scratch as backing store. + struct LevelSummaryStorage { + char buffer[1000]; + }; + struct FileSummaryStorage { + char buffer[3000]; + }; + const char* LevelSummary(LevelSummaryStorage* scratch) const; + // Return a human-readable short (single-line) summary of files + // in a specified level. Uses *scratch as backing store. + const char* LevelFileSummary(FileSummaryStorage* scratch, int level) const; + + // Return the maximum overlapping data (in bytes) at next level for any + // file at a level >= 1. + uint64_t MaxNextLevelOverlappingBytes(); + + // Return a human readable string that describes this version's contents. + std::string DebugString(bool hex = false) const; + + uint64_t GetAverageValueSize() const { + if (accumulated_num_non_deletions_ == 0) { + return 0; + } + assert(accumulated_raw_key_size_ + accumulated_raw_value_size_ > 0); + assert(accumulated_file_size_ > 0); + return accumulated_raw_value_size_ / accumulated_num_non_deletions_ * + accumulated_file_size_ / + (accumulated_raw_key_size_ + accumulated_raw_value_size_); + } + + uint64_t GetEstimatedActiveKeys() const; + + double GetEstimatedCompressionRatioAtLevel(int level) const; + + // re-initializes the index that is used to offset into + // files_by_compaction_pri_ + // to find the next compaction candidate file. + void ResetNextCompactionIndex(int level) { + next_file_to_compact_by_size_[level] = 0; + } + + const InternalKeyComparator* InternalComparator() const { + return internal_comparator_; + } + + // Returns maximum total bytes of data on a given level. + uint64_t MaxBytesForLevel(int level) const; + + // Returns an estimate of the amount of live data in bytes. + uint64_t EstimateLiveDataSize() const; + + uint64_t estimated_compaction_needed_bytes() const { + return estimated_compaction_needed_bytes_; + } + + void TEST_set_estimated_compaction_needed_bytes(uint64_t v) { + estimated_compaction_needed_bytes_ = v; + } + + bool force_consistency_checks() const { return force_consistency_checks_; } + + SequenceNumber bottommost_files_mark_threshold() const { + return bottommost_files_mark_threshold_; + } + + // Returns whether any key in [`smallest_key`, `largest_key`] could appear in + // an older L0 file than `last_l0_idx` or in a greater level than `last_level` + // + // @param last_level Level after which we check for overlap + // @param last_l0_idx If `last_level == 0`, index of L0 file after which we + // check for overlap; otherwise, must be -1 + bool RangeMightExistAfterSortedRun(const Slice& smallest_user_key, + const Slice& largest_user_key, + int last_level, int last_l0_idx); + + private: + void ComputeCompensatedSizes(); + void UpdateNumNonEmptyLevels(); + void CalculateBaseBytes(const ImmutableOptions& ioptions, + const MutableCFOptions& options); + void UpdateFilesByCompactionPri(const ImmutableOptions& immutable_options, + const MutableCFOptions& mutable_cf_options); + + void GenerateFileIndexer() { + file_indexer_.UpdateIndex(&arena_, num_non_empty_levels_, files_); + } + + void GenerateLevelFilesBrief(); + void GenerateLevel0NonOverlapping(); + void GenerateBottommostFiles(); + void GenerateFileLocationIndex(); + + const InternalKeyComparator* internal_comparator_; + const Comparator* user_comparator_; + int num_levels_; // Number of levels + int num_non_empty_levels_; // Number of levels. Any level larger than it + // is guaranteed to be empty. + // Per-level max bytes + std::vector level_max_bytes_; + + // A short brief metadata of files per level + autovector level_files_brief_; + FileIndexer file_indexer_; + Arena arena_; // Used to allocate space for file_levels_ + + CompactionStyle compaction_style_; + + // List of files per level, files in each level are arranged + // in increasing order of keys + std::vector* files_; + + // Map of all table files in version. Maps file number to (level, position on + // level). + using FileLocations = UnorderedMap; + FileLocations file_locations_; + + // Vector of blob files in version sorted by blob file number. + BlobFiles blob_files_; + + // Level that L0 data should be compacted to. All levels < base_level_ should + // be empty. -1 if it is not level-compaction so it's not applicable. + int base_level_; + + // Applies to level compaction when + // `level_compaction_dynamic_level_bytes=true`. All non-empty levels <= + // lowest_unnecessary_level_ are not needed and will be drained automatically. + // -1 if there is no unnecessary level, + int lowest_unnecessary_level_; + + double level_multiplier_; + + // A list for the same set of files that are stored in files_, + // but files in each level are now sorted based on file + // size. The file with the largest size is at the front. + // This vector stores the index of the file from files_. + std::vector> files_by_compaction_pri_; + + // If true, means that files in L0 have keys with non overlapping ranges + bool level0_non_overlapping_; + + // An index into files_by_compaction_pri_ that specifies the first + // file that is not yet compacted + std::vector next_file_to_compact_by_size_; + + // Only the first few entries of files_by_compaction_pri_ are sorted. + // There is no need to sort all the files because it is likely + // that on a running system, we need to look at only the first + // few largest files because a new version is created every few + // seconds/minutes (because of concurrent compactions). + static const size_t number_of_files_to_sort_ = 50; + + // This vector contains list of files marked for compaction and also not + // currently being compacted. It is protected by DB mutex. It is calculated in + // ComputeCompactionScore(). Used by Leveled and Universal Compaction. + autovector> files_marked_for_compaction_; + + autovector> expired_ttl_files_; + + autovector> + files_marked_for_periodic_compaction_; + + // These files are considered bottommost because none of their keys can exist + // at lower levels. They are not necessarily all in the same level. The marked + // ones are eligible for compaction because they contain duplicate key + // versions that are no longer protected by snapshot. These variables are + // protected by DB mutex and are calculated in `GenerateBottommostFiles()` and + // `ComputeBottommostFilesMarkedForCompaction()`. + autovector> bottommost_files_; + autovector> + bottommost_files_marked_for_compaction_; + + autovector> files_marked_for_forced_blob_gc_; + + // Threshold for needing to mark another bottommost file. Maintain it so we + // can quickly check when releasing a snapshot whether more bottommost files + // became eligible for compaction. It's defined as the min of the max nonzero + // seqnums of unmarked bottommost files. + SequenceNumber bottommost_files_mark_threshold_ = kMaxSequenceNumber; + + // Monotonically increases as we release old snapshots. Zero indicates no + // snapshots have been released yet. When no snapshots remain we set it to the + // current seqnum, which needs to be protected as a snapshot can still be + // created that references it. + SequenceNumber oldest_snapshot_seqnum_ = 0; + + // Level that should be compacted next and its compaction score. + // Score < 1 means compaction is not strictly needed. These fields + // are initialized by ComputeCompactionScore. + // The most critical level to be compacted is listed first + // These are used to pick the best compaction level + std::vector compaction_score_; + std::vector compaction_level_; + int l0_delay_trigger_count_ = 0; // Count used to trigger slow down and stop + // for number of L0 files. + + // Compact cursors for round-robin compactions in each level + std::vector compact_cursor_; + + // the following are the sampled temporary stats. + // the current accumulated size of sampled files. + uint64_t accumulated_file_size_; + // the current accumulated size of all raw keys based on the sampled files. + uint64_t accumulated_raw_key_size_; + // the current accumulated size of all raw keys based on the sampled files. + uint64_t accumulated_raw_value_size_; + // total number of non-deletion entries + uint64_t accumulated_num_non_deletions_; + // total number of deletion entries + uint64_t accumulated_num_deletions_; + // current number of non_deletion entries + uint64_t current_num_non_deletions_; + // current number of deletion entries + uint64_t current_num_deletions_; + // current number of file samples + uint64_t current_num_samples_; + // Estimated bytes needed to be compacted until all levels' size is down to + // target sizes. + uint64_t estimated_compaction_needed_bytes_; + + bool finalized_; + + // If set to true, we will run consistency checks even if RocksDB + // is compiled in release mode + bool force_consistency_checks_; + + EpochNumberRequirement epoch_number_requirement_; + + friend class Version; + friend class VersionSet; +}; + +struct ObsoleteFileInfo { + FileMetaData* metadata; + std::string path; + // If true, the FileMataData should be destroyed but the file should + // not be deleted. This is because another FileMetaData still references + // the file, usually because the file is trivial moved so two FileMetadata + // is managing the file. + bool only_delete_metadata = false; + + ObsoleteFileInfo() noexcept + : metadata(nullptr), only_delete_metadata(false) {} + ObsoleteFileInfo(FileMetaData* f, const std::string& file_path, + std::shared_ptr + file_metadata_cache_res_mgr_arg = nullptr) + : metadata(f), + path(file_path), + only_delete_metadata(false), + file_metadata_cache_res_mgr(file_metadata_cache_res_mgr_arg) {} + + ObsoleteFileInfo(const ObsoleteFileInfo&) = delete; + ObsoleteFileInfo& operator=(const ObsoleteFileInfo&) = delete; + + ObsoleteFileInfo(ObsoleteFileInfo&& rhs) noexcept : ObsoleteFileInfo() { + *this = std::move(rhs); + } + + ObsoleteFileInfo& operator=(ObsoleteFileInfo&& rhs) noexcept { + path = std::move(rhs.path); + metadata = rhs.metadata; + rhs.metadata = nullptr; + file_metadata_cache_res_mgr = rhs.file_metadata_cache_res_mgr; + rhs.file_metadata_cache_res_mgr = nullptr; + + return *this; + } + void DeleteMetadata() { + if (file_metadata_cache_res_mgr) { + Status s = file_metadata_cache_res_mgr->UpdateCacheReservation( + metadata->ApproximateMemoryUsage(), false /* increase */); + s.PermitUncheckedError(); + } + delete metadata; + metadata = nullptr; + } + + private: + std::shared_ptr file_metadata_cache_res_mgr; +}; + +class ObsoleteBlobFileInfo { + public: + ObsoleteBlobFileInfo(uint64_t blob_file_number, std::string path) + : blob_file_number_(blob_file_number), path_(std::move(path)) {} + + uint64_t GetBlobFileNumber() const { return blob_file_number_; } + const std::string& GetPath() const { return path_; } + + private: + uint64_t blob_file_number_; + std::string path_; +}; + +using MultiGetRange = MultiGetContext::Range; +// A column family's version consists of the table and blob files owned by +// the column family at a certain point in time. +class Version { + public: + // Append to *iters a sequence of iterators that will + // yield the contents of this Version when merged together. + // @param read_options Must outlive any iterator built by + // `merger_iter_builder`. + void AddIterators(const ReadOptions& read_options, + const FileOptions& soptions, + MergeIteratorBuilder* merger_iter_builder, + bool allow_unprepared_value); + + // @param read_options Must outlive any iterator built by + // `merger_iter_builder`. + void AddIteratorsForLevel(const ReadOptions& read_options, + const FileOptions& soptions, + MergeIteratorBuilder* merger_iter_builder, + int level, bool allow_unprepared_value); + + Status OverlapWithLevelIterator(const ReadOptions&, const FileOptions&, + const Slice& smallest_user_key, + const Slice& largest_user_key, int level, + bool* overlap); + + // Lookup the value for key or get all merge operands for key. + // If do_merge = true (default) then lookup value for key. + // Behavior if do_merge = true: + // If found, store it in *value and + // return OK. Else return a non-OK status. + // Uses *operands to store merge_operator operations to apply later. + // + // If the ReadOptions.read_tier is set to do a read-only fetch, then + // *value_found will be set to false if it cannot be determined whether + // this value exists without doing IO. + // + // If the key is Deleted, *status will be set to NotFound and + // *key_exists will be set to true. + // If no key was found, *status will be set to NotFound and + // *key_exists will be set to false. + // If seq is non-null, *seq will be set to the sequence number found + // for the key if a key was found. + // Behavior if do_merge = false + // If the key has any merge operands then store them in + // merge_context.operands_list and don't merge the operands + // REQUIRES: lock is not held + // REQUIRES: pinned_iters_mgr != nullptr + void Get(const ReadOptions&, const LookupKey& key, PinnableSlice* value, + PinnableWideColumns* columns, std::string* timestamp, Status* status, + MergeContext* merge_context, + SequenceNumber* max_covering_tombstone_seq, + PinnedIteratorsManager* pinned_iters_mgr, + bool* value_found = nullptr, bool* key_exists = nullptr, + SequenceNumber* seq = nullptr, ReadCallback* callback = nullptr, + bool* is_blob = nullptr, bool do_merge = true); + + void MultiGet(const ReadOptions&, MultiGetRange* range, + ReadCallback* callback = nullptr); + + // Interprets blob_index_slice as a blob reference, and (assuming the + // corresponding blob file is part of this Version) retrieves the blob and + // saves it in *value. + // REQUIRES: blob_index_slice stores an encoded blob reference + Status GetBlob(const ReadOptions& read_options, const Slice& user_key, + const Slice& blob_index_slice, + FilePrefetchBuffer* prefetch_buffer, PinnableSlice* value, + uint64_t* bytes_read) const; + + // Retrieves a blob using a blob reference and saves it in *value, + // assuming the corresponding blob file is part of this Version. + Status GetBlob(const ReadOptions& read_options, const Slice& user_key, + const BlobIndex& blob_index, + FilePrefetchBuffer* prefetch_buffer, PinnableSlice* value, + uint64_t* bytes_read) const; + + struct BlobReadContext { + BlobReadContext(const BlobIndex& blob_idx, const KeyContext* key_ctx) + : blob_index(blob_idx), key_context(key_ctx) {} + + BlobIndex blob_index; + const KeyContext* key_context; + PinnableSlice result; + }; + + using BlobReadContexts = std::vector; + void MultiGetBlob(const ReadOptions& read_options, MultiGetRange& range, + std::unordered_map& blob_ctxs); + + // Loads some stats information from files (if update_stats is set) and + // populates derived data structures. Call without mutex held. It needs to be + // called before appending the version to the version set. + void PrepareAppend(const MutableCFOptions& mutable_cf_options, + const ReadOptions& read_options, bool update_stats); + + // Reference count management (so Versions do not disappear out from + // under live iterators) + void Ref(); + // Decrease reference count. Delete the object if no reference left + // and return true. Otherwise, return false. + bool Unref(); + + // Add all files listed in the current version to *live_table_files and + // *live_blob_files. + void AddLiveFiles(std::vector* live_table_files, + std::vector* live_blob_files) const; + + // Remove live files that are in the delete candidate lists. + void RemoveLiveFiles( + std::vector& sst_delete_candidates, + std::vector& blob_delete_candidates) const; + + // Return a human readable string that describes this version's contents. + std::string DebugString(bool hex = false, bool print_stats = false) const; + + // Returns the version number of this version + uint64_t GetVersionNumber() const { return version_number_; } + + // REQUIRES: lock is held + // On success, "tp" will contains the table properties of the file + // specified in "file_meta". If the file name of "file_meta" is + // known ahead, passing it by a non-null "fname" can save a + // file-name conversion. + Status GetTableProperties(const ReadOptions& read_options, + std::shared_ptr* tp, + const FileMetaData* file_meta, + const std::string* fname = nullptr) const; + + // REQUIRES: lock is held + // On success, *props will be populated with all SSTables' table properties. + // The keys of `props` are the sst file name, the values of `props` are the + // tables' properties, represented as std::shared_ptr. + Status GetPropertiesOfAllTables(const ReadOptions& read_options, + TablePropertiesCollection* props); + Status GetPropertiesOfAllTables(const ReadOptions& read_options, + TablePropertiesCollection* props, int level); + Status GetPropertiesOfTablesInRange(const ReadOptions& read_options, + const Range* range, std::size_t n, + TablePropertiesCollection* props) const; + + // Print summary of range delete tombstones in SST files into out_str, + // with maximum max_entries_to_print entries printed out. + Status TablesRangeTombstoneSummary(int max_entries_to_print, + std::string* out_str); + + // REQUIRES: lock is held + // On success, "tp" will contains the aggregated table property among + // the table properties of all sst files in this version. + Status GetAggregatedTableProperties( + const ReadOptions& read_options, + std::shared_ptr* tp, int level = -1); + + uint64_t GetEstimatedActiveKeys() { + return storage_info_.GetEstimatedActiveKeys(); + } + + size_t GetMemoryUsageByTableReaders(const ReadOptions& read_options); + + ColumnFamilyData* cfd() const { return cfd_; } + + // Return the next Version in the linked list. + Version* Next() const { return next_; } + + int TEST_refs() const { return refs_; } + + VersionStorageInfo* storage_info() { return &storage_info_; } + const VersionStorageInfo* storage_info() const { return &storage_info_; } + + VersionSet* version_set() { return vset_; } + + void GetColumnFamilyMetaData(ColumnFamilyMetaData* cf_meta); + + void GetSstFilesBoundaryKeys(Slice* smallest_user_key, + Slice* largest_user_key); + + uint64_t GetSstFilesSize(); + + // Retrieves the file_creation_time of the oldest file in the DB. + // Prerequisite for this API is max_open_files = -1 + void GetCreationTimeOfOldestFile(uint64_t* creation_time); + + const MutableCFOptions& GetMutableCFOptions() { return mutable_cf_options_; } + + InternalIterator* TEST_GetLevelIterator( + const ReadOptions& read_options, MergeIteratorBuilder* merge_iter_builder, + int level, bool allow_unprepared_value); + + private: + Env* env_; + SystemClock* clock_; + + friend class ReactiveVersionSet; + friend class VersionSet; + friend class VersionEditHandler; + friend class VersionEditHandlerPointInTime; + + const InternalKeyComparator* internal_comparator() const { + return storage_info_.internal_comparator_; + } + const Comparator* user_comparator() const { + return storage_info_.user_comparator_; + } + + // Returns true if the filter blocks in the specified level will not be + // checked during read operations. In certain cases (trivial move or preload), + // the filter block may already be cached, but we still do not access it such + // that it eventually expires from the cache. + bool IsFilterSkipped(int level, bool is_file_last_in_level = false); + + // The helper function of UpdateAccumulatedStats, which may fill the missing + // fields of file_meta from its associated TableProperties. + // Returns true if it does initialize FileMetaData. + bool MaybeInitializeFileMetaData(const ReadOptions& read_options, + FileMetaData* file_meta); + + // Update the accumulated stats associated with the current version. + // This accumulated stats will be used in compaction. + void UpdateAccumulatedStats(const ReadOptions& read_options); + + DECLARE_SYNC_AND_ASYNC( + /* ret_type */ Status, /* func_name */ MultiGetFromSST, + const ReadOptions& read_options, MultiGetRange file_range, + int hit_file_level, bool skip_filters, bool skip_range_deletions, + FdWithKeyRange* f, + std::unordered_map& blob_ctxs, + TableCache::TypedHandle* table_handle, uint64_t& num_filter_read, + uint64_t& num_index_read, uint64_t& num_sst_read); + +#ifdef USE_COROUTINES + // MultiGet using async IO to read data blocks from SST files in parallel + // within and across levels + Status MultiGetAsync( + const ReadOptions& options, MultiGetRange* range, + std::unordered_map* blob_ctxs); + + // A helper function to lookup a batch of keys in a single level. It will + // queue coroutine tasks to mget_tasks. It may also split the input batch + // by creating a new batch with keys definitely not in this level and + // enqueuing it to to_process. + Status ProcessBatch( + const ReadOptions& read_options, FilePickerMultiGet* batch, + std::vector>& mget_tasks, + std::unordered_map* blob_ctxs, + autovector& batches, std::deque& waiting, + std::deque& to_process, unsigned int& num_tasks_queued, + std::unordered_map>& + mget_stats); +#endif + + ColumnFamilyData* cfd_; // ColumnFamilyData to which this Version belongs + Logger* info_log_; + Statistics* db_statistics_; + TableCache* table_cache_; + BlobSource* blob_source_; + const MergeOperator* merge_operator_; + + VersionStorageInfo storage_info_; + VersionSet* vset_; // VersionSet to which this Version belongs + Version* next_; // Next version in linked list + Version* prev_; // Previous version in linked list + int refs_; // Number of live refs to this version + const FileOptions file_options_; + const MutableCFOptions mutable_cf_options_; + // Cached value to avoid recomputing it on every read. + const size_t max_file_size_for_l0_meta_pin_; + + // A version number that uniquely represents this version. This is + // used for debugging and logging purposes only. + uint64_t version_number_; + std::shared_ptr io_tracer_; + bool use_async_io_; + + Version(ColumnFamilyData* cfd, VersionSet* vset, const FileOptions& file_opt, + MutableCFOptions mutable_cf_options, + const std::shared_ptr& io_tracer, + uint64_t version_number = 0, + EpochNumberRequirement epoch_number_requirement = + EpochNumberRequirement::kMustPresent); + + ~Version(); + + // No copying allowed + Version(const Version&) = delete; + void operator=(const Version&) = delete; +}; + +class BaseReferencedVersionBuilder; + +class AtomicGroupReadBuffer { + public: + AtomicGroupReadBuffer() = default; + Status AddEdit(VersionEdit* edit); + void Clear(); + bool IsFull() const; + bool IsEmpty() const; + + uint64_t TEST_read_edits_in_atomic_group() const { + return read_edits_in_atomic_group_; + } + std::vector& replay_buffer() { return replay_buffer_; } + + private: + uint64_t read_edits_in_atomic_group_ = 0; + std::vector replay_buffer_; +}; + +// VersionSet is the collection of versions of all the column families of the +// database. Each database owns one VersionSet. A VersionSet has access to all +// column families via ColumnFamilySet, i.e. set of the column families. +class VersionSet { + public: + VersionSet(const std::string& dbname, const ImmutableDBOptions* db_options, + const FileOptions& file_options, Cache* table_cache, + WriteBufferManager* write_buffer_manager, + WriteController* write_controller, + BlockCacheTracer* const block_cache_tracer, + const std::shared_ptr& io_tracer, + const std::string& db_id, const std::string& db_session_id); + // No copying allowed + VersionSet(const VersionSet&) = delete; + void operator=(const VersionSet&) = delete; + + virtual ~VersionSet(); + + Status LogAndApplyToDefaultColumnFamily( + const ReadOptions& read_options, VersionEdit* edit, InstrumentedMutex* mu, + FSDirectory* dir_contains_current_file, bool new_descriptor_log = false, + const ColumnFamilyOptions* column_family_options = nullptr) { + ColumnFamilyData* default_cf = GetColumnFamilySet()->GetDefault(); + const MutableCFOptions* cf_options = + default_cf->GetLatestMutableCFOptions(); + return LogAndApply(default_cf, *cf_options, read_options, edit, mu, + dir_contains_current_file, new_descriptor_log, + column_family_options); + } + + // Apply *edit to the current version to form a new descriptor that + // is both saved to persistent state and installed as the new + // current version. Will release *mu while actually writing to the file. + // column_family_options has to be set if edit is column family add + // REQUIRES: *mu is held on entry. + // REQUIRES: no other thread concurrently calls LogAndApply() + Status LogAndApply( + ColumnFamilyData* column_family_data, + const MutableCFOptions& mutable_cf_options, + const ReadOptions& read_options, VersionEdit* edit, InstrumentedMutex* mu, + FSDirectory* dir_contains_current_file, bool new_descriptor_log = false, + const ColumnFamilyOptions* column_family_options = nullptr) { + autovector cfds; + cfds.emplace_back(column_family_data); + autovector mutable_cf_options_list; + mutable_cf_options_list.emplace_back(&mutable_cf_options); + autovector> edit_lists; + autovector edit_list; + edit_list.emplace_back(edit); + edit_lists.emplace_back(edit_list); + return LogAndApply(cfds, mutable_cf_options_list, read_options, edit_lists, + mu, dir_contains_current_file, new_descriptor_log, + column_family_options); + } + // The batch version. If edit_list.size() > 1, caller must ensure that + // no edit in the list column family add or drop + Status LogAndApply( + ColumnFamilyData* column_family_data, + const MutableCFOptions& mutable_cf_options, + const ReadOptions& read_options, + const autovector& edit_list, InstrumentedMutex* mu, + FSDirectory* dir_contains_current_file, bool new_descriptor_log = false, + const ColumnFamilyOptions* column_family_options = nullptr, + const std::function& manifest_wcb = {}) { + autovector cfds; + cfds.emplace_back(column_family_data); + autovector mutable_cf_options_list; + mutable_cf_options_list.emplace_back(&mutable_cf_options); + autovector> edit_lists; + edit_lists.emplace_back(edit_list); + return LogAndApply(cfds, mutable_cf_options_list, read_options, edit_lists, + mu, dir_contains_current_file, new_descriptor_log, + column_family_options, {manifest_wcb}); + } + + // The across-multi-cf batch version. If edit_lists contain more than + // 1 version edits, caller must ensure that no edit in the []list is column + // family manipulation. + virtual Status LogAndApply( + const autovector& cfds, + const autovector& mutable_cf_options_list, + const ReadOptions& read_options, + const autovector>& edit_lists, + InstrumentedMutex* mu, FSDirectory* dir_contains_current_file, + bool new_descriptor_log = false, + const ColumnFamilyOptions* new_cf_options = nullptr, + const std::vector>& manifest_wcbs = + {}); + + static Status GetCurrentManifestPath(const std::string& dbname, + FileSystem* fs, + std::string* manifest_filename, + uint64_t* manifest_file_number); + void WakeUpWaitingManifestWriters(); + + // Recover the last saved descriptor (MANIFEST) from persistent storage. + // If read_only == true, Recover() will not complain if some column families + // are not opened + Status Recover(const std::vector& column_families, + bool read_only = false, std::string* db_id = nullptr, + bool no_error_if_files_missing = false); + + Status TryRecover(const std::vector& column_families, + bool read_only, + const std::vector& files_in_dbname, + std::string* db_id, bool* has_missing_table_file); + + // Try to recover the version set to the most recent consistent state + // recorded in the specified manifest. + Status TryRecoverFromOneManifest( + const std::string& manifest_path, + const std::vector& column_families, + bool read_only, std::string* db_id, bool* has_missing_table_file); + + // Recover the next epoch number of each CFs and epoch number + // of their files (if missing) + void RecoverEpochNumbers(); + + // Reads a manifest file and returns a list of column families in + // column_families. + static Status ListColumnFamilies(std::vector* column_families, + const std::string& dbname, FileSystem* fs); + static Status ListColumnFamiliesFromManifest( + const std::string& manifest_path, FileSystem* fs, + std::vector* column_families); + + // Try to reduce the number of levels. This call is valid when + // only one level from the new max level to the old + // max level containing files. + // The call is static, since number of levels is immutable during + // the lifetime of a RocksDB instance. It reduces number of levels + // in a DB by applying changes to manifest. + // For example, a db currently has 7 levels [0-6], and a call to + // to reduce to 5 [0-4] can only be executed when only one level + // among [4-6] contains files. + static Status ReduceNumberOfLevels(const std::string& dbname, + const Options* options, + const FileOptions& file_options, + int new_levels); + + // Get the checksum information of all live files + Status GetLiveFilesChecksumInfo(FileChecksumList* checksum_list); + + // printf contents (for debugging) + Status DumpManifest(Options& options, std::string& manifestFileName, + bool verbose, bool hex = false, bool json = false, + const std::vector& cf_descs = {}); + + const std::string& DbSessionId() const { return db_session_id_; } + + // Return the current manifest file number + uint64_t manifest_file_number() const { return manifest_file_number_; } + + uint64_t options_file_number() const { return options_file_number_; } + + uint64_t pending_manifest_file_number() const { + return pending_manifest_file_number_; + } + + uint64_t current_next_file_number() const { return next_file_number_.load(); } + + uint64_t min_log_number_to_keep() const { + return min_log_number_to_keep_.load(); + } + + // Allocate and return a new file number + uint64_t NewFileNumber() { return next_file_number_.fetch_add(1); } + + // Fetch And Add n new file number + uint64_t FetchAddFileNumber(uint64_t n) { + return next_file_number_.fetch_add(n); + } + + // Return the last sequence number. + uint64_t LastSequence() const { + return last_sequence_.load(std::memory_order_acquire); + } + + // Note: memory_order_acquire must be sufficient. + uint64_t LastAllocatedSequence() const { + return last_allocated_sequence_.load(std::memory_order_seq_cst); + } + + // Note: memory_order_acquire must be sufficient. + uint64_t LastPublishedSequence() const { + return last_published_sequence_.load(std::memory_order_seq_cst); + } + + // Set the last sequence number to s. + void SetLastSequence(uint64_t s) { + assert(s >= last_sequence_); + // Last visible sequence must always be less than last written seq + assert(!db_options_->two_write_queues || s <= last_allocated_sequence_); + last_sequence_.store(s, std::memory_order_release); + } + + // Note: memory_order_release must be sufficient + void SetLastPublishedSequence(uint64_t s) { + assert(s >= last_published_sequence_); + last_published_sequence_.store(s, std::memory_order_seq_cst); + } + + // Note: memory_order_release must be sufficient + void SetLastAllocatedSequence(uint64_t s) { + assert(s >= last_allocated_sequence_); + last_allocated_sequence_.store(s, std::memory_order_seq_cst); + } + + // Note: memory_order_release must be sufficient + uint64_t FetchAddLastAllocatedSequence(uint64_t s) { + return last_allocated_sequence_.fetch_add(s, std::memory_order_seq_cst); + } + + // Mark the specified file number as used. + // REQUIRED: this is only called during single-threaded recovery or repair. + void MarkFileNumberUsed(uint64_t number); + + // Mark the specified log number as deleted + // REQUIRED: this is only called during single-threaded recovery or repair, or + // from ::LogAndApply where the global mutex is held. + void MarkMinLogNumberToKeep(uint64_t number); + + // Return the log file number for the log file that is currently + // being compacted, or zero if there is no such log file. + uint64_t prev_log_number() const { return prev_log_number_; } + + // Returns the minimum log number which still has data not flushed to any SST + // file. + // In non-2PC mode, all the log numbers smaller than this number can be safely + // deleted, although we still use `min_log_number_to_keep_` to determine when + // to delete a WAL file. + uint64_t MinLogNumberWithUnflushedData() const { + return PreComputeMinLogNumberWithUnflushedData(nullptr); + } + + // Returns the minimum log number which still has data not flushed to any SST + // file. + // Empty column families' log number is considered to be + // new_log_number_for_empty_cf. + uint64_t PreComputeMinLogNumberWithUnflushedData( + uint64_t new_log_number_for_empty_cf) const { + uint64_t min_log_num = std::numeric_limits::max(); + for (auto cfd : *column_family_set_) { + // It's safe to ignore dropped column families here: + // cfd->IsDropped() becomes true after the drop is persisted in MANIFEST. + uint64_t num = + cfd->IsEmpty() ? new_log_number_for_empty_cf : cfd->GetLogNumber(); + if (min_log_num > num && !cfd->IsDropped()) { + min_log_num = num; + } + } + return min_log_num; + } + // Returns the minimum log number which still has data not flushed to any SST + // file, except data from `cfd_to_skip`. + uint64_t PreComputeMinLogNumberWithUnflushedData( + const ColumnFamilyData* cfd_to_skip) const { + uint64_t min_log_num = std::numeric_limits::max(); + for (auto cfd : *column_family_set_) { + if (cfd == cfd_to_skip) { + continue; + } + // It's safe to ignore dropped column families here: + // cfd->IsDropped() becomes true after the drop is persisted in MANIFEST. + if (min_log_num > cfd->GetLogNumber() && !cfd->IsDropped()) { + min_log_num = cfd->GetLogNumber(); + } + } + return min_log_num; + } + // Returns the minimum log number which still has data not flushed to any SST + // file, except data from `cfds_to_skip`. + uint64_t PreComputeMinLogNumberWithUnflushedData( + const std::unordered_set& cfds_to_skip) const { + uint64_t min_log_num = std::numeric_limits::max(); + for (auto cfd : *column_family_set_) { + if (cfds_to_skip.count(cfd)) { + continue; + } + // It's safe to ignore dropped column families here: + // cfd->IsDropped() becomes true after the drop is persisted in MANIFEST. + if (min_log_num > cfd->GetLogNumber() && !cfd->IsDropped()) { + min_log_num = cfd->GetLogNumber(); + } + } + return min_log_num; + } + + // Create an iterator that reads over the compaction inputs for "*c". + // The caller should delete the iterator when no longer needed. + // @param read_options Must outlive the returned iterator. + // @param start, end indicates compaction range + InternalIterator* MakeInputIterator( + const ReadOptions& read_options, const Compaction* c, + RangeDelAggregator* range_del_agg, + const FileOptions& file_options_compactions, + const std::optional& start, + const std::optional& end); + + // Add all files listed in any live version to *live_table_files and + // *live_blob_files. Note that these lists may contain duplicates. + void AddLiveFiles(std::vector* live_table_files, + std::vector* live_blob_files) const; + + // Remove live files that are in the delete candidate lists. + void RemoveLiveFiles( + std::vector& sst_delete_candidates, + std::vector& blob_delete_candidates) const; + + // Return the approximate size of data to be scanned for range [start, end) + // in levels [start_level, end_level). If end_level == -1 it will search + // through all non-empty levels + uint64_t ApproximateSize(const SizeApproximationOptions& options, + const ReadOptions& read_options, Version* v, + const Slice& start, const Slice& end, + int start_level, int end_level, + TableReaderCaller caller); + + // Return the size of the current manifest file + uint64_t manifest_file_size() const { return manifest_file_size_; } + + Status GetMetadataForFile(uint64_t number, int* filelevel, + FileMetaData** metadata, ColumnFamilyData** cfd); + + // This function doesn't support leveldb SST filenames + void GetLiveFilesMetaData(std::vector* metadata); + + void AddObsoleteBlobFile(uint64_t blob_file_number, std::string path) { + assert(table_cache_); + + table_cache_->Erase(GetSliceForKey(&blob_file_number)); + + obsolete_blob_files_.emplace_back(blob_file_number, std::move(path)); + } + + void GetObsoleteFiles(std::vector* files, + std::vector* blob_files, + std::vector* manifest_filenames, + uint64_t min_pending_output); + + // REQUIRES: DB mutex held + uint64_t GetObsoleteSstFilesSize() const; + + ColumnFamilySet* GetColumnFamilySet() { return column_family_set_.get(); } + + const UnorderedMap& GetRunningColumnFamiliesTimestampSize() + const { + return column_family_set_->GetRunningColumnFamiliesTimestampSize(); + } + + const UnorderedMap& + GetColumnFamiliesTimestampSizeForRecord() const { + return column_family_set_->GetColumnFamiliesTimestampSizeForRecord(); + } + + RefedColumnFamilySet GetRefedColumnFamilySet() { + return RefedColumnFamilySet(GetColumnFamilySet()); + } + + const FileOptions& file_options() { return file_options_; } + void ChangeFileOptions(const MutableDBOptions& new_options) { + file_options_.writable_file_max_buffer_size = + new_options.writable_file_max_buffer_size; + } + + const ImmutableDBOptions* db_options() const { return db_options_; } + + static uint64_t GetNumLiveVersions(Version* dummy_versions); + + static uint64_t GetTotalSstFilesSize(Version* dummy_versions); + + static uint64_t GetTotalBlobFileSize(Version* dummy_versions); + + // Get the IO Status returned by written Manifest. + const IOStatus& io_status() const { return io_status_; } + + // The returned WalSet needs to be accessed with DB mutex held. + const WalSet& GetWalSet() const { return wals_; } + + void TEST_CreateAndAppendVersion(ColumnFamilyData* cfd) { + assert(cfd); + + const auto& mutable_cf_options = *cfd->GetLatestMutableCFOptions(); + Version* const version = + new Version(cfd, this, file_options_, mutable_cf_options, io_tracer_); + + constexpr bool update_stats = false; + const ReadOptions read_options; + version->PrepareAppend(mutable_cf_options, read_options, update_stats); + AppendVersion(cfd, version); + } + + protected: + using VersionBuilderMap = + UnorderedMap>; + + struct ManifestWriter; + + friend class Version; + friend class VersionEditHandler; + friend class VersionEditHandlerPointInTime; + friend class DumpManifestHandler; + friend class DBImpl; + friend class DBImplReadOnly; + + struct LogReporter : public log::Reader::Reporter { + Status* status; + virtual void Corruption(size_t /*bytes*/, const Status& s) override { + if (status->ok()) { + *status = s; + } + } + }; + + void Reset(); + + // Returns approximated offset of a key in a file for a given version. + uint64_t ApproximateOffsetOf(const ReadOptions& read_options, Version* v, + const FdWithKeyRange& f, const Slice& key, + TableReaderCaller caller); + + // Returns approximated data size between start and end keys in a file + // for a given version. + uint64_t ApproximateSize(const ReadOptions& read_options, Version* v, + const FdWithKeyRange& f, const Slice& start, + const Slice& end, TableReaderCaller caller); + + struct MutableCFState { + uint64_t log_number; + std::string full_history_ts_low; + + explicit MutableCFState() = default; + explicit MutableCFState(uint64_t _log_number, std::string ts_low) + : log_number(_log_number), full_history_ts_low(std::move(ts_low)) {} + }; + + // Save current contents to *log + Status WriteCurrentStateToManifest( + const std::unordered_map& curr_state, + const VersionEdit& wal_additions, log::Writer* log, IOStatus& io_s); + + void AppendVersion(ColumnFamilyData* column_family_data, Version* v); + + ColumnFamilyData* CreateColumnFamily(const ColumnFamilyOptions& cf_options, + const ReadOptions& read_options, + const VersionEdit* edit); + + Status VerifyFileMetadata(const ReadOptions& read_options, + ColumnFamilyData* cfd, const std::string& fpath, + int level, const FileMetaData& meta); + + // Protected by DB mutex. + WalSet wals_; + + std::unique_ptr column_family_set_; + Cache* table_cache_; + Env* const env_; + FileSystemPtr const fs_; + SystemClock* const clock_; + const std::string dbname_; + std::string db_id_; + const ImmutableDBOptions* const db_options_; + std::atomic next_file_number_; + // Any WAL number smaller than this should be ignored during recovery, + // and is qualified for being deleted. + std::atomic min_log_number_to_keep_ = {0}; + uint64_t manifest_file_number_; + uint64_t options_file_number_; + uint64_t options_file_size_; + uint64_t pending_manifest_file_number_; + // The last seq visible to reads. It normally indicates the last sequence in + // the memtable but when using two write queues it could also indicate the + // last sequence in the WAL visible to reads. + std::atomic last_sequence_; + // The last sequence number of data committed to the descriptor (manifest + // file). + SequenceNumber descriptor_last_sequence_ = 0; + // The last seq that is already allocated. It is applicable only when we have + // two write queues. In that case seq might or might not have appreated in + // memtable but it is expected to appear in the WAL. + // We have last_sequence <= last_allocated_sequence_ + std::atomic last_allocated_sequence_; + // The last allocated sequence that is also published to the readers. This is + // applicable only when last_seq_same_as_publish_seq_ is not set. Otherwise + // last_sequence_ also indicates the last published seq. + // We have last_sequence <= last_published_sequence_ <= + // last_allocated_sequence_ + std::atomic last_published_sequence_; + uint64_t prev_log_number_; // 0 or backing store for memtable being compacted + + // Opened lazily + std::unique_ptr descriptor_log_; + + // generates a increasing version number for every new version + uint64_t current_version_number_; + + // Queue of writers to the manifest file + std::deque manifest_writers_; + + // Current size of manifest file + uint64_t manifest_file_size_; + + std::vector obsolete_files_; + std::vector obsolete_blob_files_; + std::vector obsolete_manifests_; + + // env options for all reads and writes except compactions + FileOptions file_options_; + + BlockCacheTracer* const block_cache_tracer_; + + // Store the IO status when Manifest is written + IOStatus io_status_; + + std::shared_ptr io_tracer_; + + std::string db_session_id_; + + private: + // REQUIRES db mutex at beginning. may release and re-acquire db mutex + Status ProcessManifestWrites(std::deque& writers, + InstrumentedMutex* mu, + FSDirectory* dir_contains_current_file, + bool new_descriptor_log, + const ColumnFamilyOptions* new_cf_options, + const ReadOptions& read_options); + + void LogAndApplyCFHelper(VersionEdit* edit, + SequenceNumber* max_last_sequence); + Status LogAndApplyHelper(ColumnFamilyData* cfd, VersionBuilder* b, + VersionEdit* edit, SequenceNumber* max_last_sequence, + InstrumentedMutex* mu); +}; + +// ReactiveVersionSet represents a collection of versions of the column +// families of the database. Users of ReactiveVersionSet, e.g. DBImplSecondary, +// need to replay the MANIFEST (description log in older terms) in order to +// reconstruct and install versions. +class ReactiveVersionSet : public VersionSet { + public: + ReactiveVersionSet(const std::string& dbname, + const ImmutableDBOptions* _db_options, + const FileOptions& _file_options, Cache* table_cache, + WriteBufferManager* write_buffer_manager, + WriteController* write_controller, + const std::shared_ptr& io_tracer); + + ~ReactiveVersionSet() override; + + Status ReadAndApply( + InstrumentedMutex* mu, + std::unique_ptr* manifest_reader, + Status* manifest_read_status, + std::unordered_set* cfds_changed); + + Status Recover(const std::vector& column_families, + std::unique_ptr* manifest_reader, + std::unique_ptr* manifest_reporter, + std::unique_ptr* manifest_reader_status); +#ifndef NDEBUG + uint64_t TEST_read_edits_in_atomic_group() const; +#endif //! NDEBUG + + std::vector& replay_buffer(); + + protected: + // REQUIRES db mutex + Status ApplyOneVersionEditToBuilder( + VersionEdit& edit, std::unordered_set* cfds_changed, + VersionEdit* version_edit); + + Status MaybeSwitchManifest( + log::Reader::Reporter* reporter, + std::unique_ptr* manifest_reader); + + private: + std::unique_ptr manifest_tailer_; + // TODO: plumb Env::IOActivity + const ReadOptions read_options_; + using VersionSet::LogAndApply; + using VersionSet::Recover; + + Status LogAndApply( + const autovector& /*cfds*/, + const autovector& /*mutable_cf_options_list*/, + const ReadOptions& /* read_options */, + const autovector>& /*edit_lists*/, + InstrumentedMutex* /*mu*/, FSDirectory* /*dir_contains_current_file*/, + bool /*new_descriptor_log*/, const ColumnFamilyOptions* /*new_cf_option*/, + const std::vector>& /*manifest_wcbs*/) + override { + return Status::NotSupported("not supported in reactive mode"); + } + + // No copy allowed + ReactiveVersionSet(const ReactiveVersionSet&); + ReactiveVersionSet& operator=(const ReactiveVersionSet&); +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/version_set_sync_and_async.h b/librocksdb-sys/rocksdb/db/version_set_sync_and_async.h new file mode 100644 index 0000000..2507762 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/version_set_sync_and_async.h @@ -0,0 +1,172 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "util/coro_utils.h" + +#if defined(WITHOUT_COROUTINES) || \ + (defined(USE_COROUTINES) && defined(WITH_COROUTINES)) + +namespace ROCKSDB_NAMESPACE { + +// Lookup a batch of keys in a single SST file +DEFINE_SYNC_AND_ASYNC(Status, Version::MultiGetFromSST) +(const ReadOptions& read_options, MultiGetRange file_range, int hit_file_level, + bool skip_filters, bool skip_range_deletions, FdWithKeyRange* f, + std::unordered_map& blob_ctxs, + TableCache::TypedHandle* table_handle, uint64_t& num_filter_read, + uint64_t& num_index_read, uint64_t& num_sst_read) { + bool timer_enabled = GetPerfLevel() >= PerfLevel::kEnableTimeExceptForMutex && + get_perf_context()->per_level_perf_context_enabled; + + Status s; + StopWatchNano timer(clock_, timer_enabled /* auto_start */); + s = CO_AWAIT(table_cache_->MultiGet)( + read_options, *internal_comparator(), *f->file_metadata, &file_range, + mutable_cf_options_.block_protection_bytes_per_key, + mutable_cf_options_.prefix_extractor, + cfd_->internal_stats()->GetFileReadHist(hit_file_level), skip_filters, + skip_range_deletions, hit_file_level, table_handle); + // TODO: examine the behavior for corrupted key + if (timer_enabled) { + PERF_COUNTER_BY_LEVEL_ADD(get_from_table_nanos, timer.ElapsedNanos(), + hit_file_level); + } + if (!s.ok()) { + // TODO: Set status for individual keys appropriately + for (auto iter = file_range.begin(); iter != file_range.end(); ++iter) { + *iter->s = s; + file_range.MarkKeyDone(iter); + } + CO_RETURN s; + } + uint64_t batch_size = 0; + for (auto iter = file_range.begin(); s.ok() && iter != file_range.end(); + ++iter) { + GetContext& get_context = *iter->get_context; + Status* status = iter->s; + // The Status in the KeyContext takes precedence over GetContext state + // Status may be an error if there were any IO errors in the table + // reader. We never expect Status to be NotFound(), as that is + // determined by get_context + assert(!status->IsNotFound()); + if (!status->ok()) { + file_range.MarkKeyDone(iter); + continue; + } + + if (get_context.sample()) { + sample_file_read_inc(f->file_metadata); + } + batch_size++; + num_index_read += get_context.get_context_stats_.num_index_read; + num_filter_read += get_context.get_context_stats_.num_filter_read; + num_sst_read += get_context.get_context_stats_.num_sst_read; + // Reset these stats since they're specific to a level + get_context.get_context_stats_.num_index_read = 0; + get_context.get_context_stats_.num_filter_read = 0; + get_context.get_context_stats_.num_sst_read = 0; + + // report the counters before returning + if (get_context.State() != GetContext::kNotFound && + get_context.State() != GetContext::kMerge && + db_statistics_ != nullptr) { + get_context.ReportCounters(); + } else { + if (iter->max_covering_tombstone_seq > 0) { + // The remaining files we look at will only contain covered keys, so + // we stop here for this key + file_range.SkipKey(iter); + } + } + switch (get_context.State()) { + case GetContext::kNotFound: + // Keep searching in other files + break; + case GetContext::kMerge: + // TODO: update per-level perfcontext user_key_return_count for kMerge + break; + case GetContext::kFound: + if (hit_file_level == 0) { + RecordTick(db_statistics_, GET_HIT_L0); + } else if (hit_file_level == 1) { + RecordTick(db_statistics_, GET_HIT_L1); + } else if (hit_file_level >= 2) { + RecordTick(db_statistics_, GET_HIT_L2_AND_UP); + } + + PERF_COUNTER_BY_LEVEL_ADD(user_key_return_count, 1, hit_file_level); + + file_range.MarkKeyDone(iter); + + if (iter->is_blob_index) { + BlobIndex blob_index; + Status tmp_s; + + if (iter->value) { + TEST_SYNC_POINT_CALLBACK("Version::MultiGet::TamperWithBlobIndex", + &(*iter)); + + tmp_s = blob_index.DecodeFrom(*(iter->value)); + + } else { + assert(iter->columns); + assert(!iter->columns->columns().empty()); + assert(iter->columns->columns().front().name() == + kDefaultWideColumnName); + + tmp_s = + blob_index.DecodeFrom(iter->columns->columns().front().value()); + } + + if (tmp_s.ok()) { + const uint64_t blob_file_num = blob_index.file_number(); + blob_ctxs[blob_file_num].emplace_back(blob_index, &*iter); + } else { + *(iter->s) = tmp_s; + } + } else { + if (iter->value) { + file_range.AddValueSize(iter->value->size()); + } else { + assert(iter->columns); + file_range.AddValueSize(iter->columns->serialized_size()); + } + + if (file_range.GetValueSize() > read_options.value_size_soft_limit) { + s = Status::Aborted(); + break; + } + } + continue; + case GetContext::kDeleted: + // Use empty error message for speed + *status = Status::NotFound(); + file_range.MarkKeyDone(iter); + continue; + case GetContext::kCorrupt: + *status = + Status::Corruption("corrupted key for ", iter->lkey->user_key()); + file_range.MarkKeyDone(iter); + continue; + case GetContext::kUnexpectedBlobIndex: + ROCKS_LOG_ERROR(info_log_, "Encounter unexpected blob index."); + *status = Status::NotSupported( + "Encounter unexpected blob index. Please open DB with " + "ROCKSDB_NAMESPACE::blob_db::BlobDB instead."); + file_range.MarkKeyDone(iter); + continue; + case GetContext::kMergeOperatorFailed: + *status = Status::Corruption(Status::SubCode::kMergeOperatorFailed); + file_range.MarkKeyDone(iter); + continue; + } + } + + RecordInHistogram(db_statistics_, SST_BATCH_SIZE, batch_size); + CO_RETURN s; +} +} // namespace ROCKSDB_NAMESPACE +#endif diff --git a/librocksdb-sys/rocksdb/db/version_set_test.cc b/librocksdb-sys/rocksdb/db/version_set_test.cc new file mode 100644 index 0000000..135b2d6 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/version_set_test.cc @@ -0,0 +1,3666 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/version_set.h" + +#include + +#include "db/db_impl/db_impl.h" +#include "db/db_test_util.h" +#include "db/log_writer.h" +#include "db/version_edit.h" +#include "rocksdb/advanced_options.h" +#include "rocksdb/convenience.h" +#include "rocksdb/file_system.h" +#include "table/block_based/block_based_table_factory.h" +#include "table/mock_table.h" +#include "table/unique_id_impl.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +class GenerateLevelFilesBriefTest : public testing::Test { + public: + std::vector files_; + LevelFilesBrief file_level_; + Arena arena_; + + GenerateLevelFilesBriefTest() {} + + ~GenerateLevelFilesBriefTest() override { + for (size_t i = 0; i < files_.size(); i++) { + delete files_[i]; + } + } + + void Add(const char* smallest, const char* largest, + SequenceNumber smallest_seq = 100, + SequenceNumber largest_seq = 100) { + FileMetaData* f = new FileMetaData( + files_.size() + 1, 0, 0, + InternalKey(smallest, smallest_seq, kTypeValue), + InternalKey(largest, largest_seq, kTypeValue), smallest_seq, + largest_seq, /* marked_for_compact */ false, Temperature::kUnknown, + kInvalidBlobFileNumber, kUnknownOldestAncesterTime, + kUnknownFileCreationTime, kUnknownEpochNumber, kUnknownFileChecksum, + kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0, + /* user_defined_timestamps_persisted */ true); + files_.push_back(f); + } + + int Compare() { + int diff = 0; + for (size_t i = 0; i < files_.size(); i++) { + if (file_level_.files[i].fd.GetNumber() != files_[i]->fd.GetNumber()) { + diff++; + } + } + return diff; + } +}; + +TEST_F(GenerateLevelFilesBriefTest, Empty) { + DoGenerateLevelFilesBrief(&file_level_, files_, &arena_); + ASSERT_EQ(0u, file_level_.num_files); + ASSERT_EQ(0, Compare()); +} + +TEST_F(GenerateLevelFilesBriefTest, Single) { + Add("p", "q"); + DoGenerateLevelFilesBrief(&file_level_, files_, &arena_); + ASSERT_EQ(1u, file_level_.num_files); + ASSERT_EQ(0, Compare()); +} + +TEST_F(GenerateLevelFilesBriefTest, Multiple) { + Add("150", "200"); + Add("200", "250"); + Add("300", "350"); + Add("400", "450"); + DoGenerateLevelFilesBrief(&file_level_, files_, &arena_); + ASSERT_EQ(4u, file_level_.num_files); + ASSERT_EQ(0, Compare()); +} + +class CountingLogger : public Logger { + public: + CountingLogger() : log_count(0) {} + using Logger::Logv; + void Logv(const char* /*format*/, va_list /*ap*/) override { log_count++; } + int log_count; +}; + +Options GetOptionsWithNumLevels(int num_levels, + std::shared_ptr logger) { + Options opt; + opt.num_levels = num_levels; + opt.info_log = logger; + return opt; +} + +class VersionStorageInfoTestBase : public testing::Test { + public: + const Comparator* ucmp_; + InternalKeyComparator icmp_; + std::shared_ptr logger_; + Options options_; + ImmutableOptions ioptions_; + MutableCFOptions mutable_cf_options_; + VersionStorageInfo vstorage_; + + InternalKey GetInternalKey(const char* ukey, + SequenceNumber smallest_seq = 100) { + return InternalKey(ukey, smallest_seq, kTypeValue); + } + + explicit VersionStorageInfoTestBase(const Comparator* ucmp) + : ucmp_(ucmp), + icmp_(ucmp_), + logger_(new CountingLogger()), + options_(GetOptionsWithNumLevels(6, logger_)), + ioptions_(options_), + mutable_cf_options_(options_), + vstorage_(&icmp_, ucmp_, 6, kCompactionStyleLevel, + /*src_vstorage=*/nullptr, + /*_force_consistency_checks=*/false) {} + + ~VersionStorageInfoTestBase() override { + for (int i = 0; i < vstorage_.num_levels(); ++i) { + for (auto* f : vstorage_.LevelFiles(i)) { + if (--f->refs == 0) { + delete f; + } + } + } + } + + void Add(int level, uint32_t file_number, const char* smallest, + const char* largest, uint64_t file_size = 0, + uint64_t oldest_blob_file_number = kInvalidBlobFileNumber, + uint64_t compensated_range_deletion_size = 0) { + constexpr SequenceNumber dummy_seq = 0; + + Add(level, file_number, GetInternalKey(smallest, dummy_seq), + GetInternalKey(largest, dummy_seq), file_size, oldest_blob_file_number, + compensated_range_deletion_size); + } + + void Add(int level, uint32_t file_number, const InternalKey& smallest, + const InternalKey& largest, uint64_t file_size = 0, + uint64_t oldest_blob_file_number = kInvalidBlobFileNumber, + uint64_t compensated_range_deletion_size = 0) { + assert(level < vstorage_.num_levels()); + FileMetaData* f = new FileMetaData( + file_number, 0, file_size, smallest, largest, /* smallest_seq */ 0, + /* largest_seq */ 0, /* marked_for_compact */ false, + Temperature::kUnknown, oldest_blob_file_number, + kUnknownOldestAncesterTime, kUnknownFileCreationTime, + kUnknownEpochNumber, kUnknownFileChecksum, kUnknownFileChecksumFuncName, + kNullUniqueId64x2, compensated_range_deletion_size, 0, + /* user_defined_timestamps_persisted */ true); + vstorage_.AddFile(level, f); + } + + void AddBlob(uint64_t blob_file_number, uint64_t total_blob_count, + uint64_t total_blob_bytes, + BlobFileMetaData::LinkedSsts linked_ssts, + uint64_t garbage_blob_count, uint64_t garbage_blob_bytes) { + auto shared_meta = SharedBlobFileMetaData::Create( + blob_file_number, total_blob_count, total_blob_bytes, + /* checksum_method */ std::string(), + /* checksum_value */ std::string()); + auto meta = + BlobFileMetaData::Create(std::move(shared_meta), std::move(linked_ssts), + garbage_blob_count, garbage_blob_bytes); + + vstorage_.AddBlobFile(std::move(meta)); + } + + void UpdateVersionStorageInfo() { + vstorage_.PrepareForVersionAppend(ioptions_, mutable_cf_options_); + vstorage_.SetFinalized(); + } + + std::string GetOverlappingFiles(int level, const InternalKey& begin, + const InternalKey& end) { + std::vector inputs; + vstorage_.GetOverlappingInputs(level, &begin, &end, &inputs); + + std::string result; + for (size_t i = 0; i < inputs.size(); ++i) { + if (i > 0) { + result += ","; + } + AppendNumberTo(&result, inputs[i]->fd.GetNumber()); + } + return result; + } +}; + +class VersionStorageInfoTest : public VersionStorageInfoTestBase { + public: + VersionStorageInfoTest() : VersionStorageInfoTestBase(BytewiseComparator()) {} + + ~VersionStorageInfoTest() override {} +}; + +TEST_F(VersionStorageInfoTest, MaxBytesForLevelStatic) { + ioptions_.level_compaction_dynamic_level_bytes = false; + mutable_cf_options_.max_bytes_for_level_base = 10; + mutable_cf_options_.max_bytes_for_level_multiplier = 5; + + Add(4, 100U, "1", "2", 100U); + Add(5, 101U, "1", "2", 100U); + + UpdateVersionStorageInfo(); + + ASSERT_EQ(vstorage_.MaxBytesForLevel(1), 10U); + ASSERT_EQ(vstorage_.MaxBytesForLevel(2), 50U); + ASSERT_EQ(vstorage_.MaxBytesForLevel(3), 250U); + ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 1250U); + + ASSERT_EQ(0, logger_->log_count); +} + +TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamic_1) { + ioptions_.level_compaction_dynamic_level_bytes = true; + mutable_cf_options_.max_bytes_for_level_base = 1000; + mutable_cf_options_.max_bytes_for_level_multiplier = 5; + + Add(5, 1U, "1", "2", 500U); + + UpdateVersionStorageInfo(); + + ASSERT_EQ(0, logger_->log_count); + ASSERT_EQ(vstorage_.base_level(), 5); +} + +TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamic_2) { + ioptions_.level_compaction_dynamic_level_bytes = true; + mutable_cf_options_.max_bytes_for_level_base = 1000; + mutable_cf_options_.max_bytes_for_level_multiplier = 5; + + Add(5, 1U, "1", "2", 500U); + Add(5, 2U, "3", "4", 550U); + + UpdateVersionStorageInfo(); + + ASSERT_EQ(0, logger_->log_count); + ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 1000U); + ASSERT_EQ(vstorage_.base_level(), 4); +} + +TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamic_3) { + ioptions_.level_compaction_dynamic_level_bytes = true; + mutable_cf_options_.max_bytes_for_level_base = 1000; + mutable_cf_options_.max_bytes_for_level_multiplier = 5; + + Add(5, 1U, "1", "2", 500U); + Add(5, 2U, "3", "4", 550U); + Add(4, 3U, "3", "4", 550U); + + UpdateVersionStorageInfo(); + + ASSERT_EQ(0, logger_->log_count); + ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 1000U); + ASSERT_EQ(vstorage_.base_level(), 4); +} + +TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamic_4) { + ioptions_.level_compaction_dynamic_level_bytes = true; + mutable_cf_options_.max_bytes_for_level_base = 1000; + mutable_cf_options_.max_bytes_for_level_multiplier = 5; + + Add(5, 1U, "1", "2", 500U); + Add(5, 2U, "3", "4", 550U); + Add(4, 3U, "3", "4", 550U); + Add(3, 4U, "3", "4", 250U); + Add(3, 5U, "5", "7", 300U); + + UpdateVersionStorageInfo(); + + ASSERT_EQ(1, logger_->log_count); + ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 1005U); + ASSERT_EQ(vstorage_.MaxBytesForLevel(3), 1000U); + ASSERT_EQ(vstorage_.base_level(), 3); +} + +TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamic_5) { + ioptions_.level_compaction_dynamic_level_bytes = true; + mutable_cf_options_.max_bytes_for_level_base = 1000; + mutable_cf_options_.max_bytes_for_level_multiplier = 5; + + Add(5, 1U, "1", "2", 500U); + Add(5, 2U, "3", "4", 550U); + Add(4, 3U, "3", "4", 550U); + Add(3, 4U, "3", "4", 250U); + Add(3, 5U, "5", "7", 300U); + Add(1, 6U, "3", "4", 5U); + Add(1, 7U, "8", "9", 5U); + + UpdateVersionStorageInfo(); + + ASSERT_EQ(1, logger_->log_count); + ASSERT_GT(vstorage_.MaxBytesForLevel(4), 1005U); + ASSERT_GT(vstorage_.MaxBytesForLevel(3), 1005U); + ASSERT_EQ(vstorage_.MaxBytesForLevel(2), 1005U); + ASSERT_EQ(vstorage_.MaxBytesForLevel(1), 1000U); + ASSERT_EQ(vstorage_.base_level(), 1); +} + +TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamicLotsOfData) { + ioptions_.level_compaction_dynamic_level_bytes = true; + mutable_cf_options_.max_bytes_for_level_base = 100; + mutable_cf_options_.max_bytes_for_level_multiplier = 2; + + Add(0, 1U, "1", "2", 50U); + Add(1, 2U, "1", "2", 50U); + Add(2, 3U, "1", "2", 500U); + Add(3, 4U, "1", "2", 500U); + Add(4, 5U, "1", "2", 1700U); + Add(5, 6U, "1", "2", 500U); + + UpdateVersionStorageInfo(); + + ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 800U); + ASSERT_EQ(vstorage_.MaxBytesForLevel(3), 400U); + ASSERT_EQ(vstorage_.MaxBytesForLevel(2), 200U); + ASSERT_EQ(vstorage_.MaxBytesForLevel(1), 100U); + ASSERT_EQ(vstorage_.base_level(), 1); + ASSERT_EQ(0, logger_->log_count); +} + +TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamicLargeLevel) { + uint64_t kOneGB = 1000U * 1000U * 1000U; + ioptions_.level_compaction_dynamic_level_bytes = true; + mutable_cf_options_.max_bytes_for_level_base = 10U * kOneGB; + mutable_cf_options_.max_bytes_for_level_multiplier = 10; + + Add(0, 1U, "1", "2", 50U); + Add(3, 4U, "1", "2", 32U * kOneGB); + Add(4, 5U, "1", "2", 500U * kOneGB); + Add(5, 6U, "1", "2", 3000U * kOneGB); + + UpdateVersionStorageInfo(); + + ASSERT_EQ(vstorage_.MaxBytesForLevel(5), 3000U * kOneGB); + ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 300U * kOneGB); + ASSERT_EQ(vstorage_.MaxBytesForLevel(3), 30U * kOneGB); + ASSERT_EQ(vstorage_.MaxBytesForLevel(2), 10U * kOneGB); + ASSERT_EQ(vstorage_.base_level(), 2); + ASSERT_EQ(0, logger_->log_count); +} + +TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamicWithLargeL0_1) { + ioptions_.level_compaction_dynamic_level_bytes = true; + mutable_cf_options_.max_bytes_for_level_base = 40000; + mutable_cf_options_.max_bytes_for_level_multiplier = 5; + mutable_cf_options_.level0_file_num_compaction_trigger = 2; + + Add(0, 1U, "1", "2", 10000U); + Add(0, 2U, "1", "2", 10000U); + Add(0, 3U, "1", "2", 10000U); + + Add(5, 4U, "1", "2", 1286250U); + Add(4, 5U, "1", "2", 200000U); + Add(3, 6U, "1", "2", 40000U); + Add(2, 7U, "1", "2", 8000U); + + UpdateVersionStorageInfo(); + + ASSERT_EQ(0, logger_->log_count); + ASSERT_EQ(2, vstorage_.base_level()); + // level multiplier should be 3.5 + ASSERT_EQ(vstorage_.level_multiplier(), 5.0); + ASSERT_EQ(40000U, vstorage_.MaxBytesForLevel(2)); + ASSERT_EQ(51450U, vstorage_.MaxBytesForLevel(3)); + ASSERT_EQ(257250U, vstorage_.MaxBytesForLevel(4)); + + vstorage_.ComputeCompactionScore(ioptions_, mutable_cf_options_); + // Only L0 hits compaction. + ASSERT_EQ(vstorage_.CompactionScoreLevel(0), 0); +} + +TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamicWithLargeL0_2) { + ioptions_.level_compaction_dynamic_level_bytes = true; + mutable_cf_options_.max_bytes_for_level_base = 10000; + mutable_cf_options_.max_bytes_for_level_multiplier = 5; + mutable_cf_options_.level0_file_num_compaction_trigger = 4; + + Add(0, 11U, "1", "2", 10000U); + Add(0, 12U, "1", "2", 10000U); + Add(0, 13U, "1", "2", 10000U); + + // Level size should be around 10,000, 10,290, 51,450, 257,250 + Add(5, 4U, "1", "2", 1286250U); + Add(4, 5U, "1", "2", 258000U); // unadjusted score 1.003 + Add(3, 6U, "1", "2", 53000U); // unadjusted score 1.03 + Add(2, 7U, "1", "2", 20000U); // unadjusted score 1.94 + + UpdateVersionStorageInfo(); + + ASSERT_EQ(0, logger_->log_count); + ASSERT_EQ(1, vstorage_.base_level()); + ASSERT_EQ(10000U, vstorage_.MaxBytesForLevel(1)); + ASSERT_EQ(10290U, vstorage_.MaxBytesForLevel(2)); + ASSERT_EQ(51450U, vstorage_.MaxBytesForLevel(3)); + ASSERT_EQ(257250U, vstorage_.MaxBytesForLevel(4)); + + vstorage_.ComputeCompactionScore(ioptions_, mutable_cf_options_); + // Although L2 and l3 have higher unadjusted compaction score, considering + // a relatively large L0 being compacted down soon, L4 is picked up for + // compaction. + // L0 is still picked up for oversizing. + ASSERT_EQ(0, vstorage_.CompactionScoreLevel(0)); + ASSERT_EQ(4, vstorage_.CompactionScoreLevel(1)); +} + +TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamicWithLargeL0_3) { + ioptions_.level_compaction_dynamic_level_bytes = true; + mutable_cf_options_.max_bytes_for_level_base = 20000; + mutable_cf_options_.max_bytes_for_level_multiplier = 5; + mutable_cf_options_.level0_file_num_compaction_trigger = 5; + + Add(0, 11U, "1", "2", 2500U); + Add(0, 12U, "1", "2", 2500U); + Add(0, 13U, "1", "2", 2500U); + Add(0, 14U, "1", "2", 2500U); + + // Level size should be around 20,000, 53000, 258000 + Add(5, 4U, "1", "2", 1286250U); + Add(4, 5U, "1", "2", 260000U); // Unadjusted score 1.01, adjusted about 4.3 + Add(3, 6U, "1", "2", 85000U); // Unadjusted score 1.42, adjusted about 11.6 + Add(2, 7U, "1", "2", 30000); // Unadjusted score 1.5, adjusted about 10.0 + + UpdateVersionStorageInfo(); + + ASSERT_EQ(0, logger_->log_count); + ASSERT_EQ(2, vstorage_.base_level()); + ASSERT_EQ(20000U, vstorage_.MaxBytesForLevel(2)); + + vstorage_.ComputeCompactionScore(ioptions_, mutable_cf_options_); + // Although L2 has higher unadjusted compaction score, considering + // a relatively large L0 being compacted down soon, L3 is picked up for + // compaction. + + ASSERT_EQ(3, vstorage_.CompactionScoreLevel(0)); + ASSERT_EQ(2, vstorage_.CompactionScoreLevel(1)); + ASSERT_EQ(4, vstorage_.CompactionScoreLevel(2)); +} + +TEST_F(VersionStorageInfoTest, DrainUnnecessaryLevel) { + ioptions_.level_compaction_dynamic_level_bytes = true; + mutable_cf_options_.max_bytes_for_level_base = 1000; + mutable_cf_options_.max_bytes_for_level_multiplier = 10; + + // Create a few unnecessary levels. + // See if score is calculated correctly. + Add(5, 1U, "1", "2", 2000U); // target size 1010000 + Add(4, 2U, "1", "2", 200U); // target size 101000 + // Unnecessary levels + Add(3, 3U, "1", "2", 100U); // target size 10100 + // Level 2: target size 1010 + Add(1, 4U, "1", "2", + 10U); // target size 1000 = max(base_bytes_min + 1, base_bytes_max) + + UpdateVersionStorageInfo(); + + ASSERT_EQ(1, vstorage_.base_level()); + ASSERT_EQ(1000, vstorage_.MaxBytesForLevel(1)); + ASSERT_EQ(10100, vstorage_.MaxBytesForLevel(3)); + vstorage_.ComputeCompactionScore(ioptions_, mutable_cf_options_); + + // Tests that levels 1 and 3 are eligible for compaction. + // Levels 1 and 3 are much smaller than target size, + // so size does not contribute to a high compaction score. + ASSERT_EQ(1, vstorage_.CompactionScoreLevel(0)); + ASSERT_GT(vstorage_.CompactionScore(0), 10); + ASSERT_EQ(3, vstorage_.CompactionScoreLevel(1)); + ASSERT_GT(vstorage_.CompactionScore(1), 10); +} + +TEST_F(VersionStorageInfoTest, EstimateLiveDataSize) { + // Test whether the overlaps are detected as expected + Add(1, 1U, "4", "7", 1U); // Perfect overlap with last level + Add(2, 2U, "3", "5", 1U); // Partial overlap with last level + Add(2, 3U, "6", "8", 1U); // Partial overlap with last level + Add(3, 4U, "1", "9", 1U); // Contains range of last level + Add(4, 5U, "4", "5", 1U); // Inside range of last level + Add(4, 6U, "6", "7", 1U); // Inside range of last level + Add(5, 7U, "4", "7", 10U); + + UpdateVersionStorageInfo(); + + ASSERT_EQ(10U, vstorage_.EstimateLiveDataSize()); +} + +TEST_F(VersionStorageInfoTest, EstimateLiveDataSize2) { + Add(0, 1U, "9", "9", 1U); // Level 0 is not ordered + Add(0, 2U, "5", "6", 1U); // Ignored because of [5,6] in l1 + Add(1, 3U, "1", "2", 1U); // Ignored because of [2,3] in l2 + Add(1, 4U, "3", "4", 1U); // Ignored because of [2,3] in l2 + Add(1, 5U, "5", "6", 1U); + Add(2, 6U, "2", "3", 1U); + Add(3, 7U, "7", "8", 1U); + + UpdateVersionStorageInfo(); + + ASSERT_EQ(4U, vstorage_.EstimateLiveDataSize()); +} + +TEST_F(VersionStorageInfoTest, GetOverlappingInputs) { + // Two files that overlap at the range deletion tombstone sentinel. + Add(1, 1U, {"a", 0, kTypeValue}, + {"b", kMaxSequenceNumber, kTypeRangeDeletion}, 1); + Add(1, 2U, {"b", 0, kTypeValue}, {"c", 0, kTypeValue}, 1); + // Two files that overlap at the same user key. + Add(1, 3U, {"d", 0, kTypeValue}, {"e", kMaxSequenceNumber, kTypeValue}, 1); + Add(1, 4U, {"e", 0, kTypeValue}, {"f", 0, kTypeValue}, 1); + // Two files that do not overlap. + Add(1, 5U, {"g", 0, kTypeValue}, {"h", 0, kTypeValue}, 1); + Add(1, 6U, {"i", 0, kTypeValue}, {"j", 0, kTypeValue}, 1); + + UpdateVersionStorageInfo(); + + ASSERT_EQ("1,2", + GetOverlappingFiles(1, {"a", 0, kTypeValue}, {"b", 0, kTypeValue})); + ASSERT_EQ("1", + GetOverlappingFiles(1, {"a", 0, kTypeValue}, + {"b", kMaxSequenceNumber, kTypeRangeDeletion})); + ASSERT_EQ("2", GetOverlappingFiles(1, {"b", kMaxSequenceNumber, kTypeValue}, + {"c", 0, kTypeValue})); + ASSERT_EQ("3,4", + GetOverlappingFiles(1, {"d", 0, kTypeValue}, {"e", 0, kTypeValue})); + ASSERT_EQ("3", + GetOverlappingFiles(1, {"d", 0, kTypeValue}, + {"e", kMaxSequenceNumber, kTypeRangeDeletion})); + ASSERT_EQ("3,4", GetOverlappingFiles(1, {"e", kMaxSequenceNumber, kTypeValue}, + {"f", 0, kTypeValue})); + ASSERT_EQ("3,4", + GetOverlappingFiles(1, {"e", 0, kTypeValue}, {"f", 0, kTypeValue})); + ASSERT_EQ("5", + GetOverlappingFiles(1, {"g", 0, kTypeValue}, {"h", 0, kTypeValue})); + ASSERT_EQ("6", + GetOverlappingFiles(1, {"i", 0, kTypeValue}, {"j", 0, kTypeValue})); +} + +TEST_F(VersionStorageInfoTest, FileLocationAndMetaDataByNumber) { + Add(0, 11U, "1", "2", 5000U); + Add(0, 12U, "1", "2", 5000U); + + Add(2, 7U, "1", "2", 8000U); + + UpdateVersionStorageInfo(); + + ASSERT_EQ(vstorage_.GetFileLocation(11U), + VersionStorageInfo::FileLocation(0, 0)); + ASSERT_NE(vstorage_.GetFileMetaDataByNumber(11U), nullptr); + + ASSERT_EQ(vstorage_.GetFileLocation(12U), + VersionStorageInfo::FileLocation(0, 1)); + ASSERT_NE(vstorage_.GetFileMetaDataByNumber(12U), nullptr); + + ASSERT_EQ(vstorage_.GetFileLocation(7U), + VersionStorageInfo::FileLocation(2, 0)); + ASSERT_NE(vstorage_.GetFileMetaDataByNumber(7U), nullptr); + + ASSERT_FALSE(vstorage_.GetFileLocation(999U).IsValid()); + ASSERT_EQ(vstorage_.GetFileMetaDataByNumber(999U), nullptr); +} + +TEST_F(VersionStorageInfoTest, ForcedBlobGCEmpty) { + // No SST or blob files in VersionStorageInfo + UpdateVersionStorageInfo(); + + constexpr double age_cutoff = 0.5; + constexpr double force_threshold = 0.75; + vstorage_.ComputeFilesMarkedForForcedBlobGC(age_cutoff, force_threshold); + + ASSERT_TRUE(vstorage_.FilesMarkedForForcedBlobGC().empty()); +} + +TEST_F(VersionStorageInfoTest, ForcedBlobGCSingleBatch) { + // Test the edge case when all blob files are part of the oldest batch. + // We have one L0 SST file #1, and four blob files #10, #11, #12, and #13. + // The oldest blob file used by SST #1 is blob file #10. + + constexpr int level = 0; + + constexpr uint64_t sst = 1; + + constexpr uint64_t first_blob = 10; + constexpr uint64_t second_blob = 11; + constexpr uint64_t third_blob = 12; + constexpr uint64_t fourth_blob = 13; + + { + constexpr char smallest[] = "bar1"; + constexpr char largest[] = "foo1"; + constexpr uint64_t file_size = 1000; + + Add(level, sst, smallest, largest, file_size, first_blob); + } + + { + constexpr uint64_t total_blob_count = 10; + constexpr uint64_t total_blob_bytes = 100000; + constexpr uint64_t garbage_blob_count = 2; + constexpr uint64_t garbage_blob_bytes = 15000; + + AddBlob(first_blob, total_blob_count, total_blob_bytes, + BlobFileMetaData::LinkedSsts{sst}, garbage_blob_count, + garbage_blob_bytes); + } + + { + constexpr uint64_t total_blob_count = 4; + constexpr uint64_t total_blob_bytes = 400000; + constexpr uint64_t garbage_blob_count = 3; + constexpr uint64_t garbage_blob_bytes = 235000; + + AddBlob(second_blob, total_blob_count, total_blob_bytes, + BlobFileMetaData::LinkedSsts{}, garbage_blob_count, + garbage_blob_bytes); + } + + { + constexpr uint64_t total_blob_count = 20; + constexpr uint64_t total_blob_bytes = 1000000; + constexpr uint64_t garbage_blob_count = 8; + constexpr uint64_t garbage_blob_bytes = 400000; + + AddBlob(third_blob, total_blob_count, total_blob_bytes, + BlobFileMetaData::LinkedSsts{}, garbage_blob_count, + garbage_blob_bytes); + } + + { + constexpr uint64_t total_blob_count = 128; + constexpr uint64_t total_blob_bytes = 1000000; + constexpr uint64_t garbage_blob_count = 67; + constexpr uint64_t garbage_blob_bytes = 600000; + + AddBlob(fourth_blob, total_blob_count, total_blob_bytes, + BlobFileMetaData::LinkedSsts{}, garbage_blob_count, + garbage_blob_bytes); + } + + UpdateVersionStorageInfo(); + + assert(vstorage_.num_levels() > 0); + const auto& level_files = vstorage_.LevelFiles(level); + + assert(level_files.size() == 1); + assert(level_files[0] && level_files[0]->fd.GetNumber() == sst); + + // No blob files eligible for GC due to the age cutoff + + { + constexpr double age_cutoff = 0.1; + constexpr double force_threshold = 0.0; + vstorage_.ComputeFilesMarkedForForcedBlobGC(age_cutoff, force_threshold); + + ASSERT_TRUE(vstorage_.FilesMarkedForForcedBlobGC().empty()); + } + + // Part of the oldest batch of blob files (specifically, #12 and #13) is + // ineligible for GC due to the age cutoff + + { + constexpr double age_cutoff = 0.5; + constexpr double force_threshold = 0.0; + vstorage_.ComputeFilesMarkedForForcedBlobGC(age_cutoff, force_threshold); + + ASSERT_TRUE(vstorage_.FilesMarkedForForcedBlobGC().empty()); + } + + // Oldest batch is eligible based on age cutoff but its overall garbage ratio + // is below threshold + + { + constexpr double age_cutoff = 1.0; + constexpr double force_threshold = 0.6; + vstorage_.ComputeFilesMarkedForForcedBlobGC(age_cutoff, force_threshold); + + ASSERT_TRUE(vstorage_.FilesMarkedForForcedBlobGC().empty()); + } + + // Oldest batch is eligible based on age cutoff and its overall garbage ratio + // meets threshold + + { + constexpr double age_cutoff = 1.0; + constexpr double force_threshold = 0.5; + vstorage_.ComputeFilesMarkedForForcedBlobGC(age_cutoff, force_threshold); + + auto ssts_to_be_compacted = vstorage_.FilesMarkedForForcedBlobGC(); + ASSERT_EQ(ssts_to_be_compacted.size(), 1); + + const autovector> + expected_ssts_to_be_compacted{{level, level_files[0]}}; + + ASSERT_EQ(ssts_to_be_compacted[0], expected_ssts_to_be_compacted[0]); + } +} + +TEST_F(VersionStorageInfoTest, ForcedBlobGCMultipleBatches) { + // Add three L0 SSTs (1, 2, and 3) and four blob files (10, 11, 12, and 13). + // The first two SSTs have the same oldest blob file, namely, the very oldest + // one (10), while the third SST's oldest blob file reference points to the + // third blob file (12). Thus, the oldest batch of blob files contains the + // first two blob files 10 and 11, and assuming they are eligible for GC based + // on the age cutoff, compacting away the SSTs 1 and 2 will eliminate them. + + constexpr int level = 0; + + constexpr uint64_t first_sst = 1; + constexpr uint64_t second_sst = 2; + constexpr uint64_t third_sst = 3; + + constexpr uint64_t first_blob = 10; + constexpr uint64_t second_blob = 11; + constexpr uint64_t third_blob = 12; + constexpr uint64_t fourth_blob = 13; + + { + constexpr char smallest[] = "bar1"; + constexpr char largest[] = "foo1"; + constexpr uint64_t file_size = 1000; + + Add(level, first_sst, smallest, largest, file_size, first_blob); + } + + { + constexpr char smallest[] = "bar2"; + constexpr char largest[] = "foo2"; + constexpr uint64_t file_size = 2000; + + Add(level, second_sst, smallest, largest, file_size, first_blob); + } + + { + constexpr char smallest[] = "bar3"; + constexpr char largest[] = "foo3"; + constexpr uint64_t file_size = 3000; + + Add(level, third_sst, smallest, largest, file_size, third_blob); + } + + { + constexpr uint64_t total_blob_count = 10; + constexpr uint64_t total_blob_bytes = 100000; + constexpr uint64_t garbage_blob_count = 2; + constexpr uint64_t garbage_blob_bytes = 15000; + + AddBlob(first_blob, total_blob_count, total_blob_bytes, + BlobFileMetaData::LinkedSsts{first_sst, second_sst}, + garbage_blob_count, garbage_blob_bytes); + } + + { + constexpr uint64_t total_blob_count = 4; + constexpr uint64_t total_blob_bytes = 400000; + constexpr uint64_t garbage_blob_count = 3; + constexpr uint64_t garbage_blob_bytes = 235000; + + AddBlob(second_blob, total_blob_count, total_blob_bytes, + BlobFileMetaData::LinkedSsts{}, garbage_blob_count, + garbage_blob_bytes); + } + + { + constexpr uint64_t total_blob_count = 20; + constexpr uint64_t total_blob_bytes = 1000000; + constexpr uint64_t garbage_blob_count = 8; + constexpr uint64_t garbage_blob_bytes = 123456; + + AddBlob(third_blob, total_blob_count, total_blob_bytes, + BlobFileMetaData::LinkedSsts{third_sst}, garbage_blob_count, + garbage_blob_bytes); + } + + { + constexpr uint64_t total_blob_count = 128; + constexpr uint64_t total_blob_bytes = 789012345; + constexpr uint64_t garbage_blob_count = 67; + constexpr uint64_t garbage_blob_bytes = 88888888; + + AddBlob(fourth_blob, total_blob_count, total_blob_bytes, + BlobFileMetaData::LinkedSsts{}, garbage_blob_count, + garbage_blob_bytes); + } + + UpdateVersionStorageInfo(); + + assert(vstorage_.num_levels() > 0); + const auto& level_files = vstorage_.LevelFiles(level); + + assert(level_files.size() == 3); + assert(level_files[0] && level_files[0]->fd.GetNumber() == first_sst); + assert(level_files[1] && level_files[1]->fd.GetNumber() == second_sst); + assert(level_files[2] && level_files[2]->fd.GetNumber() == third_sst); + + // No blob files eligible for GC due to the age cutoff + + { + constexpr double age_cutoff = 0.1; + constexpr double force_threshold = 0.0; + vstorage_.ComputeFilesMarkedForForcedBlobGC(age_cutoff, force_threshold); + + ASSERT_TRUE(vstorage_.FilesMarkedForForcedBlobGC().empty()); + } + + // Part of the oldest batch of blob files (specifically, the second file) is + // ineligible for GC due to the age cutoff + + { + constexpr double age_cutoff = 0.25; + constexpr double force_threshold = 0.0; + vstorage_.ComputeFilesMarkedForForcedBlobGC(age_cutoff, force_threshold); + + ASSERT_TRUE(vstorage_.FilesMarkedForForcedBlobGC().empty()); + } + + // Oldest batch is eligible based on age cutoff but its overall garbage ratio + // is below threshold + + { + constexpr double age_cutoff = 0.5; + constexpr double force_threshold = 0.6; + vstorage_.ComputeFilesMarkedForForcedBlobGC(age_cutoff, force_threshold); + + ASSERT_TRUE(vstorage_.FilesMarkedForForcedBlobGC().empty()); + } + + // Oldest batch is eligible based on age cutoff and its overall garbage ratio + // meets threshold + + { + constexpr double age_cutoff = 0.5; + constexpr double force_threshold = 0.5; + vstorage_.ComputeFilesMarkedForForcedBlobGC(age_cutoff, force_threshold); + + auto ssts_to_be_compacted = vstorage_.FilesMarkedForForcedBlobGC(); + ASSERT_EQ(ssts_to_be_compacted.size(), 2); + + std::sort(ssts_to_be_compacted.begin(), ssts_to_be_compacted.end(), + [](const std::pair& lhs, + const std::pair& rhs) { + assert(lhs.second); + assert(rhs.second); + return lhs.second->fd.GetNumber() < rhs.second->fd.GetNumber(); + }); + + const autovector> + expected_ssts_to_be_compacted{{level, level_files[0]}, + {level, level_files[1]}}; + + ASSERT_EQ(ssts_to_be_compacted[0], expected_ssts_to_be_compacted[0]); + ASSERT_EQ(ssts_to_be_compacted[1], expected_ssts_to_be_compacted[1]); + } + + // Now try the last two cases again with a greater than necessary age cutoff + + // Oldest batch is eligible based on age cutoff but its overall garbage ratio + // is below threshold + + { + constexpr double age_cutoff = 0.75; + constexpr double force_threshold = 0.6; + vstorage_.ComputeFilesMarkedForForcedBlobGC(age_cutoff, force_threshold); + + ASSERT_TRUE(vstorage_.FilesMarkedForForcedBlobGC().empty()); + } + + // Oldest batch is eligible based on age cutoff and its overall garbage ratio + // meets threshold + + { + constexpr double age_cutoff = 0.75; + constexpr double force_threshold = 0.5; + vstorage_.ComputeFilesMarkedForForcedBlobGC(age_cutoff, force_threshold); + + auto ssts_to_be_compacted = vstorage_.FilesMarkedForForcedBlobGC(); + ASSERT_EQ(ssts_to_be_compacted.size(), 2); + + std::sort(ssts_to_be_compacted.begin(), ssts_to_be_compacted.end(), + [](const std::pair& lhs, + const std::pair& rhs) { + assert(lhs.second); + assert(rhs.second); + return lhs.second->fd.GetNumber() < rhs.second->fd.GetNumber(); + }); + + const autovector> + expected_ssts_to_be_compacted{{level, level_files[0]}, + {level, level_files[1]}}; + + ASSERT_EQ(ssts_to_be_compacted[0], expected_ssts_to_be_compacted[0]); + ASSERT_EQ(ssts_to_be_compacted[1], expected_ssts_to_be_compacted[1]); + } +} + +class VersionStorageInfoTimestampTest : public VersionStorageInfoTestBase { + public: + VersionStorageInfoTimestampTest() + : VersionStorageInfoTestBase(test::BytewiseComparatorWithU64TsWrapper()) { + } + ~VersionStorageInfoTimestampTest() override {} + std::string Timestamp(uint64_t ts) const { + std::string ret; + PutFixed64(&ret, ts); + return ret; + } + std::string PackUserKeyAndTimestamp(const Slice& ukey, uint64_t ts) const { + std::string ret; + ret.assign(ukey.data(), ukey.size()); + PutFixed64(&ret, ts); + return ret; + } +}; + +TEST_F(VersionStorageInfoTimestampTest, GetOverlappingInputs) { + Add(/*level=*/1, /*file_number=*/1, /*smallest=*/ + {PackUserKeyAndTimestamp("a", /*ts=*/9), /*s=*/0, kTypeValue}, + /*largest=*/ + {PackUserKeyAndTimestamp("a", /*ts=*/8), /*s=*/0, kTypeValue}, + /*file_size=*/100); + Add(/*level=*/1, /*file_number=*/2, /*smallest=*/ + {PackUserKeyAndTimestamp("a", /*ts=*/5), /*s=*/0, kTypeValue}, + /*largest=*/ + {PackUserKeyAndTimestamp("b", /*ts=*/10), /*s=*/0, kTypeValue}, + /*file_size=*/100); + Add(/*level=*/1, /*file_number=*/3, /*smallest=*/ + {PackUserKeyAndTimestamp("c", /*ts=*/12), /*s=*/0, kTypeValue}, + /*largest=*/ + {PackUserKeyAndTimestamp("d", /*ts=*/1), /*s=*/0, kTypeValue}, + /*file_size=*/100); + + UpdateVersionStorageInfo(); + + ASSERT_EQ( + "1,2", + GetOverlappingFiles( + /*level=*/1, + {PackUserKeyAndTimestamp("a", /*ts=*/12), /*s=*/0, kTypeValue}, + {PackUserKeyAndTimestamp("a", /*ts=*/11), /*s=*/0, kTypeValue})); + ASSERT_EQ("3", + GetOverlappingFiles( + /*level=*/1, + {PackUserKeyAndTimestamp("c", /*ts=*/15), /*s=*/0, kTypeValue}, + {PackUserKeyAndTimestamp("c", /*ts=*/2), /*s=*/0, kTypeValue})); +} + +class FindLevelFileTest : public testing::Test { + public: + LevelFilesBrief file_level_; + bool disjoint_sorted_files_; + Arena arena_; + + FindLevelFileTest() : disjoint_sorted_files_(true) {} + + ~FindLevelFileTest() override {} + + void LevelFileInit(size_t num = 0) { + char* mem = arena_.AllocateAligned(num * sizeof(FdWithKeyRange)); + file_level_.files = new (mem) FdWithKeyRange[num]; + file_level_.num_files = 0; + } + + void Add(const char* smallest, const char* largest, + SequenceNumber smallest_seq = 100, + SequenceNumber largest_seq = 100) { + InternalKey smallest_key = InternalKey(smallest, smallest_seq, kTypeValue); + InternalKey largest_key = InternalKey(largest, largest_seq, kTypeValue); + + Slice smallest_slice = smallest_key.Encode(); + Slice largest_slice = largest_key.Encode(); + + char* mem = + arena_.AllocateAligned(smallest_slice.size() + largest_slice.size()); + memcpy(mem, smallest_slice.data(), smallest_slice.size()); + memcpy(mem + smallest_slice.size(), largest_slice.data(), + largest_slice.size()); + + // add to file_level_ + size_t num = file_level_.num_files; + auto& file = file_level_.files[num]; + file.fd = FileDescriptor(num + 1, 0, 0); + file.smallest_key = Slice(mem, smallest_slice.size()); + file.largest_key = Slice(mem + smallest_slice.size(), largest_slice.size()); + file_level_.num_files++; + } + + int Find(const char* key) { + InternalKey target(key, 100, kTypeValue); + InternalKeyComparator cmp(BytewiseComparator()); + return FindFile(cmp, file_level_, target.Encode()); + } + + bool Overlaps(const char* smallest, const char* largest) { + InternalKeyComparator cmp(BytewiseComparator()); + Slice s(smallest != nullptr ? smallest : ""); + Slice l(largest != nullptr ? largest : ""); + return SomeFileOverlapsRange(cmp, disjoint_sorted_files_, file_level_, + (smallest != nullptr ? &s : nullptr), + (largest != nullptr ? &l : nullptr)); + } +}; + +TEST_F(FindLevelFileTest, LevelEmpty) { + LevelFileInit(0); + + ASSERT_EQ(0, Find("foo")); + ASSERT_TRUE(!Overlaps("a", "z")); + ASSERT_TRUE(!Overlaps(nullptr, "z")); + ASSERT_TRUE(!Overlaps("a", nullptr)); + ASSERT_TRUE(!Overlaps(nullptr, nullptr)); +} + +TEST_F(FindLevelFileTest, LevelSingle) { + LevelFileInit(1); + + Add("p", "q"); + ASSERT_EQ(0, Find("a")); + ASSERT_EQ(0, Find("p")); + ASSERT_EQ(0, Find("p1")); + ASSERT_EQ(0, Find("q")); + ASSERT_EQ(1, Find("q1")); + ASSERT_EQ(1, Find("z")); + + ASSERT_TRUE(!Overlaps("a", "b")); + ASSERT_TRUE(!Overlaps("z1", "z2")); + ASSERT_TRUE(Overlaps("a", "p")); + ASSERT_TRUE(Overlaps("a", "q")); + ASSERT_TRUE(Overlaps("a", "z")); + ASSERT_TRUE(Overlaps("p", "p1")); + ASSERT_TRUE(Overlaps("p", "q")); + ASSERT_TRUE(Overlaps("p", "z")); + ASSERT_TRUE(Overlaps("p1", "p2")); + ASSERT_TRUE(Overlaps("p1", "z")); + ASSERT_TRUE(Overlaps("q", "q")); + ASSERT_TRUE(Overlaps("q", "q1")); + + ASSERT_TRUE(!Overlaps(nullptr, "j")); + ASSERT_TRUE(!Overlaps("r", nullptr)); + ASSERT_TRUE(Overlaps(nullptr, "p")); + ASSERT_TRUE(Overlaps(nullptr, "p1")); + ASSERT_TRUE(Overlaps("q", nullptr)); + ASSERT_TRUE(Overlaps(nullptr, nullptr)); +} + +TEST_F(FindLevelFileTest, LevelMultiple) { + LevelFileInit(4); + + Add("150", "200"); + Add("200", "250"); + Add("300", "350"); + Add("400", "450"); + ASSERT_EQ(0, Find("100")); + ASSERT_EQ(0, Find("150")); + ASSERT_EQ(0, Find("151")); + ASSERT_EQ(0, Find("199")); + ASSERT_EQ(0, Find("200")); + ASSERT_EQ(1, Find("201")); + ASSERT_EQ(1, Find("249")); + ASSERT_EQ(1, Find("250")); + ASSERT_EQ(2, Find("251")); + ASSERT_EQ(2, Find("299")); + ASSERT_EQ(2, Find("300")); + ASSERT_EQ(2, Find("349")); + ASSERT_EQ(2, Find("350")); + ASSERT_EQ(3, Find("351")); + ASSERT_EQ(3, Find("400")); + ASSERT_EQ(3, Find("450")); + ASSERT_EQ(4, Find("451")); + + ASSERT_TRUE(!Overlaps("100", "149")); + ASSERT_TRUE(!Overlaps("251", "299")); + ASSERT_TRUE(!Overlaps("451", "500")); + ASSERT_TRUE(!Overlaps("351", "399")); + + ASSERT_TRUE(Overlaps("100", "150")); + ASSERT_TRUE(Overlaps("100", "200")); + ASSERT_TRUE(Overlaps("100", "300")); + ASSERT_TRUE(Overlaps("100", "400")); + ASSERT_TRUE(Overlaps("100", "500")); + ASSERT_TRUE(Overlaps("375", "400")); + ASSERT_TRUE(Overlaps("450", "450")); + ASSERT_TRUE(Overlaps("450", "500")); +} + +TEST_F(FindLevelFileTest, LevelMultipleNullBoundaries) { + LevelFileInit(4); + + Add("150", "200"); + Add("200", "250"); + Add("300", "350"); + Add("400", "450"); + ASSERT_TRUE(!Overlaps(nullptr, "149")); + ASSERT_TRUE(!Overlaps("451", nullptr)); + ASSERT_TRUE(Overlaps(nullptr, nullptr)); + ASSERT_TRUE(Overlaps(nullptr, "150")); + ASSERT_TRUE(Overlaps(nullptr, "199")); + ASSERT_TRUE(Overlaps(nullptr, "200")); + ASSERT_TRUE(Overlaps(nullptr, "201")); + ASSERT_TRUE(Overlaps(nullptr, "400")); + ASSERT_TRUE(Overlaps(nullptr, "800")); + ASSERT_TRUE(Overlaps("100", nullptr)); + ASSERT_TRUE(Overlaps("200", nullptr)); + ASSERT_TRUE(Overlaps("449", nullptr)); + ASSERT_TRUE(Overlaps("450", nullptr)); +} + +TEST_F(FindLevelFileTest, LevelOverlapSequenceChecks) { + LevelFileInit(1); + + Add("200", "200", 5000, 3000); + ASSERT_TRUE(!Overlaps("199", "199")); + ASSERT_TRUE(!Overlaps("201", "300")); + ASSERT_TRUE(Overlaps("200", "200")); + ASSERT_TRUE(Overlaps("190", "200")); + ASSERT_TRUE(Overlaps("200", "210")); +} + +TEST_F(FindLevelFileTest, LevelOverlappingFiles) { + LevelFileInit(2); + + Add("150", "600"); + Add("400", "500"); + disjoint_sorted_files_ = false; + ASSERT_TRUE(!Overlaps("100", "149")); + ASSERT_TRUE(!Overlaps("601", "700")); + ASSERT_TRUE(Overlaps("100", "150")); + ASSERT_TRUE(Overlaps("100", "200")); + ASSERT_TRUE(Overlaps("100", "300")); + ASSERT_TRUE(Overlaps("100", "400")); + ASSERT_TRUE(Overlaps("100", "500")); + ASSERT_TRUE(Overlaps("375", "400")); + ASSERT_TRUE(Overlaps("450", "450")); + ASSERT_TRUE(Overlaps("450", "500")); + ASSERT_TRUE(Overlaps("450", "700")); + ASSERT_TRUE(Overlaps("600", "700")); +} + +class VersionSetTestBase { + public: + const static std::string kColumnFamilyName1; + const static std::string kColumnFamilyName2; + const static std::string kColumnFamilyName3; + int num_initial_edits_; + + explicit VersionSetTestBase(const std::string& name) + : env_(nullptr), + dbname_(test::PerThreadDBPath(name)), + options_(), + db_options_(options_), + cf_options_(options_), + immutable_options_(db_options_, cf_options_), + mutable_cf_options_(cf_options_), + table_cache_(NewLRUCache(50000, 16)), + write_buffer_manager_(db_options_.db_write_buffer_size), + shutting_down_(false), + mock_table_factory_(std::make_shared()) { + EXPECT_OK(test::CreateEnvFromSystem(ConfigOptions(), &env_, &env_guard_)); + if (env_ == Env::Default() && getenv("MEM_ENV")) { + env_guard_.reset(NewMemEnv(Env::Default())); + env_ = env_guard_.get(); + } + EXPECT_NE(nullptr, env_); + + fs_ = env_->GetFileSystem(); + EXPECT_OK(fs_->CreateDirIfMissing(dbname_, IOOptions(), nullptr)); + + options_.env = env_; + db_options_.env = env_; + db_options_.fs = fs_; + immutable_options_.env = env_; + immutable_options_.fs = fs_; + immutable_options_.clock = env_->GetSystemClock().get(); + + versions_.reset( + new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id*/ "", /*db_session_id*/ "")); + reactive_versions_ = std::make_shared( + dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, nullptr); + db_options_.db_paths.emplace_back(dbname_, + std::numeric_limits::max()); + } + + virtual ~VersionSetTestBase() { + if (getenv("KEEP_DB")) { + fprintf(stdout, "DB is still at %s\n", dbname_.c_str()); + } else { + Options options; + options.env = env_; + EXPECT_OK(DestroyDB(dbname_, options)); + } + } + + protected: + virtual void PrepareManifest( + std::vector* column_families, + SequenceNumber* last_seqno, std::unique_ptr* log_writer) { + assert(column_families != nullptr); + assert(last_seqno != nullptr); + assert(log_writer != nullptr); + VersionEdit new_db; + if (db_options_.write_dbid_to_manifest) { + DBOptions tmp_db_options; + tmp_db_options.env = env_; + std::unique_ptr impl(new DBImpl(tmp_db_options, dbname_)); + std::string db_id; + impl->GetDbIdentityFromIdentityFile(&db_id); + new_db.SetDBId(db_id); + } + new_db.SetLogNumber(0); + new_db.SetNextFile(2); + new_db.SetLastSequence(0); + + const std::vector cf_names = { + kDefaultColumnFamilyName, kColumnFamilyName1, kColumnFamilyName2, + kColumnFamilyName3}; + const int kInitialNumOfCfs = static_cast(cf_names.size()); + autovector new_cfs; + uint64_t last_seq = 1; + uint32_t cf_id = 1; + for (int i = 1; i != kInitialNumOfCfs; ++i) { + VersionEdit new_cf; + new_cf.AddColumnFamily(cf_names[i]); + new_cf.SetColumnFamily(cf_id++); + new_cf.SetLogNumber(0); + new_cf.SetNextFile(2); + new_cf.SetLastSequence(last_seq++); + new_cfs.emplace_back(new_cf); + } + *last_seqno = last_seq; + num_initial_edits_ = static_cast(new_cfs.size() + 1); + std::unique_ptr file_writer; + const std::string manifest = DescriptorFileName(dbname_, 1); + const auto& fs = env_->GetFileSystem(); + Status s = WritableFileWriter::Create( + fs, manifest, fs->OptimizeForManifestWrite(env_options_), &file_writer, + nullptr); + ASSERT_OK(s); + { + log_writer->reset(new log::Writer(std::move(file_writer), 0, false)); + std::string record; + new_db.EncodeTo(&record); + s = (*log_writer)->AddRecord(record); + for (const auto& e : new_cfs) { + record.clear(); + e.EncodeTo(&record); + s = (*log_writer)->AddRecord(record); + ASSERT_OK(s); + } + } + ASSERT_OK(s); + + cf_options_.table_factory = mock_table_factory_; + for (const auto& cf_name : cf_names) { + column_families->emplace_back(cf_name, cf_options_); + } + } + + // Create DB with 3 column families. + void NewDB() { + SequenceNumber last_seqno; + std::unique_ptr log_writer; + SetIdentityFile(env_, dbname_); + PrepareManifest(&column_families_, &last_seqno, &log_writer); + log_writer.reset(); + // Make "CURRENT" file point to the new manifest file. + Status s = SetCurrentFile(fs_.get(), dbname_, 1, nullptr); + ASSERT_OK(s); + + EXPECT_OK(versions_->Recover(column_families_, false)); + EXPECT_EQ(column_families_.size(), + versions_->GetColumnFamilySet()->NumberOfColumnFamilies()); + } + + void ReopenDB() { + versions_.reset( + new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id*/ "", /*db_session_id*/ "")); + EXPECT_OK(versions_->Recover(column_families_, false)); + } + + void VerifyManifest(std::string* manifest_path) const { + assert(manifest_path != nullptr); + uint64_t manifest_file_number = 0; + Status s = versions_->GetCurrentManifestPath( + dbname_, fs_.get(), manifest_path, &manifest_file_number); + ASSERT_OK(s); + ASSERT_EQ(1, manifest_file_number); + } + + Status LogAndApplyToDefaultCF(VersionEdit& edit) { + mutex_.Lock(); + Status s = versions_->LogAndApply( + versions_->GetColumnFamilySet()->GetDefault(), mutable_cf_options_, + read_options_, &edit, &mutex_, nullptr); + mutex_.Unlock(); + return s; + } + + Status LogAndApplyToDefaultCF( + const autovector>& edits) { + autovector vedits; + for (auto& e : edits) { + vedits.push_back(e.get()); + } + mutex_.Lock(); + Status s = versions_->LogAndApply( + versions_->GetColumnFamilySet()->GetDefault(), mutable_cf_options_, + read_options_, vedits, &mutex_, nullptr); + mutex_.Unlock(); + return s; + } + + void CreateNewManifest() { + constexpr FSDirectory* db_directory = nullptr; + constexpr bool new_descriptor_log = true; + mutex_.Lock(); + VersionEdit dummy; + ASSERT_OK(versions_->LogAndApply( + versions_->GetColumnFamilySet()->GetDefault(), mutable_cf_options_, + read_options_, &dummy, &mutex_, db_directory, new_descriptor_log)); + mutex_.Unlock(); + } + + ColumnFamilyData* CreateColumnFamily(const std::string& cf_name, + const ColumnFamilyOptions& cf_options) { + VersionEdit new_cf; + new_cf.AddColumnFamily(cf_name); + uint32_t new_id = versions_->GetColumnFamilySet()->GetNextColumnFamilyID(); + new_cf.SetColumnFamily(new_id); + new_cf.SetLogNumber(0); + new_cf.SetComparatorName(cf_options.comparator->Name()); + new_cf.SetPersistUserDefinedTimestamps( + cf_options.persist_user_defined_timestamps); + Status s; + mutex_.Lock(); + s = versions_->LogAndApply(/*column_family_data=*/nullptr, + MutableCFOptions(cf_options), read_options_, + &new_cf, &mutex_, + /*db_directory=*/nullptr, + /*new_descriptor_log=*/false, &cf_options); + mutex_.Unlock(); + EXPECT_OK(s); + ColumnFamilyData* cfd = + versions_->GetColumnFamilySet()->GetColumnFamily(cf_name); + EXPECT_NE(nullptr, cfd); + return cfd; + } + + Env* mem_env_; + Env* env_; + std::shared_ptr env_guard_; + std::shared_ptr fs_; + const std::string dbname_; + EnvOptions env_options_; + Options options_; + ImmutableDBOptions db_options_; + ColumnFamilyOptions cf_options_; + ImmutableOptions immutable_options_; + MutableCFOptions mutable_cf_options_; + const ReadOptions read_options_; + std::shared_ptr table_cache_; + WriteController write_controller_; + WriteBufferManager write_buffer_manager_; + std::shared_ptr versions_; + std::shared_ptr reactive_versions_; + InstrumentedMutex mutex_; + std::atomic shutting_down_; + std::shared_ptr mock_table_factory_; + std::vector column_families_; +}; + +const std::string VersionSetTestBase::kColumnFamilyName1 = "alice"; +const std::string VersionSetTestBase::kColumnFamilyName2 = "bob"; +const std::string VersionSetTestBase::kColumnFamilyName3 = "charles"; + +class VersionSetTest : public VersionSetTestBase, public testing::Test { + public: + VersionSetTest() : VersionSetTestBase("version_set_test") {} +}; + +TEST_F(VersionSetTest, SameColumnFamilyGroupCommit) { + NewDB(); + const int kGroupSize = 5; + const ReadOptions read_options; + + autovector edits; + for (int i = 0; i != kGroupSize; ++i) { + edits.emplace_back(VersionEdit()); + } + autovector cfds; + autovector all_mutable_cf_options; + autovector> edit_lists; + for (int i = 0; i != kGroupSize; ++i) { + cfds.emplace_back(versions_->GetColumnFamilySet()->GetDefault()); + all_mutable_cf_options.emplace_back(&mutable_cf_options_); + autovector edit_list; + edit_list.emplace_back(&edits[i]); + edit_lists.emplace_back(edit_list); + } + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + int count = 0; + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::ProcessManifestWrites:SameColumnFamily", [&](void* arg) { + uint32_t* cf_id = reinterpret_cast(arg); + EXPECT_EQ(0u, *cf_id); + ++count; + }); + SyncPoint::GetInstance()->EnableProcessing(); + mutex_.Lock(); + Status s = versions_->LogAndApply(cfds, all_mutable_cf_options, read_options, + edit_lists, &mutex_, nullptr); + mutex_.Unlock(); + EXPECT_OK(s); + EXPECT_EQ(kGroupSize - 1, count); +} + +TEST_F(VersionSetTest, PersistBlobFileStateInNewManifest) { + // Initialize the database and add a couple of blob files, one with some + // garbage in it, and one without any garbage. + NewDB(); + + assert(versions_); + assert(versions_->GetColumnFamilySet()); + + ColumnFamilyData* const cfd = versions_->GetColumnFamilySet()->GetDefault(); + assert(cfd); + + Version* const version = cfd->current(); + assert(version); + + VersionStorageInfo* const storage_info = version->storage_info(); + assert(storage_info); + + { + constexpr uint64_t blob_file_number = 123; + constexpr uint64_t total_blob_count = 456; + constexpr uint64_t total_blob_bytes = 77777777; + constexpr char checksum_method[] = "SHA1"; + constexpr char checksum_value[] = + "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c" + "\x52\x5c\xbd"; + + auto shared_meta = SharedBlobFileMetaData::Create( + blob_file_number, total_blob_count, total_blob_bytes, checksum_method, + checksum_value); + + constexpr uint64_t garbage_blob_count = 89; + constexpr uint64_t garbage_blob_bytes = 1000000; + + auto meta = BlobFileMetaData::Create( + std::move(shared_meta), BlobFileMetaData::LinkedSsts(), + garbage_blob_count, garbage_blob_bytes); + + storage_info->AddBlobFile(std::move(meta)); + } + + { + constexpr uint64_t blob_file_number = 234; + constexpr uint64_t total_blob_count = 555; + constexpr uint64_t total_blob_bytes = 66666; + constexpr char checksum_method[] = "CRC32"; + constexpr char checksum_value[] = "\x3d\x87\xff\x57"; + + auto shared_meta = SharedBlobFileMetaData::Create( + blob_file_number, total_blob_count, total_blob_bytes, checksum_method, + checksum_value); + + constexpr uint64_t garbage_blob_count = 0; + constexpr uint64_t garbage_blob_bytes = 0; + + auto meta = BlobFileMetaData::Create( + std::move(shared_meta), BlobFileMetaData::LinkedSsts(), + garbage_blob_count, garbage_blob_bytes); + + storage_info->AddBlobFile(std::move(meta)); + } + + // Force the creation of a new manifest file and make sure metadata for + // the blob files is re-persisted. + size_t addition_encoded = 0; + SyncPoint::GetInstance()->SetCallBack( + "BlobFileAddition::EncodeTo::CustomFields", + [&](void* /* arg */) { ++addition_encoded; }); + + size_t garbage_encoded = 0; + SyncPoint::GetInstance()->SetCallBack( + "BlobFileGarbage::EncodeTo::CustomFields", + [&](void* /* arg */) { ++garbage_encoded; }); + SyncPoint::GetInstance()->EnableProcessing(); + + CreateNewManifest(); + + ASSERT_EQ(addition_encoded, 2); + ASSERT_EQ(garbage_encoded, 1); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(VersionSetTest, AddLiveBlobFiles) { + // Initialize the database and add a blob file. + NewDB(); + + assert(versions_); + assert(versions_->GetColumnFamilySet()); + + ColumnFamilyData* const cfd = versions_->GetColumnFamilySet()->GetDefault(); + assert(cfd); + + Version* const first_version = cfd->current(); + assert(first_version); + + VersionStorageInfo* const first_storage_info = first_version->storage_info(); + assert(first_storage_info); + + constexpr uint64_t first_blob_file_number = 234; + constexpr uint64_t first_total_blob_count = 555; + constexpr uint64_t first_total_blob_bytes = 66666; + constexpr char first_checksum_method[] = "CRC32"; + constexpr char first_checksum_value[] = "\x3d\x87\xff\x57"; + + auto first_shared_meta = SharedBlobFileMetaData::Create( + first_blob_file_number, first_total_blob_count, first_total_blob_bytes, + first_checksum_method, first_checksum_value); + + constexpr uint64_t garbage_blob_count = 0; + constexpr uint64_t garbage_blob_bytes = 0; + + auto first_meta = BlobFileMetaData::Create( + std::move(first_shared_meta), BlobFileMetaData::LinkedSsts(), + garbage_blob_count, garbage_blob_bytes); + + first_storage_info->AddBlobFile(first_meta); + + // Reference the version so it stays alive even after the following version + // edit. + first_version->Ref(); + + // Get live files directly from version. + std::vector version_table_files; + std::vector version_blob_files; + + first_version->AddLiveFiles(&version_table_files, &version_blob_files); + + ASSERT_EQ(version_blob_files.size(), 1); + ASSERT_EQ(version_blob_files[0], first_blob_file_number); + + // Create a new version containing an additional blob file. + versions_->TEST_CreateAndAppendVersion(cfd); + + Version* const second_version = cfd->current(); + assert(second_version); + assert(second_version != first_version); + + VersionStorageInfo* const second_storage_info = + second_version->storage_info(); + assert(second_storage_info); + + constexpr uint64_t second_blob_file_number = 456; + constexpr uint64_t second_total_blob_count = 100; + constexpr uint64_t second_total_blob_bytes = 2000000; + constexpr char second_checksum_method[] = "CRC32B"; + constexpr char second_checksum_value[] = "\x6d\xbd\xf2\x3a"; + + auto second_shared_meta = SharedBlobFileMetaData::Create( + second_blob_file_number, second_total_blob_count, second_total_blob_bytes, + second_checksum_method, second_checksum_value); + + auto second_meta = BlobFileMetaData::Create( + std::move(second_shared_meta), BlobFileMetaData::LinkedSsts(), + garbage_blob_count, garbage_blob_bytes); + + second_storage_info->AddBlobFile(std::move(first_meta)); + second_storage_info->AddBlobFile(std::move(second_meta)); + + // Get all live files from version set. Note that the result contains + // duplicates. + std::vector all_table_files; + std::vector all_blob_files; + + versions_->AddLiveFiles(&all_table_files, &all_blob_files); + + ASSERT_EQ(all_blob_files.size(), 3); + ASSERT_EQ(all_blob_files[0], first_blob_file_number); + ASSERT_EQ(all_blob_files[1], first_blob_file_number); + ASSERT_EQ(all_blob_files[2], second_blob_file_number); + + // Clean up previous version. + first_version->Unref(); +} + +TEST_F(VersionSetTest, ObsoleteBlobFile) { + // Initialize the database and add a blob file that is entirely garbage + // and thus can immediately be marked obsolete. + NewDB(); + + VersionEdit edit; + + constexpr uint64_t blob_file_number = 234; + constexpr uint64_t total_blob_count = 555; + constexpr uint64_t total_blob_bytes = 66666; + constexpr char checksum_method[] = "CRC32"; + constexpr char checksum_value[] = "\x3d\x87\xff\x57"; + + edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes, + checksum_method, checksum_value); + + edit.AddBlobFileGarbage(blob_file_number, total_blob_count, total_blob_bytes); + + mutex_.Lock(); + Status s = versions_->LogAndApply( + versions_->GetColumnFamilySet()->GetDefault(), mutable_cf_options_, + read_options_, &edit, &mutex_, nullptr); + mutex_.Unlock(); + + ASSERT_OK(s); + + // Make sure blob files from the pending number range are not returned + // as obsolete. + { + std::vector table_files; + std::vector blob_files; + std::vector manifest_files; + constexpr uint64_t min_pending_output = blob_file_number; + + versions_->GetObsoleteFiles(&table_files, &blob_files, &manifest_files, + min_pending_output); + + ASSERT_TRUE(blob_files.empty()); + } + + // Make sure the blob file is returned as obsolete if it's not in the pending + // range. + { + std::vector table_files; + std::vector blob_files; + std::vector manifest_files; + constexpr uint64_t min_pending_output = blob_file_number + 1; + + versions_->GetObsoleteFiles(&table_files, &blob_files, &manifest_files, + min_pending_output); + + ASSERT_EQ(blob_files.size(), 1); + ASSERT_EQ(blob_files[0].GetBlobFileNumber(), blob_file_number); + } + + // Make sure it's not returned a second time. + { + std::vector table_files; + std::vector blob_files; + std::vector manifest_files; + constexpr uint64_t min_pending_output = blob_file_number + 1; + + versions_->GetObsoleteFiles(&table_files, &blob_files, &manifest_files, + min_pending_output); + + ASSERT_TRUE(blob_files.empty()); + } +} + +TEST_F(VersionSetTest, WalEditsNotAppliedToVersion) { + NewDB(); + + constexpr uint64_t kNumWals = 5; + + autovector> edits; + // Add some WALs. + for (uint64_t i = 1; i <= kNumWals; i++) { + edits.emplace_back(new VersionEdit); + // WAL's size equals its log number. + edits.back()->AddWal(i, WalMetadata(i)); + } + // Delete the first half of the WALs. + edits.emplace_back(new VersionEdit); + edits.back()->DeleteWalsBefore(kNumWals / 2 + 1); + + autovector versions; + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::ProcessManifestWrites:NewVersion", + [&](void* arg) { versions.push_back(reinterpret_cast(arg)); }); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(LogAndApplyToDefaultCF(edits)); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + // Since the edits are all WAL edits, no version should be created. + ASSERT_EQ(versions.size(), 1); + ASSERT_EQ(versions[0], nullptr); +} + +// Similar to WalEditsNotAppliedToVersion, but contains a non-WAL edit. +TEST_F(VersionSetTest, NonWalEditsAppliedToVersion) { + NewDB(); + + const std::string kDBId = "db_db"; + constexpr uint64_t kNumWals = 5; + + autovector> edits; + // Add some WALs. + for (uint64_t i = 1; i <= kNumWals; i++) { + edits.emplace_back(new VersionEdit); + // WAL's size equals its log number. + edits.back()->AddWal(i, WalMetadata(i)); + } + // Delete the first half of the WALs. + edits.emplace_back(new VersionEdit); + edits.back()->DeleteWalsBefore(kNumWals / 2 + 1); + edits.emplace_back(new VersionEdit); + edits.back()->SetDBId(kDBId); + + autovector versions; + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::ProcessManifestWrites:NewVersion", + [&](void* arg) { versions.push_back(reinterpret_cast(arg)); }); + SyncPoint::GetInstance()->EnableProcessing(); + + ASSERT_OK(LogAndApplyToDefaultCF(edits)); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + // Since the edits are all WAL edits, no version should be created. + ASSERT_EQ(versions.size(), 1); + ASSERT_NE(versions[0], nullptr); +} + +TEST_F(VersionSetTest, WalAddition) { + NewDB(); + + constexpr WalNumber kLogNumber = 10; + constexpr uint64_t kSizeInBytes = 111; + + // A WAL is just created. + { + VersionEdit edit; + edit.AddWal(kLogNumber); + + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + + const auto& wals = versions_->GetWalSet().GetWals(); + ASSERT_EQ(wals.size(), 1); + ASSERT_TRUE(wals.find(kLogNumber) != wals.end()); + ASSERT_FALSE(wals.at(kLogNumber).HasSyncedSize()); + } + + // The WAL is synced for several times before closing. + { + for (uint64_t size_delta = 100; size_delta > 0; size_delta /= 2) { + uint64_t size = kSizeInBytes - size_delta; + WalMetadata wal(size); + VersionEdit edit; + edit.AddWal(kLogNumber, wal); + + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + + const auto& wals = versions_->GetWalSet().GetWals(); + ASSERT_EQ(wals.size(), 1); + ASSERT_TRUE(wals.find(kLogNumber) != wals.end()); + ASSERT_TRUE(wals.at(kLogNumber).HasSyncedSize()); + ASSERT_EQ(wals.at(kLogNumber).GetSyncedSizeInBytes(), size); + } + } + + // The WAL is closed. + { + WalMetadata wal(kSizeInBytes); + VersionEdit edit; + edit.AddWal(kLogNumber, wal); + + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + + const auto& wals = versions_->GetWalSet().GetWals(); + ASSERT_EQ(wals.size(), 1); + ASSERT_TRUE(wals.find(kLogNumber) != wals.end()); + ASSERT_TRUE(wals.at(kLogNumber).HasSyncedSize()); + ASSERT_EQ(wals.at(kLogNumber).GetSyncedSizeInBytes(), kSizeInBytes); + } + + // Recover a new VersionSet. + { + std::unique_ptr new_versions( + new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id*/ "", /*db_session_id*/ "")); + ASSERT_OK(new_versions->Recover(column_families_, /*read_only=*/false)); + const auto& wals = new_versions->GetWalSet().GetWals(); + ASSERT_EQ(wals.size(), 1); + ASSERT_TRUE(wals.find(kLogNumber) != wals.end()); + ASSERT_TRUE(wals.at(kLogNumber).HasSyncedSize()); + ASSERT_EQ(wals.at(kLogNumber).GetSyncedSizeInBytes(), kSizeInBytes); + } +} + +TEST_F(VersionSetTest, WalCloseWithoutSync) { + NewDB(); + + constexpr WalNumber kLogNumber = 10; + constexpr uint64_t kSizeInBytes = 111; + constexpr uint64_t kSyncedSizeInBytes = kSizeInBytes / 2; + + // A WAL is just created. + { + VersionEdit edit; + edit.AddWal(kLogNumber); + + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + + const auto& wals = versions_->GetWalSet().GetWals(); + ASSERT_EQ(wals.size(), 1); + ASSERT_TRUE(wals.find(kLogNumber) != wals.end()); + ASSERT_FALSE(wals.at(kLogNumber).HasSyncedSize()); + } + + // The WAL is synced before closing. + { + WalMetadata wal(kSyncedSizeInBytes); + VersionEdit edit; + edit.AddWal(kLogNumber, wal); + + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + + const auto& wals = versions_->GetWalSet().GetWals(); + ASSERT_EQ(wals.size(), 1); + ASSERT_TRUE(wals.find(kLogNumber) != wals.end()); + ASSERT_TRUE(wals.at(kLogNumber).HasSyncedSize()); + ASSERT_EQ(wals.at(kLogNumber).GetSyncedSizeInBytes(), kSyncedSizeInBytes); + } + + // A new WAL with larger log number is created, + // implicitly marking the current WAL closed. + { + VersionEdit edit; + edit.AddWal(kLogNumber + 1); + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + + const auto& wals = versions_->GetWalSet().GetWals(); + ASSERT_EQ(wals.size(), 2); + ASSERT_TRUE(wals.find(kLogNumber) != wals.end()); + ASSERT_TRUE(wals.at(kLogNumber).HasSyncedSize()); + ASSERT_EQ(wals.at(kLogNumber).GetSyncedSizeInBytes(), kSyncedSizeInBytes); + ASSERT_TRUE(wals.find(kLogNumber + 1) != wals.end()); + ASSERT_FALSE(wals.at(kLogNumber + 1).HasSyncedSize()); + } + + // Recover a new VersionSet. + { + std::unique_ptr new_versions( + new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id*/ "", /*db_session_id*/ "")); + ASSERT_OK(new_versions->Recover(column_families_, false)); + const auto& wals = new_versions->GetWalSet().GetWals(); + ASSERT_EQ(wals.size(), 2); + ASSERT_TRUE(wals.find(kLogNumber) != wals.end()); + ASSERT_TRUE(wals.at(kLogNumber).HasSyncedSize()); + ASSERT_EQ(wals.at(kLogNumber).GetSyncedSizeInBytes(), kSyncedSizeInBytes); + } +} + +TEST_F(VersionSetTest, WalDeletion) { + NewDB(); + + constexpr WalNumber kClosedLogNumber = 10; + constexpr WalNumber kNonClosedLogNumber = 20; + constexpr uint64_t kSizeInBytes = 111; + + // Add a non-closed and a closed WAL. + { + VersionEdit edit; + edit.AddWal(kClosedLogNumber, WalMetadata(kSizeInBytes)); + edit.AddWal(kNonClosedLogNumber); + + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + + const auto& wals = versions_->GetWalSet().GetWals(); + ASSERT_EQ(wals.size(), 2); + ASSERT_TRUE(wals.find(kNonClosedLogNumber) != wals.end()); + ASSERT_TRUE(wals.find(kClosedLogNumber) != wals.end()); + ASSERT_FALSE(wals.at(kNonClosedLogNumber).HasSyncedSize()); + ASSERT_TRUE(wals.at(kClosedLogNumber).HasSyncedSize()); + ASSERT_EQ(wals.at(kClosedLogNumber).GetSyncedSizeInBytes(), kSizeInBytes); + } + + // Delete the closed WAL. + { + VersionEdit edit; + edit.DeleteWalsBefore(kNonClosedLogNumber); + + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + + const auto& wals = versions_->GetWalSet().GetWals(); + ASSERT_EQ(wals.size(), 1); + ASSERT_TRUE(wals.find(kNonClosedLogNumber) != wals.end()); + ASSERT_FALSE(wals.at(kNonClosedLogNumber).HasSyncedSize()); + } + + // Recover a new VersionSet, only the non-closed WAL should show up. + { + std::unique_ptr new_versions( + new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id*/ "", /*db_session_id*/ "")); + ASSERT_OK(new_versions->Recover(column_families_, false)); + const auto& wals = new_versions->GetWalSet().GetWals(); + ASSERT_EQ(wals.size(), 1); + ASSERT_TRUE(wals.find(kNonClosedLogNumber) != wals.end()); + ASSERT_FALSE(wals.at(kNonClosedLogNumber).HasSyncedSize()); + } + + // Force the creation of a new MANIFEST file, + // only the non-closed WAL should be written to the new MANIFEST. + { + std::vector wal_additions; + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::WriteCurrentStateToManifest:SaveWal", [&](void* arg) { + VersionEdit* edit = reinterpret_cast(arg); + ASSERT_TRUE(edit->IsWalAddition()); + for (auto& addition : edit->GetWalAdditions()) { + wal_additions.push_back(addition); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + + CreateNewManifest(); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + ASSERT_EQ(wal_additions.size(), 1); + ASSERT_EQ(wal_additions[0].GetLogNumber(), kNonClosedLogNumber); + ASSERT_FALSE(wal_additions[0].GetMetadata().HasSyncedSize()); + } + + // Recover from the new MANIFEST, only the non-closed WAL should show up. + { + std::unique_ptr new_versions( + new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id*/ "", /*db_session_id*/ "")); + ASSERT_OK(new_versions->Recover(column_families_, false)); + const auto& wals = new_versions->GetWalSet().GetWals(); + ASSERT_EQ(wals.size(), 1); + ASSERT_TRUE(wals.find(kNonClosedLogNumber) != wals.end()); + ASSERT_FALSE(wals.at(kNonClosedLogNumber).HasSyncedSize()); + } +} + +TEST_F(VersionSetTest, WalCreateTwice) { + NewDB(); + + constexpr WalNumber kLogNumber = 10; + + VersionEdit edit; + edit.AddWal(kLogNumber); + + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + + Status s = LogAndApplyToDefaultCF(edit); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(s.ToString().find("WAL 10 is created more than once") != + std::string::npos) + << s.ToString(); +} + +TEST_F(VersionSetTest, WalCreateAfterClose) { + NewDB(); + + constexpr WalNumber kLogNumber = 10; + constexpr uint64_t kSizeInBytes = 111; + + { + // Add a closed WAL. + VersionEdit edit; + edit.AddWal(kLogNumber); + WalMetadata wal(kSizeInBytes); + edit.AddWal(kLogNumber, wal); + + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + } + + { + // Create the same WAL again. + VersionEdit edit; + edit.AddWal(kLogNumber); + + Status s = LogAndApplyToDefaultCF(edit); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(s.ToString().find("WAL 10 is created more than once") != + std::string::npos) + << s.ToString(); + } +} + +TEST_F(VersionSetTest, AddWalWithSmallerSize) { + NewDB(); + assert(versions_); + + constexpr WalNumber kLogNumber = 10; + constexpr uint64_t kSizeInBytes = 111; + + { + // Add a closed WAL. + VersionEdit edit; + WalMetadata wal(kSizeInBytes); + edit.AddWal(kLogNumber, wal); + + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + } + // Copy for future comparison. + const std::map wals1 = + versions_->GetWalSet().GetWals(); + + { + // Add the same WAL with smaller synced size. + VersionEdit edit; + WalMetadata wal(kSizeInBytes / 2); + edit.AddWal(kLogNumber, wal); + + Status s = LogAndApplyToDefaultCF(edit); + ASSERT_OK(s); + } + const std::map wals2 = + versions_->GetWalSet().GetWals(); + ASSERT_EQ(wals1, wals2); +} + +TEST_F(VersionSetTest, DeleteWalsBeforeNonExistingWalNumber) { + NewDB(); + + constexpr WalNumber kLogNumber0 = 10; + constexpr WalNumber kLogNumber1 = 20; + constexpr WalNumber kNonExistingNumber = 15; + constexpr uint64_t kSizeInBytes = 111; + + { + // Add closed WALs. + VersionEdit edit; + WalMetadata wal(kSizeInBytes); + edit.AddWal(kLogNumber0, wal); + edit.AddWal(kLogNumber1, wal); + + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + } + + { + // Delete WALs before a non-existing WAL. + VersionEdit edit; + edit.DeleteWalsBefore(kNonExistingNumber); + + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + } + + // Recover a new VersionSet, WAL0 is deleted, WAL1 is not. + { + std::unique_ptr new_versions( + new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id*/ "", /*db_session_id*/ "")); + ASSERT_OK(new_versions->Recover(column_families_, false)); + const auto& wals = new_versions->GetWalSet().GetWals(); + ASSERT_EQ(wals.size(), 1); + ASSERT_TRUE(wals.find(kLogNumber1) != wals.end()); + } +} + +TEST_F(VersionSetTest, DeleteAllWals) { + NewDB(); + + constexpr WalNumber kMaxLogNumber = 10; + constexpr uint64_t kSizeInBytes = 111; + + { + // Add a closed WAL. + VersionEdit edit; + WalMetadata wal(kSizeInBytes); + edit.AddWal(kMaxLogNumber, wal); + + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + } + + { + VersionEdit edit; + edit.DeleteWalsBefore(kMaxLogNumber + 10); + + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + } + + // Recover a new VersionSet, all WALs are deleted. + { + std::unique_ptr new_versions( + new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id*/ "", /*db_session_id*/ "")); + ASSERT_OK(new_versions->Recover(column_families_, false)); + const auto& wals = new_versions->GetWalSet().GetWals(); + ASSERT_EQ(wals.size(), 0); + } +} + +TEST_F(VersionSetTest, AtomicGroupWithWalEdits) { + NewDB(); + + constexpr int kAtomicGroupSize = 7; + constexpr uint64_t kNumWals = 5; + const std::string kDBId = "db_db"; + + int remaining = kAtomicGroupSize; + autovector> edits; + // Add 5 WALs. + for (uint64_t i = 1; i <= kNumWals; i++) { + edits.emplace_back(new VersionEdit); + // WAL's size equals its log number. + edits.back()->AddWal(i, WalMetadata(i)); + edits.back()->MarkAtomicGroup(--remaining); + } + // One edit with the min log number set. + edits.emplace_back(new VersionEdit); + edits.back()->SetDBId(kDBId); + edits.back()->MarkAtomicGroup(--remaining); + // Delete the first added 4 WALs. + edits.emplace_back(new VersionEdit); + edits.back()->DeleteWalsBefore(kNumWals); + edits.back()->MarkAtomicGroup(--remaining); + ASSERT_EQ(remaining, 0); + + ASSERT_OK(LogAndApplyToDefaultCF(edits)); + + // Recover a new VersionSet, the min log number and the last WAL should be + // kept. + { + std::unique_ptr new_versions( + new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id*/ "", /*db_session_id*/ "")); + std::string db_id; + ASSERT_OK( + new_versions->Recover(column_families_, /*read_only=*/false, &db_id)); + + ASSERT_EQ(db_id, kDBId); + + const auto& wals = new_versions->GetWalSet().GetWals(); + ASSERT_EQ(wals.size(), 1); + ASSERT_TRUE(wals.find(kNumWals) != wals.end()); + ASSERT_TRUE(wals.at(kNumWals).HasSyncedSize()); + ASSERT_EQ(wals.at(kNumWals).GetSyncedSizeInBytes(), kNumWals); + } +} + +TEST_F(VersionStorageInfoTest, AddRangeDeletionCompensatedFileSize) { + // Tests that compensated range deletion size is added to compensated file + // size. + Add(4, 100U, "1", "2", 100U, kInvalidBlobFileNumber, 1000U); + + UpdateVersionStorageInfo(); + + auto meta = vstorage_.GetFileMetaDataByNumber(100U); + ASSERT_EQ(meta->compensated_file_size, 100U + 1000U); +} + +class VersionSetWithTimestampTest : public VersionSetTest { + public: + static const std::string kNewCfName; + + explicit VersionSetWithTimestampTest() : VersionSetTest() {} + + void SetUp() override { + NewDB(); + Options options; + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + cfd_ = CreateColumnFamily(kNewCfName, options); + EXPECT_NE(nullptr, cfd_); + EXPECT_NE(nullptr, cfd_->GetLatestMutableCFOptions()); + column_families_.emplace_back(kNewCfName, options); + } + + void TearDown() override { + for (auto* e : edits_) { + delete e; + } + edits_.clear(); + } + + void GenVersionEditsToSetFullHistoryTsLow( + const std::vector& ts_lbs) { + for (const auto ts_lb : ts_lbs) { + VersionEdit* edit = new VersionEdit; + edit->SetColumnFamily(cfd_->GetID()); + std::string ts_str = test::EncodeInt(ts_lb); + edit->SetFullHistoryTsLow(ts_str); + edits_.emplace_back(edit); + } + } + + void VerifyFullHistoryTsLow(uint64_t expected_ts_low) { + std::unique_ptr vset( + new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id*/ "", /*db_session_id*/ "")); + ASSERT_OK(vset->Recover(column_families_, /*read_only=*/false, + /*db_id=*/nullptr)); + for (auto* cfd : *(vset->GetColumnFamilySet())) { + ASSERT_NE(nullptr, cfd); + if (cfd->GetName() == kNewCfName) { + ASSERT_EQ(test::EncodeInt(expected_ts_low), cfd->GetFullHistoryTsLow()); + } else { + ASSERT_TRUE(cfd->GetFullHistoryTsLow().empty()); + } + } + } + + void DoTest(const std::vector& ts_lbs) { + if (ts_lbs.empty()) { + return; + } + + GenVersionEditsToSetFullHistoryTsLow(ts_lbs); + + Status s; + mutex_.Lock(); + s = versions_->LogAndApply(cfd_, *(cfd_->GetLatestMutableCFOptions()), + read_options_, edits_, &mutex_, nullptr); + mutex_.Unlock(); + ASSERT_OK(s); + VerifyFullHistoryTsLow(*std::max_element(ts_lbs.begin(), ts_lbs.end())); + } + + protected: + ColumnFamilyData* cfd_{nullptr}; + // edits_ must contain and own pointers to heap-alloc VersionEdit objects. + autovector edits_; + + private: + const ReadOptions read_options_; +}; + +const std::string VersionSetWithTimestampTest::kNewCfName("new_cf"); + +TEST_F(VersionSetWithTimestampTest, SetFullHistoryTsLbOnce) { + constexpr uint64_t kTsLow = 100; + DoTest({kTsLow}); +} + +// Simulate the application increasing full_history_ts_low. +TEST_F(VersionSetWithTimestampTest, IncreaseFullHistoryTsLb) { + const std::vector ts_lbs = {100, 101, 102, 103}; + DoTest(ts_lbs); +} + +// Simulate the application trying to decrease full_history_ts_low +// unsuccessfully. If the application calls public API sequentially to +// decrease the lower bound ts, RocksDB will return an InvalidArgument +// status before involving VersionSet. Only when multiple threads trying +// to decrease the lower bound concurrently will this case ever happen. Even +// so, the lower bound cannot be decreased. The application will be notified +// via return value of the API. +TEST_F(VersionSetWithTimestampTest, TryDecreaseFullHistoryTsLb) { + const std::vector ts_lbs = {103, 102, 101, 100}; + DoTest(ts_lbs); +} + +class VersionSetAtomicGroupTest : public VersionSetTestBase, + public testing::Test { + public: + VersionSetAtomicGroupTest() + : VersionSetTestBase("version_set_atomic_group_test") {} + + void SetUp() override { + PrepareManifest(&column_families_, &last_seqno_, &log_writer_); + SetupTestSyncPoints(); + } + + void SetupValidAtomicGroup(int atomic_group_size) { + edits_.resize(atomic_group_size); + int remaining = atomic_group_size; + for (size_t i = 0; i != edits_.size(); ++i) { + edits_[i].SetLogNumber(0); + edits_[i].SetNextFile(2); + edits_[i].MarkAtomicGroup(--remaining); + edits_[i].SetLastSequence(last_seqno_++); + } + ASSERT_OK(SetCurrentFile(fs_.get(), dbname_, 1, nullptr)); + } + + void SetupIncompleteTrailingAtomicGroup(int atomic_group_size) { + edits_.resize(atomic_group_size); + int remaining = atomic_group_size; + for (size_t i = 0; i != edits_.size(); ++i) { + edits_[i].SetLogNumber(0); + edits_[i].SetNextFile(2); + edits_[i].MarkAtomicGroup(--remaining); + edits_[i].SetLastSequence(last_seqno_++); + } + ASSERT_OK(SetCurrentFile(fs_.get(), dbname_, 1, nullptr)); + } + + void SetupCorruptedAtomicGroup(int atomic_group_size) { + edits_.resize(atomic_group_size); + int remaining = atomic_group_size; + for (size_t i = 0; i != edits_.size(); ++i) { + edits_[i].SetLogNumber(0); + edits_[i].SetNextFile(2); + if (i != ((size_t)atomic_group_size / 2)) { + edits_[i].MarkAtomicGroup(--remaining); + } + edits_[i].SetLastSequence(last_seqno_++); + } + ASSERT_OK(SetCurrentFile(fs_.get(), dbname_, 1, nullptr)); + } + + void SetupIncorrectAtomicGroup(int atomic_group_size) { + edits_.resize(atomic_group_size); + int remaining = atomic_group_size; + for (size_t i = 0; i != edits_.size(); ++i) { + edits_[i].SetLogNumber(0); + edits_[i].SetNextFile(2); + if (i != 1) { + edits_[i].MarkAtomicGroup(--remaining); + } else { + edits_[i].MarkAtomicGroup(remaining--); + } + edits_[i].SetLastSequence(last_seqno_++); + } + ASSERT_OK(SetCurrentFile(fs_.get(), dbname_, 1, nullptr)); + } + + void SetupTestSyncPoints() { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "AtomicGroupReadBuffer::AddEdit:FirstInAtomicGroup", [&](void* arg) { + VersionEdit* e = reinterpret_cast(arg); + EXPECT_EQ(edits_.front().DebugString(), + e->DebugString()); // compare based on value + first_in_atomic_group_ = true; + }); + SyncPoint::GetInstance()->SetCallBack( + "AtomicGroupReadBuffer::AddEdit:LastInAtomicGroup", [&](void* arg) { + VersionEdit* e = reinterpret_cast(arg); + EXPECT_EQ(edits_.back().DebugString(), + e->DebugString()); // compare based on value + EXPECT_TRUE(first_in_atomic_group_); + last_in_atomic_group_ = true; + }); + SyncPoint::GetInstance()->SetCallBack( + "VersionEditHandlerBase::Iterate:Finish", [&](void* arg) { + num_recovered_edits_ = *reinterpret_cast(arg); + }); + SyncPoint::GetInstance()->SetCallBack( + "AtomicGroupReadBuffer::AddEdit:AtomicGroup", + [&](void* /* arg */) { ++num_edits_in_atomic_group_; }); + SyncPoint::GetInstance()->SetCallBack( + "AtomicGroupReadBuffer::AddEdit:AtomicGroupMixedWithNormalEdits", + [&](void* arg) { + corrupted_edit_ = *reinterpret_cast(arg); + }); + SyncPoint::GetInstance()->SetCallBack( + "AtomicGroupReadBuffer::AddEdit:IncorrectAtomicGroupSize", + [&](void* arg) { + edit_with_incorrect_group_size_ = + *reinterpret_cast(arg); + }); + SyncPoint::GetInstance()->EnableProcessing(); + } + + void AddNewEditsToLog(int num_edits) { + for (int i = 0; i < num_edits; i++) { + std::string record; + edits_[i].EncodeTo(&record); + ASSERT_OK(log_writer_->AddRecord(record)); + } + } + + void TearDown() override { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + log_writer_.reset(); + } + + protected: + std::vector column_families_; + SequenceNumber last_seqno_; + std::vector edits_; + bool first_in_atomic_group_ = false; + bool last_in_atomic_group_ = false; + int num_edits_in_atomic_group_ = 0; + size_t num_recovered_edits_ = 0; + VersionEdit corrupted_edit_; + VersionEdit edit_with_incorrect_group_size_; + std::unique_ptr log_writer_; +}; + +TEST_F(VersionSetAtomicGroupTest, HandleValidAtomicGroupWithVersionSetRecover) { + const int kAtomicGroupSize = 3; + SetupValidAtomicGroup(kAtomicGroupSize); + AddNewEditsToLog(kAtomicGroupSize); + EXPECT_OK(versions_->Recover(column_families_, false)); + EXPECT_EQ(column_families_.size(), + versions_->GetColumnFamilySet()->NumberOfColumnFamilies()); + EXPECT_TRUE(first_in_atomic_group_); + EXPECT_TRUE(last_in_atomic_group_); + EXPECT_EQ(num_initial_edits_ + kAtomicGroupSize, num_recovered_edits_); +} + +TEST_F(VersionSetAtomicGroupTest, + HandleValidAtomicGroupWithReactiveVersionSetRecover) { + const int kAtomicGroupSize = 3; + SetupValidAtomicGroup(kAtomicGroupSize); + AddNewEditsToLog(kAtomicGroupSize); + std::unique_ptr manifest_reader; + std::unique_ptr manifest_reporter; + std::unique_ptr manifest_reader_status; + EXPECT_OK(reactive_versions_->Recover(column_families_, &manifest_reader, + &manifest_reporter, + &manifest_reader_status)); + EXPECT_EQ(column_families_.size(), + reactive_versions_->GetColumnFamilySet()->NumberOfColumnFamilies()); + EXPECT_TRUE(first_in_atomic_group_); + EXPECT_TRUE(last_in_atomic_group_); + // The recover should clean up the replay buffer. + EXPECT_TRUE(reactive_versions_->TEST_read_edits_in_atomic_group() == 0); + EXPECT_TRUE(reactive_versions_->replay_buffer().size() == 0); + EXPECT_EQ(num_initial_edits_ + kAtomicGroupSize, num_recovered_edits_); +} + +TEST_F(VersionSetAtomicGroupTest, + HandleValidAtomicGroupWithReactiveVersionSetReadAndApply) { + const int kAtomicGroupSize = 3; + SetupValidAtomicGroup(kAtomicGroupSize); + std::unique_ptr manifest_reader; + std::unique_ptr manifest_reporter; + std::unique_ptr manifest_reader_status; + EXPECT_OK(reactive_versions_->Recover(column_families_, &manifest_reader, + &manifest_reporter, + &manifest_reader_status)); + EXPECT_EQ(num_initial_edits_, num_recovered_edits_); + AddNewEditsToLog(kAtomicGroupSize); + InstrumentedMutex mu; + std::unordered_set cfds_changed; + mu.Lock(); + EXPECT_OK(reactive_versions_->ReadAndApply( + &mu, &manifest_reader, manifest_reader_status.get(), &cfds_changed)); + mu.Unlock(); + EXPECT_TRUE(first_in_atomic_group_); + EXPECT_TRUE(last_in_atomic_group_); + // The recover should clean up the replay buffer. + EXPECT_TRUE(reactive_versions_->TEST_read_edits_in_atomic_group() == 0); + EXPECT_TRUE(reactive_versions_->replay_buffer().size() == 0); + EXPECT_EQ(kAtomicGroupSize, num_recovered_edits_); +} + +TEST_F(VersionSetAtomicGroupTest, + HandleIncompleteTrailingAtomicGroupWithVersionSetRecover) { + const int kAtomicGroupSize = 4; + const int kNumberOfPersistedVersionEdits = kAtomicGroupSize - 1; + SetupIncompleteTrailingAtomicGroup(kAtomicGroupSize); + AddNewEditsToLog(kNumberOfPersistedVersionEdits); + EXPECT_OK(versions_->Recover(column_families_, false)); + EXPECT_EQ(column_families_.size(), + versions_->GetColumnFamilySet()->NumberOfColumnFamilies()); + EXPECT_TRUE(first_in_atomic_group_); + EXPECT_FALSE(last_in_atomic_group_); + EXPECT_EQ(kNumberOfPersistedVersionEdits, num_edits_in_atomic_group_); + EXPECT_EQ(num_initial_edits_, num_recovered_edits_); +} + +TEST_F(VersionSetAtomicGroupTest, + HandleIncompleteTrailingAtomicGroupWithReactiveVersionSetRecover) { + const int kAtomicGroupSize = 4; + const int kNumberOfPersistedVersionEdits = kAtomicGroupSize - 1; + SetupIncompleteTrailingAtomicGroup(kAtomicGroupSize); + AddNewEditsToLog(kNumberOfPersistedVersionEdits); + std::unique_ptr manifest_reader; + std::unique_ptr manifest_reporter; + std::unique_ptr manifest_reader_status; + EXPECT_OK(reactive_versions_->Recover(column_families_, &manifest_reader, + &manifest_reporter, + &manifest_reader_status)); + EXPECT_EQ(column_families_.size(), + reactive_versions_->GetColumnFamilySet()->NumberOfColumnFamilies()); + EXPECT_TRUE(first_in_atomic_group_); + EXPECT_FALSE(last_in_atomic_group_); + EXPECT_EQ(kNumberOfPersistedVersionEdits, num_edits_in_atomic_group_); + // Reactive version set should store the edits in the replay buffer. + EXPECT_TRUE(reactive_versions_->TEST_read_edits_in_atomic_group() == + kNumberOfPersistedVersionEdits); + EXPECT_TRUE(reactive_versions_->replay_buffer().size() == kAtomicGroupSize); + // Write the last record. The reactive version set should now apply all + // edits. + std::string last_record; + edits_[kAtomicGroupSize - 1].EncodeTo(&last_record); + EXPECT_OK(log_writer_->AddRecord(last_record)); + InstrumentedMutex mu; + std::unordered_set cfds_changed; + mu.Lock(); + EXPECT_OK(reactive_versions_->ReadAndApply( + &mu, &manifest_reader, manifest_reader_status.get(), &cfds_changed)); + mu.Unlock(); + // Reactive version set should be empty now. + EXPECT_TRUE(reactive_versions_->TEST_read_edits_in_atomic_group() == 0); + EXPECT_TRUE(reactive_versions_->replay_buffer().size() == 0); + EXPECT_EQ(num_initial_edits_, num_recovered_edits_); +} + +TEST_F(VersionSetAtomicGroupTest, + HandleIncompleteTrailingAtomicGroupWithReactiveVersionSetReadAndApply) { + const int kAtomicGroupSize = 4; + const int kNumberOfPersistedVersionEdits = kAtomicGroupSize - 1; + SetupIncompleteTrailingAtomicGroup(kAtomicGroupSize); + std::unique_ptr manifest_reader; + std::unique_ptr manifest_reporter; + std::unique_ptr manifest_reader_status; + // No edits in an atomic group. + EXPECT_OK(reactive_versions_->Recover(column_families_, &manifest_reader, + &manifest_reporter, + &manifest_reader_status)); + EXPECT_EQ(column_families_.size(), + reactive_versions_->GetColumnFamilySet()->NumberOfColumnFamilies()); + EXPECT_EQ(num_initial_edits_, num_recovered_edits_); + // Write a few edits in an atomic group. + AddNewEditsToLog(kNumberOfPersistedVersionEdits); + InstrumentedMutex mu; + std::unordered_set cfds_changed; + mu.Lock(); + EXPECT_OK(reactive_versions_->ReadAndApply( + &mu, &manifest_reader, manifest_reader_status.get(), &cfds_changed)); + mu.Unlock(); + EXPECT_TRUE(first_in_atomic_group_); + EXPECT_FALSE(last_in_atomic_group_); + EXPECT_EQ(kNumberOfPersistedVersionEdits, num_edits_in_atomic_group_); + // Reactive version set should store the edits in the replay buffer. + EXPECT_TRUE(reactive_versions_->TEST_read_edits_in_atomic_group() == + kNumberOfPersistedVersionEdits); + EXPECT_TRUE(reactive_versions_->replay_buffer().size() == kAtomicGroupSize); +} + +TEST_F(VersionSetAtomicGroupTest, + HandleCorruptedAtomicGroupWithVersionSetRecover) { + const int kAtomicGroupSize = 4; + SetupCorruptedAtomicGroup(kAtomicGroupSize); + AddNewEditsToLog(kAtomicGroupSize); + EXPECT_NOK(versions_->Recover(column_families_, false)); + EXPECT_EQ(column_families_.size(), + versions_->GetColumnFamilySet()->NumberOfColumnFamilies()); + EXPECT_EQ(edits_[kAtomicGroupSize / 2].DebugString(), + corrupted_edit_.DebugString()); +} + +TEST_F(VersionSetAtomicGroupTest, + HandleCorruptedAtomicGroupWithReactiveVersionSetRecover) { + const int kAtomicGroupSize = 4; + SetupCorruptedAtomicGroup(kAtomicGroupSize); + AddNewEditsToLog(kAtomicGroupSize); + std::unique_ptr manifest_reader; + std::unique_ptr manifest_reporter; + std::unique_ptr manifest_reader_status; + EXPECT_NOK(reactive_versions_->Recover(column_families_, &manifest_reader, + &manifest_reporter, + &manifest_reader_status)); + EXPECT_EQ(column_families_.size(), + reactive_versions_->GetColumnFamilySet()->NumberOfColumnFamilies()); + EXPECT_EQ(edits_[kAtomicGroupSize / 2].DebugString(), + corrupted_edit_.DebugString()); +} + +TEST_F(VersionSetAtomicGroupTest, + HandleCorruptedAtomicGroupWithReactiveVersionSetReadAndApply) { + const int kAtomicGroupSize = 4; + SetupCorruptedAtomicGroup(kAtomicGroupSize); + InstrumentedMutex mu; + std::unordered_set cfds_changed; + std::unique_ptr manifest_reader; + std::unique_ptr manifest_reporter; + std::unique_ptr manifest_reader_status; + EXPECT_OK(reactive_versions_->Recover(column_families_, &manifest_reader, + &manifest_reporter, + &manifest_reader_status)); + // Write the corrupted edits. + AddNewEditsToLog(kAtomicGroupSize); + mu.Lock(); + EXPECT_NOK(reactive_versions_->ReadAndApply( + &mu, &manifest_reader, manifest_reader_status.get(), &cfds_changed)); + mu.Unlock(); + EXPECT_EQ(edits_[kAtomicGroupSize / 2].DebugString(), + corrupted_edit_.DebugString()); +} + +TEST_F(VersionSetAtomicGroupTest, + HandleIncorrectAtomicGroupSizeWithVersionSetRecover) { + const int kAtomicGroupSize = 4; + SetupIncorrectAtomicGroup(kAtomicGroupSize); + AddNewEditsToLog(kAtomicGroupSize); + EXPECT_NOK(versions_->Recover(column_families_, false)); + EXPECT_EQ(column_families_.size(), + versions_->GetColumnFamilySet()->NumberOfColumnFamilies()); + EXPECT_EQ(edits_[1].DebugString(), + edit_with_incorrect_group_size_.DebugString()); +} + +TEST_F(VersionSetAtomicGroupTest, + HandleIncorrectAtomicGroupSizeWithReactiveVersionSetRecover) { + const int kAtomicGroupSize = 4; + SetupIncorrectAtomicGroup(kAtomicGroupSize); + AddNewEditsToLog(kAtomicGroupSize); + std::unique_ptr manifest_reader; + std::unique_ptr manifest_reporter; + std::unique_ptr manifest_reader_status; + EXPECT_NOK(reactive_versions_->Recover(column_families_, &manifest_reader, + &manifest_reporter, + &manifest_reader_status)); + EXPECT_EQ(column_families_.size(), + reactive_versions_->GetColumnFamilySet()->NumberOfColumnFamilies()); + EXPECT_EQ(edits_[1].DebugString(), + edit_with_incorrect_group_size_.DebugString()); +} + +TEST_F(VersionSetAtomicGroupTest, + HandleIncorrectAtomicGroupSizeWithReactiveVersionSetReadAndApply) { + const int kAtomicGroupSize = 4; + SetupIncorrectAtomicGroup(kAtomicGroupSize); + InstrumentedMutex mu; + std::unordered_set cfds_changed; + std::unique_ptr manifest_reader; + std::unique_ptr manifest_reporter; + std::unique_ptr manifest_reader_status; + EXPECT_OK(reactive_versions_->Recover(column_families_, &manifest_reader, + &manifest_reporter, + &manifest_reader_status)); + AddNewEditsToLog(kAtomicGroupSize); + mu.Lock(); + EXPECT_NOK(reactive_versions_->ReadAndApply( + &mu, &manifest_reader, manifest_reader_status.get(), &cfds_changed)); + mu.Unlock(); + EXPECT_EQ(edits_[1].DebugString(), + edit_with_incorrect_group_size_.DebugString()); +} + +class VersionSetTestDropOneCF : public VersionSetTestBase, + public testing::TestWithParam { + public: + VersionSetTestDropOneCF() + : VersionSetTestBase("version_set_test_drop_one_cf") {} +}; + +// This test simulates the following execution sequence +// Time thread1 bg_flush_thr +// | Prepare version edits (e1,e2,e3) for atomic +// | flush cf1, cf2, cf3 +// | Enqueue e to drop cfi +// | to manifest_writers_ +// | Enqueue (e1,e2,e3) to manifest_writers_ +// | +// | Apply e, +// | cfi.IsDropped() is true +// | Apply (e1,e2,e3), +// | since cfi.IsDropped() == true, we need to +// | drop ei and write the rest to MANIFEST. +// V +// +// Repeat the test for i = 1, 2, 3 to simulate dropping the first, middle and +// last column family in an atomic group. +TEST_P(VersionSetTestDropOneCF, HandleDroppedColumnFamilyInAtomicGroup) { + const ReadOptions read_options; + + std::vector column_families; + SequenceNumber last_seqno; + std::unique_ptr log_writer; + PrepareManifest(&column_families, &last_seqno, &log_writer); + Status s = SetCurrentFile(fs_.get(), dbname_, 1, nullptr); + ASSERT_OK(s); + + EXPECT_OK(versions_->Recover(column_families, false /* read_only */)); + EXPECT_EQ(column_families.size(), + versions_->GetColumnFamilySet()->NumberOfColumnFamilies()); + + const int kAtomicGroupSize = 3; + const std::vector non_default_cf_names = { + kColumnFamilyName1, kColumnFamilyName2, kColumnFamilyName3}; + + // Drop one column family + VersionEdit drop_cf_edit; + drop_cf_edit.DropColumnFamily(); + const std::string cf_to_drop_name(GetParam()); + auto cfd_to_drop = + versions_->GetColumnFamilySet()->GetColumnFamily(cf_to_drop_name); + ASSERT_NE(nullptr, cfd_to_drop); + // Increase its refcount because cfd_to_drop is used later, and we need to + // prevent it from being deleted. + cfd_to_drop->Ref(); + drop_cf_edit.SetColumnFamily(cfd_to_drop->GetID()); + mutex_.Lock(); + s = versions_->LogAndApply(cfd_to_drop, + *cfd_to_drop->GetLatestMutableCFOptions(), + read_options, &drop_cf_edit, &mutex_, nullptr); + mutex_.Unlock(); + ASSERT_OK(s); + + std::vector edits(kAtomicGroupSize); + uint32_t remaining = kAtomicGroupSize; + size_t i = 0; + autovector cfds; + autovector mutable_cf_options_list; + autovector> edit_lists; + for (const auto& cf_name : non_default_cf_names) { + auto cfd = (cf_name != cf_to_drop_name) + ? versions_->GetColumnFamilySet()->GetColumnFamily(cf_name) + : cfd_to_drop; + ASSERT_NE(nullptr, cfd); + cfds.push_back(cfd); + mutable_cf_options_list.emplace_back(cfd->GetLatestMutableCFOptions()); + edits[i].SetColumnFamily(cfd->GetID()); + edits[i].SetLogNumber(0); + edits[i].SetNextFile(2); + edits[i].MarkAtomicGroup(--remaining); + edits[i].SetLastSequence(last_seqno++); + autovector tmp_edits; + tmp_edits.push_back(&edits[i]); + edit_lists.emplace_back(tmp_edits); + ++i; + } + int called = 0; + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::ProcessManifestWrites:CheckOneAtomicGroup", [&](void* arg) { + std::vector* tmp_edits = + reinterpret_cast*>(arg); + EXPECT_EQ(kAtomicGroupSize - 1, tmp_edits->size()); + for (const auto e : *tmp_edits) { + bool found = false; + for (const auto& e2 : edits) { + if (&e2 == e) { + found = true; + break; + } + } + ASSERT_TRUE(found); + } + ++called; + }); + SyncPoint::GetInstance()->EnableProcessing(); + mutex_.Lock(); + s = versions_->LogAndApply(cfds, mutable_cf_options_list, read_options, + edit_lists, &mutex_, nullptr); + mutex_.Unlock(); + ASSERT_OK(s); + ASSERT_EQ(1, called); + cfd_to_drop->UnrefAndTryDelete(); +} + +INSTANTIATE_TEST_CASE_P( + AtomicGroup, VersionSetTestDropOneCF, + testing::Values(VersionSetTestBase::kColumnFamilyName1, + VersionSetTestBase::kColumnFamilyName2, + VersionSetTestBase::kColumnFamilyName3)); + +class EmptyDefaultCfNewManifest : public VersionSetTestBase, + public testing::Test { + public: + EmptyDefaultCfNewManifest() : VersionSetTestBase("version_set_new_db_test") {} + // Emulate DBImpl::NewDB() + void PrepareManifest(std::vector* /*column_families*/, + SequenceNumber* /*last_seqno*/, + std::unique_ptr* log_writer) override { + assert(log_writer != nullptr); + VersionEdit new_db; + new_db.SetLogNumber(0); + const std::string manifest_path = DescriptorFileName(dbname_, 1); + const auto& fs = env_->GetFileSystem(); + std::unique_ptr file_writer; + Status s = WritableFileWriter::Create( + fs, manifest_path, fs->OptimizeForManifestWrite(env_options_), + &file_writer, nullptr); + ASSERT_OK(s); + log_writer->reset(new log::Writer(std::move(file_writer), 0, true)); + std::string record; + ASSERT_TRUE(new_db.EncodeTo(&record)); + s = (*log_writer)->AddRecord(record); + ASSERT_OK(s); + // Create new column family + VersionEdit new_cf; + new_cf.AddColumnFamily(VersionSetTestBase::kColumnFamilyName1); + new_cf.SetColumnFamily(1); + new_cf.SetLastSequence(2); + new_cf.SetNextFile(2); + record.clear(); + ASSERT_TRUE(new_cf.EncodeTo(&record)); + s = (*log_writer)->AddRecord(record); + ASSERT_OK(s); + } + + protected: + bool write_dbid_to_manifest_ = false; + std::unique_ptr log_writer_; +}; + +// Create db, create column family. Cf creation will switch to a new MANIFEST. +// Then reopen db, trying to recover. +TEST_F(EmptyDefaultCfNewManifest, Recover) { + PrepareManifest(nullptr, nullptr, &log_writer_); + log_writer_.reset(); + Status s = + SetCurrentFile(fs_.get(), dbname_, 1, /*directory_to_fsync=*/nullptr); + ASSERT_OK(s); + std::string manifest_path; + VerifyManifest(&manifest_path); + std::vector column_families; + column_families.emplace_back(kDefaultColumnFamilyName, cf_options_); + column_families.emplace_back(VersionSetTestBase::kColumnFamilyName1, + cf_options_); + std::string db_id; + bool has_missing_table_file = false; + s = versions_->TryRecoverFromOneManifest( + manifest_path, column_families, false, &db_id, &has_missing_table_file); + ASSERT_OK(s); + ASSERT_FALSE(has_missing_table_file); +} + +class VersionSetTestEmptyDb + : public VersionSetTestBase, + public testing::TestWithParam< + std::tuple>> { + public: + static const std::string kUnknownColumnFamilyName; + VersionSetTestEmptyDb() : VersionSetTestBase("version_set_test_empty_db") {} + + protected: + void PrepareManifest(std::vector* /*column_families*/, + SequenceNumber* /*last_seqno*/, + std::unique_ptr* log_writer) override { + assert(nullptr != log_writer); + VersionEdit new_db; + if (db_options_.write_dbid_to_manifest) { + DBOptions tmp_db_options; + tmp_db_options.env = env_; + std::unique_ptr impl(new DBImpl(tmp_db_options, dbname_)); + std::string db_id; + impl->GetDbIdentityFromIdentityFile(&db_id); + new_db.SetDBId(db_id); + } + const std::string manifest_path = DescriptorFileName(dbname_, 1); + const auto& fs = env_->GetFileSystem(); + std::unique_ptr file_writer; + Status s = WritableFileWriter::Create( + fs, manifest_path, fs->OptimizeForManifestWrite(env_options_), + &file_writer, nullptr); + ASSERT_OK(s); + { + log_writer->reset(new log::Writer(std::move(file_writer), 0, false)); + std::string record; + new_db.EncodeTo(&record); + s = (*log_writer)->AddRecord(record); + ASSERT_OK(s); + } + } + + std::unique_ptr log_writer_; +}; + +const std::string VersionSetTestEmptyDb::kUnknownColumnFamilyName = "unknown"; + +TEST_P(VersionSetTestEmptyDb, OpenFromIncompleteManifest0) { + db_options_.write_dbid_to_manifest = std::get<0>(GetParam()); + PrepareManifest(nullptr, nullptr, &log_writer_); + log_writer_.reset(); + Status s = + SetCurrentFile(fs_.get(), dbname_, 1, /*directory_to_fsync=*/nullptr); + ASSERT_OK(s); + + std::string manifest_path; + VerifyManifest(&manifest_path); + + bool read_only = std::get<1>(GetParam()); + const std::vector cf_names = std::get<2>(GetParam()); + + std::vector column_families; + for (const auto& cf_name : cf_names) { + column_families.emplace_back(cf_name, cf_options_); + } + + std::string db_id; + bool has_missing_table_file = false; + s = versions_->TryRecoverFromOneManifest(manifest_path, column_families, + read_only, &db_id, + &has_missing_table_file); + auto iter = + std::find(cf_names.begin(), cf_names.end(), kDefaultColumnFamilyName); + if (iter == cf_names.end()) { + ASSERT_TRUE(s.IsInvalidArgument()); + } else { + ASSERT_NE(s.ToString().find(manifest_path), std::string::npos); + ASSERT_TRUE(s.IsCorruption()); + } +} + +TEST_P(VersionSetTestEmptyDb, OpenFromIncompleteManifest1) { + db_options_.write_dbid_to_manifest = std::get<0>(GetParam()); + PrepareManifest(nullptr, nullptr, &log_writer_); + // Only a subset of column families in the MANIFEST. + VersionEdit new_cf1; + new_cf1.AddColumnFamily(VersionSetTestBase::kColumnFamilyName1); + new_cf1.SetColumnFamily(1); + Status s; + { + std::string record; + new_cf1.EncodeTo(&record); + s = log_writer_->AddRecord(record); + ASSERT_OK(s); + } + log_writer_.reset(); + s = SetCurrentFile(fs_.get(), dbname_, 1, /*directory_to_fsync=*/nullptr); + ASSERT_OK(s); + + std::string manifest_path; + VerifyManifest(&manifest_path); + + bool read_only = std::get<1>(GetParam()); + const std::vector& cf_names = std::get<2>(GetParam()); + std::vector column_families; + for (const auto& cf_name : cf_names) { + column_families.emplace_back(cf_name, cf_options_); + } + std::string db_id; + bool has_missing_table_file = false; + s = versions_->TryRecoverFromOneManifest(manifest_path, column_families, + read_only, &db_id, + &has_missing_table_file); + auto iter = + std::find(cf_names.begin(), cf_names.end(), kDefaultColumnFamilyName); + if (iter == cf_names.end()) { + ASSERT_TRUE(s.IsInvalidArgument()); + } else { + ASSERT_NE(s.ToString().find(manifest_path), std::string::npos); + ASSERT_TRUE(s.IsCorruption()); + } +} + +TEST_P(VersionSetTestEmptyDb, OpenFromInCompleteManifest2) { + db_options_.write_dbid_to_manifest = std::get<0>(GetParam()); + PrepareManifest(nullptr, nullptr, &log_writer_); + // Write all column families but no log_number, next_file_number and + // last_sequence. + const std::vector all_cf_names = { + kDefaultColumnFamilyName, kColumnFamilyName1, kColumnFamilyName2, + kColumnFamilyName3}; + uint32_t cf_id = 1; + Status s; + for (size_t i = 1; i != all_cf_names.size(); ++i) { + VersionEdit new_cf; + new_cf.AddColumnFamily(all_cf_names[i]); + new_cf.SetColumnFamily(cf_id++); + std::string record; + ASSERT_TRUE(new_cf.EncodeTo(&record)); + s = log_writer_->AddRecord(record); + ASSERT_OK(s); + } + log_writer_.reset(); + s = SetCurrentFile(fs_.get(), dbname_, 1, /*directory_to_fsync=*/nullptr); + ASSERT_OK(s); + + std::string manifest_path; + VerifyManifest(&manifest_path); + + bool read_only = std::get<1>(GetParam()); + const std::vector& cf_names = std::get<2>(GetParam()); + std::vector column_families; + for (const auto& cf_name : cf_names) { + column_families.emplace_back(cf_name, cf_options_); + } + std::string db_id; + bool has_missing_table_file = false; + s = versions_->TryRecoverFromOneManifest(manifest_path, column_families, + read_only, &db_id, + &has_missing_table_file); + auto iter = + std::find(cf_names.begin(), cf_names.end(), kDefaultColumnFamilyName); + if (iter == cf_names.end()) { + ASSERT_TRUE(s.IsInvalidArgument()); + } else { + ASSERT_NE(s.ToString().find(manifest_path), std::string::npos); + ASSERT_TRUE(s.IsCorruption()); + } +} + +TEST_P(VersionSetTestEmptyDb, OpenManifestWithUnknownCF) { + db_options_.write_dbid_to_manifest = std::get<0>(GetParam()); + PrepareManifest(nullptr, nullptr, &log_writer_); + // Write all column families but no log_number, next_file_number and + // last_sequence. + const std::vector all_cf_names = { + kDefaultColumnFamilyName, kColumnFamilyName1, kColumnFamilyName2, + kColumnFamilyName3}; + uint32_t cf_id = 1; + Status s; + for (size_t i = 1; i != all_cf_names.size(); ++i) { + VersionEdit new_cf; + new_cf.AddColumnFamily(all_cf_names[i]); + new_cf.SetColumnFamily(cf_id++); + std::string record; + ASSERT_TRUE(new_cf.EncodeTo(&record)); + s = log_writer_->AddRecord(record); + ASSERT_OK(s); + } + { + VersionEdit tmp_edit; + tmp_edit.SetColumnFamily(4); + tmp_edit.SetLogNumber(0); + tmp_edit.SetNextFile(2); + tmp_edit.SetLastSequence(0); + std::string record; + ASSERT_TRUE(tmp_edit.EncodeTo(&record)); + s = log_writer_->AddRecord(record); + ASSERT_OK(s); + } + log_writer_.reset(); + s = SetCurrentFile(fs_.get(), dbname_, 1, /*directory_to_fsync=*/nullptr); + ASSERT_OK(s); + + std::string manifest_path; + VerifyManifest(&manifest_path); + + bool read_only = std::get<1>(GetParam()); + const std::vector& cf_names = std::get<2>(GetParam()); + std::vector column_families; + for (const auto& cf_name : cf_names) { + column_families.emplace_back(cf_name, cf_options_); + } + std::string db_id; + bool has_missing_table_file = false; + s = versions_->TryRecoverFromOneManifest(manifest_path, column_families, + read_only, &db_id, + &has_missing_table_file); + auto iter = + std::find(cf_names.begin(), cf_names.end(), kDefaultColumnFamilyName); + if (iter == cf_names.end()) { + ASSERT_TRUE(s.IsInvalidArgument()); + } else { + ASSERT_NE(s.ToString().find(manifest_path), std::string::npos); + ASSERT_TRUE(s.IsCorruption()); + } +} + +TEST_P(VersionSetTestEmptyDb, OpenCompleteManifest) { + db_options_.write_dbid_to_manifest = std::get<0>(GetParam()); + PrepareManifest(nullptr, nullptr, &log_writer_); + // Write all column families but no log_number, next_file_number and + // last_sequence. + const std::vector all_cf_names = { + kDefaultColumnFamilyName, kColumnFamilyName1, kColumnFamilyName2, + kColumnFamilyName3}; + uint32_t cf_id = 1; + Status s; + for (size_t i = 1; i != all_cf_names.size(); ++i) { + VersionEdit new_cf; + new_cf.AddColumnFamily(all_cf_names[i]); + new_cf.SetColumnFamily(cf_id++); + std::string record; + ASSERT_TRUE(new_cf.EncodeTo(&record)); + s = log_writer_->AddRecord(record); + ASSERT_OK(s); + } + { + VersionEdit tmp_edit; + tmp_edit.SetLogNumber(0); + tmp_edit.SetNextFile(2); + tmp_edit.SetLastSequence(0); + std::string record; + ASSERT_TRUE(tmp_edit.EncodeTo(&record)); + s = log_writer_->AddRecord(record); + ASSERT_OK(s); + } + log_writer_.reset(); + s = SetCurrentFile(fs_.get(), dbname_, 1, /*directory_to_fsync=*/nullptr); + ASSERT_OK(s); + + std::string manifest_path; + VerifyManifest(&manifest_path); + + bool read_only = std::get<1>(GetParam()); + const std::vector& cf_names = std::get<2>(GetParam()); + std::vector column_families; + for (const auto& cf_name : cf_names) { + column_families.emplace_back(cf_name, cf_options_); + } + std::string db_id; + bool has_missing_table_file = false; + s = versions_->TryRecoverFromOneManifest(manifest_path, column_families, + read_only, &db_id, + &has_missing_table_file); + auto iter = + std::find(cf_names.begin(), cf_names.end(), kDefaultColumnFamilyName); + if (iter == cf_names.end()) { + ASSERT_TRUE(s.IsInvalidArgument()); + } else if (read_only) { + ASSERT_OK(s); + ASSERT_FALSE(has_missing_table_file); + } else if (cf_names.size() == all_cf_names.size()) { + ASSERT_OK(s); + ASSERT_FALSE(has_missing_table_file); + } else if (cf_names.size() < all_cf_names.size()) { + ASSERT_TRUE(s.IsInvalidArgument()); + } else { + ASSERT_OK(s); + ASSERT_FALSE(has_missing_table_file); + ColumnFamilyData* cfd = versions_->GetColumnFamilySet()->GetColumnFamily( + kUnknownColumnFamilyName); + ASSERT_EQ(nullptr, cfd); + } +} + +INSTANTIATE_TEST_CASE_P( + BestEffortRecovery, VersionSetTestEmptyDb, + testing::Combine( + /*write_dbid_to_manifest=*/testing::Bool(), + /*read_only=*/testing::Bool(), + /*cf_names=*/ + testing::Values( + std::vector(), + std::vector({kDefaultColumnFamilyName}), + std::vector({VersionSetTestBase::kColumnFamilyName1, + VersionSetTestBase::kColumnFamilyName2, + VersionSetTestBase::kColumnFamilyName3}), + std::vector({kDefaultColumnFamilyName, + VersionSetTestBase::kColumnFamilyName1}), + std::vector({kDefaultColumnFamilyName, + VersionSetTestBase::kColumnFamilyName1, + VersionSetTestBase::kColumnFamilyName2, + VersionSetTestBase::kColumnFamilyName3}), + std::vector( + {kDefaultColumnFamilyName, + VersionSetTestBase::kColumnFamilyName1, + VersionSetTestBase::kColumnFamilyName2, + VersionSetTestBase::kColumnFamilyName3, + VersionSetTestEmptyDb::kUnknownColumnFamilyName})))); + +class VersionSetTestMissingFiles : public VersionSetTestBase, + public testing::Test { + public: + VersionSetTestMissingFiles() + : VersionSetTestBase("version_set_test_missing_files"), + block_based_table_options_(), + table_factory_(std::make_shared( + block_based_table_options_)), + internal_comparator_( + std::make_shared(options_.comparator)) {} + + protected: + void PrepareManifest(std::vector* column_families, + SequenceNumber* last_seqno, + std::unique_ptr* log_writer) override { + assert(column_families != nullptr); + assert(last_seqno != nullptr); + assert(log_writer != nullptr); + const std::string manifest = DescriptorFileName(dbname_, 1); + const auto& fs = env_->GetFileSystem(); + std::unique_ptr file_writer; + Status s = WritableFileWriter::Create( + fs, manifest, fs->OptimizeForManifestWrite(env_options_), &file_writer, + nullptr); + ASSERT_OK(s); + log_writer->reset(new log::Writer(std::move(file_writer), 0, false)); + VersionEdit new_db; + if (db_options_.write_dbid_to_manifest) { + DBOptions tmp_db_options; + tmp_db_options.env = env_; + std::unique_ptr impl(new DBImpl(tmp_db_options, dbname_)); + std::string db_id; + impl->GetDbIdentityFromIdentityFile(&db_id); + new_db.SetDBId(db_id); + } + { + std::string record; + ASSERT_TRUE(new_db.EncodeTo(&record)); + s = (*log_writer)->AddRecord(record); + ASSERT_OK(s); + } + const std::vector cf_names = { + kDefaultColumnFamilyName, kColumnFamilyName1, kColumnFamilyName2, + kColumnFamilyName3}; + uint32_t cf_id = 1; // default cf id is 0 + cf_options_.table_factory = table_factory_; + for (const auto& cf_name : cf_names) { + column_families->emplace_back(cf_name, cf_options_); + if (cf_name == kDefaultColumnFamilyName) { + continue; + } + VersionEdit new_cf; + new_cf.AddColumnFamily(cf_name); + new_cf.SetColumnFamily(cf_id); + std::string record; + ASSERT_TRUE(new_cf.EncodeTo(&record)); + s = (*log_writer)->AddRecord(record); + ASSERT_OK(s); + + VersionEdit cf_files; + cf_files.SetColumnFamily(cf_id); + cf_files.SetLogNumber(0); + record.clear(); + ASSERT_TRUE(cf_files.EncodeTo(&record)); + s = (*log_writer)->AddRecord(record); + ASSERT_OK(s); + ++cf_id; + } + SequenceNumber seq = 2; + { + VersionEdit edit; + edit.SetNextFile(7); + edit.SetLastSequence(seq); + std::string record; + ASSERT_TRUE(edit.EncodeTo(&record)); + s = (*log_writer)->AddRecord(record); + ASSERT_OK(s); + } + *last_seqno = seq + 1; + } + + struct SstInfo { + uint64_t file_number; + std::string column_family; + std::string key; // the only key + int level = 0; + uint64_t epoch_number; + SstInfo(uint64_t file_num, const std::string& cf_name, + const std::string& _key, + uint64_t _epoch_number = kUnknownEpochNumber) + : SstInfo(file_num, cf_name, _key, 0, _epoch_number) {} + SstInfo(uint64_t file_num, const std::string& cf_name, + const std::string& _key, int lvl, + uint64_t _epoch_number = kUnknownEpochNumber) + : file_number(file_num), + column_family(cf_name), + key(_key), + level(lvl), + epoch_number(_epoch_number) {} + }; + + // Create dummy sst, return their metadata. Note that only file name and size + // are used. + void CreateDummyTableFiles(const std::vector& file_infos, + std::vector* file_metas) { + assert(file_metas != nullptr); + for (const auto& info : file_infos) { + uint64_t file_num = info.file_number; + std::string fname = MakeTableFileName(dbname_, file_num); + std::unique_ptr file; + Status s = fs_->NewWritableFile(fname, FileOptions(), &file, nullptr); + ASSERT_OK(s); + std::unique_ptr fwriter(new WritableFileWriter( + std::move(file), fname, FileOptions(), env_->GetSystemClock().get())); + IntTblPropCollectorFactories int_tbl_prop_collector_factories; + + std::unique_ptr builder(table_factory_->NewTableBuilder( + TableBuilderOptions( + immutable_options_, mutable_cf_options_, *internal_comparator_, + &int_tbl_prop_collector_factories, kNoCompression, + CompressionOptions(), + TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, + info.column_family, info.level), + fwriter.get())); + InternalKey ikey(info.key, 0, ValueType::kTypeValue); + builder->Add(ikey.Encode(), "value"); + ASSERT_OK(builder->Finish()); + ASSERT_OK(fwriter->Flush()); + uint64_t file_size = 0; + s = fs_->GetFileSize(fname, IOOptions(), &file_size, nullptr); + ASSERT_OK(s); + ASSERT_NE(0, file_size); + file_metas->emplace_back( + file_num, /*file_path_id=*/0, file_size, ikey, ikey, 0, 0, false, + Temperature::kUnknown, 0, 0, 0, info.epoch_number, + kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2, + 0, 0, /* user_defined_timestamps_persisted */ true); + } + } + + // This method updates last_sequence_. + void WriteFileAdditionAndDeletionToManifest( + uint32_t cf, const std::vector>& added_files, + const std::vector>& deleted_files) { + VersionEdit edit; + edit.SetColumnFamily(cf); + for (const auto& elem : added_files) { + int level = elem.first; + edit.AddFile(level, elem.second); + } + for (const auto& elem : deleted_files) { + int level = elem.first; + edit.DeleteFile(level, elem.second); + } + edit.SetLastSequence(last_seqno_); + ++last_seqno_; + assert(log_writer_.get() != nullptr); + std::string record; + ASSERT_TRUE(edit.EncodeTo(&record, 0 /* ts_sz */)); + Status s = log_writer_->AddRecord(record); + ASSERT_OK(s); + } + + BlockBasedTableOptions block_based_table_options_; + std::shared_ptr table_factory_; + std::shared_ptr internal_comparator_; + std::vector column_families_; + SequenceNumber last_seqno_; + std::unique_ptr log_writer_; +}; + +TEST_F(VersionSetTestMissingFiles, ManifestFarBehindSst) { + std::vector existing_files = { + SstInfo(100, kDefaultColumnFamilyName, "a", 100 /* epoch_number */), + SstInfo(102, kDefaultColumnFamilyName, "b", 102 /* epoch_number */), + SstInfo(103, kDefaultColumnFamilyName, "c", 103 /* epoch_number */), + SstInfo(107, kDefaultColumnFamilyName, "d", 107 /* epoch_number */), + SstInfo(110, kDefaultColumnFamilyName, "e", 110 /* epoch_number */)}; + std::vector file_metas; + CreateDummyTableFiles(existing_files, &file_metas); + + PrepareManifest(&column_families_, &last_seqno_, &log_writer_); + std::vector> added_files; + for (uint64_t file_num = 10; file_num < 15; ++file_num) { + std::string smallest_ukey = "a"; + std::string largest_ukey = "b"; + InternalKey smallest_ikey(smallest_ukey, 1, ValueType::kTypeValue); + InternalKey largest_ikey(largest_ukey, 1, ValueType::kTypeValue); + + FileMetaData meta = FileMetaData( + file_num, /*file_path_id=*/0, /*file_size=*/12, smallest_ikey, + largest_ikey, 0, 0, false, Temperature::kUnknown, 0, 0, 0, + file_num /* epoch_number */, kUnknownFileChecksum, + kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0, + /* user_defined_timestamps_persisted */ true); + added_files.emplace_back(0, meta); + } + WriteFileAdditionAndDeletionToManifest( + /*cf=*/0, added_files, std::vector>()); + std::vector> deleted_files; + deleted_files.emplace_back(0, 10); + WriteFileAdditionAndDeletionToManifest( + /*cf=*/0, std::vector>(), deleted_files); + log_writer_.reset(); + Status s = SetCurrentFile(fs_.get(), dbname_, 1, nullptr); + ASSERT_OK(s); + std::string manifest_path; + VerifyManifest(&manifest_path); + std::string db_id; + bool has_missing_table_file = false; + s = versions_->TryRecoverFromOneManifest(manifest_path, column_families_, + /*read_only=*/false, &db_id, + &has_missing_table_file); + ASSERT_OK(s); + ASSERT_TRUE(has_missing_table_file); + for (ColumnFamilyData* cfd : *(versions_->GetColumnFamilySet())) { + VersionStorageInfo* vstorage = cfd->current()->storage_info(); + const std::vector& files = vstorage->LevelFiles(0); + ASSERT_TRUE(files.empty()); + } +} + +TEST_F(VersionSetTestMissingFiles, ManifestAheadofSst) { + std::vector existing_files = { + SstInfo(100, kDefaultColumnFamilyName, "a", 0 /* level */, + 100 /* epoch_number */), + SstInfo(102, kDefaultColumnFamilyName, "b", 0 /* level */, + 102 /* epoch_number */), + SstInfo(103, kDefaultColumnFamilyName, "c", 0 /* level */, + 103 /* epoch_number */), + SstInfo(107, kDefaultColumnFamilyName, "d", 0 /* level */, + 107 /* epoch_number */), + SstInfo(110, kDefaultColumnFamilyName, "e", 0 /* level */, + 110 /* epoch_number */)}; + std::vector file_metas; + CreateDummyTableFiles(existing_files, &file_metas); + + PrepareManifest(&column_families_, &last_seqno_, &log_writer_); + std::vector> added_files; + for (size_t i = 3; i != 5; ++i) { + added_files.emplace_back(0, file_metas[i]); + } + WriteFileAdditionAndDeletionToManifest( + /*cf=*/0, added_files, std::vector>()); + + added_files.clear(); + for (uint64_t file_num = 120; file_num < 130; ++file_num) { + std::string smallest_ukey = "a"; + std::string largest_ukey = "b"; + InternalKey smallest_ikey(smallest_ukey, 1, ValueType::kTypeValue); + InternalKey largest_ikey(largest_ukey, 1, ValueType::kTypeValue); + FileMetaData meta = FileMetaData( + file_num, /*file_path_id=*/0, /*file_size=*/12, smallest_ikey, + largest_ikey, 0, 0, false, Temperature::kUnknown, 0, 0, 0, + file_num /* epoch_number */, kUnknownFileChecksum, + kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0, + /* user_defined_timestamps_persisted */ true); + added_files.emplace_back(0, meta); + } + WriteFileAdditionAndDeletionToManifest( + /*cf=*/0, added_files, std::vector>()); + log_writer_.reset(); + Status s = SetCurrentFile(fs_.get(), dbname_, 1, nullptr); + ASSERT_OK(s); + std::string manifest_path; + VerifyManifest(&manifest_path); + std::string db_id; + bool has_missing_table_file = false; + s = versions_->TryRecoverFromOneManifest(manifest_path, column_families_, + /*read_only=*/false, &db_id, + &has_missing_table_file); + ASSERT_OK(s); + ASSERT_TRUE(has_missing_table_file); + for (ColumnFamilyData* cfd : *(versions_->GetColumnFamilySet())) { + VersionStorageInfo* vstorage = cfd->current()->storage_info(); + const std::vector& files = vstorage->LevelFiles(0); + if (cfd->GetName() == kDefaultColumnFamilyName) { + ASSERT_EQ(2, files.size()); + for (const auto* fmeta : files) { + if (fmeta->fd.GetNumber() != 107 && fmeta->fd.GetNumber() != 110) { + ASSERT_FALSE(true); + } + } + } else { + ASSERT_TRUE(files.empty()); + } + } +} + +TEST_F(VersionSetTestMissingFiles, NoFileMissing) { + std::vector existing_files = { + SstInfo(100, kDefaultColumnFamilyName, "a", 0 /* level */, + 100 /* epoch_number */), + SstInfo(102, kDefaultColumnFamilyName, "b", 0 /* level */, + 102 /* epoch_number */), + SstInfo(103, kDefaultColumnFamilyName, "c", 0 /* level */, + 103 /* epoch_number */), + SstInfo(107, kDefaultColumnFamilyName, "d", 0 /* level */, + 107 /* epoch_number */), + SstInfo(110, kDefaultColumnFamilyName, "e", 0 /* level */, + 110 /* epoch_number */)}; + std::vector file_metas; + CreateDummyTableFiles(existing_files, &file_metas); + + PrepareManifest(&column_families_, &last_seqno_, &log_writer_); + std::vector> added_files; + for (const auto& meta : file_metas) { + added_files.emplace_back(0, meta); + } + WriteFileAdditionAndDeletionToManifest( + /*cf=*/0, added_files, std::vector>()); + std::vector> deleted_files; + deleted_files.emplace_back(/*level=*/0, 100); + WriteFileAdditionAndDeletionToManifest( + /*cf=*/0, std::vector>(), deleted_files); + log_writer_.reset(); + Status s = SetCurrentFile(fs_.get(), dbname_, 1, nullptr); + ASSERT_OK(s); + std::string manifest_path; + VerifyManifest(&manifest_path); + std::string db_id; + bool has_missing_table_file = false; + s = versions_->TryRecoverFromOneManifest(manifest_path, column_families_, + /*read_only=*/false, &db_id, + &has_missing_table_file); + ASSERT_OK(s); + ASSERT_FALSE(has_missing_table_file); + for (ColumnFamilyData* cfd : *(versions_->GetColumnFamilySet())) { + VersionStorageInfo* vstorage = cfd->current()->storage_info(); + const std::vector& files = vstorage->LevelFiles(0); + if (cfd->GetName() == kDefaultColumnFamilyName) { + ASSERT_EQ(existing_files.size() - deleted_files.size(), files.size()); + bool has_deleted_file = false; + for (const auto* fmeta : files) { + if (fmeta->fd.GetNumber() == 100) { + has_deleted_file = true; + break; + } + } + ASSERT_FALSE(has_deleted_file); + } else { + ASSERT_TRUE(files.empty()); + } + } +} + +TEST_F(VersionSetTestMissingFiles, MinLogNumberToKeep2PC) { + db_options_.allow_2pc = true; + NewDB(); + + SstInfo sst(100, kDefaultColumnFamilyName, "a", 0 /* level */, + 100 /* epoch_number */); + std::vector file_metas; + CreateDummyTableFiles({sst}, &file_metas); + + constexpr WalNumber kMinWalNumberToKeep2PC = 10; + VersionEdit edit; + edit.AddFile(0, file_metas[0]); + edit.SetMinLogNumberToKeep(kMinWalNumberToKeep2PC); + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + ASSERT_EQ(versions_->min_log_number_to_keep(), kMinWalNumberToKeep2PC); + + for (int i = 0; i < 3; i++) { + CreateNewManifest(); + ReopenDB(); + ASSERT_EQ(versions_->min_log_number_to_keep(), kMinWalNumberToKeep2PC); + } +} + +class ChargeFileMetadataTest : public DBTestBase { + public: + ChargeFileMetadataTest() + : DBTestBase("charge_file_metadata_test", /*env_do_fsync=*/true) {} +}; + +class ChargeFileMetadataTestWithParam + : public ChargeFileMetadataTest, + public testing::WithParamInterface { + public: + ChargeFileMetadataTestWithParam() {} +}; + +INSTANTIATE_TEST_CASE_P( + ChargeFileMetadataTestWithParam, ChargeFileMetadataTestWithParam, + ::testing::Values(CacheEntryRoleOptions::Decision::kEnabled, + CacheEntryRoleOptions::Decision::kDisabled)); + +TEST_P(ChargeFileMetadataTestWithParam, Basic) { + Options options; + options.level_compaction_dynamic_level_bytes = false; + BlockBasedTableOptions table_options; + CacheEntryRoleOptions::Decision charge_file_metadata = GetParam(); + table_options.cache_usage_options.options_overrides.insert( + {CacheEntryRole::kFileMetadata, {/*.charged = */ charge_file_metadata}}); + std::shared_ptr> + file_metadata_charge_only_cache = std::make_shared< + TargetCacheChargeTrackingCache>( + NewLRUCache( + 4 * CacheReservationManagerImpl< + CacheEntryRole::kFileMetadata>::GetDummyEntrySize(), + 0 /* num_shard_bits */, true /* strict_capacity_limit */)); + table_options.block_cache = file_metadata_charge_only_cache; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.create_if_missing = true; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + + // Create 128 file metadata, each of which is roughly 1024 bytes. + // This results in 1 * + // CacheReservationManagerImpl::GetDummyEntrySize() + // cache reservation for file metadata. + for (int i = 1; i <= 128; ++i) { + ASSERT_OK(Put(std::string(1024, 'a'), "va")); + ASSERT_OK(Put("b", "vb")); + ASSERT_OK(Flush()); + } + if (charge_file_metadata == CacheEntryRoleOptions::Decision::kEnabled) { + EXPECT_EQ(file_metadata_charge_only_cache->GetCacheCharge(), + 1 * CacheReservationManagerImpl< + CacheEntryRole::kFileMetadata>::GetDummyEntrySize()); + + } else { + EXPECT_EQ(file_metadata_charge_only_cache->GetCacheCharge(), 0); + } + + // Create another 128 file metadata. + // This increases the file metadata cache reservation to 2 * + // CacheReservationManagerImpl::GetDummyEntrySize(). + for (int i = 1; i <= 128; ++i) { + ASSERT_OK(Put(std::string(1024, 'a'), "vva")); + ASSERT_OK(Put("b", "vvb")); + ASSERT_OK(Flush()); + } + if (charge_file_metadata == CacheEntryRoleOptions::Decision::kEnabled) { + EXPECT_EQ(file_metadata_charge_only_cache->GetCacheCharge(), + 2 * CacheReservationManagerImpl< + CacheEntryRole::kFileMetadata>::GetDummyEntrySize()); + } else { + EXPECT_EQ(file_metadata_charge_only_cache->GetCacheCharge(), 0); + } + // Compaction will create 1 new file metadata, obsolete and delete all 256 + // file metadata above. This results in 1 * + // CacheReservationManagerImpl::GetDummyEntrySize() + // cache reservation for file metadata. + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BackgroundCallCompaction:PurgedObsoleteFiles", + "ChargeFileMetadataTestWithParam::" + "PreVerifyingCacheReservationRelease"}}); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("0,1", FilesPerLevel(0)); + TEST_SYNC_POINT( + "ChargeFileMetadataTestWithParam::PreVerifyingCacheReservationRelease"); + if (charge_file_metadata == CacheEntryRoleOptions::Decision::kEnabled) { + EXPECT_EQ(file_metadata_charge_only_cache->GetCacheCharge(), + 1 * CacheReservationManagerImpl< + CacheEntryRole::kFileMetadata>::GetDummyEntrySize()); + } else { + EXPECT_EQ(file_metadata_charge_only_cache->GetCacheCharge(), 0); + } + SyncPoint::GetInstance()->DisableProcessing(); + + // Destroying the db will delete the remaining 1 new file metadata + // This results in no cache reservation for file metadata. + Destroy(options); + EXPECT_EQ(file_metadata_charge_only_cache->GetCacheCharge(), + 0 * CacheReservationManagerImpl< + CacheEntryRole::kFileMetadata>::GetDummyEntrySize()); + + // Reopen the db with a smaller cache in order to test failure in allocating + // file metadata due to memory limit based on cache capacity + file_metadata_charge_only_cache = std::make_shared< + TargetCacheChargeTrackingCache>( + NewLRUCache(1 * CacheReservationManagerImpl< + CacheEntryRole::kFileMetadata>::GetDummyEntrySize(), + 0 /* num_shard_bits */, true /* strict_capacity_limit */)); + table_options.block_cache = file_metadata_charge_only_cache; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Reopen(options); + ASSERT_OK(Put(std::string(1024, 'a'), "va")); + ASSERT_OK(Put("b", "vb")); + Status s = Flush(); + if (charge_file_metadata == CacheEntryRoleOptions::Decision::kEnabled) { + EXPECT_TRUE(s.IsMemoryLimit()); + EXPECT_TRUE(s.ToString().find( + kCacheEntryRoleToCamelString[static_cast( + CacheEntryRole::kFileMetadata)]) != std::string::npos); + EXPECT_TRUE(s.ToString().find("memory limit based on cache capacity") != + std::string::npos); + } else { + EXPECT_TRUE(s.ok()); + } +} +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/version_util.h b/librocksdb-sys/rocksdb/db/version_util.h new file mode 100644 index 0000000..e39f255 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/version_util.h @@ -0,0 +1,72 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "db/version_set.h" + +namespace ROCKSDB_NAMESPACE { + +// Instead of opening a `DB` to perform certain manifest updates, this +// uses the underlying `VersionSet` API to read and modify the MANIFEST. This +// allows us to use the user's real options, while not having to worry about +// the DB persisting new SST files via flush/compaction or attempting to read/ +// compact files which may fail, particularly for the file we intend to remove +// (the user may want to remove an already deleted file from MANIFEST). +class OfflineManifestWriter { + public: + OfflineManifestWriter(const DBOptions& options, const std::string& db_path) + : wc_(options.delayed_write_rate), + wb_(options.db_write_buffer_size), + immutable_db_options_(WithDbPath(options, db_path)), + tc_(NewLRUCache(1 << 20 /* capacity */, + options.table_cache_numshardbits)), + versions_(db_path, &immutable_db_options_, sopt_, tc_.get(), &wb_, &wc_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id*/ "", /*db_session_id*/ "") {} + + Status Recover(const std::vector& column_families) { + return versions_.Recover(column_families, /*read_only*/ false, + /*db_id*/ nullptr, + /*no_error_if_files_missing*/ true); + } + + Status LogAndApply(const ReadOptions& read_options, ColumnFamilyData* cfd, + VersionEdit* edit, + FSDirectory* dir_contains_current_file) { + // Use `mutex` to imitate a locked DB mutex when calling `LogAndApply()`. + InstrumentedMutex mutex; + mutex.Lock(); + Status s = versions_.LogAndApply( + cfd, *cfd->GetLatestMutableCFOptions(), read_options, edit, &mutex, + dir_contains_current_file, false /* new_descriptor_log */); + mutex.Unlock(); + return s; + } + + VersionSet& Versions() { return versions_; } + const ImmutableDBOptions& IOptions() { return immutable_db_options_; } + + private: + WriteController wc_; + WriteBufferManager wb_; + ImmutableDBOptions immutable_db_options_; + std::shared_ptr tc_; + EnvOptions sopt_; + VersionSet versions_; + + static ImmutableDBOptions WithDbPath(const DBOptions& options, + const std::string& db_path) { + ImmutableDBOptions rv(options); + if (rv.db_paths.empty()) { + // `VersionSet` expects options that have been through + // `SanitizeOptions()`, which would sanitize an empty `db_paths`. + rv.db_paths.emplace_back(db_path, 0 /* target_size */); + } + return rv; + } +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/wal_edit.cc b/librocksdb-sys/rocksdb/db/wal_edit.cc new file mode 100644 index 0000000..2525be6 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/wal_edit.cc @@ -0,0 +1,211 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/wal_edit.h" + +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { + +void WalAddition::EncodeTo(std::string* dst) const { + PutVarint64(dst, number_); + + if (metadata_.HasSyncedSize()) { + PutVarint32(dst, static_cast(WalAdditionTag::kSyncedSize)); + PutVarint64(dst, metadata_.GetSyncedSizeInBytes()); + } + + PutVarint32(dst, static_cast(WalAdditionTag::kTerminate)); +} + +Status WalAddition::DecodeFrom(Slice* src) { + constexpr char class_name[] = "WalAddition"; + + if (!GetVarint64(src, &number_)) { + return Status::Corruption(class_name, "Error decoding WAL log number"); + } + + while (true) { + uint32_t tag_value = 0; + if (!GetVarint32(src, &tag_value)) { + return Status::Corruption(class_name, "Error decoding tag"); + } + WalAdditionTag tag = static_cast(tag_value); + switch (tag) { + case WalAdditionTag::kSyncedSize: { + uint64_t size = 0; + if (!GetVarint64(src, &size)) { + return Status::Corruption(class_name, "Error decoding WAL file size"); + } + metadata_.SetSyncedSizeInBytes(size); + break; + } + // TODO: process future tags such as checksum. + case WalAdditionTag::kTerminate: + return Status::OK(); + default: { + std::stringstream ss; + ss << "Unknown tag " << tag_value; + return Status::Corruption(class_name, ss.str()); + } + } + } +} + +JSONWriter& operator<<(JSONWriter& jw, const WalAddition& wal) { + jw << "LogNumber" << wal.GetLogNumber() << "SyncedSizeInBytes" + << wal.GetMetadata().GetSyncedSizeInBytes(); + return jw; +} + +std::ostream& operator<<(std::ostream& os, const WalAddition& wal) { + os << "log_number: " << wal.GetLogNumber() + << " synced_size_in_bytes: " << wal.GetMetadata().GetSyncedSizeInBytes(); + return os; +} + +std::string WalAddition::DebugString() const { + std::ostringstream oss; + oss << *this; + return oss.str(); +} + +void WalDeletion::EncodeTo(std::string* dst) const { + PutVarint64(dst, number_); +} + +Status WalDeletion::DecodeFrom(Slice* src) { + constexpr char class_name[] = "WalDeletion"; + + if (!GetVarint64(src, &number_)) { + return Status::Corruption(class_name, "Error decoding WAL log number"); + } + + return Status::OK(); +} + +JSONWriter& operator<<(JSONWriter& jw, const WalDeletion& wal) { + jw << "LogNumber" << wal.GetLogNumber(); + return jw; +} + +std::ostream& operator<<(std::ostream& os, const WalDeletion& wal) { + os << "log_number: " << wal.GetLogNumber(); + return os; +} + +std::string WalDeletion::DebugString() const { + std::ostringstream oss; + oss << *this; + return oss.str(); +} + +Status WalSet::AddWal(const WalAddition& wal) { + if (wal.GetLogNumber() < min_wal_number_to_keep_) { + // The WAL has been obsolete, ignore it. + return Status::OK(); + } + + auto it = wals_.lower_bound(wal.GetLogNumber()); + bool existing = it != wals_.end() && it->first == wal.GetLogNumber(); + + if (!existing) { + wals_.insert(it, {wal.GetLogNumber(), wal.GetMetadata()}); + return Status::OK(); + } + + assert(existing); + if (!wal.GetMetadata().HasSyncedSize()) { + std::stringstream ss; + ss << "WAL " << wal.GetLogNumber() << " is created more than once"; + return Status::Corruption("WalSet::AddWal", ss.str()); + } + + assert(wal.GetMetadata().HasSyncedSize()); + if (it->second.HasSyncedSize() && wal.GetMetadata().GetSyncedSizeInBytes() <= + it->second.GetSyncedSizeInBytes()) { + // This is possible because version edits with different synced WAL sizes + // for the same WAL can be committed out-of-order. For example, thread + // 1 synces the first 10 bytes of 1.log, while thread 2 synces the first 20 + // bytes of 1.log. It's possible that thread 1 calls LogAndApply() after + // thread 2. + // In this case, just return ok. + return Status::OK(); + } + + // Update synced size for the given WAL. + it->second.SetSyncedSizeInBytes(wal.GetMetadata().GetSyncedSizeInBytes()); + return Status::OK(); +} + +Status WalSet::AddWals(const WalAdditions& wals) { + Status s; + for (const WalAddition& wal : wals) { + s = AddWal(wal); + if (!s.ok()) { + break; + } + } + return s; +} + +Status WalSet::DeleteWalsBefore(WalNumber wal) { + if (wal > min_wal_number_to_keep_) { + min_wal_number_to_keep_ = wal; + wals_.erase(wals_.begin(), wals_.lower_bound(wal)); + } + return Status::OK(); +} + +void WalSet::Reset() { + wals_.clear(); + min_wal_number_to_keep_ = 0; +} + +Status WalSet::CheckWals( + Env* env, + const std::unordered_map& logs_on_disk) const { + assert(env != nullptr); + + Status s; + for (const auto& wal : wals_) { + const uint64_t log_number = wal.first; + const WalMetadata& wal_meta = wal.second; + + if (!wal_meta.HasSyncedSize()) { + // The WAL and WAL directory is not even synced, + // so the WAL's inode may not be persisted, + // then the WAL might not show up when listing WAL directory. + continue; + } + + if (logs_on_disk.find(log_number) == logs_on_disk.end()) { + std::stringstream ss; + ss << "Missing WAL with log number: " << log_number << "."; + s = Status::Corruption(ss.str()); + break; + } + + uint64_t log_file_size = 0; + s = env->GetFileSize(logs_on_disk.at(log_number), &log_file_size); + if (!s.ok()) { + break; + } + if (log_file_size < wal_meta.GetSyncedSizeInBytes()) { + std::stringstream ss; + ss << "Size mismatch: WAL (log number: " << log_number + << ") in MANIFEST is " << wal_meta.GetSyncedSizeInBytes() + << " bytes , but actually is " << log_file_size << " bytes on disk."; + s = Status::Corruption(ss.str()); + break; + } + } + + return s; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/wal_edit.h b/librocksdb-sys/rocksdb/db/wal_edit.h new file mode 100644 index 0000000..bb5c5e2 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/wal_edit.h @@ -0,0 +1,177 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +// WAL related classes used in VersionEdit and VersionSet. +// Modifications to WalAddition and WalDeletion may need to update +// VersionEdit and its related tests. + +#pragma once + +#include +#include +#include +#include +#include + +#include "logging/event_logger.h" +#include "port/port.h" +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +class JSONWriter; +class Slice; +class Status; + +using WalNumber = uint64_t; + +// Metadata of a WAL. +class WalMetadata { + public: + WalMetadata() = default; + + explicit WalMetadata(uint64_t synced_size_bytes) + : synced_size_bytes_(synced_size_bytes) {} + + bool HasSyncedSize() const { return synced_size_bytes_ != kUnknownWalSize; } + + void SetSyncedSizeInBytes(uint64_t bytes) { synced_size_bytes_ = bytes; } + + uint64_t GetSyncedSizeInBytes() const { return synced_size_bytes_; } + + private: + friend bool operator==(const WalMetadata& lhs, const WalMetadata& rhs); + friend bool operator!=(const WalMetadata& lhs, const WalMetadata& rhs); + // The size of WAL is unknown, used when the WAL is not synced yet or is + // empty. + constexpr static uint64_t kUnknownWalSize = + std::numeric_limits::max(); + + // Size of the most recently synced WAL in bytes. + uint64_t synced_size_bytes_ = kUnknownWalSize; +}; + +inline bool operator==(const WalMetadata& lhs, const WalMetadata& rhs) { + return lhs.synced_size_bytes_ == rhs.synced_size_bytes_; +} + +inline bool operator!=(const WalMetadata& lhs, const WalMetadata& rhs) { + return !(lhs == rhs); +} + +// These tags are persisted to MANIFEST, so it's part of the user API. +enum class WalAdditionTag : uint32_t { + // Indicates that there are no more tags. + kTerminate = 1, + // Synced Size in bytes. + kSyncedSize = 2, + // Add tags in the future, such as checksum? +}; + +// Records the event of adding a WAL in VersionEdit. +class WalAddition { + public: + WalAddition() : number_(0), metadata_() {} + + explicit WalAddition(WalNumber number) : number_(number), metadata_() {} + + WalAddition(WalNumber number, WalMetadata meta) + : number_(number), metadata_(std::move(meta)) {} + + WalNumber GetLogNumber() const { return number_; } + + const WalMetadata& GetMetadata() const { return metadata_; } + + void EncodeTo(std::string* dst) const; + + Status DecodeFrom(Slice* src); + + std::string DebugString() const; + + private: + WalNumber number_; + WalMetadata metadata_; +}; + +std::ostream& operator<<(std::ostream& os, const WalAddition& wal); +JSONWriter& operator<<(JSONWriter& jw, const WalAddition& wal); + +using WalAdditions = std::vector; + +// Records the event of deleting WALs before the specified log number. +class WalDeletion { + public: + WalDeletion() : number_(kEmpty) {} + + explicit WalDeletion(WalNumber number) : number_(number) {} + + WalNumber GetLogNumber() const { return number_; } + + void EncodeTo(std::string* dst) const; + + Status DecodeFrom(Slice* src); + + std::string DebugString() const; + + bool IsEmpty() const { return number_ == kEmpty; } + + void Reset() { number_ = kEmpty; } + + private: + static constexpr WalNumber kEmpty = 0; + + WalNumber number_; +}; + +std::ostream& operator<<(std::ostream& os, const WalDeletion& wal); +JSONWriter& operator<<(JSONWriter& jw, const WalDeletion& wal); + +// Used in VersionSet to keep the current set of WALs. +// +// When a WAL is synced or becomes obsoleted, +// a VersionEdit is logged to MANIFEST and +// the WAL is added to or deleted from WalSet. +// +// Not thread safe, needs external synchronization such as holding DB mutex. +class WalSet { + public: + // Add WAL(s). + // If the WAL is closed, + // then there must be an existing unclosed WAL, + // otherwise, return Status::Corruption. + // Can happen when applying a VersionEdit or recovering from MANIFEST. + Status AddWal(const WalAddition& wal); + Status AddWals(const WalAdditions& wals); + + // Delete WALs with log number smaller than the specified wal number. + // Can happen when applying a VersionEdit or recovering from MANIFEST. + Status DeleteWalsBefore(WalNumber wal); + + // Resets the internal state. + void Reset(); + + // WALs with number less than MinWalNumberToKeep should not exist in WalSet. + WalNumber GetMinWalNumberToKeep() const { return min_wal_number_to_keep_; } + + const std::map& GetWals() const { return wals_; } + + // Checks whether there are missing or corrupted WALs. + // Returns Status::OK if there is no missing nor corrupted WAL, + // otherwise returns Status::Corruption. + // logs_on_disk is a map from log number to the log filename. + // Note that logs_on_disk may contain logs that is obsolete but + // haven't been deleted from disk. + Status CheckWals( + Env* env, + const std::unordered_map& logs_on_disk) const; + + private: + std::map wals_; + // WAL number < min_wal_number_to_keep_ should not exist in wals_. + // It's monotonically increasing, in-memory only, not written to MANIFEST. + WalNumber min_wal_number_to_keep_ = 0; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/wal_edit_test.cc b/librocksdb-sys/rocksdb/db/wal_edit_test.cc new file mode 100644 index 0000000..0c18fb1 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/wal_edit_test.cc @@ -0,0 +1,213 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/wal_edit.h" + +#include "db/db_test_util.h" +#include "file/file_util.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" + +namespace ROCKSDB_NAMESPACE { + +TEST(WalSet, AddDeleteReset) { + WalSet wals; + ASSERT_TRUE(wals.GetWals().empty()); + + // Create WAL 1 - 10. + for (WalNumber log_number = 1; log_number <= 10; log_number++) { + wals.AddWal(WalAddition(log_number)); + } + ASSERT_EQ(wals.GetWals().size(), 10); + + // Delete WAL 1 - 5. + wals.DeleteWalsBefore(6); + ASSERT_EQ(wals.GetWals().size(), 5); + + WalNumber expected_log_number = 6; + for (auto it : wals.GetWals()) { + WalNumber log_number = it.first; + ASSERT_EQ(log_number, expected_log_number++); + } + + wals.Reset(); + ASSERT_TRUE(wals.GetWals().empty()); +} + +TEST(WalSet, Overwrite) { + constexpr WalNumber kNumber = 100; + constexpr uint64_t kBytes = 200; + WalSet wals; + wals.AddWal(WalAddition(kNumber)); + ASSERT_FALSE(wals.GetWals().at(kNumber).HasSyncedSize()); + wals.AddWal(WalAddition(kNumber, WalMetadata(kBytes))); + ASSERT_TRUE(wals.GetWals().at(kNumber).HasSyncedSize()); + ASSERT_EQ(wals.GetWals().at(kNumber).GetSyncedSizeInBytes(), kBytes); +} + +TEST(WalSet, SmallerSyncedSize) { + constexpr WalNumber kNumber = 100; + constexpr uint64_t kBytes = 100; + WalSet wals; + ASSERT_OK(wals.AddWal(WalAddition(kNumber, WalMetadata(kBytes)))); + const auto wals1 = wals.GetWals(); + Status s = wals.AddWal(WalAddition(kNumber, WalMetadata(0))); + const auto wals2 = wals.GetWals(); + ASSERT_OK(s); + ASSERT_EQ(wals1, wals2); +} + +TEST(WalSet, CreateTwice) { + constexpr WalNumber kNumber = 100; + WalSet wals; + ASSERT_OK(wals.AddWal(WalAddition(kNumber))); + Status s = wals.AddWal(WalAddition(kNumber)); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(s.ToString().find("WAL 100 is created more than once") != + std::string::npos); +} + +TEST(WalSet, DeleteAllWals) { + constexpr WalNumber kMaxWalNumber = 10; + WalSet wals; + for (WalNumber i = 1; i <= kMaxWalNumber; i++) { + wals.AddWal(WalAddition(i)); + } + ASSERT_OK(wals.DeleteWalsBefore(kMaxWalNumber + 1)); +} + +TEST(WalSet, AddObsoleteWal) { + constexpr WalNumber kNumber = 100; + WalSet wals; + ASSERT_OK(wals.DeleteWalsBefore(kNumber + 1)); + ASSERT_OK(wals.AddWal(WalAddition(kNumber))); + ASSERT_TRUE(wals.GetWals().empty()); +} + +TEST(WalSet, MinWalNumberToKeep) { + constexpr WalNumber kNumber = 100; + WalSet wals; + ASSERT_EQ(wals.GetMinWalNumberToKeep(), 0); + ASSERT_OK(wals.DeleteWalsBefore(kNumber)); + ASSERT_EQ(wals.GetMinWalNumberToKeep(), kNumber); + ASSERT_OK(wals.DeleteWalsBefore(kNumber - 1)); + ASSERT_EQ(wals.GetMinWalNumberToKeep(), kNumber); + ASSERT_OK(wals.DeleteWalsBefore(kNumber + 1)); + ASSERT_EQ(wals.GetMinWalNumberToKeep(), kNumber + 1); +} + +class WalSetTest : public DBTestBase { + public: + WalSetTest() : DBTestBase("WalSetTest", /* env_do_fsync */ true) {} + + void SetUp() override { + test_dir_ = test::PerThreadDBPath("wal_set_test"); + ASSERT_OK(env_->CreateDir(test_dir_)); + } + + void TearDown() override { + EXPECT_OK(DestroyDir(env_, test_dir_)); + logs_on_disk_.clear(); + wals_.Reset(); + } + + void CreateWalOnDisk(WalNumber number, const std::string& fname, + uint64_t size_bytes) { + std::unique_ptr f; + std::string fpath = Path(fname); + ASSERT_OK(env_->NewWritableFile(fpath, &f, EnvOptions())); + std::string content(size_bytes, '0'); + ASSERT_OK(f->Append(content)); + ASSERT_OK(f->Close()); + + logs_on_disk_[number] = fpath; + } + + void AddWalToWalSet(WalNumber number, uint64_t size_bytes) { + // Create WAL. + ASSERT_OK(wals_.AddWal(WalAddition(number))); + // Close WAL. + WalMetadata wal(size_bytes); + ASSERT_OK(wals_.AddWal(WalAddition(number, wal))); + } + + Status CheckWals() const { return wals_.CheckWals(env_, logs_on_disk_); } + + private: + std::string test_dir_; + std::unordered_map logs_on_disk_; + WalSet wals_; + + std::string Path(const std::string& fname) { return test_dir_ + "/" + fname; } +}; + +TEST_F(WalSetTest, CheckEmptyWals) { ASSERT_OK(CheckWals()); } + +TEST_F(WalSetTest, CheckWals) { + for (int number = 1; number < 10; number++) { + uint64_t size = rand() % 100; + std::stringstream ss; + ss << "log" << number; + std::string fname = ss.str(); + CreateWalOnDisk(number, fname, size); + // log 0 - 5 are obsolete. + if (number > 5) { + AddWalToWalSet(number, size); + } + } + ASSERT_OK(CheckWals()); +} + +TEST_F(WalSetTest, CheckMissingWals) { + for (int number = 1; number < 10; number++) { + uint64_t size = rand() % 100; + AddWalToWalSet(number, size); + // logs with even number are missing from disk. + if (number % 2) { + std::stringstream ss; + ss << "log" << number; + std::string fname = ss.str(); + CreateWalOnDisk(number, fname, size); + } + } + + Status s = CheckWals(); + ASSERT_TRUE(s.IsCorruption()) << s.ToString(); + // The first log with even number is missing. + std::stringstream expected_err; + expected_err << "Missing WAL with log number: " << 2; + ASSERT_TRUE(s.ToString().find(expected_err.str()) != std::string::npos) + << s.ToString(); +} + +TEST_F(WalSetTest, CheckWalsWithShrinkedSize) { + for (int number = 1; number < 10; number++) { + uint64_t size = rand() % 100 + 1; + AddWalToWalSet(number, size); + // logs with even number have shrinked size. + std::stringstream ss; + ss << "log" << number; + std::string fname = ss.str(); + CreateWalOnDisk(number, fname, (number % 2) ? size : size - 1); + } + + Status s = CheckWals(); + ASSERT_TRUE(s.IsCorruption()) << s.ToString(); + // The first log with even number has wrong size. + std::stringstream expected_err; + expected_err << "Size mismatch: WAL (log number: " << 2 << ")"; + ASSERT_TRUE(s.ToString().find(expected_err.str()) != std::string::npos) + << s.ToString(); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/wal_manager.cc b/librocksdb-sys/rocksdb/db/wal_manager.cc new file mode 100644 index 0000000..400b2e5 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/wal_manager.cc @@ -0,0 +1,527 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/wal_manager.h" + +#include +#include +#include +#include + +#include "db/log_reader.h" +#include "db/log_writer.h" +#include "db/transaction_log_impl.h" +#include "db/write_batch_internal.h" +#include "file/file_util.h" +#include "file/filename.h" +#include "file/sequence_file_reader.h" +#include "logging/logging.h" +#include "port/port.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "rocksdb/write_batch.h" +#include "test_util/sync_point.h" +#include "util/cast_util.h" +#include "util/coding.h" +#include "util/mutexlock.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + + +Status WalManager::DeleteFile(const std::string& fname, uint64_t number) { + auto s = env_->DeleteFile(wal_dir_ + "/" + fname); + if (s.ok()) { + MutexLock l(&read_first_record_cache_mutex_); + read_first_record_cache_.erase(number); + } + return s; +} + +Status WalManager::GetSortedWalFiles(VectorLogPtr& files) { + // First get sorted files in db dir, then get sorted files from archived + // dir, to avoid a race condition where a log file is moved to archived + // dir in between. + Status s; + // list wal files in main db dir. + VectorLogPtr logs; + s = GetSortedWalsOfType(wal_dir_, logs, kAliveLogFile); + if (!s.ok()) { + return s; + } + + // Reproduce the race condition where a log file is moved + // to archived dir, between these two sync points, used in + // (DBTest,TransactionLogIteratorRace) + TEST_SYNC_POINT("WalManager::GetSortedWalFiles:1"); + TEST_SYNC_POINT("WalManager::GetSortedWalFiles:2"); + + files.clear(); + // list wal files in archive dir. + std::string archivedir = ArchivalDirectory(wal_dir_); + Status exists = env_->FileExists(archivedir); + if (exists.ok()) { + s = GetSortedWalsOfType(archivedir, files, kArchivedLogFile); + if (!s.ok()) { + return s; + } + } else if (!exists.IsNotFound()) { + assert(s.IsIOError()); + return s; + } + + uint64_t latest_archived_log_number = 0; + if (!files.empty()) { + latest_archived_log_number = files.back()->LogNumber(); + ROCKS_LOG_INFO(db_options_.info_log, "Latest Archived log: %" PRIu64, + latest_archived_log_number); + } + + files.reserve(files.size() + logs.size()); + for (auto& log : logs) { + if (log->LogNumber() > latest_archived_log_number) { + files.push_back(std::move(log)); + } else { + // When the race condition happens, we could see the + // same log in both db dir and archived dir. Simply + // ignore the one in db dir. Note that, if we read + // archived dir first, we would have missed the log file. + ROCKS_LOG_WARN(db_options_.info_log, "%s already moved to archive", + log->PathName().c_str()); + } + } + + return s; +} + +Status WalManager::GetUpdatesSince( + SequenceNumber seq, std::unique_ptr* iter, + const TransactionLogIterator::ReadOptions& read_options, + VersionSet* version_set) { + if (seq_per_batch_) { + return Status::NotSupported(); + } + + assert(!seq_per_batch_); + + // Get all sorted Wal Files. + // Do binary search and open files and find the seq number. + + std::unique_ptr wal_files(new VectorLogPtr); + Status s = GetSortedWalFiles(*wal_files); + if (!s.ok()) { + return s; + } + + s = RetainProbableWalFiles(*wal_files, seq); + if (!s.ok()) { + return s; + } + iter->reset(new TransactionLogIteratorImpl( + wal_dir_, &db_options_, read_options, file_options_, seq, + std::move(wal_files), version_set, seq_per_batch_, io_tracer_)); + return (*iter)->status(); +} + +// 1. Go through all archived files and +// a. if ttl is enabled, delete outdated files +// b. if archive size limit is enabled, delete empty files, +// compute file number and size. +// 2. If size limit is enabled: +// a. compute how many files should be deleted +// b. get sorted non-empty archived logs +// c. delete what should be deleted +void WalManager::PurgeObsoleteWALFiles() { + bool const ttl_enabled = db_options_.WAL_ttl_seconds > 0; + bool const size_limit_enabled = db_options_.WAL_size_limit_MB > 0; + if (!ttl_enabled && !size_limit_enabled) { + return; + } + + int64_t current_time = 0; + Status s = db_options_.clock->GetCurrentTime(¤t_time); + if (!s.ok()) { + ROCKS_LOG_ERROR(db_options_.info_log, "Can't get current time: %s", + s.ToString().c_str()); + assert(false); + return; + } + uint64_t const now_seconds = static_cast(current_time); + uint64_t const time_to_check = (ttl_enabled && !size_limit_enabled) + ? db_options_.WAL_ttl_seconds / 2 + : kDefaultIntervalToDeleteObsoleteWAL; + + if (purge_wal_files_last_run_ + time_to_check > now_seconds) { + return; + } + + purge_wal_files_last_run_ = now_seconds; + + std::string archival_dir = ArchivalDirectory(wal_dir_); + std::vector files; + s = env_->GetChildren(archival_dir, &files); + if (!s.ok()) { + ROCKS_LOG_ERROR(db_options_.info_log, "Can't get archive files: %s", + s.ToString().c_str()); + assert(false); + return; + } + + size_t log_files_num = 0; + uint64_t log_file_size = 0; + for (auto& f : files) { + uint64_t number; + FileType type; + if (ParseFileName(f, &number, &type) && type == kWalFile) { + std::string const file_path = archival_dir + "/" + f; + if (ttl_enabled) { + uint64_t file_m_time; + s = env_->GetFileModificationTime(file_path, &file_m_time); + if (!s.ok()) { + ROCKS_LOG_WARN(db_options_.info_log, + "Can't get file mod time: %s: %s", file_path.c_str(), + s.ToString().c_str()); + continue; + } + if (now_seconds - file_m_time > db_options_.WAL_ttl_seconds) { + s = DeleteDBFile(&db_options_, file_path, archival_dir, false, + /*force_fg=*/!wal_in_db_path_); + if (!s.ok()) { + ROCKS_LOG_WARN(db_options_.info_log, "Can't delete file: %s: %s", + file_path.c_str(), s.ToString().c_str()); + continue; + } else { + MutexLock l(&read_first_record_cache_mutex_); + read_first_record_cache_.erase(number); + } + continue; + } + } + + if (size_limit_enabled) { + uint64_t file_size; + s = env_->GetFileSize(file_path, &file_size); + if (!s.ok()) { + ROCKS_LOG_ERROR(db_options_.info_log, + "Unable to get file size: %s: %s", file_path.c_str(), + s.ToString().c_str()); + return; + } else { + if (file_size > 0) { + log_file_size = std::max(log_file_size, file_size); + ++log_files_num; + } else { + s = DeleteDBFile(&db_options_, file_path, archival_dir, false, + /*force_fg=*/!wal_in_db_path_); + if (!s.ok()) { + ROCKS_LOG_WARN(db_options_.info_log, + "Unable to delete file: %s: %s", file_path.c_str(), + s.ToString().c_str()); + continue; + } else { + MutexLock l(&read_first_record_cache_mutex_); + read_first_record_cache_.erase(number); + } + } + } + } + } + } + + if (0 == log_files_num || !size_limit_enabled) { + return; + } + + size_t const files_keep_num = static_cast( + db_options_.WAL_size_limit_MB * 1024 * 1024 / log_file_size); + if (log_files_num <= files_keep_num) { + return; + } + + size_t files_del_num = log_files_num - files_keep_num; + VectorLogPtr archived_logs; + s = GetSortedWalsOfType(archival_dir, archived_logs, kArchivedLogFile); + if (!s.ok()) { + ROCKS_LOG_WARN(db_options_.info_log, + "Unable to get archived WALs from: %s: %s", + archival_dir.c_str(), s.ToString().c_str()); + files_del_num = 0; + } else if (files_del_num > archived_logs.size()) { + ROCKS_LOG_WARN(db_options_.info_log, + "Trying to delete more archived log files than " + "exist. Deleting all"); + files_del_num = archived_logs.size(); + } + + for (size_t i = 0; i < files_del_num; ++i) { + std::string const file_path = archived_logs[i]->PathName(); + s = DeleteDBFile(&db_options_, wal_dir_ + "/" + file_path, wal_dir_, false, + /*force_fg=*/!wal_in_db_path_); + if (!s.ok()) { + ROCKS_LOG_WARN(db_options_.info_log, "Unable to delete file: %s: %s", + file_path.c_str(), s.ToString().c_str()); + continue; + } else { + MutexLock l(&read_first_record_cache_mutex_); + read_first_record_cache_.erase(archived_logs[i]->LogNumber()); + } + } +} + +void WalManager::ArchiveWALFile(const std::string& fname, uint64_t number) { + auto archived_log_name = ArchivedLogFileName(wal_dir_, number); + // The sync point below is used in (DBTest,TransactionLogIteratorRace) + TEST_SYNC_POINT("WalManager::PurgeObsoleteFiles:1"); + Status s = env_->RenameFile(fname, archived_log_name); + // The sync point below is used in (DBTest,TransactionLogIteratorRace) + TEST_SYNC_POINT("WalManager::PurgeObsoleteFiles:2"); + ROCKS_LOG_INFO(db_options_.info_log, "Move log file %s to %s -- %s\n", + fname.c_str(), archived_log_name.c_str(), + s.ToString().c_str()); +} + +Status WalManager::GetSortedWalsOfType(const std::string& path, + VectorLogPtr& log_files, + WalFileType log_type) { + std::vector all_files; + const Status status = env_->GetChildren(path, &all_files); + if (!status.ok()) { + return status; + } + log_files.reserve(all_files.size()); + for (const auto& f : all_files) { + uint64_t number; + FileType type; + if (ParseFileName(f, &number, &type) && type == kWalFile) { + SequenceNumber sequence; + Status s = ReadFirstRecord(log_type, number, &sequence); + if (!s.ok()) { + return s; + } + if (sequence == 0) { + // empty file + continue; + } + + // Reproduce the race condition where a log file is moved + // to archived dir, between these two sync points, used in + // (DBTest,TransactionLogIteratorRace) + TEST_SYNC_POINT("WalManager::GetSortedWalsOfType:1"); + TEST_SYNC_POINT("WalManager::GetSortedWalsOfType:2"); + + uint64_t size_bytes; + s = env_->GetFileSize(LogFileName(path, number), &size_bytes); + // re-try in case the alive log file has been moved to archive. + if (!s.ok() && log_type == kAliveLogFile) { + std::string archived_file = ArchivedLogFileName(path, number); + if (env_->FileExists(archived_file).ok()) { + s = env_->GetFileSize(archived_file, &size_bytes); + if (!s.ok() && env_->FileExists(archived_file).IsNotFound()) { + // oops, the file just got deleted from archived dir! move on + s = Status::OK(); + continue; + } + } + } + if (!s.ok()) { + return s; + } + + log_files.push_back(std::unique_ptr( + new LogFileImpl(number, log_type, sequence, size_bytes))); + } + } + std::sort( + log_files.begin(), log_files.end(), + [](const std::unique_ptr& a, const std::unique_ptr& b) { + LogFileImpl* a_impl = static_cast_with_check(a.get()); + LogFileImpl* b_impl = static_cast_with_check(b.get()); + return *a_impl < *b_impl; + }); + return status; +} + +Status WalManager::RetainProbableWalFiles(VectorLogPtr& all_logs, + const SequenceNumber target) { + int64_t start = 0; // signed to avoid overflow when target is < first file. + int64_t end = static_cast(all_logs.size()) - 1; + // Binary Search. avoid opening all files. + while (end >= start) { + int64_t mid = start + (end - start) / 2; // Avoid overflow. + SequenceNumber current_seq_num = + all_logs.at(static_cast(mid))->StartSequence(); + if (current_seq_num == target) { + end = mid; + break; + } else if (current_seq_num < target) { + start = mid + 1; + } else { + end = mid - 1; + } + } + // end could be -ve. + size_t start_index = + static_cast(std::max(static_cast(0), end)); + // The last wal file is always included + all_logs.erase(all_logs.begin(), all_logs.begin() + start_index); + return Status::OK(); +} + +Status WalManager::ReadFirstRecord(const WalFileType type, + const uint64_t number, + SequenceNumber* sequence) { + *sequence = 0; + if (type != kAliveLogFile && type != kArchivedLogFile) { + ROCKS_LOG_ERROR(db_options_.info_log, "[WalManger] Unknown file type %s", + std::to_string(type).c_str()); + return Status::NotSupported("File Type Not Known " + std::to_string(type)); + } + { + MutexLock l(&read_first_record_cache_mutex_); + auto itr = read_first_record_cache_.find(number); + if (itr != read_first_record_cache_.end()) { + *sequence = itr->second; + return Status::OK(); + } + } + Status s; + if (type == kAliveLogFile) { + std::string fname = LogFileName(wal_dir_, number); + s = ReadFirstLine(fname, number, sequence); + if (!s.ok() && env_->FileExists(fname).ok()) { + // return any error that is not caused by non-existing file + return s; + } + } + + if (type == kArchivedLogFile || !s.ok()) { + // check if the file got moved to archive. + std::string archived_file = ArchivedLogFileName(wal_dir_, number); + s = ReadFirstLine(archived_file, number, sequence); + // maybe the file was deleted from archive dir. If that's the case, return + // Status::OK(). The caller with identify this as empty file because + // *sequence == 0 + if (!s.ok() && env_->FileExists(archived_file).IsNotFound()) { + return Status::OK(); + } + } + + if (s.ok() && *sequence != 0) { + MutexLock l(&read_first_record_cache_mutex_); + read_first_record_cache_.insert({number, *sequence}); + } + return s; +} + +Status WalManager::GetLiveWalFile(uint64_t number, + std::unique_ptr* log_file) { + if (!log_file) { + return Status::InvalidArgument("log_file not preallocated."); + } + + if (!number) { + return Status::PathNotFound("log file not available"); + } + + Status s; + + uint64_t size_bytes; + s = env_->GetFileSize(LogFileName(wal_dir_, number), &size_bytes); + + if (!s.ok()) { + return s; + } + + log_file->reset(new LogFileImpl(number, kAliveLogFile, + 0, // SequenceNumber + size_bytes)); + + return Status::OK(); +} + +// the function returns status.ok() and sequence == 0 if the file exists, but is +// empty +Status WalManager::ReadFirstLine(const std::string& fname, + const uint64_t number, + SequenceNumber* sequence) { + struct LogReporter : public log::Reader::Reporter { + Env* env; + Logger* info_log; + const char* fname; + + Status* status; + bool ignore_error; // true if db_options_.paranoid_checks==false + void Corruption(size_t bytes, const Status& s) override { + ROCKS_LOG_WARN(info_log, "[WalManager] %s%s: dropping %d bytes; %s", + (this->ignore_error ? "(ignoring error) " : ""), fname, + static_cast(bytes), s.ToString().c_str()); + if (this->status->ok()) { + // only keep the first error + *this->status = s; + } + } + }; + + std::unique_ptr file; + Status status = fs_->NewSequentialFile( + fname, fs_->OptimizeForLogRead(file_options_), &file, nullptr); + std::unique_ptr file_reader( + new SequentialFileReader(std::move(file), fname, io_tracer_)); + + if (!status.ok()) { + return status; + } + + LogReporter reporter; + reporter.env = env_; + reporter.info_log = db_options_.info_log.get(); + reporter.fname = fname.c_str(); + reporter.status = &status; + reporter.ignore_error = !db_options_.paranoid_checks; + log::Reader reader(db_options_.info_log, std::move(file_reader), &reporter, + true /*checksum*/, number); + std::string scratch; + Slice record; + + if (reader.ReadRecord(&record, &scratch) && + (status.ok() || !db_options_.paranoid_checks)) { + if (record.size() < WriteBatchInternal::kHeader) { + reporter.Corruption(record.size(), + Status::Corruption("log record too small")); + // TODO read record's till the first no corrupt entry? + } else { + WriteBatch batch; + // We can overwrite an existing non-OK Status since it'd only reach here + // with `paranoid_checks == false`. + status = WriteBatchInternal::SetContents(&batch, record); + if (status.ok()) { + *sequence = WriteBatchInternal::Sequence(&batch); + return status; + } + } + } + + if (status.ok() && reader.IsCompressedAndEmptyFile()) { + // In case of wal_compression, it writes a `kSetCompressionType` record + // which is not associated with any sequence number. As result for an empty + // file, GetSortedWalsOfType() will skip these WALs causing the operations + // to fail. + // Therefore, in order to avoid that failure, it sets sequence_number to 1 + // indicating those WALs should be included. + *sequence = 1; + } else { + // ReadRecord might have returned false on EOF, which means that the log + // file is empty. Or, a failure may have occurred while processing the first + // entry. In any case, return status and set sequence number to 0. + *sequence = 0; + } + return status; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/wal_manager.h b/librocksdb-sys/rocksdb/db/wal_manager.h new file mode 100644 index 0000000..ab79bf0 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/wal_manager.h @@ -0,0 +1,136 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "db/version_set.h" +#include "file/file_util.h" +#include "options/db_options.h" +#include "port/port.h" +#include "rocksdb/env.h" +#include "rocksdb/status.h" +#include "rocksdb/transaction_log.h" +#include "rocksdb/types.h" + +namespace ROCKSDB_NAMESPACE { + + +// WAL manager provides the abstraction for reading the WAL files as a single +// unit. Internally, it opens and reads the files using Reader or Writer +// abstraction. +class WalManager { + public: + WalManager(const ImmutableDBOptions& db_options, + const FileOptions& file_options, + const std::shared_ptr& io_tracer, + const bool seq_per_batch = false) + : db_options_(db_options), + file_options_(file_options), + env_(db_options.env), + fs_(db_options.fs, io_tracer), + purge_wal_files_last_run_(0), + seq_per_batch_(seq_per_batch), + wal_dir_(db_options_.GetWalDir()), + wal_in_db_path_(db_options_.IsWalDirSameAsDBPath()), + io_tracer_(io_tracer) {} + + Status GetSortedWalFiles(VectorLogPtr& files); + + // Allow user to tail transaction log to find all recent changes to the + // database that are newer than `seq_number`. + Status GetUpdatesSince( + SequenceNumber seq_number, std::unique_ptr* iter, + const TransactionLogIterator::ReadOptions& read_options, + VersionSet* version_set); + + void PurgeObsoleteWALFiles(); + + void ArchiveWALFile(const std::string& fname, uint64_t number); + + Status DeleteFile(const std::string& fname, uint64_t number); + + Status GetLiveWalFile(uint64_t number, std::unique_ptr* log_file); + + Status TEST_ReadFirstRecord(const WalFileType type, const uint64_t number, + SequenceNumber* sequence) { + return ReadFirstRecord(type, number, sequence); + } + + Status TEST_ReadFirstLine(const std::string& fname, const uint64_t number, + SequenceNumber* sequence) { + return ReadFirstLine(fname, number, sequence); + } + + private: + Status GetSortedWalsOfType(const std::string& path, VectorLogPtr& log_files, + WalFileType type); + // Requires: all_logs should be sorted with earliest log file first + // Retains all log files in all_logs which contain updates with seq no. + // Greater Than or Equal to the requested SequenceNumber. + Status RetainProbableWalFiles(VectorLogPtr& all_logs, + const SequenceNumber target); + + // ReadFirstRecord checks the read_first_record_cache_ to see if the entry + // exists or not. If not, it will read the WAL file. + // In case of wal_compression, WAL contains a `kSetCompressionType` record + // which is not associated with any sequence number. So the sequence_number is + // set to 1 if that WAL doesn't include any other record (basically empty) in + // order to include that WAL and is inserted in read_first_record_cache_. + // Therefore, sequence_number is used as boolean if WAL should be included or + // not and that sequence_number shouldn't be use for any other purpose. + Status ReadFirstRecord(const WalFileType type, const uint64_t number, + SequenceNumber* sequence); + + // In case of no wal_compression, ReadFirstLine returns status.ok() and + // sequence == 0 if the file exists, but is empty. + // In case of wal_compression, WAL contains + // `kSetCompressionType` record which is not associated with any sequence + // number if that WAL doesn't include any other record (basically empty). As + // result for an empty file, GetSortedWalsOfType() will skip these WALs + // causing the operations to fail. To avoid that, it sets sequence_number to + // 1 inorder to include that WAL. + Status ReadFirstLine(const std::string& fname, const uint64_t number, + SequenceNumber* sequence); + + // ------- state from DBImpl ------ + const ImmutableDBOptions& db_options_; + const FileOptions file_options_; + Env* env_; + const FileSystemPtr fs_; + + // ------- WalManager state ------- + // cache for ReadFirstRecord() calls + std::unordered_map read_first_record_cache_; + port::Mutex read_first_record_cache_mutex_; + + // last time when PurgeObsoleteWALFiles ran. + uint64_t purge_wal_files_last_run_; + + bool seq_per_batch_; + + const std::string& wal_dir_; + + bool wal_in_db_path_; + + // obsolete files will be deleted every this seconds if ttl deletion is + // enabled and archive size_limit is disabled. + static constexpr uint64_t kDefaultIntervalToDeleteObsoleteWAL = 600; + + std::shared_ptr io_tracer_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/wal_manager_test.cc b/librocksdb-sys/rocksdb/db/wal_manager_test.cc new file mode 100644 index 0000000..0144e18 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/wal_manager_test.cc @@ -0,0 +1,336 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + + +#include "db/wal_manager.h" + +#include +#include + +#include "db/column_family.h" +#include "db/db_impl/db_impl.h" +#include "db/log_writer.h" +#include "db/version_set.h" +#include "env/mock_env.h" +#include "file/writable_file_writer.h" +#include "rocksdb/cache.h" +#include "rocksdb/file_system.h" +#include "rocksdb/write_batch.h" +#include "rocksdb/write_buffer_manager.h" +#include "table/mock_table.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +// TODO(icanadi) mock out VersionSet +// TODO(icanadi) move other WalManager-specific tests from db_test here +class WalManagerTest : public testing::Test { + public: + WalManagerTest() + : dbname_(test::PerThreadDBPath("wal_manager_test")), + db_options_(), + table_cache_(NewLRUCache(50000, 16)), + write_buffer_manager_(db_options_.db_write_buffer_size), + current_log_number_(0) { + env_.reset(MockEnv::Create(Env::Default())); + EXPECT_OK(DestroyDB(dbname_, Options())); + } + + void Init() { + ASSERT_OK(env_->CreateDirIfMissing(dbname_)); + ASSERT_OK(env_->CreateDirIfMissing(ArchivalDirectory(dbname_))); + db_options_.db_paths.emplace_back(dbname_, + std::numeric_limits::max()); + db_options_.wal_dir = dbname_; + db_options_.env = env_.get(); + db_options_.fs = env_->GetFileSystem(); + db_options_.clock = env_->GetSystemClock().get(); + + versions_.reset( + new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), + &write_buffer_manager_, &write_controller_, + /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, + /*db_id*/ "", /*db_session_id*/ "")); + + wal_manager_.reset( + new WalManager(db_options_, env_options_, nullptr /*IOTracer*/)); + } + + void Reopen() { + wal_manager_.reset( + new WalManager(db_options_, env_options_, nullptr /*IOTracer*/)); + } + + // NOT thread safe + void Put(const std::string& key, const std::string& value) { + assert(current_log_writer_.get() != nullptr); + uint64_t seq = versions_->LastSequence() + 1; + WriteBatch batch; + ASSERT_OK(batch.Put(key, value)); + WriteBatchInternal::SetSequence(&batch, seq); + ASSERT_OK( + current_log_writer_->AddRecord(WriteBatchInternal::Contents(&batch))); + versions_->SetLastAllocatedSequence(seq); + versions_->SetLastPublishedSequence(seq); + versions_->SetLastSequence(seq); + } + + // NOT thread safe + void RollTheLog(bool /*archived*/) { + current_log_number_++; + std::string fname = ArchivedLogFileName(dbname_, current_log_number_); + const auto& fs = env_->GetFileSystem(); + std::unique_ptr file_writer; + ASSERT_OK(WritableFileWriter::Create(fs, fname, env_options_, &file_writer, + nullptr)); + current_log_writer_.reset( + new log::Writer(std::move(file_writer), 0, false)); + } + + void CreateArchiveLogs(int num_logs, int entries_per_log) { + for (int i = 1; i <= num_logs; ++i) { + RollTheLog(true); + for (int k = 0; k < entries_per_log; ++k) { + Put(std::to_string(k), std::string(1024, 'a')); + } + } + } + + std::unique_ptr OpenTransactionLogIter( + const SequenceNumber seq) { + std::unique_ptr iter; + Status status = wal_manager_->GetUpdatesSince( + seq, &iter, TransactionLogIterator::ReadOptions(), versions_.get()); + EXPECT_OK(status); + return iter; + } + + std::unique_ptr env_; + std::string dbname_; + ImmutableDBOptions db_options_; + WriteController write_controller_; + EnvOptions env_options_; + std::shared_ptr table_cache_; + WriteBufferManager write_buffer_manager_; + std::unique_ptr versions_; + std::unique_ptr wal_manager_; + + std::unique_ptr current_log_writer_; + uint64_t current_log_number_; +}; + +TEST_F(WalManagerTest, ReadFirstRecordCache) { + Init(); + std::string path = dbname_ + "/000001.log"; + std::unique_ptr file; + ASSERT_OK(env_->GetFileSystem()->NewWritableFile(path, FileOptions(), &file, + nullptr)); + + SequenceNumber s; + ASSERT_OK(wal_manager_->TEST_ReadFirstLine(path, 1 /* number */, &s)); + ASSERT_EQ(s, 0U); + + ASSERT_OK( + wal_manager_->TEST_ReadFirstRecord(kAliveLogFile, 1 /* number */, &s)); + ASSERT_EQ(s, 0U); + + std::unique_ptr file_writer( + new WritableFileWriter(std::move(file), path, FileOptions())); + log::Writer writer(std::move(file_writer), 1, + db_options_.recycle_log_file_num > 0); + WriteBatch batch; + ASSERT_OK(batch.Put("foo", "bar")); + WriteBatchInternal::SetSequence(&batch, 10); + ASSERT_OK(writer.AddRecord(WriteBatchInternal::Contents(&batch))); + + // TODO(icanadi) move SpecialEnv outside of db_test, so we can reuse it here. + // Waiting for lei to finish with db_test + // env_->count_sequential_reads_ = true; + // sequential_read_counter_ sanity test + // ASSERT_EQ(env_->sequential_read_counter_.Read(), 0); + + ASSERT_OK(wal_manager_->TEST_ReadFirstRecord(kAliveLogFile, 1, &s)); + ASSERT_EQ(s, 10U); + // did a read + // TODO(icanadi) move SpecialEnv outside of db_test, so we can reuse it here + // ASSERT_EQ(env_->sequential_read_counter_.Read(), 1); + + ASSERT_OK(wal_manager_->TEST_ReadFirstRecord(kAliveLogFile, 1, &s)); + ASSERT_EQ(s, 10U); + // no new reads since the value is cached + // TODO(icanadi) move SpecialEnv outside of db_test, so we can reuse it here + // ASSERT_EQ(env_->sequential_read_counter_.Read(), 1); +} + +namespace { +uint64_t GetLogDirSize(std::string dir_path, Env* env) { + uint64_t dir_size = 0; + std::vector files; + EXPECT_OK(env->GetChildren(dir_path, &files)); + for (auto& f : files) { + uint64_t number; + FileType type; + if (ParseFileName(f, &number, &type) && type == kWalFile) { + std::string const file_path = dir_path + "/" + f; + uint64_t file_size; + EXPECT_OK(env->GetFileSize(file_path, &file_size)); + dir_size += file_size; + } + } + return dir_size; +} +std::vector ListSpecificFiles( + Env* env, const std::string& path, const FileType expected_file_type) { + std::vector files; + std::vector file_numbers; + uint64_t number; + FileType type; + EXPECT_OK(env->GetChildren(path, &files)); + for (size_t i = 0; i < files.size(); ++i) { + if (ParseFileName(files[i], &number, &type)) { + if (type == expected_file_type) { + file_numbers.push_back(number); + } + } + } + return file_numbers; +} + +int CountRecords(TransactionLogIterator* iter) { + int count = 0; + SequenceNumber lastSequence = 0; + BatchResult res; + while (iter->Valid()) { + res = iter->GetBatch(); + EXPECT_TRUE(res.sequence > lastSequence); + ++count; + lastSequence = res.sequence; + EXPECT_OK(iter->status()); + iter->Next(); + } + EXPECT_OK(iter->status()); + return count; +} +} // anonymous namespace + +TEST_F(WalManagerTest, WALArchivalSizeLimit) { + db_options_.WAL_ttl_seconds = 0; + db_options_.WAL_size_limit_MB = 1000; + Init(); + + // TEST : Create WalManager with huge size limit and no ttl. + // Create some archived files and call PurgeObsoleteWALFiles(). + // Count the archived log files that survived. + // Assert that all of them did. + // Change size limit. Re-open WalManager. + // Assert that archive is not greater than WAL_size_limit_MB after + // PurgeObsoleteWALFiles() + // Set ttl and time_to_check_ to small values. Re-open db. + // Assert that there are no archived logs left. + + std::string archive_dir = ArchivalDirectory(dbname_); + CreateArchiveLogs(20, 5000); + + std::vector log_files = + ListSpecificFiles(env_.get(), archive_dir, kWalFile); + ASSERT_EQ(log_files.size(), 20U); + + db_options_.WAL_size_limit_MB = 8; + Reopen(); + wal_manager_->PurgeObsoleteWALFiles(); + + uint64_t archive_size = GetLogDirSize(archive_dir, env_.get()); + ASSERT_TRUE(archive_size <= db_options_.WAL_size_limit_MB * 1024 * 1024); + + db_options_.WAL_ttl_seconds = 1; + env_->SleepForMicroseconds(2 * 1000 * 1000); + Reopen(); + wal_manager_->PurgeObsoleteWALFiles(); + + log_files = ListSpecificFiles(env_.get(), archive_dir, kWalFile); + ASSERT_TRUE(log_files.empty()); +} + +TEST_F(WalManagerTest, WALArchivalTtl) { + db_options_.WAL_ttl_seconds = 1000; + Init(); + + // TEST : Create WalManager with a ttl and no size limit. + // Create some archived log files and call PurgeObsoleteWALFiles(). + // Assert that files are not deleted + // Reopen db with small ttl. + // Assert that all archived logs was removed. + + std::string archive_dir = ArchivalDirectory(dbname_); + CreateArchiveLogs(20, 5000); + + std::vector log_files = + ListSpecificFiles(env_.get(), archive_dir, kWalFile); + ASSERT_GT(log_files.size(), 0U); + + db_options_.WAL_ttl_seconds = 1; + env_->SleepForMicroseconds(3 * 1000 * 1000); + Reopen(); + wal_manager_->PurgeObsoleteWALFiles(); + + log_files = ListSpecificFiles(env_.get(), archive_dir, kWalFile); + ASSERT_TRUE(log_files.empty()); +} + +TEST_F(WalManagerTest, TransactionLogIteratorMoveOverZeroFiles) { + Init(); + RollTheLog(false); + Put("key1", std::string(1024, 'a')); + // Create a zero record WAL file. + RollTheLog(false); + RollTheLog(false); + + Put("key2", std::string(1024, 'a')); + + auto iter = OpenTransactionLogIter(0); + ASSERT_EQ(2, CountRecords(iter.get())); +} + +TEST_F(WalManagerTest, TransactionLogIteratorJustEmptyFile) { + Init(); + RollTheLog(false); + auto iter = OpenTransactionLogIter(0); + // Check that an empty iterator is returned + ASSERT_TRUE(!iter->Valid()); +} + +TEST_F(WalManagerTest, TransactionLogIteratorNewFileWhileScanning) { + Init(); + CreateArchiveLogs(2, 100); + auto iter = OpenTransactionLogIter(0); + CreateArchiveLogs(1, 100); + int i = 0; + for (; iter->Valid(); iter->Next()) { + i++; + } + ASSERT_EQ(i, 200); + // A new log file was added after the iterator was created. + // TryAgain indicates a new iterator is needed to fetch the new data + ASSERT_TRUE(iter->status().IsTryAgain()); + + iter = OpenTransactionLogIter(0); + i = 0; + for (; iter->Valid(); iter->Next()) { + i++; + } + ASSERT_EQ(i, 300); + ASSERT_TRUE(iter->status().ok()); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/librocksdb-sys/rocksdb/db/wide/db_wide_basic_test.cc b/librocksdb-sys/rocksdb/db/wide/db_wide_basic_test.cc new file mode 100644 index 0000000..536a543 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/wide/db_wide_basic_test.cc @@ -0,0 +1,1042 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include +#include +#include + +#include "db/db_test_util.h" +#include "port/stack_trace.h" +#include "test_util/testutil.h" +#include "utilities/merge_operators.h" + +namespace ROCKSDB_NAMESPACE { + +class DBWideBasicTest : public DBTestBase { + protected: + explicit DBWideBasicTest() + : DBTestBase("db_wide_basic_test", /* env_do_fsync */ false) {} +}; + +TEST_F(DBWideBasicTest, PutEntity) { + Options options = GetDefaultOptions(); + + // Write a couple of wide-column entities and a plain old key-value, then read + // them back. + constexpr char first_key[] = "first"; + constexpr char first_value_of_default_column[] = "hello"; + WideColumns first_columns{ + {kDefaultWideColumnName, first_value_of_default_column}, + {"attr_name1", "foo"}, + {"attr_name2", "bar"}}; + + constexpr char second_key[] = "second"; + WideColumns second_columns{{"attr_one", "two"}, {"attr_three", "four"}}; + + constexpr char third_key[] = "third"; + constexpr char third_value[] = "baz"; + + auto verify = [&]() { + const WideColumns expected_third_columns{ + {kDefaultWideColumnName, third_value}}; + + { + PinnableSlice result; + ASSERT_OK(db_->Get(ReadOptions(), db_->DefaultColumnFamily(), first_key, + &result)); + ASSERT_EQ(result, first_value_of_default_column); + } + + { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + first_key, &result)); + ASSERT_EQ(result.columns(), first_columns); + } + + { + PinnableSlice result; + ASSERT_OK(db_->Get(ReadOptions(), db_->DefaultColumnFamily(), second_key, + &result)); + ASSERT_TRUE(result.empty()); + } + + { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + second_key, &result)); + ASSERT_EQ(result.columns(), second_columns); + } + + { + PinnableSlice result; + ASSERT_OK(db_->Get(ReadOptions(), db_->DefaultColumnFamily(), third_key, + &result)); + ASSERT_EQ(result, third_value); + } + + { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + third_key, &result)); + + ASSERT_EQ(result.columns(), expected_third_columns); + } + + { + constexpr size_t num_keys = 3; + + std::array keys{{first_key, second_key, third_key}}; + std::array values; + std::array statuses; + + db_->MultiGet(ReadOptions(), db_->DefaultColumnFamily(), num_keys, + &keys[0], &values[0], &statuses[0]); + + ASSERT_OK(statuses[0]); + ASSERT_EQ(values[0], first_value_of_default_column); + + ASSERT_OK(statuses[1]); + ASSERT_TRUE(values[1].empty()); + + ASSERT_OK(statuses[2]); + ASSERT_EQ(values[2], third_value); + } + + { + constexpr size_t num_keys = 3; + + std::array keys{{first_key, second_key, third_key}}; + std::array results; + std::array statuses; + + db_->MultiGetEntity(ReadOptions(), db_->DefaultColumnFamily(), num_keys, + &keys[0], &results[0], &statuses[0]); + + ASSERT_OK(statuses[0]); + ASSERT_EQ(results[0].columns(), first_columns); + + ASSERT_OK(statuses[1]); + ASSERT_EQ(results[1].columns(), second_columns); + + ASSERT_OK(statuses[2]); + ASSERT_EQ(results[2].columns(), expected_third_columns); + } + + { + std::unique_ptr iter(db_->NewIterator(ReadOptions())); + + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), first_key); + ASSERT_EQ(iter->value(), first_value_of_default_column); + ASSERT_EQ(iter->columns(), first_columns); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), second_key); + ASSERT_TRUE(iter->value().empty()); + ASSERT_EQ(iter->columns(), second_columns); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), third_key); + ASSERT_EQ(iter->value(), third_value); + ASSERT_EQ(iter->columns(), expected_third_columns); + + iter->Next(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + + iter->SeekToLast(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), third_key); + ASSERT_EQ(iter->value(), third_value); + ASSERT_EQ(iter->columns(), expected_third_columns); + + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), second_key); + ASSERT_TRUE(iter->value().empty()); + ASSERT_EQ(iter->columns(), second_columns); + + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), first_key); + ASSERT_EQ(iter->value(), first_value_of_default_column); + ASSERT_EQ(iter->columns(), first_columns); + + iter->Prev(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + } + }; + + // Use the DB::PutEntity API to write the first entity + ASSERT_OK(db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), + first_key, first_columns)); + + // Use WriteBatch to write the second entity + WriteBatch batch; + ASSERT_OK( + batch.PutEntity(db_->DefaultColumnFamily(), second_key, second_columns)); + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + + // Use Put to write the plain key-value + ASSERT_OK(db_->Put(WriteOptions(), db_->DefaultColumnFamily(), third_key, + third_value)); + + // Try reading from memtable + verify(); + + // Try reading after recovery + Close(); + options.avoid_flush_during_recovery = true; + Reopen(options); + + verify(); + + // Try reading from storage + ASSERT_OK(Flush()); + + verify(); +} + +TEST_F(DBWideBasicTest, PutEntityColumnFamily) { + Options options = GetDefaultOptions(); + CreateAndReopenWithCF({"corinthian"}, options); + + // Use the DB::PutEntity API + constexpr char first_key[] = "first"; + WideColumns first_columns{{"attr_name1", "foo"}, {"attr_name2", "bar"}}; + + ASSERT_OK( + db_->PutEntity(WriteOptions(), handles_[1], first_key, first_columns)); + + // Use WriteBatch + constexpr char second_key[] = "second"; + WideColumns second_columns{{"attr_one", "two"}, {"attr_three", "four"}}; + + WriteBatch batch; + ASSERT_OK(batch.PutEntity(handles_[1], second_key, second_columns)); + ASSERT_OK(db_->Write(WriteOptions(), &batch)); +} + +TEST_F(DBWideBasicTest, MultiCFMultiGetEntity) { + Options options = GetDefaultOptions(); + CreateAndReopenWithCF({"corinthian"}, options); + + constexpr char first_key[] = "first"; + WideColumns first_columns{{"attr_name1", "foo"}, {"attr_name2", "bar"}}; + + ASSERT_OK(db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), + first_key, first_columns)); + + constexpr char second_key[] = "second"; + WideColumns second_columns{{"attr_one", "two"}, {"attr_three", "four"}}; + + ASSERT_OK( + db_->PutEntity(WriteOptions(), handles_[1], second_key, second_columns)); + + constexpr size_t num_keys = 2; + + std::array column_families{ + {db_->DefaultColumnFamily(), handles_[1]}}; + std::array keys{{first_key, second_key}}; + std::array results; + std::array statuses; + + db_->MultiGetEntity(ReadOptions(), num_keys, &column_families[0], &keys[0], + &results[0], &statuses[0]); + + ASSERT_OK(statuses[0]); + ASSERT_EQ(results[0].columns(), first_columns); + + ASSERT_OK(statuses[1]); + ASSERT_EQ(results[1].columns(), second_columns); +} + +TEST_F(DBWideBasicTest, MergePlainKeyValue) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + Reopen(options); + + // Put + Merge + constexpr char first_key[] = "first"; + constexpr char first_base_value[] = "hello"; + constexpr char first_merge_op[] = "world"; + + // Delete + Merge + constexpr char second_key[] = "second"; + constexpr char second_merge_op[] = "foo"; + + // Merge without any preceding KV + constexpr char third_key[] = "third"; + constexpr char third_merge_op[] = "bar"; + + auto write_base = [&]() { + // Write "base" KVs: a Put for the 1st key and a Delete for the 2nd one; + // note there is no "base" KV for the 3rd + ASSERT_OK(db_->Put(WriteOptions(), db_->DefaultColumnFamily(), first_key, + first_base_value)); + ASSERT_OK( + db_->Delete(WriteOptions(), db_->DefaultColumnFamily(), second_key)); + }; + + auto write_merge = [&]() { + // Write Merge operands + ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), first_key, + first_merge_op)); + ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), second_key, + second_merge_op)); + ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), third_key, + third_merge_op)); + }; + + const std::string expected_first_column(std::string(first_base_value) + "," + + first_merge_op); + const WideColumns expected_first_columns{ + {kDefaultWideColumnName, expected_first_column}}; + const WideColumns expected_second_columns{ + {kDefaultWideColumnName, second_merge_op}}; + const WideColumns expected_third_columns{ + {kDefaultWideColumnName, third_merge_op}}; + + auto verify = [&]() { + { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + first_key, &result)); + ASSERT_EQ(result.columns(), expected_first_columns); + } + + { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + second_key, &result)); + ASSERT_EQ(result.columns(), expected_second_columns); + } + + { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + third_key, &result)); + + ASSERT_EQ(result.columns(), expected_third_columns); + } + + { + constexpr size_t num_keys = 3; + + std::array keys{{first_key, second_key, third_key}}; + std::array results; + std::array statuses; + + db_->MultiGetEntity(ReadOptions(), db_->DefaultColumnFamily(), num_keys, + &keys[0], &results[0], &statuses[0]); + + ASSERT_OK(statuses[0]); + ASSERT_EQ(results[0].columns(), expected_first_columns); + + ASSERT_OK(statuses[1]); + ASSERT_EQ(results[1].columns(), expected_second_columns); + + ASSERT_OK(statuses[2]); + ASSERT_EQ(results[2].columns(), expected_third_columns); + } + + { + std::unique_ptr iter(db_->NewIterator(ReadOptions())); + + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), first_key); + ASSERT_EQ(iter->value(), expected_first_columns[0].value()); + ASSERT_EQ(iter->columns(), expected_first_columns); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), second_key); + ASSERT_EQ(iter->value(), expected_second_columns[0].value()); + ASSERT_EQ(iter->columns(), expected_second_columns); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), third_key); + ASSERT_EQ(iter->value(), expected_third_columns[0].value()); + ASSERT_EQ(iter->columns(), expected_third_columns); + + iter->Next(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + + iter->SeekToLast(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), third_key); + ASSERT_EQ(iter->value(), expected_third_columns[0].value()); + ASSERT_EQ(iter->columns(), expected_third_columns); + + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), second_key); + ASSERT_EQ(iter->value(), expected_second_columns[0].value()); + ASSERT_EQ(iter->columns(), expected_second_columns); + + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), first_key); + ASSERT_EQ(iter->value(), expected_first_columns[0].value()); + ASSERT_EQ(iter->columns(), expected_first_columns); + + iter->Prev(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + } + }; + + { + // Base KVs (if any) and Merge operands both in memtable (note: we take a + // snapshot in between to make sure they do not get reconciled during the + // subsequent flush) + write_base(); + ManagedSnapshot snapshot(db_); + write_merge(); + verify(); + + // Base KVs (if any) and Merge operands both in storage + ASSERT_OK(Flush()); + verify(); + } + + // Base KVs (if any) in storage, Merge operands in memtable + DestroyAndReopen(options); + write_base(); + ASSERT_OK(Flush()); + write_merge(); + verify(); +} + +TEST_F(DBWideBasicTest, MergeEntity) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + + const std::string delim("|"); + options.merge_operator = MergeOperators::CreateStringAppendOperator(delim); + + Reopen(options); + + // Test Merge with two entities: one that has the default column and one that + // doesn't + constexpr char first_key[] = "first"; + WideColumns first_columns{{kDefaultWideColumnName, "a"}, + {"attr_name1", "foo"}, + {"attr_name2", "bar"}}; + constexpr char first_merge_operand[] = "bla1"; + + constexpr char second_key[] = "second"; + WideColumns second_columns{{"attr_one", "two"}, {"attr_three", "four"}}; + constexpr char second_merge_operand[] = "bla2"; + + auto write_base = [&]() { + // Use the DB::PutEntity API + ASSERT_OK(db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), + first_key, first_columns)); + + // Use WriteBatch + WriteBatch batch; + ASSERT_OK(batch.PutEntity(db_->DefaultColumnFamily(), second_key, + second_columns)); + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + }; + + auto write_merge = [&]() { + ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), first_key, + first_merge_operand)); + ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), second_key, + second_merge_operand)); + }; + + const std::string first_expected_default(first_columns[0].value().ToString() + + delim + first_merge_operand); + const std::string second_expected_default(delim + second_merge_operand); + + auto verify_basic = [&]() { + WideColumns first_expected_columns{ + {kDefaultWideColumnName, first_expected_default}, + first_columns[1], + first_columns[2]}; + + WideColumns second_expected_columns{ + {kDefaultWideColumnName, second_expected_default}, + second_columns[0], + second_columns[1]}; + + { + PinnableSlice result; + ASSERT_OK(db_->Get(ReadOptions(), db_->DefaultColumnFamily(), first_key, + &result)); + ASSERT_EQ(result, first_expected_default); + } + + { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + first_key, &result)); + ASSERT_EQ(result.columns(), first_expected_columns); + } + + { + PinnableSlice result; + ASSERT_OK(db_->Get(ReadOptions(), db_->DefaultColumnFamily(), second_key, + &result)); + ASSERT_EQ(result, second_expected_default); + } + + { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + second_key, &result)); + ASSERT_EQ(result.columns(), second_expected_columns); + } + + { + constexpr size_t num_keys = 2; + + std::array keys{{first_key, second_key}}; + std::array values; + std::array statuses; + + db_->MultiGet(ReadOptions(), db_->DefaultColumnFamily(), num_keys, + &keys[0], &values[0], &statuses[0]); + + ASSERT_EQ(values[0], first_expected_default); + ASSERT_OK(statuses[0]); + + ASSERT_EQ(values[1], second_expected_default); + ASSERT_OK(statuses[1]); + } + + { + constexpr size_t num_keys = 2; + + std::array keys{{first_key, second_key}}; + std::array results; + std::array statuses; + + db_->MultiGetEntity(ReadOptions(), db_->DefaultColumnFamily(), num_keys, + &keys[0], &results[0], &statuses[0]); + + ASSERT_OK(statuses[0]); + ASSERT_EQ(results[0].columns(), first_expected_columns); + + ASSERT_OK(statuses[1]); + ASSERT_EQ(results[1].columns(), second_expected_columns); + } + + { + std::unique_ptr iter(db_->NewIterator(ReadOptions())); + + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), first_key); + ASSERT_EQ(iter->value(), first_expected_default); + ASSERT_EQ(iter->columns(), first_expected_columns); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), second_key); + ASSERT_EQ(iter->value(), second_expected_default); + ASSERT_EQ(iter->columns(), second_expected_columns); + + iter->Next(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + + iter->SeekToLast(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), second_key); + ASSERT_EQ(iter->value(), second_expected_default); + ASSERT_EQ(iter->columns(), second_expected_columns); + + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_EQ(iter->key(), first_key); + ASSERT_EQ(iter->value(), first_expected_default); + ASSERT_EQ(iter->columns(), first_expected_columns); + + iter->Prev(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + } + }; + + auto verify_merge_ops_pre_compaction = [&]() { + constexpr size_t num_merge_operands = 2; + + GetMergeOperandsOptions get_merge_opts; + get_merge_opts.expected_max_number_of_operands = num_merge_operands; + + { + std::array merge_operands; + int number_of_operands = 0; + + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + first_key, &merge_operands[0], + &get_merge_opts, &number_of_operands)); + + ASSERT_EQ(number_of_operands, num_merge_operands); + ASSERT_EQ(merge_operands[0], first_columns[0].value()); + ASSERT_EQ(merge_operands[1], first_merge_operand); + } + + { + std::array merge_operands; + int number_of_operands = 0; + + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + second_key, &merge_operands[0], + &get_merge_opts, &number_of_operands)); + + ASSERT_EQ(number_of_operands, num_merge_operands); + ASSERT_TRUE(merge_operands[0].empty()); + ASSERT_EQ(merge_operands[1], second_merge_operand); + } + }; + + auto verify_merge_ops_post_compaction = [&]() { + constexpr size_t num_merge_operands = 1; + + GetMergeOperandsOptions get_merge_opts; + get_merge_opts.expected_max_number_of_operands = num_merge_operands; + + { + std::array merge_operands; + int number_of_operands = 0; + + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + first_key, &merge_operands[0], + &get_merge_opts, &number_of_operands)); + + ASSERT_EQ(number_of_operands, num_merge_operands); + ASSERT_EQ(merge_operands[0], first_expected_default); + } + + { + std::array merge_operands; + int number_of_operands = 0; + + ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), + second_key, &merge_operands[0], + &get_merge_opts, &number_of_operands)); + + ASSERT_EQ(number_of_operands, num_merge_operands); + ASSERT_EQ(merge_operands[0], second_expected_default); + } + }; + + { + // Base KVs and Merge operands both in memtable (note: we take a snapshot in + // between to make sure they do not get reconciled during the subsequent + // flush) + write_base(); + ManagedSnapshot snapshot(db_); + write_merge(); + verify_basic(); + verify_merge_ops_pre_compaction(); + + // Base KVs and Merge operands both in storage + ASSERT_OK(Flush()); + verify_basic(); + verify_merge_ops_pre_compaction(); + } + + // Base KVs in storage, Merge operands in memtable + DestroyAndReopen(options); + write_base(); + ASSERT_OK(Flush()); + write_merge(); + verify_basic(); + verify_merge_ops_pre_compaction(); + + // Flush and compact + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), /* begin */ nullptr, + /* end */ nullptr)); + verify_basic(); + verify_merge_ops_post_compaction(); +} + +TEST_F(DBWideBasicTest, CompactionFilter) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + + // Wide-column entity with default column + constexpr char first_key[] = "first"; + WideColumns first_columns{{kDefaultWideColumnName, "a"}, + {"attr_name1", "foo"}, + {"attr_name2", "bar"}}; + WideColumns first_columns_uppercase{{kDefaultWideColumnName, "A"}, + {"attr_name1", "FOO"}, + {"attr_name2", "BAR"}}; + + // Wide-column entity without default column + constexpr char second_key[] = "second"; + WideColumns second_columns{{"attr_one", "two"}, {"attr_three", "four"}}; + WideColumns second_columns_uppercase{{"attr_one", "TWO"}, + {"attr_three", "FOUR"}}; + + // Plain old key-value + constexpr char last_key[] = "last"; + constexpr char last_value[] = "baz"; + constexpr char last_value_uppercase[] = "BAZ"; + + auto write = [&] { + ASSERT_OK(db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), + first_key, first_columns)); + ASSERT_OK(db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), + second_key, second_columns)); + + ASSERT_OK(Flush()); + + ASSERT_OK(db_->Put(WriteOptions(), db_->DefaultColumnFamily(), last_key, + last_value)); + + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), /* begin */ nullptr, + /* end */ nullptr)); + }; + + // Test a compaction filter that keeps all entries + { + class KeepFilter : public CompactionFilter { + public: + Decision FilterV3( + int /* level */, const Slice& /* key */, ValueType /* value_type */, + const Slice* /* existing_value */, + const WideColumns* /* existing_columns */, + std::string* /* new_value */, + std::vector>* /* new_columns */, + std::string* /* skip_until */) const override { + return Decision::kKeep; + } + + const char* Name() const override { return "KeepFilter"; } + }; + + KeepFilter filter; + options.compaction_filter = &filter; + + DestroyAndReopen(options); + + write(); + + { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + first_key, &result)); + ASSERT_EQ(result.columns(), first_columns); + } + + { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + second_key, &result)); + ASSERT_EQ(result.columns(), second_columns); + } + + // Note: GetEntity should return an entity with a single default column, + // since last_key is a plain key-value + { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + last_key, &result)); + + WideColumns expected_columns{{kDefaultWideColumnName, last_value}}; + ASSERT_EQ(result.columns(), expected_columns); + } + } + + // Test a compaction filter that removes all entries + { + class RemoveFilter : public CompactionFilter { + public: + Decision FilterV3( + int /* level */, const Slice& /* key */, ValueType /* value_type */, + const Slice* /* existing_value */, + const WideColumns* /* existing_columns */, + std::string* /* new_value */, + std::vector>* /* new_columns */, + std::string* /* skip_until */) const override { + return Decision::kRemove; + } + + const char* Name() const override { return "RemoveFilter"; } + }; + + RemoveFilter filter; + options.compaction_filter = &filter; + + DestroyAndReopen(options); + + write(); + + { + PinnableWideColumns result; + ASSERT_TRUE(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + first_key, &result) + .IsNotFound()); + } + + { + PinnableWideColumns result; + ASSERT_TRUE(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + second_key, &result) + .IsNotFound()); + } + + { + PinnableWideColumns result; + ASSERT_TRUE(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + last_key, &result) + .IsNotFound()); + } + } + + // Test a compaction filter that changes the values of entries to uppercase. + // The new entry is always a plain key-value; if the existing entry is a + // wide-column entity, only the value of its first column is kept. + { + class ChangeValueFilter : public CompactionFilter { + public: + Decision FilterV3( + int /* level */, const Slice& /* key */, ValueType value_type, + const Slice* existing_value, const WideColumns* existing_columns, + std::string* new_value, + std::vector>* /* new_columns */, + std::string* /* skip_until */) const override { + assert(new_value); + + auto upper = [](const std::string& str) { + std::string result(str); + + for (char& c : result) { + c = static_cast(std::toupper(static_cast(c))); + } + + return result; + }; + + if (value_type == ValueType::kWideColumnEntity) { + assert(existing_columns); + + if (!existing_columns->empty()) { + *new_value = upper(existing_columns->front().value().ToString()); + } + } else { + assert(existing_value); + + *new_value = upper(existing_value->ToString()); + } + + return Decision::kChangeValue; + } + + const char* Name() const override { return "ChangeValueFilter"; } + }; + + ChangeValueFilter filter; + options.compaction_filter = &filter; + + DestroyAndReopen(options); + + write(); + + // Note: GetEntity should return entities with a single default column, + // since all entries are now plain key-values + { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + first_key, &result)); + + WideColumns expected_columns{ + {kDefaultWideColumnName, first_columns_uppercase[0].value()}}; + ASSERT_EQ(result.columns(), expected_columns); + } + + { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + second_key, &result)); + + WideColumns expected_columns{ + {kDefaultWideColumnName, second_columns_uppercase[0].value()}}; + ASSERT_EQ(result.columns(), expected_columns); + } + + { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + last_key, &result)); + + WideColumns expected_columns{ + {kDefaultWideColumnName, last_value_uppercase}}; + ASSERT_EQ(result.columns(), expected_columns); + } + } + + // Test a compaction filter that changes the column values of entries to + // uppercase. The new entry is always a wide-column entity; if the existing + // entry is a plain key-value, it is converted to a wide-column entity with a + // single default column. + { + class ChangeEntityFilter : public CompactionFilter { + public: + Decision FilterV3( + int /* level */, const Slice& /* key */, ValueType value_type, + const Slice* existing_value, const WideColumns* existing_columns, + std::string* /* new_value */, + std::vector>* new_columns, + std::string* /* skip_until */) const override { + assert(new_columns); + + auto upper = [](const std::string& str) { + std::string result(str); + + for (char& c : result) { + c = static_cast(std::toupper(static_cast(c))); + } + + return result; + }; + + if (value_type == ValueType::kWideColumnEntity) { + assert(existing_columns); + + for (const auto& column : *existing_columns) { + new_columns->emplace_back(column.name().ToString(), + upper(column.value().ToString())); + } + } else { + assert(existing_value); + + new_columns->emplace_back(kDefaultWideColumnName.ToString(), + upper(existing_value->ToString())); + } + + return Decision::kChangeWideColumnEntity; + } + + const char* Name() const override { return "ChangeEntityFilter"; } + }; + + ChangeEntityFilter filter; + options.compaction_filter = &filter; + + DestroyAndReopen(options); + + write(); + + { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + first_key, &result)); + ASSERT_EQ(result.columns(), first_columns_uppercase); + } + + { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + second_key, &result)); + ASSERT_EQ(result.columns(), second_columns_uppercase); + } + + { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + last_key, &result)); + + WideColumns expected_columns{ + {kDefaultWideColumnName, last_value_uppercase}}; + ASSERT_EQ(result.columns(), expected_columns); + } + } +} + +TEST_F(DBWideBasicTest, PutEntityTimestampError) { + // Note: timestamps are currently not supported + + Options options = GetDefaultOptions(); + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + + ColumnFamilyHandle* handle = nullptr; + ASSERT_OK(db_->CreateColumnFamily(options, "corinthian", &handle)); + std::unique_ptr handle_guard(handle); + + // Use the DB::PutEntity API + constexpr char first_key[] = "first"; + WideColumns first_columns{{"attr_name1", "foo"}, {"attr_name2", "bar"}}; + + ASSERT_TRUE(db_->PutEntity(WriteOptions(), handle, first_key, first_columns) + .IsInvalidArgument()); + + // Use WriteBatch + constexpr char second_key[] = "second"; + WideColumns second_columns{{"doric", "column"}, {"ionic", "column"}}; + + WriteBatch batch; + ASSERT_TRUE( + batch.PutEntity(handle, second_key, second_columns).IsInvalidArgument()); + ASSERT_OK(db_->Write(WriteOptions(), &batch)); +} + +TEST_F(DBWideBasicTest, PutEntitySerializationError) { + // Make sure duplicate columns are caught + + Options options = GetDefaultOptions(); + + // Use the DB::PutEntity API + constexpr char first_key[] = "first"; + WideColumns first_columns{{"foo", "bar"}, {"foo", "baz"}}; + + ASSERT_TRUE(db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), + first_key, first_columns) + .IsCorruption()); + + // Use WriteBatch + constexpr char second_key[] = "second"; + WideColumns second_columns{{"column", "doric"}, {"column", "ionic"}}; + + WriteBatch batch; + ASSERT_TRUE( + batch.PutEntity(db_->DefaultColumnFamily(), second_key, second_columns) + .IsCorruption()); + ASSERT_OK(db_->Write(WriteOptions(), &batch)); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/wide/wide_column_serialization.cc b/librocksdb-sys/rocksdb/db/wide/wide_column_serialization.cc new file mode 100644 index 0000000..f62143c --- /dev/null +++ b/librocksdb-sys/rocksdb/db/wide/wide_column_serialization.cc @@ -0,0 +1,182 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/wide/wide_column_serialization.h" + +#include +#include +#include + +#include "rocksdb/slice.h" +#include "util/autovector.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { + +Status WideColumnSerialization::SerializeImpl(const Slice* value_of_default, + const WideColumns& columns, + std::string& output) { + const size_t num_columns = + value_of_default ? columns.size() + 1 : columns.size(); + + if (num_columns > static_cast(std::numeric_limits::max())) { + return Status::InvalidArgument("Too many wide columns"); + } + + PutVarint32(&output, kCurrentVersion); + + PutVarint32(&output, static_cast(num_columns)); + + const Slice* prev_name = nullptr; + if (value_of_default) { + if (value_of_default->size() > + static_cast(std::numeric_limits::max())) { + return Status::InvalidArgument("Wide column value too long"); + } + + PutLengthPrefixedSlice(&output, kDefaultWideColumnName); + PutVarint32(&output, static_cast(value_of_default->size())); + + prev_name = &kDefaultWideColumnName; + } + + for (size_t i = 0; i < columns.size(); ++i) { + const WideColumn& column = columns[i]; + + const Slice& name = column.name(); + if (name.size() > + static_cast(std::numeric_limits::max())) { + return Status::InvalidArgument("Wide column name too long"); + } + + if (prev_name && prev_name->compare(name) >= 0) { + return Status::Corruption("Wide columns out of order"); + } + + const Slice& value = column.value(); + if (value.size() > + static_cast(std::numeric_limits::max())) { + return Status::InvalidArgument("Wide column value too long"); + } + + PutLengthPrefixedSlice(&output, name); + PutVarint32(&output, static_cast(value.size())); + + prev_name = &name; + } + + if (value_of_default) { + output.append(value_of_default->data(), value_of_default->size()); + } + + for (const auto& column : columns) { + const Slice& value = column.value(); + + output.append(value.data(), value.size()); + } + + return Status::OK(); +} + +Status WideColumnSerialization::Deserialize(Slice& input, + WideColumns& columns) { + assert(columns.empty()); + + uint32_t version = 0; + if (!GetVarint32(&input, &version)) { + return Status::Corruption("Error decoding wide column version"); + } + + if (version > kCurrentVersion) { + return Status::NotSupported("Unsupported wide column version"); + } + + uint32_t num_columns = 0; + if (!GetVarint32(&input, &num_columns)) { + return Status::Corruption("Error decoding number of wide columns"); + } + + if (!num_columns) { + return Status::OK(); + } + + columns.reserve(num_columns); + + autovector column_value_sizes; + column_value_sizes.reserve(num_columns); + + for (uint32_t i = 0; i < num_columns; ++i) { + Slice name; + if (!GetLengthPrefixedSlice(&input, &name)) { + return Status::Corruption("Error decoding wide column name"); + } + + if (!columns.empty() && columns.back().name().compare(name) >= 0) { + return Status::Corruption("Wide columns out of order"); + } + + columns.emplace_back(name, Slice()); + + uint32_t value_size = 0; + if (!GetVarint32(&input, &value_size)) { + return Status::Corruption("Error decoding wide column value size"); + } + + column_value_sizes.emplace_back(value_size); + } + + const Slice data(input); + size_t pos = 0; + + for (uint32_t i = 0; i < num_columns; ++i) { + const uint32_t value_size = column_value_sizes[i]; + + if (pos + value_size > data.size()) { + return Status::Corruption("Error decoding wide column value payload"); + } + + columns[i].value() = Slice(data.data() + pos, value_size); + + pos += value_size; + } + + return Status::OK(); +} + +WideColumns::const_iterator WideColumnSerialization::Find( + const WideColumns& columns, const Slice& column_name) { + const auto it = + std::lower_bound(columns.cbegin(), columns.cend(), column_name, + [](const WideColumn& lhs, const Slice& rhs) { + return lhs.name().compare(rhs) < 0; + }); + + if (it == columns.cend() || it->name() != column_name) { + return columns.cend(); + } + + return it; +} + +Status WideColumnSerialization::GetValueOfDefaultColumn(Slice& input, + Slice& value) { + WideColumns columns; + + const Status s = Deserialize(input, columns); + if (!s.ok()) { + return s; + } + + if (columns.empty() || columns[0].name() != kDefaultWideColumnName) { + value.clear(); + return Status::OK(); + } + + value = columns[0].value(); + + return Status::OK(); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/wide/wide_column_serialization.h b/librocksdb-sys/rocksdb/db/wide/wide_column_serialization.h new file mode 100644 index 0000000..f0ffbd3 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/wide/wide_column_serialization.h @@ -0,0 +1,77 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include + +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/status.h" +#include "rocksdb/wide_columns.h" + +namespace ROCKSDB_NAMESPACE { + +class Slice; + +// Wide-column serialization/deserialization primitives. +// +// The two main parts of the layout are 1) a sorted index containing the column +// names and column value sizes and 2) the column values themselves. Keeping the +// index and the values separate will enable selectively reading column values +// down the line. Note that currently the index has to be fully parsed in order +// to find out the offset of each column value. +// +// Legend: cn = column name, cv = column value, cns = column name size, cvs = +// column value size. +// +// +----------+--------------+----------+-------+----------+---... +// | version | # of columns | cns 1 | cn 1 | cvs 1 | +// +----------+--------------+------------------+--------- +---... +// | varint32 | varint32 | varint32 | bytes | varint32 | +// +----------+--------------+----------+-------+----------+---... +// +// ... continued ... +// +// ...---+----------+-------+----------+-------+---...---+-------+ +// | cns N | cn N | cvs N | cv 1 | | cv N | +// ...---+----------+-------+----------+-------+---...---+-------+ +// | varint32 | bytes | varint32 | bytes | | bytes | +// ...---+----------+-------+----------+-------+---...---+-------+ + +class WideColumnSerialization { + public: + static Status Serialize(const WideColumns& columns, std::string& output); + static Status Serialize(const Slice& value_of_default, + const WideColumns& other_columns, + std::string& output); + + static Status Deserialize(Slice& input, WideColumns& columns); + + static WideColumns::const_iterator Find(const WideColumns& columns, + const Slice& column_name); + static Status GetValueOfDefaultColumn(Slice& input, Slice& value); + + static constexpr uint32_t kCurrentVersion = 1; + + private: + static Status SerializeImpl(const Slice* value_of_default, + const WideColumns& columns, std::string& output); +}; + +inline Status WideColumnSerialization::Serialize(const WideColumns& columns, + std::string& output) { + constexpr Slice* value_of_default = nullptr; + + return SerializeImpl(value_of_default, columns, output); +} + +inline Status WideColumnSerialization::Serialize( + const Slice& value_of_default, const WideColumns& other_columns, + std::string& output) { + return SerializeImpl(&value_of_default, other_columns, output); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/wide/wide_column_serialization_test.cc b/librocksdb-sys/rocksdb/db/wide/wide_column_serialization_test.cc new file mode 100644 index 0000000..8060d2f --- /dev/null +++ b/librocksdb-sys/rocksdb/db/wide/wide_column_serialization_test.cc @@ -0,0 +1,338 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/wide/wide_column_serialization.h" + +#include "test_util/testharness.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { + +TEST(WideColumnSerializationTest, Construct) { + constexpr char foo[] = "foo"; + constexpr char bar[] = "bar"; + + const std::string foo_str(foo); + const std::string bar_str(bar); + + const Slice foo_slice(foo_str); + const Slice bar_slice(bar_str); + + { + WideColumn column(foo, bar); + ASSERT_EQ(column.name(), foo); + ASSERT_EQ(column.value(), bar); + } + + { + WideColumn column(foo_str, bar); + ASSERT_EQ(column.name(), foo_str); + ASSERT_EQ(column.value(), bar); + } + + { + WideColumn column(foo_slice, bar); + ASSERT_EQ(column.name(), foo_slice); + ASSERT_EQ(column.value(), bar); + } + + { + WideColumn column(foo, bar_str); + ASSERT_EQ(column.name(), foo); + ASSERT_EQ(column.value(), bar_str); + } + + { + WideColumn column(foo_str, bar_str); + ASSERT_EQ(column.name(), foo_str); + ASSERT_EQ(column.value(), bar_str); + } + + { + WideColumn column(foo_slice, bar_str); + ASSERT_EQ(column.name(), foo_slice); + ASSERT_EQ(column.value(), bar_str); + } + + { + WideColumn column(foo, bar_slice); + ASSERT_EQ(column.name(), foo); + ASSERT_EQ(column.value(), bar_slice); + } + + { + WideColumn column(foo_str, bar_slice); + ASSERT_EQ(column.name(), foo_str); + ASSERT_EQ(column.value(), bar_slice); + } + + { + WideColumn column(foo_slice, bar_slice); + ASSERT_EQ(column.name(), foo_slice); + ASSERT_EQ(column.value(), bar_slice); + } + + { + constexpr char foo_name[] = "foo_name"; + constexpr char bar_value[] = "bar_value"; + + WideColumn column(std::piecewise_construct, + std::forward_as_tuple(foo_name, sizeof(foo) - 1), + std::forward_as_tuple(bar_value, sizeof(bar) - 1)); + ASSERT_EQ(column.name(), foo); + ASSERT_EQ(column.value(), bar); + } +} + +TEST(WideColumnSerializationTest, SerializeDeserialize) { + WideColumns columns{{"foo", "bar"}, {"hello", "world"}}; + std::string output; + + ASSERT_OK(WideColumnSerialization::Serialize(columns, output)); + + Slice input(output); + WideColumns deserialized_columns; + + ASSERT_OK(WideColumnSerialization::Deserialize(input, deserialized_columns)); + ASSERT_EQ(columns, deserialized_columns); + + { + const auto it = WideColumnSerialization::Find(deserialized_columns, "foo"); + ASSERT_NE(it, deserialized_columns.cend()); + ASSERT_EQ(*it, deserialized_columns.front()); + } + + { + const auto it = + WideColumnSerialization::Find(deserialized_columns, "hello"); + ASSERT_NE(it, deserialized_columns.cend()); + ASSERT_EQ(*it, deserialized_columns.back()); + } + + { + const auto it = + WideColumnSerialization::Find(deserialized_columns, "fubar"); + ASSERT_EQ(it, deserialized_columns.cend()); + } + + { + const auto it = + WideColumnSerialization::Find(deserialized_columns, "snafu"); + ASSERT_EQ(it, deserialized_columns.cend()); + } +} + +TEST(WideColumnSerializationTest, SerializeWithPrepend) { + Slice value_of_default("baz"); + WideColumns other_columns{{"foo", "bar"}, {"hello", "world"}}; + + std::string output; + ASSERT_OK(WideColumnSerialization::Serialize(value_of_default, other_columns, + output)); + + Slice input(output); + + WideColumns deserialized_columns; + ASSERT_OK(WideColumnSerialization::Deserialize(input, deserialized_columns)); + + WideColumns expected_columns{{kDefaultWideColumnName, value_of_default}, + other_columns[0], + other_columns[1]}; + ASSERT_EQ(deserialized_columns, expected_columns); +} + +TEST(WideColumnSerializationTest, SerializeDuplicateError) { + WideColumns columns{{"foo", "bar"}, {"foo", "baz"}}; + std::string output; + + ASSERT_TRUE( + WideColumnSerialization::Serialize(columns, output).IsCorruption()); +} + +TEST(WideColumnSerializationTest, SerializeWithPrependDuplicateError) { + Slice value_of_default("baz"); + WideColumns other_columns{{kDefaultWideColumnName, "dup"}, {"foo", "bar"}}; + + std::string output; + ASSERT_TRUE(WideColumnSerialization::Serialize(value_of_default, + other_columns, output) + .IsCorruption()); +} + +TEST(WideColumnSerializationTest, SerializeOutOfOrderError) { + WideColumns columns{{"hello", "world"}, {"foo", "bar"}}; + std::string output; + + ASSERT_TRUE( + WideColumnSerialization::Serialize(columns, output).IsCorruption()); +} + +TEST(WideColumnSerializationTest, DeserializeVersionError) { + // Can't decode version + + std::string buf; + + Slice input(buf); + WideColumns columns; + + const Status s = WideColumnSerialization::Deserialize(input, columns); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "version")); +} + +TEST(WideColumnSerializationTest, DeserializeUnsupportedVersion) { + // Unsupported version + constexpr uint32_t future_version = 1000; + + std::string buf; + PutVarint32(&buf, future_version); + + Slice input(buf); + WideColumns columns; + + const Status s = WideColumnSerialization::Deserialize(input, columns); + ASSERT_TRUE(s.IsNotSupported()); + ASSERT_TRUE(std::strstr(s.getState(), "version")); +} + +TEST(WideColumnSerializationTest, DeserializeNumberOfColumnsError) { + // Can't decode number of columns + + std::string buf; + PutVarint32(&buf, WideColumnSerialization::kCurrentVersion); + + Slice input(buf); + WideColumns columns; + + const Status s = WideColumnSerialization::Deserialize(input, columns); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "number")); +} + +TEST(WideColumnSerializationTest, DeserializeColumnsError) { + std::string buf; + + PutVarint32(&buf, WideColumnSerialization::kCurrentVersion); + + constexpr uint32_t num_columns = 2; + PutVarint32(&buf, num_columns); + + // Can't decode the first column name + { + Slice input(buf); + WideColumns columns; + + const Status s = WideColumnSerialization::Deserialize(input, columns); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "name")); + } + + constexpr char first_column_name[] = "foo"; + PutLengthPrefixedSlice(&buf, first_column_name); + + // Can't decode the size of the first column value + { + Slice input(buf); + WideColumns columns; + + const Status s = WideColumnSerialization::Deserialize(input, columns); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "value size")); + } + + constexpr uint32_t first_value_size = 16; + PutVarint32(&buf, first_value_size); + + // Can't decode the second column name + { + Slice input(buf); + WideColumns columns; + + const Status s = WideColumnSerialization::Deserialize(input, columns); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "name")); + } + + constexpr char second_column_name[] = "hello"; + PutLengthPrefixedSlice(&buf, second_column_name); + + // Can't decode the size of the second column value + { + Slice input(buf); + WideColumns columns; + + const Status s = WideColumnSerialization::Deserialize(input, columns); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "value size")); + } + + constexpr uint32_t second_value_size = 64; + PutVarint32(&buf, second_value_size); + + // Can't decode the payload of the first column + { + Slice input(buf); + WideColumns columns; + + const Status s = WideColumnSerialization::Deserialize(input, columns); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "payload")); + } + + buf.append(first_value_size, '0'); + + // Can't decode the payload of the second column + { + Slice input(buf); + WideColumns columns; + + const Status s = WideColumnSerialization::Deserialize(input, columns); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "payload")); + } + + buf.append(second_value_size, 'x'); + + // Success + { + Slice input(buf); + WideColumns columns; + + ASSERT_OK(WideColumnSerialization::Deserialize(input, columns)); + } +} + +TEST(WideColumnSerializationTest, DeserializeColumnsOutOfOrder) { + std::string buf; + + PutVarint32(&buf, WideColumnSerialization::kCurrentVersion); + + constexpr uint32_t num_columns = 2; + PutVarint32(&buf, num_columns); + + constexpr char first_column_name[] = "b"; + PutLengthPrefixedSlice(&buf, first_column_name); + + constexpr uint32_t first_value_size = 16; + PutVarint32(&buf, first_value_size); + + constexpr char second_column_name[] = "a"; + PutLengthPrefixedSlice(&buf, second_column_name); + + Slice input(buf); + WideColumns columns; + + const Status s = WideColumnSerialization::Deserialize(input, columns); + ASSERT_TRUE(s.IsCorruption()); + ASSERT_TRUE(std::strstr(s.getState(), "order")); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/wide/wide_columns.cc b/librocksdb-sys/rocksdb/db/wide/wide_columns.cc new file mode 100644 index 0000000..186be7f --- /dev/null +++ b/librocksdb-sys/rocksdb/db/wide/wide_columns.cc @@ -0,0 +1,22 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "rocksdb/wide_columns.h" + +#include "db/wide/wide_column_serialization.h" + +namespace ROCKSDB_NAMESPACE { + +const Slice kDefaultWideColumnName; + +const WideColumns kNoWideColumns; + +Status PinnableWideColumns::CreateIndexForWideColumns() { + Slice value_copy = value_; + + return WideColumnSerialization::Deserialize(value_copy, columns_); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/write_batch.cc b/librocksdb-sys/rocksdb/db/write_batch.cc new file mode 100644 index 0000000..47e34e6 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/write_batch.cc @@ -0,0 +1,3148 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// WriteBatch::rep_ := +// sequence: fixed64 +// count: fixed32 +// data: record[count] +// record := +// kTypeValue varstring varstring +// kTypeDeletion varstring +// kTypeSingleDeletion varstring +// kTypeRangeDeletion varstring varstring +// kTypeMerge varstring varstring +// kTypeColumnFamilyValue varint32 varstring varstring +// kTypeColumnFamilyDeletion varint32 varstring +// kTypeColumnFamilySingleDeletion varint32 varstring +// kTypeColumnFamilyRangeDeletion varint32 varstring varstring +// kTypeColumnFamilyMerge varint32 varstring varstring +// kTypeBeginPrepareXID +// kTypeEndPrepareXID varstring +// kTypeCommitXID varstring +// kTypeCommitXIDAndTimestamp varstring varstring +// kTypeRollbackXID varstring +// kTypeBeginPersistedPrepareXID +// kTypeBeginUnprepareXID +// kTypeWideColumnEntity varstring varstring +// kTypeColumnFamilyWideColumnEntity varint32 varstring varstring +// kTypeNoop +// varstring := +// len: varint32 +// data: uint8[len] + +#include "rocksdb/write_batch.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "db/column_family.h" +#include "db/db_impl/db_impl.h" +#include "db/dbformat.h" +#include "db/flush_scheduler.h" +#include "db/kv_checksum.h" +#include "db/memtable.h" +#include "db/merge_context.h" +#include "db/snapshot_impl.h" +#include "db/trim_history_scheduler.h" +#include "db/wide/wide_column_serialization.h" +#include "db/write_batch_internal.h" +#include "monitoring/perf_context_imp.h" +#include "monitoring/statistics_impl.h" +#include "port/lang.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/system_clock.h" +#include "util/autovector.h" +#include "util/cast_util.h" +#include "util/coding.h" +#include "util/duplicate_detector.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +// anon namespace for file-local types +namespace { + +enum ContentFlags : uint32_t { + DEFERRED = 1 << 0, + HAS_PUT = 1 << 1, + HAS_DELETE = 1 << 2, + HAS_SINGLE_DELETE = 1 << 3, + HAS_MERGE = 1 << 4, + HAS_BEGIN_PREPARE = 1 << 5, + HAS_END_PREPARE = 1 << 6, + HAS_COMMIT = 1 << 7, + HAS_ROLLBACK = 1 << 8, + HAS_DELETE_RANGE = 1 << 9, + HAS_BLOB_INDEX = 1 << 10, + HAS_BEGIN_UNPREPARE = 1 << 11, + HAS_PUT_ENTITY = 1 << 12, +}; + +struct BatchContentClassifier : public WriteBatch::Handler { + uint32_t content_flags = 0; + + Status PutCF(uint32_t, const Slice&, const Slice&) override { + content_flags |= ContentFlags::HAS_PUT; + return Status::OK(); + } + + Status PutEntityCF(uint32_t /* column_family_id */, const Slice& /* key */, + const Slice& /* entity */) override { + content_flags |= ContentFlags::HAS_PUT_ENTITY; + return Status::OK(); + } + + Status DeleteCF(uint32_t, const Slice&) override { + content_flags |= ContentFlags::HAS_DELETE; + return Status::OK(); + } + + Status SingleDeleteCF(uint32_t, const Slice&) override { + content_flags |= ContentFlags::HAS_SINGLE_DELETE; + return Status::OK(); + } + + Status DeleteRangeCF(uint32_t, const Slice&, const Slice&) override { + content_flags |= ContentFlags::HAS_DELETE_RANGE; + return Status::OK(); + } + + Status MergeCF(uint32_t, const Slice&, const Slice&) override { + content_flags |= ContentFlags::HAS_MERGE; + return Status::OK(); + } + + Status PutBlobIndexCF(uint32_t, const Slice&, const Slice&) override { + content_flags |= ContentFlags::HAS_BLOB_INDEX; + return Status::OK(); + } + + Status MarkBeginPrepare(bool unprepare) override { + content_flags |= ContentFlags::HAS_BEGIN_PREPARE; + if (unprepare) { + content_flags |= ContentFlags::HAS_BEGIN_UNPREPARE; + } + return Status::OK(); + } + + Status MarkEndPrepare(const Slice&) override { + content_flags |= ContentFlags::HAS_END_PREPARE; + return Status::OK(); + } + + Status MarkCommit(const Slice&) override { + content_flags |= ContentFlags::HAS_COMMIT; + return Status::OK(); + } + + Status MarkCommitWithTimestamp(const Slice&, const Slice&) override { + content_flags |= ContentFlags::HAS_COMMIT; + return Status::OK(); + } + + Status MarkRollback(const Slice&) override { + content_flags |= ContentFlags::HAS_ROLLBACK; + return Status::OK(); + } +}; + +} // anonymous namespace + +struct SavePoints { + std::stack> stack; +}; + +WriteBatch::WriteBatch(size_t reserved_bytes, size_t max_bytes, + size_t protection_bytes_per_key, size_t default_cf_ts_sz) + : content_flags_(0), + max_bytes_(max_bytes), + default_cf_ts_sz_(default_cf_ts_sz), + rep_() { + // Currently `protection_bytes_per_key` can only be enabled at 8 bytes per + // entry. + assert(protection_bytes_per_key == 0 || protection_bytes_per_key == 8); + if (protection_bytes_per_key != 0) { + prot_info_.reset(new WriteBatch::ProtectionInfo()); + } + rep_.reserve((reserved_bytes > WriteBatchInternal::kHeader) + ? reserved_bytes + : WriteBatchInternal::kHeader); + rep_.resize(WriteBatchInternal::kHeader); +} + +WriteBatch::WriteBatch(const std::string& rep) + : content_flags_(ContentFlags::DEFERRED), max_bytes_(0), rep_(rep) {} + +WriteBatch::WriteBatch(std::string&& rep) + : content_flags_(ContentFlags::DEFERRED), + max_bytes_(0), + rep_(std::move(rep)) {} + +WriteBatch::WriteBatch(const WriteBatch& src) + : wal_term_point_(src.wal_term_point_), + content_flags_(src.content_flags_.load(std::memory_order_relaxed)), + max_bytes_(src.max_bytes_), + default_cf_ts_sz_(src.default_cf_ts_sz_), + rep_(src.rep_) { + if (src.save_points_ != nullptr) { + save_points_.reset(new SavePoints()); + save_points_->stack = src.save_points_->stack; + } + if (src.prot_info_ != nullptr) { + prot_info_.reset(new WriteBatch::ProtectionInfo()); + prot_info_->entries_ = src.prot_info_->entries_; + } +} + +WriteBatch::WriteBatch(WriteBatch&& src) noexcept + : save_points_(std::move(src.save_points_)), + wal_term_point_(std::move(src.wal_term_point_)), + content_flags_(src.content_flags_.load(std::memory_order_relaxed)), + max_bytes_(src.max_bytes_), + prot_info_(std::move(src.prot_info_)), + default_cf_ts_sz_(src.default_cf_ts_sz_), + rep_(std::move(src.rep_)) {} + +WriteBatch& WriteBatch::operator=(const WriteBatch& src) { + if (&src != this) { + this->~WriteBatch(); + new (this) WriteBatch(src); + } + return *this; +} + +WriteBatch& WriteBatch::operator=(WriteBatch&& src) { + if (&src != this) { + this->~WriteBatch(); + new (this) WriteBatch(std::move(src)); + } + return *this; +} + +WriteBatch::~WriteBatch() {} + +WriteBatch::Handler::~Handler() {} + +void WriteBatch::Handler::LogData(const Slice& /*blob*/) { + // If the user has not specified something to do with blobs, then we ignore + // them. +} + +bool WriteBatch::Handler::Continue() { return true; } + +void WriteBatch::Clear() { + rep_.clear(); + rep_.resize(WriteBatchInternal::kHeader); + + content_flags_.store(0, std::memory_order_relaxed); + + if (save_points_ != nullptr) { + while (!save_points_->stack.empty()) { + save_points_->stack.pop(); + } + } + + if (prot_info_ != nullptr) { + prot_info_->entries_.clear(); + } + wal_term_point_.clear(); + default_cf_ts_sz_ = 0; +} + +uint32_t WriteBatch::Count() const { return WriteBatchInternal::Count(this); } + +uint32_t WriteBatch::ComputeContentFlags() const { + auto rv = content_flags_.load(std::memory_order_relaxed); + if ((rv & ContentFlags::DEFERRED) != 0) { + BatchContentClassifier classifier; + // Should we handle status here? + Iterate(&classifier).PermitUncheckedError(); + rv = classifier.content_flags; + + // this method is conceptually const, because it is performing a lazy + // computation that doesn't affect the abstract state of the batch. + // content_flags_ is marked mutable so that we can perform the + // following assignment + content_flags_.store(rv, std::memory_order_relaxed); + } + return rv; +} + +void WriteBatch::MarkWalTerminationPoint() { + wal_term_point_.size = GetDataSize(); + wal_term_point_.count = Count(); + wal_term_point_.content_flags = content_flags_; +} + +size_t WriteBatch::GetProtectionBytesPerKey() const { + if (prot_info_ != nullptr) { + return prot_info_->GetBytesPerKey(); + } + return 0; +} + +std::string WriteBatch::Release() { + std::string ret = std::move(rep_); + Clear(); + return ret; +} + +bool WriteBatch::HasPut() const { + return (ComputeContentFlags() & ContentFlags::HAS_PUT) != 0; +} + +bool WriteBatch::HasPutEntity() const { + return (ComputeContentFlags() & ContentFlags::HAS_PUT_ENTITY) != 0; +} + +bool WriteBatch::HasDelete() const { + return (ComputeContentFlags() & ContentFlags::HAS_DELETE) != 0; +} + +bool WriteBatch::HasSingleDelete() const { + return (ComputeContentFlags() & ContentFlags::HAS_SINGLE_DELETE) != 0; +} + +bool WriteBatch::HasDeleteRange() const { + return (ComputeContentFlags() & ContentFlags::HAS_DELETE_RANGE) != 0; +} + +bool WriteBatch::HasMerge() const { + return (ComputeContentFlags() & ContentFlags::HAS_MERGE) != 0; +} + +bool ReadKeyFromWriteBatchEntry(Slice* input, Slice* key, bool cf_record) { + assert(input != nullptr && key != nullptr); + // Skip tag byte + input->remove_prefix(1); + + if (cf_record) { + // Skip column_family bytes + uint32_t cf; + if (!GetVarint32(input, &cf)) { + return false; + } + } + + // Extract key + return GetLengthPrefixedSlice(input, key); +} + +bool WriteBatch::HasBeginPrepare() const { + return (ComputeContentFlags() & ContentFlags::HAS_BEGIN_PREPARE) != 0; +} + +bool WriteBatch::HasEndPrepare() const { + return (ComputeContentFlags() & ContentFlags::HAS_END_PREPARE) != 0; +} + +bool WriteBatch::HasCommit() const { + return (ComputeContentFlags() & ContentFlags::HAS_COMMIT) != 0; +} + +bool WriteBatch::HasRollback() const { + return (ComputeContentFlags() & ContentFlags::HAS_ROLLBACK) != 0; +} + +Status ReadRecordFromWriteBatch(Slice* input, char* tag, + uint32_t* column_family, Slice* key, + Slice* value, Slice* blob, Slice* xid) { + assert(key != nullptr && value != nullptr); + *tag = (*input)[0]; + input->remove_prefix(1); + *column_family = 0; // default + switch (*tag) { + case kTypeColumnFamilyValue: + if (!GetVarint32(input, column_family)) { + return Status::Corruption("bad WriteBatch Put"); + } + FALLTHROUGH_INTENDED; + case kTypeValue: + if (!GetLengthPrefixedSlice(input, key) || + !GetLengthPrefixedSlice(input, value)) { + return Status::Corruption("bad WriteBatch Put"); + } + break; + case kTypeColumnFamilyDeletion: + case kTypeColumnFamilySingleDeletion: + if (!GetVarint32(input, column_family)) { + return Status::Corruption("bad WriteBatch Delete"); + } + FALLTHROUGH_INTENDED; + case kTypeDeletion: + case kTypeSingleDeletion: + if (!GetLengthPrefixedSlice(input, key)) { + return Status::Corruption("bad WriteBatch Delete"); + } + break; + case kTypeColumnFamilyRangeDeletion: + if (!GetVarint32(input, column_family)) { + return Status::Corruption("bad WriteBatch DeleteRange"); + } + FALLTHROUGH_INTENDED; + case kTypeRangeDeletion: + // for range delete, "key" is begin_key, "value" is end_key + if (!GetLengthPrefixedSlice(input, key) || + !GetLengthPrefixedSlice(input, value)) { + return Status::Corruption("bad WriteBatch DeleteRange"); + } + break; + case kTypeColumnFamilyMerge: + if (!GetVarint32(input, column_family)) { + return Status::Corruption("bad WriteBatch Merge"); + } + FALLTHROUGH_INTENDED; + case kTypeMerge: + if (!GetLengthPrefixedSlice(input, key) || + !GetLengthPrefixedSlice(input, value)) { + return Status::Corruption("bad WriteBatch Merge"); + } + break; + case kTypeColumnFamilyBlobIndex: + if (!GetVarint32(input, column_family)) { + return Status::Corruption("bad WriteBatch BlobIndex"); + } + FALLTHROUGH_INTENDED; + case kTypeBlobIndex: + if (!GetLengthPrefixedSlice(input, key) || + !GetLengthPrefixedSlice(input, value)) { + return Status::Corruption("bad WriteBatch BlobIndex"); + } + break; + case kTypeLogData: + assert(blob != nullptr); + if (!GetLengthPrefixedSlice(input, blob)) { + return Status::Corruption("bad WriteBatch Blob"); + } + break; + case kTypeNoop: + case kTypeBeginPrepareXID: + // This indicates that the prepared batch is also persisted in the db. + // This is used in WritePreparedTxn + case kTypeBeginPersistedPrepareXID: + // This is used in WriteUnpreparedTxn + case kTypeBeginUnprepareXID: + break; + case kTypeEndPrepareXID: + if (!GetLengthPrefixedSlice(input, xid)) { + return Status::Corruption("bad EndPrepare XID"); + } + break; + case kTypeCommitXIDAndTimestamp: + if (!GetLengthPrefixedSlice(input, key)) { + return Status::Corruption("bad commit timestamp"); + } + FALLTHROUGH_INTENDED; + case kTypeCommitXID: + if (!GetLengthPrefixedSlice(input, xid)) { + return Status::Corruption("bad Commit XID"); + } + break; + case kTypeRollbackXID: + if (!GetLengthPrefixedSlice(input, xid)) { + return Status::Corruption("bad Rollback XID"); + } + break; + case kTypeColumnFamilyWideColumnEntity: + if (!GetVarint32(input, column_family)) { + return Status::Corruption("bad WriteBatch PutEntity"); + } + FALLTHROUGH_INTENDED; + case kTypeWideColumnEntity: + if (!GetLengthPrefixedSlice(input, key) || + !GetLengthPrefixedSlice(input, value)) { + return Status::Corruption("bad WriteBatch PutEntity"); + } + break; + default: + return Status::Corruption("unknown WriteBatch tag"); + } + return Status::OK(); +} + +Status WriteBatch::Iterate(Handler* handler) const { + if (rep_.size() < WriteBatchInternal::kHeader) { + return Status::Corruption("malformed WriteBatch (too small)"); + } + + return WriteBatchInternal::Iterate(this, handler, WriteBatchInternal::kHeader, + rep_.size()); +} + +Status WriteBatchInternal::Iterate(const WriteBatch* wb, + WriteBatch::Handler* handler, size_t begin, + size_t end) { + if (begin > wb->rep_.size() || end > wb->rep_.size() || end < begin) { + return Status::Corruption("Invalid start/end bounds for Iterate"); + } + assert(begin <= end); + Slice input(wb->rep_.data() + begin, static_cast(end - begin)); + bool whole_batch = + (begin == WriteBatchInternal::kHeader) && (end == wb->rep_.size()); + + Slice key, value, blob, xid; + + // Sometimes a sub-batch starts with a Noop. We want to exclude such Noops as + // the batch boundary symbols otherwise we would mis-count the number of + // batches. We do that by checking whether the accumulated batch is empty + // before seeing the next Noop. + bool empty_batch = true; + uint32_t found = 0; + Status s; + char tag = 0; + uint32_t column_family = 0; // default + bool last_was_try_again = false; + bool handler_continue = true; + while (((s.ok() && !input.empty()) || UNLIKELY(s.IsTryAgain()))) { + handler_continue = handler->Continue(); + if (!handler_continue) { + break; + } + + if (LIKELY(!s.IsTryAgain())) { + last_was_try_again = false; + tag = 0; + column_family = 0; // default + + s = ReadRecordFromWriteBatch(&input, &tag, &column_family, &key, &value, + &blob, &xid); + if (!s.ok()) { + return s; + } + } else { + assert(s.IsTryAgain()); + assert(!last_was_try_again); // to detect infinite loop bugs + if (UNLIKELY(last_was_try_again)) { + return Status::Corruption( + "two consecutive TryAgain in WriteBatch handler; this is either a " + "software bug or data corruption."); + } + last_was_try_again = true; + s = Status::OK(); + } + + switch (tag) { + case kTypeColumnFamilyValue: + case kTypeValue: + assert(wb->content_flags_.load(std::memory_order_relaxed) & + (ContentFlags::DEFERRED | ContentFlags::HAS_PUT)); + s = handler->PutCF(column_family, key, value); + if (LIKELY(s.ok())) { + empty_batch = false; + found++; + } + break; + case kTypeColumnFamilyDeletion: + case kTypeDeletion: + assert(wb->content_flags_.load(std::memory_order_relaxed) & + (ContentFlags::DEFERRED | ContentFlags::HAS_DELETE)); + s = handler->DeleteCF(column_family, key); + if (LIKELY(s.ok())) { + empty_batch = false; + found++; + } + break; + case kTypeColumnFamilySingleDeletion: + case kTypeSingleDeletion: + assert(wb->content_flags_.load(std::memory_order_relaxed) & + (ContentFlags::DEFERRED | ContentFlags::HAS_SINGLE_DELETE)); + s = handler->SingleDeleteCF(column_family, key); + if (LIKELY(s.ok())) { + empty_batch = false; + found++; + } + break; + case kTypeColumnFamilyRangeDeletion: + case kTypeRangeDeletion: + assert(wb->content_flags_.load(std::memory_order_relaxed) & + (ContentFlags::DEFERRED | ContentFlags::HAS_DELETE_RANGE)); + s = handler->DeleteRangeCF(column_family, key, value); + if (LIKELY(s.ok())) { + empty_batch = false; + found++; + } + break; + case kTypeColumnFamilyMerge: + case kTypeMerge: + assert(wb->content_flags_.load(std::memory_order_relaxed) & + (ContentFlags::DEFERRED | ContentFlags::HAS_MERGE)); + s = handler->MergeCF(column_family, key, value); + if (LIKELY(s.ok())) { + empty_batch = false; + found++; + } + break; + case kTypeColumnFamilyBlobIndex: + case kTypeBlobIndex: + assert(wb->content_flags_.load(std::memory_order_relaxed) & + (ContentFlags::DEFERRED | ContentFlags::HAS_BLOB_INDEX)); + s = handler->PutBlobIndexCF(column_family, key, value); + if (LIKELY(s.ok())) { + found++; + } + break; + case kTypeLogData: + handler->LogData(blob); + // A batch might have nothing but LogData. It is still a batch. + empty_batch = false; + break; + case kTypeBeginPrepareXID: + assert(wb->content_flags_.load(std::memory_order_relaxed) & + (ContentFlags::DEFERRED | ContentFlags::HAS_BEGIN_PREPARE)); + s = handler->MarkBeginPrepare(); + assert(s.ok()); + empty_batch = false; + if (handler->WriteAfterCommit() == + WriteBatch::Handler::OptionState::kDisabled) { + s = Status::NotSupported( + "WriteCommitted txn tag when write_after_commit_ is disabled (in " + "WritePrepared/WriteUnprepared mode). If it is not due to " + "corruption, the WAL must be emptied before changing the " + "WritePolicy."); + } + if (handler->WriteBeforePrepare() == + WriteBatch::Handler::OptionState::kEnabled) { + s = Status::NotSupported( + "WriteCommitted txn tag when write_before_prepare_ is enabled " + "(in WriteUnprepared mode). If it is not due to corruption, the " + "WAL must be emptied before changing the WritePolicy."); + } + break; + case kTypeBeginPersistedPrepareXID: + assert(wb->content_flags_.load(std::memory_order_relaxed) & + (ContentFlags::DEFERRED | ContentFlags::HAS_BEGIN_PREPARE)); + s = handler->MarkBeginPrepare(); + assert(s.ok()); + empty_batch = false; + if (handler->WriteAfterCommit() == + WriteBatch::Handler::OptionState::kEnabled) { + s = Status::NotSupported( + "WritePrepared/WriteUnprepared txn tag when write_after_commit_ " + "is enabled (in default WriteCommitted mode). If it is not due " + "to corruption, the WAL must be emptied before changing the " + "WritePolicy."); + } + break; + case kTypeBeginUnprepareXID: + assert(wb->content_flags_.load(std::memory_order_relaxed) & + (ContentFlags::DEFERRED | ContentFlags::HAS_BEGIN_UNPREPARE)); + s = handler->MarkBeginPrepare(true /* unprepared */); + assert(s.ok()); + empty_batch = false; + if (handler->WriteAfterCommit() == + WriteBatch::Handler::OptionState::kEnabled) { + s = Status::NotSupported( + "WriteUnprepared txn tag when write_after_commit_ is enabled (in " + "default WriteCommitted mode). If it is not due to corruption, " + "the WAL must be emptied before changing the WritePolicy."); + } + if (handler->WriteBeforePrepare() == + WriteBatch::Handler::OptionState::kDisabled) { + s = Status::NotSupported( + "WriteUnprepared txn tag when write_before_prepare_ is disabled " + "(in WriteCommitted/WritePrepared mode). If it is not due to " + "corruption, the WAL must be emptied before changing the " + "WritePolicy."); + } + break; + case kTypeEndPrepareXID: + assert(wb->content_flags_.load(std::memory_order_relaxed) & + (ContentFlags::DEFERRED | ContentFlags::HAS_END_PREPARE)); + s = handler->MarkEndPrepare(xid); + assert(s.ok()); + empty_batch = true; + break; + case kTypeCommitXID: + assert(wb->content_flags_.load(std::memory_order_relaxed) & + (ContentFlags::DEFERRED | ContentFlags::HAS_COMMIT)); + s = handler->MarkCommit(xid); + assert(s.ok()); + empty_batch = true; + break; + case kTypeCommitXIDAndTimestamp: + assert(wb->content_flags_.load(std::memory_order_relaxed) & + (ContentFlags::DEFERRED | ContentFlags::HAS_COMMIT)); + // key stores the commit timestamp. + assert(!key.empty()); + s = handler->MarkCommitWithTimestamp(xid, key); + if (LIKELY(s.ok())) { + empty_batch = true; + } + break; + case kTypeRollbackXID: + assert(wb->content_flags_.load(std::memory_order_relaxed) & + (ContentFlags::DEFERRED | ContentFlags::HAS_ROLLBACK)); + s = handler->MarkRollback(xid); + assert(s.ok()); + empty_batch = true; + break; + case kTypeNoop: + s = handler->MarkNoop(empty_batch); + assert(s.ok()); + empty_batch = true; + break; + case kTypeWideColumnEntity: + case kTypeColumnFamilyWideColumnEntity: + assert(wb->content_flags_.load(std::memory_order_relaxed) & + (ContentFlags::DEFERRED | ContentFlags::HAS_PUT_ENTITY)); + s = handler->PutEntityCF(column_family, key, value); + if (LIKELY(s.ok())) { + empty_batch = false; + ++found; + } + break; + default: + return Status::Corruption("unknown WriteBatch tag"); + } + } + if (!s.ok()) { + return s; + } + if (handler_continue && whole_batch && + found != WriteBatchInternal::Count(wb)) { + return Status::Corruption("WriteBatch has wrong count"); + } else { + return Status::OK(); + } +} + +bool WriteBatchInternal::IsLatestPersistentState(const WriteBatch* b) { + return b->is_latest_persistent_state_; +} + +void WriteBatchInternal::SetAsLatestPersistentState(WriteBatch* b) { + b->is_latest_persistent_state_ = true; +} + +uint32_t WriteBatchInternal::Count(const WriteBatch* b) { + return DecodeFixed32(b->rep_.data() + 8); +} + +void WriteBatchInternal::SetCount(WriteBatch* b, uint32_t n) { + EncodeFixed32(&b->rep_[8], n); +} + +SequenceNumber WriteBatchInternal::Sequence(const WriteBatch* b) { + return SequenceNumber(DecodeFixed64(b->rep_.data())); +} + +void WriteBatchInternal::SetSequence(WriteBatch* b, SequenceNumber seq) { + EncodeFixed64(&b->rep_[0], seq); +} + +size_t WriteBatchInternal::GetFirstOffset(WriteBatch* /*b*/) { + return WriteBatchInternal::kHeader; +} + +std::tuple +WriteBatchInternal::GetColumnFamilyIdAndTimestampSize( + WriteBatch* b, ColumnFamilyHandle* column_family) { + uint32_t cf_id = GetColumnFamilyID(column_family); + size_t ts_sz = 0; + Status s; + if (column_family) { + const Comparator* const ucmp = column_family->GetComparator(); + if (ucmp) { + ts_sz = ucmp->timestamp_size(); + if (0 == cf_id && b->default_cf_ts_sz_ != ts_sz) { + s = Status::InvalidArgument("Default cf timestamp size mismatch"); + } + } + } else if (b->default_cf_ts_sz_ > 0) { + ts_sz = b->default_cf_ts_sz_; + } + return std::make_tuple(s, cf_id, ts_sz); +} + +namespace { +Status CheckColumnFamilyTimestampSize(ColumnFamilyHandle* column_family, + const Slice& ts) { + if (!column_family) { + return Status::InvalidArgument("column family handle cannot be null"); + } + const Comparator* const ucmp = column_family->GetComparator(); + assert(ucmp); + size_t cf_ts_sz = ucmp->timestamp_size(); + if (0 == cf_ts_sz) { + return Status::InvalidArgument("timestamp disabled"); + } + if (cf_ts_sz != ts.size()) { + return Status::InvalidArgument("timestamp size mismatch"); + } + return Status::OK(); +} +} // anonymous namespace + +Status WriteBatchInternal::Put(WriteBatch* b, uint32_t column_family_id, + const Slice& key, const Slice& value) { + if (key.size() > size_t{std::numeric_limits::max()}) { + return Status::InvalidArgument("key is too large"); + } + if (value.size() > size_t{std::numeric_limits::max()}) { + return Status::InvalidArgument("value is too large"); + } + + LocalSavePoint save(b); + WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1); + if (column_family_id == 0) { + b->rep_.push_back(static_cast(kTypeValue)); + } else { + b->rep_.push_back(static_cast(kTypeColumnFamilyValue)); + PutVarint32(&b->rep_, column_family_id); + } + PutLengthPrefixedSlice(&b->rep_, key); + PutLengthPrefixedSlice(&b->rep_, value); + b->content_flags_.store( + b->content_flags_.load(std::memory_order_relaxed) | ContentFlags::HAS_PUT, + std::memory_order_relaxed); + if (b->prot_info_ != nullptr) { + // Technically the optype could've been `kTypeColumnFamilyValue` with the + // CF ID encoded in the `WriteBatch`. That distinction is unimportant + // however since we verify CF ID is correct, as well as all other fields + // (a missing/extra encoded CF ID would corrupt another field). It is + // convenient to consolidate on `kTypeValue` here as that is what will be + // inserted into memtable. + b->prot_info_->entries_.emplace_back(ProtectionInfo64() + .ProtectKVO(key, value, kTypeValue) + .ProtectC(column_family_id)); + } + return save.commit(); +} + +Status WriteBatch::Put(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& value) { + size_t ts_sz = 0; + uint32_t cf_id = 0; + Status s; + + std::tie(s, cf_id, ts_sz) = + WriteBatchInternal::GetColumnFamilyIdAndTimestampSize(this, + column_family); + + if (!s.ok()) { + return s; + } + + if (0 == ts_sz) { + return WriteBatchInternal::Put(this, cf_id, key, value); + } + + needs_in_place_update_ts_ = true; + has_key_with_ts_ = true; + std::string dummy_ts(ts_sz, '\0'); + std::array key_with_ts{{key, dummy_ts}}; + return WriteBatchInternal::Put(this, cf_id, SliceParts(key_with_ts.data(), 2), + SliceParts(&value, 1)); +} + +Status WriteBatch::Put(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& ts, const Slice& value) { + const Status s = CheckColumnFamilyTimestampSize(column_family, ts); + if (!s.ok()) { + return s; + } + has_key_with_ts_ = true; + assert(column_family); + uint32_t cf_id = column_family->GetID(); + std::array key_with_ts{{key, ts}}; + return WriteBatchInternal::Put(this, cf_id, SliceParts(key_with_ts.data(), 2), + SliceParts(&value, 1)); +} + +Status WriteBatchInternal::CheckSlicePartsLength(const SliceParts& key, + const SliceParts& value) { + size_t total_key_bytes = 0; + for (int i = 0; i < key.num_parts; ++i) { + total_key_bytes += key.parts[i].size(); + } + if (total_key_bytes >= size_t{std::numeric_limits::max()}) { + return Status::InvalidArgument("key is too large"); + } + + size_t total_value_bytes = 0; + for (int i = 0; i < value.num_parts; ++i) { + total_value_bytes += value.parts[i].size(); + } + if (total_value_bytes >= size_t{std::numeric_limits::max()}) { + return Status::InvalidArgument("value is too large"); + } + return Status::OK(); +} + +Status WriteBatchInternal::Put(WriteBatch* b, uint32_t column_family_id, + const SliceParts& key, const SliceParts& value) { + Status s = CheckSlicePartsLength(key, value); + if (!s.ok()) { + return s; + } + + LocalSavePoint save(b); + WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1); + if (column_family_id == 0) { + b->rep_.push_back(static_cast(kTypeValue)); + } else { + b->rep_.push_back(static_cast(kTypeColumnFamilyValue)); + PutVarint32(&b->rep_, column_family_id); + } + PutLengthPrefixedSliceParts(&b->rep_, key); + PutLengthPrefixedSliceParts(&b->rep_, value); + b->content_flags_.store( + b->content_flags_.load(std::memory_order_relaxed) | ContentFlags::HAS_PUT, + std::memory_order_relaxed); + if (b->prot_info_ != nullptr) { + // See comment in first `WriteBatchInternal::Put()` overload concerning the + // `ValueType` argument passed to `ProtectKVO()`. + b->prot_info_->entries_.emplace_back(ProtectionInfo64() + .ProtectKVO(key, value, kTypeValue) + .ProtectC(column_family_id)); + } + return save.commit(); +} + +Status WriteBatch::Put(ColumnFamilyHandle* column_family, const SliceParts& key, + const SliceParts& value) { + size_t ts_sz = 0; + uint32_t cf_id = 0; + Status s; + + std::tie(s, cf_id, ts_sz) = + WriteBatchInternal::GetColumnFamilyIdAndTimestampSize(this, + column_family); + + if (!s.ok()) { + return s; + } + + if (ts_sz == 0) { + return WriteBatchInternal::Put(this, cf_id, key, value); + } + + return Status::InvalidArgument( + "Cannot call this method on column family enabling timestamp"); +} + +Status WriteBatchInternal::PutEntity(WriteBatch* b, uint32_t column_family_id, + const Slice& key, + const WideColumns& columns) { + assert(b); + + if (key.size() > size_t{std::numeric_limits::max()}) { + return Status::InvalidArgument("key is too large"); + } + + WideColumns sorted_columns(columns); + std::sort(sorted_columns.begin(), sorted_columns.end(), + [](const WideColumn& lhs, const WideColumn& rhs) { + return lhs.name().compare(rhs.name()) < 0; + }); + + std::string entity; + const Status s = WideColumnSerialization::Serialize(sorted_columns, entity); + if (!s.ok()) { + return s; + } + + if (entity.size() > size_t{std::numeric_limits::max()}) { + return Status::InvalidArgument("wide column entity is too large"); + } + + LocalSavePoint save(b); + + WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1); + + if (column_family_id == 0) { + b->rep_.push_back(static_cast(kTypeWideColumnEntity)); + } else { + b->rep_.push_back(static_cast(kTypeColumnFamilyWideColumnEntity)); + PutVarint32(&b->rep_, column_family_id); + } + + PutLengthPrefixedSlice(&b->rep_, key); + PutLengthPrefixedSlice(&b->rep_, entity); + + b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | + ContentFlags::HAS_PUT_ENTITY, + std::memory_order_relaxed); + + if (b->prot_info_ != nullptr) { + b->prot_info_->entries_.emplace_back( + ProtectionInfo64() + .ProtectKVO(key, entity, kTypeWideColumnEntity) + .ProtectC(column_family_id)); + } + + return save.commit(); +} + +Status WriteBatch::PutEntity(ColumnFamilyHandle* column_family, + const Slice& key, const WideColumns& columns) { + if (!column_family) { + return Status::InvalidArgument( + "Cannot call this method without a column family handle"); + } + + Status s; + uint32_t cf_id = 0; + size_t ts_sz = 0; + + std::tie(s, cf_id, ts_sz) = + WriteBatchInternal::GetColumnFamilyIdAndTimestampSize(this, + column_family); + + if (!s.ok()) { + return s; + } + + if (ts_sz) { + return Status::InvalidArgument( + "Cannot call this method on column family enabling timestamp"); + } + + return WriteBatchInternal::PutEntity(this, cf_id, key, columns); +} + +Status WriteBatchInternal::InsertNoop(WriteBatch* b) { + b->rep_.push_back(static_cast(kTypeNoop)); + return Status::OK(); +} + +Status WriteBatchInternal::MarkEndPrepare(WriteBatch* b, const Slice& xid, + bool write_after_commit, + bool unprepared_batch) { + // a manually constructed batch can only contain one prepare section + assert(b->rep_[12] == static_cast(kTypeNoop)); + + // all savepoints up to this point are cleared + if (b->save_points_ != nullptr) { + while (!b->save_points_->stack.empty()) { + b->save_points_->stack.pop(); + } + } + + // rewrite noop as begin marker + b->rep_[12] = static_cast( + write_after_commit ? kTypeBeginPrepareXID + : (unprepared_batch ? kTypeBeginUnprepareXID + : kTypeBeginPersistedPrepareXID)); + b->rep_.push_back(static_cast(kTypeEndPrepareXID)); + PutLengthPrefixedSlice(&b->rep_, xid); + b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | + ContentFlags::HAS_END_PREPARE | + ContentFlags::HAS_BEGIN_PREPARE, + std::memory_order_relaxed); + if (unprepared_batch) { + b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | + ContentFlags::HAS_BEGIN_UNPREPARE, + std::memory_order_relaxed); + } + return Status::OK(); +} + +Status WriteBatchInternal::MarkCommit(WriteBatch* b, const Slice& xid) { + b->rep_.push_back(static_cast(kTypeCommitXID)); + PutLengthPrefixedSlice(&b->rep_, xid); + b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | + ContentFlags::HAS_COMMIT, + std::memory_order_relaxed); + return Status::OK(); +} + +Status WriteBatchInternal::MarkCommitWithTimestamp(WriteBatch* b, + const Slice& xid, + const Slice& commit_ts) { + assert(!commit_ts.empty()); + b->rep_.push_back(static_cast(kTypeCommitXIDAndTimestamp)); + PutLengthPrefixedSlice(&b->rep_, commit_ts); + PutLengthPrefixedSlice(&b->rep_, xid); + b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | + ContentFlags::HAS_COMMIT, + std::memory_order_relaxed); + return Status::OK(); +} + +Status WriteBatchInternal::MarkRollback(WriteBatch* b, const Slice& xid) { + b->rep_.push_back(static_cast(kTypeRollbackXID)); + PutLengthPrefixedSlice(&b->rep_, xid); + b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | + ContentFlags::HAS_ROLLBACK, + std::memory_order_relaxed); + return Status::OK(); +} + +Status WriteBatchInternal::Delete(WriteBatch* b, uint32_t column_family_id, + const Slice& key) { + LocalSavePoint save(b); + WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1); + if (column_family_id == 0) { + b->rep_.push_back(static_cast(kTypeDeletion)); + } else { + b->rep_.push_back(static_cast(kTypeColumnFamilyDeletion)); + PutVarint32(&b->rep_, column_family_id); + } + PutLengthPrefixedSlice(&b->rep_, key); + b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | + ContentFlags::HAS_DELETE, + std::memory_order_relaxed); + if (b->prot_info_ != nullptr) { + // See comment in first `WriteBatchInternal::Put()` overload concerning the + // `ValueType` argument passed to `ProtectKVO()`. + b->prot_info_->entries_.emplace_back( + ProtectionInfo64() + .ProtectKVO(key, "" /* value */, kTypeDeletion) + .ProtectC(column_family_id)); + } + return save.commit(); +} + +Status WriteBatch::Delete(ColumnFamilyHandle* column_family, const Slice& key) { + size_t ts_sz = 0; + uint32_t cf_id = 0; + Status s; + + std::tie(s, cf_id, ts_sz) = + WriteBatchInternal::GetColumnFamilyIdAndTimestampSize(this, + column_family); + + if (!s.ok()) { + return s; + } + + if (0 == ts_sz) { + return WriteBatchInternal::Delete(this, cf_id, key); + } + + needs_in_place_update_ts_ = true; + has_key_with_ts_ = true; + std::string dummy_ts(ts_sz, '\0'); + std::array key_with_ts{{key, dummy_ts}}; + return WriteBatchInternal::Delete(this, cf_id, + SliceParts(key_with_ts.data(), 2)); +} + +Status WriteBatch::Delete(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& ts) { + const Status s = CheckColumnFamilyTimestampSize(column_family, ts); + if (!s.ok()) { + return s; + } + assert(column_family); + has_key_with_ts_ = true; + uint32_t cf_id = column_family->GetID(); + std::array key_with_ts{{key, ts}}; + return WriteBatchInternal::Delete(this, cf_id, + SliceParts(key_with_ts.data(), 2)); +} + +Status WriteBatchInternal::Delete(WriteBatch* b, uint32_t column_family_id, + const SliceParts& key) { + LocalSavePoint save(b); + WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1); + if (column_family_id == 0) { + b->rep_.push_back(static_cast(kTypeDeletion)); + } else { + b->rep_.push_back(static_cast(kTypeColumnFamilyDeletion)); + PutVarint32(&b->rep_, column_family_id); + } + PutLengthPrefixedSliceParts(&b->rep_, key); + b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | + ContentFlags::HAS_DELETE, + std::memory_order_relaxed); + if (b->prot_info_ != nullptr) { + // See comment in first `WriteBatchInternal::Put()` overload concerning the + // `ValueType` argument passed to `ProtectKVO()`. + b->prot_info_->entries_.emplace_back( + ProtectionInfo64() + .ProtectKVO(key, + SliceParts(nullptr /* _parts */, 0 /* _num_parts */), + kTypeDeletion) + .ProtectC(column_family_id)); + } + return save.commit(); +} + +Status WriteBatch::Delete(ColumnFamilyHandle* column_family, + const SliceParts& key) { + size_t ts_sz = 0; + uint32_t cf_id = 0; + Status s; + + std::tie(s, cf_id, ts_sz) = + WriteBatchInternal::GetColumnFamilyIdAndTimestampSize(this, + column_family); + + if (!s.ok()) { + return s; + } + + if (0 == ts_sz) { + return WriteBatchInternal::Delete(this, cf_id, key); + } + + return Status::InvalidArgument( + "Cannot call this method on column family enabling timestamp"); +} + +Status WriteBatchInternal::SingleDelete(WriteBatch* b, + uint32_t column_family_id, + const Slice& key) { + LocalSavePoint save(b); + WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1); + if (column_family_id == 0) { + b->rep_.push_back(static_cast(kTypeSingleDeletion)); + } else { + b->rep_.push_back(static_cast(kTypeColumnFamilySingleDeletion)); + PutVarint32(&b->rep_, column_family_id); + } + PutLengthPrefixedSlice(&b->rep_, key); + b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | + ContentFlags::HAS_SINGLE_DELETE, + std::memory_order_relaxed); + if (b->prot_info_ != nullptr) { + // See comment in first `WriteBatchInternal::Put()` overload concerning the + // `ValueType` argument passed to `ProtectKVO()`. + b->prot_info_->entries_.emplace_back( + ProtectionInfo64() + .ProtectKVO(key, "" /* value */, kTypeSingleDeletion) + .ProtectC(column_family_id)); + } + return save.commit(); +} + +Status WriteBatch::SingleDelete(ColumnFamilyHandle* column_family, + const Slice& key) { + size_t ts_sz = 0; + uint32_t cf_id = 0; + Status s; + + std::tie(s, cf_id, ts_sz) = + WriteBatchInternal::GetColumnFamilyIdAndTimestampSize(this, + column_family); + + if (!s.ok()) { + return s; + } + + if (0 == ts_sz) { + return WriteBatchInternal::SingleDelete(this, cf_id, key); + } + + needs_in_place_update_ts_ = true; + has_key_with_ts_ = true; + std::string dummy_ts(ts_sz, '\0'); + std::array key_with_ts{{key, dummy_ts}}; + return WriteBatchInternal::SingleDelete(this, cf_id, + SliceParts(key_with_ts.data(), 2)); +} + +Status WriteBatch::SingleDelete(ColumnFamilyHandle* column_family, + const Slice& key, const Slice& ts) { + const Status s = CheckColumnFamilyTimestampSize(column_family, ts); + if (!s.ok()) { + return s; + } + has_key_with_ts_ = true; + assert(column_family); + uint32_t cf_id = column_family->GetID(); + std::array key_with_ts{{key, ts}}; + return WriteBatchInternal::SingleDelete(this, cf_id, + SliceParts(key_with_ts.data(), 2)); +} + +Status WriteBatchInternal::SingleDelete(WriteBatch* b, + uint32_t column_family_id, + const SliceParts& key) { + LocalSavePoint save(b); + WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1); + if (column_family_id == 0) { + b->rep_.push_back(static_cast(kTypeSingleDeletion)); + } else { + b->rep_.push_back(static_cast(kTypeColumnFamilySingleDeletion)); + PutVarint32(&b->rep_, column_family_id); + } + PutLengthPrefixedSliceParts(&b->rep_, key); + b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | + ContentFlags::HAS_SINGLE_DELETE, + std::memory_order_relaxed); + if (b->prot_info_ != nullptr) { + // See comment in first `WriteBatchInternal::Put()` overload concerning the + // `ValueType` argument passed to `ProtectKVO()`. + b->prot_info_->entries_.emplace_back( + ProtectionInfo64() + .ProtectKVO(key, + SliceParts(nullptr /* _parts */, + 0 /* _num_parts */) /* value */, + kTypeSingleDeletion) + .ProtectC(column_family_id)); + } + return save.commit(); +} + +Status WriteBatch::SingleDelete(ColumnFamilyHandle* column_family, + const SliceParts& key) { + size_t ts_sz = 0; + uint32_t cf_id = 0; + Status s; + + std::tie(s, cf_id, ts_sz) = + WriteBatchInternal::GetColumnFamilyIdAndTimestampSize(this, + column_family); + + if (!s.ok()) { + return s; + } + + if (0 == ts_sz) { + return WriteBatchInternal::SingleDelete(this, cf_id, key); + } + + return Status::InvalidArgument( + "Cannot call this method on column family enabling timestamp"); +} + +Status WriteBatchInternal::DeleteRange(WriteBatch* b, uint32_t column_family_id, + const Slice& begin_key, + const Slice& end_key) { + LocalSavePoint save(b); + WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1); + if (column_family_id == 0) { + b->rep_.push_back(static_cast(kTypeRangeDeletion)); + } else { + b->rep_.push_back(static_cast(kTypeColumnFamilyRangeDeletion)); + PutVarint32(&b->rep_, column_family_id); + } + PutLengthPrefixedSlice(&b->rep_, begin_key); + PutLengthPrefixedSlice(&b->rep_, end_key); + b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | + ContentFlags::HAS_DELETE_RANGE, + std::memory_order_relaxed); + if (b->prot_info_ != nullptr) { + // See comment in first `WriteBatchInternal::Put()` overload concerning the + // `ValueType` argument passed to `ProtectKVO()`. + // In `DeleteRange()`, the end key is treated as the value. + b->prot_info_->entries_.emplace_back( + ProtectionInfo64() + .ProtectKVO(begin_key, end_key, kTypeRangeDeletion) + .ProtectC(column_family_id)); + } + return save.commit(); +} + +Status WriteBatch::DeleteRange(ColumnFamilyHandle* column_family, + const Slice& begin_key, const Slice& end_key) { + size_t ts_sz = 0; + uint32_t cf_id = 0; + Status s; + + std::tie(s, cf_id, ts_sz) = + WriteBatchInternal::GetColumnFamilyIdAndTimestampSize(this, + column_family); + + if (!s.ok()) { + return s; + } + + if (0 == ts_sz) { + return WriteBatchInternal::DeleteRange(this, cf_id, begin_key, end_key); + } + + needs_in_place_update_ts_ = true; + has_key_with_ts_ = true; + std::string dummy_ts(ts_sz, '\0'); + std::array begin_key_with_ts{{begin_key, dummy_ts}}; + std::array end_key_with_ts{{end_key, dummy_ts}}; + return WriteBatchInternal::DeleteRange( + this, cf_id, SliceParts(begin_key_with_ts.data(), 2), + SliceParts(end_key_with_ts.data(), 2)); +} + +Status WriteBatch::DeleteRange(ColumnFamilyHandle* column_family, + const Slice& begin_key, const Slice& end_key, + const Slice& ts) { + const Status s = CheckColumnFamilyTimestampSize(column_family, ts); + if (!s.ok()) { + return s; + } + assert(column_family); + has_key_with_ts_ = true; + uint32_t cf_id = column_family->GetID(); + std::array key_with_ts{{begin_key, ts}}; + std::array end_key_with_ts{{end_key, ts}}; + return WriteBatchInternal::DeleteRange(this, cf_id, + SliceParts(key_with_ts.data(), 2), + SliceParts(end_key_with_ts.data(), 2)); +} + +Status WriteBatchInternal::DeleteRange(WriteBatch* b, uint32_t column_family_id, + const SliceParts& begin_key, + const SliceParts& end_key) { + LocalSavePoint save(b); + WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1); + if (column_family_id == 0) { + b->rep_.push_back(static_cast(kTypeRangeDeletion)); + } else { + b->rep_.push_back(static_cast(kTypeColumnFamilyRangeDeletion)); + PutVarint32(&b->rep_, column_family_id); + } + PutLengthPrefixedSliceParts(&b->rep_, begin_key); + PutLengthPrefixedSliceParts(&b->rep_, end_key); + b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | + ContentFlags::HAS_DELETE_RANGE, + std::memory_order_relaxed); + if (b->prot_info_ != nullptr) { + // See comment in first `WriteBatchInternal::Put()` overload concerning the + // `ValueType` argument passed to `ProtectKVO()`. + // In `DeleteRange()`, the end key is treated as the value. + b->prot_info_->entries_.emplace_back( + ProtectionInfo64() + .ProtectKVO(begin_key, end_key, kTypeRangeDeletion) + .ProtectC(column_family_id)); + } + return save.commit(); +} + +Status WriteBatch::DeleteRange(ColumnFamilyHandle* column_family, + const SliceParts& begin_key, + const SliceParts& end_key) { + size_t ts_sz = 0; + uint32_t cf_id = 0; + Status s; + + std::tie(s, cf_id, ts_sz) = + WriteBatchInternal::GetColumnFamilyIdAndTimestampSize(this, + column_family); + + if (!s.ok()) { + return s; + } + + if (0 == ts_sz) { + return WriteBatchInternal::DeleteRange(this, cf_id, begin_key, end_key); + } + + return Status::InvalidArgument( + "Cannot call this method on column family enabling timestamp"); +} + +Status WriteBatchInternal::Merge(WriteBatch* b, uint32_t column_family_id, + const Slice& key, const Slice& value) { + if (key.size() > size_t{std::numeric_limits::max()}) { + return Status::InvalidArgument("key is too large"); + } + if (value.size() > size_t{std::numeric_limits::max()}) { + return Status::InvalidArgument("value is too large"); + } + + LocalSavePoint save(b); + WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1); + if (column_family_id == 0) { + b->rep_.push_back(static_cast(kTypeMerge)); + } else { + b->rep_.push_back(static_cast(kTypeColumnFamilyMerge)); + PutVarint32(&b->rep_, column_family_id); + } + PutLengthPrefixedSlice(&b->rep_, key); + PutLengthPrefixedSlice(&b->rep_, value); + b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | + ContentFlags::HAS_MERGE, + std::memory_order_relaxed); + if (b->prot_info_ != nullptr) { + // See comment in first `WriteBatchInternal::Put()` overload concerning the + // `ValueType` argument passed to `ProtectKVO()`. + b->prot_info_->entries_.emplace_back(ProtectionInfo64() + .ProtectKVO(key, value, kTypeMerge) + .ProtectC(column_family_id)); + } + return save.commit(); +} + +Status WriteBatch::Merge(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& value) { + size_t ts_sz = 0; + uint32_t cf_id = 0; + Status s; + + std::tie(s, cf_id, ts_sz) = + WriteBatchInternal::GetColumnFamilyIdAndTimestampSize(this, + column_family); + + if (!s.ok()) { + return s; + } + + if (0 == ts_sz) { + return WriteBatchInternal::Merge(this, cf_id, key, value); + } + + needs_in_place_update_ts_ = true; + has_key_with_ts_ = true; + std::string dummy_ts(ts_sz, '\0'); + std::array key_with_ts{{key, dummy_ts}}; + + return WriteBatchInternal::Merge( + this, cf_id, SliceParts(key_with_ts.data(), 2), SliceParts(&value, 1)); +} + +Status WriteBatch::Merge(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& ts, const Slice& value) { + const Status s = CheckColumnFamilyTimestampSize(column_family, ts); + if (!s.ok()) { + return s; + } + has_key_with_ts_ = true; + assert(column_family); + uint32_t cf_id = column_family->GetID(); + std::array key_with_ts{{key, ts}}; + return WriteBatchInternal::Merge( + this, cf_id, SliceParts(key_with_ts.data(), 2), SliceParts(&value, 1)); +} + +Status WriteBatchInternal::Merge(WriteBatch* b, uint32_t column_family_id, + const SliceParts& key, + const SliceParts& value) { + Status s = CheckSlicePartsLength(key, value); + if (!s.ok()) { + return s; + } + + LocalSavePoint save(b); + WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1); + if (column_family_id == 0) { + b->rep_.push_back(static_cast(kTypeMerge)); + } else { + b->rep_.push_back(static_cast(kTypeColumnFamilyMerge)); + PutVarint32(&b->rep_, column_family_id); + } + PutLengthPrefixedSliceParts(&b->rep_, key); + PutLengthPrefixedSliceParts(&b->rep_, value); + b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | + ContentFlags::HAS_MERGE, + std::memory_order_relaxed); + if (b->prot_info_ != nullptr) { + // See comment in first `WriteBatchInternal::Put()` overload concerning the + // `ValueType` argument passed to `ProtectKVO()`. + b->prot_info_->entries_.emplace_back(ProtectionInfo64() + .ProtectKVO(key, value, kTypeMerge) + .ProtectC(column_family_id)); + } + return save.commit(); +} + +Status WriteBatch::Merge(ColumnFamilyHandle* column_family, + const SliceParts& key, const SliceParts& value) { + size_t ts_sz = 0; + uint32_t cf_id = 0; + Status s; + + std::tie(s, cf_id, ts_sz) = + WriteBatchInternal::GetColumnFamilyIdAndTimestampSize(this, + column_family); + + if (!s.ok()) { + return s; + } + + if (0 == ts_sz) { + return WriteBatchInternal::Merge(this, cf_id, key, value); + } + + return Status::InvalidArgument( + "Cannot call this method on column family enabling timestamp"); +} + +Status WriteBatchInternal::PutBlobIndex(WriteBatch* b, + uint32_t column_family_id, + const Slice& key, const Slice& value) { + LocalSavePoint save(b); + WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1); + if (column_family_id == 0) { + b->rep_.push_back(static_cast(kTypeBlobIndex)); + } else { + b->rep_.push_back(static_cast(kTypeColumnFamilyBlobIndex)); + PutVarint32(&b->rep_, column_family_id); + } + PutLengthPrefixedSlice(&b->rep_, key); + PutLengthPrefixedSlice(&b->rep_, value); + b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | + ContentFlags::HAS_BLOB_INDEX, + std::memory_order_relaxed); + if (b->prot_info_ != nullptr) { + // See comment in first `WriteBatchInternal::Put()` overload concerning the + // `ValueType` argument passed to `ProtectKVO()`. + b->prot_info_->entries_.emplace_back( + ProtectionInfo64() + .ProtectKVO(key, value, kTypeBlobIndex) + .ProtectC(column_family_id)); + } + return save.commit(); +} + +Status WriteBatch::PutLogData(const Slice& blob) { + LocalSavePoint save(this); + rep_.push_back(static_cast(kTypeLogData)); + PutLengthPrefixedSlice(&rep_, blob); + return save.commit(); +} + +void WriteBatch::SetSavePoint() { + if (save_points_ == nullptr) { + save_points_.reset(new SavePoints()); + } + // Record length and count of current batch of writes. + save_points_->stack.push(SavePoint( + GetDataSize(), Count(), content_flags_.load(std::memory_order_relaxed))); +} + +Status WriteBatch::RollbackToSavePoint() { + if (save_points_ == nullptr || save_points_->stack.size() == 0) { + return Status::NotFound(); + } + + // Pop the most recent savepoint off the stack + SavePoint savepoint = save_points_->stack.top(); + save_points_->stack.pop(); + + assert(savepoint.size <= rep_.size()); + assert(static_cast(savepoint.count) <= Count()); + + if (savepoint.size == rep_.size()) { + // No changes to rollback + } else if (savepoint.size == 0) { + // Rollback everything + Clear(); + } else { + rep_.resize(savepoint.size); + if (prot_info_ != nullptr) { + prot_info_->entries_.resize(savepoint.count); + } + WriteBatchInternal::SetCount(this, savepoint.count); + content_flags_.store(savepoint.content_flags, std::memory_order_relaxed); + } + + return Status::OK(); +} + +Status WriteBatch::PopSavePoint() { + if (save_points_ == nullptr || save_points_->stack.size() == 0) { + return Status::NotFound(); + } + + // Pop the most recent savepoint off the stack + save_points_->stack.pop(); + + return Status::OK(); +} + +Status WriteBatch::UpdateTimestamps( + const Slice& ts, std::function ts_sz_func) { + TimestampUpdater ts_updater(prot_info_.get(), + std::move(ts_sz_func), ts); + const Status s = Iterate(&ts_updater); + if (s.ok()) { + needs_in_place_update_ts_ = false; + } + return s; +} + +Status WriteBatch::VerifyChecksum() const { + if (prot_info_ == nullptr) { + return Status::OK(); + } + Slice input(rep_.data() + WriteBatchInternal::kHeader, + rep_.size() - WriteBatchInternal::kHeader); + Slice key, value, blob, xid; + char tag = 0; + uint32_t column_family = 0; // default + Status s; + size_t prot_info_idx = 0; + bool checksum_protected = true; + while (!input.empty() && prot_info_idx < prot_info_->entries_.size()) { + // In case key/value/column_family are not updated by + // ReadRecordFromWriteBatch + key.clear(); + value.clear(); + column_family = 0; + s = ReadRecordFromWriteBatch(&input, &tag, &column_family, &key, &value, + &blob, &xid); + if (!s.ok()) { + return s; + } + checksum_protected = true; + // Write batch checksum uses op_type without ColumnFamily (e.g., if op_type + // in the write batch is kTypeColumnFamilyValue, kTypeValue is used to + // compute the checksum), and encodes column family id separately. See + // comment in first `WriteBatchInternal::Put()` for more detail. + switch (tag) { + case kTypeColumnFamilyValue: + case kTypeValue: + tag = kTypeValue; + break; + case kTypeColumnFamilyDeletion: + case kTypeDeletion: + tag = kTypeDeletion; + break; + case kTypeColumnFamilySingleDeletion: + case kTypeSingleDeletion: + tag = kTypeSingleDeletion; + break; + case kTypeColumnFamilyRangeDeletion: + case kTypeRangeDeletion: + tag = kTypeRangeDeletion; + break; + case kTypeColumnFamilyMerge: + case kTypeMerge: + tag = kTypeMerge; + break; + case kTypeColumnFamilyBlobIndex: + case kTypeBlobIndex: + tag = kTypeBlobIndex; + break; + case kTypeLogData: + case kTypeBeginPrepareXID: + case kTypeEndPrepareXID: + case kTypeCommitXID: + case kTypeRollbackXID: + case kTypeNoop: + case kTypeBeginPersistedPrepareXID: + case kTypeBeginUnprepareXID: + case kTypeDeletionWithTimestamp: + case kTypeCommitXIDAndTimestamp: + checksum_protected = false; + break; + case kTypeColumnFamilyWideColumnEntity: + case kTypeWideColumnEntity: + tag = kTypeWideColumnEntity; + break; + default: + return Status::Corruption( + "unknown WriteBatch tag", + std::to_string(static_cast(tag))); + } + if (checksum_protected) { + s = prot_info_->entries_[prot_info_idx++] + .StripC(column_family) + .StripKVO(key, value, static_cast(tag)) + .GetStatus(); + if (!s.ok()) { + return s; + } + } + } + + if (prot_info_idx != WriteBatchInternal::Count(this)) { + return Status::Corruption("WriteBatch has wrong count"); + } + assert(WriteBatchInternal::Count(this) == prot_info_->entries_.size()); + return Status::OK(); +} + +namespace { + +class MemTableInserter : public WriteBatch::Handler { + SequenceNumber sequence_; + ColumnFamilyMemTables* const cf_mems_; + FlushScheduler* const flush_scheduler_; + TrimHistoryScheduler* const trim_history_scheduler_; + const bool ignore_missing_column_families_; + const uint64_t recovering_log_number_; + // log number that all Memtables inserted into should reference + uint64_t log_number_ref_; + DBImpl* db_; + const bool concurrent_memtable_writes_; + bool post_info_created_; + const WriteBatch::ProtectionInfo* prot_info_; + size_t prot_info_idx_; + + bool* has_valid_writes_; + // On some (!) platforms just default creating + // a map is too expensive in the Write() path as they + // cause memory allocations though unused. + // Make creation optional but do not incur + // std::unique_ptr additional allocation + using MemPostInfoMap = std::map; + using PostMapType = std::aligned_storage::type; + PostMapType mem_post_info_map_; + // current recovered transaction we are rebuilding (recovery) + WriteBatch* rebuilding_trx_; + SequenceNumber rebuilding_trx_seq_; + // Increase seq number once per each write batch. Otherwise increase it once + // per key. + bool seq_per_batch_; + // Whether the memtable write will be done only after the commit + bool write_after_commit_; + // Whether memtable write can be done before prepare + bool write_before_prepare_; + // Whether this batch was unprepared or not + bool unprepared_batch_; + using DupDetector = std::aligned_storage::type; + DupDetector duplicate_detector_; + bool dup_dectector_on_; + + bool hint_per_batch_; + bool hint_created_; + // Hints for this batch + using HintMap = std::unordered_map; + using HintMapType = std::aligned_storage::type; + HintMapType hint_; + + HintMap& GetHintMap() { + assert(hint_per_batch_); + if (!hint_created_) { + new (&hint_) HintMap(); + hint_created_ = true; + } + return *reinterpret_cast(&hint_); + } + + MemPostInfoMap& GetPostMap() { + assert(concurrent_memtable_writes_); + if (!post_info_created_) { + new (&mem_post_info_map_) MemPostInfoMap(); + post_info_created_ = true; + } + return *reinterpret_cast(&mem_post_info_map_); + } + + bool IsDuplicateKeySeq(uint32_t column_family_id, const Slice& key) { + assert(!write_after_commit_); + assert(rebuilding_trx_ != nullptr); + if (!dup_dectector_on_) { + new (&duplicate_detector_) DuplicateDetector(db_); + dup_dectector_on_ = true; + } + return reinterpret_cast(&duplicate_detector_) + ->IsDuplicateKeySeq(column_family_id, key, sequence_); + } + + const ProtectionInfoKVOC64* NextProtectionInfo() { + const ProtectionInfoKVOC64* res = nullptr; + if (prot_info_ != nullptr) { + assert(prot_info_idx_ < prot_info_->entries_.size()); + res = &prot_info_->entries_[prot_info_idx_]; + ++prot_info_idx_; + } + return res; + } + + void DecrementProtectionInfoIdxForTryAgain() { + if (prot_info_ != nullptr) --prot_info_idx_; + } + + void ResetProtectionInfo() { + prot_info_idx_ = 0; + prot_info_ = nullptr; + } + + protected: + Handler::OptionState WriteBeforePrepare() const override { + return write_before_prepare_ ? Handler::OptionState::kEnabled + : Handler::OptionState::kDisabled; + } + Handler::OptionState WriteAfterCommit() const override { + return write_after_commit_ ? Handler::OptionState::kEnabled + : Handler::OptionState::kDisabled; + } + + public: + // cf_mems should not be shared with concurrent inserters + MemTableInserter(SequenceNumber _sequence, ColumnFamilyMemTables* cf_mems, + FlushScheduler* flush_scheduler, + TrimHistoryScheduler* trim_history_scheduler, + bool ignore_missing_column_families, + uint64_t recovering_log_number, DB* db, + bool concurrent_memtable_writes, + const WriteBatch::ProtectionInfo* prot_info, + bool* has_valid_writes = nullptr, bool seq_per_batch = false, + bool batch_per_txn = true, bool hint_per_batch = false) + : sequence_(_sequence), + cf_mems_(cf_mems), + flush_scheduler_(flush_scheduler), + trim_history_scheduler_(trim_history_scheduler), + ignore_missing_column_families_(ignore_missing_column_families), + recovering_log_number_(recovering_log_number), + log_number_ref_(0), + db_(static_cast_with_check(db)), + concurrent_memtable_writes_(concurrent_memtable_writes), + post_info_created_(false), + prot_info_(prot_info), + prot_info_idx_(0), + has_valid_writes_(has_valid_writes), + rebuilding_trx_(nullptr), + rebuilding_trx_seq_(0), + seq_per_batch_(seq_per_batch), + // Write after commit currently uses one seq per key (instead of per + // batch). So seq_per_batch being false indicates write_after_commit + // approach. + write_after_commit_(!seq_per_batch), + // WriteUnprepared can write WriteBatches per transaction, so + // batch_per_txn being false indicates write_before_prepare. + write_before_prepare_(!batch_per_txn), + unprepared_batch_(false), + duplicate_detector_(), + dup_dectector_on_(false), + hint_per_batch_(hint_per_batch), + hint_created_(false) { + assert(cf_mems_); + } + + ~MemTableInserter() override { + if (dup_dectector_on_) { + reinterpret_cast(&duplicate_detector_) + ->~DuplicateDetector(); + } + if (post_info_created_) { + reinterpret_cast(&mem_post_info_map_)->~MemPostInfoMap(); + } + if (hint_created_) { + for (auto iter : GetHintMap()) { + delete[] reinterpret_cast(iter.second); + } + reinterpret_cast(&hint_)->~HintMap(); + } + delete rebuilding_trx_; + } + + MemTableInserter(const MemTableInserter&) = delete; + MemTableInserter& operator=(const MemTableInserter&) = delete; + + // The batch seq is regularly restarted; In normal mode it is set when + // MemTableInserter is constructed in the write thread and in recovery mode it + // is set when a batch, which is tagged with seq, is read from the WAL. + // Within a sequenced batch, which could be a merge of multiple batches, we + // have two policies to advance the seq: i) seq_per_key (default) and ii) + // seq_per_batch. To implement the latter we need to mark the boundary between + // the individual batches. The approach is this: 1) Use the terminating + // markers to indicate the boundary (kTypeEndPrepareXID, kTypeCommitXID, + // kTypeRollbackXID) 2) Terminate a batch with kTypeNoop in the absence of a + // natural boundary marker. + void MaybeAdvanceSeq(bool batch_boundry = false) { + if (batch_boundry == seq_per_batch_) { + sequence_++; + } + } + + void set_log_number_ref(uint64_t log) { log_number_ref_ = log; } + void set_prot_info(const WriteBatch::ProtectionInfo* prot_info) { + prot_info_ = prot_info; + prot_info_idx_ = 0; + } + + SequenceNumber sequence() const { return sequence_; } + + void PostProcess() { + assert(concurrent_memtable_writes_); + // If post info was not created there is nothing + // to process and no need to create on demand + if (post_info_created_) { + for (auto& pair : GetPostMap()) { + pair.first->BatchPostProcess(pair.second); + } + } + } + + bool SeekToColumnFamily(uint32_t column_family_id, Status* s) { + // If we are in a concurrent mode, it is the caller's responsibility + // to clone the original ColumnFamilyMemTables so that each thread + // has its own instance. Otherwise, it must be guaranteed that there + // is no concurrent access + bool found = cf_mems_->Seek(column_family_id); + if (!found) { + if (ignore_missing_column_families_) { + *s = Status::OK(); + } else { + *s = Status::InvalidArgument( + "Invalid column family specified in write batch"); + } + return false; + } + if (recovering_log_number_ != 0 && + recovering_log_number_ < cf_mems_->GetLogNumber()) { + // This is true only in recovery environment (recovering_log_number_ is + // always 0 in + // non-recovery, regular write code-path) + // * If recovering_log_number_ < cf_mems_->GetLogNumber(), this means that + // column family already contains updates from this log. We can't apply + // updates twice because of update-in-place or merge workloads -- ignore + // the update + *s = Status::OK(); + return false; + } + + if (has_valid_writes_ != nullptr) { + *has_valid_writes_ = true; + } + + if (log_number_ref_ > 0) { + cf_mems_->GetMemTable()->RefLogContainingPrepSection(log_number_ref_); + } + + return true; + } + + Status PutCFImpl(uint32_t column_family_id, const Slice& key, + const Slice& value, ValueType value_type, + const ProtectionInfoKVOS64* kv_prot_info) { + // optimize for non-recovery mode + if (UNLIKELY(write_after_commit_ && rebuilding_trx_ != nullptr)) { + // TODO(ajkr): propagate `ProtectionInfoKVOS64`. + return WriteBatchInternal::Put(rebuilding_trx_, column_family_id, key, + value); + // else insert the values to the memtable right away + } + + Status ret_status; + if (UNLIKELY(!SeekToColumnFamily(column_family_id, &ret_status))) { + if (ret_status.ok() && rebuilding_trx_ != nullptr) { + assert(!write_after_commit_); + // The CF is probably flushed and hence no need for insert but we still + // need to keep track of the keys for upcoming rollback/commit. + // TODO(ajkr): propagate `ProtectionInfoKVOS64`. + ret_status = WriteBatchInternal::Put(rebuilding_trx_, column_family_id, + key, value); + if (ret_status.ok()) { + MaybeAdvanceSeq(IsDuplicateKeySeq(column_family_id, key)); + } + } else if (ret_status.ok()) { + MaybeAdvanceSeq(false /* batch_boundary */); + } + return ret_status; + } + assert(ret_status.ok()); + + MemTable* mem = cf_mems_->GetMemTable(); + auto* moptions = mem->GetImmutableMemTableOptions(); + // inplace_update_support is inconsistent with snapshots, and therefore with + // any kind of transactions including the ones that use seq_per_batch + assert(!seq_per_batch_ || !moptions->inplace_update_support); + if (!moptions->inplace_update_support) { + ret_status = + mem->Add(sequence_, value_type, key, value, kv_prot_info, + concurrent_memtable_writes_, get_post_process_info(mem), + hint_per_batch_ ? &GetHintMap()[mem] : nullptr); + } else if (moptions->inplace_callback == nullptr || + value_type != kTypeValue) { + assert(!concurrent_memtable_writes_); + ret_status = mem->Update(sequence_, value_type, key, value, kv_prot_info); + } else { + assert(!concurrent_memtable_writes_); + assert(value_type == kTypeValue); + ret_status = mem->UpdateCallback(sequence_, key, value, kv_prot_info); + if (ret_status.IsNotFound()) { + // key not found in memtable. Do sst get, update, add + SnapshotImpl read_from_snapshot; + read_from_snapshot.number_ = sequence_; + // TODO: plumb Env::IOActivity + ReadOptions ropts; + // it's going to be overwritten for sure, so no point caching data block + // containing the old version + ropts.fill_cache = false; + ropts.snapshot = &read_from_snapshot; + + std::string prev_value; + std::string merged_value; + + auto cf_handle = cf_mems_->GetColumnFamilyHandle(); + Status get_status = Status::NotSupported(); + if (db_ != nullptr && recovering_log_number_ == 0) { + if (cf_handle == nullptr) { + cf_handle = db_->DefaultColumnFamily(); + } + // TODO (yanqin): fix when user-defined timestamp is enabled. + get_status = db_->Get(ropts, cf_handle, key, &prev_value); + } + // Intentionally overwrites the `NotFound` in `ret_status`. + if (!get_status.ok() && !get_status.IsNotFound()) { + ret_status = get_status; + } else { + ret_status = Status::OK(); + } + if (ret_status.ok()) { + UpdateStatus update_status; + char* prev_buffer = const_cast(prev_value.c_str()); + uint32_t prev_size = static_cast(prev_value.size()); + if (get_status.ok()) { + update_status = moptions->inplace_callback(prev_buffer, &prev_size, + value, &merged_value); + } else { + update_status = moptions->inplace_callback( + nullptr /* existing_value */, nullptr /* existing_value_size */, + value, &merged_value); + } + if (update_status == UpdateStatus::UPDATED_INPLACE) { + assert(get_status.ok()); + if (kv_prot_info != nullptr) { + ProtectionInfoKVOS64 updated_kv_prot_info(*kv_prot_info); + updated_kv_prot_info.UpdateV(value, + Slice(prev_buffer, prev_size)); + // prev_value is updated in-place with final value. + ret_status = mem->Add(sequence_, value_type, key, + Slice(prev_buffer, prev_size), + &updated_kv_prot_info); + } else { + ret_status = mem->Add(sequence_, value_type, key, + Slice(prev_buffer, prev_size), + nullptr /* kv_prot_info */); + } + if (ret_status.ok()) { + RecordTick(moptions->statistics, NUMBER_KEYS_WRITTEN); + } + } else if (update_status == UpdateStatus::UPDATED) { + if (kv_prot_info != nullptr) { + ProtectionInfoKVOS64 updated_kv_prot_info(*kv_prot_info); + updated_kv_prot_info.UpdateV(value, merged_value); + // merged_value contains the final value. + ret_status = mem->Add(sequence_, value_type, key, + Slice(merged_value), &updated_kv_prot_info); + } else { + // merged_value contains the final value. + ret_status = + mem->Add(sequence_, value_type, key, Slice(merged_value), + nullptr /* kv_prot_info */); + } + if (ret_status.ok()) { + RecordTick(moptions->statistics, NUMBER_KEYS_WRITTEN); + } + } + } + } + } + if (UNLIKELY(ret_status.IsTryAgain())) { + assert(seq_per_batch_); + const bool kBatchBoundary = true; + MaybeAdvanceSeq(kBatchBoundary); + } else if (ret_status.ok()) { + MaybeAdvanceSeq(); + CheckMemtableFull(); + } + // optimize for non-recovery mode + // If `ret_status` is `TryAgain` then the next (successful) try will add + // the key to the rebuilding transaction object. If `ret_status` is + // another non-OK `Status`, then the `rebuilding_trx_` will be thrown + // away. So we only need to add to it when `ret_status.ok()`. + if (UNLIKELY(ret_status.ok() && rebuilding_trx_ != nullptr)) { + assert(!write_after_commit_); + // TODO(ajkr): propagate `ProtectionInfoKVOS64`. + ret_status = WriteBatchInternal::Put(rebuilding_trx_, column_family_id, + key, value); + } + return ret_status; + } + + Status PutCF(uint32_t column_family_id, const Slice& key, + const Slice& value) override { + const auto* kv_prot_info = NextProtectionInfo(); + Status ret_status; + if (kv_prot_info != nullptr) { + // Memtable needs seqno, doesn't need CF ID + auto mem_kv_prot_info = + kv_prot_info->StripC(column_family_id).ProtectS(sequence_); + ret_status = PutCFImpl(column_family_id, key, value, kTypeValue, + &mem_kv_prot_info); + } else { + ret_status = PutCFImpl(column_family_id, key, value, kTypeValue, + nullptr /* kv_prot_info */); + } + // TODO: this assumes that if TryAgain status is returned to the caller, + // the operation is actually tried again. The proper way to do this is to + // pass a `try_again` parameter to the operation itself and decrement + // prot_info_idx_ based on that + if (UNLIKELY(ret_status.IsTryAgain())) { + DecrementProtectionInfoIdxForTryAgain(); + } + return ret_status; + } + + Status PutEntityCF(uint32_t column_family_id, const Slice& key, + const Slice& value) override { + const auto* kv_prot_info = NextProtectionInfo(); + + Status s; + if (kv_prot_info) { + // Memtable needs seqno, doesn't need CF ID + auto mem_kv_prot_info = + kv_prot_info->StripC(column_family_id).ProtectS(sequence_); + s = PutCFImpl(column_family_id, key, value, kTypeWideColumnEntity, + &mem_kv_prot_info); + } else { + s = PutCFImpl(column_family_id, key, value, kTypeWideColumnEntity, + /* kv_prot_info */ nullptr); + } + + if (UNLIKELY(s.IsTryAgain())) { + DecrementProtectionInfoIdxForTryAgain(); + } + + return s; + } + + Status DeleteImpl(uint32_t /*column_family_id*/, const Slice& key, + const Slice& value, ValueType delete_type, + const ProtectionInfoKVOS64* kv_prot_info) { + Status ret_status; + MemTable* mem = cf_mems_->GetMemTable(); + ret_status = + mem->Add(sequence_, delete_type, key, value, kv_prot_info, + concurrent_memtable_writes_, get_post_process_info(mem), + hint_per_batch_ ? &GetHintMap()[mem] : nullptr); + if (UNLIKELY(ret_status.IsTryAgain())) { + assert(seq_per_batch_); + const bool kBatchBoundary = true; + MaybeAdvanceSeq(kBatchBoundary); + } else if (ret_status.ok()) { + MaybeAdvanceSeq(); + CheckMemtableFull(); + } + return ret_status; + } + + Status DeleteCF(uint32_t column_family_id, const Slice& key) override { + const auto* kv_prot_info = NextProtectionInfo(); + // optimize for non-recovery mode + if (UNLIKELY(write_after_commit_ && rebuilding_trx_ != nullptr)) { + // TODO(ajkr): propagate `ProtectionInfoKVOS64`. + return WriteBatchInternal::Delete(rebuilding_trx_, column_family_id, key); + // else insert the values to the memtable right away + } + + Status ret_status; + if (UNLIKELY(!SeekToColumnFamily(column_family_id, &ret_status))) { + if (ret_status.ok() && rebuilding_trx_ != nullptr) { + assert(!write_after_commit_); + // The CF is probably flushed and hence no need for insert but we still + // need to keep track of the keys for upcoming rollback/commit. + // TODO(ajkr): propagate `ProtectionInfoKVOS64`. + ret_status = + WriteBatchInternal::Delete(rebuilding_trx_, column_family_id, key); + if (ret_status.ok()) { + MaybeAdvanceSeq(IsDuplicateKeySeq(column_family_id, key)); + } + } else if (ret_status.ok()) { + MaybeAdvanceSeq(false /* batch_boundary */); + } + if (UNLIKELY(ret_status.IsTryAgain())) { + DecrementProtectionInfoIdxForTryAgain(); + } + return ret_status; + } + + ColumnFamilyData* cfd = cf_mems_->current(); + assert(!cfd || cfd->user_comparator()); + const size_t ts_sz = (cfd && cfd->user_comparator()) + ? cfd->user_comparator()->timestamp_size() + : 0; + const ValueType delete_type = + (0 == ts_sz) ? kTypeDeletion : kTypeDeletionWithTimestamp; + if (kv_prot_info != nullptr) { + auto mem_kv_prot_info = + kv_prot_info->StripC(column_family_id).ProtectS(sequence_); + mem_kv_prot_info.UpdateO(kTypeDeletion, delete_type); + ret_status = DeleteImpl(column_family_id, key, Slice(), delete_type, + &mem_kv_prot_info); + } else { + ret_status = DeleteImpl(column_family_id, key, Slice(), delete_type, + nullptr /* kv_prot_info */); + } + // optimize for non-recovery mode + // If `ret_status` is `TryAgain` then the next (successful) try will add + // the key to the rebuilding transaction object. If `ret_status` is + // another non-OK `Status`, then the `rebuilding_trx_` will be thrown + // away. So we only need to add to it when `ret_status.ok()`. + if (UNLIKELY(ret_status.ok() && rebuilding_trx_ != nullptr)) { + assert(!write_after_commit_); + // TODO(ajkr): propagate `ProtectionInfoKVOS64`. + ret_status = + WriteBatchInternal::Delete(rebuilding_trx_, column_family_id, key); + } + if (UNLIKELY(ret_status.IsTryAgain())) { + DecrementProtectionInfoIdxForTryAgain(); + } + return ret_status; + } + + Status SingleDeleteCF(uint32_t column_family_id, const Slice& key) override { + const auto* kv_prot_info = NextProtectionInfo(); + // optimize for non-recovery mode + if (UNLIKELY(write_after_commit_ && rebuilding_trx_ != nullptr)) { + // TODO(ajkr): propagate `ProtectionInfoKVOS64`. + return WriteBatchInternal::SingleDelete(rebuilding_trx_, column_family_id, + key); + // else insert the values to the memtable right away + } + + Status ret_status; + if (UNLIKELY(!SeekToColumnFamily(column_family_id, &ret_status))) { + if (ret_status.ok() && rebuilding_trx_ != nullptr) { + assert(!write_after_commit_); + // The CF is probably flushed and hence no need for insert but we still + // need to keep track of the keys for upcoming rollback/commit. + // TODO(ajkr): propagate `ProtectionInfoKVOS64`. + ret_status = WriteBatchInternal::SingleDelete(rebuilding_trx_, + column_family_id, key); + if (ret_status.ok()) { + MaybeAdvanceSeq(IsDuplicateKeySeq(column_family_id, key)); + } + } else if (ret_status.ok()) { + MaybeAdvanceSeq(false /* batch_boundary */); + } + if (UNLIKELY(ret_status.IsTryAgain())) { + DecrementProtectionInfoIdxForTryAgain(); + } + return ret_status; + } + assert(ret_status.ok()); + + if (kv_prot_info != nullptr) { + auto mem_kv_prot_info = + kv_prot_info->StripC(column_family_id).ProtectS(sequence_); + ret_status = DeleteImpl(column_family_id, key, Slice(), + kTypeSingleDeletion, &mem_kv_prot_info); + } else { + ret_status = DeleteImpl(column_family_id, key, Slice(), + kTypeSingleDeletion, nullptr /* kv_prot_info */); + } + // optimize for non-recovery mode + // If `ret_status` is `TryAgain` then the next (successful) try will add + // the key to the rebuilding transaction object. If `ret_status` is + // another non-OK `Status`, then the `rebuilding_trx_` will be thrown + // away. So we only need to add to it when `ret_status.ok()`. + if (UNLIKELY(ret_status.ok() && rebuilding_trx_ != nullptr)) { + assert(!write_after_commit_); + // TODO(ajkr): propagate `ProtectionInfoKVOS64`. + ret_status = WriteBatchInternal::SingleDelete(rebuilding_trx_, + column_family_id, key); + } + if (UNLIKELY(ret_status.IsTryAgain())) { + DecrementProtectionInfoIdxForTryAgain(); + } + return ret_status; + } + + Status DeleteRangeCF(uint32_t column_family_id, const Slice& begin_key, + const Slice& end_key) override { + const auto* kv_prot_info = NextProtectionInfo(); + // optimize for non-recovery mode + if (UNLIKELY(write_after_commit_ && rebuilding_trx_ != nullptr)) { + // TODO(ajkr): propagate `ProtectionInfoKVOS64`. + return WriteBatchInternal::DeleteRange(rebuilding_trx_, column_family_id, + begin_key, end_key); + // else insert the values to the memtable right away + } + + Status ret_status; + if (UNLIKELY(!SeekToColumnFamily(column_family_id, &ret_status))) { + if (ret_status.ok() && rebuilding_trx_ != nullptr) { + assert(!write_after_commit_); + // The CF is probably flushed and hence no need for insert but we still + // need to keep track of the keys for upcoming rollback/commit. + // TODO(ajkr): propagate `ProtectionInfoKVOS64`. + ret_status = WriteBatchInternal::DeleteRange( + rebuilding_trx_, column_family_id, begin_key, end_key); + if (ret_status.ok()) { + MaybeAdvanceSeq(IsDuplicateKeySeq(column_family_id, begin_key)); + } + } else if (ret_status.ok()) { + MaybeAdvanceSeq(false /* batch_boundary */); + } + if (UNLIKELY(ret_status.IsTryAgain())) { + DecrementProtectionInfoIdxForTryAgain(); + } + return ret_status; + } + assert(ret_status.ok()); + + if (db_ != nullptr) { + auto cf_handle = cf_mems_->GetColumnFamilyHandle(); + if (cf_handle == nullptr) { + cf_handle = db_->DefaultColumnFamily(); + } + auto* cfd = + static_cast_with_check(cf_handle)->cfd(); + if (!cfd->is_delete_range_supported()) { + // TODO(ajkr): refactor `SeekToColumnFamily()` so it returns a `Status`. + ret_status.PermitUncheckedError(); + return Status::NotSupported( + std::string("DeleteRange not supported for table type ") + + cfd->ioptions()->table_factory->Name() + " in CF " + + cfd->GetName()); + } + int cmp = + cfd->user_comparator()->CompareWithoutTimestamp(begin_key, end_key); + if (cmp > 0) { + // TODO(ajkr): refactor `SeekToColumnFamily()` so it returns a `Status`. + ret_status.PermitUncheckedError(); + // It's an empty range where endpoints appear mistaken. Don't bother + // applying it to the DB, and return an error to the user. + return Status::InvalidArgument("end key comes before start key"); + } else if (cmp == 0) { + // TODO(ajkr): refactor `SeekToColumnFamily()` so it returns a `Status`. + ret_status.PermitUncheckedError(); + // It's an empty range. Don't bother applying it to the DB. + return Status::OK(); + } + } + + if (kv_prot_info != nullptr) { + auto mem_kv_prot_info = + kv_prot_info->StripC(column_family_id).ProtectS(sequence_); + ret_status = DeleteImpl(column_family_id, begin_key, end_key, + kTypeRangeDeletion, &mem_kv_prot_info); + } else { + ret_status = DeleteImpl(column_family_id, begin_key, end_key, + kTypeRangeDeletion, nullptr /* kv_prot_info */); + } + // optimize for non-recovery mode + // If `ret_status` is `TryAgain` then the next (successful) try will add + // the key to the rebuilding transaction object. If `ret_status` is + // another non-OK `Status`, then the `rebuilding_trx_` will be thrown + // away. So we only need to add to it when `ret_status.ok()`. + if (UNLIKELY(!ret_status.IsTryAgain() && rebuilding_trx_ != nullptr)) { + assert(!write_after_commit_); + // TODO(ajkr): propagate `ProtectionInfoKVOS64`. + ret_status = WriteBatchInternal::DeleteRange( + rebuilding_trx_, column_family_id, begin_key, end_key); + } + if (UNLIKELY(ret_status.IsTryAgain())) { + DecrementProtectionInfoIdxForTryAgain(); + } + return ret_status; + } + + Status MergeCF(uint32_t column_family_id, const Slice& key, + const Slice& value) override { + const auto* kv_prot_info = NextProtectionInfo(); + // optimize for non-recovery mode + if (UNLIKELY(write_after_commit_ && rebuilding_trx_ != nullptr)) { + // TODO(ajkr): propagate `ProtectionInfoKVOS64`. + return WriteBatchInternal::Merge(rebuilding_trx_, column_family_id, key, + value); + // else insert the values to the memtable right away + } + + Status ret_status; + if (UNLIKELY(!SeekToColumnFamily(column_family_id, &ret_status))) { + if (ret_status.ok() && rebuilding_trx_ != nullptr) { + assert(!write_after_commit_); + // The CF is probably flushed and hence no need for insert but we still + // need to keep track of the keys for upcoming rollback/commit. + // TODO(ajkr): propagate `ProtectionInfoKVOS64`. + ret_status = WriteBatchInternal::Merge(rebuilding_trx_, + column_family_id, key, value); + if (ret_status.ok()) { + MaybeAdvanceSeq(IsDuplicateKeySeq(column_family_id, key)); + } + } else if (ret_status.ok()) { + MaybeAdvanceSeq(false /* batch_boundary */); + } + if (UNLIKELY(ret_status.IsTryAgain())) { + DecrementProtectionInfoIdxForTryAgain(); + } + return ret_status; + } + assert(ret_status.ok()); + + MemTable* mem = cf_mems_->GetMemTable(); + auto* moptions = mem->GetImmutableMemTableOptions(); + if (moptions->merge_operator == nullptr) { + return Status::InvalidArgument( + "Merge requires `ColumnFamilyOptions::merge_operator != nullptr`"); + } + bool perform_merge = false; + assert(!concurrent_memtable_writes_ || + moptions->max_successive_merges == 0); + + // If we pass DB through and options.max_successive_merges is hit + // during recovery, Get() will be issued which will try to acquire + // DB mutex and cause deadlock, as DB mutex is already held. + // So we disable merge in recovery + if (moptions->max_successive_merges > 0 && db_ != nullptr && + recovering_log_number_ == 0) { + assert(!concurrent_memtable_writes_); + LookupKey lkey(key, sequence_); + + // Count the number of successive merges at the head + // of the key in the memtable + size_t num_merges = mem->CountSuccessiveMergeEntries(lkey); + + if (num_merges >= moptions->max_successive_merges) { + perform_merge = true; + } + } + + if (perform_merge) { + // 1) Get the existing value + std::string get_value; + + // Pass in the sequence number so that we also include previous merge + // operations in the same batch. + SnapshotImpl read_from_snapshot; + read_from_snapshot.number_ = sequence_; + // TODO: plumb Env::IOActivity + ReadOptions read_options; + read_options.snapshot = &read_from_snapshot; + + auto cf_handle = cf_mems_->GetColumnFamilyHandle(); + if (cf_handle == nullptr) { + cf_handle = db_->DefaultColumnFamily(); + } + Status get_status = db_->Get(read_options, cf_handle, key, &get_value); + if (!get_status.ok()) { + // Failed to read a key we know exists. Store the delta in memtable. + perform_merge = false; + } else { + Slice get_value_slice = Slice(get_value); + + // 2) Apply this merge + auto merge_operator = moptions->merge_operator; + assert(merge_operator); + + std::string new_value; + // `op_failure_scope` (an output parameter) is not provided (set to + // nullptr) since a failure must be propagated regardless of its value. + Status merge_status = MergeHelper::TimedFullMerge( + merge_operator, key, &get_value_slice, {value}, &new_value, + moptions->info_log, moptions->statistics, + SystemClock::Default().get(), /* result_operand */ nullptr, + /* update_num_ops_stats */ false, + /* op_failure_scope */ nullptr); + + if (!merge_status.ok()) { + // Failed to merge! + // Store the delta in memtable + perform_merge = false; + } else { + // 3) Add value to memtable + assert(!concurrent_memtable_writes_); + if (kv_prot_info != nullptr) { + auto merged_kv_prot_info = + kv_prot_info->StripC(column_family_id).ProtectS(sequence_); + merged_kv_prot_info.UpdateV(value, new_value); + merged_kv_prot_info.UpdateO(kTypeMerge, kTypeValue); + ret_status = mem->Add(sequence_, kTypeValue, key, new_value, + &merged_kv_prot_info); + } else { + ret_status = mem->Add(sequence_, kTypeValue, key, new_value, + nullptr /* kv_prot_info */); + } + } + } + } + + if (!perform_merge) { + assert(ret_status.ok()); + // Add merge operand to memtable + if (kv_prot_info != nullptr) { + auto mem_kv_prot_info = + kv_prot_info->StripC(column_family_id).ProtectS(sequence_); + ret_status = + mem->Add(sequence_, kTypeMerge, key, value, &mem_kv_prot_info, + concurrent_memtable_writes_, get_post_process_info(mem)); + } else { + ret_status = mem->Add( + sequence_, kTypeMerge, key, value, nullptr /* kv_prot_info */, + concurrent_memtable_writes_, get_post_process_info(mem)); + } + } + + if (UNLIKELY(ret_status.IsTryAgain())) { + assert(seq_per_batch_); + const bool kBatchBoundary = true; + MaybeAdvanceSeq(kBatchBoundary); + } else if (ret_status.ok()) { + MaybeAdvanceSeq(); + CheckMemtableFull(); + } + // optimize for non-recovery mode + // If `ret_status` is `TryAgain` then the next (successful) try will add + // the key to the rebuilding transaction object. If `ret_status` is + // another non-OK `Status`, then the `rebuilding_trx_` will be thrown + // away. So we only need to add to it when `ret_status.ok()`. + if (UNLIKELY(ret_status.ok() && rebuilding_trx_ != nullptr)) { + assert(!write_after_commit_); + // TODO(ajkr): propagate `ProtectionInfoKVOS64`. + ret_status = WriteBatchInternal::Merge(rebuilding_trx_, column_family_id, + key, value); + } + if (UNLIKELY(ret_status.IsTryAgain())) { + DecrementProtectionInfoIdxForTryAgain(); + } + return ret_status; + } + + Status PutBlobIndexCF(uint32_t column_family_id, const Slice& key, + const Slice& value) override { + const auto* kv_prot_info = NextProtectionInfo(); + Status ret_status; + if (kv_prot_info != nullptr) { + // Memtable needs seqno, doesn't need CF ID + auto mem_kv_prot_info = + kv_prot_info->StripC(column_family_id).ProtectS(sequence_); + // Same as PutCF except for value type. + ret_status = PutCFImpl(column_family_id, key, value, kTypeBlobIndex, + &mem_kv_prot_info); + } else { + ret_status = PutCFImpl(column_family_id, key, value, kTypeBlobIndex, + nullptr /* kv_prot_info */); + } + if (UNLIKELY(ret_status.IsTryAgain())) { + DecrementProtectionInfoIdxForTryAgain(); + } + return ret_status; + } + + void CheckMemtableFull() { + if (flush_scheduler_ != nullptr) { + auto* cfd = cf_mems_->current(); + assert(cfd != nullptr); + if (cfd->mem()->ShouldScheduleFlush() && + cfd->mem()->MarkFlushScheduled()) { + // MarkFlushScheduled only returns true if we are the one that + // should take action, so no need to dedup further + flush_scheduler_->ScheduleWork(cfd); + } + } + // check if memtable_list size exceeds max_write_buffer_size_to_maintain + if (trim_history_scheduler_ != nullptr) { + auto* cfd = cf_mems_->current(); + + assert(cfd); + assert(cfd->ioptions()); + + const size_t size_to_maintain = static_cast( + cfd->ioptions()->max_write_buffer_size_to_maintain); + + if (size_to_maintain > 0) { + MemTableList* const imm = cfd->imm(); + assert(imm); + + if (imm->HasHistory()) { + const MemTable* const mem = cfd->mem(); + assert(mem); + + if (mem->MemoryAllocatedBytes() + + imm->MemoryAllocatedBytesExcludingLast() >= + size_to_maintain && + imm->MarkTrimHistoryNeeded()) { + trim_history_scheduler_->ScheduleWork(cfd); + } + } + } + } + } + + // The write batch handler calls MarkBeginPrepare with unprepare set to true + // if it encounters the kTypeBeginUnprepareXID marker. + Status MarkBeginPrepare(bool unprepare) override { + assert(rebuilding_trx_ == nullptr); + assert(db_); + + if (recovering_log_number_ != 0) { + db_->mutex()->AssertHeld(); + // during recovery we rebuild a hollow transaction + // from all encountered prepare sections of the wal + if (db_->allow_2pc() == false) { + return Status::NotSupported( + "WAL contains prepared transactions. Open with " + "TransactionDB::Open()."); + } + + // we are now iterating through a prepared section + rebuilding_trx_ = new WriteBatch(); + rebuilding_trx_seq_ = sequence_; + // Verify that we have matching MarkBeginPrepare/MarkEndPrepare markers. + // unprepared_batch_ should be false because it is false by default, and + // gets reset to false in MarkEndPrepare. + assert(!unprepared_batch_); + unprepared_batch_ = unprepare; + + if (has_valid_writes_ != nullptr) { + *has_valid_writes_ = true; + } + } + + return Status::OK(); + } + + Status MarkEndPrepare(const Slice& name) override { + assert(db_); + assert((rebuilding_trx_ != nullptr) == (recovering_log_number_ != 0)); + + if (recovering_log_number_ != 0) { + db_->mutex()->AssertHeld(); + assert(db_->allow_2pc()); + size_t batch_cnt = + write_after_commit_ + ? 0 // 0 will disable further checks + : static_cast(sequence_ - rebuilding_trx_seq_ + 1); + db_->InsertRecoveredTransaction(recovering_log_number_, name.ToString(), + rebuilding_trx_, rebuilding_trx_seq_, + batch_cnt, unprepared_batch_); + unprepared_batch_ = false; + rebuilding_trx_ = nullptr; + } else { + assert(rebuilding_trx_ == nullptr); + } + const bool batch_boundry = true; + MaybeAdvanceSeq(batch_boundry); + + return Status::OK(); + } + + Status MarkNoop(bool empty_batch) override { + if (recovering_log_number_ != 0) { + db_->mutex()->AssertHeld(); + } + // A hack in pessimistic transaction could result into a noop at the start + // of the write batch, that should be ignored. + if (!empty_batch) { + // In the absence of Prepare markers, a kTypeNoop tag indicates the end of + // a batch. This happens when write batch commits skipping the prepare + // phase. + const bool batch_boundry = true; + MaybeAdvanceSeq(batch_boundry); + } + return Status::OK(); + } + + Status MarkCommit(const Slice& name) override { + assert(db_); + + Status s; + + if (recovering_log_number_ != 0) { + // We must hold db mutex in recovery. + db_->mutex()->AssertHeld(); + // in recovery when we encounter a commit marker + // we lookup this transaction in our set of rebuilt transactions + // and commit. + auto trx = db_->GetRecoveredTransaction(name.ToString()); + + // the log containing the prepared section may have + // been released in the last incarnation because the + // data was flushed to L0 + if (trx != nullptr) { + // at this point individual CF lognumbers will prevent + // duplicate re-insertion of values. + assert(log_number_ref_ == 0); + if (write_after_commit_) { + // write_after_commit_ can only have one batch in trx. + assert(trx->batches_.size() == 1); + const auto& batch_info = trx->batches_.begin()->second; + // all inserts must reference this trx log number + log_number_ref_ = batch_info.log_number_; + ResetProtectionInfo(); + s = batch_info.batch_->Iterate(this); + log_number_ref_ = 0; + } + // else the values are already inserted before the commit + + if (s.ok()) { + db_->DeleteRecoveredTransaction(name.ToString()); + } + if (has_valid_writes_ != nullptr) { + *has_valid_writes_ = true; + } + } + } else { + // When writes are not delayed until commit, there is no disconnect + // between a memtable write and the WAL that supports it. So the commit + // need not reference any log as the only log to which it depends. + assert(!write_after_commit_ || log_number_ref_ > 0); + } + const bool batch_boundry = true; + MaybeAdvanceSeq(batch_boundry); + + if (UNLIKELY(s.IsTryAgain())) { + DecrementProtectionInfoIdxForTryAgain(); + } + + return s; + } + + Status MarkCommitWithTimestamp(const Slice& name, + const Slice& commit_ts) override { + assert(db_); + + Status s; + + if (recovering_log_number_ != 0) { + // In recovery, db mutex must be held. + db_->mutex()->AssertHeld(); + // in recovery when we encounter a commit marker + // we lookup this transaction in our set of rebuilt transactions + // and commit. + auto trx = db_->GetRecoveredTransaction(name.ToString()); + // the log containing the prepared section may have + // been released in the last incarnation because the + // data was flushed to L0 + if (trx) { + // at this point individual CF lognumbers will prevent + // duplicate re-insertion of values. + assert(0 == log_number_ref_); + if (write_after_commit_) { + // write_after_commit_ can only have one batch in trx. + assert(trx->batches_.size() == 1); + const auto& batch_info = trx->batches_.begin()->second; + // all inserts must reference this trx log number + log_number_ref_ = batch_info.log_number_; + + s = batch_info.batch_->UpdateTimestamps( + commit_ts, [this](uint32_t cf) { + assert(db_); + VersionSet* const vset = db_->GetVersionSet(); + assert(vset); + ColumnFamilySet* const cf_set = vset->GetColumnFamilySet(); + assert(cf_set); + ColumnFamilyData* cfd = cf_set->GetColumnFamily(cf); + assert(cfd); + const auto* const ucmp = cfd->user_comparator(); + assert(ucmp); + return ucmp->timestamp_size(); + }); + if (s.ok()) { + ResetProtectionInfo(); + s = batch_info.batch_->Iterate(this); + log_number_ref_ = 0; + } + } + // else the values are already inserted before the commit + + if (s.ok()) { + db_->DeleteRecoveredTransaction(name.ToString()); + } + if (has_valid_writes_) { + *has_valid_writes_ = true; + } + } + } else { + // When writes are not delayed until commit, there is no connection + // between a memtable write and the WAL that supports it. So the commit + // need not reference any log as the only log to which it depends. + assert(!write_after_commit_ || log_number_ref_ > 0); + } + constexpr bool batch_boundary = true; + MaybeAdvanceSeq(batch_boundary); + + if (UNLIKELY(s.IsTryAgain())) { + DecrementProtectionInfoIdxForTryAgain(); + } + + return s; + } + + Status MarkRollback(const Slice& name) override { + assert(db_); + + if (recovering_log_number_ != 0) { + auto trx = db_->GetRecoveredTransaction(name.ToString()); + + // the log containing the transactions prep section + // may have been released in the previous incarnation + // because we knew it had been rolled back + if (trx != nullptr) { + db_->DeleteRecoveredTransaction(name.ToString()); + } + } else { + // in non recovery we simply ignore this tag + } + + const bool batch_boundry = true; + MaybeAdvanceSeq(batch_boundry); + + return Status::OK(); + } + + private: + MemTablePostProcessInfo* get_post_process_info(MemTable* mem) { + if (!concurrent_memtable_writes_) { + // No need to batch counters locally if we don't use concurrent mode. + return nullptr; + } + return &GetPostMap()[mem]; + } +}; + +} // anonymous namespace + +// This function can only be called in these conditions: +// 1) During Recovery() +// 2) During Write(), in a single-threaded write thread +// 3) During Write(), in a concurrent context where memtables has been cloned +// The reason is that it calls memtables->Seek(), which has a stateful cache +Status WriteBatchInternal::InsertInto( + WriteThread::WriteGroup& write_group, SequenceNumber sequence, + ColumnFamilyMemTables* memtables, FlushScheduler* flush_scheduler, + TrimHistoryScheduler* trim_history_scheduler, + bool ignore_missing_column_families, uint64_t recovery_log_number, DB* db, + bool concurrent_memtable_writes, bool seq_per_batch, bool batch_per_txn) { + MemTableInserter inserter( + sequence, memtables, flush_scheduler, trim_history_scheduler, + ignore_missing_column_families, recovery_log_number, db, + concurrent_memtable_writes, nullptr /* prot_info */, + nullptr /*has_valid_writes*/, seq_per_batch, batch_per_txn); + for (auto w : write_group) { + if (w->CallbackFailed()) { + continue; + } + w->sequence = inserter.sequence(); + if (!w->ShouldWriteToMemtable()) { + // In seq_per_batch_ mode this advances the seq by one. + inserter.MaybeAdvanceSeq(true); + continue; + } + SetSequence(w->batch, inserter.sequence()); + inserter.set_log_number_ref(w->log_ref); + inserter.set_prot_info(w->batch->prot_info_.get()); + w->status = w->batch->Iterate(&inserter); + if (!w->status.ok()) { + return w->status; + } + assert(!seq_per_batch || w->batch_cnt != 0); + assert(!seq_per_batch || inserter.sequence() - w->sequence == w->batch_cnt); + } + return Status::OK(); +} + +Status WriteBatchInternal::InsertInto( + WriteThread::Writer* writer, SequenceNumber sequence, + ColumnFamilyMemTables* memtables, FlushScheduler* flush_scheduler, + TrimHistoryScheduler* trim_history_scheduler, + bool ignore_missing_column_families, uint64_t log_number, DB* db, + bool concurrent_memtable_writes, bool seq_per_batch, size_t batch_cnt, + bool batch_per_txn, bool hint_per_batch) { +#ifdef NDEBUG + (void)batch_cnt; +#endif + assert(writer->ShouldWriteToMemtable()); + MemTableInserter inserter(sequence, memtables, flush_scheduler, + trim_history_scheduler, + ignore_missing_column_families, log_number, db, + concurrent_memtable_writes, nullptr /* prot_info */, + nullptr /*has_valid_writes*/, seq_per_batch, + batch_per_txn, hint_per_batch); + SetSequence(writer->batch, sequence); + inserter.set_log_number_ref(writer->log_ref); + inserter.set_prot_info(writer->batch->prot_info_.get()); + Status s = writer->batch->Iterate(&inserter); + assert(!seq_per_batch || batch_cnt != 0); + assert(!seq_per_batch || inserter.sequence() - sequence == batch_cnt); + if (concurrent_memtable_writes) { + inserter.PostProcess(); + } + return s; +} + +Status WriteBatchInternal::InsertInto( + const WriteBatch* batch, ColumnFamilyMemTables* memtables, + FlushScheduler* flush_scheduler, + TrimHistoryScheduler* trim_history_scheduler, + bool ignore_missing_column_families, uint64_t log_number, DB* db, + bool concurrent_memtable_writes, SequenceNumber* next_seq, + bool* has_valid_writes, bool seq_per_batch, bool batch_per_txn) { + MemTableInserter inserter(Sequence(batch), memtables, flush_scheduler, + trim_history_scheduler, + ignore_missing_column_families, log_number, db, + concurrent_memtable_writes, batch->prot_info_.get(), + has_valid_writes, seq_per_batch, batch_per_txn); + Status s = batch->Iterate(&inserter); + if (next_seq != nullptr) { + *next_seq = inserter.sequence(); + } + if (concurrent_memtable_writes) { + inserter.PostProcess(); + } + return s; +} + +namespace { + +// This class updates protection info for a WriteBatch. +class ProtectionInfoUpdater : public WriteBatch::Handler { + public: + explicit ProtectionInfoUpdater(WriteBatch::ProtectionInfo* prot_info) + : prot_info_(prot_info) {} + + ~ProtectionInfoUpdater() override {} + + Status PutCF(uint32_t cf, const Slice& key, const Slice& val) override { + return UpdateProtInfo(cf, key, val, kTypeValue); + } + + Status PutEntityCF(uint32_t cf, const Slice& key, + const Slice& entity) override { + return UpdateProtInfo(cf, key, entity, kTypeWideColumnEntity); + } + + Status DeleteCF(uint32_t cf, const Slice& key) override { + return UpdateProtInfo(cf, key, "", kTypeDeletion); + } + + Status SingleDeleteCF(uint32_t cf, const Slice& key) override { + return UpdateProtInfo(cf, key, "", kTypeSingleDeletion); + } + + Status DeleteRangeCF(uint32_t cf, const Slice& begin_key, + const Slice& end_key) override { + return UpdateProtInfo(cf, begin_key, end_key, kTypeRangeDeletion); + } + + Status MergeCF(uint32_t cf, const Slice& key, const Slice& val) override { + return UpdateProtInfo(cf, key, val, kTypeMerge); + } + + Status PutBlobIndexCF(uint32_t cf, const Slice& key, + const Slice& val) override { + return UpdateProtInfo(cf, key, val, kTypeBlobIndex); + } + + Status MarkBeginPrepare(bool /* unprepare */) override { + return Status::OK(); + } + + Status MarkEndPrepare(const Slice& /* xid */) override { + return Status::OK(); + } + + Status MarkCommit(const Slice& /* xid */) override { return Status::OK(); } + + Status MarkCommitWithTimestamp(const Slice& /* xid */, + const Slice& /* ts */) override { + return Status::OK(); + } + + Status MarkRollback(const Slice& /* xid */) override { return Status::OK(); } + + Status MarkNoop(bool /* empty_batch */) override { return Status::OK(); } + + private: + Status UpdateProtInfo(uint32_t cf, const Slice& key, const Slice& val, + const ValueType op_type) { + if (prot_info_) { + prot_info_->entries_.emplace_back( + ProtectionInfo64().ProtectKVO(key, val, op_type).ProtectC(cf)); + } + return Status::OK(); + } + + // No copy or move. + ProtectionInfoUpdater(const ProtectionInfoUpdater&) = delete; + ProtectionInfoUpdater(ProtectionInfoUpdater&&) = delete; + ProtectionInfoUpdater& operator=(const ProtectionInfoUpdater&) = delete; + ProtectionInfoUpdater& operator=(ProtectionInfoUpdater&&) = delete; + + WriteBatch::ProtectionInfo* const prot_info_ = nullptr; +}; + +} // anonymous namespace + +Status WriteBatchInternal::SetContents(WriteBatch* b, const Slice& contents) { + assert(contents.size() >= WriteBatchInternal::kHeader); + assert(b->prot_info_ == nullptr); + + b->rep_.assign(contents.data(), contents.size()); + b->content_flags_.store(ContentFlags::DEFERRED, std::memory_order_relaxed); + return Status::OK(); +} + +Status WriteBatchInternal::Append(WriteBatch* dst, const WriteBatch* src, + const bool wal_only) { + assert(dst->Count() == 0 || + (dst->prot_info_ == nullptr) == (src->prot_info_ == nullptr)); + if ((src->prot_info_ != nullptr && + src->prot_info_->entries_.size() != src->Count()) || + (dst->prot_info_ != nullptr && + dst->prot_info_->entries_.size() != dst->Count())) { + return Status::Corruption( + "Write batch has inconsistent count and number of checksums"); + } + + size_t src_len; + int src_count; + uint32_t src_flags; + + const SavePoint& batch_end = src->GetWalTerminationPoint(); + + if (wal_only && !batch_end.is_cleared()) { + src_len = batch_end.size - WriteBatchInternal::kHeader; + src_count = batch_end.count; + src_flags = batch_end.content_flags; + } else { + src_len = src->rep_.size() - WriteBatchInternal::kHeader; + src_count = Count(src); + src_flags = src->content_flags_.load(std::memory_order_relaxed); + } + + if (src->prot_info_ != nullptr) { + if (dst->prot_info_ == nullptr) { + dst->prot_info_.reset(new WriteBatch::ProtectionInfo()); + } + std::copy(src->prot_info_->entries_.begin(), + src->prot_info_->entries_.begin() + src_count, + std::back_inserter(dst->prot_info_->entries_)); + } else if (dst->prot_info_ != nullptr) { + // dst has empty prot_info->entries + // In this special case, we allow write batch without prot_info to + // be appende to write batch with empty prot_info + dst->prot_info_ = nullptr; + } + SetCount(dst, Count(dst) + src_count); + assert(src->rep_.size() >= WriteBatchInternal::kHeader); + dst->rep_.append(src->rep_.data() + WriteBatchInternal::kHeader, src_len); + dst->content_flags_.store( + dst->content_flags_.load(std::memory_order_relaxed) | src_flags, + std::memory_order_relaxed); + return Status::OK(); +} + +size_t WriteBatchInternal::AppendedByteSize(size_t leftByteSize, + size_t rightByteSize) { + if (leftByteSize == 0 || rightByteSize == 0) { + return leftByteSize + rightByteSize; + } else { + return leftByteSize + rightByteSize - WriteBatchInternal::kHeader; + } +} + +Status WriteBatchInternal::UpdateProtectionInfo(WriteBatch* wb, + size_t bytes_per_key, + uint64_t* checksum) { + if (bytes_per_key == 0) { + if (wb->prot_info_ != nullptr) { + wb->prot_info_.reset(); + return Status::OK(); + } else { + // Already not protected. + return Status::OK(); + } + } else if (bytes_per_key == 8) { + if (wb->prot_info_ == nullptr) { + wb->prot_info_.reset(new WriteBatch::ProtectionInfo()); + ProtectionInfoUpdater prot_info_updater(wb->prot_info_.get()); + Status s = wb->Iterate(&prot_info_updater); + if (s.ok() && checksum != nullptr) { + uint64_t expected_hash = XXH3_64bits(wb->rep_.data(), wb->rep_.size()); + if (expected_hash != *checksum) { + return Status::Corruption("Write batch content corrupted."); + } + } + return s; + } else { + // Already protected. + return Status::OK(); + } + } + return Status::NotSupported( + "WriteBatch protection info must be zero or eight bytes/key"); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/write_batch_base.cc b/librocksdb-sys/rocksdb/db/write_batch_base.cc new file mode 100644 index 0000000..e4c0e74 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/write_batch_base.cc @@ -0,0 +1,94 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "rocksdb/write_batch_base.h" + +#include + +#include "rocksdb/slice.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +// Simple implementation of SlicePart variants of Put(). Child classes +// can override these method with more performant solutions if they choose. +Status WriteBatchBase::Put(ColumnFamilyHandle* column_family, + const SliceParts& key, const SliceParts& value) { + std::string key_buf, value_buf; + Slice key_slice(key, &key_buf); + Slice value_slice(value, &value_buf); + + return Put(column_family, key_slice, value_slice); +} + +Status WriteBatchBase::Put(const SliceParts& key, const SliceParts& value) { + std::string key_buf, value_buf; + Slice key_slice(key, &key_buf); + Slice value_slice(value, &value_buf); + + return Put(key_slice, value_slice); +} + +Status WriteBatchBase::Delete(ColumnFamilyHandle* column_family, + const SliceParts& key) { + std::string key_buf; + Slice key_slice(key, &key_buf); + return Delete(column_family, key_slice); +} + +Status WriteBatchBase::Delete(const SliceParts& key) { + std::string key_buf; + Slice key_slice(key, &key_buf); + return Delete(key_slice); +} + +Status WriteBatchBase::SingleDelete(ColumnFamilyHandle* column_family, + const SliceParts& key) { + std::string key_buf; + Slice key_slice(key, &key_buf); + return SingleDelete(column_family, key_slice); +} + +Status WriteBatchBase::SingleDelete(const SliceParts& key) { + std::string key_buf; + Slice key_slice(key, &key_buf); + return SingleDelete(key_slice); +} + +Status WriteBatchBase::DeleteRange(ColumnFamilyHandle* column_family, + const SliceParts& begin_key, + const SliceParts& end_key) { + std::string begin_key_buf, end_key_buf; + Slice begin_key_slice(begin_key, &begin_key_buf); + Slice end_key_slice(end_key, &end_key_buf); + return DeleteRange(column_family, begin_key_slice, end_key_slice); +} + +Status WriteBatchBase::DeleteRange(const SliceParts& begin_key, + const SliceParts& end_key) { + std::string begin_key_buf, end_key_buf; + Slice begin_key_slice(begin_key, &begin_key_buf); + Slice end_key_slice(end_key, &end_key_buf); + return DeleteRange(begin_key_slice, end_key_slice); +} + +Status WriteBatchBase::Merge(ColumnFamilyHandle* column_family, + const SliceParts& key, const SliceParts& value) { + std::string key_buf, value_buf; + Slice key_slice(key, &key_buf); + Slice value_slice(value, &value_buf); + + return Merge(column_family, key_slice, value_slice); +} + +Status WriteBatchBase::Merge(const SliceParts& key, const SliceParts& value) { + std::string key_buf, value_buf; + Slice key_slice(key, &key_buf); + Slice value_slice(value, &value_buf); + + return Merge(key_slice, value_slice); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/write_batch_internal.h b/librocksdb-sys/rocksdb/db/write_batch_internal.h new file mode 100644 index 0000000..1be0bd1 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/write_batch_internal.h @@ -0,0 +1,401 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include +#include + +#include "db/flush_scheduler.h" +#include "db/kv_checksum.h" +#include "db/trim_history_scheduler.h" +#include "db/write_thread.h" +#include "rocksdb/db.h" +#include "rocksdb/options.h" +#include "rocksdb/types.h" +#include "rocksdb/write_batch.h" +#include "util/autovector.h" +#include "util/cast_util.h" + +namespace ROCKSDB_NAMESPACE { + +class MemTable; +class FlushScheduler; +class ColumnFamilyData; + +class ColumnFamilyMemTables { + public: + virtual ~ColumnFamilyMemTables() {} + virtual bool Seek(uint32_t column_family_id) = 0; + // returns true if the update to memtable should be ignored + // (useful when recovering from log whose updates have already + // been processed) + virtual uint64_t GetLogNumber() const = 0; + virtual MemTable* GetMemTable() const = 0; + virtual ColumnFamilyHandle* GetColumnFamilyHandle() = 0; + virtual ColumnFamilyData* current() { return nullptr; } +}; + +class ColumnFamilyMemTablesDefault : public ColumnFamilyMemTables { + public: + explicit ColumnFamilyMemTablesDefault(MemTable* mem) + : ok_(false), mem_(mem) {} + + bool Seek(uint32_t column_family_id) override { + ok_ = (column_family_id == 0); + return ok_; + } + + uint64_t GetLogNumber() const override { return 0; } + + MemTable* GetMemTable() const override { + assert(ok_); + return mem_; + } + + ColumnFamilyHandle* GetColumnFamilyHandle() override { return nullptr; } + + private: + bool ok_; + MemTable* mem_; +}; + +struct WriteBatch::ProtectionInfo { + // `WriteBatch` usually doesn't contain a huge number of keys so protecting + // with a fixed, non-configurable eight bytes per key may work well enough. + autovector entries_; + + size_t GetBytesPerKey() const { return 8; } +}; + +// WriteBatchInternal provides static methods for manipulating a +// WriteBatch that we don't want in the public WriteBatch interface. +class WriteBatchInternal { + public: + // WriteBatch header has an 8-byte sequence number followed by a 4-byte count. + static constexpr size_t kHeader = 12; + + // WriteBatch methods with column_family_id instead of ColumnFamilyHandle* + static Status Put(WriteBatch* batch, uint32_t column_family_id, + const Slice& key, const Slice& value); + + static Status Put(WriteBatch* batch, uint32_t column_family_id, + const SliceParts& key, const SliceParts& value); + + static Status PutEntity(WriteBatch* batch, uint32_t column_family_id, + const Slice& key, const WideColumns& columns); + + static Status Delete(WriteBatch* batch, uint32_t column_family_id, + const SliceParts& key); + + static Status Delete(WriteBatch* batch, uint32_t column_family_id, + const Slice& key); + + static Status SingleDelete(WriteBatch* batch, uint32_t column_family_id, + const SliceParts& key); + + static Status SingleDelete(WriteBatch* batch, uint32_t column_family_id, + const Slice& key); + + static Status DeleteRange(WriteBatch* b, uint32_t column_family_id, + const Slice& begin_key, const Slice& end_key); + + static Status DeleteRange(WriteBatch* b, uint32_t column_family_id, + const SliceParts& begin_key, + const SliceParts& end_key); + + static Status Merge(WriteBatch* batch, uint32_t column_family_id, + const Slice& key, const Slice& value); + + static Status Merge(WriteBatch* batch, uint32_t column_family_id, + const SliceParts& key, const SliceParts& value); + + static Status PutBlobIndex(WriteBatch* batch, uint32_t column_family_id, + const Slice& key, const Slice& value); + + static Status MarkEndPrepare(WriteBatch* batch, const Slice& xid, + const bool write_after_commit = true, + const bool unprepared_batch = false); + + static Status MarkRollback(WriteBatch* batch, const Slice& xid); + + static Status MarkCommit(WriteBatch* batch, const Slice& xid); + + static Status MarkCommitWithTimestamp(WriteBatch* batch, const Slice& xid, + const Slice& commit_ts); + + static Status InsertNoop(WriteBatch* batch); + + // Return the number of entries in the batch. + static uint32_t Count(const WriteBatch* batch); + + // Set the count for the number of entries in the batch. + static void SetCount(WriteBatch* batch, uint32_t n); + + // Return the sequence number for the start of this batch. + static SequenceNumber Sequence(const WriteBatch* batch); + + // Store the specified number as the sequence number for the start of + // this batch. + static void SetSequence(WriteBatch* batch, SequenceNumber seq); + + // Returns the offset of the first entry in the batch. + // This offset is only valid if the batch is not empty. + static size_t GetFirstOffset(WriteBatch* batch); + + static Slice Contents(const WriteBatch* batch) { return Slice(batch->rep_); } + + static size_t ByteSize(const WriteBatch* batch) { return batch->rep_.size(); } + + static Status SetContents(WriteBatch* batch, const Slice& contents); + + static Status CheckSlicePartsLength(const SliceParts& key, + const SliceParts& value); + + // Inserts batches[i] into memtable, for i in 0..num_batches-1 inclusive. + // + // If ignore_missing_column_families == true. WriteBatch + // referencing non-existing column family will be ignored. + // If ignore_missing_column_families == false, processing of the + // batches will be stopped if a reference is found to a non-existing + // column family and InvalidArgument() will be returned. The writes + // in batches may be only partially applied at that point. + // + // If log_number is non-zero, the memtable will be updated only if + // memtables->GetLogNumber() >= log_number. + // + // If flush_scheduler is non-null, it will be invoked if the memtable + // should be flushed. + // + // Under concurrent use, the caller is responsible for making sure that + // the memtables object itself is thread-local. + static Status InsertInto( + WriteThread::WriteGroup& write_group, SequenceNumber sequence, + ColumnFamilyMemTables* memtables, FlushScheduler* flush_scheduler, + TrimHistoryScheduler* trim_history_scheduler, + bool ignore_missing_column_families = false, uint64_t log_number = 0, + DB* db = nullptr, bool concurrent_memtable_writes = false, + bool seq_per_batch = false, bool batch_per_txn = true); + + // Convenience form of InsertInto when you have only one batch + // next_seq returns the seq after last sequence number used in MemTable insert + static Status InsertInto( + const WriteBatch* batch, ColumnFamilyMemTables* memtables, + FlushScheduler* flush_scheduler, + TrimHistoryScheduler* trim_history_scheduler, + bool ignore_missing_column_families = false, uint64_t log_number = 0, + DB* db = nullptr, bool concurrent_memtable_writes = false, + SequenceNumber* next_seq = nullptr, bool* has_valid_writes = nullptr, + bool seq_per_batch = false, bool batch_per_txn = true); + + static Status InsertInto(WriteThread::Writer* writer, SequenceNumber sequence, + ColumnFamilyMemTables* memtables, + FlushScheduler* flush_scheduler, + TrimHistoryScheduler* trim_history_scheduler, + bool ignore_missing_column_families = false, + uint64_t log_number = 0, DB* db = nullptr, + bool concurrent_memtable_writes = false, + bool seq_per_batch = false, size_t batch_cnt = 0, + bool batch_per_txn = true, + bool hint_per_batch = false); + + // Appends src write batch to dst write batch and updates count in dst + // write batch. Returns OK if the append is successful. Checks number of + // checksum against count in dst and src write batches, and returns Corruption + // if the count is inconsistent. + static Status Append(WriteBatch* dst, const WriteBatch* src, + const bool WAL_only = false); + + // Returns the byte size of appending a WriteBatch with ByteSize + // leftByteSize and a WriteBatch with ByteSize rightByteSize + static size_t AppendedByteSize(size_t leftByteSize, size_t rightByteSize); + + // Iterate over [begin, end) range of a write batch + static Status Iterate(const WriteBatch* wb, WriteBatch::Handler* handler, + size_t begin, size_t end); + + // This write batch includes the latest state that should be persisted. Such + // state meant to be used only during recovery. + static void SetAsLatestPersistentState(WriteBatch* b); + static bool IsLatestPersistentState(const WriteBatch* b); + + static std::tuple GetColumnFamilyIdAndTimestampSize( + WriteBatch* b, ColumnFamilyHandle* column_family); + + static bool TimestampsUpdateNeeded(const WriteBatch& wb) { + return wb.needs_in_place_update_ts_; + } + + static bool HasKeyWithTimestamp(const WriteBatch& wb) { + return wb.has_key_with_ts_; + } + + // Update per-key value protection information on this write batch. + // If checksum is provided, the batch content is verfied against the checksum. + static Status UpdateProtectionInfo(WriteBatch* wb, size_t bytes_per_key, + uint64_t* checksum = nullptr); +}; + +// LocalSavePoint is similar to a scope guard +class LocalSavePoint { + public: + explicit LocalSavePoint(WriteBatch* batch) + : batch_(batch), + savepoint_(batch->GetDataSize(), batch->Count(), + batch->content_flags_.load(std::memory_order_relaxed)) +#ifndef NDEBUG + , + committed_(false) +#endif + { + } + +#ifndef NDEBUG + ~LocalSavePoint() { assert(committed_); } +#endif + Status commit() { +#ifndef NDEBUG + committed_ = true; +#endif + if (batch_->max_bytes_ && batch_->rep_.size() > batch_->max_bytes_) { + batch_->rep_.resize(savepoint_.size); + WriteBatchInternal::SetCount(batch_, savepoint_.count); + if (batch_->prot_info_ != nullptr) { + batch_->prot_info_->entries_.resize(savepoint_.count); + } + batch_->content_flags_.store(savepoint_.content_flags, + std::memory_order_relaxed); + return Status::MemoryLimit(); + } + return Status::OK(); + } + + private: + WriteBatch* batch_; + SavePoint savepoint_; +#ifndef NDEBUG + bool committed_; +#endif +}; + +template +class TimestampUpdater : public WriteBatch::Handler { + public: + explicit TimestampUpdater(WriteBatch::ProtectionInfo* prot_info, + TimestampSizeFuncType&& ts_sz_func, const Slice& ts) + : prot_info_(prot_info), + ts_sz_func_(std::move(ts_sz_func)), + timestamp_(ts) { + assert(!timestamp_.empty()); + } + + ~TimestampUpdater() override {} + + Status PutCF(uint32_t cf, const Slice& key, const Slice&) override { + return UpdateTimestamp(cf, key); + } + + Status DeleteCF(uint32_t cf, const Slice& key) override { + return UpdateTimestamp(cf, key); + } + + Status SingleDeleteCF(uint32_t cf, const Slice& key) override { + return UpdateTimestamp(cf, key); + } + + Status DeleteRangeCF(uint32_t cf, const Slice& begin_key, + const Slice& end_key) override { + Status s = UpdateTimestamp(cf, begin_key, true /* is_key */); + if (s.ok()) { + s = UpdateTimestamp(cf, end_key, false /* is_key */); + } + return s; + } + + Status MergeCF(uint32_t cf, const Slice& key, const Slice&) override { + return UpdateTimestamp(cf, key); + } + + Status PutBlobIndexCF(uint32_t cf, const Slice& key, const Slice&) override { + return UpdateTimestamp(cf, key); + } + + Status MarkBeginPrepare(bool) override { return Status::OK(); } + + Status MarkEndPrepare(const Slice&) override { return Status::OK(); } + + Status MarkCommit(const Slice&) override { return Status::OK(); } + + Status MarkCommitWithTimestamp(const Slice&, const Slice&) override { + return Status::OK(); + } + + Status MarkRollback(const Slice&) override { return Status::OK(); } + + Status MarkNoop(bool /*empty_batch*/) override { return Status::OK(); } + + private: + // @param is_key specifies whether the update is for key or value. + Status UpdateTimestamp(uint32_t cf, const Slice& buf, bool is_key = true) { + Status s = UpdateTimestampImpl(cf, buf, idx_, is_key); + ++idx_; + return s; + } + + Status UpdateTimestampImpl(uint32_t cf, const Slice& buf, size_t /*idx*/, + bool is_key) { + if (timestamp_.empty()) { + return Status::InvalidArgument("Timestamp is empty"); + } + size_t cf_ts_sz = ts_sz_func_(cf); + if (0 == cf_ts_sz) { + // Skip this column family. + return Status::OK(); + } else if (std::numeric_limits::max() == cf_ts_sz) { + // Column family timestamp info not found. + return Status::NotFound(); + } else if (cf_ts_sz != timestamp_.size()) { + return Status::InvalidArgument("timestamp size mismatch"); + } + UpdateProtectionInformationIfNeeded(buf, timestamp_, is_key); + + char* ptr = const_cast(buf.data() + buf.size() - cf_ts_sz); + assert(ptr); + memcpy(ptr, timestamp_.data(), timestamp_.size()); + return Status::OK(); + } + + void UpdateProtectionInformationIfNeeded(const Slice& buf, const Slice& ts, + bool is_key) { + if (prot_info_ != nullptr) { + const size_t ts_sz = ts.size(); + SliceParts old(&buf, 1); + Slice old_no_ts(buf.data(), buf.size() - ts_sz); + std::array new_key_cmpts{{old_no_ts, ts}}; + SliceParts new_parts(new_key_cmpts.data(), 2); + if (is_key) { + prot_info_->entries_[idx_].UpdateK(old, new_parts); + } else { + prot_info_->entries_[idx_].UpdateV(old, new_parts); + } + } + } + + // No copy or move. + TimestampUpdater(const TimestampUpdater&) = delete; + TimestampUpdater(TimestampUpdater&&) = delete; + TimestampUpdater& operator=(const TimestampUpdater&) = delete; + TimestampUpdater& operator=(TimestampUpdater&&) = delete; + + WriteBatch::ProtectionInfo* const prot_info_ = nullptr; + const TimestampSizeFuncType ts_sz_func_{}; + const Slice timestamp_; + size_t idx_ = 0; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/write_batch_test.cc b/librocksdb-sys/rocksdb/db/write_batch_test.cc new file mode 100644 index 0000000..1740526 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/write_batch_test.cc @@ -0,0 +1,1128 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include + +#include "db/column_family.h" +#include "db/db_test_util.h" +#include "db/memtable.h" +#include "db/write_batch_internal.h" +#include "rocksdb/comparator.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/utilities/write_batch_with_index.h" +#include "rocksdb/write_buffer_manager.h" +#include "table/scoped_arena_iterator.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +static std::string PrintContents(WriteBatch* b, + bool merge_operator_supported = true) { + InternalKeyComparator cmp(BytewiseComparator()); + auto factory = std::make_shared(); + Options options; + options.memtable_factory = factory; + if (merge_operator_supported) { + options.merge_operator.reset(new TestPutOperator()); + } + ImmutableOptions ioptions(options); + WriteBufferManager wb(options.db_write_buffer_size); + MemTable* mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb, + kMaxSequenceNumber, 0 /* column_family_id */); + mem->Ref(); + std::string state; + ColumnFamilyMemTablesDefault cf_mems_default(mem); + Status s = + WriteBatchInternal::InsertInto(b, &cf_mems_default, nullptr, nullptr); + uint32_t count = 0; + int put_count = 0; + int delete_count = 0; + int single_delete_count = 0; + int delete_range_count = 0; + int merge_count = 0; + for (int i = 0; i < 2; ++i) { + Arena arena; + ScopedArenaIterator arena_iter_guard; + std::unique_ptr iter_guard; + InternalIterator* iter; + if (i == 0) { + iter = mem->NewIterator(ReadOptions(), &arena); + arena_iter_guard.set(iter); + } else { + iter = mem->NewRangeTombstoneIterator(ReadOptions(), + kMaxSequenceNumber /* read_seq */, + false /* immutable_memtable */); + iter_guard.reset(iter); + } + if (iter == nullptr) { + continue; + } + EXPECT_OK(iter->status()); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ParsedInternalKey ikey; + ikey.clear(); + EXPECT_OK(ParseInternalKey(iter->key(), &ikey, true /* log_err_key */)); + switch (ikey.type) { + case kTypeValue: + state.append("Put("); + state.append(ikey.user_key.ToString()); + state.append(", "); + state.append(iter->value().ToString()); + state.append(")"); + count++; + put_count++; + break; + case kTypeDeletion: + state.append("Delete("); + state.append(ikey.user_key.ToString()); + state.append(")"); + count++; + delete_count++; + break; + case kTypeSingleDeletion: + state.append("SingleDelete("); + state.append(ikey.user_key.ToString()); + state.append(")"); + count++; + single_delete_count++; + break; + case kTypeRangeDeletion: + state.append("DeleteRange("); + state.append(ikey.user_key.ToString()); + state.append(", "); + state.append(iter->value().ToString()); + state.append(")"); + count++; + delete_range_count++; + break; + case kTypeMerge: + state.append("Merge("); + state.append(ikey.user_key.ToString()); + state.append(", "); + state.append(iter->value().ToString()); + state.append(")"); + count++; + merge_count++; + break; + default: + assert(false); + break; + } + state.append("@"); + state.append(std::to_string(ikey.sequence)); + } + EXPECT_OK(iter->status()); + } + if (s.ok()) { + EXPECT_EQ(b->HasPut(), put_count > 0); + EXPECT_EQ(b->HasDelete(), delete_count > 0); + EXPECT_EQ(b->HasSingleDelete(), single_delete_count > 0); + EXPECT_EQ(b->HasDeleteRange(), delete_range_count > 0); + EXPECT_EQ(b->HasMerge(), merge_count > 0); + if (count != WriteBatchInternal::Count(b)) { + state.append("CountMismatch()"); + } + } else { + state.append(s.ToString()); + } + delete mem->Unref(); + return state; +} + +class WriteBatchTest : public testing::Test {}; + +TEST_F(WriteBatchTest, Empty) { + WriteBatch batch; + ASSERT_EQ("", PrintContents(&batch)); + ASSERT_EQ(0u, WriteBatchInternal::Count(&batch)); + ASSERT_EQ(0u, batch.Count()); +} + +TEST_F(WriteBatchTest, Multiple) { + WriteBatch batch; + ASSERT_OK(batch.Put(Slice("foo"), Slice("bar"))); + ASSERT_OK(batch.Delete(Slice("box"))); + ASSERT_OK(batch.DeleteRange(Slice("bar"), Slice("foo"))); + ASSERT_OK(batch.Put(Slice("baz"), Slice("boo"))); + WriteBatchInternal::SetSequence(&batch, 100); + ASSERT_EQ(100U, WriteBatchInternal::Sequence(&batch)); + ASSERT_EQ(4u, WriteBatchInternal::Count(&batch)); + ASSERT_EQ( + "Put(baz, boo)@103" + "Delete(box)@101" + "Put(foo, bar)@100" + "DeleteRange(bar, foo)@102", + PrintContents(&batch)); + ASSERT_EQ(4u, batch.Count()); +} + +TEST_F(WriteBatchTest, Corruption) { + WriteBatch batch; + ASSERT_OK(batch.Put(Slice("foo"), Slice("bar"))); + ASSERT_OK(batch.Delete(Slice("box"))); + WriteBatchInternal::SetSequence(&batch, 200); + Slice contents = WriteBatchInternal::Contents(&batch); + ASSERT_OK(WriteBatchInternal::SetContents( + &batch, Slice(contents.data(), contents.size() - 1))); + ASSERT_EQ( + "Put(foo, bar)@200" + "Corruption: bad WriteBatch Delete", + PrintContents(&batch)); +} + +TEST_F(WriteBatchTest, Append) { + WriteBatch b1, b2; + WriteBatchInternal::SetSequence(&b1, 200); + WriteBatchInternal::SetSequence(&b2, 300); + ASSERT_OK(WriteBatchInternal::Append(&b1, &b2)); + ASSERT_EQ("", PrintContents(&b1)); + ASSERT_EQ(0u, b1.Count()); + ASSERT_OK(b2.Put("a", "va")); + ASSERT_OK(WriteBatchInternal::Append(&b1, &b2)); + ASSERT_EQ("Put(a, va)@200", PrintContents(&b1)); + ASSERT_EQ(1u, b1.Count()); + b2.Clear(); + ASSERT_OK(b2.Put("b", "vb")); + ASSERT_OK(WriteBatchInternal::Append(&b1, &b2)); + ASSERT_EQ( + "Put(a, va)@200" + "Put(b, vb)@201", + PrintContents(&b1)); + ASSERT_EQ(2u, b1.Count()); + ASSERT_OK(b2.Delete("foo")); + ASSERT_OK(WriteBatchInternal::Append(&b1, &b2)); + ASSERT_EQ( + "Put(a, va)@200" + "Put(b, vb)@202" + "Put(b, vb)@201" + "Delete(foo)@203", + PrintContents(&b1)); + ASSERT_EQ(4u, b1.Count()); + b2.Clear(); + ASSERT_OK(b2.Put("c", "cc")); + ASSERT_OK(b2.Put("d", "dd")); + b2.MarkWalTerminationPoint(); + ASSERT_OK(b2.Put("e", "ee")); + ASSERT_OK(WriteBatchInternal::Append(&b1, &b2, /*wal only*/ true)); + ASSERT_EQ( + "Put(a, va)@200" + "Put(b, vb)@202" + "Put(b, vb)@201" + "Put(c, cc)@204" + "Put(d, dd)@205" + "Delete(foo)@203", + PrintContents(&b1)); + ASSERT_EQ(6u, b1.Count()); + ASSERT_EQ( + "Put(c, cc)@0" + "Put(d, dd)@1" + "Put(e, ee)@2", + PrintContents(&b2)); + ASSERT_EQ(3u, b2.Count()); +} + +TEST_F(WriteBatchTest, SingleDeletion) { + WriteBatch batch; + WriteBatchInternal::SetSequence(&batch, 100); + ASSERT_EQ("", PrintContents(&batch)); + ASSERT_EQ(0u, batch.Count()); + ASSERT_OK(batch.Put("a", "va")); + ASSERT_EQ("Put(a, va)@100", PrintContents(&batch)); + ASSERT_EQ(1u, batch.Count()); + ASSERT_OK(batch.SingleDelete("a")); + ASSERT_EQ( + "SingleDelete(a)@101" + "Put(a, va)@100", + PrintContents(&batch)); + ASSERT_EQ(2u, batch.Count()); +} + +TEST_F(WriteBatchTest, OwnershipTransfer) { + Random rnd(301); + WriteBatch put_batch; + ASSERT_OK(put_batch.Put(rnd.RandomString(16) /* key */, + rnd.RandomString(1024) /* value */)); + + // (1) Verify `Release()` transfers string data ownership + const char* expected_data = put_batch.Data().data(); + std::string batch_str = put_batch.Release(); + ASSERT_EQ(expected_data, batch_str.data()); + + // (2) Verify constructor transfers string data ownership + WriteBatch move_batch(std::move(batch_str)); + ASSERT_EQ(expected_data, move_batch.Data().data()); +} + +namespace { +struct TestHandler : public WriteBatch::Handler { + std::string seen; + Status PutCF(uint32_t column_family_id, const Slice& key, + const Slice& value) override { + if (column_family_id == 0) { + seen += "Put(" + key.ToString() + ", " + value.ToString() + ")"; + } else { + seen += "PutCF(" + std::to_string(column_family_id) + ", " + + key.ToString() + ", " + value.ToString() + ")"; + } + return Status::OK(); + } + Status DeleteCF(uint32_t column_family_id, const Slice& key) override { + if (column_family_id == 0) { + seen += "Delete(" + key.ToString() + ")"; + } else { + seen += "DeleteCF(" + std::to_string(column_family_id) + ", " + + key.ToString() + ")"; + } + return Status::OK(); + } + Status SingleDeleteCF(uint32_t column_family_id, const Slice& key) override { + if (column_family_id == 0) { + seen += "SingleDelete(" + key.ToString() + ")"; + } else { + seen += "SingleDeleteCF(" + std::to_string(column_family_id) + ", " + + key.ToString() + ")"; + } + return Status::OK(); + } + Status DeleteRangeCF(uint32_t column_family_id, const Slice& begin_key, + const Slice& end_key) override { + if (column_family_id == 0) { + seen += "DeleteRange(" + begin_key.ToString() + ", " + + end_key.ToString() + ")"; + } else { + seen += "DeleteRangeCF(" + std::to_string(column_family_id) + ", " + + begin_key.ToString() + ", " + end_key.ToString() + ")"; + } + return Status::OK(); + } + Status MergeCF(uint32_t column_family_id, const Slice& key, + const Slice& value) override { + if (column_family_id == 0) { + seen += "Merge(" + key.ToString() + ", " + value.ToString() + ")"; + } else { + seen += "MergeCF(" + std::to_string(column_family_id) + ", " + + key.ToString() + ", " + value.ToString() + ")"; + } + return Status::OK(); + } + void LogData(const Slice& blob) override { + seen += "LogData(" + blob.ToString() + ")"; + } + Status MarkBeginPrepare(bool unprepare) override { + seen += + "MarkBeginPrepare(" + std::string(unprepare ? "true" : "false") + ")"; + return Status::OK(); + } + Status MarkEndPrepare(const Slice& xid) override { + seen += "MarkEndPrepare(" + xid.ToString() + ")"; + return Status::OK(); + } + Status MarkNoop(bool empty_batch) override { + seen += "MarkNoop(" + std::string(empty_batch ? "true" : "false") + ")"; + return Status::OK(); + } + Status MarkCommit(const Slice& xid) override { + seen += "MarkCommit(" + xid.ToString() + ")"; + return Status::OK(); + } + Status MarkCommitWithTimestamp(const Slice& xid, const Slice& ts) override { + seen += "MarkCommitWithTimestamp(" + xid.ToString() + ", " + + ts.ToString(true) + ")"; + return Status::OK(); + } + Status MarkRollback(const Slice& xid) override { + seen += "MarkRollback(" + xid.ToString() + ")"; + return Status::OK(); + } +}; +} // anonymous namespace + +TEST_F(WriteBatchTest, PutNotImplemented) { + WriteBatch batch; + ASSERT_OK(batch.Put(Slice("k1"), Slice("v1"))); + ASSERT_EQ(1u, batch.Count()); + ASSERT_EQ("Put(k1, v1)@0", PrintContents(&batch)); + + WriteBatch::Handler handler; + ASSERT_OK(batch.Iterate(&handler)); +} + +TEST_F(WriteBatchTest, DeleteNotImplemented) { + WriteBatch batch; + ASSERT_OK(batch.Delete(Slice("k2"))); + ASSERT_EQ(1u, batch.Count()); + ASSERT_EQ("Delete(k2)@0", PrintContents(&batch)); + + WriteBatch::Handler handler; + ASSERT_OK(batch.Iterate(&handler)); +} + +TEST_F(WriteBatchTest, SingleDeleteNotImplemented) { + WriteBatch batch; + ASSERT_OK(batch.SingleDelete(Slice("k2"))); + ASSERT_EQ(1u, batch.Count()); + ASSERT_EQ("SingleDelete(k2)@0", PrintContents(&batch)); + + WriteBatch::Handler handler; + ASSERT_OK(batch.Iterate(&handler)); +} + +TEST_F(WriteBatchTest, MergeNotImplemented) { + WriteBatch batch; + ASSERT_OK(batch.Merge(Slice("foo"), Slice("bar"))); + ASSERT_EQ(1u, batch.Count()); + ASSERT_EQ("Merge(foo, bar)@0", PrintContents(&batch)); + + WriteBatch::Handler handler; + ASSERT_OK(batch.Iterate(&handler)); +} + +TEST_F(WriteBatchTest, MergeWithoutOperatorInsertionFailure) { + WriteBatch batch; + ASSERT_OK(batch.Merge(Slice("foo"), Slice("bar"))); + ASSERT_EQ(1u, batch.Count()); + ASSERT_EQ( + "Invalid argument: Merge requires `ColumnFamilyOptions::merge_operator " + "!= nullptr`", + PrintContents(&batch, false /* merge_operator_supported */)); +} + +TEST_F(WriteBatchTest, Blob) { + WriteBatch batch; + ASSERT_OK(batch.Put(Slice("k1"), Slice("v1"))); + ASSERT_OK(batch.Put(Slice("k2"), Slice("v2"))); + ASSERT_OK(batch.Put(Slice("k3"), Slice("v3"))); + ASSERT_OK(batch.PutLogData(Slice("blob1"))); + ASSERT_OK(batch.Delete(Slice("k2"))); + ASSERT_OK(batch.SingleDelete(Slice("k3"))); + ASSERT_OK(batch.PutLogData(Slice("blob2"))); + ASSERT_OK(batch.Merge(Slice("foo"), Slice("bar"))); + ASSERT_EQ(6u, batch.Count()); + ASSERT_EQ( + "Merge(foo, bar)@5" + "Put(k1, v1)@0" + "Delete(k2)@3" + "Put(k2, v2)@1" + "SingleDelete(k3)@4" + "Put(k3, v3)@2", + PrintContents(&batch)); + + TestHandler handler; + ASSERT_OK(batch.Iterate(&handler)); + ASSERT_EQ( + "Put(k1, v1)" + "Put(k2, v2)" + "Put(k3, v3)" + "LogData(blob1)" + "Delete(k2)" + "SingleDelete(k3)" + "LogData(blob2)" + "Merge(foo, bar)", + handler.seen); +} + +TEST_F(WriteBatchTest, PrepareCommit) { + WriteBatch batch; + ASSERT_OK(WriteBatchInternal::InsertNoop(&batch)); + ASSERT_OK(batch.Put(Slice("k1"), Slice("v1"))); + ASSERT_OK(batch.Put(Slice("k2"), Slice("v2"))); + batch.SetSavePoint(); + ASSERT_OK(WriteBatchInternal::MarkEndPrepare(&batch, Slice("xid1"))); + Status s = batch.RollbackToSavePoint(); + ASSERT_EQ(s, Status::NotFound()); + ASSERT_OK(WriteBatchInternal::MarkCommit(&batch, Slice("xid1"))); + ASSERT_OK(WriteBatchInternal::MarkRollback(&batch, Slice("xid1"))); + ASSERT_EQ(2u, batch.Count()); + + TestHandler handler; + ASSERT_OK(batch.Iterate(&handler)); + ASSERT_EQ( + "MarkBeginPrepare(false)" + "Put(k1, v1)" + "Put(k2, v2)" + "MarkEndPrepare(xid1)" + "MarkCommit(xid1)" + "MarkRollback(xid1)", + handler.seen); +} + +// It requires more than 30GB of memory to run the test. With single memory +// allocation of more than 30GB. +// Not all platform can run it. Also it runs a long time. So disable it. +TEST_F(WriteBatchTest, DISABLED_ManyUpdates) { + // Insert key and value of 3GB and push total batch size to 12GB. + static const size_t kKeyValueSize = 4u; + static const uint32_t kNumUpdates = uint32_t{3} << 30; + std::string raw(kKeyValueSize, 'A'); + WriteBatch batch(kNumUpdates * (4 + kKeyValueSize * 2) + 1024u); + char c = 'A'; + for (uint32_t i = 0; i < kNumUpdates; i++) { + if (c > 'Z') { + c = 'A'; + } + raw[0] = c; + raw[raw.length() - 1] = c; + c++; + ASSERT_OK(batch.Put(raw, raw)); + } + + ASSERT_EQ(kNumUpdates, batch.Count()); + + struct NoopHandler : public WriteBatch::Handler { + uint32_t num_seen = 0; + char expected_char = 'A'; + Status PutCF(uint32_t /*column_family_id*/, const Slice& key, + const Slice& value) override { + EXPECT_EQ(kKeyValueSize, key.size()); + EXPECT_EQ(kKeyValueSize, value.size()); + EXPECT_EQ(expected_char, key[0]); + EXPECT_EQ(expected_char, value[0]); + EXPECT_EQ(expected_char, key[kKeyValueSize - 1]); + EXPECT_EQ(expected_char, value[kKeyValueSize - 1]); + expected_char++; + if (expected_char > 'Z') { + expected_char = 'A'; + } + ++num_seen; + return Status::OK(); + } + Status DeleteCF(uint32_t /*column_family_id*/, + const Slice& /*key*/) override { + ADD_FAILURE(); + return Status::OK(); + } + Status SingleDeleteCF(uint32_t /*column_family_id*/, + const Slice& /*key*/) override { + ADD_FAILURE(); + return Status::OK(); + } + Status MergeCF(uint32_t /*column_family_id*/, const Slice& /*key*/, + const Slice& /*value*/) override { + ADD_FAILURE(); + return Status::OK(); + } + void LogData(const Slice& /*blob*/) override { ADD_FAILURE(); } + bool Continue() override { return num_seen < kNumUpdates; } + } handler; + + ASSERT_OK(batch.Iterate(&handler)); + ASSERT_EQ(kNumUpdates, handler.num_seen); +} + +// The test requires more than 18GB memory to run it, with single memory +// allocation of more than 12GB. Not all the platform can run it. So disable it. +TEST_F(WriteBatchTest, DISABLED_LargeKeyValue) { + // Insert key and value of 3GB and push total batch size to 12GB. + static const size_t kKeyValueSize = 3221225472u; + std::string raw(kKeyValueSize, 'A'); + WriteBatch batch(size_t(12884901888ull + 1024u)); + for (char i = 0; i < 2; i++) { + raw[0] = 'A' + i; + raw[raw.length() - 1] = 'A' - i; + ASSERT_OK(batch.Put(raw, raw)); + } + + ASSERT_EQ(2u, batch.Count()); + + struct NoopHandler : public WriteBatch::Handler { + int num_seen = 0; + Status PutCF(uint32_t /*column_family_id*/, const Slice& key, + const Slice& value) override { + EXPECT_EQ(kKeyValueSize, key.size()); + EXPECT_EQ(kKeyValueSize, value.size()); + EXPECT_EQ('A' + num_seen, key[0]); + EXPECT_EQ('A' + num_seen, value[0]); + EXPECT_EQ('A' - num_seen, key[kKeyValueSize - 1]); + EXPECT_EQ('A' - num_seen, value[kKeyValueSize - 1]); + ++num_seen; + return Status::OK(); + } + Status DeleteCF(uint32_t /*column_family_id*/, + const Slice& /*key*/) override { + ADD_FAILURE(); + return Status::OK(); + } + Status SingleDeleteCF(uint32_t /*column_family_id*/, + const Slice& /*key*/) override { + ADD_FAILURE(); + return Status::OK(); + } + Status MergeCF(uint32_t /*column_family_id*/, const Slice& /*key*/, + const Slice& /*value*/) override { + ADD_FAILURE(); + return Status::OK(); + } + void LogData(const Slice& /*blob*/) override { ADD_FAILURE(); } + bool Continue() override { return num_seen < 2; } + } handler; + + ASSERT_OK(batch.Iterate(&handler)); + ASSERT_EQ(2, handler.num_seen); +} + +TEST_F(WriteBatchTest, Continue) { + WriteBatch batch; + + struct Handler : public TestHandler { + int num_seen = 0; + Status PutCF(uint32_t column_family_id, const Slice& key, + const Slice& value) override { + ++num_seen; + return TestHandler::PutCF(column_family_id, key, value); + } + Status DeleteCF(uint32_t column_family_id, const Slice& key) override { + ++num_seen; + return TestHandler::DeleteCF(column_family_id, key); + } + Status SingleDeleteCF(uint32_t column_family_id, + const Slice& key) override { + ++num_seen; + return TestHandler::SingleDeleteCF(column_family_id, key); + } + Status MergeCF(uint32_t column_family_id, const Slice& key, + const Slice& value) override { + ++num_seen; + return TestHandler::MergeCF(column_family_id, key, value); + } + void LogData(const Slice& blob) override { + ++num_seen; + TestHandler::LogData(blob); + } + bool Continue() override { return num_seen < 5; } + } handler; + + ASSERT_OK(batch.Put(Slice("k1"), Slice("v1"))); + ASSERT_OK(batch.Put(Slice("k2"), Slice("v2"))); + ASSERT_OK(batch.PutLogData(Slice("blob1"))); + ASSERT_OK(batch.Delete(Slice("k1"))); + ASSERT_OK(batch.SingleDelete(Slice("k2"))); + ASSERT_OK(batch.PutLogData(Slice("blob2"))); + ASSERT_OK(batch.Merge(Slice("foo"), Slice("bar"))); + ASSERT_OK(batch.Iterate(&handler)); + ASSERT_EQ( + "Put(k1, v1)" + "Put(k2, v2)" + "LogData(blob1)" + "Delete(k1)" + "SingleDelete(k2)", + handler.seen); +} + +TEST_F(WriteBatchTest, PutGatherSlices) { + WriteBatch batch; + ASSERT_OK(batch.Put(Slice("foo"), Slice("bar"))); + + { + // Try a write where the key is one slice but the value is two + Slice key_slice("baz"); + Slice value_slices[2] = {Slice("header"), Slice("payload")}; + ASSERT_OK( + batch.Put(SliceParts(&key_slice, 1), SliceParts(value_slices, 2))); + } + + { + // One where the key is composite but the value is a single slice + Slice key_slices[3] = {Slice("key"), Slice("part2"), Slice("part3")}; + Slice value_slice("value"); + ASSERT_OK( + batch.Put(SliceParts(key_slices, 3), SliceParts(&value_slice, 1))); + } + + WriteBatchInternal::SetSequence(&batch, 100); + ASSERT_EQ( + "Put(baz, headerpayload)@101" + "Put(foo, bar)@100" + "Put(keypart2part3, value)@102", + PrintContents(&batch)); + ASSERT_EQ(3u, batch.Count()); +} + +namespace { +class ColumnFamilyHandleImplDummy : public ColumnFamilyHandleImpl { + public: + explicit ColumnFamilyHandleImplDummy(int id) + : ColumnFamilyHandleImpl(nullptr, nullptr, nullptr), id_(id) {} + explicit ColumnFamilyHandleImplDummy(int id, const Comparator* ucmp) + : ColumnFamilyHandleImpl(nullptr, nullptr, nullptr), + id_(id), + ucmp_(ucmp) {} + uint32_t GetID() const override { return id_; } + const Comparator* GetComparator() const override { return ucmp_; } + + private: + uint32_t id_; + const Comparator* const ucmp_ = BytewiseComparator(); +}; +} // anonymous namespace + +TEST_F(WriteBatchTest, ColumnFamiliesBatchTest) { + WriteBatch batch; + ColumnFamilyHandleImplDummy zero(0), two(2), three(3), eight(8); + ASSERT_OK(batch.Put(&zero, Slice("foo"), Slice("bar"))); + ASSERT_OK(batch.Put(&two, Slice("twofoo"), Slice("bar2"))); + ASSERT_OK(batch.Put(&eight, Slice("eightfoo"), Slice("bar8"))); + ASSERT_OK(batch.Delete(&eight, Slice("eightfoo"))); + ASSERT_OK(batch.SingleDelete(&two, Slice("twofoo"))); + ASSERT_OK(batch.DeleteRange(&two, Slice("3foo"), Slice("4foo"))); + ASSERT_OK(batch.Merge(&three, Slice("threethree"), Slice("3three"))); + ASSERT_OK(batch.Put(&zero, Slice("foo"), Slice("bar"))); + ASSERT_OK(batch.Merge(Slice("omom"), Slice("nom"))); + + TestHandler handler; + ASSERT_OK(batch.Iterate(&handler)); + ASSERT_EQ( + "Put(foo, bar)" + "PutCF(2, twofoo, bar2)" + "PutCF(8, eightfoo, bar8)" + "DeleteCF(8, eightfoo)" + "SingleDeleteCF(2, twofoo)" + "DeleteRangeCF(2, 3foo, 4foo)" + "MergeCF(3, threethree, 3three)" + "Put(foo, bar)" + "Merge(omom, nom)", + handler.seen); +} + +TEST_F(WriteBatchTest, ColumnFamiliesBatchWithIndexTest) { + WriteBatchWithIndex batch; + ColumnFamilyHandleImplDummy zero(0), two(2), three(3), eight(8); + ASSERT_OK(batch.Put(&zero, Slice("foo"), Slice("bar"))); + ASSERT_OK(batch.Put(&two, Slice("twofoo"), Slice("bar2"))); + ASSERT_OK(batch.Put(&eight, Slice("eightfoo"), Slice("bar8"))); + ASSERT_OK(batch.Delete(&eight, Slice("eightfoo"))); + ASSERT_OK(batch.SingleDelete(&two, Slice("twofoo"))); + ASSERT_OK(batch.Merge(&three, Slice("threethree"), Slice("3three"))); + ASSERT_OK(batch.Put(&zero, Slice("foo"), Slice("bar"))); + ASSERT_OK(batch.Merge(Slice("omom"), Slice("nom"))); + + std::unique_ptr iter; + + iter.reset(batch.NewIterator(&eight)); + iter->Seek("eightfoo"); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(WriteType::kPutRecord, iter->Entry().type); + ASSERT_EQ("eightfoo", iter->Entry().key.ToString()); + ASSERT_EQ("bar8", iter->Entry().value.ToString()); + + iter->Next(); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(WriteType::kDeleteRecord, iter->Entry().type); + ASSERT_EQ("eightfoo", iter->Entry().key.ToString()); + + iter->Next(); + ASSERT_OK(iter->status()); + ASSERT_TRUE(!iter->Valid()); + + iter.reset(batch.NewIterator(&two)); + iter->Seek("twofoo"); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(WriteType::kPutRecord, iter->Entry().type); + ASSERT_EQ("twofoo", iter->Entry().key.ToString()); + ASSERT_EQ("bar2", iter->Entry().value.ToString()); + + iter->Next(); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(WriteType::kSingleDeleteRecord, iter->Entry().type); + ASSERT_EQ("twofoo", iter->Entry().key.ToString()); + + iter->Next(); + ASSERT_OK(iter->status()); + ASSERT_TRUE(!iter->Valid()); + + iter.reset(batch.NewIterator()); + iter->Seek("gggg"); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(WriteType::kMergeRecord, iter->Entry().type); + ASSERT_EQ("omom", iter->Entry().key.ToString()); + ASSERT_EQ("nom", iter->Entry().value.ToString()); + + iter->Next(); + ASSERT_OK(iter->status()); + ASSERT_TRUE(!iter->Valid()); + + iter.reset(batch.NewIterator(&zero)); + iter->Seek("foo"); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(WriteType::kPutRecord, iter->Entry().type); + ASSERT_EQ("foo", iter->Entry().key.ToString()); + ASSERT_EQ("bar", iter->Entry().value.ToString()); + + iter->Next(); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(WriteType::kPutRecord, iter->Entry().type); + ASSERT_EQ("foo", iter->Entry().key.ToString()); + ASSERT_EQ("bar", iter->Entry().value.ToString()); + + iter->Next(); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(WriteType::kMergeRecord, iter->Entry().type); + ASSERT_EQ("omom", iter->Entry().key.ToString()); + ASSERT_EQ("nom", iter->Entry().value.ToString()); + + iter->Next(); + ASSERT_OK(iter->status()); + ASSERT_TRUE(!iter->Valid()); + + TestHandler handler; + ASSERT_OK(batch.GetWriteBatch()->Iterate(&handler)); + ASSERT_EQ( + "Put(foo, bar)" + "PutCF(2, twofoo, bar2)" + "PutCF(8, eightfoo, bar8)" + "DeleteCF(8, eightfoo)" + "SingleDeleteCF(2, twofoo)" + "MergeCF(3, threethree, 3three)" + "Put(foo, bar)" + "Merge(omom, nom)", + handler.seen); +} + +TEST_F(WriteBatchTest, SavePointTest) { + Status s; + WriteBatch batch; + batch.SetSavePoint(); + + ASSERT_OK(batch.Put("A", "a")); + ASSERT_OK(batch.Put("B", "b")); + batch.SetSavePoint(); + + ASSERT_OK(batch.Put("C", "c")); + ASSERT_OK(batch.Delete("A")); + batch.SetSavePoint(); + batch.SetSavePoint(); + + ASSERT_OK(batch.RollbackToSavePoint()); + ASSERT_EQ( + "Delete(A)@3" + "Put(A, a)@0" + "Put(B, b)@1" + "Put(C, c)@2", + PrintContents(&batch)); + + ASSERT_OK(batch.RollbackToSavePoint()); + ASSERT_OK(batch.RollbackToSavePoint()); + ASSERT_EQ( + "Put(A, a)@0" + "Put(B, b)@1", + PrintContents(&batch)); + + ASSERT_OK(batch.Delete("A")); + ASSERT_OK(batch.Put("B", "bb")); + + ASSERT_OK(batch.RollbackToSavePoint()); + ASSERT_EQ("", PrintContents(&batch)); + + s = batch.RollbackToSavePoint(); + ASSERT_TRUE(s.IsNotFound()); + ASSERT_EQ("", PrintContents(&batch)); + + ASSERT_OK(batch.Put("D", "d")); + ASSERT_OK(batch.Delete("A")); + + batch.SetSavePoint(); + + ASSERT_OK(batch.Put("A", "aaa")); + + ASSERT_OK(batch.RollbackToSavePoint()); + ASSERT_EQ( + "Delete(A)@1" + "Put(D, d)@0", + PrintContents(&batch)); + + batch.SetSavePoint(); + + ASSERT_OK(batch.Put("D", "d")); + ASSERT_OK(batch.Delete("A")); + + ASSERT_OK(batch.RollbackToSavePoint()); + ASSERT_EQ( + "Delete(A)@1" + "Put(D, d)@0", + PrintContents(&batch)); + + s = batch.RollbackToSavePoint(); + ASSERT_TRUE(s.IsNotFound()); + ASSERT_EQ( + "Delete(A)@1" + "Put(D, d)@0", + PrintContents(&batch)); + + WriteBatch batch2; + + s = batch2.RollbackToSavePoint(); + ASSERT_TRUE(s.IsNotFound()); + ASSERT_EQ("", PrintContents(&batch2)); + + ASSERT_OK(batch2.Delete("A")); + batch2.SetSavePoint(); + + s = batch2.RollbackToSavePoint(); + ASSERT_OK(s); + ASSERT_EQ("Delete(A)@0", PrintContents(&batch2)); + + batch2.Clear(); + ASSERT_EQ("", PrintContents(&batch2)); + + batch2.SetSavePoint(); + + ASSERT_OK(batch2.Delete("B")); + ASSERT_EQ("Delete(B)@0", PrintContents(&batch2)); + + batch2.SetSavePoint(); + s = batch2.RollbackToSavePoint(); + ASSERT_OK(s); + ASSERT_EQ("Delete(B)@0", PrintContents(&batch2)); + + s = batch2.RollbackToSavePoint(); + ASSERT_OK(s); + ASSERT_EQ("", PrintContents(&batch2)); + + s = batch2.RollbackToSavePoint(); + ASSERT_TRUE(s.IsNotFound()); + ASSERT_EQ("", PrintContents(&batch2)); + + WriteBatch batch3; + + s = batch3.PopSavePoint(); + ASSERT_TRUE(s.IsNotFound()); + ASSERT_EQ("", PrintContents(&batch3)); + + batch3.SetSavePoint(); + ASSERT_OK(batch3.Delete("A")); + + s = batch3.PopSavePoint(); + ASSERT_OK(s); + ASSERT_EQ("Delete(A)@0", PrintContents(&batch3)); +} + +TEST_F(WriteBatchTest, MemoryLimitTest) { + Status s; + // The header size is 12 bytes. The two Puts take 8 bytes which gives total + // of 12 + 8 * 2 = 28 bytes. + WriteBatch batch(0, 28); + + ASSERT_OK(batch.Put("a", "....")); + ASSERT_OK(batch.Put("b", "....")); + s = batch.Put("c", "...."); + ASSERT_TRUE(s.IsMemoryLimit()); +} + +namespace { +class TimestampChecker : public WriteBatch::Handler { + public: + explicit TimestampChecker( + std::unordered_map cf_to_ucmps, Slice ts) + : cf_to_ucmps_(std::move(cf_to_ucmps)), timestamp_(std::move(ts)) {} + Status PutCF(uint32_t cf, const Slice& key, const Slice& /*value*/) override { + auto cf_iter = cf_to_ucmps_.find(cf); + if (cf_iter == cf_to_ucmps_.end()) { + return Status::Corruption(); + } + const Comparator* const ucmp = cf_iter->second; + assert(ucmp); + size_t ts_sz = ucmp->timestamp_size(); + if (ts_sz == 0) { + return Status::OK(); + } + if (key.size() < ts_sz) { + return Status::Corruption(); + } + Slice ts = ExtractTimestampFromUserKey(key, ts_sz); + if (ts.compare(timestamp_) != 0) { + return Status::Corruption(); + } + return Status::OK(); + } + + private: + std::unordered_map cf_to_ucmps_; + Slice timestamp_; +}; + +Status CheckTimestampsInWriteBatch( + WriteBatch& wb, Slice timestamp, + std::unordered_map cf_to_ucmps) { + TimestampChecker ts_checker(cf_to_ucmps, timestamp); + return wb.Iterate(&ts_checker); +} +} // anonymous namespace + +TEST_F(WriteBatchTest, SanityChecks) { + ColumnFamilyHandleImplDummy cf0(0, + test::BytewiseComparatorWithU64TsWrapper()); + ColumnFamilyHandleImplDummy cf4(4); + + WriteBatch wb(0, 0, 0, /*default_cf_ts_sz=*/sizeof(uint64_t)); + + // Sanity checks for the new WriteBatch APIs with extra 'ts' arg. + ASSERT_TRUE(wb.Put(nullptr, "key", "ts", "value").IsInvalidArgument()); + ASSERT_TRUE(wb.Delete(nullptr, "key", "ts").IsInvalidArgument()); + ASSERT_TRUE(wb.SingleDelete(nullptr, "key", "ts").IsInvalidArgument()); + ASSERT_TRUE(wb.Merge(nullptr, "key", "ts", "value").IsInvalidArgument()); + ASSERT_TRUE(wb.DeleteRange(nullptr, "begin_key", "end_key", "ts") + .IsInvalidArgument()); + + ASSERT_TRUE(wb.Put(&cf4, "key", "ts", "value").IsInvalidArgument()); + ASSERT_TRUE(wb.Delete(&cf4, "key", "ts").IsInvalidArgument()); + ASSERT_TRUE(wb.SingleDelete(&cf4, "key", "ts").IsInvalidArgument()); + ASSERT_TRUE(wb.Merge(&cf4, "key", "ts", "value").IsInvalidArgument()); + ASSERT_TRUE( + wb.DeleteRange(&cf4, "begin_key", "end_key", "ts").IsInvalidArgument()); + + constexpr size_t wrong_ts_sz = 1 + sizeof(uint64_t); + std::string ts(wrong_ts_sz, '\0'); + + ASSERT_TRUE(wb.Put(&cf0, "key", ts, "value").IsInvalidArgument()); + ASSERT_TRUE(wb.Delete(&cf0, "key", ts).IsInvalidArgument()); + ASSERT_TRUE(wb.SingleDelete(&cf0, "key", ts).IsInvalidArgument()); + ASSERT_TRUE(wb.Merge(&cf0, "key", ts, "value").IsInvalidArgument()); + ASSERT_TRUE( + wb.DeleteRange(&cf0, "begin_key", "end_key", ts).IsInvalidArgument()); + + // Sanity checks for the new WriteBatch APIs without extra 'ts' arg. + WriteBatch wb1(0, 0, 0, wrong_ts_sz); + ASSERT_TRUE(wb1.Put(&cf0, "key", "value").IsInvalidArgument()); + ASSERT_TRUE(wb1.Delete(&cf0, "key").IsInvalidArgument()); + ASSERT_TRUE(wb1.SingleDelete(&cf0, "key").IsInvalidArgument()); + ASSERT_TRUE(wb1.Merge(&cf0, "key", "value").IsInvalidArgument()); + ASSERT_TRUE( + wb1.DeleteRange(&cf0, "begin_key", "end_key").IsInvalidArgument()); +} + +TEST_F(WriteBatchTest, UpdateTimestamps) { + // We assume the last eight bytes of each key is reserved for timestamps. + // Therefore, we must make sure each key is longer than eight bytes. + constexpr size_t key_size = 16; + constexpr size_t num_of_keys = 10; + std::vector key_strs(num_of_keys, std::string(key_size, '\0')); + + ColumnFamilyHandleImplDummy cf0(0); + ColumnFamilyHandleImplDummy cf4(4, + test::BytewiseComparatorWithU64TsWrapper()); + ColumnFamilyHandleImplDummy cf5(5, + test::BytewiseComparatorWithU64TsWrapper()); + + const std::unordered_map cf_to_ucmps = { + {0, cf0.GetComparator()}, + {4, cf4.GetComparator()}, + {5, cf5.GetComparator()}}; + + static constexpr size_t timestamp_size = sizeof(uint64_t); + + { + WriteBatch wb1, wb2, wb3, wb4, wb5, wb6, wb7; + ASSERT_OK(wb1.Put(&cf0, "key", "value")); + ASSERT_FALSE(WriteBatchInternal::HasKeyWithTimestamp(wb1)); + ASSERT_OK(wb2.Put(&cf4, "key", "value")); + ASSERT_TRUE(WriteBatchInternal::HasKeyWithTimestamp(wb2)); + ASSERT_OK(wb3.Put(&cf4, "key", /*ts=*/std::string(timestamp_size, '\xfe'), + "value")); + ASSERT_TRUE(WriteBatchInternal::HasKeyWithTimestamp(wb3)); + ASSERT_OK(wb4.Delete(&cf4, "key", + /*ts=*/std::string(timestamp_size, '\xfe'))); + ASSERT_TRUE(WriteBatchInternal::HasKeyWithTimestamp(wb4)); + ASSERT_OK(wb5.Delete(&cf4, "key")); + ASSERT_TRUE(WriteBatchInternal::HasKeyWithTimestamp(wb5)); + ASSERT_OK(wb6.SingleDelete(&cf4, "key")); + ASSERT_TRUE(WriteBatchInternal::HasKeyWithTimestamp(wb6)); + ASSERT_OK(wb7.SingleDelete(&cf4, "key", + /*ts=*/std::string(timestamp_size, '\xfe'))); + ASSERT_TRUE(WriteBatchInternal::HasKeyWithTimestamp(wb7)); + } + + WriteBatch batch; + // Write to the batch. We will assign timestamps later. + for (const auto& key_str : key_strs) { + ASSERT_OK(batch.Put(&cf0, key_str, "value")); + ASSERT_OK(batch.Put(&cf4, key_str, "value")); + ASSERT_OK(batch.Put(&cf5, key_str, "value")); + } + + const auto checker1 = [](uint32_t cf) { + if (cf == 4 || cf == 5) { + return timestamp_size; + } else if (cf == 0) { + return static_cast(0); + } else { + return std::numeric_limits::max(); + } + }; + ASSERT_OK( + batch.UpdateTimestamps(std::string(timestamp_size, '\xfe'), checker1)); + ASSERT_OK(CheckTimestampsInWriteBatch( + batch, std::string(timestamp_size, '\xfe'), cf_to_ucmps)); + + // We use indexed_cf_to_ucmps, non_indexed_cfs_with_ts and timestamp_size to + // simulate the case in which a transaction enables indexing for some writes + // while disables indexing for other writes. A transaction uses a + // WriteBatchWithIndex object to buffer writes (we consider Write-committed + // policy only). If indexing is enabled, then writes go through + // WriteBatchWithIndex API populating a WBWI internal data structure, i.e. a + // mapping from cf to user comparators. If indexing is disabled, a transaction + // writes directly to the underlying raw WriteBatch. We will need to track the + // comparator information for the column families to which un-indexed writes + // are performed. When calling UpdateTimestamp API of WriteBatch, we need + // indexed_cf_to_ucmps, non_indexed_cfs_with_ts, and timestamp_size to perform + // checking. + std::unordered_map indexed_cf_to_ucmps = { + {0, cf0.GetComparator()}, {4, cf4.GetComparator()}}; + std::unordered_set non_indexed_cfs_with_ts = {cf5.GetID()}; + const auto checker2 = [&indexed_cf_to_ucmps, + &non_indexed_cfs_with_ts](uint32_t cf) { + if (non_indexed_cfs_with_ts.count(cf) > 0) { + return timestamp_size; + } + auto cf_iter = indexed_cf_to_ucmps.find(cf); + if (cf_iter == indexed_cf_to_ucmps.end()) { + assert(false); + return std::numeric_limits::max(); + } + const Comparator* const ucmp = cf_iter->second; + assert(ucmp); + return ucmp->timestamp_size(); + }; + ASSERT_OK( + batch.UpdateTimestamps(std::string(timestamp_size, '\xef'), checker2)); + ASSERT_OK(CheckTimestampsInWriteBatch( + batch, std::string(timestamp_size, '\xef'), cf_to_ucmps)); +} + +TEST_F(WriteBatchTest, CommitWithTimestamp) { + WriteBatch wb; + const std::string txn_name = "xid1"; + std::string ts; + constexpr uint64_t commit_ts = 23; + PutFixed64(&ts, commit_ts); + ASSERT_OK(WriteBatchInternal::MarkCommitWithTimestamp(&wb, txn_name, ts)); + TestHandler handler; + ASSERT_OK(wb.Iterate(&handler)); + ASSERT_EQ("MarkCommitWithTimestamp(" + txn_name + ", " + + Slice(ts).ToString(true) + ")", + handler.seen); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/write_callback.h b/librocksdb-sys/rocksdb/db/write_callback.h new file mode 100644 index 0000000..106d020 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/write_callback.h @@ -0,0 +1,27 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +class DB; + +class WriteCallback { + public: + virtual ~WriteCallback() {} + + // Will be called while on the write thread before the write executes. If + // this function returns a non-OK status, the write will be aborted and this + // status will be returned to the caller of DB::Write(). + virtual Status Callback(DB* db) = 0; + + // return true if writes with this callback can be batched with other writes + virtual bool AllowWriteBatching() = 0; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/write_callback_test.cc b/librocksdb-sys/rocksdb/db/write_callback_test.cc new file mode 100644 index 0000000..1be8593 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/write_callback_test.cc @@ -0,0 +1,454 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + + +#include "db/write_callback.h" + +#include +#include +#include +#include +#include + +#include "db/db_impl/db_impl.h" +#include "port/port.h" +#include "rocksdb/db.h" +#include "rocksdb/write_batch.h" +#include "test_util/sync_point.h" +#include "test_util/testharness.h" +#include "util/random.h" + +using std::string; + +namespace ROCKSDB_NAMESPACE { + +class WriteCallbackTest : public testing::Test { + public: + string dbname; + + WriteCallbackTest() { + dbname = test::PerThreadDBPath("write_callback_testdb"); + } +}; + +class WriteCallbackTestWriteCallback1 : public WriteCallback { + public: + bool was_called = false; + + Status Callback(DB* db) override { + was_called = true; + + // Make sure db is a DBImpl + DBImpl* db_impl = dynamic_cast(db); + if (db_impl == nullptr) { + return Status::InvalidArgument(""); + } + + return Status::OK(); + } + + bool AllowWriteBatching() override { return true; } +}; + +class WriteCallbackTestWriteCallback2 : public WriteCallback { + public: + Status Callback(DB* /*db*/) override { return Status::Busy(); } + bool AllowWriteBatching() override { return true; } +}; + +class MockWriteCallback : public WriteCallback { + public: + bool should_fail_ = false; + bool allow_batching_ = false; + std::atomic was_called_{false}; + + MockWriteCallback() {} + + MockWriteCallback(const MockWriteCallback& other) { + should_fail_ = other.should_fail_; + allow_batching_ = other.allow_batching_; + was_called_.store(other.was_called_.load()); + } + + Status Callback(DB* /*db*/) override { + was_called_.store(true); + if (should_fail_) { + return Status::Busy(); + } else { + return Status::OK(); + } + } + + bool AllowWriteBatching() override { return allow_batching_; } +}; + +#if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) +class WriteCallbackPTest + : public WriteCallbackTest, + public ::testing::WithParamInterface< + std::tuple> { + public: + WriteCallbackPTest() { + std::tie(unordered_write_, seq_per_batch_, two_queues_, allow_parallel_, + allow_batching_, enable_WAL_, enable_pipelined_write_) = + GetParam(); + } + + protected: + bool unordered_write_; + bool seq_per_batch_; + bool two_queues_; + bool allow_parallel_; + bool allow_batching_; + bool enable_WAL_; + bool enable_pipelined_write_; +}; + +TEST_P(WriteCallbackPTest, WriteWithCallbackTest) { + struct WriteOP { + WriteOP(bool should_fail = false) { callback_.should_fail_ = should_fail; } + + void Put(const string& key, const string& val) { + kvs_.push_back(std::make_pair(key, val)); + ASSERT_OK(write_batch_.Put(key, val)); + } + + void Clear() { + kvs_.clear(); + write_batch_.Clear(); + callback_.was_called_.store(false); + } + + MockWriteCallback callback_; + WriteBatch write_batch_; + std::vector> kvs_; + }; + + // In each scenario we'll launch multiple threads to write. + // The size of each array equals to number of threads, and + // each boolean in it denote whether callback of corresponding + // thread should succeed or fail. + std::vector> write_scenarios = { + {true}, + {false}, + {false, false}, + {true, true}, + {true, false}, + {false, true}, + {false, false, false}, + {true, true, true}, + {false, true, false}, + {true, false, true}, + {true, false, false, false, false}, + {false, false, false, false, true}, + {false, false, true, false, true}, + }; + + for (auto& write_group : write_scenarios) { + Options options; + options.create_if_missing = true; + options.unordered_write = unordered_write_; + options.allow_concurrent_memtable_write = allow_parallel_; + options.enable_pipelined_write = enable_pipelined_write_; + options.two_write_queues = two_queues_; + // Skip unsupported combinations + if (options.enable_pipelined_write && seq_per_batch_) { + continue; + } + if (options.enable_pipelined_write && options.two_write_queues) { + continue; + } + if (options.unordered_write && !options.allow_concurrent_memtable_write) { + continue; + } + if (options.unordered_write && options.enable_pipelined_write) { + continue; + } + + ReadOptions read_options; + DB* db; + DBImpl* db_impl; + + ASSERT_OK(DestroyDB(dbname, options)); + + DBOptions db_options(options); + ColumnFamilyOptions cf_options(options); + std::vector column_families; + column_families.push_back( + ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options)); + std::vector handles; + auto open_s = DBImpl::Open(db_options, dbname, column_families, &handles, + &db, seq_per_batch_, true /* batch_per_txn */); + ASSERT_OK(open_s); + assert(handles.size() == 1); + delete handles[0]; + + db_impl = dynamic_cast(db); + ASSERT_TRUE(db_impl); + + // Writers that have called JoinBatchGroup. + std::atomic threads_joining(0); + // Writers that have linked to the queue + std::atomic threads_linked(0); + // Writers that pass WriteThread::JoinBatchGroup:Wait sync-point. + std::atomic threads_verified(0); + + std::atomic seq(db_impl->GetLatestSequenceNumber()); + ASSERT_EQ(db_impl->GetLatestSequenceNumber(), 0); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WriteThread::JoinBatchGroup:Start", [&](void*) { + uint64_t cur_threads_joining = threads_joining.fetch_add(1); + // Wait for the last joined writer to link to the queue. + // In this way the writers link to the queue one by one. + // This allows us to confidently detect the first writer + // who increases threads_linked as the leader. + while (threads_linked.load() < cur_threads_joining) { + } + }); + + // Verification once writers call JoinBatchGroup. + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WriteThread::JoinBatchGroup:Wait", [&](void* arg) { + uint64_t cur_threads_linked = threads_linked.fetch_add(1); + bool is_leader = false; + bool is_last = false; + + // who am i + is_leader = (cur_threads_linked == 0); + is_last = (cur_threads_linked == write_group.size() - 1); + + // check my state + auto* writer = reinterpret_cast(arg); + + if (is_leader) { + ASSERT_TRUE(writer->state == + WriteThread::State::STATE_GROUP_LEADER); + } else { + ASSERT_TRUE(writer->state == WriteThread::State::STATE_INIT); + } + + // (meta test) the first WriteOP should indeed be the first + // and the last should be the last (all others can be out of + // order) + if (is_leader) { + ASSERT_TRUE(writer->callback->Callback(nullptr).ok() == + !write_group.front().callback_.should_fail_); + } else if (is_last) { + ASSERT_TRUE(writer->callback->Callback(nullptr).ok() == + !write_group.back().callback_.should_fail_); + } + + threads_verified.fetch_add(1); + // Wait here until all verification in this sync-point + // callback finish for all writers. + while (threads_verified.load() < write_group.size()) { + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "WriteThread::JoinBatchGroup:DoneWaiting", [&](void* arg) { + // check my state + auto* writer = reinterpret_cast(arg); + + if (!allow_batching_) { + // no batching so everyone should be a leader + ASSERT_TRUE(writer->state == + WriteThread::State::STATE_GROUP_LEADER); + } else if (!allow_parallel_) { + ASSERT_TRUE(writer->state == WriteThread::State::STATE_COMPLETED || + (enable_pipelined_write_ && + writer->state == + WriteThread::State::STATE_MEMTABLE_WRITER_LEADER)); + } + }); + + std::atomic thread_num(0); + std::atomic dummy_key(0); + + // Each write thread create a random write batch and write to DB + // with a write callback. + std::function write_with_callback_func = [&]() { + uint32_t i = thread_num.fetch_add(1); + Random rnd(i); + + // leaders gotta lead + while (i > 0 && threads_verified.load() < 1) { + } + + // loser has to lose + while (i == write_group.size() - 1 && + threads_verified.load() < write_group.size() - 1) { + } + + auto& write_op = write_group.at(i); + write_op.Clear(); + write_op.callback_.allow_batching_ = allow_batching_; + + // insert some keys + for (uint32_t j = 0; j < rnd.Next() % 50; j++) { + // grab unique key + char my_key = dummy_key.fetch_add(1); + + string skey(5, my_key); + string sval(10, my_key); + write_op.Put(skey, sval); + + if (!write_op.callback_.should_fail_ && !seq_per_batch_) { + seq.fetch_add(1); + } + } + if (!write_op.callback_.should_fail_ && seq_per_batch_) { + seq.fetch_add(1); + } + + WriteOptions woptions; + woptions.disableWAL = !enable_WAL_; + woptions.sync = enable_WAL_; + if (woptions.protection_bytes_per_key > 0) { + ASSERT_OK(WriteBatchInternal::UpdateProtectionInfo( + &write_op.write_batch_, woptions.protection_bytes_per_key)); + } + Status s; + if (seq_per_batch_) { + class PublishSeqCallback : public PreReleaseCallback { + public: + PublishSeqCallback(DBImpl* db_impl_in) : db_impl_(db_impl_in) {} + Status Callback(SequenceNumber last_seq, bool /*not used*/, uint64_t, + size_t /*index*/, size_t /*total*/) override { + db_impl_->SetLastPublishedSequence(last_seq); + return Status::OK(); + } + DBImpl* db_impl_; + } publish_seq_callback(db_impl); + // seq_per_batch_ requires a natural batch separator or Noop + ASSERT_OK(WriteBatchInternal::InsertNoop(&write_op.write_batch_)); + const size_t ONE_BATCH = 1; + s = db_impl->WriteImpl(woptions, &write_op.write_batch_, + &write_op.callback_, nullptr, 0, false, nullptr, + ONE_BATCH, + two_queues_ ? &publish_seq_callback : nullptr); + } else { + s = db_impl->WriteWithCallback(woptions, &write_op.write_batch_, + &write_op.callback_); + } + + if (write_op.callback_.should_fail_) { + ASSERT_TRUE(s.IsBusy()); + } else { + ASSERT_OK(s); + } + }; + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // do all the writes + std::vector threads; + for (uint32_t i = 0; i < write_group.size(); i++) { + threads.emplace_back(write_with_callback_func); + } + for (auto& t : threads) { + t.join(); + } + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + + // check for keys + string value; + for (auto& w : write_group) { + ASSERT_TRUE(w.callback_.was_called_.load()); + for (auto& kvp : w.kvs_) { + if (w.callback_.should_fail_) { + ASSERT_TRUE(db->Get(read_options, kvp.first, &value).IsNotFound()); + } else { + ASSERT_OK(db->Get(read_options, kvp.first, &value)); + ASSERT_EQ(value, kvp.second); + } + } + } + + ASSERT_EQ(seq.load(), db_impl->TEST_GetLastVisibleSequence()); + + delete db; + ASSERT_OK(DestroyDB(dbname, options)); + } +} + +INSTANTIATE_TEST_CASE_P(WriteCallbackPTest, WriteCallbackPTest, + ::testing::Combine(::testing::Bool(), ::testing::Bool(), + ::testing::Bool(), ::testing::Bool(), + ::testing::Bool(), ::testing::Bool(), + ::testing::Bool())); +#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) + +TEST_F(WriteCallbackTest, WriteCallBackTest) { + Options options; + WriteOptions write_options; + ReadOptions read_options; + string value; + DB* db; + DBImpl* db_impl; + + ASSERT_OK(DestroyDB(dbname, options)); + + options.create_if_missing = true; + Status s = DB::Open(options, dbname, &db); + ASSERT_OK(s); + + db_impl = dynamic_cast(db); + ASSERT_TRUE(db_impl); + + WriteBatch wb; + + ASSERT_OK(wb.Put("a", "value.a")); + ASSERT_OK(wb.Delete("x")); + + // Test a simple Write + s = db->Write(write_options, &wb); + ASSERT_OK(s); + + s = db->Get(read_options, "a", &value); + ASSERT_OK(s); + ASSERT_EQ("value.a", value); + + // Test WriteWithCallback + WriteCallbackTestWriteCallback1 callback1; + WriteBatch wb2; + + ASSERT_OK(wb2.Put("a", "value.a2")); + + s = db_impl->WriteWithCallback(write_options, &wb2, &callback1); + ASSERT_OK(s); + ASSERT_TRUE(callback1.was_called); + + s = db->Get(read_options, "a", &value); + ASSERT_OK(s); + ASSERT_EQ("value.a2", value); + + // Test WriteWithCallback for a callback that fails + WriteCallbackTestWriteCallback2 callback2; + WriteBatch wb3; + + ASSERT_OK(wb3.Put("a", "value.a3")); + + s = db_impl->WriteWithCallback(write_options, &wb3, &callback2); + ASSERT_NOK(s); + + s = db->Get(read_options, "a", &value); + ASSERT_OK(s); + ASSERT_EQ("value.a2", value); + + delete db; + ASSERT_OK(DestroyDB(dbname, options)); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/librocksdb-sys/rocksdb/db/write_controller.cc b/librocksdb-sys/rocksdb/db/write_controller.cc new file mode 100644 index 0000000..c5f7443 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/write_controller.cc @@ -0,0 +1,121 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/write_controller.h" + +#include +#include +#include +#include + +#include "rocksdb/system_clock.h" + +namespace ROCKSDB_NAMESPACE { + +std::unique_ptr WriteController::GetStopToken() { + ++total_stopped_; + return std::unique_ptr(new StopWriteToken(this)); +} + +std::unique_ptr WriteController::GetDelayToken( + uint64_t write_rate) { + if (0 == total_delayed_++) { + // Starting delay, so reset counters. + next_refill_time_ = 0; + credit_in_bytes_ = 0; + } + // NOTE: for simplicity, any current credit_in_bytes_ or "debt" in + // next_refill_time_ will be based on an old rate. This rate will apply + // for subsequent additional debts and for the next refill. + set_delayed_write_rate(write_rate); + return std::unique_ptr(new DelayWriteToken(this)); +} + +std::unique_ptr +WriteController::GetCompactionPressureToken() { + ++total_compaction_pressure_; + return std::unique_ptr( + new CompactionPressureToken(this)); +} + +bool WriteController::IsStopped() const { + return total_stopped_.load(std::memory_order_relaxed) > 0; +} +// This is inside DB mutex, so we can't sleep and need to minimize +// frequency to get time. +// If it turns out to be a performance issue, we can redesign the thread +// synchronization model here. +// The function trust caller will sleep micros returned. +uint64_t WriteController::GetDelay(SystemClock* clock, uint64_t num_bytes) { + if (total_stopped_.load(std::memory_order_relaxed) > 0) { + return 0; + } + if (total_delayed_.load(std::memory_order_relaxed) == 0) { + return 0; + } + + if (credit_in_bytes_ >= num_bytes) { + credit_in_bytes_ -= num_bytes; + return 0; + } + // The frequency to get time inside DB mutex is less than one per refill + // interval. + auto time_now = NowMicrosMonotonic(clock); + + const uint64_t kMicrosPerSecond = 1000000; + // Refill every 1 ms + const uint64_t kMicrosPerRefill = 1000; + + if (next_refill_time_ == 0) { + // Start with an initial allotment of bytes for one interval + next_refill_time_ = time_now; + } + if (next_refill_time_ <= time_now) { + // Refill based on time interval plus any extra elapsed + uint64_t elapsed = time_now - next_refill_time_ + kMicrosPerRefill; + credit_in_bytes_ += static_cast( + 1.0 * elapsed / kMicrosPerSecond * delayed_write_rate_ + 0.999999); + next_refill_time_ = time_now + kMicrosPerRefill; + + if (credit_in_bytes_ >= num_bytes) { + // Avoid delay if possible, to reduce DB mutex release & re-aquire. + credit_in_bytes_ -= num_bytes; + return 0; + } + } + + // We need to delay to avoid exceeding write rate. + assert(num_bytes > credit_in_bytes_); + uint64_t bytes_over_budget = num_bytes - credit_in_bytes_; + uint64_t needed_delay = static_cast( + 1.0 * bytes_over_budget / delayed_write_rate_ * kMicrosPerSecond); + + credit_in_bytes_ = 0; + next_refill_time_ += needed_delay; + + // Minimum delay of refill interval, to reduce DB mutex contention. + return std::max(next_refill_time_ - time_now, kMicrosPerRefill); +} + +uint64_t WriteController::NowMicrosMonotonic(SystemClock* clock) { + return clock->NowNanos() / std::milli::den; +} + +StopWriteToken::~StopWriteToken() { + assert(controller_->total_stopped_ >= 1); + --controller_->total_stopped_; +} + +DelayWriteToken::~DelayWriteToken() { + controller_->total_delayed_--; + assert(controller_->total_delayed_.load() >= 0); +} + +CompactionPressureToken::~CompactionPressureToken() { + controller_->total_compaction_pressure_--; + assert(controller_->total_compaction_pressure_ >= 0); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/write_controller.h b/librocksdb-sys/rocksdb/db/write_controller.h new file mode 100644 index 0000000..bcead16 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/write_controller.h @@ -0,0 +1,148 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include +#include + +#include "rocksdb/rate_limiter.h" + +namespace ROCKSDB_NAMESPACE { + +class SystemClock; +class WriteControllerToken; + +// WriteController is controlling write stalls in our write code-path. Write +// stalls happen when compaction can't keep up with write rate. +// All of the methods here (including WriteControllerToken's destructors) need +// to be called while holding DB mutex +class WriteController { + public: + explicit WriteController(uint64_t _delayed_write_rate = 1024u * 1024u * 32u, + int64_t low_pri_rate_bytes_per_sec = 1024 * 1024) + : total_stopped_(0), + total_delayed_(0), + total_compaction_pressure_(0), + credit_in_bytes_(0), + next_refill_time_(0), + low_pri_rate_limiter_( + NewGenericRateLimiter(low_pri_rate_bytes_per_sec)) { + set_max_delayed_write_rate(_delayed_write_rate); + } + ~WriteController() = default; + + // When an actor (column family) requests a stop token, all writes will be + // stopped until the stop token is released (deleted) + std::unique_ptr GetStopToken(); + // When an actor (column family) requests a delay token, total delay for all + // writes to the DB will be controlled under the delayed write rate. Every + // write needs to call GetDelay() with number of bytes writing to the DB, + // which returns number of microseconds to sleep. + std::unique_ptr GetDelayToken( + uint64_t delayed_write_rate); + // When an actor (column family) requests a moderate token, compaction + // threads will be increased + std::unique_ptr GetCompactionPressureToken(); + + // these three metods are querying the state of the WriteController + bool IsStopped() const; + bool NeedsDelay() const { return total_delayed_.load() > 0; } + bool NeedSpeedupCompaction() const { + return IsStopped() || NeedsDelay() || total_compaction_pressure_.load() > 0; + } + // return how many microseconds the caller needs to sleep after the call + // num_bytes: how many number of bytes to put into the DB. + // Prerequisite: DB mutex held. + uint64_t GetDelay(SystemClock* clock, uint64_t num_bytes); + void set_delayed_write_rate(uint64_t write_rate) { + // avoid divide 0 + if (write_rate == 0) { + write_rate = 1u; + } else if (write_rate > max_delayed_write_rate()) { + write_rate = max_delayed_write_rate(); + } + delayed_write_rate_ = write_rate; + } + + void set_max_delayed_write_rate(uint64_t write_rate) { + // avoid divide 0 + if (write_rate == 0) { + write_rate = 1u; + } + max_delayed_write_rate_ = write_rate; + // update delayed_write_rate_ as well + delayed_write_rate_ = write_rate; + } + + uint64_t delayed_write_rate() const { return delayed_write_rate_; } + + uint64_t max_delayed_write_rate() const { return max_delayed_write_rate_; } + + RateLimiter* low_pri_rate_limiter() { return low_pri_rate_limiter_.get(); } + + private: + uint64_t NowMicrosMonotonic(SystemClock* clock); + + friend class WriteControllerToken; + friend class StopWriteToken; + friend class DelayWriteToken; + friend class CompactionPressureToken; + + std::atomic total_stopped_; + std::atomic total_delayed_; + std::atomic total_compaction_pressure_; + + // Number of bytes allowed to write without delay + uint64_t credit_in_bytes_; + // Next time that we can add more credit of bytes + uint64_t next_refill_time_; + // Write rate set when initialization or by `DBImpl::SetDBOptions` + uint64_t max_delayed_write_rate_; + // Current write rate (bytes / second) + uint64_t delayed_write_rate_; + + std::unique_ptr low_pri_rate_limiter_; +}; + +class WriteControllerToken { + public: + explicit WriteControllerToken(WriteController* controller) + : controller_(controller) {} + virtual ~WriteControllerToken() {} + + protected: + WriteController* controller_; + + private: + // no copying allowed + WriteControllerToken(const WriteControllerToken&) = delete; + void operator=(const WriteControllerToken&) = delete; +}; + +class StopWriteToken : public WriteControllerToken { + public: + explicit StopWriteToken(WriteController* controller) + : WriteControllerToken(controller) {} + virtual ~StopWriteToken(); +}; + +class DelayWriteToken : public WriteControllerToken { + public: + explicit DelayWriteToken(WriteController* controller) + : WriteControllerToken(controller) {} + virtual ~DelayWriteToken(); +}; + +class CompactionPressureToken : public WriteControllerToken { + public: + explicit CompactionPressureToken(WriteController* controller) + : WriteControllerToken(controller) {} + virtual ~CompactionPressureToken(); +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/write_controller_test.cc b/librocksdb-sys/rocksdb/db/write_controller_test.cc new file mode 100644 index 0000000..b6321a3 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/write_controller_test.cc @@ -0,0 +1,248 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#include "db/write_controller.h" + +#include +#include + +#include "rocksdb/system_clock.h" +#include "test_util/testharness.h" + +namespace ROCKSDB_NAMESPACE { +namespace { +class TimeSetClock : public SystemClockWrapper { + public: + explicit TimeSetClock() : SystemClockWrapper(nullptr) {} + const char* Name() const override { return "TimeSetClock"; } + uint64_t now_micros_ = 6666; + uint64_t NowNanos() override { return now_micros_ * std::milli::den; } +}; +} // anonymous namespace +class WriteControllerTest : public testing::Test { + public: + WriteControllerTest() { clock_ = std::make_shared(); } + std::shared_ptr clock_; +}; + +// Make tests easier to read +#define MILLION *1000000u +#define MB MILLION +#define MBPS MILLION +#define SECS MILLION // in microseconds + +TEST_F(WriteControllerTest, BasicAPI) { + WriteController controller(40 MBPS); // also set max delayed rate + EXPECT_EQ(controller.delayed_write_rate(), 40 MBPS); + EXPECT_FALSE(controller.IsStopped()); + EXPECT_FALSE(controller.NeedsDelay()); + EXPECT_EQ(0, controller.GetDelay(clock_.get(), 100 MB)); + + // set, get + controller.set_delayed_write_rate(20 MBPS); + EXPECT_EQ(controller.delayed_write_rate(), 20 MBPS); + EXPECT_FALSE(controller.IsStopped()); + EXPECT_FALSE(controller.NeedsDelay()); + EXPECT_EQ(0, controller.GetDelay(clock_.get(), 100 MB)); + + { + // set with token, get + auto delay_token_0 = controller.GetDelayToken(10 MBPS); + EXPECT_EQ(controller.delayed_write_rate(), 10 MBPS); + EXPECT_FALSE(controller.IsStopped()); + EXPECT_TRUE(controller.NeedsDelay()); + // test with delay + EXPECT_EQ(2 SECS, controller.GetDelay(clock_.get(), 20 MB)); + clock_->now_micros_ += 2 SECS; // pay the "debt" + + auto delay_token_1 = controller.GetDelayToken(2 MBPS); + EXPECT_EQ(10 SECS, controller.GetDelay(clock_.get(), 20 MB)); + clock_->now_micros_ += 10 SECS; // pay the "debt" + + auto delay_token_2 = controller.GetDelayToken(1 MBPS); + EXPECT_EQ(20 SECS, controller.GetDelay(clock_.get(), 20 MB)); + clock_->now_micros_ += 20 SECS; // pay the "debt" + + auto delay_token_3 = controller.GetDelayToken(20 MBPS); + EXPECT_EQ(1 SECS, controller.GetDelay(clock_.get(), 20 MB)); + clock_->now_micros_ += 1 SECS; // pay the "debt" + + // 60M is more than the max rate of 40M. Max rate will be used. + EXPECT_EQ(controller.delayed_write_rate(), 20 MBPS); + auto delay_token_4 = + controller.GetDelayToken(controller.delayed_write_rate() * 3); + EXPECT_EQ(controller.delayed_write_rate(), 40 MBPS); + EXPECT_EQ(static_cast(0.5 SECS), + controller.GetDelay(clock_.get(), 20 MB)); + + EXPECT_FALSE(controller.IsStopped()); + EXPECT_TRUE(controller.NeedsDelay()); + + // Test stop tokens + { + auto stop_token_1 = controller.GetStopToken(); + EXPECT_TRUE(controller.IsStopped()); + EXPECT_EQ(0, controller.GetDelay(clock_.get(), 100 MB)); + { + auto stop_token_2 = controller.GetStopToken(); + EXPECT_TRUE(controller.IsStopped()); + EXPECT_EQ(0, controller.GetDelay(clock_.get(), 100 MB)); + } + EXPECT_TRUE(controller.IsStopped()); + EXPECT_EQ(0, controller.GetDelay(clock_.get(), 100 MB)); + } + // Stop tokens released + EXPECT_FALSE(controller.IsStopped()); + EXPECT_TRUE(controller.NeedsDelay()); + EXPECT_EQ(controller.delayed_write_rate(), 40 MBPS); + // pay the previous "debt" + clock_->now_micros_ += static_cast(0.5 SECS); + EXPECT_EQ(1 SECS, controller.GetDelay(clock_.get(), 40 MB)); + } + + // Delay tokens released + EXPECT_FALSE(controller.NeedsDelay()); +} + +TEST_F(WriteControllerTest, StartFilled) { + WriteController controller(10 MBPS); + + // Attempt to write two things that combined would be allowed within + // a single refill interval + auto delay_token_0 = + controller.GetDelayToken(controller.delayed_write_rate()); + + // Verify no delay because write rate has not been exceeded within + // refill interval. + EXPECT_EQ(0U, controller.GetDelay(clock_.get(), 2000u /*bytes*/)); + EXPECT_EQ(0U, controller.GetDelay(clock_.get(), 2000u /*bytes*/)); + + // Allow refill (kMicrosPerRefill) + clock_->now_micros_ += 1000; + + // Again + EXPECT_EQ(0U, controller.GetDelay(clock_.get(), 2000u /*bytes*/)); + EXPECT_EQ(0U, controller.GetDelay(clock_.get(), 2000u /*bytes*/)); + + // Control: something bigger that would exceed write rate within interval + uint64_t delay = controller.GetDelay(clock_.get(), 10 MB); + EXPECT_GT(1.0 * delay, 0.999 SECS); + EXPECT_LT(1.0 * delay, 1.001 SECS); +} + +TEST_F(WriteControllerTest, DebtAccumulation) { + WriteController controller(10 MBPS); + + std::array, 10> tokens; + + // Accumulate a time delay debt with no passage of time, like many column + // families delaying writes simultaneously. (Old versions of WriteController + // would reset the debt on every GetDelayToken.) + uint64_t debt = 0; + for (unsigned i = 0; i < tokens.size(); ++i) { + tokens[i] = controller.GetDelayToken((i + 1u) MBPS); + uint64_t delay = controller.GetDelay(clock_.get(), 63 MB); + ASSERT_GT(delay, debt); + uint64_t incremental = delay - debt; + ASSERT_EQ(incremental, (63 SECS) / (i + 1u)); + debt += incremental; + } + + // Pay down the debt + clock_->now_micros_ += debt; + debt = 0; + + // Now accumulate debt with some passage of time. + for (unsigned i = 0; i < tokens.size(); ++i) { + // Debt is accumulated in time, not in bytes, so this new write + // limit is not applied to prior requested delays, even it they are + // in progress. + tokens[i] = controller.GetDelayToken((i + 1u) MBPS); + uint64_t delay = controller.GetDelay(clock_.get(), 63 MB); + ASSERT_GT(delay, debt); + uint64_t incremental = delay - debt; + ASSERT_EQ(incremental, (63 SECS) / (i + 1u)); + debt += incremental; + uint64_t credit = debt / 2; + clock_->now_micros_ += credit; + debt -= credit; + } + + // Pay down the debt + clock_->now_micros_ += debt; + debt = 0; // consistent state + (void)debt; // appease clang-analyze + + // Verify paid down + EXPECT_EQ(0U, controller.GetDelay(clock_.get(), 100u /*small bytes*/)); + + // Accumulate another debt, without accounting, and releasing tokens + for (unsigned i = 0; i < tokens.size(); ++i) { + // Big and small are delayed + ASSERT_LT(0U, controller.GetDelay(clock_.get(), 63 MB)); + ASSERT_LT(0U, controller.GetDelay(clock_.get(), 100u /*small bytes*/)); + tokens[i].reset(); + } + // All tokens released. + // Verify that releasing all tokens pays down debt, even with no time passage. + tokens[0] = controller.GetDelayToken(1 MBPS); + ASSERT_EQ(0U, controller.GetDelay(clock_.get(), 100u /*small bytes*/)); +} + +// This may or may not be a "good" feature, but it's an old feature +TEST_F(WriteControllerTest, CreditAccumulation) { + WriteController controller(10 MBPS); + + std::array, 10> tokens; + + // Ensure started + tokens[0] = controller.GetDelayToken(1 MBPS); + ASSERT_EQ(10 SECS, controller.GetDelay(clock_.get(), 10 MB)); + clock_->now_micros_ += 10 SECS; + + // Accumulate a credit + uint64_t credit = 1000 SECS /* see below: * 1 MB / 1 SEC */; + clock_->now_micros_ += credit; + + // Spend some credit (burst of I/O) + for (unsigned i = 0; i < tokens.size(); ++i) { + tokens[i] = controller.GetDelayToken((i + 1u) MBPS); + ASSERT_EQ(0U, controller.GetDelay(clock_.get(), 63 MB)); + // In WriteController, credit is accumulated in bytes, not in time. + // After an "unnecessary" delay, all of our time credit will be + // translated to bytes on the next operation, in this case with + // setting 1 MBPS. So regardless of the rate at delay time, we just + // account for the bytes. + credit -= 63 MB; + } + // Spend remaining credit + tokens[0] = controller.GetDelayToken(1 MBPS); + ASSERT_EQ(0U, controller.GetDelay(clock_.get(), credit)); + // Verify + ASSERT_EQ(10 SECS, controller.GetDelay(clock_.get(), 10 MB)); + clock_->now_micros_ += 10 SECS; + + // Accumulate a credit, no accounting + clock_->now_micros_ += 1000 SECS; + + // Spend a small amount, releasing tokens + for (unsigned i = 0; i < tokens.size(); ++i) { + ASSERT_EQ(0U, controller.GetDelay(clock_.get(), 3 MB)); + tokens[i].reset(); + } + + // All tokens released. + // Verify credit is wiped away on new delay. + tokens[0] = controller.GetDelayToken(1 MBPS); + ASSERT_EQ(10 SECS, controller.GetDelay(clock_.get(), 10 MB)); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/db/write_stall_stats.cc b/librocksdb-sys/rocksdb/db/write_stall_stats.cc new file mode 100644 index 0000000..3973df7 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/write_stall_stats.cc @@ -0,0 +1,179 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/write_stall_stats.h" + +namespace ROCKSDB_NAMESPACE { +const std::string& InvalidWriteStallHyphenString() { + static const std::string kInvalidWriteStallHyphenString = "invalid"; + return kInvalidWriteStallHyphenString; +} + +const std::string& WriteStallCauseToHyphenString(WriteStallCause cause) { + static const std::string kMemtableLimit = "memtable-limit"; + static const std::string kL0FileCountLimit = "l0-file-count-limit"; + static const std::string kPendingCompactionBytes = "pending-compaction-bytes"; + static const std::string kWriteBufferManagerLimit = + "write-buffer-manager-limit"; + switch (cause) { + case WriteStallCause::kMemtableLimit: + return kMemtableLimit; + case WriteStallCause::kL0FileCountLimit: + return kL0FileCountLimit; + case WriteStallCause::kPendingCompactionBytes: + return kPendingCompactionBytes; + case WriteStallCause::kWriteBufferManagerLimit: + return kWriteBufferManagerLimit; + default: + break; + } + return InvalidWriteStallHyphenString(); +} + +const std::string& WriteStallConditionToHyphenString( + WriteStallCondition condition) { + static const std::string kDelayed = "delays"; + static const std::string kStopped = "stops"; + switch (condition) { + case WriteStallCondition::kDelayed: + return kDelayed; + case WriteStallCondition::kStopped: + return kStopped; + default: + break; + } + return InvalidWriteStallHyphenString(); +} + +InternalStats::InternalCFStatsType InternalCFStat( + WriteStallCause cause, WriteStallCondition condition) { + switch (cause) { + case WriteStallCause::kMemtableLimit: { + switch (condition) { + case WriteStallCondition::kDelayed: + return InternalStats::MEMTABLE_LIMIT_DELAYS; + case WriteStallCondition::kStopped: + return InternalStats::MEMTABLE_LIMIT_STOPS; + case WriteStallCondition::kNormal: + break; + } + break; + } + case WriteStallCause::kL0FileCountLimit: { + switch (condition) { + case WriteStallCondition::kDelayed: + return InternalStats::L0_FILE_COUNT_LIMIT_DELAYS; + case WriteStallCondition::kStopped: + return InternalStats::L0_FILE_COUNT_LIMIT_STOPS; + case WriteStallCondition::kNormal: + break; + } + break; + } + case WriteStallCause::kPendingCompactionBytes: { + switch (condition) { + case WriteStallCondition::kDelayed: + return InternalStats::PENDING_COMPACTION_BYTES_LIMIT_DELAYS; + case WriteStallCondition::kStopped: + return InternalStats::PENDING_COMPACTION_BYTES_LIMIT_STOPS; + case WriteStallCondition::kNormal: + break; + } + break; + } + default: + break; + } + return InternalStats::INTERNAL_CF_STATS_ENUM_MAX; +} + +InternalStats::InternalDBStatsType InternalDBStat( + WriteStallCause cause, WriteStallCondition condition) { + switch (cause) { + case WriteStallCause::kWriteBufferManagerLimit: { + switch (condition) { + case WriteStallCondition::kStopped: + return InternalStats::kIntStatsWriteBufferManagerLimitStopsCounts; + default: + break; + } + break; + } + default: + break; + } + return InternalStats::kIntStatsNumMax; +} + +bool isCFScopeWriteStallCause(WriteStallCause cause) { + uint32_t int_cause = static_cast(cause); + uint32_t lower_bound = + static_cast(WriteStallCause::kCFScopeWriteStallCauseEnumMax) - + kNumCFScopeWriteStallCauses; + uint32_t upper_bound = + static_cast(WriteStallCause::kCFScopeWriteStallCauseEnumMax) - + 1; + return lower_bound <= int_cause && int_cause <= upper_bound; +} + +bool isDBScopeWriteStallCause(WriteStallCause cause) { + uint32_t int_cause = static_cast(cause); + uint32_t lower_bound = + static_cast(WriteStallCause::kDBScopeWriteStallCauseEnumMax) - + kNumDBScopeWriteStallCauses; + uint32_t upper_bound = + static_cast(WriteStallCause::kDBScopeWriteStallCauseEnumMax) - + 1; + return lower_bound <= int_cause && int_cause <= upper_bound; +} + +const std::string& WriteStallStatsMapKeys::TotalStops() { + static const std::string kTotalStops = "total-stops"; + return kTotalStops; +} + +const std::string& WriteStallStatsMapKeys::TotalDelays() { + static const std::string kTotalDelays = "total-delays"; + return kTotalDelays; +} + +const std::string& +WriteStallStatsMapKeys::CFL0FileCountLimitDelaysWithOngoingCompaction() { + static const std::string ret = + "cf-l0-file-count-limit-delays-with-ongoing-compaction"; + return ret; +} + +const std::string& +WriteStallStatsMapKeys::CFL0FileCountLimitStopsWithOngoingCompaction() { + static const std::string ret = + "cf-l0-file-count-limit-stops-with-ongoing-compaction"; + return ret; +} + +std::string WriteStallStatsMapKeys::CauseConditionCount( + WriteStallCause cause, WriteStallCondition condition) { + std::string cause_condition_count_name; + + std::string cause_name; + if (isCFScopeWriteStallCause(cause) || isDBScopeWriteStallCause(cause)) { + cause_name = WriteStallCauseToHyphenString(cause); + } else { + assert(false); + return ""; + } + + const std::string& condition_name = + WriteStallConditionToHyphenString(condition); + + cause_condition_count_name.reserve(cause_name.size() + 1 + + condition_name.size()); + cause_condition_count_name.append(cause_name); + cause_condition_count_name.append("-"); + cause_condition_count_name.append(condition_name); + + return cause_condition_count_name; +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/write_stall_stats.h b/librocksdb-sys/rocksdb/db/write_stall_stats.h new file mode 100644 index 0000000..6394abb --- /dev/null +++ b/librocksdb-sys/rocksdb/db/write_stall_stats.h @@ -0,0 +1,47 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "db/internal_stats.h" +#include "rocksdb/types.h" + +namespace ROCKSDB_NAMESPACE { +extern const std::string& InvalidWriteStallHyphenString(); + +extern const std::string& WriteStallCauseToHyphenString(WriteStallCause cause); + +extern const std::string& WriteStallConditionToHyphenString( + WriteStallCondition condition); + +// REQUIRES: +// cause` is CF-scope `WriteStallCause`, see `WriteStallCause` for more +// +// REQUIRES: +// `condition` != `WriteStallCondition::kNormal` +extern InternalStats::InternalCFStatsType InternalCFStat( + WriteStallCause cause, WriteStallCondition condition); + +// REQUIRES: +// cause` is DB-scope `WriteStallCause`, see `WriteStallCause` for more +// +// REQUIRES: +// `condition` != `WriteStallCondition::kNormal` +extern InternalStats::InternalDBStatsType InternalDBStat( + WriteStallCause cause, WriteStallCondition condition); + +extern bool isCFScopeWriteStallCause(WriteStallCause cause); +extern bool isDBScopeWriteStallCause(WriteStallCause cause); + +constexpr uint32_t kNumCFScopeWriteStallCauses = + static_cast(WriteStallCause::kCFScopeWriteStallCauseEnumMax) - + static_cast(WriteStallCause::kMemtableLimit); + +constexpr uint32_t kNumDBScopeWriteStallCauses = + static_cast(WriteStallCause::kDBScopeWriteStallCauseEnumMax) - + static_cast(WriteStallCause::kWriteBufferManagerLimit); +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/write_thread.cc b/librocksdb-sys/rocksdb/db/write_thread.cc new file mode 100644 index 0000000..7987007 --- /dev/null +++ b/librocksdb-sys/rocksdb/db/write_thread.cc @@ -0,0 +1,844 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/write_thread.h" + +#include +#include + +#include "db/column_family.h" +#include "monitoring/perf_context_imp.h" +#include "port/port.h" +#include "test_util/sync_point.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +WriteThread::WriteThread(const ImmutableDBOptions& db_options) + : max_yield_usec_(db_options.enable_write_thread_adaptive_yield + ? db_options.write_thread_max_yield_usec + : 0), + slow_yield_usec_(db_options.write_thread_slow_yield_usec), + allow_concurrent_memtable_write_( + db_options.allow_concurrent_memtable_write), + enable_pipelined_write_(db_options.enable_pipelined_write), + max_write_batch_group_size_bytes( + db_options.max_write_batch_group_size_bytes), + newest_writer_(nullptr), + newest_memtable_writer_(nullptr), + last_sequence_(0), + write_stall_dummy_(), + stall_mu_(), + stall_cv_(&stall_mu_) {} + +uint8_t WriteThread::BlockingAwaitState(Writer* w, uint8_t goal_mask) { + // We're going to block. Lazily create the mutex. We guarantee + // propagation of this construction to the waker via the + // STATE_LOCKED_WAITING state. The waker won't try to touch the mutex + // or the condvar unless they CAS away the STATE_LOCKED_WAITING that + // we install below. + w->CreateMutex(); + + auto state = w->state.load(std::memory_order_acquire); + assert(state != STATE_LOCKED_WAITING); + if ((state & goal_mask) == 0 && + w->state.compare_exchange_strong(state, STATE_LOCKED_WAITING)) { + // we have permission (and an obligation) to use StateMutex + std::unique_lock guard(w->StateMutex()); + w->StateCV().wait(guard, [w] { + return w->state.load(std::memory_order_relaxed) != STATE_LOCKED_WAITING; + }); + state = w->state.load(std::memory_order_relaxed); + } + // else tricky. Goal is met or CAS failed. In the latter case the waker + // must have changed the state, and compare_exchange_strong has updated + // our local variable with the new one. At the moment WriteThread never + // waits for a transition across intermediate states, so we know that + // since a state change has occurred the goal must have been met. + assert((state & goal_mask) != 0); + return state; +} + +uint8_t WriteThread::AwaitState(Writer* w, uint8_t goal_mask, + AdaptationContext* ctx) { + uint8_t state = 0; + + // 1. Busy loop using "pause" for 1 micro sec + // 2. Else SOMETIMES busy loop using "yield" for 100 micro sec (default) + // 3. Else blocking wait + + // On a modern Xeon each loop takes about 7 nanoseconds (most of which + // is the effect of the pause instruction), so 200 iterations is a bit + // more than a microsecond. This is long enough that waits longer than + // this can amortize the cost of accessing the clock and yielding. + for (uint32_t tries = 0; tries < 200; ++tries) { + state = w->state.load(std::memory_order_acquire); + if ((state & goal_mask) != 0) { + return state; + } + port::AsmVolatilePause(); + } + + // This is below the fast path, so that the stat is zero when all writes are + // from the same thread. + PERF_TIMER_GUARD(write_thread_wait_nanos); + + // If we're only going to end up waiting a short period of time, + // it can be a lot more efficient to call std::this_thread::yield() + // in a loop than to block in StateMutex(). For reference, on my 4.0 + // SELinux test server with support for syscall auditing enabled, the + // minimum latency between FUTEX_WAKE to returning from FUTEX_WAIT is + // 2.7 usec, and the average is more like 10 usec. That can be a big + // drag on RockDB's single-writer design. Of course, spinning is a + // bad idea if other threads are waiting to run or if we're going to + // wait for a long time. How do we decide? + // + // We break waiting into 3 categories: short-uncontended, + // short-contended, and long. If we had an oracle, then we would always + // spin for short-uncontended, always block for long, and our choice for + // short-contended might depend on whether we were trying to optimize + // RocksDB throughput or avoid being greedy with system resources. + // + // Bucketing into short or long is easy by measuring elapsed time. + // Differentiating short-uncontended from short-contended is a bit + // trickier, but not too bad. We could look for involuntary context + // switches using getrusage(RUSAGE_THREAD, ..), but it's less work + // (portability code and CPU) to just look for yield calls that take + // longer than we expect. sched_yield() doesn't actually result in any + // context switch overhead if there are no other runnable processes + // on the current core, in which case it usually takes less than + // a microsecond. + // + // There are two primary tunables here: the threshold between "short" + // and "long" waits, and the threshold at which we suspect that a yield + // is slow enough to indicate we should probably block. If these + // thresholds are chosen well then CPU-bound workloads that don't + // have more threads than cores will experience few context switches + // (voluntary or involuntary), and the total number of context switches + // (voluntary and involuntary) will not be dramatically larger (maybe + // 2x) than the number of voluntary context switches that occur when + // --max_yield_wait_micros=0. + // + // There's another constant, which is the number of slow yields we will + // tolerate before reversing our previous decision. Solitary slow + // yields are pretty common (low-priority small jobs ready to run), + // so this should be at least 2. We set this conservatively to 3 so + // that we can also immediately schedule a ctx adaptation, rather than + // waiting for the next update_ctx. + + const size_t kMaxSlowYieldsWhileSpinning = 3; + + // Whether the yield approach has any credit in this context. The credit is + // added by yield being succesfull before timing out, and decreased otherwise. + auto& yield_credit = ctx->value; + // Update the yield_credit based on sample runs or right after a hard failure + bool update_ctx = false; + // Should we reinforce the yield credit + bool would_spin_again = false; + // The samling base for updating the yeild credit. The sampling rate would be + // 1/sampling_base. + const int sampling_base = 256; + + if (max_yield_usec_ > 0) { + update_ctx = Random::GetTLSInstance()->OneIn(sampling_base); + + if (update_ctx || yield_credit.load(std::memory_order_relaxed) >= 0) { + // we're updating the adaptation statistics, or spinning has > + // 50% chance of being shorter than max_yield_usec_ and causing no + // involuntary context switches + auto spin_begin = std::chrono::steady_clock::now(); + + // this variable doesn't include the final yield (if any) that + // causes the goal to be met + size_t slow_yield_count = 0; + + auto iter_begin = spin_begin; + while ((iter_begin - spin_begin) <= + std::chrono::microseconds(max_yield_usec_)) { + std::this_thread::yield(); + + state = w->state.load(std::memory_order_acquire); + if ((state & goal_mask) != 0) { + // success + would_spin_again = true; + break; + } + + auto now = std::chrono::steady_clock::now(); + if (now == iter_begin || + now - iter_begin >= std::chrono::microseconds(slow_yield_usec_)) { + // conservatively count it as a slow yield if our clock isn't + // accurate enough to measure the yield duration + ++slow_yield_count; + if (slow_yield_count >= kMaxSlowYieldsWhileSpinning) { + // Not just one ivcsw, but several. Immediately update yield_credit + // and fall back to blocking + update_ctx = true; + break; + } + } + iter_begin = now; + } + } + } + + if ((state & goal_mask) == 0) { + TEST_SYNC_POINT_CALLBACK("WriteThread::AwaitState:BlockingWaiting", w); + state = BlockingAwaitState(w, goal_mask); + } + + if (update_ctx) { + // Since our update is sample based, it is ok if a thread overwrites the + // updates by other threads. Thus the update does not have to be atomic. + auto v = yield_credit.load(std::memory_order_relaxed); + // fixed point exponential decay with decay constant 1/1024, with +1 + // and -1 scaled to avoid overflow for int32_t + // + // On each update the positive credit is decayed by a facor of 1/1024 (i.e., + // 0.1%). If the sampled yield was successful, the credit is also increased + // by X. Setting X=2^17 ensures that the credit never exceeds + // 2^17*2^10=2^27, which is lower than 2^31 the upperbound of int32_t. Same + // logic applies to negative credits. + v = v - (v / 1024) + (would_spin_again ? 1 : -1) * 131072; + yield_credit.store(v, std::memory_order_relaxed); + } + + assert((state & goal_mask) != 0); + return state; +} + +void WriteThread::SetState(Writer* w, uint8_t new_state) { + assert(w); + auto state = w->state.load(std::memory_order_acquire); + if (state == STATE_LOCKED_WAITING || + !w->state.compare_exchange_strong(state, new_state)) { + assert(state == STATE_LOCKED_WAITING); + + std::lock_guard guard(w->StateMutex()); + assert(w->state.load(std::memory_order_relaxed) != new_state); + w->state.store(new_state, std::memory_order_relaxed); + w->StateCV().notify_one(); + } +} + +bool WriteThread::LinkOne(Writer* w, std::atomic* newest_writer) { + assert(newest_writer != nullptr); + assert(w->state == STATE_INIT); + Writer* writers = newest_writer->load(std::memory_order_relaxed); + while (true) { + assert(writers != w); + // If write stall in effect, and w->no_slowdown is not true, + // block here until stall is cleared. If its true, then return + // immediately + if (writers == &write_stall_dummy_) { + if (w->no_slowdown) { + w->status = Status::Incomplete("Write stall"); + SetState(w, STATE_COMPLETED); + return false; + } + // Since no_slowdown is false, wait here to be notified of the write + // stall clearing + { + MutexLock lock(&stall_mu_); + writers = newest_writer->load(std::memory_order_relaxed); + if (writers == &write_stall_dummy_) { + TEST_SYNC_POINT_CALLBACK("WriteThread::WriteStall::Wait", w); + stall_cv_.Wait(); + // Load newest_writers_ again since it may have changed + writers = newest_writer->load(std::memory_order_relaxed); + continue; + } + } + } + w->link_older = writers; + if (newest_writer->compare_exchange_weak(writers, w)) { + return (writers == nullptr); + } + } +} + +bool WriteThread::LinkGroup(WriteGroup& write_group, + std::atomic* newest_writer) { + assert(newest_writer != nullptr); + Writer* leader = write_group.leader; + Writer* last_writer = write_group.last_writer; + Writer* w = last_writer; + while (true) { + // Unset link_newer pointers to make sure when we call + // CreateMissingNewerLinks later it create all missing links. + w->link_newer = nullptr; + w->write_group = nullptr; + if (w == leader) { + break; + } + w = w->link_older; + } + Writer* newest = newest_writer->load(std::memory_order_relaxed); + while (true) { + leader->link_older = newest; + if (newest_writer->compare_exchange_weak(newest, last_writer)) { + return (newest == nullptr); + } + } +} + +void WriteThread::CreateMissingNewerLinks(Writer* head) { + while (true) { + Writer* next = head->link_older; + if (next == nullptr || next->link_newer != nullptr) { + assert(next == nullptr || next->link_newer == head); + break; + } + next->link_newer = head; + head = next; + } +} + +void WriteThread::CompleteLeader(WriteGroup& write_group) { + assert(write_group.size > 0); + Writer* leader = write_group.leader; + if (write_group.size == 1) { + write_group.leader = nullptr; + write_group.last_writer = nullptr; + } else { + assert(leader->link_newer != nullptr); + leader->link_newer->link_older = nullptr; + write_group.leader = leader->link_newer; + } + write_group.size -= 1; + SetState(leader, STATE_COMPLETED); +} + +void WriteThread::CompleteFollower(Writer* w, WriteGroup& write_group) { + assert(write_group.size > 1); + assert(w != write_group.leader); + if (w == write_group.last_writer) { + w->link_older->link_newer = nullptr; + write_group.last_writer = w->link_older; + } else { + w->link_older->link_newer = w->link_newer; + w->link_newer->link_older = w->link_older; + } + write_group.size -= 1; + SetState(w, STATE_COMPLETED); +} + +void WriteThread::BeginWriteStall() { + ++stall_begun_count_; + LinkOne(&write_stall_dummy_, &newest_writer_); + + // Walk writer list until w->write_group != nullptr. The current write group + // will not have a mix of slowdown/no_slowdown, so its ok to stop at that + // point + Writer* w = write_stall_dummy_.link_older; + Writer* prev = &write_stall_dummy_; + while (w != nullptr && w->write_group == nullptr) { + if (w->no_slowdown) { + prev->link_older = w->link_older; + w->status = Status::Incomplete("Write stall"); + SetState(w, STATE_COMPLETED); + // Only update `link_newer` if it's already set. + // `CreateMissingNewerLinks()` will update the nullptr `link_newer` later, + // which assumes the the first non-nullptr `link_newer` is the last + // nullptr link in the writer list. + // If `link_newer` is set here, `CreateMissingNewerLinks()` may stop + // updating the whole list when it sees the first non nullptr link. + if (prev->link_older && prev->link_older->link_newer) { + prev->link_older->link_newer = prev; + } + w = prev->link_older; + } else { + prev = w; + w = w->link_older; + } + } +} + +void WriteThread::EndWriteStall() { + MutexLock lock(&stall_mu_); + + // Unlink write_stall_dummy_ from the write queue. This will unblock + // pending write threads to enqueue themselves + assert(newest_writer_.load(std::memory_order_relaxed) == &write_stall_dummy_); + // write_stall_dummy_.link_older can be nullptr only if LockWAL() has been + // called. + if (write_stall_dummy_.link_older) { + write_stall_dummy_.link_older->link_newer = write_stall_dummy_.link_newer; + } + newest_writer_.exchange(write_stall_dummy_.link_older); + + ++stall_ended_count_; + + // Wake up writers + stall_cv_.SignalAll(); +} + +uint64_t WriteThread::GetBegunCountOfOutstandingStall() { + if (stall_begun_count_ > stall_ended_count_) { + // Oustanding stall in queue + assert(newest_writer_.load(std::memory_order_relaxed) == + &write_stall_dummy_); + return stall_begun_count_; + } else { + // No stall in queue + assert(newest_writer_.load(std::memory_order_relaxed) != + &write_stall_dummy_); + return 0; + } +} + +void WriteThread::WaitForStallEndedCount(uint64_t stall_count) { + MutexLock lock(&stall_mu_); + + while (stall_ended_count_ < stall_count) { + stall_cv_.Wait(); + } +} + +static WriteThread::AdaptationContext jbg_ctx("JoinBatchGroup"); +void WriteThread::JoinBatchGroup(Writer* w) { + TEST_SYNC_POINT_CALLBACK("WriteThread::JoinBatchGroup:Start", w); + assert(w->batch != nullptr); + + bool linked_as_leader = LinkOne(w, &newest_writer_); + + if (linked_as_leader) { + SetState(w, STATE_GROUP_LEADER); + } + + TEST_SYNC_POINT_CALLBACK("WriteThread::JoinBatchGroup:Wait", w); + TEST_SYNC_POINT_CALLBACK("WriteThread::JoinBatchGroup:Wait2", w); + + if (!linked_as_leader) { + /** + * Wait util: + * 1) An existing leader pick us as the new leader when it finishes + * 2) An existing leader pick us as its follewer and + * 2.1) finishes the memtable writes on our behalf + * 2.2) Or tell us to finish the memtable writes in pralallel + * 3) (pipelined write) An existing leader pick us as its follower and + * finish book-keeping and WAL write for us, enqueue us as pending + * memtable writer, and + * 3.1) we become memtable writer group leader, or + * 3.2) an existing memtable writer group leader tell us to finish memtable + * writes in parallel. + */ + TEST_SYNC_POINT_CALLBACK("WriteThread::JoinBatchGroup:BeganWaiting", w); + AwaitState(w, + STATE_GROUP_LEADER | STATE_MEMTABLE_WRITER_LEADER | + STATE_PARALLEL_MEMTABLE_WRITER | STATE_COMPLETED, + &jbg_ctx); + TEST_SYNC_POINT_CALLBACK("WriteThread::JoinBatchGroup:DoneWaiting", w); + } +} + +size_t WriteThread::EnterAsBatchGroupLeader(Writer* leader, + WriteGroup* write_group) { + assert(leader->link_older == nullptr); + assert(leader->batch != nullptr); + assert(write_group != nullptr); + + size_t size = WriteBatchInternal::ByteSize(leader->batch); + + // Allow the group to grow up to a maximum size, but if the + // original write is small, limit the growth so we do not slow + // down the small write too much. + size_t max_size = max_write_batch_group_size_bytes; + const uint64_t min_batch_size_bytes = max_write_batch_group_size_bytes / 8; + if (size <= min_batch_size_bytes) { + max_size = size + min_batch_size_bytes; + } + + leader->write_group = write_group; + write_group->leader = leader; + write_group->last_writer = leader; + write_group->size = 1; + Writer* newest_writer = newest_writer_.load(std::memory_order_acquire); + + // This is safe regardless of any db mutex status of the caller. Previous + // calls to ExitAsGroupLeader either didn't call CreateMissingNewerLinks + // (they emptied the list and then we added ourself as leader) or had to + // explicitly wake us up (the list was non-empty when we added ourself, + // so we have already received our MarkJoined). + CreateMissingNewerLinks(newest_writer); + + // Tricky. Iteration start (leader) is exclusive and finish + // (newest_writer) is inclusive. Iteration goes from old to new. + Writer* w = leader; + while (w != newest_writer) { + assert(w->link_newer); + w = w->link_newer; + + if (w->sync && !leader->sync) { + // Do not include a sync write into a batch handled by a non-sync write. + break; + } + + if (w->no_slowdown != leader->no_slowdown) { + // Do not mix writes that are ok with delays with the ones that + // request fail on delays. + break; + } + + if (w->disable_wal != leader->disable_wal) { + // Do not mix writes that enable WAL with the ones whose + // WAL disabled. + break; + } + + if (w->protection_bytes_per_key != leader->protection_bytes_per_key) { + // Do not mix writes with different levels of integrity protection. + break; + } + + if (w->rate_limiter_priority != leader->rate_limiter_priority) { + // Do not mix writes with different rate limiter priorities. + break; + } + + if (w->batch == nullptr) { + // Do not include those writes with nullptr batch. Those are not writes, + // those are something else. They want to be alone + break; + } + + if (w->callback != nullptr && !w->callback->AllowWriteBatching()) { + // don't batch writes that don't want to be batched + break; + } + + auto batch_size = WriteBatchInternal::ByteSize(w->batch); + if (size + batch_size > max_size) { + // Do not make batch too big + break; + } + + w->write_group = write_group; + size += batch_size; + write_group->last_writer = w; + write_group->size++; + } + TEST_SYNC_POINT_CALLBACK("WriteThread::EnterAsBatchGroupLeader:End", w); + return size; +} + +void WriteThread::EnterAsMemTableWriter(Writer* leader, + WriteGroup* write_group) { + assert(leader != nullptr); + assert(leader->link_older == nullptr); + assert(leader->batch != nullptr); + assert(write_group != nullptr); + + size_t size = WriteBatchInternal::ByteSize(leader->batch); + + // Allow the group to grow up to a maximum size, but if the + // original write is small, limit the growth so we do not slow + // down the small write too much. + size_t max_size = max_write_batch_group_size_bytes; + const uint64_t min_batch_size_bytes = max_write_batch_group_size_bytes / 8; + if (size <= min_batch_size_bytes) { + max_size = size + min_batch_size_bytes; + } + + leader->write_group = write_group; + write_group->leader = leader; + write_group->size = 1; + Writer* last_writer = leader; + + if (!allow_concurrent_memtable_write_ || !leader->batch->HasMerge()) { + Writer* newest_writer = newest_memtable_writer_.load(); + CreateMissingNewerLinks(newest_writer); + + Writer* w = leader; + while (w != newest_writer) { + assert(w->link_newer); + w = w->link_newer; + + if (w->batch == nullptr) { + break; + } + + if (w->batch->HasMerge()) { + break; + } + + if (!allow_concurrent_memtable_write_) { + auto batch_size = WriteBatchInternal::ByteSize(w->batch); + if (size + batch_size > max_size) { + // Do not make batch too big + break; + } + size += batch_size; + } + + w->write_group = write_group; + last_writer = w; + write_group->size++; + } + } + + write_group->last_writer = last_writer; + write_group->last_sequence = + last_writer->sequence + WriteBatchInternal::Count(last_writer->batch) - 1; +} + +void WriteThread::ExitAsMemTableWriter(Writer* /*self*/, + WriteGroup& write_group) { + Writer* leader = write_group.leader; + Writer* last_writer = write_group.last_writer; + + Writer* newest_writer = last_writer; + if (!newest_memtable_writer_.compare_exchange_strong(newest_writer, + nullptr)) { + CreateMissingNewerLinks(newest_writer); + Writer* next_leader = last_writer->link_newer; + assert(next_leader != nullptr); + next_leader->link_older = nullptr; + SetState(next_leader, STATE_MEMTABLE_WRITER_LEADER); + } + Writer* w = leader; + while (true) { + if (!write_group.status.ok()) { + w->status = write_group.status; + } + Writer* next = w->link_newer; + if (w != leader) { + SetState(w, STATE_COMPLETED); + } + if (w == last_writer) { + break; + } + assert(next); + w = next; + } + // Note that leader has to exit last, since it owns the write group. + SetState(leader, STATE_COMPLETED); +} + +void WriteThread::LaunchParallelMemTableWriters(WriteGroup* write_group) { + assert(write_group != nullptr); + write_group->running.store(write_group->size); + for (auto w : *write_group) { + SetState(w, STATE_PARALLEL_MEMTABLE_WRITER); + } +} + +static WriteThread::AdaptationContext cpmtw_ctx( + "CompleteParallelMemTableWriter"); +// This method is called by both the leader and parallel followers +bool WriteThread::CompleteParallelMemTableWriter(Writer* w) { + auto* write_group = w->write_group; + if (!w->status.ok()) { + std::lock_guard guard(write_group->leader->StateMutex()); + write_group->status = w->status; + } + + if (write_group->running-- > 1) { + // we're not the last one + AwaitState(w, STATE_COMPLETED, &cpmtw_ctx); + return false; + } + // else we're the last parallel worker and should perform exit duties. + w->status = write_group->status; + // Callers of this function must ensure w->status is checked. + write_group->status.PermitUncheckedError(); + return true; +} + +void WriteThread::ExitAsBatchGroupFollower(Writer* w) { + auto* write_group = w->write_group; + + assert(w->state == STATE_PARALLEL_MEMTABLE_WRITER); + assert(write_group->status.ok()); + ExitAsBatchGroupLeader(*write_group, write_group->status); + assert(w->status.ok()); + assert(w->state == STATE_COMPLETED); + SetState(write_group->leader, STATE_COMPLETED); +} + +static WriteThread::AdaptationContext eabgl_ctx("ExitAsBatchGroupLeader"); +void WriteThread::ExitAsBatchGroupLeader(WriteGroup& write_group, + Status& status) { + TEST_SYNC_POINT_CALLBACK("WriteThread::ExitAsBatchGroupLeader:Start", + &write_group); + + Writer* leader = write_group.leader; + Writer* last_writer = write_group.last_writer; + assert(leader->link_older == nullptr); + + // If status is non-ok already, then write_group.status won't have the chance + // of being propagated to caller. + if (!status.ok()) { + write_group.status.PermitUncheckedError(); + } + + // Propagate memtable write error to the whole group. + if (status.ok() && !write_group.status.ok()) { + status = write_group.status; + } + + if (enable_pipelined_write_) { + // We insert a dummy Writer right before our current write_group. This + // allows us to unlink our write_group without the risk that a subsequent + // writer becomes a new leader and might overtake us and add itself to the + // memtable-writer-list before we can do so. This ensures that writers are + // added to the memtable-writer-list in the exact same order in which they + // were in the newest_writer list. + // This must happen before completing the writers from our group to prevent + // a race where the owning thread of one of these writers can start a new + // write operation. + Writer dummy; + Writer* head = newest_writer_.load(std::memory_order_acquire); + if (head != last_writer || + !newest_writer_.compare_exchange_strong(head, &dummy)) { + // Either last_writer wasn't the head during the load(), or it was the + // head during the load() but somebody else pushed onto the list before + // we did the compare_exchange_strong (causing it to fail). In the latter + // case compare_exchange_strong has the effect of re-reading its first + // param (head). No need to retry a failing CAS, because only a departing + // leader (which we are at the moment) can remove nodes from the list. + assert(head != last_writer); + + // After walking link_older starting from head (if not already done) we + // will be able to traverse w->link_newer below. + CreateMissingNewerLinks(head); + assert(last_writer->link_newer != nullptr); + last_writer->link_newer->link_older = &dummy; + dummy.link_newer = last_writer->link_newer; + } + + // Complete writers that don't write to memtable + for (Writer* w = last_writer; w != leader;) { + Writer* next = w->link_older; + w->status = status; + if (!w->ShouldWriteToMemtable()) { + CompleteFollower(w, write_group); + } + w = next; + } + if (!leader->ShouldWriteToMemtable()) { + CompleteLeader(write_group); + } + + TEST_SYNC_POINT_CALLBACK( + "WriteThread::ExitAsBatchGroupLeader:AfterCompleteWriters", + &write_group); + + // Link the remaining of the group to memtable writer list. + // We have to link our group to memtable writer queue before wake up the + // next leader or set newest_writer_ to null, otherwise the next leader + // can run ahead of us and link to memtable writer queue before we do. + if (write_group.size > 0) { + if (LinkGroup(write_group, &newest_memtable_writer_)) { + // The leader can now be different from current writer. + SetState(write_group.leader, STATE_MEMTABLE_WRITER_LEADER); + } + } + + // Unlink the dummy writer from the list and identify the new leader + head = newest_writer_.load(std::memory_order_acquire); + if (head != &dummy || + !newest_writer_.compare_exchange_strong(head, nullptr)) { + CreateMissingNewerLinks(head); + Writer* new_leader = dummy.link_newer; + assert(new_leader != nullptr); + new_leader->link_older = nullptr; + SetState(new_leader, STATE_GROUP_LEADER); + } + + AwaitState(leader, + STATE_MEMTABLE_WRITER_LEADER | STATE_PARALLEL_MEMTABLE_WRITER | + STATE_COMPLETED, + &eabgl_ctx); + } else { + Writer* head = newest_writer_.load(std::memory_order_acquire); + if (head != last_writer || + !newest_writer_.compare_exchange_strong(head, nullptr)) { + // Either last_writer wasn't the head during the load(), or it was the + // head during the load() but somebody else pushed onto the list before + // we did the compare_exchange_strong (causing it to fail). In the + // latter case compare_exchange_strong has the effect of re-reading + // its first param (head). No need to retry a failing CAS, because + // only a departing leader (which we are at the moment) can remove + // nodes from the list. + assert(head != last_writer); + + // After walking link_older starting from head (if not already done) + // we will be able to traverse w->link_newer below. This function + // can only be called from an active leader, only a leader can + // clear newest_writer_, we didn't, and only a clear newest_writer_ + // could cause the next leader to start their work without a call + // to MarkJoined, so we can definitely conclude that no other leader + // work is going on here (with or without db mutex). + CreateMissingNewerLinks(head); + assert(last_writer->link_newer != nullptr); + assert(last_writer->link_newer->link_older == last_writer); + last_writer->link_newer->link_older = nullptr; + + // Next leader didn't self-identify, because newest_writer_ wasn't + // nullptr when they enqueued (we were definitely enqueued before them + // and are still in the list). That means leader handoff occurs when + // we call MarkJoined + SetState(last_writer->link_newer, STATE_GROUP_LEADER); + } + // else nobody else was waiting, although there might already be a new + // leader now + + while (last_writer != leader) { + assert(last_writer); + last_writer->status = status; + // we need to read link_older before calling SetState, because as soon + // as it is marked committed the other thread's Await may return and + // deallocate the Writer. + auto next = last_writer->link_older; + SetState(last_writer, STATE_COMPLETED); + + last_writer = next; + } + } +} + +static WriteThread::AdaptationContext eu_ctx("EnterUnbatched"); +void WriteThread::EnterUnbatched(Writer* w, InstrumentedMutex* mu) { + assert(w != nullptr && w->batch == nullptr); + mu->Unlock(); + bool linked_as_leader = LinkOne(w, &newest_writer_); + if (!linked_as_leader) { + TEST_SYNC_POINT("WriteThread::EnterUnbatched:Wait"); + // Last leader will not pick us as a follower since our batch is nullptr + AwaitState(w, STATE_GROUP_LEADER, &eu_ctx); + } + if (enable_pipelined_write_) { + WaitForMemTableWriters(); + } + mu->Lock(); +} + +void WriteThread::ExitUnbatched(Writer* w) { + assert(w != nullptr); + Writer* newest_writer = w; + if (!newest_writer_.compare_exchange_strong(newest_writer, nullptr)) { + CreateMissingNewerLinks(newest_writer); + Writer* next_leader = w->link_newer; + assert(next_leader != nullptr); + next_leader->link_older = nullptr; + SetState(next_leader, STATE_GROUP_LEADER); + } +} + +static WriteThread::AdaptationContext wfmw_ctx("WaitForMemTableWriters"); +void WriteThread::WaitForMemTableWriters() { + assert(enable_pipelined_write_); + if (newest_memtable_writer_.load() == nullptr) { + return; + } + Writer w; + if (!LinkOne(&w, &newest_memtable_writer_)) { + AwaitState(&w, STATE_MEMTABLE_WRITER_LEADER, &wfmw_ctx); + } + newest_memtable_writer_.store(nullptr); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db/write_thread.h b/librocksdb-sys/rocksdb/db/write_thread.h new file mode 100644 index 0000000..6e5805e --- /dev/null +++ b/librocksdb-sys/rocksdb/db/write_thread.h @@ -0,0 +1,464 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "db/dbformat.h" +#include "db/post_memtable_callback.h" +#include "db/pre_release_callback.h" +#include "db/write_callback.h" +#include "monitoring/instrumented_mutex.h" +#include "rocksdb/options.h" +#include "rocksdb/status.h" +#include "rocksdb/types.h" +#include "rocksdb/write_batch.h" +#include "util/autovector.h" + +namespace ROCKSDB_NAMESPACE { + +class WriteThread { + public: + enum State : uint8_t { + // The initial state of a writer. This is a Writer that is + // waiting in JoinBatchGroup. This state can be left when another + // thread informs the waiter that it has become a group leader + // (-> STATE_GROUP_LEADER), when a leader that has chosen to be + // non-parallel informs a follower that its writes have been committed + // (-> STATE_COMPLETED), or when a leader that has chosen to perform + // updates in parallel and needs this Writer to apply its batch (-> + // STATE_PARALLEL_MEMTABLE_WRITER). + STATE_INIT = 1, + + // The state used to inform a waiting Writer that it has become the + // leader, and it should now build a write batch group. Tricky: + // this state is not used if newest_writer_ is empty when a writer + // enqueues itself, because there is no need to wait (or even to + // create the mutex and condvar used to wait) in that case. This is + // a terminal state unless the leader chooses to make this a parallel + // batch, in which case the last parallel worker to finish will move + // the leader to STATE_COMPLETED. + STATE_GROUP_LEADER = 2, + + // The state used to inform a waiting writer that it has become the + // leader of memtable writer group. The leader will either write + // memtable for the whole group, or launch a parallel group write + // to memtable by calling LaunchParallelMemTableWrite. + STATE_MEMTABLE_WRITER_LEADER = 4, + + // The state used to inform a waiting writer that it has become a + // parallel memtable writer. It can be the group leader who launch the + // parallel writer group, or one of the followers. The writer should then + // apply its batch to the memtable concurrently and call + // CompleteParallelMemTableWriter. + STATE_PARALLEL_MEMTABLE_WRITER = 8, + + // A follower whose writes have been applied, or a parallel leader + // whose followers have all finished their work. This is a terminal + // state. + STATE_COMPLETED = 16, + + // A state indicating that the thread may be waiting using StateMutex() + // and StateCondVar() + STATE_LOCKED_WAITING = 32, + }; + + struct Writer; + + struct WriteGroup { + Writer* leader = nullptr; + Writer* last_writer = nullptr; + SequenceNumber last_sequence; + // before running goes to zero, status needs leader->StateMutex() + Status status; + std::atomic running; + size_t size = 0; + + struct Iterator { + Writer* writer; + Writer* last_writer; + + explicit Iterator(Writer* w, Writer* last) + : writer(w), last_writer(last) {} + + Writer* operator*() const { return writer; } + + Iterator& operator++() { + assert(writer != nullptr); + if (writer == last_writer) { + writer = nullptr; + } else { + writer = writer->link_newer; + } + return *this; + } + + bool operator!=(const Iterator& other) const { + return writer != other.writer; + } + }; + + Iterator begin() const { return Iterator(leader, last_writer); } + Iterator end() const { return Iterator(nullptr, nullptr); } + }; + + // Information kept for every waiting writer. + struct Writer { + WriteBatch* batch; + bool sync; + bool no_slowdown; + bool disable_wal; + Env::IOPriority rate_limiter_priority; + bool disable_memtable; + size_t batch_cnt; // if non-zero, number of sub-batches in the write batch + size_t protection_bytes_per_key; + PreReleaseCallback* pre_release_callback; + PostMemTableCallback* post_memtable_callback; + uint64_t log_used; // log number that this batch was inserted into + uint64_t log_ref; // log number that memtable insert should reference + WriteCallback* callback; + bool made_waitable; // records lazy construction of mutex and cv + std::atomic state; // write under StateMutex() or pre-link + WriteGroup* write_group; + SequenceNumber sequence; // the sequence number to use for the first key + Status status; + Status callback_status; // status returned by callback->Callback() + + std::aligned_storage::type state_mutex_bytes; + std::aligned_storage::type state_cv_bytes; + Writer* link_older; // read/write only before linking, or as leader + Writer* link_newer; // lazy, read/write only before linking, or as leader + + Writer() + : batch(nullptr), + sync(false), + no_slowdown(false), + disable_wal(false), + rate_limiter_priority(Env::IOPriority::IO_TOTAL), + disable_memtable(false), + batch_cnt(0), + protection_bytes_per_key(0), + pre_release_callback(nullptr), + post_memtable_callback(nullptr), + log_used(0), + log_ref(0), + callback(nullptr), + made_waitable(false), + state(STATE_INIT), + write_group(nullptr), + sequence(kMaxSequenceNumber), + link_older(nullptr), + link_newer(nullptr) {} + + Writer(const WriteOptions& write_options, WriteBatch* _batch, + WriteCallback* _callback, uint64_t _log_ref, bool _disable_memtable, + size_t _batch_cnt = 0, + PreReleaseCallback* _pre_release_callback = nullptr, + PostMemTableCallback* _post_memtable_callback = nullptr) + : batch(_batch), + sync(write_options.sync), + no_slowdown(write_options.no_slowdown), + disable_wal(write_options.disableWAL), + rate_limiter_priority(write_options.rate_limiter_priority), + disable_memtable(_disable_memtable), + batch_cnt(_batch_cnt), + protection_bytes_per_key(_batch->GetProtectionBytesPerKey()), + pre_release_callback(_pre_release_callback), + post_memtable_callback(_post_memtable_callback), + log_used(0), + log_ref(_log_ref), + callback(_callback), + made_waitable(false), + state(STATE_INIT), + write_group(nullptr), + sequence(kMaxSequenceNumber), + link_older(nullptr), + link_newer(nullptr) {} + + ~Writer() { + if (made_waitable) { + StateMutex().~mutex(); + StateCV().~condition_variable(); + } + status.PermitUncheckedError(); + callback_status.PermitUncheckedError(); + } + + bool CheckCallback(DB* db) { + if (callback != nullptr) { + callback_status = callback->Callback(db); + } + return callback_status.ok(); + } + + void CreateMutex() { + if (!made_waitable) { + // Note that made_waitable is tracked separately from state + // transitions, because we can't atomically create the mutex and + // link into the list. + made_waitable = true; + new (&state_mutex_bytes) std::mutex; + new (&state_cv_bytes) std::condition_variable; + } + } + + // returns the aggregate status of this Writer + Status FinalStatus() { + if (!status.ok()) { + // a non-ok memtable write status takes presidence + assert(callback == nullptr || callback_status.ok()); + return status; + } else if (!callback_status.ok()) { + // if the callback failed then that is the status we want + // because a memtable insert should not have been attempted + assert(callback != nullptr); + assert(status.ok()); + return callback_status; + } else { + // if there is no callback then we only care about + // the memtable insert status + assert(callback == nullptr || callback_status.ok()); + return status; + } + } + + bool CallbackFailed() { + return (callback != nullptr) && !callback_status.ok(); + } + + bool ShouldWriteToMemtable() { + return status.ok() && !CallbackFailed() && !disable_memtable; + } + + bool ShouldWriteToWAL() { + return status.ok() && !CallbackFailed() && !disable_wal; + } + + // No other mutexes may be acquired while holding StateMutex(), it is + // always last in the order + std::mutex& StateMutex() { + assert(made_waitable); + return *static_cast(static_cast(&state_mutex_bytes)); + } + + std::condition_variable& StateCV() { + assert(made_waitable); + return *static_cast( + static_cast(&state_cv_bytes)); + } + }; + + struct AdaptationContext { + const char* name; + std::atomic value; + + explicit AdaptationContext(const char* name0) : name(name0), value(0) {} + }; + + explicit WriteThread(const ImmutableDBOptions& db_options); + + virtual ~WriteThread() = default; + + // IMPORTANT: None of the methods in this class rely on the db mutex + // for correctness. All of the methods except JoinBatchGroup and + // EnterUnbatched may be called either with or without the db mutex held. + // Correctness is maintained by ensuring that only a single thread is + // a leader at a time. + + // Registers w as ready to become part of a batch group, waits until the + // caller should perform some work, and returns the current state of the + // writer. If w has become the leader of a write batch group, returns + // STATE_GROUP_LEADER. If w has been made part of a sequential batch + // group and the leader has performed the write, returns STATE_DONE. + // If w has been made part of a parallel batch group and is responsible + // for updating the memtable, returns STATE_PARALLEL_MEMTABLE_WRITER. + // + // The db mutex SHOULD NOT be held when calling this function, because + // it will block. + // + // Writer* w: Writer to be executed as part of a batch group + void JoinBatchGroup(Writer* w); + + // Constructs a write batch group led by leader, which should be a + // Writer passed to JoinBatchGroup on the current thread. + // + // Writer* leader: Writer that is STATE_GROUP_LEADER + // WriteGroup* write_group: Out-param of group members + // returns: Total batch group byte size + size_t EnterAsBatchGroupLeader(Writer* leader, WriteGroup* write_group); + + // Unlinks the Writer-s in a batch group, wakes up the non-leaders, + // and wakes up the next leader (if any). + // + // WriteGroup* write_group: the write group + // Status status: Status of write operation + void ExitAsBatchGroupLeader(WriteGroup& write_group, Status& status); + + // Exit batch group on behalf of batch group leader. + void ExitAsBatchGroupFollower(Writer* w); + + // Constructs a write batch group led by leader from newest_memtable_writers_ + // list. The leader should either write memtable for the whole group and + // call ExitAsMemTableWriter, or launch parallel memtable write through + // LaunchParallelMemTableWriters. + void EnterAsMemTableWriter(Writer* leader, WriteGroup* write_grup); + + // Memtable writer group leader, or the last finished writer in a parallel + // write group, exit from the newest_memtable_writers_ list, and wake up + // the next leader if needed. + void ExitAsMemTableWriter(Writer* self, WriteGroup& write_group); + + // Causes JoinBatchGroup to return STATE_PARALLEL_MEMTABLE_WRITER for all of + // the non-leader members of this write batch group. Sets Writer::sequence + // before waking them up. + // + // WriteGroup* write_group: Extra state used to coordinate the parallel add + void LaunchParallelMemTableWriters(WriteGroup* write_group); + + // Reports the completion of w's batch to the parallel group leader, and + // waits for the rest of the parallel batch to complete. Returns true + // if this thread is the last to complete, and hence should advance + // the sequence number and then call EarlyExitParallelGroup, false if + // someone else has already taken responsibility for that. + bool CompleteParallelMemTableWriter(Writer* w); + + // Waits for all preceding writers (unlocking mu while waiting), then + // registers w as the currently proceeding writer. + // + // Writer* w: A Writer not eligible for batching + // InstrumentedMutex* mu: The db mutex, to unlock while waiting + // REQUIRES: db mutex held + void EnterUnbatched(Writer* w, InstrumentedMutex* mu); + + // Completes a Writer begun with EnterUnbatched, unblocking subsequent + // writers. + void ExitUnbatched(Writer* w); + + // Wait for all parallel memtable writers to finish, in case pipelined + // write is enabled. + void WaitForMemTableWriters(); + + SequenceNumber UpdateLastSequence(SequenceNumber sequence) { + if (sequence > last_sequence_) { + last_sequence_ = sequence; + } + return last_sequence_; + } + + // Insert a dummy writer at the tail of the write queue to indicate a write + // stall, and fail any writers in the queue with no_slowdown set to true + // REQUIRES: db mutex held, no other stall on this queue outstanding + void BeginWriteStall(); + + // Remove the dummy writer and wake up waiting writers + // REQUIRES: db mutex held + void EndWriteStall(); + + // Number of BeginWriteStall(), or 0 if there is no active stall in the + // write queue. + // REQUIRES: db mutex held + uint64_t GetBegunCountOfOutstandingStall(); + + // Wait for number of completed EndWriteStall() to reach >= `stall_count`, + // which will generally have come from GetBegunCountOfOutstandingStall(). + // (Does not require db mutex held) + void WaitForStallEndedCount(uint64_t stall_count); + + private: + // See AwaitState. + const uint64_t max_yield_usec_; + const uint64_t slow_yield_usec_; + + // Allow multiple writers write to memtable concurrently. + const bool allow_concurrent_memtable_write_; + + // Enable pipelined write to WAL and memtable. + const bool enable_pipelined_write_; + + // The maximum limit of number of bytes that are written in a single batch + // of WAL or memtable write. It is followed when the leader write size + // is larger than 1/8 of this limit. + const uint64_t max_write_batch_group_size_bytes; + + // Points to the newest pending writer. Only leader can remove + // elements, adding can be done lock-free by anybody. + std::atomic newest_writer_; + + // Points to the newest pending memtable writer. Used only when pipelined + // write is enabled. + std::atomic newest_memtable_writer_; + + // The last sequence that have been consumed by a writer. The sequence + // is not necessary visible to reads because the writer can be ongoing. + SequenceNumber last_sequence_; + + // A dummy writer to indicate a write stall condition. This will be inserted + // at the tail of the writer queue by the leader, so newer writers can just + // check for this and bail + Writer write_stall_dummy_; + + // Mutex and condvar for writers to block on a write stall. During a write + // stall, writers with no_slowdown set to false will wait on this rather + // on the writer queue + port::Mutex stall_mu_; + port::CondVar stall_cv_; + + // Count the number of stalls begun, so that we can check whether + // a particular stall has cleared (even if caught in another stall). + // Controlled by DB mutex. + // Because of the contract on BeginWriteStall() / EndWriteStall(), + // stall_ended_count_ <= stall_begun_count_ <= stall_ended_count_ + 1. + uint64_t stall_begun_count_ = 0; + // Count the number of stalls ended, so that we can check whether + // a particular stall has cleared (even if caught in another stall). + // Writes controlled by DB mutex + stall_mu_, signalled by stall_cv_. + // Read with stall_mu or DB mutex. + uint64_t stall_ended_count_ = 0; + + // Waits for w->state & goal_mask using w->StateMutex(). Returns + // the state that satisfies goal_mask. + uint8_t BlockingAwaitState(Writer* w, uint8_t goal_mask); + + // Blocks until w->state & goal_mask, returning the state value + // that satisfied the predicate. Uses ctx to adaptively use + // std::this_thread::yield() to avoid mutex overheads. ctx should be + // a context-dependent static. + uint8_t AwaitState(Writer* w, uint8_t goal_mask, AdaptationContext* ctx); + + // Set writer state and wake the writer up if it is waiting. + void SetState(Writer* w, uint8_t new_state); + + // Links w into the newest_writer list. Return true if w was linked directly + // into the leader position. Safe to call from multiple threads without + // external locking. + bool LinkOne(Writer* w, std::atomic* newest_writer); + + // Link write group into the newest_writer list as a whole, while keeping the + // order of the writers unchanged. Return true if the group was linked + // directly into the leader position. + bool LinkGroup(WriteGroup& write_group, std::atomic* newest_writer); + + // Computes any missing link_newer links. Should not be called + // concurrently with itself. + void CreateMissingNewerLinks(Writer* head); + + // Set the leader in write_group to completed state and remove it from the + // write group. + void CompleteLeader(WriteGroup& write_group); + + // Set a follower in write_group to completed state and remove it from the + // write group. + void CompleteFollower(Writer* w, WriteGroup& write_group); +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db_stress_tool/CMakeLists.txt b/librocksdb-sys/rocksdb/db_stress_tool/CMakeLists.txt new file mode 100644 index 0000000..51d6ea0 --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/CMakeLists.txt @@ -0,0 +1,18 @@ +add_executable(db_stress${ARTIFACT_SUFFIX} + batched_ops_stress.cc + cf_consistency_stress.cc + db_stress.cc + db_stress_common.cc + db_stress_driver.cc + db_stress_gflags.cc + db_stress_listener.cc + db_stress_shared_state.cc + db_stress_stat.cc + db_stress_test_base.cc + db_stress_tool.cc + expected_state.cc + expected_value.cc + multi_ops_txns_stress.cc + no_batched_ops_stress.cc) +target_link_libraries(db_stress${ARTIFACT_SUFFIX} ${ROCKSDB_LIB} ${THIRDPARTY_LIBS}) +list(APPEND tool_deps db_stress) diff --git a/librocksdb-sys/rocksdb/db_stress_tool/batched_ops_stress.cc b/librocksdb-sys/rocksdb/db_stress_tool/batched_ops_stress.cc new file mode 100644 index 0000000..0872f28 --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/batched_ops_stress.cc @@ -0,0 +1,593 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifdef GFLAGS +#include "db_stress_tool/db_stress_common.h" + +namespace ROCKSDB_NAMESPACE { +class BatchedOpsStressTest : public StressTest { + public: + BatchedOpsStressTest() {} + virtual ~BatchedOpsStressTest() {} + + bool IsStateTracked() const override { return false; } + + // Given a key K and value V, this puts ("0"+K, V+"0"), ("1"+K, V+"1"), ..., + // ("9"+K, V+"9") in DB atomically i.e in a single batch. + // Also refer BatchedOpsStressTest::TestGet + Status TestPut(ThreadState* thread, WriteOptions& write_opts, + const ReadOptions& /* read_opts */, + const std::vector& rand_column_families, + const std::vector& rand_keys, + char (&value)[100]) override { + assert(!rand_column_families.empty()); + assert(!rand_keys.empty()); + + const std::string key_body = Key(rand_keys[0]); + + const uint32_t value_base = thread->rand.Next(); + const size_t sz = GenerateValue(value_base, value, sizeof(value)); + const std::string value_body = Slice(value, sz).ToString(); + + WriteBatch batch(0 /* reserved_bytes */, 0 /* max_bytes */, + FLAGS_batch_protection_bytes_per_key, + FLAGS_user_timestamp_size); + + ColumnFamilyHandle* const cfh = column_families_[rand_column_families[0]]; + assert(cfh); + + for (int i = 9; i >= 0; --i) { + const std::string num = std::to_string(i); + + // Note: the digit in num is prepended to the key; however, it is appended + // to the value because we want the "value base" to be encoded uniformly + // at the beginning of the value for all types of stress tests (e.g. + // batched, non-batched, CF consistency). + const std::string k = num + key_body; + const std::string v = value_body + num; + + if (FLAGS_use_merge) { + batch.Merge(cfh, k, v); + } else if (FLAGS_use_put_entity_one_in > 0 && + (value_base % FLAGS_use_put_entity_one_in) == 0) { + batch.PutEntity(cfh, k, GenerateWideColumns(value_base, v)); + } else { + batch.Put(cfh, k, v); + } + } + + const Status s = db_->Write(write_opts, &batch); + + if (!s.ok()) { + fprintf(stderr, "multiput error: %s\n", s.ToString().c_str()); + thread->stats.AddErrors(1); + } else { + // we did 10 writes each of size sz + 1 + thread->stats.AddBytesForWrites(10, (sz + 1) * 10); + } + + return s; + } + + // Given a key K, this deletes ("0"+K), ("1"+K), ..., ("9"+K) + // in DB atomically i.e in a single batch. Also refer MultiGet. + Status TestDelete(ThreadState* thread, WriteOptions& writeoptions, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + std::string keys[10] = {"9", "7", "5", "3", "1", "8", "6", "4", "2", "0"}; + + WriteBatch batch(0 /* reserved_bytes */, 0 /* max_bytes */, + FLAGS_batch_protection_bytes_per_key, + FLAGS_user_timestamp_size); + Status s; + auto cfh = column_families_[rand_column_families[0]]; + std::string key_str = Key(rand_keys[0]); + for (int i = 0; i < 10; i++) { + keys[i] += key_str; + batch.Delete(cfh, keys[i]); + } + + s = db_->Write(writeoptions, &batch); + if (!s.ok()) { + fprintf(stderr, "multidelete error: %s\n", s.ToString().c_str()); + thread->stats.AddErrors(1); + } else { + thread->stats.AddDeletes(10); + } + + return s; + } + + Status TestDeleteRange(ThreadState* /* thread */, + WriteOptions& /* write_opts */, + const std::vector& /* rand_column_families */, + const std::vector& /* rand_keys */) override { + assert(false); + return Status::NotSupported( + "BatchedOpsStressTest does not support " + "TestDeleteRange"); + } + + void TestIngestExternalFile( + ThreadState* /* thread */, + const std::vector& /* rand_column_families */, + const std::vector& /* rand_keys */) override { + assert(false); + fprintf(stderr, + "BatchedOpsStressTest does not support " + "TestIngestExternalFile\n"); + std::terminate(); + } + + // Given a key K, this gets values for "0"+K, "1"+K, ..., "9"+K + // in the same snapshot, and verifies that all the values are of the form + // V+"0", V+"1", ..., V+"9". + // ASSUMES that BatchedOpsStressTest::TestPut was used to put (K, V) into + // the DB. + Status TestGet(ThreadState* thread, const ReadOptions& readoptions, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + std::string keys[10] = {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9"}; + Slice key_slices[10]; + std::string values[10]; + ReadOptions readoptionscopy = readoptions; + readoptionscopy.snapshot = db_->GetSnapshot(); + std::string key_str = Key(rand_keys[0]); + Slice key = key_str; + auto cfh = column_families_[rand_column_families[0]]; + std::string from_db; + Status s; + for (int i = 0; i < 10; i++) { + keys[i] += key.ToString(); + key_slices[i] = keys[i]; + s = db_->Get(readoptionscopy, cfh, key_slices[i], &from_db); + if (!s.ok() && !s.IsNotFound()) { + fprintf(stderr, "get error: %s\n", s.ToString().c_str()); + values[i] = ""; + thread->stats.AddErrors(1); + // we continue after error rather than exiting so that we can + // find more errors if any + } else if (s.IsNotFound()) { + values[i] = ""; + thread->stats.AddGets(1, 0); + } else { + values[i] = from_db; + + assert(!keys[i].empty()); + assert(!values[i].empty()); + + const char expected = keys[i].front(); + const char actual = values[i].back(); + + if (expected != actual) { + fprintf(stderr, "get error expected = %c actual = %c\n", expected, + actual); + } + + values[i].pop_back(); // get rid of the differing character + + thread->stats.AddGets(1, 1); + } + } + db_->ReleaseSnapshot(readoptionscopy.snapshot); + + // Now that we retrieved all values, check that they all match + for (int i = 1; i < 10; i++) { + if (values[i] != values[0]) { + fprintf(stderr, "get error: inconsistent values for key %s: %s, %s\n", + key.ToString(true).c_str(), StringToHex(values[0]).c_str(), + StringToHex(values[i]).c_str()); + // we continue after error rather than exiting so that we can + // find more errors if any + } + } + + return s; + } + + std::vector TestMultiGet( + ThreadState* thread, const ReadOptions& readoptions, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + size_t num_keys = rand_keys.size(); + std::vector ret_status(num_keys); + std::array keys = { + {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9"}}; + size_t num_prefixes = keys.size(); + for (size_t rand_key = 0; rand_key < num_keys; ++rand_key) { + std::vector key_slices; + std::vector values(num_prefixes); + std::vector statuses(num_prefixes); + ReadOptions readoptionscopy = readoptions; + readoptionscopy.snapshot = db_->GetSnapshot(); + readoptionscopy.rate_limiter_priority = + FLAGS_rate_limit_user_ops ? Env::IO_USER : Env::IO_TOTAL; + std::vector key_str; + key_str.reserve(num_prefixes); + key_slices.reserve(num_prefixes); + std::string from_db; + ColumnFamilyHandle* cfh = column_families_[rand_column_families[0]]; + + for (size_t key = 0; key < num_prefixes; ++key) { + key_str.emplace_back(keys[key] + Key(rand_keys[rand_key])); + key_slices.emplace_back(key_str.back()); + } + db_->MultiGet(readoptionscopy, cfh, num_prefixes, key_slices.data(), + values.data(), statuses.data()); + for (size_t i = 0; i < num_prefixes; i++) { + Status s = statuses[i]; + if (!s.ok() && !s.IsNotFound()) { + fprintf(stderr, "multiget error: %s\n", s.ToString().c_str()); + thread->stats.AddErrors(1); + ret_status[rand_key] = s; + // we continue after error rather than exiting so that we can + // find more errors if any + } else if (s.IsNotFound()) { + thread->stats.AddGets(1, 0); + ret_status[rand_key] = s; + } else { + assert(!keys[i].empty()); + assert(!values[i].empty()); + + const char expected = keys[i][0]; + const char actual = values[i][values[i].size() - 1]; + + if (expected != actual) { + fprintf(stderr, "multiget error expected = %c actual = %c\n", + expected, actual); + } + + values[i].remove_suffix(1); // get rid of the differing character + + thread->stats.AddGets(1, 1); + } + } + db_->ReleaseSnapshot(readoptionscopy.snapshot); + + // Now that we retrieved all values, check that they all match + for (size_t i = 1; i < num_prefixes; i++) { + if (values[i] != values[0]) { + fprintf(stderr, + "multiget error: inconsistent values for key %s: %s, %s\n", + StringToHex(key_str[i]).c_str(), + StringToHex(values[0].ToString()).c_str(), + StringToHex(values[i].ToString()).c_str()); + // we continue after error rather than exiting so that we can + // find more errors if any + } + } + } + + return ret_status; + } + + void TestGetEntity(ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + assert(thread); + + ManagedSnapshot snapshot_guard(db_); + + ReadOptions read_opts_copy(read_opts); + read_opts_copy.snapshot = snapshot_guard.snapshot(); + + assert(!rand_keys.empty()); + + const std::string key_suffix = Key(rand_keys[0]); + + assert(!rand_column_families.empty()); + assert(rand_column_families[0] >= 0); + assert(rand_column_families[0] < static_cast(column_families_.size())); + + ColumnFamilyHandle* const cfh = column_families_[rand_column_families[0]]; + assert(cfh); + + constexpr size_t num_keys = 10; + + std::array results; + + for (size_t i = 0; i < num_keys; ++i) { + const std::string key = std::to_string(i) + key_suffix; + + const Status s = db_->GetEntity(read_opts_copy, cfh, key, &results[i]); + + if (!s.ok() && !s.IsNotFound()) { + fprintf(stderr, "GetEntity error: %s\n", s.ToString().c_str()); + thread->stats.AddErrors(1); + } else if (s.IsNotFound()) { + thread->stats.AddGets(1, 0); + } else { + thread->stats.AddGets(1, 1); + } + } + + for (size_t i = 0; i < num_keys; ++i) { + const WideColumns& columns = results[i].columns(); + + if (!CompareColumns(results[0].columns(), columns)) { + fprintf(stderr, + "GetEntity error: inconsistent entities for key %s: %s, %s\n", + StringToHex(key_suffix).c_str(), + WideColumnsToHex(results[0].columns()).c_str(), + WideColumnsToHex(columns).c_str()); + } + + if (!columns.empty()) { + // The last character of each column value should be 'i' as a decimal + // digit + const char expected = static_cast('0' + i); + + for (const auto& column : columns) { + const Slice& value = column.value(); + + if (value.empty() || value[value.size() - 1] != expected) { + fprintf(stderr, + "GetEntity error: incorrect column value for key " + "%s, entity %s, column value %s, expected %c\n", + StringToHex(key_suffix).c_str(), + WideColumnsToHex(columns).c_str(), + value.ToString(/* hex */ true).c_str(), expected); + } + } + + if (!VerifyWideColumns(columns)) { + fprintf( + stderr, + "GetEntity error: inconsistent columns for key %s, entity %s\n", + StringToHex(key_suffix).c_str(), + WideColumnsToHex(columns).c_str()); + } + } + } + } + + void TestMultiGetEntity(ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + assert(thread); + + assert(!rand_column_families.empty()); + assert(rand_column_families[0] >= 0); + assert(rand_column_families[0] < static_cast(column_families_.size())); + + ColumnFamilyHandle* const cfh = column_families_[rand_column_families[0]]; + assert(cfh); + + assert(!rand_keys.empty()); + + ManagedSnapshot snapshot_guard(db_); + + ReadOptions read_opts_copy(read_opts); + read_opts_copy.snapshot = snapshot_guard.snapshot(); + + const size_t num_keys = rand_keys.size(); + + for (size_t i = 0; i < num_keys; ++i) { + const std::string key_suffix = Key(rand_keys[i]); + + constexpr size_t num_prefixes = 10; + + std::array keys; + std::array key_slices; + std::array results; + std::array statuses; + + for (size_t j = 0; j < num_prefixes; ++j) { + keys[j] = std::to_string(j) + key_suffix; + key_slices[j] = keys[j]; + } + + db_->MultiGetEntity(read_opts_copy, cfh, num_prefixes, key_slices.data(), + results.data(), statuses.data()); + + for (size_t j = 0; j < num_prefixes; ++j) { + const Status& s = statuses[j]; + + if (!s.ok() && !s.IsNotFound()) { + fprintf(stderr, "MultiGetEntity error: %s\n", s.ToString().c_str()); + thread->stats.AddErrors(1); + } else if (s.IsNotFound()) { + thread->stats.AddGets(1, 0); + } else { + thread->stats.AddGets(1, 1); + } + + const WideColumns& cmp_columns = results[0].columns(); + const WideColumns& columns = results[j].columns(); + + if (!CompareColumns(cmp_columns, columns)) { + fprintf(stderr, + "MultiGetEntity error: inconsistent entities for key %s: %s, " + "%s\n", + StringToHex(key_suffix).c_str(), + WideColumnsToHex(cmp_columns).c_str(), + WideColumnsToHex(columns).c_str()); + } + + if (!columns.empty()) { + // The last character of each column value should be 'j' as a decimal + // digit + const char expected = static_cast('0' + j); + + for (const auto& column : columns) { + const Slice& value = column.value(); + + if (value.empty() || value[value.size() - 1] != expected) { + fprintf(stderr, + "MultiGetEntity error: incorrect column value for key " + "%s, entity %s, column value %s, expected %c\n", + StringToHex(key_suffix).c_str(), + WideColumnsToHex(columns).c_str(), + value.ToString(/* hex */ true).c_str(), expected); + } + } + + if (!VerifyWideColumns(columns)) { + fprintf(stderr, + "MultiGetEntity error: inconsistent columns for key %s, " + "entity %s\n", + StringToHex(key_suffix).c_str(), + WideColumnsToHex(columns).c_str()); + } + } + } + } + } + + // Given a key, this does prefix scans for "0"+P, "1"+P, ..., "9"+P + // in the same snapshot where P is the first FLAGS_prefix_size - 1 bytes + // of the key. Each of these 10 scans returns a series of values; + // each series should be the same length, and it is verified for each + // index i that all the i'th values are of the form V+"0", V+"1", ..., V+"9". + // ASSUMES that MultiPut was used to put (K, V) + Status TestPrefixScan(ThreadState* thread, const ReadOptions& readoptions, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + assert(!rand_column_families.empty()); + assert(!rand_keys.empty()); + + const std::string key = Key(rand_keys[0]); + + assert(FLAGS_prefix_size > 0); + const size_t prefix_to_use = static_cast(FLAGS_prefix_size); + + constexpr size_t num_prefixes = 10; + + std::array prefixes; + std::array prefix_slices; + std::array ro_copies; + std::array upper_bounds; + std::array ub_slices; + std::array, num_prefixes> iters; + + const Snapshot* const snapshot = db_->GetSnapshot(); + + ColumnFamilyHandle* const cfh = column_families_[rand_column_families[0]]; + assert(cfh); + + for (size_t i = 0; i < num_prefixes; ++i) { + prefixes[i] = std::to_string(i) + key; + prefix_slices[i] = Slice(prefixes[i].data(), prefix_to_use); + + ro_copies[i] = readoptions; + ro_copies[i].snapshot = snapshot; + if (thread->rand.OneIn(2) && + GetNextPrefix(prefix_slices[i], &(upper_bounds[i]))) { + // For half of the time, set the upper bound to the next prefix + ub_slices[i] = upper_bounds[i]; + ro_copies[i].iterate_upper_bound = &(ub_slices[i]); + } + + iters[i].reset(db_->NewIterator(ro_copies[i], cfh)); + iters[i]->Seek(prefix_slices[i]); + } + + uint64_t count = 0; + + while (iters[0]->Valid() && iters[0]->key().starts_with(prefix_slices[0])) { + ++count; + + std::array values; + + // get list of all values for this iteration + for (size_t i = 0; i < num_prefixes; ++i) { + // no iterator should finish before the first one + assert(iters[i]->Valid() && + iters[i]->key().starts_with(prefix_slices[i])); + values[i] = iters[i]->value().ToString(); + + // make sure the last character of the value is the expected digit + assert(!prefixes[i].empty()); + assert(!values[i].empty()); + + const char expected = prefixes[i].front(); + const char actual = values[i].back(); + + if (expected != actual) { + fprintf(stderr, "prefix scan error expected = %c actual = %c\n", + expected, actual); + } + + values[i].pop_back(); // get rid of the differing character + + // make sure all values are equivalent + if (values[i] != values[0]) { + fprintf(stderr, + "prefix scan error : %" ROCKSDB_PRIszt + ", inconsistent values for prefix %s: %s, %s\n", + i, prefix_slices[i].ToString(/* hex */ true).c_str(), + StringToHex(values[0]).c_str(), + StringToHex(values[i]).c_str()); + // we continue after error rather than exiting so that we can + // find more errors if any + } + + // make sure value() and columns() are consistent + if (!VerifyWideColumns(iters[i]->value(), iters[i]->columns())) { + fprintf(stderr, + "prefix scan error : %" ROCKSDB_PRIszt + ", value and columns inconsistent for prefix %s: value: %s, " + "columns: %s\n", + i, prefix_slices[i].ToString(/* hex */ true).c_str(), + iters[i]->value().ToString(/* hex */ true).c_str(), + WideColumnsToHex(iters[i]->columns()).c_str()); + } + + iters[i]->Next(); + } + } + + // cleanup iterators and snapshot + for (size_t i = 0; i < num_prefixes; ++i) { + // if the first iterator finished, they should have all finished + assert(!iters[i]->Valid() || + !iters[i]->key().starts_with(prefix_slices[i])); + assert(iters[i]->status().ok()); + } + + db_->ReleaseSnapshot(snapshot); + + thread->stats.AddPrefixes(1, count); + + return Status::OK(); + } + + void VerifyDb(ThreadState* /* thread */) const override {} + + void ContinuouslyVerifyDb(ThreadState* /* thread */) const override {} + + // Compare columns ignoring the last character of column values + bool CompareColumns(const WideColumns& lhs, const WideColumns& rhs) { + if (lhs.size() != rhs.size()) { + return false; + } + + for (size_t i = 0; i < lhs.size(); ++i) { + if (lhs[i].name() != rhs[i].name()) { + return false; + } + + if (lhs[i].value().size() != rhs[i].value().size()) { + return false; + } + + if (lhs[i].value().difference_offset(rhs[i].value()) < + lhs[i].value().size() - 1) { + return false; + } + } + + return true; + } +}; + +StressTest* CreateBatchedOpsStressTest() { return new BatchedOpsStressTest(); } + +} // namespace ROCKSDB_NAMESPACE +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db_stress_tool/cf_consistency_stress.cc b/librocksdb-sys/rocksdb/db_stress_tool/cf_consistency_stress.cc new file mode 100644 index 0000000..f3d9b71 --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/cf_consistency_stress.cc @@ -0,0 +1,892 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifdef GFLAGS +#include "db_stress_tool/db_stress_common.h" +#include "file/file_util.h" + +namespace ROCKSDB_NAMESPACE { +class CfConsistencyStressTest : public StressTest { + public: + CfConsistencyStressTest() : batch_id_(0) {} + + ~CfConsistencyStressTest() override {} + + bool IsStateTracked() const override { return false; } + + Status TestPut(ThreadState* thread, WriteOptions& write_opts, + const ReadOptions& /* read_opts */, + const std::vector& rand_column_families, + const std::vector& rand_keys, + char (&value)[100]) override { + assert(!rand_column_families.empty()); + assert(!rand_keys.empty()); + + const std::string k = Key(rand_keys[0]); + + const uint32_t value_base = batch_id_.fetch_add(1); + const size_t sz = GenerateValue(value_base, value, sizeof(value)); + const Slice v(value, sz); + + WriteBatch batch; + + const bool use_put_entity = !FLAGS_use_merge && + FLAGS_use_put_entity_one_in > 0 && + (value_base % FLAGS_use_put_entity_one_in) == 0; + + for (auto cf : rand_column_families) { + ColumnFamilyHandle* const cfh = column_families_[cf]; + assert(cfh); + + if (FLAGS_use_merge) { + batch.Merge(cfh, k, v); + } else if (use_put_entity) { + batch.PutEntity(cfh, k, GenerateWideColumns(value_base, v)); + } else { + batch.Put(cfh, k, v); + } + } + + Status s = db_->Write(write_opts, &batch); + + if (!s.ok()) { + fprintf(stderr, "multi put or merge error: %s\n", s.ToString().c_str()); + thread->stats.AddErrors(1); + } else { + auto num = static_cast(rand_column_families.size()); + thread->stats.AddBytesForWrites(num, (sz + 1) * num); + } + + return s; + } + + Status TestDelete(ThreadState* thread, WriteOptions& write_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + std::string key_str = Key(rand_keys[0]); + Slice key = key_str; + WriteBatch batch; + for (auto cf : rand_column_families) { + ColumnFamilyHandle* cfh = column_families_[cf]; + batch.Delete(cfh, key); + } + Status s = db_->Write(write_opts, &batch); + if (!s.ok()) { + fprintf(stderr, "multidel error: %s\n", s.ToString().c_str()); + thread->stats.AddErrors(1); + } else { + thread->stats.AddDeletes(static_cast(rand_column_families.size())); + } + return s; + } + + Status TestDeleteRange(ThreadState* thread, WriteOptions& write_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + int64_t rand_key = rand_keys[0]; + auto shared = thread->shared; + int64_t max_key = shared->GetMaxKey(); + if (rand_key > max_key - FLAGS_range_deletion_width) { + rand_key = + thread->rand.Next() % (max_key - FLAGS_range_deletion_width + 1); + } + std::string key_str = Key(rand_key); + Slice key = key_str; + std::string end_key_str = Key(rand_key + FLAGS_range_deletion_width); + Slice end_key = end_key_str; + WriteBatch batch; + for (auto cf : rand_column_families) { + ColumnFamilyHandle* cfh = column_families_[rand_column_families[cf]]; + batch.DeleteRange(cfh, key, end_key); + } + Status s = db_->Write(write_opts, &batch); + if (!s.ok()) { + fprintf(stderr, "multi del range error: %s\n", s.ToString().c_str()); + thread->stats.AddErrors(1); + } else { + thread->stats.AddRangeDeletions( + static_cast(rand_column_families.size())); + } + return s; + } + + void TestIngestExternalFile( + ThreadState* /* thread */, + const std::vector& /* rand_column_families */, + const std::vector& /* rand_keys */) override { + assert(false); + fprintf(stderr, + "CfConsistencyStressTest does not support TestIngestExternalFile " + "because it's not possible to verify the result\n"); + std::terminate(); + } + + Status TestGet(ThreadState* thread, const ReadOptions& readoptions, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + std::string key_str = Key(rand_keys[0]); + Slice key = key_str; + Status s; + bool is_consistent = true; + + if (thread->rand.OneIn(2)) { + // 1/2 chance, does a random read from random CF + auto cfh = + column_families_[rand_column_families[thread->rand.Next() % + rand_column_families.size()]]; + std::string from_db; + s = db_->Get(readoptions, cfh, key, &from_db); + } else { + // 1/2 chance, comparing one key is the same across all CFs + const Snapshot* snapshot = db_->GetSnapshot(); + ReadOptions readoptionscopy = readoptions; + readoptionscopy.snapshot = snapshot; + + std::string value0; + s = db_->Get(readoptionscopy, column_families_[rand_column_families[0]], + key, &value0); + if (s.ok() || s.IsNotFound()) { + bool found = s.ok(); + for (size_t i = 1; i < rand_column_families.size(); i++) { + std::string value1; + s = db_->Get(readoptionscopy, + column_families_[rand_column_families[i]], key, &value1); + if (!s.ok() && !s.IsNotFound()) { + break; + } + if (!found && s.ok()) { + fprintf(stderr, "Get() return different results with key %s\n", + Slice(key_str).ToString(true).c_str()); + fprintf(stderr, "CF %s is not found\n", + column_family_names_[0].c_str()); + fprintf(stderr, "CF %s returns value %s\n", + column_family_names_[i].c_str(), + Slice(value1).ToString(true).c_str()); + is_consistent = false; + } else if (found && s.IsNotFound()) { + fprintf(stderr, "Get() return different results with key %s\n", + Slice(key_str).ToString(true).c_str()); + fprintf(stderr, "CF %s returns value %s\n", + column_family_names_[0].c_str(), + Slice(value0).ToString(true).c_str()); + fprintf(stderr, "CF %s is not found\n", + column_family_names_[i].c_str()); + is_consistent = false; + } else if (s.ok() && value0 != value1) { + fprintf(stderr, "Get() return different results with key %s\n", + Slice(key_str).ToString(true).c_str()); + fprintf(stderr, "CF %s returns value %s\n", + column_family_names_[0].c_str(), + Slice(value0).ToString(true).c_str()); + fprintf(stderr, "CF %s returns value %s\n", + column_family_names_[i].c_str(), + Slice(value1).ToString(true).c_str()); + is_consistent = false; + } + if (!is_consistent) { + break; + } + } + } + + db_->ReleaseSnapshot(snapshot); + } + if (!is_consistent) { + fprintf(stderr, "TestGet error: is_consistent is false\n"); + thread->stats.AddErrors(1); + // Fail fast to preserve the DB state. + thread->shared->SetVerificationFailure(); + } else if (s.ok()) { + thread->stats.AddGets(1, 1); + } else if (s.IsNotFound()) { + thread->stats.AddGets(1, 0); + } else { + fprintf(stderr, "TestGet error: %s\n", s.ToString().c_str()); + thread->stats.AddErrors(1); + } + return s; + } + + std::vector TestMultiGet( + ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + size_t num_keys = rand_keys.size(); + std::vector key_str; + std::vector keys; + keys.reserve(num_keys); + key_str.reserve(num_keys); + std::vector values(num_keys); + std::vector statuses(num_keys); + ColumnFamilyHandle* cfh = column_families_[rand_column_families[0]]; + ReadOptions readoptionscopy = read_opts; + readoptionscopy.rate_limiter_priority = + FLAGS_rate_limit_user_ops ? Env::IO_USER : Env::IO_TOTAL; + + for (size_t i = 0; i < num_keys; ++i) { + key_str.emplace_back(Key(rand_keys[i])); + keys.emplace_back(key_str.back()); + } + db_->MultiGet(readoptionscopy, cfh, num_keys, keys.data(), values.data(), + statuses.data()); + for (auto s : statuses) { + if (s.ok()) { + // found case + thread->stats.AddGets(1, 1); + } else if (s.IsNotFound()) { + // not found case + thread->stats.AddGets(1, 0); + } else { + // errors case + fprintf(stderr, "MultiGet error: %s\n", s.ToString().c_str()); + thread->stats.AddErrors(1); + } + } + return statuses; + } + + void TestGetEntity(ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + assert(thread); + assert(!rand_column_families.empty()); + assert(!rand_keys.empty()); + + const std::string key = Key(rand_keys[0]); + + Status s; + bool is_consistent = true; + + if (thread->rand.OneIn(2)) { + // With a 1/2 chance, do a random read from a random CF + const size_t cf_id = thread->rand.Next() % rand_column_families.size(); + + assert(rand_column_families[cf_id] >= 0); + assert(rand_column_families[cf_id] < + static_cast(column_families_.size())); + + ColumnFamilyHandle* const cfh = + column_families_[rand_column_families[cf_id]]; + assert(cfh); + + PinnableWideColumns result; + s = db_->GetEntity(read_opts, cfh, key, &result); + + if (s.ok()) { + if (!VerifyWideColumns(result.columns())) { + fprintf( + stderr, + "GetEntity error: inconsistent columns for key %s, entity %s\n", + StringToHex(key).c_str(), + WideColumnsToHex(result.columns()).c_str()); + is_consistent = false; + } + } + } else { + // With a 1/2 chance, compare one key across all CFs + ManagedSnapshot snapshot_guard(db_); + + ReadOptions read_opts_copy = read_opts; + read_opts_copy.snapshot = snapshot_guard.snapshot(); + + assert(rand_column_families[0] >= 0); + assert(rand_column_families[0] < + static_cast(column_families_.size())); + + PinnableWideColumns cmp_result; + s = db_->GetEntity(read_opts_copy, + column_families_[rand_column_families[0]], key, + &cmp_result); + + if (s.ok() || s.IsNotFound()) { + const bool cmp_found = s.ok(); + + if (cmp_found) { + if (!VerifyWideColumns(cmp_result.columns())) { + fprintf(stderr, + "GetEntity error: inconsistent columns for key %s, " + "entity %s\n", + StringToHex(key).c_str(), + WideColumnsToHex(cmp_result.columns()).c_str()); + is_consistent = false; + } + } + + if (is_consistent) { + for (size_t i = 1; i < rand_column_families.size(); ++i) { + assert(rand_column_families[i] >= 0); + assert(rand_column_families[i] < + static_cast(column_families_.size())); + + PinnableWideColumns result; + s = db_->GetEntity(read_opts_copy, + column_families_[rand_column_families[i]], key, + &result); + + if (!s.ok() && !s.IsNotFound()) { + break; + } + + const bool found = s.ok(); + + assert(!column_family_names_.empty()); + assert(i < column_family_names_.size()); + + if (!cmp_found && found) { + fprintf(stderr, + "GetEntity returns different results for key %s: CF %s " + "returns not found, CF %s returns entity %s\n", + StringToHex(key).c_str(), column_family_names_[0].c_str(), + column_family_names_[i].c_str(), + WideColumnsToHex(result.columns()).c_str()); + is_consistent = false; + break; + } + + if (cmp_found && !found) { + fprintf(stderr, + "GetEntity returns different results for key %s: CF %s " + "returns entity %s, CF %s returns not found\n", + StringToHex(key).c_str(), column_family_names_[0].c_str(), + WideColumnsToHex(cmp_result.columns()).c_str(), + column_family_names_[i].c_str()); + is_consistent = false; + break; + } + + if (found && result != cmp_result) { + fprintf(stderr, + "GetEntity returns different results for key %s: CF %s " + "returns entity %s, CF %s returns entity %s\n", + StringToHex(key).c_str(), column_family_names_[0].c_str(), + WideColumnsToHex(cmp_result.columns()).c_str(), + column_family_names_[i].c_str(), + WideColumnsToHex(result.columns()).c_str()); + is_consistent = false; + break; + } + } + } + } + } + + if (!is_consistent) { + fprintf(stderr, "TestGetEntity error: results are not consistent\n"); + thread->stats.AddErrors(1); + // Fail fast to preserve the DB state. + thread->shared->SetVerificationFailure(); + } else if (s.ok()) { + thread->stats.AddGets(1, 1); + } else if (s.IsNotFound()) { + thread->stats.AddGets(1, 0); + } else { + fprintf(stderr, "TestGetEntity error: %s\n", s.ToString().c_str()); + thread->stats.AddErrors(1); + } + } + + void TestMultiGetEntity(ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + assert(thread); + assert(thread->shared); + assert(!rand_column_families.empty()); + assert(!rand_keys.empty()); + + ManagedSnapshot snapshot_guard(db_); + + ReadOptions read_opts_copy = read_opts; + read_opts_copy.snapshot = snapshot_guard.snapshot(); + + const size_t num_cfs = rand_column_families.size(); + + std::vector cfhs; + cfhs.reserve(num_cfs); + + for (size_t j = 0; j < num_cfs; ++j) { + assert(rand_column_families[j] >= 0); + assert(rand_column_families[j] < + static_cast(column_families_.size())); + + ColumnFamilyHandle* const cfh = column_families_[rand_column_families[j]]; + assert(cfh); + + cfhs.emplace_back(cfh); + } + + const size_t num_keys = rand_keys.size(); + + for (size_t i = 0; i < num_keys; ++i) { + const std::string key = Key(rand_keys[i]); + + std::vector key_slices(num_cfs, key); + std::vector results(num_cfs); + std::vector statuses(num_cfs); + + db_->MultiGetEntity(read_opts_copy, num_cfs, cfhs.data(), + key_slices.data(), results.data(), statuses.data()); + + bool is_consistent = true; + + for (size_t j = 0; j < num_cfs; ++j) { + const Status& s = statuses[j]; + const Status& cmp_s = statuses[0]; + const WideColumns& columns = results[j].columns(); + const WideColumns& cmp_columns = results[0].columns(); + + if (!s.ok() && !s.IsNotFound()) { + fprintf(stderr, "TestMultiGetEntity error: %s\n", + s.ToString().c_str()); + thread->stats.AddErrors(1); + break; + } + + assert(cmp_s.ok() || cmp_s.IsNotFound()); + + if (s.IsNotFound()) { + if (cmp_s.ok()) { + fprintf( + stderr, + "MultiGetEntity returns different results for key %s: CF %s " + "returns entity %s, CF %s returns not found\n", + StringToHex(key).c_str(), column_family_names_[0].c_str(), + WideColumnsToHex(cmp_columns).c_str(), + column_family_names_[j].c_str()); + is_consistent = false; + break; + } + + continue; + } + + assert(s.ok()); + if (cmp_s.IsNotFound()) { + fprintf(stderr, + "MultiGetEntity returns different results for key %s: CF %s " + "returns not found, CF %s returns entity %s\n", + StringToHex(key).c_str(), column_family_names_[0].c_str(), + column_family_names_[j].c_str(), + WideColumnsToHex(columns).c_str()); + is_consistent = false; + break; + } + + if (columns != cmp_columns) { + fprintf(stderr, + "MultiGetEntity returns different results for key %s: CF %s " + "returns entity %s, CF %s returns entity %s\n", + StringToHex(key).c_str(), column_family_names_[0].c_str(), + WideColumnsToHex(cmp_columns).c_str(), + column_family_names_[j].c_str(), + WideColumnsToHex(columns).c_str()); + is_consistent = false; + break; + } + + if (!VerifyWideColumns(columns)) { + fprintf(stderr, + "MultiGetEntity error: inconsistent columns for key %s, " + "entity %s\n", + StringToHex(key).c_str(), WideColumnsToHex(columns).c_str()); + is_consistent = false; + break; + } + } + + if (!is_consistent) { + fprintf(stderr, + "TestMultiGetEntity error: results are not consistent\n"); + thread->stats.AddErrors(1); + // Fail fast to preserve the DB state. + thread->shared->SetVerificationFailure(); + break; + } else if (statuses[0].ok()) { + thread->stats.AddGets(1, 1); + } else if (statuses[0].IsNotFound()) { + thread->stats.AddGets(1, 0); + } + } + } + + Status TestPrefixScan(ThreadState* thread, const ReadOptions& readoptions, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + assert(!rand_column_families.empty()); + assert(!rand_keys.empty()); + + const std::string key = Key(rand_keys[0]); + + const size_t prefix_to_use = + (FLAGS_prefix_size < 0) ? 7 : static_cast(FLAGS_prefix_size); + + const Slice prefix(key.data(), prefix_to_use); + + std::string upper_bound; + Slice ub_slice; + + ReadOptions ro_copy = readoptions; + + // Get the next prefix first and then see if we want to set upper bound. + // We'll use the next prefix in an assertion later on + if (GetNextPrefix(prefix, &upper_bound) && thread->rand.OneIn(2)) { + ub_slice = Slice(upper_bound); + ro_copy.iterate_upper_bound = &ub_slice; + } + + ColumnFamilyHandle* const cfh = + column_families_[rand_column_families[thread->rand.Uniform( + static_cast(rand_column_families.size()))]]; + assert(cfh); + + std::unique_ptr iter(db_->NewIterator(ro_copy, cfh)); + + uint64_t count = 0; + Status s; + + for (iter->Seek(prefix); iter->Valid() && iter->key().starts_with(prefix); + iter->Next()) { + ++count; + + if (!VerifyWideColumns(iter->value(), iter->columns())) { + s = Status::Corruption("Value and columns inconsistent", + DebugString(iter->value(), iter->columns())); + break; + } + } + + assert(prefix_to_use == 0 || + count <= GetPrefixKeyCount(prefix.ToString(), upper_bound)); + + if (s.ok()) { + s = iter->status(); + } + + if (!s.ok()) { + fprintf(stderr, "TestPrefixScan error: %s\n", s.ToString().c_str()); + thread->stats.AddErrors(1); + + return s; + } + + thread->stats.AddPrefixes(1, count); + + return Status::OK(); + } + + ColumnFamilyHandle* GetControlCfh(ThreadState* thread, + int /*column_family_id*/ + ) override { + // All column families should contain the same data. Randomly pick one. + return column_families_[thread->rand.Next() % column_families_.size()]; + } + + void VerifyDb(ThreadState* thread) const override { + // This `ReadOptions` is for validation purposes. Ignore + // `FLAGS_rate_limit_user_ops` to avoid slowing any validation. + ReadOptions options(FLAGS_verify_checksum, true); + + // We must set total_order_seek to true because we are doing a SeekToFirst + // on a column family whose memtables may support (by default) prefix-based + // iterator. In this case, NewIterator with options.total_order_seek being + // false returns a prefix-based iterator. Calling SeekToFirst using this + // iterator causes the iterator to become invalid. That means we cannot + // iterate the memtable using this iterator any more, although the memtable + // contains the most up-to-date key-values. + options.total_order_seek = true; + + ManagedSnapshot snapshot_guard(db_); + options.snapshot = snapshot_guard.snapshot(); + + const size_t num = column_families_.size(); + + std::vector> iters; + iters.reserve(num); + + for (size_t i = 0; i < num; ++i) { + iters.emplace_back(db_->NewIterator(options, column_families_[i])); + iters.back()->SeekToFirst(); + } + + std::vector statuses(num, Status::OK()); + + assert(thread); + + auto shared = thread->shared; + assert(shared); + + do { + if (shared->HasVerificationFailedYet()) { + break; + } + + size_t valid_cnt = 0; + + for (size_t i = 0; i < num; ++i) { + const auto& iter = iters[i]; + assert(iter); + + if (iter->Valid()) { + if (!VerifyWideColumns(iter->value(), iter->columns())) { + statuses[i] = + Status::Corruption("Value and columns inconsistent", + DebugString(iter->value(), iter->columns())); + } else { + ++valid_cnt; + } + } else { + statuses[i] = iter->status(); + } + } + + if (valid_cnt == 0) { + for (size_t i = 0; i < num; ++i) { + const auto& s = statuses[i]; + if (!s.ok()) { + fprintf(stderr, "Iterator on cf %s has error: %s\n", + column_families_[i]->GetName().c_str(), + s.ToString().c_str()); + shared->SetVerificationFailure(); + } + } + + break; + } + + if (valid_cnt < num) { + shared->SetVerificationFailure(); + + for (size_t i = 0; i < num; ++i) { + assert(iters[i]); + + if (!iters[i]->Valid()) { + if (statuses[i].ok()) { + fprintf(stderr, "Finished scanning cf %s\n", + column_families_[i]->GetName().c_str()); + } else { + fprintf(stderr, "Iterator on cf %s has error: %s\n", + column_families_[i]->GetName().c_str(), + statuses[i].ToString().c_str()); + } + } else { + fprintf(stderr, "cf %s has remaining data to scan\n", + column_families_[i]->GetName().c_str()); + } + } + + break; + } + + if (shared->HasVerificationFailedYet()) { + break; + } + + // If the program reaches here, then all column families' iterators are + // still valid. + assert(valid_cnt == num); + + if (shared->PrintingVerificationResults()) { + continue; + } + + assert(iters[0]); + + const Slice key = iters[0]->key(); + const Slice value = iters[0]->value(); + + int num_mismatched_cfs = 0; + + for (size_t i = 1; i < num; ++i) { + assert(iters[i]); + + const int cmp = key.compare(iters[i]->key()); + + if (cmp != 0) { + ++num_mismatched_cfs; + + if (1 == num_mismatched_cfs) { + fprintf(stderr, "Verification failed\n"); + fprintf(stderr, "Latest Sequence Number: %" PRIu64 "\n", + db_->GetLatestSequenceNumber()); + fprintf(stderr, "[%s] %s => %s\n", + column_families_[0]->GetName().c_str(), + key.ToString(true /* hex */).c_str(), + value.ToString(true /* hex */).c_str()); + } + + fprintf(stderr, "[%s] %s => %s\n", + column_families_[i]->GetName().c_str(), + iters[i]->key().ToString(true /* hex */).c_str(), + iters[i]->value().ToString(true /* hex */).c_str()); + + Slice begin_key; + Slice end_key; + if (cmp < 0) { + begin_key = key; + end_key = iters[i]->key(); + } else { + begin_key = iters[i]->key(); + end_key = key; + } + + const auto print_key_versions = [&](ColumnFamilyHandle* cfh) { + constexpr size_t kMaxNumIKeys = 8; + + std::vector versions; + const Status s = GetAllKeyVersions(db_, cfh, begin_key, end_key, + kMaxNumIKeys, &versions); + if (!s.ok()) { + fprintf(stderr, "%s\n", s.ToString().c_str()); + return; + } + + assert(cfh); + + fprintf(stderr, + "Internal keys in CF '%s', [%s, %s] (max %" ROCKSDB_PRIszt + ")\n", + cfh->GetName().c_str(), + begin_key.ToString(true /* hex */).c_str(), + end_key.ToString(true /* hex */).c_str(), kMaxNumIKeys); + + for (const KeyVersion& kv : versions) { + fprintf(stderr, " key %s seq %" PRIu64 " type %d\n", + Slice(kv.user_key).ToString(true).c_str(), kv.sequence, + kv.type); + } + }; + + if (1 == num_mismatched_cfs) { + print_key_versions(column_families_[0]); + } + + print_key_versions(column_families_[i]); + + shared->SetVerificationFailure(); + } + } + + shared->FinishPrintingVerificationResults(); + + for (auto& iter : iters) { + assert(iter); + iter->Next(); + } + } while (true); + } + + void ContinuouslyVerifyDb(ThreadState* thread) const override { + assert(thread); + Status status; + + DB* db_ptr = cmp_db_ ? cmp_db_ : db_; + const auto& cfhs = cmp_db_ ? cmp_cfhs_ : column_families_; + + // Take a snapshot to preserve the state of primary db. + ManagedSnapshot snapshot_guard(db_); + + SharedState* shared = thread->shared; + assert(shared); + + if (cmp_db_) { + status = cmp_db_->TryCatchUpWithPrimary(); + if (!status.ok()) { + fprintf(stderr, "TryCatchUpWithPrimary: %s\n", + status.ToString().c_str()); + shared->SetShouldStopTest(); + assert(false); + return; + } + } + + const auto checksum_column_family = [](Iterator* iter, + uint32_t* checksum) -> Status { + assert(nullptr != checksum); + + uint32_t ret = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ret = crc32c::Extend(ret, iter->key().data(), iter->key().size()); + ret = crc32c::Extend(ret, iter->value().data(), iter->value().size()); + + for (const auto& column : iter->columns()) { + ret = crc32c::Extend(ret, column.name().data(), column.name().size()); + ret = + crc32c::Extend(ret, column.value().data(), column.value().size()); + } + } + + *checksum = ret; + return iter->status(); + }; + // This `ReadOptions` is for validation purposes. Ignore + // `FLAGS_rate_limit_user_ops` to avoid slowing any validation. + ReadOptions ropts(FLAGS_verify_checksum, true); + ropts.total_order_seek = true; + if (nullptr == cmp_db_) { + ropts.snapshot = snapshot_guard.snapshot(); + } + uint32_t crc = 0; + { + // Compute crc for all key-values of default column family. + std::unique_ptr it(db_ptr->NewIterator(ropts)); + status = checksum_column_family(it.get(), &crc); + if (!status.ok()) { + fprintf(stderr, "Computing checksum of default cf: %s\n", + status.ToString().c_str()); + assert(false); + } + } + // Since we currently intentionally disallow reading from the secondary + // instance with snapshot, we cannot achieve cross-cf consistency if WAL is + // enabled because there is no guarantee that secondary instance replays + // the primary's WAL to a consistent point where all cfs have the same + // data. + if (status.ok() && FLAGS_disable_wal) { + uint32_t tmp_crc = 0; + for (ColumnFamilyHandle* cfh : cfhs) { + if (cfh == db_ptr->DefaultColumnFamily()) { + continue; + } + std::unique_ptr it(db_ptr->NewIterator(ropts, cfh)); + status = checksum_column_family(it.get(), &tmp_crc); + if (!status.ok() || tmp_crc != crc) { + break; + } + } + if (!status.ok()) { + fprintf(stderr, "status: %s\n", status.ToString().c_str()); + shared->SetShouldStopTest(); + assert(false); + } else if (tmp_crc != crc) { + fprintf(stderr, "tmp_crc=%" PRIu32 " crc=%" PRIu32 "\n", tmp_crc, crc); + shared->SetShouldStopTest(); + assert(false); + } + } + } + + std::vector GenerateColumnFamilies( + const int /* num_column_families */, + int /* rand_column_family */) const override { + std::vector ret; + int num = static_cast(column_families_.size()); + int k = 0; + std::generate_n(back_inserter(ret), num, [&k]() -> int { return k++; }); + return ret; + } + + private: + std::atomic batch_id_; +}; + +StressTest* CreateCfConsistencyStressTest() { + return new CfConsistencyStressTest(); +} + +} // namespace ROCKSDB_NAMESPACE +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db_stress_tool/db_stress.cc b/librocksdb-sys/rocksdb/db_stress_tool/db_stress.cc new file mode 100644 index 0000000..2d03f5d --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/db_stress.cc @@ -0,0 +1,25 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef GFLAGS +#include + +int main() { + fprintf(stderr, "Please install gflags to run rocksdb tools\n"); + return 1; +} +#else +#include "port/stack_trace.h" +#include "rocksdb/db_stress_tool.h" + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + return ROCKSDB_NAMESPACE::db_stress_tool(argc, argv); +} +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db_stress_tool/db_stress_common.cc b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_common.cc new file mode 100644 index 0000000..93436d0 --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_common.cc @@ -0,0 +1,491 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// + +#ifdef GFLAGS +#include "db_stress_tool/db_stress_common.h" + +#include + +#include "util/file_checksum_helper.h" +#include "util/xxhash.h" + +ROCKSDB_NAMESPACE::Env* db_stress_listener_env = nullptr; +ROCKSDB_NAMESPACE::Env* db_stress_env = nullptr; +// If non-null, injects read error at a rate specified by the +// read_fault_one_in or write_fault_one_in flag +std::shared_ptr fault_fs_guard; +enum ROCKSDB_NAMESPACE::CompressionType compression_type_e = + ROCKSDB_NAMESPACE::kSnappyCompression; +enum ROCKSDB_NAMESPACE::CompressionType bottommost_compression_type_e = + ROCKSDB_NAMESPACE::kSnappyCompression; +enum ROCKSDB_NAMESPACE::ChecksumType checksum_type_e = + ROCKSDB_NAMESPACE::kCRC32c; +enum RepFactory FLAGS_rep_factory = kSkipList; +std::vector sum_probs(100001); +constexpr int64_t zipf_sum_size = 100000; + +namespace ROCKSDB_NAMESPACE { + +// Zipfian distribution is generated based on a pre-calculated array. +// It should be used before start the stress test. +// First, the probability distribution function (PDF) of this Zipfian follows +// power low. P(x) = 1/(x^alpha). +// So we calculate the PDF when x is from 0 to zipf_sum_size in first for loop +// and add the PDF value togetger as c. So we get the total probability in c. +// Next, we calculate inverse CDF of Zipfian and store the value of each in +// an array (sum_probs). The rank is from 0 to zipf_sum_size. For example, for +// integer k, its Zipfian CDF value is sum_probs[k]. +// Third, when we need to get an integer whose probability follows Zipfian +// distribution, we use a rand_seed [0,1] which follows uniform distribution +// as a seed and search it in the sum_probs via binary search. When we find +// the closest sum_probs[i] of rand_seed, i is the integer that in +// [0, zipf_sum_size] following Zipfian distribution with parameter alpha. +// Finally, we can scale i to [0, max_key] scale. +// In order to avoid that hot keys are close to each other and skew towards 0, +// we use Rando64 to shuffle it. +void InitializeHotKeyGenerator(double alpha) { + double c = 0; + for (int64_t i = 1; i <= zipf_sum_size; i++) { + c = c + (1.0 / std::pow(static_cast(i), alpha)); + } + c = 1.0 / c; + + sum_probs[0] = 0; + for (int64_t i = 1; i <= zipf_sum_size; i++) { + sum_probs[i] = + sum_probs[i - 1] + c / std::pow(static_cast(i), alpha); + } +} + +// Generate one key that follows the Zipfian distribution. The skewness +// is decided by the parameter alpha. Input is the rand_seed [0,1] and +// the max of the key to be generated. If we directly return tmp_zipf_seed, +// the closer to 0, the higher probability will be. To randomly distribute +// the hot keys in [0, max_key], we use Random64 to shuffle it. +int64_t GetOneHotKeyID(double rand_seed, int64_t max_key) { + int64_t low = 1, mid, high = zipf_sum_size, zipf = 0; + while (low <= high) { + mid = (low + high) / 2; + if (sum_probs[mid] >= rand_seed && sum_probs[mid - 1] < rand_seed) { + zipf = mid; + break; + } else if (sum_probs[mid] >= rand_seed) { + high = mid - 1; + } else { + low = mid + 1; + } + } + int64_t tmp_zipf_seed = zipf * max_key / zipf_sum_size; + Random64 rand_local(tmp_zipf_seed); + return rand_local.Next() % max_key; +} + +void PoolSizeChangeThread(void* v) { + assert(FLAGS_compaction_thread_pool_adjust_interval > 0); + ThreadState* thread = reinterpret_cast(v); + SharedState* shared = thread->shared; + + while (true) { + { + MutexLock l(shared->GetMutex()); + if (shared->ShouldStopBgThread()) { + shared->IncBgThreadsFinished(); + if (shared->BgThreadsFinished()) { + shared->GetCondVar()->SignalAll(); + } + return; + } + } + + auto thread_pool_size_base = FLAGS_max_background_compactions; + auto thread_pool_size_var = FLAGS_compaction_thread_pool_variations; + int new_thread_pool_size = + thread_pool_size_base - thread_pool_size_var + + thread->rand.Next() % (thread_pool_size_var * 2 + 1); + if (new_thread_pool_size < 1) { + new_thread_pool_size = 1; + } + db_stress_env->SetBackgroundThreads(new_thread_pool_size, + ROCKSDB_NAMESPACE::Env::Priority::LOW); + // Sleep up to 3 seconds + db_stress_env->SleepForMicroseconds( + thread->rand.Next() % FLAGS_compaction_thread_pool_adjust_interval * + 1000 + + 1); + } +} + +void DbVerificationThread(void* v) { + assert(FLAGS_continuous_verification_interval > 0); + auto* thread = reinterpret_cast(v); + SharedState* shared = thread->shared; + StressTest* stress_test = shared->GetStressTest(); + assert(stress_test != nullptr); + while (true) { + { + MutexLock l(shared->GetMutex()); + if (shared->ShouldStopBgThread()) { + shared->IncBgThreadsFinished(); + if (shared->BgThreadsFinished()) { + shared->GetCondVar()->SignalAll(); + } + return; + } + } + if (!shared->HasVerificationFailedYet()) { + stress_test->ContinuouslyVerifyDb(thread); + } + db_stress_env->SleepForMicroseconds( + thread->rand.Next() % FLAGS_continuous_verification_interval * 1000 + + 1); + } +} + +void PrintKeyValue(int cf, uint64_t key, const char* value, size_t sz) { + if (!FLAGS_verbose) { + return; + } + std::string tmp; + tmp.reserve(sz * 2 + 16); + char buf[4]; + for (size_t i = 0; i < sz; i++) { + snprintf(buf, 4, "%X", value[i]); + tmp.append(buf); + } + auto key_str = Key(key); + Slice key_slice = key_str; + fprintf(stdout, "[CF %d] %s (%" PRIi64 ") == > (%" ROCKSDB_PRIszt ") %s\n", + cf, key_slice.ToString(true).c_str(), key, sz, tmp.c_str()); +} + +// Note that if hot_key_alpha != 0, it generates the key based on Zipfian +// distribution. Keys are randomly scattered to [0, FLAGS_max_key]. It does +// not ensure the order of the keys being generated and the keys does not have +// the active range which is related to FLAGS_active_width. +int64_t GenerateOneKey(ThreadState* thread, uint64_t iteration) { + const double completed_ratio = + static_cast(iteration) / FLAGS_ops_per_thread; + const int64_t base_key = static_cast( + completed_ratio * (FLAGS_max_key - FLAGS_active_width)); + int64_t rand_seed = base_key + thread->rand.Next() % FLAGS_active_width; + int64_t cur_key = rand_seed; + if (FLAGS_hot_key_alpha != 0) { + // If set the Zipfian distribution Alpha to non 0, use Zipfian + double float_rand = + (static_cast(thread->rand.Next() % FLAGS_max_key)) / + FLAGS_max_key; + cur_key = GetOneHotKeyID(float_rand, FLAGS_max_key); + } + return cur_key; +} + +// Note that if hot_key_alpha != 0, it generates the key based on Zipfian +// distribution. Keys being generated are in random order. +// If user want to generate keys based on uniform distribution, user needs to +// set hot_key_alpha == 0. It will generate the random keys in increasing +// order in the key array (ensure key[i] >= key[i+1]) and constrained in a +// range related to FLAGS_active_width. +std::vector GenerateNKeys(ThreadState* thread, int num_keys, + uint64_t iteration) { + const double completed_ratio = + static_cast(iteration) / FLAGS_ops_per_thread; + const int64_t base_key = static_cast( + completed_ratio * (FLAGS_max_key - FLAGS_active_width)); + std::vector keys; + keys.reserve(num_keys); + int64_t next_key = base_key + thread->rand.Next() % FLAGS_active_width; + keys.push_back(next_key); + for (int i = 1; i < num_keys; ++i) { + // Generate the key follows zipfian distribution + if (FLAGS_hot_key_alpha != 0) { + double float_rand = + (static_cast(thread->rand.Next() % FLAGS_max_key)) / + FLAGS_max_key; + next_key = GetOneHotKeyID(float_rand, FLAGS_max_key); + } else { + // This may result in some duplicate keys + next_key = next_key + thread->rand.Next() % + (FLAGS_active_width - (next_key - base_key)); + } + keys.push_back(next_key); + } + return keys; +} + +size_t GenerateValue(uint32_t rand, char* v, size_t max_sz) { + size_t value_sz = + ((rand % kRandomValueMaxFactor) + 1) * FLAGS_value_size_mult; + assert(value_sz <= max_sz && value_sz >= sizeof(uint32_t)); + (void)max_sz; + PutUnaligned(reinterpret_cast(v), rand); + for (size_t i = sizeof(uint32_t); i < value_sz; i++) { + v[i] = (char)(rand ^ i); + } + v[value_sz] = '\0'; + return value_sz; // the size of the value set. +} + +uint32_t GetValueBase(Slice s) { + assert(s.size() >= sizeof(uint32_t)); + uint32_t res; + GetUnaligned(reinterpret_cast(s.data()), &res); + return res; +} + +WideColumns GenerateWideColumns(uint32_t value_base, const Slice& slice) { + WideColumns columns; + + constexpr size_t max_columns = 4; + const size_t num_columns = (value_base % max_columns) + 1; + + columns.reserve(num_columns); + + assert(slice.size() >= num_columns); + + columns.emplace_back(kDefaultWideColumnName, slice); + + for (size_t i = 1; i < num_columns; ++i) { + const Slice name(slice.data(), i); + const Slice value(slice.data() + i, slice.size() - i); + + columns.emplace_back(name, value); + } + + return columns; +} + +WideColumns GenerateExpectedWideColumns(uint32_t value_base, + const Slice& slice) { + if (FLAGS_use_put_entity_one_in == 0 || + (value_base % FLAGS_use_put_entity_one_in) != 0) { + return WideColumns{{kDefaultWideColumnName, slice}}; + } + + WideColumns columns = GenerateWideColumns(value_base, slice); + + std::sort(columns.begin(), columns.end(), + [](const WideColumn& lhs, const WideColumn& rhs) { + return lhs.name().compare(rhs.name()) < 0; + }); + + return columns; +} + +bool VerifyWideColumns(const Slice& value, const WideColumns& columns) { + if (value.size() < sizeof(uint32_t)) { + return false; + } + + const uint32_t value_base = GetValueBase(value); + + const WideColumns expected_columns = + GenerateExpectedWideColumns(value_base, value); + + if (columns != expected_columns) { + return false; + } + + return true; +} + +bool VerifyWideColumns(const WideColumns& columns) { + if (columns.empty()) { + return false; + } + + if (columns.front().name() != kDefaultWideColumnName) { + return false; + } + + const Slice& value_of_default = columns.front().value(); + + return VerifyWideColumns(value_of_default, columns); +} + +std::string GetNowNanos() { + uint64_t t = db_stress_env->NowNanos(); + std::string ret; + PutFixed64(&ret, t); + return ret; +} + +namespace { + +class MyXXH64Checksum : public FileChecksumGenerator { + public: + explicit MyXXH64Checksum(bool big) : big_(big) { + state_ = XXH64_createState(); + XXH64_reset(state_, 0); + } + + virtual ~MyXXH64Checksum() override { XXH64_freeState(state_); } + + void Update(const char* data, size_t n) override { + XXH64_update(state_, data, n); + } + + void Finalize() override { + assert(str_.empty()); + uint64_t digest = XXH64_digest(state_); + // Store as little endian raw bytes + PutFixed64(&str_, digest); + if (big_) { + // Throw in some more data for stress testing (448 bits total) + PutFixed64(&str_, GetSliceHash64(str_)); + PutFixed64(&str_, GetSliceHash64(str_)); + PutFixed64(&str_, GetSliceHash64(str_)); + PutFixed64(&str_, GetSliceHash64(str_)); + PutFixed64(&str_, GetSliceHash64(str_)); + PutFixed64(&str_, GetSliceHash64(str_)); + } + } + + std::string GetChecksum() const override { + assert(!str_.empty()); + return str_; + } + + const char* Name() const override { + return big_ ? "MyBigChecksum" : "MyXXH64Checksum"; + } + + private: + bool big_; + XXH64_state_t* state_; + std::string str_; +}; + +class DbStressChecksumGenFactory : public FileChecksumGenFactory { + std::string default_func_name_; + + std::unique_ptr CreateFromFuncName( + const std::string& func_name) { + std::unique_ptr rv; + if (func_name == "FileChecksumCrc32c") { + rv.reset(new FileChecksumGenCrc32c(FileChecksumGenContext())); + } else if (func_name == "MyXXH64Checksum") { + rv.reset(new MyXXH64Checksum(false /* big */)); + } else if (func_name == "MyBigChecksum") { + rv.reset(new MyXXH64Checksum(true /* big */)); + } else { + // Should be a recognized function when we get here + assert(false); + } + return rv; + } + + public: + explicit DbStressChecksumGenFactory(const std::string& default_func_name) + : default_func_name_(default_func_name) {} + + std::unique_ptr CreateFileChecksumGenerator( + const FileChecksumGenContext& context) override { + if (context.requested_checksum_func_name.empty()) { + return CreateFromFuncName(default_func_name_); + } else { + return CreateFromFuncName(context.requested_checksum_func_name); + } + } + + const char* Name() const override { return "FileChecksumGenCrc32cFactory"; } +}; + +} // namespace + +std::shared_ptr GetFileChecksumImpl( + const std::string& name) { + // Translate from friendly names to internal names + std::string internal_name; + if (name == "crc32c") { + internal_name = "FileChecksumCrc32c"; + } else if (name == "xxh64") { + internal_name = "MyXXH64Checksum"; + } else if (name == "big") { + internal_name = "MyBigChecksum"; + } else { + assert(name.empty() || name == "none"); + return nullptr; + } + return std::make_shared(internal_name); +} + +Status DeleteFilesInDirectory(const std::string& dirname) { + std::vector filenames; + Status s = Env::Default()->GetChildren(dirname, &filenames); + for (size_t i = 0; s.ok() && i < filenames.size(); ++i) { + s = Env::Default()->DeleteFile(dirname + "/" + filenames[i]); + } + return s; +} + +Status SaveFilesInDirectory(const std::string& src_dirname, + const std::string& dst_dirname) { + std::vector filenames; + Status s = Env::Default()->GetChildren(src_dirname, &filenames); + for (size_t i = 0; s.ok() && i < filenames.size(); ++i) { + bool is_dir = false; + s = Env::Default()->IsDirectory(src_dirname + "/" + filenames[i], &is_dir); + if (s.ok()) { + if (is_dir) { + continue; + } + s = Env::Default()->LinkFile(src_dirname + "/" + filenames[i], + dst_dirname + "/" + filenames[i]); + } + } + return s; +} + +Status InitUnverifiedSubdir(const std::string& dirname) { + Status s = Env::Default()->FileExists(dirname); + if (s.IsNotFound()) { + return Status::OK(); + } + + const std::string kUnverifiedDirname = dirname + "/unverified"; + if (s.ok()) { + s = Env::Default()->CreateDirIfMissing(kUnverifiedDirname); + } + if (s.ok()) { + // It might already exist with some stale contents. Delete any such + // contents. + s = DeleteFilesInDirectory(kUnverifiedDirname); + } + if (s.ok()) { + s = SaveFilesInDirectory(dirname, kUnverifiedDirname); + } + return s; +} + +Status DestroyUnverifiedSubdir(const std::string& dirname) { + Status s = Env::Default()->FileExists(dirname); + if (s.IsNotFound()) { + return Status::OK(); + } + + const std::string kUnverifiedDirname = dirname + "/unverified"; + if (s.ok()) { + s = Env::Default()->FileExists(kUnverifiedDirname); + } + if (s.IsNotFound()) { + return Status::OK(); + } + + if (s.ok()) { + s = DeleteFilesInDirectory(kUnverifiedDirname); + } + if (s.ok()) { + s = Env::Default()->DeleteDir(kUnverifiedDirname); + } + return s; +} + +} // namespace ROCKSDB_NAMESPACE +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db_stress_tool/db_stress_common.h b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_common.h new file mode 100644 index 0000000..0a9dd42 --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_common.h @@ -0,0 +1,689 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// The test uses an array to compare against values written to the database. +// Keys written to the array are in 1:1 correspondence to the actual values in +// the database according to the formula in the function GenerateValue. + +// Space is reserved in the array from 0 to FLAGS_max_key and values are +// randomly written/deleted/read from those positions. During verification we +// compare all the positions in the array. To shorten/elongate the running +// time, you could change the settings: FLAGS_max_key, FLAGS_ops_per_thread, +// (sometimes also FLAGS_threads). +// +// NOTE that if FLAGS_test_batches_snapshots is set, the test will have +// different behavior. See comment of the flag for details. + +#ifdef GFLAGS +#pragma once +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "db/db_impl/db_impl.h" +#include "db/version_set.h" +#include "db_stress_tool/db_stress_env_wrapper.h" +#include "db_stress_tool/db_stress_listener.h" +#include "db_stress_tool/db_stress_shared_state.h" +#include "db_stress_tool/db_stress_test_base.h" +#include "logging/logging.h" +#include "monitoring/histogram.h" +#include "options/options_helper.h" +#include "port/port.h" +#include "rocksdb/cache.h" +#include "rocksdb/env.h" +#include "rocksdb/slice.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/statistics.h" +#include "rocksdb/utilities/backup_engine.h" +#include "rocksdb/utilities/checkpoint.h" +#include "rocksdb/utilities/db_ttl.h" +#include "rocksdb/utilities/debug.h" +#include "rocksdb/utilities/optimistic_transaction_db.h" +#include "rocksdb/utilities/options_util.h" +#include "rocksdb/utilities/transaction.h" +#include "rocksdb/utilities/transaction_db.h" +#include "rocksdb/write_batch.h" +#include "test_util/testutil.h" +#include "util/coding.h" +#include "util/compression.h" +#include "util/crc32c.h" +#include "util/gflags_compat.h" +#include "util/mutexlock.h" +#include "util/random.h" +#include "util/string_util.h" +#include "utilities/blob_db/blob_db.h" +#include "utilities/fault_injection_fs.h" +#include "utilities/merge_operators.h" + +using GFLAGS_NAMESPACE::ParseCommandLineFlags; +using GFLAGS_NAMESPACE::RegisterFlagValidator; +using GFLAGS_NAMESPACE::SetUsageMessage; + +DECLARE_uint64(seed); +DECLARE_bool(read_only); +DECLARE_int64(max_key); +DECLARE_double(hot_key_alpha); +DECLARE_int32(max_key_len); +DECLARE_string(key_len_percent_dist); +DECLARE_int32(key_window_scale_factor); +DECLARE_int32(column_families); +DECLARE_string(options_file); +DECLARE_int64(active_width); +DECLARE_bool(test_batches_snapshots); +DECLARE_bool(atomic_flush); +DECLARE_int32(manual_wal_flush_one_in); +DECLARE_int32(lock_wal_one_in); +DECLARE_bool(test_cf_consistency); +DECLARE_bool(test_multi_ops_txns); +DECLARE_int32(threads); +DECLARE_int32(ttl); +DECLARE_int32(value_size_mult); +DECLARE_int32(compaction_readahead_size); +DECLARE_bool(enable_pipelined_write); +DECLARE_bool(verify_before_write); +DECLARE_bool(histogram); +DECLARE_bool(destroy_db_initially); +DECLARE_bool(verbose); +DECLARE_bool(progress_reports); +DECLARE_uint64(db_write_buffer_size); +DECLARE_int32(write_buffer_size); +DECLARE_int32(max_write_buffer_number); +DECLARE_int32(min_write_buffer_number_to_merge); +DECLARE_int32(max_write_buffer_number_to_maintain); +DECLARE_int64(max_write_buffer_size_to_maintain); +DECLARE_double(memtable_prefix_bloom_size_ratio); +DECLARE_bool(memtable_whole_key_filtering); +DECLARE_int32(open_files); +DECLARE_int64(compressed_cache_size); +DECLARE_int32(compressed_cache_numshardbits); +DECLARE_int32(compaction_style); +DECLARE_int32(compaction_pri); +DECLARE_int32(num_levels); +DECLARE_int32(level0_file_num_compaction_trigger); +DECLARE_int32(level0_slowdown_writes_trigger); +DECLARE_int32(level0_stop_writes_trigger); +DECLARE_int32(block_size); +DECLARE_int32(format_version); +DECLARE_int32(index_block_restart_interval); +DECLARE_bool(disable_auto_compactions); +DECLARE_int32(max_background_compactions); +DECLARE_int32(num_bottom_pri_threads); +DECLARE_int32(compaction_thread_pool_adjust_interval); +DECLARE_int32(compaction_thread_pool_variations); +DECLARE_int32(max_background_flushes); +DECLARE_int32(universal_size_ratio); +DECLARE_int32(universal_min_merge_width); +DECLARE_int32(universal_max_merge_width); +DECLARE_int32(universal_max_size_amplification_percent); +DECLARE_int32(clear_column_family_one_in); +DECLARE_int32(get_live_files_one_in); +DECLARE_int32(get_sorted_wal_files_one_in); +DECLARE_int32(get_current_wal_file_one_in); +DECLARE_int32(set_options_one_in); +DECLARE_int32(set_in_place_one_in); +DECLARE_int64(cache_size); +DECLARE_int32(cache_numshardbits); +DECLARE_bool(cache_index_and_filter_blocks); +DECLARE_bool(charge_compression_dictionary_building_buffer); +DECLARE_bool(charge_filter_construction); +DECLARE_bool(charge_table_reader); +DECLARE_bool(charge_file_metadata); +DECLARE_bool(charge_blob_cache); +DECLARE_int32(top_level_index_pinning); +DECLARE_int32(partition_pinning); +DECLARE_int32(unpartitioned_pinning); +DECLARE_string(cache_type); +DECLARE_uint64(subcompactions); +DECLARE_uint64(periodic_compaction_seconds); +DECLARE_uint64(compaction_ttl); +DECLARE_bool(fifo_allow_compaction); +DECLARE_bool(allow_concurrent_memtable_write); +DECLARE_double(experimental_mempurge_threshold); +DECLARE_bool(enable_write_thread_adaptive_yield); +DECLARE_int32(reopen); +DECLARE_double(bloom_bits); +DECLARE_int32(ribbon_starting_level); +DECLARE_bool(partition_filters); +DECLARE_bool(optimize_filters_for_memory); +DECLARE_bool(detect_filter_construct_corruption); +DECLARE_int32(index_type); +DECLARE_int32(data_block_index_type); +DECLARE_string(db); +DECLARE_string(secondaries_base); +DECLARE_bool(test_secondary); +DECLARE_string(expected_values_dir); +DECLARE_bool(verify_checksum); +DECLARE_bool(mmap_read); +DECLARE_bool(mmap_write); +DECLARE_bool(use_direct_reads); +DECLARE_bool(use_direct_io_for_flush_and_compaction); +DECLARE_bool(mock_direct_io); +DECLARE_bool(statistics); +DECLARE_bool(sync); +DECLARE_bool(use_fsync); +DECLARE_uint64(stats_dump_period_sec); +DECLARE_uint64(bytes_per_sync); +DECLARE_uint64(wal_bytes_per_sync); +DECLARE_int32(kill_random_test); +DECLARE_string(kill_exclude_prefixes); +DECLARE_bool(disable_wal); +DECLARE_uint64(recycle_log_file_num); +DECLARE_int64(target_file_size_base); +DECLARE_int32(target_file_size_multiplier); +DECLARE_uint64(max_bytes_for_level_base); +DECLARE_double(max_bytes_for_level_multiplier); +DECLARE_int32(range_deletion_width); +DECLARE_uint64(rate_limiter_bytes_per_sec); +DECLARE_bool(rate_limit_bg_reads); +DECLARE_bool(rate_limit_user_ops); +DECLARE_bool(rate_limit_auto_wal_flush); +DECLARE_uint64(sst_file_manager_bytes_per_sec); +DECLARE_uint64(sst_file_manager_bytes_per_truncate); +DECLARE_int32(backup_one_in); +DECLARE_uint64(backup_max_size); +DECLARE_int32(checkpoint_one_in); +DECLARE_int32(ingest_external_file_one_in); +DECLARE_int32(ingest_external_file_width); +DECLARE_int32(compact_files_one_in); +DECLARE_int32(compact_range_one_in); +DECLARE_int32(mark_for_compaction_one_file_in); +DECLARE_int32(flush_one_in); +DECLARE_int32(pause_background_one_in); +DECLARE_int32(compact_range_width); +DECLARE_int32(acquire_snapshot_one_in); +DECLARE_bool(compare_full_db_state_snapshot); +DECLARE_uint64(snapshot_hold_ops); +DECLARE_bool(long_running_snapshots); +DECLARE_bool(use_multiget); +DECLARE_bool(use_get_entity); +DECLARE_bool(use_multi_get_entity); +DECLARE_int32(readpercent); +DECLARE_int32(prefixpercent); +DECLARE_int32(writepercent); +DECLARE_int32(delpercent); +DECLARE_int32(delrangepercent); +DECLARE_int32(nooverwritepercent); +DECLARE_int32(iterpercent); +DECLARE_uint64(num_iterations); +DECLARE_int32(customopspercent); +DECLARE_string(compression_type); +DECLARE_string(bottommost_compression_type); +DECLARE_int32(compression_max_dict_bytes); +DECLARE_int32(compression_zstd_max_train_bytes); +DECLARE_int32(compression_parallel_threads); +DECLARE_uint64(compression_max_dict_buffer_bytes); +DECLARE_bool(compression_use_zstd_dict_trainer); +DECLARE_string(checksum_type); +DECLARE_string(env_uri); +DECLARE_string(fs_uri); +DECLARE_uint64(ops_per_thread); +DECLARE_uint64(log2_keys_per_lock); +DECLARE_uint64(max_manifest_file_size); +DECLARE_bool(in_place_update); +DECLARE_string(memtablerep); +DECLARE_int32(prefix_size); +DECLARE_bool(use_merge); +DECLARE_uint32(use_put_entity_one_in); +DECLARE_bool(use_full_merge_v1); +DECLARE_int32(sync_wal_one_in); +DECLARE_bool(avoid_unnecessary_blocking_io); +DECLARE_bool(write_dbid_to_manifest); +DECLARE_bool(avoid_flush_during_recovery); +DECLARE_uint64(max_write_batch_group_size_bytes); +DECLARE_bool(level_compaction_dynamic_level_bytes); +DECLARE_int32(verify_checksum_one_in); +DECLARE_int32(verify_db_one_in); +DECLARE_int32(continuous_verification_interval); +DECLARE_int32(get_property_one_in); +DECLARE_string(file_checksum_impl); + +// Options for transaction dbs. +// Use TransactionDB (a.k.a. Pessimistic Transaction DB) +// OR OptimisticTransactionDB +DECLARE_bool(use_txn); + +// Options for TransactionDB (a.k.a. Pessimistic Transaction DB) +DECLARE_uint64(txn_write_policy); +DECLARE_bool(unordered_write); + +// Options for OptimisticTransactionDB +DECLARE_bool(use_optimistic_txn); +DECLARE_uint64(occ_validation_policy); +DECLARE_bool(share_occ_lock_buckets); +DECLARE_uint32(occ_lock_bucket_count); + +// Options for StackableDB-based BlobDB +DECLARE_bool(use_blob_db); +DECLARE_uint64(blob_db_min_blob_size); +DECLARE_uint64(blob_db_bytes_per_sync); +DECLARE_uint64(blob_db_file_size); +DECLARE_bool(blob_db_enable_gc); +DECLARE_double(blob_db_gc_cutoff); + +// Options for integrated BlobDB +DECLARE_bool(allow_setting_blob_options_dynamically); +DECLARE_bool(enable_blob_files); +DECLARE_uint64(min_blob_size); +DECLARE_uint64(blob_file_size); +DECLARE_string(blob_compression_type); +DECLARE_bool(enable_blob_garbage_collection); +DECLARE_double(blob_garbage_collection_age_cutoff); +DECLARE_double(blob_garbage_collection_force_threshold); +DECLARE_uint64(blob_compaction_readahead_size); +DECLARE_int32(blob_file_starting_level); +DECLARE_bool(use_blob_cache); +DECLARE_bool(use_shared_block_and_blob_cache); +DECLARE_uint64(blob_cache_size); +DECLARE_int32(blob_cache_numshardbits); +DECLARE_int32(prepopulate_blob_cache); + +DECLARE_int32(approximate_size_one_in); +DECLARE_bool(sync_fault_injection); + +DECLARE_bool(best_efforts_recovery); +DECLARE_bool(skip_verifydb); +DECLARE_bool(enable_compaction_filter); +DECLARE_bool(paranoid_file_checks); +DECLARE_bool(fail_if_options_file_error); +DECLARE_uint64(batch_protection_bytes_per_key); +DECLARE_uint32(memtable_protection_bytes_per_key); +DECLARE_uint32(block_protection_bytes_per_key); + +DECLARE_uint64(user_timestamp_size); +DECLARE_string(secondary_cache_uri); +DECLARE_int32(secondary_cache_fault_one_in); + +DECLARE_int32(prepopulate_block_cache); + +DECLARE_bool(two_write_queues); +DECLARE_bool(use_only_the_last_commit_time_batch_for_recovery); +DECLARE_uint64(wp_snapshot_cache_bits); +DECLARE_uint64(wp_commit_cache_bits); + +DECLARE_bool(adaptive_readahead); +DECLARE_bool(async_io); +DECLARE_string(wal_compression); +DECLARE_bool(verify_sst_unique_id_in_manifest); + +DECLARE_int32(create_timestamped_snapshot_one_in); + +DECLARE_bool(allow_data_in_errors); + +DECLARE_bool(enable_thread_tracking); + +DECLARE_uint32(memtable_max_range_deletions); + +// Tiered storage +DECLARE_bool(enable_tiered_storage); // set last_level_temperature +DECLARE_int64(preclude_last_level_data_seconds); +DECLARE_int64(preserve_internal_time_seconds); + +DECLARE_int32(verify_iterator_with_expected_state_one_in); +DECLARE_bool(preserve_unverified_changes); + +DECLARE_uint64(readahead_size); +DECLARE_uint64(initial_auto_readahead_size); +DECLARE_uint64(max_auto_readahead_size); +DECLARE_uint64(num_file_reads_for_auto_readahead); +DECLARE_bool(use_io_uring); + +constexpr long KB = 1024; +constexpr int kRandomValueMaxFactor = 3; +constexpr int kValueMaxLen = 100; + +// wrapped posix environment +extern ROCKSDB_NAMESPACE::Env* db_stress_env; +extern ROCKSDB_NAMESPACE::Env* db_stress_listener_env; +extern std::shared_ptr fault_fs_guard; + +extern enum ROCKSDB_NAMESPACE::CompressionType compression_type_e; +extern enum ROCKSDB_NAMESPACE::CompressionType bottommost_compression_type_e; +extern enum ROCKSDB_NAMESPACE::ChecksumType checksum_type_e; + +enum RepFactory { kSkipList, kHashSkipList, kVectorRep }; + +inline enum RepFactory StringToRepFactory(const char* ctype) { + assert(ctype); + + if (!strcasecmp(ctype, "skip_list")) + return kSkipList; + else if (!strcasecmp(ctype, "prefix_hash")) + return kHashSkipList; + else if (!strcasecmp(ctype, "vector")) + return kVectorRep; + + fprintf(stdout, "Cannot parse memreptable %s\n", ctype); + return kSkipList; +} + +extern enum RepFactory FLAGS_rep_factory; + +namespace ROCKSDB_NAMESPACE { +inline enum ROCKSDB_NAMESPACE::CompressionType StringToCompressionType( + const char* ctype) { + assert(ctype); + + ROCKSDB_NAMESPACE::CompressionType ret_compression_type; + + if (!strcasecmp(ctype, "disable")) { + ret_compression_type = ROCKSDB_NAMESPACE::kDisableCompressionOption; + } else if (!strcasecmp(ctype, "none")) { + ret_compression_type = ROCKSDB_NAMESPACE::kNoCompression; + } else if (!strcasecmp(ctype, "snappy")) { + ret_compression_type = ROCKSDB_NAMESPACE::kSnappyCompression; + } else if (!strcasecmp(ctype, "zlib")) { + ret_compression_type = ROCKSDB_NAMESPACE::kZlibCompression; + } else if (!strcasecmp(ctype, "bzip2")) { + ret_compression_type = ROCKSDB_NAMESPACE::kBZip2Compression; + } else if (!strcasecmp(ctype, "lz4")) { + ret_compression_type = ROCKSDB_NAMESPACE::kLZ4Compression; + } else if (!strcasecmp(ctype, "lz4hc")) { + ret_compression_type = ROCKSDB_NAMESPACE::kLZ4HCCompression; + } else if (!strcasecmp(ctype, "xpress")) { + ret_compression_type = ROCKSDB_NAMESPACE::kXpressCompression; + } else if (!strcasecmp(ctype, "zstd")) { + ret_compression_type = ROCKSDB_NAMESPACE::kZSTD; + } else { + fprintf(stderr, "Cannot parse compression type '%s'\n", ctype); + ret_compression_type = + ROCKSDB_NAMESPACE::kSnappyCompression; // default value + } + if (ret_compression_type != ROCKSDB_NAMESPACE::kDisableCompressionOption && + !CompressionTypeSupported(ret_compression_type)) { + // Use no compression will be more portable but considering this is + // only a stress test and snappy is widely available. Use snappy here. + ret_compression_type = ROCKSDB_NAMESPACE::kSnappyCompression; + } + return ret_compression_type; +} + +inline enum ROCKSDB_NAMESPACE::ChecksumType StringToChecksumType( + const char* ctype) { + assert(ctype); + auto iter = ROCKSDB_NAMESPACE::checksum_type_string_map.find(ctype); + if (iter != ROCKSDB_NAMESPACE::checksum_type_string_map.end()) { + return iter->second; + } + fprintf(stderr, "Cannot parse checksum type '%s'\n", ctype); + return ROCKSDB_NAMESPACE::kCRC32c; +} + +inline std::string ChecksumTypeToString(ROCKSDB_NAMESPACE::ChecksumType ctype) { + auto iter = std::find_if( + ROCKSDB_NAMESPACE::checksum_type_string_map.begin(), + ROCKSDB_NAMESPACE::checksum_type_string_map.end(), + [&](const std::pair& + name_and_enum_val) { return name_and_enum_val.second == ctype; }); + assert(iter != ROCKSDB_NAMESPACE::checksum_type_string_map.end()); + return iter->first; +} + +inline std::vector SplitString(std::string src) { + std::vector ret; + if (src.empty()) { + return ret; + } + size_t pos = 0; + size_t pos_comma; + while ((pos_comma = src.find(',', pos)) != std::string::npos) { + ret.push_back(src.substr(pos, pos_comma - pos)); + pos = pos_comma + 1; + } + ret.push_back(src.substr(pos, src.length())); + return ret; +} + +#ifdef _MSC_VER +#pragma warning(push) +// truncation of constant value on static_cast +#pragma warning(disable : 4309) +#endif +inline bool GetNextPrefix(const ROCKSDB_NAMESPACE::Slice& src, std::string* v) { + std::string ret = src.ToString(); + for (int i = static_cast(ret.size()) - 1; i >= 0; i--) { + if (ret[i] != static_cast(255)) { + ret[i] = ret[i] + 1; + break; + } else if (i != 0) { + ret[i] = 0; + } else { + // all FF. No next prefix + return false; + } + } + *v = ret; + return true; +} +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +// Append `val` to `*key` in fixed-width big-endian format +extern inline void AppendIntToString(uint64_t val, std::string* key) { + // PutFixed64 uses little endian + PutFixed64(key, val); + // Reverse to get big endian + char* int_data = &((*key)[key->size() - sizeof(uint64_t)]); + for (size_t i = 0; i < sizeof(uint64_t) / 2; ++i) { + std::swap(int_data[i], int_data[sizeof(uint64_t) - 1 - i]); + } +} + +// A struct for maintaining the parameters for generating variable length keys +struct KeyGenContext { + // Number of adjacent keys in one cycle of key lengths + uint64_t window; + // Number of keys of each possible length in a given window + std::vector weights; +}; +extern KeyGenContext key_gen_ctx; + +// Generate a variable length key string from the given int64 val. The +// order of the keys is preserved. The key could be anywhere from 8 to +// max_key_len * 8 bytes. +// The algorithm picks the length based on the +// offset of the val within a configured window and the distribution of the +// number of keys of various lengths in that window. For example, if x, y, x are +// the weights assigned to each possible key length, the keys generated would be +// - {0}...{x-1} +// {(x-1),0}..{(x-1),(y-1)},{(x-1),(y-1),0}..{(x-1),(y-1),(z-1)} and so on. +// Additionally, a trailer of 0-7 bytes could be appended. +extern inline std::string Key(int64_t val) { + uint64_t window = key_gen_ctx.window; + size_t levels = key_gen_ctx.weights.size(); + std::string key; + // Over-reserve and for now do not bother `shrink_to_fit()` since the key + // strings are transient. + key.reserve(FLAGS_max_key_len * 8); + + uint64_t window_idx = static_cast(val) / window; + uint64_t offset = static_cast(val) % window; + for (size_t level = 0; level < levels; ++level) { + uint64_t weight = key_gen_ctx.weights[level]; + uint64_t pfx; + if (level == 0) { + pfx = window_idx * weight; + } else { + pfx = 0; + } + pfx += offset >= weight ? weight - 1 : offset; + AppendIntToString(pfx, &key); + if (offset < weight) { + // Use the bottom 3 bits of offset as the number of trailing 'x's in the + // key. If the next key is going to be of the next level, then skip the + // trailer as it would break ordering. If the key length is already at + // max, skip the trailer. + if (offset < weight - 1 && level < levels - 1) { + size_t trailer_len = offset & 0x7; + key.append(trailer_len, 'x'); + } + break; + } + offset -= weight; + } + + return key; +} + +// Given a string key, map it to an index into the expected values buffer +extern inline bool GetIntVal(std::string big_endian_key, uint64_t* key_p) { + size_t size_key = big_endian_key.size(); + std::vector prefixes; + + assert(size_key <= key_gen_ctx.weights.size() * sizeof(uint64_t)); + + std::string little_endian_key; + little_endian_key.resize(size_key); + for (size_t start = 0; start + sizeof(uint64_t) <= size_key; + start += sizeof(uint64_t)) { + size_t end = start + sizeof(uint64_t); + for (size_t i = 0; i < sizeof(uint64_t); ++i) { + little_endian_key[start + i] = big_endian_key[end - 1 - i]; + } + Slice little_endian_slice = + Slice(&little_endian_key[start], sizeof(uint64_t)); + uint64_t pfx; + if (!GetFixed64(&little_endian_slice, &pfx)) { + return false; + } + prefixes.emplace_back(pfx); + } + + uint64_t key = 0; + for (size_t i = 0; i < prefixes.size(); ++i) { + uint64_t pfx = prefixes[i]; + key += (pfx / key_gen_ctx.weights[i]) * key_gen_ctx.window + + pfx % key_gen_ctx.weights[i]; + if (i < prefixes.size() - 1) { + // The encoding writes a `key_gen_ctx.weights[i] - 1` that counts for + // `key_gen_ctx.weights[i]` when there are more prefixes to come. So we + // need to add back the one here as we're at a non-last prefix. + ++key; + } + } + *key_p = key; + return true; +} + +// Given a string prefix, map it to the first corresponding index in the +// expected values buffer. +inline bool GetFirstIntValInPrefix(std::string big_endian_prefix, + uint64_t* key_p) { + size_t size_key = big_endian_prefix.size(); + // Pad with zeros to make it a multiple of 8. This function may be called + // with a prefix, in which case we return the first index that falls + // inside or outside that prefix, dependeing on whether the prefix is + // the start of upper bound of a scan + unsigned int pad = sizeof(uint64_t) - (size_key % sizeof(uint64_t)); + if (pad < sizeof(uint64_t)) { + big_endian_prefix.append(pad, '\0'); + } + return GetIntVal(std::move(big_endian_prefix), key_p); +} + +extern inline uint64_t GetPrefixKeyCount(const std::string& prefix, + const std::string& ub) { + uint64_t start = 0; + uint64_t end = 0; + + if (!GetFirstIntValInPrefix(prefix, &start) || + !GetFirstIntValInPrefix(ub, &end)) { + return 0; + } + + return end - start; +} + +extern inline std::string StringToHex(const std::string& str) { + std::string result = "0x"; + result.append(Slice(str).ToString(true)); + return result; +} + +inline std::string WideColumnsToHex(const WideColumns& columns) { + if (columns.empty()) { + return std::string(); + } + + std::ostringstream oss; + + oss << std::hex; + + auto it = columns.begin(); + oss << *it; + for (++it; it != columns.end(); ++it) { + oss << ' ' << *it; + } + + return oss.str(); +} + +// Unified output format for double parameters +extern inline std::string FormatDoubleParam(double param) { + return std::to_string(param); +} + +// Make sure that double parameter is a value we can reproduce by +// re-inputting the value printed. +extern inline void SanitizeDoubleParam(double* param) { + *param = std::atof(FormatDoubleParam(*param).c_str()); +} + +extern void PoolSizeChangeThread(void* v); + +extern void DbVerificationThread(void* v); + +extern void TimestampedSnapshotsThread(void* v); + +extern void PrintKeyValue(int cf, uint64_t key, const char* value, size_t sz); + +extern int64_t GenerateOneKey(ThreadState* thread, uint64_t iteration); + +extern std::vector GenerateNKeys(ThreadState* thread, int num_keys, + uint64_t iteration); + +extern size_t GenerateValue(uint32_t rand, char* v, size_t max_sz); +extern uint32_t GetValueBase(Slice s); + +extern WideColumns GenerateWideColumns(uint32_t value_base, const Slice& slice); +extern WideColumns GenerateExpectedWideColumns(uint32_t value_base, + const Slice& slice); +extern bool VerifyWideColumns(const Slice& value, const WideColumns& columns); +extern bool VerifyWideColumns(const WideColumns& columns); + +extern StressTest* CreateCfConsistencyStressTest(); +extern StressTest* CreateBatchedOpsStressTest(); +extern StressTest* CreateNonBatchedOpsStressTest(); +extern StressTest* CreateMultiOpsTxnsStressTest(); +extern void CheckAndSetOptionsForMultiOpsTxnStressTest(); +extern void InitializeHotKeyGenerator(double alpha); +extern int64_t GetOneHotKeyID(double rand_seed, int64_t max_key); + +extern std::string GetNowNanos(); + +std::shared_ptr GetFileChecksumImpl( + const std::string& name); + +Status DeleteFilesInDirectory(const std::string& dirname); +Status SaveFilesInDirectory(const std::string& src_dirname, + const std::string& dst_dirname); +Status DestroyUnverifiedSubdir(const std::string& dirname); +Status InitUnverifiedSubdir(const std::string& dirname); +} // namespace ROCKSDB_NAMESPACE +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db_stress_tool/db_stress_compaction_filter.h b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_compaction_filter.h new file mode 100644 index 0000000..408bb48 --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_compaction_filter.h @@ -0,0 +1,96 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include "db_stress_tool/db_stress_common.h" +#include "db_stress_tool/db_stress_shared_state.h" +#include "rocksdb/compaction_filter.h" + +namespace ROCKSDB_NAMESPACE { + +// DbStressCompactionFilter is safe to use with db_stress as it does not perform +// any mutation. It only makes `kRemove` decisions for keys that are already +// non-existent according to the `SharedState`. +class DbStressCompactionFilter : public CompactionFilter { + public: + DbStressCompactionFilter(SharedState* state, int cf_id) + : state_(state), cf_id_(cf_id) {} + + Decision FilterV2(int /*level*/, const Slice& key, ValueType /*value_type*/, + const Slice& /*existing_value*/, std::string* /*new_value*/, + std::string* /*skip_until*/) const override { + if (state_ == nullptr) { + return Decision::kKeep; + } + if (key.empty() || ('0' <= key[0] && key[0] <= '9')) { + // It is likely leftover from a test_batches_snapshots run. Below this + // conditional, the test_batches_snapshots key format is not handled + // properly. Just keep it to be safe. + return Decision::kKeep; + } + uint64_t key_num = 0; + { + Slice ukey_without_ts = key; + assert(ukey_without_ts.size() >= FLAGS_user_timestamp_size); + ukey_without_ts.remove_suffix(FLAGS_user_timestamp_size); + [[maybe_unused]] bool ok = + GetIntVal(ukey_without_ts.ToString(), &key_num); + assert(ok); + } + port::Mutex* key_mutex = state_->GetMutexForKey(cf_id_, key_num); + if (!key_mutex->TryLock()) { + return Decision::kKeep; + } + // Reaching here means we acquired the lock. + + bool key_exists = state_->Exists(cf_id_, key_num); + const bool allow_overwrite = state_->AllowsOverwrite(key_num); + + key_mutex->Unlock(); + + if (!key_exists) { + return allow_overwrite ? Decision::kRemove : Decision::kPurge; + } + return Decision::kKeep; + } + + const char* Name() const override { return "DbStressCompactionFilter"; } + + private: + SharedState* const state_; + const int cf_id_; +}; + +class DbStressCompactionFilterFactory : public CompactionFilterFactory { + public: + DbStressCompactionFilterFactory() : state_(nullptr) {} + + void SetSharedState(SharedState* state) { + MutexLock state_mutex_guard(&state_mutex_); + state_ = state; + } + + std::unique_ptr CreateCompactionFilter( + const CompactionFilter::Context& context) override { + MutexLock state_mutex_guard(&state_mutex_); + return std::unique_ptr( + new DbStressCompactionFilter(state_, context.column_family_id)); + } + + const char* Name() const override { + return "DbStressCompactionFilterFactory"; + } + + private: + port::Mutex state_mutex_; + SharedState* state_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db_stress_tool/db_stress_driver.cc b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_driver.cc new file mode 100644 index 0000000..4bf82c9 --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_driver.cc @@ -0,0 +1,216 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// + +#ifdef GFLAGS +#include "db_stress_tool/db_stress_common.h" +#include "utilities/fault_injection_fs.h" + +namespace ROCKSDB_NAMESPACE { +void ThreadBody(void* v) { + ThreadState* thread = reinterpret_cast(v); + SharedState* shared = thread->shared; + + if (!FLAGS_skip_verifydb && shared->ShouldVerifyAtBeginning()) { + thread->shared->GetStressTest()->VerifyDb(thread); + } + { + MutexLock l(shared->GetMutex()); + shared->IncInitialized(); + if (shared->AllInitialized()) { + shared->GetCondVar()->SignalAll(); + } + while (!shared->Started()) { + shared->GetCondVar()->Wait(); + } + } + thread->shared->GetStressTest()->OperateDb(thread); + + { + MutexLock l(shared->GetMutex()); + shared->IncOperated(); + if (shared->AllOperated()) { + shared->GetCondVar()->SignalAll(); + } + while (!shared->VerifyStarted()) { + shared->GetCondVar()->Wait(); + } + } + + if (!FLAGS_skip_verifydb) { + thread->shared->GetStressTest()->VerifyDb(thread); + } + + { + MutexLock l(shared->GetMutex()); + shared->IncDone(); + if (shared->AllDone()) { + shared->GetCondVar()->SignalAll(); + } + } +} +bool RunStressTestImpl(SharedState* shared) { + SystemClock* clock = db_stress_env->GetSystemClock().get(); + StressTest* stress = shared->GetStressTest(); + + if (shared->ShouldVerifyAtBeginning() && FLAGS_preserve_unverified_changes) { + Status s = InitUnverifiedSubdir(FLAGS_db); + if (s.ok() && !FLAGS_expected_values_dir.empty()) { + s = InitUnverifiedSubdir(FLAGS_expected_values_dir); + } + if (!s.ok()) { + fprintf(stderr, "Failed to setup unverified state dir: %s\n", + s.ToString().c_str()); + exit(1); + } + } + + stress->InitDb(shared); + stress->FinishInitDb(shared); + + if (FLAGS_sync_fault_injection) { + fault_fs_guard->SetFilesystemDirectWritable(false); + } + if (FLAGS_write_fault_one_in) { + fault_fs_guard->EnableWriteErrorInjection(); + } + + uint32_t n = FLAGS_threads; + uint64_t now = clock->NowMicros(); + fprintf(stdout, "%s Initializing worker threads\n", + clock->TimeToString(now / 1000000).c_str()); + + shared->SetThreads(n); + + if (FLAGS_compaction_thread_pool_adjust_interval > 0) { + shared->IncBgThreads(); + } + + if (FLAGS_continuous_verification_interval > 0) { + shared->IncBgThreads(); + } + + std::vector threads(n); + for (uint32_t i = 0; i < n; i++) { + threads[i] = new ThreadState(i, shared); + db_stress_env->StartThread(ThreadBody, threads[i]); + } + + ThreadState bg_thread(0, shared); + if (FLAGS_compaction_thread_pool_adjust_interval > 0) { + db_stress_env->StartThread(PoolSizeChangeThread, &bg_thread); + } + + ThreadState continuous_verification_thread(0, shared); + if (FLAGS_continuous_verification_interval > 0) { + db_stress_env->StartThread(DbVerificationThread, + &continuous_verification_thread); + } + + // Each thread goes through the following states: + // initializing -> wait for others to init -> read/populate/depopulate + // wait for others to operate -> verify -> done + + { + MutexLock l(shared->GetMutex()); + while (!shared->AllInitialized()) { + shared->GetCondVar()->Wait(); + } + if (shared->ShouldVerifyAtBeginning()) { + if (shared->HasVerificationFailedYet()) { + fprintf(stderr, "Crash-recovery verification failed :(\n"); + } else { + fprintf(stdout, "Crash-recovery verification passed :)\n"); + Status s = DestroyUnverifiedSubdir(FLAGS_db); + if (s.ok() && !FLAGS_expected_values_dir.empty()) { + s = DestroyUnverifiedSubdir(FLAGS_expected_values_dir); + } + if (!s.ok()) { + fprintf(stderr, "Failed to cleanup unverified state dir: %s\n", + s.ToString().c_str()); + exit(1); + } + } + } + + // This is after the verification step to avoid making all those `Get()`s + // and `MultiGet()`s contend on the DB-wide trace mutex. + if (!FLAGS_expected_values_dir.empty()) { + stress->TrackExpectedState(shared); + } + + now = clock->NowMicros(); + fprintf(stdout, "%s Starting database operations\n", + clock->TimeToString(now / 1000000).c_str()); + + shared->SetStart(); + shared->GetCondVar()->SignalAll(); + while (!shared->AllOperated()) { + shared->GetCondVar()->Wait(); + } + + now = clock->NowMicros(); + if (FLAGS_test_batches_snapshots) { + fprintf(stdout, "%s Limited verification already done during gets\n", + clock->TimeToString((uint64_t)now / 1000000).c_str()); + } else if (FLAGS_skip_verifydb) { + fprintf(stdout, "%s Verification skipped\n", + clock->TimeToString((uint64_t)now / 1000000).c_str()); + } else { + fprintf(stdout, "%s Starting verification\n", + clock->TimeToString((uint64_t)now / 1000000).c_str()); + } + + shared->SetStartVerify(); + shared->GetCondVar()->SignalAll(); + while (!shared->AllDone()) { + shared->GetCondVar()->Wait(); + } + } + + for (unsigned int i = 1; i < n; i++) { + threads[0]->stats.Merge(threads[i]->stats); + } + threads[0]->stats.Report("Stress Test"); + + for (unsigned int i = 0; i < n; i++) { + delete threads[i]; + threads[i] = nullptr; + } + now = clock->NowMicros(); + if (!FLAGS_skip_verifydb && !FLAGS_test_batches_snapshots && + !shared->HasVerificationFailedYet()) { + fprintf(stdout, "%s Verification successful\n", + clock->TimeToString(now / 1000000).c_str()); + } + stress->PrintStatistics(); + + if (FLAGS_compaction_thread_pool_adjust_interval > 0 || + FLAGS_continuous_verification_interval > 0) { + MutexLock l(shared->GetMutex()); + shared->SetShouldStopBgThread(); + while (!shared->BgThreadsFinished()) { + shared->GetCondVar()->Wait(); + } + } + + if (shared->HasVerificationFailedYet()) { + fprintf(stderr, "Verification failed :(\n"); + return false; + } + return true; +} +bool RunStressTest(SharedState* shared) { + ThreadStatusUtil::RegisterThread(db_stress_env, ThreadStatus::USER); + bool result = RunStressTestImpl(shared); + ThreadStatusUtil::UnregisterThread(); + return result; +} +} // namespace ROCKSDB_NAMESPACE +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db_stress_tool/db_stress_driver.h b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_driver.h new file mode 100644 index 0000000..a173470 --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_driver.h @@ -0,0 +1,18 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db_stress_tool/db_stress_shared_state.h" +#ifdef GFLAGS +#pragma once +#include "db_stress_tool/db_stress_test_base.h" +namespace ROCKSDB_NAMESPACE { +extern void ThreadBody(void* /*thread_state*/); +extern bool RunStressTest(SharedState*); +} // namespace ROCKSDB_NAMESPACE +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db_stress_tool/db_stress_env_wrapper.h b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_env_wrapper.h new file mode 100644 index 0000000..612d9fc --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_env_wrapper.h @@ -0,0 +1,78 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifdef GFLAGS +#pragma once +#include "db_stress_tool/db_stress_common.h" +#include "monitoring/thread_status_util.h" + +namespace ROCKSDB_NAMESPACE { +class DbStressRandomAccessFileWrapper : public FSRandomAccessFileOwnerWrapper { + public: + explicit DbStressRandomAccessFileWrapper( + std::unique_ptr&& target) + : FSRandomAccessFileOwnerWrapper(std::move(target)) {} + + IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const override { +#ifndef NDEBUG + const ThreadStatus::OperationType thread_op = + ThreadStatusUtil::GetThreadOperation(); + Env::IOActivity io_activity = + ThreadStatusUtil::TEST_GetExpectedIOActivity(thread_op); + assert(io_activity == Env::IOActivity::kUnknown || + io_activity == options.io_activity); +#endif + return target()->Read(offset, n, options, result, scratch, dbg); + } +}; + +class DbStressFSWrapper : public FileSystemWrapper { + public: + explicit DbStressFSWrapper(const std::shared_ptr& t) + : FileSystemWrapper(t) {} + static const char* kClassName() { return "DbStressFS"; } + const char* Name() const override { return kClassName(); } + + IOStatus NewRandomAccessFile(const std::string& f, + const FileOptions& file_opts, + std::unique_ptr* r, + IODebugContext* dbg) override { + std::unique_ptr file; + IOStatus s = target()->NewRandomAccessFile(f, file_opts, &file, dbg); + if (s.ok()) { + r->reset(new DbStressRandomAccessFileWrapper(std::move(file))); + } + return s; + } + + IOStatus DeleteFile(const std::string& f, const IOOptions& opts, + IODebugContext* dbg) override { + // We determine whether it is a manifest file by searching a strong, + // so that there will be false positive if the directory path contains the + // keyword but it is unlikely. + // Checkpoint, backup, and restore directories needs to be exempted. + if (!if_preserve_all_manifests || + f.find("MANIFEST-") == std::string::npos || + f.find("checkpoint") != std::string::npos || + f.find(".backup") != std::string::npos || + f.find(".restore") != std::string::npos) { + return target()->DeleteFile(f, opts, dbg); + } + // Rename the file instead of deletion to keep the history, and + // at the same time it is not visible to RocksDB. + return target()->RenameFile(f, f + "_renamed_", opts, dbg); + } + + // If true, all manifest files will not be delted in DeleteFile(). + bool if_preserve_all_manifests = true; +}; +} // namespace ROCKSDB_NAMESPACE +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db_stress_tool/db_stress_gflags.cc b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_gflags.cc new file mode 100644 index 0000000..3276418 --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_gflags.cc @@ -0,0 +1,1109 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifdef GFLAGS +#include "db_stress_tool/db_stress_common.h" + +static bool ValidateUint32Range(const char* flagname, uint64_t value) { + if (value > std::numeric_limits::max()) { + fprintf(stderr, "Invalid value for --%s: %lu, overflow\n", flagname, + (unsigned long)value); + return false; + } + return true; +} + +DEFINE_uint64(seed, 2341234, + "Seed for PRNG. When --nooverwritepercent is " + "nonzero and --expected_values_dir is nonempty, this value " + "must be fixed across invocations."); +static const bool FLAGS_seed_dummy __attribute__((__unused__)) = + RegisterFlagValidator(&FLAGS_seed, &ValidateUint32Range); + +DEFINE_bool(read_only, false, "True if open DB in read-only mode during tests"); + +DEFINE_int64(max_key, 1 * KB * KB, + "Max number of key/values to place in database"); + +DEFINE_int32(max_key_len, 3, "Maximum length of a key in 8-byte units"); + +DEFINE_string(key_len_percent_dist, "", + "Percentages of keys of various lengths. For example, 1,30,69 " + "means 1% of keys are 8 bytes, 30% are 16 bytes, and 69% are " + "24 bytes. If not specified, it will be evenly distributed"); + +DEFINE_int32(key_window_scale_factor, 10, + "This value will be multiplied by 100 to come up with a window " + "size for varying the key length"); + +DEFINE_int32(column_families, 10, "Number of column families"); + +DEFINE_double( + hot_key_alpha, 0, + "Use Zipfian distribution to generate the key " + "distribution. If it is not specified, write path will use random " + "distribution to generate the keys. The parameter is [0, double_max]). " + "However, the larger alpha is, the more shewed will be. If alpha is " + "larger than 2, it is likely that only 1 key will be accessed. The " + "Recommended value is [0.8-1.5]. The distribution is also related to " + "max_key and total iterations of generating the hot key. "); + +DEFINE_string( + options_file, "", + "The path to a RocksDB options file. If specified, then db_stress will " + "run with the RocksDB options in the default column family of the " + "specified options file. Note that, when an options file is provided, " + "db_stress will ignore the flag values for all options that may be passed " + "via options file."); + +DEFINE_int64( + active_width, 0, + "Number of keys in active span of the key-range at any given time. The " + "span begins with its left endpoint at key 0, gradually moves rightwards, " + "and ends with its right endpoint at max_key. If set to 0, active_width " + "will be sanitized to be equal to max_key."); + +// TODO(noetzli) Add support for single deletes +DEFINE_bool(test_batches_snapshots, false, + "If set, the test uses MultiGet(), MultiPut() and MultiDelete()" + " which read/write/delete multiple keys in a batch. In this mode," + " we do not verify db content by comparing the content with the " + "pre-allocated array. Instead, we do partial verification inside" + " MultiGet() by checking various values in a batch. Benefit of" + " this mode:\n" + "\t(a) No need to acquire mutexes during writes (less cache " + "flushes in multi-core leading to speed up)\n" + "\t(b) No long validation at the end (more speed up)\n" + "\t(c) Test snapshot and atomicity of batch writes"); + +DEFINE_bool(atomic_flush, false, + "If set, enables atomic flush in the options.\n"); + +DEFINE_int32( + manual_wal_flush_one_in, 0, + "If non-zero, then `FlushWAL(bool sync)`, where `bool sync` is randomly " + "decided, will be explictly called in db stress once for every N ops " + "on average. Setting `manual_wal_flush_one_in` to be greater than 0 " + "implies `Options::manual_wal_flush = true` is set."); + +DEFINE_int32(lock_wal_one_in, 1000000, + "If non-zero, then `LockWAL()` + `UnlockWAL()` will be called in " + "db_stress once for every N ops on average."); + +DEFINE_bool(test_cf_consistency, false, + "If set, runs the stress test dedicated to verifying writes to " + "multiple column families are consistent. Setting this implies " + "`atomic_flush=true` is set true if `disable_wal=false`.\n"); + +DEFINE_bool(test_multi_ops_txns, false, + "If set, runs stress test dedicated to verifying multi-ops " + "transactions on a simple relational table with primary and " + "secondary index."); + +DEFINE_int32(threads, 32, "Number of concurrent threads to run."); + +DEFINE_int32(ttl, -1, + "Opens the db with this ttl value if this is not -1. " + "Carefully specify a large value such that verifications on " + "deleted values don't fail"); + +DEFINE_int32(value_size_mult, 8, + "Size of value will be this number times rand_int(1,3) bytes"); + +DEFINE_int32(compaction_readahead_size, 0, "Compaction readahead size"); + +DEFINE_bool(enable_pipelined_write, false, "Pipeline WAL/memtable writes"); + +DEFINE_bool(verify_before_write, false, "Verify before write"); + +DEFINE_bool(histogram, false, "Print histogram of operation timings"); + +DEFINE_bool(destroy_db_initially, true, + "Destroys the database dir before start if this is true"); + +DEFINE_bool(verbose, false, "Verbose"); + +DEFINE_bool(progress_reports, true, + "If true, db_stress will report number of finished operations"); + +DEFINE_uint64(db_write_buffer_size, + ROCKSDB_NAMESPACE::Options().db_write_buffer_size, + "Number of bytes to buffer in all memtables before compacting"); + +DEFINE_int32( + write_buffer_size, + static_cast(ROCKSDB_NAMESPACE::Options().write_buffer_size), + "Number of bytes to buffer in memtable before compacting"); + +DEFINE_int32(max_write_buffer_number, + ROCKSDB_NAMESPACE::Options().max_write_buffer_number, + "The number of in-memory memtables. " + "Each memtable is of size FLAGS_write_buffer_size."); + +DEFINE_int32(min_write_buffer_number_to_merge, + ROCKSDB_NAMESPACE::Options().min_write_buffer_number_to_merge, + "The minimum number of write buffers that will be merged together " + "before writing to storage. This is cheap because it is an " + "in-memory merge. If this feature is not enabled, then all these " + "write buffers are flushed to L0 as separate files and this " + "increases read amplification because a get request has to check " + "in all of these files. Also, an in-memory merge may result in " + "writing less data to storage if there are duplicate records in" + " each of these individual write buffers."); + +DEFINE_int32(max_write_buffer_number_to_maintain, + ROCKSDB_NAMESPACE::Options().max_write_buffer_number_to_maintain, + "The total maximum number of write buffers to maintain in memory " + "including copies of buffers that have already been flushed. " + "Unlike max_write_buffer_number, this parameter does not affect " + "flushing. This controls the minimum amount of write history " + "that will be available in memory for conflict checking when " + "Transactions are used. If this value is too low, some " + "transactions may fail at commit time due to not being able to " + "determine whether there were any write conflicts. Setting this " + "value to 0 will cause write buffers to be freed immediately " + "after they are flushed. If this value is set to -1, " + "'max_write_buffer_number' will be used."); + +DEFINE_int64(max_write_buffer_size_to_maintain, + ROCKSDB_NAMESPACE::Options().max_write_buffer_size_to_maintain, + "The total maximum size of write buffers to maintain in memory " + "including copies of buffers that have already been flushed. " + "Unlike max_write_buffer_number, this parameter does not affect " + "flushing. This controls the minimum amount of write history " + "that will be available in memory for conflict checking when " + "Transactions are used. If this value is too low, some " + "transactions may fail at commit time due to not being able to " + "determine whether there were any write conflicts. Setting this " + "value to 0 will cause write buffers to be freed immediately " + "after they are flushed. If this value is set to -1, " + "'max_write_buffer_number' will be used."); + +DEFINE_double(memtable_prefix_bloom_size_ratio, + ROCKSDB_NAMESPACE::Options().memtable_prefix_bloom_size_ratio, + "creates prefix blooms for memtables, each with size " + "`write_buffer_size * memtable_prefix_bloom_size_ratio`."); + +DEFINE_bool(memtable_whole_key_filtering, + ROCKSDB_NAMESPACE::Options().memtable_whole_key_filtering, + "Enable whole key filtering in memtables."); + +DEFINE_int32(open_files, ROCKSDB_NAMESPACE::Options().max_open_files, + "Maximum number of files to keep open at the same time " + "(use default if == 0)"); + +DEFINE_int64(compressed_cache_size, 0, + "Number of bytes to use as a cache of compressed data." + " 0 means use default settings."); + +DEFINE_int32( + compressed_cache_numshardbits, -1, + "Number of shards for the compressed block cache is 2 ** " + "compressed_cache_numshardbits. Negative value means default settings. " + "This is applied only if compressed_cache_size is greater than 0."); + +DEFINE_int32(compaction_style, ROCKSDB_NAMESPACE::Options().compaction_style, + ""); + +DEFINE_int32(compaction_pri, ROCKSDB_NAMESPACE::Options().compaction_pri, + "Which file from a level should be picked to merge to the next " + "level in level-based compaction"); + +DEFINE_int32(num_levels, ROCKSDB_NAMESPACE::Options().num_levels, + "Number of levels in the DB"); + +DEFINE_int32(level0_file_num_compaction_trigger, + ROCKSDB_NAMESPACE::Options().level0_file_num_compaction_trigger, + "Level0 compaction start trigger"); + +DEFINE_int32(level0_slowdown_writes_trigger, + ROCKSDB_NAMESPACE::Options().level0_slowdown_writes_trigger, + "Number of files in level-0 that will slow down writes"); + +DEFINE_int32(level0_stop_writes_trigger, + ROCKSDB_NAMESPACE::Options().level0_stop_writes_trigger, + "Number of files in level-0 that will trigger put stop."); + +DEFINE_int32(block_size, + static_cast( + ROCKSDB_NAMESPACE::BlockBasedTableOptions().block_size), + "Number of bytes in a block."); + +DEFINE_int32(format_version, + static_cast( + ROCKSDB_NAMESPACE::BlockBasedTableOptions().format_version), + "Format version of SST files."); + +DEFINE_int32( + index_block_restart_interval, + ROCKSDB_NAMESPACE::BlockBasedTableOptions().index_block_restart_interval, + "Number of keys between restart points " + "for delta encoding of keys in index block."); + +DEFINE_bool(disable_auto_compactions, + ROCKSDB_NAMESPACE::Options().disable_auto_compactions, + "If true, RocksDB internally will not trigger compactions."); + +DEFINE_int32(max_background_compactions, + ROCKSDB_NAMESPACE::Options().max_background_compactions, + "The maximum number of concurrent background compactions " + "that can occur in parallel."); + +DEFINE_int32(num_bottom_pri_threads, 0, + "The number of threads in the bottom-priority thread pool (used " + "by universal compaction only)."); + +DEFINE_int32(compaction_thread_pool_adjust_interval, 0, + "The interval (in milliseconds) to adjust compaction thread pool " + "size. Don't change it periodically if the value is 0."); + +DEFINE_int32(compaction_thread_pool_variations, 2, + "Range of background thread pool size variations when adjusted " + "periodically."); + +DEFINE_int32(max_background_flushes, + ROCKSDB_NAMESPACE::Options().max_background_flushes, + "The maximum number of concurrent background flushes " + "that can occur in parallel."); + +DEFINE_int32(universal_size_ratio, 0, + "The ratio of file sizes that trigger" + " compaction in universal style"); + +DEFINE_int32(universal_min_merge_width, 0, + "The minimum number of files to " + "compact in universal style compaction"); + +DEFINE_int32(universal_max_merge_width, 0, + "The max number of files to compact" + " in universal style compaction"); + +DEFINE_int32(universal_max_size_amplification_percent, 0, + "The max size amplification for universal style compaction"); + +DEFINE_int32(clear_column_family_one_in, 1000000, + "With a chance of 1/N, delete a column family and then recreate " + "it again. If N == 0, never drop/create column families. " + "When test_batches_snapshots is true, this flag has no effect"); + +DEFINE_int32(get_live_files_one_in, 1000000, + "With a chance of 1/N, call GetLiveFiles to verify if it returns " + "correctly. If N == 0, do not call the interface."); + +DEFINE_int32( + get_sorted_wal_files_one_in, 1000000, + "With a chance of 1/N, call GetSortedWalFiles to verify if it returns " + "correctly. (Note that this API may legitimately return an error.) If N == " + "0, do not call the interface."); + +DEFINE_int32( + get_current_wal_file_one_in, 1000000, + "With a chance of 1/N, call GetCurrentWalFile to verify if it returns " + "correctly. (Note that this API may legitimately return an error.) If N == " + "0, do not call the interface."); + +DEFINE_int32(set_options_one_in, 0, + "With a chance of 1/N, change some random options"); + +DEFINE_int32(set_in_place_one_in, 0, + "With a chance of 1/N, toggle in place support option"); + +DEFINE_int64(cache_size, 2LL * KB * KB * KB, + "Number of bytes to use as a cache of uncompressed data."); + +DEFINE_int32(cache_numshardbits, 6, + "Number of shards for the block cache" + " is 2 ** cache_numshardbits. Negative means use default settings." + " This is applied only if FLAGS_cache_size is greater than 0."); + +DEFINE_bool(cache_index_and_filter_blocks, false, + "True if indexes/filters should be cached in block cache."); + +DEFINE_bool(charge_compression_dictionary_building_buffer, false, + "Setting for " + "CacheEntryRoleOptions::charged of " + "CacheEntryRole::kCompressionDictionaryBuildingBuffer"); + +DEFINE_bool(charge_filter_construction, false, + "Setting for " + "CacheEntryRoleOptions::charged of " + "CacheEntryRole::kFilterConstruction"); + +DEFINE_bool(charge_table_reader, false, + "Setting for " + "CacheEntryRoleOptions::charged of " + "CacheEntryRole::kBlockBasedTableReader"); + +DEFINE_bool(charge_file_metadata, false, + "Setting for " + "CacheEntryRoleOptions::charged of " + "kFileMetadata"); + +DEFINE_bool(charge_blob_cache, false, + "Setting for " + "CacheEntryRoleOptions::charged of " + "kBlobCache"); + +DEFINE_int32( + top_level_index_pinning, + static_cast(ROCKSDB_NAMESPACE::PinningTier::kFallback), + "Type of pinning for top-level indexes into metadata partitions (see " + "`enum PinningTier` in table.h)"); + +DEFINE_int32( + partition_pinning, + static_cast(ROCKSDB_NAMESPACE::PinningTier::kFallback), + "Type of pinning for metadata partitions (see `enum PinningTier` in " + "table.h)"); + +DEFINE_int32( + unpartitioned_pinning, + static_cast(ROCKSDB_NAMESPACE::PinningTier::kFallback), + "Type of pinning for unpartitioned metadata blocks (see `enum PinningTier` " + "in table.h)"); + +DEFINE_string(cache_type, "lru_cache", "Type of block cache."); + +DEFINE_uint64(subcompactions, 1, + "Maximum number of subcompactions to divide L0-L1 compactions " + "into."); + +DEFINE_uint64(periodic_compaction_seconds, 1000, + "Files older than this value will be picked up for compaction."); + +DEFINE_uint64(compaction_ttl, 1000, + "Files older than TTL will be compacted to the next level."); + +DEFINE_bool(fifo_allow_compaction, false, + "If true, set `Options::compaction_options_fifo.allow_compaction = " + "true`. It only take effect when FIFO compaction is used."); + +DEFINE_bool(allow_concurrent_memtable_write, false, + "Allow multi-writers to update mem tables in parallel."); + +DEFINE_double(experimental_mempurge_threshold, 0.0, + "Maximum estimated useful payload that triggers a " + "mempurge process to collect memtable garbage bytes."); + +DEFINE_bool(enable_write_thread_adaptive_yield, true, + "Use a yielding spin loop for brief writer thread waits."); + +// Options for StackableDB-based BlobDB +DEFINE_bool(use_blob_db, false, "[Stacked BlobDB] Use BlobDB."); + +DEFINE_uint64( + blob_db_min_blob_size, + ROCKSDB_NAMESPACE::blob_db::BlobDBOptions().min_blob_size, + "[Stacked BlobDB] Smallest blob to store in a file. Blobs " + "smaller than this will be inlined with the key in the LSM tree."); + +DEFINE_uint64( + blob_db_bytes_per_sync, + ROCKSDB_NAMESPACE::blob_db::BlobDBOptions().bytes_per_sync, + "[Stacked BlobDB] Sync blob files once per every N bytes written."); + +DEFINE_uint64(blob_db_file_size, + ROCKSDB_NAMESPACE::blob_db::BlobDBOptions().blob_file_size, + "[Stacked BlobDB] Target size of each blob file."); + +DEFINE_bool( + blob_db_enable_gc, + ROCKSDB_NAMESPACE::blob_db::BlobDBOptions().enable_garbage_collection, + "[Stacked BlobDB] Enable BlobDB garbage collection."); + +DEFINE_double( + blob_db_gc_cutoff, + ROCKSDB_NAMESPACE::blob_db::BlobDBOptions().garbage_collection_cutoff, + "[Stacked BlobDB] Cutoff ratio for BlobDB garbage collection."); + +// Options for integrated BlobDB +DEFINE_bool(allow_setting_blob_options_dynamically, false, + "[Integrated BlobDB] Allow setting blob options dynamically."); + +DEFINE_bool( + enable_blob_files, + ROCKSDB_NAMESPACE::AdvancedColumnFamilyOptions().enable_blob_files, + "[Integrated BlobDB] Enable writing large values to separate blob files."); + +DEFINE_uint64(min_blob_size, + ROCKSDB_NAMESPACE::AdvancedColumnFamilyOptions().min_blob_size, + "[Integrated BlobDB] The size of the smallest value to be stored " + "separately in a blob file."); + +DEFINE_uint64(blob_file_size, + ROCKSDB_NAMESPACE::AdvancedColumnFamilyOptions().blob_file_size, + "[Integrated BlobDB] The size limit for blob files."); + +DEFINE_string(blob_compression_type, "none", + "[Integrated BlobDB] The compression algorithm to use for large " + "values stored in blob files."); + +DEFINE_bool(enable_blob_garbage_collection, + ROCKSDB_NAMESPACE::AdvancedColumnFamilyOptions() + .enable_blob_garbage_collection, + "[Integrated BlobDB] Enable blob garbage collection."); + +DEFINE_double(blob_garbage_collection_age_cutoff, + ROCKSDB_NAMESPACE::AdvancedColumnFamilyOptions() + .blob_garbage_collection_age_cutoff, + "[Integrated BlobDB] The cutoff in terms of blob file age for " + "garbage collection."); + +DEFINE_double(blob_garbage_collection_force_threshold, + ROCKSDB_NAMESPACE::AdvancedColumnFamilyOptions() + .blob_garbage_collection_force_threshold, + "[Integrated BlobDB] The threshold for the ratio of garbage in " + "the oldest blob files for forcing garbage collection."); + +DEFINE_uint64(blob_compaction_readahead_size, + ROCKSDB_NAMESPACE::AdvancedColumnFamilyOptions() + .blob_compaction_readahead_size, + "[Integrated BlobDB] Compaction readahead for blob files."); + +DEFINE_int32( + blob_file_starting_level, + ROCKSDB_NAMESPACE::AdvancedColumnFamilyOptions().blob_file_starting_level, + "[Integrated BlobDB] Enable writing blob files during flushes and " + "compactions starting from the specified level."); + +DEFINE_bool(use_blob_cache, false, "[Integrated BlobDB] Enable blob cache."); + +DEFINE_bool( + use_shared_block_and_blob_cache, true, + "[Integrated BlobDB] Use a shared backing cache for both block " + "cache and blob cache. It only takes effect if use_blob_cache is enabled."); + +DEFINE_uint64( + blob_cache_size, 2LL * KB * KB * KB, + "[Integrated BlobDB] Number of bytes to use as a cache of blobs. It only " + "takes effect if the block and blob caches are different " + "(use_shared_block_and_blob_cache = false)."); + +DEFINE_int32(blob_cache_numshardbits, 6, + "[Integrated BlobDB] Number of shards for the blob cache is 2 ** " + "blob_cache_numshardbits. Negative means use default settings. " + "It only takes effect if blob_cache_size is greater than 0, and " + "the block and blob caches are different " + "(use_shared_block_and_blob_cache = false)."); + +DEFINE_int32(prepopulate_blob_cache, 0, + "[Integrated BlobDB] Pre-populate hot/warm blobs in blob cache. 0 " + "to disable and 1 to insert during flush."); + +DEFINE_bool(enable_tiered_storage, false, "Set last_level_temperature"); + +DEFINE_int64(preclude_last_level_data_seconds, 0, + "Preclude data from the last level. Used with tiered storage " + "feature to preclude new data from comacting to the last level."); + +DEFINE_int64( + preserve_internal_time_seconds, 0, + "Preserve internal time information which is attached to each SST."); + +static const bool FLAGS_subcompactions_dummy __attribute__((__unused__)) = + RegisterFlagValidator(&FLAGS_subcompactions, &ValidateUint32Range); + +static bool ValidateInt32Positive(const char* flagname, int32_t value) { + if (value < 0) { + fprintf(stderr, "Invalid value for --%s: %d, must be >=0\n", flagname, + value); + return false; + } + return true; +} +DEFINE_int32(reopen, 10, "Number of times database reopens"); +static const bool FLAGS_reopen_dummy __attribute__((__unused__)) = + RegisterFlagValidator(&FLAGS_reopen, &ValidateInt32Positive); + +DEFINE_double(bloom_bits, 10, + "Bloom filter bits per key. " + "Negative means use default settings."); + +DEFINE_int32( + ribbon_starting_level, 999, + "Use Bloom filter on levels below specified and Ribbon beginning on level " + "specified. Flush is considered level -1. 999 or more -> always Bloom. 0 " + "-> Ribbon except Bloom for flush. -1 -> always Ribbon."); + +DEFINE_bool(partition_filters, false, + "use partitioned filters " + "for block-based table"); + +DEFINE_bool( + optimize_filters_for_memory, + ROCKSDB_NAMESPACE::BlockBasedTableOptions().optimize_filters_for_memory, + "Minimize memory footprint of filters"); + +DEFINE_bool( + detect_filter_construct_corruption, + ROCKSDB_NAMESPACE::BlockBasedTableOptions() + .detect_filter_construct_corruption, + "Detect corruption during new Bloom Filter and Ribbon Filter construction"); + +DEFINE_int32( + index_type, + static_cast( + ROCKSDB_NAMESPACE::BlockBasedTableOptions().index_type), + "Type of block-based table index (see `enum IndexType` in table.h)"); + +DEFINE_int32( + data_block_index_type, + static_cast( + ROCKSDB_NAMESPACE::BlockBasedTableOptions().data_block_index_type), + "Index type for data blocks (see `enum DataBlockIndexType` in table.h)"); + +DEFINE_string(db, "", "Use the db with the following name."); + +DEFINE_string(secondaries_base, "", + "Use this path as the base path for secondary instances."); + +DEFINE_bool(test_secondary, false, + "If true, start an additional secondary instance which can be used " + "for verification."); + +DEFINE_string( + expected_values_dir, "", + "Dir where files containing info about the latest/historical values will " + "be stored. If provided and non-empty, the DB state will be verified " + "against values from these files after recovery. --max_key and " + "--column_family must be kept the same across invocations of this program " + "that use the same --expected_values_dir. Currently historical values are " + "only tracked when --sync_fault_injection is set. See --seed and " + "--nooverwritepercent for further requirements."); + +DEFINE_bool(verify_checksum, false, + "Verify checksum for every block read from storage"); + +DEFINE_bool(mmap_read, ROCKSDB_NAMESPACE::Options().allow_mmap_reads, + "Allow reads to occur via mmap-ing files"); + +DEFINE_bool(mmap_write, ROCKSDB_NAMESPACE::Options().allow_mmap_writes, + "Allow writes to occur via mmap-ing files"); + +DEFINE_bool(use_direct_reads, ROCKSDB_NAMESPACE::Options().use_direct_reads, + "Use O_DIRECT for reading data"); + +DEFINE_bool(use_direct_io_for_flush_and_compaction, + ROCKSDB_NAMESPACE::Options().use_direct_io_for_flush_and_compaction, + "Use O_DIRECT for writing data"); + +DEFINE_bool(mock_direct_io, false, + "Mock direct IO by not using O_DIRECT for direct IO read"); + +DEFINE_bool(statistics, false, "Create database statistics"); + +DEFINE_bool(sync, false, "Sync all writes to disk"); + +DEFINE_bool(use_fsync, false, "If true, issue fsync instead of fdatasync"); + +DEFINE_uint64(bytes_per_sync, ROCKSDB_NAMESPACE::Options().bytes_per_sync, + "If nonzero, sync SST file data incrementally after every " + "`bytes_per_sync` bytes are written"); + +DEFINE_uint64(wal_bytes_per_sync, + ROCKSDB_NAMESPACE::Options().wal_bytes_per_sync, + "If nonzero, sync WAL file data incrementally after every " + "`bytes_per_sync` bytes are written"); + +DEFINE_int32(kill_random_test, 0, + "If non-zero, kill at various points in source code with " + "probability 1/this"); +static const bool FLAGS_kill_random_test_dummy __attribute__((__unused__)) = + RegisterFlagValidator(&FLAGS_kill_random_test, &ValidateInt32Positive); + +DEFINE_string(kill_exclude_prefixes, "", + "If non-empty, kill points with prefix in the list given will be" + " skipped. Items are comma-separated."); +extern std::vector rocksdb_kill_exclude_prefixes; + +DEFINE_bool(disable_wal, false, "If true, do not write WAL for write."); + +DEFINE_uint64(recycle_log_file_num, + ROCKSDB_NAMESPACE::Options().recycle_log_file_num, + "Number of old WAL files to keep around for later recycling"); + +DEFINE_int64(target_file_size_base, + ROCKSDB_NAMESPACE::Options().target_file_size_base, + "Target level-1 file size for compaction"); + +DEFINE_int32(target_file_size_multiplier, 1, + "A multiplier to compute target level-N file size (N >= 2)"); + +DEFINE_uint64(max_bytes_for_level_base, + ROCKSDB_NAMESPACE::Options().max_bytes_for_level_base, + "Max bytes for level-1"); + +DEFINE_double(max_bytes_for_level_multiplier, 2, + "A multiplier to compute max bytes for level-N (N >= 2)"); + +DEFINE_int32(range_deletion_width, 10, + "The width of the range deletion intervals."); + +DEFINE_uint64(rate_limiter_bytes_per_sec, 0, "Set options.rate_limiter value."); + +DEFINE_bool(rate_limit_bg_reads, false, + "Use options.rate_limiter on compaction reads"); + +DEFINE_bool(rate_limit_user_ops, false, + "When true use Env::IO_USER priority level to charge internal rate " + "limiter for reads associated with user operations."); + +DEFINE_bool(rate_limit_auto_wal_flush, false, + "When true use Env::IO_USER priority level to charge internal rate " + "limiter for automatic WAL flush (`Options::manual_wal_flush` == " + "false) after the user " + "write operation."); + +DEFINE_uint64(sst_file_manager_bytes_per_sec, 0, + "Set `Options::sst_file_manager` to delete at this rate. By " + "default the deletion rate is unbounded."); + +DEFINE_uint64(sst_file_manager_bytes_per_truncate, 0, + "Set `Options::sst_file_manager` to delete in chunks of this " + "many bytes. By default whole files will be deleted."); + +DEFINE_bool(use_txn, false, + "Use TransactionDB or OptimisticTransactionDB. When " + "use_optimistic_txn == false (by default), " + "it's (Pessimistic) TransactionDB"); + +DEFINE_uint64(txn_write_policy, 0, + "The transaction write policy. Default is " + "TxnDBWritePolicy::WRITE_COMMITTED. Note that this should not be " + "changed across crashes."); + +DEFINE_bool(use_optimistic_txn, false, "Use OptimisticTransactionDB."); +DEFINE_uint64(occ_validation_policy, 1, + "Optimistic Concurrency Control Validation Policy for " + "OptimisticTransactionDB"); +DEFINE_bool(share_occ_lock_buckets, false, + "Share a pool of locks across DB instances for buckets"); +DEFINE_uint32( + occ_lock_bucket_count, 500, + "Bucket Count for shared Optimistic Concurrency Control (OCC) locks"); + +DEFINE_bool(unordered_write, false, + "Turn on the unordered_write feature. This options is currently " + "tested only in combination with use_txn=true and " + "txn_write_policy=TxnDBWritePolicy::WRITE_PREPARED."); + +DEFINE_int32(backup_one_in, 0, + "If non-zero, then CreateNewBackup() will be called once for " + "every N operations on average. 0 indicates CreateNewBackup() " + "is disabled."); + +DEFINE_uint64(backup_max_size, 100 * 1024 * 1024, + "If non-zero, skip checking backup/restore when DB size in " + "bytes exceeds this setting."); + +DEFINE_int32(checkpoint_one_in, 0, + "If non-zero, then CreateCheckpoint() will be called once for " + "every N operations on average. 0 indicates CreateCheckpoint() " + "is disabled."); + +DEFINE_int32(ingest_external_file_one_in, 0, + "If non-zero, then IngestExternalFile() will be called once for " + "every N operations on average. 0 indicates IngestExternalFile() " + "is disabled."); + +DEFINE_int32(ingest_external_file_width, 100, + "The width of the ingested external files."); + +DEFINE_int32(compact_files_one_in, 0, + "If non-zero, then CompactFiles() will be called once for every N " + "operations on average. 0 indicates CompactFiles() is disabled."); + +DEFINE_int32(compact_range_one_in, 0, + "If non-zero, then CompactRange() will be called once for every N " + "operations on average. 0 indicates CompactRange() is disabled."); + +DEFINE_int32(mark_for_compaction_one_file_in, 0, + "A `TablePropertiesCollectorFactory` will be registered, which " + "creates a `TablePropertiesCollector` with `NeedCompact()` " + "returning true once for every N files on average. 0 or negative " + "mean `NeedCompact()` always returns false."); + +DEFINE_int32(flush_one_in, 0, + "If non-zero, then Flush() will be called once for every N ops " + "on average. 0 indicates calls to Flush() are disabled."); + +DEFINE_int32(pause_background_one_in, 0, + "If non-zero, then PauseBackgroundWork()+Continue will be called " + "once for every N ops on average. 0 disables."); + +DEFINE_int32(compact_range_width, 10000, + "The width of the ranges passed to CompactRange()."); + +DEFINE_int32(acquire_snapshot_one_in, 0, + "If non-zero, then acquires a snapshot once every N operations on " + "average."); + +DEFINE_bool(compare_full_db_state_snapshot, false, + "If set we compare state of entire db (in one of the threads) with" + "each snapshot."); + +DEFINE_uint64(snapshot_hold_ops, 0, + "If non-zero, then releases snapshots N operations after they're " + "acquired."); + +DEFINE_bool(long_running_snapshots, false, + "If set, hold on some some snapshots for much longer time."); + +DEFINE_bool(use_multiget, false, + "If set, use the batched MultiGet API for reads"); + +DEFINE_bool(use_get_entity, false, "If set, use the GetEntity API for reads"); + +DEFINE_bool(use_multi_get_entity, false, + "If set, use the MultiGetEntity API for reads"); + +static bool ValidateInt32Percent(const char* flagname, int32_t value) { + if (value < 0 || value > 100) { + fprintf(stderr, "Invalid value for --%s: %d, 0<= pct <=100 \n", flagname, + value); + return false; + } + return true; +} + +DEFINE_int32(readpercent, 10, + "Ratio of reads to total workload (expressed as a percentage)"); +static const bool FLAGS_readpercent_dummy __attribute__((__unused__)) = + RegisterFlagValidator(&FLAGS_readpercent, &ValidateInt32Percent); + +DEFINE_int32(prefixpercent, 20, + "Ratio of prefix iterators to total workload (expressed as a" + " percentage)"); +static const bool FLAGS_prefixpercent_dummy __attribute__((__unused__)) = + RegisterFlagValidator(&FLAGS_prefixpercent, &ValidateInt32Percent); + +DEFINE_int32(writepercent, 45, + "Ratio of writes to total workload (expressed as a percentage)"); +static const bool FLAGS_writepercent_dummy __attribute__((__unused__)) = + RegisterFlagValidator(&FLAGS_writepercent, &ValidateInt32Percent); + +DEFINE_int32(delpercent, 15, + "Ratio of deletes to total workload (expressed as a percentage)"); +static const bool FLAGS_delpercent_dummy __attribute__((__unused__)) = + RegisterFlagValidator(&FLAGS_delpercent, &ValidateInt32Percent); + +DEFINE_int32(delrangepercent, 0, + "Ratio of range deletions to total workload (expressed as a " + "percentage). Cannot be used with test_batches_snapshots"); +static const bool FLAGS_delrangepercent_dummy __attribute__((__unused__)) = + RegisterFlagValidator(&FLAGS_delrangepercent, &ValidateInt32Percent); + +DEFINE_int32(nooverwritepercent, 60, + "Ratio of keys without overwrite to total workload (expressed as " + "a percentage). When --expected_values_dir is nonempty, must " + "keep this value constant across invocations."); +static const bool FLAGS_nooverwritepercent_dummy __attribute__((__unused__)) = + RegisterFlagValidator(&FLAGS_nooverwritepercent, &ValidateInt32Percent); + +DEFINE_int32(iterpercent, 10, + "Ratio of iterations to total workload" + " (expressed as a percentage)"); +static const bool FLAGS_iterpercent_dummy __attribute__((__unused__)) = + RegisterFlagValidator(&FLAGS_iterpercent, &ValidateInt32Percent); + +DEFINE_uint64(num_iterations, 10, "Number of iterations per MultiIterate run"); +static const bool FLAGS_num_iterations_dummy __attribute__((__unused__)) = + RegisterFlagValidator(&FLAGS_num_iterations, &ValidateUint32Range); + +DEFINE_int32( + customopspercent, 0, + "Ratio of custom operations to total workload (expressed as a percentage)"); + +DEFINE_string(compression_type, "snappy", + "Algorithm to use to compress the database"); + +DEFINE_int32(compression_max_dict_bytes, 0, + "Maximum size of dictionary used to prime the compression " + "library."); + +DEFINE_int32(compression_zstd_max_train_bytes, 0, + "Maximum size of training data passed to zstd's dictionary " + "trainer."); + +DEFINE_int32(compression_parallel_threads, 1, + "Number of threads for parallel compression."); + +DEFINE_uint64(compression_max_dict_buffer_bytes, 0, + "Buffering limit for SST file data to sample for dictionary " + "compression."); + +DEFINE_bool( + compression_use_zstd_dict_trainer, true, + "Use zstd's trainer to generate dictionary. If the options is false, " + "zstd's finalizeDictionary() API is used to generate dictionary. " + "ZSTD 1.4.5+ is required. If ZSTD 1.4.5+ is not linked with the binary, " + "this flag will have the default value true."); + +DEFINE_string(bottommost_compression_type, "disable", + "Algorithm to use to compress bottommost level of the database. " + "\"disable\" means disabling the feature"); + +DEFINE_string(checksum_type, "kCRC32c", "Algorithm to use to checksum blocks"); + +DEFINE_string(env_uri, "", + "URI for env lookup. Mutually exclusive with --fs_uri"); + +DEFINE_string(fs_uri, "", + "URI for registry Filesystem lookup. Mutually exclusive" + " with --env_uri." + " Creates a default environment with the specified filesystem."); + +DEFINE_uint64(ops_per_thread, 1200000, "Number of operations per thread."); +static const bool FLAGS_ops_per_thread_dummy __attribute__((__unused__)) = + RegisterFlagValidator(&FLAGS_ops_per_thread, &ValidateUint32Range); + +DEFINE_uint64(log2_keys_per_lock, 2, "Log2 of number of keys per lock"); +static const bool FLAGS_log2_keys_per_lock_dummy __attribute__((__unused__)) = + RegisterFlagValidator(&FLAGS_log2_keys_per_lock, &ValidateUint32Range); + +DEFINE_uint64(max_manifest_file_size, 16384, "Maximum size of a MANIFEST file"); + +DEFINE_bool(in_place_update, false, "On true, does inplace update in memtable"); + +DEFINE_string(memtablerep, "skip_list", ""); + +inline static bool ValidatePrefixSize(const char* flagname, int32_t value) { + if (value < -1 || value > 8) { + fprintf(stderr, "Invalid value for --%s: %d. -1 <= PrefixSize <= 8\n", + flagname, value); + return false; + } + return true; +} +DEFINE_int32(prefix_size, 7, + "Control the prefix size for HashSkipListRep. " + "-1 is disabled."); +static const bool FLAGS_prefix_size_dummy __attribute__((__unused__)) = + RegisterFlagValidator(&FLAGS_prefix_size, &ValidatePrefixSize); + +DEFINE_bool(use_merge, false, + "On true, replaces all writes with a Merge " + "that behaves like a Put"); + +DEFINE_uint32(use_put_entity_one_in, 0, + "If greater than zero, PutEntity will be used once per every N " + "write ops on average."); + +DEFINE_bool(use_full_merge_v1, false, + "On true, use a merge operator that implement the deprecated " + "version of FullMerge"); + +DEFINE_int32(sync_wal_one_in, 0, + "If non-zero, then SyncWAL() will be called once for every N ops " + "on average. 0 indicates that calls to SyncWAL() are disabled."); + +DEFINE_bool(avoid_unnecessary_blocking_io, + ROCKSDB_NAMESPACE::Options().avoid_unnecessary_blocking_io, + "If true, some expensive cleaning up operations will be moved from " + "user reads to high-pri background threads."); + +DEFINE_bool(write_dbid_to_manifest, + ROCKSDB_NAMESPACE::Options().write_dbid_to_manifest, + "Write DB_ID to manifest"); + +DEFINE_bool(avoid_flush_during_recovery, + ROCKSDB_NAMESPACE::Options().avoid_flush_during_recovery, + "Avoid flush during recovery"); + +DEFINE_uint64(max_write_batch_group_size_bytes, + ROCKSDB_NAMESPACE::Options().max_write_batch_group_size_bytes, + "Max write batch group size"); + +DEFINE_bool(level_compaction_dynamic_level_bytes, + ROCKSDB_NAMESPACE::Options().level_compaction_dynamic_level_bytes, + "Use dynamic level"); + +DEFINE_int32(verify_checksum_one_in, 0, + "If non-zero, then DB::VerifyChecksum() will be called to do" + " checksum verification of all the files in the database once for" + " every N ops on average. 0 indicates that calls to" + " VerifyChecksum() are disabled."); +DEFINE_int32(verify_db_one_in, 0, + "If non-zero, call VerifyDb() once for every N ops. 0 indicates " + "that VerifyDb() will not be called in OperateDb(). Note that " + "enabling this can slow down tests."); + +DEFINE_int32(continuous_verification_interval, 1000, + "While test is running, verify db every N milliseconds. 0 " + "disables continuous verification."); + +DEFINE_int32(approximate_size_one_in, 64, + "If non-zero, DB::GetApproximateSizes() will be called against" + " random key ranges."); + +DEFINE_int32(read_fault_one_in, 1000, + "On non-zero, enables fault injection on read"); + +DEFINE_int32(get_property_one_in, 1000, + "If non-zero, then DB::GetProperty() will be called to get various" + " properties for every N ops on average. 0 indicates that" + " GetProperty() will be not be called."); + +DEFINE_bool(sync_fault_injection, false, + "If true, FaultInjectionTestFS will be used for write operations, " + "and unsynced data in DB will lost after crash. In such a case we " + "track DB changes in a trace file (\"*.trace\") in " + "--expected_values_dir for verifying there are no holes in the " + "recovered data."); + +DEFINE_bool(best_efforts_recovery, false, + "If true, use best efforts recovery."); +DEFINE_bool(skip_verifydb, false, + "If true, skip VerifyDb() calls and Get()/Iterator verifications" + "against expected state."); + +DEFINE_bool(enable_compaction_filter, false, + "If true, configures a compaction filter that returns a kRemove " + "decision for deleted keys."); + +DEFINE_bool(paranoid_file_checks, true, + "After writing every SST file, reopen it and read all the keys " + "and validate checksums"); + +DEFINE_bool(fail_if_options_file_error, false, + "Fail operations that fail to detect or properly persist options " + "file."); + +DEFINE_uint64(batch_protection_bytes_per_key, 0, + "If nonzero, enables integrity protection in `WriteBatch` at the " + "specified number of bytes per key. Currently the only supported " + "nonzero value is eight."); + +DEFINE_uint32( + memtable_protection_bytes_per_key, 0, + "If nonzero, enables integrity protection in memtable entries at the " + "specified number of bytes per key. Currently the supported " + "nonzero values are 1, 2, 4 and 8."); + +DEFINE_uint32(block_protection_bytes_per_key, 0, + "If nonzero, enables integrity protection in blocks at the " + "specified number of bytes per key. Currently the supported " + "nonzero values are 1, 2, 4 and 8."); + +DEFINE_string(file_checksum_impl, "none", + "Name of an implementation for file_checksum_gen_factory, or " + "\"none\" for null."); + +DEFINE_int32(write_fault_one_in, 0, + "On non-zero, enables fault injection on write"); + +DEFINE_uint64(user_timestamp_size, 0, + "Number of bytes for a user-defined timestamp. Currently, only " + "8-byte is supported"); + +DEFINE_int32(open_metadata_write_fault_one_in, 0, + "On non-zero, enables fault injection on file metadata write " + "during DB reopen."); + +DEFINE_string(secondary_cache_uri, "", + "Full URI for creating a customized secondary cache object"); +DEFINE_int32(secondary_cache_fault_one_in, 0, + "On non-zero, enables fault injection in secondary cache inserts" + " and lookups"); +DEFINE_int32(open_write_fault_one_in, 0, + "On non-zero, enables fault injection on file writes " + "during DB reopen."); +DEFINE_int32(open_read_fault_one_in, 0, + "On non-zero, enables fault injection on file reads " + "during DB reopen."); +DEFINE_int32(injest_error_severity, 1, + "The severity of the injested IO Error. 1 is soft error (e.g. " + "retryable error), 2 is fatal error, and the default is " + "retryable error."); +DEFINE_int32(prepopulate_block_cache, + static_cast(ROCKSDB_NAMESPACE::BlockBasedTableOptions:: + PrepopulateBlockCache::kDisable), + "Options related to cache warming (see `enum " + "PrepopulateBlockCache` in table.h)"); + +DEFINE_bool(two_write_queues, false, + "Set to true to enable two write queues. Default: false"); + +DEFINE_bool(use_only_the_last_commit_time_batch_for_recovery, false, + "If true, the commit-time write batch will not be immediately " + "inserted into the memtables. Default: false"); + +DEFINE_uint64( + wp_snapshot_cache_bits, 7ull, + "Number of bits to represent write-prepared transaction db's snapshot " + "cache. Default: 7 (128 entries)"); + +DEFINE_uint64(wp_commit_cache_bits, 23ull, + "Number of bits to represent write-prepared transaction db's " + "commit cache. Default: 23 (8M entries)"); + +DEFINE_bool(adaptive_readahead, false, + "Carry forward internal auto readahead size from one file to next " + "file at each level during iteration"); +DEFINE_bool( + async_io, false, + "Does asynchronous prefetching when internal auto readahead is enabled"); + +DEFINE_string(wal_compression, "none", + "Algorithm to use for WAL compression. none to disable."); + +DEFINE_bool( + verify_sst_unique_id_in_manifest, false, + "Enable DB options `verify_sst_unique_id_in_manifest`, if true, during " + "DB-open try verifying the SST unique id between MANIFEST and SST " + "properties."); + +DEFINE_int32( + create_timestamped_snapshot_one_in, 0, + "On non-zero, create timestamped snapshots upon transaction commits."); + +DEFINE_bool(allow_data_in_errors, + ROCKSDB_NAMESPACE::Options().allow_data_in_errors, + "If true, allow logging data, e.g. key, value in LOG files."); + +DEFINE_bool(enable_thread_tracking, + ROCKSDB_NAMESPACE::Options().enable_thread_tracking, + "If true, the status of the threads involved in this DB will be " + "tracked and available via GetThreadList() API."); + +DEFINE_int32(verify_iterator_with_expected_state_one_in, 0, + "If non-zero, when TestIterate() is to be called, there is a " + "1/verify_iterator_with_expected_state_one_in " + "chance that the iterator is verified against the expected state " + "file, instead of comparing keys between two iterators."); + +DEFINE_uint64(readahead_size, 0, "Iterator readahead size"); +DEFINE_uint64(initial_auto_readahead_size, 0, + "Initial auto readahead size for prefetching during Iteration"); +DEFINE_uint64(max_auto_readahead_size, 0, + "Max auto readahead size for prefetching during Iteration"); +DEFINE_uint64( + num_file_reads_for_auto_readahead, 0, + "Num of sequential reads to enable auto prefetching during Iteration"); + +DEFINE_bool( + preserve_unverified_changes, false, + "DB files of the current run will all be preserved in `FLAGS_db`. DB files " + "from the last run will be preserved in `FLAGS_db/unverified` until the " + "first verification succeeds. Expected state files from the last run will " + "be preserved similarly under `FLAGS_expected_values_dir/unverified` when " + "`--expected_values_dir` is nonempty."); + +DEFINE_uint64(stats_dump_period_sec, + ROCKSDB_NAMESPACE::Options().stats_dump_period_sec, + "Gap between printing stats to log in seconds"); + +DEFINE_bool(use_io_uring, false, "Enable the use of IO uring on Posix"); +extern "C" bool RocksDbIOUringEnable() { return FLAGS_use_io_uring; } + +DEFINE_uint32(memtable_max_range_deletions, 0, + "If nonzero, RocksDB will try to flush the current memtable" + "after the number of range deletions is >= this limit"); + +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db_stress_tool/db_stress_listener.cc b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_listener.cc new file mode 100644 index 0000000..e2838c5 --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_listener.cc @@ -0,0 +1,189 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db_stress_tool/db_stress_listener.h" + +#include + +#include "file/file_util.h" +#include "rocksdb/file_system.h" +#include "util/coding_lean.h" + +namespace ROCKSDB_NAMESPACE { + +#ifdef GFLAGS + +// TODO: consider using expected_values_dir instead, but this is more +// convenient for now. +UniqueIdVerifier::UniqueIdVerifier(const std::string& db_name, Env* env) + : path_(db_name + "/.unique_ids") { + // We expect such a small number of files generated during this test + // (thousands?), checking full 192-bit IDs for uniqueness is a very + // weak check. For a stronger check, we pick a specific 64-bit + // subsequence from the ID to check for uniqueness. All bits of the + // ID should be high quality, and 64 bits should be unique with + // very good probability for the quantities in this test. + offset_ = Random::GetTLSInstance()->Uniform(17); // 0 to 16 + + const std::shared_ptr fs = env->GetFileSystem(); + IOOptions opts; + + Status st = fs->CreateDirIfMissing(db_name, opts, nullptr); + if (!st.ok()) { + fprintf(stderr, "Failed to create directory %s: %s\n", db_name.c_str(), + st.ToString().c_str()); + exit(1); + } + + // Avoid relying on ReopenWritableFile which is not supported by all + // file systems. Create a new file and copy the old file contents to it. + std::string tmp_path = path_ + ".tmp"; + st = fs->FileExists(tmp_path, opts, /*dbg*/ nullptr); + if (st.IsNotFound()) { + st = fs->RenameFile(path_, tmp_path, opts, /*dbg*/ nullptr); + // Either it should succeed or fail because src path doesn't exist + assert(st.ok() || st.IsPathNotFound()); + } else { + // If path_ and tmp_path both exist, retain tmp_path as its + // guaranteed to be more complete. The order of operations are - + // 1. Rename path_ to tmp_path + // 2. Parse tmp_path contents + // 3. Create path_ + // 4. Copy tmp_path contents to path_ + // 5. Delete tmp_path + st = fs->DeleteFile(path_, opts, /*dbg*/ nullptr); + assert(st.ok() || st.IsPathNotFound()); + } + + uint64_t size = 0; + { + std::unique_ptr reader; + Status s = fs->NewSequentialFile(tmp_path, FileOptions(), &reader, + /*dbg*/ nullptr); + if (s.ok()) { + // Load from file + std::string id(24U, '\0'); + Slice result; + for (;;) { + s = reader->Read(id.size(), opts, &result, &id[0], /*dbg*/ nullptr); + if (!s.ok()) { + fprintf(stderr, "Error reading unique id file: %s\n", + s.ToString().c_str()); + assert(false); + } + if (result.size() < id.size()) { + // EOF + if (result.size() != 0) { + // Corrupt file. Not a DB bug but could happen if OS doesn't provide + // good guarantees on process crash. + fprintf(stdout, "Warning: clearing corrupt unique id file\n"); + id_set_.clear(); + reader.reset(); + s = fs->DeleteFile(tmp_path, opts, /*dbg*/ nullptr); + assert(s.ok()); + size = 0; + } + break; + } + size += 24U; + VerifyNoWrite(id); + } + } else { + // Newly created is ok. + // But FileSystem doesn't tell us whether non-existence was the cause of + // the failure. (Issue #9021) + Status s2 = fs->FileExists(tmp_path, opts, /*dbg*/ nullptr); + if (!s2.IsNotFound()) { + fprintf(stderr, "Error opening unique id file: %s\n", + s.ToString().c_str()); + assert(false); + } + size = 0; + } + } + fprintf(stdout, "(Re-)verified %zu unique IDs\n", id_set_.size()); + + std::unique_ptr file_writer; + st = fs->NewWritableFile(path_, FileOptions(), &file_writer, /*dbg*/ nullptr); + if (!st.ok()) { + fprintf(stderr, "Error creating the unique ids file: %s\n", + st.ToString().c_str()); + assert(false); + } + data_file_writer_.reset( + new WritableFileWriter(std::move(file_writer), path_, FileOptions())); + + if (size > 0) { + st = CopyFile(fs.get(), tmp_path, data_file_writer_, size, + /*use_fsync*/ true, /*io_tracer*/ nullptr, + /*temparature*/ Temperature::kHot); + if (!st.ok()) { + fprintf(stderr, "Error copying contents of old unique id file: %s\n", + st.ToString().c_str()); + assert(false); + } + } + st = fs->DeleteFile(tmp_path, opts, /*dbg*/ nullptr); + assert(st.ok() || st.IsPathNotFound()); +} + +UniqueIdVerifier::~UniqueIdVerifier() { + IOStatus s = data_file_writer_->Close(); + assert(s.ok()); +} + +void UniqueIdVerifier::VerifyNoWrite(const std::string& id) { + assert(id.size() == 24); + bool is_new = id_set_.insert(DecodeFixed64(&id[offset_])).second; + if (!is_new) { + fprintf(stderr, + "Duplicate partial unique ID found (offset=%zu, count=%zu)\n", + offset_, id_set_.size()); + assert(false); + } +} + +void UniqueIdVerifier::Verify(const std::string& id) { + assert(id.size() == 24); + std::lock_guard lock(mutex_); + // If we accumulate more than ~4 million IDs, there would be > 1 in 1M + // natural chance of collision. Thus, simply stop checking at that point. + if (id_set_.size() >= 4294967) { + return; + } + IOStatus s = data_file_writer_->Append(Slice(id)); + if (!s.ok()) { + fprintf(stderr, "Error writing to unique id file: %s\n", + s.ToString().c_str()); + assert(false); + } + s = data_file_writer_->Flush(); + if (!s.ok()) { + fprintf(stderr, "Error flushing unique id file: %s\n", + s.ToString().c_str()); + assert(false); + } + VerifyNoWrite(id); +} + +void DbStressListener::VerifyTableFileUniqueId( + const TableProperties& new_file_properties, const std::string& file_path) { + // Verify unique ID + std::string id; + // Unit tests verify that GetUniqueIdFromTableProperties returns just a + // substring of this, and we're only going to pull out 64 bits, so using + // GetExtendedUniqueIdFromTableProperties is arguably stronger testing here. + Status s = GetExtendedUniqueIdFromTableProperties(new_file_properties, &id); + if (!s.ok()) { + fprintf(stderr, "Error getting SST unique id for %s: %s\n", + file_path.c_str(), s.ToString().c_str()); + assert(false); + } + unique_ids_.Verify(id); +} + +#endif // GFLAGS + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db_stress_tool/db_stress_listener.h b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_listener.h new file mode 100644 index 0000000..97bbdae --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_listener.h @@ -0,0 +1,269 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#ifdef GFLAGS +#pragma once + +#include +#include + +#include "file/filename.h" +#include "file/writable_file_writer.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/file_system.h" +#include "rocksdb/listener.h" +#include "rocksdb/table_properties.h" +#include "rocksdb/unique_id.h" +#include "util/gflags_compat.h" +#include "util/random.h" + +DECLARE_int32(compact_files_one_in); + +namespace ROCKSDB_NAMESPACE { + +// Verify across process executions that all seen IDs are unique +class UniqueIdVerifier { + public: + explicit UniqueIdVerifier(const std::string& db_name, Env* env); + ~UniqueIdVerifier(); + + void Verify(const std::string& id); + + private: + void VerifyNoWrite(const std::string& id); + + private: + std::mutex mutex_; + // IDs persisted to a hidden file inside DB dir + std::string path_; + std::unique_ptr data_file_writer_; + // Starting byte for which 8 bytes to check in memory within 24 byte ID + size_t offset_; + // Working copy of the set of 8 byte pieces + std::unordered_set id_set_; +}; + +class DbStressListener : public EventListener { + public: + DbStressListener(const std::string& db_name, + const std::vector& db_paths, + const std::vector& column_families, + Env* env) + : db_name_(db_name), + db_paths_(db_paths), + column_families_(column_families), + num_pending_file_creations_(0), + unique_ids_(db_name, env) {} + + const char* Name() const override { return kClassName(); } + static const char* kClassName() { return "DBStressListener"; } + + ~DbStressListener() override { assert(num_pending_file_creations_ == 0); } + void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& info) override { + assert(IsValidColumnFamilyName(info.cf_name)); + VerifyFilePath(info.file_path); + // pretending doing some work here + RandomSleep(); + } + + void OnFlushBegin(DB* /*db*/, + const FlushJobInfo& /*flush_job_info*/) override { + RandomSleep(); + } + + void OnTableFileDeleted(const TableFileDeletionInfo& /*info*/) override { + RandomSleep(); + } + + void OnCompactionBegin(DB* /*db*/, const CompactionJobInfo& /*ci*/) override { + RandomSleep(); + } + + void OnCompactionCompleted(DB* /*db*/, const CompactionJobInfo& ci) override { + assert(IsValidColumnFamilyName(ci.cf_name)); + assert(ci.input_files.size() + ci.output_files.size() > 0U); + for (const auto& file_path : ci.input_files) { + VerifyFilePath(file_path); + } + for (const auto& file_path : ci.output_files) { + VerifyFilePath(file_path); + } + // pretending doing some work here + RandomSleep(); + } + + void OnTableFileCreationStarted( + const TableFileCreationBriefInfo& /*info*/) override { + ++num_pending_file_creations_; + } + + void OnTableFileCreated(const TableFileCreationInfo& info) override { + assert(info.db_name == db_name_); + assert(IsValidColumnFamilyName(info.cf_name)); + assert(info.job_id > 0 || FLAGS_compact_files_one_in > 0); + if (info.status.ok()) { + assert(info.file_size > 0); + VerifyFilePath(info.file_path); + assert(info.table_properties.data_size > 0 || + info.table_properties.num_range_deletions > 0); + assert(info.table_properties.raw_key_size > 0); + assert(info.table_properties.num_entries > 0); + VerifyTableFileUniqueId(info.table_properties, info.file_path); + } + --num_pending_file_creations_; + } + + void OnMemTableSealed(const MemTableInfo& /*info*/) override { + RandomSleep(); + } + + void OnColumnFamilyHandleDeletionStarted( + ColumnFamilyHandle* /*handle*/) override { + RandomSleep(); + } + + void OnExternalFileIngested(DB* /*db*/, + const ExternalFileIngestionInfo& info) override { + RandomSleep(); + // Here we assume that each generated external file is ingested + // exactly once (or thrown away in case of crash) + VerifyTableFileUniqueId(info.table_properties, info.internal_file_path); + } + + void OnBackgroundError(BackgroundErrorReason /* reason */, + Status* /* bg_error */) override { + RandomSleep(); + } + + void OnStallConditionsChanged(const WriteStallInfo& /*info*/) override { + RandomSleep(); + } + + void OnFileReadFinish(const FileOperationInfo& info) override { + // Even empty callback is valuable because sometimes some locks are + // released in order to make the callback. + + // Sleep carefully here as it is a frequent operation and we don't want + // to slow down the tests. We always sleep when the read is large. + // When read is small, sleep in a small chance. + size_t length_read = info.length; + if (length_read >= 1000000 || Random::GetTLSInstance()->OneIn(1000)) { + RandomSleep(); + } + } + + void OnFileWriteFinish(const FileOperationInfo& info) override { + // Even empty callback is valuable because sometimes some locks are + // released in order to make the callback. + + // Sleep carefully here as it is a frequent operation and we don't want + // to slow down the tests. When the write is large, always sleep. + // Otherwise, sleep in a relatively small chance. + size_t length_write = info.length; + if (length_write >= 1000000 || Random::GetTLSInstance()->OneIn(64)) { + RandomSleep(); + } + } + + bool ShouldBeNotifiedOnFileIO() override { + RandomSleep(); + return static_cast(Random::GetTLSInstance()->OneIn(1)); + } + + void OnErrorRecoveryBegin(BackgroundErrorReason /* reason */, + Status /* bg_error */, + bool* /* auto_recovery */) override { + RandomSleep(); + } + + void OnErrorRecoveryCompleted(Status /* old_bg_error */) override { + RandomSleep(); + } + + protected: + bool IsValidColumnFamilyName(const std::string& cf_name) const { + if (cf_name == kDefaultColumnFamilyName) { + return true; + } + // The column family names in the stress tests are numbers. + for (size_t i = 0; i < cf_name.size(); ++i) { + if (cf_name[i] < '0' || cf_name[i] > '9') { + return false; + } + } + return true; + } + + void VerifyFileDir(const std::string& file_dir) { +#ifndef NDEBUG + if (db_name_ == file_dir) { + return; + } + for (const auto& db_path : db_paths_) { + if (db_path.path == file_dir) { + return; + } + } + for (auto& cf : column_families_) { + for (const auto& cf_path : cf.options.cf_paths) { + if (cf_path.path == file_dir) { + return; + } + } + } + assert(false); +#else + (void)file_dir; +#endif // !NDEBUG + } + + void VerifyFileName(const std::string& file_name) { +#ifndef NDEBUG + uint64_t file_number; + FileType file_type; + bool result = ParseFileName(file_name, &file_number, &file_type); + assert(result); + assert(file_type == kTableFile); +#else + (void)file_name; +#endif // !NDEBUG + } + + void VerifyFilePath(const std::string& file_path) { +#ifndef NDEBUG + size_t pos = file_path.find_last_of("/"); + if (pos == std::string::npos) { + VerifyFileName(file_path); + } else { + if (pos > 0) { + VerifyFileDir(file_path.substr(0, pos)); + } + VerifyFileName(file_path.substr(pos)); + } +#else + (void)file_path; +#endif // !NDEBUG + } + + // Unique id is verified using the TableProperties. file_path is only used + // for reporting. + void VerifyTableFileUniqueId(const TableProperties& new_file_properties, + const std::string& file_path); + + void RandomSleep() { + std::this_thread::sleep_for( + std::chrono::microseconds(Random::GetTLSInstance()->Uniform(5000))); + } + + private: + std::string db_name_; + std::vector db_paths_; + std::vector column_families_; + std::atomic num_pending_file_creations_; + UniqueIdVerifier unique_ids_; +}; +} // namespace ROCKSDB_NAMESPACE +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db_stress_tool/db_stress_shared_state.cc b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_shared_state.cc new file mode 100644 index 0000000..a27f6ac --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_shared_state.cc @@ -0,0 +1,17 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// + +#ifdef GFLAGS +#include "db_stress_tool/db_stress_shared_state.h" + +namespace ROCKSDB_NAMESPACE { +thread_local bool SharedState::ignore_read_error; +} // namespace ROCKSDB_NAMESPACE +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db_stress_tool/db_stress_shared_state.h b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_shared_state.h new file mode 100644 index 0000000..604e8c6 --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_shared_state.h @@ -0,0 +1,444 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors + +#ifdef GFLAGS +#pragma once + +#include "db_stress_tool/db_stress_stat.h" +#include "db_stress_tool/expected_state.h" +// SyncPoint is not supported in Released Windows Mode. +#if !(defined NDEBUG) || !defined(OS_WIN) +#include "test_util/sync_point.h" +#endif // !(defined NDEBUG) || !defined(OS_WIN) +#include "util/gflags_compat.h" + +DECLARE_uint64(seed); +DECLARE_int64(max_key); +DECLARE_uint64(log2_keys_per_lock); +DECLARE_int32(threads); +DECLARE_int32(column_families); +DECLARE_int32(nooverwritepercent); +DECLARE_string(expected_values_dir); +DECLARE_int32(clear_column_family_one_in); +DECLARE_bool(test_batches_snapshots); +DECLARE_int32(compaction_thread_pool_adjust_interval); +DECLARE_int32(continuous_verification_interval); +DECLARE_int32(read_fault_one_in); +DECLARE_int32(write_fault_one_in); +DECLARE_int32(open_metadata_write_fault_one_in); +DECLARE_int32(open_write_fault_one_in); +DECLARE_int32(open_read_fault_one_in); + +DECLARE_int32(injest_error_severity); + +namespace ROCKSDB_NAMESPACE { +class StressTest; + +// State shared by all concurrent executions of the same benchmark. +class SharedState { + public: + // Errors when reading filter blocks are ignored, so we use a thread + // local variable updated via sync points to keep track of errors injected + // while reading filter blocks in order to ignore the Get/MultiGet result + // for those calls + static thread_local bool ignore_read_error; + + SharedState(Env* /*env*/, StressTest* stress_test) + : cv_(&mu_), + seed_(static_cast(FLAGS_seed)), + max_key_(FLAGS_max_key), + log2_keys_per_lock_(static_cast(FLAGS_log2_keys_per_lock)), + num_threads_(0), + num_initialized_(0), + num_populated_(0), + vote_reopen_(0), + num_done_(0), + start_(false), + start_verify_(false), + num_bg_threads_(0), + should_stop_bg_thread_(false), + bg_thread_finished_(0), + stress_test_(stress_test), + verification_failure_(false), + should_stop_test_(false), + no_overwrite_ids_(GenerateNoOverwriteIds()), + expected_state_manager_(nullptr), + printing_verification_results_(false), + start_timestamp_(Env::Default()->NowNanos()) { + Status status; + // TODO: We should introduce a way to explicitly disable verification + // during shutdown. When that is disabled and FLAGS_expected_values_dir + // is empty (disabling verification at startup), we can skip tracking + // expected state. Only then should we permit bypassing the below feature + // compatibility checks. + if (!FLAGS_expected_values_dir.empty()) { + if (!std::atomic{}.is_lock_free()) { + status = Status::InvalidArgument( + "Cannot use --expected_values_dir on platforms without lock-free " + "std::atomic"); + } + if (status.ok() && FLAGS_clear_column_family_one_in > 0) { + status = Status::InvalidArgument( + "Cannot use --expected_values_dir on when " + "--clear_column_family_one_in is greater than zero."); + } + } + if (status.ok()) { + if (FLAGS_expected_values_dir.empty()) { + expected_state_manager_.reset( + new AnonExpectedStateManager(FLAGS_max_key, FLAGS_column_families)); + } else { + expected_state_manager_.reset(new FileExpectedStateManager( + FLAGS_max_key, FLAGS_column_families, FLAGS_expected_values_dir)); + } + status = expected_state_manager_->Open(); + } + if (!status.ok()) { + fprintf(stderr, "Failed setting up expected state with error: %s\n", + status.ToString().c_str()); + exit(1); + } + + if (FLAGS_test_batches_snapshots) { + fprintf(stdout, "No lock creation because test_batches_snapshots set\n"); + return; + } + + long num_locks = static_cast(max_key_ >> log2_keys_per_lock_); + if (max_key_ & ((1 << log2_keys_per_lock_) - 1)) { + num_locks++; + } + fprintf(stdout, "Creating %ld locks\n", num_locks * FLAGS_column_families); + key_locks_.resize(FLAGS_column_families); + + for (int i = 0; i < FLAGS_column_families; ++i) { + key_locks_[i].reset(new port::Mutex[num_locks]); + } + if (FLAGS_read_fault_one_in) { +#ifdef NDEBUG + // Unsupported in release mode because it relies on + // `IGNORE_STATUS_IF_ERROR` to distinguish faults not expected to lead to + // failure. + fprintf(stderr, + "Cannot set nonzero value for --read_fault_one_in in " + "release mode."); + exit(1); +#else // NDEBUG + SyncPoint::GetInstance()->SetCallBack("FaultInjectionIgnoreError", + IgnoreReadErrorCallback); + SyncPoint::GetInstance()->EnableProcessing(); +#endif // NDEBUG + } + } + + ~SharedState() { +#ifndef NDEBUG + if (FLAGS_read_fault_one_in) { + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); + } +#endif + } + + port::Mutex* GetMutex() { return &mu_; } + + port::CondVar* GetCondVar() { return &cv_; } + + StressTest* GetStressTest() const { return stress_test_; } + + int64_t GetMaxKey() const { return max_key_; } + + uint32_t GetNumThreads() const { return num_threads_; } + + void SetThreads(int num_threads) { num_threads_ = num_threads; } + + void IncInitialized() { num_initialized_++; } + + void IncOperated() { num_populated_++; } + + void IncDone() { num_done_++; } + + void IncVotedReopen() { vote_reopen_ = (vote_reopen_ + 1) % num_threads_; } + + bool AllInitialized() const { return num_initialized_ >= num_threads_; } + + bool AllOperated() const { return num_populated_ >= num_threads_; } + + bool AllDone() const { return num_done_ >= num_threads_; } + + bool AllVotedReopen() { return (vote_reopen_ == 0); } + + void SetStart() { start_ = true; } + + void SetStartVerify() { start_verify_ = true; } + + bool Started() const { return start_; } + + bool VerifyStarted() const { return start_verify_; } + + void SetVerificationFailure() { verification_failure_.store(true); } + + bool HasVerificationFailedYet() const { return verification_failure_.load(); } + + void SetShouldStopTest() { should_stop_test_.store(true); } + + bool ShouldStopTest() const { return should_stop_test_.load(); } + + // Returns a lock covering `key` in `cf`. + port::Mutex* GetMutexForKey(int cf, int64_t key) { + return &key_locks_[cf][key >> log2_keys_per_lock_]; + } + + // Acquires locks for all keys in `cf`. + void LockColumnFamily(int cf) { + for (int i = 0; i < max_key_ >> log2_keys_per_lock_; ++i) { + key_locks_[cf][i].Lock(); + } + } + + // Releases locks for all keys in `cf`. + void UnlockColumnFamily(int cf) { + for (int i = 0; i < max_key_ >> log2_keys_per_lock_; ++i) { + key_locks_[cf][i].Unlock(); + } + } + + // Returns a collection of mutex locks covering the key range [start, end) in + // `cf`. + std::vector> GetLocksForKeyRange(int cf, + int64_t start, + int64_t end) { + std::vector> range_locks; + + if (start >= end) { + return range_locks; + } + + const int64_t start_idx = start >> log2_keys_per_lock_; + + int64_t end_idx = end >> log2_keys_per_lock_; + if ((end & ((1 << log2_keys_per_lock_) - 1)) == 0) { + --end_idx; + } + + for (int64_t idx = start_idx; idx <= end_idx; ++idx) { + range_locks.emplace_back( + std::make_unique(&key_locks_[cf][idx])); + } + + return range_locks; + } + + Status SaveAtAndAfter(DB* db) { + return expected_state_manager_->SaveAtAndAfter(db); + } + + bool HasHistory() { return expected_state_manager_->HasHistory(); } + + Status Restore(DB* db) { return expected_state_manager_->Restore(db); } + + // Requires external locking covering all keys in `cf`. + void ClearColumnFamily(int cf) { + return expected_state_manager_->ClearColumnFamily(cf); + } + + // Prepare a Put that will be started but not finish yet + // This is useful for crash-recovery testing when the process may crash + // before updating the corresponding expected value + // + // Requires external locking covering `key` in `cf` to prevent concurrent + // write or delete to the same `key`. + PendingExpectedValue PreparePut(int cf, int64_t key) { + return expected_state_manager_->PreparePut(cf, key); + } + + // Does not requires external locking. + ExpectedValue Get(int cf, int64_t key) { + return expected_state_manager_->Get(cf, key); + } + + // Prepare a Delete that will be started but not finish yet + // This is useful for crash-recovery testing when the process may crash + // before updating the corresponding expected value + // + // Requires external locking covering `key` in `cf` to prevent concurrent + // write or delete to the same `key`. + PendingExpectedValue PrepareDelete(int cf, int64_t key) { + return expected_state_manager_->PrepareDelete(cf, key); + } + + // Requires external locking covering `key` in `cf` to prevent concurrent + // write or delete to the same `key`. + PendingExpectedValue PrepareSingleDelete(int cf, int64_t key) { + return expected_state_manager_->PrepareSingleDelete(cf, key); + } + + // Requires external locking covering keys in `[begin_key, end_key)` in `cf` + // to prevent concurrent write or delete to the same `key`. + std::vector PrepareDeleteRange(int cf, + int64_t begin_key, + int64_t end_key) { + return expected_state_manager_->PrepareDeleteRange(cf, begin_key, end_key); + } + + bool AllowsOverwrite(int64_t key) const { + return no_overwrite_ids_.find(key) == no_overwrite_ids_.end(); + } + + // Requires external locking covering `key` in `cf` to prevent concurrent + // delete to the same `key`. + bool Exists(int cf, int64_t key) { + return expected_state_manager_->Exists(cf, key); + } + + // Sync the `value_base` to the corresponding expected value + void SyncPut(int cf, int64_t key, uint32_t value_base) { + return expected_state_manager_->SyncPut(cf, key, value_base); + } + + // Sync the corresponding expected value to be pending Put + void SyncPendingPut(int cf, int64_t key) { + return expected_state_manager_->SyncPendingPut(cf, key); + } + + // Sync the corresponding expected value to be deleted + void SyncDelete(int cf, int64_t key) { + return expected_state_manager_->SyncDelete(cf, key); + } + + uint32_t GetSeed() const { return seed_; } + + void SetShouldStopBgThread() { should_stop_bg_thread_ = true; } + + bool ShouldStopBgThread() { return should_stop_bg_thread_; } + + void IncBgThreads() { ++num_bg_threads_; } + + void IncBgThreadsFinished() { ++bg_thread_finished_; } + + bool BgThreadsFinished() const { + return bg_thread_finished_ == num_bg_threads_; + } + + bool ShouldVerifyAtBeginning() const { + return !FLAGS_expected_values_dir.empty(); + } + + bool PrintingVerificationResults() { + bool tmp = false; + return !printing_verification_results_.compare_exchange_strong( + tmp, true, std::memory_order_relaxed); + } + + void FinishPrintingVerificationResults() { + printing_verification_results_.store(false, std::memory_order_relaxed); + } + + uint64_t GetStartTimestamp() const { return start_timestamp_; } + + void SafeTerminate() { + // Grab mutex so that we don't call terminate while another thread is + // attempting to print a stack trace due to the first one + MutexLock l(&mu_); + std::terminate(); + } + + private: + static void IgnoreReadErrorCallback(void*) { ignore_read_error = true; } + + // Pick random keys in each column family that will not experience overwrite. + std::unordered_set GenerateNoOverwriteIds() const { + fprintf(stdout, "Choosing random keys with no overwrite\n"); + // Start with the identity permutation. Subsequent iterations of + // for loop below will start with perm of previous for loop + std::vector permutation(max_key_); + for (int64_t i = 0; i < max_key_; ++i) { + permutation[i] = i; + } + // Now do the Knuth shuffle + const int64_t num_no_overwrite_keys = + (max_key_ * FLAGS_nooverwritepercent) / 100; + // Only need to figure out first num_no_overwrite_keys of permutation + std::unordered_set ret; + ret.reserve(num_no_overwrite_keys); + Random64 rnd(seed_); + for (int64_t i = 0; i < num_no_overwrite_keys; i++) { + assert(i < max_key_); + int64_t rand_index = i + rnd.Next() % (max_key_ - i); + // Swap i and rand_index; + int64_t temp = permutation[i]; + permutation[i] = permutation[rand_index]; + permutation[rand_index] = temp; + // Fill no_overwrite_ids_ with the first num_no_overwrite_keys of + // permutation + ret.insert(permutation[i]); + } + return ret; + } + + port::Mutex mu_; + port::CondVar cv_; + const uint32_t seed_; + const int64_t max_key_; + const uint32_t log2_keys_per_lock_; + int num_threads_; + long num_initialized_; + long num_populated_; + long vote_reopen_; + long num_done_; + bool start_; + bool start_verify_; + int num_bg_threads_; + bool should_stop_bg_thread_; + int bg_thread_finished_; + StressTest* stress_test_; + std::atomic verification_failure_; + std::atomic should_stop_test_; + + // Keys that should not be overwritten + const std::unordered_set no_overwrite_ids_; + + std::unique_ptr expected_state_manager_; + // Cannot store `port::Mutex` directly in vector since it is not copyable + // and storing it in the container may require copying depending on the impl. + std::vector> key_locks_; + std::atomic printing_verification_results_; + const uint64_t start_timestamp_; +}; + +// Per-thread state for concurrent executions of the same benchmark. +struct ThreadState { + uint32_t tid; // 0..n-1 + Random rand; // Has different seeds for different threads + SharedState* shared; + Stats stats; + struct SnapshotState { + const Snapshot* snapshot; + // The cf from which we did a Get at this snapshot + int cf_at; + // The name of the cf at the time that we did a read + std::string cf_at_name; + // The key with which we did a Get at this snapshot + std::string key; + // The status of the Get + Status status; + // The value of the Get + std::string value; + // optional state of all keys in the db + std::vector* key_vec; + + std::string timestamp; + }; + std::queue> snapshot_queue; + + ThreadState(uint32_t index, SharedState* _shared) + : tid(index), rand(1000 + index + _shared->GetSeed()), shared(_shared) {} +}; +} // namespace ROCKSDB_NAMESPACE +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db_stress_tool/db_stress_stat.cc b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_stat.cc new file mode 100644 index 0000000..6a7883a --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_stat.cc @@ -0,0 +1,17 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#ifdef GFLAGS + +#include "db_stress_tool/db_stress_stat.h" + +namespace ROCKSDB_NAMESPACE { + +std::shared_ptr dbstats; +std::shared_ptr dbstats_secondaries; + +} // namespace ROCKSDB_NAMESPACE + +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db_stress_tool/db_stress_stat.h b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_stat.h new file mode 100644 index 0000000..5b38c6e --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_stat.h @@ -0,0 +1,219 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once +#include +#include +#include +#include + +#include "monitoring/histogram.h" +#include "port/port.h" +#include "rocksdb/snapshot.h" +#include "rocksdb/statistics.h" +#include "rocksdb/system_clock.h" +#include "util/gflags_compat.h" +#include "util/random.h" + +DECLARE_bool(histogram); +DECLARE_bool(progress_reports); + +namespace ROCKSDB_NAMESPACE { + +// Database statistics +extern std::shared_ptr dbstats; +extern std::shared_ptr dbstats_secondaries; + +class Stats { + private: + uint64_t start_; + uint64_t finish_; + double seconds_; + long done_; + long gets_; + long prefixes_; + long writes_; + long deletes_; + size_t single_deletes_; + long iterator_size_sums_; + long founds_; + long iterations_; + long range_deletions_; + long covered_by_range_deletions_; + long errors_; + long verified_errors_; + long num_compact_files_succeed_; + long num_compact_files_failed_; + int next_report_; + size_t bytes_; + uint64_t last_op_finish_; + HistogramImpl hist_; + + public: + Stats() {} + + void Start() { + next_report_ = 100; + hist_.Clear(); + done_ = 0; + gets_ = 0; + prefixes_ = 0; + writes_ = 0; + deletes_ = 0; + single_deletes_ = 0; + iterator_size_sums_ = 0; + founds_ = 0; + iterations_ = 0; + range_deletions_ = 0; + covered_by_range_deletions_ = 0; + errors_ = 0; + verified_errors_ = 0; + bytes_ = 0; + seconds_ = 0; + num_compact_files_succeed_ = 0; + num_compact_files_failed_ = 0; + start_ = SystemClock::Default()->NowMicros(); + last_op_finish_ = start_; + finish_ = start_; + } + + void Merge(const Stats& other) { + hist_.Merge(other.hist_); + done_ += other.done_; + gets_ += other.gets_; + prefixes_ += other.prefixes_; + writes_ += other.writes_; + deletes_ += other.deletes_; + single_deletes_ += other.single_deletes_; + iterator_size_sums_ += other.iterator_size_sums_; + founds_ += other.founds_; + iterations_ += other.iterations_; + range_deletions_ += other.range_deletions_; + covered_by_range_deletions_ = other.covered_by_range_deletions_; + errors_ += other.errors_; + verified_errors_ += other.verified_errors_; + bytes_ += other.bytes_; + seconds_ += other.seconds_; + num_compact_files_succeed_ += other.num_compact_files_succeed_; + num_compact_files_failed_ += other.num_compact_files_failed_; + if (other.start_ < start_) start_ = other.start_; + if (other.finish_ > finish_) finish_ = other.finish_; + } + + void Stop() { + finish_ = SystemClock::Default()->NowMicros(); + seconds_ = (finish_ - start_) * 1e-6; + } + + void FinishedSingleOp() { + if (FLAGS_histogram) { + auto now = SystemClock::Default()->NowMicros(); + auto micros = now - last_op_finish_; + hist_.Add(micros); + if (micros > 20000) { + fprintf(stdout, "long op: %" PRIu64 " micros%30s\r", micros, ""); + } + last_op_finish_ = now; + } + + done_++; + if (FLAGS_progress_reports) { + if (done_ >= next_report_) { + if (next_report_ < 1000) + next_report_ += 100; + else if (next_report_ < 5000) + next_report_ += 500; + else if (next_report_ < 10000) + next_report_ += 1000; + else if (next_report_ < 50000) + next_report_ += 5000; + else if (next_report_ < 100000) + next_report_ += 10000; + else if (next_report_ < 500000) + next_report_ += 50000; + else + next_report_ += 100000; + fprintf(stdout, "... finished %ld ops%30s\r", done_, ""); + } + } + } + + void AddBytesForWrites(long nwrites, size_t nbytes) { + writes_ += nwrites; + bytes_ += nbytes; + } + + void AddGets(long ngets, long nfounds) { + founds_ += nfounds; + gets_ += ngets; + } + + void AddPrefixes(long nprefixes, long count) { + prefixes_ += nprefixes; + iterator_size_sums_ += count; + } + + void AddIterations(long n) { iterations_ += n; } + + void AddDeletes(long n) { deletes_ += n; } + + void AddSingleDeletes(size_t n) { single_deletes_ += n; } + + void AddRangeDeletions(long n) { range_deletions_ += n; } + + void AddCoveredByRangeDeletions(long n) { covered_by_range_deletions_ += n; } + + void AddErrors(long n) { errors_ += n; } + + void AddVerifiedErrors(long n) { verified_errors_ += n; } + + void AddNumCompactFilesSucceed(long n) { num_compact_files_succeed_ += n; } + + void AddNumCompactFilesFailed(long n) { num_compact_files_failed_ += n; } + + void Report(const char* name) { + std::string extra; + if (bytes_ < 1 || done_ < 1) { + fprintf(stderr, "No writes or ops?\n"); + return; + } + + double elapsed = (finish_ - start_) * 1e-6; + double bytes_mb = bytes_ / 1048576.0; + double rate = bytes_mb / elapsed; + double throughput = (double)done_ / elapsed; + + fprintf(stdout, "%-12s: ", name); + fprintf(stdout, "%.3f micros/op %ld ops/sec\n", seconds_ * 1e6 / done_, + (long)throughput); + fprintf(stdout, "%-12s: Wrote %.2f MB (%.2f MB/sec) (%ld%% of %ld ops)\n", + "", bytes_mb, rate, (100 * writes_) / done_, done_); + fprintf(stdout, "%-12s: Wrote %ld times\n", "", writes_); + fprintf(stdout, "%-12s: Deleted %ld times\n", "", deletes_); + fprintf(stdout, "%-12s: Single deleted %" ROCKSDB_PRIszt " times\n", "", + single_deletes_); + fprintf(stdout, "%-12s: %ld read and %ld found the key\n", "", gets_, + founds_); + fprintf(stdout, "%-12s: Prefix scanned %ld times\n", "", prefixes_); + fprintf(stdout, "%-12s: Iterator size sum is %ld\n", "", + iterator_size_sums_); + fprintf(stdout, "%-12s: Iterated %ld times\n", "", iterations_); + fprintf(stdout, "%-12s: Deleted %ld key-ranges\n", "", range_deletions_); + fprintf(stdout, "%-12s: Range deletions covered %ld keys\n", "", + covered_by_range_deletions_); + + fprintf(stdout, "%-12s: Got errors %ld times\n", "", errors_); + fprintf(stdout, "%-12s: %ld CompactFiles() succeed\n", "", + num_compact_files_succeed_); + fprintf(stdout, "%-12s: %ld CompactFiles() did not succeed\n", "", + num_compact_files_failed_); + + if (FLAGS_histogram) { + fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str()); + } + fflush(stdout); + } +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db_stress_tool/db_stress_table_properties_collector.h b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_table_properties_collector.h new file mode 100644 index 0000000..d1758cb --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_table_properties_collector.h @@ -0,0 +1,65 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "rocksdb/table.h" +#include "util/gflags_compat.h" +#include "util/random.h" + +DECLARE_int32(mark_for_compaction_one_file_in); + +namespace ROCKSDB_NAMESPACE { + +// A `DbStressTablePropertiesCollector` ignores what keys/values were added to +// the table, adds no properties to the table, and decides at random whether the +// table will be marked for compaction according to +// `FLAGS_mark_for_compaction_one_file_in`. +class DbStressTablePropertiesCollector : public TablePropertiesCollector { + public: + DbStressTablePropertiesCollector() + : need_compact_(Random::GetTLSInstance()->OneInOpt( + FLAGS_mark_for_compaction_one_file_in)) {} + + virtual Status AddUserKey(const Slice& /* key */, const Slice& /* value */, + EntryType /*type*/, SequenceNumber /*seq*/, + uint64_t /*file_size*/) override { + return Status::OK(); + } + + virtual Status Finish(UserCollectedProperties* /* properties */) override { + return Status::OK(); + } + + virtual UserCollectedProperties GetReadableProperties() const override { + return UserCollectedProperties{}; + } + + virtual const char* Name() const override { + return "DbStressTablePropertiesCollector"; + } + + virtual bool NeedCompact() const override { return need_compact_; } + + private: + const bool need_compact_; +}; + +// A `DbStressTablePropertiesCollectorFactory` creates +// `DbStressTablePropertiesCollectorFactory`s. +class DbStressTablePropertiesCollectorFactory + : public TablePropertiesCollectorFactory { + public: + virtual TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context /* context */) override { + return new DbStressTablePropertiesCollector(); + } + + virtual const char* Name() const override { + return "DbStressTablePropertiesCollectorFactory"; + } +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/db_stress_tool/db_stress_test_base.cc b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_test_base.cc new file mode 100644 index 0000000..01026a3 --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_test_base.cc @@ -0,0 +1,3364 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// + +#include +#include + +#include "util/compression.h" +#ifdef GFLAGS +#include "db_stress_tool/db_stress_common.h" +#include "db_stress_tool/db_stress_compaction_filter.h" +#include "db_stress_tool/db_stress_driver.h" +#include "db_stress_tool/db_stress_table_properties_collector.h" +#include "rocksdb/convenience.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/secondary_cache.h" +#include "rocksdb/sst_file_manager.h" +#include "rocksdb/types.h" +#include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/write_batch_with_index.h" +#include "test_util/testutil.h" +#include "util/cast_util.h" +#include "utilities/backup/backup_engine_impl.h" +#include "utilities/fault_injection_fs.h" +#include "utilities/fault_injection_secondary_cache.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { + +std::shared_ptr CreateFilterPolicy() { + if (FLAGS_bloom_bits < 0) { + return BlockBasedTableOptions().filter_policy; + } + const FilterPolicy* new_policy; + if (FLAGS_ribbon_starting_level >= 999) { + // Use Bloom API + new_policy = NewBloomFilterPolicy(FLAGS_bloom_bits, false); + } else { + new_policy = NewRibbonFilterPolicy( + FLAGS_bloom_bits, /* bloom_before_level */ FLAGS_ribbon_starting_level); + } + return std::shared_ptr(new_policy); +} + +} // namespace + +StressTest::StressTest() + : cache_(NewCache(FLAGS_cache_size, FLAGS_cache_numshardbits)), + filter_policy_(CreateFilterPolicy()), + db_(nullptr), + txn_db_(nullptr), + optimistic_txn_db_(nullptr), + db_aptr_(nullptr), + clock_(db_stress_env->GetSystemClock().get()), + new_column_family_name_(1), + num_times_reopened_(0), + db_preload_finished_(false), + cmp_db_(nullptr), + is_db_stopped_(false) { + if (FLAGS_destroy_db_initially) { + std::vector files; + db_stress_env->GetChildren(FLAGS_db, &files); + for (unsigned int i = 0; i < files.size(); i++) { + if (Slice(files[i]).starts_with("heap-")) { + db_stress_env->DeleteFile(FLAGS_db + "/" + files[i]); + } + } + + Options options; + options.env = db_stress_env; + // Remove files without preserving manfiest files + const Status s = !FLAGS_use_blob_db + ? DestroyDB(FLAGS_db, options) + : blob_db::DestroyBlobDB(FLAGS_db, options, + blob_db::BlobDBOptions()); + + if (!s.ok()) { + fprintf(stderr, "Cannot destroy original db: %s\n", s.ToString().c_str()); + exit(1); + } + } +} + +StressTest::~StressTest() { + for (auto cf : column_families_) { + delete cf; + } + column_families_.clear(); + delete db_; + + for (auto* cf : cmp_cfhs_) { + delete cf; + } + cmp_cfhs_.clear(); + delete cmp_db_; +} + +std::shared_ptr StressTest::NewCache(size_t capacity, + int32_t num_shard_bits) { + ConfigOptions config_options; + if (capacity <= 0) { + return nullptr; + } + + std::shared_ptr secondary_cache; + if (!FLAGS_secondary_cache_uri.empty()) { + Status s = SecondaryCache::CreateFromString( + config_options, FLAGS_secondary_cache_uri, &secondary_cache); + if (secondary_cache == nullptr) { + fprintf(stderr, + "No secondary cache registered matching string: %s status=%s\n", + FLAGS_secondary_cache_uri.c_str(), s.ToString().c_str()); + exit(1); + } + if (FLAGS_secondary_cache_fault_one_in > 0) { + secondary_cache = std::make_shared( + secondary_cache, static_cast(FLAGS_seed), + FLAGS_secondary_cache_fault_one_in); + } + } + + if (FLAGS_cache_type == "clock_cache") { + fprintf(stderr, "Old clock cache implementation has been removed.\n"); + exit(1); + } else if (FLAGS_cache_type == "hyper_clock_cache") { + HyperClockCacheOptions opts(static_cast(capacity), + FLAGS_block_size /*estimated_entry_charge*/, + num_shard_bits); + opts.secondary_cache = std::move(secondary_cache); + return opts.MakeSharedCache(); + } else if (FLAGS_cache_type == "lru_cache") { + LRUCacheOptions opts; + opts.capacity = capacity; + opts.num_shard_bits = num_shard_bits; + opts.secondary_cache = std::move(secondary_cache); + return NewLRUCache(opts); + } else { + fprintf(stderr, "Cache type not supported."); + exit(1); + } +} + +std::vector StressTest::GetBlobCompressionTags() { + std::vector compression_tags{"kNoCompression"}; + + if (Snappy_Supported()) { + compression_tags.emplace_back("kSnappyCompression"); + } + if (LZ4_Supported()) { + compression_tags.emplace_back("kLZ4Compression"); + } + if (ZSTD_Supported()) { + compression_tags.emplace_back("kZSTD"); + } + + return compression_tags; +} + +bool StressTest::BuildOptionsTable() { + if (FLAGS_set_options_one_in <= 0) { + return true; + } + + std::unordered_map> options_tbl = { + {"write_buffer_size", + {std::to_string(options_.write_buffer_size), + std::to_string(options_.write_buffer_size * 2), + std::to_string(options_.write_buffer_size * 4)}}, + {"max_write_buffer_number", + {std::to_string(options_.max_write_buffer_number), + std::to_string(options_.max_write_buffer_number * 2), + std::to_string(options_.max_write_buffer_number * 4)}}, + {"arena_block_size", + { + std::to_string(options_.arena_block_size), + std::to_string(options_.write_buffer_size / 4), + std::to_string(options_.write_buffer_size / 8), + }}, + {"memtable_huge_page_size", {"0", std::to_string(2 * 1024 * 1024)}}, + {"max_successive_merges", {"0", "2", "4"}}, + {"inplace_update_num_locks", {"100", "200", "300"}}, + // TODO: re-enable once internal task T124324915 is fixed. + // {"experimental_mempurge_threshold", {"0.0", "1.0"}}, + // TODO(ljin): enable test for this option + // {"disable_auto_compactions", {"100", "200", "300"}}, + {"level0_file_num_compaction_trigger", + { + std::to_string(options_.level0_file_num_compaction_trigger), + std::to_string(options_.level0_file_num_compaction_trigger + 2), + std::to_string(options_.level0_file_num_compaction_trigger + 4), + }}, + {"level0_slowdown_writes_trigger", + { + std::to_string(options_.level0_slowdown_writes_trigger), + std::to_string(options_.level0_slowdown_writes_trigger + 2), + std::to_string(options_.level0_slowdown_writes_trigger + 4), + }}, + {"level0_stop_writes_trigger", + { + std::to_string(options_.level0_stop_writes_trigger), + std::to_string(options_.level0_stop_writes_trigger + 2), + std::to_string(options_.level0_stop_writes_trigger + 4), + }}, + {"max_compaction_bytes", + { + std::to_string(options_.target_file_size_base * 5), + std::to_string(options_.target_file_size_base * 15), + std::to_string(options_.target_file_size_base * 100), + }}, + {"target_file_size_base", + { + std::to_string(options_.target_file_size_base), + std::to_string(options_.target_file_size_base * 2), + std::to_string(options_.target_file_size_base * 4), + }}, + {"target_file_size_multiplier", + { + std::to_string(options_.target_file_size_multiplier), + "1", + "2", + }}, + {"max_bytes_for_level_base", + { + std::to_string(options_.max_bytes_for_level_base / 2), + std::to_string(options_.max_bytes_for_level_base), + std::to_string(options_.max_bytes_for_level_base * 2), + }}, + {"max_bytes_for_level_multiplier", + { + std::to_string(options_.max_bytes_for_level_multiplier), + "1", + "2", + }}, + {"max_sequential_skip_in_iterations", {"4", "8", "12"}}, + }; + + if (FLAGS_allow_setting_blob_options_dynamically) { + options_tbl.emplace("enable_blob_files", + std::vector{"false", "true"}); + options_tbl.emplace("min_blob_size", + std::vector{"0", "8", "16"}); + options_tbl.emplace("blob_file_size", + std::vector{"1M", "16M", "256M", "1G"}); + options_tbl.emplace("blob_compression_type", GetBlobCompressionTags()); + options_tbl.emplace("enable_blob_garbage_collection", + std::vector{"false", "true"}); + options_tbl.emplace( + "blob_garbage_collection_age_cutoff", + std::vector{"0.0", "0.25", "0.5", "0.75", "1.0"}); + options_tbl.emplace("blob_garbage_collection_force_threshold", + std::vector{"0.5", "0.75", "1.0"}); + options_tbl.emplace("blob_compaction_readahead_size", + std::vector{"0", "1M", "4M"}); + options_tbl.emplace("blob_file_starting_level", + std::vector{"0", "1", "2"}); + options_tbl.emplace("prepopulate_blob_cache", + std::vector{"kDisable", "kFlushOnly"}); + } + + options_table_ = std::move(options_tbl); + + for (const auto& iter : options_table_) { + options_index_.push_back(iter.first); + } + return true; +} + +void StressTest::InitDb(SharedState* shared) { + uint64_t now = clock_->NowMicros(); + fprintf(stdout, "%s Initializing db_stress\n", + clock_->TimeToString(now / 1000000).c_str()); + PrintEnv(); + Open(shared); + BuildOptionsTable(); +} + +void StressTest::FinishInitDb(SharedState* shared) { + if (FLAGS_read_only) { + uint64_t now = clock_->NowMicros(); + fprintf(stdout, "%s Preloading db with %" PRIu64 " KVs\n", + clock_->TimeToString(now / 1000000).c_str(), FLAGS_max_key); + PreloadDbAndReopenAsReadOnly(FLAGS_max_key, shared); + } + + if (shared->HasHistory()) { + // The way it works right now is, if there's any history, that means the + // previous run mutating the DB had all its operations traced, in which case + // we should always be able to `Restore()` the expected values to match the + // `db_`'s current seqno. + Status s = shared->Restore(db_); + if (!s.ok()) { + fprintf(stderr, "Error restoring historical expected values: %s\n", + s.ToString().c_str()); + exit(1); + } + } + if (FLAGS_use_txn && !FLAGS_use_optimistic_txn) { + // It's OK here without sync because unsynced data cannot be lost at this + // point + // - even with sync_fault_injection=1 as the + // file is still directly writable until after FinishInitDb() + ProcessRecoveredPreparedTxns(shared); + } + + if (FLAGS_enable_compaction_filter) { + auto* compaction_filter_factory = + reinterpret_cast( + options_.compaction_filter_factory.get()); + assert(compaction_filter_factory); + // This must be called only after any potential `SharedState::Restore()` has + // completed in order for the `compaction_filter_factory` to operate on the + // correct latest values file. + compaction_filter_factory->SetSharedState(shared); + fprintf(stdout, "Compaction filter factory: %s\n", + compaction_filter_factory->Name()); + } +} + +void StressTest::TrackExpectedState(SharedState* shared) { + // For `FLAGS_manual_wal_flush_one_inWAL` + // data can be lost when `manual_wal_flush_one_in > 0` and `FlushWAL()` is not + // explictly called by users of RocksDB (in our case, db stress). + // Therefore recovery from such potential WAL data loss is a prefix recovery + // that requires tracing + if ((FLAGS_sync_fault_injection || FLAGS_disable_wal || + FLAGS_manual_wal_flush_one_in > 0) && + IsStateTracked()) { + Status s = shared->SaveAtAndAfter(db_); + if (!s.ok()) { + fprintf(stderr, "Error enabling history tracing: %s\n", + s.ToString().c_str()); + exit(1); + } + } +} + +Status StressTest::AssertSame(DB* db, ColumnFamilyHandle* cf, + ThreadState::SnapshotState& snap_state) { + Status s; + if (cf->GetName() != snap_state.cf_at_name) { + return s; + } + // This `ReadOptions` is for validation purposes. Ignore + // `FLAGS_rate_limit_user_ops` to avoid slowing any validation. + ReadOptions ropt; + ropt.snapshot = snap_state.snapshot; + Slice ts; + if (!snap_state.timestamp.empty()) { + ts = snap_state.timestamp; + ropt.timestamp = &ts; + } + PinnableSlice exp_v(&snap_state.value); + exp_v.PinSelf(); + PinnableSlice v; + s = db->Get(ropt, cf, snap_state.key, &v); + if (!s.ok() && !s.IsNotFound()) { + return s; + } + if (snap_state.status != s) { + return Status::Corruption( + "The snapshot gave inconsistent results for key " + + std::to_string(Hash(snap_state.key.c_str(), snap_state.key.size(), 0)) + + " in cf " + cf->GetName() + ": (" + snap_state.status.ToString() + + ") vs. (" + s.ToString() + ")"); + } + if (s.ok()) { + if (exp_v != v) { + return Status::Corruption("The snapshot gave inconsistent values: (" + + exp_v.ToString() + ") vs. (" + v.ToString() + + ")"); + } + } + if (snap_state.key_vec != nullptr) { + // When `prefix_extractor` is set, seeking to beginning and scanning + // across prefixes are only supported with `total_order_seek` set. + ropt.total_order_seek = true; + std::unique_ptr iterator(db->NewIterator(ropt)); + std::unique_ptr> tmp_bitvec( + new std::vector(FLAGS_max_key)); + for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) { + uint64_t key_val; + if (GetIntVal(iterator->key().ToString(), &key_val)) { + (*tmp_bitvec.get())[key_val] = true; + } + } + if (!std::equal(snap_state.key_vec->begin(), snap_state.key_vec->end(), + tmp_bitvec.get()->begin())) { + return Status::Corruption("Found inconsistent keys at this snapshot"); + } + } + return Status::OK(); +} + +void StressTest::VerificationAbort(SharedState* shared, std::string msg, + Status s) const { + fprintf(stderr, "Verification failed: %s. Status is %s\n", msg.c_str(), + s.ToString().c_str()); + shared->SetVerificationFailure(); +} + +void StressTest::VerificationAbort(SharedState* shared, std::string msg, int cf, + int64_t key) const { + auto key_str = Key(key); + Slice key_slice = key_str; + fprintf(stderr, + "Verification failed for column family %d key %s (%" PRIi64 "): %s\n", + cf, key_slice.ToString(true).c_str(), key, msg.c_str()); + shared->SetVerificationFailure(); +} + +void StressTest::VerificationAbort(SharedState* shared, std::string msg, int cf, + int64_t key, Slice value_from_db, + Slice value_from_expected) const { + auto key_str = Key(key); + fprintf(stderr, + "Verification failed for column family %d key %s (%" PRIi64 + "): value_from_db: %s, value_from_expected: %s, msg: %s\n", + cf, Slice(key_str).ToString(true).c_str(), key, + value_from_db.ToString(true).c_str(), + value_from_expected.ToString(true).c_str(), msg.c_str()); + shared->SetVerificationFailure(); +} + +void StressTest::VerificationAbort(SharedState* shared, int cf, int64_t key, + const Slice& value, + const WideColumns& columns) const { + assert(shared); + + auto key_str = Key(key); + + fprintf(stderr, + "Verification failed for column family %d key %s (%" PRIi64 + "): Value and columns inconsistent: value: %s, columns: %s\n", + cf, Slice(key_str).ToString(/* hex */ true).c_str(), key, + value.ToString(/* hex */ true).c_str(), + WideColumnsToHex(columns).c_str()); + + shared->SetVerificationFailure(); +} + +std::string StressTest::DebugString(const Slice& value, + const WideColumns& columns) { + std::ostringstream oss; + + oss << "value: " << value.ToString(/* hex */ true) + << ", columns: " << WideColumnsToHex(columns); + + return oss.str(); +} + +void StressTest::PrintStatistics() { + if (dbstats) { + fprintf(stdout, "STATISTICS:\n%s\n", dbstats->ToString().c_str()); + } + if (dbstats_secondaries) { + fprintf(stdout, "Secondary instances STATISTICS:\n%s\n", + dbstats_secondaries->ToString().c_str()); + } +} + +// Currently PreloadDb has to be single-threaded. +void StressTest::PreloadDbAndReopenAsReadOnly(int64_t number_of_keys, + SharedState* shared) { + WriteOptions write_opts; + write_opts.disableWAL = FLAGS_disable_wal; + if (FLAGS_sync) { + write_opts.sync = true; + } + if (FLAGS_rate_limit_auto_wal_flush) { + write_opts.rate_limiter_priority = Env::IO_USER; + } + char value[100]; + int cf_idx = 0; + Status s; + for (auto cfh : column_families_) { + for (int64_t k = 0; k != number_of_keys; ++k) { + const std::string key = Key(k); + + PendingExpectedValue pending_expected_value = + shared->PreparePut(cf_idx, k); + const uint32_t value_base = pending_expected_value.GetFinalValueBase(); + const size_t sz = GenerateValue(value_base, value, sizeof(value)); + + const Slice v(value, sz); + + + std::string ts; + if (FLAGS_user_timestamp_size > 0) { + ts = GetNowNanos(); + } + + if (FLAGS_use_merge) { + if (!FLAGS_use_txn) { + if (FLAGS_user_timestamp_size > 0) { + s = db_->Merge(write_opts, cfh, key, ts, v); + } else { + s = db_->Merge(write_opts, cfh, key, v); + } + } else { + s = ExecuteTransaction( + write_opts, /*thread=*/nullptr, + [&](Transaction& txn) { return txn.Merge(cfh, key, v); }); + } + } else if (FLAGS_use_put_entity_one_in > 0) { + s = db_->PutEntity(write_opts, cfh, key, + GenerateWideColumns(value_base, v)); + } else { + if (!FLAGS_use_txn) { + if (FLAGS_user_timestamp_size > 0) { + s = db_->Put(write_opts, cfh, key, ts, v); + } else { + s = db_->Put(write_opts, cfh, key, v); + } + } else { + s = ExecuteTransaction( + write_opts, /*thread=*/nullptr, + [&](Transaction& txn) { return txn.Put(cfh, key, v); }); + } + } + + pending_expected_value.Commit(); + if (!s.ok()) { + break; + } + } + if (!s.ok()) { + break; + } + ++cf_idx; + } + if (s.ok()) { + s = db_->Flush(FlushOptions(), column_families_); + } + if (s.ok()) { + for (auto cf : column_families_) { + delete cf; + } + column_families_.clear(); + delete db_; + db_ = nullptr; + txn_db_ = nullptr; + optimistic_txn_db_ = nullptr; + + db_preload_finished_.store(true); + auto now = clock_->NowMicros(); + fprintf(stdout, "%s Reopening database in read-only\n", + clock_->TimeToString(now / 1000000).c_str()); + // Reopen as read-only, can ignore all options related to updates + Open(shared); + } else { + fprintf(stderr, "Failed to preload db"); + exit(1); + } +} + +Status StressTest::SetOptions(ThreadState* thread) { + assert(FLAGS_set_options_one_in > 0); + std::unordered_map opts; + std::string name = + options_index_[thread->rand.Next() % options_index_.size()]; + int value_idx = thread->rand.Next() % options_table_[name].size(); + if (name == "level0_file_num_compaction_trigger" || + name == "level0_slowdown_writes_trigger" || + name == "level0_stop_writes_trigger") { + opts["level0_file_num_compaction_trigger"] = + options_table_["level0_file_num_compaction_trigger"][value_idx]; + opts["level0_slowdown_writes_trigger"] = + options_table_["level0_slowdown_writes_trigger"][value_idx]; + opts["level0_stop_writes_trigger"] = + options_table_["level0_stop_writes_trigger"][value_idx]; + } else { + opts[name] = options_table_[name][value_idx]; + } + + int rand_cf_idx = thread->rand.Next() % FLAGS_column_families; + auto cfh = column_families_[rand_cf_idx]; + return db_->SetOptions(cfh, opts); +} + +void StressTest::ProcessRecoveredPreparedTxns(SharedState* shared) { + assert(txn_db_); + std::vector recovered_prepared_trans; + txn_db_->GetAllPreparedTransactions(&recovered_prepared_trans); + for (Transaction* txn : recovered_prepared_trans) { + ProcessRecoveredPreparedTxnsHelper(txn, shared); + delete txn; + } + recovered_prepared_trans.clear(); + txn_db_->GetAllPreparedTransactions(&recovered_prepared_trans); + assert(recovered_prepared_trans.size() == 0); +} + +void StressTest::ProcessRecoveredPreparedTxnsHelper(Transaction* txn, + SharedState* shared) { + thread_local Random rand(static_cast(FLAGS_seed)); + for (size_t i = 0; i < column_families_.size(); ++i) { + std::unique_ptr wbwi_iter( + txn->GetWriteBatch()->NewIterator(column_families_[i])); + for (wbwi_iter->SeekToFirst(); wbwi_iter->Valid(); wbwi_iter->Next()) { + uint64_t key_val; + if (GetIntVal(wbwi_iter->Entry().key.ToString(), &key_val)) { + shared->SyncPendingPut(static_cast(i) /* cf_idx */, key_val); + } + } + } + if (rand.OneIn(2)) { + Status s = txn->Commit(); + assert(s.ok()); + } else { + Status s = txn->Rollback(); + assert(s.ok()); + } +} + +Status StressTest::NewTxn(WriteOptions& write_opts, + std::unique_ptr* out_txn) { + if (!FLAGS_use_txn) { + return Status::InvalidArgument("NewTxn when FLAGS_use_txn is not set"); + } + write_opts.disableWAL = FLAGS_disable_wal; + static std::atomic txn_id = {0}; + if (FLAGS_use_optimistic_txn) { + out_txn->reset(optimistic_txn_db_->BeginTransaction(write_opts)); + return Status::OK(); + } else { + TransactionOptions txn_options; + txn_options.use_only_the_last_commit_time_batch_for_recovery = + FLAGS_use_only_the_last_commit_time_batch_for_recovery; + txn_options.lock_timeout = 600000; // 10 min + txn_options.deadlock_detect = true; + out_txn->reset(txn_db_->BeginTransaction(write_opts, txn_options)); + auto istr = std::to_string(txn_id.fetch_add(1)); + Status s = (*out_txn)->SetName("xid" + istr); + return s; + } +} + +Status StressTest::CommitTxn(Transaction& txn, ThreadState* thread) { + if (!FLAGS_use_txn) { + return Status::InvalidArgument("CommitTxn when FLAGS_use_txn is not set"); + } + Status s = Status::OK(); + if (FLAGS_use_optimistic_txn) { + assert(optimistic_txn_db_); + s = txn.Commit(); + } else { + assert(txn_db_); + s = txn.Prepare(); + std::shared_ptr timestamped_snapshot; + if (s.ok()) { + if (thread && FLAGS_create_timestamped_snapshot_one_in && + thread->rand.OneIn(FLAGS_create_timestamped_snapshot_one_in)) { + uint64_t ts = db_stress_env->NowNanos(); + s = txn.CommitAndTryCreateSnapshot(/*notifier=*/nullptr, ts, + ×tamped_snapshot); + + std::pair> res; + if (thread->tid == 0) { + uint64_t now = db_stress_env->NowNanos(); + res = txn_db_->CreateTimestampedSnapshot(now); + if (res.first.ok()) { + assert(res.second); + assert(res.second->GetTimestamp() == now); + if (timestamped_snapshot) { + assert(res.second->GetTimestamp() > + timestamped_snapshot->GetTimestamp()); + } + } else { + assert(!res.second); + } + } + } else { + s = txn.Commit(); + } + } + if (thread && FLAGS_create_timestamped_snapshot_one_in > 0 && + thread->rand.OneInOpt(50000)) { + uint64_t now = db_stress_env->NowNanos(); + constexpr uint64_t time_diff = static_cast(1000) * 1000 * 1000; + txn_db_->ReleaseTimestampedSnapshotsOlderThan(now - time_diff); + } + } + return s; +} + +Status StressTest::ExecuteTransaction( + WriteOptions& write_opts, ThreadState* thread, + std::function&& ops) { + std::unique_ptr txn; + Status s = NewTxn(write_opts, &txn); + if (s.ok()) { + for (int tries = 1;; ++tries) { + s = ops(*txn); + if (s.ok()) { + s = CommitTxn(*txn, thread); + if (s.ok()) { + break; + } + } + // Optimistic txn might return TryAgain, in which case rollback + // and try again. But that shouldn't happen too many times in a row. + if (!s.IsTryAgain() || !FLAGS_use_optimistic_txn) { + break; + } + if (tries >= 5) { + break; + } + s = txn->Rollback(); + if (!s.ok()) { + break; + } + } + } + return s; +} + +void StressTest::OperateDb(ThreadState* thread) { + ReadOptions read_opts(FLAGS_verify_checksum, true); + read_opts.rate_limiter_priority = + FLAGS_rate_limit_user_ops ? Env::IO_USER : Env::IO_TOTAL; + read_opts.async_io = FLAGS_async_io; + read_opts.adaptive_readahead = FLAGS_adaptive_readahead; + read_opts.readahead_size = FLAGS_readahead_size; + WriteOptions write_opts; + if (FLAGS_rate_limit_auto_wal_flush) { + write_opts.rate_limiter_priority = Env::IO_USER; + } + auto shared = thread->shared; + char value[100]; + std::string from_db; + if (FLAGS_sync) { + write_opts.sync = true; + } + write_opts.disableWAL = FLAGS_disable_wal; + write_opts.protection_bytes_per_key = FLAGS_batch_protection_bytes_per_key; + const int prefix_bound = static_cast(FLAGS_readpercent) + + static_cast(FLAGS_prefixpercent); + const int write_bound = prefix_bound + static_cast(FLAGS_writepercent); + const int del_bound = write_bound + static_cast(FLAGS_delpercent); + const int delrange_bound = + del_bound + static_cast(FLAGS_delrangepercent); + const int iterate_bound = + delrange_bound + static_cast(FLAGS_iterpercent); + + const uint64_t ops_per_open = FLAGS_ops_per_thread / (FLAGS_reopen + 1); + +#ifndef NDEBUG + if (FLAGS_read_fault_one_in) { + fault_fs_guard->SetThreadLocalReadErrorContext(thread->shared->GetSeed(), + FLAGS_read_fault_one_in); + } +#endif // NDEBUG + if (FLAGS_write_fault_one_in) { + IOStatus error_msg; + if (FLAGS_injest_error_severity <= 1 || FLAGS_injest_error_severity > 2) { + error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + } else if (FLAGS_injest_error_severity == 2) { + // Ingest the fatal error + error_msg = IOStatus::IOError("Fatal IO Error"); + error_msg.SetDataLoss(true); + } + std::vector types = {FileType::kTableFile, + FileType::kDescriptorFile, + FileType::kCurrentFile}; + fault_fs_guard->SetRandomWriteError( + thread->shared->GetSeed(), FLAGS_write_fault_one_in, error_msg, + /*inject_for_all_file_types=*/false, types); + } + thread->stats.Start(); + for (int open_cnt = 0; open_cnt <= FLAGS_reopen; ++open_cnt) { + if (thread->shared->HasVerificationFailedYet() || + thread->shared->ShouldStopTest()) { + break; + } + if (open_cnt != 0) { + thread->stats.FinishedSingleOp(); + MutexLock l(thread->shared->GetMutex()); + while (!thread->snapshot_queue.empty()) { + db_->ReleaseSnapshot(thread->snapshot_queue.front().second.snapshot); + delete thread->snapshot_queue.front().second.key_vec; + thread->snapshot_queue.pop(); + } + thread->shared->IncVotedReopen(); + if (thread->shared->AllVotedReopen()) { + thread->shared->GetStressTest()->Reopen(thread); + thread->shared->GetCondVar()->SignalAll(); + } else { + thread->shared->GetCondVar()->Wait(); + } + // Commenting this out as we don't want to reset stats on each open. + // thread->stats.Start(); + } + + for (uint64_t i = 0; i < ops_per_open; i++) { + if (thread->shared->HasVerificationFailedYet()) { + break; + } + + // Change Options + if (thread->rand.OneInOpt(FLAGS_set_options_one_in)) { + SetOptions(thread); + } + + if (thread->rand.OneInOpt(FLAGS_set_in_place_one_in)) { + options_.inplace_update_support ^= options_.inplace_update_support; + } + + if (thread->tid == 0 && FLAGS_verify_db_one_in > 0 && + thread->rand.OneIn(FLAGS_verify_db_one_in)) { + ContinuouslyVerifyDb(thread); + if (thread->shared->ShouldStopTest()) { + break; + } + } + + MaybeClearOneColumnFamily(thread); + + if (thread->rand.OneInOpt(FLAGS_manual_wal_flush_one_in)) { + bool sync = thread->rand.OneIn(2) ? true : false; + Status s = db_->FlushWAL(sync); + if (!s.ok() && !(sync && s.IsNotSupported())) { + fprintf(stderr, "FlushWAL(sync=%s) failed: %s\n", + (sync ? "true" : "false"), s.ToString().c_str()); + } + } + + if (thread->rand.OneInOpt(FLAGS_lock_wal_one_in)) { + Status s = db_->LockWAL(); + if (!s.ok()) { + fprintf(stderr, "LockWAL() failed: %s\n", s.ToString().c_str()); + } else { + auto old_seqno = db_->GetLatestSequenceNumber(); + // Yield for a while + do { + std::this_thread::yield(); + } while (thread->rand.OneIn(2)); + // Latest seqno should not have changed + auto new_seqno = db_->GetLatestSequenceNumber(); + if (old_seqno != new_seqno) { + fprintf( + stderr, + "Failure: latest seqno changed from %u to %u with WAL locked\n", + (unsigned)old_seqno, (unsigned)new_seqno); + } + s = db_->UnlockWAL(); + if (!s.ok()) { + fprintf(stderr, "UnlockWAL() failed: %s\n", s.ToString().c_str()); + } + } + } + + if (thread->rand.OneInOpt(FLAGS_sync_wal_one_in)) { + Status s = db_->SyncWAL(); + if (!s.ok() && !s.IsNotSupported()) { + fprintf(stderr, "SyncWAL() failed: %s\n", s.ToString().c_str()); + } + } + + int rand_column_family = thread->rand.Next() % FLAGS_column_families; + ColumnFamilyHandle* column_family = column_families_[rand_column_family]; + + if (thread->rand.OneInOpt(FLAGS_compact_files_one_in)) { + TestCompactFiles(thread, column_family); + } + + int64_t rand_key = GenerateOneKey(thread, i); + std::string keystr = Key(rand_key); + Slice key = keystr; + + if (thread->rand.OneInOpt(FLAGS_compact_range_one_in)) { + TestCompactRange(thread, rand_key, key, column_family); + if (thread->shared->HasVerificationFailedYet()) { + break; + } + } + + std::vector rand_column_families = + GenerateColumnFamilies(FLAGS_column_families, rand_column_family); + + if (thread->rand.OneInOpt(FLAGS_flush_one_in)) { + Status status = TestFlush(rand_column_families); + if (!status.ok()) { + fprintf(stdout, "Unable to perform Flush(): %s\n", + status.ToString().c_str()); + } + } + + // Verify GetLiveFiles with a 1 in N chance. + if (thread->rand.OneInOpt(FLAGS_get_live_files_one_in) && + !FLAGS_write_fault_one_in) { + Status status = VerifyGetLiveFiles(); + if (!status.ok()) { + VerificationAbort(shared, "VerifyGetLiveFiles status not OK", status); + } + } + + // Verify GetSortedWalFiles with a 1 in N chance. + if (thread->rand.OneInOpt(FLAGS_get_sorted_wal_files_one_in)) { + Status status = VerifyGetSortedWalFiles(); + if (!status.ok()) { + VerificationAbort(shared, "VerifyGetSortedWalFiles status not OK", + status); + } + } + + // Verify GetCurrentWalFile with a 1 in N chance. + if (thread->rand.OneInOpt(FLAGS_get_current_wal_file_one_in)) { + Status status = VerifyGetCurrentWalFile(); + if (!status.ok()) { + VerificationAbort(shared, "VerifyGetCurrentWalFile status not OK", + status); + } + } + + if (thread->rand.OneInOpt(FLAGS_pause_background_one_in)) { + Status status = TestPauseBackground(thread); + if (!status.ok()) { + VerificationAbort( + shared, "Pause/ContinueBackgroundWork status not OK", status); + } + } + + if (thread->rand.OneInOpt(FLAGS_verify_checksum_one_in)) { + Status status = db_->VerifyChecksum(); + if (!status.ok()) { + VerificationAbort(shared, "VerifyChecksum status not OK", status); + } + } + + if (thread->rand.OneInOpt(FLAGS_get_property_one_in)) { + TestGetProperty(thread); + } + + std::vector rand_keys = GenerateKeys(rand_key); + + if (thread->rand.OneInOpt(FLAGS_ingest_external_file_one_in)) { + TestIngestExternalFile(thread, rand_column_families, rand_keys); + } + + if (thread->rand.OneInOpt(FLAGS_backup_one_in)) { + // Beyond a certain DB size threshold, this test becomes heavier than + // it's worth. + uint64_t total_size = 0; + if (FLAGS_backup_max_size > 0) { + std::vector files; + db_stress_env->GetChildrenFileAttributes(FLAGS_db, &files); + for (auto& file : files) { + total_size += file.size_bytes; + } + } + + if (total_size <= FLAGS_backup_max_size) { + Status s = TestBackupRestore(thread, rand_column_families, rand_keys); + if (!s.ok()) { + VerificationAbort(shared, "Backup/restore gave inconsistent state", + s); + } + } + } + + if (thread->rand.OneInOpt(FLAGS_checkpoint_one_in)) { + Status s = TestCheckpoint(thread, rand_column_families, rand_keys); + if (!s.ok()) { + VerificationAbort(shared, "Checkpoint gave inconsistent state", s); + } + } + + if (thread->rand.OneInOpt(FLAGS_approximate_size_one_in)) { + Status s = + TestApproximateSize(thread, i, rand_column_families, rand_keys); + if (!s.ok()) { + VerificationAbort(shared, "ApproximateSize Failed", s); + } + } + if (thread->rand.OneInOpt(FLAGS_acquire_snapshot_one_in)) { + TestAcquireSnapshot(thread, rand_column_family, keystr, i); + } + + /*always*/ { + Status s = MaybeReleaseSnapshots(thread, i); + if (!s.ok()) { + VerificationAbort(shared, "Snapshot gave inconsistent state", s); + } + } + + // Assign timestamps if necessary. + std::string read_ts_str; + Slice read_ts; + if (FLAGS_user_timestamp_size > 0) { + read_ts_str = GetNowNanos(); + read_ts = read_ts_str; + read_opts.timestamp = &read_ts; + } + + int prob_op = thread->rand.Uniform(100); + // Reset this in case we pick something other than a read op. We don't + // want to use a stale value when deciding at the beginning of the loop + // whether to vote to reopen + if (prob_op >= 0 && prob_op < static_cast(FLAGS_readpercent)) { + assert(0 <= prob_op); + // OPERATION read + if (FLAGS_use_multi_get_entity) { + constexpr uint64_t max_batch_size = 64; + const uint64_t batch_size = std::min( + static_cast(thread->rand.Uniform(max_batch_size)) + 1, + ops_per_open - i); + assert(batch_size >= 1); + assert(batch_size <= max_batch_size); + assert(i + batch_size <= ops_per_open); + + rand_keys = GenerateNKeys(thread, static_cast(batch_size), i); + + TestMultiGetEntity(thread, read_opts, rand_column_families, + rand_keys); + + i += batch_size - 1; + } else if (FLAGS_use_get_entity) { + TestGetEntity(thread, read_opts, rand_column_families, rand_keys); + } else if (FLAGS_use_multiget) { + // Leave room for one more iteration of the loop with a single key + // batch. This is to ensure that each thread does exactly the same + // number of ops + int multiget_batch_size = static_cast( + std::min(static_cast(thread->rand.Uniform(64)), + FLAGS_ops_per_thread - i - 1)); + // If its the last iteration, ensure that multiget_batch_size is 1 + multiget_batch_size = std::max(multiget_batch_size, 1); + rand_keys = GenerateNKeys(thread, multiget_batch_size, i); + TestMultiGet(thread, read_opts, rand_column_families, rand_keys); + i += multiget_batch_size - 1; + } else { + TestGet(thread, read_opts, rand_column_families, rand_keys); + } + } else if (prob_op < prefix_bound) { + assert(static_cast(FLAGS_readpercent) <= prob_op); + // OPERATION prefix scan + // keys are 8 bytes long, prefix size is FLAGS_prefix_size. There are + // (8 - FLAGS_prefix_size) bytes besides the prefix. So there will + // be 2 ^ ((8 - FLAGS_prefix_size) * 8) possible keys with the same + // prefix + TestPrefixScan(thread, read_opts, rand_column_families, rand_keys); + } else if (prob_op < write_bound) { + assert(prefix_bound <= prob_op); + // OPERATION write + TestPut(thread, write_opts, read_opts, rand_column_families, rand_keys, + value); + } else if (prob_op < del_bound) { + assert(write_bound <= prob_op); + // OPERATION delete + TestDelete(thread, write_opts, rand_column_families, rand_keys); + } else if (prob_op < delrange_bound) { + assert(del_bound <= prob_op); + // OPERATION delete range + TestDeleteRange(thread, write_opts, rand_column_families, rand_keys); + } else if (prob_op < iterate_bound) { + assert(delrange_bound <= prob_op); + // OPERATION iterate + if (!FLAGS_skip_verifydb && + thread->rand.OneInOpt( + FLAGS_verify_iterator_with_expected_state_one_in)) { + TestIterateAgainstExpected(thread, read_opts, rand_column_families, + rand_keys); + } else { + int num_seeks = static_cast(std::min( + std::max(static_cast(thread->rand.Uniform(4)), + static_cast(1)), + std::max(static_cast(FLAGS_ops_per_thread - i - 1), + static_cast(1)))); + rand_keys = GenerateNKeys(thread, num_seeks, i); + i += num_seeks - 1; + TestIterate(thread, read_opts, rand_column_families, rand_keys); + } + } else { + assert(iterate_bound <= prob_op); + TestCustomOperations(thread, rand_column_families); + } + thread->stats.FinishedSingleOp(); + } + } + while (!thread->snapshot_queue.empty()) { + db_->ReleaseSnapshot(thread->snapshot_queue.front().second.snapshot); + delete thread->snapshot_queue.front().second.key_vec; + thread->snapshot_queue.pop(); + } + + thread->stats.Stop(); +} + +// Generated a list of keys that close to boundaries of SST keys. +// If there isn't any SST file in the DB, return empty list. +std::vector StressTest::GetWhiteBoxKeys(ThreadState* thread, + DB* db, + ColumnFamilyHandle* cfh, + size_t num_keys) { + ColumnFamilyMetaData cfmd; + db->GetColumnFamilyMetaData(cfh, &cfmd); + std::vector boundaries; + for (const LevelMetaData& lmd : cfmd.levels) { + for (const SstFileMetaData& sfmd : lmd.files) { + // If FLAGS_user_timestamp_size > 0, then both smallestkey and largestkey + // have timestamps. + const auto& skey = sfmd.smallestkey; + const auto& lkey = sfmd.largestkey; + assert(skey.size() >= FLAGS_user_timestamp_size); + assert(lkey.size() >= FLAGS_user_timestamp_size); + boundaries.push_back( + skey.substr(0, skey.size() - FLAGS_user_timestamp_size)); + boundaries.push_back( + lkey.substr(0, lkey.size() - FLAGS_user_timestamp_size)); + } + } + if (boundaries.empty()) { + return {}; + } + + std::vector ret; + for (size_t j = 0; j < num_keys; j++) { + std::string k = + boundaries[thread->rand.Uniform(static_cast(boundaries.size()))]; + if (thread->rand.OneIn(3)) { + // Reduce one byte from the string + for (int i = static_cast(k.length()) - 1; i >= 0; i--) { + uint8_t cur = k[i]; + if (cur > 0) { + k[i] = static_cast(cur - 1); + break; + } else if (i > 0) { + k[i] = 0xFFu; + } + } + } else if (thread->rand.OneIn(2)) { + // Add one byte to the string + for (int i = static_cast(k.length()) - 1; i >= 0; i--) { + uint8_t cur = k[i]; + if (cur < 255) { + k[i] = static_cast(cur + 1); + break; + } else if (i > 0) { + k[i] = 0x00; + } + } + } + ret.push_back(k); + } + return ret; +} + +// Given a key K, this creates an iterator which scans to K and then +// does a random sequence of Next/Prev operations. +Status StressTest::TestIterate(ThreadState* thread, + const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) { + assert(!rand_column_families.empty()); + assert(!rand_keys.empty()); + + ManagedSnapshot snapshot_guard(db_); + + ReadOptions ro = read_opts; + ro.snapshot = snapshot_guard.snapshot(); + + std::string read_ts_str; + Slice read_ts_slice; + MaybeUseOlderTimestampForRangeScan(thread, read_ts_str, read_ts_slice, ro); + + bool expect_total_order = false; + if (thread->rand.OneIn(16)) { + // When prefix extractor is used, it's useful to cover total order seek. + ro.total_order_seek = true; + expect_total_order = true; + } else if (thread->rand.OneIn(4)) { + ro.total_order_seek = false; + ro.auto_prefix_mode = true; + expect_total_order = true; + } else if (options_.prefix_extractor.get() == nullptr) { + expect_total_order = true; + } + + std::string upper_bound_str; + Slice upper_bound; + if (thread->rand.OneIn(16)) { + // With a 1/16 chance, set an iterator upper bound. + // Note: upper_bound can be smaller than the seek key. + const int64_t rand_upper_key = GenerateOneKey(thread, FLAGS_ops_per_thread); + upper_bound_str = Key(rand_upper_key); + upper_bound = Slice(upper_bound_str); + ro.iterate_upper_bound = &upper_bound; + } + std::string lower_bound_str; + Slice lower_bound; + if (thread->rand.OneIn(16)) { + // With a 1/16 chance, enable iterator lower bound. + // Note: lower_bound can be greater than the seek key. + const int64_t rand_lower_key = GenerateOneKey(thread, FLAGS_ops_per_thread); + lower_bound_str = Key(rand_lower_key); + lower_bound = Slice(lower_bound_str); + ro.iterate_lower_bound = &lower_bound; + } + + ColumnFamilyHandle* const cfh = column_families_[rand_column_families[0]]; + assert(cfh); + + std::unique_ptr iter(db_->NewIterator(ro, cfh)); + + std::vector key_strs; + if (thread->rand.OneIn(16)) { + // Generate keys close to lower or upper bound of SST files. + key_strs = GetWhiteBoxKeys(thread, db_, cfh, rand_keys.size()); + } + if (key_strs.empty()) { + // Use the random keys passed in. + for (int64_t rkey : rand_keys) { + key_strs.push_back(Key(rkey)); + } + } + + std::string op_logs; + constexpr size_t kOpLogsLimit = 10000; + + for (const std::string& key_str : key_strs) { + if (op_logs.size() > kOpLogsLimit) { + // Shouldn't take too much memory for the history log. Clear it. + op_logs = "(cleared...)\n"; + } + + if (ro.iterate_upper_bound != nullptr && thread->rand.OneIn(2)) { + // With a 1/2 chance, change the upper bound. + // It is possible that it is changed before first use, but there is no + // problem with that. + const int64_t rand_upper_key = + GenerateOneKey(thread, FLAGS_ops_per_thread); + upper_bound_str = Key(rand_upper_key); + upper_bound = Slice(upper_bound_str); + } + if (ro.iterate_lower_bound != nullptr && thread->rand.OneIn(4)) { + // With a 1/4 chance, change the lower bound. + // It is possible that it is changed before first use, but there is no + // problem with that. + const int64_t rand_lower_key = + GenerateOneKey(thread, FLAGS_ops_per_thread); + lower_bound_str = Key(rand_lower_key); + lower_bound = Slice(lower_bound_str); + } + + // Record some options to op_logs + op_logs += "total_order_seek: "; + op_logs += (ro.total_order_seek ? "1 " : "0 "); + op_logs += "auto_prefix_mode: "; + op_logs += (ro.auto_prefix_mode ? "1 " : "0 "); + if (ro.iterate_upper_bound != nullptr) { + op_logs += "ub: " + upper_bound.ToString(true) + " "; + } + if (ro.iterate_lower_bound != nullptr) { + op_logs += "lb: " + lower_bound.ToString(true) + " "; + } + + // Set up an iterator, perform the same operations without bounds and with + // total order seek, and compare the results. This is to identify bugs + // related to bounds, prefix extractor, or reseeking. Sometimes we are + // comparing iterators with the same set-up, and it doesn't hurt to check + // them to be equal. + // + // This `ReadOptions` is for validation purposes. Ignore + // `FLAGS_rate_limit_user_ops` to avoid slowing any validation. + ReadOptions cmp_ro; + cmp_ro.timestamp = ro.timestamp; + cmp_ro.iter_start_ts = ro.iter_start_ts; + cmp_ro.snapshot = snapshot_guard.snapshot(); + cmp_ro.total_order_seek = true; + + ColumnFamilyHandle* const cmp_cfh = + GetControlCfh(thread, rand_column_families[0]); + assert(cmp_cfh); + + std::unique_ptr cmp_iter(db_->NewIterator(cmp_ro, cmp_cfh)); + + bool diverged = false; + + Slice key(key_str); + + const bool support_seek_first_or_last = expect_total_order; + + LastIterateOp last_op; + if (support_seek_first_or_last && thread->rand.OneIn(100)) { + iter->SeekToFirst(); + cmp_iter->SeekToFirst(); + last_op = kLastOpSeekToFirst; + op_logs += "STF "; + } else if (support_seek_first_or_last && thread->rand.OneIn(100)) { + iter->SeekToLast(); + cmp_iter->SeekToLast(); + last_op = kLastOpSeekToLast; + op_logs += "STL "; + } else if (thread->rand.OneIn(8)) { + iter->SeekForPrev(key); + cmp_iter->SeekForPrev(key); + last_op = kLastOpSeekForPrev; + op_logs += "SFP " + key.ToString(true) + " "; + } else { + iter->Seek(key); + cmp_iter->Seek(key); + last_op = kLastOpSeek; + op_logs += "S " + key.ToString(true) + " "; + } + + VerifyIterator(thread, cmp_cfh, ro, iter.get(), cmp_iter.get(), last_op, + key, op_logs, &diverged); + + const bool no_reverse = + (FLAGS_memtablerep == "prefix_hash" && !expect_total_order); + for (uint64_t i = 0; i < FLAGS_num_iterations && iter->Valid(); ++i) { + if (no_reverse || thread->rand.OneIn(2)) { + iter->Next(); + if (!diverged) { + assert(cmp_iter->Valid()); + cmp_iter->Next(); + } + op_logs += "N"; + } else { + iter->Prev(); + if (!diverged) { + assert(cmp_iter->Valid()); + cmp_iter->Prev(); + } + op_logs += "P"; + } + + last_op = kLastOpNextOrPrev; + + VerifyIterator(thread, cmp_cfh, ro, iter.get(), cmp_iter.get(), last_op, + key, op_logs, &diverged); + } + + thread->stats.AddIterations(1); + + op_logs += "; "; + } + + return Status::OK(); +} + +// Test the return status of GetLiveFiles. +Status StressTest::VerifyGetLiveFiles() const { + std::vector live_file; + uint64_t manifest_size = 0; + return db_->GetLiveFiles(live_file, &manifest_size); +} + +// Test the return status of GetSortedWalFiles. +Status StressTest::VerifyGetSortedWalFiles() const { + VectorLogPtr log_ptr; + return db_->GetSortedWalFiles(log_ptr); +} + +// Test the return status of GetCurrentWalFile. +Status StressTest::VerifyGetCurrentWalFile() const { + std::unique_ptr cur_wal_file; + return db_->GetCurrentWalFile(&cur_wal_file); +} + +// Compare the two iterator, iter and cmp_iter are in the same position, +// unless iter might be made invalidate or undefined because of +// upper or lower bounds, or prefix extractor. +// Will flag failure if the verification fails. +// diverged = true if the two iterator is already diverged. +// True if verification passed, false if not. +void StressTest::VerifyIterator(ThreadState* thread, + ColumnFamilyHandle* cmp_cfh, + const ReadOptions& ro, Iterator* iter, + Iterator* cmp_iter, LastIterateOp op, + const Slice& seek_key, + const std::string& op_logs, bool* diverged) { + assert(diverged); + + if (*diverged) { + return; + } + + if (ro.iter_start_ts != nullptr) { + assert(FLAGS_user_timestamp_size > 0); + // We currently do not verify iterator when dumping history of internal + // keys. + *diverged = true; + return; + } + + if (op == kLastOpSeekToFirst && ro.iterate_lower_bound != nullptr) { + // SeekToFirst() with lower bound is not well defined. + *diverged = true; + return; + } else if (op == kLastOpSeekToLast && ro.iterate_upper_bound != nullptr) { + // SeekToLast() with higher bound is not well defined. + *diverged = true; + return; + } else if (op == kLastOpSeek && ro.iterate_lower_bound != nullptr && + (options_.comparator->CompareWithoutTimestamp( + *ro.iterate_lower_bound, /*a_has_ts=*/false, seek_key, + /*b_has_ts=*/false) >= 0 || + (ro.iterate_upper_bound != nullptr && + options_.comparator->CompareWithoutTimestamp( + *ro.iterate_lower_bound, /*a_has_ts=*/false, + *ro.iterate_upper_bound, /*b_has_ts*/ false) >= 0))) { + // Lower bound behavior is not well defined if it is larger than + // seek key or upper bound. Disable the check for now. + *diverged = true; + return; + } else if (op == kLastOpSeekForPrev && ro.iterate_upper_bound != nullptr && + (options_.comparator->CompareWithoutTimestamp( + *ro.iterate_upper_bound, /*a_has_ts=*/false, seek_key, + /*b_has_ts=*/false) <= 0 || + (ro.iterate_lower_bound != nullptr && + options_.comparator->CompareWithoutTimestamp( + *ro.iterate_lower_bound, /*a_has_ts=*/false, + *ro.iterate_upper_bound, /*b_has_ts=*/false) >= 0))) { + // Uppder bound behavior is not well defined if it is smaller than + // seek key or lower bound. Disable the check for now. + *diverged = true; + return; + } + + const SliceTransform* pe = (ro.total_order_seek || ro.auto_prefix_mode) + ? nullptr + : options_.prefix_extractor.get(); + const Comparator* cmp = options_.comparator; + + if (iter->Valid() && !cmp_iter->Valid()) { + if (pe != nullptr) { + if (!pe->InDomain(seek_key)) { + // Prefix seek a non-in-domain key is undefined. Skip checking for + // this scenario. + *diverged = true; + return; + } else if (!pe->InDomain(iter->key())) { + // out of range is iterator key is not in domain anymore. + *diverged = true; + return; + } else if (pe->Transform(iter->key()) != pe->Transform(seek_key)) { + *diverged = true; + return; + } + } + fprintf(stderr, + "Control interator is invalid but iterator has key %s " + "%s\n", + iter->key().ToString(true).c_str(), op_logs.c_str()); + + *diverged = true; + } else if (cmp_iter->Valid()) { + // Iterator is not valid. It can be legimate if it has already been + // out of upper or lower bound, or filtered out by prefix iterator. + const Slice& total_order_key = cmp_iter->key(); + + if (pe != nullptr) { + if (!pe->InDomain(seek_key)) { + // Prefix seek a non-in-domain key is undefined. Skip checking for + // this scenario. + *diverged = true; + return; + } + + if (!pe->InDomain(total_order_key) || + pe->Transform(total_order_key) != pe->Transform(seek_key)) { + // If the prefix is exhausted, the only thing needs to check + // is the iterator isn't return a position in prefix. + // Either way, checking can stop from here. + *diverged = true; + if (!iter->Valid() || !pe->InDomain(iter->key()) || + pe->Transform(iter->key()) != pe->Transform(seek_key)) { + return; + } + fprintf(stderr, + "Iterator stays in prefix but contol doesn't" + " iterator key %s control iterator key %s %s\n", + iter->key().ToString(true).c_str(), + cmp_iter->key().ToString(true).c_str(), op_logs.c_str()); + } + } + // Check upper or lower bounds. + if (!*diverged) { + if ((iter->Valid() && iter->key() != cmp_iter->key()) || + (!iter->Valid() && + (ro.iterate_upper_bound == nullptr || + cmp->CompareWithoutTimestamp(total_order_key, /*a_has_ts=*/false, + *ro.iterate_upper_bound, + /*b_has_ts=*/false) < 0) && + (ro.iterate_lower_bound == nullptr || + cmp->CompareWithoutTimestamp(total_order_key, /*a_has_ts=*/false, + *ro.iterate_lower_bound, + /*b_has_ts=*/false) > 0))) { + fprintf(stderr, + "Iterator diverged from control iterator which" + " has value %s %s\n", + total_order_key.ToString(true).c_str(), op_logs.c_str()); + if (iter->Valid()) { + fprintf(stderr, "iterator has value %s\n", + iter->key().ToString(true).c_str()); + } else { + fprintf(stderr, "iterator is not valid\n"); + } + *diverged = true; + } + } + } + + if (!*diverged && iter->Valid()) { + if (!VerifyWideColumns(iter->value(), iter->columns())) { + fprintf(stderr, + "Value and columns inconsistent for iterator: value: %s, " + "columns: %s\n", + iter->value().ToString(/* hex */ true).c_str(), + WideColumnsToHex(iter->columns()).c_str()); + + *diverged = true; + } + } + + if (*diverged) { + fprintf(stderr, "Control CF %s\n", cmp_cfh->GetName().c_str()); + thread->stats.AddErrors(1); + // Fail fast to preserve the DB state. + thread->shared->SetVerificationFailure(); + } +} + +Status StressTest::TestBackupRestore( + ThreadState* thread, const std::vector& rand_column_families, + const std::vector& rand_keys) { + std::vector> locks; + if (ShouldAcquireMutexOnKey()) { + for (int rand_column_family : rand_column_families) { + // `rand_keys[0]` on each chosen CF will be verified. + locks.emplace_back(new MutexLock( + thread->shared->GetMutexForKey(rand_column_family, rand_keys[0]))); + } + } + + const std::string backup_dir = + FLAGS_db + "/.backup" + std::to_string(thread->tid); + const std::string restore_dir = + FLAGS_db + "/.restore" + std::to_string(thread->tid); + BackupEngineOptions backup_opts(backup_dir); + // For debugging, get info_log from live options + backup_opts.info_log = db_->GetDBOptions().info_log.get(); + if (thread->rand.OneIn(10)) { + backup_opts.share_table_files = false; + } else { + backup_opts.share_table_files = true; + if (thread->rand.OneIn(5)) { + backup_opts.share_files_with_checksum = false; + } else { + backup_opts.share_files_with_checksum = true; + if (thread->rand.OneIn(2)) { + // old + backup_opts.share_files_with_checksum_naming = + BackupEngineOptions::kLegacyCrc32cAndFileSize; + } else { + // new + backup_opts.share_files_with_checksum_naming = + BackupEngineOptions::kUseDbSessionId; + } + if (thread->rand.OneIn(2)) { + backup_opts.share_files_with_checksum_naming = + backup_opts.share_files_with_checksum_naming | + BackupEngineOptions::kFlagIncludeFileSize; + } + } + } + if (thread->rand.OneIn(2)) { + backup_opts.schema_version = 1; + } else { + backup_opts.schema_version = 2; + } + BackupEngine* backup_engine = nullptr; + std::string from = "a backup/restore operation"; + Status s = BackupEngine::Open(db_stress_env, backup_opts, &backup_engine); + if (!s.ok()) { + from = "BackupEngine::Open"; + } + if (s.ok()) { + if (backup_opts.schema_version >= 2 && thread->rand.OneIn(2)) { + TEST_BackupMetaSchemaOptions test_opts; + test_opts.crc32c_checksums = thread->rand.OneIn(2) == 0; + test_opts.file_sizes = thread->rand.OneIn(2) == 0; + TEST_SetBackupMetaSchemaOptions(backup_engine, test_opts); + } + CreateBackupOptions create_opts; + if (FLAGS_disable_wal) { + // The verification can only work when latest value of `key` is backed up, + // which requires flushing in case of WAL disabled. + // + // Note this triggers a flush with a key lock held. Meanwhile, operations + // like flush/compaction may attempt to grab key locks like in + // `DbStressCompactionFilter`. The philosophy around preventing deadlock + // is the background operation key lock acquisition only tries but does + // not wait for the lock. So here in the foreground it is OK to hold the + // lock and wait on a background operation (flush). + create_opts.flush_before_backup = true; + } + s = backup_engine->CreateNewBackup(create_opts, db_); + if (!s.ok()) { + from = "BackupEngine::CreateNewBackup"; + } + } + if (s.ok()) { + delete backup_engine; + backup_engine = nullptr; + s = BackupEngine::Open(db_stress_env, backup_opts, &backup_engine); + if (!s.ok()) { + from = "BackupEngine::Open (again)"; + } + } + std::vector backup_info; + // If inplace_not_restore, we verify the backup by opening it as a + // read-only DB. If !inplace_not_restore, we restore it to a temporary + // directory for verification. + bool inplace_not_restore = thread->rand.OneIn(3); + if (s.ok()) { + backup_engine->GetBackupInfo(&backup_info, + /*include_file_details*/ inplace_not_restore); + if (backup_info.empty()) { + s = Status::NotFound("no backups found"); + from = "BackupEngine::GetBackupInfo"; + } + } + if (s.ok() && thread->rand.OneIn(2)) { + s = backup_engine->VerifyBackup( + backup_info.front().backup_id, + thread->rand.OneIn(2) /* verify_with_checksum */); + if (!s.ok()) { + from = "BackupEngine::VerifyBackup"; + } + } + const bool allow_persistent = thread->tid == 0; // not too many + bool from_latest = false; + int count = static_cast(backup_info.size()); + if (s.ok() && !inplace_not_restore) { + if (count > 1) { + s = backup_engine->RestoreDBFromBackup( + RestoreOptions(), backup_info[thread->rand.Uniform(count)].backup_id, + restore_dir /* db_dir */, restore_dir /* wal_dir */); + if (!s.ok()) { + from = "BackupEngine::RestoreDBFromBackup"; + } + } else { + from_latest = true; + s = backup_engine->RestoreDBFromLatestBackup(RestoreOptions(), + restore_dir /* db_dir */, + restore_dir /* wal_dir */); + if (!s.ok()) { + from = "BackupEngine::RestoreDBFromLatestBackup"; + } + } + } + if (s.ok() && !inplace_not_restore) { + // Purge early if restoring, to ensure the restored directory doesn't + // have some secret dependency on the backup directory. + uint32_t to_keep = 0; + if (allow_persistent) { + // allow one thread to keep up to 2 backups + to_keep = thread->rand.Uniform(3); + } + s = backup_engine->PurgeOldBackups(to_keep); + if (!s.ok()) { + from = "BackupEngine::PurgeOldBackups"; + } + } + DB* restored_db = nullptr; + std::vector restored_cf_handles; + // Not yet implemented: opening restored BlobDB or TransactionDB + if (s.ok() && !FLAGS_use_txn && !FLAGS_use_blob_db) { + Options restore_options(options_); + restore_options.best_efforts_recovery = false; + restore_options.listeners.clear(); + // Avoid dangling/shared file descriptors, for reliable destroy + restore_options.sst_file_manager = nullptr; + std::vector cf_descriptors; + // TODO(ajkr): `column_family_names_` is not safe to access here when + // `clear_column_family_one_in != 0`. But we can't easily switch to + // `ListColumnFamilies` to get names because it won't necessarily give + // the same order as `column_family_names_`. + assert(FLAGS_clear_column_family_one_in == 0); + for (auto name : column_family_names_) { + cf_descriptors.emplace_back(name, ColumnFamilyOptions(restore_options)); + } + if (inplace_not_restore) { + BackupInfo& info = backup_info[thread->rand.Uniform(count)]; + restore_options.env = info.env_for_open.get(); + s = DB::OpenForReadOnly(DBOptions(restore_options), info.name_for_open, + cf_descriptors, &restored_cf_handles, + &restored_db); + if (!s.ok()) { + from = "DB::OpenForReadOnly in backup/restore"; + } + } else { + s = DB::Open(DBOptions(restore_options), restore_dir, cf_descriptors, + &restored_cf_handles, &restored_db); + if (!s.ok()) { + from = "DB::Open in backup/restore"; + } + } + } + // Note the column families chosen by `rand_column_families` cannot be + // dropped while the locks for `rand_keys` are held. So we should not have + // to worry about accessing those column families throughout this function. + // + // For simplicity, currently only verifies existence/non-existence of a + // single key + for (size_t i = 0; restored_db && s.ok() && i < rand_column_families.size(); + ++i) { + std::string key_str = Key(rand_keys[0]); + Slice key = key_str; + std::string restored_value; + // This `ReadOptions` is for validation purposes. Ignore + // `FLAGS_rate_limit_user_ops` to avoid slowing any validation. + ReadOptions read_opts; + std::string ts_str; + Slice ts; + if (FLAGS_user_timestamp_size > 0) { + ts_str = GetNowNanos(); + ts = ts_str; + read_opts.timestamp = &ts; + } + Status get_status = restored_db->Get( + read_opts, restored_cf_handles[rand_column_families[i]], key, + &restored_value); + bool exists = thread->shared->Exists(rand_column_families[i], rand_keys[0]); + if (get_status.ok()) { + if (!exists && from_latest && ShouldAcquireMutexOnKey()) { + std::ostringstream oss; + oss << "0x" << key.ToString(true) + << " exists in restore but not in original db"; + s = Status::Corruption(oss.str()); + } + } else if (get_status.IsNotFound()) { + if (exists && from_latest && ShouldAcquireMutexOnKey()) { + std::ostringstream oss; + oss << "0x" << key.ToString(true) + << " exists in original db but not in restore"; + s = Status::Corruption(oss.str()); + } + } else { + s = get_status; + if (!s.ok()) { + from = "DB::Get in backup/restore"; + } + } + } + if (restored_db != nullptr) { + for (auto* cf_handle : restored_cf_handles) { + restored_db->DestroyColumnFamilyHandle(cf_handle); + } + delete restored_db; + restored_db = nullptr; + } + if (s.ok() && inplace_not_restore) { + // Purge late if inplace open read-only + uint32_t to_keep = 0; + if (allow_persistent) { + // allow one thread to keep up to 2 backups + to_keep = thread->rand.Uniform(3); + } + s = backup_engine->PurgeOldBackups(to_keep); + if (!s.ok()) { + from = "BackupEngine::PurgeOldBackups"; + } + } + if (backup_engine != nullptr) { + delete backup_engine; + backup_engine = nullptr; + } + if (s.ok()) { + // Preserve directories on failure, or allowed persistent backup + if (!allow_persistent) { + s = DestroyDir(db_stress_env, backup_dir); + if (!s.ok()) { + from = "Destroy backup dir"; + } + } + } + if (s.ok()) { + s = DestroyDir(db_stress_env, restore_dir); + if (!s.ok()) { + from = "Destroy restore dir"; + } + } + if (!s.ok()) { + fprintf(stderr, "Failure in %s with: %s\n", from.c_str(), + s.ToString().c_str()); + } + return s; +} + +Status StressTest::TestApproximateSize( + ThreadState* thread, uint64_t iteration, + const std::vector& rand_column_families, + const std::vector& rand_keys) { + // rand_keys likely only has one key. Just use the first one. + assert(!rand_keys.empty()); + assert(!rand_column_families.empty()); + int64_t key1 = rand_keys[0]; + int64_t key2; + if (thread->rand.OneIn(2)) { + // Two totally random keys. This tends to cover large ranges. + key2 = GenerateOneKey(thread, iteration); + if (key2 < key1) { + std::swap(key1, key2); + } + } else { + // Unless users pass a very large FLAGS_max_key, it we should not worry + // about overflow. It is for testing, so we skip the overflow checking + // for simplicity. + key2 = key1 + static_cast(thread->rand.Uniform(1000)); + } + std::string key1_str = Key(key1); + std::string key2_str = Key(key2); + Range range{Slice(key1_str), Slice(key2_str)}; + SizeApproximationOptions sao; + sao.include_memtables = thread->rand.OneIn(2); + if (sao.include_memtables) { + sao.include_files = thread->rand.OneIn(2); + } + if (thread->rand.OneIn(2)) { + if (thread->rand.OneIn(2)) { + sao.files_size_error_margin = 0.0; + } else { + sao.files_size_error_margin = + static_cast(thread->rand.Uniform(3)); + } + } + uint64_t result; + return db_->GetApproximateSizes( + sao, column_families_[rand_column_families[0]], &range, 1, &result); +} + +Status StressTest::TestCheckpoint(ThreadState* thread, + const std::vector& rand_column_families, + const std::vector& rand_keys) { + std::vector> locks; + if (ShouldAcquireMutexOnKey()) { + for (int rand_column_family : rand_column_families) { + // `rand_keys[0]` on each chosen CF will be verified. + locks.emplace_back(new MutexLock( + thread->shared->GetMutexForKey(rand_column_family, rand_keys[0]))); + } + } + + std::string checkpoint_dir = + FLAGS_db + "/.checkpoint" + std::to_string(thread->tid); + Options tmp_opts(options_); + tmp_opts.listeners.clear(); + tmp_opts.env = db_stress_env; + // Avoid delayed deletion so whole directory can be deleted + tmp_opts.sst_file_manager.reset(); + + DestroyDB(checkpoint_dir, tmp_opts); + + Checkpoint* checkpoint = nullptr; + Status s = Checkpoint::Create(db_, &checkpoint); + if (s.ok()) { + s = checkpoint->CreateCheckpoint(checkpoint_dir); + if (!s.ok()) { + fprintf(stderr, "Fail to create checkpoint to %s\n", + checkpoint_dir.c_str()); + std::vector files; + Status my_s = db_stress_env->GetChildren(checkpoint_dir, &files); + if (my_s.ok()) { + for (const auto& f : files) { + fprintf(stderr, " %s\n", f.c_str()); + } + } else { + fprintf(stderr, "Fail to get files under the directory to %s\n", + my_s.ToString().c_str()); + } + } + } + delete checkpoint; + checkpoint = nullptr; + std::vector cf_handles; + DB* checkpoint_db = nullptr; + if (s.ok()) { + Options options(options_); + options.best_efforts_recovery = false; + options.listeners.clear(); + // Avoid race condition in trash handling after delete checkpoint_db + options.sst_file_manager.reset(); + std::vector cf_descs; + // TODO(ajkr): `column_family_names_` is not safe to access here when + // `clear_column_family_one_in != 0`. But we can't easily switch to + // `ListColumnFamilies` to get names because it won't necessarily give + // the same order as `column_family_names_`. + assert(FLAGS_clear_column_family_one_in == 0); + if (FLAGS_clear_column_family_one_in == 0) { + for (const auto& name : column_family_names_) { + cf_descs.emplace_back(name, ColumnFamilyOptions(options)); + } + s = DB::OpenForReadOnly(DBOptions(options), checkpoint_dir, cf_descs, + &cf_handles, &checkpoint_db); + } + } + if (checkpoint_db != nullptr) { + // Note the column families chosen by `rand_column_families` cannot be + // dropped while the locks for `rand_keys` are held. So we should not have + // to worry about accessing those column families throughout this function. + for (size_t i = 0; s.ok() && i < rand_column_families.size(); ++i) { + std::string key_str = Key(rand_keys[0]); + Slice key = key_str; + std::string ts_str; + Slice ts; + ReadOptions read_opts; + if (FLAGS_user_timestamp_size > 0) { + ts_str = GetNowNanos(); + ts = ts_str; + read_opts.timestamp = &ts; + } + std::string value; + Status get_status = checkpoint_db->Get( + read_opts, cf_handles[rand_column_families[i]], key, &value); + bool exists = + thread->shared->Exists(rand_column_families[i], rand_keys[0]); + if (get_status.ok()) { + if (!exists && ShouldAcquireMutexOnKey()) { + std::ostringstream oss; + oss << "0x" << key.ToString(true) << " exists in checkpoint " + << checkpoint_dir << " but not in original db"; + s = Status::Corruption(oss.str()); + } + } else if (get_status.IsNotFound()) { + if (exists && ShouldAcquireMutexOnKey()) { + std::ostringstream oss; + oss << "0x" << key.ToString(true) + << " exists in original db but not in checkpoint " + << checkpoint_dir; + s = Status::Corruption(oss.str()); + } + } else { + s = get_status; + } + } + for (auto cfh : cf_handles) { + delete cfh; + } + cf_handles.clear(); + delete checkpoint_db; + checkpoint_db = nullptr; + } + + if (!s.ok()) { + fprintf(stderr, "A checkpoint operation failed with: %s\n", + s.ToString().c_str()); + } else { + DestroyDB(checkpoint_dir, tmp_opts); + } + return s; +} + +void StressTest::TestGetProperty(ThreadState* thread) const { + std::unordered_set levelPropertyNames = { + DB::Properties::kAggregatedTablePropertiesAtLevel, + DB::Properties::kCompressionRatioAtLevelPrefix, + DB::Properties::kNumFilesAtLevelPrefix, + }; + std::unordered_set unknownPropertyNames = { + DB::Properties::kEstimateOldestKeyTime, + DB::Properties::kOptionsStatistics, + DB::Properties:: + kLiveSstFilesSizeAtTemperature, // similar to levelPropertyNames, it + // requires a number suffix + }; + unknownPropertyNames.insert(levelPropertyNames.begin(), + levelPropertyNames.end()); + + std::unordered_set blobCachePropertyNames = { + DB::Properties::kBlobCacheCapacity, + DB::Properties::kBlobCacheUsage, + DB::Properties::kBlobCachePinnedUsage, + }; + if (db_->GetOptions().blob_cache == nullptr) { + unknownPropertyNames.insert(blobCachePropertyNames.begin(), + blobCachePropertyNames.end()); + } + + std::string prop; + for (const auto& ppt_name_and_info : InternalStats::ppt_name_to_info) { + bool res = db_->GetProperty(ppt_name_and_info.first, &prop); + if (unknownPropertyNames.find(ppt_name_and_info.first) == + unknownPropertyNames.end()) { + if (!res) { + fprintf(stderr, "Failed to get DB property: %s\n", + ppt_name_and_info.first.c_str()); + thread->shared->SetVerificationFailure(); + } + if (ppt_name_and_info.second.handle_int != nullptr) { + uint64_t prop_int; + if (!db_->GetIntProperty(ppt_name_and_info.first, &prop_int)) { + fprintf(stderr, "Failed to get Int property: %s\n", + ppt_name_and_info.first.c_str()); + thread->shared->SetVerificationFailure(); + } + } + if (ppt_name_and_info.second.handle_map != nullptr) { + std::map prop_map; + if (!db_->GetMapProperty(ppt_name_and_info.first, &prop_map)) { + fprintf(stderr, "Failed to get Map property: %s\n", + ppt_name_and_info.first.c_str()); + thread->shared->SetVerificationFailure(); + } + } + } + } + + ROCKSDB_NAMESPACE::ColumnFamilyMetaData cf_meta_data; + db_->GetColumnFamilyMetaData(&cf_meta_data); + int level_size = static_cast(cf_meta_data.levels.size()); + for (int level = 0; level < level_size; level++) { + for (const auto& ppt_name : levelPropertyNames) { + bool res = db_->GetProperty(ppt_name + std::to_string(level), &prop); + if (!res) { + fprintf(stderr, "Failed to get DB property: %s\n", + (ppt_name + std::to_string(level)).c_str()); + thread->shared->SetVerificationFailure(); + } + } + } + + // Test for an invalid property name + if (thread->rand.OneIn(100)) { + if (db_->GetProperty("rocksdb.invalid_property_name", &prop)) { + fprintf(stderr, "Failed to return false for invalid property name\n"); + thread->shared->SetVerificationFailure(); + } + } +} + +void StressTest::TestCompactFiles(ThreadState* thread, + ColumnFamilyHandle* column_family) { + ROCKSDB_NAMESPACE::ColumnFamilyMetaData cf_meta_data; + db_->GetColumnFamilyMetaData(column_family, &cf_meta_data); + + if (cf_meta_data.levels.empty()) { + return; + } + + // Randomly compact up to three consecutive files from a level + const int kMaxRetry = 3; + for (int attempt = 0; attempt < kMaxRetry; ++attempt) { + size_t random_level = + thread->rand.Uniform(static_cast(cf_meta_data.levels.size())); + + const auto& files = cf_meta_data.levels[random_level].files; + if (files.size() > 0) { + size_t random_file_index = + thread->rand.Uniform(static_cast(files.size())); + if (files[random_file_index].being_compacted) { + // Retry as the selected file is currently being compacted + continue; + } + + std::vector input_files; + input_files.push_back(files[random_file_index].name); + if (random_file_index > 0 && + !files[random_file_index - 1].being_compacted) { + input_files.push_back(files[random_file_index - 1].name); + } + if (random_file_index + 1 < files.size() && + !files[random_file_index + 1].being_compacted) { + input_files.push_back(files[random_file_index + 1].name); + } + + size_t output_level = + std::min(random_level + 1, cf_meta_data.levels.size() - 1); + auto s = db_->CompactFiles(CompactionOptions(), column_family, + input_files, static_cast(output_level)); + if (!s.ok()) { + fprintf(stdout, "Unable to perform CompactFiles(): %s\n", + s.ToString().c_str()); + thread->stats.AddNumCompactFilesFailed(1); + } else { + thread->stats.AddNumCompactFilesSucceed(1); + } + break; + } + } +} + +Status StressTest::TestFlush(const std::vector& rand_column_families) { + FlushOptions flush_opts; + if (FLAGS_atomic_flush) { + return db_->Flush(flush_opts, column_families_); + } + std::vector cfhs; + std::for_each(rand_column_families.begin(), rand_column_families.end(), + [this, &cfhs](int k) { cfhs.push_back(column_families_[k]); }); + return db_->Flush(flush_opts, cfhs); +} + +Status StressTest::TestPauseBackground(ThreadState* thread) { + Status status = db_->PauseBackgroundWork(); + if (!status.ok()) { + return status; + } + // To avoid stalling/deadlocking ourself in this thread, just + // sleep here during pause and let other threads do db operations. + // Sleep up to ~16 seconds (2**24 microseconds), but very skewed + // toward short pause. (1 chance in 25 of pausing >= 1s; + // 1 chance in 625 of pausing full 16s.) + int pwr2_micros = + std::min(thread->rand.Uniform(25), thread->rand.Uniform(25)); + clock_->SleepForMicroseconds(1 << pwr2_micros); + return db_->ContinueBackgroundWork(); +} + +void StressTest::TestAcquireSnapshot(ThreadState* thread, + int rand_column_family, + const std::string& keystr, uint64_t i) { + Slice key = keystr; + ColumnFamilyHandle* column_family = column_families_[rand_column_family]; + // This `ReadOptions` is for validation purposes. Ignore + // `FLAGS_rate_limit_user_ops` to avoid slowing any validation. + ReadOptions ropt; + auto db_impl = static_cast_with_check(db_->GetRootDB()); + const bool ww_snapshot = thread->rand.OneIn(10); + const Snapshot* snapshot = + ww_snapshot ? db_impl->GetSnapshotForWriteConflictBoundary() + : db_->GetSnapshot(); + ropt.snapshot = snapshot; + + // Ideally, we want snapshot taking and timestamp generation to be atomic + // here, so that the snapshot corresponds to the timestamp. However, it is + // not possible with current GetSnapshot() API. + std::string ts_str; + Slice ts; + if (FLAGS_user_timestamp_size > 0) { + ts_str = GetNowNanos(); + ts = ts_str; + ropt.timestamp = &ts; + } + + std::string value_at; + // When taking a snapshot, we also read a key from that snapshot. We + // will later read the same key before releasing the snapshot and + // verify that the results are the same. + auto status_at = db_->Get(ropt, column_family, key, &value_at); + std::vector* key_vec = nullptr; + + if (FLAGS_compare_full_db_state_snapshot && (thread->tid == 0)) { + key_vec = new std::vector(FLAGS_max_key); + // When `prefix_extractor` is set, seeking to beginning and scanning + // across prefixes are only supported with `total_order_seek` set. + ropt.total_order_seek = true; + std::unique_ptr iterator(db_->NewIterator(ropt)); + for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) { + uint64_t key_val; + if (GetIntVal(iterator->key().ToString(), &key_val)) { + (*key_vec)[key_val] = true; + } + } + } + + ThreadState::SnapshotState snap_state = {snapshot, + rand_column_family, + column_family->GetName(), + keystr, + status_at, + value_at, + key_vec, + ts_str}; + uint64_t hold_for = FLAGS_snapshot_hold_ops; + if (FLAGS_long_running_snapshots) { + // Hold 10% of snapshots for 10x more + if (thread->rand.OneIn(10)) { + assert(hold_for < std::numeric_limits::max() / 10); + hold_for *= 10; + // Hold 1% of snapshots for 100x more + if (thread->rand.OneIn(10)) { + assert(hold_for < std::numeric_limits::max() / 10); + hold_for *= 10; + } + } + } + uint64_t release_at = std::min(FLAGS_ops_per_thread - 1, i + hold_for); + thread->snapshot_queue.emplace(release_at, snap_state); +} + +Status StressTest::MaybeReleaseSnapshots(ThreadState* thread, uint64_t i) { + while (!thread->snapshot_queue.empty() && + i >= thread->snapshot_queue.front().first) { + auto snap_state = thread->snapshot_queue.front().second; + assert(snap_state.snapshot); + // Note: this is unsafe as the cf might be dropped concurrently. But + // it is ok since unclean cf drop is cunnrently not supported by write + // prepared transactions. + Status s = AssertSame(db_, column_families_[snap_state.cf_at], snap_state); + db_->ReleaseSnapshot(snap_state.snapshot); + delete snap_state.key_vec; + thread->snapshot_queue.pop(); + if (!s.ok()) { + return s; + } + } + return Status::OK(); +} + +void StressTest::TestCompactRange(ThreadState* thread, int64_t rand_key, + const Slice& start_key, + ColumnFamilyHandle* column_family) { + int64_t end_key_num; + if (std::numeric_limits::max() - rand_key < + FLAGS_compact_range_width) { + end_key_num = std::numeric_limits::max(); + } else { + end_key_num = FLAGS_compact_range_width + rand_key; + } + std::string end_key_buf = Key(end_key_num); + Slice end_key(end_key_buf); + + CompactRangeOptions cro; + cro.exclusive_manual_compaction = static_cast(thread->rand.Next() % 2); + cro.change_level = static_cast(thread->rand.Next() % 2); + std::vector bottom_level_styles = { + BottommostLevelCompaction::kSkip, + BottommostLevelCompaction::kIfHaveCompactionFilter, + BottommostLevelCompaction::kForce, + BottommostLevelCompaction::kForceOptimized}; + cro.bottommost_level_compaction = + bottom_level_styles[thread->rand.Next() % + static_cast(bottom_level_styles.size())]; + cro.allow_write_stall = static_cast(thread->rand.Next() % 2); + cro.max_subcompactions = static_cast(thread->rand.Next() % 4); + std::vector blob_gc_policies = { + BlobGarbageCollectionPolicy::kForce, + BlobGarbageCollectionPolicy::kDisable, + BlobGarbageCollectionPolicy::kUseDefault}; + cro.blob_garbage_collection_policy = + blob_gc_policies[thread->rand.Next() % + static_cast(blob_gc_policies.size())]; + cro.blob_garbage_collection_age_cutoff = + static_cast(thread->rand.Next() % 100) / 100.0; + + const Snapshot* pre_snapshot = nullptr; + uint32_t pre_hash = 0; + if (thread->rand.OneIn(2)) { + // Do some validation by declaring a snapshot and compare the data before + // and after the compaction + pre_snapshot = db_->GetSnapshot(); + pre_hash = + GetRangeHash(thread, pre_snapshot, column_family, start_key, end_key); + } + + Status status = db_->CompactRange(cro, column_family, &start_key, &end_key); + + if (!status.ok()) { + fprintf(stdout, "Unable to perform CompactRange(): %s\n", + status.ToString().c_str()); + } + + if (pre_snapshot != nullptr) { + uint32_t post_hash = + GetRangeHash(thread, pre_snapshot, column_family, start_key, end_key); + if (pre_hash != post_hash) { + fprintf(stderr, + "Data hash different before and after compact range " + "start_key %s end_key %s\n", + start_key.ToString(true).c_str(), end_key.ToString(true).c_str()); + thread->stats.AddErrors(1); + // Fail fast to preserve the DB state. + thread->shared->SetVerificationFailure(); + } + db_->ReleaseSnapshot(pre_snapshot); + } +} + +uint32_t StressTest::GetRangeHash(ThreadState* thread, const Snapshot* snapshot, + ColumnFamilyHandle* column_family, + const Slice& start_key, + const Slice& end_key) { + // This `ReadOptions` is for validation purposes. Ignore + // `FLAGS_rate_limit_user_ops` to avoid slowing any validation. + ReadOptions ro; + ro.snapshot = snapshot; + ro.total_order_seek = true; + std::string ts_str; + Slice ts; + if (FLAGS_user_timestamp_size > 0) { + ts_str = GetNowNanos(); + ts = ts_str; + ro.timestamp = &ts; + } + + std::unique_ptr it(db_->NewIterator(ro, column_family)); + + constexpr char kCrcCalculatorSepearator = ';'; + + uint32_t crc = 0; + + for (it->Seek(start_key); + it->Valid() && options_.comparator->Compare(it->key(), end_key) <= 0; + it->Next()) { + crc = crc32c::Extend(crc, it->key().data(), it->key().size()); + crc = crc32c::Extend(crc, &kCrcCalculatorSepearator, sizeof(char)); + crc = crc32c::Extend(crc, it->value().data(), it->value().size()); + crc = crc32c::Extend(crc, &kCrcCalculatorSepearator, sizeof(char)); + + for (const auto& column : it->columns()) { + crc = crc32c::Extend(crc, column.name().data(), column.name().size()); + crc = crc32c::Extend(crc, &kCrcCalculatorSepearator, sizeof(char)); + crc = crc32c::Extend(crc, column.value().data(), column.value().size()); + crc = crc32c::Extend(crc, &kCrcCalculatorSepearator, sizeof(char)); + } + } + + if (!it->status().ok()) { + fprintf(stderr, "Iterator non-OK when calculating range CRC: %s\n", + it->status().ToString().c_str()); + thread->stats.AddErrors(1); + // Fail fast to preserve the DB state. + thread->shared->SetVerificationFailure(); + } + + return crc; +} + +void StressTest::PrintEnv() const { + fprintf(stdout, "RocksDB version : %d.%d\n", kMajorVersion, + kMinorVersion); + fprintf(stdout, "Format version : %d\n", FLAGS_format_version); + fprintf(stdout, "TransactionDB : %s\n", + FLAGS_use_txn ? "true" : "false"); + if (FLAGS_use_txn) { + fprintf(stdout, "TransactionDB Type : %s\n", + FLAGS_use_optimistic_txn ? "Optimistic" : "Pessimistic"); + if (FLAGS_use_optimistic_txn) { + fprintf(stdout, "OCC Validation Type : %d\n", + static_cast(FLAGS_occ_validation_policy)); + if (static_cast(OccValidationPolicy::kValidateParallel) == + FLAGS_occ_validation_policy) { + fprintf(stdout, "Share Lock Buckets : %s\n", + FLAGS_share_occ_lock_buckets ? "true" : "false"); + if (FLAGS_share_occ_lock_buckets) { + fprintf(stdout, "Lock Bucket Count : %d\n", + static_cast(FLAGS_occ_lock_bucket_count)); + } + } + } else { + fprintf(stdout, "Two write queues: : %s\n", + FLAGS_two_write_queues ? "true" : "false"); + fprintf(stdout, "Write policy : %d\n", + static_cast(FLAGS_txn_write_policy)); + if (static_cast(TxnDBWritePolicy::WRITE_PREPARED) == + FLAGS_txn_write_policy || + static_cast(TxnDBWritePolicy::WRITE_UNPREPARED) == + FLAGS_txn_write_policy) { + fprintf(stdout, "Snapshot cache bits : %d\n", + static_cast(FLAGS_wp_snapshot_cache_bits)); + fprintf(stdout, "Commit cache bits : %d\n", + static_cast(FLAGS_wp_commit_cache_bits)); + } + fprintf(stdout, "last cwb for recovery : %s\n", + FLAGS_use_only_the_last_commit_time_batch_for_recovery ? "true" + : "false"); + } + } + + fprintf(stdout, "Stacked BlobDB : %s\n", + FLAGS_use_blob_db ? "true" : "false"); + fprintf(stdout, "Read only mode : %s\n", + FLAGS_read_only ? "true" : "false"); + fprintf(stdout, "Atomic flush : %s\n", + FLAGS_atomic_flush ? "true" : "false"); + fprintf(stdout, "Manual WAL flush : %s\n", + FLAGS_manual_wal_flush_one_in > 0 ? "true" : "false"); + fprintf(stdout, "Column families : %d\n", FLAGS_column_families); + if (!FLAGS_test_batches_snapshots) { + fprintf(stdout, "Clear CFs one in : %d\n", + FLAGS_clear_column_family_one_in); + } + fprintf(stdout, "Number of threads : %d\n", FLAGS_threads); + fprintf(stdout, "Ops per thread : %lu\n", + (unsigned long)FLAGS_ops_per_thread); + std::string ttl_state("unused"); + if (FLAGS_ttl > 0) { + ttl_state = std::to_string(FLAGS_ttl); + } + fprintf(stdout, "Time to live(sec) : %s\n", ttl_state.c_str()); + fprintf(stdout, "Read percentage : %d%%\n", FLAGS_readpercent); + fprintf(stdout, "Prefix percentage : %d%%\n", FLAGS_prefixpercent); + fprintf(stdout, "Write percentage : %d%%\n", FLAGS_writepercent); + fprintf(stdout, "Delete percentage : %d%%\n", FLAGS_delpercent); + fprintf(stdout, "Delete range percentage : %d%%\n", FLAGS_delrangepercent); + fprintf(stdout, "No overwrite percentage : %d%%\n", + FLAGS_nooverwritepercent); + fprintf(stdout, "Iterate percentage : %d%%\n", FLAGS_iterpercent); + fprintf(stdout, "Custom ops percentage : %d%%\n", FLAGS_customopspercent); + fprintf(stdout, "DB-write-buffer-size : %" PRIu64 "\n", + FLAGS_db_write_buffer_size); + fprintf(stdout, "Write-buffer-size : %d\n", FLAGS_write_buffer_size); + fprintf(stdout, "Iterations : %lu\n", + (unsigned long)FLAGS_num_iterations); + fprintf(stdout, "Max key : %lu\n", + (unsigned long)FLAGS_max_key); + fprintf(stdout, "Ratio #ops/#keys : %f\n", + (1.0 * FLAGS_ops_per_thread * FLAGS_threads) / FLAGS_max_key); + fprintf(stdout, "Num times DB reopens : %d\n", FLAGS_reopen); + fprintf(stdout, "Batches/snapshots : %d\n", + FLAGS_test_batches_snapshots); + fprintf(stdout, "Do update in place : %d\n", FLAGS_in_place_update); + fprintf(stdout, "Num keys per lock : %d\n", + 1 << FLAGS_log2_keys_per_lock); + std::string compression = CompressionTypeToString(compression_type_e); + fprintf(stdout, "Compression : %s\n", compression.c_str()); + std::string bottommost_compression = + CompressionTypeToString(bottommost_compression_type_e); + fprintf(stdout, "Bottommost Compression : %s\n", + bottommost_compression.c_str()); + std::string checksum = ChecksumTypeToString(checksum_type_e); + fprintf(stdout, "Checksum type : %s\n", checksum.c_str()); + fprintf(stdout, "File checksum impl : %s\n", + FLAGS_file_checksum_impl.c_str()); + fprintf(stdout, "Bloom bits / key : %s\n", + FormatDoubleParam(FLAGS_bloom_bits).c_str()); + fprintf(stdout, "Max subcompactions : %" PRIu64 "\n", + FLAGS_subcompactions); + fprintf(stdout, "Use MultiGet : %s\n", + FLAGS_use_multiget ? "true" : "false"); + fprintf(stdout, "Use GetEntity : %s\n", + FLAGS_use_get_entity ? "true" : "false"); + fprintf(stdout, "Use MultiGetEntity : %s\n", + FLAGS_use_multi_get_entity ? "true" : "false"); + + const char* memtablerep = ""; + switch (FLAGS_rep_factory) { + case kSkipList: + memtablerep = "skip_list"; + break; + case kHashSkipList: + memtablerep = "prefix_hash"; + break; + case kVectorRep: + memtablerep = "vector"; + break; + } + + fprintf(stdout, "Memtablerep : %s\n", memtablerep); + +#ifndef NDEBUG + KillPoint* kp = KillPoint::GetInstance(); + fprintf(stdout, "Test kill odd : %d\n", kp->rocksdb_kill_odds); + if (!kp->rocksdb_kill_exclude_prefixes.empty()) { + fprintf(stdout, "Skipping kill points prefixes:\n"); + for (auto& p : kp->rocksdb_kill_exclude_prefixes) { + fprintf(stdout, " %s\n", p.c_str()); + } + } +#endif + fprintf(stdout, "Periodic Compaction Secs : %" PRIu64 "\n", + FLAGS_periodic_compaction_seconds); + fprintf(stdout, "Compaction TTL : %" PRIu64 "\n", + FLAGS_compaction_ttl); + const char* compaction_pri = ""; + switch (FLAGS_compaction_pri) { + case kByCompensatedSize: + compaction_pri = "kByCompensatedSize"; + break; + case kOldestLargestSeqFirst: + compaction_pri = "kOldestLargestSeqFirst"; + break; + case kOldestSmallestSeqFirst: + compaction_pri = "kOldestSmallestSeqFirst"; + break; + case kMinOverlappingRatio: + compaction_pri = "kMinOverlappingRatio"; + break; + case kRoundRobin: + compaction_pri = "kRoundRobin"; + break; + } + fprintf(stdout, "Compaction Pri : %s\n", compaction_pri); + fprintf(stdout, "Background Purge : %d\n", + static_cast(FLAGS_avoid_unnecessary_blocking_io)); + fprintf(stdout, "Write DB ID to manifest : %d\n", + static_cast(FLAGS_write_dbid_to_manifest)); + fprintf(stdout, "Max Write Batch Group Size: %" PRIu64 "\n", + FLAGS_max_write_batch_group_size_bytes); + fprintf(stdout, "Use dynamic level : %d\n", + static_cast(FLAGS_level_compaction_dynamic_level_bytes)); + fprintf(stdout, "Read fault one in : %d\n", FLAGS_read_fault_one_in); + fprintf(stdout, "Write fault one in : %d\n", FLAGS_write_fault_one_in); + fprintf(stdout, "Open metadata write fault one in:\n"); + fprintf(stdout, " %d\n", + FLAGS_open_metadata_write_fault_one_in); + fprintf(stdout, "Sync fault injection : %d\n", + FLAGS_sync_fault_injection); + fprintf(stdout, "Best efforts recovery : %d\n", + static_cast(FLAGS_best_efforts_recovery)); + fprintf(stdout, "Fail if OPTIONS file error: %d\n", + static_cast(FLAGS_fail_if_options_file_error)); + fprintf(stdout, "User timestamp size bytes : %d\n", + static_cast(FLAGS_user_timestamp_size)); + fprintf(stdout, "WAL compression : %s\n", + FLAGS_wal_compression.c_str()); + fprintf(stdout, "Try verify sst unique id : %d\n", + static_cast(FLAGS_verify_sst_unique_id_in_manifest)); + + fprintf(stdout, "------------------------------------------------\n"); +} + +void StressTest::Open(SharedState* shared) { + assert(db_ == nullptr); + assert(txn_db_ == nullptr); + assert(optimistic_txn_db_ == nullptr); + if (!InitializeOptionsFromFile(options_)) { + InitializeOptionsFromFlags(cache_, filter_policy_, options_); + } + InitializeOptionsGeneral(cache_, filter_policy_, options_); + + if (FLAGS_prefix_size == 0 && FLAGS_rep_factory == kHashSkipList) { + fprintf(stderr, + "prefeix_size cannot be zero if memtablerep == prefix_hash\n"); + exit(1); + } + if (FLAGS_prefix_size != 0 && FLAGS_rep_factory != kHashSkipList) { + fprintf(stderr, + "WARNING: prefix_size is non-zero but " + "memtablerep != prefix_hash\n"); + } + + if ((options_.enable_blob_files || options_.enable_blob_garbage_collection || + FLAGS_allow_setting_blob_options_dynamically) && + FLAGS_best_efforts_recovery) { + fprintf(stderr, + "Integrated BlobDB is currently incompatible with best-effort " + "recovery\n"); + exit(1); + } + + fprintf(stdout, + "Integrated BlobDB: blob files enabled %d, min blob size %" PRIu64 + ", blob file size %" PRIu64 + ", blob compression type %s, blob GC enabled %d, cutoff %f, force " + "threshold %f, blob compaction readahead size %" PRIu64 + ", blob file starting level %d\n", + options_.enable_blob_files, options_.min_blob_size, + options_.blob_file_size, + CompressionTypeToString(options_.blob_compression_type).c_str(), + options_.enable_blob_garbage_collection, + options_.blob_garbage_collection_age_cutoff, + options_.blob_garbage_collection_force_threshold, + options_.blob_compaction_readahead_size, + options_.blob_file_starting_level); + + if (FLAGS_use_blob_cache) { + fprintf(stdout, + "Integrated BlobDB: blob cache enabled" + ", block and blob caches shared: %d", + FLAGS_use_shared_block_and_blob_cache); + if (!FLAGS_use_shared_block_and_blob_cache) { + fprintf(stdout, + ", blob cache size %" PRIu64 ", blob cache num shard bits: %d", + FLAGS_blob_cache_size, FLAGS_blob_cache_numshardbits); + } + fprintf(stdout, ", blob cache prepopulated: %d\n", + FLAGS_prepopulate_blob_cache); + } else { + fprintf(stdout, "Integrated BlobDB: blob cache disabled\n"); + } + + fprintf(stdout, "DB path: [%s]\n", FLAGS_db.c_str()); + + Status s; + + if (FLAGS_ttl == -1) { + std::vector existing_column_families; + s = DB::ListColumnFamilies(DBOptions(options_), FLAGS_db, + &existing_column_families); // ignore errors + if (!s.ok()) { + // DB doesn't exist + assert(existing_column_families.empty()); + assert(column_family_names_.empty()); + column_family_names_.push_back(kDefaultColumnFamilyName); + } else if (column_family_names_.empty()) { + // this is the first call to the function Open() + column_family_names_ = existing_column_families; + } else { + // this is a reopen. just assert that existing column_family_names are + // equivalent to what we remember + auto sorted_cfn = column_family_names_; + std::sort(sorted_cfn.begin(), sorted_cfn.end()); + std::sort(existing_column_families.begin(), + existing_column_families.end()); + if (sorted_cfn != existing_column_families) { + fprintf(stderr, "Expected column families differ from the existing:\n"); + fprintf(stderr, "Expected: {"); + for (auto cf : sorted_cfn) { + fprintf(stderr, "%s ", cf.c_str()); + } + fprintf(stderr, "}\n"); + fprintf(stderr, "Existing: {"); + for (auto cf : existing_column_families) { + fprintf(stderr, "%s ", cf.c_str()); + } + fprintf(stderr, "}\n"); + } + assert(sorted_cfn == existing_column_families); + } + std::vector cf_descriptors; + for (auto name : column_family_names_) { + if (name != kDefaultColumnFamilyName) { + new_column_family_name_ = + std::max(new_column_family_name_.load(), std::stoi(name) + 1); + } + cf_descriptors.emplace_back(name, ColumnFamilyOptions(options_)); + } + while (cf_descriptors.size() < (size_t)FLAGS_column_families) { + std::string name = std::to_string(new_column_family_name_.load()); + new_column_family_name_++; + cf_descriptors.emplace_back(name, ColumnFamilyOptions(options_)); + column_family_names_.push_back(name); + } + + options_.listeners.clear(); + options_.listeners.emplace_back(new DbStressListener( + FLAGS_db, options_.db_paths, cf_descriptors, db_stress_listener_env)); + RegisterAdditionalListeners(); + + if (!FLAGS_use_txn) { + // Determine whether we need to ingest file metadata write failures + // during DB reopen. If it does, enable it. + // Only ingest metadata error if it is reopening, as initial open + // failure doesn't need to be handled. + // TODO cover transaction DB is not covered in this fault test too. + bool ingest_meta_error = false; + bool ingest_write_error = false; + bool ingest_read_error = false; + if ((FLAGS_open_metadata_write_fault_one_in || + FLAGS_open_write_fault_one_in || FLAGS_open_read_fault_one_in) && + fault_fs_guard + ->FileExists(FLAGS_db + "/CURRENT", IOOptions(), nullptr) + .ok()) { + if (!FLAGS_sync) { + // When DB Stress is not sync mode, we expect all WAL writes to + // WAL is durable. Buffering unsynced writes will cause false + // positive in crash tests. Before we figure out a way to + // solve it, skip WAL from failure injection. + fault_fs_guard->SetSkipDirectWritableTypes({kWalFile}); + } + ingest_meta_error = FLAGS_open_metadata_write_fault_one_in; + ingest_write_error = FLAGS_open_write_fault_one_in; + ingest_read_error = FLAGS_open_read_fault_one_in; + if (ingest_meta_error) { + fault_fs_guard->EnableMetadataWriteErrorInjection(); + fault_fs_guard->SetRandomMetadataWriteError( + FLAGS_open_metadata_write_fault_one_in); + } + if (ingest_write_error) { + fault_fs_guard->SetFilesystemDirectWritable(false); + fault_fs_guard->EnableWriteErrorInjection(); + fault_fs_guard->SetRandomWriteError( + static_cast(FLAGS_seed), FLAGS_open_write_fault_one_in, + IOStatus::IOError("Injected Open Error"), + /*inject_for_all_file_types=*/true, /*types=*/{}); + } + if (ingest_read_error) { + fault_fs_guard->SetRandomReadError(FLAGS_open_read_fault_one_in); + } + } + while (true) { + // StackableDB-based BlobDB + if (FLAGS_use_blob_db) { + blob_db::BlobDBOptions blob_db_options; + blob_db_options.min_blob_size = FLAGS_blob_db_min_blob_size; + blob_db_options.bytes_per_sync = FLAGS_blob_db_bytes_per_sync; + blob_db_options.blob_file_size = FLAGS_blob_db_file_size; + blob_db_options.enable_garbage_collection = FLAGS_blob_db_enable_gc; + blob_db_options.garbage_collection_cutoff = FLAGS_blob_db_gc_cutoff; + + blob_db::BlobDB* blob_db = nullptr; + s = blob_db::BlobDB::Open(options_, blob_db_options, FLAGS_db, + cf_descriptors, &column_families_, + &blob_db); + if (s.ok()) { + db_ = blob_db; + } + } else + { + if (db_preload_finished_.load() && FLAGS_read_only) { + s = DB::OpenForReadOnly(DBOptions(options_), FLAGS_db, + cf_descriptors, &column_families_, &db_); + } else { + s = DB::Open(DBOptions(options_), FLAGS_db, cf_descriptors, + &column_families_, &db_); + } + } + + if (ingest_meta_error || ingest_write_error || ingest_read_error) { + fault_fs_guard->SetFilesystemDirectWritable(true); + fault_fs_guard->DisableMetadataWriteErrorInjection(); + fault_fs_guard->DisableWriteErrorInjection(); + fault_fs_guard->SetSkipDirectWritableTypes({}); + fault_fs_guard->SetRandomReadError(0); + if (s.ok()) { + // Ingested errors might happen in background compactions. We + // wait for all compactions to finish to make sure DB is in + // clean state before executing queries. + s = db_->GetRootDB()->WaitForCompact(WaitForCompactOptions()); + if (!s.ok()) { + for (auto cf : column_families_) { + delete cf; + } + column_families_.clear(); + delete db_; + db_ = nullptr; + } + } + if (!s.ok()) { + // After failure to opening a DB due to IO error, retry should + // successfully open the DB with correct data if no IO error shows + // up. + ingest_meta_error = false; + ingest_write_error = false; + ingest_read_error = false; + + Random rand(static_cast(FLAGS_seed)); + if (rand.OneIn(2)) { + fault_fs_guard->DeleteFilesCreatedAfterLastDirSync(IOOptions(), + nullptr); + } + if (rand.OneIn(3)) { + fault_fs_guard->DropUnsyncedFileData(); + } else if (rand.OneIn(2)) { + fault_fs_guard->DropRandomUnsyncedFileData(&rand); + } + continue; + } + } + break; + } + } else { + if (FLAGS_use_optimistic_txn) { + OptimisticTransactionDBOptions optimistic_txn_db_options; + optimistic_txn_db_options.validate_policy = + static_cast(FLAGS_occ_validation_policy); + + if (FLAGS_share_occ_lock_buckets) { + optimistic_txn_db_options.shared_lock_buckets = + MakeSharedOccLockBuckets(FLAGS_occ_lock_bucket_count); + } else { + optimistic_txn_db_options.occ_lock_buckets = + FLAGS_occ_lock_bucket_count; + optimistic_txn_db_options.shared_lock_buckets = nullptr; + } + s = OptimisticTransactionDB::Open( + options_, optimistic_txn_db_options, FLAGS_db, cf_descriptors, + &column_families_, &optimistic_txn_db_); + if (!s.ok()) { + fprintf(stderr, "Error in opening the OptimisticTransactionDB [%s]\n", + s.ToString().c_str()); + fflush(stderr); + } + assert(s.ok()); + { + db_ = optimistic_txn_db_; + db_aptr_.store(optimistic_txn_db_, std::memory_order_release); + } + } else { + TransactionDBOptions txn_db_options; + assert(FLAGS_txn_write_policy <= TxnDBWritePolicy::WRITE_UNPREPARED); + txn_db_options.write_policy = + static_cast(FLAGS_txn_write_policy); + if (FLAGS_unordered_write) { + assert(txn_db_options.write_policy == + TxnDBWritePolicy::WRITE_PREPARED); + options_.unordered_write = true; + options_.two_write_queues = true; + txn_db_options.skip_concurrency_control = true; + } else { + options_.two_write_queues = FLAGS_two_write_queues; + } + txn_db_options.wp_snapshot_cache_bits = + static_cast(FLAGS_wp_snapshot_cache_bits); + txn_db_options.wp_commit_cache_bits = + static_cast(FLAGS_wp_commit_cache_bits); + PrepareTxnDbOptions(shared, txn_db_options); + s = TransactionDB::Open(options_, txn_db_options, FLAGS_db, + cf_descriptors, &column_families_, &txn_db_); + if (!s.ok()) { + fprintf(stderr, "Error in opening the TransactionDB [%s]\n", + s.ToString().c_str()); + fflush(stderr); + } + assert(s.ok()); + + // Do not swap the order of the following. + { + db_ = txn_db_; + db_aptr_.store(txn_db_, std::memory_order_release); + } + } + } + if (!s.ok()) { + fprintf(stderr, "Error in opening the DB [%s]\n", s.ToString().c_str()); + fflush(stderr); + } + assert(s.ok()); + assert(column_families_.size() == + static_cast(FLAGS_column_families)); + + // Secondary instance does not support write-prepared/write-unprepared + // transactions, thus just disable secondary instance if we use + // transaction. + if (s.ok() && FLAGS_test_secondary && !FLAGS_use_txn) { + Options tmp_opts; + // TODO(yanqin) support max_open_files != -1 for secondary instance. + tmp_opts.max_open_files = -1; + tmp_opts.env = db_stress_env; + const std::string& secondary_path = FLAGS_secondaries_base; + s = DB::OpenAsSecondary(tmp_opts, FLAGS_db, secondary_path, + cf_descriptors, &cmp_cfhs_, &cmp_db_); + assert(s.ok()); + assert(cmp_cfhs_.size() == static_cast(FLAGS_column_families)); + } + } else { + DBWithTTL* db_with_ttl; + s = DBWithTTL::Open(options_, FLAGS_db, &db_with_ttl, FLAGS_ttl); + db_ = db_with_ttl; + } + + if (FLAGS_preserve_unverified_changes) { + // Up until now, no live file should have become obsolete due to these + // options. After `DisableFileDeletions()` we can reenable auto compactions + // since, even if live files become obsolete, they won't be deleted. + assert(options_.avoid_flush_during_recovery); + assert(options_.disable_auto_compactions); + if (s.ok()) { + s = db_->DisableFileDeletions(); + } + if (s.ok()) { + s = db_->EnableAutoCompaction(column_families_); + } + } + + if (!s.ok()) { + fprintf(stderr, "open error: %s\n", s.ToString().c_str()); + exit(1); + } +} + +void StressTest::Reopen(ThreadState* thread) { + // BG jobs in WritePrepared must be canceled first because i) they can access + // the db via a callbac ii) they hold on to a snapshot and the upcoming + // ::Close would complain about it. + const bool write_prepared = FLAGS_use_txn && FLAGS_txn_write_policy != 0; + bool bg_canceled __attribute__((unused)) = false; + if (write_prepared || thread->rand.OneIn(2)) { + const bool wait = + write_prepared || static_cast(thread->rand.OneIn(2)); + CancelAllBackgroundWork(db_, wait); + bg_canceled = wait; + } + assert(!write_prepared || bg_canceled); + + for (auto cf : column_families_) { + delete cf; + } + column_families_.clear(); + + if (thread->rand.OneIn(2)) { + Status s = db_->Close(); + if (!s.ok()) { + fprintf(stderr, "Non-ok close status: %s\n", s.ToString().c_str()); + fflush(stderr); + } + assert(s.ok()); + } + assert((txn_db_ == nullptr && optimistic_txn_db_ == nullptr) || + (db_ == txn_db_ || db_ == optimistic_txn_db_)); + delete db_; + db_ = nullptr; + txn_db_ = nullptr; + optimistic_txn_db_ = nullptr; + + num_times_reopened_++; + auto now = clock_->NowMicros(); + fprintf(stdout, "%s Reopening database for the %dth time\n", + clock_->TimeToString(now / 1000000).c_str(), num_times_reopened_); + Open(thread->shared); + + if ((FLAGS_sync_fault_injection || FLAGS_disable_wal || + FLAGS_manual_wal_flush_one_in > 0) && + IsStateTracked()) { + Status s = thread->shared->SaveAtAndAfter(db_); + if (!s.ok()) { + fprintf(stderr, "Error enabling history tracing: %s\n", + s.ToString().c_str()); + exit(1); + } + } +} + +bool StressTest::MaybeUseOlderTimestampForPointLookup(ThreadState* thread, + std::string& ts_str, + Slice& ts_slice, + ReadOptions& read_opts) { + if (FLAGS_user_timestamp_size == 0) { + return false; + } + + assert(thread); + if (!thread->rand.OneInOpt(3)) { + return false; + } + + const SharedState* const shared = thread->shared; + assert(shared); + const uint64_t start_ts = shared->GetStartTimestamp(); + + uint64_t now = db_stress_env->NowNanos(); + + assert(now > start_ts); + uint64_t time_diff = now - start_ts; + uint64_t ts = start_ts + (thread->rand.Next64() % time_diff); + ts_str.clear(); + PutFixed64(&ts_str, ts); + ts_slice = ts_str; + read_opts.timestamp = &ts_slice; + return true; +} + +void StressTest::MaybeUseOlderTimestampForRangeScan(ThreadState* thread, + std::string& ts_str, + Slice& ts_slice, + ReadOptions& read_opts) { + if (FLAGS_user_timestamp_size == 0) { + return; + } + + assert(thread); + if (!thread->rand.OneInOpt(3)) { + return; + } + + const Slice* const saved_ts = read_opts.timestamp; + assert(saved_ts != nullptr); + + const SharedState* const shared = thread->shared; + assert(shared); + const uint64_t start_ts = shared->GetStartTimestamp(); + + uint64_t now = db_stress_env->NowNanos(); + + assert(now > start_ts); + uint64_t time_diff = now - start_ts; + uint64_t ts = start_ts + (thread->rand.Next64() % time_diff); + ts_str.clear(); + PutFixed64(&ts_str, ts); + ts_slice = ts_str; + read_opts.timestamp = &ts_slice; + + // TODO (yanqin): support Merge with iter_start_ts + if (!thread->rand.OneInOpt(3) || FLAGS_use_merge || FLAGS_use_full_merge_v1) { + return; + } + + ts_str.clear(); + PutFixed64(&ts_str, start_ts); + ts_slice = ts_str; + read_opts.iter_start_ts = &ts_slice; + read_opts.timestamp = saved_ts; +} + +void CheckAndSetOptionsForUserTimestamp(Options& options) { + assert(FLAGS_user_timestamp_size > 0); + const Comparator* const cmp = test::BytewiseComparatorWithU64TsWrapper(); + assert(cmp); + if (FLAGS_user_timestamp_size != cmp->timestamp_size()) { + fprintf(stderr, + "Only -user_timestamp_size=%d is supported in stress test.\n", + static_cast(cmp->timestamp_size())); + exit(1); + } + if (FLAGS_use_txn) { + fprintf(stderr, "TransactionDB does not support timestamp yet.\n"); + exit(1); + } + if (FLAGS_test_cf_consistency || FLAGS_test_batches_snapshots) { + fprintf(stderr, + "Due to per-key ts-seq ordering constraint, only the (default) " + "non-batched test is supported with timestamp.\n"); + exit(1); + } + if (FLAGS_ingest_external_file_one_in > 0) { + fprintf(stderr, "Bulk loading may not support timestamp yet.\n"); + exit(1); + } + options.comparator = cmp; +} + +bool InitializeOptionsFromFile(Options& options) { + DBOptions db_options; + ConfigOptions config_options; + config_options.ignore_unknown_options = false; + config_options.input_strings_escaped = true; + config_options.env = db_stress_env; + std::vector cf_descriptors; + if (!FLAGS_options_file.empty()) { + Status s = LoadOptionsFromFile(config_options, FLAGS_options_file, + &db_options, &cf_descriptors); + if (!s.ok()) { + fprintf(stderr, "Unable to load options file %s --- %s\n", + FLAGS_options_file.c_str(), s.ToString().c_str()); + exit(1); + } + db_options.env = new CompositeEnvWrapper(db_stress_env); + options = Options(db_options, cf_descriptors[0].options); + return true; + } + return false; +} + +void InitializeOptionsFromFlags( + const std::shared_ptr& cache, + const std::shared_ptr& filter_policy, + Options& options) { + BlockBasedTableOptions block_based_options; + block_based_options.block_cache = cache; + block_based_options.cache_index_and_filter_blocks = + FLAGS_cache_index_and_filter_blocks; + block_based_options.metadata_cache_options.top_level_index_pinning = + static_cast(FLAGS_top_level_index_pinning); + block_based_options.metadata_cache_options.partition_pinning = + static_cast(FLAGS_partition_pinning); + block_based_options.metadata_cache_options.unpartitioned_pinning = + static_cast(FLAGS_unpartitioned_pinning); + block_based_options.checksum = checksum_type_e; + block_based_options.block_size = FLAGS_block_size; + block_based_options.cache_usage_options.options_overrides.insert( + {CacheEntryRole::kCompressionDictionaryBuildingBuffer, + {/*.charged = */ FLAGS_charge_compression_dictionary_building_buffer + ? CacheEntryRoleOptions::Decision::kEnabled + : CacheEntryRoleOptions::Decision::kDisabled}}); + block_based_options.cache_usage_options.options_overrides.insert( + {CacheEntryRole::kFilterConstruction, + {/*.charged = */ FLAGS_charge_filter_construction + ? CacheEntryRoleOptions::Decision::kEnabled + : CacheEntryRoleOptions::Decision::kDisabled}}); + block_based_options.cache_usage_options.options_overrides.insert( + {CacheEntryRole::kBlockBasedTableReader, + {/*.charged = */ FLAGS_charge_table_reader + ? CacheEntryRoleOptions::Decision::kEnabled + : CacheEntryRoleOptions::Decision::kDisabled}}); + block_based_options.cache_usage_options.options_overrides.insert( + {CacheEntryRole::kFileMetadata, + {/*.charged = */ FLAGS_charge_file_metadata + ? CacheEntryRoleOptions::Decision::kEnabled + : CacheEntryRoleOptions::Decision::kDisabled}}); + block_based_options.cache_usage_options.options_overrides.insert( + {CacheEntryRole::kBlobCache, + {/*.charged = */ FLAGS_charge_blob_cache + ? CacheEntryRoleOptions::Decision::kEnabled + : CacheEntryRoleOptions::Decision::kDisabled}}); + block_based_options.format_version = + static_cast(FLAGS_format_version); + block_based_options.index_block_restart_interval = + static_cast(FLAGS_index_block_restart_interval); + block_based_options.filter_policy = filter_policy; + block_based_options.partition_filters = FLAGS_partition_filters; + block_based_options.optimize_filters_for_memory = + FLAGS_optimize_filters_for_memory; + block_based_options.detect_filter_construct_corruption = + FLAGS_detect_filter_construct_corruption; + block_based_options.index_type = + static_cast(FLAGS_index_type); + block_based_options.data_block_index_type = + static_cast( + FLAGS_data_block_index_type); + block_based_options.prepopulate_block_cache = + static_cast( + FLAGS_prepopulate_block_cache); + block_based_options.initial_auto_readahead_size = + FLAGS_initial_auto_readahead_size; + block_based_options.max_auto_readahead_size = FLAGS_max_auto_readahead_size; + block_based_options.num_file_reads_for_auto_readahead = + FLAGS_num_file_reads_for_auto_readahead; + options.table_factory.reset(NewBlockBasedTableFactory(block_based_options)); + options.db_write_buffer_size = FLAGS_db_write_buffer_size; + options.write_buffer_size = FLAGS_write_buffer_size; + options.max_write_buffer_number = FLAGS_max_write_buffer_number; + options.min_write_buffer_number_to_merge = + FLAGS_min_write_buffer_number_to_merge; + options.max_write_buffer_number_to_maintain = + FLAGS_max_write_buffer_number_to_maintain; + options.max_write_buffer_size_to_maintain = + FLAGS_max_write_buffer_size_to_maintain; + options.memtable_prefix_bloom_size_ratio = + FLAGS_memtable_prefix_bloom_size_ratio; + options.memtable_whole_key_filtering = FLAGS_memtable_whole_key_filtering; + options.disable_auto_compactions = FLAGS_disable_auto_compactions; + options.max_background_compactions = FLAGS_max_background_compactions; + options.max_background_flushes = FLAGS_max_background_flushes; + options.compaction_style = + static_cast(FLAGS_compaction_style); + if (options.compaction_style == + ROCKSDB_NAMESPACE::CompactionStyle::kCompactionStyleFIFO) { + options.compaction_options_fifo.allow_compaction = + FLAGS_fifo_allow_compaction; + } + options.compaction_pri = + static_cast(FLAGS_compaction_pri); + options.num_levels = FLAGS_num_levels; + if (FLAGS_prefix_size >= 0) { + options.prefix_extractor.reset(NewFixedPrefixTransform(FLAGS_prefix_size)); + } + options.max_open_files = FLAGS_open_files; + options.statistics = dbstats; + options.env = db_stress_env; + options.use_fsync = FLAGS_use_fsync; + options.compaction_readahead_size = FLAGS_compaction_readahead_size; + options.allow_mmap_reads = FLAGS_mmap_read; + options.allow_mmap_writes = FLAGS_mmap_write; + options.use_direct_reads = FLAGS_use_direct_reads; + options.use_direct_io_for_flush_and_compaction = + FLAGS_use_direct_io_for_flush_and_compaction; + options.recycle_log_file_num = + static_cast(FLAGS_recycle_log_file_num); + options.target_file_size_base = FLAGS_target_file_size_base; + options.target_file_size_multiplier = FLAGS_target_file_size_multiplier; + options.max_bytes_for_level_base = FLAGS_max_bytes_for_level_base; + options.max_bytes_for_level_multiplier = FLAGS_max_bytes_for_level_multiplier; + options.level0_stop_writes_trigger = FLAGS_level0_stop_writes_trigger; + options.level0_slowdown_writes_trigger = FLAGS_level0_slowdown_writes_trigger; + options.level0_file_num_compaction_trigger = + FLAGS_level0_file_num_compaction_trigger; + options.compression = compression_type_e; + options.bottommost_compression = bottommost_compression_type_e; + options.compression_opts.max_dict_bytes = FLAGS_compression_max_dict_bytes; + options.compression_opts.zstd_max_train_bytes = + FLAGS_compression_zstd_max_train_bytes; + options.compression_opts.parallel_threads = + FLAGS_compression_parallel_threads; + options.compression_opts.max_dict_buffer_bytes = + FLAGS_compression_max_dict_buffer_bytes; + if (ZSTD_FinalizeDictionarySupported()) { + options.compression_opts.use_zstd_dict_trainer = + FLAGS_compression_use_zstd_dict_trainer; + } else if (!FLAGS_compression_use_zstd_dict_trainer) { + fprintf( + stderr, + "WARNING: use_zstd_dict_trainer is false but zstd finalizeDictionary " + "cannot be used because ZSTD 1.4.5+ is not linked with the binary." + " zstd dictionary trainer will be used.\n"); + } + options.max_manifest_file_size = FLAGS_max_manifest_file_size; + options.inplace_update_support = FLAGS_in_place_update; + options.max_subcompactions = static_cast(FLAGS_subcompactions); + options.allow_concurrent_memtable_write = + FLAGS_allow_concurrent_memtable_write; + options.experimental_mempurge_threshold = + FLAGS_experimental_mempurge_threshold; + options.periodic_compaction_seconds = FLAGS_periodic_compaction_seconds; + options.stats_dump_period_sec = + static_cast(FLAGS_stats_dump_period_sec); + options.ttl = FLAGS_compaction_ttl; + options.enable_pipelined_write = FLAGS_enable_pipelined_write; + options.enable_write_thread_adaptive_yield = + FLAGS_enable_write_thread_adaptive_yield; + options.compaction_options_universal.size_ratio = FLAGS_universal_size_ratio; + options.compaction_options_universal.min_merge_width = + FLAGS_universal_min_merge_width; + options.compaction_options_universal.max_merge_width = + FLAGS_universal_max_merge_width; + options.compaction_options_universal.max_size_amplification_percent = + FLAGS_universal_max_size_amplification_percent; + options.atomic_flush = FLAGS_atomic_flush; + options.manual_wal_flush = FLAGS_manual_wal_flush_one_in > 0 ? true : false; + options.avoid_unnecessary_blocking_io = FLAGS_avoid_unnecessary_blocking_io; + options.write_dbid_to_manifest = FLAGS_write_dbid_to_manifest; + options.avoid_flush_during_recovery = FLAGS_avoid_flush_during_recovery; + options.max_write_batch_group_size_bytes = + FLAGS_max_write_batch_group_size_bytes; + options.level_compaction_dynamic_level_bytes = + FLAGS_level_compaction_dynamic_level_bytes; + options.track_and_verify_wals_in_manifest = true; + options.verify_sst_unique_id_in_manifest = + FLAGS_verify_sst_unique_id_in_manifest; + options.memtable_protection_bytes_per_key = + FLAGS_memtable_protection_bytes_per_key; + options.block_protection_bytes_per_key = FLAGS_block_protection_bytes_per_key; + + // Integrated BlobDB + options.enable_blob_files = FLAGS_enable_blob_files; + options.min_blob_size = FLAGS_min_blob_size; + options.blob_file_size = FLAGS_blob_file_size; + options.blob_compression_type = + StringToCompressionType(FLAGS_blob_compression_type.c_str()); + options.enable_blob_garbage_collection = FLAGS_enable_blob_garbage_collection; + options.blob_garbage_collection_age_cutoff = + FLAGS_blob_garbage_collection_age_cutoff; + options.blob_garbage_collection_force_threshold = + FLAGS_blob_garbage_collection_force_threshold; + options.blob_compaction_readahead_size = FLAGS_blob_compaction_readahead_size; + options.blob_file_starting_level = FLAGS_blob_file_starting_level; + + if (FLAGS_use_blob_cache) { + if (FLAGS_use_shared_block_and_blob_cache) { + options.blob_cache = cache; + } else { + if (FLAGS_blob_cache_size > 0) { + LRUCacheOptions co; + co.capacity = FLAGS_blob_cache_size; + co.num_shard_bits = FLAGS_blob_cache_numshardbits; + options.blob_cache = NewLRUCache(co); + } else { + fprintf(stderr, + "Unable to create a standalone blob cache if blob_cache_size " + "<= 0.\n"); + exit(1); + } + } + switch (FLAGS_prepopulate_blob_cache) { + case 0: + options.prepopulate_blob_cache = PrepopulateBlobCache::kDisable; + break; + case 1: + options.prepopulate_blob_cache = PrepopulateBlobCache::kFlushOnly; + break; + default: + fprintf(stderr, "Unknown prepopulate blob cache mode\n"); + exit(1); + } + } + + options.wal_compression = + StringToCompressionType(FLAGS_wal_compression.c_str()); + + if (FLAGS_enable_tiered_storage) { + options.bottommost_temperature = Temperature::kCold; + } + options.preclude_last_level_data_seconds = + FLAGS_preclude_last_level_data_seconds; + options.preserve_internal_time_seconds = FLAGS_preserve_internal_time_seconds; + + switch (FLAGS_rep_factory) { + case kSkipList: + // no need to do anything + break; + case kHashSkipList: + options.memtable_factory.reset(NewHashSkipListRepFactory(10000)); + break; + case kVectorRep: + options.memtable_factory.reset(new VectorRepFactory()); + break; + } + if (FLAGS_use_full_merge_v1) { + options.merge_operator = MergeOperators::CreateDeprecatedPutOperator(); + } else { + options.merge_operator = MergeOperators::CreatePutOperator(); + } + + if (FLAGS_enable_compaction_filter) { + options.compaction_filter_factory = + std::make_shared(); + } + + options.best_efforts_recovery = FLAGS_best_efforts_recovery; + options.paranoid_file_checks = FLAGS_paranoid_file_checks; + options.fail_if_options_file_error = FLAGS_fail_if_options_file_error; + + if (FLAGS_user_timestamp_size > 0) { + CheckAndSetOptionsForUserTimestamp(options); + } + + options.allow_data_in_errors = FLAGS_allow_data_in_errors; + + options.enable_thread_tracking = FLAGS_enable_thread_tracking; + + options.memtable_max_range_deletions = FLAGS_memtable_max_range_deletions; +} + +void InitializeOptionsGeneral( + const std::shared_ptr& cache, + const std::shared_ptr& filter_policy, + Options& options) { + options.create_missing_column_families = true; + options.create_if_missing = true; + + if (!options.statistics) { + options.statistics = dbstats; + } + + if (options.env == Options().env) { + options.env = db_stress_env; + } + + assert(options.table_factory); + auto table_options = + options.table_factory->GetOptions(); + if (table_options) { + if (FLAGS_cache_size > 0) { + table_options->block_cache = cache; + } + if (!table_options->filter_policy) { + table_options->filter_policy = filter_policy; + } + } + + // TODO: row_cache, thread-pool IO priority, CPU priority. + + if (!options.rate_limiter) { + if (FLAGS_rate_limiter_bytes_per_sec > 0) { + options.rate_limiter.reset(NewGenericRateLimiter( + FLAGS_rate_limiter_bytes_per_sec, 1000 /* refill_period_us */, + 10 /* fairness */, + FLAGS_rate_limit_bg_reads ? RateLimiter::Mode::kReadsOnly + : RateLimiter::Mode::kWritesOnly)); + } + } + + if (!options.file_checksum_gen_factory) { + options.file_checksum_gen_factory = + GetFileChecksumImpl(FLAGS_file_checksum_impl); + } + + if (FLAGS_sst_file_manager_bytes_per_sec > 0 || + FLAGS_sst_file_manager_bytes_per_truncate > 0) { + Status status; + options.sst_file_manager.reset(NewSstFileManager( + db_stress_env, options.info_log, "" /* trash_dir */, + static_cast(FLAGS_sst_file_manager_bytes_per_sec), + true /* delete_existing_trash */, &status, + 0.25 /* max_trash_db_ratio */, + FLAGS_sst_file_manager_bytes_per_truncate)); + if (!status.ok()) { + fprintf(stderr, "SstFileManager creation failed: %s\n", + status.ToString().c_str()); + exit(1); + } + } + + if (FLAGS_preserve_unverified_changes) { + if (!options.avoid_flush_during_recovery) { + fprintf(stderr, + "WARNING: flipping `avoid_flush_during_recovery` to true for " + "`preserve_unverified_changes` to keep all files\n"); + options.avoid_flush_during_recovery = true; + } + // Together with `avoid_flush_during_recovery == true`, this will prevent + // live files from becoming obsolete and deleted between `DB::Open()` and + // `DisableFileDeletions()` due to flush or compaction. We do not need to + // warn the user since we will reenable compaction soon. + options.disable_auto_compactions = true; + } + + options.table_properties_collector_factories.emplace_back( + std::make_shared()); +} + +} // namespace ROCKSDB_NAMESPACE +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db_stress_tool/db_stress_test_base.h b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_test_base.h new file mode 100644 index 0000000..dc235fc --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_test_base.h @@ -0,0 +1,332 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifdef GFLAGS +#pragma once + +#include "db_stress_tool/db_stress_common.h" +#include "db_stress_tool/db_stress_shared_state.h" + +namespace ROCKSDB_NAMESPACE { +class SystemClock; +class Transaction; +class TransactionDB; +class OptimisticTransactionDB; +struct TransactionDBOptions; + +class StressTest { + public: + StressTest(); + + virtual ~StressTest(); + + std::shared_ptr NewCache(size_t capacity, int32_t num_shard_bits); + + static std::vector GetBlobCompressionTags(); + + bool BuildOptionsTable(); + + void InitDb(SharedState*); + // The initialization work is split into two parts to avoid a circular + // dependency with `SharedState`. + virtual void FinishInitDb(SharedState*); + void TrackExpectedState(SharedState* shared); + void OperateDb(ThreadState* thread); + virtual void VerifyDb(ThreadState* thread) const = 0; + virtual void ContinuouslyVerifyDb(ThreadState* /*thread*/) const = 0; + void PrintStatistics(); + + protected: + Status AssertSame(DB* db, ColumnFamilyHandle* cf, + ThreadState::SnapshotState& snap_state); + + // Currently PreloadDb has to be single-threaded. + void PreloadDbAndReopenAsReadOnly(int64_t number_of_keys, + SharedState* shared); + + Status SetOptions(ThreadState* thread); + + // For transactionsDB, there can be txns prepared but not yet committeed + // right before previous stress run crash. + // They will be recovered and processed through + // ProcessRecoveredPreparedTxnsHelper on the start of current stress run. + void ProcessRecoveredPreparedTxns(SharedState* shared); + + // Default implementation will first update ExpectedState to be + // `SharedState::UNKNOWN` for each keys in `txn` and then randomly + // commit or rollback `txn`. + virtual void ProcessRecoveredPreparedTxnsHelper(Transaction* txn, + SharedState* shared); + + // ExecuteTransaction is recommended instead + Status NewTxn(WriteOptions& write_opts, + std::unique_ptr* out_txn); + Status CommitTxn(Transaction& txn, ThreadState* thread = nullptr); + + // Creates a transaction, executes `ops`, and tries to commit + Status ExecuteTransaction(WriteOptions& write_opts, ThreadState* thread, + std::function&& ops); + + virtual void MaybeClearOneColumnFamily(ThreadState* /* thread */) {} + + virtual bool ShouldAcquireMutexOnKey() const { return false; } + + // Returns true if DB state is tracked by the stress test. + virtual bool IsStateTracked() const = 0; + + virtual std::vector GenerateColumnFamilies( + const int /* num_column_families */, int rand_column_family) const { + return {rand_column_family}; + } + + virtual std::vector GenerateKeys(int64_t rand_key) const { + return {rand_key}; + } + + virtual Status TestGet(ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) = 0; + + virtual std::vector TestMultiGet( + ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) = 0; + + virtual void TestGetEntity(ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) = 0; + + virtual void TestMultiGetEntity(ThreadState* thread, + const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) = 0; + + virtual Status TestPrefixScan(ThreadState* thread, + const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) = 0; + + virtual Status TestPut(ThreadState* thread, WriteOptions& write_opts, + const ReadOptions& read_opts, + const std::vector& cf_ids, + const std::vector& keys, + char (&value)[100]) = 0; + + virtual Status TestDelete(ThreadState* thread, WriteOptions& write_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) = 0; + + virtual Status TestDeleteRange(ThreadState* thread, WriteOptions& write_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) = 0; + + virtual void TestIngestExternalFile( + ThreadState* thread, const std::vector& rand_column_families, + const std::vector& rand_keys) = 0; + + // Issue compact range, starting with start_key, whose integer value + // is rand_key. + virtual void TestCompactRange(ThreadState* thread, int64_t rand_key, + const Slice& start_key, + ColumnFamilyHandle* column_family); + + // Calculate a hash value for all keys in range [start_key, end_key] + // at a certain snapshot. + uint32_t GetRangeHash(ThreadState* thread, const Snapshot* snapshot, + ColumnFamilyHandle* column_family, + const Slice& start_key, const Slice& end_key); + + // Return a column family handle that mirrors what is pointed by + // `column_family_id`, which will be used to validate data to be correct. + // By default, the column family itself will be returned. + virtual ColumnFamilyHandle* GetControlCfh(ThreadState* /* thread*/, + int column_family_id) { + return column_families_[column_family_id]; + } + + // Generated a list of keys that close to boundaries of SST keys. + // If there isn't any SST file in the DB, return empty list. + std::vector GetWhiteBoxKeys(ThreadState* thread, DB* db, + ColumnFamilyHandle* cfh, + size_t num_keys); + + // Given a key K, this creates an iterator which scans to K and then + // does a random sequence of Next/Prev operations. + virtual Status TestIterate(ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys); + + virtual Status TestIterateAgainstExpected( + ThreadState* /* thread */, const ReadOptions& /* read_opts */, + const std::vector& /* rand_column_families */, + const std::vector& /* rand_keys */) { + return Status::NotSupported(); + } + + // Enum used by VerifyIterator() to identify the mode to validate. + enum LastIterateOp { + kLastOpSeek, + kLastOpSeekForPrev, + kLastOpNextOrPrev, + kLastOpSeekToFirst, + kLastOpSeekToLast + }; + + // Compare the two iterator, iter and cmp_iter are in the same position, + // unless iter might be made invalidate or undefined because of + // upper or lower bounds, or prefix extractor. + // Will flag failure if the verification fails. + // diverged = true if the two iterator is already diverged. + // True if verification passed, false if not. + // op_logs is the information to print when validation fails. + void VerifyIterator(ThreadState* thread, ColumnFamilyHandle* cmp_cfh, + const ReadOptions& ro, Iterator* iter, Iterator* cmp_iter, + LastIterateOp op, const Slice& seek_key, + const std::string& op_logs, bool* diverged); + + virtual Status TestBackupRestore(ThreadState* thread, + const std::vector& rand_column_families, + const std::vector& rand_keys); + + virtual Status TestCheckpoint(ThreadState* thread, + const std::vector& rand_column_families, + const std::vector& rand_keys); + + void TestCompactFiles(ThreadState* thread, ColumnFamilyHandle* column_family); + + Status TestFlush(const std::vector& rand_column_families); + + Status TestPauseBackground(ThreadState* thread); + + void TestAcquireSnapshot(ThreadState* thread, int rand_column_family, + const std::string& keystr, uint64_t i); + + Status MaybeReleaseSnapshots(ThreadState* thread, uint64_t i); + Status VerifyGetLiveFiles() const; + Status VerifyGetSortedWalFiles() const; + Status VerifyGetCurrentWalFile() const; + void TestGetProperty(ThreadState* thread) const; + + virtual Status TestApproximateSize( + ThreadState* thread, uint64_t iteration, + const std::vector& rand_column_families, + const std::vector& rand_keys); + + virtual Status TestCustomOperations( + ThreadState* /*thread*/, + const std::vector& /*rand_column_families*/) { + return Status::NotSupported("TestCustomOperations() must be overridden"); + } + + void VerificationAbort(SharedState* shared, std::string msg, Status s) const; + + void VerificationAbort(SharedState* shared, std::string msg, int cf, + int64_t key) const; + + void VerificationAbort(SharedState* shared, std::string msg, int cf, + int64_t key, Slice value_from_db, + Slice value_from_expected) const; + + void VerificationAbort(SharedState* shared, int cf, int64_t key, + const Slice& value, const WideColumns& columns) const; + + static std::string DebugString(const Slice& value, + const WideColumns& columns); + + void PrintEnv() const; + + void Open(SharedState* shared); + + void Reopen(ThreadState* thread); + + virtual void RegisterAdditionalListeners() {} + + virtual void PrepareTxnDbOptions(SharedState* /*shared*/, + TransactionDBOptions& /*txn_db_opts*/) {} + + // Returns whether the timestamp of read_opts is updated. + bool MaybeUseOlderTimestampForPointLookup(ThreadState* thread, + std::string& ts_str, + Slice& ts_slice, + ReadOptions& read_opts); + + void MaybeUseOlderTimestampForRangeScan(ThreadState* thread, + std::string& ts_str, Slice& ts_slice, + ReadOptions& read_opts); + + std::shared_ptr cache_; + std::shared_ptr compressed_cache_; + std::shared_ptr filter_policy_; + DB* db_; + TransactionDB* txn_db_; + OptimisticTransactionDB* optimistic_txn_db_; + + // Currently only used in MultiOpsTxnsStressTest + std::atomic db_aptr_; + + Options options_; + SystemClock* clock_; + std::vector column_families_; + std::vector column_family_names_; + std::atomic new_column_family_name_; + int num_times_reopened_; + std::unordered_map> options_table_; + std::vector options_index_; + std::atomic db_preload_finished_; + + // Fields used for continuous verification from another thread + DB* cmp_db_; + std::vector cmp_cfhs_; + bool is_db_stopped_; +}; + +// Load options from OPTIONS file and populate `options`. +extern bool InitializeOptionsFromFile(Options& options); + +// Initialize `options` using command line arguments. +// When this function is called, `cache`, `block_cache_compressed`, +// `filter_policy` have all been initialized. Therefore, we just pass them as +// input arguments. +extern void InitializeOptionsFromFlags( + const std::shared_ptr& cache, + const std::shared_ptr& filter_policy, Options& options); + +// Initialize `options` on which `InitializeOptionsFromFile()` and +// `InitializeOptionsFromFlags()` have both been called already. +// There are two cases. +// Case 1: OPTIONS file is not specified. Command line arguments have been used +// to initialize `options`. InitializeOptionsGeneral() will use +// `cache` and `filter_policy` to initialize +// corresponding fields of `options`. InitializeOptionsGeneral() will +// also set up other fields of `options` so that stress test can run. +// Examples include `create_if_missing` and +// `create_missing_column_families`, etc. +// Case 2: OPTIONS file is specified. It is possible that, after loading from +// the given OPTIONS files, some shared object fields are still not +// initialized because they are not set in the OPTIONS file. In this +// case, if command line arguments indicate that the user wants to set +// up such shared objects, e.g. block cache, compressed block cache, +// row cache, filter policy, then InitializeOptionsGeneral() will honor +// the user's choice, thus passing `cache`, +// `filter_policy` as input arguments. +// +// InitializeOptionsGeneral() must not overwrite fields of `options` loaded +// from OPTIONS file. +extern void InitializeOptionsGeneral( + const std::shared_ptr& cache, + const std::shared_ptr& filter_policy, Options& options); + +// If no OPTIONS file is specified, set up `options` so that we can test +// user-defined timestamp which requires `-user_timestamp_size=8`. +// This function also checks for known (currently) incompatible features with +// user-defined timestamp. +extern void CheckAndSetOptionsForUserTimestamp(Options& options); + +} // namespace ROCKSDB_NAMESPACE +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db_stress_tool/db_stress_tool.cc b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_tool.cc new file mode 100644 index 0000000..f2b124b --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/db_stress_tool.cc @@ -0,0 +1,379 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// The test uses an array to compare against values written to the database. +// Keys written to the array are in 1:1 correspondence to the actual values in +// the database according to the formula in the function GenerateValue. + +// Space is reserved in the array from 0 to FLAGS_max_key and values are +// randomly written/deleted/read from those positions. During verification we +// compare all the positions in the array. To shorten/elongate the running +// time, you could change the settings: FLAGS_max_key, FLAGS_ops_per_thread, +// (sometimes also FLAGS_threads). +// +// NOTE that if FLAGS_test_batches_snapshots is set, the test will have +// different behavior. See comment of the flag for details. + +#include "db_stress_tool/db_stress_shared_state.h" +#ifdef GFLAGS +#include "db_stress_tool/db_stress_common.h" +#include "db_stress_tool/db_stress_driver.h" +#include "rocksdb/convenience.h" +#include "utilities/fault_injection_fs.h" + +namespace ROCKSDB_NAMESPACE { +namespace { +static std::shared_ptr env_guard; +static std::shared_ptr env_wrapper_guard; +static std::shared_ptr legacy_env_wrapper_guard; +static std::shared_ptr + dbsl_env_wrapper_guard; +static std::shared_ptr fault_env_guard; +} // namespace + +KeyGenContext key_gen_ctx; + +int db_stress_tool(int argc, char** argv) { + SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) + + " [OPTIONS]..."); + ParseCommandLineFlags(&argc, &argv, true); + + SanitizeDoubleParam(&FLAGS_bloom_bits); + SanitizeDoubleParam(&FLAGS_memtable_prefix_bloom_size_ratio); + SanitizeDoubleParam(&FLAGS_max_bytes_for_level_multiplier); + +#ifndef NDEBUG + if (FLAGS_mock_direct_io) { + SetupSyncPointsToMockDirectIO(); + } +#endif + if (FLAGS_statistics) { + dbstats = ROCKSDB_NAMESPACE::CreateDBStatistics(); + if (FLAGS_test_secondary) { + dbstats_secondaries = ROCKSDB_NAMESPACE::CreateDBStatistics(); + } + } + compression_type_e = StringToCompressionType(FLAGS_compression_type.c_str()); + bottommost_compression_type_e = + StringToCompressionType(FLAGS_bottommost_compression_type.c_str()); + checksum_type_e = StringToChecksumType(FLAGS_checksum_type.c_str()); + + Env* raw_env; + + int env_opts = !FLAGS_env_uri.empty() + !FLAGS_fs_uri.empty(); + if (env_opts > 1) { + fprintf(stderr, "Error: --env_uri and --fs_uri are mutually exclusive\n"); + exit(1); + } + + Status s = Env::CreateFromUri(ConfigOptions(), FLAGS_env_uri, FLAGS_fs_uri, + &raw_env, &env_guard); + if (!s.ok()) { + fprintf(stderr, "Error Creating Env URI: %s: %s\n", FLAGS_env_uri.c_str(), + s.ToString().c_str()); + exit(1); + } + dbsl_env_wrapper_guard = std::make_shared(raw_env); + db_stress_listener_env = dbsl_env_wrapper_guard.get(); + + if (FLAGS_read_fault_one_in || FLAGS_sync_fault_injection || + FLAGS_write_fault_one_in || FLAGS_open_metadata_write_fault_one_in || + FLAGS_open_write_fault_one_in || FLAGS_open_read_fault_one_in) { + FaultInjectionTestFS* fs = + new FaultInjectionTestFS(raw_env->GetFileSystem()); + fault_fs_guard.reset(fs); + if (FLAGS_write_fault_one_in) { + fault_fs_guard->SetFilesystemDirectWritable(false); + } else { + fault_fs_guard->SetFilesystemDirectWritable(true); + } + fault_env_guard = + std::make_shared(raw_env, fault_fs_guard); + raw_env = fault_env_guard.get(); + } + + env_wrapper_guard = std::make_shared( + raw_env, std::make_shared(raw_env->GetFileSystem())); + if (!env_opts && !FLAGS_use_io_uring) { + // If using the default Env (Posix), wrap DbStressEnvWrapper with the + // legacy EnvWrapper. This is a workaround to prevent MultiGet and scans + // from failing when IO uring is disabled. The EnvWrapper + // has a default implementation of ReadAsync that redirects to Read. + legacy_env_wrapper_guard = std::make_shared(raw_env); + env_wrapper_guard = std::make_shared( + legacy_env_wrapper_guard, + std::make_shared( + legacy_env_wrapper_guard->GetFileSystem())); + } + db_stress_env = env_wrapper_guard.get(); + + FLAGS_rep_factory = StringToRepFactory(FLAGS_memtablerep.c_str()); + + // The number of background threads should be at least as much the + // max number of concurrent compactions. + db_stress_env->SetBackgroundThreads(FLAGS_max_background_compactions, + ROCKSDB_NAMESPACE::Env::Priority::LOW); + db_stress_env->SetBackgroundThreads(FLAGS_num_bottom_pri_threads, + ROCKSDB_NAMESPACE::Env::Priority::BOTTOM); + if (FLAGS_prefixpercent > 0 && FLAGS_prefix_size < 0) { + fprintf(stderr, + "Error: prefixpercent is non-zero while prefix_size is " + "not positive!\n"); + exit(1); + } + if (FLAGS_test_batches_snapshots && FLAGS_prefix_size <= 0) { + fprintf(stderr, + "Error: please specify prefix_size for " + "test_batches_snapshots test!\n"); + exit(1); + } + if (FLAGS_memtable_prefix_bloom_size_ratio > 0.0 && FLAGS_prefix_size < 0 && + !FLAGS_memtable_whole_key_filtering) { + fprintf(stderr, + "Error: please specify positive prefix_size or enable whole key " + "filtering in order to use memtable_prefix_bloom_size_ratio\n"); + exit(1); + } + if ((FLAGS_readpercent + FLAGS_prefixpercent + FLAGS_writepercent + + FLAGS_delpercent + FLAGS_delrangepercent + FLAGS_iterpercent + + FLAGS_customopspercent) != 100) { + fprintf( + stderr, + "Error: " + "Read(-readpercent=%d)+Prefix(-prefixpercent=%d)+Write(-writepercent=%" + "d)+Delete(-delpercent=%d)+DeleteRange(-delrangepercent=%d)" + "+Iterate(-iterpercent=%d)+CustomOps(-customopspercent=%d) percents != " + "100!\n", + FLAGS_readpercent, FLAGS_prefixpercent, FLAGS_writepercent, + FLAGS_delpercent, FLAGS_delrangepercent, FLAGS_iterpercent, + FLAGS_customopspercent); + exit(1); + } + if (FLAGS_disable_wal == 1 && FLAGS_reopen > 0) { + fprintf(stderr, "Error: Db cannot reopen safely with disable_wal set!\n"); + exit(1); + } + if ((unsigned)FLAGS_reopen >= FLAGS_ops_per_thread) { + fprintf(stderr, + "Error: #DB-reopens should be < ops_per_thread\n" + "Provided reopens = %d and ops_per_thread = %lu\n", + FLAGS_reopen, (unsigned long)FLAGS_ops_per_thread); + exit(1); + } + if (FLAGS_test_batches_snapshots && FLAGS_delrangepercent > 0) { + fprintf(stderr, + "Error: nonzero delrangepercent unsupported in " + "test_batches_snapshots mode\n"); + exit(1); + } + if (FLAGS_active_width > FLAGS_max_key) { + fprintf(stderr, "Error: active_width can be at most max_key\n"); + exit(1); + } else if (FLAGS_active_width == 0) { + FLAGS_active_width = FLAGS_max_key; + } + if (FLAGS_value_size_mult * kRandomValueMaxFactor > kValueMaxLen) { + fprintf(stderr, "Error: value_size_mult can be at most %d\n", + kValueMaxLen / kRandomValueMaxFactor); + exit(1); + } + if (FLAGS_use_merge && FLAGS_nooverwritepercent == 100) { + fprintf( + stderr, + "Error: nooverwritepercent must not be 100 when using merge operands"); + exit(1); + } + if (FLAGS_ingest_external_file_one_in > 0 && + FLAGS_nooverwritepercent == 100) { + fprintf( + stderr, + "Error: nooverwritepercent must not be 100 when using file ingestion"); + exit(1); + } + if (FLAGS_clear_column_family_one_in > 0 && FLAGS_backup_one_in > 0) { + fprintf(stderr, + "Error: clear_column_family_one_in must be 0 when using backup\n"); + exit(1); + } + if (FLAGS_test_cf_consistency && FLAGS_disable_wal) { + FLAGS_atomic_flush = true; + } + + if (FLAGS_read_only) { + if (FLAGS_writepercent != 0 || FLAGS_delpercent != 0 || + FLAGS_delrangepercent != 0) { + fprintf(stderr, "Error: updates are not supported in read only mode\n"); + exit(1); + } else if (FLAGS_checkpoint_one_in > 0 && + FLAGS_clear_column_family_one_in > 0) { + fprintf(stdout, + "Warn: checkpoint won't be validated since column families may " + "be dropped.\n"); + } + } + + // Choose a location for the test database if none given with --db= + if (FLAGS_db.empty()) { + std::string default_db_path; + db_stress_env->GetTestDirectory(&default_db_path); + default_db_path += "/dbstress"; + FLAGS_db = default_db_path; + } + + if ((FLAGS_test_secondary || FLAGS_continuous_verification_interval > 0) && + FLAGS_secondaries_base.empty()) { + std::string default_secondaries_path; + db_stress_env->GetTestDirectory(&default_secondaries_path); + default_secondaries_path += "/dbstress_secondaries"; + s = db_stress_env->CreateDirIfMissing(default_secondaries_path); + if (!s.ok()) { + fprintf(stderr, "Failed to create directory %s: %s\n", + default_secondaries_path.c_str(), s.ToString().c_str()); + exit(1); + } + FLAGS_secondaries_base = default_secondaries_path; + } + + if (FLAGS_best_efforts_recovery && !FLAGS_skip_verifydb && + !FLAGS_disable_wal) { + fprintf(stderr, + "With best-efforts recovery, either skip_verifydb or disable_wal " + "should be set to true.\n"); + exit(1); + } + if (FLAGS_skip_verifydb) { + if (FLAGS_verify_db_one_in > 0) { + fprintf(stderr, + "Must set -verify_db_one_in=0 if skip_verifydb is true.\n"); + exit(1); + } + if (FLAGS_continuous_verification_interval > 0) { + fprintf(stderr, + "Must set -continuous_verification_interval=0 if skip_verifydb " + "is true.\n"); + exit(1); + } + } + if (FLAGS_enable_compaction_filter && + (FLAGS_acquire_snapshot_one_in > 0 || FLAGS_compact_range_one_in > 0 || + FLAGS_iterpercent > 0 || FLAGS_test_batches_snapshots || + FLAGS_test_cf_consistency)) { + fprintf( + stderr, + "Error: acquire_snapshot_one_in, compact_range_one_in, iterpercent, " + "test_batches_snapshots must all be 0 when using compaction filter\n"); + exit(1); + } + if (FLAGS_test_multi_ops_txns) { + CheckAndSetOptionsForMultiOpsTxnStressTest(); + } + + if (!FLAGS_use_txn && FLAGS_use_optimistic_txn) { + fprintf( + stderr, + "You cannot set use_optimistic_txn true while use_txn is false. Please " + "set use_txn true if you want to use OptimisticTransactionDB\n"); + exit(1); + } + + if (FLAGS_create_timestamped_snapshot_one_in > 0) { + if (!FLAGS_use_txn) { + fprintf(stderr, "timestamped snapshot supported only in TransactionDB\n"); + exit(1); + } else if (FLAGS_txn_write_policy != 0) { + fprintf(stderr, + "timestamped snapshot supported only in write-committed\n"); + exit(1); + } + } + + if (FLAGS_preserve_unverified_changes && FLAGS_reopen != 0) { + fprintf(stderr, + "Reopen DB is incompatible with preserving unverified changes\n"); + exit(1); + } + + if (FLAGS_use_txn && !FLAGS_use_optimistic_txn && + FLAGS_sync_fault_injection && FLAGS_txn_write_policy != 0) { + fprintf(stderr, + "For TransactionDB, correctness testing with unsync data loss is " + "currently compatible with only write committed policy\n"); + exit(1); + } + + if (FLAGS_use_put_entity_one_in > 0 && + (FLAGS_ingest_external_file_one_in > 0 || FLAGS_use_merge || + FLAGS_use_full_merge_v1 || FLAGS_use_txn || FLAGS_test_multi_ops_txns || + FLAGS_user_timestamp_size > 0)) { + fprintf(stderr, + "PutEntity is currently incompatible with SstFileWriter, Merge," + " transactions, and user-defined timestamps\n"); + exit(1); + } + +#ifndef NDEBUG + KillPoint* kp = KillPoint::GetInstance(); + kp->rocksdb_kill_odds = FLAGS_kill_random_test; + kp->rocksdb_kill_exclude_prefixes = SplitString(FLAGS_kill_exclude_prefixes); +#endif + + unsigned int levels = FLAGS_max_key_len; + std::vector weights; + uint64_t scale_factor = FLAGS_key_window_scale_factor; + key_gen_ctx.window = scale_factor * 100; + if (!FLAGS_key_len_percent_dist.empty()) { + weights = SplitString(FLAGS_key_len_percent_dist); + if (weights.size() != levels) { + fprintf(stderr, + "Number of weights in key_len_dist should be equal to" + " max_key_len"); + exit(1); + } + + uint64_t total_weight = 0; + for (std::string& weight : weights) { + uint64_t val = std::stoull(weight); + key_gen_ctx.weights.emplace_back(val * scale_factor); + total_weight += val; + } + if (total_weight != 100) { + fprintf(stderr, "Sum of all weights in key_len_dist should be 100"); + exit(1); + } + } else { + uint64_t keys_per_level = key_gen_ctx.window / levels; + for (unsigned int level = 0; level + 1 < levels; ++level) { + key_gen_ctx.weights.emplace_back(keys_per_level); + } + key_gen_ctx.weights.emplace_back(key_gen_ctx.window - + keys_per_level * (levels - 1)); + } + std::unique_ptr shared; + std::unique_ptr stress; + if (FLAGS_test_cf_consistency) { + stress.reset(CreateCfConsistencyStressTest()); + } else if (FLAGS_test_batches_snapshots) { + stress.reset(CreateBatchedOpsStressTest()); + } else if (FLAGS_test_multi_ops_txns) { + stress.reset(CreateMultiOpsTxnsStressTest()); + } else { + stress.reset(CreateNonBatchedOpsStressTest()); + } + // Initialize the Zipfian pre-calculated array + InitializeHotKeyGenerator(FLAGS_hot_key_alpha); + shared.reset(new SharedState(db_stress_env, stress.get())); + if (RunStressTest(shared.get())) { + return 0; + } else { + return 1; + } +} + +} // namespace ROCKSDB_NAMESPACE +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db_stress_tool/expected_state.cc b/librocksdb-sys/rocksdb/db_stress_tool/expected_state.cc new file mode 100644 index 0000000..dd210ca --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/expected_state.cc @@ -0,0 +1,780 @@ +// Copyright (c) 2021-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include +#ifdef GFLAGS + + +#include "db/wide/wide_column_serialization.h" +#include "db_stress_tool/db_stress_common.h" +#include "db_stress_tool/db_stress_shared_state.h" +#include "db_stress_tool/expected_state.h" +#include "rocksdb/trace_reader_writer.h" +#include "rocksdb/trace_record_result.h" + +namespace ROCKSDB_NAMESPACE { +ExpectedState::ExpectedState(size_t max_key, size_t num_column_families) + : max_key_(max_key), + num_column_families_(num_column_families), + values_(nullptr) {} + +void ExpectedState::ClearColumnFamily(int cf) { + const uint32_t del_mask = ExpectedValue::GetDelMask(); + std::fill(&Value(cf, 0 /* key */), &Value(cf + 1, 0 /* key */), del_mask); +} + +void ExpectedState::Precommit(int cf, int64_t key, const ExpectedValue& value) { + Value(cf, key).store(value.Read()); + // To prevent low-level instruction reordering that results + // in db write happens before setting pending state in expected value + std::atomic_thread_fence(std::memory_order_release); +} + +PendingExpectedValue ExpectedState::PreparePut(int cf, int64_t key) { + ExpectedValue expected_value = Load(cf, key); + const ExpectedValue orig_expected_value = expected_value; + expected_value.Put(true /* pending */); + const ExpectedValue pending_expected_value = expected_value; + expected_value.Put(false /* pending */); + const ExpectedValue final_expected_value = expected_value; + Precommit(cf, key, pending_expected_value); + return PendingExpectedValue(&Value(cf, key), orig_expected_value, + final_expected_value); +} + +ExpectedValue ExpectedState::Get(int cf, int64_t key) { return Load(cf, key); } + +PendingExpectedValue ExpectedState::PrepareDelete(int cf, int64_t key, + bool* prepared) { + ExpectedValue expected_value = Load(cf, key); + const ExpectedValue orig_expected_value = expected_value; + bool res = expected_value.Delete(true /* pending */); + if (prepared) { + *prepared = res; + } + if (!res) { + return PendingExpectedValue(&Value(cf, key), orig_expected_value, + orig_expected_value); + } + const ExpectedValue pending_expected_value = expected_value; + expected_value.Delete(false /* pending */); + const ExpectedValue final_expected_value = expected_value; + Precommit(cf, key, pending_expected_value); + return PendingExpectedValue(&Value(cf, key), orig_expected_value, + final_expected_value); +} + +PendingExpectedValue ExpectedState::PrepareSingleDelete(int cf, int64_t key) { + return PrepareDelete(cf, key); +} + +std::vector ExpectedState::PrepareDeleteRange( + int cf, int64_t begin_key, int64_t end_key) { + std::vector pending_expected_values; + for (int64_t key = begin_key; key < end_key; ++key) { + bool prepared = false; + PendingExpectedValue pending_expected_value = + PrepareDelete(cf, key, &prepared); + if (prepared) { + pending_expected_values.push_back(pending_expected_value); + } + } + return pending_expected_values; +} + +bool ExpectedState::Exists(int cf, int64_t key) { + return Load(cf, key).Exists(); +} + +void ExpectedState::Reset() { + const uint32_t del_mask = ExpectedValue::GetDelMask(); + for (size_t i = 0; i < num_column_families_; ++i) { + for (size_t j = 0; j < max_key_; ++j) { + Value(static_cast(i), j).store(del_mask, std::memory_order_relaxed); + } + } +} + +void ExpectedState::SyncPut(int cf, int64_t key, uint32_t value_base) { + ExpectedValue expected_value = Load(cf, key); + expected_value.SyncPut(value_base); + Value(cf, key).store(expected_value.Read()); +} + +void ExpectedState::SyncPendingPut(int cf, int64_t key) { + ExpectedValue expected_value = Load(cf, key); + expected_value.SyncPendingPut(); + Value(cf, key).store(expected_value.Read()); +} + +void ExpectedState::SyncDelete(int cf, int64_t key) { + ExpectedValue expected_value = Load(cf, key); + expected_value.SyncDelete(); + Value(cf, key).store(expected_value.Read()); +} + +void ExpectedState::SyncDeleteRange(int cf, int64_t begin_key, + int64_t end_key) { + for (int64_t key = begin_key; key < end_key; ++key) { + SyncDelete(cf, key); + } +} + +FileExpectedState::FileExpectedState(std::string expected_state_file_path, + size_t max_key, size_t num_column_families) + : ExpectedState(max_key, num_column_families), + expected_state_file_path_(expected_state_file_path) {} + +Status FileExpectedState::Open(bool create) { + size_t expected_values_size = GetValuesLen(); + + Env* default_env = Env::Default(); + + Status status; + if (create) { + std::unique_ptr wfile; + const EnvOptions soptions; + status = default_env->NewWritableFile(expected_state_file_path_, &wfile, + soptions); + if (status.ok()) { + std::string buf(expected_values_size, '\0'); + status = wfile->Append(buf); + } + } + if (status.ok()) { + status = default_env->NewMemoryMappedFileBuffer( + expected_state_file_path_, &expected_state_mmap_buffer_); + } + if (status.ok()) { + assert(expected_state_mmap_buffer_->GetLen() == expected_values_size); + values_ = static_cast*>( + expected_state_mmap_buffer_->GetBase()); + assert(values_ != nullptr); + if (create) { + Reset(); + } + } else { + assert(values_ == nullptr); + } + return status; +} + +AnonExpectedState::AnonExpectedState(size_t max_key, size_t num_column_families) + : ExpectedState(max_key, num_column_families) {} + +#ifndef NDEBUG +Status AnonExpectedState::Open(bool create) { +#else +Status AnonExpectedState::Open(bool /* create */) { +#endif + // AnonExpectedState only supports being freshly created. + assert(create); + values_allocation_.reset( + new std::atomic[GetValuesLen() / + sizeof(std::atomic)]); + values_ = &values_allocation_[0]; + Reset(); + return Status::OK(); +} + +ExpectedStateManager::ExpectedStateManager(size_t max_key, + size_t num_column_families) + : max_key_(max_key), + num_column_families_(num_column_families), + latest_(nullptr) {} + +ExpectedStateManager::~ExpectedStateManager() {} + +const std::string FileExpectedStateManager::kLatestBasename = "LATEST"; +const std::string FileExpectedStateManager::kStateFilenameSuffix = ".state"; +const std::string FileExpectedStateManager::kTraceFilenameSuffix = ".trace"; +const std::string FileExpectedStateManager::kTempFilenamePrefix = "."; +const std::string FileExpectedStateManager::kTempFilenameSuffix = ".tmp"; + +FileExpectedStateManager::FileExpectedStateManager( + size_t max_key, size_t num_column_families, + std::string expected_state_dir_path) + : ExpectedStateManager(max_key, num_column_families), + expected_state_dir_path_(std::move(expected_state_dir_path)) { + assert(!expected_state_dir_path_.empty()); +} + +Status FileExpectedStateManager::Open() { + // Before doing anything, sync directory state with ours. That is, determine + // `saved_seqno_`, and create any necessary missing files. + std::vector expected_state_dir_children; + Status s = Env::Default()->GetChildren(expected_state_dir_path_, + &expected_state_dir_children); + bool found_trace = false; + if (s.ok()) { + for (size_t i = 0; i < expected_state_dir_children.size(); ++i) { + const auto& filename = expected_state_dir_children[i]; + if (filename.size() >= kStateFilenameSuffix.size() && + filename.rfind(kStateFilenameSuffix) == + filename.size() - kStateFilenameSuffix.size() && + filename.rfind(kLatestBasename, 0) == std::string::npos) { + SequenceNumber found_seqno = ParseUint64( + filename.substr(0, filename.size() - kStateFilenameSuffix.size())); + if (saved_seqno_ == kMaxSequenceNumber || found_seqno > saved_seqno_) { + saved_seqno_ = found_seqno; + } + } + } + // Check if crash happened after creating state file but before creating + // trace file. + if (saved_seqno_ != kMaxSequenceNumber) { + std::string saved_seqno_trace_path = GetPathForFilename( + std::to_string(saved_seqno_) + kTraceFilenameSuffix); + Status exists_status = Env::Default()->FileExists(saved_seqno_trace_path); + if (exists_status.ok()) { + found_trace = true; + } else if (exists_status.IsNotFound()) { + found_trace = false; + } else { + s = exists_status; + } + } + } + if (s.ok() && saved_seqno_ != kMaxSequenceNumber && !found_trace) { + // Create an empty trace file so later logic does not need to distinguish + // missing vs. empty trace file. + std::unique_ptr wfile; + const EnvOptions soptions; + std::string saved_seqno_trace_path = + GetPathForFilename(std::to_string(saved_seqno_) + kTraceFilenameSuffix); + s = Env::Default()->NewWritableFile(saved_seqno_trace_path, &wfile, + soptions); + } + + if (s.ok()) { + s = Clean(); + } + + std::string expected_state_file_path = + GetPathForFilename(kLatestBasename + kStateFilenameSuffix); + bool found = false; + if (s.ok()) { + Status exists_status = Env::Default()->FileExists(expected_state_file_path); + if (exists_status.ok()) { + found = true; + } else if (exists_status.IsNotFound()) { + found = false; + } else { + s = exists_status; + } + } + + if (!found) { + // Initialize the file in a temp path and then rename it. That way, in case + // this process is killed during setup, `Clean()` will take care of removing + // the incomplete expected values file. + std::string temp_expected_state_file_path = + GetTempPathForFilename(kLatestBasename + kStateFilenameSuffix); + FileExpectedState temp_expected_state(temp_expected_state_file_path, + max_key_, num_column_families_); + if (s.ok()) { + s = temp_expected_state.Open(true /* create */); + } + if (s.ok()) { + s = Env::Default()->RenameFile(temp_expected_state_file_path, + expected_state_file_path); + } + } + + if (s.ok()) { + latest_.reset(new FileExpectedState(std::move(expected_state_file_path), + max_key_, num_column_families_)); + s = latest_->Open(false /* create */); + } + return s; +} + +Status FileExpectedStateManager::SaveAtAndAfter(DB* db) { + SequenceNumber seqno = db->GetLatestSequenceNumber(); + + std::string state_filename = std::to_string(seqno) + kStateFilenameSuffix; + std::string state_file_temp_path = GetTempPathForFilename(state_filename); + std::string state_file_path = GetPathForFilename(state_filename); + + std::string latest_file_path = + GetPathForFilename(kLatestBasename + kStateFilenameSuffix); + + std::string trace_filename = std::to_string(seqno) + kTraceFilenameSuffix; + std::string trace_file_path = GetPathForFilename(trace_filename); + + // Populate a tempfile and then rename it to atomically create ".state" + // with contents from "LATEST.state" + Status s = CopyFile(FileSystem::Default(), latest_file_path, + state_file_temp_path, 0 /* size */, false /* use_fsync */, + nullptr /* io_tracer */, Temperature::kUnknown); + if (s.ok()) { + s = FileSystem::Default()->RenameFile(state_file_temp_path, state_file_path, + IOOptions(), nullptr /* dbg */); + } + SequenceNumber old_saved_seqno = 0; + if (s.ok()) { + old_saved_seqno = saved_seqno_; + saved_seqno_ = seqno; + } + + // If there is a crash now, i.e., after ".state" was created but before + // ".trace" is created, it will be treated as if ".trace" were + // present but empty. + + // Create ".trace" directly. It is initially empty so no need for + // tempfile. + std::unique_ptr trace_writer; + if (s.ok()) { + EnvOptions soptions; + // Disable buffering so traces will not get stuck in application buffer. + soptions.writable_file_max_buffer_size = 0; + s = NewFileTraceWriter(Env::Default(), soptions, trace_file_path, + &trace_writer); + } + if (s.ok()) { + TraceOptions trace_opts; + trace_opts.filter |= kTraceFilterGet; + trace_opts.filter |= kTraceFilterMultiGet; + trace_opts.filter |= kTraceFilterIteratorSeek; + trace_opts.filter |= kTraceFilterIteratorSeekForPrev; + trace_opts.preserve_write_order = true; + s = db->StartTrace(trace_opts, std::move(trace_writer)); + } + + // Delete old state/trace files. Deletion order does not matter since we only + // delete after successfully saving new files, so old files will never be used + // again, even if we crash. + if (s.ok() && old_saved_seqno != kMaxSequenceNumber && + old_saved_seqno != saved_seqno_) { + s = Env::Default()->DeleteFile(GetPathForFilename( + std::to_string(old_saved_seqno) + kStateFilenameSuffix)); + } + if (s.ok() && old_saved_seqno != kMaxSequenceNumber && + old_saved_seqno != saved_seqno_) { + s = Env::Default()->DeleteFile(GetPathForFilename( + std::to_string(old_saved_seqno) + kTraceFilenameSuffix)); + } + return s; +} + +bool FileExpectedStateManager::HasHistory() { + return saved_seqno_ != kMaxSequenceNumber; +} + + +namespace { + +// An `ExpectedStateTraceRecordHandler` applies a configurable number of +// write operation trace records to the configured expected state. It is used in +// `FileExpectedStateManager::Restore()` to sync the expected state with the +// DB's post-recovery state. +class ExpectedStateTraceRecordHandler : public TraceRecord::Handler, + public WriteBatch::Handler { + public: + ExpectedStateTraceRecordHandler(uint64_t max_write_ops, ExpectedState* state) + : max_write_ops_(max_write_ops), + state_(state), + buffered_writes_(nullptr) {} + + ~ExpectedStateTraceRecordHandler() { assert(IsDone()); } + + // True if we have already reached the limit on write operations to apply. + bool IsDone() { return num_write_ops_ == max_write_ops_; } + + Status Handle(const WriteQueryTraceRecord& record, + std::unique_ptr* /* result */) override { + if (IsDone()) { + return Status::OK(); + } + WriteBatch batch(record.GetWriteBatchRep().ToString()); + return batch.Iterate(this); + } + + // Ignore reads. + Status Handle(const GetQueryTraceRecord& /* record */, + std::unique_ptr* /* result */) override { + return Status::OK(); + } + + // Ignore reads. + Status Handle(const IteratorSeekQueryTraceRecord& /* record */, + std::unique_ptr* /* result */) override { + return Status::OK(); + } + + // Ignore reads. + Status Handle(const MultiGetQueryTraceRecord& /* record */, + std::unique_ptr* /* result */) override { + return Status::OK(); + } + + // Below are the WriteBatch::Handler overrides. We could use a separate + // object, but it's convenient and works to share state with the + // `TraceRecord::Handler`. + + Status PutCF(uint32_t column_family_id, const Slice& key_with_ts, + const Slice& value) override { + Slice key = + StripTimestampFromUserKey(key_with_ts, FLAGS_user_timestamp_size); + uint64_t key_id; + if (!GetIntVal(key.ToString(), &key_id)) { + return Status::Corruption("unable to parse key", key.ToString()); + } + uint32_t value_base = GetValueBase(value); + + bool should_buffer_write = !(buffered_writes_ == nullptr); + if (should_buffer_write) { + return WriteBatchInternal::Put(buffered_writes_.get(), column_family_id, + key, value); + } + + state_->SyncPut(column_family_id, static_cast(key_id), value_base); + ++num_write_ops_; + return Status::OK(); + } + + Status PutEntityCF(uint32_t column_family_id, const Slice& key_with_ts, + const Slice& entity) override { + Slice key = + StripTimestampFromUserKey(key_with_ts, FLAGS_user_timestamp_size); + + uint64_t key_id = 0; + if (!GetIntVal(key.ToString(), &key_id)) { + return Status::Corruption("Unable to parse key", key.ToString()); + } + + Slice entity_copy = entity; + WideColumns columns; + if (!WideColumnSerialization::Deserialize(entity_copy, columns).ok()) { + return Status::Corruption("Unable to deserialize entity", + entity.ToString(/* hex */ true)); + } + + if (!VerifyWideColumns(columns)) { + return Status::Corruption("Wide columns in entity inconsistent", + entity.ToString(/* hex */ true)); + } + + if (buffered_writes_) { + return WriteBatchInternal::PutEntity(buffered_writes_.get(), + column_family_id, key, columns); + } + + assert(!columns.empty()); + assert(columns.front().name() == kDefaultWideColumnName); + + const uint32_t value_base = GetValueBase(columns.front().value()); + + state_->SyncPut(column_family_id, static_cast(key_id), value_base); + + ++num_write_ops_; + + return Status::OK(); + } + + Status DeleteCF(uint32_t column_family_id, + const Slice& key_with_ts) override { + Slice key = + StripTimestampFromUserKey(key_with_ts, FLAGS_user_timestamp_size); + uint64_t key_id; + if (!GetIntVal(key.ToString(), &key_id)) { + return Status::Corruption("unable to parse key", key.ToString()); + } + + bool should_buffer_write = !(buffered_writes_ == nullptr); + if (should_buffer_write) { + return WriteBatchInternal::Delete(buffered_writes_.get(), + column_family_id, key); + } + + state_->SyncDelete(column_family_id, static_cast(key_id)); + ++num_write_ops_; + return Status::OK(); + } + + Status SingleDeleteCF(uint32_t column_family_id, + const Slice& key_with_ts) override { + bool should_buffer_write = !(buffered_writes_ == nullptr); + if (should_buffer_write) { + Slice key = + StripTimestampFromUserKey(key_with_ts, FLAGS_user_timestamp_size); + Slice ts = + ExtractTimestampFromUserKey(key_with_ts, FLAGS_user_timestamp_size); + std::array key_with_ts_arr{{key, ts}}; + return WriteBatchInternal::SingleDelete( + buffered_writes_.get(), column_family_id, + SliceParts(key_with_ts_arr.data(), 2)); + } + + return DeleteCF(column_family_id, key_with_ts); + } + + Status DeleteRangeCF(uint32_t column_family_id, + const Slice& begin_key_with_ts, + const Slice& end_key_with_ts) override { + Slice begin_key = + StripTimestampFromUserKey(begin_key_with_ts, FLAGS_user_timestamp_size); + Slice end_key = + StripTimestampFromUserKey(end_key_with_ts, FLAGS_user_timestamp_size); + uint64_t begin_key_id, end_key_id; + if (!GetIntVal(begin_key.ToString(), &begin_key_id)) { + return Status::Corruption("unable to parse begin key", + begin_key.ToString()); + } + if (!GetIntVal(end_key.ToString(), &end_key_id)) { + return Status::Corruption("unable to parse end key", end_key.ToString()); + } + + bool should_buffer_write = !(buffered_writes_ == nullptr); + if (should_buffer_write) { + return WriteBatchInternal::DeleteRange( + buffered_writes_.get(), column_family_id, begin_key, end_key); + } + + state_->SyncDeleteRange(column_family_id, + static_cast(begin_key_id), + static_cast(end_key_id)); + ++num_write_ops_; + return Status::OK(); + } + + Status MergeCF(uint32_t column_family_id, const Slice& key_with_ts, + const Slice& value) override { + Slice key = + StripTimestampFromUserKey(key_with_ts, FLAGS_user_timestamp_size); + + bool should_buffer_write = !(buffered_writes_ == nullptr); + if (should_buffer_write) { + return WriteBatchInternal::Merge(buffered_writes_.get(), column_family_id, + key, value); + } + + return PutCF(column_family_id, key, value); + } + + Status MarkBeginPrepare(bool = false) override { + assert(!buffered_writes_); + buffered_writes_.reset(new WriteBatch()); + return Status::OK(); + } + + Status MarkEndPrepare(const Slice& xid) override { + assert(buffered_writes_); + std::string xid_str = xid.ToString(); + assert(xid_to_buffered_writes_.find(xid_str) == + xid_to_buffered_writes_.end()); + + xid_to_buffered_writes_[xid_str].swap(buffered_writes_); + + buffered_writes_.reset(); + + return Status::OK(); + } + + Status MarkCommit(const Slice& xid) override { + std::string xid_str = xid.ToString(); + assert(xid_to_buffered_writes_.find(xid_str) != + xid_to_buffered_writes_.end()); + assert(xid_to_buffered_writes_.at(xid_str)); + + Status s = xid_to_buffered_writes_.at(xid_str)->Iterate(this); + xid_to_buffered_writes_.erase(xid_str); + + return s; + } + + Status MarkRollback(const Slice& xid) override { + std::string xid_str = xid.ToString(); + assert(xid_to_buffered_writes_.find(xid_str) != + xid_to_buffered_writes_.end()); + assert(xid_to_buffered_writes_.at(xid_str)); + xid_to_buffered_writes_.erase(xid_str); + + return Status::OK(); + } + + private: + uint64_t num_write_ops_ = 0; + uint64_t max_write_ops_; + ExpectedState* state_; + std::unordered_map> + xid_to_buffered_writes_; + std::unique_ptr buffered_writes_; +}; + +} // anonymous namespace + +Status FileExpectedStateManager::Restore(DB* db) { + assert(HasHistory()); + SequenceNumber seqno = db->GetLatestSequenceNumber(); + if (seqno < saved_seqno_) { + return Status::Corruption("DB is older than any restorable expected state"); + } + + std::string state_filename = + std::to_string(saved_seqno_) + kStateFilenameSuffix; + std::string state_file_path = GetPathForFilename(state_filename); + + std::string latest_file_temp_path = + GetTempPathForFilename(kLatestBasename + kStateFilenameSuffix); + std::string latest_file_path = + GetPathForFilename(kLatestBasename + kStateFilenameSuffix); + + std::string trace_filename = + std::to_string(saved_seqno_) + kTraceFilenameSuffix; + std::string trace_file_path = GetPathForFilename(trace_filename); + + std::unique_ptr trace_reader; + Status s = NewFileTraceReader(Env::Default(), EnvOptions(), trace_file_path, + &trace_reader); + + if (s.ok()) { + // We are going to replay on top of "`seqno`.state" to create a new + // "LATEST.state". Start off by creating a tempfile so we can later make the + // new "LATEST.state" appear atomically using `RenameFile()`. + s = CopyFile(FileSystem::Default(), state_file_path, latest_file_temp_path, + 0 /* size */, false /* use_fsync */, nullptr /* io_tracer */, + Temperature::kUnknown); + } + + { + std::unique_ptr replayer; + std::unique_ptr state; + std::unique_ptr handler; + if (s.ok()) { + state.reset(new FileExpectedState(latest_file_temp_path, max_key_, + num_column_families_)); + s = state->Open(false /* create */); + } + if (s.ok()) { + handler.reset(new ExpectedStateTraceRecordHandler(seqno - saved_seqno_, + state.get())); + // TODO(ajkr): An API limitation requires we provide `handles` although + // they will be unused since we only use the replayer for reading records. + // Just give a default CFH for now to satisfy the requirement. + s = db->NewDefaultReplayer({db->DefaultColumnFamily()} /* handles */, + std::move(trace_reader), &replayer); + } + + if (s.ok()) { + s = replayer->Prepare(); + } + for (;;) { + std::unique_ptr record; + s = replayer->Next(&record); + if (!s.ok()) { + break; + } + std::unique_ptr res; + record->Accept(handler.get(), &res); + } + if (s.IsCorruption() && handler->IsDone()) { + // There could be a corruption reading the tail record of the trace due to + // `db_stress` crashing while writing it. It shouldn't matter as long as + // we already found all the write ops we need to catch up the expected + // state. + s = Status::OK(); + } + if (s.IsIncomplete()) { + // OK because `Status::Incomplete` is expected upon finishing all the + // trace records. + s = Status::OK(); + } + } + + if (s.ok()) { + s = FileSystem::Default()->RenameFile(latest_file_temp_path, + latest_file_path, IOOptions(), + nullptr /* dbg */); + } + if (s.ok()) { + latest_.reset(new FileExpectedState(latest_file_path, max_key_, + num_column_families_)); + s = latest_->Open(false /* create */); + } + + // Delete old state/trace files. We must delete the state file first. + // Otherwise, a crash-recovery immediately after deleting the trace file could + // lead to `Restore()` unable to replay to `seqno`. + if (s.ok()) { + s = Env::Default()->DeleteFile(state_file_path); + } + if (s.ok()) { + saved_seqno_ = kMaxSequenceNumber; + s = Env::Default()->DeleteFile(trace_file_path); + } + return s; +} + +Status FileExpectedStateManager::Clean() { + std::vector expected_state_dir_children; + Status s = Env::Default()->GetChildren(expected_state_dir_path_, + &expected_state_dir_children); + // An incomplete `Open()` or incomplete `SaveAtAndAfter()` could have left + // behind invalid temporary files. An incomplete `SaveAtAndAfter()` could have + // also left behind stale state/trace files. An incomplete `Restore()` could + // have left behind stale trace files. + for (size_t i = 0; s.ok() && i < expected_state_dir_children.size(); ++i) { + const auto& filename = expected_state_dir_children[i]; + if (filename.rfind(kTempFilenamePrefix, 0 /* pos */) == 0 && + filename.size() >= kTempFilenameSuffix.size() && + filename.rfind(kTempFilenameSuffix) == + filename.size() - kTempFilenameSuffix.size()) { + // Delete all temp files. + s = Env::Default()->DeleteFile(GetPathForFilename(filename)); + } else if (filename.size() >= kStateFilenameSuffix.size() && + filename.rfind(kStateFilenameSuffix) == + filename.size() - kStateFilenameSuffix.size() && + filename.rfind(kLatestBasename, 0) == std::string::npos && + ParseUint64(filename.substr( + 0, filename.size() - kStateFilenameSuffix.size())) < + saved_seqno_) { + assert(saved_seqno_ != kMaxSequenceNumber); + // Delete stale state files. + s = Env::Default()->DeleteFile(GetPathForFilename(filename)); + } else if (filename.size() >= kTraceFilenameSuffix.size() && + filename.rfind(kTraceFilenameSuffix) == + filename.size() - kTraceFilenameSuffix.size() && + ParseUint64(filename.substr( + 0, filename.size() - kTraceFilenameSuffix.size())) < + saved_seqno_) { + // Delete stale trace files. + s = Env::Default()->DeleteFile(GetPathForFilename(filename)); + } + } + return s; +} + +std::string FileExpectedStateManager::GetTempPathForFilename( + const std::string& filename) { + assert(!expected_state_dir_path_.empty()); + std::string expected_state_dir_path_slash = + expected_state_dir_path_.back() == '/' ? expected_state_dir_path_ + : expected_state_dir_path_ + "/"; + return expected_state_dir_path_slash + kTempFilenamePrefix + filename + + kTempFilenameSuffix; +} + +std::string FileExpectedStateManager::GetPathForFilename( + const std::string& filename) { + assert(!expected_state_dir_path_.empty()); + std::string expected_state_dir_path_slash = + expected_state_dir_path_.back() == '/' ? expected_state_dir_path_ + : expected_state_dir_path_ + "/"; + return expected_state_dir_path_slash + filename; +} + +AnonExpectedStateManager::AnonExpectedStateManager(size_t max_key, + size_t num_column_families) + : ExpectedStateManager(max_key, num_column_families) {} + +Status AnonExpectedStateManager::Open() { + latest_.reset(new AnonExpectedState(max_key_, num_column_families_)); + return latest_->Open(true /* create */); +} + +} // namespace ROCKSDB_NAMESPACE + +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db_stress_tool/expected_state.h b/librocksdb-sys/rocksdb/db_stress_tool/expected_state.h new file mode 100644 index 0000000..f3f924c --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/expected_state.h @@ -0,0 +1,329 @@ +// Copyright (c) 2021-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#ifdef GFLAGS + +#pragma once + +#include + +#include +#include + +#include "db/dbformat.h" +#include "db_stress_tool/expected_value.h" +#include "file/file_util.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/file_system.h" +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/types.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { +// `ExpectedState` provides read/write access to expected values stored in +// `ExpectedState` for every key. +class ExpectedState { + public: + explicit ExpectedState(size_t max_key, size_t num_column_families); + + virtual ~ExpectedState() {} + + // Requires external locking preventing concurrent execution with any other + // member function. + virtual Status Open(bool create) = 0; + + // Requires external locking covering all keys in `cf`. + void ClearColumnFamily(int cf); + + // Prepare a Put that will be started but not finished yet + // This is useful for crash-recovery testing when the process may crash + // before updating the corresponding expected value + // + // Requires external locking covering `key` in `cf` to prevent concurrent + // write or delete to the same `key`. + PendingExpectedValue PreparePut(int cf, int64_t key); + + // Does not requires external locking. + ExpectedValue Get(int cf, int64_t key); + + // Prepare a Delete that will be started but not finished yet + // This is useful for crash-recovery testing when the process may crash + // before updating the corresponding expected value + // + // Requires external locking covering `key` in `cf` to prevent concurrent + // write or delete to the same `key`. + PendingExpectedValue PrepareDelete(int cf, int64_t key, + bool* prepared = nullptr); + + // Requires external locking covering `key` in `cf` to prevent concurrent + // write or delete to the same `key`. + PendingExpectedValue PrepareSingleDelete(int cf, int64_t key); + + // Requires external locking covering keys in `[begin_key, end_key)` in `cf` + // to prevent concurrent write or delete to the same `key`. + std::vector PrepareDeleteRange(int cf, + int64_t begin_key, + int64_t end_key); + + // Update the expected value for start of an incomplete write or delete + // operation on the key assoicated with this expected value + void Precommit(int cf, int64_t key, const ExpectedValue& value); + + // Requires external locking covering `key` in `cf` to prevent concurrent + // delete to the same `key`. + bool Exists(int cf, int64_t key); + + // Sync the `value_base` to the corresponding expected value + // + // Requires external locking covering `key` in `cf` or be in single thread + // to prevent concurrent write or delete to the same `key` + void SyncPut(int cf, int64_t key, uint32_t value_base); + + // Sync the corresponding expected value to be pending Put + // + // Requires external locking covering `key` in `cf` or be in single thread + // to prevent concurrent write or delete to the same `key` + void SyncPendingPut(int cf, int64_t key); + + // Sync the corresponding expected value to be deleted + // + // Requires external locking covering `key` in `cf` or be in single thread + // to prevent concurrent write or delete to the same `key` + void SyncDelete(int cf, int64_t key); + + // Sync the corresponding expected values to be deleted + // + // Requires external locking covering keys in `[begin_key, end_key)` in `cf` + // to prevent concurrent write or delete to the same `key` + void SyncDeleteRange(int cf, int64_t begin_key, int64_t end_key); + + private: + // Does not requires external locking. + std::atomic& Value(int cf, int64_t key) const { + return values_[cf * max_key_ + key]; + } + + // Does not requires external locking + ExpectedValue Load(int cf, int64_t key) const { + return ExpectedValue(Value(cf, key).load()); + } + + const size_t max_key_; + const size_t num_column_families_; + + protected: + size_t GetValuesLen() const { + return sizeof(std::atomic) * num_column_families_ * max_key_; + } + + // Requires external locking preventing concurrent execution with any other + // member function. + void Reset(); + + std::atomic* values_; +}; + +// A `FileExpectedState` implements `ExpectedState` backed by a file. +class FileExpectedState : public ExpectedState { + public: + explicit FileExpectedState(std::string expected_state_file_path, + size_t max_key, size_t num_column_families); + + // Requires external locking preventing concurrent execution with any other + // member function. + Status Open(bool create) override; + + private: + const std::string expected_state_file_path_; + std::unique_ptr expected_state_mmap_buffer_; +}; + +// An `AnonExpectedState` implements `ExpectedState` backed by a memory +// allocation. +class AnonExpectedState : public ExpectedState { + public: + explicit AnonExpectedState(size_t max_key, size_t num_column_families); + + // Requires external locking preventing concurrent execution with any other + // member function. + Status Open(bool create) override; + + private: + std::unique_ptr[]> values_allocation_; +}; + +// An `ExpectedStateManager` manages data about the expected state of the +// database. It exposes operations for reading and modifying the latest +// expected state. +class ExpectedStateManager { + public: + explicit ExpectedStateManager(size_t max_key, size_t num_column_families); + + virtual ~ExpectedStateManager(); + + // Requires external locking preventing concurrent execution with any other + // member function. + virtual Status Open() = 0; + + // Saves expected values for the current state of `db` and begins tracking + // changes. Following a successful `SaveAtAndAfter()`, `Restore()` can be + // called on the same DB, as long as its state does not roll back to before + // its current state. + // + // Requires external locking preventing concurrent execution with any other + // member function. Furthermore, `db` must not be mutated while this function + // is executing. + virtual Status SaveAtAndAfter(DB* db) = 0; + + // Returns true if at least one state of historical expected values can be + // restored. + // + // Requires external locking preventing concurrent execution with any other + // member function. + virtual bool HasHistory() = 0; + + // Restores expected values according to the current state of `db`. See + // `SaveAtAndAfter()` for conditions where this can be called. + // + // Requires external locking preventing concurrent execution with any other + // member function. Furthermore, `db` must not be mutated while this function + // is executing. + virtual Status Restore(DB* db) = 0; + + // Requires external locking covering all keys in `cf`. + void ClearColumnFamily(int cf) { return latest_->ClearColumnFamily(cf); } + + // See ExpectedState::PreparePut() + PendingExpectedValue PreparePut(int cf, int64_t key) { + return latest_->PreparePut(cf, key); + } + + // See ExpectedState::Get() + ExpectedValue Get(int cf, int64_t key) { return latest_->Get(cf, key); } + + // See ExpectedState::PrepareDelete() + PendingExpectedValue PrepareDelete(int cf, int64_t key) { + return latest_->PrepareDelete(cf, key); + } + + // See ExpectedState::PrepareSingleDelete() + PendingExpectedValue PrepareSingleDelete(int cf, int64_t key) { + return latest_->PrepareSingleDelete(cf, key); + } + + // See ExpectedState::PrepareDeleteRange() + std::vector PrepareDeleteRange(int cf, + int64_t begin_key, + int64_t end_key) { + return latest_->PrepareDeleteRange(cf, begin_key, end_key); + } + + // See ExpectedState::Exists() + bool Exists(int cf, int64_t key) { return latest_->Exists(cf, key); } + + // See ExpectedState::SyncPut() + void SyncPut(int cf, int64_t key, uint32_t value_base) { + return latest_->SyncPut(cf, key, value_base); + } + + // See ExpectedState::SyncPendingPut() + void SyncPendingPut(int cf, int64_t key) { + return latest_->SyncPendingPut(cf, key); + } + + // See ExpectedState::SyncDelete() + void SyncDelete(int cf, int64_t key) { return latest_->SyncDelete(cf, key); } + + // See ExpectedState::SyncDeleteRange() + void SyncDeleteRange(int cf, int64_t begin_key, int64_t end_key) { + return latest_->SyncDeleteRange(cf, begin_key, end_key); + } + + protected: + const size_t max_key_; + const size_t num_column_families_; + std::unique_ptr latest_; +}; + +// A `FileExpectedStateManager` implements an `ExpectedStateManager` backed by +// a directory of files containing data about the expected state of the +// database. +class FileExpectedStateManager : public ExpectedStateManager { + public: + explicit FileExpectedStateManager(size_t max_key, size_t num_column_families, + std::string expected_state_dir_path); + + // Requires external locking preventing concurrent execution with any other + // member function. + Status Open() override; + + // See `ExpectedStateManager::SaveAtAndAfter()` API doc. + // + // This implementation makes a copy of "LATEST.state" into + // ".state", and starts a trace in ".trace". + // Due to using external files, a following `Restore()` can happen even + // from a different process. + Status SaveAtAndAfter(DB* db) override; + + // See `ExpectedStateManager::HasHistory()` API doc. + bool HasHistory() override; + + // See `ExpectedStateManager::Restore()` API doc. + // + // Say `db->GetLatestSequenceNumber()` was `a` last time `SaveAtAndAfter()` + // was called and now it is `b`. Then this function replays `b - a` write + // operations from "`a`.trace" onto "`a`.state", and then copies the resulting + // file into "LATEST.state". + Status Restore(DB* db) override; + + private: + // Requires external locking preventing concurrent execution with any other + // member function. + Status Clean(); + + std::string GetTempPathForFilename(const std::string& filename); + std::string GetPathForFilename(const std::string& filename); + + static const std::string kLatestBasename; + static const std::string kStateFilenameSuffix; + static const std::string kTraceFilenameSuffix; + static const std::string kTempFilenamePrefix; + static const std::string kTempFilenameSuffix; + + const std::string expected_state_dir_path_; + SequenceNumber saved_seqno_ = kMaxSequenceNumber; +}; + +// An `AnonExpectedStateManager` implements an `ExpectedStateManager` backed by +// a memory allocation containing data about the expected state of the database. +class AnonExpectedStateManager : public ExpectedStateManager { + public: + explicit AnonExpectedStateManager(size_t max_key, size_t num_column_families); + + // See `ExpectedStateManager::SaveAtAndAfter()` API doc. + // + // This implementation returns `Status::NotSupported` since we do not + // currently have a need to keep history of expected state within a process. + Status SaveAtAndAfter(DB* /* db */) override { + return Status::NotSupported(); + } + + // See `ExpectedStateManager::HasHistory()` API doc. + bool HasHistory() override { return false; } + + // See `ExpectedStateManager::Restore()` API doc. + // + // This implementation returns `Status::NotSupported` since we do not + // currently have a need to keep history of expected state within a process. + Status Restore(DB* /* db */) override { return Status::NotSupported(); } + + // Requires external locking preventing concurrent execution with any other + // member function. + Status Open() override; +}; +} // namespace ROCKSDB_NAMESPACE + +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db_stress_tool/expected_value.cc b/librocksdb-sys/rocksdb/db_stress_tool/expected_value.cc new file mode 100644 index 0000000..d280055 --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/expected_value.cc @@ -0,0 +1,122 @@ +// Copyright (c) 2021-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#ifdef GFLAGS + +#include "db_stress_tool/expected_value.h" + +#include + +namespace ROCKSDB_NAMESPACE { +void ExpectedValue::Put(bool pending) { + if (pending) { + SetPendingWrite(); + } else { + SetValueBase(NextValueBase()); + ClearDeleted(); + ClearPendingWrite(); + } +} + +bool ExpectedValue::Delete(bool pending) { + if (!Exists()) { + return false; + } + if (pending) { + SetPendingDel(); + } else { + SetDelCounter(NextDelCounter()); + SetDeleted(); + ClearPendingDel(); + } + return true; +} + +void ExpectedValue::SyncPut(uint32_t value_base) { + assert(ExpectedValue::IsValueBaseValid(value_base)); + + SetValueBase(value_base); + ClearDeleted(); + ClearPendingWrite(); + + // This is needed in case crash happens during a pending delete of the key + // assocated with this expected value + ClearPendingDel(); +} + +void ExpectedValue::SyncPendingPut() { Put(true /* pending */); } + +void ExpectedValue::SyncDelete() { + Delete(false /* pending */); + // This is needed in case crash happens during a pending write of the key + // assocated with this expected value + ClearPendingWrite(); +} + +uint32_t ExpectedValue::GetFinalValueBase() const { + return PendingWrite() ? NextValueBase() : GetValueBase(); +} + +uint32_t ExpectedValue::GetFinalDelCounter() const { + return PendingDelete() ? NextDelCounter() : GetDelCounter(); +} + +bool ExpectedValueHelper::MustHaveNotExisted( + ExpectedValue pre_read_expected_value, + ExpectedValue post_read_expected_value) { + const bool pre_read_expected_deleted = pre_read_expected_value.IsDeleted(); + + const uint32_t pre_read_expected_value_base = + pre_read_expected_value.GetValueBase(); + + const uint32_t post_read_expected_final_value_base = + post_read_expected_value.GetFinalValueBase(); + + const bool during_read_no_write_happened = + (pre_read_expected_value_base == post_read_expected_final_value_base); + return pre_read_expected_deleted && during_read_no_write_happened; +} + +bool ExpectedValueHelper::MustHaveExisted( + ExpectedValue pre_read_expected_value, + ExpectedValue post_read_expected_value) { + const bool pre_read_expected_not_deleted = + !pre_read_expected_value.IsDeleted(); + + const uint32_t pre_read_expected_del_counter = + pre_read_expected_value.GetDelCounter(); + const uint32_t post_read_expected_final_del_counter = + post_read_expected_value.GetFinalDelCounter(); + + const bool during_read_no_delete_happened = + (pre_read_expected_del_counter == post_read_expected_final_del_counter); + + return pre_read_expected_not_deleted && during_read_no_delete_happened; +} + +bool ExpectedValueHelper::InExpectedValueBaseRange( + uint32_t value_base, ExpectedValue pre_read_expected_value, + ExpectedValue post_read_expected_value) { + assert(ExpectedValue::IsValueBaseValid(value_base)); + + const uint32_t pre_read_expected_value_base = + pre_read_expected_value.GetValueBase(); + const uint32_t post_read_expected_final_value_base = + post_read_expected_value.GetFinalValueBase(); + + if (pre_read_expected_value_base <= post_read_expected_final_value_base) { + const uint32_t lower_bound = pre_read_expected_value_base; + const uint32_t upper_bound = post_read_expected_final_value_base; + return lower_bound <= value_base && value_base <= upper_bound; + } else { + const uint32_t upper_bound_1 = post_read_expected_final_value_base; + const uint32_t lower_bound_2 = pre_read_expected_value_base; + const uint32_t upper_bound_2 = ExpectedValue::GetValueBaseMask(); + return (value_base <= upper_bound_1) || + (lower_bound_2 <= value_base && value_base <= upper_bound_2); + } +} +} // namespace ROCKSDB_NAMESPACE +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db_stress_tool/expected_value.h b/librocksdb-sys/rocksdb/db_stress_tool/expected_value.h new file mode 100644 index 0000000..338afc0 --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/expected_value.h @@ -0,0 +1,208 @@ +// Copyright (c) 2021-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#ifdef GFLAGS + +#pragma once + +#include + +#include +#include +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { +// `ExpectedValue` represents the expected value of a key used in db stress, +// which provides APIs to obtain various information e.g, value base, existence, +// pending operation status and APIs to edit expected value. +// +// This class is not thread-safe. +class ExpectedValue { + public: + static uint32_t GetValueBaseMask() { return VALUE_BASE_MASK; } + static uint32_t GetValueBaseDelta() { return VALUE_BASE_DELTA; } + static uint32_t GetDelCounterDelta() { return DEL_COUNTER_DELTA; } + static uint32_t GetDelMask() { return DEL_MASK; } + static bool IsValueBaseValid(uint32_t value_base) { + return IsValuePartValid(value_base, VALUE_BASE_MASK); + } + + ExpectedValue() : expected_value_(DEL_MASK) {} + + explicit ExpectedValue(uint32_t expected_value) + : expected_value_(expected_value) {} + + bool Exists() const { return PendingWrite() || !IsDeleted(); } + + uint32_t Read() const { return expected_value_; } + + void Put(bool pending); + + bool Delete(bool pending); + + void SyncPut(uint32_t value_base); + + void SyncPendingPut(); + + void SyncDelete(); + + uint32_t GetValueBase() const { return GetValuePart(VALUE_BASE_MASK); } + + uint32_t NextValueBase() const { + return GetIncrementedValuePart(VALUE_BASE_MASK, VALUE_BASE_DELTA); + } + + void SetValueBase(uint32_t new_value_base) { + SetValuePart(VALUE_BASE_MASK, new_value_base); + } + + bool PendingWrite() const { + const uint32_t pending_write = GetValuePart(PENDING_WRITE_MASK); + return pending_write != 0; + } + + void SetPendingWrite() { + SetValuePart(PENDING_WRITE_MASK, PENDING_WRITE_MASK); + } + + void ClearPendingWrite() { ClearValuePart(PENDING_WRITE_MASK); } + + uint32_t GetDelCounter() const { return GetValuePart(DEL_COUNTER_MASK); } + + uint32_t NextDelCounter() const { + return GetIncrementedValuePart(DEL_COUNTER_MASK, DEL_COUNTER_DELTA); + } + + void SetDelCounter(uint32_t new_del_counter) { + SetValuePart(DEL_COUNTER_MASK, new_del_counter); + } + + bool PendingDelete() const { + const uint32_t pending_del = GetValuePart(PENDING_DEL_MASK); + return pending_del != 0; + } + + void SetPendingDel() { SetValuePart(PENDING_DEL_MASK, PENDING_DEL_MASK); } + + void ClearPendingDel() { ClearValuePart(PENDING_DEL_MASK); } + + bool IsDeleted() const { + const uint32_t deleted = GetValuePart(DEL_MASK); + return deleted != 0; + } + + void SetDeleted() { SetValuePart(DEL_MASK, DEL_MASK); } + + void ClearDeleted() { ClearValuePart(DEL_MASK); } + + uint32_t GetFinalValueBase() const; + + uint32_t GetFinalDelCounter() const; + + private: + static bool IsValuePartValid(uint32_t value_part, uint32_t value_part_mask) { + return (value_part & (~value_part_mask)) == 0; + } + + // The 32-bit expected_value_ is divided into following parts: + // Bit 0 - 14: value base + static constexpr uint32_t VALUE_BASE_MASK = 0x7fff; + static constexpr uint32_t VALUE_BASE_DELTA = 1; + // Bit 15: whether write to this value base is pending (0 equals `false`) + static constexpr uint32_t PENDING_WRITE_MASK = (uint32_t)1 << 15; + // Bit 16 - 29: deletion counter (i.e, number of times this value base has + // been deleted) + static constexpr uint32_t DEL_COUNTER_MASK = 0x3fff0000; + static constexpr uint32_t DEL_COUNTER_DELTA = (uint32_t)1 << 16; + // Bit 30: whether deletion of this value base is pending (0 equals `false`) + static constexpr uint32_t PENDING_DEL_MASK = (uint32_t)1 << 30; + // Bit 31: whether this value base is deleted (0 equals `false`) + static constexpr uint32_t DEL_MASK = (uint32_t)1 << 31; + + uint32_t GetValuePart(uint32_t value_part_mask) const { + return expected_value_ & value_part_mask; + } + + uint32_t GetIncrementedValuePart(uint32_t value_part_mask, + uint32_t value_part_delta) const { + uint32_t current_value_part = GetValuePart(value_part_mask); + ExpectedValue temp_expected_value(current_value_part + value_part_delta); + return temp_expected_value.GetValuePart(value_part_mask); + } + + void SetValuePart(uint32_t value_part_mask, uint32_t new_value_part) { + assert(IsValuePartValid(new_value_part, value_part_mask)); + ClearValuePart(value_part_mask); + expected_value_ |= new_value_part; + } + + void ClearValuePart(uint32_t value_part_mask) { + expected_value_ &= (~value_part_mask); + } + + uint32_t expected_value_; +}; + +// `PendingExpectedValue` represents the expected value of a key undergoing a +// pending operation in db stress. +// +// This class is not thread-safe. +class PendingExpectedValue { + public: + explicit PendingExpectedValue(std::atomic* value_ptr, + ExpectedValue orig_value, + ExpectedValue final_value) + : value_ptr_(value_ptr), + orig_value_(orig_value), + final_value_(final_value) {} + + void Commit() { + // To prevent low-level instruction reordering that results + // in setting expected value happens before db write + std::atomic_thread_fence(std::memory_order_release); + value_ptr_->store(final_value_.Read()); + } + + uint32_t GetFinalValueBase() { return final_value_.GetValueBase(); } + + private: + std::atomic* const value_ptr_; + const ExpectedValue orig_value_; + const ExpectedValue final_value_; +}; + +// `ExpectedValueHelper` provides utils to parse `ExpectedValue` to obtain +// useful info about it in db stress +class ExpectedValueHelper { + public: + // Return whether the key associated with `pre_read_expected_value` and + // `post_read_expected_value` is expected not to exist from begining till the + // end of the read + // + // The negation of `MustHaveNotExisted()` is "may have not existed". + // To assert some key must have existsed, please use `MustHaveExisted()` + static bool MustHaveNotExisted(ExpectedValue pre_read_expected_value, + ExpectedValue post_read_expected_value); + + // Return whether the key associated with `pre_read_expected_value` and + // `post_read_expected_value` is expected to exist from begining till the end + // of the read. + // + // The negation of `MustHaveExisted()` is "may have existed". + // To assert some key must have not existsed, please use + // `MustHaveNotExisted()` + static bool MustHaveExisted(ExpectedValue pre_read_expected_value, + ExpectedValue post_read_expected_value); + + // Return whether the `value_base` falls within the expected value base + static bool InExpectedValueBaseRange(uint32_t value_base, + ExpectedValue pre_read_expected_value, + ExpectedValue post_read_expected_value); +}; +} // namespace ROCKSDB_NAMESPACE + +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db_stress_tool/multi_ops_txns_stress.cc b/librocksdb-sys/rocksdb/db_stress_tool/multi_ops_txns_stress.cc new file mode 100644 index 0000000..1591a52 --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/multi_ops_txns_stress.cc @@ -0,0 +1,1749 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifdef GFLAGS +#include "db_stress_tool/multi_ops_txns_stress.h" + +#include "rocksdb/utilities/write_batch_with_index.h" +#include "util/defer.h" +#include "utilities/fault_injection_fs.h" +#include "utilities/transactions/write_prepared_txn_db.h" + +namespace ROCKSDB_NAMESPACE { + +// The description of A and C can be found in multi_ops_txns_stress.h +DEFINE_int32(lb_a, 0, "(Inclusive) lower bound of A"); +DEFINE_int32(ub_a, 1000, "(Exclusive) upper bound of A"); +DEFINE_int32(lb_c, 0, "(Inclusive) lower bound of C"); +DEFINE_int32(ub_c, 1000, "(Exclusive) upper bound of C"); + +DEFINE_string(key_spaces_path, "", + "Path to file describing the lower and upper bounds of A and C"); + +DEFINE_int32(delay_snapshot_read_one_in, 0, + "With a chance of 1/N, inject a random delay between taking " + "snapshot and read."); + +DEFINE_int32(rollback_one_in, 0, + "If non-zero, rollback non-read-only transactions with a " + "probability of 1/N."); + +DEFINE_int32(clear_wp_commit_cache_one_in, 0, + "If non-zero, evict all commit entries from commit cache with a " + "probability of 1/N. This options applies to write-prepared and " + "write-unprepared transactions."); + +extern "C" bool rocksdb_write_prepared_TEST_ShouldClearCommitCache(void) { + static Random rand(static_cast(db_stress_env->NowMicros())); + return FLAGS_clear_wp_commit_cache_one_in > 0 && + rand.OneIn(FLAGS_clear_wp_commit_cache_one_in); +} + +// MultiOpsTxnsStressTest can either operate on a database with pre-populated +// data (possibly from previous ones), or create a new db and preload it with +// data specified via `-lb_a`, `-ub_a`, `-lb_c`, `-ub_c`, etc. Among these, we +// define the test key spaces as two key ranges: [lb_a, ub_a) and [lb_c, ub_c). +// The key spaces specification is persisted in a file whose absolute path can +// be specified via `-key_spaces_path`. +// +// Whether an existing db is used or a new one is created, key_spaces_path will +// be used. In the former case, the test reads the key spaces specification +// from `-key_spaces_path` and decodes [lb_a, ub_a) and [lb_c, ub_c). In the +// latter case, the test writes a key spaces specification to a file at the +// location, and this file will be used by future runs until a new db is +// created. +// +// Create a fresh new database (-destroy_db_initially=1 or there is no database +// in the location specified by -db). See PreloadDb(). +// +// Use an existing, non-empty database. See ScanExistingDb(). +// +// This test is multi-threaded, and thread count can be specified via +// `-threads`. For simplicity, we partition the key ranges and each thread +// operates on a subrange independently. +// Within each subrange, a KeyGenerator object is responsible for key +// generation. A KeyGenerator maintains two sets: set of existing keys within +// [low, high), set of non-existing keys within [low, high). [low, high) is the +// subrange. The test initialization makes sure there is at least one +// non-existing key, otherwise the test will return an error and exit before +// any test thread is spawned. + +void MultiOpsTxnsStressTest::KeyGenerator::FinishInit() { + assert(existing_.empty()); + assert(!existing_uniq_.empty()); + assert(low_ < high_); + for (auto v : existing_uniq_) { + assert(low_ <= v); + assert(high_ > v); + existing_.push_back(v); + } + if (non_existing_uniq_.empty()) { + fprintf( + stderr, + "Cannot allocate key in [%u, %u)\nStart with a new DB or try change " + "the number of threads for testing via -threads=<#threads>\n", + static_cast(low_), static_cast(high_)); + fflush(stdout); + fflush(stderr); + assert(false); + } + initialized_ = true; +} + +std::pair +MultiOpsTxnsStressTest::KeyGenerator::ChooseExisting() { + assert(initialized_); + const size_t N = existing_.size(); + assert(N > 0); + uint32_t rnd = rand_.Uniform(static_cast(N)); + assert(rnd < N); + return std::make_pair(existing_[rnd], rnd); +} + +uint32_t MultiOpsTxnsStressTest::KeyGenerator::Allocate() { + assert(initialized_); + auto it = non_existing_uniq_.begin(); + assert(non_existing_uniq_.end() != it); + uint32_t ret = *it; + // Remove this element from non_existing_. + // Need to call UndoAllocation() if the calling transaction does not commit. + non_existing_uniq_.erase(it); + return ret; +} + +void MultiOpsTxnsStressTest::KeyGenerator::Replace(uint32_t old_val, + uint32_t old_pos, + uint32_t new_val) { + assert(initialized_); + { + auto it = existing_uniq_.find(old_val); + assert(it != existing_uniq_.end()); + existing_uniq_.erase(it); + } + + { + assert(0 == existing_uniq_.count(new_val)); + existing_uniq_.insert(new_val); + existing_[old_pos] = new_val; + } + + { + assert(0 == non_existing_uniq_.count(old_val)); + non_existing_uniq_.insert(old_val); + } +} + +void MultiOpsTxnsStressTest::KeyGenerator::UndoAllocation(uint32_t new_val) { + assert(initialized_); + assert(0 == non_existing_uniq_.count(new_val)); + non_existing_uniq_.insert(new_val); +} + +std::string MultiOpsTxnsStressTest::Record::EncodePrimaryKey(uint32_t a) { + std::string ret; + PutFixed32(&ret, kPrimaryIndexId); + PutFixed32(&ret, a); + + char* const buf = &ret[0]; + std::reverse(buf, buf + sizeof(kPrimaryIndexId)); + std::reverse(buf + sizeof(kPrimaryIndexId), + buf + sizeof(kPrimaryIndexId) + sizeof(a)); + return ret; +} + +std::string MultiOpsTxnsStressTest::Record::EncodeSecondaryKey(uint32_t c) { + std::string ret; + PutFixed32(&ret, kSecondaryIndexId); + PutFixed32(&ret, c); + + char* const buf = &ret[0]; + std::reverse(buf, buf + sizeof(kSecondaryIndexId)); + std::reverse(buf + sizeof(kSecondaryIndexId), + buf + sizeof(kSecondaryIndexId) + sizeof(c)); + return ret; +} + +std::string MultiOpsTxnsStressTest::Record::EncodeSecondaryKey(uint32_t c, + uint32_t a) { + std::string ret; + PutFixed32(&ret, kSecondaryIndexId); + PutFixed32(&ret, c); + PutFixed32(&ret, a); + + char* const buf = &ret[0]; + std::reverse(buf, buf + sizeof(kSecondaryIndexId)); + std::reverse(buf + sizeof(kSecondaryIndexId), + buf + sizeof(kSecondaryIndexId) + sizeof(c)); + std::reverse(buf + sizeof(kSecondaryIndexId) + sizeof(c), + buf + sizeof(kSecondaryIndexId) + sizeof(c) + sizeof(a)); + return ret; +} + +std::tuple +MultiOpsTxnsStressTest::Record::DecodePrimaryIndexValue( + Slice primary_index_value) { + if (primary_index_value.size() != 8) { + return std::tuple{Status::Corruption(""), 0, 0}; + } + uint32_t b = 0; + uint32_t c = 0; + if (!GetFixed32(&primary_index_value, &b) || + !GetFixed32(&primary_index_value, &c)) { + assert(false); + return std::tuple{Status::Corruption(""), 0, 0}; + } + return std::tuple{Status::OK(), b, c}; +} + +std::pair +MultiOpsTxnsStressTest::Record::DecodeSecondaryIndexValue( + Slice secondary_index_value) { + if (secondary_index_value.size() != 4) { + return std::make_pair(Status::Corruption(""), 0); + } + uint32_t crc = 0; + bool result __attribute__((unused)) = + GetFixed32(&secondary_index_value, &crc); + assert(result); + return std::make_pair(Status::OK(), crc); +} + +std::pair +MultiOpsTxnsStressTest::Record::EncodePrimaryIndexEntry() const { + std::string primary_index_key = EncodePrimaryKey(); + std::string primary_index_value = EncodePrimaryIndexValue(); + return std::make_pair(primary_index_key, primary_index_value); +} + +std::string MultiOpsTxnsStressTest::Record::EncodePrimaryKey() const { + return EncodePrimaryKey(a_); +} + +std::string MultiOpsTxnsStressTest::Record::EncodePrimaryIndexValue() const { + std::string ret; + PutFixed32(&ret, b_); + PutFixed32(&ret, c_); + return ret; +} + +std::pair +MultiOpsTxnsStressTest::Record::EncodeSecondaryIndexEntry() const { + std::string secondary_index_key = EncodeSecondaryKey(c_, a_); + + // Secondary index value is always 4-byte crc32 of the secondary key + std::string secondary_index_value; + uint32_t crc = + crc32c::Value(secondary_index_key.data(), secondary_index_key.size()); + PutFixed32(&secondary_index_value, crc); + return std::make_pair(std::move(secondary_index_key), secondary_index_value); +} + +std::string MultiOpsTxnsStressTest::Record::EncodeSecondaryKey() const { + return EncodeSecondaryKey(c_, a_); +} + +Status MultiOpsTxnsStressTest::Record::DecodePrimaryIndexEntry( + Slice primary_index_key, Slice primary_index_value) { + if (primary_index_key.size() != 8) { + assert(false); + return Status::Corruption("Primary index key length is not 8"); + } + + uint32_t index_id = 0; + + [[maybe_unused]] bool res = GetFixed32(&primary_index_key, &index_id); + assert(res); + index_id = EndianSwapValue(index_id); + + if (index_id != kPrimaryIndexId) { + std::ostringstream oss; + oss << "Unexpected primary index id: " << index_id; + return Status::Corruption(oss.str()); + } + + res = GetFixed32(&primary_index_key, &a_); + assert(res); + a_ = EndianSwapValue(a_); + assert(primary_index_key.empty()); + + if (primary_index_value.size() != 8) { + return Status::Corruption("Primary index value length is not 8"); + } + GetFixed32(&primary_index_value, &b_); + GetFixed32(&primary_index_value, &c_); + return Status::OK(); +} + +Status MultiOpsTxnsStressTest::Record::DecodeSecondaryIndexEntry( + Slice secondary_index_key, Slice secondary_index_value) { + if (secondary_index_key.size() != 12) { + return Status::Corruption("Secondary index key length is not 12"); + } + uint32_t crc = + crc32c::Value(secondary_index_key.data(), secondary_index_key.size()); + + uint32_t index_id = 0; + + [[maybe_unused]] bool res = GetFixed32(&secondary_index_key, &index_id); + assert(res); + index_id = EndianSwapValue(index_id); + + if (index_id != kSecondaryIndexId) { + std::ostringstream oss; + oss << "Unexpected secondary index id: " << index_id; + return Status::Corruption(oss.str()); + } + + assert(secondary_index_key.size() == 8); + res = GetFixed32(&secondary_index_key, &c_); + assert(res); + c_ = EndianSwapValue(c_); + + assert(secondary_index_key.size() == 4); + res = GetFixed32(&secondary_index_key, &a_); + assert(res); + a_ = EndianSwapValue(a_); + assert(secondary_index_key.empty()); + + if (secondary_index_value.size() != 4) { + return Status::Corruption("Secondary index value length is not 4"); + } + uint32_t val = 0; + GetFixed32(&secondary_index_value, &val); + if (val != crc) { + std::ostringstream oss; + oss << "Secondary index key checksum mismatch, stored: " << val + << ", recomputed: " << crc; + return Status::Corruption(oss.str()); + } + return Status::OK(); +} + +void MultiOpsTxnsStressTest::FinishInitDb(SharedState* shared) { + if (FLAGS_enable_compaction_filter) { + // TODO (yanqin) enable compaction filter + } + ProcessRecoveredPreparedTxns(shared); + + ReopenAndPreloadDbIfNeeded(shared); + // TODO (yanqin) parallelize if key space is large + for (auto& key_gen : key_gen_for_a_) { + assert(key_gen); + key_gen->FinishInit(); + } + // TODO (yanqin) parallelize if key space is large + for (auto& key_gen : key_gen_for_c_) { + assert(key_gen); + key_gen->FinishInit(); + } +} + +void MultiOpsTxnsStressTest::ReopenAndPreloadDbIfNeeded(SharedState* shared) { + (void)shared; + bool db_empty = false; + { + std::unique_ptr iter(db_->NewIterator(ReadOptions())); + iter->SeekToFirst(); + if (!iter->Valid()) { + db_empty = true; + } + } + + if (db_empty) { + PreloadDb(shared, FLAGS_threads, FLAGS_lb_a, FLAGS_ub_a, FLAGS_lb_c, + FLAGS_ub_c); + } else { + fprintf(stdout, + "Key ranges will be read from %s.\n-lb_a, -ub_a, -lb_c, -ub_c will " + "be ignored\n", + FLAGS_key_spaces_path.c_str()); + fflush(stdout); + ScanExistingDb(shared, FLAGS_threads); + } +} + +// Used for point-lookup transaction +Status MultiOpsTxnsStressTest::TestGet( + ThreadState* thread, const ReadOptions& read_opts, + const std::vector& /*rand_column_families*/, + const std::vector& /*rand_keys*/) { + uint32_t a = 0; + uint32_t pos = 0; + std::tie(a, pos) = ChooseExistingA(thread); + return PointLookupTxn(thread, read_opts, a); +} + +// Not used. +std::vector MultiOpsTxnsStressTest::TestMultiGet( + ThreadState* /*thread*/, const ReadOptions& /*read_opts*/, + const std::vector& /*rand_column_families*/, + const std::vector& /*rand_keys*/) { + return std::vector{Status::NotSupported()}; +} + +// Wide columns are currently not supported by transactions. +void MultiOpsTxnsStressTest::TestGetEntity( + ThreadState* /* thread */, const ReadOptions& /* read_opts */, + const std::vector& /* rand_column_families */, + const std::vector& /* rand_keys */) {} + +// Wide columns are currently not supported by transactions. +void MultiOpsTxnsStressTest::TestMultiGetEntity( + ThreadState* /* thread */, const ReadOptions& /* read_opts */, + const std::vector& /* rand_column_families */, + const std::vector& /* rand_keys */) {} + +Status MultiOpsTxnsStressTest::TestPrefixScan( + ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) { + (void)thread; + (void)read_opts; + (void)rand_column_families; + (void)rand_keys; + return Status::OK(); +} + +// Given a key K, this creates an iterator which scans to K and then +// does a random sequence of Next/Prev operations. +Status MultiOpsTxnsStressTest::TestIterate( + ThreadState* thread, const ReadOptions& read_opts, + const std::vector& /*rand_column_families*/, + const std::vector& /*rand_keys*/) { + uint32_t c = 0; + uint32_t pos = 0; + std::tie(c, pos) = ChooseExistingC(thread); + return RangeScanTxn(thread, read_opts, c); +} + +// Not intended for use. +Status MultiOpsTxnsStressTest::TestPut(ThreadState* /*thread*/, + WriteOptions& /*write_opts*/, + const ReadOptions& /*read_opts*/, + const std::vector& /*cf_ids*/, + const std::vector& /*keys*/, + char (&value)[100]) { + (void)value; + return Status::NotSupported(); +} + +// Not intended for use. +Status MultiOpsTxnsStressTest::TestDelete( + ThreadState* /*thread*/, WriteOptions& /*write_opts*/, + const std::vector& /*rand_column_families*/, + const std::vector& /*rand_keys*/) { + return Status::NotSupported(); +} + +// Not intended for use. +Status MultiOpsTxnsStressTest::TestDeleteRange( + ThreadState* /*thread*/, WriteOptions& /*write_opts*/, + const std::vector& /*rand_column_families*/, + const std::vector& /*rand_keys*/) { + return Status::NotSupported(); +} + +void MultiOpsTxnsStressTest::TestIngestExternalFile( + ThreadState* thread, const std::vector& rand_column_families, + const std::vector& /*rand_keys*/) { + // TODO (yanqin) + (void)thread; + (void)rand_column_families; +} + +void MultiOpsTxnsStressTest::TestCompactRange( + ThreadState* thread, int64_t /*rand_key*/, const Slice& /*start_key*/, + ColumnFamilyHandle* column_family) { + // TODO (yanqin). + // May use GetRangeHash() for validation before and after DB::CompactRange() + // completes. + (void)thread; + (void)column_family; +} + +Status MultiOpsTxnsStressTest::TestBackupRestore( + ThreadState* thread, const std::vector& rand_column_families, + const std::vector& /*rand_keys*/) { + // TODO (yanqin) + (void)thread; + (void)rand_column_families; + return Status::OK(); +} + +Status MultiOpsTxnsStressTest::TestCheckpoint( + ThreadState* thread, const std::vector& rand_column_families, + const std::vector& /*rand_keys*/) { + // TODO (yanqin) + (void)thread; + (void)rand_column_families; + return Status::OK(); +} + +Status MultiOpsTxnsStressTest::TestApproximateSize( + ThreadState* thread, uint64_t iteration, + const std::vector& rand_column_families, + const std::vector& /*rand_keys*/) { + // TODO (yanqin) + (void)thread; + (void)iteration; + (void)rand_column_families; + return Status::OK(); +} + +Status MultiOpsTxnsStressTest::TestCustomOperations( + ThreadState* thread, const std::vector& rand_column_families) { + (void)rand_column_families; + // Randomly choose from 0, 1, and 2. + // TODO (yanqin) allow user to configure probability of each operation. + uint32_t rand = thread->rand.Uniform(3); + Status s; + if (0 == rand) { + // Update primary key. + uint32_t old_a = 0; + uint32_t pos = 0; + std::tie(old_a, pos) = ChooseExistingA(thread); + uint32_t new_a = GenerateNextA(thread); + s = PrimaryKeyUpdateTxn(thread, old_a, pos, new_a); + } else if (1 == rand) { + // Update secondary key. + uint32_t old_c = 0; + uint32_t pos = 0; + std::tie(old_c, pos) = ChooseExistingC(thread); + uint32_t new_c = GenerateNextC(thread); + s = SecondaryKeyUpdateTxn(thread, old_c, pos, new_c); + } else if (2 == rand) { + // Update primary index value. + uint32_t a = 0; + uint32_t pos = 0; + std::tie(a, pos) = ChooseExistingA(thread); + s = UpdatePrimaryIndexValueTxn(thread, a, /*b_delta=*/1); + } else { + // Should never reach here. + assert(false); + } + + return s; +} + +void MultiOpsTxnsStressTest::RegisterAdditionalListeners() { + options_.listeners.emplace_back(new MultiOpsTxnsStressListener(this)); +} + +void MultiOpsTxnsStressTest::PrepareTxnDbOptions( + SharedState* /*shared*/, TransactionDBOptions& txn_db_opts) { + // MultiOpsTxnStressTest uses SingleDelete to delete secondary keys, thus we + // register this callback to let TxnDb know that when rolling back + // a transaction, use only SingleDelete to cancel prior Put from the same + // transaction if applicable. + txn_db_opts.rollback_deletion_type_callback = + [](TransactionDB* /*db*/, ColumnFamilyHandle* /*column_family*/, + const Slice& key) { + Slice ks = key; + uint32_t index_id = 0; + [[maybe_unused]] bool res = GetFixed32(&ks, &index_id); + assert(res); + index_id = EndianSwapValue(index_id); + assert(index_id <= Record::kSecondaryIndexId); + return index_id == Record::kSecondaryIndexId; + }; +} + +Status MultiOpsTxnsStressTest::PrimaryKeyUpdateTxn(ThreadState* thread, + uint32_t old_a, + uint32_t old_a_pos, + uint32_t new_a) { + std::string old_pk = Record::EncodePrimaryKey(old_a); + std::string new_pk = Record::EncodePrimaryKey(new_a); + std::unique_ptr txn; + WriteOptions wopts; + Status s = NewTxn(wopts, &txn); + if (!s.ok()) { + assert(!txn); + thread->stats.AddErrors(1); + return s; + } + + assert(txn); + txn->SetSnapshotOnNextOperation(/*notifier=*/nullptr); + + const Defer cleanup([new_a, &s, thread, this, &txn]() { + if (s.ok()) { + // Two gets, one for existing pk, one for locking potential new pk. + thread->stats.AddGets(/*ngets=*/2, /*nfounds=*/1); + thread->stats.AddDeletes(1); + thread->stats.AddBytesForWrites( + /*nwrites=*/2, + Record::kPrimaryIndexEntrySize + Record::kSecondaryIndexEntrySize); + thread->stats.AddSingleDeletes(1); + return; + } + if (s.IsNotFound()) { + thread->stats.AddGets(/*ngets=*/1, /*nfounds=*/0); + } else if (s.IsBusy() || s.IsIncomplete()) { + // ignore. + // Incomplete also means rollback by application. See the transaction + // implementations. + } else { + thread->stats.AddErrors(1); + } + auto& key_gen = key_gen_for_a_[thread->tid]; + key_gen->UndoAllocation(new_a); + txn->Rollback().PermitUncheckedError(); + }); + + ReadOptions ropts; + ropts.rate_limiter_priority = + FLAGS_rate_limit_user_ops ? Env::IO_USER : Env::IO_TOTAL; + std::string value; + s = txn->GetForUpdate(ropts, old_pk, &value); + if (!s.ok()) { + return s; + } + std::string empty_value; + s = txn->GetForUpdate(ropts, new_pk, &empty_value); + if (s.ok()) { + assert(!empty_value.empty()); + s = Status::Busy(); + return s; + } else if (!s.IsNotFound()) { + return s; + } + + auto result = Record::DecodePrimaryIndexValue(value); + s = std::get<0>(result); + if (!s.ok()) { + return s; + } + uint32_t b = std::get<1>(result); + uint32_t c = std::get<2>(result); + + ColumnFamilyHandle* cf = db_->DefaultColumnFamily(); + s = txn->Delete(cf, old_pk, /*assume_tracked=*/true); + if (!s.ok()) { + return s; + } + s = txn->Put(cf, new_pk, value, /*assume_tracked=*/true); + if (!s.ok()) { + return s; + } + + auto* wb = txn->GetWriteBatch(); + assert(wb); + + std::string old_sk = Record::EncodeSecondaryKey(c, old_a); + s = wb->SingleDelete(old_sk); + if (!s.ok()) { + return s; + } + + Record record(new_a, b, c); + std::string new_sk; + std::string new_crc; + std::tie(new_sk, new_crc) = record.EncodeSecondaryIndexEntry(); + s = wb->Put(new_sk, new_crc); + if (!s.ok()) { + return s; + } + + s = txn->Prepare(); + + if (!s.ok()) { + return s; + } + + if (FLAGS_rollback_one_in > 0 && thread->rand.OneIn(FLAGS_rollback_one_in)) { + s = Status::Incomplete(); + return s; + } + + s = WriteToCommitTimeWriteBatch(*txn); + if (!s.ok()) { + return s; + } + + s = CommitAndCreateTimestampedSnapshotIfNeeded(thread, *txn); + + auto& key_gen = key_gen_for_a_.at(thread->tid); + if (s.ok()) { + key_gen->Replace(old_a, old_a_pos, new_a); + } + return s; +} + +Status MultiOpsTxnsStressTest::SecondaryKeyUpdateTxn(ThreadState* thread, + uint32_t old_c, + uint32_t old_c_pos, + uint32_t new_c) { + std::unique_ptr txn; + WriteOptions wopts; + Status s = NewTxn(wopts, &txn); + if (!s.ok()) { + assert(!txn); + thread->stats.AddErrors(1); + return s; + } + + assert(txn); + + Iterator* it = nullptr; + long iterations = 0; + const Defer cleanup([new_c, &s, thread, &txn, &it, this, &iterations]() { + delete it; + if (s.ok()) { + thread->stats.AddIterations(iterations); + thread->stats.AddGets(/*ngets=*/1, /*nfounds=*/1); + thread->stats.AddSingleDeletes(1); + thread->stats.AddBytesForWrites( + /*nwrites=*/2, + Record::kPrimaryIndexEntrySize + Record::kSecondaryIndexEntrySize); + return; + } else if (s.IsBusy() || s.IsTimedOut() || s.IsTryAgain() || + s.IsMergeInProgress() || s.IsIncomplete()) { + // ww-conflict detected, or + // lock cannot be acquired, or + // memtable history is not large enough for conflict checking, or + // Merge operation cannot be resolved, or + // application rollback. + // TODO (yanqin) add stats for other cases? + } else if (s.IsNotFound()) { + // ignore. + } else { + thread->stats.AddErrors(1); + } + auto& key_gen = key_gen_for_c_[thread->tid]; + key_gen->UndoAllocation(new_c); + txn->Rollback().PermitUncheckedError(); + }); + + // TODO (yanqin) try SetSnapshotOnNextOperation(). We currently need to take + // a snapshot here because we will later verify that point lookup in the + // primary index using GetForUpdate() returns the same value for 'c' as the + // iterator. The iterator does not need a snapshot though, because it will be + // assigned the current latest (published) sequence in the db, which will be + // no smaller than the snapshot created here. The GetForUpdate will perform + // ww conflict checking to ensure GetForUpdate() (using the snapshot) sees + // the same data as this iterator. + txn->SetSnapshot(); + std::string old_sk_prefix = Record::EncodeSecondaryKey(old_c); + std::string iter_ub_str = Record::EncodeSecondaryKey(old_c + 1); + Slice iter_ub = iter_ub_str; + ReadOptions ropts; + ropts.snapshot = txn->GetSnapshot(); + ropts.total_order_seek = true; + ropts.iterate_upper_bound = &iter_ub; + ropts.rate_limiter_priority = + FLAGS_rate_limit_user_ops ? Env::IO_USER : Env::IO_TOTAL; + it = txn->GetIterator(ropts); + + assert(it); + it->Seek(old_sk_prefix); + if (!it->Valid()) { + s = Status::NotFound(); + return s; + } + auto* wb = txn->GetWriteBatch(); + assert(wb); + + do { + ++iterations; + Record record; + s = record.DecodeSecondaryIndexEntry(it->key(), it->value()); + if (!s.ok()) { + fprintf(stderr, "Cannot decode secondary key (%s => %s): %s\n", + it->key().ToString(true).c_str(), + it->value().ToString(true).c_str(), s.ToString().c_str()); + assert(false); + break; + } + // At this point, record.b is not known yet, thus we need to access + // primary index. + std::string pk = Record::EncodePrimaryKey(record.a_value()); + std::string value; + ReadOptions read_opts; + read_opts.rate_limiter_priority = + FLAGS_rate_limit_user_ops ? Env::IO_USER : Env::IO_TOTAL; + read_opts.snapshot = txn->GetSnapshot(); + s = txn->GetForUpdate(read_opts, pk, &value); + if (s.IsBusy() || s.IsTimedOut() || s.IsTryAgain() || + s.IsMergeInProgress()) { + // Write conflict, or cannot acquire lock, or memtable size is not large + // enough, or merge cannot be resolved. + break; + } else if (s.IsNotFound()) { + // We can also fail verification here. + std::ostringstream oss; + auto* dbimpl = static_cast_with_check(db_->GetRootDB()); + assert(dbimpl); + oss << "snap " << read_opts.snapshot->GetSequenceNumber() + << " (published " << dbimpl->GetLastPublishedSequence() + << "), pk should exist: " << Slice(pk).ToString(true); + fprintf(stderr, "%s\n", oss.str().c_str()); + assert(false); + break; + } + if (!s.ok()) { + std::ostringstream oss; + auto* dbimpl = static_cast_with_check(db_->GetRootDB()); + assert(dbimpl); + oss << "snap " << read_opts.snapshot->GetSequenceNumber() + << " (published " << dbimpl->GetLastPublishedSequence() << "), " + << s.ToString(); + fprintf(stderr, "%s\n", oss.str().c_str()); + assert(false); + break; + } + auto result = Record::DecodePrimaryIndexValue(value); + s = std::get<0>(result); + if (!s.ok()) { + fprintf(stderr, "Cannot decode primary index value %s: %s\n", + Slice(value).ToString(true).c_str(), s.ToString().c_str()); + assert(false); + break; + } + uint32_t b = std::get<1>(result); + uint32_t c = std::get<2>(result); + if (c != old_c) { + std::ostringstream oss; + auto* dbimpl = static_cast_with_check(db_->GetRootDB()); + assert(dbimpl); + oss << "snap " << read_opts.snapshot->GetSequenceNumber() + << " (published " << dbimpl->GetLastPublishedSequence() + << "), pk/sk mismatch. pk: (a=" << record.a_value() << ", " + << "c=" << c << "), sk: (c=" << old_c << ")"; + s = Status::Corruption(); + fprintf(stderr, "%s\n", oss.str().c_str()); + assert(false); + break; + } + Record new_rec(record.a_value(), b, new_c); + std::string new_primary_index_value = new_rec.EncodePrimaryIndexValue(); + ColumnFamilyHandle* cf = db_->DefaultColumnFamily(); + s = txn->Put(cf, pk, new_primary_index_value, /*assume_tracked=*/true); + if (!s.ok()) { + break; + } + std::string old_sk = it->key().ToString(/*hex=*/false); + std::string new_sk; + std::string new_crc; + std::tie(new_sk, new_crc) = new_rec.EncodeSecondaryIndexEntry(); + s = wb->SingleDelete(old_sk); + if (!s.ok()) { + break; + } + s = wb->Put(new_sk, new_crc); + if (!s.ok()) { + break; + } + + it->Next(); + } while (it->Valid()); + + if (!s.ok()) { + return s; + } + + s = txn->Prepare(); + + if (!s.ok()) { + return s; + } + + if (FLAGS_rollback_one_in > 0 && thread->rand.OneIn(FLAGS_rollback_one_in)) { + s = Status::Incomplete(); + return s; + } + + s = WriteToCommitTimeWriteBatch(*txn); + if (!s.ok()) { + return s; + } + + s = CommitAndCreateTimestampedSnapshotIfNeeded(thread, *txn); + + if (s.ok()) { + auto& key_gen = key_gen_for_c_.at(thread->tid); + key_gen->Replace(old_c, old_c_pos, new_c); + } + + return s; +} + +Status MultiOpsTxnsStressTest::UpdatePrimaryIndexValueTxn(ThreadState* thread, + uint32_t a, + uint32_t b_delta) { + std::string pk_str = Record::EncodePrimaryKey(a); + std::unique_ptr txn; + WriteOptions wopts; + Status s = NewTxn(wopts, &txn); + if (!s.ok()) { + assert(!txn); + thread->stats.AddErrors(1); + return s; + } + + assert(txn); + + const Defer cleanup([&s, thread, &txn]() { + if (s.ok()) { + thread->stats.AddGets(/*ngets=*/1, /*nfounds=*/1); + thread->stats.AddBytesForWrites( + /*nwrites=*/1, /*nbytes=*/Record::kPrimaryIndexEntrySize); + return; + } + if (s.IsNotFound()) { + thread->stats.AddGets(/*ngets=*/1, /*nfounds=*/0); + } else if (s.IsInvalidArgument()) { + // ignored. + } else if (s.IsBusy() || s.IsTimedOut() || s.IsTryAgain() || + s.IsMergeInProgress() || s.IsIncomplete()) { + // ignored. + } else { + thread->stats.AddErrors(1); + } + txn->Rollback().PermitUncheckedError(); + }); + ReadOptions ropts; + ropts.rate_limiter_priority = + FLAGS_rate_limit_user_ops ? Env::IO_USER : Env::IO_TOTAL; + std::string value; + s = txn->GetForUpdate(ropts, pk_str, &value); + if (!s.ok()) { + return s; + } + auto result = Record::DecodePrimaryIndexValue(value); + if (!std::get<0>(result).ok()) { + s = std::get<0>(result); + fprintf(stderr, "Cannot decode primary index value %s: %s\n", + Slice(value).ToString(true).c_str(), s.ToString().c_str()); + assert(false); + return s; + } + uint32_t b = std::get<1>(result) + b_delta; + uint32_t c = std::get<2>(result); + Record record(a, b, c); + std::string primary_index_value = record.EncodePrimaryIndexValue(); + ColumnFamilyHandle* cf = db_->DefaultColumnFamily(); + s = txn->Put(cf, pk_str, primary_index_value, /*assume_tracked=*/true); + if (!s.ok()) { + return s; + } + s = txn->Prepare(); + if (!s.ok()) { + return s; + } + + if (FLAGS_rollback_one_in > 0 && thread->rand.OneIn(FLAGS_rollback_one_in)) { + s = Status::Incomplete(); + return s; + } + + s = WriteToCommitTimeWriteBatch(*txn); + if (!s.ok()) { + return s; + } + + s = CommitAndCreateTimestampedSnapshotIfNeeded(thread, *txn); + + return s; +} + +Status MultiOpsTxnsStressTest::PointLookupTxn(ThreadState* thread, + ReadOptions ropts, uint32_t a) { + std::string pk_str = Record::EncodePrimaryKey(a); + // pk may or may not exist + PinnableSlice value; + + std::unique_ptr txn; + WriteOptions wopts; + Status s = NewTxn(wopts, &txn); + if (!s.ok()) { + assert(!txn); + thread->stats.AddErrors(1); + return s; + } + + assert(txn); + + const Defer cleanup([&s, thread, &txn]() { + if (s.ok()) { + thread->stats.AddGets(/*ngets=*/1, /*nfounds=*/1); + return; + } else if (s.IsNotFound()) { + thread->stats.AddGets(/*ngets=*/1, /*nfounds=*/0); + } else { + thread->stats.AddErrors(1); + } + txn->Rollback().PermitUncheckedError(); + }); + + std::shared_ptr snapshot; + SetupSnapshot(thread, ropts, *txn, snapshot); + + if (FLAGS_delay_snapshot_read_one_in > 0 && + thread->rand.OneIn(FLAGS_delay_snapshot_read_one_in)) { + uint64_t delay_ms = thread->rand.Uniform(100) + 1; + db_->GetDBOptions().env->SleepForMicroseconds( + static_cast(delay_ms * 1000)); + } + + s = txn->Get(ropts, db_->DefaultColumnFamily(), pk_str, &value); + if (s.ok()) { + s = txn->Commit(); + } + return s; +} + +Status MultiOpsTxnsStressTest::RangeScanTxn(ThreadState* thread, + ReadOptions ropts, uint32_t c) { + std::string sk = Record::EncodeSecondaryKey(c); + + std::unique_ptr txn; + WriteOptions wopts; + Status s = NewTxn(wopts, &txn); + if (!s.ok()) { + assert(!txn); + thread->stats.AddErrors(1); + return s; + } + + assert(txn); + + const Defer cleanup([&s, thread, &txn]() { + if (s.ok()) { + thread->stats.AddIterations(1); + return; + } + thread->stats.AddErrors(1); + txn->Rollback().PermitUncheckedError(); + }); + + std::shared_ptr snapshot; + SetupSnapshot(thread, ropts, *txn, snapshot); + + if (FLAGS_delay_snapshot_read_one_in > 0 && + thread->rand.OneIn(FLAGS_delay_snapshot_read_one_in)) { + uint64_t delay_ms = thread->rand.Uniform(100) + 1; + db_->GetDBOptions().env->SleepForMicroseconds( + static_cast(delay_ms * 1000)); + } + + std::unique_ptr iter(txn->GetIterator(ropts)); + + constexpr size_t total_nexts = 10; + size_t nexts = 0; + for (iter->Seek(sk); + iter->Valid() && nexts < total_nexts && iter->status().ok(); + iter->Next(), ++nexts) { + } + + if (iter->status().ok()) { + s = txn->Commit(); + } else { + s = iter->status(); + } + + return s; +} + +void MultiOpsTxnsStressTest::VerifyDb(ThreadState* thread) const { + if (thread->shared->HasVerificationFailedYet()) { + return; + } + const Snapshot* const snapshot = db_->GetSnapshot(); + assert(snapshot); + ManagedSnapshot snapshot_guard(db_, snapshot); + + std::ostringstream oss; + oss << "[snap=" << snapshot->GetSequenceNumber() << ","; + + auto* dbimpl = static_cast_with_check(db_->GetRootDB()); + assert(dbimpl); + + oss << " last_published=" << dbimpl->GetLastPublishedSequence() << "] "; + + if (FLAGS_delay_snapshot_read_one_in > 0 && + thread->rand.OneIn(FLAGS_delay_snapshot_read_one_in)) { + uint64_t delay_ms = thread->rand.Uniform(100) + 1; + db_->GetDBOptions().env->SleepForMicroseconds( + static_cast(delay_ms * 1000)); + } + + // TODO (yanqin) with a probability, we can use either forward or backward + // iterator in subsequent checks. We can also use more advanced features in + // range scan. For now, let's just use simple forward iteration with + // total_order_seek = true. + + // First, iterate primary index. + size_t primary_index_entries_count = 0; + { + std::string iter_ub_str; + PutFixed32(&iter_ub_str, Record::kPrimaryIndexId + 1); + std::reverse(iter_ub_str.begin(), iter_ub_str.end()); + Slice iter_ub = iter_ub_str; + + std::string start_key; + PutFixed32(&start_key, Record::kPrimaryIndexId); + std::reverse(start_key.begin(), start_key.end()); + + // This `ReadOptions` is for validation purposes. Ignore + // `FLAGS_rate_limit_user_ops` to avoid slowing any validation. + ReadOptions ropts; + ropts.snapshot = snapshot; + ropts.total_order_seek = true; + ropts.iterate_upper_bound = &iter_ub; + + std::unique_ptr it(db_->NewIterator(ropts)); + for (it->Seek(start_key); it->Valid(); it->Next()) { + Record record; + Status s = record.DecodePrimaryIndexEntry(it->key(), it->value()); + if (!s.ok()) { + oss << "Cannot decode primary index entry " << it->key().ToString(true) + << "=>" << it->value().ToString(true); + VerificationAbort(thread->shared, oss.str(), s); + assert(false); + return; + } + ++primary_index_entries_count; + + // Search secondary index. + uint32_t a = record.a_value(); + uint32_t c = record.c_value(); + char sk_buf[12]; + EncodeFixed32(sk_buf, Record::kSecondaryIndexId); + std::reverse(sk_buf, sk_buf + sizeof(uint32_t)); + EncodeFixed32(sk_buf + sizeof(uint32_t), c); + std::reverse(sk_buf + sizeof(uint32_t), sk_buf + 2 * sizeof(uint32_t)); + EncodeFixed32(sk_buf + 2 * sizeof(uint32_t), a); + std::reverse(sk_buf + 2 * sizeof(uint32_t), sk_buf + sizeof(sk_buf)); + Slice sk(sk_buf, sizeof(sk_buf)); + std::string value; + s = db_->Get(ropts, sk, &value); + if (!s.ok()) { + oss << "Cannot find secondary index entry " << sk.ToString(true); + VerificationAbort(thread->shared, oss.str(), s); + assert(false); + return; + } + } + } + + // Second, iterate secondary index. + size_t secondary_index_entries_count = 0; + { + std::string start_key; + PutFixed32(&start_key, Record::kSecondaryIndexId); + std::reverse(start_key.begin(), start_key.end()); + + // This `ReadOptions` is for validation purposes. Ignore + // `FLAGS_rate_limit_user_ops` to avoid slowing any validation. + ReadOptions ropts; + ropts.snapshot = snapshot; + ropts.total_order_seek = true; + + std::unique_ptr it(db_->NewIterator(ropts)); + for (it->Seek(start_key); it->Valid(); it->Next()) { + ++secondary_index_entries_count; + Record record; + Status s = record.DecodeSecondaryIndexEntry(it->key(), it->value()); + if (!s.ok()) { + oss << "Cannot decode secondary index entry " + << it->key().ToString(true) << "=>" << it->value().ToString(true); + VerificationAbort(thread->shared, oss.str(), s); + assert(false); + return; + } + // After decoding secondary index entry, we know a and c. Crc is verified + // in decoding phase. + // + // Form a primary key and search in the primary index. + std::string pk = Record::EncodePrimaryKey(record.a_value()); + std::string value; + s = db_->Get(ropts, pk, &value); + if (!s.ok()) { + oss << "Error searching pk " << Slice(pk).ToString(true) << ". " + << s.ToString() << ". sk " << it->key().ToString(true); + VerificationAbort(thread->shared, oss.str(), s); + assert(false); + return; + } + auto result = Record::DecodePrimaryIndexValue(value); + s = std::get<0>(result); + if (!s.ok()) { + oss << "Error decoding primary index value " + << Slice(value).ToString(true) << ". " << s.ToString(); + VerificationAbort(thread->shared, oss.str(), s); + assert(false); + return; + } + uint32_t c_in_primary = std::get<2>(result); + if (c_in_primary != record.c_value()) { + oss << "Pk/sk mismatch. pk: " << Slice(pk).ToString(true) << "=>" + << Slice(value).ToString(true) << " (a=" << record.a_value() + << ", c=" << c_in_primary << "), sk: " << it->key().ToString(true) + << " (c=" << record.c_value() << ")"; + VerificationAbort(thread->shared, oss.str(), s); + assert(false); + return; + } + } + } + + if (secondary_index_entries_count != primary_index_entries_count) { + oss << "Pk/sk mismatch: primary index has " << primary_index_entries_count + << " entries. Secondary index has " << secondary_index_entries_count + << " entries."; + VerificationAbort(thread->shared, oss.str(), Status::OK()); + assert(false); + return; + } +} + +// VerifyPkSkFast() can be called by MultiOpsTxnsStressListener's callbacks +// which can be called before TransactionDB::Open() returns to caller. +// Therefore, at that time, db_ and txn_db_ may still be nullptr. +// Caller has to make sure that the race condition does not happen. +void MultiOpsTxnsStressTest::VerifyPkSkFast(const ReadOptions& read_options, + int job_id) { + DB* const db = db_aptr_.load(std::memory_order_acquire); + if (db == nullptr) { + return; + } + + assert(db_ == db); + assert(db_ != nullptr); + + const Snapshot* const snapshot = db_->GetSnapshot(); + assert(snapshot); + ManagedSnapshot snapshot_guard(db_, snapshot); + + std::ostringstream oss; + auto* dbimpl = static_cast_with_check(db_->GetRootDB()); + assert(dbimpl); + + oss << "Job " << job_id << ": [" << snapshot->GetSequenceNumber() << "," + << dbimpl->GetLastPublishedSequence() << "] "; + + std::string start_key; + PutFixed32(&start_key, Record::kSecondaryIndexId); + std::reverse(start_key.begin(), start_key.end()); + + // This `ReadOptions` is for validation purposes. Ignore + // `FLAGS_rate_limit_user_ops` to avoid slowing any validation. + ReadOptions ropts; + ropts.snapshot = snapshot; + ropts.total_order_seek = true; + ropts.io_activity = read_options.io_activity; + + std::unique_ptr it(db_->NewIterator(ropts)); + for (it->Seek(start_key); it->Valid(); it->Next()) { + Record record; + Status s = record.DecodeSecondaryIndexEntry(it->key(), it->value()); + if (!s.ok()) { + oss << "Cannot decode secondary index entry " << it->key().ToString(true) + << "=>" << it->value().ToString(true); + fprintf(stderr, "%s\n", oss.str().c_str()); + fflush(stderr); + assert(false); + } + // After decoding secondary index entry, we know a and c. Crc is verified + // in decoding phase. + // + // Form a primary key and search in the primary index. + std::string pk = Record::EncodePrimaryKey(record.a_value()); + std::string value; + s = db_->Get(ropts, pk, &value); + if (!s.ok()) { + oss << "Error searching pk " << Slice(pk).ToString(true) << ". " + << s.ToString() << ". sk " << it->key().ToString(true); + fprintf(stderr, "%s\n", oss.str().c_str()); + fflush(stderr); + assert(false); + } + auto result = Record::DecodePrimaryIndexValue(value); + s = std::get<0>(result); + if (!s.ok()) { + oss << "Error decoding primary index value " + << Slice(value).ToString(true) << ". " << s.ToString(); + fprintf(stderr, "%s\n", oss.str().c_str()); + fflush(stderr); + assert(false); + } + uint32_t c_in_primary = std::get<2>(result); + if (c_in_primary != record.c_value()) { + oss << "Pk/sk mismatch. pk: " << Slice(pk).ToString(true) << "=>" + << Slice(value).ToString(true) << " (a=" << record.a_value() + << ", c=" << c_in_primary << "), sk: " << it->key().ToString(true) + << " (c=" << record.c_value() << ")"; + fprintf(stderr, "%s\n", oss.str().c_str()); + fflush(stderr); + assert(false); + } + } +} + +std::pair MultiOpsTxnsStressTest::ChooseExistingA( + ThreadState* thread) { + uint32_t tid = thread->tid; + auto& key_gen = key_gen_for_a_.at(tid); + return key_gen->ChooseExisting(); +} + +uint32_t MultiOpsTxnsStressTest::GenerateNextA(ThreadState* thread) { + uint32_t tid = thread->tid; + auto& key_gen = key_gen_for_a_.at(tid); + return key_gen->Allocate(); +} + +std::pair MultiOpsTxnsStressTest::ChooseExistingC( + ThreadState* thread) { + uint32_t tid = thread->tid; + auto& key_gen = key_gen_for_c_.at(tid); + return key_gen->ChooseExisting(); +} + +uint32_t MultiOpsTxnsStressTest::GenerateNextC(ThreadState* thread) { + uint32_t tid = thread->tid; + auto& key_gen = key_gen_for_c_.at(tid); + return key_gen->Allocate(); +} + +void MultiOpsTxnsStressTest::ProcessRecoveredPreparedTxnsHelper( + Transaction* txn, SharedState*) { + thread_local Random rand(static_cast(FLAGS_seed)); + if (rand.OneIn(2)) { + Status s = txn->Commit(); + assert(s.ok()); + } else { + Status s = txn->Rollback(); + assert(s.ok()); + } +} + +Status MultiOpsTxnsStressTest::WriteToCommitTimeWriteBatch(Transaction& txn) { + WriteBatch* ctwb = txn.GetCommitTimeWriteBatch(); + assert(ctwb); + // Do not change the content in key_buf. + static constexpr char key_buf[sizeof(Record::kMetadataPrefix) + 4] = { + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\xff'}; + + uint64_t counter_val = counter_.Next(); + char val_buf[sizeof(counter_val)]; + EncodeFixed64(val_buf, counter_val); + return ctwb->Put(Slice(key_buf, sizeof(key_buf)), + Slice(val_buf, sizeof(val_buf))); +} + +Status MultiOpsTxnsStressTest::CommitAndCreateTimestampedSnapshotIfNeeded( + ThreadState* thread, Transaction& txn) { + Status s; + if (FLAGS_create_timestamped_snapshot_one_in > 0 && + thread->rand.OneInOpt(FLAGS_create_timestamped_snapshot_one_in)) { + uint64_t ts = db_stress_env->NowNanos(); + std::shared_ptr snapshot; + s = txn.CommitAndTryCreateSnapshot(/*notifier=*/nullptr, ts, &snapshot); + } else { + s = txn.Commit(); + } + assert(txn_db_); + if (FLAGS_create_timestamped_snapshot_one_in > 0 && + thread->rand.OneInOpt(50000)) { + uint64_t now = db_stress_env->NowNanos(); + constexpr uint64_t time_diff = static_cast(1000) * 1000 * 1000; + txn_db_->ReleaseTimestampedSnapshotsOlderThan(now - time_diff); + } + return s; +} + +void MultiOpsTxnsStressTest::SetupSnapshot( + ThreadState* thread, ReadOptions& read_opts, Transaction& txn, + std::shared_ptr& snapshot) { + if (thread->rand.OneInOpt(2)) { + snapshot = txn_db_->GetLatestTimestampedSnapshot(); + } + + if (snapshot) { + read_opts.snapshot = snapshot.get(); + } else { + txn.SetSnapshot(); + read_opts.snapshot = txn.GetSnapshot(); + } +} + +std::string MultiOpsTxnsStressTest::KeySpaces::EncodeTo() const { + std::string result; + PutFixed32(&result, lb_a); + PutFixed32(&result, ub_a); + PutFixed32(&result, lb_c); + PutFixed32(&result, ub_c); + return result; +} + +bool MultiOpsTxnsStressTest::KeySpaces::DecodeFrom(Slice data) { + if (!GetFixed32(&data, &lb_a) || !GetFixed32(&data, &ub_a) || + !GetFixed32(&data, &lb_c) || !GetFixed32(&data, &ub_c)) { + return false; + } + return true; +} + +void MultiOpsTxnsStressTest::PersistKeySpacesDesc( + const std::string& key_spaces_path, uint32_t lb_a, uint32_t ub_a, + uint32_t lb_c, uint32_t ub_c) { + KeySpaces key_spaces(lb_a, ub_a, lb_c, ub_c); + std::string key_spaces_rep = key_spaces.EncodeTo(); + + std::unique_ptr wfile; + Status s1 = + Env::Default()->NewWritableFile(key_spaces_path, &wfile, EnvOptions()); + assert(s1.ok()); + assert(wfile); + s1 = wfile->Append(key_spaces_rep); + assert(s1.ok()); +} + +MultiOpsTxnsStressTest::KeySpaces MultiOpsTxnsStressTest::ReadKeySpacesDesc( + const std::string& key_spaces_path) { + KeySpaces key_spaces; + std::unique_ptr sfile; + Status s1 = + Env::Default()->NewSequentialFile(key_spaces_path, &sfile, EnvOptions()); + assert(s1.ok()); + assert(sfile); + char buf[16]; + Slice result; + s1 = sfile->Read(sizeof(buf), &result, buf); + assert(s1.ok()); + if (!key_spaces.DecodeFrom(result)) { + assert(false); + } + return key_spaces; +} + +// Create an empty database if necessary and preload it with initial test data. +// Key range [lb_a, ub_a), [lb_c, ub_c). The key ranges will be shared by +// 'threads' threads. +// PreloadDb() also sets up KeyGenerator objects for each sub key range +// operated on by each thread. +// Both [lb_a, ub_a) and [lb_c, ub_c) are partitioned. Each thread operates on +// one sub range, using KeyGenerators to generate keys. +// For example, we choose a from [0, 10000) and c from [0, 100). Number of +// threads is 32, their tids range from 0 to 31. +// Thread k chooses a from [312*k,312*(k+1)) and c from [3*k,3*(k+1)) if k<31. +// Thread 31 chooses a from [9672, 10000) and c from [93, 100). +// Within each subrange: a from [low1, high1), c from [low2, high2). +// high1 - low1 > high2 - low2 +// We reserve {high1 - 1} and {high2 - 1} as unallocated. +// The records are , , ..., +// , ... +void MultiOpsTxnsStressTest::PreloadDb(SharedState* shared, int threads, + uint32_t lb_a, uint32_t ub_a, + uint32_t lb_c, uint32_t ub_c) { + key_gen_for_a_.resize(threads); + key_gen_for_c_.resize(threads); + + assert(ub_a > lb_a && ub_a > lb_a + threads); + assert(ub_c > lb_c && ub_c > lb_c + threads); + + PersistKeySpacesDesc(FLAGS_key_spaces_path, lb_a, ub_a, lb_c, ub_c); + + fprintf(stdout, "a from [%u, %u), c from [%u, %u)\n", + static_cast(lb_a), static_cast(ub_a), + static_cast(lb_c), static_cast(ub_c)); + + const uint32_t num_c = ub_c - lb_c; + const uint32_t num_c_per_thread = num_c / threads; + const uint32_t num_a = ub_a - lb_a; + const uint32_t num_a_per_thread = num_a / threads; + + WriteOptions wopts; + wopts.disableWAL = FLAGS_disable_wal; + Random rnd(shared->GetSeed()); + assert(txn_db_); + + std::vector existing_a_uniqs(threads); + std::vector non_existing_a_uniqs(threads); + std::vector existing_c_uniqs(threads); + std::vector non_existing_c_uniqs(threads); + + for (uint32_t a = lb_a; a < ub_a; ++a) { + uint32_t tid = (a - lb_a) / num_a_per_thread; + if (tid >= static_cast(threads)) { + tid = threads - 1; + } + + uint32_t a_base = lb_a + tid * num_a_per_thread; + uint32_t a_hi = (tid < static_cast(threads - 1)) + ? (a_base + num_a_per_thread) + : ub_a; + uint32_t a_delta = a - a_base; + + if (a == a_hi - 1) { + non_existing_a_uniqs[tid].insert(a); + continue; + } + + uint32_t c_base = lb_c + tid * num_c_per_thread; + uint32_t c_hi = (tid < static_cast(threads - 1)) + ? (c_base + num_c_per_thread) + : ub_c; + uint32_t c_delta = a_delta % (c_hi - c_base - 1); + uint32_t c = c_base + c_delta; + + uint32_t b = rnd.Next(); + Record record(a, b, c); + WriteBatch wb; + const auto primary_index_entry = record.EncodePrimaryIndexEntry(); + Status s = wb.Put(primary_index_entry.first, primary_index_entry.second); + assert(s.ok()); + + const auto secondary_index_entry = record.EncodeSecondaryIndexEntry(); + s = wb.Put(secondary_index_entry.first, secondary_index_entry.second); + assert(s.ok()); + + s = txn_db_->Write(wopts, &wb); + assert(s.ok()); + + // TODO (yanqin): make the following check optional, especially when data + // size is large. + Record tmp_rec; + tmp_rec.SetB(record.b_value()); + s = tmp_rec.DecodeSecondaryIndexEntry(secondary_index_entry.first, + secondary_index_entry.second); + assert(s.ok()); + assert(tmp_rec == record); + + existing_a_uniqs[tid].insert(a); + existing_c_uniqs[tid].insert(c); + } + + for (int i = 0; i < threads; ++i) { + uint32_t my_seed = i + shared->GetSeed(); + + auto& key_gen_for_a = key_gen_for_a_[i]; + assert(!key_gen_for_a); + uint32_t low = lb_a + i * num_a_per_thread; + uint32_t high = (i < threads - 1) ? (low + num_a_per_thread) : ub_a; + assert(existing_a_uniqs[i].size() == high - low - 1); + assert(non_existing_a_uniqs[i].size() == 1); + key_gen_for_a = std::make_unique( + my_seed, low, high, std::move(existing_a_uniqs[i]), + std::move(non_existing_a_uniqs[i])); + + auto& key_gen_for_c = key_gen_for_c_[i]; + assert(!key_gen_for_c); + low = lb_c + i * num_c_per_thread; + high = (i < threads - 1) ? (low + num_c_per_thread) : ub_c; + non_existing_c_uniqs[i].insert(high - 1); + assert(existing_c_uniqs[i].size() == high - low - 1); + assert(non_existing_c_uniqs[i].size() == 1); + key_gen_for_c = std::make_unique( + my_seed, low, high, std::move(existing_c_uniqs[i]), + std::move(non_existing_c_uniqs[i])); + } +} + +// Scan an existing, non-empty database. +// Set up [lb_a, ub_a) and [lb_c, ub_c) as test key ranges. +// Set up KeyGenerator objects for each sub key range operated on by each +// thread. +// Scan the entire database and for each subrange, populate the existing keys +// and non-existing keys. We currently require the non-existing keys be +// non-empty after initialization. +void MultiOpsTxnsStressTest::ScanExistingDb(SharedState* shared, int threads) { + key_gen_for_a_.resize(threads); + key_gen_for_c_.resize(threads); + + KeySpaces key_spaces = ReadKeySpacesDesc(FLAGS_key_spaces_path); + + const uint32_t lb_a = key_spaces.lb_a; + const uint32_t ub_a = key_spaces.ub_a; + const uint32_t lb_c = key_spaces.lb_c; + const uint32_t ub_c = key_spaces.ub_c; + + assert(lb_a < ub_a && lb_c < ub_c); + + fprintf(stdout, "a from [%u, %u), c from [%u, %u)\n", + static_cast(lb_a), static_cast(ub_a), + static_cast(lb_c), static_cast(ub_c)); + + assert(ub_a > lb_a && ub_a > lb_a + threads); + assert(ub_c > lb_c && ub_c > lb_c + threads); + + const uint32_t num_c = ub_c - lb_c; + const uint32_t num_c_per_thread = num_c / threads; + const uint32_t num_a = ub_a - lb_a; + const uint32_t num_a_per_thread = num_a / threads; + + assert(db_); + ReadOptions ropts; + std::vector existing_a_uniqs(threads); + std::vector non_existing_a_uniqs(threads); + std::vector existing_c_uniqs(threads); + std::vector non_existing_c_uniqs(threads); + { + std::string pk_lb_str = Record::EncodePrimaryKey(0); + std::string pk_ub_str = + Record::EncodePrimaryKey(std::numeric_limits::max()); + Slice pk_lb = pk_lb_str; + Slice pk_ub = pk_ub_str; + ropts.iterate_lower_bound = &pk_lb; + ropts.iterate_upper_bound = &pk_ub; + ropts.total_order_seek = true; + std::unique_ptr it(db_->NewIterator(ropts)); + + for (it->SeekToFirst(); it->Valid(); it->Next()) { + Record record; + Status s = record.DecodePrimaryIndexEntry(it->key(), it->value()); + if (!s.ok()) { + fprintf(stderr, "Cannot decode primary index entry (%s => %s): %s\n", + it->key().ToString(true).c_str(), + it->value().ToString(true).c_str(), s.ToString().c_str()); + assert(false); + } + uint32_t a = record.a_value(); + assert(a >= lb_a); + assert(a < ub_a); + uint32_t tid = (a - lb_a) / num_a_per_thread; + if (tid >= static_cast(threads)) { + tid = threads - 1; + } + + existing_a_uniqs[tid].insert(a); + + uint32_t c = record.c_value(); + assert(c >= lb_c); + assert(c < ub_c); + tid = (c - lb_c) / num_c_per_thread; + if (tid >= static_cast(threads)) { + tid = threads - 1; + } + auto& existing_c_uniq = existing_c_uniqs[tid]; + existing_c_uniq.insert(c); + } + + for (uint32_t a = lb_a; a < ub_a; ++a) { + uint32_t tid = (a - lb_a) / num_a_per_thread; + if (tid >= static_cast(threads)) { + tid = threads - 1; + } + if (0 == existing_a_uniqs[tid].count(a)) { + non_existing_a_uniqs[tid].insert(a); + } + } + + for (uint32_t c = lb_c; c < ub_c; ++c) { + uint32_t tid = (c - lb_c) / num_c_per_thread; + if (tid >= static_cast(threads)) { + tid = threads - 1; + } + if (0 == existing_c_uniqs[tid].count(c)) { + non_existing_c_uniqs[tid].insert(c); + } + } + + for (int i = 0; i < threads; ++i) { + uint32_t my_seed = i + shared->GetSeed(); + auto& key_gen_for_a = key_gen_for_a_[i]; + assert(!key_gen_for_a); + uint32_t low = lb_a + i * num_a_per_thread; + uint32_t high = (i < threads - 1) ? (low + num_a_per_thread) : ub_a; + + // The following two assertions assume the test thread count and key + // space remain the same across different runs. Will need to relax. + assert(existing_a_uniqs[i].size() == high - low - 1); + assert(non_existing_a_uniqs[i].size() == 1); + + key_gen_for_a = std::make_unique( + my_seed, low, high, std::move(existing_a_uniqs[i]), + std::move(non_existing_a_uniqs[i])); + + auto& key_gen_for_c = key_gen_for_c_[i]; + assert(!key_gen_for_c); + low = lb_c + i * num_c_per_thread; + high = (i < threads - 1) ? (low + num_c_per_thread) : ub_c; + + // The following two assertions assume the test thread count and key + // space remain the same across different runs. Will need to relax. + assert(existing_c_uniqs[i].size() == high - low - 1); + assert(non_existing_c_uniqs[i].size() == 1); + + key_gen_for_c = std::make_unique( + my_seed, low, high, std::move(existing_c_uniqs[i]), + std::move(non_existing_c_uniqs[i])); + } + } +} + +StressTest* CreateMultiOpsTxnsStressTest() { + return new MultiOpsTxnsStressTest(); +} + +void CheckAndSetOptionsForMultiOpsTxnStressTest() { + if (FLAGS_test_batches_snapshots || FLAGS_test_cf_consistency) { + fprintf(stderr, + "-test_multi_ops_txns is not compatible with " + "-test_bathces_snapshots and -test_cf_consistency\n"); + exit(1); + } + if (!FLAGS_use_txn) { + fprintf(stderr, "-use_txn must be true if -test_multi_ops_txns\n"); + exit(1); + } else if (FLAGS_test_secondary > 0) { + fprintf( + stderr, + "secondary instance does not support replaying logs (MANIFEST + WAL) " + "of TransactionDB with write-prepared/write-unprepared policy\n"); + exit(1); + } + if (FLAGS_clear_column_family_one_in > 0) { + fprintf(stderr, + "-test_multi_ops_txns is not compatible with clearing column " + "families\n"); + exit(1); + } + if (FLAGS_column_families > 1) { + // TODO (yanqin) support separating primary index and secondary index in + // different column families. + fprintf(stderr, + "-test_multi_ops_txns currently does not use more than one column " + "family\n"); + exit(1); + } + if (FLAGS_writepercent > 0 || FLAGS_delpercent > 0 || + FLAGS_delrangepercent > 0) { + fprintf(stderr, + "-test_multi_ops_txns requires that -writepercent, -delpercent and " + "-delrangepercent be 0\n"); + exit(1); + } + if (FLAGS_key_spaces_path.empty()) { + fprintf(stderr, + "Must specify a file to store ranges of A and C via " + "-key_spaces_path\n"); + exit(1); + } + if (FLAGS_create_timestamped_snapshot_one_in > 0) { + if (FLAGS_txn_write_policy != + static_cast(TxnDBWritePolicy::WRITE_COMMITTED)) { + fprintf(stderr, + "Timestamped snapshot is not yet supported by " + "write-prepared/write-unprepared transactions\n"); + exit(1); + } + } + if (FLAGS_sync_fault_injection == 1) { + fprintf(stderr, + "Sync fault injection is currently not supported in " + "-test_multi_ops_txns\n"); + exit(1); + } +} +} // namespace ROCKSDB_NAMESPACE + +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db_stress_tool/multi_ops_txns_stress.h b/librocksdb-sys/rocksdb/db_stress_tool/multi_ops_txns_stress.h new file mode 100644 index 0000000..12c45aa --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/multi_ops_txns_stress.h @@ -0,0 +1,446 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifdef GFLAGS +#include "db_stress_tool/db_stress_common.h" + +namespace ROCKSDB_NAMESPACE { + +// This file defines MultiOpsTxnsStress so that we can stress test RocksDB +// transactions on a simple, emulated relational table. +// +// The record format is similar to the example found at +// https://github.com/facebook/mysql-5.6/wiki/MyRocks-record-format. +// +// The table is created by +// ``` +// create table t1 ( +// a int primary key, +// b int, +// c int, +// key(c), +// ) +// ``` +// +// (For simplicity, we use uint32_t for int here.) +// +// For this table, there is a primary index using `a`, as well as a secondary +// index using `c` and `a`. +// +// Primary key format: +// | index id | M(a) | +// Primary index value: +// | b | c | +// M(a) represents the big-endian format of a. +// +// Secondary key format: +// | index id | M(c) | M(a) | +// Secondary index value: +// | crc32 | +// Similarly to M(a), M(c) is the big-endian format of c. +// +// The in-memory representation of a record is defined in class +// MultiOpsTxnsStress:Record that includes a number of helper methods to +// encode/decode primary index keys, primary index values, secondary index keys, +// secondary index values, etc. +// +// Sometimes primary index and secondary index reside on different column +// families, but sometimes they colocate in the same column family. Current +// implementation puts them in the same (default) column family, and this is +// subject to future change if we find it interesting to test the other case. +// +// Class MultiOpsTxnsStressTest has the following transactions for testing. +// +// 1. Primary key update +// UPDATE t1 SET a = 3 WHERE a = 2; +// ``` +// tx->GetForUpdate(primary key a=2) +// tx->GetForUpdate(primary key a=3) +// tx->Delete(primary key a=2) +// tx->Put(primary key a=3, value) +// tx->batch->SingleDelete(secondary key a=2) +// tx->batch->Put(secondary key a=3, value) +// tx->Prepare() +// Tx->Commit() +// ``` +// +// 2. Secondary key update +// UPDATE t1 SET c = 3 WHERE c = 2; +// ``` +// iter->Seek(secondary key) +// // Get corresponding primary key value(s) from iterator +// tx->GetForUpdate(primary key) +// tx->Put(primary key, value c=3) +// tx->batch->SingleDelete(secondary key c=2) +// tx->batch->Put(secondary key c=3) +// tx->Prepare() +// tx->Commit() +// ``` +// +// 3. Primary index value update +// UPDATE t1 SET b = b + 1 WHERE a = 2; +// ``` +// tx->GetForUpdate(primary key a=2) +// tx->Put(primary key a=2, value b=b+1) +// tx->Prepare() +// tx->Commit() +// ``` +// +// 4. Point lookup +// SELECT * FROM t1 WHERE a = 3; +// ``` +// tx->Get(primary key a=3) +// tx->Commit() +// ``` +// +// 5. Range scan +// SELECT * FROM t1 WHERE c = 2; +// ``` +// it = tx->GetIterator() +// it->Seek(secondary key c=2) +// tx->Commit() +// ``` + +class MultiOpsTxnsStressTest : public StressTest { + public: + class Record { + public: + static constexpr uint32_t kMetadataPrefix = 0; + static constexpr uint32_t kPrimaryIndexId = 1; + static constexpr uint32_t kSecondaryIndexId = 2; + + static constexpr size_t kPrimaryIndexEntrySize = 8 + 8; + static constexpr size_t kSecondaryIndexEntrySize = 12 + 4; + + static_assert(kPrimaryIndexId < kSecondaryIndexId, + "kPrimaryIndexId must be smaller than kSecondaryIndexId"); + + static_assert(sizeof(kPrimaryIndexId) == sizeof(uint32_t), + "kPrimaryIndexId must be 4 bytes"); + static_assert(sizeof(kSecondaryIndexId) == sizeof(uint32_t), + "kSecondaryIndexId must be 4 bytes"); + + // Used for generating search key to probe primary index. + static std::string EncodePrimaryKey(uint32_t a); + // Used for generating search prefix to probe secondary index. + static std::string EncodeSecondaryKey(uint32_t c); + // Used for generating search key to probe secondary index. + static std::string EncodeSecondaryKey(uint32_t c, uint32_t a); + + static std::tuple DecodePrimaryIndexValue( + Slice primary_index_value); + + static std::pair DecodeSecondaryIndexValue( + Slice secondary_index_value); + + Record() = default; + Record(uint32_t _a, uint32_t _b, uint32_t _c) : a_(_a), b_(_b), c_(_c) {} + + bool operator==(const Record& other) const { + return a_ == other.a_ && b_ == other.b_ && c_ == other.c_; + } + + bool operator!=(const Record& other) const { return !(*this == other); } + + std::pair EncodePrimaryIndexEntry() const; + + std::string EncodePrimaryKey() const; + + std::string EncodePrimaryIndexValue() const; + + std::pair EncodeSecondaryIndexEntry() const; + + std::string EncodeSecondaryKey() const; + + Status DecodePrimaryIndexEntry(Slice primary_index_key, + Slice primary_index_value); + + Status DecodeSecondaryIndexEntry(Slice secondary_index_key, + Slice secondary_index_value); + + uint32_t a_value() const { return a_; } + uint32_t b_value() const { return b_; } + uint32_t c_value() const { return c_; } + + void SetA(uint32_t _a) { a_ = _a; } + void SetB(uint32_t _b) { b_ = _b; } + void SetC(uint32_t _c) { c_ = _c; } + + std::string ToString() const { + std::string ret("("); + ret.append(std::to_string(a_)); + ret.append(","); + ret.append(std::to_string(b_)); + ret.append(","); + ret.append(std::to_string(c_)); + ret.append(")"); + return ret; + } + + private: + friend class InvariantChecker; + + uint32_t a_{0}; + uint32_t b_{0}; + uint32_t c_{0}; + }; + + MultiOpsTxnsStressTest() {} + + ~MultiOpsTxnsStressTest() override {} + + void FinishInitDb(SharedState*) override; + + void ReopenAndPreloadDbIfNeeded(SharedState* shared); + + bool IsStateTracked() const override { return false; } + + Status TestGet(ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override; + + std::vector TestMultiGet( + ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override; + + void TestGetEntity(ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override; + + void TestMultiGetEntity(ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override; + + Status TestPrefixScan(ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override; + + // Given a key K, this creates an iterator which scans to K and then + // does a random sequence of Next/Prev operations. + Status TestIterate(ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override; + + Status TestPut(ThreadState* thread, WriteOptions& write_opts, + const ReadOptions& read_opts, const std::vector& cf_ids, + const std::vector& keys, char (&value)[100]) override; + + Status TestDelete(ThreadState* thread, WriteOptions& write_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override; + + Status TestDeleteRange(ThreadState* thread, WriteOptions& write_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override; + + void TestIngestExternalFile(ThreadState* thread, + const std::vector& rand_column_families, + const std::vector& rand_keys) override; + + void TestCompactRange(ThreadState* thread, int64_t rand_key, + const Slice& start_key, + ColumnFamilyHandle* column_family) override; + + Status TestBackupRestore(ThreadState* thread, + const std::vector& rand_column_families, + const std::vector& rand_keys) override; + + Status TestCheckpoint(ThreadState* thread, + const std::vector& rand_column_families, + const std::vector& rand_keys) override; + + Status TestApproximateSize(ThreadState* thread, uint64_t iteration, + const std::vector& rand_column_families, + const std::vector& rand_keys) override; + + Status TestCustomOperations( + ThreadState* thread, + const std::vector& rand_column_families) override; + + void RegisterAdditionalListeners() override; + + void PrepareTxnDbOptions(SharedState* /*shared*/, + TransactionDBOptions& txn_db_opts) override; + + Status PrimaryKeyUpdateTxn(ThreadState* thread, uint32_t old_a, + uint32_t old_a_pos, uint32_t new_a); + + Status SecondaryKeyUpdateTxn(ThreadState* thread, uint32_t old_c, + uint32_t old_c_pos, uint32_t new_c); + + Status UpdatePrimaryIndexValueTxn(ThreadState* thread, uint32_t a, + uint32_t b_delta); + + Status PointLookupTxn(ThreadState* thread, ReadOptions ropts, uint32_t a); + + Status RangeScanTxn(ThreadState* thread, ReadOptions ropts, uint32_t c); + + void VerifyDb(ThreadState* thread) const override; + + void ContinuouslyVerifyDb(ThreadState* thread) const override { + VerifyDb(thread); + } + + void VerifyPkSkFast(const ReadOptions& read_options, int job_id); + + protected: + class Counter { + public: + uint64_t Next() { return value_.fetch_add(1); } + + private: + std::atomic value_ = Env::Default()->NowNanos(); + }; + + using KeySet = std::set; + class KeyGenerator { + public: + explicit KeyGenerator(uint32_t s, uint32_t low, uint32_t high, + KeySet&& existing_uniq, KeySet&& non_existing_uniq) + : rand_(s), + low_(low), + high_(high), + existing_uniq_(std::move(existing_uniq)), + non_existing_uniq_(std::move(non_existing_uniq)) {} + ~KeyGenerator() { + assert(!existing_uniq_.empty()); + assert(!non_existing_uniq_.empty()); + } + void FinishInit(); + + std::pair ChooseExisting(); + void Replace(uint32_t old_val, uint32_t old_pos, uint32_t new_val); + uint32_t Allocate(); + void UndoAllocation(uint32_t new_val); + + std::string ToString() const { + std::ostringstream oss; + oss << "[" << low_ << ", " << high_ << "): " << existing_.size() + << " elements, " << existing_uniq_.size() << " unique values, " + << non_existing_uniq_.size() << " unique non-existing values"; + return oss.str(); + } + + private: + Random rand_; + uint32_t low_ = 0; + uint32_t high_ = 0; + std::vector existing_{}; + KeySet existing_uniq_{}; + KeySet non_existing_uniq_{}; + bool initialized_ = false; + }; + + // Return + std::pair ChooseExistingA(ThreadState* thread); + + uint32_t GenerateNextA(ThreadState* thread); + + // Return + std::pair ChooseExistingC(ThreadState* thread); + + uint32_t GenerateNextC(ThreadState* thread); + + // Randomly commit or rollback `txn` + void ProcessRecoveredPreparedTxnsHelper(Transaction* txn, + SharedState*) override; + + // Some applications, e.g. MyRocks writes a KV pair to the database via + // commit-time-write-batch (ctwb) in additional to the transaction's regular + // write batch. The key is usually constant representing some system + // metadata, while the value is monoticailly increasing which represents the + // actual value of the metadata. Method WriteToCommitTimeWriteBatch() + // emulates this scenario. + Status WriteToCommitTimeWriteBatch(Transaction& txn); + + Status CommitAndCreateTimestampedSnapshotIfNeeded(ThreadState* thread, + Transaction& txn); + + void SetupSnapshot(ThreadState* thread, ReadOptions& read_opts, + Transaction& txn, + std::shared_ptr& snapshot); + + std::vector> key_gen_for_a_; + std::vector> key_gen_for_c_; + + Counter counter_{}; + + private: + struct KeySpaces { + uint32_t lb_a = 0; + uint32_t ub_a = 0; + uint32_t lb_c = 0; + uint32_t ub_c = 0; + + explicit KeySpaces() = default; + explicit KeySpaces(uint32_t _lb_a, uint32_t _ub_a, uint32_t _lb_c, + uint32_t _ub_c) + : lb_a(_lb_a), ub_a(_ub_a), lb_c(_lb_c), ub_c(_ub_c) {} + + std::string EncodeTo() const; + bool DecodeFrom(Slice data); + }; + + void PersistKeySpacesDesc(const std::string& key_spaces_path, uint32_t lb_a, + uint32_t ub_a, uint32_t lb_c, uint32_t ub_c); + + KeySpaces ReadKeySpacesDesc(const std::string& key_spaces_path); + + void PreloadDb(SharedState* shared, int threads, uint32_t lb_a, uint32_t ub_a, + uint32_t lb_c, uint32_t ub_c); + + void ScanExistingDb(SharedState* shared, int threads); +}; + +class InvariantChecker { + public: + static_assert(sizeof(MultiOpsTxnsStressTest::Record().a_) == sizeof(uint32_t), + "MultiOpsTxnsStressTest::Record::a_ must be 4 bytes"); + static_assert(sizeof(MultiOpsTxnsStressTest::Record().b_) == sizeof(uint32_t), + "MultiOpsTxnsStressTest::Record::b_ must be 4 bytes"); + static_assert(sizeof(MultiOpsTxnsStressTest::Record().c_) == sizeof(uint32_t), + "MultiOpsTxnsStressTest::Record::c_ must be 4 bytes"); +}; + +class MultiOpsTxnsStressListener : public EventListener { + public: + explicit MultiOpsTxnsStressListener(MultiOpsTxnsStressTest* stress_test) + : stress_test_(stress_test) { + assert(stress_test_); + } + + ~MultiOpsTxnsStressListener() override {} + + void OnFlushCompleted(DB* db, const FlushJobInfo& info) override { + assert(db); +#ifdef NDEBUG + (void)db; +#endif + assert(info.cf_id == 0); + const ReadOptions read_options(Env::IOActivity::kFlush); + stress_test_->VerifyPkSkFast(read_options, info.job_id); + } + + void OnCompactionCompleted(DB* db, const CompactionJobInfo& info) override { + assert(db); +#ifdef NDEBUG + (void)db; +#endif + assert(info.cf_id == 0); + const ReadOptions read_options(Env::IOActivity::kCompaction); + stress_test_->VerifyPkSkFast(read_options, info.job_id); + } + + private: + MultiOpsTxnsStressTest* const stress_test_ = nullptr; +}; + +} // namespace ROCKSDB_NAMESPACE +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/db_stress_tool/no_batched_ops_stress.cc b/librocksdb-sys/rocksdb/db_stress_tool/no_batched_ops_stress.cc new file mode 100644 index 0000000..5d0ee22 --- /dev/null +++ b/librocksdb-sys/rocksdb/db_stress_tool/no_batched_ops_stress.cc @@ -0,0 +1,1999 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db_stress_tool/expected_state.h" +#ifdef GFLAGS +#include "db_stress_tool/db_stress_common.h" +#include "rocksdb/utilities/transaction_db.h" +#include "utilities/fault_injection_fs.h" + +namespace ROCKSDB_NAMESPACE { +class NonBatchedOpsStressTest : public StressTest { + public: + NonBatchedOpsStressTest() {} + + virtual ~NonBatchedOpsStressTest() {} + + void VerifyDb(ThreadState* thread) const override { + // This `ReadOptions` is for validation purposes. Ignore + // `FLAGS_rate_limit_user_ops` to avoid slowing any validation. + ReadOptions options(FLAGS_verify_checksum, true); + std::string ts_str; + Slice ts; + if (FLAGS_user_timestamp_size > 0) { + ts_str = GetNowNanos(); + ts = ts_str; + options.timestamp = &ts; + } + + auto shared = thread->shared; + const int64_t max_key = shared->GetMaxKey(); + const int64_t keys_per_thread = max_key / shared->GetNumThreads(); + int64_t start = keys_per_thread * thread->tid; + int64_t end = start + keys_per_thread; + uint64_t prefix_to_use = + (FLAGS_prefix_size < 0) ? 1 : static_cast(FLAGS_prefix_size); + + if (thread->tid == shared->GetNumThreads() - 1) { + end = max_key; + } + + for (size_t cf = 0; cf < column_families_.size(); ++cf) { + if (thread->shared->HasVerificationFailedYet()) { + break; + } + + enum class VerificationMethod { + kIterator, + kGet, + kGetEntity, + kMultiGet, + kMultiGetEntity, + kGetMergeOperands, + // Add any new items above kNumberOfMethods + kNumberOfMethods + }; + + constexpr int num_methods = + static_cast(VerificationMethod::kNumberOfMethods); + + const VerificationMethod method = + static_cast(thread->rand.Uniform( + (FLAGS_user_timestamp_size > 0) ? num_methods - 1 : num_methods)); + + if (method == VerificationMethod::kIterator) { + std::unique_ptr iter( + db_->NewIterator(options, column_families_[cf])); + + std::string seek_key = Key(start); + iter->Seek(seek_key); + + Slice prefix(seek_key.data(), prefix_to_use); + + for (int64_t i = start; i < end; ++i) { + if (thread->shared->HasVerificationFailedYet()) { + break; + } + + const std::string key = Key(i); + const Slice k(key); + const Slice pfx(key.data(), prefix_to_use); + + // Reseek when the prefix changes + if (prefix_to_use > 0 && prefix.compare(pfx) != 0) { + iter->Seek(k); + seek_key = key; + prefix = Slice(seek_key.data(), prefix_to_use); + } + + Status s = iter->status(); + + std::string from_db; + + if (iter->Valid()) { + const int diff = iter->key().compare(k); + + if (diff > 0) { + s = Status::NotFound(); + } else if (diff == 0) { + if (!VerifyWideColumns(iter->value(), iter->columns())) { + VerificationAbort(shared, static_cast(cf), i, + iter->value(), iter->columns()); + } + + from_db = iter->value().ToString(); + iter->Next(); + } else { + assert(diff < 0); + + VerificationAbort(shared, "An out of range key was found", + static_cast(cf), i); + } + } else { + // The iterator found no value for the key in question, so do not + // move to the next item in the iterator + s = Status::NotFound(); + } + + VerifyOrSyncValue(static_cast(cf), i, options, shared, from_db, + /* msg_prefix */ "Iterator verification", s); + + if (!from_db.empty()) { + PrintKeyValue(static_cast(cf), static_cast(i), + from_db.data(), from_db.size()); + } + } + } else if (method == VerificationMethod::kGet) { + for (int64_t i = start; i < end; ++i) { + if (thread->shared->HasVerificationFailedYet()) { + break; + } + + const std::string key = Key(i); + std::string from_db; + + Status s = db_->Get(options, column_families_[cf], key, &from_db); + + VerifyOrSyncValue(static_cast(cf), i, options, shared, from_db, + /* msg_prefix */ "Get verification", s); + + if (!from_db.empty()) { + PrintKeyValue(static_cast(cf), static_cast(i), + from_db.data(), from_db.size()); + } + } + } else if (method == VerificationMethod::kGetEntity) { + for (int64_t i = start; i < end; ++i) { + if (thread->shared->HasVerificationFailedYet()) { + break; + } + + const std::string key = Key(i); + PinnableWideColumns result; + + Status s = + db_->GetEntity(options, column_families_[cf], key, &result); + + std::string from_db; + + if (s.ok()) { + const WideColumns& columns = result.columns(); + + if (!columns.empty() && + columns.front().name() == kDefaultWideColumnName) { + from_db = columns.front().value().ToString(); + } + + if (!VerifyWideColumns(columns)) { + VerificationAbort(shared, static_cast(cf), i, from_db, + columns); + } + } + + VerifyOrSyncValue(static_cast(cf), i, options, shared, from_db, + /* msg_prefix */ "GetEntity verification", s); + + if (!from_db.empty()) { + PrintKeyValue(static_cast(cf), static_cast(i), + from_db.data(), from_db.size()); + } + } + } else if (method == VerificationMethod::kMultiGet) { + for (int64_t i = start; i < end;) { + if (thread->shared->HasVerificationFailedYet()) { + break; + } + + // Keep the batch size to some reasonable value + size_t batch_size = thread->rand.Uniform(128) + 1; + batch_size = std::min(batch_size, end - i); + + std::vector key_strs(batch_size); + std::vector keys(batch_size); + std::vector values(batch_size); + std::vector statuses(batch_size); + + for (size_t j = 0; j < batch_size; ++j) { + key_strs[j] = Key(i + j); + keys[j] = Slice(key_strs[j]); + } + + db_->MultiGet(options, column_families_[cf], batch_size, keys.data(), + values.data(), statuses.data()); + + for (size_t j = 0; j < batch_size; ++j) { + const std::string from_db = values[j].ToString(); + + VerifyOrSyncValue(static_cast(cf), i + j, options, shared, + from_db, /* msg_prefix */ "MultiGet verification", + statuses[j]); + + if (!from_db.empty()) { + PrintKeyValue(static_cast(cf), static_cast(i + j), + from_db.data(), from_db.size()); + } + } + + i += batch_size; + } + } else if (method == VerificationMethod::kMultiGetEntity) { + for (int64_t i = start; i < end;) { + if (thread->shared->HasVerificationFailedYet()) { + break; + } + + // Keep the batch size to some reasonable value + size_t batch_size = thread->rand.Uniform(128) + 1; + batch_size = std::min(batch_size, end - i); + + std::vector key_strs(batch_size); + std::vector keys(batch_size); + std::vector results(batch_size); + std::vector statuses(batch_size); + + for (size_t j = 0; j < batch_size; ++j) { + key_strs[j] = Key(i + j); + keys[j] = Slice(key_strs[j]); + } + + db_->MultiGetEntity(options, column_families_[cf], batch_size, + keys.data(), results.data(), statuses.data()); + + for (size_t j = 0; j < batch_size; ++j) { + std::string from_db; + + if (statuses[j].ok()) { + const WideColumns& columns = results[j].columns(); + + if (!columns.empty() && + columns.front().name() == kDefaultWideColumnName) { + from_db = columns.front().value().ToString(); + } + + if (!VerifyWideColumns(columns)) { + VerificationAbort(shared, static_cast(cf), i, from_db, + columns); + } + } + + VerifyOrSyncValue( + static_cast(cf), i + j, options, shared, from_db, + /* msg_prefix */ "MultiGetEntity verification", statuses[j]); + + if (!from_db.empty()) { + PrintKeyValue(static_cast(cf), static_cast(i + j), + from_db.data(), from_db.size()); + } + } + + i += batch_size; + } + } else { + assert(method == VerificationMethod::kGetMergeOperands); + + // Start off with small size that will be increased later if necessary + std::vector values(4); + + GetMergeOperandsOptions merge_operands_info; + merge_operands_info.expected_max_number_of_operands = + static_cast(values.size()); + + for (int64_t i = start; i < end; ++i) { + if (thread->shared->HasVerificationFailedYet()) { + break; + } + + const std::string key = Key(i); + const Slice k(key); + std::string from_db; + int number_of_operands = 0; + + Status s = db_->GetMergeOperands(options, column_families_[cf], k, + values.data(), &merge_operands_info, + &number_of_operands); + + if (s.IsIncomplete()) { + // Need to resize values as there are more than values.size() merge + // operands on this key. Should only happen a few times when we + // encounter a key that had more merge operands than any key seen so + // far + values.resize(number_of_operands); + merge_operands_info.expected_max_number_of_operands = + static_cast(number_of_operands); + s = db_->GetMergeOperands(options, column_families_[cf], k, + values.data(), &merge_operands_info, + &number_of_operands); + } + // Assumed here that GetMergeOperands always sets number_of_operand + if (number_of_operands) { + from_db = values[number_of_operands - 1].ToString(); + } + + VerifyOrSyncValue(static_cast(cf), i, options, shared, from_db, + /* msg_prefix */ "GetMergeOperands verification", + s); + + if (!from_db.empty()) { + PrintKeyValue(static_cast(cf), static_cast(i), + from_db.data(), from_db.size()); + } + } + } + } + } + + void ContinuouslyVerifyDb(ThreadState* thread) const override { + if (!cmp_db_) { + return; + } + assert(cmp_db_); + assert(!cmp_cfhs_.empty()); + Status s = cmp_db_->TryCatchUpWithPrimary(); + if (!s.ok()) { + assert(false); + exit(1); + } + + const auto checksum_column_family = [](Iterator* iter, + uint32_t* checksum) -> Status { + assert(nullptr != checksum); + uint32_t ret = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ret = crc32c::Extend(ret, iter->key().data(), iter->key().size()); + ret = crc32c::Extend(ret, iter->value().data(), iter->value().size()); + } + *checksum = ret; + return iter->status(); + }; + + auto* shared = thread->shared; + assert(shared); + const int64_t max_key = shared->GetMaxKey(); + ReadOptions read_opts(FLAGS_verify_checksum, true); + std::string ts_str; + Slice ts; + if (FLAGS_user_timestamp_size > 0) { + ts_str = GetNowNanos(); + ts = ts_str; + read_opts.timestamp = &ts; + } + + static Random64 rand64(shared->GetSeed()); + + { + uint32_t crc = 0; + std::unique_ptr it(cmp_db_->NewIterator(read_opts)); + s = checksum_column_family(it.get(), &crc); + if (!s.ok()) { + fprintf(stderr, "Computing checksum of default cf: %s\n", + s.ToString().c_str()); + assert(false); + } + } + + for (auto* handle : cmp_cfhs_) { + if (thread->rand.OneInOpt(3)) { + // Use Get() + uint64_t key = rand64.Uniform(static_cast(max_key)); + std::string key_str = Key(key); + std::string value; + std::string key_ts; + s = cmp_db_->Get(read_opts, handle, key_str, &value, + FLAGS_user_timestamp_size > 0 ? &key_ts : nullptr); + s.PermitUncheckedError(); + } else { + // Use range scan + std::unique_ptr iter(cmp_db_->NewIterator(read_opts, handle)); + uint32_t rnd = (thread->rand.Next()) % 4; + if (0 == rnd) { + // SeekToFirst() + Next()*5 + read_opts.total_order_seek = true; + iter->SeekToFirst(); + for (int i = 0; i < 5 && iter->Valid(); ++i, iter->Next()) { + } + } else if (1 == rnd) { + // SeekToLast() + Prev()*5 + read_opts.total_order_seek = true; + iter->SeekToLast(); + for (int i = 0; i < 5 && iter->Valid(); ++i, iter->Prev()) { + } + } else if (2 == rnd) { + // Seek() +Next()*5 + uint64_t key = rand64.Uniform(static_cast(max_key)); + std::string key_str = Key(key); + iter->Seek(key_str); + for (int i = 0; i < 5 && iter->Valid(); ++i, iter->Next()) { + } + } else { + // SeekForPrev() + Prev()*5 + uint64_t key = rand64.Uniform(static_cast(max_key)); + std::string key_str = Key(key); + iter->SeekForPrev(key_str); + for (int i = 0; i < 5 && iter->Valid(); ++i, iter->Prev()) { + } + } + } + } + } + + void MaybeClearOneColumnFamily(ThreadState* thread) override { + if (FLAGS_column_families > 1) { + if (thread->rand.OneInOpt(FLAGS_clear_column_family_one_in)) { + // drop column family and then create it again (can't drop default) + int cf = thread->rand.Next() % (FLAGS_column_families - 1) + 1; + std::string new_name = + std::to_string(new_column_family_name_.fetch_add(1)); + { + MutexLock l(thread->shared->GetMutex()); + fprintf( + stdout, + "[CF %d] Dropping and recreating column family. new name: %s\n", + cf, new_name.c_str()); + } + thread->shared->LockColumnFamily(cf); + Status s = db_->DropColumnFamily(column_families_[cf]); + delete column_families_[cf]; + if (!s.ok()) { + fprintf(stderr, "dropping column family error: %s\n", + s.ToString().c_str()); + thread->shared->SafeTerminate(); + } + s = db_->CreateColumnFamily(ColumnFamilyOptions(options_), new_name, + &column_families_[cf]); + column_family_names_[cf] = new_name; + thread->shared->ClearColumnFamily(cf); + if (!s.ok()) { + fprintf(stderr, "creating column family error: %s\n", + s.ToString().c_str()); + thread->shared->SafeTerminate(); + } + thread->shared->UnlockColumnFamily(cf); + } + } + } + + bool ShouldAcquireMutexOnKey() const override { return true; } + + bool IsStateTracked() const override { return true; } + + Status TestGet(ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + auto cfh = column_families_[rand_column_families[0]]; + std::string key_str = Key(rand_keys[0]); + Slice key = key_str; + std::string from_db; + int error_count = 0; + + if (fault_fs_guard) { + fault_fs_guard->EnableErrorInjection(); + SharedState::ignore_read_error = false; + } + + ReadOptions read_opts_copy = read_opts; + std::string read_ts_str; + Slice read_ts_slice; + if (FLAGS_user_timestamp_size > 0) { + read_ts_str = GetNowNanos(); + read_ts_slice = read_ts_str; + read_opts_copy.timestamp = &read_ts_slice; + } + bool read_older_ts = MaybeUseOlderTimestampForPointLookup( + thread, read_ts_str, read_ts_slice, read_opts_copy); + + const ExpectedValue pre_read_expected_value = + thread->shared->Get(rand_column_families[0], rand_keys[0]); + Status s = db_->Get(read_opts_copy, cfh, key, &from_db); + const ExpectedValue post_read_expected_value = + thread->shared->Get(rand_column_families[0], rand_keys[0]); + if (fault_fs_guard) { + error_count = fault_fs_guard->GetAndResetErrorCount(); + } + if (s.ok()) { + if (fault_fs_guard) { + if (error_count && !SharedState::ignore_read_error) { + // Grab mutex so multiple thread don't try to print the + // stack trace at the same time + MutexLock l(thread->shared->GetMutex()); + fprintf(stderr, "Didn't get expected error from Get\n"); + fprintf(stderr, "Callstack that injected the fault\n"); + fault_fs_guard->PrintFaultBacktrace(); + std::terminate(); + } + } + // found case + thread->stats.AddGets(1, 1); + // we only have the latest expected state + if (!FLAGS_skip_verifydb && !read_older_ts) { + if (ExpectedValueHelper::MustHaveNotExisted(pre_read_expected_value, + post_read_expected_value)) { + thread->shared->SetVerificationFailure(); + fprintf(stderr, + "error : inconsistent values for key %s: Get returns %s, " + "but expected state is \"deleted\".\n", + key.ToString(true).c_str(), StringToHex(from_db).c_str()); + } + Slice from_db_slice(from_db); + uint32_t value_base_from_db = GetValueBase(from_db_slice); + if (!ExpectedValueHelper::InExpectedValueBaseRange( + value_base_from_db, pre_read_expected_value, + post_read_expected_value)) { + thread->shared->SetVerificationFailure(); + fprintf(stderr, + "error : inconsistent values for key %s: Get returns %s with " + "value base %d that falls out of expected state's value base " + "range.\n", + key.ToString(true).c_str(), StringToHex(from_db).c_str(), + value_base_from_db); + } + } + } else if (s.IsNotFound()) { + // not found case + thread->stats.AddGets(1, 0); + if (!FLAGS_skip_verifydb && !read_older_ts) { + if (ExpectedValueHelper::MustHaveExisted(pre_read_expected_value, + post_read_expected_value)) { + thread->shared->SetVerificationFailure(); + fprintf(stderr, + "error : inconsistent values for key %s: expected state has " + "the key, Get() returns NotFound.\n", + key.ToString(true).c_str()); + } + } + } else { + if (error_count == 0) { + // errors case + thread->stats.AddErrors(1); + } else { + thread->stats.AddVerifiedErrors(1); + } + } + if (fault_fs_guard) { + fault_fs_guard->DisableErrorInjection(); + } + return s; + } + + std::vector TestMultiGet( + ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + size_t num_keys = rand_keys.size(); + std::vector key_str; + std::vector keys; + key_str.reserve(num_keys); + keys.reserve(num_keys); + std::vector values(num_keys); + std::vector statuses(num_keys); + // When Flags_use_txn is enabled, we also do a read your write check. + std::vector> ryw_expected_values; + ryw_expected_values.reserve(num_keys); + + SharedState* shared = thread->shared; + + int column_family = rand_column_families[0]; + ColumnFamilyHandle* cfh = column_families_[column_family]; + int error_count = 0; + // Do a consistency check between Get and MultiGet. Don't do it too + // often as it will slow db_stress down + bool do_consistency_check = thread->rand.OneIn(4); + + ReadOptions readoptionscopy = read_opts; + if (do_consistency_check) { + readoptionscopy.snapshot = db_->GetSnapshot(); + } + + std::string read_ts_str; + Slice read_ts_slice; + MaybeUseOlderTimestampForPointLookup(thread, read_ts_str, read_ts_slice, + readoptionscopy); + + readoptionscopy.rate_limiter_priority = + FLAGS_rate_limit_user_ops ? Env::IO_USER : Env::IO_TOTAL; + + // To appease clang analyzer + const bool use_txn = FLAGS_use_txn; + + // Create a transaction in order to write some data. The purpose is to + // exercise WriteBatchWithIndex::MultiGetFromBatchAndDB. The transaction + // will be rolled back once MultiGet returns. + std::unique_ptr txn; + if (use_txn) { + WriteOptions wo; + if (FLAGS_rate_limit_auto_wal_flush) { + wo.rate_limiter_priority = Env::IO_USER; + } + Status s = NewTxn(wo, &txn); + if (!s.ok()) { + fprintf(stderr, "NewTxn: %s\n", s.ToString().c_str()); + thread->shared->SafeTerminate(); + } + } + for (size_t i = 0; i < num_keys; ++i) { + uint64_t rand_key = rand_keys[i]; + key_str.emplace_back(Key(rand_key)); + keys.emplace_back(key_str.back()); + if (use_txn) { + if (!shared->AllowsOverwrite(rand_key) && + shared->Exists(column_family, rand_key)) { + // Just do read your write checks for keys that allow overwrites. + ryw_expected_values.push_back(std::nullopt); + continue; + } + // With a 1 in 10 probability, insert the just added key in the batch + // into the transaction. This will create an overlap with the MultiGet + // keys and exercise some corner cases in the code + if (thread->rand.OneIn(10)) { + int op = thread->rand.Uniform(2); + Status s; + assert(txn); + switch (op) { + case 0: + case 1: { + ExpectedValue put_value; + put_value.Put(false /* pending */); + ryw_expected_values.emplace_back(put_value); + char value[100]; + size_t sz = + GenerateValue(put_value.GetValueBase(), value, sizeof(value)); + Slice v(value, sz); + if (op == 0) { + s = txn->Put(cfh, keys.back(), v); + } else { + s = txn->Merge(cfh, keys.back(), v); + } + break; + } + case 2: { + ExpectedValue delete_value; + delete_value.Delete(false /* pending */); + ryw_expected_values.emplace_back(delete_value); + s = txn->Delete(cfh, keys.back()); + break; + } + default: + assert(false); + } + if (!s.ok()) { + fprintf(stderr, "Transaction put: %s\n", s.ToString().c_str()); + thread->shared->SafeTerminate(); + } + } else { + ryw_expected_values.push_back(std::nullopt); + } + } + } + + if (!use_txn) { + if (fault_fs_guard) { + fault_fs_guard->EnableErrorInjection(); + SharedState::ignore_read_error = false; + } + db_->MultiGet(readoptionscopy, cfh, num_keys, keys.data(), values.data(), + statuses.data()); + if (fault_fs_guard) { + error_count = fault_fs_guard->GetAndResetErrorCount(); + } + } else { + assert(txn); + txn->MultiGet(readoptionscopy, cfh, num_keys, keys.data(), values.data(), + statuses.data()); + } + + if (fault_fs_guard && error_count && !SharedState::ignore_read_error) { + int stat_nok = 0; + for (const auto& s : statuses) { + if (!s.ok() && !s.IsNotFound()) { + stat_nok++; + } + } + + if (stat_nok < error_count) { + // Grab mutex so multiple thread don't try to print the + // stack trace at the same time + MutexLock l(thread->shared->GetMutex()); + fprintf(stderr, "Didn't get expected error from MultiGet. \n"); + fprintf(stderr, "num_keys %zu Expected %d errors, seen %d\n", num_keys, + error_count, stat_nok); + fprintf(stderr, "Callstack that injected the fault\n"); + fault_fs_guard->PrintFaultBacktrace(); + std::terminate(); + } + } + if (fault_fs_guard) { + fault_fs_guard->DisableErrorInjection(); + } + + auto ryw_check = + [](const Slice& key, const PinnableSlice& value, const Status& s, + const std::optional& ryw_expected_value) -> bool { + if (!ryw_expected_value.has_value()) { + return true; + } + const ExpectedValue& expected = ryw_expected_value.value(); + char expected_value[100]; + if (s.ok() && + ExpectedValueHelper::MustHaveNotExisted(expected, expected)) { + fprintf(stderr, + "MultiGet returned value different from what was " + "written for key %s\n", + key.ToString(true).c_str()); + fprintf(stderr, + "MultiGet returned ok, transaction has non-committed " + "delete.\n"); + return false; + } else if (s.IsNotFound() && + ExpectedValueHelper::MustHaveExisted(expected, expected)) { + fprintf(stderr, + "MultiGet returned value different from what was " + "written for key %s\n", + key.ToString(true).c_str()); + fprintf(stderr, + "MultiGet returned not found, transaction has " + "non-committed value.\n"); + return false; + } else if (s.ok() && + ExpectedValueHelper::MustHaveExisted(expected, expected)) { + Slice from_txn_slice(value); + size_t sz = GenerateValue(expected.GetValueBase(), expected_value, + sizeof(expected_value)); + Slice expected_value_slice(expected_value, sz); + if (expected_value_slice.compare(from_txn_slice) == 0) { + return true; + } + fprintf(stderr, + "MultiGet returned value different from what was " + "written for key %s\n", + key.ToString(true /* hex */).c_str()); + fprintf(stderr, "MultiGet returned value %s\n", + from_txn_slice.ToString(true /* hex */).c_str()); + fprintf(stderr, "Transaction has non-committed value %s\n", + expected_value_slice.ToString(true /* hex */).c_str()); + return false; + } + return true; + }; + + auto check_multiget = + [&](const Slice& key, const PinnableSlice& expected_value, + const Status& s, + const std::optional& ryw_expected_value) -> bool { + bool is_consistent = true; + bool is_ryw_correct = true; + // Only do the consistency check if no error was injected and + // MultiGet didn't return an unexpected error. If test does not use + // transaction, the consistency check for each key included check results + // from db `Get` and db `MultiGet` are consistent. + // If test use transaction, after consistency check, also do a read your + // own write check. + if (do_consistency_check && !error_count && (s.ok() || s.IsNotFound())) { + Status tmp_s; + std::string value; + + if (use_txn) { + assert(txn); + tmp_s = txn->Get(readoptionscopy, cfh, key, &value); + } else { + tmp_s = db_->Get(readoptionscopy, cfh, key, &value); + } + if (!tmp_s.ok() && !tmp_s.IsNotFound()) { + fprintf(stderr, "Get error: %s\n", s.ToString().c_str()); + is_consistent = false; + } else if (!s.ok() && tmp_s.ok()) { + fprintf(stderr, "MultiGet returned different results with key %s\n", + key.ToString(true).c_str()); + fprintf(stderr, "Get returned ok, MultiGet returned not found\n"); + is_consistent = false; + } else if (s.ok() && tmp_s.IsNotFound()) { + fprintf(stderr, "MultiGet returned different results with key %s\n", + key.ToString(true).c_str()); + fprintf(stderr, "MultiGet returned ok, Get returned not found\n"); + is_consistent = false; + } else if (s.ok() && value != expected_value.ToString()) { + fprintf(stderr, "MultiGet returned different results with key %s\n", + key.ToString(true).c_str()); + fprintf(stderr, "MultiGet returned value %s\n", + expected_value.ToString(true).c_str()); + fprintf(stderr, "Get returned value %s\n", + Slice(value).ToString(true /* hex */).c_str()); + is_consistent = false; + } + } + + // If test uses transaction, continue to do a read your own write check. + if (is_consistent && use_txn) { + is_ryw_correct = ryw_check(key, expected_value, s, ryw_expected_value); + } + + if (!is_consistent) { + fprintf(stderr, "TestMultiGet error: is_consistent is false\n"); + thread->stats.AddErrors(1); + // Fail fast to preserve the DB state + thread->shared->SetVerificationFailure(); + return false; + } else if (!is_ryw_correct) { + fprintf(stderr, "TestMultiGet error: is_ryw_correct is false\n"); + thread->stats.AddErrors(1); + // Fail fast to preserve the DB state + thread->shared->SetVerificationFailure(); + return false; + } else if (s.ok()) { + // found case + thread->stats.AddGets(1, 1); + } else if (s.IsNotFound()) { + // not found case + thread->stats.AddGets(1, 0); + } else if (s.IsMergeInProgress() && use_txn) { + // With txn this is sometimes expected. + thread->stats.AddGets(1, 1); + } else { + if (error_count == 0) { + // errors case + fprintf(stderr, "MultiGet error: %s\n", s.ToString().c_str()); + thread->stats.AddErrors(1); + } else { + thread->stats.AddVerifiedErrors(1); + } + } + return true; + }; + + size_t num_of_keys = keys.size(); + assert(values.size() == num_of_keys); + assert(statuses.size() == num_of_keys); + for (size_t i = 0; i < num_of_keys; ++i) { + bool check_result = true; + if (use_txn) { + assert(ryw_expected_values.size() == num_of_keys); + check_result = check_multiget(keys[i], values[i], statuses[i], + ryw_expected_values[i]); + } else { + check_result = check_multiget(keys[i], values[i], statuses[i], + std::nullopt /* ryw_expected_value */); + } + if (!check_result) { + break; + } + } + + if (readoptionscopy.snapshot) { + db_->ReleaseSnapshot(readoptionscopy.snapshot); + } + if (use_txn) { + txn->Rollback().PermitUncheckedError(); + } + return statuses; + } + + void TestGetEntity(ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + if (fault_fs_guard) { + fault_fs_guard->EnableErrorInjection(); + SharedState::ignore_read_error = false; + } + + assert(thread); + + SharedState* const shared = thread->shared; + assert(shared); + + assert(!rand_column_families.empty()); + assert(!rand_keys.empty()); + + std::unique_ptr lock(new MutexLock( + shared->GetMutexForKey(rand_column_families[0], rand_keys[0]))); + + assert(rand_column_families[0] >= 0); + assert(rand_column_families[0] < static_cast(column_families_.size())); + + ColumnFamilyHandle* const cfh = column_families_[rand_column_families[0]]; + assert(cfh); + + const std::string key = Key(rand_keys[0]); + + PinnableWideColumns from_db; + + const Status s = db_->GetEntity(read_opts, cfh, key, &from_db); + + int error_count = 0; + + if (fault_fs_guard) { + error_count = fault_fs_guard->GetAndResetErrorCount(); + } + + if (s.ok()) { + if (fault_fs_guard) { + if (error_count && !SharedState::ignore_read_error) { + // Grab mutex so multiple threads don't try to print the + // stack trace at the same time + MutexLock l(shared->GetMutex()); + fprintf(stderr, "Didn't get expected error from GetEntity\n"); + fprintf(stderr, "Call stack that injected the fault\n"); + fault_fs_guard->PrintFaultBacktrace(); + std::terminate(); + } + } + + thread->stats.AddGets(1, 1); + + if (!FLAGS_skip_verifydb) { + const WideColumns& columns = from_db.columns(); + ExpectedValue expected = + shared->Get(rand_column_families[0], rand_keys[0]); + if (!VerifyWideColumns(columns)) { + shared->SetVerificationFailure(); + fprintf(stderr, + "error : inconsistent columns returned by GetEntity for key " + "%s: %s\n", + StringToHex(key).c_str(), WideColumnsToHex(columns).c_str()); + } else if (ExpectedValueHelper::MustHaveNotExisted(expected, + expected)) { + shared->SetVerificationFailure(); + fprintf( + stderr, + "error : inconsistent values for key %s: GetEntity returns %s, " + "expected state does not have the key.\n", + StringToHex(key).c_str(), WideColumnsToHex(columns).c_str()); + } + } + } else if (s.IsNotFound()) { + thread->stats.AddGets(1, 0); + + if (!FLAGS_skip_verifydb) { + ExpectedValue expected = + shared->Get(rand_column_families[0], rand_keys[0]); + if (ExpectedValueHelper::MustHaveExisted(expected, expected)) { + shared->SetVerificationFailure(); + fprintf(stderr, + "error : inconsistent values for key %s: expected state has " + "the key, GetEntity returns NotFound.\n", + StringToHex(key).c_str()); + } + } + } else { + if (error_count == 0) { + thread->stats.AddErrors(1); + } else { + thread->stats.AddVerifiedErrors(1); + } + } + + if (fault_fs_guard) { + fault_fs_guard->DisableErrorInjection(); + } + } + + void TestMultiGetEntity(ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + assert(thread); + + ManagedSnapshot snapshot_guard(db_); + + ReadOptions read_opts_copy(read_opts); + read_opts_copy.snapshot = snapshot_guard.snapshot(); + + assert(!rand_column_families.empty()); + assert(rand_column_families[0] >= 0); + assert(rand_column_families[0] < static_cast(column_families_.size())); + + ColumnFamilyHandle* const cfh = column_families_[rand_column_families[0]]; + assert(cfh); + + assert(!rand_keys.empty()); + + const size_t num_keys = rand_keys.size(); + + std::vector keys(num_keys); + std::vector key_slices(num_keys); + + for (size_t i = 0; i < num_keys; ++i) { + keys[i] = Key(rand_keys[i]); + key_slices[i] = keys[i]; + } + + std::vector results(num_keys); + std::vector statuses(num_keys); + + if (fault_fs_guard) { + fault_fs_guard->EnableErrorInjection(); + SharedState::ignore_read_error = false; + } + + db_->MultiGetEntity(read_opts_copy, cfh, num_keys, key_slices.data(), + results.data(), statuses.data()); + + int error_count = 0; + + if (fault_fs_guard) { + error_count = fault_fs_guard->GetAndResetErrorCount(); + + if (error_count && !SharedState::ignore_read_error) { + int stat_nok = 0; + for (const auto& s : statuses) { + if (!s.ok() && !s.IsNotFound()) { + stat_nok++; + } + } + + if (stat_nok < error_count) { + // Grab mutex so multiple threads don't try to print the + // stack trace at the same time + assert(thread->shared); + MutexLock l(thread->shared->GetMutex()); + + fprintf(stderr, "Didn't get expected error from MultiGetEntity\n"); + fprintf(stderr, "num_keys %zu Expected %d errors, seen %d\n", + num_keys, error_count, stat_nok); + fprintf(stderr, "Call stack that injected the fault\n"); + fault_fs_guard->PrintFaultBacktrace(); + std::terminate(); + } + } + + fault_fs_guard->DisableErrorInjection(); + } + + const bool check_get_entity = !error_count && thread->rand.OneIn(4); + + for (size_t i = 0; i < num_keys; ++i) { + const Status& s = statuses[i]; + + bool is_consistent = true; + + if (s.ok() && !VerifyWideColumns(results[i].columns())) { + fprintf( + stderr, + "error : inconsistent columns returned by MultiGetEntity for key " + "%s: %s\n", + StringToHex(keys[i]).c_str(), + WideColumnsToHex(results[i].columns()).c_str()); + is_consistent = false; + } else if (check_get_entity && (s.ok() || s.IsNotFound())) { + PinnableWideColumns cmp_result; + + const Status cmp_s = + db_->GetEntity(read_opts_copy, cfh, key_slices[i], &cmp_result); + + if (!cmp_s.ok() && !cmp_s.IsNotFound()) { + fprintf(stderr, "GetEntity error: %s\n", cmp_s.ToString().c_str()); + is_consistent = false; + } else if (cmp_s.IsNotFound()) { + if (s.ok()) { + fprintf(stderr, + "Inconsistent results for key %s: MultiGetEntity returned " + "ok, GetEntity returned not found\n", + StringToHex(keys[i]).c_str()); + is_consistent = false; + } + } else { + assert(cmp_s.ok()); + + if (s.IsNotFound()) { + fprintf(stderr, + "Inconsistent results for key %s: MultiGetEntity returned " + "not found, GetEntity returned ok\n", + StringToHex(keys[i]).c_str()); + is_consistent = false; + } else { + assert(s.ok()); + + if (results[i] != cmp_result) { + fprintf( + stderr, + "Inconsistent results for key %s: MultiGetEntity returned " + "%s, GetEntity returned %s\n", + StringToHex(keys[i]).c_str(), + WideColumnsToHex(results[i].columns()).c_str(), + WideColumnsToHex(cmp_result.columns()).c_str()); + is_consistent = false; + } + } + } + } + + if (!is_consistent) { + fprintf(stderr, + "TestMultiGetEntity error: results are not consistent\n"); + thread->stats.AddErrors(1); + // Fail fast to preserve the DB state + thread->shared->SetVerificationFailure(); + break; + } else if (s.ok()) { + thread->stats.AddGets(1, 1); + } else if (s.IsNotFound()) { + thread->stats.AddGets(1, 0); + } else { + if (error_count == 0) { + fprintf(stderr, "MultiGetEntity error: %s\n", s.ToString().c_str()); + thread->stats.AddErrors(1); + } else { + thread->stats.AddVerifiedErrors(1); + } + } + } + } + + Status TestPrefixScan(ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + assert(!rand_column_families.empty()); + assert(!rand_keys.empty()); + + ColumnFamilyHandle* const cfh = column_families_[rand_column_families[0]]; + assert(cfh); + + const std::string key = Key(rand_keys[0]); + const Slice prefix(key.data(), FLAGS_prefix_size); + + std::string upper_bound; + Slice ub_slice; + ReadOptions ro_copy = read_opts; + + // Get the next prefix first and then see if we want to set upper bound. + // We'll use the next prefix in an assertion later on + if (GetNextPrefix(prefix, &upper_bound) && thread->rand.OneIn(2)) { + // For half of the time, set the upper bound to the next prefix + ub_slice = Slice(upper_bound); + ro_copy.iterate_upper_bound = &ub_slice; + } + + std::string read_ts_str; + Slice read_ts_slice; + MaybeUseOlderTimestampForRangeScan(thread, read_ts_str, read_ts_slice, + ro_copy); + + std::unique_ptr iter(db_->NewIterator(ro_copy, cfh)); + + uint64_t count = 0; + Status s; + + if (fault_fs_guard) { + fault_fs_guard->EnableErrorInjection(); + SharedState::ignore_read_error = false; + } + + for (iter->Seek(prefix); iter->Valid() && iter->key().starts_with(prefix); + iter->Next()) { + ++count; + + // When iter_start_ts is set, iterator exposes internal keys, including + // tombstones; however, we want to perform column validation only for + // value-like types. + if (ro_copy.iter_start_ts) { + const ValueType value_type = ExtractValueType(iter->key()); + if (value_type != kTypeValue && value_type != kTypeBlobIndex && + value_type != kTypeWideColumnEntity) { + continue; + } + } + + if (!VerifyWideColumns(iter->value(), iter->columns())) { + s = Status::Corruption("Value and columns inconsistent", + DebugString(iter->value(), iter->columns())); + break; + } + } + + if (ro_copy.iter_start_ts == nullptr) { + assert(count <= GetPrefixKeyCount(prefix.ToString(), upper_bound)); + } + + if (s.ok()) { + s = iter->status(); + } + + uint64_t error_count = 0; + if (fault_fs_guard) { + error_count = fault_fs_guard->GetAndResetErrorCount(); + } + if (!s.ok() && (!fault_fs_guard || (fault_fs_guard && !error_count))) { + fprintf(stderr, "TestPrefixScan error: %s\n", s.ToString().c_str()); + thread->stats.AddErrors(1); + + return s; + } + + if (fault_fs_guard) { + fault_fs_guard->DisableErrorInjection(); + } + thread->stats.AddPrefixes(1, count); + + return Status::OK(); + } + + Status TestPut(ThreadState* thread, WriteOptions& write_opts, + const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys, + char (&value)[100]) override { + assert(!rand_column_families.empty()); + assert(!rand_keys.empty()); + + auto shared = thread->shared; + assert(shared); + + const int64_t max_key = shared->GetMaxKey(); + + int64_t rand_key = rand_keys[0]; + int rand_column_family = rand_column_families[0]; + std::string write_ts; + + std::unique_ptr lock( + new MutexLock(shared->GetMutexForKey(rand_column_family, rand_key))); + while (!shared->AllowsOverwrite(rand_key) && + (FLAGS_use_merge || shared->Exists(rand_column_family, rand_key))) { + lock.reset(); + + rand_key = thread->rand.Next() % max_key; + rand_column_family = thread->rand.Next() % FLAGS_column_families; + + lock.reset( + new MutexLock(shared->GetMutexForKey(rand_column_family, rand_key))); + if (FLAGS_user_timestamp_size > 0) { + write_ts = GetNowNanos(); + } + } + + if (write_ts.empty() && FLAGS_user_timestamp_size) { + write_ts = GetNowNanos(); + } + + const std::string k = Key(rand_key); + + ColumnFamilyHandle* const cfh = column_families_[rand_column_family]; + assert(cfh); + + if (FLAGS_verify_before_write) { + std::string from_db; + Status s = db_->Get(read_opts, cfh, k, &from_db); + if (!VerifyOrSyncValue(rand_column_family, rand_key, read_opts, shared, + /* msg_prefix */ "Pre-Put Get verification", + from_db, s)) { + return s; + } + } + + PendingExpectedValue pending_expected_value = + shared->PreparePut(rand_column_family, rand_key); + const uint32_t value_base = pending_expected_value.GetFinalValueBase(); + const size_t sz = GenerateValue(value_base, value, sizeof(value)); + const Slice v(value, sz); + + + Status s; + + if (FLAGS_use_merge) { + if (!FLAGS_use_txn) { + if (FLAGS_user_timestamp_size == 0) { + s = db_->Merge(write_opts, cfh, k, v); + } else { + s = db_->Merge(write_opts, cfh, k, write_ts, v); + } + } else { + s = ExecuteTransaction(write_opts, thread, [&](Transaction& txn) { + return txn.Merge(cfh, k, v); + }); + } + } else if (FLAGS_use_put_entity_one_in > 0 && + (value_base % FLAGS_use_put_entity_one_in) == 0) { + s = db_->PutEntity(write_opts, cfh, k, + GenerateWideColumns(value_base, v)); + } else { + if (!FLAGS_use_txn) { + if (FLAGS_user_timestamp_size == 0) { + s = db_->Put(write_opts, cfh, k, v); + } else { + s = db_->Put(write_opts, cfh, k, write_ts, v); + } + } else { + s = ExecuteTransaction(write_opts, thread, [&](Transaction& txn) { + return txn.Put(cfh, k, v); + }); + } + } + + pending_expected_value.Commit(); + + if (!s.ok()) { + if (FLAGS_injest_error_severity >= 2) { + if (!is_db_stopped_ && s.severity() >= Status::Severity::kFatalError) { + is_db_stopped_ = true; + } else if (!is_db_stopped_ || + s.severity() < Status::Severity::kFatalError) { + fprintf(stderr, "put or merge error: %s\n", s.ToString().c_str()); + thread->shared->SafeTerminate(); + } + } else { + fprintf(stderr, "put or merge error: %s\n", s.ToString().c_str()); + thread->shared->SafeTerminate(); + } + } + + thread->stats.AddBytesForWrites(1, sz); + PrintKeyValue(rand_column_family, static_cast(rand_key), value, + sz); + return s; + } + + Status TestDelete(ThreadState* thread, WriteOptions& write_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + int64_t rand_key = rand_keys[0]; + int rand_column_family = rand_column_families[0]; + auto shared = thread->shared; + + std::unique_ptr lock( + new MutexLock(shared->GetMutexForKey(rand_column_family, rand_key))); + + // OPERATION delete + std::string write_ts_str = GetNowNanos(); + Slice write_ts = write_ts_str; + + std::string key_str = Key(rand_key); + Slice key = key_str; + auto cfh = column_families_[rand_column_family]; + + // Use delete if the key may be overwritten and a single deletion + // otherwise. + Status s; + if (shared->AllowsOverwrite(rand_key)) { + PendingExpectedValue pending_expected_value = + shared->PrepareDelete(rand_column_family, rand_key); + if (!FLAGS_use_txn) { + if (FLAGS_user_timestamp_size == 0) { + s = db_->Delete(write_opts, cfh, key); + } else { + s = db_->Delete(write_opts, cfh, key, write_ts); + } + } else { + s = ExecuteTransaction(write_opts, thread, [&](Transaction& txn) { + return txn.Delete(cfh, key); + }); + } + pending_expected_value.Commit(); + + thread->stats.AddDeletes(1); + if (!s.ok()) { + if (FLAGS_injest_error_severity >= 2) { + if (!is_db_stopped_ && + s.severity() >= Status::Severity::kFatalError) { + is_db_stopped_ = true; + } else if (!is_db_stopped_ || + s.severity() < Status::Severity::kFatalError) { + fprintf(stderr, "delete error: %s\n", s.ToString().c_str()); + thread->shared->SafeTerminate(); + } + } else { + fprintf(stderr, "delete error: %s\n", s.ToString().c_str()); + thread->shared->SafeTerminate(); + } + } + } else { + PendingExpectedValue pending_expected_value = + shared->PrepareSingleDelete(rand_column_family, rand_key); + if (!FLAGS_use_txn) { + if (FLAGS_user_timestamp_size == 0) { + s = db_->SingleDelete(write_opts, cfh, key); + } else { + s = db_->SingleDelete(write_opts, cfh, key, write_ts); + } + } else { + s = ExecuteTransaction(write_opts, thread, [&](Transaction& txn) { + return txn.SingleDelete(cfh, key); + }); + } + pending_expected_value.Commit(); + thread->stats.AddSingleDeletes(1); + if (!s.ok()) { + if (FLAGS_injest_error_severity >= 2) { + if (!is_db_stopped_ && + s.severity() >= Status::Severity::kFatalError) { + is_db_stopped_ = true; + } else if (!is_db_stopped_ || + s.severity() < Status::Severity::kFatalError) { + fprintf(stderr, "single delete error: %s\n", s.ToString().c_str()); + thread->shared->SafeTerminate(); + } + } else { + fprintf(stderr, "single delete error: %s\n", s.ToString().c_str()); + thread->shared->SafeTerminate(); + } + } + } + return s; + } + + Status TestDeleteRange(ThreadState* thread, WriteOptions& write_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + // OPERATION delete range + std::vector> range_locks; + // delete range does not respect disallowed overwrites. the keys for + // which overwrites are disallowed are randomly distributed so it + // could be expensive to find a range where each key allows + // overwrites. + int64_t rand_key = rand_keys[0]; + int rand_column_family = rand_column_families[0]; + auto shared = thread->shared; + int64_t max_key = shared->GetMaxKey(); + if (rand_key > max_key - FLAGS_range_deletion_width) { + rand_key = + thread->rand.Next() % (max_key - FLAGS_range_deletion_width + 1); + } + for (int j = 0; j < FLAGS_range_deletion_width; ++j) { + if (j == 0 || + ((rand_key + j) & ((1 << FLAGS_log2_keys_per_lock) - 1)) == 0) { + range_locks.emplace_back(new MutexLock( + shared->GetMutexForKey(rand_column_family, rand_key + j))); + } + } + std::vector pending_expected_values = + shared->PrepareDeleteRange(rand_column_family, rand_key, + rand_key + FLAGS_range_deletion_width); + const int covered = static_cast(pending_expected_values.size()); + std::string keystr = Key(rand_key); + Slice key = keystr; + auto cfh = column_families_[rand_column_family]; + std::string end_keystr = Key(rand_key + FLAGS_range_deletion_width); + Slice end_key = end_keystr; + std::string write_ts_str; + Slice write_ts; + Status s; + if (FLAGS_user_timestamp_size) { + write_ts_str = GetNowNanos(); + write_ts = write_ts_str; + s = db_->DeleteRange(write_opts, cfh, key, end_key, write_ts); + } else { + s = db_->DeleteRange(write_opts, cfh, key, end_key); + } + if (!s.ok()) { + if (FLAGS_injest_error_severity >= 2) { + if (!is_db_stopped_ && s.severity() >= Status::Severity::kFatalError) { + is_db_stopped_ = true; + } else if (!is_db_stopped_ || + s.severity() < Status::Severity::kFatalError) { + fprintf(stderr, "delete range error: %s\n", s.ToString().c_str()); + thread->shared->SafeTerminate(); + } + } else { + fprintf(stderr, "delete range error: %s\n", s.ToString().c_str()); + thread->shared->SafeTerminate(); + } + } + for (PendingExpectedValue& pending_expected_value : + pending_expected_values) { + pending_expected_value.Commit(); + } + thread->stats.AddRangeDeletions(1); + thread->stats.AddCoveredByRangeDeletions(covered); + return s; + } + + void TestIngestExternalFile(ThreadState* thread, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + const std::string sst_filename = + FLAGS_db + "/." + std::to_string(thread->tid) + ".sst"; + Status s; + if (db_stress_env->FileExists(sst_filename).ok()) { + // Maybe we terminated abnormally before, so cleanup to give this file + // ingestion a clean slate + s = db_stress_env->DeleteFile(sst_filename); + } + + SstFileWriter sst_file_writer(EnvOptions(options_), options_); + if (s.ok()) { + s = sst_file_writer.Open(sst_filename); + } + int64_t key_base = rand_keys[0]; + int column_family = rand_column_families[0]; + std::vector> range_locks; + range_locks.reserve(FLAGS_ingest_external_file_width); + std::vector keys; + keys.reserve(FLAGS_ingest_external_file_width); + std::vector values; + values.reserve(FLAGS_ingest_external_file_width); + std::vector pending_expected_values; + pending_expected_values.reserve(FLAGS_ingest_external_file_width); + SharedState* shared = thread->shared; + + assert(FLAGS_nooverwritepercent < 100); + // Grab locks, set pending state on expected values, and add keys + for (int64_t key = key_base; + s.ok() && key < shared->GetMaxKey() && + static_cast(keys.size()) < FLAGS_ingest_external_file_width; + ++key) { + if (key == key_base || + (key & ((1 << FLAGS_log2_keys_per_lock) - 1)) == 0) { + range_locks.emplace_back( + new MutexLock(shared->GetMutexForKey(column_family, key))); + } + if (!shared->AllowsOverwrite(key)) { + // We could alternatively include `key` that is deleted. + continue; + } + keys.push_back(key); + + PendingExpectedValue pending_expected_value = + shared->PreparePut(column_family, key); + const uint32_t value_base = pending_expected_value.GetFinalValueBase(); + values.push_back(value_base); + pending_expected_values.push_back(pending_expected_value); + + char value[100]; + size_t value_len = GenerateValue(value_base, value, sizeof(value)); + auto key_str = Key(key); + s = sst_file_writer.Put(Slice(key_str), Slice(value, value_len)); + } + + if (s.ok() && keys.empty()) { + return; + } + + if (s.ok()) { + s = sst_file_writer.Finish(); + } + if (s.ok()) { + s = db_->IngestExternalFile(column_families_[column_family], + {sst_filename}, IngestExternalFileOptions()); + } + if (!s.ok()) { + fprintf(stderr, "file ingestion error: %s\n", s.ToString().c_str()); + thread->shared->SafeTerminate(); + } + + for (size_t i = 0; i < pending_expected_values.size(); ++i) { + pending_expected_values[i].Commit(); + } + } + + // Given a key K, this creates an iterator which scans the range + // [K, K + FLAGS_num_iterations) forward and backward. + // Then does a random sequence of Next/Prev operations. + Status TestIterateAgainstExpected( + ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + assert(thread); + assert(!rand_column_families.empty()); + assert(!rand_keys.empty()); + + auto shared = thread->shared; + assert(shared); + + int64_t max_key = shared->GetMaxKey(); + + const int64_t num_iter = static_cast(FLAGS_num_iterations); + + int64_t lb = rand_keys[0]; + if (lb > max_key - num_iter) { + lb = thread->rand.Next() % (max_key - num_iter + 1); + } + + const int64_t ub = lb + num_iter; + + // Lock the whole range over which we might iterate to ensure it doesn't + // change under us. + const int rand_column_family = rand_column_families[0]; + + // Testing parallel read and write to the same key with user timestamp + // is not currently supported + std::vector> range_locks; + if (FLAGS_user_timestamp_size > 0) { + range_locks = shared->GetLocksForKeyRange(rand_column_family, lb, ub); + } + + ReadOptions ro(read_opts); + ro.total_order_seek = true; + + std::string read_ts_str; + Slice read_ts; + if (FLAGS_user_timestamp_size > 0) { + read_ts_str = GetNowNanos(); + read_ts = read_ts_str; + ro.timestamp = &read_ts; + } + + std::string max_key_str; + Slice max_key_slice; + if (!FLAGS_destroy_db_initially) { + max_key_str = Key(max_key); + max_key_slice = max_key_str; + // to restrict iterator from reading keys written in batched_op_stress + // that do not have expected state updated and may not be parseable by + // GetIntVal(). + ro.iterate_upper_bound = &max_key_slice; + } + + ColumnFamilyHandle* const cfh = column_families_[rand_column_family]; + assert(cfh); + + const std::size_t expected_values_size = static_cast(ub - lb); + std::vector pre_read_expected_values; + std::vector post_read_expected_values; + + for (int64_t i = 0; i < static_cast(expected_values_size); ++i) { + pre_read_expected_values.push_back( + shared->Get(rand_column_family, i + lb)); + } + std::unique_ptr iter(db_->NewIterator(ro, cfh)); + for (int64_t i = 0; i < static_cast(expected_values_size); ++i) { + post_read_expected_values.push_back( + shared->Get(rand_column_family, i + lb)); + } + + assert(pre_read_expected_values.size() == expected_values_size && + pre_read_expected_values.size() == post_read_expected_values.size()); + + std::string op_logs; + + auto check_columns = [&]() { + assert(iter); + assert(iter->Valid()); + + if (!VerifyWideColumns(iter->value(), iter->columns())) { + shared->SetVerificationFailure(); + + fprintf(stderr, + "Verification failed for key %s: " + "Value and columns inconsistent: value: %s, columns: %s\n", + Slice(iter->key()).ToString(/* hex */ true).c_str(), + iter->value().ToString(/* hex */ true).c_str(), + WideColumnsToHex(iter->columns()).c_str()); + fprintf(stderr, "Column family: %s, op_logs: %s\n", + cfh->GetName().c_str(), op_logs.c_str()); + + thread->stats.AddErrors(1); + + return false; + } + + return true; + }; + + auto check_no_key_in_range = [&](int64_t start, int64_t end) { + for (auto j = std::max(start, lb); j < std::min(end, ub); ++j) { + std::size_t index = static_cast(j - lb); + assert(index < pre_read_expected_values.size() && + index < post_read_expected_values.size()); + const ExpectedValue pre_read_expected_value = + pre_read_expected_values[index]; + const ExpectedValue post_read_expected_value = + post_read_expected_values[index]; + if (ExpectedValueHelper::MustHaveExisted(pre_read_expected_value, + post_read_expected_value)) { + // Fail fast to preserve the DB state. + thread->shared->SetVerificationFailure(); + if (iter->Valid()) { + fprintf(stderr, + "Expected state has key %s, iterator is at key %s\n", + Slice(Key(j)).ToString(true).c_str(), + iter->key().ToString(true).c_str()); + } else { + fprintf(stderr, "Expected state has key %s, iterator is invalid\n", + Slice(Key(j)).ToString(true).c_str()); + } + fprintf(stderr, "Column family: %s, op_logs: %s\n", + cfh->GetName().c_str(), op_logs.c_str()); + thread->stats.AddErrors(1); + return false; + } + } + return true; + }; + + // Forward and backward scan to ensure we cover the entire range [lb, ub). + // The random sequence Next and Prev test below tends to be very short + // ranged. + int64_t last_key = lb - 1; + + std::string key_str = Key(lb); + iter->Seek(key_str); + + op_logs += "S " + Slice(key_str).ToString(true) + " "; + + uint64_t curr = 0; + while (true) { + if (!iter->Valid()) { + if (!iter->status().ok()) { + thread->shared->SetVerificationFailure(); + fprintf(stderr, "TestIterate against expected state error: %s\n", + iter->status().ToString().c_str()); + fprintf(stderr, "Column family: %s, op_logs: %s\n", + cfh->GetName().c_str(), op_logs.c_str()); + thread->stats.AddErrors(1); + return iter->status(); + } + if (!check_no_key_in_range(last_key + 1, ub)) { + return Status::OK(); + } + break; + } + + if (!check_columns()) { + return Status::OK(); + } + + // iter is valid, the range (last_key, current key) was skipped + GetIntVal(iter->key().ToString(), &curr); + if (!check_no_key_in_range(last_key + 1, static_cast(curr))) { + return Status::OK(); + } + + last_key = static_cast(curr); + if (last_key >= ub - 1) { + break; + } + + iter->Next(); + + op_logs += "N"; + } + + // backward scan + key_str = Key(ub - 1); + iter->SeekForPrev(key_str); + + op_logs += " SFP " + Slice(key_str).ToString(true) + " "; + + last_key = ub; + while (true) { + if (!iter->Valid()) { + if (!iter->status().ok()) { + thread->shared->SetVerificationFailure(); + fprintf(stderr, "TestIterate against expected state error: %s\n", + iter->status().ToString().c_str()); + fprintf(stderr, "Column family: %s, op_logs: %s\n", + cfh->GetName().c_str(), op_logs.c_str()); + thread->stats.AddErrors(1); + return iter->status(); + } + if (!check_no_key_in_range(lb, last_key)) { + return Status::OK(); + } + break; + } + + if (!check_columns()) { + return Status::OK(); + } + + // the range (current key, last key) was skipped + GetIntVal(iter->key().ToString(), &curr); + if (!check_no_key_in_range(static_cast(curr + 1), last_key)) { + return Status::OK(); + } + + last_key = static_cast(curr); + if (last_key <= lb) { + break; + } + + iter->Prev(); + + op_logs += "P"; + } + + if (thread->rand.OneIn(2)) { + pre_read_expected_values.clear(); + post_read_expected_values.clear(); + // Refresh after forward/backward scan to allow higher chance of SV + // change. + for (int64_t i = 0; i < static_cast(expected_values_size); ++i) { + pre_read_expected_values.push_back( + shared->Get(rand_column_family, i + lb)); + } + iter->Refresh(); + for (int64_t i = 0; i < static_cast(expected_values_size); ++i) { + post_read_expected_values.push_back( + shared->Get(rand_column_family, i + lb)); + } + + assert(pre_read_expected_values.size() == expected_values_size && + pre_read_expected_values.size() == + post_read_expected_values.size()); + } + + // start from middle of [lb, ub) otherwise it is easy to iterate out of + // locked range + const int64_t mid = lb + num_iter / 2; + + key_str = Key(mid); + const Slice key(key_str); + + if (thread->rand.OneIn(2)) { + iter->Seek(key); + op_logs += " S " + key.ToString(true) + " "; + if (!iter->Valid() && iter->status().ok()) { + if (!check_no_key_in_range(mid, ub)) { + return Status::OK(); + } + } + } else { + iter->SeekForPrev(key); + op_logs += " SFP " + key.ToString(true) + " "; + if (!iter->Valid() && iter->status().ok()) { + // iterator says nothing <= mid + if (!check_no_key_in_range(lb, mid + 1)) { + return Status::OK(); + } + } + } + + for (int64_t i = 0; i < num_iter && iter->Valid(); ++i) { + if (!check_columns()) { + return Status::OK(); + } + + GetIntVal(iter->key().ToString(), &curr); + if (static_cast(curr) < lb) { + iter->Next(); + op_logs += "N"; + } else if (static_cast(curr) >= ub) { + iter->Prev(); + op_logs += "P"; + } else { + const uint32_t value_base_from_db = GetValueBase(iter->value()); + std::size_t index = static_cast(curr - lb); + assert(index < pre_read_expected_values.size() && + index < post_read_expected_values.size()); + const ExpectedValue pre_read_expected_value = + pre_read_expected_values[index]; + const ExpectedValue post_read_expected_value = + post_read_expected_values[index]; + if (ExpectedValueHelper::MustHaveNotExisted(pre_read_expected_value, + post_read_expected_value) || + !ExpectedValueHelper::InExpectedValueBaseRange( + value_base_from_db, pre_read_expected_value, + post_read_expected_value)) { + // Fail fast to preserve the DB state. + thread->shared->SetVerificationFailure(); + fprintf(stderr, "Iterator has key %s, but expected state does not.\n", + iter->key().ToString(true).c_str()); + fprintf(stderr, "Column family: %s, op_logs: %s\n", + cfh->GetName().c_str(), op_logs.c_str()); + thread->stats.AddErrors(1); + break; + } + + if (thread->rand.OneIn(2)) { + iter->Next(); + op_logs += "N"; + if (!iter->Valid()) { + break; + } + uint64_t next = 0; + GetIntVal(iter->key().ToString(), &next); + if (!check_no_key_in_range(static_cast(curr + 1), + static_cast(next))) { + return Status::OK(); + } + } else { + iter->Prev(); + op_logs += "P"; + if (!iter->Valid()) { + break; + } + uint64_t prev = 0; + GetIntVal(iter->key().ToString(), &prev); + if (!check_no_key_in_range(static_cast(prev + 1), + static_cast(curr))) { + return Status::OK(); + } + } + } + } + + if (!iter->status().ok()) { + thread->shared->SetVerificationFailure(); + fprintf(stderr, "TestIterate against expected state error: %s\n", + iter->status().ToString().c_str()); + fprintf(stderr, "Column family: %s, op_logs: %s\n", + cfh->GetName().c_str(), op_logs.c_str()); + thread->stats.AddErrors(1); + return iter->status(); + } + + thread->stats.AddIterations(1); + + return Status::OK(); + } + + bool VerifyOrSyncValue(int cf, int64_t key, const ReadOptions& /*opts*/, + SharedState* shared, const std::string& value_from_db, + std::string msg_prefix, const Status& s) const { + if (shared->HasVerificationFailedYet()) { + return false; + } + const ExpectedValue expected_value = shared->Get(cf, key); + + if (expected_value.PendingWrite() || expected_value.PendingDelete()) { + if (s.ok()) { + // Value exists in db, update state to reflect that + Slice slice(value_from_db); + uint32_t value_base = GetValueBase(slice); + shared->SyncPut(cf, key, value_base); + } else if (s.IsNotFound()) { + // Value doesn't exist in db, update state to reflect that + shared->SyncDelete(cf, key); + } + return true; + } + + // compare value_from_db with the value in the shared state + if (s.ok()) { + const Slice slice(value_from_db); + const uint32_t value_base_from_db = GetValueBase(slice); + if (ExpectedValueHelper::MustHaveNotExisted(expected_value, + expected_value) || + !ExpectedValueHelper::InExpectedValueBaseRange( + value_base_from_db, expected_value, expected_value)) { + VerificationAbort(shared, msg_prefix + ": Unexpected value found", cf, + key, value_from_db, ""); + return false; + } + char expected_value_data[kValueMaxLen]; + size_t expected_value_data_size = + GenerateValue(expected_value.GetValueBase(), expected_value_data, + sizeof(expected_value_data)); + if (value_from_db.length() != expected_value_data_size) { + VerificationAbort(shared, + msg_prefix + ": Length of value read is not equal", + cf, key, value_from_db, + Slice(expected_value_data, expected_value_data_size)); + return false; + } + if (memcmp(value_from_db.data(), expected_value_data, + expected_value_data_size) != 0) { + VerificationAbort(shared, + msg_prefix + ": Contents of value read don't match", + cf, key, value_from_db, + Slice(expected_value_data, expected_value_data_size)); + return false; + } + } else if (s.IsNotFound()) { + if (ExpectedValueHelper::MustHaveExisted(expected_value, + expected_value)) { + char expected_value_data[kValueMaxLen]; + size_t expected_value_data_size = + GenerateValue(expected_value.GetValueBase(), expected_value_data, + sizeof(expected_value_data)); + VerificationAbort( + shared, msg_prefix + ": Value not found: " + s.ToString(), cf, key, + "", Slice(expected_value_data, expected_value_data_size)); + return false; + } + } else { + assert(false); + } + return true; + } + + void PrepareTxnDbOptions(SharedState* shared, + TransactionDBOptions& txn_db_opts) override { + txn_db_opts.rollback_deletion_type_callback = + [shared](TransactionDB*, ColumnFamilyHandle*, const Slice& key) { + assert(shared); + uint64_t key_num = 0; + bool ok = GetIntVal(key.ToString(), &key_num); + assert(ok); + (void)ok; + return !shared->AllowsOverwrite(key_num); + }; + } +}; + +StressTest* CreateNonBatchedOpsStressTest() { + return new NonBatchedOpsStressTest(); +} + +} // namespace ROCKSDB_NAMESPACE +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/docs/.gitignore b/librocksdb-sys/rocksdb/docs/.gitignore new file mode 100644 index 0000000..3938549 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/.gitignore @@ -0,0 +1,8 @@ +.DS_STORE +_site/ +*.swo +*.swp +_site +.sass-cache +*.psd +*~ diff --git a/librocksdb-sys/rocksdb/docs/CNAME b/librocksdb-sys/rocksdb/docs/CNAME new file mode 100644 index 0000000..827d1c0 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/CNAME @@ -0,0 +1 @@ +rocksdb.org \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/CONTRIBUTING.md b/librocksdb-sys/rocksdb/docs/CONTRIBUTING.md new file mode 100644 index 0000000..2c5842f --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/CONTRIBUTING.md @@ -0,0 +1,115 @@ +This provides guidance on how to contribute various content to `rocksdb.org`. + +## Getting started + +You should only have to do these one time. + +- Rename this file to `CONTRIBUTING.md`. +- Rename `EXAMPLE-README-FOR-RUNNING-DOCS.md` to `README.md` (replacing the existing `README.md` that came with the template). +- Rename `EXAMPLE-LICENSE` to `LICENSE`. +- Review the [template information](./TEMPLATE-INFORMATION.md). +- Review `./_config.yml`. +- Make sure you update `title`, `description`, `tagline` and `gacode` (Google Analytics) in `./_config.yml`. + +## Basic Structure + +Most content is written in markdown. You name the file `something.md`, then have a header that looks like this: + +``` +--- +docid: getting-started +title: Getting started with ProjectName +layout: docs +permalink: /docs/getting-started.html +--- +``` + +Customize these values for each document, blog post, etc. + +> The filename of the `.md` file doesn't actually matter; what is important is the `docid` being unique and the `permalink` correct and unique too). + +## Landing page + +Modify `index.md` with your new or updated content. + +If you want a `GridBlock` as part of your content, you can do so directly with HTML: + +``` +
+
+
+

Your Features

+ +
+
+ +
+
+

More information

+

+ Stuff here +

+
+
+
+``` + +or with a combination of changing `./_data/features.yml` and adding some Liquid to `index.md`, such as: + +``` +{% include content/gridblocks.html data_source=site.data.features imagealign="bottom"%} +``` + +## Blog + +To modify a blog post, edit the appopriate markdown file in `./_posts/`. + +Adding a new blog post is a four-step process. + +> Some posts have a `permalink` and `comments` in the blog post YAML header. You will not need these for new blog posts. These are an artifact of migrating the blog from Wordpress to gh-pages. + +1. Create your blog post in `./_posts/` in markdown (file extension `.md` or `.markdown`). See current posts in that folder or `./doc-type-examples/2016-04-07-blog-post-example.md` for an example of the YAML format. **If the `./_posts` directory does not exist, create it**. + - You can add a `` tag in the middle of your post such that you show only the excerpt above that tag in the main `/blog` index on your page. +1. If you have not authored a blog post before, modify the `./_data/authors.yml` file with the `author` id you used in your blog post, along with your full name and Facebook ID to get your profile picture. +1. [Run the site locally](./README.md) to test your changes. It will be at `http://127.0.0.1/blog/your-new-blog-post-title.html` +1. Push your changes to GitHub. + +## Docs + +To modify docs, edit the appropriate markdown file in `./_docs/`. + +To add docs to the site.... + +1. Add your markdown file to the `./_docs/` folder. See `./doc-type-examples/docs-hello-world.md` for an example of the YAML header format. **If the `./_docs/` directory does not exist, create it**. + - You can use folders in the `./_docs/` directory to organize your content if you want. +1. Update `_data/nav_docs.yml` to add your new document to the navigation bar. Use the `docid` you put in your doc markdown in as the `id` in the `_data/nav_docs.yml` file. +1. [Run the site locally](./README.md) to test your changes. It will be at `http://127.0.0.1/docs/your-new-doc-permalink.html` +1. Push your changes to GitHub. + +## Header Bar + +To modify the header bar, change `./_data/nav.yml`. + +## Top Level Page + +To modify a top-level page, edit the appropriate markdown file in `./top-level/` + +If you want a top-level page (e.g., http://your-site.com/top-level.html) -- not in `/blog/` or `/docs/`.... + +1. Create a markdown file in the root `./top-level/`. See `./doc-type-examples/top-level-example.md` for more information. +1. If you want a visible link to that file, update `_data/nav.yml` to add a link to your new top-level document in the header bar. + + > This is not necessary if you just want to have a page that is linked to from another page, but not exposed as direct link to the user. + +1. [Run the site locally](./README.md) to test your changes. It will be at `http://127.0.0.1/your-top-level-page-permalink.html` +1. Push your changes to GitHub. + +## Other Changes + +- CSS: `./css/main.css` or `./_sass/*.scss`. +- Images: `./static/images/[docs | posts]/....` +- Main Blog post HTML: `./_includes/post.html` +- Main Docs HTML: `./_includes/doc.html` diff --git a/librocksdb-sys/rocksdb/docs/Gemfile b/librocksdb-sys/rocksdb/docs/Gemfile new file mode 100644 index 0000000..dfb1cfd --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/Gemfile @@ -0,0 +1,4 @@ +source 'https://rubygems.org' +gem 'github-pages', '~> 227' + +gem "webrick", "~> 1.7" diff --git a/librocksdb-sys/rocksdb/docs/LICENSE-DOCUMENTATION b/librocksdb-sys/rocksdb/docs/LICENSE-DOCUMENTATION new file mode 100644 index 0000000..1f255c9 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/LICENSE-DOCUMENTATION @@ -0,0 +1,385 @@ +Attribution 4.0 International + +======================================================================= + +Creative Commons Corporation ("Creative Commons") is not a law firm and +does not provide legal services or legal advice. Distribution of +Creative Commons public licenses does not create a lawyer-client or +other relationship. Creative Commons makes its licenses and related +information available on an "as-is" basis. Creative Commons gives no +warranties regarding its licenses, any material licensed under their +terms and conditions, or any related information. Creative Commons +disclaims all liability for damages resulting from their use to the +fullest extent possible. + +Using Creative Commons Public Licenses + +Creative Commons public licenses provide a standard set of terms and +conditions that creators and other rights holders may use to share +original works of authorship and other material subject to copyright +and certain other rights specified in the public license below. The +following considerations are for informational purposes only, are not +exhaustive, and do not form part of our licenses. + + Considerations for licensors: Our public licenses are + intended for use by those authorized to give the public + permission to use material in ways otherwise restricted by + copyright and certain other rights. Our licenses are + irrevocable. Licensors should read and understand the terms + and conditions of the license they choose before applying it. + Licensors should also secure all rights necessary before + applying our licenses so that the public can reuse the + material as expected. Licensors should clearly mark any + material not subject to the license. This includes other CC- + licensed material, or material used under an exception or + limitation to copyright. More considerations for licensors: + wiki.creativecommons.org/Considerations_for_licensors + + Considerations for the public: By using one of our public + licenses, a licensor grants the public permission to use the + licensed material under specified terms and conditions. If + the licensor's permission is not necessary for any reason--for + example, because of any applicable exception or limitation to + copyright--then that use is not regulated by the license. Our + licenses grant only permissions under copyright and certain + other rights that a licensor has authority to grant. Use of + the licensed material may still be restricted for other + reasons, including because others have copyright or other + rights in the material. A licensor may make special requests, + such as asking that all changes be marked or described. + Although not required by our licenses, you are encouraged to + respect those requests where reasonable. More_considerations + for the public: + wiki.creativecommons.org/Considerations_for_licensees + +======================================================================= + +Creative Commons Attribution 4.0 International Public License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution 4.0 International Public License ("Public License"). To the +extent this Public License may be interpreted as a contract, You are +granted the Licensed Rights in consideration of Your acceptance of +these terms and conditions, and the Licensor grants You such rights in +consideration of benefits the Licensor receives from making the +Licensed Material available under these terms and conditions. + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + +b. Adapter's License means the license You apply to Your Copyright + and Similar Rights in Your contributions to Adapted Material in + accordance with the terms and conditions of this Public License. + +c. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + +d. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + +e. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + +f. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + +g. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + +h. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + +i. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + +j. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + +k. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + +Section 2 -- Scope. + +a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part; and + + b. produce, reproduce, and Share Adapted Material. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + +b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties. + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + +a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + 4. If You Share Adapted Material You produce, the Adapter's + License You apply must not prevent recipients of the Adapted + Material from complying with this Public License. + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + +a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database; + +b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material; and + +c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + +a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + +b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + +c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + +Section 6 -- Term and Termination. + +a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + +b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + +c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + +d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + +Section 7 -- Other Terms and Conditions. + +a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + +b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + +Section 8 -- Interpretation. + +a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + +b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + +c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + +d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + +======================================================================= + +Creative Commons is not a party to its public licenses. +Notwithstanding, Creative Commons may elect to apply one of its public +licenses to material it publishes and in those instances will be +considered the "Licensor." Except for the limited purpose of indicating +that material is shared under a Creative Commons public license or as +otherwise permitted by the Creative Commons policies published at +creativecommons.org/policies, Creative Commons does not authorize the +use of the trademark "Creative Commons" or any other trademark or logo +of Creative Commons without its prior written consent including, +without limitation, in connection with any unauthorized modifications +to any of its public licenses or any other arrangements, +understandings, or agreements concerning use of licensed material. For +the avoidance of doubt, this paragraph does not form part of the public +licenses. + +Creative Commons may be contacted at creativecommons.org. + diff --git a/librocksdb-sys/rocksdb/docs/README.md b/librocksdb-sys/rocksdb/docs/README.md new file mode 100644 index 0000000..0ae8978 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/README.md @@ -0,0 +1,80 @@ +## User Documentation for rocksdb.org + +This directory will contain the user and feature documentation for RocksDB. The documentation will be hosted on GitHub pages. + +### Contributing + +See [CONTRIBUTING.md](./CONTRIBUTING.md) for details on how to add or modify content. + +### Run the Site Locally + +The requirements for running a GitHub pages site locally is described in [GitHub help](https://help.github.com/articles/setting-up-your-github-pages-site-locally-with-jekyll/#requirements). The steps below summarize these steps. + +> If you have run the site before, you can start with step 1 and then move on to step 5. + +1. Ensure that you are in the `/docs` directory in your local RocksDB clone (i.e., the same directory where this `README.md` exists). The below RubyGems commands, etc. must be run from there. + +1. Make sure you have Ruby and [RubyGems](https://rubygems.org/) installed. + + > Ruby >= 2.2 is required for the gems. On the latest versions of Mac OS X, Ruby 2.0 is the + > default. Use `brew install ruby` (or your preferred upgrade mechanism) to install a newer + > version of Ruby for your Mac OS X system. + +1. Make sure you have [Bundler](http://bundler.io/) installed. + + ``` + # may require sudo + gem install bundler + ``` +1. Install the project's dependencies + + ``` + # run this in the 'docs' directory + bundle install + ``` + + > If you get an error when installing `nokogiri`, you may be running into the problem described + > in [this nokogiri issue](https://github.com/sparklemotion/nokogiri/issues/1483). You can + > either `brew uninstall xz` (and then `brew install xz` after the bundle is installed) or + > `xcode-select --install` (although this may not work if you have already installed command + > line tools). + +1. Run Jekyll's server. + + - On first runs or for structural changes to the documentation (e.g., new sidebar menu item), do a full build. + + ``` + bundle exec jekyll serve + ``` + + - For content changes only, you can use `--incremental` for faster builds. + + ``` + bundle exec jekyll serve --incremental + ``` + + > We use `bundle exec` instead of running straight `jekyll` because `bundle exec` will always use the version of Jekyll from our `Gemfile`. Just running `jekyll` will use the system version and may not necessarily be compatible. + + - To run using an actual IP address, you can use `--host=0.0.0.0` + + ``` + bundle exec jekyll serve --host=0.0.0.0 + ``` + + This will allow you to use the IP address associated with your machine in the URL. That way you could share it with other people. + + e.g., on a Mac, you can your IP address with something like `ifconfig | grep "inet " | grep -v 127.0.0.1`. + +1. Either of commands in the previous step will serve up the site on your local device at http://127.0.0.1:4000/ or http://localhost:4000. + +### Updating the Bundle + +The site depends on Github Pages and the installed bundle is based on the `github-pages` gem. +Occasionally that gem might get updated with new or changed functionality. If that is the case, +you can run: + +``` +bundle update +``` + +to get the latest packages for the installation. diff --git a/librocksdb-sys/rocksdb/docs/TEMPLATE-INFORMATION.md b/librocksdb-sys/rocksdb/docs/TEMPLATE-INFORMATION.md new file mode 100644 index 0000000..9175bc0 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/TEMPLATE-INFORMATION.md @@ -0,0 +1,17 @@ +## Template Details + +First, go through `_config.yml` and adjust the available settings to your project's standard. When you make changes here, you'll have to kill the `jekyll serve` instance and restart it to see those changes, but that's only the case with the config file. + +Next, update some image assets - you'll want to update `favicon.png`, `logo.svg`, and `og_image.png` (used for Like button stories and Shares on Facbeook) in the `static` folder with your own logos. + +Next, if you're going to have docs on your site, keep the `_docs` and `docs` folders, if not, you can safely remove them (or you can safely leave them and not include them in your navigation - Jekyll renders all of this before a client views the site anyway, so there's no performance hit from just leaving it there for a future expansion). + +Same thing with a blog section, either keep or delete the `_posts` and `blog` folders. + +You can customize your homepage in three parts - the first in the homepage header, which is mostly automatically derived from the elements you insert into your config file. However, you can also specify a series of 'promotional' elements in `_data/promo.yml`. You can read that file for more information. + +The second place for your homepage is in `index.md` which contains the bulk of the main content below the header. This is all markdown if you want, but you can use HTML and Jekyll's template tags (called Liquid) in there too. Checkout this folder's index.md for an example of one common template tag that we use on our sites called gridblocks. + +The third and last place is in the `_data/powered_by.yml` and `_data/powered_by_highlight.yml` files. Both these files combine to create a section on the homepage that is intended to show a list of companies or apps that are using your project. The `powered_by_highlight` file is a list of curated companies/apps that you want to show as a highlight at the top of this section, including their logos in whatever format you want. The `powered_by` file is a more open list that is just text links to the companies/apps and can be updated via Pull Request by the community. If you don't want these sections on your homepage, just empty out both files and leave them blank. + +The last thing you'll want to do is setup your top level navigation bar. You can do this by editing `nav.yml` and keeping the existing title/href/category structure used there. Although the nav is responsive and fairly flexible design-wise, no more than 5 or 6 nav items is recommended. diff --git a/librocksdb-sys/rocksdb/docs/_config.yml b/librocksdb-sys/rocksdb/docs/_config.yml new file mode 100644 index 0000000..a4055fd --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_config.yml @@ -0,0 +1,85 @@ +# Site settings +permalink: /blog/:year/:month/:day/:title.html +title: RocksDB +tagline: A persistent key-value store for fast storage environments +description: > + RocksDB is an embeddable persistent key-value store for fast storage. +fbappid: "1615782811974223" +gacode: "UA-49459723-1" +# baseurl determines the subpath of your site. For example if you're using an +# organisation.github.io/reponame/ basic site URL, then baseurl would be set +# as "/reponame" but leave blank if you have a top-level domain URL as it is +# now set to "" by default as discussed in: +# http://jekyllrb.com/news/2016/10/06/jekyll-3-3-is-here/ +baseurl: "" + +# the base hostname & protocol for your site +# If baseurl is set, then the absolute url for your site would be url/baseurl +# This was also be set to the right thing automatically for local development +# https://github.com/blog/2277-what-s-new-in-github-pages-with-jekyll-3-3 +# http://jekyllrb.com/news/2016/10/06/jekyll-3-3-is-here/ +url: "http://rocksdb.org" + +# Note: There are new filters in Jekyll 3.3 to help with absolute and relative urls +# absolute_url +# relative_url +# So you will see these used throughout the Jekyll code in this template. +# no more need for | prepend: site.url | prepend: site.baseurl +# http://jekyllrb.com/news/2016/10/06/jekyll-3-3-is-here/ +#https://github.com/blog/2277-what-s-new-in-github-pages-with-jekyll-3-3 + +# The GitHub repo for your project +ghrepo: "facebook/rocksdb" + +# Use these color settings to determine your colour scheme for the site. +color: + # primary should be a vivid color that reflects the project's brand + primary: "#2a2a2a" + # secondary should be a subtle light or dark color used on page backgrounds + secondary: "#f9f9f9" + # Use the following to specify whether the previous two colours are 'light' + # or 'dark' and therefore what colors can be overlaid on them + primary-overlay: "dark" + secondary-overlay: "light" + +#Uncomment this if you want to enable Algolia doc search with your own values +#searchconfig: +# apikey: "" +# indexname: "" + +# Blog posts are builtin to Jekyll by default, with the `_posts` directory. +# Here you can specify other types of documentation. The names here are `docs` +# and `top-level`. This means their content will be in `_docs` and `_top-level`. +# The permalink format is also given. +# http://ben.balter.com/2015/02/20/jekyll-collections/ +collections: + docs: + output: true + permalink: /docs/:name/ + top-level: + output: true + permalink: :name.html + +# DO NOT ADJUST BELOW THIS LINE UNLESS YOU KNOW WHAT YOU ARE CHANGING + +markdown: kramdown +kramdown: + input: GFM + syntax_highlighter: rouge + + syntax_highlighter_opts: + css_class: 'rougeHighlight' + span: + line_numbers: false + block: + line_numbers: true + start_line: 1 + +sass: + style: :compressed + +redcarpet: + extensions: [with_toc_data] + +plugins: + - jekyll-redirect-from diff --git a/librocksdb-sys/rocksdb/docs/_data/authors.yml b/librocksdb-sys/rocksdb/docs/_data/authors.yml new file mode 100644 index 0000000..210987c --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_data/authors.yml @@ -0,0 +1,81 @@ +icanadi: + full_name: Igor Canadi + fbid: 706165749 + +xjin: + full_name: Xing Jin + fbid: 100000739847320 + +leijin: + full_name: Lei Jin + fbid: 634570164 + +yhciang: + full_name: Yueh-Hsuan Chiang + fbid: 1619020986 + +radheshyam: + full_name: Radheshyam Balasundaram + fbid: 800837305 + +zagfox: + full_name: Feng Zhu + fbid: 100006493823622 + +lgalanis: + full_name: Leonidas Galanis + fbid: 8649950 + +sdong: + full_name: Siying Dong + fbid: 9805119 + +dmitrism: + full_name: Dmitri Smirnov + +rven2: + full_name: Venkatesh Radhakrishnan + fbid: 100008352697325 + +yiwu: + full_name: Yi Wu + fbid: 100000476362039 + +maysamyabandeh: + full_name: Maysam Yabandeh + fbid: 100003482360101 + +IslamAbdelRahman: + full_name: Islam AbdelRahman + fbid: 642759407 + +ajkr: + full_name: Andrew Kryczka + fbid: 568694102 + +abhimadan: + full_name: Abhishek Madan + fbid: 1850247869 + +sagar0: + full_name: Sagar Vemuri + fbid: 2419111 + +lightmark: + full_name: Aaron Gao + fbid: 1351549072 + +fgwu: + full_name: Fenggang Wu + fbid: 100002297362180 + +ltamasi: + full_name: Levi Tamasi + +cbi42: + full_name: Changyu Bi + fbid: 100078474793041 + +zjay: + full_name: Jay Zhuang + fbid: 100032386042884 diff --git a/librocksdb-sys/rocksdb/docs/_data/features.yml b/librocksdb-sys/rocksdb/docs/_data/features.yml new file mode 100644 index 0000000..d692c18 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_data/features.yml @@ -0,0 +1,19 @@ +- title: High Performance + text: | + RocksDB uses a log structured database engine, written entirely in C++, for maximum performance. Keys and values are just arbitrarily-sized byte streams. + image: images/promo-performance.svg + +- title: Optimized for Fast Storage + text: | + RocksDB is optimized for fast, low latency storage such as flash drives and high-speed disk drives. RocksDB exploits the full potential of high read/write rates offered by flash or RAM. + image: images/promo-flash.svg + +- title: Adaptable + text: | + RocksDB is adaptable to different workloads. From database storage engines such as [MyRocks](https://github.com/facebook/mysql-5.6) to [application data caching](http://techblog.netflix.com/2016/05/application-data-caching-using-ssds.html) to embedded workloads, RocksDB can be used for a variety of data needs. + image: images/promo-adapt.svg + +- title: Basic and Advanced Database Operations + text: | + RocksDB provides basic operations such as opening and closing a database, reading and writing to more advanced operations such as merging and compaction filters. + image: images/promo-operations.svg diff --git a/librocksdb-sys/rocksdb/docs/_data/nav.yml b/librocksdb-sys/rocksdb/docs/_data/nav.yml new file mode 100644 index 0000000..b70c65f --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_data/nav.yml @@ -0,0 +1,30 @@ +- title: Docs + href: /docs/ + category: docs + +- title: GitHub + href: https://github.com/facebook/rocksdb/ + category: external + +- title: API (C++) + href: https://github.com/facebook/rocksdb/tree/main/include/rocksdb + category: external + +- title: API (Java) + href: https://github.com/facebook/rocksdb/tree/main/java/src/main/java/org/rocksdb + category: external + +- title: Support + href: /support.html + category: support + +- title: Blog + href: /blog/ + category: blog + +- title: Facebook + href: https://www.facebook.com/groups/rocksdb.dev/ + category: external + +# Use external for external links not associated with the paths of the current site. +# If a category is external, site urls, for example, are not prepended to the href, etc.. diff --git a/librocksdb-sys/rocksdb/docs/_data/nav_docs.yml b/librocksdb-sys/rocksdb/docs/_data/nav_docs.yml new file mode 100644 index 0000000..8cdfd2d --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_data/nav_docs.yml @@ -0,0 +1,3 @@ +- title: Quick Start + items: + - id: getting-started diff --git a/librocksdb-sys/rocksdb/docs/_data/powered_by.yml b/librocksdb-sys/rocksdb/docs/_data/powered_by.yml new file mode 100644 index 0000000..a780cfe --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_data/powered_by.yml @@ -0,0 +1 @@ +# Fill in later if desired diff --git a/librocksdb-sys/rocksdb/docs/_data/powered_by_highlight.yml b/librocksdb-sys/rocksdb/docs/_data/powered_by_highlight.yml new file mode 100644 index 0000000..a780cfe --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_data/powered_by_highlight.yml @@ -0,0 +1 @@ +# Fill in later if desired diff --git a/librocksdb-sys/rocksdb/docs/_data/promo.yml b/librocksdb-sys/rocksdb/docs/_data/promo.yml new file mode 100644 index 0000000..9a72aa8 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_data/promo.yml @@ -0,0 +1,6 @@ +# This file determines the list of promotional elements added to the header of \ +# your site's homepage. Full list of plugins are shown + +- type: button + href: docs/getting-started.html + text: Get Started diff --git a/librocksdb-sys/rocksdb/docs/_docs/faq.md b/librocksdb-sys/rocksdb/docs/_docs/faq.md new file mode 100644 index 0000000..0887a09 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_docs/faq.md @@ -0,0 +1,48 @@ +--- +docid: support-faq +title: FAQ +layout: docs +permalink: /docs/support/faq.html +--- + +Here is an ever-growing list of frequently asked questions around RocksDB + +## What is RocksDB? + +RocksDB is an embeddable persistent key-value store for fast storage. RocksDB can also be the foundation for a client-server database but our current focus is on embedded workloads. + +RocksDB builds on [LevelDB](https://code.google.com/p/leveldb/) to be scalable to run on servers with many CPU cores, to efficiently use fast storage, to support IO-bound, in-memory and write-once workloads, and to be flexible to allow for innovation. + +For the latest details, watch [Mark Callaghan’s and Igor Canadi’s talk at CMU on 10/2015](https://scs.hosted.panopto.com/Panopto/Pages/Viewer.aspx?id=f4e0eb37-ae18-468f-9248-cb73edad3e56). [Dhruba Borthakur’s introductory talk](https://github.com/facebook/rocksdb/blob/gh-pages-old/intro.pdf?raw=true) from the Data @ Scale 2013 conference provides some perspective about how RocksDB has evolved. + +## How does performance compare? + +We benchmarked LevelDB and found that it was unsuitable for our server workloads. The [benchmark results](http://leveldb.googlecode.com/svn/trunk/doc/benchmark.html) look awesome at first sight, but we quickly realized that those results were for a database whose size was smaller than the size of RAM on the test machine – where the entire database could fit in the OS page cache. When we performed the same benchmarks on a database that was at least 5 times larger than main memory, the performance results were dismal. + +By contrast, we’ve published the [RocksDB benchmark results](https://github.com/facebook/rocksdb/wiki/Performance-Benchmarks) for server side workloads on Flash. We also measured the performance of LevelDB on these server-workload benchmarks and found that RocksDB solidly outperforms LevelDB for these IO bound workloads. We found that LevelDB’s single-threaded compaction process was insufficient to drive server workloads. We saw frequent write-stalls with LevelDB that caused 99-percentile latency to be tremendously large. We found that mmap-ing a file into the OS cache introduced performance bottlenecks for reads. We could not make LevelDB consume all the IOs offered by the underlying Flash storage. + +## What is RocksDB suitable for? + +RocksDB can be used by applications that need low latency database accesses. Possibilities include: + +* A user-facing application that stores the viewing history and state of users of a website. +* A spam detection application that needs fast access to big data sets. +* A graph-search query that needs to scan a data set in realtime. +* A cache data from Hadoop, thereby allowing applications to query Hadoop data in realtime. +* A message-queue that supports a high number of inserts and deletes. + +## How big is RocksDB adoption? + +RocksDB is an embedded storage engine that is used in a number of backend systems at Facebook. In the Facebook newsfeed’s backend, it replaced another internal storage engine called Centrifuge and is one of the many components used. ZippyDB, a distributed key value store service used by Facebook products relies RocksDB. Details on ZippyDB are in [Muthu Annamalai’s talk at Data@Scale in Seattle](https://youtu.be/DfiN7pG0D0k). Dragon, a distributed graph query engine part of the social graph infrastructure, is using RocksDB to store data. Parse has been running [MongoDB on RocksDB in production](http://blog.parse.com/announcements/mongodb-rocksdb-parse/) since early 2015. + +RocksDB is proving to be a useful component for a lot of other groups in the industry. For a list of projects currently using RocksDB, take a look at our USERS.md list on github. + +## How good is RocksDB as a database storage engine? + +Our engineering team at Facebook firmly believes that RocksDB has great potential as storage engine for databases. It has been proven in production with MongoDB: [MongoRocks](https://github.com/mongodb-partners/mongo-rocks) is the RocksDB based storage engine for MongoDB. + +[MyRocks](https://code.facebook.com/posts/190251048047090/myrocks-a-space-and-write-optimized-mysql-database/) is the RocksDB based storage engine for MySQL. Using RocksDB we have managed to achieve 2x better compression and 10x less write amplification for our benchmarks compared to our existing MySQL setup. Given our current results, work is currently underway to develop MyRocks into a production ready solution for web-scale MySQL workloads. Follow along on [GitHub](https://github.com/facebook/mysql-5.6)! + +## Why is RocksDB open sourced? + +We are open sourcing this project on [GitHub](http://github.com/facebook/rocksdb) because we think it will be useful beyond Facebook. We are hoping that software programmers and database developers will use, enhance, and customize RocksDB for their use-cases. We would also like to engage with the academic community on topics related to efficiency for modern database algorithms. diff --git a/librocksdb-sys/rocksdb/docs/_docs/getting-started.md b/librocksdb-sys/rocksdb/docs/_docs/getting-started.md new file mode 100644 index 0000000..efd17c0 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_docs/getting-started.md @@ -0,0 +1,78 @@ +--- +docid: getting-started +title: Getting started +layout: docs +permalink: /docs/getting-started.html +--- + +## Overview + +The RocksDB library provides a persistent key value store. Keys and values are arbitrary byte arrays. The keys are ordered within the key value store according to a user-specified comparator function. + +The library is maintained by the Facebook Database Engineering Team, and is based on [LevelDB](https://github.com/google/leveldb), by Sanjay Ghemawat and Jeff Dean at Google. + +This overview gives some simple examples of how RocksDB is used. For the story of why RocksDB was created in the first place, see [Dhruba Borthakur’s introductory talk](https://github.com/facebook/rocksdb/blob/gh-pages-old/intro.pdf?raw=true) from the Data @ Scale 2013 conference. + +## Opening A Database + +A rocksdb database has a name which corresponds to a file system directory. All of the contents of database are stored in this directory. The following example shows how to open a database, creating it if necessary: + +```c++ +#include +#include "rocksdb/db.h" + +rocksdb::DB* db; +rocksdb::Options options; +options.create_if_missing = true; +rocksdb::Status status = + rocksdb::DB::Open(options, "/tmp/testdb", &db); +assert(status.ok()); +... +``` + +If you want to raise an error if the database already exists, add the following line before the rocksdb::DB::Open call: + +```c++ +options.error_if_exists = true; +``` + +## Status + +You may have noticed the `rocksdb::Status` type above. Values of this type are returned by most functions in RocksDB that may encounter +an error. You can check if such a result is ok, and also print an associated error message: + +```c++ +rocksdb::Status s = ...; +if (!s.ok()) cerr << s.ToString() << endl; +``` + +## Closing A Database + +When you are done with a database, just delete the database object. For example: + +```c++ +/* open the db as described above */ +/* do something with db */ +delete db; +``` + +## Reads And Writes + +The database provides Put, Delete, and Get methods to modify/query the database. For example, the following code moves the value stored under `key1` to `key2`. + +```c++ +std::string value; +rocksdb::Status s = db->Get(rocksdb::ReadOptions(), key1, &value); +if (s.ok()) s = db->Put(rocksdb::WriteOptions(), key2, value); +if (s.ok()) s = db->Delete(rocksdb::WriteOptions(), key1); +``` + +## Further documentation + +These are just simple examples of how RocksDB is used. The full documentation is currently on the [GitHub wiki](https://github.com/facebook/rocksdb/wiki). + +Here are some specific details about the RocksDB implementation: + +- [RocksDB Overview](https://github.com/facebook/rocksdb/wiki/RocksDB-Overview) +- [Immutable BlockBased Table file format](https://github.com/facebook/rocksdb/wiki/Rocksdb-BlockBasedTable-Format) +- [Log file format](https://github.com/facebook/rocksdb/wiki/Write-Ahead-Log-File-Format) diff --git a/librocksdb-sys/rocksdb/docs/_includes/blog_pagination.html b/librocksdb-sys/rocksdb/docs/_includes/blog_pagination.html new file mode 100644 index 0000000..6a1f334 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/blog_pagination.html @@ -0,0 +1,28 @@ + +{% if paginator.total_pages > 1 %} +
+ +
+{% endif %} diff --git a/librocksdb-sys/rocksdb/docs/_includes/content/gridblocks.html b/librocksdb-sys/rocksdb/docs/_includes/content/gridblocks.html new file mode 100644 index 0000000..49c5e59 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/content/gridblocks.html @@ -0,0 +1,5 @@ +
+{% for item in {{include.data_source}} %} + {% include content/items/gridblock.html item=item layout=include.layout imagealign=include.imagealign align=include.align %} +{% endfor %} +
\ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_includes/content/items/gridblock.html b/librocksdb-sys/rocksdb/docs/_includes/content/items/gridblock.html new file mode 100644 index 0000000..58c9e7f --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/content/items/gridblock.html @@ -0,0 +1,37 @@ +{% if include.layout == "fourColumn" %} + {% assign layout = "fourByGridBlock" %} +{% else %} + {% assign layout = "twoByGridBlock" %} +{% endif %} + +{% if include.imagealign == "side" %} + {% assign imagealign = "imageAlignSide" %} +{% else %} + {% if item.image %} + {% assign imagealign = "imageAlignTop" %} + {% else %} + {% assign imagealign = "" %} + {% endif %} +{% endif %} + +{% if include.align == "right" %} + {% assign align = "alignRight" %} +{% elsif include.align == "center" %} + {% assign align = "alignCenter" %} +{% else %} + {% assign align = "alignLeft" %} +{% endif %} + +
+ {% if item.image %} +
+ {{ item.title }} +
+ {% endif %} +
+

{{ item.title }}

+ {% if item.text %} + {{ item.text | markdownify }} + {% endif %} +
+
diff --git a/librocksdb-sys/rocksdb/docs/_includes/doc.html b/librocksdb-sys/rocksdb/docs/_includes/doc.html new file mode 100644 index 0000000..31e365f --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/doc.html @@ -0,0 +1,25 @@ +
+
+

{% if include.truncate %}{{ page.title }}{% else %}{{ page.title }}{% endif %}

+
+ +
+ {% if include.truncate %} + {% if page.content contains '' %} + {{ page.content | split:'' | first }} +
+ + ...Read More + +
+ {% else %} + {{ page.content }} + {% endif %} + {% else %} + {{ content }} + +

Edit on GitHub

+ {% endif %} +
+ {% include doc_paging.html %} +
diff --git a/librocksdb-sys/rocksdb/docs/_includes/doc_paging.html b/librocksdb-sys/rocksdb/docs/_includes/doc_paging.html new file mode 100644 index 0000000..e69de29 diff --git a/librocksdb-sys/rocksdb/docs/_includes/footer.html b/librocksdb-sys/rocksdb/docs/_includes/footer.html new file mode 100644 index 0000000..f5b78ba --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/footer.html @@ -0,0 +1,34 @@ +
+ +
+ diff --git a/librocksdb-sys/rocksdb/docs/_includes/head.html b/librocksdb-sys/rocksdb/docs/_includes/head.html new file mode 100644 index 0000000..10845ec --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/head.html @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + {% if site.searchconfig %} + + {% endif %} + + {% if page.title %}{{ page.title }} | {{ site.title }}{% else %}{{ site.title }}{% endif %} + + + + + diff --git a/librocksdb-sys/rocksdb/docs/_includes/header.html b/librocksdb-sys/rocksdb/docs/_includes/header.html new file mode 100644 index 0000000..8108d22 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/header.html @@ -0,0 +1,19 @@ +
+
+
+ +

{{ site.title }}

+

{{ site.tagline }}

+ +
+

{% if page.excerpt %}{{ page.excerpt | strip_html }}{% else %}{{ site.description }}{% endif %}

+
+
+ {% for promo in site.data.promo %} + {% include plugins/{{promo.type}}.html button_href=promo.href button_text=promo.text %} +
+ {% endfor %} +
+
+
+
diff --git a/librocksdb-sys/rocksdb/docs/_includes/hero.html b/librocksdb-sys/rocksdb/docs/_includes/hero.html new file mode 100644 index 0000000..e69de29 diff --git a/librocksdb-sys/rocksdb/docs/_includes/home_header.html b/librocksdb-sys/rocksdb/docs/_includes/home_header.html new file mode 100644 index 0000000..90880d1 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/home_header.html @@ -0,0 +1,22 @@ +
+
+
+
+

{{ site.tagline }}

+
+

{% if page.excerpt %}{{ page.excerpt | strip_html }}{% else %}{{ site.description }}{% endif %}

+
+
+ {% for promo in site.data.promo %} +
+ {% include plugins/{{promo.type}}.html href=promo.href text=promo.text children=promo.children %} +
+ {% endfor %} +
+
+ +
+
+
diff --git a/librocksdb-sys/rocksdb/docs/_includes/katex_import.html b/librocksdb-sys/rocksdb/docs/_includes/katex_import.html new file mode 100644 index 0000000..6d6b7cf --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/katex_import.html @@ -0,0 +1,3 @@ + + + diff --git a/librocksdb-sys/rocksdb/docs/_includes/katex_render.html b/librocksdb-sys/rocksdb/docs/_includes/katex_render.html new file mode 100644 index 0000000..56e2e89 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/katex_render.html @@ -0,0 +1,210 @@ + diff --git a/librocksdb-sys/rocksdb/docs/_includes/nav.html b/librocksdb-sys/rocksdb/docs/_includes/nav.html new file mode 100644 index 0000000..9c6fed0 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/nav.html @@ -0,0 +1,37 @@ +
+
+
+ + +

{{ site.title }}

+ + + + +
+
+
diff --git a/librocksdb-sys/rocksdb/docs/_includes/nav/collection_nav.html b/librocksdb-sys/rocksdb/docs/_includes/nav/collection_nav.html new file mode 100644 index 0000000..a3c7a2d --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/nav/collection_nav.html @@ -0,0 +1,64 @@ +
+ +
+ diff --git a/librocksdb-sys/rocksdb/docs/_includes/nav/collection_nav_group.html b/librocksdb-sys/rocksdb/docs/_includes/nav/collection_nav_group.html new file mode 100644 index 0000000..b236ac5 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/nav/collection_nav_group.html @@ -0,0 +1,19 @@ +
+

+{{ grouptitle }}

+
    + {% if include.data_collection %} + {% for item in group.items %} + {% for collectionitem in include.data_collection %} + {% if collectionitem.docid == item.id %} + {% assign groupitem = collectionitem %} + {% include nav/collection_nav_group_item.html %} + {% endif %} + {% endfor %} + {% endfor %} + {% else %} + {% for groupitem in groupitems %} + {% include nav/collection_nav_group_item.html %} + {% endfor %} + {% endif %} +
+
\ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_includes/nav/collection_nav_group_item.html b/librocksdb-sys/rocksdb/docs/_includes/nav/collection_nav_group_item.html new file mode 100644 index 0000000..fbb063d --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/nav/collection_nav_group_item.html @@ -0,0 +1 @@ +
  • {{ groupitem.title }}
    • diff --git a/librocksdb-sys/rocksdb/docs/_includes/nav/header_nav.html b/librocksdb-sys/rocksdb/docs/_includes/nav/header_nav.html new file mode 100644 index 0000000..0fe945c --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/nav/header_nav.html @@ -0,0 +1,30 @@ +
    +
    + +
    + +
    + \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_includes/nav_search.html b/librocksdb-sys/rocksdb/docs/_includes/nav_search.html new file mode 100644 index 0000000..84956b9 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/nav_search.html @@ -0,0 +1,15 @@ +
  • + +
    • + + \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_includes/plugins/all_share.html b/librocksdb-sys/rocksdb/docs/_includes/plugins/all_share.html new file mode 100644 index 0000000..59b00d6 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/plugins/all_share.html @@ -0,0 +1,3 @@ +
    + {% include plugins/like_button.html %}{% include plugins/twitter_share.html %}{% include plugins/google_share.html %} +
    \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_includes/plugins/ascii_cinema.html b/librocksdb-sys/rocksdb/docs/_includes/plugins/ascii_cinema.html new file mode 100644 index 0000000..7d3f971 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/plugins/ascii_cinema.html @@ -0,0 +1,2 @@ +
    + \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_includes/plugins/button.html b/librocksdb-sys/rocksdb/docs/_includes/plugins/button.html new file mode 100644 index 0000000..9e499fe --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/plugins/button.html @@ -0,0 +1,6 @@ +
    + {{ include.text }} +
    \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_includes/plugins/github_star.html b/librocksdb-sys/rocksdb/docs/_includes/plugins/github_star.html new file mode 100644 index 0000000..6aea70f --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/plugins/github_star.html @@ -0,0 +1,4 @@ +
    + Star +
    + \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_includes/plugins/github_watch.html b/librocksdb-sys/rocksdb/docs/_includes/plugins/github_watch.html new file mode 100644 index 0000000..64233b5 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/plugins/github_watch.html @@ -0,0 +1,4 @@ +
    + Watch +
    + \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_includes/plugins/google_share.html b/librocksdb-sys/rocksdb/docs/_includes/plugins/google_share.html new file mode 100644 index 0000000..1b557db --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/plugins/google_share.html @@ -0,0 +1,5 @@ +
    +
    +
    + + diff --git a/librocksdb-sys/rocksdb/docs/_includes/plugins/iframe.html b/librocksdb-sys/rocksdb/docs/_includes/plugins/iframe.html new file mode 100644 index 0000000..525b59f --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/plugins/iframe.html @@ -0,0 +1,6 @@ +
    + +
    +
    + {% include plugins/button.html href=include.href text=include.text %} +
    \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_includes/plugins/like_button.html b/librocksdb-sys/rocksdb/docs/_includes/plugins/like_button.html new file mode 100644 index 0000000..bcb8a7b --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/plugins/like_button.html @@ -0,0 +1,18 @@ +
    + \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_includes/plugins/plugin_row.html b/librocksdb-sys/rocksdb/docs/_includes/plugins/plugin_row.html new file mode 100644 index 0000000..800f50b --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/plugins/plugin_row.html @@ -0,0 +1,5 @@ +
    +{% for child in include.children %} + {% include plugins/{{child.type}}.html href=child.href text=child.text %} +{% endfor %} +
    \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_includes/plugins/post_social_plugins.html b/librocksdb-sys/rocksdb/docs/_includes/plugins/post_social_plugins.html new file mode 100644 index 0000000..a2ecb90 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/plugins/post_social_plugins.html @@ -0,0 +1,41 @@ +
    + +
    +
    + + + diff --git a/librocksdb-sys/rocksdb/docs/_includes/plugins/slideshow.html b/librocksdb-sys/rocksdb/docs/_includes/plugins/slideshow.html new file mode 100644 index 0000000..69fa2b3 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/plugins/slideshow.html @@ -0,0 +1,88 @@ +
    + + + \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_includes/plugins/twitter_follow.html b/librocksdb-sys/rocksdb/docs/_includes/plugins/twitter_follow.html new file mode 100644 index 0000000..b0f25dc --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/plugins/twitter_follow.html @@ -0,0 +1,12 @@ +
    + +
    + + diff --git a/librocksdb-sys/rocksdb/docs/_includes/plugins/twitter_share.html b/librocksdb-sys/rocksdb/docs/_includes/plugins/twitter_share.html new file mode 100644 index 0000000..a60f2a8 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/plugins/twitter_share.html @@ -0,0 +1,11 @@ +
    + +
    + diff --git a/librocksdb-sys/rocksdb/docs/_includes/post.html b/librocksdb-sys/rocksdb/docs/_includes/post.html new file mode 100644 index 0000000..3ae0a2a --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/post.html @@ -0,0 +1,40 @@ +
    +
    +
    + {% for author_idx in page.author %} +
    + {% assign author = site.data.authors[author_idx] %} + {% if author.fbid %} +
    + {{ author.fullname }} +
    + {% endif %} + {% if author.full_name %} +

    {{ author.full_name }}

    + {% endif %} +
    + {% endfor %} +
    +

    {% if include.truncate %}{{ page.title }}{% else %}{{ page.title }}{% endif %}

    +

    Posted {{ page.date | date: '%B %d, %Y' }}{% if page.meta %} • {{ page.meta }}{% endif %}

    +
    +
    + {% if include.truncate %} + {% if page.content contains '' %} + {{ page.content | split:'' | first | markdownify }} +
    + + Read More + +
    + {% else %} + {{ page.content | markdownify }} + {% endif %} + {% else %} + {{ content }} + {% endif %} + {% unless include.truncate %} + {% include plugins/like_button.html %} + {% endunless %} +
    +
    diff --git a/librocksdb-sys/rocksdb/docs/_includes/powered_by.html b/librocksdb-sys/rocksdb/docs/_includes/powered_by.html new file mode 100644 index 0000000..c629429 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/powered_by.html @@ -0,0 +1,28 @@ +{% if site.data.powered_by.first.items or site.data.powered_by_highlight.first.items %} +
    +
    + {% if site.data.powered_by_highlight.first.title %} +

    {{ site.data.powered_by_highlight.first.title }}

    + {% else %} +

    {{ site.data.powered_by.first.title }}

    + {% endif %} + {% if site.data.powered_by_highlight.first.items %} +
    + {% for item in site.data.powered_by_highlight.first.items %} +
    + {{ item.name }} +
    + {% endfor %} +
    + {% endif %} +
    + {% for item in site.data.powered_by.first.items %} +
    + {{ item.name }} +
    + {% endfor %} +
    +
    Does your app use {{ site.title }}? Add it to this list with a pull request!
    +
    +
    +{% endif %} diff --git a/librocksdb-sys/rocksdb/docs/_includes/social_plugins.html b/librocksdb-sys/rocksdb/docs/_includes/social_plugins.html new file mode 100644 index 0000000..9b36580 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/social_plugins.html @@ -0,0 +1,31 @@ + +
    + +
    + + + diff --git a/librocksdb-sys/rocksdb/docs/_includes/ui/button.html b/librocksdb-sys/rocksdb/docs/_includes/ui/button.html new file mode 100644 index 0000000..729ccc3 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_includes/ui/button.html @@ -0,0 +1 @@ +{{ include.button_text }} \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_layouts/basic.html b/librocksdb-sys/rocksdb/docs/_layouts/basic.html new file mode 100644 index 0000000..65bd210 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_layouts/basic.html @@ -0,0 +1,12 @@ +--- +layout: doc_default +--- + +
    +
    +
    + {{ content }} +
    +
    +
    + diff --git a/librocksdb-sys/rocksdb/docs/_layouts/blog.html b/librocksdb-sys/rocksdb/docs/_layouts/blog.html new file mode 100644 index 0000000..1b0da41 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_layouts/blog.html @@ -0,0 +1,11 @@ +--- +category: blog +layout: blog_default +--- + +
    +
    + {{ content }} +
    +
    + diff --git a/librocksdb-sys/rocksdb/docs/_layouts/blog_default.html b/librocksdb-sys/rocksdb/docs/_layouts/blog_default.html new file mode 100644 index 0000000..a29d58d --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_layouts/blog_default.html @@ -0,0 +1,14 @@ + + + {% include head.html %} + + {% include nav.html alwayson=true %} +
    +
    + {% include nav/collection_nav.html navdata=site.posts type="blog" sectionpath="/blog/" sectiontitle="Blog" %} + {{ content }} +
    + {% include footer.html %} +
    + + diff --git a/librocksdb-sys/rocksdb/docs/_layouts/default.html b/librocksdb-sys/rocksdb/docs/_layouts/default.html new file mode 100644 index 0000000..0167d9f --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_layouts/default.html @@ -0,0 +1,12 @@ + + + {% include head.html %} + + {% include nav.html alwayson=true %} +
    + {{ content }} + {% include footer.html %} +
    + + + diff --git a/librocksdb-sys/rocksdb/docs/_layouts/doc_default.html b/librocksdb-sys/rocksdb/docs/_layouts/doc_default.html new file mode 100644 index 0000000..4a41392 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_layouts/doc_default.html @@ -0,0 +1,14 @@ + + + {% include head.html %} + + {% include nav.html alwayson=true %} +
    +
    + {% include nav/collection_nav.html navdata=site.data.nav_docs type="docs" sectionpath="/docs/" sectiontitle="Docs" data_collection=site.docs %} + {{ content }} +
    + {% include footer.html %} +
    + + diff --git a/librocksdb-sys/rocksdb/docs/_layouts/doc_page.html b/librocksdb-sys/rocksdb/docs/_layouts/doc_page.html new file mode 100644 index 0000000..dba761e --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_layouts/doc_page.html @@ -0,0 +1,10 @@ +--- +layout: doc_default +--- + +
    +
    + {{ content }} +
    +
    + diff --git a/librocksdb-sys/rocksdb/docs/_layouts/docs.html b/librocksdb-sys/rocksdb/docs/_layouts/docs.html new file mode 100644 index 0000000..749dafa --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_layouts/docs.html @@ -0,0 +1,5 @@ +--- +layout: doc_page +--- + +{% include doc.html %} \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_layouts/home.html b/librocksdb-sys/rocksdb/docs/_layouts/home.html new file mode 100644 index 0000000..b17732f --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_layouts/home.html @@ -0,0 +1,26 @@ + + + {% include head.html %} +
    +
    + Support Ukraine 🇺🇦 + + Help Provide Humanitarian Aid to Ukraine + + . +
    +
    + + {% include nav.html alwayson=true %} +
    + {% include home_header.html %} +
    +
    + {{ content }} +
    + {% include powered_by.html %} +
    + {% include footer.html %} +
    + + diff --git a/librocksdb-sys/rocksdb/docs/_layouts/page.html b/librocksdb-sys/rocksdb/docs/_layouts/page.html new file mode 100644 index 0000000..bec3680 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_layouts/page.html @@ -0,0 +1,3 @@ +--- +layout: blog +--- diff --git a/librocksdb-sys/rocksdb/docs/_layouts/plain.html b/librocksdb-sys/rocksdb/docs/_layouts/plain.html new file mode 100644 index 0000000..fccc02c --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_layouts/plain.html @@ -0,0 +1,10 @@ +--- +layout: default +--- + +
    +
    + {{ content }} +
    +
    + diff --git a/librocksdb-sys/rocksdb/docs/_layouts/post.html b/librocksdb-sys/rocksdb/docs/_layouts/post.html new file mode 100644 index 0000000..4c92cf2 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_layouts/post.html @@ -0,0 +1,8 @@ +--- +collection: blog +layout: blog +--- + +
    +{% include post.html %} +
    \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_layouts/redirect.html b/librocksdb-sys/rocksdb/docs/_layouts/redirect.html new file mode 100644 index 0000000..c24f817 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_layouts/redirect.html @@ -0,0 +1,6 @@ + + + + + + diff --git a/librocksdb-sys/rocksdb/docs/_layouts/top-level.html b/librocksdb-sys/rocksdb/docs/_layouts/top-level.html new file mode 100644 index 0000000..fccc02c --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_layouts/top-level.html @@ -0,0 +1,10 @@ +--- +layout: default +--- + +
    +
    + {{ content }} +
    +
    + diff --git a/librocksdb-sys/rocksdb/docs/_posts/2014-03-27-how-to-backup-rocksdb.markdown b/librocksdb-sys/rocksdb/docs/_posts/2014-03-27-how-to-backup-rocksdb.markdown new file mode 100644 index 0000000..f9e4a54 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2014-03-27-how-to-backup-rocksdb.markdown @@ -0,0 +1,135 @@ +--- +title: How to backup RocksDB? +layout: post +author: icanadi +category: blog +redirect_from: + - /blog/191/how-to-backup-rocksdb/ +--- + +In RocksDB, we have implemented an easy way to backup your DB. Here is a simple example: + + + + #include "rocksdb/db.h" + #include "utilities/backupable_db.h" + using namespace rocksdb; + + DB* db; + DB::Open(Options(), "/tmp/rocksdb", &db); + BackupableDB* backupable_db = new BackupableDB(db, BackupableDBOptions("/tmp/rocksdb_backup")); + backupable_db->Put(...); // do your thing + backupable_db->CreateNewBackup(); + delete backupable_db; // no need to also delete db + + + + +This simple example will create a backup of your DB in "/tmp/rocksdb_backup". Creating new BackupableDB consumes DB* and you should be calling all the DB methods on object `backupable_db` going forward. + +Restoring is also easy: + + + + RestoreBackupableDB* restore = new RestoreBackupableDB(Env::Default(), BackupableDBOptions("/tmp/rocksdb_backup")); + restore->RestoreDBFromLatestBackup("/tmp/rocksdb", "/tmp/rocksdb"); + delete restore; + + + + +This code will restore the backup back to "/tmp/rocksdb". The second parameter is the location of log files (In some DBs they are different from DB directory, but usually they are the same. See Options::wal_dir for more info). + +An alternative API for backups is to use BackupEngine directly: + + + + #include "rocksdb/db.h" + #include "utilities/backupable_db.h" + using namespace rocksdb; + + DB* db; + DB::Open(Options(), "/tmp/rocksdb", &db); + db->Put(...); // do your thing + BackupEngine* backup_engine = BackupEngine::NewBackupEngine(Env::Default(), BackupableDBOptions("/tmp/rocksdb_backup")); + backup_engine->CreateNewBackup(db); + delete db; + delete backup_engine; + + + + +Restoring with BackupEngine is similar to RestoreBackupableDB: + + + + BackupEngine* backup_engine = BackupEngine::NewBackupEngine(Env::Default(), BackupableDBOptions("/tmp/rocksdb_backup")); + backup_engine->RestoreDBFromLatestBackup("/tmp/rocksdb", "/tmp/rocksdb"); + delete backup_engine; + + + + +Backups are incremental. You can create a new backup with `CreateNewBackup()` and only the new data will be copied to backup directory (for more details on what gets copied, see "Under the hood"). Checksum is always calculated for any backuped file (including sst, log, and etc). It is used to make sure files are kept sound in the file system. Checksum is also verified for files from the previous backups even though they do not need to be copied. A checksum mismatch aborts the current backup (see "Under the hood" for more details). Once you have more backups saved, you can issue `GetBackupInfo()` call to get a list of all backups together with information on timestamp of the backup and the size (please note that sum of all backups' sizes is bigger than the actual size of the backup directory because some data is shared by multiple backups). Backups are identified by their always-increasing IDs. `GetBackupInfo()` is available both in `BackupableDB` and `RestoreBackupableDB`. + +You probably want to keep around only small number of backups. To delete old backups, just call `PurgeOldBackups(N)`, where N is how many backups you'd like to keep. All backups except the N newest ones will be deleted. You can also choose to delete arbitrary backup with call `DeleteBackup(id)`. + +`RestoreDBFromLatestBackup()` will restore the DB from the latest consistent backup. An alternative is `RestoreDBFromBackup()` which takes a backup ID and restores that particular backup. Checksum is calculated for any restored file and compared against the one stored during the backup time. If a checksum mismatch is detected, the restore process is aborted and `Status::Corruption` is returned. Very important thing to note here: Let's say you have backups 1, 2, 3, 4. If you restore from backup 2 and start writing more data to your database, newly created backup will delete old backups 3 and 4 and create new backup 3 on top of 2. + + + +## Advanced usage + + +Let's say you want to backup your DB to HDFS. There is an option in `BackupableDBOptions` to set `backup_env`, which will be used for all file I/O related to backup dir (writes when backuping, reads when restoring). If you set it to HDFS Env, all the backups will be stored in HDFS. + +`BackupableDBOptions::info_log` is a Logger object that is used to print out LOG messages if not-nullptr. + +If `BackupableDBOptions::sync` is true, we will sync data to disk after every file write, guaranteeing that backups will be consistent after a reboot or if machine crashes. Setting it to false will speed things up a bit, but some (newer) backups might be inconsistent. In most cases, everything should be fine, though. + +If you set `BackupableDBOptions::destroy_old_data` to true, creating new `BackupableDB` will delete all the old backups in the backup directory. + +`BackupableDB::CreateNewBackup()` method takes a parameter `flush_before_backup`, which is false by default. When `flush_before_backup` is true, `BackupableDB` will first issue a memtable flush and only then copy the DB files to the backup directory. Doing so will prevent log files from being copied to the backup directory (since flush will delete them). If `flush_before_backup` is false, backup will not issue flush before starting the backup. In that case, the backup will also include log files corresponding to live memtables. Backup will be consistent with current state of the database regardless of `flush_before_backup` parameter. + + + +## Under the hood + + +`BackupableDB` implements `DB` interface and adds four methods to it: `CreateNewBackup()`, `GetBackupInfo()`, `PurgeOldBackups()`, `DeleteBackup()`. Any `DB` interface calls will get forwarded to underlying `DB` object. + +When you call `BackupableDB::CreateNewBackup()`, it does the following: + + + + + + 1. Disable file deletions + + + + 2. Get live files (this includes table files, current and manifest file). + + + + 3. Copy live files to the backup directory. Since table files are immutable and filenames unique, we don't copy a table file that is already present in the backup directory. For example, if there is a file `00050.sst` already backed up and `GetLiveFiles()` returns `00050.sst`, we will not copy that file to the backup directory. However, checksum is calculated for all files regardless if a file needs to be copied or not. If a file is already present, the calculated checksum is compared against previously calculated checksum to make sure nothing crazy happened between backups. If a mismatch is detected, backup is aborted and the system is restored back to the state before `BackupableDB::CreateNewBackup()` is called. One thing to note is that a backup abortion could mean a corruption from a file in backup directory or the corresponding live file in current DB. Both manifest and current files are copied, since they are not immutable. + + + + 4. If `flush_before_backup` was set to false, we also need to copy log files to the backup directory. We call `GetSortedWalFiles()` and copy all live files to the backup directory. + + + + 5. Enable file deletions + + + + +Backup IDs are always increasing and we have a file `LATEST_BACKUP` that contains the ID of the latest backup. If we crash in middle of backing up, on a restart we will detect that there are newer backup files than `LATEST_BACKUP` claims there are. In that case, we will delete any backup newer than `LATEST_BACKUP` and clean up all the files since some of the table files might be corrupted. Having corrupted table files in the backup directory is dangerous because of our deduplication strategy. + + + +## Further reading + + +For the API details, see `include/utilities/backupable_db.h`. For the implementation, see `utilities/backupable/backupable_db.cc`. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2014-03-27-how-to-persist-in-memory-rocksdb-database.markdown b/librocksdb-sys/rocksdb/docs/_posts/2014-03-27-how-to-persist-in-memory-rocksdb-database.markdown new file mode 100644 index 0000000..89ffb2d --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2014-03-27-how-to-persist-in-memory-rocksdb-database.markdown @@ -0,0 +1,54 @@ +--- +title: How to persist in-memory RocksDB database? +layout: post +author: icanadi +category: blog +redirect_from: + - /blog/245/how-to-persist-in-memory-rocksdb-database/ +--- + +In recent months, we have focused on optimizing RocksDB for in-memory workloads. With growing RAM sizes and strict low-latency requirements, lots of applications decide to keep their entire data in memory. Running in-memory database with RocksDB is easy -- just mount your RocksDB directory on tmpfs or ramfs [1]. Even if the process crashes, RocksDB can recover all of your data from in-memory filesystem. However, what happens if the machine reboots? + + + +In this article we will explain how you can recover your in-memory RocksDB database even after a machine reboot. + +Every update to RocksDB is written to two places - one is an in-memory data structure called memtable and second is write-ahead log. Write-ahead log can be used to completely recover the data in memtable. By default, when we flush the memtable to table file, we also delete the current log, since we don't need it anymore for recovery (the data from the log is "persisted" in the table file -- we say that the log file is obsolete). However, if your table file is stored in in-memory file system, you may need the obsolete write-ahead log to recover the data after the machine reboots. Here's how you can do that. + +Options::wal_dir is the directory where RocksDB stores write-ahead log files. If you configure this directory to be on flash or disk, you will not lose current log file on machine reboot. +Options::WAL_ttl_seconds is the timeout when we delete the archived log files. If the timeout is non-zero, obsolete log files will be moved to `archive/` directory under Options::wal_dir. Those archived log files will only be deleted after the specified timeout. + +Let's assume Options::wal_dir is a directory on persistent storage and Options::WAL_ttl_seconds is set to one day. To fully recover the DB, we also need to backup the current snapshot of the database (containing table and metadata files) with a frequency of less than one day. RocksDB provides an utility that enables you to easily backup the snapshot of your database. You can learn more about it here: [How to backup RocksDB?](https://github.com/facebook/rocksdb/wiki/How-to-backup-RocksDB%3F) + +You should configure the backup process to avoid backing up log files, since they are already stored in persistent storage. To do that, set BackupableDBOptions::backup_log_files to false. + +Restore process by default cleans up entire DB and WAL directory. Since we didn't include log files in the backup, we need to make sure that restoring the database doesn't delete log files in WAL directory. When restoring, configure RestoreOptions::keep_log_file to true. That option will also move any archived log files back to WAL directory, enabling RocksDB to replay all archived log files and rebuild the in-memory database state. + +To reiterate, here's what you have to do: + + + + + * Set DB directory to tmpfs or ramfs mounted drive + + + + * Set Options::wal_log to a directory on persistent storage + + + + * Set Options::WAL_ttl_seconds to T seconds + + + + * Backup RocksDB every T/2 seconds, with BackupableDBOptions::backup_log_files = false + + + + * When you lose data, restore from backup with RestoreOptions::keep_log_file = true + + + + + +[1] You might also want to consider using [PlainTable format](https://github.com/facebook/rocksdb/wiki/PlainTable-Format) for table files diff --git a/librocksdb-sys/rocksdb/docs/_posts/2014-04-02-the-1st-rocksdb-local-meetup-held-on-march-27-2014.markdown b/librocksdb-sys/rocksdb/docs/_posts/2014-04-02-the-1st-rocksdb-local-meetup-held-on-march-27-2014.markdown new file mode 100644 index 0000000..7ccbdba --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2014-04-02-the-1st-rocksdb-local-meetup-held-on-march-27-2014.markdown @@ -0,0 +1,53 @@ +--- +title: The 1st RocksDB Local Meetup Held on March 27, 2014 +layout: post +author: xjin +category: blog +redirect_from: + - /blog/323/the-1st-rocksdb-local-meetup-held-on-march-27-2014/ +--- + +On Mar 27, 2014, RocksDB team @ Facebook held the 1st RocksDB local meetup in FB HQ (Menlo Park, California). We invited around 80 guests from 20+ local companies, including LinkedIn, Twitter, Dropbox, Square, Pinterest, MapR, Microsoft and IBM. Finally around 50 guests showed up, totaling around 60% show-up rate. + + + +[![Resize of 20140327_200754](/static/images/Resize-of-20140327_200754-300x225.jpg)](/static/images/Resize-of-20140327_200754-300x225.jpg) + +RocksDB team @ Facebook gave four talks about the latest progress and experience on RocksDB: + + + + + * [Supporting a 1PB In-Memory Workload](https://github.com/facebook/rocksdb/raw/gh-pages/talks/2014-03-27-RocksDB-Meetup-Haobo-RocksDB-In-Memory.pdf) + + + + + * [Column Families in RocksDB](https://github.com/facebook/rocksdb/raw/gh-pages/talks/2014-03-27-RocksDB-Meetup-Igor-Column-Families.pdf) + + + + + * ["Lockless" Get() in RocksDB?](https://github.com/facebook/rocksdb/raw/gh-pages/talks/2014-03-27-RocksDB-Meetup-Lei-Lockless-Get.pdf) + + + + + * [Prefix Hashing in RocksDB](https://github.com/facebook/rocksdb/raw/gh-pages/talks/2014-03-27-RocksDB-Meetup-Siying-Prefix-Hash.pdf) + + +A very interesting question asked by a massive number of guests is: does RocksDB plan to provide replication functionality? Obviously, many applications need a resilient and distributed storage solution, not just single-node storage. We are considering how to approach this issue. + +When will be the next meetup? We haven't decided yet. We will see whether the community is interested in it and how it can help RocksDB grow. + +If you have any questions or feedback for the meetup or RocksDB, please let us know in [our Facebook group](https://www.facebook.com/groups/rocksdb.dev/). + +### Comments + +**[Rajiv](geetasen@gmail.com)** + +Have any of these talks been recorded and if so will they be published? + +**[Igor Canadi](icanadi@fb.com)** + +Yes, I think we plan to publish them soon. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2014-04-07-rocksdb-2-8-release.markdown b/librocksdb-sys/rocksdb/docs/_posts/2014-04-07-rocksdb-2-8-release.markdown new file mode 100644 index 0000000..7be7842 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2014-04-07-rocksdb-2-8-release.markdown @@ -0,0 +1,40 @@ +--- +title: RocksDB 2.8 release +layout: post +author: icanadi +category: blog +redirect_from: + - /blog/371/rocksdb-2-8-release/ +--- + +Check out the new RocksDB 2.8 release on [Github](https://github.com/facebook/rocksdb/releases/tag/2.8.fb). + +RocksDB 2.8. is mostly focused on improving performance for in-memory workloads. We are seeing read QPS as high as 5M (we will write a separate blog post on this). + + + +Here is the summary of new features: + + * Added a new table format called PlainTable, which is optimized for RAM storage (ramfs or tmpfs). You can read more details about it on [our wiki](https://github.com/facebook/rocksdb/wiki/PlainTable-Format). + + + * New prefixed memtable format HashLinkedList, which is optimized for cases where there are only a few keys for each prefix. + + + * Merge operator supports a new function PartialMergeMulti() that allows users to do partial merges against multiple operands. This function enables big speedups for workloads that use merge operators. + + + * Added a V2 compaction filter interface. It buffers the kv-pairs sharing the same key prefix, process them in batches, and return the batched results back to DB. + + + * Geo-spatial support for locations and radial-search. + + + * Improved read performance using thread local cache for frequently accessed data. + + + * Stability improvements -- we're now ignoring partially written tailing record to MANIFEST or WAL files. + + + +We have also introduced small incompatible API changes (mostly for advanced users). You can see full release notes in our [HISTORY.my](https://github.com/facebook/rocksdb/blob/2.8.fb/HISTORY.md) file. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2014-04-21-indexing-sst-files-for-better-lookup-performance.markdown b/librocksdb-sys/rocksdb/docs/_posts/2014-04-21-indexing-sst-files-for-better-lookup-performance.markdown new file mode 100644 index 0000000..368055d --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2014-04-21-indexing-sst-files-for-better-lookup-performance.markdown @@ -0,0 +1,28 @@ +--- +title: Indexing SST Files for Better Lookup Performance +layout: post +author: leijin +category: blog +redirect_from: + - /blog/431/indexing-sst-files-for-better-lookup-performance/ +--- + +For a `Get()` request, RocksDB goes through mutable memtable, list of immutable memtables, and SST files to look up the target key. SST files are organized in levels. + +On level 0, files are sorted based on the time they are flushed. Their key range (as defined by FileMetaData.smallest and FileMetaData.largest) are mostly overlapped with each other. So it needs to look up every L0 file. + + + +Compaction is scheduled periodically to pick up files from an upper level and merges them with files from lower level. As a result, key/values are moved from L0 down the LSM tree gradually. Compaction sorts key/values and split them into files. From level 1 and below, SST files are sorted based on key. Their key range are mutually exclusive. Instead of scanning through each SST file and checking if a key falls into its range, RocksDB performs a binary search based on FileMetaData.largest to locate a candidate file that can potentially contain the target key. This reduces complexity from O(N) to O(log(N)). However, log(N) can still be large for bottom levels. For a fan-out ratio of 10, level 3 can have 1000 files. That requires 10 comparisons to locate a candidate file. This is a significant cost for an in-memory database when you can do [several million gets per second](https://github.com/facebook/rocksdb/wiki/RocksDB-In-Memory-Workload-Performance-Benchmarks). + +One observation to this problem is that: after the LSM tree is built, an SST file's position in its level is fixed. Furthermore, its order relative to files from the next level is also fixed. Based on this idea, we can perform [fractional cascading](http://en.wikipedia.org/wiki/Fractional_cascading) kind of optimization to narrow down the binary search range. Here is an example: + +[![tree_example](/static/images/tree_example1.png)](/static/images/tree_example1.png) + +Level 1 has 2 files and level 2 has 8 files. Now, we want to look up key 80. A binary search based FileMetaData.largest tells you file 1 is the candidate. Then key 80 is compared with its FileMetaData.smallest and FileMetaData.largest to decide if it falls into the range. The comparison shows 80 is less than FileMetaData.smallest (100), so file 1 does not possibly contain key 80. We to proceed to check level 2. Usually, we need to do binary search among all 8 files on level 2. But since we already know target key 80 is less than 100 and only file 1 to file 3 can contain key less than 100, we can safely exclude other files from the search. As a result we cut down the search space from 8 files to 3 files. + +Let's look at another example. We want to get key 230. A binary search on level 1 locates to file 2 (this also implies key 230 is larger than file 1's FileMetaData.largest 200). A comparison with file 2's range shows the target key is smaller than file 2's FileMetaData.smallest 300. Even though, we couldn't find key on level 1, we have derived hints that target key is in range between 200 and 300. Any files on level 2 that cannot overlap with [200, 300] can be safely excluded. As a result, we only need to look at file 5 and file 6 on level 2. + +Inspired by this concept, we pre-build pointers at compaction time on level 1 files that point to a range of files on level 2. For example, file 1 on level 1 points to file 3 (on level 2) on the left and file 4 on the right. File 2 will point to level 2 files 6 and 7. At query time, these pointers are used to determine the actual binary search range based on comparison result. + +Our benchmark shows that this optimization improves lookup QPS by ~5% for similar setup mentioned [here](https://github.com/facebook/rocksdb/wiki/RocksDB-In-Memory-Workload-Performance-Benchmarks). diff --git a/librocksdb-sys/rocksdb/docs/_posts/2014-05-14-lock.markdown b/librocksdb-sys/rocksdb/docs/_posts/2014-05-14-lock.markdown new file mode 100644 index 0000000..12009cc --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2014-05-14-lock.markdown @@ -0,0 +1,88 @@ +--- +title: Reducing Lock Contention in RocksDB +layout: post +author: sdong +category: blog +redirect_from: + - /blog/521/lock/ +--- + +In this post, we briefly introduce the recent improvements we did to RocksDB to improve the issue of lock contention costs. + +RocksDB has a simple thread synchronization mechanism (See [RocksDB Architecture Guide](https://github.com/facebook/rocksdb/wiki/Rocksdb-Architecture-Guide)  to understand terms used below, like SST tables or mem tables). SST tables are immutable after being written and mem tables are lock-free data structures supporting single writer and multiple readers. There is only one single major lock, the DB mutex (DBImpl.mutex_) protecting all the meta operations, including: + + + + * Increase or decrease reference counters of mem tables and SST tables + + + * Change and check meta data structures, before and after finishing compactions, flushes and new mem table creations + + + * Coordinating writers + + +This DB mutex used to be scalability bottleneck preventing us from scaling to more than 16 threads. To address the issue, we improved RocksDB in several ways. + +1. Consolidate reference counters and introduce "super version". For every read operation, mutex was acquired, and reference counters for each mem table and each SST table were increased. One such operation is not expensive but if you are building a high throughput server with lots of reads, the lock contention will become the bottleneck. This is especially true if you store all your data in RAM. + +To solve this problem, we created a meta-meta data structure called “[super version](https://reviews.facebook.net/rROCKSDB1fdb3f7dc60e96394e3e5b69a46ede5d67fb976c)”, which holds reference counters to all those mem table and SST tables, so that readers only need to increase the reference counters for this single data structure. In RocksDB, list of live mem tables and SST tables only changes infrequently, which would happen when new mem tables are created or flush/compaction happens. Now, at those times, a new super version is created with their reference counters increased. A super version lists live mem tables and SST tables so a reader only needs acquire the lock in order to find the latest super version and increase its reference counter. From the super version, the reader can find all the mem and SST tables which are safety accessible as long as the reader holds the reference count for the super version. + +2. We replace some reference counters to stc::atomic objects, so that decreasing reference count of an object usually doesn’t need to be inside the mutex any more. + +3. Make fetching super version and reference counting lock-free in read queries. After consolidating reference counting to one single super version and removing the locking for decreasing reference counts, in read case, we only acquire mutex for one thing: fetch the latest super version and increase the reference count for that (dereference the counter is done in an atomic decrease). We designed and implemented a (mostly) lock-free approach to do it. See [details](https://github.com/facebook/rocksdb/raw/gh-pages/talks/2014-03-27-RocksDB-Meetup-Lei-Lockless-Get.pdf). We will write a separate blog post for that. + +4. Avoid disk I/O inside the mutex. As we know, each disk I/O to hard drives takes several milliseconds. It can be even longer if file system journal is involved or I/Os are queued. Even occasional disk I/O within mutex can cause huge performance outliers. +We identified in two situations, we might do disk I/O inside mutex and we removed them: +(1) Opening and closing transactional log files. We moved those operations out of the mutex. +(2) Information logging. In multiple places we write to logs within mutex. There is a chance that file write will wait for disk I/O to finish before finishing, even if fsync() is not issued, especially in EXT systems. We occasionally see 100+ milliseconds write() latency on EXT. Instead of removing those logging, we came up with a solution of delay logging. When inside mutex, instead of directly writing to the log file, we write to a log buffer, with the timing information. As soon as mutex is released, we flush the log buffer to log files. + +5. Reduce object creation inside the mutex. +Object creation can be slow because it involves malloc (in our case). Malloc sometimes is slow because it needs to lock some shared data structures. Allocating can also be slow because we sometimes do expensive operations in some of our classes' constructors. For these reasons, we try to reduce object creations inside the mutex. Here are two examples: + +(1) std::vector uses malloc inside. We introduced “[autovector](https://reviews.facebook.net/rROCKSDBc01676e46d3be08c3c140361ef1f5884f47d3b3c)” data structure, in which memory for first a few elements are pre-allocated as members of the autovector class. When an autovector is used as a stack variable, no malloc will be needed unless the pre-allocated buffer is used up. This autovector is quite useful for manipulating those meta data structures. Those meta operations are often locked inside DB mutex. + +(2) When building an iterator, we used to creating iterator of every live men table and SST table within the mutex and a merging iterator on top of them. Besides malloc, some of those iterators can be quite expensive to create, like sorting. Now, instead of doing that, we simply increase the reference counters of them, and release the mutex before creating any iterator. + +6. Deal with mutexes in LRU caches. +When I said there was only one single major lock, I was lying. In RocksDB, all LRU caches had exclusive mutexes within to protect writes to the LRU lists, which are done in both of read and write operations. LRU caches are used in block cache and table cache. Both of them are accessed more frequently than DB data structures. Lock contention of these two locks are as intense as the DB mutex. Even if LRU cache is sharded into ShardedLRUCache, we can still see lock contentions, especially table caches. We further address this issue in two way: +(1) Bypassing table caches. A table cache maintains list of SST table’s read handlers. Those handlers contain SST files’ descriptors, table metadata, and possibly data indexes, as well as bloom filters. When the table handler needs to be evicted based on LRU, those information is cleared. When the SST table needs to be read and its table handler is not in LRU cache, the table is opened and those metadata is loaded. In some cases, users want to tune the system in a way that table handler evictions should never happen. It is common for high-throughput, low-latency servers. We introduce a mode where table cache is bypassed in read queries. In this mode, all table handlers are cached and accessed directly, so there is no need to query and adjust table caches for reading the database. It is the users’ responsibility to reserve enough resource for it. This mode can be turned on by setting options.max_open_files=-1. + +(2) [New PlainTable format](//github.com/facebook/rocksdb/wiki/PlainTable-Format) (optimized for SST in ramfs/tmpfs) does not organize data by blocks. Data are located by memory addresses so no block cache is needed. + +With all of those improvements, lock contention is not a bottleneck anymore, which is shown in our [memory-only benchmark](https://github.com/facebook/rocksdb/wiki/RocksDB-In-Memory-Workload-Performance-Benchmarks) . Furthermore, lock contentions are not causing some huge (50 milliseconds+) latency outliers they used to cause. + +### Comments + +**[Lee Hounshell](lee@apsalar.com)** + +Please post an example of reading the same rocksdb concurrently. + +We are using the latest 3.0 rocksdb; however, when two separate processes +try and open the same rocksdb for reading, only one of the open requests +succeed. The other open always fails with “db/LOCK: Resource temporarily unavailable” So far we have not found an option that allows sharing the rocksdb for reads. An example would be most appreciated. + +**[Siying Dong](siying.d@fb.com)** + +Sorry for the delay. We don’t have feature support for this scenario yet. Here is an example you can work around this problem. You can build a snapshot of the DB by doing this: + +1. create a separate directory on the same host for a snapshot of the DB. +1. call `DB::DisableFileDeletions()` +1. call `DB::GetLiveFiles()` to get a full list of the files. +1. for all the files except manifest, add a hardlink file in your new directory pointing to the original file +1. copy the manifest file and truncate the size (you can read the comments of `DB::GetLiveFiles()` for more information) +1. call `DB::EnableFileDeletions()` +1. now you can open the snapshot directory in another process to access those files. Please remember to delete the directory after reading the data to allow those files to be recycled. + +By the way, the best way to ask those questions is in our [facebook group](https://www.facebook.com/groups/rocksdb.dev/). Let us know if you need any further help. + +**[Darshan](darshan.ghumare@gmail.com)** + +Will this consistency problem of RocksDB all occurs in case of single put/write? +What all ACID properties is supported by RocksDB, only durability irrespective of single or batch write? + +**[Siying Dong](siying.d@fb.com)** + +We recently [introduced optimistic transaction](https://reviews.facebook.net/D33435) which can help you ensure all of ACID. + +This blog post is mainly about optimizations in implementation. The RocksDB consistency semantic is not changed. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2014-05-19-rocksdb-3-0-release.markdown b/librocksdb-sys/rocksdb/docs/_posts/2014-05-19-rocksdb-3-0-release.markdown new file mode 100644 index 0000000..61c90dc --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2014-05-19-rocksdb-3-0-release.markdown @@ -0,0 +1,24 @@ +--- +title: RocksDB 3.0 release +layout: post +author: icanadi +category: blog +redirect_from: + - /blog/557/rocksdb-3-0-release/ +--- + +Check out new RocksDB release on [Github](https://github.com/facebook/rocksdb/releases/tag/3.0.fb)! + +New features in RocksDB 3.0: + + * [Column Family support](https://github.com/facebook/rocksdb/wiki/Column-Families) + + + * [Ability to chose different checksum function](https://github.com/facebook/rocksdb/commit/0afc8bc29a5800e3212388c327c750d32e31f3d6) + + + * Deprecated ReadOptions::prefix_seek and ReadOptions::prefix + + + +Check out the full [change log](https://github.com/facebook/rocksdb/blob/3.0.fb/HISTORY.md). diff --git a/librocksdb-sys/rocksdb/docs/_posts/2014-05-22-rocksdb-3-1-release.markdown b/librocksdb-sys/rocksdb/docs/_posts/2014-05-22-rocksdb-3-1-release.markdown new file mode 100644 index 0000000..3015674 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2014-05-22-rocksdb-3-1-release.markdown @@ -0,0 +1,20 @@ +--- +title: RocksDB 3.1 release +layout: post +author: icanadi +category: blog +redirect_from: + - /blog/575/rocksdb-3-1-release/ +--- + +Check out the new release on [Github](https://github.com/facebook/rocksdb/releases/tag/rocksdb-3.1)! + +New features in RocksDB 3.1: + + * [Materialized hash index](https://github.com/facebook/rocksdb/commit/0b3d03d026a7248e438341264b4c6df339edc1d7) + + + * [FIFO compaction style](https://github.com/facebook/rocksdb/wiki/FIFO-compaction-style) + + +We released 3.1 so fast after 3.0 because one of our internal customers needed materialized hash index. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2014-06-23-plaintable-a-new-file-format.markdown b/librocksdb-sys/rocksdb/docs/_posts/2014-06-23-plaintable-a-new-file-format.markdown new file mode 100644 index 0000000..6a641f2 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2014-06-23-plaintable-a-new-file-format.markdown @@ -0,0 +1,47 @@ +--- +title: PlainTable — A New File Format +layout: post +author: sdong +category: blog +redirect_from: + - /blog/599/plaintable-a-new-file-format/ +--- + +In this post, we are introducing "PlainTable" -- a file format we designed for RocksDB, initially to satisfy a production use case at Facebook. + +Design goals: + +1. All data stored in memory, in files stored in tmpfs/ramfs. Support DBs larger than 100GB (may be sharded across multiple RocksDB instance). +1. Optimize for [prefix hashing](https://github.com/facebook/rocksdb/raw/gh-pages/talks/2014-03-27-RocksDB-Meetup-Siying-Prefix-Hash.pdf) +1. Less than or around 1 micro-second average latency for single Get() or Seek(). +1. Minimize memory consumption. +1. Queries efficiently return empty results + + + +Notice that our priority was not to maximize query performance, but to strike a balance between query performance and memory consumption. PlainTable query performance is not as good as you would see with a nicely-designed hash table, but they are of the same order of magnitude, while keeping memory overhead to a minimum. + +Since we are targeting micro-second latency, it is on the level of the number of CPU cache misses (if they cannot be parallellized, which are usually the case for index look-ups). On our target hardware with Intel CPUs of multiple sockets with NUMA, we can only allow 4-5 CPU cache misses (including costs of data TLB). + +To meet our requirements, given that only hash prefix iterating is needed, we made two decisions: + +1. to use a hash index, which is +1. directly addressed to rows, with no block structure. + +Having addressed our latency goal, the next task was to design a very compact hash index to minimize memory consumption. Some tricks we used to meet this goal: + +1. We only use 32-bit integers for data and index offsets.The first bit serves as a flag, so we can avoid using 8-byte pointers. +1. We never copy keys or parts of keys to index search structures. We store only offsets from which keys can be retrieved, to make comparisons with search keys. +1. Since our file is immutable, we can accurately estimate the number of hash buckets needed. + +To make sure the format works efficiently with empty queries, we added a bloom filter check before the query. This adds only one cache miss for non-empty cases [1], but avoids multiple cache misses for most empty results queries. This is a good trade-off for use cases with a large percentage of empty results. + +These are the design goals and basic ideas of PlainTable file format. For detailed information, see [this wiki page](https://github.com/facebook/rocksdb/wiki/PlainTable-Format). + +[1] Bloom filter checks typically require multiple memory access. However, because they are independent, they usually do not make the CPU pipeline stale. In any case, we improved the bloom filter to improve data locality - we may cover this further in a future blog post. + +### Comments + +**[Siying Dong](siying.d@fb.com)** + +Does [http://rocksdb.org/feed/](http://rocksdb.org/feed/) work? diff --git a/librocksdb-sys/rocksdb/docs/_posts/2014-06-27-avoid-expensive-locks-in-get.markdown b/librocksdb-sys/rocksdb/docs/_posts/2014-06-27-avoid-expensive-locks-in-get.markdown new file mode 100644 index 0000000..4411c7a --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2014-06-27-avoid-expensive-locks-in-get.markdown @@ -0,0 +1,89 @@ +--- +title: Avoid Expensive Locks in Get() +layout: post +author: leijin +category: blog +redirect_from: + - /blog/677/avoid-expensive-locks-in-get/ +--- + +As promised in the previous [blog post](blog/2014/05/14/lock.html)! + +RocksDB employs a multiversion concurrency control strategy. Before reading data, it needs to grab the current version, which is encapsulated in a data structure called [SuperVersion](https://reviews.facebook.net/rROCKSDB1fdb3f7dc60e96394e3e5b69a46ede5d67fb976c). + + + +At the beginning of `GetImpl()`, it used to do this: + + + mutex_.Lock(); + auto* s = super_version_->Ref(); + mutex_.Unlock(); + + +The lock is necessary because pointer super_version_ may be updated, the corresponding SuperVersion may be deleted while Ref() is in progress. + + +`Ref()` simply increases the reference counter and returns “this” pointer. However, this simple operation posed big challenges for in-memory workload and stopped RocksDB from scaling read throughput beyond 8 cores. Running 32 read threads on a 32-core CPU leads to [70% system CPU usage](https://github.com/facebook/rocksdb/raw/gh-pages/talks/2014-03-27-RocksDB-Meetup-Lei-Lockless-Get.pdf). This is outrageous! + + + + +Luckily, we found a way to circumvent this problem by using [thread local storage](http://en.wikipedia.org/wiki/Thread-local_storage). Version change is a rare event comparable to millions of read requests. On the very first Get() request, each thread pays the mutex cost to acquire a reference to the new super version. Instead of releasing the reference after use, the reference is cached in thread’s local storage. An atomic variable is used to track global super version number. Subsequent reads simply compare the local super version number against the global super version number. If they are the same, the cached super version reference may be used directly, at no cost. If a version change is detected, mutex must be acquired to update the reference. The cost of mutex lock is amortized among millions of reads and becomes negligible. + + + + +The code looks something like this: + + + + + + SuperVersion* s = thread_local_->Get(); + if (s->version_number != super_version_number_.load()) { + // slow path, cleanup of current super version is omitted + mutex_.Lock(); + s = super_version_->Ref(); + mutex_.Unlock(); + } + + + + +The result is quite amazing. RocksDB can nicely [scale to 32 cores](https://github.com/facebook/rocksdb/raw/gh-pages/talks/2014-03-27-RocksDB-Meetup-Lei-Lockless-Get.pdf) and most CPU time is spent in user land. + + + + +Daryl Grove gives a pretty good [comparison between mutex and atomic](https://blogs.oracle.com/d/entry/the_cost_of_mutexes). However, the real cost difference lies beyond what is shown in the assembly code. Mutex can keep threads spinning on CPU or even trigger thread context switches in which all readers compete to access the critical area. Our approach prevents mutual competition by directing threads to check against a global version which does not change at high frequency, and is therefore much more cache-friendly. + + + + +The new approach entails one issue: a thread can visit GetImpl() once but can never come back again. SuperVersion is referenced and cached in its thread local storage. All resources (e.g., memtables, files) which belong to that version are frozen. A “supervisor” is required to visit each thread’s local storage and free its resources without incurring a lock. We designed a lockless sweep using CAS (compare and switch instruction). Here is how it works: + + + + +(1) A reader thread uses CAS to acquire SuperVersion from its local storage and to put in a special flag (SuperVersion::kSVInUse). + + + + +(2) Upon completion of GetImpl(), the reader thread tries to return SuperVersion to local storage by CAS, expecting the special flag (SuperVersion::kSVInUse) in its local storage. If it does not see SuperVersion::kSVInUse, that means a “sweep” was done and the reader thread is responsible for cleanup (this is expensive, but does not happen often on the hot path). + + + + +(3) After any flush/compaction, the background thread performs a sweep (CAS) across all threads’ local storage and frees encountered SuperVersion. A reader thread must re-acquire a new SuperVersion reference on its next visit. + +### Comments + +**[David Barbour](dmbarbour@gmail.com)** + +Please post an example of reading the same rocksdb concurrently. + +We are using the latest 3.0 rocksdb; however, when two separate processes +try and open the same rocksdb for reading, only one of the open requests +succeed. The other open always fails with “db/LOCK: Resource temporarily unavailable” So far we have not found an option that allows sharing the rocksdb for reads. An example would be most appreciated. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2014-06-27-rocksdb-3-2-release.markdown b/librocksdb-sys/rocksdb/docs/_posts/2014-06-27-rocksdb-3-2-release.markdown new file mode 100644 index 0000000..e4eba6a --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2014-06-27-rocksdb-3-2-release.markdown @@ -0,0 +1,30 @@ +--- +title: RocksDB 3.2 release +layout: post +author: leijin +category: blog +redirect_from: + - /blog/647/rocksdb-3-2-release/ +--- + +Check out new RocksDB release on [GitHub](https://github.com/facebook/rocksdb/releases/tag/rocksdb-3.2)! + +New Features in RocksDB 3.2: + + * PlainTable now supports a new key encoding: for keys of the same prefix, the prefix is only written once. It can be enabled through encoding_type paramter of NewPlainTableFactory() + + + * Add AdaptiveTableFactory, which is used to convert from a DB of PlainTable to BlockBasedTabe, or vise versa. It can be created using NewAdaptiveTableFactory() + + + +Public API changes: + + + * We removed seek compaction as a concept from RocksDB + + + * Add two paramters to NewHashLinkListRepFactory() for logging on too many entries in a hash bucket when flushing + + + * Added new option BlockBasedTableOptions::hash_index_allow_collision. When enabled, prefix hash index for block-based table will not store prefix and allow hash collision, reducing memory consumption diff --git a/librocksdb-sys/rocksdb/docs/_posts/2014-07-29-rocksdb-3-3-release.markdown b/librocksdb-sys/rocksdb/docs/_posts/2014-07-29-rocksdb-3-3-release.markdown new file mode 100644 index 0000000..d858e4f --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2014-07-29-rocksdb-3-3-release.markdown @@ -0,0 +1,34 @@ +--- +title: RocksDB 3.3 Release +layout: post +author: yhciang +category: blog +redirect_from: + - /blog/1301/rocksdb-3-3-release/ +--- + +Check out new RocksDB release on [GitHub](https://github.com/facebook/rocksdb/releases/tag/rocksdb-3.3)! + +New Features in RocksDB 3.3: + + * **JSON API prototype**. + + + * **Performance improvement on HashLinkList**: We addressed performance outlier of HashLinkList caused by skewed bucket by switching data in the bucket from linked list to skip list. Add parameter threshold_use_skiplist in NewHashLinkListRepFactory(). + + + + * **More effective on storage space reclaim**: RocksDB is now able to reclaim storage space more effectively during the compaction process. This is done by compensating the size of each deletion entry by the 2X average value size, which makes compaction to be triggerred by deletion entries more easily. + + + * **TimeOut API to write**: Now WriteOptions have a variable called timeout_hint_us. With timeout_hint_us set to non-zero, any write associated with this timeout_hint_us may be aborted when it runs longer than the specified timeout_hint_us, and it is guaranteed that any write completes earlier than the specified time-out will not be aborted due to the time-out condition. + + + * **rate_limiter option**: We added an option that controls total throughput of flush and compaction. The throughput is specified in bytes/sec. Flush always has precedence over compaction when available bandwidth is constrained. + + + +Public API changes: + + + * Removed NewTotalOrderPlainTableFactory because it is not used and implemented semantically incorrect. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2014-09-12-cuckoo.markdown b/librocksdb-sys/rocksdb/docs/_posts/2014-09-12-cuckoo.markdown new file mode 100644 index 0000000..22178f7 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2014-09-12-cuckoo.markdown @@ -0,0 +1,74 @@ +--- +title: Cuckoo Hashing Table Format +layout: post +author: radheshyam +category: blog +redirect_from: + - /blog/1427/new-bloom-filter-format/ +--- + +## Introduction + +We recently introduced a new [Cuckoo Hashing](http://en.wikipedia.org/wiki/Cuckoo_hashing) based SST file format which is optimized for fast point lookups. The new format was built for applications which require very high point lookup rates (~4Mqps) in read only mode but do not use operations like range scan, merge operator, etc. But, the existing RocksDB file formats were built to support range scan and other operations and the current best point lookup in RocksDB is 1.2 Mqps given by [PlainTable](https://github.com/facebook/rocksdb/wiki/PlainTable-Format)[ format](https://github.com/facebook/rocksdb/wiki/PlainTable-Format). This prompted a hashing based file format, which we present here. The new table format uses a cache friendly version of Cuckoo Hashing algorithm with only 1 or 2 memory accesses per lookup. + + + +Goals: + + * Reduce memory accesses per lookup to 1 or 2 + + + * Get an end to end point lookup rate of at least 4 Mqps + + + * Minimize database size + + +Assumptions: + + * Key length and value length are fixed + + + * The database is operated in read only mode + + +Non-goals: + + + * While optimizing the performance of Get() operation was our primary goal, compaction and build times were secondary. We may work on improving them in future. + + +Details for setting up the table format can be found in [GitHub](https://github.com/facebook/rocksdb/wiki/CuckooTable-Format). + + +## Cuckoo Hashing Algorithm + +In order to achieve high lookup speeds, we did multiple optimizations, including a cache friendly cuckoo hash algorithm. Cuckoo Hashing uses multiple hash functions, _h1, ..., __hn._ + +### Original Cuckoo Hashing + +To insert any new key _k_, we compute hashes of the key _h1(k), ..., __hn__(k)_. We insert the key in the first hash location that is free. If all the locations are blocked, we try to move one of the colliding keys to a different location by trying to re-insert it. + +Finding smallest set of keys to displace in order to accommodate the new key is naturally a shortest path problem in a directed graph where nodes are buckets of hash table and there is an edge from bucket _A_ to bucket _B_ if the element stored in bucket _A_ can be accommodated in bucket _B_ using one of the hash functions. The source nodes are the possible hash locations for the given key _k_ and destination is any one of the empty buckets. We use this algorithm to handle collision. + +To retrieve a key _k_, we compute hashes, _h1(k), ..., __hn__(k)_ and the key must be present in one of these locations. + +Our goal is to minimize average (and maximum) number of hash functions required and hence the number of memory accesses. In our experiments, with a hash utilization of 90%, we found that the average number of lookups is 1.8 and maximum is 3. Around 44% of keys are accommodated in first hash location and 33% in second location. + + +### Cache Friendly Cuckoo Hashing + +We noticed the following two sub-optimal properties in original Cuckoo implementation: + + + * If the key is not present in first hash location, we jump to second hash location which may not be in cache. This results in many cache misses. + + + * Because only 44% of keys are located in first cuckoo block, we couldn't have an optimal prefetching strategy - prefetching all hash locations for a key is wasteful. But prefetching only the first hash location helps only 44% of cases. + + + +The solution is to insert more keys near first location. In case of collision in the first hash location - _h1(k)_, we try to insert it in next few buckets, _h1(k)+1, _h1(k)+2, _..., h1(k)+t-1_. If all of these _t_ locations are occupied, we skip over to next hash function _h2_ and repeat the process. We call the set of _t_ buckets as a _Cuckoo Block_. We chose _t_ such that size of a block is not bigger than a cache line and we prefetch the first cuckoo block. + + +With the new algorithm, for 90% hash utilization, we found that 85% of keys are accommodated in first Cuckoo Block. Prefetching the first cuckoo block yields best results. For a database of 100 million keys with key length 8 and value length 4, the hash algorithm alone can achieve 9.6 Mqps and we are working on improving it further. End to end RocksDB performance results can be found [here](https://github.com/facebook/rocksdb/wiki/CuckooTable-Format). diff --git a/librocksdb-sys/rocksdb/docs/_posts/2014-09-12-new-bloom-filter-format.markdown b/librocksdb-sys/rocksdb/docs/_posts/2014-09-12-new-bloom-filter-format.markdown new file mode 100644 index 0000000..96fa50a --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2014-09-12-new-bloom-filter-format.markdown @@ -0,0 +1,52 @@ +--- +title: New Bloom Filter Format +layout: post +author: zagfox +category: blog +redirect_from: + - /blog/1367/cuckoo/ +--- + +## Introduction + +In this post, we are introducing "full filter block" --- a new bloom filter format for [block based table](https://github.com/facebook/rocksdb/wiki/Rocksdb-BlockBasedTable-Format). This could bring about 40% of improvement for key query under in-memory (all data stored in memory, files stored in tmpfs/ramfs, an [example](https://github.com/facebook/rocksdb/wiki/RocksDB-In-Memory-Workload-Performance-Benchmarks) workload. The main idea behind is to generate a big filter that covers all the keys in SST file to avoid lots of unnecessary memory look ups. + + + + +## What is Bloom Filter + +In brief, [bloom filter](https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter) is a bits array generated for a set of keys that could tell if an arbitrary key may exist in that set. + +In RocksDB, we generate such a bloom filter for each SST file. When we conduct a query for a key, we first goes to the bloom filter block of SST file. If key may exist in filter, we goes into data block in SST file to search for the key. If not, we would return directly. So it could help speed up point look up operation a lot. + +## Original Bloom Filter Format + +Original bloom filter creates filters for each individual data block in SST file. It has complex structure (ref [here](https://github.com/facebook/rocksdb/wiki/Rocksdb-BlockBasedTable-Format#filter-meta-block)) which results in a lot of non-adjacent memory look ups. + +Here's the work flow for checking original bloom filter in block based table: + +1. Given the target key, we goes to the index block to get the "data block ID" where this key may reside. +1. Using the "data block ID", we goes to the filter block and get the correct "offset of filter". +1. Using the "offset of filter", we goes to the actual filter and do the checking. + +## New Bloom Filter Format + +New bloom filter creates filter for all keys in SST file and we name it "full filter". The data structure of full filter is very simple, there is just one big filter: + +    [ full filter ] + +In this way, the work flow of bloom filter checking is much simplified. + +(1) Given the target key, we goes directly to the filter block and conduct the filter checking. + +To be specific, there would be no checking for index block and no address jumping inside of filter block. + +Though it is a big filter, the total filter size would be the same as the original filter. + +One little draw back is that the new bloom filter introduces more memory consumption when building SST file because we need to buffer keys (or their hashes) before generating filter. Original filter just creates a bunch of small filters so it just buffer a small amount of keys. For full filter, we buffer hashes of all keys, which would take more memory when SST file size increases. + + +## Usage & Customization + +You can refer to the document here for [usage](https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter#usage-of-new-bloom-filter) and [customization](https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter#customize-your-own-filterpolicy). diff --git a/librocksdb-sys/rocksdb/docs/_posts/2014-09-15-rocksdb-3-5-release.markdown b/librocksdb-sys/rocksdb/docs/_posts/2014-09-15-rocksdb-3-5-release.markdown new file mode 100644 index 0000000..1878a5a --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2014-09-15-rocksdb-3-5-release.markdown @@ -0,0 +1,38 @@ +--- +title: RocksDB 3.5 Release! +layout: post +author: leijin +category: blog +redirect_from: + - /blog/1547/rocksdb-3-5-release/ +--- + +New RocksDB release - 3.5! + + +**New Features** + + + 1. Add include/utilities/write_batch_with_index.h, providing a utility class to query data out of WriteBatch when building it. + + + 2. new ReadOptions.total_order_seek to force total order seek when block-based table is built with hash index. + + + +**Public API changes** + + + 1. The Prefix Extractor used with V2 compaction filters is now passed user key to SliceTransform::Transform instead of unparsed RocksDB key. + + + 2. Move BlockBasedTable related options to BlockBasedTableOptions from Options. Change corresponding JNI interface. Options affected include: no_block_cache, block_cache, block_cache_compressed, block_size, block_size_deviation, block_restart_interval, filter_policy, whole_key_filtering. filter_policy is changed to shared_ptr from a raw pointer. + + + 3. Remove deprecated options: disable_seek_compaction and db_stats_log_interval + + + 4. OptimizeForPointLookup() takes one parameter for block cache size. It now builds hash index, bloom filter, and block cache. + + +[https://github.com/facebook/rocksdb/releases/tag/v3.5](https://github.com/facebook/rocksdb/releases/tag/rocksdb-3.5) diff --git a/librocksdb-sys/rocksdb/docs/_posts/2015-01-16-migrating-from-leveldb-to-rocksdb-2.markdown b/librocksdb-sys/rocksdb/docs/_posts/2015-01-16-migrating-from-leveldb-to-rocksdb-2.markdown new file mode 100644 index 0000000..f18de0b --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2015-01-16-migrating-from-leveldb-to-rocksdb-2.markdown @@ -0,0 +1,112 @@ +--- +title: Migrating from LevelDB to RocksDB +layout: post +author: lgalanis +category: blog +redirect_from: + - /blog/1811/migrating-from-leveldb-to-rocksdb-2/ +--- + +If you have an existing application that uses LevelDB and would like to migrate to using RocksDB, one problem you need to overcome is to map the options for LevelDB to proper options for RocksDB. As of release 3.9 this can be automatically done by using our option conversion utility found in rocksdb/utilities/leveldb_options.h. What is needed, is to first replace `leveldb::Options` with `rocksdb::LevelDBOptions`. Then, use `rocksdb::ConvertOptions( )` to convert the `LevelDBOptions` struct into appropriate RocksDB options. Here is an example: + + + +LevelDB code: + +```c++ +#include +#include "leveldb/db.h" + +using namespace leveldb; + +int main(int argc, char** argv) { + DB *db; + + Options opt; + opt.create_if_missing = true; + opt.max_open_files = 1000; + opt.block_size = 4096; + + Status s = DB::Open(opt, "/tmp/mydb", &db); + + delete db; +} +``` + +RocksDB code: + +```c++ +#include +#include "rocksdb/db.h" +#include "rocksdb/utilities/leveldb_options.h" + +using namespace rocksdb; + +int main(int argc, char** argv) { + DB *db; + + LevelDBOptions opt; + opt.create_if_missing = true; + opt.max_open_files = 1000; + opt.block_size = 4096; + + Options rocksdb_options = ConvertOptions(opt); + // add rocksdb specific options here + + Status s = DB::Open(rocksdb_options, "/tmp/mydb_rocks", &db); + + delete db; +} +``` + +The difference is: + +```diff +-#include "leveldb/db.h" ++#include "rocksdb/db.h" ++#include "rocksdb/utilities/leveldb_options.h" + +-using namespace leveldb; ++using namespace rocksdb; + +- Options opt; ++ LevelDBOptions opt; + +- Status s = DB::Open(opt, "/tmp/mydb", &db); ++ Options rocksdb_options = ConvertOptions(opt); ++ // add rockdb specific options here ++ ++ Status s = DB::Open(rocksdb_options, "/tmp/mydb_rocks", &db); +``` + +Once you get up and running with RocksDB you can then focus on tuning RocksDB further by modifying the converted options struct. + +The reason why ConvertOptions is handy is because a lot of individual options in RocksDB have moved to other structures in different components. For example, block_size is not available in struct rocksdb::Options. It resides in struct rocksdb::BlockBasedTableOptions, which is used to create a TableFactory object that RocksDB uses internally to create the proper TableBuilder objects. If you were to write your application from scratch it would look like this: + +RocksDB code from scratch: + +```c++ +#include +#include "rocksdb/db.h" +#include "rocksdb/table.h" + +using namespace rocksdb; + +int main(int argc, char** argv) { + DB *db; + + Options opt; + opt.create_if_missing = true; + opt.max_open_files = 1000; + + BlockBasedTableOptions topt; + topt.block_size = 4096; + opt.table_factory.reset(NewBlockBasedTableFactory(topt)); + + Status s = DB::Open(opt, "/tmp/mydb_rocks", &db); + + delete db; +} +``` + +The LevelDBOptions utility can ease migration to RocksDB from LevelDB and allows us to break down the various options across classes as it is needed. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2015-02-24-reading-rocksdb-options-from-a-file.markdown b/librocksdb-sys/rocksdb/docs/_posts/2015-02-24-reading-rocksdb-options-from-a-file.markdown new file mode 100644 index 0000000..cddc0dd --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2015-02-24-reading-rocksdb-options-from-a-file.markdown @@ -0,0 +1,41 @@ +--- +title: Reading RocksDB options from a file +layout: post +author: lgalanis +category: blog +redirect_from: + - /blog/1883/reading-rocksdb-options-from-a-file/ +--- + +RocksDB options can be provided using a file or any string to RocksDB. The format is straightforward: `write_buffer_size=1024;max_write_buffer_number=2`. Any whitespace around `=` and `;` is OK. Moreover, options can be nested as necessary. For example `BlockBasedTableOptions` can be nested as follows: `write_buffer_size=1024; max_write_buffer_number=2; block_based_table_factory={block_size=4k};`. Similarly any white space around `{` or `}` is ok. Here is what it looks like in code: + + + +```c++ +#include +#include "rocksdb/db.h" +#include "rocksdb/table.h" +#include "rocksdb/utilities/convenience.h" + +using namespace rocksdb; + +int main(int argc, char** argv) { + DB *db; + + Options opt; + + std::string options_string = + "create_if_missing=true;max_open_files=1000;" + "block_based_table_factory={block_size=4096}"; + + Status s = GetDBOptionsFromString(opt, options_string, &opt); + + s = DB::Open(opt, "/tmp/mydb_rocks", &db); + + // use db + + delete db; +} +``` + +Using `GetDBOptionsFromString` is a convenient way of changing options for your RocksDB application without needing to resort to recompilation or tedious command line parsing. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2015-02-27-write-batch-with-index.markdown b/librocksdb-sys/rocksdb/docs/_posts/2015-02-27-write-batch-with-index.markdown new file mode 100644 index 0000000..7f9f776 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2015-02-27-write-batch-with-index.markdown @@ -0,0 +1,20 @@ +--- +title: 'WriteBatchWithIndex: Utility for Implementing Read-Your-Own-Writes' +layout: post +author: sdong +category: blog +redirect_from: + - /blog/1901/write-batch-with-index/ +--- + +RocksDB can be used as a storage engine of a higher level database. In fact, we are currently plugging RocksDB into MySQL and MongoDB as one of their storage engines. RocksDB can help with guaranteeing some of the ACID properties: durability is guaranteed by RocksDB by design; while consistency and isolation need to be enforced by concurrency controls on top of RocksDB; Atomicity can be implemented by committing a transaction's writes with one write batch to RocksDB in the end. + + + +However, if we enforce atomicity by only committing all writes in the end of the transaction in one batch, you cannot get the updated value from RocksDB previously written by the same transaction (read-your-own-write). To read the updated value, the databases on top of RocksDB need to maintain an internal buffer for all the written keys, and when a read happens they need to merge the result from RocksDB and from this buffer. This is a problem we faced when building the RocksDB storage engine in MongoDB. We solved it by creating a utility class, WriteBatchWithIndex (a write batch with a searchable index) and made it part of public API so that the community can also benefit from it. + +Before talking about the index part, let me introduce write batch first. The write batch class, `WriteBatch`, is a RocksDB data structure for atomic writes of multiple keys. Users can buffer their updates to a `WriteBatch` by calling `write_batch.Put("key1", "value1")` or `write_batch.Delete("key2")`, similar as calling RocksDB's functions of the same names. In the end, they call `db->Write(write_batch)` to atomically update all those batched operations to the DB. It is how a database can guarantee atomicity, as shown above. Adding a searchable index to `WriteBatch`, we now have `WriteBatchWithIndex`. Users can put updates to WriteBatchIndex in the same way as to `WriteBatch`. In the end, users can get a `WriteBatch` object from it and issue `db->Write()`. Additionally, users can create an iterator of a WriteBatchWithIndex, seek to any key location and iterate from there. + +To implement read-your-own-write using `WriteBatchWithIndex`, every time the user creates a transaction, we create a `WriteBatchWithIndex` attached to it. All the writes of the transaction go to the `WriteBatchWithIndex` first. When we commit the transaction, we atomically write the batch to RocksDB. When the user wants to call `Get()`, we first check if the value exists in the `WriteBatchWithIndex` and return the value if existing, by seeking and reading from an iterator of the write batch, before checking data in RocksDB. For example, here is the we implement it in MongoDB's RocksDB storage engine: [link](https://github.com/mongodb/mongo/blob/a31cc114a89a3645e97645805ba77db32c433dce/src/mongo/db/storage/rocks/rocks_recovery_unit.cpp#L245-L260). If a range query comes, we pass a DB's iterator to `WriteBatchWithIndex`, which creates a super iterator which combines the results from the DB iterator with the batch's iterator. Using this super iterator, we can iterate the DB with the transaction's own writes. Here is the iterator creation codes in MongoDB's RocksDB storage engine: [link](https://github.com/mongodb/mongo/blob/a31cc114a89a3645e97645805ba77db32c433dce/src/mongo/db/storage/rocks/rocks_recovery_unit.cpp#L266-L269). In this way, the database can solve the read-your-own-write problem by using RocksDB to handle a transaction's uncommitted writes. + +Using `WriteBatchWithIndex`, we successfully implemented read-your-own-writes in the RocksDB storage engine of MongoDB. If you also have a read-your-own-write problem, `WriteBatchWithIndex` can help you implement it quickly and correctly. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2015-04-22-integrating-rocksdb-with-mongodb-2.markdown b/librocksdb-sys/rocksdb/docs/_posts/2015-04-22-integrating-rocksdb-with-mongodb-2.markdown new file mode 100644 index 0000000..1ffe2c5 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2015-04-22-integrating-rocksdb-with-mongodb-2.markdown @@ -0,0 +1,16 @@ +--- +title: Integrating RocksDB with MongoDB +layout: post +author: icanadi +category: blog +redirect_from: + - /blog/1967/integrating-rocksdb-with-mongodb-2/ +--- + +Over the last couple of years, we have been busy integrating RocksDB with various services here at Facebook that needed to store key-value pairs locally. We have also seen other companies using RocksDB as local storage components of their distributed systems. + + + +The next big challenge for us is to bring RocksDB storage engine to general purpose databases. Today we have an exciting milestone to share with our community! We're running MongoDB with RocksDB in production and seeing great results! You can read more about it here: [http://blog.parse.com/announcements/mongodb-rocksdb-parse/](http://blog.parse.com/announcements/mongodb-rocksdb-parse/) + +Keep tuned for benchmarks and more stability and performance improvements. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2015-06-12-rocksdb-in-osquery.markdown b/librocksdb-sys/rocksdb/docs/_posts/2015-06-12-rocksdb-in-osquery.markdown new file mode 100644 index 0000000..f3a55fa --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2015-06-12-rocksdb-in-osquery.markdown @@ -0,0 +1,10 @@ +--- +title: RocksDB in osquery +layout: post +author: icanadi +category: lgalanis +redirect_from: + - /blog/1997/rocksdb-in-osquery/ +--- + +Check out [this](https://code.facebook.com/posts/1411870269134471/how-rocksdb-is-used-in-osquery/) blog post by [Mike Arpaia](https://www.facebook.com/mike.arpaia) and [Ted Reed](https://www.facebook.com/treeded) about how osquery leverages RocksDB to build an embedded pub-sub system. This article is a great read and contains insights on how to properly use RocksDB. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2015-07-15-rocksdb-2015-h2-roadmap.markdown b/librocksdb-sys/rocksdb/docs/_posts/2015-07-15-rocksdb-2015-h2-roadmap.markdown new file mode 100644 index 0000000..b3e2703 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2015-07-15-rocksdb-2015-h2-roadmap.markdown @@ -0,0 +1,92 @@ +--- +title: RocksDB 2015 H2 roadmap +layout: post +author: icanadi +category: blog +redirect_from: + - /blog/2015/rocksdb-2015-h2-roadmap/ +--- + +Every 6 months, RocksDB team gets together to prioritize the work ahead of us. We just went through this exercise and we wanted to share the results with the community. Here's what RocksDB team will be focusing on for the next 6 months: + + + +**MyRocks** + +As you might know, we're working hard to integrate RocksDB as a storage engine for MySQL. This project is pretty important for us because we're heavy users of MySQL. We're already getting pretty good performance results, but there is more work to be done. We need to focus on both performance and stability. The most high priority items on are list are: + + + + + 1. Reduce CPU costs of RocksDB as a MySQL storage engine + + + 2. Implement pessimistic concurrency control to support repeatable read isolation level in MyRocks + + + 3. Reduce P99 read latency, which is high mostly because of lingering tombstones + + + 4. Port ZSTD compression + + +**MongoRocks** + +Another database that we're working on is MongoDB. The project of integrating MongoDB with RocksDB storage engine is called MongoRocks. It's already running in production at Parse [1] and we're seeing surprisingly few issues. Our plans for the next half: + + + + + 1. Keep improving performance and stability, possibly reuse work done on MyRocks (workloads are pretty similar). + + + 2. Increase internal and external adoption. + + + 3. Support new MongoDB 3.2. + + +**RocksDB on cheaper storage media** + +Up to now, our mission was to build the best key-value store “for fast storage” (flash and in-memory). However, there are some use-cases at Facebook that don't need expensive high-end storage. In the next six months, we plan to deploy RocksDB on cheaper storage media. We will optimize performance to RocksDB on either or both: + + + + + 1. Hard drive storage array. + + + 2. Tiered Storage. + + +**Quality of Service** + +When talking to our customers, there are couple of issues that keep reoccurring. We need to fix them to make our customers happy. We will improve RocksDB to provide better assurance of performance and resource usage. Non-exhaustive list includes: + + + + + 1. Iterate P99 can be high due to the presence of tombstones. + + + 2. Write stalls can happen during high write loads. + + + 3. Better control of memory and disk usage. + + + 4. Service quality and performance of backup engine. + + +**Operation's user experience** + +As we increase deployment of RocksDB, engineers are spending more time on debugging RocksDB issues. We plan to improve user experience when running RocksDB. The goal is to reduce TTD (time-to-debug). The work includes monitoring, visualizations and documentations. + +[1]( http://blog.parse.com/announcements/mongodb-rocksdb-parse/](http://blog.parse.com/announcements/mongodb-rocksdb-parse/) + + +### Comments + +**[Mike](allspace2012@outlook.com)** + +What’s the status of this roadmap? “RocksDB on cheaper storage media”, has this been implemented? diff --git a/librocksdb-sys/rocksdb/docs/_posts/2015-07-17-spatial-indexing-in-rocksdb.markdown b/librocksdb-sys/rocksdb/docs/_posts/2015-07-17-spatial-indexing-in-rocksdb.markdown new file mode 100644 index 0000000..53c1f5a --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2015-07-17-spatial-indexing-in-rocksdb.markdown @@ -0,0 +1,78 @@ +--- +title: Spatial indexing in RocksDB +layout: post +author: icanadi +category: blog +redirect_from: + - /blog/2039/spatial-indexing-in-rocksdb/ +--- + +About a year ago, there was a need to develop a spatial database at Facebook. We needed to store and index Earth's map data. Before building our own, we looked at the existing spatial databases. They were all very good technology, but also general purpose. We could sacrifice a general-purpose API, so we thought we could build a more performant database, since it would be specifically designed for our use-case. Furthermore, we decided to build the spatial database on top of RocksDB, because we have a lot of operational experience with running and tuning RocksDB at a large scale. + + + +When we started looking at this project, the first thing that surprised us was that our planet is not that big. Earth's entire map data can fit in memory on a reasonably high-end machine. Thus, we also decided to build a spatial database optimized for memory-resident dataset. + +The first use-case of our spatial database was an experimental map renderer. As part of our project, we successfully loaded [Open Street Maps](https://www.openstreetmap.org/) dataset and hooked it up with [Mapnik](http://mapnik.org/), a map rendering engine. + +The usual Mapnik workflow is to load the map data into a SQL-based database and then define map layers with SQL statements. To render a tile, Mapnik needs to execute a couple of SQL queries. The benefit of this approach is that you don't need to reload your database when you change your map style. You can just change your SQL query and Mapnik picks it up. In our model, we decided to precompute the features we need for each tile. We need to know the map style before we create the database. However, when rendering the map tile, we only fetch the features that we need to render. + +We haven't open sourced the RocksDB Mapnik plugin or the database loading pipeline. However, the spatial indexing is available in RocksDB under a name [SpatialDB](https://github.com/facebook/rocksdb/blob/main/include/rocksdb/utilities/spatial_db.h). The API is focused on map rendering use-case, but we hope that it can also be used for other spatial-based applications. + +Let's take a tour of the API. When you create a spatial database, you specify the spatial indexes that need to be built. Each spatial index is defined by a bounding box and granularity. For map rendering, we create a spatial index for each zoom levels. Higher zoom levels have more granularity. + + + + SpatialDB::Create( + SpatialDBOptions(), + "/data/map", { + SpatialIndexOptions("zoom10", BoundingBox(0, 0, 100, 100), 10), + SpatialIndexOptions("zoom16", BoundingBox(0, 0, 100, 100), 16) + } + ); + + + + +When you insert a feature (building, street, country border) into SpatialDB, you need to specify the list of spatial indexes that will index the feature. In the loading phase we process the map style to determine the list of zoom levels on which we'll render the feature. For example, we will not render the building on zoom level that shows an entire country. Building will only be indexed on higher zoom level's index. Country borders will be indexes on all zoom levels. + + + + FeatureSet feature; + feature.Set("type", "building"); + feature.Set("height", 6); + db->Insert(WriteOptions(), BoundingBox(5, 5, 10, 10), + well_known_binary_blob, feature, {"zoom16"}); + + + + +The indexing part is pretty simple. For each feature, we first find a list of index tiles that it intersects. Then, we add a link from the tile's [quad key](https://msdn.microsoft.com/en-us/library/bb259689.aspx) to the feature's primary key. Using quad keys improves data locality, i.e. features closer together geographically will have similar quad keys. Even though we're optimizing for a memory-resident dataset, data locality is still very important due to different caching effects. + +After you're done inserting all the features, you can call an API Compact() that will compact the dataset and speed up read queries. + + + + db->Compact(); + + + + +SpatialDB's query specifies: 1) bounding box we're interested in, and 2) a zoom level. We find all tiles that intersect with the query's bounding box and return all features in those tiles. + + + + + Cursor* c = db_->Query(ReadOptions(), BoundingBox(1, 1, 7, 7), "zoom16"); + for (c->Valid(); c->Next()) { + Render(c->blob(), c->feature_set()); + } + + + + +Note: `Render()` function is not part of RocksDB. You will need to use one of many open source map renderers, for example check out [Mapnik](http://mapnik.org/). + +TL;DR If you need an embedded spatial database, check out RocksDB's SpatialDB. [Let us know](https://www.facebook.com/groups/rocksdb.dev/) how we can make it better. + +If you're interested in learning more, check out this [talk](https://www.youtube.com/watch?v=T1jWsDMONM8). diff --git a/librocksdb-sys/rocksdb/docs/_posts/2015-07-22-rocksdb-is-now-available-in-windows-platform.markdown b/librocksdb-sys/rocksdb/docs/_posts/2015-07-22-rocksdb-is-now-available-in-windows-platform.markdown new file mode 100644 index 0000000..b6bb47d --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2015-07-22-rocksdb-is-now-available-in-windows-platform.markdown @@ -0,0 +1,30 @@ +--- +title: RocksDB is now available in Windows Platform +layout: post +author: dmitrism +category: blog +redirect_from: + - /blog/2033/rocksdb-is-now-available-in-windows-platform/ +--- + +Over the past 6 months we have seen a number of use cases where RocksDB is successfully used by the community and various companies to achieve high throughput and volume in a modern server environment. + +We at Microsoft Bing could not be left behind. As a result we are happy to [announce](http://bit.ly/1OmWBT9) the availability of the Windows Port created here at Microsoft which we intend to use as a storage option for one of our key/value data stores. + + + +We are happy to make this available for the community. Keep tuned for more announcements to come. + +### Comments + +**[Siying Dong](siying.d@fb.com)** + +Appreciate your contributions to RocksDB project! I believe it will benefits many users! + +**[empresas sevilla](oxofkx@gmail.com)** + +Magnifico artículo|, un placer leer el blog + +**[jak usunac](tomogedac@o2.pl)** + +I believe it will benefits too diff --git a/librocksdb-sys/rocksdb/docs/_posts/2015-07-23-dynamic-level.markdown b/librocksdb-sys/rocksdb/docs/_posts/2015-07-23-dynamic-level.markdown new file mode 100644 index 0000000..0ff3a05 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2015-07-23-dynamic-level.markdown @@ -0,0 +1,29 @@ +--- +title: Dynamic Level Size for Level-Based Compaction +layout: post +author: sdong +category: blog +redirect_from: + - /blog/2207/dynamic-level/ +--- + +In this article, we follow up on the first part of an answer to one of the questions in our [AMA](https://www.reddit.com/r/IAmA/comments/3de3cv/we_are_rocksdb_engineering_team_ask_us_anything/ct4a8tb), the dynamic level size in level-based compaction. + + + +Level-based compaction is the original LevelDB compaction style and one of the two major compaction styles in RocksDB (See [our wiki](https://github.com/facebook/rocksdb/wiki/RocksDB-Basics#multi-threaded-compactions)). In RocksDB we introduced parallelism and more configurable options to it but the main algorithm stayed the same, until we recently introduced the dynamic level size mode. + + +In level-based compaction, we organize data to different sorted runs, called levels. Each level has a target size.  Usually target size of levels increases by the same size multiplier. For example, you can set target size of level 1 to be 1GB, and size multiplier to be 10, and the target size of level 1, 2, 3, 4 will be 1GB, 10GB, 100GB and 1000GB. Before level 1, there will be some staging file flushed from mem tables, called Level 0 files, which will later be merged to level 1. Compactions will be triggered as soon as actual size of a level exceeds its target size. We will merge a subset of data of that level to next level, to reduce size of the level. More compactions will be triggered until sizes of all the levels are lower than their target sizes. In a steady state, the size of each level will be around the same size of the size of level targets. + + +Level-based compaction’s advantage is its good space efficiency. We usually use the metric space amplification to measure the space efficiency. In this article ignore the effects of data compression so space amplification= size_on_file_system / size_of_user_data. + + +How do we estimate space amplification of level-based compaction? We focus specifically on the databases in steady state, which means database size is stable or grows slowly over time. This means updates will add roughly the same or little more data than what is removed by deletes. Given that, if we compact all the data all to the last level, the size of level will be equal as the size of last level before the compaction. On the other hand, the size of user data will be approximately the size of DB if we compact all the levels down to the last level. So the size of the last level will be a good estimation of user data size. So total size of the DB divided by the size of the last level will be a good estimation of space amplification. + + +Applying the equation, if we have four non-zero levels, their sizes are 1GB, 10GB, 100GB, 1000GB, the size amplification will be approximately (1000GB + 100GB + 10GB + 1GB) / 1000GB = 1.111, which is a very good number. However, there is a catch here: how to make sure the last level’s size is 1000GB, the same as the level’s size target? A user has to fine tune level sizes to achieve this number and will need to re-tune if DB size changes. The theoretic number 1.11 is hard to achieve in practice. In a worse case, if you have the target size of last level to be 1000GB but the user data is only 200GB, then the actual space amplification will be (200GB + 100GB + 10GB + 1GB) / 200GB = 1.555, a much worse number. + + +To solve this problem, my colleague Igor Kabiljo came up with a solution of dynamic level size target mode. You can enable it by setting options.level_compaction_dynamic_level_bytes=true. In this mode, size target of levels are changed dynamically based on size of the last level. Suppose the level size multiplier to be 10, and the DB size is 200GB. The target size of the last level is automatically set to be the actual size of the level, which is 200GB, the second to last level’s size target will be automatically set to be size_last_level / 10 = 20GB, the third last level’s will be size_last_level/100 = 2GB, and next level to be size_last_level/1000 = 200MB. We stop here because 200MB is within the range of the first level. In this way, we can achieve the 1.111 space amplification, without fine tuning of the level size targets. More details can be found in [code comments of the option](https://github.com/facebook/rocksdb/blob/v3.11/include/rocksdb/options.h#L366-L423) in the header file. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2015-10-27-getthreadlist.markdown b/librocksdb-sys/rocksdb/docs/_posts/2015-10-27-getthreadlist.markdown new file mode 100644 index 0000000..92f743a --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2015-10-27-getthreadlist.markdown @@ -0,0 +1,193 @@ +--- +title: GetThreadList +layout: post +author: yhciang +category: blog +redirect_from: + - /blog/2261/getthreadlist/ +--- + +We recently added a new API, called `GetThreadList()`, that exposes the RocksDB background thread activity. With this feature, developers will be able to obtain the real-time information about the currently running compactions and flushes such as the input / output size, elapsed time, the number of bytes it has written. Below is an example output of `GetThreadList`. To better illustrate the example, we have put a sample output of `GetThreadList` into a table where each column represents a thread status: + + + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    ThreadID +140716395198208 +140716416169728 +
    DB +db1 +db2 +
    CF +default +picachu +
    ThreadType +High Pri +Low Pri +
    Operation +Flush +Compaction +
    ElapsedTime +143.459 ms +607.538 ms +
    Stage +FlushJob::WriteLevel0Table +CompactionJob::Install +
    OperationProperties + +BytesMemtables 4092938 +BytesWritten 1050701 + +BaseInputLevel 1 +BytesRead 4876417 +BytesWritten 4140109 +IsDeletion 0 +IsManual 0 +IsTrivialMove 0 +JobID 146 +OutputLevel 2 +TotalInputBytes 4883044 +
    + +In the above output, we can see `GetThreadList()` reports the activity of two threads: one thread running flush job (middle column) and the other thread running a compaction job (right-most column). In each thread status, it shows basic information about the thread such as thread id, it's target db / column family, and the job it is currently doing and the current status of the job. For instance, we can see thread 140716416169728 is doing compaction on the `picachu` column family in database `db2`. In addition, we can see the compaction has been running for 600 ms, and it has read 4876417 bytes out of 4883044 bytes. This indicates the compaction is about to complete. The stage property indicates which code block the thread is currently executing. For instance, thread 140716416169728 is currently running `CompactionJob::Install`, which further indicates the compaction job is almost done. + +Below we briefly describe its API. + + +## How to Enable it? + + +To enable thread-tracking of a rocksdb instance, simply set `enable_thread_tracking` to true in its DBOptions: + +```c++ +// If true, then the status of the threads involved in this DB will +// be tracked and available via GetThreadList() API. +// +// Default: false +bool enable_thread_tracking; +``` + + + +## The API + + +The GetThreadList API is defined in [include/rocksdb/env.h](https://github.com/facebook/rocksdb/blob/main/include/rocksdb/env.h#L317-L318), which is an Env +function: + +```c++ +virtual Status GetThreadList(std::vector* thread_list) +``` + +Since an Env can be shared across multiple rocksdb instances, the output of +`GetThreadList()` include the background activity of all the rocksdb instances +that using the same Env. + +The `GetThreadList()` API simply returns a vector of `ThreadStatus`, each describes +the current status of a thread. The `ThreadStatus` structure, defined in +[include/rocksdb/thread_status.h](https://github.com/facebook/rocksdb/blob/main/include/rocksdb/thread_status.h), contains the following information: + +```c++ +// An unique ID for the thread. +const uint64_t thread_id; + +// The type of the thread, it could be HIGH_PRIORITY, +// LOW_PRIORITY, and USER +const ThreadType thread_type; + +// The name of the DB instance where the thread is currently +// involved with. It would be set to empty string if the thread +// does not involve in any DB operation. +const std::string db_name; + +// The name of the column family where the thread is currently +// It would be set to empty string if the thread does not involve +// in any column family. +const std::string cf_name; + +// The operation (high-level action) that the current thread is involved. +const OperationType operation_type; + +// The elapsed time in micros of the current thread operation. +const uint64_t op_elapsed_micros; + +// An integer showing the current stage where the thread is involved +// in the current operation. +const OperationStage operation_stage; + +// A list of properties that describe some details about the current +// operation. Same field in op_properties[] might have different +// meanings for different operations. +uint64_t op_properties[kNumOperationProperties]; + +// The state (lower-level action) that the current thread is involved. +const StateType state_type; +``` + +If you are interested in the background thread activity of your RocksDB application, please feel free to give `GetThreadList()` a try :) diff --git a/librocksdb-sys/rocksdb/docs/_posts/2015-11-10-use-checkpoints-for-efficient-snapshots.markdown b/librocksdb-sys/rocksdb/docs/_posts/2015-11-10-use-checkpoints-for-efficient-snapshots.markdown new file mode 100644 index 0000000..6852b8f --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2015-11-10-use-checkpoints-for-efficient-snapshots.markdown @@ -0,0 +1,45 @@ +--- +title: Use Checkpoints for Efficient Snapshots +layout: post +author: rven2 +category: blog +redirect_from: + - /blog/2609/use-checkpoints-for-efficient-snapshots/ +--- + +**Checkpoint** is a feature in RocksDB which provides the ability to take a snapshot of a running RocksDB database in a separate directory. Checkpoints can be used as a point in time snapshot, which can be opened Read-only to query rows as of the point in time or as a Writeable snapshot by opening it Read-Write. Checkpoints can be used for both full and incremental backups. + + + + +The Checkpoint feature enables RocksDB to create a consistent snapshot of a given RocksDB database in the specified directory. If the snapshot is on the same filesystem as the original database, the SST files will be hard-linked, otherwise SST files will be copied. The manifest and CURRENT files will be copied. In addition, if there are multiple column families, log files will be copied for the period covering the start and end of the checkpoint, in order to provide a consistent snapshot across column families. + + + + +A Checkpoint object needs to be created for a database before checkpoints are created. The API is as follows: + + + + +`Status Create(DB* db, Checkpoint** checkpoint_ptr);` + + + + +Given a checkpoint object and a directory, the CreateCheckpoint function creates a consistent snapshot of the database in the given directory. + + + + +`Status CreateCheckpoint(const std::string& checkpoint_dir);` + + + + +The directory should not already exist and will be created by this API. The directory will be an absolute path. The checkpoint can be used as a ​read-only copy of the DB or can be opened as a standalone DB. When opened read/write, the SST files continue to be hard links and these links are removed when the files are obsoleted. When the user is done with the snapshot, the user can delete the directory to remove the snapshot. + + + + +Checkpoints are used for online backup in ​MyRocks. which is MySQL using RocksDB as the storage engine . ([MySQL on RocksDB](https://github.com/facebook/mysql-5.6)) ​ diff --git a/librocksdb-sys/rocksdb/docs/_posts/2015-11-16-analysis-file-read-latency-by-level.markdown b/librocksdb-sys/rocksdb/docs/_posts/2015-11-16-analysis-file-read-latency-by-level.markdown new file mode 100644 index 0000000..b21b04f --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2015-11-16-analysis-file-read-latency-by-level.markdown @@ -0,0 +1,244 @@ +--- +title: Analysis File Read Latency by Level +layout: post +author: sdong +category: blog +redirect_from: + - /blog/2537/analysis-file-read-latency-by-level/ +--- + +In many use cases of RocksDB, people rely on OS page cache for caching compressed data. With this approach, verifying effective of the OS page caching is challenging, because file system is a black box to users. + +As an example, a user can tune the DB as following: use level-based compaction, with L1 - L4 sizes to be 1GB, 10GB, 100GB and 1TB. And they reserve about 20GB memory as OS page cache, expecting level 0, 1 and 2 are mostly cached in memory, leaving only reads from level 3 and 4 requiring disk I/Os. However, in practice, it's not easy to verify whether OS page cache does exactly what we expect. For example, if we end up with doing 4 instead of 2 I/Os per query, it's not easy for users to figure out whether the it's because of efficiency of OS page cache or reading multiple blocks for a level. Analysis like it is especially important if users run RocksDB on hard drive disks, for the gap of latency between hard drives and memory is much higher than flash-based SSDs. + + + +In order to make tuning easier, we added new instrumentation to help users analysis latency distribution of file reads in different levels. If users turn DB statistics on, we always keep track of distribution of file read latency for each level. Users can retrieve the information by querying DB property “rocksdb.stats” ( [https://github.com/facebook/rocksdb/blob/v3.13.1/include/rocksdb/db.h#L315-L316](https://github.com/facebook/rocksdb/blob/v3.13.1/include/rocksdb/db.h#L315-L316) ). It will also printed out as a part of compaction summary in info logs periodically. + +The output looks like this: + + +``` +** Level 0 read latency histogram (micros): +Count: 696 Average: 489.8118 StdDev: 222.40 +Min: 3.0000 Median: 452.3077 Max: 1896.0000 +Percentiles: P50: 452.31 P75: 641.30 P99: 1068.00 P99.9: 1860.80 P99.99: 1896.00 +------------------------------------------------------ +[ 2, 3 ) 1 0.144% 0.144% +[ 18, 20 ) 1 0.144% 0.287% +[ 45, 50 ) 5 0.718% 1.006% +[ 50, 60 ) 26 3.736% 4.741% # +[ 60, 70 ) 6 0.862% 5.603% +[ 90, 100 ) 1 0.144% 5.747% +[ 120, 140 ) 2 0.287% 6.034% +[ 140, 160 ) 1 0.144% 6.178% +[ 160, 180 ) 1 0.144% 6.322% +[ 200, 250 ) 9 1.293% 7.615% +[ 250, 300 ) 45 6.466% 14.080% # +[ 300, 350 ) 88 12.644% 26.724% ### +[ 350, 400 ) 88 12.644% 39.368% ### +[ 400, 450 ) 71 10.201% 49.569% ## +[ 450, 500 ) 65 9.339% 58.908% ## +[ 500, 600 ) 74 10.632% 69.540% ## +[ 600, 700 ) 92 13.218% 82.759% ### +[ 700, 800 ) 64 9.195% 91.954% ## +[ 800, 900 ) 35 5.029% 96.983% # +[ 900, 1000 ) 12 1.724% 98.707% +[ 1000, 1200 ) 6 0.862% 99.569% +[ 1200, 1400 ) 2 0.287% 99.856% +[ 1800, 2000 ) 1 0.144% 100.000% + +** Level 1 read latency histogram (micros): +(......not pasted.....) + +** Level 2 read latency histogram (micros): +(......not pasted.....) + +** Level 3 read latency histogram (micros): +(......not pasted.....) + +** Level 4 read latency histogram (micros): +(......not pasted.....) + +** Level 5 read latency histogram (micros): +Count: 25583746 Average: 421.1326 StdDev: 385.11 +Min: 1.0000 Median: 376.0011 Max: 202444.0000 +Percentiles: P50: 376.00 P75: 438.00 P99: 1421.68 P99.9: 4164.43 P99.99: 9056.52 +------------------------------------------------------ +[ 0, 1 ) 2351 0.009% 0.009% +[ 1, 2 ) 6077 0.024% 0.033% +[ 2, 3 ) 8471 0.033% 0.066% +[ 3, 4 ) 788 0.003% 0.069% +[ 4, 5 ) 393 0.002% 0.071% +[ 5, 6 ) 786 0.003% 0.074% +[ 6, 7 ) 1709 0.007% 0.080% +[ 7, 8 ) 1769 0.007% 0.087% +[ 8, 9 ) 1573 0.006% 0.093% +[ 9, 10 ) 1495 0.006% 0.099% +[ 10, 12 ) 3043 0.012% 0.111% +[ 12, 14 ) 2259 0.009% 0.120% +[ 14, 16 ) 1233 0.005% 0.125% +[ 16, 18 ) 762 0.003% 0.128% +[ 18, 20 ) 451 0.002% 0.130% +[ 20, 25 ) 794 0.003% 0.133% +[ 25, 30 ) 1279 0.005% 0.138% +[ 30, 35 ) 1172 0.005% 0.142% +[ 35, 40 ) 1363 0.005% 0.148% +[ 40, 45 ) 409 0.002% 0.149% +[ 45, 50 ) 105 0.000% 0.150% +[ 50, 60 ) 80 0.000% 0.150% +[ 60, 70 ) 280 0.001% 0.151% +[ 70, 80 ) 1583 0.006% 0.157% +[ 80, 90 ) 4245 0.017% 0.174% +[ 90, 100 ) 6572 0.026% 0.200% +[ 100, 120 ) 9724 0.038% 0.238% +[ 120, 140 ) 3713 0.015% 0.252% +[ 140, 160 ) 2383 0.009% 0.261% +[ 160, 180 ) 18344 0.072% 0.333% +[ 180, 200 ) 51873 0.203% 0.536% +[ 200, 250 ) 631722 2.469% 3.005% +[ 250, 300 ) 2721970 10.639% 13.644% ## +[ 300, 350 ) 5909249 23.098% 36.742% ##### +[ 350, 400 ) 6522507 25.495% 62.237% ##### +[ 400, 450 ) 4296332 16.793% 79.030% ### +[ 450, 500 ) 2130323 8.327% 87.357% ## +[ 500, 600 ) 1553208 6.071% 93.428% # +[ 600, 700 ) 642129 2.510% 95.938% # +[ 700, 800 ) 372428 1.456% 97.394% +[ 800, 900 ) 187561 0.733% 98.127% +[ 900, 1000 ) 85858 0.336% 98.462% +[ 1000, 1200 ) 82730 0.323% 98.786% +[ 1200, 1400 ) 50691 0.198% 98.984% +[ 1400, 1600 ) 38026 0.149% 99.133% +[ 1600, 1800 ) 32991 0.129% 99.261% +[ 1800, 2000 ) 30200 0.118% 99.380% +[ 2000, 2500 ) 62195 0.243% 99.623% +[ 2500, 3000 ) 36684 0.143% 99.766% +[ 3000, 3500 ) 21317 0.083% 99.849% +[ 3500, 4000 ) 10216 0.040% 99.889% +[ 4000, 4500 ) 8351 0.033% 99.922% +[ 4500, 5000 ) 4152 0.016% 99.938% +[ 5000, 6000 ) 6328 0.025% 99.963% +[ 6000, 7000 ) 3253 0.013% 99.976% +[ 7000, 8000 ) 2082 0.008% 99.984% +[ 8000, 9000 ) 1546 0.006% 99.990% +[ 9000, 10000 ) 1055 0.004% 99.994% +[ 10000, 12000 ) 1566 0.006% 100.000% +[ 12000, 14000 ) 761 0.003% 100.003% +[ 14000, 16000 ) 462 0.002% 100.005% +[ 16000, 18000 ) 226 0.001% 100.006% +[ 18000, 20000 ) 126 0.000% 100.006% +[ 20000, 25000 ) 107 0.000% 100.007% +[ 25000, 30000 ) 43 0.000% 100.007% +[ 30000, 35000 ) 15 0.000% 100.007% +[ 35000, 40000 ) 14 0.000% 100.007% +[ 40000, 45000 ) 16 0.000% 100.007% +[ 45000, 50000 ) 1 0.000% 100.007% +[ 50000, 60000 ) 22 0.000% 100.007% +[ 60000, 70000 ) 10 0.000% 100.007% +[ 70000, 80000 ) 5 0.000% 100.007% +[ 80000, 90000 ) 14 0.000% 100.007% +[ 90000, 100000 ) 11 0.000% 100.007% +[ 100000, 120000 ) 33 0.000% 100.007% +[ 120000, 140000 ) 6 0.000% 100.007% +[ 140000, 160000 ) 3 0.000% 100.007% +[ 160000, 180000 ) 7 0.000% 100.007% +[ 200000, 250000 ) 2 0.000% 100.007% +``` + + +In this example, you can see we only issued 696 reads from level 0 while issued 25 million reads from level 5. The latency distribution is also clearly shown among those reads. This will be helpful for users to analysis OS page cache efficiency. + +Currently the read latency per level includes reads from data blocks, index blocks, as well as bloom filter blocks. We are also working on a feature to break down those three type of blocks. + +### Comments + +**[Tao Feng](fengtao04@gmail.com)** + +Is this feature also included in RocksJava? + +**[Siying Dong](siying.d@fb.com)** + +Should be. As long as you enable statistics, you should be able to get the value from `RocksDB.getProperty()` with property `rocksdb.dbstats`. Let me know if you can’t find it. + +**[chiddu](cnbscience@gmail.com)** + +> In this example, you can see we only issued 696 reads from level 0 while issued 256K reads from level 5. + +Isn’t it 2.5 M of reads instead of 256K ? . + +Also could anyone please provide more description on the histogram ? especially + +> Count: 25583746 Average: 421.1326 StdDev: 385.11 +> Min: 1.0000 Median: 376.0011 Max: 202444.0000 +> Percentiles: P50: 376.00 P75: 438.00 P99: 1421.68 P99.9: 4164.43 P99.99: 9056.52 + +and + +> [ 0, 1 ) 2351 0.009% 0.009% +> [ 1, 2 ) 6077 0.024% 0.033% +> [ 2, 3 ) 8471 0.033% 0.066% +> [ 3, 4 ) 788 0.003% 0.069%” + +thanks in advance + +**[Siying Dong](siying.d@fb.com)** + +Thank you for pointing out the mistake. I fixed it now. + +In this output, there are 2.5 million samples, average latency is 421 micro seconds, with standard deviation 385. Median is 376, max value is 202 milliseconds. 0.009% has value of 1, 0.024% has value of 1, 0.033% has value of 2. Accumulated value from 0 to 2 is 0.066%. + +Hope it helps. + +**[chiddu](cnbscience@gmail.com)** + +Thank you Siying for the quick reply, I was running couple of benchmark testing to check the performance of rocksdb on SSD. One of the test is similar to what is mentioned in the wiki, TEST 4 : Random read , except the key_size is 10 and value_size is 20. I am inserting 1 billion hashes and reading 1 billion hashes with 32 threads. The histogram shows something like this + +``` +Level 5 read latency histogram (micros): +Count: 7133903059 Average: 480.4357 StdDev: 309.18 +Min: 0.0000 Median: 551.1491 Max: 224142.0000 +Percentiles: P50: 551.15 P75: 651.44 P99: 996.52 P99.9: 2073.07 P99.99: 3196.32 +—————————————————— +[ 0, 1 ) 28587385 0.401% 0.401% +[ 1, 2 ) 686572516 9.624% 10.025% ## +[ 2, 3 ) 567317522 7.952% 17.977% ## +[ 3, 4 ) 44979472 0.631% 18.608% +[ 4, 5 ) 50379685 0.706% 19.314% +[ 5, 6 ) 64930061 0.910% 20.224% +[ 6, 7 ) 22613561 0.317% 20.541% +…………more…………. +``` + +If I understand your previous comment correctly, + +1. How is it that the count is around 7 billion when I have only inserted 1 billion hashes ? is the stat broken ? +1. What does the percentiles and the numbers signify ? +1. 0, 1 ) 28587385 0.401% 0.401% what does this “28587385” stand for in the histogram row ? + +**[Siying Dong](siying.d@fb.com)** + +If I remember correctly, with db_bench, if you specify –num=1000000000 –threads=32, it is every thread reading one billion keys, total of 32 billions. Is it the case you ran into? + +28,587,385 means that number of data points take the value [0,1) +28,587,385 / 7,133,903,058 = 0.401% provides percentage. + +**[chiddu](cnbscience@gmail.com)** + +I do have `num=1000000000` and `t=32`. The script says reading 1 billion hashes and not 32 billion hashes. + +this is the script on which I have used + +``` +echo “Load 1B keys sequentially into database…..” +bpl=10485760;overlap=10;mcz=2;del=300000000;levels=6;ctrig=4; delay=8; stop=12; wbn=3; mbc=20; mb=67108864;wbs=134217728; dds=1; sync=0; r=1000000000; t=1; vs=20; bs=4096; cs=1048576; of=500000; si=1000000; ./db_bench –benchmarks=fillseq –disable_seek_compaction=1 –mmap_read=0 –statistics=1 –histogram=1 –num=$r –threads=$t –value_size=$vs –block_size=$bs –cache_size=$cs –bloom_bits=10 –cache_numshardbits=6 –open_files=$of –verify_checksum=1 –db=/data/mysql/leveldb/test –sync=$sync –disable_wal=1 –compression_type=none –stats_interval=$si –compression_ratio=0.5 –disable_data_sync=$dds –write_buffer_size=$wbs –target_file_size_base=$mb –max_write_buffer_number=$wbn –max_background_compactions=$mbc –level0_file_num_compaction_trigger=$ctrig –level0_slowdown_writes_trigger=$delay –level0_stop_writes_trigger=$stop –num_levels=$levels –delete_obsolete_files_period_micros=$del –min_level_to_compress=$mcz –max_grandparent_overlap_factor=$overlap –stats_per_interval=1 –max_bytes_for_level_base=$bpl –use_existing_db=0 –key_size=10 + +echo “Reading 1B keys in database in random order….” +bpl=10485760;overlap=10;mcz=2;del=300000000;levels=6;ctrig=4; delay=8; stop=12; wbn=3; mbc=20; mb=67108864;wbs=134217728; dds=0; sync=0; r=1000000000; t=32; vs=20; bs=4096; cs=1048576; of=500000; si=1000000; ./db_bench –benchmarks=readrandom –disable_seek_compaction=1 –mmap_read=0 –statistics=1 –histogram=1 –num=$r –threads=$t –value_size=$vs –block_size=$bs –cache_size=$cs –bloom_bits=10 –cache_numshardbits=6 –open_files=$of –verify_checksum=1 –db=/some_data_base –sync=$sync –disable_wal=1 –compression_type=none –stats_interval=$si –compression_ratio=0.5 –disable_data_sync=$dds –write_buffer_size=$wbs –target_file_size_base=$mb –max_write_buffer_number=$wbn –max_background_compactions=$mbc –level0_file_num_compaction_trigger=$ctrig –level0_slowdown_writes_trigger=$delay –level0_stop_writes_trigger=$stop –num_levels=$levels –delete_obsolete_files_period_micros=$del –min_level_to_compress=$mcz –max_grandparent_overlap_factor=$overlap –stats_per_interval=1 –max_bytes_for_level_base=$bpl –use_existing_db=1 –key_size=10 +``` + +After running this script, there were no issues wrt to loading billion hashes , but when it came to reading part, its been almost 4 days and still I have only read 7 billion hashes and have read 200 million hashes in 2 and half days. Is there something which is missing in db_bench or something which I am missing ? + +**[Siying Dong](siying.d@fb.com)** + +It’s a printing error then. If you have `num=1000000000` and `t=32`, it will be 32 threads, and each reads 1 billion keys. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2016-01-29-compaction_pri.markdown b/librocksdb-sys/rocksdb/docs/_posts/2016-01-29-compaction_pri.markdown new file mode 100644 index 0000000..ba9ee62 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2016-01-29-compaction_pri.markdown @@ -0,0 +1,51 @@ +--- +title: Option of Compaction Priority +layout: post +author: sdong +category: blog +redirect_from: + - /blog/2921/compaction_pri/ +--- + +The most popular compaction style of RocksDB is level-based compaction, which is an improved version of LevelDB's compaction algorithm. Page 9- 16 of this [slides](https://github.com/facebook/rocksdb/blob/gh-pages/talks/2015-09-29-HPTS-Siying-RocksDB.pdf) gives an illustrated introduction of this compaction style. The basic idea that: data is organized by multiple levels with exponential increasing target size. Except a special level 0, every level is key-range partitioned into many files. When size of a level exceeds its target size, we pick one or more of its files, and merge the file into the next level. + + + +Which file to pick to compact is an interesting question. LevelDB only uses one thread for compaction and it always picks files in round robin manner. We implemented multi-thread compaction in RocksDB by picking multiple files from the same level and compact them in parallel. We had to move away from LevelDB's file picking approach. Recently, we created an option [options.compaction_pri](https://github.com/facebook/rocksdb/blob/d6c838f1e130d8860407bc771fa6d4ac238859ba/include/rocksdb/options.h#L83-L93), which indicated three different algorithms to pick files to compact. + +Why do we need to multiple algorithms to choose from? Because there are different factors to consider when picking the files, and we now don't yet know how to balance them automatically, so we expose it to users to choose. Here are factors to consider: + +**Write amplification** + +When we estimate write amplification, we usually simplify the problem by assuming keys are uniformly distributed inside each level. In reality, it is not the case, even if user updates are uniformly distributed across the whole key range. For instance, when we compact one file of a level to the next level, it creates a hole. Over time, incoming compaction will fill data to the hole, but the density will still be lower for a while. Picking a file with keys least densely populated is more expensive to get the file to the next level, because there will be more overlapping files in the next level so we need to rewrite more data. For example, assume a file is 100MB, if an L2 file overlaps with 8 L3 files, we need to rewrite about 800MB of data to get the file to L3. If the file overlaps with 12 L3 files, we'll need to rewrite about 1200MB to get a file of the same size out of L2. It uses 50% more writes. (This analysis ignores the key density of the next level, because the range covers N times of files in that level so one hole only impacts write amplification by 1/N) + +If all the updates are uniformly distributed, LevelDB's approach optimizes write amplification, because a file being picked covers a range whose last compaction time to the next level is the oldest, so the range will accumulated keys from incoming compactions for the longest and the density is the highest. + +We created a compaction priority **kOldestSmallestSeqFirst** for the same effect. With this mode, we always pick the file covers the oldest updates in the level, which usually is contains the densest key range. If you have a use case where writes are uniformly distributed across the key space and you want to reduce write amplification, you should set options.compaction_pri=kOldestSmallestSeqFirst. + +**Optimize for small working set** + +We are assuming updates are uniformly distributed across the whole key space in previous analysis. However, in many use cases, there are subset of keys that are frequently updated while other key ranges are very cold. In this case, keeping hot key ranges from compacting to deeper levels will benefit write amplification, as well as space amplification. For example, if in a DB only key 150-160 are updated and other keys are seldom updated. If level 1 contains 20 keys, we want to keep 150-160 all stay in level 1. Because when next level 0 -> 1 compaction comes, it will simply overwrite existing keys so size level 1 doesn't increase, so no need to schedule further compaction for level 1->2. On the other hand, if we compact key 150-155 to level2, when a new Level 1->2 compaction comes, it increases the size of level 1, making size of level 1 exceed target size and more compactions will be needed, which generates more writes. + +The compaction priority **kOldestLargestSeqFirst** optimizes this use case. In this mode, we will pick a file whose latest update is the oldest. It means there is no incoming data for the range for the longest. Usually it is the coldest range. By compacting coldest range first, we leave the hot ranges in the level. If your use case is to overwrite existing keys in a small range, try options.compaction_pri=kOldestLargestSeqFirst**.** + +**Drop delete marker sooner** + +If one file contains a lot of delete markers, it may slow down iterating over this area, because we still need to iterate those deleted keys just to ignore them. Furthermore, the sooner we compact delete keys into the last level, the sooner the disk space is reclaimed, so it is good for space efficiency. + +Our default compaction priority **kByCompensatedSize** considers the case. If number of deletes in a file exceeds number of inserts, it is more likely to be picked for compaction. The more number of deletes exceed inserts, the more likely it is being compacted. The optimization is added to avoid the worst performance of space efficiency and query performance when a large percentage of the DB is deleted. + +**Efficiency of compaction filter** + +Usually people use [compaction filters](https://github.com/facebook/rocksdb/blob/v4.1/include/rocksdb/options.h#L201-L226) to clean up old data to free up space. Picking files to compact may impact space efficiency. We don't yet have a a compaction priority to optimize this case. In some of our use cases, we solved the problem in a different way: we have an external service checking modify time of all SST files. If any of the files is too old, we force the single file to compaction by calling DB::CompactFiles() using the single file. In this way, we can provide a time bound of data passing through compaction filters. + + +In all, there three choices of compaction priority modes optimizing different scenarios. if you have a new use case, we suggest you start with `options.compaction_pri=kOldestSmallestSeqFirst` (note it is not the default one for backward compatible reason). If you want to further optimize your use case, you can try other two use cases if your use cases apply. + +If you have good ideas about better compaction picker approach, you are welcome to implement and benchmark it. We'll be glad to review and merge your a pull requests. + +### Comments + +**[Mark Callaghan](mdcallag@gmail.com)** + +Performance results for compaction_pri values and linkbench are explained at [http://smalldatum.blogspot.com/2016/02/compaction-priority-in-rocksdb.html](http://smalldatum.blogspot.com/2016/02/compaction-priority-in-rocksdb.html) diff --git a/librocksdb-sys/rocksdb/docs/_posts/2016-02-24-rocksdb-4-2-release.markdown b/librocksdb-sys/rocksdb/docs/_posts/2016-02-24-rocksdb-4-2-release.markdown new file mode 100644 index 0000000..409015c --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2016-02-24-rocksdb-4-2-release.markdown @@ -0,0 +1,41 @@ +--- +title: RocksDB 4.2 Release! +layout: post +author: sdong +category: blog +redirect_from: + - /blog/3017/rocksdb-4-2-release/ +--- + +New RocksDB release - 4.2! + + +**New Features** + + 1. Introduce CreateLoggerFromOptions(), this function create a Logger for provided DBOptions. + + + 2. Add GetAggregatedIntProperty(), which returns the sum of the GetIntProperty of all the column families. + + + 3. Add MemoryUtil in rocksdb/utilities/memory.h. It currently offers a way to get the memory usage by type from a list rocksdb instances. + + + + + +**Public API changes** + + 1. CompactionFilter::Context includes information of Column Family ID + + + 2. The need-compaction hint given by TablePropertiesCollector::NeedCompact() will be persistent and recoverable after DB recovery. This introduces a breaking format change. If you use this experimental feature, including NewCompactOnDeletionCollectorFactory() in the new version, you may not be able to directly downgrade the DB back to version 4.0 or lower. + + + 3. TablePropertiesCollectorFactory::CreateTablePropertiesCollector() now takes an option Context, containing the information of column family ID for the file being written. + + + 4. Remove DefaultCompactionFilterFactory. + + +[https://github.com/facebook/rocksdb/releases/tag/v4.2](https://github.com/facebook/rocksdb/releases/tag/v4.2) diff --git a/librocksdb-sys/rocksdb/docs/_posts/2016-02-25-rocksdb-ama.markdown b/librocksdb-sys/rocksdb/docs/_posts/2016-02-25-rocksdb-ama.markdown new file mode 100644 index 0000000..2ba04f3 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2016-02-25-rocksdb-ama.markdown @@ -0,0 +1,20 @@ +--- +title: RocksDB AMA +layout: post +author: yhchiang +category: blog +redirect_from: + - /blog/3065/rocksdb-ama/ +--- + +RocksDB developers are doing a Reddit Ask-Me-Anything now at 10AM – 11AM PDT! We welcome you to stop by and ask any RocksDB related questions, including existing / upcoming features, tuning tips, or database design. + +Here are some enhancements that we'd like to focus on over the next six months: + +* 2-Phase Commit +* Lua support in some custom functions +* Backup and repair tools +* Direct I/O to bypass OS cache +* RocksDB Java API + +[https://www.reddit.com/r/IAmA/comments/47k1si/we_are_rocksdb_developers_ask_us_anything/](https://www.reddit.com/r/IAmA/comments/47k1si/we_are_rocksdb_developers_ask_us_anything/) diff --git a/librocksdb-sys/rocksdb/docs/_posts/2016-03-07-rocksdb-options-file.markdown b/librocksdb-sys/rocksdb/docs/_posts/2016-03-07-rocksdb-options-file.markdown new file mode 100644 index 0000000..703449b --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2016-03-07-rocksdb-options-file.markdown @@ -0,0 +1,24 @@ +--- +title: RocksDB Options File +layout: post +author: yhciang +category: blog +redirect_from: + - /blog/3089/rocksdb-options-file/ +--- + +In RocksDB 4.3, we added a new set of features that makes managing RocksDB options easier. Specifically: + + * **Persisting Options Automatically**: Each RocksDB database will now automatically persist its current set of options into an INI file on every successful call of DB::Open(), SetOptions(), and CreateColumnFamily() / DropColumnFamily(). + + + + * **Load Options from File**: We added [LoadLatestOptions() / LoadOptionsFromFile()](https://github.com/facebook/rocksdb/blob/4.3.fb/include/rocksdb/utilities/options_util.h#L48-L58) that enables developers to construct RocksDB options object from an options file. + + + + * **Sanity Check Options**: We added [CheckOptionsCompatibility](https://github.com/facebook/rocksdb/blob/4.3.fb/include/rocksdb/utilities/options_util.h#L64-L77) that performs compatibility check on two sets of RocksDB options. + + + +Want to know more about how to use this new features? Check out the [RocksDB Options File wiki page](https://github.com/facebook/rocksdb/wiki/RocksDB-Options-File) and start using this new feature today! diff --git a/librocksdb-sys/rocksdb/docs/_posts/2016-04-26-rocksdb-4-5-1-released.markdown b/librocksdb-sys/rocksdb/docs/_posts/2016-04-26-rocksdb-4-5-1-released.markdown new file mode 100644 index 0000000..247768d --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2016-04-26-rocksdb-4-5-1-released.markdown @@ -0,0 +1,60 @@ +--- +title: RocksDB 4.5.1 Released! +layout: post +author: sdong +category: blog +redirect_from: + - /blog/3179/rocksdb-4-5-1-released/ +--- + +## 4.5.1 (3/25/2016) + +### Bug Fixes + + *  Fix failures caused by the destorying order of singleton objects. + +
    + +## 4.5.0 (2/5/2016) + +### Public API Changes + + * Add a new perf context level between kEnableCount and kEnableTime. Level 2 now does not include timers for mutexes. + * Statistics of mutex operation durations will not be measured by default. If you want to have them enabled, you need to set Statistics::stats_level_ to kAll. + * DBOptions::delete_scheduler and NewDeleteScheduler() are removed, please use DBOptions::sst_file_manager and NewSstFileManager() instead + +### New Features + * ldb tool now supports operations to non-default column families. + * Add kPersistedTier to ReadTier. This option allows Get and MultiGet to read only the persited data and skip mem-tables if writes were done with disableWAL = true. + * Add DBOptions::sst_file_manager. Use NewSstFileManager() in include/rocksdb/sst_file_manager.h to create a SstFileManager that can be used to track the total size of SST files and control the SST files deletion rate. + +
    + + + +## 4.4.0 (1/14/2016) + +### Public API Changes + + * Change names in CompactionPri and add a new one. + * Deprecate options.soft_rate_limit and add options.soft_pending_compaction_bytes_limit. + * If options.max_write_buffer_number > 3, writes will be slowed down when writing to the last write buffer to delay a full stop. + * Introduce CompactionJobInfo::compaction_reason, this field include the reason to trigger the compaction. + * After slow down is triggered, if estimated pending compaction bytes keep increasing, slowdown more. + * Increase default options.delayed_write_rate to 2MB/s. + * Added a new parameter --path to ldb tool. --path accepts the name of either MANIFEST, SST or a WAL file. Either --db or --path can be used when calling ldb. + +
    + +## 4.3.0 (12/8/2015) + +### New Features + + * CompactionFilter has new member function called IgnoreSnapshots which allows CompactionFilter to be called even if there are snapshots later than the key. + * RocksDB will now persist options under the same directory as the RocksDB database on successful DB::Open, CreateColumnFamily, DropColumnFamily, and SetOptions. + * Introduce LoadLatestOptions() in rocksdb/utilities/options_util.h. This function can construct the latest DBOptions / ColumnFamilyOptions used by the specified RocksDB intance. + * Introduce CheckOptionsCompatibility() in rocksdb/utilities/options_util.h. This function checks whether the input set of options is able to open the specified DB successfully. + +### Public API Changes + + * When options.db_write_buffer_size triggers, only the column family with the largest column family size will be flushed, not all the column families. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2016-07-26-rocksdb-4-8-released.markdown b/librocksdb-sys/rocksdb/docs/_posts/2016-07-26-rocksdb-4-8-released.markdown new file mode 100644 index 0000000..0db275d --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2016-07-26-rocksdb-4-8-released.markdown @@ -0,0 +1,48 @@ +--- +title: RocksDB 4.8 Released! +layout: post +author: yiwu +category: blog +redirect_from: + - /blog/3239/rocksdb-4-8-released/ +--- + +## 4.8.0 (5/2/2016) + +### [](https://github.com/facebook/rocksdb/blob/main/HISTORY.md#public-api-change-1)Public API Change + + * Allow preset compression dictionary for improved compression of block-based tables. This is supported for zlib, zstd, and lz4. The compression dictionary's size is configurable via CompressionOptions::max_dict_bytes. + * Delete deprecated classes for creating backups (BackupableDB) and restoring from backups (RestoreBackupableDB). Now, BackupEngine should be used for creating backups, and BackupEngineReadOnly should be used for restorations. For more details, see [https://github.com/facebook/rocksdb/wiki/How-to-backup-RocksDB%3F](https://github.com/facebook/rocksdb/wiki/How-to-backup-RocksDB%3F) + * Expose estimate of per-level compression ratio via DB property: "rocksdb.compression-ratio-at-levelN". + * Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status. + +### [](https://github.com/facebook/rocksdb/blob/main/HISTORY.md#new-features-2)New Features + + * Add ReadOptions::readahead_size. If non-zero, NewIterator will create a new table reader which performs reads of the given size. + +
    + + + +## [](https://github.com/facebook/rocksdb/blob/main/HISTORY.md#470-482016)4.7.0 (4/8/2016) + +### [](https://github.com/facebook/rocksdb/blob/main/HISTORY.md#public-api-change-2)Public API Change + + * rename options compaction_measure_io_stats to report_bg_io_stats and include flush too. + * Change some default options. Now default options will optimize for server-workloads. Also enable slowdown and full stop triggers for pending compaction bytes. These changes may cause sub-optimal performance or significant increase of resource usage. To avoid these risks, users can open existing RocksDB with options extracted from RocksDB option files. See [https://github.com/facebook/rocksdb/wiki/RocksDB-Options-File](https://github.com/facebook/rocksdb/wiki/RocksDB-Options-File) for how to use RocksDB option files. Or you can call Options.OldDefaults() to recover old defaults. DEFAULT_OPTIONS_HISTORY.md will track change history of default options. + +
    + +## [](https://github.com/facebook/rocksdb/blob/main/HISTORY.md#460-3102016)4.6.0 (3/10/2016) + +### [](https://github.com/facebook/rocksdb/blob/main/HISTORY.md#public-api-changes-1)Public API Changes + + * Change default of BlockBasedTableOptions.format_version to 2. It means default DB created by 4.6 or up cannot be opened by RocksDB version 3.9 or earlier + * Added strict_capacity_limit option to NewLRUCache. If the flag is set to true, insert to cache will fail if no enough capacity can be free. Signature of Cache::Insert() is updated accordingly. + * Tickers [NUMBER_DB_NEXT, NUMBER_DB_PREV, NUMBER_DB_NEXT_FOUND, NUMBER_DB_PREV_FOUND, ITER_BYTES_READ] are not updated immediately. The are updated when the Iterator is deleted. + * Add monotonically increasing counter (DB property "rocksdb.current-super-version-number") that increments upon any change to the LSM tree. + +### [](https://github.com/facebook/rocksdb/blob/main/HISTORY.md#new-features-3)New Features + + * Add CompactionPri::kMinOverlappingRatio, a compaction picking mode friendly to write amplification. + * Deprecate Iterator::IsKeyPinned() and replace it with Iterator::GetProperty() with prop_name="rocksdb.iterator.is.key.pinned" diff --git a/librocksdb-sys/rocksdb/docs/_posts/2016-09-28-rocksdb-4-11-2-released.markdown b/librocksdb-sys/rocksdb/docs/_posts/2016-09-28-rocksdb-4-11-2-released.markdown new file mode 100644 index 0000000..87c20eb --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2016-09-28-rocksdb-4-11-2-released.markdown @@ -0,0 +1,49 @@ +--- +title: RocksDB 4.11.2 Released! +layout: post +author: sdong +category: blog +--- +We abandoned release candidates 4.10.x and directly go to 4.11.2 from 4.9, to make sure the latest release is stable. In 4.11.2, we fixed several data corruption related bugs introduced in 4.9.0. + +## 4.11.2 (9/15/2016) + +### Bug fixes + + * Segfault when failing to open an SST file for read-ahead iterators. + * WAL without data for all CFs is not deleted after recovery. + + + +## 4.11.1 (8/30/2016) + +### Bug Fixes + + * Mitigate the regression bug of deadlock condition during recovery when options.max_successive_merges hits. + * Fix data race condition related to hash index in block based table when putting indexes in the block cache. + +## 4.11.0 (8/1/2016) + +### Public API Change + + * options.memtable_prefix_bloom_huge_page_tlb_size => memtable_huge_page_size. When it is set, RocksDB will try to allocate memory from huge page for memtable too, rather than just memtable bloom filter. + +### New Features + + * A tool to migrate DB after options change. See include/rocksdb/utilities/option_change_migration.h. + * Add ReadOptions.background_purge_on_iterator_cleanup. If true, we avoid file deletion when destorying iterators. + +## 4.10.0 (7/5/2016) + +### Public API Change + + * options.memtable_prefix_bloom_bits changes to options.memtable_prefix_bloom_bits_ratio and deprecate options.memtable_prefix_bloom_probes + * enum type CompressionType and PerfLevel changes from char to unsigned char. Value of all PerfLevel shift by one. + * Deprecate options.filter_deletes. + +### New Features + + * Add avoid_flush_during_recovery option. + * Add a read option background_purge_on_iterator_cleanup to avoid deleting files in foreground when destroying iterators. Instead, a job is scheduled in high priority queue and would be executed in a separate background thread. + * RepairDB support for column families. RepairDB now associates data with non-default column families using information embedded in the SST/WAL files (4.7 or later). For data written by 4.6 or earlier, RepairDB associates it with the default column family. + * Add options.write_buffer_manager which allows users to control total memtable sizes across multiple DB instances. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2017-01-06-rocksdb-5-0-1-released.markdown b/librocksdb-sys/rocksdb/docs/_posts/2017-01-06-rocksdb-5-0-1-released.markdown new file mode 100644 index 0000000..fb04130 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2017-01-06-rocksdb-5-0-1-released.markdown @@ -0,0 +1,26 @@ +--- +title: RocksDB 5.0.1 Released! +layout: post +author: yiwu +category: blog +--- + +### Public API Change + + * Options::max_bytes_for_level_multiplier is now a double along with all getters and setters. + * Support dynamically change `delayed_write_rate` and `max_total_wal_size` options via SetDBOptions(). + * Introduce DB::DeleteRange for optimized deletion of large ranges of contiguous keys. + * Support dynamically change `delayed_write_rate` option via SetDBOptions(). + * Options::allow_concurrent_memtable_write and Options::enable_write_thread_adaptive_yield are now true by default. + * Remove Tickers::SEQUENCE_NUMBER to avoid confusion if statistics object is shared among RocksDB instance. Alternatively DB::GetLatestSequenceNumber() can be used to get the same value. + * Options.level0_stop_writes_trigger default value changes from 24 to 32. + * New compaction filter API: CompactionFilter::FilterV2(). Allows to drop ranges of keys. + * Removed flashcache support. + * DB::AddFile() is deprecated and is replaced with DB::IngestExternalFile(). DB::IngestExternalFile() remove all the restrictions that existed for DB::AddFile. + +### New Features + + * Add avoid_flush_during_shutdown option, which speeds up DB shutdown by not flushing unpersisted data (i.e. with disableWAL = true). Unpersisted data will be lost. The options is dynamically changeable via SetDBOptions(). + * Add memtable_insert_with_hint_prefix_extractor option. The option is mean to reduce CPU usage for inserting keys into memtable, if keys can be group by prefix and insert for each prefix are sequential or almost sequential. See include/rocksdb/options.h for more details. + * Add LuaCompactionFilter in utilities. This allows developers to write compaction filters in Lua. To use this feature, LUA_PATH needs to be set to the root directory of Lua. + * No longer populate "LATEST_BACKUP" file in backup directory, which formerly contained the number of the latest backup. The latest backup can be determined by finding the highest numbered file in the "meta/" subdirectory. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2017-02-07-rocksdb-5-1-2-released.markdown b/librocksdb-sys/rocksdb/docs/_posts/2017-02-07-rocksdb-5-1-2-released.markdown new file mode 100644 index 0000000..35bafb2 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2017-02-07-rocksdb-5-1-2-released.markdown @@ -0,0 +1,15 @@ +--- +title: RocksDB 5.1.2 Released! +layout: post +author: maysamyabandeh +category: blog +--- + +### Public API Change +* Support dynamically change `delete_obsolete_files_period_micros` option via SetDBOptions(). +* Added EventListener::OnExternalFileIngested which will be called when IngestExternalFile() add a file successfully. +* BackupEngine::Open and BackupEngineReadOnly::Open now always return error statuses matching those of the backup Env. + +### Bug Fixes +* Fix the bug that if 2PC is enabled, checkpoints may loss some recent transactions. +* When file copying is needed when creating checkpoints or bulk loading files, fsync the file after the file copying. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2017-02-17-bulkoad-ingest-sst-file.markdown b/librocksdb-sys/rocksdb/docs/_posts/2017-02-17-bulkoad-ingest-sst-file.markdown new file mode 100644 index 0000000..9a43a84 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2017-02-17-bulkoad-ingest-sst-file.markdown @@ -0,0 +1,50 @@ +--- +title: Bulkloading by ingesting external SST files +layout: post +author: IslamAbdelRahman +category: blog +--- + +## Introduction + +One of the basic operations of RocksDB is writing to RocksDB, Writes happen when user call (DB::Put, DB::Write, DB::Delete ... ), but what happens when you write to RocksDB ? .. this is a brief description of what happens. +- User insert a new key/value by calling DB::Put() (or DB::Write()) +- We create a new entry for the new key/value in our in-memory structure (memtable / SkipList by default) and we assign it a new sequence number. +- When the memtable exceeds a specific size (64 MB for example), we convert this memtable to a SST file, and put this file in level 0 of our LSM-Tree +- Later, compaction will kick in and move data from level 0 to level 1, and then from level 1 to level 2 .. and so on + +But what if we can skip these steps and add data to the lowest possible level directly ? This is what bulk-loading does + +## Bulkloading + +- Write all of our keys and values into SST file outside of the DB +- Add the SST file into the LSM directly + +This is bulk-loading, and in specific use-cases it allow users to achieve faster data loading and better write-amplification. + +and doing it is as simple as +```cpp +Options options; +SstFileWriter sst_file_writer(EnvOptions(), options, options.comparator); +Status s = sst_file_writer.Open(file_path); +assert(s.ok()); + +// Insert rows into the SST file, note that inserted keys must be +// strictly increasing (based on options.comparator) +for (...) { + s = sst_file_writer.Add(key, value); + assert(s.ok()); +} + +// Ingest the external SST file into the DB +s = db_->IngestExternalFile({"/home/usr/file1.sst"}, IngestExternalFileOptions()); +assert(s.ok()); +``` + +You can find more details about how to generate SST files and ingesting them into RocksDB in this [wiki page](https://github.com/facebook/rocksdb/wiki/Creating-and-Ingesting-SST-files) + +## Use cases +There are multiple use cases where bulkloading could be useful, for example +- Generating SST files in offline jobs in Hadoop, then downloading and ingesting the SST files into RocksDB +- Migrating shards between machines by dumping key-range in SST File and loading the file in a different machine +- Migrating from a different storage (InnoDB to RocksDB migration in MyRocks) diff --git a/librocksdb-sys/rocksdb/docs/_posts/2017-03-02-rocksdb-5-2-1-released.markdown b/librocksdb-sys/rocksdb/docs/_posts/2017-03-02-rocksdb-5-2-1-released.markdown new file mode 100644 index 0000000..c6ce27d --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2017-03-02-rocksdb-5-2-1-released.markdown @@ -0,0 +1,22 @@ +--- +title: RocksDB 5.2.1 Released! +layout: post +author: sdong +category: blog +--- + +### Public API Change +* NewLRUCache() will determine number of shard bits automatically based on capacity, if the user doesn't pass one. This also impacts the default block cache when the user doesn't explict provide one. +* Change the default of delayed slowdown value to 16MB/s and further increase the L0 stop condition to 36 files. + +### New Features +* Added new overloaded function GetApproximateSizes that allows to specify if memtable stats should be computed only without computing SST files' stats approximations. +* Added new function GetApproximateMemTableStats that approximates both number of records and size of memtables. +* (Experimental) Two-level indexing that partition the index and creates a 2nd level index on the partitions. The feature can be enabled by setting kTwoLevelIndexSearch as IndexType and configuring index_per_partition. + +### Bug Fixes +* RangeSync() should work if ROCKSDB_FALLOCATE_PRESENT is not set +* Fix wrong results in a data race case in Get() +* Some fixes related to 2PC. +* Fix several bugs in Direct I/O supports. +* Fix a regression bug which can cause Seek() to miss some keys if the return key has been updated many times after the snapshot which is used by the iterator. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2017-05-12-partitioned-index-filter.markdown b/librocksdb-sys/rocksdb/docs/_posts/2017-05-12-partitioned-index-filter.markdown new file mode 100644 index 0000000..a537feb --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2017-05-12-partitioned-index-filter.markdown @@ -0,0 +1,34 @@ +--- +title: Partitioned Index/Filters +layout: post +author: maysamyabandeh +category: blog +--- + +As DB/mem ratio gets larger, the memory footprint of filter/index blocks becomes non-trivial. Although `cache_index_and_filter_blocks` allows storing only a subset of them in block cache, their relatively large size negatively affects the performance by i) occupying the block cache space that could otherwise be used for caching data, ii) increasing the load on the disk storage by loading them into the cache after a miss. Here we illustrate these problems in more detail and explain how partitioning index/filters alleviates the overhead. + +### How large are the index/filter blocks? + +RocksDB has by default one index/filter block per SST file. The size of the index/filter varies based on the configuration but for a SST of size 256MB the index/filter block of size 0.5/5MB is typical, which is much larger than the typical data block size of 4-32KB. That is fine when all index/filters fit perfectly into memory and hence are read once per SST lifetime, not so much when they compete with data blocks for the block cache space and are also likely to be re-read many times from the disk. + +### What is the big deal with large index/filter blocks? + +When index/filter blocks are stored in block cache they are effectively competing with data blocks (as well as with each other) on this scarce resource. A filter of size 5MB is occupying the space that could otherwise be used to cache 1000s of data blocks (of size 4KB). This would result in more cache misses for data blocks. The large index/filters also kick each other out of the block cache more often and exacerbate their own cache miss rate too. This is while only a small part of the index/filter block might have been actually used during its lifetime in the cache. + +After the cache miss of an index/filter, it has to be reloaded from the disk, and its large size is not helping in reducing the IO cost. While a simple point lookup might need at most a couple of data block reads (of size 4KB) one from each layer of LSM, it might end up also loading multiple megabytes of index/filter blocks. If that happens often then the disk is spending more time serving index/filters rather than the actual data blocks. + +## What is partitioned index/filters? + +With partitioning, the index/filter of a SST file is partitioned into smaller blocks with an additional top-level index on them. When reading an index/filter, only top-level index is loaded into memory. The partitioned index/filter then uses the top-level index to load on demand into the block cache the partitions that are required to perform the index/filter query. The top-level index, which has much smaller memory footprint, can be stored in heap or block cache depending on the `cache_index_and_filter_blocks` setting. + +### Success stories + +#### HDD, 100TB DB + +In this example we have a DB of size 86G on HDD and emulate the small memory that is present to a node with 100TB of data by using direct IO (skipping OS file cache) and a very small block cache of size 60MB. Partitioning improves throughput by 11x from 5 op/s to 55 op/s. + +#### SSD, Linkbench + +In this example we have a DB of size 300G on SSD and emulate the small memory that would be available in presence of other DBs on the same node by by using direct IO (skipping OS file cache) and block cache of size 6G and 2G. Without partitioning the linkbench throughput drops from 38k tps to 23k when reducing block cache size from 6G to 2G. With partitioning the throughput drops from 38k to only 30k. + +Learn more [here](https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters). diff --git a/librocksdb-sys/rocksdb/docs/_posts/2017-05-14-core-local-stats.markdown b/librocksdb-sys/rocksdb/docs/_posts/2017-05-14-core-local-stats.markdown new file mode 100644 index 0000000..a806541 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2017-05-14-core-local-stats.markdown @@ -0,0 +1,106 @@ +--- +title: Core-local Statistics +layout: post +author: ajkr +category: blog +--- + +## Origins: Global Atomics + +Until RocksDB 4.12, ticker/histogram statistics were implemented with std::atomic values shared across the entire program. A ticker consists of a single atomic, while a histogram consists of several atomics to represent things like min/max/per-bucket counters. These statistics could be updated by all user/background threads. + +For concurrent/high-throughput workloads, cache line bouncing of atomics caused high CPU utilization. For example, we have tickers that count block cache hits and misses. Almost every user read increments these tickers a few times. Many concurrent user reads would cause the cache lines containing these atomics to bounce between cores. + +### Performance + +Here are perf results for 32 reader threads where most reads (99%+) are served by uncompressed block cache. Such a scenario stresses the statistics code heavily. + +Benchmark command: `TEST_TMPDIR=/dev/shm/ perf record -g ./db_bench -statistics -use_existing_db=true -benchmarks=readrandom -threads=32 -cache_size=1048576000 -num=1000000 -reads=1000000 && perf report -g --children` + +Perf snippet for "cycles" event: + +``` + Children Self Command Shared Object Symbol ++ 30.33% 30.17% db_bench db_bench [.] rocksdb::StatisticsImpl::recordTick ++ 3.65% 0.98% db_bench db_bench [.] rocksdb::StatisticsImpl::measureTime +``` + +Perf snippet for "cache-misses" event: + +``` + Children Self Command Shared Object Symbol ++ 19.54% 19.50% db_bench db_bench [.] rocksdb::StatisticsImpl::recordTick ++ 3.44% 0.57% db_bench db_bench [.] rocksdb::StatisticsImpl::measureTime +``` + +The high CPU overhead for updating tickers and histograms corresponds well to the high cache misses. + +## Thread-locals: Faster Updates + +Since RocksDB 4.12, ticker/histogram statistics use thread-local storage. Each thread has a local set of atomic values that no other thread can update. This prevents the cache line bouncing problem described above. Even though updates to a given value are always made by the same thread, atomics are still useful to synchronize with aggregations for querying statistics. + +Implementing this approach involved a couple challenges. First, each query for a statistic's global value must aggregate all threads' local values. This adds some overhead, which may pass unnoticed if statistics are queried infrequently. Second, exited threads' local values are still needed to provide accurate statistics. We handle this by merging a thread's local values into process-wide variables upon thread exit. + +### Performance + +Update benchmark setup is same as before. CPU overhead improved 7.8x compared to global atomics, corresponding to a 17.8x reduction in cache-misses overhead. + +Perf snippet for "cycles" event: + +``` + Children Self Command Shared Object Symbol ++ 2.96% 0.87% db_bench db_bench [.] rocksdb::StatisticsImpl::recordTick ++ 1.37% 0.10% db_bench db_bench [.] rocksdb::StatisticsImpl::measureTime +``` + +Perf snippet for "cache-misses" event: + +``` + Children Self Command Shared Object Symbol ++ 1.21% 0.65% db_bench db_bench [.] rocksdb::StatisticsImpl::recordTick + 0.08% 0.00% db_bench db_bench [.] rocksdb::StatisticsImpl::measureTime +``` + +To measure statistics query latency, we ran sysbench with 4K OLTP clients concurrently with one client that queries statistics repeatedly. Times shown are in milliseconds. + +``` + min: 18.45 + avg: 27.91 + max: 231.65 + 95th percentile: 55.82 +``` + +## Core-locals: Faster Querying + +The thread-local approach is working well for applications calling RocksDB from only a few threads, or polling statistics infrequently. Eventually, though, we found use cases where those assumptions do not hold. For example, one application has per-connection threads and typically runs into performance issues when connection count grows very high. For debugging such issues, they want high-frequency statistics polling to correlate issues in their application with changes in RocksDB's state. + +Once [PR #2258](https://github.com/facebook/rocksdb/pull/2258) lands, ticker/histogram statistics will be local to each CPU core. Similarly to thread-local, each core updates only its local values, thus avoiding cache line bouncing. Local values are still atomics to make aggregation possible. With this change, query work depends only on number of cores, not the number of threads. So, applications with many more threads than cores can no longer impact statistics query latency. + +### Performance + +Update benchmark setup is same as before. CPU overhead worsened ~23% compared to thread-local, while cache performance was unchanged. + +Perf snippet for "cycles" event: + +``` + Children Self Command Shared Object Symbol ++ 2.96% 0.87% db_bench db_bench [.] rocksdb::StatisticsImpl::recordTick ++ 1.37% 0.10% db_bench db_bench [.] rocksdb::StatisticsImpl::measureTime +``` + +Perf snippet for "cache-misses" event: + +``` + Children Self Command Shared Object Symbol ++ 1.21% 0.65% db_bench db_bench [.] rocksdb::StatisticsImpl::recordTick + 0.08% 0.00% db_bench db_bench [.] rocksdb::StatisticsImpl::measureTime +``` + +Query latency is measured same as before with times in milliseconds. Average latency improved by 6.3x compared to thread-local. + +``` + min: 2.47 + avg: 4.45 + max: 91.13 + 95th percentile: 7.56 +``` diff --git a/librocksdb-sys/rocksdb/docs/_posts/2017-05-26-rocksdb-5-4-5-released.markdown b/librocksdb-sys/rocksdb/docs/_posts/2017-05-26-rocksdb-5-4-5-released.markdown new file mode 100644 index 0000000..561dab4 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2017-05-26-rocksdb-5-4-5-released.markdown @@ -0,0 +1,39 @@ +--- +title: RocksDB 5.4.5 Released! +layout: post +author: sagar0 +category: blog +--- + +### Public API Change +* Support dynamically changing `stats_dump_period_sec` option via SetDBOptions(). +* Added ReadOptions::max_skippable_internal_keys to set a threshold to fail a request as incomplete when too many keys are being skipped while using iterators. +* DB::Get in place of std::string accepts PinnableSlice, which avoids the extra memcpy of value to std::string in most of cases. + * PinnableSlice releases the pinned resources that contain the value when it is destructed or when ::Reset() is called on it. + * The old API that accepts std::string, although discouraged, is still supported. +* Replace Options::use_direct_writes with Options::use_direct_io_for_flush_and_compaction. See Direct IO wiki for details. + +### New Features +* Memtable flush can be avoided during checkpoint creation if total log file size is smaller than a threshold specified by the user. +* Introduce level-based L0->L0 compactions to reduce file count, so write delays are incurred less often. +* (Experimental) Partitioning filters which creates an index on the partitions. The feature can be enabled by setting partition_filters when using kFullFilter. Currently the feature also requires two-level indexing to be enabled. Number of partitions is the same as the number of partitions for indexes, which is controlled by metadata_block_size. +* DB::ResetStats() to reset internal stats. +* Added CompactionEventListener and EventListener::OnFlushBegin interfaces. +* Added DB::CreateColumnFamilie() and DB::DropColumnFamilies() to bulk create/drop column families. +* Facility for cross-building RocksJava using Docker. + +### Bug Fixes +* Fix WriteBatchWithIndex address use after scope error. +* Fix WritableFile buffer size in direct IO. +* Add prefetch to PosixRandomAccessFile in buffered io. +* Fix PinnableSlice access invalid address when row cache is enabled. +* Fix huge fallocate calls fail and make XFS unhappy. +* Fix memory alignment with logical sector size. +* Fix alignment in ReadaheadRandomAccessFile. +* Fix bias with read amplification stats (READ_AMP_ESTIMATE_USEFUL_BYTES and READ_AMP_TOTAL_READ_BYTES). +* Fix a manual / auto compaction data race. +* Fix CentOS 5 cross-building of RocksJava. +* Build and link with ZStd when creating the static RocksJava build. +* Fix snprintf's usage to be cross-platform. +* Fix build errors with blob DB. +* Fix readamp test type inconsistency. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2017-06-26-17-level-based-changes.markdown b/librocksdb-sys/rocksdb/docs/_posts/2017-06-26-17-level-based-changes.markdown new file mode 100644 index 0000000..9e838eb --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2017-06-26-17-level-based-changes.markdown @@ -0,0 +1,60 @@ +--- +title: Level-based Compaction Changes +layout: post +author: ajkr +category: blog +--- + +### Introduction + +RocksDB provides an option to limit the number of L0 files, which bounds read-amplification. Since L0 files (unlike files at lower levels) can span the entire key-range, a key might be in any file, thus reads need to check them one-by-one. Users often wish to configure a low limit to improve their read latency. + +Although, the mechanism with which we enforce L0's file count limit may be unappealing. When the limit is reached, RocksDB intentionally delays user writes. This slows down accumulation of files in L0, and frees up resources for compacting files down to lower levels. But adding delays will significantly increase user-visible write latency jitter. + +Also, due to how L0 files can span the entire key-range, compaction parallelization is limited. Files at L0 or L1 may be locked due to involvement in pending L0->L1 or L1->L2 compactions. We can only schedule a parallel L0->L1 compaction if it does not require any of the locked files, which is typically not the case. + +To handle these constraints better, we added a new type of compaction, L0->L0. It quickly reduces file count in L0 and can be scheduled even when L1 files are locked, unlike L0->L1. We also changed the L0->L1 picking algorithm to increase opportunities for parallelism. + +### Old L0->L1 Picking Logic + +Previously, our logic for picking which L0 file to compact was the same as every other level: pick the largest file in the level. One special property of L0->L1 compaction is that files can overlap in the input level, so those overlapping files must be pulled in as well. For example, a compaction may look like this: + +![full-range.png](/static/images/compaction/full-range.png) + +This compaction pulls in every L0 and L1 file. This happens regardless of which L0 file is initially chosen as each file overlaps with every other file. + +Users may insert their data less uniformly in the key-range. For example, a database may look like this during L0->L1 compaction: + +![part-range-old.png](/static/images/compaction/part-range-old.png) + +Let's say the third file from the top is the largest, and let's say the top two files are created after the compaction started. When the compaction is picked, the fourth L0 file and six rightmost L1 files are pulled in due to overlap. Notice this leaves the database in a state where we might not be able to schedule parallel compactions. For example, if the sixth file from the top is the next largest, we can't compact it because it overlaps with the top two files, which overlap with the locked L0 files. + +We can now see the high-level problems with this approach more clearly. First, locked files in L0 or L1 prevent us from parallelizing compactions. When locked files block L0->L1 compaction, there is nothing we can do to eliminate L0 files. Second, L0->L1 compactions are relatively slow. As we saw, when keys are uniformly distributed, L0->L1 compacts two entire levels. While this is happening, new files are being flushed to L0, advancing towards the file count limit. + +### New L0->L0 Algorithm + +We introduced compaction within L0 to improve both parallelization and speed of reducing L0 file count. An L0->L0 compaction may look like this: + +![l1-l2-contend.png](/static/images/compaction/l1-l2-contend.png) + +Say the L1->L2 compaction started first. Now L0->L1 is prevented by the locked L1 file. In this case, we compact files within L0. This allows us to start the work for eliminating L0 files earlier. It also lets us do less work since we don't pull in any L1 files, whereas L0->L1 compaction would've pulled in all of them. This lets us quickly reduce L0 file count to keep read-amp low while sustaining large bursts of writes (i.e., fast accumulation of L0 files). + +The tradeoff is this increases total compaction work, as we're now compacting files without contributing towards our eventual goal of moving them towards lower levels. Our benchmarks, though, consistently show less compaction stalls and improved write throughput. One justification is that L0 file data is highly likely in page cache and/or block cache due to it being recently written and frequently accessed. So, this type of compaction is relatively cheap compared to compactions at lower levels. + +This feature is available since RocksDB 5.4. + +### New L0->L1 Picking Logic + +Recall how the old L0->L1 picking algorithm chose the largest L0 file for compaction. This didn't fit well with L0->L0 compaction, which operates on a span of files. That span begins at the newest L0 file, and expands towards older files as long as they're not being compacted. Since the largest file may be anywhere, the old L0->L1 picking logic could arbitrarily prevent us from getting a long span of files. See the second illustration in this post for a scenario where this would happen. + +So, we changed the L0->L1 picking algorithm to start from the oldest file and expand towards newer files as long as they're not being compacted. For example: + +![l0-l1-contend.png](/static/images/compaction/l0-l1-contend.png) + +Now, there can never be L0 files unreachable for L0->L0 due to L0->L1 selecting files in the middle. When longer spans of files are available for L0->L0, we perform less compaction work per deleted L0 file, thus improving efficiency. + +This feature will be available in RocksDB 5.7. + +### Performance Changes + +Mark Callaghan did the most extensive benchmarking of this feature's impact on MyRocks. See his results [here](http://smalldatum.blogspot.com/2017/05/innodb-myrocks-and-tokudb-on-insert.html). Note the primary change between his March 17 and April 14 builds is the latter performs L0->L0 compaction. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2017-06-29-rocksdb-5-5-1-released.markdown b/librocksdb-sys/rocksdb/docs/_posts/2017-06-29-rocksdb-5-5-1-released.markdown new file mode 100644 index 0000000..d785608 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2017-06-29-rocksdb-5-5-1-released.markdown @@ -0,0 +1,22 @@ +--- +title: RocksDB 5.5.1 Released! +layout: post +author: lightmark +category: blog +--- + +### New Features +* FIFO compaction to support Intra L0 compaction too with CompactionOptionsFIFO.allow_compaction=true. +* Statistics::Reset() to reset user stats. +* ldb add option --try_load_options, which will open DB with its own option file. +* Introduce WriteBatch::PopSavePoint to pop the most recent save point explicitly. +* Support dynamically change `max_open_files` option via SetDBOptions() +* Added DB::CreateColumnFamilie() and DB::DropColumnFamilies() to bulk create/drop column families. +* Add debugging function `GetAllKeyVersions` to see internal versions of a range of keys. +* Support file ingestion with universal compaction style +* Support file ingestion behind with option `allow_ingest_behind` +* New option enable_pipelined_write which may improve write throughput in case writing from multiple threads and WAL enabled. + +### Bug Fixes +* Fix the bug that Direct I/O uses direct reads for non-SST file +* Fix the bug that flush doesn't respond to fsync result diff --git a/librocksdb-sys/rocksdb/docs/_posts/2017-07-25-rocksdb-5-6-1-released.markdown b/librocksdb-sys/rocksdb/docs/_posts/2017-07-25-rocksdb-5-6-1-released.markdown new file mode 100644 index 0000000..3b54ffd --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2017-07-25-rocksdb-5-6-1-released.markdown @@ -0,0 +1,22 @@ +--- +title: RocksDB 5.6.1 Released! +layout: post +author: yiwu +category: blog +--- + +### Public API Change +* Scheduling flushes and compactions in the same thread pool is no longer supported by setting `max_background_flushes=0`. Instead, users can achieve this by configuring their high-pri thread pool to have zero threads. See https://github.com/facebook/rocksdb/wiki/Thread-Pool for more details. +* Replace `Options::max_background_flushes`, `Options::max_background_compactions`, and `Options::base_background_compactions` all with `Options::max_background_jobs`, which automatically decides how many threads to allocate towards flush/compaction. +* options.delayed_write_rate by default take the value of options.rate_limiter rate. +* Replace global variable `IOStatsContext iostats_context` with `IOStatsContext* get_iostats_context()`; replace global variable `PerfContext perf_context` with `PerfContext* get_perf_context()`. + +### New Features +* Change ticker/histogram statistics implementations to use core-local storage. This improves aggregation speed compared to our previous thread-local approach, particularly for applications with many threads. See http://rocksdb.org/blog/2017/05/14/core-local-stats.html for more details. +* Users can pass a cache object to write buffer manager, so that they can cap memory usage for memtable and block cache using one single limit. +* Flush will be triggered when 7/8 of the limit introduced by write_buffer_manager or db_write_buffer_size is triggered, so that the hard threshold is hard to hit. See https://github.com/facebook/rocksdb/wiki/Write-Buffer-Manager for more details. +* Introduce WriteOptions.low_pri. If it is true, low priority writes will be throttled if the compaction is behind. See https://github.com/facebook/rocksdb/wiki/Low-Priority-Write for more details. +* `DB::IngestExternalFile()` now supports ingesting files into a database containing range deletions. + +### Bug Fixes +* Shouldn't ignore return value of fsync() in flush. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2017-08-24-pinnableslice.markdown b/librocksdb-sys/rocksdb/docs/_posts/2017-08-24-pinnableslice.markdown new file mode 100644 index 0000000..06e0bcb --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2017-08-24-pinnableslice.markdown @@ -0,0 +1,37 @@ +--- +title: PinnableSlice; less memcpy with point lookups +layout: post +author: maysamyabandeh +category: blog +--- + +The classic API for [DB::Get](https://github.com/facebook/rocksdb/blob/9e583711144f580390ce21a49a8ceacca338fcd5/include/rocksdb/db.h#L310) receives a std::string as argument to which it will copy the value. The memcpy overhead could be non-trivial when the value is large. The [new API](https://github.com/facebook/rocksdb/blob/9e583711144f580390ce21a49a8ceacca338fcd5/include/rocksdb/db.h#L322) receives a PinnableSlice instead, which avoids memcpy in most of the cases. + +### What is PinnableSlice? + +Similarly to Slice, PinnableSlice refers to some in-memory data so it does not incur the memcpy cost. To ensure that the data will not be erased while it is being processed by the user, PinnableSlice, as its name suggests, has the data pinned in memory. The pinned data are released when PinnableSlice object is destructed or when ::Reset is invoked explicitly on it. + +### How good is it? + +Here are the improvements in throughput for an [in-memory benchmark](https://github.com/facebook/rocksdb/pull/1756#issuecomment-286201693): +* value 1k byte: 14% +* value 10k byte: 34% + +### Any limitations? + +PinnableSlice tries to avoid memcpy as much as possible. The primary gain is when reading large values from the block cache. There are however cases that it would still have to copy the data into its internal buffer. The reason is mainly the complexity of implementation and if there is enough motivation on the application side. the scope of PinnableSlice could be extended to such cases too. These include: +* Merged values +* Reads from memtables + +### How to use it? + +```cpp +PinnableSlice pinnable_val; +while (!stopped) { + auto s = db->Get(opt, cf, key, &pinnable_val); + // ... use it + pinnable_val.Reset(); // then release it immediately +} +``` + +You can also [initialize the internal buffer](https://github.com/facebook/rocksdb/blob/9e583711144f580390ce21a49a8ceacca338fcd5/include/rocksdb/db.h#L314) of PinnableSlice by passing your own string in the constructor. [simple_example.cc](https://github.com/facebook/rocksdb/blob/main/examples/simple_example.cc) demonstrates that with more examples. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2017-08-25-flushwal.markdown b/librocksdb-sys/rocksdb/docs/_posts/2017-08-25-flushwal.markdown new file mode 100644 index 0000000..751fe52 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2017-08-25-flushwal.markdown @@ -0,0 +1,26 @@ +--- +title: FlushWAL; less fwrite, faster writes +layout: post +author: maysamyabandeh +category: blog +--- + +When `DB::Put` is called, the data is written to both memtable (to be flushed to SST files later) and the WAL (write-ahead log) if it is enabled. In the case of a crash, RocksDB can recover as much as the memtable state that is reflected into the WAL. By default RocksDB automatically flushes the WAL from the application memory to the OS buffer after each `::Put`. It however can be configured to perform the flush manually after an explicit call to `::FlushWAL`. Not doing fwrite syscall after each `::Put` offers a tradeoff between reliability and write latency for the general case. As we explain below, some applications such as MyRocks benefit from this API to gain higher write throughput with however no compromise in reliability. + +### How much is the gain? + +Using `::FlushWAL` API along with setting `DBOptions.concurrent_prepare`, MyRocks achieves 40% higher throughput in Sysbench's [update-nonindex](https://github.com/akopytov/sysbench/blob/master/src/lua/oltp_update_non_index.lua) benchmark. + +### Write, Flush, and Sync + +The write to the WAL is first written to the application memory buffer. The buffer in the next step is "flushed" to OS buffer by calling fwrite syscall. The OS buffer is later "synced" to the persistent storage. The data in the OS buffer, although not persisted yet, will survive the application crash. By default, the flush occurs automatically upon each call to `DB::Put` or `DB::Write`. The user can additionally request sync after each write by setting `WriteOptions::sync`. + +### FlushWAL API + +The user can turn off the automatic flush of the WAL by setting `DBOptions::manual_wal_flush`. In that case, the WAL buffer is flushed when it is either full or `DB::FlushWAL` is called by the user. The API also accepts a boolean argument should we want to sync right after the flush: `::FlushWAL(true)`. + +### Success story: MyRocks + +Some applications that use RocksDB, already have other machinsims in place to provide reliability. MySQL for example uses 2PC (two-phase commit) to write to both binlog as well as the storage engine such as InnoDB and MyRocks. The group commit logic in MySQL allows the 1st phase (Prepare) to be run in parallel but after a commit group is formed performs the 2nd phase (Commit) in a serial manner. This makes low commit latency in the storage engine essential for achieving high throughput. The commit in MyRocks includes writing to the RocksDB WAL, which as explaiend above, by default incures the latency of flushing the WAL new appends to the OS buffer. + +Since binlog helps in recovering from some failure scenarios, MySQL can provide reliability without however needing a storage WAL flush after each individual commit. MyRocks benefits from this property, disables automatic WAL flush in RocksDB, and manually calls `::FlushWAL` when requested by MySQL. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2017-09-28-rocksdb-5-8-released.markdown b/librocksdb-sys/rocksdb/docs/_posts/2017-09-28-rocksdb-5-8-released.markdown new file mode 100644 index 0000000..a22dcaa --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2017-09-28-rocksdb-5-8-released.markdown @@ -0,0 +1,25 @@ +--- +title: RocksDB 5.8 Released! +layout: post +author: maysamyabandeh +category: blog +--- + +### Public API Change +* Users of `Statistics::getHistogramString()` will see fewer histogram buckets and different bucket endpoints. +* `Slice::compare` and BytewiseComparator `Compare` no longer accept `Slice`s containing nullptr. +* `Transaction::Get` and `Transaction::GetForUpdate` variants with `PinnableSlice` added. + +### New Features +* Add Iterator::Refresh(), which allows users to update the iterator state so that they can avoid some initialization costs of recreating iterators. +* Replace dynamic_cast<> (except unit test) so people can choose to build with RTTI off. With make, release mode is by default built with -fno-rtti and debug mode is built without it. Users can override it by setting USE_RTTI=0 or 1. +* Universal compactions including the bottom level can be executed in a dedicated thread pool. This alleviates head-of-line blocking in the compaction queue, which cause write stalling, particularly in multi-instance use cases. Users can enable this feature via `Env::SetBackgroundThreads(N, Env::Priority::BOTTOM)`, where `N > 0`. +* Allow merge operator to be called even with a single merge operand during compactions, by appropriately overriding `MergeOperator::AllowSingleOperand`. +* Add `DB::VerifyChecksum()`, which verifies the checksums in all SST files in a running DB. +* Block-based table support for disabling checksums by setting `BlockBasedTableOptions::checksum = kNoChecksum`. + +### Bug Fixes +* Fix wrong latencies in `rocksdb.db.get.micros`, `rocksdb.db.write.micros`, and `rocksdb.sst.read.micros`. +* Fix incorrect dropping of deletions during intra-L0 compaction. +* Fix transient reappearance of keys covered by range deletions when memtable prefix bloom filter is enabled. +* Fix potentially wrong file smallest key when range deletions separated by snapshot are written together. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2017-12-18-17-auto-tuned-rate-limiter.markdown b/librocksdb-sys/rocksdb/docs/_posts/2017-12-18-17-auto-tuned-rate-limiter.markdown new file mode 100644 index 0000000..d2e6204 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2017-12-18-17-auto-tuned-rate-limiter.markdown @@ -0,0 +1,28 @@ +--- +title: Auto-tuned Rate Limiter +layout: post +author: ajkr +category: blog +--- + +### Introduction + +Our rate limiter has been hard to configure since users need to pick a value that is low enough to prevent background I/O spikes, which can impact user-visible read/write latencies. Meanwhile, picking too low a value can cause memtables and L0 files to pile up, eventually leading to writes stalling. Tuning the rate limiter has been especially difficult for users whose DB instances have different workloads, or have workloads that vary over time, or commonly both. + +To address this, in RocksDB 5.9 we released a dynamic rate limiter that adjusts itself over time according to demand for background I/O. It can be enabled simply by passing `auto_tuned=true` in the `NewGenericRateLimiter()` call. In this case `rate_bytes_per_sec` will indicate the upper-bound of the window within which a rate limit will be picked dynamically. The chosen rate limit will be much lower unless absolutely necessary, so setting this to the device's maximum throughput is a reasonable choice on dedicated hosts. + +### Algorithm + +We use a simple multiplicative-increase, multiplicative-decrease algorithm. We measure demand for background I/O as the ratio of intervals where the rate limiter is drained. There are low and high watermarks for this ratio, which will trigger a change in rate limit when breached. The rate limit can move within a window bounded by the user-specified upper-bound, and a lower-bound that we derive internally. Users can expect this lower bound to be 1-2 orders of magnitude less than the provided upper-bound (so don't provide INT64_MAX as your upper-bound), although it's subject to change. + +### Benchmark Results + +Data is ingested at 10MB/s and the rate limiter was created with 1000MB/s as its upper bound. The dynamically chosen rate limit hovers around 125MB/s. The other clustering of points at 50MB/s is due to number of compaction threads being reduced to one when there's no compaction pressure. + +![](/static/images/rate-limiter/write-KBps-series.png) + +![](/static/images/rate-limiter/auto-tuned-write-KBps-series.png) + +The following graph summarizes the above two time series graphs in CDF form. In particular, notice the p90 - p100 for background write rate are significantly lower with auto-tuned rate limiter enabled. + +![](/static/images/rate-limiter/write-KBps-cdf.png) diff --git a/librocksdb-sys/rocksdb/docs/_posts/2017-12-19-write-prepared-txn.markdown b/librocksdb-sys/rocksdb/docs/_posts/2017-12-19-write-prepared-txn.markdown new file mode 100644 index 0000000..439b3f8 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2017-12-19-write-prepared-txn.markdown @@ -0,0 +1,41 @@ +--- +title: WritePrepared Transactions +layout: post +author: maysamyabandeh +category: blog +--- + +RocksDB supports both optimistic and pessimistic concurrency controls. The pessimistic transactions make use of locks to provide isolation between the transactions. The default write policy in pessimistic transactions is _WriteCommitted_, which means that the data is written to the DB, i.e., the memtable, only after the transaction is committed. This policy simplified the implementation but came with some limitations in throughput, transaction size, and variety in supported isolation levels. In the below, we explain these in detail and present the other write policies, _WritePrepared_ and _WriteUnprepared_. We then dive into the design of _WritePrepared_ transactions. + +### WriteCommitted, Pros and Cons + +With _WriteCommitted_ write policy, the data is written to the memtable only after the transaction commits. This greatly simplifies the read path as any data that is read by other transactions can be assumed to be committed. This write policy, however, implies that the writes are buffered in memory in the meanwhile. This makes memory a bottleneck for large transactions. The delay of the commit phase in 2PC (two-phase commit) also becomes noticeable since most of the work, i.e., writing to memtable, is done at the commit phase. When the commit of multiple transactions are done in a serial fashion, such as in 2PC implementation of MySQL, the lengthy commit latency becomes a major contributor to lower throughput. Moreover this write policy cannot provide weaker isolation levels, such as READ UNCOMMITTED, that could potentially provide higher throughput for some applications. + +### Alternatives: _WritePrepared_ and _WriteUnprepared_ + +To tackle the lengthy commit issue, we should do memtable writes at earlier phases of 2PC so that the commit phase become lightweight and fast. 2PC is composed of Write stage, where the transaction `::Put` is invoked, the prepare phase, where `::Prepare` is invoked (upon which the DB promises to commit the transaction if later is requested), and commit phase, where `::Commit` is invoked and the transaction writes become visible to all readers. To make the commit phase lightweight, the memtable write could be done at either `::Prepare` or `::Put` stages, resulting into _WritePrepared_ and _WriteUnprepared_ write policies respectively. The downside is that when another transaction is reading data, it would need a way to tell apart which data is committed, and if they are, whether they are committed before the transaction's start, i.e., in the read snapshot of the transaction. _WritePrepared_ would still have the issue of buffering the data, which makes the memory the bottleneck for large transactions. It however provides a good milestone for transitioning from _WriteCommitted_ to _WriteUnprepared_ write policy. Here we explain the design of _WritePrepared_ policy. We will cover the changes that make the design to also supported _WriteUnprepared_ in an upcoming post. + +### _WritePrepared_ in a nutshell + +These are the primary design questions that needs to be addressed: +1) How do we identify the key/values in the DB with transactions that wrote them? +2) How do we figure if a key/value written by transaction Txn_w is in the read snapshot of the reading transaction Txn_r? +3) How do we rollback the data written by aborted transactions? + +With _WritePrepared_, a transaction still buffers the writes in a write batch object in memory. When 2PC `::Prepare` is called, it writes the in-memory write batch to the WAL (write-ahead log) as well as to the memtable(s) (one memtable per column family); We reuse the existing notion of sequence numbers in RocksDB to tag all the key/values in the same write batch with the same sequence number, `prepare_seq`, which is also used as the identifier for the transaction. At commit time, it writes a commit marker to the WAL, whose sequence number, `commit_seq`, will be used as the commit timestamp of the transaction. Before releasing the commit sequence number to the readers, it stores a mapping from `prepare_seq` to `commit_seq` in an in-memory data structure that we call _CommitCache_. When a transaction reading values from the DB (tagged with `prepare_seq`) it makes use of the _CommitCache_ to figure if `commit_seq` of the value is in its read snapshot. To rollback an aborted transaction, we apply the status before the transaction by making another write that cancels out the writes of the aborted transaction. + +The _CommitCache_ is a lock-free data structure that caches the recent commit entries. Looking up the entries in the cache must be enough for almost all th transactions that commit in a timely manner. When evicting the older entries from the cache, it still maintains some other data structures to cover the corner cases for transactions that takes abnormally too long to finish. We will cover them in the design details below. + +### Benchmark Results +Here we presents the improvements observed in MyRocks with sysbench and linkbench: +* benchmark...........tps.........p95 latency....cpu/query +* insert...................68% +* update-noindex...30%......38% +* update-index.......61%.......28% +* read-write............6%........3.5% +* read-only...........-1.2%.....-1.8% +* linkbench.............1.9%......+overall........0.6% + +Here are also the detailed results for [In-Memory Sysbench](https://gist.github.com/maysamyabandeh/bdb868091b2929a6d938615fdcf58424) and [SSD Sysbench](https://gist.github.com/maysamyabandeh/ff94f378ab48925025c34c47eff99306) curtesy of [@mdcallag](https://github.com/mdcallag). + +Learn more [here](https://github.com/facebook/rocksdb/wiki/WritePrepared-Transactions). diff --git a/librocksdb-sys/rocksdb/docs/_posts/2018-02-05-rocksdb-5-10-2-released.markdown b/librocksdb-sys/rocksdb/docs/_posts/2018-02-05-rocksdb-5-10-2-released.markdown new file mode 100644 index 0000000..9f32d3f --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2018-02-05-rocksdb-5-10-2-released.markdown @@ -0,0 +1,22 @@ +--- +title: RocksDB 5.10.2 Released! +layout: post +author: siying +category: blog +--- + +### Public API Change +* When running `make` with environment variable `USE_SSE` set and `PORTABLE` unset, will use all machine features available locally. Previously this combination only compiled SSE-related features. + +### New Features +* CRC32C is now using the 3-way pipelined SSE algorithm `crc32c_3way` on supported platforms to improve performance. The system will choose to use this algorithm on supported platforms automatically whenever possible. If PCLMULQDQ is not supported it will fall back to the old Fast_CRC32 algorithm. +* Provide lifetime hints when writing files on Linux. This reduces hardware write-amp on storage devices supporting multiple streams. +* Add a DB stat, `NUMBER_ITER_SKIP`, which returns how many internal keys were skipped during iterations (e.g., due to being tombstones or duplicate versions of a key). +* Add PerfContext counters, `key_lock_wait_count` and `key_lock_wait_time`, which measure the number of times transactions wait on key locks and total amount of time waiting. + +### Bug Fixes +* Fix IOError on WAL write doesn't propagate to write group follower +* Make iterator invalid on merge error. +* Fix performance issue in `IngestExternalFile()` affecting databases with large number of SST files. +* Fix possible corruption to LSM structure when `DeleteFilesInRange()` deletes a subset of files spanned by a `DeleteRange()` marker. +* Fix DB::Flush() keep waiting after flush finish under certain condition. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2018-08-01-rocksdb-tuning-advisor.markdown b/librocksdb-sys/rocksdb/docs/_posts/2018-08-01-rocksdb-tuning-advisor.markdown new file mode 100644 index 0000000..ff9b1e4 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2018-08-01-rocksdb-tuning-advisor.markdown @@ -0,0 +1,58 @@ +--- +title: Rocksdb Tuning Advisor +layout: post +author: poojam23 +category: blog +--- + +The performance of Rocksdb is contingent on its tuning. However, because +of the complexity of its underlying technology and a large number of +configurable parameters, a good configuration is sometimes hard to obtain. The aim of +the python command-line tool, Rocksdb Advisor, is to automate the process of +suggesting improvements in the configuration based on advice from Rocksdb +experts. + +### Overview + +Experts share their wisdom as rules comprising of conditions and suggestions in the INI format (refer +[rules.ini](https://github.com/facebook/rocksdb/blob/main/tools/advisor/advisor/rules.ini)). +Users provide the Rocksdb configuration that they want to improve upon (as the +familiar Rocksdb OPTIONS file — +[example](https://github.com/facebook/rocksdb/blob/main/examples/rocksdb_option_file_example.ini)) +and the path of the file which contains Rocksdb logs and statistics. +The [Advisor](https://github.com/facebook/rocksdb/blob/main/tools/advisor/advisor/rule_parser_example.py) +creates appropriate DataSource objects (for Rocksdb +[logs](https://github.com/facebook/rocksdb/blob/main/tools/advisor/advisor/db_log_parser.py), +[options](https://github.com/facebook/rocksdb/blob/main/tools/advisor/advisor/db_options_parser.py), +[statistics](https://github.com/facebook/rocksdb/blob/main/tools/advisor/advisor/db_stats_fetcher.py) etc.) +and provides them to the [Rules Engine](https://github.com/facebook/rocksdb/blob/main/tools/advisor/advisor/rule_parser.py). +The Rules uses rules from experts to parse data-sources and trigger appropriate rules. +The Advisor's output gives information about which rules were triggered, +why they were triggered and what each of them suggests. Each suggestion +provided by a triggered rule advises some action on a Rocksdb +configuration option, for example, increase CFOptions.write_buffer_size, +set bloom_bits to 2 etc. + +### Usage + +An example command to run the tool: + +```shell +cd rocksdb/tools/advisor +python3 -m advisor.rule_parser_example --rules_spec=advisor/rules.ini --rocksdb_options=test/input_files/OPTIONS-000005 --log_files_path_prefix=test/input_files/LOG-0 --stats_dump_period_sec=20 +``` + +Sample output where a Rocksdb log-based rule has been triggered : + +```shell +Rule: stall-too-many-memtables +LogCondition: stall-too-many-memtables regex: Stopping writes because we have \d+ immutable memtables \(waiting for flush\), max_write_buffer_number is set to \d+ +Suggestion: inc-bg-flush option : DBOptions.max_background_flushes action : increase suggested_values : ['2'] +Suggestion: inc-write-buffer option : CFOptions.max_write_buffer_number action : increase +scope: col_fam: +{'default'} +``` + +### Read more + +For more information, refer to [advisor](https://github.com/facebook/rocksdb/tree/main/tools/advisor/README.md). diff --git a/librocksdb-sys/rocksdb/docs/_posts/2018-08-23-data-block-hash-index.markdown b/librocksdb-sys/rocksdb/docs/_posts/2018-08-23-data-block-hash-index.markdown new file mode 100644 index 0000000..c4b24ec --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2018-08-23-data-block-hash-index.markdown @@ -0,0 +1,118 @@ +--- +title: Improving Point-Lookup Using Data Block Hash Index +layout: post +author: fgwu +category: blog +--- +We've designed and implemented a _data block hash index_ in RocksDB that has the benefit of both reducing the CPU util and increasing the throughput for point lookup queries with a reasonable and tunable space overhead. + +Specifially, we append a compact hash table to the end of the data block for efficient indexing. It is backward compatible with the data base created without this feature. After turned on the hash index feature, existing data will be gradually converted to the hash index format. + +Benchmarks with `db_bench` show the CPU utilization of one of the main functions in the point lookup code path, `DataBlockIter::Seek()`, is reduced by 21.8%, and the overall RocksDB throughput is increased by 10% under purely cached workloads, at an overhead of 4.6% more space. Shadow testing with Facebook production traffic shows good CPU improvements too. + + +### How to use it +Two new options are added as part of this feature: `BlockBasedTableOptions::data_block_index_type` and `BlockBasedTableOptions::data_block_hash_table_util_ratio`. + +The hash index is disabled by default unless `BlockBasedTableOptions::data_block_index_type` is set to `data_block_index_type = kDataBlockBinaryAndHash`. The hash table utilization ratio is adjustable using `BlockBasedTableOptions::data_block_hash_table_util_ratio`, which is valid only if `data_block_index_type = kDataBlockBinaryAndHash`. + + +``` +// the definitions can be found in include/rocksdb/table.h + +// The index type that will be used for the data block. +enum DataBlockIndexType : char { + kDataBlockBinarySearch = 0, // traditional block type + kDataBlockBinaryAndHash = 1, // additional hash index +}; + +// Set to kDataBlockBinaryAndHash to enable hash index +DataBlockIndexType data_block_index_type = kDataBlockBinarySearch; + +// #entries/#buckets. It is valid only when data_block_hash_index_type is +// kDataBlockBinaryAndHash. +double data_block_hash_table_util_ratio = 0.75; + +``` + + +### Data Block Hash Index Design + +Current data block format groups adjacent keys together as a restart interval. One block consists of multiple restart intervals. The byte offset of the beginning of each restart interval, i.e. a restart point, is stored in an array called restart interval index or binary seek index. RocksDB does a binary search when performing point lookup for keys in data blocks to find the right restart interval the key may reside. We will use binary seek and binary search interchangeably in this post. + +In order to find the right location where the key may reside using binary search, multiple key parsing and comparison are needed. Each binary search branching triggers CPU cache miss, causing much CPU utilization. We have seen that this binary search takes up considerable CPU in production use-cases. + +![](/static/images/data-block-hash-index/block-format-binary-seek.png) + +We implemented a hash map at the end of the block to index the key to reduce the CPU overhead of the binary search. The hash index is just an array of pointers pointing into the binary seek index. + +![](/static/images/data-block-hash-index/block-format-hash-index.png) + + +Each array element is considered as a hash bucket when storing the location of a key (or more precisely, the restart index of the restart interval where the key resides). When multiple keys happen to hash into the same bucket (hash collision), we just mark the bucket as “collision”. So that when later querying on that key, the hash table lookup knows that there was a hash collision happened so it can fall back to the traditional binary search to find the location of the key. + +We define hash table utilization ratio as the #keys/#buckets. If a utilization ratio is 0.5 and there are 100 buckets, 50 keys are stored in the bucket. The less the util ratio, the less hash collision, and the less chance for a point lookup falls back to binary seek (fall back ratio) due to the collision. So a small util ratio has more benefit to reduce the CPU time but introduces more space overhead. + +Space overhead depends on the util ratio. Each bucket is a `uint8_t` (i.e. one byte). For a util ratio of 1, the space overhead is 1Byte per key, the fall back ratio observed is ~52%. + +![](/static/images/data-block-hash-index/hash-index-data-structure.png) + +### Things that Need Attention + +**Customized Comparator** + +Hash index will hash different keys (keys with different content, or byte sequence) into different hash values. This assumes the comparator will not treat different keys as equal if they have different content. + +The default bytewise comparator orders the keys in alphabetical order and works well with hash index, as different keys will never be regarded as equal. However, some specially crafted comparators will do. For example, say, a `StringToIntComparator` can convert a string into an integer, and use the integer to perform the comparison. Key string “16” and “0x10” is equal to each other as seen by this `StringToIntComparator`, but they probably hash to different value. Later queries to one form of the key will not be able to find the existing key been stored in the other format. + +We add a new function member to the comparator interface: + +``` +virtual bool CanKeysWithDifferentByteContentsBeEqual() const { return true; } +``` + + +Every comparator implementation should override this function and specify the behavior of the comparator. If a comparator can regard different keys equal, the function returns true, and as a result the hash index feature will not be enabled, and vice versa. + +NOTE: to use the hash index feature, one should 1) have a comparator that can never treat different keys as equal; and 2) override the `CanKeysWithDifferentByteContentsBeEqual()` function to return `false`, so the hash index can be enabled. + + +**Util Ratio's Impact on Data Block Cache** + +Adding the hash index to the end of the data block essentially takes up the data block cache space, making the effective data block cache size smaller and increasing the data block cache miss ratio. Therefore, a very small util ratio will result in a large data block cache miss ratio, and the extra I/O may drag down the throughput gain achieved by the hash index lookup. Besides, when compression is enabled, cache miss also incurs data block decompression, which is CPU-consuming. Therefore the CPU may even increase if using a too small util ratio. The best util ratio depends on workloads, cache to data ratio, disk bandwidth/latency etc. In our experiment, we found util ratio = 0.5 ~ 1 is a good range to explore that brings both CPU and throughput gains. + + +### Limitations + +As we use `uint8_t` to store binary seek index, i.e. restart interval index, the total number of restart intervals cannot be more than 253 (we reserved 255 and 254 as special flags). For blocks having a larger number of restart intervals, the hash index will not be created and the point lookup will be done by traditional binary seek. + +Data block hash index only supports point lookup. We do not support range lookup. Range lookup request will fall back to BinarySeek. + +RocksDB supports many types of records, such as `Put`, `Delete`, `Merge`, etc (visit [here](https://github.com/facebook/rocksdb/wiki/rocksdb-basics) for more information). Currently we only support `Put` and `Delete`, but not `Merge`. Internally we have a limited set of supported record types: + + +``` +kPutRecord, <=== supported +kDeleteRecord, <=== supported +kSingleDeleteRecord, <=== supported +kTypeBlobIndex, <=== supported +``` + +For records not supported, the searching process will fall back to the traditional binary seek. + + + +### Evaluation +To evaluate the CPU util reduction and isolate other factors such as disk I/O and block decompression, we first evaluate the hash idnex in a purely cached workload. We observe that the CPU utilization of one of the main functions in the point lookup code path, DataBlockIter::Seek(), is reduced by 21.8% and the overall throughput is increased by 10% at an overhead of 4.6% more space. + +However, general worload is not always purely cached. So we also evaluate the performance under different cache space pressure. In the following test, we use `db_bench` with RocksDB deployed on SSDs. The total DB size is 5~6GB, and it is about 14GB if decompressed. Different block cache sizes are used, ranging from 14GB down to 2GB, with an increasing cache miss ratio. + +Orange bars are representing our hash index performance. We use a hash util ratio of 1.0 in this test. Block size are set to 16KiB with the restart interval as 16. + +![](/static/images/data-block-hash-index/perf-throughput.png) +![](/static/images/data-block-hash-index/perf-cache-miss.png) + +We can see that if cache size is greater than 8GB, hash index can bring throughput gain. Cache size greater than 8GB can be translated to a cache miss ratio smaller than 40%. So if the workload has a cache miss ratio smaller than 40%, hash index is able to increase the throughput. + +Besides, shadow testing with Facebook production traffic shows good CPU improvements too. + diff --git a/librocksdb-sys/rocksdb/docs/_posts/2018-11-21-delete-range.markdown b/librocksdb-sys/rocksdb/docs/_posts/2018-11-21-delete-range.markdown new file mode 100644 index 0000000..96fc356 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2018-11-21-delete-range.markdown @@ -0,0 +1,292 @@ +--- +title: "DeleteRange: A New Native RocksDB Operation" +layout: post +author: +- abhimadan +- ajkr +category: blog +--- +## Motivation + +### Deletion patterns in LSM + +Deleting a range of keys is a common pattern in RocksDB. Most systems built on top of +RocksDB have multi-component key schemas, where keys sharing a common prefix are +logically related. Here are some examples. + +MyRocks is a MySQL fork using RocksDB as its storage engine. Each key's first +four bytes identify the table or index to which that key belongs. Thus dropping +a table or index involves deleting all the keys with that prefix. + +Rockssandra is a Cassandra variant that uses RocksDB as its storage engine. One +of its admin tool commands, `nodetool cleanup`, removes key-ranges that have been migrated +to other nodes in the cluster. + +Marketplace uses RocksDB to store product data. Its key begins with product ID, +and it stores various data associated with the product in separate keys. When a +product is removed, all these keys must be deleted. + +When we decide what to improve, we try to find a use case that's common across +users, since we want to build a generally useful system, not one that has many +one-off features for individual users. The range deletion pattern is common as +illustrated above, so from this perspective it's a good target for optimization. + +### Existing mechanisms: challenges and opportunities + +The most common pattern we see is scan-and-delete, i.e., advance an iterator +through the to-be-deleted range, and issue a `Delete` for each key. This is +slow (involves read I/O) so cannot be done in any critical path. Additionally, +it creates many tombstones, which slows down iterators and doesn't offer a deadline +for space reclamation. + +Another common pattern is using a custom compaction filter that drops keys in +the deleted range(s). This deletes the range asynchronously, so cannot be used +in cases where readers must not see keys in deleted ranges. Further, it has the +disadvantage of outputting tombstones to all but the bottom level. That's +because compaction cannot detect whether dropping a key would cause an older +version at a lower level to reappear. + +If space reclamation time is important, or it is important that the deleted +range not affect iterators, the user can trigger `CompactRange` on the deleted +range. This can involve arbitrarily long waits in the compaction queue, and +increases write-amp. By the time it's finished, however, the range is completely +gone from the LSM. + +`DeleteFilesInRange` can be used prior to compacting the deleted range as long +as snapshot readers do not need to access them. It drops files that are +completely contained in the deleted range. That saves write-amp because, in +`CompactRange`, the file data would have to be rewritten several times before it +reaches the bottom of the LSM, where tombstones can finally be dropped. + +In addition to the above approaches having various drawbacks, they are quite +complicated to reason about and implement. In an ideal world, deleting a range +of keys would be (1) simple, i.e., a single API call; (2) synchronous, i.e., +when the call finishes, the keys are guaranteed to be wiped from the DB; (3) low +latency so it can be used in critical paths; and (4) a first-class operation +with all the guarantees of any other write, like atomicity, crash-recovery, etc. + +## v1: Getting it to work + +### Where to persist them? + +The first place we thought about storing them is inline with the data blocks. +We could not think of a good way to do it, however, since the start of a range +tombstone covering a key could be anywhere, making binary search impossible. +So, we decided to investigate segregated storage. + +A second solution we considered is appending to the manifest. This file is +append-only, periodically compacted, and stores metadata like the level to which +each SST belongs. This is tempting because it leverages an existing file, which +is maintained in the background and fully read when the DB is opened. However, +it conceptually violates the manifest's purpose, which is to store metadata. It +also has no way to detect when a range tombstone no longer covers anything and +is droppable. Further, it'd be possible for keys above a range tombstone to disappear +when they have their seqnums zeroed upon compaction to the bottommost level. + +A third candidate is using a separate column family. This has similar problems +to the manifest approach. That is, we cannot easily detect when a range +tombstone is obsolete, and seqnum zeroing can cause a key +to go from above a range tombstone to below, i.e., disappearing. The upside is +we can reuse logic for memory buffering, consistent reads/writes, etc. + +The problems with the second and third solutions indicate a need for range +tombstones to be aware of flush/compaction. An easy way to achieve this is put +them in the SST files themselves - but not in the data blocks, as explained for +the first solution. So, we introduced a separate meta-block for range tombstones. +This resolved the problem of when to obsolete range tombstones, as it's simple: +when they're compacted to the bottom level. We also reused the LSM invariants +that newer versions of a key are always in a higher level to prevent the seqnum +zeroing problem. This approach has the side benefit of constraining the range +tombstones seen during reads to ones in a similar key-range. + +![](/static/images/delrange/delrange_sst_blocks.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +*When there are range tombstones in an SST, they are segregated in a separate meta-block* +{: style="text-align: center"} + +![](/static/images/delrange/delrange_key_schema.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +*Logical range tombstones (left) and their corresponding physical key-value representation (right)* +{: style="text-align: center"} + +### Write path + +`WriteBatch` stores range tombstones in its buffer which are logged to the WAL and +then applied to a dedicated range tombstone memtable during `Write`. Later in +the background the range tombstone memtable and its corresponding data memtable +are flushed together into a single SST with a range tombstone meta-block. SSTs +periodically undergo compaction which rewrites SSTs with point data and range +tombstones dropped or merged wherever possible. + +We chose to use a dedicated memtable for range tombstones. The memtable +representation is always skiplist in order to minimize overhead in the usual +case, which is the memtable contains zero or a small number of range tombstones. +The range tombstones are segregated to a separate memtable for the same reason +we segregated range tombstones in SSTs. That is, we did not know how to +interleave the range tombstone with point data in a way that we would be able to +find it for arbitrary keys that it covers. + +![](/static/images/delrange/delrange_write_path.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 70%"} + +*Lifetime of point keys and range tombstones in RocksDB* +{: style="text-align: center"} + +During flush and compaction, we chose to write out all non-obsolete range +tombstones unsorted. Sorting by a single dimension is easy to implement, but +doesn't bring asymptotic improvement to queries over range data. Ideally, we +want to store skylines (see “Read Path” subsection below) computed over our ranges so we can binary search. +However, a couple of concerns cause doing this in flush and compaction to feel +unsatisfactory: (1) we need to store multiple skylines, one for each snapshot, +which further complicates the range tombstone meta-block encoding; and (2) even +if we implement this, the range tombstone memtable still needs to be linearly +scanned. Given these concerns we decided to defer collapsing work to the read +side, hoping a good caching strategy could optimize this at some future point. + + +### Read path + +In point lookups, we aggregate range tombstones in an unordered vector as we +search through live memtable, immutable memtables, and then SSTs. When a key is +found that matches the lookup key, we do a scan through the vector, checking +whether the key is deleted. + +In iterators, we aggregate range tombstones into a skyline as we visit live +memtable, immutable memtables, and SSTs. The skyline is expensive to construct but fast to determine whether a key is covered. The skyline keeps track of the most recent range tombstone found to optimize `Next` and `Prev`. + +|![](/static/images/delrange/delrange_uncollapsed.png) |![](/static/images/delrange/delrange_collapsed.png) | + +*([Image source: Leetcode](https://leetcode.com/problems/the-skyline-problem/description/)) The skyline problem involves taking building location/height data in the +unsearchable form of A and converting it to the form of B, which is +binary-searchable. With overlapping range tombstones, to achieve efficient +searching we need to solve an analogous problem, where the x-axis is the +key-space and the y-axis is the sequence number.* +{: style="text-align: center"} + +### Performance characteristics + +For the v1 implementation, writes are much faster compared to the scan and +delete (optionally within a transaction) pattern. `DeleteRange` only logs to WAL +and applies to memtable. Logging to WAL always `fflush`es, and optionally +`fsync`s or `fdatasync`s. Applying to memtable is always an in-memory operation. +Since range tombstones have a dedicated skiplist memtable, the complexity of inserting is O(log(T)), where T is the number of existing buffered range tombstones. + +Reading in the presence of v1 range tombstones, however, is much slower than reads +in a database where scan-and-delete has happened, due to the linear scan over +range tombstone memtables/meta-blocks. + +Iterating in a database with v1 range tombstones is usually slower than in a +scan-and-delete database, although the gap lessens as iterations grow longer. +When an iterator is first created and seeked, we construct a skyline over its +tombstones. This operation is O(T\*log(T)) where T is the number of tombstones +found across live memtable, immutable memtable, L0 files, and one file from each +of the L1+ levels. However, moving the iterator forwards or backwards is simply +a constant-time operation (excluding edge cases, e.g., many range tombstones +between consecutive point keys). + +## v2: Making it fast + +`DeleteRange`’s negative impact on read perf is a barrier to its adoption. The +root cause is range tombstones are not stored or cached in a format that can be +efficiently searched. We needed to design DeleteRange so that we could maintain +write performance while making read performance competitive with workarounds +used in production (e.g., scan-and-delete). + +### Representations + +The key idea of the redesign is that, instead of globally collapsing range tombstones, + we can locally “fragment” them for each SST file and memtable to guarantee that: + +* no range tombstones overlap; and +* range tombstones are ordered by start key. + +Combined, these properties make range tombstones binary searchable. This + fragmentation will happen on the read path, but unlike the previous design, we can + easily cache many of these range tombstone fragments on the read path. + +### Write path + +The write path remains unchanged. + +### Read path + +When an SST file is opened, its range tombstones are fragmented and cached. For point + lookups, we binary search each file's fragmented range tombstones for one that covers + the lookup key. Unlike the old design, once we find a tombstone, we no longer need to + search for the key in lower levels, since we know that any keys on those levels will be + covered (though we do still check the current level since there may be keys written after + the range tombstone). + +For range scans, we create iterators over all the fragmented range + tombstones and store them in a list, seeking each one to cover the start key of the range + scan (if possible), and query each encountered key in this structure as in the old design, + advancing range tombstone iterators as necessary. In effect, we implicitly create a skyline. + This requires significantly less work on iterator creation, but since each memtable/SST has +its own range tombstone iterator, querying range tombstones requires key comparisons (and +possibly iterator increments) for several iterators (as opposed to v1, where we had a global +collapsed representation of all range tombstones). As a result, very long range scans may become + slower than before, but short range scans are an order of magnitude faster, which are the + more common class of range scan. + +## Benchmarks + +To understand the performance of this new design, we used `db_bench` to compare point lookup, short range scan, + and long range scan performance across: + +* the v1 DeleteRange design, +* the scan-and-delete workaround, and +* the v2 DeleteRange design. + +In these benchmarks, we used a database with 5 million data keys, and 10000 range tombstones (ignoring +those dropped during compaction) that were written in regular intervals after 4.5 million data keys were written. +Writing the range tombstones ensures that most of them are not compacted away, and we have more tombstones +in higher levels that cover keys in lower levels, which allows the benchmarks to exercise more interesting behavior +when reading deleted keys. + +Point lookup benchmarks read 100000 keys from a database using `readwhilewriting`. Range scan benchmarks used +`seekrandomwhilewriting` and seeked 100000 times, and advanced up to 10 keys away from the seek position for short range scans, and advanced up to 1000 keys away from the seek position for long range scans. + +The results are summarized in the tables below, averaged over 10 runs (note the +different SHAs for v1 benchmarks are due to a new `db_bench` flag that was added in order to compare performance with databases with no tombstones; for brevity, those results are not reported here). Also note that the block cache was large enough to hold the entire db, so the large throughput is due to limited I/Os and little time spent on decompression. The range tombstone blocks are always pinned uncompressed in memory. We believe these setup details should not affect relative performance between versions. + +### Point Lookups + +|Name |SHA |avg micros/op |avg ops/sec | +|v1 |35cd754a6 |1.3179 |759,830.90 | +|scan-del |7528130e3 |0.6036 |1,667,237.70 | +|v2 |7528130e3 |0.6128 |1,634,633.40 | + +### Short Range Scans + +|Name |SHA |avg micros/op |avg ops/sec | +|v1 |0ed738fdd |6.23 |176,562.00 | +|scan-del |PR 4677 |2.6844 |377,313.00 | +|v2 |PR 4677 |2.8226 |361,249.70 | + +### Long Range scans + +|Name |SHA |avg micros/op |avg ops/sec | +|v1 |0ed738fdd |52.7066 |19,074.00 | +|scan-del |PR 4677 |38.0325 |26,648.60 | +|v2 |PR 4677 |41.2882 |24,714.70 | + +## Future Work + +Note that memtable range tombstones are fragmented every read; for now this is acceptable, + since we expect there to be relatively few range tombstones in memtables (and users can + enforce this by keeping track of the number of memtable range deletions and manually flushing + after it passes a threshold). In the future, a specialized data structure can be used for storing + range tombstones in memory to avoid this work. + +Another future optimization is to create a new format version that requires range tombstones to + be stored in a fragmented form. This would save time when opening SST files, and when `max_open_files` +is not -1 (i.e., files may be opened several times). + +## Acknowledgements + +Special thanks to Peter Mattis and Nikhil Benesch from Cockroach Labs, who were early users of +DeleteRange v1 in production, contributed the cleanest/most efficient v1 aggregation implementation, found and fixed bugs, and provided initial DeleteRange v2 design and continued help. + +Thanks to Huachao Huang and Jinpeng Zhang from PingCAP for early DeleteRange v1 adoption, bug reports, and fixes. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2019-03-08-format-version-4.markdown b/librocksdb-sys/rocksdb/docs/_posts/2019-03-08-format-version-4.markdown new file mode 100644 index 0000000..ce65769 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2019-03-08-format-version-4.markdown @@ -0,0 +1,36 @@ +--- +title: format_version 4 +layout: post +author: maysamyabandeh +category: blog +--- + +The data blocks in RocksDB consist of a sequence of key/values pairs sorted by key, where the pairs are grouped into _restart intervals_ specified by `block_restart_interval`. Up to RocksDB version 5.14, where the latest and default value of `BlockBasedTableOptions::format_version` is 2, the format of index and data blocks are the same: index blocks use the same key format of <`user_key`,`seq`> and encode pointers to data blocks, <`offset`,`size`>, to a byte string and use them as values. The only difference is that the index blocks use `index_block_restart_interval` for the size of _restart intervals_. `format_version=`3,4 offer more optimized, backward-compatible, yet forward-incompatible format for index blocks. + +### Pros + +Using `format_version`=4 significantly reduces the index block size, in some cases around 4-5x. This frees more space in block cache, which would result in higher hit rate for data and filter blocks, or offer the same performance with a smaller block cache size. + +### Cons + +Being _forward-incompatible_ means that if you enable `format_version=`4 you cannot downgrade to a RocksDB version lower than 5.16. + +### How to use it? + +- `BlockBasedTableOptions::format_version` = 4 +- `BlockBasedTableOptions::index_block_restart_interval` = 16 + +### What is format_version 3? +(Since RocksDB 5.15) In most cases, the sequence number `seq` is not necessary for keys in the index blocks. In such cases, `format_version`=3 skips encoding the sequence number and sets `index_key_is_user_key` in TableProperties, which is used by the reader to know how to decode the index block. + +### What is format_version 4? +(Since RocksDB 5.16) Changes the format of index blocks by delta encoding the index values, which are the block handles. This saves the encoding of `BlockHandle::offset` of the non-head index entries in each restart interval. If used, `TableProperties::index_value_is_delta_encoded` is set, which is used by the reader to know how to decode the index block. The format of each key is (shared_size, non_shared_size, shared, non_shared). The format of each value, i.e., block handle, is (offset, size) whenever the shared_size is 0, which included the first entry in each restart point. Otherwise the format is delta-size = block handle size - size of last block handle. + +The index format in `format_version=4` would be as follows: + + restart_point 0: k, v (off, sz), k, v (delta-sz), ..., k, v (delta-sz) + restart_point 1: k, v (off, sz), k, v (delta-sz), ..., k, v (delta-sz) + ... + restart_point n-1: k, v (off, sz), k, v (delta-sz), ..., k, v (delta-sz) + where, k is key, v is value, and its encoding is in parenthesis. + diff --git a/librocksdb-sys/rocksdb/docs/_posts/2019-08-15-unordered-write.markdown b/librocksdb-sys/rocksdb/docs/_posts/2019-08-15-unordered-write.markdown new file mode 100644 index 0000000..5f0eb28 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2019-08-15-unordered-write.markdown @@ -0,0 +1,56 @@ +--- +title: Higher write throughput with `unordered_write` feature +layout: post +author: maysamyabandeh +category: blog +--- + +Since RocksDB 6.3, The `unordered_write=`true option together with WritePrepared transactions offers 34-42% higher write throughput compared to vanilla RocksDB. If the application can handle more relaxed ordering guarantees, the gain in throughput would increase to 63-131%. + +### Background + +Currently RocksDB API delivers the following powerful guarantees: +- Atomic reads: Either all of a write batch is visible to reads or none of it. +- Read-your-own writes: When a write thread returns to the user, a subsequent read by the same thread will be able to see its own writes. +- Immutable Snapshots: The reads visible to the snapshot are immutable in the sense that it will not be affected by any in-flight or future writes. + +### `unordered_write` + +The `unordered_write` feature, when turned on, relaxes the default guarantees of RocksDB. While it still gives read-your-own-write property, neither atomic reads nor the immutable snapshot properties are provided any longer. However, RocksDB users could still get read-your-own-write and immutable snapshots when using this feature in conjunction with TransactionDB configured with WritePrepared transactions and `two_write_queues`. You can read [here](https://github.com/facebook/rocksdb/wiki/unordered_write) to learn about the design of `unordered_write` and [here](https://github.com/facebook/rocksdb/wiki/WritePrepared-Transactions) to learn more about WritePrepared transactions. + +### How to use it? + +To get the same guarantees as vanilla RocksdB: + + DBOptions db_options; + db_options.unordered_write = true; + db_options.two_write_queues = true; + DB* db; + { + TransactionDBOptions txn_db_options; + txn_db_options.write_policy = TxnDBWritePolicy::WRITE_PREPARED; + txn_db_options.skip_concurrency_control = true; + TransactionDB* txn_db; + TransactionDB::Open(options, txn_db_options, kDBPath, &txn_db); + db = txn_db; + } + db->Write(...); + +To get relaxed guarantees: + + DBOptions db_options; + db_options.unordered_write = true; + DB* db; + DB::Open(db_options, kDBPath, &db); + db->Write(...); + +# Benchmarks + + TEST_TMPDIR=/dev/shm/ ~/db_bench --benchmarks=fillrandom --threads=32 --num=10000000 -max_write_buffer_number=16 --max_background_jobs=64 --batch_size=8 --writes=3000000 -level0_file_num_compaction_trigger=99999 --level0_slowdown_writes_trigger=99999 --level0_stop_writes_trigger=99999 -enable_pipelined_write=false -disable_auto_compactions --transaction_db=true --unordered_write=1 --disable_wal=0 + +Throughput with `unordered_write`=true and using WritePrepared transaction: +- WAL: +42% +- No-WAL: +34% +Throughput with `unordered_write`=true +- WAL: +63% +- NoWAL: +131% diff --git a/librocksdb-sys/rocksdb/docs/_posts/2021-04-12-universal-improvements.markdown b/librocksdb-sys/rocksdb/docs/_posts/2021-04-12-universal-improvements.markdown new file mode 100644 index 0000000..fa4e9d4 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2021-04-12-universal-improvements.markdown @@ -0,0 +1,46 @@ +--- +title: (Call For Contribution) Make Universal Compaction More Incremental +layout: post +author: sdong +category: blog +--- + +### Motivation + +Universal Compaction is an important compaction style, but few changes were made after we made the structure multi-leveled. Yet the major restriction of always compacting full sorted run is not relaxed. Compared to Leveled Compaction, where we usually only compile several SST files together, in universal compaction, we frequently compact GBs of data. Two issues with this gap: 1. it makes it harder to unify universal and leveled compaction; 2. periodically data is fully compacted, and in the mean time space is doubled. To ease the problem, we can break the restriction and do similar as leveled compaction, and bring it closer to unified compaction. + +We call for help for making following improvements. + + +### How Universal Compaction Works + +In universal, whole levels are compacted together to satisfy two conditions (See [wiki page](https://github.com/facebook/rocksdb/wiki/Universal-Compaction) for more details): + +1. total size / bottommost level size > a threshold, or +2. total number of sorted runs (non-0 levels + L0 files) is within a threshold + +1 is to limit extra space overhead used for dead data and 2 is for read performance. + +If 1 is triggered, likely a full compaction will be triggered. If 2 is triggered, RocksDB compact some sorted runs to bring the number down. It does it by using a simple heuristic so that less writes needed for that purpose over time: it starts from compacting smaller files, but if total size to compact is similar to or larger than size of the next level, it will take that level together, as soon on (whether it is the best heuristic is another question and we’ve never seriously looked at it). + +### How We Can Improve? + +Let’s start from condition 1. Here we do full compaction but is not necessary. A simple optimization would be to compact so that just enough files are merged into the bottommost level (Lmax) to satisfy condition 1. It would work if we only need to pick some files from Lmax-1, or if it is cheaper over time, we can pick some files from other levels too. + +Then condition 2. If we finish condition 1, there might be holes in some ranges in older levels. These holes might make it possible that only by compacting some sub ranges, we can fix the LSM-tree for condition 2. RocksDB can take single files into consideration and apply more sophisticated heuristic. + +This new approach makes universal compaction closer to leveled compaction. The operation for 1 is closer to how Leveled compaction triggeres Lmax-1 to Lmax compaction. And 2 can potentially be implemented as something similar to level picking in Leveled Compaction. In fact, all those file picking can co-existing in one single compaction style and there isn’t fundamental conflicts to that. + +### Limitation + +There are two limitations: + +* Periodic automatic full compaction is unpleasant but at the same time is pleasant in another way. Some users might uses it to reason that everything is periodically collapsed so dead data is gone and old data is rewritten. We need to make sure periodic compaction works to continue with that. +* L0 to the first non-L0 level compaction is the first time data is partitioned in LSM-tree so that incremental compaction by range is possible. We might need to do more of these compactions in order to make incremental possible, which will increase compaction slightly. +* Compacting subset of a level would introduce some extra overhead for unaligned files, just as in leveled compaction. More SST boundary cutting heuristic can reduce this overhead but it will be there. + +But I believe the benefits would outweight the limitations. Reducing temporary space doubling and moving towards to unified compaction would be important achievements. + +### Interested in Help? + +Compaction is the core of LSM-tree, but its improvements are far overdue. If you are a user of universal compaction and would be able to benefit from those improvements, we will be happy to work with you on speeding up the project and bring them to RocksDB sooner. Feel free to communicate with us in [this issue](https://github.com/facebook/rocksdb/issues/8181). diff --git a/librocksdb-sys/rocksdb/docs/_posts/2021-05-26-integrated-blob-db.markdown b/librocksdb-sys/rocksdb/docs/_posts/2021-05-26-integrated-blob-db.markdown new file mode 100644 index 0000000..9f3a22f --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2021-05-26-integrated-blob-db.markdown @@ -0,0 +1,101 @@ +--- +title: Integrated BlobDB +layout: post +author: ltamasi +category: blog +--- +## Background + +BlobDB is essentially RocksDB for large-value use cases. The basic idea, which was proposed in the [WiscKey paper](https://www.usenix.org/system/files/conference/fast16/fast16-papers-lu.pdf), is key-value separation: by storing large values in dedicated blob files and storing only small pointers to them in the LSM tree, we avoid copying the values over and over again during compaction, thus reducing write amplification. Historically, BlobDB supported only FIFO and TTL based use cases that can tolerate some data loss. In addition, it was incompatible with many widely used RocksDB features, and required users to adopt a custom API. In 2020, we decided to rearchitect BlobDB from the ground up, taking the lessons learned from WiscKey and the original BlobDB but also drawing inspiration and incorporating ideas from other similar systems. Our goals were to eliminate the above limitations and to create a new integrated version that enables customers to use the well-known RocksDB API, has feature parity with the core of RocksDB, and offers better performance. This new implementation is now available and provides the following improvements over the original: + +* **API.** In contrast with the legacy BlobDB implementation, which had its own `StackableDB`-based interface (`rocksdb::blob_db::BlobDB`), the new version can be used via the well-known `rocksdb::DB` API, and can be configured simply by using a few column family options. +* **Consistency.** With the integrated BlobDB implementation, RocksDB’s consistency guarantees and various write options (like using the WAL or synchronous writes) now apply to blobs as well. Moreover, the new BlobDB keeps track of blob files in the RocksDB MANIFEST. +* **Write performance.** When using the old BlobDB, blobs are extracted and immediately written to blob files by the BlobDB layer *in the application thread*. This has multiple drawbacks from a performance perspective: first, it requires synchronization; second, it means that expensive operations like compression are performed in the application thread; and finally, it involves flushing the blob file after each blob. The new code takes a completely different approach by *offloading blob file building to RocksDB’s background jobs*, i.e. flushes and compactions. This means that similarly to SSTs, any given blob file is now written by a single background thread, eliminating the need for locking, flushing, or performing compression in the foreground. Note that this approach is also a better fit for network-based file systems where small writes might be expensive and opens up the possibility of file format optimizations that involve buffering (like dictionary compression). +* **Read performance.** The old code relies on each read (i.e. `Get`, `MultiGet`, or iterator) taking a snapshot and uses those snapshots when deciding which obsolete blob files can be removed. The new BlobDB improves this by generalizing RocksDB’s Version concept, which historically referred to the set of live SST files at a given point in time, to include the set of live blob files as well. This has performance benefits like [making the read path mostly lock-free by utilizing thread-local storage](https://rocksdb.org/blog/2014/06/27/avoid-expensive-locks-in-get.html). We have also introduced a blob file cache that can be utilized to keep frequently accessed blob files open. +* **Garbage collection.** Key-value separation means that if a key pointing to a blob gets overwritten or deleted, the blob becomes unreferenced garbage. To be able to reclaim this space, BlobDB now has garbage collection capabilities. GC is integrated into the compaction process and works by relocating valid blobs residing in old blob files as they are encountered during compaction. Blob files can be marked obsolete (and eventually deleted in one shot) once they contain nothing but garbage. This is more efficient than the method used by WiscKey, which involves performing a `Get` operation to find out whether a blob is still referenced followed by a `Put` to update the reference, which in turn results in garbage collection competing and potentially conflicting with the application’s writes. +* **Feature parity with the RocksDB core.** The new BlobDB supports way more features than the original and is near feature parity with vanilla RocksDB. In particular, we support all basic read/write APIs (with the exception of `Merge`, which is coming soon), recovery, compression, atomic flush, column families, compaction filters, checkpoints, backup/restore, transactions, per-file checksums, and the SST file manager. In addition, the new BlobDB’s options can be dynamically adjusted using the `SetOptions` interface. + +## API + +The new BlobDB can be configured (on a per-column family basis if needed) simply by using the following options: + +* `enable_blob_files`: set it to `true` to enable key-value separation. +* `min_blob_size`: values at or above this threshold will be written to blob files during flush or compaction. +* `blob_file_size`: the size limit for blob files. +* `blob_compression_type`: the compression type to use for blob files. All blobs in the same file are compressed using the same algorithm. +* `enable_blob_garbage_collection`: set this to `true` to make BlobDB actively relocate valid blobs from the oldest blob files as they are encountered during compaction. +* `blob_garbage_collection_age_cutoff`: the threshold that the GC logic uses to determine which blob files should be considered “old.” For example, the default value of 0.25 signals to RocksDB that blobs residing in the oldest 25% of blob files should be relocated by GC. This parameter can be tuned to adjust the trade-off between write amplification and space amplification. + +The above options are all dynamically adjustable via the `SetOptions` API; changing them will affect subsequent flushes and compactions but not ones that are already in progress. + +In terms of compaction styles, we recommend using leveled compaction with BlobDB. The rationale behind universal compaction in general is to provide lower write amplification at the expense of higher read amplification; however, as we will see later in the Performance section, BlobDB can provide very low write amp and good read performance with leveled compaction. Therefore, there is really no reason to take the hit in read performance that comes with universal compaction. + +In addition to the above, consider tuning the following non-BlobDB specific options: + +* `write_buffer_size`: this is the memtable size. You might want to increase it for large-value workloads to ensure that SST and blob files contain a decent number of keys. +* `target_file_size_base`: the target size of SST files. Note that even when using BlobDB, it is important to have an LSM tree with a “nice” shape and multiple levels and files per level to prevent heavy compactions. Since BlobDB extracts and writes large values to blob files, it makes sense to make this parameter significantly smaller than the memtable size. One guideline is to set `blob_file_size` to the same value as `write_buffer_size` (adjusted for compression if needed) and make `target_file_size_base` proportionally smaller based on the ratio of key size to value size. +* `max_bytes_for_level_base`: consider setting this to a multiple (e.g. 8x or 10x) of `target_file_size_base`. + +As mentioned above, the new BlobDB now also supports compaction filters. Key-value separation actually enables an optimization here: if the compaction filter of an application can make a decision about a key-value solely based on the key, it is unnecessary to read the value from the blob file. Applications can take advantage of this optimization by implementing the new `FilterBlobByKey` method of the `CompactionFilter` interface. This method gets called by RocksDB first whenever it encounters a key-value where the value is stored in a blob file. If this method returns a “final” decision like `kKeep`, `kRemove`, `kChangeValue`, or `kRemoveAndSkipUntil`, RocksDB will honor that decision; on the other hand, if the method returns `kUndetermined`, RocksDB will read the blob from the blob file and call `FilterV2` with the value in the usual fashion. + +## Performance + +We tested the performance of the new BlobDB for six different value sizes between 1 KB and 1 MB using a customized version of our [standard benchmark suite](https://github.com/facebook/rocksdb/wiki/Performance-Benchmarks) on a box with an 18-core Skylake DE CPU (running at 1.6 GHz, with hyperthreading enabled), 64 GB RAM, a 512 GB boot SSD, and two 1.88 TB M.2 SSDs in a RAID0 configuration for data. The RocksDB version used was equivalent to 6.18.1, with some benchmarking and statistics related enhancements. Leveled and universal compaction without key-value separation were used as reference points. Note that for simplicity, we use “leveled compaction” and “universal compaction” as shorthand for leveled and universal compaction without key-value separation, respectively, and “BlobDB” for BlobDB with leveled compaction. + +Our benchmarks cycled through six different workloads: two write-only ones (initial load and overwrite), two read/write ones (point lookup/write mix and range scan/write mix), and finally two read-only ones (point lookups and range scans). The first two phases performed a fixed amount of work (see below), while the final four were run for a fixed amount of time, namely 30 minutes each. Each phase other than the first one started with the database state left behind by the previous one. Here’s a brief description of the workloads: + +* **Initial load**: this workload has two distinct stages, a single-threaded random write stage during which compactions are disabled (so all data is flushed to L0, where it remains for the rest of the stage), followed by a full manual compaction. The random writes are performed with load-optimized settings, namely using the vector memtable implementation and with concurrent memtable writes and WAL disabled. This stage was used to populate the database with 1 TB worth of raw values, e.g. 2^30 (~1 billion) 1 KB values or 2^20 (~1 million) 1 MB values. +* **Overwrite**: this is a multi-threaded random write workload using the usual skiplist memtable, with compactions, WAL, and concurrent memtable writes enabled. In our tests, 16 writer threads were used. The total number of writes was set to the same number as in the initial load stage and split up evenly between the writer threads. For instance, for the 1 MB value size, we had 2^20 writes divided up between the 16 threads, resulting in each thread performing 2^16 write operations. At the end of this phase, a “wait for compactions” step was added to prevent this workload from exhibiting artificially low write amp or conversely, the next phase showing inflated write amp. +* **Point lookup/write mix**: a single writer thread performing random writes while N (in our case, 16) threads perform random point lookups. WAL is enabled and all writes are synced. +* **Range scan/write mix**: similar to the above, with one writer thread and N reader threads (where N was again set to 16 in our tests). The reader threads perform random range scans, with 10 `Next` calls per `Seek`. Again, WAL is enabled, and sync writes are used. +* **Point lookups (read-only)**: N=16 threads perform random point lookups. +* **Range scans (read-only)**: N=16 threads execute random range scans, with 10 `Next`s per `Seek` like above. + +With that out of the way, let’s see how the new BlobDB performs against traditional leveled and universal compaction. In the next few sections, we’ll be looking at write amplification as well as read and write performance. We’ll also briefly compare the write performance of the new BlobDB with the legacy implementation. + +### Write amplification + +Reducing write amp is the original motivation for key-value separation. Here, we follow RocksDB’s definition of write amplification (as used in compaction statistics and the info log). That is, we define write amp as the total amount of data written by flushes and compactions divided by the amount of data written by flushes, where “data written” includes SST files and blob files as well (if applicable). The following charts show that BlobDB significantly reduces write amplification for all of our (non-read only) workloads. + +For the initial load, where due to the nature of the workload both leveled and universal already have a low write amp factor of 1.6, BlobDB has a write amp close to the theoretical minimum of 1.0, namely in the 1.0..1.02 range, depending on value size. How is this possible? Well, the trick is that when key-value separation is used, the full compaction step only has to sort the keys but not the values. This results in a write amp that is about **36% lower** than the already low write amp you get with either leveled or universal. + +In the case of the overwrite workload, BlobDB had a write amp between 1.4 and 1.7 depending on value size. This is around **75-78% lower** than the write amp of leveled compaction (6.1 to 6.8) and **70-77% lower** than universal (5.7 to 6.2); for this workload, there wasn’t a huge difference between the performance of leveled and universal. + +When it comes to the point lookup/write mix workload, BlobDB had a write amp between 1.4 and 1.8. This is **83-88% lower** than the write amp of leveled compaction, which had values between 10.8 and 12.5. Universal fared much better than leveled under this workload, and had write amp in the 2.2..6.6 range; however, BlobDB still provided significant gains for all value sizes we tested: namely, write amp was **18-77% lower** than that of universal, depending on value size. + +As for the range scan/write mix workload, BlobDB again had a write amp between 1.4 and 1.8, while leveled had values between 13.6 and 14.9, and universal was between 2.8 and 5.0. In other words, BlobDB’s write amp was **88-90% lower** than that of leveled, and **46-70% lower** than that of universal. + +![Write amplification](/static/images/integrated-blob-db/BlobDB_Benchmarks_Write_Amp.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +### Write performance + +In terms of write performance, there are other factors to consider besides write amplification. The following charts show some interesting metrics for the two write-only workloads (initial load and overwrite). As discussed earlier, these two workloads perform a fixed amount of work; the two charts in the top row show how long it took BlobDB, leveled, and universal to complete that work. Note that each bar is broken down into two, corresponding to the two stages of each workload (random write and full compaction for initial load, and random write and waiting for compactions for overwrite). + +For initial load, note that the random write stage takes the same amount of time regardless of which algorithm is used. This is not surprising considering the fact that compactions are disabled during this stage and thus RocksDB is simply writing L0 files (and in BlobDB’s case, blob files) as fast as it can. The second stage, on the other hand, is very different: as mentioned above, BlobDB essentially only needs to read, sort, and rewrite the keys during compaction, which can be done much much faster (with 1 MB values, more than a hundred times faster) than doing the same for large key-values. Due to this, initial load completed **2.3x to 4.7x faster** overall when using BlobDB. + +As for the overwrite workload, BlobDB performs much better during both stages. The two charts in the bottom row help explain why. In the case of both leveled and universal compaction, compactions can’t keep up with the write rate, which eventually leads to back pressure in the form of write stalls. As shown in the chart below, both leveled and universal stall between ~40% and ~70% of the time; on the other hand, BlobDB is stall-free except for the largest value size tested (1 MB). This naturally leads to higher throughput, namely **2.1x to 3.5x higher** throughput compared to leveled, and **1.6x to 3.0x higher** throughput compared to universal. The overwrite time chart also shows that the catch-up stage that waits for all compactions to finish is much shorter (and in fact, at larger value sizes, negligible) with BlobDB. + +![Write performance](/static/images/integrated-blob-db/BlobDB_Benchmarks_Write_Perf.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +### Read/write and read-only performance + +The charts below show the read performance (in terms of operations per second) of BlobDB versus leveled and universal compaction under the two read/write workloads and the two read-only workloads. BlobDB meets or exceeds the read performance of leveled compaction, except for workloads involving range scans at the two smallest value sizes tested (1 KB and 4 KB). It also provides better (in some cases, much better) read performance than universal across the board. In particular, BlobDB provides up **1.4x higher** read performance than leveled (for larger values), and up to **5.6x higher** than universal. + +![Read-write and read-only performance](/static/images/integrated-blob-db/BlobDB_Benchmarks_RW_RO_Perf.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +### Comparing the two BlobDB implementations + +To compare the write performance of the new BlobDB with the legacy implementation, we ran two versions of the first (single-threaded random write) stage of the initial load benchmark using 1 KB values: one with WAL disabled, and one with WAL enabled. The new implementation completed the load **4.6x faster** than the old one without WAL, and **2.3x faster** with WAL. + +![Comparing the two BlobDB implementations](/static/images/integrated-blob-db/BlobDB_Benchmarks_Legacy_Vs_Integrated.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +## Future work + +There are a few remaining features that are not yet supported by the new BlobDB. The most important one is `Merge` (and the related `GetMergeOperands` API); in addition, we don’t currently support the `EventListener` interface, the `GetLiveFilesMetaData` and `GetColumnFamilyMetaData` APIs, secondary instances, and ingestion of blob files. We will continue to work on closing this gap. + +We also have further plans when it comes to performance. These include optimizing garbage collection, introducing a dedicated cache for blobs, improving iterator and `MultiGet` performance, and evolving the blob file format amongst others. + diff --git a/librocksdb-sys/rocksdb/docs/_posts/2021-05-26-online-validation.markdown b/librocksdb-sys/rocksdb/docs/_posts/2021-05-26-online-validation.markdown new file mode 100644 index 0000000..33e9dfc --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2021-05-26-online-validation.markdown @@ -0,0 +1,17 @@ +--- +title: Online Validation +layout: post +author: sdong +category: blog +--- +To prevent or mitigate data corrution in RocksDB when some software or hardware issues happens, we keep adding online consistency checks and improving existing ones. + +We improved ColumnFamilyOptions::force_consistency_checks and enabled it by default. The option does some basic consistency checks to LSM-tree, e.g., files in one level are not overlapping. The DB will be frozen from new writes if a violation is detected. Previously, the feature’s check was too limited and didn’t always freeze the DB in a timely manner. Last year, we made the checking stricter so that it can [catch much more corrupted LSM-tree structures](https://github.com/facebook/rocksdb/pull/6901). We also fixed several issues where the checking failure was swallowed without freezing the DB. After making force_consistency_checks more reliable, we changed the default value to be on. + +ColumnFamilyOptions::paranoid_file_checks does some more expensive extra checking when generating a new SST file. Last year, we advanced coverage to this feature: after every SST file is generated, the SST file is created, read back keys one by one and check two things: (1) the keys are in comparator order (also available and enabled by default during file write via ColumnFamilyOptions::check_flush_compaction_key_order); (2) the hash of all the KVs is the same as calculated when we add KVs into it. These checks detect certain corruptions so we can prevent the corrupt files from being applied to the DB. We suggest users turn it on at least in shadow environments, and consider to run it in production too if you can afford the overheads. + +A recent feature is added to check the count of entries added into memtable while flushing it into an SST file. This feature is to have some online coverage to memtable corruption, caused by either software bug or hardware issue. This feature will be released in the coming release (6.21) and by default on. In the future, we will check more counters during memtables, e.g. number of puts or number of deletes. + +We also improved the reporting of online validation errors to improve debuggability. For example, failure to parse a corrupt key now reports details about the corrupt key. Since we did not want to expose key data in logs, error messages, etc., by default, this reporting is opt-in via DBOptions::allow_data_in_errors. + +More online checking features are planned and some are more sophisticated, including key/value checksums and sample based query validation. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2021-05-27-rocksdb-secondary-cache.markdown b/librocksdb-sys/rocksdb/docs/_posts/2021-05-27-rocksdb-secondary-cache.markdown new file mode 100644 index 0000000..3ad1141 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2021-05-27-rocksdb-secondary-cache.markdown @@ -0,0 +1,195 @@ +--- +title: RocksDB Secondary Cache +layout: post +author: anand1976 +category: blog +--- +## Introduction + +The RocksDB team is implementing support for a block cache on non-volatile media, such as a local flash device or NVM/SCM. It can be viewed as an extension of RocksDB’s current volatile block cache (LRUCache or ClockCache). The non-volatile block cache acts as a second tier cache that contains blocks evicted from the volatile cache. Those blocks are then promoted to the volatile cache as they become hotter due to access. + +This feature is meant for cases where the DB is located on remote storage or cloud storage. The non-volatile cache is officially referred to in RocksDB as the SecondaryCache. By maintaining a SecondaryCache that’s an order of magnitude larger than DRAM, fewer reads would be required from remote storage, thus reducing read latency as well as network bandwidth consumption. + +From the user point of view, the local flash cache will support the following requirements - + +1. Provide a pointer to a secondary cache when opening a DB +2. Be able to share the secondary cache across DBs in the same process +3. Have multiple secondary caches on a host +4. Support persisting the cache across process restarts and reboots by ensuring repeatability of the cache key + +![Architecture](/static/images/rocksdb-secondary-cache/arch_diagram.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +## Design + +When designing the API for a SecondaryCache, we had a choice between making it visible to the RocksDB code (table reader) or hiding it behind the RocksDB block cache. There are several advantages of hiding it behind the block cache - + +* Allows flexibility in insertion of blocks into the secondary cache. A block can be inserted on eviction from the RAM tier, or it could be eagerly inserted. +* It makes the rest of the RocksDB code less complex by providing a uniform interface regardless of whether a secondary cache is configured or not +* Makes parallel reads, peeking in the cache for prefetching, failure handling etc. easier +* Makes it easier to extend to compressed data if needed, and allows other persistent media, such as PM, to be added as an additional tier + + +We decided to make the secondary cache transparent to the rest of RocksDB code by hiding it behind the block cache. A key issue that we needed to address was the allocation and ownership of memory of the cached items - insertion into the secondary cache may require that memory be allocated by the same. This means that parts of the cached object that can be transferred to the secondary cache needs to be copied out (referred to as **unpacking**), and on a lookup the data stored in the secondary cache needs to be provided to the object constructor (referred to as **packing**). For RocksDB cached objects such as data blocks, index and filter blocks, and compression dictionaries, unpacking involves copying out the raw uncompressed BlockContents of the block, and packing involves constructing the corresponding block/index/filter/dictionary object using the raw uncompressed data. + +Another alternative we considered was the existing PersistentCache interface. However, we decided to not pursue it and eventually deprecate it for the following reasons - +* It is exposed directly to the table reader code, which makes it more difficult to implement different policies such as inclusive/exclusive cache, as well as extending it to more sophisticated admission control policies +* The interface does not allow for custom memory allocation and object packing/unpacking, so new APIs would have to be defined anyway +* The current PersistentCache implementation is very simple and does not have any admission control policies + +## API + +The interface between RocksDB’s block cache and the secondary cache is designed to allow pluggable implementations. For FB internal usage, we plan to use Cachelib with a wrapper to provide the plug-in implementation and use folly and other fbcode libraries, which cannot be used directly by RocksDB, to efficiently implement the cache operations. The following diagrams show the flow of insertion and lookup of a block. + +![Insert flow](/static/images/rocksdb-secondary-cache/insert_flow.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +![Lookup flow](/static/images/rocksdb-secondary-cache/lookup_flow.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +An item in the secondary cache is referenced by a SecondaryCacheHandle. The handle may not be immediately ready or have a valid value. The caller can call IsReady() to determine if its ready, and can call Wait() in order to block until it becomes ready. The caller must call Value() after it becomes ready to determine if the item was successfully read. Value() must return nullptr on failure. + +``` +class SecondaryCacheHandle { + public: + virtual ~SecondaryCacheHandle() {} + + // Returns whether the handle is ready or not + virtual bool IsReady() = 0; + + // Block until handle becomes ready + virtual void Wait() = 0; + + // Return the value. If nullptr, it means the lookup was unsuccessful + virtual void* Value() = 0; + + // Return the size of value + virtual size_t Size() = 0; +}; +``` + +The user of the secondary cache (for example, BlockBasedTableReader indirectly through LRUCache) must implement the callbacks defined in CacheItemHelper, in order to facilitate the unpacking/packing of objects for saving to and restoring from the secondary cache. The CreateCallback must be implemented to construct a cacheable object from the raw data in secondary cache. + +``` + // The SizeCallback takes a void* pointer to the object and returns the size + // of the persistable data. It can be used by the secondary cache to allocate + // memory if needed. + using SizeCallback = size_t (*)(void* obj); + + // The SaveToCallback takes a void* object pointer and saves the persistable + // data into a buffer. The secondary cache may decide to not store it in a + // contiguous buffer, in which case this callback will be called multiple + // times with increasing offset + using SaveToCallback = Status (*)(void* from_obj, size_t from_offset, + size_t length, void* out); + + // A function pointer type for custom destruction of an entry's + // value. The Cache is responsible for copying and reclaiming space + // for the key, but values are managed by the caller. + using DeleterFn = void (*)(const Slice& key, void* value); + + // A struct with pointers to helper functions for spilling items from the + // cache into the secondary cache. May be extended in the future. An + // instance of this struct is expected to outlive the cache. + struct CacheItemHelper { + SizeCallback size_cb; + SaveToCallback saveto_cb; + DeleterFn del_cb; + + CacheItemHelper() : size_cb(nullptr), saveto_cb(nullptr), del_cb(nullptr) {} + CacheItemHelper(SizeCallback _size_cb, SaveToCallback _saveto_cb, + DeleterFn _del_cb) + : size_cb(_size_cb), saveto_cb(_saveto_cb), del_cb(_del_cb) {} + }; + + // The CreateCallback is passed by the block cache user to Lookup(). It + // takes in a buffer from the NVM cache and constructs an object using + // it. The callback doesn't have ownership of the buffer and should + // copy the contents into its own buffer. + // typedef std::function + // CreateCallback; + using CreateCallback = std::function; +``` + +The secondary cache provider must provide a concrete implementation of the SecondaryCache abstract class. + +``` +// SecondaryCache +// +// Cache interface for caching blocks on a secondary tier (which can include +// non-volatile media, or alternate forms of caching such as compressed data) +class SecondaryCache { + public: + virtual ~SecondaryCache() {} + + virtual std::string Name() = 0; + + static const std::string Type() { return "SecondaryCache"; } + + // Insert the given value into this cache. The value is not written + // directly. Rather, the SaveToCallback provided by helper_cb will be + // used to extract the persistable data in value, which will be written + // to this tier. The implementation may or may not write it to cache + // depending on the admission control policy, even if the return status is + // success. + virtual Status Insert(const Slice& key, void* value, + const Cache::CacheItemHelper* helper) = 0; + + // Lookup the data for the given key in this cache. The create_cb + // will be used to create the object. The handle returned may not be + // ready yet, unless wait=true, in which case Lookup() will block until + // the handle is ready + virtual std::unique_ptr Lookup( + const Slice& key, const Cache::CreateCallback& create_cb, bool wait) = 0; + + // At the discretion of the implementation, erase the data associated + // with key + virtual void Erase(const Slice& key) = 0; + + // Wait for a collection of handles to become ready. This would be used + // by MultiGet, for example, to read multitple data blocks in parallel + virtual void WaitAll(std::vector handles) = 0; + + virtual std::string GetPrintableOptions() const = 0; +}; +``` + +A SecondaryCache is configured by the user by providing a pointer to it in LRUCacheOptions - +``` +struct LRUCacheOptions { + ... + // A SecondaryCache instance to use as an additional cache tier + std::shared_ptr secondary_cache; + ... +}; +``` + +## Current Status + +The initial RocksDB support for the secondary cache has been merged into the main branch, and will be available in the 6.21 release. This includes providing a way for the user to configure a secondary cache when instantiating RocksDB’s LRU cache (volatile block cache), spilling blocks evicted from the LRU cache to the flash cache, promoting a block read from the SecondaryCache to the LRU cache, update tools such as cache_bench and db_bench to specify a flash cache. The relevant PRs are [#8271](https://github.com/facebook/rocksdb/pull/8271), [#8191](https://github.com/facebook/rocksdb/pull/8191), and [#8312](https://github.com/facebook/rocksdb/pull/8312). + +We prototyped an end-to-end solution, with the above PRs as well as a Cachelib based implementation of the SecondaryCache. We ran a mixgraph benchmark to simulate a realistic read/write workload. The results showed a 15% gain with the local flash cache over no local cache, and a ~25-30% reduction in network reads with a corresponding decrease in cache misses. + +![Throughput](/static/images/rocksdb-secondary-cache/Mixgraph_throughput.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +![Hit Rate](/static/images/rocksdb-secondary-cache/Mixgraph_hit_rate.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +## Future Work + +In the short term, we plan to do the following in order to fully integrate the SecondaryCache with RocksDB - + +1. Use DB session ID as the cache key prefix to ensure uniqueness and repeatability +2. Optimize flash cache usage of MultiGet and iterator workloads +3. Stress testing +4. More benchmarking + +Longer term, we plan to deploy this in production at Facebook. + +## Call to Action + +We are hoping for a community contribution of a secondary cache implementation, which would make this feature usable by the broader RocksDB userbase. If you are interested in contributing, please reach out to us in [this issue](https://github.com/facebook/rocksdb/issues/8347). + diff --git a/librocksdb-sys/rocksdb/docs/_posts/2021-05-31-dictionary-compression.markdown b/librocksdb-sys/rocksdb/docs/_posts/2021-05-31-dictionary-compression.markdown new file mode 100644 index 0000000..9b0f452 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2021-05-31-dictionary-compression.markdown @@ -0,0 +1,157 @@ +--- +title: Preset Dictionary Compression +layout: post +author: ajkr +category: blog +--- + +## Summary + +Compression algorithms relying on an adaptive dictionary, such as LZ4, zstd, and zlib, struggle to achieve good compression ratios on small inputs when using the basic compress API. +With the basic compress API, the compressor starts with an empty dictionary. +With small inputs, not much content gets added to the dictionary during the compression. +Combined, these factors suggest the dictionary will never have enough contents to achieve great compression ratios. + +RocksDB groups key-value pairs into data blocks before storing them in files. +For use cases that are heavy on random accesses, smaller data block size is sometimes desirable for reducing I/O and CPU spent reading blocks. +However, as explained above, smaller data block size comes with the downside of worse compression ratio when using the basic compress API. + +Fortunately, zstd and other libraries offer advanced compress APIs that preset the dictionary. +A preset dictionary makes it possible for the compressor to start from a useful state instead of from an empty one, making compression immediately effective. + +RocksDB now optionally takes advantage of these dictionary presetting APIs. +The challenges in integrating this feature into the storage engine were more substantial than apparent on the surface. +First, we need to target a preset dictionary to the relevant data. +Second, preset dictionaries need to be trained from data samples, which need to be gathered. +Third, preset dictionaries need to be persisted since they are needed at decompression time. +Fourth, overhead in accessing the preset dictionary must be minimized to prevent regression in critical code paths. +Fifth, we need easy-to-use measurement to evaluate candidate use cases and production impact. + +In production, we have deployed dictionary presetting to save space in multiple RocksDB use cases with data block size 8KB or smaller. +We have measured meaningful benefit to compression ratio in use cases with data block size up to 16KB. +We have also measured a use case that can save both CPU and space by reducing data block size and turning on dictionary presetting at the same time. + +## Feature design +#### Targeting + +Over time we have considered a few possibilities for the scope of a dictionary. + +- Subcompaction +- SST file +- Column family + +The original choice was subcompaction scope. +This enabled an approach with minimal buffering overhead because we could collect samples while generating the first output SST file. +The dictionary could then be trained and applied to subsequent SST files in the same subcompaction. + +However, we found a large use case where the proximity of data in the keyspace was more correlated with its similarity than we had predicted. +In particular, the approach of training a dictionary on an adjacent file yielded substantially worse ratios than training the dictionary on the same file it would be used to compress. +In response to this finding, we changed the preset dictionary scope to per SST file. + +With this change in approach, we had to face the problem we had hoped to avoid: how can we compress all of an SST file's data blocks with the same preset dictionary while that dictionary can only be trained after many data blocks have been sampled? +The solutions we considered both involved a new overhead. +We could read the input more than once and introduce I/O overhead, or we could buffer the uncompressed output file data blocks until a dictionary is trained, introducing memory overhead. +We chose to take the hit on memory overhead. + +Another approach that we considered was associating multiple dictionaries with a column family. +For example, in MyRocks there could be a dictionary trained on data from each large table. +When compressing a data block, we would look at the table to which its data belongs and pick the corresponding dictionary. +However, this approach would introduce many challenges. +RocksDB would need to be aware of the key schema to know where are the table boundaries. +RocksDB would also need to periodically update the dictionaries to account for changes in data pattern. +It would need somewhere to store dictionaries at column family scope. +Overall, we thought these challenges were too difficult to pursue the approach. + +#### Training + +![](/static/images/dictcmp/dictcmp_raw_sampled.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} +

    +Raw samples mode (`zstd_max_train_bytes == 0`) +

    + +As mentioned earlier, the approach we took is to build the dictionary from buffered uncompressed data blocks. +The first row of data blocks in these diagrams illustrate this buffering. +The second row illustrates training samples selected from the buffered blocks. +In raw samples mode (above), the final dictionary is simply the concatenation of these samples. +Whereas, in zstd training mode (below), these samples will be passed to the trainer to produce the final dictionary. + +![](/static/images/dictcmp/dictcmp_zstd_trained.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} +

    +zstd training mode (`zstd_max_train_bytes > 0`) +

    + +#### Compression path + +Once the preset dictionary is generated by the above process, we apply it to the buffered data blocks and write them to the output file. +Thereafter, newly generated data blocks are immediately compressed and written out. + +One optimization here is available to zstd v0.7.0+ users. +Instead of deserializing the dictionary on each compress invocation, we can do that work once and reuse it. +A `ZSTD_CDict` holds this digested dictionary state and is passed to the compress API. + +#### Persistence + +When an SST file's data blocks are compressed using a preset dictionary, that dictionary is stored inside the file for later use in decompression. + +![](/static/images/dictcmp/dictcmp_sst_blocks.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} +

    +SST file layout with the preset dictionary in its own (uncompressed) block +

    + +#### Decompression path + +To decompress, we need to provide both the data block and the dictionary used to compress it. +Since dictionaries are just blocks in a file, we access them through block cache. +However this additional load on block cache can be problematic. +It can be alleviated by pinning the dictionaries to avoid going through the LRU locks. + +An optimization analogous to the digested dictionary exists for certain zstd users (see User API section for details). +When enabled, the block cache stores the digested dictionary state for decompression (`ZSTD_DDict`) instead of the block contents. +In some cases we have seen decompression CPU decrease overall when enabling dictionary thanks to this optimization. + +#### Measurement + +Typically our first step in evaluating a candidate use case is an offline analysis of the data. +This gives us a quick idea whether presetting dictionary will be beneficial without any code, config, or data changes. +Our `sst_dump` tool reports what size SST files would have been using specified compression libraries and options. +We can select random SST files and compare the size with vs. without dictionary. + +When that goes well, the next step is to see how it works in a live DB, like a production shadow or canary. +There we can observe how it affects application/system metrics. + +Even after dictionary is enabled, there is the question of how much space was finally saved. +We provide a way to A/B test size with vs. without dictionary while running in production. +This feature picks a sample of data blocks to compress in multiple ways -- one of the outputs is stored, while the other outputs are thrown away after counting their size. +Due to API limitations, the stored output always has to be the dictionary-compressed one, so this feature can only be used after enabling dictionary. +The size with and without dictionary are stored in the SST file as table properties. +These properties can be aggregated across all SST files in a DB (and across all DBs in a tier) to learn the final space saving. + +## User API + +RocksDB allows presetting compression dictionary for users of LZ4, zstd, and zlib. +The most advanced capabilities are available to zstd v1.1.4+ users who statically link (see below). +Newer versions of zstd (v1.3.6+) have internal changes to the dictionary trainer and digested dictionary management, which significantly improve memory and CPU efficiency. + +Run-time settings: + +- `CompressionOptions::max_dict_bytes`: Limit on per-SST file dictionary size. Increasing this causes dictionaries to consume more space and memory for the possibility of better data block compression. A typical value we use is 16KB. +- (**zstd only**) `CompressionOptions::zstd_max_train_bytes`: Limit on training data passed to zstd dictionary trainer. Larger values cause the training to consume more CPU (and take longer) while generating more effective dictionaries. The starting point guidance we received from zstd team is to set it to 100x `CompressionOptions::max_dict_bytes`. +- `CompressionOptions::max_dict_buffer_bytes`: Limit on data buffering from which training samples are gathered. By default we buffer up to the target file size per ongoing background job. If this amount of memory is concerning, this option can constrain the buffering with the downside that training samples will cover a smaller portion of the SST file. Work is ongoing to charge this memory usage to block cache so it will not need to be accounted for separately. +- `BlockBasedTableOptions::cache_index_and_filter_blocks`: Controls whether metadata blocks including dictionary are accessed through block cache or held in table reader memory (yes, its name is outdated). +- `BlockBasedTableOptions::metadata_cache_options`: Controls what metadata blocks are pinned in block cache. Pinning avoids LRU contention at the risk of cold blocks holding memory. +- `ColumnFamilyOptions::sample_for_compression`: Controls frequency of measuring extra compressions on data blocks using various libraries with default settings (i.e., without preset dictionary). + +Compile-time setting: + +- (**zstd only**) `EXTRA_CXXFLAGS=-DZSTD_STATIC_LINKING_ONLY`: Hold digested dictionaries in block cache to save repetitive deserialization overhead. This saves a lot of CPU for read-heavy workloads. This compiler flag is necessary because one of the digested dictionary APIs we use is marked as experimental. We still use it in production, however. + +Function: + +- `DB::GetPropertiesOfAllTables()`: The properties `kSlowCompressionEstimatedDataSize` and `kFastCompressionEstimatedDataSize` estimate what the data block size (`kDataSize`) would have been if the corresponding compression library had been used. These properties are only present when `ColumnFamilyOptions::sample_for_compression` causes one or more samples to be measured, and they become more accurate with higher sampling frequency. + +Tool: + +- `sst_dump --command=recompress`: Offline analysis tool that reports what the SST file size would have been using the specified compression library and options. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2021-12-29-ribbon-filter.markdown b/librocksdb-sys/rocksdb/docs/_posts/2021-12-29-ribbon-filter.markdown new file mode 100644 index 0000000..c6a52ce --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2021-12-29-ribbon-filter.markdown @@ -0,0 +1,281 @@ +--- +title: Ribbon Filter +layout: post +author: pdillinger +category: blog +--- + +## Summary +Since version 6.15 last year, RocksDB supports Ribbon filters, a new +alternative to Bloom filters that save space, especially memory, at +the cost of more CPU usage, mostly in constructing the filters in the +background. Most applications with long-lived data (many hours or +longer) will likely benefit from adopting a Ribbon+Bloom hybrid filter +policy. Here we explain why and how. + +[Ribbon filter on RocksDB wiki](https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter#ribbon-filter) + +[Ribbon filter paper](https://arxiv.org/abs/2103.02515) + +## Problem & background +Bloom filters play a critical role in optimizing point queries and +some range queries in LSM-tree storage systems like RocksDB. Very +large DBs can use 10% or more of their RAM memory for (Bloom) filters, +so that (average case) read performance can be very good despite high +(worst case) read amplification, [which is useful for lowering write +and/or space +amplification](http://smalldatum.blogspot.com/2015/11/read-write-space-amplification-pick-2_23.html). +Although the `format_version=5` Bloom filter in RocksDB is extremely +fast, all Bloom filters use around 50% more space than is +theoretically possible for a hashed structure configured for the same +false positive (FP) rate and number of keys added. What would it take +to save that significant share of “wasted” filter memory, and when +does it make sense to use such a Bloom alternative? + +A number of alternatives to Bloom filters were known, especially for +static filters (not modified after construction), but all the +previously known structures were unsatisfying for SSTs because of some +combination of +* Not enough space savings for CPU increase. For example, [Xor + filters](https://arxiv.org/abs/1912.08258) use 3-4x more CPU than + Bloom but only save 15-20% of + space. [GOV](https://arxiv.org/pdf/1603.04330.pdf) can save around + 30% space but requires around 10x more CPU than Bloom. +* Inconsistent space savings. [Cuckoo + filters](https://www.cs.cmu.edu/~dga/papers/cuckoo-conext2014.pdf) + and Xor+ filters offer significant space savings for very low FP + rates (high bits per key) but little or no savings for higher FP + rates (low bits per key). ([Higher FP rates are considered best for + largest levels of + LSM.](https://stratos.seas.harvard.edu/files/stratos/files/monkeykeyvaluestore.pdf)) + [Spatially-coupled Xor + filters](https://arxiv.org/pdf/2001.10500.pdf) require very large + number of keys per filter for large space savings. +* Inflexible configuration. No published alternatives offered the same + continuous configurability of Bloom filters, where any FP rate and + any fractional bits per key could be chosen. This flexibility + improves memory efficiency with the `optimize_filters_for_memory` + option that minimizes internal fragmentation on filters. + +## Ribbon filter development and implementation +The Ribbon filter came about when I developed a faster, simpler, and +more adaptable algorithm for constructing a little-known [Xor-based +structure from Dietzfelbinger and +Walzer](https://arxiv.org/pdf/1907.04750.pdf). It has very good space +usage for required CPU time (~30% space savings for 3-4x CPU) and, +with some engineering, Bloom-like configurability. The complications +were managable for use in RocksDB: +* Ribbon space efficiency does not naturally scale to very large + number of keys in a single filter (whole SST file or partition), but + with the current 128-bit Ribbon implementation in RocksDB, even 100 + million keys in one filter saves 27% space vs. Bloom rather than 30% + for 100,000 keys in a filter. +* More temporary memory is required during construction, ~230 bits per + key for 128-bit Ribbon vs. ~75 bits per key for Bloom filter. A + quick calculation shows that if you are saving 3 bits per key on the + generated filter, you only need about 50 generated filters in memory + to offset this temporary memory usage. (Thousands of filters in + memory is typical.) Starting in RocksDB version 6.27, this temporary + memory can be accounted for under block cache using + `BlockBasedTableOptions::reserve_table_builder_memory`. +* Ribbon filter queries use relatively more CPU for lower FP rates + (but still O(1) relative to number of keys added to filter). This + should be OK because lower FP rates are only appropriate when then + cost of a false positive is very high (worth extra query time) or + memory is not so constrained (can use Bloom instead). + +Future: data in [the paper](https://arxiv.org/abs/2103.02515) suggests +that 32-bit Balanced Ribbon (new name: [Bump-Once +Ribbon](https://arxiv.org/pdf/2109.01892.pdf)) would improve all of +these issues and be better all around (except for code complexity). + +## Ribbon vs. Bloom in RocksDB configuration +Different applications and hardware configurations have different +constraints, but we can use hardware costs to examine and better +understand the trade-off between Bloom and Ribbon. + +### Same FP rate, RAM vs. CPU hardware cost +Under ideal conditions where we can adjust our hardware to suit the +application, in terms of dollars, how much does it cost to construct, +query, and keep in memory a Bloom filter vs. a Ribbon filter? The +Ribbon filter costs more for CPU but less for RAM. Importantly, the +RAM cost directly depends on how long the filter is kept in memory, +which in RocksDB is essentially the lifetime of the filter. +(Temporary RAM during construction is so short-lived that it is +ignored.) Using some consumer hardware and electricity prices and a +predicted balance between construction and queries, we can compute a +“break even” duration in memory. To minimize cost, filters with a +lifetime shorter than this should be Bloom and filters with a lifetime +longer than this should be Ribbon. (Python code) + +``` +# Commodity prices based roughly on consumer prices and rough guesses +# Upfront cost of a CPU per hardware thread +upfront_dollars_per_cpu_thread = 30.0 + +# CPU average power usage per hardware thread +watts_per_cpu_thread = 3.5 + +# Upfront cost of a GB of RAM +upfront_dollars_per_gb_ram = 8.0 + +# RAM average power usage per GB +# https://www.crucial.com/support/articles-faq-memory/how-much-power-does-memory-use +watts_per_gb_ram = 0.375 + +# Estimated price of power per kilowatt-hour, including overheads like conversion losses and cooling +dollars_per_kwh = 0.35 + +# Assume 3 year hardware lifetime +hours_per_lifetime = 3 * 365 * 24 +seconds_per_lifetime = hours_per_lifetime * 60 * 60 + +# Number of filter queries per key added in filter construction is heavily dependent on workload. +# When replication is in layer above RocksDB, it will be low, likely < 1. When replication is in +# storage layer below RocksDB, it will likely be > 1. Using a rough and general guesstimate. +key_query_per_construct = 1.0 + +#================================== +# Bloom & Ribbon filter performance +typical_bloom_bits_per_key = 10.0 +typical_ribbon_bits_per_key = 7.0 + +# Speeds here are sensitive to many variables, especially query speed because it +# is so dependent on memory latency. Using this benchmark here: +# for IMPL in 2 3; do +# ./filter_bench -impl=$IMPL -quick -m_keys_total_max=200 -use_full_block_reader +# done +# and "Random filter" queries. +nanoseconds_per_construct_bloom_key = 32.0 +nanoseconds_per_construct_ribbon_key = 140.0 + +nanoseconds_per_query_bloom_key = 500.0 +nanoseconds_per_query_ribbon_key = 600.0 + +#================================== +# Some constants +kwh_per_watt_lifetime = hours_per_lifetime / 1000.0 +bits_per_gb = 8 * 1024 * 1024 * 1024 + +#================================== +# Crunching the numbers +# on CPU for constructing filters +dollars_per_cpu_thread_lifetime = upfront_dollars_per_cpu_thread + watts_per_cpu_thread * kwh_per_watt_lifetime * dollars_per_kwh +dollars_per_cpu_thread_second = dollars_per_cpu_thread_lifetime / seconds_per_lifetime + +dollars_per_construct_bloom_key = dollars_per_cpu_thread_second * nanoseconds_per_construct_bloom_key / 10**9 +dollars_per_construct_ribbon_key = dollars_per_cpu_thread_second * nanoseconds_per_construct_ribbon_key / 10**9 + +dollars_per_query_bloom_key = dollars_per_cpu_thread_second * nanoseconds_per_query_bloom_key / 10**9 +dollars_per_query_ribbon_key = dollars_per_cpu_thread_second * nanoseconds_per_query_ribbon_key / 10**9 + +dollars_per_bloom_key_cpu = dollars_per_construct_bloom_key + key_query_per_construct * dollars_per_query_bloom_key +dollars_per_ribbon_key_cpu = dollars_per_construct_ribbon_key + key_query_per_construct * dollars_per_query_ribbon_key + +# on holding filters in RAM +dollars_per_gb_ram_lifetime = upfront_dollars_per_gb_ram + watts_per_gb_ram * kwh_per_watt_lifetime * dollars_per_kwh +dollars_per_gb_ram_second = dollars_per_gb_ram_lifetime / seconds_per_lifetime + +dollars_per_bloom_key_in_ram_second = dollars_per_gb_ram_second / bits_per_gb * typical_bloom_bits_per_key +dollars_per_ribbon_key_in_ram_second = dollars_per_gb_ram_second / bits_per_gb * typical_ribbon_bits_per_key + +#================================== +# How many seconds does it take for the added cost of constructing a ribbon filter instead +# of bloom to be offset by the added cost of holding the bloom filter in memory? +break_even_seconds = (dollars_per_ribbon_key_cpu - dollars_per_bloom_key_cpu) / (dollars_per_bloom_key_in_ram_second - dollars_per_ribbon_key_in_ram_second) +print(break_even_seconds) +# -> 3235.1647730256936 +``` + +So roughly speaking, filters that live in memory for more than an hour +should be Ribbon, and filters that live less than an hour should be +Bloom. This is very interesting, but how long do filters live in +RocksDB? + +First let's consider the average case. Write-heavy RocksDB loads are +often backed by flash storage, which has some specified write +endurance for its intended lifetime. This can be expressed as *device +writes per day* (DWPD), and supported DWPD is typically < 10.0 even +for high end devices (excluding NVRAM). Roughly speaking, the DB would +need to be writing at a rate of 20+ DWPD for data to have an average +lifetime of less than one hour. Thus, unless you are prematurely +burning out your flash or massively under-utilizing available storage, +using the Ribbon filter has the better cost profile *on average*. + +### Predictable lifetime +But we can do even better than optimizing for the average case. LSM +levels give us very strong data lifetime hints. Data in L0 might live +for minutes or a small number of hours. Data in Lmax might live for +days or weeks. So even if Ribbon filters weren't the best choice on +average for a workload, they almost certainly make sense for the +larger, longer-lived levels of the LSM. As of RocksDB 6.24, you can +specify a minimum LSM level for Ribbon filters with +`NewRibbonFilterPolicy`, and earlier levels will use Bloom filters. + +### Resident filter memory +The above analysis assumes that nearly all filters for all live SST +files are resident in memory. This is true if using +`cache_index_and_filter_blocks=0` and `max_open_files=-1` (defaults), +but `cache_index_and_filter_blocks=1` is popular. In that case, +if you use `optimize_filters_for_hits=1` and non-partitioned filters +(a popular MyRocks configuration), it is also likely that nearly all +live filters are in memory. However, if you don't use +`optimize_filters_for_hits` and use partitioned filters, then +cold data (by age or by key range) can lead to only a portion of +filters being resident in memory. In that case, benefit from Ribbon +filter is not as clear, though because Ribbon filters are smaller, +they are more efficient to read into memory. + +RocksDB version 6.21 and later include a rough feature to determine +block cache usage for data blocks, filter blocks, index blocks, etc. +Data like this is periodically dumped to LOG file +(`stats_dump_period_sec`): + +``` +Block cache entry stats(count,size,portion): DataBlock(441761,6.82 GB,75.765%) FilterBlock(3002,1.27 GB,14.1387%) IndexBlock(17777,887.75 MB,9.63267%) Misc(1,0.00 KB,0%) +Block cache LRUCache@0x7fdd08104290#7004432 capacity: 9.00 GB collections: 2573 last_copies: 10 last_secs: 0.143248 secs_since: 0 +``` + +This indicates that at this moment in time, the block cache object +identified by `LRUCache@0x7fdd08104290#7004432` (potentially used +by multiple DBs) uses roughly 14% of its 9GB, about 1.27 GB, on filter +blocks. This same data is available through `DB::GetMapProperty` with +`DB::Properties::kBlockCacheEntryStats`, and (with some effort) can +be compared to total size of all filters (not necessarily in memory) +using `rocksdb.filter.size` from +`DB::Properties::kAggregatedTableProperties`. + +### Sanity checking lifetime +Can we be sure that using filters even makes sense for such long-lived +data? We can apply [the current 5 minute rule for caching SSD data in +RAM](http://renata.borovica-gajic.com/data/adms2017_5minuterule.pdf). A +4KB filter page holds data for roughly 4K keys. If we assume at least +one negative (useful) filter query in its lifetime per added key, it +can satisfy the 5 minute rule with a lifetime of up to about two +weeks. Thus, the lifetime threshold for “no filter” is about 300x +higher than the lifetime threshold for Ribbon filter. + +### What to do with saved memory +The default way to improve overall RocksDB performance with more +available memory is to use more space for caching, which improves +latency, CPU load, read IOs, etc. With +`cache_index_and_filter_blocks=1`, savings in filters will +automatically make room for caching more data blocks in block +cache. With `cache_index_and_filter_blocks=0`, consider increasing +block cache size. + +Using the space savings to lower filter FP rates is also an option, +but there is less evidence for this commonly improving existing +*optimized* configurations. + +## Generic recommendation +If using `NewBloomFilterPolicy(bpk)` for a large persistent DB using +compression, try using `NewRibbonFilterPolicy(bpk)` instead, which +will generate Ribbon filters during compaction and Bloom filters +for flush, both with the same FP rate as the old setting. Once new SST +files are generated under the new policy, this should free up some +memory for more caching without much effect on burst or sustained +write speed. Both kinds of filters can be read under either policy, so +there's always an option to adjust settings or gracefully roll back to +using Bloom filter only (keeping in mind that SST files must be +replaced to see effect of that change). diff --git a/librocksdb-sys/rocksdb/docs/_posts/2022-07-18-per-key-value-checksum.markdown b/librocksdb-sys/rocksdb/docs/_posts/2022-07-18-per-key-value-checksum.markdown new file mode 100644 index 0000000..6b9ad80 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2022-07-18-per-key-value-checksum.markdown @@ -0,0 +1,142 @@ +--- +title: "Per Key-Value Checksum" +layout: post +author: +- cbi42 +- ajkr +category: blog +--- + +## Summary + +Silent data corruptions can severely impact RocksDB users. As a key-value library, RocksDB resides at the bottom of the user space software stack for many diverse applications. Returning wrong query results can cause unpredictable consequences for our users so must be avoided. + +To prevent and detect corruption, RocksDB has several consistency checks [1], especially focusing on the storage layer. For example, SST files contain block checksums that are verified during reads, and each SST file has a full file checksum that can be verified when files are transferred. + +Other sources of corruptions, such as those from faulty CPU/memory or heap corruptions, pose risks for which protections are relatively underdeveloped. Meanwhile, recent work [2] suggests one per thousand machines in our fleet will at some point experience a hardware error that is exposed to an application. Additionally, software bugs can increase the risk of heap corruptions at any time. + +Hardware/heap corruptions are naturally difficult to detect in the application layer since they can compromise any data or control flow. Some factors we take into account when choosing where to add protection are the volume of data, the importance of the data, the CPU instructions that operate on the data, and the duration it resides in memory. One recently added protection, `detect_filter_construct_corruption`, has proven itself useful in preventing corrupt filters from being persisted. We have seen hardware encounter machine-check exceptions a few hours after we detected a corrupt filter. + +The next way we intend to detect hardware and heap corruptions before they cause queries to return wrong results is through developing a new feature: per key-value checksum. This feature will eventually provide optional end-to-end integrity protection for every key-value pair. RocksDB 7.4 offers substantial coverage of the user write and recovery paths with per key-value checksum protection. + +## User API + +For integrity protection during recovery, no change is required. Recovery is always protected. + +For user write protection, RocksDB allows the user to specify per key-value protection through `WriteOptions::protection_bytes_per_key` or pass in `protection_bytes_per_key` to `WriteBatch` constructor when creating a `WriteBatch` directly. Currently, only 0 (default, no protection) and 8 bytes per key are supported. This should be fine for write batches as they do not usually contain a huge number of keys. We are working on supporting more settings as 8 bytes per key might cause considerable memory overhead when the protection is extended to memtable entries. + +## Feature Design + +### Data Structures + +#### Protection info + +For protecting key-value pairs, we chose to use a hashing algorithm, xxh3 [3], for its good efficiency without relying on special hardware. While algorithms like crc32c can guarantee detection of certain patterns of bit flips, xxh3 offers no such guarantees. This is acceptable for us as we do not expect any particular error pattern [4], and even if we did, xxh3 can achieve a collision probability close enough to zero for us by tuning the number of protection bytes per key-value. + +Key-value pairs have multiple representations in RocksDB: in [WriteBatch](https://github.com/facebook/rocksdb/blob/7d0ecab570742c7280628b08ddc03cfd692f484f/db/write_batch.cc#L14-L31), in memtable [entries](https://github.com/facebook/rocksdb/blob/fc51b7f33adcba7ac725ed0e7fe8b8155aaeaee4/db/memtable.cc#L541-L545) and in [data blocks](https://github.com/facebook/rocksdb/blob/fc51b7f33adcba7ac725ed0e7fe8b8155aaeaee4/table/block_based/block_builder.cc#L21-L27). In this post we focus on key-values in write batches and memtable as in-memory data blocks are not yet protected. + +Besides user key and value, RocksDB includes internal metadata in the per key-value checksum calculation. Depending on the representation, internal metadata consists of some combination of sequence number, operation type, and column family ID. Note that since timestamp (when enabled) is part of the user key it is protected as well. + +The protection info consists of the XOR’d result of the xxh3 hash for all the protected components. This allows us to efficiently transform protection info for different representations. See below for an example converting WriteBatch protection info to memtable protection info. + +A risk of using XOR is the possibility of swapping corruptions (e.g., key becomes the value and the value becomes the key). To mitigate this risk, we use an independent seed for hashing each type of component. + +The following two figures illustrate how protection info in WriteBatch and memtable are calculated from a key-value’s components. + +![](/static/images/kv-checksum/ProtInfo-Writebatch.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +*Protection info for a key-value in a WriteBatch* +{: style="text-align: center"} + +![](/static/images/kv-checksum/ProtInfo-Memtable.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +*Protection info for a key-value in a memtable* +{: style="text-align: center"} + +The next figure illustrates how protection info for a key-value can be transformed to protect that same key-value in a different representation. Note this is done without recalculating the hash for all the key-value’s components. + +![](/static/images/kv-checksum/ProtInfo-Writebatch-to-Memtable.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +*Protection info for a key-value in a memtable derived from an existing WriteBatch protection info* +{: style="text-align: center"} + +Above, we see two (small) components are hashed: column family ID and sequence number. When a key-value is inserted from WriteBatch into memtable, it is assigned a sequence number and drops the column family ID since each memtable is associated with one column family. Recall the xxh3 of column family ID was included in the WriteBatch protection info, which is canceled out by the column family ID xxh3 included in the XOR. + +#### WAL fragment + +WAL (Write-ahead-log) persists write batches that correspond to operations in memtables and enables consistent database recovery after restart. RocksDB writes to WAL in chunks of some [fixed block size](https://github.com/facebook/rocksdb/blob/fc51b7f33adcba7ac725ed0e7fe8b8155aaeaee4/db/log_writer.h#L44) for efficiency. It is possible that some write batch does not fit into the space left in the current block and/or is larger than the fixed block size. Thus, serialized write batches (WAL records) are divided into WAL fragments before being written to WAL. The format of a WAL fragment is in the following diagram (there is another legacy format detailed in code [comments](https://github.com/facebook/rocksdb/blob/fc51b7f33adcba7ac725ed0e7fe8b8155aaeaee4/db/log_writer.h#L47-L59)). Roughly, the `Type` field indicates whether a fragment is at the beginning, middle or end of a record, and is used to group fragments. + +![](/static/images/kv-checksum/WAL-fragment.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +Note that each fragment is prefixed by a crc32c checksum that is calculated over `Type`, `Log #` and `Payload`. This ensures that RocksDB can detect corruptions that happened to the WAL in the storage layer. + +#### Write batch + +As mentioned above, a WAL record is a serialized `WriteBatch` that is split into physical fragments during writes to WAL. During DB recovery, once a WAL record is reconstructed from one or more fragments, it is [copied](https://github.com/facebook/rocksdb/blob/fc51b7f33adcba7ac725ed0e7fe8b8155aaeaee4/db/db_impl/db_impl_open.cc#L1127) into the content of a `WriteBatch`. The write batch will then be used to restore the memtable states. + +Besides the recovery path, a write batch is always constructed during user writes. Firstly, RocksDB allows users to construct a write batch directly, and pass it to DB through `DB::Write()` API for execution. Higher-level buffered write APIs like Transaction rely on a write batch to buffer writes prior to executing them. For unbuffered write APIs like `DB::Put()`, RocksDB constructs a write batch internally with the input user key and value. + +![](/static/images/kv-checksum/Write-batch.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +The above diagram shows a rough representation of a write batch in memory. `Contents` is the concatenation of serialized user operations in this write batch. Each operation consists of user key, value, op_type and optionally column family ID. With per key-value checksum protection enabled, a vector of ProtectionInfo is stored in the write batch, one for each user operation. + +#### Memtable entry + +![](/static/images/kv-checksum/Memtable-entry.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +A memtable entry is similar to write batch content, except that it captures only a single user operation and that it does not contain column family ID (since memtable is per column family). User key and value are length-prefixed, and seqno and optype are combined in a fixed 8 bytes representation. + +### Processes + +In order to protect user writes and recovery, per key-value checksum is covered in the following code paths. + +#### WriteBatch write + +Per key-value checksum coverage starts with the user buffers that contain user key and/or value. When users call DB Write APIs (e.g., `DB::Put()`), or when users add operations into write batches directly (e.g. `WriteBatch::Put()`), RocksDB constructs `ProtectionInfo` from the user buffer (e.g. [here](https://github.com/facebook/rocksdb/blob/96206531bc0bb56d87012921c5458c8a3047a6b3/db/write_batch.cc#L813)) and [stores](https://github.com/facebook/rocksdb/blob/96206531bc0bb56d87012921c5458c8a3047a6b3/include/rocksdb/write_batch.h#L478) the protection information within the corresponding `WriteBatch` object as diagramed below. Then the user key and/or value are copied into the `WriteBatch`, thus starting per key-value checksum protection from user buffer. + +![](/static/images/kv-checksum/Writebatch-write.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + + +#### WAL write + +Before a `WriteBatch` leaves RocksDB and be persisted in a WAL file, it is verified against its `ProtectionInfo` to ensure its content is not corrupted. We added `WriteBatch::VerifyChecksum()` for this purpose. Once we verify the content of a `WriteBatch`, it is then divided into potentially multiple WAL fragments and persisted in the underlying file system. From that point on, the integrity protection is handed off to the per fragment crc32c checksum that is persisted in WAL too. + +![](/static/images/kv-checksum/WAL-write.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +#### Memtable write + +Similar to the WAL write path, `ProtectionInfo` is verified before an entry is inserted into a memtable. The difference here is that an memtable entry has its own buffer, and the content of a `WriteBatch` is copied into the memtable entry. So the `ProtectionInfo` is verified against the memtable entry buffer instead. The current per key-value checksum protection ends at this verification on the buffer containing a memtable entry, and one of the future work is to extend the coverage to key-value pairs in memtables. + +![](/static/images/kv-checksum/Memtable-write.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +#### WAL read + +This is for the DB recovery path: WAL fragments are read into memory, concatenated together to form WAL records, and then `WriteBatch`es are constructed from WAL records and added to memtables. In RocksDB 7.4, once a `WriteBatch` copies its content from a WAL record, `ProtectionInfo` is constructed from the `WriteBatch` content and per key-value protection starts. However, this copy operation is not protected, neither is the reconstruction of a WAL record from WAL fragments. To provide protection from silent data corruption during these memory copying operations, we added checksum handshake detailed below in RocksDB 7.5. + +When a WAL fragment is first read into memory, its crc32c checksum is [verified](https://github.com/facebook/rocksdb/blob/2f13f5f7d09c589d5adebf0cbc42fadf0da0f00e/db/log_reader.cc#L483). The WAL fragment is then appended to the buffer containing a WAL record. RocksDB uses xxh3’s streaming API to calculate the checksum of the WAL record and updates the streaming hash state with the new WAL fragment content whenever it is appended to the WAL record buffer (e.g. [here](https://github.com/facebook/rocksdb/blob/2f13f5f7d09c589d5adebf0cbc42fadf0da0f00e/db/log_reader.cc#L135)). After the WAL record is constructed, it is copied into a `WriteBatch` and `ProtectionInfo` is constructed from the write batch content. Then, the xxh3 checksum of the WAL record is [verified](https://github.com/facebook/rocksdb/blob/2f13f5f7d09c589d5adebf0cbc42fadf0da0f00e/db/write_batch.cc#L3081-L3085) against the write batch content to complete the checksum handshake. If the checksum verification succeeds, then we are more confident that `ProtectionInfo` is calculated based on uncorrupted data, and the protection coverage continues with the newly constructed `ProtectionInfo` along the write code paths mentioned above. + +![](/static/images/kv-checksum/WAL-read.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +## Future work + +Future coverage expansion will cover memtable KVs, flush, compaction and user reads etc. + +## References + +[1] http://rocksdb.org/blog/2021/05/26/online-validation.html + +[2] H. D. Dixit, L. Boyle, G. Vunnam, S. Pendharkar, M. Beadon, and S. Sankar, ‘Detecting silent data corruptions in the wild’. arXiv, 2022. + +[3] https://github.com/Cyan4973/xxHash + +[4] https://github.com/Cyan4973/xxHash/issues/229#issuecomment-511956403 diff --git a/librocksdb-sys/rocksdb/docs/_posts/2022-10-05-lost-buffered-write-recovery.markdown b/librocksdb-sys/rocksdb/docs/_posts/2022-10-05-lost-buffered-write-recovery.markdown new file mode 100644 index 0000000..fca3ea7 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2022-10-05-lost-buffered-write-recovery.markdown @@ -0,0 +1,123 @@ +--- +title: "Verifying crash-recovery with lost buffered writes" +layout: post +author: +- ajkr +category: blog +--- + +## Introduction + +Writes to a RocksDB instance go through multiple layers before they are fully persisted. +Those layers may buffer writes, delaying their persistence. +Depending on the layer, buffered writes may be lost in a process or system crash. +A process crash loses writes buffered in process memory only. +A system crash additionally loses writes buffered in OS memory. + +The new test coverage introduced in this post verifies there is no hole in the recovered data in either type of crash. +A hole would exist if any recovered write were newer than any lost write, as illustrated below. +This guarantee is important for many applications, such as those that use the newest recovered write to determine the starting point for replication. + +![](/static/images/lost-buffered-write-recovery/happy-cat.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +*Valid (no hole) recovery: all recovered writes (1 and 2) are older than all lost writes (3 and 4)* +{: style="text-align: center"} + +![](/static/images/lost-buffered-write-recovery/angry-cat.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +*Invalid (hole) recovery: a recovered write (4) is newer than a lost write (3)* +{: style="text-align: center"} + +The new test coverage assumes all writes use the same options related to buffering/persistence. +For example, we do not cover the case of alternating writes with WAL disabled and WAL enabled (`WriteOptions::disableWAL`). +It also assumes the crash does not have any unexpected consequences like corrupting persisted data. + +Testing for holes in the recovery is challenging because there are many valid recovery outcomes. +Our solution involves tracing all the writes and then verifying the recovery matches a prefix of the trace. +This proves there are no holes in the recovery. +See "Extensions for lost buffered writes" subsection below for more details. + +Testing actual system crashes would be operationally difficult. +Our solution simulates system crash by buffering written but unsynced data in process memory such that it is lost in a process crash. +See "Simulating system crash" subsection below for more details. + +## Scenarios covered + +We began testing recovery has no hole in the following new scenarios. +This coverage is included in our internal CI that periodically runs against the latest commit on the main branch. + +1. **Process crash with WAL disabled** (`WriteOptions::disableWAL=1`), which loses writes since the last memtable flush. +2. **System crash with WAL enabled** (`WriteOptions::disableWAL=0`), which loses writes since the last memtable flush or WAL sync (`WriteOptions::sync=1`, `SyncWAL()`, or `FlushWAL(true /* sync */)`). +3. **Process crash with manual WAL flush** (`DBOptions::manual_wal_flush=1`), which loses writes since the last memtable flush or manual WAL flush (`FlushWAL()`). +4. **System crash with manual WAL flush** (`DBOptions::manual_wal_flush=1`), which loses writes since the last memtable flush or synced manual WAL flush (`FlushWAL(true /* sync */)`, or `FlushWAL(false /* sync */)` followed by WAL sync). + +## Issues found + +* [False detection of corruption after system crash due to race condition with WAL sync and `track_and_verify_wals_in_manifest](https://github.com/facebook/rocksdb/pull/10185) +* [Undetected hole in recovery after system crash due to race condition in WAL sync](https://github.com/facebook/rocksdb/pull/10560) +* [Recovery failure after system crash due to missing directory sync for critical metadata file](https://github.com/facebook/rocksdb/pull/10573) + +## Solution details + +### Basic setup + +![](/static/images/lost-buffered-write-recovery/basic-setup.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +Our correctness testing framework consists of a stress test program (`db_stress`) and a wrapper script (`db_crashtest.py`). +`db_crashtest.py` manages instances of `db_stress`, starting them and injecting crashes. +`db_stress` operates a DB and test oracle ("Latest values file"). + +At startup, `db_stress` verifies the DB using the test oracle, skipping keys that had pending writes when the last crash happened. +`db_stress` then stresses the DB with random operations, keeping the test oracle up-to-date. + +As the name "Latest values file" implies, this test oracle only tracks the latest value for each key. +As a result, this setup is unable to verify recoveries involving lost buffered writes, where recovering older values is tolerated as long as there is no hole. + +### Extensions for lost buffered writes + +To accommodate lost buffered writes, we extended the test oracle to include two new files: "`verifiedSeqno`.state" and "`verifiedSeqno`.trace". +`verifiedSeqno` is the sequence number of the last successful verification. +"`verifiedSeqno`.state" is the expected values file at that sequence number, and "`verifiedSeqno`.trace" is the trace file of all operations that happened after that sequence number. + +![](/static/images/lost-buffered-write-recovery/replay-extension.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +When buffered writes may have been lost by the previous `db_stress` instance, the current `db_stress` instance must reconstruct the latest values file before startup verification. +M is the recovery sequence number of the current `db_stress` instance and N is the recovery sequence number of the previous `db_stress` instance. +M is learned from the DB, while N is learned from the filesystem by parsing the "*.{trace,state}" filenames. +Then, the latest values file ("LATEST.state") can be reconstructed by replaying the first M-N traced operations (in "N.trace") on top of the last instance's starting point ("N.state"). + +![](/static/images/lost-buffered-write-recovery/trace-extension.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +When buffered writes may be lost by the current `db_stress` instance, we save the current expected values into "M.state" and begin tracing newer operations in "M.trace". + +### Simulating system crash + +When simulating system crash, we send file writes to a `TestFSWritableFile`, which buffers unsynced writes in process memory. +That way, the existing `db_stress` process crash mechanism will lose unsynced writes. + +![](/static/images/lost-buffered-write-recovery/test-fs-writable-file.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +`TestFSWritableFile` is implemented as follows. + +* `Append()` buffers the write in a local `std::string` rather than calling `write()`. +* `Sync()` transfers the local `std::string`s content to `PosixWritableFile::Append()`, which will then `write()` it to the OS page cache. + +## Next steps +An untested guarantee is that RocksDB recovers all writes that the user explicitly flushed out of the buffers lost in the crash. +We may recover more writes than these due to internal flushing of buffers, but never less. +Our test oracle needs to be further extended to track the lower bound on the sequence number that is expected to survive a crash. + +We would also like to make our system crash simulation more realistic. +Currently we only drop unsynced regular file data, but we should drop unsynced directory entries as well. + +## Acknowledgements + +Hui Xiao added the manual WAL flush coverage and compatibility with `TransactionDB`. +Zhichao Cao added the system crash simulation. +Several RocksDB team members contributed to this feature's dependencies. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2022-10-07-asynchronous-io-in-rocksdb.markdown b/librocksdb-sys/rocksdb/docs/_posts/2022-10-07-asynchronous-io-in-rocksdb.markdown new file mode 100644 index 0000000..0586f1c --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2022-10-07-asynchronous-io-in-rocksdb.markdown @@ -0,0 +1,133 @@ +--- +title: Asynchronous IO in RocksDB +layout: post +author: +- akankshamahajan15 +- anand1976 +category: blog +--- +## Summary + +RocksDB provides several APIs to read KV pairs from a database, including Get and MultiGet for point lookups and Iterator for sequential scanning. These APIs may result in RocksDB reading blocks from SST files on disk storage. The types of blocks and the frequency with which they are read from storage is workload dependent. Some workloads may have a small working set and thus may be able to cache most of the data required, while others may have large working sets and have to read from disk more often. In the latter case, the latency would be much higher and throughput would be lower than the former. They would also be dependent on the characteristics of the underlying storage media, making it difficult to migrate from one medium to another, for example, local flash to disaggregated flash. + +One way to mitigate the impact of storage latency is to read asynchronously and in parallel as much as possible, in order to hide IO latency. We have implemented this in RocksDB in Iterators and MultiGet. In Iterators, we prefetch data asynchronously in the background for each file being iterated on, unlike the current implementation that does prefetching synchronously, thus blocking the iterator thread. In MultiGet, we determine the set of files that a given batch of keys overlaps, and read the necessary data blocks from those files in parallel using an asynchronous file system API. These optimizations have significantly decreased the overall latency of the RocksDB MultiGet and iteration APIs on slower storage compared to local flash. + +The optimizations described here are in the internal implementation of Iterator and MultiGet in RocksDB. The user API is still synchronous, so existing code can easily benefit from it. We might consider async user APIs in the future. + + +## Design + +### API + +A new flag in `ReadOptions`, `async_io`, controls the usage of async IO. This flag, when set, enables async IO in Iterators and MultiGet. For MultiGet, an additional `ReadOptions` flag, `optimize_multiget_for_io` (defaults to true), controls how aggressively to use async IO. If the flag is not set, files in the same level are read in parallel but not different levels. If the flag is set, the level restriction is removed and as many files as possible are read in parallel, regardless of level. The latter might have a higher CPU cost depending on the workload. + +At the FileSystem layer, we use the `FSRandomAccessFile::ReadAsync` API to start an async read, providing a completion callback. + +### Scan + +A RocksDB scan usually involves the allocation of a new iterator, followed by a Seek call with a target key to position the iterator, followed by multiple Next calls to iterate through the keys sequentially. Both the Seek and Next operations present opportunities to read asynchronously, thereby reducing the scan latency. + +A scan usually involves iterating through keys in multiple entities - the active memtable, sealed and unflushed memtables, every L0 file, and every non-empty non-zero level. The first two are completely in memory and thus not impacted by IO latency. The latter two involve reading from SST files. This means that an increase in IO latency has a multiplier effect, since multiple L0 files and levels have to be iterated on. + +Some factors, such as block cache and prefix bloom filters, can reduce the number of files to iterate and number of reads from the files. Nevertheless, even a few reads from disk can dominate the overall latency. RocksDB uses async IO in both Seek and Next to mitigate the latency impact, as described below. + + +#### Seek + +A RocksDB iterator maintains a collection of child iterators, one for each L0 file and for each non-empty non-zero levels. For a Seek operation every child iterator has to Seek to the target key. This is normally done serially, by doing synchronous reads from SST files when the required data blocks are not in cache. When the async_io option is enabled, RocksDB performs the Seek in 2 phases - 1) Locate the data block required for Seek in each file/level and issue an async read, and 2) in the second phase, reseek with the same key, which will wait for the async read to finish at each level and position the table iterator. Phase 1 reads multiple blocks in parallel, reducing overall Seek latency. + + +#### Next + +For the iterator Next operation, RocksDB tries to reduce the latency due to IO by prefetching data from the file. This prefetching occurs when a data block required by Next is not present in the cache. The reads from file and prefetching is managed by the FilePrefetchBuffer, which is an object that’s created per table iterator (BlockBasedTableIterator). The FilePrefetchBuffer reads the required data block, and an additional amount of data that varies depending on the options provided by the user in ReadOptions and BlockBasedTableOptions. The default behavior is to start prefetching on the third read from a file, with an initial prefetch size of 8KB and doubling it on every subsequent read, upto a max of 256KB. + +While the prefetching in the previous paragraph helps, it is still synchronous and contributes to the iterator latency. When the async_io option is enabled, RocksDB prefetches in the background, i.e while the iterator is scanning KV pairs. This is accomplished in FilePrefetchBuffer by maintaining two prefetch buffers. The prefetch size is calculated as usual, but its then split across the two buffers. As the iteration proceeds and data in the first buffer is consumed, the buffer is cleared and an async read is scheduled to prefetch additional data. This read continues in the background while the iterator continues to process data in the second buffer. At this point, the roles of the two buffers are reversed. This does not completely hide the IO latency, since the iterator would have to wait for an async read to complete after the data in memory has been consumed. However, it does hide some of it by overlapping CPU and IO, and async prefetch can be happening on multiple levels in parallel, further reducing the latency. + +![Scan flow](/static/images/asynchronous-io/scan_async.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +### MultiGet + +The MultiGet API accepts a batch of keys as input. Its a more efficient way of looking up multiple keys compared to a loop of Gets. One way MultiGet is more efficient is by reading multiple data blocks from an SST file in a batch, for keys in the same file. This greatly reduces the latency of the request, compared to a loop of Gets. The MultiRead FileSystem API is used to read a batch of data blocks. + +![MultiGet flow](/static/images/asynchronous-io/mget_async.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +Even with the MultiRead optimization, subset of keys that are in different files still need to be read serially. We can take this one step further and read multiple files in parallel. In order to do this, a few fundamental changes were required in the MultiGet implementation - + +1. Coroutines - A MultiGet involves determining the set of keys in a batch that overlap an SST file, and then calling TableReader::MultiGet to do the actual lookup. The TableReader probes the bloom filter, traverses the index block, looks up the block cache for the necessary, reads the missing data blocks from the SST file, and then searches for the keys in the data blocks. There is a significant amount of context that’s accumulated at each stage, and it would be rather complex to interleave data blocks reads by multiple TableReaders. In order to simplify it, we used async IO with C++ coroutines. The TableReader::MultiGet is implemented as a coroutine, and the coroutine is suspended after issuing async reads for missing data blocks. This allows the top-level MultiGet to iterate through the TableReaders for all the keys, before waiting for the reads to finish and resuming the coroutines. +2. Filtering - The downside of using coroutines is the CPU overhead, which is non-trivial. To minimize the overhead, its desirable to not use coroutines as much as possible. One scenario in which we can completely avoid the call to a TableReader::MultiGet coroutine is if we know that none of the overlapping keys are actually present in the SST file. This can easily determined by probing the bloom filter. In the previous implementation, the bloom filter lookup was embedded in TableReader::MultiGet. However, we could easily implement is as a separate step, before calling TableReader::MultiGet. +3. Splitting batches - The default strategy of MultiGet is to lookup keys in one level (or L0 file), before moving on to the next. This limits the amount of IO parallelism we can exploit. For example, the keys in a batch may not be clustered together, and may be scattered over multiple files. Even if they are clustered together in the key space, they may not all be in the same level. In order to optimize for these situations, we determine the subset of keys that are likely to be in a given level, and then split the MultiGet batch into 2 - the subset in that level, and the remainder. The batch containing the remainder can then be processed in parallel. The subset of keys likely to be in a level is determined by the filtering step. + +Together, these changes enabled two types of latency optimization in MultiGet using async IO - single-level and multi-level. The former reads data blocks in parallel from multiple files in the same LSM level, while the latter reads in parallel from multiple files in multiple levels. + +## Results + +Command used to generate the database: + +`buck-out/opt/gen/rocks/tools/rocks_db_bench —db=/rocks_db_team/prefix_scan —env_uri=ws://ws.flash.ftw3preprod1 -logtostderr=false -benchmarks="fillseqdeterministic" -key_size=32 -value_size=512 -num=5000000 -num_levels=4 -multiread_batched=true -use_direct_reads=false -adaptive_readahead=true -threads=1 -cache_size=10485760000 -async_io=false -multiread_stride=40000 -disable_auto_compactions=true -compaction_style=1 -bloom_bits=10` + +Structure of the database: + +`Level[0]: /000233.sst(size: 24828520 bytes)` +`Level[0]: /000232.sst(size: 49874113 bytes)` +`Level[0]: /000231.sst(size: 100243447 bytes)` +`Level[0]: /000230.sst(size: 201507232 bytes)` +`Level[1]: /000224.sst - /000229.sst(total size: 405046844 bytes)` +`Level[2]: /000211.sst - /000223.sst(total size: 814190051 bytes)` +`Level[3]: /000188.sst - /000210.sst(total size: 1515327216 bytes)` + + +### MultiGet + +MultiGet benchmark command: + +`buck-out/opt/gen/rocks/tools/rocks_db_bench -use_existing_db=true —db=/rocks_db_team/prefix_scan -benchmarks="multireadrandom" -key_size=32 -value_size=512 -num=5000000 -batch_size=8 -multiread_batched=true -use_direct_reads=false -duration=60 -ops_between_duration_checks=1 -readonly=true -threads=4 -cache_size=300000000 -async_io=true -multiread_stride=40000 -statistics —env_uri=ws://ws.flash.ftw3preprod1 -logtostderr=false -adaptive_readahead=true -bloom_bits=10` + +#### Single-file + +The default MultiGet implementation of reading from one file at a time had a latency of 1292 micros/op. + +`multireadrandom : 1291.992 micros/op 3095 ops/sec 60.007 seconds 185768 operations; 1.6 MB/s (46768 of 46768 found) ` +`rocksdb.db.multiget.micros P50 : 9664.419795 P95 : 20757.097056 P99 : 29329.444444 P100 : 46162.000000 COUNT : 23221 SUM : 239839394` + +#### Single-level + +MultiGet with async_io=true and optimize_multiget_for_io=false had a latency of 775 micros/op. + +`multireadrandom : 774.587 micros/op 5163 ops/sec 60.009 seconds 309864 operations; 2.7 MB/s (77816 of 77816 found)` +`rocksdb.db.multiget.micros P50 : [6029.601964](tel:6029601964) P95 : 10727.467932 P99 : 13986.683940 P100 : 47466.000000 COUNT : 38733 SUM : 239750172` + +#### Multi-level + +With all optimizations turned on, MultiGet had the lowest latency of 508 micros/op. + +`multireadrandom : 507.533 micros/op 7881 ops/sec 60.003 seconds 472896 operations; 4.1 MB/s (117536 of 117536 found)` +`rocksdb.db.multiget.micros P50 : 3923.819467 P95 : 7356.182075 P99 : 10880.728723 P100 : 28511.000000 COUNT : 59112 SUM : 239642721` + +### Scan + +Benchmark command: + +`buck-out/opt/gen/rocks/tools/rocks_db_bench -use_existing_db=true —db=/rocks_db_team/prefix_scan -ben``chmarks="seekrandom" -key_size=32 -value_size=512 -num=5000000 -batch_size=8 -multiread_batched=true -use_direct_reads=false -duration=60 -ops_between_duration_che``cks=1 -readonly=true -threads=4 -cache_size=300000000 -async_io=true -multiread_stride=40000 -statistics —env_uri=ws://ws.flash.ftw3preprod1 -logtostderr=false -a``daptive_readahead=true -bloom_bits=10 -seek_nexts=65536` + +### With async scan + +`seekrandom : 414442.303 micros/op 9 ops/sec 60.288 seconds 581 operations; 326.2 MB/s (145 of 145 found)` + +### Without async scan + +`seekrandom : 848858.669 micros/op 4 ops/sec 60.529 seconds 284 operations; 158.1 MB/s (74 of 74 found)` + +## Known Limitations + +These optimizations apply only to block based table SSTs. File system support for the `ReadAsync` and `Poll` interfaces is required. Currently, it is available only for `PosixFileSystem`. + +The MultiGet async IO optimization has a few additional limitations - + +1. Depends on folly, which introduces a few additional build steps +2. Higher CPU overhead due to coroutines. The CPU overhead of MultiGet may increase 6-15%, with the worst case being a single threaded MultiGet batch of keys with 1 key/file intersection and 100% cache hit rate. A more realistic case of multiple threads with a few keys (~4) overlap per file should see ~6% higher CPU util. +3. No parallelization of metadata reads. A metadata read will block the thread. +4. A few other cases will also be in serial, such as additional block reads for merge operands. + + diff --git a/librocksdb-sys/rocksdb/docs/_posts/2022-10-31-align-compaction-output-file.markdown b/librocksdb-sys/rocksdb/docs/_posts/2022-10-31-align-compaction-output-file.markdown new file mode 100644 index 0000000..a2db41b --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2022-10-31-align-compaction-output-file.markdown @@ -0,0 +1,107 @@ +--- +title: Reduce Write Amplification by Aligning Compaction Output File Boundaries +layout: post +author: +- zjay +category: blog +--- +## TL;DR +By cutting the compaction output file earlier and allowing larger than targeted_file_size to align the compaction output files to the next level files, it can **reduce WA (Write Amplification) by more than 10%**. The feature is **enabled by default** after the user upgrades RocksDB to version `7.8.0+`. + +## Background +RocksDB level compaction picks one file from the source level and compacts to the next level, which is a typical partial merge compaction algorithm. Compared to the full merge compaction strategy for example [universal compaction](https://github.com/facebook/rocksdb/wiki/Universal-Compaction), it has the benefits of smaller compaction size, better parallelism, etc. But it also has a larger write amplification (typically 20-30 times user data). One of the problems is wasted compaction at the beginning and ending: + +![](/static/images/align-compaction-output/file_cut_normal.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +In the diagram above, `SST11` is selected for the compaction, it overlaps with `SST20` to `SST23`, so all these files are selected for compaction. But the beginning and ending of the SST on Level 2 are wasted, which also means it will be compacted again when `SST10` is compacting down. If the file boundaries are aligned, then the wasted compaction size could be reduced. On average, the wasted compaction is `1` file size: `0.5` at the beginning, and `0.5` at the end. Typically the average compaction fan-out is about 6 (with the default max_bytes_for_level_multiplier = 10), then `1 / (6 + 1) ~= 14%` of compaction is wasted. +## implementation +To reduce such wasted compaction, RocksDB now tries to align the compaction output file to the next level's file. So future compactions will have fewer wasted compaction. For example, the above case might be cut like this: + +![](/static/images/align-compaction-output/file_cut_align.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +The trade-off is the file won't be cut exactly after it exceeds target_file_size_base, instead, it will be more likely cut when it's aligned with the next level file's boundary, so the file size might be more varied. It could be as small as 50% of `target_file_size` or as large as `2x target_file_size`. It will only impact non-bottommost-level files, which should be only `~11%` of the data. +Internally, RocksDB tries to cut the file so its size is close to the `target_file_size` setting but also aligned with the next level boundary. When the compaction output file hit a next-level file boundary, either the beginning or ending boundary, it will cut if: +``` +current_size > ((5 * min(bounderies_num, 8) + 50) / 100) * target_file_size +``` +([details](https://github.com/facebook/rocksdb/blob/23fa5b7789d6acd0c211d6bdd41448bbf1513bb6/db/compaction/compaction_outputs.cc#L270-L290)) + +The file size is also capped at `2x target_file_size`: [details](https://github.com/facebook/rocksdb/blob/f726d29a8268ae4e2ffeec09172383cff2ab4db9/db/compaction/compaction.cc#L273-L277). +Another benefit of cutting the file earlier is having more trivial move compaction, which is moving the file from a high level to a low level without compacting anything. Based on a compaction simulator test, the trivial move data is increased by 30% (but still less than 1% compaction data is trivial move): + +![](/static/images/align-compaction-output/file_cut_trival_move.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +Based on the db_bench test, it can save `~12%` compaction load, here is the test command and result: +``` +TEST_TMPDIR=/data/dbbench ./db_bench --benchmarks=fillrandom,readrandom -max_background_jobs=12 -num=400000000 -target_file_size_base=33554432 + +# baseline: +Flush(GB): cumulative 25.882, interval 7.216 +Cumulative compaction: 285.90 GB write, 162.36 MB/s write, 269.68 GB read, 153.15 MB/s read, 2926.7 seconds + +# with this change: +Flush(GB): cumulative 25.882, interval 7.753 +Cumulative compaction: 249.97 GB write, 141.96 MB/s write, 233.74 GB read, 132.74 MB/s read, 2534.9 seconds +``` + +The feature is enabled by default by upgrading to RocksDB 7.8 or later versions, as the feature should have a limited impact on the file size and have great write amplification improvements. If in a rare case, it needs to opt out, set +``` +options.level_compaction_dynamic_file_size = false; +``` + +## Other Options and Benchmark +We also tested a few other options, starting with a fixed threshold: 75% of the target_file_size and 50%. Then with a dynamic threshold that is explained, but still limiting file size smaller than the target_file_size. +1. Baseline (main branch before [PR#10655](https://github.com/facebook/rocksdb/pull/10655)); +2. Fixed Threshold `75%`: after 75% of target file size, cut the file whenever it aligns with a low level file boundary; +3. Fixed Threshold `50%`: reduce the threshold to 50% of target file size; +4. Dynamic Threshold `(5*bounderies_num + 50)` percent of target file size and maxed at 90%; +5. Dynamic Threshold + allow 2x the target file size (chosen option). + +### Test Environment and Data +To speed up the benchmark, we introduced a compaction simulator within Rocksdb ([details](https://github.com/jay-zhuang/rocksdb/tree/compaction_sim)), which replaced the physical SST with in-memory data (a large bitset). Which can test compaction more consistently. As it's a simulator, it has its limitations: + +it assumes each key-value has the same size; +1. no deletion (but has override); +2. doesn't consider data compression; +3. single-threaded and finish all compactions before the next flush (so no write stall). + +We use 3 kinds of the dataset for tests: +1. Random Data, has an override, evenly distributed; +2. Zipf distribution with alpha = 1.01, moderately skewed; +3. Zipf distribution with alpha = 1.2, highly skewed. + +#### Write Amplification + +![](/static/images/align-compaction-output/write_amp_compare.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 100%"} + +As we can see, all options are better than the baseline. Option5 (brown) and option3 (green) have similar WA improvements. (The sudden WA drop during ~40G Random Dataset is because we enabled `level_compaction_dynamic_level_bytes` and the level number was increased from 3 to 4, the similar test result without enabling `level_compaction_dynamic_level_bytes`). + +#### File Size Distribution at the End of Test +This is the file size distribution at the end of the test, which loads about 100G data. As this change only impacts the non-bottommost file size, and the majority of the SST files are bottommost, there're no significant differences: + +![](/static/images/align-compaction-output/file_size_compare.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 100%"} + +#### All Compaction Generated File Sizes +The high-level files are much more likely to be compacted, so all compaction-generated files size has more significant change: + +![](/static/images/align-compaction-output/compaction_output_file_size_compare.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 100%"} + +Overall option5 has most of the file size close to the target file size. vs. option3 has a much smaller size. Here are more detailed stats for compaction output file size: +``` + base 50p 75p dynamic 2xdynamic +count 1.656000e+03 1.960000e+03 1.770000e+03 1.687000e+03 1.705000e+03 +mean 3.116062e+07 2.634125e+07 2.917876e+07 3.060135e+07 3.028076e+07 +std 7.145242e+06 1.065134e+07 8.800474e+06 7.612939e+06 8.046139e+06 +``` + +## Summary +Allowing more dynamic file size and aligning the compaction output file to the next level file's boundary improves the RocksDB write amplification by more than 10%, which will be enabled by default in `7.8.0` release. We picked a simple algorithm to decide when to cut the output file, which can be further improved. For example, by estimating output file size with index information. Any suggestions or PR are welcomed. + +## Acknowledgements +We thank Siying Dong for initializing the file-cutting idea and thank Andrew Kryczka, Mark Callaghan for contributing to the ideas. And Changyu Bi for the detailed code review. diff --git a/librocksdb-sys/rocksdb/docs/_posts/2022-11-09-time-aware-tiered-storage.markdown b/librocksdb-sys/rocksdb/docs/_posts/2022-11-09-time-aware-tiered-storage.markdown new file mode 100644 index 0000000..03a6b02 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_posts/2022-11-09-time-aware-tiered-storage.markdown @@ -0,0 +1,121 @@ +--- +title: Time-Aware Tiered Storage in RocksDB +layout: post +author: +- zjay +category: blog +--- +## TL:DR +Tiered storage is now natively supported in the RocksDB with the option [`last_level_temperature`](https://github.com/facebook/rocksdb/blob/b0d9776b704af01c2b5385e9d53754e0c8176373/include/rocksdb/advanced_options.h#L910), time-aware Tiered storage feature guarantees the recently written data are put in the hot tier storage with the option [`preclude_last_level_data_seconds`](https://github.com/facebook/rocksdb/blob/b0d9776b704af01c2b5385e9d53754e0c8176373/include/rocksdb/advanced_options.h#L927). + +## Background +RocksDB Tiered Storage assigns a data temperature when creating the new SST which [hints the file system](https://github.com/facebook/rocksdb/blob/b0d9776b704af01c2b5385e9d53754e0c8176373/include/rocksdb/file_system.h#L162) to put the data on the corresponding storage media, so the data in a single DB instance can be placed on different storage media. Before the feature, the user typically creates multiple DB instances for different storage media, for example, one DB instance stores the recent hot data and migrates the data to another cold DB instance when the data becomes cold. Tracking and migrating the data could be challenging. With the RocksDB tiered storage feature, RocksDB compaction migrates the data from hot storage to cold storage. + +![](/static/images/time-aware-tiered-storage/tiered_storage_overview.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +Currently, RocksDB supports assigning the last level file temperature. In an LSM tree, typically the last level data is most likely the coldest. As the most recent data is on the higher level and gradually compacted to the lower level. The higher level data is more likely to be read, because: +1. RocksDB read always queries from the higher level to the lower level until it finds the data; +2. The high-level data is much more likely to be read and written by the compactions. + +### Problem +Generally in the LSM tree, hotter data is likely on the higher levels as mentioned before, **but it is not always the case**, for example for the skewed dataset, the recent data could be compacted to the last level first. For the universal compaction, a major compaction would compact all data to the last level (the cold tier) which includes both recent data that should be cataloged as hot data. In production, **we found the majority of the compaction load is actually major compaction (more than 80%)**. + +![](/static/images/time-aware-tiered-storage/tiered_storage_problem.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +### Goal and Non-goals +It’s hard to predict the hot and cold data. The most frequently accessed data should be cataloged as hot data. But it is hard to predict which key is going to be accessed most, it is also hard to track the per-key based access history. The time-aware tiered storage feature is only **focusing on the use cases that the more recent data is more likely to be accessed**. Which is the majority of the cases, but not all. + +## User APIs +Here are the 3 main tiered storage options: +```c++ +Temperature last_level_temperature = Temperature::kUnknown; +uint64_t preclude_last_level_data_seconds = 0; +uint64_t preserve_internal_time_seconds = 0; +``` +[`last_level_temperature`](https://github.com/facebook/rocksdb/blob/b0d9776b704af01c2b5385e9d53754e0c8176373/include/rocksdb/advanced_options.h#L910) defines the data temperature for the last level SST files, which is typically kCold or kWarm. RocksDB doesn’t check the option value, instead it just passes that to the file_system API with [`FileOptions.temperature`](https://github.com/facebook/rocksdb/blob/b0d9776b704af01c2b5385e9d53754e0c8176373/include/rocksdb/file_system.h#L162) when creating the last level SST files. For all the other files, non-last-level SST files, and non-SST files like manifest files, the temperature is set to kUnknown, which typically maps to hot data. +The user can also get each SST’s temperature information through APIs: +```c++ +db.GetLiveFilesStorageInfo(); +db.GetLiveFilesMetaData(); +db.GetColumnFamilyMetaData(); +``` + +### User Metrics +Here are the tiered storage related statistics: +```c++ +HOT_FILE_READ_BYTES, +WARM_FILE_READ_BYTES, +COLD_FILE_READ_BYTES, +HOT_FILE_READ_COUNT, +WARM_FILE_READ_COUNT, +COLD_FILE_READ_COUNT, +// Last level and non-last level statistics +LAST_LEVEL_READ_BYTES, +LAST_LEVEL_READ_COUNT, +NON_LAST_LEVEL_READ_BYTES, +NON_LAST_LEVEL_READ_COUNT, +``` + +And more details from `IOStats`: +```c++ +struct FileIOByTemperature { +// the number of bytes read to Temperature::kHot file +uint64_t hot_file_bytes_read; +// the number of bytes read to Temperature::kWarm file +uint64_t warm_file_bytes_read; +// the number of bytes read to Temperature::kCold file +uint64_t cold_file_bytes_read; +// total number of reads to Temperature::kHot file +uint64_t hot_file_read_count; +// total number of reads to Temperature::kWarm file +uint64_t warm_file_read_count; +// total number of reads to Temperature::kCold file +uint64_t cold_file_read_count; +``` + +## Implementation +There are 2 main components for this feature. One is the **time-tracking**, and another is the **per-key based placement compaction**. These 2 components are relatively independent and linked together during the compaction initialization phase which gets the sequence number for splitting the hot and cold data. The time-tracking components can even be enabled independently by setting the option [`preserve_internal_time_seconds`](https://github.com/facebook/rocksdb/blob/b0d9776b704af01c2b5385e9d53754e0c8176373/include/rocksdb/advanced_options.h#L950). The purpose of that is before migrating existing user cases to the tiered storage feature and avoid compacting the existing hot data to the cold tier (detailed in the migration session below). + +Unlike the user-defined timestamp feature, the time tracking feature doesn’t have accurate time information for each key. It only samples the time information and gives a rough estimation for the key write time. Here is the high-level graph for the implementation: + +![](/static/images/time-aware-tiered-storage/tiered_storage_design.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +### Time Tracking +Time tracking information is recorded by a [periodic task](https://github.com/facebook/rocksdb/blob/d9e71fb2c53726d9c5ed73b4ec962a7ed6ef15ec/db/periodic_task_scheduler.cc#L36) which gets the latest sequence number and the current time and then stores it in an in-memory data structure. The interval of the periodic task is determined by the user setting [`preserve_internal_time_seconds`](https://github.com/facebook/rocksdb/blob/b0d9776b704af01c2b5385e9d53754e0c8176373/include/rocksdb/advanced_options.h#L950) and dividing that by 100. For example, if 3 days of data should be precluded from the last level, then the interval of the periodic task is about 0.7 hours (3 * 24 / 100 ~= 0.72), which also means only the latest 100 seq->time pairs needed in memory. + +Currently, the in-memory seq_time_mapping is only used during Flush() and encoded to the SST property. The data is delta encoded and again maximum 100 pairs are stored, so the extra data size is pretty small (far less than 1KB per SST) and only non-last-level SSTs need to have that information. Internally, RocksDB also uses the minimal sequence number and SST creation time from the SST metadata to improve the time accuracy. +**The sequence number to time information is distributed in each SST**, ranging from the min seqno to max seqno for that SST file, so each SST has its self-contained time information. This also means there could be redundancy for the time information, for example, if 2 SSTs have an overlapped sequence number (which is very likely for non-L0 files), the same seq->time pair may exist in both SSTs. +For the future, the time information could also be useful for other potential features like a better estimate of the oldest timestamp for an SST which is critical for the RocksDB TTL feature. + +### Per-Key Placement Compaction + +![](/static/images/time-aware-tiered-storage/per_key_placement_compaction.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +Compare to normal compaction which only outputs the data to a single level, Per-key placement compaction can output data to 2 different levels, as per-per placement compaction is only for the last level compaction, so the 2 output levels would **always be the penultimate level, and the last level**. The compaction places the key to its corresponding tier by simply checking the key’s sequence number. + +At the beginning of the compaction, the compaction job collects all seq to time information from every input SSTs and merges them together, then based on the current time to get the oldest sequence number that should be put into non-last-level (hot tier). During the last level compaction, as long as the key is newer than the oldest_sequence_number, it will be placed in the penultimate level (hot tier) instead of the last level (cold tier). + +Note, RocksDB also places the keys that are within the user snapshot in the hot tier, there’re a few reasons for that: +1. It’s reasonable to assume snapshot-protected data are hot data; +2. Avoid mixing the sequence number not zeroed out data with old last-level data, which is desirable to reduce the oldest obsolete data time (it’s defined as the oldest SST time that has a non-zero sequence number). It also means tombstones are always placed in the hot tier, which is also desirable as it should be pretty small. +3. The original motivation was to avoid moving data from the lower level to a higher level in case the user increases the [`preclude_last_level_data_seconds`](https://github.com/facebook/rocksdb/blob/b0d9776b704af01c2b5385e9d53754e0c8176373/include/rocksdb/advanced_options.h#L927), so the snapshot-protected data in the last level will become hot again, and moving data to a higher level. It’s not always safe to move data from a lower level to a higher level in the LSM tree which could cause key conflict. Later we added a conflict check to allow the data to move up as long as there’s no key conflict, but then the movement is not guaranteed (see Migration for details) + +### Migration +Once the user enables the feature, it enables both time tracking and per-key placement compaction **at the same time**. As the existing data, it can still be mismarked as cold data. To have a smooth migration to the feature. The user can enable the time-tracking feature first. For example, if the user plans to set [`preclude_last_level_data_seconds`](https://github.com/facebook/rocksdb/blob/b0d9776b704af01c2b5385e9d53754e0c8176373/include/rocksdb/advanced_options.h#L927) to 3 days, the user can enable time tracking 3 days earlier with [`preserve_internal_time_seconds`](https://github.com/facebook/rocksdb/blob/b0d9776b704af01c2b5385e9d53754e0c8176373/include/rocksdb/advanced_options.h#L950). Then when enabling the tiered storage feature, it already has the time information for the last 3 days' hot data, then per-key placement compaction won’t compact them to the last level. + +Just preserving the time information won’t prevent the data from compacting to the last level (which should be still on the hot tier). Once the [`preclude_last_level_data_seconds`](https://github.com/facebook/rocksdb/blob/b0d9776b704af01c2b5385e9d53754e0c8176373/include/rocksdb/advanced_options.h#L927) and [`last_level_temperature`](https://github.com/facebook/rocksdb/blob/b0d9776b704af01c2b5385e9d53754e0c8176373/include/rocksdb/advanced_options.h#L910) features are enabled, some of the last-level data might need to move up. Currently, RocksDB just does a conflict check, the hot/cold split in this case is not guaranteed. + +![](/static/images/time-aware-tiered-storage/compaction_moving_up_conflict.png) +{: style="display: block; margin-left: auto; margin-right: auto; width: 80%"} + +## Summary +Time-aware tired storage feature guarantees the new data is placed in the hot tier, which **is ideal for the tiering use cases where the most recent data is likely the hot data**. It’s done by tracking the write time information and per-key placement compaction to split the hot/cold data. + +The tiered storage feature is actively being developed, any suggestions or PRs will be welcomed. + +## Acknowledgements +We thank Siying Dong and Andrew Kryczka for brainstorming and reviewing the feature design and implementation. And it was my fortune to work with the RocksDB team members! \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_sass/_base.scss b/librocksdb-sys/rocksdb/docs/_sass/_base.scss new file mode 100644 index 0000000..6d26d9f --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_sass/_base.scss @@ -0,0 +1,492 @@ +body { + background: $secondary-bg; + color: $text; + font: normal #{$base-font-size}/#{$base-line-height} $base-font-family; + height: 100vh; + text-align: left; + text-rendering: optimizeLegibility; +} + +img { + max-width: 100%; +} + +article { + p { + img { + max-width: 100%; + display:block; + margin-left: auto; + margin-right: auto; + } + } +} + +a { + border-bottom: 1px dotted $primary-bg; + color: $text; + text-decoration: none; + -webkit-transition: background 0.3s, color 0.3s; + transition: background 0.3s, color 0.3s; +} + +blockquote { + padding: 15px 30px 15px 15px; + margin: 20px 0 0 10px; + background-color: rgba(204, 122, 111, 0.1); + border-left: 10px solid rgba(191, 87, 73, 0.2); +} + +#fb_oss a { + border: 0; +} + +h1, h2, h3, h4 { + font-family: $header-font-family; + font-weight: 900; +} + +.navPusher { + border-top: $header-height + $header-ptop + $header-pbot solid $primary-bg; + height: 100%; + left: 0; + position: relative; + z-index: 99; +} + +.homeContainer { + background: $primary-bg; + color: $primary-overlay; + + a { + color: $primary-overlay; + } + + .homeSplashFade { + color: white; + } + + .homeWrapper { + padding: 2em 10px; + text-align: left; + + .wrapper { + margin: 0px auto; + max-width: $content-width; + padding: 0 20px; + } + + .projectLogo { + img { + height: 100px; + margin-bottom: 0px; + } + } + + h1#project_title { + font-family: $header-font-family; + font-size: 300%; + letter-spacing: -0.08em; + line-height: 1em; + margin-bottom: 80px; + } + + h2#project_tagline { + font-family: $header-font-family; + font-size: 200%; + letter-spacing: -0.04em; + line-height: 1em; + } + } +} + +.wrapper { + margin: 0px auto; + max-width: $content-width; + padding: 0 10px; +} + +.projectLogo { + display: none; + + img { + height: 100px; + margin-bottom: 0px; + } +} + +section#intro { + margin: 40px 0; +} + +.fbossFontLight { + font-family: $base-font-family; + font-weight: 300; + font-style: normal; +} + +.fb-like { + display: block; + margin-bottom: 20px; + width: 100%; +} + +.center { + display: block; + text-align: center; +} + +.mainContainer { + background: $secondary-bg; + overflow: auto; + + .mainWrapper { + padding: 4vh 10px; + text-align: left; + + .allShareBlock { + padding: 10px 0; + + .pluginBlock { + margin: 12px 0; + padding: 0; + } + } + + a { + &:hover, + &:focus { + background: $primary-bg; + color: $primary-overlay; + } + } + + em, i { + font-style: italic; + } + + strong, b { + font-weight: bold; + } + + h1 { + font-size: 300%; + line-height: 1em; + padding: 1.4em 0 1em; + text-align: left; + } + + h2 { + font-size: 250%; + line-height: 1em; + margin-bottom: 20px; + padding: 1.4em 0 20px; + text-align: left; + + & { + border-bottom: 1px solid darken($primary-bg, 10%); + color: darken($primary-bg, 10%); + font-size: 22px; + padding: 10px 0; + } + + &.blockHeader { + border-bottom: 1px solid white; + color: white; + font-size: 22px; + margin-bottom: 20px; + padding: 10px 0; + } + } + + h3 { + font-size: 150%; + line-height: 1.2em; + padding: 1em 0 0.8em; + } + + h4 { + font-size: 130%; + line-height: 1.2em; + padding: 1em 0 0.8em; + } + + p { + padding: 0.8em 0; + } + + ul { + list-style: disc; + } + + ol, ul { + padding-left: 24px; + li { + padding-bottom: 4px; + padding-left: 6px; + } + } + + strong { + font-weight: bold; + } + + .post { + position: relative; + + .katex { + font-weight: 700; + } + + &.basicPost { + margin-top: 30px; + } + + a { + color: $primary-bg; + + &:hover, + &:focus { + color: #fff; + } + } + + h2 { + border-bottom: 4px solid $primary-bg; + font-size: 130%; + } + + h3 { + border-bottom: 1px solid $primary-bg; + font-size: 110%; + } + + ol { + list-style: decimal outside none; + } + + .post-header { + padding: 1em 0; + + h1 { + font-size: 150%; + line-height: 1em; + padding: 0.4em 0 0; + + a { + border: none; + } + } + + .post-meta { + color: $primary-bg; + font-family: $header-font-family; + text-align: center; + } + } + + .postSocialPlugins { + padding-top: 1em; + } + + .docPagination { + background: $primary-bg; + bottom: 0px; + left: 0px; + position: absolute; + right: 0px; + + .pager { + display: inline-block; + width: 50%; + } + + .pagingNext { + float: right; + text-align: right; + } + + a { + border: none; + color: $primary-overlay; + display: block; + padding: 4px 12px; + + &:hover { + background-color: $secondary-bg; + color: $text; + } + + .pagerLabel { + display: inline; + } + + .pagerTitle { + display: none; + } + } + } + } + + .posts { + .post { + margin-bottom: 6vh; + } + } + } +} + +#integrations_title { + font-size: 250%; + margin: 80px 0; +} + +.ytVideo { + height: 0; + overflow: hidden; + padding-bottom: 53.4%; /* 16:9 */ + padding-top: 25px; + position: relative; +} + +.ytVideo iframe, +.ytVideo object, +.ytVideo embed { + height: 100%; + left: 0; + position: absolute; + top: 0; + width: 100%; +} + +@media only screen and (min-width: 480px) { + h1#project_title { + font-size: 500%; + } + + h2#project_tagline { + font-size: 250%; + } + + .projectLogo { + img { + margin-bottom: 10px; + height: 200px; + } + } + + .homeContainer .homeWrapper { + padding-left: 10px; + padding-right: 10px; + } + + .mainContainer { + .mainWrapper { + .post { + h2 { + font-size: 180%; + } + + h3 { + font-size: 120%; + } + + .docPagination { + a { + .pagerLabel { + display: none; + } + .pagerTitle { + display: inline; + } + } + } + } + } + } +} + +@media only screen and (min-width: 900px) { + .homeContainer { + .homeWrapper { + position: relative; + + #inner { + box-sizing: border-box; + max-width: 600px; + padding-right: 40px; + } + + .projectLogo { + align-items: center; + bottom: 0; + display: flex; + justify-content: flex-end; + left: 0; + padding: 2em 20px 4em; + position: absolute; + right: 20px; + top: 0; + + img { + height: 100%; + max-height: 250px; + } + } + } + } +} + +@media only screen and (min-width: 1024px) { + .mainContainer { + .mainWrapper { + .post { + box-sizing: border-box; + display: block; + + .post-header { + h1 { + font-size: 250%; + } + } + } + + .posts { + .post { + margin-bottom: 4vh; + width: 100%; + } + } + } + } +} + +@media only screen and (min-width: 1200px) { + .homeContainer { + .homeWrapper { + #inner { + max-width: 750px; + } + } + } + + .wrapper { + max-width: 1100px; + } +} + +@media only screen and (min-width: 1500px) { + .homeContainer { + .homeWrapper { + #inner { + max-width: 1100px; + padding-bottom: 40px; + padding-top: 40px; + } + } + } + + .wrapper { + max-width: 1400px; + } +} diff --git a/librocksdb-sys/rocksdb/docs/_sass/_blog.scss b/librocksdb-sys/rocksdb/docs/_sass/_blog.scss new file mode 100644 index 0000000..12a73c1 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_sass/_blog.scss @@ -0,0 +1,47 @@ +.blogContainer { + .posts { + margin-top: 60px; + + .post { + border: 1px solid $primary-bg; + border-radius: 3px; + padding: 10px 20px 20px; + } + } + + .lonePost { + margin-top: 60px; + + .post { + padding: 10px 0px 0px; + } + } + + .post-header { + h1 { + text-align: center; + } + + .post-authorName { + color: rgba($text, 0.7); + font-size: 14px; + font-weight: 900; + margin-top: 0; + padding: 0; + text-align: center; + } + + .authorPhoto { + border-radius: 50%; + height: 50px; + left: 50%; + margin-left: auto; + margin-right: auto; + display: inline-block; + overflow: hidden; + position: static; + top: -25px; + width: 50px; + } + } +} diff --git a/librocksdb-sys/rocksdb/docs/_sass/_buttons.scss b/librocksdb-sys/rocksdb/docs/_sass/_buttons.scss new file mode 100644 index 0000000..a037161 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_sass/_buttons.scss @@ -0,0 +1,47 @@ +.button { + border: 1px solid $primary-bg; + border-radius: 3px; + color: $primary-bg; + display: inline-block; + font-size: 14px; + font-weight: 900; + line-height: 1.2em; + padding: 10px; + text-transform: uppercase; + transition: background 0.3s, color 0.3s; + + &:hover { + background: $primary-bg; + color: $primary-overlay; + } +} + +.homeContainer { + .button { + border-color: $primary-overlay; + border-width: 1px; + color: $primary-overlay; + + &:hover { + background: $primary-overlay; + color: $primary-bg; + } + } +} + +.blockButton { + display: block; +} + +.edit-page-link { + float: right; + font-size: 14px; + font-weight: normal; + line-height: 20px; + opacity: 0.6; + transition: opacity 0.5s; +} + +.edit-page-link:hover { + opacity: 1; +} diff --git a/librocksdb-sys/rocksdb/docs/_sass/_footer.scss b/librocksdb-sys/rocksdb/docs/_sass/_footer.scss new file mode 100644 index 0000000..5b74395 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_sass/_footer.scss @@ -0,0 +1,82 @@ +.footerContainer { + background: $secondary-bg; + color: $primary-bg; + overflow: hidden; + padding: 0 10px; + text-align: left; + + .footerWrapper { + border-top: 1px solid $primary-bg; + padding: 0; + + .footerBlocks { + align-items: center; + align-content: center; + display: flex; + flex-flow: row wrap; + margin: 0 -20px; + padding: 10px 0; + } + + .footerSection { + box-sizing: border-box; + flex: 1 1 25%; + font-size: 14px; + min-width: 275px; + padding: 0px 20px; + + a { + border: 0; + color: inherit; + display: inline-block; + line-height: 1.2em; + } + + .footerLink { + padding-right: 20px; + } + } + + .fbOpenSourceFooter { + align-items: center; + display: flex; + flex-flow: row nowrap; + max-width: 25%; + + .facebookOSSLogoSvg { + flex: 0 0 31px; + height: 30px; + margin-right: 10px; + width: 31px; + + path { + fill: $primary-bg; + } + + .middleRing { + opacity: 0.7; + } + + .innerRing { + opacity: 0.45; + } + } + + h2 { + display: block; + font-weight: 900; + line-height: 1em; + } + } + } +} + +@media only screen and (min-width: 900px) { + .footerSection { + &.rightAlign { + margin-left: auto; + max-width: 25%; + text-align: right; + } + } +} \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_sass/_gridBlock.scss b/librocksdb-sys/rocksdb/docs/_sass/_gridBlock.scss new file mode 100644 index 0000000..679b31c --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_sass/_gridBlock.scss @@ -0,0 +1,115 @@ +.gridBlock { + margin: -5px 0; + padding: 0; + padding-bottom: 20px; + + .blockElement { + padding: 5px 0; + + img { + max-width: 100%; + } + + h3 { + border-bottom: 1px solid rgba($primary-bg, 0.5); + color: $primary-bg; + font-size: 18px; + margin: 0; + padding: 10px 0; + } + } + + .gridClear { + clear: both; + } + +} + +.gridBlock .alignCenter { + text-align: center; +} +.gridBlock .alignRight { + text-align: right; +} +.gridBlock .imageAlignSide { + align-items: center; + display: flex; + flex-flow: row wrap; +} +.blockImage { + max-width: 150px; + width: 50%; +} +.imageAlignTop .blockImage { + margin-bottom: 20px; +} +.imageAlignTop.alignCenter .blockImage { + margin-left: auto; + margin-right: auto; +} +.imageAlignSide .blockImage { + flex: 0 1 100px; + margin-right: 20px; +} +.imageAlignSide .blockContent { + flex: 1 1; +} + +@media only screen and (max-width: 1023px) { + .responsiveList .blockContent { + position: relative; + } + .responsiveList .blockContent > div { + padding-left: 20px; + } + .responsiveList .blockContent::before { + content: "\2022"; + position: absolute; + } +} + +@media only screen and (min-width: 1024px) { + .gridBlock { + display: flex; + flex-direction: row; + flex-wrap: wrap; + margin: -10px -10px 10px -10px; + + .twoByGridBlock { + box-sizing: border-box; + flex: 1 0 50%; + padding: 10px; + } + + .fourByGridBlock { + box-sizing: border-box; + flex: 1 0 25%; + padding: 10px; + } + } + + h2 + .gridBlock { + padding-top: 20px; + } +} + +@media only screen and (min-width: 1400px) { + .gridBlock { + display: flex; + flex-direction: row; + flex-wrap: wrap; + margin: -10px -20px 10px -20px; + + .twoByGridBlock { + box-sizing: border-box; + flex: 1 0 50%; + padding: 10px 20px; + } + + .fourByGridBlock { + box-sizing: border-box; + flex: 1 0 25%; + padding: 10px 20px; + } + } +} \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_sass/_header.scss b/librocksdb-sys/rocksdb/docs/_sass/_header.scss new file mode 100644 index 0000000..ac79390 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_sass/_header.scss @@ -0,0 +1,139 @@ +.fixedHeaderContainer { + background: $primary-bg; + color: $primary-overlay; + height: $header-height; + padding: $header-ptop 0 $header-pbot; + position: sticky; + top: 0; + width: 100%; + z-index: 9999; + + a { + align-items: center; + border: 0; + color: $primary-overlay; + display: flex; + flex-flow: row nowrap; + height: $header-height; + } + + header { + display: flex; + flex-flow: row nowrap; + position: relative; + text-align: left; + + img { + height: 24px; + margin-right: 10px; + } + + h2 { + display: block; + font-family: $header-font-family; + font-weight: 900; + line-height: 18px; + position: relative; + } + } +} + +.navigationFull { + height: 34px; + margin-left: auto; + + nav { + position: relative; + + ul { + display: flex; + flex-flow: row nowrap; + margin: 0 -10px; + + li { + padding: 0 10px; + display: block; + + a { + border: 0; + color: $primary-overlay-special; + font-size: 16px; + font-weight: 400; + line-height: 1.2em; + + &:hover { + border-bottom: 2px solid $primary-overlay; + color: $primary-overlay; + } + } + + &.navItemActive { + a { + color: $primary-overlay; + } + } + } + } + } +} + +/* 900px + + + .fixedHeaderContainer { + .navigationWrapper { + nav { + padding: 0 1em; + position: relative; + top: -9px; + + ul { + margin: 0 -0.4em; + li { + display: inline-block; + + a { + padding: 14px 0.4em; + border: 0; + color: $primary-overlay-special; + display: inline-block; + + &:hover { + color: $primary-overlay; + } + } + + &.navItemActive { + a { + color: $primary-overlay; + } + } + } + } + } + + &.navigationFull { + display: inline-block; + } + + &.navigationSlider { + display: none; + } + } + } + + 1200px + + .fixedHeaderContainer { + header { + max-width: 1100px; + } + } + + 1500px + .fixedHeaderContainer { + header { + max-width: 1400px; + } + } + */ diff --git a/librocksdb-sys/rocksdb/docs/_sass/_poweredby.scss b/librocksdb-sys/rocksdb/docs/_sass/_poweredby.scss new file mode 100644 index 0000000..4155b60 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_sass/_poweredby.scss @@ -0,0 +1,69 @@ +.poweredByContainer { + background: $primary-bg; + color: $primary-overlay; + margin-bottom: 20px; + + a { + color: $primary-overlay; + } + + .poweredByWrapper { + h2 { + border-color: $primary-overlay-special; + color: $primary-overlay-special; + } + } + + .poweredByMessage { + color: $primary-overlay-special; + font-size: 14px; + padding-top: 20px; + } +} + +.poweredByItems { + display: flex; + flex-flow: row wrap; + margin: 0 -10px; +} + +.poweredByItem { + box-sizing: border-box; + flex: 1 0 50%; + line-height: 1.1em; + padding: 5px 10px; + + &.itemLarge { + flex-basis: 100%; + padding: 10px; + text-align: center; + + &:nth-child(4) { + padding-bottom: 20px; + } + + img { + max-height: 30px; + } + } +} + +@media only screen and (min-width: 480px) { + .itemLarge { + flex-basis: 50%; + max-width: 50%; + } +} + +@media only screen and (min-width: 1024px) { + .poweredByItem { + flex-basis: 25%; + max-width: 25%; + + &.itemLarge { + padding-bottom: 20px; + text-align: left; + } + } +} + diff --git a/librocksdb-sys/rocksdb/docs/_sass/_promo.scss b/librocksdb-sys/rocksdb/docs/_sass/_promo.scss new file mode 100644 index 0000000..8c9a809 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_sass/_promo.scss @@ -0,0 +1,55 @@ +.promoSection { + display: flex; + flex-flow: column wrap; + font-size: 125%; + line-height: 1.6em; + margin: -10px 0; + position: relative; + z-index: 99; + + .promoRow { + padding: 10px 0; + + .pluginWrapper { + display: block; + + &.ghWatchWrapper, &.ghStarWrapper { + height: 28px; + } + } + + .pluginRowBlock { + display: flex; + flex-flow: row wrap; + margin: 0 -2px; + + .pluginWrapper { + padding: 0 2px; + } + } + } +} + +iframe.pluginIframe { + height: 500px; + margin-top: 20px; + width: 100%; +} + +.iframeContent { + display: none; +} + +.iframePreview { + display: inline-block; + margin-top: 20px; +} + +@media only screen and (min-width: 1024px) { + .iframeContent { + display: block; + } + .iframePreview { + display: none; + } +} \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_sass/_react_docs_nav.scss b/librocksdb-sys/rocksdb/docs/_sass/_react_docs_nav.scss new file mode 100644 index 0000000..f0a651e --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_sass/_react_docs_nav.scss @@ -0,0 +1,332 @@ +.docsNavContainer { + background: $sidenav; + height: 35px; + left: 0; + position: fixed; + width: 100%; + z-index: 100; +} + +.docMainWrapper { + .wrapper { + &.mainWrapper { + padding-left: 0; + padding-right: 0; + padding-top: 10px; + } + } +} + +.docsSliderActive { + .docsNavContainer { + box-sizing: border-box; + height: 100%; + overflow-y: auto; + -webkit-overflow-scrolling: touch; + padding-bottom: 50px; + } + + .mainContainer { + display: none; + } +} + +.navBreadcrumb { + box-sizing: border-box; + display: flex; + flex-flow: row nowrap; + font-size: 12px; + height: 35px; + overflow: hidden; + padding: 5px 10px; + + a, span { + border: 0; + color: $sidenav-text; + } + + i { + padding: 0 3px; + } +} + +nav.toc { + position: relative; + + section { + padding: 0px; + position: relative; + + .navGroups { + display: none; + padding: 40px 10px 10px; + } + } + + .toggleNav { + background: $sidenav; + color: $sidenav-text; + position: relative; + transition: background-color 0.3s, color 0.3s; + + .navToggle { + cursor: pointer; + height: 24px; + margin-right: 10px; + position: relative; + text-align: left; + width: 18px; + + &::before, &::after { + content: ""; + position: absolute; + top: 50%; + left: 0; + left: 8px; + width: 3px; + height: 6px; + border: 5px solid $sidenav-text; + border-width: 5px 0; + margin-top: -8px; + transform: rotate(45deg); + z-index: 1; + } + + &::after { + transform: rotate(-45deg); + } + + i { + &::before, &::after { + content: ""; + position: absolute; + top: 50%; + left: 2px; + background: transparent; + border-width: 0 5px 5px; + border-style: solid; + border-color: transparent $sidenav-text; + height: 0; + margin-top: -7px; + opacity: 1; + width: 5px; + z-index: 10; + } + + &::after { + border-width: 5px 5px 0; + margin-top: 2px; + } + } + } + + .navGroup { + background: $sidenav-overlay; + margin: 1px 0; + + ul { + display: none; + } + + h3 { + background: $sidenav-overlay; + color: $sidenav-text; + cursor: pointer; + font-size: 14px; + font-weight: 400; + line-height: 1.2em; + padding: 10px; + transition: color 0.2s; + + i:not(:empty) { + width: 16px; + height: 16px; + display: inline-block; + box-sizing: border-box; + text-align: center; + color: rgba($sidenav-text, 0.5); + margin-right: 10px; + transition: color 0.2s; + } + + &:hover { + color: $primary-bg; + + i:not(:empty) { + color: $primary-bg; + } + } + } + + &.navGroupActive { + background: $sidenav-active; + color: $sidenav-text; + + ul { + display: block; + padding-bottom: 10px; + padding-top: 10px; + } + + h3 { + background: $primary-bg; + color: $primary-overlay; + + i { + display: none; + } + } + } + } + + ul { + padding-left: 0; + padding-right: 24px; + + li { + list-style-type: none; + padding-bottom: 0; + padding-left: 0; + + a { + border: none; + color: $sidenav-text; + display: inline-block; + font-size: 14px; + line-height: 1.1em; + margin: 2px 10px 5px; + padding: 5px 0 2px; + transition: color 0.3s; + + &:hover, + &:focus { + color: $primary-bg; + } + + &.navItemActive { + color: $primary-bg; + font-weight: 900; + } + } + } + } + } + + .toggleNavActive { + .navBreadcrumb { + background: $sidenav; + margin-bottom: 20px; + position: fixed; + width: 100%; + } + + section { + .navGroups { + display: block; + } + } + + + .navToggle { + &::before, &::after { + border-width: 6px 0; + height: 0px; + margin-top: -6px; + } + + i { + opacity: 0; + } + } + } +} + +.docsNavVisible { + .navPusher { + .mainContainer { + padding-top: 35px; + } + } +} + +@media only screen and (min-width: 900px) { + .navBreadcrumb { + padding: 5px 0; + } + + nav.toc { + section { + .navGroups { + padding: 40px 0 0; + } + } + } +} + +@media only screen and (min-width: 1024px) { + .navToggle { + display: none; + } + + .docsSliderActive { + .mainContainer { + display: block; + } + } + + .docsNavVisible { + .navPusher { + .mainContainer { + padding-top: 0; + } + } + } + + .docsNavContainer { + background: none; + box-sizing: border-box; + height: auto; + margin: 40px 40px 0 0; + overflow-y: auto; + position: relative; + width: 300px; + } + + nav.toc { + section { + .navGroups { + display: block; + padding-top: 0px; + } + } + + .toggleNavActive { + .navBreadcrumb { + margin-bottom: 0; + position: relative; + } + } + } + + .docMainWrapper { + display: flex; + flex-flow: row nowrap; + margin-bottom: 40px; + + .wrapper { + padding-left: 0; + padding-right: 0; + + &.mainWrapper { + padding-top: 0; + } + } + } + + .navBreadcrumb { + display: none; + h2 { + padding: 0 10px; + } + } +} \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_sass/_react_header_nav.scss b/librocksdb-sys/rocksdb/docs/_sass/_react_header_nav.scss new file mode 100644 index 0000000..13c0e56 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_sass/_react_header_nav.scss @@ -0,0 +1,141 @@ +.navigationFull { + display: none; +} + +.navigationSlider { + position: absolute; + right: 0px; + + .navSlideout { + cursor: pointer; + padding-top: 4px; + position: absolute; + right: 10px; + top: 0; + transition: top 0.3s; + z-index: 101; + } + + .slidingNav { + background: $secondary-bg; + box-sizing: border-box; + height: 0px; + overflow-x: hidden; + padding: 0; + position: absolute; + right: 0px; + top: 0; + transition: height 0.3s cubic-bezier(0.68, -0.55, 0.265, 1.55), width 0.3s cubic-bezier(0.68, -0.55, 0.265, 1.55); + width: 0; + + ul { + flex-flow: column nowrap; + list-style: none; + padding: 10px; + + li { + margin: 0; + padding: 2px 0; + + a { + color: $primary-bg; + display: inline; + margin: 3px 5px; + padding: 2px 0px; + transition: background-color 0.3s; + + &:focus, + &:hover { + border-bottom: 2px solid $primary-bg; + } + } + } + } + } + + .navSlideoutActive { + .slidingNav { + height: auto; + padding-top: $header-height + $header-pbot; + width: 300px; + } + + .navSlideout { + top: -2px; + .menuExpand { + span:nth-child(1) { + background-color: $text; + top: 16px; + transform: rotate(45deg); + } + span:nth-child(2) { + opacity: 0; + } + span:nth-child(3) { + background-color: $text; + transform: rotate(-45deg); + } + } + } + } +} + +.menuExpand { + display: flex; + flex-flow: column nowrap; + height: 20px; + justify-content: space-between; + + span { + background: $primary-overlay; + border-radius: 3px; + display: block; + flex: 0 0 4px; + height: 4px; + position: relative; + top: 0; + transition: background-color 0.3s, top 0.3s, opacity 0.3s, transform 0.3s; + width: 20px; + } +} + +.navPusher { + border-top: $header-height + $header-ptop + $header-pbot solid $primary-bg; + position: relative; + left: 0; + z-index: 99; + height: 100%; + + &::after { + position: absolute; + top: 0; + right: 0; + width: 0; + height: 0; + background: rgba(0,0,0,0.4); + content: ''; + opacity: 0; + -webkit-transition: opacity 0.5s, width 0.1s 0.5s, height 0.1s 0.5s; + transition: opacity 0.5s, width 0.1s 0.5s, height 0.1s 0.5s; + } + + .sliderActive &::after { + width: 100%; + height: 100%; + opacity: 1; + -webkit-transition: opacity 0.5s; + transition: opacity 0.5s; + z-index: 100; + } +} + + +@media only screen and (min-width: 1024px) { + .navigationFull { + display: block; + } + + .navigationSlider { + display: none; + } +} \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_sass/_reset.scss b/librocksdb-sys/rocksdb/docs/_sass/_reset.scss new file mode 100644 index 0000000..0e5f2e0 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_sass/_reset.scss @@ -0,0 +1,43 @@ +html, body, div, span, applet, object, iframe, +h1, h2, h3, h4, h5, h6, p, blockquote, pre, +a, abbr, acronym, address, big, cite, code, +del, dfn, em, img, ins, kbd, q, s, samp, +small, strike, strong, sub, sup, tt, var, +b, u, i, center, +dl, dt, dd, ol, ul, li, +fieldset, form, label, legend, +table, caption, tbody, tfoot, thead, tr, th, td, +article, aside, canvas, details, embed, +figure, figcaption, footer, header, hgroup, +menu, nav, output, ruby, section, summary, +time, mark, audio, video { + margin: 0; + padding: 0; + border: 0; + font-size: 100%; + font: inherit; + vertical-align: baseline; +} +/* HTML5 display-role reset for older browsers */ +article, aside, details, figcaption, figure, +footer, header, hgroup, menu, nav, section { + display: block; +} +body { + line-height: 1; +} +ol, ul { + list-style: none; +} +blockquote, q { + quotes: none; +} +blockquote:before, blockquote:after, +q:before, q:after { + content: ''; + content: none; +} +table { + border-collapse: collapse; + border-spacing: 0; +} diff --git a/librocksdb-sys/rocksdb/docs/_sass/_search.scss b/librocksdb-sys/rocksdb/docs/_sass/_search.scss new file mode 100644 index 0000000..eadfa11 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_sass/_search.scss @@ -0,0 +1,142 @@ +input[type="search"] { + -moz-appearance: none; + -webkit-appearance: none; +} + +.navSearchWrapper { + align-self: center; + position: relative; + + &::before { + border: 3px solid $primary-overlay-special; + border-radius: 50%; + content: " "; + display: block; + height: 6px; + left: 15px; + width: 6px; + position: absolute; + top: 4px; + z-index: 1; + } + + &::after { + background: $primary-overlay-special; + content: " "; + height: 7px; + left: 24px; + position: absolute; + transform: rotate(-45deg); + top: 12px; + width: 3px; + z-index: 1; + } + + .aa-dropdown-menu { + background: $secondary-bg; + border: 3px solid rgba($text, 0.25); + color: $text; + font-size: 14px; + left: auto !important; + line-height: 1.2em; + right: 0 !important; + + .algolia-docsearch-suggestion--category-header { + background: $primary-overlay-special; + color: $primary-bg; + + .algolia-docsearch-suggestion--highlight { + background-color: $primary-bg; + color: $primary-overlay; + } + } + + .algolia-docsearch-suggestion--title .algolia-docsearch-suggestion--highlight, + .algolia-docsearch-suggestion--subcategory-column .algolia-docsearch-suggestion--highlight { + color: $primary-bg; + } + + .algolia-docsearch-suggestion__secondary, + .algolia-docsearch-suggestion--subcategory-column { + border-color: rgba($text, 0.3); + } + } +} + +input#search_input { + padding-left: 25px; + font-size: 14px; + line-height: 20px; + border-radius: 20px; + background-color: rgba($primary-overlay-special, 0.25); + border: none; + color: rgba($primary-overlay-special, 0); + outline: none; + position: relative; + transition: background-color .2s cubic-bezier(0.68, -0.55, 0.265, 1.55), width .2s cubic-bezier(0.68, -0.55, 0.265, 1.55), color .2s ease; + width: 60px; + + &:focus, &:active { + background-color: $secondary-bg; + color: $text; + width: 240px; + } +} + +.navigationSlider { + .navSearchWrapper { + &::before { + left: 6px; + top: 6px; + } + + &::after { + left: 15px; + top: 14px; + } + } + + input#search_input_react { + box-sizing: border-box; + padding-left: 25px; + font-size: 14px; + line-height: 20px; + border-radius: 20px; + background-color: rgba($primary-overlay-special, 0.25); + border: none; + color: $text; + outline: none; + position: relative; + transition: background-color .2s cubic-bezier(0.68, -0.55, 0.265, 1.55), width .2s cubic-bezier(0.68, -0.55, 0.265, 1.55), color .2s ease; + width: 100%; + + &:focus, &:active { + background-color: $primary-bg; + color: $primary-overlay; + } + } + + .algolia-docsearch-suggestion--subcategory-inline { + display: none; + } + + & > span { + width: 100%; + } + + .aa-dropdown-menu { + background: $secondary-bg; + border: 0px solid $secondary-bg; + color: $text; + font-size: 12px; + line-height: 2em; + max-height: 140px; + min-width: auto; + overflow-y: scroll; + -webkit-overflow-scrolling: touch; + padding: 0; + border-radius: 0; + position: relative !important; + width: 100%; + } +} \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_sass/_slideshow.scss b/librocksdb-sys/rocksdb/docs/_sass/_slideshow.scss new file mode 100644 index 0000000..cd98a6c --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_sass/_slideshow.scss @@ -0,0 +1,48 @@ +.slideshow { + position: relative; + + .slide { + display: none; + + img { + display: block; + margin: 0 auto; + } + + &.slideActive { + display: block; + } + + a { + border: none; + display: block; + } + } + + .pagination { + display: block; + margin: -10px; + padding: 1em 0; + text-align: center; + width: 100%; + + .pager { + background: transparent; + border: 2px solid rgba(255, 255, 255, 0.5); + border-radius: 50%; + cursor: pointer; + display: inline-block; + height: 12px; + margin: 10px; + transition: background-color 0.3s, border-color 0.3s; + width: 12px; + + &.pagerActive { + background: rgba(255, 255, 255, 0.5); + border-width: 4px; + height: 8px; + width: 8px; + } + } + } +} diff --git a/librocksdb-sys/rocksdb/docs/_sass/_syntax-highlighting.scss b/librocksdb-sys/rocksdb/docs/_sass/_syntax-highlighting.scss new file mode 100644 index 0000000..e55c88a --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_sass/_syntax-highlighting.scss @@ -0,0 +1,129 @@ + + +.rougeHighlight { background-color: $code-bg; color: #93a1a1 } +.rougeHighlight .c { color: #586e75 } /* Comment */ +.rougeHighlight .err { color: #93a1a1 } /* Error */ +.rougeHighlight .g { color: #93a1a1 } /* Generic */ +.rougeHighlight .k { color: #859900 } /* Keyword */ +.rougeHighlight .l { color: #93a1a1 } /* Literal */ +.rougeHighlight .n { color: #93a1a1 } /* Name */ +.rougeHighlight .o { color: #859900 } /* Operator */ +.rougeHighlight .x { color: #cb4b16 } /* Other */ +.rougeHighlight .p { color: #93a1a1 } /* Punctuation */ +.rougeHighlight .cm { color: #586e75 } /* Comment.Multiline */ +.rougeHighlight .cp { color: #859900 } /* Comment.Preproc */ +.rougeHighlight .c1 { color: #72c02c; } /* Comment.Single */ +.rougeHighlight .cs { color: #859900 } /* Comment.Special */ +.rougeHighlight .gd { color: #2aa198 } /* Generic.Deleted */ +.rougeHighlight .ge { color: #93a1a1; font-style: italic } /* Generic.Emph */ +.rougeHighlight .gr { color: #dc322f } /* Generic.Error */ +.rougeHighlight .gh { color: #cb4b16 } /* Generic.Heading */ +.rougeHighlight .gi { color: #859900 } /* Generic.Inserted */ +.rougeHighlight .go { color: #93a1a1 } /* Generic.Output */ +.rougeHighlight .gp { color: #93a1a1 } /* Generic.Prompt */ +.rougeHighlight .gs { color: #93a1a1; font-weight: bold } /* Generic.Strong */ +.rougeHighlight .gu { color: #cb4b16 } /* Generic.Subheading */ +.rougeHighlight .gt { color: #93a1a1 } /* Generic.Traceback */ +.rougeHighlight .kc { color: #cb4b16 } /* Keyword.Constant */ +.rougeHighlight .kd { color: #268bd2 } /* Keyword.Declaration */ +.rougeHighlight .kn { color: #859900 } /* Keyword.Namespace */ +.rougeHighlight .kp { color: #859900 } /* Keyword.Pseudo */ +.rougeHighlight .kr { color: #268bd2 } /* Keyword.Reserved */ +.rougeHighlight .kt { color: #dc322f } /* Keyword.Type */ +.rougeHighlight .ld { color: #93a1a1 } /* Literal.Date */ +.rougeHighlight .m { color: #2aa198 } /* Literal.Number */ +.rougeHighlight .s { color: #2aa198 } /* Literal.String */ +.rougeHighlight .na { color: #93a1a1 } /* Name.Attribute */ +.rougeHighlight .nb { color: #B58900 } /* Name.Builtin */ +.rougeHighlight .nc { color: #268bd2 } /* Name.Class */ +.rougeHighlight .no { color: #cb4b16 } /* Name.Constant */ +.rougeHighlight .nd { color: #268bd2 } /* Name.Decorator */ +.rougeHighlight .ni { color: #cb4b16 } /* Name.Entity */ +.rougeHighlight .ne { color: #cb4b16 } /* Name.Exception */ +.rougeHighlight .nf { color: #268bd2 } /* Name.Function */ +.rougeHighlight .nl { color: #93a1a1 } /* Name.Label */ +.rougeHighlight .nn { color: #93a1a1 } /* Name.Namespace */ +.rougeHighlight .nx { color: #93a1a1 } /* Name.Other */ +.rougeHighlight .py { color: #93a1a1 } /* Name.Property */ +.rougeHighlight .nt { color: #268bd2 } /* Name.Tag */ +.rougeHighlight .nv { color: #268bd2 } /* Name.Variable */ +.rougeHighlight .ow { color: #859900 } /* Operator.Word */ +.rougeHighlight .w { color: #93a1a1 } /* Text.Whitespace */ +.rougeHighlight .mf { color: #2aa198 } /* Literal.Number.Float */ +.rougeHighlight .mh { color: #2aa198 } /* Literal.Number.Hex */ +.rougeHighlight .mi { color: #2aa198 } /* Literal.Number.Integer */ +.rougeHighlight .mo { color: #2aa198 } /* Literal.Number.Oct */ +.rougeHighlight .sb { color: #586e75 } /* Literal.String.Backtick */ +.rougeHighlight .sc { color: #2aa198 } /* Literal.String.Char */ +.rougeHighlight .sd { color: #93a1a1 } /* Literal.String.Doc */ +.rougeHighlight .s2 { color: #2aa198 } /* Literal.String.Double */ +.rougeHighlight .se { color: #cb4b16 } /* Literal.String.Escape */ +.rougeHighlight .sh { color: #93a1a1 } /* Literal.String.Heredoc */ +.rougeHighlight .si { color: #2aa198 } /* Literal.String.Interpol */ +.rougeHighlight .sx { color: #2aa198 } /* Literal.String.Other */ +.rougeHighlight .sr { color: #dc322f } /* Literal.String.Regex */ +.rougeHighlight .s1 { color: #2aa198 } /* Literal.String.Single */ +.rougeHighlight .ss { color: #2aa198 } /* Literal.String.Symbol */ +.rougeHighlight .bp { color: #268bd2 } /* Name.Builtin.Pseudo */ +.rougeHighlight .vc { color: #268bd2 } /* Name.Variable.Class */ +.rougeHighlight .vg { color: #268bd2 } /* Name.Variable.Global */ +.rougeHighlight .vi { color: #268bd2 } /* Name.Variable.Instance */ +.rougeHighlight .il { color: #2aa198 } /* Literal.Number.Integer.Long */ + +.highlighter-rouge { + color: darken(#72c02c, 8%); + font: 800 12px/1.5em Hack, monospace; + max-width: 100%; + + .rougeHighlight { + border-radius: 3px; + margin: 20px 0; + padding: 0px; + overflow-x: scroll; + -webkit-overflow-scrolling: touch; + + table { + background: none; + border: none; + + tbody { + tr { + background: none; + display: flex; + flex-flow: row nowrap; + + td { + display: block; + flex: 1 1; + + &.gutter { + border-right: 1px solid lighten($code-bg, 10%); + color: lighten($code-bg, 15%); + margin-right: 10px; + max-width: 40px; + padding-right: 10px; + + pre { + max-width: 20px; + } + } + } + } + } + } + } +} + +p > .highlighter-rouge, +li > .highlighter-rouge, +a > .highlighter-rouge { + font-size: 16px; + font-weight: 400; + line-height: inherit; +} + +a:hover { + .highlighter-rouge { + color: white; + } +} \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_sass/_tables.scss b/librocksdb-sys/rocksdb/docs/_sass/_tables.scss new file mode 100644 index 0000000..f847c70 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_sass/_tables.scss @@ -0,0 +1,47 @@ +table { + background: $lightergrey; + border: 1px solid $lightgrey; + border-collapse: collapse; + display:table; + margin: 20px 0; + + thead { + border-bottom: 1px solid $lightgrey; + display: table-header-group; + } + tbody { + display: table-row-group; + } + tr { + display: table-row; + &:nth-of-type(odd) { + background: $greyish; + } + + th, td { + border-right: 1px dotted $lightgrey; + display: table-cell; + font-size: 14px; + line-height: 1.3em; + padding: 10px; + text-align: left; + + &:last-of-type { + border-right: 0; + } + + code { + color: $green; + display: inline-block; + font-size: 12px; + } + } + + th { + color: #000000; + font-weight: bold; + font-family: $header-font-family; + text-transform: uppercase; + } + } +} \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/docs/_top-level/support.md b/librocksdb-sys/rocksdb/docs/_top-level/support.md new file mode 100644 index 0000000..05c39be --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/_top-level/support.md @@ -0,0 +1,22 @@ +--- +layout: top-level +title: Support +id: support +category: support +--- + +## Need help? + +Do not hesitate to ask questions if you are having trouble with RocksDB. + +### GitHub issues + +Use [GitHub issues](https://github.com/facebook/rocksdb/issues) to report bugs, issues and feature requests for the RocksDB codebase. + +### Facebook Group + +Use the [RocksDB Facebook group](https://www.facebook.com/groups/rocksdb.dev/) for general questions and discussion about RocksDB. + +### FAQ + +Check out a list of [commonly asked questions](https://github.com/facebook/rocksdb/wiki/RocksDB-FAQ) about RocksDB. diff --git a/librocksdb-sys/rocksdb/docs/blog/all.html b/librocksdb-sys/rocksdb/docs/blog/all.html new file mode 100644 index 0000000..3be2d3b --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/blog/all.html @@ -0,0 +1,20 @@ +--- +id: all +layout: blog +category: blog +--- + +
    +
    +

    All Posts

    + {% for post in site.posts %} + {% assign author = site.data.authors[post.author] %} +

    + + {{ post.title }} + + on {{ post.date | date: "%B %e, %Y" }} by {{ author.display_name }} +

    + {% endfor %} +
    +
    diff --git a/librocksdb-sys/rocksdb/docs/blog/index.html b/librocksdb-sys/rocksdb/docs/blog/index.html new file mode 100644 index 0000000..9f6b25d --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/blog/index.html @@ -0,0 +1,12 @@ +--- +id: blog +title: Blog +layout: blog +category: blog +--- + +
    + {% for page in site.posts %} + {% include post.html truncate=true %} + {% endfor %} +
    diff --git a/librocksdb-sys/rocksdb/docs/css/main.scss b/librocksdb-sys/rocksdb/docs/css/main.scss new file mode 100644 index 0000000..88ab4e8 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/css/main.scss @@ -0,0 +1,159 @@ +--- +# Only the main Sass file needs front matter (the dashes are enough) +--- +@charset "utf-8"; + +@font-face { + font-family: 'Lato'; + src: url("{{ '/static/fonts/LatoLatin-Italic.woff2' }}") format('woff2'), + url("{{ '/static/fonts/LatoLatin-Italic.woff' }}") format('woff'); + font-weight: normal; + font-style: italic; +} + +@font-face { + font-family: 'Lato'; + src: url("{{ '/static/fonts/LatoLatin-Black.woff2' }}") format('woff2'), + url("{{ '/static/fonts/LatoLatin-Black.woff' }}") format('woff'); + font-weight: 900; + font-style: normal; +} + +@font-face { + font-family: 'Lato'; + src: url("{{ '/static/fonts/LatoLatin-BlackItalic.woff2' }}") format('woff2'), + url("{{ '/static/fonts/LatoLatin-BlackItalic.woff' }}") format('woff'); + font-weight: 900; + font-style: italic; +} + +@font-face { + font-family: 'Lato'; + src: url("{{ '/static/fonts/LatoLatin-Light.woff2' }}") format('woff2'), + url("{{ '/static/fonts/LatoLatin-Light.woff' }}") format('woff'); + font-weight: 300; + font-style: normal; +} + +@font-face { + font-family: 'Lato'; + src: url("{{ '/static/fonts/LatoLatin-Regular.woff2' }}") format('woff2'), + url("{{ '/static/fonts/LatoLatin-Regular.woff' }}") format('woff'); + font-weight: normal; + font-style: normal; +} + +// Our variables +$base-font-family: 'Lato', Calibri, Arial, sans-serif; +$header-font-family: 'Lato', 'Helvetica Neue', Arial, sans-serif; +$base-font-size: 18px; +$small-font-size: $base-font-size * 0.875; +$base-line-height: 1.4em; + +$spacing-unit: 12px; + +// Two configured colors (see _config.yml) +$primary-bg: {{ site.color.primary }}; +$secondary-bg: {{ site.color.secondary }}; + +// $primary-bg overlays +{% if site.color.primary-overlay == 'light' %} +$primary-overlay: darken($primary-bg, 70%); +$primary-overlay-special: darken($primary-bg, 40%); +{% else %} +$primary-overlay: #fff; +$primary-overlay-special: lighten($primary-bg, 30%); +{% endif %} + +// $secondary-bg overlays +{% if site.color.secondary-overlay == 'light' %} +$text: #393939; +$sidenav: darken($secondary-bg, 20%); +$sidenav-text: $text; +$sidenav-overlay: darken($sidenav, 10%); +$sidenav-active: lighten($sidenav, 10%); +{% else %} +$text: #fff; +$sidenav: lighten($secondary-bg, 20%); +$sidenav-text: $text; +$sidenav-overlay: lighten($sidenav, 10%); +$sidenav-active: darken($sidenav, 10%); +{% endif %} + +$code-bg: #002b36; + +$header-height: 34px; +$header-ptop: 10px; +$header-pbot: 8px; + +// Width of the content area +$content-width: 900px; + +// Table setting variables +$lightergrey: #F8F8F8; +$greyish: #E8E8E8; +$lightgrey: #B0B0B0; +$green: #2db04b; + +// Using media queries with like this: +// @include media-query($on-palm) { +// .wrapper { +// padding-right: $spacing-unit / 2; +// padding-left: $spacing-unit / 2; +// } +// } +@mixin media-query($device) { + @media screen and (max-width: $device) { + @content; + } +} + + + +// Import partials from `sass_dir` (defaults to `_sass`) +@import + "reset", + "base", + "header", + "search", + "syntax-highlighting", + "promo", + "buttons", + "gridBlock", + "poweredby", + "footer", + "react_header_nav", + "react_docs_nav", + "tables", + "blog" +; + +// Anchor links +// http://ben.balter.com/2014/03/13/pages-anchor-links/ +.header-link { + position: absolute; + margin-left: 0.2em; + opacity: 0; + + -webkit-transition: opacity 0.2s ease-in-out 0.1s; + -moz-transition: opacity 0.2s ease-in-out 0.1s; + -ms-transition: opacity 0.2s ease-in-out 0.1s; +} + +h2:hover .header-link, +h3:hover .header-link, +h4:hover .header-link, +h5:hover .header-link, +h6:hover .header-link { + opacity: 1; +} + +/* Social Banner */ +.socialBanner { + font-weight: bold; + font-size: 20px; + padding: 20px; + max-width: 768px; + margin: 0 auto; + text-align: center; + } diff --git a/librocksdb-sys/rocksdb/docs/doc-type-examples/2016-04-07-blog-post-example.md b/librocksdb-sys/rocksdb/docs/doc-type-examples/2016-04-07-blog-post-example.md new file mode 100644 index 0000000..ef954d6 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/doc-type-examples/2016-04-07-blog-post-example.md @@ -0,0 +1,21 @@ +--- +title: Blog Post Example +layout: post +author: exampleauthor +category: blog +--- + +Any local blog posts would go in the `_posts` directory. + +This is an example blog post introduction, try to keep it short and about a paragraph long, to encourage people to click through to read the entire post. + + + +Everything below the `` tag will only show on the actual blog post page, not on the `/blog/` index. + +Author is defined in `_data/authors.yml` + + +## No posts? + +If you have no blog for your site, you can remove the entire `_posts` folder. Otherwise add markdown files in here. See CONTRIBUTING.md for details. diff --git a/librocksdb-sys/rocksdb/docs/doc-type-examples/docs-hello-world.md b/librocksdb-sys/rocksdb/docs/doc-type-examples/docs-hello-world.md new file mode 100644 index 0000000..c7094ba --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/doc-type-examples/docs-hello-world.md @@ -0,0 +1,12 @@ +--- +docid: hello-world +title: Hello, World! +layout: docs +permalink: /docs/hello-world.html +--- + +Any local docs would go in the `_docs` directory. + +## No documentation? + +If you have no documentation for your site, you can remove the entire `_docs` folder. Otherwise add markdown files in here. See CONTRIBUTING.md for details. diff --git a/librocksdb-sys/rocksdb/docs/doc-type-examples/top-level-example.md b/librocksdb-sys/rocksdb/docs/doc-type-examples/top-level-example.md new file mode 100644 index 0000000..67b1fa7 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/doc-type-examples/top-level-example.md @@ -0,0 +1,8 @@ +--- +layout: top-level +title: Support Example +id: top-level-example +category: top-level +--- + +This is a static page disconnected from the blog or docs collections that can be added at a top-level (i.e., the same level as `index.md`). diff --git a/librocksdb-sys/rocksdb/docs/docs/index.html b/librocksdb-sys/rocksdb/docs/docs/index.html new file mode 100644 index 0000000..fa6ec8b --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/docs/index.html @@ -0,0 +1,6 @@ +--- +id: docs +title: Docs +layout: redirect +destination: getting-started.html +--- diff --git a/librocksdb-sys/rocksdb/docs/feed.xml b/librocksdb-sys/rocksdb/docs/feed.xml new file mode 100644 index 0000000..725f005 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/feed.xml @@ -0,0 +1,30 @@ +--- +layout: null +--- + + + + {{ site.title | xml_escape }} + {{ site.description | xml_escape }} + https://rocksdb.org/feed.xml + + {{ site.time | date_to_rfc822 }} + {{ site.time | date_to_rfc822 }} + Jekyll v{{ jekyll.version }} + {% for post in site.posts limit:10 %} + + {{ post.title | xml_escape }} + {{ post.content | xml_escape }} + {{ post.date | date_to_rfc822 }} + {{ post.url | absolute_url }} + {{ post.url | absolute_url }} + {% for tag in post.tags %} + {{ tag | xml_escape }} + {% endfor %} + {% for cat in post.categories %} + {{ cat | xml_escape }} + {% endfor %} + + {% endfor %} + + diff --git a/librocksdb-sys/rocksdb/docs/index.md b/librocksdb-sys/rocksdb/docs/index.md new file mode 100644 index 0000000..2b9570d --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/index.md @@ -0,0 +1,9 @@ +--- +layout: home +title: RocksDB | A persistent key-value store +id: home +--- + +## Features + +{% include content/gridblocks.html data_source=site.data.features align="center" %} diff --git a/librocksdb-sys/rocksdb/docs/static/favicon.png b/librocksdb-sys/rocksdb/docs/static/favicon.png new file mode 100644 index 0000000..7f668f3 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/favicon.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Black.woff b/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Black.woff new file mode 100644 index 0000000..d1e2579 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Black.woff differ diff --git a/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Black.woff2 b/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Black.woff2 new file mode 100644 index 0000000..4127b4d Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Black.woff2 differ diff --git a/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-BlackItalic.woff b/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-BlackItalic.woff new file mode 100644 index 0000000..142c1c9 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-BlackItalic.woff differ diff --git a/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-BlackItalic.woff2 b/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-BlackItalic.woff2 new file mode 100644 index 0000000..e9862e6 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-BlackItalic.woff2 differ diff --git a/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Italic.woff b/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Italic.woff new file mode 100644 index 0000000..d8cf84c Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Italic.woff differ diff --git a/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Italic.woff2 b/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Italic.woff2 new file mode 100644 index 0000000..aaa5a35 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Italic.woff2 differ diff --git a/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Light.woff b/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Light.woff new file mode 100644 index 0000000..e7d4278 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Light.woff differ diff --git a/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Light.woff2 b/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Light.woff2 new file mode 100644 index 0000000..b6d0288 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Light.woff2 differ diff --git a/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Regular.woff b/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Regular.woff new file mode 100644 index 0000000..bf73a6d Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Regular.woff differ diff --git a/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Regular.woff2 b/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Regular.woff2 new file mode 100644 index 0000000..a4d084b Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/fonts/LatoLatin-Regular.woff2 differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/Resize-of-20140327_200754-300x225.jpg b/librocksdb-sys/rocksdb/docs/static/images/Resize-of-20140327_200754-300x225.jpg new file mode 100644 index 0000000..9f93151 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/Resize-of-20140327_200754-300x225.jpg differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/align-compaction-output/compaction_output_file_size_compare.png b/librocksdb-sys/rocksdb/docs/static/images/align-compaction-output/compaction_output_file_size_compare.png new file mode 100644 index 0000000..2ce86fb Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/align-compaction-output/compaction_output_file_size_compare.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/align-compaction-output/file_cut_align.png b/librocksdb-sys/rocksdb/docs/static/images/align-compaction-output/file_cut_align.png new file mode 100644 index 0000000..bc3e899 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/align-compaction-output/file_cut_align.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/align-compaction-output/file_cut_normal.png b/librocksdb-sys/rocksdb/docs/static/images/align-compaction-output/file_cut_normal.png new file mode 100644 index 0000000..e17133e Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/align-compaction-output/file_cut_normal.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/align-compaction-output/file_cut_trival_move.png b/librocksdb-sys/rocksdb/docs/static/images/align-compaction-output/file_cut_trival_move.png new file mode 100644 index 0000000..7aca9ae Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/align-compaction-output/file_cut_trival_move.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/align-compaction-output/file_size_compare.png b/librocksdb-sys/rocksdb/docs/static/images/align-compaction-output/file_size_compare.png new file mode 100644 index 0000000..5f39a80 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/align-compaction-output/file_size_compare.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/align-compaction-output/write_amp_compare.png b/librocksdb-sys/rocksdb/docs/static/images/align-compaction-output/write_amp_compare.png new file mode 100644 index 0000000..8b20f2a Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/align-compaction-output/write_amp_compare.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/asynchronous-io/mget_async.png b/librocksdb-sys/rocksdb/docs/static/images/asynchronous-io/mget_async.png new file mode 100644 index 0000000..79d1a85 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/asynchronous-io/mget_async.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/asynchronous-io/scan_async.png b/librocksdb-sys/rocksdb/docs/static/images/asynchronous-io/scan_async.png new file mode 100644 index 0000000..ee84189 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/asynchronous-io/scan_async.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/binaryseek.png b/librocksdb-sys/rocksdb/docs/static/images/binaryseek.png new file mode 100644 index 0000000..0e213f0 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/binaryseek.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/bloom_fp_vs_bpk.png b/librocksdb-sys/rocksdb/docs/static/images/bloom_fp_vs_bpk.png new file mode 100644 index 0000000..e83f4d0 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/bloom_fp_vs_bpk.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/compaction/full-range.png b/librocksdb-sys/rocksdb/docs/static/images/compaction/full-range.png new file mode 100644 index 0000000..5b2c9fc Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/compaction/full-range.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/compaction/l0-l1-contend.png b/librocksdb-sys/rocksdb/docs/static/images/compaction/l0-l1-contend.png new file mode 100644 index 0000000..bcf8ec7 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/compaction/l0-l1-contend.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/compaction/l1-l2-contend.png b/librocksdb-sys/rocksdb/docs/static/images/compaction/l1-l2-contend.png new file mode 100644 index 0000000..6dafbbb Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/compaction/l1-l2-contend.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/compaction/part-range-old.png b/librocksdb-sys/rocksdb/docs/static/images/compaction/part-range-old.png new file mode 100644 index 0000000..1cc723d Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/compaction/part-range-old.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/data-block-hash-index/block-format-binary-seek.png b/librocksdb-sys/rocksdb/docs/static/images/data-block-hash-index/block-format-binary-seek.png new file mode 100644 index 0000000..0e213f0 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/data-block-hash-index/block-format-binary-seek.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/data-block-hash-index/block-format-hash-index.png b/librocksdb-sys/rocksdb/docs/static/images/data-block-hash-index/block-format-hash-index.png new file mode 100644 index 0000000..accb863 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/data-block-hash-index/block-format-hash-index.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/data-block-hash-index/hash-index-data-structure.png b/librocksdb-sys/rocksdb/docs/static/images/data-block-hash-index/hash-index-data-structure.png new file mode 100644 index 0000000..9acc71d Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/data-block-hash-index/hash-index-data-structure.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/data-block-hash-index/perf-cache-miss.png b/librocksdb-sys/rocksdb/docs/static/images/data-block-hash-index/perf-cache-miss.png new file mode 100644 index 0000000..7178873 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/data-block-hash-index/perf-cache-miss.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/data-block-hash-index/perf-throughput.png b/librocksdb-sys/rocksdb/docs/static/images/data-block-hash-index/perf-throughput.png new file mode 100644 index 0000000..54948af Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/data-block-hash-index/perf-throughput.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/delrange/delrange_collapsed.png b/librocksdb-sys/rocksdb/docs/static/images/delrange/delrange_collapsed.png new file mode 100644 index 0000000..52246c2 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/delrange/delrange_collapsed.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/delrange/delrange_key_schema.png b/librocksdb-sys/rocksdb/docs/static/images/delrange/delrange_key_schema.png new file mode 100644 index 0000000..0a14d4a Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/delrange/delrange_key_schema.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/delrange/delrange_sst_blocks.png b/librocksdb-sys/rocksdb/docs/static/images/delrange/delrange_sst_blocks.png new file mode 100644 index 0000000..6003e42 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/delrange/delrange_sst_blocks.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/delrange/delrange_uncollapsed.png b/librocksdb-sys/rocksdb/docs/static/images/delrange/delrange_uncollapsed.png new file mode 100644 index 0000000..39c7097 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/delrange/delrange_uncollapsed.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/delrange/delrange_write_path.png b/librocksdb-sys/rocksdb/docs/static/images/delrange/delrange_write_path.png new file mode 100644 index 0000000..229dfb3 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/delrange/delrange_write_path.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/dictcmp/dictcmp_raw_sampled.png b/librocksdb-sys/rocksdb/docs/static/images/dictcmp/dictcmp_raw_sampled.png new file mode 100644 index 0000000..2eb6463 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/dictcmp/dictcmp_raw_sampled.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/dictcmp/dictcmp_sst_blocks.png b/librocksdb-sys/rocksdb/docs/static/images/dictcmp/dictcmp_sst_blocks.png new file mode 100644 index 0000000..551860b Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/dictcmp/dictcmp_sst_blocks.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/dictcmp/dictcmp_zstd_trained.png b/librocksdb-sys/rocksdb/docs/static/images/dictcmp/dictcmp_zstd_trained.png new file mode 100644 index 0000000..966c7fe Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/dictcmp/dictcmp_zstd_trained.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/integrated-blob-db/BlobDB_Benchmarks_Legacy_Vs_Integrated.png b/librocksdb-sys/rocksdb/docs/static/images/integrated-blob-db/BlobDB_Benchmarks_Legacy_Vs_Integrated.png new file mode 100644 index 0000000..7215390 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/integrated-blob-db/BlobDB_Benchmarks_Legacy_Vs_Integrated.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/integrated-blob-db/BlobDB_Benchmarks_RW_RO_Perf.png b/librocksdb-sys/rocksdb/docs/static/images/integrated-blob-db/BlobDB_Benchmarks_RW_RO_Perf.png new file mode 100644 index 0000000..f412ee6 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/integrated-blob-db/BlobDB_Benchmarks_RW_RO_Perf.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/integrated-blob-db/BlobDB_Benchmarks_Write_Amp.png b/librocksdb-sys/rocksdb/docs/static/images/integrated-blob-db/BlobDB_Benchmarks_Write_Amp.png new file mode 100644 index 0000000..19f40b0 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/integrated-blob-db/BlobDB_Benchmarks_Write_Amp.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/integrated-blob-db/BlobDB_Benchmarks_Write_Perf.png b/librocksdb-sys/rocksdb/docs/static/images/integrated-blob-db/BlobDB_Benchmarks_Write_Perf.png new file mode 100644 index 0000000..a1d43da Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/integrated-blob-db/BlobDB_Benchmarks_Write_Perf.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/Memtable-entry.png b/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/Memtable-entry.png new file mode 100644 index 0000000..31eb727 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/Memtable-entry.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/Memtable-write.png b/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/Memtable-write.png new file mode 100644 index 0000000..32f526f Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/Memtable-write.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/ProtInfo-Memtable.png b/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/ProtInfo-Memtable.png new file mode 100644 index 0000000..c2e21bb Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/ProtInfo-Memtable.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/ProtInfo-Writebatch-to-Memtable.png b/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/ProtInfo-Writebatch-to-Memtable.png new file mode 100644 index 0000000..91ad93b Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/ProtInfo-Writebatch-to-Memtable.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/ProtInfo-Writebatch.png b/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/ProtInfo-Writebatch.png new file mode 100644 index 0000000..b3cd531 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/ProtInfo-Writebatch.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/WAL-fragment.png b/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/WAL-fragment.png new file mode 100644 index 0000000..9bbacca Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/WAL-fragment.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/WAL-read.png b/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/WAL-read.png new file mode 100644 index 0000000..e130733 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/WAL-read.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/WAL-write.png b/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/WAL-write.png new file mode 100644 index 0000000..fb9fd8f Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/WAL-write.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/Write-batch.png b/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/Write-batch.png new file mode 100644 index 0000000..121d425 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/Write-batch.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/Writebatch-write.png b/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/Writebatch-write.png new file mode 100644 index 0000000..b10ab35 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/kv-checksum/Writebatch-write.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/lost-buffered-write-recovery/angry-cat.png b/librocksdb-sys/rocksdb/docs/static/images/lost-buffered-write-recovery/angry-cat.png new file mode 100644 index 0000000..e956fb6 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/lost-buffered-write-recovery/angry-cat.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/lost-buffered-write-recovery/basic-setup.png b/librocksdb-sys/rocksdb/docs/static/images/lost-buffered-write-recovery/basic-setup.png new file mode 100644 index 0000000..f79831a Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/lost-buffered-write-recovery/basic-setup.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/lost-buffered-write-recovery/happy-cat.png b/librocksdb-sys/rocksdb/docs/static/images/lost-buffered-write-recovery/happy-cat.png new file mode 100644 index 0000000..155b534 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/lost-buffered-write-recovery/happy-cat.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/lost-buffered-write-recovery/replay-extension.png b/librocksdb-sys/rocksdb/docs/static/images/lost-buffered-write-recovery/replay-extension.png new file mode 100644 index 0000000..5bedd94 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/lost-buffered-write-recovery/replay-extension.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/lost-buffered-write-recovery/test-fs-writable-file.png b/librocksdb-sys/rocksdb/docs/static/images/lost-buffered-write-recovery/test-fs-writable-file.png new file mode 100644 index 0000000..58db8e2 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/lost-buffered-write-recovery/test-fs-writable-file.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/lost-buffered-write-recovery/trace-extension.png b/librocksdb-sys/rocksdb/docs/static/images/lost-buffered-write-recovery/trace-extension.png new file mode 100644 index 0000000..f782955 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/lost-buffered-write-recovery/trace-extension.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/pcache-blockindex.jpg b/librocksdb-sys/rocksdb/docs/static/images/pcache-blockindex.jpg new file mode 100644 index 0000000..9c18bde Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/pcache-blockindex.jpg differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/pcache-fileindex.jpg b/librocksdb-sys/rocksdb/docs/static/images/pcache-fileindex.jpg new file mode 100644 index 0000000..51f4e09 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/pcache-fileindex.jpg differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/pcache-filelayout.jpg b/librocksdb-sys/rocksdb/docs/static/images/pcache-filelayout.jpg new file mode 100644 index 0000000..771ee60 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/pcache-filelayout.jpg differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/pcache-readiopath.jpg b/librocksdb-sys/rocksdb/docs/static/images/pcache-readiopath.jpg new file mode 100644 index 0000000..4993f00 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/pcache-readiopath.jpg differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/pcache-tieredstorage.jpg b/librocksdb-sys/rocksdb/docs/static/images/pcache-tieredstorage.jpg new file mode 100644 index 0000000..c362a2d Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/pcache-tieredstorage.jpg differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/pcache-writeiopath.jpg b/librocksdb-sys/rocksdb/docs/static/images/pcache-writeiopath.jpg new file mode 100644 index 0000000..561b551 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/pcache-writeiopath.jpg differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/promo-adapt.svg b/librocksdb-sys/rocksdb/docs/static/images/promo-adapt.svg new file mode 100644 index 0000000..7cd4443 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/static/images/promo-adapt.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/librocksdb-sys/rocksdb/docs/static/images/promo-flash.svg b/librocksdb-sys/rocksdb/docs/static/images/promo-flash.svg new file mode 100644 index 0000000..79810c3 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/static/images/promo-flash.svg @@ -0,0 +1,28 @@ + + + + + + + + + + +]> + + + + + + + + + + + diff --git a/librocksdb-sys/rocksdb/docs/static/images/promo-operations.svg b/librocksdb-sys/rocksdb/docs/static/images/promo-operations.svg new file mode 100644 index 0000000..3036294 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/static/images/promo-operations.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/librocksdb-sys/rocksdb/docs/static/images/promo-performance.svg b/librocksdb-sys/rocksdb/docs/static/images/promo-performance.svg new file mode 100644 index 0000000..be8a101 --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/static/images/promo-performance.svg @@ -0,0 +1,134 @@ + + + + + + + + + + +netalloy chequered flag + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/librocksdb-sys/rocksdb/docs/static/images/rate-limiter/auto-tuned-write-KBps-series.png b/librocksdb-sys/rocksdb/docs/static/images/rate-limiter/auto-tuned-write-KBps-series.png new file mode 100644 index 0000000..b4b2484 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/rate-limiter/auto-tuned-write-KBps-series.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/rate-limiter/write-KBps-cdf.png b/librocksdb-sys/rocksdb/docs/static/images/rate-limiter/write-KBps-cdf.png new file mode 100644 index 0000000..742f985 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/rate-limiter/write-KBps-cdf.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/rate-limiter/write-KBps-series.png b/librocksdb-sys/rocksdb/docs/static/images/rate-limiter/write-KBps-series.png new file mode 100644 index 0000000..c7bdcb9 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/rate-limiter/write-KBps-series.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/rocksdb-secondary-cache/Mixgraph_hit_rate.png b/librocksdb-sys/rocksdb/docs/static/images/rocksdb-secondary-cache/Mixgraph_hit_rate.png new file mode 100644 index 0000000..10fa737 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/rocksdb-secondary-cache/Mixgraph_hit_rate.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/rocksdb-secondary-cache/Mixgraph_throughput.png b/librocksdb-sys/rocksdb/docs/static/images/rocksdb-secondary-cache/Mixgraph_throughput.png new file mode 100644 index 0000000..df2e333 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/rocksdb-secondary-cache/Mixgraph_throughput.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/rocksdb-secondary-cache/arch_diagram.png b/librocksdb-sys/rocksdb/docs/static/images/rocksdb-secondary-cache/arch_diagram.png new file mode 100644 index 0000000..696a376 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/rocksdb-secondary-cache/arch_diagram.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/rocksdb-secondary-cache/insert_flow.png b/librocksdb-sys/rocksdb/docs/static/images/rocksdb-secondary-cache/insert_flow.png new file mode 100644 index 0000000..f02e7e4 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/rocksdb-secondary-cache/insert_flow.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/rocksdb-secondary-cache/lookup_flow.png b/librocksdb-sys/rocksdb/docs/static/images/rocksdb-secondary-cache/lookup_flow.png new file mode 100644 index 0000000..2b3c70e Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/rocksdb-secondary-cache/lookup_flow.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/time-aware-tiered-storage/compaction_moving_up_conflict.png b/librocksdb-sys/rocksdb/docs/static/images/time-aware-tiered-storage/compaction_moving_up_conflict.png new file mode 100644 index 0000000..8feaef2 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/time-aware-tiered-storage/compaction_moving_up_conflict.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/time-aware-tiered-storage/per_key_placement_compaction.png b/librocksdb-sys/rocksdb/docs/static/images/time-aware-tiered-storage/per_key_placement_compaction.png new file mode 100644 index 0000000..0b232d1 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/time-aware-tiered-storage/per_key_placement_compaction.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/time-aware-tiered-storage/tiered_storage_design.png b/librocksdb-sys/rocksdb/docs/static/images/time-aware-tiered-storage/tiered_storage_design.png new file mode 100644 index 0000000..7e5158c Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/time-aware-tiered-storage/tiered_storage_design.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/time-aware-tiered-storage/tiered_storage_overview.png b/librocksdb-sys/rocksdb/docs/static/images/time-aware-tiered-storage/tiered_storage_overview.png new file mode 100644 index 0000000..7d115e6 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/time-aware-tiered-storage/tiered_storage_overview.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/time-aware-tiered-storage/tiered_storage_problem.png b/librocksdb-sys/rocksdb/docs/static/images/time-aware-tiered-storage/tiered_storage_problem.png new file mode 100644 index 0000000..dbe2ae5 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/time-aware-tiered-storage/tiered_storage_problem.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/images/tree_example1.png b/librocksdb-sys/rocksdb/docs/static/images/tree_example1.png new file mode 100644 index 0000000..9f72586 Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/images/tree_example1.png differ diff --git a/librocksdb-sys/rocksdb/docs/static/logo.svg b/librocksdb-sys/rocksdb/docs/static/logo.svg new file mode 100644 index 0000000..e6e1e8a --- /dev/null +++ b/librocksdb-sys/rocksdb/docs/static/logo.svg @@ -0,0 +1,76 @@ + + + + + + + + + + + + + + + + diff --git a/librocksdb-sys/rocksdb/docs/static/og_image.png b/librocksdb-sys/rocksdb/docs/static/og_image.png new file mode 100644 index 0000000..4e2759e Binary files /dev/null and b/librocksdb-sys/rocksdb/docs/static/og_image.png differ diff --git a/librocksdb-sys/rocksdb/env/composite_env.cc b/librocksdb-sys/rocksdb/env/composite_env.cc new file mode 100644 index 0000000..8ddc9a1 --- /dev/null +++ b/librocksdb-sys/rocksdb/env/composite_env.cc @@ -0,0 +1,534 @@ +// Copyright (c) 2019-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#include "env/composite_env_wrapper.h" +#include "rocksdb/utilities/options_type.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { +namespace { +// The CompositeEnvWrapper class provides an interface that is compatible +// with the old monolithic Env API, and an implementation that wraps around +// the new Env that provides threading and other OS related functionality, and +// the new FileSystem API that provides storage functionality. By +// providing the old Env interface, it allows the rest of RocksDB code to +// be agnostic of whether the underlying Env implementation is a monolithic +// Env or an Env + FileSystem. In the former case, the user will specify +// Options::env only, whereas in the latter case, the user will specify +// Options::env and Options::file_system. + +class CompositeSequentialFileWrapper : public SequentialFile { + public: + explicit CompositeSequentialFileWrapper( + std::unique_ptr& target) + : target_(std::move(target)) {} + + Status Read(size_t n, Slice* result, char* scratch) override { + IOOptions io_opts; + IODebugContext dbg; + return target_->Read(n, io_opts, result, scratch, &dbg); + } + Status Skip(uint64_t n) override { return target_->Skip(n); } + bool use_direct_io() const override { return target_->use_direct_io(); } + size_t GetRequiredBufferAlignment() const override { + return target_->GetRequiredBufferAlignment(); + } + Status InvalidateCache(size_t offset, size_t length) override { + return target_->InvalidateCache(offset, length); + } + Status PositionedRead(uint64_t offset, size_t n, Slice* result, + char* scratch) override { + IOOptions io_opts; + IODebugContext dbg; + return target_->PositionedRead(offset, n, io_opts, result, scratch, &dbg); + } + + private: + std::unique_ptr target_; +}; + +class CompositeRandomAccessFileWrapper : public RandomAccessFile { + public: + explicit CompositeRandomAccessFileWrapper( + std::unique_ptr& target) + : target_(std::move(target)) {} + + Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const override { + IOOptions io_opts; + IODebugContext dbg; + return target_->Read(offset, n, io_opts, result, scratch, &dbg); + } + Status MultiRead(ReadRequest* reqs, size_t num_reqs) override { + IOOptions io_opts; + IODebugContext dbg; + std::vector fs_reqs; + Status status; + + fs_reqs.resize(num_reqs); + for (size_t i = 0; i < num_reqs; ++i) { + fs_reqs[i].offset = reqs[i].offset; + fs_reqs[i].len = reqs[i].len; + fs_reqs[i].scratch = reqs[i].scratch; + fs_reqs[i].status = IOStatus::OK(); + } + status = target_->MultiRead(fs_reqs.data(), num_reqs, io_opts, &dbg); + for (size_t i = 0; i < num_reqs; ++i) { + reqs[i].result = fs_reqs[i].result; + reqs[i].status = fs_reqs[i].status; + } + return status; + } + Status Prefetch(uint64_t offset, size_t n) override { + IOOptions io_opts; + IODebugContext dbg; + return target_->Prefetch(offset, n, io_opts, &dbg); + } + size_t GetUniqueId(char* id, size_t max_size) const override { + return target_->GetUniqueId(id, max_size); + } + void Hint(AccessPattern pattern) override { + target_->Hint((FSRandomAccessFile::AccessPattern)pattern); + } + bool use_direct_io() const override { return target_->use_direct_io(); } + size_t GetRequiredBufferAlignment() const override { + return target_->GetRequiredBufferAlignment(); + } + Status InvalidateCache(size_t offset, size_t length) override { + return target_->InvalidateCache(offset, length); + } + + private: + std::unique_ptr target_; +}; + +class CompositeWritableFileWrapper : public WritableFile { + public: + explicit CompositeWritableFileWrapper(std::unique_ptr& t) + : target_(std::move(t)) {} + + Status Append(const Slice& data) override { + IOOptions io_opts; + IODebugContext dbg; + return target_->Append(data, io_opts, &dbg); + } + Status Append(const Slice& data, + const DataVerificationInfo& verification_info) override { + IOOptions io_opts; + IODebugContext dbg; + return target_->Append(data, io_opts, verification_info, &dbg); + } + Status PositionedAppend(const Slice& data, uint64_t offset) override { + IOOptions io_opts; + IODebugContext dbg; + return target_->PositionedAppend(data, offset, io_opts, &dbg); + } + Status PositionedAppend( + const Slice& data, uint64_t offset, + const DataVerificationInfo& verification_info) override { + IOOptions io_opts; + IODebugContext dbg; + return target_->PositionedAppend(data, offset, io_opts, verification_info, + &dbg); + } + Status Truncate(uint64_t size) override { + IOOptions io_opts; + IODebugContext dbg; + return target_->Truncate(size, io_opts, &dbg); + } + Status Close() override { + IOOptions io_opts; + IODebugContext dbg; + return target_->Close(io_opts, &dbg); + } + Status Flush() override { + IOOptions io_opts; + IODebugContext dbg; + return target_->Flush(io_opts, &dbg); + } + Status Sync() override { + IOOptions io_opts; + IODebugContext dbg; + return target_->Sync(io_opts, &dbg); + } + Status Fsync() override { + IOOptions io_opts; + IODebugContext dbg; + return target_->Fsync(io_opts, &dbg); + } + bool IsSyncThreadSafe() const override { return target_->IsSyncThreadSafe(); } + + bool use_direct_io() const override { return target_->use_direct_io(); } + + size_t GetRequiredBufferAlignment() const override { + return target_->GetRequiredBufferAlignment(); + } + + void SetWriteLifeTimeHint(Env::WriteLifeTimeHint hint) override { + target_->SetWriteLifeTimeHint(hint); + } + + Env::WriteLifeTimeHint GetWriteLifeTimeHint() override { + return target_->GetWriteLifeTimeHint(); + } + + uint64_t GetFileSize() override { + IOOptions io_opts; + IODebugContext dbg; + return target_->GetFileSize(io_opts, &dbg); + } + + void SetPreallocationBlockSize(size_t size) override { + target_->SetPreallocationBlockSize(size); + } + + void GetPreallocationStatus(size_t* block_size, + size_t* last_allocated_block) override { + target_->GetPreallocationStatus(block_size, last_allocated_block); + } + + size_t GetUniqueId(char* id, size_t max_size) const override { + return target_->GetUniqueId(id, max_size); + } + + Status InvalidateCache(size_t offset, size_t length) override { + return target_->InvalidateCache(offset, length); + } + + Status RangeSync(uint64_t offset, uint64_t nbytes) override { + IOOptions io_opts; + IODebugContext dbg; + return target_->RangeSync(offset, nbytes, io_opts, &dbg); + } + + void PrepareWrite(size_t offset, size_t len) override { + IOOptions io_opts; + IODebugContext dbg; + target_->PrepareWrite(offset, len, io_opts, &dbg); + } + + Status Allocate(uint64_t offset, uint64_t len) override { + IOOptions io_opts; + IODebugContext dbg; + return target_->Allocate(offset, len, io_opts, &dbg); + } + + std::unique_ptr* target() { return &target_; } + + private: + std::unique_ptr target_; +}; + +class CompositeRandomRWFileWrapper : public RandomRWFile { + public: + explicit CompositeRandomRWFileWrapper(std::unique_ptr& target) + : target_(std::move(target)) {} + + bool use_direct_io() const override { return target_->use_direct_io(); } + size_t GetRequiredBufferAlignment() const override { + return target_->GetRequiredBufferAlignment(); + } + Status Write(uint64_t offset, const Slice& data) override { + IOOptions io_opts; + IODebugContext dbg; + return target_->Write(offset, data, io_opts, &dbg); + } + Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const override { + IOOptions io_opts; + IODebugContext dbg; + return target_->Read(offset, n, io_opts, result, scratch, &dbg); + } + Status Flush() override { + IOOptions io_opts; + IODebugContext dbg; + return target_->Flush(io_opts, &dbg); + } + Status Sync() override { + IOOptions io_opts; + IODebugContext dbg; + return target_->Sync(io_opts, &dbg); + } + Status Fsync() override { + IOOptions io_opts; + IODebugContext dbg; + return target_->Fsync(io_opts, &dbg); + } + Status Close() override { + IOOptions io_opts; + IODebugContext dbg; + return target_->Close(io_opts, &dbg); + } + + private: + std::unique_ptr target_; +}; + +class CompositeDirectoryWrapper : public Directory { + public: + explicit CompositeDirectoryWrapper(std::unique_ptr& target) + : target_(std::move(target)) {} + + Status Fsync() override { + IOOptions io_opts; + IODebugContext dbg; + return target_->FsyncWithDirOptions(io_opts, &dbg, DirFsyncOptions()); + } + + Status Close() override { + IOOptions io_opts; + IODebugContext dbg; + return target_->Close(io_opts, &dbg); + } + + size_t GetUniqueId(char* id, size_t max_size) const override { + return target_->GetUniqueId(id, max_size); + } + + private: + std::unique_ptr target_; +}; +} // namespace + +Status CompositeEnv::NewSequentialFile(const std::string& f, + std::unique_ptr* r, + const EnvOptions& options) { + IODebugContext dbg; + std::unique_ptr file; + Status status; + status = + file_system_->NewSequentialFile(f, FileOptions(options), &file, &dbg); + if (status.ok()) { + r->reset(new CompositeSequentialFileWrapper(file)); + } + return status; +} + +Status CompositeEnv::NewRandomAccessFile(const std::string& f, + std::unique_ptr* r, + const EnvOptions& options) { + IODebugContext dbg; + std::unique_ptr file; + Status status; + status = + file_system_->NewRandomAccessFile(f, FileOptions(options), &file, &dbg); + if (status.ok()) { + r->reset(new CompositeRandomAccessFileWrapper(file)); + } + return status; +} + +Status CompositeEnv::NewWritableFile(const std::string& f, + std::unique_ptr* r, + const EnvOptions& options) { + IODebugContext dbg; + std::unique_ptr file; + Status status; + status = file_system_->NewWritableFile(f, FileOptions(options), &file, &dbg); + if (status.ok()) { + r->reset(new CompositeWritableFileWrapper(file)); + } + return status; +} + +Status CompositeEnv::ReopenWritableFile(const std::string& fname, + std::unique_ptr* result, + const EnvOptions& options) { + IODebugContext dbg; + Status status; + std::unique_ptr file; + status = file_system_->ReopenWritableFile(fname, FileOptions(options), &file, + &dbg); + if (status.ok()) { + result->reset(new CompositeWritableFileWrapper(file)); + } + return status; +} + +Status CompositeEnv::ReuseWritableFile(const std::string& fname, + const std::string& old_fname, + std::unique_ptr* r, + const EnvOptions& options) { + IODebugContext dbg; + Status status; + std::unique_ptr file; + status = file_system_->ReuseWritableFile(fname, old_fname, + FileOptions(options), &file, &dbg); + if (status.ok()) { + r->reset(new CompositeWritableFileWrapper(file)); + } + return status; +} + +Status CompositeEnv::NewRandomRWFile(const std::string& fname, + std::unique_ptr* result, + const EnvOptions& options) { + IODebugContext dbg; + std::unique_ptr file; + Status status; + status = + file_system_->NewRandomRWFile(fname, FileOptions(options), &file, &dbg); + if (status.ok()) { + result->reset(new CompositeRandomRWFileWrapper(file)); + } + return status; +} + +Status CompositeEnv::NewDirectory(const std::string& name, + std::unique_ptr* result) { + IOOptions io_opts; + IODebugContext dbg; + std::unique_ptr dir; + Status status; + status = file_system_->NewDirectory(name, io_opts, &dir, &dbg); + if (status.ok()) { + result->reset(new CompositeDirectoryWrapper(dir)); + } + return status; +} + +namespace { +static std::unordered_map env_wrapper_type_info = { + {"target", + OptionTypeInfo(0, OptionType::kUnknown, OptionVerificationType::kByName, + OptionTypeFlags::kDontSerialize) + .SetParseFunc([](const ConfigOptions& opts, + const std::string& /*name*/, const std::string& value, + void* addr) { + auto target = static_cast(addr); + return Env::CreateFromString(opts, value, &(target->env), + &(target->guard)); + }) + .SetEqualsFunc([](const ConfigOptions& opts, + const std::string& /*name*/, const void* addr1, + const void* addr2, std::string* mismatch) { + const auto target1 = static_cast(addr1); + const auto target2 = static_cast(addr2); + if (target1->env != nullptr) { + return target1->env->AreEquivalent(opts, target2->env, mismatch); + } else { + return (target2->env == nullptr); + } + }) + .SetPrepareFunc([](const ConfigOptions& opts, + const std::string& /*name*/, void* addr) { + auto target = static_cast(addr); + if (target->guard.get() != nullptr) { + target->env = target->guard.get(); + } else if (target->env == nullptr) { + target->env = Env::Default(); + } + return target->env->PrepareOptions(opts); + }) + .SetValidateFunc([](const DBOptions& db_opts, + const ColumnFamilyOptions& cf_opts, + const std::string& /*name*/, const void* addr) { + const auto target = static_cast(addr); + if (target->env == nullptr) { + return Status::InvalidArgument("Target Env not specified"); + } else { + return target->env->ValidateOptions(db_opts, cf_opts); + } + })}, +}; +static std::unordered_map + composite_fs_wrapper_type_info = { + {"file_system", + OptionTypeInfo::AsCustomSharedPtr( + 0, OptionVerificationType::kByName, OptionTypeFlags::kNone)}, +}; + +static std::unordered_map + composite_clock_wrapper_type_info = { + {"clock", + OptionTypeInfo::AsCustomSharedPtr( + 0, OptionVerificationType::kByName, OptionTypeFlags::kNone)}, +}; + +} // namespace + +std::unique_ptr NewCompositeEnv(const std::shared_ptr& fs) { + return std::unique_ptr(new CompositeEnvWrapper(Env::Default(), fs)); +} + +CompositeEnvWrapper::CompositeEnvWrapper(Env* env, + const std::shared_ptr& fs, + const std::shared_ptr& sc) + : CompositeEnv(fs, sc), target_(env) { + RegisterOptions("", &target_, &env_wrapper_type_info); + RegisterOptions("", &file_system_, &composite_fs_wrapper_type_info); + RegisterOptions("", &system_clock_, &composite_clock_wrapper_type_info); +} + +CompositeEnvWrapper::CompositeEnvWrapper(const std::shared_ptr& env, + const std::shared_ptr& fs, + const std::shared_ptr& sc) + : CompositeEnv(fs, sc), target_(env) { + RegisterOptions("", &target_, &env_wrapper_type_info); + RegisterOptions("", &file_system_, &composite_fs_wrapper_type_info); + RegisterOptions("", &system_clock_, &composite_clock_wrapper_type_info); +} + +Status CompositeEnvWrapper::PrepareOptions(const ConfigOptions& options) { + target_.Prepare(); + if (file_system_ == nullptr) { + file_system_ = target_.env->GetFileSystem(); + } + if (system_clock_ == nullptr) { + system_clock_ = target_.env->GetSystemClock(); + } + return Env::PrepareOptions(options); +} + +std::string CompositeEnvWrapper::SerializeOptions( + const ConfigOptions& config_options, const std::string& header) const { + auto options = CompositeEnv::SerializeOptions(config_options, header); + if (target_.env != nullptr && target_.env != Env::Default()) { + options.append("target="); + options.append(target_.env->ToString(config_options)); + } + return options; +} + +EnvWrapper::EnvWrapper(Env* t) : target_(t) { + RegisterOptions("", &target_, &env_wrapper_type_info); +} + +EnvWrapper::EnvWrapper(std::unique_ptr&& t) : target_(std::move(t)) { + RegisterOptions("", &target_, &env_wrapper_type_info); +} + +EnvWrapper::EnvWrapper(const std::shared_ptr& t) : target_(t) { + RegisterOptions("", &target_, &env_wrapper_type_info); +} + +EnvWrapper::~EnvWrapper() {} + +Status EnvWrapper::PrepareOptions(const ConfigOptions& options) { + target_.Prepare(); + return Env::PrepareOptions(options); +} + +std::string EnvWrapper::SerializeOptions(const ConfigOptions& config_options, + const std::string& header) const { + auto parent = Env::SerializeOptions(config_options, ""); + if (config_options.IsShallow() || target_.env == nullptr || + target_.env == Env::Default()) { + return parent; + } else { + std::string result = header; + if (!StartsWith(parent, OptionTypeInfo::kIdPropName())) { + result.append(OptionTypeInfo::kIdPropName()).append("="); + } + result.append(parent); + if (!EndsWith(result, config_options.delimiter)) { + result.append(config_options.delimiter); + } + result.append("target=").append(target_.env->ToString(config_options)); + return result; + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/env/composite_env_wrapper.h b/librocksdb-sys/rocksdb/env/composite_env_wrapper.h new file mode 100644 index 0000000..003c506 --- /dev/null +++ b/librocksdb-sys/rocksdb/env/composite_env_wrapper.h @@ -0,0 +1,378 @@ +// Copyright (c) 2019-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "rocksdb/env.h" +#include "rocksdb/file_system.h" +#include "rocksdb/system_clock.h" + +#ifdef _WIN32 +// Windows API macro interference +#undef DeleteFile +#undef GetCurrentTime +#undef LoadLibrary +#endif + +namespace ROCKSDB_NAMESPACE { + +class CompositeEnv : public Env { + public: + // Initialize a CompositeEnvWrapper that delegates all thread/time related + // calls to env, and all file operations to fs + explicit CompositeEnv(const std::shared_ptr& fs, + const std::shared_ptr& clock) + : Env(fs, clock) {} + + Status RegisterDbPaths(const std::vector& paths) override { + return file_system_->RegisterDbPaths(paths); + } + Status UnregisterDbPaths(const std::vector& paths) override { + return file_system_->UnregisterDbPaths(paths); + } + + // The following text is boilerplate that forwards all methods to target() + Status NewSequentialFile(const std::string& f, + std::unique_ptr* r, + const EnvOptions& options) override; + + Status NewRandomAccessFile(const std::string& f, + std::unique_ptr* r, + const EnvOptions& options) override; + + Status NewWritableFile(const std::string& f, std::unique_ptr* r, + const EnvOptions& options) override; + + Status ReopenWritableFile(const std::string& fname, + std::unique_ptr* result, + const EnvOptions& options) override; + + Status ReuseWritableFile(const std::string& fname, + const std::string& old_fname, + std::unique_ptr* r, + const EnvOptions& options) override; + + Status NewRandomRWFile(const std::string& fname, + std::unique_ptr* result, + const EnvOptions& options) override; + + Status NewMemoryMappedFileBuffer( + const std::string& fname, + std::unique_ptr* result) override { + return file_system_->NewMemoryMappedFileBuffer(fname, result); + } + + Status NewDirectory(const std::string& name, + std::unique_ptr* result) override; + + Status FileExists(const std::string& f) override { + IOOptions io_opts; + IODebugContext dbg; + return file_system_->FileExists(f, io_opts, &dbg); + } + Status GetChildren(const std::string& dir, + std::vector* r) override { + IOOptions io_opts; + IODebugContext dbg; + return file_system_->GetChildren(dir, io_opts, r, &dbg); + } + Status GetChildrenFileAttributes( + const std::string& dir, std::vector* result) override { + IOOptions io_opts; + IODebugContext dbg; + return file_system_->GetChildrenFileAttributes(dir, io_opts, result, &dbg); + } + Status DeleteFile(const std::string& f) override { + IOOptions io_opts; + IODebugContext dbg; + return file_system_->DeleteFile(f, io_opts, &dbg); + } + Status Truncate(const std::string& fname, size_t size) override { + IOOptions io_opts; + IODebugContext dbg; + return file_system_->Truncate(fname, size, io_opts, &dbg); + } + Status CreateDir(const std::string& d) override { + IOOptions io_opts; + IODebugContext dbg; + return file_system_->CreateDir(d, io_opts, &dbg); + } + Status CreateDirIfMissing(const std::string& d) override { + IOOptions io_opts; + IODebugContext dbg; + return file_system_->CreateDirIfMissing(d, io_opts, &dbg); + } + Status DeleteDir(const std::string& d) override { + IOOptions io_opts; + IODebugContext dbg; + return file_system_->DeleteDir(d, io_opts, &dbg); + } + Status GetFileSize(const std::string& f, uint64_t* s) override { + IOOptions io_opts; + IODebugContext dbg; + return file_system_->GetFileSize(f, io_opts, s, &dbg); + } + + Status GetFileModificationTime(const std::string& fname, + uint64_t* file_mtime) override { + IOOptions io_opts; + IODebugContext dbg; + return file_system_->GetFileModificationTime(fname, io_opts, file_mtime, + &dbg); + } + + Status RenameFile(const std::string& s, const std::string& t) override { + IOOptions io_opts; + IODebugContext dbg; + return file_system_->RenameFile(s, t, io_opts, &dbg); + } + + Status LinkFile(const std::string& s, const std::string& t) override { + IOOptions io_opts; + IODebugContext dbg; + return file_system_->LinkFile(s, t, io_opts, &dbg); + } + + Status NumFileLinks(const std::string& fname, uint64_t* count) override { + IOOptions io_opts; + IODebugContext dbg; + return file_system_->NumFileLinks(fname, io_opts, count, &dbg); + } + + Status AreFilesSame(const std::string& first, const std::string& second, + bool* res) override { + IOOptions io_opts; + IODebugContext dbg; + return file_system_->AreFilesSame(first, second, io_opts, res, &dbg); + } + + Status LockFile(const std::string& f, FileLock** l) override { + IOOptions io_opts; + IODebugContext dbg; + return file_system_->LockFile(f, io_opts, l, &dbg); + } + + Status UnlockFile(FileLock* l) override { + IOOptions io_opts; + IODebugContext dbg; + return file_system_->UnlockFile(l, io_opts, &dbg); + } + + Status GetAbsolutePath(const std::string& db_path, + std::string* output_path) override { + IOOptions io_opts; + IODebugContext dbg; + return file_system_->GetAbsolutePath(db_path, io_opts, output_path, &dbg); + } + + Status NewLogger(const std::string& fname, + std::shared_ptr* result) override { + IOOptions io_opts; + IODebugContext dbg; + return file_system_->NewLogger(fname, io_opts, result, &dbg); + } + + Status IsDirectory(const std::string& path, bool* is_dir) override { + IOOptions io_opts; + IODebugContext dbg; + return file_system_->IsDirectory(path, io_opts, is_dir, &dbg); + } + + Status GetTestDirectory(std::string* path) override { + IOOptions io_opts; + IODebugContext dbg; + return file_system_->GetTestDirectory(io_opts, path, &dbg); + } + + EnvOptions OptimizeForLogRead(const EnvOptions& env_options) const override { + return file_system_->OptimizeForLogRead(FileOptions(env_options)); + } + + EnvOptions OptimizeForManifestRead( + const EnvOptions& env_options) const override { + return file_system_->OptimizeForManifestRead(FileOptions(env_options)); + } + + EnvOptions OptimizeForLogWrite(const EnvOptions& env_options, + const DBOptions& db_options) const override { + return file_system_->OptimizeForLogWrite(FileOptions(env_options), + db_options); + } + + EnvOptions OptimizeForManifestWrite( + const EnvOptions& env_options) const override { + return file_system_->OptimizeForManifestWrite(FileOptions(env_options)); + } + + EnvOptions OptimizeForCompactionTableWrite( + const EnvOptions& env_options, + const ImmutableDBOptions& immutable_ops) const override { + return file_system_->OptimizeForCompactionTableWrite( + FileOptions(env_options), immutable_ops); + } + EnvOptions OptimizeForCompactionTableRead( + const EnvOptions& env_options, + const ImmutableDBOptions& db_options) const override { + return file_system_->OptimizeForCompactionTableRead( + FileOptions(env_options), db_options); + } + EnvOptions OptimizeForBlobFileRead( + const EnvOptions& env_options, + const ImmutableDBOptions& db_options) const override { + return file_system_->OptimizeForBlobFileRead(FileOptions(env_options), + db_options); + } + // This seems to clash with a macro on Windows, so #undef it here +#ifdef GetFreeSpace +#undef GetFreeSpace +#endif + Status GetFreeSpace(const std::string& path, uint64_t* diskfree) override { + IOOptions io_opts; + IODebugContext dbg; + return file_system_->GetFreeSpace(path, io_opts, diskfree, &dbg); + } + uint64_t NowMicros() override { return system_clock_->NowMicros(); } + uint64_t NowNanos() override { return system_clock_->NowNanos(); } + + uint64_t NowCPUNanos() override { return system_clock_->CPUNanos(); } + + void SleepForMicroseconds(int micros) override { + system_clock_->SleepForMicroseconds(micros); + } + + Status GetCurrentTime(int64_t* unix_time) override { + return system_clock_->GetCurrentTime(unix_time); + } + std::string TimeToString(uint64_t time) override { + return system_clock_->TimeToString(time); + } +}; + +class CompositeEnvWrapper : public CompositeEnv { + public: + // Initialize a CompositeEnvWrapper that delegates all thread/time related + // calls to env, and all file operations to fs + explicit CompositeEnvWrapper(Env* env) + : CompositeEnvWrapper(env, env->GetFileSystem(), env->GetSystemClock()) {} + explicit CompositeEnvWrapper(Env* env, const std::shared_ptr& fs) + : CompositeEnvWrapper(env, fs, env->GetSystemClock()) {} + + explicit CompositeEnvWrapper(Env* env, const std::shared_ptr& sc) + : CompositeEnvWrapper(env, env->GetFileSystem(), sc) {} + + explicit CompositeEnvWrapper(Env* env, const std::shared_ptr& fs, + const std::shared_ptr& sc); + + explicit CompositeEnvWrapper(const std::shared_ptr& env, + const std::shared_ptr& fs) + : CompositeEnvWrapper(env, fs, env->GetSystemClock()) {} + + explicit CompositeEnvWrapper(const std::shared_ptr& env, + const std::shared_ptr& sc) + : CompositeEnvWrapper(env, env->GetFileSystem(), sc) {} + + explicit CompositeEnvWrapper(const std::shared_ptr& env, + const std::shared_ptr& fs, + const std::shared_ptr& sc); + + static const char* kClassName() { return "CompositeEnv"; } + const char* Name() const override { return kClassName(); } + bool IsInstanceOf(const std::string& name) const override { + if (name == kClassName()) { + return true; + } else { + return CompositeEnv::IsInstanceOf(name); + } + } + const Customizable* Inner() const override { return target_.env; } + + Status PrepareOptions(const ConfigOptions& options) override; + std::string SerializeOptions(const ConfigOptions& config_options, + const std::string& header) const override; + + // Return the target to which this Env forwards all calls + Env* env_target() const { return target_.env; } + +#if !defined(OS_WIN) && !defined(ROCKSDB_NO_DYNAMIC_EXTENSION) + Status LoadLibrary(const std::string& lib_name, + const std::string& search_path, + std::shared_ptr* result) override { + return target_.env->LoadLibrary(lib_name, search_path, result); + } +#endif + + void Schedule(void (*f)(void* arg), void* a, Priority pri, + void* tag = nullptr, void (*u)(void* arg) = nullptr) override { + return target_.env->Schedule(f, a, pri, tag, u); + } + + int UnSchedule(void* tag, Priority pri) override { + return target_.env->UnSchedule(tag, pri); + } + + void StartThread(void (*f)(void*), void* a) override { + return target_.env->StartThread(f, a); + } + void WaitForJoin() override { return target_.env->WaitForJoin(); } + unsigned int GetThreadPoolQueueLen(Priority pri = LOW) const override { + return target_.env->GetThreadPoolQueueLen(pri); + } + + int ReserveThreads(int threads_to_be_reserved, Priority pri) override { + return target_.env->ReserveThreads(threads_to_be_reserved, pri); + } + + int ReleaseThreads(int threads_to_be_released, Priority pri) override { + return target_.env->ReleaseThreads(threads_to_be_released, pri); + } + + Status GetHostName(char* name, uint64_t len) override { + return target_.env->GetHostName(name, len); + } + void SetBackgroundThreads(int num, Priority pri) override { + return target_.env->SetBackgroundThreads(num, pri); + } + int GetBackgroundThreads(Priority pri) override { + return target_.env->GetBackgroundThreads(pri); + } + + Status SetAllowNonOwnerAccess(bool allow_non_owner_access) override { + return target_.env->SetAllowNonOwnerAccess(allow_non_owner_access); + } + + void IncBackgroundThreadsIfNeeded(int num, Priority pri) override { + return target_.env->IncBackgroundThreadsIfNeeded(num, pri); + } + + void LowerThreadPoolIOPriority(Priority pool) override { + target_.env->LowerThreadPoolIOPriority(pool); + } + + void LowerThreadPoolCPUPriority(Priority pool) override { + target_.env->LowerThreadPoolCPUPriority(pool); + } + + Status LowerThreadPoolCPUPriority(Priority pool, CpuPriority pri) override { + return target_.env->LowerThreadPoolCPUPriority(pool, pri); + } + + Status GetThreadList(std::vector* thread_list) override { + return target_.env->GetThreadList(thread_list); + } + + ThreadStatusUpdater* GetThreadStatusUpdater() const override { + return target_.env->GetThreadStatusUpdater(); + } + + uint64_t GetThreadID() const override { return target_.env->GetThreadID(); } + + std::string GenerateUniqueId() override { + return target_.env->GenerateUniqueId(); + } + + private: + EnvWrapper::Target target_; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/env/emulated_clock.h b/librocksdb-sys/rocksdb/env/emulated_clock.h new file mode 100644 index 0000000..6227376 --- /dev/null +++ b/librocksdb-sys/rocksdb/env/emulated_clock.h @@ -0,0 +1,114 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once + +#include +#include + +#include "rocksdb/status.h" +#include "rocksdb/system_clock.h" + +namespace ROCKSDB_NAMESPACE { +// A SystemClock that can "mock" sleep and counts its operations. +class EmulatedSystemClock : public SystemClockWrapper { + private: + // Something to return when mocking current time + const int64_t maybe_starting_time_; + std::atomic sleep_counter_{0}; + std::atomic cpu_counter_{0}; + std::atomic addon_microseconds_{0}; + // Do not modify in the env of a running DB (could cause deadlock) + std::atomic time_elapse_only_sleep_; + bool no_slowdown_; + + public: + explicit EmulatedSystemClock(const std::shared_ptr& base, + bool time_elapse_only_sleep = false); + + static const char* kClassName() { return "TimeEmulatedSystemClock"; } + const char* Name() const override { return kClassName(); } + + virtual void SleepForMicroseconds(int micros) override { + sleep_counter_++; + if (no_slowdown_ || time_elapse_only_sleep_) { + addon_microseconds_.fetch_add(micros); + } + if (!no_slowdown_) { + SystemClockWrapper::SleepForMicroseconds(micros); + } + } + + void MockSleepForMicroseconds(int64_t micros) { + sleep_counter_++; + assert(no_slowdown_); + addon_microseconds_.fetch_add(micros); + } + + void MockSleepForSeconds(int64_t seconds) { + sleep_counter_++; + assert(no_slowdown_); + addon_microseconds_.fetch_add(seconds * 1000000); + } + + void SetTimeElapseOnlySleep(bool enabled) { + // We cannot set these before destroying the last DB because they might + // cause a deadlock or similar without the appropriate options set in + // the DB. + time_elapse_only_sleep_ = enabled; + no_slowdown_ = enabled; + } + + bool IsTimeElapseOnlySleep() const { return time_elapse_only_sleep_.load(); } + void SetMockSleep(bool enabled = true) { no_slowdown_ = enabled; } + bool IsMockSleepEnabled() const { return no_slowdown_; } + + int GetSleepCounter() const { return sleep_counter_.load(); } + + virtual Status GetCurrentTime(int64_t* unix_time) override { + Status s; + if (time_elapse_only_sleep_) { + *unix_time = maybe_starting_time_; + } else { + s = SystemClockWrapper::GetCurrentTime(unix_time); + } + if (s.ok()) { + // mock microseconds elapsed to seconds of time + *unix_time += addon_microseconds_.load() / 1000000; + } + return s; + } + + virtual uint64_t CPUNanos() override { + cpu_counter_++; + return SystemClockWrapper::CPUNanos(); + } + + virtual uint64_t CPUMicros() override { + cpu_counter_++; + return SystemClockWrapper::CPUMicros(); + } + + virtual uint64_t NowNanos() override { + return (time_elapse_only_sleep_ ? 0 : SystemClockWrapper::NowNanos()) + + addon_microseconds_.load() * 1000; + } + + virtual uint64_t NowMicros() override { + return (time_elapse_only_sleep_ ? 0 : SystemClockWrapper::NowMicros()) + + addon_microseconds_.load(); + } + + int GetCpuCounter() const { return cpu_counter_.load(); } + + void ResetCounters() { + cpu_counter_.store(0); + sleep_counter_.store(0); + } +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/env/env.cc b/librocksdb-sys/rocksdb/env/env.cc new file mode 100644 index 0000000..937be43 --- /dev/null +++ b/librocksdb-sys/rocksdb/env/env.cc @@ -0,0 +1,1232 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "rocksdb/env.h" + +#include + +#include "env/composite_env_wrapper.h" +#include "env/emulated_clock.h" +#include "env/mock_env.h" +#include "env/unique_id_gen.h" +#include "logging/env_logger.h" +#include "memory/arena.h" +#include "options/db_options.h" +#include "port/port.h" +#include "rocksdb/convenience.h" +#include "rocksdb/options.h" +#include "rocksdb/system_clock.h" +#include "rocksdb/utilities/customizable_util.h" +#include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/options_type.h" +#include "util/autovector.h" + +namespace ROCKSDB_NAMESPACE { +namespace { +static int RegisterBuiltinEnvs(ObjectLibrary& library, + const std::string& /*arg*/) { + library.AddFactory(MockEnv::kClassName(), [](const std::string& /*uri*/, + std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(MockEnv::Create(Env::Default())); + return guard->get(); + }); + library.AddFactory( + CompositeEnvWrapper::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new CompositeEnvWrapper(Env::Default())); + return guard->get(); + }); + size_t num_types; + return static_cast(library.GetFactoryCount(&num_types)); +} + +static void RegisterSystemEnvs() { + static std::once_flag loaded; + std::call_once(loaded, [&]() { + RegisterBuiltinEnvs(*(ObjectLibrary::Default().get()), ""); + }); +} + +class LegacySystemClock : public SystemClock { + private: + Env* env_; + + public: + explicit LegacySystemClock(Env* env) : env_(env) {} + const char* Name() const override { return "LegacySystemClock"; } + + // Returns the number of micro-seconds since some fixed point in time. + // It is often used as system time such as in GenericRateLimiter + // and other places so a port needs to return system time in order to work. + uint64_t NowMicros() override { return env_->NowMicros(); } + + // Returns the number of nano-seconds since some fixed point in time. Only + // useful for computing deltas of time in one run. + // Default implementation simply relies on NowMicros. + // In platform-specific implementations, NowNanos() should return time points + // that are MONOTONIC. + uint64_t NowNanos() override { return env_->NowNanos(); } + + uint64_t CPUMicros() override { return CPUNanos() / 1000; } + uint64_t CPUNanos() override { return env_->NowCPUNanos(); } + + // Sleep/delay the thread for the prescribed number of micro-seconds. + void SleepForMicroseconds(int micros) override { + env_->SleepForMicroseconds(micros); + } + + // Get the number of seconds since the Epoch, 1970-01-01 00:00:00 (UTC). + // Only overwrites *unix_time on success. + Status GetCurrentTime(int64_t* unix_time) override { + return env_->GetCurrentTime(unix_time); + } + // Converts seconds-since-Jan-01-1970 to a printable string + std::string TimeToString(uint64_t time) override { + return env_->TimeToString(time); + } + + std::string SerializeOptions(const ConfigOptions& /*config_options*/, + const std::string& /*prefix*/) const override { + // We do not want the LegacySystemClock to appear in the serialized output. + // This clock is an internal class for those who do not implement one and + // would be part of the Env. As such, do not serialize it here. + return ""; + } +}; + +class LegacySequentialFileWrapper : public FSSequentialFile { + public: + explicit LegacySequentialFileWrapper( + std::unique_ptr&& _target) + : target_(std::move(_target)) {} + + IOStatus Read(size_t n, const IOOptions& /*options*/, Slice* result, + char* scratch, IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->Read(n, result, scratch)); + } + IOStatus Skip(uint64_t n) override { + return status_to_io_status(target_->Skip(n)); + } + bool use_direct_io() const override { return target_->use_direct_io(); } + size_t GetRequiredBufferAlignment() const override { + return target_->GetRequiredBufferAlignment(); + } + IOStatus InvalidateCache(size_t offset, size_t length) override { + return status_to_io_status(target_->InvalidateCache(offset, length)); + } + IOStatus PositionedRead(uint64_t offset, size_t n, + const IOOptions& /*options*/, Slice* result, + char* scratch, IODebugContext* /*dbg*/) override { + return status_to_io_status( + target_->PositionedRead(offset, n, result, scratch)); + } + + private: + std::unique_ptr target_; +}; + +class LegacyRandomAccessFileWrapper : public FSRandomAccessFile { + public: + explicit LegacyRandomAccessFileWrapper( + std::unique_ptr&& target) + : target_(std::move(target)) {} + + IOStatus Read(uint64_t offset, size_t n, const IOOptions& /*options*/, + Slice* result, char* scratch, + IODebugContext* /*dbg*/) const override { + return status_to_io_status(target_->Read(offset, n, result, scratch)); + } + + IOStatus MultiRead(FSReadRequest* fs_reqs, size_t num_reqs, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + std::vector reqs; + Status status; + + reqs.reserve(num_reqs); + for (size_t i = 0; i < num_reqs; ++i) { + ReadRequest req; + + req.offset = fs_reqs[i].offset; + req.len = fs_reqs[i].len; + req.scratch = fs_reqs[i].scratch; + req.status = Status::OK(); + reqs.emplace_back(std::move(req)); + } + status = target_->MultiRead(reqs.data(), num_reqs); + for (size_t i = 0; i < num_reqs; ++i) { + fs_reqs[i].result = reqs[i].result; + fs_reqs[i].status = status_to_io_status(std::move(reqs[i].status)); + } + return status_to_io_status(std::move(status)); + } + + IOStatus Prefetch(uint64_t offset, size_t n, const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->Prefetch(offset, n)); + } + size_t GetUniqueId(char* id, size_t max_size) const override { + return target_->GetUniqueId(id, max_size); + } + void Hint(AccessPattern pattern) override { + target_->Hint((RandomAccessFile::AccessPattern)pattern); + } + bool use_direct_io() const override { return target_->use_direct_io(); } + size_t GetRequiredBufferAlignment() const override { + return target_->GetRequiredBufferAlignment(); + } + IOStatus InvalidateCache(size_t offset, size_t length) override { + return status_to_io_status(target_->InvalidateCache(offset, length)); + } + + private: + std::unique_ptr target_; +}; + +class LegacyRandomRWFileWrapper : public FSRandomRWFile { + public: + explicit LegacyRandomRWFileWrapper(std::unique_ptr&& target) + : target_(std::move(target)) {} + + bool use_direct_io() const override { return target_->use_direct_io(); } + size_t GetRequiredBufferAlignment() const override { + return target_->GetRequiredBufferAlignment(); + } + IOStatus Write(uint64_t offset, const Slice& data, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->Write(offset, data)); + } + IOStatus Read(uint64_t offset, size_t n, const IOOptions& /*options*/, + Slice* result, char* scratch, + IODebugContext* /*dbg*/) const override { + return status_to_io_status(target_->Read(offset, n, result, scratch)); + } + IOStatus Flush(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->Flush()); + } + IOStatus Sync(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->Sync()); + } + IOStatus Fsync(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->Fsync()); + } + IOStatus Close(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->Close()); + } + + private: + std::unique_ptr target_; +}; + +class LegacyWritableFileWrapper : public FSWritableFile { + public: + explicit LegacyWritableFileWrapper(std::unique_ptr&& _target) + : target_(std::move(_target)) {} + + IOStatus Append(const Slice& data, const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->Append(data)); + } + IOStatus Append(const Slice& data, const IOOptions& /*options*/, + const DataVerificationInfo& /*verification_info*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->Append(data)); + } + IOStatus PositionedAppend(const Slice& data, uint64_t offset, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->PositionedAppend(data, offset)); + } + IOStatus PositionedAppend(const Slice& data, uint64_t offset, + const IOOptions& /*options*/, + const DataVerificationInfo& /*verification_info*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->PositionedAppend(data, offset)); + } + IOStatus Truncate(uint64_t size, const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->Truncate(size)); + } + IOStatus Close(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->Close()); + } + IOStatus Flush(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->Flush()); + } + IOStatus Sync(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->Sync()); + } + IOStatus Fsync(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->Fsync()); + } + bool IsSyncThreadSafe() const override { return target_->IsSyncThreadSafe(); } + + bool use_direct_io() const override { return target_->use_direct_io(); } + + size_t GetRequiredBufferAlignment() const override { + return target_->GetRequiredBufferAlignment(); + } + + void SetWriteLifeTimeHint(Env::WriteLifeTimeHint hint) override { + target_->SetWriteLifeTimeHint(hint); + } + + Env::WriteLifeTimeHint GetWriteLifeTimeHint() override { + return target_->GetWriteLifeTimeHint(); + } + + uint64_t GetFileSize(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return target_->GetFileSize(); + } + + void SetPreallocationBlockSize(size_t size) override { + target_->SetPreallocationBlockSize(size); + } + + void GetPreallocationStatus(size_t* block_size, + size_t* last_allocated_block) override { + target_->GetPreallocationStatus(block_size, last_allocated_block); + } + + size_t GetUniqueId(char* id, size_t max_size) const override { + return target_->GetUniqueId(id, max_size); + } + + IOStatus InvalidateCache(size_t offset, size_t length) override { + return status_to_io_status(target_->InvalidateCache(offset, length)); + } + + IOStatus RangeSync(uint64_t offset, uint64_t nbytes, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->RangeSync(offset, nbytes)); + } + + void PrepareWrite(size_t offset, size_t len, const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + target_->PrepareWrite(offset, len); + } + + IOStatus Allocate(uint64_t offset, uint64_t len, const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->Allocate(offset, len)); + } + + private: + std::unique_ptr target_; +}; + +class LegacyDirectoryWrapper : public FSDirectory { + public: + explicit LegacyDirectoryWrapper(std::unique_ptr&& target) + : target_(std::move(target)) {} + + IOStatus Fsync(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->Fsync()); + } + IOStatus Close(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->Close()); + } + size_t GetUniqueId(char* id, size_t max_size) const override { + return target_->GetUniqueId(id, max_size); + } + + private: + std::unique_ptr target_; +}; + +class LegacyFileSystemWrapper : public FileSystem { + public: + // Initialize an EnvWrapper that delegates all calls to *t + explicit LegacyFileSystemWrapper(Env* t) : target_(t) {} + ~LegacyFileSystemWrapper() override {} + + static const char* kClassName() { return "LegacyFileSystem"; } + const char* Name() const override { return kClassName(); } + + // Return the target to which this Env forwards all calls + Env* target() const { return target_; } + + // The following text is boilerplate that forwards all methods to target() + IOStatus NewSequentialFile(const std::string& f, const FileOptions& file_opts, + std::unique_ptr* r, + IODebugContext* /*dbg*/) override { + std::unique_ptr file; + Status s = target_->NewSequentialFile(f, &file, file_opts); + if (s.ok()) { + r->reset(new LegacySequentialFileWrapper(std::move(file))); + } + return status_to_io_status(std::move(s)); + } + IOStatus NewRandomAccessFile(const std::string& f, + const FileOptions& file_opts, + std::unique_ptr* r, + IODebugContext* /*dbg*/) override { + std::unique_ptr file; + Status s = target_->NewRandomAccessFile(f, &file, file_opts); + if (s.ok()) { + r->reset(new LegacyRandomAccessFileWrapper(std::move(file))); + } + return status_to_io_status(std::move(s)); + } + IOStatus NewWritableFile(const std::string& f, const FileOptions& file_opts, + std::unique_ptr* r, + IODebugContext* /*dbg*/) override { + std::unique_ptr file; + Status s = target_->NewWritableFile(f, &file, file_opts); + if (s.ok()) { + r->reset(new LegacyWritableFileWrapper(std::move(file))); + } + return status_to_io_status(std::move(s)); + } + IOStatus ReopenWritableFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* /*dbg*/) override { + std::unique_ptr file; + Status s = target_->ReopenWritableFile(fname, &file, file_opts); + if (s.ok()) { + result->reset(new LegacyWritableFileWrapper(std::move(file))); + } + return status_to_io_status(std::move(s)); + } + IOStatus ReuseWritableFile(const std::string& fname, + const std::string& old_fname, + const FileOptions& file_opts, + std::unique_ptr* r, + IODebugContext* /*dbg*/) override { + std::unique_ptr file; + Status s = target_->ReuseWritableFile(fname, old_fname, &file, file_opts); + if (s.ok()) { + r->reset(new LegacyWritableFileWrapper(std::move(file))); + } + return status_to_io_status(std::move(s)); + } + IOStatus NewRandomRWFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* /*dbg*/) override { + std::unique_ptr file; + Status s = target_->NewRandomRWFile(fname, &file, file_opts); + if (s.ok()) { + result->reset(new LegacyRandomRWFileWrapper(std::move(file))); + } + return status_to_io_status(std::move(s)); + } + IOStatus NewMemoryMappedFileBuffer( + const std::string& fname, + std::unique_ptr* result) override { + return status_to_io_status( + target_->NewMemoryMappedFileBuffer(fname, result)); + } + IOStatus NewDirectory(const std::string& name, const IOOptions& /*io_opts*/, + std::unique_ptr* result, + IODebugContext* /*dbg*/) override { + std::unique_ptr dir; + Status s = target_->NewDirectory(name, &dir); + if (s.ok()) { + result->reset(new LegacyDirectoryWrapper(std::move(dir))); + } + return status_to_io_status(std::move(s)); + } + IOStatus FileExists(const std::string& f, const IOOptions& /*io_opts*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->FileExists(f)); + } + IOStatus GetChildren(const std::string& dir, const IOOptions& /*io_opts*/, + std::vector* r, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->GetChildren(dir, r)); + } + IOStatus GetChildrenFileAttributes(const std::string& dir, + const IOOptions& /*options*/, + std::vector* result, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->GetChildrenFileAttributes(dir, result)); + } + IOStatus DeleteFile(const std::string& f, const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->DeleteFile(f)); + } + IOStatus Truncate(const std::string& fname, size_t size, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->Truncate(fname, size)); + } + IOStatus CreateDir(const std::string& d, const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->CreateDir(d)); + } + IOStatus CreateDirIfMissing(const std::string& d, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->CreateDirIfMissing(d)); + } + IOStatus DeleteDir(const std::string& d, const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->DeleteDir(d)); + } + IOStatus GetFileSize(const std::string& f, const IOOptions& /*options*/, + uint64_t* s, IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->GetFileSize(f, s)); + } + + IOStatus GetFileModificationTime(const std::string& fname, + const IOOptions& /*options*/, + uint64_t* file_mtime, + IODebugContext* /*dbg*/) override { + return status_to_io_status( + target_->GetFileModificationTime(fname, file_mtime)); + } + + IOStatus GetAbsolutePath(const std::string& db_path, + const IOOptions& /*options*/, + std::string* output_path, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->GetAbsolutePath(db_path, output_path)); + } + + IOStatus RenameFile(const std::string& s, const std::string& t, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->RenameFile(s, t)); + } + + IOStatus LinkFile(const std::string& s, const std::string& t, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->LinkFile(s, t)); + } + + IOStatus NumFileLinks(const std::string& fname, const IOOptions& /*options*/, + uint64_t* count, IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->NumFileLinks(fname, count)); + } + + IOStatus AreFilesSame(const std::string& first, const std::string& second, + const IOOptions& /*options*/, bool* res, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->AreFilesSame(first, second, res)); + } + + IOStatus LockFile(const std::string& f, const IOOptions& /*options*/, + FileLock** l, IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->LockFile(f, l)); + } + + IOStatus UnlockFile(FileLock* l, const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->UnlockFile(l)); + } + + IOStatus GetTestDirectory(const IOOptions& /*options*/, std::string* path, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->GetTestDirectory(path)); + } + IOStatus NewLogger(const std::string& fname, const IOOptions& /*options*/, + std::shared_ptr* result, + IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->NewLogger(fname, result)); + } + + void SanitizeFileOptions(FileOptions* opts) const override { + target_->SanitizeEnvOptions(opts); + } + + FileOptions OptimizeForLogRead( + const FileOptions& file_options) const override { + return target_->OptimizeForLogRead(file_options); + } + FileOptions OptimizeForManifestRead( + const FileOptions& file_options) const override { + return target_->OptimizeForManifestRead(file_options); + } + FileOptions OptimizeForLogWrite(const FileOptions& file_options, + const DBOptions& db_options) const override { + return target_->OptimizeForLogWrite(file_options, db_options); + } + FileOptions OptimizeForManifestWrite( + const FileOptions& file_options) const override { + return target_->OptimizeForManifestWrite(file_options); + } + FileOptions OptimizeForCompactionTableWrite( + const FileOptions& file_options, + const ImmutableDBOptions& immutable_ops) const override { + return target_->OptimizeForCompactionTableWrite(file_options, + immutable_ops); + } + FileOptions OptimizeForCompactionTableRead( + const FileOptions& file_options, + const ImmutableDBOptions& db_options) const override { + return target_->OptimizeForCompactionTableRead(file_options, db_options); + } + FileOptions OptimizeForBlobFileRead( + const FileOptions& file_options, + const ImmutableDBOptions& db_options) const override { + return target_->OptimizeForBlobFileRead(file_options, db_options); + } + +#ifdef GetFreeSpace +#undef GetFreeSpace +#endif + IOStatus GetFreeSpace(const std::string& path, const IOOptions& /*options*/, + uint64_t* diskfree, IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->GetFreeSpace(path, diskfree)); + } + IOStatus IsDirectory(const std::string& path, const IOOptions& /*options*/, + bool* is_dir, IODebugContext* /*dbg*/) override { + return status_to_io_status(target_->IsDirectory(path, is_dir)); + } + + std::string SerializeOptions(const ConfigOptions& /*config_options*/, + const std::string& /*prefix*/) const override { + // We do not want the LegacyFileSystem to appear in the serialized output. + // This clock is an internal class for those who do not implement one and + // would be part of the Env. As such, do not serialize it here. + return ""; + } + private: + Env* target_; +}; +} // end anonymous namespace + +Env::Env() : thread_status_updater_(nullptr) { + file_system_ = std::make_shared(this); + system_clock_ = std::make_shared(this); +} + +Env::Env(const std::shared_ptr& fs) + : thread_status_updater_(nullptr), file_system_(fs) { + system_clock_ = std::make_shared(this); +} + +Env::Env(const std::shared_ptr& fs, + const std::shared_ptr& clock) + : thread_status_updater_(nullptr), file_system_(fs), system_clock_(clock) {} + +Env::~Env() {} + +Status Env::NewLogger(const std::string& fname, + std::shared_ptr* result) { + return NewEnvLogger(fname, this, result); +} + +Status Env::CreateFromString(const ConfigOptions& config_options, + const std::string& value, Env** result) { + Env* base = Env::Default(); + if (value.empty() || base->IsInstanceOf(value)) { + *result = base; + return Status::OK(); + } else { + RegisterSystemEnvs(); + Env* env = *result; + Status s = LoadStaticObject(config_options, value, &env); + if (s.ok()) { + *result = env; + } + return s; + } +} + +Status Env::CreateFromString(const ConfigOptions& config_options, + const std::string& value, Env** result, + std::shared_ptr* guard) { + assert(result); + assert(guard != nullptr); + std::unique_ptr uniq; + + Env* env = *result; + std::string id; + std::unordered_map opt_map; + + Status status = + Customizable::GetOptionsMap(config_options, env, value, &id, &opt_map); + if (!status.ok()) { // GetOptionsMap failed + return status; + } + Env* base = Env::Default(); + if (id.empty() || base->IsInstanceOf(id)) { + env = base; + status = Status::OK(); + } else { + RegisterSystemEnvs(); + // First, try to load the Env as a unique object. + status = config_options.registry->NewObject(id, &env, &uniq); + } + if (config_options.ignore_unsupported_options && status.IsNotSupported()) { + status = Status::OK(); + } else if (status.ok()) { + status = Customizable::ConfigureNewObject(config_options, env, opt_map); + } + if (status.ok()) { + guard->reset(uniq.release()); + *result = env; + } + return status; +} + +Status Env::CreateFromUri(const ConfigOptions& config_options, + const std::string& env_uri, const std::string& fs_uri, + Env** result, std::shared_ptr* guard) { + *result = config_options.env; + if (env_uri.empty() && fs_uri.empty()) { + // Neither specified. Use the default + guard->reset(); + return Status::OK(); + } else if (!env_uri.empty() && !fs_uri.empty()) { + // Both specified. Cannot choose. Return Invalid + return Status::InvalidArgument("cannot specify both fs_uri and env_uri"); + } else if (fs_uri.empty()) { // Only have an ENV URI. Create an Env from it + return CreateFromString(config_options, env_uri, result, guard); + } else { + std::shared_ptr fs; + Status s = FileSystem::CreateFromString(config_options, fs_uri, &fs); + if (s.ok()) { + guard->reset(new CompositeEnvWrapper(*result, fs)); + *result = guard->get(); + } + return s; + } +} + +std::string Env::PriorityToString(Env::Priority priority) { + switch (priority) { + case Env::Priority::BOTTOM: + return "Bottom"; + case Env::Priority::LOW: + return "Low"; + case Env::Priority::HIGH: + return "High"; + case Env::Priority::USER: + return "User"; + case Env::Priority::TOTAL: + assert(false); + } + return "Invalid"; +} + +uint64_t Env::GetThreadID() const { + std::hash hasher; + return hasher(std::this_thread::get_id()); +} + +Status Env::ReuseWritableFile(const std::string& fname, + const std::string& old_fname, + std::unique_ptr* result, + const EnvOptions& options) { + Status s = RenameFile(old_fname, fname); + if (!s.ok()) { + return s; + } + return NewWritableFile(fname, result, options); +} + +Status Env::GetChildrenFileAttributes(const std::string& dir, + std::vector* result) { + assert(result != nullptr); + std::vector child_fnames; + Status s = GetChildren(dir, &child_fnames); + if (!s.ok()) { + return s; + } + result->resize(child_fnames.size()); + size_t result_size = 0; + for (size_t i = 0; i < child_fnames.size(); ++i) { + const std::string path = dir + "/" + child_fnames[i]; + if (!(s = GetFileSize(path, &(*result)[result_size].size_bytes)).ok()) { + if (FileExists(path).IsNotFound()) { + // The file may have been deleted since we listed the directory + continue; + } + return s; + } + (*result)[result_size].name = std::move(child_fnames[i]); + result_size++; + } + result->resize(result_size); + return Status::OK(); +} + +Status Env::GetHostNameString(std::string* result) { + std::array hostname_buf{}; + Status s = GetHostName(hostname_buf.data(), hostname_buf.size()); + if (s.ok()) { + hostname_buf[hostname_buf.size() - 1] = '\0'; + result->assign(hostname_buf.data()); + } + return s; +} + +std::string Env::GenerateUniqueId() { + std::string result; + bool success = port::GenerateRfcUuid(&result); + if (!success) { + // Fall back on our own way of generating a unique ID and adapt it to + // RFC 4122 variant 1 version 4 (a random ID). + // https://en.wikipedia.org/wiki/Universally_unique_identifier + // We already tried GenerateRfcUuid so no need to try it again in + // GenerateRawUniqueId + constexpr bool exclude_port_uuid = true; + uint64_t upper, lower; + GenerateRawUniqueId(&upper, &lower, exclude_port_uuid); + + // Set 4-bit version to 4 + upper = (upper & (~uint64_t{0xf000})) | 0x4000; + // Set unary-encoded variant to 1 (0b10) + lower = (lower & (~(uint64_t{3} << 62))) | (uint64_t{2} << 62); + + // Use 36 character format of RFC 4122 + result.resize(36U); + char* buf = &result[0]; + PutBaseChars<16>(&buf, 8, upper >> 32, /*!uppercase*/ false); + *(buf++) = '-'; + PutBaseChars<16>(&buf, 4, upper >> 16, /*!uppercase*/ false); + *(buf++) = '-'; + PutBaseChars<16>(&buf, 4, upper, /*!uppercase*/ false); + *(buf++) = '-'; + PutBaseChars<16>(&buf, 4, lower >> 48, /*!uppercase*/ false); + *(buf++) = '-'; + PutBaseChars<16>(&buf, 12, lower, /*!uppercase*/ false); + assert(buf == &result[36]); + + // Verify variant 1 version 4 + assert(result[14] == '4'); + assert(result[19] == '8' || result[19] == '9' || result[19] == 'a' || + result[19] == 'b'); + } + return result; +} + +SequentialFile::~SequentialFile() {} + +RandomAccessFile::~RandomAccessFile() {} + +WritableFile::~WritableFile() {} + +MemoryMappedFileBuffer::~MemoryMappedFileBuffer() {} + +Logger::~Logger() {} + +Status Logger::Close() { + if (!closed_) { + closed_ = true; + return CloseImpl(); + } else { + return Status::OK(); + } +} + +Status Logger::CloseImpl() { return Status::NotSupported(); } + +FileLock::~FileLock() {} + +void LogFlush(Logger* info_log) { + if (info_log) { + info_log->Flush(); + } +} + +static void Logv(Logger* info_log, const char* format, va_list ap) { + if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::INFO_LEVEL) { + info_log->Logv(InfoLogLevel::INFO_LEVEL, format, ap); + } +} + +void Log(Logger* info_log, const char* format, ...) { + va_list ap; + va_start(ap, format); + Logv(info_log, format, ap); + va_end(ap); +} + +void Logger::Logv(const InfoLogLevel log_level, const char* format, + va_list ap) { + static const char* kInfoLogLevelNames[5] = {"DEBUG", "INFO", "WARN", "ERROR", + "FATAL"}; + if (log_level < log_level_) { + return; + } + + if (log_level == InfoLogLevel::INFO_LEVEL) { + // Doesn't print log level if it is INFO level. + // This is to avoid unexpected performance regression after we add + // the feature of log level. All the logs before we add the feature + // are INFO level. We don't want to add extra costs to those existing + // logging. + Logv(format, ap); + } else if (log_level == InfoLogLevel::HEADER_LEVEL) { + LogHeader(format, ap); + } else { + char new_format[500]; + snprintf(new_format, sizeof(new_format) - 1, "[%s] %s", + kInfoLogLevelNames[log_level], format); + Logv(new_format, ap); + } + + if (log_level >= InfoLogLevel::WARN_LEVEL && + log_level != InfoLogLevel::HEADER_LEVEL) { + // Log messages with severity of warning or higher should be rare and are + // sometimes followed by an unclean crash. We want to be sure important + // messages are not lost in an application buffer when that happens. + Flush(); + } +} + +static void Logv(const InfoLogLevel log_level, Logger* info_log, + const char* format, va_list ap) { + if (info_log && info_log->GetInfoLogLevel() <= log_level) { + if (log_level == InfoLogLevel::HEADER_LEVEL) { + info_log->LogHeader(format, ap); + } else { + info_log->Logv(log_level, format, ap); + } + } +} + +void Log(const InfoLogLevel log_level, Logger* info_log, const char* format, + ...) { + va_list ap; + va_start(ap, format); + Logv(log_level, info_log, format, ap); + va_end(ap); +} + +static void Headerv(Logger* info_log, const char* format, va_list ap) { + if (info_log) { + info_log->LogHeader(format, ap); + } +} + +void Header(Logger* info_log, const char* format, ...) { + va_list ap; + va_start(ap, format); + Headerv(info_log, format, ap); + va_end(ap); +} + +static void Debugv(Logger* info_log, const char* format, va_list ap) { + if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::DEBUG_LEVEL) { + info_log->Logv(InfoLogLevel::DEBUG_LEVEL, format, ap); + } +} + +void Debug(Logger* info_log, const char* format, ...) { + va_list ap; + va_start(ap, format); + Debugv(info_log, format, ap); + va_end(ap); +} + +static void Infov(Logger* info_log, const char* format, va_list ap) { + if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::INFO_LEVEL) { + info_log->Logv(InfoLogLevel::INFO_LEVEL, format, ap); + } +} + +void Info(Logger* info_log, const char* format, ...) { + va_list ap; + va_start(ap, format); + Infov(info_log, format, ap); + va_end(ap); +} + +static void Warnv(Logger* info_log, const char* format, va_list ap) { + if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::WARN_LEVEL) { + info_log->Logv(InfoLogLevel::WARN_LEVEL, format, ap); + } +} + +void Warn(Logger* info_log, const char* format, ...) { + va_list ap; + va_start(ap, format); + Warnv(info_log, format, ap); + va_end(ap); +} + +static void Errorv(Logger* info_log, const char* format, va_list ap) { + if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::ERROR_LEVEL) { + info_log->Logv(InfoLogLevel::ERROR_LEVEL, format, ap); + } +} + +void Error(Logger* info_log, const char* format, ...) { + va_list ap; + va_start(ap, format); + Errorv(info_log, format, ap); + va_end(ap); +} + +static void Fatalv(Logger* info_log, const char* format, va_list ap) { + if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::FATAL_LEVEL) { + info_log->Logv(InfoLogLevel::FATAL_LEVEL, format, ap); + } +} + +void Fatal(Logger* info_log, const char* format, ...) { + va_list ap; + va_start(ap, format); + Fatalv(info_log, format, ap); + va_end(ap); +} + +void LogFlush(const std::shared_ptr& info_log) { + LogFlush(info_log.get()); +} + +void Log(const InfoLogLevel log_level, const std::shared_ptr& info_log, + const char* format, ...) { + va_list ap; + va_start(ap, format); + Logv(log_level, info_log.get(), format, ap); + va_end(ap); +} + +void Header(const std::shared_ptr& info_log, const char* format, ...) { + va_list ap; + va_start(ap, format); + Headerv(info_log.get(), format, ap); + va_end(ap); +} + +void Debug(const std::shared_ptr& info_log, const char* format, ...) { + va_list ap; + va_start(ap, format); + Debugv(info_log.get(), format, ap); + va_end(ap); +} + +void Info(const std::shared_ptr& info_log, const char* format, ...) { + va_list ap; + va_start(ap, format); + Infov(info_log.get(), format, ap); + va_end(ap); +} + +void Warn(const std::shared_ptr& info_log, const char* format, ...) { + va_list ap; + va_start(ap, format); + Warnv(info_log.get(), format, ap); + va_end(ap); +} + +void Error(const std::shared_ptr& info_log, const char* format, ...) { + va_list ap; + va_start(ap, format); + Errorv(info_log.get(), format, ap); + va_end(ap); +} + +void Fatal(const std::shared_ptr& info_log, const char* format, ...) { + va_list ap; + va_start(ap, format); + Fatalv(info_log.get(), format, ap); + va_end(ap); +} + +void Log(const std::shared_ptr& info_log, const char* format, ...) { + va_list ap; + va_start(ap, format); + Logv(info_log.get(), format, ap); + va_end(ap); +} + +Status WriteStringToFile(Env* env, const Slice& data, const std::string& fname, + bool should_sync) { + const auto& fs = env->GetFileSystem(); + return WriteStringToFile(fs.get(), data, fname, should_sync); +} + +Status ReadFileToString(Env* env, const std::string& fname, std::string* data) { + const auto& fs = env->GetFileSystem(); + return ReadFileToString(fs.get(), fname, data); +} + +namespace { // anonymous namespace + +void AssignEnvOptions(EnvOptions* env_options, const DBOptions& options) { + env_options->use_mmap_reads = options.allow_mmap_reads; + env_options->use_mmap_writes = options.allow_mmap_writes; + env_options->use_direct_reads = options.use_direct_reads; + env_options->set_fd_cloexec = options.is_fd_close_on_exec; + env_options->bytes_per_sync = options.bytes_per_sync; + env_options->compaction_readahead_size = options.compaction_readahead_size; + env_options->random_access_max_buffer_size = + options.random_access_max_buffer_size; + env_options->rate_limiter = options.rate_limiter.get(); + env_options->writable_file_max_buffer_size = + options.writable_file_max_buffer_size; + env_options->allow_fallocate = options.allow_fallocate; + env_options->strict_bytes_per_sync = options.strict_bytes_per_sync; + options.env->SanitizeEnvOptions(env_options); +} + +} // namespace + +EnvOptions Env::OptimizeForLogWrite(const EnvOptions& env_options, + const DBOptions& db_options) const { + EnvOptions optimized_env_options(env_options); + optimized_env_options.bytes_per_sync = db_options.wal_bytes_per_sync; + optimized_env_options.writable_file_max_buffer_size = + db_options.writable_file_max_buffer_size; + return optimized_env_options; +} + +EnvOptions Env::OptimizeForManifestWrite(const EnvOptions& env_options) const { + return env_options; +} + +EnvOptions Env::OptimizeForLogRead(const EnvOptions& env_options) const { + EnvOptions optimized_env_options(env_options); + optimized_env_options.use_direct_reads = false; + return optimized_env_options; +} + +EnvOptions Env::OptimizeForManifestRead(const EnvOptions& env_options) const { + EnvOptions optimized_env_options(env_options); + optimized_env_options.use_direct_reads = false; + return optimized_env_options; +} + +EnvOptions Env::OptimizeForCompactionTableWrite( + const EnvOptions& env_options, const ImmutableDBOptions& db_options) const { + EnvOptions optimized_env_options(env_options); + optimized_env_options.use_direct_writes = + db_options.use_direct_io_for_flush_and_compaction; + return optimized_env_options; +} + +EnvOptions Env::OptimizeForCompactionTableRead( + const EnvOptions& env_options, const ImmutableDBOptions& db_options) const { + EnvOptions optimized_env_options(env_options); + optimized_env_options.use_direct_reads = db_options.use_direct_reads; + return optimized_env_options; +} +EnvOptions Env::OptimizeForBlobFileRead( + const EnvOptions& env_options, const ImmutableDBOptions& db_options) const { + EnvOptions optimized_env_options(env_options); + optimized_env_options.use_direct_reads = db_options.use_direct_reads; + return optimized_env_options; +} + +EnvOptions::EnvOptions(const DBOptions& options) { + AssignEnvOptions(this, options); +} + +EnvOptions::EnvOptions() { + DBOptions options; + AssignEnvOptions(this, options); +} + +Status NewEnvLogger(const std::string& fname, Env* env, + std::shared_ptr* result) { + FileOptions options; + // TODO: Tune the buffer size. + options.writable_file_max_buffer_size = 1024 * 1024; + std::unique_ptr writable_file; + const auto status = env->GetFileSystem()->NewWritableFile( + fname, options, &writable_file, nullptr); + if (!status.ok()) { + return status; + } + + *result = std::make_shared(std::move(writable_file), fname, + options, env); + return Status::OK(); +} + +const std::shared_ptr& Env::GetFileSystem() const { + return file_system_; +} + +const std::shared_ptr& Env::GetSystemClock() const { + return system_clock_; +} +namespace { +static std::unordered_map sc_wrapper_type_info = { + {"target", + OptionTypeInfo::AsCustomSharedPtr( + 0, OptionVerificationType::kByName, OptionTypeFlags::kDontSerialize)}, +}; + +} // namespace +SystemClockWrapper::SystemClockWrapper(const std::shared_ptr& t) + : target_(t) { + RegisterOptions("", &target_, &sc_wrapper_type_info); +} + +Status SystemClockWrapper::PrepareOptions(const ConfigOptions& options) { + if (target_ == nullptr) { + target_ = SystemClock::Default(); + } + return SystemClock::PrepareOptions(options); +} + +std::string SystemClockWrapper::SerializeOptions( + const ConfigOptions& config_options, const std::string& header) const { + auto parent = SystemClock::SerializeOptions(config_options, ""); + if (config_options.IsShallow() || target_ == nullptr || + target_->IsInstanceOf(SystemClock::kDefaultName())) { + return parent; + } else { + std::string result = header; + if (!StartsWith(parent, OptionTypeInfo::kIdPropName())) { + result.append(OptionTypeInfo::kIdPropName()).append("="); + } + result.append(parent); + if (!EndsWith(result, config_options.delimiter)) { + result.append(config_options.delimiter); + } + result.append("target=").append(target_->ToString(config_options)); + return result; + } +} + +static int RegisterBuiltinSystemClocks(ObjectLibrary& library, + const std::string& /*arg*/) { + library.AddFactory( + EmulatedSystemClock::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new EmulatedSystemClock(SystemClock::Default())); + return guard->get(); + }); + size_t num_types; + return static_cast(library.GetFactoryCount(&num_types)); +} + +Status SystemClock::CreateFromString(const ConfigOptions& config_options, + const std::string& value, + std::shared_ptr* result) { + auto clock = SystemClock::Default(); + if (clock->IsInstanceOf(value)) { + *result = clock; + return Status::OK(); + } else { + static std::once_flag once; + std::call_once(once, [&]() { + RegisterBuiltinSystemClocks(*(ObjectLibrary::Default().get()), ""); + }); + return LoadSharedObject(config_options, value, result); + } +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/env/env_basic_test.cc b/librocksdb-sys/rocksdb/env/env_basic_test.cc new file mode 100644 index 0000000..11b0750 --- /dev/null +++ b/librocksdb-sys/rocksdb/env/env_basic_test.cc @@ -0,0 +1,397 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +#include +#include +#include +#include + +#include "env/mock_env.h" +#include "file/file_util.h" +#include "rocksdb/convenience.h" +#include "rocksdb/env.h" +#include "rocksdb/env_encryption.h" +#include "test_util/testharness.h" + +namespace ROCKSDB_NAMESPACE { +namespace { +using CreateEnvFunc = Env*(); + +// These functions are used to create the various environments under which this +// test can execute. These functions are used to allow the test cases to be +// created without the Env being initialized, thereby eliminating a potential +// static initialization fiasco/race condition when attempting to get a +// custom/configured env prior to main being invoked. + +static Env* GetDefaultEnv() { return Env::Default(); } + +static Env* GetMockEnv() { + static std::unique_ptr mock_env(MockEnv::Create(Env::Default())); + return mock_env.get(); +} +static Env* NewTestEncryptedEnv(Env* base, const std::string& provider_id) { + ConfigOptions config_opts; + config_opts.invoke_prepare_options = false; + + std::shared_ptr provider; + EXPECT_OK(EncryptionProvider::CreateFromString(config_opts, provider_id, + &provider)); + return NewEncryptedEnv(base, provider); +} + +static Env* GetCtrEncryptedEnv() { + static std::unique_ptr ctr_encrypt_env( + NewTestEncryptedEnv(Env::Default(), "CTR://test")); + return ctr_encrypt_env.get(); +} + +static Env* GetMemoryEnv() { + static std::unique_ptr mem_env(NewMemEnv(Env::Default())); + return mem_env.get(); +} + +static Env* GetTestEnv() { + static std::shared_ptr env_guard; + static Env* custom_env = nullptr; + if (custom_env == nullptr) { + const char* uri = getenv("TEST_ENV_URI"); + if (uri != nullptr) { + EXPECT_OK(Env::CreateFromUri(ConfigOptions(), uri, "", &custom_env, + &env_guard)); + } + } + EXPECT_NE(custom_env, nullptr); + return custom_env; +} + +static Env* GetTestFS() { + static std::shared_ptr fs_env_guard; + static Env* fs_env = nullptr; + if (fs_env == nullptr) { + const char* uri = getenv("TEST_FS_URI"); + if (uri != nullptr) { + EXPECT_OK( + Env::CreateFromUri(ConfigOptions(), uri, "", &fs_env, &fs_env_guard)); + } + } + EXPECT_NE(fs_env, nullptr); + return fs_env; +} + +} // namespace +class EnvBasicTestWithParam + : public testing::Test, + public ::testing::WithParamInterface { + public: + Env* env_; + const EnvOptions soptions_; + std::string test_dir_; + + EnvBasicTestWithParam() : env_(GetParam()()) { + test_dir_ = test::PerThreadDBPath(env_, "env_basic_test"); + } + + void SetUp() override { ASSERT_OK(env_->CreateDirIfMissing(test_dir_)); } + + void TearDown() override { ASSERT_OK(DestroyDir(env_, test_dir_)); } +}; + +class EnvMoreTestWithParam : public EnvBasicTestWithParam {}; + +INSTANTIATE_TEST_CASE_P(EnvDefault, EnvBasicTestWithParam, + ::testing::Values(&GetDefaultEnv)); +INSTANTIATE_TEST_CASE_P(EnvDefault, EnvMoreTestWithParam, + ::testing::Values(&GetDefaultEnv)); + +INSTANTIATE_TEST_CASE_P(MockEnv, EnvBasicTestWithParam, + ::testing::Values(&GetMockEnv)); + +// next statements run env test against default encryption code. +INSTANTIATE_TEST_CASE_P(EncryptedEnv, EnvBasicTestWithParam, + ::testing::Values(&GetCtrEncryptedEnv)); +INSTANTIATE_TEST_CASE_P(EncryptedEnv, EnvMoreTestWithParam, + ::testing::Values(&GetCtrEncryptedEnv)); + +INSTANTIATE_TEST_CASE_P(MemEnv, EnvBasicTestWithParam, + ::testing::Values(&GetMemoryEnv)); + +namespace { + +// Returns a vector of 0 or 1 Env*, depending whether an Env is registered for +// TEST_ENV_URI. +// +// The purpose of returning an empty vector (instead of nullptr) is that gtest +// ValuesIn() will skip running tests when given an empty collection. +std::vector GetCustomEnvs() { + std::vector res; + const char* uri = getenv("TEST_ENV_URI"); + if (uri != nullptr) { + res.push_back(&GetTestEnv); + } + uri = getenv("TEST_FS_URI"); + if (uri != nullptr) { + res.push_back(&GetTestFS); + } + return res; +} + +} // anonymous namespace + +INSTANTIATE_TEST_CASE_P(CustomEnv, EnvBasicTestWithParam, + ::testing::ValuesIn(GetCustomEnvs())); + +INSTANTIATE_TEST_CASE_P(CustomEnv, EnvMoreTestWithParam, + ::testing::ValuesIn(GetCustomEnvs())); + +TEST_P(EnvBasicTestWithParam, Basics) { + uint64_t file_size; + std::unique_ptr writable_file; + std::vector children; + + // Check that the directory is empty. + ASSERT_EQ(Status::NotFound(), env_->FileExists(test_dir_ + "/non_existent")); + ASSERT_TRUE(!env_->GetFileSize(test_dir_ + "/non_existent", &file_size).ok()); + ASSERT_OK(env_->GetChildren(test_dir_, &children)); + ASSERT_EQ(0U, children.size()); + + // Create a file. + ASSERT_OK(env_->NewWritableFile(test_dir_ + "/f", &writable_file, soptions_)); + ASSERT_OK(writable_file->Close()); + writable_file.reset(); + + // Check that the file exists. + ASSERT_OK(env_->FileExists(test_dir_ + "/f")); + ASSERT_OK(env_->GetFileSize(test_dir_ + "/f", &file_size)); + ASSERT_EQ(0U, file_size); + ASSERT_OK(env_->GetChildren(test_dir_, &children)); + ASSERT_EQ(1U, children.size()); + ASSERT_EQ("f", children[0]); + ASSERT_OK(env_->DeleteFile(test_dir_ + "/f")); + + // Write to the file. + ASSERT_OK( + env_->NewWritableFile(test_dir_ + "/f1", &writable_file, soptions_)); + ASSERT_OK(writable_file->Append("abc")); + ASSERT_OK(writable_file->Close()); + writable_file.reset(); + ASSERT_OK( + env_->NewWritableFile(test_dir_ + "/f2", &writable_file, soptions_)); + ASSERT_OK(writable_file->Close()); + writable_file.reset(); + + // Check for expected size. + ASSERT_OK(env_->GetFileSize(test_dir_ + "/f1", &file_size)); + ASSERT_EQ(3U, file_size); + + // Check that renaming works. + ASSERT_TRUE( + !env_->RenameFile(test_dir_ + "/non_existent", test_dir_ + "/g").ok()); + ASSERT_OK(env_->RenameFile(test_dir_ + "/f1", test_dir_ + "/g")); + ASSERT_EQ(Status::NotFound(), env_->FileExists(test_dir_ + "/f1")); + ASSERT_OK(env_->FileExists(test_dir_ + "/g")); + ASSERT_OK(env_->GetFileSize(test_dir_ + "/g", &file_size)); + ASSERT_EQ(3U, file_size); + + // Check that renaming overwriting works + ASSERT_OK(env_->RenameFile(test_dir_ + "/f2", test_dir_ + "/g")); + ASSERT_OK(env_->GetFileSize(test_dir_ + "/g", &file_size)); + ASSERT_EQ(0U, file_size); + + // Check that opening non-existent file fails. + std::unique_ptr seq_file; + std::unique_ptr rand_file; + ASSERT_TRUE(!env_->NewSequentialFile(test_dir_ + "/non_existent", &seq_file, + soptions_) + .ok()); + ASSERT_TRUE(!seq_file); + ASSERT_NOK(env_->NewRandomAccessFile(test_dir_ + "/non_existent", &rand_file, + soptions_)); + ASSERT_TRUE(!rand_file); + + // Check that deleting works. + ASSERT_NOK(env_->DeleteFile(test_dir_ + "/non_existent")); + ASSERT_OK(env_->DeleteFile(test_dir_ + "/g")); + ASSERT_EQ(Status::NotFound(), env_->FileExists(test_dir_ + "/g")); + ASSERT_OK(env_->GetChildren(test_dir_, &children)); + ASSERT_EQ(0U, children.size()); + Status s = env_->GetChildren(test_dir_ + "/non_existent", &children); + ASSERT_TRUE(s.IsNotFound()); +} + +TEST_P(EnvBasicTestWithParam, ReadWrite) { + std::unique_ptr writable_file; + std::unique_ptr seq_file; + std::unique_ptr rand_file; + Slice result; + char scratch[100]; + + ASSERT_OK(env_->NewWritableFile(test_dir_ + "/f", &writable_file, soptions_)); + ASSERT_OK(writable_file->Append("hello ")); + ASSERT_OK(writable_file->Append("world")); + ASSERT_OK(writable_file->Close()); + writable_file.reset(); + + // Read sequentially. + ASSERT_OK(env_->NewSequentialFile(test_dir_ + "/f", &seq_file, soptions_)); + ASSERT_OK(seq_file->Read(5, &result, scratch)); // Read "hello". + ASSERT_EQ(0, result.compare("hello")); + ASSERT_OK(seq_file->Skip(1)); + ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Read "world". + ASSERT_EQ(0, result.compare("world")); + ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Try reading past EOF. + ASSERT_EQ(0U, result.size()); + ASSERT_OK(seq_file->Skip(100)); // Try to skip past end of file. + ASSERT_OK(seq_file->Read(1000, &result, scratch)); + ASSERT_EQ(0U, result.size()); + + // Random reads. + ASSERT_OK(env_->NewRandomAccessFile(test_dir_ + "/f", &rand_file, soptions_)); + ASSERT_OK(rand_file->Read(6, 5, &result, scratch)); // Read "world". + ASSERT_EQ(0, result.compare("world")); + ASSERT_OK(rand_file->Read(0, 5, &result, scratch)); // Read "hello". + ASSERT_EQ(0, result.compare("hello")); + ASSERT_OK(rand_file->Read(10, 100, &result, scratch)); // Read "d". + ASSERT_EQ(0, result.compare("d")); + + // Too high offset. + ASSERT_TRUE(rand_file->Read(1000, 5, &result, scratch).ok()); +} + +TEST_P(EnvBasicTestWithParam, Misc) { + std::unique_ptr writable_file; + ASSERT_OK(env_->NewWritableFile(test_dir_ + "/b", &writable_file, soptions_)); + + // These are no-ops, but we test they return success. + ASSERT_OK(writable_file->Sync()); + ASSERT_OK(writable_file->Flush()); + ASSERT_OK(writable_file->Close()); + writable_file.reset(); +} + +TEST_P(EnvBasicTestWithParam, LargeWrite) { + const size_t kWriteSize = 300 * 1024; + char* scratch = new char[kWriteSize * 2]; + + std::string write_data; + for (size_t i = 0; i < kWriteSize; ++i) { + write_data.append(1, static_cast(i)); + } + + std::unique_ptr writable_file; + ASSERT_OK(env_->NewWritableFile(test_dir_ + "/f", &writable_file, soptions_)); + ASSERT_OK(writable_file->Append("foo")); + ASSERT_OK(writable_file->Append(write_data)); + ASSERT_OK(writable_file->Close()); + writable_file.reset(); + + std::unique_ptr seq_file; + Slice result; + ASSERT_OK(env_->NewSequentialFile(test_dir_ + "/f", &seq_file, soptions_)); + ASSERT_OK(seq_file->Read(3, &result, scratch)); // Read "foo". + ASSERT_EQ(0, result.compare("foo")); + + size_t read = 0; + std::string read_data; + while (read < kWriteSize) { + ASSERT_OK(seq_file->Read(kWriteSize - read, &result, scratch)); + read_data.append(result.data(), result.size()); + read += result.size(); + } + ASSERT_TRUE(write_data == read_data); + delete[] scratch; +} + +TEST_P(EnvMoreTestWithParam, GetModTime) { + ASSERT_OK(env_->CreateDirIfMissing(test_dir_ + "/dir1")); + uint64_t mtime1 = 0x0; + ASSERT_OK(env_->GetFileModificationTime(test_dir_ + "/dir1", &mtime1)); +} + +TEST_P(EnvMoreTestWithParam, MakeDir) { + ASSERT_OK(env_->CreateDir(test_dir_ + "/j")); + ASSERT_OK(env_->FileExists(test_dir_ + "/j")); + std::vector children; + ASSERT_OK(env_->GetChildren(test_dir_, &children)); + ASSERT_EQ(1U, children.size()); + // fail because file already exists + ASSERT_TRUE(!env_->CreateDir(test_dir_ + "/j").ok()); + ASSERT_OK(env_->CreateDirIfMissing(test_dir_ + "/j")); + ASSERT_OK(env_->DeleteDir(test_dir_ + "/j")); + ASSERT_EQ(Status::NotFound(), env_->FileExists(test_dir_ + "/j")); +} + +TEST_P(EnvMoreTestWithParam, GetChildren) { + // empty folder returns empty vector + std::vector children; + std::vector childAttr; + ASSERT_OK(env_->CreateDirIfMissing(test_dir_)); + ASSERT_OK(env_->GetChildren(test_dir_, &children)); + ASSERT_OK(env_->FileExists(test_dir_)); + ASSERT_OK(env_->GetChildrenFileAttributes(test_dir_, &childAttr)); + ASSERT_EQ(0U, children.size()); + ASSERT_EQ(0U, childAttr.size()); + + // folder with contents returns relative path to test dir + ASSERT_OK(env_->CreateDirIfMissing(test_dir_ + "/niu")); + ASSERT_OK(env_->CreateDirIfMissing(test_dir_ + "/you")); + ASSERT_OK(env_->CreateDirIfMissing(test_dir_ + "/guo")); + ASSERT_OK(env_->GetChildren(test_dir_, &children)); + ASSERT_OK(env_->GetChildrenFileAttributes(test_dir_, &childAttr)); + ASSERT_EQ(3U, children.size()); + ASSERT_EQ(3U, childAttr.size()); + for (auto each : children) { + env_->DeleteDir(test_dir_ + "/" + each).PermitUncheckedError(); + } // necessary for default POSIX env + + // non-exist directory returns IOError + ASSERT_OK(env_->DeleteDir(test_dir_)); + ASSERT_NOK(env_->FileExists(test_dir_)); + ASSERT_NOK(env_->GetChildren(test_dir_, &children)); + ASSERT_NOK(env_->GetChildrenFileAttributes(test_dir_, &childAttr)); + + // if dir is a file, returns IOError + ASSERT_OK(env_->CreateDir(test_dir_)); + std::unique_ptr writable_file; + ASSERT_OK( + env_->NewWritableFile(test_dir_ + "/file", &writable_file, soptions_)); + ASSERT_OK(writable_file->Close()); + writable_file.reset(); + ASSERT_NOK(env_->GetChildren(test_dir_ + "/file", &children)); + ASSERT_EQ(0U, children.size()); +} + +TEST_P(EnvMoreTestWithParam, GetChildrenIgnoresDotAndDotDot) { + auto* env = Env::Default(); + ASSERT_OK(env->CreateDirIfMissing(test_dir_)); + + // Create a single file + std::string path = test_dir_; + const EnvOptions soptions; +#ifdef OS_WIN + path.append("\\test_file"); +#else + path.append("/test_file"); +#endif + std::string data("test data"); + std::unique_ptr file; + ASSERT_OK(env->NewWritableFile(path, &file, soptions)); + ASSERT_OK(file->Append("test data")); + + // get the children + std::vector result; + ASSERT_OK(env->GetChildren(test_dir_, &result)); + + // expect only one file named `test_data`, i.e. no `.` or `..` names + ASSERT_EQ(result.size(), 1); + ASSERT_EQ(result.at(0), "test_file"); +} + +} // namespace ROCKSDB_NAMESPACE +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/env/env_chroot.cc b/librocksdb-sys/rocksdb/env/env_chroot.cc new file mode 100644 index 0000000..5ff32a7 --- /dev/null +++ b/librocksdb-sys/rocksdb/env/env_chroot.cc @@ -0,0 +1,148 @@ +// Copyright (c) 2016-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#if !defined(OS_WIN) + +#include "env/env_chroot.h" + +#include // errno +#include // realpath, free +#include // geteuid + +#include "env/composite_env_wrapper.h" +#include "env/fs_remap.h" +#include "rocksdb/utilities/options_type.h" +#include "util/string_util.h" // errnoStr + +namespace ROCKSDB_NAMESPACE { +namespace { +static std::unordered_map chroot_fs_type_info = { + {"chroot_dir", {0, OptionType::kString}}}; +} // namespace +ChrootFileSystem::ChrootFileSystem(const std::shared_ptr& base, + const std::string& chroot_dir) + : RemapFileSystem(base), chroot_dir_(chroot_dir) { + RegisterOptions("chroot_dir", &chroot_dir_, &chroot_fs_type_info); +} + +Status ChrootFileSystem::PrepareOptions(const ConfigOptions& options) { + Status s = FileSystemWrapper::PrepareOptions(options); + if (!s.ok()) { + return s; + } else if (chroot_dir_.empty()) { + s = Status::InvalidArgument("ChRootFileSystem requires a chroot dir"); + } else { + s = target_->FileExists(chroot_dir_, IOOptions(), nullptr); + } + if (s.ok()) { +#if defined(OS_AIX) + char resolvedName[PATH_MAX]; + char* real_chroot_dir = realpath(chroot_dir_.c_str(), resolvedName); +#else + char* real_chroot_dir = realpath(chroot_dir_.c_str(), nullptr); +#endif + // chroot_dir must exist so realpath() returns non-nullptr. + assert(real_chroot_dir != nullptr); + chroot_dir_ = real_chroot_dir; +#if !defined(OS_AIX) + free(real_chroot_dir); +#endif + } + return s; +} + +IOStatus ChrootFileSystem::GetTestDirectory(const IOOptions& options, + std::string* path, + IODebugContext* dbg) { + // Adapted from PosixEnv's implementation since it doesn't provide a way to + // create directory in the chroot. + char buf[256]; + snprintf(buf, sizeof(buf), "/rocksdbtest-%d", static_cast(geteuid())); + *path = buf; + + // Directory may already exist, so ignore return + return CreateDirIfMissing(*path, options, dbg); +} + +// Returns status and expanded absolute path including the chroot directory. +// Checks whether the provided path breaks out of the chroot. If it returns +// non-OK status, the returned path should not be used. +std::pair ChrootFileSystem::EncodePath( + const std::string& path) { + if (path.empty() || path[0] != '/') { + return {IOStatus::InvalidArgument(path, "Not an absolute path"), ""}; + } + std::pair res; + res.second = chroot_dir_ + path; +#if defined(OS_AIX) + char resolvedName[PATH_MAX]; + char* normalized_path = realpath(res.second.c_str(), resolvedName); +#else + char* normalized_path = realpath(res.second.c_str(), nullptr); +#endif + if (normalized_path == nullptr) { + res.first = IOStatus::NotFound(res.second, errnoStr(errno).c_str()); + } else if (strlen(normalized_path) < chroot_dir_.size() || + strncmp(normalized_path, chroot_dir_.c_str(), + chroot_dir_.size()) != 0) { + res.first = IOStatus::IOError(res.second, + "Attempted to access path outside chroot"); + } else { + res.first = IOStatus::OK(); + } +#if !defined(OS_AIX) + free(normalized_path); +#endif + return res; +} + +// Similar to EncodePath() except assumes the basename in the path hasn't been +// created yet. +std::pair ChrootFileSystem::EncodePathWithNewBasename( + const std::string& path) { + if (path.empty() || path[0] != '/') { + return {IOStatus::InvalidArgument(path, "Not an absolute path"), ""}; + } + // Basename may be followed by trailing slashes + size_t final_idx = path.find_last_not_of('/'); + if (final_idx == std::string::npos) { + // It's only slashes so no basename to extract + return EncodePath(path); + } + + // Pull off the basename temporarily since realname(3) (used by + // EncodePath()) requires a path that exists + size_t base_sep = path.rfind('/', final_idx); + auto status_and_enc_path = EncodePath(path.substr(0, base_sep + 1)); + status_and_enc_path.second.append(path.substr(base_sep + 1)); + return status_and_enc_path; +} + +std::shared_ptr NewChrootFileSystem( + const std::shared_ptr& base, const std::string& chroot_dir) { + auto chroot_fs = std::make_shared(base, chroot_dir); + Status s = chroot_fs->PrepareOptions(ConfigOptions()); + if (s.ok()) { + return chroot_fs; + } else { + return nullptr; + } +} + +Env* NewChrootEnv(Env* base_env, const std::string& chroot_dir) { + if (!base_env->FileExists(chroot_dir).ok()) { + return nullptr; + } + auto chroot_fs = NewChrootFileSystem(base_env->GetFileSystem(), chroot_dir); + if (chroot_fs != nullptr) { + return new CompositeEnvWrapper(base_env, chroot_fs); + } else { + return nullptr; + } +} + +} // namespace ROCKSDB_NAMESPACE + +#endif // !defined(OS_WIN) diff --git a/librocksdb-sys/rocksdb/env/env_chroot.h b/librocksdb-sys/rocksdb/env/env_chroot.h new file mode 100644 index 0000000..9cead15 --- /dev/null +++ b/librocksdb-sys/rocksdb/env/env_chroot.h @@ -0,0 +1,55 @@ +// Copyright (c) 2016-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#if !defined(OS_WIN) + +#include + +#include "env/fs_remap.h" +#include "rocksdb/file_system.h" + +namespace ROCKSDB_NAMESPACE { +class ChrootFileSystem : public RemapFileSystem { + public: + ChrootFileSystem(const std::shared_ptr& base, + const std::string& chroot_dir); + + static const char* kClassName() { return "ChrootFS"; } + const char* Name() const override { return kClassName(); } + + IOStatus GetTestDirectory(const IOOptions& options, std::string* path, + IODebugContext* dbg) override; + + Status PrepareOptions(const ConfigOptions& options) override; + + protected: + // Returns status and expanded absolute path including the chroot directory. + // Checks whether the provided path breaks out of the chroot. If it returns + // non-OK status, the returned path should not be used. + std::pair EncodePath(const std::string& path) override; + + // Similar to EncodePath() except assumes the basename in the path hasn't been + // created yet. + std::pair EncodePathWithNewBasename( + const std::string& path) override; + + private: + std::string chroot_dir_; +}; + +// Returns an Env that translates paths such that the root directory appears to +// be chroot_dir. chroot_dir should refer to an existing directory. +// +// This class has not been fully analyzed for providing strong security +// guarantees. +Env* NewChrootEnv(Env* base_env, const std::string& chroot_dir); +std::shared_ptr NewChrootFileSystem( + const std::shared_ptr& base, const std::string& chroot_dir); + +} // namespace ROCKSDB_NAMESPACE + +#endif // !defined(OS_WIN) diff --git a/librocksdb-sys/rocksdb/env/env_encryption.cc b/librocksdb-sys/rocksdb/env/env_encryption.cc new file mode 100644 index 0000000..80b9793 --- /dev/null +++ b/librocksdb-sys/rocksdb/env/env_encryption.cc @@ -0,0 +1,1187 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + + +#include "rocksdb/env_encryption.h" + +#include +#include +#include +#include + +#include "env/composite_env_wrapper.h" +#include "env/env_encryption_ctr.h" +#include "monitoring/perf_context_imp.h" +#include "rocksdb/convenience.h" +#include "rocksdb/io_status.h" +#include "rocksdb/system_clock.h" +#include "rocksdb/utilities/customizable_util.h" +#include "rocksdb/utilities/options_type.h" +#include "util/aligned_buffer.h" +#include "util/coding.h" +#include "util/random.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { +std::shared_ptr EncryptionProvider::NewCTRProvider( + const std::shared_ptr& cipher) { + return std::make_shared(cipher); +} + +IOStatus EncryptedSequentialFile::Read(size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) { + assert(scratch); + IOStatus io_s = file_->Read(n, options, result, scratch, dbg); + if (!io_s.ok()) { + return io_s; + } + { + PERF_TIMER_GUARD(decrypt_data_nanos); + io_s = status_to_io_status( + stream_->Decrypt(offset_, (char*)result->data(), result->size())); + } + if (io_s.ok()) { + offset_ += result->size(); // We've already read data from disk, so update + // offset_ even if decryption fails. + } + return io_s; +} + +IOStatus EncryptedSequentialFile::Skip(uint64_t n) { + auto status = file_->Skip(n); + if (!status.ok()) { + return status; + } + offset_ += n; + return status; +} + +bool EncryptedSequentialFile::use_direct_io() const { + return file_->use_direct_io(); +} + +size_t EncryptedSequentialFile::GetRequiredBufferAlignment() const { + return file_->GetRequiredBufferAlignment(); +} + +IOStatus EncryptedSequentialFile::InvalidateCache(size_t offset, + size_t length) { + return file_->InvalidateCache(offset + prefixLength_, length); +} + +IOStatus EncryptedSequentialFile::PositionedRead(uint64_t offset, size_t n, + const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) { + assert(scratch); + offset += prefixLength_; // Skip prefix + auto io_s = file_->PositionedRead(offset, n, options, result, scratch, dbg); + if (!io_s.ok()) { + return io_s; + } + offset_ = offset + result->size(); + { + PERF_TIMER_GUARD(decrypt_data_nanos); + io_s = status_to_io_status( + stream_->Decrypt(offset, (char*)result->data(), result->size())); + } + return io_s; +} + +IOStatus EncryptedRandomAccessFile::Read(uint64_t offset, size_t n, + const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const { + assert(scratch); + offset += prefixLength_; + auto io_s = file_->Read(offset, n, options, result, scratch, dbg); + if (!io_s.ok()) { + return io_s; + } + { + PERF_TIMER_GUARD(decrypt_data_nanos); + io_s = status_to_io_status( + stream_->Decrypt(offset, (char*)result->data(), result->size())); + } + return io_s; +} + +IOStatus EncryptedRandomAccessFile::Prefetch(uint64_t offset, size_t n, + const IOOptions& options, + IODebugContext* dbg) { + return file_->Prefetch(offset + prefixLength_, n, options, dbg); +} + +size_t EncryptedRandomAccessFile::GetUniqueId(char* id, size_t max_size) const { + return file_->GetUniqueId(id, max_size); +}; + +void EncryptedRandomAccessFile::Hint(AccessPattern pattern) { + file_->Hint(pattern); +} + +bool EncryptedRandomAccessFile::use_direct_io() const { + return file_->use_direct_io(); +} + +size_t EncryptedRandomAccessFile::GetRequiredBufferAlignment() const { + return file_->GetRequiredBufferAlignment(); +} + +IOStatus EncryptedRandomAccessFile::InvalidateCache(size_t offset, + size_t length) { + return file_->InvalidateCache(offset + prefixLength_, length); +} + +IOStatus EncryptedWritableFile::Append(const Slice& data, + const IOOptions& options, + IODebugContext* dbg) { + AlignedBuffer buf; + Slice dataToAppend(data); + if (data.size() > 0) { + auto offset = file_->GetFileSize(options, dbg); // size including prefix + // Encrypt in cloned buffer + buf.Alignment(GetRequiredBufferAlignment()); + buf.AllocateNewBuffer(data.size()); + // TODO (sagar0): Modify AlignedBuffer.Append to allow doing a memmove + // so that the next two lines can be replaced with buf.Append(). + memmove(buf.BufferStart(), data.data(), data.size()); + buf.Size(data.size()); + IOStatus io_s; + { + PERF_TIMER_GUARD(encrypt_data_nanos); + io_s = status_to_io_status( + stream_->Encrypt(offset, buf.BufferStart(), buf.CurrentSize())); + } + if (!io_s.ok()) { + return io_s; + } + dataToAppend = Slice(buf.BufferStart(), buf.CurrentSize()); + } + return file_->Append(dataToAppend, options, dbg); +} + +IOStatus EncryptedWritableFile::PositionedAppend(const Slice& data, + uint64_t offset, + const IOOptions& options, + IODebugContext* dbg) { + AlignedBuffer buf; + Slice dataToAppend(data); + offset += prefixLength_; + if (data.size() > 0) { + // Encrypt in cloned buffer + buf.Alignment(GetRequiredBufferAlignment()); + buf.AllocateNewBuffer(data.size()); + memmove(buf.BufferStart(), data.data(), data.size()); + buf.Size(data.size()); + IOStatus io_s; + { + PERF_TIMER_GUARD(encrypt_data_nanos); + io_s = status_to_io_status( + stream_->Encrypt(offset, buf.BufferStart(), buf.CurrentSize())); + } + if (!io_s.ok()) { + return io_s; + } + dataToAppend = Slice(buf.BufferStart(), buf.CurrentSize()); + } + return file_->PositionedAppend(dataToAppend, offset, options, dbg); +} + +bool EncryptedWritableFile::use_direct_io() const { + return file_->use_direct_io(); +} + +bool EncryptedWritableFile::IsSyncThreadSafe() const { + return file_->IsSyncThreadSafe(); +} + +size_t EncryptedWritableFile::GetRequiredBufferAlignment() const { + return file_->GetRequiredBufferAlignment(); +} + +uint64_t EncryptedWritableFile::GetFileSize(const IOOptions& options, + IODebugContext* dbg) { + return file_->GetFileSize(options, dbg) - prefixLength_; +} + +IOStatus EncryptedWritableFile::Truncate(uint64_t size, + const IOOptions& options, + IODebugContext* dbg) { + return file_->Truncate(size + prefixLength_, options, dbg); +} + +IOStatus EncryptedWritableFile::InvalidateCache(size_t offset, size_t length) { + return file_->InvalidateCache(offset + prefixLength_, length); +} + +IOStatus EncryptedWritableFile::RangeSync(uint64_t offset, uint64_t nbytes, + const IOOptions& options, + IODebugContext* dbg) { + return file_->RangeSync(offset + prefixLength_, nbytes, options, dbg); +} + +void EncryptedWritableFile::PrepareWrite(size_t offset, size_t len, + const IOOptions& options, + IODebugContext* dbg) { + file_->PrepareWrite(offset + prefixLength_, len, options, dbg); +} + +void EncryptedWritableFile::SetPreallocationBlockSize(size_t size) { + // the size here doesn't need to include prefixLength_, as it's a + // configuration will be use for `PrepareWrite()`. + file_->SetPreallocationBlockSize(size); +} + +void EncryptedWritableFile::GetPreallocationStatus( + size_t* block_size, size_t* last_allocated_block) { + file_->GetPreallocationStatus(block_size, last_allocated_block); +} + +IOStatus EncryptedWritableFile::Allocate(uint64_t offset, uint64_t len, + const IOOptions& options, + IODebugContext* dbg) { + return file_->Allocate(offset + prefixLength_, len, options, dbg); +} + +IOStatus EncryptedWritableFile::Flush(const IOOptions& options, + IODebugContext* dbg) { + return file_->Flush(options, dbg); +} + +IOStatus EncryptedWritableFile::Sync(const IOOptions& options, + IODebugContext* dbg) { + return file_->Sync(options, dbg); +} + +IOStatus EncryptedWritableFile::Close(const IOOptions& options, + IODebugContext* dbg) { + return file_->Close(options, dbg); +} + +bool EncryptedRandomRWFile::use_direct_io() const { + return file_->use_direct_io(); +} + +size_t EncryptedRandomRWFile::GetRequiredBufferAlignment() const { + return file_->GetRequiredBufferAlignment(); +} + +IOStatus EncryptedRandomRWFile::Write(uint64_t offset, const Slice& data, + const IOOptions& options, + IODebugContext* dbg) { + AlignedBuffer buf; + Slice dataToWrite(data); + offset += prefixLength_; + if (data.size() > 0) { + // Encrypt in cloned buffer + buf.Alignment(GetRequiredBufferAlignment()); + buf.AllocateNewBuffer(data.size()); + memmove(buf.BufferStart(), data.data(), data.size()); + buf.Size(data.size()); + IOStatus io_s; + { + PERF_TIMER_GUARD(encrypt_data_nanos); + io_s = status_to_io_status( + stream_->Encrypt(offset, buf.BufferStart(), buf.CurrentSize())); + } + if (!io_s.ok()) { + return io_s; + } + dataToWrite = Slice(buf.BufferStart(), buf.CurrentSize()); + } + return file_->Write(offset, dataToWrite, options, dbg); +} + +IOStatus EncryptedRandomRWFile::Read(uint64_t offset, size_t n, + const IOOptions& options, Slice* result, + char* scratch, IODebugContext* dbg) const { + assert(scratch); + offset += prefixLength_; + auto status = file_->Read(offset, n, options, result, scratch, dbg); + if (!status.ok()) { + return status; + } + { + PERF_TIMER_GUARD(decrypt_data_nanos); + status = status_to_io_status( + stream_->Decrypt(offset, (char*)result->data(), result->size())); + } + return status; +} + +IOStatus EncryptedRandomRWFile::Flush(const IOOptions& options, + IODebugContext* dbg) { + return file_->Flush(options, dbg); +} + +IOStatus EncryptedRandomRWFile::Sync(const IOOptions& options, + IODebugContext* dbg) { + return file_->Sync(options, dbg); +} + +IOStatus EncryptedRandomRWFile::Fsync(const IOOptions& options, + IODebugContext* dbg) { + return file_->Fsync(options, dbg); +} + +IOStatus EncryptedRandomRWFile::Close(const IOOptions& options, + IODebugContext* dbg) { + return file_->Close(options, dbg); +} + +namespace { +static std::unordered_map encrypted_fs_type_info = + { + {"provider", + OptionTypeInfo::AsCustomSharedPtr( + 0 /* No offset, whole struct*/, OptionVerificationType::kByName, + OptionTypeFlags::kNone)}, +}; +// EncryptedFileSystemImpl implements an FileSystemWrapper that adds encryption +// to files stored on disk. +class EncryptedFileSystemImpl : public EncryptedFileSystem { + public: + const char* Name() const override { + return EncryptedFileSystem::kClassName(); + } + // Returns the raw encryption provider that should be used to write the input + // encrypted file. If there is no such provider, NotFound is returned. + IOStatus GetWritableProvider(const std::string& /*fname*/, + EncryptionProvider** result) { + if (provider_) { + *result = provider_.get(); + return IOStatus::OK(); + } else { + *result = nullptr; + return IOStatus::NotFound("No WriteProvider specified"); + } + } + + // Returns the raw encryption provider that should be used to read the input + // encrypted file. If there is no such provider, NotFound is returned. + IOStatus GetReadableProvider(const std::string& /*fname*/, + EncryptionProvider** result) { + if (provider_) { + *result = provider_.get(); + return IOStatus::OK(); + } else { + *result = nullptr; + return IOStatus::NotFound("No Provider specified"); + } + } + + // Creates a CipherStream for the underlying file/name using the options + // If a writable provider is found and encryption is enabled, uses + // this provider to create a cipher stream. + // @param fname Name of the writable file + // @param underlying The underlying "raw" file + // @param options Options for creating the file/cipher + // @param prefix_length Returns the length of the encryption prefix used for + // this file + // @param stream Returns the cipher stream to use for this file if it + // should be encrypted + // @return OK on success, non-OK on failure. + template + IOStatus CreateWritableCipherStream( + const std::string& fname, const std::unique_ptr& underlying, + const FileOptions& options, size_t* prefix_length, + std::unique_ptr* stream, IODebugContext* dbg) { + EncryptionProvider* provider = nullptr; + *prefix_length = 0; + IOStatus status = GetWritableProvider(fname, &provider); + if (!status.ok()) { + return status; + } else if (provider != nullptr) { + // Initialize & write prefix (if needed) + AlignedBuffer buffer; + Slice prefix; + *prefix_length = provider->GetPrefixLength(); + if (*prefix_length > 0) { + // Initialize prefix + buffer.Alignment(underlying->GetRequiredBufferAlignment()); + buffer.AllocateNewBuffer(*prefix_length); + status = status_to_io_status(provider->CreateNewPrefix( + fname, buffer.BufferStart(), *prefix_length)); + if (status.ok()) { + buffer.Size(*prefix_length); + prefix = Slice(buffer.BufferStart(), buffer.CurrentSize()); + // Write prefix + status = underlying->Append(prefix, options.io_options, dbg); + } + if (!status.ok()) { + return status; + } + } + // Create cipher stream + status = status_to_io_status( + provider->CreateCipherStream(fname, options, prefix, stream)); + } + return status; + } + + template + IOStatus CreateWritableEncryptedFile(const std::string& fname, + std::unique_ptr& underlying, + const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) { + // Create cipher stream + std::unique_ptr stream; + size_t prefix_length; + IOStatus status = CreateWritableCipherStream(fname, underlying, options, + &prefix_length, &stream, dbg); + if (status.ok()) { + if (stream) { + result->reset(new EncryptedWritableFile( + std::move(underlying), std::move(stream), prefix_length)); + } else { + result->reset(underlying.release()); + } + } + return status; + } + + // Creates a CipherStream for the underlying file/name using the options + // If a writable provider is found and encryption is enabled, uses + // this provider to create a cipher stream. + // @param fname Name of the writable file + // @param underlying The underlying "raw" file + // @param options Options for creating the file/cipher + // @param prefix_length Returns the length of the encryption prefix used for + // this file + // @param stream Returns the cipher stream to use for this file if it + // should be encrypted + // @return OK on success, non-OK on failure. + template + IOStatus CreateRandomWriteCipherStream( + const std::string& fname, const std::unique_ptr& underlying, + const FileOptions& options, size_t* prefix_length, + std::unique_ptr* stream, IODebugContext* dbg) { + EncryptionProvider* provider = nullptr; + *prefix_length = 0; + IOStatus io_s = GetWritableProvider(fname, &provider); + if (!io_s.ok()) { + return io_s; + } else if (provider != nullptr) { + // Initialize & write prefix (if needed) + AlignedBuffer buffer; + Slice prefix; + *prefix_length = provider->GetPrefixLength(); + if (*prefix_length > 0) { + // Initialize prefix + buffer.Alignment(underlying->GetRequiredBufferAlignment()); + buffer.AllocateNewBuffer(*prefix_length); + io_s = status_to_io_status(provider->CreateNewPrefix( + fname, buffer.BufferStart(), *prefix_length)); + if (io_s.ok()) { + buffer.Size(*prefix_length); + prefix = Slice(buffer.BufferStart(), buffer.CurrentSize()); + // Write prefix + io_s = underlying->Write(0, prefix, options.io_options, dbg); + } + if (!io_s.ok()) { + return io_s; + } + } + // Create cipher stream + io_s = status_to_io_status( + provider->CreateCipherStream(fname, options, prefix, stream)); + } + return io_s; + } + + // Creates a CipherStream for the underlying file/name using the options + // If a readable provider is found and the file is encrypted, uses + // this provider to create a cipher stream. + // @param fname Name of the writable file + // @param underlying The underlying "raw" file + // @param options Options for creating the file/cipher + // @param prefix_length Returns the length of the encryption prefix used for + // this file + // @param stream Returns the cipher stream to use for this file if it + // is encrypted + // @return OK on success, non-OK on failure. + template + IOStatus CreateSequentialCipherStream( + const std::string& fname, const std::unique_ptr& underlying, + const FileOptions& options, size_t* prefix_length, + std::unique_ptr* stream, IODebugContext* dbg) { + // Read prefix (if needed) + AlignedBuffer buffer; + Slice prefix; + *prefix_length = provider_->GetPrefixLength(); + if (*prefix_length > 0) { + // Read prefix + buffer.Alignment(underlying->GetRequiredBufferAlignment()); + buffer.AllocateNewBuffer(*prefix_length); + IOStatus status = underlying->Read(*prefix_length, options.io_options, + &prefix, buffer.BufferStart(), dbg); + if (!status.ok()) { + return status; + } + buffer.Size(*prefix_length); + } + return status_to_io_status( + provider_->CreateCipherStream(fname, options, prefix, stream)); + } + + // Creates a CipherStream for the underlying file/name using the options + // If a readable provider is found and the file is encrypted, uses + // this provider to create a cipher stream. + // @param fname Name of the writable file + // @param underlying The underlying "raw" file + // @param options Options for creating the file/cipher + // @param prefix_length Returns the length of the encryption prefix used for + // this file + // @param stream Returns the cipher stream to use for this file if it + // is encrypted + // @return OK on success, non-OK on failure. + template + IOStatus CreateRandomReadCipherStream( + const std::string& fname, const std::unique_ptr& underlying, + const FileOptions& options, size_t* prefix_length, + std::unique_ptr* stream, IODebugContext* dbg) { + // Read prefix (if needed) + AlignedBuffer buffer; + Slice prefix; + *prefix_length = provider_->GetPrefixLength(); + if (*prefix_length > 0) { + // Read prefix + buffer.Alignment(underlying->GetRequiredBufferAlignment()); + buffer.AllocateNewBuffer(*prefix_length); + IOStatus status = underlying->Read(0, *prefix_length, options.io_options, + &prefix, buffer.BufferStart(), dbg); + if (!status.ok()) { + return status; + } + buffer.Size(*prefix_length); + } + return status_to_io_status( + provider_->CreateCipherStream(fname, options, prefix, stream)); + } + + public: + EncryptedFileSystemImpl(const std::shared_ptr& base, + const std::shared_ptr& provider) + : EncryptedFileSystem(base) { + provider_ = provider; + RegisterOptions("EncryptionProvider", &provider_, &encrypted_fs_type_info); + } + + Status AddCipher(const std::string& descriptor, const char* cipher, + size_t len, bool for_write) override { + return provider_->AddCipher(descriptor, cipher, len, for_write); + } + + IOStatus NewSequentialFile(const std::string& fname, + const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override { + result->reset(); + if (options.use_mmap_reads) { + return IOStatus::InvalidArgument(); + } + // Open file using underlying Env implementation + std::unique_ptr underlying; + auto status = + FileSystemWrapper::NewSequentialFile(fname, options, &underlying, dbg); + if (!status.ok()) { + return status; + } + uint64_t file_size; + status = FileSystemWrapper::GetFileSize(fname, options.io_options, + &file_size, dbg); + if (!status.ok()) { + return status; + } + if (!file_size) { + *result = std::move(underlying); + return status; + } + // Create cipher stream + std::unique_ptr stream; + size_t prefix_length; + status = CreateSequentialCipherStream(fname, underlying, options, + &prefix_length, &stream, dbg); + if (status.ok()) { + result->reset(new EncryptedSequentialFile( + std::move(underlying), std::move(stream), prefix_length)); + } + return status; + } + + IOStatus NewRandomAccessFile(const std::string& fname, + const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override { + result->reset(); + if (options.use_mmap_reads) { + return IOStatus::InvalidArgument(); + } + // Open file using underlying Env implementation + std::unique_ptr underlying; + auto status = FileSystemWrapper::NewRandomAccessFile(fname, options, + &underlying, dbg); + if (!status.ok()) { + return status; + } + std::unique_ptr stream; + size_t prefix_length; + status = CreateRandomReadCipherStream(fname, underlying, options, + &prefix_length, &stream, dbg); + if (status.ok()) { + if (stream) { + result->reset(new EncryptedRandomAccessFile( + std::move(underlying), std::move(stream), prefix_length)); + } else { + result->reset(underlying.release()); + } + } + return status; + } + + IOStatus NewWritableFile(const std::string& fname, const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override { + result->reset(); + if (options.use_mmap_writes) { + return IOStatus::InvalidArgument(); + } + // Open file using underlying Env implementation + std::unique_ptr underlying; + IOStatus status = + FileSystemWrapper::NewWritableFile(fname, options, &underlying, dbg); + if (!status.ok()) { + return status; + } + return CreateWritableEncryptedFile(fname, underlying, options, result, dbg); + } + + IOStatus ReopenWritableFile(const std::string& fname, + const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override { + result->reset(); + if (options.use_mmap_writes) { + return IOStatus::InvalidArgument(); + } + // Open file using underlying Env implementation + std::unique_ptr underlying; + IOStatus status = + FileSystemWrapper::ReopenWritableFile(fname, options, &underlying, dbg); + if (!status.ok()) { + return status; + } + return CreateWritableEncryptedFile(fname, underlying, options, result, dbg); + } + + IOStatus ReuseWritableFile(const std::string& fname, + const std::string& old_fname, + const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override { + result->reset(); + if (options.use_mmap_writes) { + return IOStatus::InvalidArgument(); + } + // Open file using underlying Env implementation + std::unique_ptr underlying; + auto status = FileSystemWrapper::ReuseWritableFile( + fname, old_fname, options, &underlying, dbg); + if (!status.ok()) { + return status; + } + return CreateWritableEncryptedFile(fname, underlying, options, result, dbg); + } + + IOStatus NewRandomRWFile(const std::string& fname, const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override { + result->reset(); + if (options.use_mmap_reads || options.use_mmap_writes) { + return IOStatus::InvalidArgument(); + } + // Check file exists + bool isNewFile = !FileExists(fname, options.io_options, dbg).ok(); + + // Open file using underlying Env implementation + std::unique_ptr underlying; + auto status = + FileSystemWrapper::NewRandomRWFile(fname, options, &underlying, dbg); + if (!status.ok()) { + return status; + } + // Create cipher stream + std::unique_ptr stream; + size_t prefix_length = 0; + if (!isNewFile) { + // File already exists, read prefix + status = CreateRandomReadCipherStream(fname, underlying, options, + &prefix_length, &stream, dbg); + } else { + status = CreateRandomWriteCipherStream(fname, underlying, options, + &prefix_length, &stream, dbg); + } + if (status.ok()) { + if (stream) { + result->reset(new EncryptedRandomRWFile( + std::move(underlying), std::move(stream), prefix_length)); + } else { + result->reset(underlying.release()); + } + } + return status; + } + + IOStatus GetChildrenFileAttributes(const std::string& dir, + const IOOptions& options, + std::vector* result, + IODebugContext* dbg) override { + auto status = + FileSystemWrapper::GetChildrenFileAttributes(dir, options, result, dbg); + if (!status.ok()) { + return status; + } + for (auto it = std::begin(*result); it != std::end(*result); ++it) { + // assert(it->size_bytes >= prefixLength); + // breaks env_basic_test when called on directory containing + // directories + // which makes subtraction of prefixLength worrisome since + // FileAttributes does not identify directories + EncryptionProvider* provider; + status = GetReadableProvider(it->name, &provider); + if (!status.ok()) { + return status; + } else if (provider != nullptr) { + it->size_bytes -= provider->GetPrefixLength(); + } + } + return IOStatus::OK(); + } + + IOStatus GetFileSize(const std::string& fname, const IOOptions& options, + uint64_t* file_size, IODebugContext* dbg) override { + auto status = + FileSystemWrapper::GetFileSize(fname, options, file_size, dbg); + if (!status.ok() || !(*file_size)) { + return status; + } + EncryptionProvider* provider; + status = GetReadableProvider(fname, &provider); + if (provider != nullptr && status.ok()) { + size_t prefixLength = provider->GetPrefixLength(); + assert(*file_size >= prefixLength); + *file_size -= prefixLength; + } + return status; + } + + private: + std::shared_ptr provider_; +}; +} // namespace + +Status NewEncryptedFileSystemImpl( + const std::shared_ptr& base, + const std::shared_ptr& provider, + std::unique_ptr* result) { + result->reset(new EncryptedFileSystemImpl(base, provider)); + return Status::OK(); +} + +std::shared_ptr NewEncryptedFS( + const std::shared_ptr& base, + const std::shared_ptr& provider) { + std::unique_ptr efs; + Status s = NewEncryptedFileSystemImpl(base, provider, &efs); + if (s.ok()) { + s = efs->PrepareOptions(ConfigOptions()); + } + if (s.ok()) { + std::shared_ptr result(efs.release()); + return result; + } else { + return nullptr; + } +} + +Env* NewEncryptedEnv(Env* base_env, + const std::shared_ptr& provider) { + return new CompositeEnvWrapper( + base_env, NewEncryptedFS(base_env->GetFileSystem(), provider)); +} + +Status BlockAccessCipherStream::Encrypt(uint64_t fileOffset, char* data, + size_t dataSize) { + // Calculate block index + auto blockSize = BlockSize(); + uint64_t blockIndex = fileOffset / blockSize; + size_t blockOffset = fileOffset % blockSize; + std::unique_ptr blockBuffer; + + std::string scratch; + AllocateScratch(scratch); + + // Encrypt individual blocks. + while (1) { + char* block = data; + size_t n = std::min(dataSize, blockSize - blockOffset); + if (n != blockSize) { + // We're not encrypting a full block. + // Copy data to blockBuffer + if (!blockBuffer) { + // Allocate buffer + blockBuffer = std::unique_ptr(new char[blockSize]); + } + block = blockBuffer.get(); + // Copy plain data to block buffer + memmove(block + blockOffset, data, n); + } + auto status = EncryptBlock(blockIndex, block, (char*)scratch.data()); + if (!status.ok()) { + return status; + } + if (block != data) { + // Copy encrypted data back to `data`. + memmove(data, block + blockOffset, n); + } + dataSize -= n; + if (dataSize == 0) { + return Status::OK(); + } + data += n; + blockOffset = 0; + blockIndex++; + } +} + +Status BlockAccessCipherStream::Decrypt(uint64_t fileOffset, char* data, + size_t dataSize) { + // Calculate block index + auto blockSize = BlockSize(); + uint64_t blockIndex = fileOffset / blockSize; + size_t blockOffset = fileOffset % blockSize; + std::unique_ptr blockBuffer; + + std::string scratch; + AllocateScratch(scratch); + + // Decrypt individual blocks. + while (1) { + char* block = data; + size_t n = std::min(dataSize, blockSize - blockOffset); + if (n != blockSize) { + // We're not decrypting a full block. + // Copy data to blockBuffer + if (!blockBuffer) { + // Allocate buffer + blockBuffer = std::unique_ptr(new char[blockSize]); + } + block = blockBuffer.get(); + // Copy encrypted data to block buffer + memmove(block + blockOffset, data, n); + } + auto status = DecryptBlock(blockIndex, block, (char*)scratch.data()); + if (!status.ok()) { + return status; + } + if (block != data) { + // Copy decrypted data back to `data`. + memmove(data, block + blockOffset, n); + } + + // Simply decrementing dataSize by n could cause it to underflow, + // which will very likely make it read over the original bounds later + assert(dataSize >= n); + if (dataSize < n) { + return Status::Corruption("Cannot decrypt data at given offset"); + } + + dataSize -= n; + if (dataSize == 0) { + return Status::OK(); + } + data += n; + blockOffset = 0; + blockIndex++; + } +} + +namespace { +static std::unordered_map + rot13_block_cipher_type_info = { + {"block_size", + {0 /* No offset, whole struct*/, OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, +}; + +// Implements a BlockCipher using ROT13. +// +// Note: This is a sample implementation of BlockCipher, +// it is NOT considered safe and should NOT be used in production. +class ROT13BlockCipher : public BlockCipher { + private: + size_t blockSize_; + + public: + explicit ROT13BlockCipher(size_t blockSize) : blockSize_(blockSize) { + RegisterOptions("ROT13BlockCipherOptions", &blockSize_, + &rot13_block_cipher_type_info); + } + + static const char* kClassName() { return "ROT13"; } + const char* Name() const override { return kClassName(); } + + size_t BlockSize() override { return blockSize_; } + Status Encrypt(char* data) override { + for (size_t i = 0; i < blockSize_; ++i) { + data[i] += 13; + } + return Status::OK(); + } + Status Decrypt(char* data) override { return Encrypt(data); } +}; + +static const std::unordered_map + ctr_encryption_provider_type_info = { + {"cipher", + OptionTypeInfo::AsCustomSharedPtr( + 0 /* No offset, whole struct*/, OptionVerificationType::kByName, + OptionTypeFlags::kNone)}, +}; +} // anonymous namespace + +void CTRCipherStream::AllocateScratch(std::string& scratch) { + auto blockSize = cipher_->BlockSize(); + scratch.reserve(blockSize); +} + +Status CTRCipherStream::EncryptBlock(uint64_t blockIndex, char* data, + char* scratch) { + // Create nonce + counter + auto blockSize = cipher_->BlockSize(); + memmove(scratch, iv_.data(), blockSize); + EncodeFixed64(scratch, blockIndex + initialCounter_); + + // Encrypt nonce + counter + auto status = cipher_->Encrypt(scratch); + if (!status.ok()) { + return status; + } + + // XOR data with ciphertext. + for (size_t i = 0; i < blockSize; i++) { + data[i] = data[i] ^ scratch[i]; + } + return Status::OK(); +} + +Status CTRCipherStream::DecryptBlock(uint64_t blockIndex, char* data, + char* scratch) { + // For CTR decryption & encryption are the same + return EncryptBlock(blockIndex, data, scratch); +} + +CTREncryptionProvider::CTREncryptionProvider( + const std::shared_ptr& c) + : cipher_(c) { + RegisterOptions("Cipher", &cipher_, &ctr_encryption_provider_type_info); +} + +bool CTREncryptionProvider::IsInstanceOf(const std::string& name) const { + // Special case for test purposes. + if (name == "1://test" && cipher_ != nullptr) { + return cipher_->IsInstanceOf(ROT13BlockCipher::kClassName()); + } else { + return EncryptionProvider::IsInstanceOf(name); + } +} + +size_t CTREncryptionProvider::GetPrefixLength() const { + return defaultPrefixLength; +} + +Status CTREncryptionProvider::AddCipher(const std::string& /*descriptor*/, + const char* cipher, size_t len, + bool /*for_write*/) { + if (cipher_) { + return Status::NotSupported("Cannot add keys to CTREncryptionProvider"); + } else if (strcmp(ROT13BlockCipher::kClassName(), cipher) == 0) { + cipher_.reset(new ROT13BlockCipher(len)); + return Status::OK(); + } else { + return BlockCipher::CreateFromString(ConfigOptions(), std::string(cipher), + &cipher_); + } +} + +// decodeCTRParameters decodes the initial counter & IV from the given +// (plain text) prefix. +static void decodeCTRParameters(const char* prefix, size_t blockSize, + uint64_t& initialCounter, Slice& iv) { + // First block contains 64-bit initial counter + initialCounter = DecodeFixed64(prefix); + // Second block contains IV + iv = Slice(prefix + blockSize, blockSize); +} + +Status CTREncryptionProvider::CreateNewPrefix(const std::string& /*fname*/, + char* prefix, + size_t prefixLength) const { + if (!cipher_) { + return Status::InvalidArgument("Encryption Cipher is missing"); + } + // Create & seed rnd. + Random rnd((uint32_t)SystemClock::Default()->NowMicros()); + // Fill entire prefix block with random values. + for (size_t i = 0; i < prefixLength; i++) { + prefix[i] = rnd.Uniform(256) & 0xFF; + } + // Take random data to extract initial counter & IV + auto blockSize = cipher_->BlockSize(); + uint64_t initialCounter; + Slice prefixIV; + decodeCTRParameters(prefix, blockSize, initialCounter, prefixIV); + + // Now populate the rest of the prefix, starting from the third block. + PopulateSecretPrefixPart(prefix + (2 * blockSize), + prefixLength - (2 * blockSize), blockSize); + + // Encrypt the prefix, starting from block 2 (leave block 0, 1 with initial + // counter & IV unencrypted) + CTRCipherStream cipherStream(cipher_, prefixIV.data(), initialCounter); + Status status; + { + PERF_TIMER_GUARD(encrypt_data_nanos); + status = cipherStream.Encrypt(0, prefix + (2 * blockSize), + prefixLength - (2 * blockSize)); + } + + return status; +} + +// PopulateSecretPrefixPart initializes the data into a new prefix block +// in plain text. +// Returns the amount of space (starting from the start of the prefix) +// that has been initialized. +size_t CTREncryptionProvider::PopulateSecretPrefixPart( + char* /*prefix*/, size_t /*prefixLength*/, size_t /*blockSize*/) const { + // Nothing to do here, put in custom data in override when needed. + return 0; +} + +Status CTREncryptionProvider::CreateCipherStream( + const std::string& fname, const EnvOptions& options, Slice& prefix, + std::unique_ptr* result) { + if (!cipher_) { + return Status::InvalidArgument("Encryption Cipher is missing"); + } + // Read plain text part of prefix. + auto blockSize = cipher_->BlockSize(); + uint64_t initialCounter; + Slice iv; + decodeCTRParameters(prefix.data(), blockSize, initialCounter, iv); + + // If the prefix is smaller than twice the block size, we would below read a + // very large chunk of the file (and very likely read over the bounds) + assert(prefix.size() >= 2 * blockSize); + if (prefix.size() < 2 * blockSize) { + return Status::Corruption("Unable to read from file " + fname + + ": read attempt would read beyond file bounds"); + } + + // Decrypt the encrypted part of the prefix, starting from block 2 (block 0, 1 + // with initial counter & IV are unencrypted) + CTRCipherStream cipherStream(cipher_, iv.data(), initialCounter); + Status status; + { + PERF_TIMER_GUARD(decrypt_data_nanos); + status = cipherStream.Decrypt(0, (char*)prefix.data() + (2 * blockSize), + prefix.size() - (2 * blockSize)); + } + if (!status.ok()) { + return status; + } + + // Create cipher stream + return CreateCipherStreamFromPrefix(fname, options, initialCounter, iv, + prefix, result); +} + +// CreateCipherStreamFromPrefix creates a block access cipher stream for a file +// given name and options. The given prefix is already decrypted. +Status CTREncryptionProvider::CreateCipherStreamFromPrefix( + const std::string& /*fname*/, const EnvOptions& /*options*/, + uint64_t initialCounter, const Slice& iv, const Slice& /*prefix*/, + std::unique_ptr* result) { + (*result) = std::unique_ptr( + new CTRCipherStream(cipher_, iv.data(), initialCounter)); + return Status::OK(); +} + +// namespace { +// static void RegisterEncryptionBuiltins() { +// static std::once_flag once; +// std::call_once(once, [&]() { +// auto lib = ObjectRegistry::Default()->AddLibrary("encryption"); +// // Match "CTR" or "CTR://test" +// lib->AddFactory( +// ObjectLibrary::PatternEntry(CTREncryptionProvider::kClassName(), true) +// .AddSuffix("://test"), +// [](const std::string& uri, std::unique_ptr* guard, +// std::string* /*errmsg*/) { +// if (EndsWith(uri, "://test")) { +// std::shared_ptr cipher = +// std::make_shared(32); +// guard->reset(new CTREncryptionProvider(cipher)); +// } else { +// guard->reset(new CTREncryptionProvider()); +// } +// return guard->get(); +// }); + +// lib->AddFactory( +// "1://test", [](const std::string& /*uri*/, +// std::unique_ptr* guard, +// std::string* /*errmsg*/) { +// std::shared_ptr cipher = +// std::make_shared(32); +// guard->reset(new CTREncryptionProvider(cipher)); +// return guard->get(); +// }); + +// // Match "ROT13" or "ROT13:[0-9]+" +// lib->AddFactory( +// ObjectLibrary::PatternEntry(ROT13BlockCipher::kClassName(), true) +// .AddNumber(":"), +// [](const std::string& uri, std::unique_ptr* guard, +// std::string* /* errmsg */) { +// size_t colon = uri.find(':'); +// if (colon != std::string::npos) { +// size_t block_size = ParseSizeT(uri.substr(colon + 1)); +// guard->reset(new ROT13BlockCipher(block_size)); +// } else { +// guard->reset(new ROT13BlockCipher(32)); +// } + +// return guard->get(); +// }); +// }); +// } +// } // namespace + +Status BlockCipher::CreateFromString(const ConfigOptions& config_options, + const std::string& value, + std::shared_ptr* result) { + //RegisterEncryptionBuiltins(); + return LoadSharedObject(config_options, value, result); +} + + + + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/env/env_encryption_ctr.h b/librocksdb-sys/rocksdb/env/env_encryption_ctr.h new file mode 100644 index 0000000..b4342f7 --- /dev/null +++ b/librocksdb-sys/rocksdb/env/env_encryption_ctr.h @@ -0,0 +1,97 @@ +// Copyright (c) 2016-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + + +#include "rocksdb/env_encryption.h" + +namespace ROCKSDB_NAMESPACE { +// CTRCipherStream implements BlockAccessCipherStream using an +// Counter operations mode. +// See https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation +// +// Note: This is a possible implementation of BlockAccessCipherStream, +// it is considered suitable for use. +class CTRCipherStream final : public BlockAccessCipherStream { + private: + std::shared_ptr cipher_; + std::string iv_; + uint64_t initialCounter_; + + public: + CTRCipherStream(const std::shared_ptr& c, const char* iv, + uint64_t initialCounter) + : cipher_(c), iv_(iv, c->BlockSize()), initialCounter_(initialCounter){}; + virtual ~CTRCipherStream(){}; + + size_t BlockSize() override { return cipher_->BlockSize(); } + + protected: + void AllocateScratch(std::string&) override; + + Status EncryptBlock(uint64_t blockIndex, char* data, char* scratch) override; + + Status DecryptBlock(uint64_t blockIndex, char* data, char* scratch) override; +}; + +// This encryption provider uses a CTR cipher stream, with a given block cipher +// and IV. +// +// Note: This is a possible implementation of EncryptionProvider, +// it is considered suitable for use, provided a safe BlockCipher is used. +class CTREncryptionProvider : public EncryptionProvider { + private: + std::shared_ptr cipher_; + + protected: + // For optimal performance when using direct IO, the prefix length should be a + // multiple of the page size. This size is to ensure the first real data byte + // is placed at largest known alignment point for direct io. + const static size_t defaultPrefixLength = 4096; + + public: + explicit CTREncryptionProvider( + const std::shared_ptr& c = nullptr); + virtual ~CTREncryptionProvider() {} + + static const char* kClassName() { return "CTR"; } + const char* Name() const override { return kClassName(); } + bool IsInstanceOf(const std::string& name) const override; + size_t GetPrefixLength() const override; + Status CreateNewPrefix(const std::string& fname, char* prefix, + size_t prefixLength) const override; + Status CreateCipherStream( + const std::string& fname, const EnvOptions& options, Slice& prefix, + std::unique_ptr* result) override; + + Status AddCipher(const std::string& descriptor, const char* /*cipher*/, + size_t /*len*/, bool /*for_write*/) override; + + protected: + // PopulateSecretPrefixPart initializes the data into a new prefix block + // that will be encrypted. This function will store the data in plain text. + // It will be encrypted later (before written to disk). + // Returns the amount of space (starting from the start of the prefix) + // that has been initialized. + virtual size_t PopulateSecretPrefixPart(char* prefix, size_t prefixLength, + size_t blockSize) const; + + // CreateCipherStreamFromPrefix creates a block access cipher stream for a + // file given + // given name and options. The given prefix is already decrypted. + virtual Status CreateCipherStreamFromPrefix( + const std::string& fname, const EnvOptions& options, + uint64_t initialCounter, const Slice& iv, const Slice& prefix, + std::unique_ptr* result); +}; + +Status NewEncryptedFileSystemImpl( + const std::shared_ptr& base_fs, + const std::shared_ptr& provider, + std::unique_ptr* fs); + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/env/env_posix.cc b/librocksdb-sys/rocksdb/env/env_posix.cc new file mode 100644 index 0000000..ae2f903 --- /dev/null +++ b/librocksdb-sys/rocksdb/env/env_posix.cc @@ -0,0 +1,528 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors + +#include "port/lang.h" +#if !defined(OS_WIN) + +#include +#ifndef ROCKSDB_NO_DYNAMIC_EXTENSION +#include +#endif +#include +#include + +#if defined(ROCKSDB_IOURING_PRESENT) +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#if defined(OS_LINUX) || defined(OS_SOLARIS) || defined(OS_ANDROID) +#include +#endif +#include +#include +#include +#if defined(ROCKSDB_IOURING_PRESENT) +#include +#endif +#include +#include + +#include +// Get nano time includes +#if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_GNU_KFREEBSD) +#elif defined(__MACH__) +#include +#include +#include +#else +#include +#endif +#include +#include +#include + +#include "env/composite_env_wrapper.h" +#include "env/io_posix.h" +#include "monitoring/iostats_context_imp.h" +#include "monitoring/thread_status_updater.h" +#include "port/port.h" +#include "port/sys_time.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "rocksdb/system_clock.h" +#include "test_util/sync_point.h" +#include "util/coding.h" +#include "util/compression_context_cache.h" +#include "util/random.h" +#include "util/string_util.h" +#include "util/thread_local.h" +#include "util/threadpool_imp.h" + +#if !defined(TMPFS_MAGIC) +#define TMPFS_MAGIC 0x01021994 +#endif +#if !defined(XFS_SUPER_MAGIC) +#define XFS_SUPER_MAGIC 0x58465342 +#endif +#if !defined(EXT4_SUPER_MAGIC) +#define EXT4_SUPER_MAGIC 0xEF53 +#endif + +namespace ROCKSDB_NAMESPACE { +#if defined(OS_WIN) +static const std::string kSharedLibExt = ".dll"; +static const char kPathSeparator = ';'; +#else +static const char kPathSeparator = ':'; +#if defined(OS_MACOSX) +static const std::string kSharedLibExt = ".dylib"; +#else +static const std::string kSharedLibExt = ".so"; +#endif +#endif + +namespace { + +ThreadStatusUpdater* CreateThreadStatusUpdater() { + return new ThreadStatusUpdater(); +} + +#ifndef ROCKSDB_NO_DYNAMIC_EXTENSION +class PosixDynamicLibrary : public DynamicLibrary { + public: + PosixDynamicLibrary(const std::string& name, void* handle) + : name_(name), handle_(handle) {} + ~PosixDynamicLibrary() override { dlclose(handle_); } + + Status LoadSymbol(const std::string& sym_name, void** func) override { + assert(nullptr != func); + dlerror(); // Clear any old error + *func = dlsym(handle_, sym_name.c_str()); + if (*func != nullptr) { + return Status::OK(); + } else { + char* err = dlerror(); + return Status::NotFound("Error finding symbol: " + sym_name, err); + } + } + + const char* Name() const override { return name_.c_str(); } + + private: + std::string name_; + void* handle_; +}; +#endif // !ROCKSDB_NO_DYNAMIC_EXTENSION + +class PosixClock : public SystemClock { + public: + static const char* kClassName() { return "PosixClock"; } + const char* Name() const override { return kDefaultName(); } + const char* NickName() const override { return kClassName(); } + + uint64_t NowMicros() override { + port::TimeVal tv; + port::GetTimeOfDay(&tv, nullptr); + return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; + } + + uint64_t NowNanos() override { +#if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_GNU_KFREEBSD) || \ + defined(OS_AIX) + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return static_cast(ts.tv_sec) * 1000000000 + ts.tv_nsec; +#elif defined(OS_SOLARIS) + return gethrtime(); +#elif defined(__MACH__) + clock_serv_t cclock; + mach_timespec_t ts; + host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock); + clock_get_time(cclock, &ts); + mach_port_deallocate(mach_task_self(), cclock); + return static_cast(ts.tv_sec) * 1000000000 + ts.tv_nsec; +#else + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); +#endif + } + + uint64_t CPUMicros() override { +#if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_GNU_KFREEBSD) || \ + defined(OS_AIX) || (defined(__MACH__) && defined(__MAC_10_12)) + struct timespec ts; + clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts); + return (static_cast(ts.tv_sec) * 1000000000 + ts.tv_nsec) / 1000; +#endif + return 0; + } + + uint64_t CPUNanos() override { +#if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_GNU_KFREEBSD) || \ + defined(OS_AIX) || (defined(__MACH__) && defined(__MAC_10_12)) + struct timespec ts; + clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts); + return static_cast(ts.tv_sec) * 1000000000 + ts.tv_nsec; +#endif + return 0; + } + + void SleepForMicroseconds(int micros) override { usleep(micros); } + + Status GetCurrentTime(int64_t* unix_time) override { + time_t ret = time(nullptr); + if (ret == (time_t)-1) { + return IOError("GetCurrentTime", "", errno); + } + *unix_time = (int64_t)ret; + return Status::OK(); + } + + std::string TimeToString(uint64_t secondsSince1970) override { + const time_t seconds = (time_t)secondsSince1970; + struct tm t; + int maxsize = 64; + std::string dummy; + dummy.reserve(maxsize); + dummy.resize(maxsize); + char* p = &dummy[0]; + port::LocalTimeR(&seconds, &t); + snprintf(p, maxsize, "%04d/%02d/%02d-%02d:%02d:%02d ", t.tm_year + 1900, + t.tm_mon + 1, t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec); + return dummy; + } +}; + +class PosixEnv : public CompositeEnv { + public: + static const char* kClassName() { return "PosixEnv"; } + const char* Name() const override { return kClassName(); } + const char* NickName() const override { return kDefaultName(); } + + struct JoinThreadsOnExit { + explicit JoinThreadsOnExit(PosixEnv& _deflt) : deflt(_deflt) {} + ~JoinThreadsOnExit() { + for (const auto tid : deflt.threads_to_join_) { + pthread_join(tid, nullptr); + } + for (int pool_id = 0; pool_id < Env::Priority::TOTAL; ++pool_id) { + deflt.thread_pools_[pool_id].JoinAllThreads(); + } + // Do not delete the thread_status_updater_ in order to avoid the + // free after use when Env::Default() is destructed while some other + // child threads are still trying to update thread status. All + // PosixEnv instances use the same thread_status_updater_, so never + // explicitly delete it. + } + PosixEnv& deflt; + }; + + void SetFD_CLOEXEC(int fd, const EnvOptions* options) { + if ((options == nullptr || options->set_fd_cloexec) && fd > 0) { + fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC); + } + } + +#ifndef ROCKSDB_NO_DYNAMIC_EXTENSION + // Loads the named library into the result. + // If the input name is empty, the current executable is loaded + // On *nix systems, a "lib" prefix is added to the name if one is not supplied + // Comparably, the appropriate shared library extension is added to the name + // if not supplied. If search_path is not specified, the shared library will + // be loaded using the default path (LD_LIBRARY_PATH) If search_path is + // specified, the shared library will be searched for in the directories + // provided by the search path + Status LoadLibrary(const std::string& name, const std::string& path, + std::shared_ptr* result) override { + assert(result != nullptr); + if (name.empty()) { + void* hndl = dlopen(NULL, RTLD_NOW); + if (hndl != nullptr) { + result->reset(new PosixDynamicLibrary(name, hndl)); + return Status::OK(); + } + } else { + std::string library_name = name; + if (library_name.find(kSharedLibExt) == std::string::npos) { + library_name = library_name + kSharedLibExt; + } +#if !defined(OS_WIN) + if (library_name.find('/') == std::string::npos && + library_name.compare(0, 3, "lib") != 0) { + library_name = "lib" + library_name; + } +#endif + if (path.empty()) { + void* hndl = dlopen(library_name.c_str(), RTLD_NOW); + if (hndl != nullptr) { + result->reset(new PosixDynamicLibrary(library_name, hndl)); + return Status::OK(); + } + } else { + std::string local_path; + std::stringstream ss(path); + while (getline(ss, local_path, kPathSeparator)) { + if (!path.empty()) { + std::string full_name = local_path + "/" + library_name; + void* hndl = dlopen(full_name.c_str(), RTLD_NOW); + if (hndl != nullptr) { + result->reset(new PosixDynamicLibrary(full_name, hndl)); + return Status::OK(); + } + } + } + } + } + return Status::IOError( + IOErrorMsg("Failed to open shared library: xs", name), dlerror()); + } +#endif // !ROCKSDB_NO_DYNAMIC_EXTENSION + + void Schedule(void (*function)(void* arg1), void* arg, Priority pri = LOW, + void* tag = nullptr, + void (*unschedFunction)(void* arg) = nullptr) override; + + int UnSchedule(void* arg, Priority pri) override; + + void StartThread(void (*function)(void* arg), void* arg) override; + + void WaitForJoin() override; + + unsigned int GetThreadPoolQueueLen(Priority pri = LOW) const override; + + int ReserveThreads(int threads_to_be_reserved, Priority pri) override; + + int ReleaseThreads(int threads_to_be_released, Priority pri) override; + + Status GetThreadList(std::vector* thread_list) override { + assert(thread_status_updater_); + return thread_status_updater_->GetThreadList(thread_list); + } + + uint64_t GetThreadID() const override { + uint64_t thread_id = 0; +#if defined(_GNU_SOURCE) && defined(__GLIBC_PREREQ) +#if __GLIBC_PREREQ(2, 30) + thread_id = ::gettid(); +#else // __GLIBC_PREREQ(2, 30) + pthread_t tid = pthread_self(); + memcpy(&thread_id, &tid, std::min(sizeof(thread_id), sizeof(tid))); +#endif // __GLIBC_PREREQ(2, 30) +#else // defined(_GNU_SOURCE) && defined(__GLIBC_PREREQ) + pthread_t tid = pthread_self(); + memcpy(&thread_id, &tid, std::min(sizeof(thread_id), sizeof(tid))); +#endif // defined(_GNU_SOURCE) && defined(__GLIBC_PREREQ) + return thread_id; + } + + Status GetHostName(char* name, uint64_t len) override { + const size_t max_len = static_cast(len); + int ret = gethostname(name, max_len); + if (ret < 0) { + if (errno == EFAULT || errno == EINVAL) { + return Status::InvalidArgument(errnoStr(errno).c_str()); + } else if (errno == ENAMETOOLONG) { + return IOError("GetHostName", std::string(name, strnlen(name, max_len)), + errno); + } else { + return IOError("GetHostName", "", errno); + } + } + return Status::OK(); + } + + ThreadStatusUpdater* GetThreadStatusUpdater() const override { + return Env::GetThreadStatusUpdater(); + } + + std::string GenerateUniqueId() override { return Env::GenerateUniqueId(); } + + // Allow increasing the number of worker threads. + void SetBackgroundThreads(int num, Priority pri) override { + assert(pri >= Priority::BOTTOM && pri <= Priority::HIGH); + thread_pools_[pri].SetBackgroundThreads(num); + } + + int GetBackgroundThreads(Priority pri) override { + assert(pri >= Priority::BOTTOM && pri <= Priority::HIGH); + return thread_pools_[pri].GetBackgroundThreads(); + } + + Status SetAllowNonOwnerAccess(bool allow_non_owner_access) override { + allow_non_owner_access_ = allow_non_owner_access; + return Status::OK(); + } + + // Allow increasing the number of worker threads. + void IncBackgroundThreadsIfNeeded(int num, Priority pri) override { + assert(pri >= Priority::BOTTOM && pri <= Priority::HIGH); + thread_pools_[pri].IncBackgroundThreadsIfNeeded(num); + } + + void LowerThreadPoolIOPriority(Priority pool) override { + assert(pool >= Priority::BOTTOM && pool <= Priority::HIGH); +#ifdef OS_LINUX + thread_pools_[pool].LowerIOPriority(); +#else + (void)pool; +#endif + } + + void LowerThreadPoolCPUPriority(Priority pool) override { + assert(pool >= Priority::BOTTOM && pool <= Priority::HIGH); + thread_pools_[pool].LowerCPUPriority(CpuPriority::kLow); + } + + Status LowerThreadPoolCPUPriority(Priority pool, CpuPriority pri) override { + assert(pool >= Priority::BOTTOM && pool <= Priority::HIGH); + thread_pools_[pool].LowerCPUPriority(pri); + return Status::OK(); + } + + private: + friend Env* Env::Default(); + // Constructs the default Env, a singleton + PosixEnv(); + + // The below 4 members are only used by the default PosixEnv instance. + // Non-default instances simply maintain references to the backing + // members in te default instance + std::vector thread_pools_storage_; + pthread_mutex_t mu_storage_; + std::vector threads_to_join_storage_; + bool allow_non_owner_access_storage_; + + std::vector& thread_pools_; + pthread_mutex_t& mu_; + std::vector& threads_to_join_; + // If true, allow non owner read access for db files. Otherwise, non-owner + // has no access to db files. + bool& allow_non_owner_access_; +}; + +PosixEnv::PosixEnv() + : CompositeEnv(FileSystem::Default(), SystemClock::Default()), + thread_pools_storage_(Priority::TOTAL), + allow_non_owner_access_storage_(true), + thread_pools_(thread_pools_storage_), + mu_(mu_storage_), + threads_to_join_(threads_to_join_storage_), + allow_non_owner_access_(allow_non_owner_access_storage_) { + ThreadPoolImpl::PthreadCall("mutex_init", pthread_mutex_init(&mu_, nullptr)); + for (int pool_id = 0; pool_id < Env::Priority::TOTAL; ++pool_id) { + thread_pools_[pool_id].SetThreadPriority( + static_cast(pool_id)); + // This allows later initializing the thread-local-env of each thread. + thread_pools_[pool_id].SetHostEnv(this); + } + thread_status_updater_ = CreateThreadStatusUpdater(); +} + +void PosixEnv::Schedule(void (*function)(void* arg1), void* arg, Priority pri, + void* tag, void (*unschedFunction)(void* arg)) { + assert(pri >= Priority::BOTTOM && pri <= Priority::HIGH); + thread_pools_[pri].Schedule(function, arg, tag, unschedFunction); +} + +int PosixEnv::UnSchedule(void* arg, Priority pri) { + return thread_pools_[pri].UnSchedule(arg); +} + +unsigned int PosixEnv::GetThreadPoolQueueLen(Priority pri) const { + assert(pri >= Priority::BOTTOM && pri <= Priority::HIGH); + return thread_pools_[pri].GetQueueLen(); +} + +int PosixEnv::ReserveThreads(int threads_to_reserved, Priority pri) { + assert(pri >= Priority::BOTTOM && pri <= Priority::HIGH); + return thread_pools_[pri].ReserveThreads(threads_to_reserved); +} + +int PosixEnv::ReleaseThreads(int threads_to_released, Priority pri) { + assert(pri >= Priority::BOTTOM && pri <= Priority::HIGH); + return thread_pools_[pri].ReleaseThreads(threads_to_released); +} + +struct StartThreadState { + void (*user_function)(void*); + void* arg; +}; + +static void* StartThreadWrapper(void* arg) { + StartThreadState* state = reinterpret_cast(arg); + state->user_function(state->arg); + delete state; + return nullptr; +} + +void PosixEnv::StartThread(void (*function)(void* arg), void* arg) { + pthread_t t; + StartThreadState* state = new StartThreadState; + state->user_function = function; + state->arg = arg; + ThreadPoolImpl::PthreadCall( + "start thread", pthread_create(&t, nullptr, &StartThreadWrapper, state)); + ThreadPoolImpl::PthreadCall("lock", pthread_mutex_lock(&mu_)); + threads_to_join_.push_back(t); + ThreadPoolImpl::PthreadCall("unlock", pthread_mutex_unlock(&mu_)); +} + +void PosixEnv::WaitForJoin() { + for (const auto tid : threads_to_join_) { + pthread_join(tid, nullptr); + } + threads_to_join_.clear(); +} + +} // namespace + +// +// Default Posix Env +// +Env* Env::Default() { + // The following function call initializes the singletons of ThreadLocalPtr + // right before the static default_env. This guarantees default_env will + // always being destructed before the ThreadLocalPtr singletons get + // destructed as C++ guarantees that the destructions of static variables + // is in the reverse order of their constructions. + // + // Since static members are destructed in the reverse order + // of their construction, having this call here guarantees that + // the destructor of static PosixEnv will go first, then the + // the singletons of ThreadLocalPtr. + ThreadLocalPtr::InitSingletons(); + CompressionContextCache::InitSingleton(); + INIT_SYNC_POINT_SINGLETONS(); + // Avoid problems with accessing most members of Env::Default() during + // static destruction. + STATIC_AVOID_DESTRUCTION(PosixEnv, default_env); + // This destructor must be called on exit + static PosixEnv::JoinThreadsOnExit thread_joiner(default_env); + return &default_env; +} + +// +// Default Posix SystemClock +// +const std::shared_ptr& SystemClock::Default() { + STATIC_AVOID_DESTRUCTION(std::shared_ptr, instance) + (std::make_shared()); + return instance; +} +} // namespace ROCKSDB_NAMESPACE + +#endif diff --git a/librocksdb-sys/rocksdb/env/env_test.cc b/librocksdb-sys/rocksdb/env/env_test.cc new file mode 100644 index 0000000..fb23bae --- /dev/null +++ b/librocksdb-sys/rocksdb/env/env_test.cc @@ -0,0 +1,3661 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef OS_WIN +#include +#endif + +#if defined(ROCKSDB_IOURING_PRESENT) +#include +#include +#endif + +#include + +#include +#include +#include +#include + +#ifdef OS_LINUX +#include +#include +#include +#include +#include +#endif + +#ifdef ROCKSDB_FALLOCATE_PRESENT +#include +#endif + +#include "db/db_impl/db_impl.h" +#include "env/emulated_clock.h" +#include "env/env_chroot.h" +#include "env/env_encryption_ctr.h" +#include "env/fs_readonly.h" +#include "env/mock_env.h" +#include "env/unique_id_gen.h" +#include "logging/log_buffer.h" +#include "logging/logging.h" +#include "options/options_helper.h" +#include "port/malloc.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/convenience.h" +#include "rocksdb/env.h" +#include "rocksdb/env_encryption.h" +#include "rocksdb/file_system.h" +#include "rocksdb/system_clock.h" +#include "rocksdb/utilities/object_registry.h" +#include "test_util/mock_time_env.h" +#include "test_util/sync_point.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/coding.h" +#include "util/crc32c.h" +#include "util/mutexlock.h" +#include "util/random.h" +#include "util/string_util.h" +#include "utilities/counted_fs.h" +#include "utilities/env_timed.h" +#include "utilities/fault_injection_env.h" +#include "utilities/fault_injection_fs.h" + +namespace ROCKSDB_NAMESPACE { + +using port::kPageSize; + +static const int kDelayMicros = 100000; + +struct Deleter { + explicit Deleter(void (*fn)(void*)) : fn_(fn) {} + + void operator()(void* ptr) { + assert(fn_); + assert(ptr); + (*fn_)(ptr); + } + + void (*fn_)(void*); +}; + +extern "C" bool RocksDbIOUringEnable() { return true; } + +std::unique_ptr NewAligned(const size_t size, const char ch) { + char* ptr = nullptr; +#ifdef OS_WIN + if (nullptr == + (ptr = reinterpret_cast(_aligned_malloc(size, kPageSize)))) { + return std::unique_ptr(nullptr, Deleter(_aligned_free)); + } + std::unique_ptr uptr(ptr, Deleter(_aligned_free)); +#else + if (posix_memalign(reinterpret_cast(&ptr), kPageSize, size) != 0) { + return std::unique_ptr(nullptr, Deleter(free)); + } + std::unique_ptr uptr(ptr, Deleter(free)); +#endif + memset(uptr.get(), ch, size); + return uptr; +} + +class EnvPosixTest : public testing::Test { + private: + port::Mutex mu_; + std::string events_; + + public: + Env* env_; + bool direct_io_; + EnvPosixTest() : env_(Env::Default()), direct_io_(false) {} + ~EnvPosixTest() { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->LoadDependency({}); + SyncPoint::GetInstance()->ClearAllCallBacks(); + } +}; + +class EnvPosixTestWithParam + : public EnvPosixTest, + public ::testing::WithParamInterface> { + public: + EnvPosixTestWithParam() { + std::pair param_pair = GetParam(); + env_ = param_pair.first; + direct_io_ = param_pair.second; + } + + void WaitThreadPoolsEmpty() { + // Wait until the thread pools are empty. + while (env_->GetThreadPoolQueueLen(Env::Priority::LOW) != 0) { + Env::Default()->SleepForMicroseconds(kDelayMicros); + } + while (env_->GetThreadPoolQueueLen(Env::Priority::HIGH) != 0) { + Env::Default()->SleepForMicroseconds(kDelayMicros); + } + } + + ~EnvPosixTestWithParam() override { WaitThreadPoolsEmpty(); } +}; + +static void SetBool(void* ptr) { + reinterpret_cast*>(ptr)->store(true); +} + +TEST_F(EnvPosixTest, DISABLED_RunImmediately) { + for (int pri = Env::BOTTOM; pri < Env::TOTAL; ++pri) { + std::atomic called(false); + env_->SetBackgroundThreads(1, static_cast(pri)); + env_->Schedule(&SetBool, &called, static_cast(pri)); + Env::Default()->SleepForMicroseconds(kDelayMicros); + ASSERT_TRUE(called.load()); + } +} + +TEST_F(EnvPosixTest, RunEventually) { + std::atomic called(false); + env_->StartThread(&SetBool, &called); + env_->WaitForJoin(); + ASSERT_TRUE(called.load()); +} + +#ifdef OS_WIN +TEST_F(EnvPosixTest, AreFilesSame) { + { + bool tmp; + if (env_->AreFilesSame("", "", &tmp).IsNotSupported()) { + fprintf(stderr, + "skipping EnvBasicTestWithParam.AreFilesSame due to " + "unsupported Env::AreFilesSame\n"); + return; + } + } + + const EnvOptions soptions; + auto* env = Env::Default(); + std::string same_file_name = test::PerThreadDBPath(env, "same_file"); + std::string same_file_link_name = same_file_name + "_link"; + + std::unique_ptr same_file; + ASSERT_OK(env->NewWritableFile(same_file_name, &same_file, soptions)); + same_file->Append("random_data"); + ASSERT_OK(same_file->Flush()); + same_file.reset(); + + ASSERT_OK(env->LinkFile(same_file_name, same_file_link_name)); + bool result = false; + ASSERT_OK(env->AreFilesSame(same_file_name, same_file_link_name, &result)); + ASSERT_TRUE(result); +} +#endif + +#ifdef OS_LINUX +TEST_F(EnvPosixTest, DISABLED_FilePermission) { + // Only works for Linux environment + if (env_ == Env::Default()) { + EnvOptions soptions; + std::vector fileNames{ + test::PerThreadDBPath(env_, "testfile"), + test::PerThreadDBPath(env_, "testfile1")}; + std::unique_ptr wfile; + ASSERT_OK(env_->NewWritableFile(fileNames[0], &wfile, soptions)); + ASSERT_OK(env_->NewWritableFile(fileNames[1], &wfile, soptions)); + wfile.reset(); + std::unique_ptr rwfile; + ASSERT_OK(env_->NewRandomRWFile(fileNames[1], &rwfile, soptions)); + + struct stat sb; + for (const auto& filename : fileNames) { + if (::stat(filename.c_str(), &sb) == 0) { + ASSERT_EQ(sb.st_mode & 0777, 0644); + } + ASSERT_OK(env_->DeleteFile(filename)); + } + + env_->SetAllowNonOwnerAccess(false); + ASSERT_OK(env_->NewWritableFile(fileNames[0], &wfile, soptions)); + ASSERT_OK(env_->NewWritableFile(fileNames[1], &wfile, soptions)); + wfile.reset(); + ASSERT_OK(env_->NewRandomRWFile(fileNames[1], &rwfile, soptions)); + + for (const auto& filename : fileNames) { + if (::stat(filename.c_str(), &sb) == 0) { + ASSERT_EQ(sb.st_mode & 0777, 0600); + } + ASSERT_OK(env_->DeleteFile(filename)); + } + } +} + +TEST_F(EnvPosixTest, LowerThreadPoolCpuPriority) { + std::atomic from_priority(CpuPriority::kNormal); + std::atomic to_priority(CpuPriority::kNormal); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "ThreadPoolImpl::BGThread::BeforeSetCpuPriority", [&](void* pri) { + from_priority.store(*reinterpret_cast(pri)); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "ThreadPoolImpl::BGThread::AfterSetCpuPriority", [&](void* pri) { + to_priority.store(*reinterpret_cast(pri)); + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + env_->SetBackgroundThreads(1, Env::BOTTOM); + env_->SetBackgroundThreads(1, Env::HIGH); + + auto RunTask = [&](Env::Priority pool) { + std::atomic called(false); + env_->Schedule(&SetBool, &called, pool); + for (int i = 0; i < kDelayMicros; i++) { + if (called.load()) { + break; + } + Env::Default()->SleepForMicroseconds(1); + } + ASSERT_TRUE(called.load()); + }; + + { + // Same priority, no-op. + env_->LowerThreadPoolCPUPriority(Env::Priority::BOTTOM, + CpuPriority::kNormal) + .PermitUncheckedError(); + RunTask(Env::Priority::BOTTOM); + ASSERT_EQ(from_priority, CpuPriority::kNormal); + ASSERT_EQ(to_priority, CpuPriority::kNormal); + } + + { + // Higher priority, no-op. + env_->LowerThreadPoolCPUPriority(Env::Priority::BOTTOM, CpuPriority::kHigh) + .PermitUncheckedError(); + RunTask(Env::Priority::BOTTOM); + ASSERT_EQ(from_priority, CpuPriority::kNormal); + ASSERT_EQ(to_priority, CpuPriority::kNormal); + } + + { + // Lower priority from kNormal -> kLow. + env_->LowerThreadPoolCPUPriority(Env::Priority::BOTTOM, CpuPriority::kLow) + .PermitUncheckedError(); + RunTask(Env::Priority::BOTTOM); + ASSERT_EQ(from_priority, CpuPriority::kNormal); + ASSERT_EQ(to_priority, CpuPriority::kLow); + } + + { + // Lower priority from kLow -> kIdle. + env_->LowerThreadPoolCPUPriority(Env::Priority::BOTTOM, CpuPriority::kIdle) + .PermitUncheckedError(); + RunTask(Env::Priority::BOTTOM); + ASSERT_EQ(from_priority, CpuPriority::kLow); + ASSERT_EQ(to_priority, CpuPriority::kIdle); + } + + { + // Lower priority from kNormal -> kIdle for another pool. + env_->LowerThreadPoolCPUPriority(Env::Priority::HIGH, CpuPriority::kIdle) + .PermitUncheckedError(); + RunTask(Env::Priority::HIGH); + ASSERT_EQ(from_priority, CpuPriority::kNormal); + ASSERT_EQ(to_priority, CpuPriority::kIdle); + } + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} +#endif + +TEST_F(EnvPosixTest, MemoryMappedFileBuffer) { + const int kFileBytes = 1 << 15; // 32 KB + std::string expected_data; + std::string fname = test::PerThreadDBPath(env_, "testfile"); + { + std::unique_ptr wfile; + const EnvOptions soptions; + ASSERT_OK(env_->NewWritableFile(fname, &wfile, soptions)); + + Random rnd(301); + expected_data = rnd.RandomString(kFileBytes); + ASSERT_OK(wfile->Append(expected_data)); + } + + std::unique_ptr mmap_buffer; + Status status = env_->NewMemoryMappedFileBuffer(fname, &mmap_buffer); + // it should be supported at least on linux +#if !defined(OS_LINUX) + if (status.IsNotSupported()) { + fprintf(stderr, + "skipping EnvPosixTest.MemoryMappedFileBuffer due to " + "unsupported Env::NewMemoryMappedFileBuffer\n"); + return; + } +#endif // !defined(OS_LINUX) + + ASSERT_OK(status); + ASSERT_NE(nullptr, mmap_buffer.get()); + ASSERT_NE(nullptr, mmap_buffer->GetBase()); + ASSERT_EQ(kFileBytes, mmap_buffer->GetLen()); + std::string actual_data(reinterpret_cast(mmap_buffer->GetBase()), + mmap_buffer->GetLen()); + ASSERT_EQ(expected_data, actual_data); +} + +#ifndef ROCKSDB_NO_DYNAMIC_EXTENSION +TEST_F(EnvPosixTest, LoadRocksDBLibrary) { + std::shared_ptr library; + std::function function; + Status status = env_->LoadLibrary("no-such-library", "", &library); + ASSERT_NOK(status); + ASSERT_EQ(nullptr, library.get()); + status = env_->LoadLibrary("rocksdb", "", &library); + if (status.ok()) { // If we have can find a rocksdb shared library + ASSERT_NE(nullptr, library.get()); + ASSERT_OK(library->LoadFunction("rocksdb_create_default_env", + &function)); // from C definition + ASSERT_NE(nullptr, function); + ASSERT_NOK(library->LoadFunction("no-such-method", &function)); + ASSERT_EQ(nullptr, function); + ASSERT_OK(env_->LoadLibrary(library->Name(), "", &library)); + } else { + ASSERT_EQ(nullptr, library.get()); + } +} +#endif // !ROCKSDB_NO_DYNAMIC_EXTENSION + +#if !defined(OS_WIN) && !defined(ROCKSDB_NO_DYNAMIC_EXTENSION) +TEST_F(EnvPosixTest, LoadRocksDBLibraryWithSearchPath) { + std::shared_ptr library; + std::function function; + ASSERT_NOK(env_->LoadLibrary("no-such-library", "/tmp", &library)); + ASSERT_EQ(nullptr, library.get()); + ASSERT_NOK(env_->LoadLibrary("dl", "/tmp", &library)); + ASSERT_EQ(nullptr, library.get()); + Status status = env_->LoadLibrary("rocksdb", "/tmp:./", &library); + if (status.ok()) { + ASSERT_NE(nullptr, library.get()); + ASSERT_OK(env_->LoadLibrary(library->Name(), "", &library)); + } + char buff[1024]; + std::string cwd = getcwd(buff, sizeof(buff)); + + status = env_->LoadLibrary("rocksdb", "/tmp:" + cwd, &library); + if (status.ok()) { + ASSERT_NE(nullptr, library.get()); + ASSERT_OK(env_->LoadLibrary(library->Name(), "", &library)); + } +} +#endif // !OS_WIN && !ROCKSDB_NO_DYNAMIC_EXTENSION + +TEST_P(EnvPosixTestWithParam, UnSchedule) { + std::atomic called(false); + env_->SetBackgroundThreads(1, Env::LOW); + + /* Block the low priority queue */ + test::SleepingBackgroundTask sleeping_task, sleeping_task1; + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task, + Env::Priority::LOW); + + /* Schedule another task */ + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task1, + Env::Priority::LOW, &sleeping_task1); + + /* Remove it with a different tag */ + ASSERT_EQ(0, env_->UnSchedule(&called, Env::Priority::LOW)); + + /* Remove it from the queue with the right tag */ + ASSERT_EQ(1, env_->UnSchedule(&sleeping_task1, Env::Priority::LOW)); + + // Unblock background thread + sleeping_task.WakeUp(); + + /* Schedule another task */ + env_->Schedule(&SetBool, &called); + for (int i = 0; i < kDelayMicros; i++) { + if (called.load()) { + break; + } + Env::Default()->SleepForMicroseconds(1); + } + ASSERT_TRUE(called.load()); + + ASSERT_TRUE(!sleeping_task.IsSleeping() && !sleeping_task1.IsSleeping()); + WaitThreadPoolsEmpty(); +} + +// This tests assumes that the last scheduled +// task will run last. In fact, in the allotted +// sleeping time nothing may actually run or they may +// run in any order. The purpose of the test is unclear. +#ifndef OS_WIN +TEST_P(EnvPosixTestWithParam, RunMany) { + env_->SetBackgroundThreads(1, Env::LOW); + std::atomic last_id(0); + + struct CB { + std::atomic* last_id_ptr; // Pointer to shared slot + int id; // Order# for the execution of this callback + + CB(std::atomic* p, int i) : last_id_ptr(p), id(i) {} + + static void Run(void* v) { + CB* cb = reinterpret_cast(v); + int cur = cb->last_id_ptr->load(); + ASSERT_EQ(cb->id - 1, cur); + cb->last_id_ptr->store(cb->id); + } + }; + + // Schedule in different order than start time + CB cb1(&last_id, 1); + CB cb2(&last_id, 2); + CB cb3(&last_id, 3); + CB cb4(&last_id, 4); + env_->Schedule(&CB::Run, &cb1); + env_->Schedule(&CB::Run, &cb2); + env_->Schedule(&CB::Run, &cb3); + env_->Schedule(&CB::Run, &cb4); + // thread-pool pops a thread function and then run the function, which may + // cause threadpool is empty but the last function is still running. Add a + // dummy function at the end, to make sure the last callback is finished + // before threadpool is empty. + struct DummyCB { + static void Run(void*) {} + }; + env_->Schedule(&DummyCB::Run, nullptr); + + WaitThreadPoolsEmpty(); + ASSERT_EQ(4, last_id.load(std::memory_order_acquire)); +} +#endif + +struct State { + port::Mutex mu; + int val; + int num_running; +}; + +static void ThreadBody(void* arg) { + State* s = reinterpret_cast(arg); + s->mu.Lock(); + s->val += 1; + s->num_running -= 1; + s->mu.Unlock(); +} + +TEST_P(EnvPosixTestWithParam, StartThread) { + State state; + state.val = 0; + state.num_running = 3; + for (int i = 0; i < 3; i++) { + env_->StartThread(&ThreadBody, &state); + } + while (true) { + state.mu.Lock(); + int num = state.num_running; + state.mu.Unlock(); + if (num == 0) { + break; + } + Env::Default()->SleepForMicroseconds(kDelayMicros); + } + ASSERT_EQ(state.val, 3); + WaitThreadPoolsEmpty(); +} + +TEST_P(EnvPosixTestWithParam, TwoPools) { + // Data structures to signal tasks to run. + port::Mutex mutex; + port::CondVar cv(&mutex); + bool should_start = false; + + class CB { + public: + CB(const std::string& pool_name, int pool_size, port::Mutex* trigger_mu, + port::CondVar* trigger_cv, bool* _should_start) + : mu_(), + num_running_(0), + num_finished_(0), + pool_size_(pool_size), + pool_name_(pool_name), + trigger_mu_(trigger_mu), + trigger_cv_(trigger_cv), + should_start_(_should_start) {} + + static void Run(void* v) { + CB* cb = reinterpret_cast(v); + cb->Run(); + } + + void Run() { + { + MutexLock l(&mu_); + num_running_++; + // make sure we don't have more than pool_size_ jobs running. + ASSERT_LE(num_running_, pool_size_.load()); + } + + { + MutexLock l(trigger_mu_); + while (!(*should_start_)) { + trigger_cv_->Wait(); + } + } + + { + MutexLock l(&mu_); + num_running_--; + num_finished_++; + } + } + + int NumFinished() { + MutexLock l(&mu_); + return num_finished_; + } + + void Reset(int pool_size) { + pool_size_.store(pool_size); + num_finished_ = 0; + } + + private: + port::Mutex mu_; + int num_running_; + int num_finished_; + std::atomic pool_size_; + std::string pool_name_; + port::Mutex* trigger_mu_; + port::CondVar* trigger_cv_; + bool* should_start_; + }; + + const int kLowPoolSize = 2; + const int kHighPoolSize = 4; + const int kJobs = 8; + + CB low_pool_job("low", kLowPoolSize, &mutex, &cv, &should_start); + CB high_pool_job("high", kHighPoolSize, &mutex, &cv, &should_start); + + env_->SetBackgroundThreads(kLowPoolSize); + env_->SetBackgroundThreads(kHighPoolSize, Env::Priority::HIGH); + + ASSERT_EQ(0U, env_->GetThreadPoolQueueLen(Env::Priority::LOW)); + ASSERT_EQ(0U, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + + // schedule same number of jobs in each pool + for (int i = 0; i < kJobs; i++) { + env_->Schedule(&CB::Run, &low_pool_job); + env_->Schedule(&CB::Run, &high_pool_job, Env::Priority::HIGH); + } + // Wait a short while for the jobs to be dispatched. + int sleep_count = 0; + while ((unsigned int)(kJobs - kLowPoolSize) != + env_->GetThreadPoolQueueLen(Env::Priority::LOW) || + (unsigned int)(kJobs - kHighPoolSize) != + env_->GetThreadPoolQueueLen(Env::Priority::HIGH)) { + env_->SleepForMicroseconds(kDelayMicros); + if (++sleep_count > 100) { + break; + } + } + + ASSERT_EQ((unsigned int)(kJobs - kLowPoolSize), + env_->GetThreadPoolQueueLen()); + ASSERT_EQ((unsigned int)(kJobs - kLowPoolSize), + env_->GetThreadPoolQueueLen(Env::Priority::LOW)); + ASSERT_EQ((unsigned int)(kJobs - kHighPoolSize), + env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + + // Trigger jobs to run. + { + MutexLock l(&mutex); + should_start = true; + cv.SignalAll(); + } + + // wait for all jobs to finish + while (low_pool_job.NumFinished() < kJobs || + high_pool_job.NumFinished() < kJobs) { + env_->SleepForMicroseconds(kDelayMicros); + } + + ASSERT_EQ(0U, env_->GetThreadPoolQueueLen(Env::Priority::LOW)); + ASSERT_EQ(0U, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + + // Hold jobs to schedule; + should_start = false; + + // call IncBackgroundThreadsIfNeeded to two pools. One increasing and + // the other decreasing + env_->IncBackgroundThreadsIfNeeded(kLowPoolSize - 1, Env::Priority::LOW); + env_->IncBackgroundThreadsIfNeeded(kHighPoolSize + 1, Env::Priority::HIGH); + high_pool_job.Reset(kHighPoolSize + 1); + low_pool_job.Reset(kLowPoolSize); + + // schedule same number of jobs in each pool + for (int i = 0; i < kJobs; i++) { + env_->Schedule(&CB::Run, &low_pool_job); + env_->Schedule(&CB::Run, &high_pool_job, Env::Priority::HIGH); + } + // Wait a short while for the jobs to be dispatched. + sleep_count = 0; + while ((unsigned int)(kJobs - kLowPoolSize) != + env_->GetThreadPoolQueueLen(Env::Priority::LOW) || + (unsigned int)(kJobs - (kHighPoolSize + 1)) != + env_->GetThreadPoolQueueLen(Env::Priority::HIGH)) { + env_->SleepForMicroseconds(kDelayMicros); + if (++sleep_count > 100) { + break; + } + } + ASSERT_EQ((unsigned int)(kJobs - kLowPoolSize), + env_->GetThreadPoolQueueLen()); + ASSERT_EQ((unsigned int)(kJobs - kLowPoolSize), + env_->GetThreadPoolQueueLen(Env::Priority::LOW)); + ASSERT_EQ((unsigned int)(kJobs - (kHighPoolSize + 1)), + env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + + // Trigger jobs to run. + { + MutexLock l(&mutex); + should_start = true; + cv.SignalAll(); + } + + // wait for all jobs to finish + while (low_pool_job.NumFinished() < kJobs || + high_pool_job.NumFinished() < kJobs) { + env_->SleepForMicroseconds(kDelayMicros); + } + + env_->SetBackgroundThreads(kHighPoolSize, Env::Priority::HIGH); + WaitThreadPoolsEmpty(); +} + +TEST_P(EnvPosixTestWithParam, DecreaseNumBgThreads) { + constexpr int kWaitMicros = 60000000; // 1min + + std::vector tasks(10); + + // Set number of thread to 1 first. + env_->SetBackgroundThreads(1, Env::Priority::HIGH); + + // Schedule 3 tasks. 0 running; Task 1, 2 waiting. + for (size_t i = 0; i < 3; i++) { + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &tasks[i], + Env::Priority::HIGH); + } + ASSERT_FALSE(tasks[0].TimedWaitUntilSleeping(kWaitMicros)); + ASSERT_EQ(2U, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + ASSERT_TRUE(tasks[0].IsSleeping()); + ASSERT_TRUE(!tasks[1].IsSleeping()); + ASSERT_TRUE(!tasks[2].IsSleeping()); + + // Increase to 2 threads. Task 0, 1 running; 2 waiting + env_->SetBackgroundThreads(2, Env::Priority::HIGH); + ASSERT_FALSE(tasks[1].TimedWaitUntilSleeping(kWaitMicros)); + ASSERT_EQ(1U, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + ASSERT_TRUE(tasks[0].IsSleeping()); + ASSERT_TRUE(tasks[1].IsSleeping()); + ASSERT_TRUE(!tasks[2].IsSleeping()); + + // Shrink back to 1 thread. Still task 0, 1 running, 2 waiting + env_->SetBackgroundThreads(1, Env::Priority::HIGH); + Env::Default()->SleepForMicroseconds(kDelayMicros); + ASSERT_EQ(1U, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + ASSERT_TRUE(tasks[0].IsSleeping()); + ASSERT_TRUE(tasks[1].IsSleeping()); + ASSERT_TRUE(!tasks[2].IsSleeping()); + + // The last task finishes. Task 0 running, 2 waiting. + tasks[1].WakeUp(); + ASSERT_FALSE(tasks[1].TimedWaitUntilDone(kWaitMicros)); + ASSERT_EQ(1U, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + ASSERT_TRUE(tasks[0].IsSleeping()); + ASSERT_TRUE(!tasks[1].IsSleeping()); + ASSERT_TRUE(!tasks[2].IsSleeping()); + + // Increase to 5 threads. Task 0 and 2 running. + env_->SetBackgroundThreads(5, Env::Priority::HIGH); + ASSERT_FALSE(tasks[2].TimedWaitUntilSleeping(kWaitMicros)); + ASSERT_EQ(0U, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + ASSERT_TRUE(tasks[0].IsSleeping()); + ASSERT_TRUE(!tasks[1].IsSleeping()); + ASSERT_TRUE(tasks[2].IsSleeping()); + + // Change number of threads a couple of times while there is no sufficient + // tasks. + env_->SetBackgroundThreads(7, Env::Priority::HIGH); + tasks[2].WakeUp(); + ASSERT_FALSE(tasks[2].TimedWaitUntilDone(kWaitMicros)); + ASSERT_EQ(0U, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + env_->SetBackgroundThreads(3, Env::Priority::HIGH); + Env::Default()->SleepForMicroseconds(kDelayMicros); + ASSERT_EQ(0U, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + env_->SetBackgroundThreads(4, Env::Priority::HIGH); + Env::Default()->SleepForMicroseconds(kDelayMicros); + ASSERT_EQ(0U, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + env_->SetBackgroundThreads(5, Env::Priority::HIGH); + Env::Default()->SleepForMicroseconds(kDelayMicros); + ASSERT_EQ(0U, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + env_->SetBackgroundThreads(4, Env::Priority::HIGH); + Env::Default()->SleepForMicroseconds(kDelayMicros); + ASSERT_EQ(0U, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + + Env::Default()->SleepForMicroseconds(kDelayMicros * 50); + + // Enqueue 5 more tasks. Thread pool size now is 4. + // Task 0, 3, 4, 5 running;6, 7 waiting. + for (size_t i = 3; i < 8; i++) { + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &tasks[i], + Env::Priority::HIGH); + } + for (size_t i = 3; i <= 5; i++) { + ASSERT_FALSE(tasks[i].TimedWaitUntilSleeping(kWaitMicros)); + } + ASSERT_EQ(2U, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + ASSERT_TRUE(tasks[0].IsSleeping()); + ASSERT_TRUE(!tasks[1].IsSleeping()); + ASSERT_TRUE(!tasks[2].IsSleeping()); + ASSERT_TRUE(tasks[3].IsSleeping()); + ASSERT_TRUE(tasks[4].IsSleeping()); + ASSERT_TRUE(tasks[5].IsSleeping()); + ASSERT_TRUE(!tasks[6].IsSleeping()); + ASSERT_TRUE(!tasks[7].IsSleeping()); + + // Wake up task 0, 3 and 4. Task 5, 6, 7 running. + tasks[0].WakeUp(); + tasks[3].WakeUp(); + tasks[4].WakeUp(); + + for (size_t i = 5; i < 8; i++) { + ASSERT_FALSE(tasks[i].TimedWaitUntilSleeping(kWaitMicros)); + } + ASSERT_EQ(0U, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + for (size_t i = 5; i < 8; i++) { + ASSERT_TRUE(tasks[i].IsSleeping()); + } + + // Shrink back to 1 thread. Still task 5, 6, 7 running + env_->SetBackgroundThreads(1, Env::Priority::HIGH); + Env::Default()->SleepForMicroseconds(kDelayMicros); + ASSERT_TRUE(tasks[5].IsSleeping()); + ASSERT_TRUE(tasks[6].IsSleeping()); + ASSERT_TRUE(tasks[7].IsSleeping()); + + // Wake up task 6. Task 5, 7 running + tasks[6].WakeUp(); + ASSERT_FALSE(tasks[6].TimedWaitUntilDone(kWaitMicros)); + ASSERT_TRUE(tasks[5].IsSleeping()); + ASSERT_TRUE(!tasks[6].IsSleeping()); + ASSERT_TRUE(tasks[7].IsSleeping()); + + // Wake up threads 7. Task 5 running + tasks[7].WakeUp(); + ASSERT_FALSE(tasks[7].TimedWaitUntilDone(kWaitMicros)); + ASSERT_TRUE(!tasks[7].IsSleeping()); + + // Enqueue thread 8 and 9. Task 5 running; one of 8, 9 might be running. + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &tasks[8], + Env::Priority::HIGH); + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &tasks[9], + Env::Priority::HIGH); + Env::Default()->SleepForMicroseconds(kDelayMicros); + ASSERT_GT(env_->GetThreadPoolQueueLen(Env::Priority::HIGH), (unsigned int)0); + ASSERT_TRUE(!tasks[8].IsSleeping() || !tasks[9].IsSleeping()); + + // Increase to 4 threads. Task 5, 8, 9 running. + env_->SetBackgroundThreads(4, Env::Priority::HIGH); + Env::Default()->SleepForMicroseconds(kDelayMicros); + ASSERT_EQ((unsigned int)0, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + ASSERT_TRUE(tasks[8].IsSleeping()); + ASSERT_TRUE(tasks[9].IsSleeping()); + + // Shrink to 1 thread + env_->SetBackgroundThreads(1, Env::Priority::HIGH); + + // Wake up thread 9. + tasks[9].WakeUp(); + ASSERT_FALSE(tasks[9].TimedWaitUntilDone(kWaitMicros)); + ASSERT_TRUE(!tasks[9].IsSleeping()); + ASSERT_TRUE(tasks[8].IsSleeping()); + + // Wake up thread 8 + tasks[8].WakeUp(); + ASSERT_FALSE(tasks[8].TimedWaitUntilDone(kWaitMicros)); + ASSERT_TRUE(!tasks[8].IsSleeping()); + + // Wake up the last thread + tasks[5].WakeUp(); + ASSERT_FALSE(tasks[5].TimedWaitUntilDone(kWaitMicros)); + WaitThreadPoolsEmpty(); +} + +TEST_P(EnvPosixTestWithParam, ReserveThreads) { + // Initialize the background thread to 1 in case other threads exist + // from the last unit test + env_->SetBackgroundThreads(1, Env::Priority::HIGH); + ASSERT_EQ(env_->GetBackgroundThreads(Env::HIGH), 1); + constexpr int kWaitMicros = 10000000; // 10seconds + std::vector tasks(4); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + // Set the sync point to ensure thread 0 can terminate + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"ThreadPoolImpl::BGThread::Termination:th0", + "EnvTest::ReserveThreads:0"}}); + // Empty the thread pool to ensure all the threads can start later + env_->SetBackgroundThreads(0, Env::Priority::HIGH); + TEST_SYNC_POINT("EnvTest::ReserveThreads:0"); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + // Set the sync point to ensure threads start and pass the sync point + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"ThreadPoolImpl::BGThread::Start:th0", "EnvTest::ReserveThreads:1"}, + {"ThreadPoolImpl::BGThread::Start:th1", "EnvTest::ReserveThreads:2"}, + {"ThreadPoolImpl::BGThread::Start:th2", "EnvTest::ReserveThreads:3"}, + {"ThreadPoolImpl::BGThread::Start:th3", "EnvTest::ReserveThreads:4"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Set number of thread to 3 first. + env_->SetBackgroundThreads(3, Env::Priority::HIGH); + ASSERT_EQ(env_->GetBackgroundThreads(Env::HIGH), 3); + // Add sync points to ensure all 3 threads start + TEST_SYNC_POINT("EnvTest::ReserveThreads:1"); + TEST_SYNC_POINT("EnvTest::ReserveThreads:2"); + TEST_SYNC_POINT("EnvTest::ReserveThreads:3"); + // Reserve 2 threads + ASSERT_EQ(2, env_->ReserveThreads(2, Env::Priority::HIGH)); + + // Schedule 3 tasks. Task 0 running (in this context, doing + // SleepingBackgroundTask); Task 1, 2 waiting; 3 reserved threads. + for (size_t i = 0; i < 3; i++) { + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &tasks[i], + Env::Priority::HIGH); + } + ASSERT_FALSE(tasks[0].TimedWaitUntilSleeping(kWaitMicros)); + ASSERT_EQ(2U, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + ASSERT_TRUE(tasks[0].IsSleeping()); + ASSERT_TRUE(!tasks[1].IsSleeping()); + ASSERT_TRUE(!tasks[2].IsSleeping()); + + // Release 2 threads. Task 0, 1, 2 running; 0 reserved thread. + ASSERT_EQ(2, env_->ReleaseThreads(2, Env::Priority::HIGH)); + ASSERT_FALSE(tasks[1].TimedWaitUntilSleeping(kWaitMicros)); + ASSERT_FALSE(tasks[2].TimedWaitUntilSleeping(kWaitMicros)); + ASSERT_EQ(0U, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + ASSERT_TRUE(tasks[1].IsSleeping()); + ASSERT_TRUE(tasks[2].IsSleeping()); + // No more threads can be reserved + ASSERT_EQ(0, env_->ReserveThreads(3, Env::Priority::HIGH)); + // Expand the number of background threads so that the last thread + // is waiting + env_->SetBackgroundThreads(4, Env::Priority::HIGH); + // Add sync point to ensure the 4th thread starts + TEST_SYNC_POINT("EnvTest::ReserveThreads:4"); + // As the thread pool is expanded, we can reserve one more thread + ASSERT_EQ(1, env_->ReserveThreads(3, Env::Priority::HIGH)); + // No more threads can be reserved + ASSERT_EQ(0, env_->ReserveThreads(3, Env::Priority::HIGH)); + + // Reset the sync points for the next iteration in BGThread or the + // next time Submit() is called + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"ThreadPoolImpl::BGThread::WaitingThreadsInc", + "EnvTest::ReserveThreads:5"}, + {"ThreadPoolImpl::BGThread::Termination", "EnvTest::ReserveThreads:6"}, + {"ThreadPoolImpl::Submit::Enqueue", "EnvTest::ReserveThreads:7"}}); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + tasks[0].WakeUp(); + ASSERT_FALSE(tasks[0].TimedWaitUntilDone(kWaitMicros)); + // Add sync point to ensure the number of waiting threads increases + TEST_SYNC_POINT("EnvTest::ReserveThreads:5"); + // 1 more thread can be reserved + ASSERT_EQ(1, env_->ReserveThreads(3, Env::Priority::HIGH)); + // 2 reserved threads now + + // Currently, two threads are blocked since the number of waiting + // threads is equal to the number of reserved threads (i.e., 2). + // If we reduce the number of background thread to 1, at least one thread + // will be the last excessive thread (here we have no control over the + // number of excessive threads because thread order does not + // necessarily follows the schedule order, but we ensure that the last thread + // shall not run any task by expanding the thread pool after we schedule + // the tasks), and thus they(it) become(s) unblocked, the number of waiting + // threads decreases to 0 or 1, but the number of reserved threads is still 2 + env_->SetBackgroundThreads(1, Env::Priority::HIGH); + + // Task 1,2 running; 2 reserved threads, however, in fact, we only have + // 0 or 1 waiting thread in the thread pool, proved by the + // following test, we CANNOT reserve 2 threads even though we just + // release 2 + TEST_SYNC_POINT("EnvTest::ReserveThreads:6"); + ASSERT_EQ(2, env_->ReleaseThreads(2, Env::Priority::HIGH)); + ASSERT_GT(2, env_->ReserveThreads(2, Env::Priority::HIGH)); + + // Every new task will be put into the queue at this point + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &tasks[3], + Env::Priority::HIGH); + TEST_SYNC_POINT("EnvTest::ReserveThreads:7"); + ASSERT_EQ(1U, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + ASSERT_TRUE(!tasks[3].IsSleeping()); + + // Set the number of threads to 3 so that Task 3 can dequeue + env_->SetBackgroundThreads(3, Env::Priority::HIGH); + // Wakup Task 1 + tasks[1].WakeUp(); + ASSERT_FALSE(tasks[1].TimedWaitUntilDone(kWaitMicros)); + // Task 2, 3 running (Task 3 dequeue); 0 or 1 reserved thread + ASSERT_FALSE(tasks[3].TimedWaitUntilSleeping(kWaitMicros)); + ASSERT_TRUE(tasks[3].IsSleeping()); + ASSERT_EQ(0U, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + + // At most 1 thread can be released + ASSERT_GT(2, env_->ReleaseThreads(3, Env::Priority::HIGH)); + tasks[2].WakeUp(); + ASSERT_FALSE(tasks[2].TimedWaitUntilDone(kWaitMicros)); + tasks[3].WakeUp(); + ASSERT_FALSE(tasks[3].TimedWaitUntilDone(kWaitMicros)); + WaitThreadPoolsEmpty(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +#if (defined OS_LINUX || defined OS_WIN) +namespace { +bool IsSingleVarint(const std::string& s) { + Slice slice(s); + + uint64_t v; + if (!GetVarint64(&slice, &v)) { + return false; + } + + return slice.size() == 0; +} + +bool IsUniqueIDValid(const std::string& s) { + return !s.empty() && !IsSingleVarint(s); +} + +const size_t MAX_ID_SIZE = 100; +char temp_id[MAX_ID_SIZE]; + +} // namespace + +// Determine whether we can use the FS_IOC_GETVERSION ioctl +// on a file in directory DIR. Create a temporary file therein, +// try to apply the ioctl (save that result), cleanup and +// return the result. Return true if it is supported, and +// false if anything fails. +// Note that this function "knows" that dir has just been created +// and is empty, so we create a simply-named test file: "f". +bool ioctl_support__FS_IOC_GETVERSION(const std::string& dir) { +#ifdef OS_WIN + return true; +#else + const std::string file = dir + "/f"; + int fd; + do { + fd = open(file.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644); + } while (fd < 0 && errno == EINTR); + long int version; + bool ok = (fd >= 0 && ioctl(fd, FS_IOC_GETVERSION, &version) >= 0); + + close(fd); + unlink(file.c_str()); + + return ok; +#endif +} + +// To ensure that Env::GetUniqueId-related tests work correctly, the files +// should be stored in regular storage like "hard disk" or "flash device", +// and not on a tmpfs file system (like /dev/shm and /tmp on some systems). +// Otherwise we cannot get the correct id. +// +// This function serves as the replacement for test::TmpDir(), which may be +// customized to be on a file system that doesn't work with GetUniqueId(). + +class IoctlFriendlyTmpdir { + public: + explicit IoctlFriendlyTmpdir() { + char dir_buf[100]; + + const char* fmt = "%s/rocksdb.XXXXXX"; + const char* tmp = getenv("TEST_IOCTL_FRIENDLY_TMPDIR"); + +#ifdef OS_WIN +#define rmdir _rmdir + if (tmp == nullptr) { + tmp = getenv("TMP"); + } + + snprintf(dir_buf, sizeof dir_buf, fmt, tmp); + auto result = _mktemp(dir_buf); + assert(result != nullptr); + BOOL ret = CreateDirectory(dir_buf, NULL); + assert(ret == TRUE); + dir_ = dir_buf; +#else + std::list candidate_dir_list = {"/var/tmp", "/tmp"}; + + // If $TEST_IOCTL_FRIENDLY_TMPDIR/rocksdb.XXXXXX fits, use + // $TEST_IOCTL_FRIENDLY_TMPDIR; subtract 2 for the "%s", and + // add 1 for the trailing NUL byte. + if (tmp && strlen(tmp) + strlen(fmt) - 2 + 1 <= sizeof dir_buf) { + // use $TEST_IOCTL_FRIENDLY_TMPDIR value + candidate_dir_list.push_front(tmp); + } + + for (const std::string& d : candidate_dir_list) { + snprintf(dir_buf, sizeof dir_buf, fmt, d.c_str()); + if (mkdtemp(dir_buf)) { + if (ioctl_support__FS_IOC_GETVERSION(dir_buf)) { + dir_ = dir_buf; + return; + } else { + // Diagnose ioctl-related failure only if this is the + // directory specified via that envvar. + if (tmp && tmp == d) { + fprintf(stderr, + "TEST_IOCTL_FRIENDLY_TMPDIR-specified directory is " + "not suitable: %s\n", + d.c_str()); + } + rmdir(dir_buf); // ignore failure + } + } else { + // mkdtemp failed: diagnose it, but don't give up. + fprintf(stderr, "mkdtemp(%s/...) failed: %s\n", d.c_str(), + errnoStr(errno).c_str()); + } + } + + // check if it's running test within a docker container, in which case, the + // file system inside `overlayfs` may not support FS_IOC_GETVERSION + // skip the tests + struct stat buffer; + if (stat("/.dockerenv", &buffer) == 0) { + is_supported_ = false; + return; + } + + fprintf(stderr, + "failed to find an ioctl-friendly temporary directory;" + " specify one via the TEST_IOCTL_FRIENDLY_TMPDIR envvar\n"); + std::abort(); +#endif + } + + ~IoctlFriendlyTmpdir() { rmdir(dir_.c_str()); } + + const std::string& name() const { return dir_; } + + bool is_supported() const { return is_supported_; } + + private: + std::string dir_; + + bool is_supported_ = true; +}; + +TEST_F(EnvPosixTest, PositionedAppend) { + std::unique_ptr writable_file; + EnvOptions options; + options.use_direct_writes = true; + options.use_mmap_writes = false; + std::string fname = test::PerThreadDBPath(env_, "positioned_append"); + SetupSyncPointsToMockDirectIO(); + + ASSERT_OK(env_->NewWritableFile(fname, &writable_file, options)); + const size_t kBlockSize = 4096; + const size_t kDataSize = kPageSize; + // Write a page worth of 'a' + auto data_ptr = NewAligned(kDataSize, 'a'); + Slice data_a(data_ptr.get(), kDataSize); + ASSERT_OK(writable_file->PositionedAppend(data_a, 0U)); + // Write a page worth of 'b' right after the first sector + data_ptr = NewAligned(kDataSize, 'b'); + Slice data_b(data_ptr.get(), kDataSize); + ASSERT_OK(writable_file->PositionedAppend(data_b, kBlockSize)); + ASSERT_OK(writable_file->Close()); + // The file now has 1 sector worth of a followed by a page worth of b + + // Verify the above + std::unique_ptr seq_file; + ASSERT_OK(env_->NewSequentialFile(fname, &seq_file, options)); + size_t scratch_len = kPageSize * 2; + std::unique_ptr scratch(new char[scratch_len]); + Slice result; + ASSERT_OK(seq_file->Read(scratch_len, &result, scratch.get())); + ASSERT_EQ(kPageSize + kBlockSize, result.size()); + ASSERT_EQ('a', result[kBlockSize - 1]); + ASSERT_EQ('b', result[kBlockSize]); +} + +// `GetUniqueId()` temporarily returns zero on Windows. `BlockBasedTable` can +// handle a return value of zero but this test case cannot. +#ifndef OS_WIN +TEST_P(EnvPosixTestWithParam, RandomAccessUniqueID) { + // Create file. + if (env_ == Env::Default()) { + EnvOptions soptions; + soptions.use_direct_reads = soptions.use_direct_writes = direct_io_; + IoctlFriendlyTmpdir ift; + if (!ift.is_supported()) { + ROCKSDB_GTEST_BYPASS( + "FS_IOC_GETVERSION is not supported by the filesystem"); + return; + } + std::string fname = ift.name() + "/testfile"; + std::unique_ptr wfile; + ASSERT_OK(env_->NewWritableFile(fname, &wfile, soptions)); + + std::unique_ptr file; + + // Get Unique ID + ASSERT_OK(env_->NewRandomAccessFile(fname, &file, soptions)); + size_t id_size = file->GetUniqueId(temp_id, MAX_ID_SIZE); + ASSERT_TRUE(id_size > 0); + std::string unique_id1(temp_id, id_size); + ASSERT_TRUE(IsUniqueIDValid(unique_id1)); + + // Get Unique ID again + ASSERT_OK(env_->NewRandomAccessFile(fname, &file, soptions)); + id_size = file->GetUniqueId(temp_id, MAX_ID_SIZE); + ASSERT_TRUE(id_size > 0); + std::string unique_id2(temp_id, id_size); + ASSERT_TRUE(IsUniqueIDValid(unique_id2)); + + // Get Unique ID again after waiting some time. + env_->SleepForMicroseconds(1000000); + ASSERT_OK(env_->NewRandomAccessFile(fname, &file, soptions)); + id_size = file->GetUniqueId(temp_id, MAX_ID_SIZE); + ASSERT_TRUE(id_size > 0); + std::string unique_id3(temp_id, id_size); + ASSERT_TRUE(IsUniqueIDValid(unique_id3)); + + // Check IDs are the same. + ASSERT_EQ(unique_id1, unique_id2); + ASSERT_EQ(unique_id2, unique_id3); + + // Delete the file + ASSERT_OK(env_->DeleteFile(fname)); + } +} +#endif // !defined(OS_WIN) + +// only works in linux platforms +#ifdef ROCKSDB_FALLOCATE_PRESENT +TEST_P(EnvPosixTestWithParam, AllocateTest) { + if (env_ == Env::Default()) { + SetupSyncPointsToMockDirectIO(); + std::string fname = test::PerThreadDBPath(env_, "preallocate_testfile"); + // Try fallocate in a file to see whether the target file system supports + // it. + // Skip the test if fallocate is not supported. + std::string fname_test_fallocate = + test::PerThreadDBPath(env_, "preallocate_testfile_2"); + int fd = -1; + do { + fd = open(fname_test_fallocate.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644); + } while (fd < 0 && errno == EINTR); + ASSERT_GT(fd, 0); + + int alloc_status = fallocate(fd, 0, 0, 1); + + int err_number = 0; + if (alloc_status != 0) { + err_number = errno; + fprintf(stderr, "Warning: fallocate() fails, %s\n", + errnoStr(err_number).c_str()); + } + close(fd); + ASSERT_OK(env_->DeleteFile(fname_test_fallocate)); + if (alloc_status != 0 && err_number == EOPNOTSUPP) { + // The filesystem containing the file does not support fallocate + return; + } + + EnvOptions soptions; + soptions.use_mmap_writes = false; + soptions.use_direct_reads = soptions.use_direct_writes = direct_io_; + std::unique_ptr wfile; + ASSERT_OK(env_->NewWritableFile(fname, &wfile, soptions)); + + // allocate 100 MB + size_t kPreallocateSize = 100 * 1024 * 1024; + size_t kBlockSize = 512; + size_t kDataSize = 1024 * 1024; + auto data_ptr = NewAligned(kDataSize, 'A'); + Slice data(data_ptr.get(), kDataSize); + wfile->SetPreallocationBlockSize(kPreallocateSize); + wfile->PrepareWrite(wfile->GetFileSize(), kDataSize); + ASSERT_OK(wfile->Append(data)); + ASSERT_OK(wfile->Flush()); + + struct stat f_stat; + ASSERT_EQ(stat(fname.c_str(), &f_stat), 0); + ASSERT_EQ((unsigned int)kDataSize, f_stat.st_size); + // verify that blocks are preallocated + // Note here that we don't check the exact number of blocks preallocated -- + // we only require that number of allocated blocks is at least what we + // expect. + // It looks like some FS give us more blocks that we asked for. That's fine. + // It might be worth investigating further. + ASSERT_LE((unsigned int)(kPreallocateSize / kBlockSize), f_stat.st_blocks); + + // close the file, should deallocate the blocks + wfile.reset(); + + stat(fname.c_str(), &f_stat); + ASSERT_EQ((unsigned int)kDataSize, f_stat.st_size); + // verify that preallocated blocks were deallocated on file close + // Because the FS might give us more blocks, we add a full page to the size + // and expect the number of blocks to be less or equal to that. + ASSERT_GE((f_stat.st_size + kPageSize + kBlockSize - 1) / kBlockSize, + (unsigned int)f_stat.st_blocks); + } +} +#endif // ROCKSDB_FALLOCATE_PRESENT + +// Returns true if any of the strings in ss are the prefix of another string. +bool HasPrefix(const std::unordered_set& ss) { + for (const std::string& s : ss) { + if (s.empty()) { + return true; + } + for (size_t i = 1; i < s.size(); ++i) { + if (ss.count(s.substr(0, i)) != 0) { + return true; + } + } + } + return false; +} + +// `GetUniqueId()` temporarily returns zero on Windows. `BlockBasedTable` can +// handle a return value of zero but this test case cannot. +#ifndef OS_WIN +TEST_P(EnvPosixTestWithParam, RandomAccessUniqueIDConcurrent) { + if (env_ == Env::Default()) { + // Check whether a bunch of concurrently existing files have unique IDs. + EnvOptions soptions; + soptions.use_direct_reads = soptions.use_direct_writes = direct_io_; + + // Create the files + IoctlFriendlyTmpdir ift; + if (!ift.is_supported()) { + ROCKSDB_GTEST_BYPASS( + "FS_IOC_GETVERSION is not supported by the filesystem"); + return; + } + std::vector fnames; + for (int i = 0; i < 1000; ++i) { + fnames.push_back(ift.name() + "/" + "testfile" + std::to_string(i)); + + // Create file. + std::unique_ptr wfile; + ASSERT_OK(env_->NewWritableFile(fnames[i], &wfile, soptions)); + } + + // Collect and check whether the IDs are unique. + std::unordered_set ids; + for (const std::string& fname : fnames) { + std::unique_ptr file; + std::string unique_id; + ASSERT_OK(env_->NewRandomAccessFile(fname, &file, soptions)); + size_t id_size = file->GetUniqueId(temp_id, MAX_ID_SIZE); + ASSERT_TRUE(id_size > 0); + unique_id = std::string(temp_id, id_size); + ASSERT_TRUE(IsUniqueIDValid(unique_id)); + + ASSERT_TRUE(ids.count(unique_id) == 0); + ids.insert(unique_id); + } + + // Delete the files + for (const std::string& fname : fnames) { + ASSERT_OK(env_->DeleteFile(fname)); + } + + ASSERT_TRUE(!HasPrefix(ids)); + } +} + +// TODO: Disable the flaky test, it's a known issue that ext4 may return same +// key after file deletion. The issue is tracked in #7405, #7470. +TEST_P(EnvPosixTestWithParam, DISABLED_RandomAccessUniqueIDDeletes) { + if (env_ == Env::Default()) { + EnvOptions soptions; + soptions.use_direct_reads = soptions.use_direct_writes = direct_io_; + + IoctlFriendlyTmpdir ift; + if (!ift.is_supported()) { + ROCKSDB_GTEST_BYPASS( + "FS_IOC_GETVERSION is not supported by the filesystem"); + return; + } + std::string fname = ift.name() + "/" + "testfile"; + + // Check that after file is deleted we don't get same ID again in a new + // file. + std::unordered_set ids; + for (int i = 0; i < 1000; ++i) { + // Create file. + { + std::unique_ptr wfile; + ASSERT_OK(env_->NewWritableFile(fname, &wfile, soptions)); + } + + // Get Unique ID + std::string unique_id; + { + std::unique_ptr file; + ASSERT_OK(env_->NewRandomAccessFile(fname, &file, soptions)); + size_t id_size = file->GetUniqueId(temp_id, MAX_ID_SIZE); + ASSERT_TRUE(id_size > 0); + unique_id = std::string(temp_id, id_size); + } + + ASSERT_TRUE(IsUniqueIDValid(unique_id)); + ASSERT_TRUE(ids.count(unique_id) == 0); + ids.insert(unique_id); + + // Delete the file + ASSERT_OK(env_->DeleteFile(fname)); + } + + ASSERT_TRUE(!HasPrefix(ids)); + } +} +#endif // !defined(OS_WIN) + +TEST_P(EnvPosixTestWithParam, MultiRead) { + EnvOptions soptions; + soptions.use_direct_reads = soptions.use_direct_writes = direct_io_; + std::string fname = test::PerThreadDBPath(env_, "testfile"); + + const size_t kSectorSize = 4096; + const size_t kNumSectors = 8; + + // Create file. + { + std::unique_ptr wfile; +#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && \ + !defined(OS_AIX) + if (soptions.use_direct_writes) { + soptions.use_direct_writes = false; + } +#endif + ASSERT_OK(env_->NewWritableFile(fname, &wfile, soptions)); + for (size_t i = 0; i < kNumSectors; ++i) { + auto data = NewAligned(kSectorSize * 8, static_cast(i + 1)); + Slice slice(data.get(), kSectorSize); + ASSERT_OK(wfile->Append(slice)); + } + ASSERT_OK(wfile->Close()); + } + + // More attempts to simulate more partial result sequences. + for (uint32_t attempt = 0; attempt < 20; attempt++) { + // Random Read + Random rnd(301 + attempt); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "UpdateResults::io_uring_result", [&](void* arg) { + if (attempt > 0) { + // No failure in the first attempt. + size_t& bytes_read = *static_cast(arg); + if (rnd.OneIn(4)) { + bytes_read = 0; + } else if (rnd.OneIn(3)) { + bytes_read = static_cast( + rnd.Uniform(static_cast(bytes_read))); + } + } + }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + std::unique_ptr file; + std::vector reqs(3); + std::vector> data; + uint64_t offset = 0; + for (size_t i = 0; i < reqs.size(); ++i) { + reqs[i].offset = offset; + offset += 2 * kSectorSize; + reqs[i].len = kSectorSize; + data.emplace_back(NewAligned(kSectorSize, 0)); + reqs[i].scratch = data.back().get(); + } +#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && \ + !defined(OS_AIX) + if (soptions.use_direct_reads) { + soptions.use_direct_reads = false; + } +#endif + ASSERT_OK(env_->NewRandomAccessFile(fname, &file, soptions)); + ASSERT_OK(file->MultiRead(reqs.data(), reqs.size())); + for (size_t i = 0; i < reqs.size(); ++i) { + auto buf = NewAligned(kSectorSize * 8, static_cast(i * 2 + 1)); + ASSERT_OK(reqs[i].status); + ASSERT_EQ(memcmp(reqs[i].scratch, buf.get(), kSectorSize), 0); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + } +} + +TEST_F(EnvPosixTest, MultiReadNonAlignedLargeNum) { + // In this test we don't do aligned read, so it doesn't work for + // direct I/O case. + EnvOptions soptions; + soptions.use_direct_reads = soptions.use_direct_writes = false; + std::string fname = test::PerThreadDBPath(env_, "testfile"); + + const size_t kTotalSize = 81920; + Random rnd(301); + std::string expected_data = rnd.RandomString(kTotalSize); + + // Create file. + { + std::unique_ptr wfile; + ASSERT_OK(env_->NewWritableFile(fname, &wfile, soptions)); + ASSERT_OK(wfile->Append(expected_data)); + ASSERT_OK(wfile->Close()); + } + + // More attempts to simulate more partial result sequences. + for (uint32_t attempt = 0; attempt < 25; attempt++) { + // Right now kIoUringDepth is hard coded as 256, so we need very large + // number of keys to cover the case of multiple rounds of submissions. + // Right now the test latency is still acceptable. If it ends up with + // too long, we can modify the io uring depth with SyncPoint here. + const int num_reads = rnd.Uniform(512) + 1; + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "UpdateResults::io_uring_result", [&](void* arg) { + if (attempt > 5) { + // Improve partial result rates in second half of the run to + // cover the case of repeated partial results. + int odd = (attempt < 15) ? num_reads / 2 : 4; + // No failure in first several attempts. + size_t& bytes_read = *static_cast(arg); + if (rnd.OneIn(odd)) { + bytes_read = 0; + } else if (rnd.OneIn(odd / 2)) { + bytes_read = static_cast( + rnd.Uniform(static_cast(bytes_read))); + } + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Generate (offset, len) pairs + std::set start_offsets; + for (int i = 0; i < num_reads; i++) { + int rnd_off; + // No repeat offsets. + while (start_offsets.find(rnd_off = rnd.Uniform(81920)) != + start_offsets.end()) { + } + start_offsets.insert(rnd_off); + } + std::vector offsets; + std::vector lens; + // std::set already sorted the offsets. + for (int so : start_offsets) { + offsets.push_back(so); + } + for (size_t i = 0; i + 1 < offsets.size(); i++) { + lens.push_back(static_cast( + rnd.Uniform(static_cast(offsets[i + 1] - offsets[i])) + 1)); + } + lens.push_back(static_cast( + rnd.Uniform(static_cast(kTotalSize - offsets.back())) + 1)); + ASSERT_EQ(num_reads, lens.size()); + + // Create requests + std::vector scratches; + scratches.reserve(num_reads); + std::vector reqs(num_reads); + for (size_t i = 0; i < reqs.size(); ++i) { + reqs[i].offset = offsets[i]; + reqs[i].len = lens[i]; + scratches.emplace_back(reqs[i].len, ' '); + reqs[i].scratch = const_cast(scratches.back().data()); + } + + // Query the data + std::unique_ptr file; + ASSERT_OK(env_->NewRandomAccessFile(fname, &file, soptions)); + ASSERT_OK(file->MultiRead(reqs.data(), reqs.size())); + + // Validate results + for (int i = 0; i < num_reads; ++i) { + ASSERT_OK(reqs[i].status); + ASSERT_EQ( + Slice(expected_data.data() + offsets[i], lens[i]).ToString(true), + reqs[i].result.ToString(true)); + } + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + } +} + +TEST_F(EnvPosixTest, NonAlignedDirectIOMultiReadBeyondFileSize) { + EnvOptions soptions; + soptions.use_direct_reads = true; + soptions.use_direct_writes = false; + std::string fname = test::PerThreadDBPath(env_, "testfile"); + + Random rnd(301); + std::unique_ptr wfile; + size_t alignment = 0; + // Create file. + { + ASSERT_OK(env_->NewWritableFile(fname, &wfile, soptions)); + auto data_ptr = NewAligned(4095, 'b'); + Slice data_b(data_ptr.get(), 4095); + ASSERT_OK(wfile->PositionedAppend(data_b, 0U)); + ASSERT_OK(wfile->Close()); + } + +#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && \ + !defined(OS_AIX) && !defined(OS_OPENBSD) && !defined(OS_FREEBSD) + if (soptions.use_direct_reads) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "NewRandomAccessFile:O_DIRECT", [&](void* arg) { + int* val = static_cast(arg); + *val &= ~O_DIRECT; + }); + } +#endif + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + const int num_reads = 2; + // Create requests + std::vector scratches; + scratches.reserve(num_reads); + std::vector reqs(num_reads); + + std::unique_ptr file; + ASSERT_OK(env_->NewRandomAccessFile(fname, &file, soptions)); + alignment = file->GetRequiredBufferAlignment(); + ASSERT_EQ(num_reads, reqs.size()); + + std::vector> data; + + std::vector offsets = {0, 2047}; + std::vector lens = {2047, 4096 - 2047}; + + for (size_t i = 0; i < num_reads; i++) { + // Do alignment + reqs[i].offset = static_cast(TruncateToPageBoundary( + alignment, static_cast(/*offset=*/offsets[i]))); + reqs[i].len = + Roundup(static_cast(/*offset=*/offsets[i]) + /*length=*/lens[i], + alignment) - + reqs[i].offset; + + size_t new_capacity = Roundup(reqs[i].len, alignment); + data.emplace_back(NewAligned(new_capacity, 0)); + reqs[i].scratch = data.back().get(); + } + + // Query the data + ASSERT_OK(file->MultiRead(reqs.data(), reqs.size())); + + // Validate results + for (size_t i = 0; i < num_reads; ++i) { + ASSERT_OK(reqs[i].status); + } + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +#if defined(ROCKSDB_IOURING_PRESENT) +void GenerateFilesAndRequest(Env* env, const std::string& fname, + std::vector* ret_reqs, + std::vector* scratches) { + const size_t kTotalSize = 81920; + Random rnd(301); + std::string expected_data = rnd.RandomString(kTotalSize); + + // Create file. + { + std::unique_ptr wfile; + ASSERT_OK(env->NewWritableFile(fname, &wfile, EnvOptions())); + ASSERT_OK(wfile->Append(expected_data)); + ASSERT_OK(wfile->Close()); + } + + // Right now kIoUringDepth is hard coded as 256, so we need very large + // number of keys to cover the case of multiple rounds of submissions. + // Right now the test latency is still acceptable. If it ends up with + // too long, we can modify the io uring depth with SyncPoint here. + const int num_reads = 3; + std::vector offsets = {10000, 20000, 30000}; + std::vector lens = {3000, 200, 100}; + + // Create requests + scratches->reserve(num_reads); + std::vector& reqs = *ret_reqs; + reqs.resize(num_reads); + for (int i = 0; i < num_reads; ++i) { + reqs[i].offset = offsets[i]; + reqs[i].len = lens[i]; + scratches->emplace_back(reqs[i].len, ' '); + reqs[i].scratch = const_cast(scratches->back().data()); + } +} + +TEST_F(EnvPosixTest, MultiReadIOUringError) { + // In this test we don't do aligned read, so we can't do direct I/O. + EnvOptions soptions; + soptions.use_direct_reads = soptions.use_direct_writes = false; + std::string fname = test::PerThreadDBPath(env_, "testfile"); + + std::vector scratches; + std::vector reqs; + GenerateFilesAndRequest(env_, fname, &reqs, &scratches); + // Query the data + std::unique_ptr file; + ASSERT_OK(env_->NewRandomAccessFile(fname, &file, soptions)); + + bool io_uring_wait_cqe_called = false; + SyncPoint::GetInstance()->SetCallBack( + "PosixRandomAccessFile::MultiRead:io_uring_wait_cqe:return", + [&](void* arg) { + if (!io_uring_wait_cqe_called) { + io_uring_wait_cqe_called = true; + ssize_t& ret = *(static_cast(arg)); + ret = 1; + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + + Status s = file->MultiRead(reqs.data(), reqs.size()); + if (io_uring_wait_cqe_called) { + ASSERT_NOK(s); + } else { + s.PermitUncheckedError(); + } + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(EnvPosixTest, MultiReadIOUringError2) { + // In this test we don't do aligned read, so we can't do direct I/O. + EnvOptions soptions; + soptions.use_direct_reads = soptions.use_direct_writes = false; + std::string fname = test::PerThreadDBPath(env_, "testfile"); + + std::vector scratches; + std::vector reqs; + GenerateFilesAndRequest(env_, fname, &reqs, &scratches); + // Query the data + std::unique_ptr file; + ASSERT_OK(env_->NewRandomAccessFile(fname, &file, soptions)); + + bool io_uring_submit_and_wait_called = false; + SyncPoint::GetInstance()->SetCallBack( + "PosixRandomAccessFile::MultiRead:io_uring_submit_and_wait:return1", + [&](void* arg) { + io_uring_submit_and_wait_called = true; + ssize_t* ret = static_cast(arg); + (*ret)--; + }); + SyncPoint::GetInstance()->SetCallBack( + "PosixRandomAccessFile::MultiRead:io_uring_submit_and_wait:return2", + [&](void* arg) { + struct io_uring* iu = static_cast(arg); + struct io_uring_cqe* cqe; + assert(io_uring_wait_cqe(iu, &cqe) == 0); + io_uring_cqe_seen(iu, cqe); + }); + SyncPoint::GetInstance()->EnableProcessing(); + + Status s = file->MultiRead(reqs.data(), reqs.size()); + if (io_uring_submit_and_wait_called) { + ASSERT_NOK(s); + } else { + s.PermitUncheckedError(); + } + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} +#endif // ROCKSDB_IOURING_PRESENT + +// Only works in linux platforms +#ifdef OS_WIN +TEST_P(EnvPosixTestWithParam, DISABLED_InvalidateCache) { +#else +TEST_P(EnvPosixTestWithParam, InvalidateCache) { +#endif + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + EnvOptions soptions; + soptions.use_direct_reads = soptions.use_direct_writes = direct_io_; + std::string fname = test::PerThreadDBPath(env_, "testfile"); + + const size_t kSectorSize = 512; + auto data = NewAligned(kSectorSize, 0); + Slice slice(data.get(), kSectorSize); + + // Create file. + { + std::unique_ptr wfile; +#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && \ + !defined(OS_AIX) + if (soptions.use_direct_writes) { + soptions.use_direct_writes = false; + } +#endif + ASSERT_OK(env_->NewWritableFile(fname, &wfile, soptions)); + ASSERT_OK(wfile->Append(slice)); + ASSERT_OK(wfile->InvalidateCache(0, 0)); + ASSERT_OK(wfile->Close()); + } + + // Random Read + { + std::unique_ptr file; + auto scratch = NewAligned(kSectorSize, 0); + Slice result; +#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && \ + !defined(OS_AIX) + if (soptions.use_direct_reads) { + soptions.use_direct_reads = false; + } +#endif + ASSERT_OK(env_->NewRandomAccessFile(fname, &file, soptions)); + ASSERT_OK(file->Read(0, kSectorSize, &result, scratch.get())); + ASSERT_EQ(memcmp(scratch.get(), data.get(), kSectorSize), 0); + ASSERT_OK(file->InvalidateCache(0, 11)); + ASSERT_OK(file->InvalidateCache(0, 0)); + } + + // Sequential Read + { + std::unique_ptr file; + auto scratch = NewAligned(kSectorSize, 0); + Slice result; +#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && \ + !defined(OS_AIX) + if (soptions.use_direct_reads) { + soptions.use_direct_reads = false; + } +#endif + ASSERT_OK(env_->NewSequentialFile(fname, &file, soptions)); + if (file->use_direct_io()) { + ASSERT_OK(file->PositionedRead(0, kSectorSize, &result, scratch.get())); + } else { + ASSERT_OK(file->Read(kSectorSize, &result, scratch.get())); + } + ASSERT_EQ(memcmp(scratch.get(), data.get(), kSectorSize), 0); + ASSERT_OK(file->InvalidateCache(0, 11)); + ASSERT_OK(file->InvalidateCache(0, 0)); + } + // Delete the file + ASSERT_OK(env_->DeleteFile(fname)); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace(); +} +#endif // OS_LINUX || OS_WIN + +class TestLogger : public Logger { + public: + using Logger::Logv; + void Logv(const char* format, va_list ap) override { + log_count++; + + char new_format[550]; + std::fill_n(new_format, sizeof(new_format), '2'); + { + va_list backup_ap; + va_copy(backup_ap, ap); + int n = vsnprintf(new_format, sizeof(new_format) - 1, format, backup_ap); + // 48 bytes for extra information + bytes allocated + +// When we have n == -1 there is not a terminating zero expected +#ifdef OS_WIN + if (n < 0) { + char_0_count++; + } +#endif + + if (new_format[0] == '[') { + // "[DEBUG] " + ASSERT_TRUE(n <= 56 + (512 - static_cast(sizeof(port::TimeVal)))); + } else { + ASSERT_TRUE(n <= 48 + (512 - static_cast(sizeof(port::TimeVal)))); + } + va_end(backup_ap); + } + + for (size_t i = 0; i < sizeof(new_format); i++) { + if (new_format[i] == 'x') { + char_x_count++; + } else if (new_format[i] == '\0') { + char_0_count++; + } + } + } + int log_count; + int char_x_count; + int char_0_count; +}; + +TEST_P(EnvPosixTestWithParam, LogBufferTest) { + TestLogger test_logger; + test_logger.SetInfoLogLevel(InfoLogLevel::INFO_LEVEL); + test_logger.log_count = 0; + test_logger.char_x_count = 0; + test_logger.char_0_count = 0; + LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, &test_logger); + LogBuffer log_buffer_debug(DEBUG_LEVEL, &test_logger); + + char bytes200[200]; + std::fill_n(bytes200, sizeof(bytes200), '1'); + bytes200[sizeof(bytes200) - 1] = '\0'; + char bytes600[600]; + std::fill_n(bytes600, sizeof(bytes600), '1'); + bytes600[sizeof(bytes600) - 1] = '\0'; + char bytes9000[9000]; + std::fill_n(bytes9000, sizeof(bytes9000), '1'); + bytes9000[sizeof(bytes9000) - 1] = '\0'; + + ROCKS_LOG_BUFFER(&log_buffer, "x%sx", bytes200); + ROCKS_LOG_BUFFER(&log_buffer, "x%sx", bytes600); + ROCKS_LOG_BUFFER(&log_buffer, "x%sx%sx%sx", bytes200, bytes200, bytes200); + ROCKS_LOG_BUFFER(&log_buffer, "x%sx%sx", bytes200, bytes600); + ROCKS_LOG_BUFFER(&log_buffer, "x%sx%sx", bytes600, bytes9000); + + ROCKS_LOG_BUFFER(&log_buffer_debug, "x%sx", bytes200); + test_logger.SetInfoLogLevel(DEBUG_LEVEL); + ROCKS_LOG_BUFFER(&log_buffer_debug, "x%sx%sx%sx", bytes600, bytes9000, + bytes200); + + ASSERT_EQ(0, test_logger.log_count); + log_buffer.FlushBufferToLog(); + log_buffer_debug.FlushBufferToLog(); + ASSERT_EQ(6, test_logger.log_count); + ASSERT_EQ(6, test_logger.char_0_count); + ASSERT_EQ(10, test_logger.char_x_count); +} + +class TestLogger2 : public Logger { + public: + explicit TestLogger2(size_t max_log_size) : max_log_size_(max_log_size) {} + using Logger::Logv; + void Logv(const char* format, va_list ap) override { + char new_format[2000]; + std::fill_n(new_format, sizeof(new_format), '2'); + { + va_list backup_ap; + va_copy(backup_ap, ap); + int n = vsnprintf(new_format, sizeof(new_format) - 1, format, backup_ap); + // 48 bytes for extra information + bytes allocated + ASSERT_TRUE(n <= + 48 + static_cast(max_log_size_ - sizeof(port::TimeVal))); + ASSERT_TRUE(n > static_cast(max_log_size_ - sizeof(port::TimeVal))); + va_end(backup_ap); + } + } + size_t max_log_size_; +}; + +TEST_P(EnvPosixTestWithParam, LogBufferMaxSizeTest) { + char bytes9000[9000]; + std::fill_n(bytes9000, sizeof(bytes9000), '1'); + bytes9000[sizeof(bytes9000) - 1] = '\0'; + + for (size_t max_log_size = 256; max_log_size <= 1024; + max_log_size += 1024 - 256) { + TestLogger2 test_logger(max_log_size); + test_logger.SetInfoLogLevel(InfoLogLevel::INFO_LEVEL); + LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, &test_logger); + ROCKS_LOG_BUFFER_MAX_SZ(&log_buffer, max_log_size, "%s", bytes9000); + log_buffer.FlushBufferToLog(); + } +} + +TEST_P(EnvPosixTestWithParam, Preallocation) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + const std::string src = test::PerThreadDBPath(env_, "testfile"); + std::unique_ptr srcfile; + EnvOptions soptions; + soptions.use_direct_reads = soptions.use_direct_writes = direct_io_; +#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && \ + !defined(OS_AIX) && !defined(OS_OPENBSD) && !defined(OS_FREEBSD) + if (soptions.use_direct_writes) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "NewWritableFile:O_DIRECT", [&](void* arg) { + int* val = static_cast(arg); + *val &= ~O_DIRECT; + }); + } +#endif + ASSERT_OK(env_->NewWritableFile(src, &srcfile, soptions)); + srcfile->SetPreallocationBlockSize(1024 * 1024); + + // No writes should mean no preallocation + size_t block_size, last_allocated_block; + srcfile->GetPreallocationStatus(&block_size, &last_allocated_block); + ASSERT_EQ(last_allocated_block, 0UL); + + // Small write should preallocate one block + size_t kStrSize = 4096; + auto data = NewAligned(kStrSize, 'A'); + Slice str(data.get(), kStrSize); + srcfile->PrepareWrite(srcfile->GetFileSize(), kStrSize); + ASSERT_OK(srcfile->Append(str)); + srcfile->GetPreallocationStatus(&block_size, &last_allocated_block); + ASSERT_EQ(last_allocated_block, 1UL); + + // Write an entire preallocation block, make sure we increased by two. + { + auto buf_ptr = NewAligned(block_size, ' '); + Slice buf(buf_ptr.get(), block_size); + srcfile->PrepareWrite(srcfile->GetFileSize(), block_size); + ASSERT_OK(srcfile->Append(buf)); + srcfile->GetPreallocationStatus(&block_size, &last_allocated_block); + ASSERT_EQ(last_allocated_block, 2UL); + } + + // Write five more blocks at once, ensure we're where we need to be. + { + auto buf_ptr = NewAligned(block_size * 5, ' '); + Slice buf = Slice(buf_ptr.get(), block_size * 5); + srcfile->PrepareWrite(srcfile->GetFileSize(), buf.size()); + ASSERT_OK(srcfile->Append(buf)); + srcfile->GetPreallocationStatus(&block_size, &last_allocated_block); + ASSERT_EQ(last_allocated_block, 7UL); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace(); +} + +// Test that the two ways to get children file attributes (in bulk or +// individually) behave consistently. +TEST_P(EnvPosixTestWithParam, ConsistentChildrenAttributes) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + EnvOptions soptions; + soptions.use_direct_reads = soptions.use_direct_writes = direct_io_; + const int kNumChildren = 10; + + std::string data; + std::string test_base_dir = test::PerThreadDBPath(env_, "env_test_chr_attr"); + env_->CreateDir(test_base_dir).PermitUncheckedError(); + for (int i = 0; i < kNumChildren; ++i) { + const std::string path = test_base_dir + "/testfile_" + std::to_string(i); + std::unique_ptr file; +#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && \ + !defined(OS_AIX) && !defined(OS_OPENBSD) && !defined(OS_FREEBSD) + if (soptions.use_direct_writes) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "NewWritableFile:O_DIRECT", [&](void* arg) { + int* val = static_cast(arg); + *val &= ~O_DIRECT; + }); + } +#endif + ASSERT_OK(env_->NewWritableFile(path, &file, soptions)); + auto buf_ptr = NewAligned(data.size(), 'T'); + Slice buf(buf_ptr.get(), data.size()); + ASSERT_OK(file->Append(buf)); + data.append(std::string(4096, 'T')); + } + + std::vector file_attrs; + ASSERT_OK(env_->GetChildrenFileAttributes(test_base_dir, &file_attrs)); + for (int i = 0; i < kNumChildren; ++i) { + const std::string name = "testfile_" + std::to_string(i); + const std::string path = test_base_dir + "/" + name; + + auto file_attrs_iter = std::find_if( + file_attrs.begin(), file_attrs.end(), + [&name](const Env::FileAttributes& fm) { return fm.name == name; }); + ASSERT_TRUE(file_attrs_iter != file_attrs.end()); + uint64_t size; + ASSERT_OK(env_->GetFileSize(path, &size)); + ASSERT_EQ(size, 4096 * i); + ASSERT_EQ(size, file_attrs_iter->size_bytes); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace(); +} + +// Test that all WritableFileWrapper forwards all calls to WritableFile. +TEST_P(EnvPosixTestWithParam, WritableFileWrapper) { + class Base : public WritableFile { + public: + mutable int* step_; + + void inc(int x) const { EXPECT_EQ(x, (*step_)++); } + + explicit Base(int* step) : step_(step) { inc(0); } + + Status Append(const Slice& /*data*/) override { + inc(1); + return Status::OK(); + } + + Status Append( + const Slice& /*data*/, + const DataVerificationInfo& /* verification_info */) override { + inc(1); + return Status::OK(); + } + + Status PositionedAppend(const Slice& /*data*/, + uint64_t /*offset*/) override { + inc(2); + return Status::OK(); + } + + Status PositionedAppend( + const Slice& /*data*/, uint64_t /*offset*/, + const DataVerificationInfo& /* verification_info */) override { + inc(2); + return Status::OK(); + } + + Status Truncate(uint64_t /*size*/) override { + inc(3); + return Status::OK(); + } + + Status Close() override { + inc(4); + return Status::OK(); + } + + Status Flush() override { + inc(5); + return Status::OK(); + } + + Status Sync() override { + inc(6); + return Status::OK(); + } + + Status Fsync() override { + inc(7); + return Status::OK(); + } + + bool IsSyncThreadSafe() const override { + inc(8); + return true; + } + + bool use_direct_io() const override { + inc(9); + return true; + } + + size_t GetRequiredBufferAlignment() const override { + inc(10); + return 0; + } + + void SetIOPriority(Env::IOPriority /*pri*/) override { inc(11); } + + Env::IOPriority GetIOPriority() override { + inc(12); + return Env::IOPriority::IO_LOW; + } + + void SetWriteLifeTimeHint(Env::WriteLifeTimeHint /*hint*/) override { + inc(13); + } + + Env::WriteLifeTimeHint GetWriteLifeTimeHint() override { + inc(14); + return Env::WriteLifeTimeHint::WLTH_NOT_SET; + } + + uint64_t GetFileSize() override { + inc(15); + return 0; + } + + void SetPreallocationBlockSize(size_t /*size*/) override { inc(16); } + + void GetPreallocationStatus(size_t* /*block_size*/, + size_t* /*last_allocated_block*/) override { + inc(17); + } + + size_t GetUniqueId(char* /*id*/, size_t /*max_size*/) const override { + inc(18); + return 0; + } + + Status InvalidateCache(size_t /*offset*/, size_t /*length*/) override { + inc(19); + return Status::OK(); + } + + Status RangeSync(uint64_t /*offset*/, uint64_t /*nbytes*/) override { + inc(20); + return Status::OK(); + } + + void PrepareWrite(size_t /*offset*/, size_t /*len*/) override { inc(21); } + + Status Allocate(uint64_t /*offset*/, uint64_t /*len*/) override { + inc(22); + return Status::OK(); + } + + public: + ~Base() override { inc(23); } + }; + + class Wrapper : public WritableFileWrapper { + public: + explicit Wrapper(WritableFile* target) : WritableFileWrapper(target) {} + }; + + int step = 0; + + { + Base b(&step); + Wrapper w(&b); + ASSERT_OK(w.Append(Slice())); + ASSERT_OK(w.PositionedAppend(Slice(), 0)); + ASSERT_OK(w.Truncate(0)); + ASSERT_OK(w.Close()); + ASSERT_OK(w.Flush()); + ASSERT_OK(w.Sync()); + ASSERT_OK(w.Fsync()); + w.IsSyncThreadSafe(); + w.use_direct_io(); + w.GetRequiredBufferAlignment(); + w.SetIOPriority(Env::IOPriority::IO_HIGH); + w.GetIOPriority(); + w.SetWriteLifeTimeHint(Env::WriteLifeTimeHint::WLTH_NOT_SET); + w.GetWriteLifeTimeHint(); + w.GetFileSize(); + w.SetPreallocationBlockSize(0); + w.GetPreallocationStatus(nullptr, nullptr); + w.GetUniqueId(nullptr, 0); + ASSERT_OK(w.InvalidateCache(0, 0)); + ASSERT_OK(w.RangeSync(0, 0)); + w.PrepareWrite(0, 0); + ASSERT_OK(w.Allocate(0, 0)); + } + + EXPECT_EQ(24, step); +} + +TEST_P(EnvPosixTestWithParam, PosixRandomRWFile) { + const std::string path = test::PerThreadDBPath(env_, "random_rw_file"); + + env_->DeleteFile(path).PermitUncheckedError(); + + std::unique_ptr file; + + // Cannot open non-existing file. + ASSERT_NOK(env_->NewRandomRWFile(path, &file, EnvOptions())); + + // Create the file using WritableFile + { + std::unique_ptr wf; + ASSERT_OK(env_->NewWritableFile(path, &wf, EnvOptions())); + } + + ASSERT_OK(env_->NewRandomRWFile(path, &file, EnvOptions())); + + char buf[10000]; + Slice read_res; + + ASSERT_OK(file->Write(0, "ABCD")); + ASSERT_OK(file->Read(0, 10, &read_res, buf)); + ASSERT_EQ(read_res.ToString(), "ABCD"); + + ASSERT_OK(file->Write(2, "XXXX")); + ASSERT_OK(file->Read(0, 10, &read_res, buf)); + ASSERT_EQ(read_res.ToString(), "ABXXXX"); + + ASSERT_OK(file->Write(10, "ZZZ")); + ASSERT_OK(file->Read(10, 10, &read_res, buf)); + ASSERT_EQ(read_res.ToString(), "ZZZ"); + + ASSERT_OK(file->Write(11, "Y")); + ASSERT_OK(file->Read(10, 10, &read_res, buf)); + ASSERT_EQ(read_res.ToString(), "ZYZ"); + + ASSERT_OK(file->Write(200, "FFFFF")); + ASSERT_OK(file->Read(200, 10, &read_res, buf)); + ASSERT_EQ(read_res.ToString(), "FFFFF"); + + ASSERT_OK(file->Write(205, "XXXX")); + ASSERT_OK(file->Read(200, 10, &read_res, buf)); + ASSERT_EQ(read_res.ToString(), "FFFFFXXXX"); + + ASSERT_OK(file->Write(5, "QQQQ")); + ASSERT_OK(file->Read(0, 9, &read_res, buf)); + ASSERT_EQ(read_res.ToString(), "ABXXXQQQQ"); + + ASSERT_OK(file->Read(2, 4, &read_res, buf)); + ASSERT_EQ(read_res.ToString(), "XXXQ"); + + // Close file and reopen it + ASSERT_OK(file->Close()); + ASSERT_OK(env_->NewRandomRWFile(path, &file, EnvOptions())); + + ASSERT_OK(file->Read(0, 9, &read_res, buf)); + ASSERT_EQ(read_res.ToString(), "ABXXXQQQQ"); + + ASSERT_OK(file->Read(10, 3, &read_res, buf)); + ASSERT_EQ(read_res.ToString(), "ZYZ"); + + ASSERT_OK(file->Read(200, 9, &read_res, buf)); + ASSERT_EQ(read_res.ToString(), "FFFFFXXXX"); + + ASSERT_OK(file->Write(4, "TTTTTTTTTTTTTTTT")); + ASSERT_OK(file->Read(0, 10, &read_res, buf)); + ASSERT_EQ(read_res.ToString(), "ABXXTTTTTT"); + + // Clean up + ASSERT_OK(env_->DeleteFile(path)); +} + +class RandomRWFileWithMirrorString { + public: + explicit RandomRWFileWithMirrorString(RandomRWFile* _file) : file_(_file) {} + + void Write(size_t offset, const std::string& data) { + // Write to mirror string + StringWrite(offset, data); + + // Write to file + Status s = file_->Write(offset, data); + ASSERT_OK(s) << s.ToString(); + } + + void Read(size_t offset = 0, size_t n = 1000000) { + Slice str_res(nullptr, 0); + if (offset < file_mirror_.size()) { + size_t str_res_sz = std::min(file_mirror_.size() - offset, n); + str_res = Slice(file_mirror_.data() + offset, str_res_sz); + StopSliceAtNull(&str_res); + } + + Slice file_res; + Status s = file_->Read(offset, n, &file_res, buf_); + ASSERT_OK(s) << s.ToString(); + StopSliceAtNull(&file_res); + + ASSERT_EQ(str_res.ToString(), file_res.ToString()) << offset << " " << n; + } + + void SetFile(RandomRWFile* _file) { file_ = _file; } + + private: + void StringWrite(size_t offset, const std::string& src) { + if (offset + src.size() > file_mirror_.size()) { + file_mirror_.resize(offset + src.size(), '\0'); + } + + char* pos = const_cast(file_mirror_.data() + offset); + memcpy(pos, src.data(), src.size()); + } + + void StopSliceAtNull(Slice* slc) { + for (size_t i = 0; i < slc->size(); i++) { + if ((*slc)[i] == '\0') { + *slc = Slice(slc->data(), i); + break; + } + } + } + + char buf_[10000]; + RandomRWFile* file_; + std::string file_mirror_; +}; + +TEST_P(EnvPosixTestWithParam, PosixRandomRWFileRandomized) { + const std::string path = test::PerThreadDBPath(env_, "random_rw_file_rand"); + env_->DeleteFile(path).PermitUncheckedError(); + + std::unique_ptr file; + +#ifdef OS_LINUX + // Cannot open non-existing file. + ASSERT_NOK(env_->NewRandomRWFile(path, &file, EnvOptions())); +#endif + + // Create the file using WritableFile + { + std::unique_ptr wf; + ASSERT_OK(env_->NewWritableFile(path, &wf, EnvOptions())); + } + + ASSERT_OK(env_->NewRandomRWFile(path, &file, EnvOptions())); + RandomRWFileWithMirrorString file_with_mirror(file.get()); + + Random rnd(301); + std::string buf; + for (int i = 0; i < 10000; i++) { + // Genrate random data + buf = rnd.RandomString(10); + + // Pick random offset for write + size_t write_off = rnd.Next() % 1000; + file_with_mirror.Write(write_off, buf); + + // Pick random offset for read + size_t read_off = rnd.Next() % 1000; + size_t read_sz = rnd.Next() % 20; + file_with_mirror.Read(read_off, read_sz); + + if (i % 500 == 0) { + // Reopen the file every 500 iters + ASSERT_OK(env_->NewRandomRWFile(path, &file, EnvOptions())); + file_with_mirror.SetFile(file.get()); + } + } + + // clean up + ASSERT_OK(env_->DeleteFile(path)); +} + +class TestEnv : public EnvWrapper { + public: + explicit TestEnv() : EnvWrapper(Env::Default()), close_count(0) {} + const char* Name() const override { return "TestEnv"; } + class TestLogger : public Logger { + public: + using Logger::Logv; + explicit TestLogger(TestEnv* env_ptr) : Logger() { env = env_ptr; } + ~TestLogger() override { + if (!closed_) { + Status s = CloseHelper(); + s.PermitUncheckedError(); + } + } + void Logv(const char* /*format*/, va_list /*ap*/) override {} + + protected: + Status CloseImpl() override { return CloseHelper(); } + + private: + Status CloseHelper() { + env->CloseCountInc(); + return Status::OK(); + } + TestEnv* env; + }; + + void CloseCountInc() { close_count++; } + + int GetCloseCount() { return close_count; } + + Status NewLogger(const std::string& /*fname*/, + std::shared_ptr* result) override { + result->reset(new TestLogger(this)); + return Status::OK(); + } + + private: + int close_count; +}; + +class EnvTest : public testing::Test { + public: + EnvTest() : test_directory_(test::PerThreadDBPath("env_test")) {} + + protected: + const std::string test_directory_; +}; + +TEST_F(EnvTest, Close) { + TestEnv* env = new TestEnv(); + std::shared_ptr logger; + Status s; + + s = env->NewLogger("", &logger); + ASSERT_OK(s); + ASSERT_OK(logger.get()->Close()); + ASSERT_EQ(env->GetCloseCount(), 1); + // Call Close() again. CloseHelper() should not be called again + ASSERT_OK(logger.get()->Close()); + ASSERT_EQ(env->GetCloseCount(), 1); + logger.reset(); + ASSERT_EQ(env->GetCloseCount(), 1); + + s = env->NewLogger("", &logger); + ASSERT_OK(s); + logger.reset(); + ASSERT_EQ(env->GetCloseCount(), 2); + + delete env; +} + +class LogvWithInfoLogLevelLogger : public Logger { + public: + using Logger::Logv; + void Logv(const InfoLogLevel /* log_level */, const char* /* format */, + va_list /* ap */) override {} +}; + +TEST_F(EnvTest, LogvWithInfoLogLevel) { + // Verifies the log functions work on a `Logger` that only overrides the + // `Logv()` overload including `InfoLogLevel`. + const std::string kSampleMessage("sample log message"); + LogvWithInfoLogLevelLogger logger; + ROCKS_LOG_HEADER(&logger, "%s", kSampleMessage.c_str()); + ROCKS_LOG_DEBUG(&logger, "%s", kSampleMessage.c_str()); + ROCKS_LOG_INFO(&logger, "%s", kSampleMessage.c_str()); + ROCKS_LOG_WARN(&logger, "%s", kSampleMessage.c_str()); + ROCKS_LOG_ERROR(&logger, "%s", kSampleMessage.c_str()); + ROCKS_LOG_FATAL(&logger, "%s", kSampleMessage.c_str()); +} + +INSTANTIATE_TEST_CASE_P(DefaultEnvWithoutDirectIO, EnvPosixTestWithParam, + ::testing::Values(std::pair(Env::Default(), + false))); +INSTANTIATE_TEST_CASE_P(DefaultEnvWithDirectIO, EnvPosixTestWithParam, + ::testing::Values(std::pair(Env::Default(), + true))); + +#if !defined(OS_WIN) +static Env* GetChrootEnv() { + static std::unique_ptr chroot_env( + NewChrootEnv(Env::Default(), test::TmpDir(Env::Default()))); + return chroot_env.get(); +} +INSTANTIATE_TEST_CASE_P(ChrootEnvWithoutDirectIO, EnvPosixTestWithParam, + ::testing::Values(std::pair(GetChrootEnv(), + false))); +INSTANTIATE_TEST_CASE_P(ChrootEnvWithDirectIO, EnvPosixTestWithParam, + ::testing::Values(std::pair(GetChrootEnv(), + true))); +#endif // !defined(OS_WIN) + +class EnvFSTestWithParam + : public ::testing::Test, + public ::testing::WithParamInterface> { + public: + EnvFSTestWithParam() { + bool env_non_null = std::get<0>(GetParam()); + bool env_default = std::get<1>(GetParam()); + bool fs_default = std::get<2>(GetParam()); + + env_ = env_non_null ? (env_default ? Env::Default() : nullptr) : nullptr; + fs_ = fs_default + ? FileSystem::Default() + : std::make_shared(FileSystem::Default()); + if (env_non_null && env_default && !fs_default) { + env_ptr_ = NewCompositeEnv(fs_); + } + if (env_non_null && !env_default && fs_default) { + env_ptr_ = + std::unique_ptr(new FaultInjectionTestEnv(Env::Default())); + fs_.reset(); + } + if (env_non_null && !env_default && !fs_default) { + env_ptr_.reset(new FaultInjectionTestEnv(Env::Default())); + composite_env_ptr_.reset(new CompositeEnvWrapper(env_ptr_.get(), fs_)); + env_ = composite_env_ptr_.get(); + } else { + env_ = env_ptr_.get(); + } + + dbname1_ = test::PerThreadDBPath("env_fs_test1"); + dbname2_ = test::PerThreadDBPath("env_fs_test2"); + } + + ~EnvFSTestWithParam() = default; + + Env* env_; + std::unique_ptr env_ptr_; + std::unique_ptr composite_env_ptr_; + std::shared_ptr fs_; + std::string dbname1_; + std::string dbname2_; +}; + +TEST_P(EnvFSTestWithParam, OptionsTest) { + Options opts; + opts.env = env_; + opts.create_if_missing = true; + std::string dbname = dbname1_; + + if (env_) { + if (fs_) { + ASSERT_EQ(fs_.get(), env_->GetFileSystem().get()); + } else { + ASSERT_NE(FileSystem::Default().get(), env_->GetFileSystem().get()); + } + } + for (int i = 0; i < 2; ++i) { + DB* db; + Status s = DB::Open(opts, dbname, &db); + ASSERT_OK(s); + + WriteOptions wo; + ASSERT_OK(db->Put(wo, "a", "a")); + ASSERT_OK(db->Flush(FlushOptions())); + ASSERT_OK(db->Put(wo, "b", "b")); + ASSERT_OK(db->Flush(FlushOptions())); + ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + std::string val; + ASSERT_OK(db->Get(ReadOptions(), "a", &val)); + ASSERT_EQ("a", val); + ASSERT_OK(db->Get(ReadOptions(), "b", &val)); + ASSERT_EQ("b", val); + + ASSERT_OK(db->Close()); + delete db; + ASSERT_OK(DestroyDB(dbname, opts)); + + dbname = dbname2_; + } +} + +// The parameters are as follows - +// 1. True means Options::env is non-null, false means null +// 2. True means use Env::Default, false means custom +// 3. True means use FileSystem::Default, false means custom +INSTANTIATE_TEST_CASE_P(EnvFSTest, EnvFSTestWithParam, + ::testing::Combine(::testing::Bool(), ::testing::Bool(), + ::testing::Bool())); +// This test ensures that default Env and those allocated by +// NewCompositeEnv() all share the same threadpool +TEST_F(EnvTest, MultipleCompositeEnv) { + std::shared_ptr fs1 = + std::make_shared(FileSystem::Default()); + std::shared_ptr fs2 = + std::make_shared(FileSystem::Default()); + std::unique_ptr env1 = NewCompositeEnv(fs1); + std::unique_ptr env2 = NewCompositeEnv(fs2); + Env::Default()->SetBackgroundThreads(8, Env::HIGH); + Env::Default()->SetBackgroundThreads(16, Env::LOW); + ASSERT_EQ(env1->GetBackgroundThreads(Env::LOW), 16); + ASSERT_EQ(env1->GetBackgroundThreads(Env::HIGH), 8); + ASSERT_EQ(env2->GetBackgroundThreads(Env::LOW), 16); + ASSERT_EQ(env2->GetBackgroundThreads(Env::HIGH), 8); +} + +TEST_F(EnvTest, IsDirectory) { + Status s = Env::Default()->CreateDirIfMissing(test_directory_); + ASSERT_OK(s); + const std::string test_sub_dir = test_directory_ + "sub1"; + const std::string test_file_path = test_directory_ + "file1"; + ASSERT_OK(Env::Default()->CreateDirIfMissing(test_sub_dir)); + bool is_dir = false; + ASSERT_OK(Env::Default()->IsDirectory(test_sub_dir, &is_dir)); + ASSERT_TRUE(is_dir); + { + std::unique_ptr wfile; + s = Env::Default()->GetFileSystem()->NewWritableFile( + test_file_path, FileOptions(), &wfile, /*dbg=*/nullptr); + ASSERT_OK(s); + std::unique_ptr fwriter; + fwriter.reset(new WritableFileWriter(std::move(wfile), test_file_path, + FileOptions(), + SystemClock::Default().get())); + constexpr char buf[] = "test"; + s = fwriter->Append(buf); + ASSERT_OK(s); + } + ASSERT_OK(Env::Default()->IsDirectory(test_file_path, &is_dir)); + ASSERT_FALSE(is_dir); +} + +TEST_F(EnvTest, EnvWriteVerificationTest) { + Status s = Env::Default()->CreateDirIfMissing(test_directory_); + const std::string test_file_path = test_directory_ + "file1"; + ASSERT_OK(s); + std::shared_ptr fault_fs( + new FaultInjectionTestFS(FileSystem::Default())); + fault_fs->SetChecksumHandoffFuncType(ChecksumType::kCRC32c); + std::unique_ptr fault_fs_env(NewCompositeEnv(fault_fs)); + std::unique_ptr file; + s = fault_fs_env->NewWritableFile(test_file_path, &file, EnvOptions()); + ASSERT_OK(s); + + DataVerificationInfo v_info; + std::string test_data = "test"; + std::string checksum; + uint32_t v_crc32c = crc32c::Extend(0, test_data.c_str(), test_data.size()); + PutFixed32(&checksum, v_crc32c); + v_info.checksum = Slice(checksum); + s = file->Append(Slice(test_data), v_info); + ASSERT_OK(s); +} + +class CreateEnvTest : public testing::Test { + public: + CreateEnvTest() { + config_options_.ignore_unknown_options = false; + config_options_.ignore_unsupported_options = false; + } + ConfigOptions config_options_; +}; + +TEST_F(CreateEnvTest, LoadCTRProvider) { + config_options_.invoke_prepare_options = false; + std::string CTR = CTREncryptionProvider::kClassName(); + std::shared_ptr provider; + // Test a provider with no cipher + ASSERT_OK( + EncryptionProvider::CreateFromString(config_options_, CTR, &provider)); + ASSERT_NE(provider, nullptr); + ASSERT_EQ(provider->Name(), CTR); + ASSERT_NOK(provider->PrepareOptions(config_options_)); + ASSERT_NOK(provider->ValidateOptions(DBOptions(), ColumnFamilyOptions())); + auto cipher = provider->GetOptions>("Cipher"); + ASSERT_NE(cipher, nullptr); + ASSERT_EQ(cipher->get(), nullptr); + provider.reset(); + + ASSERT_OK(EncryptionProvider::CreateFromString(config_options_, + CTR + "://test", &provider)); + ASSERT_NE(provider, nullptr); + ASSERT_EQ(provider->Name(), CTR); + ASSERT_OK(provider->PrepareOptions(config_options_)); + ASSERT_OK(provider->ValidateOptions(DBOptions(), ColumnFamilyOptions())); + cipher = provider->GetOptions>("Cipher"); + ASSERT_NE(cipher, nullptr); + ASSERT_NE(cipher->get(), nullptr); + ASSERT_STREQ(cipher->get()->Name(), "ROT13"); + provider.reset(); + + ASSERT_OK(EncryptionProvider::CreateFromString(config_options_, "1://test", + &provider)); + ASSERT_NE(provider, nullptr); + ASSERT_EQ(provider->Name(), CTR); + ASSERT_OK(provider->PrepareOptions(config_options_)); + ASSERT_OK(provider->ValidateOptions(DBOptions(), ColumnFamilyOptions())); + cipher = provider->GetOptions>("Cipher"); + ASSERT_NE(cipher, nullptr); + ASSERT_NE(cipher->get(), nullptr); + ASSERT_STREQ(cipher->get()->Name(), "ROT13"); + provider.reset(); + + ASSERT_OK(EncryptionProvider::CreateFromString( + config_options_, "id=" + CTR + "; cipher=ROT13", &provider)); + ASSERT_NE(provider, nullptr); + ASSERT_EQ(provider->Name(), CTR); + cipher = provider->GetOptions>("Cipher"); + ASSERT_NE(cipher, nullptr); + ASSERT_NE(cipher->get(), nullptr); + ASSERT_STREQ(cipher->get()->Name(), "ROT13"); + provider.reset(); +} + +TEST_F(CreateEnvTest, LoadROT13Cipher) { + std::shared_ptr cipher; + // Test a provider with no cipher + ASSERT_OK(BlockCipher::CreateFromString(config_options_, "ROT13", &cipher)); + ASSERT_NE(cipher, nullptr); + ASSERT_STREQ(cipher->Name(), "ROT13"); +} + +TEST_F(CreateEnvTest, CreateDefaultSystemClock) { + std::shared_ptr clock, copy; + ASSERT_OK(SystemClock::CreateFromString(config_options_, + SystemClock::kDefaultName(), &clock)); + ASSERT_NE(clock, nullptr); + ASSERT_EQ(clock, SystemClock::Default()); + std::string opts_str = clock->ToString(config_options_); + std::string mismatch; + ASSERT_OK(SystemClock::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(clock->AreEquivalent(config_options_, copy.get(), &mismatch)); +} + +TEST_F(CreateEnvTest, CreateMockSystemClock) { + std::shared_ptr mock, copy; + + config_options_.registry->AddLibrary("test")->AddFactory( + MockSystemClock::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new MockSystemClock(nullptr)); + return guard->get(); + }); + ASSERT_OK(SystemClock::CreateFromString( + config_options_, EmulatedSystemClock::kClassName(), &mock)); + ASSERT_NE(mock, nullptr); + ASSERT_STREQ(mock->Name(), EmulatedSystemClock::kClassName()); + ASSERT_EQ(mock->Inner(), SystemClock::Default().get()); + std::string opts_str = mock->ToString(config_options_); + std::string mismatch; + ASSERT_OK(SystemClock::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(mock->AreEquivalent(config_options_, copy.get(), &mismatch)); + + std::string id = std::string("id=") + EmulatedSystemClock::kClassName() + + ";target=" + MockSystemClock::kClassName(); + + ASSERT_OK(SystemClock::CreateFromString(config_options_, id, &mock)); + ASSERT_NE(mock, nullptr); + ASSERT_STREQ(mock->Name(), EmulatedSystemClock::kClassName()); + ASSERT_NE(mock->Inner(), nullptr); + ASSERT_STREQ(mock->Inner()->Name(), MockSystemClock::kClassName()); + ASSERT_EQ(mock->Inner()->Inner(), SystemClock::Default().get()); + opts_str = mock->ToString(config_options_); + ASSERT_OK(SystemClock::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(mock->AreEquivalent(config_options_, copy.get(), &mismatch)); + ASSERT_OK(SystemClock::CreateFromString( + config_options_, EmulatedSystemClock::kClassName(), &mock)); +} + +TEST_F(CreateEnvTest, CreateReadOnlyFileSystem) { + std::shared_ptr fs, copy; + + ASSERT_OK(FileSystem::CreateFromString( + config_options_, ReadOnlyFileSystem::kClassName(), &fs)); + ASSERT_NE(fs, nullptr); + ASSERT_STREQ(fs->Name(), ReadOnlyFileSystem::kClassName()); + ASSERT_EQ(fs->Inner(), FileSystem::Default().get()); + + std::string opts_str = fs->ToString(config_options_); + std::string mismatch; + + ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch)); + + ASSERT_OK(FileSystem::CreateFromString( + config_options_, + std::string("id=") + ReadOnlyFileSystem::kClassName() + + "; target=" + TimedFileSystem::kClassName(), + &fs)); + ASSERT_NE(fs, nullptr); + opts_str = fs->ToString(config_options_); + ASSERT_STREQ(fs->Name(), ReadOnlyFileSystem::kClassName()); + ASSERT_NE(fs->Inner(), nullptr); + ASSERT_STREQ(fs->Inner()->Name(), TimedFileSystem::kClassName()); + ASSERT_EQ(fs->Inner()->Inner(), FileSystem::Default().get()); + ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch)); +} + +TEST_F(CreateEnvTest, CreateTimedFileSystem) { + std::shared_ptr fs, copy; + + ASSERT_OK(FileSystem::CreateFromString(config_options_, + TimedFileSystem::kClassName(), &fs)); + ASSERT_NE(fs, nullptr); + ASSERT_STREQ(fs->Name(), TimedFileSystem::kClassName()); + ASSERT_EQ(fs->Inner(), FileSystem::Default().get()); + + std::string opts_str = fs->ToString(config_options_); + std::string mismatch; + + ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch)); + + ASSERT_OK(FileSystem::CreateFromString( + config_options_, + std::string("id=") + TimedFileSystem::kClassName() + + "; target=" + ReadOnlyFileSystem::kClassName(), + &fs)); + ASSERT_NE(fs, nullptr); + opts_str = fs->ToString(config_options_); + ASSERT_STREQ(fs->Name(), TimedFileSystem::kClassName()); + ASSERT_NE(fs->Inner(), nullptr); + ASSERT_STREQ(fs->Inner()->Name(), ReadOnlyFileSystem::kClassName()); + ASSERT_EQ(fs->Inner()->Inner(), FileSystem::Default().get()); + ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch)); +} + +TEST_F(CreateEnvTest, CreateCountedFileSystem) { + std::shared_ptr fs, copy; + + ASSERT_OK(FileSystem::CreateFromString(config_options_, + CountedFileSystem::kClassName(), &fs)); + ASSERT_NE(fs, nullptr); + ASSERT_STREQ(fs->Name(), CountedFileSystem::kClassName()); + ASSERT_EQ(fs->Inner(), FileSystem::Default().get()); + + std::string opts_str = fs->ToString(config_options_); + std::string mismatch; + + ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch)); + + ASSERT_OK(FileSystem::CreateFromString( + config_options_, + std::string("id=") + CountedFileSystem::kClassName() + + "; target=" + ReadOnlyFileSystem::kClassName(), + &fs)); + ASSERT_NE(fs, nullptr); + opts_str = fs->ToString(config_options_); + ASSERT_STREQ(fs->Name(), CountedFileSystem::kClassName()); + ASSERT_NE(fs->Inner(), nullptr); + ASSERT_STREQ(fs->Inner()->Name(), ReadOnlyFileSystem::kClassName()); + ASSERT_EQ(fs->Inner()->Inner(), FileSystem::Default().get()); + ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch)); +} + +#ifndef OS_WIN +TEST_F(CreateEnvTest, CreateChrootFileSystem) { + std::shared_ptr fs, copy; + auto tmp_dir = test::TmpDir(Env::Default()); + // The Chroot FileSystem has a required "chroot_dir" option. + ASSERT_NOK(FileSystem::CreateFromString(config_options_, + ChrootFileSystem::kClassName(), &fs)); + + // ChrootFileSystem fails with an invalid directory + ASSERT_NOK(FileSystem::CreateFromString( + config_options_, + std::string("chroot_dir=/No/Such/Directory; id=") + + ChrootFileSystem::kClassName(), + &fs)); + std::string chroot_opts = std::string("chroot_dir=") + tmp_dir + + std::string("; id=") + + ChrootFileSystem::kClassName(); + + // Create a valid ChrootFileSystem with an inner Default + ASSERT_OK(FileSystem::CreateFromString(config_options_, chroot_opts, &fs)); + ASSERT_NE(fs, nullptr); + ASSERT_STREQ(fs->Name(), ChrootFileSystem::kClassName()); + ASSERT_EQ(fs->Inner(), FileSystem::Default().get()); + std::string opts_str = fs->ToString(config_options_); + std::string mismatch; + ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch)); + + // Create a valid ChrootFileSystem with an inner TimedFileSystem + ASSERT_OK(FileSystem::CreateFromString( + config_options_, + chroot_opts + "; target=" + TimedFileSystem::kClassName(), &fs)); + ASSERT_NE(fs, nullptr); + ASSERT_STREQ(fs->Name(), ChrootFileSystem::kClassName()); + ASSERT_NE(fs->Inner(), nullptr); + ASSERT_STREQ(fs->Inner()->Name(), TimedFileSystem::kClassName()); + ASSERT_EQ(fs->Inner()->Inner(), FileSystem::Default().get()); + opts_str = fs->ToString(config_options_); + ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch)); + + // Create a TimedFileSystem with an inner ChrootFileSystem + ASSERT_OK(FileSystem::CreateFromString( + config_options_, + "target={" + chroot_opts + "}; id=" + TimedFileSystem::kClassName(), + &fs)); + ASSERT_NE(fs, nullptr); + ASSERT_STREQ(fs->Name(), TimedFileSystem::kClassName()); + ASSERT_NE(fs->Inner(), nullptr); + ASSERT_STREQ(fs->Inner()->Name(), ChrootFileSystem::kClassName()); + ASSERT_EQ(fs->Inner()->Inner(), FileSystem::Default().get()); + opts_str = fs->ToString(config_options_); + ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch)); +} +#endif // OS_WIN + +TEST_F(CreateEnvTest, CreateEncryptedFileSystem) { + std::shared_ptr fs, copy; + + std::string base_opts = + std::string("provider=1://test; id=") + EncryptedFileSystem::kClassName(); + // The EncryptedFileSystem requires a "provider" option. + ASSERT_NOK(FileSystem::CreateFromString( + config_options_, EncryptedFileSystem::kClassName(), &fs)); + + ASSERT_OK(FileSystem::CreateFromString(config_options_, base_opts, &fs)); + + ASSERT_NE(fs, nullptr); + ASSERT_STREQ(fs->Name(), EncryptedFileSystem::kClassName()); + ASSERT_EQ(fs->Inner(), FileSystem::Default().get()); + std::string opts_str = fs->ToString(config_options_); + std::string mismatch; + ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch)); + ASSERT_OK(FileSystem::CreateFromString( + config_options_, base_opts + "; target=" + TimedFileSystem::kClassName(), + &fs)); + ASSERT_NE(fs, nullptr); + ASSERT_STREQ(fs->Name(), EncryptedFileSystem::kClassName()); + ASSERT_NE(fs->Inner(), nullptr); + ASSERT_STREQ(fs->Inner()->Name(), TimedFileSystem::kClassName()); + ASSERT_EQ(fs->Inner()->Inner(), FileSystem::Default().get()); + opts_str = fs->ToString(config_options_); + ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch)); +} + + +namespace { + +constexpr size_t kThreads = 8; +constexpr size_t kIdsPerThread = 1000; + +// This is a mini-stress test to check for duplicates in functions like +// GenerateUniqueId() +template > +struct NoDuplicateMiniStressTest { + std::unordered_set ids; + std::mutex mutex; + Env* env; + + NoDuplicateMiniStressTest() { env = Env::Default(); } + + virtual ~NoDuplicateMiniStressTest() {} + + void Run() { + std::array threads; + for (size_t i = 0; i < kThreads; ++i) { + threads[i] = std::thread([&]() { ThreadFn(); }); + } + for (auto& thread : threads) { + thread.join(); + } + // All must be unique + ASSERT_EQ(ids.size(), kThreads * kIdsPerThread); + } + + void ThreadFn() { + std::array my_ids; + // Generate in parallel threads as fast as possible + for (size_t i = 0; i < kIdsPerThread; ++i) { + my_ids[i] = Generate(); + } + // Now collate + std::lock_guard lock(mutex); + for (auto& id : my_ids) { + ids.insert(id); + } + } + + virtual IdType Generate() = 0; +}; + +void VerifyRfcUuids(const std::unordered_set& uuids) { + if (uuids.empty()) { + return; + } +} + +using uint64_pair_t = std::pair; +struct HashUint64Pair { + std::size_t operator()( + std::pair const& u) const noexcept { + // Assume suitable distribution already + return static_cast(u.first ^ u.second); + } +}; + +} // namespace + +TEST_F(EnvTest, GenerateUniqueId) { + struct MyStressTest : public NoDuplicateMiniStressTest { + std::string Generate() override { return env->GenerateUniqueId(); } + }; + + MyStressTest t; + t.Run(); + + // Basically verify RFC-4122 format + for (auto& uuid : t.ids) { + ASSERT_EQ(36U, uuid.size()); + ASSERT_EQ('-', uuid[8]); + ASSERT_EQ('-', uuid[13]); + ASSERT_EQ('-', uuid[18]); + ASSERT_EQ('-', uuid[23]); + } +} + +TEST_F(EnvTest, GenerateDbSessionId) { + struct MyStressTest : public NoDuplicateMiniStressTest { + std::string Generate() override { return DBImpl::GenerateDbSessionId(env); } + }; + + MyStressTest t; + t.Run(); + + // Basically verify session ID + for (auto& id : t.ids) { + ASSERT_EQ(20U, id.size()); + } +} + +constexpr bool kRequirePortGenerateRfcUuid = +#if defined(OS_LINUX) || defined(OS_ANDROID) || defined(OS_WIN) + true; +#else + false; +#endif + +TEST_F(EnvTest, PortGenerateRfcUuid) { + if (!kRequirePortGenerateRfcUuid) { + ROCKSDB_GTEST_SKIP("Not supported/expected on this platform"); + return; + } + struct MyStressTest : public NoDuplicateMiniStressTest { + std::string Generate() override { + std::string u; + assert(port::GenerateRfcUuid(&u)); + return u; + } + }; + + MyStressTest t; + t.Run(); + + // Extra verification on versions and variants + VerifyRfcUuids(t.ids); +} + +// Test the atomic, linear generation of GenerateRawUniqueId +TEST_F(EnvTest, GenerateRawUniqueId) { + struct MyStressTest + : public NoDuplicateMiniStressTest { + uint64_pair_t Generate() override { + uint64_pair_t p; + GenerateRawUniqueId(&p.first, &p.second); + return p; + } + }; + + MyStressTest t; + t.Run(); +} + +// Test that each entropy source ("track") is at least adequate +TEST_F(EnvTest, GenerateRawUniqueIdTrackPortUuidOnly) { + if (!kRequirePortGenerateRfcUuid) { + ROCKSDB_GTEST_SKIP("Not supported/expected on this platform"); + return; + } + + struct MyStressTest + : public NoDuplicateMiniStressTest { + uint64_pair_t Generate() override { + uint64_pair_t p; + TEST_GenerateRawUniqueId(&p.first, &p.second, false, true, true); + return p; + } + }; + + MyStressTest t; + t.Run(); +} + +TEST_F(EnvTest, GenerateRawUniqueIdTrackEnvDetailsOnly) { + struct MyStressTest + : public NoDuplicateMiniStressTest { + uint64_pair_t Generate() override { + uint64_pair_t p; + TEST_GenerateRawUniqueId(&p.first, &p.second, true, false, true); + return p; + } + }; + + MyStressTest t; + t.Run(); +} + +TEST_F(EnvTest, GenerateRawUniqueIdTrackRandomDeviceOnly) { + struct MyStressTest + : public NoDuplicateMiniStressTest { + uint64_pair_t Generate() override { + uint64_pair_t p; + TEST_GenerateRawUniqueId(&p.first, &p.second, true, true, false); + return p; + } + }; + + MyStressTest t; + t.Run(); +} + +TEST_F(EnvTest, SemiStructuredUniqueIdGenTest) { + // Must be thread safe and usable as a static + static SemiStructuredUniqueIdGen gen; + + struct MyStressTest + : public NoDuplicateMiniStressTest { + uint64_pair_t Generate() override { + uint64_pair_t p; + gen.GenerateNext(&p.first, &p.second); + return p; + } + }; + + MyStressTest t; + t.Run(); +} + +TEST_F(EnvTest, SemiStructuredUniqueIdGenTestSmaller) { + // For small generated types, will cycle through all the possible values. + SemiStructuredUniqueIdGen gen; + std::vector hit(256); + for (int i = 0; i < 256; ++i) { + auto val = gen.GenerateNext(); + ASSERT_FALSE(hit[val]); + hit[val] = true; + } + for (int i = 0; i < 256; ++i) { + ASSERT_TRUE(hit[i]); + } +} + +TEST_F(EnvTest, UnpredictableUniqueIdGenTest1) { + // Must be thread safe and usable as a static. + static UnpredictableUniqueIdGen gen; + + struct MyStressTest + : public NoDuplicateMiniStressTest { + uint64_pair_t Generate() override { + uint64_pair_t p; + gen.GenerateNext(&p.first, &p.second); + return p; + } + }; + + MyStressTest t; + t.Run(); +} + +TEST_F(EnvTest, UnpredictableUniqueIdGenTest2) { + // Even if we completely strip the seeding and entropy of the structure + // down to a bare minimum, we still get quality pseudorandom results. + static UnpredictableUniqueIdGen gen{ + UnpredictableUniqueIdGen::TEST_ZeroInitialized{}}; + + struct MyStressTest + : public NoDuplicateMiniStressTest { + uint64_pair_t Generate() override { + uint64_pair_t p; + // No extra entropy is required to get quality pseudorandom results + gen.GenerateNextWithEntropy(&p.first, &p.second, /*no extra entropy*/ 0); + return p; + } + }; + + MyStressTest t; + t.Run(); +} + +TEST_F(EnvTest, UnpredictableUniqueIdGenTest3) { + struct MyStressTest + : public NoDuplicateMiniStressTest { + uint64_pair_t Generate() override { + uint64_pair_t p; + thread_local UnpredictableUniqueIdGen gen{ + UnpredictableUniqueIdGen::TEST_ZeroInitialized{}}; + // Even without the counter (reset it to thread id), we get quality + // single-threaded results (because part of each result is fed back + // into pool). + gen.TEST_counter().store(Env::Default()->GetThreadID()); + gen.GenerateNext(&p.first, &p.second); + return p; + } + }; + + MyStressTest t; + t.Run(); +} + +TEST_F(EnvTest, UnpredictableUniqueIdGenTest4) { + struct MyStressTest + : public NoDuplicateMiniStressTest { + uint64_pair_t Generate() override { + uint64_pair_t p; + // Even if we reset the state to thread ID each time, RDTSC instruction + // suffices for quality single-threaded results. + UnpredictableUniqueIdGen gen{ + UnpredictableUniqueIdGen::TEST_ZeroInitialized{}}; + gen.TEST_counter().store(Env::Default()->GetThreadID()); + gen.GenerateNext(&p.first, &p.second); + return p; + } + }; + + MyStressTest t; +#ifdef __SSE4_2__ // Our rough check for RDTSC + t.Run(); +#else + ROCKSDB_GTEST_BYPASS("Requires IA32 with RDTSC"); + // because nanosecond time might not be high enough fidelity to have + // incremented after a few hundred instructions, especially in cases where + // we really only have microsecond fidelity. Also, wall clock might not be + // monotonic. +#endif +} + +TEST_F(EnvTest, FailureToCreateLockFile) { + auto env = Env::Default(); + auto fs = env->GetFileSystem(); + std::string dir = test::PerThreadDBPath(env, "lockdir"); + std::string file = dir + "/lockfile"; + + // Ensure directory doesn't exist + ASSERT_OK(DestroyDir(env, dir)); + + // Make sure that we can acquire a file lock after the first attempt fails + FileLock* lock = nullptr; + ASSERT_NOK(fs->LockFile(file, IOOptions(), &lock, /*dbg*/ nullptr)); + ASSERT_FALSE(lock); + + ASSERT_OK(fs->CreateDir(dir, IOOptions(), /*dbg*/ nullptr)); + ASSERT_OK(fs->LockFile(file, IOOptions(), &lock, /*dbg*/ nullptr)); + ASSERT_OK(fs->UnlockFile(lock, IOOptions(), /*dbg*/ nullptr)); + + // Clean up + ASSERT_OK(DestroyDir(env, dir)); +} + +TEST_F(CreateEnvTest, CreateDefaultEnv) { + ConfigOptions options; + options.ignore_unsupported_options = false; + + std::shared_ptr guard; + Env* env = nullptr; + ASSERT_OK(Env::CreateFromString(options, "", &env)); + ASSERT_EQ(env, Env::Default()); + + env = nullptr; + ASSERT_OK(Env::CreateFromString(options, Env::kDefaultName(), &env)); + ASSERT_EQ(env, Env::Default()); + + env = nullptr; + ASSERT_OK(Env::CreateFromString(options, "", &env, &guard)); + ASSERT_EQ(env, Env::Default()); + ASSERT_EQ(guard, nullptr); + + env = nullptr; + ASSERT_OK(Env::CreateFromString(options, Env::kDefaultName(), &env, &guard)); + ASSERT_EQ(env, Env::Default()); + ASSERT_EQ(guard, nullptr); + + std::string opt_str = env->ToString(options); + ASSERT_OK(Env::CreateFromString(options, opt_str, &env)); + ASSERT_EQ(env, Env::Default()); + ASSERT_OK(Env::CreateFromString(options, opt_str, &env, &guard)); + ASSERT_EQ(env, Env::Default()); + ASSERT_EQ(guard, nullptr); +} + +namespace { +class WrappedEnv : public EnvWrapper { + public: + explicit WrappedEnv(Env* t) : EnvWrapper(t) {} + explicit WrappedEnv(const std::shared_ptr& t) : EnvWrapper(t) {} + static const char* kClassName() { return "WrappedEnv"; } + const char* Name() const override { return kClassName(); } + static void Register(ObjectLibrary& lib, const std::string& /*arg*/) { + lib.AddFactory( + WrappedEnv::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new WrappedEnv(nullptr)); + return guard->get(); + }); + } +}; +} // namespace +TEST_F(CreateEnvTest, CreateMockEnv) { + ConfigOptions options; + options.ignore_unsupported_options = false; + WrappedEnv::Register(*(options.registry->AddLibrary("test")), ""); + std::shared_ptr guard, copy; + std::string opt_str; + + Env* env = nullptr; + ASSERT_NOK(Env::CreateFromString(options, MockEnv::kClassName(), &env)); + ASSERT_OK( + Env::CreateFromString(options, MockEnv::kClassName(), &env, &guard)); + ASSERT_NE(env, nullptr); + ASSERT_NE(env, Env::Default()); + opt_str = env->ToString(options); + ASSERT_OK(Env::CreateFromString(options, opt_str, &env, ©)); + ASSERT_NE(copy, guard); + std::string mismatch; + ASSERT_TRUE(guard->AreEquivalent(options, copy.get(), &mismatch)); + guard.reset(MockEnv::Create(Env::Default(), SystemClock::Default())); + opt_str = guard->ToString(options); + ASSERT_OK(Env::CreateFromString(options, opt_str, &env, ©)); + std::unique_ptr wrapped_env(new WrappedEnv(Env::Default())); + guard.reset(MockEnv::Create(wrapped_env.get(), SystemClock::Default())); + opt_str = guard->ToString(options); + ASSERT_OK(Env::CreateFromString(options, opt_str, &env, ©)); + opt_str = copy->ToString(options); +} + +TEST_F(CreateEnvTest, CreateWrappedEnv) { + ConfigOptions options; + options.ignore_unsupported_options = false; + WrappedEnv::Register(*(options.registry->AddLibrary("test")), ""); + Env* env = nullptr; + std::shared_ptr guard, copy; + std::string opt_str; + std::string mismatch; + + ASSERT_NOK(Env::CreateFromString(options, WrappedEnv::kClassName(), &env)); + ASSERT_OK( + Env::CreateFromString(options, WrappedEnv::kClassName(), &env, &guard)); + ASSERT_NE(env, nullptr); + ASSERT_NE(env, Env::Default()); + ASSERT_FALSE(guard->AreEquivalent(options, Env::Default(), &mismatch)); + + opt_str = env->ToString(options); + ASSERT_OK(Env::CreateFromString(options, opt_str, &env, ©)); + ASSERT_NE(copy, guard); + ASSERT_TRUE(guard->AreEquivalent(options, copy.get(), &mismatch)); + + guard.reset(new WrappedEnv(std::make_shared(Env::Default()))); + ASSERT_NE(guard.get(), env); + opt_str = guard->ToString(options); + ASSERT_OK(Env::CreateFromString(options, opt_str, &env, ©)); + ASSERT_NE(copy, guard); + ASSERT_TRUE(guard->AreEquivalent(options, copy.get(), &mismatch)); + + guard.reset(new WrappedEnv(std::make_shared( + std::make_shared(Env::Default())))); + ASSERT_NE(guard.get(), env); + opt_str = guard->ToString(options); + ASSERT_OK(Env::CreateFromString(options, opt_str, &env, ©)); + ASSERT_NE(copy, guard); + ASSERT_TRUE(guard->AreEquivalent(options, copy.get(), &mismatch)); +} + +TEST_F(CreateEnvTest, CreateCompositeEnv) { + ConfigOptions options; + options.ignore_unsupported_options = false; + std::shared_ptr guard, copy; + Env* env = nullptr; + std::string mismatch, opt_str; + + WrappedEnv::Register(*(options.registry->AddLibrary("test")), ""); + std::unique_ptr base(NewCompositeEnv(FileSystem::Default())); + std::unique_ptr wrapped(new WrappedEnv(Env::Default())); + std::shared_ptr timed_fs = + std::make_shared(FileSystem::Default()); + std::shared_ptr clock = + std::make_shared(SystemClock::Default()); + + opt_str = base->ToString(options); + ASSERT_NOK(Env::CreateFromString(options, opt_str, &env)); + ASSERT_OK(Env::CreateFromString(options, opt_str, &env, &guard)); + ASSERT_NE(env, nullptr); + ASSERT_NE(env, Env::Default()); + ASSERT_EQ(env->GetFileSystem(), FileSystem::Default()); + ASSERT_EQ(env->GetSystemClock(), SystemClock::Default()); + + base = NewCompositeEnv(timed_fs); + opt_str = base->ToString(options); + ASSERT_NOK(Env::CreateFromString(options, opt_str, &env)); + ASSERT_OK(Env::CreateFromString(options, opt_str, &env, &guard)); + ASSERT_NE(env, nullptr); + ASSERT_NE(env, Env::Default()); + ASSERT_NE(env->GetFileSystem(), FileSystem::Default()); + ASSERT_EQ(env->GetSystemClock(), SystemClock::Default()); + + env = nullptr; + guard.reset(new CompositeEnvWrapper(wrapped.get(), timed_fs)); + opt_str = guard->ToString(options); + ASSERT_OK(Env::CreateFromString(options, opt_str, &env, ©)); + ASSERT_NE(env, nullptr); + ASSERT_NE(env, Env::Default()); + ASSERT_TRUE(guard->AreEquivalent(options, copy.get(), &mismatch)); + + env = nullptr; + guard.reset(new CompositeEnvWrapper(wrapped.get(), clock)); + opt_str = guard->ToString(options); + ASSERT_OK(Env::CreateFromString(options, opt_str, &env, ©)); + ASSERT_NE(env, nullptr); + ASSERT_NE(env, Env::Default()); + ASSERT_TRUE(guard->AreEquivalent(options, copy.get(), &mismatch)); + + env = nullptr; + guard.reset(new CompositeEnvWrapper(wrapped.get(), timed_fs, clock)); + opt_str = guard->ToString(options); + ASSERT_OK(Env::CreateFromString(options, opt_str, &env, ©)); + ASSERT_NE(env, nullptr); + ASSERT_NE(env, Env::Default()); + ASSERT_TRUE(guard->AreEquivalent(options, copy.get(), &mismatch)); + + guard.reset(new CompositeEnvWrapper(nullptr, timed_fs, clock)); + ColumnFamilyOptions cf_opts; + DBOptions db_opts; + db_opts.env = guard.get(); + auto comp = db_opts.env->CheckedCast(); + ASSERT_NE(comp, nullptr); + ASSERT_EQ(comp->Inner(), nullptr); + ASSERT_NOK(ValidateOptions(db_opts, cf_opts)); + ASSERT_OK(db_opts.env->PrepareOptions(options)); + ASSERT_NE(comp->Inner(), nullptr); + ASSERT_OK(ValidateOptions(db_opts, cf_opts)); +} + +// Forward declaration +class ReadAsyncFS; + +struct MockIOHandle { + std::function cb; + void* cb_arg; + bool create_io_error; +}; + +// ReadAsyncFS and ReadAsyncRandomAccessFile mocks the FS doing asynchronous +// reads by creating threads that submit read requests and then calling Poll API +// to obtain those results. +class ReadAsyncRandomAccessFile : public FSRandomAccessFileOwnerWrapper { + public: + ReadAsyncRandomAccessFile(ReadAsyncFS& fs, + std::unique_ptr& file) + : FSRandomAccessFileOwnerWrapper(std::move(file)), fs_(fs) {} + + IOStatus ReadAsync(FSReadRequest& req, const IOOptions& opts, + std::function cb, + void* cb_arg, void** io_handle, IOHandleDeleter* del_fn, + IODebugContext* dbg) override; + + private: + ReadAsyncFS& fs_; + std::unique_ptr file_; + int counter = 0; +}; + +class ReadAsyncFS : public FileSystemWrapper { + public: + explicit ReadAsyncFS(const std::shared_ptr& wrapped) + : FileSystemWrapper(wrapped) {} + + static const char* kClassName() { return "ReadAsyncFS"; } + const char* Name() const override { return kClassName(); } + + IOStatus NewRandomAccessFile(const std::string& fname, + const FileOptions& opts, + std::unique_ptr* result, + IODebugContext* dbg) override { + std::unique_ptr file; + IOStatus s = target()->NewRandomAccessFile(fname, opts, &file, dbg); + EXPECT_OK(s); + result->reset(new ReadAsyncRandomAccessFile(*this, file)); + return s; + } + + IOStatus Poll(std::vector& io_handles, + size_t /*min_completions*/) override { + // Wait for the threads completion. + for (auto& t : workers) { + t.join(); + } + + for (size_t i = 0; i < io_handles.size(); i++) { + MockIOHandle* handle = static_cast(io_handles[i]); + if (handle->create_io_error) { + FSReadRequest req; + req.status = IOStatus::IOError(); + handle->cb(req, handle->cb_arg); + } + } + return IOStatus::OK(); + } + + std::vector workers; +}; + +IOStatus ReadAsyncRandomAccessFile::ReadAsync( + FSReadRequest& req, const IOOptions& opts, + std::function cb, void* cb_arg, + void** io_handle, IOHandleDeleter* del_fn, IODebugContext* dbg) { + IOHandleDeleter deletefn = [](void* args) -> void { + delete (static_cast(args)); + args = nullptr; + }; + *del_fn = deletefn; + + // Allocate and populate io_handle. + MockIOHandle* mock_handle = new MockIOHandle(); + bool create_io_error = false; + if (counter % 2) { + create_io_error = true; + } + mock_handle->create_io_error = create_io_error; + mock_handle->cb = cb; + mock_handle->cb_arg = cb_arg; + *io_handle = static_cast(mock_handle); + counter++; + + // Submit read request asynchronously. + std::function submit_request = + [&opts, cb, cb_arg, dbg, create_io_error, this](FSReadRequest _req) { + if (!create_io_error) { + _req.status = target()->Read(_req.offset, _req.len, opts, + &(_req.result), _req.scratch, dbg); + cb(_req, cb_arg); + } + }; + + fs_.workers.emplace_back(submit_request, std::move(req)); + return IOStatus::OK(); +} + +class TestAsyncRead : public testing::Test { + public: + TestAsyncRead() { env_ = Env::Default(); } + Env* env_; +}; + +// Tests the default implementation of ReadAsync API. +TEST_F(TestAsyncRead, ReadAsync) { + EnvOptions soptions; + std::shared_ptr fs = + std::make_shared(env_->GetFileSystem()); + + std::string fname = test::PerThreadDBPath(env_, "testfile"); + + const size_t kSectorSize = 4096; + const size_t kNumSectors = 8; + + // 1. create & write to a file. + { + std::unique_ptr wfile; + ASSERT_OK( + fs->NewWritableFile(fname, FileOptions(), &wfile, nullptr /*dbg*/)); + + for (size_t i = 0; i < kNumSectors; ++i) { + auto data = NewAligned(kSectorSize * 8, static_cast(i + 1)); + Slice slice(data.get(), kSectorSize); + ASSERT_OK(wfile->Append(slice, IOOptions(), nullptr)); + } + ASSERT_OK(wfile->Close(IOOptions(), nullptr)); + } + // 2. Read file + { + std::unique_ptr file; + ASSERT_OK(fs->NewRandomAccessFile(fname, FileOptions(), &file, nullptr)); + + IOOptions opts; + std::vector io_handles(kNumSectors); + std::vector reqs(kNumSectors); + std::vector> data; + std::vector vals; + IOHandleDeleter del_fn; + uint64_t offset = 0; + + // Initialize read requests + for (size_t i = 0; i < kNumSectors; i++) { + reqs[i].offset = offset; + reqs[i].len = kSectorSize; + data.emplace_back(NewAligned(kSectorSize, 0)); + reqs[i].scratch = data.back().get(); + vals.push_back(i); + offset += kSectorSize; + } + + // callback function passed to async read. + std::function callback = + [&](const FSReadRequest& req, void* cb_arg) { + assert(cb_arg != nullptr); + size_t i = *(reinterpret_cast(cb_arg)); + reqs[i].offset = req.offset; + reqs[i].result = req.result; + reqs[i].status = req.status; + }; + + // Submit asynchronous read requests. + for (size_t i = 0; i < kNumSectors; i++) { + void* cb_arg = static_cast(&(vals[i])); + ASSERT_OK(file->ReadAsync(reqs[i], opts, callback, cb_arg, + &(io_handles[i]), &del_fn, nullptr)); + } + + // Poll for the submitted requests. + fs->Poll(io_handles, kNumSectors); + + // Check the status of read requests. + for (size_t i = 0; i < kNumSectors; i++) { + if (i % 2) { + ASSERT_EQ(reqs[i].status, IOStatus::IOError()); + } else { + auto buf = NewAligned(kSectorSize * 8, static_cast(i + 1)); + Slice expected_data(buf.get(), kSectorSize); + + ASSERT_EQ(reqs[i].offset, i * kSectorSize); + ASSERT_OK(reqs[i].status); + ASSERT_EQ(expected_data.ToString(), reqs[i].result.ToString()); + } + } + + // Delete io_handles. + for (size_t i = 0; i < io_handles.size(); i++) { + del_fn(io_handles[i]); + } + } +} + +struct StaticDestructionTester { + bool activated = false; + ~StaticDestructionTester() { + if (activated && !kMustFreeHeapAllocations) { + // Make sure we can still call some things on default Env. + std::string hostname; + Env::Default()->GetHostNameString(&hostname); + } + } +} static_destruction_tester; + +TEST(EnvTestMisc, StaticDestruction) { + // Check for any crashes during static destruction. + static_destruction_tester.activated = true; +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/env/file_system.cc b/librocksdb-sys/rocksdb/env/file_system.cc new file mode 100644 index 0000000..71fb4d5 --- /dev/null +++ b/librocksdb-sys/rocksdb/env/file_system.cc @@ -0,0 +1,277 @@ +// Copyright (c) 2019-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#include "rocksdb/file_system.h" + +#include "env/composite_env_wrapper.h" +#include "env/env_chroot.h" +#include "env/env_encryption_ctr.h" +#include "env/fs_readonly.h" +#include "env/mock_env.h" +#include "logging/env_logger.h" +#include "options/db_options.h" +#include "rocksdb/convenience.h" +#include "rocksdb/utilities/customizable_util.h" +#include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/options_type.h" +#include "util/string_util.h" +#include "utilities/counted_fs.h" +#include "utilities/env_timed.h" + +namespace ROCKSDB_NAMESPACE { + +FileSystem::FileSystem() {} + +FileSystem::~FileSystem() {} + +static int RegisterBuiltinFileSystems(ObjectLibrary& library, + const std::string& /*arg*/) { + library.AddFactory( + TimedFileSystem::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new TimedFileSystem(nullptr)); + return guard->get(); + }); + library.AddFactory( + ReadOnlyFileSystem::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new ReadOnlyFileSystem(nullptr)); + return guard->get(); + }); + library.AddFactory( + EncryptedFileSystem::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* errmsg) { + Status s = NewEncryptedFileSystemImpl(nullptr, nullptr, guard); + if (!s.ok()) { + *errmsg = s.ToString(); + } + return guard->get(); + }); + library.AddFactory( + CountedFileSystem::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /*errmsg*/) { + guard->reset(new CountedFileSystem(FileSystem::Default())); + return guard->get(); + }); + library.AddFactory( + MockFileSystem::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /*errmsg*/) { + guard->reset(new MockFileSystem(SystemClock::Default())); + return guard->get(); + }); +#ifndef OS_WIN + library.AddFactory( + ChrootFileSystem::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new ChrootFileSystem(nullptr, "")); + return guard->get(); + }); +#endif // OS_WIN + size_t num_types; + return static_cast(library.GetFactoryCount(&num_types)); +} + +Status FileSystem::CreateFromString(const ConfigOptions& config_options, + const std::string& value, + std::shared_ptr* result) { + auto default_fs = FileSystem::Default(); + if (default_fs->IsInstanceOf(value)) { + *result = default_fs; + return Status::OK(); + } else { + static std::once_flag once; + std::call_once(once, [&]() { + RegisterBuiltinFileSystems(*(ObjectLibrary::Default().get()), ""); + }); + return LoadSharedObject(config_options, value, result); + } +} + +IOStatus FileSystem::ReuseWritableFile(const std::string& fname, + const std::string& old_fname, + const FileOptions& opts, + std::unique_ptr* result, + IODebugContext* dbg) { + IOStatus s = RenameFile(old_fname, fname, opts.io_options, dbg); + if (!s.ok()) { + return s; + } + return NewWritableFile(fname, opts, result, dbg); +} + +IOStatus FileSystem::NewLogger(const std::string& fname, + const IOOptions& io_opts, + std::shared_ptr* result, + IODebugContext* dbg) { + FileOptions options; + options.io_options = io_opts; + // TODO: Tune the buffer size. + options.writable_file_max_buffer_size = 1024 * 1024; + std::unique_ptr writable_file; + const IOStatus status = NewWritableFile(fname, options, &writable_file, dbg); + if (!status.ok()) { + return status; + } + + *result = std::make_shared(std::move(writable_file), fname, + options, Env::Default()); + return IOStatus::OK(); +} + +FileOptions FileSystem::OptimizeForLogRead( + const FileOptions& file_options) const { + FileOptions optimized_file_options(file_options); + optimized_file_options.use_direct_reads = false; + return optimized_file_options; +} + +FileOptions FileSystem::OptimizeForManifestRead( + const FileOptions& file_options) const { + FileOptions optimized_file_options(file_options); + optimized_file_options.use_direct_reads = false; + return optimized_file_options; +} + +FileOptions FileSystem::OptimizeForLogWrite(const FileOptions& file_options, + const DBOptions& db_options) const { + FileOptions optimized_file_options(file_options); + optimized_file_options.bytes_per_sync = db_options.wal_bytes_per_sync; + optimized_file_options.writable_file_max_buffer_size = + db_options.writable_file_max_buffer_size; + return optimized_file_options; +} + +FileOptions FileSystem::OptimizeForManifestWrite( + const FileOptions& file_options) const { + return file_options; +} + +FileOptions FileSystem::OptimizeForCompactionTableWrite( + const FileOptions& file_options, + const ImmutableDBOptions& db_options) const { + FileOptions optimized_file_options(file_options); + optimized_file_options.use_direct_writes = + db_options.use_direct_io_for_flush_and_compaction; + return optimized_file_options; +} + +FileOptions FileSystem::OptimizeForCompactionTableRead( + const FileOptions& file_options, + const ImmutableDBOptions& db_options) const { + FileOptions optimized_file_options(file_options); + optimized_file_options.use_direct_reads = db_options.use_direct_reads; + return optimized_file_options; +} + +FileOptions FileSystem::OptimizeForBlobFileRead( + const FileOptions& file_options, + const ImmutableDBOptions& db_options) const { + FileOptions optimized_file_options(file_options); + optimized_file_options.use_direct_reads = db_options.use_direct_reads; + return optimized_file_options; +} + +IOStatus WriteStringToFile(FileSystem* fs, const Slice& data, + const std::string& fname, bool should_sync) { + std::unique_ptr file; + EnvOptions soptions; + IOStatus s = fs->NewWritableFile(fname, soptions, &file, nullptr); + if (!s.ok()) { + return s; + } + s = file->Append(data, IOOptions(), nullptr); + if (s.ok() && should_sync) { + s = file->Sync(IOOptions(), nullptr); + } + if (!s.ok()) { + fs->DeleteFile(fname, IOOptions(), nullptr); + } + return s; +} + +IOStatus ReadFileToString(FileSystem* fs, const std::string& fname, + std::string* data) { + FileOptions soptions; + data->clear(); + std::unique_ptr file; + IOStatus s = status_to_io_status( + fs->NewSequentialFile(fname, soptions, &file, nullptr)); + if (!s.ok()) { + return s; + } + static const int kBufferSize = 8192; + char* space = new char[kBufferSize]; + while (true) { + Slice fragment; + s = file->Read(kBufferSize, IOOptions(), &fragment, space, nullptr); + if (!s.ok()) { + break; + } + data->append(fragment.data(), fragment.size()); + if (fragment.empty()) { + break; + } + } + delete[] space; + return s; +} + +namespace { +static std::unordered_map fs_wrapper_type_info = { + {"target", + OptionTypeInfo::AsCustomSharedPtr( + 0, OptionVerificationType::kByName, OptionTypeFlags::kDontSerialize)}, +}; +} // namespace +FileSystemWrapper::FileSystemWrapper(const std::shared_ptr& t) + : target_(t) { + RegisterOptions("", &target_, &fs_wrapper_type_info); +} + +Status FileSystemWrapper::PrepareOptions(const ConfigOptions& options) { + if (target_ == nullptr) { + target_ = FileSystem::Default(); + } + return FileSystem::PrepareOptions(options); +} + +std::string FileSystemWrapper::SerializeOptions( + const ConfigOptions& config_options, const std::string& header) const { + auto parent = FileSystem::SerializeOptions(config_options, ""); + if (config_options.IsShallow() || target_ == nullptr || + target_->IsInstanceOf(FileSystem::kDefaultName())) { + return parent; + } else { + std::string result = header; + if (!StartsWith(parent, OptionTypeInfo::kIdPropName())) { + result.append(OptionTypeInfo::kIdPropName()).append("="); + } + result.append(parent); + if (!EndsWith(result, config_options.delimiter)) { + result.append(config_options.delimiter); + } + result.append("target=").append(target_->ToString(config_options)); + return result; + } +} + +DirFsyncOptions::DirFsyncOptions() { reason = kDefault; } + +DirFsyncOptions::DirFsyncOptions(std::string file_renamed_new_name) { + reason = kFileRenamed; + renamed_new_name = file_renamed_new_name; +} + +DirFsyncOptions::DirFsyncOptions(FsyncReason fsync_reason) { + assert(fsync_reason != kFileRenamed); + reason = fsync_reason; +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/env/file_system_tracer.cc b/librocksdb-sys/rocksdb/env/file_system_tracer.cc new file mode 100644 index 0000000..d0c45c5 --- /dev/null +++ b/librocksdb-sys/rocksdb/env/file_system_tracer.cc @@ -0,0 +1,564 @@ +// Copyright (c) 2019-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "env/file_system_tracer.h" + +#include "rocksdb/file_system.h" +#include "rocksdb/system_clock.h" +#include "rocksdb/trace_record.h" + +namespace ROCKSDB_NAMESPACE { + +IOStatus FileSystemTracingWrapper::NewSequentialFile( + const std::string& fname, const FileOptions& file_opts, + std::unique_ptr* result, IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->NewSequentialFile(fname, file_opts, result, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, + 0 /*io_op_data*/, __func__, elapsed, s.ToString(), + fname.substr(fname.find_last_of("/\\") + 1)); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FileSystemTracingWrapper::NewRandomAccessFile( + const std::string& fname, const FileOptions& file_opts, + std::unique_ptr* result, IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->NewRandomAccessFile(fname, file_opts, result, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, + 0 /*io_op_data*/, __func__, elapsed, s.ToString(), + fname.substr(fname.find_last_of("/\\") + 1)); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FileSystemTracingWrapper::NewWritableFile( + const std::string& fname, const FileOptions& file_opts, + std::unique_ptr* result, IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->NewWritableFile(fname, file_opts, result, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, + 0 /*io_op_data*/, __func__, elapsed, s.ToString(), + fname.substr(fname.find_last_of("/\\") + 1)); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FileSystemTracingWrapper::ReopenWritableFile( + const std::string& fname, const FileOptions& file_opts, + std::unique_ptr* result, IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->ReopenWritableFile(fname, file_opts, result, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, + 0 /*io_op_data*/, __func__, elapsed, s.ToString(), + fname.substr(fname.find_last_of("/\\") + 1)); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FileSystemTracingWrapper::ReuseWritableFile( + const std::string& fname, const std::string& old_fname, + const FileOptions& file_opts, std::unique_ptr* result, + IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = + target()->ReuseWritableFile(fname, old_fname, file_opts, result, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, + 0 /*io_op_data*/, __func__, elapsed, s.ToString(), + fname.substr(fname.find_last_of("/\\") + 1)); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FileSystemTracingWrapper::NewRandomRWFile( + const std::string& fname, const FileOptions& file_opts, + std::unique_ptr* result, IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->NewRandomRWFile(fname, file_opts, result, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, + 0 /*io_op_data*/, __func__, elapsed, s.ToString(), + fname.substr(fname.find_last_of("/\\") + 1)); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FileSystemTracingWrapper::NewDirectory( + const std::string& name, const IOOptions& io_opts, + std::unique_ptr* result, IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->NewDirectory(name, io_opts, result, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, + 0 /*io_op_data*/, __func__, elapsed, s.ToString(), + name.substr(name.find_last_of("/\\") + 1)); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FileSystemTracingWrapper::GetChildren(const std::string& dir, + const IOOptions& io_opts, + std::vector* r, + IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->GetChildren(dir, io_opts, r, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, + 0 /*io_op_data*/, __func__, elapsed, s.ToString(), + dir.substr(dir.find_last_of("/\\") + 1)); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FileSystemTracingWrapper::DeleteFile(const std::string& fname, + const IOOptions& options, + IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->DeleteFile(fname, options, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, + 0 /*io_op_data*/, __func__, elapsed, s.ToString(), + fname.substr(fname.find_last_of("/\\") + 1)); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FileSystemTracingWrapper::CreateDir(const std::string& dirname, + const IOOptions& options, + IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->CreateDir(dirname, options, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, + 0 /*io_op_data*/, __func__, elapsed, s.ToString(), + dirname.substr(dirname.find_last_of("/\\") + 1)); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FileSystemTracingWrapper::CreateDirIfMissing( + const std::string& dirname, const IOOptions& options, IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->CreateDirIfMissing(dirname, options, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, + 0 /*io_op_data*/, __func__, elapsed, s.ToString(), + dirname.substr(dirname.find_last_of("/\\") + 1)); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FileSystemTracingWrapper::DeleteDir(const std::string& dirname, + const IOOptions& options, + IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->DeleteDir(dirname, options, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, + 0 /*io_op_data*/, __func__, elapsed, s.ToString(), + dirname.substr(dirname.find_last_of("/\\") + 1)); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FileSystemTracingWrapper::GetFileSize(const std::string& fname, + const IOOptions& options, + uint64_t* file_size, + IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->GetFileSize(fname, options, file_size, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + uint64_t io_op_data = 0; + io_op_data |= (1 << IOTraceOp::kIOFileSize); + IOTraceRecord io_record( + clock_->NowNanos(), TraceType::kIOTracer, io_op_data, __func__, elapsed, + s.ToString(), fname.substr(fname.find_last_of("/\\") + 1), *file_size); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FileSystemTracingWrapper::Truncate(const std::string& fname, + size_t size, + const IOOptions& options, + IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->Truncate(fname, size, options, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + uint64_t io_op_data = 0; + io_op_data |= (1 << IOTraceOp::kIOFileSize); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, io_op_data, + __func__, elapsed, s.ToString(), + fname.substr(fname.find_last_of("/\\") + 1), size); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FSSequentialFileTracingWrapper::Read(size_t n, + const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->Read(n, options, result, scratch, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + uint64_t io_op_data = 0; + io_op_data |= (1 << IOTraceOp::kIOLen); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, io_op_data, + __func__, elapsed, s.ToString(), file_name_, + result->size(), 0 /*Offset*/); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FSSequentialFileTracingWrapper::InvalidateCache(size_t offset, + size_t length) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->InvalidateCache(offset, length); + uint64_t elapsed = timer.ElapsedNanos(); + uint64_t io_op_data = 0; + io_op_data |= (1 << IOTraceOp::kIOLen); + io_op_data |= (1 << IOTraceOp::kIOOffset); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, io_op_data, + __func__, elapsed, s.ToString(), file_name_, length, + offset); + io_tracer_->WriteIOOp(io_record, nullptr /*dbg*/); + return s; +} + +IOStatus FSSequentialFileTracingWrapper::PositionedRead( + uint64_t offset, size_t n, const IOOptions& options, Slice* result, + char* scratch, IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = + target()->PositionedRead(offset, n, options, result, scratch, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + uint64_t io_op_data = 0; + io_op_data |= (1 << IOTraceOp::kIOLen); + io_op_data |= (1 << IOTraceOp::kIOOffset); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, io_op_data, + __func__, elapsed, s.ToString(), file_name_, + result->size(), offset); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FSRandomAccessFileTracingWrapper::Read(uint64_t offset, size_t n, + const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->Read(offset, n, options, result, scratch, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + uint64_t io_op_data = 0; + io_op_data |= (1 << IOTraceOp::kIOLen); + io_op_data |= (1 << IOTraceOp::kIOOffset); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, io_op_data, + __func__, elapsed, s.ToString(), file_name_, n, + offset); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FSRandomAccessFileTracingWrapper::MultiRead(FSReadRequest* reqs, + size_t num_reqs, + const IOOptions& options, + IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->MultiRead(reqs, num_reqs, options, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + uint64_t latency = elapsed; + uint64_t io_op_data = 0; + io_op_data |= (1 << IOTraceOp::kIOLen); + io_op_data |= (1 << IOTraceOp::kIOOffset); + for (size_t i = 0; i < num_reqs; i++) { + IOTraceRecord io_record( + clock_->NowNanos(), TraceType::kIOTracer, io_op_data, __func__, latency, + reqs[i].status.ToString(), file_name_, reqs[i].len, reqs[i].offset); + io_tracer_->WriteIOOp(io_record, dbg); + } + return s; +} + +IOStatus FSRandomAccessFileTracingWrapper::Prefetch(uint64_t offset, size_t n, + const IOOptions& options, + IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->Prefetch(offset, n, options, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + uint64_t io_op_data = 0; + io_op_data |= (1 << IOTraceOp::kIOLen); + io_op_data |= (1 << IOTraceOp::kIOOffset); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, io_op_data, + __func__, elapsed, s.ToString(), file_name_, n, + offset); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FSRandomAccessFileTracingWrapper::InvalidateCache(size_t offset, + size_t length) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->InvalidateCache(offset, length); + uint64_t elapsed = timer.ElapsedNanos(); + uint64_t io_op_data = 0; + io_op_data |= (1 << IOTraceOp::kIOLen); + io_op_data |= (1 << IOTraceOp::kIOOffset); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, io_op_data, + __func__, elapsed, s.ToString(), file_name_, length, + static_cast(offset)); + io_tracer_->WriteIOOp(io_record, nullptr /*dbg*/); + return s; +} + +IOStatus FSRandomAccessFileTracingWrapper::ReadAsync( + FSReadRequest& req, const IOOptions& opts, + std::function cb, void* cb_arg, + void** io_handle, IOHandleDeleter* del_fn, IODebugContext* dbg) { + // Create a callback and populate info. + auto read_async_callback = + std::bind(&FSRandomAccessFileTracingWrapper::ReadAsyncCallback, this, + std::placeholders::_1, std::placeholders::_2); + ReadAsyncCallbackInfo* read_async_cb_info = new ReadAsyncCallbackInfo; + read_async_cb_info->cb_ = cb; + read_async_cb_info->cb_arg_ = cb_arg; + read_async_cb_info->start_time_ = clock_->NowNanos(); + read_async_cb_info->file_op_ = __func__; + + IOStatus s = target()->ReadAsync(req, opts, read_async_callback, + read_async_cb_info, io_handle, del_fn, dbg); + + if (!s.ok()) { + delete read_async_cb_info; + } + return s; +} + +void FSRandomAccessFileTracingWrapper::ReadAsyncCallback( + const FSReadRequest& req, void* cb_arg) { + ReadAsyncCallbackInfo* read_async_cb_info = + static_cast(cb_arg); + assert(read_async_cb_info); + assert(read_async_cb_info->cb_); + + uint64_t elapsed = clock_->NowNanos() - read_async_cb_info->start_time_; + uint64_t io_op_data = 0; + io_op_data |= (1 << IOTraceOp::kIOLen); + io_op_data |= (1 << IOTraceOp::kIOOffset); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, io_op_data, + read_async_cb_info->file_op_, elapsed, + req.status.ToString(), file_name_, req.result.size(), + req.offset); + io_tracer_->WriteIOOp(io_record, nullptr /*dbg*/); + + // call the underlying callback. + read_async_cb_info->cb_(req, read_async_cb_info->cb_arg_); + delete read_async_cb_info; +} + +IOStatus FSWritableFileTracingWrapper::Append(const Slice& data, + const IOOptions& options, + IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->Append(data, options, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + uint64_t io_op_data = 0; + io_op_data |= (1 << IOTraceOp::kIOLen); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, io_op_data, + __func__, elapsed, s.ToString(), file_name_, + data.size(), 0 /*Offset*/); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FSWritableFileTracingWrapper::PositionedAppend( + const Slice& data, uint64_t offset, const IOOptions& options, + IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->PositionedAppend(data, offset, options, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + uint64_t io_op_data = 0; + io_op_data |= (1 << IOTraceOp::kIOLen); + io_op_data |= (1 << IOTraceOp::kIOOffset); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, io_op_data, + __func__, elapsed, s.ToString(), file_name_, + data.size(), offset); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FSWritableFileTracingWrapper::Truncate(uint64_t size, + const IOOptions& options, + IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->Truncate(size, options, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + uint64_t io_op_data = 0; + io_op_data |= (1 << IOTraceOp::kIOLen); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, io_op_data, + __func__, elapsed, s.ToString(), file_name_, size, + 0 /*Offset*/); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FSWritableFileTracingWrapper::Close(const IOOptions& options, + IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->Close(options, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, + 0 /*io_op_data*/, __func__, elapsed, s.ToString(), + file_name_); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +uint64_t FSWritableFileTracingWrapper::GetFileSize(const IOOptions& options, + IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + uint64_t file_size = target()->GetFileSize(options, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + uint64_t io_op_data = 0; + io_op_data |= (1 << IOTraceOp::kIOFileSize); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, io_op_data, + __func__, elapsed, "OK", file_name_, file_size); + io_tracer_->WriteIOOp(io_record, dbg); + return file_size; +} + +IOStatus FSWritableFileTracingWrapper::InvalidateCache(size_t offset, + size_t length) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->InvalidateCache(offset, length); + uint64_t elapsed = timer.ElapsedNanos(); + uint64_t io_op_data = 0; + io_op_data |= (1 << IOTraceOp::kIOLen); + io_op_data |= (1 << IOTraceOp::kIOOffset); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, io_op_data, + __func__, elapsed, s.ToString(), file_name_, length, + static_cast(offset)); + io_tracer_->WriteIOOp(io_record, nullptr /*dbg*/); + return s; +} + +IOStatus FSRandomRWFileTracingWrapper::Write(uint64_t offset, const Slice& data, + const IOOptions& options, + IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->Write(offset, data, options, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + uint64_t io_op_data = 0; + io_op_data |= (1 << IOTraceOp::kIOLen); + io_op_data |= (1 << IOTraceOp::kIOOffset); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, io_op_data, + __func__, elapsed, s.ToString(), file_name_, + data.size(), offset); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FSRandomRWFileTracingWrapper::Read(uint64_t offset, size_t n, + const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->Read(offset, n, options, result, scratch, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + uint64_t io_op_data = 0; + io_op_data |= (1 << IOTraceOp::kIOLen); + io_op_data |= (1 << IOTraceOp::kIOOffset); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, io_op_data, + __func__, elapsed, s.ToString(), file_name_, n, + offset); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FSRandomRWFileTracingWrapper::Flush(const IOOptions& options, + IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->Flush(options, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, + 0 /*io_op_data*/, __func__, elapsed, s.ToString(), + file_name_); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FSRandomRWFileTracingWrapper::Close(const IOOptions& options, + IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->Close(options, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, + 0 /*io_op_data*/, __func__, elapsed, s.ToString(), + file_name_); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FSRandomRWFileTracingWrapper::Sync(const IOOptions& options, + IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->Sync(options, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, + 0 /*io_op_data*/, __func__, elapsed, s.ToString(), + file_name_); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} + +IOStatus FSRandomRWFileTracingWrapper::Fsync(const IOOptions& options, + IODebugContext* dbg) { + StopWatchNano timer(clock_); + timer.Start(); + IOStatus s = target()->Fsync(options, dbg); + uint64_t elapsed = timer.ElapsedNanos(); + IOTraceRecord io_record(clock_->NowNanos(), TraceType::kIOTracer, + 0 /*io_op_data*/, __func__, elapsed, s.ToString(), + file_name_); + io_tracer_->WriteIOOp(io_record, dbg); + return s; +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/env/file_system_tracer.h b/librocksdb-sys/rocksdb/env/file_system_tracer.h new file mode 100644 index 0000000..979a0bf --- /dev/null +++ b/librocksdb-sys/rocksdb/env/file_system_tracer.h @@ -0,0 +1,461 @@ +// Copyright (c) 2019-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "rocksdb/file_system.h" +#include "rocksdb/system_clock.h" +#include "trace_replay/io_tracer.h" + +namespace ROCKSDB_NAMESPACE { + +// FileSystemTracingWrapper is a wrapper class above FileSystem that forwards +// the call to the underlying storage system. It then invokes IOTracer to record +// file operations and other contextual information in a binary format for +// tracing. It overrides methods we are interested in tracing and extends +// FileSystemWrapper, which forwards all methods that are not explicitly +// overridden. +class FileSystemTracingWrapper : public FileSystemWrapper { + public: + FileSystemTracingWrapper(const std::shared_ptr& t, + const std::shared_ptr& io_tracer) + : FileSystemWrapper(t), + io_tracer_(io_tracer), + clock_(SystemClock::Default().get()) {} + + ~FileSystemTracingWrapper() override {} + + static const char* kClassName() { return "FileSystemTracing"; } + const char* Name() const override { return kClassName(); } + + IOStatus NewSequentialFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus NewRandomAccessFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus NewWritableFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus ReopenWritableFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus ReuseWritableFile(const std::string& fname, + const std::string& old_fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus NewRandomRWFile(const std::string& fname, const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus NewDirectory(const std::string& name, const IOOptions& io_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus GetChildren(const std::string& dir, const IOOptions& io_opts, + std::vector* r, + IODebugContext* dbg) override; + + IOStatus DeleteFile(const std::string& fname, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus CreateDir(const std::string& dirname, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus CreateDirIfMissing(const std::string& dirname, + const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus DeleteDir(const std::string& dirname, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus GetFileSize(const std::string& fname, const IOOptions& options, + uint64_t* file_size, IODebugContext* dbg) override; + + IOStatus Truncate(const std::string& fname, size_t size, + const IOOptions& options, IODebugContext* dbg) override; + + private: + std::shared_ptr io_tracer_; + SystemClock* clock_; +}; + +// The FileSystemPtr is a wrapper class that takes pointer to storage systems +// (such as posix filesystems). It overloads operator -> and returns a pointer +// of either FileSystem or FileSystemTracingWrapper based on whether tracing is +// enabled or not. It is added to bypass FileSystemTracingWrapper when tracing +// is disabled. +class FileSystemPtr { + public: + FileSystemPtr(std::shared_ptr fs, + const std::shared_ptr& io_tracer) + : fs_(fs), io_tracer_(io_tracer) { + fs_tracer_ = std::make_shared(fs_, io_tracer_); + } + + std::shared_ptr operator->() const { + if (io_tracer_ && io_tracer_->is_tracing_enabled()) { + return fs_tracer_; + } else { + return fs_; + } + } + + /* Returns the underlying File System pointer */ + FileSystem* get() const { + if (io_tracer_ && io_tracer_->is_tracing_enabled()) { + return fs_tracer_.get(); + } else { + return fs_.get(); + } + } + + private: + std::shared_ptr fs_; + std::shared_ptr io_tracer_; + std::shared_ptr fs_tracer_; +}; + +// FSSequentialFileTracingWrapper is a wrapper class above FSSequentialFile that +// forwards the call to the underlying storage system. It then invokes IOTracer +// to record file operations and other contextual information in a binary format +// for tracing. It overrides methods we are interested in tracing and extends +// FSSequentialFileWrapper, which forwards all methods that are not explicitly +// overridden. +class FSSequentialFileTracingWrapper : public FSSequentialFileOwnerWrapper { + public: + FSSequentialFileTracingWrapper(std::unique_ptr&& t, + std::shared_ptr io_tracer, + const std::string& file_name) + : FSSequentialFileOwnerWrapper(std::move(t)), + io_tracer_(io_tracer), + clock_(SystemClock::Default().get()), + file_name_(file_name) {} + + ~FSSequentialFileTracingWrapper() override {} + + IOStatus Read(size_t n, const IOOptions& options, Slice* result, + char* scratch, IODebugContext* dbg) override; + + IOStatus InvalidateCache(size_t offset, size_t length) override; + + IOStatus PositionedRead(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) override; + + private: + std::shared_ptr io_tracer_; + SystemClock* clock_; + std::string file_name_; +}; + +// The FSSequentialFilePtr is a wrapper class that takes pointer to storage +// systems (such as posix filesystems). It overloads operator -> and returns a +// pointer of either FSSequentialFile or FSSequentialFileTracingWrapper based on +// whether tracing is enabled or not. It is added to bypass +// FSSequentialFileTracingWrapper when tracing is disabled. +class FSSequentialFilePtr { + public: + FSSequentialFilePtr() = delete; + FSSequentialFilePtr(std::unique_ptr&& fs, + const std::shared_ptr& io_tracer, + const std::string& file_name) + : io_tracer_(io_tracer), + fs_tracer_(std::move(fs), io_tracer_, + file_name.substr(file_name.find_last_of("/\\") + + 1) /* pass file name */) {} + + FSSequentialFile* operator->() const { + if (io_tracer_ && io_tracer_->is_tracing_enabled()) { + return const_cast(&fs_tracer_); + } else { + return fs_tracer_.target(); + } + } + + FSSequentialFile* get() const { + if (io_tracer_ && io_tracer_->is_tracing_enabled()) { + return const_cast(&fs_tracer_); + } else { + return fs_tracer_.target(); + } + } + + private: + std::shared_ptr io_tracer_; + FSSequentialFileTracingWrapper fs_tracer_; +}; + +// FSRandomAccessFileTracingWrapper is a wrapper class above FSRandomAccessFile +// that forwards the call to the underlying storage system. It then invokes +// IOTracer to record file operations and other contextual information in a +// binary format for tracing. It overrides methods we are interested in tracing +// and extends FSRandomAccessFileWrapper, which forwards all methods that are +// not explicitly overridden. +class FSRandomAccessFileTracingWrapper : public FSRandomAccessFileOwnerWrapper { + public: + FSRandomAccessFileTracingWrapper(std::unique_ptr&& t, + std::shared_ptr io_tracer, + const std::string& file_name) + : FSRandomAccessFileOwnerWrapper(std::move(t)), + io_tracer_(io_tracer), + clock_(SystemClock::Default().get()), + file_name_(file_name) {} + + ~FSRandomAccessFileTracingWrapper() override {} + + IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const override; + + IOStatus MultiRead(FSReadRequest* reqs, size_t num_reqs, + const IOOptions& options, IODebugContext* dbg) override; + + IOStatus Prefetch(uint64_t offset, size_t n, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus InvalidateCache(size_t offset, size_t length) override; + + IOStatus ReadAsync(FSReadRequest& req, const IOOptions& opts, + std::function cb, + void* cb_arg, void** io_handle, IOHandleDeleter* del_fn, + IODebugContext* dbg) override; + + void ReadAsyncCallback(const FSReadRequest& req, void* cb_arg); + + private: + std::shared_ptr io_tracer_; + SystemClock* clock_; + // Stores file name instead of full path. + std::string file_name_; + + struct ReadAsyncCallbackInfo { + uint64_t start_time_; + std::function cb_; + void* cb_arg_; + std::string file_op_; + }; +}; + +// The FSRandomAccessFilePtr is a wrapper class that takes pointer to storage +// systems (such as posix filesystems). It overloads operator -> and returns a +// pointer of either FSRandomAccessFile or FSRandomAccessFileTracingWrapper +// based on whether tracing is enabled or not. It is added to bypass +// FSRandomAccessFileTracingWrapper when tracing is disabled. +class FSRandomAccessFilePtr { + public: + FSRandomAccessFilePtr(std::unique_ptr&& fs, + const std::shared_ptr& io_tracer, + const std::string& file_name) + : io_tracer_(io_tracer), + fs_tracer_(std::move(fs), io_tracer_, + file_name.substr(file_name.find_last_of("/\\") + + 1) /* pass file name */) {} + + FSRandomAccessFile* operator->() const { + if (io_tracer_ && io_tracer_->is_tracing_enabled()) { + return const_cast(&fs_tracer_); + } else { + return fs_tracer_.target(); + } + } + + FSRandomAccessFile* get() const { + if (io_tracer_ && io_tracer_->is_tracing_enabled()) { + return const_cast(&fs_tracer_); + } else { + return fs_tracer_.target(); + } + } + + private: + std::shared_ptr io_tracer_; + FSRandomAccessFileTracingWrapper fs_tracer_; +}; + +// FSWritableFileTracingWrapper is a wrapper class above FSWritableFile that +// forwards the call to the underlying storage system. It then invokes IOTracer +// to record file operations and other contextual information in a binary format +// for tracing. It overrides methods we are interested in tracing and extends +// FSWritableFileWrapper, which forwards all methods that are not explicitly +// overridden. +class FSWritableFileTracingWrapper : public FSWritableFileOwnerWrapper { + public: + FSWritableFileTracingWrapper(std::unique_ptr&& t, + std::shared_ptr io_tracer, + const std::string& file_name) + : FSWritableFileOwnerWrapper(std::move(t)), + io_tracer_(io_tracer), + clock_(SystemClock::Default().get()), + file_name_(file_name) {} + + ~FSWritableFileTracingWrapper() override {} + + IOStatus Append(const Slice& data, const IOOptions& options, + IODebugContext* dbg) override; + IOStatus Append(const Slice& data, const IOOptions& options, + const DataVerificationInfo& /*verification_info*/, + IODebugContext* dbg) override { + return Append(data, options, dbg); + } + + IOStatus PositionedAppend(const Slice& data, uint64_t offset, + const IOOptions& options, + IODebugContext* dbg) override; + IOStatus PositionedAppend(const Slice& data, uint64_t offset, + const IOOptions& options, + const DataVerificationInfo& /*verification_info*/, + IODebugContext* dbg) override { + return PositionedAppend(data, offset, options, dbg); + } + + IOStatus Truncate(uint64_t size, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus Close(const IOOptions& options, IODebugContext* dbg) override; + + uint64_t GetFileSize(const IOOptions& options, IODebugContext* dbg) override; + + IOStatus InvalidateCache(size_t offset, size_t length) override; + + private: + std::shared_ptr io_tracer_; + SystemClock* clock_; + // Stores file name instead of full path. + std::string file_name_; +}; + +// The FSWritableFilePtr is a wrapper class that takes pointer to storage +// systems (such as posix filesystems). It overloads operator -> and returns a +// pointer of either FSWritableFile or FSWritableFileTracingWrapper based on +// whether tracing is enabled or not. It is added to bypass +// FSWritableFileTracingWrapper when tracing is disabled. +class FSWritableFilePtr { + public: + FSWritableFilePtr(std::unique_ptr&& fs, + const std::shared_ptr& io_tracer, + const std::string& file_name) + : io_tracer_(io_tracer) { + fs_tracer_.reset(new FSWritableFileTracingWrapper( + std::move(fs), io_tracer_, + file_name.substr(file_name.find_last_of("/\\") + + 1) /* pass file name */)); + } + + FSWritableFile* operator->() const { + if (io_tracer_ && io_tracer_->is_tracing_enabled()) { + return fs_tracer_.get(); + } else { + return fs_tracer_->target(); + } + } + + FSWritableFile* get() const { + if (io_tracer_ && io_tracer_->is_tracing_enabled()) { + return fs_tracer_.get(); + } else if (fs_tracer_) { + return fs_tracer_->target(); + } else { + return nullptr; + } + } + + void reset() { + fs_tracer_.reset(); + io_tracer_ = nullptr; + } + + private: + std::shared_ptr io_tracer_; + std::unique_ptr fs_tracer_; +}; + +// FSRandomRWFileTracingWrapper is a wrapper class above FSRandomRWFile that +// forwards the call to the underlying storage system. It then invokes IOTracer +// to record file operations and other contextual information in a binary format +// for tracing. It overrides methods we are interested in tracing and extends +// FSRandomRWFileWrapper, which forwards all methods that are not explicitly +// overridden. +class FSRandomRWFileTracingWrapper : public FSRandomRWFileOwnerWrapper { + public: + FSRandomRWFileTracingWrapper(std::unique_ptr&& t, + std::shared_ptr io_tracer, + const std::string& file_name) + : FSRandomRWFileOwnerWrapper(std::move(t)), + io_tracer_(io_tracer), + clock_(SystemClock::Default().get()), + file_name_(file_name) {} + + ~FSRandomRWFileTracingWrapper() override {} + + IOStatus Write(uint64_t offset, const Slice& data, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const override; + + IOStatus Flush(const IOOptions& options, IODebugContext* dbg) override; + + IOStatus Close(const IOOptions& options, IODebugContext* dbg) override; + + IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override; + + IOStatus Fsync(const IOOptions& options, IODebugContext* dbg) override; + + private: + std::shared_ptr io_tracer_; + SystemClock* clock_; + // Stores file name instead of full path. + std::string file_name_; +}; + +// The FSRandomRWFilePtr is a wrapper class that takes pointer to storage +// systems (such as posix filesystems). It overloads operator -> and returns a +// pointer of either FSRandomRWFile or FSRandomRWFileTracingWrapper based on +// whether tracing is enabled or not. It is added to bypass +// FSRandomRWFileTracingWrapper when tracing is disabled. +class FSRandomRWFilePtr { + public: + FSRandomRWFilePtr(std::unique_ptr&& fs, + std::shared_ptr io_tracer, + const std::string& file_name) + : io_tracer_(io_tracer), + fs_tracer_(std::move(fs), io_tracer_, + file_name.substr(file_name.find_last_of("/\\") + + 1) /* pass file name */) {} + + FSRandomRWFile* operator->() const { + if (io_tracer_ && io_tracer_->is_tracing_enabled()) { + return const_cast(&fs_tracer_); + } else { + return fs_tracer_.target(); + } + } + + FSRandomRWFile* get() const { + if (io_tracer_ && io_tracer_->is_tracing_enabled()) { + return const_cast(&fs_tracer_); + } else { + return fs_tracer_.target(); + } + } + + private: + std::shared_ptr io_tracer_; + FSRandomRWFileTracingWrapper fs_tracer_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/env/fs_posix.cc b/librocksdb-sys/rocksdb/env/fs_posix.cc new file mode 100644 index 0000000..dd2f749 --- /dev/null +++ b/librocksdb-sys/rocksdb/env/fs_posix.cc @@ -0,0 +1,1286 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors + +#if !defined(OS_WIN) + +#include +#ifndef ROCKSDB_NO_DYNAMIC_EXTENSION +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if defined(OS_LINUX) || defined(OS_SOLARIS) || defined(OS_ANDROID) +#include +#include +#endif +#include +#include +#include +#include + +#include +// Get nano time includes +#if defined(OS_LINUX) || defined(OS_FREEBSD) +#elif defined(__MACH__) +#include +#include +#include +#else +#include +#endif +#include +#include +#include + +#include "env/composite_env_wrapper.h" +#include "env/io_posix.h" +#include "monitoring/iostats_context_imp.h" +#include "monitoring/thread_status_updater.h" +#include "port/lang.h" +#include "port/port.h" +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "rocksdb/utilities/object_registry.h" +#include "test_util/sync_point.h" +#include "util/coding.h" +#include "util/compression_context_cache.h" +#include "util/random.h" +#include "util/string_util.h" +#include "util/thread_local.h" +#include "util/threadpool_imp.h" + +#if !defined(TMPFS_MAGIC) +#define TMPFS_MAGIC 0x01021994 +#endif +#if !defined(XFS_SUPER_MAGIC) +#define XFS_SUPER_MAGIC 0x58465342 +#endif +#if !defined(EXT4_SUPER_MAGIC) +#define EXT4_SUPER_MAGIC 0xEF53 +#endif + +extern "C" bool RocksDbIOUringEnable() __attribute__((__weak__)); + +namespace ROCKSDB_NAMESPACE { + +namespace { + +inline mode_t GetDBFileMode(bool allow_non_owner_access) { + return allow_non_owner_access ? 0644 : 0600; +} + +// list of pathnames that are locked +// Only used for error message. +struct LockHoldingInfo { + int64_t acquire_time; + uint64_t acquiring_thread; +}; +static std::map locked_files; +static port::Mutex mutex_locked_files; + +static int LockOrUnlock(int fd, bool lock) { + errno = 0; + struct flock f; + memset(&f, 0, sizeof(f)); + f.l_type = (lock ? F_WRLCK : F_UNLCK); + f.l_whence = SEEK_SET; + f.l_start = 0; + f.l_len = 0; // Lock/unlock entire file + int value = fcntl(fd, F_SETLK, &f); + + return value; +} + +class PosixFileLock : public FileLock { + public: + int fd_ = /*invalid*/ -1; + std::string filename; + + void Clear() { + fd_ = -1; + filename.clear(); + } + + virtual ~PosixFileLock() override { + // Check for destruction without UnlockFile + assert(fd_ == -1); + } +}; + +int cloexec_flags(int flags, const EnvOptions* options) { + // If the system supports opening the file with cloexec enabled, + // do so, as this avoids a race condition if a db is opened around + // the same time that a child process is forked +#ifdef O_CLOEXEC + if (options == nullptr || options->set_fd_cloexec) { + flags |= O_CLOEXEC; + } +#else + (void)options; +#endif + return flags; +} + +class PosixFileSystem : public FileSystem { + public: + PosixFileSystem(); + + static const char* kClassName() { return "PosixFileSystem"; } + const char* Name() const override { return kClassName(); } + const char* NickName() const override { return kDefaultName(); } + + ~PosixFileSystem() override {} + bool IsInstanceOf(const std::string& name) const override { + if (name == "posix") { + return true; + } else { + return FileSystem::IsInstanceOf(name); + } + } + + void SetFD_CLOEXEC(int fd, const EnvOptions* options) { + if ((options == nullptr || options->set_fd_cloexec) && fd > 0) { + fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC); + } + } + + IOStatus NewSequentialFile(const std::string& fname, + const FileOptions& options, + std::unique_ptr* result, + IODebugContext* /*dbg*/) override { + result->reset(); + int fd = -1; + int flags = cloexec_flags(O_RDONLY, &options); + FILE* file = nullptr; + + if (options.use_direct_reads && !options.use_mmap_reads) { +#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS) + flags |= O_DIRECT; + TEST_SYNC_POINT_CALLBACK("NewSequentialFile:O_DIRECT", &flags); +#endif + } + + do { + IOSTATS_TIMER_GUARD(open_nanos); + fd = open(fname.c_str(), flags, GetDBFileMode(allow_non_owner_access_)); + } while (fd < 0 && errno == EINTR); + if (fd < 0) { + return IOError("While opening a file for sequentially reading", fname, + errno); + } + + SetFD_CLOEXEC(fd, &options); + + if (options.use_direct_reads && !options.use_mmap_reads) { +#ifdef OS_MACOSX + if (fcntl(fd, F_NOCACHE, 1) == -1) { + close(fd); + return IOError("While fcntl NoCache", fname, errno); + } +#endif + } else { + do { + IOSTATS_TIMER_GUARD(open_nanos); + file = fdopen(fd, "r"); + } while (file == nullptr && errno == EINTR); + if (file == nullptr) { + close(fd); + return IOError("While opening file for sequentially read", fname, + errno); + } + } + result->reset(new PosixSequentialFile( + fname, file, fd, GetLogicalBlockSizeForReadIfNeeded(options, fname, fd), + options)); + return IOStatus::OK(); + } + + IOStatus NewRandomAccessFile(const std::string& fname, + const FileOptions& options, + std::unique_ptr* result, + IODebugContext* /*dbg*/) override { + result->reset(); + IOStatus s = IOStatus::OK(); + int fd; + int flags = cloexec_flags(O_RDONLY, &options); + + if (options.use_direct_reads && !options.use_mmap_reads) { +#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS) + flags |= O_DIRECT; + TEST_SYNC_POINT_CALLBACK("NewRandomAccessFile:O_DIRECT", &flags); +#endif + } + + do { + IOSTATS_TIMER_GUARD(open_nanos); + fd = open(fname.c_str(), flags, GetDBFileMode(allow_non_owner_access_)); + } while (fd < 0 && errno == EINTR); + if (fd < 0) { + s = IOError("While open a file for random read", fname, errno); + return s; + } + SetFD_CLOEXEC(fd, &options); + + if (options.use_mmap_reads) { + // Use of mmap for random reads has been removed because it + // kills performance when storage is fast. + // Use mmap when virtual address-space is plentiful. + uint64_t size; + IOOptions opts; + s = GetFileSize(fname, opts, &size, nullptr); + if (s.ok()) { + void* base = mmap(nullptr, size, PROT_READ, MAP_SHARED, fd, 0); + if (base != MAP_FAILED) { + result->reset( + new PosixMmapReadableFile(fd, fname, base, size, options)); + } else { + s = IOError("while mmap file for read", fname, errno); + close(fd); + } + } else { + close(fd); + } + } else { + if (options.use_direct_reads && !options.use_mmap_reads) { +#ifdef OS_MACOSX + if (fcntl(fd, F_NOCACHE, 1) == -1) { + close(fd); + return IOError("while fcntl NoCache", fname, errno); + } +#endif + } + result->reset(new PosixRandomAccessFile( + fname, fd, GetLogicalBlockSizeForReadIfNeeded(options, fname, fd), + options +#if defined(ROCKSDB_IOURING_PRESENT) + , + !IsIOUringEnabled() ? nullptr : thread_local_io_urings_.get() +#endif + )); + } + return s; + } + + virtual IOStatus OpenWritableFile(const std::string& fname, + const FileOptions& options, bool reopen, + std::unique_ptr* result, + IODebugContext* /*dbg*/) { + result->reset(); + IOStatus s; + int fd = -1; + int flags = (reopen) ? (O_CREAT | O_APPEND) : (O_CREAT | O_TRUNC); + // Direct IO mode with O_DIRECT flag or F_NOCAHCE (MAC OSX) + if (options.use_direct_writes && !options.use_mmap_writes) { + // Note: we should avoid O_APPEND here due to ta the following bug: + // POSIX requires that opening a file with the O_APPEND flag should + // have no affect on the location at which pwrite() writes data. + // However, on Linux, if a file is opened with O_APPEND, pwrite() + // appends data to the end of the file, regardless of the value of + // offset. + // More info here: https://linux.die.net/man/2/pwrite + flags |= O_WRONLY; +#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS) + flags |= O_DIRECT; +#endif + TEST_SYNC_POINT_CALLBACK("NewWritableFile:O_DIRECT", &flags); + } else if (options.use_mmap_writes) { + // non-direct I/O + flags |= O_RDWR; + } else { + flags |= O_WRONLY; + } + + flags = cloexec_flags(flags, &options); + + do { + IOSTATS_TIMER_GUARD(open_nanos); + fd = open(fname.c_str(), flags, GetDBFileMode(allow_non_owner_access_)); + } while (fd < 0 && errno == EINTR); + + if (fd < 0) { + s = IOError("While open a file for appending", fname, errno); + return s; + } + SetFD_CLOEXEC(fd, &options); + + if (options.use_mmap_writes) { + MaybeForceDisableMmap(fd); + } + if (options.use_mmap_writes && !forceMmapOff_) { + result->reset(new PosixMmapFile(fname, fd, page_size_, options)); + } else if (options.use_direct_writes && !options.use_mmap_writes) { +#ifdef OS_MACOSX + if (fcntl(fd, F_NOCACHE, 1) == -1) { + close(fd); + s = IOError("While fcntl NoCache an opened file for appending", fname, + errno); + return s; + } +#elif defined(OS_SOLARIS) + if (directio(fd, DIRECTIO_ON) == -1) { + if (errno != ENOTTY) { // ZFS filesystems don't support DIRECTIO_ON + close(fd); + s = IOError("While calling directio()", fname, errno); + return s; + } + } +#endif + result->reset(new PosixWritableFile( + fname, fd, GetLogicalBlockSizeForWriteIfNeeded(options, fname, fd), + options)); + } else { + // disable mmap writes + EnvOptions no_mmap_writes_options = options; + no_mmap_writes_options.use_mmap_writes = false; + result->reset( + new PosixWritableFile(fname, fd, + GetLogicalBlockSizeForWriteIfNeeded( + no_mmap_writes_options, fname, fd), + no_mmap_writes_options)); + } + return s; + } + + IOStatus NewWritableFile(const std::string& fname, const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override { + return OpenWritableFile(fname, options, false, result, dbg); + } + + IOStatus ReopenWritableFile(const std::string& fname, + const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override { + return OpenWritableFile(fname, options, true, result, dbg); + } + + IOStatus ReuseWritableFile(const std::string& fname, + const std::string& old_fname, + const FileOptions& options, + std::unique_ptr* result, + IODebugContext* /*dbg*/) override { + result->reset(); + IOStatus s; + int fd = -1; + + int flags = 0; + // Direct IO mode with O_DIRECT flag or F_NOCAHCE (MAC OSX) + if (options.use_direct_writes && !options.use_mmap_writes) { + flags |= O_WRONLY; +#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS) + flags |= O_DIRECT; +#endif + TEST_SYNC_POINT_CALLBACK("NewWritableFile:O_DIRECT", &flags); + } else if (options.use_mmap_writes) { + // mmap needs O_RDWR mode + flags |= O_RDWR; + } else { + flags |= O_WRONLY; + } + + flags = cloexec_flags(flags, &options); + + do { + IOSTATS_TIMER_GUARD(open_nanos); + fd = open(old_fname.c_str(), flags, + GetDBFileMode(allow_non_owner_access_)); + } while (fd < 0 && errno == EINTR); + if (fd < 0) { + s = IOError("while reopen file for write", fname, errno); + return s; + } + + SetFD_CLOEXEC(fd, &options); + // rename into place + if (rename(old_fname.c_str(), fname.c_str()) != 0) { + s = IOError("while rename file to " + fname, old_fname, errno); + close(fd); + return s; + } + + if (options.use_mmap_writes) { + MaybeForceDisableMmap(fd); + } + if (options.use_mmap_writes && !forceMmapOff_) { + result->reset(new PosixMmapFile(fname, fd, page_size_, options)); + } else if (options.use_direct_writes && !options.use_mmap_writes) { +#ifdef OS_MACOSX + if (fcntl(fd, F_NOCACHE, 1) == -1) { + close(fd); + s = IOError("while fcntl NoCache for reopened file for append", fname, + errno); + return s; + } +#elif defined(OS_SOLARIS) + if (directio(fd, DIRECTIO_ON) == -1) { + if (errno != ENOTTY) { // ZFS filesystems don't support DIRECTIO_ON + close(fd); + s = IOError("while calling directio()", fname, errno); + return s; + } + } +#endif + result->reset(new PosixWritableFile( + fname, fd, GetLogicalBlockSizeForWriteIfNeeded(options, fname, fd), + options)); + } else { + // disable mmap writes + FileOptions no_mmap_writes_options = options; + no_mmap_writes_options.use_mmap_writes = false; + result->reset( + new PosixWritableFile(fname, fd, + GetLogicalBlockSizeForWriteIfNeeded( + no_mmap_writes_options, fname, fd), + no_mmap_writes_options)); + } + return s; + } + + IOStatus NewRandomRWFile(const std::string& fname, const FileOptions& options, + std::unique_ptr* result, + IODebugContext* /*dbg*/) override { + int fd = -1; + int flags = cloexec_flags(O_RDWR, &options); + + while (fd < 0) { + IOSTATS_TIMER_GUARD(open_nanos); + + fd = open(fname.c_str(), flags, GetDBFileMode(allow_non_owner_access_)); + if (fd < 0) { + // Error while opening the file + if (errno == EINTR) { + continue; + } + return IOError("While open file for random read/write", fname, errno); + } + } + + SetFD_CLOEXEC(fd, &options); + result->reset(new PosixRandomRWFile(fname, fd, options)); + return IOStatus::OK(); + } + + IOStatus NewMemoryMappedFileBuffer( + const std::string& fname, + std::unique_ptr* result) override { + int fd = -1; + IOStatus status; + int flags = cloexec_flags(O_RDWR, nullptr); + + while (fd < 0) { + IOSTATS_TIMER_GUARD(open_nanos); + fd = open(fname.c_str(), flags, 0644); + if (fd < 0) { + // Error while opening the file + if (errno == EINTR) { + continue; + } + status = + IOError("While open file for raw mmap buffer access", fname, errno); + break; + } + } + uint64_t size; + if (status.ok()) { + IOOptions opts; + status = GetFileSize(fname, opts, &size, nullptr); + } + void* base = nullptr; + if (status.ok()) { + base = mmap(nullptr, static_cast(size), PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (base == MAP_FAILED) { + status = IOError("while mmap file for read", fname, errno); + } + } + if (status.ok()) { + result->reset( + new PosixMemoryMappedFileBuffer(base, static_cast(size))); + } + if (fd >= 0) { + // don't need to keep it open after mmap has been called + close(fd); + } + return status; + } + + IOStatus NewDirectory(const std::string& name, const IOOptions& /*opts*/, + std::unique_ptr* result, + IODebugContext* /*dbg*/) override { + result->reset(); + int fd; + int flags = cloexec_flags(0, nullptr); + { + IOSTATS_TIMER_GUARD(open_nanos); + fd = open(name.c_str(), flags); + } + if (fd < 0) { + return IOError("While open directory", name, errno); + } else { + result->reset(new PosixDirectory(fd, name)); + } + return IOStatus::OK(); + } + + IOStatus FileExists(const std::string& fname, const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) override { + int result = access(fname.c_str(), F_OK); + + if (result == 0) { + return IOStatus::OK(); + } + + int err = errno; + switch (err) { + case EACCES: + case ELOOP: + case ENAMETOOLONG: + case ENOENT: + case ENOTDIR: + return IOStatus::NotFound(); + default: + assert(err == EIO || err == ENOMEM); + return IOStatus::IOError("Unexpected error(" + std::to_string(err) + + ") accessing file `" + fname + "' "); + } + } + + IOStatus GetChildren(const std::string& dir, const IOOptions& opts, + std::vector* result, + IODebugContext* /*dbg*/) override { + result->clear(); + + DIR* d = opendir(dir.c_str()); + if (d == nullptr) { + switch (errno) { + case EACCES: + case ENOENT: + case ENOTDIR: + return IOStatus::NotFound(); + default: + return IOError("While opendir", dir, errno); + } + } + + // reset errno before calling readdir() + errno = 0; + struct dirent* entry; + + while ((entry = readdir(d)) != nullptr) { + // filter out '.' and '..' directory entries + // which appear only on some platforms + const bool ignore = + entry->d_type == DT_DIR && + (strcmp(entry->d_name, ".") == 0 || + strcmp(entry->d_name, "..") == 0 +#ifndef ASSERT_STATUS_CHECKED + // In case of ASSERT_STATUS_CHECKED, GetChildren support older + // version of API for debugging purpose. + || opts.do_not_recurse +#endif + ); + if (!ignore) { + result->push_back(entry->d_name); + } + errno = 0; // reset errno if readdir() success + } + + // always attempt to close the dir + const auto pre_close_errno = errno; // errno may be modified by closedir + const int close_result = closedir(d); + + if (pre_close_errno != 0) { + // error occurred during readdir + return IOError("While readdir", dir, pre_close_errno); + } + + if (close_result != 0) { + // error occurred during closedir + return IOError("While closedir", dir, errno); + } + + return IOStatus::OK(); + } + + IOStatus DeleteFile(const std::string& fname, const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) override { + IOStatus result; + if (unlink(fname.c_str()) != 0) { + result = IOError("while unlink() file", fname, errno); + } + return result; + } + + IOStatus CreateDir(const std::string& name, const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) override { + if (mkdir(name.c_str(), 0755) != 0) { + return IOError("While mkdir", name, errno); + } + return IOStatus::OK(); + } + + IOStatus CreateDirIfMissing(const std::string& name, + const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) override { + if (mkdir(name.c_str(), 0755) != 0) { + if (errno != EEXIST) { + return IOError("While mkdir if missing", name, errno); + } else if (!DirExists(name)) { // Check that name is actually a + // directory. + // Message is taken from mkdir + return IOStatus::IOError("`" + name + + "' exists but is not a directory"); + } + } + return IOStatus::OK(); + } + + IOStatus DeleteDir(const std::string& name, const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) override { + if (rmdir(name.c_str()) != 0) { + return IOError("file rmdir", name, errno); + } + return IOStatus::OK(); + } + + IOStatus GetFileSize(const std::string& fname, const IOOptions& /*opts*/, + uint64_t* size, IODebugContext* /*dbg*/) override { + struct stat sbuf; + if (stat(fname.c_str(), &sbuf) != 0) { + *size = 0; + return IOError("while stat a file for size", fname, errno); + } else { + *size = sbuf.st_size; + } + return IOStatus::OK(); + } + + IOStatus GetFileModificationTime(const std::string& fname, + const IOOptions& /*opts*/, + uint64_t* file_mtime, + IODebugContext* /*dbg*/) override { + struct stat s; + if (stat(fname.c_str(), &s) != 0) { + return IOError("while stat a file for modification time", fname, errno); + } + *file_mtime = static_cast(s.st_mtime); + return IOStatus::OK(); + } + + IOStatus RenameFile(const std::string& src, const std::string& target, + const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) override { + if (rename(src.c_str(), target.c_str()) != 0) { + return IOError("While renaming a file to " + target, src, errno); + } + return IOStatus::OK(); + } + + IOStatus LinkFile(const std::string& src, const std::string& target, + const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) override { + if (link(src.c_str(), target.c_str()) != 0) { + if (errno == EXDEV || errno == ENOTSUP) { + return IOStatus::NotSupported(errno == EXDEV + ? "No cross FS links allowed" + : "Links not supported by FS"); + } + return IOError("while link file to " + target, src, errno); + } + return IOStatus::OK(); + } + + IOStatus NumFileLinks(const std::string& fname, const IOOptions& /*opts*/, + uint64_t* count, IODebugContext* /*dbg*/) override { + struct stat s; + if (stat(fname.c_str(), &s) != 0) { + return IOError("while stat a file for num file links", fname, errno); + } + *count = static_cast(s.st_nlink); + return IOStatus::OK(); + } + + IOStatus AreFilesSame(const std::string& first, const std::string& second, + const IOOptions& /*opts*/, bool* res, + IODebugContext* /*dbg*/) override { + struct stat statbuf[2]; + if (stat(first.c_str(), &statbuf[0]) != 0) { + return IOError("stat file", first, errno); + } + if (stat(second.c_str(), &statbuf[1]) != 0) { + return IOError("stat file", second, errno); + } + + if (major(statbuf[0].st_dev) != major(statbuf[1].st_dev) || + minor(statbuf[0].st_dev) != minor(statbuf[1].st_dev) || + statbuf[0].st_ino != statbuf[1].st_ino) { + *res = false; + } else { + *res = true; + } + return IOStatus::OK(); + } + + IOStatus LockFile(const std::string& fname, const IOOptions& /*opts*/, + FileLock** lock, IODebugContext* /*dbg*/) override { + *lock = nullptr; + + LockHoldingInfo lhi; + int64_t current_time = 0; + // Ignore status code as the time is only used for error message. + SystemClock::Default() + ->GetCurrentTime(¤t_time) + .PermitUncheckedError(); + lhi.acquire_time = current_time; + lhi.acquiring_thread = Env::Default()->GetThreadID(); + + mutex_locked_files.Lock(); + // If it already exists in the locked_files set, then it is already locked, + // and fail this lock attempt. Otherwise, insert it into locked_files. + // This check is needed because fcntl() does not detect lock conflict + // if the fcntl is issued by the same thread that earlier acquired + // this lock. + // We must do this check *before* opening the file: + // Otherwise, we will open a new file descriptor. Locks are associated with + // a process, not a file descriptor and when *any* file descriptor is + // closed, all locks the process holds for that *file* are released + const auto it_success = locked_files.insert({fname, lhi}); + if (it_success.second == false) { + LockHoldingInfo prev_info = it_success.first->second; + mutex_locked_files.Unlock(); + errno = ENOLCK; + // Note that the thread ID printed is the same one as the one in + // posix logger, but posix logger prints it hex format. + return IOError("lock hold by current process, acquire time " + + std::to_string(prev_info.acquire_time) + + " acquiring thread " + + std::to_string(prev_info.acquiring_thread), + fname, errno); + } + + IOStatus result = IOStatus::OK(); + int fd; + int flags = cloexec_flags(O_RDWR | O_CREAT, nullptr); + + { + IOSTATS_TIMER_GUARD(open_nanos); + fd = open(fname.c_str(), flags, 0644); + } + if (fd < 0) { + result = IOError("while open a file for lock", fname, errno); + } else if (LockOrUnlock(fd, true) == -1) { + result = IOError("While lock file", fname, errno); + close(fd); + } else { + SetFD_CLOEXEC(fd, nullptr); + PosixFileLock* my_lock = new PosixFileLock; + my_lock->fd_ = fd; + my_lock->filename = fname; + *lock = my_lock; + } + if (!result.ok()) { + // If there is an error in locking, then remove the pathname from + // locked_files. (If we got this far, it did not exist in locked_files + // before this call.) + locked_files.erase(fname); + } + + mutex_locked_files.Unlock(); + return result; + } + + IOStatus UnlockFile(FileLock* lock, const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) override { + PosixFileLock* my_lock = reinterpret_cast(lock); + IOStatus result; + mutex_locked_files.Lock(); + // If we are unlocking, then verify that we had locked it earlier, + // it should already exist in locked_files. Remove it from locked_files. + if (locked_files.erase(my_lock->filename) != 1) { + errno = ENOLCK; + result = IOError("unlock", my_lock->filename, errno); + } else if (LockOrUnlock(my_lock->fd_, false) == -1) { + result = IOError("unlock", my_lock->filename, errno); + } + close(my_lock->fd_); + my_lock->Clear(); + delete my_lock; + mutex_locked_files.Unlock(); + return result; + } + + IOStatus GetAbsolutePath(const std::string& db_path, + const IOOptions& /*opts*/, std::string* output_path, + IODebugContext* /*dbg*/) override { + if (!db_path.empty() && db_path[0] == '/') { + *output_path = db_path; + return IOStatus::OK(); + } + + char the_path[4096]; + char* ret = getcwd(the_path, 4096); + if (ret == nullptr) { + return IOStatus::IOError(errnoStr(errno).c_str()); + } + + *output_path = ret; + return IOStatus::OK(); + } + + IOStatus GetTestDirectory(const IOOptions& /*opts*/, std::string* result, + IODebugContext* /*dbg*/) override { + const char* env = getenv("TEST_TMPDIR"); + if (env && env[0] != '\0') { + *result = env; + } else { + char buf[100]; + snprintf(buf, sizeof(buf), "/tmp/rocksdbtest-%d", int(geteuid())); + *result = buf; + } + // Directory may already exist + { + IOOptions opts; + return CreateDirIfMissing(*result, opts, nullptr); + } + return IOStatus::OK(); + } + + IOStatus GetFreeSpace(const std::string& fname, const IOOptions& /*opts*/, + uint64_t* free_space, + IODebugContext* /*dbg*/) override { + struct statvfs sbuf; + + if (statvfs(fname.c_str(), &sbuf) < 0) { + return IOError("While doing statvfs", fname, errno); + } + + // sbuf.bfree is total free space available to root + // sbuf.bavail is total free space available to unprivileged user + // sbuf.bavail <= sbuf.bfree ... pick correct based upon effective user id + if (geteuid()) { + // non-zero user is unprivileged, or -1 if error. take more conservative + // size + *free_space = ((uint64_t)sbuf.f_bsize * sbuf.f_bavail); + } else { + // root user can access all disk space + *free_space = ((uint64_t)sbuf.f_bsize * sbuf.f_bfree); + } + return IOStatus::OK(); + } + + IOStatus IsDirectory(const std::string& path, const IOOptions& /*opts*/, + bool* is_dir, IODebugContext* /*dbg*/) override { + // First open + int fd = -1; + int flags = cloexec_flags(O_RDONLY, nullptr); + { + IOSTATS_TIMER_GUARD(open_nanos); + fd = open(path.c_str(), flags); + } + if (fd < 0) { + return IOError("While open for IsDirectory()", path, errno); + } + IOStatus io_s; + struct stat sbuf; + if (fstat(fd, &sbuf) < 0) { + io_s = IOError("While doing stat for IsDirectory()", path, errno); + } + close(fd); + if (io_s.ok() && nullptr != is_dir) { + *is_dir = S_ISDIR(sbuf.st_mode); + } + return io_s; + } + + FileOptions OptimizeForLogWrite(const FileOptions& file_options, + const DBOptions& db_options) const override { + FileOptions optimized = file_options; + optimized.use_mmap_writes = false; + optimized.use_direct_writes = false; + optimized.bytes_per_sync = db_options.wal_bytes_per_sync; + // TODO(icanadi) it's faster if fallocate_with_keep_size is false, but it + // breaks TransactionLogIteratorStallAtLastRecord unit test. Fix the unit + // test and make this false + optimized.fallocate_with_keep_size = true; + optimized.writable_file_max_buffer_size = + db_options.writable_file_max_buffer_size; + return optimized; + } + + FileOptions OptimizeForManifestWrite( + const FileOptions& file_options) const override { + FileOptions optimized = file_options; + optimized.use_mmap_writes = false; + optimized.use_direct_writes = false; + optimized.fallocate_with_keep_size = true; + return optimized; + } +#ifdef OS_LINUX + Status RegisterDbPaths(const std::vector& paths) override { + return logical_block_size_cache_.RefAndCacheLogicalBlockSize(paths); + } + Status UnregisterDbPaths(const std::vector& paths) override { + logical_block_size_cache_.UnrefAndTryRemoveCachedLogicalBlockSize(paths); + return Status::OK(); + } +#endif + private: + bool forceMmapOff_ = false; // do we override Env options? + + // Returns true iff the named directory exists and is a directory. + virtual bool DirExists(const std::string& dname) { + struct stat statbuf; + if (stat(dname.c_str(), &statbuf) == 0) { + return S_ISDIR(statbuf.st_mode); + } + return false; // stat() failed return false + } + + bool SupportsFastAllocate(int fd) { +#ifdef ROCKSDB_FALLOCATE_PRESENT + struct statfs s; + if (fstatfs(fd, &s)) { + return false; + } + switch (s.f_type) { + case EXT4_SUPER_MAGIC: + return true; + case XFS_SUPER_MAGIC: + return true; + case TMPFS_MAGIC: + return true; + default: + return false; + } +#else + (void)fd; + return false; +#endif + } + + void MaybeForceDisableMmap(int fd) { + static std::once_flag s_check_disk_for_mmap_once; + assert(this == FileSystem::Default().get()); + std::call_once( + s_check_disk_for_mmap_once, + [this](int fdesc) { + // this will be executed once in the program's lifetime. + // do not use mmapWrite on non ext-3/xfs/tmpfs systems. + if (!SupportsFastAllocate(fdesc)) { + forceMmapOff_ = true; + } + }, + fd); + } + +#ifdef ROCKSDB_IOURING_PRESENT + bool IsIOUringEnabled() { + if (RocksDbIOUringEnable && RocksDbIOUringEnable()) { + return true; + } else { + return false; + } + } +#endif // ROCKSDB_IOURING_PRESENT + + // EXPERIMENTAL + // + // TODO akankshamahajan: + // 1. Update Poll API to take into account min_completions + // and returns if number of handles in io_handles (any order) completed is + // equal to atleast min_completions. + // 2. Currently in case of direct_io, Read API is called because of which call + // to Poll API fails as it expects IOHandle to be populated. + virtual IOStatus Poll(std::vector& io_handles, + size_t /*min_completions*/) override { +#if defined(ROCKSDB_IOURING_PRESENT) + // io_uring_queue_init. + struct io_uring* iu = nullptr; + if (thread_local_io_urings_) { + iu = static_cast(thread_local_io_urings_->Get()); + } + + // Init failed, platform doesn't support io_uring. + if (iu == nullptr) { + return IOStatus::NotSupported("Poll"); + } + + for (size_t i = 0; i < io_handles.size(); i++) { + // The request has been completed in earlier runs. + if ((static_cast(io_handles[i]))->is_finished) { + continue; + } + // Loop until IO for io_handles[i] is completed. + while (true) { + // io_uring_wait_cqe. + struct io_uring_cqe* cqe = nullptr; + ssize_t ret = io_uring_wait_cqe(iu, &cqe); + if (ret) { + // abort as it shouldn't be in indeterminate state and there is no + // good way currently to handle this error. + abort(); + } + + // Step 3: Populate the request. + assert(cqe != nullptr); + Posix_IOHandle* posix_handle = + static_cast(io_uring_cqe_get_data(cqe)); + assert(posix_handle->iu == iu); + if (posix_handle->iu != iu) { + return IOStatus::IOError(""); + } + // Reset cqe data to catch any stray reuse of it + static_cast(cqe)->user_data = 0xd5d5d5d5d5d5d5d5; + + FSReadRequest req; + req.scratch = posix_handle->scratch; + req.offset = posix_handle->offset; + req.len = posix_handle->len; + + size_t finished_len = 0; + size_t bytes_read = 0; + bool read_again = false; + UpdateResult(cqe, "", req.len, posix_handle->iov.iov_len, + true /*async_read*/, posix_handle->use_direct_io, + posix_handle->alignment, finished_len, &req, bytes_read, + read_again); + posix_handle->is_finished = true; + io_uring_cqe_seen(iu, cqe); + posix_handle->cb(req, posix_handle->cb_arg); + + (void)finished_len; + (void)bytes_read; + (void)read_again; + + if (static_cast(io_handles[i]) == posix_handle) { + break; + } + } + } + return IOStatus::OK(); +#else + (void)io_handles; + return IOStatus::NotSupported("Poll"); +#endif + } + + virtual IOStatus AbortIO(std::vector& io_handles) override { +#if defined(ROCKSDB_IOURING_PRESENT) + // io_uring_queue_init. + struct io_uring* iu = nullptr; + if (thread_local_io_urings_) { + iu = static_cast(thread_local_io_urings_->Get()); + } + + // Init failed, platform doesn't support io_uring. + // If Poll is not supported then it didn't submit any request and it should + // return OK. + if (iu == nullptr) { + return IOStatus::OK(); + } + + for (size_t i = 0; i < io_handles.size(); i++) { + Posix_IOHandle* posix_handle = + static_cast(io_handles[i]); + if (posix_handle->is_finished == true) { + continue; + } + assert(posix_handle->iu == iu); + if (posix_handle->iu != iu) { + return IOStatus::IOError(""); + } + + // Prepare the cancel request. + struct io_uring_sqe* sqe; + sqe = io_uring_get_sqe(iu); + + // In order to cancel the request, sqe->addr of cancel request should + // match with the read request submitted which is posix_handle->iov. + io_uring_prep_cancel(sqe, &posix_handle->iov, 0); + // Sets sqe->user_data to posix_handle. + io_uring_sqe_set_data(sqe, posix_handle); + + // submit the request. + ssize_t ret = io_uring_submit(iu); + if (ret < 0) { + fprintf(stderr, "io_uring_submit error: %ld\n", long(ret)); + return IOStatus::IOError("io_uring_submit() requested but returned " + + std::to_string(ret)); + } + } + + // After submitting the requests, wait for the requests. + for (size_t i = 0; i < io_handles.size(); i++) { + if ((static_cast(io_handles[i]))->is_finished) { + continue; + } + + while (true) { + struct io_uring_cqe* cqe = nullptr; + ssize_t ret = io_uring_wait_cqe(iu, &cqe); + if (ret) { + // abort as it shouldn't be in indeterminate state and there is no + // good way currently to handle this error. + abort(); + } + assert(cqe != nullptr); + + // Returns cqe->user_data. + Posix_IOHandle* posix_handle = + static_cast(io_uring_cqe_get_data(cqe)); + assert(posix_handle->iu == iu); + if (posix_handle->iu != iu) { + return IOStatus::IOError(""); + } + posix_handle->req_count++; + + // Reset cqe data to catch any stray reuse of it + static_cast(cqe)->user_data = 0xd5d5d5d5d5d5d5d5; + io_uring_cqe_seen(iu, cqe); + + // - If the request is cancelled successfully, the original request is + // completed with -ECANCELED and the cancel request is completed with + // a result of 0. + // - If the request was already running, the original may or + // may not complete in error. The cancel request will complete with + // -EALREADY for that case. + // - And finally, if the request to cancel wasn't + // found, the cancel request is completed with -ENOENT. + // + // Every handle has to wait for 2 requests completion: original one and + // the cancel request which is tracked by PosixHandle::req_count. + if (posix_handle->req_count == 2 && + static_cast(io_handles[i]) == posix_handle) { + posix_handle->is_finished = true; + FSReadRequest req; + req.status = IOStatus::Aborted(); + posix_handle->cb(req, posix_handle->cb_arg); + + break; + } + } + } + return IOStatus::OK(); +#else + // If Poll is not supported then it didn't submit any request and it should + // return OK. + (void)io_handles; + return IOStatus::OK(); +#endif + } + + void SupportedOps(int64_t& supported_ops) override { + supported_ops = 0; +#if defined(ROCKSDB_IOURING_PRESENT) + if (IsIOUringEnabled()) { + // Underlying FS supports async_io + supported_ops |= (1 << FSSupportedOps::kAsyncIO); + } +#endif + } + +#if defined(ROCKSDB_IOURING_PRESENT) + // io_uring instance + std::unique_ptr thread_local_io_urings_; +#endif + + size_t page_size_; + + // If true, allow non owner read access for db files. Otherwise, non-owner + // has no access to db files. + bool allow_non_owner_access_; + +#ifdef OS_LINUX + static LogicalBlockSizeCache logical_block_size_cache_; +#endif + static size_t GetLogicalBlockSize(const std::string& fname, int fd); + // In non-direct IO mode, this directly returns kDefaultPageSize. + // Otherwise call GetLogicalBlockSize. + static size_t GetLogicalBlockSizeForReadIfNeeded(const EnvOptions& options, + const std::string& fname, + int fd); + static size_t GetLogicalBlockSizeForWriteIfNeeded(const EnvOptions& options, + const std::string& fname, + int fd); +}; + +#ifdef OS_LINUX +LogicalBlockSizeCache PosixFileSystem::logical_block_size_cache_; +#endif + +size_t PosixFileSystem::GetLogicalBlockSize(const std::string& fname, int fd) { +#ifdef OS_LINUX + return logical_block_size_cache_.GetLogicalBlockSize(fname, fd); +#else + (void)fname; + return PosixHelper::GetLogicalBlockSizeOfFd(fd); +#endif +} + +size_t PosixFileSystem::GetLogicalBlockSizeForReadIfNeeded( + const EnvOptions& options, const std::string& fname, int fd) { + return options.use_direct_reads + ? PosixFileSystem::GetLogicalBlockSize(fname, fd) + : kDefaultPageSize; +} + +size_t PosixFileSystem::GetLogicalBlockSizeForWriteIfNeeded( + const EnvOptions& options, const std::string& fname, int fd) { + return options.use_direct_writes + ? PosixFileSystem::GetLogicalBlockSize(fname, fd) + : kDefaultPageSize; +} + +PosixFileSystem::PosixFileSystem() + : forceMmapOff_(false), + page_size_(getpagesize()), + allow_non_owner_access_(true) { +#if defined(ROCKSDB_IOURING_PRESENT) + // Test whether IOUring is supported, and if it does, create a managing + // object for thread local point so that in the future thread-local + // io_uring can be created. + struct io_uring* new_io_uring = CreateIOUring(); + if (new_io_uring != nullptr) { + thread_local_io_urings_.reset(new ThreadLocalPtr(DeleteIOUring)); + delete new_io_uring; + } +#endif +} + +} // namespace + +// +// Default Posix FileSystem +// +std::shared_ptr FileSystem::Default() { + STATIC_AVOID_DESTRUCTION(std::shared_ptr, instance) + (std::make_shared()); + return instance; +} + +static FactoryFunc posix_filesystem_reg = + ObjectLibrary::Default()->AddFactory( + ObjectLibrary::PatternEntry("posix").AddSeparator("://", false), + [](const std::string& /* uri */, std::unique_ptr* f, + std::string* /* errmsg */) { + f->reset(new PosixFileSystem()); + return f->get(); + }); + +} // namespace ROCKSDB_NAMESPACE + +#endif diff --git a/librocksdb-sys/rocksdb/env/fs_readonly.h b/librocksdb-sys/rocksdb/env/fs_readonly.h new file mode 100644 index 0000000..7a04aea --- /dev/null +++ b/librocksdb-sys/rocksdb/env/fs_readonly.h @@ -0,0 +1,105 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + + +#include "rocksdb/file_system.h" + +namespace ROCKSDB_NAMESPACE { + +// A FileSystem wrapper that only allows read-only operation. +// +// This class has not been fully analyzed for providing strong security +// guarantees. +class ReadOnlyFileSystem : public FileSystemWrapper { + static inline IOStatus FailReadOnly() { + IOStatus s = IOStatus::IOError("Attempted write to ReadOnlyFileSystem"); + assert(s.GetRetryable() == false); + return s; + } + + public: + explicit ReadOnlyFileSystem(const std::shared_ptr& base) + : FileSystemWrapper(base) {} + + static const char* kClassName() { return "ReadOnlyFileSystem"; } + const char* Name() const override { return kClassName(); } + + IOStatus NewWritableFile(const std::string& /*fname*/, + const FileOptions& /*options*/, + std::unique_ptr* /*result*/, + IODebugContext* /*dbg*/) override { + return FailReadOnly(); + } + IOStatus ReuseWritableFile(const std::string& /*fname*/, + const std::string& /*old_fname*/, + const FileOptions& /*options*/, + std::unique_ptr* /*result*/, + IODebugContext* /*dbg*/) override { + return FailReadOnly(); + } + IOStatus NewRandomRWFile(const std::string& /*fname*/, + const FileOptions& /*options*/, + std::unique_ptr* /*result*/, + IODebugContext* /*dbg*/) override { + return FailReadOnly(); + } + IOStatus NewDirectory(const std::string& /*dir*/, + const IOOptions& /*options*/, + std::unique_ptr* /*result*/, + IODebugContext* /*dbg*/) override { + return FailReadOnly(); + } + IOStatus DeleteFile(const std::string& /*fname*/, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return FailReadOnly(); + } + IOStatus CreateDir(const std::string& /*dirname*/, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return FailReadOnly(); + } + IOStatus CreateDirIfMissing(const std::string& dirname, + const IOOptions& options, + IODebugContext* dbg) override { + // Allow if dir already exists + bool is_dir = false; + IOStatus s = IsDirectory(dirname, options, &is_dir, dbg); + if (s.ok() && is_dir) { + return s; + } else { + return FailReadOnly(); + } + } + IOStatus DeleteDir(const std::string& /*dirname*/, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return FailReadOnly(); + } + IOStatus RenameFile(const std::string& /*src*/, const std::string& /*dest*/, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return FailReadOnly(); + } + IOStatus LinkFile(const std::string& /*src*/, const std::string& /*dest*/, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return FailReadOnly(); + } + IOStatus LockFile(const std::string& /*fname*/, const IOOptions& /*options*/, + FileLock** /*lock*/, IODebugContext* /*dbg*/) override { + return FailReadOnly(); + } + IOStatus NewLogger(const std::string& /*fname*/, const IOOptions& /*options*/, + std::shared_ptr* /*result*/, + IODebugContext* /*dbg*/) override { + return FailReadOnly(); + } +}; + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/env/fs_remap.cc b/librocksdb-sys/rocksdb/env/fs_remap.cc new file mode 100644 index 0000000..b9832e6 --- /dev/null +++ b/librocksdb-sys/rocksdb/env/fs_remap.cc @@ -0,0 +1,341 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + + +#include "env/fs_remap.h" + +namespace ROCKSDB_NAMESPACE { + +RemapFileSystem::RemapFileSystem(const std::shared_ptr& base) + : FileSystemWrapper(base) {} + +std::pair RemapFileSystem::EncodePathWithNewBasename( + const std::string& path) { + // No difference by default + return EncodePath(path); +} + +Status RemapFileSystem::RegisterDbPaths(const std::vector& paths) { + std::vector encoded_paths; + encoded_paths.reserve(paths.size()); + for (auto& path : paths) { + auto status_and_enc_path = EncodePathWithNewBasename(path); + if (!status_and_enc_path.first.ok()) { + return status_and_enc_path.first; + } + encoded_paths.emplace_back(status_and_enc_path.second); + } + return FileSystemWrapper::RegisterDbPaths(encoded_paths); +} + +Status RemapFileSystem::UnregisterDbPaths( + const std::vector& paths) { + std::vector encoded_paths; + encoded_paths.reserve(paths.size()); + for (auto& path : paths) { + auto status_and_enc_path = EncodePathWithNewBasename(path); + if (!status_and_enc_path.first.ok()) { + return status_and_enc_path.first; + } + encoded_paths.emplace_back(status_and_enc_path.second); + } + return FileSystemWrapper::UnregisterDbPaths(encoded_paths); +} + +IOStatus RemapFileSystem::NewSequentialFile( + const std::string& fname, const FileOptions& options, + std::unique_ptr* result, IODebugContext* dbg) { + auto status_and_enc_path = EncodePathWithNewBasename(fname); + if (!status_and_enc_path.first.ok()) { + return status_and_enc_path.first; + } + return FileSystemWrapper::NewSequentialFile(status_and_enc_path.second, + options, result, dbg); +} + +IOStatus RemapFileSystem::NewRandomAccessFile( + const std::string& fname, const FileOptions& options, + std::unique_ptr* result, IODebugContext* dbg) { + auto status_and_enc_path = EncodePathWithNewBasename(fname); + if (!status_and_enc_path.first.ok()) { + return status_and_enc_path.first; + } + return FileSystemWrapper::NewRandomAccessFile(status_and_enc_path.second, + options, result, dbg); +} + +IOStatus RemapFileSystem::NewWritableFile( + const std::string& fname, const FileOptions& options, + std::unique_ptr* result, IODebugContext* dbg) { + auto status_and_enc_path = EncodePathWithNewBasename(fname); + if (!status_and_enc_path.first.ok()) { + return status_and_enc_path.first; + } + return FileSystemWrapper::NewWritableFile(status_and_enc_path.second, options, + result, dbg); +} + +IOStatus RemapFileSystem::ReuseWritableFile( + const std::string& fname, const std::string& old_fname, + const FileOptions& options, std::unique_ptr* result, + IODebugContext* dbg) { + auto status_and_enc_path = EncodePathWithNewBasename(fname); + if (!status_and_enc_path.first.ok()) { + return status_and_enc_path.first; + } + auto status_and_old_enc_path = EncodePath(old_fname); + if (!status_and_old_enc_path.first.ok()) { + return status_and_old_enc_path.first; + } + return FileSystemWrapper::ReuseWritableFile(status_and_old_enc_path.second, + status_and_old_enc_path.second, + options, result, dbg); +} + +IOStatus RemapFileSystem::NewRandomRWFile( + const std::string& fname, const FileOptions& options, + std::unique_ptr* result, IODebugContext* dbg) { + auto status_and_enc_path = EncodePathWithNewBasename(fname); + if (!status_and_enc_path.first.ok()) { + return status_and_enc_path.first; + } + return FileSystemWrapper::NewRandomRWFile(status_and_enc_path.second, options, + result, dbg); +} + +IOStatus RemapFileSystem::NewDirectory(const std::string& dir, + const IOOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) { + // A hassle to remap DirFsyncOptions::renamed_new_name + class RemapFSDirectory : public FSDirectoryWrapper { + public: + RemapFSDirectory(RemapFileSystem* fs, std::unique_ptr&& t) + : FSDirectoryWrapper(std::move(t)), fs_(fs) {} + IOStatus FsyncWithDirOptions( + const IOOptions& options, IODebugContext* dbg, + const DirFsyncOptions& dir_fsync_options) override { + if (dir_fsync_options.renamed_new_name.empty()) { + return FSDirectoryWrapper::FsyncWithDirOptions(options, dbg, + dir_fsync_options); + } else { + auto status_and_enc_path = + fs_->EncodePath(dir_fsync_options.renamed_new_name); + if (status_and_enc_path.first.ok()) { + DirFsyncOptions mapped_options = dir_fsync_options; + mapped_options.renamed_new_name = status_and_enc_path.second; + return FSDirectoryWrapper::FsyncWithDirOptions(options, dbg, + mapped_options); + } else { + return status_and_enc_path.first; + } + } + } + + private: + RemapFileSystem* const fs_; + }; + + auto status_and_enc_path = EncodePathWithNewBasename(dir); + if (!status_and_enc_path.first.ok()) { + return status_and_enc_path.first; + } + IOStatus ios = FileSystemWrapper::NewDirectory(status_and_enc_path.second, + options, result, dbg); + if (ios.ok()) { + *result = std::make_unique(this, std::move(*result)); + } + return ios; +} + +IOStatus RemapFileSystem::FileExists(const std::string& fname, + const IOOptions& options, + IODebugContext* dbg) { + auto status_and_enc_path = EncodePathWithNewBasename(fname); + if (!status_and_enc_path.first.ok()) { + return status_and_enc_path.first; + } + return FileSystemWrapper::FileExists(status_and_enc_path.second, options, + dbg); +} + +IOStatus RemapFileSystem::GetChildren(const std::string& dir, + const IOOptions& options, + std::vector* result, + IODebugContext* dbg) { + auto status_and_enc_path = EncodePath(dir); + if (!status_and_enc_path.first.ok()) { + return status_and_enc_path.first; + } + return FileSystemWrapper::GetChildren(status_and_enc_path.second, options, + result, dbg); +} + +IOStatus RemapFileSystem::GetChildrenFileAttributes( + const std::string& dir, const IOOptions& options, + std::vector* result, IODebugContext* dbg) { + auto status_and_enc_path = EncodePath(dir); + if (!status_and_enc_path.first.ok()) { + return status_and_enc_path.first; + } + return FileSystemWrapper::GetChildrenFileAttributes( + status_and_enc_path.second, options, result, dbg); +} + +IOStatus RemapFileSystem::DeleteFile(const std::string& fname, + const IOOptions& options, + IODebugContext* dbg) { + auto status_and_enc_path = EncodePath(fname); + if (!status_and_enc_path.first.ok()) { + return status_and_enc_path.first; + } + return FileSystemWrapper::DeleteFile(status_and_enc_path.second, options, + dbg); +} + +IOStatus RemapFileSystem::CreateDir(const std::string& dirname, + const IOOptions& options, + IODebugContext* dbg) { + auto status_and_enc_path = EncodePathWithNewBasename(dirname); + if (!status_and_enc_path.first.ok()) { + return status_and_enc_path.first; + } + return FileSystemWrapper::CreateDir(status_and_enc_path.second, options, dbg); +} + +IOStatus RemapFileSystem::CreateDirIfMissing(const std::string& dirname, + const IOOptions& options, + IODebugContext* dbg) { + auto status_and_enc_path = EncodePathWithNewBasename(dirname); + if (!status_and_enc_path.first.ok()) { + return status_and_enc_path.first; + } + return FileSystemWrapper::CreateDirIfMissing(status_and_enc_path.second, + options, dbg); +} + +IOStatus RemapFileSystem::DeleteDir(const std::string& dirname, + const IOOptions& options, + IODebugContext* dbg) { + auto status_and_enc_path = EncodePath(dirname); + if (!status_and_enc_path.first.ok()) { + return status_and_enc_path.first; + } + return FileSystemWrapper::DeleteDir(status_and_enc_path.second, options, dbg); +} + +IOStatus RemapFileSystem::GetFileSize(const std::string& fname, + const IOOptions& options, + uint64_t* file_size, + IODebugContext* dbg) { + auto status_and_enc_path = EncodePath(fname); + if (!status_and_enc_path.first.ok()) { + return status_and_enc_path.first; + } + return FileSystemWrapper::GetFileSize(status_and_enc_path.second, options, + file_size, dbg); +} + +IOStatus RemapFileSystem::GetFileModificationTime(const std::string& fname, + const IOOptions& options, + uint64_t* file_mtime, + IODebugContext* dbg) { + auto status_and_enc_path = EncodePath(fname); + if (!status_and_enc_path.first.ok()) { + return status_and_enc_path.first; + } + return FileSystemWrapper::GetFileModificationTime(status_and_enc_path.second, + options, file_mtime, dbg); +} + +IOStatus RemapFileSystem::IsDirectory(const std::string& path, + const IOOptions& options, bool* is_dir, + IODebugContext* dbg) { + auto status_and_enc_path = EncodePath(path); + if (!status_and_enc_path.first.ok()) { + return status_and_enc_path.first; + } + return FileSystemWrapper::IsDirectory(status_and_enc_path.second, options, + is_dir, dbg); +} + +IOStatus RemapFileSystem::RenameFile(const std::string& src, + const std::string& dest, + const IOOptions& options, + IODebugContext* dbg) { + auto status_and_src_enc_path = EncodePath(src); + if (!status_and_src_enc_path.first.ok()) { + if (status_and_src_enc_path.first.IsNotFound()) { + const IOStatus& s = status_and_src_enc_path.first; + status_and_src_enc_path.first = IOStatus::PathNotFound(s.ToString()); + } + return status_and_src_enc_path.first; + } + auto status_and_dest_enc_path = EncodePathWithNewBasename(dest); + if (!status_and_dest_enc_path.first.ok()) { + return status_and_dest_enc_path.first; + } + return FileSystemWrapper::RenameFile(status_and_src_enc_path.second, + status_and_dest_enc_path.second, options, + dbg); +} + +IOStatus RemapFileSystem::LinkFile(const std::string& src, + const std::string& dest, + const IOOptions& options, + IODebugContext* dbg) { + auto status_and_src_enc_path = EncodePath(src); + if (!status_and_src_enc_path.first.ok()) { + return status_and_src_enc_path.first; + } + auto status_and_dest_enc_path = EncodePathWithNewBasename(dest); + if (!status_and_dest_enc_path.first.ok()) { + return status_and_dest_enc_path.first; + } + return FileSystemWrapper::LinkFile(status_and_src_enc_path.second, + status_and_dest_enc_path.second, options, + dbg); +} + +IOStatus RemapFileSystem::LockFile(const std::string& fname, + const IOOptions& options, FileLock** lock, + IODebugContext* dbg) { + auto status_and_enc_path = EncodePathWithNewBasename(fname); + if (!status_and_enc_path.first.ok()) { + return status_and_enc_path.first; + } + // FileLock subclasses may store path (e.g., PosixFileLock stores it). We + // can skip stripping the chroot directory from this path because callers + // shouldn't use it. + return FileSystemWrapper::LockFile(status_and_enc_path.second, options, lock, + dbg); +} + +IOStatus RemapFileSystem::NewLogger(const std::string& fname, + const IOOptions& options, + std::shared_ptr* result, + IODebugContext* dbg) { + auto status_and_enc_path = EncodePathWithNewBasename(fname); + if (!status_and_enc_path.first.ok()) { + return status_and_enc_path.first; + } + return FileSystemWrapper::NewLogger(status_and_enc_path.second, options, + result, dbg); +} + +IOStatus RemapFileSystem::GetAbsolutePath(const std::string& db_path, + const IOOptions& options, + std::string* output_path, + IODebugContext* dbg) { + auto status_and_enc_path = EncodePathWithNewBasename(db_path); + if (!status_and_enc_path.first.ok()) { + return status_and_enc_path.first; + } + return FileSystemWrapper::GetAbsolutePath(status_and_enc_path.second, options, + output_path, dbg); +} + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/env/fs_remap.h b/librocksdb-sys/rocksdb/env/fs_remap.h new file mode 100644 index 0000000..a3c9982 --- /dev/null +++ b/librocksdb-sys/rocksdb/env/fs_remap.h @@ -0,0 +1,137 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + + +#include + +#include "rocksdb/file_system.h" + +namespace ROCKSDB_NAMESPACE { + +// An abstract FileSystem wrapper that creates a view of an existing +// FileSystem by remapping names in some way. +// +// This class has not been fully analyzed for providing strong security +// guarantees. +class RemapFileSystem : public FileSystemWrapper { + public: + explicit RemapFileSystem(const std::shared_ptr& base); + + protected: + // Returns status and mapped-to path in the wrapped filesystem. + // If it returns non-OK status, the returned path should not be used. + virtual std::pair EncodePath( + const std::string& path) = 0; + + // Similar to EncodePath() except used in cases in which it is OK for + // no file or directory on 'path' to already exist, such as if the + // operation would create one. However, the parent of 'path' is expected + // to exist for the operation to succeed. + // Default implementation: call EncodePath + virtual std::pair EncodePathWithNewBasename( + const std::string& path); + + public: + // Left abstract: + // const char* Name() const override { ... } + static const char* kClassName() { return "RemapFileSystem"; } + bool IsInstanceOf(const std::string& id) const override { + if (id == kClassName()) { + return true; + } else { + return FileSystemWrapper::IsInstanceOf(id); + } + } + + Status RegisterDbPaths(const std::vector& paths) override; + + Status UnregisterDbPaths(const std::vector& paths) override; + + IOStatus NewSequentialFile(const std::string& fname, + const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus NewRandomAccessFile(const std::string& fname, + const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus NewWritableFile(const std::string& fname, const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus ReuseWritableFile(const std::string& fname, + const std::string& old_fname, + const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus NewRandomRWFile(const std::string& fname, const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus NewDirectory(const std::string& dir, const IOOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus FileExists(const std::string& fname, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus GetChildren(const std::string& dir, const IOOptions& options, + std::vector* result, + IODebugContext* dbg) override; + + IOStatus GetChildrenFileAttributes(const std::string& dir, + const IOOptions& options, + std::vector* result, + IODebugContext* dbg) override; + + IOStatus DeleteFile(const std::string& fname, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus CreateDir(const std::string& dirname, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus CreateDirIfMissing(const std::string& dirname, + const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus DeleteDir(const std::string& dirname, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus GetFileSize(const std::string& fname, const IOOptions& options, + uint64_t* file_size, IODebugContext* dbg) override; + + IOStatus GetFileModificationTime(const std::string& fname, + const IOOptions& options, + uint64_t* file_mtime, + IODebugContext* dbg) override; + + IOStatus IsDirectory(const std::string& path, const IOOptions& options, + bool* is_dir, IODebugContext* dbg) override; + + IOStatus RenameFile(const std::string& src, const std::string& dest, + const IOOptions& options, IODebugContext* dbg) override; + + IOStatus LinkFile(const std::string& src, const std::string& dest, + const IOOptions& options, IODebugContext* dbg) override; + + IOStatus LockFile(const std::string& fname, const IOOptions& options, + FileLock** lock, IODebugContext* dbg) override; + + IOStatus NewLogger(const std::string& fname, const IOOptions& options, + std::shared_ptr* result, + IODebugContext* dbg) override; + + IOStatus GetAbsolutePath(const std::string& db_path, const IOOptions& options, + std::string* output_path, + IODebugContext* dbg) override; +}; + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/env/io_posix.cc b/librocksdb-sys/rocksdb/env/io_posix.cc new file mode 100644 index 0000000..0ec0e9c --- /dev/null +++ b/librocksdb-sys/rocksdb/env/io_posix.cc @@ -0,0 +1,1733 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifdef ROCKSDB_LIB_IO_POSIX +#include "env/io_posix.h" + +#include +#include + +#include +#if defined(OS_LINUX) +#include +#ifndef FALLOC_FL_KEEP_SIZE +#include +#endif +#endif +#include +#include +#include +#include +#include +#include +#include +#ifdef OS_LINUX +#include +#include +#endif +#include "monitoring/iostats_context_imp.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/slice.h" +#include "test_util/sync_point.h" +#include "util/autovector.h" +#include "util/coding.h" +#include "util/string_util.h" + +#if defined(OS_LINUX) && !defined(F_SET_RW_HINT) +#define F_LINUX_SPECIFIC_BASE 1024 +#define F_SET_RW_HINT (F_LINUX_SPECIFIC_BASE + 12) +#endif + +namespace ROCKSDB_NAMESPACE { + +std::string IOErrorMsg(const std::string& context, + const std::string& file_name) { + if (file_name.empty()) { + return context; + } + return context + ": " + file_name; +} + +// file_name can be left empty if it is not unkown. +IOStatus IOError(const std::string& context, const std::string& file_name, + int err_number) { + switch (err_number) { + case ENOSPC: { + IOStatus s = IOStatus::NoSpace(IOErrorMsg(context, file_name), + errnoStr(err_number).c_str()); + s.SetRetryable(true); + return s; + } + case ESTALE: + return IOStatus::IOError(IOStatus::kStaleFile); + case ENOENT: + return IOStatus::PathNotFound(IOErrorMsg(context, file_name), + errnoStr(err_number).c_str()); + default: + return IOStatus::IOError(IOErrorMsg(context, file_name), + errnoStr(err_number).c_str()); + } +} + +// A wrapper for fadvise, if the platform doesn't support fadvise, +// it will simply return 0. +int Fadvise(int fd, off_t offset, size_t len, int advice) { +#ifdef OS_LINUX + return posix_fadvise(fd, offset, len, advice); +#else + (void)fd; + (void)offset; + (void)len; + (void)advice; + return 0; // simply do nothing. +#endif +} + +// A wrapper for fadvise, if the platform doesn't support fadvise, +// it will simply return 0. +int Madvise(void* addr, size_t len, int advice) { +#ifdef OS_LINUX + return posix_madvise(addr, len, advice); +#else + (void)addr; + (void)len; + (void)advice; + return 0; // simply do nothing. +#endif +} + +namespace { + +// On MacOS (and probably *BSD), the posix write and pwrite calls do not support +// buffers larger than 2^31-1 bytes. These two wrappers fix this issue by +// cutting the buffer in 1GB chunks. We use this chunk size to be sure to keep +// the writes aligned. + +bool PosixWrite(int fd, const char* buf, size_t nbyte) { + const size_t kLimit1Gb = 1UL << 30; + + const char* src = buf; + size_t left = nbyte; + + while (left != 0) { + size_t bytes_to_write = std::min(left, kLimit1Gb); + + ssize_t done = write(fd, src, bytes_to_write); + if (done < 0) { + if (errno == EINTR) { + continue; + } + return false; + } + left -= done; + src += done; + } + return true; +} + +bool PosixPositionedWrite(int fd, const char* buf, size_t nbyte, off_t offset) { + const size_t kLimit1Gb = 1UL << 30; + + const char* src = buf; + size_t left = nbyte; + + while (left != 0) { + size_t bytes_to_write = std::min(left, kLimit1Gb); + + ssize_t done = pwrite(fd, src, bytes_to_write, offset); + if (done < 0) { + if (errno == EINTR) { + continue; + } + return false; + } + left -= done; + offset += done; + src += done; + } + + return true; +} + +#ifdef ROCKSDB_RANGESYNC_PRESENT + +#if !defined(ZFS_SUPER_MAGIC) +// The magic number for ZFS was not exposed until recently. It should be fixed +// forever so we can just copy the magic number here. +#define ZFS_SUPER_MAGIC 0x2fc12fc1 +#endif + +bool IsSyncFileRangeSupported(int fd) { + // This function tracks and checks for cases where we know `sync_file_range` + // definitely will not work properly despite passing the compile-time check + // (`ROCKSDB_RANGESYNC_PRESENT`). If we are unsure, or if any of the checks + // fail in unexpected ways, we allow `sync_file_range` to be used. This way + // should minimize risk of impacting existing use cases. + struct statfs buf; + int ret = fstatfs(fd, &buf); + assert(ret == 0); + if (ret == 0 && buf.f_type == ZFS_SUPER_MAGIC) { + // Testing on ZFS showed the writeback did not happen asynchronously when + // `sync_file_range` was called, even though it returned success. Avoid it + // and use `fdatasync` instead to preserve the contract of `bytes_per_sync`, + // even though this'll incur extra I/O for metadata. + return false; + } + + ret = sync_file_range(fd, 0 /* offset */, 0 /* nbytes */, 0 /* flags */); + assert(!(ret == -1 && errno != ENOSYS)); + if (ret == -1 && errno == ENOSYS) { + // `sync_file_range` is not implemented on all platforms even if + // compile-time checks pass and a supported filesystem is in-use. For + // example, using ext4 on WSL (Windows Subsystem for Linux), + // `sync_file_range()` returns `ENOSYS` + // ("Function not implemented"). + return false; + } + // None of the known cases matched, so allow `sync_file_range` use. + return true; +} + +#undef ZFS_SUPER_MAGIC + +#endif // ROCKSDB_RANGESYNC_PRESENT + +} // anonymous namespace + +/* + * PosixSequentialFile + */ +PosixSequentialFile::PosixSequentialFile(const std::string& fname, FILE* file, + int fd, size_t logical_block_size, + const EnvOptions& options) + : filename_(fname), + file_(file), + fd_(fd), + use_direct_io_(options.use_direct_reads), + logical_sector_size_(logical_block_size) { + assert(!options.use_direct_reads || !options.use_mmap_reads); +} + +PosixSequentialFile::~PosixSequentialFile() { + if (!use_direct_io()) { + assert(file_); + fclose(file_); + } else { + assert(fd_); + close(fd_); + } +} + +IOStatus PosixSequentialFile::Read(size_t n, const IOOptions& /*opts*/, + Slice* result, char* scratch, + IODebugContext* /*dbg*/) { + assert(result != nullptr && !use_direct_io()); + IOStatus s; + size_t r = 0; + do { + clearerr(file_); + r = fread_unlocked(scratch, 1, n, file_); + } while (r == 0 && ferror(file_) && errno == EINTR); + *result = Slice(scratch, r); + if (r < n) { + if (feof(file_)) { + // We leave status as ok if we hit the end of the file + // We also clear the error so that the reads can continue + // if a new data is written to the file + clearerr(file_); + } else { + // A partial read with an error: return a non-ok status + s = IOError("While reading file sequentially", filename_, errno); + } + } + return s; +} + +IOStatus PosixSequentialFile::PositionedRead(uint64_t offset, size_t n, + const IOOptions& /*opts*/, + Slice* result, char* scratch, + IODebugContext* /*dbg*/) { + assert(use_direct_io()); + assert(IsSectorAligned(offset, GetRequiredBufferAlignment())); + assert(IsSectorAligned(n, GetRequiredBufferAlignment())); + assert(IsSectorAligned(scratch, GetRequiredBufferAlignment())); + + IOStatus s; + ssize_t r = -1; + size_t left = n; + char* ptr = scratch; + while (left > 0) { + r = pread(fd_, ptr, left, static_cast(offset)); + if (r <= 0) { + if (r == -1 && errno == EINTR) { + continue; + } + break; + } + ptr += r; + offset += r; + left -= r; + if (!IsSectorAligned(r, GetRequiredBufferAlignment())) { + // Bytes reads don't fill sectors. Should only happen at the end + // of the file. + break; + } + } + if (r < 0) { + // An error: return a non-ok status + s = IOError("While pread " + std::to_string(n) + " bytes from offset " + + std::to_string(offset), + filename_, errno); + } + *result = Slice(scratch, (r < 0) ? 0 : n - left); + return s; +} + +IOStatus PosixSequentialFile::Skip(uint64_t n) { + if (fseek(file_, static_cast(n), SEEK_CUR)) { + return IOError("While fseek to skip " + std::to_string(n) + " bytes", + filename_, errno); + } + return IOStatus::OK(); +} + +IOStatus PosixSequentialFile::InvalidateCache(size_t offset, size_t length) { +#ifndef OS_LINUX + (void)offset; + (void)length; + return IOStatus::OK(); +#else + if (!use_direct_io()) { + // free OS pages + int ret = Fadvise(fd_, offset, length, POSIX_FADV_DONTNEED); + if (ret != 0) { + return IOError("While fadvise NotNeeded offset " + + std::to_string(offset) + " len " + + std::to_string(length), + filename_, errno); + } + } + return IOStatus::OK(); +#endif +} + +/* + * PosixRandomAccessFile + */ +#if defined(OS_LINUX) +size_t PosixHelper::GetUniqueIdFromFile(int fd, char* id, size_t max_size) { + if (max_size < kMaxVarint64Length * 3) { + return 0; + } + + struct stat buf; + int result = fstat(fd, &buf); + if (result == -1) { + return 0; + } + + long version = 0; + result = ioctl(fd, FS_IOC_GETVERSION, &version); + TEST_SYNC_POINT_CALLBACK("GetUniqueIdFromFile:FS_IOC_GETVERSION", &result); + if (result == -1) { + return 0; + } + uint64_t uversion = (uint64_t)version; + + char* rid = id; + rid = EncodeVarint64(rid, buf.st_dev); + rid = EncodeVarint64(rid, buf.st_ino); + rid = EncodeVarint64(rid, uversion); + assert(rid >= id); + return static_cast(rid - id); +} +#endif + +#if defined(OS_MACOSX) || defined(OS_AIX) +size_t PosixHelper::GetUniqueIdFromFile(int fd, char* id, size_t max_size) { + if (max_size < kMaxVarint64Length * 3) { + return 0; + } + + struct stat buf; + int result = fstat(fd, &buf); + if (result == -1) { + return 0; + } + + char* rid = id; + rid = EncodeVarint64(rid, buf.st_dev); + rid = EncodeVarint64(rid, buf.st_ino); + rid = EncodeVarint64(rid, buf.st_gen); + assert(rid >= id); + return static_cast(rid - id); +} +#endif + +#ifdef OS_LINUX +std::string RemoveTrailingSlash(const std::string& path) { + std::string p = path; + if (p.size() > 1 && p.back() == '/') { + p.pop_back(); + } + return p; +} + +Status LogicalBlockSizeCache::RefAndCacheLogicalBlockSize( + const std::vector& directories) { + std::vector dirs; + dirs.reserve(directories.size()); + for (auto& d : directories) { + dirs.emplace_back(RemoveTrailingSlash(d)); + } + + std::map dir_sizes; + { + ReadLock lock(&cache_mutex_); + for (const auto& dir : dirs) { + if (cache_.find(dir) == cache_.end()) { + dir_sizes.emplace(dir, 0); + } + } + } + + Status s; + for (auto& dir_size : dir_sizes) { + s = get_logical_block_size_of_directory_(dir_size.first, &dir_size.second); + if (!s.ok()) { + return s; + } + } + + WriteLock lock(&cache_mutex_); + for (const auto& dir : dirs) { + auto& v = cache_[dir]; + v.ref++; + auto dir_size = dir_sizes.find(dir); + if (dir_size != dir_sizes.end()) { + v.size = dir_size->second; + } + } + return s; +} + +void LogicalBlockSizeCache::UnrefAndTryRemoveCachedLogicalBlockSize( + const std::vector& directories) { + std::vector dirs; + dirs.reserve(directories.size()); + for (auto& dir : directories) { + dirs.emplace_back(RemoveTrailingSlash(dir)); + } + + WriteLock lock(&cache_mutex_); + for (const auto& dir : dirs) { + auto it = cache_.find(dir); + if (it != cache_.end() && !(--(it->second.ref))) { + cache_.erase(it); + } + } +} + +size_t LogicalBlockSizeCache::GetLogicalBlockSize(const std::string& fname, + int fd) { + std::string dir = fname.substr(0, fname.find_last_of("/")); + if (dir.empty()) { + dir = "/"; + } + { + ReadLock lock(&cache_mutex_); + auto it = cache_.find(dir); + if (it != cache_.end()) { + return it->second.size; + } + } + return get_logical_block_size_of_fd_(fd); +} +#endif + +Status PosixHelper::GetLogicalBlockSizeOfDirectory(const std::string& directory, + size_t* size) { + int fd = open(directory.c_str(), O_DIRECTORY | O_RDONLY); + if (fd == -1) { + close(fd); + return Status::IOError("Cannot open directory " + directory); + } + *size = PosixHelper::GetLogicalBlockSizeOfFd(fd); + close(fd); + return Status::OK(); +} + +size_t PosixHelper::GetLogicalBlockSizeOfFd(int fd) { +#ifdef OS_LINUX + struct stat buf; + int result = fstat(fd, &buf); + if (result == -1) { + return kDefaultPageSize; + } + if (major(buf.st_dev) == 0) { + // Unnamed devices (e.g. non-device mounts), reserved as null device number. + // These don't have an entry in /sys/dev/block/. Return a sensible default. + return kDefaultPageSize; + } + + // Reading queue/logical_block_size does not require special permissions. + const int kBufferSize = 100; + char path[kBufferSize]; + char real_path[PATH_MAX + 1]; + snprintf(path, kBufferSize, "/sys/dev/block/%u:%u", major(buf.st_dev), + minor(buf.st_dev)); + if (realpath(path, real_path) == nullptr) { + return kDefaultPageSize; + } + std::string device_dir(real_path); + if (!device_dir.empty() && device_dir.back() == '/') { + device_dir.pop_back(); + } + // NOTE: sda3 and nvme0n1p1 do not have a `queue/` subdir, only the parent sda + // and nvme0n1 have it. + // $ ls -al '/sys/dev/block/8:3' + // lrwxrwxrwx. 1 root root 0 Jun 26 01:38 /sys/dev/block/8:3 -> + // ../../block/sda/sda3 + // $ ls -al '/sys/dev/block/259:4' + // lrwxrwxrwx 1 root root 0 Jan 31 16:04 /sys/dev/block/259:4 -> + // ../../devices/pci0000:17/0000:17:00.0/0000:18:00.0/nvme/nvme0/nvme0n1/nvme0n1p1 + size_t parent_end = device_dir.rfind('/', device_dir.length() - 1); + if (parent_end == std::string::npos) { + return kDefaultPageSize; + } + size_t parent_begin = device_dir.rfind('/', parent_end - 1); + if (parent_begin == std::string::npos) { + return kDefaultPageSize; + } + std::string parent = + device_dir.substr(parent_begin + 1, parent_end - parent_begin - 1); + std::string child = device_dir.substr(parent_end + 1, std::string::npos); + if (parent != "block" && + (child.compare(0, 4, "nvme") || child.find('p') != std::string::npos)) { + device_dir = device_dir.substr(0, parent_end); + } + std::string fname = device_dir + "/queue/logical_block_size"; + FILE* fp; + size_t size = 0; + fp = fopen(fname.c_str(), "r"); + if (fp != nullptr) { + char* line = nullptr; + size_t len = 0; + if (getline(&line, &len, fp) != -1) { + sscanf(line, "%zu", &size); + } + free(line); + fclose(fp); + } + if (size != 0 && (size & (size - 1)) == 0) { + return size; + } +#endif + (void)fd; + return kDefaultPageSize; +} + +/* + * PosixRandomAccessFile + * + * pread() based random-access + */ +PosixRandomAccessFile::PosixRandomAccessFile( + const std::string& fname, int fd, size_t logical_block_size, + const EnvOptions& options +#if defined(ROCKSDB_IOURING_PRESENT) + , + ThreadLocalPtr* thread_local_io_urings +#endif + ) + : filename_(fname), + fd_(fd), + use_direct_io_(options.use_direct_reads), + logical_sector_size_(logical_block_size) +#if defined(ROCKSDB_IOURING_PRESENT) + , + thread_local_io_urings_(thread_local_io_urings) +#endif +{ + assert(!options.use_direct_reads || !options.use_mmap_reads); + assert(!options.use_mmap_reads); +} + +PosixRandomAccessFile::~PosixRandomAccessFile() { close(fd_); } + +IOStatus PosixRandomAccessFile::Read(uint64_t offset, size_t n, + const IOOptions& /*opts*/, Slice* result, + char* scratch, + IODebugContext* /*dbg*/) const { + if (use_direct_io()) { + assert(IsSectorAligned(offset, GetRequiredBufferAlignment())); + assert(IsSectorAligned(n, GetRequiredBufferAlignment())); + assert(IsSectorAligned(scratch, GetRequiredBufferAlignment())); + } + IOStatus s; + ssize_t r = -1; + size_t left = n; + char* ptr = scratch; + while (left > 0) { + r = pread(fd_, ptr, left, static_cast(offset)); + if (r <= 0) { + if (r == -1 && errno == EINTR) { + continue; + } + break; + } + ptr += r; + offset += r; + left -= r; + if (use_direct_io() && + r % static_cast(GetRequiredBufferAlignment()) != 0) { + // Bytes reads don't fill sectors. Should only happen at the end + // of the file. + break; + } + } + if (r < 0) { + // An error: return a non-ok status + s = IOError("While pread offset " + std::to_string(offset) + " len " + + std::to_string(n), + filename_, errno); + } + *result = Slice(scratch, (r < 0) ? 0 : n - left); + return s; +} + +IOStatus PosixRandomAccessFile::MultiRead(FSReadRequest* reqs, size_t num_reqs, + const IOOptions& options, + IODebugContext* dbg) { + if (use_direct_io()) { + for (size_t i = 0; i < num_reqs; i++) { + assert(IsSectorAligned(reqs[i].offset, GetRequiredBufferAlignment())); + assert(IsSectorAligned(reqs[i].len, GetRequiredBufferAlignment())); + assert(IsSectorAligned(reqs[i].scratch, GetRequiredBufferAlignment())); + } + } + +#if defined(ROCKSDB_IOURING_PRESENT) + struct io_uring* iu = nullptr; + if (thread_local_io_urings_) { + iu = static_cast(thread_local_io_urings_->Get()); + if (iu == nullptr) { + iu = CreateIOUring(); + if (iu != nullptr) { + thread_local_io_urings_->Reset(iu); + } + } + } + + // Init failed, platform doesn't support io_uring. Fall back to + // serialized reads + if (iu == nullptr) { + return FSRandomAccessFile::MultiRead(reqs, num_reqs, options, dbg); + } + + IOStatus ios = IOStatus::OK(); + + struct WrappedReadRequest { + FSReadRequest* req; + struct iovec iov; + size_t finished_len; + explicit WrappedReadRequest(FSReadRequest* r) : req(r), finished_len(0) {} + }; + + autovector req_wraps; + autovector incomplete_rq_list; + std::unordered_set wrap_cache; + + for (size_t i = 0; i < num_reqs; i++) { + req_wraps.emplace_back(&reqs[i]); + } + + size_t reqs_off = 0; + while (num_reqs > reqs_off || !incomplete_rq_list.empty()) { + size_t this_reqs = (num_reqs - reqs_off) + incomplete_rq_list.size(); + + // If requests exceed depth, split it into batches + if (this_reqs > kIoUringDepth) this_reqs = kIoUringDepth; + + assert(incomplete_rq_list.size() <= this_reqs); + for (size_t i = 0; i < this_reqs; i++) { + WrappedReadRequest* rep_to_submit; + if (i < incomplete_rq_list.size()) { + rep_to_submit = incomplete_rq_list[i]; + } else { + rep_to_submit = &req_wraps[reqs_off++]; + } + assert(rep_to_submit->req->len > rep_to_submit->finished_len); + rep_to_submit->iov.iov_base = + rep_to_submit->req->scratch + rep_to_submit->finished_len; + rep_to_submit->iov.iov_len = + rep_to_submit->req->len - rep_to_submit->finished_len; + + struct io_uring_sqe* sqe; + sqe = io_uring_get_sqe(iu); + io_uring_prep_readv( + sqe, fd_, &rep_to_submit->iov, 1, + rep_to_submit->req->offset + rep_to_submit->finished_len); + io_uring_sqe_set_data(sqe, rep_to_submit); + wrap_cache.emplace(rep_to_submit); + } + incomplete_rq_list.clear(); + + ssize_t ret = + io_uring_submit_and_wait(iu, static_cast(this_reqs)); + TEST_SYNC_POINT_CALLBACK( + "PosixRandomAccessFile::MultiRead:io_uring_submit_and_wait:return1", + &ret); + TEST_SYNC_POINT_CALLBACK( + "PosixRandomAccessFile::MultiRead:io_uring_submit_and_wait:return2", + iu); + + if (static_cast(ret) != this_reqs) { + fprintf(stderr, "ret = %ld this_reqs: %ld\n", (long)ret, (long)this_reqs); + // If error happens and we submitted fewer than expected, it is an + // exception case and we don't retry here. We should still consume + // what is is submitted in the ring. + for (ssize_t i = 0; i < ret; i++) { + struct io_uring_cqe* cqe = nullptr; + io_uring_wait_cqe(iu, &cqe); + if (cqe != nullptr) { + io_uring_cqe_seen(iu, cqe); + } + } + return IOStatus::IOError("io_uring_submit_and_wait() requested " + + std::to_string(this_reqs) + " but returned " + + std::to_string(ret)); + } + + for (size_t i = 0; i < this_reqs; i++) { + struct io_uring_cqe* cqe = nullptr; + WrappedReadRequest* req_wrap; + + // We could use the peek variant here, but this seems safer in terms + // of our initial wait not reaping all completions + ret = io_uring_wait_cqe(iu, &cqe); + TEST_SYNC_POINT_CALLBACK( + "PosixRandomAccessFile::MultiRead:io_uring_wait_cqe:return", &ret); + if (ret) { + ios = IOStatus::IOError("io_uring_wait_cqe() returns " + + std::to_string(ret)); + + if (cqe != nullptr) { + io_uring_cqe_seen(iu, cqe); + } + continue; + } + + req_wrap = static_cast(io_uring_cqe_get_data(cqe)); + // Reset cqe data to catch any stray reuse of it + static_cast(cqe)->user_data = 0xd5d5d5d5d5d5d5d5; + // Check that we got a valid unique cqe data + auto wrap_check = wrap_cache.find(req_wrap); + if (wrap_check == wrap_cache.end()) { + fprintf(stderr, + "PosixRandomAccessFile::MultiRead: " + "Bad cqe data from IO uring - %p\n", + req_wrap); + port::PrintStack(); + ios = IOStatus::IOError("io_uring_cqe_get_data() returned " + + std::to_string((uint64_t)req_wrap)); + continue; + } + wrap_cache.erase(wrap_check); + + FSReadRequest* req = req_wrap->req; + size_t bytes_read = 0; + bool read_again = false; + UpdateResult(cqe, filename_, req->len, req_wrap->iov.iov_len, + false /*async_read*/, use_direct_io(), + GetRequiredBufferAlignment(), req_wrap->finished_len, req, + bytes_read, read_again); + int32_t res = cqe->res; + if (res >= 0) { + if (bytes_read == 0) { + if (read_again) { + Slice tmp_slice; + req->status = + Read(req->offset + req_wrap->finished_len, + req->len - req_wrap->finished_len, options, &tmp_slice, + req->scratch + req_wrap->finished_len, dbg); + req->result = + Slice(req->scratch, req_wrap->finished_len + tmp_slice.size()); + } + // else It means EOF so no need to do anything. + } else if (bytes_read < req_wrap->iov.iov_len) { + incomplete_rq_list.push_back(req_wrap); + } + } + io_uring_cqe_seen(iu, cqe); + } + wrap_cache.clear(); + } + return ios; +#else + return FSRandomAccessFile::MultiRead(reqs, num_reqs, options, dbg); +#endif +} + +IOStatus PosixRandomAccessFile::Prefetch(uint64_t offset, size_t n, + const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + IOStatus s; + if (!use_direct_io()) { + ssize_t r = 0; +#ifdef OS_LINUX + r = readahead(fd_, offset, n); +#endif +#ifdef OS_MACOSX + radvisory advice; + advice.ra_offset = static_cast(offset); + advice.ra_count = static_cast(n); + r = fcntl(fd_, F_RDADVISE, &advice); +#endif + if (r == -1) { + s = IOError("While prefetching offset " + std::to_string(offset) + + " len " + std::to_string(n), + filename_, errno); + } + } + return s; +} + +#if defined(OS_LINUX) || defined(OS_MACOSX) || defined(OS_AIX) +size_t PosixRandomAccessFile::GetUniqueId(char* id, size_t max_size) const { + return PosixHelper::GetUniqueIdFromFile(fd_, id, max_size); +} +#endif + +void PosixRandomAccessFile::Hint(AccessPattern pattern) { + if (use_direct_io()) { + return; + } + switch (pattern) { + case kNormal: + Fadvise(fd_, 0, 0, POSIX_FADV_NORMAL); + break; + case kRandom: + Fadvise(fd_, 0, 0, POSIX_FADV_RANDOM); + break; + case kSequential: + Fadvise(fd_, 0, 0, POSIX_FADV_SEQUENTIAL); + break; + case kWillNeed: + Fadvise(fd_, 0, 0, POSIX_FADV_WILLNEED); + break; + case kWontNeed: + Fadvise(fd_, 0, 0, POSIX_FADV_DONTNEED); + break; + default: + assert(false); + break; + } +} + +IOStatus PosixRandomAccessFile::InvalidateCache(size_t offset, size_t length) { + if (use_direct_io()) { + return IOStatus::OK(); + } +#ifndef OS_LINUX + (void)offset; + (void)length; + return IOStatus::OK(); +#else + // free OS pages + int ret = Fadvise(fd_, offset, length, POSIX_FADV_DONTNEED); + if (ret == 0) { + return IOStatus::OK(); + } + return IOError("While fadvise NotNeeded offset " + std::to_string(offset) + + " len " + std::to_string(length), + filename_, errno); +#endif +} + +IOStatus PosixRandomAccessFile::ReadAsync( + FSReadRequest& req, const IOOptions& /*opts*/, + std::function cb, void* cb_arg, + void** io_handle, IOHandleDeleter* del_fn, IODebugContext* /*dbg*/) { + if (use_direct_io()) { + assert(IsSectorAligned(req.offset, GetRequiredBufferAlignment())); + assert(IsSectorAligned(req.len, GetRequiredBufferAlignment())); + assert(IsSectorAligned(req.scratch, GetRequiredBufferAlignment())); + } + +#if defined(ROCKSDB_IOURING_PRESENT) + // io_uring_queue_init. + struct io_uring* iu = nullptr; + if (thread_local_io_urings_) { + iu = static_cast(thread_local_io_urings_->Get()); + if (iu == nullptr) { + iu = CreateIOUring(); + if (iu != nullptr) { + thread_local_io_urings_->Reset(iu); + } + } + } + + // Init failed, platform doesn't support io_uring. + if (iu == nullptr) { + return IOStatus::NotSupported("ReadAsync"); + } + + // Allocate io_handle. + IOHandleDeleter deletefn = [](void* args) -> void { + delete (static_cast(args)); + args = nullptr; + }; + + // Initialize Posix_IOHandle. + Posix_IOHandle* posix_handle = + new Posix_IOHandle(iu, cb, cb_arg, req.offset, req.len, req.scratch, + use_direct_io(), GetRequiredBufferAlignment()); + posix_handle->iov.iov_base = req.scratch; + posix_handle->iov.iov_len = req.len; + + *io_handle = static_cast(posix_handle); + *del_fn = deletefn; + + // Step 3: io_uring_sqe_set_data + struct io_uring_sqe* sqe; + sqe = io_uring_get_sqe(iu); + + io_uring_prep_readv(sqe, fd_, /*sqe->addr=*/&posix_handle->iov, + /*sqe->len=*/1, /*sqe->offset=*/posix_handle->offset); + + // Sets sqe->user_data to posix_handle. + io_uring_sqe_set_data(sqe, posix_handle); + + // Step 4: io_uring_submit + ssize_t ret = io_uring_submit(iu); + if (ret < 0) { + fprintf(stderr, "io_uring_submit error: %ld\n", long(ret)); + return IOStatus::IOError("io_uring_submit() requested but returned " + + std::to_string(ret)); + } + return IOStatus::OK(); +#else + (void)req; + (void)cb; + (void)cb_arg; + (void)io_handle; + (void)del_fn; + return IOStatus::NotSupported("ReadAsync"); +#endif +} + +/* + * PosixMmapReadableFile + * + * mmap() based random-access + */ +// base[0,length-1] contains the mmapped contents of the file. +PosixMmapReadableFile::PosixMmapReadableFile(const int fd, + const std::string& fname, + void* base, size_t length, + const EnvOptions& options) + : fd_(fd), filename_(fname), mmapped_region_(base), length_(length) { +#ifdef NDEBUG + (void)options; +#endif + fd_ = fd_ + 0; // suppress the warning for used variables + assert(options.use_mmap_reads); + assert(!options.use_direct_reads); +} + +PosixMmapReadableFile::~PosixMmapReadableFile() { + int ret = munmap(mmapped_region_, length_); + if (ret != 0) { + fprintf(stdout, "failed to munmap %p length %" ROCKSDB_PRIszt " \n", + mmapped_region_, length_); + } + close(fd_); +} + +IOStatus PosixMmapReadableFile::Read(uint64_t offset, size_t n, + const IOOptions& /*opts*/, Slice* result, + char* /*scratch*/, + IODebugContext* /*dbg*/) const { + IOStatus s; + if (offset > length_) { + *result = Slice(); + return IOError("While mmap read offset " + std::to_string(offset) + + " larger than file length " + std::to_string(length_), + filename_, EINVAL); + } else if (offset + n > length_) { + n = static_cast(length_ - offset); + } + *result = Slice(reinterpret_cast(mmapped_region_) + offset, n); + return s; +} + +void PosixMmapReadableFile::Hint(AccessPattern pattern) { + switch (pattern) { + case kNormal: + Madvise(mmapped_region_, length_, POSIX_MADV_NORMAL); + break; + case kRandom: + Madvise(mmapped_region_, length_, POSIX_MADV_RANDOM); + break; + case kSequential: + Madvise(mmapped_region_, length_, POSIX_MADV_SEQUENTIAL); + break; + case kWillNeed: + Madvise(mmapped_region_, length_, POSIX_MADV_WILLNEED); + break; + case kWontNeed: + Madvise(mmapped_region_, length_, POSIX_MADV_DONTNEED); + break; + default: + assert(false); + break; + } +} + +IOStatus PosixMmapReadableFile::InvalidateCache(size_t offset, size_t length) { +#ifndef OS_LINUX + (void)offset; + (void)length; + return IOStatus::OK(); +#else + // free OS pages + int ret = Fadvise(fd_, offset, length, POSIX_FADV_DONTNEED); + if (ret == 0) { + return IOStatus::OK(); + } + return IOError("While fadvise not needed. Offset " + std::to_string(offset) + + " len" + std::to_string(length), + filename_, errno); +#endif +} + +/* + * PosixMmapFile + * + * We preallocate up to an extra megabyte and use memcpy to append new + * data to the file. This is safe since we either properly close the + * file before reading from it, or for log files, the reading code + * knows enough to skip zero suffixes. + */ +IOStatus PosixMmapFile::UnmapCurrentRegion() { + TEST_KILL_RANDOM("PosixMmapFile::UnmapCurrentRegion:0"); + if (base_ != nullptr) { + int munmap_status = munmap(base_, limit_ - base_); + if (munmap_status != 0) { + return IOError("While munmap", filename_, munmap_status); + } + file_offset_ += limit_ - base_; + base_ = nullptr; + limit_ = nullptr; + last_sync_ = nullptr; + dst_ = nullptr; + + // Increase the amount we map the next time, but capped at 1MB + if (map_size_ < (1 << 20)) { + map_size_ *= 2; + } + } + return IOStatus::OK(); +} + +IOStatus PosixMmapFile::MapNewRegion() { +#ifdef ROCKSDB_FALLOCATE_PRESENT + assert(base_ == nullptr); + TEST_KILL_RANDOM("PosixMmapFile::UnmapCurrentRegion:0"); + // we can't fallocate with FALLOC_FL_KEEP_SIZE here + if (allow_fallocate_) { + IOSTATS_TIMER_GUARD(allocate_nanos); + int alloc_status = fallocate(fd_, 0, file_offset_, map_size_); + if (alloc_status != 0) { + // fallback to posix_fallocate + alloc_status = posix_fallocate(fd_, file_offset_, map_size_); + } + if (alloc_status != 0) { + return IOStatus::IOError("Error allocating space to file : " + filename_ + + "Error : " + errnoStr(alloc_status).c_str()); + } + } + + TEST_KILL_RANDOM("PosixMmapFile::Append:1"); + void* ptr = mmap(nullptr, map_size_, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, + file_offset_); + if (ptr == MAP_FAILED) { + return IOStatus::IOError("MMap failed on " + filename_); + } + TEST_KILL_RANDOM("PosixMmapFile::Append:2"); + + base_ = reinterpret_cast(ptr); + limit_ = base_ + map_size_; + dst_ = base_; + last_sync_ = base_; + return IOStatus::OK(); +#else + return IOStatus::NotSupported("This platform doesn't support fallocate()"); +#endif +} + +IOStatus PosixMmapFile::Msync() { + if (dst_ == last_sync_) { + return IOStatus::OK(); + } + // Find the beginnings of the pages that contain the first and last + // bytes to be synced. + size_t p1 = TruncateToPageBoundary(last_sync_ - base_); + size_t p2 = TruncateToPageBoundary(dst_ - base_ - 1); + last_sync_ = dst_; + TEST_KILL_RANDOM("PosixMmapFile::Msync:0"); + if (msync(base_ + p1, p2 - p1 + page_size_, MS_SYNC) < 0) { + return IOError("While msync", filename_, errno); + } + return IOStatus::OK(); +} + +PosixMmapFile::PosixMmapFile(const std::string& fname, int fd, size_t page_size, + const EnvOptions& options) + : filename_(fname), + fd_(fd), + page_size_(page_size), + map_size_(Roundup(65536, page_size)), + base_(nullptr), + limit_(nullptr), + dst_(nullptr), + last_sync_(nullptr), + file_offset_(0) { +#ifdef ROCKSDB_FALLOCATE_PRESENT + allow_fallocate_ = options.allow_fallocate; + fallocate_with_keep_size_ = options.fallocate_with_keep_size; +#else + (void)options; +#endif + assert((page_size & (page_size - 1)) == 0); + assert(options.use_mmap_writes); + assert(!options.use_direct_writes); +} + +PosixMmapFile::~PosixMmapFile() { + if (fd_ >= 0) { + IOStatus s = PosixMmapFile::Close(IOOptions(), nullptr); + s.PermitUncheckedError(); + } +} + +IOStatus PosixMmapFile::Append(const Slice& data, const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + const char* src = data.data(); + size_t left = data.size(); + while (left > 0) { + assert(base_ <= dst_); + assert(dst_ <= limit_); + size_t avail = limit_ - dst_; + if (avail == 0) { + IOStatus s = UnmapCurrentRegion(); + if (!s.ok()) { + return s; + } + s = MapNewRegion(); + if (!s.ok()) { + return s; + } + TEST_KILL_RANDOM("PosixMmapFile::Append:0"); + } + + size_t n = (left <= avail) ? left : avail; + assert(dst_); + memcpy(dst_, src, n); + dst_ += n; + src += n; + left -= n; + } + return IOStatus::OK(); +} + +IOStatus PosixMmapFile::Close(const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + IOStatus s; + size_t unused = limit_ - dst_; + + s = UnmapCurrentRegion(); + if (!s.ok()) { + s = IOError("While closing mmapped file", filename_, errno); + } else if (unused > 0) { + // Trim the extra space at the end of the file + if (ftruncate(fd_, file_offset_ - unused) < 0) { + s = IOError("While ftruncating mmaped file", filename_, errno); + } + } + + if (close(fd_) < 0) { + if (s.ok()) { + s = IOError("While closing mmapped file", filename_, errno); + } + } + + fd_ = -1; + base_ = nullptr; + limit_ = nullptr; + return s; +} + +IOStatus PosixMmapFile::Flush(const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + return IOStatus::OK(); +} + +IOStatus PosixMmapFile::Sync(const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { +#ifdef HAVE_FULLFSYNC + if (::fcntl(fd_, F_FULLFSYNC) < 0) { + return IOError("while fcntl(F_FULLSYNC) mmapped file", filename_, errno); + } +#else // HAVE_FULLFSYNC + if (fdatasync(fd_) < 0) { + return IOError("While fdatasync mmapped file", filename_, errno); + } +#endif // HAVE_FULLFSYNC + + return Msync(); +} + +/** + * Flush data as well as metadata to stable storage. + */ +IOStatus PosixMmapFile::Fsync(const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { +#ifdef HAVE_FULLFSYNC + if (::fcntl(fd_, F_FULLFSYNC) < 0) { + return IOError("While fcntl(F_FULLSYNC) on mmaped file", filename_, errno); + } +#else // HAVE_FULLFSYNC + if (fsync(fd_) < 0) { + return IOError("While fsync mmaped file", filename_, errno); + } +#endif // HAVE_FULLFSYNC + + return Msync(); +} + +/** + * Get the size of valid data in the file. This will not match the + * size that is returned from the filesystem because we use mmap + * to extend file by map_size every time. + */ +uint64_t PosixMmapFile::GetFileSize(const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + size_t used = dst_ - base_; + return file_offset_ + used; +} + +IOStatus PosixMmapFile::InvalidateCache(size_t offset, size_t length) { +#ifndef OS_LINUX + (void)offset; + (void)length; + return IOStatus::OK(); +#else + // free OS pages + int ret = Fadvise(fd_, offset, length, POSIX_FADV_DONTNEED); + if (ret == 0) { + return IOStatus::OK(); + } + return IOError("While fadvise NotNeeded mmapped file", filename_, errno); +#endif +} + +#ifdef ROCKSDB_FALLOCATE_PRESENT +IOStatus PosixMmapFile::Allocate(uint64_t offset, uint64_t len, + const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + assert(offset <= static_cast(std::numeric_limits::max())); + assert(len <= static_cast(std::numeric_limits::max())); + TEST_KILL_RANDOM("PosixMmapFile::Allocate:0"); + int alloc_status = 0; + if (allow_fallocate_) { + alloc_status = + fallocate(fd_, fallocate_with_keep_size_ ? FALLOC_FL_KEEP_SIZE : 0, + static_cast(offset), static_cast(len)); + } + if (alloc_status == 0) { + return IOStatus::OK(); + } else { + return IOError("While fallocate offset " + std::to_string(offset) + + " len " + std::to_string(len), + filename_, errno); + } +} +#endif + +/* + * PosixWritableFile + * + * Use posix write to write data to a file. + */ +PosixWritableFile::PosixWritableFile(const std::string& fname, int fd, + size_t logical_block_size, + const EnvOptions& options) + : FSWritableFile(options), + filename_(fname), + use_direct_io_(options.use_direct_writes), + fd_(fd), + filesize_(0), + logical_sector_size_(logical_block_size) { +#ifdef ROCKSDB_FALLOCATE_PRESENT + allow_fallocate_ = options.allow_fallocate; + fallocate_with_keep_size_ = options.fallocate_with_keep_size; +#endif +#ifdef ROCKSDB_RANGESYNC_PRESENT + sync_file_range_supported_ = IsSyncFileRangeSupported(fd_); +#endif // ROCKSDB_RANGESYNC_PRESENT + assert(!options.use_mmap_writes); +} + +PosixWritableFile::~PosixWritableFile() { + if (fd_ >= 0) { + IOStatus s = PosixWritableFile::Close(IOOptions(), nullptr); + s.PermitUncheckedError(); + } +} + +IOStatus PosixWritableFile::Append(const Slice& data, const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + if (use_direct_io()) { + assert(IsSectorAligned(data.size(), GetRequiredBufferAlignment())); + assert(IsSectorAligned(data.data(), GetRequiredBufferAlignment())); + } + const char* src = data.data(); + size_t nbytes = data.size(); + + if (!PosixWrite(fd_, src, nbytes)) { + return IOError("While appending to file", filename_, errno); + } + + filesize_ += nbytes; + return IOStatus::OK(); +} + +IOStatus PosixWritableFile::PositionedAppend(const Slice& data, uint64_t offset, + const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + if (use_direct_io()) { + assert(IsSectorAligned(offset, GetRequiredBufferAlignment())); + assert(IsSectorAligned(data.size(), GetRequiredBufferAlignment())); + assert(IsSectorAligned(data.data(), GetRequiredBufferAlignment())); + } + assert(offset <= static_cast(std::numeric_limits::max())); + const char* src = data.data(); + size_t nbytes = data.size(); + if (!PosixPositionedWrite(fd_, src, nbytes, static_cast(offset))) { + return IOError("While pwrite to file at offset " + std::to_string(offset), + filename_, errno); + } + filesize_ = offset + nbytes; + return IOStatus::OK(); +} + +IOStatus PosixWritableFile::Truncate(uint64_t size, const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + IOStatus s; + int r = ftruncate(fd_, size); + if (r < 0) { + s = IOError("While ftruncate file to size " + std::to_string(size), + filename_, errno); + } else { + filesize_ = size; + } + return s; +} + +IOStatus PosixWritableFile::Close(const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + IOStatus s; + + size_t block_size; + size_t last_allocated_block; + GetPreallocationStatus(&block_size, &last_allocated_block); + TEST_SYNC_POINT_CALLBACK("PosixWritableFile::Close", &last_allocated_block); + if (last_allocated_block > 0) { + // trim the extra space preallocated at the end of the file + // NOTE(ljin): we probably don't want to surface failure as an IOError, + // but it will be nice to log these errors. + int dummy __attribute__((__unused__)); + dummy = ftruncate(fd_, filesize_); +#if defined(ROCKSDB_FALLOCATE_PRESENT) && defined(FALLOC_FL_PUNCH_HOLE) + // in some file systems, ftruncate only trims trailing space if the + // new file size is smaller than the current size. Calling fallocate + // with FALLOC_FL_PUNCH_HOLE flag to explicitly release these unused + // blocks. FALLOC_FL_PUNCH_HOLE is supported on at least the following + // filesystems: + // XFS (since Linux 2.6.38) + // ext4 (since Linux 3.0) + // Btrfs (since Linux 3.7) + // tmpfs (since Linux 3.5) + // We ignore error since failure of this operation does not affect + // correctness. + struct stat file_stats; + int result = fstat(fd_, &file_stats); + // After ftruncate, we check whether ftruncate has the correct behavior. + // If not, we should hack it with FALLOC_FL_PUNCH_HOLE + if (result == 0 && + (file_stats.st_size + file_stats.st_blksize - 1) / + file_stats.st_blksize != + file_stats.st_blocks / (file_stats.st_blksize / 512)) { + IOSTATS_TIMER_GUARD(allocate_nanos); + if (allow_fallocate_) { + fallocate(fd_, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, filesize_, + block_size * last_allocated_block - filesize_); + } + } +#endif + } + + if (close(fd_) < 0) { + s = IOError("While closing file after writing", filename_, errno); + } + fd_ = -1; + return s; +} + +// write out the cached data to the OS cache +IOStatus PosixWritableFile::Flush(const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + return IOStatus::OK(); +} + +IOStatus PosixWritableFile::Sync(const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { +#ifdef HAVE_FULLFSYNC + if (::fcntl(fd_, F_FULLFSYNC) < 0) { + return IOError("while fcntl(F_FULLFSYNC)", filename_, errno); + } +#else // HAVE_FULLFSYNC + if (fdatasync(fd_) < 0) { + return IOError("While fdatasync", filename_, errno); + } +#endif // HAVE_FULLFSYNC + return IOStatus::OK(); +} + +IOStatus PosixWritableFile::Fsync(const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { +#ifdef HAVE_FULLFSYNC + if (::fcntl(fd_, F_FULLFSYNC) < 0) { + return IOError("while fcntl(F_FULLFSYNC)", filename_, errno); + } +#else // HAVE_FULLFSYNC + if (fsync(fd_) < 0) { + return IOError("While fsync", filename_, errno); + } +#endif // HAVE_FULLFSYNC + return IOStatus::OK(); +} + +bool PosixWritableFile::IsSyncThreadSafe() const { return true; } + +uint64_t PosixWritableFile::GetFileSize(const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + return filesize_; +} + +void PosixWritableFile::SetWriteLifeTimeHint(Env::WriteLifeTimeHint hint) { +#ifdef OS_LINUX +// Suppress Valgrind "Unimplemented functionality" error. +#ifndef ROCKSDB_VALGRIND_RUN + if (hint == write_hint_) { + return; + } + if (fcntl(fd_, F_SET_RW_HINT, &hint) == 0) { + write_hint_ = hint; + } +#else + (void)hint; +#endif // ROCKSDB_VALGRIND_RUN +#else + (void)hint; +#endif // OS_LINUX +} + +IOStatus PosixWritableFile::InvalidateCache(size_t offset, size_t length) { + if (use_direct_io()) { + return IOStatus::OK(); + } +#ifndef OS_LINUX + (void)offset; + (void)length; + return IOStatus::OK(); +#else + // free OS pages + int ret = Fadvise(fd_, offset, length, POSIX_FADV_DONTNEED); + if (ret == 0) { + return IOStatus::OK(); + } + return IOError("While fadvise NotNeeded", filename_, errno); +#endif +} + +#ifdef ROCKSDB_FALLOCATE_PRESENT +IOStatus PosixWritableFile::Allocate(uint64_t offset, uint64_t len, + const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + assert(offset <= static_cast(std::numeric_limits::max())); + assert(len <= static_cast(std::numeric_limits::max())); + TEST_KILL_RANDOM("PosixWritableFile::Allocate:0"); + IOSTATS_TIMER_GUARD(allocate_nanos); + int alloc_status = 0; + if (allow_fallocate_) { + alloc_status = + fallocate(fd_, fallocate_with_keep_size_ ? FALLOC_FL_KEEP_SIZE : 0, + static_cast(offset), static_cast(len)); + } + if (alloc_status == 0) { + return IOStatus::OK(); + } else { + return IOError("While fallocate offset " + std::to_string(offset) + + " len " + std::to_string(len), + filename_, errno); + } +} +#endif + +IOStatus PosixWritableFile::RangeSync(uint64_t offset, uint64_t nbytes, + const IOOptions& opts, + IODebugContext* dbg) { +#ifdef ROCKSDB_RANGESYNC_PRESENT + assert(offset <= static_cast(std::numeric_limits::max())); + assert(nbytes <= static_cast(std::numeric_limits::max())); + if (sync_file_range_supported_) { + int ret; + if (strict_bytes_per_sync_) { + // Specifying `SYNC_FILE_RANGE_WAIT_BEFORE` together with an offset/length + // that spans all bytes written so far tells `sync_file_range` to wait for + // any outstanding writeback requests to finish before issuing a new one. + ret = + sync_file_range(fd_, 0, static_cast(offset + nbytes), + SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WRITE); + } else { + ret = sync_file_range(fd_, static_cast(offset), + static_cast(nbytes), SYNC_FILE_RANGE_WRITE); + } + if (ret != 0) { + return IOError("While sync_file_range returned " + std::to_string(ret), + filename_, errno); + } + return IOStatus::OK(); + } +#endif // ROCKSDB_RANGESYNC_PRESENT + return FSWritableFile::RangeSync(offset, nbytes, opts, dbg); +} + +#ifdef OS_LINUX +size_t PosixWritableFile::GetUniqueId(char* id, size_t max_size) const { + return PosixHelper::GetUniqueIdFromFile(fd_, id, max_size); +} +#endif + +/* + * PosixRandomRWFile + */ + +PosixRandomRWFile::PosixRandomRWFile(const std::string& fname, int fd, + const EnvOptions& /*options*/) + : filename_(fname), fd_(fd) {} + +PosixRandomRWFile::~PosixRandomRWFile() { + if (fd_ >= 0) { + IOStatus s = Close(IOOptions(), nullptr); + s.PermitUncheckedError(); + } +} + +IOStatus PosixRandomRWFile::Write(uint64_t offset, const Slice& data, + const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + const char* src = data.data(); + size_t nbytes = data.size(); + if (!PosixPositionedWrite(fd_, src, nbytes, static_cast(offset))) { + return IOError("While write random read/write file at offset " + + std::to_string(offset), + filename_, errno); + } + + return IOStatus::OK(); +} + +IOStatus PosixRandomRWFile::Read(uint64_t offset, size_t n, + const IOOptions& /*opts*/, Slice* result, + char* scratch, IODebugContext* /*dbg*/) const { + size_t left = n; + char* ptr = scratch; + while (left > 0) { + ssize_t done = pread(fd_, ptr, left, offset); + if (done < 0) { + // error while reading from file + if (errno == EINTR) { + // read was interrupted, try again. + continue; + } + return IOError("While reading random read/write file offset " + + std::to_string(offset) + " len " + std::to_string(n), + filename_, errno); + } else if (done == 0) { + // Nothing more to read + break; + } + + // Read `done` bytes + ptr += done; + offset += done; + left -= done; + } + + *result = Slice(scratch, n - left); + return IOStatus::OK(); +} + +IOStatus PosixRandomRWFile::Flush(const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + return IOStatus::OK(); +} + +IOStatus PosixRandomRWFile::Sync(const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { +#ifdef HAVE_FULLFSYNC + if (::fcntl(fd_, F_FULLFSYNC) < 0) { + return IOError("while fcntl(F_FULLFSYNC) random rw file", filename_, errno); + } +#else // HAVE_FULLFSYNC + if (fdatasync(fd_) < 0) { + return IOError("While fdatasync random read/write file", filename_, errno); + } +#endif // HAVE_FULLFSYNC + return IOStatus::OK(); +} + +IOStatus PosixRandomRWFile::Fsync(const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { +#ifdef HAVE_FULLFSYNC + if (::fcntl(fd_, F_FULLFSYNC) < 0) { + return IOError("While fcntl(F_FULLSYNC) random rw file", filename_, errno); + } +#else // HAVE_FULLFSYNC + if (fsync(fd_) < 0) { + return IOError("While fsync random read/write file", filename_, errno); + } +#endif // HAVE_FULLFSYNC + return IOStatus::OK(); +} + +IOStatus PosixRandomRWFile::Close(const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + if (close(fd_) < 0) { + return IOError("While close random read/write file", filename_, errno); + } + fd_ = -1; + return IOStatus::OK(); +} + +PosixMemoryMappedFileBuffer::~PosixMemoryMappedFileBuffer() { + // TODO should have error handling though not much we can do... + munmap(this->base_, length_); +} + +/* + * PosixDirectory + */ +#if !defined(BTRFS_SUPER_MAGIC) +// The magic number for BTRFS is fixed, if it's not defined, define it here +#define BTRFS_SUPER_MAGIC 0x9123683E +#endif +PosixDirectory::PosixDirectory(int fd, const std::string& directory_name) + : fd_(fd), directory_name_(directory_name) { + is_btrfs_ = false; +#ifdef OS_LINUX + struct statfs buf; + int ret = fstatfs(fd, &buf); + is_btrfs_ = (ret == 0 && buf.f_type == static_cast( + BTRFS_SUPER_MAGIC)); +#endif +} + +PosixDirectory::~PosixDirectory() { + if (fd_ >= 0) { + IOStatus s = PosixDirectory::Close(IOOptions(), nullptr); + s.PermitUncheckedError(); + } +} + +IOStatus PosixDirectory::Fsync(const IOOptions& opts, IODebugContext* dbg) { + return FsyncWithDirOptions(opts, dbg, DirFsyncOptions()); +} + +// Users who want the file entries synced in Directory project must call a +// Fsync or FsyncWithDirOptions function before Close +IOStatus PosixDirectory::Close(const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + IOStatus s = IOStatus::OK(); + if (close(fd_) < 0) { + s = IOError("While closing directory ", directory_name_, errno); + } else { + fd_ = -1; + } + return s; +} + +IOStatus PosixDirectory::FsyncWithDirOptions( + const IOOptions& /*opts*/, IODebugContext* /*dbg*/, + const DirFsyncOptions& dir_fsync_options) { + assert(fd_ >= 0); // Check use after close + IOStatus s = IOStatus::OK(); +#ifndef OS_AIX + if (is_btrfs_) { + // skip dir fsync for new file creation, which is not needed for btrfs + if (dir_fsync_options.reason == DirFsyncOptions::kNewFileSynced) { + return s; + } + // skip dir fsync for renaming file, only need to sync new file + if (dir_fsync_options.reason == DirFsyncOptions::kFileRenamed) { + std::string new_name = dir_fsync_options.renamed_new_name; + assert(!new_name.empty()); + int fd; + do { + IOSTATS_TIMER_GUARD(open_nanos); + fd = open(new_name.c_str(), O_RDONLY); + } while (fd < 0 && errno == EINTR); + if (fd < 0) { + s = IOError("While open renaming file", new_name, errno); + } else if (fsync(fd) < 0) { + s = IOError("While fsync renaming file", new_name, errno); + } + if (close(fd) < 0) { + s = IOError("While closing file after fsync", new_name, errno); + } + return s; + } + // fallback to dir-fsync for kDefault, kDirRenamed and kFileDeleted + } + + // skip fsync/fcntl when fd_ == -1 since this file descriptor has been closed + // in either the de-construction or the close function, data must have been + // fsync-ed before de-construction and close is called +#ifdef HAVE_FULLFSYNC + // btrfs is a Linux file system, while currently F_FULLFSYNC is available on + // Mac OS. + assert(!is_btrfs_); + if (fd_ != -1 && ::fcntl(fd_, F_FULLFSYNC) < 0) { + return IOError("while fcntl(F_FULLFSYNC)", "a directory", errno); + } +#else // HAVE_FULLFSYNC + if (fd_ != -1 && fsync(fd_) == -1) { + s = IOError("While fsync", "a directory", errno); + } +#endif // HAVE_FULLFSYNC +#endif // OS_AIX + return s; +} +} // namespace ROCKSDB_NAMESPACE +#endif diff --git a/librocksdb-sys/rocksdb/env/io_posix.h b/librocksdb-sys/rocksdb/env/io_posix.h new file mode 100644 index 0000000..f129668 --- /dev/null +++ b/librocksdb-sys/rocksdb/env/io_posix.h @@ -0,0 +1,523 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once +#include +#if defined(ROCKSDB_IOURING_PRESENT) +#include +#include +#endif +#include + +#include +#include +#include +#include + +#include "port/port.h" +#include "rocksdb/env.h" +#include "rocksdb/file_system.h" +#include "rocksdb/io_status.h" +#include "test_util/sync_point.h" +#include "util/mutexlock.h" +#include "util/thread_local.h" + +// For non linux platform, the following macros are used only as place +// holder. +#if !(defined OS_LINUX) && !(defined CYGWIN) && !(defined OS_AIX) +#define POSIX_FADV_NORMAL 0 /* [MC1] no further special treatment */ +#define POSIX_FADV_RANDOM 1 /* [MC1] expect random page refs */ +#define POSIX_FADV_SEQUENTIAL 2 /* [MC1] expect sequential page refs */ +#define POSIX_FADV_WILLNEED 3 /* [MC1] will need these pages */ +#define POSIX_FADV_DONTNEED 4 /* [MC1] don't need these pages */ + +#define POSIX_MADV_NORMAL 0 /* [MC1] no further special treatment */ +#define POSIX_MADV_RANDOM 1 /* [MC1] expect random page refs */ +#define POSIX_MADV_SEQUENTIAL 2 /* [MC1] expect sequential page refs */ +#define POSIX_MADV_WILLNEED 3 /* [MC1] will need these pages */ +#define POSIX_MADV_DONTNEED 4 /* [MC1] don't need these pages */ +#endif + +namespace ROCKSDB_NAMESPACE { +std::string IOErrorMsg(const std::string& context, + const std::string& file_name); +// file_name can be left empty if it is not unkown. +IOStatus IOError(const std::string& context, const std::string& file_name, + int err_number); + +class PosixHelper { + public: + static size_t GetUniqueIdFromFile(int fd, char* id, size_t max_size); + static size_t GetLogicalBlockSizeOfFd(int fd); + static Status GetLogicalBlockSizeOfDirectory(const std::string& directory, + size_t* size); +}; + +/* + * DirectIOHelper + */ +inline bool IsSectorAligned(const size_t off, size_t sector_size) { + assert((sector_size & (sector_size - 1)) == 0); + return (off & (sector_size - 1)) == 0; +} + +#ifndef NDEBUG +inline bool IsSectorAligned(const void* ptr, size_t sector_size) { + return uintptr_t(ptr) % sector_size == 0; +} +#endif + +#if defined(ROCKSDB_IOURING_PRESENT) +struct Posix_IOHandle { + Posix_IOHandle(struct io_uring* _iu, + std::function _cb, + void* _cb_arg, uint64_t _offset, size_t _len, char* _scratch, + bool _use_direct_io, size_t _alignment) + : iu(_iu), + cb(_cb), + cb_arg(_cb_arg), + offset(_offset), + len(_len), + scratch(_scratch), + use_direct_io(_use_direct_io), + alignment(_alignment), + is_finished(false), + req_count(0) {} + + struct iovec iov; + struct io_uring* iu; + std::function cb; + void* cb_arg; + uint64_t offset; + size_t len; + char* scratch; + bool use_direct_io; + size_t alignment; + bool is_finished; + // req_count is used by AbortIO API to keep track of number of requests. + uint32_t req_count; +}; + +inline void UpdateResult(struct io_uring_cqe* cqe, const std::string& file_name, + size_t len, size_t iov_len, bool async_read, + bool use_direct_io, size_t alignment, + size_t& finished_len, FSReadRequest* req, + size_t& bytes_read, bool& read_again) { + read_again = false; + if (cqe->res < 0) { + req->result = Slice(req->scratch, 0); + req->status = IOError("Req failed", file_name, cqe->res); + } else { + bytes_read = static_cast(cqe->res); + TEST_SYNC_POINT_CALLBACK("UpdateResults::io_uring_result", &bytes_read); + if (bytes_read == iov_len) { + req->result = Slice(req->scratch, req->len); + req->status = IOStatus::OK(); + } else if (bytes_read == 0) { + /// cqe->res == 0 can means EOF, or can mean partial results. See + // comment + // https://github.com/facebook/rocksdb/pull/6441#issuecomment-589843435 + // Fall back to pread in this case. + if (use_direct_io && !IsSectorAligned(finished_len, alignment)) { + // Bytes reads don't fill sectors. Should only happen at the end + // of the file. + req->result = Slice(req->scratch, finished_len); + req->status = IOStatus::OK(); + } else { + if (async_read) { + // No bytes read. It can means EOF. In case of partial results, it's + // caller responsibility to call read/readasync again. + req->result = Slice(req->scratch, 0); + req->status = IOStatus::OK(); + } else { + read_again = true; + } + } + } else if (bytes_read < iov_len) { + assert(bytes_read > 0); + if (async_read) { + req->result = Slice(req->scratch, bytes_read); + req->status = IOStatus::OK(); + } else { + assert(bytes_read + finished_len < len); + finished_len += bytes_read; + } + } else { + req->result = Slice(req->scratch, 0); + req->status = IOError("Req returned more bytes than requested", file_name, + cqe->res); + } + } +#ifdef NDEBUG + (void)len; +#endif +} +#endif + +#ifdef OS_LINUX +// Files under a specific directory have the same logical block size. +// This class caches the logical block size for the specified directories to +// save the CPU cost of computing the size. +// Safe for concurrent access from multiple threads without any external +// synchronization. +class LogicalBlockSizeCache { + public: + LogicalBlockSizeCache( + std::function get_logical_block_size_of_fd = + PosixHelper::GetLogicalBlockSizeOfFd, + std::function + get_logical_block_size_of_directory = + PosixHelper::GetLogicalBlockSizeOfDirectory) + : get_logical_block_size_of_fd_(get_logical_block_size_of_fd), + get_logical_block_size_of_directory_( + get_logical_block_size_of_directory) {} + + // Takes the following actions: + // 1. Increases reference count of the directories; + // 2. If the directory's logical block size is not cached, + // compute the buffer size and cache the result. + Status RefAndCacheLogicalBlockSize( + const std::vector& directories); + + // Takes the following actions: + // 1. Decreases reference count of the directories; + // 2. If the reference count of a directory reaches 0, remove the directory + // from the cache. + void UnrefAndTryRemoveCachedLogicalBlockSize( + const std::vector& directories); + + // Returns the logical block size for the file. + // + // If the file is under a cached directory, return the cached size. + // Otherwise, the size is computed. + size_t GetLogicalBlockSize(const std::string& fname, int fd); + + int GetRefCount(const std::string& dir) { + ReadLock lock(&cache_mutex_); + auto it = cache_.find(dir); + if (it == cache_.end()) { + return 0; + } + return it->second.ref; + } + + size_t Size() const { return cache_.size(); } + + bool Contains(const std::string& dir) { + ReadLock lock(&cache_mutex_); + return cache_.find(dir) != cache_.end(); + } + + private: + struct CacheValue { + CacheValue() : size(0), ref(0) {} + + // Logical block size of the directory. + size_t size; + // Reference count of the directory. + int ref; + }; + + std::function get_logical_block_size_of_fd_; + std::function + get_logical_block_size_of_directory_; + + std::map cache_; + port::RWMutex cache_mutex_; +}; +#endif + +class PosixSequentialFile : public FSSequentialFile { + private: + std::string filename_; + FILE* file_; + int fd_; + bool use_direct_io_; + size_t logical_sector_size_; + + public: + PosixSequentialFile(const std::string& fname, FILE* file, int fd, + size_t logical_block_size, const EnvOptions& options); + virtual ~PosixSequentialFile(); + + virtual IOStatus Read(size_t n, const IOOptions& opts, Slice* result, + char* scratch, IODebugContext* dbg) override; + virtual IOStatus PositionedRead(uint64_t offset, size_t n, + const IOOptions& opts, Slice* result, + char* scratch, IODebugContext* dbg) override; + virtual IOStatus Skip(uint64_t n) override; + virtual IOStatus InvalidateCache(size_t offset, size_t length) override; + virtual bool use_direct_io() const override { return use_direct_io_; } + virtual size_t GetRequiredBufferAlignment() const override { + return logical_sector_size_; + } +}; + +#if defined(ROCKSDB_IOURING_PRESENT) +// io_uring instance queue depth +const unsigned int kIoUringDepth = 256; + +inline void DeleteIOUring(void* p) { + struct io_uring* iu = static_cast(p); + delete iu; +} + +inline struct io_uring* CreateIOUring() { + struct io_uring* new_io_uring = new struct io_uring; + int ret = io_uring_queue_init(kIoUringDepth, new_io_uring, 0); + if (ret) { + delete new_io_uring; + new_io_uring = nullptr; + } + return new_io_uring; +} +#endif // defined(ROCKSDB_IOURING_PRESENT) + +class PosixRandomAccessFile : public FSRandomAccessFile { + protected: + std::string filename_; + int fd_; + bool use_direct_io_; + size_t logical_sector_size_; +#if defined(ROCKSDB_IOURING_PRESENT) + ThreadLocalPtr* thread_local_io_urings_; +#endif + + public: + PosixRandomAccessFile(const std::string& fname, int fd, + size_t logical_block_size, const EnvOptions& options +#if defined(ROCKSDB_IOURING_PRESENT) + , + ThreadLocalPtr* thread_local_io_urings +#endif + ); + virtual ~PosixRandomAccessFile(); + + virtual IOStatus Read(uint64_t offset, size_t n, const IOOptions& opts, + Slice* result, char* scratch, + IODebugContext* dbg) const override; + + virtual IOStatus MultiRead(FSReadRequest* reqs, size_t num_reqs, + const IOOptions& options, + IODebugContext* dbg) override; + + virtual IOStatus Prefetch(uint64_t offset, size_t n, const IOOptions& opts, + IODebugContext* dbg) override; + +#if defined(OS_LINUX) || defined(OS_MACOSX) || defined(OS_AIX) + virtual size_t GetUniqueId(char* id, size_t max_size) const override; +#endif + virtual void Hint(AccessPattern pattern) override; + virtual IOStatus InvalidateCache(size_t offset, size_t length) override; + virtual bool use_direct_io() const override { return use_direct_io_; } + virtual size_t GetRequiredBufferAlignment() const override { + return logical_sector_size_; + } + // EXPERIMENTAL + virtual IOStatus ReadAsync( + FSReadRequest& req, const IOOptions& opts, + std::function cb, void* cb_arg, + void** io_handle, IOHandleDeleter* del_fn, IODebugContext* dbg) override; +}; + +class PosixWritableFile : public FSWritableFile { + protected: + const std::string filename_; + const bool use_direct_io_; + int fd_; + uint64_t filesize_; + size_t logical_sector_size_; +#ifdef ROCKSDB_FALLOCATE_PRESENT + bool allow_fallocate_; + bool fallocate_with_keep_size_; +#endif +#ifdef ROCKSDB_RANGESYNC_PRESENT + // Even if the syscall is present, the filesystem may still not properly + // support it, so we need to do a dynamic check too. + bool sync_file_range_supported_; +#endif // ROCKSDB_RANGESYNC_PRESENT + + public: + explicit PosixWritableFile(const std::string& fname, int fd, + size_t logical_block_size, + const EnvOptions& options); + virtual ~PosixWritableFile(); + + // Need to implement this so the file is truncated correctly + // with direct I/O + virtual IOStatus Truncate(uint64_t size, const IOOptions& opts, + IODebugContext* dbg) override; + virtual IOStatus Close(const IOOptions& opts, IODebugContext* dbg) override; + virtual IOStatus Append(const Slice& data, const IOOptions& opts, + IODebugContext* dbg) override; + virtual IOStatus Append(const Slice& data, const IOOptions& opts, + const DataVerificationInfo& /* verification_info */, + IODebugContext* dbg) override { + return Append(data, opts, dbg); + } + virtual IOStatus PositionedAppend(const Slice& data, uint64_t offset, + const IOOptions& opts, + IODebugContext* dbg) override; + virtual IOStatus PositionedAppend( + const Slice& data, uint64_t offset, const IOOptions& opts, + const DataVerificationInfo& /* verification_info */, + IODebugContext* dbg) override { + return PositionedAppend(data, offset, opts, dbg); + } + virtual IOStatus Flush(const IOOptions& opts, IODebugContext* dbg) override; + virtual IOStatus Sync(const IOOptions& opts, IODebugContext* dbg) override; + virtual IOStatus Fsync(const IOOptions& opts, IODebugContext* dbg) override; + virtual bool IsSyncThreadSafe() const override; + virtual bool use_direct_io() const override { return use_direct_io_; } + virtual void SetWriteLifeTimeHint(Env::WriteLifeTimeHint hint) override; + virtual uint64_t GetFileSize(const IOOptions& opts, + IODebugContext* dbg) override; + virtual IOStatus InvalidateCache(size_t offset, size_t length) override; + virtual size_t GetRequiredBufferAlignment() const override { + return logical_sector_size_; + } +#ifdef ROCKSDB_FALLOCATE_PRESENT + virtual IOStatus Allocate(uint64_t offset, uint64_t len, + const IOOptions& opts, + IODebugContext* dbg) override; +#endif + virtual IOStatus RangeSync(uint64_t offset, uint64_t nbytes, + const IOOptions& opts, + IODebugContext* dbg) override; +#ifdef OS_LINUX + virtual size_t GetUniqueId(char* id, size_t max_size) const override; +#endif +}; + +// mmap() based random-access +class PosixMmapReadableFile : public FSRandomAccessFile { + private: + int fd_; + std::string filename_; + void* mmapped_region_; + size_t length_; + + public: + PosixMmapReadableFile(const int fd, const std::string& fname, void* base, + size_t length, const EnvOptions& options); + virtual ~PosixMmapReadableFile(); + IOStatus Read(uint64_t offset, size_t n, const IOOptions& opts, Slice* result, + char* scratch, IODebugContext* dbg) const override; + void Hint(AccessPattern pattern) override; + IOStatus InvalidateCache(size_t offset, size_t length) override; +}; + +class PosixMmapFile : public FSWritableFile { + private: + std::string filename_; + int fd_; + size_t page_size_; + size_t map_size_; // How much extra memory to map at a time + char* base_; // The mapped region + char* limit_; // Limit of the mapped region + char* dst_; // Where to write next (in range [base_,limit_]) + char* last_sync_; // Where have we synced up to + uint64_t file_offset_; // Offset of base_ in file +#ifdef ROCKSDB_FALLOCATE_PRESENT + bool allow_fallocate_; // If false, fallocate calls are bypassed + bool fallocate_with_keep_size_; +#endif + + // Roundup x to a multiple of y + static size_t Roundup(size_t x, size_t y) { return ((x + y - 1) / y) * y; } + + size_t TruncateToPageBoundary(size_t s) { + s -= (s & (page_size_ - 1)); + assert((s % page_size_) == 0); + return s; + } + + IOStatus MapNewRegion(); + IOStatus UnmapCurrentRegion(); + IOStatus Msync(); + + public: + PosixMmapFile(const std::string& fname, int fd, size_t page_size, + const EnvOptions& options); + ~PosixMmapFile(); + + // Means Close() will properly take care of truncate + // and it does not need any additional information + virtual IOStatus Truncate(uint64_t /*size*/, const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) override { + return IOStatus::OK(); + } + virtual IOStatus Close(const IOOptions& opts, IODebugContext* dbg) override; + virtual IOStatus Append(const Slice& data, const IOOptions& opts, + IODebugContext* dbg) override; + virtual IOStatus Append(const Slice& data, const IOOptions& opts, + const DataVerificationInfo& /* verification_info */, + IODebugContext* dbg) override { + return Append(data, opts, dbg); + } + virtual IOStatus Flush(const IOOptions& opts, IODebugContext* dbg) override; + virtual IOStatus Sync(const IOOptions& opts, IODebugContext* dbg) override; + virtual IOStatus Fsync(const IOOptions& opts, IODebugContext* dbg) override; + virtual uint64_t GetFileSize(const IOOptions& opts, + IODebugContext* dbg) override; + virtual IOStatus InvalidateCache(size_t offset, size_t length) override; +#ifdef ROCKSDB_FALLOCATE_PRESENT + virtual IOStatus Allocate(uint64_t offset, uint64_t len, + const IOOptions& opts, + IODebugContext* dbg) override; +#endif +}; + +class PosixRandomRWFile : public FSRandomRWFile { + public: + explicit PosixRandomRWFile(const std::string& fname, int fd, + const EnvOptions& options); + virtual ~PosixRandomRWFile(); + + virtual IOStatus Write(uint64_t offset, const Slice& data, + const IOOptions& opts, IODebugContext* dbg) override; + + virtual IOStatus Read(uint64_t offset, size_t n, const IOOptions& opts, + Slice* result, char* scratch, + IODebugContext* dbg) const override; + + virtual IOStatus Flush(const IOOptions& opts, IODebugContext* dbg) override; + virtual IOStatus Sync(const IOOptions& opts, IODebugContext* dbg) override; + virtual IOStatus Fsync(const IOOptions& opts, IODebugContext* dbg) override; + virtual IOStatus Close(const IOOptions& opts, IODebugContext* dbg) override; + + private: + const std::string filename_; + int fd_; +}; + +struct PosixMemoryMappedFileBuffer : public MemoryMappedFileBuffer { + PosixMemoryMappedFileBuffer(void* _base, size_t _length) + : MemoryMappedFileBuffer(_base, _length) {} + virtual ~PosixMemoryMappedFileBuffer(); +}; + +class PosixDirectory : public FSDirectory { + public: + explicit PosixDirectory(int fd, const std::string& directory_name); + ~PosixDirectory(); + virtual IOStatus Fsync(const IOOptions& opts, IODebugContext* dbg) override; + + virtual IOStatus Close(const IOOptions& opts, IODebugContext* dbg) override; + + virtual IOStatus FsyncWithDirOptions( + const IOOptions&, IODebugContext*, + const DirFsyncOptions& dir_fsync_options) override; + + private: + int fd_; + bool is_btrfs_; + const std::string directory_name_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/env/io_posix_test.cc b/librocksdb-sys/rocksdb/env/io_posix_test.cc new file mode 100644 index 0000000..81ce505 --- /dev/null +++ b/librocksdb-sys/rocksdb/env/io_posix_test.cc @@ -0,0 +1,141 @@ +// Copyright (c) 2020-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "test_util/testharness.h" + +#ifdef ROCKSDB_LIB_IO_POSIX +#include "env/io_posix.h" + +namespace ROCKSDB_NAMESPACE { + +#ifdef OS_LINUX +class LogicalBlockSizeCacheTest : public testing::Test {}; + +// Tests the caching behavior. +TEST_F(LogicalBlockSizeCacheTest, Cache) { + int ncall = 0; + auto get_fd_block_size = [&](int fd) { + ncall++; + return fd; + }; + std::map dir_fds{ + {"/", 0}, + {"/db", 1}, + {"/db1", 2}, + {"/db2", 3}, + }; + auto get_dir_block_size = [&](const std::string& dir, size_t* size) { + ncall++; + *size = dir_fds[dir]; + return Status::OK(); + }; + LogicalBlockSizeCache cache(get_fd_block_size, get_dir_block_size); + ASSERT_EQ(0, ncall); + ASSERT_EQ(0, cache.Size()); + + ASSERT_EQ(6, cache.GetLogicalBlockSize("/sst", 6)); + ASSERT_EQ(1, ncall); + ASSERT_EQ(7, cache.GetLogicalBlockSize("/db/sst1", 7)); + ASSERT_EQ(2, ncall); + ASSERT_EQ(8, cache.GetLogicalBlockSize("/db/sst2", 8)); + ASSERT_EQ(3, ncall); + + ASSERT_OK(cache.RefAndCacheLogicalBlockSize({"/", "/db1/", "/db2"})); + ASSERT_EQ(3, cache.Size()); + ASSERT_TRUE(cache.Contains("/")); + ASSERT_TRUE(cache.Contains("/db1")); + ASSERT_TRUE(cache.Contains("/db2")); + ASSERT_EQ(6, ncall); + // Block size for / is cached. + ASSERT_EQ(0, cache.GetLogicalBlockSize("/sst", 6)); + ASSERT_EQ(6, ncall); + // No cached size for /db. + ASSERT_EQ(7, cache.GetLogicalBlockSize("/db/sst1", 7)); + ASSERT_EQ(7, ncall); + ASSERT_EQ(8, cache.GetLogicalBlockSize("/db/sst2", 8)); + ASSERT_EQ(8, ncall); + // Block size for /db1 is cached. + ASSERT_EQ(2, cache.GetLogicalBlockSize("/db1/sst1", 4)); + ASSERT_EQ(8, ncall); + ASSERT_EQ(2, cache.GetLogicalBlockSize("/db1/sst2", 5)); + ASSERT_EQ(8, ncall); + // Block size for /db2 is cached. + ASSERT_EQ(3, cache.GetLogicalBlockSize("/db2/sst1", 6)); + ASSERT_EQ(8, ncall); + ASSERT_EQ(3, cache.GetLogicalBlockSize("/db2/sst2", 7)); + ASSERT_EQ(8, ncall); + + ASSERT_OK(cache.RefAndCacheLogicalBlockSize({"/db"})); + ASSERT_EQ(4, cache.Size()); + ASSERT_TRUE(cache.Contains("/")); + ASSERT_TRUE(cache.Contains("/db1")); + ASSERT_TRUE(cache.Contains("/db2")); + ASSERT_TRUE(cache.Contains("/db")); + + ASSERT_EQ(9, ncall); + // Block size for /db is cached. + ASSERT_EQ(1, cache.GetLogicalBlockSize("/db/sst1", 7)); + ASSERT_EQ(9, ncall); + ASSERT_EQ(1, cache.GetLogicalBlockSize("/db/sst2", 8)); + ASSERT_EQ(9, ncall); +} + +// Tests the reference counting behavior. +TEST_F(LogicalBlockSizeCacheTest, Ref) { + int ncall = 0; + auto get_fd_block_size = [&](int fd) { + ncall++; + return fd; + }; + std::map dir_fds{ + {"/db", 0}, + }; + auto get_dir_block_size = [&](const std::string& dir, size_t* size) { + ncall++; + *size = dir_fds[dir]; + return Status::OK(); + }; + LogicalBlockSizeCache cache(get_fd_block_size, get_dir_block_size); + + ASSERT_EQ(0, ncall); + + ASSERT_EQ(1, cache.GetLogicalBlockSize("/db/sst0", 1)); + ASSERT_EQ(1, ncall); + + ASSERT_OK(cache.RefAndCacheLogicalBlockSize({"/db"})); + ASSERT_EQ(2, ncall); + ASSERT_EQ(1, cache.GetRefCount("/db")); + // Block size for /db is cached. Ref count = 1. + ASSERT_EQ(0, cache.GetLogicalBlockSize("/db/sst1", 1)); + ASSERT_EQ(2, ncall); + + // Ref count = 2, but won't recompute the cached buffer size. + ASSERT_OK(cache.RefAndCacheLogicalBlockSize({"/db"})); + ASSERT_EQ(2, cache.GetRefCount("/db")); + ASSERT_EQ(2, ncall); + + // Ref count = 1. + cache.UnrefAndTryRemoveCachedLogicalBlockSize({"/db"}); + ASSERT_EQ(1, cache.GetRefCount("/db")); + // Block size for /db is still cached. + ASSERT_EQ(0, cache.GetLogicalBlockSize("/db/sst2", 1)); + ASSERT_EQ(2, ncall); + + // Ref count = 0 and cached buffer size for /db is removed. + cache.UnrefAndTryRemoveCachedLogicalBlockSize({"/db"}); + ASSERT_EQ(0, cache.Size()); + ASSERT_EQ(1, cache.GetLogicalBlockSize("/db/sst0", 1)); + ASSERT_EQ(3, ncall); +} +#endif + +} // namespace ROCKSDB_NAMESPACE +#endif + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/env/mock_env.cc b/librocksdb-sys/rocksdb/env/mock_env.cc new file mode 100644 index 0000000..9d6438a --- /dev/null +++ b/librocksdb-sys/rocksdb/env/mock_env.cc @@ -0,0 +1,1062 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "env/mock_env.h" + +#include +#include + +#include "env/emulated_clock.h" +#include "file/filename.h" +#include "port/sys_time.h" +#include "rocksdb/file_system.h" +#include "rocksdb/utilities/options_type.h" +#include "test_util/sync_point.h" +#include "util/cast_util.h" +#include "util/hash.h" +#include "util/random.h" +#include "util/rate_limiter_impl.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { +namespace { +int64_t MaybeCurrentTime(const std::shared_ptr& clock) { + int64_t time = 1337346000; // arbitrary fallback default + clock->GetCurrentTime(&time).PermitUncheckedError(); + return time; +} + +static std::unordered_map time_elapse_type_info = { + {"time_elapse_only_sleep", + {0, OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kCompareNever, + [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + const std::string& value, void* addr) { + auto clock = static_cast(addr); + clock->SetTimeElapseOnlySleep(ParseBoolean("", value)); + return Status::OK(); + }, + [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + const void* addr, std::string* value) { + const auto clock = static_cast(addr); + *value = clock->IsTimeElapseOnlySleep() ? "true" : "false"; + return Status::OK(); + }, + nullptr}}, +}; +static std::unordered_map mock_sleep_type_info = { + {"mock_sleep", + {0, OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kCompareNever, + [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + const std::string& value, void* addr) { + auto clock = static_cast(addr); + clock->SetMockSleep(ParseBoolean("", value)); + return Status::OK(); + }, + [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + const void* addr, std::string* value) { + const auto clock = static_cast(addr); + *value = clock->IsMockSleepEnabled() ? "true" : "false"; + return Status::OK(); + }, + nullptr}}, +}; +} // namespace + +EmulatedSystemClock::EmulatedSystemClock( + const std::shared_ptr& base, bool time_elapse_only_sleep) + : SystemClockWrapper(base), + maybe_starting_time_(MaybeCurrentTime(base)), + time_elapse_only_sleep_(time_elapse_only_sleep), + no_slowdown_(time_elapse_only_sleep) { + RegisterOptions("", this, &time_elapse_type_info); + RegisterOptions("", this, &mock_sleep_type_info); +} + +class MemFile { + public: + explicit MemFile(SystemClock* clock, const std::string& fn, + bool _is_lock_file = false) + : clock_(clock), + fn_(fn), + refs_(0), + is_lock_file_(_is_lock_file), + locked_(false), + size_(0), + modified_time_(Now()), + rnd_(Lower32of64(GetSliceNPHash64(fn))), + fsynced_bytes_(0) {} + // No copying allowed. + MemFile(const MemFile&) = delete; + void operator=(const MemFile&) = delete; + + void Ref() { + MutexLock lock(&mutex_); + ++refs_; + } + + bool is_lock_file() const { return is_lock_file_; } + + bool Lock() { + assert(is_lock_file_); + MutexLock lock(&mutex_); + if (locked_) { + return false; + } else { + locked_ = true; + return true; + } + } + + void Unlock() { + assert(is_lock_file_); + MutexLock lock(&mutex_); + locked_ = false; + } + + void Unref() { + bool do_delete = false; + { + MutexLock lock(&mutex_); + --refs_; + assert(refs_ >= 0); + if (refs_ <= 0) { + do_delete = true; + } + } + + if (do_delete) { + delete this; + } + } + + uint64_t Size() const { return size_; } + + void Truncate(size_t size, const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + MutexLock lock(&mutex_); + if (size < size_) { + data_.resize(size); + size_ = size; + } + } + + void CorruptBuffer() { + if (fsynced_bytes_ >= size_) { + return; + } + uint64_t buffered_bytes = size_ - fsynced_bytes_; + uint64_t start = + fsynced_bytes_ + rnd_.Uniform(static_cast(buffered_bytes)); + uint64_t end = std::min(start + 512, size_.load()); + MutexLock lock(&mutex_); + for (uint64_t pos = start; pos < end; ++pos) { + data_[static_cast(pos)] = static_cast(rnd_.Uniform(256)); + } + } + + IOStatus Read(uint64_t offset, size_t n, const IOOptions& /*options*/, + Slice* result, char* scratch, IODebugContext* /*dbg*/) const { + { + IOStatus s; + TEST_SYNC_POINT_CALLBACK("MemFile::Read:IOStatus", &s); + if (!s.ok()) { + // with sync point only + *result = Slice(); + return s; + } + } + MutexLock lock(&mutex_); + const uint64_t available = Size() - std::min(Size(), offset); + size_t offset_ = static_cast(offset); + if (n > available) { + n = static_cast(available); + } + if (n == 0) { + *result = Slice(); + return IOStatus::OK(); + } + if (scratch) { + memcpy(scratch, &(data_[offset_]), n); + *result = Slice(scratch, n); + } else { + *result = Slice(&(data_[offset_]), n); + } + return IOStatus::OK(); + } + + IOStatus Write(uint64_t offset, const Slice& data, + const IOOptions& /*options*/, IODebugContext* /*dbg*/) { + MutexLock lock(&mutex_); + size_t offset_ = static_cast(offset); + if (offset + data.size() > data_.size()) { + data_.resize(offset_ + data.size()); + } + data_.replace(offset_, data.size(), data.data(), data.size()); + size_ = data_.size(); + modified_time_ = Now(); + return IOStatus::OK(); + } + + IOStatus Append(const Slice& data, const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + MutexLock lock(&mutex_); + data_.append(data.data(), data.size()); + size_ = data_.size(); + modified_time_ = Now(); + return IOStatus::OK(); + } + + IOStatus Fsync(const IOOptions& /*options*/, IODebugContext* /*dbg*/) { + fsynced_bytes_ = size_.load(); + return IOStatus::OK(); + } + + uint64_t ModifiedTime() const { return modified_time_; } + + private: + uint64_t Now() { + int64_t unix_time = 0; + auto s = clock_->GetCurrentTime(&unix_time); + assert(s.ok()); + return static_cast(unix_time); + } + + // Private since only Unref() should be used to delete it. + ~MemFile() { assert(refs_ == 0); } + + SystemClock* clock_; + const std::string fn_; + mutable port::Mutex mutex_; + int refs_; + bool is_lock_file_; + bool locked_; + + // Data written into this file, all bytes before fsynced_bytes are + // persistent. + std::string data_; + std::atomic size_; + std::atomic modified_time_; + + Random rnd_; + std::atomic fsynced_bytes_; +}; + +namespace { + +class MockSequentialFile : public FSSequentialFile { + public: + explicit MockSequentialFile(MemFile* file, const FileOptions& opts) + : file_(file), + use_direct_io_(opts.use_direct_reads), + use_mmap_read_(opts.use_mmap_reads), + pos_(0) { + file_->Ref(); + } + + ~MockSequentialFile() override { file_->Unref(); } + + IOStatus Read(size_t n, const IOOptions& options, Slice* result, + char* scratch, IODebugContext* dbg) override { + IOStatus s = file_->Read(pos_, n, options, result, + (use_mmap_read_) ? nullptr : scratch, dbg); + if (s.ok()) { + pos_ += result->size(); + } + return s; + } + + bool use_direct_io() const override { return use_direct_io_; } + IOStatus Skip(uint64_t n) override { + if (pos_ > file_->Size()) { + return IOStatus::IOError("pos_ > file_->Size()"); + } + const uint64_t available = file_->Size() - pos_; + if (n > available) { + n = available; + } + pos_ += static_cast(n); + return IOStatus::OK(); + } + + private: + MemFile* file_; + bool use_direct_io_; + bool use_mmap_read_; + size_t pos_; +}; + +class MockRandomAccessFile : public FSRandomAccessFile { + public: + explicit MockRandomAccessFile(MemFile* file, const FileOptions& opts) + : file_(file), + use_direct_io_(opts.use_direct_reads), + use_mmap_read_(opts.use_mmap_reads) { + file_->Ref(); + } + + ~MockRandomAccessFile() override { file_->Unref(); } + + bool use_direct_io() const override { return use_direct_io_; } + + IOStatus Prefetch(uint64_t /*offset*/, size_t /*n*/, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return IOStatus::OK(); + } + + IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const override { + if (use_mmap_read_) { + return file_->Read(offset, n, options, result, nullptr, dbg); + } else { + return file_->Read(offset, n, options, result, scratch, dbg); + } + } + + private: + MemFile* file_; + bool use_direct_io_; + bool use_mmap_read_; +}; + +class MockRandomRWFile : public FSRandomRWFile { + public: + explicit MockRandomRWFile(MemFile* file) : file_(file) { file_->Ref(); } + + ~MockRandomRWFile() override { file_->Unref(); } + + IOStatus Write(uint64_t offset, const Slice& data, const IOOptions& options, + IODebugContext* dbg) override { + return file_->Write(offset, data, options, dbg); + } + + IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const override { + return file_->Read(offset, n, options, result, scratch, dbg); + } + + IOStatus Close(const IOOptions& options, IODebugContext* dbg) override { + return file_->Fsync(options, dbg); + } + + IOStatus Flush(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return IOStatus::OK(); + } + + IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override { + return file_->Fsync(options, dbg); + } + + private: + MemFile* file_; +}; + +class MockWritableFile : public FSWritableFile { + public: + MockWritableFile(MemFile* file, const FileOptions& opts) + : file_(file), + use_direct_io_(opts.use_direct_writes), + rate_limiter_(opts.rate_limiter) { + file_->Ref(); + } + + ~MockWritableFile() override { file_->Unref(); } + + bool use_direct_io() const override { return false && use_direct_io_; } + + using FSWritableFile::Append; + IOStatus Append(const Slice& data, const IOOptions& options, + IODebugContext* dbg) override { + size_t bytes_written = 0; + while (bytes_written < data.size()) { + auto bytes = RequestToken(data.size() - bytes_written); + IOStatus s = file_->Append(Slice(data.data() + bytes_written, bytes), + options, dbg); + if (!s.ok()) { + return s; + } + bytes_written += bytes; + } + return IOStatus::OK(); + } + + using FSWritableFile::PositionedAppend; + IOStatus PositionedAppend(const Slice& data, uint64_t /*offset*/, + const IOOptions& options, + IODebugContext* dbg) override { + assert(use_direct_io_); + return Append(data, options, dbg); + } + + IOStatus Truncate(uint64_t size, const IOOptions& options, + IODebugContext* dbg) override { + file_->Truncate(static_cast(size), options, dbg); + return IOStatus::OK(); + } + IOStatus Close(const IOOptions& options, IODebugContext* dbg) override { + return file_->Fsync(options, dbg); + } + + IOStatus Flush(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return IOStatus::OK(); + } + + IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override { + return file_->Fsync(options, dbg); + } + + uint64_t GetFileSize(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return file_->Size(); + } + + private: + inline size_t RequestToken(size_t bytes) { + if (rate_limiter_ && io_priority_ < Env::IO_TOTAL) { + bytes = std::min( + bytes, static_cast(rate_limiter_->GetSingleBurstBytes())); + rate_limiter_->Request(bytes, io_priority_); + } + return bytes; + } + + MemFile* file_; + bool use_direct_io_; + RateLimiter* rate_limiter_; +}; + +class MockEnvDirectory : public FSDirectory { + public: + IOStatus Fsync(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return IOStatus::OK(); + } + + IOStatus Close(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return IOStatus::OK(); + } +}; + +class MockEnvFileLock : public FileLock { + public: + explicit MockEnvFileLock(const std::string& fname) : fname_(fname) {} + + std::string FileName() const { return fname_; } + + private: + const std::string fname_; +}; + +class TestMemLogger : public Logger { + private: + std::unique_ptr file_; + std::atomic_size_t log_size_; + static const uint64_t flush_every_seconds_ = 5; + std::atomic_uint_fast64_t last_flush_micros_; + SystemClock* clock_; + IOOptions options_; + IODebugContext* dbg_; + std::atomic flush_pending_; + + public: + TestMemLogger(std::unique_ptr f, SystemClock* clock, + const IOOptions& options, IODebugContext* dbg, + const InfoLogLevel log_level = InfoLogLevel::ERROR_LEVEL) + : Logger(log_level), + file_(std::move(f)), + log_size_(0), + last_flush_micros_(0), + clock_(clock), + options_(options), + dbg_(dbg), + flush_pending_(false) {} + ~TestMemLogger() override {} + + void Flush() override { + if (flush_pending_) { + flush_pending_ = false; + } + last_flush_micros_ = clock_->NowMicros(); + } + + using Logger::Logv; + void Logv(const char* format, va_list ap) override { + // We try twice: the first time with a fixed-size stack allocated buffer, + // and the second time with a much larger dynamically allocated buffer. + char buffer[500]; + for (int iter = 0; iter < 2; iter++) { + char* base; + int bufsize; + if (iter == 0) { + bufsize = sizeof(buffer); + base = buffer; + } else { + bufsize = 30000; + base = new char[bufsize]; + } + char* p = base; + char* limit = base + bufsize; + + port::TimeVal now_tv; + port::GetTimeOfDay(&now_tv, nullptr); + const time_t seconds = now_tv.tv_sec; + struct tm t; + memset(&t, 0, sizeof(t)); + struct tm* ret __attribute__((__unused__)); + ret = port::LocalTimeR(&seconds, &t); + assert(ret); + p += snprintf(p, limit - p, "%04d/%02d/%02d-%02d:%02d:%02d.%06d ", + t.tm_year + 1900, t.tm_mon + 1, t.tm_mday, t.tm_hour, + t.tm_min, t.tm_sec, static_cast(now_tv.tv_usec)); + + // Print the message + if (p < limit) { + va_list backup_ap; + va_copy(backup_ap, ap); + p += vsnprintf(p, limit - p, format, backup_ap); + va_end(backup_ap); + } + + // Truncate to available space if necessary + if (p >= limit) { + if (iter == 0) { + continue; // Try again with larger buffer + } else { + p = limit - 1; + } + } + + // Add newline if necessary + if (p == base || p[-1] != '\n') { + *p++ = '\n'; + } + + assert(p <= limit); + const size_t write_size = p - base; + + Status s = file_->Append(Slice(base, write_size), options_, dbg_); + if (s.ok()) { + flush_pending_ = true; + log_size_ += write_size; + } + uint64_t now_micros = + static_cast(now_tv.tv_sec) * 1000000 + now_tv.tv_usec; + if (now_micros - last_flush_micros_ >= flush_every_seconds_ * 1000000) { + flush_pending_ = false; + last_flush_micros_ = now_micros; + } + if (base != buffer) { + delete[] base; + } + break; + } + } + size_t GetLogFileSize() const override { return log_size_; } +}; + +static std::unordered_map mock_fs_type_info = { + {"supports_direct_io", + {0, OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, +}; +} // namespace + +MockFileSystem::MockFileSystem(const std::shared_ptr& clock, + bool supports_direct_io) + : system_clock_(clock), supports_direct_io_(supports_direct_io) { + clock_ = system_clock_.get(); + RegisterOptions("", &supports_direct_io_, &mock_fs_type_info); +} + +MockFileSystem::~MockFileSystem() { + for (auto i = file_map_.begin(); i != file_map_.end(); ++i) { + i->second->Unref(); + } +} + +Status MockFileSystem::PrepareOptions(const ConfigOptions& options) { + Status s = FileSystem::PrepareOptions(options); + if (s.ok() && system_clock_ == SystemClock::Default()) { + system_clock_ = options.env->GetSystemClock(); + clock_ = system_clock_.get(); + } + return s; +} + +IOStatus MockFileSystem::GetAbsolutePath(const std::string& db_path, + const IOOptions& /*options*/, + std::string* output_path, + IODebugContext* /*dbg*/) { + *output_path = NormalizeMockPath(db_path); +#ifdef OS_WIN + return IOStatus::OK(); // TODO +#else + if (output_path->at(0) != '/') { + return IOStatus::NotSupported("GetAbsolutePath"); + } else { + return IOStatus::OK(); + } +#endif +} + +std::string MockFileSystem::NormalizeMockPath(const std::string& path) { + std::string p = NormalizePath(path); + if (p.back() == kFilePathSeparator && p.size() > 1) { + p.pop_back(); + } + return p; +} + +// Partial implementation of the FileSystem interface. +IOStatus MockFileSystem::NewSequentialFile( + const std::string& fname, const FileOptions& file_opts, + std::unique_ptr* result, IODebugContext* /*dbg*/) { + auto fn = NormalizeMockPath(fname); + + MutexLock lock(&mutex_); + if (file_map_.find(fn) == file_map_.end()) { + *result = nullptr; + return IOStatus::PathNotFound(fn); + } + auto* f = file_map_[fn]; + if (f->is_lock_file()) { + return IOStatus::InvalidArgument(fn, "Cannot open a lock file."); + } else if (file_opts.use_direct_reads && !supports_direct_io_) { + return IOStatus::NotSupported("Direct I/O Not Supported"); + } else { + result->reset(new MockSequentialFile(f, file_opts)); + return IOStatus::OK(); + } +} + +IOStatus MockFileSystem::NewRandomAccessFile( + const std::string& fname, const FileOptions& file_opts, + std::unique_ptr* result, IODebugContext* /*dbg*/) { + auto fn = NormalizeMockPath(fname); + MutexLock lock(&mutex_); + if (file_map_.find(fn) == file_map_.end()) { + *result = nullptr; + return IOStatus::PathNotFound(fn); + } + auto* f = file_map_[fn]; + if (f->is_lock_file()) { + return IOStatus::InvalidArgument(fn, "Cannot open a lock file."); + } else if (file_opts.use_direct_reads && !supports_direct_io_) { + return IOStatus::NotSupported("Direct I/O Not Supported"); + } else { + result->reset(new MockRandomAccessFile(f, file_opts)); + return IOStatus::OK(); + } +} + +IOStatus MockFileSystem::NewRandomRWFile( + const std::string& fname, const FileOptions& /*file_opts*/, + std::unique_ptr* result, IODebugContext* /*dbg*/) { + auto fn = NormalizeMockPath(fname); + MutexLock lock(&mutex_); + if (file_map_.find(fn) == file_map_.end()) { + *result = nullptr; + return IOStatus::PathNotFound(fn); + } + auto* f = file_map_[fn]; + if (f->is_lock_file()) { + return IOStatus::InvalidArgument(fn, "Cannot open a lock file."); + } + result->reset(new MockRandomRWFile(f)); + return IOStatus::OK(); +} + +IOStatus MockFileSystem::ReuseWritableFile( + const std::string& fname, const std::string& old_fname, + const FileOptions& options, std::unique_ptr* result, + IODebugContext* dbg) { + auto s = RenameFile(old_fname, fname, IOOptions(), dbg); + if (!s.ok()) { + return s; + } else { + result->reset(); + return NewWritableFile(fname, options, result, dbg); + } +} + +IOStatus MockFileSystem::NewWritableFile( + const std::string& fname, const FileOptions& file_opts, + std::unique_ptr* result, IODebugContext* /*dbg*/) { + auto fn = NormalizeMockPath(fname); + MutexLock lock(&mutex_); + if (file_map_.find(fn) != file_map_.end()) { + DeleteFileInternal(fn); + } + MemFile* file = new MemFile(clock_, fn, false); + file->Ref(); + file_map_[fn] = file; + if (file_opts.use_direct_writes && !supports_direct_io_) { + return IOStatus::NotSupported("Direct I/O Not Supported"); + } else { + result->reset(new MockWritableFile(file, file_opts)); + return IOStatus::OK(); + } +} + +IOStatus MockFileSystem::ReopenWritableFile( + const std::string& fname, const FileOptions& file_opts, + std::unique_ptr* result, IODebugContext* /*dbg*/) { + auto fn = NormalizeMockPath(fname); + MutexLock lock(&mutex_); + MemFile* file = nullptr; + if (file_map_.find(fn) == file_map_.end()) { + file = new MemFile(clock_, fn, false); + // Only take a reference when we create the file objectt + file->Ref(); + file_map_[fn] = file; + } else { + file = file_map_[fn]; + } + if (file_opts.use_direct_writes && !supports_direct_io_) { + return IOStatus::NotSupported("Direct I/O Not Supported"); + } else { + result->reset(new MockWritableFile(file, file_opts)); + return IOStatus::OK(); + } +} + +IOStatus MockFileSystem::NewDirectory(const std::string& /*name*/, + const IOOptions& /*io_opts*/, + std::unique_ptr* result, + IODebugContext* /*dbg*/) { + result->reset(new MockEnvDirectory()); + return IOStatus::OK(); +} + +IOStatus MockFileSystem::FileExists(const std::string& fname, + const IOOptions& /*io_opts*/, + IODebugContext* /*dbg*/) { + auto fn = NormalizeMockPath(fname); + MutexLock lock(&mutex_); + if (file_map_.find(fn) != file_map_.end()) { + // File exists + return IOStatus::OK(); + } + // Now also check if fn exists as a dir + for (const auto& iter : file_map_) { + const std::string& filename = iter.first; + if (filename.size() >= fn.size() + 1 && filename[fn.size()] == '/' && + Slice(filename).starts_with(Slice(fn))) { + return IOStatus::OK(); + } + } + return IOStatus::NotFound(); +} + +bool MockFileSystem::GetChildrenInternal(const std::string& dir, + std::vector* result) { + auto d = NormalizeMockPath(dir); + bool found_dir = false; + result->clear(); + for (const auto& iter : file_map_) { + const std::string& filename = iter.first; + + if (filename == d) { + found_dir = true; + } else if (filename.size() >= d.size() + 1 && filename[d.size()] == '/' && + Slice(filename).starts_with(Slice(d))) { + found_dir = true; + size_t next_slash = filename.find('/', d.size() + 1); + if (next_slash != std::string::npos) { + result->push_back( + filename.substr(d.size() + 1, next_slash - d.size() - 1)); + } else { + result->push_back(filename.substr(d.size() + 1)); + } + } + } + result->erase(std::unique(result->begin(), result->end()), result->end()); + return found_dir; +} + +IOStatus MockFileSystem::GetChildren(const std::string& dir, + const IOOptions& /*options*/, + std::vector* result, + IODebugContext* /*dbg*/) { + MutexLock lock(&mutex_); + bool found_dir = GetChildrenInternal(dir, result); +#ifndef __clang_analyzer__ + return found_dir ? IOStatus::OK() : IOStatus::NotFound(dir); +#else + return found_dir ? IOStatus::OK() : IOStatus::NotFound(); +#endif +} + +void MockFileSystem::DeleteFileInternal(const std::string& fname) { + assert(fname == NormalizeMockPath(fname)); + const auto& pair = file_map_.find(fname); + if (pair != file_map_.end()) { + pair->second->Unref(); + file_map_.erase(fname); + } +} + +IOStatus MockFileSystem::DeleteFile(const std::string& fname, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + auto fn = NormalizeMockPath(fname); + MutexLock lock(&mutex_); + if (file_map_.find(fn) == file_map_.end()) { + return IOStatus::PathNotFound(fn); + } + + DeleteFileInternal(fn); + return IOStatus::OK(); +} + +IOStatus MockFileSystem::Truncate(const std::string& fname, size_t size, + const IOOptions& options, + IODebugContext* dbg) { + auto fn = NormalizeMockPath(fname); + MutexLock lock(&mutex_); + auto iter = file_map_.find(fn); + if (iter == file_map_.end()) { + return IOStatus::PathNotFound(fn); + } + iter->second->Truncate(size, options, dbg); + return IOStatus::OK(); +} + +IOStatus MockFileSystem::CreateDir(const std::string& dirname, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + auto dn = NormalizeMockPath(dirname); + MutexLock lock(&mutex_); + if (file_map_.find(dn) == file_map_.end()) { + MemFile* file = new MemFile(clock_, dn, false); + file->Ref(); + file_map_[dn] = file; + } else { + return IOStatus::IOError(); + } + return IOStatus::OK(); +} + +IOStatus MockFileSystem::CreateDirIfMissing(const std::string& dirname, + const IOOptions& options, + IODebugContext* dbg) { + CreateDir(dirname, options, dbg).PermitUncheckedError(); + return IOStatus::OK(); +} + +IOStatus MockFileSystem::DeleteDir(const std::string& dirname, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + auto dir = NormalizeMockPath(dirname); + MutexLock lock(&mutex_); + if (file_map_.find(dir) == file_map_.end()) { + return IOStatus::PathNotFound(dir); + } else { + std::vector children; + if (GetChildrenInternal(dir, &children)) { + for (const auto& child : children) { + DeleteFileInternal(child); + } + } + DeleteFileInternal(dir); + return IOStatus::OK(); + } +} + +IOStatus MockFileSystem::GetFileSize(const std::string& fname, + const IOOptions& /*options*/, + uint64_t* file_size, + IODebugContext* /*dbg*/) { + auto fn = NormalizeMockPath(fname); + TEST_SYNC_POINT_CALLBACK("MockFileSystem::GetFileSize:CheckFileType", &fn); + MutexLock lock(&mutex_); + auto iter = file_map_.find(fn); + if (iter == file_map_.end()) { + return IOStatus::PathNotFound(fn); + } + + *file_size = iter->second->Size(); + return IOStatus::OK(); +} + +IOStatus MockFileSystem::GetFileModificationTime(const std::string& fname, + const IOOptions& /*options*/, + uint64_t* time, + IODebugContext* /*dbg*/) { + auto fn = NormalizeMockPath(fname); + MutexLock lock(&mutex_); + auto iter = file_map_.find(fn); + if (iter == file_map_.end()) { + return IOStatus::PathNotFound(fn); + } + *time = iter->second->ModifiedTime(); + return IOStatus::OK(); +} + +bool MockFileSystem::RenameFileInternal(const std::string& src, + const std::string& dest) { + if (file_map_.find(src) == file_map_.end()) { + return false; + } else { + std::vector children; + if (GetChildrenInternal(src, &children)) { + for (const auto& child : children) { + RenameFileInternal(src + "/" + child, dest + "/" + child); + } + } + DeleteFileInternal(dest); + file_map_[dest] = file_map_[src]; + file_map_.erase(src); + return true; + } +} + +IOStatus MockFileSystem::RenameFile(const std::string& src, + const std::string& dest, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + auto s = NormalizeMockPath(src); + auto t = NormalizeMockPath(dest); + MutexLock lock(&mutex_); + bool found = RenameFileInternal(s, t); + if (!found) { + return IOStatus::PathNotFound(s); + } else { + return IOStatus::OK(); + } +} + +IOStatus MockFileSystem::LinkFile(const std::string& src, + const std::string& dest, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + auto s = NormalizeMockPath(src); + auto t = NormalizeMockPath(dest); + MutexLock lock(&mutex_); + if (file_map_.find(s) == file_map_.end()) { + return IOStatus::PathNotFound(s); + } + + DeleteFileInternal(t); + file_map_[t] = file_map_[s]; + file_map_[t]->Ref(); // Otherwise it might get deleted when noone uses s + return IOStatus::OK(); +} + +IOStatus MockFileSystem::NewLogger(const std::string& fname, + const IOOptions& io_opts, + std::shared_ptr* result, + IODebugContext* dbg) { + auto fn = NormalizeMockPath(fname); + MutexLock lock(&mutex_); + auto iter = file_map_.find(fn); + MemFile* file = nullptr; + if (iter == file_map_.end()) { + file = new MemFile(clock_, fn, false); + file->Ref(); + file_map_[fn] = file; + } else { + file = iter->second; + } + std::unique_ptr f(new MockWritableFile(file, FileOptions())); + result->reset(new TestMemLogger(std::move(f), clock_, io_opts, dbg)); + return IOStatus::OK(); +} + +IOStatus MockFileSystem::LockFile(const std::string& fname, + const IOOptions& /*options*/, + FileLock** flock, IODebugContext* /*dbg*/) { + auto fn = NormalizeMockPath(fname); + { + MutexLock lock(&mutex_); + if (file_map_.find(fn) != file_map_.end()) { + if (!file_map_[fn]->is_lock_file()) { + return IOStatus::InvalidArgument(fname, "Not a lock file."); + } + if (!file_map_[fn]->Lock()) { + return IOStatus::IOError(fn, "lock is already held."); + } + } else { + auto* file = new MemFile(clock_, fn, true); + file->Ref(); + file->Lock(); + file_map_[fn] = file; + } + } + *flock = new MockEnvFileLock(fn); + return IOStatus::OK(); +} + +IOStatus MockFileSystem::UnlockFile(FileLock* flock, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + std::string fn = static_cast_with_check(flock)->FileName(); + { + MutexLock lock(&mutex_); + if (file_map_.find(fn) != file_map_.end()) { + if (!file_map_[fn]->is_lock_file()) { + return IOStatus::InvalidArgument(fn, "Not a lock file."); + } + file_map_[fn]->Unlock(); + } + } + delete flock; + return IOStatus::OK(); +} + +IOStatus MockFileSystem::GetTestDirectory(const IOOptions& /*options*/, + std::string* path, + IODebugContext* /*dbg*/) { + *path = "/test"; + return IOStatus::OK(); +} + +Status MockFileSystem::CorruptBuffer(const std::string& fname) { + auto fn = NormalizeMockPath(fname); + MutexLock lock(&mutex_); + auto iter = file_map_.find(fn); + if (iter == file_map_.end()) { + return Status::IOError(fn, "File not found"); + } + iter->second->CorruptBuffer(); + return Status::OK(); +} + +MockEnv::MockEnv(Env* env, const std::shared_ptr& fs, + const std::shared_ptr& clock) + : CompositeEnvWrapper(env, fs, clock) {} + +MockEnv* MockEnv::Create(Env* env) { + auto clock = + std::make_shared(env->GetSystemClock(), true); + return MockEnv::Create(env, clock); +} + +MockEnv* MockEnv::Create(Env* env, const std::shared_ptr& clock) { + auto fs = std::make_shared(clock); + return new MockEnv(env, fs, clock); +} + +Status MockEnv::CorruptBuffer(const std::string& fname) { + auto mock = static_cast_with_check(GetFileSystem().get()); + return mock->CorruptBuffer(fname); +} + +// This is to maintain the behavior before swithcing from InMemoryEnv to MockEnv +Env* NewMemEnv(Env* base_env) { return MockEnv::Create(base_env); } + + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/env/mock_env.h b/librocksdb-sys/rocksdb/env/mock_env.h new file mode 100644 index 0000000..406a31f --- /dev/null +++ b/librocksdb-sys/rocksdb/env/mock_env.h @@ -0,0 +1,144 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once + +#include +#include +#include +#include + +#include "env/composite_env_wrapper.h" +#include "port/port.h" +#include "rocksdb/env.h" +#include "rocksdb/status.h" +#include "rocksdb/system_clock.h" + +namespace ROCKSDB_NAMESPACE { +class MemFile; +class MockFileSystem : public FileSystem { + public: + explicit MockFileSystem(const std::shared_ptr& clock, + bool supports_direct_io = true); + ~MockFileSystem() override; + + static const char* kClassName() { return "MemoryFileSystem"; } + const char* Name() const override { return kClassName(); } + IOStatus NewSequentialFile(const std::string& f, const FileOptions& file_opts, + std::unique_ptr* r, + IODebugContext* dbg) override; + IOStatus NewRandomAccessFile(const std::string& f, + const FileOptions& file_opts, + std::unique_ptr* r, + IODebugContext* dbg) override; + + IOStatus NewRandomRWFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + IOStatus ReuseWritableFile(const std::string& fname, + const std::string& old_fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + IOStatus NewWritableFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + IOStatus ReopenWritableFile(const std::string& fname, + const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override; + IOStatus NewDirectory(const std::string& /*name*/, const IOOptions& io_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + IOStatus FileExists(const std::string& fname, const IOOptions& /*io_opts*/, + IODebugContext* /*dbg*/) override; + IOStatus GetChildren(const std::string& dir, const IOOptions& options, + std::vector* result, + IODebugContext* dbg) override; + IOStatus DeleteFile(const std::string& fname, const IOOptions& options, + IODebugContext* dbg) override; + IOStatus Truncate(const std::string& fname, size_t size, + const IOOptions& options, IODebugContext* dbg) override; + IOStatus CreateDir(const std::string& dirname, const IOOptions& options, + IODebugContext* dbg) override; + IOStatus CreateDirIfMissing(const std::string& dirname, + const IOOptions& options, + IODebugContext* dbg) override; + IOStatus DeleteDir(const std::string& dirname, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus GetFileSize(const std::string& fname, const IOOptions& options, + uint64_t* file_size, IODebugContext* dbg) override; + + IOStatus GetFileModificationTime(const std::string& fname, + const IOOptions& options, + uint64_t* file_mtime, + IODebugContext* dbg) override; + IOStatus RenameFile(const std::string& src, const std::string& target, + const IOOptions& options, IODebugContext* dbg) override; + IOStatus LinkFile(const std::string& /*src*/, const std::string& /*target*/, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override; + IOStatus LockFile(const std::string& fname, const IOOptions& options, + FileLock** lock, IODebugContext* dbg) override; + IOStatus UnlockFile(FileLock* lock, const IOOptions& options, + IODebugContext* dbg) override; + IOStatus GetTestDirectory(const IOOptions& options, std::string* path, + IODebugContext* dbg) override; + IOStatus NewLogger(const std::string& fname, const IOOptions& io_opts, + std::shared_ptr* result, + IODebugContext* dbg) override; + // Get full directory name for this db. + IOStatus GetAbsolutePath(const std::string& db_path, + const IOOptions& /*options*/, + std::string* output_path, + IODebugContext* /*dbg*/) override; + IOStatus IsDirectory(const std::string& /*path*/, + const IOOptions& /*options*/, bool* /*is_dir*/, + IODebugContext* /*dgb*/) override { + return IOStatus::NotSupported("IsDirectory"); + } + + Status CorruptBuffer(const std::string& fname); + Status PrepareOptions(const ConfigOptions& options) override; + + private: + bool RenameFileInternal(const std::string& src, const std::string& dest); + void DeleteFileInternal(const std::string& fname); + bool GetChildrenInternal(const std::string& fname, + std::vector* results); + + std::string NormalizeMockPath(const std::string& path); + + private: + // Map from filenames to MemFile objects, representing a simple file system. + port::Mutex mutex_; + std::map file_map_; // Protected by mutex_. + std::shared_ptr system_clock_; + SystemClock* clock_; + bool supports_direct_io_; +}; + +class MockEnv : public CompositeEnvWrapper { + public: + static MockEnv* Create(Env* base); + static MockEnv* Create(Env* base, const std::shared_ptr& clock); + + static const char* kClassName() { return "MockEnv"; } + const char* Name() const override { return kClassName(); } + + Status CorruptBuffer(const std::string& fname); + + private: + MockEnv(Env* env, const std::shared_ptr& fs, + const std::shared_ptr& clock); +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/env/mock_env_test.cc b/librocksdb-sys/rocksdb/env/mock_env_test.cc new file mode 100644 index 0000000..be174bd --- /dev/null +++ b/librocksdb-sys/rocksdb/env/mock_env_test.cc @@ -0,0 +1,84 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +#include "env/mock_env.h" + +#include +#include + +#include "rocksdb/env.h" +#include "test_util/testharness.h" + +namespace ROCKSDB_NAMESPACE { + +class MockEnvTest : public testing::Test { + public: + MockEnv* env_; + const EnvOptions soptions_; + + MockEnvTest() : env_(MockEnv::Create(Env::Default())) {} + ~MockEnvTest() override { delete env_; } +}; + +TEST_F(MockEnvTest, Corrupt) { + const std::string kGood = "this is a good string, synced to disk"; + const std::string kCorrupted = "this part may be corrupted"; + const std::string kFileName = "/dir/f"; + std::unique_ptr writable_file; + ASSERT_OK(env_->NewWritableFile(kFileName, &writable_file, soptions_)); + ASSERT_OK(writable_file->Append(kGood)); + ASSERT_TRUE(writable_file->GetFileSize() == kGood.size()); + + std::string scratch; + scratch.resize(kGood.size() + kCorrupted.size() + 16); + Slice result; + std::unique_ptr rand_file; + ASSERT_OK(env_->NewRandomAccessFile(kFileName, &rand_file, soptions_)); + ASSERT_OK(rand_file->Read(0, kGood.size(), &result, &(scratch[0]))); + ASSERT_EQ(result.compare(kGood), 0); + + // Sync + corrupt => no change + ASSERT_OK(writable_file->Fsync()); + ASSERT_OK(dynamic_cast(env_)->CorruptBuffer(kFileName)); + result.clear(); + ASSERT_OK(rand_file->Read(0, kGood.size(), &result, &(scratch[0]))); + ASSERT_EQ(result.compare(kGood), 0); + + // Add new data and corrupt it + ASSERT_OK(writable_file->Append(kCorrupted)); + ASSERT_TRUE(writable_file->GetFileSize() == kGood.size() + kCorrupted.size()); + result.clear(); + ASSERT_OK( + rand_file->Read(kGood.size(), kCorrupted.size(), &result, &(scratch[0]))); + ASSERT_EQ(result.compare(kCorrupted), 0); + // Corrupted + ASSERT_OK(dynamic_cast(env_)->CorruptBuffer(kFileName)); + result.clear(); + ASSERT_OK( + rand_file->Read(kGood.size(), kCorrupted.size(), &result, &(scratch[0]))); + ASSERT_NE(result.compare(kCorrupted), 0); +} + +TEST_F(MockEnvTest, FakeSleeping) { + int64_t now = 0; + auto s = env_->GetCurrentTime(&now); + ASSERT_OK(s); + env_->SleepForMicroseconds(3 * 1000 * 1000); + int64_t after_sleep = 0; + s = env_->GetCurrentTime(&after_sleep); + ASSERT_OK(s); + auto delta = after_sleep - now; + // this will be true unless test runs for 2 seconds + ASSERT_TRUE(delta == 3 || delta == 4); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/env/unique_id_gen.cc b/librocksdb-sys/rocksdb/env/unique_id_gen.cc new file mode 100644 index 0000000..7d221d3 --- /dev/null +++ b/librocksdb-sys/rocksdb/env/unique_id_gen.cc @@ -0,0 +1,242 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "env/unique_id_gen.h" + +#include +#include +#include +#include +#include +#include + +#include "port/lang.h" +#include "port/port.h" +#include "rocksdb/env.h" +#include "rocksdb/version.h" +#include "util/hash.h" + +#ifdef __SSE4_2__ +#ifdef _WIN32 +#include +#else +#include +#endif +#else +#include "rocksdb/system_clock.h" +#endif + +namespace ROCKSDB_NAMESPACE { + +namespace { + +struct GenerateRawUniqueIdOpts { + Env* env = Env::Default(); + bool exclude_port_uuid = false; + bool exclude_env_details = false; + bool exclude_random_device = false; +}; + +// Each of these "tracks" below should be sufficient for generating 128 bits +// of entropy, after hashing the raw bytes. The tracks are separable for +// testing purposes, but in production we combine as many tracks as possible +// to ensure quality results even if some environments have degraded +// capabilities or quality in some APIs. +// +// This approach has not been validated for use in cryptography. The goal is +// generating globally unique values with high probability without coordination +// between instances. +// +// Linux performance: EntropyTrackRandomDevice is much faster than +// EntropyTrackEnvDetails, which is much faster than EntropyTrackPortUuid. + +struct EntropyTrackPortUuid { + std::array uuid; + + void Populate(const GenerateRawUniqueIdOpts& opts) { + if (opts.exclude_port_uuid) { + return; + } + std::string s; + port::GenerateRfcUuid(&s); + if (s.size() >= uuid.size()) { + std::copy_n(s.begin(), uuid.size(), uuid.begin()); + } + } +}; + +struct EntropyTrackEnvDetails { + std::array hostname_buf; + int64_t process_id; + uint64_t thread_id; + int64_t unix_time; + uint64_t nano_time; + + void Populate(const GenerateRawUniqueIdOpts& opts) { + if (opts.exclude_env_details) { + return; + } + opts.env->GetHostName(hostname_buf.data(), hostname_buf.size()) + .PermitUncheckedError(); + process_id = port::GetProcessID(); + thread_id = opts.env->GetThreadID(); + opts.env->GetCurrentTime(&unix_time).PermitUncheckedError(); + nano_time = opts.env->NowNanos(); + } +}; + +struct EntropyTrackRandomDevice { + using RandType = std::random_device::result_type; + static constexpr size_t kNumRandVals = + /* generous bits */ 192U / (8U * sizeof(RandType)); + std::array rand_vals; + + void Populate(const GenerateRawUniqueIdOpts& opts) { + if (opts.exclude_random_device) { + return; + } + std::random_device r; + for (auto& val : rand_vals) { + val = r(); + } + } +}; + +struct Entropy { + uint64_t version_identifier; + EntropyTrackRandomDevice et1; + EntropyTrackEnvDetails et2; + EntropyTrackPortUuid et3; + + void Populate(const GenerateRawUniqueIdOpts& opts) { + // If we change the format of what goes into the entropy inputs, it's + // conceivable there could be a physical collision in the hash input + // even though they are logically different. This value should change + // if there's a change to the "schema" here, including byte order. + version_identifier = (uint64_t{ROCKSDB_MAJOR} << 32) + + (uint64_t{ROCKSDB_MINOR} << 16) + + uint64_t{ROCKSDB_PATCH}; + et1.Populate(opts); + et2.Populate(opts); + et3.Populate(opts); + } +}; + +void GenerateRawUniqueIdImpl(uint64_t* a, uint64_t* b, + const GenerateRawUniqueIdOpts& opts) { + Entropy e; + std::memset(&e, 0, sizeof(e)); + e.Populate(opts); + Hash2x64(reinterpret_cast(&e), sizeof(e), a, b); +} + +} // namespace + +void GenerateRawUniqueId(uint64_t* a, uint64_t* b, bool exclude_port_uuid) { + GenerateRawUniqueIdOpts opts; + opts.exclude_port_uuid = exclude_port_uuid; + assert(!opts.exclude_env_details); + assert(!opts.exclude_random_device); + GenerateRawUniqueIdImpl(a, b, opts); +} + +#ifndef NDEBUG +void TEST_GenerateRawUniqueId(uint64_t* a, uint64_t* b, bool exclude_port_uuid, + bool exclude_env_details, + bool exclude_random_device) { + GenerateRawUniqueIdOpts opts; + opts.exclude_port_uuid = exclude_port_uuid; + opts.exclude_env_details = exclude_env_details; + opts.exclude_random_device = exclude_random_device; + GenerateRawUniqueIdImpl(a, b, opts); +} +#endif + +void SemiStructuredUniqueIdGen::Reset() { + saved_process_id_ = port::GetProcessID(); + GenerateRawUniqueId(&base_upper_, &base_lower_); + counter_ = 0; +} + +void SemiStructuredUniqueIdGen::GenerateNext(uint64_t* upper, uint64_t* lower) { + if (port::GetProcessID() == saved_process_id_) { + // Safe to increment the atomic for guaranteed uniqueness within this + // process lifetime. Xor slightly better than +. See + // https://github.com/pdillinger/unique_id + *lower = base_lower_ ^ counter_.fetch_add(1); + *upper = base_upper_; + } else { + // There must have been a fork() or something. Rather than attempting to + // update in a thread-safe way, simply fall back on GenerateRawUniqueId. + GenerateRawUniqueId(upper, lower); + } +} + +void UnpredictableUniqueIdGen::Reset() { + for (size_t i = 0; i < pool_.size(); i += 2) { + assert(i + 1 < pool_.size()); + uint64_t a, b; + GenerateRawUniqueId(&a, &b); + pool_[i] = a; + pool_[i + 1] = b; + } +} + +void UnpredictableUniqueIdGen::GenerateNext(uint64_t* upper, uint64_t* lower) { + uint64_t extra_entropy; + // Use timing information (if available) to add to entropy. (Not a disaster + // if unavailable on some platforms. High performance is important.) +#ifdef __SSE4_2__ // More than enough to guarantee rdtsc instruction + extra_entropy = static_cast(_rdtsc()); +#else + extra_entropy = SystemClock::Default()->NowNanos(); +#endif + + GenerateNextWithEntropy(upper, lower, extra_entropy); +} + +void UnpredictableUniqueIdGen::GenerateNextWithEntropy(uint64_t* upper, + uint64_t* lower, + uint64_t extra_entropy) { + // To efficiently ensure unique inputs to the hash function in the presence + // of multithreading, we do not require atomicity on the whole entropy pool, + // but instead only a piece of it (a 64-bit counter) that is sufficient to + // guarantee uniqueness. + uint64_t count = counter_.fetch_add(1, std::memory_order_relaxed); + uint64_t a = count; + uint64_t b = extra_entropy; + // Invoking the hash function several times avoids copying all the inputs + // to a contiguous, non-atomic buffer. + BijectiveHash2x64(a, b, &a, &b); // Based on XXH128 + + // In hashing the rest of the pool with that, we don't need to worry about + // races, but use atomic operations for sanitizer-friendliness. + for (size_t i = 0; i < pool_.size(); i += 2) { + assert(i + 1 < pool_.size()); + a ^= pool_[i].load(std::memory_order_relaxed); + b ^= pool_[i + 1].load(std::memory_order_relaxed); + BijectiveHash2x64(a, b, &a, &b); // Based on XXH128 + } + + // Return result + *lower = a; + *upper = b; + + // Add some back into pool. We don't really care that there's a race in + // storing the result back and another thread computing the next value. + // It's just an entropy pool. + pool_[count & (pool_.size() - 1)].fetch_add(a, std::memory_order_relaxed); +} + +#ifndef NDEBUG +UnpredictableUniqueIdGen::UnpredictableUniqueIdGen(TEST_ZeroInitialized) { + for (auto& p : pool_) { + p.store(0); + } + counter_.store(0); +} +#endif + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/env/unique_id_gen.h b/librocksdb-sys/rocksdb/env/unique_id_gen.h new file mode 100644 index 0000000..f654c7b --- /dev/null +++ b/librocksdb-sys/rocksdb/env/unique_id_gen.h @@ -0,0 +1,119 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +// This file is for functions that generate unique identifiers by +// (at least in part) by extracting novel entropy or sources of uniqueness +// from the execution environment. (By contrast, random.h is for algorithmic +// pseudorandomness.) +// +// These functions could eventually migrate to public APIs, such as in Env. + +#pragma once + +#include +#include +#include +#include + +#include "port/port.h" +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +// Generates a new 128-bit identifier that is universally unique +// (with high probability) for each call. The result is split into +// two 64-bit pieces. This function has NOT been validated for use in +// cryptography. +// +// This is used in generating DB session IDs and by Env::GenerateUniqueId +// (used for DB IDENTITY) if the platform does not provide a generator of +// RFC 4122 UUIDs or fails somehow. (Set exclude_port_uuid=true if this +// function is used as a fallback for GenerateRfcUuid, because no need +// trying it again.) +void GenerateRawUniqueId(uint64_t* a, uint64_t* b, + bool exclude_port_uuid = false); + +#ifndef NDEBUG +// A version of above with options for challenge testing +void TEST_GenerateRawUniqueId(uint64_t* a, uint64_t* b, bool exclude_port_uuid, + bool exclude_env_details, + bool exclude_random_device); +#endif + +// Generates globally unique ids with lower probability of any collisions +// vs. each unique id being independently random (GenerateRawUniqueId). +// We call this "semi-structured" because between different +// SemiStructuredUniqueIdGen objects, the IDs are separated by random +// intervals (unstructured), but within a single SemiStructuredUniqueIdGen +// object, the generated IDs are trivially related (structured). See +// https://github.com/pdillinger/unique_id for how this improves probability +// of no collision. In short, if we have n SemiStructuredUniqueIdGen +// objects each generating m IDs, the first collision is expected at +// around n = sqrt(2^128 / m), equivalently n * sqrt(m) = 2^64, +// rather than n * m = 2^64 for fully random IDs. +class SemiStructuredUniqueIdGen { + public: + // Initializes with random starting state (from GenerateRawUniqueId) + SemiStructuredUniqueIdGen() { Reset(); } + // Re-initializes, but not thread safe + void Reset(); + + // Assuming no fork(), `lower` is guaranteed unique from one call + // to the next (thread safe). + void GenerateNext(uint64_t* upper, uint64_t* lower); + + // For generating smaller values. Will cycle through all the possibilities + // before repeating. + template + T GenerateNext() { + static_assert(sizeof(T) <= sizeof(uint64_t)); + static_assert(std::is_integral_v); + uint64_t ignore, val; + GenerateNext(&ignore, &val); + return static_cast(val); + } + + uint64_t GetBaseUpper() const { return base_upper_; } + + private: + uint64_t base_upper_; + uint64_t base_lower_; + std::atomic counter_; + int64_t saved_process_id_; +}; + +// A unique id generator that should provide reasonable security against +// predicting the output from previous outputs, but is NOT known to be +// cryptographically secure. Unlike std::random_device, this is guaranteed +// not to block once initialized. +class ALIGN_AS(CACHE_LINE_SIZE) UnpredictableUniqueIdGen { + public: + // Initializes with random starting state (from several GenerateRawUniqueId) + UnpredictableUniqueIdGen() { Reset(); } + // Re-initializes, but not thread safe + void Reset(); + + // Generate next probabilistically unique value. Thread safe. Uses timing + // information to add to the entropy pool. + void GenerateNext(uint64_t* upper, uint64_t* lower); + + // Explicitly include given value for entropy pool instead of timing + // information. + void GenerateNextWithEntropy(uint64_t* upper, uint64_t* lower, + uint64_t extra_entropy); + +#ifndef NDEBUG + struct TEST_ZeroInitialized {}; + explicit UnpredictableUniqueIdGen(TEST_ZeroInitialized); + std::atomic& TEST_counter() { return counter_; } +#endif + private: + // 256 bit entropy pool + std::array, 4> pool_; + // Counter to ensure unique hash inputs + std::atomic counter_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/examples/.gitignore b/librocksdb-sys/rocksdb/examples/.gitignore new file mode 100644 index 0000000..39da06a --- /dev/null +++ b/librocksdb-sys/rocksdb/examples/.gitignore @@ -0,0 +1,10 @@ +c_simple_example +column_families_example +compact_files_example +compaction_filter_example +multi_processes_example +optimistic_transaction_example +options_file_example +rocksdb_backup_restore_example +simple_example +transaction_example diff --git a/librocksdb-sys/rocksdb/examples/CMakeLists.txt b/librocksdb-sys/rocksdb/examples/CMakeLists.txt new file mode 100644 index 0000000..0b93a6d --- /dev/null +++ b/librocksdb-sys/rocksdb/examples/CMakeLists.txt @@ -0,0 +1,45 @@ +add_executable(simple_example + simple_example.cc) +target_link_libraries(simple_example + ${ROCKSDB_LIB}) + +add_executable(column_families_example + column_families_example.cc) +target_link_libraries(column_families_example + ${ROCKSDB_LIB}) + +add_executable(compact_files_example + compact_files_example.cc) +target_link_libraries(compact_files_example + ${ROCKSDB_LIB}) + +add_executable(c_simple_example + c_simple_example.c) +target_link_libraries(c_simple_example + ${ROCKSDB_LIB}) + +add_executable(optimistic_transaction_example + optimistic_transaction_example.cc) +target_link_libraries(optimistic_transaction_example + ${ROCKSDB_LIB}) + +add_executable(transaction_example + transaction_example.cc) +target_link_libraries(transaction_example + ${ROCKSDB_LIB}) + +add_executable(compaction_filter_example + compaction_filter_example.cc) +target_link_libraries(compaction_filter_example + ${ROCKSDB_LIB}) + +add_executable(options_file_example + options_file_example.cc) +target_link_libraries(options_file_example + ${ROCKSDB_LIB}) + +add_executable(multi_processes_example + EXCLUDE_FROM_ALL + multi_processes_example.cc) +target_link_libraries(multi_processes_example + ${ROCKSDB_LIB}) diff --git a/librocksdb-sys/rocksdb/examples/Makefile b/librocksdb-sys/rocksdb/examples/Makefile new file mode 100644 index 0000000..b056508 --- /dev/null +++ b/librocksdb-sys/rocksdb/examples/Makefile @@ -0,0 +1,58 @@ +include ../make_config.mk + +ifndef DISABLE_JEMALLOC + ifdef JEMALLOC + PLATFORM_CXXFLAGS += -DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE + endif + EXEC_LDFLAGS := $(JEMALLOC_LIB) $(EXEC_LDFLAGS) -lpthread + PLATFORM_CXXFLAGS += $(JEMALLOC_INCLUDE) +endif + +ifneq ($(USE_RTTI), 1) + CXXFLAGS += -fno-rtti +endif + +CFLAGS += -Wstrict-prototypes + +.PHONY: clean librocksdb + +all: simple_example column_families_example compact_files_example c_simple_example optimistic_transaction_example transaction_example compaction_filter_example options_file_example rocksdb_backup_restore_example + +simple_example: librocksdb simple_example.cc + $(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++17 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS) + +column_families_example: librocksdb column_families_example.cc + $(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++17 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS) + +compaction_filter_example: librocksdb compaction_filter_example.cc + $(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++17 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS) + +compact_files_example: librocksdb compact_files_example.cc + $(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++17 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS) + +.c.o: + $(CC) $(CFLAGS) -c $< -o $@ -I../include + +c_simple_example: librocksdb c_simple_example.o + $(CXX) $@.o -o$@ ../librocksdb.a $(PLATFORM_LDFLAGS) $(EXEC_LDFLAGS) + +optimistic_transaction_example: librocksdb optimistic_transaction_example.cc + $(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++17 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS) + +transaction_example: librocksdb transaction_example.cc + $(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++17 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS) + +options_file_example: librocksdb options_file_example.cc + $(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++17 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS) + +multi_processes_example: librocksdb multi_processes_example.cc + $(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++17 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS) + +rocksdb_backup_restore_example: librocksdb rocksdb_backup_restore_example.cc + $(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++17 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS) + +clean: + rm -rf ./simple_example ./column_families_example ./compact_files_example ./compaction_filter_example ./c_simple_example c_simple_example.o ./optimistic_transaction_example ./transaction_example ./options_file_example ./multi_processes_example ./rocksdb_backup_restore_example + +librocksdb: + cd .. && $(MAKE) static_lib diff --git a/librocksdb-sys/rocksdb/examples/README.md b/librocksdb-sys/rocksdb/examples/README.md new file mode 100644 index 0000000..f4ba238 --- /dev/null +++ b/librocksdb-sys/rocksdb/examples/README.md @@ -0,0 +1,2 @@ +1. Compile RocksDB first by executing `make static_lib` in parent dir +2. Compile all examples: `cd examples/; make all` diff --git a/librocksdb-sys/rocksdb/examples/c_simple_example.c b/librocksdb-sys/rocksdb/examples/c_simple_example.c new file mode 100644 index 0000000..fe2f917 --- /dev/null +++ b/librocksdb-sys/rocksdb/examples/c_simple_example.c @@ -0,0 +1,96 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include +#include +#include +#include + +#include "rocksdb/c.h" + +#if defined(OS_WIN) +#include +#else +#include // sysconf() - get CPU count +#endif + +#if defined(OS_WIN) +const char DBPath[] = "C:\\Windows\\TEMP\\rocksdb_c_simple_example"; +const char DBBackupPath[] = + "C:\\Windows\\TEMP\\rocksdb_c_simple_example_backup"; +#else +const char DBPath[] = "/tmp/rocksdb_c_simple_example"; +const char DBBackupPath[] = "/tmp/rocksdb_c_simple_example_backup"; +#endif + +int main(int argc, char **argv) { + rocksdb_t *db; + rocksdb_backup_engine_t *be; + rocksdb_options_t *options = rocksdb_options_create(); + // Optimize RocksDB. This is the easiest way to + // get RocksDB to perform well. +#if defined(OS_WIN) + SYSTEM_INFO system_info; + GetSystemInfo(&system_info); + long cpus = system_info.dwNumberOfProcessors; +#else + long cpus = sysconf(_SC_NPROCESSORS_ONLN); +#endif + // Set # of online cores + rocksdb_options_increase_parallelism(options, (int)(cpus)); + rocksdb_options_optimize_level_style_compaction(options, 0); + // create the DB if it's not already present + rocksdb_options_set_create_if_missing(options, 1); + + // open DB + char *err = NULL; + db = rocksdb_open(options, DBPath, &err); + assert(!err); + + // open Backup Engine that we will use for backing up our database + be = rocksdb_backup_engine_open(options, DBBackupPath, &err); + assert(!err); + + // Put key-value + rocksdb_writeoptions_t *writeoptions = rocksdb_writeoptions_create(); + const char key[] = "key"; + const char *value = "value"; + rocksdb_put(db, writeoptions, key, strlen(key), value, strlen(value) + 1, + &err); + assert(!err); + // Get value + rocksdb_readoptions_t *readoptions = rocksdb_readoptions_create(); + size_t len; + char *returned_value = + rocksdb_get(db, readoptions, key, strlen(key), &len, &err); + assert(!err); + assert(strcmp(returned_value, "value") == 0); + free(returned_value); + + // create new backup in a directory specified by DBBackupPath + rocksdb_backup_engine_create_new_backup(be, db, &err); + assert(!err); + + rocksdb_close(db); + + // If something is wrong, you might want to restore data from last backup + rocksdb_restore_options_t *restore_options = rocksdb_restore_options_create(); + rocksdb_backup_engine_restore_db_from_latest_backup(be, DBPath, DBPath, + restore_options, &err); + assert(!err); + rocksdb_restore_options_destroy(restore_options); + + db = rocksdb_open(options, DBPath, &err); + assert(!err); + + // cleanup + rocksdb_writeoptions_destroy(writeoptions); + rocksdb_readoptions_destroy(readoptions); + rocksdb_options_destroy(options); + rocksdb_backup_engine_close(be); + rocksdb_close(db); + + return 0; +} diff --git a/librocksdb-sys/rocksdb/examples/column_families_example.cc b/librocksdb-sys/rocksdb/examples/column_families_example.cc new file mode 100644 index 0000000..3828d3f --- /dev/null +++ b/librocksdb-sys/rocksdb/examples/column_families_example.cc @@ -0,0 +1,88 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#include +#include +#include + +#include "rocksdb/db.h" +#include "rocksdb/options.h" +#include "rocksdb/slice.h" + +#if defined(OS_WIN) +std::string kDBPath = "C:\\Windows\\TEMP\\rocksdb_column_families_example"; +#else +std::string kDBPath = "/tmp/rocksdb_column_families_example"; +#endif + +using ROCKSDB_NAMESPACE::ColumnFamilyDescriptor; +using ROCKSDB_NAMESPACE::ColumnFamilyHandle; +using ROCKSDB_NAMESPACE::ColumnFamilyOptions; +using ROCKSDB_NAMESPACE::DB; +using ROCKSDB_NAMESPACE::DBOptions; +using ROCKSDB_NAMESPACE::Options; +using ROCKSDB_NAMESPACE::ReadOptions; +using ROCKSDB_NAMESPACE::Slice; +using ROCKSDB_NAMESPACE::Status; +using ROCKSDB_NAMESPACE::WriteBatch; +using ROCKSDB_NAMESPACE::WriteOptions; + +int main() { + // open DB + Options options; + options.create_if_missing = true; + DB* db; + Status s = DB::Open(options, kDBPath, &db); + assert(s.ok()); + + // create column family + ColumnFamilyHandle* cf; + s = db->CreateColumnFamily(ColumnFamilyOptions(), "new_cf", &cf); + assert(s.ok()); + + // close DB + s = db->DestroyColumnFamilyHandle(cf); + assert(s.ok()); + delete db; + + // open DB with two column families + std::vector column_families; + // have to open default column family + column_families.push_back(ColumnFamilyDescriptor( + ROCKSDB_NAMESPACE::kDefaultColumnFamilyName, ColumnFamilyOptions())); + // open the new one, too + column_families.push_back( + ColumnFamilyDescriptor("new_cf", ColumnFamilyOptions())); + std::vector handles; + s = DB::Open(DBOptions(), kDBPath, column_families, &handles, &db); + assert(s.ok()); + + // put and get from non-default column family + s = db->Put(WriteOptions(), handles[1], Slice("key"), Slice("value")); + assert(s.ok()); + std::string value; + s = db->Get(ReadOptions(), handles[1], Slice("key"), &value); + assert(s.ok()); + + // atomic write + WriteBatch batch; + batch.Put(handles[0], Slice("key2"), Slice("value2")); + batch.Put(handles[1], Slice("key3"), Slice("value3")); + batch.Delete(handles[0], Slice("key")); + s = db->Write(WriteOptions(), &batch); + assert(s.ok()); + + // drop column family + s = db->DropColumnFamily(handles[1]); + assert(s.ok()); + + // close db + for (auto handle : handles) { + s = db->DestroyColumnFamilyHandle(handle); + assert(s.ok()); + } + delete db; + + return 0; +} diff --git a/librocksdb-sys/rocksdb/examples/compact_files_example.cc b/librocksdb-sys/rocksdb/examples/compact_files_example.cc new file mode 100644 index 0000000..1ecf8c7 --- /dev/null +++ b/librocksdb-sys/rocksdb/examples/compact_files_example.cc @@ -0,0 +1,177 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// An example code demonstrating how to use CompactFiles, EventListener, +// and GetColumnFamilyMetaData APIs to implement custom compaction algorithm. + +#include +#include + +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" + +using ROCKSDB_NAMESPACE::ColumnFamilyMetaData; +using ROCKSDB_NAMESPACE::CompactionOptions; +using ROCKSDB_NAMESPACE::DB; +using ROCKSDB_NAMESPACE::EventListener; +using ROCKSDB_NAMESPACE::FlushJobInfo; +using ROCKSDB_NAMESPACE::Options; +using ROCKSDB_NAMESPACE::ReadOptions; +using ROCKSDB_NAMESPACE::Status; +using ROCKSDB_NAMESPACE::WriteOptions; + +#if defined(OS_WIN) +std::string kDBPath = "C:\\Windows\\TEMP\\rocksdb_compact_files_example"; +#else +std::string kDBPath = "/tmp/rocksdb_compact_files_example"; +#endif + +struct CompactionTask; + +// This is an example interface of external-compaction algorithm. +// Compaction algorithm can be implemented outside the core-RocksDB +// code by using the pluggable compaction APIs that RocksDb provides. +class Compactor : public EventListener { + public: + // Picks and returns a compaction task given the specified DB + // and column family. It is the caller's responsibility to + // destroy the returned CompactionTask. Returns "nullptr" + // if it cannot find a proper compaction task. + virtual CompactionTask* PickCompaction(DB* db, + const std::string& cf_name) = 0; + + // Schedule and run the specified compaction task in background. + virtual void ScheduleCompaction(CompactionTask* task) = 0; +}; + +// Example structure that describes a compaction task. +struct CompactionTask { + CompactionTask(DB* _db, Compactor* _compactor, + const std::string& _column_family_name, + const std::vector& _input_file_names, + const int _output_level, + const CompactionOptions& _compact_options, bool _retry_on_fail) + : db(_db), + compactor(_compactor), + column_family_name(_column_family_name), + input_file_names(_input_file_names), + output_level(_output_level), + compact_options(_compact_options), + retry_on_fail(_retry_on_fail) {} + DB* db; + Compactor* compactor; + const std::string& column_family_name; + std::vector input_file_names; + int output_level; + CompactionOptions compact_options; + bool retry_on_fail; +}; + +// A simple compaction algorithm that always compacts everything +// to the highest level whenever possible. +class FullCompactor : public Compactor { + public: + explicit FullCompactor(const Options options) : options_(options) { + compact_options_.compression = options_.compression; + compact_options_.output_file_size_limit = options_.target_file_size_base; + } + + // When flush happens, it determines whether to trigger compaction. If + // triggered_writes_stop is true, it will also set the retry flag of + // compaction-task to true. + void OnFlushCompleted(DB* db, const FlushJobInfo& info) override { + CompactionTask* task = PickCompaction(db, info.cf_name); + if (task != nullptr) { + if (info.triggered_writes_stop) { + task->retry_on_fail = true; + } + // Schedule compaction in a different thread. + ScheduleCompaction(task); + } + } + + // Always pick a compaction which includes all files whenever possible. + CompactionTask* PickCompaction(DB* db, const std::string& cf_name) override { + ColumnFamilyMetaData cf_meta; + db->GetColumnFamilyMetaData(&cf_meta); + + std::vector input_file_names; + for (auto level : cf_meta.levels) { + for (auto file : level.files) { + if (file.being_compacted) { + return nullptr; + } + input_file_names.push_back(file.name); + } + } + return new CompactionTask(db, this, cf_name, input_file_names, + options_.num_levels - 1, compact_options_, false); + } + + // Schedule the specified compaction task in background. + void ScheduleCompaction(CompactionTask* task) override { + options_.env->Schedule(&FullCompactor::CompactFiles, task); + } + + static void CompactFiles(void* arg) { + std::unique_ptr task( + reinterpret_cast(arg)); + assert(task); + assert(task->db); + Status s = task->db->CompactFiles( + task->compact_options, task->input_file_names, task->output_level); + printf("CompactFiles() finished with status %s\n", s.ToString().c_str()); + if (!s.ok() && !s.IsIOError() && task->retry_on_fail) { + // If a compaction task with its retry_on_fail=true failed, + // try to schedule another compaction in case the reason + // is not an IO error. + CompactionTask* new_task = + task->compactor->PickCompaction(task->db, task->column_family_name); + task->compactor->ScheduleCompaction(new_task); + } + } + + private: + Options options_; + CompactionOptions compact_options_; +}; + +int main() { + Options options; + options.create_if_missing = true; + // Disable RocksDB background compaction. + options.compaction_style = ROCKSDB_NAMESPACE::kCompactionStyleNone; + // Small slowdown and stop trigger for experimental purpose. + options.level0_slowdown_writes_trigger = 3; + options.level0_stop_writes_trigger = 5; + options.IncreaseParallelism(5); + options.listeners.emplace_back(new FullCompactor(options)); + + DB* db = nullptr; + ROCKSDB_NAMESPACE::DestroyDB(kDBPath, options); + Status s = DB::Open(options, kDBPath, &db); + assert(s.ok()); + assert(db); + + // if background compaction is not working, write will stall + // because of options.level0_stop_writes_trigger + for (int i = 1000; i < 99999; ++i) { + db->Put(WriteOptions(), std::to_string(i), + std::string(500, 'a' + (i % 26))); + } + + // verify the values are still there + std::string value; + for (int i = 1000; i < 99999; ++i) { + db->Get(ReadOptions(), std::to_string(i), &value); + assert(value == std::string(500, 'a' + (i % 26))); + } + + // close the db. + delete db; + + return 0; +} diff --git a/librocksdb-sys/rocksdb/examples/compaction_filter_example.cc b/librocksdb-sys/rocksdb/examples/compaction_filter_example.cc new file mode 100644 index 0000000..ed1ada8 --- /dev/null +++ b/librocksdb-sys/rocksdb/examples/compaction_filter_example.cc @@ -0,0 +1,96 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "rocksdb/compaction_filter.h" +#include "rocksdb/db.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/options.h" + +class MyMerge : public ROCKSDB_NAMESPACE::MergeOperator { + public: + virtual bool FullMergeV2(const MergeOperationInput& merge_in, + MergeOperationOutput* merge_out) const override { + merge_out->new_value.clear(); + if (merge_in.existing_value != nullptr) { + merge_out->new_value.assign(merge_in.existing_value->data(), + merge_in.existing_value->size()); + } + for (const ROCKSDB_NAMESPACE::Slice& m : merge_in.operand_list) { + fprintf(stderr, "Merge(%s)\n", m.ToString().c_str()); + // the compaction filter filters out bad values + assert(m.ToString() != "bad"); + merge_out->new_value.assign(m.data(), m.size()); + } + return true; + } + + const char* Name() const override { return "MyMerge"; } +}; + +class MyFilter : public ROCKSDB_NAMESPACE::CompactionFilter { + public: + bool Filter(int level, const ROCKSDB_NAMESPACE::Slice& key, + const ROCKSDB_NAMESPACE::Slice& existing_value, + std::string* new_value, bool* value_changed) const override { + fprintf(stderr, "Filter(%s)\n", key.ToString().c_str()); + ++count_; + assert(*value_changed == false); + return false; + } + + bool FilterMergeOperand( + int level, const ROCKSDB_NAMESPACE::Slice& key, + const ROCKSDB_NAMESPACE::Slice& existing_value) const override { + fprintf(stderr, "FilterMerge(%s)\n", key.ToString().c_str()); + ++merge_count_; + return existing_value == "bad"; + } + + const char* Name() const override { return "MyFilter"; } + + mutable int count_ = 0; + mutable int merge_count_ = 0; +}; + +#if defined(OS_WIN) +std::string kDBPath = "C:\\Windows\\TEMP\\rocksmergetest"; +std::string kRemoveDirCommand = "rmdir /Q /S "; +#else +std::string kDBPath = "/tmp/rocksmergetest"; +std::string kRemoveDirCommand = "rm -rf "; +#endif + +int main() { + ROCKSDB_NAMESPACE::DB* raw_db; + ROCKSDB_NAMESPACE::Status status; + + MyFilter filter; + + std::string rm_cmd = kRemoveDirCommand + kDBPath; + int ret = system(rm_cmd.c_str()); + if (ret != 0) { + fprintf(stderr, "Error deleting %s, code: %d\n", kDBPath.c_str(), ret); + } + ROCKSDB_NAMESPACE::Options options; + options.create_if_missing = true; + options.merge_operator.reset(new MyMerge); + options.compaction_filter = &filter; + status = ROCKSDB_NAMESPACE::DB::Open(options, kDBPath, &raw_db); + assert(status.ok()); + std::unique_ptr db(raw_db); + + ROCKSDB_NAMESPACE::WriteOptions wopts; + db->Merge(wopts, "0", "bad"); // This is filtered out + db->Merge(wopts, "1", "data1"); + db->Merge(wopts, "1", "bad"); + db->Merge(wopts, "1", "data2"); + db->Merge(wopts, "1", "bad"); + db->Merge(wopts, "3", "data3"); + db->CompactRange(ROCKSDB_NAMESPACE::CompactRangeOptions(), nullptr, nullptr); + fprintf(stderr, "filter.count_ = %d\n", filter.count_); + assert(filter.count_ == 0); + fprintf(stderr, "filter.merge_count_ = %d\n", filter.merge_count_); + assert(filter.merge_count_ == 6); +} diff --git a/librocksdb-sys/rocksdb/examples/multi_processes_example.cc b/librocksdb-sys/rocksdb/examples/multi_processes_example.cc new file mode 100644 index 0000000..93c54d7 --- /dev/null +++ b/librocksdb-sys/rocksdb/examples/multi_processes_example.cc @@ -0,0 +1,393 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +// How to use this example +// Open two terminals, in one of them, run `./multi_processes_example 0` to +// start a process running the primary instance. This will create a new DB in +// kDBPath. The process will run for a while inserting keys to the normal +// RocksDB database. +// Next, go to the other terminal and run `./multi_processes_example 1` to +// start a process running the secondary instance. This will create a secondary +// instance following the aforementioned primary instance. This process will +// run for a while, tailing the logs of the primary. After process with primary +// instance exits, this process will keep running until you hit 'CTRL+C'. + +#include +#include +#include +#include +#include +#include +#include +#include + +// TODO: port this example to other systems. It should be straightforward for +// POSIX-compliant systems. +#if defined(OS_LINUX) +#include +#include +#include +#include +#include +#include + +#include "rocksdb/db.h" +#include "rocksdb/options.h" +#include "rocksdb/slice.h" + +using ROCKSDB_NAMESPACE::ColumnFamilyDescriptor; +using ROCKSDB_NAMESPACE::ColumnFamilyHandle; +using ROCKSDB_NAMESPACE::ColumnFamilyOptions; +using ROCKSDB_NAMESPACE::DB; +using ROCKSDB_NAMESPACE::FlushOptions; +using ROCKSDB_NAMESPACE::Iterator; +using ROCKSDB_NAMESPACE::Options; +using ROCKSDB_NAMESPACE::ReadOptions; +using ROCKSDB_NAMESPACE::Slice; +using ROCKSDB_NAMESPACE::Status; +using ROCKSDB_NAMESPACE::WriteOptions; + +const std::string kDBPath = "/tmp/rocksdb_multi_processes_example"; +const std::string kPrimaryStatusFile = + "/tmp/rocksdb_multi_processes_example_primary_status"; +const uint64_t kMaxKey = 600000; +const size_t kMaxValueLength = 256; +const size_t kNumKeysPerFlush = 1000; + +const std::vector& GetColumnFamilyNames() { + static std::vector column_family_names = { + ROCKSDB_NAMESPACE::kDefaultColumnFamilyName, "pikachu"}; + return column_family_names; +} + +inline bool IsLittleEndian() { + uint32_t x = 1; + return *reinterpret_cast(&x) != 0; +} + +static std::atomic& ShouldSecondaryWait() { + static std::atomic should_secondary_wait{1}; + return should_secondary_wait; +} + +static std::string Key(uint64_t k) { + std::string ret; + if (IsLittleEndian()) { + ret.append(reinterpret_cast(&k), sizeof(k)); + } else { + char buf[sizeof(k)]; + buf[0] = k & 0xff; + buf[1] = (k >> 8) & 0xff; + buf[2] = (k >> 16) & 0xff; + buf[3] = (k >> 24) & 0xff; + buf[4] = (k >> 32) & 0xff; + buf[5] = (k >> 40) & 0xff; + buf[6] = (k >> 48) & 0xff; + buf[7] = (k >> 56) & 0xff; + ret.append(buf, sizeof(k)); + } + size_t i = 0, j = ret.size() - 1; + while (i < j) { + char tmp = ret[i]; + ret[i] = ret[j]; + ret[j] = tmp; + ++i; + --j; + } + return ret; +} + +static uint64_t Key(std::string key) { + assert(key.size() == sizeof(uint64_t)); + size_t i = 0, j = key.size() - 1; + while (i < j) { + char tmp = key[i]; + key[i] = key[j]; + key[j] = tmp; + ++i; + --j; + } + uint64_t ret = 0; + if (IsLittleEndian()) { + memcpy(&ret, key.c_str(), sizeof(uint64_t)); + } else { + const char* buf = key.c_str(); + ret |= static_cast(buf[0]); + ret |= (static_cast(buf[1]) << 8); + ret |= (static_cast(buf[2]) << 16); + ret |= (static_cast(buf[3]) << 24); + ret |= (static_cast(buf[4]) << 32); + ret |= (static_cast(buf[5]) << 40); + ret |= (static_cast(buf[6]) << 48); + ret |= (static_cast(buf[7]) << 56); + } + return ret; +} + +static Slice GenerateRandomValue(const size_t max_length, char scratch[]) { + size_t sz = 1 + (std::rand() % max_length); + int rnd = std::rand(); + for (size_t i = 0; i != sz; ++i) { + scratch[i] = static_cast(rnd ^ i); + } + return Slice(scratch, sz); +} + +static bool ShouldCloseDB() { return true; } + +void CreateDB() { + long my_pid = static_cast(getpid()); + Options options; + Status s = ROCKSDB_NAMESPACE::DestroyDB(kDBPath, options); + if (!s.ok()) { + fprintf(stderr, "[process %ld] Failed to destroy DB: %s\n", my_pid, + s.ToString().c_str()); + assert(false); + } + options.create_if_missing = true; + DB* db = nullptr; + s = DB::Open(options, kDBPath, &db); + if (!s.ok()) { + fprintf(stderr, "[process %ld] Failed to open DB: %s\n", my_pid, + s.ToString().c_str()); + assert(false); + } + std::vector handles; + ColumnFamilyOptions cf_opts(options); + for (const auto& cf_name : GetColumnFamilyNames()) { + if (ROCKSDB_NAMESPACE::kDefaultColumnFamilyName != cf_name) { + ColumnFamilyHandle* handle = nullptr; + s = db->CreateColumnFamily(cf_opts, cf_name, &handle); + if (!s.ok()) { + fprintf(stderr, "[process %ld] Failed to create CF %s: %s\n", my_pid, + cf_name.c_str(), s.ToString().c_str()); + assert(false); + } + handles.push_back(handle); + } + } + fprintf(stdout, "[process %ld] Column families created\n", my_pid); + for (auto h : handles) { + delete h; + } + handles.clear(); + delete db; +} + +void RunPrimary() { + long my_pid = static_cast(getpid()); + fprintf(stdout, "[process %ld] Primary instance starts\n", my_pid); + CreateDB(); + std::srand(time(nullptr)); + DB* db = nullptr; + Options options; + options.create_if_missing = false; + std::vector column_families; + for (const auto& cf_name : GetColumnFamilyNames()) { + column_families.push_back(ColumnFamilyDescriptor(cf_name, options)); + } + std::vector handles; + WriteOptions write_opts; + char val_buf[kMaxValueLength] = {0}; + uint64_t curr_key = 0; + while (curr_key < kMaxKey) { + Status s; + if (nullptr == db) { + s = DB::Open(options, kDBPath, column_families, &handles, &db); + if (!s.ok()) { + fprintf(stderr, "[process %ld] Failed to open DB: %s\n", my_pid, + s.ToString().c_str()); + assert(false); + } + } + assert(nullptr != db); + assert(handles.size() == GetColumnFamilyNames().size()); + for (auto h : handles) { + assert(nullptr != h); + for (size_t i = 0; i != kNumKeysPerFlush; ++i) { + Slice key = Key(curr_key + static_cast(i)); + Slice value = GenerateRandomValue(kMaxValueLength, val_buf); + s = db->Put(write_opts, h, key, value); + if (!s.ok()) { + fprintf(stderr, "[process %ld] Failed to insert\n", my_pid); + assert(false); + } + } + s = db->Flush(FlushOptions(), h); + if (!s.ok()) { + fprintf(stderr, "[process %ld] Failed to flush\n", my_pid); + assert(false); + } + } + curr_key += static_cast(kNumKeysPerFlush); + if (ShouldCloseDB()) { + for (auto h : handles) { + delete h; + } + handles.clear(); + delete db; + db = nullptr; + } + } + if (nullptr != db) { + for (auto h : handles) { + delete h; + } + handles.clear(); + delete db; + db = nullptr; + } + fprintf(stdout, "[process %ld] Finished adding keys\n", my_pid); +} + +void secondary_instance_sigint_handler(int signal) { + ShouldSecondaryWait().store(0, std::memory_order_relaxed); + fprintf(stdout, "\n"); + fflush(stdout); +}; + +void RunSecondary() { + ::signal(SIGINT, secondary_instance_sigint_handler); + long my_pid = static_cast(getpid()); + const std::string kSecondaryPath = + "/tmp/rocksdb_multi_processes_example_secondary"; + // Create directory if necessary + if (nullptr == opendir(kSecondaryPath.c_str())) { + int ret = + mkdir(kSecondaryPath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); + if (ret < 0) { + perror("failed to create directory for secondary instance"); + exit(0); + } + } + DB* db = nullptr; + Options options; + options.create_if_missing = false; + options.max_open_files = -1; + Status s = DB::OpenAsSecondary(options, kDBPath, kSecondaryPath, &db); + if (!s.ok()) { + fprintf(stderr, "[process %ld] Failed to open in secondary mode: %s\n", + my_pid, s.ToString().c_str()); + assert(false); + } else { + fprintf(stdout, "[process %ld] Secondary instance starts\n", my_pid); + } + + ReadOptions ropts; + ropts.verify_checksums = true; + ropts.total_order_seek = true; + + std::vector test_threads; + test_threads.emplace_back([&]() { + while (1 == ShouldSecondaryWait().load(std::memory_order_relaxed)) { + std::unique_ptr iter(db->NewIterator(ropts)); + iter->SeekToFirst(); + size_t count = 0; + for (; iter->Valid(); iter->Next()) { + ++count; + } + } + fprintf(stdout, "[process %ld] Range_scan thread finished\n", my_pid); + }); + + test_threads.emplace_back([&]() { + std::srand(time(nullptr)); + while (1 == ShouldSecondaryWait().load(std::memory_order_relaxed)) { + Slice key = Key(std::rand() % kMaxKey); + std::string value; + db->Get(ropts, key, &value); + } + fprintf(stdout, "[process %ld] Point lookup thread finished\n", my_pid); + }); + + uint64_t curr_key = 0; + while (1 == ShouldSecondaryWait().load(std::memory_order_relaxed)) { + s = db->TryCatchUpWithPrimary(); + if (!s.ok()) { + fprintf(stderr, + "[process %ld] error while trying to catch up with " + "primary %s\n", + my_pid, s.ToString().c_str()); + assert(false); + } + { + std::unique_ptr iter(db->NewIterator(ropts)); + if (!iter) { + fprintf(stderr, "[process %ld] Failed to create iterator\n", my_pid); + assert(false); + } + iter->SeekToLast(); + if (iter->Valid()) { + uint64_t curr_max_key = Key(iter->key().ToString()); + if (curr_max_key != curr_key) { + fprintf(stdout, "[process %ld] Observed key %" PRIu64 "\n", my_pid, + curr_key); + curr_key = curr_max_key; + } + } + } + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + s = db->TryCatchUpWithPrimary(); + if (!s.ok()) { + fprintf(stderr, + "[process %ld] error while trying to catch up with " + "primary %s\n", + my_pid, s.ToString().c_str()); + assert(false); + } + + std::vector column_families; + for (const auto& cf_name : GetColumnFamilyNames()) { + column_families.push_back(ColumnFamilyDescriptor(cf_name, options)); + } + std::vector handles; + DB* verification_db = nullptr; + s = DB::OpenForReadOnly(options, kDBPath, column_families, &handles, + &verification_db); + assert(s.ok()); + Iterator* iter1 = verification_db->NewIterator(ropts); + iter1->SeekToFirst(); + + Iterator* iter = db->NewIterator(ropts); + iter->SeekToFirst(); + for (; iter->Valid() && iter1->Valid(); iter->Next(), iter1->Next()) { + if (iter->key().ToString() != iter1->key().ToString()) { + fprintf(stderr, "%" PRIu64 "!= %" PRIu64 "\n", + Key(iter->key().ToString()), Key(iter1->key().ToString())); + assert(false); + } else if (iter->value().ToString() != iter1->value().ToString()) { + fprintf(stderr, "Value mismatch\n"); + assert(false); + } + } + fprintf(stdout, "[process %ld] Verification succeeded\n", my_pid); + for (auto& thr : test_threads) { + thr.join(); + } + delete iter; + delete iter1; + delete db; + delete verification_db; +} + +int main(int argc, char** argv) { + if (argc < 2) { + fprintf(stderr, "%s <0 for primary, 1 for secondary>\n", argv[0]); + return 0; + } + if (atoi(argv[1]) == 0) { + RunPrimary(); + } else { + RunSecondary(); + } + return 0; +} +#else // OS_LINUX +int main() { + fprintf(stderr, "Not implemented.\n"); + return 0; +} +#endif // !OS_LINUX diff --git a/librocksdb-sys/rocksdb/examples/optimistic_transaction_example.cc b/librocksdb-sys/rocksdb/examples/optimistic_transaction_example.cc new file mode 100644 index 0000000..0795727 --- /dev/null +++ b/librocksdb-sys/rocksdb/examples/optimistic_transaction_example.cc @@ -0,0 +1,190 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + + +#include "rocksdb/db.h" +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "rocksdb/utilities/optimistic_transaction_db.h" +#include "rocksdb/utilities/transaction.h" + +using ROCKSDB_NAMESPACE::DB; +using ROCKSDB_NAMESPACE::OptimisticTransactionDB; +using ROCKSDB_NAMESPACE::OptimisticTransactionOptions; +using ROCKSDB_NAMESPACE::Options; +using ROCKSDB_NAMESPACE::ReadOptions; +using ROCKSDB_NAMESPACE::Snapshot; +using ROCKSDB_NAMESPACE::Status; +using ROCKSDB_NAMESPACE::Transaction; +using ROCKSDB_NAMESPACE::WriteOptions; + +#if defined(OS_WIN) +std::string kDBPath = "C:\\Windows\\TEMP\\rocksdb_transaction_example"; +#else +std::string kDBPath = "/tmp/rocksdb_transaction_example"; +#endif + +int main() { + // open DB + Options options; + options.create_if_missing = true; + DB* db; + OptimisticTransactionDB* txn_db; + + Status s = OptimisticTransactionDB::Open(options, kDBPath, &txn_db); + assert(s.ok()); + db = txn_db->GetBaseDB(); + + WriteOptions write_options; + ReadOptions read_options; + OptimisticTransactionOptions txn_options; + std::string value; + + //////////////////////////////////////////////////////// + // + // Simple OptimisticTransaction Example ("Read Committed") + // + //////////////////////////////////////////////////////// + + // Start a transaction + Transaction* txn = txn_db->BeginTransaction(write_options); + assert(txn); + + // Read a key in this transaction + s = txn->Get(read_options, "abc", &value); + assert(s.IsNotFound()); + + // Write a key in this transaction + s = txn->Put("abc", "xyz"); + assert(s.ok()); + + // Read a key OUTSIDE this transaction. Does not affect txn. + s = db->Get(read_options, "abc", &value); + assert(s.IsNotFound()); + + // Write a key OUTSIDE of this transaction. + // Does not affect txn since this is an unrelated key. If we wrote key 'abc' + // here, the transaction would fail to commit. + s = db->Put(write_options, "xyz", "zzz"); + assert(s.ok()); + s = db->Put(write_options, "abc", "def"); + assert(s.ok()); + + // Commit transaction + s = txn->Commit(); + assert(s.IsBusy()); + delete txn; + + s = db->Get(read_options, "xyz", &value); + assert(s.ok()); + assert(value == "zzz"); + + s = db->Get(read_options, "abc", &value); + assert(s.ok()); + assert(value == "def"); + + //////////////////////////////////////////////////////// + // + // "Repeatable Read" (Snapshot Isolation) Example + // -- Using a single Snapshot + // + //////////////////////////////////////////////////////// + + // Set a snapshot at start of transaction by setting set_snapshot=true + txn_options.set_snapshot = true; + txn = txn_db->BeginTransaction(write_options, txn_options); + + const Snapshot* snapshot = txn->GetSnapshot(); + + // Write a key OUTSIDE of transaction + s = db->Put(write_options, "abc", "xyz"); + assert(s.ok()); + + // Read a key using the snapshot + read_options.snapshot = snapshot; + s = txn->GetForUpdate(read_options, "abc", &value); + assert(s.ok()); + assert(value == "def"); + + // Attempt to commit transaction + s = txn->Commit(); + + // Transaction could not commit since the write outside of the txn conflicted + // with the read! + assert(s.IsBusy()); + + delete txn; + // Clear snapshot from read options since it is no longer valid + read_options.snapshot = nullptr; + snapshot = nullptr; + + s = db->Get(read_options, "abc", &value); + assert(s.ok()); + assert(value == "xyz"); + + //////////////////////////////////////////////////////// + // + // "Read Committed" (Monotonic Atomic Views) Example + // --Using multiple Snapshots + // + //////////////////////////////////////////////////////// + + // In this example, we set the snapshot multiple times. This is probably + // only necessary if you have very strict isolation requirements to + // implement. + + // Set a snapshot at start of transaction + txn_options.set_snapshot = true; + txn = txn_db->BeginTransaction(write_options, txn_options); + + // Do some reads and writes to key "x" + read_options.snapshot = db->GetSnapshot(); + s = txn->Get(read_options, "x", &value); + assert(s.IsNotFound()); + s = txn->Put("x", "x"); + assert(s.ok()); + + // The transaction hasn't committed, so the write is not visible + // outside of txn. + s = db->Get(read_options, "x", &value); + assert(s.IsNotFound()); + + // Do a write outside of the transaction to key "y" + s = db->Put(write_options, "y", "z"); + assert(s.ok()); + + // Set a new snapshot in the transaction + txn->SetSnapshot(); + read_options.snapshot = db->GetSnapshot(); + + // Do some reads and writes to key "y" + s = txn->GetForUpdate(read_options, "y", &value); + assert(s.ok()); + assert(value == "z"); + txn->Put("y", "y"); + + // Commit. Since the snapshot was advanced, the write done outside of the + // transaction does not prevent this transaction from Committing. + s = txn->Commit(); + assert(s.ok()); + delete txn; + // Clear snapshot from read options since it is no longer valid + read_options.snapshot = nullptr; + + // txn is committed, read the latest values. + s = db->Get(read_options, "x", &value); + assert(s.ok()); + assert(value == "x"); + + s = db->Get(read_options, "y", &value); + assert(s.ok()); + assert(value == "y"); + + // Cleanup + delete txn_db; + DestroyDB(kDBPath, options); + return 0; +} + diff --git a/librocksdb-sys/rocksdb/examples/options_file_example.cc b/librocksdb-sys/rocksdb/examples/options_file_example.cc new file mode 100644 index 0000000..00632f3 --- /dev/null +++ b/librocksdb-sys/rocksdb/examples/options_file_example.cc @@ -0,0 +1,132 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file demonstrates how to use the utility functions defined in +// rocksdb/utilities/options_util.h to open a rocksdb database without +// remembering all the rocksdb options. +#include +#include +#include + +#include "rocksdb/cache.h" +#include "rocksdb/compaction_filter.h" +#include "rocksdb/db.h" +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "rocksdb/table.h" +#include "rocksdb/utilities/options_util.h" + +using ROCKSDB_NAMESPACE::BlockBasedTableOptions; +using ROCKSDB_NAMESPACE::ColumnFamilyDescriptor; +using ROCKSDB_NAMESPACE::ColumnFamilyHandle; +using ROCKSDB_NAMESPACE::ColumnFamilyOptions; +using ROCKSDB_NAMESPACE::CompactionFilter; +using ROCKSDB_NAMESPACE::ConfigOptions; +using ROCKSDB_NAMESPACE::DB; +using ROCKSDB_NAMESPACE::DBOptions; +using ROCKSDB_NAMESPACE::NewLRUCache; +using ROCKSDB_NAMESPACE::Options; +using ROCKSDB_NAMESPACE::Slice; +using ROCKSDB_NAMESPACE::Status; + +#if defined(OS_WIN) +std::string kDBPath = "C:\\Windows\\TEMP\\rocksdb_options_file_example"; +#else +std::string kDBPath = "/tmp/rocksdb_options_file_example"; +#endif + +namespace { +// A dummy compaction filter +class DummyCompactionFilter : public CompactionFilter { + public: + virtual ~DummyCompactionFilter() {} + virtual bool Filter(int level, const Slice& key, const Slice& existing_value, + std::string* new_value, bool* value_changed) const { + return false; + } + virtual const char* Name() const { return "DummyCompactionFilter"; } +}; + +} // namespace + +int main() { + DBOptions db_opt; + db_opt.create_if_missing = true; + + std::vector cf_descs; + cf_descs.push_back( + {ROCKSDB_NAMESPACE::kDefaultColumnFamilyName, ColumnFamilyOptions()}); + cf_descs.push_back({"new_cf", ColumnFamilyOptions()}); + + // initialize BlockBasedTableOptions + auto cache = NewLRUCache(1 * 1024 * 1024 * 1024); + BlockBasedTableOptions bbt_opts; + bbt_opts.block_size = 32 * 1024; + bbt_opts.block_cache = cache; + + // initialize column families options + std::unique_ptr compaction_filter; + compaction_filter.reset(new DummyCompactionFilter()); + cf_descs[0].options.table_factory.reset(NewBlockBasedTableFactory(bbt_opts)); + cf_descs[0].options.compaction_filter = compaction_filter.get(); + cf_descs[1].options.table_factory.reset(NewBlockBasedTableFactory(bbt_opts)); + + // destroy and open DB + DB* db; + Status s = ROCKSDB_NAMESPACE::DestroyDB(kDBPath, + Options(db_opt, cf_descs[0].options)); + assert(s.ok()); + s = DB::Open(Options(db_opt, cf_descs[0].options), kDBPath, &db); + assert(s.ok()); + + // Create column family, and rocksdb will persist the options. + ColumnFamilyHandle* cf; + s = db->CreateColumnFamily(ColumnFamilyOptions(), "new_cf", &cf); + assert(s.ok()); + + // close DB + delete cf; + delete db; + + // In the following code, we will reopen the rocksdb instance using + // the options file stored in the db directory. + + // Load the options file. + DBOptions loaded_db_opt; + std::vector loaded_cf_descs; + ConfigOptions config_options; + s = LoadLatestOptions(config_options, kDBPath, &loaded_db_opt, + &loaded_cf_descs); + assert(s.ok()); + assert(loaded_db_opt.create_if_missing == db_opt.create_if_missing); + + // Initialize pointer options for each column family + for (size_t i = 0; i < loaded_cf_descs.size(); ++i) { + auto* loaded_bbt_opt = + loaded_cf_descs[0] + .options.table_factory->GetOptions(); + // Expect the same as BlockBasedTableOptions will be loaded form file. + assert(loaded_bbt_opt->block_size == bbt_opts.block_size); + // However, block_cache needs to be manually initialized as documented + // in rocksdb/utilities/options_util.h. + loaded_bbt_opt->block_cache = cache; + } + // In addition, as pointer options are initialized with default value, + // we need to properly initialized all the pointer options if non-defalut + // values are used before calling DB::Open(). + assert(loaded_cf_descs[0].options.compaction_filter == nullptr); + loaded_cf_descs[0].options.compaction_filter = compaction_filter.get(); + + // reopen the db using the loaded options. + std::vector handles; + s = DB::Open(loaded_db_opt, kDBPath, loaded_cf_descs, &handles, &db); + assert(s.ok()); + + // close DB + for (auto* handle : handles) { + delete handle; + } + delete db; +} diff --git a/librocksdb-sys/rocksdb/examples/rocksdb_backup_restore_example.cc b/librocksdb-sys/rocksdb/examples/rocksdb_backup_restore_example.cc new file mode 100644 index 0000000..c833ed1 --- /dev/null +++ b/librocksdb-sys/rocksdb/examples/rocksdb_backup_restore_example.cc @@ -0,0 +1,99 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include +#include +#include + +#include "rocksdb/db.h" +#include "rocksdb/options.h" +#include "rocksdb/utilities/backup_engine.h" + +using ROCKSDB_NAMESPACE::BackupEngine; +using ROCKSDB_NAMESPACE::BackupEngineOptions; +using ROCKSDB_NAMESPACE::BackupEngineReadOnly; +using ROCKSDB_NAMESPACE::BackupInfo; +using ROCKSDB_NAMESPACE::DB; +using ROCKSDB_NAMESPACE::Env; +using ROCKSDB_NAMESPACE::Options; +using ROCKSDB_NAMESPACE::ReadOptions; +using ROCKSDB_NAMESPACE::Status; +using ROCKSDB_NAMESPACE::WriteOptions; + +#if defined(OS_WIN) +std::string kDBPath = "C:\\Windows\\TEMP\\rocksdb_example"; +#else +std::string kDBPath = "/tmp/rocksdb_example"; +#endif + +int main() { + DB* db; + Options options; + // Optimize RocksDB. This is the easiest way to get RocksDB to perform well + options.IncreaseParallelism(); + options.OptimizeLevelStyleCompaction(); + // create the DB if it's not already present + options.create_if_missing = true; + + // open DB + Status s = DB::Open(options, kDBPath, &db); + assert(s.ok()); + + // Put key-value + db->Put(WriteOptions(), "key1", "value1"); + assert(s.ok()); + + // create backup + BackupEngine* backup_engine; + s = BackupEngine::Open(Env::Default(), + BackupEngineOptions("/tmp/rocksdb_example_backup"), + &backup_engine); + assert(s.ok()); + + backup_engine->CreateNewBackup(db); + assert(s.ok()); + + std::vector backup_info; + backup_engine->GetBackupInfo(&backup_info); + + s = backup_engine->VerifyBackup(1); + assert(s.ok()); + + // Put key-value + db->Put(WriteOptions(), "key2", "value2"); + assert(s.ok()); + + db->Close(); + delete db; + db = nullptr; + + // restore db to backup 1 + BackupEngineReadOnly* backup_engine_ro; + s = BackupEngineReadOnly::Open( + Env::Default(), BackupEngineOptions("/tmp/rocksdb_example_backup"), + &backup_engine_ro); + assert(s.ok()); + + s = backup_engine_ro->RestoreDBFromBackup(1, "/tmp/rocksdb_example", + "/tmp/rocksdb_example"); + assert(s.ok()); + + // open db again + s = DB::Open(options, kDBPath, &db); + assert(s.ok()); + + std::string value; + s = db->Get(ReadOptions(), "key1", &value); + assert(!s.IsNotFound()); + + s = db->Get(ReadOptions(), "key2", &value); + assert(s.IsNotFound()); + + delete backup_engine; + delete backup_engine_ro; + delete db; + + return 0; +} diff --git a/librocksdb-sys/rocksdb/examples/rocksdb_option_file_example.ini b/librocksdb-sys/rocksdb/examples/rocksdb_option_file_example.ini new file mode 100644 index 0000000..351890e --- /dev/null +++ b/librocksdb-sys/rocksdb/examples/rocksdb_option_file_example.ini @@ -0,0 +1,142 @@ +# This is a RocksDB option file. +# +# A typical RocksDB options file has four sections, which are +# Version section, DBOptions section, at least one CFOptions +# section, and one TableOptions section for each column family. +# The RocksDB options file in general follows the basic INI +# file format with the following extensions / modifications: +# +# * Escaped characters +# We escaped the following characters: +# - \n -- line feed - new line +# - \r -- carriage return +# - \\ -- backslash \ +# - \: -- colon symbol : +# - \# -- hash tag # +# * Comments +# We support # style comments. Comments can appear at the ending +# part of a line. +# * Statements +# A statement is of the form option_name = value. +# Each statement contains a '=', where extra white-spaces +# are supported. However, we don't support multi-lined statement. +# Furthermore, each line can only contain at most one statement. +# * Sections +# Sections are of the form [SecitonTitle "SectionArgument"], +# where section argument is optional. +# * List +# We use colon-separated string to represent a list. +# For instance, n1:n2:n3:n4 is a list containing four values. +# +# Below is an example of a RocksDB options file: +[Version] + rocksdb_version=4.3.0 + options_file_version=1.1 + +[DBOptions] + stats_dump_period_sec=600 + max_manifest_file_size=18446744073709551615 + bytes_per_sync=8388608 + delayed_write_rate=2097152 + WAL_ttl_seconds=0 + WAL_size_limit_MB=0 + max_subcompactions=1 + wal_dir= + wal_bytes_per_sync=0 + db_write_buffer_size=0 + keep_log_file_num=1000 + table_cache_numshardbits=4 + max_file_opening_threads=1 + writable_file_max_buffer_size=1048576 + random_access_max_buffer_size=1048576 + use_fsync=false + max_total_wal_size=0 + max_open_files=-1 + skip_stats_update_on_db_open=false + max_background_compactions=16 + manifest_preallocation_size=4194304 + max_background_flushes=7 + is_fd_close_on_exec=true + max_log_file_size=0 + advise_random_on_open=true + create_missing_column_families=false + paranoid_checks=true + delete_obsolete_files_period_micros=21600000000 + log_file_time_to_roll=0 + compaction_readahead_size=0 + create_if_missing=false + use_adaptive_mutex=false + enable_thread_tracking=false + allow_fallocate=true + error_if_exists=false + recycle_log_file_num=0 + db_log_dir= + skip_log_error_on_recovery=false + new_table_reader_for_compaction_inputs=true + allow_mmap_reads=false + allow_mmap_writes=false + use_direct_reads=false + use_direct_writes=false + + +[CFOptions "default"] + compaction_style=kCompactionStyleLevel + compaction_filter=nullptr + num_levels=6 + table_factory=BlockBasedTable + comparator=leveldb.BytewiseComparator + max_sequential_skip_in_iterations=8 + max_bytes_for_level_base=1073741824 + memtable_prefix_bloom_probes=6 + memtable_prefix_bloom_bits=0 + memtable_prefix_bloom_huge_page_tlb_size=0 + max_successive_merges=0 + arena_block_size=16777216 + min_write_buffer_number_to_merge=1 + target_file_size_multiplier=1 + source_compaction_factor=1 + max_bytes_for_level_multiplier=8 + max_bytes_for_level_multiplier_additional=2:3:5 + compaction_filter_factory=nullptr + max_write_buffer_number=8 + level0_stop_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + purge_redundant_kvs_while_flush=true + max_write_buffer_size_to_maintain=0 + memtable_factory=SkipListFactory + max_grandparent_overlap_factor=8 + expanded_compaction_factor=25 + hard_pending_compaction_bytes_limit=137438953472 + inplace_update_num_locks=10000 + level_compaction_dynamic_level_bytes=true + level0_slowdown_writes_trigger=12 + filter_deletes=false + verify_checksums_in_compaction=true + min_partial_merge_operands=2 + paranoid_file_checks=false + target_file_size_base=134217728 + optimize_filters_for_hits=false + merge_operator=PutOperator + compression_per_level=kNoCompression:kNoCompression:kNoCompression:kSnappyCompression:kSnappyCompression:kSnappyCompression + compaction_measure_io_stats=false + prefix_extractor=nullptr + bloom_locality=0 + write_buffer_size=134217728 + disable_auto_compactions=false + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + format_version=2 + whole_key_filtering=true + no_block_cache=false + checksum=kCRC32c + filter_policy=rocksdb.BuiltinBloomFilter + block_size_deviation=10 + block_size=8192 + block_restart_interval=16 + cache_index_and_filter_blocks=false + pin_l0_filter_and_index_blocks_in_cache=false + pin_top_level_index_and_filter=false + index_type=kBinarySearch + flush_block_policy_factory=FlushBlockBySizePolicyFactory diff --git a/librocksdb-sys/rocksdb/examples/simple_example.cc b/librocksdb-sys/rocksdb/examples/simple_example.cc new file mode 100644 index 0000000..2d49c4d --- /dev/null +++ b/librocksdb-sys/rocksdb/examples/simple_example.cc @@ -0,0 +1,93 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include +#include + +#include "rocksdb/db.h" +#include "rocksdb/options.h" +#include "rocksdb/slice.h" + +using ROCKSDB_NAMESPACE::DB; +using ROCKSDB_NAMESPACE::Options; +using ROCKSDB_NAMESPACE::PinnableSlice; +using ROCKSDB_NAMESPACE::ReadOptions; +using ROCKSDB_NAMESPACE::Status; +using ROCKSDB_NAMESPACE::WriteBatch; +using ROCKSDB_NAMESPACE::WriteOptions; + +#if defined(OS_WIN) +std::string kDBPath = "C:\\Windows\\TEMP\\rocksdb_simple_example"; +#else +std::string kDBPath = "/tmp/rocksdb_simple_example"; +#endif + +int main() { + DB* db; + Options options; + // Optimize RocksDB. This is the easiest way to get RocksDB to perform well + options.IncreaseParallelism(); + options.OptimizeLevelStyleCompaction(); + // create the DB if it's not already present + options.create_if_missing = true; + + // open DB + Status s = DB::Open(options, kDBPath, &db); + assert(s.ok()); + + // Put key-value + s = db->Put(WriteOptions(), "key1", "value"); + assert(s.ok()); + std::string value; + // get value + s = db->Get(ReadOptions(), "key1", &value); + assert(s.ok()); + assert(value == "value"); + + // atomically apply a set of updates + { + WriteBatch batch; + batch.Delete("key1"); + batch.Put("key2", value); + s = db->Write(WriteOptions(), &batch); + } + + s = db->Get(ReadOptions(), "key1", &value); + assert(s.IsNotFound()); + + db->Get(ReadOptions(), "key2", &value); + assert(value == "value"); + + { + PinnableSlice pinnable_val; + db->Get(ReadOptions(), db->DefaultColumnFamily(), "key2", &pinnable_val); + assert(pinnable_val == "value"); + } + + { + std::string string_val; + // If it cannot pin the value, it copies the value to its internal buffer. + // The intenral buffer could be set during construction. + PinnableSlice pinnable_val(&string_val); + db->Get(ReadOptions(), db->DefaultColumnFamily(), "key2", &pinnable_val); + assert(pinnable_val == "value"); + // If the value is not pinned, the internal buffer must have the value. + assert(pinnable_val.IsPinned() || string_val == "value"); + } + + PinnableSlice pinnable_val; + s = db->Get(ReadOptions(), db->DefaultColumnFamily(), "key1", &pinnable_val); + assert(s.IsNotFound()); + // Reset PinnableSlice after each use and before each reuse + pinnable_val.Reset(); + db->Get(ReadOptions(), db->DefaultColumnFamily(), "key2", &pinnable_val); + assert(pinnable_val == "value"); + pinnable_val.Reset(); + // The Slice pointed by pinnable_val is not valid after this point + + delete db; + + return 0; +} diff --git a/librocksdb-sys/rocksdb/examples/transaction_example.cc b/librocksdb-sys/rocksdb/examples/transaction_example.cc new file mode 100644 index 0000000..541b13f --- /dev/null +++ b/librocksdb-sys/rocksdb/examples/transaction_example.cc @@ -0,0 +1,196 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + + +#include "rocksdb/db.h" +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "rocksdb/utilities/transaction.h" +#include "rocksdb/utilities/transaction_db.h" + +using ROCKSDB_NAMESPACE::Options; +using ROCKSDB_NAMESPACE::ReadOptions; +using ROCKSDB_NAMESPACE::Snapshot; +using ROCKSDB_NAMESPACE::Status; +using ROCKSDB_NAMESPACE::Transaction; +using ROCKSDB_NAMESPACE::TransactionDB; +using ROCKSDB_NAMESPACE::TransactionDBOptions; +using ROCKSDB_NAMESPACE::TransactionOptions; +using ROCKSDB_NAMESPACE::WriteOptions; + +#if defined(OS_WIN) +std::string kDBPath = "C:\\Windows\\TEMP\\rocksdb_transaction_example"; +#else +std::string kDBPath = "/tmp/rocksdb_transaction_example"; +#endif + +int main() { + // open DB + Options options; + TransactionDBOptions txn_db_options; + options.create_if_missing = true; + TransactionDB* txn_db; + + Status s = TransactionDB::Open(options, txn_db_options, kDBPath, &txn_db); + assert(s.ok()); + + WriteOptions write_options; + ReadOptions read_options; + TransactionOptions txn_options; + std::string value; + + //////////////////////////////////////////////////////// + // + // Simple Transaction Example ("Read Committed") + // + //////////////////////////////////////////////////////// + + // Start a transaction + Transaction* txn = txn_db->BeginTransaction(write_options); + assert(txn); + + // Read a key in this transaction + s = txn->Get(read_options, "abc", &value); + assert(s.IsNotFound()); + + // Write a key in this transaction + s = txn->Put("abc", "def"); + assert(s.ok()); + + // Read a key OUTSIDE this transaction. Does not affect txn. + s = txn_db->Get(read_options, "abc", &value); + assert(s.IsNotFound()); + + // Write a key OUTSIDE of this transaction. + // Does not affect txn since this is an unrelated key. + s = txn_db->Put(write_options, "xyz", "zzz"); + assert(s.ok()); + + // Write a key OUTSIDE of this transaction. + // Fail because the key conflicts with the key written in txn. + s = txn_db->Put(write_options, "abc", "def"); + assert(s.subcode() == Status::kLockTimeout); + + // Value for key "xyz" has been committed, can be read in txn. + s = txn->Get(read_options, "xyz", &value); + assert(s.ok()); + assert(value == "zzz"); + + // Commit transaction + s = txn->Commit(); + assert(s.ok()); + delete txn; + + // Value is committed, can be read now. + s = txn_db->Get(read_options, "abc", &value); + assert(s.ok()); + assert(value == "def"); + + //////////////////////////////////////////////////////// + // + // "Repeatable Read" (Snapshot Isolation) Example + // -- Using a single Snapshot + // + //////////////////////////////////////////////////////// + + // Set a snapshot at start of transaction by setting set_snapshot=true + txn_options.set_snapshot = true; + txn = txn_db->BeginTransaction(write_options, txn_options); + + const Snapshot* snapshot = txn->GetSnapshot(); + + // Write a key OUTSIDE of transaction + s = txn_db->Put(write_options, "abc", "xyz"); + assert(s.ok()); + + // Read the latest committed value. + s = txn->Get(read_options, "abc", &value); + assert(s.ok()); + assert(value == "xyz"); + + // Read the snapshotted value. + read_options.snapshot = snapshot; + s = txn->Get(read_options, "abc", &value); + assert(s.ok()); + assert(value == "def"); + + // Attempt to read a key using the snapshot. This will fail since + // the previous write outside this txn conflicts with this read. + s = txn->GetForUpdate(read_options, "abc", &value); + assert(s.IsBusy()); + + txn->Rollback(); + + // Snapshot will be released upon deleting the transaction. + delete txn; + // Clear snapshot from read options since it is no longer valid + read_options.snapshot = nullptr; + snapshot = nullptr; + + //////////////////////////////////////////////////////// + // + // "Read Committed" (Monotonic Atomic Views) Example + // --Using multiple Snapshots + // + //////////////////////////////////////////////////////// + + // In this example, we set the snapshot multiple times. This is probably + // only necessary if you have very strict isolation requirements to + // implement. + + // Set a snapshot at start of transaction + txn_options.set_snapshot = true; + txn = txn_db->BeginTransaction(write_options, txn_options); + + // Do some reads and writes to key "x" + read_options.snapshot = txn_db->GetSnapshot(); + s = txn->Get(read_options, "x", &value); + assert(s.IsNotFound()); + s = txn->Put("x", "x"); + assert(s.ok()); + + // Do a write outside of the transaction to key "y" + s = txn_db->Put(write_options, "y", "y1"); + assert(s.ok()); + + // Set a new snapshot in the transaction + txn->SetSnapshot(); + txn->SetSavePoint(); + read_options.snapshot = txn_db->GetSnapshot(); + + // Do some reads and writes to key "y" + // Since the snapshot was advanced, the write done outside of the + // transaction does not conflict. + s = txn->GetForUpdate(read_options, "y", &value); + assert(s.ok()); + assert(value == "y1"); + s = txn->Put("y", "y2"); + assert(s.ok()); + + // Decide we want to revert the last write from this transaction. + txn->RollbackToSavePoint(); + + // Commit. + s = txn->Commit(); + assert(s.ok()); + delete txn; + // Clear snapshot from read options since it is no longer valid + read_options.snapshot = nullptr; + + // db state is at the save point. + s = txn_db->Get(read_options, "x", &value); + assert(s.ok()); + assert(value == "x"); + + s = txn_db->Get(read_options, "y", &value); + assert(s.ok()); + assert(value == "y1"); + + // Cleanup + delete txn_db; + ROCKSDB_NAMESPACE::DestroyDB(kDBPath, options); + return 0; +} + diff --git a/librocksdb-sys/rocksdb/file/delete_scheduler.cc b/librocksdb-sys/rocksdb/file/delete_scheduler.cc new file mode 100644 index 0000000..78ea6f7 --- /dev/null +++ b/librocksdb-sys/rocksdb/file/delete_scheduler.cc @@ -0,0 +1,410 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + + +#include "file/delete_scheduler.h" + +#include +#include +#include + +#include "file/sst_file_manager_impl.h" +#include "logging/logging.h" +#include "port/port.h" +#include "rocksdb/env.h" +#include "rocksdb/file_system.h" +#include "rocksdb/system_clock.h" +#include "test_util/sync_point.h" +#include "util/mutexlock.h" + +namespace ROCKSDB_NAMESPACE { + +DeleteScheduler::DeleteScheduler(SystemClock* clock, FileSystem* fs, + int64_t rate_bytes_per_sec, Logger* info_log, + SstFileManagerImpl* sst_file_manager, + double max_trash_db_ratio, + uint64_t bytes_max_delete_chunk) + : clock_(clock), + fs_(fs), + total_trash_size_(0), + rate_bytes_per_sec_(rate_bytes_per_sec), + pending_files_(0), + bytes_max_delete_chunk_(bytes_max_delete_chunk), + closing_(false), + cv_(&mu_), + bg_thread_(nullptr), + info_log_(info_log), + sst_file_manager_(sst_file_manager), + max_trash_db_ratio_(max_trash_db_ratio) { + assert(sst_file_manager != nullptr); + assert(max_trash_db_ratio >= 0); + MaybeCreateBackgroundThread(); +} + +DeleteScheduler::~DeleteScheduler() { + { + InstrumentedMutexLock l(&mu_); + closing_ = true; + cv_.SignalAll(); + } + if (bg_thread_) { + bg_thread_->join(); + } + for (const auto& it : bg_errors_) { + it.second.PermitUncheckedError(); + } +} + +Status DeleteScheduler::DeleteFile(const std::string& file_path, + const std::string& dir_to_sync, + const bool force_bg) { + if (rate_bytes_per_sec_.load() <= 0 || + (!force_bg && + total_trash_size_.load() > + sst_file_manager_->GetTotalSize() * max_trash_db_ratio_.load())) { + // Rate limiting is disabled or trash size makes up more than + // max_trash_db_ratio_ (default 25%) of the total DB size + TEST_SYNC_POINT("DeleteScheduler::DeleteFile"); + Status s = fs_->DeleteFile(file_path, IOOptions(), nullptr); + if (s.ok()) { + s = sst_file_manager_->OnDeleteFile(file_path); + ROCKS_LOG_INFO(info_log_, + "Deleted file %s immediately, rate_bytes_per_sec %" PRIi64 + ", total_trash_size %" PRIu64 " max_trash_db_ratio %lf", + file_path.c_str(), rate_bytes_per_sec_.load(), + total_trash_size_.load(), max_trash_db_ratio_.load()); + InstrumentedMutexLock l(&mu_); + RecordTick(stats_.get(), FILES_DELETED_IMMEDIATELY); + } + return s; + } + + // Move file to trash + std::string trash_file; + Status s = MarkAsTrash(file_path, &trash_file); + ROCKS_LOG_INFO(info_log_, "Mark file: %s as trash -- %s", trash_file.c_str(), + s.ToString().c_str()); + + if (!s.ok()) { + ROCKS_LOG_ERROR(info_log_, "Failed to mark %s as trash -- %s", + file_path.c_str(), s.ToString().c_str()); + s = fs_->DeleteFile(file_path, IOOptions(), nullptr); + if (s.ok()) { + s = sst_file_manager_->OnDeleteFile(file_path); + ROCKS_LOG_INFO(info_log_, "Deleted file %s immediately", + trash_file.c_str()); + InstrumentedMutexLock l(&mu_); + RecordTick(stats_.get(), FILES_DELETED_IMMEDIATELY); + } + return s; + } + + // Update the total trash size + uint64_t trash_file_size = 0; + IOStatus io_s = + fs_->GetFileSize(trash_file, IOOptions(), &trash_file_size, nullptr); + if (io_s.ok()) { + total_trash_size_.fetch_add(trash_file_size); + } + //**TODO: What should we do if we failed to + // get the file size? + + // Add file to delete queue + { + InstrumentedMutexLock l(&mu_); + RecordTick(stats_.get(), FILES_MARKED_TRASH); + queue_.emplace(trash_file, dir_to_sync); + pending_files_++; + if (pending_files_ == 1) { + cv_.SignalAll(); + } + } + return s; +} + +std::map DeleteScheduler::GetBackgroundErrors() { + InstrumentedMutexLock l(&mu_); + return bg_errors_; +} + +const std::string DeleteScheduler::kTrashExtension = ".trash"; +bool DeleteScheduler::IsTrashFile(const std::string& file_path) { + return (file_path.size() >= kTrashExtension.size() && + file_path.rfind(kTrashExtension) == + file_path.size() - kTrashExtension.size()); +} + +Status DeleteScheduler::CleanupDirectory(Env* env, SstFileManagerImpl* sfm, + const std::string& path) { + Status s; + // Check if there are any files marked as trash in this path + std::vector files_in_path; + const auto& fs = env->GetFileSystem(); + IOOptions io_opts; + io_opts.do_not_recurse = true; + s = fs->GetChildren(path, io_opts, &files_in_path, + /*IODebugContext*=*/nullptr); + if (!s.ok()) { + return s; + } + for (const std::string& current_file : files_in_path) { + if (!DeleteScheduler::IsTrashFile(current_file)) { + // not a trash file, skip + continue; + } + + Status file_delete; + std::string trash_file = path + "/" + current_file; + if (sfm) { + // We have an SstFileManager that will schedule the file delete + s = sfm->OnAddFile(trash_file); + file_delete = sfm->ScheduleFileDeletion(trash_file, path); + } else { + // Delete the file immediately + file_delete = env->DeleteFile(trash_file); + } + + if (s.ok() && !file_delete.ok()) { + s = file_delete; + } + } + + return s; +} + +Status DeleteScheduler::MarkAsTrash(const std::string& file_path, + std::string* trash_file) { + // Sanity check of the path + size_t idx = file_path.rfind("/"); + if (idx == std::string::npos || idx == file_path.size() - 1) { + return Status::InvalidArgument("file_path is corrupted"); + } + + if (DeleteScheduler::IsTrashFile(file_path)) { + // This is already a trash file + *trash_file = file_path; + return Status::OK(); + } + + *trash_file = file_path + kTrashExtension; + // TODO(tec) : Implement Env::RenameFileIfNotExist and remove + // file_move_mu mutex. + int cnt = 0; + Status s; + InstrumentedMutexLock l(&file_move_mu_); + while (true) { + s = fs_->FileExists(*trash_file, IOOptions(), nullptr); + if (s.IsNotFound()) { + // We found a path for our file in trash + s = fs_->RenameFile(file_path, *trash_file, IOOptions(), nullptr); + break; + } else if (s.ok()) { + // Name conflict, generate new random suffix + *trash_file = file_path + std::to_string(cnt) + kTrashExtension; + } else { + // Error during FileExists call, we cannot continue + break; + } + cnt++; + } + if (s.ok()) { + s = sst_file_manager_->OnMoveFile(file_path, *trash_file); + } + return s; +} + +void DeleteScheduler::BackgroundEmptyTrash() { + TEST_SYNC_POINT("DeleteScheduler::BackgroundEmptyTrash"); + + while (true) { + InstrumentedMutexLock l(&mu_); + while (queue_.empty() && !closing_) { + cv_.Wait(); + } + + if (closing_) { + return; + } + + // Delete all files in queue_ + uint64_t start_time = clock_->NowMicros(); + uint64_t total_deleted_bytes = 0; + int64_t current_delete_rate = rate_bytes_per_sec_.load(); + while (!queue_.empty() && !closing_) { + if (current_delete_rate != rate_bytes_per_sec_.load()) { + // User changed the delete rate + current_delete_rate = rate_bytes_per_sec_.load(); + start_time = clock_->NowMicros(); + total_deleted_bytes = 0; + ROCKS_LOG_INFO(info_log_, "rate_bytes_per_sec is changed to %" PRIi64, + current_delete_rate); + } + + // Get new file to delete + const FileAndDir& fad = queue_.front(); + std::string path_in_trash = fad.fname; + + // We don't need to hold the lock while deleting the file + mu_.Unlock(); + uint64_t deleted_bytes = 0; + bool is_complete = true; + // Delete file from trash and update total_penlty value + Status s = + DeleteTrashFile(path_in_trash, fad.dir, &deleted_bytes, &is_complete); + total_deleted_bytes += deleted_bytes; + mu_.Lock(); + if (is_complete) { + RecordTick(stats_.get(), FILES_DELETED_FROM_TRASH_QUEUE); + queue_.pop(); + } + + if (!s.ok()) { + bg_errors_[path_in_trash] = s; + } + + // Apply penalty if necessary + uint64_t total_penalty; + if (current_delete_rate > 0) { + // rate limiting is enabled + total_penalty = + ((total_deleted_bytes * kMicrosInSecond) / current_delete_rate); + ROCKS_LOG_INFO(info_log_, + "Rate limiting is enabled with penalty %" PRIu64 + " after deleting file %s", + total_penalty, path_in_trash.c_str()); + while (!closing_ && !cv_.TimedWait(start_time + total_penalty)) { + } + } else { + // rate limiting is disabled + total_penalty = 0; + ROCKS_LOG_INFO(info_log_, + "Rate limiting is disabled after deleting file %s", + path_in_trash.c_str()); + } + TEST_SYNC_POINT_CALLBACK("DeleteScheduler::BackgroundEmptyTrash:Wait", + &total_penalty); + + if (is_complete) { + pending_files_--; + } + if (pending_files_ == 0) { + // Unblock WaitForEmptyTrash since there are no more files waiting + // to be deleted + cv_.SignalAll(); + } + } + } +} + +Status DeleteScheduler::DeleteTrashFile(const std::string& path_in_trash, + const std::string& dir_to_sync, + uint64_t* deleted_bytes, + bool* is_complete) { + uint64_t file_size; + Status s = fs_->GetFileSize(path_in_trash, IOOptions(), &file_size, nullptr); + *is_complete = true; + TEST_SYNC_POINT("DeleteScheduler::DeleteTrashFile:DeleteFile"); + if (s.ok()) { + bool need_full_delete = true; + if (bytes_max_delete_chunk_ != 0 && file_size > bytes_max_delete_chunk_) { + uint64_t num_hard_links = 2; + // We don't have to worry aobut data race between linking a new + // file after the number of file link check and ftruncte because + // the file is now in trash and no hardlink is supposed to create + // to trash files by RocksDB. + Status my_status = fs_->NumFileLinks(path_in_trash, IOOptions(), + &num_hard_links, nullptr); + if (my_status.ok()) { + if (num_hard_links == 1) { + std::unique_ptr wf; + my_status = fs_->ReopenWritableFile(path_in_trash, FileOptions(), &wf, + nullptr); + if (my_status.ok()) { + my_status = wf->Truncate(file_size - bytes_max_delete_chunk_, + IOOptions(), nullptr); + if (my_status.ok()) { + TEST_SYNC_POINT("DeleteScheduler::DeleteTrashFile:Fsync"); + my_status = wf->Fsync(IOOptions(), nullptr); + } + } + if (my_status.ok()) { + *deleted_bytes = bytes_max_delete_chunk_; + need_full_delete = false; + *is_complete = false; + } else { + ROCKS_LOG_WARN(info_log_, + "Failed to partially delete %s from trash -- %s", + path_in_trash.c_str(), my_status.ToString().c_str()); + } + } else { + ROCKS_LOG_INFO(info_log_, + "Cannot delete %s slowly through ftruncate from trash " + "as it has other links", + path_in_trash.c_str()); + } + } else if (!num_link_error_printed_) { + ROCKS_LOG_INFO( + info_log_, + "Cannot delete files slowly through ftruncate from trash " + "as Env::NumFileLinks() returns error: %s", + my_status.ToString().c_str()); + num_link_error_printed_ = true; + } + } + + if (need_full_delete) { + s = fs_->DeleteFile(path_in_trash, IOOptions(), nullptr); + if (!dir_to_sync.empty()) { + std::unique_ptr dir_obj; + if (s.ok()) { + s = fs_->NewDirectory(dir_to_sync, IOOptions(), &dir_obj, nullptr); + } + if (s.ok()) { + s = dir_obj->FsyncWithDirOptions( + IOOptions(), nullptr, + DirFsyncOptions(DirFsyncOptions::FsyncReason::kFileDeleted)); + TEST_SYNC_POINT_CALLBACK( + "DeleteScheduler::DeleteTrashFile::AfterSyncDir", + reinterpret_cast(const_cast(&dir_to_sync))); + } + } + if (s.ok()) { + *deleted_bytes = file_size; + s = sst_file_manager_->OnDeleteFile(path_in_trash); + } + } + } + if (!s.ok()) { + // Error while getting file size or while deleting + ROCKS_LOG_ERROR(info_log_, "Failed to delete %s from trash -- %s", + path_in_trash.c_str(), s.ToString().c_str()); + *deleted_bytes = 0; + } else { + total_trash_size_.fetch_sub(*deleted_bytes); + } + + return s; +} + +void DeleteScheduler::WaitForEmptyTrash() { + InstrumentedMutexLock l(&mu_); + while (pending_files_ > 0 && !closing_) { + cv_.Wait(); + } +} + +void DeleteScheduler::MaybeCreateBackgroundThread() { + if (bg_thread_ == nullptr && rate_bytes_per_sec_.load() > 0) { + bg_thread_.reset( + new port::Thread(&DeleteScheduler::BackgroundEmptyTrash, this)); + ROCKS_LOG_INFO(info_log_, + "Created background thread for deletion scheduler with " + "rate_bytes_per_sec: %" PRIi64, + rate_bytes_per_sec_.load()); + } +} + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/file/delete_scheduler.h b/librocksdb-sys/rocksdb/file/delete_scheduler.h new file mode 100644 index 0000000..da3735a --- /dev/null +++ b/librocksdb-sys/rocksdb/file/delete_scheduler.h @@ -0,0 +1,147 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + + +#include +#include +#include +#include + +#include "monitoring/instrumented_mutex.h" +#include "port/port.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +class Env; +class FileSystem; +class Logger; +class SstFileManagerImpl; +class SystemClock; + +// DeleteScheduler allows the DB to enforce a rate limit on file deletion, +// Instead of deleteing files immediately, files are marked as trash +// and deleted in a background thread that apply sleep penalty between deletes +// if they are happening in a rate faster than rate_bytes_per_sec, +// +// Rate limiting can be turned off by setting rate_bytes_per_sec = 0, In this +// case DeleteScheduler will delete files immediately. +class DeleteScheduler { + public: + DeleteScheduler(SystemClock* clock, FileSystem* fs, + int64_t rate_bytes_per_sec, Logger* info_log, + SstFileManagerImpl* sst_file_manager, + double max_trash_db_ratio, uint64_t bytes_max_delete_chunk); + + ~DeleteScheduler(); + + // Return delete rate limit in bytes per second + int64_t GetRateBytesPerSecond() { return rate_bytes_per_sec_.load(); } + + // Set delete rate limit in bytes per second + void SetRateBytesPerSecond(int64_t bytes_per_sec) { + rate_bytes_per_sec_.store(bytes_per_sec); + MaybeCreateBackgroundThread(); + } + + // Mark file as trash directory and schedule its deletion. If force_bg is + // set, it forces the file to always be deleted in the background thread, + // except when rate limiting is disabled + Status DeleteFile(const std::string& fname, const std::string& dir_to_sync, + const bool force_bg = false); + + // Wait for all files being deleteing in the background to finish or for + // destructor to be called. + void WaitForEmptyTrash(); + + // Return a map containing errors that happened in BackgroundEmptyTrash + // file_path => error status + std::map GetBackgroundErrors(); + + uint64_t GetTotalTrashSize() { return total_trash_size_.load(); } + + // Return trash/DB size ratio where new files will be deleted immediately + double GetMaxTrashDBRatio() { return max_trash_db_ratio_.load(); } + + // Update trash/DB size ratio where new files will be deleted immediately + void SetMaxTrashDBRatio(double r) { + assert(r >= 0); + max_trash_db_ratio_.store(r); + } + + static const std::string kTrashExtension; + static bool IsTrashFile(const std::string& file_path); + + // Check if there are any .trash files in path, and schedule their deletion + // Or delete immediately if sst_file_manager is nullptr + static Status CleanupDirectory(Env* env, SstFileManagerImpl* sfm, + const std::string& path); + + void SetStatisticsPtr(const std::shared_ptr& stats) { + InstrumentedMutexLock l(&mu_); + stats_ = stats; + } + + private: + Status MarkAsTrash(const std::string& file_path, std::string* path_in_trash); + + Status DeleteTrashFile(const std::string& path_in_trash, + const std::string& dir_to_sync, + uint64_t* deleted_bytes, bool* is_complete); + + void BackgroundEmptyTrash(); + + void MaybeCreateBackgroundThread(); + + SystemClock* clock_; + FileSystem* fs_; + + // total size of trash files + std::atomic total_trash_size_; + // Maximum number of bytes that should be deleted per second + std::atomic rate_bytes_per_sec_; + // Mutex to protect queue_, pending_files_, bg_errors_, closing_, stats_ + InstrumentedMutex mu_; + + struct FileAndDir { + FileAndDir(const std::string& f, const std::string& d) : fname(f), dir(d) {} + std::string fname; + std::string dir; // empty will be skipped. + }; + + // Queue of trash files that need to be deleted + std::queue queue_; + // Number of trash files that are waiting to be deleted + int32_t pending_files_; + uint64_t bytes_max_delete_chunk_; + // Errors that happened in BackgroundEmptyTrash (file_path => error) + std::map bg_errors_; + + bool num_link_error_printed_ = false; + // Set to true in ~DeleteScheduler() to force BackgroundEmptyTrash to stop + bool closing_; + // Condition variable signaled in these conditions + // - pending_files_ value change from 0 => 1 + // - pending_files_ value change from 1 => 0 + // - closing_ value is set to true + InstrumentedCondVar cv_; + // Background thread running BackgroundEmptyTrash + std::unique_ptr bg_thread_; + // Mutex to protect threads from file name conflicts + InstrumentedMutex file_move_mu_; + Logger* info_log_; + SstFileManagerImpl* sst_file_manager_; + // If the trash size constitutes for more than this fraction of the total DB + // size we will start deleting new files passed to DeleteScheduler + // immediately + std::atomic max_trash_db_ratio_; + static const uint64_t kMicrosInSecond = 1000 * 1000LL; + std::shared_ptr stats_; +}; + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/file/delete_scheduler_test.cc b/librocksdb-sys/rocksdb/file/delete_scheduler_test.cc new file mode 100644 index 0000000..62de5d2 --- /dev/null +++ b/librocksdb-sys/rocksdb/file/delete_scheduler_test.cc @@ -0,0 +1,728 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "file/delete_scheduler.h" + +#include +#include +#include +#include + +#include "file/file_util.h" +#include "file/sst_file_manager_impl.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "test_util/sync_point.h" +#include "test_util/testharness.h" +#include "util/string_util.h" + + +namespace ROCKSDB_NAMESPACE { + +class DeleteSchedulerTest : public testing::Test { + public: + DeleteSchedulerTest() : env_(Env::Default()) { + const int kNumDataDirs = 3; + dummy_files_dirs_.reserve(kNumDataDirs); + for (size_t i = 0; i < kNumDataDirs; ++i) { + dummy_files_dirs_.emplace_back( + test::PerThreadDBPath(env_, "delete_scheduler_dummy_data_dir") + + std::to_string(i)); + DestroyAndCreateDir(dummy_files_dirs_.back()); + } + stats_ = ROCKSDB_NAMESPACE::CreateDBStatistics(); + } + + ~DeleteSchedulerTest() override { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); + for (const auto& dummy_files_dir : dummy_files_dirs_) { + DestroyDir(env_, dummy_files_dir); + } + } + + void DestroyAndCreateDir(const std::string& dir) { + ASSERT_OK(DestroyDir(env_, dir)); + EXPECT_OK(env_->CreateDir(dir)); + } + + int CountNormalFiles(size_t dummy_files_dirs_idx = 0) { + std::vector files_in_dir; + EXPECT_OK(env_->GetChildren(dummy_files_dirs_[dummy_files_dirs_idx], + &files_in_dir)); + + int normal_cnt = 0; + for (auto& f : files_in_dir) { + if (!DeleteScheduler::IsTrashFile(f)) { + normal_cnt++; + } + } + return normal_cnt; + } + + int CountTrashFiles(size_t dummy_files_dirs_idx = 0) { + std::vector files_in_dir; + EXPECT_OK(env_->GetChildren(dummy_files_dirs_[dummy_files_dirs_idx], + &files_in_dir)); + + int trash_cnt = 0; + for (auto& f : files_in_dir) { + if (DeleteScheduler::IsTrashFile(f)) { + trash_cnt++; + } + } + return trash_cnt; + } + + std::string NewDummyFile(const std::string& file_name, uint64_t size = 1024, + size_t dummy_files_dirs_idx = 0) { + std::string file_path = + dummy_files_dirs_[dummy_files_dirs_idx] + "/" + file_name; + std::unique_ptr f; + env_->NewWritableFile(file_path, &f, EnvOptions()); + std::string data(size, 'A'); + EXPECT_OK(f->Append(data)); + EXPECT_OK(f->Close()); + sst_file_mgr_->OnAddFile(file_path); + return file_path; + } + + void NewDeleteScheduler() { + // Tests in this file are for DeleteScheduler component and don't create any + // DBs, so we need to set max_trash_db_ratio to 100% (instead of default + // 25%) + sst_file_mgr_.reset( + new SstFileManagerImpl(env_->GetSystemClock(), env_->GetFileSystem(), + nullptr, rate_bytes_per_sec_, + /* max_trash_db_ratio= */ 1.1, 128 * 1024)); + delete_scheduler_ = sst_file_mgr_->delete_scheduler(); + sst_file_mgr_->SetStatisticsPtr(stats_); + } + + Env* env_; + std::vector dummy_files_dirs_; + int64_t rate_bytes_per_sec_; + DeleteScheduler* delete_scheduler_; + std::unique_ptr sst_file_mgr_; + std::shared_ptr stats_; +}; + +// Test the basic functionality of DeleteScheduler (Rate Limiting). +// 1- Create 100 dummy files +// 2- Delete the 100 dummy files using DeleteScheduler +// --- Hold DeleteScheduler::BackgroundEmptyTrash --- +// 3- Wait for DeleteScheduler to delete all files in trash +// 4- Verify that BackgroundEmptyTrash used to correct penlties for the files +// 5- Make sure that all created files were completely deleted +TEST_F(DeleteSchedulerTest, BasicRateLimiting) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"DeleteSchedulerTest::BasicRateLimiting:1", + "DeleteScheduler::BackgroundEmptyTrash"}, + }); + + std::vector penalties; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DeleteScheduler::BackgroundEmptyTrash:Wait", + [&](void* arg) { penalties.push_back(*(static_cast(arg))); }); + int dir_synced = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DeleteScheduler::DeleteTrashFile::AfterSyncDir", [&](void* arg) { + dir_synced++; + std::string* dir = reinterpret_cast(arg); + EXPECT_EQ(dummy_files_dirs_[0], *dir); + }); + + int num_files = 100; // 100 files + uint64_t file_size = 1024; // every file is 1 kb + std::vector delete_kbs_per_sec = {512, 200, 100, 50, 25}; + + for (size_t t = 0; t < delete_kbs_per_sec.size(); t++) { + penalties.clear(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + DestroyAndCreateDir(dummy_files_dirs_[0]); + rate_bytes_per_sec_ = delete_kbs_per_sec[t] * 1024; + NewDeleteScheduler(); + + dir_synced = 0; + // Create 100 dummy files, every file is 1 Kb + std::vector generated_files; + for (int i = 0; i < num_files; i++) { + std::string file_name = "file" + std::to_string(i) + ".data"; + generated_files.push_back(NewDummyFile(file_name, file_size)); + } + + // Delete dummy files and measure time spent to empty trash + for (int i = 0; i < num_files; i++) { + ASSERT_OK(delete_scheduler_->DeleteFile(generated_files[i], + dummy_files_dirs_[0])); + } + ASSERT_EQ(CountNormalFiles(), 0); + + uint64_t delete_start_time = env_->NowMicros(); + TEST_SYNC_POINT("DeleteSchedulerTest::BasicRateLimiting:1"); + delete_scheduler_->WaitForEmptyTrash(); + uint64_t time_spent_deleting = env_->NowMicros() - delete_start_time; + + auto bg_errors = delete_scheduler_->GetBackgroundErrors(); + ASSERT_EQ(bg_errors.size(), 0); + + uint64_t total_files_size = 0; + uint64_t expected_penlty = 0; + ASSERT_EQ(penalties.size(), num_files); + for (int i = 0; i < num_files; i++) { + total_files_size += file_size; + expected_penlty = ((total_files_size * 1000000) / rate_bytes_per_sec_); + ASSERT_EQ(expected_penlty, penalties[i]); + } + ASSERT_GT(time_spent_deleting, expected_penlty * 0.9); + + ASSERT_EQ(num_files, dir_synced); + + ASSERT_EQ(CountTrashFiles(), 0); + ASSERT_EQ(num_files, stats_->getAndResetTickerCount(FILES_MARKED_TRASH)); + ASSERT_EQ(num_files, + stats_->getAndResetTickerCount(FILES_DELETED_FROM_TRASH_QUEUE)); + ASSERT_EQ(0, stats_->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY)); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + } +} + +TEST_F(DeleteSchedulerTest, MultiDirectoryDeletionsScheduled) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"DeleteSchedulerTest::MultiDbPathDeletionsScheduled:1", + "DeleteScheduler::BackgroundEmptyTrash"}, + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + rate_bytes_per_sec_ = 1 << 20; // 1MB + NewDeleteScheduler(); + + // Generate dummy files in multiple directories + const size_t kNumFiles = dummy_files_dirs_.size(); + const size_t kFileSize = 1 << 10; // 1KB + std::vector generated_files; + for (size_t i = 0; i < kNumFiles; i++) { + generated_files.push_back(NewDummyFile("file", kFileSize, i)); + ASSERT_EQ(1, CountNormalFiles(i)); + } + + // Mark dummy files as trash + for (size_t i = 0; i < kNumFiles; i++) { + ASSERT_OK(delete_scheduler_->DeleteFile(generated_files[i], "")); + ASSERT_EQ(0, CountNormalFiles(i)); + ASSERT_EQ(1, CountTrashFiles(i)); + } + TEST_SYNC_POINT("DeleteSchedulerTest::MultiDbPathDeletionsScheduled:1"); + delete_scheduler_->WaitForEmptyTrash(); + + // Verify dummy files eventually got deleted + for (size_t i = 0; i < kNumFiles; i++) { + ASSERT_EQ(0, CountNormalFiles(i)); + ASSERT_EQ(0, CountTrashFiles(i)); + } + + ASSERT_EQ(kNumFiles, stats_->getAndResetTickerCount(FILES_MARKED_TRASH)); + ASSERT_EQ(kNumFiles, + stats_->getAndResetTickerCount(FILES_DELETED_FROM_TRASH_QUEUE)); + ASSERT_EQ(0, stats_->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY)); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +// Same as the BasicRateLimiting test but delete files in multiple threads. +// 1- Create 100 dummy files +// 2- Delete the 100 dummy files using DeleteScheduler using 10 threads +// --- Hold DeleteScheduler::BackgroundEmptyTrash --- +// 3- Wait for DeleteScheduler to delete all files in queue +// 4- Verify that BackgroundEmptyTrash used to correct penlties for the files +// 5- Make sure that all created files were completely deleted +TEST_F(DeleteSchedulerTest, RateLimitingMultiThreaded) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"DeleteSchedulerTest::RateLimitingMultiThreaded:1", + "DeleteScheduler::BackgroundEmptyTrash"}, + }); + + std::vector penalties; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DeleteScheduler::BackgroundEmptyTrash:Wait", + [&](void* arg) { penalties.push_back(*(static_cast(arg))); }); + + int thread_cnt = 10; + int num_files = 10; // 10 files per thread + uint64_t file_size = 1024; // every file is 1 kb + + std::vector delete_kbs_per_sec = {512, 200, 100, 50, 25}; + for (size_t t = 0; t < delete_kbs_per_sec.size(); t++) { + penalties.clear(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + DestroyAndCreateDir(dummy_files_dirs_[0]); + rate_bytes_per_sec_ = delete_kbs_per_sec[t] * 1024; + NewDeleteScheduler(); + + // Create 100 dummy files, every file is 1 Kb + std::vector generated_files; + for (int i = 0; i < num_files * thread_cnt; i++) { + std::string file_name = "file" + std::to_string(i) + ".data"; + generated_files.push_back(NewDummyFile(file_name, file_size)); + } + + // Delete dummy files using 10 threads and measure time spent to empty trash + std::atomic thread_num(0); + std::vector threads; + std::function delete_thread = [&]() { + int idx = thread_num.fetch_add(1); + int range_start = idx * num_files; + int range_end = range_start + num_files; + for (int j = range_start; j < range_end; j++) { + ASSERT_OK(delete_scheduler_->DeleteFile(generated_files[j], "")); + } + }; + + for (int i = 0; i < thread_cnt; i++) { + threads.emplace_back(delete_thread); + } + + for (size_t i = 0; i < threads.size(); i++) { + threads[i].join(); + } + + uint64_t delete_start_time = env_->NowMicros(); + TEST_SYNC_POINT("DeleteSchedulerTest::RateLimitingMultiThreaded:1"); + delete_scheduler_->WaitForEmptyTrash(); + uint64_t time_spent_deleting = env_->NowMicros() - delete_start_time; + + auto bg_errors = delete_scheduler_->GetBackgroundErrors(); + ASSERT_EQ(bg_errors.size(), 0); + + uint64_t total_files_size = 0; + uint64_t expected_penlty = 0; + ASSERT_EQ(penalties.size(), num_files * thread_cnt); + for (int i = 0; i < num_files * thread_cnt; i++) { + total_files_size += file_size; + expected_penlty = ((total_files_size * 1000000) / rate_bytes_per_sec_); + ASSERT_EQ(expected_penlty, penalties[i]); + } + ASSERT_GT(time_spent_deleting, expected_penlty * 0.9); + + ASSERT_EQ(CountNormalFiles(), 0); + ASSERT_EQ(CountTrashFiles(), 0); + int total_num_files = num_files * thread_cnt; + ASSERT_EQ(total_num_files, + stats_->getAndResetTickerCount(FILES_MARKED_TRASH)); + ASSERT_EQ(total_num_files, + stats_->getAndResetTickerCount(FILES_DELETED_FROM_TRASH_QUEUE)); + ASSERT_EQ(0, stats_->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY)); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + } +} + +// Disable rate limiting by setting rate_bytes_per_sec_ to 0 and make sure +// that when DeleteScheduler delete a file it delete it immediately and don't +// move it to trash +TEST_F(DeleteSchedulerTest, DisableRateLimiting) { + int bg_delete_file = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DeleteScheduler::DeleteTrashFile:DeleteFile", + [&](void* /*arg*/) { bg_delete_file++; }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + rate_bytes_per_sec_ = 0; + NewDeleteScheduler(); + constexpr int num_files = 10; + + for (int i = 0; i < num_files; i++) { + // Every file we delete will be deleted immediately + std::string dummy_file = NewDummyFile("dummy.data"); + ASSERT_OK(delete_scheduler_->DeleteFile(dummy_file, "")); + ASSERT_TRUE(env_->FileExists(dummy_file).IsNotFound()); + ASSERT_EQ(CountNormalFiles(), 0); + ASSERT_EQ(CountTrashFiles(), 0); + } + + ASSERT_EQ(bg_delete_file, 0); + ASSERT_EQ(0, stats_->getAndResetTickerCount(FILES_MARKED_TRASH)); + ASSERT_EQ(0, stats_->getAndResetTickerCount(FILES_DELETED_FROM_TRASH_QUEUE)); + ASSERT_EQ(num_files, + stats_->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY)); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +// Testing that moving files to trash with the same name is not a problem +// 1- Create 10 files with the same name "conflict.data" +// 2- Delete the 10 files using DeleteScheduler +// 3- Make sure that trash directory contain 10 files ("conflict.data" x 10) +// --- Hold DeleteScheduler::BackgroundEmptyTrash --- +// 4- Make sure that files are deleted from trash +TEST_F(DeleteSchedulerTest, ConflictNames) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"DeleteSchedulerTest::ConflictNames:1", + "DeleteScheduler::BackgroundEmptyTrash"}, + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + rate_bytes_per_sec_ = 1024 * 1024; // 1 Mb/sec + NewDeleteScheduler(); + + // Create "conflict.data" and move it to trash 10 times + for (int i = 0; i < 10; i++) { + std::string dummy_file = NewDummyFile("conflict.data"); + ASSERT_OK(delete_scheduler_->DeleteFile(dummy_file, "")); + } + ASSERT_EQ(CountNormalFiles(), 0); + // 10 files ("conflict.data" x 10) in trash + ASSERT_EQ(CountTrashFiles(), 10); + + // Hold BackgroundEmptyTrash + TEST_SYNC_POINT("DeleteSchedulerTest::ConflictNames:1"); + delete_scheduler_->WaitForEmptyTrash(); + ASSERT_EQ(CountTrashFiles(), 0); + + auto bg_errors = delete_scheduler_->GetBackgroundErrors(); + ASSERT_EQ(bg_errors.size(), 0); + ASSERT_EQ(10, stats_->getAndResetTickerCount(FILES_MARKED_TRASH)); + ASSERT_EQ(10, stats_->getAndResetTickerCount(FILES_DELETED_FROM_TRASH_QUEUE)); + ASSERT_EQ(0, stats_->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY)); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +// 1- Create 10 dummy files +// 2- Delete the 10 files using DeleteScheduler (move them to trsah) +// 3- Delete the 10 files directly (using env_->DeleteFile) +// --- Hold DeleteScheduler::BackgroundEmptyTrash --- +// 4- Make sure that DeleteScheduler failed to delete the 10 files and +// reported 10 background errors +TEST_F(DeleteSchedulerTest, BackgroundError) { + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"DeleteSchedulerTest::BackgroundError:1", + "DeleteScheduler::BackgroundEmptyTrash"}, + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + rate_bytes_per_sec_ = 1024 * 1024; // 1 Mb/sec + NewDeleteScheduler(); + + // Generate 10 dummy files and move them to trash + for (int i = 0; i < 10; i++) { + std::string file_name = "data_" + std::to_string(i) + ".data"; + ASSERT_OK(delete_scheduler_->DeleteFile(NewDummyFile(file_name), "")); + } + ASSERT_EQ(CountNormalFiles(), 0); + ASSERT_EQ(CountTrashFiles(), 10); + + // Delete 10 files from trash, this will cause background errors in + // BackgroundEmptyTrash since we already deleted the files it was + // goind to delete + for (int i = 0; i < 10; i++) { + std::string file_name = "data_" + std::to_string(i) + ".data.trash"; + ASSERT_OK(env_->DeleteFile(dummy_files_dirs_[0] + "/" + file_name)); + } + + // Hold BackgroundEmptyTrash + TEST_SYNC_POINT("DeleteSchedulerTest::BackgroundError:1"); + delete_scheduler_->WaitForEmptyTrash(); + auto bg_errors = delete_scheduler_->GetBackgroundErrors(); + ASSERT_EQ(bg_errors.size(), 10); + for (const auto& it : bg_errors) { + ASSERT_TRUE(it.second.IsPathNotFound()); + } + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +// 1- Create kTestFileNum dummy files +// 2- Delete kTestFileNum dummy files using DeleteScheduler +// 3- Wait for DeleteScheduler to delete all files in queue +// 4- Make sure all files in trash directory were deleted +// 5- Repeat previous steps 5 times +TEST_F(DeleteSchedulerTest, StartBGEmptyTrashMultipleTimes) { + constexpr int kTestFileNum = 10; + std::atomic_int bg_delete_file = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DeleteScheduler::DeleteTrashFile:DeleteFile", + [&](void* /*arg*/) { bg_delete_file++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + rate_bytes_per_sec_ = 1024 * 1024; // 1 MB / sec + NewDeleteScheduler(); + + // If trash file is generated faster than deleting, delete_scheduler will + // delete it directly instead of waiting for background trash empty thread to + // clean it. Set the ratio higher to avoid that. + sst_file_mgr_->SetMaxTrashDBRatio(kTestFileNum + 1); + + // Move files to trash, wait for empty trash, start again + for (int run = 1; run <= 5; run++) { + // Generate kTestFileNum dummy files and move them to trash + for (int i = 0; i < kTestFileNum; i++) { + std::string file_name = "data_" + std::to_string(i) + ".data"; + ASSERT_OK(delete_scheduler_->DeleteFile(NewDummyFile(file_name), "")); + } + ASSERT_EQ(CountNormalFiles(), 0); + delete_scheduler_->WaitForEmptyTrash(); + ASSERT_EQ(bg_delete_file, kTestFileNum * run); + ASSERT_EQ(CountTrashFiles(), 0); + + auto bg_errors = delete_scheduler_->GetBackgroundErrors(); + ASSERT_EQ(bg_errors.size(), 0); + ASSERT_EQ(kTestFileNum, stats_->getAndResetTickerCount(FILES_MARKED_TRASH)); + ASSERT_EQ(kTestFileNum, + stats_->getAndResetTickerCount(FILES_DELETED_FROM_TRASH_QUEUE)); + ASSERT_EQ(0, stats_->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY)); + } + + ASSERT_EQ(bg_delete_file, 5 * kTestFileNum); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); +} + +TEST_F(DeleteSchedulerTest, DeletePartialFile) { + int bg_delete_file = 0; + int bg_fsync = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DeleteScheduler::DeleteTrashFile:DeleteFile", + [&](void*) { bg_delete_file++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DeleteScheduler::DeleteTrashFile:Fsync", [&](void*) { bg_fsync++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + rate_bytes_per_sec_ = 1024 * 1024; // 1 MB / sec + NewDeleteScheduler(); + + // Should delete in 4 batch + ASSERT_OK( + delete_scheduler_->DeleteFile(NewDummyFile("data_1", 500 * 1024), "")); + ASSERT_OK( + delete_scheduler_->DeleteFile(NewDummyFile("data_2", 100 * 1024), "")); + // Should delete in 2 batch + ASSERT_OK( + delete_scheduler_->DeleteFile(NewDummyFile("data_2", 200 * 1024), "")); + + delete_scheduler_->WaitForEmptyTrash(); + + auto bg_errors = delete_scheduler_->GetBackgroundErrors(); + ASSERT_EQ(bg_errors.size(), 0); + ASSERT_EQ(7, bg_delete_file); + ASSERT_EQ(4, bg_fsync); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); +} + +#ifdef OS_LINUX +TEST_F(DeleteSchedulerTest, NoPartialDeleteWithLink) { + int bg_delete_file = 0; + int bg_fsync = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DeleteScheduler::DeleteTrashFile:DeleteFile", + [&](void*) { bg_delete_file++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DeleteScheduler::DeleteTrashFile:Fsync", [&](void*) { bg_fsync++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + rate_bytes_per_sec_ = 1024 * 1024; // 1 MB / sec + NewDeleteScheduler(); + + std::string file1 = NewDummyFile("data_1", 500 * 1024); + std::string file2 = NewDummyFile("data_2", 100 * 1024); + + ASSERT_OK(env_->LinkFile(file1, dummy_files_dirs_[0] + "/data_1b")); + ASSERT_OK(env_->LinkFile(file2, dummy_files_dirs_[0] + "/data_2b")); + + // Should delete in 4 batch if there is no hardlink + ASSERT_OK(delete_scheduler_->DeleteFile(file1, "")); + ASSERT_OK(delete_scheduler_->DeleteFile(file2, "")); + + delete_scheduler_->WaitForEmptyTrash(); + + auto bg_errors = delete_scheduler_->GetBackgroundErrors(); + ASSERT_EQ(bg_errors.size(), 0); + ASSERT_EQ(2, bg_delete_file); + ASSERT_EQ(0, bg_fsync); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); +} +#endif + +// 1- Create a DeleteScheduler with very slow rate limit (1 Byte / sec) +// 2- Delete 100 files using DeleteScheduler +// 3- Delete the DeleteScheduler (call the destructor while queue is not empty) +// 4- Make sure that not all files were deleted from trash and that +// DeleteScheduler background thread did not delete all files +TEST_F(DeleteSchedulerTest, DestructorWithNonEmptyQueue) { + int bg_delete_file = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DeleteScheduler::DeleteTrashFile:DeleteFile", + [&](void* /*arg*/) { bg_delete_file++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + rate_bytes_per_sec_ = 1; // 1 Byte / sec + NewDeleteScheduler(); + + for (int i = 0; i < 100; i++) { + std::string file_name = "data_" + std::to_string(i) + ".data"; + ASSERT_OK(delete_scheduler_->DeleteFile(NewDummyFile(file_name), "")); + } + + // Deleting 100 files will need >28 hours to delete + // we will delete the DeleteScheduler while delete queue is not empty + sst_file_mgr_.reset(); + + ASSERT_LT(bg_delete_file, 100); + ASSERT_GT(CountTrashFiles(), 0); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DeleteSchedulerTest, DISABLED_DynamicRateLimiting1) { + std::vector penalties; + int bg_delete_file = 0; + int fg_delete_file = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DeleteScheduler::DeleteTrashFile:DeleteFile", + [&](void* /*arg*/) { bg_delete_file++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DeleteScheduler::DeleteFile", [&](void* /*arg*/) { fg_delete_file++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DeleteScheduler::BackgroundEmptyTrash:Wait", + [&](void* arg) { penalties.push_back(*(static_cast(arg))); }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ + {"DeleteSchedulerTest::DynamicRateLimiting1:1", + "DeleteScheduler::BackgroundEmptyTrash"}, + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + rate_bytes_per_sec_ = 0; // Disable rate limiting initially + NewDeleteScheduler(); + + int num_files = 10; // 10 files + uint64_t file_size = 1024; // every file is 1 kb + + std::vector delete_kbs_per_sec = {512, 200, 0, 100, 50, -2, 25}; + for (size_t t = 0; t < delete_kbs_per_sec.size(); t++) { + penalties.clear(); + bg_delete_file = 0; + fg_delete_file = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + DestroyAndCreateDir(dummy_files_dirs_[0]); + rate_bytes_per_sec_ = delete_kbs_per_sec[t] * 1024; + delete_scheduler_->SetRateBytesPerSecond(rate_bytes_per_sec_); + + // Create 100 dummy files, every file is 1 Kb + std::vector generated_files; + for (int i = 0; i < num_files; i++) { + std::string file_name = "file" + std::to_string(i) + ".data"; + generated_files.push_back(NewDummyFile(file_name, file_size)); + } + + // Delete dummy files and measure time spent to empty trash + for (int i = 0; i < num_files; i++) { + ASSERT_OK(delete_scheduler_->DeleteFile(generated_files[i], "")); + } + ASSERT_EQ(CountNormalFiles(), 0); + + if (rate_bytes_per_sec_ > 0) { + uint64_t delete_start_time = env_->NowMicros(); + TEST_SYNC_POINT("DeleteSchedulerTest::DynamicRateLimiting1:1"); + delete_scheduler_->WaitForEmptyTrash(); + uint64_t time_spent_deleting = env_->NowMicros() - delete_start_time; + + auto bg_errors = delete_scheduler_->GetBackgroundErrors(); + ASSERT_EQ(bg_errors.size(), 0); + + uint64_t total_files_size = 0; + uint64_t expected_penlty = 0; + ASSERT_EQ(penalties.size(), num_files); + for (int i = 0; i < num_files; i++) { + total_files_size += file_size; + expected_penlty = ((total_files_size * 1000000) / rate_bytes_per_sec_); + ASSERT_EQ(expected_penlty, penalties[i]); + } + ASSERT_GT(time_spent_deleting, expected_penlty * 0.9); + ASSERT_EQ(bg_delete_file, num_files); + ASSERT_EQ(fg_delete_file, 0); + } else { + ASSERT_EQ(penalties.size(), 0); + ASSERT_EQ(bg_delete_file, 0); + ASSERT_EQ(fg_delete_file, num_files); + } + + ASSERT_EQ(CountTrashFiles(), 0); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + } +} + +TEST_F(DeleteSchedulerTest, ImmediateDeleteOn25PercDBSize) { + int bg_delete_file = 0; + int fg_delete_file = 0; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DeleteScheduler::DeleteTrashFile:DeleteFile", + [&](void* /*arg*/) { bg_delete_file++; }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "DeleteScheduler::DeleteFile", [&](void* /*arg*/) { fg_delete_file++; }); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + int num_files = 100; // 100 files + uint64_t file_size = 1024 * 10; // 100 KB as a file size + rate_bytes_per_sec_ = 1; // 1 byte per sec (very slow trash delete) + + NewDeleteScheduler(); + delete_scheduler_->SetMaxTrashDBRatio(0.25); + + std::vector generated_files; + for (int i = 0; i < num_files; i++) { + std::string file_name = "file" + std::to_string(i) + ".data"; + generated_files.push_back(NewDummyFile(file_name, file_size)); + } + + for (std::string& file_name : generated_files) { + ASSERT_OK(delete_scheduler_->DeleteFile(file_name, "")); + } + + // When we end up with 26 files in trash we will start + // deleting new files immediately + ASSERT_EQ(fg_delete_file, 74); + ASSERT_EQ(26, stats_->getAndResetTickerCount(FILES_MARKED_TRASH)); + ASSERT_EQ(74, stats_->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY)); + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +TEST_F(DeleteSchedulerTest, IsTrashCheck) { + // Trash files + ASSERT_TRUE(DeleteScheduler::IsTrashFile("x.trash")); + ASSERT_TRUE(DeleteScheduler::IsTrashFile(".trash")); + ASSERT_TRUE(DeleteScheduler::IsTrashFile("abc.sst.trash")); + ASSERT_TRUE(DeleteScheduler::IsTrashFile("/a/b/c/abc..sst.trash")); + ASSERT_TRUE(DeleteScheduler::IsTrashFile("log.trash")); + ASSERT_TRUE(DeleteScheduler::IsTrashFile("^^^^^.log.trash")); + ASSERT_TRUE(DeleteScheduler::IsTrashFile("abc.t.trash")); + + // Not trash files + ASSERT_FALSE(DeleteScheduler::IsTrashFile("abc.sst")); + ASSERT_FALSE(DeleteScheduler::IsTrashFile("abc.txt")); + ASSERT_FALSE(DeleteScheduler::IsTrashFile("/a/b/c/abc.sst")); + ASSERT_FALSE(DeleteScheduler::IsTrashFile("/a/b/c/abc.sstrash")); + ASSERT_FALSE(DeleteScheduler::IsTrashFile("^^^^^.trashh")); + ASSERT_FALSE(DeleteScheduler::IsTrashFile("abc.ttrash")); + ASSERT_FALSE(DeleteScheduler::IsTrashFile(".ttrash")); + ASSERT_FALSE(DeleteScheduler::IsTrashFile("abc.trashx")); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/librocksdb-sys/rocksdb/file/file_prefetch_buffer.cc b/librocksdb-sys/rocksdb/file/file_prefetch_buffer.cc new file mode 100644 index 0000000..59fbf12 --- /dev/null +++ b/librocksdb-sys/rocksdb/file/file_prefetch_buffer.cc @@ -0,0 +1,956 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "file/file_prefetch_buffer.h" + +#include +#include + +#include "file/random_access_file_reader.h" +#include "monitoring/histogram.h" +#include "monitoring/iostats_context_imp.h" +#include "port/port.h" +#include "test_util/sync_point.h" +#include "util/random.h" +#include "util/rate_limiter_impl.h" + +namespace ROCKSDB_NAMESPACE { + +void FilePrefetchBuffer::CalculateOffsetAndLen(size_t alignment, + uint64_t offset, + size_t roundup_len, + uint32_t index, bool refit_tail, + uint64_t& chunk_len) { + uint64_t chunk_offset_in_buffer = 0; + bool copy_data_to_new_buffer = false; + // Check if requested bytes are in the existing buffer_. + // If only a few bytes exist -- reuse them & read only what is really needed. + // This is typically the case of incremental reading of data. + // If no bytes exist in buffer -- full pread. + if (DoesBufferContainData(index) && IsOffsetInBuffer(offset, index)) { + // Only a few requested bytes are in the buffer. memmove those chunk of + // bytes to the beginning, and memcpy them back into the new buffer if a + // new buffer is created. + chunk_offset_in_buffer = Rounddown( + static_cast(offset - bufs_[index].offset_), alignment); + chunk_len = static_cast(bufs_[index].buffer_.CurrentSize()) - + chunk_offset_in_buffer; + assert(chunk_offset_in_buffer % alignment == 0); + assert(chunk_len % alignment == 0); + assert(chunk_offset_in_buffer + chunk_len <= + bufs_[index].offset_ + bufs_[index].buffer_.CurrentSize()); + if (chunk_len > 0) { + copy_data_to_new_buffer = true; + } else { + // this reset is not necessary, but just to be safe. + chunk_offset_in_buffer = 0; + } + } + + // Create a new buffer only if current capacity is not sufficient, and memcopy + // bytes from old buffer if needed (i.e., if chunk_len is greater than 0). + if (bufs_[index].buffer_.Capacity() < roundup_len) { + bufs_[index].buffer_.Alignment(alignment); + bufs_[index].buffer_.AllocateNewBuffer( + static_cast(roundup_len), copy_data_to_new_buffer, + chunk_offset_in_buffer, static_cast(chunk_len)); + } else if (chunk_len > 0 && refit_tail) { + // New buffer not needed. But memmove bytes from tail to the beginning since + // chunk_len is greater than 0. + bufs_[index].buffer_.RefitTail(static_cast(chunk_offset_in_buffer), + static_cast(chunk_len)); + } else if (chunk_len > 0) { + // For async prefetching, it doesn't call RefitTail with chunk_len > 0. + // Allocate new buffer if needed because aligned buffer calculate remaining + // buffer as capacity_ - cursize_ which might not be the case in this as we + // are not refitting. + // TODO akanksha: Update the condition when asynchronous prefetching is + // stable. + bufs_[index].buffer_.Alignment(alignment); + bufs_[index].buffer_.AllocateNewBuffer( + static_cast(roundup_len), copy_data_to_new_buffer, + chunk_offset_in_buffer, static_cast(chunk_len)); + } +} + +Status FilePrefetchBuffer::Read(const IOOptions& opts, + RandomAccessFileReader* reader, + Env::IOPriority rate_limiter_priority, + uint64_t read_len, uint64_t chunk_len, + uint64_t rounddown_start, uint32_t index) { + Slice result; + Status s = reader->Read(opts, rounddown_start + chunk_len, read_len, &result, + bufs_[index].buffer_.BufferStart() + chunk_len, + /*aligned_buf=*/nullptr, rate_limiter_priority); +#ifndef NDEBUG + if (result.size() < read_len) { + // Fake an IO error to force db_stress fault injection to ignore + // truncated read errors + IGNORE_STATUS_IF_ERROR(Status::IOError()); + } +#endif + if (!s.ok()) { + return s; + } + + // Update the buffer offset and size. + bufs_[index].offset_ = rounddown_start; + bufs_[index].buffer_.Size(static_cast(chunk_len) + result.size()); + return s; +} + +Status FilePrefetchBuffer::ReadAsync(const IOOptions& opts, + RandomAccessFileReader* reader, + uint64_t read_len, + uint64_t rounddown_start, uint32_t index) { + TEST_SYNC_POINT("FilePrefetchBuffer::ReadAsync"); + // callback for async read request. + auto fp = std::bind(&FilePrefetchBuffer::PrefetchAsyncCallback, this, + std::placeholders::_1, std::placeholders::_2); + FSReadRequest req; + Slice result; + req.len = read_len; + req.offset = rounddown_start; + req.result = result; + req.scratch = bufs_[index].buffer_.BufferStart(); + bufs_[index].async_req_len_ = req.len; + + Status s = + reader->ReadAsync(req, opts, fp, &(bufs_[index].pos_), + &(bufs_[index].io_handle_), &(bufs_[index].del_fn_), + /*aligned_buf=*/nullptr); + req.status.PermitUncheckedError(); + if (s.ok()) { + bufs_[index].async_read_in_progress_ = true; + } + return s; +} + +Status FilePrefetchBuffer::Prefetch(const IOOptions& opts, + RandomAccessFileReader* reader, + uint64_t offset, size_t n, + Env::IOPriority rate_limiter_priority) { + if (!enable_ || reader == nullptr) { + return Status::OK(); + } + TEST_SYNC_POINT("FilePrefetchBuffer::Prefetch:Start"); + + if (offset + n <= bufs_[curr_].offset_ + bufs_[curr_].buffer_.CurrentSize()) { + // All requested bytes are already in the curr_ buffer. So no need to Read + // again. + return Status::OK(); + } + + size_t alignment = reader->file()->GetRequiredBufferAlignment(); + size_t offset_ = static_cast(offset); + uint64_t rounddown_offset = Rounddown(offset_, alignment); + uint64_t roundup_end = Roundup(offset_ + n, alignment); + uint64_t roundup_len = roundup_end - rounddown_offset; + assert(roundup_len >= alignment); + assert(roundup_len % alignment == 0); + + uint64_t chunk_len = 0; + CalculateOffsetAndLen(alignment, offset, roundup_len, curr_, + true /*refit_tail*/, chunk_len); + size_t read_len = static_cast(roundup_len - chunk_len); + + Status s = Read(opts, reader, rate_limiter_priority, read_len, chunk_len, + rounddown_offset, curr_); + if (usage_ == FilePrefetchBufferUsage::kTableOpenPrefetchTail && s.ok()) { + RecordInHistogram(stats_, TABLE_OPEN_PREFETCH_TAIL_READ_BYTES, read_len); + } + return s; +} + +// Copy data from src to third buffer. +void FilePrefetchBuffer::CopyDataToBuffer(uint32_t src, uint64_t& offset, + size_t& length) { + if (length == 0) { + return; + } + uint64_t copy_offset = (offset - bufs_[src].offset_); + size_t copy_len = 0; + if (IsDataBlockInBuffer(offset, length, src)) { + // All the bytes are in src. + copy_len = length; + } else { + copy_len = bufs_[src].buffer_.CurrentSize() - copy_offset; + } + + memcpy(bufs_[2].buffer_.BufferStart() + bufs_[2].buffer_.CurrentSize(), + bufs_[src].buffer_.BufferStart() + copy_offset, copy_len); + + bufs_[2].buffer_.Size(bufs_[2].buffer_.CurrentSize() + copy_len); + + // Update offset and length. + offset += copy_len; + length -= copy_len; + + // length > 0 indicates it has consumed all data from the src buffer and it + // still needs to read more other buffer. + if (length > 0) { + bufs_[src].buffer_.Clear(); + } +} + +// Clear the buffers if it contains outdated data. Outdated data can be +// because previous sequential reads were read from the cache instead of these +// buffer. In that case outdated IOs should be aborted. +void FilePrefetchBuffer::AbortIOIfNeeded(uint64_t offset) { + uint32_t second = curr_ ^ 1; + std::vector handles; + autovector buf_pos; + if (IsBufferOutdatedWithAsyncProgress(offset, curr_)) { + handles.emplace_back(bufs_[curr_].io_handle_); + buf_pos.emplace_back(curr_); + } + if (IsBufferOutdatedWithAsyncProgress(offset, second)) { + handles.emplace_back(bufs_[second].io_handle_); + buf_pos.emplace_back(second); + } + if (!handles.empty()) { + StopWatch sw(clock_, stats_, ASYNC_PREFETCH_ABORT_MICROS); + Status s = fs_->AbortIO(handles); + assert(s.ok()); + } + + for (auto& pos : buf_pos) { + // Release io_handle. + DestroyAndClearIOHandle(pos); + } + + if (bufs_[second].io_handle_ == nullptr) { + bufs_[second].async_read_in_progress_ = false; + } + + if (bufs_[curr_].io_handle_ == nullptr) { + bufs_[curr_].async_read_in_progress_ = false; + } +} + +void FilePrefetchBuffer::AbortAllIOs() { + uint32_t second = curr_ ^ 1; + std::vector handles; + for (uint32_t i = 0; i < 2; i++) { + if (bufs_[i].async_read_in_progress_ && bufs_[i].io_handle_ != nullptr) { + handles.emplace_back(bufs_[i].io_handle_); + } + } + if (!handles.empty()) { + StopWatch sw(clock_, stats_, ASYNC_PREFETCH_ABORT_MICROS); + Status s = fs_->AbortIO(handles); + assert(s.ok()); + } + + // Release io_handles. + if (bufs_[curr_].io_handle_ != nullptr && bufs_[curr_].del_fn_ != nullptr) { + DestroyAndClearIOHandle(curr_); + } else { + bufs_[curr_].async_read_in_progress_ = false; + } + + if (bufs_[second].io_handle_ != nullptr && bufs_[second].del_fn_ != nullptr) { + DestroyAndClearIOHandle(second); + } else { + bufs_[second].async_read_in_progress_ = false; + } +} + +// Clear the buffers if it contains outdated data. Outdated data can be +// because previous sequential reads were read from the cache instead of these +// buffer. +void FilePrefetchBuffer::UpdateBuffersIfNeeded(uint64_t offset) { + uint32_t second = curr_ ^ 1; + if (IsBufferOutdated(offset, curr_)) { + bufs_[curr_].buffer_.Clear(); + } + if (IsBufferOutdated(offset, second)) { + bufs_[second].buffer_.Clear(); + } + + { + // In case buffers do not align, reset second buffer. This can happen in + // case readahead_size is set. + if (!bufs_[second].async_read_in_progress_ && + !bufs_[curr_].async_read_in_progress_) { + if (DoesBufferContainData(curr_)) { + if (bufs_[curr_].offset_ + bufs_[curr_].buffer_.CurrentSize() != + bufs_[second].offset_) { + bufs_[second].buffer_.Clear(); + } + } else { + if (!IsOffsetInBuffer(offset, second)) { + bufs_[second].buffer_.Clear(); + } + } + } + } + + // If data starts from second buffer, make it curr_. Second buffer can be + // either partial filled, full or async read is in progress. + if (bufs_[second].async_read_in_progress_) { + if (IsOffsetInBufferWithAsyncProgress(offset, second)) { + curr_ = curr_ ^ 1; + } + } else { + if (DoesBufferContainData(second) && IsOffsetInBuffer(offset, second)) { + assert(bufs_[curr_].async_read_in_progress_ || + bufs_[curr_].buffer_.CurrentSize() == 0); + curr_ = curr_ ^ 1; + } + } +} + +void FilePrefetchBuffer::PollAndUpdateBuffersIfNeeded(uint64_t offset) { + if (bufs_[curr_].async_read_in_progress_ && fs_ != nullptr) { + if (bufs_[curr_].io_handle_ != nullptr) { + // Wait for prefetch data to complete. + // No mutex is needed as async_read_in_progress behaves as mutex and is + // updated by main thread only. + std::vector handles; + handles.emplace_back(bufs_[curr_].io_handle_); + StopWatch sw(clock_, stats_, POLL_WAIT_MICROS); + fs_->Poll(handles, 1).PermitUncheckedError(); + } + + // Reset and Release io_handle after the Poll API as request has been + // completed. + DestroyAndClearIOHandle(curr_); + } + UpdateBuffersIfNeeded(offset); +} + +Status FilePrefetchBuffer::HandleOverlappingData( + const IOOptions& opts, RandomAccessFileReader* reader, uint64_t offset, + size_t length, size_t readahead_size, + Env::IOPriority /*rate_limiter_priority*/, bool& copy_to_third_buffer, + uint64_t& tmp_offset, size_t& tmp_length) { + Status s; + size_t alignment = reader->file()->GetRequiredBufferAlignment(); + uint32_t second; + + // Check if the first buffer has the required offset and the async read is + // still in progress. This should only happen if a prefetch was initiated + // by Seek, but the next access is at another offset. + if (bufs_[curr_].async_read_in_progress_ && + IsOffsetInBufferWithAsyncProgress(offset, curr_)) { + PollAndUpdateBuffersIfNeeded(offset); + } + second = curr_ ^ 1; + + // If data is overlapping over two buffers, copy the data from curr_ and + // call ReadAsync on curr_. + if (!bufs_[curr_].async_read_in_progress_ && DoesBufferContainData(curr_) && + IsOffsetInBuffer(offset, curr_) && + (/*Data extends over curr_ buffer and second buffer either has data or in + process of population=*/ + (offset + length > bufs_[second].offset_) && + (bufs_[second].async_read_in_progress_ || + DoesBufferContainData(second)))) { + // Allocate new buffer to third buffer; + bufs_[2].buffer_.Clear(); + bufs_[2].buffer_.Alignment(alignment); + bufs_[2].buffer_.AllocateNewBuffer(length); + bufs_[2].offset_ = offset; + copy_to_third_buffer = true; + + CopyDataToBuffer(curr_, tmp_offset, tmp_length); + + // Call async prefetching on curr_ since data has been consumed in curr_ + // only if data lies within second buffer. + size_t second_size = bufs_[second].async_read_in_progress_ + ? bufs_[second].async_req_len_ + : bufs_[second].buffer_.CurrentSize(); + if (tmp_offset + tmp_length <= bufs_[second].offset_ + second_size) { + uint64_t rounddown_start = bufs_[second].offset_ + second_size; + uint64_t roundup_end = + Roundup(rounddown_start + readahead_size, alignment); + uint64_t roundup_len = roundup_end - rounddown_start; + uint64_t chunk_len = 0; + CalculateOffsetAndLen(alignment, rounddown_start, roundup_len, curr_, + false, chunk_len); + assert(chunk_len == 0); + assert(roundup_len >= chunk_len); + + bufs_[curr_].offset_ = rounddown_start; + uint64_t read_len = static_cast(roundup_len - chunk_len); + s = ReadAsync(opts, reader, read_len, rounddown_start, curr_); + if (!s.ok()) { + DestroyAndClearIOHandle(curr_); + bufs_[curr_].buffer_.Clear(); + return s; + } + } + curr_ = curr_ ^ 1; + } + return s; +} +// If async_io is enabled in case of sequential reads, PrefetchAsyncInternal is +// called. When buffers are switched, we clear the curr_ buffer as we assume the +// data has been consumed because of sequential reads. +// Data in buffers will always be sequential with curr_ following second and +// not vice versa. +// +// Scenarios for prefetching asynchronously: +// Case1: If both buffers are empty, prefetch n + readahead_size_/2 bytes +// synchronously in curr_ and prefetch readahead_size_/2 async in second +// buffer. +// Case2: If second buffer has partial or full data, make it current and +// prefetch readahead_size_/2 async in second buffer. In case of +// partial data, prefetch remaining bytes from size n synchronously to +// fulfill the requested bytes request. +// Case3: If curr_ has partial data, prefetch remaining bytes from size n +// synchronously in curr_ to fulfill the requested bytes request and +// prefetch readahead_size_/2 bytes async in second buffer. +// Case4: (Special case) If data is in both buffers, copy requested data from +// curr_, send async request on curr_, wait for poll to fill second +// buffer (if any), and copy remaining data from second buffer to third +// buffer. +Status FilePrefetchBuffer::PrefetchAsyncInternal( + const IOOptions& opts, RandomAccessFileReader* reader, uint64_t offset, + size_t length, size_t readahead_size, Env::IOPriority rate_limiter_priority, + bool& copy_to_third_buffer) { + if (!enable_) { + return Status::OK(); + } + + TEST_SYNC_POINT("FilePrefetchBuffer::PrefetchAsyncInternal:Start"); + + size_t alignment = reader->file()->GetRequiredBufferAlignment(); + Status s; + uint64_t tmp_offset = offset; + size_t tmp_length = length; + + // 1. Abort IO and swap buffers if needed to point curr_ to first buffer with + // data. + if (!explicit_prefetch_submitted_) { + AbortIOIfNeeded(offset); + } + UpdateBuffersIfNeeded(offset); + + // 2. Handle overlapping data over two buffers. If data is overlapping then + // during this call: + // - data from curr_ is copied into third buffer, + // - curr_ is send for async prefetching of further data if second buffer + // contains remaining requested data or in progress for async prefetch, + // - switch buffers and curr_ now points to second buffer to copy remaining + // data. + s = HandleOverlappingData(opts, reader, offset, length, readahead_size, + rate_limiter_priority, copy_to_third_buffer, + tmp_offset, tmp_length); + if (!s.ok()) { + return s; + } + + // 3. Call Poll only if data is needed for the second buffer. + // - Return if whole data is in curr_ and second buffer is in progress or + // already full. + // - If second buffer is empty, it will go for ReadAsync for second buffer. + if (!bufs_[curr_].async_read_in_progress_ && DoesBufferContainData(curr_) && + IsDataBlockInBuffer(offset, length, curr_)) { + // Whole data is in curr_. + UpdateBuffersIfNeeded(offset); + if (!IsSecondBuffEligibleForPrefetching()) { + return s; + } + } else { + // After poll request, curr_ might be empty because of IOError in + // callback while reading or may contain required data. + PollAndUpdateBuffersIfNeeded(offset); + } + + if (copy_to_third_buffer) { + offset = tmp_offset; + length = tmp_length; + } + + // 4. After polling and swapping buffers, if all the requested bytes are in + // curr_, it will only go for async prefetching. + // copy_to_third_buffer is a special case so it will be handled separately. + if (!copy_to_third_buffer && DoesBufferContainData(curr_) && + IsDataBlockInBuffer(offset, length, curr_)) { + offset += length; + length = 0; + + // Since async request was submitted directly by calling PrefetchAsync in + // last call, we don't need to prefetch further as this call is to poll + // the data submitted in previous call. + if (explicit_prefetch_submitted_) { + return s; + } + if (!IsSecondBuffEligibleForPrefetching()) { + return s; + } + } + + uint32_t second = curr_ ^ 1; + assert(!bufs_[curr_].async_read_in_progress_); + + // In case because of some IOError curr_ got empty, abort IO for second as + // well. Otherwise data might not align if more data needs to be read in curr_ + // which might overlap with second buffer. + if (!DoesBufferContainData(curr_) && bufs_[second].async_read_in_progress_) { + if (bufs_[second].io_handle_ != nullptr) { + std::vector handles; + handles.emplace_back(bufs_[second].io_handle_); + { + StopWatch sw(clock_, stats_, ASYNC_PREFETCH_ABORT_MICROS); + Status status = fs_->AbortIO(handles); + assert(status.ok()); + } + } + DestroyAndClearIOHandle(second); + bufs_[second].buffer_.Clear(); + } + + // 5. Data is overlapping i.e. some of the data has been copied to third + // buffer and remaining will be updated below. + if (copy_to_third_buffer && DoesBufferContainData(curr_)) { + CopyDataToBuffer(curr_, offset, length); + + // Length == 0: All the requested data has been copied to third buffer and + // it has already gone for async prefetching. It can return without doing + // anything further. + // Length > 0: More data needs to be consumed so it will continue async + // and sync prefetching and copy the remaining data to third buffer in the + // end. + if (length == 0) { + return s; + } + } + + // 6. Go for ReadAsync and Read (if needed). + size_t prefetch_size = length + readahead_size; + size_t _offset = static_cast(offset); + + // offset and size alignment for curr_ buffer with synchronous prefetching + uint64_t rounddown_start1 = Rounddown(_offset, alignment); + uint64_t roundup_end1 = Roundup(_offset + prefetch_size, alignment); + uint64_t roundup_len1 = roundup_end1 - rounddown_start1; + assert(roundup_len1 >= alignment); + assert(roundup_len1 % alignment == 0); + uint64_t chunk_len1 = 0; + uint64_t read_len1 = 0; + + assert(!bufs_[second].async_read_in_progress_ && + !DoesBufferContainData(second)); + + // For length == 0, skip the synchronous prefetching. read_len1 will be 0. + if (length > 0) { + CalculateOffsetAndLen(alignment, offset, roundup_len1, curr_, + false /*refit_tail*/, chunk_len1); + assert(roundup_len1 >= chunk_len1); + read_len1 = static_cast(roundup_len1 - chunk_len1); + } + { + // offset and size alignment for second buffer for asynchronous + // prefetching + uint64_t rounddown_start2 = roundup_end1; + uint64_t roundup_end2 = + Roundup(rounddown_start2 + readahead_size, alignment); + + // For length == 0, do the asynchronous prefetching in second instead of + // synchronous prefetching in curr_. + if (length == 0) { + rounddown_start2 = + bufs_[curr_].offset_ + bufs_[curr_].buffer_.CurrentSize(); + roundup_end2 = Roundup(rounddown_start2 + prefetch_size, alignment); + } + + uint64_t roundup_len2 = roundup_end2 - rounddown_start2; + uint64_t chunk_len2 = 0; + CalculateOffsetAndLen(alignment, rounddown_start2, roundup_len2, second, + false /*refit_tail*/, chunk_len2); + assert(chunk_len2 == 0); + // Update the buffer offset. + bufs_[second].offset_ = rounddown_start2; + assert(roundup_len2 >= chunk_len2); + uint64_t read_len2 = static_cast(roundup_len2 - chunk_len2); + s = ReadAsync(opts, reader, read_len2, rounddown_start2, second); + if (!s.ok()) { + DestroyAndClearIOHandle(second); + bufs_[second].buffer_.Clear(); + return s; + } + } + + if (read_len1 > 0) { + s = Read(opts, reader, rate_limiter_priority, read_len1, chunk_len1, + rounddown_start1, curr_); + if (!s.ok()) { + if (bufs_[second].io_handle_ != nullptr) { + std::vector handles; + handles.emplace_back(bufs_[second].io_handle_); + { + StopWatch sw(clock_, stats_, ASYNC_PREFETCH_ABORT_MICROS); + Status status = fs_->AbortIO(handles); + assert(status.ok()); + } + } + DestroyAndClearIOHandle(second); + bufs_[second].buffer_.Clear(); + bufs_[curr_].buffer_.Clear(); + return s; + } + } + // Copy remaining requested bytes to third_buffer. + if (copy_to_third_buffer && length > 0) { + CopyDataToBuffer(curr_, offset, length); + } + return s; +} + +bool FilePrefetchBuffer::TryReadFromCache(const IOOptions& opts, + RandomAccessFileReader* reader, + uint64_t offset, size_t n, + Slice* result, Status* status, + Env::IOPriority rate_limiter_priority, + bool for_compaction /* = false */) { + bool ret = TryReadFromCacheUntracked(opts, reader, offset, n, result, status, + rate_limiter_priority, for_compaction); + if (usage_ == FilePrefetchBufferUsage::kTableOpenPrefetchTail && enable_) { + if (ret) { + RecordTick(stats_, TABLE_OPEN_PREFETCH_TAIL_HIT); + } else { + RecordTick(stats_, TABLE_OPEN_PREFETCH_TAIL_MISS); + } + } + return ret; +} + +bool FilePrefetchBuffer::TryReadFromCacheUntracked( + const IOOptions& opts, RandomAccessFileReader* reader, uint64_t offset, + size_t n, Slice* result, Status* status, + Env::IOPriority rate_limiter_priority, bool for_compaction /* = false */) { + if (track_min_offset_ && offset < min_offset_read_) { + min_offset_read_ = static_cast(offset); + } + if (!enable_ || (offset < bufs_[curr_].offset_)) { + return false; + } + + // If the buffer contains only a few of the requested bytes: + // If readahead is enabled: prefetch the remaining bytes + readahead bytes + // and satisfy the request. + // If readahead is not enabled: return false. + TEST_SYNC_POINT_CALLBACK("FilePrefetchBuffer::TryReadFromCache", + &readahead_size_); + if (offset + n > bufs_[curr_].offset_ + bufs_[curr_].buffer_.CurrentSize()) { + if (readahead_size_ > 0) { + Status s; + assert(reader != nullptr); + assert(max_readahead_size_ >= readahead_size_); + if (for_compaction) { + s = Prefetch(opts, reader, offset, std::max(n, readahead_size_), + rate_limiter_priority); + } else { + if (implicit_auto_readahead_) { + if (!IsEligibleForPrefetch(offset, n)) { + // Ignore status as Prefetch is not called. + s.PermitUncheckedError(); + return false; + } + } + s = Prefetch(opts, reader, offset, n + readahead_size_, + rate_limiter_priority); + } + if (!s.ok()) { + if (status) { + *status = s; + } +#ifndef NDEBUG + IGNORE_STATUS_IF_ERROR(s); +#endif + return false; + } + readahead_size_ = std::min(max_readahead_size_, readahead_size_ * 2); + } else { + return false; + } + } + UpdateReadPattern(offset, n, false /*decrease_readaheadsize*/); + + uint64_t offset_in_buffer = offset - bufs_[curr_].offset_; + *result = Slice(bufs_[curr_].buffer_.BufferStart() + offset_in_buffer, n); + return true; +} + +bool FilePrefetchBuffer::TryReadFromCacheAsync( + const IOOptions& opts, RandomAccessFileReader* reader, uint64_t offset, + size_t n, Slice* result, Status* status, + Env::IOPriority rate_limiter_priority) { + bool ret = TryReadFromCacheAsyncUntracked(opts, reader, offset, n, result, + status, rate_limiter_priority); + if (usage_ == FilePrefetchBufferUsage::kTableOpenPrefetchTail && enable_) { + if (ret) { + RecordTick(stats_, TABLE_OPEN_PREFETCH_TAIL_HIT); + } else { + RecordTick(stats_, TABLE_OPEN_PREFETCH_TAIL_MISS); + } + } + return ret; +} + +bool FilePrefetchBuffer::TryReadFromCacheAsyncUntracked( + const IOOptions& opts, RandomAccessFileReader* reader, uint64_t offset, + size_t n, Slice* result, Status* status, + Env::IOPriority rate_limiter_priority) { + if (track_min_offset_ && offset < min_offset_read_) { + min_offset_read_ = static_cast(offset); + } + + if (!enable_) { + return false; + } + + if (explicit_prefetch_submitted_) { + // explicit_prefetch_submitted_ is special case where it expects request + // submitted in PrefetchAsync should match with this request. Otherwise + // buffers will be outdated. + // Random offset called. So abort the IOs. + if (prev_offset_ != offset) { + AbortAllIOs(); + bufs_[curr_].buffer_.Clear(); + bufs_[curr_ ^ 1].buffer_.Clear(); + explicit_prefetch_submitted_ = false; + return false; + } + } + + if (!explicit_prefetch_submitted_ && offset < bufs_[curr_].offset_) { + return false; + } + + bool prefetched = false; + bool copy_to_third_buffer = false; + // If the buffer contains only a few of the requested bytes: + // If readahead is enabled: prefetch the remaining bytes + readahead bytes + // and satisfy the request. + // If readahead is not enabled: return false. + TEST_SYNC_POINT_CALLBACK("FilePrefetchBuffer::TryReadFromCache", + &readahead_size_); + + if (explicit_prefetch_submitted_ || + (bufs_[curr_].async_read_in_progress_ || + offset + n > + bufs_[curr_].offset_ + bufs_[curr_].buffer_.CurrentSize())) { + if (readahead_size_ > 0) { + Status s; + assert(reader != nullptr); + assert(max_readahead_size_ >= readahead_size_); + + if (implicit_auto_readahead_) { + if (!IsEligibleForPrefetch(offset, n)) { + // Ignore status as Prefetch is not called. + s.PermitUncheckedError(); + return false; + } + } + // Prefetch n + readahead_size_/2 synchronously as remaining + // readahead_size_/2 will be prefetched asynchronously. + s = PrefetchAsyncInternal(opts, reader, offset, n, readahead_size_ / 2, + rate_limiter_priority, copy_to_third_buffer); + explicit_prefetch_submitted_ = false; + if (!s.ok()) { + if (status) { + *status = s; + } +#ifndef NDEBUG + IGNORE_STATUS_IF_ERROR(s); +#endif + return false; + } + prefetched = explicit_prefetch_submitted_ ? false : true; + } else { + return false; + } + } + + UpdateReadPattern(offset, n, false /*decrease_readaheadsize*/); + + uint32_t index = curr_; + if (copy_to_third_buffer) { + index = 2; + } + uint64_t offset_in_buffer = offset - bufs_[index].offset_; + *result = Slice(bufs_[index].buffer_.BufferStart() + offset_in_buffer, n); + if (prefetched) { + readahead_size_ = std::min(max_readahead_size_, readahead_size_ * 2); + } + return true; +} + +void FilePrefetchBuffer::PrefetchAsyncCallback(const FSReadRequest& req, + void* cb_arg) { + uint32_t index = *(static_cast(cb_arg)); +#ifndef NDEBUG + if (req.result.size() < req.len) { + // Fake an IO error to force db_stress fault injection to ignore + // truncated read errors + IGNORE_STATUS_IF_ERROR(Status::IOError()); + } + IGNORE_STATUS_IF_ERROR(req.status); +#endif + + if (req.status.ok()) { + if (req.offset + req.result.size() <= + bufs_[index].offset_ + bufs_[index].buffer_.CurrentSize()) { + // All requested bytes are already in the buffer or no data is read + // because of EOF. So no need to update. + return; + } + if (req.offset < bufs_[index].offset_) { + // Next block to be read has changed (Recent read was not a sequential + // read). So ignore this read. + return; + } + size_t current_size = bufs_[index].buffer_.CurrentSize(); + bufs_[index].buffer_.Size(current_size + req.result.size()); + } +} + +Status FilePrefetchBuffer::PrefetchAsync(const IOOptions& opts, + RandomAccessFileReader* reader, + uint64_t offset, size_t n, + Slice* result) { + assert(reader != nullptr); + if (!enable_) { + return Status::NotSupported(); + } + + TEST_SYNC_POINT("FilePrefetchBuffer::PrefetchAsync:Start"); + + num_file_reads_ = 0; + explicit_prefetch_submitted_ = false; + bool is_eligible_for_prefetching = false; + if (readahead_size_ > 0 && + (!implicit_auto_readahead_ || + num_file_reads_ >= num_file_reads_for_auto_readahead_)) { + is_eligible_for_prefetching = true; + } + + // 1. Cancel any pending async read to make code simpler as buffers can be out + // of sync. + AbortAllIOs(); + + // 2. Clear outdated data. + UpdateBuffersIfNeeded(offset); + uint32_t second = curr_ ^ 1; + // Since PrefetchAsync can be called on non sequential reads. So offset can + // be less than curr_ buffers' offset. In that case also it clears both + // buffers. + if (DoesBufferContainData(curr_) && !IsOffsetInBuffer(offset, curr_)) { + bufs_[curr_].buffer_.Clear(); + bufs_[second].buffer_.Clear(); + } + + UpdateReadPattern(offset, n, /*decrease_readaheadsize=*/false); + + bool data_found = false; + + // 3. If curr_ has full data. + if (DoesBufferContainData(curr_) && IsDataBlockInBuffer(offset, n, curr_)) { + uint64_t offset_in_buffer = offset - bufs_[curr_].offset_; + *result = Slice(bufs_[curr_].buffer_.BufferStart() + offset_in_buffer, n); + data_found = true; + // Update num_file_reads_ as TryReadFromCacheAsync won't be called for + // poll and update num_file_reads_ if data is found. + num_file_reads_++; + + // 3.1 If second also has some data or is not eligible for prefetching, + // return. + if (!is_eligible_for_prefetching || DoesBufferContainData(second)) { + return Status::OK(); + } + } else { + // Partial data in curr_. + bufs_[curr_].buffer_.Clear(); + } + bufs_[second].buffer_.Clear(); + + Status s; + size_t alignment = reader->file()->GetRequiredBufferAlignment(); + size_t prefetch_size = is_eligible_for_prefetching ? readahead_size_ / 2 : 0; + size_t offset_to_read = static_cast(offset); + uint64_t rounddown_start1 = 0; + uint64_t roundup_end1 = 0; + uint64_t rounddown_start2 = 0; + uint64_t roundup_end2 = 0; + uint64_t chunk_len1 = 0; + uint64_t chunk_len2 = 0; + size_t read_len1 = 0; + size_t read_len2 = 0; + + // - If curr_ is empty. + // - Call async read for full data + prefetch_size on curr_. + // - Call async read for prefetch_size on second if eligible. + // - If curr_ is filled. + // - prefetch_size on second. + // Calculate length and offsets for reading. + if (!DoesBufferContainData(curr_)) { + // Prefetch full data + prefetch_size in curr_. + rounddown_start1 = Rounddown(offset_to_read, alignment); + roundup_end1 = Roundup(offset_to_read + n + prefetch_size, alignment); + uint64_t roundup_len1 = roundup_end1 - rounddown_start1; + assert(roundup_len1 >= alignment); + assert(roundup_len1 % alignment == 0); + + CalculateOffsetAndLen(alignment, rounddown_start1, roundup_len1, curr_, + false, chunk_len1); + assert(chunk_len1 == 0); + assert(roundup_len1 >= chunk_len1); + read_len1 = static_cast(roundup_len1 - chunk_len1); + bufs_[curr_].offset_ = rounddown_start1; + } + + if (is_eligible_for_prefetching) { + if (DoesBufferContainData(curr_)) { + rounddown_start2 = + bufs_[curr_].offset_ + bufs_[curr_].buffer_.CurrentSize(); + } else { + rounddown_start2 = roundup_end1; + } + + roundup_end2 = Roundup(rounddown_start2 + prefetch_size, alignment); + uint64_t roundup_len2 = roundup_end2 - rounddown_start2; + + assert(roundup_len2 >= alignment); + CalculateOffsetAndLen(alignment, rounddown_start2, roundup_len2, second, + false, chunk_len2); + assert(chunk_len2 == 0); + assert(roundup_len2 >= chunk_len2); + read_len2 = static_cast(roundup_len2 - chunk_len2); + // Update the buffer offset. + bufs_[second].offset_ = rounddown_start2; + } + + if (read_len1) { + s = ReadAsync(opts, reader, read_len1, rounddown_start1, curr_); + if (!s.ok()) { + DestroyAndClearIOHandle(curr_); + bufs_[curr_].buffer_.Clear(); + return s; + } + explicit_prefetch_submitted_ = true; + prev_len_ = 0; + } + if (read_len2) { + TEST_SYNC_POINT("FilePrefetchBuffer::PrefetchAsync:ExtraPrefetching"); + s = ReadAsync(opts, reader, read_len2, rounddown_start2, second); + if (!s.ok()) { + DestroyAndClearIOHandle(second); + bufs_[second].buffer_.Clear(); + return s; + } + readahead_size_ = std::min(max_readahead_size_, readahead_size_ * 2); + } + return (data_found ? Status::OK() : Status::TryAgain()); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/file/file_prefetch_buffer.h b/librocksdb-sys/rocksdb/file/file_prefetch_buffer.h new file mode 100644 index 0000000..ae84724 --- /dev/null +++ b/librocksdb-sys/rocksdb/file/file_prefetch_buffer.h @@ -0,0 +1,471 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include +#include +#include +#include + +#include "file/readahead_file_info.h" +#include "monitoring/statistics_impl.h" +#include "port/port.h" +#include "rocksdb/env.h" +#include "rocksdb/file_system.h" +#include "rocksdb/options.h" +#include "util/aligned_buffer.h" +#include "util/autovector.h" +#include "util/stop_watch.h" + +namespace ROCKSDB_NAMESPACE { + +#define DEFAULT_DECREMENT 8 * 1024 + +struct IOOptions; +class RandomAccessFileReader; + +struct BufferInfo { + AlignedBuffer buffer_; + + uint64_t offset_ = 0; + + // Below parameters are used in case of async read flow. + // Length requested for in ReadAsync. + size_t async_req_len_ = 0; + + // async_read_in_progress can be used as mutex. Callback can update the buffer + // and its size but async_read_in_progress is only set by main thread. + bool async_read_in_progress_ = false; + + // io_handle is allocated and used by underlying file system in case of + // asynchronous reads. + void* io_handle_ = nullptr; + + IOHandleDeleter del_fn_ = nullptr; + + // pos represents the index of this buffer in vector of BufferInfo. + uint32_t pos_ = 0; +}; + +enum class FilePrefetchBufferUsage { + kTableOpenPrefetchTail, + kUnknown, +}; + +// FilePrefetchBuffer is a smart buffer to store and read data from a file. +class FilePrefetchBuffer { + public: + // Constructor. + // + // All arguments are optional. + // readahead_size : the initial readahead size. + // max_readahead_size : the maximum readahead size. + // If max_readahead_size > readahead_size, the readahead size will be + // doubled on every IO until max_readahead_size is hit. + // Typically this is set as a multiple of readahead_size. + // max_readahead_size should be greater than equal to readahead_size. + // enable : controls whether reading from the buffer is enabled. + // If false, TryReadFromCache() always return false, and we only take stats + // for the minimum offset if track_min_offset = true. + // track_min_offset : Track the minimum offset ever read and collect stats on + // it. Used for adaptable readahead of the file footer/metadata. + // implicit_auto_readahead : Readahead is enabled implicitly by rocksdb after + // doing sequential scans for two times. + // + // Automatic readhead is enabled for a file if readahead_size + // and max_readahead_size are passed in. + // A user can construct a FilePrefetchBuffer without any arguments, but use + // `Prefetch` to load data into the buffer. + FilePrefetchBuffer( + size_t readahead_size = 0, size_t max_readahead_size = 0, + bool enable = true, bool track_min_offset = false, + bool implicit_auto_readahead = false, uint64_t num_file_reads = 0, + uint64_t num_file_reads_for_auto_readahead = 0, FileSystem* fs = nullptr, + SystemClock* clock = nullptr, Statistics* stats = nullptr, + FilePrefetchBufferUsage usage = FilePrefetchBufferUsage::kUnknown) + : curr_(0), + readahead_size_(readahead_size), + initial_auto_readahead_size_(readahead_size), + max_readahead_size_(max_readahead_size), + min_offset_read_(std::numeric_limits::max()), + enable_(enable), + track_min_offset_(track_min_offset), + implicit_auto_readahead_(implicit_auto_readahead), + prev_offset_(0), + prev_len_(0), + num_file_reads_for_auto_readahead_(num_file_reads_for_auto_readahead), + num_file_reads_(num_file_reads), + explicit_prefetch_submitted_(false), + fs_(fs), + clock_(clock), + stats_(stats), + usage_(usage) { + assert((num_file_reads_ >= num_file_reads_for_auto_readahead_ + 1) || + (num_file_reads_ == 0)); + // If ReadOptions.async_io is enabled, data is asynchronously filled in + // second buffer while curr_ is being consumed. If data is overlapping in + // two buffers, data is copied to third buffer to return continuous buffer. + bufs_.resize(3); + for (uint32_t i = 0; i < 2; i++) { + bufs_[i].pos_ = i; + } + } + + ~FilePrefetchBuffer() { + // Abort any pending async read request before destroying the class object. + if (fs_ != nullptr) { + std::vector handles; + for (uint32_t i = 0; i < 2; i++) { + if (bufs_[i].async_read_in_progress_ && + bufs_[i].io_handle_ != nullptr) { + handles.emplace_back(bufs_[i].io_handle_); + } + } + if (!handles.empty()) { + StopWatch sw(clock_, stats_, ASYNC_PREFETCH_ABORT_MICROS); + Status s = fs_->AbortIO(handles); + assert(s.ok()); + } + } + + // Prefetch buffer bytes discarded. + uint64_t bytes_discarded = 0; + // Iterated over 2 buffers. + for (int i = 0; i < 2; i++) { + int first = i; + int second = i ^ 1; + + if (DoesBufferContainData(first)) { + // If last block was read completely from first and some bytes in + // first buffer are still unconsumed. + if (prev_offset_ >= bufs_[first].offset_ && + prev_offset_ + prev_len_ < + bufs_[first].offset_ + bufs_[first].buffer_.CurrentSize()) { + bytes_discarded += bufs_[first].buffer_.CurrentSize() - + (prev_offset_ + prev_len_ - bufs_[first].offset_); + } + // If data was in second buffer and some/whole block bytes were read + // from second buffer. + else if (prev_offset_ < bufs_[first].offset_ && + !DoesBufferContainData(second)) { + // If last block read was completely from different buffer, this + // buffer is unconsumed. + if (prev_offset_ + prev_len_ <= bufs_[first].offset_) { + bytes_discarded += bufs_[first].buffer_.CurrentSize(); + } + // If last block read overlaps with this buffer and some data is + // still unconsumed and previous buffer (second) is not cleared. + else if (prev_offset_ + prev_len_ > bufs_[first].offset_ && + bufs_[first].offset_ + bufs_[first].buffer_.CurrentSize() == + bufs_[second].offset_) { + bytes_discarded += bufs_[first].buffer_.CurrentSize() - + (/*bytes read from this buffer=*/prev_len_ - + (bufs_[first].offset_ - prev_offset_)); + } + } + } + } + + for (uint32_t i = 0; i < 2; i++) { + // Release io_handle. + DestroyAndClearIOHandle(i); + } + RecordInHistogram(stats_, PREFETCHED_BYTES_DISCARDED, bytes_discarded); + } + + bool Enabled() const { return enable_; } + + // Load data into the buffer from a file. + // reader : the file reader. + // offset : the file offset to start reading from. + // n : the number of bytes to read. + // rate_limiter_priority : rate limiting priority, or `Env::IO_TOTAL` to + // bypass. + Status Prefetch(const IOOptions& opts, RandomAccessFileReader* reader, + uint64_t offset, size_t n, + Env::IOPriority rate_limiter_priority); + + // Request for reading the data from a file asynchronously. + // If data already exists in the buffer, result will be updated. + // reader : the file reader. + // offset : the file offset to start reading from. + // n : the number of bytes to read. + // result : if data already exists in the buffer, result will + // be updated with the data. + // + // If data already exist in the buffer, it will return Status::OK, otherwise + // it will send asynchronous request and return Status::TryAgain. + Status PrefetchAsync(const IOOptions& opts, RandomAccessFileReader* reader, + uint64_t offset, size_t n, Slice* result); + + // Tries returning the data for a file read from this buffer if that data is + // in the buffer. + // It handles tracking the minimum read offset if track_min_offset = true. + // It also does the exponential readahead when readahead_size is set as part + // of the constructor. + // + // opts : the IO options to use. + // reader : the file reader. + // offset : the file offset. + // n : the number of bytes. + // result : output buffer to put the data into. + // s : output status. + // rate_limiter_priority : rate limiting priority, or `Env::IO_TOTAL` to + // bypass. + // for_compaction : true if cache read is done for compaction read. + bool TryReadFromCache(const IOOptions& opts, RandomAccessFileReader* reader, + uint64_t offset, size_t n, Slice* result, Status* s, + Env::IOPriority rate_limiter_priority, + bool for_compaction = false); + + bool TryReadFromCacheAsync(const IOOptions& opts, + RandomAccessFileReader* reader, uint64_t offset, + size_t n, Slice* result, Status* status, + Env::IOPriority rate_limiter_priority); + + // The minimum `offset` ever passed to TryReadFromCache(). This will nly be + // tracked if track_min_offset = true. + size_t min_offset_read() const { return min_offset_read_; } + + size_t GetPrefetchOffset() const { return bufs_[curr_].offset_; } + + // Called in case of implicit auto prefetching. + void UpdateReadPattern(const uint64_t& offset, const size_t& len, + bool decrease_readaheadsize) { + if (decrease_readaheadsize) { + // Since this block was eligible for prefetch but it was found in + // cache, so check and decrease the readahead_size by 8KB (default) + // if eligible. + DecreaseReadAheadIfEligible(offset, len); + } + prev_offset_ = offset; + prev_len_ = len; + explicit_prefetch_submitted_ = false; + } + + void GetReadaheadState(ReadaheadFileInfo::ReadaheadInfo* readahead_info) { + readahead_info->readahead_size = readahead_size_; + readahead_info->num_file_reads = num_file_reads_; + } + + void DecreaseReadAheadIfEligible(uint64_t offset, size_t size, + size_t value = DEFAULT_DECREMENT) { + // Decrease the readahead_size if + // - its enabled internally by RocksDB (implicit_auto_readahead_) and, + // - readahead_size is greater than 0 and, + // - this block would have called prefetch API if not found in cache for + // which conditions are: + // - few/no bytes are in buffer and, + // - block is sequential with the previous read and, + // - num_file_reads_ + 1 (including this read) > + // num_file_reads_for_auto_readahead_ + size_t curr_size = bufs_[curr_].async_read_in_progress_ + ? bufs_[curr_].async_req_len_ + : bufs_[curr_].buffer_.CurrentSize(); + if (implicit_auto_readahead_ && readahead_size_ > 0) { + if ((offset + size > bufs_[curr_].offset_ + curr_size) && + IsBlockSequential(offset) && + (num_file_reads_ + 1 > num_file_reads_for_auto_readahead_)) { + readahead_size_ = + std::max(initial_auto_readahead_size_, + (readahead_size_ >= value ? readahead_size_ - value : 0)); + } + } + } + + // Callback function passed to underlying FS in case of asynchronous reads. + void PrefetchAsyncCallback(const FSReadRequest& req, void* cb_arg); + + private: + // Calculates roundoff offset and length to be prefetched based on alignment + // and data present in buffer_. It also allocates new buffer or refit tail if + // required. + void CalculateOffsetAndLen(size_t alignment, uint64_t offset, + size_t roundup_len, uint32_t index, + bool refit_tail, uint64_t& chunk_len); + + void AbortIOIfNeeded(uint64_t offset); + + void AbortAllIOs(); + + void UpdateBuffersIfNeeded(uint64_t offset); + + // It calls Poll API if any there is any pending asynchronous request. It then + // checks if data is in any buffer. It clears the outdated data and swaps the + // buffers if required. + void PollAndUpdateBuffersIfNeeded(uint64_t offset); + + Status PrefetchAsyncInternal(const IOOptions& opts, + RandomAccessFileReader* reader, uint64_t offset, + size_t length, size_t readahead_size, + Env::IOPriority rate_limiter_priority, + bool& copy_to_third_buffer); + + Status Read(const IOOptions& opts, RandomAccessFileReader* reader, + Env::IOPriority rate_limiter_priority, uint64_t read_len, + uint64_t chunk_len, uint64_t rounddown_start, uint32_t index); + + Status ReadAsync(const IOOptions& opts, RandomAccessFileReader* reader, + uint64_t read_len, uint64_t rounddown_start, uint32_t index); + + // Copy the data from src to third buffer. + void CopyDataToBuffer(uint32_t src, uint64_t& offset, size_t& length); + + bool IsBlockSequential(const size_t& offset) { + return (prev_len_ == 0 || (prev_offset_ + prev_len_ == offset)); + } + + // Called in case of implicit auto prefetching. + void ResetValues() { + num_file_reads_ = 1; + readahead_size_ = initial_auto_readahead_size_; + } + + // Called in case of implicit auto prefetching. + bool IsEligibleForPrefetch(uint64_t offset, size_t n) { + // Prefetch only if this read is sequential otherwise reset readahead_size_ + // to initial value. + if (!IsBlockSequential(offset)) { + UpdateReadPattern(offset, n, false /*decrease_readaheadsize*/); + ResetValues(); + return false; + } + num_file_reads_++; + + // Since async request was submitted in last call directly by calling + // PrefetchAsync, it skips num_file_reads_ check as this call is to poll the + // data submitted in previous call. + if (explicit_prefetch_submitted_) { + return true; + } + if (num_file_reads_ <= num_file_reads_for_auto_readahead_) { + UpdateReadPattern(offset, n, false /*decrease_readaheadsize*/); + return false; + } + return true; + } + + // Helper functions. + bool IsDataBlockInBuffer(uint64_t offset, size_t length, uint32_t index) { + return (offset >= bufs_[index].offset_ && + offset + length <= + bufs_[index].offset_ + bufs_[index].buffer_.CurrentSize()); + } + bool IsOffsetInBuffer(uint64_t offset, uint32_t index) { + return (offset >= bufs_[index].offset_ && + offset < bufs_[index].offset_ + bufs_[index].buffer_.CurrentSize()); + } + bool DoesBufferContainData(uint32_t index) { + return bufs_[index].buffer_.CurrentSize() > 0; + } + bool IsBufferOutdated(uint64_t offset, uint32_t index) { + return ( + !bufs_[index].async_read_in_progress_ && DoesBufferContainData(index) && + offset >= bufs_[index].offset_ + bufs_[index].buffer_.CurrentSize()); + } + bool IsBufferOutdatedWithAsyncProgress(uint64_t offset, uint32_t index) { + return (bufs_[index].async_read_in_progress_ && + bufs_[index].io_handle_ != nullptr && + offset >= bufs_[index].offset_ + bufs_[index].async_req_len_); + } + bool IsOffsetInBufferWithAsyncProgress(uint64_t offset, uint32_t index) { + return (bufs_[index].async_read_in_progress_ && + offset >= bufs_[index].offset_ && + offset < bufs_[index].offset_ + bufs_[index].async_req_len_); + } + + bool IsSecondBuffEligibleForPrefetching() { + uint32_t second = curr_ ^ 1; + if (bufs_[second].async_read_in_progress_) { + return false; + } + assert(!bufs_[curr_].async_read_in_progress_); + + if (DoesBufferContainData(curr_) && DoesBufferContainData(second) && + (bufs_[curr_].offset_ + bufs_[curr_].buffer_.CurrentSize() == + bufs_[second].offset_)) { + return false; + } + bufs_[second].buffer_.Clear(); + return true; + } + + void DestroyAndClearIOHandle(uint32_t index) { + if (bufs_[index].io_handle_ != nullptr && bufs_[index].del_fn_ != nullptr) { + bufs_[index].del_fn_(bufs_[index].io_handle_); + bufs_[index].io_handle_ = nullptr; + bufs_[index].del_fn_ = nullptr; + } + bufs_[index].async_read_in_progress_ = false; + } + + Status HandleOverlappingData(const IOOptions& opts, + RandomAccessFileReader* reader, uint64_t offset, + size_t length, size_t readahead_size, + Env::IOPriority rate_limiter_priority, + bool& copy_to_third_buffer, uint64_t& tmp_offset, + size_t& tmp_length); + + bool TryReadFromCacheUntracked(const IOOptions& opts, + RandomAccessFileReader* reader, + uint64_t offset, size_t n, Slice* result, + Status* s, + Env::IOPriority rate_limiter_priority, + bool for_compaction = false); + + bool TryReadFromCacheAsyncUntracked(const IOOptions& opts, + RandomAccessFileReader* reader, + uint64_t offset, size_t n, Slice* result, + Status* status, + Env::IOPriority rate_limiter_priority); + + std::vector bufs_; + // curr_ represents the index for bufs_ indicating which buffer is being + // consumed currently. + uint32_t curr_; + + size_t readahead_size_; + size_t initial_auto_readahead_size_; + // FilePrefetchBuffer object won't be created from Iterator flow if + // max_readahead_size_ = 0. + size_t max_readahead_size_; + + // The minimum `offset` ever passed to TryReadFromCache(). + size_t min_offset_read_; + // if false, TryReadFromCache() always return false, and we only take stats + // for track_min_offset_ if track_min_offset_ = true + bool enable_; + // If true, track minimum `offset` ever passed to TryReadFromCache(), which + // can be fetched from min_offset_read(). + bool track_min_offset_; + + // implicit_auto_readahead is enabled by rocksdb internally after 2 + // sequential IOs. + bool implicit_auto_readahead_; + uint64_t prev_offset_; + size_t prev_len_; + // num_file_reads_ and num_file_reads_for_auto_readahead_ is only used when + // implicit_auto_readahead_ is set. + uint64_t num_file_reads_for_auto_readahead_; + uint64_t num_file_reads_; + + // If explicit_prefetch_submitted_ is set then it indicates RocksDB called + // PrefetchAsync to submit request. It needs to call TryReadFromCacheAsync to + // poll the submitted request without checking if data is sequential and + // num_file_reads_. + bool explicit_prefetch_submitted_; + + FileSystem* fs_; + SystemClock* clock_; + Statistics* stats_; + + FilePrefetchBufferUsage usage_; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/file/file_util.cc b/librocksdb-sys/rocksdb/file/file_util.cc new file mode 100644 index 0000000..46faac6 --- /dev/null +++ b/librocksdb-sys/rocksdb/file/file_util.cc @@ -0,0 +1,277 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#include "file/file_util.h" + +#include +#include + +#include "file/random_access_file_reader.h" +#include "file/sequence_file_reader.h" +#include "file/sst_file_manager_impl.h" +#include "file/writable_file_writer.h" +#include "rocksdb/env.h" + +namespace ROCKSDB_NAMESPACE { + +// Utility function to copy a file up to a specified length +IOStatus CopyFile(FileSystem* fs, const std::string& source, + std::unique_ptr& dest_writer, + uint64_t size, bool use_fsync, + const std::shared_ptr& io_tracer, + const Temperature temperature) { + FileOptions soptions; + IOStatus io_s; + std::unique_ptr src_reader; + + { + soptions.temperature = temperature; + std::unique_ptr srcfile; + io_s = fs->NewSequentialFile(source, soptions, &srcfile, nullptr); + if (!io_s.ok()) { + return io_s; + } + + if (size == 0) { + // default argument means copy everything + io_s = fs->GetFileSize(source, IOOptions(), &size, nullptr); + if (!io_s.ok()) { + return io_s; + } + } + src_reader.reset( + new SequentialFileReader(std::move(srcfile), source, io_tracer)); + } + + char buffer[4096]; + Slice slice; + while (size > 0) { + size_t bytes_to_read = std::min(sizeof(buffer), static_cast(size)); + // TODO: rate limit copy file + io_s = status_to_io_status( + src_reader->Read(bytes_to_read, &slice, buffer, + Env::IO_TOTAL /* rate_limiter_priority */)); + if (!io_s.ok()) { + return io_s; + } + if (slice.size() == 0) { + return IOStatus::Corruption("file too small"); + } + io_s = dest_writer->Append(slice); + if (!io_s.ok()) { + return io_s; + } + size -= slice.size(); + } + return dest_writer->Sync(use_fsync); +} + +IOStatus CopyFile(FileSystem* fs, const std::string& source, + const std::string& destination, uint64_t size, bool use_fsync, + const std::shared_ptr& io_tracer, + const Temperature temperature) { + FileOptions options; + IOStatus io_s; + std::unique_ptr dest_writer; + + { + options.temperature = temperature; + std::unique_ptr destfile; + io_s = fs->NewWritableFile(destination, options, &destfile, nullptr); + if (!io_s.ok()) { + return io_s; + } + + dest_writer.reset( + new WritableFileWriter(std::move(destfile), destination, options)); + } + + return CopyFile(fs, source, dest_writer, size, use_fsync, io_tracer, + temperature); +} + +// Utility function to create a file with the provided contents +IOStatus CreateFile(FileSystem* fs, const std::string& destination, + const std::string& contents, bool use_fsync) { + const EnvOptions soptions; + IOStatus io_s; + std::unique_ptr dest_writer; + + std::unique_ptr destfile; + io_s = fs->NewWritableFile(destination, soptions, &destfile, nullptr); + if (!io_s.ok()) { + return io_s; + } + dest_writer.reset( + new WritableFileWriter(std::move(destfile), destination, soptions)); + io_s = dest_writer->Append(Slice(contents)); + if (!io_s.ok()) { + return io_s; + } + return dest_writer->Sync(use_fsync); +} + +Status DeleteDBFile(const ImmutableDBOptions* db_options, + const std::string& fname, const std::string& dir_to_sync, + const bool force_bg, const bool force_fg) { + SstFileManagerImpl* sfm = + static_cast(db_options->sst_file_manager.get()); + if (sfm && !force_fg) { + return sfm->ScheduleFileDeletion(fname, dir_to_sync, force_bg); + } else { + return db_options->env->DeleteFile(fname); + } +} + +// requested_checksum_func_name brings the function name of the checksum +// generator in checksum_factory. Empty string is permitted, in which case the +// name of the generator created by the factory is unchecked. When +// `requested_checksum_func_name` is non-empty, however, the created generator's +// name must match it, otherwise an `InvalidArgument` error is returned. +IOStatus GenerateOneFileChecksum( + FileSystem* fs, const std::string& file_path, + FileChecksumGenFactory* checksum_factory, + const std::string& requested_checksum_func_name, std::string* file_checksum, + std::string* file_checksum_func_name, + size_t verify_checksums_readahead_size, bool /*allow_mmap_reads*/, + std::shared_ptr& io_tracer, RateLimiter* rate_limiter, + Env::IOPriority rate_limiter_priority) { + if (checksum_factory == nullptr) { + return IOStatus::InvalidArgument("Checksum factory is invalid"); + } + assert(file_checksum != nullptr); + assert(file_checksum_func_name != nullptr); + + FileChecksumGenContext gen_context; + gen_context.requested_checksum_func_name = requested_checksum_func_name; + gen_context.file_name = file_path; + std::unique_ptr checksum_generator = + checksum_factory->CreateFileChecksumGenerator(gen_context); + if (checksum_generator == nullptr) { + std::string msg = + "Cannot get the file checksum generator based on the requested " + "checksum function name: " + + requested_checksum_func_name + + " from checksum factory: " + checksum_factory->Name(); + return IOStatus::InvalidArgument(msg); + } else { + // For backward compatibility and use in file ingestion clients where there + // is no stored checksum function name, `requested_checksum_func_name` can + // be empty. If we give the requested checksum function name, we expect it + // is the same name of the checksum generator. + if (!requested_checksum_func_name.empty() && + checksum_generator->Name() != requested_checksum_func_name) { + std::string msg = "Expected file checksum generator named '" + + requested_checksum_func_name + + "', while the factory created one " + "named '" + + checksum_generator->Name() + "'"; + return IOStatus::InvalidArgument(msg); + } + } + + uint64_t size; + IOStatus io_s; + std::unique_ptr reader; + { + std::unique_ptr r_file; + io_s = fs->NewRandomAccessFile(file_path, FileOptions(), &r_file, nullptr); + if (!io_s.ok()) { + return io_s; + } + io_s = fs->GetFileSize(file_path, IOOptions(), &size, nullptr); + if (!io_s.ok()) { + return io_s; + } + reader.reset(new RandomAccessFileReader( + std::move(r_file), file_path, nullptr /*Env*/, io_tracer, nullptr, + Histograms::HISTOGRAM_ENUM_MAX, nullptr, rate_limiter)); + } + + // Found that 256 KB readahead size provides the best performance, based on + // experiments, for auto readahead. Experiment data is in PR #3282. + size_t default_max_read_ahead_size = 256 * 1024; + size_t readahead_size = (verify_checksums_readahead_size != 0) + ? verify_checksums_readahead_size + : default_max_read_ahead_size; + std::unique_ptr buf; + if (reader->use_direct_io()) { + size_t alignment = reader->file()->GetRequiredBufferAlignment(); + readahead_size = (readahead_size + alignment - 1) & ~(alignment - 1); + } + buf.reset(new char[readahead_size]); + + Slice slice; + uint64_t offset = 0; + IOOptions opts; + while (size > 0) { + size_t bytes_to_read = + static_cast(std::min(uint64_t{readahead_size}, size)); + io_s = reader->Read(opts, offset, bytes_to_read, &slice, buf.get(), nullptr, + rate_limiter_priority); + if (!io_s.ok()) { + return IOStatus::Corruption("file read failed with error: " + + io_s.ToString()); + } + if (slice.size() == 0) { + return IOStatus::Corruption("file too small"); + } + checksum_generator->Update(slice.data(), slice.size()); + size -= slice.size(); + offset += slice.size(); + + TEST_SYNC_POINT("GenerateOneFileChecksum::Chunk:0"); + } + checksum_generator->Finalize(); + *file_checksum = checksum_generator->GetChecksum(); + *file_checksum_func_name = checksum_generator->Name(); + return IOStatus::OK(); +} + +Status DestroyDir(Env* env, const std::string& dir) { + Status s; + if (env->FileExists(dir).IsNotFound()) { + return s; + } + std::vector files_in_dir; + s = env->GetChildren(dir, &files_in_dir); + if (s.ok()) { + for (auto& file_in_dir : files_in_dir) { + std::string path = dir + "/" + file_in_dir; + bool is_dir = false; + s = env->IsDirectory(path, &is_dir); + if (s.ok()) { + if (is_dir) { + s = DestroyDir(env, path); + } else { + s = env->DeleteFile(path); + } + } else if (s.IsNotSupported()) { + s = Status::OK(); + } + if (!s.ok()) { + // IsDirectory, etc. might not report NotFound + if (s.IsNotFound() || env->FileExists(path).IsNotFound()) { + // Allow files to be deleted externally + s = Status::OK(); + } else { + break; + } + } + } + } + + if (s.ok()) { + s = env->DeleteDir(dir); + // DeleteDir might or might not report NotFound + if (!s.ok() && (s.IsNotFound() || env->FileExists(dir).IsNotFound())) { + // Allow to be deleted externally + s = Status::OK(); + } + } + return s; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/file/file_util.h b/librocksdb-sys/rocksdb/file/file_util.h new file mode 100644 index 0000000..8b59731 --- /dev/null +++ b/librocksdb-sys/rocksdb/file/file_util.h @@ -0,0 +1,101 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#pragma once +#include + +#include "file/filename.h" +#include "options/db_options.h" +#include "rocksdb/env.h" +#include "rocksdb/file_system.h" +#include "rocksdb/sst_file_writer.h" +#include "rocksdb/status.h" +#include "rocksdb/system_clock.h" +#include "rocksdb/types.h" +#include "trace_replay/io_tracer.h" + +namespace ROCKSDB_NAMESPACE { +// use_fsync maps to options.use_fsync, which determines the way that +// the file is synced after copying. +extern IOStatus CopyFile(FileSystem* fs, const std::string& source, + std::unique_ptr& dest_writer, + uint64_t size, bool use_fsync, + const std::shared_ptr& io_tracer, + const Temperature temperature); +extern IOStatus CopyFile(FileSystem* fs, const std::string& source, + const std::string& destination, uint64_t size, + bool use_fsync, + const std::shared_ptr& io_tracer, + const Temperature temperature); +inline IOStatus CopyFile(const std::shared_ptr& fs, + const std::string& source, + const std::string& destination, uint64_t size, + bool use_fsync, + const std::shared_ptr& io_tracer, + const Temperature temperature) { + return CopyFile(fs.get(), source, destination, size, use_fsync, io_tracer, + temperature); +} +extern IOStatus CreateFile(FileSystem* fs, const std::string& destination, + const std::string& contents, bool use_fsync); + +inline IOStatus CreateFile(const std::shared_ptr& fs, + const std::string& destination, + const std::string& contents, bool use_fsync) { + return CreateFile(fs.get(), destination, contents, use_fsync); +} + +extern Status DeleteDBFile(const ImmutableDBOptions* db_options, + const std::string& fname, + const std::string& path_to_sync, const bool force_bg, + const bool force_fg); + +extern IOStatus GenerateOneFileChecksum( + FileSystem* fs, const std::string& file_path, + FileChecksumGenFactory* checksum_factory, + const std::string& requested_checksum_func_name, std::string* file_checksum, + std::string* file_checksum_func_name, + size_t verify_checksums_readahead_size, bool allow_mmap_reads, + std::shared_ptr& io_tracer, RateLimiter* rate_limiter, + Env::IOPriority rate_limiter_priority); + +inline IOStatus PrepareIOFromReadOptions(const ReadOptions& ro, + SystemClock* clock, IOOptions& opts) { + if (ro.deadline.count()) { + std::chrono::microseconds now = + std::chrono::microseconds(clock->NowMicros()); + // Ensure there is atleast 1us available. We don't want to pass a value of + // 0 as that means no timeout + if (now >= ro.deadline) { + return IOStatus::TimedOut("Deadline exceeded"); + } + opts.timeout = ro.deadline - now; + } + + if (ro.io_timeout.count() && + (!opts.timeout.count() || ro.io_timeout < opts.timeout)) { + opts.timeout = ro.io_timeout; + } + + opts.rate_limiter_priority = ro.rate_limiter_priority; + opts.io_activity = ro.io_activity; + + return IOStatus::OK(); +} + +// Test method to delete the input directory and all of its contents. +// This method is destructive and is meant for use only in tests!!! +Status DestroyDir(Env* env, const std::string& dir); + +inline bool CheckFSFeatureSupport(FileSystem* fs, FSSupportedOps feat) { + int64_t supported_ops = 0; + fs->SupportedOps(supported_ops); + if (supported_ops & (1ULL << feat)) { + return true; + } + return false; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/file/filename.cc b/librocksdb-sys/rocksdb/file/filename.cc new file mode 100644 index 0000000..1e04c73 --- /dev/null +++ b/librocksdb-sys/rocksdb/file/filename.cc @@ -0,0 +1,523 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "file/filename.h" + +#include +#include + +#include +#include + +#include "file/writable_file_writer.h" +#include "rocksdb/env.h" +#include "test_util/sync_point.h" +#include "util/stop_watch.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +const std::string kCurrentFileName = "CURRENT"; +const std::string kOptionsFileNamePrefix = "OPTIONS-"; +const std::string kTempFileNameSuffix = "dbtmp"; + +static const std::string kRocksDbTFileExt = "sst"; +static const std::string kLevelDbTFileExt = "ldb"; +static const std::string kRocksDBBlobFileExt = "blob"; +static const std::string kArchivalDirName = "archive"; + +// Given a path, flatten the path name by replacing all chars not in +// {[0-9,a-z,A-Z,-,_,.]} with _. And append '_LOG\0' at the end. +// Return the number of chars stored in dest not including the trailing '\0'. +static size_t GetInfoLogPrefix(const std::string& path, char* dest, int len) { + const char suffix[] = "_LOG"; + + size_t write_idx = 0; + size_t i = 0; + size_t src_len = path.size(); + + while (i < src_len && write_idx < len - sizeof(suffix)) { + if ((path[i] >= 'a' && path[i] <= 'z') || + (path[i] >= '0' && path[i] <= '9') || + (path[i] >= 'A' && path[i] <= 'Z') || path[i] == '-' || + path[i] == '.' || path[i] == '_') { + dest[write_idx++] = path[i]; + } else { + if (i > 0) { + dest[write_idx++] = '_'; + } + } + i++; + } + assert(sizeof(suffix) <= len - write_idx); + // "\0" is automatically added by snprintf + snprintf(dest + write_idx, len - write_idx, suffix); + write_idx += sizeof(suffix) - 1; + return write_idx; +} + +static std::string MakeFileName(uint64_t number, const char* suffix) { + char buf[100]; + snprintf(buf, sizeof(buf), "%06llu.%s", + static_cast(number), suffix); + return buf; +} + +static std::string MakeFileName(const std::string& name, uint64_t number, + const char* suffix) { + return name + "/" + MakeFileName(number, suffix); +} + +std::string LogFileName(const std::string& name, uint64_t number) { + assert(number > 0); + return MakeFileName(name, number, "log"); +} + +std::string LogFileName(uint64_t number) { + assert(number > 0); + return MakeFileName(number, "log"); +} + +std::string BlobFileName(uint64_t number) { + assert(number > 0); + return MakeFileName(number, kRocksDBBlobFileExt.c_str()); +} + +std::string BlobFileName(const std::string& blobdirname, uint64_t number) { + assert(number > 0); + return MakeFileName(blobdirname, number, kRocksDBBlobFileExt.c_str()); +} + +std::string BlobFileName(const std::string& dbname, const std::string& blob_dir, + uint64_t number) { + assert(number > 0); + return MakeFileName(dbname + "/" + blob_dir, number, + kRocksDBBlobFileExt.c_str()); +} + +std::string ArchivalDirectory(const std::string& dir) { + return dir + "/" + kArchivalDirName; +} +std::string ArchivedLogFileName(const std::string& name, uint64_t number) { + assert(number > 0); + return MakeFileName(name + "/" + kArchivalDirName, number, "log"); +} + +std::string MakeTableFileName(const std::string& path, uint64_t number) { + return MakeFileName(path, number, kRocksDbTFileExt.c_str()); +} + +std::string MakeTableFileName(uint64_t number) { + return MakeFileName(number, kRocksDbTFileExt.c_str()); +} + +std::string Rocks2LevelTableFileName(const std::string& fullname) { + assert(fullname.size() > kRocksDbTFileExt.size() + 1); + if (fullname.size() <= kRocksDbTFileExt.size() + 1) { + return ""; + } + return fullname.substr(0, fullname.size() - kRocksDbTFileExt.size()) + + kLevelDbTFileExt; +} + +uint64_t TableFileNameToNumber(const std::string& name) { + uint64_t number = 0; + uint64_t base = 1; + int pos = static_cast(name.find_last_of('.')); + while (--pos >= 0 && name[pos] >= '0' && name[pos] <= '9') { + number += (name[pos] - '0') * base; + base *= 10; + } + return number; +} + +std::string TableFileName(const std::vector& db_paths, uint64_t number, + uint32_t path_id) { + assert(number > 0); + std::string path; + if (path_id >= db_paths.size()) { + path = db_paths.back().path; + } else { + path = db_paths[path_id].path; + } + return MakeTableFileName(path, number); +} + +void FormatFileNumber(uint64_t number, uint32_t path_id, char* out_buf, + size_t out_buf_size) { + if (path_id == 0) { + snprintf(out_buf, out_buf_size, "%" PRIu64, number); + } else { + snprintf(out_buf, out_buf_size, + "%" PRIu64 + "(path " + "%" PRIu32 ")", + number, path_id); + } +} + +std::string DescriptorFileName(uint64_t number) { + assert(number > 0); + char buf[100]; + snprintf(buf, sizeof(buf), "MANIFEST-%06llu", + static_cast(number)); + return buf; +} + +std::string DescriptorFileName(const std::string& dbname, uint64_t number) { + return dbname + "/" + DescriptorFileName(number); +} + +std::string CurrentFileName(const std::string& dbname) { + return dbname + "/" + kCurrentFileName; +} + +std::string LockFileName(const std::string& dbname) { return dbname + "/LOCK"; } + +std::string TempFileName(const std::string& dbname, uint64_t number) { + return MakeFileName(dbname, number, kTempFileNameSuffix.c_str()); +} + +InfoLogPrefix::InfoLogPrefix(bool has_log_dir, + const std::string& db_absolute_path) { + if (!has_log_dir) { + const char kInfoLogPrefix[] = "LOG"; + // "\0" is automatically added to the end + snprintf(buf, sizeof(buf), kInfoLogPrefix); + prefix = Slice(buf, sizeof(kInfoLogPrefix) - 1); + } else { + size_t len = + GetInfoLogPrefix(NormalizePath(db_absolute_path), buf, sizeof(buf)); + prefix = Slice(buf, len); + } +} + +std::string InfoLogFileName(const std::string& dbname, + const std::string& db_path, + const std::string& log_dir) { + if (log_dir.empty()) { + return dbname + "/LOG"; + } + + InfoLogPrefix info_log_prefix(true, db_path); + return log_dir + "/" + info_log_prefix.buf; +} + +// Return the name of the old info log file for "dbname". +std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts, + const std::string& db_path, + const std::string& log_dir) { + char buf[50]; + snprintf(buf, sizeof(buf), "%llu", static_cast(ts)); + + if (log_dir.empty()) { + return dbname + "/LOG.old." + buf; + } + + InfoLogPrefix info_log_prefix(true, db_path); + return log_dir + "/" + info_log_prefix.buf + ".old." + buf; +} + +std::string OptionsFileName(uint64_t file_num) { + char buffer[256]; + snprintf(buffer, sizeof(buffer), "%s%06" PRIu64, + kOptionsFileNamePrefix.c_str(), file_num); + return buffer; +} +std::string OptionsFileName(const std::string& dbname, uint64_t file_num) { + return dbname + "/" + OptionsFileName(file_num); +} + +std::string TempOptionsFileName(const std::string& dbname, uint64_t file_num) { + char buffer[256]; + snprintf(buffer, sizeof(buffer), "%s%06" PRIu64 ".%s", + kOptionsFileNamePrefix.c_str(), file_num, + kTempFileNameSuffix.c_str()); + return dbname + "/" + buffer; +} + +std::string MetaDatabaseName(const std::string& dbname, uint64_t number) { + char buf[100]; + snprintf(buf, sizeof(buf), "/METADB-%llu", + static_cast(number)); + return dbname + buf; +} + +std::string IdentityFileName(const std::string& dbname) { + return dbname + "/IDENTITY"; +} + +// Owned filenames have the form: +// dbname/IDENTITY +// dbname/CURRENT +// dbname/LOCK +// dbname/ +// dbname/.old.[0-9]+ +// dbname/MANIFEST-[0-9]+ +// dbname/[0-9]+.(log|sst|blob) +// dbname/METADB-[0-9]+ +// dbname/OPTIONS-[0-9]+ +// dbname/OPTIONS-[0-9]+.dbtmp +// Disregards / at the beginning +bool ParseFileName(const std::string& fname, uint64_t* number, FileType* type, + WalFileType* log_type) { + return ParseFileName(fname, number, "", type, log_type); +} + +bool ParseFileName(const std::string& fname, uint64_t* number, + const Slice& info_log_name_prefix, FileType* type, + WalFileType* log_type) { + Slice rest(fname); + if (fname.length() > 1 && fname[0] == '/') { + rest.remove_prefix(1); + } + if (rest == "IDENTITY") { + *number = 0; + *type = kIdentityFile; + } else if (rest == "CURRENT") { + *number = 0; + *type = kCurrentFile; + } else if (rest == "LOCK") { + *number = 0; + *type = kDBLockFile; + } else if (info_log_name_prefix.size() > 0 && + rest.starts_with(info_log_name_prefix)) { + rest.remove_prefix(info_log_name_prefix.size()); + if (rest == "" || rest == ".old") { + *number = 0; + *type = kInfoLogFile; + } else if (rest.starts_with(".old.")) { + uint64_t ts_suffix; + // sizeof also counts the trailing '\0'. + rest.remove_prefix(sizeof(".old.") - 1); + if (!ConsumeDecimalNumber(&rest, &ts_suffix)) { + return false; + } + *number = ts_suffix; + *type = kInfoLogFile; + } + } else if (rest.starts_with("MANIFEST-")) { + rest.remove_prefix(strlen("MANIFEST-")); + uint64_t num; + if (!ConsumeDecimalNumber(&rest, &num)) { + return false; + } + if (!rest.empty()) { + return false; + } + *type = kDescriptorFile; + *number = num; + } else if (rest.starts_with("METADB-")) { + rest.remove_prefix(strlen("METADB-")); + uint64_t num; + if (!ConsumeDecimalNumber(&rest, &num)) { + return false; + } + if (!rest.empty()) { + return false; + } + *type = kMetaDatabase; + *number = num; + } else if (rest.starts_with(kOptionsFileNamePrefix)) { + uint64_t ts_suffix; + bool is_temp_file = false; + rest.remove_prefix(kOptionsFileNamePrefix.size()); + const std::string kTempFileNameSuffixWithDot = + std::string(".") + kTempFileNameSuffix; + if (rest.ends_with(kTempFileNameSuffixWithDot)) { + rest.remove_suffix(kTempFileNameSuffixWithDot.size()); + is_temp_file = true; + } + if (!ConsumeDecimalNumber(&rest, &ts_suffix)) { + return false; + } + *number = ts_suffix; + *type = is_temp_file ? kTempFile : kOptionsFile; + } else { + // Avoid strtoull() to keep filename format independent of the + // current locale + bool archive_dir_found = false; + if (rest.starts_with(kArchivalDirName)) { + if (rest.size() <= kArchivalDirName.size()) { + return false; + } + rest.remove_prefix(kArchivalDirName.size() + + 1); // Add 1 to remove / also + if (log_type) { + *log_type = kArchivedLogFile; + } + archive_dir_found = true; + } + uint64_t num; + if (!ConsumeDecimalNumber(&rest, &num)) { + return false; + } + if (rest.size() <= 1 || rest[0] != '.') { + return false; + } + rest.remove_prefix(1); + + Slice suffix = rest; + if (suffix == Slice("log")) { + *type = kWalFile; + if (log_type && !archive_dir_found) { + *log_type = kAliveLogFile; + } + } else if (archive_dir_found) { + return false; // Archive dir can contain only log files + } else if (suffix == Slice(kRocksDbTFileExt) || + suffix == Slice(kLevelDbTFileExt)) { + *type = kTableFile; + } else if (suffix == Slice(kRocksDBBlobFileExt)) { + *type = kBlobFile; + } else if (suffix == Slice(kTempFileNameSuffix)) { + *type = kTempFile; + } else { + return false; + } + *number = num; + } + return true; +} + +IOStatus SetCurrentFile(FileSystem* fs, const std::string& dbname, + uint64_t descriptor_number, + FSDirectory* dir_contains_current_file) { + // Remove leading "dbname/" and add newline to manifest file name + std::string manifest = DescriptorFileName(dbname, descriptor_number); + Slice contents = manifest; + assert(contents.starts_with(dbname + "/")); + contents.remove_prefix(dbname.size() + 1); + std::string tmp = TempFileName(dbname, descriptor_number); + IOStatus s = WriteStringToFile(fs, contents.ToString() + "\n", tmp, true); + TEST_SYNC_POINT_CALLBACK("SetCurrentFile:BeforeRename", &s); + if (s.ok()) { + TEST_KILL_RANDOM_WITH_WEIGHT("SetCurrentFile:0", REDUCE_ODDS2); + s = fs->RenameFile(tmp, CurrentFileName(dbname), IOOptions(), nullptr); + TEST_KILL_RANDOM_WITH_WEIGHT("SetCurrentFile:1", REDUCE_ODDS2); + TEST_SYNC_POINT_CALLBACK("SetCurrentFile:AfterRename", &s); + } + if (s.ok()) { + if (dir_contains_current_file != nullptr) { + s = dir_contains_current_file->FsyncWithDirOptions( + IOOptions(), nullptr, DirFsyncOptions(CurrentFileName(dbname))); + } + } else { + fs->DeleteFile(tmp, IOOptions(), nullptr) + .PermitUncheckedError(); // NOTE: PermitUncheckedError is acceptable + // here as we are already handling an error + // case, and this is just a best-attempt + // effort at some cleanup + } + return s; +} + +Status SetIdentityFile(Env* env, const std::string& dbname, + const std::string& db_id) { + std::string id; + if (db_id.empty()) { + id = env->GenerateUniqueId(); + } else { + id = db_id; + } + assert(!id.empty()); + // Reserve the filename dbname/000000.dbtmp for the temporary identity file + std::string tmp = TempFileName(dbname, 0); + std::string identify_file_name = IdentityFileName(dbname); + Status s = WriteStringToFile(env, id, tmp, true); + if (s.ok()) { + s = env->RenameFile(tmp, identify_file_name); + } + std::unique_ptr dir_obj; + if (s.ok()) { + s = env->GetFileSystem()->NewDirectory(dbname, IOOptions(), &dir_obj, + nullptr); + } + if (s.ok()) { + s = dir_obj->FsyncWithDirOptions(IOOptions(), nullptr, + DirFsyncOptions(identify_file_name)); + } + + // The default Close() could return "NotSupported" and we bypass it + // if it is not impelmented. Detailed explanations can be found in + // db/db_impl/db_impl.h + if (s.ok()) { + Status temp_s = dir_obj->Close(IOOptions(), nullptr); + if (!temp_s.ok()) { + if (temp_s.IsNotSupported()) { + temp_s.PermitUncheckedError(); + } else { + s = temp_s; + } + } + } + if (!s.ok()) { + env->DeleteFile(tmp).PermitUncheckedError(); + } + return s; +} + +IOStatus SyncManifest(const ImmutableDBOptions* db_options, + WritableFileWriter* file) { + TEST_KILL_RANDOM_WITH_WEIGHT("SyncManifest:0", REDUCE_ODDS2); + StopWatch sw(db_options->clock, db_options->stats, MANIFEST_FILE_SYNC_MICROS); + return file->Sync(db_options->use_fsync); +} + +Status GetInfoLogFiles(const std::shared_ptr& fs, + const std::string& db_log_dir, const std::string& dbname, + std::string* parent_dir, + std::vector* info_log_list) { + assert(parent_dir != nullptr); + assert(info_log_list != nullptr); + uint64_t number = 0; + FileType type = kWalFile; + + if (!db_log_dir.empty()) { + *parent_dir = db_log_dir; + } else { + *parent_dir = dbname; + } + + InfoLogPrefix info_log_prefix(!db_log_dir.empty(), dbname); + + std::vector file_names; + Status s = fs->GetChildren(*parent_dir, IOOptions(), &file_names, nullptr); + + if (!s.ok()) { + return s; + } + + for (auto& f : file_names) { + if (ParseFileName(f, &number, info_log_prefix.prefix, &type) && + (type == kInfoLogFile)) { + info_log_list->push_back(f); + } + } + return Status::OK(); +} + +std::string NormalizePath(const std::string& path) { + std::string dst; + + if (path.length() > 2 && path[0] == kFilePathSeparator && + path[1] == kFilePathSeparator) { // Handle UNC names + dst.append(2, kFilePathSeparator); + } + + for (auto c : path) { + if (!dst.empty() && (c == kFilePathSeparator || c == '/') && + (dst.back() == kFilePathSeparator || dst.back() == '/')) { + continue; + } + dst.push_back(c); + } + return dst; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/file/filename.h b/librocksdb-sys/rocksdb/file/filename.h new file mode 100644 index 0000000..2eb125b --- /dev/null +++ b/librocksdb-sys/rocksdb/file/filename.h @@ -0,0 +1,188 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// File names used by DB code + +#pragma once +#include + +#include +#include +#include + +#include "options/db_options.h" +#include "port/port.h" +#include "rocksdb/file_system.h" +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "rocksdb/transaction_log.h" + +namespace ROCKSDB_NAMESPACE { + +class Env; +class Directory; +class SystemClock; +class WritableFileWriter; + +#ifdef OS_WIN +constexpr char kFilePathSeparator = '\\'; +#else +constexpr char kFilePathSeparator = '/'; +#endif + +// Return the name of the log file with the specified number +// in the db named by "dbname". The result will be prefixed with +// "dbname". +extern std::string LogFileName(const std::string& dbname, uint64_t number); + +extern std::string LogFileName(uint64_t number); + +extern std::string BlobFileName(uint64_t number); + +extern std::string BlobFileName(const std::string& bdirname, uint64_t number); + +extern std::string BlobFileName(const std::string& dbname, + const std::string& blob_dir, uint64_t number); + +extern std::string ArchivalDirectory(const std::string& dbname); + +// Return the name of the archived log file with the specified number +// in the db named by "dbname". The result will be prefixed with "dbname". +extern std::string ArchivedLogFileName(const std::string& dbname, uint64_t num); + +extern std::string MakeTableFileName(const std::string& name, uint64_t number); + +extern std::string MakeTableFileName(uint64_t number); + +// Return the name of sstable with LevelDB suffix +// created from RocksDB sstable suffixed name +extern std::string Rocks2LevelTableFileName(const std::string& fullname); + +// the reverse function of MakeTableFileName +// TODO(yhchiang): could merge this function with ParseFileName() +extern uint64_t TableFileNameToNumber(const std::string& name); + +// Return the name of the sstable with the specified number +// in the db named by "dbname". The result will be prefixed with +// "dbname". +extern std::string TableFileName(const std::vector& db_paths, + uint64_t number, uint32_t path_id); + +// Sufficient buffer size for FormatFileNumber. +const size_t kFormatFileNumberBufSize = 38; + +extern void FormatFileNumber(uint64_t number, uint32_t path_id, char* out_buf, + size_t out_buf_size); + +// Return the name of the descriptor file for the db named by +// "dbname" and the specified incarnation number. The result will be +// prefixed with "dbname". +extern std::string DescriptorFileName(const std::string& dbname, + uint64_t number); + +extern std::string DescriptorFileName(uint64_t number); + +extern const std::string kCurrentFileName; // = "CURRENT" + +// Return the name of the current file. This file contains the name +// of the current manifest file. The result will be prefixed with +// "dbname". +extern std::string CurrentFileName(const std::string& dbname); + +// Return the name of the lock file for the db named by +// "dbname". The result will be prefixed with "dbname". +extern std::string LockFileName(const std::string& dbname); + +// Return the name of a temporary file owned by the db named "dbname". +// The result will be prefixed with "dbname". +extern std::string TempFileName(const std::string& dbname, uint64_t number); + +// A helper structure for prefix of info log names. +struct InfoLogPrefix { + char buf[260]; + Slice prefix; + // Prefix with DB absolute path encoded + explicit InfoLogPrefix(bool has_log_dir, const std::string& db_absolute_path); + // Default Prefix + explicit InfoLogPrefix(); +}; + +// Return the name of the info log file for "dbname". +extern std::string InfoLogFileName(const std::string& dbname, + const std::string& db_path = "", + const std::string& log_dir = ""); + +// Return the name of the old info log file for "dbname". +extern std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts, + const std::string& db_path = "", + const std::string& log_dir = ""); + +extern const std::string kOptionsFileNamePrefix; // = "OPTIONS-" +extern const std::string kTempFileNameSuffix; // = "dbtmp" + +// Return a options file name given the "dbname" and file number. +// Format: OPTIONS-[number].dbtmp +extern std::string OptionsFileName(const std::string& dbname, + uint64_t file_num); +extern std::string OptionsFileName(uint64_t file_num); + +// Return a temp options file name given the "dbname" and file number. +// Format: OPTIONS-[number] +extern std::string TempOptionsFileName(const std::string& dbname, + uint64_t file_num); + +// Return the name to use for a metadatabase. The result will be prefixed with +// "dbname". +extern std::string MetaDatabaseName(const std::string& dbname, uint64_t number); + +// Return the name of the Identity file which stores a unique number for the db +// that will get regenerated if the db loses all its data and is recreated fresh +// either from a backup-image or empty +extern std::string IdentityFileName(const std::string& dbname); + +// If filename is a rocksdb file, store the type of the file in *type. +// The number encoded in the filename is stored in *number. If the +// filename was successfully parsed, returns true. Else return false. +// info_log_name_prefix is the path of info logs. +extern bool ParseFileName(const std::string& filename, uint64_t* number, + const Slice& info_log_name_prefix, FileType* type, + WalFileType* log_type = nullptr); +// Same as previous function, but skip info log files. +extern bool ParseFileName(const std::string& filename, uint64_t* number, + FileType* type, WalFileType* log_type = nullptr); + +// Make the CURRENT file point to the descriptor file with the +// specified number. On its success and when dir_contains_current_file is not +// nullptr, the function will fsync the directory containing the CURRENT file +// when +extern IOStatus SetCurrentFile(FileSystem* fs, const std::string& dbname, + uint64_t descriptor_number, + FSDirectory* dir_contains_current_file); + +// Make the IDENTITY file for the db +extern Status SetIdentityFile(Env* env, const std::string& dbname, + const std::string& db_id = {}); + +// Sync manifest file `file`. +extern IOStatus SyncManifest(const ImmutableDBOptions* db_options, + WritableFileWriter* file); + +// Return list of file names of info logs in `file_names`. +// The list only contains file name. The parent directory name is stored +// in `parent_dir`. +// `db_log_dir` should be the one as in options.db_log_dir +extern Status GetInfoLogFiles(const std::shared_ptr& fs, + const std::string& db_log_dir, + const std::string& dbname, + std::string* parent_dir, + std::vector* file_names); + +extern std::string NormalizePath(const std::string& path); +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/file/line_file_reader.cc b/librocksdb-sys/rocksdb/file/line_file_reader.cc new file mode 100644 index 0000000..50c415d --- /dev/null +++ b/librocksdb-sys/rocksdb/file/line_file_reader.cc @@ -0,0 +1,73 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "file/line_file_reader.h" + +#include + +#include "monitoring/iostats_context_imp.h" + +namespace ROCKSDB_NAMESPACE { + +IOStatus LineFileReader::Create(const std::shared_ptr& fs, + const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* reader, + IODebugContext* dbg, + RateLimiter* rate_limiter) { + std::unique_ptr file; + IOStatus io_s = fs->NewSequentialFile(fname, file_opts, &file, dbg); + if (io_s.ok()) { + reader->reset(new LineFileReader( + std::move(file), fname, nullptr, + std::vector>{}, rate_limiter)); + } + return io_s; +} + +bool LineFileReader::ReadLine(std::string* out, + Env::IOPriority rate_limiter_priority) { + assert(out); + if (!io_status_.ok()) { + // Status should be checked (or permit unchecked) any time we return false. + io_status_.MustCheck(); + return false; + } + out->clear(); + for (;;) { + // Look for line delimiter + const char* found = static_cast( + std::memchr(buf_begin_, '\n', buf_end_ - buf_begin_)); + if (found) { + size_t len = found - buf_begin_; + out->append(buf_begin_, len); + buf_begin_ += len + /*delim*/ 1; + ++line_number_; + return true; + } + if (at_eof_) { + io_status_.MustCheck(); + return false; + } + // else flush and reload buffer + out->append(buf_begin_, buf_end_ - buf_begin_); + Slice result; + io_status_ = + sfr_.Read(buf_.size(), &result, buf_.data(), rate_limiter_priority); + IOSTATS_ADD(bytes_read, result.size()); + if (!io_status_.ok()) { + io_status_.MustCheck(); + return false; + } + if (result.size() != buf_.size()) { + // The obscure way of indicating EOF + at_eof_ = true; + } + buf_begin_ = result.data(); + buf_end_ = result.data() + result.size(); + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/file/line_file_reader.h b/librocksdb-sys/rocksdb/file/line_file_reader.h new file mode 100644 index 0000000..cc302d3 --- /dev/null +++ b/librocksdb-sys/rocksdb/file/line_file_reader.h @@ -0,0 +1,60 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once +#include + +#include "file/sequence_file_reader.h" + +namespace ROCKSDB_NAMESPACE { + +// A wrapper on top of Env::SequentialFile for reading text lines from a file. +// Lines are delimited by '\n'. The last line may or may not include a +// trailing newline. Uses SequentialFileReader internally. +class LineFileReader { + private: + std::array buf_; + SequentialFileReader sfr_; + IOStatus io_status_; + const char* buf_begin_ = buf_.data(); + const char* buf_end_ = buf_.data(); + size_t line_number_ = 0; + bool at_eof_ = false; + + public: + // See SequentialFileReader constructors + template + explicit LineFileReader(Args&&... args) + : sfr_(std::forward(args)...) {} + + static IOStatus Create(const std::shared_ptr& fs, + const std::string& fname, const FileOptions& file_opts, + std::unique_ptr* reader, + IODebugContext* dbg, RateLimiter* rate_limiter); + + LineFileReader(const LineFileReader&) = delete; + LineFileReader& operator=(const LineFileReader&) = delete; + + // Reads another line from the file, returning true on success and saving + // the line to `out`, without delimiter, or returning false on failure. You + // must check GetStatus() to determine whether the failure was just + // end-of-file (OK status) or an I/O error (another status). + // The internal rate limiter will be charged at the specified priority. + bool ReadLine(std::string* out, Env::IOPriority rate_limiter_priority); + + // Returns the number of the line most recently returned from ReadLine. + // Return value is unspecified if ReadLine has returned false due to + // I/O error. After ReadLine returns false due to end-of-file, return + // value is the last returned line number, or equivalently the total + // number of lines returned. + size_t GetLineNumber() const { return line_number_; } + + // Returns any error encountered during read. The error is considered + // permanent and no retry or recovery is attempted with the same + // LineFileReader. + const IOStatus& GetStatus() const { return io_status_; } +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/file/prefetch_test.cc b/librocksdb-sys/rocksdb/file/prefetch_test.cc new file mode 100644 index 0000000..15b1c6b --- /dev/null +++ b/librocksdb-sys/rocksdb/file/prefetch_test.cc @@ -0,0 +1,2670 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/db_test_util.h" +#include "file/file_prefetch_buffer.h" +#include "file/file_util.h" +#include "rocksdb/file_system.h" +#include "test_util/sync_point.h" +#ifdef GFLAGS +#include "tools/io_tracer_parser_tool.h" +#endif +#include "util/random.h" + +namespace { +static bool enable_io_uring = true; +extern "C" bool RocksDbIOUringEnable() { return enable_io_uring; } +} // namespace + +namespace ROCKSDB_NAMESPACE { + +class MockFS; + +class MockRandomAccessFile : public FSRandomAccessFileOwnerWrapper { + public: + MockRandomAccessFile(std::unique_ptr& file, + bool support_prefetch, std::atomic_int& prefetch_count, + bool small_buffer_alignment = false) + : FSRandomAccessFileOwnerWrapper(std::move(file)), + support_prefetch_(support_prefetch), + prefetch_count_(prefetch_count), + small_buffer_alignment_(small_buffer_alignment) {} + + IOStatus Prefetch(uint64_t offset, size_t n, const IOOptions& options, + IODebugContext* dbg) override { + if (support_prefetch_) { + prefetch_count_.fetch_add(1); + return target()->Prefetch(offset, n, options, dbg); + } else { + return IOStatus::NotSupported("Prefetch not supported"); + } + } + + size_t GetRequiredBufferAlignment() const override { + return small_buffer_alignment_ + ? 1 + : FSRandomAccessFileOwnerWrapper::GetRequiredBufferAlignment(); + } + + private: + const bool support_prefetch_; + std::atomic_int& prefetch_count_; + const bool small_buffer_alignment_; +}; + +class MockFS : public FileSystemWrapper { + public: + explicit MockFS(const std::shared_ptr& wrapped, + bool support_prefetch, bool small_buffer_alignment = false) + : FileSystemWrapper(wrapped), + support_prefetch_(support_prefetch), + small_buffer_alignment_(small_buffer_alignment) {} + + static const char* kClassName() { return "MockFS"; } + const char* Name() const override { return kClassName(); } + + IOStatus NewRandomAccessFile(const std::string& fname, + const FileOptions& opts, + std::unique_ptr* result, + IODebugContext* dbg) override { + std::unique_ptr file; + IOStatus s; + s = target()->NewRandomAccessFile(fname, opts, &file, dbg); + result->reset(new MockRandomAccessFile( + file, support_prefetch_, prefetch_count_, small_buffer_alignment_)); + return s; + } + + void ClearPrefetchCount() { prefetch_count_ = 0; } + + bool IsPrefetchCalled() { return prefetch_count_ > 0; } + + int GetPrefetchCount() { + return prefetch_count_.load(std::memory_order_relaxed); + } + + private: + const bool support_prefetch_; + const bool small_buffer_alignment_; + std::atomic_int prefetch_count_{0}; +}; + +class PrefetchTest + : public DBTestBase, + public ::testing::WithParamInterface> { + public: + PrefetchTest() : DBTestBase("prefetch_test", true) {} + + virtual void SetGenericOptions(Env* env, bool use_direct_io, + Options& options) { + options = CurrentOptions(); + options.write_buffer_size = 1024; + options.create_if_missing = true; + options.compression = kNoCompression; + options.env = env; + options.disable_auto_compactions = true; + if (use_direct_io) { + options.use_direct_reads = true; + options.use_direct_io_for_flush_and_compaction = true; + } + } + + void SetBlockBasedTableOptions(BlockBasedTableOptions& table_options) { + table_options.no_block_cache = true; + table_options.cache_index_and_filter_blocks = false; + table_options.metadata_block_size = 1024; + table_options.index_type = + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + } +}; + +INSTANTIATE_TEST_CASE_P(PrefetchTest, PrefetchTest, + ::testing::Combine(::testing::Bool(), + ::testing::Bool())); + +std::string BuildKey(int num, std::string postfix = "") { + return "my_key_" + std::to_string(num) + postfix; +} + +// This test verifies the following basic functionalities of prefetching: +// (1) If underline file system supports prefetch, and directIO is not enabled +// make sure prefetch() is called and FilePrefetchBuffer is not used. +// (2) If underline file system doesn't support prefetch, or directIO is +// enabled, make sure prefetch() is not called and FilePrefetchBuffer is +// used. +// (3) Measure read bytes, hit and miss of SST's tail prefetching during table +// open. +TEST_P(PrefetchTest, Basic) { + // First param is if the mockFS support_prefetch or not + bool support_prefetch = + std::get<0>(GetParam()) && + test::IsPrefetchSupported(env_->GetFileSystem(), dbname_); + std::shared_ptr fs = + std::make_shared(env_->GetFileSystem(), support_prefetch); + + // Second param is if directIO is enabled or not + bool use_direct_io = std::get<1>(GetParam()); + + std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); + Options options; + SetGenericOptions(env.get(), use_direct_io, options); + options.statistics = CreateDBStatistics(); + + const int kNumKeys = 1100; + int buff_prefetch_count = 0; + SyncPoint::GetInstance()->SetCallBack("FilePrefetchBuffer::Prefetch:Start", + [&](void*) { buff_prefetch_count++; }); + SyncPoint::GetInstance()->EnableProcessing(); + + Status s = TryReopen(options); + if (use_direct_io && (s.IsNotSupported() || s.IsInvalidArgument())) { + // If direct IO is not supported, skip the test + return; + } else { + ASSERT_OK(s); + } + + // create first key range + WriteBatch batch; + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(batch.Put(BuildKey(i), "value for range 1 key")); + } + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + + // create second key range + batch.Clear(); + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(batch.Put(BuildKey(i, "key2"), "value for range 2 key")); + } + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + + // delete second key range + batch.Clear(); + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(batch.Delete(BuildKey(i, "key2"))); + } + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + ASSERT_OK(db_->Flush(FlushOptions())); + + // To verify SST file tail prefetch (once per file) during flush output + // verification + if (support_prefetch && !use_direct_io) { + ASSERT_TRUE(fs->IsPrefetchCalled()); + ASSERT_EQ(3, fs->GetPrefetchCount()); + ASSERT_EQ(0, buff_prefetch_count); + fs->ClearPrefetchCount(); + } else { + ASSERT_FALSE(fs->IsPrefetchCalled()); + ASSERT_EQ(buff_prefetch_count, 3); + buff_prefetch_count = 0; + } + + // compact database + std::string start_key = BuildKey(0); + std::string end_key = BuildKey(kNumKeys - 1); + Slice least(start_key.data(), start_key.size()); + Slice greatest(end_key.data(), end_key.size()); + + HistogramData prev_table_open_prefetch_tail_read; + options.statistics->histogramData(TABLE_OPEN_PREFETCH_TAIL_READ_BYTES, + &prev_table_open_prefetch_tail_read); + const uint64_t prev_table_open_prefetch_tail_miss = + options.statistics->getTickerCount(TABLE_OPEN_PREFETCH_TAIL_MISS); + const uint64_t prev_table_open_prefetch_tail_hit = + options.statistics->getTickerCount(TABLE_OPEN_PREFETCH_TAIL_HIT); + + // commenting out the line below causes the example to work correctly + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &least, &greatest)); + + HistogramData cur_table_open_prefetch_tail_read; + options.statistics->histogramData(TABLE_OPEN_PREFETCH_TAIL_READ_BYTES, + &cur_table_open_prefetch_tail_read); + const uint64_t cur_table_open_prefetch_tail_miss = + options.statistics->getTickerCount(TABLE_OPEN_PREFETCH_TAIL_MISS); + const uint64_t cur_table_open_prefetch_tail_hit = + options.statistics->getTickerCount(TABLE_OPEN_PREFETCH_TAIL_HIT); + + // To verify prefetch during compaction input read + if (support_prefetch && !use_direct_io) { + ASSERT_TRUE(fs->IsPrefetchCalled()); + // To rule out false positive by the SST file tail prefetch during + // compaction output verification + ASSERT_GT(fs->GetPrefetchCount(), 1); + ASSERT_EQ(0, buff_prefetch_count); + fs->ClearPrefetchCount(); + } else { + ASSERT_FALSE(fs->IsPrefetchCalled()); + // To rule out false positive by the SST file tail prefetch during + // compaction output verification + ASSERT_GT(buff_prefetch_count, 1); + buff_prefetch_count = 0; + + ASSERT_GT(cur_table_open_prefetch_tail_read.count, + prev_table_open_prefetch_tail_read.count); + ASSERT_GT(cur_table_open_prefetch_tail_hit, + prev_table_open_prefetch_tail_hit); + ASSERT_GE(cur_table_open_prefetch_tail_miss, + prev_table_open_prefetch_tail_miss); + } + + // count the keys + { + auto iter = std::unique_ptr(db_->NewIterator(ReadOptions())); + int num_keys = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + num_keys++; + } + (void)num_keys; + } + + // To verify prefetch during user scan + if (support_prefetch && !use_direct_io) { + ASSERT_TRUE(fs->IsPrefetchCalled()); + fs->ClearPrefetchCount(); + ASSERT_EQ(0, buff_prefetch_count); + } else { + ASSERT_FALSE(fs->IsPrefetchCalled()); + ASSERT_GT(buff_prefetch_count, 0); + buff_prefetch_count = 0; + } + Close(); +} + +class PrefetchTailTest : public PrefetchTest { + public: + bool SupportPrefetch() const { + return std::get<0>(GetParam()) && + test::IsPrefetchSupported(env_->GetFileSystem(), dbname_); + } + + bool UseDirectIO() const { return std::get<1>(GetParam()); } + + bool UseFilePrefetchBuffer() const { + return !SupportPrefetch() || UseDirectIO(); + } + + Env* GetEnv(bool small_buffer_alignment = false) const { + std::shared_ptr fs = std::make_shared( + env_->GetFileSystem(), SupportPrefetch(), small_buffer_alignment); + + return new CompositeEnvWrapper(env_, fs); + } + + void SetGenericOptions(Env* env, bool use_direct_io, + Options& options) override { + PrefetchTest::SetGenericOptions(env, use_direct_io, options); + options.statistics = CreateDBStatistics(); + } + + void SetBlockBasedTableOptions( + BlockBasedTableOptions& table_options, bool partition_filters = true, + uint64_t metadata_block_size = + BlockBasedTableOptions().metadata_block_size, + bool use_small_cache = false) { + table_options.index_type = BlockBasedTableOptions::kTwoLevelIndexSearch; + table_options.partition_filters = partition_filters; + if (table_options.partition_filters) { + table_options.filter_policy.reset(NewBloomFilterPolicy(10, false)); + } + table_options.metadata_block_size = metadata_block_size; + + if (use_small_cache) { + LRUCacheOptions co; + co.capacity = 1; + std::shared_ptr cache = NewLRUCache(co); + table_options.block_cache = cache; + } + } + + int64_t GetNumIndexPartition() const { + int64_t index_partition_counts = 0; + TablePropertiesCollection all_table_props; + assert(db_->GetPropertiesOfAllTables(&all_table_props).ok()); + for (const auto& name_and_table_props : all_table_props) { + const auto& table_props = name_and_table_props.second; + index_partition_counts += table_props->index_partitions; + } + return index_partition_counts; + } +}; + +INSTANTIATE_TEST_CASE_P(PrefetchTailTest, PrefetchTailTest, + ::testing::Combine(::testing::Bool(), + ::testing::Bool())); + +TEST_P(PrefetchTailTest, Basic) { + std::unique_ptr env(GetEnv()); + Options options; + SetGenericOptions(env.get(), UseDirectIO(), options); + + BlockBasedTableOptions bbto; + SetBlockBasedTableOptions(bbto); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + + Status s = TryReopen(options); + if (UseDirectIO() && (s.IsNotSupported() || s.IsInvalidArgument())) { + // If direct IO is not supported, skip the test + ROCKSDB_GTEST_BYPASS("Direct IO is not supported"); + return; + } else { + ASSERT_OK(s); + } + + ASSERT_OK(Put("k1", "v1")); + + HistogramData pre_flush_file_read; + options.statistics->histogramData(FILE_READ_FLUSH_MICROS, + &pre_flush_file_read); + ASSERT_OK(Flush()); + HistogramData post_flush_file_read; + options.statistics->histogramData(FILE_READ_FLUSH_MICROS, + &post_flush_file_read); + if (UseFilePrefetchBuffer()) { + // `PartitionedFilterBlockReader/PartitionIndexReader::CacheDependencies()` + // should read from the prefetched tail in file prefetch buffer instead of + // initiating extra SST reads. Therefore `BlockBasedTable::PrefetchTail()` + // should be the only SST read in table verification during flush. + ASSERT_EQ(post_flush_file_read.count - pre_flush_file_read.count, 1); + } else { + // Without the prefetched tail in file prefetch buffer, + // `PartitionedFilterBlockReader/PartitionIndexReader::CacheDependencies()` + // will initiate extra SST reads + ASSERT_GT(post_flush_file_read.count - pre_flush_file_read.count, 1); + } + ASSERT_OK(Put("k1", "v2")); + ASSERT_OK(Put("k2", "v2")); + ASSERT_OK(Flush()); + + CompactRangeOptions cro; + HistogramData pre_compaction_file_read; + options.statistics->histogramData(FILE_READ_COMPACTION_MICROS, + &pre_compaction_file_read); + ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr)); + HistogramData post_compaction_file_read; + options.statistics->histogramData(FILE_READ_COMPACTION_MICROS, + &post_compaction_file_read); + if (UseFilePrefetchBuffer()) { + // `PartitionedFilterBlockReader/PartitionIndexReader::CacheDependencies()` + // should read from the prefetched tail in file prefetch buffer instead of + // initiating extra SST reads. + // + // Therefore the 3 reads are + // (1) `ProcessKeyValueCompaction()` of input file 1 + // (2) `ProcessKeyValueCompaction()` of input file 2 + // (3) `BlockBasedTable::PrefetchTail()` of output file during table + // verification in compaction + ASSERT_EQ(post_compaction_file_read.count - pre_compaction_file_read.count, + 3); + } else { + // Without the prefetched tail in file prefetch buffer, + // `PartitionedFilterBlockReader/PartitionIndexReader::CacheDependencies()` + // as well as reading other parts of the tail (e.g, footer, table + // properties..) will initiate extra SST reads + ASSERT_GT(post_compaction_file_read.count - pre_compaction_file_read.count, + 3); + } + Close(); +} + +TEST_P(PrefetchTailTest, UpgradeToTailSizeInManifest) { + if (!UseFilePrefetchBuffer()) { + ROCKSDB_GTEST_BYPASS( + "Upgrade to tail size in manifest is only relevant when RocksDB file " + "prefetch buffer is used."); + } + if (UseDirectIO()) { + ROCKSDB_GTEST_BYPASS( + "To simplify testing logics with setting file's buffer alignment to be " + "1, direct IO is required to be disabled."); + } + + std::unique_ptr env(GetEnv(true /* small_buffer_alignment */)); + Options options; + SetGenericOptions(env.get(), false /* use_direct_io*/, options); + options.max_open_files = -1; + options.write_buffer_size = 1024 * 1024; + + BlockBasedTableOptions table_options; + SetBlockBasedTableOptions(table_options, false /* partition_filters */, + 1 /* metadata_block_size*/, + true /* use_small_cache */); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + SyncPoint::GetInstance()->EnableProcessing(); + // To simulate a pre-upgrade DB where file tail size is not recorded in + // manifest + SyncPoint::GetInstance()->SetCallBack( + "FileMetaData::FileMetaData", [&](void* arg) { + FileMetaData* meta = static_cast(arg); + meta->tail_size = 0; + }); + + ASSERT_OK(TryReopen(options)); + for (int i = 0; i < 10000; ++i) { + ASSERT_OK(Put("k" + std::to_string(i), "v")); + } + ASSERT_OK(Flush()); + + SyncPoint::GetInstance()->ClearAllCallBacks(); + + // To simulate a DB undergoing the upgrade where tail size to prefetch is + // inferred to be a small number for files with no tail size recorded in + // manifest. + // "1" is chosen to be such number so that with `small_buffer_alignment == + // true` and `use_small_cache == true`, it would have caused one file read per + // index partition during db open if the upgrade is done wrong. + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTable::Open::TailPrefetchLen", [&](void* arg) { + std::pair* prefetch_off_len_pair = + static_cast*>(arg); + size_t* prefetch_off = prefetch_off_len_pair->first; + size_t* tail_size = prefetch_off_len_pair->second; + const size_t file_size = *prefetch_off + *tail_size; + + *tail_size = 1; + *prefetch_off = file_size - (*tail_size); + }); + + ASSERT_OK(TryReopen(options)); + + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->DisableProcessing(); + + HistogramData db_open_file_read; + options.statistics->histogramData(FILE_READ_DB_OPEN_MICROS, + &db_open_file_read); + + int64_t num_index_partition = GetNumIndexPartition(); + // If the upgrade is done right, db open will prefetch all the index + // partitions at once, instead of doing one read per partition. + // That is, together with `metadata_block_size == 1`, there will be more index + // partitions than number of non index partitions reads. + ASSERT_LT(db_open_file_read.count, num_index_partition); + + Close(); +} + +// This test verifies BlockBasedTableOptions.max_auto_readahead_size is +// configured dynamically. +TEST_P(PrefetchTest, ConfigureAutoMaxReadaheadSize) { + // First param is if the mockFS support_prefetch or not + bool support_prefetch = + std::get<0>(GetParam()) && + test::IsPrefetchSupported(env_->GetFileSystem(), dbname_); + std::shared_ptr fs = + std::make_shared(env_->GetFileSystem(), support_prefetch); + + // Second param is if directIO is enabled or not + bool use_direct_io = std::get<1>(GetParam()); + + std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); + Options options; + SetGenericOptions(env.get(), use_direct_io, options); + BlockBasedTableOptions table_options; + SetBlockBasedTableOptions(table_options); + table_options.max_auto_readahead_size = 0; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + int buff_prefetch_count = 0; + SyncPoint::GetInstance()->SetCallBack("FilePrefetchBuffer::Prefetch:Start", + [&](void*) { buff_prefetch_count++; }); + + // DB open will create table readers unless we reduce the table cache + // capacity. SanitizeOptions will set max_open_files to minimum of 20. Table + // cache is allocated with max_open_files - 10 as capacity. So override + // max_open_files to 10 so table cache capacity will become 0. This will + // prevent file open during DB open and force the file to be opened during + // Iteration. + SyncPoint::GetInstance()->SetCallBack( + "SanitizeOptions::AfterChangeMaxOpenFiles", [&](void* arg) { + int* max_open_files = (int*)arg; + *max_open_files = 11; + }); + + SyncPoint::GetInstance()->EnableProcessing(); + + Status s = TryReopen(options); + + if (use_direct_io && (s.IsNotSupported() || s.IsInvalidArgument())) { + // If direct IO is not supported, skip the test + return; + } else { + ASSERT_OK(s); + } + + Random rnd(309); + int key_count = 0; + const int num_keys_per_level = 100; + // Level 0 : Keys in range [0, 99], Level 1:[100, 199], Level 2:[200, 299]. + for (int level = 2; level >= 0; level--) { + key_count = level * num_keys_per_level; + for (int i = 0; i < num_keys_per_level; ++i) { + ASSERT_OK(Put(Key(key_count++), rnd.RandomString(500))); + } + ASSERT_OK(Flush()); + MoveFilesToLevel(level); + } + Close(); + std::vector buff_prefectch_level_count = {0, 0, 0}; + TryReopen(options); + { + auto iter = std::unique_ptr(db_->NewIterator(ReadOptions())); + fs->ClearPrefetchCount(); + buff_prefetch_count = 0; + + for (int level = 2; level >= 0; level--) { + key_count = level * num_keys_per_level; + switch (level) { + case 0: + // max_auto_readahead_size is set 0 so data and index blocks are not + // prefetched. + ASSERT_OK(db_->SetOptions( + {{"block_based_table_factory", "{max_auto_readahead_size=0;}"}})); + break; + case 1: + // max_auto_readahead_size is set less than + // initial_auto_readahead_size. So readahead_size remains equal to + // max_auto_readahead_size. + ASSERT_OK(db_->SetOptions({{"block_based_table_factory", + "{max_auto_readahead_size=4096;}"}})); + break; + case 2: + ASSERT_OK(db_->SetOptions({{"block_based_table_factory", + "{max_auto_readahead_size=65536;}"}})); + break; + default: + assert(false); + } + + for (int i = 0; i < num_keys_per_level; ++i) { + iter->Seek(Key(key_count++)); + iter->Next(); + } + + buff_prefectch_level_count[level] = buff_prefetch_count; + if (support_prefetch && !use_direct_io) { + if (level == 0) { + ASSERT_FALSE(fs->IsPrefetchCalled()); + } else { + ASSERT_TRUE(fs->IsPrefetchCalled()); + } + fs->ClearPrefetchCount(); + } else { + ASSERT_FALSE(fs->IsPrefetchCalled()); + if (level == 0) { + ASSERT_EQ(buff_prefetch_count, 0); + } else { + ASSERT_GT(buff_prefetch_count, 0); + } + buff_prefetch_count = 0; + } + } + } + + if (!support_prefetch) { + ASSERT_GT(buff_prefectch_level_count[1], buff_prefectch_level_count[2]); + } + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + Close(); +} + +// This test verifies BlockBasedTableOptions.initial_auto_readahead_size is +// configured dynamically. +TEST_P(PrefetchTest, ConfigureInternalAutoReadaheadSize) { + // First param is if the mockFS support_prefetch or not + bool support_prefetch = + std::get<0>(GetParam()) && + test::IsPrefetchSupported(env_->GetFileSystem(), dbname_); + + // Second param is if directIO is enabled or not + bool use_direct_io = std::get<1>(GetParam()); + + std::shared_ptr fs = + std::make_shared(env_->GetFileSystem(), support_prefetch); + std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); + Options options; + SetGenericOptions(env.get(), use_direct_io, options); + BlockBasedTableOptions table_options; + SetBlockBasedTableOptions(table_options); + table_options.initial_auto_readahead_size = 0; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + int buff_prefetch_count = 0; + // DB open will create table readers unless we reduce the table cache + // capacity. SanitizeOptions will set max_open_files to minimum of 20. + // Table cache is allocated with max_open_files - 10 as capacity. So + // override max_open_files to 10 so table cache capacity will become 0. + // This will prevent file open during DB open and force the file to be + // opened during Iteration. + SyncPoint::GetInstance()->SetCallBack( + "SanitizeOptions::AfterChangeMaxOpenFiles", [&](void* arg) { + int* max_open_files = (int*)arg; + *max_open_files = 11; + }); + + SyncPoint::GetInstance()->SetCallBack("FilePrefetchBuffer::Prefetch:Start", + [&](void*) { buff_prefetch_count++; }); + + SyncPoint::GetInstance()->EnableProcessing(); + + SyncPoint::GetInstance()->EnableProcessing(); + + Status s = TryReopen(options); + + if (use_direct_io && (s.IsNotSupported() || s.IsInvalidArgument())) { + // If direct IO is not supported, skip the test + return; + } else { + ASSERT_OK(s); + } + + Random rnd(309); + int key_count = 0; + const int num_keys_per_level = 100; + // Level 0 : Keys in range [0, 99], Level 1:[100, 199], Level 2:[200, 299]. + for (int level = 2; level >= 0; level--) { + key_count = level * num_keys_per_level; + for (int i = 0; i < num_keys_per_level; ++i) { + ASSERT_OK(Put(Key(key_count++), rnd.RandomString(500))); + } + ASSERT_OK(Flush()); + MoveFilesToLevel(level); + } + Close(); + + TryReopen(options); + { + auto iter = std::unique_ptr(db_->NewIterator(ReadOptions())); + fs->ClearPrefetchCount(); + buff_prefetch_count = 0; + std::vector buff_prefetch_level_count = {0, 0, 0}; + + for (int level = 2; level >= 0; level--) { + key_count = level * num_keys_per_level; + switch (level) { + case 0: + // initial_auto_readahead_size is set 0 so data and index blocks are + // not prefetched. + ASSERT_OK(db_->SetOptions({{"block_based_table_factory", + "{initial_auto_readahead_size=0;}"}})); + break; + case 1: + // intial_auto_readahead_size and max_auto_readahead_size are set same + // so readahead_size remains same. + ASSERT_OK(db_->SetOptions({{"block_based_table_factory", + "{initial_auto_readahead_size=4096;max_" + "auto_readahead_size=4096;}"}})); + break; + case 2: + ASSERT_OK( + db_->SetOptions({{"block_based_table_factory", + "{initial_auto_readahead_size=65536;}"}})); + break; + default: + assert(false); + } + + for (int i = 0; i < num_keys_per_level; ++i) { + iter->Seek(Key(key_count++)); + iter->Next(); + } + + buff_prefetch_level_count[level] = buff_prefetch_count; + if (support_prefetch && !use_direct_io) { + if (level == 0) { + ASSERT_FALSE(fs->IsPrefetchCalled()); + } else { + ASSERT_TRUE(fs->IsPrefetchCalled()); + } + fs->ClearPrefetchCount(); + } else { + ASSERT_FALSE(fs->IsPrefetchCalled()); + if (level == 0) { + ASSERT_EQ(buff_prefetch_count, 0); + } else { + ASSERT_GT(buff_prefetch_count, 0); + } + buff_prefetch_count = 0; + } + } + if (!support_prefetch) { + ASSERT_GT(buff_prefetch_level_count[1], buff_prefetch_level_count[2]); + } + } + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + Close(); +} + +// This test verifies BlockBasedTableOptions.num_file_reads_for_auto_readahead +// is configured dynamically. +TEST_P(PrefetchTest, ConfigureNumFilesReadsForReadaheadSize) { + // First param is if the mockFS support_prefetch or not + bool support_prefetch = + std::get<0>(GetParam()) && + test::IsPrefetchSupported(env_->GetFileSystem(), dbname_); + + const int kNumKeys = 2000; + std::shared_ptr fs = + std::make_shared(env_->GetFileSystem(), support_prefetch); + std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); + + // Second param is if directIO is enabled or not + bool use_direct_io = std::get<1>(GetParam()); + + Options options; + SetGenericOptions(env.get(), use_direct_io, options); + BlockBasedTableOptions table_options; + SetBlockBasedTableOptions(table_options); + table_options.num_file_reads_for_auto_readahead = 0; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + int buff_prefetch_count = 0; + SyncPoint::GetInstance()->SetCallBack("FilePrefetchBuffer::Prefetch:Start", + [&](void*) { buff_prefetch_count++; }); + SyncPoint::GetInstance()->EnableProcessing(); + + Status s = TryReopen(options); + if (use_direct_io && (s.IsNotSupported() || s.IsInvalidArgument())) { + // If direct IO is not supported, skip the test + return; + } else { + ASSERT_OK(s); + } + + WriteBatch batch; + Random rnd(309); + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(batch.Put(BuildKey(i), rnd.RandomString(1000))); + } + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + + std::string start_key = BuildKey(0); + std::string end_key = BuildKey(kNumKeys - 1); + Slice least(start_key.data(), start_key.size()); + Slice greatest(end_key.data(), end_key.size()); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &least, &greatest)); + + Close(); + TryReopen(options); + + fs->ClearPrefetchCount(); + buff_prefetch_count = 0; + + { + auto iter = std::unique_ptr(db_->NewIterator(ReadOptions())); + /* + * Reseek keys from sequential Data Blocks within same partitioned + * index. It will prefetch the data block at the first seek since + * num_file_reads_for_auto_readahead = 0. Data Block size is nearly 4076 so + * readahead will fetch 8 * 1024 data more initially (2 more data blocks). + */ + iter->Seek(BuildKey(0)); // Prefetch data + index block since + // num_file_reads_for_auto_readahead = 0. + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1000)); // In buffer + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1004)); // In buffer + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1008)); // Prefetch Data + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1011)); // In buffer + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1015)); // In buffer + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1019)); // In buffer + ASSERT_TRUE(iter->Valid()); + // Missed 2 blocks but they are already in buffer so no reset. + iter->Seek(BuildKey(103)); // Already in buffer. + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1033)); // Prefetch Data. + ASSERT_TRUE(iter->Valid()); + if (support_prefetch && !use_direct_io) { + ASSERT_EQ(fs->GetPrefetchCount(), 4); + fs->ClearPrefetchCount(); + } else { + ASSERT_EQ(buff_prefetch_count, 4); + buff_prefetch_count = 0; + } + } + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + Close(); +} + +// This test verifies the basic functionality of implicit autoreadahead: +// - Enable implicit autoreadahead and prefetch only if sequential blocks are +// read, +// - If data is already in buffer and few blocks are not requested to read, +// don't reset, +// - If data blocks are sequential during read after enabling implicit +// autoreadahead, reset readahead parameters. +TEST_P(PrefetchTest, PrefetchWhenReseek) { + // First param is if the mockFS support_prefetch or not + bool support_prefetch = + std::get<0>(GetParam()) && + test::IsPrefetchSupported(env_->GetFileSystem(), dbname_); + + const int kNumKeys = 2000; + std::shared_ptr fs = + std::make_shared(env_->GetFileSystem(), support_prefetch); + std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); + + // Second param is if directIO is enabled or not + bool use_direct_io = std::get<1>(GetParam()); + + Options options; + SetGenericOptions(env.get(), use_direct_io, options); + BlockBasedTableOptions table_options; + SetBlockBasedTableOptions(table_options); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + int buff_prefetch_count = 0; + SyncPoint::GetInstance()->SetCallBack("FilePrefetchBuffer::Prefetch:Start", + [&](void*) { buff_prefetch_count++; }); + SyncPoint::GetInstance()->EnableProcessing(); + + Status s = TryReopen(options); + if (use_direct_io && (s.IsNotSupported() || s.IsInvalidArgument())) { + // If direct IO is not supported, skip the test + return; + } else { + ASSERT_OK(s); + } + + WriteBatch batch; + Random rnd(309); + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(batch.Put(BuildKey(i), rnd.RandomString(1000))); + } + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + + std::string start_key = BuildKey(0); + std::string end_key = BuildKey(kNumKeys - 1); + Slice least(start_key.data(), start_key.size()); + Slice greatest(end_key.data(), end_key.size()); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &least, &greatest)); + + fs->ClearPrefetchCount(); + buff_prefetch_count = 0; + + { + auto iter = std::unique_ptr(db_->NewIterator(ReadOptions())); + /* + * Reseek keys from sequential Data Blocks within same partitioned + * index. After 2 sequential reads it will prefetch the data block. + * Data Block size is nearly 4076 so readahead will fetch 8 * 1024 data more + * initially (2 more data blocks). + */ + iter->Seek(BuildKey(0)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1000)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1004)); // Prefetch Data + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1008)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1011)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1015)); // Prefetch Data + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1019)); + ASSERT_TRUE(iter->Valid()); + // Missed 2 blocks but they are already in buffer so no reset. + iter->Seek(BuildKey(103)); // Already in buffer. + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1033)); // Prefetch Data + ASSERT_TRUE(iter->Valid()); + if (support_prefetch && !use_direct_io) { + ASSERT_EQ(fs->GetPrefetchCount(), 3); + fs->ClearPrefetchCount(); + } else { + ASSERT_EQ(buff_prefetch_count, 3); + buff_prefetch_count = 0; + } + } + { + /* + * Reseek keys from non sequential data blocks within same partitioned + * index. buff_prefetch_count will be 0 in that case. + */ + auto iter = std::unique_ptr(db_->NewIterator(ReadOptions())); + iter->Seek(BuildKey(0)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1008)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1019)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1033)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1048)); + ASSERT_TRUE(iter->Valid()); + if (support_prefetch && !use_direct_io) { + ASSERT_EQ(fs->GetPrefetchCount(), 0); + fs->ClearPrefetchCount(); + } else { + ASSERT_EQ(buff_prefetch_count, 0); + buff_prefetch_count = 0; + } + } + { + /* + * Reesek keys from Single Data Block. + */ + auto iter = std::unique_ptr(db_->NewIterator(ReadOptions())); + iter->Seek(BuildKey(0)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(10)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(100)); + ASSERT_TRUE(iter->Valid()); + if (support_prefetch && !use_direct_io) { + ASSERT_EQ(fs->GetPrefetchCount(), 0); + fs->ClearPrefetchCount(); + } else { + ASSERT_EQ(buff_prefetch_count, 0); + buff_prefetch_count = 0; + } + } + { + /* + * Reseek keys from sequential data blocks to set implicit auto readahead + * and prefetch data but after that iterate over different (non sequential) + * data blocks which won't prefetch any data further. So buff_prefetch_count + * will be 1 for the first one. + */ + auto iter = std::unique_ptr(db_->NewIterator(ReadOptions())); + iter->Seek(BuildKey(0)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1000)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1004)); // This iteration will prefetch buffer + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1008)); + ASSERT_TRUE(iter->Valid()); + iter->Seek( + BuildKey(996)); // Reseek won't prefetch any data and + // readahead_size will be initiallized to 8*1024. + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(992)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(989)); + ASSERT_TRUE(iter->Valid()); + if (support_prefetch && !use_direct_io) { + ASSERT_EQ(fs->GetPrefetchCount(), 1); + fs->ClearPrefetchCount(); + } else { + ASSERT_EQ(buff_prefetch_count, 1); + buff_prefetch_count = 0; + } + + // Read sequentially to confirm readahead_size is reset to initial value (2 + // more data blocks) + iter->Seek(BuildKey(1011)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1015)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1019)); // Prefetch Data + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1022)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1026)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(103)); // Prefetch Data + ASSERT_TRUE(iter->Valid()); + if (support_prefetch && !use_direct_io) { + ASSERT_EQ(fs->GetPrefetchCount(), 2); + fs->ClearPrefetchCount(); + } else { + ASSERT_EQ(buff_prefetch_count, 2); + buff_prefetch_count = 0; + } + } + { + /* Reseek keys from sequential partitioned index block. Since partitioned + * index fetch are sequential, buff_prefetch_count will be 1. + */ + auto iter = std::unique_ptr(db_->NewIterator(ReadOptions())); + iter->Seek(BuildKey(0)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1167)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1334)); // This iteration will prefetch buffer + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1499)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1667)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1847)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1999)); + ASSERT_TRUE(iter->Valid()); + if (support_prefetch && !use_direct_io) { + ASSERT_EQ(fs->GetPrefetchCount(), 1); + fs->ClearPrefetchCount(); + } else { + ASSERT_EQ(buff_prefetch_count, 1); + buff_prefetch_count = 0; + } + } + { + /* + * Reseek over different keys from different blocks. buff_prefetch_count is + * set 0. + */ + auto iter = std::unique_ptr(db_->NewIterator(ReadOptions())); + int i = 0; + int j = 1000; + do { + iter->Seek(BuildKey(i)); + if (!iter->Valid()) { + break; + } + i = i + 100; + iter->Seek(BuildKey(j)); + j = j + 100; + } while (i < 1000 && j < kNumKeys && iter->Valid()); + if (support_prefetch && !use_direct_io) { + ASSERT_EQ(fs->GetPrefetchCount(), 0); + fs->ClearPrefetchCount(); + } else { + ASSERT_EQ(buff_prefetch_count, 0); + buff_prefetch_count = 0; + } + } + { + /* Iterates sequentially over all keys. It will prefetch the buffer.*/ + auto iter = std::unique_ptr(db_->NewIterator(ReadOptions())); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + } + if (support_prefetch && !use_direct_io) { + ASSERT_EQ(fs->GetPrefetchCount(), 13); + fs->ClearPrefetchCount(); + } else { + ASSERT_EQ(buff_prefetch_count, 13); + buff_prefetch_count = 0; + } + } + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + Close(); +} + +// This test verifies the functionality of implicit autoreadahead when caching +// is enabled: +// - If data is already in buffer and few blocks are not requested to read, +// don't reset, +// - If block was eligible for prefetching/in buffer but found in cache, don't +// prefetch and reset. +TEST_P(PrefetchTest, PrefetchWhenReseekwithCache) { + // First param is if the mockFS support_prefetch or not + bool support_prefetch = + std::get<0>(GetParam()) && + test::IsPrefetchSupported(env_->GetFileSystem(), dbname_); + + const int kNumKeys = 2000; + std::shared_ptr fs = + std::make_shared(env_->GetFileSystem(), support_prefetch); + std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); + + // Second param is if directIO is enabled or not + bool use_direct_io = std::get<1>(GetParam()); + + Options options; + SetGenericOptions(env.get(), use_direct_io, options); + BlockBasedTableOptions table_options; + SetBlockBasedTableOptions(table_options); + std::shared_ptr cache = NewLRUCache(4 * 1024 * 1024, 2); // 8MB + table_options.block_cache = cache; + table_options.no_block_cache = false; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + int buff_prefetch_count = 0; + SyncPoint::GetInstance()->SetCallBack("FilePrefetchBuffer::Prefetch:Start", + [&](void*) { buff_prefetch_count++; }); + SyncPoint::GetInstance()->EnableProcessing(); + + Status s = TryReopen(options); + if (use_direct_io && (s.IsNotSupported() || s.IsInvalidArgument())) { + // If direct IO is not supported, skip the test + return; + } else { + ASSERT_OK(s); + } + + WriteBatch batch; + Random rnd(309); + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(batch.Put(BuildKey(i), rnd.RandomString(1000))); + } + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + + std::string start_key = BuildKey(0); + std::string end_key = BuildKey(kNumKeys - 1); + Slice least(start_key.data(), start_key.size()); + Slice greatest(end_key.data(), end_key.size()); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &least, &greatest)); + + fs->ClearPrefetchCount(); + buff_prefetch_count = 0; + + { + /* + * Reseek keys from sequential Data Blocks within same partitioned + * index. After 2 sequential reads it will prefetch the data block. + * Data Block size is nearly 4076 so readahead will fetch 8 * 1024 data more + * initially (2 more data blocks). + */ + auto iter = std::unique_ptr(db_->NewIterator(ReadOptions())); + // Warm up the cache + iter->Seek(BuildKey(1011)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1015)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1019)); + ASSERT_TRUE(iter->Valid()); + if (support_prefetch && !use_direct_io) { + ASSERT_EQ(fs->GetPrefetchCount(), 1); + fs->ClearPrefetchCount(); + } else { + ASSERT_EQ(buff_prefetch_count, 1); + buff_prefetch_count = 0; + } + } + { + // After caching, blocks will be read from cache (Sequential blocks) + auto iter = std::unique_ptr(db_->NewIterator(ReadOptions())); + iter->Seek(BuildKey(0)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1000)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1004)); // Prefetch data (not in cache). + ASSERT_TRUE(iter->Valid()); + // Missed one sequential block but next is in already in buffer so readahead + // will not be reset. + iter->Seek(BuildKey(1011)); + ASSERT_TRUE(iter->Valid()); + // Prefetch data but blocks are in cache so no prefetch and reset. + iter->Seek(BuildKey(1015)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1019)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1022)); + ASSERT_TRUE(iter->Valid()); + // Prefetch data with readahead_size = 4 blocks. + iter->Seek(BuildKey(1026)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(103)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1033)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1037)); + ASSERT_TRUE(iter->Valid()); + + if (support_prefetch && !use_direct_io) { + ASSERT_EQ(fs->GetPrefetchCount(), 3); + fs->ClearPrefetchCount(); + } else { + ASSERT_EQ(buff_prefetch_count, 2); + buff_prefetch_count = 0; + } + } + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + Close(); +} + +// This test verifies the functionality of ReadOptions.adaptive_readahead. +TEST_P(PrefetchTest, DBIterLevelReadAhead) { + const int kNumKeys = 1000; + // Set options + std::shared_ptr fs = + std::make_shared(env_->GetFileSystem(), false); + std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); + + bool use_direct_io = std::get<0>(GetParam()); + bool is_adaptive_readahead = std::get<1>(GetParam()); + + Options options; + SetGenericOptions(env.get(), use_direct_io, options); + options.statistics = CreateDBStatistics(); + BlockBasedTableOptions table_options; + SetBlockBasedTableOptions(table_options); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + Status s = TryReopen(options); + if (use_direct_io && (s.IsNotSupported() || s.IsInvalidArgument())) { + // If direct IO is not supported, skip the test + return; + } else { + ASSERT_OK(s); + } + + WriteBatch batch; + Random rnd(309); + int total_keys = 0; + for (int j = 0; j < 5; j++) { + for (int i = j * kNumKeys; i < (j + 1) * kNumKeys; i++) { + ASSERT_OK(batch.Put(BuildKey(i), rnd.RandomString(1000))); + total_keys++; + } + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + ASSERT_OK(Flush()); + } + MoveFilesToLevel(2); + int buff_prefetch_count = 0; + int readahead_carry_over_count = 0; + int num_sst_files = NumTableFilesAtLevel(2); + size_t current_readahead_size = 0; + + // Test - Iterate over the keys sequentially. + { + SyncPoint::GetInstance()->SetCallBack( + "FilePrefetchBuffer::Prefetch:Start", + [&](void*) { buff_prefetch_count++; }); + + // The callback checks, since reads are sequential, readahead_size doesn't + // start from 8KB when iterator moves to next file and its called + // num_sst_files-1 times (excluding for first file). + SyncPoint::GetInstance()->SetCallBack( + "BlockPrefetcher::SetReadaheadState", [&](void* arg) { + readahead_carry_over_count++; + size_t readahead_size = *reinterpret_cast(arg); + if (readahead_carry_over_count) { + ASSERT_GT(readahead_size, 8 * 1024); + } + }); + + SyncPoint::GetInstance()->SetCallBack( + "FilePrefetchBuffer::TryReadFromCache", [&](void* arg) { + current_readahead_size = *reinterpret_cast(arg); + ASSERT_GT(current_readahead_size, 0); + }); + + SyncPoint::GetInstance()->EnableProcessing(); + + ReadOptions ro; + if (is_adaptive_readahead) { + ro.adaptive_readahead = true; + } + + ASSERT_OK(options.statistics->Reset()); + + auto iter = std::unique_ptr(db_->NewIterator(ro)); + int num_keys = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_OK(iter->status()); + num_keys++; + } + ASSERT_EQ(num_keys, total_keys); + + // For index and data blocks. + if (is_adaptive_readahead) { + ASSERT_EQ(readahead_carry_over_count, 2 * (num_sst_files - 1)); + } else { + ASSERT_GT(buff_prefetch_count, 0); + ASSERT_EQ(readahead_carry_over_count, 0); + } + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + } + Close(); +} + +// This test verifies the functionality of ReadOptions.adaptive_readahead when +// async_io is enabled. +TEST_P(PrefetchTest, DBIterLevelReadAheadWithAsyncIO) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_BYPASS("Test requires non-mem or non-encrypted environment"); + return; + } + const int kNumKeys = 1000; + // Set options + std::shared_ptr fs = + std::make_shared(FileSystem::Default(), false); + std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); + + bool use_direct_io = std::get<0>(GetParam()); + bool is_adaptive_readahead = std::get<1>(GetParam()); + + Options options; + SetGenericOptions(env.get(), use_direct_io, options); + options.statistics = CreateDBStatistics(); + BlockBasedTableOptions table_options; + SetBlockBasedTableOptions(table_options); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + Status s = TryReopen(options); + if (use_direct_io && (s.IsNotSupported() || s.IsInvalidArgument())) { + // If direct IO is not supported, skip the test + return; + } else { + ASSERT_OK(s); + } + + WriteBatch batch; + Random rnd(309); + int total_keys = 0; + for (int j = 0; j < 5; j++) { + for (int i = j * kNumKeys; i < (j + 1) * kNumKeys; i++) { + ASSERT_OK(batch.Put(BuildKey(i), rnd.RandomString(1000))); + total_keys++; + } + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + ASSERT_OK(Flush()); + } + MoveFilesToLevel(2); + int buff_async_prefetch_count = 0; + int buff_prefetch_count = 0; + int readahead_carry_over_count = 0; + int num_sst_files = NumTableFilesAtLevel(2); + size_t current_readahead_size = 0; + bool read_async_called = false; + + // Test - Iterate over the keys sequentially. + { + SyncPoint::GetInstance()->SetCallBack( + "FilePrefetchBuffer::Prefetch:Start", + [&](void*) { buff_prefetch_count++; }); + + SyncPoint::GetInstance()->SetCallBack( + "FilePrefetchBuffer::PrefetchAsyncInternal:Start", + [&](void*) { buff_async_prefetch_count++; }); + + SyncPoint::GetInstance()->SetCallBack( + "UpdateResults::io_uring_result", + [&](void* /*arg*/) { read_async_called = true; }); + + // The callback checks, since reads are sequential, readahead_size doesn't + // start from 8KB when iterator moves to next file and its called + // num_sst_files-1 times (excluding for first file). + SyncPoint::GetInstance()->SetCallBack( + "BlockPrefetcher::SetReadaheadState", [&](void* arg) { + readahead_carry_over_count++; + size_t readahead_size = *reinterpret_cast(arg); + if (readahead_carry_over_count) { + ASSERT_GT(readahead_size, 8 * 1024); + } + }); + + SyncPoint::GetInstance()->SetCallBack( + "FilePrefetchBuffer::TryReadFromCache", [&](void* arg) { + current_readahead_size = *reinterpret_cast(arg); + ASSERT_GT(current_readahead_size, 0); + }); + + SyncPoint::GetInstance()->EnableProcessing(); + + ReadOptions ro; + if (is_adaptive_readahead) { + ro.adaptive_readahead = true; + } + ro.async_io = true; + + ASSERT_OK(options.statistics->Reset()); + + auto iter = std::unique_ptr(db_->NewIterator(ro)); + int num_keys = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_OK(iter->status()); + num_keys++; + } + ASSERT_EQ(num_keys, total_keys); + + // For index and data blocks. + if (is_adaptive_readahead) { + ASSERT_EQ(readahead_carry_over_count, 2 * (num_sst_files - 1)); + } else { + ASSERT_EQ(readahead_carry_over_count, 0); + } + + // Check stats to make sure async prefetch is done. + { + HistogramData async_read_bytes; + options.statistics->histogramData(ASYNC_READ_BYTES, &async_read_bytes); + // Not all platforms support iouring. In that case, ReadAsync in posix + // won't submit async requests. + if (read_async_called) { + ASSERT_GT(buff_async_prefetch_count, 0); + ASSERT_GT(async_read_bytes.count, 0); + } else { + ASSERT_GT(buff_prefetch_count, 0); + ASSERT_EQ(async_read_bytes.count, 0); + } + } + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + } + Close(); +} + +TEST_P(PrefetchTest, DBIterAsyncIONoIOUring) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_SKIP("Test requires non-mem or non-encrypted environment"); + return; + } + + const int kNumKeys = 1000; + // Set options + bool use_direct_io = std::get<0>(GetParam()); + bool is_adaptive_readahead = std::get<1>(GetParam()); + + Options options; + SetGenericOptions(Env::Default(), use_direct_io, options); + options.statistics = CreateDBStatistics(); + BlockBasedTableOptions table_options; + SetBlockBasedTableOptions(table_options); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + enable_io_uring = false; + Status s = TryReopen(options); + if (use_direct_io && (s.IsNotSupported() || s.IsInvalidArgument())) { + // If direct IO is not supported, skip the test + enable_io_uring = true; + return; + } else { + ASSERT_OK(s); + } + + WriteBatch batch; + Random rnd(309); + int total_keys = 0; + for (int j = 0; j < 5; j++) { + for (int i = j * kNumKeys; i < (j + 1) * kNumKeys; i++) { + ASSERT_OK(batch.Put(BuildKey(i), rnd.RandomString(1000))); + total_keys++; + } + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + ASSERT_OK(Flush()); + } + MoveFilesToLevel(2); + + // Test - Iterate over the keys sequentially. + { + ReadOptions ro; + if (is_adaptive_readahead) { + ro.adaptive_readahead = true; + } + ro.async_io = true; + + ASSERT_OK(options.statistics->Reset()); + + auto iter = std::unique_ptr(db_->NewIterator(ro)); + int num_keys = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_OK(iter->status()); + num_keys++; + } + ASSERT_EQ(num_keys, total_keys); + + // Check stats to make sure async prefetch is done. + { + HistogramData async_read_bytes; + options.statistics->histogramData(ASYNC_READ_BYTES, &async_read_bytes); + ASSERT_EQ(async_read_bytes.count, 0); + ASSERT_EQ(options.statistics->getTickerCount(READ_ASYNC_MICROS), 0); + } + } + + { + ReadOptions ro; + if (is_adaptive_readahead) { + ro.adaptive_readahead = true; + } + ro.async_io = true; + ro.tailing = true; + + ASSERT_OK(options.statistics->Reset()); + + auto iter = std::unique_ptr(db_->NewIterator(ro)); + int num_keys = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_OK(iter->status()); + num_keys++; + } + ASSERT_EQ(num_keys, total_keys); + + // Check stats to make sure async prefetch is done. + { + HistogramData async_read_bytes; + options.statistics->histogramData(ASYNC_READ_BYTES, &async_read_bytes); + ASSERT_EQ(async_read_bytes.count, 0); + ASSERT_EQ(options.statistics->getTickerCount(READ_ASYNC_MICROS), 0); + } + } + Close(); + + enable_io_uring = true; +} + +class PrefetchTest1 : public DBTestBase, + public ::testing::WithParamInterface { + public: + PrefetchTest1() : DBTestBase("prefetch_test1", true) {} + + virtual void SetGenericOptions(Env* env, bool use_direct_io, + Options& options) { + options = CurrentOptions(); + options.write_buffer_size = 1024; + options.create_if_missing = true; + options.compression = kNoCompression; + options.env = env; + options.disable_auto_compactions = true; + if (use_direct_io) { + options.use_direct_reads = true; + options.use_direct_io_for_flush_and_compaction = true; + } + } + + void SetBlockBasedTableOptions(BlockBasedTableOptions& table_options) { + table_options.no_block_cache = true; + table_options.cache_index_and_filter_blocks = false; + table_options.metadata_block_size = 1024; + table_options.index_type = + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + } +}; + +INSTANTIATE_TEST_CASE_P(PrefetchTest1, PrefetchTest1, ::testing::Bool()); + +TEST_P(PrefetchTest1, SeekWithExtraPrefetchAsyncIO) { + const int kNumKeys = 2000; + // Set options + std::shared_ptr fs = + std::make_shared(env_->GetFileSystem(), false); + std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); + + Options options; + SetGenericOptions(env.get(), GetParam(), options); + options.statistics = CreateDBStatistics(); + BlockBasedTableOptions table_options; + SetBlockBasedTableOptions(table_options); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + Status s = TryReopen(options); + if (GetParam() && (s.IsNotSupported() || s.IsInvalidArgument())) { + // If direct IO is not supported, skip the test + return; + } else { + ASSERT_OK(s); + } + + WriteBatch batch; + Random rnd(309); + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(batch.Put(BuildKey(i), rnd.RandomString(1000))); + } + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + + std::string start_key = BuildKey(0); + std::string end_key = BuildKey(kNumKeys - 1); + Slice least(start_key.data(), start_key.size()); + Slice greatest(end_key.data(), end_key.size()); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &least, &greatest)); + Close(); + + for (size_t i = 0; i < 3; i++) { + table_options.num_file_reads_for_auto_readahead = i; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + s = TryReopen(options); + ASSERT_OK(s); + + int buff_prefetch_count = 0; + int extra_prefetch_buff_cnt = 0; + SyncPoint::GetInstance()->SetCallBack( + "FilePrefetchBuffer::PrefetchAsync:ExtraPrefetching", + [&](void*) { extra_prefetch_buff_cnt++; }); + + SyncPoint::GetInstance()->SetCallBack( + "FilePrefetchBuffer::PrefetchAsyncInternal:Start", + [&](void*) { buff_prefetch_count++; }); + + SyncPoint::GetInstance()->EnableProcessing(); + + ReadOptions ro; + ro.async_io = true; + { + auto iter = std::unique_ptr(db_->NewIterator(ro)); + // First Seek + iter->Seek(BuildKey( + 0)); // Prefetch data on seek because of seek parallelization. + ASSERT_TRUE(iter->Valid()); + + // Do extra prefetching in Seek only if num_file_reads_for_auto_readahead + // = 0. + ASSERT_EQ(extra_prefetch_buff_cnt, (i == 0 ? 1 : 0)); + // buff_prefetch_count is 2 because of index block when + // num_file_reads_for_auto_readahead = 0. + // If num_file_reads_for_auto_readahead > 0, index block isn't prefetched. + ASSERT_EQ(buff_prefetch_count, i == 0 ? 2 : 1); + + extra_prefetch_buff_cnt = 0; + buff_prefetch_count = 0; + // Reset all values of FilePrefetchBuffer on new seek. + iter->Seek( + BuildKey(22)); // Prefetch data because of seek parallelization. + ASSERT_TRUE(iter->Valid()); + // Do extra prefetching in Seek only if num_file_reads_for_auto_readahead + // = 0. + ASSERT_EQ(extra_prefetch_buff_cnt, (i == 0 ? 1 : 0)); + ASSERT_EQ(buff_prefetch_count, 1); + + extra_prefetch_buff_cnt = 0; + buff_prefetch_count = 0; + // Reset all values of FilePrefetchBuffer on new seek. + iter->Seek( + BuildKey(33)); // Prefetch data because of seek parallelization. + ASSERT_TRUE(iter->Valid()); + // Do extra prefetching in Seek only if num_file_reads_for_auto_readahead + // = 0. + ASSERT_EQ(extra_prefetch_buff_cnt, (i == 0 ? 1 : 0)); + ASSERT_EQ(buff_prefetch_count, 1); + } + Close(); + } +} + +// This test verifies the functionality of ReadOptions.adaptive_readahead when +// reads are not sequential. +TEST_P(PrefetchTest1, NonSequentialReadsWithAdaptiveReadahead) { + const int kNumKeys = 1000; + // Set options + std::shared_ptr fs = + std::make_shared(env_->GetFileSystem(), false); + std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); + + Options options; + SetGenericOptions(env.get(), GetParam(), options); + BlockBasedTableOptions table_options; + SetBlockBasedTableOptions(table_options); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + Status s = TryReopen(options); + if (GetParam() && (s.IsNotSupported() || s.IsInvalidArgument())) { + // If direct IO is not supported, skip the test + return; + } else { + ASSERT_OK(s); + } + + WriteBatch batch; + Random rnd(309); + for (int j = 0; j < 5; j++) { + for (int i = j * kNumKeys; i < (j + 1) * kNumKeys; i++) { + ASSERT_OK(batch.Put(BuildKey(i), rnd.RandomString(1000))); + } + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + ASSERT_OK(Flush()); + } + MoveFilesToLevel(2); + + int buff_prefetch_count = 0; + int set_readahead = 0; + size_t readahead_size = 0; + + SyncPoint::GetInstance()->SetCallBack("FilePrefetchBuffer::Prefetch:Start", + [&](void*) { buff_prefetch_count++; }); + SyncPoint::GetInstance()->SetCallBack( + "BlockPrefetcher::SetReadaheadState", + [&](void* /*arg*/) { set_readahead++; }); + SyncPoint::GetInstance()->SetCallBack( + "FilePrefetchBuffer::TryReadFromCache", + [&](void* arg) { readahead_size = *reinterpret_cast(arg); }); + + SyncPoint::GetInstance()->EnableProcessing(); + + { + // Iterate until prefetch is done. + ReadOptions ro; + ro.adaptive_readahead = true; + auto iter = std::unique_ptr(db_->NewIterator(ro)); + + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + + while (iter->Valid() && buff_prefetch_count == 0) { + iter->Next(); + } + + ASSERT_EQ(readahead_size, 8 * 1024); + ASSERT_EQ(buff_prefetch_count, 1); + ASSERT_EQ(set_readahead, 0); + buff_prefetch_count = 0; + + // Move to last file and check readahead size fallbacks to 8KB. So next + // readahead size after prefetch should be 8 * 1024; + iter->Seek(BuildKey(4004)); + ASSERT_TRUE(iter->Valid()); + + while (iter->Valid() && buff_prefetch_count == 0) { + iter->Next(); + } + + ASSERT_EQ(readahead_size, 8 * 1024); + ASSERT_EQ(set_readahead, 0); + ASSERT_EQ(buff_prefetch_count, 1); + } + Close(); +} + +// This test verifies the functionality of adaptive_readaheadsize with cache and +// if block is found in cache, decrease the readahead_size if +// - its enabled internally by RocksDB (implicit_auto_readahead_) and, +// - readahead_size is greater than 0 and, +// - the block would have called prefetch API if not found in cache for +// which conditions are: +// - few/no bytes are in buffer and, +// - block is sequential with the previous read and, +// - num_file_reads_ + 1 (including this read) > +// num_file_reads_for_auto_readahead_ +TEST_P(PrefetchTest1, DecreaseReadAheadIfInCache) { + const int kNumKeys = 2000; + // Set options + std::shared_ptr fs = + std::make_shared(env_->GetFileSystem(), false); + std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); + + Options options; + SetGenericOptions(env.get(), GetParam(), options); + options.statistics = CreateDBStatistics(); + BlockBasedTableOptions table_options; + SetBlockBasedTableOptions(table_options); + std::shared_ptr cache = NewLRUCache(4 * 1024 * 1024, 2); // 8MB + table_options.block_cache = cache; + table_options.no_block_cache = false; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + Status s = TryReopen(options); + if (GetParam() && (s.IsNotSupported() || s.IsInvalidArgument())) { + // If direct IO is not supported, skip the test + return; + } else { + ASSERT_OK(s); + } + + WriteBatch batch; + Random rnd(309); + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(batch.Put(BuildKey(i), rnd.RandomString(1000))); + } + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + + std::string start_key = BuildKey(0); + std::string end_key = BuildKey(kNumKeys - 1); + Slice least(start_key.data(), start_key.size()); + Slice greatest(end_key.data(), end_key.size()); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &least, &greatest)); + + int buff_prefetch_count = 0; + size_t current_readahead_size = 0; + size_t expected_current_readahead_size = 8 * 1024; + size_t decrease_readahead_size = 8 * 1024; + + SyncPoint::GetInstance()->SetCallBack("FilePrefetchBuffer::Prefetch:Start", + [&](void*) { buff_prefetch_count++; }); + SyncPoint::GetInstance()->SetCallBack( + "FilePrefetchBuffer::TryReadFromCache", [&](void* arg) { + current_readahead_size = *reinterpret_cast(arg); + }); + + SyncPoint::GetInstance()->EnableProcessing(); + ReadOptions ro; + ro.adaptive_readahead = true; + { + /* + * Reseek keys from sequential Data Blocks within same partitioned + * index. After 2 sequential reads it will prefetch the data block. + * Data Block size is nearly 4076 so readahead will fetch 8 * 1024 data + * more initially (2 more data blocks). + */ + auto iter = std::unique_ptr(db_->NewIterator(ro)); + // Warm up the cache + iter->Seek(BuildKey(1011)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1015)); + ASSERT_TRUE(iter->Valid()); + iter->Seek(BuildKey(1019)); + ASSERT_TRUE(iter->Valid()); + buff_prefetch_count = 0; + } + + { + ASSERT_OK(options.statistics->Reset()); + // After caching, blocks will be read from cache (Sequential blocks) + auto iter = std::unique_ptr(db_->NewIterator(ro)); + iter->Seek( + BuildKey(0)); // In cache so it will decrease the readahead_size. + ASSERT_TRUE(iter->Valid()); + expected_current_readahead_size = std::max( + decrease_readahead_size, + (expected_current_readahead_size >= decrease_readahead_size + ? (expected_current_readahead_size - decrease_readahead_size) + : 0)); + + iter->Seek(BuildKey(1000)); // Won't prefetch the block. + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(current_readahead_size, expected_current_readahead_size); + + iter->Seek(BuildKey(1004)); // Prefetch the block. + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(current_readahead_size, expected_current_readahead_size); + expected_current_readahead_size *= 2; + + iter->Seek(BuildKey(1011)); + ASSERT_TRUE(iter->Valid()); + + // Eligible to Prefetch data (not in buffer) but block is in cache so no + // prefetch will happen and will result in decrease in readahead_size. + // readahead_size will be 8 * 1024 + iter->Seek(BuildKey(1015)); + ASSERT_TRUE(iter->Valid()); + expected_current_readahead_size = std::max( + decrease_readahead_size, + (expected_current_readahead_size >= decrease_readahead_size + ? (expected_current_readahead_size - decrease_readahead_size) + : 0)); + + // 1016 is the same block as 1015. So no change in readahead_size. + iter->Seek(BuildKey(1016)); + ASSERT_TRUE(iter->Valid()); + + // Prefetch data (not in buffer) but found in cache. So decrease + // readahead_size. Since it will 0 after decrementing so readahead_size will + // be set to initial value. + iter->Seek(BuildKey(1019)); + ASSERT_TRUE(iter->Valid()); + expected_current_readahead_size = std::max( + decrease_readahead_size, + (expected_current_readahead_size >= decrease_readahead_size + ? (expected_current_readahead_size - decrease_readahead_size) + : 0)); + + // Prefetch next sequential data. + iter->Seek(BuildKey(1022)); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(current_readahead_size, expected_current_readahead_size); + ASSERT_EQ(buff_prefetch_count, 2); + + buff_prefetch_count = 0; + } + Close(); +} + +// This test verifies the basic functionality of seek parallelization for +// async_io. +TEST_P(PrefetchTest1, SeekParallelizationTest) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_BYPASS("Test requires non-mem or non-encrypted environment"); + return; + } + const int kNumKeys = 2000; + // Set options + std::shared_ptr fs = std::make_shared( + FileSystem::Default(), /*support_prefetch=*/false); + std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); + + Options options; + SetGenericOptions(env.get(), GetParam(), options); + options.statistics = CreateDBStatistics(); + BlockBasedTableOptions table_options; + SetBlockBasedTableOptions(table_options); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + Status s = TryReopen(options); + if (GetParam() && (s.IsNotSupported() || s.IsInvalidArgument())) { + // If direct IO is not supported, skip the test + return; + } else { + ASSERT_OK(s); + } + + WriteBatch batch; + Random rnd(309); + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(batch.Put(BuildKey(i), rnd.RandomString(1000))); + } + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + + std::string start_key = BuildKey(0); + std::string end_key = BuildKey(kNumKeys - 1); + Slice least(start_key.data(), start_key.size()); + Slice greatest(end_key.data(), end_key.size()); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &least, &greatest)); + + int buff_prefetch_count = 0; + int buff_prefetch_async_count = 0; + + SyncPoint::GetInstance()->SetCallBack( + "FilePrefetchBuffer::PrefetchAsyncInternal:Start", + [&](void*) { buff_prefetch_async_count++; }); + + SyncPoint::GetInstance()->SetCallBack("FilePrefetchBuffer::Prefetch:Start", + [&](void*) { buff_prefetch_count++; }); + + bool read_async_called = false; + SyncPoint::GetInstance()->SetCallBack( + "UpdateResults::io_uring_result", + [&](void* /*arg*/) { read_async_called = true; }); + + SyncPoint::GetInstance()->EnableProcessing(); + ReadOptions ro; + ro.adaptive_readahead = true; + ro.async_io = true; + + { + ASSERT_OK(options.statistics->Reset()); + // Each block contains around 4 keys. + auto iter = std::unique_ptr(db_->NewIterator(ro)); + iter->Seek(BuildKey(0)); // Prefetch data because of seek parallelization. + ASSERT_TRUE(iter->Valid()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + + // New data block. Since num_file_reads in FilePrefetch after this read is + // 2, it won't go for prefetching. + iter->Next(); + ASSERT_TRUE(iter->Valid()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + + // Prefetch data. + iter->Next(); + ASSERT_TRUE(iter->Valid()); + + HistogramData async_read_bytes; + options.statistics->histogramData(ASYNC_READ_BYTES, &async_read_bytes); + // not all platforms support io_uring. In that case it'll fallback to normal + // prefetching without async_io. + if (read_async_called) { + ASSERT_EQ(buff_prefetch_async_count, 2); + ASSERT_GT(async_read_bytes.count, 0); + ASSERT_GT(get_perf_context()->number_async_seek, 0); + } else { + ASSERT_EQ(buff_prefetch_count, 1); + } + } + Close(); +} + +namespace { +#ifdef GFLAGS +const int kMaxArgCount = 100; +const size_t kArgBufferSize = 100000; + +void RunIOTracerParserTool(std::string trace_file) { + std::vector params = {"./io_tracer_parser", + "-io_trace_file=" + trace_file}; + + char arg_buffer[kArgBufferSize]; + char* argv[kMaxArgCount]; + int argc = 0; + int cursor = 0; + for (const auto& arg : params) { + ASSERT_LE(cursor + arg.size() + 1, kArgBufferSize); + ASSERT_LE(argc + 1, kMaxArgCount); + + snprintf(arg_buffer + cursor, arg.size() + 1, "%s", arg.c_str()); + + argv[argc++] = arg_buffer + cursor; + cursor += static_cast(arg.size()) + 1; + } + ASSERT_EQ(0, ROCKSDB_NAMESPACE::io_tracer_parser(argc, argv)); +} +#endif // GFLAGS +} // namespace + +// Tests the default implementation of ReadAsync API with PosixFileSystem during +// prefetching. +TEST_P(PrefetchTest, ReadAsyncWithPosixFS) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_SKIP("Test requires non-mem or non-encrypted environment"); + return; + } + + const int kNumKeys = 1000; + std::shared_ptr fs = std::make_shared( + FileSystem::Default(), /*support_prefetch=*/false); + std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); + + bool use_direct_io = std::get<0>(GetParam()); + Options options; + SetGenericOptions(env.get(), use_direct_io, options); + options.statistics = CreateDBStatistics(); + BlockBasedTableOptions table_options; + SetBlockBasedTableOptions(table_options); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + Status s = TryReopen(options); + if (use_direct_io && (s.IsNotSupported() || s.IsInvalidArgument())) { + // If direct IO is not supported, skip the test + return; + } else { + ASSERT_OK(s); + } + + int total_keys = 0; + // Write the keys. + { + WriteBatch batch; + Random rnd(309); + for (int j = 0; j < 5; j++) { + for (int i = j * kNumKeys; i < (j + 1) * kNumKeys; i++) { + ASSERT_OK(batch.Put(BuildKey(i), rnd.RandomString(1000))); + total_keys++; + } + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + ASSERT_OK(Flush()); + } + MoveFilesToLevel(2); + } + + int buff_prefetch_count = 0; + bool read_async_called = false; + ReadOptions ro; + ro.adaptive_readahead = true; + ro.async_io = true; + + if (std::get<1>(GetParam())) { + ro.readahead_size = 16 * 1024; + } + + SyncPoint::GetInstance()->SetCallBack( + "FilePrefetchBuffer::PrefetchAsyncInternal:Start", + [&](void*) { buff_prefetch_count++; }); + + SyncPoint::GetInstance()->SetCallBack( + "UpdateResults::io_uring_result", + [&](void* /*arg*/) { read_async_called = true; }); + SyncPoint::GetInstance()->EnableProcessing(); + + // Read the keys. + { + ASSERT_OK(options.statistics->Reset()); + get_perf_context()->Reset(); + + auto iter = std::unique_ptr(db_->NewIterator(ro)); + int num_keys = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_OK(iter->status()); + num_keys++; + } + + if (read_async_called) { + ASSERT_EQ(num_keys, total_keys); + ASSERT_GT(buff_prefetch_count, 0); + // Check stats to make sure async prefetch is done. + HistogramData async_read_bytes; + options.statistics->histogramData(ASYNC_READ_BYTES, &async_read_bytes); + HistogramData prefetched_bytes_discarded; + options.statistics->histogramData(PREFETCHED_BYTES_DISCARDED, + &prefetched_bytes_discarded); + ASSERT_GT(async_read_bytes.count, 0); + ASSERT_GT(prefetched_bytes_discarded.count, 0); + ASSERT_EQ(get_perf_context()->number_async_seek, 0); + } else { + // Not all platforms support iouring. In that case, ReadAsync in posix + // won't submit async requests. + ASSERT_EQ(num_keys, total_keys); + ASSERT_EQ(buff_prefetch_count, 0); + } + } + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + Close(); +} + +// This test verifies implementation of seek parallelization with +// PosixFileSystem during prefetching. +TEST_P(PrefetchTest, MultipleSeekWithPosixFS) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_SKIP("Test requires non-mem or non-encrypted environment"); + return; + } + + const int kNumKeys = 1000; + std::shared_ptr fs = std::make_shared( + FileSystem::Default(), /*support_prefetch=*/false); + std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); + + bool use_direct_io = std::get<0>(GetParam()); + Options options; + SetGenericOptions(env.get(), use_direct_io, options); + options.statistics = CreateDBStatistics(); + BlockBasedTableOptions table_options; + SetBlockBasedTableOptions(table_options); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + Status s = TryReopen(options); + if (use_direct_io && (s.IsNotSupported() || s.IsInvalidArgument())) { + // If direct IO is not supported, skip the test + return; + } else { + ASSERT_OK(s); + } + + int total_keys = 0; + // Write the keys. + { + WriteBatch batch; + Random rnd(309); + for (int j = 0; j < 5; j++) { + for (int i = j * kNumKeys; i < (j + 1) * kNumKeys; i++) { + ASSERT_OK(batch.Put(BuildKey(i), rnd.RandomString(1000))); + total_keys++; + } + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + ASSERT_OK(Flush()); + } + MoveFilesToLevel(2); + } + (void)total_keys; + + int num_keys_first_batch = 0; + int num_keys_second_batch = 0; + // Calculate number of keys without async_io for correctness validation. + { + auto iter = std::unique_ptr(db_->NewIterator(ReadOptions())); + // First Seek. + iter->Seek(BuildKey(450)); + while (iter->Valid() && num_keys_first_batch < 100) { + ASSERT_OK(iter->status()); + num_keys_first_batch++; + iter->Next(); + } + ASSERT_OK(iter->status()); + + iter->Seek(BuildKey(942)); + while (iter->Valid()) { + ASSERT_OK(iter->status()); + num_keys_second_batch++; + iter->Next(); + } + ASSERT_OK(iter->status()); + } + + int buff_prefetch_count = 0; + bool read_async_called = false; + ReadOptions ro; + ro.adaptive_readahead = true; + ro.async_io = true; + + if (std::get<1>(GetParam())) { + ro.readahead_size = 16 * 1024; + } + + SyncPoint::GetInstance()->SetCallBack( + "FilePrefetchBuffer::PrefetchAsyncInternal:Start", + [&](void*) { buff_prefetch_count++; }); + + SyncPoint::GetInstance()->SetCallBack( + "UpdateResults::io_uring_result", + [&](void* /*arg*/) { read_async_called = true; }); + SyncPoint::GetInstance()->EnableProcessing(); + + // Read the keys using seek. + { + ASSERT_OK(options.statistics->Reset()); + get_perf_context()->Reset(); + + auto iter = std::unique_ptr(db_->NewIterator(ro)); + int num_keys = 0; + // First Seek. + { + iter->Seek(BuildKey(450)); + while (iter->Valid() && num_keys < 100) { + ASSERT_OK(iter->status()); + num_keys++; + iter->Next(); + } + + ASSERT_OK(iter->status()); + ASSERT_EQ(num_keys, num_keys_first_batch); + // Check stats to make sure async prefetch is done. + HistogramData async_read_bytes; + options.statistics->histogramData(ASYNC_READ_BYTES, &async_read_bytes); + if (read_async_called) { + ASSERT_GT(async_read_bytes.count, 0); + ASSERT_GT(get_perf_context()->number_async_seek, 0); + } else { + // Not all platforms support iouring. In that case, ReadAsync in posix + // won't submit async requests. + ASSERT_EQ(async_read_bytes.count, 0); + ASSERT_EQ(get_perf_context()->number_async_seek, 0); + } + } + + // Second Seek. + { + num_keys = 0; + ASSERT_OK(options.statistics->Reset()); + get_perf_context()->Reset(); + + iter->Seek(BuildKey(942)); + while (iter->Valid()) { + ASSERT_OK(iter->status()); + num_keys++; + iter->Next(); + } + + ASSERT_OK(iter->status()); + ASSERT_EQ(num_keys, num_keys_second_batch); + HistogramData async_read_bytes; + options.statistics->histogramData(ASYNC_READ_BYTES, &async_read_bytes); + HistogramData prefetched_bytes_discarded; + options.statistics->histogramData(PREFETCHED_BYTES_DISCARDED, + &prefetched_bytes_discarded); + ASSERT_GT(prefetched_bytes_discarded.count, 0); + + if (read_async_called) { + ASSERT_GT(buff_prefetch_count, 0); + + // Check stats to make sure async prefetch is done. + ASSERT_GT(async_read_bytes.count, 0); + ASSERT_GT(get_perf_context()->number_async_seek, 0); + } else { + // Not all platforms support iouring. In that case, ReadAsync in posix + // won't submit async requests. + ASSERT_EQ(async_read_bytes.count, 0); + ASSERT_EQ(get_perf_context()->number_async_seek, 0); + } + } + } + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + Close(); +} + +// This test verifies implementation of seek parallelization with +// PosixFileSystem during prefetching. +TEST_P(PrefetchTest, SeekParallelizationTestWithPosix) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_SKIP("Test requires non-mem or non-encrypted environment"); + return; + } + const int kNumKeys = 2000; + // Set options + std::shared_ptr fs = std::make_shared( + FileSystem::Default(), /*support_prefetch=*/false); + std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); + + bool use_direct_io = std::get<0>(GetParam()); + Options options; + SetGenericOptions(env.get(), use_direct_io, options); + options.statistics = CreateDBStatistics(); + BlockBasedTableOptions table_options; + SetBlockBasedTableOptions(table_options); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + Status s = TryReopen(options); + if (use_direct_io && (s.IsNotSupported() || s.IsInvalidArgument())) { + // If direct IO is not supported, skip the test + return; + } else { + ASSERT_OK(s); + } + + WriteBatch batch; + Random rnd(309); + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(batch.Put(BuildKey(i), rnd.RandomString(1000))); + } + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + + std::string start_key = BuildKey(0); + std::string end_key = BuildKey(kNumKeys - 1); + Slice least(start_key.data(), start_key.size()); + Slice greatest(end_key.data(), end_key.size()); + + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &least, &greatest)); + + int buff_prefetch_count = 0; + + SyncPoint::GetInstance()->SetCallBack( + "FilePrefetchBuffer::PrefetchAsyncInternal:Start", + [&](void*) { buff_prefetch_count++; }); + + bool read_async_called = false; + SyncPoint::GetInstance()->SetCallBack( + "UpdateResults::io_uring_result", + [&](void* /*arg*/) { read_async_called = true; }); + SyncPoint::GetInstance()->EnableProcessing(); + + SyncPoint::GetInstance()->EnableProcessing(); + ReadOptions ro; + ro.adaptive_readahead = true; + ro.async_io = true; + + if (std::get<1>(GetParam())) { + ro.readahead_size = 16 * 1024; + } + + { + ASSERT_OK(options.statistics->Reset()); + // Each block contains around 4 keys. + auto iter = std::unique_ptr(db_->NewIterator(ro)); + iter->Seek(BuildKey(0)); // Prefetch data because of seek parallelization. + ASSERT_TRUE(iter->Valid()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + + // New data block. Since num_file_reads in FilePrefetch after this read is + // 2, it won't go for prefetching. + iter->Next(); + ASSERT_TRUE(iter->Valid()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + + // Prefetch data. + iter->Next(); + + ASSERT_TRUE(iter->Valid()); + HistogramData async_read_bytes; + options.statistics->histogramData(ASYNC_READ_BYTES, &async_read_bytes); + if (read_async_called) { + ASSERT_GT(async_read_bytes.count, 0); + ASSERT_GT(get_perf_context()->number_async_seek, 0); + if (std::get<1>(GetParam())) { + ASSERT_EQ(buff_prefetch_count, 1); + } else { + ASSERT_EQ(buff_prefetch_count, 2); + } + } else { + // Not all platforms support iouring. In that case, ReadAsync in posix + // won't submit async requests. + ASSERT_EQ(async_read_bytes.count, 0); + ASSERT_EQ(get_perf_context()->number_async_seek, 0); + } + } + Close(); +} + +#ifdef GFLAGS +// This test verifies io_tracing with PosixFileSystem during prefetching. +TEST_P(PrefetchTest, TraceReadAsyncWithCallbackWrapper) { + if (mem_env_ || encrypted_env_) { + ROCKSDB_GTEST_SKIP("Test requires non-mem or non-encrypted environment"); + return; + } + + const int kNumKeys = 1000; + std::shared_ptr fs = std::make_shared( + FileSystem::Default(), /*support_prefetch=*/false); + std::unique_ptr env(new CompositeEnvWrapper(env_, fs)); + + bool use_direct_io = std::get<0>(GetParam()); + Options options; + SetGenericOptions(env.get(), use_direct_io, options); + options.statistics = CreateDBStatistics(); + BlockBasedTableOptions table_options; + SetBlockBasedTableOptions(table_options); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + Status s = TryReopen(options); + if (use_direct_io && (s.IsNotSupported() || s.IsInvalidArgument())) { + // If direct IO is not supported, skip the test + return; + } else { + ASSERT_OK(s); + } + + int total_keys = 0; + // Write the keys. + { + WriteBatch batch; + Random rnd(309); + for (int j = 0; j < 5; j++) { + for (int i = j * kNumKeys; i < (j + 1) * kNumKeys; i++) { + ASSERT_OK(batch.Put(BuildKey(i), rnd.RandomString(1000))); + total_keys++; + } + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + ASSERT_OK(Flush()); + } + MoveFilesToLevel(2); + } + + int buff_prefetch_count = 0; + bool read_async_called = false; + ReadOptions ro; + ro.adaptive_readahead = true; + ro.async_io = true; + + if (std::get<1>(GetParam())) { + ro.readahead_size = 16 * 1024; + } + + SyncPoint::GetInstance()->SetCallBack( + "FilePrefetchBuffer::PrefetchAsyncInternal:Start", + [&](void*) { buff_prefetch_count++; }); + + SyncPoint::GetInstance()->SetCallBack( + "UpdateResults::io_uring_result", + [&](void* /*arg*/) { read_async_called = true; }); + SyncPoint::GetInstance()->EnableProcessing(); + + // Read the keys. + { + // Start io_tracing. + WriteOptions write_opt; + TraceOptions trace_opt; + std::unique_ptr trace_writer; + std::string trace_file_path = dbname_ + "/io_trace_file"; + + ASSERT_OK( + NewFileTraceWriter(env_, EnvOptions(), trace_file_path, &trace_writer)); + ASSERT_OK(db_->StartIOTrace(trace_opt, std::move(trace_writer))); + ASSERT_OK(options.statistics->Reset()); + + auto iter = std::unique_ptr(db_->NewIterator(ro)); + int num_keys = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ASSERT_OK(iter->status()); + num_keys++; + } + + // End the tracing. + ASSERT_OK(db_->EndIOTrace()); + ASSERT_OK(env_->FileExists(trace_file_path)); + + ASSERT_EQ(num_keys, total_keys); + HistogramData async_read_bytes; + options.statistics->histogramData(ASYNC_READ_BYTES, &async_read_bytes); + if (read_async_called) { + ASSERT_GT(buff_prefetch_count, 0); + // Check stats to make sure async prefetch is done. + ASSERT_GT(async_read_bytes.count, 0); + } else { + // Not all platforms support iouring. In that case, ReadAsync in posix + // won't submit async requests. + ASSERT_EQ(async_read_bytes.count, 0); + } + + // Check the file to see if ReadAsync is logged. + RunIOTracerParserTool(trace_file_path); + } + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + Close(); +} +#endif // GFLAGS + +class FilePrefetchBufferTest : public testing::Test { + public: + void SetUp() override { + SetupSyncPointsToMockDirectIO(); + env_ = Env::Default(); + fs_ = FileSystem::Default(); + test_dir_ = test::PerThreadDBPath("file_prefetch_buffer_test"); + ASSERT_OK(fs_->CreateDir(test_dir_, IOOptions(), nullptr)); + stats_ = CreateDBStatistics(); + } + + void TearDown() override { EXPECT_OK(DestroyDir(env_, test_dir_)); } + + void Write(const std::string& fname, const std::string& content) { + std::unique_ptr f; + ASSERT_OK(fs_->NewWritableFile(Path(fname), FileOptions(), &f, nullptr)); + ASSERT_OK(f->Append(content, IOOptions(), nullptr)); + ASSERT_OK(f->Close(IOOptions(), nullptr)); + } + + void Read(const std::string& fname, const FileOptions& opts, + std::unique_ptr* reader) { + std::string fpath = Path(fname); + std::unique_ptr f; + ASSERT_OK(fs_->NewRandomAccessFile(fpath, opts, &f, nullptr)); + reader->reset(new RandomAccessFileReader( + std::move(f), fpath, env_->GetSystemClock().get(), + /*io_tracer=*/nullptr, stats_.get())); + } + + void AssertResult(const std::string& content, + const std::vector& reqs) { + for (const auto& r : reqs) { + ASSERT_OK(r.status); + ASSERT_EQ(r.len, r.result.size()); + ASSERT_EQ(content.substr(r.offset, r.len), r.result.ToString()); + } + } + + FileSystem* fs() { return fs_.get(); } + Statistics* stats() { return stats_.get(); } + + private: + Env* env_; + std::shared_ptr fs_; + std::string test_dir_; + std::shared_ptr stats_; + + std::string Path(const std::string& fname) { return test_dir_ + "/" + fname; } +}; + +TEST_F(FilePrefetchBufferTest, SeekWithBlockCacheHit) { + std::string fname = "seek-with-block-cache-hit"; + Random rand(0); + std::string content = rand.RandomString(32768); + Write(fname, content); + + FileOptions opts; + std::unique_ptr r; + Read(fname, opts, &r); + + FilePrefetchBuffer fpb(16384, 16384, true, false, false, 0, 0, fs()); + Slice result; + // Simulate a seek of 4096 bytes at offset 0. Due to the readahead settings, + // it will do two reads of 4096+8192 and 8192 + Status s = fpb.PrefetchAsync(IOOptions(), r.get(), 0, 4096, &result); + + // Platforms that don't have IO uring may not support async IO. + if (s.IsNotSupported()) { + return; + } + + ASSERT_TRUE(s.IsTryAgain()); + // Simulate a block cache hit + fpb.UpdateReadPattern(0, 4096, false); + // Now read some data that straddles the two prefetch buffers - offset 8192 to + // 16384 + ASSERT_TRUE(fpb.TryReadFromCacheAsync(IOOptions(), r.get(), 8192, 8192, + &result, &s, Env::IOPriority::IO_LOW)); +} + +TEST_F(FilePrefetchBufferTest, NoSyncWithAsyncIO) { + std::string fname = "seek-with-block-cache-hit"; + Random rand(0); + std::string content = rand.RandomString(32768); + Write(fname, content); + + FileOptions opts; + std::unique_ptr r; + Read(fname, opts, &r); + + FilePrefetchBuffer fpb( + /*readahead_size=*/8192, /*max_readahead_size=*/16384, /*enable=*/true, + /*track_min_offset=*/false, /*implicit_auto_readahead=*/false, + /*num_file_reads=*/0, /*num_file_reads_for_auto_readahead=*/0, fs()); + + int read_async_called = 0; + SyncPoint::GetInstance()->SetCallBack( + "FilePrefetchBuffer::ReadAsync", + [&](void* /*arg*/) { read_async_called++; }); + SyncPoint::GetInstance()->EnableProcessing(); + + Slice async_result; + // Simulate a seek of 4000 bytes at offset 3000. Due to the readahead + // settings, it will do two reads of 4000+4096 and 4096 + Status s = fpb.PrefetchAsync(IOOptions(), r.get(), 3000, 4000, &async_result); + + // Platforms that don't have IO uring may not support async IO + if (s.IsNotSupported()) { + return; + } + + ASSERT_TRUE(s.IsTryAgain()); + ASSERT_TRUE(fpb.TryReadFromCacheAsync(IOOptions(), r.get(), /*offset=*/3000, + /*length=*/4000, &async_result, &s, + Env::IOPriority::IO_LOW)); + // No sync call should be made. + HistogramData sst_read_micros; + stats()->histogramData(SST_READ_MICROS, &sst_read_micros); + ASSERT_EQ(sst_read_micros.count, 0); + + // Number of async calls should be. + ASSERT_EQ(read_async_called, 2); + // Length should be 4000. + ASSERT_EQ(async_result.size(), 4000); + // Data correctness. + Slice result(content.c_str() + 3000, 4000); + ASSERT_EQ(result.size(), 4000); + ASSERT_EQ(result, async_result); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/file/random_access_file_reader.cc b/librocksdb-sys/rocksdb/file/random_access_file_reader.cc new file mode 100644 index 0000000..e38e9ec --- /dev/null +++ b/librocksdb-sys/rocksdb/file/random_access_file_reader.cc @@ -0,0 +1,607 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "file/random_access_file_reader.h" + +#include +#include + +#include "file/file_util.h" +#include "monitoring/histogram.h" +#include "monitoring/iostats_context_imp.h" +#include "port/port.h" +#include "table/format.h" +#include "test_util/sync_point.h" +#include "util/random.h" +#include "util/rate_limiter_impl.h" + +namespace ROCKSDB_NAMESPACE { +const std::array + kReadHistograms{{ + FILE_READ_FLUSH_MICROS, + FILE_READ_COMPACTION_MICROS, + FILE_READ_DB_OPEN_MICROS, + }}; +inline void RecordIOStats(Statistics* stats, Temperature file_temperature, + bool is_last_level, size_t size) { + IOSTATS_ADD(bytes_read, size); + // record for last/non-last level + if (is_last_level) { + RecordTick(stats, LAST_LEVEL_READ_BYTES, size); + RecordTick(stats, LAST_LEVEL_READ_COUNT, 1); + } else { + RecordTick(stats, NON_LAST_LEVEL_READ_BYTES, size); + RecordTick(stats, NON_LAST_LEVEL_READ_COUNT, 1); + } + + // record for temperature file + if (file_temperature != Temperature::kUnknown) { + switch (file_temperature) { + case Temperature::kHot: + IOSTATS_ADD(file_io_stats_by_temperature.hot_file_bytes_read, size); + IOSTATS_ADD(file_io_stats_by_temperature.hot_file_read_count, 1); + RecordTick(stats, HOT_FILE_READ_BYTES, size); + RecordTick(stats, HOT_FILE_READ_COUNT, 1); + break; + case Temperature::kWarm: + IOSTATS_ADD(file_io_stats_by_temperature.warm_file_bytes_read, size); + IOSTATS_ADD(file_io_stats_by_temperature.warm_file_read_count, 1); + RecordTick(stats, WARM_FILE_READ_BYTES, size); + RecordTick(stats, WARM_FILE_READ_COUNT, 1); + break; + case Temperature::kCold: + IOSTATS_ADD(file_io_stats_by_temperature.cold_file_bytes_read, size); + IOSTATS_ADD(file_io_stats_by_temperature.cold_file_read_count, 1); + RecordTick(stats, COLD_FILE_READ_BYTES, size); + RecordTick(stats, COLD_FILE_READ_COUNT, 1); + break; + default: + break; + } + } +} + +IOStatus RandomAccessFileReader::Create( + const std::shared_ptr& fs, const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* reader, IODebugContext* dbg) { + std::unique_ptr file; + IOStatus io_s = fs->NewRandomAccessFile(fname, file_opts, &file, dbg); + if (io_s.ok()) { + reader->reset(new RandomAccessFileReader(std::move(file), fname)); + } + return io_s; +} + +IOStatus RandomAccessFileReader::Read( + const IOOptions& opts, uint64_t offset, size_t n, Slice* result, + char* scratch, AlignedBuf* aligned_buf, + Env::IOPriority rate_limiter_priority) const { + (void)aligned_buf; + + TEST_SYNC_POINT_CALLBACK("RandomAccessFileReader::Read", nullptr); + + // To be paranoid: modify scratch a little bit, so in case underlying + // FileSystem doesn't fill the buffer but return success and `scratch` returns + // contains a previous block, returned value will not pass checksum. + if (n > 0 && scratch != nullptr) { + // This byte might not change anything for direct I/O case, but it's OK. + scratch[0]++; + } + + IOStatus io_s; + uint64_t elapsed = 0; + size_t alignment = file_->GetRequiredBufferAlignment(); + bool is_aligned = false; + if (scratch != nullptr) { + // Check if offset, length and buffer are aligned. + is_aligned = (offset & (alignment - 1)) == 0 && + (n & (alignment - 1)) == 0 && + (uintptr_t(scratch) & (alignment - 1)) == 0; + } + + { + StopWatch sw(clock_, stats_, hist_type_, + (opts.io_activity != Env::IOActivity::kUnknown) + ? kReadHistograms[(std::size_t)(opts.io_activity)] + : Histograms::HISTOGRAM_ENUM_MAX, + (stats_ != nullptr) ? &elapsed : nullptr, true /*overwrite*/, + true /*delay_enabled*/); + auto prev_perf_level = GetPerfLevel(); + IOSTATS_TIMER_GUARD(read_nanos); + if (use_direct_io() && is_aligned == false) { + size_t aligned_offset = + TruncateToPageBoundary(alignment, static_cast(offset)); + size_t offset_advance = static_cast(offset) - aligned_offset; + size_t read_size = + Roundup(static_cast(offset + n), alignment) - aligned_offset; + AlignedBuffer buf; + buf.Alignment(alignment); + buf.AllocateNewBuffer(read_size); + while (buf.CurrentSize() < read_size) { + size_t allowed; + if (rate_limiter_priority != Env::IO_TOTAL && + rate_limiter_ != nullptr) { + allowed = rate_limiter_->RequestToken( + buf.Capacity() - buf.CurrentSize(), buf.Alignment(), + rate_limiter_priority, stats_, RateLimiter::OpType::kRead); + } else { + assert(buf.CurrentSize() == 0); + allowed = read_size; + } + Slice tmp; + + FileOperationInfo::StartTimePoint start_ts; + uint64_t orig_offset = 0; + if (ShouldNotifyListeners()) { + start_ts = FileOperationInfo::StartNow(); + orig_offset = aligned_offset + buf.CurrentSize(); + } + + { + IOSTATS_CPU_TIMER_GUARD(cpu_read_nanos, clock_); + // Only user reads are expected to specify a timeout. And user reads + // are not subjected to rate_limiter and should go through only + // one iteration of this loop, so we don't need to check and adjust + // the opts.timeout before calling file_->Read + assert(!opts.timeout.count() || allowed == read_size); + io_s = file_->Read(aligned_offset + buf.CurrentSize(), allowed, opts, + &tmp, buf.Destination(), nullptr); + } + if (ShouldNotifyListeners()) { + auto finish_ts = FileOperationInfo::FinishNow(); + NotifyOnFileReadFinish(orig_offset, tmp.size(), start_ts, finish_ts, + io_s); + if (!io_s.ok()) { + NotifyOnIOError(io_s, FileOperationType::kRead, file_name(), + tmp.size(), orig_offset); + } + } + + buf.Size(buf.CurrentSize() + tmp.size()); + if (!io_s.ok() || tmp.size() < allowed) { + break; + } + } + size_t res_len = 0; + if (io_s.ok() && offset_advance < buf.CurrentSize()) { + res_len = std::min(buf.CurrentSize() - offset_advance, n); + if (aligned_buf == nullptr) { + buf.Read(scratch, offset_advance, res_len); + } else { + scratch = buf.BufferStart() + offset_advance; + aligned_buf->reset(buf.Release()); + } + } + *result = Slice(scratch, res_len); + } else { + size_t pos = 0; + const char* res_scratch = nullptr; + while (pos < n) { + size_t allowed; + if (rate_limiter_priority != Env::IO_TOTAL && + rate_limiter_ != nullptr) { + if (rate_limiter_->IsRateLimited(RateLimiter::OpType::kRead)) { + sw.DelayStart(); + } + allowed = rate_limiter_->RequestToken( + n - pos, (use_direct_io() ? alignment : 0), rate_limiter_priority, + stats_, RateLimiter::OpType::kRead); + if (rate_limiter_->IsRateLimited(RateLimiter::OpType::kRead)) { + sw.DelayStop(); + } + } else { + allowed = n; + } + Slice tmp_result; + + FileOperationInfo::StartTimePoint start_ts; + if (ShouldNotifyListeners()) { + start_ts = FileOperationInfo::StartNow(); + } + + { + IOSTATS_CPU_TIMER_GUARD(cpu_read_nanos, clock_); + // Only user reads are expected to specify a timeout. And user reads + // are not subjected to rate_limiter and should go through only + // one iteration of this loop, so we don't need to check and adjust + // the opts.timeout before calling file_->Read + assert(!opts.timeout.count() || allowed == n); + io_s = file_->Read(offset + pos, allowed, opts, &tmp_result, + scratch + pos, nullptr); + } + if (ShouldNotifyListeners()) { + auto finish_ts = FileOperationInfo::FinishNow(); + NotifyOnFileReadFinish(offset + pos, tmp_result.size(), start_ts, + finish_ts, io_s); + + if (!io_s.ok()) { + NotifyOnIOError(io_s, FileOperationType::kRead, file_name(), + tmp_result.size(), offset + pos); + } + } + if (res_scratch == nullptr) { + // we can't simply use `scratch` because reads of mmap'd files return + // data in a different buffer. + res_scratch = tmp_result.data(); + } else { + // make sure chunks are inserted contiguously into `res_scratch`. + assert(tmp_result.data() == res_scratch + pos); + } + pos += tmp_result.size(); + if (!io_s.ok() || tmp_result.size() < allowed) { + break; + } + } + *result = Slice(res_scratch, io_s.ok() ? pos : 0); + } + RecordIOStats(stats_, file_temperature_, is_last_level_, result->size()); + SetPerfLevel(prev_perf_level); + } + if (stats_ != nullptr && file_read_hist_ != nullptr) { + file_read_hist_->Add(elapsed); + } + + return io_s; +} + +size_t End(const FSReadRequest& r) { + return static_cast(r.offset) + r.len; +} + +FSReadRequest Align(const FSReadRequest& r, size_t alignment) { + FSReadRequest req; + req.offset = static_cast( + TruncateToPageBoundary(alignment, static_cast(r.offset))); + req.len = Roundup(End(r), alignment) - req.offset; + req.scratch = nullptr; + return req; +} + +bool TryMerge(FSReadRequest* dest, const FSReadRequest& src) { + size_t dest_offset = static_cast(dest->offset); + size_t src_offset = static_cast(src.offset); + size_t dest_end = End(*dest); + size_t src_end = End(src); + if (std::max(dest_offset, src_offset) > std::min(dest_end, src_end)) { + return false; + } + dest->offset = static_cast(std::min(dest_offset, src_offset)); + dest->len = std::max(dest_end, src_end) - dest->offset; + return true; +} + +IOStatus RandomAccessFileReader::MultiRead( + const IOOptions& opts, FSReadRequest* read_reqs, size_t num_reqs, + AlignedBuf* aligned_buf, Env::IOPriority rate_limiter_priority) const { + (void)aligned_buf; // suppress warning of unused variable in LITE mode + assert(num_reqs > 0); + +#ifndef NDEBUG + for (size_t i = 0; i < num_reqs - 1; ++i) { + assert(read_reqs[i].offset <= read_reqs[i + 1].offset); + } +#endif // !NDEBUG + + // To be paranoid modify scratch a little bit, so in case underlying + // FileSystem doesn't fill the buffer but return success and `scratch` returns + // contains a previous block, returned value will not pass checksum. + // This byte might not change anything for direct I/O case, but it's OK. + for (size_t i = 0; i < num_reqs; i++) { + FSReadRequest& r = read_reqs[i]; + if (r.len > 0 && r.scratch != nullptr) { + r.scratch[0]++; + } + } + + IOStatus io_s; + uint64_t elapsed = 0; + { + StopWatch sw(clock_, stats_, hist_type_, + (opts.io_activity != Env::IOActivity::kUnknown) + ? kReadHistograms[(std::size_t)(opts.io_activity)] + : Histograms::HISTOGRAM_ENUM_MAX, + (stats_ != nullptr) ? &elapsed : nullptr, true /*overwrite*/, + true /*delay_enabled*/); + auto prev_perf_level = GetPerfLevel(); + IOSTATS_TIMER_GUARD(read_nanos); + + FSReadRequest* fs_reqs = read_reqs; + size_t num_fs_reqs = num_reqs; + std::vector aligned_reqs; + if (use_direct_io()) { + // num_reqs is the max possible size, + // this can reduce std::vecector's internal resize operations. + aligned_reqs.reserve(num_reqs); + // Align and merge the read requests. + size_t alignment = file_->GetRequiredBufferAlignment(); + for (size_t i = 0; i < num_reqs; i++) { + FSReadRequest r = Align(read_reqs[i], alignment); + if (i == 0) { + // head + aligned_reqs.push_back(std::move(r)); + + } else if (!TryMerge(&aligned_reqs.back(), r)) { + // head + n + aligned_reqs.push_back(std::move(r)); + + } else { + // unused + r.status.PermitUncheckedError(); + } + } + TEST_SYNC_POINT_CALLBACK("RandomAccessFileReader::MultiRead:AlignedReqs", + &aligned_reqs); + + // Allocate aligned buffer and let scratch buffers point to it. + size_t total_len = 0; + for (const auto& r : aligned_reqs) { + total_len += r.len; + } + AlignedBuffer buf; + buf.Alignment(alignment); + buf.AllocateNewBuffer(total_len); + char* scratch = buf.BufferStart(); + for (auto& r : aligned_reqs) { + r.scratch = scratch; + scratch += r.len; + } + + aligned_buf->reset(buf.Release()); + fs_reqs = aligned_reqs.data(); + num_fs_reqs = aligned_reqs.size(); + } + + FileOperationInfo::StartTimePoint start_ts; + if (ShouldNotifyListeners()) { + start_ts = FileOperationInfo::StartNow(); + } + + { + IOSTATS_CPU_TIMER_GUARD(cpu_read_nanos, clock_); + if (rate_limiter_priority != Env::IO_TOTAL && rate_limiter_ != nullptr) { + // TODO: ideally we should call `RateLimiter::RequestToken()` for + // allowed bytes to multi-read and then consume those bytes by + // satisfying as many requests in `MultiRead()` as possible, instead of + // what we do here, which can cause burst when the + // `total_multi_read_size` is big. + size_t total_multi_read_size = 0; + assert(fs_reqs != nullptr); + for (size_t i = 0; i < num_fs_reqs; ++i) { + FSReadRequest& req = fs_reqs[i]; + total_multi_read_size += req.len; + } + size_t remaining_bytes = total_multi_read_size; + size_t request_bytes = 0; + while (remaining_bytes > 0) { + request_bytes = std::min( + static_cast(rate_limiter_->GetSingleBurstBytes()), + remaining_bytes); + rate_limiter_->Request(request_bytes, rate_limiter_priority, + nullptr /* stats */, + RateLimiter::OpType::kRead); + remaining_bytes -= request_bytes; + } + } + io_s = file_->MultiRead(fs_reqs, num_fs_reqs, opts, nullptr); + RecordInHistogram(stats_, MULTIGET_IO_BATCH_SIZE, num_fs_reqs); + } + + if (use_direct_io()) { + // Populate results in the unaligned read requests. + size_t aligned_i = 0; + for (size_t i = 0; i < num_reqs; i++) { + auto& r = read_reqs[i]; + if (static_cast(r.offset) > End(aligned_reqs[aligned_i])) { + aligned_i++; + } + const auto& fs_r = fs_reqs[aligned_i]; + r.status = fs_r.status; + if (r.status.ok()) { + uint64_t offset = r.offset - fs_r.offset; + if (fs_r.result.size() <= offset) { + // No byte in the read range is returned. + r.result = Slice(); + } else { + size_t len = std::min( + r.len, static_cast(fs_r.result.size() - offset)); + r.result = Slice(fs_r.scratch + offset, len); + } + } else { + r.result = Slice(); + } + } + } + + for (size_t i = 0; i < num_reqs; ++i) { + if (ShouldNotifyListeners()) { + auto finish_ts = FileOperationInfo::FinishNow(); + NotifyOnFileReadFinish(read_reqs[i].offset, read_reqs[i].result.size(), + start_ts, finish_ts, read_reqs[i].status); + } + if (!read_reqs[i].status.ok()) { + NotifyOnIOError(read_reqs[i].status, FileOperationType::kRead, + file_name(), read_reqs[i].result.size(), + read_reqs[i].offset); + } + + RecordIOStats(stats_, file_temperature_, is_last_level_, + read_reqs[i].result.size()); + } + SetPerfLevel(prev_perf_level); + } + if (stats_ != nullptr && file_read_hist_ != nullptr) { + file_read_hist_->Add(elapsed); + } + + return io_s; +} + +IOStatus RandomAccessFileReader::PrepareIOOptions(const ReadOptions& ro, + IOOptions& opts) const { + if (clock_ != nullptr) { + return PrepareIOFromReadOptions(ro, clock_, opts); + } else { + return PrepareIOFromReadOptions(ro, SystemClock::Default().get(), opts); + } +} + +IOStatus RandomAccessFileReader::ReadAsync( + FSReadRequest& req, const IOOptions& opts, + std::function cb, void* cb_arg, + void** io_handle, IOHandleDeleter* del_fn, AlignedBuf* aligned_buf) { + IOStatus s; + // Create a callback and populate info. + auto read_async_callback = + std::bind(&RandomAccessFileReader::ReadAsyncCallback, this, + std::placeholders::_1, std::placeholders::_2); + ReadAsyncInfo* read_async_info = + new ReadAsyncInfo(cb, cb_arg, clock_->NowMicros()); + + if (ShouldNotifyListeners()) { + read_async_info->fs_start_ts_ = FileOperationInfo::StartNow(); + } + + size_t alignment = file_->GetRequiredBufferAlignment(); + bool is_aligned = (req.offset & (alignment - 1)) == 0 && + (req.len & (alignment - 1)) == 0 && + (uintptr_t(req.scratch) & (alignment - 1)) == 0; + read_async_info->is_aligned_ = is_aligned; + + uint64_t elapsed = 0; + if (use_direct_io() && is_aligned == false) { + FSReadRequest aligned_req = Align(req, alignment); + aligned_req.status.PermitUncheckedError(); + + // Allocate aligned buffer. + read_async_info->buf_.Alignment(alignment); + read_async_info->buf_.AllocateNewBuffer(aligned_req.len); + + // Set rem fields in aligned FSReadRequest. + aligned_req.scratch = read_async_info->buf_.BufferStart(); + + // Set user provided fields to populate back in callback. + read_async_info->user_scratch_ = req.scratch; + read_async_info->user_aligned_buf_ = aligned_buf; + read_async_info->user_len_ = req.len; + read_async_info->user_offset_ = req.offset; + read_async_info->user_result_ = req.result; + + assert(read_async_info->buf_.CurrentSize() == 0); + + StopWatch sw(clock_, nullptr /*stats*/, + Histograms::HISTOGRAM_ENUM_MAX /*hist_type*/, + Histograms::HISTOGRAM_ENUM_MAX, &elapsed, true /*overwrite*/, + true /*delay_enabled*/); + s = file_->ReadAsync(aligned_req, opts, read_async_callback, + read_async_info, io_handle, del_fn, nullptr /*dbg*/); + } else { + StopWatch sw(clock_, nullptr /*stats*/, + Histograms::HISTOGRAM_ENUM_MAX /*hist_type*/, + Histograms::HISTOGRAM_ENUM_MAX, &elapsed, true /*overwrite*/, + true /*delay_enabled*/); + s = file_->ReadAsync(req, opts, read_async_callback, read_async_info, + io_handle, del_fn, nullptr /*dbg*/); + } + RecordTick(stats_, READ_ASYNC_MICROS, elapsed); + +// Suppress false positive clang analyzer warnings. +// Memory is not released if file_->ReadAsync returns !s.ok(), because +// ReadAsyncCallback is never called in that case. If ReadAsyncCallback is +// called then ReadAsync should always return IOStatus::OK(). +#ifndef __clang_analyzer__ + if (!s.ok()) { + delete read_async_info; + } +#endif // __clang_analyzer__ + + return s; +} + +void RandomAccessFileReader::ReadAsyncCallback(const FSReadRequest& req, + void* cb_arg) { + ReadAsyncInfo* read_async_info = static_cast(cb_arg); + assert(read_async_info); + assert(read_async_info->cb_); + + if (use_direct_io() && read_async_info->is_aligned_ == false) { + // Create FSReadRequest with user provided fields. + FSReadRequest user_req; + user_req.scratch = read_async_info->user_scratch_; + user_req.offset = read_async_info->user_offset_; + user_req.len = read_async_info->user_len_; + + // Update results in user_req. + user_req.result = req.result; + user_req.status = req.status; + + read_async_info->buf_.Size(read_async_info->buf_.CurrentSize() + + req.result.size()); + + size_t offset_advance_len = static_cast( + /*offset_passed_by_user=*/read_async_info->user_offset_ - + /*aligned_offset=*/req.offset); + + size_t res_len = 0; + if (req.status.ok() && + offset_advance_len < read_async_info->buf_.CurrentSize()) { + res_len = + std::min(read_async_info->buf_.CurrentSize() - offset_advance_len, + read_async_info->user_len_); + if (read_async_info->user_aligned_buf_ == nullptr) { + // Copy the data into user's scratch. +// Clang analyzer assumes that it will take use_direct_io() == false in +// ReadAsync and use_direct_io() == true in Callback which cannot be true. +#ifndef __clang_analyzer__ + read_async_info->buf_.Read(user_req.scratch, offset_advance_len, + res_len); +#endif // __clang_analyzer__ + } else { + // Set aligned_buf provided by user without additional copy. + user_req.scratch = + read_async_info->buf_.BufferStart() + offset_advance_len; + read_async_info->user_aligned_buf_->reset( + read_async_info->buf_.Release()); + } + user_req.result = Slice(user_req.scratch, res_len); + } else { + // Either req.status is not ok or data was not read. + user_req.result = Slice(); + } + read_async_info->cb_(user_req, read_async_info->cb_arg_); + } else { + read_async_info->cb_(req, read_async_info->cb_arg_); + } + + // Update stats and notify listeners. + if (stats_ != nullptr && file_read_hist_ != nullptr) { + // elapsed doesn't take into account delay and overwrite as StopWatch does + // in Read. + uint64_t elapsed = clock_->NowMicros() - read_async_info->start_time_; + file_read_hist_->Add(elapsed); + } + if (req.status.ok()) { + RecordInHistogram(stats_, ASYNC_READ_BYTES, req.result.size()); + } else if (!req.status.IsAborted()) { + RecordTick(stats_, ASYNC_READ_ERROR_COUNT, 1); + } + if (ShouldNotifyListeners()) { + auto finish_ts = FileOperationInfo::FinishNow(); + NotifyOnFileReadFinish(req.offset, req.result.size(), + read_async_info->fs_start_ts_, finish_ts, + req.status); + } + if (!req.status.ok()) { + NotifyOnIOError(req.status, FileOperationType::kRead, file_name(), + req.result.size(), req.offset); + } + RecordIOStats(stats_, file_temperature_, is_last_level_, req.result.size()); + delete read_async_info; +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/file/random_access_file_reader.h b/librocksdb-sys/rocksdb/file/random_access_file_reader.h new file mode 100644 index 0000000..ab4d1e7 --- /dev/null +++ b/librocksdb-sys/rocksdb/file/random_access_file_reader.h @@ -0,0 +1,210 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include +#include +#include + +#include "env/file_system_tracer.h" +#include "port/port.h" +#include "rocksdb/file_system.h" +#include "rocksdb/listener.h" +#include "rocksdb/options.h" +#include "rocksdb/rate_limiter.h" +#include "util/aligned_buffer.h" + +namespace ROCKSDB_NAMESPACE { +class Statistics; +class HistogramImpl; +class SystemClock; + +using AlignedBuf = std::unique_ptr; + +// Align the request r according to alignment and return the aligned result. +FSReadRequest Align(const FSReadRequest& r, size_t alignment); + +// Try to merge src to dest if they have overlap. +// +// Each request represents an inclusive interval [offset, offset + len]. +// If the intervals have overlap, update offset and len to represent the +// merged interval, and return true. +// Otherwise, do nothing and return false. +bool TryMerge(FSReadRequest* dest, const FSReadRequest& src); + +// RandomAccessFileReader is a wrapper on top of FSRandomAccessFile. It is +// responsible for: +// - Handling Buffered and Direct reads appropriately. +// - Rate limiting compaction reads. +// - Notifying any interested listeners on the completion of a read. +// - Updating IO stats. +class RandomAccessFileReader { + private: + void NotifyOnFileReadFinish( + uint64_t offset, size_t length, + const FileOperationInfo::StartTimePoint& start_ts, + const FileOperationInfo::FinishTimePoint& finish_ts, + const Status& status) const { + FileOperationInfo info(FileOperationType::kRead, file_name_, start_ts, + finish_ts, status, file_temperature_); + info.offset = offset; + info.length = length; + + for (auto& listener : listeners_) { + listener->OnFileReadFinish(info); + } + info.status.PermitUncheckedError(); + } + + void NotifyOnIOError(const IOStatus& io_status, FileOperationType operation, + const std::string& file_path, size_t length, + uint64_t offset) const { + if (listeners_.empty()) { + return; + } + IOErrorInfo io_error_info(io_status, operation, file_path, length, offset); + + for (auto& listener : listeners_) { + listener->OnIOError(io_error_info); + } + io_status.PermitUncheckedError(); + } + + + bool ShouldNotifyListeners() const { return !listeners_.empty(); } + + FSRandomAccessFilePtr file_; + std::string file_name_; + SystemClock* clock_; + Statistics* stats_; + uint32_t hist_type_; + HistogramImpl* file_read_hist_; + RateLimiter* rate_limiter_; + std::vector> listeners_; + const Temperature file_temperature_; + const bool is_last_level_; + + struct ReadAsyncInfo { + ReadAsyncInfo(std::function cb, + void* cb_arg, uint64_t start_time) + : cb_(cb), + cb_arg_(cb_arg), + start_time_(start_time), + user_scratch_(nullptr), + user_aligned_buf_(nullptr), + user_offset_(0), + user_len_(0), + is_aligned_(false) {} + + std::function cb_; + void* cb_arg_; + uint64_t start_time_; + FileOperationInfo::StartTimePoint fs_start_ts_; + // Below fields stores the parameters passed by caller in case of direct_io. + char* user_scratch_; + AlignedBuf* user_aligned_buf_; + uint64_t user_offset_; + size_t user_len_; + Slice user_result_; + // Used in case of direct_io + AlignedBuffer buf_; + bool is_aligned_; + }; + + public: + explicit RandomAccessFileReader( + std::unique_ptr&& raf, const std::string& _file_name, + SystemClock* clock = nullptr, + const std::shared_ptr& io_tracer = nullptr, + Statistics* stats = nullptr, + uint32_t hist_type = Histograms::HISTOGRAM_ENUM_MAX, + HistogramImpl* file_read_hist = nullptr, + RateLimiter* rate_limiter = nullptr, + const std::vector>& listeners = {}, + Temperature file_temperature = Temperature::kUnknown, + bool is_last_level = false) + : file_(std::move(raf), io_tracer, _file_name), + file_name_(std::move(_file_name)), + clock_(clock), + stats_(stats), + hist_type_(hist_type), + file_read_hist_(file_read_hist), + rate_limiter_(rate_limiter), + listeners_(), + file_temperature_(file_temperature), + is_last_level_(is_last_level) { + std::for_each(listeners.begin(), listeners.end(), + [this](const std::shared_ptr& e) { + if (e->ShouldBeNotifiedOnFileIO()) { + listeners_.emplace_back(e); + } + }); + } + + static IOStatus Create(const std::shared_ptr& fs, + const std::string& fname, const FileOptions& file_opts, + std::unique_ptr* reader, + IODebugContext* dbg); + RandomAccessFileReader(const RandomAccessFileReader&) = delete; + RandomAccessFileReader& operator=(const RandomAccessFileReader&) = delete; + + // In non-direct IO mode, + // 1. if using mmap, result is stored in a buffer other than scratch; + // 2. if not using mmap, result is stored in the buffer starting from scratch. + // + // In direct IO mode, an aligned buffer is allocated internally. + // 1. If aligned_buf is null, then results are copied to the buffer + // starting from scratch; + // 2. Otherwise, scratch is not used and can be null, the aligned_buf owns + // the internally allocated buffer on return, and the result refers to a + // region in aligned_buf. + // + // `rate_limiter_priority` is used to charge the internal rate limiter when + // enabled. The special value `Env::IO_TOTAL` makes this operation bypass the + // rate limiter. + IOStatus Read(const IOOptions& opts, uint64_t offset, size_t n, Slice* result, + char* scratch, AlignedBuf* aligned_buf, + Env::IOPriority rate_limiter_priority) const; + + // REQUIRES: + // num_reqs > 0, reqs do not overlap, and offsets in reqs are increasing. + // In non-direct IO mode, aligned_buf should be null; + // In direct IO mode, aligned_buf stores the aligned buffer allocated inside + // MultiRead, the result Slices in reqs refer to aligned_buf. + // + // `rate_limiter_priority` will be used to charge the internal rate limiter. + // It is not yet supported so the client must provide the special value + // `Env::IO_TOTAL` to bypass the rate limiter. + IOStatus MultiRead(const IOOptions& opts, FSReadRequest* reqs, + size_t num_reqs, AlignedBuf* aligned_buf, + Env::IOPriority rate_limiter_priority) const; + + IOStatus Prefetch(uint64_t offset, size_t n, + const Env::IOPriority rate_limiter_priority) const { + IOOptions opts; + opts.rate_limiter_priority = rate_limiter_priority; + return file_->Prefetch(offset, n, opts, nullptr); + } + + FSRandomAccessFile* file() { return file_.get(); } + + const std::string& file_name() const { return file_name_; } + + bool use_direct_io() const { return file_->use_direct_io(); } + + IOStatus PrepareIOOptions(const ReadOptions& ro, IOOptions& opts) const; + + IOStatus ReadAsync(FSReadRequest& req, const IOOptions& opts, + std::function cb, + void* cb_arg, void** io_handle, IOHandleDeleter* del_fn, + AlignedBuf* aligned_buf); + + void ReadAsyncCallback(const FSReadRequest& req, void* cb_arg); +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/file/random_access_file_reader_test.cc b/librocksdb-sys/rocksdb/file/random_access_file_reader_test.cc new file mode 100644 index 0000000..82ddcff --- /dev/null +++ b/librocksdb-sys/rocksdb/file/random_access_file_reader_test.cc @@ -0,0 +1,489 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "file/random_access_file_reader.h" + +#include + +#include "file/file_util.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/file_system.h" +#include "test_util/sync_point.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +class RandomAccessFileReaderTest : public testing::Test { + public: + void SetUp() override { + SetupSyncPointsToMockDirectIO(); + env_ = Env::Default(); + fs_ = FileSystem::Default(); + test_dir_ = test::PerThreadDBPath("random_access_file_reader_test"); + ASSERT_OK(fs_->CreateDir(test_dir_, IOOptions(), nullptr)); + } + + void TearDown() override { EXPECT_OK(DestroyDir(env_, test_dir_)); } + + void Write(const std::string& fname, const std::string& content) { + std::unique_ptr f; + ASSERT_OK(fs_->NewWritableFile(Path(fname), FileOptions(), &f, nullptr)); + ASSERT_OK(f->Append(content, IOOptions(), nullptr)); + ASSERT_OK(f->Close(IOOptions(), nullptr)); + } + + void Read(const std::string& fname, const FileOptions& opts, + std::unique_ptr* reader) { + std::string fpath = Path(fname); + std::unique_ptr f; + ASSERT_OK(fs_->NewRandomAccessFile(fpath, opts, &f, nullptr)); + reader->reset(new RandomAccessFileReader(std::move(f), fpath, + env_->GetSystemClock().get())); + } + + void AssertResult(const std::string& content, + const std::vector& reqs) { + for (const auto& r : reqs) { + ASSERT_OK(r.status); + ASSERT_EQ(r.len, r.result.size()); + ASSERT_EQ(content.substr(r.offset, r.len), r.result.ToString()); + } + } + + private: + Env* env_; + std::shared_ptr fs_; + std::string test_dir_; + + std::string Path(const std::string& fname) { return test_dir_ + "/" + fname; } +}; + +// Skip the following tests in lite mode since direct I/O is unsupported. + +TEST_F(RandomAccessFileReaderTest, ReadDirectIO) { + std::string fname = "read-direct-io"; + Random rand(0); + std::string content = rand.RandomString(kDefaultPageSize); + Write(fname, content); + + FileOptions opts; + opts.use_direct_reads = true; + std::unique_ptr r; + Read(fname, opts, &r); + ASSERT_TRUE(r->use_direct_io()); + + const size_t page_size = r->file()->GetRequiredBufferAlignment(); + size_t offset = page_size / 2; + size_t len = page_size / 3; + Slice result; + AlignedBuf buf; + for (Env::IOPriority rate_limiter_priority : {Env::IO_LOW, Env::IO_TOTAL}) { + ASSERT_OK(r->Read(IOOptions(), offset, len, &result, nullptr, &buf, + rate_limiter_priority)); + ASSERT_EQ(result.ToString(), content.substr(offset, len)); + } +} + +TEST_F(RandomAccessFileReaderTest, MultiReadDirectIO) { + std::vector aligned_reqs; + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "RandomAccessFileReader::MultiRead:AlignedReqs", [&](void* reqs) { + // Copy reqs, since it's allocated on stack inside MultiRead, which will + // be deallocated after MultiRead returns. + size_t i = 0; + aligned_reqs.resize( + (*reinterpret_cast*>(reqs)).size()); + for (auto& req : + (*reinterpret_cast*>(reqs))) { + aligned_reqs[i].offset = req.offset; + aligned_reqs[i].len = req.len; + aligned_reqs[i].result = req.result; + aligned_reqs[i].status = req.status; + aligned_reqs[i].scratch = req.scratch; + i++; + } + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + // Creates a file with 3 pages. + std::string fname = "multi-read-direct-io"; + Random rand(0); + std::string content = rand.RandomString(3 * kDefaultPageSize); + Write(fname, content); + + FileOptions opts; + opts.use_direct_reads = true; + std::unique_ptr r; + Read(fname, opts, &r); + ASSERT_TRUE(r->use_direct_io()); + + const size_t page_size = r->file()->GetRequiredBufferAlignment(); + + { + // Reads 2 blocks in the 1st page. + // The results should be SharedSlices of the same underlying buffer. + // + // Illustration (each x is a 1/4 page) + // First page: xxxx + // 1st block: x + // 2nd block: xx + FSReadRequest r0; + r0.offset = 0; + r0.len = page_size / 4; + r0.scratch = nullptr; + + FSReadRequest r1; + r1.offset = page_size / 2; + r1.len = page_size / 2; + r1.scratch = nullptr; + + std::vector reqs; + reqs.push_back(std::move(r0)); + reqs.push_back(std::move(r1)); + AlignedBuf aligned_buf; + ASSERT_OK(r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf, + Env::IO_TOTAL /*rate_limiter_priority*/)); + + AssertResult(content, reqs); + + // Reads the first page internally. + ASSERT_EQ(aligned_reqs.size(), 1); + const FSReadRequest& aligned_r = aligned_reqs[0]; + ASSERT_OK(aligned_r.status); + ASSERT_EQ(aligned_r.offset, 0); + ASSERT_EQ(aligned_r.len, page_size); + } + + { + // Reads 3 blocks: + // 1st block in the 1st page; + // 2nd block from the middle of the 1st page to the middle of the 2nd page; + // 3rd block in the 2nd page. + // The results should be SharedSlices of the same underlying buffer. + // + // Illustration (each x is a 1/4 page) + // 2 pages: xxxxxxxx + // 1st block: x + // 2nd block: xxxx + // 3rd block: x + FSReadRequest r0; + r0.offset = 0; + r0.len = page_size / 4; + r0.scratch = nullptr; + + FSReadRequest r1; + r1.offset = page_size / 2; + r1.len = page_size; + r1.scratch = nullptr; + + FSReadRequest r2; + r2.offset = 2 * page_size - page_size / 4; + r2.len = page_size / 4; + r2.scratch = nullptr; + + std::vector reqs; + reqs.push_back(std::move(r0)); + reqs.push_back(std::move(r1)); + reqs.push_back(std::move(r2)); + AlignedBuf aligned_buf; + ASSERT_OK(r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf, + Env::IO_TOTAL /*rate_limiter_priority*/)); + + AssertResult(content, reqs); + + // Reads the first two pages in one request internally. + ASSERT_EQ(aligned_reqs.size(), 1); + const FSReadRequest& aligned_r = aligned_reqs[0]; + ASSERT_OK(aligned_r.status); + ASSERT_EQ(aligned_r.offset, 0); + ASSERT_EQ(aligned_r.len, 2 * page_size); + } + + { + // Reads 3 blocks: + // 1st block in the middle of the 1st page; + // 2nd block in the middle of the 2nd page; + // 3rd block in the middle of the 3rd page. + // The results should be SharedSlices of the same underlying buffer. + // + // Illustration (each x is a 1/4 page) + // 3 pages: xxxxxxxxxxxx + // 1st block: xx + // 2nd block: xx + // 3rd block: xx + FSReadRequest r0; + r0.offset = page_size / 4; + r0.len = page_size / 2; + r0.scratch = nullptr; + + FSReadRequest r1; + r1.offset = page_size + page_size / 4; + r1.len = page_size / 2; + r1.scratch = nullptr; + + FSReadRequest r2; + r2.offset = 2 * page_size + page_size / 4; + r2.len = page_size / 2; + r2.scratch = nullptr; + + std::vector reqs; + reqs.push_back(std::move(r0)); + reqs.push_back(std::move(r1)); + reqs.push_back(std::move(r2)); + AlignedBuf aligned_buf; + ASSERT_OK(r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf, + Env::IO_TOTAL /*rate_limiter_priority*/)); + + AssertResult(content, reqs); + + // Reads the first 3 pages in one request internally. + ASSERT_EQ(aligned_reqs.size(), 1); + const FSReadRequest& aligned_r = aligned_reqs[0]; + ASSERT_OK(aligned_r.status); + ASSERT_EQ(aligned_r.offset, 0); + ASSERT_EQ(aligned_r.len, 3 * page_size); + } + + { + // Reads 2 blocks: + // 1st block in the middle of the 1st page; + // 2nd block in the middle of the 3rd page. + // The results are two different buffers. + // + // Illustration (each x is a 1/4 page) + // 3 pages: xxxxxxxxxxxx + // 1st block: xx + // 2nd block: xx + FSReadRequest r0; + r0.offset = page_size / 4; + r0.len = page_size / 2; + r0.scratch = nullptr; + + FSReadRequest r1; + r1.offset = 2 * page_size + page_size / 4; + r1.len = page_size / 2; + r1.scratch = nullptr; + + std::vector reqs; + reqs.push_back(std::move(r0)); + reqs.push_back(std::move(r1)); + AlignedBuf aligned_buf; + ASSERT_OK(r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf, + Env::IO_TOTAL /*rate_limiter_priority*/)); + + AssertResult(content, reqs); + + // Reads the 1st and 3rd pages in two requests internally. + ASSERT_EQ(aligned_reqs.size(), 2); + const FSReadRequest& aligned_r0 = aligned_reqs[0]; + const FSReadRequest& aligned_r1 = aligned_reqs[1]; + ASSERT_OK(aligned_r0.status); + ASSERT_EQ(aligned_r0.offset, 0); + ASSERT_EQ(aligned_r0.len, page_size); + ASSERT_OK(aligned_r1.status); + ASSERT_EQ(aligned_r1.offset, 2 * page_size); + ASSERT_EQ(aligned_r1.len, page_size); + } + + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST(FSReadRequest, Align) { + FSReadRequest r; + r.offset = 2000; + r.len = 2000; + r.scratch = nullptr; + ASSERT_OK(r.status); + + FSReadRequest aligned_r = Align(r, 1024); + ASSERT_OK(r.status); + ASSERT_OK(aligned_r.status); + ASSERT_EQ(aligned_r.offset, 1024); + ASSERT_EQ(aligned_r.len, 3072); +} + +TEST(FSReadRequest, TryMerge) { + // reverse means merging dest into src. + for (bool reverse : {true, false}) { + { + // dest: [ ] + // src: [ ] + FSReadRequest dest; + dest.offset = 0; + dest.len = 10; + dest.scratch = nullptr; + ASSERT_OK(dest.status); + + FSReadRequest src; + src.offset = 15; + src.len = 10; + src.scratch = nullptr; + ASSERT_OK(src.status); + + if (reverse) { + std::swap(dest, src); + } + ASSERT_FALSE(TryMerge(&dest, src)); + ASSERT_OK(dest.status); + ASSERT_OK(src.status); + } + + { + // dest: [ ] + // src: [ ] + FSReadRequest dest; + dest.offset = 0; + dest.len = 10; + dest.scratch = nullptr; + ASSERT_OK(dest.status); + + FSReadRequest src; + src.offset = 10; + src.len = 10; + src.scratch = nullptr; + ASSERT_OK(src.status); + + if (reverse) { + std::swap(dest, src); + } + ASSERT_TRUE(TryMerge(&dest, src)); + ASSERT_EQ(dest.offset, 0); + ASSERT_EQ(dest.len, 20); + ASSERT_OK(dest.status); + ASSERT_OK(src.status); + } + + { + // dest: [ ] + // src: [ ] + FSReadRequest dest; + dest.offset = 0; + dest.len = 10; + dest.scratch = nullptr; + ASSERT_OK(dest.status); + + FSReadRequest src; + src.offset = 5; + src.len = 10; + src.scratch = nullptr; + ASSERT_OK(src.status); + + if (reverse) { + std::swap(dest, src); + } + ASSERT_TRUE(TryMerge(&dest, src)); + ASSERT_EQ(dest.offset, 0); + ASSERT_EQ(dest.len, 15); + ASSERT_OK(dest.status); + ASSERT_OK(src.status); + } + + { + // dest: [ ] + // src: [ ] + FSReadRequest dest; + dest.offset = 0; + dest.len = 10; + dest.scratch = nullptr; + ASSERT_OK(dest.status); + + FSReadRequest src; + src.offset = 5; + src.len = 5; + src.scratch = nullptr; + ASSERT_OK(src.status); + + if (reverse) { + std::swap(dest, src); + } + ASSERT_TRUE(TryMerge(&dest, src)); + ASSERT_EQ(dest.offset, 0); + ASSERT_EQ(dest.len, 10); + ASSERT_OK(dest.status); + ASSERT_OK(src.status); + } + + { + // dest: [ ] + // src: [ ] + FSReadRequest dest; + dest.offset = 0; + dest.len = 10; + dest.scratch = nullptr; + ASSERT_OK(dest.status); + + FSReadRequest src; + src.offset = 5; + src.len = 1; + src.scratch = nullptr; + ASSERT_OK(src.status); + + if (reverse) std::swap(dest, src); + ASSERT_TRUE(TryMerge(&dest, src)); + ASSERT_EQ(dest.offset, 0); + ASSERT_EQ(dest.len, 10); + ASSERT_OK(dest.status); + ASSERT_OK(src.status); + } + + { + // dest: [ ] + // src: [ ] + FSReadRequest dest; + dest.offset = 0; + dest.len = 10; + dest.scratch = nullptr; + ASSERT_OK(dest.status); + + FSReadRequest src; + src.offset = 0; + src.len = 10; + src.scratch = nullptr; + ASSERT_OK(src.status); + + if (reverse) std::swap(dest, src); + ASSERT_TRUE(TryMerge(&dest, src)); + ASSERT_EQ(dest.offset, 0); + ASSERT_EQ(dest.len, 10); + ASSERT_OK(dest.status); + ASSERT_OK(src.status); + } + + { + // dest: [ ] + // src: [ ] + FSReadRequest dest; + dest.offset = 0; + dest.len = 10; + dest.scratch = nullptr; + ASSERT_OK(dest.status); + + FSReadRequest src; + src.offset = 0; + src.len = 5; + src.scratch = nullptr; + ASSERT_OK(src.status); + + if (reverse) std::swap(dest, src); + ASSERT_TRUE(TryMerge(&dest, src)); + ASSERT_EQ(dest.offset, 0); + ASSERT_EQ(dest.len, 10); + ASSERT_OK(dest.status); + ASSERT_OK(src.status); + } + } +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/file/read_write_util.cc b/librocksdb-sys/rocksdb/file/read_write_util.cc new file mode 100644 index 0000000..3617a35 --- /dev/null +++ b/librocksdb-sys/rocksdb/file/read_write_util.cc @@ -0,0 +1,33 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "file/read_write_util.h" + +#include + +#include "test_util/sync_point.h" + +namespace ROCKSDB_NAMESPACE { + +IOStatus NewWritableFile(FileSystem* fs, const std::string& fname, + std::unique_ptr* result, + const FileOptions& options) { + TEST_SYNC_POINT_CALLBACK("NewWritableFile::FileOptions.temperature", + const_cast(&options.temperature)); + IOStatus s = fs->NewWritableFile(fname, options, result, nullptr); + TEST_KILL_RANDOM_WITH_WEIGHT("NewWritableFile:0", REDUCE_ODDS2); + return s; +} + +#ifndef NDEBUG +bool IsFileSectorAligned(const size_t off, size_t sector_size) { + return off % sector_size == 0; +} +#endif // NDEBUG +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/file/read_write_util.h b/librocksdb-sys/rocksdb/file/read_write_util.h new file mode 100644 index 0000000..9f034b7 --- /dev/null +++ b/librocksdb-sys/rocksdb/file/read_write_util.h @@ -0,0 +1,31 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include + +#include "file/sequence_file_reader.h" +#include "rocksdb/env.h" +#include "rocksdb/file_system.h" + +namespace ROCKSDB_NAMESPACE { +// Returns a WritableFile. +// +// env : the Env. +// fname : the file name. +// result : output arg. A WritableFile based on `fname` returned. +// options : the Env Options. +extern IOStatus NewWritableFile(FileSystem* fs, const std::string& fname, + std::unique_ptr* result, + const FileOptions& options); + +#ifndef NDEBUG +bool IsFileSectorAligned(const size_t off, size_t sector_size); +#endif // NDEBUG +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/file/readahead_file_info.h b/librocksdb-sys/rocksdb/file/readahead_file_info.h new file mode 100644 index 0000000..f0208bf --- /dev/null +++ b/librocksdb-sys/rocksdb/file/readahead_file_info.h @@ -0,0 +1,33 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +// struct ReadaheadFileInfo contains readahead information that is passed from +// one file to another file per level during iterations. This information helps +// iterators to carry forward the internal automatic prefetching readahead value +// to next file during sequential reads instead of starting from the scratch. + +struct ReadaheadFileInfo { + struct ReadaheadInfo { + size_t readahead_size = 0; + int64_t num_file_reads = 0; + }; + + // Used by Data block iterators to update readahead info. + ReadaheadInfo data_block_readahead_info; + + // Used by Index block iterators to update readahead info. + ReadaheadInfo index_block_readahead_info; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/file/readahead_raf.cc b/librocksdb-sys/rocksdb/file/readahead_raf.cc new file mode 100644 index 0000000..dd09822 --- /dev/null +++ b/librocksdb-sys/rocksdb/file/readahead_raf.cc @@ -0,0 +1,169 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "file/readahead_raf.h" + +#include +#include + +#include "file/read_write_util.h" +#include "rocksdb/file_system.h" +#include "util/aligned_buffer.h" +#include "util/rate_limiter_impl.h" + +namespace ROCKSDB_NAMESPACE { +namespace { +class ReadaheadRandomAccessFile : public FSRandomAccessFile { + public: + ReadaheadRandomAccessFile(std::unique_ptr&& file, + size_t readahead_size) + : file_(std::move(file)), + alignment_(file_->GetRequiredBufferAlignment()), + readahead_size_(Roundup(readahead_size, alignment_)), + buffer_(), + buffer_offset_(0) { + buffer_.Alignment(alignment_); + buffer_.AllocateNewBuffer(readahead_size_); + } + + ReadaheadRandomAccessFile(const ReadaheadRandomAccessFile&) = delete; + + ReadaheadRandomAccessFile& operator=(const ReadaheadRandomAccessFile&) = + delete; + + IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const override { + // Read-ahead only make sense if we have some slack left after reading + if (n + alignment_ >= readahead_size_) { + return file_->Read(offset, n, options, result, scratch, dbg); + } + + std::unique_lock lk(lock_); + + size_t cached_len = 0; + // Check if there is a cache hit, meaning that [offset, offset + n) is + // either completely or partially in the buffer. If it's completely cached, + // including end of file case when offset + n is greater than EOF, then + // return. + if (TryReadFromCache(offset, n, &cached_len, scratch) && + (cached_len == n || buffer_.CurrentSize() < readahead_size_)) { + // We read exactly what we needed, or we hit end of file - return. + *result = Slice(scratch, cached_len); + return IOStatus::OK(); + } + size_t advanced_offset = static_cast(offset + cached_len); + // In the case of cache hit advanced_offset is already aligned, means that + // chunk_offset equals to advanced_offset + size_t chunk_offset = TruncateToPageBoundary(alignment_, advanced_offset); + + IOStatus s = ReadIntoBuffer(chunk_offset, readahead_size_, options, dbg); + if (s.ok()) { + // The data we need is now in cache, so we can safely read it + size_t remaining_len; + TryReadFromCache(advanced_offset, n - cached_len, &remaining_len, + scratch + cached_len); + *result = Slice(scratch, cached_len + remaining_len); + } + return s; + } + + IOStatus Prefetch(uint64_t offset, size_t n, const IOOptions& options, + IODebugContext* dbg) override { + if (n < readahead_size_) { + // Don't allow smaller prefetches than the configured `readahead_size_`. + // `Read()` assumes a smaller prefetch buffer indicates EOF was reached. + return IOStatus::OK(); + } + + std::unique_lock lk(lock_); + + size_t offset_ = static_cast(offset); + size_t prefetch_offset = TruncateToPageBoundary(alignment_, offset_); + if (prefetch_offset == buffer_offset_) { + return IOStatus::OK(); + } + return ReadIntoBuffer(prefetch_offset, + Roundup(offset_ + n, alignment_) - prefetch_offset, + options, dbg); + } + + size_t GetUniqueId(char* id, size_t max_size) const override { + return file_->GetUniqueId(id, max_size); + } + + void Hint(AccessPattern pattern) override { file_->Hint(pattern); } + + IOStatus InvalidateCache(size_t offset, size_t length) override { + std::unique_lock lk(lock_); + buffer_.Clear(); + return file_->InvalidateCache(offset, length); + } + + bool use_direct_io() const override { return file_->use_direct_io(); } + + private: + // Tries to read from buffer_ n bytes starting at offset. If anything was read + // from the cache, it sets cached_len to the number of bytes actually read, + // copies these number of bytes to scratch and returns true. + // If nothing was read sets cached_len to 0 and returns false. + bool TryReadFromCache(uint64_t offset, size_t n, size_t* cached_len, + char* scratch) const { + if (offset < buffer_offset_ || + offset >= buffer_offset_ + buffer_.CurrentSize()) { + *cached_len = 0; + return false; + } + uint64_t offset_in_buffer = offset - buffer_offset_; + *cached_len = std::min( + buffer_.CurrentSize() - static_cast(offset_in_buffer), n); + memcpy(scratch, buffer_.BufferStart() + offset_in_buffer, *cached_len); + return true; + } + + // Reads into buffer_ the next n bytes from file_ starting at offset. + // Can actually read less if EOF was reached. + // Returns the status of the read operastion on the file. + IOStatus ReadIntoBuffer(uint64_t offset, size_t n, const IOOptions& options, + IODebugContext* dbg) const { + if (n > buffer_.Capacity()) { + n = buffer_.Capacity(); + } + assert(IsFileSectorAligned(offset, alignment_)); + assert(IsFileSectorAligned(n, alignment_)); + Slice result; + IOStatus s = + file_->Read(offset, n, options, &result, buffer_.BufferStart(), dbg); + if (s.ok()) { + buffer_offset_ = offset; + buffer_.Size(result.size()); + assert(result.size() == 0 || buffer_.BufferStart() == result.data()); + } + return s; + } + + const std::unique_ptr file_; + const size_t alignment_; + const size_t readahead_size_; + + mutable std::mutex lock_; + // The buffer storing the prefetched data + mutable AlignedBuffer buffer_; + // The offset in file_, corresponding to data stored in buffer_ + mutable uint64_t buffer_offset_; +}; +} // namespace + +std::unique_ptr NewReadaheadRandomAccessFile( + std::unique_ptr&& file, size_t readahead_size) { + std::unique_ptr result( + new ReadaheadRandomAccessFile(std::move(file), readahead_size)); + return result; +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/file/readahead_raf.h b/librocksdb-sys/rocksdb/file/readahead_raf.h new file mode 100644 index 0000000..dfaf2b4 --- /dev/null +++ b/librocksdb-sys/rocksdb/file/readahead_raf.h @@ -0,0 +1,29 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { +class FSRandomAccessFile; +// This file provides the following main abstractions: +// SequentialFileReader : wrapper over Env::SequentialFile +// RandomAccessFileReader : wrapper over Env::RandomAccessFile +// WritableFileWriter : wrapper over Env::WritableFile +// In addition, it also exposed NewReadaheadRandomAccessFile, NewWritableFile, +// and ReadOneLine primitives. + +// NewReadaheadRandomAccessFile provides a wrapper over RandomAccessFile to +// always prefetch additional data with every read. This is mainly used in +// Compaction Table Readers. +std::unique_ptr NewReadaheadRandomAccessFile( + std::unique_ptr&& file, size_t readahead_size); +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/file/sequence_file_reader.cc b/librocksdb-sys/rocksdb/file/sequence_file_reader.cc new file mode 100644 index 0000000..a753c1d --- /dev/null +++ b/librocksdb-sys/rocksdb/file/sequence_file_reader.cc @@ -0,0 +1,320 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "file/sequence_file_reader.h" + +#include +#include + +#include "file/read_write_util.h" +#include "monitoring/histogram.h" +#include "monitoring/iostats_context_imp.h" +#include "port/port.h" +#include "test_util/sync_point.h" +#include "util/aligned_buffer.h" +#include "util/random.h" +#include "util/rate_limiter_impl.h" + +namespace ROCKSDB_NAMESPACE { +IOStatus SequentialFileReader::Create( + const std::shared_ptr& fs, const std::string& fname, + const FileOptions& file_opts, std::unique_ptr* reader, + IODebugContext* dbg, RateLimiter* rate_limiter) { + std::unique_ptr file; + IOStatus io_s = fs->NewSequentialFile(fname, file_opts, &file, dbg); + if (io_s.ok()) { + reader->reset(new SequentialFileReader(std::move(file), fname, nullptr, {}, + rate_limiter)); + } + return io_s; +} + +IOStatus SequentialFileReader::Read(size_t n, Slice* result, char* scratch, + Env::IOPriority rate_limiter_priority) { + IOStatus io_s; + if (use_direct_io()) { + // + // |-offset_advance-|---bytes returned--| + // |----------------------buf size-------------------------| + // | | | | + // aligned offset offset + n Roundup(offset + n, + // offset alignment) + // + size_t offset = offset_.fetch_add(n); + size_t alignment = file_->GetRequiredBufferAlignment(); + size_t aligned_offset = TruncateToPageBoundary(alignment, offset); + size_t offset_advance = offset - aligned_offset; + size_t size = Roundup(offset + n, alignment) - aligned_offset; + size_t r = 0; + AlignedBuffer buf; + buf.Alignment(alignment); + buf.AllocateNewBuffer(size); + + while (buf.CurrentSize() < size) { + size_t allowed; + if (rate_limiter_priority != Env::IO_TOTAL && rate_limiter_ != nullptr) { + allowed = rate_limiter_->RequestToken( + buf.Capacity() - buf.CurrentSize(), buf.Alignment(), + rate_limiter_priority, nullptr /* stats */, + RateLimiter::OpType::kRead); + } else { + assert(buf.CurrentSize() == 0); + allowed = size; + } + + Slice tmp; + uint64_t orig_offset = 0; + FileOperationInfo::StartTimePoint start_ts; + if (ShouldNotifyListeners()) { + orig_offset = aligned_offset + buf.CurrentSize(); + start_ts = FileOperationInfo::StartNow(); + } + io_s = file_->PositionedRead(aligned_offset + buf.CurrentSize(), allowed, + IOOptions(), &tmp, buf.Destination(), + nullptr /* dbg */); + if (ShouldNotifyListeners()) { + auto finish_ts = FileOperationInfo::FinishNow(); + NotifyOnFileReadFinish(orig_offset, tmp.size(), start_ts, finish_ts, + io_s); + } + buf.Size(buf.CurrentSize() + tmp.size()); + if (!io_s.ok() || tmp.size() < allowed) { + break; + } + } + + if (io_s.ok() && offset_advance < buf.CurrentSize()) { + r = buf.Read(scratch, offset_advance, + std::min(buf.CurrentSize() - offset_advance, n)); + } + *result = Slice(scratch, r); + } else { + // To be paranoid, modify scratch a little bit, so in case underlying + // FileSystem doesn't fill the buffer but return success and `scratch` + // returns contains a previous block, returned value will not pass + // checksum. + // It's hard to find useful byte for direct I/O case, so we skip it. + if (n > 0 && scratch != nullptr) { + scratch[0]++; + } + + size_t read = 0; + while (read < n) { + size_t allowed; + if (rate_limiter_priority != Env::IO_TOTAL && rate_limiter_ != nullptr) { + allowed = rate_limiter_->RequestToken( + n - read, 0 /* alignment */, rate_limiter_priority, + nullptr /* stats */, RateLimiter::OpType::kRead); + } else { + allowed = n; + } + FileOperationInfo::StartTimePoint start_ts; + if (ShouldNotifyListeners()) { + start_ts = FileOperationInfo::StartNow(); + } + Slice tmp; + io_s = file_->Read(allowed, IOOptions(), &tmp, scratch + read, + nullptr /* dbg */); + if (ShouldNotifyListeners()) { + auto finish_ts = FileOperationInfo::FinishNow(); + size_t offset = offset_.fetch_add(tmp.size()); + NotifyOnFileReadFinish(offset, tmp.size(), start_ts, finish_ts, io_s); + } + read += tmp.size(); + if (!io_s.ok() || tmp.size() < allowed) { + break; + } + } + *result = Slice(scratch, read); + } + IOSTATS_ADD(bytes_read, result->size()); + return io_s; +} + +IOStatus SequentialFileReader::Skip(uint64_t n) { + if (use_direct_io()) { + offset_ += static_cast(n); + return IOStatus::OK(); + } + return file_->Skip(n); +} + +namespace { +// This class wraps a SequentialFile, exposing same API, with the differenece +// of being able to prefetch up to readahead_size bytes and then serve them +// from memory, avoiding the entire round-trip if, for example, the data for the +// file is actually remote. +class ReadaheadSequentialFile : public FSSequentialFile { + public: + ReadaheadSequentialFile(std::unique_ptr&& file, + size_t readahead_size) + : file_(std::move(file)), + alignment_(file_->GetRequiredBufferAlignment()), + readahead_size_(Roundup(readahead_size, alignment_)), + buffer_(), + buffer_offset_(0), + read_offset_(0) { + buffer_.Alignment(alignment_); + buffer_.AllocateNewBuffer(readahead_size_); + } + + ReadaheadSequentialFile(const ReadaheadSequentialFile&) = delete; + + ReadaheadSequentialFile& operator=(const ReadaheadSequentialFile&) = delete; + + IOStatus Read(size_t n, const IOOptions& opts, Slice* result, char* scratch, + IODebugContext* dbg) override { + std::unique_lock lk(lock_); + + size_t cached_len = 0; + // Check if there is a cache hit, meaning that [offset, offset + n) is + // either completely or partially in the buffer. If it's completely cached, + // including end of file case when offset + n is greater than EOF, then + // return. + if (TryReadFromCache(n, &cached_len, scratch) && + (cached_len == n || buffer_.CurrentSize() < readahead_size_)) { + // We read exactly what we needed, or we hit end of file - return. + *result = Slice(scratch, cached_len); + return IOStatus::OK(); + } + n -= cached_len; + + IOStatus s; + // Read-ahead only make sense if we have some slack left after reading + if (n + alignment_ >= readahead_size_) { + s = file_->Read(n, opts, result, scratch + cached_len, dbg); + if (s.ok()) { + read_offset_ += result->size(); + *result = Slice(scratch, cached_len + result->size()); + } + buffer_.Clear(); + return s; + } + + s = ReadIntoBuffer(readahead_size_, opts, dbg); + if (s.ok()) { + // The data we need is now in cache, so we can safely read it + size_t remaining_len; + TryReadFromCache(n, &remaining_len, scratch + cached_len); + *result = Slice(scratch, cached_len + remaining_len); + } + return s; + } + + IOStatus Skip(uint64_t n) override { + std::unique_lock lk(lock_); + IOStatus s = IOStatus::OK(); + // First check if we need to skip already cached data + if (buffer_.CurrentSize() > 0) { + // Do we need to skip beyond cached data? + if (read_offset_ + n >= buffer_offset_ + buffer_.CurrentSize()) { + // Yes. Skip whaterver is in memory and adjust offset accordingly + n -= buffer_offset_ + buffer_.CurrentSize() - read_offset_; + read_offset_ = buffer_offset_ + buffer_.CurrentSize(); + } else { + // No. The entire section to be skipped is entirely i cache. + read_offset_ += n; + n = 0; + } + } + if (n > 0) { + // We still need to skip more, so call the file API for skipping + s = file_->Skip(n); + if (s.ok()) { + read_offset_ += n; + } + buffer_.Clear(); + } + return s; + } + + IOStatus PositionedRead(uint64_t offset, size_t n, const IOOptions& opts, + Slice* result, char* scratch, + IODebugContext* dbg) override { + return file_->PositionedRead(offset, n, opts, result, scratch, dbg); + } + + IOStatus InvalidateCache(size_t offset, size_t length) override { + std::unique_lock lk(lock_); + buffer_.Clear(); + return file_->InvalidateCache(offset, length); + } + + bool use_direct_io() const override { return file_->use_direct_io(); } + + private: + // Tries to read from buffer_ n bytes. If anything was read from the cache, it + // sets cached_len to the number of bytes actually read, copies these number + // of bytes to scratch and returns true. + // If nothing was read sets cached_len to 0 and returns false. + bool TryReadFromCache(size_t n, size_t* cached_len, char* scratch) { + if (read_offset_ < buffer_offset_ || + read_offset_ >= buffer_offset_ + buffer_.CurrentSize()) { + *cached_len = 0; + return false; + } + uint64_t offset_in_buffer = read_offset_ - buffer_offset_; + *cached_len = std::min( + buffer_.CurrentSize() - static_cast(offset_in_buffer), n); + memcpy(scratch, buffer_.BufferStart() + offset_in_buffer, *cached_len); + read_offset_ += *cached_len; + return true; + } + + // Reads into buffer_ the next n bytes from file_. + // Can actually read less if EOF was reached. + // Returns the status of the read operastion on the file. + IOStatus ReadIntoBuffer(size_t n, const IOOptions& opts, + IODebugContext* dbg) { + if (n > buffer_.Capacity()) { + n = buffer_.Capacity(); + } + assert(IsFileSectorAligned(n, alignment_)); + Slice result; + IOStatus s = file_->Read(n, opts, &result, buffer_.BufferStart(), dbg); + if (s.ok()) { + buffer_offset_ = read_offset_; + buffer_.Size(result.size()); + assert(result.size() == 0 || buffer_.BufferStart() == result.data()); + } + return s; + } + + const std::unique_ptr file_; + const size_t alignment_; + const size_t readahead_size_; + + std::mutex lock_; + // The buffer storing the prefetched data + AlignedBuffer buffer_; + // The offset in file_, corresponding to data stored in buffer_ + uint64_t buffer_offset_; + // The offset up to which data was read from file_. In fact, it can be larger + // than the actual file size, since the file_->Skip(n) call doesn't return the + // actual number of bytes that were skipped, which can be less than n. + // This is not a problemm since read_offset_ is monotonically increasing and + // its only use is to figure out if next piece of data should be read from + // buffer_ or file_ directly. + uint64_t read_offset_; +}; +} // namespace + +std::unique_ptr +SequentialFileReader::NewReadaheadSequentialFile( + std::unique_ptr&& file, size_t readahead_size) { + if (file->GetRequiredBufferAlignment() >= readahead_size) { + // Short-circuit and return the original file if readahead_size is + // too small and hence doesn't make sense to be used for prefetching. + return std::move(file); + } + std::unique_ptr result( + new ReadaheadSequentialFile(std::move(file), readahead_size)); + return result; +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/file/sequence_file_reader.h b/librocksdb-sys/rocksdb/file/sequence_file_reader.h new file mode 100644 index 0000000..14350e8 --- /dev/null +++ b/librocksdb-sys/rocksdb/file/sequence_file_reader.h @@ -0,0 +1,119 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include +#include + +#include "env/file_system_tracer.h" +#include "port/port.h" +#include "rocksdb/env.h" +#include "rocksdb/file_system.h" + +namespace ROCKSDB_NAMESPACE { + +// SequentialFileReader is a wrapper on top of Env::SequentialFile. It handles +// Buffered (i.e when page cache is enabled) and Direct (with O_DIRECT / page +// cache disabled) reads appropriately, and also updates the IO stats. +class SequentialFileReader { + private: + void NotifyOnFileReadFinish( + uint64_t offset, size_t length, + const FileOperationInfo::StartTimePoint& start_ts, + const FileOperationInfo::FinishTimePoint& finish_ts, + const Status& status) const { + FileOperationInfo info(FileOperationType::kRead, file_name_, start_ts, + finish_ts, status); + info.offset = offset; + info.length = length; + + for (auto& listener : listeners_) { + listener->OnFileReadFinish(info); + } + info.status.PermitUncheckedError(); + } + + void AddFileIOListeners( + const std::vector>& listeners) { + std::for_each(listeners.begin(), listeners.end(), + [this](const std::shared_ptr& e) { + if (e->ShouldBeNotifiedOnFileIO()) { + listeners_.emplace_back(e); + } + }); + } + + bool ShouldNotifyListeners() const { return !listeners_.empty(); } + + std::string file_name_; + FSSequentialFilePtr file_; + std::atomic offset_{0}; // read offset + std::vector> listeners_{}; + RateLimiter* rate_limiter_; + + public: + explicit SequentialFileReader( + std::unique_ptr&& _file, const std::string& _file_name, + const std::shared_ptr& io_tracer = nullptr, + const std::vector>& listeners = {}, + RateLimiter* rate_limiter = + nullptr) // TODO: migrate call sites to provide rate limiter + : file_name_(_file_name), + file_(std::move(_file), io_tracer, _file_name), + listeners_(), + rate_limiter_(rate_limiter) { + AddFileIOListeners(listeners); + } + + explicit SequentialFileReader( + std::unique_ptr&& _file, const std::string& _file_name, + size_t _readahead_size, + const std::shared_ptr& io_tracer = nullptr, + const std::vector>& listeners = {}, + RateLimiter* rate_limiter = + nullptr) // TODO: migrate call sites to provide rate limiter + : file_name_(_file_name), + file_(NewReadaheadSequentialFile(std::move(_file), _readahead_size), + io_tracer, _file_name), + listeners_(), + rate_limiter_(rate_limiter) { + AddFileIOListeners(listeners); + } + static IOStatus Create(const std::shared_ptr& fs, + const std::string& fname, const FileOptions& file_opts, + std::unique_ptr* reader, + IODebugContext* dbg, RateLimiter* rate_limiter); + + SequentialFileReader(const SequentialFileReader&) = delete; + SequentialFileReader& operator=(const SequentialFileReader&) = delete; + + // `rate_limiter_priority` is used to charge the internal rate limiter when + // enabled. The special value `Env::IO_TOTAL` makes this operation bypass the + // rate limiter. The amount charged to the internal rate limiter is n, even + // when less than n bytes are actually read (e.g. at end of file). To avoid + // overcharging the rate limiter, the caller can use file size to cap n to + // read until end of file. + IOStatus Read(size_t n, Slice* result, char* scratch, + Env::IOPriority rate_limiter_priority); + + IOStatus Skip(uint64_t n); + + FSSequentialFile* file() { return file_.get(); } + + std::string file_name() { return file_name_; } + + bool use_direct_io() const { return file_->use_direct_io(); } + + private: + // NewReadaheadSequentialFile provides a wrapper over SequentialFile to + // always prefetch additional data with every read. + static std::unique_ptr NewReadaheadSequentialFile( + std::unique_ptr&& file, size_t readahead_size); +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/file/sst_file_manager_impl.cc b/librocksdb-sys/rocksdb/file/sst_file_manager_impl.cc new file mode 100644 index 0000000..459ea36 --- /dev/null +++ b/librocksdb-sys/rocksdb/file/sst_file_manager_impl.cc @@ -0,0 +1,507 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "file/sst_file_manager_impl.h" + +#include +#include + +#include "db/db_impl/db_impl.h" +#include "logging/logging.h" +#include "port/port.h" +#include "rocksdb/env.h" +#include "rocksdb/sst_file_manager.h" +#include "test_util/sync_point.h" +#include "util/mutexlock.h" + +namespace ROCKSDB_NAMESPACE { + +SstFileManagerImpl::SstFileManagerImpl( + const std::shared_ptr& clock, + const std::shared_ptr& fs, + const std::shared_ptr& logger, int64_t rate_bytes_per_sec, + double max_trash_db_ratio, uint64_t bytes_max_delete_chunk) + : clock_(clock), + fs_(fs), + logger_(logger), + total_files_size_(0), + compaction_buffer_size_(0), + cur_compactions_reserved_size_(0), + max_allowed_space_(0), + delete_scheduler_(clock_.get(), fs_.get(), rate_bytes_per_sec, + logger.get(), this, max_trash_db_ratio, + bytes_max_delete_chunk), + cv_(&mu_), + closing_(false), + bg_thread_(nullptr), + reserved_disk_buffer_(0), + free_space_trigger_(0), + cur_instance_(nullptr) {} + +SstFileManagerImpl::~SstFileManagerImpl() { + Close(); + bg_err_.PermitUncheckedError(); +} + +void SstFileManagerImpl::Close() { + { + MutexLock l(&mu_); + if (closing_) { + return; + } + closing_ = true; + cv_.SignalAll(); + } + if (bg_thread_) { + bg_thread_->join(); + } +} + +Status SstFileManagerImpl::OnAddFile(const std::string& file_path) { + uint64_t file_size; + Status s = fs_->GetFileSize(file_path, IOOptions(), &file_size, nullptr); + if (s.ok()) { + MutexLock l(&mu_); + OnAddFileImpl(file_path, file_size); + } + TEST_SYNC_POINT_CALLBACK("SstFileManagerImpl::OnAddFile", + const_cast(&file_path)); + return s; +} + +Status SstFileManagerImpl::OnAddFile(const std::string& file_path, + uint64_t file_size) { + MutexLock l(&mu_); + OnAddFileImpl(file_path, file_size); + TEST_SYNC_POINT_CALLBACK("SstFileManagerImpl::OnAddFile", + const_cast(&file_path)); + return Status::OK(); +} + +Status SstFileManagerImpl::OnDeleteFile(const std::string& file_path) { + { + MutexLock l(&mu_); + OnDeleteFileImpl(file_path); + } + TEST_SYNC_POINT_CALLBACK("SstFileManagerImpl::OnDeleteFile", + const_cast(&file_path)); + return Status::OK(); +} + +void SstFileManagerImpl::OnCompactionCompletion(Compaction* c) { + MutexLock l(&mu_); + uint64_t size_added_by_compaction = 0; + for (size_t i = 0; i < c->num_input_levels(); i++) { + for (size_t j = 0; j < c->num_input_files(i); j++) { + FileMetaData* filemeta = c->input(i, j); + size_added_by_compaction += filemeta->fd.GetFileSize(); + } + } + cur_compactions_reserved_size_ -= size_added_by_compaction; +} + +Status SstFileManagerImpl::OnMoveFile(const std::string& old_path, + const std::string& new_path, + uint64_t* file_size) { + { + MutexLock l(&mu_); + if (file_size != nullptr) { + *file_size = tracked_files_[old_path]; + } + OnAddFileImpl(new_path, tracked_files_[old_path]); + OnDeleteFileImpl(old_path); + } + TEST_SYNC_POINT("SstFileManagerImpl::OnMoveFile"); + return Status::OK(); +} + +void SstFileManagerImpl::SetMaxAllowedSpaceUsage(uint64_t max_allowed_space) { + MutexLock l(&mu_); + max_allowed_space_ = max_allowed_space; +} + +void SstFileManagerImpl::SetCompactionBufferSize( + uint64_t compaction_buffer_size) { + MutexLock l(&mu_); + compaction_buffer_size_ = compaction_buffer_size; +} + +bool SstFileManagerImpl::IsMaxAllowedSpaceReached() { + MutexLock l(&mu_); + if (max_allowed_space_ <= 0) { + return false; + } + return total_files_size_ >= max_allowed_space_; +} + +bool SstFileManagerImpl::IsMaxAllowedSpaceReachedIncludingCompactions() { + MutexLock l(&mu_); + if (max_allowed_space_ <= 0) { + return false; + } + return total_files_size_ + cur_compactions_reserved_size_ >= + max_allowed_space_; +} + +bool SstFileManagerImpl::EnoughRoomForCompaction( + ColumnFamilyData* cfd, const std::vector& inputs, + const Status& bg_error) { + MutexLock l(&mu_); + uint64_t size_added_by_compaction = 0; + // First check if we even have the space to do the compaction + for (size_t i = 0; i < inputs.size(); i++) { + for (size_t j = 0; j < inputs[i].size(); j++) { + FileMetaData* filemeta = inputs[i][j]; + size_added_by_compaction += filemeta->fd.GetFileSize(); + } + } + + // Update cur_compactions_reserved_size_ so concurrent compaction + // don't max out space + size_t needed_headroom = cur_compactions_reserved_size_ + + size_added_by_compaction + compaction_buffer_size_; + if (max_allowed_space_ != 0 && + (needed_headroom + total_files_size_ > max_allowed_space_)) { + return false; + } + + // Implement more aggressive checks only if this DB instance has already + // seen a NoSpace() error. This is tin order to contain a single potentially + // misbehaving DB instance and prevent it from slowing down compactions of + // other DB instances + if (bg_error.IsNoSpace() && CheckFreeSpace()) { + auto fn = + TableFileName(cfd->ioptions()->cf_paths, inputs[0][0]->fd.GetNumber(), + inputs[0][0]->fd.GetPathId()); + uint64_t free_space = 0; + Status s = fs_->GetFreeSpace(fn, IOOptions(), &free_space, nullptr); + s.PermitUncheckedError(); // TODO: Check the status + // needed_headroom is based on current size reserved by compactions, + // minus any files created by running compactions as they would count + // against the reserved size. If user didn't specify any compaction + // buffer, add reserved_disk_buffer_ that's calculated by default so the + // compaction doesn't end up leaving nothing for logs and flush SSTs + if (compaction_buffer_size_ == 0) { + needed_headroom += reserved_disk_buffer_; + } + if (free_space < needed_headroom + size_added_by_compaction) { + // We hit the condition of not enough disk space + ROCKS_LOG_ERROR(logger_, + "free space [%" PRIu64 + " bytes] is less than " + "needed headroom [%" ROCKSDB_PRIszt " bytes]\n", + free_space, needed_headroom); + return false; + } + } + + cur_compactions_reserved_size_ += size_added_by_compaction; + // Take a snapshot of cur_compactions_reserved_size_ for when we encounter + // a NoSpace error. + free_space_trigger_ = cur_compactions_reserved_size_; + return true; +} + +uint64_t SstFileManagerImpl::GetCompactionsReservedSize() { + MutexLock l(&mu_); + return cur_compactions_reserved_size_; +} + +uint64_t SstFileManagerImpl::GetTotalSize() { + MutexLock l(&mu_); + return total_files_size_; +} + +std::unordered_map +SstFileManagerImpl::GetTrackedFiles() { + MutexLock l(&mu_); + return tracked_files_; +} + +int64_t SstFileManagerImpl::GetDeleteRateBytesPerSecond() { + return delete_scheduler_.GetRateBytesPerSecond(); +} + +void SstFileManagerImpl::SetDeleteRateBytesPerSecond(int64_t delete_rate) { + return delete_scheduler_.SetRateBytesPerSecond(delete_rate); +} + +double SstFileManagerImpl::GetMaxTrashDBRatio() { + return delete_scheduler_.GetMaxTrashDBRatio(); +} + +void SstFileManagerImpl::SetMaxTrashDBRatio(double r) { + return delete_scheduler_.SetMaxTrashDBRatio(r); +} + +uint64_t SstFileManagerImpl::GetTotalTrashSize() { + return delete_scheduler_.GetTotalTrashSize(); +} + +void SstFileManagerImpl::ReserveDiskBuffer(uint64_t size, + const std::string& path) { + MutexLock l(&mu_); + + reserved_disk_buffer_ += size; + if (path_.empty()) { + path_ = path; + } +} + +void SstFileManagerImpl::ClearError() { + while (true) { + MutexLock l(&mu_); + + if (error_handler_list_.empty() || closing_) { + return; + } + + uint64_t free_space = 0; + Status s = fs_->GetFreeSpace(path_, IOOptions(), &free_space, nullptr); + free_space = max_allowed_space_ > 0 + ? std::min(max_allowed_space_, free_space) + : free_space; + if (s.ok()) { + // In case of multi-DB instances, some of them may have experienced a + // soft error and some a hard error. In the SstFileManagerImpl, a hard + // error will basically override previously reported soft errors. Once + // we clear the hard error, we don't keep track of previous errors for + // now + if (bg_err_.severity() == Status::Severity::kHardError) { + if (free_space < reserved_disk_buffer_) { + ROCKS_LOG_ERROR(logger_, + "free space [%" PRIu64 + " bytes] is less than " + "required disk buffer [%" PRIu64 " bytes]\n", + free_space, reserved_disk_buffer_); + ROCKS_LOG_ERROR(logger_, "Cannot clear hard error\n"); + s = Status::NoSpace(); + } + } else if (bg_err_.severity() == Status::Severity::kSoftError) { + if (free_space < free_space_trigger_) { + ROCKS_LOG_WARN(logger_, + "free space [%" PRIu64 + " bytes] is less than " + "free space for compaction trigger [%" PRIu64 + " bytes]\n", + free_space, free_space_trigger_); + ROCKS_LOG_WARN(logger_, "Cannot clear soft error\n"); + s = Status::NoSpace(); + } + } + } + + // Someone could have called CancelErrorRecovery() and the list could have + // become empty, so check again here + if (s.ok()) { + assert(!error_handler_list_.empty()); + auto error_handler = error_handler_list_.front(); + // Since we will release the mutex, set cur_instance_ to signal to the + // shutdown thread, if it calls // CancelErrorRecovery() the meantime, + // to indicate that this DB instance is busy. The DB instance is + // guaranteed to not be deleted before RecoverFromBGError() returns, + // since the ErrorHandler::recovery_in_prog_ flag would be true + cur_instance_ = error_handler; + mu_.Unlock(); + s = error_handler->RecoverFromBGError(); + TEST_SYNC_POINT("SstFileManagerImpl::ErrorCleared"); + mu_.Lock(); + // The DB instance might have been deleted while we were + // waiting for the mutex, so check cur_instance_ to make sure its + // still non-null + if (cur_instance_) { + // Check for error again, since the instance may have recovered but + // immediately got another error. If that's the case, and the new + // error is also a NoSpace() non-fatal error, leave the instance in + // the list + Status err = cur_instance_->GetBGError(); + if (s.ok() && err.subcode() == IOStatus::SubCode::kNoSpace && + err.severity() < Status::Severity::kFatalError) { + s = err; + } + cur_instance_ = nullptr; + } + + if (s.ok() || s.IsShutdownInProgress() || + (!s.ok() && s.severity() >= Status::Severity::kFatalError)) { + // If shutdown is in progress, abandon this handler instance + // and continue with the others + error_handler_list_.pop_front(); + } + } + + if (!error_handler_list_.empty()) { + // If there are more instances to be recovered, reschedule after 5 + // seconds + int64_t wait_until = clock_->NowMicros() + 5000000; + cv_.TimedWait(wait_until); + } + + // Check again for error_handler_list_ empty, as a DB instance shutdown + // could have removed it from the queue while we were in timed wait + if (error_handler_list_.empty()) { + ROCKS_LOG_INFO(logger_, "Clearing error\n"); + bg_err_ = Status::OK(); + return; + } + } +} + +void SstFileManagerImpl::StartErrorRecovery(ErrorHandler* handler, + Status bg_error) { + MutexLock l(&mu_); + if (bg_error.severity() == Status::Severity::kSoftError) { + if (bg_err_.ok()) { + // Setting bg_err_ basically means we're in degraded mode + // Assume that all pending compactions will fail similarly. The trigger + // for clearing this condition is set to current compaction reserved + // size, so we stop checking disk space available in + // EnoughRoomForCompaction once this much free space is available + bg_err_ = bg_error; + } + } else if (bg_error.severity() == Status::Severity::kHardError) { + bg_err_ = bg_error; + } else { + assert(false); + } + + // If this is the first instance of this error, kick of a thread to poll + // and recover from this condition + if (error_handler_list_.empty()) { + error_handler_list_.push_back(handler); + // Release lock before calling join. Its ok to do so because + // error_handler_list_ is now non-empty, so no other invocation of this + // function will execute this piece of code + mu_.Unlock(); + if (bg_thread_) { + bg_thread_->join(); + } + // Start a new thread. The previous one would have exited. + bg_thread_.reset(new port::Thread(&SstFileManagerImpl::ClearError, this)); + mu_.Lock(); + } else { + // Check if this DB instance is already in the list + for (auto iter = error_handler_list_.begin(); + iter != error_handler_list_.end(); ++iter) { + if ((*iter) == handler) { + return; + } + } + error_handler_list_.push_back(handler); + } +} + +bool SstFileManagerImpl::CancelErrorRecovery(ErrorHandler* handler) { + MutexLock l(&mu_); + + if (cur_instance_ == handler) { + // This instance is currently busy attempting to recover + // Nullify it so the recovery thread doesn't attempt to access it again + cur_instance_ = nullptr; + return false; + } + + for (auto iter = error_handler_list_.begin(); + iter != error_handler_list_.end(); ++iter) { + if ((*iter) == handler) { + error_handler_list_.erase(iter); + return true; + } + } + return false; +} + +Status SstFileManagerImpl::ScheduleFileDeletion(const std::string& file_path, + const std::string& path_to_sync, + const bool force_bg) { + TEST_SYNC_POINT_CALLBACK("SstFileManagerImpl::ScheduleFileDeletion", + const_cast(&file_path)); + return delete_scheduler_.DeleteFile(file_path, path_to_sync, force_bg); +} + +void SstFileManagerImpl::WaitForEmptyTrash() { + delete_scheduler_.WaitForEmptyTrash(); +} + +void SstFileManagerImpl::OnAddFileImpl(const std::string& file_path, + uint64_t file_size) { + auto tracked_file = tracked_files_.find(file_path); + if (tracked_file != tracked_files_.end()) { + // File was added before, we will just update the size + total_files_size_ -= tracked_file->second; + total_files_size_ += file_size; + cur_compactions_reserved_size_ -= file_size; + } else { + total_files_size_ += file_size; + } + tracked_files_[file_path] = file_size; +} + +void SstFileManagerImpl::OnDeleteFileImpl(const std::string& file_path) { + auto tracked_file = tracked_files_.find(file_path); + if (tracked_file == tracked_files_.end()) { + // File is not tracked + return; + } + + total_files_size_ -= tracked_file->second; + tracked_files_.erase(tracked_file); +} + +SstFileManager* NewSstFileManager(Env* env, std::shared_ptr info_log, + std::string trash_dir, + int64_t rate_bytes_per_sec, + bool delete_existing_trash, Status* status, + double max_trash_db_ratio, + uint64_t bytes_max_delete_chunk) { + const auto& fs = env->GetFileSystem(); + return NewSstFileManager(env, fs, info_log, trash_dir, rate_bytes_per_sec, + delete_existing_trash, status, max_trash_db_ratio, + bytes_max_delete_chunk); +} + +SstFileManager* NewSstFileManager(Env* env, std::shared_ptr fs, + std::shared_ptr info_log, + const std::string& trash_dir, + int64_t rate_bytes_per_sec, + bool delete_existing_trash, Status* status, + double max_trash_db_ratio, + uint64_t bytes_max_delete_chunk) { + const auto& clock = env->GetSystemClock(); + SstFileManagerImpl* res = + new SstFileManagerImpl(clock, fs, info_log, rate_bytes_per_sec, + max_trash_db_ratio, bytes_max_delete_chunk); + + // trash_dir is deprecated and not needed anymore, but if user passed it + // we will still remove files in it. + Status s = Status::OK(); + if (delete_existing_trash && trash_dir != "") { + std::vector files_in_trash; + s = fs->GetChildren(trash_dir, IOOptions(), &files_in_trash, nullptr); + if (s.ok()) { + for (const std::string& trash_file : files_in_trash) { + std::string path_in_trash = trash_dir + "/" + trash_file; + res->OnAddFile(path_in_trash); + Status file_delete = + res->ScheduleFileDeletion(path_in_trash, trash_dir); + if (s.ok() && !file_delete.ok()) { + s = file_delete; + } + } + } + } + + if (status) { + *status = s; + } else { + // No one passed us a Status, so they must not care about the error... + s.PermitUncheckedError(); + } + + return res; +} + + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/file/sst_file_manager_impl.h b/librocksdb-sys/rocksdb/file/sst_file_manager_impl.h new file mode 100644 index 0000000..e85376e --- /dev/null +++ b/librocksdb-sys/rocksdb/file/sst_file_manager_impl.h @@ -0,0 +1,193 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + + +#include + +#include "db/compaction/compaction.h" +#include "file/delete_scheduler.h" +#include "port/port.h" +#include "rocksdb/sst_file_manager.h" + +namespace ROCKSDB_NAMESPACE { +class ErrorHandler; +class FileSystem; +class SystemClock; +class Logger; + +// SstFileManager is used to track SST and blob files in the DB and control +// their deletion rate. All SstFileManager public functions are thread-safe. +class SstFileManagerImpl : public SstFileManager { + public: + explicit SstFileManagerImpl(const std::shared_ptr& clock, + const std::shared_ptr& fs, + const std::shared_ptr& logger, + int64_t rate_bytes_per_sec, + double max_trash_db_ratio, + uint64_t bytes_max_delete_chunk); + + ~SstFileManagerImpl(); + + // DB will call OnAddFile whenever a new sst/blob file is added. + Status OnAddFile(const std::string& file_path); + + // Overload where size of the file is provided by the caller rather than + // queried from the filesystem. This is an optimization. + Status OnAddFile(const std::string& file_path, uint64_t file_size); + + // DB will call OnDeleteFile whenever a sst/blob file is deleted. + Status OnDeleteFile(const std::string& file_path); + + // DB will call OnMoveFile whenever a sst/blob file is move to a new path. + Status OnMoveFile(const std::string& old_path, const std::string& new_path, + uint64_t* file_size = nullptr); + + // Update the maximum allowed space that should be used by RocksDB, if + // the total size of the SST and blob files exceeds max_allowed_space, writes + // to RocksDB will fail. + // + // Setting max_allowed_space to 0 will disable this feature, maximum allowed + // space will be infinite (Default value). + // + // thread-safe. + void SetMaxAllowedSpaceUsage(uint64_t max_allowed_space) override; + + void SetCompactionBufferSize(uint64_t compaction_buffer_size) override; + + // Return true if the total size of SST and blob files exceeded the maximum + // allowed space usage. + // + // thread-safe. + bool IsMaxAllowedSpaceReached() override; + + bool IsMaxAllowedSpaceReachedIncludingCompactions() override; + + // Returns true is there is enough (approximate) space for the specified + // compaction. Space is approximate because this function conservatively + // estimates how much space is currently being used by compactions (i.e. + // if a compaction has started, this function bumps the used space by + // the full compaction size). + bool EnoughRoomForCompaction(ColumnFamilyData* cfd, + const std::vector& inputs, + const Status& bg_error); + + // Bookkeeping so total_file_sizes_ goes back to normal after compaction + // finishes + void OnCompactionCompletion(Compaction* c); + + uint64_t GetCompactionsReservedSize(); + + // Return the total size of all tracked files. + uint64_t GetTotalSize() override; + + // Return a map containing all tracked files and there corresponding sizes. + std::unordered_map GetTrackedFiles() override; + + // Return delete rate limit in bytes per second. + virtual int64_t GetDeleteRateBytesPerSecond() override; + + // Update the delete rate limit in bytes per second. + virtual void SetDeleteRateBytesPerSecond(int64_t delete_rate) override; + + // Return trash/DB size ratio where new files will be deleted immediately + virtual double GetMaxTrashDBRatio() override; + + // Update trash/DB size ratio where new files will be deleted immediately + virtual void SetMaxTrashDBRatio(double ratio) override; + + // Return the total size of trash files + uint64_t GetTotalTrashSize() override; + + // Called by each DB instance using this sst file manager to reserve + // disk buffer space for recovery from out of space errors + void ReserveDiskBuffer(uint64_t buffer, const std::string& path); + + // Set a flag upon encountering disk full. May enqueue the ErrorHandler + // instance for background polling and recovery + void StartErrorRecovery(ErrorHandler* db, Status bg_error); + + // Remove the given Errorhandler instance from the recovery queue. Its + // not guaranteed + bool CancelErrorRecovery(ErrorHandler* db); + + // Mark file as trash and schedule it's deletion. If force_bg is set, it + // forces the file to be deleting in the background regardless of DB size, + // except when rate limited delete is disabled + virtual Status ScheduleFileDeletion(const std::string& file_path, + const std::string& dir_to_sync, + const bool force_bg = false); + + // Wait for all files being deleteing in the background to finish or for + // destructor to be called. + virtual void WaitForEmptyTrash(); + + DeleteScheduler* delete_scheduler() { return &delete_scheduler_; } + + // Stop the error recovery background thread. This should be called only + // once in the object's lifetime, and before the destructor + void Close(); + + void SetStatisticsPtr(const std::shared_ptr& stats) override { + stats_ = stats; + delete_scheduler_.SetStatisticsPtr(stats); + } + + private: + // REQUIRES: mutex locked + void OnAddFileImpl(const std::string& file_path, uint64_t file_size); + // REQUIRES: mutex locked + void OnDeleteFileImpl(const std::string& file_path); + + void ClearError(); + bool CheckFreeSpace() { + return bg_err_.severity() == Status::Severity::kSoftError; + } + + std::shared_ptr clock_; + std::shared_ptr fs_; + std::shared_ptr logger_; + // Mutex to protect tracked_files_, total_files_size_ + port::Mutex mu_; + // The summation of the sizes of all files in tracked_files_ map + uint64_t total_files_size_; + // Compactions should only execute if they can leave at least + // this amount of buffer space for logs and flushes + uint64_t compaction_buffer_size_; + // Estimated size of the current ongoing compactions + uint64_t cur_compactions_reserved_size_; + // A map containing all tracked files and there sizes + // file_path => file_size + std::unordered_map tracked_files_; + // The maximum allowed space (in bytes) for sst and blob files. + uint64_t max_allowed_space_; + // DeleteScheduler used to throttle file deletion. + DeleteScheduler delete_scheduler_; + port::CondVar cv_; + // Flag to force error recovery thread to exit + bool closing_; + // Background error recovery thread + std::unique_ptr bg_thread_; + // A path in the filesystem corresponding to this SFM. This is used for + // calling Env::GetFreeSpace. Posix requires a path in the filesystem + std::string path_; + // Save the current background error + Status bg_err_; + // Amount of free disk headroom before allowing recovery from hard errors + uint64_t reserved_disk_buffer_; + // For soft errors, amount of free disk space before we can allow + // compactions to run full throttle. If disk space is below this trigger, + // compactions will be gated by free disk space > input size + uint64_t free_space_trigger_; + // List of database error handler instances tracked by this SstFileManager. + std::list error_handler_list_; + // Pointer to ErrorHandler instance that is currently processing recovery + ErrorHandler* cur_instance_; + std::shared_ptr stats_; +}; + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/file/writable_file_writer.cc b/librocksdb-sys/rocksdb/file/writable_file_writer.cc new file mode 100644 index 0000000..908878a --- /dev/null +++ b/librocksdb-sys/rocksdb/file/writable_file_writer.cc @@ -0,0 +1,989 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "file/writable_file_writer.h" + +#include +#include + +#include "db/version_edit.h" +#include "monitoring/histogram.h" +#include "monitoring/iostats_context_imp.h" +#include "port/port.h" +#include "rocksdb/io_status.h" +#include "rocksdb/system_clock.h" +#include "test_util/sync_point.h" +#include "util/crc32c.h" +#include "util/random.h" +#include "util/rate_limiter_impl.h" + +namespace ROCKSDB_NAMESPACE { +IOStatus WritableFileWriter::Create(const std::shared_ptr& fs, + const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* writer, + IODebugContext* dbg) { + if (file_opts.use_direct_writes && + 0 == file_opts.writable_file_max_buffer_size) { + return IOStatus::InvalidArgument( + "Direct write requires writable_file_max_buffer_size > 0"); + } + std::unique_ptr file; + IOStatus io_s = fs->NewWritableFile(fname, file_opts, &file, dbg); + if (io_s.ok()) { + writer->reset(new WritableFileWriter(std::move(file), fname, file_opts)); + } + return io_s; +} + +IOStatus WritableFileWriter::Append(const Slice& data, uint32_t crc32c_checksum, + Env::IOPriority op_rate_limiter_priority) { + if (seen_error()) { + return AssertFalseAndGetStatusForPrevError(); + } + + const char* src = data.data(); + size_t left = data.size(); + IOStatus s; + pending_sync_ = true; + + TEST_KILL_RANDOM_WITH_WEIGHT("WritableFileWriter::Append:0", REDUCE_ODDS2); + + // Calculate the checksum of appended data + UpdateFileChecksum(data); + + { + IOOptions io_options; + io_options.rate_limiter_priority = + WritableFileWriter::DecideRateLimiterPriority( + writable_file_->GetIOPriority(), op_rate_limiter_priority); + IOSTATS_TIMER_GUARD(prepare_write_nanos); + TEST_SYNC_POINT("WritableFileWriter::Append:BeforePrepareWrite"); + writable_file_->PrepareWrite(static_cast(GetFileSize()), left, + io_options, nullptr); + } + + // See whether we need to enlarge the buffer to avoid the flush + if (buf_.Capacity() - buf_.CurrentSize() < left) { + for (size_t cap = buf_.Capacity(); + cap < max_buffer_size_; // There is still room to increase + cap *= 2) { + // See whether the next available size is large enough. + // Buffer will never be increased to more than max_buffer_size_. + size_t desired_capacity = std::min(cap * 2, max_buffer_size_); + if (desired_capacity - buf_.CurrentSize() >= left || + (use_direct_io() && desired_capacity == max_buffer_size_)) { + buf_.AllocateNewBuffer(desired_capacity, true); + break; + } + } + } + + // Flush only when buffered I/O + if (!use_direct_io() && (buf_.Capacity() - buf_.CurrentSize()) < left) { + if (buf_.CurrentSize() > 0) { + s = Flush(op_rate_limiter_priority); + if (!s.ok()) { + set_seen_error(); + return s; + } + } + assert(buf_.CurrentSize() == 0); + } + + if (perform_data_verification_ && buffered_data_with_checksum_ && + crc32c_checksum != 0) { + // Since we want to use the checksum of the input data, we cannot break it + // into several pieces. We will only write them in the buffer when buffer + // size is enough. Otherwise, we will directly write it down. + if (use_direct_io() || (buf_.Capacity() - buf_.CurrentSize()) >= left) { + if ((buf_.Capacity() - buf_.CurrentSize()) >= left) { + size_t appended = buf_.Append(src, left); + if (appended != left) { + s = IOStatus::Corruption("Write buffer append failure"); + } + buffered_data_crc32c_checksum_ = crc32c::Crc32cCombine( + buffered_data_crc32c_checksum_, crc32c_checksum, appended); + } else { + while (left > 0) { + size_t appended = buf_.Append(src, left); + buffered_data_crc32c_checksum_ = + crc32c::Extend(buffered_data_crc32c_checksum_, src, appended); + left -= appended; + src += appended; + + if (left > 0) { + s = Flush(op_rate_limiter_priority); + if (!s.ok()) { + break; + } + } + } + } + } else { + assert(buf_.CurrentSize() == 0); + buffered_data_crc32c_checksum_ = crc32c_checksum; + s = WriteBufferedWithChecksum(src, left, op_rate_limiter_priority); + } + } else { + // In this case, either we do not need to do the data verification or + // caller does not provide the checksum of the data (crc32c_checksum = 0). + // + // We never write directly to disk with direct I/O on. + // or we simply use it for its original purpose to accumulate many small + // chunks + if (use_direct_io() || (buf_.Capacity() >= left)) { + while (left > 0) { + size_t appended = buf_.Append(src, left); + if (perform_data_verification_ && buffered_data_with_checksum_) { + buffered_data_crc32c_checksum_ = + crc32c::Extend(buffered_data_crc32c_checksum_, src, appended); + } + left -= appended; + src += appended; + + if (left > 0) { + s = Flush(op_rate_limiter_priority); + if (!s.ok()) { + break; + } + } + } + } else { + // Writing directly to file bypassing the buffer + assert(buf_.CurrentSize() == 0); + if (perform_data_verification_ && buffered_data_with_checksum_) { + buffered_data_crc32c_checksum_ = crc32c::Value(src, left); + s = WriteBufferedWithChecksum(src, left, op_rate_limiter_priority); + } else { + s = WriteBuffered(src, left, op_rate_limiter_priority); + } + } + } + + TEST_KILL_RANDOM("WritableFileWriter::Append:1"); + if (s.ok()) { + uint64_t cur_size = filesize_.load(std::memory_order_acquire); + filesize_.store(cur_size + data.size(), std::memory_order_release); + } else { + set_seen_error(); + } + return s; +} + +IOStatus WritableFileWriter::Pad(const size_t pad_bytes, + Env::IOPriority op_rate_limiter_priority) { + if (seen_error()) { + return AssertFalseAndGetStatusForPrevError(); + } + assert(pad_bytes < kDefaultPageSize); + size_t left = pad_bytes; + size_t cap = buf_.Capacity() - buf_.CurrentSize(); + size_t pad_start = buf_.CurrentSize(); + + // Assume pad_bytes is small compared to buf_ capacity. So we always + // use buf_ rather than write directly to file in certain cases like + // Append() does. + while (left) { + size_t append_bytes = std::min(cap, left); + buf_.PadWith(append_bytes, 0); + left -= append_bytes; + if (left > 0) { + IOStatus s = Flush(op_rate_limiter_priority); + if (!s.ok()) { + set_seen_error(); + return s; + } + } + cap = buf_.Capacity() - buf_.CurrentSize(); + } + pending_sync_ = true; + uint64_t cur_size = filesize_.load(std::memory_order_acquire); + filesize_.store(cur_size + pad_bytes, std::memory_order_release); + if (perform_data_verification_) { + buffered_data_crc32c_checksum_ = + crc32c::Extend(buffered_data_crc32c_checksum_, + buf_.BufferStart() + pad_start, pad_bytes); + } + return IOStatus::OK(); +} + +IOStatus WritableFileWriter::Close() { + if (seen_error()) { + IOStatus interim; + if (writable_file_.get() != nullptr) { + interim = writable_file_->Close(IOOptions(), nullptr); + writable_file_.reset(); + } + if (interim.ok()) { + return IOStatus::IOError( + "File is closed but data not flushed as writer has previous error."); + } else { + return interim; + } + } + + // Do not quit immediately on failure the file MUST be closed + + // Possible to close it twice now as we MUST close + // in __dtor, simply flushing is not enough + // Windows when pre-allocating does not fill with zeros + // also with unbuffered access we also set the end of data. + if (writable_file_.get() == nullptr) { + return IOStatus::OK(); + } + + IOStatus s; + s = Flush(); // flush cache to OS + + IOStatus interim; + IOOptions io_options; + io_options.rate_limiter_priority = writable_file_->GetIOPriority(); + // In direct I/O mode we write whole pages so + // we need to let the file know where data ends. + if (use_direct_io()) { + { + FileOperationInfo::StartTimePoint start_ts; + if (ShouldNotifyListeners()) { + start_ts = FileOperationInfo::StartNow(); + } + uint64_t filesz = filesize_.load(std::memory_order_acquire); + interim = writable_file_->Truncate(filesz, io_options, nullptr); + if (ShouldNotifyListeners()) { + auto finish_ts = FileOperationInfo::FinishNow(); + NotifyOnFileTruncateFinish(start_ts, finish_ts, s); + if (!interim.ok()) { + NotifyOnIOError(interim, FileOperationType::kTruncate, file_name(), + filesz); + } + } + } + if (interim.ok()) { + { + FileOperationInfo::StartTimePoint start_ts; + if (ShouldNotifyListeners()) { + start_ts = FileOperationInfo::StartNow(); + } + interim = writable_file_->Fsync(io_options, nullptr); + if (ShouldNotifyListeners()) { + auto finish_ts = FileOperationInfo::FinishNow(); + NotifyOnFileSyncFinish(start_ts, finish_ts, s, + FileOperationType::kFsync); + if (!interim.ok()) { + NotifyOnIOError(interim, FileOperationType::kFsync, file_name()); + } + } + } + } + if (!interim.ok() && s.ok()) { + s = interim; + } + } + + TEST_KILL_RANDOM("WritableFileWriter::Close:0"); + { + FileOperationInfo::StartTimePoint start_ts; + if (ShouldNotifyListeners()) { + start_ts = FileOperationInfo::StartNow(); + } + interim = writable_file_->Close(io_options, nullptr); + if (ShouldNotifyListeners()) { + auto finish_ts = FileOperationInfo::FinishNow(); + NotifyOnFileCloseFinish(start_ts, finish_ts, s); + if (!interim.ok()) { + NotifyOnIOError(interim, FileOperationType::kClose, file_name()); + } + } + } + if (!interim.ok() && s.ok()) { + s = interim; + } + + writable_file_.reset(); + TEST_KILL_RANDOM("WritableFileWriter::Close:1"); + + if (s.ok()) { + if (checksum_generator_ != nullptr && !checksum_finalized_) { + checksum_generator_->Finalize(); + checksum_finalized_ = true; + } + } else { + set_seen_error(); + } + + return s; +} + +// write out the cached data to the OS cache or storage if direct I/O +// enabled +IOStatus WritableFileWriter::Flush(Env::IOPriority op_rate_limiter_priority) { + if (seen_error()) { + return AssertFalseAndGetStatusForPrevError(); + } + + IOStatus s; + TEST_KILL_RANDOM_WITH_WEIGHT("WritableFileWriter::Flush:0", REDUCE_ODDS2); + + if (buf_.CurrentSize() > 0) { + if (use_direct_io()) { + if (pending_sync_) { + if (perform_data_verification_ && buffered_data_with_checksum_) { + s = WriteDirectWithChecksum(op_rate_limiter_priority); + } else { + s = WriteDirect(op_rate_limiter_priority); + } + } + } else { + if (perform_data_verification_ && buffered_data_with_checksum_) { + s = WriteBufferedWithChecksum(buf_.BufferStart(), buf_.CurrentSize(), + op_rate_limiter_priority); + } else { + s = WriteBuffered(buf_.BufferStart(), buf_.CurrentSize(), + op_rate_limiter_priority); + } + } + if (!s.ok()) { + set_seen_error(); + return s; + } + } + + { + FileOperationInfo::StartTimePoint start_ts; + if (ShouldNotifyListeners()) { + start_ts = FileOperationInfo::StartNow(); + } + IOOptions io_options; + io_options.rate_limiter_priority = + WritableFileWriter::DecideRateLimiterPriority( + writable_file_->GetIOPriority(), op_rate_limiter_priority); + s = writable_file_->Flush(io_options, nullptr); + if (ShouldNotifyListeners()) { + auto finish_ts = std::chrono::steady_clock::now(); + NotifyOnFileFlushFinish(start_ts, finish_ts, s); + if (!s.ok()) { + NotifyOnIOError(s, FileOperationType::kFlush, file_name()); + } + } + } + + if (!s.ok()) { + set_seen_error(); + return s; + } + + // sync OS cache to disk for every bytes_per_sync_ + // TODO: give log file and sst file different options (log + // files could be potentially cached in OS for their whole + // life time, thus we might not want to flush at all). + + // We try to avoid sync to the last 1MB of data. For two reasons: + // (1) avoid rewrite the same page that is modified later. + // (2) for older version of OS, write can block while writing out + // the page. + // Xfs does neighbor page flushing outside of the specified ranges. We + // need to make sure sync range is far from the write offset. + if (!use_direct_io() && bytes_per_sync_) { + const uint64_t kBytesNotSyncRange = + 1024 * 1024; // recent 1MB is not synced. + const uint64_t kBytesAlignWhenSync = 4 * 1024; // Align 4KB. + uint64_t cur_size = filesize_.load(std::memory_order_acquire); + if (cur_size > kBytesNotSyncRange) { + uint64_t offset_sync_to = cur_size - kBytesNotSyncRange; + offset_sync_to -= offset_sync_to % kBytesAlignWhenSync; + assert(offset_sync_to >= last_sync_size_); + if (offset_sync_to > 0 && + offset_sync_to - last_sync_size_ >= bytes_per_sync_) { + s = RangeSync(last_sync_size_, offset_sync_to - last_sync_size_); + if (!s.ok()) { + set_seen_error(); + } + last_sync_size_ = offset_sync_to; + } + } + } + + return s; +} + +std::string WritableFileWriter::GetFileChecksum() { + if (checksum_generator_ != nullptr) { + assert(checksum_finalized_); + return checksum_generator_->GetChecksum(); + } else { + return kUnknownFileChecksum; + } +} + +const char* WritableFileWriter::GetFileChecksumFuncName() const { + if (checksum_generator_ != nullptr) { + return checksum_generator_->Name(); + } else { + return kUnknownFileChecksumFuncName; + } +} + +IOStatus WritableFileWriter::Sync(bool use_fsync) { + if (seen_error()) { + return AssertFalseAndGetStatusForPrevError(); + } + + IOStatus s = Flush(); + if (!s.ok()) { + set_seen_error(); + return s; + } + TEST_KILL_RANDOM("WritableFileWriter::Sync:0"); + if (!use_direct_io() && pending_sync_) { + s = SyncInternal(use_fsync); + if (!s.ok()) { + set_seen_error(); + return s; + } + } + TEST_KILL_RANDOM("WritableFileWriter::Sync:1"); + pending_sync_ = false; + return IOStatus::OK(); +} + +IOStatus WritableFileWriter::SyncWithoutFlush(bool use_fsync) { + if (seen_error()) { + return AssertFalseAndGetStatusForPrevError(); + } + if (!writable_file_->IsSyncThreadSafe()) { + return IOStatus::NotSupported( + "Can't WritableFileWriter::SyncWithoutFlush() because " + "WritableFile::IsSyncThreadSafe() is false"); + } + TEST_SYNC_POINT("WritableFileWriter::SyncWithoutFlush:1"); + IOStatus s = SyncInternal(use_fsync); + TEST_SYNC_POINT("WritableFileWriter::SyncWithoutFlush:2"); + if (!s.ok()) { +#ifndef NDEBUG + sync_without_flush_called_ = true; +#endif // NDEBUG + set_seen_error(); + } + return s; +} + +IOStatus WritableFileWriter::SyncInternal(bool use_fsync) { + // Caller is supposed to check seen_error_ + IOStatus s; + IOSTATS_TIMER_GUARD(fsync_nanos); + TEST_SYNC_POINT("WritableFileWriter::SyncInternal:0"); + auto prev_perf_level = GetPerfLevel(); + + IOSTATS_CPU_TIMER_GUARD(cpu_write_nanos, clock_); + + FileOperationInfo::StartTimePoint start_ts; + if (ShouldNotifyListeners()) { + start_ts = FileOperationInfo::StartNow(); + } + + IOOptions io_options; + io_options.rate_limiter_priority = writable_file_->GetIOPriority(); + if (use_fsync) { + s = writable_file_->Fsync(io_options, nullptr); + } else { + s = writable_file_->Sync(io_options, nullptr); + } + if (ShouldNotifyListeners()) { + auto finish_ts = std::chrono::steady_clock::now(); + NotifyOnFileSyncFinish( + start_ts, finish_ts, s, + use_fsync ? FileOperationType::kFsync : FileOperationType::kSync); + if (!s.ok()) { + NotifyOnIOError( + s, (use_fsync ? FileOperationType::kFsync : FileOperationType::kSync), + file_name()); + } + } + SetPerfLevel(prev_perf_level); + + // The caller will be responsible to call set_seen_error() if s is not OK. + return s; +} + +IOStatus WritableFileWriter::RangeSync(uint64_t offset, uint64_t nbytes) { + if (seen_error()) { + return AssertFalseAndGetStatusForPrevError(); + } + + IOSTATS_TIMER_GUARD(range_sync_nanos); + TEST_SYNC_POINT("WritableFileWriter::RangeSync:0"); + FileOperationInfo::StartTimePoint start_ts; + if (ShouldNotifyListeners()) { + start_ts = FileOperationInfo::StartNow(); + } + IOOptions io_options; + io_options.rate_limiter_priority = writable_file_->GetIOPriority(); + IOStatus s = writable_file_->RangeSync(offset, nbytes, io_options, nullptr); + if (!s.ok()) { + set_seen_error(); + } + if (ShouldNotifyListeners()) { + auto finish_ts = std::chrono::steady_clock::now(); + NotifyOnFileRangeSyncFinish(offset, nbytes, start_ts, finish_ts, s); + if (!s.ok()) { + NotifyOnIOError(s, FileOperationType::kRangeSync, file_name(), nbytes, + offset); + } + } + return s; +} + +// This method writes to disk the specified data and makes use of the rate +// limiter if available +IOStatus WritableFileWriter::WriteBuffered( + const char* data, size_t size, Env::IOPriority op_rate_limiter_priority) { + if (seen_error()) { + return AssertFalseAndGetStatusForPrevError(); + } + + IOStatus s; + assert(!use_direct_io()); + const char* src = data; + size_t left = size; + DataVerificationInfo v_info; + char checksum_buf[sizeof(uint32_t)]; + Env::IOPriority rate_limiter_priority_used = + WritableFileWriter::DecideRateLimiterPriority( + writable_file_->GetIOPriority(), op_rate_limiter_priority); + IOOptions io_options; + io_options.rate_limiter_priority = rate_limiter_priority_used; + + while (left > 0) { + size_t allowed = left; + if (rate_limiter_ != nullptr && + rate_limiter_priority_used != Env::IO_TOTAL) { + allowed = rate_limiter_->RequestToken(left, 0 /* alignment */, + rate_limiter_priority_used, stats_, + RateLimiter::OpType::kWrite); + } + + { + IOSTATS_TIMER_GUARD(write_nanos); + TEST_SYNC_POINT("WritableFileWriter::Flush:BeforeAppend"); + + FileOperationInfo::StartTimePoint start_ts; + uint64_t old_size = writable_file_->GetFileSize(io_options, nullptr); + if (ShouldNotifyListeners()) { + start_ts = FileOperationInfo::StartNow(); + old_size = next_write_offset_; + } + { + auto prev_perf_level = GetPerfLevel(); + + IOSTATS_CPU_TIMER_GUARD(cpu_write_nanos, clock_); + if (perform_data_verification_) { + Crc32cHandoffChecksumCalculation(src, allowed, checksum_buf); + v_info.checksum = Slice(checksum_buf, sizeof(uint32_t)); + s = writable_file_->Append(Slice(src, allowed), io_options, v_info, + nullptr); + } else { + s = writable_file_->Append(Slice(src, allowed), io_options, nullptr); + } + if (!s.ok()) { + // If writable_file_->Append() failed, then the data may or may not + // exist in the underlying memory buffer, OS page cache, remote file + // system's buffer, etc. If WritableFileWriter keeps the data in + // buf_, then a future Close() or write retry may send the data to + // the underlying file again. If the data does exist in the + // underlying buffer and gets written to the file eventually despite + // returning error, the file may end up with two duplicate pieces of + // data. Therefore, clear the buf_ at the WritableFileWriter layer + // and let caller determine error handling. + buf_.Size(0); + buffered_data_crc32c_checksum_ = 0; + } + SetPerfLevel(prev_perf_level); + } + if (ShouldNotifyListeners()) { + auto finish_ts = std::chrono::steady_clock::now(); + NotifyOnFileWriteFinish(old_size, allowed, start_ts, finish_ts, s); + if (!s.ok()) { + NotifyOnIOError(s, FileOperationType::kAppend, file_name(), allowed, + old_size); + } + } + if (!s.ok()) { + set_seen_error(); + return s; + } + } + + IOSTATS_ADD(bytes_written, allowed); + TEST_KILL_RANDOM("WritableFileWriter::WriteBuffered:0"); + + left -= allowed; + src += allowed; + uint64_t cur_size = flushed_size_.load(std::memory_order_acquire); + flushed_size_.store(cur_size + allowed, std::memory_order_release); + } + buf_.Size(0); + buffered_data_crc32c_checksum_ = 0; + if (!s.ok()) { + set_seen_error(); + } + return s; +} + +IOStatus WritableFileWriter::WriteBufferedWithChecksum( + const char* data, size_t size, Env::IOPriority op_rate_limiter_priority) { + if (seen_error()) { + return AssertFalseAndGetStatusForPrevError(); + } + + IOStatus s; + assert(!use_direct_io()); + assert(perform_data_verification_ && buffered_data_with_checksum_); + const char* src = data; + size_t left = size; + DataVerificationInfo v_info; + char checksum_buf[sizeof(uint32_t)]; + Env::IOPriority rate_limiter_priority_used = + WritableFileWriter::DecideRateLimiterPriority( + writable_file_->GetIOPriority(), op_rate_limiter_priority); + IOOptions io_options; + io_options.rate_limiter_priority = rate_limiter_priority_used; + // Check how much is allowed. Here, we loop until the rate limiter allows to + // write the entire buffer. + // TODO: need to be improved since it sort of defeats the purpose of the rate + // limiter + size_t data_size = left; + if (rate_limiter_ != nullptr && rate_limiter_priority_used != Env::IO_TOTAL) { + while (data_size > 0) { + size_t tmp_size; + tmp_size = rate_limiter_->RequestToken(data_size, buf_.Alignment(), + rate_limiter_priority_used, stats_, + RateLimiter::OpType::kWrite); + data_size -= tmp_size; + } + } + + { + IOSTATS_TIMER_GUARD(write_nanos); + TEST_SYNC_POINT("WritableFileWriter::Flush:BeforeAppend"); + + FileOperationInfo::StartTimePoint start_ts; + uint64_t old_size = writable_file_->GetFileSize(io_options, nullptr); + if (ShouldNotifyListeners()) { + start_ts = FileOperationInfo::StartNow(); + old_size = next_write_offset_; + } + { + auto prev_perf_level = GetPerfLevel(); + + IOSTATS_CPU_TIMER_GUARD(cpu_write_nanos, clock_); + + EncodeFixed32(checksum_buf, buffered_data_crc32c_checksum_); + v_info.checksum = Slice(checksum_buf, sizeof(uint32_t)); + s = writable_file_->Append(Slice(src, left), io_options, v_info, nullptr); + SetPerfLevel(prev_perf_level); + } + if (ShouldNotifyListeners()) { + auto finish_ts = std::chrono::steady_clock::now(); + NotifyOnFileWriteFinish(old_size, left, start_ts, finish_ts, s); + if (!s.ok()) { + NotifyOnIOError(s, FileOperationType::kAppend, file_name(), left, + old_size); + } + } + if (!s.ok()) { + // If writable_file_->Append() failed, then the data may or may not + // exist in the underlying memory buffer, OS page cache, remote file + // system's buffer, etc. If WritableFileWriter keeps the data in + // buf_, then a future Close() or write retry may send the data to + // the underlying file again. If the data does exist in the + // underlying buffer and gets written to the file eventually despite + // returning error, the file may end up with two duplicate pieces of + // data. Therefore, clear the buf_ at the WritableFileWriter layer + // and let caller determine error handling. + buf_.Size(0); + buffered_data_crc32c_checksum_ = 0; + set_seen_error(); + return s; + } + } + + IOSTATS_ADD(bytes_written, left); + TEST_KILL_RANDOM("WritableFileWriter::WriteBuffered:0"); + + // Buffer write is successful, reset the buffer current size to 0 and reset + // the corresponding checksum value + buf_.Size(0); + buffered_data_crc32c_checksum_ = 0; + uint64_t cur_size = flushed_size_.load(std::memory_order_acquire); + flushed_size_.store(cur_size + left, std::memory_order_release); + if (!s.ok()) { + set_seen_error(); + } + return s; +} + +void WritableFileWriter::UpdateFileChecksum(const Slice& data) { + if (checksum_generator_ != nullptr) { + checksum_generator_->Update(data.data(), data.size()); + } +} + +// Currently, crc32c checksum is used to calculate the checksum value of the +// content in the input buffer for handoff. In the future, the checksum might be +// calculated from the existing crc32c checksums of the in WAl and Manifest +// records, or even SST file blocks. +// TODO: effectively use the existing checksum of the data being writing to +// generate the crc32c checksum instead of a raw calculation. +void WritableFileWriter::Crc32cHandoffChecksumCalculation(const char* data, + size_t size, + char* buf) { + uint32_t v_crc32c = crc32c::Extend(0, data, size); + EncodeFixed32(buf, v_crc32c); +} + +// This flushes the accumulated data in the buffer. We pad data with zeros if +// necessary to the whole page. +// However, during automatic flushes padding would not be necessary. +// We always use RateLimiter if available. We move (Refit) any buffer bytes +// that are left over the +// whole number of pages to be written again on the next flush because we can +// only write on aligned +// offsets. +IOStatus WritableFileWriter::WriteDirect( + Env::IOPriority op_rate_limiter_priority) { + if (seen_error()) { + assert(false); + + return IOStatus::IOError("Writer has previous error."); + } + + assert(use_direct_io()); + IOStatus s; + const size_t alignment = buf_.Alignment(); + assert((next_write_offset_ % alignment) == 0); + + // Calculate whole page final file advance if all writes succeed + const size_t file_advance = + TruncateToPageBoundary(alignment, buf_.CurrentSize()); + + // Calculate the leftover tail, we write it here padded with zeros BUT we + // will write it again in the future either on Close() OR when the current + // whole page fills out. + const size_t leftover_tail = buf_.CurrentSize() - file_advance; + + // Round up and pad + buf_.PadToAlignmentWith(0); + + const char* src = buf_.BufferStart(); + uint64_t write_offset = next_write_offset_; + size_t left = buf_.CurrentSize(); + DataVerificationInfo v_info; + char checksum_buf[sizeof(uint32_t)]; + Env::IOPriority rate_limiter_priority_used = + WritableFileWriter::DecideRateLimiterPriority( + writable_file_->GetIOPriority(), op_rate_limiter_priority); + IOOptions io_options; + io_options.rate_limiter_priority = rate_limiter_priority_used; + + while (left > 0) { + // Check how much is allowed + size_t size = left; + if (rate_limiter_ != nullptr && + rate_limiter_priority_used != Env::IO_TOTAL) { + size = rate_limiter_->RequestToken(left, buf_.Alignment(), + rate_limiter_priority_used, stats_, + RateLimiter::OpType::kWrite); + } + + { + IOSTATS_TIMER_GUARD(write_nanos); + TEST_SYNC_POINT("WritableFileWriter::Flush:BeforeAppend"); + FileOperationInfo::StartTimePoint start_ts; + if (ShouldNotifyListeners()) { + start_ts = FileOperationInfo::StartNow(); + } + // direct writes must be positional + if (perform_data_verification_) { + Crc32cHandoffChecksumCalculation(src, size, checksum_buf); + v_info.checksum = Slice(checksum_buf, sizeof(uint32_t)); + s = writable_file_->PositionedAppend(Slice(src, size), write_offset, + io_options, v_info, nullptr); + } else { + s = writable_file_->PositionedAppend(Slice(src, size), write_offset, + io_options, nullptr); + } + + if (ShouldNotifyListeners()) { + auto finish_ts = std::chrono::steady_clock::now(); + NotifyOnFileWriteFinish(write_offset, size, start_ts, finish_ts, s); + if (!s.ok()) { + NotifyOnIOError(s, FileOperationType::kPositionedAppend, file_name(), + size, write_offset); + } + } + if (!s.ok()) { + buf_.Size(file_advance + leftover_tail); + set_seen_error(); + return s; + } + } + + IOSTATS_ADD(bytes_written, size); + left -= size; + src += size; + write_offset += size; + uint64_t cur_size = flushed_size_.load(std::memory_order_acquire); + flushed_size_.store(cur_size + size, std::memory_order_release); + assert((next_write_offset_ % alignment) == 0); + } + + if (s.ok()) { + // Move the tail to the beginning of the buffer + // This never happens during normal Append but rather during + // explicit call to Flush()/Sync() or Close() + buf_.RefitTail(file_advance, leftover_tail); + // This is where we start writing next time which may or not be + // the actual file size on disk. They match if the buffer size + // is a multiple of whole pages otherwise filesize_ is leftover_tail + // behind + next_write_offset_ += file_advance; + } else { + set_seen_error(); + } + return s; +} + +IOStatus WritableFileWriter::WriteDirectWithChecksum( + Env::IOPriority op_rate_limiter_priority) { + if (seen_error()) { + return AssertFalseAndGetStatusForPrevError(); + } + + assert(use_direct_io()); + assert(perform_data_verification_ && buffered_data_with_checksum_); + IOStatus s; + const size_t alignment = buf_.Alignment(); + assert((next_write_offset_ % alignment) == 0); + + // Calculate whole page final file advance if all writes succeed + const size_t file_advance = + TruncateToPageBoundary(alignment, buf_.CurrentSize()); + + // Calculate the leftover tail, we write it here padded with zeros BUT we + // will write it again in the future either on Close() OR when the current + // whole page fills out. + const size_t leftover_tail = buf_.CurrentSize() - file_advance; + + // Round up, pad, and combine the checksum. + size_t last_cur_size = buf_.CurrentSize(); + buf_.PadToAlignmentWith(0); + size_t padded_size = buf_.CurrentSize() - last_cur_size; + const char* padded_start = buf_.BufferStart() + last_cur_size; + uint32_t padded_checksum = crc32c::Value(padded_start, padded_size); + buffered_data_crc32c_checksum_ = crc32c::Crc32cCombine( + buffered_data_crc32c_checksum_, padded_checksum, padded_size); + + const char* src = buf_.BufferStart(); + uint64_t write_offset = next_write_offset_; + size_t left = buf_.CurrentSize(); + DataVerificationInfo v_info; + char checksum_buf[sizeof(uint32_t)]; + + Env::IOPriority rate_limiter_priority_used = + WritableFileWriter::DecideRateLimiterPriority( + writable_file_->GetIOPriority(), op_rate_limiter_priority); + IOOptions io_options; + io_options.rate_limiter_priority = rate_limiter_priority_used; + // Check how much is allowed. Here, we loop until the rate limiter allows to + // write the entire buffer. + // TODO: need to be improved since it sort of defeats the purpose of the rate + // limiter + size_t data_size = left; + if (rate_limiter_ != nullptr && rate_limiter_priority_used != Env::IO_TOTAL) { + while (data_size > 0) { + size_t size; + size = rate_limiter_->RequestToken(data_size, buf_.Alignment(), + rate_limiter_priority_used, stats_, + RateLimiter::OpType::kWrite); + data_size -= size; + } + } + + { + IOSTATS_TIMER_GUARD(write_nanos); + TEST_SYNC_POINT("WritableFileWriter::Flush:BeforeAppend"); + FileOperationInfo::StartTimePoint start_ts; + if (ShouldNotifyListeners()) { + start_ts = FileOperationInfo::StartNow(); + } + // direct writes must be positional + EncodeFixed32(checksum_buf, buffered_data_crc32c_checksum_); + v_info.checksum = Slice(checksum_buf, sizeof(uint32_t)); + s = writable_file_->PositionedAppend(Slice(src, left), write_offset, + io_options, v_info, nullptr); + + if (ShouldNotifyListeners()) { + auto finish_ts = std::chrono::steady_clock::now(); + NotifyOnFileWriteFinish(write_offset, left, start_ts, finish_ts, s); + if (!s.ok()) { + NotifyOnIOError(s, FileOperationType::kPositionedAppend, file_name(), + left, write_offset); + } + } + if (!s.ok()) { + // In this case, we do not change buffered_data_crc32c_checksum_ because + // it still aligns with the data in the buffer. + buf_.Size(file_advance + leftover_tail); + buffered_data_crc32c_checksum_ = + crc32c::Value(buf_.BufferStart(), buf_.CurrentSize()); + set_seen_error(); + return s; + } + } + + IOSTATS_ADD(bytes_written, left); + assert((next_write_offset_ % alignment) == 0); + uint64_t cur_size = flushed_size_.load(std::memory_order_acquire); + flushed_size_.store(cur_size + left, std::memory_order_release); + + if (s.ok()) { + // Move the tail to the beginning of the buffer + // This never happens during normal Append but rather during + // explicit call to Flush()/Sync() or Close(). Also the buffer checksum will + // recalculated accordingly. + buf_.RefitTail(file_advance, leftover_tail); + // Adjust the checksum value to align with the data in the buffer + buffered_data_crc32c_checksum_ = + crc32c::Value(buf_.BufferStart(), buf_.CurrentSize()); + // This is where we start writing next time which may or not be + // the actual file size on disk. They match if the buffer size + // is a multiple of whole pages otherwise filesize_ is leftover_tail + // behind + next_write_offset_ += file_advance; + } else { + set_seen_error(); + } + return s; +} +Env::IOPriority WritableFileWriter::DecideRateLimiterPriority( + Env::IOPriority writable_file_io_priority, + Env::IOPriority op_rate_limiter_priority) { + if (writable_file_io_priority == Env::IO_TOTAL && + op_rate_limiter_priority == Env::IO_TOTAL) { + return Env::IO_TOTAL; + } else if (writable_file_io_priority == Env::IO_TOTAL) { + return op_rate_limiter_priority; + } else if (op_rate_limiter_priority == Env::IO_TOTAL) { + return writable_file_io_priority; + } else { + return op_rate_limiter_priority; + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/file/writable_file_writer.h b/librocksdb-sys/rocksdb/file/writable_file_writer.h new file mode 100644 index 0000000..aac0f59 --- /dev/null +++ b/librocksdb-sys/rocksdb/file/writable_file_writer.h @@ -0,0 +1,320 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include +#include + +#include "db/version_edit.h" +#include "env/file_system_tracer.h" +#include "port/port.h" +#include "rocksdb/file_checksum.h" +#include "rocksdb/file_system.h" +#include "rocksdb/io_status.h" +#include "rocksdb/listener.h" +#include "rocksdb/rate_limiter.h" +#include "test_util/sync_point.h" +#include "util/aligned_buffer.h" + +namespace ROCKSDB_NAMESPACE { +class Statistics; +class SystemClock; + +// WritableFileWriter is a wrapper on top of Env::WritableFile. It provides +// facilities to: +// - Handle Buffered and Direct writes. +// - Rate limit writes. +// - Flush and Sync the data to the underlying filesystem. +// - Notify any interested listeners on the completion of a write. +// - Update IO stats. +class WritableFileWriter { + private: + void NotifyOnFileWriteFinish( + uint64_t offset, size_t length, + const FileOperationInfo::StartTimePoint& start_ts, + const FileOperationInfo::FinishTimePoint& finish_ts, + const IOStatus& io_status) { + FileOperationInfo info(FileOperationType::kWrite, file_name_, start_ts, + finish_ts, io_status, temperature_); + info.offset = offset; + info.length = length; + + for (auto& listener : listeners_) { + listener->OnFileWriteFinish(info); + } + info.status.PermitUncheckedError(); + } + void NotifyOnFileFlushFinish( + const FileOperationInfo::StartTimePoint& start_ts, + const FileOperationInfo::FinishTimePoint& finish_ts, + const IOStatus& io_status) { + FileOperationInfo info(FileOperationType::kFlush, file_name_, start_ts, + finish_ts, io_status, temperature_); + + for (auto& listener : listeners_) { + listener->OnFileFlushFinish(info); + } + info.status.PermitUncheckedError(); + } + void NotifyOnFileSyncFinish( + const FileOperationInfo::StartTimePoint& start_ts, + const FileOperationInfo::FinishTimePoint& finish_ts, + const IOStatus& io_status, + FileOperationType type = FileOperationType::kSync) { + FileOperationInfo info(type, file_name_, start_ts, finish_ts, io_status, + temperature_); + + for (auto& listener : listeners_) { + listener->OnFileSyncFinish(info); + } + info.status.PermitUncheckedError(); + } + void NotifyOnFileRangeSyncFinish( + uint64_t offset, size_t length, + const FileOperationInfo::StartTimePoint& start_ts, + const FileOperationInfo::FinishTimePoint& finish_ts, + const IOStatus& io_status) { + FileOperationInfo info(FileOperationType::kRangeSync, file_name_, start_ts, + finish_ts, io_status, temperature_); + info.offset = offset; + info.length = length; + + for (auto& listener : listeners_) { + listener->OnFileRangeSyncFinish(info); + } + info.status.PermitUncheckedError(); + } + void NotifyOnFileTruncateFinish( + const FileOperationInfo::StartTimePoint& start_ts, + const FileOperationInfo::FinishTimePoint& finish_ts, + const IOStatus& io_status) { + FileOperationInfo info(FileOperationType::kTruncate, file_name_, start_ts, + finish_ts, io_status, temperature_); + + for (auto& listener : listeners_) { + listener->OnFileTruncateFinish(info); + } + info.status.PermitUncheckedError(); + } + void NotifyOnFileCloseFinish( + const FileOperationInfo::StartTimePoint& start_ts, + const FileOperationInfo::FinishTimePoint& finish_ts, + const IOStatus& io_status) { + FileOperationInfo info(FileOperationType::kClose, file_name_, start_ts, + finish_ts, io_status, temperature_); + + for (auto& listener : listeners_) { + listener->OnFileCloseFinish(info); + } + info.status.PermitUncheckedError(); + } + + void NotifyOnIOError(const IOStatus& io_status, FileOperationType operation, + const std::string& file_path, size_t length = 0, + uint64_t offset = 0) { + if (listeners_.empty()) { + return; + } + IOErrorInfo io_error_info(io_status, operation, file_path, length, offset); + for (auto& listener : listeners_) { + listener->OnIOError(io_error_info); + } + io_error_info.io_status.PermitUncheckedError(); + } + + bool ShouldNotifyListeners() const { return !listeners_.empty(); } + void UpdateFileChecksum(const Slice& data); + void Crc32cHandoffChecksumCalculation(const char* data, size_t size, + char* buf); + + std::string file_name_; + FSWritableFilePtr writable_file_; + SystemClock* clock_; + AlignedBuffer buf_; + size_t max_buffer_size_; + // Actually written data size can be used for truncate + // not counting padding data + std::atomic filesize_; + std::atomic flushed_size_; + // This is necessary when we use unbuffered access + // and writes must happen on aligned offsets + // so we need to go back and write that page again + uint64_t next_write_offset_; + bool pending_sync_; + std::atomic seen_error_; +#ifndef NDEBUG + // SyncWithoutFlush() is the function that is allowed to be called + // concurrently with other function. One of the concurrent call + // could set seen_error_, and the other one would hit assertion + // in debug mode. + std::atomic sync_without_flush_called_ = false; +#endif // NDEBUG + uint64_t last_sync_size_; + uint64_t bytes_per_sync_; + RateLimiter* rate_limiter_; + Statistics* stats_; + std::vector> listeners_; + std::unique_ptr checksum_generator_; + bool checksum_finalized_; + bool perform_data_verification_; + uint32_t buffered_data_crc32c_checksum_; + bool buffered_data_with_checksum_; + Temperature temperature_; + + public: + WritableFileWriter( + std::unique_ptr&& file, const std::string& _file_name, + const FileOptions& options, SystemClock* clock = nullptr, + const std::shared_ptr& io_tracer = nullptr, + Statistics* stats = nullptr, + const std::vector>& listeners = {}, + FileChecksumGenFactory* file_checksum_gen_factory = nullptr, + bool perform_data_verification = false, + bool buffered_data_with_checksum = false) + : file_name_(_file_name), + writable_file_(std::move(file), io_tracer, _file_name), + clock_(clock), + buf_(), + max_buffer_size_(options.writable_file_max_buffer_size), + filesize_(0), + flushed_size_(0), + next_write_offset_(0), + pending_sync_(false), + seen_error_(false), + last_sync_size_(0), + bytes_per_sync_(options.bytes_per_sync), + rate_limiter_(options.rate_limiter), + stats_(stats), + listeners_(), + checksum_generator_(nullptr), + checksum_finalized_(false), + perform_data_verification_(perform_data_verification), + buffered_data_crc32c_checksum_(0), + buffered_data_with_checksum_(buffered_data_with_checksum) { + temperature_ = options.temperature; + assert(!use_direct_io() || max_buffer_size_ > 0); + TEST_SYNC_POINT_CALLBACK("WritableFileWriter::WritableFileWriter:0", + reinterpret_cast(max_buffer_size_)); + buf_.Alignment(writable_file_->GetRequiredBufferAlignment()); + buf_.AllocateNewBuffer(std::min((size_t)65536, max_buffer_size_)); + std::for_each(listeners.begin(), listeners.end(), + [this](const std::shared_ptr& e) { + if (e->ShouldBeNotifiedOnFileIO()) { + listeners_.emplace_back(e); + } + }); + if (file_checksum_gen_factory != nullptr) { + FileChecksumGenContext checksum_gen_context; + checksum_gen_context.file_name = _file_name; + checksum_generator_ = + file_checksum_gen_factory->CreateFileChecksumGenerator( + checksum_gen_context); + } + } + + static IOStatus Create(const std::shared_ptr& fs, + const std::string& fname, const FileOptions& file_opts, + std::unique_ptr* writer, + IODebugContext* dbg); + WritableFileWriter(const WritableFileWriter&) = delete; + + WritableFileWriter& operator=(const WritableFileWriter&) = delete; + + ~WritableFileWriter() { + auto s = Close(); + s.PermitUncheckedError(); + } + + std::string file_name() const { return file_name_; } + + // When this Append API is called, if the crc32c_checksum is not provided, we + // will calculate the checksum internally. + IOStatus Append(const Slice& data, uint32_t crc32c_checksum = 0, + Env::IOPriority op_rate_limiter_priority = Env::IO_TOTAL); + + IOStatus Pad(const size_t pad_bytes, + Env::IOPriority op_rate_limiter_priority = Env::IO_TOTAL); + + IOStatus Flush(Env::IOPriority op_rate_limiter_priority = Env::IO_TOTAL); + + IOStatus Close(); + + IOStatus Sync(bool use_fsync); + + // Sync only the data that was already Flush()ed. Safe to call concurrently + // with Append() and Flush(). If !writable_file_->IsSyncThreadSafe(), + // returns NotSupported status. + IOStatus SyncWithoutFlush(bool use_fsync); + + uint64_t GetFileSize() const { + return filesize_.load(std::memory_order_acquire); + } + + // Returns the size of data flushed to the underlying `FSWritableFile`. + // Expected to match `writable_file()->GetFileSize()`. + // The return value can serve as a lower-bound for the amount of data synced + // by a future call to `SyncWithoutFlush()`. + uint64_t GetFlushedSize() const { + return flushed_size_.load(std::memory_order_acquire); + } + + IOStatus InvalidateCache(size_t offset, size_t length) { + return writable_file_->InvalidateCache(offset, length); + } + + FSWritableFile* writable_file() const { return writable_file_.get(); } + + bool use_direct_io() { return writable_file_->use_direct_io(); } + + bool BufferIsEmpty() { return buf_.CurrentSize() == 0; } + + void TEST_SetFileChecksumGenerator( + FileChecksumGenerator* checksum_generator) { + checksum_generator_.reset(checksum_generator); + } + + std::string GetFileChecksum(); + + const char* GetFileChecksumFuncName() const; + + bool seen_error() const { + return seen_error_.load(std::memory_order_relaxed); + } + // For options of relaxed consistency, users might hope to continue + // operating on the file after an error happens. + void reset_seen_error() { + seen_error_.store(false, std::memory_order_relaxed); + } + void set_seen_error() { seen_error_.store(true, std::memory_order_relaxed); } + + IOStatus AssertFalseAndGetStatusForPrevError() { + // This should only happen if SyncWithoutFlush() was called. + assert(sync_without_flush_called_); + return IOStatus::IOError("Writer has previous error."); + } + + private: + // Decide the Rate Limiter priority. + static Env::IOPriority DecideRateLimiterPriority( + Env::IOPriority writable_file_io_priority, + Env::IOPriority op_rate_limiter_priority); + + // Used when os buffering is OFF and we are writing + // DMA such as in Direct I/O mode + IOStatus WriteDirect(Env::IOPriority op_rate_limiter_priority); + IOStatus WriteDirectWithChecksum(Env::IOPriority op_rate_limiter_priority); + // Normal write. + IOStatus WriteBuffered(const char* data, size_t size, + Env::IOPriority op_rate_limiter_priority); + IOStatus WriteBufferedWithChecksum(const char* data, size_t size, + Env::IOPriority op_rate_limiter_priority); + IOStatus RangeSync(uint64_t offset, uint64_t nbytes); + IOStatus SyncInternal(bool use_fsync); +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/fuzz/.gitignore b/librocksdb-sys/rocksdb/fuzz/.gitignore new file mode 100644 index 0000000..9dab421 --- /dev/null +++ b/librocksdb-sys/rocksdb/fuzz/.gitignore @@ -0,0 +1,5 @@ +db_fuzzer +db_map_fuzzer +sst_file_writer_fuzzer + +proto/gen/* diff --git a/librocksdb-sys/rocksdb/fuzz/Makefile b/librocksdb-sys/rocksdb/fuzz/Makefile new file mode 100644 index 0000000..b830405 --- /dev/null +++ b/librocksdb-sys/rocksdb/fuzz/Makefile @@ -0,0 +1,67 @@ +# Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +# This source code is licensed under both the GPLv2 (found in the +# COPYING file in the root directory) and Apache 2.0 License +# (found in the LICENSE.Apache file in the root directory). + +ROOT_DIR = $(abspath $(shell pwd)/../) + +include $(ROOT_DIR)/make_config.mk + +PROTOBUF_CFLAGS = `pkg-config --cflags protobuf` +PROTOBUF_LDFLAGS = `pkg-config --libs protobuf` + +PROTOBUF_MUTATOR_CFLAGS = `pkg-config --cflags libprotobuf-mutator` +PROTOBUF_MUTATOR_LDFLAGS = `pkg-config --libs libprotobuf-mutator` + +ROCKSDB_INCLUDE_DIR = $(ROOT_DIR)/include +ROCKSDB_LIB_DIR = $(ROOT_DIR) + +PROTO_IN = $(ROOT_DIR)/fuzz/proto +PROTO_OUT = $(ROOT_DIR)/fuzz/proto/gen + +ifneq ($(FUZZ_ENV), ossfuzz) +CC = $(CXX) +CCFLAGS += -Wall -fsanitize=address,fuzzer +CFLAGS += $(PLATFORM_CXXFLAGS) $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKSDB_INCLUDE_DIR) -I$(ROCKSDB_LIB_DIR) +LDFLAGS += $(PLATFORM_LDFLAGS) $(PROTOBUF_MUTATOR_LDFLAGS) $(PROTOBUF_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb +else +# OSS-Fuzz sets various environment flags that are used for compilation. +# These environment flags depend on which type of sanitizer build is being +# used, however, an ASan build would set the environment flags as follows: +# CFLAGS="-O1 -fno-omit-frame-pointer -gline-tables-only \ + -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address \ + -fsanitize-address-use-after-scope -fsanitize=fuzzer-no-link" +# CXXFLAGS="-O1 -fno-omit-frame-pointer -gline-tables-only \ + -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address \ + -fsanitize-address-use-after-scope -fsanitize=fuzzer-no-link \ + -stdlib=libc++" +# LIB_FUZZING_ENGINE="-fsanitize=fuzzer" +CC = $(CXX) +CCFLAGS = $(CXXFLAGS) +CFLAGS += $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKSDB_INCLUDE_DIR) -I$(ROCKSDB_LIB_DIR) +LDFLAGS += $(PLATFORM_LDFLAGS) $(LIB_FUZZING_ENGINE) $(PROTOBUF_MUTATOR_LDFLAGS) $(PROTOBUF_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb +endif + +.PHONY: gen_proto clean + +# Set PROTOC_BIN when invoking `make` if a custom protoc is required. +PROTOC_BIN ?= protoc + +gen_proto: + mkdir -p $(PROTO_OUT) + $(PROTOC_BIN) \ + --proto_path=$(PROTO_IN) \ + --cpp_out=$(PROTO_OUT) \ + $(PROTO_IN)/*.proto + +clean: + rm -rf db_fuzzer db_map_fuzzer sst_file_writer_fuzzer $(PROTO_OUT) + +db_fuzzer: db_fuzzer.cc + $(CC) $(CCFLAGS) -o db_fuzzer db_fuzzer.cc $(CFLAGS) $(LDFLAGS) + +db_map_fuzzer: gen_proto db_map_fuzzer.cc proto/gen/db_operation.pb.cc + $(CC) $(CCFLAGS) -o db_map_fuzzer db_map_fuzzer.cc proto/gen/db_operation.pb.cc $(CFLAGS) $(LDFLAGS) + +sst_file_writer_fuzzer: gen_proto sst_file_writer_fuzzer.cc proto/gen/db_operation.pb.cc + $(CC) $(CCFLAGS) -o sst_file_writer_fuzzer sst_file_writer_fuzzer.cc proto/gen/db_operation.pb.cc $(CFLAGS) $(LDFLAGS) diff --git a/librocksdb-sys/rocksdb/fuzz/README.md b/librocksdb-sys/rocksdb/fuzz/README.md new file mode 100644 index 0000000..238b283 --- /dev/null +++ b/librocksdb-sys/rocksdb/fuzz/README.md @@ -0,0 +1,165 @@ +# Fuzzing RocksDB + +## Overview + +This directory contains [fuzz tests](https://en.wikipedia.org/wiki/Fuzzing) for RocksDB. +RocksDB testing infrastructure currently includes unit tests and [stress tests](https://github.com/facebook/rocksdb/wiki/Stress-test), +we hope fuzz testing can catch more bugs. + +## Prerequisite + +We use [LLVM libFuzzer](http://llvm.org/docs/LibFuzzer.html) as the fuzzying engine, +so make sure you have [clang](https://clang.llvm.org/get_started.html) as your compiler. + +Some tests rely on [structure aware fuzzing](https://github.com/google/fuzzing/blob/master/docs/structure-aware-fuzzing.md). +We use [protobuf](https://developers.google.com/protocol-buffers) to define structured input to the fuzzer, +and use [libprotobuf-mutator](https://github.com/google/libprotobuf-mutator) as the custom libFuzzer mutator. +So make sure you have protobuf and libprotobuf-mutator installed, and make sure `pkg-config` can find them. +On some systems, there are both protobuf2 and protobuf3 in the package management system, +make sure protobuf3 is installed. + +If you do not want to install protobuf library yourself, you can rely on libprotobuf-mutator to download protobuf +for you. For details about installation, please refer to [libprotobuf-mutator README](https://github.com/google/libprotobuf-mutator#readme) + +## Example + +This example shows you how to do structure aware fuzzing to `rocksdb::SstFileWriter`. + +After walking through the steps to create the fuzzer, we'll introduce a bug into `rocksdb::SstFileWriter::Put`, +then show that the fuzzer can catch the bug. + +### Design the test + +We want the fuzzing engine to automatically generate a list of database operations, +then we apply these operations to `SstFileWriter` in sequence, +finally, after the SST file is generated, we use `SstFileReader` to check the file's checksum. + +### Define input + +We define the database operations in protobuf, each operation has a type of operation and a key value pair, +see [proto/db_operation.proto](proto/db_operation.proto) for details. + +### Define tests with the input + +In [sst_file_writer_fuzzer.cc](sst_file_writer_fuzzer.cc), +we define the tests to be run on the generated input: + +``` +DEFINE_PROTO_FUZZER(DBOperations& input) { + // apply the operations to SstFileWriter and use SstFileReader to verify checksum. + // ... +} +``` + +`SstFileWriter` requires the keys of the operations to be unique and be in ascending order, +but the fuzzing engine generates the input randomly, so we need to process the generated input before +passing it to `DEFINE_PROTO_FUZZER`, this is accomplished by registering a post processor: + +``` +protobuf_mutator::libfuzzer::PostProcessorRegistration +``` + +### Compile and link the fuzzer + +In the rocksdb root directory, compile rocksdb library by `make static_lib`. + +Go to the `fuzz` directory, +run `make sst_file_writer_fuzzer` to generate the fuzzer, +it will compile rocksdb static library, generate protobuf, then compile and link `sst_file_writer_fuzzer`. + +### Introduce a bug + +Manually introduce a bug to `SstFileWriter::Put`: + +``` +diff --git a/table/sst_file_writer.cc b/table/sst_file_writer.cc +index ab1ee7c4e..c7da9ffa0 100644 +--- a/table/sst_file_writer.cc ++++ b/table/sst_file_writer.cc +@@ -277,6 +277,11 @@ Status SstFileWriter::Add(const Slice& user_key, const Slice& value) { + } + + Status SstFileWriter::Put(const Slice& user_key, const Slice& value) { ++ if (user_key.starts_with("!")) { ++ if (value.ends_with("!")) { ++ return Status::Corruption("bomb"); ++ } ++ } + return rep_->Add(user_key, value, ValueType::kTypeValue); + } +``` + +The bug is that for `Put`, if `user_key` starts with `!` and `value` ends with `!`, then corrupt. + +### Run fuzz testing to catch the bug + +Run the fuzzer by `time ./sst_file_writer_fuzzer`. + +Here is the output on my machine: + +``` +Corruption: bomb +==59680== ERROR: libFuzzer: deadly signal + #0 0x109487315 in __sanitizer_print_stack_trace+0x35 (libclang_rt.asan_osx_dynamic.dylib:x86_64+0x4d315) + #1 0x108d63f18 in fuzzer::PrintStackTrace() FuzzerUtil.cpp:205 + #2 0x108d47613 in fuzzer::Fuzzer::CrashCallback() FuzzerLoop.cpp:232 + #3 0x7fff6af535fc in _sigtramp+0x1c (libsystem_platform.dylib:x86_64+0x35fc) + #4 0x7ffee720f3ef () + #5 0x7fff6ae29807 in abort+0x77 (libsystem_c.dylib:x86_64+0x7f807) + #6 0x108cf1c4c in TestOneProtoInput(DBOperations&)+0x113c (sst_file_writer_fuzzer:x86_64+0x100302c4c) + #7 0x108cf09be in LLVMFuzzerTestOneInput+0x16e (sst_file_writer_fuzzer:x86_64+0x1003019be) + #8 0x108d48ce0 in fuzzer::Fuzzer::ExecuteCallback(unsigned char const*, unsigned long) FuzzerLoop.cpp:556 + #9 0x108d48425 in fuzzer::Fuzzer::RunOne(unsigned char const*, unsigned long, bool, fuzzer::InputInfo*, bool*) FuzzerLoop.cpp:470 + #10 0x108d4a626 in fuzzer::Fuzzer::MutateAndTestOne() FuzzerLoop.cpp:698 + #11 0x108d4b325 in fuzzer::Fuzzer::Loop(std::__1::vector >&) FuzzerLoop.cpp:830 + #12 0x108d37fcd in fuzzer::FuzzerDriver(int*, char***, int (*)(unsigned char const*, unsigned long)) FuzzerDriver.cpp:829 + #13 0x108d652b2 in main FuzzerMain.cpp:19 + #14 0x7fff6ad5acc8 in start+0x0 (libdyld.dylib:x86_64+0x1acc8) + +NOTE: libFuzzer has rudimentary signal handlers. + Combine libFuzzer with AddressSanitizer or similar for better crash reports. +SUMMARY: libFuzzer: deadly signal +MS: 7 Custom-CustomCrossOver-InsertByte-Custom-ChangeBit-Custom-CustomCrossOver-; base unit: 90863b4d83c3f994bba0a417d0c2ee3b68f9e795 +0x6f,0x70,0x65,0x72,0x61,0x74,0x69,0x6f,0x6e,0x73,0x20,0x7b,0xa,0x20,0x20,0x6b,0x65,0x79,0x3a,0x20,0x22,0x21,0x22,0xa,0x20,0x20,0x76,0x61,0x6c,0x75,0x65,0x3a,0x20,0x22,0x21,0x22,0xa,0x20,0x20,0x74,0x79,0x70,0x65,0x3a,0x20,0x50,0x55,0x54,0xa,0x7d,0xa,0x6f,0x70,0x65,0x72,0x61,0x74,0x69,0x6f,0x6e,0x73,0x20,0x7b,0xa,0x20,0x20,0x6b,0x65,0x79,0x3a,0x20,0x22,0x2b,0x22,0xa,0x20,0x20,0x74,0x79,0x70,0x65,0x3a,0x20,0x50,0x55,0x54,0xa,0x7d,0xa,0x6f,0x70,0x65,0x72,0x61,0x74,0x69,0x6f,0x6e,0x73,0x20,0x7b,0xa,0x20,0x20,0x6b,0x65,0x79,0x3a,0x20,0x22,0x2e,0x22,0xa,0x20,0x20,0x74,0x79,0x70,0x65,0x3a,0x20,0x50,0x55,0x54,0xa,0x7d,0xa,0x6f,0x70,0x65,0x72,0x61,0x74,0x69,0x6f,0x6e,0x73,0x20,0x7b,0xa,0x20,0x20,0x6b,0x65,0x79,0x3a,0x20,0x22,0x5c,0x32,0x35,0x33,0x22,0xa,0x20,0x20,0x74,0x79,0x70,0x65,0x3a,0x20,0x50,0x55,0x54,0xa,0x7d,0xa, +operations {\x0a key: \"!\"\x0a value: \"!\"\x0a type: PUT\x0a}\x0aoperations {\x0a key: \"+\"\x0a type: PUT\x0a}\x0aoperations {\x0a key: \".\"\x0a type: PUT\x0a}\x0aoperations {\x0a key: \"\\253\"\x0a type: PUT\x0a}\x0a +artifact_prefix='./'; Test unit written to ./crash-a1460be302d09b548e61787178d9edaa40aea467 +Base64: b3BlcmF0aW9ucyB7CiAga2V5OiAiISIKICB2YWx1ZTogIiEiCiAgdHlwZTogUFVUCn0Kb3BlcmF0aW9ucyB7CiAga2V5OiAiKyIKICB0eXBlOiBQVVQKfQpvcGVyYXRpb25zIHsKICBrZXk6ICIuIgogIHR5cGU6IFBVVAp9Cm9wZXJhdGlvbnMgewogIGtleTogIlwyNTMiCiAgdHlwZTogUFVUCn0K +./sst_file_writer_fuzzer 5.97s user 4.40s system 64% cpu 16.195 total +``` + +Within 6 seconds, it catches the bug. + +The input that triggers the bug is persisted in `./crash-a1460be302d09b548e61787178d9edaa40aea467`: + +``` +$ cat ./crash-a1460be302d09b548e61787178d9edaa40aea467 +operations { + key: "!" + value: "!" + type: PUT +} +operations { + key: "+" + type: PUT +} +operations { + key: "." + type: PUT +} +operations { + key: "\253" + type: PUT +} +``` + +### Reproduce the crash to debug + +The above crash can be reproduced by `./sst_file_writer_fuzzer ./crash-a1460be302d09b548e61787178d9edaa40aea467`, +so you can debug the crash. + +## Future Work + +According to [OSS-Fuzz](https://github.com/google/oss-fuzz), +`as of June 2020, OSS-Fuzz has found over 20,000 bugs in 300 open source projects.` + +RocksDB can join OSS-Fuzz together with other open source projects such as sqlite. diff --git a/librocksdb-sys/rocksdb/fuzz/db_fuzzer.cc b/librocksdb-sys/rocksdb/fuzz/db_fuzzer.cc new file mode 100644 index 0000000..e6d5bb6 --- /dev/null +++ b/librocksdb-sys/rocksdb/fuzz/db_fuzzer.cc @@ -0,0 +1,172 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include + +#include "rocksdb/db.h" + +enum OperationType { + kPut, + kGet, + kDelete, + kGetProperty, + kIterator, + kSnapshot, + kOpenClose, + kColumn, + kCompactRange, + kSeekForPrev, + OP_COUNT +}; + +constexpr char db_path[] = "/tmp/testdb"; + +// Fuzzes DB operations by doing interpretations on the data. Both the +// sequence of API calls to be called on the DB as well as the arguments +// to each of these APIs are interpreted by way of the data buffer. +// The operations that the fuzzer supports are given by the OperationType +// enum. The goal is to capture sanitizer bugs, so the code should be +// compiled with a given sanitizer (ASan, UBSan, MSan). +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + ROCKSDB_NAMESPACE::DB* db; + ROCKSDB_NAMESPACE::Options options; + options.create_if_missing = true; + ROCKSDB_NAMESPACE::Status status = + ROCKSDB_NAMESPACE::DB::Open(options, db_path, &db); + if (!status.ok()) { + return 0; + } + FuzzedDataProvider fuzzed_data(data, size); + + // perform a sequence of calls on our db instance + int max_iter = static_cast(data[0]); + for (int i = 0; i < max_iter && i < size; i++) { + OperationType op = static_cast(data[i] % OP_COUNT); + + switch (op) { + case kPut: { + std::string key = fuzzed_data.ConsumeRandomLengthString(); + std::string val = fuzzed_data.ConsumeRandomLengthString(); + db->Put(ROCKSDB_NAMESPACE::WriteOptions(), key, val); + break; + } + case kGet: { + std::string key = fuzzed_data.ConsumeRandomLengthString(); + std::string value; + db->Get(ROCKSDB_NAMESPACE::ReadOptions(), key, &value); + break; + } + case kDelete: { + std::string key = fuzzed_data.ConsumeRandomLengthString(); + db->Delete(ROCKSDB_NAMESPACE::WriteOptions(), key); + break; + } + case kGetProperty: { + std::string prop; + std::string property_name = fuzzed_data.ConsumeRandomLengthString(); + db->GetProperty(property_name, &prop); + break; + } + case kIterator: { + ROCKSDB_NAMESPACE::Iterator* it = + db->NewIterator(ROCKSDB_NAMESPACE::ReadOptions()); + for (it->SeekToFirst(); it->Valid(); it->Next()) { + } + delete it; + break; + } + case kSnapshot: { + ROCKSDB_NAMESPACE::ReadOptions snapshot_options; + snapshot_options.snapshot = db->GetSnapshot(); + ROCKSDB_NAMESPACE::Iterator* it = db->NewIterator(snapshot_options); + db->ReleaseSnapshot(snapshot_options.snapshot); + delete it; + break; + } + case kOpenClose: { + db->Close(); + delete db; + status = ROCKSDB_NAMESPACE::DB::Open(options, db_path, &db); + if (!status.ok()) { + ROCKSDB_NAMESPACE::DestroyDB(db_path, options); + return 0; + } + + break; + } + case kColumn: { + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf; + ROCKSDB_NAMESPACE::Status s; + s = db->CreateColumnFamily(ROCKSDB_NAMESPACE::ColumnFamilyOptions(), + "new_cf", &cf); + s = db->DestroyColumnFamilyHandle(cf); + db->Close(); + delete db; + + // open DB with two column families + std::vector column_families; + // have to open default column family + column_families.push_back(ROCKSDB_NAMESPACE::ColumnFamilyDescriptor( + ROCKSDB_NAMESPACE::kDefaultColumnFamilyName, + ROCKSDB_NAMESPACE::ColumnFamilyOptions())); + // open the new one, too + column_families.push_back(ROCKSDB_NAMESPACE::ColumnFamilyDescriptor( + "new_cf", ROCKSDB_NAMESPACE::ColumnFamilyOptions())); + std::vector handles; + s = ROCKSDB_NAMESPACE::DB::Open(ROCKSDB_NAMESPACE::DBOptions(), db_path, + column_families, &handles, &db); + + if (s.ok()) { + std::string key1 = fuzzed_data.ConsumeRandomLengthString(); + std::string val1 = fuzzed_data.ConsumeRandomLengthString(); + std::string key2 = fuzzed_data.ConsumeRandomLengthString(); + s = db->Put(ROCKSDB_NAMESPACE::WriteOptions(), handles[1], key1, + val1); + std::string value; + s = db->Get(ROCKSDB_NAMESPACE::ReadOptions(), handles[1], key2, + &value); + s = db->DropColumnFamily(handles[1]); + for (auto handle : handles) { + s = db->DestroyColumnFamilyHandle(handle); + } + } else { + status = ROCKSDB_NAMESPACE::DB::Open(options, db_path, &db); + if (!status.ok()) { + // At this point there is no saving to do. So we exit + ROCKSDB_NAMESPACE::DestroyDB(db_path, ROCKSDB_NAMESPACE::Options()); + return 0; + } + } + break; + } + case kCompactRange: { + std::string slice_start = fuzzed_data.ConsumeRandomLengthString(); + std::string slice_end = fuzzed_data.ConsumeRandomLengthString(); + + ROCKSDB_NAMESPACE::Slice begin(slice_start); + ROCKSDB_NAMESPACE::Slice end(slice_end); + ROCKSDB_NAMESPACE::CompactRangeOptions options; + ROCKSDB_NAMESPACE::Status s = db->CompactRange(options, &begin, &end); + break; + } + case kSeekForPrev: { + std::string key = fuzzed_data.ConsumeRandomLengthString(); + auto iter = db->NewIterator(ROCKSDB_NAMESPACE::ReadOptions()); + iter->SeekForPrev(key); + delete iter; + break; + } + case OP_COUNT: + break; + } + } + + // Cleanup DB + db->Close(); + delete db; + ROCKSDB_NAMESPACE::DestroyDB(db_path, options); + return 0; +} diff --git a/librocksdb-sys/rocksdb/fuzz/db_map_fuzzer.cc b/librocksdb-sys/rocksdb/fuzz/db_map_fuzzer.cc new file mode 100644 index 0000000..ed9df8f --- /dev/null +++ b/librocksdb-sys/rocksdb/fuzz/db_map_fuzzer.cc @@ -0,0 +1,107 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include +#include +#include +#include + +#include "proto/gen/db_operation.pb.h" +#include "rocksdb/db.h" +#include "rocksdb/file_system.h" +#include "src/libfuzzer/libfuzzer_macro.h" +#include "util.h" + +protobuf_mutator::libfuzzer::PostProcessorRegistration reg = { + [](DBOperations* input, unsigned int /* seed */) { + const ROCKSDB_NAMESPACE::Comparator* comparator = + ROCKSDB_NAMESPACE::BytewiseComparator(); + auto ops = input->mutable_operations(); + // Make sure begin <= end for DELETE_RANGE. + for (DBOperation& op : *ops) { + if (op.type() == OpType::DELETE_RANGE) { + auto begin = op.key(); + auto end = op.value(); + if (comparator->Compare(begin, end) > 0) { + std::swap(begin, end); + op.set_key(begin); + op.set_value(end); + } + } + } + }}; + +// Execute randomly generated operations on both a DB and a std::map, +// then reopen the DB and make sure that iterating the DB produces the +// same key-value pairs as iterating through the std::map. +DEFINE_PROTO_FUZZER(DBOperations& input) { + if (input.operations().empty()) { + return; + } + + const std::string kDbPath = "/tmp/db_map_fuzzer_test"; + auto fs = ROCKSDB_NAMESPACE::FileSystem::Default(); + if (fs->FileExists(kDbPath, ROCKSDB_NAMESPACE::IOOptions(), /*dbg=*/nullptr) + .ok()) { + std::cerr << "db path " << kDbPath << " already exists" << std::endl; + abort(); + } + + std::map kv; + ROCKSDB_NAMESPACE::DB* db = nullptr; + ROCKSDB_NAMESPACE::Options options; + options.create_if_missing = true; + CHECK_OK(ROCKSDB_NAMESPACE::DB::Open(options, kDbPath, &db)); + + for (const DBOperation& op : input.operations()) { + switch (op.type()) { + case OpType::PUT: { + CHECK_OK( + db->Put(ROCKSDB_NAMESPACE::WriteOptions(), op.key(), op.value())); + kv[op.key()] = op.value(); + break; + } + case OpType::MERGE: { + break; + } + case OpType::DELETE: { + CHECK_OK(db->Delete(ROCKSDB_NAMESPACE::WriteOptions(), op.key())); + kv.erase(op.key()); + break; + } + case OpType::DELETE_RANGE: { + // [op.key(), op.value()) corresponds to [begin, end). + CHECK_OK(db->DeleteRange(ROCKSDB_NAMESPACE::WriteOptions(), + db->DefaultColumnFamily(), op.key(), + op.value())); + kv.erase(kv.lower_bound(op.key()), kv.lower_bound(op.value())); + break; + } + default: { + std::cerr << "Unsupported operation" << static_cast(op.type()); + return; + } + } + } + CHECK_OK(db->Close()); + delete db; + db = nullptr; + + CHECK_OK(ROCKSDB_NAMESPACE::DB::Open(options, kDbPath, &db)); + auto kv_it = kv.begin(); + ROCKSDB_NAMESPACE::Iterator* it = + db->NewIterator(ROCKSDB_NAMESPACE::ReadOptions()); + for (it->SeekToFirst(); it->Valid(); it->Next(), kv_it++) { + CHECK_TRUE(kv_it != kv.end()); + CHECK_EQ(it->key().ToString(), kv_it->first); + CHECK_EQ(it->value().ToString(), kv_it->second); + } + CHECK_TRUE(kv_it == kv.end()); + delete it; + + CHECK_OK(db->Close()); + delete db; + CHECK_OK(ROCKSDB_NAMESPACE::DestroyDB(kDbPath, options)); +} diff --git a/librocksdb-sys/rocksdb/fuzz/proto/db_operation.proto b/librocksdb-sys/rocksdb/fuzz/proto/db_operation.proto new file mode 100644 index 0000000..20a55ea --- /dev/null +++ b/librocksdb-sys/rocksdb/fuzz/proto/db_operation.proto @@ -0,0 +1,28 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +// Defines database operations. +// Each operation is a key-value pair and an operation type. + +syntax = "proto2"; + +enum OpType { + PUT = 0; + MERGE = 1; + DELETE = 2; + DELETE_RANGE = 3; +} + +message DBOperation { + required string key = 1; + // value is ignored for DELETE. + // [key, value] is the range for DELETE_RANGE. + optional string value = 2; + required OpType type = 3; +} + +message DBOperations { + repeated DBOperation operations = 1; +} diff --git a/librocksdb-sys/rocksdb/fuzz/sst_file_writer_fuzzer.cc b/librocksdb-sys/rocksdb/fuzz/sst_file_writer_fuzzer.cc new file mode 100644 index 0000000..676daf5 --- /dev/null +++ b/librocksdb-sys/rocksdb/fuzz/sst_file_writer_fuzzer.cc @@ -0,0 +1,210 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include +#include +#include +#include + +#include "proto/gen/db_operation.pb.h" +#include "rocksdb/file_system.h" +#include "rocksdb/sst_file_writer.h" +#include "src/libfuzzer/libfuzzer_macro.h" +#include "table/table_builder.h" +#include "table/table_reader.h" +#include "util.h" + +using ROCKSDB_NAMESPACE::BytewiseComparator; +using ROCKSDB_NAMESPACE::Comparator; +using ROCKSDB_NAMESPACE::EnvOptions; +using ROCKSDB_NAMESPACE::ExternalSstFileInfo; +using ROCKSDB_NAMESPACE::FileOptions; +using ROCKSDB_NAMESPACE::FileSystem; +using ROCKSDB_NAMESPACE::ImmutableCFOptions; +using ROCKSDB_NAMESPACE::ImmutableOptions; +using ROCKSDB_NAMESPACE::InternalIterator; +using ROCKSDB_NAMESPACE::IOOptions; +using ROCKSDB_NAMESPACE::kMaxSequenceNumber; +using ROCKSDB_NAMESPACE::Options; +using ROCKSDB_NAMESPACE::ParsedInternalKey; +using ROCKSDB_NAMESPACE::ParseInternalKey; +using ROCKSDB_NAMESPACE::RandomAccessFileReader; +using ROCKSDB_NAMESPACE::ReadOptions; +using ROCKSDB_NAMESPACE::SstFileWriter; +using ROCKSDB_NAMESPACE::Status; +using ROCKSDB_NAMESPACE::TableReader; +using ROCKSDB_NAMESPACE::TableReaderCaller; +using ROCKSDB_NAMESPACE::TableReaderOptions; +using ROCKSDB_NAMESPACE::ValueType; + +// Keys in SST file writer operations must be unique and in ascending order. +// For each DBOperation generated by the fuzzer, this function is called on +// it to deduplicate and sort the keys in the DBOperations. +protobuf_mutator::libfuzzer::PostProcessorRegistration reg = { + [](DBOperations* input, unsigned int /* seed */) { + const Comparator* comparator = BytewiseComparator(); + auto ops = input->mutable_operations(); + + // Make sure begin <= end for DELETE_RANGE. + for (DBOperation& op : *ops) { + if (op.type() == OpType::DELETE_RANGE) { + auto begin = op.key(); + auto end = op.value(); + if (comparator->Compare(begin, end) > 0) { + std::swap(begin, end); + op.set_key(begin); + op.set_value(end); + } + } + } + + std::sort(ops->begin(), ops->end(), + [&comparator](const DBOperation& a, const DBOperation& b) { + return comparator->Compare(a.key(), b.key()) < 0; + }); + + auto last = std::unique( + ops->begin(), ops->end(), + [&comparator](const DBOperation& a, const DBOperation& b) { + return comparator->Compare(a.key(), b.key()) == 0; + }); + ops->erase(last, ops->end()); + }}; + +TableReader* NewTableReader(const std::string& sst_file_path, + const Options& options, + const EnvOptions& env_options, + const ImmutableCFOptions& cf_ioptions) { + // This code block is similar to SstFileReader::Open. + + uint64_t file_size = 0; + std::unique_ptr file_reader; + std::unique_ptr table_reader; + const auto& fs = options.env->GetFileSystem(); + FileOptions fopts(env_options); + Status s = options.env->GetFileSize(sst_file_path, &file_size); + if (s.ok()) { + s = RandomAccessFileReader::Create(fs, sst_file_path, fopts, &file_reader, + nullptr); + } + if (s.ok()) { + ImmutableOptions iopts(options, cf_ioptions); + TableReaderOptions t_opt(iopts, /*prefix_extractor=*/nullptr, env_options, + cf_ioptions.internal_comparator, + 0 /* block_protection_bytes_per_key */); + t_opt.largest_seqno = kMaxSequenceNumber; + s = options.table_factory->NewTableReader(t_opt, std::move(file_reader), + file_size, &table_reader, + /*prefetch=*/false); + } + if (!s.ok()) { + std::cerr << "Failed to create TableReader for " << sst_file_path << ": " + << s.ToString() << std::endl; + abort(); + } + return table_reader.release(); +} + +ValueType ToValueType(OpType op_type) { + switch (op_type) { + case OpType::PUT: + return ValueType::kTypeValue; + case OpType::MERGE: + return ValueType::kTypeMerge; + case OpType::DELETE: + return ValueType::kTypeDeletion; + case OpType::DELETE_RANGE: + return ValueType::kTypeRangeDeletion; + default: + std::cerr << "Unknown operation type " << static_cast(op_type) + << std::endl; + abort(); + } +} + +// Fuzzes DB operations as input, let SstFileWriter generate a SST file +// according to the operations, then let TableReader read and check all the +// key-value pairs from the generated SST file. +DEFINE_PROTO_FUZZER(DBOperations& input) { + if (input.operations().empty()) { + return; + } + + std::string sstfile; + { + auto fs = FileSystem::Default(); + std::string dir; + IOOptions opt; + CHECK_OK(fs->GetTestDirectory(opt, &dir, nullptr)); + sstfile = dir + "/SstFileWriterFuzzer.sst"; + } + + Options options; + EnvOptions env_options(options); + ImmutableCFOptions cf_ioptions(options); + + // Generate sst file. + SstFileWriter writer(env_options, options); + CHECK_OK(writer.Open(sstfile)); + for (const DBOperation& op : input.operations()) { + switch (op.type()) { + case OpType::PUT: { + CHECK_OK(writer.Put(op.key(), op.value())); + break; + } + case OpType::MERGE: { + CHECK_OK(writer.Merge(op.key(), op.value())); + break; + } + case OpType::DELETE: { + CHECK_OK(writer.Delete(op.key())); + break; + } + case OpType::DELETE_RANGE: { + CHECK_OK(writer.DeleteRange(op.key(), op.value())); + break; + } + default: { + std::cerr << "Unsupported operation" << static_cast(op.type()) + << std::endl; + abort(); + } + } + } + ExternalSstFileInfo info; + CHECK_OK(writer.Finish(&info)); + + // Iterate and verify key-value pairs. + std::unique_ptr table_reader( + ::NewTableReader(sstfile, options, env_options, cf_ioptions)); + ReadOptions roptions; + CHECK_OK(table_reader->VerifyChecksum(roptions, + TableReaderCaller::kUncategorized)); + std::unique_ptr it( + table_reader->NewIterator(roptions, /*prefix_extractor=*/nullptr, + /*arena=*/nullptr, /*skip_filters=*/true, + TableReaderCaller::kUncategorized)); + it->SeekToFirst(); + for (const DBOperation& op : input.operations()) { + if (op.type() == OpType::DELETE_RANGE) { + // InternalIterator cannot iterate over DELETE_RANGE entries. + continue; + } + CHECK_TRUE(it->Valid()); + ParsedInternalKey ikey; + CHECK_OK(ParseInternalKey(it->key(), &ikey, /*log_err_key=*/true)); + CHECK_EQ(ikey.user_key.ToString(), op.key()); + CHECK_EQ(ikey.sequence, 0); + CHECK_EQ(ikey.type, ToValueType(op.type())); + if (op.type() != OpType::DELETE) { + CHECK_EQ(op.value(), it->value().ToString()); + } + it->Next(); + } + CHECK_TRUE(!it->Valid()); + + // Delete sst file. + remove(sstfile.c_str()); +} diff --git a/librocksdb-sys/rocksdb/fuzz/util.h b/librocksdb-sys/rocksdb/fuzz/util.h new file mode 100644 index 0000000..9701182 --- /dev/null +++ b/librocksdb-sys/rocksdb/fuzz/util.h @@ -0,0 +1,29 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#define CHECK_OK(expression) \ + do { \ + auto status = (expression); \ + if (!status.ok()) { \ + std::cerr << status.ToString() << std::endl; \ + abort(); \ + } \ + } while (0) + +#define CHECK_EQ(a, b) \ + if (a != b) { \ + std::cerr << "(" << #a << "=" << a << ") != (" << #b << "=" << b << ")" \ + << std::endl; \ + abort(); \ + } + +#define CHECK_TRUE(cond) \ + if (!(cond)) { \ + std::cerr << "\"" << #cond << "\" is false" << std::endl; \ + abort(); \ + } diff --git a/librocksdb-sys/rocksdb/include/rocksdb/advanced_cache.h b/librocksdb-sys/rocksdb/include/rocksdb/advanced_cache.h new file mode 100644 index 0000000..997a414 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/advanced_cache.h @@ -0,0 +1,623 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// APIs for customizing read caches in RocksDB. + +#pragma once + +#include +#include +#include +#include + +#include "rocksdb/cache.h" +#include "rocksdb/memory_allocator.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +class Logger; +class SecondaryCacheResultHandle; +class Statistics; + +// A Cache maps keys to objects resident in memory, tracks reference counts +// on those key-object entries, and is able to remove unreferenced entries +// whenever it wants. All operations are fully thread safe except as noted. +// Inserted entries have a specified "charge" which is some quantity in +// unspecified units, typically bytes of memory used. A Cache will typically +// have a finite capacity in units of charge, and evict entries as needed +// to stay at or below that capacity. +// +// NOTE: This API is for expert use only and is intended more for customizing +// cache behavior than for calling into outside of RocksDB. It is subject to +// change as RocksDB evolves, especially the RocksDB block cache. Overriding +// CacheWrapper is the preferred way of customizing some operations on an +// existing implementation. +// +// INTERNAL: See typed_cache.h for convenient wrappers on top of this API. +// New virtual functions must also be added to CacheWrapper below. +class Cache { + public: // types hidden from API client + // Opaque handle to an entry stored in the cache. + struct Handle {}; + + public: // types hidden from Cache implementation + // Pointer to cached object of unspecified type. (This type alias is + // provided for clarity, not really for type checking.) + using ObjectPtr = void*; + + // Opaque object providing context (settings, etc.) to create objects + // for primary cache from saved (serialized) secondary cache entries. + struct CreateContext {}; + + public: // type defs + // Depending on implementation, cache entries with higher priority levels + // could be less likely to get evicted than entries with lower priority + // levels. The "high" priority level applies to certain SST metablocks (e.g. + // index and filter blocks) if the option + // cache_index_and_filter_blocks_with_high_priority is set. The "low" priority + // level is used for other kinds of SST blocks (most importantly, data + // blocks), as well as the above metablocks in case + // cache_index_and_filter_blocks_with_high_priority is + // not set. The "bottom" priority level is for BlobDB's blob values. + enum class Priority { HIGH, LOW, BOTTOM }; + + // A set of callbacks to allow objects in the primary block cache to be + // be persisted in a secondary cache. The purpose of the secondary cache + // is to support other ways of caching the object, such as persistent or + // compressed data, that may require the object to be parsed and transformed + // in some way. Since the primary cache holds C++ objects and the secondary + // cache may only hold flat data that doesn't need relocation, these + // callbacks need to be provided by the user of the block + // cache to do the conversion. + // The CacheItemHelper is passed to Insert() and Lookup(). It has pointers + // to callback functions for size, saving and deletion of the + // object. The callbacks are defined in C-style in order to make them + // stateless and not add to the cache metadata size. + // Saving multiple std::function objects will take up 32 bytes per + // function, even if its not bound to an object and does no capture. + // + // All the callbacks are C-style function pointers in order to simplify + // lifecycle management. Objects in the cache can outlive the parent DB, + // so anything required for these operations should be contained in the + // object itself. + // + // The SizeCallback takes a pointer to the object and returns the size + // of the persistable data. It can be used by the secondary cache to allocate + // memory if needed. + // + // RocksDB callbacks are NOT exception-safe. A callback completing with an + // exception can lead to undefined behavior in RocksDB, including data loss, + // unreported corruption, deadlocks, and more. + using SizeCallback = size_t (*)(ObjectPtr obj); + + // The SaveToCallback takes an object pointer and saves the persistable + // data into a buffer. The secondary cache may decide to not store it in a + // contiguous buffer, in which case this callback will be called multiple + // times with increasing offset + using SaveToCallback = Status (*)(ObjectPtr from_obj, size_t from_offset, + size_t length, char* out_buf); + + // A function pointer type for destruction of a cache object. This will + // typically call the destructor for the appropriate type of the object. + // The Cache is responsible for copying and reclaiming space for the key, + // but objects are managed in part using this callback. Generally a DeleterFn + // can be nullptr if the ObjectPtr does not need destruction (e.g. nullptr or + // pointer into static data). + using DeleterFn = void (*)(ObjectPtr obj, MemoryAllocator* allocator); + + // The CreateCallback is takes in a buffer from the NVM cache and constructs + // an object using it. The callback doesn't have ownership of the buffer and + // should copy the contents into its own buffer. The CreateContext* is + // provided by Lookup and may be used to follow DB- or CF-specific settings. + // In case of some error, non-OK is returned and the caller should ignore + // any result in out_obj. (The implementation must clean up after itself.) + using CreateCallback = Status (*)(const Slice& data, CreateContext* context, + MemoryAllocator* allocator, + ObjectPtr* out_obj, size_t* out_charge); + + // A struct with pointers to helper functions for spilling items from the + // cache into the secondary cache. May be extended in the future. An + // instance of this struct is expected to outlive the cache. + struct CacheItemHelper { + // Function for deleting an object on its removal from the Cache. + // nullptr is only for entries that require no destruction, such as + // "placeholder" cache entries with nullptr object. + DeleterFn del_cb; // (<- Most performance critical) + // Next three are used for persisting values as described above. + // If any is nullptr, then all three should be nullptr and persisting the + // entry to/from secondary cache is not supported. + SizeCallback size_cb; + SaveToCallback saveto_cb; + CreateCallback create_cb; + // Classification of the entry for monitoring purposes in block cache. + CacheEntryRole role; + // Another CacheItemHelper (or this one) without secondary cache support. + // This is provided so that items promoted from secondary cache into + // primary cache without removal from the secondary cache can be prevented + // from attempting re-insertion into secondary cache (for efficiency). + const CacheItemHelper* without_secondary_compat; + + CacheItemHelper() : CacheItemHelper(CacheEntryRole::kMisc) {} + + // For helpers without SecondaryCache support + explicit CacheItemHelper(CacheEntryRole _role, DeleterFn _del_cb = nullptr) + : CacheItemHelper(_role, _del_cb, nullptr, nullptr, nullptr, this) {} + + // For helpers with SecondaryCache support + explicit CacheItemHelper(CacheEntryRole _role, DeleterFn _del_cb, + SizeCallback _size_cb, SaveToCallback _saveto_cb, + CreateCallback _create_cb, + const CacheItemHelper* _without_secondary_compat) + : del_cb(_del_cb), + size_cb(_size_cb), + saveto_cb(_saveto_cb), + create_cb(_create_cb), + role(_role), + without_secondary_compat(_without_secondary_compat) { + // Either all three secondary cache callbacks are non-nullptr or + // all three are nullptr + assert((size_cb != nullptr) == (saveto_cb != nullptr)); + assert((size_cb != nullptr) == (create_cb != nullptr)); + // without_secondary_compat points to equivalent but without + // secondary support + assert(role == without_secondary_compat->role); + assert(del_cb == without_secondary_compat->del_cb); + assert(!without_secondary_compat->IsSecondaryCacheCompatible()); + } + inline bool IsSecondaryCacheCompatible() const { + return size_cb != nullptr; + } + }; + + public: // ctor/dtor/create + Cache(std::shared_ptr allocator = nullptr) + : memory_allocator_(std::move(allocator)) {} + // No copying allowed + Cache(const Cache&) = delete; + Cache& operator=(const Cache&) = delete; + + // Destroys all remaining entries by calling the associated "deleter" + virtual ~Cache() {} + + // Creates a new Cache based on the input value string and returns the result. + // Currently, this method can be used to create LRUCaches only + // @param config_options + // @param value The value might be: + // - an old-style cache ("1M") -- equivalent to NewLRUCache(1024*102( + // - Name-value option pairs -- "capacity=1M; num_shard_bits=4; + // For the LRUCache, the values are defined in LRUCacheOptions. + // @param result The new Cache object + // @return OK if the cache was successfully created + // @return NotFound if an invalid name was specified in the value + // @return InvalidArgument if either the options were not valid + static Status CreateFromString(const ConfigOptions& config_options, + const std::string& value, + std::shared_ptr* result); + + public: // functions + // The type of the Cache + virtual const char* Name() const = 0; + + // The Insert and Lookup APIs below are intended to allow cached objects + // to be demoted/promoted between the primary block cache and a secondary + // cache. The secondary cache could be a non-volatile cache, and will + // likely store the object in a different representation. They rely on a + // per object CacheItemHelper to do the conversions. + // The secondary cache may persist across process and system restarts, + // and may even be moved between hosts. Therefore, the cache key must + // be repeatable across restarts/reboots, and globally unique if + // multiple DBs share the same cache and the set of DBs can change + // over time. + + // Insert a mapping from key->object into the cache and assign it + // the specified charge against the total cache capacity. If + // strict_capacity_limit is true and cache reaches its full capacity, + // return Status::MemoryLimit. `obj` must be non-nullptr if compatible + // with secondary cache (helper->size_cb != nullptr), because Value() == + // nullptr is reserved for indicating some secondary cache failure cases. + // On success, returns OK and takes ownership of `obj`, eventually deleting + // it with helper->del_cb. On non-OK return, the caller maintains ownership + // of `obj` so will often need to delete it in such cases. + // + // The helper argument is saved by the cache and will be used when the + // inserted object is evicted or considered for promotion to the secondary + // cache. Promotion to secondary cache is only enabled if helper->size_cb + // != nullptr. The helper must outlive the cache. Callers may use + // &kNoopCacheItemHelper as a trivial helper (no deleter for the object, + // no secondary cache). `helper` must not be nullptr (efficiency). + // + // If `handle` is not nullptr and return status is OK, `handle` is set + // to a Handle* for the entry. The caller must call this->Release(handle) + // when the returned entry is no longer needed. If `handle` is nullptr, it is + // as if Release is called immediately after Insert. + // + // Regardless of whether the item was inserted into the cache, + // it will attempt to insert it into the secondary cache if one is + // configured, and the helper supports it. + // The cache implementation must support a secondary cache, otherwise + // the item is only inserted into the primary cache. It may + // defer the insertion to the secondary cache as it sees fit. + // + // When the inserted entry is no longer needed, it will be destroyed using + // helper->del_cb (if non-nullptr). + virtual Status Insert(const Slice& key, ObjectPtr obj, + const CacheItemHelper* helper, size_t charge, + Handle** handle = nullptr, + Priority priority = Priority::LOW) = 0; + + // Similar to Insert, but used for creating cache entries that cannot + // be found with Lookup, such as for memory charging purposes. The + // key is needed for cache sharding purposes. + // * If allow_uncharged==true or strict_capacity_limit=false, the operation + // always succeeds and returns a valid Handle. + // * If strict_capacity_limit=true and the requested charge cannot be freed + // up in the cache, then + // * If allow_uncharged==true, it's created anyway (GetCharge() == 0). + // * If allow_uncharged==false, returns nullptr to indicate failure. + virtual Handle* CreateStandalone(const Slice& key, ObjectPtr obj, + const CacheItemHelper* helper, size_t charge, + bool allow_uncharged) = 0; + + // Lookup the key, returning nullptr if not found. If found, returns + // a handle to the mapping that must eventually be passed to Release(). + // + // If a non-nullptr helper argument is provided with a non-nullptr + // create_cb, and a secondary cache is configured, then the secondary + // cache is also queried if lookup in the primary cache fails. If found + // in secondary cache, the provided create_db and create_context are + // used to promote the entry to an object in the primary cache. + // In that case, the helper may be saved and used later when the object + // is evicted, so as usual, the pointed-to helper must outlive the cache. + virtual Handle* Lookup(const Slice& key, + const CacheItemHelper* helper = nullptr, + CreateContext* create_context = nullptr, + Priority priority = Priority::LOW, + Statistics* stats = nullptr) = 0; + + // Convenience wrapper when secondary cache not supported + inline Handle* BasicLookup(const Slice& key, Statistics* stats) { + return Lookup(key, nullptr, nullptr, Priority::LOW, stats); + } + + // Increments the reference count for the handle if it refers to an entry in + // the cache. Returns true if refcount was incremented; otherwise, returns + // false. + // REQUIRES: handle must have been returned by a method on *this. + virtual bool Ref(Handle* handle) = 0; + + /** + * Release a mapping returned by a previous Lookup(). A released entry might + * still remain in cache in case it is later looked up by others. If + * erase_if_last_ref is set then it also erases it from the cache if there is + * no other reference to it. Erasing it should call the deleter function that + * was provided when the entry was inserted. + * + * Returns true if the entry was also erased. + */ + // REQUIRES: handle must not have been released yet. + // REQUIRES: handle must have been returned by a method on *this. + virtual bool Release(Handle* handle, bool erase_if_last_ref = false) = 0; + + // Return the object assiciated with a handle returned by a successful + // Lookup(). For historical reasons, this is also known at the "value" + // associated with the key. + // REQUIRES: handle must not have been released yet. + // REQUIRES: handle must have been returned by a method on *this. + virtual ObjectPtr Value(Handle* handle) = 0; + + // If the cache contains the entry for the key, erase it. Note that the + // underlying entry will be kept around until all existing handles + // to it have been released. + virtual void Erase(const Slice& key) = 0; + // Return a new numeric id. May be used by multiple clients who are + // sharding the same cache to partition the key space. Typically the + // client will allocate a new id at startup and prepend the id to + // its cache keys. + virtual uint64_t NewId() = 0; + + // sets the maximum configured capacity of the cache. When the new + // capacity is less than the old capacity and the existing usage is + // greater than new capacity, the implementation will do its best job to + // purge the released entries from the cache in order to lower the usage + virtual void SetCapacity(size_t capacity) = 0; + + // Set whether to return error on insertion when cache reaches its full + // capacity. + virtual void SetStrictCapacityLimit(bool strict_capacity_limit) = 0; + + // Get the flag whether to return error on insertion when cache reaches its + // full capacity. + virtual bool HasStrictCapacityLimit() const = 0; + + // Returns the maximum configured capacity of the cache + virtual size_t GetCapacity() const = 0; + + // Returns the memory size for the entries residing in the cache. + virtual size_t GetUsage() const = 0; + + // Returns the number of entries currently tracked in the table. SIZE_MAX + // means "not supported." This is used for inspecting the load factor, along + // with GetTableAddressCount(). + virtual size_t GetOccupancyCount() const { return SIZE_MAX; } + + // Returns the number of ways the hash function is divided for addressing + // entries. Zero means "not supported." This is used for inspecting the load + // factor, along with GetOccupancyCount(). + virtual size_t GetTableAddressCount() const { return 0; } + + // Returns the memory size for a specific entry in the cache. + virtual size_t GetUsage(Handle* handle) const = 0; + + // Returns the memory size for the entries in use by the system + virtual size_t GetPinnedUsage() const = 0; + + // Returns the charge for the specific entry in the cache. + virtual size_t GetCharge(Handle* handle) const = 0; + + // Returns the helper for the specified entry. + virtual const CacheItemHelper* GetCacheItemHelper(Handle* handle) const = 0; + + // Call this on shutdown if you want to speed it up. Cache will disown + // any underlying data and will not free it on delete. This call will leak + // memory - call this only if you're shutting down the process. + // Any attempts of using cache after this call will fail terribly. + // Always delete the DB object before calling this method! + virtual void DisownData() { + // default implementation is noop + } + + struct ApplyToAllEntriesOptions { + // If the Cache uses locks, setting `average_entries_per_lock` to + // a higher value suggests iterating over more entries each time a lock + // is acquired, likely reducing the time for ApplyToAllEntries but + // increasing latency for concurrent users of the Cache. Setting + // `average_entries_per_lock` to a smaller value could be helpful if + // callback is relatively expensive, such as using large data structures. + size_t average_entries_per_lock = 256; + }; + + // Apply a callback to all entries in the cache. The Cache must ensure + // thread safety but does not guarantee that a consistent snapshot of all + // entries is iterated over if other threads are operating on the Cache + // also. + virtual void ApplyToAllEntries( + const std::function& callback, + const ApplyToAllEntriesOptions& opts) = 0; + + // Remove all entries. + // Prerequisite: no entry is referenced. + virtual void EraseUnRefEntries() = 0; + + virtual std::string GetPrintableOptions() const { return ""; } + + // Check for any warnings or errors in the operation of the cache and + // report them to the logger. This is intended only to be called + // periodically so does not need to be very efficient. (Obscure calling + // conventions for Logger inherited from env.h) + virtual void ReportProblems( + const std::shared_ptr& /*info_log*/) const {} + + MemoryAllocator* memory_allocator() const { return memory_allocator_.get(); } + + // See ShardedCacheOptions::hash_seed + virtual uint32_t GetHashSeed() const { return 0; } + + // EXPERIMENTAL + // The following APIs are experimental and might change in the future. + + // Release a mapping returned by a previous Lookup(). The "useful" + // parameter specifies whether the data was actually used or not, + // which may be used by the cache implementation to decide whether + // to consider it as a hit for retention purposes. As noted elsewhere, + // "pending" handles require Wait()/WaitAll() before Release(). + virtual bool Release(Handle* handle, bool /*useful*/, + bool erase_if_last_ref) { + return Release(handle, erase_if_last_ref); + } + + // A temporary handle structure for managing async lookups, which callers + // of AsyncLookup() can allocate on the call stack for efficiency. + // An AsyncLookupHandle should not be used concurrently across threads. + struct AsyncLookupHandle { + // Inputs, populated by caller: + // NOTE: at least in case of stacked secondary caches, the underlying + // key buffer must last until handle is completely waited on. + Slice key; + const CacheItemHelper* helper = nullptr; + CreateContext* create_context = nullptr; + Priority priority = Priority::LOW; + Statistics* stats = nullptr; + + AsyncLookupHandle() {} + AsyncLookupHandle(const Slice& _key, const CacheItemHelper* _helper, + CreateContext* _create_context, + Priority _priority = Priority::LOW, + Statistics* _stats = nullptr) + : key(_key), + helper(_helper), + create_context(_create_context), + priority(_priority), + stats(_stats) {} + + // AsyncLookupHandle should only be destroyed when no longer pending + ~AsyncLookupHandle() { assert(!IsPending()); } + + // No copies or moves (StartAsyncLookup may save a pointer to this) + AsyncLookupHandle(const AsyncLookupHandle&) = delete; + AsyncLookupHandle operator=(const AsyncLookupHandle&) = delete; + AsyncLookupHandle(AsyncLookupHandle&&) = delete; + AsyncLookupHandle operator=(AsyncLookupHandle&&) = delete; + + // Determines if the handle returned by Lookup() can give a value without + // blocking, though Wait()/WaitAll() might be required to publish it to + // Value(). See secondary cache compatible Lookup() above for details. + // This call is not thread safe on "pending" handles. + // WART/TODO with stacked secondaries: might indicate ready when one + // result is ready (a miss) but the next lookup will block. + bool IsReady(); + + // Returns true if Wait/WaitAll is required before calling Result(). + bool IsPending(); + + // Returns a Lookup()-like result if this AsyncHandle is not pending. + // (Undefined behavior on a pending AsyncHandle.) Like Lookup(), the + // caller is responsible for eventually Release()ing a non-nullptr + // Handle* result. + Handle* Result(); + + // Implementation details, for RocksDB internal use only + Handle* result_handle = nullptr; + SecondaryCacheResultHandle* pending_handle = nullptr; + SecondaryCache* pending_cache = nullptr; + bool found_dummy_entry = false; + bool kept_in_sec_cache = false; + }; + + // Starts a potentially asynchronous Lookup(), based on the populated + // "input" fields of the async_handle. The caller is responsible for + // keeping the AsyncLookupHandle and the key it references alive through + // WaitAll(), and the AsyncLookupHandle alive through + // AsyncLookupHandle::Result(). WaitAll() can only be skipped if + // AsyncLookupHandle::IsPending() is already false after StartAsyncLookup. + // Calling AsyncLookupHandle::Result() is essentially required so that + // Release() can be called on non-nullptr Handle result. Wait() is a + // concise version of WaitAll()+Result() on a single handle. After an + // AsyncLookupHandle has completed this cycle, its input fields can be + // updated and re-used for another StartAsyncLookup. + // + // Handle is thread-safe while AsyncLookupHandle is not thread-safe. + // + // Default implementation is appropriate for Caches without + // true asynchronous support: defers to synchronous Lookup(). + // (AsyncLookupHandles will only get into the "pending" state with + // SecondaryCache configured.) + virtual void StartAsyncLookup(AsyncLookupHandle& async_handle); + + // A convenient wrapper around WaitAll() and AsyncLookupHandle::Result() + // for a single async handle. See StartAsyncLookup(). + Handle* Wait(AsyncLookupHandle& async_handle); + + // Wait for an array of async handles to get results, so that none are left + // in the "pending" state. Not thread safe. See StartAsyncLookup(). + // Default implementation is appropriate for Caches without true + // asynchronous support: asserts that all handles are not pending (or not + // expected to be handled by this cache, in case of wrapped/stacked + // WaitAlls()). + virtual void WaitAll(AsyncLookupHandle* /*async_handles*/, size_t /*count*/); + + // For a function called on cache entries about to be evicted. The function + // returns `true` if it has taken ownership of the Value (object), or + // `false` if the cache should destroy it as usual. Regardless, Ref() and + // Release() cannot be called on this Handle that is poised for eviction. + using EvictionCallback = std::function; + // Sets an eviction callback for this Cache. Not thread safe and only + // supports being set once, so should only be used during initialization + // or destruction, guaranteed before or after any thread-shared operations. + void SetEvictionCallback(EvictionCallback&& fn); + + protected: + std::shared_ptr memory_allocator_; + EvictionCallback eviction_callback_; +}; + +// A wrapper around Cache that can easily be extended with instrumentation, +// etc. +class CacheWrapper : public Cache { + public: + explicit CacheWrapper(std::shared_ptr target) + : target_(std::move(target)) {} + + // Only function that derived class must provide + // const char* Name() const override { ... } + + Status Insert(const Slice& key, ObjectPtr value, + const CacheItemHelper* helper, size_t charge, + Handle** handle = nullptr, + Priority priority = Priority::LOW) override { + return target_->Insert(key, value, helper, charge, handle, priority); + } + + Handle* CreateStandalone(const Slice& key, ObjectPtr obj, + const CacheItemHelper* helper, size_t charge, + bool allow_uncharged) override { + return target_->CreateStandalone(key, obj, helper, charge, allow_uncharged); + } + + Handle* Lookup(const Slice& key, const CacheItemHelper* helper, + CreateContext* create_context, + Priority priority = Priority::LOW, + Statistics* stats = nullptr) override { + return target_->Lookup(key, helper, create_context, priority, stats); + } + + bool Ref(Handle* handle) override { return target_->Ref(handle); } + + using Cache::Release; + bool Release(Handle* handle, bool erase_if_last_ref = false) override { + return target_->Release(handle, erase_if_last_ref); + } + + ObjectPtr Value(Handle* handle) override { return target_->Value(handle); } + + void Erase(const Slice& key) override { target_->Erase(key); } + uint64_t NewId() override { return target_->NewId(); } + + void SetCapacity(size_t capacity) override { target_->SetCapacity(capacity); } + + void SetStrictCapacityLimit(bool strict_capacity_limit) override { + target_->SetStrictCapacityLimit(strict_capacity_limit); + } + + bool HasStrictCapacityLimit() const override { + return target_->HasStrictCapacityLimit(); + } + + size_t GetCapacity() const override { return target_->GetCapacity(); } + + size_t GetUsage() const override { return target_->GetUsage(); } + + size_t GetUsage(Handle* handle) const override { + return target_->GetUsage(handle); + } + + size_t GetPinnedUsage() const override { return target_->GetPinnedUsage(); } + + size_t GetCharge(Handle* handle) const override { + return target_->GetCharge(handle); + } + + const CacheItemHelper* GetCacheItemHelper(Handle* handle) const override { + return target_->GetCacheItemHelper(handle); + } + + void ApplyToAllEntries( + const std::function& callback, + const ApplyToAllEntriesOptions& opts) override { + target_->ApplyToAllEntries(callback, opts); + } + + void EraseUnRefEntries() override { target_->EraseUnRefEntries(); } + + void StartAsyncLookup(AsyncLookupHandle& async_handle) override { + target_->StartAsyncLookup(async_handle); + } + + void WaitAll(AsyncLookupHandle* async_handles, size_t count) override { + target_->WaitAll(async_handles, count); + } + + protected: + std::shared_ptr target_; +}; + +// Useful for cache entries requiring no clean-up, such as for cache +// reservations +extern const Cache::CacheItemHelper kNoopCacheItemHelper; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/advanced_options.h b/librocksdb-sys/rocksdb/include/rocksdb/advanced_options.h new file mode 100644 index 0000000..ab39497 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/advanced_options.h @@ -0,0 +1,1212 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include + +#include "rocksdb/cache.h" +#include "rocksdb/compression_type.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/universal_compaction.h" + +namespace ROCKSDB_NAMESPACE { + +class Slice; +class SliceTransform; +class TablePropertiesCollectorFactory; +class TableFactory; +struct Options; + +enum CompactionStyle : char { + // level based compaction style + kCompactionStyleLevel = 0x0, + // Universal compaction style + kCompactionStyleUniversal = 0x1, + // FIFO compaction style + kCompactionStyleFIFO = 0x2, + // Disable background compaction. Compaction jobs are submitted + // via CompactFiles(). + kCompactionStyleNone = 0x3, +}; + +// In Level-based compaction, it Determines which file from a level to be +// picked to merge to the next level. We suggest people try +// kMinOverlappingRatio first when you tune your database. +enum CompactionPri : char { + // Slightly prioritize larger files by size compensated by #deletes + kByCompensatedSize = 0x0, + // First compact files whose data's latest update time is oldest. + // Try this if you only update some hot keys in small ranges. + kOldestLargestSeqFirst = 0x1, + // First compact files whose range hasn't been compacted to the next level + // for the longest. If your updates are random across the key space, + // write amplification is slightly better with this option. + kOldestSmallestSeqFirst = 0x2, + // First compact files whose ratio between overlapping size in next level + // and its size is the smallest. It in many cases can optimize write + // amplification. + kMinOverlappingRatio = 0x3, + // Keeps a cursor(s) of the successor of the file (key range) was/were + // compacted before, and always picks the next files (key range) in that + // level. The file picking process will cycle through all the files in a + // round-robin manner. + kRoundRobin = 0x4, +}; + +// Compression options for different compression algorithms like Zlib +struct CompressionOptions { + // ==> BEGIN options that can be set by deprecated configuration syntax, <== + // ==> e.g. compression_opts=5:6:7:8:9:10:true:11:false <== + // ==> Please use compression_opts={level=6;strategy=7;} form instead. <== + + // RocksDB's generic default compression level. Internally it'll be translated + // to the default compression level specific to the library being used (see + // comment above `ColumnFamilyOptions::compression`). + // + // The default value is the max 16-bit int as it'll be written out in OPTIONS + // file, which should be portable. + static constexpr int kDefaultCompressionLevel = 32767; + + // zlib only: windowBits parameter. See https://www.zlib.net/manual.html + int window_bits = -14; + + // Compression "level" applicable to zstd, zlib, LZ4. Except for + // kDefaultCompressionLevel (see above), the meaning of each value depends + // on the compression algorithm. + int level = kDefaultCompressionLevel; + + // zlib only: strategy parameter. See https://www.zlib.net/manual.html + int strategy = 0; + + // Maximum size of dictionaries used to prime the compression library. + // Enabling dictionary can improve compression ratios when there are + // repetitions across data blocks. + // + // The dictionary is created by sampling the SST file data. If + // `zstd_max_train_bytes` is nonzero, the samples are passed through zstd's + // dictionary generator (see comments for option `use_zstd_dict_trainer` for + // detail on dictionary generator). If `zstd_max_train_bytes` is zero, the + // random samples are used directly as the dictionary. + // + // When compression dictionary is disabled, we compress and write each block + // before buffering data for the next one. When compression dictionary is + // enabled, we buffer SST file data in-memory so we can sample it, as data + // can only be compressed and written after the dictionary has been finalized. + // + // The amount of data buffered can be limited by `max_dict_buffer_bytes`. This + // buffered memory is charged to the block cache when there is a block cache. + // If block cache insertion fails with `Status::MemoryLimit` (i.e., it is + // full), we finalize the dictionary with whatever data we have and then stop + // buffering. + uint32_t max_dict_bytes = 0; + + // Maximum size of training data passed to zstd's dictionary trainer. Using + // zstd's dictionary trainer can achieve even better compression ratio + // improvements than using `max_dict_bytes` alone. + // + // The training data will be used to generate a dictionary of max_dict_bytes. + uint32_t zstd_max_train_bytes = 0; + + // Number of threads for parallel compression. + // Parallel compression is enabled only if threads > 1. + // THE FEATURE IS STILL EXPERIMENTAL + // + // This option is valid only when BlockBasedTable is used. + // + // When parallel compression is enabled, SST size file sizes might be + // more inflated compared to the target size, because more data of unknown + // compressed size is in flight when compression is parallelized. To be + // reasonably accurate, this inflation is also estimated by using historical + // compression ratio and current bytes inflight. + uint32_t parallel_threads = 1; + + // When the compression options are set by the user, it will be set to "true". + // For bottommost_compression_opts, to enable it, user must set enabled=true. + // Otherwise, bottommost compression will use compression_opts as default + // compression options. + // + // For compression_opts, if compression_opts.enabled=false, it is still + // used as compression options for compression process. + bool enabled = false; + + // Limit on data buffering when gathering samples to build a dictionary. Zero + // means no limit. When dictionary is disabled (`max_dict_bytes == 0`), + // enabling this limit (`max_dict_buffer_bytes != 0`) has no effect. + // + // In compaction, the buffering is limited to the target file size (see + // `target_file_size_base` and `target_file_size_multiplier`) even if this + // setting permits more buffering. Since we cannot determine where the file + // should be cut until data blocks are compressed with dictionary, buffering + // more than the target file size could lead to selecting samples that belong + // to a later output SST. + // + // Limiting too strictly may harm dictionary effectiveness since it forces + // RocksDB to pick samples from the initial portion of the output SST, which + // may not be representative of the whole file. Configuring this limit below + // `zstd_max_train_bytes` (when enabled) can restrict how many samples we can + // pass to the dictionary trainer. Configuring it below `max_dict_bytes` can + // restrict the size of the final dictionary. + uint64_t max_dict_buffer_bytes = 0; + + // Use zstd trainer to generate dictionaries. When this option is set to true, + // zstd_max_train_bytes of training data sampled from max_dict_buffer_bytes + // buffered data will be passed to zstd dictionary trainer to generate a + // dictionary of size max_dict_bytes. + // + // When this option is false, zstd's API ZDICT_finalizeDictionary() will be + // called to generate dictionaries. zstd_max_train_bytes of training sampled + // data will be passed to this API. Using this API should save CPU time on + // dictionary training, but the compression ratio may not be as good as using + // a dictionary trainer. + bool use_zstd_dict_trainer = true; + + // ===> END options that can be set by deprecated configuration syntax <=== + // ===> Use compression_opts={level=6;strategy=7;} form for below opts <=== + + // Essentially specifies a minimum acceptable compression ratio. A block is + // stored uncompressed if the compressed block does not achieve this ratio, + // because the downstream cost of decompression is not considered worth such + // a small savings (if any). + // However, the ratio is specified in a way that is efficient for checking. + // An integer from 1 to 1024 indicates the maximum allowable compressed bytes + // per 1KB of input, so the minimum acceptable ratio is 1024.0 / this value. + // For example, for a minimum ratio of 1.5:1, set to 683. See SetMinRatio(). + // Default: abandon use of compression for a specific block or entry if + // compressed by less than 12.5% (minimum ratio of 1.143:1). + int max_compressed_bytes_per_kb = 1024 * 7 / 8; + + // A convenience function for setting max_compressed_bytes_per_kb based on a + // minimum acceptable compression ratio (uncompressed size over compressed + // size). + void SetMinRatio(double min_ratio) { + max_compressed_bytes_per_kb = static_cast(1024.0 / min_ratio + 0.5); + } +}; + +// Temperature of a file. Used to pass to FileSystem for a different +// placement and/or coding. +// Reserve some numbers in the middle, in case we need to insert new tier +// there. +enum class Temperature : uint8_t { + kUnknown = 0, + kHot = 0x04, + kWarm = 0x08, + kCold = 0x0C, + kLastTemperature, +}; + +struct FileTemperatureAge { + Temperature temperature = Temperature::kUnknown; + uint64_t age = 0; +}; + +struct CompactionOptionsFIFO { + // once the total sum of table files reaches this, we will delete the oldest + // table file + // Default: 1GB + uint64_t max_table_files_size; + + // If true, try to do compaction to compact smaller files into larger ones. + // Minimum files to compact follows options.level0_file_num_compaction_trigger + // and compaction won't trigger if average compact bytes per del file is + // larger than options.write_buffer_size. This is to protect large files + // from being compacted again. + // Default: false; + bool allow_compaction = false; + + // DEPRECATED + // When not 0, if the data in the file is older than this threshold, RocksDB + // will soon move the file to warm temperature. + uint64_t age_for_warm = 0; + + // EXPERIMENTAL + // Age (in seconds) threshold for different file temperatures. + // When not empty, each element specifies an age threshold `age` and a + // temperature such that if all the data in a file is older than `age`, + // RocksDB will compact the file to the specified `temperature`. + // + // Note: + // - Flushed files will always have temperature kUnknown. + // - Compaction output files will have temperature kUnknown by default, so + // only temperatures other than kUnknown needs to be specified. + // - The elements should be in increasing order with respect to `age` field. + // + // Dynamically changeable through SetOptions() API, e.g., + // SetOptions("compaction_options_fifo", + // "{file_temperature_age_thresholds={ + // {age=10;temperature=kWarm}:{age=20;temperature=kCold}}}") + // In this example, all files that are at least 20 seconds old will be + // compacted and output files will have temperature kCold. All files that are + // at least 10 seconds old but younger than 20 seconds will be compacted to + // files with temperature kWarm. + // + // Default: empty + std::vector file_temperature_age_thresholds{}; + + CompactionOptionsFIFO() : max_table_files_size(1 * 1024 * 1024 * 1024) {} + CompactionOptionsFIFO(uint64_t _max_table_files_size, bool _allow_compaction) + : max_table_files_size(_max_table_files_size), + allow_compaction(_allow_compaction) {} +}; + +// The control option of how the cache tiers will be used. Currently rocksdb +// support block cache (volatile tier), secondary cache (non-volatile tier). +// In the future, we may add more caching layers. +enum class CacheTier : uint8_t { + kVolatileTier = 0, + kNonVolatileBlockTier = 0x01, +}; + +enum UpdateStatus { // Return status For inplace update callback + UPDATE_FAILED = 0, // Nothing to update + UPDATED_INPLACE = 1, // Value updated inplace + UPDATED = 2, // No inplace update. Merged value set +}; + +enum class PrepopulateBlobCache : uint8_t { + kDisable = 0x0, // Disable prepopulate blob cache + kFlushOnly = 0x1, // Prepopulate blobs during flush only +}; + +struct AdvancedColumnFamilyOptions { + // The maximum number of write buffers that are built up in memory. + // The default and the minimum number is 2, so that when 1 write buffer + // is being flushed to storage, new writes can continue to the other + // write buffer. + // If max_write_buffer_number > 3, writing will be slowed down to + // options.delayed_write_rate if we are writing to the last write buffer + // allowed. + // + // Default: 2 + // + // Dynamically changeable through SetOptions() API + int max_write_buffer_number = 2; + + // The minimum number of write buffers that will be merged together + // before writing to storage. If set to 1, then + // all write buffers are flushed to L0 as individual files and this increases + // read amplification because a get request has to check in all of these + // files. Also, an in-memory merge may result in writing lesser + // data to storage if there are duplicate records in each of these + // individual write buffers. + // If atomic flush is enabled (options.atomic_flush == true), then this + // option will be sanitized to 1. + // Default: 1 + int min_write_buffer_number_to_merge = 1; + + // DEPRECATED + // The total maximum number of write buffers to maintain in memory including + // copies of buffers that have already been flushed. Unlike + // max_write_buffer_number, this parameter does not affect flushing. + // This parameter is being replaced by max_write_buffer_size_to_maintain. + // If both parameters are set to non-zero values, this parameter will be + // ignored. + int max_write_buffer_number_to_maintain = 0; + + // The target number of write history bytes to hold in memory. Write history + // comprises the latest write buffers (memtables). To reach the target, write + // buffers that were most recently flushed to SST files may be retained in + // memory. + // + // This controls the target amount of write history that will be available + // in memory for conflict checking when Transactions are used. + // + // This target may be undershot when the CF first opens and has not recovered + // or received enough writes to reach the target. After reaching the target + // once, it is guaranteed to never undershoot again. That guarantee is + // implemented by retaining flushed write buffers in-memory until the oldest + // one can be trimmed without dropping below the target. + // + // Examples with `max_write_buffer_size_to_maintain` set to 32MB: + // + // - One mutable memtable of 64MB, one unflushed immutable memtable of 64MB, + // and zero flushed immutable memtables. Nothing trimmable exists. + // - One mutable memtable of 16MB, zero unflushed immutable memtables, and + // one flushed immutable memtable of 64MB. Trimming is disallowed because + // dropping the earliest (only) flushed immutable memtable would result in + // write history of 16MB < 32MB. + // - One mutable memtable of 24MB, one unflushed immutable memtable of 16MB, + // and one flushed immutable memtable of 16MB. The earliest (only) flushed + // immutable memtable is trimmed because without it we still have + // 16MB + 24MB = 40MB > 32MB of write history. + // + // When using an OptimisticTransactionDB: + // If this value is too low, some transactions may fail at commit time due + // to not being able to determine whether there were any write conflicts. + // + // When using a TransactionDB: + // If Transaction::SetSnapshot is used, TransactionDB will read either + // in-memory write buffers or SST files to do write-conflict checking. + // Increasing this value can reduce the number of reads to SST files + // done for conflict detection. + // + // Setting this value to 0 will cause write buffers to be freed immediately + // after they are flushed. If this value is set to -1, + // 'max_write_buffer_number * write_buffer_size' will be used. + // + // Default: + // If using a TransactionDB/OptimisticTransactionDB, the default value will + // be set to the value of 'max_write_buffer_number * write_buffer_size' + // if it is not explicitly set by the user. Otherwise, the default is 0. + int64_t max_write_buffer_size_to_maintain = 0; + + // Allows thread-safe inplace updates. If this is true, there is no way to + // achieve point-in-time consistency using snapshot or iterator (assuming + // concurrent updates). Hence iterator and multi-get will return results + // which are not consistent as of any point-in-time. + // Backward iteration on memtables will not work either. + // If inplace_callback function is not set, + // Put(key, new_value) will update inplace the existing_value iff + // * key exists in current memtable + // * new sizeof(new_value) <= sizeof(existing_value) + // * existing_value for that key is a put i.e. kTypeValue + // If inplace_callback function is set, check doc for inplace_callback. + // Default: false. + bool inplace_update_support = false; + + // Number of locks used for inplace update + // Default: 10000, if inplace_update_support = true, else 0. + // + // Dynamically changeable through SetOptions() API + size_t inplace_update_num_locks = 10000; + + // [experimental] + // Used to activate or deactive the Mempurge feature (memtable garbage + // collection). (deactivated by default). At every flush, the total useful + // payload (total entries minus garbage entries) is estimated as a ratio + // [useful payload bytes]/[size of a memtable (in bytes)]. This ratio is then + // compared to this `threshold` value: + // - if ratio1.0 : aggressive mempurge. + // 0 < threshold < 1.0: mempurge triggered only for very low useful payload + // ratios. + // [experimental] + double experimental_mempurge_threshold = 0.0; + + // existing_value - pointer to previous value (from both memtable and sst). + // nullptr if key doesn't exist + // existing_value_size - pointer to size of existing_value). + // nullptr if key doesn't exist + // delta_value - Delta value to be merged with the existing_value. + // Stored in transaction logs. + // merged_value - Set when delta is applied on the previous value. + // + // Applicable only when inplace_update_support is true, + // this callback function is called at the time of updating the memtable + // as part of a Put operation, lets say Put(key, delta_value). It allows the + // 'delta_value' specified as part of the Put operation to be merged with + // an 'existing_value' of the key in the database. + // + // If the merged value is smaller in size that the 'existing_value', + // then this function can update the 'existing_value' buffer inplace and + // the corresponding 'existing_value'_size pointer, if it wishes to. + // The callback should return UpdateStatus::UPDATED_INPLACE. + // In this case. (In this case, the snapshot-semantics of the rocksdb + // Iterator is not atomic anymore). + // + // If the merged value is larger in size than the 'existing_value' or the + // application does not wish to modify the 'existing_value' buffer inplace, + // then the merged value should be returned via *merge_value. It is set by + // merging the 'existing_value' and the Put 'delta_value'. The callback should + // return UpdateStatus::UPDATED in this case. This merged value will be added + // to the memtable. + // + // If merging fails or the application does not wish to take any action, + // then the callback should return UpdateStatus::UPDATE_FAILED. + // + // Please remember that the original call from the application is Put(key, + // delta_value). So the transaction log (if enabled) will still contain (key, + // delta_value). The 'merged_value' is not stored in the transaction log. + // Hence the inplace_callback function should be consistent across db reopens. + // + // RocksDB callbacks are NOT exception-safe. A callback completing with an + // exception can lead to undefined behavior in RocksDB, including data loss, + // unreported corruption, deadlocks, and more. + // + // Default: nullptr + UpdateStatus (*inplace_callback)(char* existing_value, + uint32_t* existing_value_size, + Slice delta_value, + std::string* merged_value) = nullptr; + + // Should really be called `memtable_bloom_size_ratio`. Enables a dynamic + // Bloom filter in memtable to optimize many queries that must go beyond + // the memtable. The size in bytes of the filter is + // write_buffer_size * memtable_prefix_bloom_size_ratio. + // * If prefix_extractor is set, the filter includes prefixes. + // * If memtable_whole_key_filtering, the filter includes whole keys. + // * If both, the filter includes both. + // * If neither, the feature is disabled. + // + // If this value is larger than 0.25, it is sanitized to 0.25. + // + // Default: 0 (disabled) + // + // Dynamically changeable through SetOptions() API + double memtable_prefix_bloom_size_ratio = 0.0; + + // Enable whole key bloom filter in memtable. Note this will only take effect + // if memtable_prefix_bloom_size_ratio is not 0. Enabling whole key filtering + // can potentially reduce CPU usage for point-look-ups. + // + // Default: false (disabled) + // + // Dynamically changeable through SetOptions() API + bool memtable_whole_key_filtering = false; + + // Page size for huge page for the arena used by the memtable. If <=0, it + // won't allocate from huge page but from malloc. + // Users are responsible to reserve huge pages for it to be allocated. For + // example: + // sysctl -w vm.nr_hugepages=20 + // See linux doc Documentation/vm/hugetlbpage.txt + // If there isn't enough free huge page available, it will fall back to + // malloc. + // + // Dynamically changeable through SetOptions() API + size_t memtable_huge_page_size = 0; + + // If non-nullptr, memtable will use the specified function to extract + // prefixes for keys, and for each prefix maintain a hint of insert location + // to reduce CPU usage for inserting keys with the prefix. Keys out of + // domain of the prefix extractor will be insert without using hints. + // + // Currently only the default skiplist based memtable implements the feature. + // All other memtable implementation will ignore the option. It incurs ~250 + // additional bytes of memory overhead to store a hint for each prefix. + // Also concurrent writes (when allow_concurrent_memtable_write is true) will + // ignore the option. + // + // The option is best suited for workloads where keys will likely to insert + // to a location close the last inserted key with the same prefix. + // One example could be inserting keys of the form (prefix + timestamp), + // and keys of the same prefix always comes in with time order. Another + // example would be updating the same key over and over again, in which case + // the prefix can be the key itself. + // + // Default: nullptr (disabled) + std::shared_ptr + memtable_insert_with_hint_prefix_extractor = nullptr; + + // Control locality of bloom filter probes to improve CPU cache hit rate. + // This option now only applies to plaintable prefix bloom. This + // optimization is turned off when set to 0, and positive number to turn + // it on. + // Default: 0 + uint32_t bloom_locality = 0; + + // size of one block in arena memory allocation. + // If <= 0, a proper value is automatically calculated (usually 1/8 of + // writer_buffer_size, rounded up to a multiple of 4KB, or 1MB which ever is + // smaller). + // + // There are two additional restriction of the specified size: + // (1) size should be in the range of [4096, 2 << 30] and + // (2) be the multiple of the CPU word (which helps with the memory + // alignment). + // + // We'll automatically check and adjust the size number to make sure it + // conforms to the restrictions. + // + // Default: 0 + // + // Dynamically changeable through SetOptions() API + size_t arena_block_size = 0; + + // Different levels can have different compression policies. There + // are cases where most lower levels would like to use quick compression + // algorithms while the higher levels (which have more data) use + // compression algorithms that have better compression but could + // be slower. This array, if non-empty, should have an entry for + // each level of the database; these override the value specified in + // the previous field 'compression'. + // + // NOTICE if level_compaction_dynamic_level_bytes=true, + // compression_per_level[0] still determines L0, but other elements + // of the array are based on base level (the level L0 files are merged + // to), and may not match the level users see from info log for metadata. + // If L0 files are merged to level-n, then, for i>0, compression_per_level[i] + // determines compaction type for level n+i-1. + // For example, if we have three 5 levels, and we determine to merge L0 + // data to L4 (which means L1..L3 will be empty), then the new files go to + // L4 uses compression type compression_per_level[1]. + // If now L0 is merged to L2. Data goes to L2 will be compressed + // according to compression_per_level[1], L3 using compression_per_level[2] + // and L4 using compression_per_level[3]. Compaction for each level can + // change when data grows. + // + // NOTE: if the vector size is smaller than the level number, the undefined + // lower level uses the last option in the vector, for example, for 3 level + // LSM tree the following settings are the same: + // {kNoCompression, kSnappyCompression} + // {kNoCompression, kSnappyCompression, kSnappyCompression} + // + // Dynamically changeable through SetOptions() API + std::vector compression_per_level; + + // Number of levels for this database + int num_levels = 7; + + // Soft limit on number of level-0 files. We start slowing down writes at this + // point. A value <0 means that no writing slow down will be triggered by + // number of files in level-0. + // + // Default: 20 + // + // Dynamically changeable through SetOptions() API + int level0_slowdown_writes_trigger = 20; + + // Maximum number of level-0 files. We stop writes at this point. + // + // Default: 36 + // + // Dynamically changeable through SetOptions() API + int level0_stop_writes_trigger = 36; + + // Target file size for compaction. + // target_file_size_base is per-file size for level-1. + // Target file size for level L can be calculated by + // target_file_size_base * (target_file_size_multiplier ^ (L-1)) + // For example, if target_file_size_base is 2MB and + // target_file_size_multiplier is 10, then each file on level-1 will + // be 2MB, and each file on level 2 will be 20MB, + // and each file on level-3 will be 200MB. + // + // Default: 64MB. + // + // Dynamically changeable through SetOptions() API + uint64_t target_file_size_base = 64 * 1048576; + + // By default target_file_size_multiplier is 1, which means + // by default files in different levels will have similar size. + // + // Dynamically changeable through SetOptions() API + int target_file_size_multiplier = 1; + + // If true, RocksDB will pick target size of each level dynamically. + // We will pick a base level b >= 1. L0 will be directly merged into level b, + // instead of always into level 1. Level 1 to b-1 need to be empty. + // We try to pick b and its target size so that + // 1. target size is in the range of + // (max_bytes_for_level_base / max_bytes_for_level_multiplier, + // max_bytes_for_level_base] + // 2. target size of the last level (level num_levels-1) equals to the max + // size of a level in the LSM (typically the last level). + // At the same time max_bytes_for_level_multiplier is still satisfied. + // Note that max_bytes_for_level_multiplier_additional is ignored with this + // flag on. + // + // With this option on, from an empty DB, we make last level the base level, + // which means merging L0 data into the last level, until it exceeds + // max_bytes_for_level_base. And then we make the second last level to be + // base level, to start to merge L0 data to second last level, with its + // target size to be 1/max_bytes_for_level_multiplier of the last level's + // extra size. After the data accumulates more so that we need to move the + // base level to the third last one, and so on. + // + // For example, assume max_bytes_for_level_multiplier=10, num_levels=6, + // and max_bytes_for_level_base=10MB. + // Target sizes of level 1 to 5 starts with: + // [- - - - 10MB] + // with base level is level 5. Target sizes of level 1 to 4 are not applicable + // because they will not be used. + // Until the size of Level 5 grows to more than 10MB, say 11MB, we make + // base target to level 4 and now the targets looks like: + // [- - - 1.1MB 11MB] + // While data are accumulated, size targets are tuned based on actual data + // of level 5. When level 5 has 50MB of data, the target is like: + // [- - - 5MB 50MB] + // Until level 5's actual size is more than 100MB, say 101MB. Now if we keep + // level 4 to be the base level, its target size needs to be 10.1MB, which + // doesn't satisfy the target size range. So now we make level 3 the target + // size and the target sizes of the levels look like: + // [- - 1.01MB 10.1MB 101MB] + // In the same way, while level 5 further grows, all levels' targets grow, + // like + // [- - 5MB 50MB 500MB] + // Until level 5 exceeds 1000MB and becomes 1001MB, we make level 2 the + // base level and make levels' target sizes like this: + // [- 1.001MB 10.01MB 100.1MB 1001MB] + // and go on... + // + // By doing it, we give max_bytes_for_level_multiplier a priority against + // max_bytes_for_level_base, for a more predictable LSM tree shape. It is + // useful to limit worse case space amplification. + // If `allow_ingest_behind=true` or `preclude_last_level_data_seconds > 0`, + // then the last level is reserved, and we will start filling LSM from the + // second last level. + // + // With this option on, compaction is more adaptive to write traffic: + // Compaction priority will take into account estimated bytes to be compacted + // down to a level and favors compacting lower levels when there is a write + // traffic spike (and hence more compaction debt). Refer to + // https://github.com/facebook/rocksdb/wiki/Leveled-Compactio#option-level_compaction_dynamic_level_bytes-and-levels-target-size + // for more detailed description. See more implementation detail in: + // VersionStorageInfo::ComputeCompactionScore(). + // + // With this option on, unneeded levels will be drained automatically: + // Note that there may be excessive levels (where target level size is 0 when + // computed based on this feature) in the LSM. This can happen after a user + // migrates to turn this feature on or deletes a lot of data. This is + // especially likely when a user migrates from leveled compaction with a + // smaller multiplier or from universal compaction. RocksDB will gradually + // drain these unnecessary levels by compacting files down the LSM. Smaller + // number of levels should help to reduce read amplification. + // + // Migration to turn on this option: + // - Before RocksDB v8.2, users are expected to do a full manual compaction + // and then restart DB to turn on this option. + // - Since RocksDB v8.2, users can just restart DB with this option on, as + // long as num_levels is no smaller than number of non-empty levels in the + // LSM. Migration will be done automatically by RocksDB. See more in + // https://github.com/facebook/rocksdb/wiki/Leveled-Compaction#migrating-from-level_compaction_dynamic_level_bytesfalse-to-level_compaction_dynamic_level_bytestrue + // + // Default: true + bool level_compaction_dynamic_level_bytes = true; + + // Allows RocksDB to generate files that are not exactly the target_file_size + // only for the non-bottommost files. Which can reduce the write-amplification + // from compaction. The file size could be from 0 to 2x target_file_size. + // Once enabled, non-bottommost compaction will try to cut the files align + // with the next level file boundaries (grandparent level). + // + // Default: true + bool level_compaction_dynamic_file_size = true; + + // Default: 10. + // + // Dynamically changeable through SetOptions() API + double max_bytes_for_level_multiplier = 10; + + // Different max-size multipliers for different levels. + // These are multiplied by max_bytes_for_level_multiplier to arrive + // at the max-size of each level. + // This option only applies to leveled compaction with + // `level_compaction_dynamic_level_bytes = false`. + // + // Default: 1 + // + // Dynamically changeable through SetOptions() API + std::vector max_bytes_for_level_multiplier_additional = + std::vector(num_levels, 1); + + // We try to limit number of bytes in one compaction to be lower than this + // threshold. But it's not guaranteed. + // Value 0 will be sanitized. + // + // Default: target_file_size_base * 25 + // + // Dynamically changeable through SetOptions() API + uint64_t max_compaction_bytes = 0; + + // When setting up compaction input files, we ignore the + // `max_compaction_bytes` limit when pulling in input files that are entirely + // within output key range. + // + // Default: true + // + // Dynamically changeable through SetOptions() API + // We could remove this knob and always ignore the limit once it is proven + // safe. + bool ignore_max_compaction_bytes_for_input = true; + + // All writes will be slowed down to at least delayed_write_rate if estimated + // bytes needed to be compaction exceed this threshold. + // + // Default: 64GB + // + // Dynamically changeable through SetOptions() API + uint64_t soft_pending_compaction_bytes_limit = 64 * 1073741824ull; + + // All writes are stopped if estimated bytes needed to be compaction exceed + // this threshold. + // + // Default: 256GB + // + // Dynamically changeable through SetOptions() API + uint64_t hard_pending_compaction_bytes_limit = 256 * 1073741824ull; + + // The compaction style. Default: kCompactionStyleLevel + CompactionStyle compaction_style = kCompactionStyleLevel; + + // If level compaction_style = kCompactionStyleLevel, for each level, + // which files are prioritized to be picked to compact. + // Default: kMinOverlappingRatio + CompactionPri compaction_pri = kMinOverlappingRatio; + + // The options needed to support Universal Style compactions + // + // Dynamically changeable through SetOptions() API + // Dynamic change example: + // SetOptions("compaction_options_universal", "{size_ratio=2;}") + CompactionOptionsUniversal compaction_options_universal; + + // The options for FIFO compaction style + // + // Dynamically changeable through SetOptions() API + // Dynamic change example: + // SetOptions("compaction_options_fifo", "{max_table_files_size=100;}") + CompactionOptionsFIFO compaction_options_fifo; + + // An iteration->Next() sequentially skips over keys with the same + // user-key unless this option is set. This number specifies the number + // of keys (with the same userkey) that will be sequentially + // skipped before a reseek is issued. + // + // Default: 8 + // + // Dynamically changeable through SetOptions() API + uint64_t max_sequential_skip_in_iterations = 8; + + // This is a factory that provides MemTableRep objects. + // Default: a factory that provides a skip-list-based implementation of + // MemTableRep. + std::shared_ptr memtable_factory = + std::shared_ptr(new SkipListFactory); + + // Block-based table related options are moved to BlockBasedTableOptions. + // Related options that were originally here but now moved include: + // no_block_cache + // block_cache + // block_cache_compressed (removed) + // block_size + // block_size_deviation + // block_restart_interval + // filter_policy + // whole_key_filtering + // If you'd like to customize some of these options, you will need to + // use NewBlockBasedTableFactory() to construct a new table factory. + + // This option allows user to collect their own interested statistics of + // the tables. + // Default: empty vector -- no user-defined statistics collection will be + // performed. + using TablePropertiesCollectorFactories = + std::vector>; + TablePropertiesCollectorFactories table_properties_collector_factories; + + // Maximum number of successive merge operations on a key in the memtable. + // + // When a merge operation is added to the memtable and the maximum number of + // successive merges is reached, the value of the key will be calculated and + // inserted into the memtable instead of the merge operation. This will + // ensure that there are never more than max_successive_merges merge + // operations in the memtable. + // + // Default: 0 (disabled) + // + // Dynamically changeable through SetOptions() API + size_t max_successive_merges = 0; + + // This flag specifies that the implementation should optimize the filters + // mainly for cases where keys are found rather than also optimize for keys + // missed. This would be used in cases where the application knows that + // there are very few misses or the performance in the case of misses is not + // important. + // + // For now, this flag allows us to not store filters for the last level i.e + // the largest level which contains data of the LSM store. For keys which + // are hits, the filters in this level are not useful because we will search + // for the data anyway. NOTE: the filters in other levels are still useful + // even for key hit because they tell us whether to look in that level or go + // to the higher level. + // + // Default: false + bool optimize_filters_for_hits = false; + + // During flush or compaction, check whether keys inserted to output files + // are in order. + // + // Default: true + // + // Dynamically changeable through SetOptions() API + bool check_flush_compaction_key_order = true; + + // After writing every SST file, reopen it and read all the keys. + // Checks the hash of all of the keys and values written versus the + // keys in the file and signals a corruption if they do not match + // + // Default: false + // + // Dynamically changeable through SetOptions() API + bool paranoid_file_checks = false; + + // In debug mode, RocksDB runs consistency checks on the LSM every time the + // LSM changes (Flush, Compaction, AddFile). When this option is true, these + // checks are also enabled in release mode. These checks were historically + // disabled in release mode, but are now enabled by default for proactive + // corruption detection. The CPU overhead is negligible for normal mixed + // operations but can slow down saturated writing. See + // Options::DisableExtraChecks(). + // Default: true + bool force_consistency_checks = true; + + // Measure IO stats in compactions and flushes, if true. + // + // Default: false + // + // Dynamically changeable through SetOptions() API + bool report_bg_io_stats = false; + + // This option has different meanings for different compaction styles: + // + // Leveled: Non-bottom-level files with all keys older than TTL will go + // through the compaction process. This usually happens in a cascading + // way so that those entries will be compacted to bottommost level/file. + // The feature is used to remove stale entries that have been deleted or + // updated from the file system. + // + // FIFO: Files with all keys older than TTL will be deleted. TTL is only + // supported if option max_open_files is set to -1. + // + // Universal: users should only set the option `periodic_compaction_seconds` + // below instead. For backward compatibility, this option has the same + // meaning as `periodic_compaction_seconds`. See more in comments for + // `periodic_compaction_seconds` on the interaction between these two + // options. + // + // This option only supports block based table format for any compaction + // style. + // + // unit: seconds. Ex: 1 day = 1 * 24 * 60 * 60 + // 0 means disabling. + // UINT64_MAX - 1 (0xfffffffffffffffe) is special flag to allow RocksDB to + // pick default. + // + // Default: 30 days if using block based table. 0 (disable) otherwise. + // + // Dynamically changeable through SetOptions() API + // Note that dynamically changing this option only works for leveled and FIFO + // compaction. For universal compaction, dynamically changing this option has + // no effect, users should dynamically change `periodic_compaction_seconds` + // instead. + uint64_t ttl = 0xfffffffffffffffe; + + // This option has different meanings for different compaction styles: + // + // Leveled: files older than `periodic_compaction_seconds` will be picked up + // for compaction and will be re-written to the same level as they were + // before. + // + // FIFO: not supported. Setting this option has no effect for FIFO compaction. + // + // Universal: when there are files older than `periodic_compaction_seconds`, + // rocksdb will try to do as large a compaction as possible including the + // last level. Such compaction is only skipped if only last level is to + // be compacted and no file in last level is older than + // `periodic_compaction_seconds`. See more in + // UniversalCompactionBuilder::PickPeriodicCompaction(). + // For backward compatibility, the effective value of this option takes + // into account the value of option `ttl`. The logic is as follows: + // - both options are set to 30 days if they have the default value. + // - if both options are zero, zero is picked. Otherwise, we take the min + // value among non-zero options values (i.e. takes the stricter limit). + // + // One main use of the feature is to make sure a file goes through compaction + // filters periodically. Users can also use the feature to clear up SST + // files using old format. + // + // A file's age is computed by looking at file_creation_time or creation_time + // table properties in order, if they have valid non-zero values; if not, the + // age is based on the file's last modified time (given by the underlying + // Env). + // + // This option only supports block based table format for any compaction + // style. + // + // unit: seconds. Ex: 7 days = 7 * 24 * 60 * 60 + // + // Values: + // 0: Turn off Periodic compactions. + // UINT64_MAX - 1 (0xfffffffffffffffe) is special flag to allow RocksDB to + // pick default. + // + // Default: 30 days if using block based table format + compaction filter + + // leveled compaction or block based table format + universal compaction. + // 0 (disabled) otherwise. + // + // Dynamically changeable through SetOptions() API + uint64_t periodic_compaction_seconds = 0xfffffffffffffffe; + + // If this option is set then 1 in N blocks are compressed + // using a fast (lz4) and slow (zstd) compression algorithm. + // The compressibility is reported as stats and the stored + // data is left uncompressed (unless compression is also requested). + uint64_t sample_for_compression = 0; + + // EXPERIMENTAL + // The feature is still in development and is incomplete. + // If this option is set, when creating the last level files, pass this + // temperature to FileSystem used. Should be no-op for default FileSystem + // and users need to plug in their own FileSystem to take advantage of it. + // + // Note: the feature is changed from `bottommost_temperature` to + // `last_level_temperature` which now only apply for the last level files. + // The option name `bottommost_temperature` is kept only for migration, the + // behavior is the same as `last_level_temperature`. Please stop using + // `bottommost_temperature` and will be removed in next release. + // + // Dynamically changeable through the SetOptions() API + Temperature bottommost_temperature = Temperature::kUnknown; + Temperature last_level_temperature = Temperature::kUnknown; + + // EXPERIMENTAL + // The feature is still in development and is incomplete. + // If this option is set, when data insert time is within this time range, it + // will be precluded from the last level. + // 0 means no key will be precluded from the last level. + // + // Note: when enabled, universal size amplification (controlled by option + // `compaction_options_universal.max_size_amplification_percent`) calculation + // will exclude the last level. As the feature is designed for tiered storage + // and a typical setting is the last level is cold tier which is likely not + // size constrained, the size amp is going to be only for non-last levels. + // + // Default: 0 (disable the feature) + // + // Not dynamically changeable, change it requires db restart. + uint64_t preclude_last_level_data_seconds = 0; + + // EXPERIMENTAL + // If this option is set, it will preserve the internal time information about + // the data until it's older than the specified time here. + // Internally the time information is a map between sequence number and time, + // which is the same as `preclude_last_level_data_seconds`. But it won't + // preclude the data from the last level and the data in the last level won't + // have the sequence number zeroed out. + // Internally, rocksdb would sample the sequence number to time pair and store + // that in SST property "rocksdb.seqno.time.map". The information is currently + // only used for tiered storage compaction (option + // `preclude_last_level_data_seconds`). + // + // Note: if both `preclude_last_level_data_seconds` and this option is set, it + // will preserve the max time of the 2 options and compaction still preclude + // the data based on `preclude_last_level_data_seconds`. + // The higher the preserve_time is, the less the sampling frequency will be ( + // which means less accuracy of the time estimation). + // + // Default: 0 (disable the feature) + // + // Not dynamically changeable, change it requires db restart. + uint64_t preserve_internal_time_seconds = 0; + + // When set, large values (blobs) are written to separate blob files, and + // only pointers to them are stored in SST files. This can reduce write + // amplification for large-value use cases at the cost of introducing a level + // of indirection for reads. See also the options min_blob_size, + // blob_file_size, blob_compression_type, enable_blob_garbage_collection, + // blob_garbage_collection_age_cutoff, + // blob_garbage_collection_force_threshold, and blob_compaction_readahead_size + // below. + // + // Default: false + // + // Dynamically changeable through the SetOptions() API + bool enable_blob_files = false; + + // The size of the smallest value to be stored separately in a blob file. + // Values which have an uncompressed size smaller than this threshold are + // stored alongside the keys in SST files in the usual fashion. A value of + // zero for this option means that all values are stored in blob files. Note + // that enable_blob_files has to be set in order for this option to have any + // effect. + // + // Default: 0 + // + // Dynamically changeable through the SetOptions() API + uint64_t min_blob_size = 0; + + // The size limit for blob files. When writing blob files, a new file is + // opened once this limit is reached. Note that enable_blob_files has to be + // set in order for this option to have any effect. + // + // Default: 256 MB + // + // Dynamically changeable through the SetOptions() API + uint64_t blob_file_size = 1ULL << 28; + + // The compression algorithm to use for large values stored in blob files. + // Note that enable_blob_files has to be set in order for this option to have + // any effect. + // + // Default: no compression + // + // Dynamically changeable through the SetOptions() API + CompressionType blob_compression_type = kNoCompression; + + // Enables garbage collection of blobs. Blob GC is performed as part of + // compaction. Valid blobs residing in blob files older than a cutoff get + // relocated to new files as they are encountered during compaction, which + // makes it possible to clean up blob files once they contain nothing but + // obsolete/garbage blobs. See also blob_garbage_collection_age_cutoff and + // blob_garbage_collection_force_threshold below. + // + // Default: false + // + // Dynamically changeable through the SetOptions() API + bool enable_blob_garbage_collection = false; + + // The cutoff in terms of blob file age for garbage collection. Blobs in + // the oldest N blob files will be relocated when encountered during + // compaction, where N = garbage_collection_cutoff * number_of_blob_files. + // Note that enable_blob_garbage_collection has to be set in order for this + // option to have any effect. + // + // Default: 0.25 + // + // Dynamically changeable through the SetOptions() API + double blob_garbage_collection_age_cutoff = 0.25; + + // If the ratio of garbage in the oldest blob files exceeds this threshold, + // targeted compactions are scheduled in order to force garbage collecting + // the blob files in question, assuming they are all eligible based on the + // value of blob_garbage_collection_age_cutoff above. This option is + // currently only supported with leveled compactions. + // Note that enable_blob_garbage_collection has to be set in order for this + // option to have any effect. + // + // Default: 1.0 + // + // Dynamically changeable through the SetOptions() API + double blob_garbage_collection_force_threshold = 1.0; + + // Compaction readahead for blob files. + // + // Default: 0 + // + // Dynamically changeable through the SetOptions() API + uint64_t blob_compaction_readahead_size = 0; + + // Enable blob files starting from a certain LSM tree level. + // + // For certain use cases that have a mix of short-lived and long-lived values, + // it might make sense to support extracting large values only during + // compactions whose output level is greater than or equal to a specified LSM + // tree level (e.g. compactions into L1/L2/... or above). This could reduce + // the space amplification caused by large values that are turned into garbage + // shortly after being written at the price of some write amplification + // incurred by long-lived values whose extraction to blob files is delayed. + // + // Default: 0 + // + // Dynamically changeable through the SetOptions() API + int blob_file_starting_level = 0; + + // The Cache object to use for blobs. Using a dedicated object for blobs and + // using the same object for the block and blob caches are both supported. In + // the latter case, note that blobs are less valuable from a caching + // perspective than SST blocks, and some cache implementations have + // configuration options that can be used to prioritize items accordingly (see + // Cache::Priority and LRUCacheOptions::{high,low}_pri_pool_ratio). + // + // Default: nullptr (disabled) + std::shared_ptr blob_cache = nullptr; + + // Enable/disable prepopulating the blob cache. When set to kFlushOnly, BlobDB + // will insert newly written blobs into the blob cache during flush. This can + // improve performance when reading back these blobs would otherwise be + // expensive (e.g. when using direct I/O or remote storage), or when the + // workload has a high temporal locality. + // + // Default: disabled + // + // Dynamically changeable through the SetOptions() API + PrepopulateBlobCache prepopulate_blob_cache = PrepopulateBlobCache::kDisable; + + // Enable memtable per key-value checksum protection. + // + // Each entry in memtable will be suffixed by a per key-value checksum. + // This options determines the size of such checksums. + // + // It is suggested to turn on write batch per key-value + // checksum protection together with this option, so that the checksum + // computation is done outside of writer threads (memtable kv checksum can be + // computed from write batch checksum) See + // WriteOptions::protection_bytes_per_key for more detail. + // + // Default: 0 (no protection) + // Supported values: 0, 1, 2, 4, 8. + uint32_t memtable_protection_bytes_per_key = 0; + + // UNDER CONSTRUCTION -- DO NOT USE + // When the user-defined timestamp feature is enabled, this flag controls + // whether the user-defined timestamps will be persisted. + // + // When it's false, the user-defined timestamps will be removed from the user + // keys when data is flushed from memtables to SST files. Other places that + // user keys can be persisted like file boundaries in file metadata and blob + // files go through a similar process. There are two major motivations + // for this flag: + // 1) backward compatibility: if the user later decides to + // disable the user-defined timestamp feature for the column family, these SST + // files can be handled by a user comparator that is not aware of user-defined + // timestamps. + // 2) enable user-defined timestamp feature for an existing column family + // while set this flag to be `false`: user keys in the newly generated SST + // files are of the same format as the existing SST files. + // + // Currently only user comparator that formats user-defined timesamps as + // uint64_t via using one of the RocksDB provided comparator + // `ComparatorWithU64TsImpl` are supported. + // + // When setting this flag to `false`, users should also call + // `DB::IncreaseFullHistoryTsLow` to set a cutoff timestamp for flush. RocksDB + // refrains from flushing a memtable with data still above + // the cutoff timestamp with best effort. If this cutoff timestamp is not set, + // flushing continues normally. + // NOTE: in order for the cutoff timestamp to work properly, users of this + // feature need to ensure to write to a column family with globally + // non-decreasing user-defined timestamps. + // + // Users can do user-defined + // multi-versioned read above the cutoff timestamp. When users try to read + // below the cutoff timestamp, an error will be returned. + // + // Note that if WAL is enabled, unlike SST files, user-defined timestamps are + // persisted to WAL even if this flag is set to `false`. The benefit of this + // is that user-defined timestamps can be recovered with the caveat that users + // should flush all memtables so there is no active WAL files before doing a + // downgrade. + // + // Note that setting this flag to false is not supported in combination with + // atomic flush, or concurrent memtable write enabled by + // `allow_concurrent_memtable_write`. + // + // Default: true (user-defined timestamps are persisted) + // Not dynamically changeable, change it requires db restart and + // only compatible changes are allowed. + bool persist_user_defined_timestamps = true; + + // Enable/disable per key-value checksum protection for in memory blocks. + // + // Checksum is constructed when a block is loaded into memory and verification + // is done for each key read from the block. This is useful for detecting + // in-memory data corruption. Note that this feature has a non-trivial + // negative impact on read performance. Different values of the + // option have similar performance impact, but different memory cost and + // corruption detection probability (e.g. 1 byte gives 255/256 chance for + // detecting a corruption). + // + // Default: 0 (no protection) + // Supported values: 0, 1, 2, 4, 8. + uint8_t block_protection_bytes_per_key = 0; + + // Create ColumnFamilyOptions with default values for all fields + AdvancedColumnFamilyOptions(); + // Create ColumnFamilyOptions from Options + explicit AdvancedColumnFamilyOptions(const Options& options); + + // ---------------- OPTIONS NOT SUPPORTED ANYMORE ---------------- +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/block_cache_trace_writer.h b/librocksdb-sys/rocksdb/include/rocksdb/block_cache_trace_writer.h new file mode 100644 index 0000000..18d2868 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/block_cache_trace_writer.h @@ -0,0 +1,149 @@ +// Copyright (c) 2022, Meta Platforms, Inc. and affiliates. All rights +// reserved. This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "rocksdb/options.h" +#include "rocksdb/system_clock.h" +#include "rocksdb/table_reader_caller.h" +#include "rocksdb/trace_reader_writer.h" +#include "rocksdb/trace_record.h" + +namespace ROCKSDB_NAMESPACE { +// A record for block cache lookups/inserts. This is passed by the table +// reader to the BlockCacheTraceWriter for every block cache op. +struct BlockCacheTraceRecord { + // Required fields for all accesses. + uint64_t access_timestamp = 0; + + // Info related to the block being looked up or inserted + // + // 1. The cache key for the block + std::string block_key; + + // 2. The type of block + TraceType block_type = TraceType::kTraceMax; + + // 3. Size of the block + uint64_t block_size = 0; + + // Info about the SST file the block is in + // + // 1. Column family ID + uint64_t cf_id = 0; + + // 2. Column family name + std::string cf_name; + + // 3. LSM level of the file + uint32_t level = 0; + + // 4. SST file number + uint64_t sst_fd_number = 0; + + // Info about the calling context + // + // 1. The higher level request triggering the block cache request + TableReaderCaller caller = TableReaderCaller::kMaxBlockCacheLookupCaller; + + // 2. Cache lookup hit/miss. Not relevant for inserts + bool is_cache_hit = false; + + // 3. Whether this request is a lookup + bool no_insert = false; + + // Get/MultiGet specific info + // + // 1. A unique ID for Get/MultiGet + uint64_t get_id = kReservedGetId; + + // 2. Whether the Get/MultiGet is from a user-specified snapshot + bool get_from_user_specified_snapshot = false; + + // 3. The target user key in the block + std::string referenced_key; + + // Required fields for data block and user Get/Multi-Get only. + // + // 1. Size of te useful data in the block + uint64_t referenced_data_size = 0; + + // 2. Only for MultiGet, number of keys from the batch found in the block + uint64_t num_keys_in_block = 0; + + // 3. Whether the key was found in the block or not (false positive) + bool referenced_key_exist_in_block = false; + + static const uint64_t kReservedGetId; + + BlockCacheTraceRecord() {} + + BlockCacheTraceRecord(uint64_t _access_timestamp, std::string _block_key, + TraceType _block_type, uint64_t _block_size, + uint64_t _cf_id, std::string _cf_name, uint32_t _level, + uint64_t _sst_fd_number, TableReaderCaller _caller, + bool _is_cache_hit, bool _no_insert, uint64_t _get_id, + bool _get_from_user_specified_snapshot = false, + std::string _referenced_key = "", + uint64_t _referenced_data_size = 0, + uint64_t _num_keys_in_block = 0, + bool _referenced_key_exist_in_block = false) + : access_timestamp(_access_timestamp), + block_key(_block_key), + block_type(_block_type), + block_size(_block_size), + cf_id(_cf_id), + cf_name(_cf_name), + level(_level), + sst_fd_number(_sst_fd_number), + caller(_caller), + is_cache_hit(_is_cache_hit), + no_insert(_no_insert), + get_id(_get_id), + get_from_user_specified_snapshot(_get_from_user_specified_snapshot), + referenced_key(_referenced_key), + referenced_data_size(_referenced_data_size), + num_keys_in_block(_num_keys_in_block), + referenced_key_exist_in_block(_referenced_key_exist_in_block) {} +}; + +// Options for tracing block cache accesses +struct BlockCacheTraceOptions { + // Specify trace sampling option, i.e. capture one per how many requests. + // Default to 1 (capture every request). + uint64_t sampling_frequency = 1; +}; + +// Options for the built-in implementation of BlockCacheTraceWriter +struct BlockCacheTraceWriterOptions { + uint64_t max_trace_file_size = uint64_t{64} * 1024 * 1024 * 1024; +}; + +// BlockCacheTraceWriter is an abstract class that captures all RocksDB block +// cache accesses. Every RocksDB operation is passed to WriteBlockAccess() +// with a BlockCacheTraceRecord. +class BlockCacheTraceWriter { + public: + virtual ~BlockCacheTraceWriter() {} + + // Pass Slice references to avoid copy. + virtual Status WriteBlockAccess(const BlockCacheTraceRecord& record, + const Slice& block_key, const Slice& cf_name, + const Slice& referenced_key) = 0; + + // Write a trace header at the beginning, typically on initiating a trace, + // with some metadata like a magic number and RocksDB version. + virtual Status WriteHeader() = 0; +}; + +// Allocate an instance of the built-in BlockCacheTraceWriter implementation, +// that traces all block cache accesses to a user-provided TraceWriter. Each +// access is traced to a file with a timestamp and type, followed by the +// payload. +std::unique_ptr NewBlockCacheTraceWriter( + SystemClock* clock, const BlockCacheTraceWriterOptions& trace_options, + std::unique_ptr&& trace_writer); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/c.h b/librocksdb-sys/rocksdb/include/rocksdb/c.h new file mode 100644 index 0000000..6636b59 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/c.h @@ -0,0 +1,2932 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +/* Copyright (c) 2011 The LevelDB Authors. All rights reserved. + Use of this source code is governed by a BSD-style license that can be + found in the LICENSE file. See the AUTHORS file for names of contributors. + + C bindings for rocksdb. May be useful as a stable ABI that can be + used by programs that keep rocksdb in a shared library, or for + a JNI api. + + Does not support: + . getters for the option types + . custom comparators that implement key shortening + . capturing post-write-snapshot + . custom iter, db, env, cache implementations using just the C bindings + + Some conventions: + + (1) We expose just opaque struct pointers and functions to clients. + This allows us to change internal representations without having to + recompile clients. + + (2) For simplicity, there is no equivalent to the Slice type. Instead, + the caller has to pass the pointer and length as separate + arguments. + + (3) Errors are represented by a null-terminated c string. NULL + means no error. All operations that can raise an error are passed + a "char** errptr" as the last argument. One of the following must + be true on entry: + *errptr == NULL + *errptr points to a malloc()ed null-terminated error message + On success, a leveldb routine leaves *errptr unchanged. + On failure, leveldb frees the old value of *errptr and + set *errptr to a malloc()ed error message. + + (4) Bools have the type unsigned char (0 == false; rest == true) + + (5) All of the pointer arguments must be non-NULL. +*/ + +#pragma once + +#ifdef _WIN32 +#ifdef ROCKSDB_DLL +#ifdef ROCKSDB_LIBRARY_EXPORTS +#define ROCKSDB_LIBRARY_API __declspec(dllexport) +#else +#define ROCKSDB_LIBRARY_API __declspec(dllimport) +#endif +#else +#define ROCKSDB_LIBRARY_API +#endif +#else +#define ROCKSDB_LIBRARY_API +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* Exported types */ + +typedef struct rocksdb_t rocksdb_t; +typedef struct rocksdb_backup_engine_t rocksdb_backup_engine_t; +typedef struct rocksdb_backup_engine_info_t rocksdb_backup_engine_info_t; +typedef struct rocksdb_backup_engine_options_t rocksdb_backup_engine_options_t; +typedef struct rocksdb_restore_options_t rocksdb_restore_options_t; +typedef struct rocksdb_memory_allocator_t rocksdb_memory_allocator_t; +typedef struct rocksdb_lru_cache_options_t rocksdb_lru_cache_options_t; +typedef struct rocksdb_hyper_clock_cache_options_t + rocksdb_hyper_clock_cache_options_t; +typedef struct rocksdb_cache_t rocksdb_cache_t; +typedef struct rocksdb_compactionfilter_t rocksdb_compactionfilter_t; +typedef struct rocksdb_compactionfiltercontext_t + rocksdb_compactionfiltercontext_t; +typedef struct rocksdb_compactionfilterfactory_t + rocksdb_compactionfilterfactory_t; +typedef struct rocksdb_comparator_t rocksdb_comparator_t; +typedef struct rocksdb_dbpath_t rocksdb_dbpath_t; +typedef struct rocksdb_env_t rocksdb_env_t; +typedef struct rocksdb_fifo_compaction_options_t + rocksdb_fifo_compaction_options_t; +typedef struct rocksdb_filelock_t rocksdb_filelock_t; +typedef struct rocksdb_filterpolicy_t rocksdb_filterpolicy_t; +typedef struct rocksdb_flushoptions_t rocksdb_flushoptions_t; +typedef struct rocksdb_iterator_t rocksdb_iterator_t; +typedef struct rocksdb_logger_t rocksdb_logger_t; +typedef struct rocksdb_mergeoperator_t rocksdb_mergeoperator_t; +typedef struct rocksdb_options_t rocksdb_options_t; +typedef struct rocksdb_compactoptions_t rocksdb_compactoptions_t; +typedef struct rocksdb_block_based_table_options_t + rocksdb_block_based_table_options_t; +typedef struct rocksdb_cuckoo_table_options_t rocksdb_cuckoo_table_options_t; +typedef struct rocksdb_randomfile_t rocksdb_randomfile_t; +typedef struct rocksdb_readoptions_t rocksdb_readoptions_t; +typedef struct rocksdb_seqfile_t rocksdb_seqfile_t; +typedef struct rocksdb_slicetransform_t rocksdb_slicetransform_t; +typedef struct rocksdb_snapshot_t rocksdb_snapshot_t; +typedef struct rocksdb_writablefile_t rocksdb_writablefile_t; +typedef struct rocksdb_writebatch_t rocksdb_writebatch_t; +typedef struct rocksdb_writebatch_wi_t rocksdb_writebatch_wi_t; +typedef struct rocksdb_writeoptions_t rocksdb_writeoptions_t; +typedef struct rocksdb_universal_compaction_options_t + rocksdb_universal_compaction_options_t; +typedef struct rocksdb_livefiles_t rocksdb_livefiles_t; +typedef struct rocksdb_column_family_handle_t rocksdb_column_family_handle_t; +typedef struct rocksdb_column_family_metadata_t + rocksdb_column_family_metadata_t; +typedef struct rocksdb_level_metadata_t rocksdb_level_metadata_t; +typedef struct rocksdb_sst_file_metadata_t rocksdb_sst_file_metadata_t; +typedef struct rocksdb_envoptions_t rocksdb_envoptions_t; +typedef struct rocksdb_ingestexternalfileoptions_t + rocksdb_ingestexternalfileoptions_t; +typedef struct rocksdb_sstfilewriter_t rocksdb_sstfilewriter_t; +typedef struct rocksdb_ratelimiter_t rocksdb_ratelimiter_t; +typedef struct rocksdb_perfcontext_t rocksdb_perfcontext_t; +typedef struct rocksdb_pinnableslice_t rocksdb_pinnableslice_t; +typedef struct rocksdb_transactiondb_options_t rocksdb_transactiondb_options_t; +typedef struct rocksdb_transactiondb_t rocksdb_transactiondb_t; +typedef struct rocksdb_transaction_options_t rocksdb_transaction_options_t; +typedef struct rocksdb_optimistictransactiondb_t + rocksdb_optimistictransactiondb_t; +typedef struct rocksdb_optimistictransaction_options_t + rocksdb_optimistictransaction_options_t; +typedef struct rocksdb_transaction_t rocksdb_transaction_t; +typedef struct rocksdb_checkpoint_t rocksdb_checkpoint_t; +typedef struct rocksdb_wal_iterator_t rocksdb_wal_iterator_t; +typedef struct rocksdb_wal_readoptions_t rocksdb_wal_readoptions_t; +typedef struct rocksdb_memory_consumers_t rocksdb_memory_consumers_t; +typedef struct rocksdb_memory_usage_t rocksdb_memory_usage_t; +typedef struct rocksdb_statistics_histogram_data_t + rocksdb_statistics_histogram_data_t; + +/* DB operations */ + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open( + const rocksdb_options_t* options, const char* name, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_with_ttl( + const rocksdb_options_t* options, const char* name, int ttl, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_for_read_only( + const rocksdb_options_t* options, const char* name, + unsigned char error_if_wal_file_exists, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_as_secondary( + const rocksdb_options_t* options, const char* name, + const char* secondary_path, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_backup_engine_t* rocksdb_backup_engine_open( + const rocksdb_options_t* options, const char* path, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_backup_engine_t* +rocksdb_backup_engine_open_opts(const rocksdb_backup_engine_options_t* options, + rocksdb_env_t* env, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_create_new_backup( + rocksdb_backup_engine_t* be, rocksdb_t* db, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_create_new_backup_flush( + rocksdb_backup_engine_t* be, rocksdb_t* db, + unsigned char flush_before_backup, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_purge_old_backups( + rocksdb_backup_engine_t* be, uint32_t num_backups_to_keep, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_restore_options_t* +rocksdb_restore_options_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_restore_options_destroy( + rocksdb_restore_options_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_restore_options_set_keep_log_files( + rocksdb_restore_options_t* opt, int v); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_verify_backup( + rocksdb_backup_engine_t* be, uint32_t backup_id, char** errptr); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_restore_db_from_latest_backup( + rocksdb_backup_engine_t* be, const char* db_dir, const char* wal_dir, + const rocksdb_restore_options_t* restore_options, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_restore_db_from_backup( + rocksdb_backup_engine_t* be, const char* db_dir, const char* wal_dir, + const rocksdb_restore_options_t* restore_options, const uint32_t backup_id, + char** errptr); + +extern ROCKSDB_LIBRARY_API const rocksdb_backup_engine_info_t* +rocksdb_backup_engine_get_backup_info(rocksdb_backup_engine_t* be); + +extern ROCKSDB_LIBRARY_API int rocksdb_backup_engine_info_count( + const rocksdb_backup_engine_info_t* info); + +extern ROCKSDB_LIBRARY_API int64_t rocksdb_backup_engine_info_timestamp( + const rocksdb_backup_engine_info_t* info, int index); + +extern ROCKSDB_LIBRARY_API uint32_t rocksdb_backup_engine_info_backup_id( + const rocksdb_backup_engine_info_t* info, int index); + +extern ROCKSDB_LIBRARY_API uint64_t rocksdb_backup_engine_info_size( + const rocksdb_backup_engine_info_t* info, int index); + +extern ROCKSDB_LIBRARY_API uint32_t rocksdb_backup_engine_info_number_files( + const rocksdb_backup_engine_info_t* info, int index); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_info_destroy( + const rocksdb_backup_engine_info_t* info); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_close( + rocksdb_backup_engine_t* be); + +extern ROCKSDB_LIBRARY_API void rocksdb_put_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, const char* ts, size_t tslen, const char* val, size_t vallen, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_put_cf_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* ts, size_t tslen, const char* val, size_t vallen, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, const char* ts, size_t tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_cf_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* ts, size_t tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_singledelete( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_singledelete_cf( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_singledelete_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, const char* ts, size_t tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_singledelete_cf_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* ts, size_t tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_increase_full_history_ts_low( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* ts_low, size_t ts_lowlen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_get_full_history_ts_low( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + size_t* ts_lowlen, char** errptr); + +/* BackupEngineOptions */ + +extern ROCKSDB_LIBRARY_API rocksdb_backup_engine_options_t* +rocksdb_backup_engine_options_create(const char* backup_dir); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_options_set_backup_dir( + rocksdb_backup_engine_options_t* options, const char* backup_dir); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_options_set_env( + rocksdb_backup_engine_options_t* options, rocksdb_env_t* env); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_share_table_files( + rocksdb_backup_engine_options_t* options, unsigned char val); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_backup_engine_options_get_share_table_files( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_options_set_sync( + rocksdb_backup_engine_options_t* options, unsigned char val); + +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_backup_engine_options_get_sync( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_destroy_old_data( + rocksdb_backup_engine_options_t* options, unsigned char val); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_backup_engine_options_get_destroy_old_data( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_backup_log_files( + rocksdb_backup_engine_options_t* options, unsigned char val); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_backup_engine_options_get_backup_log_files( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_backup_rate_limit( + rocksdb_backup_engine_options_t* options, uint64_t limit); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_backup_engine_options_get_backup_rate_limit( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_restore_rate_limit( + rocksdb_backup_engine_options_t* options, uint64_t limit); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_backup_engine_options_get_restore_rate_limit( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_max_background_operations( + rocksdb_backup_engine_options_t* options, int val); + +extern ROCKSDB_LIBRARY_API int +rocksdb_backup_engine_options_get_max_background_operations( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_callback_trigger_interval_size( + rocksdb_backup_engine_options_t* options, uint64_t size); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_backup_engine_options_get_callback_trigger_interval_size( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_max_valid_backups_to_open( + rocksdb_backup_engine_options_t* options, int val); + +extern ROCKSDB_LIBRARY_API int +rocksdb_backup_engine_options_get_max_valid_backups_to_open( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_share_files_with_checksum_naming( + rocksdb_backup_engine_options_t* options, int val); + +extern ROCKSDB_LIBRARY_API int +rocksdb_backup_engine_options_get_share_files_with_checksum_naming( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_options_destroy( + rocksdb_backup_engine_options_t*); + +/* Checkpoint */ + +extern ROCKSDB_LIBRARY_API rocksdb_checkpoint_t* +rocksdb_checkpoint_object_create(rocksdb_t* db, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_checkpoint_create( + rocksdb_checkpoint_t* checkpoint, const char* checkpoint_dir, + uint64_t log_size_for_flush, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_checkpoint_object_destroy( + rocksdb_checkpoint_t* checkpoint); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_and_trim_history( + const rocksdb_options_t* options, const char* name, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char* trim_ts, + size_t trim_tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_column_families( + const rocksdb_options_t* options, const char* name, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_column_families_with_ttl( + const rocksdb_options_t* options, const char* name, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, const int* ttls, + char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* +rocksdb_open_for_read_only_column_families( + const rocksdb_options_t* options, const char* name, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, + unsigned char error_if_wal_file_exists, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_as_secondary_column_families( + const rocksdb_options_t* options, const char* name, + const char* secondary_path, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char** errptr); + +extern ROCKSDB_LIBRARY_API char** rocksdb_list_column_families( + const rocksdb_options_t* options, const char* name, size_t* lencf, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_list_column_families_destroy( + char** list, size_t len); + +extern ROCKSDB_LIBRARY_API rocksdb_column_family_handle_t* +rocksdb_create_column_family(rocksdb_t* db, + const rocksdb_options_t* column_family_options, + const char* column_family_name, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_column_family_handle_t** +rocksdb_create_column_families(rocksdb_t* db, + const rocksdb_options_t* column_family_options, + int num_column_families, + const char* const* column_family_names, + size_t* lencfs, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_create_column_families_destroy( + rocksdb_column_family_handle_t** list); + +extern ROCKSDB_LIBRARY_API rocksdb_column_family_handle_t* +rocksdb_create_column_family_with_ttl( + rocksdb_t* db, const rocksdb_options_t* column_family_options, + const char* column_family_name, int ttl, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_drop_column_family( + rocksdb_t* db, rocksdb_column_family_handle_t* handle, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_column_family_handle_destroy( + rocksdb_column_family_handle_t*); + +extern ROCKSDB_LIBRARY_API uint32_t +rocksdb_column_family_handle_get_id(rocksdb_column_family_handle_t* handle); + +extern ROCKSDB_LIBRARY_API char* rocksdb_column_family_handle_get_name( + rocksdb_column_family_handle_t* handle, size_t* name_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_close(rocksdb_t* db); + +extern ROCKSDB_LIBRARY_API void rocksdb_put( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, const char* val, size_t vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_put_cf( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* val, size_t vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_cf( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_range_cf( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* start_key, + size_t start_key_len, const char* end_key, size_t end_key_len, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_merge( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, const char* val, size_t vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_merge_cf( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* val, size_t vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_write( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_writebatch_t* batch, char** errptr); + +/* Returns NULL if not found. A malloc()ed array otherwise. + Stores the length of the array in *vallen. */ +extern ROCKSDB_LIBRARY_API char* rocksdb_get( + rocksdb_t* db, const rocksdb_readoptions_t* options, const char* key, + size_t keylen, size_t* vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_get_with_ts( + rocksdb_t* db, const rocksdb_readoptions_t* options, const char* key, + size_t keylen, size_t* vallen, char** ts, size_t* tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_get_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_get_cf_with_ts( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** ts, size_t* tslen, char** errptr); + +// if values_list[i] == NULL and errs[i] == NULL, +// then we got status.IsNotFound(), which we will not return. +// all errors except status status.ok() and status.IsNotFound() are returned. +// +// errs, values_list and values_list_sizes must be num_keys in length, +// allocated by the caller. +// errs is a list of strings as opposed to the conventional one error, +// where errs[i] is the status for retrieval of keys_list[i]. +// each non-NULL errs entry is a malloc()ed, null terminated string. +// each non-NULL values_list entry is a malloc()ed array, with +// the length for each stored in values_list_sizes[i]. +extern ROCKSDB_LIBRARY_API void rocksdb_multi_get( + rocksdb_t* db, const rocksdb_readoptions_t* options, size_t num_keys, + const char* const* keys_list, const size_t* keys_list_sizes, + char** values_list, size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_multi_get_with_ts( + rocksdb_t* db, const rocksdb_readoptions_t* options, size_t num_keys, + const char* const* keys_list, const size_t* keys_list_sizes, + char** values_list, size_t* values_list_sizes, char** timestamp_list, + size_t* timestamp_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_multi_get_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + const rocksdb_column_family_handle_t* const* column_families, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_multi_get_cf_with_ts( + rocksdb_t* db, const rocksdb_readoptions_t* options, + const rocksdb_column_family_handle_t* const* column_families, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** timestamps_list, + size_t* timestamps_list_sizes, char** errs); + +// The MultiGet API that improves performance by batching operations +// in the read path for greater efficiency. Currently, only the block based +// table format with full filters are supported. Other table formats such +// as plain table, block based table with block based filters and +// partitioned indexes will still work, but will not get any performance +// benefits. +// +// Note that all the keys passed to this API are restricted to a single +// column family. +// +// Parameters - +// db - the RocksDB instance. +// options - ReadOptions +// column_family - ColumnFamilyHandle* that the keys belong to. All the keys +// passed to the API are restricted to a single column family +// num_keys - Number of keys to lookup +// keys_list - Pointer to C style array of keys with num_keys elements +// keys_list_sizes - Pointer to C style array of the size of corresponding key +// in key_list with num_keys elements. +// values - Pointer to C style array of PinnableSlices with num_keys elements +// statuses - Pointer to C style array of Status with num_keys elements +// sorted_input - If true, it means the input keys are already sorted by key +// order, so the MultiGet() API doesn't have to sort them +// again. If false, the keys will be copied and sorted +// internally by the API - the input array will not be +// modified +extern ROCKSDB_LIBRARY_API void rocksdb_batched_multi_get_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, size_t num_keys, + const char* const* keys_list, const size_t* keys_list_sizes, + rocksdb_pinnableslice_t** values, char** errs, const bool sorted_input); + +// The value is only allocated (using malloc) and returned if it is found and +// value_found isn't NULL. In that case the user is responsible for freeing it. +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_key_may_exist( + rocksdb_t* db, const rocksdb_readoptions_t* options, const char* key, + size_t key_len, char** value, size_t* val_len, const char* timestamp, + size_t timestamp_len, unsigned char* value_found); + +// The value is only allocated (using malloc) and returned if it is found and +// value_found isn't NULL. In that case the user is responsible for freeing it. +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_key_may_exist_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t key_len, char** value, size_t* val_len, const char* timestamp, + size_t timestamp_len, unsigned char* value_found); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* rocksdb_create_iterator( + rocksdb_t* db, const rocksdb_readoptions_t* options); + +extern ROCKSDB_LIBRARY_API rocksdb_wal_iterator_t* rocksdb_get_updates_since( + rocksdb_t* db, uint64_t seq_number, + const rocksdb_wal_readoptions_t* options, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* rocksdb_create_iterator_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family); + +extern ROCKSDB_LIBRARY_API void rocksdb_create_iterators( + rocksdb_t* db, rocksdb_readoptions_t* opts, + rocksdb_column_family_handle_t** column_families, + rocksdb_iterator_t** iterators, size_t size, char** errptr); + +extern ROCKSDB_LIBRARY_API const rocksdb_snapshot_t* rocksdb_create_snapshot( + rocksdb_t* db); + +extern ROCKSDB_LIBRARY_API void rocksdb_release_snapshot( + rocksdb_t* db, const rocksdb_snapshot_t* snapshot); + +/* Returns NULL if property name is unknown. + Else returns a pointer to a malloc()-ed null-terminated value. */ +extern ROCKSDB_LIBRARY_API char* rocksdb_property_value(rocksdb_t* db, + const char* propname); +/* returns 0 on success, -1 otherwise */ +extern ROCKSDB_LIBRARY_API int rocksdb_property_int(rocksdb_t* db, + const char* propname, + uint64_t* out_val); + +/* returns 0 on success, -1 otherwise */ +extern ROCKSDB_LIBRARY_API int rocksdb_property_int_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* propname, uint64_t* out_val); + +extern ROCKSDB_LIBRARY_API char* rocksdb_property_value_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* propname); + +extern ROCKSDB_LIBRARY_API void rocksdb_approximate_sizes( + rocksdb_t* db, int num_ranges, const char* const* range_start_key, + const size_t* range_start_key_len, const char* const* range_limit_key, + const size_t* range_limit_key_len, uint64_t* sizes, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_approximate_sizes_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + int num_ranges, const char* const* range_start_key, + const size_t* range_start_key_len, const char* const* range_limit_key, + const size_t* range_limit_key_len, uint64_t* sizes, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_compact_range(rocksdb_t* db, + const char* start_key, + size_t start_key_len, + const char* limit_key, + size_t limit_key_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_compact_range_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* limit_key, + size_t limit_key_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_suggest_compact_range( + rocksdb_t* db, const char* start_key, size_t start_key_len, + const char* limit_key, size_t limit_key_len, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_suggest_compact_range_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* limit_key, + size_t limit_key_len, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_compact_range_opt( + rocksdb_t* db, rocksdb_compactoptions_t* opt, const char* start_key, + size_t start_key_len, const char* limit_key, size_t limit_key_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_compact_range_cf_opt( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + rocksdb_compactoptions_t* opt, const char* start_key, size_t start_key_len, + const char* limit_key, size_t limit_key_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_file(rocksdb_t* db, + const char* name); + +extern ROCKSDB_LIBRARY_API const rocksdb_livefiles_t* rocksdb_livefiles( + rocksdb_t* db); + +extern ROCKSDB_LIBRARY_API void rocksdb_flush( + rocksdb_t* db, const rocksdb_flushoptions_t* options, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_flush_cf( + rocksdb_t* db, const rocksdb_flushoptions_t* options, + rocksdb_column_family_handle_t* column_family, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_flush_cfs( + rocksdb_t* db, const rocksdb_flushoptions_t* options, + rocksdb_column_family_handle_t** column_family, int num_column_families, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_flush_wal(rocksdb_t* db, + unsigned char sync, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_disable_file_deletions(rocksdb_t* db, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_enable_file_deletions( + rocksdb_t* db, unsigned char force, char** errptr); + +/* Management operations */ + +extern ROCKSDB_LIBRARY_API void rocksdb_destroy_db( + const rocksdb_options_t* options, const char* name, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_repair_db( + const rocksdb_options_t* options, const char* name, char** errptr); + +/* Iterator */ + +extern ROCKSDB_LIBRARY_API void rocksdb_iter_destroy(rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_iter_valid( + const rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_seek_to_first(rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_seek_to_last(rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_seek(rocksdb_iterator_t*, + const char* k, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_seek_for_prev(rocksdb_iterator_t*, + const char* k, + size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_next(rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_prev(rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API const char* rocksdb_iter_key( + const rocksdb_iterator_t*, size_t* klen); +extern ROCKSDB_LIBRARY_API const char* rocksdb_iter_value( + const rocksdb_iterator_t*, size_t* vlen); +extern ROCKSDB_LIBRARY_API const char* rocksdb_iter_timestamp( + const rocksdb_iterator_t*, size_t* tslen); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_get_error( + const rocksdb_iterator_t*, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_wal_iter_next( + rocksdb_wal_iterator_t* iter); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_wal_iter_valid( + const rocksdb_wal_iterator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_wal_iter_status( + const rocksdb_wal_iterator_t* iter, char** errptr); +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_t* rocksdb_wal_iter_get_batch( + const rocksdb_wal_iterator_t* iter, uint64_t* seq); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_get_latest_sequence_number(rocksdb_t* db); +extern ROCKSDB_LIBRARY_API void rocksdb_wal_iter_destroy( + const rocksdb_wal_iterator_t* iter); + +/* Write batch */ + +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_t* rocksdb_writebatch_create( + void); +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_t* rocksdb_writebatch_create_from( + const char* rep, size_t size); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_destroy( + rocksdb_writebatch_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_clear(rocksdb_writebatch_t*); +extern ROCKSDB_LIBRARY_API int rocksdb_writebatch_count(rocksdb_writebatch_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_put(rocksdb_writebatch_t*, + const char* key, + size_t klen, + const char* val, + size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_put_cf( + rocksdb_writebatch_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_put_cf_with_ts( + rocksdb_writebatch_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* ts, size_t tslen, const char* val, + size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_putv( + rocksdb_writebatch_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, int num_values, + const char* const* values_list, const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_putv_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes, + int num_values, const char* const* values_list, + const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_merge(rocksdb_writebatch_t*, + const char* key, + size_t klen, + const char* val, + size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_merge_cf( + rocksdb_writebatch_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_mergev( + rocksdb_writebatch_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, int num_values, + const char* const* values_list, const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_mergev_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes, + int num_values, const char* const* values_list, + const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete(rocksdb_writebatch_t*, + const char* key, + size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_singledelete( + rocksdb_writebatch_t* b, const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_cf( + rocksdb_writebatch_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_cf_with_ts( + rocksdb_writebatch_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* ts, size_t tslen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_singledelete_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_singledelete_cf_with_ts( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* ts, size_t tslen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_deletev( + rocksdb_writebatch_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_deletev_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_range( + rocksdb_writebatch_t* b, const char* start_key, size_t start_key_len, + const char* end_key, size_t end_key_len); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_range_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* end_key, + size_t end_key_len); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_rangev( + rocksdb_writebatch_t* b, int num_keys, const char* const* start_keys_list, + const size_t* start_keys_list_sizes, const char* const* end_keys_list, + const size_t* end_keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_rangev_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* start_keys_list, + const size_t* start_keys_list_sizes, const char* const* end_keys_list, + const size_t* end_keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_put_log_data( + rocksdb_writebatch_t*, const char* blob, size_t len); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_iterate( + rocksdb_writebatch_t*, void* state, + void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen), + void (*deleted)(void*, const char* k, size_t klen)); +extern ROCKSDB_LIBRARY_API const char* rocksdb_writebatch_data( + rocksdb_writebatch_t*, size_t* size); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_set_save_point( + rocksdb_writebatch_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_rollback_to_save_point( + rocksdb_writebatch_t*, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_pop_save_point( + rocksdb_writebatch_t*, char** errptr); + +/* Write batch with index */ + +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_wi_t* +rocksdb_writebatch_wi_create(size_t reserved_bytes, + unsigned char overwrite_keys); +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_wi_t* +rocksdb_writebatch_wi_create_from(const char* rep, size_t size); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_destroy( + rocksdb_writebatch_wi_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_clear( + rocksdb_writebatch_wi_t*); +extern ROCKSDB_LIBRARY_API int rocksdb_writebatch_wi_count( + rocksdb_writebatch_wi_t* b); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_put( + rocksdb_writebatch_wi_t*, const char* key, size_t klen, const char* val, + size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_put_cf( + rocksdb_writebatch_wi_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_putv( + rocksdb_writebatch_wi_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, int num_values, + const char* const* values_list, const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_putv_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes, + int num_values, const char* const* values_list, + const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_merge( + rocksdb_writebatch_wi_t*, const char* key, size_t klen, const char* val, + size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_merge_cf( + rocksdb_writebatch_wi_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_mergev( + rocksdb_writebatch_wi_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, int num_values, + const char* const* values_list, const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_mergev_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes, + int num_values, const char* const* values_list, + const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete( + rocksdb_writebatch_wi_t*, const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_singledelete( + rocksdb_writebatch_wi_t*, const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete_cf( + rocksdb_writebatch_wi_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_singledelete_cf( + rocksdb_writebatch_wi_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_deletev( + rocksdb_writebatch_wi_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_deletev_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes); +// DO NOT USE - rocksdb_writebatch_wi_delete_range is not yet supported +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete_range( + rocksdb_writebatch_wi_t* b, const char* start_key, size_t start_key_len, + const char* end_key, size_t end_key_len); +// DO NOT USE - rocksdb_writebatch_wi_delete_range_cf is not yet supported +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete_range_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* end_key, + size_t end_key_len); +// DO NOT USE - rocksdb_writebatch_wi_delete_rangev is not yet supported +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete_rangev( + rocksdb_writebatch_wi_t* b, int num_keys, + const char* const* start_keys_list, const size_t* start_keys_list_sizes, + const char* const* end_keys_list, const size_t* end_keys_list_sizes); +// DO NOT USE - rocksdb_writebatch_wi_delete_rangev_cf is not yet supported +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete_rangev_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* start_keys_list, + const size_t* start_keys_list_sizes, const char* const* end_keys_list, + const size_t* end_keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_put_log_data( + rocksdb_writebatch_wi_t*, const char* blob, size_t len); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_iterate( + rocksdb_writebatch_wi_t* b, void* state, + void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen), + void (*deleted)(void*, const char* k, size_t klen)); +extern ROCKSDB_LIBRARY_API const char* rocksdb_writebatch_wi_data( + rocksdb_writebatch_wi_t* b, size_t* size); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_set_save_point( + rocksdb_writebatch_wi_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_rollback_to_save_point( + rocksdb_writebatch_wi_t*, char** errptr); +extern ROCKSDB_LIBRARY_API char* rocksdb_writebatch_wi_get_from_batch( + rocksdb_writebatch_wi_t* wbwi, const rocksdb_options_t* options, + const char* key, size_t keylen, size_t* vallen, char** errptr); +extern ROCKSDB_LIBRARY_API char* rocksdb_writebatch_wi_get_from_batch_cf( + rocksdb_writebatch_wi_t* wbwi, const rocksdb_options_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** errptr); +extern ROCKSDB_LIBRARY_API char* rocksdb_writebatch_wi_get_from_batch_and_db( + rocksdb_writebatch_wi_t* wbwi, rocksdb_t* db, + const rocksdb_readoptions_t* options, const char* key, size_t keylen, + size_t* vallen, char** errptr); +extern ROCKSDB_LIBRARY_API char* rocksdb_writebatch_wi_get_from_batch_and_db_cf( + rocksdb_writebatch_wi_t* wbwi, rocksdb_t* db, + const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_write_writebatch_wi( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_writebatch_wi_t* wbwi, char** errptr); +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_writebatch_wi_create_iterator_with_base( + rocksdb_writebatch_wi_t* wbwi, rocksdb_iterator_t* base_iterator); +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_writebatch_wi_create_iterator_with_base_cf( + rocksdb_writebatch_wi_t* wbwi, rocksdb_iterator_t* base_iterator, + rocksdb_column_family_handle_t* cf); + +/* Options utils */ + +// Load the latest rocksdb options from the specified db_path. +// +// On success, num_column_families will be updated with a non-zero +// number indicating the number of column families. +// The returned db_options, column_family_names, and column_family_options +// should be released via rocksdb_load_latest_options_destroy(). +// +// On error, a non-null errptr that includes the error message will be +// returned. db_options, column_family_names, and column_family_options +// will be set to NULL. +extern ROCKSDB_LIBRARY_API void rocksdb_load_latest_options( + const char* db_path, rocksdb_env_t* env, bool ignore_unknown_options, + rocksdb_cache_t* cache, rocksdb_options_t** db_options, + size_t* num_column_families, char*** column_family_names, + rocksdb_options_t*** column_family_options, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_load_latest_options_destroy( + rocksdb_options_t* db_options, char** list_column_family_names, + rocksdb_options_t** list_column_family_options, size_t len); + +/* Block based table options */ + +extern ROCKSDB_LIBRARY_API rocksdb_block_based_table_options_t* +rocksdb_block_based_options_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_destroy( + rocksdb_block_based_table_options_t* options); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_checksum( + rocksdb_block_based_table_options_t*, char); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_block_size( + rocksdb_block_based_table_options_t* options, size_t block_size); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_block_size_deviation( + rocksdb_block_based_table_options_t* options, int block_size_deviation); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_block_restart_interval( + rocksdb_block_based_table_options_t* options, int block_restart_interval); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_index_block_restart_interval( + rocksdb_block_based_table_options_t* options, + int index_block_restart_interval); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_metadata_block_size( + rocksdb_block_based_table_options_t* options, uint64_t metadata_block_size); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_partition_filters( + rocksdb_block_based_table_options_t* options, + unsigned char partition_filters); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_optimize_filters_for_memory( + rocksdb_block_based_table_options_t* options, + unsigned char optimize_filters_for_memory); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_use_delta_encoding( + rocksdb_block_based_table_options_t* options, + unsigned char use_delta_encoding); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_filter_policy( + rocksdb_block_based_table_options_t* options, + rocksdb_filterpolicy_t* filter_policy); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_no_block_cache( + rocksdb_block_based_table_options_t* options, unsigned char no_block_cache); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_block_cache( + rocksdb_block_based_table_options_t* options, rocksdb_cache_t* block_cache); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_whole_key_filtering( + rocksdb_block_based_table_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_format_version( + rocksdb_block_based_table_options_t*, int); +enum { + rocksdb_block_based_table_index_type_binary_search = 0, + rocksdb_block_based_table_index_type_hash_search = 1, + rocksdb_block_based_table_index_type_two_level_index_search = 2, +}; +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_index_type( + rocksdb_block_based_table_options_t*, int); // uses one of the above enums +enum { + rocksdb_block_based_table_data_block_index_type_binary_search = 0, + rocksdb_block_based_table_data_block_index_type_binary_search_and_hash = 1, +}; +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_data_block_index_type( + rocksdb_block_based_table_options_t*, int); // uses one of the above enums +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_data_block_hash_ratio( + rocksdb_block_based_table_options_t* options, double v); +// rocksdb_block_based_options_set_hash_index_allow_collision() +// is removed since BlockBasedTableOptions.hash_index_allow_collision() +// is removed +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_cache_index_and_filter_blocks( + rocksdb_block_based_table_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority( + rocksdb_block_based_table_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache( + rocksdb_block_based_table_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_pin_top_level_index_and_filter( + rocksdb_block_based_table_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_block_based_table_factory( + rocksdb_options_t* opt, rocksdb_block_based_table_options_t* table_options); + +/* Cuckoo table options */ + +extern ROCKSDB_LIBRARY_API rocksdb_cuckoo_table_options_t* +rocksdb_cuckoo_options_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_cuckoo_options_destroy( + rocksdb_cuckoo_table_options_t* options); +extern ROCKSDB_LIBRARY_API void rocksdb_cuckoo_options_set_hash_ratio( + rocksdb_cuckoo_table_options_t* options, double v); +extern ROCKSDB_LIBRARY_API void rocksdb_cuckoo_options_set_max_search_depth( + rocksdb_cuckoo_table_options_t* options, uint32_t v); +extern ROCKSDB_LIBRARY_API void rocksdb_cuckoo_options_set_cuckoo_block_size( + rocksdb_cuckoo_table_options_t* options, uint32_t v); +extern ROCKSDB_LIBRARY_API void +rocksdb_cuckoo_options_set_identity_as_first_hash( + rocksdb_cuckoo_table_options_t* options, unsigned char v); +extern ROCKSDB_LIBRARY_API void rocksdb_cuckoo_options_set_use_module_hash( + rocksdb_cuckoo_table_options_t* options, unsigned char v); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_cuckoo_table_factory( + rocksdb_options_t* opt, rocksdb_cuckoo_table_options_t* table_options); + +/* Options */ +extern ROCKSDB_LIBRARY_API void rocksdb_set_options(rocksdb_t* db, int count, + const char* const keys[], + const char* const values[], + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_set_options_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* handle, int count, + const char* const keys[], const char* const values[], char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_options_t* rocksdb_options_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_options_destroy(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API rocksdb_options_t* rocksdb_options_create_copy( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_increase_parallelism( + rocksdb_options_t* opt, int total_threads); +extern ROCKSDB_LIBRARY_API void rocksdb_options_optimize_for_point_lookup( + rocksdb_options_t* opt, uint64_t block_cache_size_mb); +extern ROCKSDB_LIBRARY_API void rocksdb_options_optimize_level_style_compaction( + rocksdb_options_t* opt, uint64_t memtable_memory_budget); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_optimize_universal_style_compaction( + rocksdb_options_t* opt, uint64_t memtable_memory_budget); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_allow_ingest_behind( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_allow_ingest_behind(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compaction_filter( + rocksdb_options_t*, rocksdb_compactionfilter_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compaction_filter_factory( + rocksdb_options_t*, rocksdb_compactionfilterfactory_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_compaction_readahead_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_compaction_readahead_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_comparator( + rocksdb_options_t*, rocksdb_comparator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_merge_operator( + rocksdb_options_t*, rocksdb_mergeoperator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_uint64add_merge_operator( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compression_per_level( + rocksdb_options_t* opt, const int* level_values, size_t num_levels); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_create_if_missing( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_create_if_missing( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_create_missing_column_families(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_create_missing_column_families(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_error_if_exists( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_error_if_exists( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_paranoid_checks( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_paranoid_checks( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_db_paths( + rocksdb_options_t*, const rocksdb_dbpath_t** path_values, size_t num_paths); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_env(rocksdb_options_t*, + rocksdb_env_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_info_log(rocksdb_options_t*, + rocksdb_logger_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_info_log_level( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_info_log_level( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_write_buffer_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_write_buffer_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_db_write_buffer_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_db_write_buffer_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_open_files( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_open_files( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_file_opening_threads( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_file_opening_threads( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_total_wal_size( + rocksdb_options_t* opt, uint64_t n); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_max_total_wal_size(rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compression_options( + rocksdb_options_t*, int, int, int, int); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_compression_options_zstd_max_train_bytes(rocksdb_options_t*, + int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_compression_options_zstd_max_train_bytes( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_compression_options_use_zstd_dict_trainer( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_compression_options_use_zstd_dict_trainer( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_compression_options_parallel_threads(rocksdb_options_t*, + int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_compression_options_parallel_threads( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_compression_options_max_dict_buffer_bytes( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_compression_options_max_dict_buffer_bytes( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_bottommost_compression_options(rocksdb_options_t*, int, int, + int, int, unsigned char); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_bottommost_compression_options_zstd_max_train_bytes( + rocksdb_options_t*, int, unsigned char); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_bottommost_compression_options_use_zstd_dict_trainer( + rocksdb_options_t*, unsigned char, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_bottommost_compression_options_use_zstd_dict_trainer( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_bottommost_compression_options_max_dict_buffer_bytes( + rocksdb_options_t*, uint64_t, unsigned char); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_prefix_extractor( + rocksdb_options_t*, rocksdb_slicetransform_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_num_levels( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_num_levels( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_level0_file_num_compaction_trigger(rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_level0_file_num_compaction_trigger(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_level0_slowdown_writes_trigger(rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_level0_slowdown_writes_trigger(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_level0_stop_writes_trigger( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_level0_stop_writes_trigger( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_target_file_size_base( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_target_file_size_base(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_target_file_size_multiplier( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_target_file_size_multiplier( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_bytes_for_level_base( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_max_bytes_for_level_base(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_level_compaction_dynamic_level_bytes(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_level_compaction_dynamic_level_bytes(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_max_bytes_for_level_multiplier(rocksdb_options_t*, double); +extern ROCKSDB_LIBRARY_API double +rocksdb_options_get_max_bytes_for_level_multiplier(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_max_bytes_for_level_multiplier_additional( + rocksdb_options_t*, int* level_values, size_t num_levels); +extern ROCKSDB_LIBRARY_API void rocksdb_options_enable_statistics( + rocksdb_options_t*); + +enum { + rocksdb_statistics_level_disable_all = 0, + rocksdb_statistics_level_except_tickers = + rocksdb_statistics_level_disable_all, + rocksdb_statistics_level_except_histogram_or_timers = 1, + rocksdb_statistics_level_except_timers = 2, + rocksdb_statistics_level_except_detailed_timers = 3, + rocksdb_statistics_level_except_time_for_mutex = 4, + rocksdb_statistics_level_all = 5, +}; + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_statistics_level( + rocksdb_options_t*, int level); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_statistics_level( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_skip_stats_update_on_db_open(rocksdb_options_t* opt, + unsigned char val); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_skip_stats_update_on_db_open(rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_skip_checking_sst_file_sizes_on_db_open( + rocksdb_options_t* opt, unsigned char val); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open( + rocksdb_options_t* opt); + +/* Blob Options Settings */ +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_enable_blob_files( + rocksdb_options_t* opt, unsigned char val); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_enable_blob_files( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_min_blob_size( + rocksdb_options_t* opt, uint64_t val); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_min_blob_size(rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_file_size( + rocksdb_options_t* opt, uint64_t val); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_blob_file_size(rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_compression_type( + rocksdb_options_t* opt, int val); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_blob_compression_type( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_enable_blob_gc( + rocksdb_options_t* opt, unsigned char val); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_enable_blob_gc( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_gc_age_cutoff( + rocksdb_options_t* opt, double val); +extern ROCKSDB_LIBRARY_API double rocksdb_options_get_blob_gc_age_cutoff( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_gc_force_threshold( + rocksdb_options_t* opt, double val); +extern ROCKSDB_LIBRARY_API double rocksdb_options_get_blob_gc_force_threshold( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_blob_compaction_readahead_size(rocksdb_options_t* opt, + uint64_t val); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_blob_compaction_readahead_size(rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_file_starting_level( + rocksdb_options_t* opt, int val); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_blob_file_starting_level( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_cache( + rocksdb_options_t* opt, rocksdb_cache_t* blob_cache); + +enum { + rocksdb_prepopulate_blob_disable = 0, + rocksdb_prepopulate_blob_flush_only = 1 +}; + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_prepopulate_blob_cache( + rocksdb_options_t* opt, int val); + +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_prepopulate_blob_cache( + rocksdb_options_t* opt); + +/* returns a pointer to a malloc()-ed, null terminated string */ +extern ROCKSDB_LIBRARY_API char* rocksdb_options_statistics_get_string( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API uint64_t rocksdb_options_statistics_get_ticker_count( + rocksdb_options_t* opt, uint32_t ticker_type); +extern ROCKSDB_LIBRARY_API void rocksdb_options_statistics_get_histogram_data( + rocksdb_options_t* opt, uint32_t histogram_type, + rocksdb_statistics_histogram_data_t* const data); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_write_buffer_number( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_write_buffer_number( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_min_write_buffer_number_to_merge(rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_min_write_buffer_number_to_merge(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_max_write_buffer_number_to_maintain(rocksdb_options_t*, + int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_max_write_buffer_number_to_maintain(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_max_write_buffer_size_to_maintain(rocksdb_options_t*, + int64_t); +extern ROCKSDB_LIBRARY_API int64_t +rocksdb_options_get_max_write_buffer_size_to_maintain(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_enable_pipelined_write( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_enable_pipelined_write(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_unordered_write( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_unordered_write( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_subcompactions( + rocksdb_options_t*, uint32_t); +extern ROCKSDB_LIBRARY_API uint32_t +rocksdb_options_get_max_subcompactions(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_background_jobs( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_background_jobs( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_background_compactions( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_background_compactions( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_background_flushes( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_background_flushes( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_log_file_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_max_log_file_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_log_file_time_to_roll( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_log_file_time_to_roll(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_keep_log_file_num( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_keep_log_file_num(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_recycle_log_file_num( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_recycle_log_file_num(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_soft_pending_compaction_bytes_limit(rocksdb_options_t* opt, + size_t v); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_soft_pending_compaction_bytes_limit(rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_hard_pending_compaction_bytes_limit(rocksdb_options_t* opt, + size_t v); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_hard_pending_compaction_bytes_limit(rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_manifest_file_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_max_manifest_file_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_table_cache_numshardbits( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_table_cache_numshardbits( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_arena_block_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_arena_block_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_use_fsync( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_use_fsync( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_db_log_dir( + rocksdb_options_t*, const char*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_wal_dir(rocksdb_options_t*, + const char*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_WAL_ttl_seconds( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_WAL_ttl_seconds(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_WAL_size_limit_MB( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_WAL_size_limit_MB(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_manifest_preallocation_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_manifest_preallocation_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_allow_mmap_reads( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_allow_mmap_reads( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_allow_mmap_writes( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_allow_mmap_writes( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_use_direct_reads( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_use_direct_reads( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_use_direct_io_for_flush_and_compaction(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_use_direct_io_for_flush_and_compaction(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_is_fd_close_on_exec( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_is_fd_close_on_exec(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_stats_dump_period_sec( + rocksdb_options_t*, unsigned int); +extern ROCKSDB_LIBRARY_API unsigned int +rocksdb_options_get_stats_dump_period_sec(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_stats_persist_period_sec( + rocksdb_options_t*, unsigned int); +extern ROCKSDB_LIBRARY_API unsigned int +rocksdb_options_get_stats_persist_period_sec(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_advise_random_on_open( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_advise_random_on_open(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_access_hint_on_compaction_start(rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_access_hint_on_compaction_start(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_use_adaptive_mutex( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_use_adaptive_mutex( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_bytes_per_sync( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_bytes_per_sync(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_wal_bytes_per_sync( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_wal_bytes_per_sync(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_writable_file_max_buffer_size(rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_writable_file_max_buffer_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_allow_concurrent_memtable_write(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_allow_concurrent_memtable_write(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_enable_write_thread_adaptive_yield(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_enable_write_thread_adaptive_yield(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_max_sequential_skip_in_iterations(rocksdb_options_t*, + uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_max_sequential_skip_in_iterations(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_disable_auto_compactions( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_disable_auto_compactions(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_optimize_filters_for_hits( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_optimize_filters_for_hits(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_delete_obsolete_files_period_micros(rocksdb_options_t*, + uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_delete_obsolete_files_period_micros(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_prepare_for_bulk_load( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_memtable_vector_rep( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_memtable_prefix_bloom_size_ratio(rocksdb_options_t*, + double); +extern ROCKSDB_LIBRARY_API double +rocksdb_options_get_memtable_prefix_bloom_size_ratio(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_compaction_bytes( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_max_compaction_bytes(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_hash_skip_list_rep( + rocksdb_options_t*, size_t, int32_t, int32_t); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_hash_link_list_rep( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_plain_table_factory( + rocksdb_options_t*, uint32_t, int, double, size_t, size_t, char, + unsigned char, unsigned char); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_min_level_to_compress( + rocksdb_options_t* opt, int level); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_memtable_huge_page_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_memtable_huge_page_size(rocksdb_options_t*); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_successive_merges( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_max_successive_merges(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_bloom_locality( + rocksdb_options_t*, uint32_t); +extern ROCKSDB_LIBRARY_API uint32_t +rocksdb_options_get_bloom_locality(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_inplace_update_support( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_inplace_update_support(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_inplace_update_num_locks( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_inplace_update_num_locks(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_report_bg_io_stats( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_report_bg_io_stats( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_avoid_unnecessary_blocking_io(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_avoid_unnecessary_blocking_io(rocksdb_options_t*); + +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_experimental_mempurge_threshold(rocksdb_options_t*, double); +extern ROCKSDB_LIBRARY_API double +rocksdb_options_get_experimental_mempurge_threshold(rocksdb_options_t*); + +enum { + rocksdb_tolerate_corrupted_tail_records_recovery = 0, + rocksdb_absolute_consistency_recovery = 1, + rocksdb_point_in_time_recovery = 2, + rocksdb_skip_any_corrupted_records_recovery = 3 +}; +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_wal_recovery_mode( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_wal_recovery_mode( + rocksdb_options_t*); + +enum { + rocksdb_no_compression = 0, + rocksdb_snappy_compression = 1, + rocksdb_zlib_compression = 2, + rocksdb_bz2_compression = 3, + rocksdb_lz4_compression = 4, + rocksdb_lz4hc_compression = 5, + rocksdb_xpress_compression = 6, + rocksdb_zstd_compression = 7 +}; +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compression( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_compression( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_bottommost_compression( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_bottommost_compression( + rocksdb_options_t*); + +enum { + rocksdb_level_compaction = 0, + rocksdb_universal_compaction = 1, + rocksdb_fifo_compaction = 2 +}; +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compaction_style( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_compaction_style( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_universal_compaction_options( + rocksdb_options_t*, rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_fifo_compaction_options( + rocksdb_options_t* opt, rocksdb_fifo_compaction_options_t* fifo); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_ratelimiter( + rocksdb_options_t* opt, rocksdb_ratelimiter_t* limiter); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_atomic_flush( + rocksdb_options_t* opt, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_atomic_flush( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_row_cache( + rocksdb_options_t* opt, rocksdb_cache_t* cache); + +extern ROCKSDB_LIBRARY_API void +rocksdb_options_add_compact_on_deletion_collector_factory( + rocksdb_options_t*, size_t window_size, size_t num_dels_trigger); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_add_compact_on_deletion_collector_factory_del_ratio( + rocksdb_options_t*, size_t window_size, size_t num_dels_trigger, + double deletion_ratio); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_manual_wal_flush( + rocksdb_options_t* opt, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_manual_wal_flush( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_wal_compression( + rocksdb_options_t* opt, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_wal_compression( + rocksdb_options_t* opt); + +/* RateLimiter */ +extern ROCKSDB_LIBRARY_API rocksdb_ratelimiter_t* rocksdb_ratelimiter_create( + int64_t rate_bytes_per_sec, int64_t refill_period_us, int32_t fairness); +extern ROCKSDB_LIBRARY_API void rocksdb_ratelimiter_destroy( + rocksdb_ratelimiter_t*); + +/* PerfContext */ +enum { + rocksdb_uninitialized = 0, + rocksdb_disable = 1, + rocksdb_enable_count = 2, + rocksdb_enable_time_except_for_mutex = 3, + rocksdb_enable_time = 4, + rocksdb_out_of_bounds = 5 +}; + +enum { + rocksdb_user_key_comparison_count = 0, + rocksdb_block_cache_hit_count, + rocksdb_block_read_count, + rocksdb_block_read_byte, + rocksdb_block_read_time, + rocksdb_block_checksum_time, + rocksdb_block_decompress_time, + rocksdb_get_read_bytes, + rocksdb_multiget_read_bytes, + rocksdb_iter_read_bytes, + rocksdb_internal_key_skipped_count, + rocksdb_internal_delete_skipped_count, + rocksdb_internal_recent_skipped_count, + rocksdb_internal_merge_count, + rocksdb_get_snapshot_time, + rocksdb_get_from_memtable_time, + rocksdb_get_from_memtable_count, + rocksdb_get_post_process_time, + rocksdb_get_from_output_files_time, + rocksdb_seek_on_memtable_time, + rocksdb_seek_on_memtable_count, + rocksdb_next_on_memtable_count, + rocksdb_prev_on_memtable_count, + rocksdb_seek_child_seek_time, + rocksdb_seek_child_seek_count, + rocksdb_seek_min_heap_time, + rocksdb_seek_max_heap_time, + rocksdb_seek_internal_seek_time, + rocksdb_find_next_user_entry_time, + rocksdb_write_wal_time, + rocksdb_write_memtable_time, + rocksdb_write_delay_time, + rocksdb_write_pre_and_post_process_time, + rocksdb_db_mutex_lock_nanos, + rocksdb_db_condition_wait_nanos, + rocksdb_merge_operator_time_nanos, + rocksdb_read_index_block_nanos, + rocksdb_read_filter_block_nanos, + rocksdb_new_table_block_iter_nanos, + rocksdb_new_table_iterator_nanos, + rocksdb_block_seek_nanos, + rocksdb_find_table_nanos, + rocksdb_bloom_memtable_hit_count, + rocksdb_bloom_memtable_miss_count, + rocksdb_bloom_sst_hit_count, + rocksdb_bloom_sst_miss_count, + rocksdb_key_lock_wait_time, + rocksdb_key_lock_wait_count, + rocksdb_env_new_sequential_file_nanos, + rocksdb_env_new_random_access_file_nanos, + rocksdb_env_new_writable_file_nanos, + rocksdb_env_reuse_writable_file_nanos, + rocksdb_env_new_random_rw_file_nanos, + rocksdb_env_new_directory_nanos, + rocksdb_env_file_exists_nanos, + rocksdb_env_get_children_nanos, + rocksdb_env_get_children_file_attributes_nanos, + rocksdb_env_delete_file_nanos, + rocksdb_env_create_dir_nanos, + rocksdb_env_create_dir_if_missing_nanos, + rocksdb_env_delete_dir_nanos, + rocksdb_env_get_file_size_nanos, + rocksdb_env_get_file_modification_time_nanos, + rocksdb_env_rename_file_nanos, + rocksdb_env_link_file_nanos, + rocksdb_env_lock_file_nanos, + rocksdb_env_unlock_file_nanos, + rocksdb_env_new_logger_nanos, + rocksdb_number_async_seek, + rocksdb_blob_cache_hit_count, + rocksdb_blob_read_count, + rocksdb_blob_read_byte, + rocksdb_blob_read_time, + rocksdb_blob_checksum_time, + rocksdb_blob_decompress_time, + rocksdb_internal_range_del_reseek_count, + rocksdb_block_read_cpu_time, + rocksdb_total_metric_count = 79 +}; + +extern ROCKSDB_LIBRARY_API void rocksdb_set_perf_level(int); +extern ROCKSDB_LIBRARY_API rocksdb_perfcontext_t* rocksdb_perfcontext_create( + void); +extern ROCKSDB_LIBRARY_API void rocksdb_perfcontext_reset( + rocksdb_perfcontext_t* context); +extern ROCKSDB_LIBRARY_API char* rocksdb_perfcontext_report( + rocksdb_perfcontext_t* context, unsigned char exclude_zero_counters); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_perfcontext_metric(rocksdb_perfcontext_t* context, int metric); +extern ROCKSDB_LIBRARY_API void rocksdb_perfcontext_destroy( + rocksdb_perfcontext_t* context); + +/* Compaction Filter */ + +extern ROCKSDB_LIBRARY_API rocksdb_compactionfilter_t* +rocksdb_compactionfilter_create( + void* state, void (*destructor)(void*), + unsigned char (*filter)(void*, int level, const char* key, + size_t key_length, const char* existing_value, + size_t value_length, char** new_value, + size_t* new_value_length, + unsigned char* value_changed), + const char* (*name)(void*)); +extern ROCKSDB_LIBRARY_API void rocksdb_compactionfilter_set_ignore_snapshots( + rocksdb_compactionfilter_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void rocksdb_compactionfilter_destroy( + rocksdb_compactionfilter_t*); + +/* Compaction Filter Context */ + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_compactionfiltercontext_is_full_compaction( + rocksdb_compactionfiltercontext_t* context); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_compactionfiltercontext_is_manual_compaction( + rocksdb_compactionfiltercontext_t* context); + +/* Compaction Filter Factory */ + +extern ROCKSDB_LIBRARY_API rocksdb_compactionfilterfactory_t* +rocksdb_compactionfilterfactory_create( + void* state, void (*destructor)(void*), + rocksdb_compactionfilter_t* (*create_compaction_filter)( + void*, rocksdb_compactionfiltercontext_t* context), + const char* (*name)(void*)); +extern ROCKSDB_LIBRARY_API void rocksdb_compactionfilterfactory_destroy( + rocksdb_compactionfilterfactory_t*); + +/* Comparator */ + +extern ROCKSDB_LIBRARY_API rocksdb_comparator_t* rocksdb_comparator_create( + void* state, void (*destructor)(void*), + int (*compare)(void*, const char* a, size_t alen, const char* b, + size_t blen), + const char* (*name)(void*)); +extern ROCKSDB_LIBRARY_API void rocksdb_comparator_destroy( + rocksdb_comparator_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_comparator_t* +rocksdb_comparator_with_ts_create( + void* state, void (*destructor)(void*), + int (*compare)(void*, const char* a, size_t alen, const char* b, + size_t blen), + int (*compare_ts)(void*, const char* a_ts, size_t a_tslen, const char* b_ts, + size_t b_tslen), + int (*compare_without_ts)(void*, const char* a, size_t alen, + unsigned char a_has_ts, const char* b, + size_t blen, unsigned char b_has_ts), + const char* (*name)(void*), size_t timestamp_size); + +/* Filter policy */ + +extern ROCKSDB_LIBRARY_API void rocksdb_filterpolicy_destroy( + rocksdb_filterpolicy_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_filterpolicy_t* +rocksdb_filterpolicy_create_bloom(double bits_per_key); +extern ROCKSDB_LIBRARY_API rocksdb_filterpolicy_t* +rocksdb_filterpolicy_create_bloom_full(double bits_per_key); +extern ROCKSDB_LIBRARY_API rocksdb_filterpolicy_t* +rocksdb_filterpolicy_create_ribbon(double bloom_equivalent_bits_per_key); +extern ROCKSDB_LIBRARY_API rocksdb_filterpolicy_t* +rocksdb_filterpolicy_create_ribbon_hybrid(double bloom_equivalent_bits_per_key, + int bloom_before_level); + +/* Merge Operator */ + +extern ROCKSDB_LIBRARY_API rocksdb_mergeoperator_t* +rocksdb_mergeoperator_create( + void* state, void (*destructor)(void*), + char* (*full_merge)(void*, const char* key, size_t key_length, + const char* existing_value, + size_t existing_value_length, + const char* const* operands_list, + const size_t* operands_list_length, int num_operands, + unsigned char* success, size_t* new_value_length), + char* (*partial_merge)(void*, const char* key, size_t key_length, + const char* const* operands_list, + const size_t* operands_list_length, int num_operands, + unsigned char* success, size_t* new_value_length), + void (*delete_value)(void*, const char* value, size_t value_length), + const char* (*name)(void*)); +extern ROCKSDB_LIBRARY_API void rocksdb_mergeoperator_destroy( + rocksdb_mergeoperator_t*); + +/* Read options */ + +extern ROCKSDB_LIBRARY_API rocksdb_readoptions_t* rocksdb_readoptions_create( + void); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_destroy( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_verify_checksums( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_readoptions_get_verify_checksums(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_fill_cache( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_readoptions_get_fill_cache( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_snapshot( + rocksdb_readoptions_t*, const rocksdb_snapshot_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_iterate_upper_bound( + rocksdb_readoptions_t*, const char* key, size_t keylen); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_iterate_lower_bound( + rocksdb_readoptions_t*, const char* key, size_t keylen); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_read_tier( + rocksdb_readoptions_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_readoptions_get_read_tier( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_tailing( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_readoptions_get_tailing( + rocksdb_readoptions_t*); +// The functionality that this option controlled has been removed. +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_managed( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_readahead_size( + rocksdb_readoptions_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_readoptions_get_readahead_size(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_prefix_same_as_start( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_readoptions_get_prefix_same_as_start(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_pin_data( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_readoptions_get_pin_data( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_total_order_seek( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_readoptions_get_total_order_seek(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_readoptions_set_max_skippable_internal_keys(rocksdb_readoptions_t*, + uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_readoptions_get_max_skippable_internal_keys(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_readoptions_set_background_purge_on_iterator_cleanup( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_readoptions_get_background_purge_on_iterator_cleanup( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_ignore_range_deletions( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_readoptions_get_ignore_range_deletions(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_deadline( + rocksdb_readoptions_t*, uint64_t microseconds); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_readoptions_get_deadline(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_io_timeout( + rocksdb_readoptions_t*, uint64_t microseconds); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_readoptions_get_io_timeout(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_async_io( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_readoptions_get_async_io( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_timestamp( + rocksdb_readoptions_t*, const char* ts, size_t tslen); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_iter_start_ts( + rocksdb_readoptions_t*, const char* ts, size_t tslen); + +/* Write options */ + +extern ROCKSDB_LIBRARY_API rocksdb_writeoptions_t* rocksdb_writeoptions_create( + void); +extern ROCKSDB_LIBRARY_API void rocksdb_writeoptions_destroy( + rocksdb_writeoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writeoptions_set_sync( + rocksdb_writeoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_writeoptions_get_sync( + rocksdb_writeoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writeoptions_disable_WAL( + rocksdb_writeoptions_t* opt, int disable); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_writeoptions_get_disable_WAL( + rocksdb_writeoptions_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_writeoptions_set_ignore_missing_column_families(rocksdb_writeoptions_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_writeoptions_get_ignore_missing_column_families( + rocksdb_writeoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writeoptions_set_no_slowdown( + rocksdb_writeoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_writeoptions_get_no_slowdown( + rocksdb_writeoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writeoptions_set_low_pri( + rocksdb_writeoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_writeoptions_get_low_pri( + rocksdb_writeoptions_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_writeoptions_set_memtable_insert_hint_per_batch(rocksdb_writeoptions_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_writeoptions_get_memtable_insert_hint_per_batch( + rocksdb_writeoptions_t*); + +/* Compact range options */ + +extern ROCKSDB_LIBRARY_API rocksdb_compactoptions_t* +rocksdb_compactoptions_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_compactoptions_destroy( + rocksdb_compactoptions_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_compactoptions_set_exclusive_manual_compaction( + rocksdb_compactoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_compactoptions_get_exclusive_manual_compaction( + rocksdb_compactoptions_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_compactoptions_set_bottommost_level_compaction( + rocksdb_compactoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_compactoptions_get_bottommost_level_compaction( + rocksdb_compactoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_compactoptions_set_change_level( + rocksdb_compactoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_compactoptions_get_change_level(rocksdb_compactoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_compactoptions_set_target_level( + rocksdb_compactoptions_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_compactoptions_get_target_level( + rocksdb_compactoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_compactoptions_set_full_history_ts_low( + rocksdb_compactoptions_t*, char* ts, size_t tslen); + +/* Flush options */ + +extern ROCKSDB_LIBRARY_API rocksdb_flushoptions_t* rocksdb_flushoptions_create( + void); +extern ROCKSDB_LIBRARY_API void rocksdb_flushoptions_destroy( + rocksdb_flushoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_flushoptions_set_wait( + rocksdb_flushoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_flushoptions_get_wait( + rocksdb_flushoptions_t*); + +/* Memory allocator */ + +extern ROCKSDB_LIBRARY_API rocksdb_memory_allocator_t* +rocksdb_jemalloc_nodump_allocator_create(char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_memory_allocator_destroy( + rocksdb_memory_allocator_t*); + +/* Cache */ + +extern ROCKSDB_LIBRARY_API rocksdb_lru_cache_options_t* +rocksdb_lru_cache_options_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_lru_cache_options_destroy( + rocksdb_lru_cache_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_lru_cache_options_set_capacity( + rocksdb_lru_cache_options_t*, size_t); +extern ROCKSDB_LIBRARY_API void rocksdb_lru_cache_options_set_num_shard_bits( + rocksdb_lru_cache_options_t*, int); +extern ROCKSDB_LIBRARY_API void rocksdb_lru_cache_options_set_memory_allocator( + rocksdb_lru_cache_options_t*, rocksdb_memory_allocator_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_cache_t* rocksdb_cache_create_lru( + size_t capacity); +extern ROCKSDB_LIBRARY_API rocksdb_cache_t* +rocksdb_cache_create_lru_with_strict_capacity_limit(size_t capacity); +extern ROCKSDB_LIBRARY_API rocksdb_cache_t* rocksdb_cache_create_lru_opts( + const rocksdb_lru_cache_options_t*); + +extern ROCKSDB_LIBRARY_API void rocksdb_cache_destroy(rocksdb_cache_t* cache); +extern ROCKSDB_LIBRARY_API void rocksdb_cache_disown_data( + rocksdb_cache_t* cache); +extern ROCKSDB_LIBRARY_API void rocksdb_cache_set_capacity( + rocksdb_cache_t* cache, size_t capacity); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_cache_get_capacity(const rocksdb_cache_t* cache); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_cache_get_usage(const rocksdb_cache_t* cache); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_cache_get_pinned_usage(const rocksdb_cache_t* cache); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_cache_get_table_address_count(const rocksdb_cache_t* cache); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_cache_get_occupancy_count(const rocksdb_cache_t* cache); + +/* HyperClockCache */ + +extern ROCKSDB_LIBRARY_API rocksdb_hyper_clock_cache_options_t* +rocksdb_hyper_clock_cache_options_create(size_t capacity, + size_t estimated_entry_charge); +extern ROCKSDB_LIBRARY_API void rocksdb_hyper_clock_cache_options_destroy( + rocksdb_hyper_clock_cache_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_hyper_clock_cache_options_set_capacity( + rocksdb_hyper_clock_cache_options_t*, size_t); +extern ROCKSDB_LIBRARY_API void +rocksdb_hyper_clock_cache_options_set_estimated_entry_charge( + rocksdb_hyper_clock_cache_options_t*, size_t); +extern ROCKSDB_LIBRARY_API void +rocksdb_hyper_clock_cache_options_set_num_shard_bits( + rocksdb_hyper_clock_cache_options_t*, int); +extern ROCKSDB_LIBRARY_API void +rocksdb_hyper_clock_cache_options_set_memory_allocator( + rocksdb_hyper_clock_cache_options_t*, rocksdb_memory_allocator_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_cache_t* rocksdb_cache_create_hyper_clock( + size_t capacity, size_t estimated_entry_charge); +extern ROCKSDB_LIBRARY_API rocksdb_cache_t* +rocksdb_cache_create_hyper_clock_opts( + const rocksdb_hyper_clock_cache_options_t*); + +/* DBPath */ + +extern ROCKSDB_LIBRARY_API rocksdb_dbpath_t* rocksdb_dbpath_create( + const char* path, uint64_t target_size); +extern ROCKSDB_LIBRARY_API void rocksdb_dbpath_destroy(rocksdb_dbpath_t*); + +/* Env */ + +extern ROCKSDB_LIBRARY_API rocksdb_env_t* rocksdb_create_default_env(void); +extern ROCKSDB_LIBRARY_API rocksdb_env_t* rocksdb_create_mem_env(void); +extern ROCKSDB_LIBRARY_API void rocksdb_env_set_background_threads( + rocksdb_env_t* env, int n); +extern ROCKSDB_LIBRARY_API int rocksdb_env_get_background_threads( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void +rocksdb_env_set_high_priority_background_threads(rocksdb_env_t* env, int n); +extern ROCKSDB_LIBRARY_API int rocksdb_env_get_high_priority_background_threads( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void rocksdb_env_set_low_priority_background_threads( + rocksdb_env_t* env, int n); +extern ROCKSDB_LIBRARY_API int rocksdb_env_get_low_priority_background_threads( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void +rocksdb_env_set_bottom_priority_background_threads(rocksdb_env_t* env, int n); +extern ROCKSDB_LIBRARY_API int +rocksdb_env_get_bottom_priority_background_threads(rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void rocksdb_env_join_all_threads( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void rocksdb_env_lower_thread_pool_io_priority( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void +rocksdb_env_lower_high_priority_thread_pool_io_priority(rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void rocksdb_env_lower_thread_pool_cpu_priority( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void +rocksdb_env_lower_high_priority_thread_pool_cpu_priority(rocksdb_env_t* env); + +extern ROCKSDB_LIBRARY_API void rocksdb_env_destroy(rocksdb_env_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_envoptions_t* rocksdb_envoptions_create( + void); +extern ROCKSDB_LIBRARY_API void rocksdb_envoptions_destroy( + rocksdb_envoptions_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_create_dir_if_missing( + rocksdb_env_t* env, const char* path, char** errptr); + +/* SstFile */ + +extern ROCKSDB_LIBRARY_API rocksdb_sstfilewriter_t* +rocksdb_sstfilewriter_create(const rocksdb_envoptions_t* env, + const rocksdb_options_t* io_options); +extern ROCKSDB_LIBRARY_API rocksdb_sstfilewriter_t* +rocksdb_sstfilewriter_create_with_comparator( + const rocksdb_envoptions_t* env, const rocksdb_options_t* io_options, + const rocksdb_comparator_t* comparator); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_open( + rocksdb_sstfilewriter_t* writer, const char* name, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_add( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + const char* val, size_t vallen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_put( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + const char* val, size_t vallen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_put_with_ts( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + const char* ts, size_t tslen, const char* val, size_t vallen, + char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_merge( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + const char* val, size_t vallen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_delete( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_delete_with_ts( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + const char* ts, size_t tslen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_delete_range( + rocksdb_sstfilewriter_t* writer, const char* begin_key, size_t begin_keylen, + const char* end_key, size_t end_keylen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_finish( + rocksdb_sstfilewriter_t* writer, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_file_size( + rocksdb_sstfilewriter_t* writer, uint64_t* file_size); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_destroy( + rocksdb_sstfilewriter_t* writer); +extern ROCKSDB_LIBRARY_API rocksdb_ingestexternalfileoptions_t* +rocksdb_ingestexternalfileoptions_create(void); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_move_files( + rocksdb_ingestexternalfileoptions_t* opt, unsigned char move_files); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_snapshot_consistency( + rocksdb_ingestexternalfileoptions_t* opt, + unsigned char snapshot_consistency); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_allow_global_seqno( + rocksdb_ingestexternalfileoptions_t* opt, unsigned char allow_global_seqno); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_allow_blocking_flush( + rocksdb_ingestexternalfileoptions_t* opt, + unsigned char allow_blocking_flush); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_ingest_behind( + rocksdb_ingestexternalfileoptions_t* opt, unsigned char ingest_behind); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_fail_if_not_bottommost_level( + rocksdb_ingestexternalfileoptions_t* opt, + unsigned char fail_if_not_bottommost_level); + +extern ROCKSDB_LIBRARY_API void rocksdb_ingestexternalfileoptions_destroy( + rocksdb_ingestexternalfileoptions_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_ingest_external_file( + rocksdb_t* db, const char* const* file_list, const size_t list_len, + const rocksdb_ingestexternalfileoptions_t* opt, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_ingest_external_file_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* handle, + const char* const* file_list, const size_t list_len, + const rocksdb_ingestexternalfileoptions_t* opt, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_try_catch_up_with_primary( + rocksdb_t* db, char** errptr); + +/* SliceTransform */ + +extern ROCKSDB_LIBRARY_API rocksdb_slicetransform_t* +rocksdb_slicetransform_create( + void* state, void (*destructor)(void*), + char* (*transform)(void*, const char* key, size_t length, + size_t* dst_length), + unsigned char (*in_domain)(void*, const char* key, size_t length), + unsigned char (*in_range)(void*, const char* key, size_t length), + const char* (*name)(void*)); +extern ROCKSDB_LIBRARY_API rocksdb_slicetransform_t* + rocksdb_slicetransform_create_fixed_prefix(size_t); +extern ROCKSDB_LIBRARY_API rocksdb_slicetransform_t* +rocksdb_slicetransform_create_noop(void); +extern ROCKSDB_LIBRARY_API void rocksdb_slicetransform_destroy( + rocksdb_slicetransform_t*); + +/* Universal Compaction options */ + +enum { + rocksdb_similar_size_compaction_stop_style = 0, + rocksdb_total_size_compaction_stop_style = 1 +}; + +extern ROCKSDB_LIBRARY_API rocksdb_universal_compaction_options_t* +rocksdb_universal_compaction_options_create(void); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_size_ratio( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_size_ratio( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_min_merge_width( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_min_merge_width( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_max_merge_width( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_max_merge_width( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_max_size_amplification_percent( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_max_size_amplification_percent( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_compression_size_percent( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_compression_size_percent( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_stop_style( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_stop_style( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_universal_compaction_options_destroy( + rocksdb_universal_compaction_options_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_fifo_compaction_options_t* +rocksdb_fifo_compaction_options_create(void); +extern ROCKSDB_LIBRARY_API void +rocksdb_fifo_compaction_options_set_allow_compaction( + rocksdb_fifo_compaction_options_t* fifo_opts, + unsigned char allow_compaction); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_fifo_compaction_options_get_allow_compaction( + rocksdb_fifo_compaction_options_t* fifo_opts); +extern ROCKSDB_LIBRARY_API void +rocksdb_fifo_compaction_options_set_max_table_files_size( + rocksdb_fifo_compaction_options_t* fifo_opts, uint64_t size); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_fifo_compaction_options_get_max_table_files_size( + rocksdb_fifo_compaction_options_t* fifo_opts); +extern ROCKSDB_LIBRARY_API void rocksdb_fifo_compaction_options_destroy( + rocksdb_fifo_compaction_options_t* fifo_opts); + +extern ROCKSDB_LIBRARY_API int rocksdb_livefiles_count( + const rocksdb_livefiles_t*); +extern ROCKSDB_LIBRARY_API const char* rocksdb_livefiles_column_family_name( + const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API const char* rocksdb_livefiles_name( + const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API int rocksdb_livefiles_level( + const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_livefiles_size(const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API const char* rocksdb_livefiles_smallestkey( + const rocksdb_livefiles_t*, int index, size_t* size); +extern ROCKSDB_LIBRARY_API const char* rocksdb_livefiles_largestkey( + const rocksdb_livefiles_t*, int index, size_t* size); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_livefiles_entries(const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_livefiles_deletions(const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API void rocksdb_livefiles_destroy( + const rocksdb_livefiles_t*); + +/* Utility Helpers */ + +extern ROCKSDB_LIBRARY_API void rocksdb_get_options_from_string( + const rocksdb_options_t* base_options, const char* opts_str, + rocksdb_options_t* new_options, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_file_in_range( + rocksdb_t* db, const char* start_key, size_t start_key_len, + const char* limit_key, size_t limit_key_len, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_file_in_range_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* limit_key, + size_t limit_key_len, char** errptr); + +/* MetaData */ + +extern ROCKSDB_LIBRARY_API rocksdb_column_family_metadata_t* +rocksdb_get_column_family_metadata(rocksdb_t* db); + +/** + * Returns the rocksdb_column_family_metadata_t of the specified + * column family. + * + * Note that the caller is responsible to release the returned memory + * using rocksdb_column_family_metadata_destroy. + */ +extern ROCKSDB_LIBRARY_API rocksdb_column_family_metadata_t* +rocksdb_get_column_family_metadata_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family); + +extern ROCKSDB_LIBRARY_API void rocksdb_column_family_metadata_destroy( + rocksdb_column_family_metadata_t* cf_meta); + +extern ROCKSDB_LIBRARY_API uint64_t rocksdb_column_family_metadata_get_size( + rocksdb_column_family_metadata_t* cf_meta); + +extern ROCKSDB_LIBRARY_API size_t rocksdb_column_family_metadata_get_file_count( + rocksdb_column_family_metadata_t* cf_meta); + +extern ROCKSDB_LIBRARY_API char* rocksdb_column_family_metadata_get_name( + rocksdb_column_family_metadata_t* cf_meta); + +extern ROCKSDB_LIBRARY_API size_t +rocksdb_column_family_metadata_get_level_count( + rocksdb_column_family_metadata_t* cf_meta); + +/** + * Returns the rocksdb_level_metadata_t of the ith level from the specified + * column family metadata. + * + * If the specified i is greater than or equal to the number of levels + * in the specified column family, then NULL will be returned. + * + * Note that the caller is responsible to release the returned memory + * using rocksdb_level_metadata_destroy before releasing its parent + * rocksdb_column_family_metadata_t. + */ +extern ROCKSDB_LIBRARY_API rocksdb_level_metadata_t* +rocksdb_column_family_metadata_get_level_metadata( + rocksdb_column_family_metadata_t* cf_meta, size_t i); + +/** + * Releases the specified rocksdb_level_metadata_t. + * + * Note that the specified rocksdb_level_metadata_t must be released + * before the release of its parent rocksdb_column_family_metadata_t. + */ +extern ROCKSDB_LIBRARY_API void rocksdb_level_metadata_destroy( + rocksdb_level_metadata_t* level_meta); + +extern ROCKSDB_LIBRARY_API int rocksdb_level_metadata_get_level( + rocksdb_level_metadata_t* level_meta); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_level_metadata_get_size(rocksdb_level_metadata_t* level_meta); + +extern ROCKSDB_LIBRARY_API size_t +rocksdb_level_metadata_get_file_count(rocksdb_level_metadata_t* level_meta); + +/** + * Returns the sst_file_metadata_t of the ith file from the specified level + * metadata. + * + * If the specified i is greater than or equal to the number of files + * in the specified level, then NULL will be returned. + * + * Note that the caller is responsible to release the returned memory + * using rocksdb_sst_file_metadata_destroy before releasing its + * parent rocksdb_level_metadata_t. + */ +extern ROCKSDB_LIBRARY_API rocksdb_sst_file_metadata_t* +rocksdb_level_metadata_get_sst_file_metadata( + rocksdb_level_metadata_t* level_meta, size_t i); + +/** + * Releases the specified rocksdb_sst_file_metadata_t. + * + * Note that the specified rocksdb_sst_file_metadata_t must be released + * before the release of its parent rocksdb_level_metadata_t. + */ +extern ROCKSDB_LIBRARY_API void rocksdb_sst_file_metadata_destroy( + rocksdb_sst_file_metadata_t* file_meta); + +extern ROCKSDB_LIBRARY_API char* +rocksdb_sst_file_metadata_get_relative_filename( + rocksdb_sst_file_metadata_t* file_meta); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_sst_file_metadata_get_size(rocksdb_sst_file_metadata_t* file_meta); + +/** + * Returns the smallest key of the specified sst file. + * The caller is responsible for releasing the returned memory. + * + * @param file_meta the metadata of an SST file to obtain its smallest key. + * @param len the out value which will contain the length of the returned key + * after the function call. + */ +extern ROCKSDB_LIBRARY_API char* rocksdb_sst_file_metadata_get_smallestkey( + rocksdb_sst_file_metadata_t* file_meta, size_t* len); + +/** + * Returns the smallest key of the specified sst file. + * The caller is responsible for releasing the returned memory. + * + * @param file_meta the metadata of an SST file to obtain its smallest key. + * @param len the out value which will contain the length of the returned key + * after the function call. + */ +extern ROCKSDB_LIBRARY_API char* rocksdb_sst_file_metadata_get_largestkey( + rocksdb_sst_file_metadata_t* file_meta, size_t* len); + +/* Transactions */ + +extern ROCKSDB_LIBRARY_API rocksdb_column_family_handle_t* +rocksdb_transactiondb_create_column_family( + rocksdb_transactiondb_t* txn_db, + const rocksdb_options_t* column_family_options, + const char* column_family_name, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_transactiondb_t* rocksdb_transactiondb_open( + const rocksdb_options_t* options, + const rocksdb_transactiondb_options_t* txn_db_options, const char* name, + char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_transactiondb_t* +rocksdb_transactiondb_open_column_families( + const rocksdb_options_t* options, + const rocksdb_transactiondb_options_t* txn_db_options, const char* name, + int num_column_families, const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char** errptr); + +extern ROCKSDB_LIBRARY_API const rocksdb_snapshot_t* +rocksdb_transactiondb_create_snapshot(rocksdb_transactiondb_t* txn_db); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_release_snapshot( + rocksdb_transactiondb_t* txn_db, const rocksdb_snapshot_t* snapshot); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transactiondb_property_value( + rocksdb_transactiondb_t* db, const char* propname); + +extern ROCKSDB_LIBRARY_API int rocksdb_transactiondb_property_int( + rocksdb_transactiondb_t* db, const char* propname, uint64_t* out_val); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_transactiondb_get_base_db( + rocksdb_transactiondb_t* txn_db); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_close_base_db( + rocksdb_t* base_db); + +extern ROCKSDB_LIBRARY_API rocksdb_transaction_t* rocksdb_transaction_begin( + rocksdb_transactiondb_t* txn_db, + const rocksdb_writeoptions_t* write_options, + const rocksdb_transaction_options_t* txn_options, + rocksdb_transaction_t* old_txn); + +extern ROCKSDB_LIBRARY_API rocksdb_transaction_t** +rocksdb_transactiondb_get_prepared_transactions(rocksdb_transactiondb_t* txn_db, + size_t* cnt); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_set_name( + rocksdb_transaction_t* txn, const char* name, size_t name_len, + char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get_name( + rocksdb_transaction_t* txn, size_t* name_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_prepare( + rocksdb_transaction_t* txn, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_commit( + rocksdb_transaction_t* txn, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_rollback( + rocksdb_transaction_t* txn, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_set_savepoint( + rocksdb_transaction_t* txn); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_rollback_to_savepoint( + rocksdb_transaction_t* txn, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_destroy( + rocksdb_transaction_t* txn); + +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_wi_t* +rocksdb_transaction_get_writebatch_wi(rocksdb_transaction_t* txn); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_rebuild_from_writebatch( + rocksdb_transaction_t* txn, rocksdb_writebatch_t* writebatch, + char** errptr); + +// This rocksdb_writebatch_wi_t should be freed with rocksdb_free +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_rebuild_from_writebatch_wi( + rocksdb_transaction_t* txn, rocksdb_writebatch_wi_t* wi, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_set_commit_timestamp( + rocksdb_transaction_t* txn, uint64_t commit_timestamp); + +extern ROCKSDB_LIBRARY_API void +rocksdb_transaction_set_read_timestamp_for_validation( + rocksdb_transaction_t* txn, uint64_t read_timestamp); + +// This snapshot should be freed using rocksdb_free +extern ROCKSDB_LIBRARY_API const rocksdb_snapshot_t* +rocksdb_transaction_get_snapshot(rocksdb_transaction_t* txn); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + const char* key, size_t klen, size_t* vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transaction_get_pinned(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options, + const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, size_t klen, + size_t* vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transaction_get_pinned_cf(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get_for_update( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + const char* key, size_t klen, size_t* vlen, unsigned char exclusive, + char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transaction_get_pinned_for_update(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options, + const char* key, size_t klen, + unsigned char exclusive, + char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get_for_update_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, size_t klen, + size_t* vlen, unsigned char exclusive, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transaction_get_pinned_for_update_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, size_t klen, + unsigned char exclusive, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_multi_get( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_multi_get_for_update( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_multi_get_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + const rocksdb_column_family_handle_t* const* column_families, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_multi_get_for_update_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + const rocksdb_column_family_handle_t* const* column_families, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transactiondb_get( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + const char* key, size_t klen, size_t* vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transactiondb_get_pinned(rocksdb_transactiondb_t* txn_db, + const rocksdb_readoptions_t* options, + const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transactiondb_get_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transactiondb_get_pinned_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_multi_get( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_multi_get_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + const rocksdb_column_family_handle_t* const* column_families, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_put( + rocksdb_transaction_t* txn, const char* key, size_t klen, const char* val, + size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_put_cf( + rocksdb_transaction_t* txn, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_put( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + const char* key, size_t klen, const char* val, size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_put_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* val, size_t vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_write( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + rocksdb_writebatch_t* batch, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_merge( + rocksdb_transaction_t* txn, const char* key, size_t klen, const char* val, + size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_merge_cf( + rocksdb_transaction_t* txn, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_merge( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + const char* key, size_t klen, const char* val, size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_merge_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, size_t klen, + const char* val, size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_delete( + rocksdb_transaction_t* txn, const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_delete_cf( + rocksdb_transaction_t* txn, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_delete( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_delete_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_transaction_create_iterator(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_transaction_create_iterator_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_transactiondb_create_iterator(rocksdb_transactiondb_t* txn_db, + const rocksdb_readoptions_t* options); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_transactiondb_create_iterator_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_close( + rocksdb_transactiondb_t* txn_db); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_flush( + rocksdb_transactiondb_t* txn_db, const rocksdb_flushoptions_t* options, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_flush_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_flushoptions_t* options, + rocksdb_column_family_handle_t* column_family, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_flush_cfs( + rocksdb_transactiondb_t* txn_db, const rocksdb_flushoptions_t* options, + rocksdb_column_family_handle_t** column_families, int num_column_families, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_flush_wal( + rocksdb_transactiondb_t* txn_db, unsigned char sync, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_checkpoint_t* +rocksdb_transactiondb_checkpoint_object_create(rocksdb_transactiondb_t* txn_db, + char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_optimistictransactiondb_t* +rocksdb_optimistictransactiondb_open(const rocksdb_options_t* options, + const char* name, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_optimistictransactiondb_t* +rocksdb_optimistictransactiondb_open_column_families( + const rocksdb_options_t* options, const char* name, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* +rocksdb_optimistictransactiondb_get_base_db( + rocksdb_optimistictransactiondb_t* otxn_db); + +extern ROCKSDB_LIBRARY_API void rocksdb_optimistictransactiondb_close_base_db( + rocksdb_t* base_db); + +extern ROCKSDB_LIBRARY_API rocksdb_transaction_t* +rocksdb_optimistictransaction_begin( + rocksdb_optimistictransactiondb_t* otxn_db, + const rocksdb_writeoptions_t* write_options, + const rocksdb_optimistictransaction_options_t* otxn_options, + rocksdb_transaction_t* old_txn); + +extern ROCKSDB_LIBRARY_API void rocksdb_optimistictransactiondb_write( + rocksdb_optimistictransactiondb_t* otxn_db, + const rocksdb_writeoptions_t* options, rocksdb_writebatch_t* batch, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_optimistictransactiondb_close( + rocksdb_optimistictransactiondb_t* otxn_db); + +extern ROCKSDB_LIBRARY_API rocksdb_checkpoint_t* +rocksdb_optimistictransactiondb_checkpoint_object_create( + rocksdb_optimistictransactiondb_t* otxn_db, char** errptr); + +/* Transaction Options */ + +extern ROCKSDB_LIBRARY_API rocksdb_transactiondb_options_t* +rocksdb_transactiondb_options_create(void); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_options_destroy( + rocksdb_transactiondb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_options_set_max_num_locks( + rocksdb_transactiondb_options_t* opt, int64_t max_num_locks); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_options_set_num_stripes( + rocksdb_transactiondb_options_t* opt, size_t num_stripes); + +extern ROCKSDB_LIBRARY_API void +rocksdb_transactiondb_options_set_transaction_lock_timeout( + rocksdb_transactiondb_options_t* opt, int64_t txn_lock_timeout); + +extern ROCKSDB_LIBRARY_API void +rocksdb_transactiondb_options_set_default_lock_timeout( + rocksdb_transactiondb_options_t* opt, int64_t default_lock_timeout); + +extern ROCKSDB_LIBRARY_API rocksdb_transaction_options_t* +rocksdb_transaction_options_create(void); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_destroy( + rocksdb_transaction_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_set_set_snapshot( + rocksdb_transaction_options_t* opt, unsigned char v); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_set_deadlock_detect( + rocksdb_transaction_options_t* opt, unsigned char v); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_set_lock_timeout( + rocksdb_transaction_options_t* opt, int64_t lock_timeout); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_set_expiration( + rocksdb_transaction_options_t* opt, int64_t expiration); + +extern ROCKSDB_LIBRARY_API void +rocksdb_transaction_options_set_deadlock_detect_depth( + rocksdb_transaction_options_t* opt, int64_t depth); + +extern ROCKSDB_LIBRARY_API void +rocksdb_transaction_options_set_max_write_batch_size( + rocksdb_transaction_options_t* opt, size_t size); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_set_skip_prepare( + rocksdb_transaction_options_t* opt, unsigned char v); + +extern ROCKSDB_LIBRARY_API rocksdb_optimistictransaction_options_t* +rocksdb_optimistictransaction_options_create(void); + +extern ROCKSDB_LIBRARY_API void rocksdb_optimistictransaction_options_destroy( + rocksdb_optimistictransaction_options_t* opt); + +extern ROCKSDB_LIBRARY_API void +rocksdb_optimistictransaction_options_set_set_snapshot( + rocksdb_optimistictransaction_options_t* opt, unsigned char v); + +extern ROCKSDB_LIBRARY_API char* rocksdb_optimistictransactiondb_property_value( + rocksdb_optimistictransactiondb_t* db, const char* propname); + +extern ROCKSDB_LIBRARY_API int rocksdb_optimistictransactiondb_property_int( + rocksdb_optimistictransactiondb_t* db, const char* propname, + uint64_t* out_val); + +// referring to convention (3), this should be used by client +// to free memory that was malloc()ed +extern ROCKSDB_LIBRARY_API void rocksdb_free(void* ptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* rocksdb_get_pinned( + rocksdb_t* db, const rocksdb_readoptions_t* options, const char* key, + size_t keylen, char** errptr); +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* rocksdb_get_pinned_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_pinnableslice_destroy( + rocksdb_pinnableslice_t* v); +extern ROCKSDB_LIBRARY_API const char* rocksdb_pinnableslice_value( + const rocksdb_pinnableslice_t* t, size_t* vlen); + +extern ROCKSDB_LIBRARY_API rocksdb_memory_consumers_t* +rocksdb_memory_consumers_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_memory_consumers_add_db( + rocksdb_memory_consumers_t* consumers, rocksdb_t* db); +extern ROCKSDB_LIBRARY_API void rocksdb_memory_consumers_add_cache( + rocksdb_memory_consumers_t* consumers, rocksdb_cache_t* cache); +extern ROCKSDB_LIBRARY_API void rocksdb_memory_consumers_destroy( + rocksdb_memory_consumers_t* consumers); +extern ROCKSDB_LIBRARY_API rocksdb_memory_usage_t* +rocksdb_approximate_memory_usage_create(rocksdb_memory_consumers_t* consumers, + char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_approximate_memory_usage_destroy( + rocksdb_memory_usage_t* usage); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_approximate_memory_usage_get_mem_table_total( + rocksdb_memory_usage_t* memory_usage); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_approximate_memory_usage_get_mem_table_unflushed( + rocksdb_memory_usage_t* memory_usage); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_approximate_memory_usage_get_mem_table_readers_total( + rocksdb_memory_usage_t* memory_usage); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_approximate_memory_usage_get_cache_total( + rocksdb_memory_usage_t* memory_usage); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_dump_malloc_stats( + rocksdb_options_t*, unsigned char); + +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_memtable_whole_key_filtering(rocksdb_options_t*, + unsigned char); + +extern ROCKSDB_LIBRARY_API void rocksdb_cancel_all_background_work( + rocksdb_t* db, unsigned char wait); + +extern ROCKSDB_LIBRARY_API void rocksdb_disable_manual_compaction( + rocksdb_t* db); + +extern ROCKSDB_LIBRARY_API void rocksdb_enable_manual_compaction(rocksdb_t* db); + +extern ROCKSDB_LIBRARY_API rocksdb_statistics_histogram_data_t* +rocksdb_statistics_histogram_data_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_statistics_histogram_data_destroy( + rocksdb_statistics_histogram_data_t* data); +extern ROCKSDB_LIBRARY_API double rocksdb_statistics_histogram_data_get_median( + rocksdb_statistics_histogram_data_t* data); +extern ROCKSDB_LIBRARY_API double rocksdb_statistics_histogram_data_get_p95( + rocksdb_statistics_histogram_data_t* data); +extern ROCKSDB_LIBRARY_API double rocksdb_statistics_histogram_data_get_p99( + rocksdb_statistics_histogram_data_t* data); +extern ROCKSDB_LIBRARY_API double rocksdb_statistics_histogram_data_get_average( + rocksdb_statistics_histogram_data_t* data); +extern ROCKSDB_LIBRARY_API double rocksdb_statistics_histogram_data_get_std_dev( + rocksdb_statistics_histogram_data_t* data); +extern ROCKSDB_LIBRARY_API double rocksdb_statistics_histogram_data_get_max( + rocksdb_statistics_histogram_data_t* data); +extern ROCKSDB_LIBRARY_API uint64_t rocksdb_statistics_histogram_data_get_count( + rocksdb_statistics_histogram_data_t* data); +extern ROCKSDB_LIBRARY_API uint64_t rocksdb_statistics_histogram_data_get_sum( + rocksdb_statistics_histogram_data_t* data); +extern ROCKSDB_LIBRARY_API double rocksdb_statistics_histogram_data_get_min( + rocksdb_statistics_histogram_data_t* data); + +#ifdef __cplusplus +} /* end extern "C" */ +#endif diff --git a/librocksdb-sys/rocksdb/include/rocksdb/cache.h b/librocksdb-sys/rocksdb/include/rocksdb/cache.h new file mode 100644 index 0000000..550b458 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/cache.h @@ -0,0 +1,464 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Various APIs for configuring, creating, and monitoring read caches. + +#pragma once + +#include +#include +#include + +#include "rocksdb/compression_type.h" +#include "rocksdb/data_structure.h" +#include "rocksdb/memory_allocator.h" + +namespace ROCKSDB_NAMESPACE { + +class Cache; // defined in advanced_cache.h +struct ConfigOptions; +class SecondaryCache; + +// These definitions begin source compatibility for a future change in which +// a specific class for block cache is split away from general caches, so that +// the block cache API can continue to become more specialized and +// customizeable, including in ways incompatible with a general cache. For +// example, HyperClockCache is not usable as a general cache because it expects +// only fixed-size block cache keys, but this limitation is not yet reflected +// in the API function signatures. +// * Phase 1 (done) - Make both BlockCache and RowCache aliases for Cache, +// and make a factory function for row caches. Encourage users of row_cache +// (not common) to switch to the factory function for row caches. +// * Phase 2 - Split off RowCache as its own class, removing secondary +// cache support features and more from the API to simplify it. Between Phase 1 +// and Phase 2 users of row_cache will need to update their code. Any time +// after Phase 2, the block cache and row cache APIs can become more specialized +// in ways incompatible with general caches. +// * Phase 3 - Move existing RocksDB uses of Cache to BlockCache, and deprecate +// (but not yet remove) Cache as an alias for BlockCache. +using BlockCache = Cache; +using RowCache = Cache; + +// Classifications of block cache entries. +// +// Developer notes: Adding a new enum to this class requires corresponding +// updates to `kCacheEntryRoleToCamelString` and +// `kCacheEntryRoleToHyphenString`. Do not add to this enum after `kMisc` since +// `kNumCacheEntryRoles` assumes `kMisc` comes last. +enum class CacheEntryRole { + // Block-based table data block + kDataBlock, + // Block-based table filter block (full or partitioned) + kFilterBlock, + // Block-based table metadata block for partitioned filter + kFilterMetaBlock, + // OBSOLETE / DEPRECATED: old/removed block-based filter + kDeprecatedFilterBlock, + // Block-based table index block + kIndexBlock, + // Other kinds of block-based table block + kOtherBlock, + // WriteBufferManager's charge to account for its memtable usage + kWriteBuffer, + // Compression dictionary building buffer's charge to account for + // its memory usage + kCompressionDictionaryBuildingBuffer, + // Filter's charge to account for + // (new) bloom and ribbon filter construction's memory usage + kFilterConstruction, + // BlockBasedTableReader's charge to account for its memory usage + kBlockBasedTableReader, + // FileMetadata's charge to account for its memory usage + kFileMetadata, + // Blob value (when using the same cache as block cache and blob cache) + kBlobValue, + // Blob cache's charge to account for its memory usage (when using a + // separate block cache and blob cache) + kBlobCache, + // Default bucket, for miscellaneous cache entries. Do not use for + // entries that could potentially add up to large usage. + kMisc, +}; +constexpr uint32_t kNumCacheEntryRoles = + static_cast(CacheEntryRole::kMisc) + 1; + +// Obtain a hyphen-separated, lowercase name of a `CacheEntryRole`. +const std::string& GetCacheEntryRoleName(CacheEntryRole); + +// A fast bit set for CacheEntryRoles +using CacheEntryRoleSet = SmallEnumSet; + +// For use with `GetMapProperty()` for property +// `DB::Properties::kBlockCacheEntryStats`. On success, the map will +// be populated with all keys that can be obtained from these functions. +struct BlockCacheEntryStatsMapKeys { + static const std::string& CacheId(); + static const std::string& CacheCapacityBytes(); + static const std::string& LastCollectionDurationSeconds(); + static const std::string& LastCollectionAgeSeconds(); + + static std::string EntryCount(CacheEntryRole); + static std::string UsedBytes(CacheEntryRole); + static std::string UsedPercent(CacheEntryRole); +}; + +extern const bool kDefaultToAdaptiveMutex; + +enum CacheMetadataChargePolicy { + // Only the `charge` of each entry inserted into a Cache counts against + // the `capacity` + kDontChargeCacheMetadata, + // In addition to the `charge`, the approximate space overheads in the + // Cache (in bytes) also count against `capacity`. These space overheads + // are for supporting fast Lookup and managing the lifetime of entries. + kFullChargeCacheMetadata +}; +const CacheMetadataChargePolicy kDefaultCacheMetadataChargePolicy = + kFullChargeCacheMetadata; + +// Options shared betweeen various cache implementations that +// divide the key space into shards using hashing. +struct ShardedCacheOptions { + // Capacity of the cache, in the same units as the `charge` of each entry. + // This is typically measured in bytes, but can be a different unit if using + // kDontChargeCacheMetadata. + size_t capacity = 0; + + // Cache is sharded into 2^num_shard_bits shards, by hash of key. + // If < 0, a good default is chosen based on the capacity and the + // implementation. (Mutex-based implementations are much more reliant + // on many shards for parallel scalability.) + int num_shard_bits = -1; + + // If strict_capacity_limit is set, Insert() will fail if there is not + // enough capacity for the new entry along with all the existing referenced + // (pinned) cache entries. (Unreferenced cache entries are evicted as + // needed, sometimes immediately.) If strict_capacity_limit == false + // (default), Insert() never fails. + bool strict_capacity_limit = false; + + // If non-nullptr, RocksDB will use this allocator instead of system + // allocator when allocating memory for cache blocks. + // + // Caveat: when the cache is used as block cache, the memory allocator is + // ignored when dealing with compression libraries that allocate memory + // internally (currently only XPRESS). + std::shared_ptr memory_allocator; + + // See CacheMetadataChargePolicy + CacheMetadataChargePolicy metadata_charge_policy = + kDefaultCacheMetadataChargePolicy; + + // A SecondaryCache instance to use the non-volatile tier. For a RowCache + // this option must be kept as default empty. + std::shared_ptr secondary_cache; + + // See hash_seed comments below + static constexpr int32_t kQuasiRandomHashSeed = -1; + static constexpr int32_t kHostHashSeed = -2; + + // EXPERT OPTION: Specifies how a hash seed should be determined for the + // cache, or specifies a specific seed (only recommended for diagnostics or + // testing). + // + // Background: it could be dangerous to have different cache instances + // access the same SST files with the same hash seed, as correlated unlucky + // hashing across hosts or restarts could cause a widespread issue, rather + // than an isolated one. For example, with smaller block caches, it is + // possible for large full Bloom filters in a set of SST files to be randomly + // clustered into one cache shard, causing mutex contention or a thrashing + // condition as there's little or no space left for other entries assigned to + // the shard. If a set of SST files is broadcast and used on many hosts, we + // should ensure all have an independent chance of balanced shards. + // + // Values >= 0 will be treated as fixed hash seeds. Values < 0 are reserved + // for methods of dynamically choosing a seed, currently: + // * kQuasiRandomHashSeed - Each cache created chooses a seed mostly randomly, + // except that within a process, no seed is repeated until all have been + // issued. + // * kHostHashSeed - The seed is determined based on hashing the host name. + // Although this is arguably slightly worse for production reliability, it + // solves the essential problem of cross-host correlation while ensuring + // repeatable behavior on a host, for diagnostic purposes. + int32_t hash_seed = kHostHashSeed; + + ShardedCacheOptions() {} + ShardedCacheOptions( + size_t _capacity, int _num_shard_bits, bool _strict_capacity_limit, + std::shared_ptr _memory_allocator = nullptr, + CacheMetadataChargePolicy _metadata_charge_policy = + kDefaultCacheMetadataChargePolicy) + : capacity(_capacity), + num_shard_bits(_num_shard_bits), + strict_capacity_limit(_strict_capacity_limit), + memory_allocator(std::move(_memory_allocator)), + metadata_charge_policy(_metadata_charge_policy) {} + // Make ShardedCacheOptions polymorphic + virtual ~ShardedCacheOptions() = default; +}; + +// LRUCache - A cache using LRU eviction to stay at or below a set capacity. +// The cache is sharded to 2^num_shard_bits shards, by hash of the key. +// The total capacity is divided and evenly assigned to each shard, and each +// shard has its own LRU list for evictions. Each shard also has a mutex for +// exclusive access during operations; even read operations need exclusive +// access in order to update the LRU list. Mutex contention is usually low +// with enough shards. +struct LRUCacheOptions : public ShardedCacheOptions { + // Ratio of cache reserved for high-priority and low-priority entries, + // respectively. (See Cache::Priority below more information on the levels.) + // Valid values are between 0 and 1 (inclusive), and the sum of the two + // values cannot exceed 1. + // + // If high_pri_pool_ratio is greater than zero, a dedicated high-priority LRU + // list is maintained by the cache. A ratio of 0.5 means non-high-priority + // entries will use midpoint insertion. Similarly, if low_pri_pool_ratio is + // greater than zero, a dedicated low-priority LRU list is maintained. + // There is also a bottom-priority LRU list, which is always enabled and not + // explicitly configurable. Entries are spilled over to the next available + // lower-priority pool if a certain pool's capacity is exceeded. + // + // Entries with cache hits are inserted into the highest priority LRU list + // available regardless of the entry's priority. Entries without hits + // are inserted into highest priority LRU list available whose priority + // does not exceed the entry's priority. (For example, high-priority items + // with no hits are placed in the high-priority pool if available; + // otherwise, they are placed in the low-priority pool if available; + // otherwise, they are placed in the bottom-priority pool.) This results + // in lower-priority entries without hits getting evicted from the cache + // sooner. + double high_pri_pool_ratio = 0.5; + double low_pri_pool_ratio = 0.0; + + // Whether to use adaptive mutexes for cache shards. Note that adaptive + // mutexes need to be supported by the platform in order for this to have any + // effect. The default value is true if RocksDB is compiled with + // -DROCKSDB_DEFAULT_TO_ADAPTIVE_MUTEX, false otherwise. + bool use_adaptive_mutex = kDefaultToAdaptiveMutex; + + LRUCacheOptions() {} + LRUCacheOptions(size_t _capacity, int _num_shard_bits, + bool _strict_capacity_limit, double _high_pri_pool_ratio, + std::shared_ptr _memory_allocator = nullptr, + bool _use_adaptive_mutex = kDefaultToAdaptiveMutex, + CacheMetadataChargePolicy _metadata_charge_policy = + kDefaultCacheMetadataChargePolicy, + double _low_pri_pool_ratio = 0.0) + : ShardedCacheOptions(_capacity, _num_shard_bits, _strict_capacity_limit, + std::move(_memory_allocator), + _metadata_charge_policy), + high_pri_pool_ratio(_high_pri_pool_ratio), + low_pri_pool_ratio(_low_pri_pool_ratio), + use_adaptive_mutex(_use_adaptive_mutex) {} + + // Construct an instance of LRUCache using these options + std::shared_ptr MakeSharedCache() const; + + // Construct an instance of LRUCache for use as a row cache, typically for + // `DBOptions::row_cache`. Some options are not relevant to row caches. + std::shared_ptr MakeSharedRowCache() const; +}; + +// DEPRECATED wrapper function +inline std::shared_ptr NewLRUCache( + size_t capacity, int num_shard_bits = -1, + bool strict_capacity_limit = false, double high_pri_pool_ratio = 0.5, + std::shared_ptr memory_allocator = nullptr, + bool use_adaptive_mutex = kDefaultToAdaptiveMutex, + CacheMetadataChargePolicy metadata_charge_policy = + kDefaultCacheMetadataChargePolicy, + double low_pri_pool_ratio = 0.0) { + return LRUCacheOptions(capacity, num_shard_bits, strict_capacity_limit, + high_pri_pool_ratio, memory_allocator, + use_adaptive_mutex, metadata_charge_policy, + low_pri_pool_ratio) + .MakeSharedCache(); +} + +// DEPRECATED wrapper function +inline std::shared_ptr NewLRUCache(const LRUCacheOptions& cache_opts) { + return cache_opts.MakeSharedCache(); +} + +// EXPERIMENTAL +// Options structure for configuring a SecondaryCache instance with in-memory +// compression. The implementation uses LRUCache so inherits its options, +// except LRUCacheOptions.secondary_cache is not used and should not be set. +struct CompressedSecondaryCacheOptions : LRUCacheOptions { + // The compression method (if any) that is used to compress data. + CompressionType compression_type = CompressionType::kLZ4Compression; + + // compress_format_version can have two values: + // compress_format_version == 1 -- decompressed size is not included in the + // block header. + // compress_format_version == 2 -- decompressed size is included in the block + // header in varint32 format. + uint32_t compress_format_version = 2; + + // Enable the custom split and merge feature, which split the compressed value + // into chunks so that they may better fit jemalloc bins. + bool enable_custom_split_merge = false; + + // Kinds of entries that should not be compressed, but can be stored. + // (Filter blocks are essentially non-compressible but others usually are.) + CacheEntryRoleSet do_not_compress_roles = {CacheEntryRole::kFilterBlock}; + + CompressedSecondaryCacheOptions() {} + CompressedSecondaryCacheOptions( + size_t _capacity, int _num_shard_bits, bool _strict_capacity_limit, + double _high_pri_pool_ratio, double _low_pri_pool_ratio = 0.0, + std::shared_ptr _memory_allocator = nullptr, + bool _use_adaptive_mutex = kDefaultToAdaptiveMutex, + CacheMetadataChargePolicy _metadata_charge_policy = + kDefaultCacheMetadataChargePolicy, + CompressionType _compression_type = CompressionType::kLZ4Compression, + uint32_t _compress_format_version = 2, + bool _enable_custom_split_merge = false, + const CacheEntryRoleSet& _do_not_compress_roles = + {CacheEntryRole::kFilterBlock}) + : LRUCacheOptions(_capacity, _num_shard_bits, _strict_capacity_limit, + _high_pri_pool_ratio, std::move(_memory_allocator), + _use_adaptive_mutex, _metadata_charge_policy, + _low_pri_pool_ratio), + compression_type(_compression_type), + compress_format_version(_compress_format_version), + enable_custom_split_merge(_enable_custom_split_merge), + do_not_compress_roles(_do_not_compress_roles) {} + + // Construct an instance of CompressedSecondaryCache using these options + std::shared_ptr MakeSharedSecondaryCache() const; + + // Avoid confusion with LRUCache + std::shared_ptr MakeSharedCache() const = delete; +}; + +// DEPRECATED wrapper function +inline std::shared_ptr NewCompressedSecondaryCache( + size_t capacity, int num_shard_bits = -1, + bool strict_capacity_limit = false, double high_pri_pool_ratio = 0.5, + double low_pri_pool_ratio = 0.0, + std::shared_ptr memory_allocator = nullptr, + bool use_adaptive_mutex = kDefaultToAdaptiveMutex, + CacheMetadataChargePolicy metadata_charge_policy = + kDefaultCacheMetadataChargePolicy, + CompressionType compression_type = CompressionType::kLZ4Compression, + uint32_t compress_format_version = 2, + bool enable_custom_split_merge = false, + const CacheEntryRoleSet& _do_not_compress_roles = { + CacheEntryRole::kFilterBlock}) { + return CompressedSecondaryCacheOptions( + capacity, num_shard_bits, strict_capacity_limit, + high_pri_pool_ratio, low_pri_pool_ratio, memory_allocator, + use_adaptive_mutex, metadata_charge_policy, compression_type, + compress_format_version, enable_custom_split_merge, + _do_not_compress_roles) + .MakeSharedSecondaryCache(); +} + +// DEPRECATED wrapper function +inline std::shared_ptr NewCompressedSecondaryCache( + const CompressedSecondaryCacheOptions& opts) { + return opts.MakeSharedSecondaryCache(); +} + +// HyperClockCache - A lock-free Cache alternative for RocksDB block cache +// that offers much improved CPU efficiency vs. LRUCache under high parallel +// load or high contention, with some caveats: +// * Not a general Cache implementation: can only be used for +// BlockBasedTableOptions::block_cache, which RocksDB uses in a way that is +// compatible with HyperClockCache. +// * Requires an extra tuning parameter: see estimated_entry_charge below. +// Similarly, substantially changing the capacity with SetCapacity could +// harm efficiency. +// * Cache priorities are less aggressively enforced, which could cause +// cache dilution from long range scans (unless they use fill_cache=false). +// * Can be worse for small caches, because if almost all of a cache shard is +// pinned (more likely with non-partitioned filters), then CLOCK eviction +// becomes very CPU intensive. +// +// See internal cache/clock_cache.h for full description. +struct HyperClockCacheOptions : public ShardedCacheOptions { + // The estimated average `charge` associated with cache entries. This is a + // critical configuration parameter for good performance from the hyper + // cache, because having a table size that is fixed at creation time greatly + // reduces the required synchronization between threads. + // * If the estimate is substantially too low (e.g. less than half the true + // average) then metadata space overhead with be substantially higher (e.g. + // 200 bytes per entry rather than 100). With kFullChargeCacheMetadata, this + // can slightly reduce cache hit rates, and slightly reduce access times due + // to the larger working memory size. + // * If the estimate is substantially too high (e.g. 25% higher than the true + // average) then there might not be sufficient slots in the hash table for + // both efficient operation and capacity utilization (hit rate). The hyper + // cache will evict entries to prevent load factors that could dramatically + // affect lookup times, instead letting the hit rate suffer by not utilizing + // the full capacity. + // + // A reasonable choice is the larger of block_size and metadata_block_size. + // When WriteBufferManager (and similar) charge memory usage to the block + // cache, this can lead to the same effect as estimate being too low, which + // is better than the opposite. Therefore, the general recommendation is to + // assume that other memory charged to block cache could be negligible, and + // ignore it in making the estimate. + // + // The best parameter choice based on a cache in use is given by + // GetUsage() / GetOccupancyCount(), ignoring metadata overheads such as + // with kDontChargeCacheMetadata. More precisely with + // kFullChargeCacheMetadata is (GetUsage() - 64 * GetTableAddressCount()) / + // GetOccupancyCount(). However, when the average value size might vary + // (e.g. balance between metadata and data blocks in cache), it is better + // to estimate toward the lower side than the higher side. + size_t estimated_entry_charge; + + HyperClockCacheOptions( + size_t _capacity, size_t _estimated_entry_charge, + int _num_shard_bits = -1, bool _strict_capacity_limit = false, + std::shared_ptr _memory_allocator = nullptr, + CacheMetadataChargePolicy _metadata_charge_policy = + kDefaultCacheMetadataChargePolicy) + : ShardedCacheOptions(_capacity, _num_shard_bits, _strict_capacity_limit, + std::move(_memory_allocator), + _metadata_charge_policy), + estimated_entry_charge(_estimated_entry_charge) {} + + // Construct an instance of HyperClockCache using these options + std::shared_ptr MakeSharedCache() const; +}; + +// DEPRECATED - The old Clock Cache implementation had an unresolved bug and +// has been removed. The new HyperClockCache requires an additional +// configuration parameter that is not provided by this API. This function +// simply returns a new LRUCache for functional compatibility. +extern std::shared_ptr NewClockCache( + size_t capacity, int num_shard_bits = -1, + bool strict_capacity_limit = false, + CacheMetadataChargePolicy metadata_charge_policy = + kDefaultCacheMetadataChargePolicy); + +enum PrimaryCacheType { + kCacheTypeLRU, // LRU cache type + kCacheTypeHCC, // Hyper Clock Cache type + kCacheTypeMax, +}; + +// A 2-tier cache with a primary block cache, and a compressed secondary +// cache. The returned cache instance will internally allocate a primary +// uncompressed cache of the specified type, and a compressed secondary +// cache. Any cache memory reservations, such as WriteBufferManager +// allocations costed to the block cache, will be distributed +// proportionally across both the primary and secondary. +struct TieredVolatileCacheOptions { + ShardedCacheOptions* cache_opts; + PrimaryCacheType cache_type; + CompressedSecondaryCacheOptions comp_cache_opts; +}; + +extern std::shared_ptr NewTieredVolatileCache( + TieredVolatileCacheOptions& cache_opts); +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/cache_bench_tool.h b/librocksdb-sys/rocksdb/include/rocksdb/cache_bench_tool.h new file mode 100644 index 0000000..413ce15 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/cache_bench_tool.h @@ -0,0 +1,14 @@ +// Copyright (c) 2013-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/status.h" +#include "rocksdb/types.h" + +namespace ROCKSDB_NAMESPACE { + +int cache_bench_tool(int argc, char** argv); +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/cleanable.h b/librocksdb-sys/rocksdb/include/rocksdb/cleanable.h new file mode 100644 index 0000000..afc7366 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/cleanable.h @@ -0,0 +1,128 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +class Cleanable { + public: + Cleanable(); + // No copy constructor and copy assignment allowed. + Cleanable(Cleanable&) = delete; + Cleanable& operator=(Cleanable&) = delete; + + // Executes all the registered cleanups + ~Cleanable(); + + // Move constructor and move assignment is allowed. + Cleanable(Cleanable&&) noexcept; + Cleanable& operator=(Cleanable&&) noexcept; + + // Clients are allowed to register function/arg1/arg2 triples that + // will be invoked when this iterator is destroyed. + // + // Note that unlike all of the preceding methods, this method is + // not abstract and therefore clients should not override it. + using CleanupFunction = void (*)(void* arg1, void* arg2); + + // Add another Cleanup to the list + void RegisterCleanup(CleanupFunction function, void* arg1, void* arg2); + + // Move the cleanups owned by this Cleanable to another Cleanable, adding to + // any existing cleanups it has + void DelegateCleanupsTo(Cleanable* other); + + // DoCleanup and also resets the pointers for reuse + inline void Reset() { + DoCleanup(); + cleanup_.function = nullptr; + cleanup_.next = nullptr; + } + + inline bool HasCleanups() { return cleanup_.function != nullptr; } + + protected: + struct Cleanup { + CleanupFunction function; + void* arg1; + void* arg2; + Cleanup* next; + }; + Cleanup cleanup_; + // It also becomes the owner of c + void RegisterCleanup(Cleanup* c); + + private: + // Performs all the cleanups. It does not reset the pointers. Making it + // private + // to prevent misuse + inline void DoCleanup() { + if (cleanup_.function != nullptr) { + (*cleanup_.function)(cleanup_.arg1, cleanup_.arg2); + for (Cleanup* c = cleanup_.next; c != nullptr;) { + (*c->function)(c->arg1, c->arg2); + Cleanup* next = c->next; + delete c; + c = next; + } + } + } +}; + +// A copyable, reference-counted pointer to a simple Cleanable that only +// performs registered cleanups after all copies are destroy. This is like +// shared_ptr but works more efficiently with wrapping the pointer +// in an outer Cleanable (see RegisterCopyWith() and MoveAsCleanupTo()). +// WARNING: if you create a reference cycle, for example: +// SharedCleanablePtr scp; +// scp.Allocate(); +// scp.RegisterCopyWith(&*scp); +// It will prevent cleanups from ever happening! +class SharedCleanablePtr { + public: + // Empy/null pointer + SharedCleanablePtr() {} + // Copy and move constructors and assignment + SharedCleanablePtr(const SharedCleanablePtr& from); + SharedCleanablePtr(SharedCleanablePtr&& from) noexcept; + SharedCleanablePtr& operator=(const SharedCleanablePtr& from); + SharedCleanablePtr& operator=(SharedCleanablePtr&& from) noexcept; + // Destructor (decrement refcount if non-null) + ~SharedCleanablePtr(); + // Create a new simple Cleanable and make this assign this pointer to it. + // (Reset()s first if necessary.) + void Allocate(); + // Reset to empty/null (decrement refcount if previously non-null) + void Reset(); + // Dereference to pointed-to Cleanable + Cleanable& operator*(); + Cleanable* operator->(); + // Get as raw pointer to Cleanable + Cleanable* get(); + + // Creates a (virtual) copy of this SharedCleanablePtr and registers its + // destruction with target, so that the cleanups registered with the + // Cleanable pointed to by this can only happen after the cleanups in the + // target Cleanable are run. + // No-op if this is empty (nullptr). + void RegisterCopyWith(Cleanable* target); + + // Moves (virtually) this shared pointer to a new cleanup in the target. + // This is essentilly a move semantics version of RegisterCopyWith(), for + // performance optimization. No-op if this is empty (nullptr). + void MoveAsCleanupTo(Cleanable* target); + + private: + struct Impl; + Impl* ptr_ = nullptr; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/compaction_filter.h b/librocksdb-sys/rocksdb/include/rocksdb/compaction_filter.h new file mode 100644 index 0000000..b1b5116 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/compaction_filter.h @@ -0,0 +1,363 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2013 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include +#include +#include +#include +#include + +#include "rocksdb/customizable.h" +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/types.h" +#include "rocksdb/wide_columns.h" + +namespace ROCKSDB_NAMESPACE { + +class Slice; +class SliceTransform; + +// CompactionFilter allows an application to modify/delete a key-value during +// table file creation. +// +// Some general notes: +// +// * RocksDB snapshots do not guarantee to preserve the state of the DB in the +// presence of CompactionFilter. Data seen from a snapshot might disappear after +// a table file created with a `CompactionFilter` is installed. If you use +// snapshots, think twice about whether you want to use `CompactionFilter` and +// whether you are using it in a safe way. +// +// * If multithreaded compaction is being used *and* a single CompactionFilter +// instance was supplied via Options::compaction_filter, CompactionFilter +// methods may be called from different threads concurrently. The application +// must ensure that such calls are thread-safe. If the CompactionFilter was +// created by a factory, then it will only ever be used by a single thread that +// is doing the table file creation, and this call does not need to be +// thread-safe. However, multiple filters may be in existence and operating +// concurrently. +// +// * The key passed to the filtering methods includes the timestamp if +// user-defined timestamps are enabled. +// +// * Exceptions MUST NOT propagate out of overridden functions into RocksDB, +// because RocksDB is not exception-safe. This could cause undefined behavior +// including data loss, unreported corruption, deadlocks, and more. +class CompactionFilter : public Customizable { + public: + // Value type of the key-value passed to the compaction filter's FilterV2/V3 + // methods. + enum ValueType { + // Plain key-value + kValue, + // Merge operand + kMergeOperand, + // Used internally by the old stacked BlobDB implementation; this value type + // is never passed to application code. Note that when using the new + // integrated BlobDB, values stored separately as blobs are retrieved and + // presented to FilterV2/V3 with the type kValue above. + kBlobIndex, + // Wide-column entity + kWideColumnEntity, + }; + + // Potential decisions that can be returned by the compaction filter's + // FilterV2/V3 and FilterBlobByKey methods. See decision-specific caveats and + // constraints below. + enum class Decision { + // Keep the current key-value as-is. + kKeep, + + // Remove the current key-value. Note that the semantics of removal are + // dependent on the value type. If the current key-value is a plain + // key-value or a wide-column entity, it is converted to a tombstone + // (Delete), resulting in the deletion of any earlier versions of the key. + // If it is a merge operand, it is simply dropped. Note: if you are using + // a TransactionDB, it is not recommended to filter out merge operands. + // If a Merge operation is filtered out, TransactionDB may not realize there + // is a write conflict and may allow a Transaction that should have failed + // to Commit. Instead, it is better to implement any Merge filtering inside + // the MergeOperator. + kRemove, + + // Change the value of the current key-value. If the current key-value is a + // plain key-value or a merge operand, its value is updated but its value + // type remains the same. If the current key-value is a wide-column entity, + // it is converted to a plain key-value with the new value specified. + kChangeValue, + + // Remove all key-values with key in [key, *skip_until). This range of keys + // will be skipped in a way that potentially avoids some IO operations + // compared to removing the keys one by one. Note that removal in this case + // means dropping the key-value regardless of value type; in other words, in + // contrast with kRemove, plain values and entities are not converted to + // tombstones. + // + // *skip_until <= key is treated the same as Decision::kKeep (since the + // range [key, *skip_until) is empty). + // + // Caveats: + // * The keys are skipped even if there are snapshots containing them, + // i.e. values removed by kRemoveAndSkipUntil can disappear from a + // snapshot - beware if you're using TransactionDB or DB::GetSnapshot(). + // * If value for a key was overwritten or merged into (multiple Put()s + // or Merge()s), and `CompactionFilter` skips this key with + // kRemoveAndSkipUntil, it's possible that it will remove only + // the new value, exposing the old value that was supposed to be + // overwritten. + // * Doesn't work with PlainTableFactory in prefix mode. + // * If you use kRemoveAndSkipUntil for table files created by compaction, + // consider also reducing compaction_readahead_size option. + kRemoveAndSkipUntil, + + // Used internally by the old stacked BlobDB implementation. Returning this + // decision from application code is not supported. + kChangeBlobIndex, + + // Used internally by the old stacked BlobDB implementation. Returning this + // decision from application code is not supported. + kIOError, + + // Remove the current key-value by converting it to a SingleDelete-type + // tombstone. Only supported for plain-key values and wide-column entities; + // not supported for merge operands. All the caveats related to + // SingleDeletes apply. + kPurge, + + // Change the current key-value to the wide-column entity specified. If the + // current key-value is already a wide-column entity, only its columns are + // updated; if it is a plain key-value, it is converted to a wide-column + // entity with the specified columns. Not supported for merge operands. + // Only applicable to FilterV3. + kChangeWideColumnEntity, + + // When using the integrated BlobDB implementation, it may be possible for + // applications to make a filtering decision for a given blob based on + // the key only without actually reading the blob value, which saves some + // I/O; see the FilterBlobByKey method below. Returning kUndetermined from + // FilterBlobByKey signals that making a decision solely based on the + // key is not possible; in this case, RocksDB reads the blob value and + // passes the key-value to the regular filtering method. Only applicable to + // FilterBlobByKey; returning this value from FilterV2/V3 is not supported. + kUndetermined, + }; + + // Used internally by the old stacked BlobDB implementation. + enum class BlobDecision { kKeep, kChangeValue, kCorruption, kIOError }; + + // Context information for a table file creation. + struct Context { + // Whether this table file is created as part of a compaction including all + // table files. + bool is_full_compaction; + // Whether this table file is created as part of a compaction requested by + // the client. + bool is_manual_compaction; + // The column family that will contain the created table file. + uint32_t column_family_id; + // Reason this table file is being created. + TableFileCreationReason reason; + }; + + virtual ~CompactionFilter() {} + static const char* Type() { return "CompactionFilter"; } + static Status CreateFromString(const ConfigOptions& config_options, + const std::string& name, + const CompactionFilter** result); + + // The table file creation process invokes this method before adding a kv to + // the table file. A return value of false indicates that the kv should be + // preserved in the new table file and a return value of true indicates + // that this key-value should be removed (that is, converted to a tombstone). + // The application can inspect the existing value of the key and make decision + // based on it. + // + // Key-Values that are results of merge operation during table file creation + // are not passed into this function. Currently, when you have a mix of Put()s + // and Merge()s on a same key, we only guarantee to process the merge operands + // through the `CompactionFilter`s. Put()s might be processed, or might not. + // + // When the value is to be preserved, the application has the option + // to modify the existing_value and pass it back through new_value. + // value_changed needs to be set to true in this case. + virtual bool Filter(int /*level*/, const Slice& /*key*/, + const Slice& /*existing_value*/, + std::string* /*new_value*/, + bool* /*value_changed*/) const { + return false; + } + + // The table file creation process invokes this method on every merge operand. + // If this method returns true, the merge operand will be ignored and not + // written out in the new table file. + // + // Note: If you are using a TransactionDB, it is not recommended to implement + // FilterMergeOperand(). If a Merge operation is filtered out, TransactionDB + // may not realize there is a write conflict and may allow a Transaction to + // Commit that should have failed. Instead, it is better to implement any + // Merge filtering inside the MergeOperator. + virtual bool FilterMergeOperand(int /*level*/, const Slice& /*key*/, + const Slice& /*operand*/) const { + return false; + } + + // A unified API for plain values and merge operands that may + // return a variety of decisions (see Decision above). The `value_type` + // parameter indicates the type of the key-value and the `existing_value` + // contains the current value or merge operand. The `new_value` output + // parameter can be used to set the updated value or merge operand when the + // kChangeValue decision is made by the filter. See the description of + // kRemoveAndSkipUntil above for the semantics of the `skip_until` output + // parameter, and see Decision above for more information on the semantics of + // the potential return values. + // + // The default implementation uses Filter() and FilterMergeOperand(). + // If you're overriding this method, no need to override the other two. + virtual Decision FilterV2(int level, const Slice& key, ValueType value_type, + const Slice& existing_value, std::string* new_value, + std::string* /*skip_until*/) const { + switch (value_type) { + case ValueType::kValue: { + bool value_changed = false; + bool rv = Filter(level, key, existing_value, new_value, &value_changed); + if (rv) { + return Decision::kRemove; + } + return value_changed ? Decision::kChangeValue : Decision::kKeep; + } + + case ValueType::kMergeOperand: { + bool rv = FilterMergeOperand(level, key, existing_value); + return rv ? Decision::kRemove : Decision::kKeep; + } + + case ValueType::kBlobIndex: + return Decision::kKeep; + + default: + assert(false); + return Decision::kKeep; + } + } + + // Wide column aware unified API. Called for plain values, merge operands, and + // wide-column entities; the `value_type` parameter indicates the type of the + // key-value. When the key-value is a plain value or a merge operand, the + // `existing_value` parameter contains the existing value and the + // `existing_columns` parameter is invalid (nullptr). When the key-value is a + // wide-column entity, the `existing_columns` parameter contains the wide + // columns of the existing entity and the `existing_value` parameter is + // invalid (nullptr). The `new_value` output parameter can be used to set the + // updated value or merge operand when the kChangeValue decision is made by + // the filter. The `new_columns` output parameter can be used to specify + // the pairs of column names and column values when the + // kChangeWideColumnEntity decision is returned. See the description of + // kRemoveAndSkipUntil above for the semantics of the `skip_until` output + // parameter, and see Decision above for more information on the semantics of + // the potential return values. + // + // For compatibility, the default implementation keeps all wide-column + // entities, and falls back to FilterV2 for plain values and merge operands. + // If you override this method, there is no need to override FilterV2 (or + // Filter/FilterMergeOperand). + virtual Decision FilterV3( + int level, const Slice& key, ValueType value_type, + const Slice* existing_value, const WideColumns* existing_columns, + std::string* new_value, + std::vector>* /* new_columns */, + std::string* skip_until) const { +#ifdef NDEBUG + (void)existing_columns; +#endif + + assert(!existing_value || !existing_columns); + assert(value_type == ValueType::kWideColumnEntity || existing_value); + assert(value_type != ValueType::kWideColumnEntity || existing_columns); + + if (value_type == ValueType::kWideColumnEntity) { + return Decision::kKeep; + } + + return FilterV2(level, key, value_type, *existing_value, new_value, + skip_until); + } + + // Internal (BlobDB) use only. Do not override in application code. + virtual BlobDecision PrepareBlobOutput(const Slice& /* key */, + const Slice& /* existing_value */, + std::string* /* new_value */) const { + return BlobDecision::kKeep; + } + + // This function is deprecated. Snapshots will always be ignored for + // `CompactionFilter`s, because we realized that not ignoring snapshots + // doesn't provide the guarantee we initially thought it would provide. + // Repeatable reads will not be guaranteed anyway. If you override the + // function and returns false, we will fail the table file creation. + virtual bool IgnoreSnapshots() const { return true; } + + // Returns a name that identifies this `CompactionFilter`. + // The name will be printed to LOG file on start up for diagnosis. + const char* Name() const override = 0; + + // Internal (BlobDB) use only. Do not override in application code. + virtual bool IsStackedBlobDbInternalCompactionFilter() const { return false; } + + // In the case of BlobDB, it may be possible to reach a decision with only + // the key without reading the actual value, saving some I/O operations. + // Keys where the value is stored separately in a blob file will be + // passed to this method. If the method returns a supported decision other + // than kUndetermined, it will be considered final and performed without + // reading the existing value. Returning kUndetermined will cause FilterV3() + // to be called to make a decision as usual. The output parameters + // `new_value` and `skip_until` are applicable to the decisions kChangeValue + // and kRemoveAndSkipUntil respectively, and have the same semantics as + // the corresponding parameters of FilterV2/V3. + virtual Decision FilterBlobByKey(int /*level*/, const Slice& /*key*/, + std::string* /*new_value*/, + std::string* /*skip_until*/) const { + return Decision::kUndetermined; + } +}; + +// Each thread of work involving creating table files will create a new +// `CompactionFilter` according to `ShouldFilterTableFileCreation()`. This +// allows the application to know about the different ongoing threads of work +// and makes it unnecessary for `CompactionFilter` to provide thread-safety. +// +// Exceptions MUST NOT propagate out of overridden functions into RocksDB, +// because RocksDB is not exception-safe. This could cause undefined behavior +// including data loss, unreported corruption, deadlocks, and more. +class CompactionFilterFactory : public Customizable { + public: + virtual ~CompactionFilterFactory() {} + static const char* Type() { return "CompactionFilterFactory"; } + static Status CreateFromString( + const ConfigOptions& config_options, const std::string& name, + std::shared_ptr* result); + + // Returns whether a thread creating table files for the specified `reason` + // should invoke `CreateCompactionFilter()` and pass KVs through the returned + // filter. + virtual bool ShouldFilterTableFileCreation( + TableFileCreationReason reason) const { + // For backward compatibility, default implementation only applies + // `CompactionFilter` to files generated by compaction. + return reason == TableFileCreationReason::kCompaction; + } + + virtual std::unique_ptr CreateCompactionFilter( + const CompactionFilter::Context& context) = 0; + + // Returns a name that identifies this `CompactionFilter` factory. + virtual const char* Name() const override = 0; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/compaction_job_stats.h b/librocksdb-sys/rocksdb/include/rocksdb/compaction_job_stats.h new file mode 100644 index 0000000..7e81530 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/compaction_job_stats.h @@ -0,0 +1,112 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once +#include +#include + +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { +struct CompactionJobStats { + CompactionJobStats() { Reset(); } + void Reset(); + // Aggregate the CompactionJobStats from another instance with this one + void Add(const CompactionJobStats& stats); + + // the elapsed time of this compaction in microseconds. + uint64_t elapsed_micros; + + // the elapsed CPU time of this compaction in microseconds. + uint64_t cpu_micros; + + // Used internally indicating whether a subcompaction's + // `num_input_records` is accurate. + bool has_num_input_records; + // the number of compaction input records. + uint64_t num_input_records; + // the number of blobs read from blob files + uint64_t num_blobs_read; + // the number of compaction input files (table files) + size_t num_input_files; + // the number of compaction input files at the output level (table files) + size_t num_input_files_at_output_level; + + // the number of compaction output records. + uint64_t num_output_records; + // the number of compaction output files (table files) + size_t num_output_files; + // the number of compaction output files (blob files) + size_t num_output_files_blob; + + // true if the compaction is a full compaction (all live SST files input) + bool is_full_compaction; + // true if the compaction is a manual compaction + bool is_manual_compaction; + + // the total size of table files in the compaction input + uint64_t total_input_bytes; + // the total size of blobs read from blob files + uint64_t total_blob_bytes_read; + // the total size of table files in the compaction output + uint64_t total_output_bytes; + // the total size of blob files in the compaction output + uint64_t total_output_bytes_blob; + + // number of records being replaced by newer record associated with same key. + // this could be a new value or a deletion entry for that key so this field + // sums up all updated and deleted keys + uint64_t num_records_replaced; + + // the sum of the uncompressed input keys in bytes. + uint64_t total_input_raw_key_bytes; + // the sum of the uncompressed input values in bytes. + uint64_t total_input_raw_value_bytes; + + // the number of deletion entries before compaction. Deletion entries + // can disappear after compaction because they expired + uint64_t num_input_deletion_records; + // number of deletion records that were found obsolete and discarded + // because it is not possible to delete any more keys with this entry + // (i.e. all possible deletions resulting from it have been completed) + uint64_t num_expired_deletion_records; + + // number of corrupt keys (ParseInternalKey returned false when applied to + // the key) encountered and written out. + uint64_t num_corrupt_keys; + + // Following counters are only populated if + // options.report_bg_io_stats = true; + + // Time spent on file's Append() call. + uint64_t file_write_nanos; + + // Time spent on sync file range. + uint64_t file_range_sync_nanos; + + // Time spent on file fsync. + uint64_t file_fsync_nanos; + + // Time spent on preparing file write (fallocate, etc) + uint64_t file_prepare_write_nanos; + + // 0-terminated strings storing the first 8 bytes of the smallest and + // largest key in the output. + static const size_t kMaxPrefixLength = 8; + + std::string smallest_output_key_prefix; + std::string largest_output_key_prefix; + + // number of single-deletes which do not meet a put + uint64_t num_single_del_fallthru; + + // number of single-deletes which meet something other than a put + uint64_t num_single_del_mismatch; + + // TODO: Add output_to_penultimate_level output information +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/comparator.h b/librocksdb-sys/rocksdb/include/rocksdb/comparator.h new file mode 100644 index 0000000..ad1e71a --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/comparator.h @@ -0,0 +1,164 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include + +#include "rocksdb/customizable.h" +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +class Slice; + +// The general interface for comparing two Slices are defined for both of +// Comparator and some internal data structures. +class CompareInterface { + public: + virtual ~CompareInterface() {} + + // Three-way comparison. Returns value: + // < 0 iff "a" < "b", + // == 0 iff "a" == "b", + // > 0 iff "a" > "b" + // Note that Compare(a, b) also compares timestamp if timestamp size is + // non-zero. For the same user key with different timestamps, larger (newer) + // timestamp comes first. + virtual int Compare(const Slice& a, const Slice& b) const = 0; +}; + +// A Comparator object provides a total order across slices that are +// used as keys in an sstable or a database. A Comparator implementation +// must be thread-safe since rocksdb may invoke its methods concurrently +// from multiple threads. +// +// Exceptions MUST NOT propagate out of overridden functions into RocksDB, +// because RocksDB is not exception-safe. This could cause undefined behavior +// including data loss, unreported corruption, deadlocks, and more. +class Comparator : public Customizable, public CompareInterface { + public: + Comparator() : timestamp_size_(0) {} + + Comparator(size_t ts_sz) : timestamp_size_(ts_sz) {} + + Comparator(const Comparator& orig) : timestamp_size_(orig.timestamp_size_) {} + + Comparator& operator=(const Comparator& rhs) { + if (this != &rhs) { + timestamp_size_ = rhs.timestamp_size_; + } + return *this; + } + + ~Comparator() override {} + + static Status CreateFromString(const ConfigOptions& opts, + const std::string& id, + const Comparator** comp); + static const char* Type() { return "Comparator"; } + + // The name of the comparator. Used to check for comparator + // mismatches (i.e., a DB created with one comparator is + // accessed using a different comparator. + // + // The client of this package should switch to a new name whenever + // the comparator implementation changes in a way that will cause + // the relative ordering of any two keys to change. + // + // Names starting with "rocksdb." are reserved and should not be used + // by any clients of this package. + const char* Name() const override = 0; + + // Compares two slices for equality. The following invariant should always + // hold (and is the default implementation): + // Equal(a, b) iff Compare(a, b) == 0 + // Overwrite only if equality comparisons can be done more efficiently than + // three-way comparisons. + virtual bool Equal(const Slice& a, const Slice& b) const { + return Compare(a, b) == 0; + } + + // Advanced functions: these are used to reduce the space requirements + // for internal data structures like index blocks. + + // If *start < limit, changes *start to a short string in [start,limit). + // Simple comparator implementations may return with *start unchanged, + // i.e., an implementation of this method that does nothing is correct. + virtual void FindShortestSeparator(std::string* start, + const Slice& limit) const = 0; + + // Changes *key to a short string >= *key. + // Simple comparator implementations may return with *key unchanged, + // i.e., an implementation of this method that does nothing is correct. + virtual void FindShortSuccessor(std::string* key) const = 0; + + // given two keys, determine if t is the successor of s + // BUG: only return true if no other keys starting with `t` are ordered + // before `t`. Otherwise, the auto_prefix_mode can omit entries within + // iterator bounds that have same prefix as upper bound but different + // prefix from seek key. + virtual bool IsSameLengthImmediateSuccessor(const Slice& /*s*/, + const Slice& /*t*/) const { + return false; + } + + // return true if two keys with different byte sequences can be regarded + // as equal by this comparator. + // The major use case is to determine if DataBlockHashIndex is compatible + // with the customized comparator. + virtual bool CanKeysWithDifferentByteContentsBeEqual() const { return true; } + + // if it is a wrapped comparator, may return the root one. + // return itself it is not wrapped. + virtual const Comparator* GetRootComparator() const { return this; } + + inline size_t timestamp_size() const { return timestamp_size_; } + + int CompareWithoutTimestamp(const Slice& a, const Slice& b) const { + return CompareWithoutTimestamp(a, /*a_has_ts=*/true, b, /*b_has_ts=*/true); + } + + // For two events e1 and e2 whose timestamps are t1 and t2 respectively, + // Returns value: + // < 0 iff t1 < t2 + // == 0 iff t1 == t2 + // > 0 iff t1 > t2 + // Note that an all-zero byte array will be the smallest (oldest) timestamp + // of the same length, and a byte array with all bits 1 will be the largest. + // In the future, we can extend Comparator so that subclasses can specify + // both largest and smallest timestamps. + virtual int CompareTimestamp(const Slice& /*ts1*/, + const Slice& /*ts2*/) const { + return 0; + } + + virtual int CompareWithoutTimestamp(const Slice& a, bool /*a_has_ts*/, + const Slice& b, bool /*b_has_ts*/) const { + return Compare(a, b); + } + + virtual bool EqualWithoutTimestamp(const Slice& a, const Slice& b) const { + return 0 == + CompareWithoutTimestamp(a, /*a_has_ts=*/true, b, /*b_has_ts=*/true); + } + + private: + size_t timestamp_size_; +}; + +// Return a builtin comparator that uses lexicographic byte-wise +// ordering. The result remains the property of this module and +// must not be deleted. +extern const Comparator* BytewiseComparator(); + +// Return a builtin comparator that uses reverse lexicographic byte-wise +// ordering. +extern const Comparator* ReverseBytewiseComparator(); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/compression_type.h b/librocksdb-sys/rocksdb/include/rocksdb/compression_type.h new file mode 100644 index 0000000..bfeb00b --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/compression_type.h @@ -0,0 +1,40 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +// DB contents are stored in a set of blocks, each of which holds a +// sequence of key,value pairs. Each block may be compressed before +// being stored in a file. The following enum describes which +// compression method (if any) is used to compress a block. + +enum CompressionType : unsigned char { + // NOTE: do not change the values of existing entries, as these are + // part of the persistent format on disk. + kNoCompression = 0x0, + kSnappyCompression = 0x1, + kZlibCompression = 0x2, + kBZip2Compression = 0x3, + kLZ4Compression = 0x4, + kLZ4HCCompression = 0x5, + kXpressCompression = 0x6, + kZSTD = 0x7, + + // Only use kZSTDNotFinalCompression if you have to use ZSTD lib older than + // 0.8.0 or consider a possibility of downgrading the service or copying + // the database files to another service running with an older version of + // RocksDB that doesn't have kZSTD. Otherwise, you should use kZSTD. We will + // eventually remove the option from the public API. + kZSTDNotFinalCompression = 0x40, + + // kDisableCompressionOption is used to disable some compression options. + kDisableCompressionOption = 0xff, +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/concurrent_task_limiter.h b/librocksdb-sys/rocksdb/include/rocksdb/concurrent_task_limiter.h new file mode 100644 index 0000000..9ad741f --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/concurrent_task_limiter.h @@ -0,0 +1,51 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include + +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +// This is NOT an extensible interface but a public interface for result of +// NewConcurrentTaskLimiter. Any derived classes must be RocksDB internal. +class ConcurrentTaskLimiter { + public: + virtual ~ConcurrentTaskLimiter() {} + + // Returns a name that identifies this concurrent task limiter. + virtual const std::string& GetName() const = 0; + + // Set max concurrent tasks. + // limit = 0 means no new task allowed. + // limit < 0 means no limitation. + virtual void SetMaxOutstandingTask(int32_t limit) = 0; + + // Reset to unlimited max concurrent task. + virtual void ResetMaxOutstandingTask() = 0; + + // Returns current outstanding task count. + virtual int32_t GetOutstandingTask() const = 0; +}; + +// Create a ConcurrentTaskLimiter that can be shared with multiple CFs +// across RocksDB instances to control concurrent tasks. +// +// @param name: Name of the limiter. +// @param limit: max concurrent tasks. +// limit = 0 means no new task allowed. +// limit < 0 means no limitation. +extern ConcurrentTaskLimiter* NewConcurrentTaskLimiter(const std::string& name, + int32_t limit); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/configurable.h b/librocksdb-sys/rocksdb/include/rocksdb/configurable.h new file mode 100644 index 0000000..a200d7e --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/configurable.h @@ -0,0 +1,390 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include +#include +#include +#include +#include + +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { +class Logger; +class ObjectRegistry; +class OptionTypeInfo; +struct ColumnFamilyOptions; +struct ConfigOptions; +struct DBOptions; + +// Configurable is a base class used by the rocksdb that describes a +// standard way of configuring objects. A Configurable object can: +// -> Populate itself given: +// - One or more "name/value" pair strings +// - A string representing the set of name=value properties +// - A map of name/value properties. +// -> Convert itself into its string representation +// -> Dump itself to a Logger +// -> Compare itself to another Configurable object to see if the two objects +// have equivalent options settings +// +// If a derived class calls RegisterOptions to register (by name) how its +// options objects are to be processed, this functionality can typically be +// handled by this class without additional overrides. Otherwise, the derived +// class will need to implement the methods for handling the corresponding +// functionality. +class Configurable { + protected: + friend class ConfigurableHelper; + struct RegisteredOptions { + // The name of the options being registered + std::string name; + // Pointer to the object being registered + void* opt_ptr; + // The map of options being registered + const std::unordered_map* type_map; + }; + + public: + virtual ~Configurable() {} + + // Returns the raw pointer of the named options that is used by this + // object, or nullptr if this function is not supported. + // Since the return value is a raw pointer, the object owns the + // pointer and the caller should not delete the pointer. + // + // Note that changing the underlying options while the object + // is currently used by any open DB is undefined behavior. + // Developers should use DB::SetOption() instead to dynamically change + // options while the DB is open. + template + const T* GetOptions() const { + return GetOptions(T::kName()); + } + template + T* GetOptions() { + return GetOptions(T::kName()); + } + template + const T* GetOptions(const std::string& name) const { + return reinterpret_cast(GetOptionsPtr(name)); + } + template + T* GetOptions(const std::string& name) { + return reinterpret_cast(const_cast(GetOptionsPtr(name))); + } + + // Configures the options for this class based on the input parameters. + // On successful completion, the object is updated with the settings from + // the opt_map. + // If this method fails, an attempt is made to revert the object to original + // state. Note that the revert may not be the original state but may be an + // equivalent. For example, if the object contains an option that is a + // shared_ptr, the shared_ptr may not be the original one but a copy (e.g. not + // the Cache object that was passed in, but a Cache object of the same size). + // + // The acceptable values of the name/value pairs are documented with the + // specific class/instance. + // + // @param config_options Controls how the arguments are processed. + // @param opt_map Name/value pairs of the options to update + // @param unused If specified, this value will return the name/value + // pairs from opt_map that were NotFound for this object. + // @return OK If all values in the map were successfully updated + // If invoke_prepare_options is true, OK also implies + // PrepareOptions ran successfully. + // @return NotFound If any of the names in the opt_map were not valid + // for this object. If unused is specified, it will contain the + // collection of NotFound names. + // @return NotSupported If any of the names are valid but the object does + // not know how to convert the value. This can happen if, for example, + // there is some nested Configurable that cannot be created. + // @return InvalidArgument If any of the values cannot be successfully + // parsed. This can also be returned if PrepareOptions encounters an + // error. + // @see ConfigOptions for a description of the controls. + Status ConfigureFromMap( + const ConfigOptions& config_options, + const std::unordered_map& opt_map); + Status ConfigureFromMap( + const ConfigOptions& config_options, + const std::unordered_map& opt_map, + std::unordered_map* unused); + + // Updates the named option to the input value, returning OK if successful. + // Note that ConfigureOption does not cause PrepareOptions to be invoked. + // @param config_options Controls how the name/value is processed. + // @param name The name of the option to update + // @param value The value to set for the named option + // @return OK If the named field was successfully updated to value. + // @return NotFound If the name is not valid for this object. + // @return NotSupported If the name is valid but the object does + // not know how to convert the value. This can happen if, for example, + // there is some nested Configurable that cannot be created. + // @return InvalidArgument If the value cannot be successfully parsed. + Status ConfigureOption(const ConfigOptions& config_options, + const std::string& name, const std::string& value); + + // Configures the options for this class based on the input parameters. + // On successful completion, the object is updated with the settings from + // the opt_map. If this method fails, an attempt is made to revert the + // object to original state. Note that the revert may not be the original + // state but may be an equivalent. + // @see ConfigureFromMap for more details + // @param config_options Controls how the arguments are processed. + // @param opt_str string containing the values to update. + // @param unused If specified, this value will return the name/value + // pairs from opt_map that were NotFound for this object. + // @return OK If all specified values were successfully updated + // If invoke_prepare_options is true, OK also implies + // PrepareOptions ran successfully. + // @return NotFound If any of the names were not valid for this object. + // If unused is specified, it will contain the collection of NotFound + // names. + // @return NotSupported If any of the names are valid but the object does + // not know how to convert the value. This can happen if, for example, + // there is some nested Configurable that cannot be created. + // @return InvalidArgument If any of the values cannot be successfully + // parsed. This can also be returned if PrepareOptions encounters an + // error. + Status ConfigureFromString(const ConfigOptions& config_options, + const std::string& opts); + + // Fills in result with the serialized options for this object. + // This is the inverse of ConfigureFromString. + // @param config_options Controls how serialization happens. + // @param result The string representation of this object. + // @return OK If the options for this object were successfully serialized. + // @return InvalidArgument If one or more of the options could not be + // serialized. + Status GetOptionString(const ConfigOptions& config_options, + std::string* result) const; + // Returns the serialized options for this object. + // This method is similar to GetOptionString with no errors. + // @param config_options Controls how serialization happens. + // @param prefix A string to prepend to every option. + // @return The serialized representation of the options for this object + std::string ToString(const ConfigOptions& config_options) const { + return ToString(config_options, ""); + } + std::string ToString(const ConfigOptions& config_options, + const std::string& prefix) const; + + // Returns the list of option names associated with this configurable + // @param config_options Controls how the names are returned + // @param result The set of option names for this object. Note that + // options that are deprecated or aliases are not returned. + // @return OK on success. + Status GetOptionNames(const ConfigOptions& config_options, + std::unordered_set* result) const; + + // Returns the value of the option associated with the input name + // This method is the functional inverse of ConfigureOption + // @param config_options Controls how the value is returned + // @param name The name of the option to return a value for. + // @param value The returned value associated with the named option. + // @return OK If the named field was successfully updated to value. + // @return NotFound If the name is not valid for this object. + // @param InvalidArgument If the name is valid for this object but + // its value cannot be serialized. + virtual Status GetOption(const ConfigOptions& config_options, + const std::string& name, std::string* value) const; + + // Checks to see if this Configurable is equivalent to other. + // This method assumes that the two objects are of the same class. + // @param config_options Controls how the options are compared. + // @param other The other object to compare to. + // @param mismatch If the objects do not match, this parameter contains + // the name of the option that triggered the match failure. + // @param True if the objects match, false otherwise. + virtual bool AreEquivalent(const ConfigOptions& config_options, + const Configurable* other, + std::string* name) const; + + // Returns a pretty-printed, human-readable version of the options. + // This method is typically used to dump the options to a log file. + // Classes should override this method + virtual std::string GetPrintableOptions() const { return ""; } + + // Validates that the settings are valid/consistent and performs any object + // initialization required by this object. This method may be called as part + // of Configure (if invoke_prepare_options is set), or may be invoked + // separately. + // + // Once an object has been prepared, non-mutable options can no longer be + // updated. + // + // Classes must override this method to provide any implementation-specific + // initialization, such as opening log files or setting up cache parameters. + // Implementations should be idempotent (e.g. don't re-open the log file or + // reconfigure the cache), as there is the potential this method can be called + // more than once. + // + // By default, this method will also prepare all nested (Inner and + // OptionType::kConfigurable) objects. + // + // @param config_options Controls how the object is prepared. Also contains + // a Logger and Env that can be used to initialize this object. + // @return OK If the object was successfully initialized. + // @return InvalidArgument If this object could not be successfully + // initialized. + virtual Status PrepareOptions(const ConfigOptions& config_options); + + // Checks to see if the settings are valid for this object. + // This method checks to see if the input DBOptions and ColumnFamilyOptions + // are valid for the settings of this object. For example, an Env might not + // support certain mmap modes or a TableFactory might require certain + // settings. + // + // By default, this method will also validate all nested (Inner and + // OptionType::kConfigurable) objects. + // + // @param db_opts The DBOptions to validate + // @param cf_opts The ColumnFamilyOptions to validate + // @return OK if the options are valid + // @return InvalidArgument If the arguments are not valid for the options + // of the current object. + virtual Status ValidateOptions(const DBOptions& db_opts, + const ColumnFamilyOptions& cf_opts) const; + + // Splits the input opt_value into the ID field and the remaining options. + // The input opt_value can be in the form of "name" or "name=value + // [;name=value]". The first form uses the "name" as an id with no options The + // latter form converts the input into a map of name=value pairs and sets "id" + // to the "id" value from the map. + // @param opt_value The value to split into id and options + // @param id The id field from the opt_value + // @param options The remaining name/value pairs from the opt_value + // @param default_id If specified and there is no id field in the map, this + // value is returned as the ID + // @return OK if the value was converted to a map successfully and an ID was + // found. + // @return InvalidArgument if the value could not be converted to a map or + // there was or there is no id property in the map. + static Status GetOptionsMap( + const std::string& opt_value, const std::string& default_id, + std::string* id, std::unordered_map* options); + + protected: + // Returns the raw pointer for the associated named option. + // The name is typically the name of an option registered via the + // Classes may override this method to provide further specialization (such as + // returning a sub-option) + // + // The default implementation looks at the registered options. If the + // input name matches that of a registered option, the pointer registered + // with that name is returned. + // e.g,, RegisterOptions("X", &my_ptr, ...); GetOptionsPtr("X") returns + // "my_ptr" + virtual const void* GetOptionsPtr(const std::string& name) const; + + // Method for allowing options to be configured outside of the normal + // registered options framework. Classes may override this method if they + // wish to support non-standard options implementations (such as configuring + // themselves from constant or simple ":"-separated strings. + // + // The default implementation does nothing and returns OK + virtual Status ParseStringOptions(const ConfigOptions& config_options, + const std::string& opts_str); + + // Internal method to configure an object from a map of name-value options. + // This method uses the input config_options to drive the configuration of + // the options in opt_map. Any option name that cannot be found from the + // input set will be returned in "unused". + // + // Classes may override this method to extend the functionality if required. + // @param config_options Controls how the options are configured and errors + // handled. + // @param opts_map The set of options to configure + // @param unused Any options from opt_map that were not configured. + // @returns a Status based on the rules outlined in ConfigureFromMap + virtual Status ConfigureOptions( + const ConfigOptions& config_options, + const std::unordered_map& opts_map, + std::unordered_map* unused); + + // Method that configures a the specific opt_name from opt_value. + // By default, this method calls opt_info.ParseOption with the + // input parameters. + // Classes may override this method to extend the functionality, or + // change the returned Status. + virtual Status ParseOption(const ConfigOptions& config_options, + const OptionTypeInfo& opt_info, + const std::string& opt_name, + const std::string& opt_value, void* opt_ptr); + + // Internal method to see if the single option name/info matches for this and + // that Classes may override this value to change its behavior. + // @param config_options Controls how the options are being matched + // @param opt_info The OptionTypeInfo registered for this option name + // that controls what field is matched (offset) and how (type). + // @param name The name associated with this opt_info. + // @param this_ptr The base pointer to compare to. This is the object + // registered for + // for this OptionTypeInfo. + // @param that_ptr The other pointer to compare to. This is the object + // registered for + // for this OptionTypeInfo. + // @param bad_name If the match fails, the name of the option that failed to + // match. + virtual bool OptionsAreEqual(const ConfigOptions& config_options, + const OptionTypeInfo& opt_info, + const std::string& name, + const void* const this_ptr, + const void* const that_ptr, + std::string* bad_name) const; + // Internal method to serialize options (ToString) + // Classes may override this value to change its behavior. + virtual std::string SerializeOptions(const ConfigOptions& config_options, + const std::string& header) const; + + // Given a name (e.g. rocksdb.my.type.opt), returns the short name (opt) + virtual std::string GetOptionName(const std::string& long_name) const; + + // Registers the input name with the options and associated map. + // When classes register their options in this manner, most of the + // functionality (excluding unknown options and validate/prepare) is + // implemented by the base class. + // + // This method should be called in the class constructor to register the + // option set for this object. For example, to register the options + // associated with the BlockBasedTableFactory, the constructor calls this + // method passing in: + // - the name of the options ("BlockBasedTableOptions"); + // - the options object (the BlockBasedTableOptions object for this object; + // - the options type map for the BlockBasedTableOptions. + // This registration allows the Configurable class to process the option + // values associated with the BlockBasedTableOptions without further code in + // the derived class. + // + // @param name The name of this set of options (@see GetOptionsPtr) + // @param opt_ptr Pointer to the options to associate with this name + // @param opt_map Options map that controls how this option is configured. + template + void RegisterOptions( + T* opt_ptr, + const std::unordered_map* opt_map) { + RegisterOptions(T::kName(), opt_ptr, opt_map); + } + void RegisterOptions( + const std::string& name, void* opt_ptr, + const std::unordered_map* opt_map); + + // Returns true if there are registered options for this Configurable object + inline bool HasRegisteredOptions() const { return !options_.empty(); } + + private: + // Contains the collection of options (name, opt_ptr, opt_map) associated with + // this object. This collection is typically set in the constructor of the + // Configurable option via + std::vector options_; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/convenience.h b/librocksdb-sys/rocksdb/include/rocksdb/convenience.h new file mode 100644 index 0000000..7ce676d --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/convenience.h @@ -0,0 +1,466 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include + +#include "rocksdb/compression_type.h" +#include "rocksdb/db.h" +#include "rocksdb/status.h" +#include "rocksdb/table.h" + +namespace ROCKSDB_NAMESPACE { +class Env; +class Logger; +class ObjectRegistry; + +struct ColumnFamilyOptions; +struct DBOptions; +struct Options; + +// ConfigOptions containing the parameters/controls for +// comparing objects and converting to/from strings. +// These settings control how the methods +// treat errors (e.g. ignore_unknown_objects), the format +// of the serialization (e.g. delimiter), and how to compare +// options (sanity_level). +struct ConfigOptions { + // Constructs a new ConfigOptions with a new object registry. + // This method should only be used when a DBOptions is not available, + // else registry settings may be lost + ConfigOptions(); + + // Constructs a new ConfigOptions using the settings from + // the input DBOptions. Currently constructs a new object registry. + explicit ConfigOptions(const DBOptions&); + + // This enum defines the RocksDB options sanity level. + enum SanityLevel : unsigned char { + kSanityLevelNone = 0x01, // Performs no sanity check at all. + // Performs minimum check to ensure the RocksDB instance can be + // opened without corrupting / mis-interpreting the data. + kSanityLevelLooselyCompatible = 0x02, + // Perform exact match sanity check. + kSanityLevelExactMatch = 0xFF, + }; + + enum Depth { + kDepthDefault, // Traverse nested options that are not flagged as "shallow" + kDepthShallow, // Do not traverse into any nested options + kDepthDetailed, // Traverse nested options, overriding the options shallow + // setting + }; + + // When true, any unused options will be ignored and OK will be returned + bool ignore_unknown_options = false; + + // When true, any unsupported options will be ignored and OK will be returned + bool ignore_unsupported_options = true; + + // If the strings are escaped (old-style?) + bool input_strings_escaped = true; + + // Whether or not to invoke PrepareOptions after configure is called. + bool invoke_prepare_options = true; + + // Options can be marked as Mutable (OptionTypeInfo::IsMutable()) or not. + // When "mutable_options_only=false", all options are evaluated. + // When "mutable_options_only="true", any option not marked as Mutable is + // either ignored (in the case of string/equals methods) or results in an + // error (in the case of Configure). + bool mutable_options_only = false; + + // The separator between options when converting to a string + std::string delimiter = ";"; + + // Controls how to traverse options during print/match stages + Depth depth = Depth::kDepthDefault; + + // Controls how options are serialized + // Controls how pedantic the comparison must be for equivalency + SanityLevel sanity_level = SanityLevel::kSanityLevelExactMatch; + // `file_readahead_size` is used for readahead for the option file. + size_t file_readahead_size = 512 * 1024; + + // The environment to use for this option + Env* env = Env::Default(); + + // The object registry to use for this options + std::shared_ptr registry; + + bool IsShallow() const { return depth == Depth::kDepthShallow; } + bool IsDetailed() const { return depth == Depth::kDepthDetailed; } + + bool IsCheckDisabled() const { + return sanity_level == SanityLevel::kSanityLevelNone; + } + + bool IsCheckEnabled(SanityLevel level) const { + return (level > SanityLevel::kSanityLevelNone && level <= sanity_level); + } +}; + + +// The following set of functions provide a way to construct RocksDB Options +// from a string or a string-to-string map. Here is the general rule of +// setting option values from strings by type. Some RocksDB types are also +// supported in these APIs. Please refer to the comment of the function itself +// to find more information about how to config those RocksDB types. +// +// * Strings: +// Strings will be used as values directly without any truncating or +// trimming. +// +// * Booleans: +// - "true" or "1" => true +// - "false" or "0" => false. +// [Example]: +// - {"optimize_filters_for_hits", "1"} in GetColumnFamilyOptionsFromMap, or +// - "optimize_filters_for_hits=true" in GetColumnFamilyOptionsFromString. +// +// * Integers: +// Integers are converted directly from string, in addition to the following +// units that we support: +// - 'k' or 'K' => 2^10 +// - 'm' or 'M' => 2^20 +// - 'g' or 'G' => 2^30 +// - 't' or 'T' => 2^40 // only for unsigned int with sufficient bits. +// [Example]: +// - {"arena_block_size", "19G"} in GetColumnFamilyOptionsFromMap, or +// - "arena_block_size=19G" in GetColumnFamilyOptionsFromString. +// +// * Doubles / Floating Points: +// Doubles / Floating Points are converted directly from string. Note that +// currently we do not support units. +// [Example]: +// - {"memtable_prefix_bloom_size_ratio", "0.1"} in +// GetColumnFamilyOptionsFromMap, or +// - "memtable_prefix_bloom_size_ratio=0.1" in +// GetColumnFamilyOptionsFromString. +// * Array / Vectors: +// An array is specified by a list of values, where ':' is used as +// the delimiter to separate each value. +// [Example]: +// - {"compression_per_level", "kNoCompression:kSnappyCompression"} +// in GetColumnFamilyOptionsFromMap, or +// - "compression_per_level=kNoCompression:kSnappyCompression" in +// GetColumnFamilyOptionsFromMapString +// * Enums: +// The valid values of each enum are identical to the names of its constants. +// [Example]: +// - CompressionType: valid values are "kNoCompression", +// "kSnappyCompression", "kZlibCompression", "kBZip2Compression", ... +// - CompactionStyle: valid values are "kCompactionStyleLevel", +// "kCompactionStyleUniversal", "kCompactionStyleFIFO", and +// "kCompactionStyleNone". +// + +// Take a ConfigOptions `config_options` and a ColumnFamilyOptions +// "base_options" as the default option in addition to a map "opts_map" of +// option name to option value to construct the new ColumnFamilyOptions +// "new_options". +// +// Below are the instructions of how to config some non-primitive-typed +// options in ColumnFamilyOptions: +// +// * table_factory: +// table_factory can be configured using our custom nested-option syntax. +// +// {option_a=value_a; option_b=value_b; option_c=value_c; ... } +// +// A nested option is enclosed by two curly braces, within which there are +// multiple option assignments. Each assignment is of the form +// "variable_name=value;". +// +// Currently we support the following types of TableFactory: +// - BlockBasedTableFactory: +// Use name "block_based_table_factory" to initialize table_factory with +// BlockBasedTableFactory. Its BlockBasedTableFactoryOptions can be +// configured using the nested-option syntax. +// [Example]: +// * {"block_based_table_factory", "{block_cache=1M;block_size=4k;}"} +// is equivalent to assigning table_factory with a BlockBasedTableFactory +// that has 1M LRU block-cache with block size equals to 4k: +// ColumnFamilyOptions cf_opt; +// BlockBasedTableOptions blk_opt; +// blk_opt.block_cache = NewLRUCache(1 * 1024 * 1024); +// blk_opt.block_size = 4 * 1024; +// cf_opt.table_factory.reset(NewBlockBasedTableFactory(blk_opt)); +// - PlainTableFactory: +// Use name "plain_table_factory" to initialize table_factory with +// PlainTableFactory. Its PlainTableFactoryOptions can be configured using +// the nested-option syntax. +// [Example]: +// * {"plain_table_factory", "{user_key_len=66;bloom_bits_per_key=20;}"} +// +// * memtable_factory: +// Use "memtable" to config memtable_factory. Here are the supported +// memtable factories: +// - SkipList: +// Pass "skip_list:" to config memtable to use SkipList, +// or simply "skip_list" to use the default SkipList. +// [Example]: +// * {"memtable", "skip_list:5"} is equivalent to setting +// memtable to SkipListFactory(5). +// - PrefixHash: +// Pass "prefix_hash:" to config memtable +// to use PrefixHash, or simply "prefix_hash" to use the default +// PrefixHash. +// [Example]: +// * {"memtable", "prefix_hash:1000"} is equivalent to setting +// memtable to NewHashSkipListRepFactory(hash_bucket_count). +// - HashLinkedList: +// Pass "hash_linkedlist:" to config memtable +// to use HashLinkedList, or simply "hash_linkedlist" to use the default +// HashLinkedList. +// [Example]: +// * {"memtable", "hash_linkedlist:1000"} is equivalent to +// setting memtable to NewHashLinkListRepFactory(1000). +// - VectorRepFactory: +// Pass "vector:" to config memtable to use VectorRepFactory, +// or simply "vector" to use the default Vector memtable. +// [Example]: +// * {"memtable", "vector:1024"} is equivalent to setting memtable +// to VectorRepFactory(1024). +// +// * compression_opts: +// Use "compression_opts" to config compression_opts. The value format +// is of the form ":::". +// [Example]: +// * {"compression_opts", "4:5:6:7"} is equivalent to setting: +// ColumnFamilyOptions cf_opt; +// cf_opt.compression_opts.window_bits = 4; +// cf_opt.compression_opts.level = 5; +// cf_opt.compression_opts.strategy = 6; +// cf_opt.compression_opts.max_dict_bytes = 7; +// +// @param config_options controls how the map is processed. +// @param base_options the default options of the output "new_options". +// @param opts_map an option name to value map for specifying how "new_options" +// should be set. +// @param new_options the resulting options based on "base_options" with the +// change specified in "opts_map". +// @param input_strings_escaped when set to true, each escaped characters +// prefixed by '\' in the values of the opts_map will be further converted +// back to the raw string before assigning to the associated options. +// @param ignore_unknown_options when set to true, unknown options are ignored +// instead of resulting in an unknown-option error. +// @return Status::OK() on success. Otherwise, a non-ok status indicating +// error will be returned, and "new_options" will be set to "base_options". +// @return Status::NotFound means the one (or more) of the option name in +// the opts_map is not valid for this option +// @return Status::NotSupported means we do not know how to parse one of the +// value for this option +// @return Status::InvalidArgument means the one of the option values is not +// valid for this option. +Status GetColumnFamilyOptionsFromMap( + const ConfigOptions& config_options, + const ColumnFamilyOptions& base_options, + const std::unordered_map& opts_map, + ColumnFamilyOptions* new_options); + +// Take a ConfigOptions `config_options` and a DBOptions "base_options" as the +// default option in addition to a map "opts_map" of option name to option value +// to construct the new DBOptions "new_options". +// +// Below are the instructions of how to config some non-primitive-typed +// options in DBOptions: +// +// * rate_limiter_bytes_per_sec: +// RateLimiter can be configured directly by specifying its bytes_per_sec. +// [Example]: +// - Passing {"rate_limiter_bytes_per_sec", "1024"} is equivalent to +// passing NewGenericRateLimiter(1024) to rate_limiter_bytes_per_sec. +// +// @param config_options controls how the map is processed. +// @param base_options the default options of the output "new_options". +// @param opts_map an option name to value map for specifying how "new_options" +// should be set. +// @param new_options the resulting options based on "base_options" with the +// change specified in "opts_map". +// @param input_strings_escaped when set to true, each escaped characters +// prefixed by '\' in the values of the opts_map will be further converted +// back to the raw string before assigning to the associated options. +// @param ignore_unknown_options when set to true, unknown options are ignored +// instead of resulting in an unknown-option error. +// @return Status::OK() on success. Otherwise, a non-ok status indicating +// error will be returned, and "new_options" will be set to "base_options". +// @return Status::NotFound means the one (or more) of the option name in +// the opts_map is not valid for this option +// @return Status::NotSupported means we do not know how to parse one of the +// value for this option +// @return Status::InvalidArgument means the one of the option values is not +// valid for this option. +Status GetDBOptionsFromMap( + const ConfigOptions& cfg_options, const DBOptions& base_options, + const std::unordered_map& opts_map, + DBOptions* new_options); + +// Take a ConfigOptions `config_options` and a BlockBasedTableOptions +// "table_options" as the default option in addition to a map "opts_map" of +// option name to option value to construct the new BlockBasedTableOptions +// "new_table_options". +// +// Below are the instructions of how to config some non-primitive-typed +// options in BlockBasedTableOptions: +// +// * filter_policy: +// We currently only support the following FilterPolicy in the convenience +// functions: +// - BloomFilter: use "bloomfilter:[bits_per_key]:[use_block_based_builder]" +// to specify BloomFilter. The above string is equivalent to calling +// NewBloomFilterPolicy(bits_per_key, use_block_based_builder). +// [Example]: +// - Pass {"filter_policy", "bloomfilter:4:true"} in +// GetBlockBasedTableOptionsFromMap to use a BloomFilter with 4-bits +// per key and use_block_based_builder enabled. +// +// * block_cache / block_cache_compressed: +// We currently only support LRU cache in the GetOptions API. The LRU +// cache can be set by directly specifying its size. +// [Example]: +// - Passing {"block_cache", "1M"} in GetBlockBasedTableOptionsFromMap is +// equivalent to setting block_cache using NewLRUCache(1024 * 1024). +// +// @param config_options controls how the map is processed. +// @param table_options the default options of the output "new_table_options". +// @param opts_map an option name to value map for specifying how +// "new_table_options" should be set. +// @param new_table_options the resulting options based on "table_options" +// with the change specified in "opts_map". +// @param input_strings_escaped when set to true, each escaped characters +// prefixed by '\' in the values of the opts_map will be further converted +// back to the raw string before assigning to the associated options. +// @param ignore_unknown_options when set to true, unknown options are ignored +// instead of resulting in an unknown-option error. +// @return Status::OK() on success. Otherwise, a non-ok status indicating +// error will be returned, and "new_table_options" will be set to +// "table_options". +Status GetBlockBasedTableOptionsFromMap( + const ConfigOptions& config_options, + const BlockBasedTableOptions& table_options, + const std::unordered_map& opts_map, + BlockBasedTableOptions* new_table_options); + +// Take a ConfigOptions `config_options` and a default PlainTableOptions +// "table_options" as the default option in addition to a map "opts_map" of +// option name to option value to construct the new PlainTableOptions +// "new_table_options". +// +// @param config_options controls how the map is processed. +// @param table_options the default options of the output "new_table_options". +// @param opts_map an option name to value map for specifying how +// "new_table_options" should be set. +// @param new_table_options the resulting options based on "table_options" +// with the change specified in "opts_map". +// @param input_strings_escaped when set to true, each escaped characters +// prefixed by '\' in the values of the opts_map will be further converted +// back to the raw string before assigning to the associated options. +// @param ignore_unknown_options when set to true, unknown options are ignored +// instead of resulting in an unknown-option error. +// @return Status::OK() on success. Otherwise, a non-ok status indicating +// error will be returned, and "new_table_options" will be set to +// "table_options". +Status GetPlainTableOptionsFromMap( + const ConfigOptions& config_options, const PlainTableOptions& table_options, + const std::unordered_map& opts_map, + PlainTableOptions* new_table_options); + +// Take a ConfigOptions `config_options`, a string representation of option +// names and values, apply them into the base_options, and return the new +// options as a result. The string has the following format: +// "write_buffer_size=1024;max_write_buffer_number=2" +// Nested options config is also possible. For example, you can define +// BlockBasedTableOptions as part of the string for block-based table factory: +// "write_buffer_size=1024;block_based_table_factory={block_size=4k};" +// "max_write_buffer_num=2" +// +Status GetColumnFamilyOptionsFromString(const ConfigOptions& config_options, + const ColumnFamilyOptions& base_options, + const std::string& opts_str, + ColumnFamilyOptions* new_options); + +Status GetDBOptionsFromString(const ConfigOptions& config_options, + const DBOptions& base_options, + const std::string& opts_str, + DBOptions* new_options); + +Status GetStringFromDBOptions(const ConfigOptions& config_options, + const DBOptions& db_options, + std::string* opts_str); + +Status GetStringFromDBOptions(std::string* opts_str, + const DBOptions& db_options, + const std::string& delimiter = "; "); + +Status GetStringFromColumnFamilyOptions(const ConfigOptions& config_options, + const ColumnFamilyOptions& cf_options, + std::string* opts_str); +Status GetStringFromColumnFamilyOptions(std::string* opts_str, + const ColumnFamilyOptions& cf_options, + const std::string& delimiter = "; "); +Status GetStringFromCompressionType(std::string* compression_str, + CompressionType compression_type); + +std::vector GetSupportedCompressions(); + +Status GetBlockBasedTableOptionsFromString( + const ConfigOptions& config_options, + const BlockBasedTableOptions& table_options, const std::string& opts_str, + BlockBasedTableOptions* new_table_options); + +Status GetPlainTableOptionsFromString(const ConfigOptions& config_options, + const PlainTableOptions& table_options, + const std::string& opts_str, + PlainTableOptions* new_table_options); + +Status GetMemTableRepFactoryFromString( + const std::string& opts_str, + std::unique_ptr* new_mem_factory); + +Status GetOptionsFromString(const Options& base_options, + const std::string& opts_str, Options* new_options); +Status GetOptionsFromString(const ConfigOptions& config_options, + const Options& base_options, + const std::string& opts_str, Options* new_options); + +Status StringToMap(const std::string& opts_str, + std::unordered_map* opts_map); + +// Request stopping background work, if wait is true wait until it's done +void CancelAllBackgroundWork(DB* db, bool wait = false); + +// Delete files which are entirely in the given range +// Could leave some keys in the range which are in files which are not +// entirely in the range. Also leaves L0 files regardless of whether they're +// in the range. +// Snapshots before the delete might not see the data in the given range. +Status DeleteFilesInRange(DB* db, ColumnFamilyHandle* column_family, + const Slice* begin, const Slice* end, + bool include_end = true); + +// Delete files in multiple ranges at once +// Delete files in a lot of ranges one at a time can be slow, use this API for +// better performance in that case. +Status DeleteFilesInRanges(DB* db, ColumnFamilyHandle* column_family, + const RangePtr* ranges, size_t n, + bool include_end = true); + +// Verify the checksum of file +Status VerifySstFileChecksum(const Options& options, + const EnvOptions& env_options, + const std::string& file_path); + +// Verify the checksum of file +Status VerifySstFileChecksum(const Options& options, + const EnvOptions& env_options, + const ReadOptions& read_options, + const std::string& file_path, + const SequenceNumber& largest_seqno = 0); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/customizable.h b/librocksdb-sys/rocksdb/include/rocksdb/customizable.h new file mode 100644 index 0000000..076aca6 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/customizable.h @@ -0,0 +1,229 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include "rocksdb/configurable.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { +/** + * Customizable a base class used by the rocksdb that describes a + * standard way of configuring and creating objects. Customizable objects + * are configurable objects that can be created from an ObjectRegistry. + * + * Customizable classes are used when there are multiple potential + * implementations of a class for use by RocksDB (e.g. Table, Cache, + * MergeOperator, etc). The abstract base class is expected to define a method + * declaring its type and a factory method for creating one of these, such as: + * static const char *Type() { return "Table"; } + * static Status CreateFromString(const ConfigOptions& options, + * const std::string& id, + * std::shared_ptr* result); + * The "Type" string is expected to be unique (no two base classes are the same + * type). This factory is expected, based on the options and id, create and + * return the appropriate derived type of the customizable class (e.g. + * BlockBasedTableFactory, PlainTableFactory, etc). For extension developers, + * helper classes and methods are provided for writing this factory. + * + * Instances of a Customizable class need to define: + * - A "static const char *kClassName()" method. This method defines the name + * of the class instance (e.g. BlockBasedTable, LRUCache) and is used by the + * CheckedCast method. + * - The Name() of the object. This name is used when creating and saving + * instances of this class. Typically this name will be the same as + * kClassName(). + * + * Additionally, Customizable classes should register any options used to + * configure themselves with the Configurable subsystem. + * + * When a Customizable is being created, the "name" property specifies + * the name of the instance being created. + * For custom objects, their configuration and name can be specified by: + * [prop]={name=X;option 1 = value1[; option2=value2...]} + * + * [prop].name=X + * [prop].option1 = value1 + * + * [prop].name=X + * X.option1 =value1 + */ +class Customizable : public Configurable { + public: + ~Customizable() override {} + + // Returns the name of this class of Customizable + virtual const char* Name() const = 0; + + // Returns an identifier for this Customizable. + // This could be its name or something more complex (like its URL/pattern). + // Used for pretty printing. + virtual std::string GetId() const { + std::string id = Name(); + return id; + } + + // This is typically determined by if the input name matches the + // name of this object. + // This method is typically used in conjunction with CheckedCast to find the + // derived class instance from its base. For example, if you have an Env + // and want the "Default" env, you would IsInstanceOf("Default") to get + // the default implementation. This method should be used when you need a + // specific derivative or implementation of a class. + // + // Intermediary caches (such as SharedCache) may wish to override this method + // to check for the intermediary name (SharedCache). Classes with multiple + // potential names (e.g. "PosixEnv", "DefaultEnv") may also wish to override + // this method. + // + // Note that IsInstanceOf only uses the "is-a" relationship and not "has-a". + // Wrapped classes that have an Inner "has-a" should not be returned. + // + // @param name The name of the instance to find. + // Returns true if the class is an instance of the input name. + virtual bool IsInstanceOf(const std::string& name) const { + if (name.empty()) { + return false; + } else if (name == Name()) { + return true; + } else { + const char* nickname = NickName(); + if (nickname != nullptr && name == nickname) { + return true; + } else { + return false; + } + } + } + + const void* GetOptionsPtr(const std::string& name) const override { + const void* ptr = Configurable::GetOptionsPtr(name); + if (ptr != nullptr) { + return ptr; + } else { + const auto inner = Inner(); + if (inner != nullptr) { + return inner->GetOptionsPtr(name); + } else { + return nullptr; + } + } + } + + // Returns the named instance of the Customizable as a T*, or nullptr if not + // found. This method uses IsInstanceOf/Inner to find the appropriate class + // instance and then casts it to the expected return type. + template + const T* CheckedCast() const { + if (IsInstanceOf(T::kClassName())) { + return static_cast(this); + } else { + const auto inner = Inner(); + if (inner != nullptr) { + return inner->CheckedCast(); + } else { + return nullptr; + } + } + } + + template + T* CheckedCast() { + if (IsInstanceOf(T::kClassName())) { + return static_cast(this); + } else { + auto inner = const_cast(Inner()); + if (inner != nullptr) { + return inner->CheckedCast(); + } else { + return nullptr; + } + } + } + + // Checks to see if this Customizable is equivalent to other. + // This method assumes that the two objects are of the same class. + // @param config_options Controls how the options are compared. + // @param other The other object to compare to. + // @param mismatch If the objects do not match, this parameter contains + // the name of the option that triggered the match failure. + // @param True if the objects match, false otherwise. + // @see Configurable::AreEquivalent for more details + bool AreEquivalent(const ConfigOptions& config_options, + const Configurable* other, + std::string* mismatch) const override; + // Gets the value of the option associated with the input name + // @see Configurable::GetOption for more details + Status GetOption(const ConfigOptions& config_options, const std::string& name, + std::string* value) const override; + // Helper method for getting for parsing the opt_value into the corresponding + // options for use in potentially creating a new Customizable object (this + // method is primarily a support method for LoadSharedObject et al for new + // Customizable objects). The opt_value may be either name-value pairs + // separated by ";" (a=b; c=d), or a simple name (a). In order to create a new + // Customizable, the ID is determined by: + // - If the value is a simple name (e.g. "BlockBasedTable"), the id is this + // name; + // - Otherwise, if there is a "id=value", the id is set to "value" + // - Otherwise, if the input customizable is not null, custom->GetId is used + // - Otherwise, an error is returned. + // + // If the opt_value is name-value pairs, these pairs will be returned in + // options (without the id pair). If the ID being returned matches the ID of + // the input custom object, then the options from the input object will also + // be added to the returned options. + // + // This method returns non-OK if the ID could not be found, or if the + // opt_value could not be parsed into name-value pairs. + static Status GetOptionsMap( + const ConfigOptions& config_options, const Customizable* custom, + const std::string& opt_value, std::string* id, + std::unordered_map* options); + + // Helper method to configure a new object with the supplied options. + // If the object is not null and invoke_prepare_options=true, the object + // will be configured and prepared. + // Returns success if the object is properly configured and (optionally) + // prepared Returns InvalidArgument if the object is nullptr and there are + // options in the map Returns the result of the ConfigureFromMap or + // PrepareOptions + static Status ConfigureNewObject( + const ConfigOptions& config_options, Customizable* object, + const std::unordered_map& options); + + // Returns the inner class when a Customizable implements a has-a (wrapped) + // relationship. Derived classes that implement a has-a must override this + // method in order to get CheckedCast to function properly. + virtual const Customizable* Inner() const { return nullptr; } + + protected: + // Generates a ID specific for this instance of the customizable. + // The unique ID is of the form :#pid, where: + // - name is the Name() of this object; + // - addr is the memory address of this object; + // - pid is the process ID of this process ID for this process. + // Note that if obj1 and obj2 have the same unique IDs, they must be the + // same. However, if an object is deleted and recreated, it may have the + // same unique ID as a predecessor + // + // This method is useful for objects (especially ManagedObjects) that + // wish to generate an ID that is specific for this instance and wish to + // override the GetId() method. + std::string GenerateIndividualId() const; + + // Some classes have both a class name (e.g. PutOperator) and a nickname + // (e.g. put). Classes can override this method to return a + // nickname. Nicknames can be used by InstanceOf and object creation. + virtual const char* NickName() const { return ""; } + // Given a name (e.g. rocksdb.my.type.opt), returns the short name (opt) + std::string GetOptionName(const std::string& long_name) const override; + std::string SerializeOptions(const ConfigOptions& options, + const std::string& prefix) const override; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/data_structure.h b/librocksdb-sys/rocksdb/include/rocksdb/data_structure.h new file mode 100644 index 0000000..ffab82c --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/data_structure.h @@ -0,0 +1,186 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include +#include +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +namespace detail { +int CountTrailingZeroBitsForSmallEnumSet(uint64_t); +} // namespace detail + +// Represents a set of values of some enum type with a small number of +// possible enumerators. For now, it supports enums where no enumerator +// exceeds 63 when converted to int. +template +class SmallEnumSet { + private: + using StateT = uint64_t; + static constexpr int kStateBits = sizeof(StateT) * 8; + static constexpr int kMaxMax = kStateBits - 1; + static constexpr int kMaxValue = static_cast(MAX_ENUMERATOR); + static_assert(kMaxValue >= 0); + static_assert(kMaxValue <= kMaxMax); + + public: + // construct / create + SmallEnumSet() : state_(0) {} + + template + /*implicit*/ constexpr SmallEnumSet(const ENUM_TYPE e, TRest... rest) { + *this = SmallEnumSet(rest...).With(e); + } + + // Return the set that includes all valid values, assuming the enum + // is "dense" (includes all values converting to 0 through kMaxValue) + static constexpr SmallEnumSet All() { + StateT tmp = StateT{1} << kMaxValue; + return SmallEnumSet(RawStateMarker(), tmp | (tmp - 1)); + } + + // equality + bool operator==(const SmallEnumSet& that) const { + return this->state_ == that.state_; + } + bool operator!=(const SmallEnumSet& that) const { return !(*this == that); } + + // query + + // Return true if the input enum is contained in the "Set". + bool Contains(const ENUM_TYPE e) const { + int value = static_cast(e); + assert(value >= 0 && value <= kMaxValue); + StateT tmp = 1; + return state_ & (tmp << value); + } + + bool empty() const { return state_ == 0; } + + // iterator + class const_iterator { + public: + // copy + const_iterator(const const_iterator& that) = default; + const_iterator& operator=(const const_iterator& that) = default; + + // move + const_iterator(const_iterator&& that) noexcept = default; + const_iterator& operator=(const_iterator&& that) noexcept = default; + + // equality + bool operator==(const const_iterator& that) const { + assert(set_ == that.set_); + return this->pos_ == that.pos_; + } + + bool operator!=(const const_iterator& that) const { + return !(*this == that); + } + + // ++iterator + const_iterator& operator++() { + if (pos_ < kMaxValue) { + pos_ = set_->SkipUnset(pos_ + 1); + } else { + pos_ = kStateBits; + } + return *this; + } + + // iterator++ + const_iterator operator++(int) { + auto old = *this; + ++*this; + return old; + } + + ENUM_TYPE operator*() const { + assert(pos_ <= kMaxValue); + return static_cast(pos_); + } + + private: + friend class SmallEnumSet; + const_iterator(const SmallEnumSet* set, int pos) : set_(set), pos_(pos) {} + const SmallEnumSet* set_; + int pos_; + }; + + const_iterator begin() const { return const_iterator(this, SkipUnset(0)); } + + const_iterator end() const { return const_iterator(this, kStateBits); } + + // mutable ops + + // Modifies the set (if needed) to include the given value. Returns true + // iff the set was modified. + bool Add(const ENUM_TYPE e) { + int value = static_cast(e); + assert(value >= 0 && value <= kMaxValue); + StateT old_state = state_; + state_ |= (StateT{1} << value); + return old_state != state_; + } + + // Modifies the set (if needed) not to include the given value. Returns true + // iff the set was modified. + bool Remove(const ENUM_TYPE e) { + int value = static_cast(e); + assert(value >= 0 && value <= kMaxValue); + StateT old_state = state_; + state_ &= ~(StateT{1} << value); + return old_state != state_; + } + + // applicative ops + + // Return a new set based on this one with the additional value(s) inserted + constexpr SmallEnumSet With(const ENUM_TYPE e) const { + int value = static_cast(e); + assert(value >= 0 && value <= kMaxValue); + return SmallEnumSet(RawStateMarker(), state_ | (StateT{1} << value)); + } + template + constexpr SmallEnumSet With(const ENUM_TYPE e1, const ENUM_TYPE e2, + TRest... rest) const { + return With(e1).With(e2, rest...); + } + + // Return a new set based on this one excluding the given value(s) + constexpr SmallEnumSet Without(const ENUM_TYPE e) const { + int value = static_cast(e); + assert(value >= 0 && value <= kMaxValue); + return SmallEnumSet(RawStateMarker(), state_ & ~(StateT{1} << value)); + } + template + constexpr SmallEnumSet Without(const ENUM_TYPE e1, const ENUM_TYPE e2, + TRest... rest) const { + return Without(e1).Without(e2, rest...); + } + + private: + int SkipUnset(int pos) const { + StateT tmp = state_ >> pos; + if (tmp == 0) { + return kStateBits; + } else { + return pos + detail::CountTrailingZeroBitsForSmallEnumSet(tmp); + } + } + struct RawStateMarker {}; + explicit SmallEnumSet(RawStateMarker, StateT state) : state_(state) {} + + StateT state_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/db.h b/librocksdb-sys/rocksdb/include/rocksdb/db.h new file mode 100644 index 0000000..436b430 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/db.h @@ -0,0 +1,2014 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +#include "rocksdb/block_cache_trace_writer.h" +#include "rocksdb/iterator.h" +#include "rocksdb/listener.h" +#include "rocksdb/metadata.h" +#include "rocksdb/options.h" +#include "rocksdb/snapshot.h" +#include "rocksdb/sst_file_writer.h" +#include "rocksdb/thread_status.h" +#include "rocksdb/transaction_log.h" +#include "rocksdb/types.h" +#include "rocksdb/version.h" +#include "rocksdb/wide_columns.h" + +#ifdef _WIN32 +// Windows API macro interference +#undef DeleteFile +#endif + +#if defined(__GNUC__) || defined(__clang__) +#define ROCKSDB_DEPRECATED_FUNC __attribute__((__deprecated__)) +#elif _WIN32 +#define ROCKSDB_DEPRECATED_FUNC __declspec(deprecated) +#endif + +namespace ROCKSDB_NAMESPACE { + +struct ColumnFamilyOptions; +struct CompactionOptions; +struct CompactRangeOptions; +struct DBOptions; +struct ExternalSstFileInfo; +struct FlushOptions; +struct Options; +struct ReadOptions; +struct TableProperties; +struct WriteOptions; +struct WaitForCompactOptions; +class Env; +class EventListener; +class FileSystem; +class Replayer; +class StatsHistoryIterator; +class TraceReader; +class TraceWriter; +class WriteBatch; + +extern const std::string kDefaultColumnFamilyName; +extern const std::string kPersistentStatsColumnFamilyName; +struct ColumnFamilyDescriptor { + std::string name; + ColumnFamilyOptions options; + ColumnFamilyDescriptor() + : name(kDefaultColumnFamilyName), options(ColumnFamilyOptions()) {} + ColumnFamilyDescriptor(const std::string& _name, + const ColumnFamilyOptions& _options) + : name(_name), options(_options) {} +}; + +class ColumnFamilyHandle { + public: + virtual ~ColumnFamilyHandle() {} + // Returns the name of the column family associated with the current handle. + virtual const std::string& GetName() const = 0; + // Returns the ID of the column family associated with the current handle. + virtual uint32_t GetID() const = 0; + // Fills "*desc" with the up-to-date descriptor of the column family + // associated with this handle. Since it fills "*desc" with the up-to-date + // information, this call might internally lock and release DB mutex to + // access the up-to-date CF options. In addition, all the pointer-typed + // options cannot be referenced any longer than the original options exist. + // + // Note that this function is not supported in RocksDBLite. + virtual Status GetDescriptor(ColumnFamilyDescriptor* desc) = 0; + // Returns the comparator of the column family associated with the + // current handle. + virtual const Comparator* GetComparator() const = 0; +}; + +static const int kMajorVersion = __ROCKSDB_MAJOR__; +static const int kMinorVersion = __ROCKSDB_MINOR__; + +// A range of keys +struct Range { + Slice start; + Slice limit; + + Range() {} + Range(const Slice& s, const Slice& l) : start(s), limit(l) {} +}; + +struct RangePtr { + const Slice* start; + const Slice* limit; + + RangePtr() : start(nullptr), limit(nullptr) {} + RangePtr(const Slice* s, const Slice* l) : start(s), limit(l) {} +}; + +// It is valid that files_checksums and files_checksum_func_names are both +// empty (no checksum information is provided for ingestion). Otherwise, +// their sizes should be the same as external_files. The file order should +// be the same in three vectors and guaranteed by the caller. +// Note that, we assume the temperatures of this batch of files to be +// ingested are the same. +struct IngestExternalFileArg { + ColumnFamilyHandle* column_family = nullptr; + std::vector external_files; + IngestExternalFileOptions options; + std::vector files_checksums; + std::vector files_checksum_func_names; + Temperature file_temperature = Temperature::kUnknown; +}; + +struct GetMergeOperandsOptions { + int expected_max_number_of_operands = 0; +}; + +// A collections of table properties objects, where +// key: is the table's file name. +// value: the table properties object of the given table. +using TablePropertiesCollection = + std::unordered_map>; + +// A DB is a persistent, versioned ordered map from keys to values. +// A DB is safe for concurrent access from multiple threads without +// any external synchronization. +// DB is an abstract base class with one primary implementation (DBImpl) +// and a number of wrapper implementations. +class DB { + public: + // Open the database with the specified "name" for reads and writes. + // Stores a pointer to a heap-allocated database in *dbptr and returns + // OK on success. + // Stores nullptr in *dbptr and returns a non-OK status on error, including + // if the DB is already open (read-write) by another DB object. (This + // guarantee depends on options.env->LockFile(), which might not provide + // this guarantee in a custom Env implementation.) + // + // Caller must delete *dbptr when it is no longer needed. + static Status Open(const Options& options, const std::string& name, + DB** dbptr); + + // Open DB with column families. + // db_options specify database specific options + // column_families is the vector of all column families in the database, + // containing column family name and options. You need to open ALL column + // families in the database. To get the list of column families, you can use + // ListColumnFamilies(). + // + // The default column family name is 'default' and it's stored + // in ROCKSDB_NAMESPACE::kDefaultColumnFamilyName. + // If everything is OK, handles will on return be the same size + // as column_families --- handles[i] will be a handle that you + // will use to operate on column family column_family[i]. + // Before delete DB, you have to close All column families by calling + // DestroyColumnFamilyHandle() with all the handles. + static Status Open(const DBOptions& db_options, const std::string& name, + const std::vector& column_families, + std::vector* handles, DB** dbptr); + + // OpenForReadOnly() creates a Read-only instance that supports reads alone. + // + // All DB interfaces that modify data, like put/delete, will return error. + // Automatic Flush and Compactions are disabled and any manual calls + // to Flush/Compaction will return error. + // + // While a given DB can be simultaneously opened via OpenForReadOnly + // by any number of readers, if a DB is simultaneously opened by Open + // and OpenForReadOnly, the read-only instance has undefined behavior + // (though can often succeed if quickly closed) and the read-write + // instance is unaffected. See also OpenAsSecondary. + + // Open the database for read only. + // + static Status OpenForReadOnly(const Options& options, const std::string& name, + DB** dbptr, + bool error_if_wal_file_exists = false); + + // Open the database for read only with column families. + // + // When opening DB with read only, you can specify only a subset of column + // families in the database that should be opened. However, you always need + // to specify default column family. The default column family name is + // 'default' and it's stored in ROCKSDB_NAMESPACE::kDefaultColumnFamilyName + // + static Status OpenForReadOnly( + const DBOptions& db_options, const std::string& name, + const std::vector& column_families, + std::vector* handles, DB** dbptr, + bool error_if_wal_file_exists = false); + + // OpenAsSecondary() creates a secondary instance that supports read-only + // operations and supports dynamic catch up with the primary (through a + // call to TryCatchUpWithPrimary()). + // + // All DB interfaces that modify data, like put/delete, will return error. + // Automatic Flush and Compactions are disabled and any manual calls + // to Flush/Compaction will return error. + // + // Multiple secondary instances can co-exist at the same time. + // + + // Open DB as secondary instance + // + // The options argument specifies the options to open the secondary instance. + // Options.max_open_files should be set to -1. + // The name argument specifies the name of the primary db that you have used + // to open the primary instance. + // The secondary_path argument points to a directory where the secondary + // instance stores its info log. + // The dbptr is an out-arg corresponding to the opened secondary instance. + // The pointer points to a heap-allocated database, and the caller should + // delete it after use. + // + // Return OK on success, non-OK on failures. + static Status OpenAsSecondary(const Options& options, const std::string& name, + const std::string& secondary_path, DB** dbptr); + + // Open DB as secondary instance with specified column families + // + // When opening DB in secondary mode, you can specify only a subset of column + // families in the database that should be opened. However, you always need + // to specify default column family. The default column family name is + // 'default' and it's stored in ROCKSDB_NAMESPACE::kDefaultColumnFamilyName + // + // Column families created by the primary after the secondary instance starts + // are currently ignored by the secondary instance. Column families opened + // by secondary and dropped by the primary will be dropped by secondary as + // well (on next invocation of TryCatchUpWithPrimary()). However the user + // of the secondary instance can still access the data of such dropped column + // family as long as they do not destroy the corresponding column family + // handle. + // + // The options argument specifies the options to open the secondary instance. + // Options.max_open_files should be set to -1. + // The name argument specifies the name of the primary db that you have used + // to open the primary instance. + // The secondary_path argument points to a directory where the secondary + // instance stores its info log. + // The column_families argument specifies a list of column families to open. + // If default column family is not specified or if any specified column + // families does not exist, the function returns non-OK status. + // The handles is an out-arg corresponding to the opened database column + // family handles. + // The dbptr is an out-arg corresponding to the opened secondary instance. + // The pointer points to a heap-allocated database, and the caller should + // delete it after use. Before deleting the dbptr, the user should also + // delete the pointers stored in handles vector. + // + // Return OK on success, non-OK on failures. + static Status OpenAsSecondary( + const DBOptions& db_options, const std::string& name, + const std::string& secondary_path, + const std::vector& column_families, + std::vector* handles, DB** dbptr); + + // Open DB and run the compaction. + // It's a read-only operation, the result won't be installed to the DB, it + // will be output to the `output_directory`. The API should only be used with + // `options.CompactionService` to run compaction triggered by + // `CompactionService`. + static Status OpenAndCompact( + const std::string& name, const std::string& output_directory, + const std::string& input, std::string* output, + const CompactionServiceOptionsOverride& override_options); + + static Status OpenAndCompact( + const OpenAndCompactOptions& options, const std::string& name, + const std::string& output_directory, const std::string& input, + std::string* output, + const CompactionServiceOptionsOverride& override_options); + + // Experimental and subject to change + // Open DB and trim data newer than specified timestamp. + // The trim_ts specified the user-defined timestamp trim bound. + // This API should only be used at timestamp enabled column families recovery. + // If some input column families do not support timestamp, nothing will + // be happened to them. The data with timestamp > trim_ts + // will be removed after this API returns successfully. + static Status OpenAndTrimHistory( + const DBOptions& db_options, const std::string& dbname, + const std::vector& column_families, + std::vector* handles, DB** dbptr, + std::string trim_ts); + + // Manually, synchronously attempt to resume DB writes after a write failure + // to the underlying filesystem. See + // https://github.com/facebook/rocksdb/wiki/Background-Error-Handling + // + // Returns OK if writes are successfully resumed, or there was no + // outstanding error to recover from. Returns underlying write error if + // it is not recoverable. + // + // WART: Does not mix well with auto-resume. Will return Busy if an + // auto-resume is in progress, without waiting for it to complete. + // See DBOptions::max_bgerror_resume_count and + // EventListener::OnErrorRecoveryBegin + virtual Status Resume() { return Status::NotSupported(); } + + // Close the DB by releasing resources, closing files etc. This should be + // called before calling the destructor so that the caller can get back a + // status in case there are any errors. This will not fsync the WAL files. + // If syncing is required, the caller must first call SyncWAL(), or Write() + // using an empty write batch with WriteOptions.sync=true. + // Regardless of the return status, the DB must be freed. + // If the return status is Aborted(), closing fails because there is + // unreleased snapshot in the system. In this case, users can release + // the unreleased snapshots and try again and expect it to succeed. For + // other status, re-calling Close() will be no-op and return the original + // close status. If the return status is NotSupported(), then the DB + // implementation does cleanup in the destructor + virtual Status Close() { return Status::NotSupported(); } + + // ListColumnFamilies will open the DB specified by argument name + // and return the list of all column families in that DB + // through column_families argument. The ordering of + // column families in column_families is unspecified. + static Status ListColumnFamilies(const DBOptions& db_options, + const std::string& name, + std::vector* column_families); + + // Abstract class ctor + DB() {} + // No copying allowed + DB(const DB&) = delete; + void operator=(const DB&) = delete; + + virtual ~DB(); + + // Create a column_family and return the handle of column family + // through the argument handle. + virtual Status CreateColumnFamily(const ColumnFamilyOptions& options, + const std::string& column_family_name, + ColumnFamilyHandle** handle); + + // Bulk create column families with the same column family options. + // Return the handles of the column families through the argument handles. + // In case of error, the request may succeed partially, and handles will + // contain column family handles that it managed to create, and have size + // equal to the number of created column families. + virtual Status CreateColumnFamilies( + const ColumnFamilyOptions& options, + const std::vector& column_family_names, + std::vector* handles); + + // Bulk create column families. + // Return the handles of the column families through the argument handles. + // In case of error, the request may succeed partially, and handles will + // contain column family handles that it managed to create, and have size + // equal to the number of created column families. + virtual Status CreateColumnFamilies( + const std::vector& column_families, + std::vector* handles); + + // Drop a column family specified by column_family handle. This call + // only records a drop record in the manifest and prevents the column + // family from flushing and compacting. + virtual Status DropColumnFamily(ColumnFamilyHandle* column_family); + + // Bulk drop column families. This call only records drop records in the + // manifest and prevents the column families from flushing and compacting. + // In case of error, the request may succeed partially. User may call + // ListColumnFamilies to check the result. + virtual Status DropColumnFamilies( + const std::vector& column_families); + + // Release and deallocate a column family handle. A column family is only + // removed once it is dropped (DropColumnFamily) and all handles have been + // destroyed (DestroyColumnFamilyHandle). Use this method to destroy + // column family handles (except for DefaultColumnFamily()!) before closing + // a DB. + virtual Status DestroyColumnFamilyHandle(ColumnFamilyHandle* column_family); + + // Set the database entry for "key" to "value". + // If "key" already exists, it will be overwritten. + // Returns OK on success, and a non-OK status on error. + // Note: consider setting options.sync = true. + virtual Status Put(const WriteOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + const Slice& value) = 0; + virtual Status Put(const WriteOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + const Slice& ts, const Slice& value) = 0; + virtual Status Put(const WriteOptions& options, const Slice& key, + const Slice& value) { + return Put(options, DefaultColumnFamily(), key, value); + } + virtual Status Put(const WriteOptions& options, const Slice& key, + const Slice& ts, const Slice& value) { + return Put(options, DefaultColumnFamily(), key, ts, value); + } + + // Set the database entry for "key" in the column family specified by + // "column_family" to the wide-column entity defined by "columns". If the key + // already exists in the column family, it will be overwritten. + // + // Returns OK on success, and a non-OK status on error. + virtual Status PutEntity(const WriteOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + const WideColumns& columns); + + // Remove the database entry (if any) for "key". Returns OK on + // success, and a non-OK status on error. It is not an error if "key" + // did not exist in the database. + // Note: consider setting options.sync = true. + virtual Status Delete(const WriteOptions& options, + ColumnFamilyHandle* column_family, + const Slice& key) = 0; + virtual Status Delete(const WriteOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + const Slice& ts) = 0; + virtual Status Delete(const WriteOptions& options, const Slice& key) { + return Delete(options, DefaultColumnFamily(), key); + } + virtual Status Delete(const WriteOptions& options, const Slice& key, + const Slice& ts) { + return Delete(options, DefaultColumnFamily(), key, ts); + } + + // Remove the database entry for "key". Requires that the key exists + // and was not overwritten. Returns OK on success, and a non-OK status + // on error. It is not an error if "key" did not exist in the database. + // + // If a key is overwritten (by calling Put() multiple times), then the result + // of calling SingleDelete() on this key is undefined. SingleDelete() only + // behaves correctly if there has been only one Put() for this key since the + // previous call to SingleDelete() for this key. + // + // This feature is currently an experimental performance optimization + // for a very specific workload. It is up to the caller to ensure that + // SingleDelete is only used for a key that is not deleted using Delete() or + // written using Merge(). Mixing SingleDelete operations with Deletes and + // Merges can result in undefined behavior. + // + // Note: consider setting options.sync = true. + virtual Status SingleDelete(const WriteOptions& options, + ColumnFamilyHandle* column_family, + const Slice& key) = 0; + virtual Status SingleDelete(const WriteOptions& options, + ColumnFamilyHandle* column_family, + const Slice& key, const Slice& ts) = 0; + virtual Status SingleDelete(const WriteOptions& options, const Slice& key) { + return SingleDelete(options, DefaultColumnFamily(), key); + } + virtual Status SingleDelete(const WriteOptions& options, const Slice& key, + const Slice& ts) { + return SingleDelete(options, DefaultColumnFamily(), key, ts); + } + + // Removes the database entries in the range ["begin_key", "end_key"), i.e., + // including "begin_key" and excluding "end_key". Returns OK on success, and + // a non-OK status on error. It is not an error if the database does not + // contain any existing data in the range ["begin_key", "end_key"). + // + // If "end_key" comes before "start_key" according to the user's comparator, + // a `Status::InvalidArgument` is returned. + // + // This feature is now usable in production, with the following caveats: + // 1) Accumulating too many range tombstones in the memtable will degrade read + // performance; this can be avoided by manually flushing occasionally. + // 2) Limiting the maximum number of open files in the presence of range + // tombstones can degrade read performance. To avoid this problem, set + // max_open_files to -1 whenever possible. + virtual Status DeleteRange(const WriteOptions& options, + ColumnFamilyHandle* column_family, + const Slice& begin_key, const Slice& end_key); + virtual Status DeleteRange(const WriteOptions& options, + ColumnFamilyHandle* column_family, + const Slice& begin_key, const Slice& end_key, + const Slice& ts); + + // Merge the database entry for "key" with "value". Returns OK on success, + // and a non-OK status on error. The semantics of this operation is + // determined by the user provided merge_operator when opening DB. + // Note: consider setting options.sync = true. + virtual Status Merge(const WriteOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + const Slice& value) = 0; + virtual Status Merge(const WriteOptions& options, const Slice& key, + const Slice& value) { + return Merge(options, DefaultColumnFamily(), key, value); + } + virtual Status Merge(const WriteOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/, + const Slice& /*key*/, const Slice& /*ts*/, + const Slice& /*value*/); + + // Apply the specified updates to the database. + // If `updates` contains no update, WAL will still be synced if + // options.sync=true. + // Returns OK on success, non-OK on failure. + // Note: consider setting options.sync = true. + virtual Status Write(const WriteOptions& options, WriteBatch* updates) = 0; + + // If the column family specified by "column_family" contains an entry for + // "key", return the corresponding value in "*value". If the entry is a plain + // key-value, return the value as-is; if it is a wide-column entity, return + // the value of its default anonymous column (see kDefaultWideColumnName) if + // any, or an empty value otherwise. + // + // If timestamp is enabled and a non-null timestamp pointer is passed in, + // timestamp is returned. + // + // Returns OK on success. Returns NotFound and an empty value in "*value" if + // there is no entry for "key". Returns some other non-OK status on error. + virtual inline Status Get(const ReadOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + std::string* value) { + assert(value != nullptr); + PinnableSlice pinnable_val(value); + assert(!pinnable_val.IsPinned()); + auto s = Get(options, column_family, key, &pinnable_val); + if (s.ok() && pinnable_val.IsPinned()) { + value->assign(pinnable_val.data(), pinnable_val.size()); + } // else value is already assigned + return s; + } + virtual Status Get(const ReadOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value) = 0; + virtual Status Get(const ReadOptions& options, const Slice& key, + std::string* value) { + return Get(options, DefaultColumnFamily(), key, value); + } + + // Get() methods that return timestamp. Derived DB classes don't need to worry + // about this group of methods if they don't care about timestamp feature. + virtual inline Status Get(const ReadOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + std::string* value, std::string* timestamp) { + assert(value != nullptr); + PinnableSlice pinnable_val(value); + assert(!pinnable_val.IsPinned()); + auto s = Get(options, column_family, key, &pinnable_val, timestamp); + if (s.ok() && pinnable_val.IsPinned()) { + value->assign(pinnable_val.data(), pinnable_val.size()); + } // else value is already assigned + return s; + } + virtual Status Get(const ReadOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/, + const Slice& /*key*/, PinnableSlice* /*value*/, + std::string* /*timestamp*/) { + return Status::NotSupported( + "Get() that returns timestamp is not implemented."); + } + virtual Status Get(const ReadOptions& options, const Slice& key, + std::string* value, std::string* timestamp) { + return Get(options, DefaultColumnFamily(), key, value, timestamp); + } + + // If the column family specified by "column_family" contains an entry for + // "key", return it as a wide-column entity in "*columns". If the entry is a + // wide-column entity, return it as-is; if it is a plain key-value, return it + // as an entity with a single anonymous column (see kDefaultWideColumnName) + // which contains the value. + // + // Returns OK on success. Returns NotFound and an empty wide-column entity in + // "*columns" if there is no entry for "key". Returns some other non-OK status + // on error. + virtual Status GetEntity(const ReadOptions& /* options */, + ColumnFamilyHandle* /* column_family */, + const Slice& /* key */, + PinnableWideColumns* /* columns */) { + return Status::NotSupported("GetEntity not supported"); + } + + // Populates the `merge_operands` array with all the merge operands in the DB + // for `key`. The `merge_operands` array will be populated in the order of + // insertion. The number of entries populated in `merge_operands` will be + // assigned to `*number_of_operands`. + // + // If the number of merge operands in DB for `key` is greater than + // `merge_operands_options.expected_max_number_of_operands`, + // `merge_operands` is not populated and the return value is + // `Status::Incomplete`. In that case, `*number_of_operands` will be assigned + // the number of merge operands found in the DB for `key`. + // + // `merge_operands`- Points to an array of at-least + // merge_operands_options.expected_max_number_of_operands and the + // caller is responsible for allocating it. + // + // The caller should delete or `Reset()` the `merge_operands` entries when + // they are no longer needed. All `merge_operands` entries must be destroyed + // or `Reset()` before this DB is closed or destroyed. + virtual Status GetMergeOperands( + const ReadOptions& options, ColumnFamilyHandle* column_family, + const Slice& key, PinnableSlice* merge_operands, + GetMergeOperandsOptions* get_merge_operands_options, + int* number_of_operands) = 0; + + // Consistent Get of many keys across column families without the need + // for an explicit snapshot. NOTE: the implementation of this MultiGet API + // does not have the performance benefits of the void-returning MultiGet + // functions. + // + // If keys[i] does not exist in the database, then the i'th returned + // status will be one for which Status::IsNotFound() is true, and + // (*values)[i] will be set to some arbitrary value (often ""). Otherwise, + // the i'th returned status will have Status::ok() true, and (*values)[i] + // will store the value associated with keys[i]. + // + // (*values) will always be resized to be the same size as (keys). + // Similarly, the number of returned statuses will be the number of keys. + // Note: keys will not be "de-duplicated". Duplicate keys will return + // duplicate values in order. + virtual std::vector MultiGet( + const ReadOptions& options, + const std::vector& column_family, + const std::vector& keys, std::vector* values) = 0; + virtual std::vector MultiGet(const ReadOptions& options, + const std::vector& keys, + std::vector* values) { + return MultiGet( + options, + std::vector(keys.size(), DefaultColumnFamily()), + keys, values); + } + + virtual std::vector MultiGet( + const ReadOptions& /*options*/, + const std::vector& /*column_family*/, + const std::vector& keys, std::vector* /*values*/, + std::vector* /*timestamps*/) { + return std::vector( + keys.size(), Status::NotSupported( + "MultiGet() returning timestamps not implemented.")); + } + virtual std::vector MultiGet(const ReadOptions& options, + const std::vector& keys, + std::vector* values, + std::vector* timestamps) { + return MultiGet( + options, + std::vector(keys.size(), DefaultColumnFamily()), + keys, values, timestamps); + } + + // Overloaded MultiGet API that improves performance by batching operations + // in the read path for greater efficiency. Currently, only the block based + // table format with full filters are supported. Other table formats such + // as plain table, block based table with block based filters and + // partitioned indexes will still work, but will not get any performance + // benefits. + // Parameters - + // options - ReadOptions + // column_family - ColumnFamilyHandle* that the keys belong to. All the keys + // passed to the API are restricted to a single column family + // num_keys - Number of keys to lookup + // keys - Pointer to C style array of key Slices with num_keys elements + // values - Pointer to C style array of PinnableSlices with num_keys elements + // statuses - Pointer to C style array of Status with num_keys elements + // sorted_input - If true, it means the input keys are already sorted by key + // order, so the MultiGet() API doesn't have to sort them + // again. If false, the keys will be copied and sorted + // internally by the API - the input array will not be + // modified + virtual void MultiGet(const ReadOptions& options, + ColumnFamilyHandle* column_family, + const size_t num_keys, const Slice* keys, + PinnableSlice* values, Status* statuses, + const bool /*sorted_input*/ = false) { + std::vector cf; + std::vector user_keys; + std::vector status; + std::vector vals; + + for (size_t i = 0; i < num_keys; ++i) { + cf.emplace_back(column_family); + user_keys.emplace_back(keys[i]); + } + status = MultiGet(options, cf, user_keys, &vals); + std::copy(status.begin(), status.end(), statuses); + for (auto& value : vals) { + values->PinSelf(value); + values++; + } + } + + virtual void MultiGet(const ReadOptions& options, + ColumnFamilyHandle* column_family, + const size_t num_keys, const Slice* keys, + PinnableSlice* values, std::string* timestamps, + Status* statuses, const bool /*sorted_input*/ = false) { + std::vector cf; + std::vector user_keys; + std::vector status; + std::vector vals; + std::vector tss; + + for (size_t i = 0; i < num_keys; ++i) { + cf.emplace_back(column_family); + user_keys.emplace_back(keys[i]); + } + status = MultiGet(options, cf, user_keys, &vals, &tss); + std::copy(status.begin(), status.end(), statuses); + std::copy(tss.begin(), tss.end(), timestamps); + for (auto& value : vals) { + values->PinSelf(value); + values++; + } + } + + // Overloaded MultiGet API that improves performance by batching operations + // in the read path for greater efficiency. Currently, only the block based + // table format with full filters are supported. Other table formats such + // as plain table, block based table with block based filters and + // partitioned indexes will still work, but will not get any performance + // benefits. + // Parameters - + // options - ReadOptions + // column_family - ColumnFamilyHandle* that the keys belong to. All the keys + // passed to the API are restricted to a single column family + // num_keys - Number of keys to lookup + // keys - Pointer to C style array of key Slices with num_keys elements + // values - Pointer to C style array of PinnableSlices with num_keys elements + // statuses - Pointer to C style array of Status with num_keys elements + // sorted_input - If true, it means the input keys are already sorted by key + // order, so the MultiGet() API doesn't have to sort them + // again. If false, the keys will be copied and sorted + // internally by the API - the input array will not be + // modified + virtual void MultiGet(const ReadOptions& options, const size_t num_keys, + ColumnFamilyHandle** column_families, const Slice* keys, + PinnableSlice* values, Status* statuses, + const bool /*sorted_input*/ = false) { + std::vector cf; + std::vector user_keys; + std::vector status; + std::vector vals; + + for (size_t i = 0; i < num_keys; ++i) { + cf.emplace_back(column_families[i]); + user_keys.emplace_back(keys[i]); + } + status = MultiGet(options, cf, user_keys, &vals); + std::copy(status.begin(), status.end(), statuses); + for (auto& value : vals) { + values->PinSelf(value); + values++; + } + } + virtual void MultiGet(const ReadOptions& options, const size_t num_keys, + ColumnFamilyHandle** column_families, const Slice* keys, + PinnableSlice* values, std::string* timestamps, + Status* statuses, const bool /*sorted_input*/ = false) { + std::vector cf; + std::vector user_keys; + std::vector status; + std::vector vals; + std::vector tss; + + for (size_t i = 0; i < num_keys; ++i) { + cf.emplace_back(column_families[i]); + user_keys.emplace_back(keys[i]); + } + status = MultiGet(options, cf, user_keys, &vals, &tss); + std::copy(status.begin(), status.end(), statuses); + std::copy(tss.begin(), tss.end(), timestamps); + for (auto& value : vals) { + values->PinSelf(value); + values++; + } + } + + // Batched MultiGet-like API that returns wide-column entities from a single + // column family. For any given "key[i]" in "keys" (where 0 <= "i" < + // "num_keys"), if the column family specified by "column_family" contains an + // entry, it is returned it as a wide-column entity in "results[i]". If the + // entry is a wide-column entity, it is returned as-is; if it is a plain + // key-value, it is returned as an entity with a single anonymous column (see + // kDefaultWideColumnName) which contains the value. + // + // "statuses[i]" is set to OK if "keys[i]" is successfully retrieved. It is + // set to NotFound and an empty wide-column entity is returned in "results[i]" + // if there is no entry for "keys[i]". Finally, "statuses[i]" is set to some + // other non-OK status on error. + // + // If "keys" are sorted according to the column family's comparator, the + // "sorted_input" flag can be set for a small performance improvement. + // + // Note that it is the caller's responsibility to ensure that "keys", + // "results", and "statuses" point to "num_keys" number of contiguous objects + // (Slices, PinnableWideColumns, and Statuses respectively). + virtual void MultiGetEntity(const ReadOptions& /* options */, + ColumnFamilyHandle* /* column_family */, + size_t num_keys, const Slice* /* keys */, + PinnableWideColumns* /* results */, + Status* statuses, + bool /* sorted_input */ = false) { + for (size_t i = 0; i < num_keys; ++i) { + statuses[i] = Status::NotSupported("MultiGetEntity not supported"); + } + } + + // Batched MultiGet-like API that returns wide-column entities potentially + // from multiple column families. For any given "key[i]" in "keys" (where 0 <= + // "i" < "num_keys"), if the column family specified by "column_families[i]" + // contains an entry, it is returned it as a wide-column entity in + // "results[i]". If the entry is a wide-column entity, it is returned as-is; + // if it is a plain key-value, it is returned as an entity with a single + // anonymous column (see kDefaultWideColumnName) which contains the value. + // + // "statuses[i]" is set to OK if "keys[i]" is successfully retrieved. It is + // set to NotFound and an empty wide-column entity is returned in "results[i]" + // if there is no entry for "keys[i]". Finally, "statuses[i]" is set to some + // other non-OK status on error. + // + // If "keys" are sorted by column family id and within each column family, + // according to the column family's comparator, the "sorted_input" flag can be + // set for a small performance improvement. + // + // Note that it is the caller's responsibility to ensure that + // "column_families", "keys", "results", and "statuses" point to "num_keys" + // number of contiguous objects (ColumnFamilyHandle pointers, Slices, + // PinnableWideColumns, and Statuses respectively). + virtual void MultiGetEntity(const ReadOptions& /* options */, size_t num_keys, + ColumnFamilyHandle** /* column_families */, + const Slice* /* keys */, + PinnableWideColumns* /* results */, + Status* statuses, + bool /* sorted_input */ = false) { + for (size_t i = 0; i < num_keys; ++i) { + statuses[i] = Status::NotSupported("MultiGetEntity not supported"); + } + } + + // If the key definitely does not exist in the database, then this method + // returns false, else true. If the caller wants to obtain value when the key + // is found in memory, a bool for 'value_found' must be passed. 'value_found' + // will be true on return if value has been set properly. + // This check is potentially lighter-weight than invoking DB::Get(). One way + // to make this lighter weight is to avoid doing any IOs. + // Default implementation here returns true and sets 'value_found' to false + virtual bool KeyMayExist(const ReadOptions& /*options*/, + ColumnFamilyHandle* /*column_family*/, + const Slice& /*key*/, std::string* /*value*/, + std::string* /*timestamp*/, + bool* value_found = nullptr) { + if (value_found != nullptr) { + *value_found = false; + } + return true; + } + + virtual bool KeyMayExist(const ReadOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + std::string* value, bool* value_found = nullptr) { + return KeyMayExist(options, column_family, key, value, + /*timestamp=*/nullptr, value_found); + } + + virtual bool KeyMayExist(const ReadOptions& options, const Slice& key, + std::string* value, bool* value_found = nullptr) { + return KeyMayExist(options, DefaultColumnFamily(), key, value, value_found); + } + + virtual bool KeyMayExist(const ReadOptions& options, const Slice& key, + std::string* value, std::string* timestamp, + bool* value_found = nullptr) { + return KeyMayExist(options, DefaultColumnFamily(), key, value, timestamp, + value_found); + } + + // Return a heap-allocated iterator over the contents of the database. + // The result of NewIterator() is initially invalid (caller must + // call one of the Seek methods on the iterator before using it). + // + // Caller should delete the iterator when it is no longer needed. + // The returned iterator should be deleted before this db is deleted. + virtual Iterator* NewIterator(const ReadOptions& options, + ColumnFamilyHandle* column_family) = 0; + virtual Iterator* NewIterator(const ReadOptions& options) { + return NewIterator(options, DefaultColumnFamily()); + } + // Returns iterators from a consistent database state across multiple + // column families. Iterators are heap allocated and need to be deleted + // before the db is deleted + virtual Status NewIterators( + const ReadOptions& options, + const std::vector& column_families, + std::vector* iterators) = 0; + + // Return a handle to the current DB state. Iterators created with + // this handle will all observe a stable snapshot of the current DB + // state. The caller must call ReleaseSnapshot(result) when the + // snapshot is no longer needed. + // + // nullptr will be returned if the DB fails to take a snapshot or does + // not support snapshot (eg: inplace_update_support enabled). + virtual const Snapshot* GetSnapshot() = 0; + + // Release a previously acquired snapshot. The caller must not + // use "snapshot" after this call. + virtual void ReleaseSnapshot(const Snapshot* snapshot) = 0; + + // Contains all valid property arguments for GetProperty() or + // GetMapProperty(). Each is a "string" property for retrieval with + // GetProperty() unless noted as a "map" property, for GetMapProperty(). + // + // NOTE: Property names cannot end in numbers since those are interpreted as + // arguments, e.g., see kNumFilesAtLevelPrefix. + struct Properties { + // "rocksdb.num-files-at-level" - returns string containing the number + // of files at level , where is an ASCII representation of a + // level number (e.g., "0"). + static const std::string kNumFilesAtLevelPrefix; + + // "rocksdb.compression-ratio-at-level" - returns string containing the + // compression ratio of data at level , where is an ASCII + // representation of a level number (e.g., "0"). Here, compression + // ratio is defined as uncompressed data size / compressed file size. + // Returns "-1.0" if no open files at level . + static const std::string kCompressionRatioAtLevelPrefix; + + // "rocksdb.stats" - returns a multi-line string containing the data + // described by kCFStats followed by the data described by kDBStats. + static const std::string kStats; + + // "rocksdb.sstables" - returns a multi-line string summarizing current + // SST files. + static const std::string kSSTables; + + // "rocksdb.cfstats" - Raw data from "rocksdb.cfstats-no-file-histogram" + // and "rocksdb.cf-file-histogram" as a "map" property. + static const std::string kCFStats; + + // "rocksdb.cfstats-no-file-histogram" - returns a multi-line string with + // general column family stats per-level over db's lifetime ("L"), + // aggregated over db's lifetime ("Sum"), and aggregated over the + // interval since the last retrieval ("Int"). + static const std::string kCFStatsNoFileHistogram; + + // "rocksdb.cf-file-histogram" - print out how many file reads to every + // level, as well as the histogram of latency of single requests. + static const std::string kCFFileHistogram; + + // "rocksdb.cf-write-stall-stats" - returns a multi-line string or + // map with statistics on CF-scope write stalls for a given CF + // See`WriteStallStatsMapKeys` for structured representation of keys + // available in the map form. + static const std::string kCFWriteStallStats; + + // "rocksdb.db-write-stall-stats" - returns a multi-line string or + // map with statistics on DB-scope write stalls + // See`WriteStallStatsMapKeys` for structured representation of keys + // available in the map form. + static const std::string kDBWriteStallStats; + + // "rocksdb.dbstats" - As a string property, returns a multi-line string + // with general database stats, both cumulative (over the db's + // lifetime) and interval (since the last retrieval of kDBStats). + // As a map property, returns cumulative stats only and does not + // update the baseline for the interval stats. + static const std::string kDBStats; + + // "rocksdb.levelstats" - returns multi-line string containing the number + // of files per level and total size of each level (MB). + static const std::string kLevelStats; + + // "rocksdb.block-cache-entry-stats" - returns a multi-line string or + // map with statistics on block cache usage. See + // `BlockCacheEntryStatsMapKeys` for structured representation of keys + // available in the map form. + static const std::string kBlockCacheEntryStats; + + // "rocksdb.fast-block-cache-entry-stats" - same as above, but returns + // stale values more frequently to reduce overhead and latency. + static const std::string kFastBlockCacheEntryStats; + + // "rocksdb.num-immutable-mem-table" - returns number of immutable + // memtables that have not yet been flushed. + static const std::string kNumImmutableMemTable; + + // "rocksdb.num-immutable-mem-table-flushed" - returns number of immutable + // memtables that have already been flushed. + static const std::string kNumImmutableMemTableFlushed; + + // "rocksdb.mem-table-flush-pending" - returns 1 if a memtable flush is + // pending; otherwise, returns 0. + static const std::string kMemTableFlushPending; + + // "rocksdb.num-running-flushes" - returns the number of currently running + // flushes. + static const std::string kNumRunningFlushes; + + // "rocksdb.compaction-pending" - returns 1 if at least one compaction is + // pending; otherwise, returns 0. + static const std::string kCompactionPending; + + // "rocksdb.num-running-compactions" - returns the number of currently + // running compactions. + static const std::string kNumRunningCompactions; + + // "rocksdb.background-errors" - returns accumulated number of background + // errors. + static const std::string kBackgroundErrors; + + // "rocksdb.cur-size-active-mem-table" - returns approximate size of active + // memtable (bytes). + static const std::string kCurSizeActiveMemTable; + + // "rocksdb.cur-size-all-mem-tables" - returns approximate size of active + // and unflushed immutable memtables (bytes). + static const std::string kCurSizeAllMemTables; + + // "rocksdb.size-all-mem-tables" - returns approximate size of active, + // unflushed immutable, and pinned immutable memtables (bytes). + static const std::string kSizeAllMemTables; + + // "rocksdb.num-entries-active-mem-table" - returns total number of entries + // in the active memtable. + static const std::string kNumEntriesActiveMemTable; + + // "rocksdb.num-entries-imm-mem-tables" - returns total number of entries + // in the unflushed immutable memtables. + static const std::string kNumEntriesImmMemTables; + + // "rocksdb.num-deletes-active-mem-table" - returns total number of delete + // entries in the active memtable. + static const std::string kNumDeletesActiveMemTable; + + // "rocksdb.num-deletes-imm-mem-tables" - returns total number of delete + // entries in the unflushed immutable memtables. + static const std::string kNumDeletesImmMemTables; + + // "rocksdb.estimate-num-keys" - returns estimated number of total keys in + // the active and unflushed immutable memtables and storage. + static const std::string kEstimateNumKeys; + + // "rocksdb.estimate-table-readers-mem" - returns estimated memory used for + // reading SST tables, excluding memory used in block cache (e.g., + // filter and index blocks). + static const std::string kEstimateTableReadersMem; + + // "rocksdb.is-file-deletions-enabled" - returns 0 if deletion of obsolete + // files is enabled; otherwise, returns a non-zero number. + // This name may be misleading because true(non-zero) means disable, + // but we keep the name for backward compatibility. + static const std::string kIsFileDeletionsEnabled; + + // "rocksdb.num-snapshots" - returns number of unreleased snapshots of the + // database. + static const std::string kNumSnapshots; + + // "rocksdb.oldest-snapshot-time" - returns number representing unix + // timestamp of oldest unreleased snapshot. + static const std::string kOldestSnapshotTime; + + // "rocksdb.oldest-snapshot-sequence" - returns number representing + // sequence number of oldest unreleased snapshot. + static const std::string kOldestSnapshotSequence; + + // "rocksdb.num-live-versions" - returns number of live versions. `Version` + // is an internal data structure. See version_set.h for details. More + // live versions often mean more SST files are held from being deleted, + // by iterators or unfinished compactions. + static const std::string kNumLiveVersions; + + // "rocksdb.current-super-version-number" - returns number of current LSM + // version. It is a uint64_t integer number, incremented after there is + // any change to the LSM tree. The number is not preserved after restarting + // the DB. After DB restart, it will start from 0 again. + static const std::string kCurrentSuperVersionNumber; + + // "rocksdb.estimate-live-data-size" - returns an estimate of the amount of + // live data in bytes. For BlobDB, it also includes the exact value of + // live bytes in the blob files of the version. + static const std::string kEstimateLiveDataSize; + + // "rocksdb.min-log-number-to-keep" - return the minimum log number of the + // log files that should be kept. + static const std::string kMinLogNumberToKeep; + + // "rocksdb.min-obsolete-sst-number-to-keep" - return the minimum file + // number for an obsolete SST to be kept. The max value of `uint64_t` + // will be returned if all obsolete files can be deleted. + static const std::string kMinObsoleteSstNumberToKeep; + + // "rocksdb.total-sst-files-size" - returns total size (bytes) of all SST + // files belonging to any of the CF's versions. + // WARNING: may slow down online queries if there are too many files. + static const std::string kTotalSstFilesSize; + + // "rocksdb.live-sst-files-size" - returns total size (bytes) of all SST + // files belong to the CF's current version. + static const std::string kLiveSstFilesSize; + + // "rocksdb.obsolete-sst-files-size" - returns total size (bytes) of all + // SST files that became obsolete but have not yet been deleted or + // scheduled for deletion. SST files can end up in this state when + // using `DisableFileDeletions()`, for example. + // + // N.B. Unlike the other "*SstFilesSize" properties, this property + // includes SST files that originated in any of the DB's CFs. + static const std::string kObsoleteSstFilesSize; + + // "rocksdb.live_sst_files_size_at_temperature" - returns total size (bytes) + // of SST files at all certain file temperature + static const std::string kLiveSstFilesSizeAtTemperature; + + // "rocksdb.base-level" - returns number of level to which L0 data will be + // compacted. + static const std::string kBaseLevel; + + // "rocksdb.estimate-pending-compaction-bytes" - returns estimated total + // number of bytes compaction needs to rewrite to get all levels down + // to under target size. Not valid for other compactions than level- + // based. + static const std::string kEstimatePendingCompactionBytes; + + // "rocksdb.aggregated-table-properties" - returns a string or map + // representation of the aggregated table properties of the target + // column family. Only properties that make sense for aggregation + // are included. + static const std::string kAggregatedTableProperties; + + // "rocksdb.aggregated-table-properties-at-level", same as the previous + // one but only returns the aggregated table properties of the + // specified level "N" at the target column family. + static const std::string kAggregatedTablePropertiesAtLevel; + + // "rocksdb.actual-delayed-write-rate" - returns the current actual delayed + // write rate. 0 means no delay. + static const std::string kActualDelayedWriteRate; + + // "rocksdb.is-write-stopped" - Return 1 if write has been stopped. + static const std::string kIsWriteStopped; + + // "rocksdb.estimate-oldest-key-time" - returns an estimation of + // oldest key timestamp in the DB. Currently only available for + // FIFO compaction with + // compaction_options_fifo.allow_compaction = false. + static const std::string kEstimateOldestKeyTime; + + // "rocksdb.block-cache-capacity" - returns block cache capacity. + static const std::string kBlockCacheCapacity; + + // "rocksdb.block-cache-usage" - returns the memory size for the entries + // residing in block cache. + static const std::string kBlockCacheUsage; + + // "rocksdb.block-cache-pinned-usage" - returns the memory size for the + // entries being pinned. + static const std::string kBlockCachePinnedUsage; + + // "rocksdb.options-statistics" - returns multi-line string + // of options.statistics + static const std::string kOptionsStatistics; + + // "rocksdb.num-blob-files" - returns number of blob files in the current + // version. + static const std::string kNumBlobFiles; + + // "rocksdb.blob-stats" - return the total number and size of all blob + // files, and total amount of garbage (bytes) in the blob files in + // the current version. + static const std::string kBlobStats; + + // "rocksdb.total-blob-file-size" - returns the total size of all blob + // files over all versions. + static const std::string kTotalBlobFileSize; + + // "rocksdb.live-blob-file-size" - returns the total size of all blob + // files in the current version. + static const std::string kLiveBlobFileSize; + + // "rocksdb.live-blob-file-garbage-size" - returns the total amount of + // garbage in the blob files in the current version. + static const std::string kLiveBlobFileGarbageSize; + + // "rocksdb.blob-cache-capacity" - returns blob cache capacity. + static const std::string kBlobCacheCapacity; + + // "rocksdb.blob-cache-usage" - returns the memory size for the entries + // residing in blob cache. + static const std::string kBlobCacheUsage; + + // "rocksdb.blob-cache-pinned-usage" - returns the memory size for the + // entries being pinned in blob cache. + static const std::string kBlobCachePinnedUsage; + }; + + // DB implementations export properties about their state via this method. + // If "property" is a valid "string" property understood by this DB + // implementation (see Properties struct above for valid options), fills + // "*value" with its current value and returns true. Otherwise, returns + // false. + virtual bool GetProperty(ColumnFamilyHandle* column_family, + const Slice& property, std::string* value) = 0; + virtual bool GetProperty(const Slice& property, std::string* value) { + return GetProperty(DefaultColumnFamily(), property, value); + } + + // Like GetProperty but for valid "map" properties. (Some properties can be + // accessed as either "string" properties or "map" properties.) + virtual bool GetMapProperty(ColumnFamilyHandle* column_family, + const Slice& property, + std::map* value) = 0; + virtual bool GetMapProperty(const Slice& property, + std::map* value) { + return GetMapProperty(DefaultColumnFamily(), property, value); + } + + // Similar to GetProperty(), but only works for a subset of properties whose + // return value is an integer. Return the value by integer. Supported + // properties: + // "rocksdb.num-immutable-mem-table" + // "rocksdb.mem-table-flush-pending" + // "rocksdb.compaction-pending" + // "rocksdb.background-errors" + // "rocksdb.cur-size-active-mem-table" + // "rocksdb.cur-size-all-mem-tables" + // "rocksdb.size-all-mem-tables" + // "rocksdb.num-entries-active-mem-table" + // "rocksdb.num-entries-imm-mem-tables" + // "rocksdb.num-deletes-active-mem-table" + // "rocksdb.num-deletes-imm-mem-tables" + // "rocksdb.estimate-num-keys" + // "rocksdb.estimate-table-readers-mem" + // "rocksdb.is-file-deletions-enabled" + // "rocksdb.num-snapshots" + // "rocksdb.oldest-snapshot-time" + // "rocksdb.num-live-versions" + // "rocksdb.current-super-version-number" + // "rocksdb.estimate-live-data-size" + // "rocksdb.min-log-number-to-keep" + // "rocksdb.min-obsolete-sst-number-to-keep" + // "rocksdb.total-sst-files-size" + // "rocksdb.live-sst-files-size" + // "rocksdb.obsolete-sst-files-size" + // "rocksdb.base-level" + // "rocksdb.estimate-pending-compaction-bytes" + // "rocksdb.num-running-compactions" + // "rocksdb.num-running-flushes" + // "rocksdb.actual-delayed-write-rate" + // "rocksdb.is-write-stopped" + // "rocksdb.estimate-oldest-key-time" + // "rocksdb.block-cache-capacity" + // "rocksdb.block-cache-usage" + // "rocksdb.block-cache-pinned-usage" + // + // Properties dedicated for BlobDB: + // "rocksdb.num-blob-files" + // "rocksdb.total-blob-file-size" + // "rocksdb.live-blob-file-size" + // "rocksdb.blob-cache-capacity" + // "rocksdb.blob-cache-usage" + // "rocksdb.blob-cache-pinned-usage" + virtual bool GetIntProperty(ColumnFamilyHandle* column_family, + const Slice& property, uint64_t* value) = 0; + virtual bool GetIntProperty(const Slice& property, uint64_t* value) { + return GetIntProperty(DefaultColumnFamily(), property, value); + } + + // Reset internal stats for DB and all column families. + // Note this doesn't reset options.statistics as it is not owned by + // DB. + virtual Status ResetStats() { + return Status::NotSupported("Not implemented"); + } + + // Same as GetIntProperty(), but this one returns the aggregated int + // property from all column families. + virtual bool GetAggregatedIntProperty(const Slice& property, + uint64_t* value) = 0; + + // Flags for DB::GetSizeApproximation that specify whether memtable + // stats should be included, or file stats approximation or both + enum class SizeApproximationFlags : uint8_t { + NONE = 0, + INCLUDE_MEMTABLES = 1 << 0, + INCLUDE_FILES = 1 << 1 + }; + + // For each i in [0,n-1], store in "sizes[i]", the approximate + // file system space used by keys in "[range[i].start .. range[i].limit)" + // in a single column family. + // + // Note that the returned sizes measure file system space usage, so + // if the user data compresses by a factor of ten, the returned + // sizes will be one-tenth the size of the corresponding user data size. + virtual Status GetApproximateSizes(const SizeApproximationOptions& options, + ColumnFamilyHandle* column_family, + const Range* ranges, int n, + uint64_t* sizes) = 0; + + // Simpler versions of the GetApproximateSizes() method above. + // The include_flags argument must of type DB::SizeApproximationFlags + // and can not be NONE. + virtual Status GetApproximateSizes(ColumnFamilyHandle* column_family, + const Range* ranges, int n, + uint64_t* sizes, + SizeApproximationFlags include_flags = + SizeApproximationFlags::INCLUDE_FILES); + + virtual Status GetApproximateSizes( + const Range* ranges, int n, uint64_t* sizes, + SizeApproximationFlags include_flags = + SizeApproximationFlags::INCLUDE_FILES) { + return GetApproximateSizes(DefaultColumnFamily(), ranges, n, sizes, + include_flags); + } + + // The method is similar to GetApproximateSizes, except it + // returns approximate number of records in memtables. + virtual void GetApproximateMemTableStats(ColumnFamilyHandle* column_family, + const Range& range, + uint64_t* const count, + uint64_t* const size) = 0; + virtual void GetApproximateMemTableStats(const Range& range, + uint64_t* const count, + uint64_t* const size) { + GetApproximateMemTableStats(DefaultColumnFamily(), range, count, size); + } + + // Compact the underlying storage for the key range [*begin,*end]. + // The actual compaction interval might be superset of [*begin, *end]. + // In particular, deleted and overwritten versions are discarded, + // and the data is rearranged to reduce the cost of operations + // needed to access the data. This operation should typically only + // be invoked by users who understand the underlying implementation. + // This call blocks until the operation completes successfully, fails, + // or is aborted (Status::Incomplete). See DisableManualCompaction. + // + // begin==nullptr is treated as a key before all keys in the database. + // end==nullptr is treated as a key after all keys in the database. + // Therefore the following call will compact the entire database: + // db->CompactRange(options, nullptr, nullptr); + // Note that after the entire database is compacted, all data are pushed + // down to the last level containing any data. If the total data size after + // compaction is reduced, that level might not be appropriate for hosting all + // the files. In this case, client could set options.change_level to true, to + // move the files back to the minimum level capable of holding the data set + // or a given level (specified by non-negative options.target_level). + virtual Status CompactRange(const CompactRangeOptions& options, + ColumnFamilyHandle* column_family, + const Slice* begin, const Slice* end) = 0; + virtual Status CompactRange(const CompactRangeOptions& options, + const Slice* begin, const Slice* end) { + return CompactRange(options, DefaultColumnFamily(), begin, end); + } + + // Dynamically change column family options or table factory options in a + // running DB, for the specified column family. Only options internally + // marked as "mutable" can be changed. Options not listed in `opts_map` will + // keep their current values. See GetColumnFamilyOptionsFromMap() in + // convenience.h for the details of `opts_map`. Not supported in LITE mode. + // + // USABILITY NOTE: SetOptions is intended only for expert users, and does + // not apply the same sanitization to options as the standard DB::Open code + // path does. Use with caution. + // + // RELIABILITY & PERFORMANCE NOTE: SetOptions is not fully stress-tested for + // reliability, and this is a slow call because a new OPTIONS file is + // serialized and persisted for each call. Use only infrequently. + // + // EXAMPLES: + // s = db->SetOptions(cfh, {{"ttl", "36000"}}); + // s = db->SetOptions(cfh, {{"block_based_table_factory", + // "{prepopulate_block_cache=kDisable;}"}}); + virtual Status SetOptions( + ColumnFamilyHandle* /*column_family*/, + const std::unordered_map& /*opts_map*/) { + return Status::NotSupported("Not implemented"); + } + // Shortcut for SetOptions on the default column family handle. + virtual Status SetOptions( + const std::unordered_map& new_options) { + return SetOptions(DefaultColumnFamily(), new_options); + } + + // Like SetOptions but for DBOptions, including the same caveats for + // usability, reliability, and performance. See GetDBOptionsFromMap() (and + // GetColumnFamilyOptionsFromMap()) in convenience.h for details on + // `opts_map`. Note supported in LITE mode. + // + // EXAMPLES: + // s = db->SetDBOptions({{"max_subcompactions", "2"}}); + // s = db->SetDBOptions({{"stats_dump_period_sec", "0"}, + // {"stats_persist_period_sec", "0"}}); + virtual Status SetDBOptions( + const std::unordered_map& new_options) = 0; + + // CompactFiles() inputs a list of files specified by file numbers and + // compacts them to the specified level. A small difference compared to + // CompactRange() is that CompactFiles() performs the compaction job + // using the CURRENT thread, so is not considered a "background" job. + // + // @see GetDataBaseMetaData + // @see GetColumnFamilyMetaData + virtual Status CompactFiles( + const CompactionOptions& compact_options, + ColumnFamilyHandle* column_family, + const std::vector& input_file_names, const int output_level, + const int output_path_id = -1, + std::vector* const output_file_names = nullptr, + CompactionJobInfo* compaction_job_info = nullptr) = 0; + + virtual Status CompactFiles( + const CompactionOptions& compact_options, + const std::vector& input_file_names, const int output_level, + const int output_path_id = -1, + std::vector* const output_file_names = nullptr, + CompactionJobInfo* compaction_job_info = nullptr) { + return CompactFiles(compact_options, DefaultColumnFamily(), + input_file_names, output_level, output_path_id, + output_file_names, compaction_job_info); + } + + // This function will wait until all currently running background processes + // finish. After it returns, no background process will be run until + // ContinueBackgroundWork is called, once for each preceding OK-returning + // call to PauseBackgroundWork. + virtual Status PauseBackgroundWork() = 0; + virtual Status ContinueBackgroundWork() = 0; + + // This function will enable automatic compactions for the given column + // families if they were previously disabled. The function will first set the + // disable_auto_compactions option for each column family to 'false', after + // which it will schedule a flush/compaction. + // + // NOTE: Setting disable_auto_compactions to 'false' through SetOptions() API + // does NOT schedule a flush/compaction afterwards, and only changes the + // parameter itself within the column family option. + // + virtual Status EnableAutoCompaction( + const std::vector& column_family_handles) = 0; + + // After this function call, CompactRange() or CompactFiles() will not + // run compactions and fail. Calling this function will tell outstanding + // manual compactions to abort and will wait for them to finish or abort + // before returning. + virtual void DisableManualCompaction() = 0; + // Re-enable CompactRange() and ComapctFiles() that are disabled by + // DisableManualCompaction(). This function must be called as many times + // as DisableManualCompaction() has been called in order to re-enable + // manual compactions, and must not be called more times than + // DisableManualCompaction() has been called. + virtual void EnableManualCompaction() = 0; + + // Wait for all flush and compactions jobs to finish. Jobs to wait include the + // unscheduled (queued, but not scheduled yet). If the db is shutting down, + // Status::ShutdownInProgress will be returned. + // + // NOTE: This may also never return if there's sufficient ongoing writes that + // keeps flush and compaction going without stopping. The user would have to + // cease all the writes to DB to make this eventually return in a stable + // state. + virtual Status WaitForCompact( + const WaitForCompactOptions& /* wait_for_compact_options */) = 0; + + // Number of levels used for this DB. + virtual int NumberLevels(ColumnFamilyHandle* column_family) = 0; + virtual int NumberLevels() { return NumberLevels(DefaultColumnFamily()); } + + // Maximum level to which a new compacted memtable is pushed if it + // does not create overlap. + virtual int MaxMemCompactionLevel(ColumnFamilyHandle* column_family) = 0; + virtual int MaxMemCompactionLevel() { + return MaxMemCompactionLevel(DefaultColumnFamily()); + } + + // Number of files in level-0 that would stop writes. + virtual int Level0StopWriteTrigger(ColumnFamilyHandle* column_family) = 0; + virtual int Level0StopWriteTrigger() { + return Level0StopWriteTrigger(DefaultColumnFamily()); + } + + // Get DB name -- the exact same name that was provided as an argument to + // DB::Open() + virtual const std::string& GetName() const = 0; + + // Get Env object from the DB + virtual Env* GetEnv() const = 0; + + // A shortcut for GetEnv()->->GetFileSystem().get(), possibly cached for + // efficiency. + virtual FileSystem* GetFileSystem() const; + + // Get DB Options that we use. During the process of opening the + // column family, the options provided when calling DB::Open() or + // DB::CreateColumnFamily() will have been "sanitized" and transformed + // in an implementation-defined manner. + virtual Options GetOptions(ColumnFamilyHandle* column_family) const = 0; + virtual Options GetOptions() const { + return GetOptions(DefaultColumnFamily()); + } + + virtual DBOptions GetDBOptions() const = 0; + + // Flush all memtable data. + // Flush a single column family, even when atomic flush is enabled. To flush + // multiple column families, use Flush(options, column_families). + virtual Status Flush(const FlushOptions& options, + ColumnFamilyHandle* column_family) = 0; + virtual Status Flush(const FlushOptions& options) { + return Flush(options, DefaultColumnFamily()); + } + // Flushes memtables of multiple column families. + // If atomic flush is not enabled, Flush(options, column_families) is + // equivalent to calling Flush(options, column_family) multiple times. + // If atomic flush is enabled, Flush(options, column_families) will flush all + // column families specified in 'column_families' up to the latest sequence + // number at the time when flush is requested. + // Note that RocksDB 5.15 and earlier may not be able to open later versions + // with atomic flush enabled. + virtual Status Flush( + const FlushOptions& options, + const std::vector& column_families) = 0; + + // When using the manual_wal_flush option, flushes RocksDB internal buffers + // of WAL data to the file, so that the data can survive process crash or be + // included in a Checkpoint or Backup. Without manual_wal_flush, there is no + // such internal buffer. If sync is true, it calls SyncWAL() afterwards. + virtual Status FlushWAL(bool /*sync*/) { + return Status::NotSupported("FlushWAL not implemented"); + } + + // Ensure all WAL writes have been synced to storage, so that (assuming OS + // and hardware support) data will survive power loss. This function does + // not imply FlushWAL, so `FlushWAL(true)` is recommended if using + // manual_wal_flush=true. Currently only works if allow_mmap_writes = false + // in Options. + // + // Note that Write() followed by SyncWAL() is not exactly the same as Write() + // with sync=true: in the latter case the changes won't be visible until the + // sync is done. + virtual Status SyncWAL() = 0; + + // Freezes the logical state of the DB (by stopping writes), and if WAL is + // enabled, ensures that state has been flushed to DB files (as in + // FlushWAL()). This can be used for taking a Checkpoint at a known DB + // state, though the user must use options to insure no DB flush is invoked + // in this frozen state. Other operations allowed on a "read only" DB should + // work while frozen. Each LockWAL() call that returns OK must eventually be + // followed by a corresponding call to UnlockWAL(). Where supported, non-OK + // status is generally only possible with some kind of corruption or I/O + // error. + virtual Status LockWAL() { + return Status::NotSupported("LockWAL not implemented"); + } + + // Unfreeze the DB state from a successful LockWAL(). + // The write stop on the database will be cleared when UnlockWAL() have been + // called for each successful LockWAL(). + virtual Status UnlockWAL() { + return Status::NotSupported("UnlockWAL not implemented"); + } + + // The sequence number of the most recent transaction. + virtual SequenceNumber GetLatestSequenceNumber() const = 0; + + // Prevent file deletions. Compactions will continue to occur, + // but no obsolete files will be deleted. Calling this multiple + // times have the same effect as calling it once. + virtual Status DisableFileDeletions() = 0; + + // Increase the full_history_ts of column family. The new ts_low value should + // be newer than current full_history_ts value. + // If another thread updates full_history_ts_low concurrently to a higher + // timestamp than the requested ts_low, a try again error will be returned. + virtual Status IncreaseFullHistoryTsLow(ColumnFamilyHandle* column_family, + std::string ts_low) = 0; + + // Get current full_history_ts value. + virtual Status GetFullHistoryTsLow(ColumnFamilyHandle* column_family, + std::string* ts_low) = 0; + + // Allow compactions to delete obsolete files. + // If force == true, the call to EnableFileDeletions() will guarantee that + // file deletions are enabled after the call, even if DisableFileDeletions() + // was called multiple times before. + // If force == false, EnableFileDeletions will only enable file deletion + // after it's been called at least as many times as DisableFileDeletions(), + // enabling the two methods to be called by two threads concurrently without + // synchronization -- i.e., file deletions will be enabled only after both + // threads call EnableFileDeletions() + virtual Status EnableFileDeletions(bool force = true) = 0; + + // Retrieves the creation time of the oldest file in the DB. + // This API only works if max_open_files = -1, if it is not then + // Status returned is Status::NotSupported() + // The file creation time is set using the env provided to the DB. + // If the DB was created from a very old release then its possible that + // the SST files might not have file_creation_time property and even after + // moving to a newer release its possible that some files never got compacted + // and may not have file_creation_time property. In both the cases + // file_creation_time is considered 0 which means this API will return + // creation_time = 0 as there wouldn't be a timestamp lower than 0. + virtual Status GetCreationTimeOfOldestFile(uint64_t* creation_time) = 0; + + // Note: this API is not yet consistent with WritePrepared transactions. + // + // Sets iter to an iterator that is positioned at a write-batch whose + // sequence number range [start_seq, end_seq] covers seq_number. If no such + // write-batch exists, then iter is positioned at the next write-batch whose + // start_seq > seq_number. + // + // Returns Status::OK if iterator is valid + // Must set WAL_ttl_seconds or WAL_size_limit_MB to large values to + // use this api, else the WAL files will get + // cleared aggressively and the iterator might keep getting invalid before + // an update is read. + virtual Status GetUpdatesSince( + SequenceNumber seq_number, std::unique_ptr* iter, + const TransactionLogIterator::ReadOptions& read_options = + TransactionLogIterator::ReadOptions()) = 0; + +// Windows API macro interference +#undef DeleteFile + // WARNING: This API is planned for removal in RocksDB 7.0 since it does not + // operate at the proper level of abstraction for a key-value store, and its + // contract/restrictions are poorly documented. For example, it returns non-OK + // `Status` for non-bottommost files and files undergoing compaction. Since we + // do not plan to maintain it, the contract will likely remain underspecified + // until its removal. Any user is encouraged to read the implementation + // carefully and migrate away from it when possible. + // + // Delete the file name from the db directory and update the internal state to + // reflect that. Supports deletion of sst and log files only. 'name' must be + // path relative to the db directory. eg. 000001.sst, /archive/000003.log + virtual Status DeleteFile(std::string name) = 0; + + // Obtains a list of all live table (SST) files and how they fit into the + // LSM-trees, such as column family, level, key range, etc. + // This builds a de-normalized form of GetAllColumnFamilyMetaData(). + // For information about all files in a DB, use GetLiveFilesStorageInfo(). + virtual void GetLiveFilesMetaData( + std::vector* /*metadata*/) {} + + // Return a list of all table (SST) and blob files checksum info. + // Note: This function might be of limited use because it cannot be + // synchronized with other "live files" APIs. GetLiveFilesStorageInfo() + // is recommended instead. + virtual Status GetLiveFilesChecksumInfo(FileChecksumList* checksum_list) = 0; + + // Get information about all live files that make up a DB, for making + // live copies (Checkpoint, backups, etc.) or other storage-related purposes. + // If creating a live copy, use DisableFileDeletions() before and + // EnableFileDeletions() after to prevent deletions. + // For LSM-tree metadata, use Get*MetaData() functions instead. + virtual Status GetLiveFilesStorageInfo( + const LiveFilesStorageInfoOptions& opts, + std::vector* files) = 0; + + // Obtains the LSM-tree meta data of the specified column family of the DB, + // including metadata for each live table (SST) file in that column family. + virtual void GetColumnFamilyMetaData(ColumnFamilyHandle* /*column_family*/, + ColumnFamilyMetaData* /*metadata*/) {} + + // Get the metadata of the default column family. + void GetColumnFamilyMetaData(ColumnFamilyMetaData* metadata) { + GetColumnFamilyMetaData(DefaultColumnFamily(), metadata); + } + + // Obtains the LSM-tree meta data of all column families of the DB, including + // metadata for each live table (SST) file and each blob file in the DB. + virtual void GetAllColumnFamilyMetaData( + std::vector* /*metadata*/) {} + + // Retrieve the list of all files in the database except WAL files. The files + // are relative to the dbname (or db_paths/cf_paths), not absolute paths. + // (Not recommended with db_paths/cf_paths because that information is not + // returned.) Despite being relative paths, the file names begin with "/". + // The valid size of the manifest file is returned in manifest_file_size. + // The manifest file is an ever growing file, but only the portion specified + // by manifest_file_size is valid for this snapshot. Setting flush_memtable + // to true does Flush before recording the live files (unless DB is + // read-only). Setting flush_memtable to false is useful when we don't want + // to wait for flush which may have to wait for compaction to complete + // taking an indeterminate time. + // + // NOTE: Although GetLiveFiles() followed by GetSortedWalFiles() can generate + // a lossless backup, GetLiveFilesStorageInfo() is strongly recommended + // instead, because it ensures a single consistent view of all files is + // captured in one call. + virtual Status GetLiveFiles(std::vector&, + uint64_t* manifest_file_size, + bool flush_memtable = true) = 0; + + // Retrieve the sorted list of all wal files with earliest file first + virtual Status GetSortedWalFiles(VectorLogPtr& files) = 0; + + // Retrieve information about the current wal file + // + // Note that the log might have rolled after this call in which case + // the current_log_file would not point to the current log file. + // + // Additionally, for the sake of optimization current_log_file->StartSequence + // would always be set to 0 + virtual Status GetCurrentWalFile( + std::unique_ptr* current_log_file) = 0; + + // IngestExternalFile() will load a list of external SST files (1) into the DB + // Two primary modes are supported: + // - Duplicate keys in the new files will overwrite exiting keys (default) + // - Duplicate keys will be skipped (set ingest_behind=true) + // In the first mode we will try to find the lowest possible level that + // the file can fit in, and ingest the file into this level (2). A file that + // have a key range that overlap with the memtable key range will require us + // to Flush the memtable first before ingesting the file. + // In the second mode we will always ingest in the bottom most level (see + // docs to IngestExternalFileOptions::ingest_behind). + // + // (1) External SST files can be created using SstFileWriter + // (2) We will try to ingest the files to the lowest possible level + // even if the file compression doesn't match the level compression + // (3) If IngestExternalFileOptions->ingest_behind is set to true, + // we always ingest at the bottommost level, which should be reserved + // for this purpose (see DBOPtions::allow_ingest_behind flag). + // (4) If IngestExternalFileOptions->fail_if_not_bottommost_level is set to + // true, then this method can return Status:TryAgain() indicating that + // the files cannot be ingested to the bottommost level, and it is the + // user's responsibility to clear the bottommost level in the overlapping + // range before re-attempting the ingestion. + virtual Status IngestExternalFile( + ColumnFamilyHandle* column_family, + const std::vector& external_files, + const IngestExternalFileOptions& options) = 0; + + virtual Status IngestExternalFile( + const std::vector& external_files, + const IngestExternalFileOptions& options) { + return IngestExternalFile(DefaultColumnFamily(), external_files, options); + } + + // IngestExternalFiles() will ingest files for multiple column families, and + // record the result atomically to the MANIFEST. + // If this function returns OK, all column families' ingestion must succeed. + // If this function returns NOK, or the process crashes, then non-of the + // files will be ingested into the database after recovery. + // Note that it is possible for application to observe a mixed state during + // the execution of this function. If the user performs range scan over the + // column families with iterators, iterator on one column family may return + // ingested data, while iterator on other column family returns old data. + // Users can use snapshot for a consistent view of data. + // If your db ingests multiple SST files using this API, i.e. args.size() + // > 1, then RocksDB 5.15 and earlier will not be able to open it. + // + // REQUIRES: each arg corresponds to a different column family: namely, for + // 0 <= i < j < len(args), args[i].column_family != args[j].column_family. + virtual Status IngestExternalFiles( + const std::vector& args) = 0; + + // CreateColumnFamilyWithImport() will create a new column family with + // column_family_name and import external SST files specified in `metadata` + // into this column family. + // (1) External SST files can be created using SstFileWriter. + // (2) External SST files can be exported from a particular column family in + // an existing DB using Checkpoint::ExportColumnFamily. `metadata` should + // be the output from Checkpoint::ExportColumnFamily. + // Option in import_options specifies whether the external files are copied or + // moved (default is copy). When option specifies copy, managing files at + // external_file_path is caller's responsibility. When option specifies a + // move, the call makes a best effort to delete the specified files at + // external_file_path on successful return, logging any failure to delete + // rather than returning in Status. Files are not modified on any error + // return, and a best effort is made to remove any newly-created files. + // On error return, column family handle returned will be nullptr. + // ColumnFamily will be present on successful return and will not be present + // on error return. ColumnFamily may be present on any crash during this call. + virtual Status CreateColumnFamilyWithImport( + const ColumnFamilyOptions& options, const std::string& column_family_name, + const ImportColumnFamilyOptions& import_options, + const ExportImportFilesMetaData& metadata, ColumnFamilyHandle** handle) { + const std::vector& metadatas{&metadata}; + return CreateColumnFamilyWithImport(options, column_family_name, + import_options, metadatas, handle); + } + + // EXPERIMENTAL + // Overload of the CreateColumnFamilyWithImport() that allows the caller to + // pass a list of ExportImportFilesMetaData pointers to support creating + // ColumnFamily by importing multiple ColumnFamilies. + // It should be noticed that if the user keys of the imported column families + // overlap with each other, an error will be returned. + virtual Status CreateColumnFamilyWithImport( + const ColumnFamilyOptions& options, const std::string& column_family_name, + const ImportColumnFamilyOptions& import_options, + const std::vector& metadatas, + ColumnFamilyHandle** handle) = 0; + + // EXPERIMENTAL + // ClipColumnFamily() will clip the entries in the CF according to the range + // [begin_key, end_key). Returns OK on success, and a non-OK status on error. + // Any entries outside this range will be completely deleted (including + // tombstones). + // The main difference between ClipColumnFamily(begin, end) and + // DeleteRange(begin, end) + // is that the former physically deletes all keys outside the range, but is + // more heavyweight than the latter. + // This feature is mainly used to ensure that there is no overlapping Key when + // calling CreateColumnFamilyWithImport() to import multiple CFs. + // Note that: concurrent updates cannot be performed during Clip. + virtual Status ClipColumnFamily(ColumnFamilyHandle* column_family, + const Slice& begin_key, + const Slice& end_key) = 0; + + // Verify the checksums of files in db. Currently the whole-file checksum of + // table files are checked. + virtual Status VerifyFileChecksums(const ReadOptions& /*read_options*/) { + return Status::NotSupported("File verification not supported"); + } + + // Verify the block checksums of files in db. The block checksums of table + // files are checked. + virtual Status VerifyChecksum(const ReadOptions& read_options) = 0; + + virtual Status VerifyChecksum() { return VerifyChecksum(ReadOptions()); } + + + // Returns the unique ID which is read from IDENTITY file during the opening + // of database by setting in the identity variable + // Returns Status::OK if identity could be set properly + virtual Status GetDbIdentity(std::string& identity) const = 0; + + // Return a unique identifier for each DB object that is opened + // This DB session ID should be unique among all open DB instances on all + // hosts, and should be unique among re-openings of the same or other DBs. + // (Two open DBs have the same identity from other function GetDbIdentity when + // one is physically copied from the other.) + virtual Status GetDbSessionId(std::string& session_id) const = 0; + + // Returns default column family handle + virtual ColumnFamilyHandle* DefaultColumnFamily() const = 0; + + + virtual Status GetPropertiesOfAllTables(ColumnFamilyHandle* column_family, + TablePropertiesCollection* props) = 0; + virtual Status GetPropertiesOfAllTables(TablePropertiesCollection* props) { + return GetPropertiesOfAllTables(DefaultColumnFamily(), props); + } + virtual Status GetPropertiesOfTablesInRange( + ColumnFamilyHandle* column_family, const Range* range, std::size_t n, + TablePropertiesCollection* props) = 0; + + virtual Status SuggestCompactRange(ColumnFamilyHandle* /*column_family*/, + const Slice* /*begin*/, + const Slice* /*end*/) { + return Status::NotSupported("SuggestCompactRange() is not implemented."); + } + + virtual Status PromoteL0(ColumnFamilyHandle* /*column_family*/, + int /*target_level*/) { + return Status::NotSupported("PromoteL0() is not implemented."); + } + + // Trace DB operations. Use EndTrace() to stop tracing. + virtual Status StartTrace(const TraceOptions& /*options*/, + std::unique_ptr&& /*trace_writer*/) { + return Status::NotSupported("StartTrace() is not implemented."); + } + + virtual Status EndTrace() { + return Status::NotSupported("EndTrace() is not implemented."); + } + + // IO Tracing operations. Use EndIOTrace() to stop tracing. + virtual Status StartIOTrace(const TraceOptions& /*options*/, + std::unique_ptr&& /*trace_writer*/) { + return Status::NotSupported("StartIOTrace() is not implemented."); + } + + virtual Status EndIOTrace() { + return Status::NotSupported("EndIOTrace() is not implemented."); + } + + // Trace block cache accesses. Use EndBlockCacheTrace() to stop tracing. + virtual Status StartBlockCacheTrace( + const TraceOptions& /*trace_options*/, + std::unique_ptr&& /*trace_writer*/) { + return Status::NotSupported("StartBlockCacheTrace() is not implemented."); + } + + virtual Status StartBlockCacheTrace( + const BlockCacheTraceOptions& /*options*/, + std::unique_ptr&& /*trace_writer*/) { + return Status::NotSupported("StartBlockCacheTrace() is not implemented."); + } + + virtual Status EndBlockCacheTrace() { + return Status::NotSupported("EndBlockCacheTrace() is not implemented."); + } + + // Create a default trace replayer. + virtual Status NewDefaultReplayer( + const std::vector& /*handles*/, + std::unique_ptr&& /*reader*/, + std::unique_ptr* /*replayer*/) { + return Status::NotSupported("NewDefaultReplayer() is not implemented."); + } + + + // Needed for StackableDB + virtual DB* GetRootDB() { return this; } + + // Given a window [start_time, end_time), setup a StatsHistoryIterator + // to access stats history. Note the start_time and end_time are epoch + // time measured in seconds, and end_time is an exclusive bound. + virtual Status GetStatsHistory( + uint64_t /*start_time*/, uint64_t /*end_time*/, + std::unique_ptr* /*stats_iterator*/) { + return Status::NotSupported("GetStatsHistory() is not implemented."); + } + + // Make the secondary instance catch up with the primary by tailing and + // replaying the MANIFEST and WAL of the primary. + // Column families created by the primary after the secondary instance starts + // will be ignored unless the secondary instance closes and restarts with the + // newly created column families. + // Column families that exist before secondary instance starts and dropped by + // the primary afterwards will be marked as dropped. However, as long as the + // secondary instance does not delete the corresponding column family + // handles, the data of the column family is still accessible to the + // secondary. + virtual Status TryCatchUpWithPrimary() { + return Status::NotSupported("Supported only by secondary instance"); + } +}; + +struct WriteStallStatsMapKeys { + static const std::string& TotalStops(); + static const std::string& TotalDelays(); + + static const std::string& CFL0FileCountLimitDelaysWithOngoingCompaction(); + static const std::string& CFL0FileCountLimitStopsWithOngoingCompaction(); + + // REQUIRES: + // `cause` isn't any of these: `WriteStallCause::kNone`, + // `WriteStallCause::kCFScopeWriteStallCauseEnumMax`, + // `WriteStallCause::kDBScopeWriteStallCauseEnumMax` + // + // REQUIRES: + // `condition` isn't any of these: `WriteStallCondition::kNormal` + static std::string CauseConditionCount(WriteStallCause cause, + WriteStallCondition condition); +}; + +// Overloaded operators for enum class SizeApproximationFlags. +inline DB::SizeApproximationFlags operator&(DB::SizeApproximationFlags lhs, + DB::SizeApproximationFlags rhs) { + return static_cast(static_cast(lhs) & + static_cast(rhs)); +} +inline DB::SizeApproximationFlags operator|(DB::SizeApproximationFlags lhs, + DB::SizeApproximationFlags rhs) { + return static_cast(static_cast(lhs) | + static_cast(rhs)); +} + +inline Status DB::GetApproximateSizes(ColumnFamilyHandle* column_family, + const Range* ranges, int n, + uint64_t* sizes, + SizeApproximationFlags include_flags) { + SizeApproximationOptions options; + options.include_memtables = + ((include_flags & SizeApproximationFlags::INCLUDE_MEMTABLES) != + SizeApproximationFlags::NONE); + options.include_files = + ((include_flags & SizeApproximationFlags::INCLUDE_FILES) != + SizeApproximationFlags::NONE); + return GetApproximateSizes(options, column_family, ranges, n, sizes); +} + +// Destroy the contents of the specified database. +// Be very careful using this method. +Status DestroyDB(const std::string& name, const Options& options, + const std::vector& column_families = + std::vector()); + +// If a DB cannot be opened, you may attempt to call this method to +// resurrect as much of the contents of the database as possible. +// Some data may be lost, so be careful when calling this function +// on a database that contains important information. +// +// With this API, we will warn and skip data associated with column families not +// specified in column_families. +// +// @param column_families Descriptors for known column families +Status RepairDB(const std::string& dbname, const DBOptions& db_options, + const std::vector& column_families); + +// @param unknown_cf_opts Options for column families encountered during the +// repair that were not specified in column_families. +Status RepairDB(const std::string& dbname, const DBOptions& db_options, + const std::vector& column_families, + const ColumnFamilyOptions& unknown_cf_opts); + +// @param options These options will be used for the database and for ALL column +// families encountered during the repair +Status RepairDB(const std::string& dbname, const Options& options); + + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/db_bench_tool.h b/librocksdb-sys/rocksdb/include/rocksdb/db_bench_tool.h new file mode 100644 index 0000000..17f4e6b --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/db_bench_tool.h @@ -0,0 +1,11 @@ +// Copyright (c) 2013-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { +int db_bench_tool(int argc, char** argv); +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/db_dump_tool.h b/librocksdb-sys/rocksdb/include/rocksdb/db_dump_tool.h new file mode 100644 index 0000000..2c97bad --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/db_dump_tool.h @@ -0,0 +1,43 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "rocksdb/db.h" + +namespace ROCKSDB_NAMESPACE { + +struct DumpOptions { + // Database that will be dumped + std::string db_path; + // File location that will contain dump output + std::string dump_location; + // Don't include db information header in the dump + bool anonymous = false; +}; + +class DbDumpTool { + public: + bool Run(const DumpOptions& dump_options, + ROCKSDB_NAMESPACE::Options options = ROCKSDB_NAMESPACE::Options()); +}; + +struct UndumpOptions { + // Database that we will load the dumped file into + std::string db_path; + // File location of the dumped file that will be loaded + std::string dump_location; + // Compact the db after loading the dumped file + bool compact_db = false; +}; + +class DbUndumpTool { + public: + bool Run(const UndumpOptions& undump_options, + ROCKSDB_NAMESPACE::Options options = ROCKSDB_NAMESPACE::Options()); +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/db_stress_tool.h b/librocksdb-sys/rocksdb/include/rocksdb/db_stress_tool.h new file mode 100644 index 0000000..7d3d42c --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/db_stress_tool.h @@ -0,0 +1,11 @@ +// Copyright (c) 2013-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { +int db_stress_tool(int argc, char** argv); +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/env.h b/librocksdb-sys/rocksdb/include/rocksdb/env.h new file mode 100644 index 0000000..ea99a7a --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/env.h @@ -0,0 +1,1905 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// An Env is an interface used by the rocksdb implementation to access +// operating system functionality like the filesystem etc. Callers +// may wish to provide a custom Env object when opening a database to +// get fine gain control; e.g., to rate limit file system operations. +// +// All Env implementations are safe for concurrent access from +// multiple threads without any external synchronization. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +#include "rocksdb/customizable.h" +#include "rocksdb/functor_wrapper.h" +#include "rocksdb/port_defs.h" +#include "rocksdb/status.h" +#include "rocksdb/thread_status.h" + +#ifdef _WIN32 +// Windows API macro interference +#undef DeleteFile +#undef GetCurrentTime +#undef LoadLibrary +#endif + +#if defined(__GNUC__) || defined(__clang__) +#define ROCKSDB_PRINTF_FORMAT_ATTR(format_param, dots_param) \ + __attribute__((__format__(__printf__, format_param, dots_param))) +#else +#define ROCKSDB_PRINTF_FORMAT_ATTR(format_param, dots_param) +#endif + +namespace ROCKSDB_NAMESPACE { + +class DynamicLibrary; +class FileLock; +class Logger; +class RandomAccessFile; +class SequentialFile; +class Slice; +struct DataVerificationInfo; +class WritableFile; +class RandomRWFile; +class MemoryMappedFileBuffer; +class Directory; +struct DBOptions; +struct ImmutableDBOptions; +struct MutableDBOptions; +class RateLimiter; +class ThreadStatusUpdater; +struct ThreadStatus; +class FileSystem; +class SystemClock; +struct ConfigOptions; + +const size_t kDefaultPageSize = 4 * 1024; + +// Options while opening a file to read/write +struct EnvOptions { + // Construct with default Options + EnvOptions(); + + // Construct from Options + explicit EnvOptions(const DBOptions& options); + + // If true, then use mmap to read data. + // Not recommended for 32-bit OS. + bool use_mmap_reads = false; + + // If true, then use mmap to write data + bool use_mmap_writes = true; + + // If true, then use O_DIRECT for reading data + bool use_direct_reads = false; + + // If true, then use O_DIRECT for writing data + bool use_direct_writes = false; + + // If false, fallocate() calls are bypassed + bool allow_fallocate = true; + + // If true, set the FD_CLOEXEC on open fd. + bool set_fd_cloexec = true; + + // Allows OS to incrementally sync files to disk while they are being + // written, in the background. Issue one request for every bytes_per_sync + // written. 0 turns it off. + // Default: 0 + uint64_t bytes_per_sync = 0; + + // When true, guarantees the file has at most `bytes_per_sync` bytes submitted + // for writeback at any given time. + // + // - If `sync_file_range` is supported it achieves this by waiting for any + // prior `sync_file_range`s to finish before proceeding. In this way, + // processing (compression, etc.) can proceed uninhibited in the gap + // between `sync_file_range`s, and we block only when I/O falls behind. + // - Otherwise the `WritableFile::Sync` method is used. Note this mechanism + // always blocks, thus preventing the interleaving of I/O and processing. + // + // Note: Enabling this option does not provide any additional persistence + // guarantees, as it may use `sync_file_range`, which does not write out + // metadata. + // + // Default: false + bool strict_bytes_per_sync = false; + + // If true, we will preallocate the file with FALLOC_FL_KEEP_SIZE flag, which + // means that file size won't change as part of preallocation. + // If false, preallocation will also change the file size. This option will + // improve the performance in workloads where you sync the data on every + // write. By default, we set it to true for MANIFEST writes and false for + // WAL writes + bool fallocate_with_keep_size = true; + + // See DBOptions doc + size_t compaction_readahead_size = 0; + + // See DBOptions doc + size_t random_access_max_buffer_size = 0; + + // See DBOptions doc + size_t writable_file_max_buffer_size = 1024 * 1024; + + // If not nullptr, write rate limiting is enabled for flush and compaction + RateLimiter* rate_limiter = nullptr; +}; + +// Exceptions MUST NOT propagate out of overridden functions into RocksDB, +// because RocksDB is not exception-safe. This could cause undefined behavior +// including data loss, unreported corruption, deadlocks, and more. +class Env : public Customizable { + public: + static const char* kDefaultName() { return "DefaultEnv"; } + struct FileAttributes { + // File name + std::string name; + + // Size of file in bytes + uint64_t size_bytes; + }; + + Env(); + // Construct an Env with a separate FileSystem and/or SystemClock + // implementation + explicit Env(const std::shared_ptr& fs); + Env(const std::shared_ptr& fs, + const std::shared_ptr& clock); + // No copying allowed + Env(const Env&) = delete; + void operator=(const Env&) = delete; + + ~Env() override; + + static const char* Type() { return "Environment"; } + + // Deprecated. Will be removed in a major release. Derived classes + // should implement this method. + const char* Name() const override { return ""; } + + // Loads the environment specified by the input value into the result + // @see Customizable for a more detailed description of the parameters and + // return codes + // + // @param config_options Controls how the environment is loaded. + // @param value the name and associated properties for the environment. + // @param result On success, the environment that was loaded. + // @param guard If specified and the loaded environment is not static, + // this value will contain the loaded environment (guard.get() == + // result). + // @return OK If the environment was successfully loaded (and optionally + // prepared) + // @return not-OK if the load failed. + static Status CreateFromString(const ConfigOptions& config_options, + const std::string& value, Env** result); + static Status CreateFromString(const ConfigOptions& config_options, + const std::string& value, Env** result, + std::shared_ptr* guard); + + // Loads the environment specified by the env and fs uri. + // If both are specified, an error is returned. + // Otherwise, the environment is created by loading (via CreateFromString) + // the appropriate env/fs from the corresponding values. + static Status CreateFromUri(const ConfigOptions& options, + const std::string& env_uri, + const std::string& fs_uri, Env** result, + std::shared_ptr* guard); + + // Return a default environment suitable for the current operating + // system. Sophisticated users may wish to provide their own Env + // implementation instead of relying on this default environment. + // + // The result of Default() belongs to rocksdb and must never be deleted. + static Env* Default(); + + // See FileSystem::RegisterDbPaths. + virtual Status RegisterDbPaths(const std::vector& /*paths*/) { + return Status::OK(); + } + // See FileSystem::UnregisterDbPaths. + virtual Status UnregisterDbPaths(const std::vector& /*paths*/) { + return Status::OK(); + } + + // Create a brand new sequentially-readable file with the specified name. + // On success, stores a pointer to the new file in *result and returns OK. + // On failure stores nullptr in *result and returns non-OK. If the file does + // not exist, returns a non-OK status. + // + // The returned file will only be accessed by one thread at a time. + virtual Status NewSequentialFile(const std::string& fname, + std::unique_ptr* result, + const EnvOptions& options) = 0; + + // Create a brand new random access read-only file with the + // specified name. On success, stores a pointer to the new file in + // *result and returns OK. On failure stores nullptr in *result and + // returns non-OK. If the file does not exist, returns a non-OK + // status. + // + // The returned file may be concurrently accessed by multiple threads. + virtual Status NewRandomAccessFile(const std::string& fname, + std::unique_ptr* result, + const EnvOptions& options) = 0; + // These values match Linux definition + // https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/include/uapi/linux/fcntl.h#n56 + enum WriteLifeTimeHint { + WLTH_NOT_SET = 0, // No hint information set + WLTH_NONE, // No hints about write life time + WLTH_SHORT, // Data written has a short life time + WLTH_MEDIUM, // Data written has a medium life time + WLTH_LONG, // Data written has a long life time + WLTH_EXTREME, // Data written has an extremely long life time + }; + + // Create an object that writes to a new file with the specified + // name. Deletes any existing file with the same name and creates a + // new file. On success, stores a pointer to the new file in + // *result and returns OK. On failure stores nullptr in *result and + // returns non-OK. + // + // The returned file will only be accessed by one thread at a time. + virtual Status NewWritableFile(const std::string& fname, + std::unique_ptr* result, + const EnvOptions& options) = 0; + + // Create an object that writes to a file with the specified name. + // `WritableFile::Append()`s will append after any existing content. If the + // file does not already exist, creates it. + // + // On success, stores a pointer to the file in *result and returns OK. On + // failure stores nullptr in *result and returns non-OK. + // + // The returned file will only be accessed by one thread at a time. + virtual Status ReopenWritableFile(const std::string& /*fname*/, + std::unique_ptr* /*result*/, + const EnvOptions& /*options*/) { + return Status::NotSupported("Env::ReopenWritableFile() not supported."); + } + + // Reuse an existing file by renaming it and opening it as writable. + virtual Status ReuseWritableFile(const std::string& fname, + const std::string& old_fname, + std::unique_ptr* result, + const EnvOptions& options); + + // Open `fname` for random read and write, if file doesn't exist the file + // will be created. On success, stores a pointer to the new file in + // *result and returns OK. On failure returns non-OK. + // + // The returned file will only be accessed by one thread at a time. + virtual Status NewRandomRWFile(const std::string& /*fname*/, + std::unique_ptr* /*result*/, + const EnvOptions& /*options*/) { + return Status::NotSupported("RandomRWFile is not implemented in this Env"); + } + + // Opens `fname` as a memory-mapped file for read and write (in-place updates + // only, i.e., no appends). On success, stores a raw buffer covering the whole + // file in `*result`. The file must exist prior to this call. + virtual Status NewMemoryMappedFileBuffer( + const std::string& /*fname*/, + std::unique_ptr* /*result*/) { + return Status::NotSupported( + "MemoryMappedFileBuffer is not implemented in this Env"); + } + + // Create an object that represents a directory. Will fail if directory + // doesn't exist. If the directory exists, it will open the directory + // and create a new Directory object. + // + // On success, stores a pointer to the new Directory in + // *result and returns OK. On failure stores nullptr in *result and + // returns non-OK. + virtual Status NewDirectory(const std::string& name, + std::unique_ptr* result) = 0; + + // Returns OK if the named file exists. + // NotFound if the named file does not exist, + // the calling process does not have permission to determine + // whether this file exists, or if the path is invalid. + // IOError if an IO Error was encountered + virtual Status FileExists(const std::string& fname) = 0; + + // Store in *result the names of the children of the specified directory. + // The names are relative to "dir", and shall never include the + // names `.` or `..`. + // Original contents of *results are dropped. + // Returns OK if "dir" exists and "*result" contains its children. + // NotFound if "dir" does not exist, the calling process does not have + // permission to access "dir", or if "dir" is invalid. + // IOError if an IO Error was encountered + virtual Status GetChildren(const std::string& dir, + std::vector* result) = 0; + + // Store in *result the attributes of the children of the specified directory. + // In case the implementation lists the directory prior to iterating the files + // and files are concurrently deleted, the deleted files will be omitted from + // result. + // The name attributes are relative to "dir", and shall never include the + // names `.` or `..`. + // Original contents of *results are dropped. + // Returns OK if "dir" exists and "*result" contains its children. + // NotFound if "dir" does not exist, the calling process does not have + // permission to access "dir", or if "dir" is invalid. + // IOError if an IO Error was encountered + virtual Status GetChildrenFileAttributes(const std::string& dir, + std::vector* result); + + // Delete the named file. + virtual Status DeleteFile(const std::string& fname) = 0; + + // Truncate the named file to the specified size. + virtual Status Truncate(const std::string& /*fname*/, size_t /*size*/) { + return Status::NotSupported("Truncate is not supported for this Env"); + } + + // Create the specified directory. Returns error if directory exists. + virtual Status CreateDir(const std::string& dirname) = 0; + + // Creates directory if missing. Return Ok if it exists, or successful in + // Creating. + virtual Status CreateDirIfMissing(const std::string& dirname) = 0; + + // Delete the specified directory. + // Many implementations of this function will only delete a directory if it is + // empty. + virtual Status DeleteDir(const std::string& dirname) = 0; + + // Store the size of fname in *file_size. + virtual Status GetFileSize(const std::string& fname, uint64_t* file_size) = 0; + + // Store the last modification time of fname in *file_mtime. + virtual Status GetFileModificationTime(const std::string& fname, + uint64_t* file_mtime) = 0; + // Rename file src to target. + virtual Status RenameFile(const std::string& src, + const std::string& target) = 0; + + // Hard Link file src to target. + virtual Status LinkFile(const std::string& /*src*/, + const std::string& /*target*/) { + return Status::NotSupported("LinkFile is not supported for this Env"); + } + + virtual Status NumFileLinks(const std::string& /*fname*/, + uint64_t* /*count*/) { + return Status::NotSupported( + "Getting number of file links is not supported for this Env"); + } + + virtual Status AreFilesSame(const std::string& /*first*/, + const std::string& /*second*/, bool* /*res*/) { + return Status::NotSupported("AreFilesSame is not supported for this Env"); + } + + // Lock the specified file. Used to prevent concurrent access to + // the same db by multiple processes. On failure, stores nullptr in + // *lock and returns non-OK. + // + // On success, stores a pointer to the object that represents the + // acquired lock in *lock and returns OK. The caller should call + // UnlockFile(*lock) to release the lock. If the process exits, + // the lock will be automatically released. + // + // If somebody else already holds the lock, finishes immediately + // with a failure. I.e., this call does not wait for existing locks + // to go away. + // + // May create the named file if it does not already exist. + virtual Status LockFile(const std::string& fname, FileLock** lock) = 0; + + // Release the lock acquired by a previous successful call to LockFile. + // REQUIRES: lock was returned by a successful LockFile() call + // REQUIRES: lock has not already been unlocked. + virtual Status UnlockFile(FileLock* lock) = 0; + + // Opens `lib_name` as a dynamic library. + // If the 'search_path' is specified, breaks the path into its components + // based on the appropriate platform separator (";" or ";") and looks for the + // library in those directories. If 'search path is not specified, uses the + // default library path search mechanism (such as LD_LIBRARY_PATH). On + // success, stores a dynamic library in `*result`. + virtual Status LoadLibrary(const std::string& /*lib_name*/, + const std::string& /*search_path */, + std::shared_ptr* /*result*/) { + return Status::NotSupported("LoadLibrary is not implemented in this Env"); + } + + // Priority for scheduling job in thread pool + enum Priority { BOTTOM, LOW, HIGH, USER, TOTAL }; + + static std::string PriorityToString(Priority priority); + + // Priority for requesting bytes in rate limiter scheduler + enum IOPriority { + IO_LOW = 0, + IO_MID = 1, + IO_HIGH = 2, + IO_USER = 3, + IO_TOTAL = 4 + }; + + // EXPERIMENTAL + enum class IOActivity : uint8_t { + kFlush = 0, + kCompaction = 1, + kDBOpen = 2, + kUnknown, // Keep last for easy array of non-unknowns + }; + + // Arrange to run "(*function)(arg)" once in a background thread, in + // the thread pool specified by pri. By default, jobs go to the 'LOW' + // priority thread pool. + + // "function" may run in an unspecified thread. Multiple functions + // added to the same Env may run concurrently in different threads. + // I.e., the caller may not assume that background work items are + // serialized. + // When the UnSchedule function is called, the unschedFunction + // registered at the time of Schedule is invoked with arg as a parameter. + virtual void Schedule(void (*function)(void* arg), void* arg, + Priority pri = LOW, void* tag = nullptr, + void (*unschedFunction)(void* arg) = nullptr) = 0; + + // Arrange to remove jobs for given arg from the queue_ if they are not + // already scheduled. Caller is expected to have exclusive lock on arg. + virtual int UnSchedule(void* /*arg*/, Priority /*pri*/) { return 0; } + + // Start a new thread, invoking "function(arg)" within the new thread. + // When "function(arg)" returns, the thread will be destroyed. + virtual void StartThread(void (*function)(void* arg), void* arg) = 0; + + // Start a new thread, invoking "function(args...)" within the new thread. + // When "function(args...)" returns, the thread will be destroyed. + template + void StartThreadTyped(FunctionT function, Args&&... args) { + using FWType = FunctorWrapper; + StartThread( + [](void* arg) { + auto* functor = static_cast(arg); + functor->invoke(); + delete functor; + }, + new FWType(std::function(function), + std::forward(args)...)); + } + + // Wait for all threads started by StartThread to terminate. + virtual void WaitForJoin() {} + + // Reserve available background threads in the specified thread pool. + virtual int ReserveThreads(int /*threads_to_be_reserved*/, Priority /*pri*/) { + return 0; + } + + // Release a specific number of reserved threads from the specified thread + // pool + virtual int ReleaseThreads(int /*threads_to_be_released*/, Priority /*pri*/) { + return 0; + } + + // Get thread pool queue length for specific thread pool. + virtual unsigned int GetThreadPoolQueueLen(Priority /*pri*/ = LOW) const { + return 0; + } + + // *path is set to a temporary directory that can be used for testing. It may + // or many not have just been created. The directory may or may not differ + // between runs of the same process, but subsequent calls will return the + // same directory. + virtual Status GetTestDirectory(std::string* path) = 0; + + // Create and returns a default logger (an instance of EnvLogger) for storing + // informational messages. Derived classes can override to provide custom + // logger. + virtual Status NewLogger(const std::string& fname, + std::shared_ptr* result); + + // Returns the number of micro-seconds since some fixed point in time. + // It is often used as system time such as in GenericRateLimiter + // and other places so a port needs to return system time in order to work. + virtual uint64_t NowMicros() = 0; + + // Returns the number of nano-seconds since some fixed point in time. Only + // useful for computing deltas of time in one run. + // Default implementation simply relies on NowMicros. + // In platform-specific implementations, NowNanos() should return time points + // that are MONOTONIC. + virtual uint64_t NowNanos() { return NowMicros() * 1000; } + + // 0 indicates not supported. + virtual uint64_t NowCPUNanos() { return 0; } + + // Sleep/delay the thread for the prescribed number of micro-seconds. + virtual void SleepForMicroseconds(int micros) = 0; + + // Get the current host name as a null terminated string iff the string + // length is < len. The hostname should otherwise be truncated to len. + virtual Status GetHostName(char* name, uint64_t len) = 0; + + // Get the current hostname from the given env as a std::string in result. + // The result may be truncated if the hostname is too + // long + virtual Status GetHostNameString(std::string* result); + + // Get the number of seconds since the Epoch, 1970-01-01 00:00:00 (UTC). + // Only overwrites *unix_time on success. + virtual Status GetCurrentTime(int64_t* unix_time) = 0; + + // Get full directory name for this db. + virtual Status GetAbsolutePath(const std::string& db_path, + std::string* output_path) = 0; + + // The number of background worker threads of a specific thread pool + // for this environment. 'LOW' is the default pool. + // default number: 1 + virtual void SetBackgroundThreads(int number, Priority pri = LOW) = 0; + virtual int GetBackgroundThreads(Priority pri = LOW) = 0; + + virtual Status SetAllowNonOwnerAccess(bool /*allow_non_owner_access*/) { + return Status::NotSupported("Env::SetAllowNonOwnerAccess() not supported."); + } + + // Enlarge number of background worker threads of a specific thread pool + // for this environment if it is smaller than specified. 'LOW' is the default + // pool. + virtual void IncBackgroundThreadsIfNeeded(int number, Priority pri) = 0; + + // Lower IO priority for threads from the specified pool. + virtual void LowerThreadPoolIOPriority(Priority /*pool*/ = LOW) {} + + // Lower CPU priority for threads from the specified pool. + virtual Status LowerThreadPoolCPUPriority(Priority /*pool*/, + CpuPriority /*pri*/) { + return Status::NotSupported( + "Env::LowerThreadPoolCPUPriority(Priority, CpuPriority) not supported"); + } + + // Lower CPU priority for threads from the specified pool. + virtual void LowerThreadPoolCPUPriority(Priority /*pool*/ = LOW) {} + + // Converts seconds-since-Jan-01-1970 to a printable string + virtual std::string TimeToString(uint64_t time) = 0; + + // Generates a human-readable unique ID that can be used to identify a DB. + // In built-in implementations, this is an RFC-4122 UUID string, but might + // not be in all implementations. Overriding is not recommended. + // NOTE: this has not be validated for use in cryptography + virtual std::string GenerateUniqueId(); + + // OptimizeForLogWrite will create a new EnvOptions object that is a copy of + // the EnvOptions in the parameters, but is optimized for reading log files. + virtual EnvOptions OptimizeForLogRead(const EnvOptions& env_options) const; + + // OptimizeForManifestRead will create a new EnvOptions object that is a copy + // of the EnvOptions in the parameters, but is optimized for reading manifest + // files. + virtual EnvOptions OptimizeForManifestRead( + const EnvOptions& env_options) const; + + // OptimizeForLogWrite will create a new EnvOptions object that is a copy of + // the EnvOptions in the parameters, but is optimized for writing log files. + // Default implementation returns the copy of the same object. + virtual EnvOptions OptimizeForLogWrite(const EnvOptions& env_options, + const DBOptions& db_options) const; + // OptimizeForManifestWrite will create a new EnvOptions object that is a copy + // of the EnvOptions in the parameters, but is optimized for writing manifest + // files. Default implementation returns the copy of the same object. + virtual EnvOptions OptimizeForManifestWrite( + const EnvOptions& env_options) const; + + // OptimizeForCompactionTableWrite will create a new EnvOptions object that is + // a copy of the EnvOptions in the parameters, but is optimized for writing + // table files. + virtual EnvOptions OptimizeForCompactionTableWrite( + const EnvOptions& env_options, + const ImmutableDBOptions& immutable_ops) const; + + // OptimizeForCompactionTableWrite will create a new EnvOptions object that + // is a copy of the EnvOptions in the parameters, but is optimized for reading + // table files. + virtual EnvOptions OptimizeForCompactionTableRead( + const EnvOptions& env_options, + const ImmutableDBOptions& db_options) const; + + // OptimizeForBlobFileRead will create a new EnvOptions object that + // is a copy of the EnvOptions in the parameters, but is optimized for reading + // blob files. + virtual EnvOptions OptimizeForBlobFileRead( + const EnvOptions& env_options, + const ImmutableDBOptions& db_options) const; + + // Returns the status of all threads that belong to the current Env. + virtual Status GetThreadList(std::vector* /*thread_list*/) { + return Status::NotSupported("Env::GetThreadList() not supported."); + } + + // Returns the pointer to ThreadStatusUpdater. This function will be + // used in RocksDB internally to update thread status and supports + // GetThreadList(). + virtual ThreadStatusUpdater* GetThreadStatusUpdater() const { + return thread_status_updater_; + } + + // Returns the ID of the current thread. + virtual uint64_t GetThreadID() const; + +// This seems to clash with a macro on Windows, so #undef it here +#undef GetFreeSpace + + // Get the amount of free disk space + virtual Status GetFreeSpace(const std::string& /*path*/, + uint64_t* /*diskfree*/) { + return Status::NotSupported("Env::GetFreeSpace() not supported."); + } + + // Check whether the specified path is a directory + virtual Status IsDirectory(const std::string& /*path*/, bool* /*is_dir*/) { + return Status::NotSupported("Env::IsDirectory() not supported."); + } + + virtual void SanitizeEnvOptions(EnvOptions* /*env_opts*/) const {} + + // Get the FileSystem implementation this Env was constructed with. It + // could be a fully implemented one, or a wrapper class around the Env + const std::shared_ptr& GetFileSystem() const; + + // Get the SystemClock implementation this Env was constructed with. It + // could be a fully implemented one, or a wrapper class around the Env + const std::shared_ptr& GetSystemClock() const; + + // If you're adding methods here, remember to add them to EnvWrapper too. + + protected: + // The pointer to an internal structure that will update the + // status of each thread. + ThreadStatusUpdater* thread_status_updater_; + + // Pointer to the underlying FileSystem implementation + std::shared_ptr file_system_; + + // Pointer to the underlying SystemClock implementation + std::shared_ptr system_clock_; + + private: + static const size_t kMaxHostNameLen = 256; +}; + +// The factory function to construct a ThreadStatusUpdater. Any Env +// that supports GetThreadList() feature should call this function in its +// constructor to initialize thread_status_updater_. +ThreadStatusUpdater* CreateThreadStatusUpdater(); + +// A file abstraction for reading sequentially through a file +class SequentialFile { + public: + SequentialFile() {} + virtual ~SequentialFile(); + + // Read up to "n" bytes from the file. "scratch[0..n-1]" may be + // written by this routine. Sets "*result" to the data that was + // read (including if fewer than "n" bytes were successfully read). + // May set "*result" to point at data in "scratch[0..n-1]", so + // "scratch[0..n-1]" must be live when "*result" is used. + // If an error was encountered, returns a non-OK status. + // + // After call, result->size() < n only if end of file has been + // reached (or non-OK status). Read might fail if called again after + // first result->size() < n. + // + // REQUIRES: External synchronization + virtual Status Read(size_t n, Slice* result, char* scratch) = 0; + + // Skip "n" bytes from the file. This is guaranteed to be no + // slower that reading the same data, but may be faster. + // + // If end of file is reached, skipping will stop at the end of the + // file, and Skip will return OK. + // + // REQUIRES: External synchronization + virtual Status Skip(uint64_t n) = 0; + + // Indicates the upper layers if the current SequentialFile implementation + // uses direct IO. + virtual bool use_direct_io() const { return false; } + + // Use the returned alignment value to allocate + // aligned buffer for Direct I/O + virtual size_t GetRequiredBufferAlignment() const { return kDefaultPageSize; } + + // Remove any kind of caching of data from the offset to offset+length + // of this file. If the length is 0, then it refers to the end of file. + // If the system is not caching the file contents, then this is a noop. + virtual Status InvalidateCache(size_t /*offset*/, size_t /*length*/) { + return Status::NotSupported( + "SequentialFile::InvalidateCache not supported."); + } + + // Positioned Read for direct I/O + // If Direct I/O enabled, offset, n, and scratch should be properly aligned + virtual Status PositionedRead(uint64_t /*offset*/, size_t /*n*/, + Slice* /*result*/, char* /*scratch*/) { + return Status::NotSupported( + "SequentialFile::PositionedRead() not supported."); + } + + // If you're adding methods here, remember to add them to + // SequentialFileWrapper too. +}; + +// A read IO request structure for use in MultiRead +struct ReadRequest { + // File offset in bytes + uint64_t offset; + + // Length to read in bytes. `result` only returns fewer bytes if end of file + // is hit (or `status` is not OK). + size_t len; + + // A buffer that MultiRead() can optionally place data in. It can + // ignore this and allocate its own buffer + char* scratch; + + // Output parameter set by MultiRead() to point to the data buffer, and + // the number of valid bytes + Slice result; + + // Status of read + Status status; +}; + +// A file abstraction for randomly reading the contents of a file. +class RandomAccessFile { + public: + RandomAccessFile() {} + virtual ~RandomAccessFile(); + + // Read up to "n" bytes from the file starting at "offset". + // "scratch[0..n-1]" may be written by this routine. Sets "*result" + // to the data that was read (including if fewer than "n" bytes were + // successfully read). May set "*result" to point at data in + // "scratch[0..n-1]", so "scratch[0..n-1]" must be live when + // "*result" is used. If an error was encountered, returns a non-OK + // status. + // + // After call, result->size() < n only if end of file has been + // reached (or non-OK status). Read might fail if called again after + // first result->size() < n. + // + // Safe for concurrent use by multiple threads. + // If Direct I/O enabled, offset, n, and scratch should be aligned properly. + virtual Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const = 0; + + // Readahead the file starting from offset by n bytes for caching. + virtual Status Prefetch(uint64_t /*offset*/, size_t /*n*/) { + return Status::OK(); + } + + // Read a bunch of blocks as described by reqs. The blocks can + // optionally be read in parallel. This is a synchronous call, i.e it + // should return after all reads have completed. The reads will be + // non-overlapping. If the function return Status is not ok, status of + // individual requests will be ignored and return status will be assumed + // for all read requests. The function return status is only meant for + // any errors that occur before even processing specific read requests + virtual Status MultiRead(ReadRequest* reqs, size_t num_reqs) { + assert(reqs != nullptr); + for (size_t i = 0; i < num_reqs; ++i) { + ReadRequest& req = reqs[i]; + req.status = Read(req.offset, req.len, &req.result, req.scratch); + } + return Status::OK(); + } + + // Tries to get an unique ID for this file that will be the same each time + // the file is opened (and will stay the same while the file is open). + // Furthermore, it tries to make this ID at most "max_size" bytes. If such an + // ID can be created this function returns the length of the ID and places it + // in "id"; otherwise, this function returns 0, in which case "id" + // may not have been modified. + // + // This function guarantees, for IDs from a given environment, two unique ids + // cannot be made equal to each other by adding arbitrary bytes to one of + // them. That is, no unique ID is the prefix of another. + // + // This function guarantees that the returned ID will not be interpretable as + // a single varint. + // + // Note: these IDs are only valid for the duration of the process. + virtual size_t GetUniqueId(char* /*id*/, size_t /*max_size*/) const { + return 0; // Default implementation to prevent issues with backwards + // compatibility. + } + + enum AccessPattern { NORMAL, RANDOM, SEQUENTIAL, WILLNEED, DONTNEED }; + + virtual void Hint(AccessPattern /*pattern*/) {} + + // Indicates the upper layers if the current RandomAccessFile implementation + // uses direct IO. + virtual bool use_direct_io() const { return false; } + + // Use the returned alignment value to allocate + // aligned buffer for Direct I/O + virtual size_t GetRequiredBufferAlignment() const { return kDefaultPageSize; } + + // Remove any kind of caching of data from the offset to offset+length + // of this file. If the length is 0, then it refers to the end of file. + // If the system is not caching the file contents, then this is a noop. + virtual Status InvalidateCache(size_t /*offset*/, size_t /*length*/) { + return Status::NotSupported( + "RandomAccessFile::InvalidateCache not supported."); + } + + // If you're adding methods here, remember to add them to + // RandomAccessFileWrapper too. +}; + +// A file abstraction for sequential writing. The implementation +// must provide buffering since callers may append small fragments +// at a time to the file. +class WritableFile { + public: + WritableFile() + : last_preallocated_block_(0), + preallocation_block_size_(0), + io_priority_(Env::IO_TOTAL), + write_hint_(Env::WLTH_NOT_SET), + strict_bytes_per_sync_(false) {} + + explicit WritableFile(const EnvOptions& options) + : last_preallocated_block_(0), + preallocation_block_size_(0), + io_priority_(Env::IO_TOTAL), + write_hint_(Env::WLTH_NOT_SET), + strict_bytes_per_sync_(options.strict_bytes_per_sync) {} + // No copying allowed + WritableFile(const WritableFile&) = delete; + void operator=(const WritableFile&) = delete; + + // For cases when Close() hasn't been called, many derived classes of + // WritableFile will need to call Close() non-virtually in their destructor, + // and ignore the result, to ensure resources are released. + virtual ~WritableFile(); + + // Append data to the end of the file + // Note: A WritableFile object must support either Append or + // PositionedAppend, so the users cannot mix the two. + virtual Status Append(const Slice& data) = 0; + + // Append data with verification information. + // Note that this API change is experimental and it might be changed in + // the future. Currently, RocksDB only generates crc32c based checksum for + // the file writes when the checksum handoff option is set. + // Expected behavior: if currently ChecksumType::kCRC32C is not supported by + // WritableFile, the information in DataVerificationInfo can be ignored + // (i.e. does not perform checksum verification). + virtual Status Append(const Slice& data, + const DataVerificationInfo& /* verification_info */) { + return Append(data); + } + + // PositionedAppend data to the specified offset. The new EOF after append + // must be larger than the previous EOF. This is to be used when writes are + // not backed by OS buffers and hence has to always start from the start of + // the sector. The implementation thus needs to also rewrite the last + // partial sector. + // Note: PositionAppend does not guarantee moving the file offset after the + // write. A WritableFile object must support either Append or + // PositionedAppend, so the users cannot mix the two. + // + // PositionedAppend() can only happen on the page/sector boundaries. For that + // reason, if the last write was an incomplete sector we still need to rewind + // back to the nearest sector/page and rewrite the portion of it with whatever + // we need to add. We need to keep where we stop writing. + // + // PositionedAppend() can only write whole sectors. For that reason we have to + // pad with zeros for the last write and trim the file when closing according + // to the position we keep in the previous step. + // + // PositionedAppend() requires aligned buffer to be passed in. The alignment + // required is queried via GetRequiredBufferAlignment() + virtual Status PositionedAppend(const Slice& /* data */, + uint64_t /* offset */) { + return Status::NotSupported( + "WritableFile::PositionedAppend() not supported."); + } + + // PositionedAppend data with verification information. + // Note that this API change is experimental and it might be changed in + // the future. Currently, RocksDB only generates crc32c based checksum for + // the file writes when the checksum handoff option is set. + // Expected behavior: if currently ChecksumType::kCRC32C is not supported by + // WritableFile, the information in DataVerificationInfo can be ignored + // (i.e. does not perform checksum verification). + virtual Status PositionedAppend( + const Slice& /* data */, uint64_t /* offset */, + const DataVerificationInfo& /* verification_info */) { + return Status::NotSupported("PositionedAppend"); + } + + // Truncate is necessary to trim the file to the correct size + // before closing. It is not always possible to keep track of the file + // size due to whole pages writes. The behavior is undefined if called + // with other writes to follow. + virtual Status Truncate(uint64_t /*size*/) { return Status::OK(); } + + // The caller should call Close() before destroying the WritableFile to + // surface any errors associated with finishing writes to the file. + // The file is considered closed regardless of return status. + // (However, implementations must also clean up properly in the destructor + // even if Close() is not called.) + virtual Status Close() = 0; + virtual Status Flush() = 0; + virtual Status Sync() = 0; // sync data + + /* + * Sync data and/or metadata as well. + * By default, sync only data. + * Override this method for environments where we need to sync + * metadata as well. + */ + virtual Status Fsync() { return Sync(); } + + // true if Sync() and Fsync() are safe to call concurrently with Append() + // and Flush(). + virtual bool IsSyncThreadSafe() const { return false; } + + // Indicates the upper layers if the current WritableFile implementation + // uses direct IO. + virtual bool use_direct_io() const { return false; } + + // Use the returned alignment value to allocate + // aligned buffer for Direct I/O + virtual size_t GetRequiredBufferAlignment() const { return kDefaultPageSize; } + + /* + * If rate limiting is enabled, change the file-granularity priority used in + * rate-limiting writes. + * + * In the presence of finer-granularity priority such as + * `WriteOptions::rate_limiter_priority`, this file-granularity priority may + * be overridden by a non-Env::IO_TOTAL finer-granularity priority and used as + * a fallback for Env::IO_TOTAL finer-granularity priority. + * + * If rate limiting is not enabled, this call has no effect. + */ + virtual void SetIOPriority(Env::IOPriority pri) { io_priority_ = pri; } + + virtual Env::IOPriority GetIOPriority() { return io_priority_; } + + virtual void SetWriteLifeTimeHint(Env::WriteLifeTimeHint hint) { + write_hint_ = hint; + } + + virtual Env::WriteLifeTimeHint GetWriteLifeTimeHint() { return write_hint_; } + /* + * Get the size of valid data in the file. + */ + virtual uint64_t GetFileSize() { return 0; } + + /* + * Get and set the default pre-allocation block size for writes to + * this file. If non-zero, then Allocate will be used to extend the + * underlying storage of a file (generally via fallocate) if the Env + * instance supports it. + */ + virtual void SetPreallocationBlockSize(size_t size) { + preallocation_block_size_ = size; + } + + virtual void GetPreallocationStatus(size_t* block_size, + size_t* last_allocated_block) { + *last_allocated_block = last_preallocated_block_; + *block_size = preallocation_block_size_; + } + + // For documentation, refer to RandomAccessFile::GetUniqueId() + virtual size_t GetUniqueId(char* /*id*/, size_t /*max_size*/) const { + return 0; // Default implementation to prevent issues with backwards + } + + // Remove any kind of caching of data from the offset to offset+length + // of this file. If the length is 0, then it refers to the end of file. + // If the system is not caching the file contents, then this is a noop. + // This call has no effect on dirty pages in the cache. + virtual Status InvalidateCache(size_t /*offset*/, size_t /*length*/) { + return Status::NotSupported("WritableFile::InvalidateCache not supported."); + } + + // Sync a file range with disk. + // offset is the starting byte of the file range to be synchronized. + // nbytes specifies the length of the range to be synchronized. + // This asks the OS to initiate flushing the cached data to disk, + // without waiting for completion. + // Default implementation does nothing. + virtual Status RangeSync(uint64_t /*offset*/, uint64_t /*nbytes*/) { + if (strict_bytes_per_sync_) { + return Sync(); + } + return Status::OK(); + } + + // PrepareWrite performs any necessary preparation for a write + // before the write actually occurs. This allows for pre-allocation + // of space on devices where it can result in less file + // fragmentation and/or less waste from over-zealous filesystem + // pre-allocation. + virtual void PrepareWrite(size_t offset, size_t len) { + if (preallocation_block_size_ == 0) { + return; + } + // If this write would cross one or more preallocation blocks, + // determine what the last preallocation block necessary to + // cover this write would be and Allocate to that point. + const auto block_size = preallocation_block_size_; + size_t new_last_preallocated_block = + (offset + len + block_size - 1) / block_size; + if (new_last_preallocated_block > last_preallocated_block_) { + size_t num_spanned_blocks = + new_last_preallocated_block - last_preallocated_block_; + // TODO: Don't ignore errors from allocate + Allocate(block_size * last_preallocated_block_, + block_size * num_spanned_blocks) + .PermitUncheckedError(); + last_preallocated_block_ = new_last_preallocated_block; + } + } + + // Pre-allocates space for a file. + virtual Status Allocate(uint64_t /*offset*/, uint64_t /*len*/) { + return Status::OK(); + } + + // If you're adding methods here, remember to add them to + // WritableFileWrapper too. + + protected: + size_t preallocation_block_size() { return preallocation_block_size_; } + + private: + size_t last_preallocated_block_; + size_t preallocation_block_size_; + + protected: + Env::IOPriority io_priority_; + Env::WriteLifeTimeHint write_hint_; + const bool strict_bytes_per_sync_; +}; + +// A file abstraction for random reading and writing. +class RandomRWFile { + public: + RandomRWFile() {} + // No copying allowed + RandomRWFile(const RandomRWFile&) = delete; + RandomRWFile& operator=(const RandomRWFile&) = delete; + + // For cases when Close() hasn't been called, many derived classes of + // RandomRWFile will need to call Close() non-virtually in their destructor, + // and ignore the result, to ensure resources are released. + virtual ~RandomRWFile() {} + + // Indicates if the class makes use of direct I/O + // If false you must pass aligned buffer to Write() + virtual bool use_direct_io() const { return false; } + + // Use the returned alignment value to allocate + // aligned buffer for Direct I/O + virtual size_t GetRequiredBufferAlignment() const { return kDefaultPageSize; } + + // Write bytes in `data` at offset `offset`, Returns Status::OK() on success. + // Pass aligned buffer when use_direct_io() returns true. + virtual Status Write(uint64_t offset, const Slice& data) = 0; + + // Read up to `n` bytes starting from offset `offset` and store them in + // result, provided `scratch` size should be at least `n`. + // + // After call, result->size() < n only if end of file has been + // reached (or non-OK status). Read might fail if called again after + // first result->size() < n. + // + // Returns Status::OK() on success. + virtual Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const = 0; + + virtual Status Flush() = 0; + + virtual Status Sync() = 0; + + virtual Status Fsync() { return Sync(); } + + // The caller should call Close() before destroying the RandomRWFile to + // surface any errors associated with finishing writes to the file. + // The file is considered closed regardless of return status. + // (However, implementations must also clean up properly in the destructor + // even if Close() is not called.) + virtual Status Close() = 0; + + // If you're adding methods here, remember to add them to + // RandomRWFileWrapper too. +}; + +// MemoryMappedFileBuffer object represents a memory-mapped file's raw buffer. +// Subclasses should release the mapping upon destruction. +class MemoryMappedFileBuffer { + public: + MemoryMappedFileBuffer(void* _base, size_t _length) + : base_(_base), length_(_length) {} + + virtual ~MemoryMappedFileBuffer() = 0; + + // We do not want to unmap this twice. We can make this class + // movable if desired, however, since + MemoryMappedFileBuffer(const MemoryMappedFileBuffer&) = delete; + MemoryMappedFileBuffer& operator=(const MemoryMappedFileBuffer&) = delete; + + void* GetBase() const { return base_; } + size_t GetLen() const { return length_; } + + protected: + void* base_; + const size_t length_; +}; + +// Directory object represents collection of files and implements +// filesystem operations that can be executed on directories. +class Directory { + public: + // Many derived classes of Directory will need to call Close() in their + // destructor, when not called already, to ensure resources are released. + virtual ~Directory() {} + // Fsync directory. Can be called concurrently from multiple threads. + virtual Status Fsync() = 0; + // Calling Close() before destroying a Directory is recommended to surface + // any errors associated with finishing writes (in case of future features). + // The directory is considered closed regardless of return status. + virtual Status Close() { return Status::NotSupported("Close"); } + + virtual size_t GetUniqueId(char* /*id*/, size_t /*max_size*/) const { + return 0; + } + + // If you're adding methods here, remember to add them to + // DirectoryWrapper too. +}; + +enum InfoLogLevel : unsigned char { + DEBUG_LEVEL = 0, + INFO_LEVEL, + WARN_LEVEL, + ERROR_LEVEL, + FATAL_LEVEL, + HEADER_LEVEL, + NUM_INFO_LOG_LEVELS, +}; + +// An interface for writing log messages. +// +// Exceptions MUST NOT propagate out of overridden functions into RocksDB, +// because RocksDB is not exception-safe. This could cause undefined behavior +// including data loss, unreported corruption, deadlocks, and more. +class Logger { + public: + size_t kDoNotSupportGetLogFileSize = (std::numeric_limits::max)(); + + explicit Logger(const InfoLogLevel log_level = InfoLogLevel::INFO_LEVEL) + : closed_(false), log_level_(log_level) {} + // No copying allowed + Logger(const Logger&) = delete; + void operator=(const Logger&) = delete; + + virtual ~Logger(); + + // Because Logger is typically a shared object, Close() may or may not be + // called before the object is destroyed, but is recommended to reveal any + // final errors in finishing outstanding writes. No other functions are + // supported after calling Close(), and the Logger is considered closed + // regardless of return status. + virtual Status Close(); + + // Write a header to the log file with the specified format + // It is recommended that you log all header information at the start of the + // application. But it is not enforced. + virtual void LogHeader(const char* format, va_list ap) { + // Default implementation does a simple INFO level log write. + // Please override as per the logger class requirement. + Logv(InfoLogLevel::INFO_LEVEL, format, ap); + } + + // Write an entry to the log file with the specified format. + // + // Users who override the `Logv()` overload taking `InfoLogLevel` do not need + // to implement this, unless they explicitly invoke it in + // `Logv(InfoLogLevel, ...)`. + virtual void Logv(const char* /* format */, va_list /* ap */) { + assert(false); + } + + // Write an entry to the log file with the specified log level + // and format. Any log with level under the internal log level + // of *this (see @SetInfoLogLevel and @GetInfoLogLevel) will not be + // printed. + virtual void Logv(const InfoLogLevel log_level, const char* format, + va_list ap); + + virtual size_t GetLogFileSize() const { return kDoNotSupportGetLogFileSize; } + // Flush to the OS buffers + virtual void Flush() {} + virtual InfoLogLevel GetInfoLogLevel() const { return log_level_; } + virtual void SetInfoLogLevel(const InfoLogLevel log_level) { + log_level_ = log_level; + } + + // If you're adding methods here, remember to add them to LoggerWrapper too. + + protected: + virtual Status CloseImpl(); + bool closed_; + + private: + InfoLogLevel log_level_; +}; + +// Identifies a locked file. Except in custom Env/Filesystem implementations, +// the lifetime of a FileLock object should be managed only by LockFile() and +// UnlockFile(). +class FileLock { + public: + FileLock() {} + virtual ~FileLock(); + + private: + // No copying allowed + FileLock(const FileLock&) = delete; + void operator=(const FileLock&) = delete; +}; + +class DynamicLibrary { + public: + virtual ~DynamicLibrary() {} + + // Returns the name of the dynamic library. + virtual const char* Name() const = 0; + + // Loads the symbol for sym_name from the library and updates the input + // function. Returns the loaded symbol. + template + Status LoadFunction(const std::string& sym_name, std::function* function) { + assert(nullptr != function); + void* ptr = nullptr; + Status s = LoadSymbol(sym_name, &ptr); + *function = reinterpret_cast(ptr); + return s; + } + // Loads and returns the symbol for sym_name from the library. + virtual Status LoadSymbol(const std::string& sym_name, void** func) = 0; +}; + +extern void LogFlush(const std::shared_ptr& info_log); + +extern void Log(const InfoLogLevel log_level, + const std::shared_ptr& info_log, const char* format, + ...) ROCKSDB_PRINTF_FORMAT_ATTR(3, 4); + +// a set of log functions with different log levels. +extern void Header(const std::shared_ptr& info_log, const char* format, + ...) ROCKSDB_PRINTF_FORMAT_ATTR(2, 3); +extern void Debug(const std::shared_ptr& info_log, const char* format, + ...) ROCKSDB_PRINTF_FORMAT_ATTR(2, 3); +extern void Info(const std::shared_ptr& info_log, const char* format, + ...) ROCKSDB_PRINTF_FORMAT_ATTR(2, 3); +extern void Warn(const std::shared_ptr& info_log, const char* format, + ...) ROCKSDB_PRINTF_FORMAT_ATTR(2, 3); +extern void Error(const std::shared_ptr& info_log, const char* format, + ...) ROCKSDB_PRINTF_FORMAT_ATTR(2, 3); +extern void Fatal(const std::shared_ptr& info_log, const char* format, + ...) ROCKSDB_PRINTF_FORMAT_ATTR(2, 3); + +// Log the specified data to *info_log if info_log is non-nullptr. +// The default info log level is InfoLogLevel::INFO_LEVEL. +extern void Log(const std::shared_ptr& info_log, const char* format, + ...) ROCKSDB_PRINTF_FORMAT_ATTR(2, 3); + +extern void LogFlush(Logger* info_log); + +extern void Log(const InfoLogLevel log_level, Logger* info_log, + const char* format, ...) ROCKSDB_PRINTF_FORMAT_ATTR(3, 4); + +// The default info log level is InfoLogLevel::INFO_LEVEL. +extern void Log(Logger* info_log, const char* format, ...) + ROCKSDB_PRINTF_FORMAT_ATTR(2, 3); + +// a set of log functions with different log levels. +extern void Header(Logger* info_log, const char* format, ...) + ROCKSDB_PRINTF_FORMAT_ATTR(2, 3); +extern void Debug(Logger* info_log, const char* format, ...) + ROCKSDB_PRINTF_FORMAT_ATTR(2, 3); +extern void Info(Logger* info_log, const char* format, ...) + ROCKSDB_PRINTF_FORMAT_ATTR(2, 3); +extern void Warn(Logger* info_log, const char* format, ...) + ROCKSDB_PRINTF_FORMAT_ATTR(2, 3); +extern void Error(Logger* info_log, const char* format, ...) + ROCKSDB_PRINTF_FORMAT_ATTR(2, 3); +extern void Fatal(Logger* info_log, const char* format, ...) + ROCKSDB_PRINTF_FORMAT_ATTR(2, 3); + +// A utility routine: write "data" to the named file. +extern Status WriteStringToFile(Env* env, const Slice& data, + const std::string& fname, + bool should_sync = false); + +// A utility routine: read contents of named file into *data +extern Status ReadFileToString(Env* env, const std::string& fname, + std::string* data); + +// Below are helpers for wrapping most of the classes in this file. +// They forward all calls to another instance of the class. +// Useful when wrapping the default implementations. +// Typical usage is to inherit your wrapper from *Wrapper, e.g.: +// +// class MySequentialFileWrapper : public +// ROCKSDB_NAMESPACE::SequentialFileWrapper { +// public: +// MySequentialFileWrapper(ROCKSDB_NAMESPACE::SequentialFile* target): +// ROCKSDB_NAMESPACE::SequentialFileWrapper(target) {} +// Status Read(size_t n, Slice* result, char* scratch) override { +// cout << "Doing a read of size " << n << "!" << endl; +// return ROCKSDB_NAMESPACE::SequentialFileWrapper::Read(n, result, +// scratch); +// } +// // All other methods are forwarded to target_ automatically. +// }; +// +// This is often more convenient than inheriting the class directly because +// (a) Don't have to override and forward all methods - the Wrapper will +// forward everything you're not explicitly overriding. +// (b) Don't need to update the wrapper when more methods are added to the +// rocksdb class. Unless you actually want to override the behavior. +// (And unless rocksdb people forgot to update the *Wrapper class.) + +// An implementation of Env that forwards all calls to another Env. +// May be useful to clients who wish to override just part of the +// functionality of another Env. +class EnvWrapper : public Env { + public: + // The Target struct allows an Env to be stored as a raw (Env*) or + // std::shared_ptr. By using this struct, the wrapping/calling + // class does not need to worry about the ownership/lifetime of the + // wrapped target env. If the guard is set, then the Env will point + // to the guard.get(). + struct Target { + Env* env; // The raw Env + std::shared_ptr guard; // The guarded Env + + // Creates a Target without assuming ownership of the target Env + explicit Target(Env* t) : env(t) {} + + // Creates a Target from the guarded env, assuming ownership + explicit Target(std::unique_ptr&& t) : guard(t.release()) { + env = guard.get(); + } + + // Creates a Target from the guarded env, assuming ownership + explicit Target(const std::shared_ptr& t) : guard(t) { + env = guard.get(); + } + + // Makes sure the raw Env is not nullptr + void Prepare() { + if (guard.get() != nullptr) { + env = guard.get(); + } else if (env == nullptr) { + env = Env::Default(); + } + } + }; + + // Initialize an EnvWrapper that delegates all calls to *t + explicit EnvWrapper(Env* t); + explicit EnvWrapper(std::unique_ptr&& t); + explicit EnvWrapper(const std::shared_ptr& t); + ~EnvWrapper() override; + + // Return the target to which this Env forwards all calls + Env* target() const { return target_.env; } + + // Deprecated. Will be removed in a major release. Derived classes + // should implement this method. + const char* Name() const override { return target_.env->Name(); } + + // The following text is boilerplate that forwards all methods to target() + Status RegisterDbPaths(const std::vector& paths) override { + return target_.env->RegisterDbPaths(paths); + } + + Status UnregisterDbPaths(const std::vector& paths) override { + return target_.env->UnregisterDbPaths(paths); + } + + Status NewSequentialFile(const std::string& f, + std::unique_ptr* r, + const EnvOptions& options) override { + return target_.env->NewSequentialFile(f, r, options); + } + Status NewRandomAccessFile(const std::string& f, + std::unique_ptr* r, + const EnvOptions& options) override { + return target_.env->NewRandomAccessFile(f, r, options); + } + Status NewWritableFile(const std::string& f, std::unique_ptr* r, + const EnvOptions& options) override { + return target_.env->NewWritableFile(f, r, options); + } + Status ReopenWritableFile(const std::string& fname, + std::unique_ptr* result, + const EnvOptions& options) override { + return target_.env->ReopenWritableFile(fname, result, options); + } + Status ReuseWritableFile(const std::string& fname, + const std::string& old_fname, + std::unique_ptr* r, + const EnvOptions& options) override { + return target_.env->ReuseWritableFile(fname, old_fname, r, options); + } + Status NewRandomRWFile(const std::string& fname, + std::unique_ptr* result, + const EnvOptions& options) override { + return target_.env->NewRandomRWFile(fname, result, options); + } + Status NewMemoryMappedFileBuffer( + const std::string& fname, + std::unique_ptr* result) override { + return target_.env->NewMemoryMappedFileBuffer(fname, result); + } + Status NewDirectory(const std::string& name, + std::unique_ptr* result) override { + return target_.env->NewDirectory(name, result); + } + Status FileExists(const std::string& f) override { + return target_.env->FileExists(f); + } + Status GetChildren(const std::string& dir, + std::vector* r) override { + return target_.env->GetChildren(dir, r); + } + Status GetChildrenFileAttributes( + const std::string& dir, std::vector* result) override { + return target_.env->GetChildrenFileAttributes(dir, result); + } + Status DeleteFile(const std::string& f) override { + return target_.env->DeleteFile(f); + } + Status Truncate(const std::string& fname, size_t size) override { + return target_.env->Truncate(fname, size); + } + Status CreateDir(const std::string& d) override { + return target_.env->CreateDir(d); + } + Status CreateDirIfMissing(const std::string& d) override { + return target_.env->CreateDirIfMissing(d); + } + Status DeleteDir(const std::string& d) override { + return target_.env->DeleteDir(d); + } + Status GetFileSize(const std::string& f, uint64_t* s) override { + return target_.env->GetFileSize(f, s); + } + + Status GetFileModificationTime(const std::string& fname, + uint64_t* file_mtime) override { + return target_.env->GetFileModificationTime(fname, file_mtime); + } + + Status RenameFile(const std::string& s, const std::string& t) override { + return target_.env->RenameFile(s, t); + } + + Status LinkFile(const std::string& s, const std::string& t) override { + return target_.env->LinkFile(s, t); + } + + Status NumFileLinks(const std::string& fname, uint64_t* count) override { + return target_.env->NumFileLinks(fname, count); + } + + Status AreFilesSame(const std::string& first, const std::string& second, + bool* res) override { + return target_.env->AreFilesSame(first, second, res); + } + + Status LockFile(const std::string& f, FileLock** l) override { + return target_.env->LockFile(f, l); + } + + Status UnlockFile(FileLock* l) override { return target_.env->UnlockFile(l); } + + Status IsDirectory(const std::string& path, bool* is_dir) override { + return target_.env->IsDirectory(path, is_dir); + } + + Status LoadLibrary(const std::string& lib_name, + const std::string& search_path, + std::shared_ptr* result) override { + return target_.env->LoadLibrary(lib_name, search_path, result); + } + + void Schedule(void (*f)(void* arg), void* a, Priority pri, + void* tag = nullptr, void (*u)(void* arg) = nullptr) override { + return target_.env->Schedule(f, a, pri, tag, u); + } + + int UnSchedule(void* tag, Priority pri) override { + return target_.env->UnSchedule(tag, pri); + } + + void StartThread(void (*f)(void*), void* a) override { + return target_.env->StartThread(f, a); + } + void WaitForJoin() override { return target_.env->WaitForJoin(); } + unsigned int GetThreadPoolQueueLen(Priority pri = LOW) const override { + return target_.env->GetThreadPoolQueueLen(pri); + } + + int ReserveThreads(int threads_to_be_reserved, Priority pri) override { + return target_.env->ReserveThreads(threads_to_be_reserved, pri); + } + + int ReleaseThreads(int threads_to_be_released, Priority pri) override { + return target_.env->ReleaseThreads(threads_to_be_released, pri); + } + + Status GetTestDirectory(std::string* path) override { + return target_.env->GetTestDirectory(path); + } + Status NewLogger(const std::string& fname, + std::shared_ptr* result) override { + return target_.env->NewLogger(fname, result); + } + uint64_t NowMicros() override { return target_.env->NowMicros(); } + uint64_t NowNanos() override { return target_.env->NowNanos(); } + uint64_t NowCPUNanos() override { return target_.env->NowCPUNanos(); } + + void SleepForMicroseconds(int micros) override { + target_.env->SleepForMicroseconds(micros); + } + Status GetHostName(char* name, uint64_t len) override { + return target_.env->GetHostName(name, len); + } + Status GetCurrentTime(int64_t* unix_time) override { + return target_.env->GetCurrentTime(unix_time); + } + Status GetAbsolutePath(const std::string& db_path, + std::string* output_path) override { + return target_.env->GetAbsolutePath(db_path, output_path); + } + void SetBackgroundThreads(int num, Priority pri) override { + return target_.env->SetBackgroundThreads(num, pri); + } + int GetBackgroundThreads(Priority pri) override { + return target_.env->GetBackgroundThreads(pri); + } + + Status SetAllowNonOwnerAccess(bool allow_non_owner_access) override { + return target_.env->SetAllowNonOwnerAccess(allow_non_owner_access); + } + + void IncBackgroundThreadsIfNeeded(int num, Priority pri) override { + return target_.env->IncBackgroundThreadsIfNeeded(num, pri); + } + + void LowerThreadPoolIOPriority(Priority pool) override { + target_.env->LowerThreadPoolIOPriority(pool); + } + + void LowerThreadPoolCPUPriority(Priority pool) override { + target_.env->LowerThreadPoolCPUPriority(pool); + } + + Status LowerThreadPoolCPUPriority(Priority pool, CpuPriority pri) override { + return target_.env->LowerThreadPoolCPUPriority(pool, pri); + } + + std::string TimeToString(uint64_t time) override { + return target_.env->TimeToString(time); + } + + Status GetThreadList(std::vector* thread_list) override { + return target_.env->GetThreadList(thread_list); + } + + ThreadStatusUpdater* GetThreadStatusUpdater() const override { + return target_.env->GetThreadStatusUpdater(); + } + + uint64_t GetThreadID() const override { return target_.env->GetThreadID(); } + + std::string GenerateUniqueId() override { + return target_.env->GenerateUniqueId(); + } + + EnvOptions OptimizeForLogRead(const EnvOptions& env_options) const override { + return target_.env->OptimizeForLogRead(env_options); + } + EnvOptions OptimizeForManifestRead( + const EnvOptions& env_options) const override { + return target_.env->OptimizeForManifestRead(env_options); + } + EnvOptions OptimizeForLogWrite(const EnvOptions& env_options, + const DBOptions& db_options) const override { + return target_.env->OptimizeForLogWrite(env_options, db_options); + } + EnvOptions OptimizeForManifestWrite( + const EnvOptions& env_options) const override { + return target_.env->OptimizeForManifestWrite(env_options); + } + EnvOptions OptimizeForCompactionTableWrite( + const EnvOptions& env_options, + const ImmutableDBOptions& immutable_ops) const override { + return target_.env->OptimizeForCompactionTableWrite(env_options, + immutable_ops); + } + EnvOptions OptimizeForCompactionTableRead( + const EnvOptions& env_options, + const ImmutableDBOptions& db_options) const override { + return target_.env->OptimizeForCompactionTableRead(env_options, db_options); + } + EnvOptions OptimizeForBlobFileRead( + const EnvOptions& env_options, + const ImmutableDBOptions& db_options) const override { + return target_.env->OptimizeForBlobFileRead(env_options, db_options); + } + Status GetFreeSpace(const std::string& path, uint64_t* diskfree) override { + return target_.env->GetFreeSpace(path, diskfree); + } + void SanitizeEnvOptions(EnvOptions* env_opts) const override { + target_.env->SanitizeEnvOptions(env_opts); + } + Status PrepareOptions(const ConfigOptions& options) override; + std::string SerializeOptions(const ConfigOptions& config_options, + const std::string& header) const override; + + private: + Target target_; +}; + +class SequentialFileWrapper : public SequentialFile { + public: + explicit SequentialFileWrapper(SequentialFile* target) : target_(target) {} + + Status Read(size_t n, Slice* result, char* scratch) override { + return target_->Read(n, result, scratch); + } + Status Skip(uint64_t n) override { return target_->Skip(n); } + bool use_direct_io() const override { return target_->use_direct_io(); } + size_t GetRequiredBufferAlignment() const override { + return target_->GetRequiredBufferAlignment(); + } + Status InvalidateCache(size_t offset, size_t length) override { + return target_->InvalidateCache(offset, length); + } + Status PositionedRead(uint64_t offset, size_t n, Slice* result, + char* scratch) override { + return target_->PositionedRead(offset, n, result, scratch); + } + + private: + SequentialFile* target_; +}; + +class RandomAccessFileWrapper : public RandomAccessFile { + public: + explicit RandomAccessFileWrapper(RandomAccessFile* target) + : target_(target) {} + + Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const override { + return target_->Read(offset, n, result, scratch); + } + Status MultiRead(ReadRequest* reqs, size_t num_reqs) override { + return target_->MultiRead(reqs, num_reqs); + } + Status Prefetch(uint64_t offset, size_t n) override { + return target_->Prefetch(offset, n); + } + size_t GetUniqueId(char* id, size_t max_size) const override { + return target_->GetUniqueId(id, max_size); + } + void Hint(AccessPattern pattern) override { target_->Hint(pattern); } + bool use_direct_io() const override { return target_->use_direct_io(); } + size_t GetRequiredBufferAlignment() const override { + return target_->GetRequiredBufferAlignment(); + } + Status InvalidateCache(size_t offset, size_t length) override { + return target_->InvalidateCache(offset, length); + } + + private: + RandomAccessFile* target_; +}; + +class WritableFileWrapper : public WritableFile { + public: + explicit WritableFileWrapper(WritableFile* t) : target_(t) {} + + Status Append(const Slice& data) override { return target_->Append(data); } + Status Append(const Slice& data, + const DataVerificationInfo& verification_info) override { + return target_->Append(data, verification_info); + } + Status PositionedAppend(const Slice& data, uint64_t offset) override { + return target_->PositionedAppend(data, offset); + } + Status PositionedAppend( + const Slice& data, uint64_t offset, + const DataVerificationInfo& verification_info) override { + return target_->PositionedAppend(data, offset, verification_info); + } + Status Truncate(uint64_t size) override { return target_->Truncate(size); } + Status Close() override { return target_->Close(); } + Status Flush() override { return target_->Flush(); } + Status Sync() override { return target_->Sync(); } + Status Fsync() override { return target_->Fsync(); } + bool IsSyncThreadSafe() const override { return target_->IsSyncThreadSafe(); } + + bool use_direct_io() const override { return target_->use_direct_io(); } + + size_t GetRequiredBufferAlignment() const override { + return target_->GetRequiredBufferAlignment(); + } + + void SetIOPriority(Env::IOPriority pri) override { + target_->SetIOPriority(pri); + } + + Env::IOPriority GetIOPriority() override { return target_->GetIOPriority(); } + + void SetWriteLifeTimeHint(Env::WriteLifeTimeHint hint) override { + target_->SetWriteLifeTimeHint(hint); + } + + Env::WriteLifeTimeHint GetWriteLifeTimeHint() override { + return target_->GetWriteLifeTimeHint(); + } + + uint64_t GetFileSize() override { return target_->GetFileSize(); } + + void SetPreallocationBlockSize(size_t size) override { + target_->SetPreallocationBlockSize(size); + } + + void GetPreallocationStatus(size_t* block_size, + size_t* last_allocated_block) override { + target_->GetPreallocationStatus(block_size, last_allocated_block); + } + + size_t GetUniqueId(char* id, size_t max_size) const override { + return target_->GetUniqueId(id, max_size); + } + + Status InvalidateCache(size_t offset, size_t length) override { + return target_->InvalidateCache(offset, length); + } + + Status RangeSync(uint64_t offset, uint64_t nbytes) override { + return target_->RangeSync(offset, nbytes); + } + + void PrepareWrite(size_t offset, size_t len) override { + target_->PrepareWrite(offset, len); + } + + Status Allocate(uint64_t offset, uint64_t len) override { + return target_->Allocate(offset, len); + } + + private: + WritableFile* target_; +}; + +class RandomRWFileWrapper : public RandomRWFile { + public: + explicit RandomRWFileWrapper(RandomRWFile* target) : target_(target) {} + + bool use_direct_io() const override { return target_->use_direct_io(); } + size_t GetRequiredBufferAlignment() const override { + return target_->GetRequiredBufferAlignment(); + } + Status Write(uint64_t offset, const Slice& data) override { + return target_->Write(offset, data); + } + Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const override { + return target_->Read(offset, n, result, scratch); + } + Status Flush() override { return target_->Flush(); } + Status Sync() override { return target_->Sync(); } + Status Fsync() override { return target_->Fsync(); } + Status Close() override { return target_->Close(); } + + private: + RandomRWFile* target_; +}; + +class DirectoryWrapper : public Directory { + public: + explicit DirectoryWrapper(Directory* target) : target_(target) {} + + Status Fsync() override { return target_->Fsync(); } + Status Close() override { return target_->Close(); } + size_t GetUniqueId(char* id, size_t max_size) const override { + return target_->GetUniqueId(id, max_size); + } + + private: + Directory* target_; +}; + +class LoggerWrapper : public Logger { + public: + explicit LoggerWrapper(Logger* target) : target_(target) {} + + Status Close() override { return target_->Close(); } + void LogHeader(const char* format, va_list ap) override { + return target_->LogHeader(format, ap); + } + void Logv(const char* format, va_list ap) override { + return target_->Logv(format, ap); + } + void Logv(const InfoLogLevel log_level, const char* format, + va_list ap) override { + return target_->Logv(log_level, format, ap); + } + size_t GetLogFileSize() const override { return target_->GetLogFileSize(); } + void Flush() override { return target_->Flush(); } + InfoLogLevel GetInfoLogLevel() const override { + return target_->GetInfoLogLevel(); + } + void SetInfoLogLevel(const InfoLogLevel log_level) override { + return target_->SetInfoLogLevel(log_level); + } + + private: + Logger* target_; +}; + +// Returns a new environment that stores its data in memory and delegates +// all non-file-storage tasks to base_env. The caller must delete the result +// when it is no longer needed. +// *base_env must remain live while the result is in use. +Env* NewMemEnv(Env* base_env); + +// Returns a new environment that measures function call times for filesystem +// operations, reporting results to variables in PerfContext. +// This is a factory method for TimedEnv defined in utilities/env_timed.cc. +Env* NewTimedEnv(Env* base_env); + +// Returns an instance of logger that can be used for storing informational +// messages. +// This is a factory method for EnvLogger declared in logging/env_logging.h +Status NewEnvLogger(const std::string& fname, Env* env, + std::shared_ptr* result); + +// Creates a new Env based on Env::Default() but modified to use the specified +// FileSystem. +std::unique_ptr NewCompositeEnv(const std::shared_ptr& fs); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/env_encryption.h b/librocksdb-sys/rocksdb/include/rocksdb/env_encryption.h new file mode 100644 index 0000000..6feae06 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/env_encryption.h @@ -0,0 +1,363 @@ +// Copyright (c) 2016-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + + +#include + +#include "rocksdb/customizable.h" +#include "rocksdb/env.h" +#include "rocksdb/file_system.h" +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +class EncryptionProvider; + +struct ConfigOptions; + +// Returns an Env that encrypts data when stored on disk and decrypts data when +// read from disk. +Env* NewEncryptedEnv(Env* base_env, + const std::shared_ptr& provider); +std::shared_ptr NewEncryptedFS( + const std::shared_ptr& base_fs, + const std::shared_ptr& provider); + +// BlockAccessCipherStream is the base class for any cipher stream that +// supports random access at block level (without requiring data from other +// blocks). E.g. CTR (Counter operation mode) supports this requirement. +class BlockAccessCipherStream { + public: + virtual ~BlockAccessCipherStream(){}; + + // BlockSize returns the size of each block supported by this cipher stream. + virtual size_t BlockSize() = 0; + + // Encrypt one or more (partial) blocks of data at the file offset. + // Length of data is given in dataSize. + virtual Status Encrypt(uint64_t fileOffset, char* data, size_t dataSize); + + // Decrypt one or more (partial) blocks of data at the file offset. + // Length of data is given in dataSize. + virtual Status Decrypt(uint64_t fileOffset, char* data, size_t dataSize); + + protected: + // Allocate scratch space which is passed to EncryptBlock/DecryptBlock. + virtual void AllocateScratch(std::string&) = 0; + + // Encrypt a block of data at the given block index. + // Length of data is equal to BlockSize(); + virtual Status EncryptBlock(uint64_t blockIndex, char* data, + char* scratch) = 0; + + // Decrypt a block of data at the given block index. + // Length of data is equal to BlockSize(); + virtual Status DecryptBlock(uint64_t blockIndex, char* data, + char* scratch) = 0; +}; + +// BlockCipher +// +// Exceptions MUST NOT propagate out of overridden functions into RocksDB, +// because RocksDB is not exception-safe. This could cause undefined behavior +// including data loss, unreported corruption, deadlocks, and more. +class BlockCipher : public Customizable { + public: + virtual ~BlockCipher() {} + + // Creates a new BlockCipher from the input config_options and value + // The value describes the type of provider (and potentially optional + // configuration parameters) used to create this provider. + // For example, if the value is "ROT13", a ROT13BlockCipher is created. + // + // @param config_options Options to control how this cipher is created + // and initialized. + // @param value The value might be: + // - ROT13 Create a ROT13 Cipher + // - ROT13:nn Create a ROT13 Cipher with block size of nn + // @param result The new cipher object + // @return OK if the cipher was successfully created + // @return NotFound if an invalid name was specified in the value + // @return InvalidArgument if either the options were not valid + static Status CreateFromString(const ConfigOptions& config_options, + const std::string& value, + std::shared_ptr* result); + + static const char* Type() { return "BlockCipher"; } + // Short-cut method to create a ROT13 BlockCipher. + // This cipher is only suitable for test purposes and should not be used in + // production!!! + static std::shared_ptr NewROT13Cipher(size_t block_size); + + // BlockSize returns the size of each block supported by this cipher stream. + virtual size_t BlockSize() = 0; + + // Encrypt a block of data. + // Length of data is equal to BlockSize(). + virtual Status Encrypt(char* data) = 0; + + // Decrypt a block of data. + // Length of data is equal to BlockSize(). + virtual Status Decrypt(char* data) = 0; +}; + +// The encryption provider is used to create a cipher stream for a specific +// file. The returned cipher stream will be used for actual +// encryption/decryption actions. +// +// Exceptions MUST NOT propagate out of overridden functions into RocksDB, +// because RocksDB is not exception-safe. This could cause undefined behavior +// including data loss, unreported corruption, deadlocks, and more. +class EncryptionProvider : public Customizable { + public: + virtual ~EncryptionProvider() {} + + // Creates a new EncryptionProvider from the input config_options and value. + // The value describes the type of provider (and potentially optional + // configuration parameters) used to create this provider. + // For example, if the value is "CTR", a CTREncryptionProvider will be + // created. If the value is end with "://test" (e.g CTR://test"), the + // provider will be initialized in "TEST" mode prior to being returned. + // + // @param config_options Options to control how this provider is created + // and initialized. + // @param value The value might be: + // - CTR Create a CTR provider + // - CTR://test Create a CTR provider and initialize it for tests. + // @param result The new provider object + // @return OK if the provider was successfully created + // @return NotFound if an invalid name was specified in the value + // @return InvalidArgument if either the options were not valid + static Status CreateFromString(const ConfigOptions& config_options, + const std::string& value, + std::shared_ptr* result); + + static const char* Type() { return "EncryptionProvider"; } + + // Short-cut method to create a CTR-provider + static std::shared_ptr NewCTRProvider( + const std::shared_ptr& cipher); + + // GetPrefixLength returns the length of the prefix that is added to every + // file and used for storing encryption options. For optimal performance, the + // prefix length should be a multiple of the page size. + virtual size_t GetPrefixLength() const = 0; + + // CreateNewPrefix initialized an allocated block of prefix memory + // for a new file. + virtual Status CreateNewPrefix(const std::string& fname, char* prefix, + size_t prefixLength) const = 0; + + // Method to add a new cipher key for use by the EncryptionProvider. + // @param descriptor Descriptor for this key + // @param cipher The cryptographic key to use + // @param len The length of the cipher key + // @param for_write If true, this cipher should be used for writing files. + // If false, this cipher should only be used for reading + // files + // @return OK if the cipher was successfully added to the provider, non-OK + // otherwise + virtual Status AddCipher(const std::string& descriptor, const char* cipher, + size_t len, bool for_write) = 0; + + // CreateCipherStream creates a block access cipher stream for a file given + // name and options. + virtual Status CreateCipherStream( + const std::string& fname, const EnvOptions& options, Slice& prefix, + std::unique_ptr* result) = 0; + + // Returns a string representing an encryption marker prefix for this + // provider. If a marker is provided, this marker can be used to tell whether + // a file is encrypted by this provider. The marker will also be part of any + // encryption prefix for this provider. + virtual std::string GetMarker() const { return ""; } +}; + +class EncryptedSequentialFile : public FSSequentialFile { + protected: + std::unique_ptr file_; + std::unique_ptr stream_; + uint64_t offset_; + const size_t prefixLength_; + + public: + // Default ctor. Given underlying sequential file is supposed to be at + // offset == prefixLength. + EncryptedSequentialFile(std::unique_ptr&& f, + std::unique_ptr&& s, + size_t prefixLength) + : file_(std::move(f)), + stream_(std::move(s)), + offset_(prefixLength), + prefixLength_(prefixLength) {} + + IOStatus Read(size_t n, const IOOptions& options, Slice* result, + char* scratch, IODebugContext* dbg) override; + + IOStatus Skip(uint64_t n) override; + + bool use_direct_io() const override; + + size_t GetRequiredBufferAlignment() const override; + + IOStatus InvalidateCache(size_t offset, size_t length) override; + + IOStatus PositionedRead(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) override; +}; + +class EncryptedRandomAccessFile : public FSRandomAccessFile { + protected: + std::unique_ptr file_; + std::unique_ptr stream_; + size_t prefixLength_; + + public: + EncryptedRandomAccessFile(std::unique_ptr&& f, + std::unique_ptr&& s, + size_t prefixLength) + : file_(std::move(f)), + stream_(std::move(s)), + prefixLength_(prefixLength) {} + + IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const override; + + IOStatus Prefetch(uint64_t offset, size_t n, const IOOptions& options, + IODebugContext* dbg) override; + + size_t GetUniqueId(char* id, size_t max_size) const override; + + void Hint(AccessPattern pattern) override; + + bool use_direct_io() const override; + + size_t GetRequiredBufferAlignment() const override; + + IOStatus InvalidateCache(size_t offset, size_t length) override; +}; + +class EncryptedWritableFile : public FSWritableFile { + protected: + std::unique_ptr file_; + std::unique_ptr stream_; + size_t prefixLength_; + + public: + // Default ctor. Prefix is assumed to be written already. + EncryptedWritableFile(std::unique_ptr&& f, + std::unique_ptr&& s, + size_t prefixLength) + : file_(std::move(f)), + stream_(std::move(s)), + prefixLength_(prefixLength) {} + + using FSWritableFile::Append; + IOStatus Append(const Slice& data, const IOOptions& options, + IODebugContext* dbg) override; + + using FSWritableFile::PositionedAppend; + IOStatus PositionedAppend(const Slice& data, uint64_t offset, + const IOOptions& options, + IODebugContext* dbg) override; + + bool IsSyncThreadSafe() const override; + + bool use_direct_io() const override; + + size_t GetRequiredBufferAlignment() const override; + + uint64_t GetFileSize(const IOOptions& options, IODebugContext* dbg) override; + + IOStatus Truncate(uint64_t size, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus InvalidateCache(size_t offset, size_t length) override; + + IOStatus RangeSync(uint64_t offset, uint64_t nbytes, const IOOptions& options, + IODebugContext* dbg) override; + + void PrepareWrite(size_t offset, size_t len, const IOOptions& options, + IODebugContext* dbg) override; + + void SetPreallocationBlockSize(size_t size) override; + + void GetPreallocationStatus(size_t* block_size, + size_t* last_allocated_block) override; + + IOStatus Allocate(uint64_t offset, uint64_t len, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus Flush(const IOOptions& options, IODebugContext* dbg) override; + + IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override; + + IOStatus Close(const IOOptions& options, IODebugContext* dbg) override; +}; + +class EncryptedRandomRWFile : public FSRandomRWFile { + protected: + std::unique_ptr file_; + std::unique_ptr stream_; + size_t prefixLength_; + + public: + EncryptedRandomRWFile(std::unique_ptr&& f, + std::unique_ptr&& s, + size_t prefixLength) + : file_(std::move(f)), + stream_(std::move(s)), + prefixLength_(prefixLength) {} + + bool use_direct_io() const override; + + size_t GetRequiredBufferAlignment() const override; + + IOStatus Write(uint64_t offset, const Slice& data, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const override; + + IOStatus Flush(const IOOptions& options, IODebugContext* dbg) override; + + IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override; + + IOStatus Fsync(const IOOptions& options, IODebugContext* dbg) override; + + IOStatus Close(const IOOptions& options, IODebugContext* dbg) override; +}; + +class EncryptedFileSystem : public FileSystemWrapper { + public: + explicit EncryptedFileSystem(const std::shared_ptr& base) + : FileSystemWrapper(base) {} + // Method to add a new cipher key for use by the EncryptionProvider. + // @param description Descriptor for this key. + // @param cipher The cryptographic key to use + // @param len The length of the cipher key + // @param for_write If true, this cipher should be used for writing files. + // If false, this cipher should only be used for reading + // files + // @return OK if the cipher was successfully added to the provider, non-OK + // otherwise + virtual Status AddCipher(const std::string& descriptor, const char* cipher, + size_t len, bool for_write) = 0; + static const char* kClassName() { return "EncryptedFileSystem"; } + bool IsInstanceOf(const std::string& name) const override { + if (name == kClassName()) { + return true; + } else { + return FileSystemWrapper::IsInstanceOf(name); + } + } +}; +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/include/rocksdb/experimental.h b/librocksdb-sys/rocksdb/include/rocksdb/experimental.h new file mode 100644 index 0000000..b593952 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/experimental.h @@ -0,0 +1,56 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "rocksdb/db.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { +namespace experimental { + +// Supported only for Leveled compaction +Status SuggestCompactRange(DB* db, ColumnFamilyHandle* column_family, + const Slice* begin, const Slice* end); +Status SuggestCompactRange(DB* db, const Slice* begin, const Slice* end); + +// Move all L0 files to target_level skipping compaction. +// This operation succeeds only if the files in L0 have disjoint ranges; this +// is guaranteed to happen, for instance, if keys are inserted in sorted +// order. Furthermore, all levels between 1 and target_level must be empty. +// If any of the above condition is violated, InvalidArgument will be +// returned. +Status PromoteL0(DB* db, ColumnFamilyHandle* column_family, + int target_level = 1); + +struct UpdateManifestForFilesStateOptions { + // When true, read current file temperatures from FileSystem and update in + // DB manifest when a temperature other than Unknown is reported and + // inconsistent with manifest. + bool update_temperatures = true; + + // TODO: new_checksums: to update files to latest file checksum algorithm +}; + +// Utility for updating manifest of DB directory (not open) for current state +// of files on filesystem. See UpdateManifestForFilesStateOptions. +// +// To minimize interference with ongoing DB operations, only the following +// guarantee is provided, assuming no IO error encountered: +// * Only files live in DB at start AND end of call to +// UpdateManifestForFilesState() are guaranteed to be updated (as needed) in +// manifest. +// * For example, new files after start of call to +// UpdateManifestForFilesState() might not be updated, but that is not +// typically required to achieve goal of manifest consistency/completeness +// (because current DB configuration would ensure new files get the desired +// consistent metadata). +Status UpdateManifestForFilesState( + const DBOptions& db_opts, const std::string& db_name, + const std::vector& column_families, + const UpdateManifestForFilesStateOptions& opts = {}); + +} // namespace experimental +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/file_checksum.h b/librocksdb-sys/rocksdb/include/rocksdb/file_checksum.h new file mode 100644 index 0000000..758bae4 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/file_checksum.h @@ -0,0 +1,146 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2013 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include +#include +#include +#include +#include + +#include "rocksdb/customizable.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +// The unknown file checksum. +constexpr char kUnknownFileChecksum[] = ""; +// The unknown sst file checksum function name. +constexpr char kUnknownFileChecksumFuncName[] = "Unknown"; +// The standard DB file checksum function name. +// This is the name of the checksum function returned by +// GetFileChecksumGenCrc32cFactory(); +constexpr char kStandardDbFileChecksumFuncName[] = "FileChecksumCrc32c"; + +struct FileChecksumGenContext { + std::string file_name; + // The name of the requested checksum generator. + // Checksum factories may use or ignore requested_checksum_func_name, + // and checksum factories written before this field was available are still + // compatible. + std::string requested_checksum_func_name; +}; + +// FileChecksumGenerator is the class to generates the checksum value +// for each file when the file is written to the file system. +// Implementations may assume that +// * Finalize is called at most once during the life of the object +// * All calls to Update come before Finalize +// * All calls to GetChecksum come after Finalize +// +// Exceptions MUST NOT propagate out of overridden functions into RocksDB, +// because RocksDB is not exception-safe. This could cause undefined behavior +// including data loss, unreported corruption, deadlocks, and more. +class FileChecksumGenerator { + public: + virtual ~FileChecksumGenerator() {} + + // Update the current result after process the data. For different checksum + // functions, the temporal results may be stored and used in Update to + // include the new data. + virtual void Update(const char* data, size_t n) = 0; + + // Generate the final results if no further new data will be updated. + virtual void Finalize() = 0; + + // Get the checksum. The result should not be the empty string and may + // include arbitrary bytes, including non-printable characters. + virtual std::string GetChecksum() const = 0; + + // Returns a name that identifies the current file checksum function. + virtual const char* Name() const = 0; +}; + +// Create the FileChecksumGenerator object for each SST file. +// +// Exceptions MUST NOT propagate out of overridden functions into RocksDB, +// because RocksDB is not exception-safe. This could cause undefined behavior +// including data loss, unreported corruption, deadlocks, and more. +class FileChecksumGenFactory : public Customizable { + public: + ~FileChecksumGenFactory() override {} + static const char* Type() { return "FileChecksumGenFactory"; } + static Status CreateFromString( + const ConfigOptions& options, const std::string& value, + std::shared_ptr* result); + + // Create a new FileChecksumGenerator. + virtual std::unique_ptr CreateFileChecksumGenerator( + const FileChecksumGenContext& context) = 0; + + // Return the name of this FileChecksumGenFactory. + const char* Name() const override = 0; +}; + +// FileChecksumList stores the checksum information of a list of files (e.g., +// SST files). The FileChecksumList can be used to store the checksum +// information of all SST file getting from the MANIFEST, which are +// the checksum information of all valid SST file of a DB instance. It can +// also be used to store the checksum information of a list of SST files to +// be ingested. +// +// Exceptions MUST NOT propagate out of overridden functions into RocksDB, +// because RocksDB is not exception-safe. This could cause undefined behavior +// including data loss, unreported corruption, deadlocks, and more. +class FileChecksumList { + public: + virtual ~FileChecksumList() {} + + // Clean the previously stored file checksum information. + virtual void reset() = 0; + + // Get the number of checksums in the checksum list + virtual size_t size() const = 0; + + // Return all the file checksum information being stored in a unordered_map. + // File_number is the key, the first part of the value is checksum value, + // and the second part of the value is checksum function name. + virtual Status GetAllFileChecksums( + std::vector* file_numbers, std::vector* checksums, + std::vector* checksum_func_names) = 0; + + // Given the file_number, it searches if the file checksum information is + // stored. + virtual Status SearchOneFileChecksum(uint64_t file_number, + std::string* checksum, + std::string* checksum_func_name) = 0; + + // Insert the checksum information of one file to the FileChecksumList. + virtual Status InsertOneFileChecksum( + uint64_t file_number, const std::string& checksum, + const std::string& checksum_func_name) = 0; + + // Remove the checksum information of one SST file. + virtual Status RemoveOneFileChecksum(uint64_t file_number) = 0; +}; + +// Create a new file checksum list. +extern FileChecksumList* NewFileChecksumList(); + +// Return a shared_ptr of the builtin Crc32c based file checksum generator +// factory object, which can be shared to create the Crc32c based checksum +// generator object. +// Note: this implementation is compatible with many other crc32c checksum +// implementations and uses big-endian encoding of the result, unlike most +// other crc32c checksums in RocksDB, which alter the result with +// crc32c::Mask and use little-endian encoding. +extern std::shared_ptr +GetFileChecksumGenCrc32cFactory(); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/file_system.h b/librocksdb-sys/rocksdb/include/rocksdb/file_system.h new file mode 100644 index 0000000..f8e3214 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/file_system.h @@ -0,0 +1,1926 @@ +// Copyright (c) 2019-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// A FileSystem is an interface used by the rocksdb implementation to access +// storage functionality like the filesystem etc. Callers +// may wish to provide a custom FileSystem object when opening a database to +// get fine gain control; e.g., to rate limit file system operations. +// +// All FileSystem implementations are safe for concurrent access from +// multiple threads without any external synchronization. +// +// WARNING: Since this is a new interface, it is expected that there will be +// some changes as storage systems are ported over. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rocksdb/customizable.h" +#include "rocksdb/env.h" +#include "rocksdb/io_status.h" +#include "rocksdb/options.h" +#include "rocksdb/table.h" +#include "rocksdb/thread_status.h" + +namespace ROCKSDB_NAMESPACE { + +class FileLock; +class FSDirectory; +class FSRandomAccessFile; +class FSRandomRWFile; +class FSSequentialFile; +class FSWritableFile; +class Logger; +class Slice; +struct ImmutableDBOptions; +struct MutableDBOptions; +class RateLimiter; +struct ConfigOptions; + +using AccessPattern = RandomAccessFile::AccessPattern; +using FileAttributes = Env::FileAttributes; + +// DEPRECATED +// Priority of an IO request. This is a hint and does not guarantee any +// particular QoS. +// IO_LOW - Typically background reads/writes such as compaction/flush +// IO_HIGH - Typically user reads/synchronous WAL writes +enum class IOPriority : uint8_t { + kIOLow, + kIOHigh, + kIOTotal, +}; + +// Type of the data begin read/written. It can be passed down as a flag +// for the FileSystem implementation to optionally handle different types in +// different ways +enum class IOType : uint8_t { + kData, + kFilter, + kIndex, + kMetadata, + kWAL, + kManifest, + kLog, + kUnknown, + kInvalid, +}; + +// enum representing various operations supported by underlying FileSystem. +// These need to be set in SupportedOps API for RocksDB to use them. +enum FSSupportedOps { kAsyncIO, kFSBuffer }; + +// Per-request options that can be passed down to the FileSystem +// implementation. These are hints and are not necessarily guaranteed to be +// honored. More hints can be added here in the future to indicate things like +// storage media (HDD/SSD) to be used, replication level etc. +struct IOOptions { + // Timeout for the operation in microseconds + std::chrono::microseconds timeout; + + // DEPRECATED + // Priority - high or low + IOPriority prio; + + // Priority used to charge rate limiter configured in file system level (if + // any) + // Limitation: right now RocksDB internal does not consider this + // rate_limiter_priority + Env::IOPriority rate_limiter_priority; + + // Type of data being read/written + IOType type; + + // EXPERIMENTAL + // An option map that's opaque to RocksDB. It can be used to implement a + // custom contract between a FileSystem user and the provider. This is only + // useful in cases where a RocksDB user directly uses the FileSystem or file + // object for their own purposes, and wants to pass extra options to APIs + // such as NewRandomAccessFile and NewWritableFile. + std::unordered_map property_bag; + + // Force directory fsync, some file systems like btrfs may skip directory + // fsync, set this to force the fsync + bool force_dir_fsync; + + // Can be used by underlying file systems to skip recursing through sub + // directories and list only files in GetChildren API. + bool do_not_recurse; + + Env::IOActivity io_activity = Env::IOActivity::kUnknown; + + IOOptions() : IOOptions(false) {} + + explicit IOOptions(bool force_dir_fsync_) + : timeout(std::chrono::microseconds::zero()), + prio(IOPriority::kIOLow), + rate_limiter_priority(Env::IO_TOTAL), + type(IOType::kUnknown), + force_dir_fsync(force_dir_fsync_), + do_not_recurse(false) {} +}; + +struct DirFsyncOptions { + enum FsyncReason : uint8_t { + kNewFileSynced, + kFileRenamed, + kDirRenamed, + kFileDeleted, + kDefault, + } reason; + + std::string renamed_new_name; // for kFileRenamed + // add other options for other FsyncReason + + DirFsyncOptions(); + + explicit DirFsyncOptions(std::string file_renamed_new_name); + + explicit DirFsyncOptions(FsyncReason fsync_reason); +}; + +// File scope options that control how a file is opened/created and accessed +// while its open. We may add more options here in the future such as +// redundancy level, media to use etc. +struct FileOptions : EnvOptions { + // Embedded IOOptions to control the parameters for any IOs that need + // to be issued for the file open/creation + IOOptions io_options; + + // EXPERIMENTAL + // The feature is in development and is subject to change. + // When creating a new file, set the temperature of the file so that + // underlying file systems can put it with appropriate storage media and/or + // coding. + Temperature temperature = Temperature::kUnknown; + + // The checksum type that is used to calculate the checksum value for + // handoff during file writes. + ChecksumType handoff_checksum_type; + + FileOptions() : EnvOptions(), handoff_checksum_type(ChecksumType::kCRC32c) {} + + FileOptions(const DBOptions& opts) + : EnvOptions(opts), handoff_checksum_type(ChecksumType::kCRC32c) {} + + FileOptions(const EnvOptions& opts) + : EnvOptions(opts), handoff_checksum_type(ChecksumType::kCRC32c) {} + + FileOptions(const FileOptions& opts) + : EnvOptions(opts), + io_options(opts.io_options), + temperature(opts.temperature), + handoff_checksum_type(opts.handoff_checksum_type) {} + + FileOptions& operator=(const FileOptions&) = default; +}; + +// A structure to pass back some debugging information from the FileSystem +// implementation to RocksDB in case of an IO error +struct IODebugContext { + // file_path to be filled in by RocksDB in case of an error + std::string file_path; + + // A map of counter names to values - set by the FileSystem implementation + std::map counters; + + // To be set by the FileSystem implementation + std::string msg; + + // To be set by the underlying FileSystem implementation. + std::string request_id; + + // In order to log required information in IO tracing for different + // operations, Each bit in trace_data stores which corresponding info from + // IODebugContext will be added in the trace. Foreg, if trace_data = 1, it + // means bit at position 0 is set so TraceData::kRequestID (request_id) will + // be logged in the trace record. + // + enum TraceData : char { + // The value of each enum represents the bitwise position for + // that information in trace_data which will be used by IOTracer for + // tracing. Make sure to add them sequentially. + kRequestID = 0, + }; + uint64_t trace_data = 0; + + IODebugContext() {} + + void AddCounter(std::string& name, uint64_t value) { + counters.emplace(name, value); + } + + // Called by underlying file system to set request_id and log request_id in + // IOTracing. + void SetRequestId(const std::string& _request_id) { + request_id = _request_id; + trace_data |= (1 << TraceData::kRequestID); + } + + std::string ToString() { + std::ostringstream ss; + ss << file_path << ", "; + for (auto counter : counters) { + ss << counter.first << " = " << counter.second << ","; + } + ss << msg; + return ss.str(); + } +}; + +// A function pointer type for custom destruction of void pointer passed to +// ReadAsync API. RocksDB/caller is responsible for deleting the void pointer +// allocated by FS in ReadAsync API. +using IOHandleDeleter = std::function; + +// The FileSystem, FSSequentialFile, FSRandomAccessFile, FSWritableFile, +// FSRandomRWFileclass, and FSDIrectory classes define the interface between +// RocksDB and storage systems, such as Posix filesystems, +// remote filesystems etc. +// The interface allows for fine grained control of individual IO operations, +// such as setting a timeout, prioritization, hints on data placement, +// different handling based on type of IO etc. +// This is accomplished by passing an instance of IOOptions to every +// API call that can potentially perform IO. Additionally, each such API is +// passed a pointer to a IODebugContext structure that can be used by the +// storage system to include troubleshooting information. The return values +// of the APIs is of type IOStatus, which can indicate an error code/sub-code, +// as well as metadata about the error such as its scope and whether its +// retryable. +// NewCompositeEnv can be used to create an Env with a custom FileSystem for +// DBOptions::env. +// +// Exceptions MUST NOT propagate out of overridden functions into RocksDB, +// because RocksDB is not exception-safe. This could cause undefined behavior +// including data loss, unreported corruption, deadlocks, and more. +class FileSystem : public Customizable { + public: + FileSystem(); + + // No copying allowed + FileSystem(const FileSystem&) = delete; + + virtual ~FileSystem(); + + static const char* Type() { return "FileSystem"; } + static const char* kDefaultName() { return "DefaultFileSystem"; } + + // Loads the FileSystem specified by the input value into the result + // @see Customizable for a more detailed description of the parameters and + // return codes + // @param config_options Controls how the FileSystem is loaded + // @param value The name and optional properties describing the file system + // to load. + // @param result On success, returns the loaded FileSystem + // @return OK if the FileSystem was successfully loaded. + // @return not-OK if the load failed. + static Status CreateFromString(const ConfigOptions& options, + const std::string& value, + std::shared_ptr* result); + + // Return a default FileSystem suitable for the current operating + // system. + static std::shared_ptr Default(); + + // Handles the event when a new DB or a new ColumnFamily starts using the + // specified data paths. + // + // The data paths might be shared by different DBs or ColumnFamilies, + // so RegisterDbPaths might be called with the same data paths. + // For example, when CreateColumnFamily is called multiple times with the same + // data path, RegisterDbPaths will also be called with the same data path. + // + // If the return status is ok, then the paths must be correspondingly + // called in UnregisterDbPaths; + // otherwise this method should have no side effect, and UnregisterDbPaths + // do not need to be called for the paths. + // + // Different implementations may take different actions. + // By default, it's a no-op and returns Status::OK. + virtual Status RegisterDbPaths(const std::vector& /*paths*/) { + return Status::OK(); + } + // Handles the event a DB or a ColumnFamily stops using the specified data + // paths. + // + // It should be called corresponding to each successful RegisterDbPaths. + // + // Different implementations may take different actions. + // By default, it's a no-op and returns Status::OK. + virtual Status UnregisterDbPaths(const std::vector& /*paths*/) { + return Status::OK(); + } + + // Create a brand new sequentially-readable file with the specified name. + // On success, stores a pointer to the new file in *result and returns OK. + // On failure stores nullptr in *result and returns non-OK. If the file does + // not exist, returns a non-OK status. + // + // The returned file will only be accessed by one thread at a time. + virtual IOStatus NewSequentialFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) = 0; + + // Create a brand new random access read-only file with the + // specified name. On success, stores a pointer to the new file in + // *result and returns OK. On failure stores nullptr in *result and + // returns non-OK. If the file does not exist, returns a non-OK + // status. + // + // The returned file may be concurrently accessed by multiple threads. + virtual IOStatus NewRandomAccessFile( + const std::string& fname, const FileOptions& file_opts, + std::unique_ptr* result, IODebugContext* dbg) = 0; + // These values match Linux definition + // https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/include/uapi/linux/fcntl.h#n56 + enum WriteLifeTimeHint { + kWLTHNotSet = 0, // No hint information set + kWLTHNone, // No hints about write life time + kWLTHShort, // Data written has a short life time + kWLTHMedium, // Data written has a medium life time + kWLTHLong, // Data written has a long life time + kWLTHExtreme, // Data written has an extremely long life time + }; + + // Create an object that writes to a new file with the specified + // name. Deletes any existing file with the same name and creates a + // new file. On success, stores a pointer to the new file in + // *result and returns OK. On failure stores nullptr in *result and + // returns non-OK. + // + // The returned file will only be accessed by one thread at a time. + virtual IOStatus NewWritableFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) = 0; + + // Create an object that writes to a file with the specified name. + // `FSWritableFile::Append()`s will append after any existing content. If the + // file does not already exist, creates it. + // + // On success, stores a pointer to the file in *result and returns OK. On + // failure stores nullptr in *result and returns non-OK. + // + // The returned file will only be accessed by one thread at a time. + virtual IOStatus ReopenWritableFile( + const std::string& /*fname*/, const FileOptions& /*options*/, + std::unique_ptr* /*result*/, IODebugContext* /*dbg*/) { + return IOStatus::NotSupported("ReopenWritableFile"); + } + + // Reuse an existing file by renaming it and opening it as writable. + virtual IOStatus ReuseWritableFile(const std::string& fname, + const std::string& old_fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg); + + // Open `fname` for random read and write, if file doesn't exist the file + // will be created. On success, stores a pointer to the new file in + // *result and returns OK. On failure returns non-OK. + // + // The returned file will only be accessed by one thread at a time. + virtual IOStatus NewRandomRWFile(const std::string& /*fname*/, + const FileOptions& /*options*/, + std::unique_ptr* /*result*/, + IODebugContext* /*dbg*/) { + return IOStatus::NotSupported( + "RandomRWFile is not implemented in this FileSystem"); + } + + // Opens `fname` as a memory-mapped file for read and write (in-place updates + // only, i.e., no appends). On success, stores a raw buffer covering the whole + // file in `*result`. The file must exist prior to this call. + virtual IOStatus NewMemoryMappedFileBuffer( + const std::string& /*fname*/, + std::unique_ptr* /*result*/) { + return IOStatus::NotSupported( + "MemoryMappedFileBuffer is not implemented in this FileSystem"); + } + + // Create an object that represents a directory. Will fail if directory + // doesn't exist. If the directory exists, it will open the directory + // and create a new Directory object. + // + // On success, stores a pointer to the new Directory in + // *result and returns OK. On failure stores nullptr in *result and + // returns non-OK. + virtual IOStatus NewDirectory(const std::string& name, + const IOOptions& io_opts, + std::unique_ptr* result, + IODebugContext* dbg) = 0; + + // Returns OK if the named file exists. + // NotFound if the named file does not exist, + // the calling process does not have permission to determine + // whether this file exists, or if the path is invalid. + // IOError if an IO Error was encountered + virtual IOStatus FileExists(const std::string& fname, + const IOOptions& options, + IODebugContext* dbg) = 0; + + // Store in *result the names of the children of the specified directory. + // The names are relative to "dir". + // Original contents of *results are dropped. + // Returns OK if "dir" exists and "*result" contains its children. + // NotFound if "dir" does not exist, the calling process does not have + // permission to access "dir", or if "dir" is invalid. + // IOError if an IO Error was encountered + virtual IOStatus GetChildren(const std::string& dir, const IOOptions& options, + std::vector* result, + IODebugContext* dbg) = 0; + + // Store in *result the attributes of the children of the specified directory. + // In case the implementation lists the directory prior to iterating the files + // and files are concurrently deleted, the deleted files will be omitted from + // result. + // The name attributes are relative to "dir". + // Original contents of *results are dropped. + // Returns OK if "dir" exists and "*result" contains its children. + // NotFound if "dir" does not exist, the calling process does not have + // permission to access "dir", or if "dir" is invalid. + // IOError if an IO Error was encountered + virtual IOStatus GetChildrenFileAttributes( + const std::string& dir, const IOOptions& options, + std::vector* result, IODebugContext* dbg) { + assert(result != nullptr); + std::vector child_fnames; + IOStatus s = GetChildren(dir, options, &child_fnames, dbg); + if (!s.ok()) { + return s; + } + result->resize(child_fnames.size()); + size_t result_size = 0; + for (size_t i = 0; i < child_fnames.size(); ++i) { + const std::string path = dir + "/" + child_fnames[i]; + if (!(s = GetFileSize(path, options, &(*result)[result_size].size_bytes, + dbg)) + .ok()) { + if (FileExists(path, options, dbg).IsNotFound()) { + // The file may have been deleted since we listed the directory + continue; + } + return s; + } + (*result)[result_size].name = std::move(child_fnames[i]); + result_size++; + } + result->resize(result_size); + return IOStatus::OK(); + } + +// This seems to clash with a macro on Windows, so #undef it here +#ifdef DeleteFile +#undef DeleteFile +#endif + // Delete the named file. + virtual IOStatus DeleteFile(const std::string& fname, + const IOOptions& options, + IODebugContext* dbg) = 0; + + // Truncate the named file to the specified size. + virtual IOStatus Truncate(const std::string& /*fname*/, size_t /*size*/, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + return IOStatus::NotSupported( + "Truncate is not supported for this FileSystem"); + } + + // Create the specified directory. Returns error if directory exists. + virtual IOStatus CreateDir(const std::string& dirname, + const IOOptions& options, IODebugContext* dbg) = 0; + + // Creates directory if missing. Return Ok if it exists, or successful in + // Creating. + virtual IOStatus CreateDirIfMissing(const std::string& dirname, + const IOOptions& options, + IODebugContext* dbg) = 0; + + // Delete the specified directory. + virtual IOStatus DeleteDir(const std::string& dirname, + const IOOptions& options, IODebugContext* dbg) = 0; + + // Store the size of fname in *file_size. + virtual IOStatus GetFileSize(const std::string& fname, + const IOOptions& options, uint64_t* file_size, + IODebugContext* dbg) = 0; + + // Store the last modification time of fname in *file_mtime. + virtual IOStatus GetFileModificationTime(const std::string& fname, + const IOOptions& options, + uint64_t* file_mtime, + IODebugContext* dbg) = 0; + // Rename file src to target. + virtual IOStatus RenameFile(const std::string& src, const std::string& target, + const IOOptions& options, + IODebugContext* dbg) = 0; + + // Hard Link file src to target. + virtual IOStatus LinkFile(const std::string& /*src*/, + const std::string& /*target*/, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + return IOStatus::NotSupported( + "LinkFile is not supported for this FileSystem"); + } + + virtual IOStatus NumFileLinks(const std::string& /*fname*/, + const IOOptions& /*options*/, + uint64_t* /*count*/, IODebugContext* /*dbg*/) { + return IOStatus::NotSupported( + "Getting number of file links is not supported for this FileSystem"); + } + + virtual IOStatus AreFilesSame(const std::string& /*first*/, + const std::string& /*second*/, + const IOOptions& /*options*/, bool* /*res*/, + IODebugContext* /*dbg*/) { + return IOStatus::NotSupported( + "AreFilesSame is not supported for this FileSystem"); + } + + // Lock the specified file. Used to prevent concurrent access to + // the same db by multiple processes. On failure, stores nullptr in + // *lock and returns non-OK. + // + // On success, stores a pointer to the object that represents the + // acquired lock in *lock and returns OK. The caller should call + // UnlockFile(*lock) to release the lock. If the process exits, + // the lock will be automatically released. + // + // If somebody else already holds the lock, finishes immediately + // with a failure. I.e., this call does not wait for existing locks + // to go away. + // + // May create the named file if it does not already exist. + virtual IOStatus LockFile(const std::string& fname, const IOOptions& options, + FileLock** lock, IODebugContext* dbg) = 0; + + // Release the lock acquired by a previous successful call to LockFile. + // REQUIRES: lock was returned by a successful LockFile() call + // REQUIRES: lock has not already been unlocked. + virtual IOStatus UnlockFile(FileLock* lock, const IOOptions& options, + IODebugContext* dbg) = 0; + + // *path is set to a temporary directory that can be used for testing. It may + // or many not have just been created. The directory may or may not differ + // between runs of the same process, but subsequent calls will return the + // same directory. + virtual IOStatus GetTestDirectory(const IOOptions& options, std::string* path, + IODebugContext* dbg) = 0; + + // Create and returns a default logger (an instance of EnvLogger) for storing + // informational messages. Derived classes can override to provide custom + // logger. + virtual IOStatus NewLogger(const std::string& fname, const IOOptions& io_opts, + std::shared_ptr* result, + IODebugContext* dbg); + + // Get full directory name for this db. + virtual IOStatus GetAbsolutePath(const std::string& db_path, + const IOOptions& options, + std::string* output_path, + IODebugContext* dbg) = 0; + + // Sanitize the FileOptions. Typically called by a FileOptions/EnvOptions + // copy constructor + virtual void SanitizeFileOptions(FileOptions* /*opts*/) const {} + + // OptimizeForLogRead will create a new FileOptions object that is a copy of + // the FileOptions in the parameters, but is optimized for reading log files. + virtual FileOptions OptimizeForLogRead(const FileOptions& file_options) const; + + // OptimizeForManifestRead will create a new FileOptions object that is a copy + // of the FileOptions in the parameters, but is optimized for reading manifest + // files. + virtual FileOptions OptimizeForManifestRead( + const FileOptions& file_options) const; + + // OptimizeForLogWrite will create a new FileOptions object that is a copy of + // the FileOptions in the parameters, but is optimized for writing log files. + // Default implementation returns the copy of the same object. + virtual FileOptions OptimizeForLogWrite(const FileOptions& file_options, + const DBOptions& db_options) const; + + // OptimizeForManifestWrite will create a new FileOptions object that is a + // copy of the FileOptions in the parameters, but is optimized for writing + // manifest files. Default implementation returns the copy of the same + // object. + virtual FileOptions OptimizeForManifestWrite( + const FileOptions& file_options) const; + + // OptimizeForCompactionTableWrite will create a new FileOptions object that + // is a copy of the FileOptions in the parameters, but is optimized for + // writing table files. + virtual FileOptions OptimizeForCompactionTableWrite( + const FileOptions& file_options, + const ImmutableDBOptions& immutable_ops) const; + + // OptimizeForCompactionTableRead will create a new FileOptions object that + // is a copy of the FileOptions in the parameters, but is optimized for + // reading table files. + virtual FileOptions OptimizeForCompactionTableRead( + const FileOptions& file_options, + const ImmutableDBOptions& db_options) const; + + // OptimizeForBlobFileRead will create a new FileOptions object that + // is a copy of the FileOptions in the parameters, but is optimized for + // reading blob files. + virtual FileOptions OptimizeForBlobFileRead( + const FileOptions& file_options, + const ImmutableDBOptions& db_options) const; + +// This seems to clash with a macro on Windows, so #undef it here +#ifdef GetFreeSpace +#undef GetFreeSpace +#endif + + // Get the amount of free disk space + virtual IOStatus GetFreeSpace(const std::string& /*path*/, + const IOOptions& /*options*/, + uint64_t* /*diskfree*/, + IODebugContext* /*dbg*/) { + return IOStatus::NotSupported("GetFreeSpace"); + } + + virtual IOStatus IsDirectory(const std::string& /*path*/, + const IOOptions& options, bool* is_dir, + IODebugContext* /*dgb*/) = 0; + + // Poll for completion of read IO requests. The Poll() method should call the + // callback functions to indicate completion of read requests. + // Underlying FS is required to support Poll API. Poll implementation should + // ensure that the callback gets called at IO completion, and return only + // after the callback has been called. + // If Poll returns partial results for any reads, its caller reponsibility to + // call Read or ReadAsync in order to get the remaining bytes. + virtual IOStatus Poll(std::vector& /*io_handles*/, + size_t /*min_completions*/) { + return IOStatus::OK(); + } + + // Abort the read IO requests submitted asynchronously. Underlying FS is + // required to support AbortIO API. AbortIO implementation should ensure that + // the all the read requests related to io_handles should be aborted and + // it shouldn't call the callback for these io_handles. + virtual IOStatus AbortIO(std::vector& /*io_handles*/) { + return IOStatus::OK(); + } + + // Indicates to upper layers which FileSystem operations mentioned in + // FSSupportedOps are supported by underlying FileSystem. Each bit in + // supported_ops argument represent corresponding FSSupportedOps operation. + // Foreg: + // If async_io is supported by the underlying FileSystem, then supported_ops + // will have corresponding bit (i.e FSSupportedOps::kAsyncIO) set to 1. + // + // By default, async_io operation is set and FS should override this API and + // set all the operations they support provided in FSSupportedOps (including + // async_io). + virtual void SupportedOps(int64_t& supported_ops) { + supported_ops = 0; + supported_ops |= (1 << FSSupportedOps::kAsyncIO); + } + + // If you're adding methods here, remember to add them to EnvWrapper too. + + private: + void operator=(const FileSystem&); +}; + +// A file abstraction for reading sequentially through a file +class FSSequentialFile { + public: + FSSequentialFile() {} + + virtual ~FSSequentialFile() {} + + // Read up to "n" bytes from the file. "scratch[0..n-1]" may be + // written by this routine. Sets "*result" to the data that was + // read (including if fewer than "n" bytes were successfully read). + // May set "*result" to point at data in "scratch[0..n-1]", so + // "scratch[0..n-1]" must be live when "*result" is used. + // If an error was encountered, returns a non-OK status. + // + // After call, result->size() < n only if end of file has been + // reached (or non-OK status). Read might fail if called again after + // first result->size() < n. + // + // REQUIRES: External synchronization + virtual IOStatus Read(size_t n, const IOOptions& options, Slice* result, + char* scratch, IODebugContext* dbg) = 0; + + // Skip "n" bytes from the file. This is guaranteed to be no + // slower that reading the same data, but may be faster. + // + // If end of file is reached, skipping will stop at the end of the + // file, and Skip will return OK. + // + // REQUIRES: External synchronization + virtual IOStatus Skip(uint64_t n) = 0; + + // Indicates the upper layers if the current SequentialFile implementation + // uses direct IO. + virtual bool use_direct_io() const { return false; } + + // Use the returned alignment value to allocate + // aligned buffer for Direct I/O + virtual size_t GetRequiredBufferAlignment() const { return kDefaultPageSize; } + + // Remove any kind of caching of data from the offset to offset+length + // of this file. If the length is 0, then it refers to the end of file. + // If the system is not caching the file contents, then this is a noop. + virtual IOStatus InvalidateCache(size_t /*offset*/, size_t /*length*/) { + return IOStatus::NotSupported("InvalidateCache not supported."); + } + + // Positioned Read for direct I/O + // If Direct I/O enabled, offset, n, and scratch should be properly aligned + virtual IOStatus PositionedRead(uint64_t /*offset*/, size_t /*n*/, + const IOOptions& /*options*/, + Slice* /*result*/, char* /*scratch*/, + IODebugContext* /*dbg*/) { + return IOStatus::NotSupported("PositionedRead"); + } + + // EXPERIMENTAL + // When available, returns the actual temperature for the file. This is + // useful in case some outside process moves a file from one tier to another, + // though the temperature is generally expected not to change while a file is + // open. + virtual Temperature GetTemperature() const { return Temperature::kUnknown; } + + // If you're adding methods here, remember to add them to + // SequentialFileWrapper too. +}; + +// A read IO request structure for use in MultiRead and asynchronous Read APIs. +struct FSReadRequest { + // Input parameter that represents the file offset in bytes. + uint64_t offset; + + // Input parameter that represents the length to read in bytes. `result` only + // returns fewer bytes if end of file is hit (or `status` is not OK). + size_t len; + + // A buffer that MultiRead() can optionally place data in. It can + // ignore this and allocate its own buffer. + // The lifecycle of scratch will be until IO is completed. + // + // In case of asynchronous reads, its an output parameter and it will be + // maintained until callback has been called. Scratch is allocated by RocksDB + // and will be passed to underlying FileSystem. + char* scratch; + + // Output parameter set by MultiRead() to point to the data buffer, and + // the number of valid bytes + // + // In case of asynchronous reads, this output parameter is set by Async Read + // APIs to point to the data buffer, and + // the number of valid bytes. + // Slice result should point to scratch i.e the data should + // always be read into scratch. + Slice result; + + // Output parameter set by underlying FileSystem that represents status of + // read request. + IOStatus status; + + // fs_scratch is a data buffer allocated and provided by underlying FileSystem + // to RocksDB during reads, when FS wants to provide its own buffer with data + // instead of using RocksDB provided FSReadRequest::scratch. + // + // FileSystem needs to provide a buffer and custom delete function. The + // lifecycle of fs_scratch until data is used by RocksDB. The buffer + // should be released by RocksDB using custom delete function provided in + // unique_ptr fs_scratch. + // + // Optimization benefits: + // This is helpful in cases where underlying FileSystem has to do additional + // copy of data to RocksDB provided buffer which can consume CPU cycles. It + // can be optimized by avoiding copying to RocksDB buffer and directly using + // FS provided buffer. + // + // How to enable: + // In order to enable this option, FS needs to override SupportedOps() API and + // set FSSupportedOps::kFSBuffer in SupportedOps() as: + // { + // supported_ops |= (1 << FSSupportedOps::kFSBuffer); + // } + // + // Work in progress: + // Right now it's only enabled for MultiReads (sync and async + // both) with non direct io. + // If RocksDB provide its own buffer (scratch) during reads, that's a + // signal for FS to use RocksDB buffer. + // If FSSupportedOps::kFSBuffer is enabled and scratch == nullptr, + // then FS have to provide its own buffer in fs_scratch. + // + // NOTE: + // - FSReadRequest::result should point to fs_scratch. + // - This is needed only if FSSupportedOps::kFSBuffer support is provided by + // underlying FS. + std::unique_ptr> fs_scratch; +}; + +// A file abstraction for randomly reading the contents of a file. +class FSRandomAccessFile { + public: + FSRandomAccessFile() {} + + virtual ~FSRandomAccessFile() {} + + // Read up to "n" bytes from the file starting at "offset". + // "scratch[0..n-1]" may be written by this routine. Sets "*result" + // to the data that was read (including if fewer than "n" bytes were + // successfully read). May set "*result" to point at data in + // "scratch[0..n-1]", so "scratch[0..n-1]" must be live when + // "*result" is used. If an error was encountered, returns a non-OK + // status. + // + // After call, result->size() < n only if end of file has been + // reached (or non-OK status). Read might fail if called again after + // first result->size() < n. + // + // Safe for concurrent use by multiple threads. + // If Direct I/O enabled, offset, n, and scratch should be aligned properly. + virtual IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const = 0; + + // Readahead the file starting from offset by n bytes for caching. + // If it's not implemented (default: `NotSupported`), RocksDB will create + // internal prefetch buffer to improve read performance. + virtual IOStatus Prefetch(uint64_t /*offset*/, size_t /*n*/, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + return IOStatus::NotSupported("Prefetch"); + } + + // Read a bunch of blocks as described by reqs. The blocks can + // optionally be read in parallel. This is a synchronous call, i.e it + // should return after all reads have completed. The reads will be + // non-overlapping but can be in any order. If the function return Status + // is not ok, status of individual requests will be ignored and return + // status will be assumed for all read requests. The function return status + // is only meant for errors that occur before processing individual read + // requests. + virtual IOStatus MultiRead(FSReadRequest* reqs, size_t num_reqs, + const IOOptions& options, IODebugContext* dbg) { + assert(reqs != nullptr); + for (size_t i = 0; i < num_reqs; ++i) { + FSReadRequest& req = reqs[i]; + req.status = + Read(req.offset, req.len, options, &req.result, req.scratch, dbg); + } + return IOStatus::OK(); + } + + // Tries to get an unique ID for this file that will be the same each time + // the file is opened (and will stay the same while the file is open). + // Furthermore, it tries to make this ID at most "max_size" bytes. If such an + // ID can be created this function returns the length of the ID and places it + // in "id"; otherwise, this function returns 0, in which case "id" + // may not have been modified. + // + // This function guarantees, for IDs from a given environment, two unique ids + // cannot be made equal to each other by adding arbitrary bytes to one of + // them. That is, no unique ID is the prefix of another. + // + // This function guarantees that the returned ID will not be interpretable as + // a single varint. + // + // Note: these IDs are only valid for the duration of the process. + virtual size_t GetUniqueId(char* /*id*/, size_t /*max_size*/) const { + return 0; // Default implementation to prevent issues with backwards + // compatibility. + }; + + enum AccessPattern { kNormal, kRandom, kSequential, kWillNeed, kWontNeed }; + + virtual void Hint(AccessPattern /*pattern*/) {} + + // Indicates the upper layers if the current RandomAccessFile implementation + // uses direct IO. + virtual bool use_direct_io() const { return false; } + + // Use the returned alignment value to allocate + // aligned buffer for Direct I/O + virtual size_t GetRequiredBufferAlignment() const { return kDefaultPageSize; } + + // Remove any kind of caching of data from the offset to offset+length + // of this file. If the length is 0, then it refers to the end of file. + // If the system is not caching the file contents, then this is a noop. + virtual IOStatus InvalidateCache(size_t /*offset*/, size_t /*length*/) { + return IOStatus::NotSupported("InvalidateCache not supported."); + } + + // This API reads the requested data in FSReadRequest asynchronously. This is + // a asynchronous call, i.e it should return after submitting the request. + // + // When the read request is completed, callback function specified in cb + // should be called with arguments cb_arg and the result populated in + // FSReadRequest with result and status fileds updated by FileSystem. + // cb_arg should be used by the callback to track the original request + // submitted. + // + // This API should also populate io_handle which should be used by + // underlying FileSystem to store the context in order to distinguish the read + // requests at their side and provide the custom deletion function in del_fn. + // RocksDB guarantees that the del_fn for io_handle will be called after + // receiving the callback. Furthermore, RocksDB guarantees that if it calls + // the Poll API for this io_handle, del_fn will be called after the Poll + // returns. RocksDB is responsible for managing the lifetime of io_handle. + // + // req contains the request offset and size passed as input parameter of read + // request and result and status fields are output parameter set by underlying + // FileSystem. The data should always be read into scratch field. + // + // How to enable: + // In order to enable ReadAsync, FS needs to override SupportedOps() API and + // set FSSupportedOps::kAsyncIO in SupportedOps() as: + // { + // supported_ops |= (1 << FSSupportedOps::kAsyncIO); + // } + // + // Note: If FS supports ReadAsync API, it should also override Poll and + // AbortIO API. + // + // Default implementation is to read the data synchronously. + virtual IOStatus ReadAsync( + FSReadRequest& req, const IOOptions& opts, + std::function cb, void* cb_arg, + void** /*io_handle*/, IOHandleDeleter* /*del_fn*/, IODebugContext* dbg) { + req.status = + Read(req.offset, req.len, opts, &(req.result), req.scratch, dbg); + cb(req, cb_arg); + return IOStatus::OK(); + } + + // EXPERIMENTAL + // When available, returns the actual temperature for the file. This is + // useful in case some outside process moves a file from one tier to another, + // though the temperature is generally expected not to change while a file is + // open. + virtual Temperature GetTemperature() const { return Temperature::kUnknown; } + + // If you're adding methods here, remember to add them to + // RandomAccessFileWrapper too. +}; + +// A data structure brings the data verification information, which is +// used together with data being written to a file. +struct DataVerificationInfo { + // checksum of the data being written. + Slice checksum; +}; + +// A file abstraction for sequential writing. The implementation +// must provide buffering since callers may append small fragments +// at a time to the file. +class FSWritableFile { + public: + FSWritableFile() + : last_preallocated_block_(0), + preallocation_block_size_(0), + io_priority_(Env::IO_TOTAL), + write_hint_(Env::WLTH_NOT_SET), + strict_bytes_per_sync_(false) {} + + explicit FSWritableFile(const FileOptions& options) + : last_preallocated_block_(0), + preallocation_block_size_(0), + io_priority_(Env::IO_TOTAL), + write_hint_(Env::WLTH_NOT_SET), + strict_bytes_per_sync_(options.strict_bytes_per_sync) {} + + // For cases when Close() hasn't been called, many derived classes of + // FSWritableFile will need to call Close() non-virtually in their destructor, + // and ignore the result, to ensure resources are released. + virtual ~FSWritableFile() {} + + // Append data to the end of the file + // Note: A WritableFile object must support either Append or + // PositionedAppend, so the users cannot mix the two. + virtual IOStatus Append(const Slice& data, const IOOptions& options, + IODebugContext* dbg) = 0; + + // Append data with verification information. + // Note that this API change is experimental and it might be changed in + // the future. Currently, RocksDB only generates crc32c based checksum for + // the file writes when the checksum handoff option is set. + // Expected behavior: if the handoff_checksum_type in FileOptions (currently, + // ChecksumType::kCRC32C is set as default) is not supported by this + // FSWritableFile, the information in DataVerificationInfo can be ignored + // (i.e. does not perform checksum verification). + virtual IOStatus Append(const Slice& data, const IOOptions& options, + const DataVerificationInfo& /* verification_info */, + IODebugContext* dbg) { + return Append(data, options, dbg); + } + + // PositionedAppend data to the specified offset. The new EOF after append + // must be larger than the previous EOF. This is to be used when writes are + // not backed by OS buffers and hence has to always start from the start of + // the sector. The implementation thus needs to also rewrite the last + // partial sector. + // Note: PositionAppend does not guarantee moving the file offset after the + // write. A WritableFile object must support either Append or + // PositionedAppend, so the users cannot mix the two. + // + // PositionedAppend() can only happen on the page/sector boundaries. For that + // reason, if the last write was an incomplete sector we still need to rewind + // back to the nearest sector/page and rewrite the portion of it with whatever + // we need to add. We need to keep where we stop writing. + // + // PositionedAppend() can only write whole sectors. For that reason we have to + // pad with zeros for the last write and trim the file when closing according + // to the position we keep in the previous step. + // + // PositionedAppend() requires aligned buffer to be passed in. The alignment + // required is queried via GetRequiredBufferAlignment() + virtual IOStatus PositionedAppend(const Slice& /* data */, + uint64_t /* offset */, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + return IOStatus::NotSupported("PositionedAppend"); + } + + // PositionedAppend data with verification information. + // Note that this API change is experimental and it might be changed in + // the future. Currently, RocksDB only generates crc32c based checksum for + // the file writes when the checksum handoff option is set. + // Expected behavior: if the handoff_checksum_type in FileOptions (currently, + // ChecksumType::kCRC32C is set as default) is not supported by this + // FSWritableFile, the information in DataVerificationInfo can be ignored + // (i.e. does not perform checksum verification). + virtual IOStatus PositionedAppend( + const Slice& /* data */, uint64_t /* offset */, + const IOOptions& /*options*/, + const DataVerificationInfo& /* verification_info */, + IODebugContext* /*dbg*/) { + return IOStatus::NotSupported("PositionedAppend"); + } + + // Truncate is necessary to trim the file to the correct size + // before closing. It is not always possible to keep track of the file + // size due to whole pages writes. The behavior is undefined if called + // with other writes to follow. + virtual IOStatus Truncate(uint64_t /*size*/, const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + return IOStatus::OK(); + } + + // The caller should call Close() before destroying the FSWritableFile to + // surface any errors associated with finishing writes to the file. + // The file is considered closed regardless of return status. + // (However, implementations must also clean up properly in the destructor + // even if Close() is not called.) + virtual IOStatus Close(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) = 0; + + virtual IOStatus Flush(const IOOptions& options, IODebugContext* dbg) = 0; + virtual IOStatus Sync(const IOOptions& options, + IODebugContext* dbg) = 0; // sync data + + /* + * Sync data and/or metadata as well. + * By default, sync only data. + * Override this method for environments where we need to sync + * metadata as well. + */ + virtual IOStatus Fsync(const IOOptions& options, IODebugContext* dbg) { + return Sync(options, dbg); + } + + // true if Sync() and Fsync() are safe to call concurrently with Append() + // and Flush(). + virtual bool IsSyncThreadSafe() const { return false; } + + // Indicates the upper layers if the current WritableFile implementation + // uses direct IO. + virtual bool use_direct_io() const { return false; } + + // Use the returned alignment value to allocate + // aligned buffer for Direct I/O + virtual size_t GetRequiredBufferAlignment() const { return kDefaultPageSize; } + + virtual void SetWriteLifeTimeHint(Env::WriteLifeTimeHint hint) { + write_hint_ = hint; + } + + /* + * If rate limiting is enabled, change the file-granularity priority used in + * rate-limiting writes. + * + * In the presence of finer-granularity priority such as + * `WriteOptions::rate_limiter_priority`, this file-granularity priority may + * be overridden by a non-Env::IO_TOTAL finer-granularity priority and used as + * a fallback for Env::IO_TOTAL finer-granularity priority. + * + * If rate limiting is not enabled, this call has no effect. + */ + virtual void SetIOPriority(Env::IOPriority pri) { io_priority_ = pri; } + + virtual Env::IOPriority GetIOPriority() { return io_priority_; } + + virtual Env::WriteLifeTimeHint GetWriteLifeTimeHint() { return write_hint_; } + /* + * Get the size of valid data in the file. + */ + virtual uint64_t GetFileSize(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + return 0; + } + + /* + * Get and set the default pre-allocation block size for writes to + * this file. If non-zero, then Allocate will be used to extend the + * underlying storage of a file (generally via fallocate) if the Env + * instance supports it. + */ + virtual void SetPreallocationBlockSize(size_t size) { + preallocation_block_size_ = size; + } + + virtual void GetPreallocationStatus(size_t* block_size, + size_t* last_allocated_block) { + *last_allocated_block = last_preallocated_block_; + *block_size = preallocation_block_size_; + } + + // For documentation, refer to RandomAccessFile::GetUniqueId() + virtual size_t GetUniqueId(char* /*id*/, size_t /*max_size*/) const { + return 0; // Default implementation to prevent issues with backwards + } + + // Remove any kind of caching of data from the offset to offset+length + // of this file. If the length is 0, then it refers to the end of file. + // If the system is not caching the file contents, then this is a noop. + // This call has no effect on dirty pages in the cache. + virtual IOStatus InvalidateCache(size_t /*offset*/, size_t /*length*/) { + return IOStatus::NotSupported("InvalidateCache not supported."); + } + + // Sync a file range with disk. + // offset is the starting byte of the file range to be synchronized. + // nbytes specifies the length of the range to be synchronized. + // This asks the OS to initiate flushing the cached data to disk, + // without waiting for completion. + // Default implementation does nothing. + virtual IOStatus RangeSync(uint64_t /*offset*/, uint64_t /*nbytes*/, + const IOOptions& options, IODebugContext* dbg) { + if (strict_bytes_per_sync_) { + return Sync(options, dbg); + } + return IOStatus::OK(); + } + + // PrepareWrite performs any necessary preparation for a write + // before the write actually occurs. This allows for pre-allocation + // of space on devices where it can result in less file + // fragmentation and/or less waste from over-zealous filesystem + // pre-allocation. + virtual void PrepareWrite(size_t offset, size_t len, const IOOptions& options, + IODebugContext* dbg) { + if (preallocation_block_size_ == 0) { + return; + } + // If this write would cross one or more preallocation blocks, + // determine what the last preallocation block necessary to + // cover this write would be and Allocate to that point. + const auto block_size = preallocation_block_size_; + size_t new_last_preallocated_block = + (offset + len + block_size - 1) / block_size; + if (new_last_preallocated_block > last_preallocated_block_) { + size_t num_spanned_blocks = + new_last_preallocated_block - last_preallocated_block_; + Allocate(block_size * last_preallocated_block_, + block_size * num_spanned_blocks, options, dbg) + .PermitUncheckedError(); + last_preallocated_block_ = new_last_preallocated_block; + } + } + + // Pre-allocates space for a file. + virtual IOStatus Allocate(uint64_t /*offset*/, uint64_t /*len*/, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + return IOStatus::OK(); + } + + // If you're adding methods here, remember to add them to + // WritableFileWrapper too. + + protected: + size_t preallocation_block_size() { return preallocation_block_size_; } + + private: + size_t last_preallocated_block_; + size_t preallocation_block_size_; + // No copying allowed + FSWritableFile(const FSWritableFile&); + void operator=(const FSWritableFile&); + + protected: + Env::IOPriority io_priority_; + Env::WriteLifeTimeHint write_hint_; + const bool strict_bytes_per_sync_; +}; + +// A file abstraction for random reading and writing. +class FSRandomRWFile { + public: + FSRandomRWFile() {} + + // For cases when Close() hasn't been called, many derived classes of + // FSRandomRWFile will need to call Close() non-virtually in their destructor, + // and ignore the result, to ensure resources are released. + virtual ~FSRandomRWFile() {} + + // Indicates if the class makes use of direct I/O + // If false you must pass aligned buffer to Write() + virtual bool use_direct_io() const { return false; } + + // Use the returned alignment value to allocate + // aligned buffer for Direct I/O + virtual size_t GetRequiredBufferAlignment() const { return kDefaultPageSize; } + + // Write bytes in `data` at offset `offset`, Returns Status::OK() on success. + // Pass aligned buffer when use_direct_io() returns true. + virtual IOStatus Write(uint64_t offset, const Slice& data, + const IOOptions& options, IODebugContext* dbg) = 0; + + // Read up to `n` bytes starting from offset `offset` and store them in + // result, provided `scratch` size should be at least `n`. + // + // After call, result->size() < n only if end of file has been + // reached (or non-OK status). Read might fail if called again after + // first result->size() < n. + // + // Returns Status::OK() on success. + virtual IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const = 0; + + virtual IOStatus Flush(const IOOptions& options, IODebugContext* dbg) = 0; + + virtual IOStatus Sync(const IOOptions& options, IODebugContext* dbg) = 0; + + virtual IOStatus Fsync(const IOOptions& options, IODebugContext* dbg) { + return Sync(options, dbg); + } + + // The caller should call Close() before destroying the FSRandomRWFile to + // surface any errors associated with finishing writes to the file. + // The file is considered closed regardless of return status. + // (However, implementations must also clean up properly in the destructor + // even if Close() is not called.) + virtual IOStatus Close(const IOOptions& options, IODebugContext* dbg) = 0; + + // EXPERIMENTAL + // When available, returns the actual temperature for the file. This is + // useful in case some outside process moves a file from one tier to another, + // though the temperature is generally expected not to change while a file is + // open. + virtual Temperature GetTemperature() const { return Temperature::kUnknown; } + + // If you're adding methods here, remember to add them to + // RandomRWFileWrapper too. + + // No copying allowed + FSRandomRWFile(const RandomRWFile&) = delete; + FSRandomRWFile& operator=(const RandomRWFile&) = delete; +}; + +// MemoryMappedFileBuffer object represents a memory-mapped file's raw buffer. +// Subclasses should release the mapping upon destruction. +class FSMemoryMappedFileBuffer { + public: + FSMemoryMappedFileBuffer(void* _base, size_t _length) + : base_(_base), length_(_length) {} + + virtual ~FSMemoryMappedFileBuffer() = 0; + + // We do not want to unmap this twice. We can make this class + // movable if desired, however, since + FSMemoryMappedFileBuffer(const FSMemoryMappedFileBuffer&) = delete; + FSMemoryMappedFileBuffer& operator=(const FSMemoryMappedFileBuffer&) = delete; + + void* GetBase() const { return base_; } + size_t GetLen() const { return length_; } + + protected: + void* base_; + const size_t length_; +}; + +// Directory object represents collection of files and implements +// filesystem operations that can be executed on directories. +class FSDirectory { + public: + // For cases when Close() hasn't been called, many derived classes of + // FSDirectory will need to call Close() non-virtually in their destructor, + // and ignore the result, to ensure resources are released. + virtual ~FSDirectory() {} + // Fsync directory. Can be called concurrently from multiple threads. + virtual IOStatus Fsync(const IOOptions& options, IODebugContext* dbg) = 0; + + // FsyncWithDirOptions after renaming a file. Depends on the filesystem, it + // may fsync directory or just the renaming file (e.g. btrfs). By default, it + // just calls directory fsync. + virtual IOStatus FsyncWithDirOptions( + const IOOptions& options, IODebugContext* dbg, + const DirFsyncOptions& /*dir_fsync_options*/) { + return Fsync(options, dbg); + } + + // Calling Close() before destroying a FSDirectory is recommended to surface + // any errors associated with finishing writes (in case of future features). + // The directory is considered closed regardless of return status. + virtual IOStatus Close(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + return IOStatus::NotSupported("Close"); + } + + virtual size_t GetUniqueId(char* /*id*/, size_t /*max_size*/) const { + return 0; + } + + // If you're adding methods here, remember to add them to + // DirectoryWrapper too. +}; + +// Below are helpers for wrapping most of the classes in this file. +// They forward all calls to another instance of the class. +// Useful when wrapping the default implementations. +// Typical usage is to inherit your wrapper from *Wrapper, e.g.: +// +// class MySequentialFileWrapper : public +// ROCKSDB_NAMESPACE::FSSequentialFileWrapper { +// public: +// MySequentialFileWrapper(ROCKSDB_NAMESPACE::FSSequentialFile* target): +// ROCKSDB_NAMESPACE::FSSequentialFileWrapper(target) {} +// Status Read(size_t n, FileSystem::IOOptions& options, Slice* result, +// char* scratch, FileSystem::IODebugContext* dbg) override { +// cout << "Doing a read of size " << n << "!" << endl; +// return ROCKSDB_NAMESPACE::FSSequentialFileWrapper::Read(n, options, +// result, +// scratch, dbg); +// } +// // All other methods are forwarded to target_ automatically. +// }; +// +// This is often more convenient than inheriting the class directly because +// (a) Don't have to override and forward all methods - the Wrapper will +// forward everything you're not explicitly overriding. +// (b) Don't need to update the wrapper when more methods are added to the +// rocksdb class. Unless you actually want to override the behavior. +// (And unless rocksdb people forgot to update the *Wrapper class.) + +// An implementation of Env that forwards all calls to another Env. +// May be useful to clients who wish to override just part of the +// functionality of another Env. +class FileSystemWrapper : public FileSystem { + public: + // Initialize an EnvWrapper that delegates all calls to *t + explicit FileSystemWrapper(const std::shared_ptr& t); + ~FileSystemWrapper() override {} + + // Return the target to which this Env forwards all calls + FileSystem* target() const { return target_.get(); } + + // The following text is boilerplate that forwards all methods to target() + IOStatus NewSequentialFile(const std::string& f, const FileOptions& file_opts, + std::unique_ptr* r, + IODebugContext* dbg) override { + return target_->NewSequentialFile(f, file_opts, r, dbg); + } + IOStatus NewRandomAccessFile(const std::string& f, + const FileOptions& file_opts, + std::unique_ptr* r, + IODebugContext* dbg) override { + return target_->NewRandomAccessFile(f, file_opts, r, dbg); + } + IOStatus NewWritableFile(const std::string& f, const FileOptions& file_opts, + std::unique_ptr* r, + IODebugContext* dbg) override { + return target_->NewWritableFile(f, file_opts, r, dbg); + } + IOStatus ReopenWritableFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override { + return target_->ReopenWritableFile(fname, file_opts, result, dbg); + } + IOStatus ReuseWritableFile(const std::string& fname, + const std::string& old_fname, + const FileOptions& file_opts, + std::unique_ptr* r, + IODebugContext* dbg) override { + return target_->ReuseWritableFile(fname, old_fname, file_opts, r, dbg); + } + IOStatus NewRandomRWFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override { + return target_->NewRandomRWFile(fname, file_opts, result, dbg); + } + IOStatus NewMemoryMappedFileBuffer( + const std::string& fname, + std::unique_ptr* result) override { + return target_->NewMemoryMappedFileBuffer(fname, result); + } + IOStatus NewDirectory(const std::string& name, const IOOptions& io_opts, + std::unique_ptr* result, + IODebugContext* dbg) override { + return target_->NewDirectory(name, io_opts, result, dbg); + } + IOStatus FileExists(const std::string& f, const IOOptions& io_opts, + IODebugContext* dbg) override { + return target_->FileExists(f, io_opts, dbg); + } + IOStatus GetChildren(const std::string& dir, const IOOptions& io_opts, + std::vector* r, + IODebugContext* dbg) override { + return target_->GetChildren(dir, io_opts, r, dbg); + } + IOStatus GetChildrenFileAttributes(const std::string& dir, + const IOOptions& options, + std::vector* result, + IODebugContext* dbg) override { + return target_->GetChildrenFileAttributes(dir, options, result, dbg); + } + IOStatus DeleteFile(const std::string& f, const IOOptions& options, + IODebugContext* dbg) override { + return target_->DeleteFile(f, options, dbg); + } + IOStatus Truncate(const std::string& fname, size_t size, + const IOOptions& options, IODebugContext* dbg) override { + return target_->Truncate(fname, size, options, dbg); + } + IOStatus CreateDir(const std::string& d, const IOOptions& options, + IODebugContext* dbg) override { + return target_->CreateDir(d, options, dbg); + } + IOStatus CreateDirIfMissing(const std::string& d, const IOOptions& options, + IODebugContext* dbg) override { + return target_->CreateDirIfMissing(d, options, dbg); + } + IOStatus DeleteDir(const std::string& d, const IOOptions& options, + IODebugContext* dbg) override { + return target_->DeleteDir(d, options, dbg); + } + IOStatus GetFileSize(const std::string& f, const IOOptions& options, + uint64_t* s, IODebugContext* dbg) override { + return target_->GetFileSize(f, options, s, dbg); + } + + IOStatus GetFileModificationTime(const std::string& fname, + const IOOptions& options, + uint64_t* file_mtime, + IODebugContext* dbg) override { + return target_->GetFileModificationTime(fname, options, file_mtime, dbg); + } + + IOStatus GetAbsolutePath(const std::string& db_path, const IOOptions& options, + std::string* output_path, + IODebugContext* dbg) override { + return target_->GetAbsolutePath(db_path, options, output_path, dbg); + } + + IOStatus RenameFile(const std::string& s, const std::string& t, + const IOOptions& options, IODebugContext* dbg) override { + return target_->RenameFile(s, t, options, dbg); + } + + IOStatus LinkFile(const std::string& s, const std::string& t, + const IOOptions& options, IODebugContext* dbg) override { + return target_->LinkFile(s, t, options, dbg); + } + + IOStatus NumFileLinks(const std::string& fname, const IOOptions& options, + uint64_t* count, IODebugContext* dbg) override { + return target_->NumFileLinks(fname, options, count, dbg); + } + + IOStatus AreFilesSame(const std::string& first, const std::string& second, + const IOOptions& options, bool* res, + IODebugContext* dbg) override { + return target_->AreFilesSame(first, second, options, res, dbg); + } + + IOStatus LockFile(const std::string& f, const IOOptions& options, + FileLock** l, IODebugContext* dbg) override { + return target_->LockFile(f, options, l, dbg); + } + + IOStatus UnlockFile(FileLock* l, const IOOptions& options, + IODebugContext* dbg) override { + return target_->UnlockFile(l, options, dbg); + } + + IOStatus GetTestDirectory(const IOOptions& options, std::string* path, + IODebugContext* dbg) override { + return target_->GetTestDirectory(options, path, dbg); + } + IOStatus NewLogger(const std::string& fname, const IOOptions& options, + std::shared_ptr* result, + IODebugContext* dbg) override { + return target_->NewLogger(fname, options, result, dbg); + } + + void SanitizeFileOptions(FileOptions* opts) const override { + target_->SanitizeFileOptions(opts); + } + + FileOptions OptimizeForLogRead( + const FileOptions& file_options) const override { + return target_->OptimizeForLogRead(file_options); + } + FileOptions OptimizeForManifestRead( + const FileOptions& file_options) const override { + return target_->OptimizeForManifestRead(file_options); + } + FileOptions OptimizeForLogWrite(const FileOptions& file_options, + const DBOptions& db_options) const override { + return target_->OptimizeForLogWrite(file_options, db_options); + } + FileOptions OptimizeForManifestWrite( + const FileOptions& file_options) const override { + return target_->OptimizeForManifestWrite(file_options); + } + FileOptions OptimizeForCompactionTableWrite( + const FileOptions& file_options, + const ImmutableDBOptions& immutable_ops) const override { + return target_->OptimizeForCompactionTableWrite(file_options, + immutable_ops); + } + FileOptions OptimizeForCompactionTableRead( + const FileOptions& file_options, + const ImmutableDBOptions& db_options) const override { + return target_->OptimizeForCompactionTableRead(file_options, db_options); + } + FileOptions OptimizeForBlobFileRead( + const FileOptions& file_options, + const ImmutableDBOptions& db_options) const override { + return target_->OptimizeForBlobFileRead(file_options, db_options); + } + IOStatus GetFreeSpace(const std::string& path, const IOOptions& options, + uint64_t* diskfree, IODebugContext* dbg) override { + return target_->GetFreeSpace(path, options, diskfree, dbg); + } + IOStatus IsDirectory(const std::string& path, const IOOptions& options, + bool* is_dir, IODebugContext* dbg) override { + return target_->IsDirectory(path, options, is_dir, dbg); + } + + const Customizable* Inner() const override { return target_.get(); } + Status PrepareOptions(const ConfigOptions& options) override; + std::string SerializeOptions(const ConfigOptions& config_options, + const std::string& header) const override; + + virtual IOStatus Poll(std::vector& io_handles, + size_t min_completions) override { + return target_->Poll(io_handles, min_completions); + } + + virtual IOStatus AbortIO(std::vector& io_handles) override { + return target_->AbortIO(io_handles); + } + + virtual void SupportedOps(int64_t& supported_ops) override { + return target_->SupportedOps(supported_ops); + } + + protected: + std::shared_ptr target_; +}; + +class FSSequentialFileWrapper : public FSSequentialFile { + public: + // Creates a FileWrapper around the input File object and without + // taking ownership of the object + explicit FSSequentialFileWrapper(FSSequentialFile* t) : target_(t) {} + + FSSequentialFile* target() const { return target_; } + + IOStatus Read(size_t n, const IOOptions& options, Slice* result, + char* scratch, IODebugContext* dbg) override { + return target_->Read(n, options, result, scratch, dbg); + } + IOStatus Skip(uint64_t n) override { return target_->Skip(n); } + bool use_direct_io() const override { return target_->use_direct_io(); } + size_t GetRequiredBufferAlignment() const override { + return target_->GetRequiredBufferAlignment(); + } + IOStatus InvalidateCache(size_t offset, size_t length) override { + return target_->InvalidateCache(offset, length); + } + IOStatus PositionedRead(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) override { + return target_->PositionedRead(offset, n, options, result, scratch, dbg); + } + Temperature GetTemperature() const override { + return target_->GetTemperature(); + } + + private: + FSSequentialFile* target_; +}; + +class FSSequentialFileOwnerWrapper : public FSSequentialFileWrapper { + public: + // Creates a FileWrapper around the input File object and takes + // ownership of the object + explicit FSSequentialFileOwnerWrapper(std::unique_ptr&& t) + : FSSequentialFileWrapper(t.get()), guard_(std::move(t)) {} + + private: + std::unique_ptr guard_; +}; + +class FSRandomAccessFileWrapper : public FSRandomAccessFile { + public: + // Creates a FileWrapper around the input File object and without + // taking ownership of the object + explicit FSRandomAccessFileWrapper(FSRandomAccessFile* t) : target_(t) {} + + FSRandomAccessFile* target() const { return target_; } + + IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const override { + return target_->Read(offset, n, options, result, scratch, dbg); + } + IOStatus MultiRead(FSReadRequest* reqs, size_t num_reqs, + const IOOptions& options, IODebugContext* dbg) override { + return target_->MultiRead(reqs, num_reqs, options, dbg); + } + IOStatus Prefetch(uint64_t offset, size_t n, const IOOptions& options, + IODebugContext* dbg) override { + return target_->Prefetch(offset, n, options, dbg); + } + size_t GetUniqueId(char* id, size_t max_size) const override { + return target_->GetUniqueId(id, max_size); + }; + void Hint(AccessPattern pattern) override { target_->Hint(pattern); } + bool use_direct_io() const override { return target_->use_direct_io(); } + size_t GetRequiredBufferAlignment() const override { + return target_->GetRequiredBufferAlignment(); + } + IOStatus InvalidateCache(size_t offset, size_t length) override { + return target_->InvalidateCache(offset, length); + } + IOStatus ReadAsync(FSReadRequest& req, const IOOptions& opts, + std::function cb, + void* cb_arg, void** io_handle, IOHandleDeleter* del_fn, + IODebugContext* dbg) override { + return target()->ReadAsync(req, opts, cb, cb_arg, io_handle, del_fn, dbg); + } + Temperature GetTemperature() const override { + return target_->GetTemperature(); + } + + private: + std::unique_ptr guard_; + FSRandomAccessFile* target_; +}; + +class FSRandomAccessFileOwnerWrapper : public FSRandomAccessFileWrapper { + public: + // Creates a FileWrapper around the input File object and takes + // ownership of the object + explicit FSRandomAccessFileOwnerWrapper( + std::unique_ptr&& t) + : FSRandomAccessFileWrapper(t.get()), guard_(std::move(t)) {} + + private: + std::unique_ptr guard_; +}; + +class FSWritableFileWrapper : public FSWritableFile { + public: + // Creates a FileWrapper around the input File object and without + // taking ownership of the object + explicit FSWritableFileWrapper(FSWritableFile* t) : target_(t) {} + + FSWritableFile* target() const { return target_; } + + IOStatus Append(const Slice& data, const IOOptions& options, + IODebugContext* dbg) override { + return target_->Append(data, options, dbg); + } + IOStatus Append(const Slice& data, const IOOptions& options, + const DataVerificationInfo& verification_info, + IODebugContext* dbg) override { + return target_->Append(data, options, verification_info, dbg); + } + IOStatus PositionedAppend(const Slice& data, uint64_t offset, + const IOOptions& options, + IODebugContext* dbg) override { + return target_->PositionedAppend(data, offset, options, dbg); + } + IOStatus PositionedAppend(const Slice& data, uint64_t offset, + const IOOptions& options, + const DataVerificationInfo& verification_info, + IODebugContext* dbg) override { + return target_->PositionedAppend(data, offset, options, verification_info, + dbg); + } + IOStatus Truncate(uint64_t size, const IOOptions& options, + IODebugContext* dbg) override { + return target_->Truncate(size, options, dbg); + } + IOStatus Close(const IOOptions& options, IODebugContext* dbg) override { + return target_->Close(options, dbg); + } + IOStatus Flush(const IOOptions& options, IODebugContext* dbg) override { + return target_->Flush(options, dbg); + } + IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override { + return target_->Sync(options, dbg); + } + IOStatus Fsync(const IOOptions& options, IODebugContext* dbg) override { + return target_->Fsync(options, dbg); + } + bool IsSyncThreadSafe() const override { return target_->IsSyncThreadSafe(); } + + bool use_direct_io() const override { return target_->use_direct_io(); } + + size_t GetRequiredBufferAlignment() const override { + return target_->GetRequiredBufferAlignment(); + } + + void SetWriteLifeTimeHint(Env::WriteLifeTimeHint hint) override { + target_->SetWriteLifeTimeHint(hint); + } + + Env::WriteLifeTimeHint GetWriteLifeTimeHint() override { + return target_->GetWriteLifeTimeHint(); + } + + uint64_t GetFileSize(const IOOptions& options, IODebugContext* dbg) override { + return target_->GetFileSize(options, dbg); + } + + void SetPreallocationBlockSize(size_t size) override { + target_->SetPreallocationBlockSize(size); + } + + void GetPreallocationStatus(size_t* block_size, + size_t* last_allocated_block) override { + target_->GetPreallocationStatus(block_size, last_allocated_block); + } + + size_t GetUniqueId(char* id, size_t max_size) const override { + return target_->GetUniqueId(id, max_size); + } + + IOStatus InvalidateCache(size_t offset, size_t length) override { + return target_->InvalidateCache(offset, length); + } + + IOStatus RangeSync(uint64_t offset, uint64_t nbytes, const IOOptions& options, + IODebugContext* dbg) override { + return target_->RangeSync(offset, nbytes, options, dbg); + } + + void PrepareWrite(size_t offset, size_t len, const IOOptions& options, + IODebugContext* dbg) override { + target_->PrepareWrite(offset, len, options, dbg); + } + + IOStatus Allocate(uint64_t offset, uint64_t len, const IOOptions& options, + IODebugContext* dbg) override { + return target_->Allocate(offset, len, options, dbg); + } + + private: + FSWritableFile* target_; +}; + +class FSWritableFileOwnerWrapper : public FSWritableFileWrapper { + public: + // Creates a FileWrapper around the input File object and takes + // ownership of the object + explicit FSWritableFileOwnerWrapper(std::unique_ptr&& t) + : FSWritableFileWrapper(t.get()), guard_(std::move(t)) {} + + private: + std::unique_ptr guard_; +}; + +class FSRandomRWFileWrapper : public FSRandomRWFile { + public: + // Creates a FileWrapper around the input File object and without + // taking ownership of the object + explicit FSRandomRWFileWrapper(FSRandomRWFile* t) : target_(t) {} + + FSRandomRWFile* target() const { return target_; } + + bool use_direct_io() const override { return target_->use_direct_io(); } + size_t GetRequiredBufferAlignment() const override { + return target_->GetRequiredBufferAlignment(); + } + IOStatus Write(uint64_t offset, const Slice& data, const IOOptions& options, + IODebugContext* dbg) override { + return target_->Write(offset, data, options, dbg); + } + IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const override { + return target_->Read(offset, n, options, result, scratch, dbg); + } + IOStatus Flush(const IOOptions& options, IODebugContext* dbg) override { + return target_->Flush(options, dbg); + } + IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override { + return target_->Sync(options, dbg); + } + IOStatus Fsync(const IOOptions& options, IODebugContext* dbg) override { + return target_->Fsync(options, dbg); + } + IOStatus Close(const IOOptions& options, IODebugContext* dbg) override { + return target_->Close(options, dbg); + } + Temperature GetTemperature() const override { + return target_->GetTemperature(); + } + + private: + FSRandomRWFile* target_; +}; + +class FSRandomRWFileOwnerWrapper : public FSRandomRWFileWrapper { + public: + // Creates a FileWrapper around the input File object and takes + // ownership of the object + explicit FSRandomRWFileOwnerWrapper(std::unique_ptr&& t) + : FSRandomRWFileWrapper(t.get()), guard_(std::move(t)) {} + + private: + std::unique_ptr guard_; +}; + +class FSDirectoryWrapper : public FSDirectory { + public: + // Creates a FileWrapper around the input File object and takes + // ownership of the object + explicit FSDirectoryWrapper(std::unique_ptr&& t) + : guard_(std::move(t)) { + target_ = guard_.get(); + } + + // Creates a FileWrapper around the input File object and without + // taking ownership of the object + explicit FSDirectoryWrapper(FSDirectory* t) : target_(t) {} + + IOStatus Fsync(const IOOptions& options, IODebugContext* dbg) override { + return target_->Fsync(options, dbg); + } + + IOStatus FsyncWithDirOptions( + const IOOptions& options, IODebugContext* dbg, + const DirFsyncOptions& dir_fsync_options) override { + return target_->FsyncWithDirOptions(options, dbg, dir_fsync_options); + } + + IOStatus Close(const IOOptions& options, IODebugContext* dbg) override { + return target_->Close(options, dbg); + } + + size_t GetUniqueId(char* id, size_t max_size) const override { + return target_->GetUniqueId(id, max_size); + } + + private: + std::unique_ptr guard_; + FSDirectory* target_; +}; + +// A utility routine: write "data" to the named file. +extern IOStatus WriteStringToFile(FileSystem* fs, const Slice& data, + const std::string& fname, + bool should_sync = false); + +// A utility routine: read contents of named file into *data +extern IOStatus ReadFileToString(FileSystem* fs, const std::string& fname, + std::string* data); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/filter_policy.h b/librocksdb-sys/rocksdb/include/rocksdb/filter_policy.h new file mode 100644 index 0000000..954d15b --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/filter_policy.h @@ -0,0 +1,206 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// A database can be configured with a custom FilterPolicy object. +// This object is responsible for creating a small filter from a set +// of keys. These filters are stored in rocksdb and are consulted +// automatically by rocksdb to decide whether or not to read some +// information from disk. In many cases, a filter can cut down the +// number of disk seeks form a handful to a single disk seek per +// DB::Get() call. +// +// Most people will want to use the builtin bloom filter support (see +// NewBloomFilterPolicy() below). + +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include "rocksdb/advanced_options.h" +#include "rocksdb/customizable.h" +#include "rocksdb/status.h" +#include "rocksdb/types.h" + +namespace ROCKSDB_NAMESPACE { + +class Slice; +struct BlockBasedTableOptions; +struct ConfigOptions; + +// As of RocksDB 7.0, the details of these classes are internal +class FilterBitsBuilder; +class FilterBitsReader; + +// Contextual information passed to BloomFilterPolicy at filter building time. +// Used in overriding FilterPolicy::GetBuilderWithContext(). References other +// structs because this is expected to be a temporary, stack-allocated object. +struct FilterBuildingContext { + // This constructor is for internal use only and subject to change. + FilterBuildingContext(const BlockBasedTableOptions& table_options); + + // Options for the table being built + const BlockBasedTableOptions& table_options; + + // BEGIN from (DB|ColumnFamily)Options in effect at table creation time + CompactionStyle compaction_style = kCompactionStyleLevel; + + // Number of LSM levels, or -1 if unknown + int num_levels = -1; + + // An optional logger for reporting errors, warnings, etc. + Logger* info_log = nullptr; + // END from (DB|ColumnFamily)Options + + // Name of the column family for the table (or empty string if unknown) + // TODO: consider changing to Slice + std::string column_family_name; + + // The table level at time of constructing the SST file, or -1 if unknown + // or N/A as in SstFileWriter. (The table file could later be used at a + // different level.) + int level_at_creation = -1; + + // True if known to be going into bottommost sorted run for applicable + // key range (which might not even be last level with data). False + // otherwise. + bool is_bottommost = false; + + // Reason for creating the file with the filter + TableFileCreationReason reason = TableFileCreationReason::kMisc; +}; + +// Determines what kind of filter (if any) to generate in SST files, and under +// which conditions. API users can create custom filter policies that +// defer to other built-in policies (see NewBloomFilterPolicy and +// NewRibbonFilterPolicy) based on the context provided to +// GetBuilderWithContext. +class FilterPolicy : public Customizable { + public: + virtual ~FilterPolicy(); + static const char* Type() { return "FilterPolicy"; } + + // The name used for identifying whether a filter on disk is readable + // by this FilterPolicy. If this FilterPolicy is part of a family that + // can read each others filters, such as built-in BloomFilterPolcy and + // RibbonFilterPolicy, the CompatibilityName is a shared family name, + // while kinds of filters in the family can have distinct Customizable + // Names. This function is pure virtual so that wrappers around built-in + // policies are prompted to defer to CompatibilityName() of the wrapped + // policy, which is important for compatibility. + // + // For custom filter policies that are not part of a read-compatible + // family (rare), implementations may return Name(). + virtual const char* CompatibilityName() const = 0; + + // Creates a new FilterPolicy based on the input value string and returns the + // result The value might be an ID, and ID with properties, or an old-style + // policy string. + // The value describes the FilterPolicy being created. + // For BloomFilters, value may be a ":"-delimited value of the form: + // "bloomfilter:[bits_per_key]", + // e.g. ""bloomfilter:4" + // The above string is equivalent to calling NewBloomFilterPolicy(4). + static Status CreateFromString(const ConfigOptions& config_options, + const std::string& value, + std::shared_ptr* result); + + // Return a new FilterBitsBuilder for constructing full or partitioned + // filter blocks, or return nullptr to indicate "no filter". Custom + // implementations should defer to a built-in FilterPolicy to get a + // new FilterBitsBuilder, but the FilterBuildingContext can be used + // to decide which built-in FilterPolicy to defer to. + virtual FilterBitsBuilder* GetBuilderWithContext( + const FilterBuildingContext&) const = 0; + + // Return a new FilterBitsReader for full or partitioned filter blocks. + // Caller retains ownership of any buffer pointed to by the input Slice. + // Custom implementation should defer to GetFilterBitsReader on any + // built-in FilterPolicy, which can read filters generated by any other + // built-in FilterPolicy. + virtual FilterBitsReader* GetFilterBitsReader( + const Slice& /*contents*/) const = 0; +}; + +// Return a new filter policy that uses a bloom filter with approximately +// the specified number of bits per key. See +// https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter +// +// bits_per_key: average bits allocated per key in bloom filter. A good +// choice is 9.9, which yields a filter with ~ 1% false positive rate. +// When format_version < 5, the value will be rounded to the nearest +// integer. Recommend using no more than three decimal digits after the +// decimal point, as in 6.667. +// +// To avoid configurations that are unlikely to produce good filtering +// value for the CPU overhead, bits_per_key < 0.5 is rounded down to 0.0 +// which means "generate no filter", and 0.5 <= bits_per_key < 1.0 is +// rounded up to 1.0, for a 62% FP rate. +// +// The caller is responsible for eventually deleting the result, though +// this is typically handled automatically with BlockBasedTableOptions: +// table_options.filter_policy.reset(NewBloomFilterPolicy(...)); +// +// As of RocksDB 7.0, the use_block_based_builder parameter is ignored. +// (The old, inefficient block-based filter is no longer accessible in +// the public API.) +// +// Note: if you are using a custom comparator that ignores some parts +// of the keys being compared, you must not use NewBloomFilterPolicy() +// and must provide your own FilterPolicy that also ignores the +// corresponding parts of the keys. For example, if the comparator +// ignores trailing spaces, it would be incorrect to use a +// FilterPolicy (like NewBloomFilterPolicy) that does not ignore +// trailing spaces in keys. +extern const FilterPolicy* NewBloomFilterPolicy( + double bits_per_key, bool IGNORED_use_block_based_builder = false); + +// A new Bloom alternative that saves about 30% space compared to +// Bloom filters, with similar query times but roughly 3-4x CPU time +// and 3x temporary space usage during construction. For example, if +// you pass in 10 for bloom_equivalent_bits_per_key, you'll get the same +// 0.95% FP rate as Bloom filter but only using about 7 bits per key. +// +// The space savings of Ribbon filters makes sense for lower (higher +// numbered; larger; longer-lived) levels of LSM, whereas the speed of +// Bloom filters make sense for highest levels of LSM. Setting +// bloom_before_level allows for this design with Level and Universal +// compaction styles. For example, bloom_before_level=1 means that Bloom +// filters will be used in level 0, including flushes, and Ribbon +// filters elsewhere, including FIFO compaction and external SST files. +// For this option, memtable flushes are considered level -1 (so that +// flushes can be distinguished from intra-L0 compaction). +// bloom_before_level=0 (default) -> Generate Bloom filters only for +// flushes under Level and Universal compaction styles. +// bloom_before_level=-1 -> Always generate Ribbon filters (except in +// some extreme or exceptional cases). +// +// Ribbon filters are compatible with RocksDB >= 6.15.0. Earlier +// versions reading the data will behave as if no filter was used +// (degraded performance until compaction rebuilds filters). All +// built-in FilterPolicies (Bloom or Ribbon) are able to read other +// kinds of built-in filters. +// +// Note: the current Ribbon filter schema uses some extra resources +// when constructing very large filters. For example, for 100 million +// keys in a single filter (one SST file without partitioned filters), +// 3GB of temporary, untracked memory is used, vs. 1GB for Bloom. +// However, the savings in filter space from just ~60 open SST files +// makes up for the additional temporary memory use. +// +// Also consider using optimize_filters_for_memory to save filter +// memory. +extern const FilterPolicy* NewRibbonFilterPolicy( + double bloom_equivalent_bits_per_key, int bloom_before_level = 0); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/flush_block_policy.h b/librocksdb-sys/rocksdb/include/rocksdb/flush_block_policy.h new file mode 100644 index 0000000..7a5dd95 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/flush_block_policy.h @@ -0,0 +1,75 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "rocksdb/customizable.h" +#include "rocksdb/table.h" + +namespace ROCKSDB_NAMESPACE { + +class Slice; +class BlockBuilder; +struct ConfigOptions; +struct Options; + +// FlushBlockPolicy provides a configurable way to determine when to flush a +// block in the block based tables. +// +// Exceptions MUST NOT propagate out of overridden functions into RocksDB, +// because RocksDB is not exception-safe. This could cause undefined behavior +// including data loss, unreported corruption, deadlocks, and more. +class FlushBlockPolicy { + public: + // Keep track of the key/value sequences and return the boolean value to + // determine if table builder should flush current data block. + virtual bool Update(const Slice& key, const Slice& value) = 0; + + virtual ~FlushBlockPolicy() {} +}; + +class FlushBlockPolicyFactory : public Customizable { + public: + static const char* Type() { return "FlushBlockPolicyFactory"; } + + // Creates a FlushBlockPolicyFactory based on the input value. + // By default, this method can create EveryKey or BySize PolicyFactory, + // which take now config_options. + static Status CreateFromString( + const ConfigOptions& config_options, const std::string& value, + std::shared_ptr* result); + + // Return a new block flush policy that flushes data blocks by data size. + // FlushBlockPolicy may need to access the metadata of the data block + // builder to determine when to flush the blocks. + // + // Callers must delete the result after any database that is using the + // result has been closed. + virtual FlushBlockPolicy* NewFlushBlockPolicy( + const BlockBasedTableOptions& table_options, + const BlockBuilder& data_block_builder) const = 0; + + virtual ~FlushBlockPolicyFactory() {} +}; + +class FlushBlockBySizePolicyFactory : public FlushBlockPolicyFactory { + public: + FlushBlockBySizePolicyFactory(); + + static const char* kClassName() { return "FlushBlockBySizePolicyFactory"; } + const char* Name() const override { return kClassName(); } + + FlushBlockPolicy* NewFlushBlockPolicy( + const BlockBasedTableOptions& table_options, + const BlockBuilder& data_block_builder) const override; + + static FlushBlockPolicy* NewFlushBlockPolicy( + const uint64_t size, const int deviation, + const BlockBuilder& data_block_builder); +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/functor_wrapper.h b/librocksdb-sys/rocksdb/include/rocksdb/functor_wrapper.h new file mode 100644 index 0000000..17b021b --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/functor_wrapper.h @@ -0,0 +1,56 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +namespace detail { +template +struct IndexSequence {}; + +template +struct IndexSequenceHelper + : public IndexSequenceHelper {}; + +template +struct IndexSequenceHelper<0U, Next...> { + using type = IndexSequence; +}; + +template +using make_index_sequence = typename IndexSequenceHelper::type; + +template +void call(Function f, Tuple t, IndexSequence) { + f(std::get(t)...); +} + +template +void call(Function f, Tuple t) { + static constexpr auto size = std::tuple_size::value; + call(f, t, make_index_sequence{}); +} +} // namespace detail + +template +class FunctorWrapper { + public: + explicit FunctorWrapper(std::function functor, Args &&...args) + : functor_(std::move(functor)), args_(std::forward(args)...) {} + + void invoke() { detail::call(functor_, args_); } + + private: + std::function functor_; + std::tuple args_; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/io_status.h b/librocksdb-sys/rocksdb/include/rocksdb/io_status.h new file mode 100644 index 0000000..0bf5e93 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/io_status.h @@ -0,0 +1,244 @@ +// Copyright (c) 2019-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// An IOStatus encapsulates the result of an operation. It may indicate +// success, or it may indicate an error with an associated error message. +// +// Multiple threads can invoke const methods on an IOStatus without +// external synchronization, but if any of the threads may call a +// non-const method, all threads accessing the same IOStatus must use +// external synchronization. + +#pragma once + +#include + +#include "rocksdb/slice.h" +#ifdef OS_WIN +#include +#endif +#include + +#include "status.h" + +namespace ROCKSDB_NAMESPACE { + +class IOStatus : public Status { + public: + using Code = Status::Code; + using SubCode = Status::SubCode; + + enum IOErrorScope : unsigned char { + kIOErrorScopeFileSystem, + kIOErrorScopeFile, + kIOErrorScopeRange, + kIOErrorScopeMax, + }; + + // Create a success status. + IOStatus() : IOStatus(kOk, kNone) {} + ~IOStatus() {} + + // Copy the specified status. + IOStatus(const IOStatus& s); + IOStatus& operator=(const IOStatus& s); + IOStatus(IOStatus&& s) noexcept; + IOStatus& operator=(IOStatus&& s) noexcept; + bool operator==(const IOStatus& rhs) const; + bool operator!=(const IOStatus& rhs) const; + + void SetRetryable(bool retryable) { retryable_ = retryable; } + void SetDataLoss(bool data_loss) { data_loss_ = data_loss; } + void SetScope(IOErrorScope scope) { + scope_ = static_cast(scope); + } + + bool GetRetryable() const { return retryable_; } + bool GetDataLoss() const { return data_loss_; } + IOErrorScope GetScope() const { return static_cast(scope_); } + + // Return a success status. + static IOStatus OK() { return IOStatus(); } + + static IOStatus NotSupported(const Slice& msg, const Slice& msg2 = Slice()) { + return IOStatus(kNotSupported, msg, msg2); + } + static IOStatus NotSupported(SubCode msg = kNone) { + return IOStatus(kNotSupported, msg); + } + + // Return error status of an appropriate type. + static IOStatus NotFound(const Slice& msg, const Slice& msg2 = Slice()) { + return IOStatus(kNotFound, msg, msg2); + } + // Fast path for not found without malloc; + static IOStatus NotFound(SubCode msg = kNone) { + return IOStatus(kNotFound, msg); + } + + static IOStatus Corruption(const Slice& msg, const Slice& msg2 = Slice()) { + return IOStatus(kCorruption, msg, msg2); + } + static IOStatus Corruption(SubCode msg = kNone) { + return IOStatus(kCorruption, msg); + } + + static IOStatus InvalidArgument(const Slice& msg, + const Slice& msg2 = Slice()) { + return IOStatus(kInvalidArgument, msg, msg2); + } + static IOStatus InvalidArgument(SubCode msg = kNone) { + return IOStatus(kInvalidArgument, msg); + } + + static IOStatus IOError(const Slice& msg, const Slice& msg2 = Slice()) { + return IOStatus(kIOError, msg, msg2); + } + static IOStatus IOError(SubCode msg = kNone) { + return IOStatus(kIOError, msg); + } + + static IOStatus Busy(SubCode msg = kNone) { return IOStatus(kBusy, msg); } + static IOStatus Busy(const Slice& msg, const Slice& msg2 = Slice()) { + return IOStatus(kBusy, msg, msg2); + } + + static IOStatus TimedOut(SubCode msg = kNone) { + return IOStatus(kTimedOut, msg); + } + static IOStatus TimedOut(const Slice& msg, const Slice& msg2 = Slice()) { + return IOStatus(kTimedOut, msg, msg2); + } + + static IOStatus NoSpace() { return IOStatus(kIOError, kNoSpace); } + static IOStatus NoSpace(const Slice& msg, const Slice& msg2 = Slice()) { + return IOStatus(kIOError, kNoSpace, msg, msg2); + } + + static IOStatus PathNotFound() { return IOStatus(kIOError, kPathNotFound); } + static IOStatus PathNotFound(const Slice& msg, const Slice& msg2 = Slice()) { + return IOStatus(kIOError, kPathNotFound, msg, msg2); + } + + static IOStatus IOFenced() { return IOStatus(kIOError, kIOFenced); } + static IOStatus IOFenced(const Slice& msg, const Slice& msg2 = Slice()) { + return IOStatus(kIOError, kIOFenced, msg, msg2); + } + + static IOStatus Aborted(SubCode msg = kNone) { + return IOStatus(kAborted, msg); + } + static IOStatus Aborted(const Slice& msg, const Slice& msg2 = Slice()) { + return IOStatus(kAborted, msg, msg2); + } + + // Return a string representation of this status suitable for printing. + // Returns the string "OK" for success. + // std::string ToString() const; + + private: + friend IOStatus status_to_io_status(Status&&); + + explicit IOStatus(Code _code, SubCode _subcode = kNone) + : Status(_code, _subcode, false, false, kIOErrorScopeFileSystem) {} + + IOStatus(Code _code, SubCode _subcode, const Slice& msg, const Slice& msg2); + IOStatus(Code _code, const Slice& msg, const Slice& msg2) + : IOStatus(_code, kNone, msg, msg2) {} +}; + +inline IOStatus::IOStatus(Code _code, SubCode _subcode, const Slice& msg, + const Slice& msg2) + : Status(_code, _subcode, false, false, kIOErrorScopeFileSystem) { + assert(code_ != kOk); + assert(subcode_ != kMaxSubCode); + const size_t len1 = msg.size(); + const size_t len2 = msg2.size(); + const size_t size = len1 + (len2 ? (2 + len2) : 0); + char* const result = new char[size + 1]; // +1 for null terminator + memcpy(result, msg.data(), len1); + if (len2) { + result[len1] = ':'; + result[len1 + 1] = ' '; + memcpy(result + len1 + 2, msg2.data(), len2); + } + result[size] = '\0'; // null terminator for C style string + state_.reset(result); +} + +inline IOStatus::IOStatus(const IOStatus& s) : Status(s.code_, s.subcode_) { +#ifdef ROCKSDB_ASSERT_STATUS_CHECKED + s.checked_ = true; +#endif // ROCKSDB_ASSERT_STATUS_CHECKED + retryable_ = s.retryable_; + data_loss_ = s.data_loss_; + scope_ = s.scope_; + state_ = (s.state_ == nullptr) ? nullptr : CopyState(s.state_.get()); +} +inline IOStatus& IOStatus::operator=(const IOStatus& s) { + // The following condition catches both aliasing (when this == &s), + // and the common case where both s and *this are ok. + if (this != &s) { +#ifdef ROCKSDB_ASSERT_STATUS_CHECKED + s.checked_ = true; + checked_ = false; +#endif // ROCKSDB_ASSERT_STATUS_CHECKED + code_ = s.code_; + subcode_ = s.subcode_; + retryable_ = s.retryable_; + data_loss_ = s.data_loss_; + scope_ = s.scope_; + state_ = (s.state_ == nullptr) ? nullptr : CopyState(s.state_.get()); + } + return *this; +} + +inline IOStatus::IOStatus(IOStatus&& s) noexcept : IOStatus() { + *this = std::move(s); +} + +inline IOStatus& IOStatus::operator=(IOStatus&& s) noexcept { + if (this != &s) { +#ifdef ROCKSDB_ASSERT_STATUS_CHECKED + s.checked_ = true; + checked_ = false; +#endif // ROCKSDB_ASSERT_STATUS_CHECKED + code_ = std::move(s.code_); + s.code_ = kOk; + subcode_ = std::move(s.subcode_); + s.subcode_ = kNone; + retryable_ = s.retryable_; + data_loss_ = s.data_loss_; + scope_ = s.scope_; + s.scope_ = kIOErrorScopeFileSystem; + state_ = std::move(s.state_); + } + return *this; +} + +inline bool IOStatus::operator==(const IOStatus& rhs) const { +#ifdef ROCKSDB_ASSERT_STATUS_CHECKED + checked_ = true; + rhs.checked_ = true; +#endif // ROCKSDB_ASSERT_STATUS_CHECKED + return (code_ == rhs.code_); +} + +inline bool IOStatus::operator!=(const IOStatus& rhs) const { +#ifdef ROCKSDB_ASSERT_STATUS_CHECKED + checked_ = true; + rhs.checked_ = true; +#endif // ROCKSDB_ASSERT_STATUS_CHECKED + return !(*this == rhs); +} + +inline IOStatus status_to_io_status(Status&& status) { + IOStatus io_s; + Status& s = io_s; + s = std::move(status); + return io_s; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/iostats_context.h b/librocksdb-sys/rocksdb/include/rocksdb/iostats_context.h new file mode 100644 index 0000000..559d44c --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/iostats_context.h @@ -0,0 +1,98 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include + +#include + +#include "rocksdb/perf_level.h" + +// A thread local context for gathering io-stats efficiently and transparently. +// Use SetPerfLevel(PerfLevel::kEnableTime) to enable time stats. + +namespace ROCKSDB_NAMESPACE { + +// EXPERIMENTAL: the IO statistics for tiered storage. It matches with each +// item in Temperature class. +struct FileIOByTemperature { + // the number of bytes read to Temperature::kHot file + uint64_t hot_file_bytes_read; + // the number of bytes read to Temperature::kWarm file + uint64_t warm_file_bytes_read; + // the number of bytes read to Temperature::kCold file + uint64_t cold_file_bytes_read; + // total number of reads to Temperature::kHot file + uint64_t hot_file_read_count; + // total number of reads to Temperature::kWarm file + uint64_t warm_file_read_count; + // total number of reads to Temperature::kCold file + uint64_t cold_file_read_count; + // reset all the statistics to 0. + void Reset() { + hot_file_bytes_read = 0; + warm_file_bytes_read = 0; + cold_file_bytes_read = 0; + hot_file_read_count = 0; + warm_file_read_count = 0; + cold_file_read_count = 0; + } +}; + +struct IOStatsContext { + // reset all io-stats counter to zero + void Reset(); + + std::string ToString(bool exclude_zero_counters = false) const; + + // the thread pool id + uint64_t thread_pool_id; + + // number of bytes that has been written. + uint64_t bytes_written; + // number of bytes that has been read. + uint64_t bytes_read; + + // time spent in open() and fopen(). + uint64_t open_nanos; + // time spent in fallocate(). + uint64_t allocate_nanos; + // time spent in write() and pwrite(). + uint64_t write_nanos; + // time spent in read() and pread() + uint64_t read_nanos; + // time spent in sync_file_range(). + uint64_t range_sync_nanos; + // time spent in fsync + uint64_t fsync_nanos; + // time spent in preparing write (fallocate etc). + uint64_t prepare_write_nanos; + // time spent in Logger::Logv(). + uint64_t logger_nanos; + // CPU time spent in write() and pwrite() + uint64_t cpu_write_nanos; + // CPU time spent in read() and pread() + uint64_t cpu_read_nanos; + + FileIOByTemperature file_io_stats_by_temperature; + + // It is not consistent that whether iostats follows PerfLevel.Timer counters + // follows it but BackupEngine relies on counter metrics to always be there. + // Here we create a backdoor option to disable some counters, so that some + // existing stats are not polluted by file operations, such as logging, by + // turning this off. + bool disable_iostats = false; +}; + +// If RocksDB is compiled with -DNIOSTATS_CONTEXT, then a pointer to a global, +// non-thread-local IOStatsContext object will be returned. Attempts to update +// this object will be ignored, and reading from it will also be no-op. +// Otherwise, a pointer to a thread-local IOStatsContext object will be +// returned. +// +// This function never returns nullptr. +IOStatsContext* get_iostats_context(); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/iterator.h b/librocksdb-sys/rocksdb/include/rocksdb/iterator.h new file mode 100644 index 0000000..9d4c9f7 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/iterator.h @@ -0,0 +1,144 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// An iterator yields a sequence of key/value pairs from a source. +// The following class defines the interface. Multiple implementations +// are provided by this library. In particular, iterators are provided +// to access the contents of a Table or a DB. +// +// Multiple threads can invoke const methods on an Iterator without +// external synchronization, but if any of the threads may call a +// non-const method, all threads accessing the same Iterator must use +// external synchronization. + +#pragma once + +#include + +#include "rocksdb/cleanable.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "rocksdb/wide_columns.h" + +namespace ROCKSDB_NAMESPACE { + +class Iterator : public Cleanable { + public: + Iterator() {} + // No copying allowed + Iterator(const Iterator&) = delete; + void operator=(const Iterator&) = delete; + + virtual ~Iterator() {} + + // An iterator is either positioned at a key/value pair, or + // not valid. This method returns true iff the iterator is valid. + // Always returns false if !status().ok(). + virtual bool Valid() const = 0; + + // Position at the first key in the source. The iterator is Valid() + // after this call iff the source is not empty. + virtual void SeekToFirst() = 0; + + // Position at the last key in the source. The iterator is + // Valid() after this call iff the source is not empty. + virtual void SeekToLast() = 0; + + // Position at the first key in the source that at or past target. + // The iterator is Valid() after this call iff the source contains + // an entry that comes at or past target. + // All Seek*() methods clear any error status() that the iterator had prior to + // the call; after the seek, status() indicates only the error (if any) that + // happened during the seek, not any past errors. + // Target does not contain timestamp. + virtual void Seek(const Slice& target) = 0; + + // Position at the last key in the source that at or before target. + // The iterator is Valid() after this call iff the source contains + // an entry that comes at or before target. + // Target does not contain timestamp. + virtual void SeekForPrev(const Slice& target) = 0; + + // Moves to the next entry in the source. After this call, Valid() is + // true iff the iterator was not positioned at the last entry in the source. + // REQUIRES: Valid() + virtual void Next() = 0; + + // Moves to the previous entry in the source. After this call, Valid() is + // true iff the iterator was not positioned at the first entry in source. + // REQUIRES: Valid() + virtual void Prev() = 0; + + // Return the key for the current entry. The underlying storage for + // the returned slice is valid only until the next modification of the + // iterator (i.e. the next SeekToFirst/SeekToLast/Seek/SeekForPrev/Next/Prev + // operation). + // REQUIRES: Valid() + virtual Slice key() const = 0; + + // Return the value for the current entry. If the entry is a plain key-value, + // return the value as-is; if it is a wide-column entity, return the value of + // the default anonymous column (see kDefaultWideColumnName) if any, or an + // empty value otherwise. The underlying storage for the returned slice is + // valid only until the next modification of the iterator (i.e. the next + // SeekToFirst/SeekToLast/Seek/SeekForPrev/Next/Prev operation). + // REQUIRES: Valid() + virtual Slice value() const = 0; + + // Return the wide columns for the current entry. If the entry is a + // wide-column entity, return it as-is; if it is a plain key-value, return it + // as an entity with a single anonymous column (see kDefaultWideColumnName) + // which contains the value. The underlying storage for the returned + // structure is valid only until the next modification of the iterator (i.e. + // the next SeekToFirst/SeekToLast/Seek/SeekForPrev/Next/Prev operation). + // REQUIRES: Valid() + virtual const WideColumns& columns() const { + assert(false); + return kNoWideColumns; + } + + // If an error has occurred, return it. Else return an ok status. + // If non-blocking IO is requested and this operation cannot be + // satisfied without doing some IO, then this returns Status::Incomplete(). + virtual Status status() const = 0; + + // If supported, renew the iterator to represent the latest state. The + // iterator will be invalidated after the call. Not supported if + // ReadOptions.snapshot is given when creating the iterator. + virtual Status Refresh() { + return Status::NotSupported("Refresh() is not supported"); + } + + // Property "rocksdb.iterator.is-key-pinned": + // If returning "1", this means that the Slice returned by key() is valid + // as long as the iterator is not deleted. + // It is guaranteed to always return "1" if + // - Iterator created with ReadOptions::pin_data = true + // - DB tables were created with + // BlockBasedTableOptions::use_delta_encoding = false. + // Property "rocksdb.iterator.super-version-number": + // LSM version used by the iterator. The same format as DB Property + // kCurrentSuperVersionNumber. See its comment for more information. + // Property "rocksdb.iterator.internal-key": + // Get the user-key portion of the internal key at which the iteration + // stopped. + virtual Status GetProperty(std::string prop_name, std::string* prop); + + virtual Slice timestamp() const { + assert(false); + return Slice(); + } +}; + +// Return an empty iterator (yields nothing). +extern Iterator* NewEmptyIterator(); + +// Return an empty iterator with the specified status. +extern Iterator* NewErrorIterator(const Status& status); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/ldb_tool.h b/librocksdb-sys/rocksdb/include/rocksdb/ldb_tool.h new file mode 100644 index 0000000..b8f2e22 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/ldb_tool.h @@ -0,0 +1,42 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once +#include +#include + +#include "rocksdb/db.h" +#include "rocksdb/options.h" + +namespace ROCKSDB_NAMESPACE { + +// An interface for converting a slice to a readable string +class SliceFormatter { + public: + virtual ~SliceFormatter() {} + virtual std::string Format(const Slice& s) const = 0; +}; + +// Options for customizing ldb tool (beyond the DB Options) +struct LDBOptions { + // Create LDBOptions with default values for all fields + LDBOptions(); + + // Key formatter that converts a slice to a readable string. + // Default: Slice::ToString() + std::shared_ptr key_formatter; + + std::string print_help_header = "ldb - RocksDB Tool"; +}; + +class LDBTool { + public: + void Run( + int argc, char** argv, Options db_options = Options(), + const LDBOptions& ldb_options = LDBOptions(), + const std::vector* column_families = nullptr); +}; + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/include/rocksdb/listener.h b/librocksdb-sys/rocksdb/include/rocksdb/listener.h new file mode 100644 index 0000000..87bc678 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/listener.h @@ -0,0 +1,840 @@ +// Copyright (c) 2014 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +#pragma once + +#include +#include +#include +#include +#include + +#include "rocksdb/advanced_options.h" +#include "rocksdb/compaction_job_stats.h" +#include "rocksdb/compression_type.h" +#include "rocksdb/customizable.h" +#include "rocksdb/io_status.h" +#include "rocksdb/status.h" +#include "rocksdb/table_properties.h" +#include "rocksdb/types.h" + +namespace ROCKSDB_NAMESPACE { + +using TablePropertiesCollection = + std::unordered_map>; + +class DB; +class ColumnFamilyHandle; +class Status; +struct CompactionJobStats; + +struct FileCreationBriefInfo { + FileCreationBriefInfo() = default; + FileCreationBriefInfo(const std::string& _db_name, + const std::string& _cf_name, + const std::string& _file_path, int _job_id) + : db_name(_db_name), + cf_name(_cf_name), + file_path(_file_path), + job_id(_job_id) {} + // the name of the database where the file was created. + std::string db_name; + // the name of the column family where the file was created. + std::string cf_name; + // the path to the created file. + std::string file_path; + // the id of the job (which could be flush or compaction) that + // created the file. + int job_id = 0; +}; + +struct TableFileCreationBriefInfo : public FileCreationBriefInfo { + // reason of creating the table. + TableFileCreationReason reason; +}; + +struct TableFileCreationInfo : public TableFileCreationBriefInfo { + TableFileCreationInfo() = default; + explicit TableFileCreationInfo(TableProperties&& prop) + : table_properties(prop) {} + // the size of the file. + uint64_t file_size; + // Detailed properties of the created file. + TableProperties table_properties; + // The status indicating whether the creation was successful or not. + Status status; + // The checksum of the table file being created + std::string file_checksum; + // The checksum function name of checksum generator used for this table file + std::string file_checksum_func_name; +}; + +struct BlobFileCreationBriefInfo : public FileCreationBriefInfo { + BlobFileCreationBriefInfo(const std::string& _db_name, + const std::string& _cf_name, + const std::string& _file_path, int _job_id, + BlobFileCreationReason _reason) + : FileCreationBriefInfo(_db_name, _cf_name, _file_path, _job_id), + reason(_reason) {} + // reason of creating the blob file. + BlobFileCreationReason reason; +}; + +struct BlobFileCreationInfo : public BlobFileCreationBriefInfo { + BlobFileCreationInfo(const std::string& _db_name, const std::string& _cf_name, + const std::string& _file_path, int _job_id, + BlobFileCreationReason _reason, + uint64_t _total_blob_count, uint64_t _total_blob_bytes, + Status _status, const std::string& _file_checksum, + const std::string& _file_checksum_func_name) + : BlobFileCreationBriefInfo(_db_name, _cf_name, _file_path, _job_id, + _reason), + total_blob_count(_total_blob_count), + total_blob_bytes(_total_blob_bytes), + status(_status), + file_checksum(_file_checksum), + file_checksum_func_name(_file_checksum_func_name) {} + + // the number of blob in a file. + uint64_t total_blob_count; + // the total bytes in a file. + uint64_t total_blob_bytes; + // The status indicating whether the creation was successful or not. + Status status; + // The checksum of the blob file being created. + std::string file_checksum; + // The checksum function name of checksum generator used for this blob file. + std::string file_checksum_func_name; +}; + +enum class CompactionReason : int { + kUnknown = 0, + // [Level] number of L0 files > level0_file_num_compaction_trigger + kLevelL0FilesNum, + // [Level] total size of level > MaxBytesForLevel() + kLevelMaxLevelSize, + // [Universal] Compacting for size amplification + kUniversalSizeAmplification, + // [Universal] Compacting for size ratio + kUniversalSizeRatio, + // [Universal] number of sorted runs > level0_file_num_compaction_trigger + kUniversalSortedRunNum, + // [FIFO] total size > max_table_files_size + kFIFOMaxSize, + // [FIFO] reduce number of files. + kFIFOReduceNumFiles, + // [FIFO] files with creation time < (current_time - interval) + kFIFOTtl, + // Manual compaction + kManualCompaction, + // DB::SuggestCompactRange() marked files for compaction + kFilesMarkedForCompaction, + // [Level] Automatic compaction within bottommost level to cleanup duplicate + // versions of same user key, usually due to a released snapshot. + kBottommostFiles, + // Compaction based on TTL + kTtl, + // According to the comments in flush_job.cc, RocksDB treats flush as + // a level 0 compaction in internal stats. + kFlush, + // [InternalOnly] External sst file ingestion treated as a compaction + // with placeholder input level L0 as file ingestion + // technically does not have an input level like other compactions. + // Used only for internal stats and conflict checking with other compactions + kExternalSstIngestion, + // Compaction due to SST file being too old + kPeriodicCompaction, + // Compaction in order to move files to temperature + kChangeTemperature, + // Compaction scheduled to force garbage collection of blob files + kForcedBlobGC, + // A special TTL compaction for RoundRobin policy, which basically the same as + // kLevelMaxLevelSize, but the goal is to compact TTLed files. + kRoundRobinTtl, + // [InternalOnly] DBImpl::ReFitLevel treated as a compaction, + // Used only for internal conflict checking with other compactions + kRefitLevel, + // total number of compaction reasons, new reasons must be added above this. + kNumOfReasons, +}; + +enum class FlushReason : int { + kOthers = 0x00, + kGetLiveFiles = 0x01, + kShutDown = 0x02, + kExternalFileIngestion = 0x03, + kManualCompaction = 0x04, + kWriteBufferManager = 0x05, + kWriteBufferFull = 0x06, + kTest = 0x07, + kDeleteFiles = 0x08, + kAutoCompaction = 0x09, + kManualFlush = 0x0a, + kErrorRecovery = 0xb, + // When set the flush reason to kErrorRecoveryRetryFlush, SwitchMemtable + // will not be called to avoid many small immutable memtables. + kErrorRecoveryRetryFlush = 0xc, + kWalFull = 0xd, +}; + +// TODO: In the future, BackgroundErrorReason will only be used to indicate +// why the BG Error is happening (e.g., flush, compaction). We may introduce +// other data structure to indicate other essential information such as +// the file type (e.g., Manifest, SST) and special context. +enum class BackgroundErrorReason { + kFlush, + kCompaction, + kWriteCallback, + kMemTable, + kManifestWrite, + kFlushNoWAL, + kManifestWriteNoWAL, +}; + +struct WriteStallInfo { + // the name of the column family + std::string cf_name; + // state of the write controller + struct { + WriteStallCondition cur; + WriteStallCondition prev; + } condition; +}; + + +struct FileDeletionInfo { + FileDeletionInfo() = default; + + FileDeletionInfo(const std::string& _db_name, const std::string& _file_path, + int _job_id, Status _status) + : db_name(_db_name), + file_path(_file_path), + job_id(_job_id), + status(_status) {} + // The name of the database where the file was deleted. + std::string db_name; + // The path to the deleted file. + std::string file_path; + // The id of the job which deleted the file. + int job_id = 0; + // The status indicating whether the deletion was successful or not. + Status status; +}; + +struct TableFileDeletionInfo : public FileDeletionInfo {}; + +struct BlobFileDeletionInfo : public FileDeletionInfo { + BlobFileDeletionInfo(const std::string& _db_name, + const std::string& _file_path, int _job_id, + Status _status) + : FileDeletionInfo(_db_name, _file_path, _job_id, _status) {} +}; + +enum class FileOperationType { + kRead, + kWrite, + kTruncate, + kClose, + kFlush, + kSync, + kFsync, + kRangeSync, + kAppend, + kPositionedAppend, + kOpen +}; + +struct FileOperationInfo { + using Duration = std::chrono::nanoseconds; + using SteadyTimePoint = + std::chrono::time_point; + using SystemTimePoint = + std::chrono::time_point; + using StartTimePoint = std::pair; + using FinishTimePoint = SteadyTimePoint; + + FileOperationType type; + const std::string& path; + // Rocksdb try to provide file temperature information, but it's not + // guaranteed. + Temperature temperature; + uint64_t offset; + size_t length; + const Duration duration; + const SystemTimePoint& start_ts; + Status status; + + FileOperationInfo(const FileOperationType _type, const std::string& _path, + const StartTimePoint& _start_ts, + const FinishTimePoint& _finish_ts, const Status& _status, + const Temperature _temperature = Temperature::kUnknown) + : type(_type), + path(_path), + temperature(_temperature), + duration(std::chrono::duration_cast( + _finish_ts - _start_ts.second)), + start_ts(_start_ts.first), + status(_status) {} + static StartTimePoint StartNow() { + return std::make_pair( + std::chrono::system_clock::now(), std::chrono::steady_clock::now()); + } + static FinishTimePoint FinishNow() { + return std::chrono::steady_clock::now(); + } +}; + +struct BlobFileInfo { + BlobFileInfo(const std::string& _blob_file_path, + const uint64_t _blob_file_number) + : blob_file_path(_blob_file_path), blob_file_number(_blob_file_number) {} + + std::string blob_file_path; + uint64_t blob_file_number; +}; + +struct BlobFileAdditionInfo : public BlobFileInfo { + BlobFileAdditionInfo(const std::string& _blob_file_path, + const uint64_t _blob_file_number, + const uint64_t _total_blob_count, + const uint64_t _total_blob_bytes) + : BlobFileInfo(_blob_file_path, _blob_file_number), + total_blob_count(_total_blob_count), + total_blob_bytes(_total_blob_bytes) {} + uint64_t total_blob_count; + uint64_t total_blob_bytes; +}; + +struct BlobFileGarbageInfo : public BlobFileInfo { + BlobFileGarbageInfo(const std::string& _blob_file_path, + const uint64_t _blob_file_number, + const uint64_t _garbage_blob_count, + const uint64_t _garbage_blob_bytes) + : BlobFileInfo(_blob_file_path, _blob_file_number), + garbage_blob_count(_garbage_blob_count), + garbage_blob_bytes(_garbage_blob_bytes) {} + uint64_t garbage_blob_count; + uint64_t garbage_blob_bytes; +}; + +struct FlushJobInfo { + // the id of the column family + uint32_t cf_id; + // the name of the column family + std::string cf_name; + // the path to the newly created file + std::string file_path; + // the file number of the newly created file + uint64_t file_number; + // the oldest blob file referenced by the newly created file + uint64_t oldest_blob_file_number; + // the id of the thread that completed this flush job. + uint64_t thread_id; + // the job id, which is unique in the same thread. + int job_id; + // If true, then rocksdb is currently slowing-down all writes to prevent + // creating too many Level 0 files as compaction seems not able to + // catch up the write request speed. This indicates that there are + // too many files in Level 0. + bool triggered_writes_slowdown; + // If true, then rocksdb is currently blocking any writes to prevent + // creating more L0 files. This indicates that there are too many + // files in level 0. Compactions should try to compact L0 files down + // to lower levels as soon as possible. + bool triggered_writes_stop; + // The smallest sequence number in the newly created file + SequenceNumber smallest_seqno; + // The largest sequence number in the newly created file + SequenceNumber largest_seqno; + // Table properties of the table being flushed + TableProperties table_properties; + + FlushReason flush_reason; + + // Compression algorithm used for blob output files + CompressionType blob_compression_type; + + // Information about blob files created during flush in Integrated BlobDB. + std::vector blob_file_addition_infos; +}; + +struct CompactionFileInfo { + // The level of the file. + int level; + + // The file number of the file. + uint64_t file_number; + + // The file number of the oldest blob file this SST file references. + uint64_t oldest_blob_file_number; +}; + +struct SubcompactionJobInfo { + ~SubcompactionJobInfo() { status.PermitUncheckedError(); } + // the id of the column family where the compaction happened. + uint32_t cf_id; + // the name of the column family where the compaction happened. + std::string cf_name; + // the status indicating whether the compaction was successful or not. + Status status; + // the id of the thread that completed this compaction job. + uint64_t thread_id; + // the job id, which is unique in the same thread. + int job_id; + + // sub-compaction job id, which is only unique within the same compaction, so + // use both 'job_id' and 'subcompaction_job_id' to identify a subcompaction + // within an instance. + // For non subcompaction job, it's set to -1. + int subcompaction_job_id; + // the smallest input level of the compaction. + int base_input_level; + // the output level of the compaction. + int output_level; + + // Reason to run the compaction + CompactionReason compaction_reason; + + // Compression algorithm used for output files + CompressionType compression; + + // Statistics and other additional details on the compaction + CompactionJobStats stats; + + // Compression algorithm used for blob output files. + CompressionType blob_compression_type; +}; + +struct CompactionJobInfo { + ~CompactionJobInfo() { status.PermitUncheckedError(); } + // the id of the column family where the compaction happened. + uint32_t cf_id; + // the name of the column family where the compaction happened. + std::string cf_name; + // the status indicating whether the compaction was successful or not. + Status status; + // the id of the thread that completed this compaction job. + uint64_t thread_id; + // the job id, which is unique in the same thread. + int job_id; + + // the smallest input level of the compaction. + int base_input_level; + // the output level of the compaction. + int output_level; + + // The following variables contain information about compaction inputs + // and outputs. A file may appear in both the input and output lists + // if it was simply moved to a different level. The order of elements + // is the same across input_files and input_file_infos; similarly, it is + // the same across output_files and output_file_infos. + + // The names of the compaction input files. + std::vector input_files; + + // Additional information about the compaction input files. + std::vector input_file_infos; + + // The names of the compaction output files. + std::vector output_files; + + // Additional information about the compaction output files. + std::vector output_file_infos; + + // Table properties for input and output tables. + // The map is keyed by values from input_files and output_files. + TablePropertiesCollection table_properties; + + // Reason to run the compaction + CompactionReason compaction_reason; + + // Compression algorithm used for output files + CompressionType compression; + + // Statistics and other additional details on the compaction + CompactionJobStats stats; + + // Compression algorithm used for blob output files. + CompressionType blob_compression_type; + + // Information about blob files created during compaction in Integrated + // BlobDB. + std::vector blob_file_addition_infos; + + // Information about blob files deleted during compaction in Integrated + // BlobDB. + std::vector blob_file_garbage_infos; +}; + +struct MemTableInfo { + // the name of the column family to which memtable belongs + std::string cf_name; + // Sequence number of the first element that was inserted + // into the memtable. + SequenceNumber first_seqno; + // Sequence number that is guaranteed to be smaller than or equal + // to the sequence number of any key that could be inserted into this + // memtable. It can then be assumed that any write with a larger(or equal) + // sequence number will be present in this memtable or a later memtable. + SequenceNumber earliest_seqno; + // Total number of entries in memtable + uint64_t num_entries; + // Total number of deletes in memtable + uint64_t num_deletes; +}; + +struct ExternalFileIngestionInfo { + // the name of the column family + std::string cf_name; + // Path of the file outside the DB + std::string external_file_path; + // Path of the file inside the DB + std::string internal_file_path; + // The global sequence number assigned to keys in this file + SequenceNumber global_seqno; + // Table properties of the table being flushed + TableProperties table_properties; +}; + +// Result of auto background error recovery +struct BackgroundErrorRecoveryInfo { + // The original error that triggered the recovery + Status old_bg_error; + + // The final bg_error after all recovery attempts. Status::OK() means + // the recovery was successful and the database is fully operational. + Status new_bg_error; +}; + +struct IOErrorInfo { + IOErrorInfo(const IOStatus& _io_status, FileOperationType _operation, + const std::string& _file_path, size_t _length, uint64_t _offset) + : io_status(_io_status), + operation(_operation), + file_path(_file_path), + length(_length), + offset(_offset) {} + + IOStatus io_status; + FileOperationType operation; + std::string file_path; + size_t length; + uint64_t offset; +}; + +// EventListener class contains a set of callback functions that will +// be called when specific RocksDB event happens such as flush. It can +// be used as a building block for developing custom features such as +// stats-collector or external compaction algorithm. +// +// IMPORTANT +// Because compaction is needed to resolve a "writes stopped" condition, +// calling or waiting for any blocking DB write function (no_slowdown=false) +// from a compaction-related listener callback can hang RocksDB. For DB +// writes from a callback we recommend a WriteBatch and no_slowdown=true, +// because the WriteBatch can accumulate writes for later in case DB::Write +// returns Status::Incomplete. Similarly, calling CompactRange or similar +// could hang by waiting for a background worker that is occupied until the +// callback returns. +// +// Otherwise, callback functions should not run for an extended period of +// time before the function returns, because this will slow RocksDB. +// +// [Threading] All EventListener callback will be called using the +// actual thread that involves in that specific event. For example, it +// is the RocksDB background flush thread that does the actual flush to +// call EventListener::OnFlushCompleted(). +// +// [Locking] All EventListener callbacks are designed to be called without +// the current thread holding any DB mutex. This is to prevent potential +// deadlock and performance issue when using EventListener callback +// in a complex way. +// +// [Exceptions] Exceptions MUST NOT propagate out of overridden functions into +// RocksDB, because RocksDB is not exception-safe. This could cause undefined +// behavior including data loss, unreported corruption, deadlocks, and more. +class EventListener : public Customizable { + public: + static const char* Type() { return "EventListener"; } + static Status CreateFromString(const ConfigOptions& options, + const std::string& id, + std::shared_ptr* result); + const char* Name() const override { + // Since EventListeners did not have a name previously, we will assume + // an empty name. Instances should override this method. + return ""; + } + // A callback function to RocksDB which will be called whenever a + // registered RocksDB flushes a file. The default implementation is + // no-op. + // + // Note that the this function must be implemented in a way such that + // it should not run for an extended period of time before the function + // returns. Otherwise, RocksDB may be blocked. + virtual void OnFlushCompleted(DB* /*db*/, + const FlushJobInfo& /*flush_job_info*/) {} + + // A callback function to RocksDB which will be called before a + // RocksDB starts to flush memtables. The default implementation is + // no-op. + // + // Note that the this function must be implemented in a way such that + // it should not run for an extended period of time before the function + // returns. Otherwise, RocksDB may be blocked. + virtual void OnFlushBegin(DB* /*db*/, + const FlushJobInfo& /*flush_job_info*/) {} + + // A callback function for RocksDB which will be called whenever + // a SST file is deleted. Different from OnCompactionCompleted and + // OnFlushCompleted, this callback is designed for external logging + // service and thus only provide string parameters instead + // of a pointer to DB. Applications that build logic basic based + // on file creations and deletions is suggested to implement + // OnFlushCompleted and OnCompactionCompleted. + // + // Note that if applications would like to use the passed reference + // outside this function call, they should make copies from the + // returned value. + virtual void OnTableFileDeleted(const TableFileDeletionInfo& /*info*/) {} + + // A callback function to RocksDB which will be called before a + // RocksDB starts to compact. The default implementation is + // no-op. + // + // Note that the this function must be implemented in a way such that + // it should not run for an extended period of time before the function + // returns. Otherwise, RocksDB may be blocked. + virtual void OnCompactionBegin(DB* /*db*/, const CompactionJobInfo& /*ci*/) {} + + // A callback function for RocksDB which will be called whenever + // a registered RocksDB compacts a file. The default implementation + // is a no-op. + // + // Note that this function must be implemented in a way such that + // it should not run for an extended period of time before the function + // returns. Otherwise, RocksDB may be blocked. + // + // @param db a pointer to the rocksdb instance which just compacted + // a file. + // @param ci a reference to a CompactionJobInfo struct. 'ci' is released + // after this function is returned, and must be copied if it is needed + // outside of this function. + virtual void OnCompactionCompleted(DB* /*db*/, + const CompactionJobInfo& /*ci*/) {} + + // A callback function to RocksDB which will be called before a sub-compaction + // begins. If a compaction is split to 2 sub-compactions, it will trigger one + // `OnCompactionBegin()` first, then two `OnSubcompactionBegin()`. + // If compaction is not split, it will still trigger one + // `OnSubcompactionBegin()`, as internally, compaction is always handled by + // sub-compaction. The default implementation is a no-op. + // + // Note that this function must be implemented in a way such that + // it should not run for an extended period of time before the function + // returns. Otherwise, RocksDB may be blocked. + // + // @param ci a reference to a CompactionJobInfo struct, it contains a + // `sub_job_id` which is only unique within the specified compaction (which + // can be identified by `job_id`). 'ci' is released after this function is + // returned, and must be copied if it's needed outside this function. + // Note: `table_properties` is not set for sub-compaction, the information + // could be got from `OnCompactionBegin()`. + virtual void OnSubcompactionBegin(const SubcompactionJobInfo& /*si*/) {} + + // A callback function to RocksDB which will be called whenever a + // sub-compaction completed. The same as `OnSubcompactionBegin()`, if a + // compaction is split to 2 sub-compactions, it will be triggered twice. If + // a compaction is not split, it will still be triggered once. + // The default implementation is a no-op. + // + // Note that this function must be implemented in a way such that + // it should not run for an extended period of time before the function + // returns. Otherwise, RocksDB may be blocked. + // + // @param ci a reference to a CompactionJobInfo struct, it contains a + // `sub_job_id` which is only unique within the specified compaction (which + // can be identified by `job_id`). 'ci' is released after this function is + // returned, and must be copied if it's needed outside this function. + // Note: `table_properties` is not set for sub-compaction, the information + // could be got from `OnCompactionCompleted()`. + virtual void OnSubcompactionCompleted(const SubcompactionJobInfo& /*si*/) {} + + // A callback function for RocksDB which will be called whenever + // a SST file is created. Different from OnCompactionCompleted and + // OnFlushCompleted, this callback is designed for external logging + // service and thus only provide string parameters instead + // of a pointer to DB. Applications that build logic basic based + // on file creations and deletions is suggested to implement + // OnFlushCompleted and OnCompactionCompleted. + // + // Historically it will only be called if the file is successfully created. + // Now it will also be called on failure case. User can check info.status + // to see if it succeeded or not. + // + // Note that if applications would like to use the passed reference + // outside this function call, they should make copies from these + // returned value. + virtual void OnTableFileCreated(const TableFileCreationInfo& /*info*/) {} + + // A callback function for RocksDB which will be called before + // a SST file is being created. It will follow by OnTableFileCreated after + // the creation finishes. + // + // Note that if applications would like to use the passed reference + // outside this function call, they should make copies from these + // returned value. + virtual void OnTableFileCreationStarted( + const TableFileCreationBriefInfo& /*info*/) {} + + // A callback function for RocksDB which will be called before + // a memtable is made immutable. + // + // Note that the this function must be implemented in a way such that + // it should not run for an extended period of time before the function + // returns. Otherwise, RocksDB may be blocked. + // + // Note that if applications would like to use the passed reference + // outside this function call, they should make copies from these + // returned value. + virtual void OnMemTableSealed(const MemTableInfo& /*info*/) {} + + // A callback function for RocksDB which will be called before + // a column family handle is deleted. + // + // Note that the this function must be implemented in a way such that + // it should not run for an extended period of time before the function + // returns. Otherwise, RocksDB may be blocked. + // @param handle is a pointer to the column family handle to be deleted + // which will become a dangling pointer after the deletion. + virtual void OnColumnFamilyHandleDeletionStarted( + ColumnFamilyHandle* /*handle*/) {} + + // A callback function for RocksDB which will be called after an external + // file is ingested using IngestExternalFile. + // + // Note that the this function will run on the same thread as + // IngestExternalFile(), if this function is blocked, IngestExternalFile() + // will be blocked from finishing. + virtual void OnExternalFileIngested( + DB* /*db*/, const ExternalFileIngestionInfo& /*info*/) {} + + // A callback function for RocksDB which will be called before setting the + // background error status to a non-OK value. The new background error status + // is provided in `bg_error` and can be modified by the callback. E.g., a + // callback can suppress errors by resetting it to Status::OK(), thus + // preventing the database from entering read-only mode. We do not provide any + // guarantee when failed flushes/compactions will be rescheduled if the user + // suppresses an error. + // + // Note that this function can run on the same threads as flush, compaction, + // and user writes. So, it is extremely important not to perform heavy + // computations or blocking calls in this function. + virtual void OnBackgroundError(BackgroundErrorReason /* reason */, + Status* /* bg_error */) {} + + // A callback function for RocksDB which will be called whenever a change + // of superversion triggers a change of the stall conditions. + // + // Note that the this function must be implemented in a way such that + // it should not run for an extended period of time before the function + // returns. Otherwise, RocksDB may be blocked. + virtual void OnStallConditionsChanged(const WriteStallInfo& /*info*/) {} + + // A callback function for RocksDB which will be called whenever a file read + // operation finishes. + virtual void OnFileReadFinish(const FileOperationInfo& /* info */) {} + + // A callback function for RocksDB which will be called whenever a file write + // operation finishes. + virtual void OnFileWriteFinish(const FileOperationInfo& /* info */) {} + + // A callback function for RocksDB which will be called whenever a file flush + // operation finishes. + virtual void OnFileFlushFinish(const FileOperationInfo& /* info */) {} + + // A callback function for RocksDB which will be called whenever a file sync + // operation finishes. + virtual void OnFileSyncFinish(const FileOperationInfo& /* info */) {} + + // A callback function for RocksDB which will be called whenever a file + // rangeSync operation finishes. + virtual void OnFileRangeSyncFinish(const FileOperationInfo& /* info */) {} + + // A callback function for RocksDB which will be called whenever a file + // truncate operation finishes. + virtual void OnFileTruncateFinish(const FileOperationInfo& /* info */) {} + + // A callback function for RocksDB which will be called whenever a file close + // operation finishes. + virtual void OnFileCloseFinish(const FileOperationInfo& /* info */) {} + + // If true, the OnFile*Finish functions will be called. If + // false, then they won't be called. + virtual bool ShouldBeNotifiedOnFileIO() { return false; } + + // A callback function for RocksDB which will be called just before + // starting the automatic recovery process for recoverable background + // errors, such as NoSpace(). The callback can suppress the automatic + // recovery by setting *auto_recovery to false. The database will then + // have to be transitioned out of read-only mode by calling DB::Resume() + virtual void OnErrorRecoveryBegin(BackgroundErrorReason /* reason */, + Status /* bg_error */, + bool* /* auto_recovery */) {} + + // DEPRECATED + // A callback function for RocksDB which will be called once the database + // is recovered from read-only mode after an error. When this is called, it + // means normal writes to the database can be issued and the user can + // initiate any further recovery actions needed + virtual void OnErrorRecoveryCompleted(Status old_bg_error) { + old_bg_error.PermitUncheckedError(); + } + + // A callback function for RocksDB which will be called once the recovery + // attempt from a background retryable error is completed. The recovery + // may have been successful or not. In either case, the callback is called + // with the old and new error. If info.new_bg_error is Status::OK(), that + // means the recovery succeeded. + virtual void OnErrorRecoveryEnd(const BackgroundErrorRecoveryInfo& /*info*/) { + } + + // A callback function for RocksDB which will be called before + // a blob file is being created. It will follow by OnBlobFileCreated after + // the creation finishes. + // + // Note that if applications would like to use the passed reference + // outside this function call, they should make copies from these + // returned value. + virtual void OnBlobFileCreationStarted( + const BlobFileCreationBriefInfo& /*info*/) {} + + // A callback function for RocksDB which will be called whenever + // a blob file is created. + // It will be called whether the file is successfully created or not. User can + // check info.status to see if it succeeded or not. + // + // Note that if applications would like to use the passed reference + // outside this function call, they should make copies from these + // returned value. + virtual void OnBlobFileCreated(const BlobFileCreationInfo& /*info*/) {} + + // A callback function for RocksDB which will be called whenever + // a blob file is deleted. + // + // Note that if applications would like to use the passed reference + // outside this function call, they should make copies from these + // returned value. + virtual void OnBlobFileDeleted(const BlobFileDeletionInfo& /*info*/) {} + + // A callback function for RocksDB which will be called whenever an IO error + // happens. ShouldBeNotifiedOnFileIO should be set to true to get a callback. + virtual void OnIOError(const IOErrorInfo& /*info*/) {} + + ~EventListener() override {} +}; + + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/memory_allocator.h b/librocksdb-sys/rocksdb/include/rocksdb/memory_allocator.h new file mode 100644 index 0000000..d126abf --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/memory_allocator.h @@ -0,0 +1,87 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "rocksdb/customizable.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +// MemoryAllocator is an interface that a client can implement to supply custom +// memory allocation and deallocation methods. See rocksdb/cache.h for more +// information. +// All methods should be thread-safe. +class MemoryAllocator : public Customizable { + public: + static const char* Type() { return "MemoryAllocator"; } + static Status CreateFromString(const ConfigOptions& options, + const std::string& value, + std::shared_ptr* result); + + // Allocate a block of at least size. Has to be thread-safe. + virtual void* Allocate(size_t size) = 0; + + // Deallocate previously allocated block. Has to be thread-safe. + virtual void Deallocate(void* p) = 0; + + // Returns the memory size of the block allocated at p. The default + // implementation that just returns the original allocation_size is fine. + virtual size_t UsableSize(void* /*p*/, size_t allocation_size) const { + // default implementation just returns the allocation size + return allocation_size; + } + + std::string GetId() const override { return GenerateIndividualId(); } +}; + +struct JemallocAllocatorOptions { + static const char* kName() { return "JemallocAllocatorOptions"; } + // Jemalloc tcache cache allocations by size class. For each size class, + // it caches between 20 (for large size classes) to 200 (for small size + // classes). To reduce tcache memory usage in case the allocator is access + // by large number of threads, we can control whether to cache an allocation + // by its size. + bool limit_tcache_size = false; + + // Lower bound of allocation size to use tcache, if limit_tcache_size=true. + // When used with block cache, it is recommended to set it to block_size/4. + size_t tcache_size_lower_bound = 1024; + + // Upper bound of allocation size to use tcache, if limit_tcache_size=true. + // When used with block cache, it is recommended to set it to block_size. + size_t tcache_size_upper_bound = 16 * 1024; + + // Number of arenas across which we spread allocation requests. Increasing + // this setting can mitigate arena mutex contention. The value must be + // positive. + size_t num_arenas = 1; +}; + +// Generate memory allocator which allocates through Jemalloc and utilize +// MADV_DONTDUMP through madvise to exclude cache items from core dump. +// Applications can use the allocator with block cache to exclude block cache +// usage from core dump. +// +// Implementation details: +// The JemallocNodumpAllocator creates a dedicated jemalloc arena, and all +// allocations of the JemallocNodumpAllocator are through the same arena. +// The memory allocator hooks memory allocation of the arena, and calls +// madvise() with MADV_DONTDUMP flag to exclude the piece of memory from +// core dump. Side benefit of using single arena would be reduction of jemalloc +// metadata for some workloads. +// +// To mitigate mutex contention for using one single arena (see also +// `JemallocAllocatorOptions::num_arenas` above), jemalloc tcache +// (thread-local cache) is enabled to cache unused allocations for future use. +// The tcache normally incurs 0.5M extra memory usage per-thread. The usage +// can be reduced by limiting allocation sizes to cache. +extern Status NewJemallocNodumpAllocator( + JemallocAllocatorOptions& options, + std::shared_ptr* memory_allocator); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/memtablerep.h b/librocksdb-sys/rocksdb/include/rocksdb/memtablerep.h new file mode 100644 index 0000000..be0f6cd --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/memtablerep.h @@ -0,0 +1,421 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file contains the interface that must be implemented by any collection +// to be used as the backing store for a MemTable. Such a collection must +// satisfy the following properties: +// (1) It does not store duplicate items. +// (2) It uses MemTableRep::KeyComparator to compare items for iteration and +// equality. +// (3) It can be accessed concurrently by multiple readers and can support +// during reads. However, it needn't support multiple concurrent writes. +// (4) Items are never deleted. +// The liberal use of assertions is encouraged to enforce (1). +// +// The factory will be passed an MemTableAllocator object when a new MemTableRep +// is requested. +// +// Users can implement their own memtable representations. We include three +// types built in: +// - SkipListRep: This is the default; it is backed by a skip list. +// - HashSkipListRep: The memtable rep that is best used for keys that are +// structured like "prefix:suffix" where iteration within a prefix is +// common and iteration across different prefixes is rare. It is backed by +// a hash map where each bucket is a skip list. +// - VectorRep: This is backed by an unordered std::vector. On iteration, the +// vector is sorted. It is intelligent about sorting; once the MarkReadOnly() +// has been called, the vector will only be sorted once. It is optimized for +// random-write-heavy workloads. +// +// The last four implementations are designed for situations in which +// iteration over the entire collection is rare since doing so requires all the +// keys to be copied into a sorted data structure. + +#pragma once + +#include +#include + +#include +#include +#include + +#include "rocksdb/customizable.h" +#include "rocksdb/slice.h" + +namespace ROCKSDB_NAMESPACE { + +class Arena; +class Allocator; +class LookupKey; +class SliceTransform; +class Logger; +struct DBOptions; + +using KeyHandle = void*; + +extern Slice GetLengthPrefixedSlice(const char* data); + +class MemTableRep { + public: + // KeyComparator provides a means to compare keys, which are internal keys + // concatenated with values. + class KeyComparator { + public: + using DecodedType = ROCKSDB_NAMESPACE::Slice; + + virtual DecodedType decode_key(const char* key) const { + // The format of key is frozen and can be treated as a part of the API + // contract. Refer to MemTable::Add for details. + return GetLengthPrefixedSlice(key); + } + + // Compare a and b. Return a negative value if a is less than b, 0 if they + // are equal, and a positive value if a is greater than b + virtual int operator()(const char* prefix_len_key1, + const char* prefix_len_key2) const = 0; + + virtual int operator()(const char* prefix_len_key, + const Slice& key) const = 0; + + virtual ~KeyComparator() {} + }; + + explicit MemTableRep(Allocator* allocator) : allocator_(allocator) {} + + // Allocate a buf of len size for storing key. The idea is that a + // specific memtable representation knows its underlying data structure + // better. By allowing it to allocate memory, it can possibly put + // correlated stuff in consecutive memory area to make processor + // prefetching more efficient. + virtual KeyHandle Allocate(const size_t len, char** buf); + + // Insert key into the collection. (The caller will pack key and value into a + // single buffer and pass that in as the parameter to Insert). + // REQUIRES: nothing that compares equal to key is currently in the + // collection, and no concurrent modifications to the table in progress + virtual void Insert(KeyHandle handle) = 0; + + // Same as ::Insert + // Returns false if MemTableRepFactory::CanHandleDuplicatedKey() is true and + // the already exists. + virtual bool InsertKey(KeyHandle handle) { + Insert(handle); + return true; + } + + // Same as Insert(), but in additional pass a hint to insert location for + // the key. If hint points to nullptr, a new hint will be populated. + // otherwise the hint will be updated to reflect the last insert location. + // + // Currently only skip-list based memtable implement the interface. Other + // implementations will fallback to Insert() by default. + virtual void InsertWithHint(KeyHandle handle, void** /*hint*/) { + // Ignore the hint by default. + Insert(handle); + } + + // Same as ::InsertWithHint + // Returns false if MemTableRepFactory::CanHandleDuplicatedKey() is true and + // the already exists. + virtual bool InsertKeyWithHint(KeyHandle handle, void** hint) { + InsertWithHint(handle, hint); + return true; + } + + // Same as ::InsertWithHint, but allow concurrent write + // + // If hint points to nullptr, a new hint will be allocated on heap, otherwise + // the hint will be updated to reflect the last insert location. The hint is + // owned by the caller and it is the caller's responsibility to delete the + // hint later. + // + // Currently only skip-list based memtable implement the interface. Other + // implementations will fallback to InsertConcurrently() by default. + virtual void InsertWithHintConcurrently(KeyHandle handle, void** /*hint*/) { + // Ignore the hint by default. + InsertConcurrently(handle); + } + + // Same as ::InsertWithHintConcurrently + // Returns false if MemTableRepFactory::CanHandleDuplicatedKey() is true and + // the already exists. + virtual bool InsertKeyWithHintConcurrently(KeyHandle handle, void** hint) { + InsertWithHintConcurrently(handle, hint); + return true; + } + + // Like Insert(handle), but may be called concurrent with other calls + // to InsertConcurrently for other handles. + // + // Returns false if MemTableRepFactory::CanHandleDuplicatedKey() is true and + // the already exists. + virtual void InsertConcurrently(KeyHandle handle); + + // Same as ::InsertConcurrently + // Returns false if MemTableRepFactory::CanHandleDuplicatedKey() is true and + // the already exists. + virtual bool InsertKeyConcurrently(KeyHandle handle) { + InsertConcurrently(handle); + return true; + } + + // Returns true iff an entry that compares equal to key is in the collection. + virtual bool Contains(const char* key) const = 0; + + // Notify this table rep that it will no longer be added to. By default, + // does nothing. After MarkReadOnly() is called, this table rep will + // not be written to (ie No more calls to Allocate(), Insert(), + // or any writes done directly to entries accessed through the iterator.) + virtual void MarkReadOnly() {} + + // Notify this table rep that it has been flushed to stable storage. + // By default, does nothing. + // + // Invariant: MarkReadOnly() is called, before MarkFlushed(). + // Note that this method if overridden, should not run for an extended period + // of time. Otherwise, RocksDB may be blocked. + virtual void MarkFlushed() {} + + // Look up key from the mem table, since the first key in the mem table whose + // user_key matches the one given k, call the function callback_func(), with + // callback_args directly forwarded as the first parameter, and the mem table + // key as the second parameter. If the return value is false, then terminates. + // Otherwise, go through the next key. + // + // It's safe for Get() to terminate after having finished all the potential + // key for the k.user_key(), or not. + // + // Default: + // Get() function with a default value of dynamically construct an iterator, + // seek and call the call back function. + virtual void Get(const LookupKey& k, void* callback_args, + bool (*callback_func)(void* arg, const char* entry)); + + virtual uint64_t ApproximateNumEntries(const Slice& /*start_ikey*/, + const Slice& /*end_key*/) { + return 0; + } + + // Returns a vector of unique random memtable entries of approximate + // size 'target_sample_size' (this size is not strictly enforced). + virtual void UniqueRandomSample(const uint64_t num_entries, + const uint64_t target_sample_size, + std::unordered_set* entries) { + (void)num_entries; + (void)target_sample_size; + (void)entries; + assert(false); + } + + // Report an approximation of how much memory has been used other than memory + // that was allocated through the allocator. Safe to call from any thread. + virtual size_t ApproximateMemoryUsage() = 0; + + virtual ~MemTableRep() {} + + // Iteration over the contents of a skip collection + class Iterator { + public: + // Initialize an iterator over the specified collection. + // The returned iterator is not valid. + // explicit Iterator(const MemTableRep* collection); + virtual ~Iterator() {} + + // Returns true iff the iterator is positioned at a valid node. + virtual bool Valid() const = 0; + + // Returns the key at the current position. + // REQUIRES: Valid() + virtual const char* key() const = 0; + + // Advances to the next position. + // REQUIRES: Valid() + virtual void Next() = 0; + + // Advances to the previous position. + // REQUIRES: Valid() + virtual void Prev() = 0; + + // Advance to the first entry with a key >= target + virtual void Seek(const Slice& internal_key, const char* memtable_key) = 0; + + // retreat to the first entry with a key <= target + virtual void SeekForPrev(const Slice& internal_key, + const char* memtable_key) = 0; + + virtual void RandomSeek() {} + + // Position at the first entry in collection. + // Final state of iterator is Valid() iff collection is not empty. + virtual void SeekToFirst() = 0; + + // Position at the last entry in collection. + // Final state of iterator is Valid() iff collection is not empty. + virtual void SeekToLast() = 0; + }; + + // Return an iterator over the keys in this representation. + // arena: If not null, the arena needs to be used to allocate the Iterator. + // When destroying the iterator, the caller will not call "delete" + // but Iterator::~Iterator() directly. The destructor needs to destroy + // all the states but those allocated in arena. + virtual Iterator* GetIterator(Arena* arena = nullptr) = 0; + + // Return an iterator that has a special Seek semantics. The result of + // a Seek might only include keys with the same prefix as the target key. + // arena: If not null, the arena is used to allocate the Iterator. + // When destroying the iterator, the caller will not call "delete" + // but Iterator::~Iterator() directly. The destructor needs to destroy + // all the states but those allocated in arena. + virtual Iterator* GetDynamicPrefixIterator(Arena* arena = nullptr) { + return GetIterator(arena); + } + + // Return true if the current MemTableRep supports merge operator. + // Default: true + virtual bool IsMergeOperatorSupported() const { return true; } + + // Return true if the current MemTableRep supports snapshot + // Default: true + virtual bool IsSnapshotSupported() const { return true; } + + protected: + // When *key is an internal key concatenated with the value, returns the + // user key. + virtual Slice UserKey(const char* key) const; + + Allocator* allocator_; +}; + +// This is the base class for all factories that are used by RocksDB to create +// new MemTableRep objects +class MemTableRepFactory : public Customizable { + public: + ~MemTableRepFactory() override {} + + static const char* Type() { return "MemTableRepFactory"; } + static Status CreateFromString(const ConfigOptions& config_options, + const std::string& id, + std::unique_ptr* factory); + static Status CreateFromString(const ConfigOptions& config_options, + const std::string& id, + std::shared_ptr* factory); + + virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&, + Allocator*, const SliceTransform*, + Logger* logger) = 0; + virtual MemTableRep* CreateMemTableRep( + const MemTableRep::KeyComparator& key_cmp, Allocator* allocator, + const SliceTransform* slice_transform, Logger* logger, + uint32_t /* column_family_id */) { + return CreateMemTableRep(key_cmp, allocator, slice_transform, logger); + } + + const char* Name() const override = 0; + + // Return true if the current MemTableRep supports concurrent inserts + // Default: false + virtual bool IsInsertConcurrentlySupported() const { return false; } + + // Return true if the current MemTableRep supports detecting duplicate + // at insertion time. If true, then MemTableRep::Insert* returns + // false when if the already exists. + // Default: false + virtual bool CanHandleDuplicatedKey() const { return false; } +}; + +// This uses a skip list to store keys. It is the default. +// +// Parameters: +// lookahead: If non-zero, each iterator's seek operation will start the +// search from the previously visited record (doing at most 'lookahead' +// steps). This is an optimization for the access pattern including many +// seeks with consecutive keys. +class SkipListFactory : public MemTableRepFactory { + public: + explicit SkipListFactory(size_t lookahead = 0); + + // Methods for Configurable/Customizable class overrides + static const char* kClassName() { return "SkipListFactory"; } + static const char* kNickName() { return "skip_list"; } + virtual const char* Name() const override { return kClassName(); } + virtual const char* NickName() const override { return kNickName(); } + std::string GetId() const override; + + // Methods for MemTableRepFactory class overrides + using MemTableRepFactory::CreateMemTableRep; + virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&, + Allocator*, const SliceTransform*, + Logger* logger) override; + + bool IsInsertConcurrentlySupported() const override { return true; } + + bool CanHandleDuplicatedKey() const override { return true; } + + private: + size_t lookahead_; +}; + +// This creates MemTableReps that are backed by an std::vector. On iteration, +// the vector is sorted. This is useful for workloads where iteration is very +// rare and writes are generally not issued after reads begin. +// +// Parameters: +// count: Passed to the constructor of the underlying std::vector of each +// VectorRep. On initialization, the underlying array will be at least count +// bytes reserved for usage. +class VectorRepFactory : public MemTableRepFactory { + size_t count_; + + public: + explicit VectorRepFactory(size_t count = 0); + + // Methods for Configurable/Customizable class overrides + static const char* kClassName() { return "VectorRepFactory"; } + static const char* kNickName() { return "vector"; } + const char* Name() const override { return kClassName(); } + const char* NickName() const override { return kNickName(); } + + // Methods for MemTableRepFactory class overrides + using MemTableRepFactory::CreateMemTableRep; + virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&, + Allocator*, const SliceTransform*, + Logger* logger) override; +}; + +// This class contains a fixed array of buckets, each +// pointing to a skiplist (null if the bucket is empty). +// bucket_count: number of fixed array buckets +// skiplist_height: the max height of the skiplist +// skiplist_branching_factor: probabilistic size ratio between adjacent +// link lists in the skiplist +extern MemTableRepFactory* NewHashSkipListRepFactory( + size_t bucket_count = 1000000, int32_t skiplist_height = 4, + int32_t skiplist_branching_factor = 4); + +// The factory is to create memtables based on a hash table: +// it contains a fixed array of buckets, each pointing to either a linked list +// or a skip list if number of entries inside the bucket exceeds +// threshold_use_skiplist. +// @bucket_count: number of fixed array buckets +// @huge_page_tlb_size: if <=0, allocate the hash table bytes from malloc. +// Otherwise from huge page TLB. The user needs to reserve +// huge pages for it to be allocated, like: +// sysctl -w vm.nr_hugepages=20 +// See linux doc Documentation/vm/hugetlbpage.txt +// @bucket_entries_logging_threshold: if number of entries in one bucket +// exceeds this number, log about it. +// @if_log_bucket_dist_when_flash: if true, log distribution of number of +// entries when flushing. +// @threshold_use_skiplist: a bucket switches to skip list if number of +// entries exceed this parameter. +extern MemTableRepFactory* NewHashLinkListRepFactory( + size_t bucket_count = 50000, size_t huge_page_tlb_size = 0, + int bucket_entries_logging_threshold = 4096, + bool if_log_bucket_dist_when_flash = true, + uint32_t threshold_use_skiplist = 256); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/merge_operator.h b/librocksdb-sys/rocksdb/include/rocksdb/merge_operator.h new file mode 100644 index 0000000..0771304 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/merge_operator.h @@ -0,0 +1,286 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include +#include + +#include "rocksdb/customizable.h" +#include "rocksdb/slice.h" + +namespace ROCKSDB_NAMESPACE { + +class Slice; +class Logger; + +// The Merge Operator +// +// Essentially, a MergeOperator specifies the SEMANTICS of a merge, which only +// client knows. It could be numeric addition, list append, string +// concatenation, edit data structure, ... , anything. +// The library, on the other hand, is concerned with the exercise of this +// interface, at the right time (during get, iteration, compaction...) +// +// To use merge, the client needs to provide an object implementing one of +// the following interfaces: +// a) AssociativeMergeOperator - for most simple semantics (always take +// two values, and merge them into one value, which is then put back +// into rocksdb); numeric addition and string concatenation are examples; +// +// b) MergeOperator - the generic class for all the more abstract / complex +// operations; one method (FullMergeV2) to merge a Put/Delete value with a +// merge operand; and another method (PartialMerge) that merges multiple +// operands together. this is especially useful if your key values have +// complex structures but you would still like to support client-specific +// incremental updates. +// +// AssociativeMergeOperator is simpler to implement. MergeOperator is simply +// more powerful. +// +// Refer to rocksdb-merge wiki for more details and example implementations. +// +// Exceptions MUST NOT propagate out of overridden functions into RocksDB, +// because RocksDB is not exception-safe. This could cause undefined behavior +// including data loss, unreported corruption, deadlocks, and more. +class MergeOperator : public Customizable { + public: + virtual ~MergeOperator() {} + static const char* Type() { return "MergeOperator"; } + static Status CreateFromString(const ConfigOptions& opts, + const std::string& id, + std::shared_ptr* result); + + // Gives the client a way to express the read -> modify -> write semantics + // key: (IN) The key that's associated with this merge operation. + // Client could multiplex the merge operator based on it + // if the key space is partitioned and different subspaces + // refer to different types of data which have different + // merge operation semantics + // existing: (IN) null indicates that the key does not exist before this op + // operand_list:(IN) the sequence of merge operations to apply, front() first. + // new_value:(OUT) Client is responsible for filling the merge result here. + // The string that new_value is pointing to will be empty. + // logger: (IN) Client could use this to log errors during merge. + // + // Return true on success. + // All values passed in will be client-specific values. So if this method + // returns false, it is because client specified bad data or there was + // internal corruption. This will be treated as an error by the library. + // + // Also make use of the *logger for error messages. + virtual bool FullMerge(const Slice& /*key*/, const Slice* /*existing_value*/, + const std::deque& /*operand_list*/, + std::string* /*new_value*/, Logger* /*logger*/) const { + // deprecated, please use FullMergeV2() + assert(false); + return false; + } + + struct MergeOperationInput { + // If user-defined timestamp is enabled, `_key` includes timestamp. + explicit MergeOperationInput(const Slice& _key, + const Slice* _existing_value, + const std::vector& _operand_list, + Logger* _logger) + : key(_key), + existing_value(_existing_value), + operand_list(_operand_list), + logger(_logger) {} + + // The key associated with the merge operation. + const Slice& key; + // The existing value of the current key, nullptr means that the + // value doesn't exist. + const Slice* existing_value; + // A list of operands to apply. + const std::vector& operand_list; + // Logger could be used by client to log any errors that happen during + // the merge operation. + Logger* logger; + }; + + enum class OpFailureScope { + kDefault, + kTryMerge, + kMustMerge, + kOpFailureScopeMax, + }; + + struct MergeOperationOutput { + explicit MergeOperationOutput(std::string& _new_value, + Slice& _existing_operand) + : new_value(_new_value), existing_operand(_existing_operand) {} + + // Client is responsible for filling the merge result here. + std::string& new_value; + // If the merge result is one of the existing operands (or existing_value), + // client can set this field to the operand (or existing_value) instead of + // using new_value. + Slice& existing_operand; + // Indicates the blast radius of the failure. It is only meaningful to + // provide a failure scope when returning `false` from the API populating + // the `MergeOperationOutput`. Currently RocksDB operations handle these + // values as follows: + // + // - `OpFailureScope::kDefault`: fallback to default + // (`OpFailureScope::kTryMerge`) + // - `OpFailureScope::kTryMerge`: operations that try to merge that key will + // fail. This includes flush and compaction, which puts the DB in + // read-only mode. + // - `OpFailureScope::kMustMerge`: operations that must merge that key will + // fail (e.g., `Get()`, `MultiGet()`, iteration). Flushes/compactions can + // still proceed by copying the original input operands to the output. + OpFailureScope op_failure_scope = OpFailureScope::kDefault; + }; + + // This function applies a stack of merge operands in chronological order + // on top of an existing value. There are two ways in which this method is + // being used: + // a) During Get() operation, it used to calculate the final value of a key + // b) During compaction, in order to collapse some operands with the based + // value. + // + // Note: The name of the method is somewhat misleading, as both in the cases + // of Get() or compaction it may be called on a subset of operands: + // K: 0 +1 +2 +7 +4 +5 2 +1 +2 + // ^ + // | + // snapshot + // In the example above, Get(K) operation will call FullMerge with a base + // value of 2 and operands [+1, +2]. Compaction process might decide to + // collapse the beginning of the history up to the snapshot by performing + // full Merge with base value of 0 and operands [+1, +2, +7, +4]. + virtual bool FullMergeV2(const MergeOperationInput& merge_in, + MergeOperationOutput* merge_out) const; + + // This function performs merge(left_op, right_op) + // when both the operands are themselves merge operation types + // that you would have passed to a DB::Merge() call in the same order + // (i.e.: DB::Merge(key,left_op), followed by DB::Merge(key,right_op)). + // + // PartialMerge should combine them into a single merge operation that is + // saved into *new_value, and then it should return true. + // *new_value should be constructed such that a call to + // DB::Merge(key, *new_value) would yield the same result as a call + // to DB::Merge(key, left_op) followed by DB::Merge(key, right_op). + // + // The string that new_value is pointing to will be empty. + // + // The default implementation of PartialMergeMulti will use this function + // as a helper, for backward compatibility. Any successor class of + // MergeOperator should either implement PartialMerge or PartialMergeMulti, + // although implementing PartialMergeMulti is suggested as it is in general + // more effective to merge multiple operands at a time instead of two + // operands at a time. + // + // If it is impossible or infeasible to combine the two operations, + // leave new_value unchanged and return false. The library will + // internally keep track of the operations, and apply them in the + // correct order once a base-value (a Put/Delete/End-of-Database) is seen. + // + // TODO: Presently there is no way to differentiate between error/corruption + // and simply "return false". For now, the client should simply return + // false in any case it cannot perform partial-merge, regardless of reason. + // If there is corruption in the data, handle it in the FullMergeV2() function + // and return false there. The default implementation of PartialMerge will + // always return false. + virtual bool PartialMerge(const Slice& /*key*/, const Slice& /*left_operand*/, + const Slice& /*right_operand*/, + std::string* /*new_value*/, + Logger* /*logger*/) const { + return false; + } + + // This function performs merge when all the operands are themselves merge + // operation types that you would have passed to a DB::Merge() call in the + // same order (front() first) + // (i.e. DB::Merge(key, operand_list[0]), followed by + // DB::Merge(key, operand_list[1]), ...) + // + // PartialMergeMulti should combine them into a single merge operation that is + // saved into *new_value, and then it should return true. *new_value should + // be constructed such that a call to DB::Merge(key, *new_value) would yield + // the same result as sequential individual calls to DB::Merge(key, operand) + // for each operand in operand_list from front() to back(). + // + // The string that new_value is pointing to will be empty. + // + // The PartialMergeMulti function will be called when there are at least two + // operands. + // + // In the default implementation, PartialMergeMulti will invoke PartialMerge + // multiple times, where each time it only merges two operands. Developers + // should either implement PartialMergeMulti, or implement PartialMerge which + // is served as the helper function of the default PartialMergeMulti. + virtual bool PartialMergeMulti(const Slice& key, + const std::deque& operand_list, + std::string* new_value, Logger* logger) const; + + // The name of the MergeOperator. Used to check for MergeOperator + // mismatches (i.e., a DB created with one MergeOperator is + // accessed using a different MergeOperator) + // TODO: the name is currently not stored persistently and thus + // no checking is enforced. Client is responsible for providing + // consistent MergeOperator between DB opens. + virtual const char* Name() const override = 0; + + // Determines whether the PartialMerge can be called with just a single + // merge operand. + // Override and return true for allowing a single operand. PartialMerge + // and PartialMergeMulti should be overridden and implemented + // correctly to properly handle a single operand. + virtual bool AllowSingleOperand() const { return false; } + + // Allows to control when to invoke a full merge during Get. + // This could be used to limit the number of merge operands that are looked at + // during a point lookup, thereby helping in limiting the number of levels to + // read from. + // Doesn't help with iterators. + // + // Note: the merge operands are passed to this function in the reversed order + // relative to how they were merged (passed to FullMerge or FullMergeV2) + // for performance reasons, see also: + // https://github.com/facebook/rocksdb/issues/3865 + virtual bool ShouldMerge(const std::vector& /*operands*/) const { + return false; + } +}; + +// The simpler, associative merge operator. +class AssociativeMergeOperator : public MergeOperator { + public: + ~AssociativeMergeOperator() override {} + + // Gives the client a way to express the read -> modify -> write semantics + // key: (IN) The key that's associated with this merge operation. + // existing_value:(IN) null indicates the key does not exist before this op + // value: (IN) the value to update/merge the existing_value with + // new_value: (OUT) Client is responsible for filling the merge result + // here. The string that new_value is pointing to will be empty. + // logger: (IN) Client could use this to log errors during merge. + // + // Return true on success. + // All values passed in will be client-specific values. So if this method + // returns false, it is because client specified bad data or there was + // internal corruption. The client should assume that this will be treated + // as an error by the library. + virtual bool Merge(const Slice& key, const Slice* existing_value, + const Slice& value, std::string* new_value, + Logger* logger) const = 0; + + private: + // Default implementations of the MergeOperator functions + bool FullMergeV2(const MergeOperationInput& merge_in, + MergeOperationOutput* merge_out) const override; + + bool PartialMerge(const Slice& key, const Slice& left_operand, + const Slice& right_operand, std::string* new_value, + Logger* logger) const override; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/metadata.h b/librocksdb-sys/rocksdb/include/rocksdb/metadata.h new file mode 100644 index 0000000..4ab3842 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/metadata.h @@ -0,0 +1,258 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include +#include +#include + +#include "rocksdb/options.h" +#include "rocksdb/types.h" + +namespace ROCKSDB_NAMESPACE { + +// Basic identifiers and metadata for a file in a DB. This only includes +// information considered relevant for taking backups, checkpoints, or other +// services relating to DB file storage. +// This is only appropriate for immutable files, such as SST files or all +// files in a backup. See also LiveFileStorageInfo. +struct FileStorageInfo { + // The name of the file within its directory (e.g. "123456.sst") + std::string relative_filename; + // The directory containing the file, without a trailing '/'. This could be + // a DB path, wal_dir, etc. + std::string directory; + + // The id of the file within a single DB. Set to 0 if the file does not have + // a number (e.g. CURRENT) + uint64_t file_number = 0; + // The type of the file as part of a DB. + FileType file_type = kTempFile; + + // File size in bytes. See also `trim_to_size`. + uint64_t size = 0; + + // This feature is experimental and subject to change. + Temperature temperature = Temperature::kUnknown; + + // The checksum of a SST file, the value is decided by the file content and + // the checksum algorithm used for this SST file. The checksum function is + // identified by the file_checksum_func_name. If the checksum function is + // not specified, file_checksum is "0" by default. + std::string file_checksum; + + // The name of the checksum function used to generate the file checksum + // value. If file checksum is not enabled (e.g., sst_file_checksum_func is + // null), file_checksum_func_name is UnknownFileChecksumFuncName, which is + // "Unknown". + std::string file_checksum_func_name; +}; + +// Adds to FileStorageInfo the ability to capture the state of files that +// might change in a running DB. +struct LiveFileStorageInfo : public FileStorageInfo { + // If non-empty, this string represents the "saved" contents of the file + // for the current context. (This field is used for checkpointing CURRENT + // file.) In that case, size == replacement_contents.size() and file on disk + // should be ignored. If empty string, the file on disk should still have + // "saved" contents. (See trim_to_size.) + std::string replacement_contents; + + // If true, the file on disk is allowed to be larger than `size` but only + // the first `size` bytes should be used for the current context. If false, + // the file is corrupt if size on disk does not equal `size`. + bool trim_to_size = false; +}; + +// The metadata that describes an SST file. (Does not need to extend +// LiveFileStorageInfo because SST files are always immutable.) +struct SstFileMetaData : public FileStorageInfo { + SstFileMetaData() { file_type = kTableFile; } + + SstFileMetaData(const std::string& _file_name, uint64_t _file_number, + const std::string& _directory, uint64_t _size, + SequenceNumber _smallest_seqno, SequenceNumber _largest_seqno, + const std::string& _smallestkey, + const std::string& _largestkey, uint64_t _num_reads_sampled, + bool _being_compacted, Temperature _temperature, + uint64_t _oldest_blob_file_number, + uint64_t _oldest_ancester_time, uint64_t _file_creation_time, + uint64_t _epoch_number, std::string& _file_checksum, + std::string& _file_checksum_func_name) + : smallest_seqno(_smallest_seqno), + largest_seqno(_largest_seqno), + smallestkey(_smallestkey), + largestkey(_largestkey), + num_reads_sampled(_num_reads_sampled), + being_compacted(_being_compacted), + num_entries(0), + num_deletions(0), + oldest_blob_file_number(_oldest_blob_file_number), + oldest_ancester_time(_oldest_ancester_time), + file_creation_time(_file_creation_time), + epoch_number(_epoch_number) { + if (!_file_name.empty()) { + if (_file_name[0] == '/') { + relative_filename = _file_name.substr(1); + name = _file_name; // Deprecated field + } else { + relative_filename = _file_name; + name = std::string("/") + _file_name; // Deprecated field + } + assert(relative_filename.size() + 1 == name.size()); + assert(relative_filename[0] != '/'); + assert(name[0] == '/'); + } + directory = _directory; + db_path = _directory; // Deprecated field + file_number = _file_number; + file_type = kTableFile; + size = _size; + temperature = _temperature; + file_checksum = _file_checksum; + file_checksum_func_name = _file_checksum_func_name; + } + + SequenceNumber smallest_seqno = 0; // Smallest sequence number in file. + SequenceNumber largest_seqno = 0; // Largest sequence number in file. + std::string smallestkey; // Smallest user defined key in the file. + std::string largestkey; // Largest user defined key in the file. + uint64_t num_reads_sampled = 0; // How many times the file is read. + bool being_compacted = + false; // true if the file is currently being compacted. + + uint64_t num_entries = 0; + uint64_t num_deletions = 0; + + uint64_t oldest_blob_file_number = 0; // The id of the oldest blob file + // referenced by the file. + // An SST file may be generated by compactions whose input files may + // in turn be generated by earlier compactions. The creation time of the + // oldest SST file that is the compaction ancestor of this file. + // The timestamp is provided SystemClock::GetCurrentTime(). + // 0 if the information is not available. + // + // Note: for TTL blob files, it contains the start of the expiration range. + uint64_t oldest_ancester_time = 0; + // Timestamp when the SST file is created, provided by + // SystemClock::GetCurrentTime(). 0 if the information is not available. + uint64_t file_creation_time = 0; + // The order of a file being flushed or ingested/imported. + // Compaction output file will be assigned with the minimum `epoch_number` + // among input files'. + // For L0, larger `epoch_number` indicates newer L0 file. + // 0 if the information is not available. + uint64_t epoch_number = 0; + + // These bounds define the effective key range for range tombstones + // in this file. + // Currently only used by CreateColumnFamilyWithImport(). + std::string smallest{}; // Smallest internal key served by table + std::string largest{}; // Largest internal key served by table + + // DEPRECATED: The name of the file within its directory with a + // leading slash (e.g. "/123456.sst"). Use relative_filename from base struct + // instead. + std::string name; + + // DEPRECATED: replaced by `directory` in base struct + std::string db_path; +}; + +// The full set of metadata associated with each SST file. +struct LiveFileMetaData : SstFileMetaData { + std::string column_family_name; // Name of the column family + int level; // Level at which this file resides. + LiveFileMetaData() : column_family_name(), level(0) {} +}; + +// The MetaData that describes a Blob file +struct BlobMetaData { + BlobMetaData() + : blob_file_number(0), + blob_file_size(0), + total_blob_count(0), + total_blob_bytes(0), + garbage_blob_count(0), + garbage_blob_bytes(0) {} + + BlobMetaData(uint64_t _file_number, const std::string& _file_name, + const std::string& _file_path, uint64_t _file_size, + uint64_t _total_blob_count, uint64_t _total_blob_bytes, + uint64_t _garbage_blob_count, uint64_t _garbage_blob_bytes, + const std::string& _file_checksum, + const std::string& _file_checksum_func_name) + : blob_file_number(_file_number), + blob_file_name(_file_name), + blob_file_path(_file_path), + blob_file_size(_file_size), + total_blob_count(_total_blob_count), + total_blob_bytes(_total_blob_bytes), + garbage_blob_count(_garbage_blob_count), + garbage_blob_bytes(_garbage_blob_bytes), + checksum_method(_file_checksum), + checksum_value(_file_checksum_func_name) {} + uint64_t blob_file_number; + std::string blob_file_name; + std::string blob_file_path; + uint64_t blob_file_size; + uint64_t total_blob_count; + uint64_t total_blob_bytes; + uint64_t garbage_blob_count; + uint64_t garbage_blob_bytes; + std::string checksum_method; + std::string checksum_value; +}; + +// The metadata that describes a level. +struct LevelMetaData { + LevelMetaData(int _level, uint64_t _size, + const std::vector&& _files) + : level(_level), size(_size), files(_files) {} + + // The level which this meta data describes. + const int level; + // The size of this level in bytes, which is equal to the sum of + // the file size of its "files". + const uint64_t size; + // The metadata of all sst files in this level. + const std::vector files; +}; + +// The metadata that describes a column family. +struct ColumnFamilyMetaData { + ColumnFamilyMetaData() : size(0), file_count(0), name("") {} + ColumnFamilyMetaData(const std::string& _name, uint64_t _size, + const std::vector&& _levels) + : size(_size), name(_name), levels(_levels) {} + + // The size of this column family in bytes, which is equal to the sum of + // the file size of its "levels". + uint64_t size; + // The number of files in this column family. + size_t file_count; + // The name of the column family. + std::string name; + // The metadata of all levels in this column family. + std::vector levels; + + // The total size of all blob files + uint64_t blob_file_size = 0; + // The number of blob files in this column family. + size_t blob_file_count = 0; + // The metadata of the blobs in this column family. + std::vector blob_files; +}; + +// Metadata returned as output from ExportColumnFamily() and used as input to +// CreateColumnFamiliesWithImport(). +struct ExportImportFilesMetaData { + std::string db_comparator_name; // Used to safety check at import. + std::vector files; // Vector of file metadata. +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/options.h b/librocksdb-sys/rocksdb/include/rocksdb/options.h new file mode 100644 index 0000000..7fac243 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/options.h @@ -0,0 +1,2130 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +#include "rocksdb/advanced_options.h" +#include "rocksdb/comparator.h" +#include "rocksdb/compression_type.h" +#include "rocksdb/customizable.h" +#include "rocksdb/data_structure.h" +#include "rocksdb/env.h" +#include "rocksdb/file_checksum.h" +#include "rocksdb/listener.h" +#include "rocksdb/sst_partitioner.h" +#include "rocksdb/types.h" +#include "rocksdb/universal_compaction.h" +#include "rocksdb/version.h" +#include "rocksdb/write_buffer_manager.h" + +#ifdef max +#undef max +#endif + +namespace ROCKSDB_NAMESPACE { + +class Cache; +class CompactionFilter; +class CompactionFilterFactory; +class Comparator; +class ConcurrentTaskLimiter; +class Env; +enum InfoLogLevel : unsigned char; +class SstFileManager; +class FilterPolicy; +class Logger; +class MergeOperator; +class Snapshot; +class MemTableRepFactory; +class RateLimiter; +class Slice; +class Statistics; +class InternalKeyComparator; +class WalFilter; +class FileSystem; + +struct Options; +struct DbPath; + +using FileTypeSet = SmallEnumSet; + +struct ColumnFamilyOptions : public AdvancedColumnFamilyOptions { + // The function recovers options to a previous version. Only 4.6 or later + // versions are supported. + // NOT MAINTAINED: This function has not been and is not maintained. + // DEPRECATED: This function might be removed in a future release. + // In general, defaults are changed to suit broad interests. Opting + // out of a change on upgrade should be deliberate and considered. + ColumnFamilyOptions* OldDefaults(int rocksdb_major_version = 4, + int rocksdb_minor_version = 6); + + // Some functions that make it easier to optimize RocksDB + // Use this if your DB is very small (like under 1GB) and you don't want to + // spend lots of memory for memtables. + // An optional cache object is passed in to be used as the block cache + ColumnFamilyOptions* OptimizeForSmallDb( + std::shared_ptr* cache = nullptr); + + // Use this if you don't need to keep the data sorted, i.e. you'll never use + // an iterator, only Put() and Get() API calls + // + ColumnFamilyOptions* OptimizeForPointLookup(uint64_t block_cache_size_mb); + + // Default values for some parameters in ColumnFamilyOptions are not + // optimized for heavy workloads and big datasets, which means you might + // observe write stalls under some conditions. As a starting point for tuning + // RocksDB options, use the following two functions: + // * OptimizeLevelStyleCompaction -- optimizes level style compaction + // * OptimizeUniversalStyleCompaction -- optimizes universal style compaction + // Universal style compaction is focused on reducing Write Amplification + // Factor for big data sets, but increases Space Amplification. You can learn + // more about the different styles here: + // https://github.com/facebook/rocksdb/wiki/Rocksdb-Architecture-Guide + // Make sure to also call IncreaseParallelism(), which will provide the + // biggest performance gains. + // Note: we might use more memory than memtable_memory_budget during high + // write rate period + ColumnFamilyOptions* OptimizeLevelStyleCompaction( + uint64_t memtable_memory_budget = 512 * 1024 * 1024); + ColumnFamilyOptions* OptimizeUniversalStyleCompaction( + uint64_t memtable_memory_budget = 512 * 1024 * 1024); + + // ------------------- + // Parameters that affect behavior + + // Comparator used to define the order of keys in the table. + // Default: a comparator that uses lexicographic byte-wise ordering + // + // REQUIRES: The client must ensure that the comparator supplied + // here has the same name and orders keys *exactly* the same as the + // comparator provided to previous open calls on the same DB. + const Comparator* comparator = BytewiseComparator(); + + // REQUIRES: The client must provide a merge operator if Merge operation + // needs to be accessed. Calling Merge on a DB without a merge operator + // would result in Status::NotSupported. The client must ensure that the + // merge operator supplied here has the same name and *exactly* the same + // semantics as the merge operator provided to previous open calls on + // the same DB. The only exception is reserved for upgrade, where a DB + // previously without a merge operator is introduced to Merge operation + // for the first time. It's necessary to specify a merge operator when + // opening the DB in this case. + // Default: nullptr + std::shared_ptr merge_operator = nullptr; + + // A single CompactionFilter instance to call into during compaction. + // Allows an application to modify/delete a key-value during background + // compaction. + // + // If the client requires a new `CompactionFilter` to be used for different + // compaction runs and/or requires a `CompactionFilter` for table file + // creations outside of compaction, it can specify compaction_filter_factory + // instead of this option. The client should specify only one of the two. + // compaction_filter takes precedence over compaction_filter_factory if + // client specifies both. + // + // If multithreaded compaction is being used, the supplied CompactionFilter + // instance may be used from different threads concurrently and so should be + // thread-safe. + // + // Default: nullptr + const CompactionFilter* compaction_filter = nullptr; + + // This is a factory that provides `CompactionFilter` objects which allow + // an application to modify/delete a key-value during table file creation. + // + // Unlike the `compaction_filter` option, which is used when compaction + // creates a table file, this factory allows using a `CompactionFilter` when a + // table file is created for various reasons. The factory can decide what + // `TableFileCreationReason`s use a `CompactionFilter`. For compatibility, by + // default the decision is to use a `CompactionFilter` for + // `TableFileCreationReason::kCompaction` only. + // + // Each thread of work involving creating table files will create a new + // `CompactionFilter` when it will be used according to the above + // `TableFileCreationReason`-based decision. This allows the application to + // know about the different ongoing threads of work and makes it unnecessary + // for `CompactionFilter` to provide thread-safety. + // + // Default: nullptr + std::shared_ptr compaction_filter_factory = nullptr; + + // ------------------- + // Parameters that affect performance + + // Amount of data to build up in memory (backed by an unsorted log + // on disk) before converting to a sorted on-disk file. + // + // Larger values increase performance, especially during bulk loads. + // Up to max_write_buffer_number write buffers may be held in memory + // at the same time, + // so you may wish to adjust this parameter to control memory usage. + // Also, a larger write buffer will result in a longer recovery time + // the next time the database is opened. + // + // Note that write_buffer_size is enforced per column family. + // See db_write_buffer_size for sharing memory across column families. + // + // Default: 64MB + // + // Dynamically changeable through SetOptions() API + size_t write_buffer_size = 64 << 20; + + // Compress blocks using the specified compression algorithm. + // + // Default: kSnappyCompression, if it's supported. If snappy is not linked + // with the library, the default is kNoCompression. + // + // Typical speeds of kSnappyCompression on an Intel(R) Core(TM)2 2.4GHz: + // ~200-500MB/s compression + // ~400-800MB/s decompression + // + // Note that these speeds are significantly faster than most + // persistent storage speeds, and therefore it is typically never + // worth switching to kNoCompression. Even if the input data is + // incompressible, the kSnappyCompression implementation will + // efficiently detect that and will switch to uncompressed mode. + // + // If you do not set `compression_opts.level`, or set it to + // `CompressionOptions::kDefaultCompressionLevel`, we will attempt to pick the + // default corresponding to `compression` as follows: + // + // - kZSTD: 3 + // - kZlibCompression: Z_DEFAULT_COMPRESSION (currently -1) + // - kLZ4HCCompression: 0 + // - For all others, we do not specify a compression level + // + // Dynamically changeable through SetOptions() API + CompressionType compression; + + // Compression algorithm that will be used for the bottommost level that + // contain files. The behavior for num_levels = 1 is not well defined. + // Right now, with num_levels = 1, all compaction outputs will use + // bottommost_compression and all flush outputs still use options.compression, + // but the behavior is subject to change. + // + // Default: kDisableCompressionOption (Disabled) + CompressionType bottommost_compression = kDisableCompressionOption; + + // different options for compression algorithms used by bottommost_compression + // if it is enabled. To enable it, please see the definition of + // CompressionOptions. Behavior for num_levels = 1 is the same as + // options.bottommost_compression. + CompressionOptions bottommost_compression_opts; + + // different options for compression algorithms + CompressionOptions compression_opts; + + // Number of files to trigger level-0 compaction. A value <0 means that + // level-0 compaction will not be triggered by number of files at all. + // + // Default: 4 + // + // Dynamically changeable through SetOptions() API + int level0_file_num_compaction_trigger = 4; + + // If non-nullptr, use the specified function to put keys in contiguous + // groups called "prefixes". These prefixes are used to place one + // representative entry for the group into the Bloom filter + // rather than an entry for each key (see whole_key_filtering). + // Under certain conditions, this enables optimizing some range queries + // (Iterators) in addition to some point lookups (Get/MultiGet). + // + // Together `prefix_extractor` and `comparator` must satisfy one essential + // property for valid prefix filtering of range queries: + // If Compare(k1, k2) <= 0 and Compare(k2, k3) <= 0 and + // InDomain(k1) and InDomain(k3) and prefix(k1) == prefix(k3), + // Then InDomain(k2) and prefix(k2) == prefix(k1) + // + // In other words, all keys with the same prefix must be in a contiguous + // group by comparator order, and cannot be interrupted by keys with no + // prefix ("out of domain"). (This makes it valid to conclude that no + // entries within some bounds are present if the upper and lower bounds + // have a common prefix and no entries with that same prefix are present.) + // + // Some other properties are recommended but not strictly required. Under + // most sensible comparators, the following will need to hold true to + // satisfy the essential property above: + // * "Prefix is a prefix": key.starts_with(prefix(key)) + // * "Prefixes preserve ordering": If Compare(k1, k2) <= 0, then + // Compare(prefix(k1), prefix(k2)) <= 0 + // + // The next two properties ensure that seeking to a prefix allows + // enumerating all entries with that prefix: + // * "Prefix starts the group": Compare(prefix(key), key) <= 0 + // * "Prefix idempotent": prefix(prefix(key)) == prefix(key) + // + // Default: nullptr + std::shared_ptr prefix_extractor = nullptr; + + // Control maximum total data size for a level. + // max_bytes_for_level_base is the max total for level-1. + // Maximum number of bytes for level L can be calculated as + // (max_bytes_for_level_base) * (max_bytes_for_level_multiplier ^ (L-1)) + // For example, if max_bytes_for_level_base is 200MB, and if + // max_bytes_for_level_multiplier is 10, total data size for level-1 + // will be 200MB, total file size for level-2 will be 2GB, + // and total file size for level-3 will be 20GB. + // + // Default: 256MB. + // + // Dynamically changeable through SetOptions() API + uint64_t max_bytes_for_level_base = 256 * 1048576; + + // Deprecated. + uint64_t snap_refresh_nanos = 0; + + // Disable automatic compactions. Manual compactions can still + // be issued on this column family + // + // Dynamically changeable through SetOptions() API + bool disable_auto_compactions = false; + + // This is a factory that provides TableFactory objects. + // Default: a block-based table factory that provides a default + // implementation of TableBuilder and TableReader with default + // BlockBasedTableOptions. + std::shared_ptr table_factory; + + // A list of paths where SST files for this column family + // can be put into, with its target size. Similar to db_paths, + // newer data is placed into paths specified earlier in the + // vector while older data gradually moves to paths specified + // later in the vector. + // Note that, if a path is supplied to multiple column + // families, it would have files and total size from all + // the column families combined. User should provision for the + // total size(from all the column families) in such cases. + // + // If left empty, db_paths will be used. + // Default: empty + std::vector cf_paths; + + // Compaction concurrent thread limiter for the column family. + // If non-nullptr, use given concurrent thread limiter to control + // the max outstanding compaction tasks. Limiter can be shared with + // multiple column families across db instances. + // + // Default: nullptr + std::shared_ptr compaction_thread_limiter = nullptr; + + // If non-nullptr, use the specified factory for a function to determine the + // partitioning of sst files. This helps compaction to split the files + // on interesting boundaries (key prefixes) to make propagation of sst + // files less write amplifying (covering the whole key space). + // THE FEATURE IS STILL EXPERIMENTAL + // + // Default: nullptr + std::shared_ptr sst_partitioner_factory = nullptr; + + // RocksDB will try to flush the current memtable after the number of range + // deletions is >= this limit. For workloads with many range + // deletions, limiting the number of range deletions in memtable can help + // prevent performance degradation and/or OOM caused by too many range + // tombstones in a single memtable. + // + // Default: 0 (disabled) + // + // Dynamically changeable through SetOptions() API + uint32_t memtable_max_range_deletions = 0; + + // Create ColumnFamilyOptions with default values for all fields + ColumnFamilyOptions(); + // Create ColumnFamilyOptions from Options + explicit ColumnFamilyOptions(const Options& options); + + void Dump(Logger* log) const; +}; + +enum class WALRecoveryMode : char { + // Original levelDB recovery + // + // We tolerate the last record in any log to be incomplete due to a crash + // while writing it. Zeroed bytes from preallocation are also tolerated in the + // trailing data of any log. + // + // Use case: Applications for which updates, once applied, must not be rolled + // back even after a crash-recovery. In this recovery mode, RocksDB guarantees + // this as long as `WritableFile::Append()` writes are durable. In case the + // user needs the guarantee in more situations (e.g., when + // `WritableFile::Append()` writes to page cache, but the user desires this + // guarantee in face of power-loss crash-recovery), RocksDB offers various + // mechanisms to additionally invoke `WritableFile::Sync()` in order to + // strengthen the guarantee. + // + // This differs from `kPointInTimeRecovery` in that, in case a corruption is + // detected during recovery, this mode will refuse to open the DB. Whereas, + // `kPointInTimeRecovery` will stop recovery just before the corruption since + // that is a valid point-in-time to which to recover. + kTolerateCorruptedTailRecords = 0x00, + // Recover from clean shutdown + // We don't expect to find any corruption in the WAL + // Use case : This is ideal for unit tests and rare applications that + // can require high consistency guarantee + kAbsoluteConsistency = 0x01, + // Recover to point-in-time consistency (default) + // We stop the WAL playback on discovering WAL inconsistency + // Use case : Ideal for systems that have disk controller cache like + // hard disk, SSD without super capacitor that store related data + kPointInTimeRecovery = 0x02, + // Recovery after a disaster + // We ignore any corruption in the WAL and try to salvage as much data as + // possible + // Use case : Ideal for last ditch effort to recover data or systems that + // operate with low grade unrelated data + kSkipAnyCorruptedRecords = 0x03, +}; + +struct DbPath { + std::string path; + uint64_t target_size; // Target size of total files under the path, in byte. + + DbPath() : target_size(0) {} + DbPath(const std::string& p, uint64_t t) : path(p), target_size(t) {} +}; + +extern const char* kHostnameForDbHostId; + +enum class CompactionServiceJobStatus : char { + kSuccess, + kFailure, + kUseLocal, +}; + +struct CompactionServiceJobInfo { + std::string db_name; + std::string db_id; + std::string db_session_id; + uint64_t job_id; // job_id is only unique within the current DB and session, + // restart DB will reset the job_id. `db_id` and + // `db_session_id` could help you build unique id across + // different DBs and sessions. + + Env::Priority priority; + + CompactionServiceJobInfo(std::string db_name_, std::string db_id_, + std::string db_session_id_, uint64_t job_id_, + Env::Priority priority_) + : db_name(std::move(db_name_)), + db_id(std::move(db_id_)), + db_session_id(std::move(db_session_id_)), + job_id(job_id_), + priority(priority_) {} +}; + +// Exceptions MUST NOT propagate out of overridden functions into RocksDB, +// because RocksDB is not exception-safe. This could cause undefined behavior +// including data loss, unreported corruption, deadlocks, and more. +class CompactionService : public Customizable { + public: + static const char* Type() { return "CompactionService"; } + + // Returns the name of this compaction service. + const char* Name() const override = 0; + + // Start the remote compaction with `compaction_service_input`, which can be + // passed to `DB::OpenAndCompact()` on the remote side. `info` provides the + // information the user might want to know, which includes `job_id`. + virtual CompactionServiceJobStatus StartV2( + const CompactionServiceJobInfo& /*info*/, + const std::string& /*compaction_service_input*/) { + return CompactionServiceJobStatus::kUseLocal; + } + + // Wait for remote compaction to finish. + virtual CompactionServiceJobStatus WaitForCompleteV2( + const CompactionServiceJobInfo& /*info*/, + std::string* /*compaction_service_result*/) { + return CompactionServiceJobStatus::kUseLocal; + } + + ~CompactionService() override = default; +}; + +struct DBOptions { + // The function recovers options to the option as in version 4.6. + // NOT MAINTAINED: This function has not been and is not maintained. + // DEPRECATED: This function might be removed in a future release. + // In general, defaults are changed to suit broad interests. Opting + // out of a change on upgrade should be deliberate and considered. + DBOptions* OldDefaults(int rocksdb_major_version = 4, + int rocksdb_minor_version = 6); + + // Some functions that make it easier to optimize RocksDB + + // Use this if your DB is very small (like under 1GB) and you don't want to + // spend lots of memory for memtables. + // An optional cache object is passed in for the memory of the + // memtable to cost to + DBOptions* OptimizeForSmallDb(std::shared_ptr* cache = nullptr); + + // By default, RocksDB uses only one background thread for flush and + // compaction. Calling this function will set it up such that total of + // `total_threads` is used. Good value for `total_threads` is the number of + // cores. You almost definitely want to call this function if your system is + // bottlenecked by RocksDB. + DBOptions* IncreaseParallelism(int total_threads = 16); + + // If true, the database will be created if it is missing. + // Default: false + bool create_if_missing = false; + + // If true, missing column families will be automatically created. + // Default: false + bool create_missing_column_families = false; + + // If true, an error is raised if the database already exists. + // Default: false + bool error_if_exists = false; + + // If true, RocksDB will aggressively check consistency of the data. + // Also, if any of the writes to the database fails (Put, Delete, Merge, + // Write), the database will switch to read-only mode and fail all other + // Write operations. + // In most cases you want this to be set to true. + // Default: true + bool paranoid_checks = true; + + // If true, during memtable flush, RocksDB will validate total entries + // read in flush, and compare with counter inserted into it. + // + // The option is here to turn the feature off in case this new validation + // feature has a bug. The option may be removed in the future once the + // feature is stable. + // + // Default: true + bool flush_verify_memtable_count = true; + + // If true, during compaction, RocksDB will count the number of entries + // read and compare it against the number of entries in the compaction + // input files. This is intended to add protection against corruption + // during compaction. Note that + // - this verification is not done for compactions during which a compaction + // filter returns kRemoveAndSkipUntil, and + // - the number of range deletions is not verified. + // + // The option is here to turn the feature off in case this new validation + // feature has a bug. The option may be removed in the future once the + // feature is stable. + // + // Default: true + bool compaction_verify_record_count = true; + + // If true, the log numbers and sizes of the synced WALs are tracked + // in MANIFEST. During DB recovery, if a synced WAL is missing + // from disk, or the WAL's size does not match the recorded size in + // MANIFEST, an error will be reported and the recovery will be aborted. + // + // This is one additional protection against WAL corruption besides the + // per-WAL-entry checksum. + // + // Note that this option does not work with secondary instance. + // Currently, only syncing closed WALs are tracked. Calling `DB::SyncWAL()`, + // etc. or writing with `WriteOptions::sync=true` to sync the live WAL is not + // tracked for performance/efficiency reasons. + // + // Default: false + bool track_and_verify_wals_in_manifest = false; + + // If true, verifies the SST unique id between MANIFEST and actual file + // each time an SST file is opened. This check ensures an SST file is not + // overwritten or misplaced. A corruption error will be reported if mismatch + // detected, but only when MANIFEST tracks the unique id, which starts from + // RocksDB version 7.3. Although the tracked internal unique id is related + // to the one returned by GetUniqueIdFromTableProperties, that is subject to + // change. + // NOTE: verification is currently only done on SST files using block-based + // table format. + // + // Setting to false should only be needed in case of unexpected problems. + // + // Although an early version of this option opened all SST files for + // verification on DB::Open, that is no longer guaranteed. However, as + // documented in an above option, if max_open_files is -1, DB will open all + // files on DB::Open(). + // + // Default: true + bool verify_sst_unique_id_in_manifest = true; + + // Use the specified object to interact with the environment, + // e.g. to read/write files, schedule background work, etc. In the near + // future, support for doing storage operations such as read/write files + // through env will be deprecated in favor of file_system (see below) + // Default: Env::Default() + Env* env = Env::Default(); + + // Limits internal file read/write bandwidth: + // + // - Flush requests write bandwidth at `Env::IOPriority::IO_HIGH` + // - Compaction requests read and write bandwidth at + // `Env::IOPriority::IO_LOW` + // - Reads associated with a `ReadOptions` can be charged at + // `ReadOptions::rate_limiter_priority` (see that option's API doc for usage + // and limitations). + // - Writes associated with a `WriteOptions` can be charged at + // `WriteOptions::rate_limiter_priority` (see that option's API doc for + // usage and limitations). + // + // Rate limiting is disabled if nullptr. If rate limiter is enabled, + // bytes_per_sync is set to 1MB by default. + // + // Default: nullptr + std::shared_ptr rate_limiter = nullptr; + + // Use to track SST files and control their file deletion rate. + // + // Features: + // - Throttle the deletion rate of the SST files. + // - Keep track the total size of all SST files. + // - Set a maximum allowed space limit for SST files that when reached + // the DB wont do any further flushes or compactions and will set the + // background error. + // - Can be shared between multiple dbs. + // Limitations: + // - Only track and throttle deletes of SST files in + // first db_path (db_name if db_paths is empty). + // + // Default: nullptr + std::shared_ptr sst_file_manager = nullptr; + + // Any internal progress/error information generated by the db will + // be written to info_log if it is non-nullptr, or to a file stored + // in the same directory as the DB contents if info_log is nullptr. + // Default: nullptr + std::shared_ptr info_log = nullptr; + +#ifdef NDEBUG + InfoLogLevel info_log_level = INFO_LEVEL; +#else + InfoLogLevel info_log_level = DEBUG_LEVEL; +#endif // NDEBUG + + // Number of open files that can be used by the DB. You may need to + // increase this if your database has a large working set. Value -1 means + // files opened are always kept open. You can estimate number of files based + // on target_file_size_base and target_file_size_multiplier for level-based + // compaction. For universal-style compaction, you can usually set it to -1. + // + // A high value or -1 for this option can cause high memory usage. + // See BlockBasedTableOptions::cache_usage_options to constrain + // memory usage in case of block based table format. + // + // Default: -1 + // + // Dynamically changeable through SetDBOptions() API. + int max_open_files = -1; + + // If max_open_files is -1, DB will open all files on DB::Open(). You can + // use this option to increase the number of threads used to open the files. + // Default: 16 + int max_file_opening_threads = 16; + + // Once write-ahead logs exceed this size, we will start forcing the flush of + // column families whose memtables are backed by the oldest live WAL file + // (i.e. the ones that are causing all the space amplification). If set to 0 + // (default), we will dynamically choose the WAL size limit to be + // [sum of all write_buffer_size * max_write_buffer_number] * 4 + // + // For example, with 15 column families, each with + // write_buffer_size = 128 MB + // max_write_buffer_number = 6 + // max_total_wal_size will be calculated to be [15 * 128MB * 6] * 4 = 45GB + // + // The RocksDB wiki has some discussion about how the WAL interacts + // with memtables and flushing of column families. + // https://github.com/facebook/rocksdb/wiki/Column-Families + // + // This option takes effect only when there are more than one column + // family as otherwise the wal size is dictated by the write_buffer_size. + // + // Default: 0 + // + // Dynamically changeable through SetDBOptions() API. + uint64_t max_total_wal_size = 0; + + // If non-null, then we should collect metrics about database operations + std::shared_ptr statistics = nullptr; + + // By default, writes to stable storage use fdatasync (on platforms + // where this function is available). If this option is true, + // fsync is used instead. + // + // fsync and fdatasync are equally safe for our purposes and fdatasync is + // faster, so it is rarely necessary to set this option. It is provided + // as a workaround for kernel/filesystem bugs, such as one that affected + // fdatasync with ext4 in kernel versions prior to 3.7. + bool use_fsync = false; + + // A list of paths where SST files can be put into, with its target size. + // Newer data is placed into paths specified earlier in the vector while + // older data gradually moves to paths specified later in the vector. + // + // For example, you have a flash device with 10GB allocated for the DB, + // as well as a hard drive of 2TB, you should config it to be: + // [{"/flash_path", 10GB}, {"/hard_drive", 2TB}] + // + // The system will try to guarantee data under each path is close to but + // not larger than the target size. But current and future file sizes used + // by determining where to place a file are based on best-effort estimation, + // which means there is a chance that the actual size under the directory + // is slightly more than target size under some workloads. User should give + // some buffer room for those cases. + // + // If none of the paths has sufficient room to place a file, the file will + // be placed to the last path anyway, despite to the target size. + // + // Placing newer data to earlier paths is also best-efforts. User should + // expect user files to be placed in higher levels in some extreme cases. + // + // If left empty, only one path will be used, which is db_name passed when + // opening the DB. + // Default: empty + std::vector db_paths; + + // This specifies the info LOG dir. + // If it is empty, the log files will be in the same dir as data. + // If it is non empty, the log files will be in the specified dir, + // and the db data dir's absolute path will be used as the log file + // name's prefix. + std::string db_log_dir = ""; + + // This specifies the absolute dir path for write-ahead logs (WAL). + // If it is empty, the log files will be in the same dir as data, + // dbname is used as the data dir by default + // If it is non empty, the log files will be in kept the specified dir. + // When destroying the db, + // all log files in wal_dir and the dir itself is deleted + std::string wal_dir = ""; + + // The periodicity when obsolete files get deleted. The default + // value is 6 hours. The files that get out of scope by compaction + // process will still get automatically delete on every compaction, + // regardless of this setting + // + // Default: 6 hours + // + // Dynamically changeable through SetDBOptions() API. + uint64_t delete_obsolete_files_period_micros = 6ULL * 60 * 60 * 1000000; + + // Maximum number of concurrent background jobs (compactions and flushes). + // + // Default: 2 + // + // Dynamically changeable through SetDBOptions() API. + int max_background_jobs = 2; + + // DEPRECATED: RocksDB automatically decides this based on the + // value of max_background_jobs. For backwards compatibility we will set + // `max_background_jobs = max_background_compactions + max_background_flushes` + // in the case where user sets at least one of `max_background_compactions` or + // `max_background_flushes` (we replace -1 by 1 in case one option is unset). + // + // Maximum number of concurrent background compaction jobs, submitted to + // the default LOW priority thread pool. + // + // If you're increasing this, also consider increasing number of threads in + // LOW priority thread pool. For more information, see + // Env::SetBackgroundThreads + // + // Default: -1 + // + // Dynamically changeable through SetDBOptions() API. + int max_background_compactions = -1; + + // This value represents the maximum number of threads that will + // concurrently perform a compaction job by breaking it into multiple, + // smaller ones that are run simultaneously. + // Default: 1 (i.e. no subcompactions) + // + // Dynamically changeable through SetDBOptions() API. + uint32_t max_subcompactions = 1; + + // DEPRECATED: RocksDB automatically decides this based on the + // value of max_background_jobs. For backwards compatibility we will set + // `max_background_jobs = max_background_compactions + max_background_flushes` + // in the case where user sets at least one of `max_background_compactions` or + // `max_background_flushes`. + // + // Maximum number of concurrent background memtable flush jobs, submitted by + // default to the HIGH priority thread pool. If the HIGH priority thread pool + // is configured to have zero threads, flush jobs will share the LOW priority + // thread pool with compaction jobs. + // + // It is important to use both thread pools when the same Env is shared by + // multiple db instances. Without a separate pool, long running compaction + // jobs could potentially block memtable flush jobs of other db instances, + // leading to unnecessary Put stalls. + // + // If you're increasing this, also consider increasing number of threads in + // HIGH priority thread pool. For more information, see + // Env::SetBackgroundThreads + // Default: -1 + int max_background_flushes = -1; + + // Specify the maximal size of the info log file. If the log file + // is larger than `max_log_file_size`, a new info log file will + // be created. + // If max_log_file_size == 0, all logs will be written to one + // log file. + size_t max_log_file_size = 0; + + // Time for the info log file to roll (in seconds). + // If specified with non-zero value, log file will be rolled + // if it has been active longer than `log_file_time_to_roll`. + // Default: 0 (disabled) + size_t log_file_time_to_roll = 0; + + // Maximal info log files to be kept. + // Default: 1000 + size_t keep_log_file_num = 1000; + + // Recycle log files. + // If non-zero, we will reuse previously written log files for new + // logs, overwriting the old data. The value indicates how many + // such files we will keep around at any point in time for later + // use. This is more efficient because the blocks are already + // allocated and fdatasync does not need to update the inode after + // each write. + // Default: 0 + size_t recycle_log_file_num = 0; + + // manifest file is rolled over on reaching this limit. + // The older manifest file be deleted. + // The default value is 1GB so that the manifest file can grow, but not + // reach the limit of storage capacity. + uint64_t max_manifest_file_size = 1024 * 1024 * 1024; + + // Number of shards used for table cache. + int table_cache_numshardbits = 6; + + // The following two fields affect how archived logs will be deleted. + // 1. If both set to 0, logs will be deleted asap and will not get into + // the archive. + // 2. If WAL_ttl_seconds is 0 and WAL_size_limit_MB is not 0, + // WAL files will be checked every 10 min and if total size is greater + // then WAL_size_limit_MB, they will be deleted starting with the + // earliest until size_limit is met. All empty files will be deleted. + // 3. If WAL_ttl_seconds is not 0 and WAL_size_limit_MB is 0, then + // WAL files will be checked every WAL_ttl_seconds / 2 and those that + // are older than WAL_ttl_seconds will be deleted. + // 4. If both are not 0, WAL files will be checked every 10 min and both + // checks will be performed with ttl being first. + uint64_t WAL_ttl_seconds = 0; + uint64_t WAL_size_limit_MB = 0; + + // Number of bytes to preallocate (via fallocate) the manifest + // files. Default is 4mb, which is reasonable to reduce random IO + // as well as prevent overallocation for mounts that preallocate + // large amounts of data (such as xfs's allocsize option). + size_t manifest_preallocation_size = 4 * 1024 * 1024; + + // Allow the OS to mmap file for reading sst tables. + // Not recommended for 32-bit OS. + // When the option is set to true and compression is disabled, the blocks + // will not be copied and will be read directly from the mmap-ed memory + // area, and the block will not be inserted into the block cache. However, + // checksums will still be checked if ReadOptions.verify_checksums is set + // to be true. It means a checksum check every time a block is read, more + // than the setup where the option is set to false and the block cache is + // used. The common use of the options is to run RocksDB on ramfs, where + // checksum verification is usually not needed. + // Default: false + bool allow_mmap_reads = false; + + // Allow the OS to mmap file for writing. + // DB::SyncWAL() only works if this is set to false. + // Default: false + bool allow_mmap_writes = false; + + // Enable direct I/O mode for read/write + // they may or may not improve performance depending on the use case + // + // Files will be opened in "direct I/O" mode + // which means that data r/w from the disk will not be cached or + // buffered. The hardware buffer of the devices may however still + // be used. Memory mapped files are not impacted by these parameters. + + // Use O_DIRECT for user and compaction reads. + // Default: false + bool use_direct_reads = false; + + // Use O_DIRECT for writes in background flush and compactions. + // Default: false + bool use_direct_io_for_flush_and_compaction = false; + + // If false, fallocate() calls are bypassed, which disables file + // preallocation. The file space preallocation is used to increase the file + // write/append performance. By default, RocksDB preallocates space for WAL, + // SST, Manifest files, the extra space is truncated when the file is written. + // Warning: if you're using btrfs, we would recommend setting + // `allow_fallocate=false` to disable preallocation. As on btrfs, the extra + // allocated space cannot be freed, which could be significant if you have + // lots of files. More details about this limitation: + // https://github.com/btrfs/btrfs-dev-docs/blob/471c5699336e043114d4bca02adcd57d9dab9c44/data-extent-reference-counts.md + bool allow_fallocate = true; + + // Disable child process inherit open files. Default: true + bool is_fd_close_on_exec = true; + + // if not zero, dump rocksdb.stats to LOG every stats_dump_period_sec + // + // Default: 600 (10 min) + // + // Dynamically changeable through SetDBOptions() API. + unsigned int stats_dump_period_sec = 600; + + // if not zero, dump rocksdb.stats to RocksDB every stats_persist_period_sec + // Default: 600 + unsigned int stats_persist_period_sec = 600; + + // If true, automatically persist stats to a hidden column family (column + // family name: ___rocksdb_stats_history___) every + // stats_persist_period_sec seconds; otherwise, write to an in-memory + // struct. User can query through `GetStatsHistory` API. + // If user attempts to create a column family with the same name on a DB + // which have previously set persist_stats_to_disk to true, the column family + // creation will fail, but the hidden column family will survive, as well as + // the previously persisted statistics. + // When peristing stats to disk, the stat name will be limited at 100 bytes. + // Default: false + bool persist_stats_to_disk = false; + + // if not zero, periodically take stats snapshots and store in memory, the + // memory size for stats snapshots is capped at stats_history_buffer_size + // Default: 1MB + size_t stats_history_buffer_size = 1024 * 1024; + + // If set true, will hint the underlying file system that the file + // access pattern is random, when a sst file is opened. + // Default: true + bool advise_random_on_open = true; + + // Amount of data to build up in memtables across all column + // families before writing to disk. + // + // This is distinct from write_buffer_size, which enforces a limit + // for a single memtable. + // + // This feature is disabled by default. Specify a non-zero value + // to enable it. + // + // Default: 0 (disabled) + size_t db_write_buffer_size = 0; + + // The memory usage of memtable will report to this object. The same object + // can be passed into multiple DBs and it will track the sum of size of all + // the DBs. If the total size of all live memtables of all the DBs exceeds + // a limit, a flush will be triggered in the next DB to which the next write + // is issued, as long as there is one or more column family not already + // flushing. + // + // If the object is only passed to one DB, the behavior is the same as + // db_write_buffer_size. When write_buffer_manager is set, the value set will + // override db_write_buffer_size. + // + // This feature is disabled by default. Specify a non-zero value + // to enable it. + // + // Default: null + std::shared_ptr write_buffer_manager = nullptr; + + // DEPRECATED + // This flag has no effect on the behavior of compaction and we plan to delete + // it in the future. + // Specify the file access pattern once a compaction is started. + // It will be applied to all input files of a compaction. + // Default: NORMAL + enum AccessHint { NONE, NORMAL, SEQUENTIAL, WILLNEED }; + AccessHint access_hint_on_compaction_start = NORMAL; + + // If non-zero, we perform bigger reads when doing compaction. If you're + // running RocksDB on spinning disks, you should set this to at least 2MB. + // That way RocksDB's compaction is doing sequential instead of random reads. + // + // Default: 0 + // + // Dynamically changeable through SetDBOptions() API. + size_t compaction_readahead_size = 0; + + // This is a maximum buffer size that is used by WinMmapReadableFile in + // unbuffered disk I/O mode. We need to maintain an aligned buffer for + // reads. We allow the buffer to grow until the specified value and then + // for bigger requests allocate one shot buffers. In unbuffered mode we + // always bypass read-ahead buffer at ReadaheadRandomAccessFile + // When read-ahead is required we then make use of compaction_readahead_size + // value and always try to read ahead. With read-ahead we always + // pre-allocate buffer to the size instead of growing it up to a limit. + // + // This option is currently honored only on Windows + // + // Default: 1 Mb + // + // Special value: 0 - means do not maintain per instance buffer. Allocate + // per request buffer and avoid locking. + size_t random_access_max_buffer_size = 1024 * 1024; + + // This is the maximum buffer size that is used by WritableFileWriter. + // With direct IO, we need to maintain an aligned buffer for writes. + // We allow the buffer to grow until it's size hits the limit in buffered + // IO and fix the buffer size when using direct IO to ensure alignment of + // write requests if the logical sector size is unusual + // + // Default: 1024 * 1024 (1 MB) + // + // Dynamically changeable through SetDBOptions() API. + size_t writable_file_max_buffer_size = 1024 * 1024; + + // Use adaptive mutex, which spins in the user space before resorting + // to kernel. This could reduce context switch when the mutex is not + // heavily contended. However, if the mutex is hot, we could end up + // wasting spin time. + // Default: false + bool use_adaptive_mutex = false; + + // Create DBOptions with default values for all fields + DBOptions(); + // Create DBOptions from Options + explicit DBOptions(const Options& options); + + void Dump(Logger* log) const; + + // Allows OS to incrementally sync files to disk while they are being + // written, asynchronously, in the background. This operation can be used + // to smooth out write I/Os over time. Users shouldn't rely on it for + // persistence guarantee. + // Issue one request for every bytes_per_sync written. 0 turns it off. + // + // You may consider using rate_limiter to regulate write rate to device. + // When rate limiter is enabled, it automatically enables bytes_per_sync + // to 1MB. + // + // This option applies to table files + // + // Default: 0, turned off + // + // Note: DOES NOT apply to WAL files. See wal_bytes_per_sync instead + // Dynamically changeable through SetDBOptions() API. + uint64_t bytes_per_sync = 0; + + // Same as bytes_per_sync, but applies to WAL files + // + // Default: 0, turned off + // + // Dynamically changeable through SetDBOptions() API. + uint64_t wal_bytes_per_sync = 0; + + // When true, guarantees WAL files have at most `wal_bytes_per_sync` + // bytes submitted for writeback at any given time, and SST files have at most + // `bytes_per_sync` bytes pending writeback at any given time. This can be + // used to handle cases where processing speed exceeds I/O speed during file + // generation, which can lead to a huge sync when the file is finished, even + // with `bytes_per_sync` / `wal_bytes_per_sync` properly configured. + // + // - If `sync_file_range` is supported it achieves this by waiting for any + // prior `sync_file_range`s to finish before proceeding. In this way, + // processing (compression, etc.) can proceed uninhibited in the gap + // between `sync_file_range`s, and we block only when I/O falls behind. + // - Otherwise the `WritableFile::Sync` method is used. Note this mechanism + // always blocks, thus preventing the interleaving of I/O and processing. + // + // Note: Enabling this option does not provide any additional persistence + // guarantees, as it may use `sync_file_range`, which does not write out + // metadata. + // + // Default: false + bool strict_bytes_per_sync = false; + + // A vector of EventListeners whose callback functions will be called + // when specific RocksDB event happens. + std::vector> listeners; + + // If true, then the status of the threads involved in this DB will + // be tracked and available via GetThreadList() API. + // + // Default: false + bool enable_thread_tracking = false; + + // The limited write rate to DB if soft_pending_compaction_bytes_limit or + // level0_slowdown_writes_trigger is triggered, or we are writing to the + // last mem table allowed and we allow more than 3 mem tables. It is + // calculated using size of user write requests before compression. + // RocksDB may decide to slow down more if the compaction still + // gets behind further. + // If the value is 0, we will infer a value from `rater_limiter` value + // if it is not empty, or 16MB if `rater_limiter` is empty. Note that + // if users change the rate in `rate_limiter` after DB is opened, + // `delayed_write_rate` won't be adjusted. + // + // Unit: byte per second. + // + // Default: 0 + // + // Dynamically changeable through SetDBOptions() API. + uint64_t delayed_write_rate = 0; + + // By default, a single write thread queue is maintained. The thread gets + // to the head of the queue becomes write batch group leader and responsible + // for writing to WAL and memtable for the batch group. + // + // If enable_pipelined_write is true, separate write thread queue is + // maintained for WAL write and memtable write. A write thread first enter WAL + // writer queue and then memtable writer queue. Pending thread on the WAL + // writer queue thus only have to wait for previous writers to finish their + // WAL writing but not the memtable writing. Enabling the feature may improve + // write throughput and reduce latency of the prepare phase of two-phase + // commit. + // + // Default: false + bool enable_pipelined_write = false; + + // Setting unordered_write to true trades higher write throughput with + // relaxing the immutability guarantee of snapshots. This violates the + // repeatability one expects from ::Get from a snapshot, as well as + // ::MultiGet and Iterator's consistent-point-in-time view property. + // If the application cannot tolerate the relaxed guarantees, it can implement + // its own mechanisms to work around that and yet benefit from the higher + // throughput. Using TransactionDB with WRITE_PREPARED write policy and + // two_write_queues=true is one way to achieve immutable snapshots despite + // unordered_write. + // + // By default, i.e., when it is false, rocksdb does not advance the sequence + // number for new snapshots unless all the writes with lower sequence numbers + // are already finished. This provides the immutability that we expect from + // snapshots. Moreover, since Iterator and MultiGet internally depend on + // snapshots, the snapshot immutability results into Iterator and MultiGet + // offering consistent-point-in-time view. If set to true, although + // Read-Your-Own-Write property is still provided, the snapshot immutability + // property is relaxed: the writes issued after the snapshot is obtained (with + // larger sequence numbers) will be still not visible to the reads from that + // snapshot, however, there still might be pending writes (with lower sequence + // number) that will change the state visible to the snapshot after they are + // landed to the memtable. + // + // Default: false + bool unordered_write = false; + + // If true, allow multi-writers to update mem tables in parallel. + // Only some memtable_factory-s support concurrent writes; currently it + // is implemented only for SkipListFactory. Concurrent memtable writes + // are not compatible with inplace_update_support or filter_deletes. + // It is strongly recommended to set enable_write_thread_adaptive_yield + // if you are going to use this feature. + // + // Default: true + bool allow_concurrent_memtable_write = true; + + // If true, threads synchronizing with the write batch group leader will + // wait for up to write_thread_max_yield_usec before blocking on a mutex. + // This can substantially improve throughput for concurrent workloads, + // regardless of whether allow_concurrent_memtable_write is enabled. + // + // Default: true + bool enable_write_thread_adaptive_yield = true; + + // The maximum limit of number of bytes that are written in a single batch + // of WAL or memtable write. It is followed when the leader write size + // is larger than 1/8 of this limit. + // + // Default: 1 MB + uint64_t max_write_batch_group_size_bytes = 1 << 20; + + // The maximum number of microseconds that a write operation will use + // a yielding spin loop to coordinate with other write threads before + // blocking on a mutex. (Assuming write_thread_slow_yield_usec is + // set properly) increasing this value is likely to increase RocksDB + // throughput at the expense of increased CPU usage. + // + // Default: 100 + uint64_t write_thread_max_yield_usec = 100; + + // The latency in microseconds after which a std::this_thread::yield + // call (sched_yield on Linux) is considered to be a signal that + // other processes or threads would like to use the current core. + // Increasing this makes writer threads more likely to take CPU + // by spinning, which will show up as an increase in the number of + // involuntary context switches. + // + // Default: 3 + uint64_t write_thread_slow_yield_usec = 3; + + // If true, then DB::Open() will not update the statistics used to optimize + // compaction decision by loading table properties from many files. + // Turning off this feature will improve DBOpen time especially in + // disk environment. + // + // Default: false + bool skip_stats_update_on_db_open = false; + + // If true, then DB::Open() will not fetch and check sizes of all sst files. + // This may significantly speed up startup if there are many sst files, + // especially when using non-default Env with expensive GetFileSize(). + // We'll still check that all required sst files exist. + // If paranoid_checks is false, this option is ignored, and sst files are + // not checked at all. + // + // Default: false + bool skip_checking_sst_file_sizes_on_db_open = false; + + // Recovery mode to control the consistency while replaying WAL + // Default: kPointInTimeRecovery + WALRecoveryMode wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery; + + // if set to false then recovery will fail when a prepared + // transaction is encountered in the WAL + bool allow_2pc = false; + + // A global cache for table-level rows. + // Default: nullptr (disabled) + std::shared_ptr row_cache = nullptr; + + // A filter object supplied to be invoked while processing write-ahead-logs + // (WALs) during recovery. The filter provides a way to inspect log + // records, ignoring a particular record or skipping replay. + // The filter is invoked at startup and is invoked from a single-thread + // currently. + WalFilter* wal_filter = nullptr; + + // If true, then DB::Open / CreateColumnFamily / DropColumnFamily + // SetOptions will fail if options file is not properly persisted. + // + // DEFAULT: false + bool fail_if_options_file_error = false; + + // If true, then print malloc stats together with rocksdb.stats + // when printing to LOG. + // DEFAULT: false + bool dump_malloc_stats = false; + + // By default RocksDB replay WAL logs and flush them on DB open, which may + // create very small SST files. If this option is enabled, RocksDB will try + // to avoid (but not guarantee not to) flush during recovery. Also, existing + // WAL logs will be kept, so that if crash happened before flush, we still + // have logs to recover from. + // + // DEFAULT: false + bool avoid_flush_during_recovery = false; + + // By default RocksDB will flush all memtables on DB close if there are + // unpersisted data (i.e. with WAL disabled) The flush can be skip to speedup + // DB close. Unpersisted data WILL BE LOST. + // + // DEFAULT: false + // + // Dynamically changeable through SetDBOptions() API. + bool avoid_flush_during_shutdown = false; + + // Set this option to true during creation of database if you want + // to be able to ingest behind (call IngestExternalFile() skipping keys + // that already exist, rather than overwriting matching keys). + // Setting this option to true has the following effects: + // 1) Disable some internal optimizations around SST file compression. + // 2) Reserve the last level for ingested files only. + // 3) Compaction will not include any file from the last level. + // Note that only Universal Compaction supports allow_ingest_behind. + // `num_levels` should be >= 3 if this option is turned on. + // + // + // DEFAULT: false + // Immutable. + bool allow_ingest_behind = false; + + // If enabled it uses two queues for writes, one for the ones with + // disable_memtable and one for the ones that also write to memtable. This + // allows the memtable writes not to lag behind other writes. It can be used + // to optimize MySQL 2PC in which only the commits, which are serial, write to + // memtable. + bool two_write_queues = false; + + // If true WAL is not flushed automatically after each write. Instead it + // relies on manual invocation of FlushWAL to write the WAL buffer to its + // file. + bool manual_wal_flush = false; + + // This feature is WORK IN PROGRESS + // If enabled WAL records will be compressed before they are written. + // Only zstd is supported. Compressed WAL records will be read in supported + // versions regardless of the wal_compression settings. + CompressionType wal_compression = kNoCompression; + + // If true, RocksDB supports flushing multiple column families and committing + // their results atomically to MANIFEST. Note that it is not + // necessary to set atomic_flush to true if WAL is always enabled since WAL + // allows the database to be restored to the last persistent state in WAL. + // This option is useful when there are column families with writes NOT + // protected by WAL. + // For manual flush, application has to specify which column families to + // flush atomically in DB::Flush. + // For auto-triggered flush, RocksDB atomically flushes ALL column families. + // + // Currently, any WAL-enabled writes after atomic flush may be replayed + // independently if the process crashes later and tries to recover. + bool atomic_flush = false; + + // If true, working thread may avoid doing unnecessary and long-latency + // operation (such as deleting obsolete files directly or deleting memtable) + // and will instead schedule a background job to do it. + // Use it if you're latency-sensitive. + // If set to true, takes precedence over + // ReadOptions::background_purge_on_iterator_cleanup. + bool avoid_unnecessary_blocking_io = false; + + // Historically DB ID has always been stored in Identity File in DB folder. + // If this flag is true, the DB ID is written to Manifest file in addition + // to the Identity file. By doing this 2 problems are solved + // 1. We don't checksum the Identity file where as Manifest file is. + // 2. Since the source of truth for DB is Manifest file DB ID will sit with + // the source of truth. Previously the Identity file could be copied + // independent of Manifest and that can result in wrong DB ID. + // We recommend setting this flag to true. + // Default: false + bool write_dbid_to_manifest = false; + + // The number of bytes to prefetch when reading the log. This is mostly useful + // for reading a remotely located log, as it can save the number of + // round-trips. If 0, then the prefetching is disabled. + // + // Default: 0 + size_t log_readahead_size = 0; + + // If user does NOT provide the checksum generator factory, the file checksum + // will NOT be used. A new file checksum generator object will be created + // when a SST file is created. Therefore, each created FileChecksumGenerator + // will only be used from a single thread and so does not need to be + // thread-safe. + // + // Default: nullptr + std::shared_ptr file_checksum_gen_factory = nullptr; + + // By default, RocksDB will attempt to detect any data losses or corruptions + // in DB files and return an error to the user, either at DB::Open time or + // later during DB operation. The exception to this policy is the WAL file, + // whose recovery is controlled by the wal_recovery_mode option. + // + // Best-efforts recovery (this option set to true) signals a preference for + // opening the DB to any point-in-time valid state for each column family, + // including the empty/new state, versus the default of returning non-WAL + // data losses to the user as errors. In terms of RocksDB user data, this + // is like applying WALRecoveryMode::kPointInTimeRecovery to each column + // family rather than just the WAL. + // + // Best-efforts recovery (BER) is specifically designed to recover a DB with + // files that are missing or truncated to some smaller size, such as the + // result of an incomplete DB "physical" (FileSystem) copy. BER can also + // detect when an SST file has been replaced with a different one of the + // same size (assuming SST unique IDs are tracked in DB manifest). + // BER is not yet designed to produce a usable DB from other corruptions to + // DB files (which should generally be detectable by DB::VerifyChecksum()), + // and BER does not yet attempt to recover any WAL files. + // + // For example, if an SST or blob file referenced by the MANIFEST is missing, + // BER might be able to find a set of files corresponding to an old "point in + // time" version of the column family, possibly from an older MANIFEST + // file. Some other kinds of DB files (e.g. CURRENT, LOCK, IDENTITY) are + // either ignored or replaced with BER, or quietly fixed regardless of BER + // setting. BER does require at least one valid MANIFEST to recover to a + // non-trivial DB state, unlike `ldb repair`. + // + // Currently, best_efforts_recovery=true is not compatible with atomic flush. + // + // Default: false + bool best_efforts_recovery = false; + + // It defines how many times DB::Resume() is called by a separate thread when + // background retryable IO Error happens. When background retryable IO + // Error happens, SetBGError is called to deal with the error. If the error + // can be auto-recovered (e.g., retryable IO Error during Flush or WAL write), + // then db resume is called in background to recover from the error. If this + // value is 0 or negative, DB::Resume() will not be called automatically. + // + // Default: INT_MAX + int max_bgerror_resume_count = INT_MAX; + + // If max_bgerror_resume_count is >= 2, db resume is called multiple times. + // This option decides how long to wait to retry the next resume if the + // previous resume fails and satisfy redo resume conditions. + // + // Default: 1000000 (microseconds). + uint64_t bgerror_resume_retry_interval = 1000000; + + // It allows user to opt-in to get error messages containing corrupted + // keys/values. Corrupt keys, values will be logged in the + // messages/logs/status that will help users with the useful information + // regarding affected data. By default value is set false to prevent users + // data to be exposed in the logs/messages etc. + // + // Default: false + bool allow_data_in_errors = false; + + // A string identifying the machine hosting the DB. This + // will be written as a property in every SST file written by the DB (or + // by offline writers such as SstFileWriter and RepairDB). It can be useful + // for troubleshooting in memory corruption caused by a failing host when + // writing a file, by tracing back to the writing host. These corruptions + // may not be caught by the checksum since they happen before checksumming. + // If left as default, the table writer will substitute it with the actual + // hostname when writing the SST file. If set to an empty string, the + // property will not be written to the SST file. + // + // Default: hostname + std::string db_host_id = kHostnameForDbHostId; + + // Use this if your DB want to enable checksum handoff for specific file + // types writes. Make sure that the File_system you use support the + // crc32c checksum verification + // Currently supported file tyes: kWALFile, kTableFile, kDescriptorFile. + // NOTE: currently RocksDB only generates crc32c based checksum for the + // handoff. If the storage layer has different checksum support, user + // should enble this set as empty. Otherwise,it may cause unexpected + // write failures. + FileTypeSet checksum_handoff_file_types; + + // EXPERIMENTAL + // CompactionService is a feature allows the user to run compactions on a + // different host or process, which offloads the background load from the + // primary host. + // It's an experimental feature, the interface will be changed without + // backward/forward compatibility support for now. Some known issues are still + // under development. + std::shared_ptr compaction_service = nullptr; + + // It indicates, which lowest cache tier we want to + // use for a certain DB. Currently we support volatile_tier and + // non_volatile_tier. They are layered. By setting it to kVolatileTier, only + // the block cache (current implemented volatile_tier) is used. So + // cache entries will not spill to secondary cache (current + // implemented non_volatile_tier), and block cache lookup misses will not + // lookup in the secondary cache. When kNonVolatileBlockTier is used, we use + // both block cache and secondary cache. + // + // Default: kNonVolatileBlockTier + CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier; + + // If set to false, when compaction or flush sees a SingleDelete followed by + // a Delete for the same user key, compaction job will not fail. + // Otherwise, compaction job will fail. + // This is a temporary option to help existing use cases migrate, and + // will be removed in a future release. + // Warning: do not set to false unless you are trying to migrate existing + // data in which the contract of single delete + // (https://github.com/facebook/rocksdb/wiki/Single-Delete) is not enforced, + // thus has Delete mixed with SingleDelete for the same user key. Violation + // of the contract leads to undefined behaviors with high possibility of data + // inconsistency, e.g. deleted old data become visible again, etc. + bool enforce_single_del_contracts = true; +}; + +// Options to control the behavior of a database (passed to DB::Open) +struct Options : public DBOptions, public ColumnFamilyOptions { + // Create an Options object with default values for all fields. + Options() : DBOptions(), ColumnFamilyOptions() {} + + Options(const DBOptions& db_options, + const ColumnFamilyOptions& column_family_options) + : DBOptions(db_options), ColumnFamilyOptions(column_family_options) {} + + // Change to some default settings from an older version. + // NOT MAINTAINED: This function has not been and is not maintained. + // DEPRECATED: This function might be removed in a future release. + // In general, defaults are changed to suit broad interests. Opting + // out of a change on upgrade should be deliberate and considered. + Options* OldDefaults(int rocksdb_major_version = 4, + int rocksdb_minor_version = 6); + + void Dump(Logger* log) const; + + void DumpCFOptions(Logger* log) const; + + // Some functions that make it easier to optimize RocksDB + + // Set appropriate parameters for bulk loading. + // The reason that this is a function that returns "this" instead of a + // constructor is to enable chaining of multiple similar calls in the future. + // + + // All data will be in level 0 without any automatic compaction. + // It's recommended to manually call CompactRange(NULL, NULL) before reading + // from the database, because otherwise the read can be very slow. + Options* PrepareForBulkLoad(); + + // Use this if your DB is very small (like under 1GB) and you don't want to + // spend lots of memory for memtables. + Options* OptimizeForSmallDb(); + + // Disable some checks that should not be necessary in the absence of + // software logic errors or CPU+memory hardware errors. This can improve + // write speeds but is only recommended for temporary use. Does not + // change protection against corrupt storage (e.g. verify_checksums). + Options* DisableExtraChecks(); +}; + +// An application can issue a read request (via Get/Iterators) and specify +// if that read should process data that ALREADY resides on a specified cache +// level. For example, if an application specifies kBlockCacheTier then the +// Get call will process data that is already processed in the memtable or +// the block cache. It will not page in data from the OS cache or data that +// resides in storage. +enum ReadTier { + kReadAllTier = 0x0, // data in memtable, block cache, OS cache or storage + kBlockCacheTier = 0x1, // data in memtable or block cache + kPersistedTier = 0x2, // persisted data. When WAL is disabled, this option + // will skip data in memtable. + // Note that this ReadTier currently only supports + // Get and MultiGet and does not support iterators. + kMemtableTier = 0x3 // data in memtable. used for memtable-only iterators. +}; + +// Options that control read operations +struct ReadOptions { + // *** BEGIN options relevant to point lookups as well as scans *** + + // If "snapshot" is non-nullptr, read as of the supplied snapshot + // (which must belong to the DB that is being read and which must + // not have been released). If "snapshot" is nullptr, use an implicit + // snapshot of the state at the beginning of this read operation. + const Snapshot* snapshot = nullptr; + + // Timestamp of operation. Read should return the latest data visible to the + // specified timestamp. All timestamps of the same database must be of the + // same length and format. The user is responsible for providing a customized + // compare function via Comparator to order tuples. + // For iterator, iter_start_ts is the lower bound (older) and timestamp + // serves as the upper bound. Versions of the same record that fall in + // the timestamp range will be returned. If iter_start_ts is nullptr, + // only the most recent version visible to timestamp is returned. + // The user-specified timestamp feature is still under active development, + // and the API is subject to change. + const Slice* timestamp = nullptr; + const Slice* iter_start_ts = nullptr; + + // Deadline for completing an API call (Get/MultiGet/Seek/Next for now) + // in microseconds. + // It should be set to microseconds since epoch, i.e, gettimeofday or + // equivalent plus allowed duration in microseconds. The best way is to use + // env->NowMicros() + some timeout. + // This is best efforts. The call may exceed the deadline if there is IO + // involved and the file system doesn't support deadlines, or due to + // checking for deadline periodically rather than for every key if + // processing a batch + std::chrono::microseconds deadline = std::chrono::microseconds::zero(); + + // A timeout in microseconds to be passed to the underlying FileSystem for + // reads. As opposed to deadline, this determines the timeout for each + // individual file read request. If a MultiGet/Get/Seek/Next etc call + // results in multiple reads, each read can last up to io_timeout us. + std::chrono::microseconds io_timeout = std::chrono::microseconds::zero(); + + // Specify if this read request should process data that ALREADY + // resides on a particular cache. If the required data is not + // found at the specified cache, then Status::Incomplete is returned. + ReadTier read_tier = kReadAllTier; + + // For file reads associated with this option, charge the internal rate + // limiter (see `DBOptions::rate_limiter`) at the specified priority. The + // special value `Env::IO_TOTAL` disables charging the rate limiter. + // + // The rate limiting is bypassed no matter this option's value for file reads + // on plain tables (these can exist when `ColumnFamilyOptions::table_factory` + // is a `PlainTableFactory`) and cuckoo tables (these can exist when + // `ColumnFamilyOptions::table_factory` is a `CuckooTableFactory`). + // + // The bytes charged to rate limiter may not exactly match the file read bytes + // since there are some seemingly insignificant reads, like for file + // headers/footers, that we currently do not charge to rate limiter. + Env::IOPriority rate_limiter_priority = Env::IO_TOTAL; + + // It limits the maximum cumulative value size of the keys in batch while + // reading through MultiGet. Once the cumulative value size exceeds this + // soft limit then all the remaining keys are returned with status Aborted. + uint64_t value_size_soft_limit = std::numeric_limits::max(); + + // If true, all data read from underlying storage will be + // verified against corresponding checksums. + bool verify_checksums = true; + + // Should the "data block"/"index block" read for this iteration be placed in + // block cache? + // Callers may wish to set this field to false for bulk scans. + // This would help not to the change eviction order of existing items in the + // block cache. + bool fill_cache = true; + + // If true, range tombstones handling will be skipped in key lookup paths. + // For DB instances that don't use DeleteRange() calls, this setting can + // be used to optimize the read performance. + // Note that, if this assumption (of no previous DeleteRange() calls) is + // broken, stale keys could be served in read paths. + bool ignore_range_deletions = false; + + // Experimental + // + // If async_io is enabled, RocksDB will prefetch some of data asynchronously. + // RocksDB apply it if reads are sequential and its internal automatic + // prefetching. + bool async_io = false; + + // Experimental + // + // If async_io is set, then this flag controls whether we read SST files + // in multiple levels asynchronously. Enabling this flag can help reduce + // MultiGet latency by maximizing the number of SST files read in + // parallel if the keys in the MultiGet batch are in different levels. It + // comes at the expense of slightly higher CPU overhead. + bool optimize_multiget_for_io = true; + + // *** END options relevant to point lookups (as well as scans) *** + // *** BEGIN options only relevant to iterators or scans *** + + // RocksDB does auto-readahead for iterators on noticing more than two reads + // for a table file. The readahead starts at 8KB and doubles on every + // additional read up to 256KB. + // This option can help if most of the range scans are large, and if it is + // determined that a larger readahead than that enabled by auto-readahead is + // needed. + // Using a large readahead size (> 2MB) can typically improve the performance + // of forward iteration on spinning disks. + size_t readahead_size = 0; + + // A threshold for the number of keys that can be skipped before failing an + // iterator seek as incomplete. The default value of 0 should be used to + // never fail a request as incomplete, even on skipping too many keys. + uint64_t max_skippable_internal_keys = 0; + + // `iterate_lower_bound` defines the smallest key at which the backward + // iterator can return an entry. Once the bound is passed, Valid() will be + // false. `iterate_lower_bound` is inclusive ie the bound value is a valid + // entry. + // + // If prefix_extractor is not null, the Seek target and `iterate_lower_bound` + // need to have the same prefix. This is because ordering is not guaranteed + // outside of prefix domain. + // + // In case of user_defined timestamp, if enabled, iterate_lower_bound should + // point to key without timestamp part. + const Slice* iterate_lower_bound = nullptr; + + // "iterate_upper_bound" defines the extent up to which the forward iterator + // can return entries. Once the bound is reached, Valid() will be false. + // "iterate_upper_bound" is exclusive ie the bound value is + // not a valid entry. If prefix_extractor is not null: + // 1. If options.auto_prefix_mode = true, iterate_upper_bound will be used + // to infer whether prefix iterating (e.g. applying prefix bloom filter) + // can be used within RocksDB. This is done by comparing + // iterate_upper_bound with the seek key. + // 2. If options.auto_prefix_mode = false, iterate_upper_bound only takes + // effect if it shares the same prefix as the seek key. If + // iterate_upper_bound is outside the prefix of the seek key, then keys + // returned outside the prefix range will be undefined, just as if + // iterate_upper_bound = null. + // If iterate_upper_bound is not null, SeekToLast() will position the iterator + // at the first key smaller than iterate_upper_bound. + // + // In case of user_defined timestamp, if enabled, iterate_upper_bound should + // point to key without timestamp part. + const Slice* iterate_upper_bound = nullptr; + + // Specify to create a tailing iterator -- a special iterator that has a + // view of the complete database (i.e. it can also be used to read newly + // added data) and is optimized for sequential reads. It will return records + // that were inserted into the database after the creation of the iterator. + bool tailing = false; + + // This options is not used anymore. It was to turn on a functionality that + // has been removed. DEPRECATED + bool managed = false; + + // Enable a total order seek regardless of index format (e.g. hash index) + // used in the table. Some table format (e.g. plain table) may not support + // this option. + // If true when calling Get(), we also skip prefix bloom when reading from + // block based table, which only affects Get() performance. + bool total_order_seek = false; + + // When true, by default use total_order_seek = true, and RocksDB can + // selectively enable prefix seek mode if won't generate a different result + // from total_order_seek, based on seek key, and iterator upper bound. + // BUG: Using Comparator::IsSameLengthImmediateSuccessor and + // SliceTransform::FullLengthEnabled to enable prefix mode in cases where + // prefix of upper bound differs from prefix of seek key has a flaw. + // If present in the DB, "short keys" (shorter than "full length" prefix) + // can be omitted from auto_prefix_mode iteration when they would be present + // in total_order_seek iteration, regardless of whether the short keys are + // "in domain" of the prefix extractor. This is not an issue if no short + // keys are added to DB or are not expected to be returned by such + // iterators. (We are also assuming the new condition on + // IsSameLengthImmediateSuccessor is satisfied; see its BUG section). + // A bug example is in DBTest2::AutoPrefixMode1, search for "BUG". + bool auto_prefix_mode = false; + + // Enforce that the iterator only iterates over the same prefix as the seek. + // This option is effective only for prefix seeks, i.e. prefix_extractor is + // non-null for the column family and total_order_seek is false. Unlike + // iterate_upper_bound, prefix_same_as_start only works within a prefix + // but in both directions. + bool prefix_same_as_start = false; + + // Keep the blocks loaded by the iterator pinned in memory as long as the + // iterator is not deleted, If used when reading from tables created with + // BlockBasedTableOptions::use_delta_encoding = false, + // Iterator's property "rocksdb.iterator.is-key-pinned" is guaranteed to + // return 1. + bool pin_data = false; + + // For iterators, RocksDB does auto-readahead on noticing more than two + // sequential reads for a table file if user doesn't provide readahead_size. + // The readahead starts at 8KB and doubles on every additional read upto + // max_auto_readahead_size only when reads are sequential. However at each + // level, if iterator moves over next file, readahead_size starts again from + // 8KB. + // + // By enabling this option, RocksDB will do some enhancements for + // prefetching the data. + bool adaptive_readahead = false; + + // If true, when PurgeObsoleteFile is called in CleanupIteratorState, we + // schedule a background job in the flush job queue and delete obsolete files + // in background. + bool background_purge_on_iterator_cleanup = false; + + // A callback to determine whether relevant keys for this scan exist in a + // given table based on the table's properties. The callback is passed the + // properties of each table during iteration. If the callback returns false, + // the table will not be scanned. This option only affects Iterators and has + // no impact on point lookups. + // Default: empty (every table will be scanned) + std::function table_filter; + + // *** END options only relevant to iterators or scans *** + + // ** For RocksDB internal use only ** + Env::IOActivity io_activity = Env::IOActivity::kUnknown; + + ReadOptions() {} + ReadOptions(bool _verify_checksums, bool _fill_cache); + explicit ReadOptions(Env::IOActivity _io_activity); +}; + +// Options that control write operations +struct WriteOptions { + // If true, the write will be flushed from the operating system + // buffer cache (by calling WritableFile::Sync()) before the write + // is considered complete. If this flag is true, writes will be + // slower. + // + // If this flag is false, and the machine crashes, some recent + // writes may be lost. Note that if it is just the process that + // crashes (i.e., the machine does not reboot), no writes will be + // lost even if sync==false. + // + // In other words, a DB write with sync==false has similar + // crash semantics as the "write()" system call. A DB write + // with sync==true has similar crash semantics to a "write()" + // system call followed by "fdatasync()". + // + // Default: false + bool sync; + + // If true, writes will not first go to the write ahead log, + // and the write may get lost after a crash. The backup engine + // relies on write-ahead logs to back up the memtable, so if + // you disable write-ahead logs, you must create backups with + // flush_before_backup=true to avoid losing unflushed memtable data. + // Default: false + bool disableWAL; + + // If true and if user is trying to write to column families that don't exist + // (they were dropped), ignore the write (don't return an error). If there + // are multiple writes in a WriteBatch, other writes will succeed. + // Default: false + bool ignore_missing_column_families; + + // If true and we need to wait or sleep for the write request, fails + // immediately with Status::Incomplete(). + // Default: false + bool no_slowdown; + + // If true, this write request is of lower priority if compaction is + // behind. In this case, no_slowdown = true, the request will be canceled + // immediately with Status::Incomplete() returned. Otherwise, it will be + // slowed down. The slowdown value is determined by RocksDB to guarantee + // it introduces minimum impacts to high priority writes. + // + // Default: false + bool low_pri; + + // If true, this writebatch will maintain the last insert positions of each + // memtable as hints in concurrent write. It can improve write performance + // in concurrent writes if keys in one writebatch are sequential. In + // non-concurrent writes (when concurrent_memtable_writes is false) this + // option will be ignored. + // + // Default: false + bool memtable_insert_hint_per_batch; + + // For writes associated with this option, charge the internal rate + // limiter (see `DBOptions::rate_limiter`) at the specified priority. The + // special value `Env::IO_TOTAL` disables charging the rate limiter. + // + // Currently the support covers automatic WAL flushes, which happen during + // live updates (`Put()`, `Write()`, `Delete()`, etc.) + // when `WriteOptions::disableWAL == false` + // and `DBOptions::manual_wal_flush == false`. + // + // Only `Env::IO_USER` and `Env::IO_TOTAL` are allowed + // due to implementation constraints. + // + // Default: `Env::IO_TOTAL` + Env::IOPriority rate_limiter_priority; + + // `protection_bytes_per_key` is the number of bytes used to store + // protection information for each key entry. Currently supported values are + // zero (disabled) and eight. + // + // Default: zero (disabled). + size_t protection_bytes_per_key; + + WriteOptions() + : sync(false), + disableWAL(false), + ignore_missing_column_families(false), + no_slowdown(false), + low_pri(false), + memtable_insert_hint_per_batch(false), + rate_limiter_priority(Env::IO_TOTAL), + protection_bytes_per_key(0) {} +}; + +// Options that control flush operations +struct FlushOptions { + // If true, the flush will wait until the flush is done. + // Default: true + bool wait; + // If true, the flush would proceed immediately even it means writes will + // stall for the duration of the flush; if false the operation will wait + // until it's possible to do flush w/o causing stall or until required flush + // is performed by someone else (foreground call or background thread). + // Default: false + bool allow_write_stall; + FlushOptions() : wait(true), allow_write_stall(false) {} +}; + +// Create a Logger from provided DBOptions +extern Status CreateLoggerFromOptions(const std::string& dbname, + const DBOptions& options, + std::shared_ptr* logger); + +// CompactionOptions are used in CompactFiles() call. +struct CompactionOptions { + // Compaction output compression type + // Default: snappy + // If set to `kDisableCompressionOption`, RocksDB will choose compression type + // according to the `ColumnFamilyOptions`, taking into account the output + // level if `compression_per_level` is specified. + CompressionType compression; + // Compaction will create files of size `output_file_size_limit`. + // Default: MAX, which means that compaction will create a single file + uint64_t output_file_size_limit; + // If > 0, it will replace the option in the DBOptions for this compaction. + uint32_t max_subcompactions; + + CompactionOptions() + : compression(kSnappyCompression), + output_file_size_limit(std::numeric_limits::max()), + max_subcompactions(0) {} +}; + +// For level based compaction, we can configure if we want to skip/force +// bottommost level compaction. +enum class BottommostLevelCompaction { + // Skip bottommost level compaction. + kSkip, + // Only compact bottommost level if there is a compaction filter. + // This is the default option. + // Similar to kForceOptimized, when compacting bottommost level, avoid + // double-compacting files + // created in the same manual compaction. + kIfHaveCompactionFilter, + // Always compact bottommost level. + kForce, + // Always compact bottommost level but in bottommost level avoid + // double-compacting files created in the same compaction. + kForceOptimized, +}; + +// For manual compaction, we can configure if we want to skip/force garbage +// collection of blob files. +enum class BlobGarbageCollectionPolicy { + // Force blob file garbage collection. + kForce, + // Skip blob file garbage collection. + kDisable, + // Inherit blob file garbage collection policy from ColumnFamilyOptions. + kUseDefault, +}; + +// CompactRangeOptions is used by CompactRange() call. +struct CompactRangeOptions { + // If true, no other compaction will run at the same time as this + // manual compaction. + // + // Default: false + bool exclusive_manual_compaction = false; + + // If true, compacted files will be moved to the minimum level capable + // of holding the data or given level (specified non-negative target_level). + bool change_level = false; + // If change_level is true and target_level have non-negative value, compacted + // files will be moved to target_level. + int target_level = -1; + // Compaction outputs will be placed in options.db_paths[target_path_id]. + // Behavior is undefined if target_path_id is out of range. + uint32_t target_path_id = 0; + // By default level based compaction will only compact the bottommost level + // if there is a compaction filter + BottommostLevelCompaction bottommost_level_compaction = + BottommostLevelCompaction::kIfHaveCompactionFilter; + // If true, will execute immediately even if doing so would cause the DB to + // enter write stall mode. Otherwise, it'll sleep until load is low enough. + bool allow_write_stall = false; + // If > 0, it will replace the option in the DBOptions for this compaction. + uint32_t max_subcompactions = 0; + // Set user-defined timestamp low bound, the data with older timestamp than + // low bound maybe GCed by compaction. Default: nullptr + const Slice* full_history_ts_low = nullptr; + + // Allows cancellation of an in-progress manual compaction. + // + // Cancellation can be delayed waiting on automatic compactions when used + // together with `exclusive_manual_compaction == true`. + std::atomic* canceled = nullptr; + // NOTE: Calling DisableManualCompaction() overwrites the uer-provided + // canceled variable in CompactRangeOptions. + // Typically, when CompactRange is being called in one thread (t1) with + // canceled = false, and DisableManualCompaction is being called in the + // other thread (t2), manual compaction is disabled normally, even if the + // compaction iterator may still scan a few items before *canceled is + // set to true + + // If set to kForce, RocksDB will override enable_blob_file_garbage_collection + // to true; if set to kDisable, RocksDB will override it to false, and + // kUseDefault leaves the setting in effect. This enables customers to both + // force-enable and force-disable GC when calling CompactRange. + BlobGarbageCollectionPolicy blob_garbage_collection_policy = + BlobGarbageCollectionPolicy::kUseDefault; + + // If set to < 0 or > 1, RocksDB leaves blob_garbage_collection_age_cutoff + // from ColumnFamilyOptions in effect. Otherwise, it will override the + // user-provided setting. This enables customers to selectively override the + // age cutoff. + double blob_garbage_collection_age_cutoff = -1; +}; + +// IngestExternalFileOptions is used by IngestExternalFile() +struct IngestExternalFileOptions { + // Can be set to true to move the files instead of copying them. + bool move_files = false; + // If set to true, ingestion falls back to copy when move fails. + bool failed_move_fall_back_to_copy = true; + // If set to false, an ingested file keys could appear in existing snapshots + // that where created before the file was ingested. + bool snapshot_consistency = true; + // If set to false, IngestExternalFile() will fail if the file key range + // overlaps with existing keys or tombstones or output of ongoing compaction + // during file ingestion in the DB (the conditions under which a global_seqno + // must be assigned to the ingested file). + bool allow_global_seqno = true; + // If set to false and the file key range overlaps with the memtable key range + // (memtable flush required), IngestExternalFile will fail. + bool allow_blocking_flush = true; + // Set to true if you would like duplicate keys in the file being ingested + // to be skipped rather than overwriting existing data under that key. + // Use case: back-fill of some historical data in the database without + // over-writing existing newer version of data. + // This option could only be used if the DB has been running + // with allow_ingest_behind=true since the dawn of time. + // All files will be ingested at the bottommost level with seqno=0. + bool ingest_behind = false; + // DEPRECATED - Set to true if you would like to write global_seqno to + // the external SST file on ingestion for backward compatibility before + // RocksDB 5.16.0. Such old versions of RocksDB expect any global_seqno to + // be written to the SST file rather than recorded in the DB manifest. + // This functionality was deprecated because (a) random writes might be + // costly or unsupported on some FileSystems, and (b) the file checksum + // changes with such a write. + bool write_global_seqno = false; + // Set to true if you would like to verify the checksums of each block of the + // external SST file before ingestion. + // Warning: setting this to true causes slowdown in file ingestion because + // the external SST file has to be read. + bool verify_checksums_before_ingest = false; + // When verify_checksums_before_ingest = true, RocksDB uses default + // readahead setting to scan the file while verifying checksums before + // ingestion. + // Users can override the default value using this option. + // Using a large readahead size (> 2MB) can typically improve the performance + // of forward iteration on spinning disks. + size_t verify_checksums_readahead_size = 0; + // Set to TRUE if user wants to verify the sst file checksum of ingested + // files. The DB checksum function will generate the checksum of each + // ingested file (if file_checksum_gen_factory is set) and compare the + // checksum function name and checksum with the ingested checksum information. + // + // If this option is set to True: 1) if DB does not enable checksum + // (file_checksum_gen_factory == nullptr), the ingested checksum information + // will be ignored; 2) If DB enable the checksum function, we calculate the + // sst file checksum after the file is moved or copied and compare the + // checksum and checksum name. If checksum or checksum function name does + // not match, ingestion will be failed. If the verification is successful, + // checksum and checksum function name will be stored in Manifest. + // If this option is set to FALSE, 1) if DB does not enable checksum, + // the ingested checksum information will be ignored; 2) if DB enable the + // checksum, we only verify the ingested checksum function name and we + // trust the ingested checksum. If the checksum function name matches, we + // store the checksum in Manifest. DB does not calculate the checksum during + // ingestion. However, if no checksum information is provided with the + // ingested files, DB will generate the checksum and store in the Manifest. + bool verify_file_checksum = true; + // Set to TRUE if user wants file to be ingested to the bottommost level. An + // error of Status::TryAgain() will be returned if a file cannot fit in the + // bottommost level when calling + // DB::IngestExternalFile()/DB::IngestExternalFiles(). The user should clear + // the bottommost level in the overlapping range before re-attempt. + // + // ingest_behind takes precedence over fail_if_not_bottommost_level. + bool fail_if_not_bottommost_level = false; +}; + +enum TraceFilterType : uint64_t { + // Trace all the operations + kTraceFilterNone = 0x0, + // Do not trace the get operations + kTraceFilterGet = 0x1 << 0, + // Do not trace the write operations + kTraceFilterWrite = 0x1 << 1, + // Do not trace the `Iterator::Seek()` operations + kTraceFilterIteratorSeek = 0x1 << 2, + // Do not trace the `Iterator::SeekForPrev()` operations + kTraceFilterIteratorSeekForPrev = 0x1 << 3, + // Do not trace the `MultiGet()` operations + kTraceFilterMultiGet = 0x1 << 4, +}; + +// TraceOptions is used for StartTrace +struct TraceOptions { + // To avoid the trace file size grows large than the storage space, + // user can set the max trace file size in Bytes. Default is 64GB + uint64_t max_trace_file_size = uint64_t{64} * 1024 * 1024 * 1024; + // Specify trace sampling option, i.e. capture one per how many requests. + // Default to 1 (capture every request). + uint64_t sampling_frequency = 1; + // Note: The filtering happens before sampling. + uint64_t filter = kTraceFilterNone; + // When true, the order of write records in the trace will match the order of + // the corresponding write records in the WAL and applied to the DB. There may + // be a performance penalty associated with preserving this ordering. + // + // Default: false. This means write records in the trace may be in an order + // different from the WAL's order. + bool preserve_write_order = false; +}; + +// ImportColumnFamilyOptions is used by ImportColumnFamily() +struct ImportColumnFamilyOptions { + // Can be set to true to move the files instead of copying them. + bool move_files = false; +}; + +// Options used with DB::GetApproximateSizes() +struct SizeApproximationOptions { + // Defines whether the returned size should include the recently written + // data in the memtables. If set to false, include_files must be true. + bool include_memtables = false; + // Defines whether the returned size should include data serialized to disk. + // If set to false, include_memtables must be true. + bool include_files = true; + // When approximating the files total size that is used to store a keys range + // using DB::GetApproximateSizes, allow approximation with an error margin of + // up to total_files_size * files_size_error_margin. This allows to take some + // shortcuts in files size approximation, resulting in better performance, + // while guaranteeing the resulting error is within a reasonable margin. + // E.g., if the value is 0.1, then the error margin of the returned files size + // approximation will be within 10%. + // If the value is non-positive - a more precise yet more CPU intensive + // estimation is performed. + double files_size_error_margin = -1.0; +}; + +struct CompactionServiceOptionsOverride { + // Currently pointer configurations are not passed to compaction service + // compaction so the user needs to set it. It will be removed once pointer + // configuration passing is supported. + Env* env = Env::Default(); + std::shared_ptr file_checksum_gen_factory = nullptr; + + const Comparator* comparator = BytewiseComparator(); + std::shared_ptr merge_operator = nullptr; + const CompactionFilter* compaction_filter = nullptr; + std::shared_ptr compaction_filter_factory = nullptr; + std::shared_ptr prefix_extractor = nullptr; + std::shared_ptr table_factory; + std::shared_ptr sst_partitioner_factory = nullptr; + + // Only subsets of events are triggered in remote compaction worker, like: + // `OnTableFileCreated`, `OnTableFileCreationStarted`, + // `ShouldBeNotifiedOnFileIO` `OnSubcompactionBegin`, + // `OnSubcompactionCompleted`, etc. Worth mentioning, `OnCompactionBegin` and + // `OnCompactionCompleted` won't be triggered. They will be triggered on the + // primary DB side. + std::vector> listeners; + + // statistics is used to collect DB operation metrics, the metrics won't be + // returned to CompactionService primary host, to collect that, the user needs + // to set it here. + std::shared_ptr statistics = nullptr; + + // Only compaction generated SST files use this user defined table properties + // collector. + std::vector> + table_properties_collector_factories; +}; + +struct OpenAndCompactOptions { + // Allows cancellation of an in-progress compaction. + std::atomic* canceled = nullptr; +}; + +struct LiveFilesStorageInfoOptions { + // Whether to populate FileStorageInfo::file_checksum* or leave blank + bool include_checksum_info = false; + // Flushes memtables if total size in bytes of live WAL files is >= this + // number (and DB is not read-only). + // Default: always force a flush without checking sizes. + uint64_t wal_size_for_flush = 0; +}; + +struct WaitForCompactOptions { + // A boolean to abort waiting in case of a pause (PauseBackgroundWork() + // called) If true, Status::Aborted will be returned immediately. If false, + // ContinueBackgroundWork() must be called to resume the background jobs. + // Otherwise, jobs that were queued, but not scheduled yet may never finish + // and WaitForCompact() may wait indefinitely. + bool abort_on_pause = false; + + // A boolean to flush all column families before starting to wait. + bool flush = false; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/perf_context.h b/librocksdb-sys/rocksdb/include/rocksdb/perf_context.h new file mode 100644 index 0000000..46266ab --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/perf_context.h @@ -0,0 +1,314 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include +#include + +#include "rocksdb/perf_level.h" + +namespace ROCKSDB_NAMESPACE { + +/* + * NOTE: + * Please do not reorder the fields in this structure. If you plan to do that or + * add/remove fields to this structure, builds would fail. The way to fix the + * builds would be to add the appropriate fields to the + * DEF_PERF_CONTEXT_LEVEL_METRICS() macro in the perf_context.cc file. + */ + +// Break down performance counters by level and store per-level perf context in +// PerfContextByLevel +struct PerfContextByLevelBase { + // These Bloom stats apply to point reads (Get/MultiGet) for whole key and + // prefix filters. + // # of times bloom filter has avoided file reads, i.e., negatives. + uint64_t bloom_filter_useful = 0; + // # of times bloom FullFilter has not avoided the reads. + uint64_t bloom_filter_full_positive = 0; + // # of times bloom FullFilter has not avoided the reads and data actually + // exist. + uint64_t bloom_filter_full_true_positive = 0; + + // total number of user key returned (only include keys that are found, does + // not include keys that are deleted or merged without a final put + uint64_t user_key_return_count = 0; + + // total nanos spent on reading data from SST files + uint64_t get_from_table_nanos = 0; + + uint64_t block_cache_hit_count = 0; // total number of block cache hits + uint64_t block_cache_miss_count = 0; // total number of block cache misses +}; + +// A thread local context for gathering performance counter efficiently +// and transparently. +// Use SetPerfLevel(PerfLevel::kEnableTime) to enable time stats. + +// Break down performance counters by level and store per-level perf context in +// PerfContextByLevel +struct PerfContextByLevel : public PerfContextByLevelBase { + void Reset(); // reset all performance counters to zero +}; + +/* + * NOTE: + * Please do not reorder the fields in this structure. If you plan to do that or + * add/remove fields to this structure, builds would fail. The way to fix the + * builds would be to add the appropriate fields to the + * DEF_PERF_CONTEXT_METRICS() macro in the perf_context.cc file. + */ + +struct PerfContextBase { + uint64_t user_key_comparison_count; // total number of user key comparisons + uint64_t block_cache_hit_count; // total number of block cache hits + uint64_t block_read_count; // total number of block reads (with IO) + uint64_t block_read_byte; // total number of bytes from block reads + uint64_t block_read_time; // total nanos spent on block reads + // total cpu time in nanos spent on block reads + uint64_t block_read_cpu_time; + uint64_t block_cache_index_hit_count; // total number of index block hits + // total number of standalone handles lookup from secondary cache + uint64_t block_cache_standalone_handle_count; + // total number of real handles lookup from secondary cache that are inserted + // into primary cache + uint64_t block_cache_real_handle_count; + uint64_t index_block_read_count; // total number of index block reads + uint64_t block_cache_filter_hit_count; // total number of filter block hits + uint64_t filter_block_read_count; // total number of filter block reads + uint64_t compression_dict_block_read_count; // total number of compression + // dictionary block reads + + uint64_t secondary_cache_hit_count; // total number of secondary cache hits + // total number of real handles inserted into secondary cache + uint64_t compressed_sec_cache_insert_real_count; + // total number of dummy handles inserted into secondary cache + uint64_t compressed_sec_cache_insert_dummy_count; + // bytes for vals before compression in secondary cache + uint64_t compressed_sec_cache_uncompressed_bytes; + // bytes for vals after compression in secondary cache + uint64_t compressed_sec_cache_compressed_bytes; + + uint64_t block_checksum_time; // total nanos spent on block checksum + uint64_t block_decompress_time; // total nanos spent on block decompression + + uint64_t get_read_bytes; // bytes for vals returned by Get + uint64_t multiget_read_bytes; // bytes for vals returned by MultiGet + uint64_t iter_read_bytes; // bytes for keys/vals decoded by iterator + + uint64_t blob_cache_hit_count; // total number of blob cache hits + uint64_t blob_read_count; // total number of blob reads (with IO) + uint64_t blob_read_byte; // total number of bytes from blob reads + uint64_t blob_read_time; // total nanos spent on blob reads + uint64_t blob_checksum_time; // total nanos spent on blob checksum + uint64_t blob_decompress_time; // total nanos spent on blob decompression + + // total number of internal keys skipped over during iteration. + // There are several reasons for it: + // 1. when calling Next(), the iterator is in the position of the previous + // key, so that we'll need to skip it. It means this counter will always + // be incremented in Next(). + // 2. when calling Next(), we need to skip internal entries for the previous + // keys that are overwritten. + // 3. when calling Next(), Seek() or SeekToFirst(), after previous key + // before calling Next(), the seek key in Seek() or the beginning for + // SeekToFirst(), there may be one or more deleted keys before the next + // valid key that the operation should place the iterator to. We need + // to skip both of the tombstone and updates hidden by the tombstones. The + // tombstones are not included in this counter, while previous updates + // hidden by the tombstones will be included here. + // 4. symmetric cases for Prev() and SeekToLast() + // internal_recent_skipped_count is not included in this counter. + // + uint64_t internal_key_skipped_count; + // Total number of deletes and single deletes skipped over during iteration + // When calling Next(), Seek() or SeekToFirst(), after previous position + // before calling Next(), the seek key in Seek() or the beginning for + // SeekToFirst(), there may be one or more deleted keys before the next valid + // key. Every deleted key is counted once. We don't recount here if there are + // still older updates invalidated by the tombstones. + // + uint64_t internal_delete_skipped_count; + // How many times iterators skipped over internal keys that are more recent + // than the snapshot that iterator is using. + // + uint64_t internal_recent_skipped_count; + // How many merge operands were fed into the merge operator by iterators. + // Note: base values are not included in the count. + // + uint64_t internal_merge_count; + // How many merge operands were fed into the merge operator by point lookups. + // Note: base values are not included in the count. + // + uint64_t internal_merge_point_lookup_count; + // Number of times we reseeked inside a merging iterator, specifically to skip + // after or before a range of keys covered by a range deletion in a newer LSM + // component. + uint64_t internal_range_del_reseek_count; + + uint64_t get_snapshot_time; // total nanos spent on getting snapshot + uint64_t get_from_memtable_time; // total nanos spent on querying memtables + uint64_t get_from_memtable_count; // number of mem tables queried + // total nanos spent after Get() finds a key + uint64_t get_post_process_time; + uint64_t get_from_output_files_time; // total nanos reading from output files + // total nanos spent on seeking memtable + uint64_t seek_on_memtable_time; + // number of seeks issued on memtable + // (including SeekForPrev but not SeekToFirst and SeekToLast) + uint64_t seek_on_memtable_count; + // number of Next()s issued on memtable + uint64_t next_on_memtable_count; + // number of Prev()s issued on memtable + uint64_t prev_on_memtable_count; + // total nanos spent on seeking child iters + uint64_t seek_child_seek_time; + // number of seek issued in child iterators + uint64_t seek_child_seek_count; + uint64_t seek_min_heap_time; // total nanos spent on the merge min heap + uint64_t seek_max_heap_time; // total nanos spent on the merge max heap + // total nanos spent on seeking the internal entries + uint64_t seek_internal_seek_time; + // total nanos spent on iterating internal entries to find the next user entry + uint64_t find_next_user_entry_time; + + // This group of stats provide a breakdown of time spent by Write(). + // May be inaccurate when 2PC, two_write_queues or enable_pipelined_write + // are enabled. + // + // total nanos spent on writing to WAL + uint64_t write_wal_time; + // total nanos spent on writing to mem tables + uint64_t write_memtable_time; + // total nanos spent on delaying or throttling write + uint64_t write_delay_time; + // total nanos spent on switching memtable/wal and scheduling + // flushes/compactions. + uint64_t write_scheduling_flushes_compactions_time; + // total nanos spent on writing a record, excluding the above four things + uint64_t write_pre_and_post_process_time; + + // time spent waiting for other threads of the batch group + uint64_t write_thread_wait_nanos; + + // time spent on acquiring DB mutex. + uint64_t db_mutex_lock_nanos; + // Time spent on waiting with a condition variable created with DB mutex. + uint64_t db_condition_wait_nanos; + // Time spent on merge operator. + uint64_t merge_operator_time_nanos; + + // Time spent on reading index block from block cache or SST file + uint64_t read_index_block_nanos; + // Time spent on reading filter block from block cache or SST file + uint64_t read_filter_block_nanos; + // Time spent on creating data block iterator + uint64_t new_table_block_iter_nanos; + // Time spent on creating a iterator of an SST file. + uint64_t new_table_iterator_nanos; + // Time spent on seeking a key in data/index blocks + uint64_t block_seek_nanos; + // Time spent on finding or creating a table reader + uint64_t find_table_nanos; + // total number of mem table bloom hits + uint64_t bloom_memtable_hit_count; + // total number of mem table bloom misses + uint64_t bloom_memtable_miss_count; + // total number of SST bloom hits + uint64_t bloom_sst_hit_count; + // total number of SST bloom misses + uint64_t bloom_sst_miss_count; + + // Time spent waiting on key locks in transaction lock manager. + uint64_t key_lock_wait_time; + // number of times acquiring a lock was blocked by another transaction. + uint64_t key_lock_wait_count; + + // Total time spent in Env filesystem operations. These are only populated + // when TimedEnv is used. + uint64_t env_new_sequential_file_nanos; + uint64_t env_new_random_access_file_nanos; + uint64_t env_new_writable_file_nanos; + uint64_t env_reuse_writable_file_nanos; + uint64_t env_new_random_rw_file_nanos; + uint64_t env_new_directory_nanos; + uint64_t env_file_exists_nanos; + uint64_t env_get_children_nanos; + uint64_t env_get_children_file_attributes_nanos; + uint64_t env_delete_file_nanos; + uint64_t env_create_dir_nanos; + uint64_t env_create_dir_if_missing_nanos; + uint64_t env_delete_dir_nanos; + uint64_t env_get_file_size_nanos; + uint64_t env_get_file_modification_time_nanos; + uint64_t env_rename_file_nanos; + uint64_t env_link_file_nanos; + uint64_t env_lock_file_nanos; + uint64_t env_unlock_file_nanos; + uint64_t env_new_logger_nanos; + + uint64_t get_cpu_nanos; + uint64_t iter_next_cpu_nanos; + uint64_t iter_prev_cpu_nanos; + uint64_t iter_seek_cpu_nanos; + + // EXPERIMENTAL + // Total number of db iterator's Next(), Prev(), Seek-related APIs being + // called + uint64_t iter_next_count; + uint64_t iter_prev_count; + uint64_t iter_seek_count; + + // Time spent in encrypting data. Populated when EncryptedEnv is used. + uint64_t encrypt_data_nanos; + // Time spent in decrypting data. Populated when EncryptedEnv is used. + uint64_t decrypt_data_nanos; + + uint64_t number_async_seek; +}; + +struct PerfContext : public PerfContextBase { + ~PerfContext(); + + PerfContext() {} + + PerfContext(const PerfContext&); + PerfContext& operator=(const PerfContext&); + PerfContext(PerfContext&&) noexcept; + + void Reset(); // reset all performance counters to zero + + std::string ToString(bool exclude_zero_counters = false) const; + + // enable per level perf context and allocate storage for PerfContextByLevel + void EnablePerLevelPerfContext(); + + // temporarily disable per level perf context by setting the flag to false + void DisablePerLevelPerfContext(); + + // free the space for PerfContextByLevel, also disable per level perf context + void ClearPerLevelPerfContext(); + + std::map* level_to_perf_context = nullptr; + bool per_level_perf_context_enabled = false; + + void copyMetrics(const PerfContext* other) noexcept; +}; + +// If RocksDB is compiled with -DNPERF_CONTEXT, then a pointer to a global, +// non-thread-local PerfContext object will be returned. Attempts to update +// this object will be ignored, and reading from it will also be no-op. +// Otherwise, +// a) if thread-local is supported on the platform, then a pointer to +// a thread-local PerfContext object will be returned. +// b) if thread-local is NOT supported, then compilation will fail. +// +// This function never returns nullptr. +PerfContext* get_perf_context(); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/perf_level.h b/librocksdb-sys/rocksdb/include/rocksdb/perf_level.h new file mode 100644 index 0000000..e7dded0 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/perf_level.h @@ -0,0 +1,36 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +// How much perf stats to collect. Affects perf_context and iostats_context. +enum PerfLevel : unsigned char { + kUninitialized = 0, // unknown setting + kDisable = 1, // disable perf stats + kEnableCount = 2, // enable only count stats + kEnableTimeExceptForMutex = 3, // Other than count stats, also enable time + // stats except for mutexes + // Other than time, also measure CPU time counters. Still don't measure + // time (neither wall time nor CPU time) for mutexes. + kEnableTimeAndCPUTimeExceptForMutex = 4, + kEnableTime = 5, // enable count and time stats + kOutOfBounds = 6 // N.B. Must always be the last value! +}; + +// set the perf stats level for current thread +void SetPerfLevel(PerfLevel level); + +// get current perf stats level for current thread +PerfLevel GetPerfLevel(); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/persistent_cache.h b/librocksdb-sys/rocksdb/include/rocksdb/persistent_cache.h new file mode 100644 index 0000000..f14f019 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/persistent_cache.h @@ -0,0 +1,74 @@ +// Copyright (c) 2013, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once + +#include + +#include +#include + +#include "rocksdb/env.h" +#include "rocksdb/slice.h" +#include "rocksdb/statistics.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +// PersistentCache +// +// Persistent cache interface for caching IO pages on a persistent medium. The +// cache interface is specifically designed for persistent read cache. +class PersistentCache { + public: + using StatsType = std::vector>; + + virtual ~PersistentCache() {} + + // Insert to page cache + // + // page_key Identifier to identify a page uniquely across restarts + // data Page data to copy (caller retains ownership) + // size Size of the page + virtual Status Insert(const Slice& key, const char* data, + const size_t size) = 0; + + // Lookup page cache by page identifier + // + // page_key Page identifier + // buf Buffer where the data should be copied + // size Size of the page + virtual Status Lookup(const Slice& key, std::unique_ptr* data, + size_t* size) = 0; + + // True if the cache is configured to store serialized blocks, which are + // potentially compressed and include a trailer (when SST format calls for + // one). False if the cache stores uncompressed blocks (no trailer). + virtual bool IsCompressed() = 0; + + // Return stats as map of {string, double} per-tier + // + // Persistent cache can be initialized as a tier of caches. The stats are per + // tire top-down + virtual StatsType Stats() = 0; + + virtual std::string GetPrintableOptions() const = 0; + + // Return a new numeric id. May be used by multiple clients who are + // sharding the same persistent cache to partition the key space. Typically + // the client will allocate a new id at startup and prepend the id to its + // cache keys. + virtual uint64_t NewId() = 0; +}; + +// Factor method to create a new persistent cache +Status NewPersistentCache(Env* const env, const std::string& path, + const uint64_t size, + const std::shared_ptr& log, + const bool optimized_for_nvm, + std::shared_ptr* cache); +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/port_defs.h b/librocksdb-sys/rocksdb/include/rocksdb/port_defs.h new file mode 100644 index 0000000..9771aac --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/port_defs.h @@ -0,0 +1,22 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file includes the common definitions used in the port/, +// the public API (this directory), and other directories + +#pragma once + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +enum class CpuPriority { + kIdle = 0, + kLow = 1, + kNormal = 2, + kHigh = 3, +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/rate_limiter.h b/librocksdb-sys/rocksdb/include/rocksdb/rate_limiter.h new file mode 100644 index 0000000..9cad6ed --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/rate_limiter.h @@ -0,0 +1,159 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include "rocksdb/env.h" +#include "rocksdb/statistics.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +// Exceptions MUST NOT propagate out of overridden functions into RocksDB, +// because RocksDB is not exception-safe. This could cause undefined behavior +// including data loss, unreported corruption, deadlocks, and more. +class RateLimiter { + public: + enum class OpType { + kRead, + kWrite, + }; + + enum class Mode { + kReadsOnly, + kWritesOnly, + kAllIo, + }; + + // For API compatibility, default to rate-limiting writes only. + explicit RateLimiter(Mode mode = Mode::kWritesOnly) : mode_(mode) {} + + virtual ~RateLimiter() {} + + // This API allows user to dynamically change rate limiter's bytes per second. + // REQUIRED: bytes_per_second > 0 + virtual void SetBytesPerSecond(int64_t bytes_per_second) = 0; + + // Deprecated. New RateLimiter derived classes should override + // Request(const int64_t, const Env::IOPriority, Statistics*) or + // Request(const int64_t, const Env::IOPriority, Statistics*, OpType) + // instead. + // + // Request for token for bytes. If this request can not be satisfied, the call + // is blocked. Caller is responsible to make sure + // bytes <= GetSingleBurstBytes() + // and bytes >= 0. + virtual void Request(const int64_t /*bytes*/, const Env::IOPriority /*pri*/) { + assert(false); + } + + // Request for token for bytes and potentially update statistics. If this + // request can not be satisfied, the call is blocked. Caller is responsible to + // make sure bytes <= GetSingleBurstBytes() + // and bytes >= 0. + virtual void Request(const int64_t bytes, const Env::IOPriority pri, + Statistics* /* stats */) { + // For API compatibility, default implementation calls the older API in + // which statistics are unsupported. + Request(bytes, pri); + } + + // Requests token to read or write bytes and potentially updates statistics. + // + // If this request can not be satisfied, the call is blocked. Caller is + // responsible to make sure bytes <= GetSingleBurstBytes() + // and bytes >= 0. + virtual void Request(const int64_t bytes, const Env::IOPriority pri, + Statistics* stats, OpType op_type) { + if (IsRateLimited(op_type)) { + Request(bytes, pri, stats); + } + } + + // Requests token to read or write bytes and potentially updates statistics. + // Takes into account GetSingleBurstBytes() and alignment (e.g., in case of + // direct I/O) to allocate an appropriate number of bytes, which may be less + // than the number of bytes requested. + virtual size_t RequestToken(size_t bytes, size_t alignment, + Env::IOPriority io_priority, Statistics* stats, + RateLimiter::OpType op_type); + + // Max bytes can be granted in a single burst + virtual int64_t GetSingleBurstBytes() const = 0; + + // Total bytes that go through rate limiter + virtual int64_t GetTotalBytesThrough( + const Env::IOPriority pri = Env::IO_TOTAL) const = 0; + + // Total # of requests that go through rate limiter + virtual int64_t GetTotalRequests( + const Env::IOPriority pri = Env::IO_TOTAL) const = 0; + + // Total # of requests that are pending for bytes in rate limiter + // For convenience, this function is supported by the RateLimiter returned + // by NewGenericRateLimiter but is not required by RocksDB. + // + // REQUIRED: total_pending_request != nullptr + virtual Status GetTotalPendingRequests( + int64_t* total_pending_requests, + const Env::IOPriority pri = Env::IO_TOTAL) const { + assert(total_pending_requests != nullptr); + (void)total_pending_requests; + (void)pri; + return Status::NotSupported(); + } + + virtual int64_t GetBytesPerSecond() const = 0; + + virtual bool IsRateLimited(OpType op_type) { + if ((mode_ == RateLimiter::Mode::kWritesOnly && + op_type == RateLimiter::OpType::kRead) || + (mode_ == RateLimiter::Mode::kReadsOnly && + op_type == RateLimiter::OpType::kWrite)) { + return false; + } + return true; + } + + protected: + Mode GetMode() { return mode_; } + + private: + const Mode mode_; +}; + +// Create a RateLimiter object, which can be shared among RocksDB instances to +// control write rate of flush and compaction. +// @rate_bytes_per_sec: this is the only parameter you want to set most of the +// time. It controls the total write rate of compaction and flush in bytes per +// second. Currently, RocksDB does not enforce rate limit for anything other +// than flush and compaction, e.g. write to WAL. +// @refill_period_us: this controls how often tokens are refilled. For example, +// when rate_bytes_per_sec is set to 10MB/s and refill_period_us is set to +// 100ms, then 1MB is refilled every 100ms internally. Larger value can lead to +// burstier writes while smaller value introduces more CPU overhead. +// The default should work for most cases. +// @fairness: RateLimiter accepts high-pri requests and low-pri requests. +// A low-pri request is usually blocked in favor of hi-pri request. Currently, +// RocksDB assigns low-pri to request from compaction and high-pri to request +// from flush. Low-pri requests can get blocked if flush requests come in +// continuously. This fairness parameter grants low-pri requests permission by +// 1/fairness chance even though high-pri requests exist to avoid starvation. +// You should be good by leaving it at default 10. +// @mode: Mode indicates which types of operations count against the limit. +// @auto_tuned: Enables dynamic adjustment of rate limit within the range +// `[rate_bytes_per_sec / 20, rate_bytes_per_sec]`, according to +// the recent demand for background I/O. +extern RateLimiter* NewGenericRateLimiter( + int64_t rate_bytes_per_sec, int64_t refill_period_us = 100 * 1000, + int32_t fairness = 10, + RateLimiter::Mode mode = RateLimiter::Mode::kWritesOnly, + bool auto_tuned = false); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/rocksdb_namespace.h b/librocksdb-sys/rocksdb/include/rocksdb/rocksdb_namespace.h new file mode 100644 index 0000000..a339ec2 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/rocksdb_namespace.h @@ -0,0 +1,16 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +// For testing purposes +#if ROCKSDB_NAMESPACE == 42 +#undef ROCKSDB_NAMESPACE +#endif + +// Normal logic +#ifndef ROCKSDB_NAMESPACE +#define ROCKSDB_NAMESPACE rocksdb +#endif diff --git a/librocksdb-sys/rocksdb/include/rocksdb/secondary_cache.h b/librocksdb-sys/rocksdb/include/rocksdb/secondary_cache.h new file mode 100644 index 0000000..7c88281 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/secondary_cache.h @@ -0,0 +1,150 @@ +// Copyright (c) 2021, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include + +#include +#include + +#include "rocksdb/advanced_cache.h" +#include "rocksdb/customizable.h" +#include "rocksdb/slice.h" +#include "rocksdb/statistics.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +// A handle for lookup result. Immediately after SecondaryCache::Lookup() with +// wait=false (and depending on the implementation), the handle could be in any +// of the below states. It must not be destroyed while in the pending state. +// * Pending state (IsReady() == false): result is not ready. Value() and Size() +// must not be called. +// * Ready + not found state (IsReady() == true, Value() == nullptr): the lookup +// has completed, finding no match. Or an error occurred that prevented +// normal completion of the Lookup. +// * Ready + found state (IsReady() == false, Value() != nullptr): the lookup +// has completed, finding an entry that has been loaded into an object that is +// now owned by the caller. +// +// Wait() or SecondaryCache::WaitAll() may be skipped if IsReady() happens to +// return true, but (depending on the implementation) IsReady() might never +// return true without Wait() or SecondaryCache::WaitAll(). After the handle +// is known ready, calling Value() is required to avoid a memory leak in case +// of a cache hit. +class SecondaryCacheResultHandle { + public: + virtual ~SecondaryCacheResultHandle() = default; + + // Returns whether the handle is ready or not + virtual bool IsReady() = 0; + + // Block until handle becomes ready + virtual void Wait() = 0; + + // Return the cache entry object (also known as value). If nullptr, it means + // the lookup was unsuccessful. + virtual Cache::ObjectPtr Value() = 0; + + // Return the out_charge from the helper->create_cb used to construct the + // object. + // WART: potentially confusing name + virtual size_t Size() = 0; +}; + +// SecondaryCache +// +// Cache interface for caching blocks on a secondary tier (which can include +// non-volatile media, or alternate forms of caching such as compressed data) +// +// Exceptions MUST NOT propagate out of overridden functions into RocksDB, +// because RocksDB is not exception-safe. This could cause undefined behavior +// including data loss, unreported corruption, deadlocks, and more. +class SecondaryCache : public Customizable { + public: + ~SecondaryCache() override = default; + + static const char* Type() { return "SecondaryCache"; } + static Status CreateFromString(const ConfigOptions& config_options, + const std::string& id, + std::shared_ptr* result); + + // Suggest inserting an entry into this cache. The caller retains ownership + // of `obj` (also called the "value"), so is only used directly by the + // SecondaryCache during Insert(). When the cache chooses to perform the + // suggested insertion, it uses the size_cb and saveto_cb provided by + // `helper` to extract the persistable data (typically an uncompressed block) + // and writes it to this cache tier. OK may be returned even if the insertion + // is not made. + virtual Status Insert(const Slice& key, Cache::ObjectPtr obj, + const Cache::CacheItemHelper* helper) = 0; + + // Insert a value from its saved/persistable data (typically uncompressed + // block), as if generated by SaveToCallback/SizeCallback. This can be used + // in "warming up" the cache from some auxiliary source, and like Insert() + // may or may not write it to cache depending on the admission control + // policy, even if the return status is success. + // + // The default implementation only assumes the entry helper's create_cb is + // called at Lookup() time and not Insert() time, so should work for all + // foreseeable implementations. + virtual Status InsertSaved(const Slice& key, const Slice& saved); + + // Lookup the data for the given key in this cache. The create_cb + // will be used to create the object. The handle returned may not be + // ready yet, unless wait=true, in which case Lookup() will block until + // the handle is ready. + // + // advise_erase is a hint from the primary cache indicating that the handle + // will be cached there, so the secondary cache is advised to drop it from + // the cache as an optimization. To use this feature, SupportForceErase() + // needs to return true. + // This hint can also be safely ignored. + // + // kept_in_sec_cache is to indicate whether the entry will be kept in the + // secondary cache after the Lookup (rather than erased because of Lookup) + virtual std::unique_ptr Lookup( + const Slice& key, const Cache::CacheItemHelper* helper, + Cache::CreateContext* create_context, bool wait, bool advise_erase, + bool& kept_in_sec_cache) = 0; + + // Indicate whether a handle can be erased in this secondary cache. + [[nodiscard]] virtual bool SupportForceErase() const = 0; + + // At the discretion of the implementation, erase the data associated + // with key. + virtual void Erase(const Slice& key) = 0; + + // Wait for a collection of handles to become ready. + virtual void WaitAll(std::vector handles) = 0; + + // Set the maximum configured capacity of the cache. + // When the new capacity is less than the old capacity and the existing usage + // is greater than new capacity, the implementation will do its best job to + // purge the released entries from the cache in order to lower the usage. + // + // The derived class can make this function no-op and return NotSupported(). + virtual Status SetCapacity(size_t /* capacity */) { + return Status::NotSupported(); + } + + // The derived class can make this function no-op and return NotSupported(). + virtual Status GetCapacity(size_t& /* capacity */) { + return Status::NotSupported(); + } + + // Temporarily decrease the cache capacity in RAM by the specified amount. + // The caller should call Inflate() to restore the cache capacity. This is + // intended to be lighter weight than SetCapacity(). The latter evenly + // distributes the new capacity across all shards and is meant for large + // changes in capacity, whereas the former is meant for relatively small + // changes and may be uneven by lowering capacity in a single shard. + virtual Status Deflate(size_t /*decrease*/) { return Status::NotSupported(); } + + // Restore the capacity reduced by a prior call to Deflate(). + virtual Status Inflate(size_t /*increase*/) { return Status::NotSupported(); } +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/slice.h b/librocksdb-sys/rocksdb/include/rocksdb/slice.h new file mode 100644 index 0000000..0d7eb59 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/slice.h @@ -0,0 +1,264 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Slice is a simple structure containing a pointer into some external +// storage and a size. The user of a Slice must ensure that the slice +// is not used after the corresponding external storage has been +// deallocated. +// +// Multiple threads can invoke const methods on a Slice without +// external synchronization, but if any of the threads may call a +// non-const method, all threads accessing the same Slice must use +// external synchronization. + +#pragma once + +#include +#include +#include +#include +#include +#include // RocksDB now requires C++17 support + +#include "rocksdb/cleanable.h" + +namespace ROCKSDB_NAMESPACE { + +class Slice { + public: + // Create an empty slice. + Slice() : data_(""), size_(0) {} + + // Create a slice that refers to d[0,n-1]. + Slice(const char* d, size_t n) : data_(d), size_(n) {} + + // Create a slice that refers to the contents of "s" + /* implicit */ + Slice(const std::string& s) : data_(s.data()), size_(s.size()) {} + + // Create a slice that refers to the same contents as "sv" + /* implicit */ + Slice(const std::string_view& sv) : data_(sv.data()), size_(sv.size()) {} + + // Create a slice that refers to s[0,strlen(s)-1] + /* implicit */ + Slice(const char* s) : data_(s) { size_ = (s == nullptr) ? 0 : strlen(s); } + + // Create a single slice from SliceParts using buf as storage. + // buf must exist as long as the returned Slice exists. + Slice(const struct SliceParts& parts, std::string* buf); + + // Return a pointer to the beginning of the referenced data + const char* data() const { return data_; } + + // Return the length (in bytes) of the referenced data + size_t size() const { return size_; } + + // Return true iff the length of the referenced data is zero + bool empty() const { return size_ == 0; } + + // Return the ith byte in the referenced data. + // REQUIRES: n < size() + char operator[](size_t n) const { + assert(n < size()); + return data_[n]; + } + + // Change this slice to refer to an empty array + void clear() { + data_ = ""; + size_ = 0; + } + + // Drop the first "n" bytes from this slice. + void remove_prefix(size_t n) { + assert(n <= size()); + data_ += n; + size_ -= n; + } + + void remove_suffix(size_t n) { + assert(n <= size()); + size_ -= n; + } + + // Return a string that contains the copy of the referenced data. + // when hex is true, returns a string of twice the length hex encoded (0-9A-F) + std::string ToString(bool hex = false) const; + + // Return a string_view that references the same data as this slice. + std::string_view ToStringView() const { + return std::string_view(data_, size_); + } + + // Decodes the current slice interpreted as an hexadecimal string into result, + // if successful returns true, if this isn't a valid hex string + // (e.g not coming from Slice::ToString(true)) DecodeHex returns false. + // This slice is expected to have an even number of 0-9A-F characters + // also accepts lowercase (a-f) + bool DecodeHex(std::string* result) const; + + // Three-way comparison. Returns value: + // < 0 iff "*this" < "b", + // == 0 iff "*this" == "b", + // > 0 iff "*this" > "b" + int compare(const Slice& b) const; + + // Return true iff "x" is a prefix of "*this" + bool starts_with(const Slice& x) const { + return ((size_ >= x.size_) && (memcmp(data_, x.data_, x.size_) == 0)); + } + + bool ends_with(const Slice& x) const { + return ((size_ >= x.size_) && + (memcmp(data_ + size_ - x.size_, x.data_, x.size_) == 0)); + } + + // Compare two slices and returns the first byte where they differ + size_t difference_offset(const Slice& b) const; + + // private: make these public for rocksdbjni access + const char* data_; + size_t size_; + + // Intentionally copyable +}; + +/** + * A Slice that can be pinned with some cleanup tasks, which will be run upon + * ::Reset() or object destruction, whichever is invoked first. This can be used + * to avoid memcpy by having the PinnableSlice object referring to the data + * that is locked in the memory and release them after the data is consumed. + */ +class PinnableSlice : public Slice, public Cleanable { + public: + PinnableSlice() { buf_ = &self_space_; } + explicit PinnableSlice(std::string* buf) { buf_ = buf; } + + PinnableSlice(PinnableSlice&& other); + PinnableSlice& operator=(PinnableSlice&& other); + + // No copy constructor and copy assignment allowed. + PinnableSlice(PinnableSlice&) = delete; + PinnableSlice& operator=(PinnableSlice&) = delete; + + inline void PinSlice(const Slice& s, CleanupFunction f, void* arg1, + void* arg2) { + assert(!pinned_); + pinned_ = true; + data_ = s.data(); + size_ = s.size(); + RegisterCleanup(f, arg1, arg2); + assert(pinned_); + } + + inline void PinSlice(const Slice& s, Cleanable* cleanable) { + assert(!pinned_); + pinned_ = true; + data_ = s.data(); + size_ = s.size(); + if (cleanable != nullptr) { + cleanable->DelegateCleanupsTo(this); + } + assert(pinned_); + } + + inline void PinSelf(const Slice& slice) { + assert(!pinned_); + buf_->assign(slice.data(), slice.size()); + data_ = buf_->data(); + size_ = buf_->size(); + assert(!pinned_); + } + + inline void PinSelf() { + assert(!pinned_); + data_ = buf_->data(); + size_ = buf_->size(); + assert(!pinned_); + } + + void remove_suffix(size_t n) { + assert(n <= size()); + if (pinned_) { + size_ -= n; + } else { + buf_->erase(size() - n, n); + PinSelf(); + } + } + + void remove_prefix(size_t n) { + assert(n <= size()); + if (pinned_) { + data_ += n; + size_ -= n; + } else { + buf_->erase(0, n); + PinSelf(); + } + } + + void Reset() { + Cleanable::Reset(); + pinned_ = false; + size_ = 0; + } + + inline std::string* GetSelf() { return buf_; } + + inline bool IsPinned() const { return pinned_; } + + private: + friend class PinnableSlice4Test; + std::string self_space_; + std::string* buf_; + bool pinned_ = false; +}; + +// A set of Slices that are virtually concatenated together. 'parts' points +// to an array of Slices. The number of elements in the array is 'num_parts'. +struct SliceParts { + SliceParts(const Slice* _parts, int _num_parts) + : parts(_parts), num_parts(_num_parts) {} + SliceParts() : parts(nullptr), num_parts(0) {} + + const Slice* parts; + int num_parts; +}; + +inline bool operator==(const Slice& x, const Slice& y) { + return ((x.size() == y.size()) && + (memcmp(x.data(), y.data(), x.size()) == 0)); +} + +inline bool operator!=(const Slice& x, const Slice& y) { return !(x == y); } + +inline int Slice::compare(const Slice& b) const { + assert(data_ != nullptr && b.data_ != nullptr); + const size_t min_len = (size_ < b.size_) ? size_ : b.size_; + int r = memcmp(data_, b.data_, min_len); + if (r == 0) { + if (size_ < b.size_) + r = -1; + else if (size_ > b.size_) + r = +1; + } + return r; +} + +inline size_t Slice::difference_offset(const Slice& b) const { + size_t off = 0; + const size_t len = (size_ < b.size_) ? size_ : b.size_; + for (; off < len; off++) { + if (data_[off] != b.data_[off]) break; + } + return off; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/slice_transform.h b/librocksdb-sys/rocksdb/include/rocksdb/slice_transform.h new file mode 100644 index 0000000..8909b9c --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/slice_transform.h @@ -0,0 +1,135 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Class for specifying user-defined functions which perform a +// transformation on a slice. It is not required that every slice +// belong to the domain and/or range of a function. Subclasses should +// define InDomain and InRange to determine which slices are in either +// of these sets respectively. + +#pragma once + +#include +#include + +#include "rocksdb/customizable.h" +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +class Slice; +struct ConfigOptions; + +// A SliceTransform is a generic pluggable way of transforming one string +// to another. Its primary use-case is in configuring RocksDB prefix Bloom +// filters, by setting prefix_extractor in ColumnFamilyOptions. +// +// Exceptions MUST NOT propagate out of overridden functions into RocksDB, +// because RocksDB is not exception-safe. This could cause undefined behavior +// including data loss, unreported corruption, deadlocks, and more. +class SliceTransform : public Customizable { + public: + virtual ~SliceTransform(){}; + + // Return the name of this transformation. + virtual const char* Name() const override = 0; + static const char* Type() { return "SliceTransform"; } + + // Creates and configures a new SliceTransform from the input options and id. + static Status CreateFromString(const ConfigOptions& config_options, + const std::string& id, + std::shared_ptr* result); + + // Returns a string representation of this SliceTransform, representing the ID + // and any additional properties. + std::string AsString() const; + + // Extract a prefix from a specified key, partial key, iterator upper bound, + // etc. This is normally used for building and checking prefix Bloom filters + // but should accept any string for which InDomain() returns true. + // See ColumnFamilyOptions::prefix_extractor for specific properties that + // must be satisfied by prefix extractors. + virtual Slice Transform(const Slice& key) const = 0; + + // Determine whether the specified key is compatible with the logic + // specified in the Transform method. Keys for which InDomain returns + // false will not be added to or queried against prefix Bloom filters. + // + // For example, if the Transform method returns a fixed length + // prefix of size 4, then an invocation to InDomain("abc") returns + // false because the specified key length(3) is shorter than the + // prefix size of 4. + // + // Wiki documentation here: + // https://github.com/facebook/rocksdb/wiki/Prefix-Seek + // + virtual bool InDomain(const Slice& key) const = 0; + + // DEPRECATED: This is currently not used and remains here for backward + // compatibility. + virtual bool InRange(const Slice& /*dst*/) const { return false; } + + // Returns information on maximum prefix length, if there is one. + // If Transform(x).size() == n for some keys and otherwise < n, + // should return true and set *len = n. Returning false is safe but + // currently disables some auto_prefix_mode filtering. + // Specifically, if the iterate_upper_bound is the immediate successor (see + // Comparator::IsSameLengthImmediateSuccessor) of the seek key's prefix, + // we require this function return true and iterate_upper_bound.size() == n + // to recognize and optimize the prefix seek. + // Otherwise (including FullLengthEnabled returns false, or prefix length is + // less than maximum), Seek with auto_prefix_mode is only optimized if the + // iterate_upper_bound and seek key have the same prefix. + // BUG: Despite all these conditions and even with the extra condition on + // IsSameLengthImmediateSuccessor (see it's "BUG" section), it is not + // sufficient to ensure auto_prefix_mode returns all entries that + // total_order_seek would return. See auto_prefix_mode "BUG" section. + virtual bool FullLengthEnabled(size_t* /*len*/) const { return false; } + + // Transform(s)=Transform(`prefix`) for any s with `prefix` as a prefix. + // + // This function is not used by RocksDB, but for users. If users pass + // Options by string to RocksDB, they might not know what prefix extractor + // they are using. This function is to help users can determine: + // if they want to iterate all keys prefixing `prefix`, whether it is + // safe to use prefix bloom filter and seek to key `prefix`. + // If this function returns true, this means a user can Seek() to a prefix + // using the bloom filter. Otherwise, user needs to skip the bloom filter + // by setting ReadOptions.total_order_seek = true. + // + // Here is an example: Suppose we implement a slice transform that returns + // the first part of the string up to and including first ",": + // 1. SameResultWhenAppended("abc,") should return true. If applying prefix + // bloom filter using it, all slices matching "abc,.*" will be extracted + // to "abc,", so any SST file or memtable containing any of those key + // will not be filtered out. + // 2. SameResultWhenAppended("abc") should return false. A user will not be + // guaranteed to see all the keys matching "abc.*" if a user prefix + // seeks to "abc" against a DB with the same setting. If one SST file + // only contains "abcd,e", the file can be filtered out and the key will + // be invisible, because the prefix according to the configured extractor + // is "abcd,". + // + // i.e., an implementation always returning false is safe. + virtual bool SameResultWhenAppended(const Slice& /*prefix*/) const { + return false; + } +}; + +// The prefix is the first `prefix_len` bytes of the key, and keys shorter +// then `prefix_len` are not InDomain. +extern const SliceTransform* NewFixedPrefixTransform(size_t prefix_len); + +// The prefix is the first min(length(key),`cap_len`) bytes of the key, and +// all keys are InDomain. +extern const SliceTransform* NewCappedPrefixTransform(size_t cap_len); + +// Prefix is equal to key. All keys are InDomain. +extern const SliceTransform* NewNoopTransform(); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/snapshot.h b/librocksdb-sys/rocksdb/include/rocksdb/snapshot.h new file mode 100644 index 0000000..1ea56e7 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/snapshot.h @@ -0,0 +1,53 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "rocksdb/types.h" + +namespace ROCKSDB_NAMESPACE { + +class DB; + +// Abstract handle to particular state of a DB. +// A Snapshot is an immutable object and can therefore be safely +// accessed from multiple threads without any external synchronization. +// +// To Create a Snapshot, call DB::GetSnapshot(). +// To Destroy a Snapshot, call DB::ReleaseSnapshot(snapshot). +class Snapshot { + public: + virtual SequenceNumber GetSequenceNumber() const = 0; + + // Returns unix time i.e. the number of seconds since the Epoch, 1970-01-01 + // 00:00:00 (UTC). + virtual int64_t GetUnixTime() const = 0; + + virtual uint64_t GetTimestamp() const = 0; + + protected: + virtual ~Snapshot(); +}; + +// Simple RAII wrapper class for Snapshot. +// Constructing this object will create a snapshot. Destructing will +// release the snapshot. +class ManagedSnapshot { + public: + explicit ManagedSnapshot(DB* db); + + // Instead of creating a snapshot, take ownership of the input snapshot. + ManagedSnapshot(DB* db, const Snapshot* _snapshot); + + ~ManagedSnapshot(); + + const Snapshot* snapshot(); + + private: + DB* db_; + const Snapshot* snapshot_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/sst_dump_tool.h b/librocksdb-sys/rocksdb/include/rocksdb/sst_dump_tool.h new file mode 100644 index 0000000..0b81833 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/sst_dump_tool.h @@ -0,0 +1,17 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include "rocksdb/options.h" + +namespace ROCKSDB_NAMESPACE { + +class SSTDumpTool { + public: + int Run(int argc, char const* const* argv, Options options = Options()); +}; + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/include/rocksdb/sst_file_manager.h b/librocksdb-sys/rocksdb/include/rocksdb/sst_file_manager.h new file mode 100644 index 0000000..b4e5a9b --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/sst_file_manager.h @@ -0,0 +1,137 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include +#include + +#include "rocksdb/file_system.h" +#include "rocksdb/statistics.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +class Env; +class Logger; + +// SstFileManager is used to track SST and blob files in the DB and control +// their deletion rate. All SstFileManager public functions are thread-safe. +// SstFileManager is NOT an extensible interface but a public interface for +// result of NewSstFileManager. Any derived classes must be RocksDB internal. +class SstFileManager { + public: + virtual ~SstFileManager() {} + + // Update the maximum allowed space that should be used by RocksDB, if + // the total size of the SST and blob files exceeds max_allowed_space, writes + // to RocksDB will fail. + // + // Setting max_allowed_space to 0 will disable this feature; maximum allowed + // space will be infinite (Default value). + // + // thread-safe. + virtual void SetMaxAllowedSpaceUsage(uint64_t max_allowed_space) = 0; + + // Set the amount of buffer room each compaction should be able to leave. + // In other words, at its maximum disk space consumption, the compaction + // should still leave compaction_buffer_size available on the disk so that + // other background functions may continue, such as logging and flushing. + virtual void SetCompactionBufferSize(uint64_t compaction_buffer_size) = 0; + + // Return true if the total size of SST and blob files exceeded the maximum + // allowed space usage. + // + // thread-safe. + virtual bool IsMaxAllowedSpaceReached() = 0; + + // Returns true if the total size of SST and blob files as well as estimated + // size of ongoing compactions exceeds the maximums allowed space usage. + virtual bool IsMaxAllowedSpaceReachedIncludingCompactions() = 0; + + // Return the total size of all tracked files. + // thread-safe + virtual uint64_t GetTotalSize() = 0; + + // Return a map containing all tracked files and their corresponding sizes. + // thread-safe + virtual std::unordered_map GetTrackedFiles() = 0; + + // Return delete rate limit in bytes per second. + // thread-safe + virtual int64_t GetDeleteRateBytesPerSecond() = 0; + + // Update the delete rate limit in bytes per second. + // zero means disable delete rate limiting and delete files immediately + // thread-safe + virtual void SetDeleteRateBytesPerSecond(int64_t delete_rate) = 0; + + // Return trash/DB size ratio where new files will be deleted immediately + // thread-safe + virtual double GetMaxTrashDBRatio() = 0; + + // Update trash/DB size ratio where new files will be deleted immediately + // thread-safe + virtual void SetMaxTrashDBRatio(double ratio) = 0; + + // Return the total size of trash files + // thread-safe + virtual uint64_t GetTotalTrashSize() = 0; + + // Set the statistics ptr to dump the stat information + virtual void SetStatisticsPtr(const std::shared_ptr& stats) = 0; +}; + +// Create a new SstFileManager that can be shared among multiple RocksDB +// instances to track SST and blob files and control there deletion rate. +// Even though SstFileManager don't track WAL files but it still control +// there deletion rate. +// +// @param env: Pointer to Env object, please see "rocksdb/env.h". +// @param fs: Pointer to FileSystem object (rocksdb/file_system.h" +// @param info_log: If not nullptr, info_log will be used to log messages of +// INFO, WARN or ERROR level with respect to info_log's info level. +// +// == Deletion rate limiting specific arguments == +// @param trash_dir: Deprecated, this argument have no effect +// @param rate_bytes_per_sec: How many bytes should be deleted per second, If +// this value is set to 1024 (1 Kb / sec) and we deleted a file of size 4 Kb +// in 1 second, we will wait for another 3 seconds before we delete other +// files, Set to 0 to disable deletion rate limiting. +// This option also affect the delete rate of WAL files in the DB. +// @param delete_existing_trash: Deprecated, this argument have no effect, but +// if user provide trash_dir we will schedule deletes for files in the dir +// @param status: If not nullptr, status will contain any errors that happened +// during creating the missing trash_dir or deleting existing files in trash. +// @param max_trash_db_ratio: If the trash size constitutes for more than this +// fraction of the total DB size we will start deleting new files passed to +// DeleteScheduler immediately +// @param bytes_max_delete_chunk: if a file to delete is larger than delete +// chunk, ftruncate the file by this size each time, rather than dropping the +// whole file. 0 means to always delete the whole file. If the file has more +// than one linked names, the file will be deleted as a whole. Either way, +// `rate_bytes_per_sec` will be appreciated. NOTE that with this option, +// files already renamed as a trash may be partial, so users should not +// directly recover them without checking. +extern SstFileManager* NewSstFileManager( + Env* env, std::shared_ptr fs, + std::shared_ptr info_log = nullptr, + const std::string& trash_dir = "", int64_t rate_bytes_per_sec = 0, + bool delete_existing_trash = true, Status* status = nullptr, + double max_trash_db_ratio = 0.25, + uint64_t bytes_max_delete_chunk = 64 * 1024 * 1024); + +// Same as above, but takes a pointer to a legacy Env object, instead of +// Env and FileSystem objects +extern SstFileManager* NewSstFileManager( + Env* env, std::shared_ptr info_log = nullptr, + std::string trash_dir = "", int64_t rate_bytes_per_sec = 0, + bool delete_existing_trash = true, Status* status = nullptr, + double max_trash_db_ratio = 0.25, + uint64_t bytes_max_delete_chunk = 64 * 1024 * 1024); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/sst_file_reader.h b/librocksdb-sys/rocksdb/include/rocksdb/sst_file_reader.h new file mode 100644 index 0000000..026ae66 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/sst_file_reader.h @@ -0,0 +1,45 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + + +#include "rocksdb/iterator.h" +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "rocksdb/table_properties.h" + +namespace ROCKSDB_NAMESPACE { + +// SstFileReader is used to read sst files that are generated by DB or +// SstFileWriter. +class SstFileReader { + public: + SstFileReader(const Options& options); + + ~SstFileReader(); + + // Prepares to read from the file located at "file_path". + Status Open(const std::string& file_path); + + // Returns a new iterator over the table contents. + // Most read options provide the same control as we read from DB. + // If "snapshot" is nullptr, the iterator returns only the latest keys. + Iterator* NewIterator(const ReadOptions& options); + + std::shared_ptr GetTableProperties() const; + + // Verifies whether there is corruption in this table. + Status VerifyChecksum(const ReadOptions& /*read_options*/); + + Status VerifyChecksum() { return VerifyChecksum(ReadOptions()); } + + private: + struct Rep; + std::unique_ptr rep_; +}; + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/include/rocksdb/sst_file_writer.h b/librocksdb-sys/rocksdb/include/rocksdb/sst_file_writer.h new file mode 100644 index 0000000..fb28068 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/sst_file_writer.h @@ -0,0 +1,189 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + + +#include +#include + +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "rocksdb/table_properties.h" +#include "rocksdb/types.h" + +#if defined(__GNUC__) || defined(__clang__) +#define ROCKSDB_DEPRECATED_FUNC __attribute__((__deprecated__)) +#elif _WIN32 +#define ROCKSDB_DEPRECATED_FUNC __declspec(deprecated) +#endif + +namespace ROCKSDB_NAMESPACE { + +class Comparator; + +// ExternalSstFileInfo include information about sst files created +// using SstFileWriter. +struct ExternalSstFileInfo { + ExternalSstFileInfo() + : file_path(""), + smallest_key(""), + largest_key(""), + smallest_range_del_key(""), + largest_range_del_key(""), + file_checksum(""), + file_checksum_func_name(""), + sequence_number(0), + file_size(0), + num_entries(0), + num_range_del_entries(0), + version(0) {} + + ExternalSstFileInfo(const std::string& _file_path, + const std::string& _smallest_key, + const std::string& _largest_key, + SequenceNumber _sequence_number, uint64_t _file_size, + int32_t _num_entries, int32_t _version) + : file_path(_file_path), + smallest_key(_smallest_key), + largest_key(_largest_key), + smallest_range_del_key(""), + largest_range_del_key(""), + file_checksum(""), + file_checksum_func_name(""), + sequence_number(_sequence_number), + file_size(_file_size), + num_entries(_num_entries), + num_range_del_entries(0), + version(_version) {} + + std::string file_path; // external sst file path + std::string smallest_key; // smallest user key in file + std::string largest_key; // largest user key in file + std::string + smallest_range_del_key; // smallest range deletion user key in file + std::string largest_range_del_key; // largest range deletion user key in file + std::string file_checksum; // sst file checksum; + std::string file_checksum_func_name; // The name of file checksum function + SequenceNumber sequence_number; // sequence number of all keys in file + uint64_t file_size; // file size in bytes + uint64_t num_entries; // number of entries in file + uint64_t num_range_del_entries; // number of range deletion entries in file + int32_t version; // file version +}; + +// SstFileWriter is used to create sst files that can be added to database later +// All keys in files generated by SstFileWriter will have sequence number = 0. +class SstFileWriter { + public: + // User can pass `column_family` to specify that the generated file will + // be ingested into this column_family, note that passing nullptr means that + // the column_family is unknown. + // If invalidate_page_cache is set to true, SstFileWriter will give the OS a + // hint that this file pages is not needed every time we write 1MB to the + // file. To use the rate limiter an io_priority smaller than IO_TOTAL can be + // passed. + // The `skip_filters` option is DEPRECATED and could be removed in the + // future. Use `BlockBasedTableOptions::filter_policy` to control filter + // generation. + SstFileWriter(const EnvOptions& env_options, const Options& options, + ColumnFamilyHandle* column_family = nullptr, + bool invalidate_page_cache = true, + Env::IOPriority io_priority = Env::IOPriority::IO_TOTAL, + bool skip_filters = false) + : SstFileWriter(env_options, options, options.comparator, column_family, + invalidate_page_cache, io_priority, skip_filters) {} + + // Deprecated API + SstFileWriter(const EnvOptions& env_options, const Options& options, + const Comparator* user_comparator, + ColumnFamilyHandle* column_family = nullptr, + bool invalidate_page_cache = true, + Env::IOPriority io_priority = Env::IOPriority::IO_TOTAL, + bool skip_filters = false); + + ~SstFileWriter(); + + // Prepare SstFileWriter to write into file located at "file_path". + Status Open(const std::string& file_path); + + // Add a Put key with value to currently opened file (deprecated) + // REQUIRES: user_key is after any previously added point (Put/Merge/Delete) + // key according to the comparator. + // REQUIRES: comparator is *not* timestamp-aware. + ROCKSDB_DEPRECATED_FUNC Status Add(const Slice& user_key, const Slice& value); + + // Add a Put key with value to currently opened file + // REQUIRES: user_key is after any previously added point (Put/Merge/Delete) + // key according to the comparator. + // REQUIRES: comparator is *not* timestamp-aware. + Status Put(const Slice& user_key, const Slice& value); + + // Add a Put (key with timestamp, value) to the currently opened file + // REQUIRES: user_key is after any previously added point (Put/Merge/Delete) + // key according to the comparator. + // REQUIRES: timestamp's size is equal to what is expected by the comparator. + Status Put(const Slice& user_key, const Slice& timestamp, const Slice& value); + + // Add a Merge key with value to currently opened file + // REQUIRES: user_key is after any previously added point (Put/Merge/Delete) + // key according to the comparator. + // REQUIRES: comparator is *not* timestamp-aware. + Status Merge(const Slice& user_key, const Slice& value); + + // Add a deletion key to currently opened file + // REQUIRES: user_key is after any previously added point (Put/Merge/Delete) + // key according to the comparator. + // REQUIRES: comparator is *not* timestamp-aware. + Status Delete(const Slice& user_key); + + // Add a deletion key with timestamp to the currently opened file + // REQUIRES: user_key is after any previously added point (Put/Merge/Delete) + // key according to the comparator. + // REQUIRES: timestamp's size is equal to what is expected by the comparator. + Status Delete(const Slice& user_key, const Slice& timestamp); + + // Add a range deletion tombstone to currently opened file. Such a range + // deletion tombstone does NOT delete point (Put/Merge/Delete) keys in the + // same file. + // + // Range deletion tombstones may be added in any order, both with respect to + // each other and with respect to the point (Put/Merge/Delete) keys in the + // same file. + // + // REQUIRES: The comparator orders `begin_key` at or before `end_key` + // REQUIRES: comparator is *not* timestamp-aware. + Status DeleteRange(const Slice& begin_key, const Slice& end_key); + + // Add a range deletion tombstone to currently opened file. Such a range + // deletion tombstone does NOT delete point (Put/Merge/Delete) keys in the + // same file. + // + // Range deletion tombstones may be added in any order, both with respect to + // each other and with respect to the point (Put/Merge/Delete) keys in the + // same file. + // + // REQUIRES: begin_key and end_key are user keys without timestamp. + // REQUIRES: The comparator orders `begin_key` at or before `end_key` + // REQUIRES: timestamp's size is equal to what is expected by the comparator. + Status DeleteRange(const Slice& begin_key, const Slice& end_key, + const Slice& timestamp); + + // Finalize writing to sst file and close file. + // + // An optional ExternalSstFileInfo pointer can be passed to the function + // which will be populated with information about the created sst file. + Status Finish(ExternalSstFileInfo* file_info = nullptr); + + // Return the current file size. + uint64_t FileSize(); + + private: + void InvalidatePageCache(bool closing); + struct Rep; + std::unique_ptr rep_; +}; +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/include/rocksdb/sst_partitioner.h b/librocksdb-sys/rocksdb/include/rocksdb/sst_partitioner.h new file mode 100644 index 0000000..3af8e94 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/sst_partitioner.h @@ -0,0 +1,142 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#pragma once + +#include +#include + +#include "rocksdb/customizable.h" +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/slice.h" + +namespace ROCKSDB_NAMESPACE { + +class Slice; + +enum PartitionerResult : char { + // Partitioner does not require to create new file + kNotRequired = 0x0, + // Partitioner is requesting forcefully to create new file + kRequired = 0x1 + // Additional constants can be added +}; + +struct PartitionerRequest { + PartitionerRequest(const Slice& prev_user_key_, + const Slice& current_user_key_, + uint64_t current_output_file_size_) + : prev_user_key(&prev_user_key_), + current_user_key(¤t_user_key_), + current_output_file_size(current_output_file_size_) {} + const Slice* prev_user_key; + const Slice* current_user_key; + uint64_t current_output_file_size; +}; + +/* + * A SstPartitioner is a generic pluggable way of defining the partition + * of SST files. Compaction job will split the SST files on partition boundary + * to lower the write amplification during SST file promote to higher level. + */ +class SstPartitioner { + public: + virtual ~SstPartitioner() {} + + // Return the name of this partitioner. + virtual const char* Name() const = 0; + + // It is called for all keys in compaction. When partitioner want to create + // new SST file it needs to return true. It means compaction job will finish + // current SST file where last key is "prev_user_key" parameter and start new + // SST file where first key is "current_user_key". Returns decision if + // partition boundary was detected and compaction should create new file. + virtual PartitionerResult ShouldPartition( + const PartitionerRequest& request) = 0; + + // Called with smallest and largest keys in SST file when compaction try to do + // trivial move. Returns true is partitioner allows to do trivial move. + virtual bool CanDoTrivialMove(const Slice& smallest_user_key, + const Slice& largest_user_key) = 0; + + // Context information of a compaction run + struct Context { + // Does this compaction run include all data files + bool is_full_compaction; + // Is this compaction requested by the client (true), + // or is it occurring as an automatic compaction process + bool is_manual_compaction; + // Output level for this compaction + int output_level; + // Smallest key for compaction + Slice smallest_user_key; + // Largest key for compaction + Slice largest_user_key; + }; +}; + +// Exceptions MUST NOT propagate out of overridden functions into RocksDB, +// because RocksDB is not exception-safe. This could cause undefined behavior +// including data loss, unreported corruption, deadlocks, and more. +class SstPartitionerFactory : public Customizable { + public: + ~SstPartitionerFactory() override {} + static const char* Type() { return "SstPartitionerFactory"; } + static Status CreateFromString( + const ConfigOptions& options, const std::string& value, + std::shared_ptr* result); + + virtual std::unique_ptr CreatePartitioner( + const SstPartitioner::Context& context) const = 0; + + // Returns a name that identifies this partitioner factory. + const char* Name() const override = 0; +}; + +/* + * Fixed key prefix partitioner. It splits the output SST files when prefix + * defined by size changes. + */ +class SstPartitionerFixedPrefix : public SstPartitioner { + public: + explicit SstPartitionerFixedPrefix(size_t len) : len_(len) {} + + virtual ~SstPartitionerFixedPrefix() override {} + + const char* Name() const override { return "SstPartitionerFixedPrefix"; } + + PartitionerResult ShouldPartition(const PartitionerRequest& request) override; + + bool CanDoTrivialMove(const Slice& smallest_user_key, + const Slice& largest_user_key) override; + + private: + size_t len_; +}; + +/* + * Factory for fixed prefix partitioner. + */ +class SstPartitionerFixedPrefixFactory : public SstPartitionerFactory { + public: + explicit SstPartitionerFixedPrefixFactory(size_t len); + + ~SstPartitionerFixedPrefixFactory() override {} + + static const char* kClassName() { return "SstPartitionerFixedPrefixFactory"; } + const char* Name() const override { return kClassName(); } + + std::unique_ptr CreatePartitioner( + const SstPartitioner::Context& /* context */) const override; + + private: + size_t len_; +}; + +extern std::shared_ptr +NewSstPartitionerFixedPrefixFactory(size_t prefix_len); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/statistics.h b/librocksdb-sys/rocksdb/include/rocksdb/statistics.h new file mode 100644 index 0000000..a56b35d --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/statistics.h @@ -0,0 +1,765 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "rocksdb/customizable.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +/** + * Keep adding tickers here. Note that the C++ enum values, unlike the values in + * the Java bindings, are not guaranteed to be stable; also, the C++ and Java + * values for any given ticker are not guaranteed to match. + * 1. Add the new ticker before TICKER_ENUM_MAX. + * 2. Add a readable string in TickersNameMap below for the newly added ticker. + * 3. Add a corresponding enum value to TickerType.java in the Java API. + * 4. Add the enum conversions from/to Java/C++ to portal.h's toJavaTickerType + * and toCppTickers. + */ +enum Tickers : uint32_t { + // total block cache misses + // REQUIRES: BLOCK_CACHE_MISS == BLOCK_CACHE_INDEX_MISS + + // BLOCK_CACHE_FILTER_MISS + + // BLOCK_CACHE_DATA_MISS; + BLOCK_CACHE_MISS = 0, + // total block cache hit + // REQUIRES: BLOCK_CACHE_HIT == BLOCK_CACHE_INDEX_HIT + + // BLOCK_CACHE_FILTER_HIT + + // BLOCK_CACHE_DATA_HIT; + BLOCK_CACHE_HIT, + // # of blocks added to block cache. + BLOCK_CACHE_ADD, + // # of failures when adding blocks to block cache. + BLOCK_CACHE_ADD_FAILURES, + // # of times cache miss when accessing index block from block cache. + BLOCK_CACHE_INDEX_MISS, + // # of times cache hit when accessing index block from block cache. + BLOCK_CACHE_INDEX_HIT, + // # of index blocks added to block cache. + BLOCK_CACHE_INDEX_ADD, + // # of bytes of index blocks inserted into cache + BLOCK_CACHE_INDEX_BYTES_INSERT, + // # of times cache miss when accessing filter block from block cache. + BLOCK_CACHE_FILTER_MISS, + // # of times cache hit when accessing filter block from block cache. + BLOCK_CACHE_FILTER_HIT, + // # of filter blocks added to block cache. + BLOCK_CACHE_FILTER_ADD, + // # of bytes of bloom filter blocks inserted into cache + BLOCK_CACHE_FILTER_BYTES_INSERT, + // # of times cache miss when accessing data block from block cache. + BLOCK_CACHE_DATA_MISS, + // # of times cache hit when accessing data block from block cache. + BLOCK_CACHE_DATA_HIT, + // # of data blocks added to block cache. + BLOCK_CACHE_DATA_ADD, + // # of bytes of data blocks inserted into cache + BLOCK_CACHE_DATA_BYTES_INSERT, + // # of bytes read from cache. + BLOCK_CACHE_BYTES_READ, + // # of bytes written into cache. + BLOCK_CACHE_BYTES_WRITE, + + // # of times bloom filter has avoided file reads, i.e., negatives. + BLOOM_FILTER_USEFUL, + // # of times bloom FullFilter has not avoided the reads. + BLOOM_FILTER_FULL_POSITIVE, + // # of times bloom FullFilter has not avoided the reads and data actually + // exist. + BLOOM_FILTER_FULL_TRUE_POSITIVE, + + // # persistent cache hit + PERSISTENT_CACHE_HIT, + // # persistent cache miss + PERSISTENT_CACHE_MISS, + + // # total simulation block cache hits + SIM_BLOCK_CACHE_HIT, + // # total simulation block cache misses + SIM_BLOCK_CACHE_MISS, + + // # of memtable hits. + MEMTABLE_HIT, + // # of memtable misses. + MEMTABLE_MISS, + + // # of Get() queries served by L0 + GET_HIT_L0, + // # of Get() queries served by L1 + GET_HIT_L1, + // # of Get() queries served by L2 and up + GET_HIT_L2_AND_UP, + + /** + * COMPACTION_KEY_DROP_* count the reasons for key drop during compaction + * There are 4 reasons currently. + */ + COMPACTION_KEY_DROP_NEWER_ENTRY, // key was written with a newer value. + // Also includes keys dropped for range del. + COMPACTION_KEY_DROP_OBSOLETE, // The key is obsolete. + COMPACTION_KEY_DROP_RANGE_DEL, // key was covered by a range tombstone. + COMPACTION_KEY_DROP_USER, // user compaction function has dropped the key. + COMPACTION_RANGE_DEL_DROP_OBSOLETE, // all keys in range were deleted. + // Deletions obsoleted before bottom level due to file gap optimization. + COMPACTION_OPTIMIZED_DEL_DROP_OBSOLETE, + // If a compaction was canceled in sfm to prevent ENOSPC + COMPACTION_CANCELLED, + + // Number of keys written to the database via the Put and Write call's + NUMBER_KEYS_WRITTEN, + // Number of Keys read, + NUMBER_KEYS_READ, + // Number keys updated, if inplace update is enabled + NUMBER_KEYS_UPDATED, + // The number of uncompressed bytes issued by DB::Put(), DB::Delete(), + // DB::Merge(), and DB::Write(). + BYTES_WRITTEN, + // The number of uncompressed bytes read from DB::Get(). It could be + // either from memtables, cache, or table files. + // For the number of logical bytes read from DB::MultiGet(), + // please use NUMBER_MULTIGET_BYTES_READ. + BYTES_READ, + // The number of calls to seek/next/prev + NUMBER_DB_SEEK, + NUMBER_DB_NEXT, + NUMBER_DB_PREV, + // The number of calls to seek/next/prev that returned data + NUMBER_DB_SEEK_FOUND, + NUMBER_DB_NEXT_FOUND, + NUMBER_DB_PREV_FOUND, + // The number of uncompressed bytes read from an iterator. + // Includes size of key and value. + ITER_BYTES_READ, + NO_FILE_OPENS, + NO_FILE_ERRORS, + // Writer has to wait for compaction or flush to finish. + STALL_MICROS, + // The wait time for db mutex. + // Disabled by default. To enable it set stats level to kAll + DB_MUTEX_WAIT_MICROS, + + // Number of MultiGet calls, keys read, and bytes read + NUMBER_MULTIGET_CALLS, + NUMBER_MULTIGET_KEYS_READ, + NUMBER_MULTIGET_BYTES_READ, + + NUMBER_MERGE_FAILURES, + + // Prefix filter stats when used for point lookups (Get / MultiGet). + // (For prefix filter stats on iterators, see *_LEVEL_SEEK_*.) + // Checked: filter was queried + BLOOM_FILTER_PREFIX_CHECKED, + // Useful: filter returned false so prevented accessing data+index blocks + BLOOM_FILTER_PREFIX_USEFUL, + // True positive: found a key matching the point query. When another key + // with the same prefix matches, it is considered a false positive by + // these statistics even though the filter returned a true positive. + BLOOM_FILTER_PREFIX_TRUE_POSITIVE, + + // Number of times we had to reseek inside an iteration to skip + // over large number of keys with same userkey. + NUMBER_OF_RESEEKS_IN_ITERATION, + + // Record the number of calls to GetUpdatesSince. Useful to keep track of + // transaction log iterator refreshes + GET_UPDATES_SINCE_CALLS, + WAL_FILE_SYNCED, // Number of times WAL sync is done + WAL_FILE_BYTES, // Number of bytes written to WAL + + // Writes can be processed by requesting thread or by the thread at the + // head of the writers queue. + WRITE_DONE_BY_SELF, + WRITE_DONE_BY_OTHER, // Equivalent to writes done for others + WRITE_WITH_WAL, // Number of Write calls that request WAL + COMPACT_READ_BYTES, // Bytes read during compaction + COMPACT_WRITE_BYTES, // Bytes written during compaction + FLUSH_WRITE_BYTES, // Bytes written during flush + + // Compaction read and write statistics broken down by CompactionReason + COMPACT_READ_BYTES_MARKED, + COMPACT_READ_BYTES_PERIODIC, + COMPACT_READ_BYTES_TTL, + COMPACT_WRITE_BYTES_MARKED, + COMPACT_WRITE_BYTES_PERIODIC, + COMPACT_WRITE_BYTES_TTL, + + // Number of table's properties loaded directly from file, without creating + // table reader object. + NUMBER_DIRECT_LOAD_TABLE_PROPERTIES, + NUMBER_SUPERVERSION_ACQUIRES, + NUMBER_SUPERVERSION_RELEASES, + NUMBER_SUPERVERSION_CLEANUPS, + + // # of compressions/decompressions executed + NUMBER_BLOCK_COMPRESSED, + NUMBER_BLOCK_DECOMPRESSED, + + // DEPRECATED / unused (see NUMBER_BLOCK_COMPRESSION_*) + NUMBER_BLOCK_NOT_COMPRESSED, + MERGE_OPERATION_TOTAL_TIME, + FILTER_OPERATION_TOTAL_TIME, + + // Row cache. + ROW_CACHE_HIT, + ROW_CACHE_MISS, + + // Read amplification statistics. + // Read amplification can be calculated using this formula + // (READ_AMP_TOTAL_READ_BYTES / READ_AMP_ESTIMATE_USEFUL_BYTES) + // + // REQUIRES: ReadOptions::read_amp_bytes_per_bit to be enabled + READ_AMP_ESTIMATE_USEFUL_BYTES, // Estimate of total bytes actually used. + READ_AMP_TOTAL_READ_BYTES, // Total size of loaded data blocks. + + // Number of refill intervals where rate limiter's bytes are fully consumed. + NUMBER_RATE_LIMITER_DRAINS, + + // Number of internal keys skipped by Iterator + NUMBER_ITER_SKIP, + + // BlobDB specific stats + // # of Put/PutTTL/PutUntil to BlobDB. Only applicable to legacy BlobDB. + BLOB_DB_NUM_PUT, + // # of Write to BlobDB. Only applicable to legacy BlobDB. + BLOB_DB_NUM_WRITE, + // # of Get to BlobDB. Only applicable to legacy BlobDB. + BLOB_DB_NUM_GET, + // # of MultiGet to BlobDB. Only applicable to legacy BlobDB. + BLOB_DB_NUM_MULTIGET, + // # of Seek/SeekToFirst/SeekToLast/SeekForPrev to BlobDB iterator. Only + // applicable to legacy BlobDB. + BLOB_DB_NUM_SEEK, + // # of Next to BlobDB iterator. Only applicable to legacy BlobDB. + BLOB_DB_NUM_NEXT, + // # of Prev to BlobDB iterator. Only applicable to legacy BlobDB. + BLOB_DB_NUM_PREV, + // # of keys written to BlobDB. Only applicable to legacy BlobDB. + BLOB_DB_NUM_KEYS_WRITTEN, + // # of keys read from BlobDB. Only applicable to legacy BlobDB. + BLOB_DB_NUM_KEYS_READ, + // # of bytes (key + value) written to BlobDB. Only applicable to legacy + // BlobDB. + BLOB_DB_BYTES_WRITTEN, + // # of bytes (keys + value) read from BlobDB. Only applicable to legacy + // BlobDB. + BLOB_DB_BYTES_READ, + // # of keys written by BlobDB as non-TTL inlined value. Only applicable to + // legacy BlobDB. + BLOB_DB_WRITE_INLINED, + // # of keys written by BlobDB as TTL inlined value. Only applicable to legacy + // BlobDB. + BLOB_DB_WRITE_INLINED_TTL, + // # of keys written by BlobDB as non-TTL blob value. Only applicable to + // legacy BlobDB. + BLOB_DB_WRITE_BLOB, + // # of keys written by BlobDB as TTL blob value. Only applicable to legacy + // BlobDB. + BLOB_DB_WRITE_BLOB_TTL, + // # of bytes written to blob file. + BLOB_DB_BLOB_FILE_BYTES_WRITTEN, + // # of bytes read from blob file. + BLOB_DB_BLOB_FILE_BYTES_READ, + // # of times a blob files being synced. + BLOB_DB_BLOB_FILE_SYNCED, + // # of blob index evicted from base DB by BlobDB compaction filter because + // of expiration. Only applicable to legacy BlobDB. + BLOB_DB_BLOB_INDEX_EXPIRED_COUNT, + // size of blob index evicted from base DB by BlobDB compaction filter + // because of expiration. Only applicable to legacy BlobDB. + BLOB_DB_BLOB_INDEX_EXPIRED_SIZE, + // # of blob index evicted from base DB by BlobDB compaction filter because + // of corresponding file deleted. Only applicable to legacy BlobDB. + BLOB_DB_BLOB_INDEX_EVICTED_COUNT, + // size of blob index evicted from base DB by BlobDB compaction filter + // because of corresponding file deleted. Only applicable to legacy BlobDB. + BLOB_DB_BLOB_INDEX_EVICTED_SIZE, + // # of blob files that were obsoleted by garbage collection. Only applicable + // to legacy BlobDB. + BLOB_DB_GC_NUM_FILES, + // # of blob files generated by garbage collection. Only applicable to legacy + // BlobDB. + BLOB_DB_GC_NUM_NEW_FILES, + // # of BlobDB garbage collection failures. Only applicable to legacy BlobDB. + BLOB_DB_GC_FAILURES, + // # of keys relocated to new blob file by garbage collection. + BLOB_DB_GC_NUM_KEYS_RELOCATED, + // # of bytes relocated to new blob file by garbage collection. + BLOB_DB_GC_BYTES_RELOCATED, + // # of blob files evicted because of BlobDB is full. Only applicable to + // legacy BlobDB. + BLOB_DB_FIFO_NUM_FILES_EVICTED, + // # of keys in the blob files evicted because of BlobDB is full. Only + // applicable to legacy BlobDB. + BLOB_DB_FIFO_NUM_KEYS_EVICTED, + // # of bytes in the blob files evicted because of BlobDB is full. Only + // applicable to legacy BlobDB. + BLOB_DB_FIFO_BYTES_EVICTED, + + // These counters indicate a performance issue in WritePrepared transactions. + // We should not seem them ticking them much. + // # of times prepare_mutex_ is acquired in the fast path. + TXN_PREPARE_MUTEX_OVERHEAD, + // # of times old_commit_map_mutex_ is acquired in the fast path. + TXN_OLD_COMMIT_MAP_MUTEX_OVERHEAD, + // # of times we checked a batch for duplicate keys. + TXN_DUPLICATE_KEY_OVERHEAD, + // # of times snapshot_mutex_ is acquired in the fast path. + TXN_SNAPSHOT_MUTEX_OVERHEAD, + // # of times ::Get returned TryAgain due to expired snapshot seq + TXN_GET_TRY_AGAIN, + + // Number of keys actually found in MultiGet calls (vs number requested by + // caller) + // NUMBER_MULTIGET_KEYS_READ gives the number requested by caller + NUMBER_MULTIGET_KEYS_FOUND, + + NO_ITERATOR_CREATED, // number of iterators created + NO_ITERATOR_DELETED, // number of iterators deleted + + BLOCK_CACHE_COMPRESSION_DICT_MISS, + BLOCK_CACHE_COMPRESSION_DICT_HIT, + BLOCK_CACHE_COMPRESSION_DICT_ADD, + BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT, + + // # of blocks redundantly inserted into block cache. + // REQUIRES: BLOCK_CACHE_ADD_REDUNDANT <= BLOCK_CACHE_ADD + BLOCK_CACHE_ADD_REDUNDANT, + // # of index blocks redundantly inserted into block cache. + // REQUIRES: BLOCK_CACHE_INDEX_ADD_REDUNDANT <= BLOCK_CACHE_INDEX_ADD + BLOCK_CACHE_INDEX_ADD_REDUNDANT, + // # of filter blocks redundantly inserted into block cache. + // REQUIRES: BLOCK_CACHE_FILTER_ADD_REDUNDANT <= BLOCK_CACHE_FILTER_ADD + BLOCK_CACHE_FILTER_ADD_REDUNDANT, + // # of data blocks redundantly inserted into block cache. + // REQUIRES: BLOCK_CACHE_DATA_ADD_REDUNDANT <= BLOCK_CACHE_DATA_ADD + BLOCK_CACHE_DATA_ADD_REDUNDANT, + // # of dict blocks redundantly inserted into block cache. + // REQUIRES: BLOCK_CACHE_COMPRESSION_DICT_ADD_REDUNDANT + // <= BLOCK_CACHE_COMPRESSION_DICT_ADD + BLOCK_CACHE_COMPRESSION_DICT_ADD_REDUNDANT, + + // # of files marked as trash by sst file manager and will be deleted + // later by background thread. + FILES_MARKED_TRASH, + // # of trash files deleted by the background thread from the trash queue. + FILES_DELETED_FROM_TRASH_QUEUE, + // # of files deleted immediately by sst file manager through delete + // scheduler. + FILES_DELETED_IMMEDIATELY, + + // The counters for error handler, not that, bg_io_error is the subset of + // bg_error and bg_retryable_io_error is the subset of bg_io_error. + // The misspelled versions are deprecated and only kept for compatibility. + // TODO: remove the misspelled tickers in the next major release. + ERROR_HANDLER_BG_ERROR_COUNT, + ERROR_HANDLER_BG_ERROR_COUNT_MISSPELLED, + ERROR_HANDLER_BG_IO_ERROR_COUNT, + ERROR_HANDLER_BG_IO_ERROR_COUNT_MISSPELLED, + ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT, + ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT_MISSPELLED, + ERROR_HANDLER_AUTORESUME_COUNT, + ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT, + ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT, + + // Statistics for memtable garbage collection: + // Raw bytes of data (payload) present on memtable at flush time. + MEMTABLE_PAYLOAD_BYTES_AT_FLUSH, + // Outdated bytes of data present on memtable at flush time. + MEMTABLE_GARBAGE_BYTES_AT_FLUSH, + + // Secondary cache statistics + SECONDARY_CACHE_HITS, + + // Bytes read by `VerifyChecksum()` and `VerifyFileChecksums()` APIs. + VERIFY_CHECKSUM_READ_BYTES, + + // Bytes read/written while creating backups + BACKUP_READ_BYTES, + BACKUP_WRITE_BYTES, + + // Remote compaction read/write statistics + REMOTE_COMPACT_READ_BYTES, + REMOTE_COMPACT_WRITE_BYTES, + + // Tiered storage related statistics + HOT_FILE_READ_BYTES, + WARM_FILE_READ_BYTES, + COLD_FILE_READ_BYTES, + HOT_FILE_READ_COUNT, + WARM_FILE_READ_COUNT, + COLD_FILE_READ_COUNT, + + // Last level and non-last level read statistics + LAST_LEVEL_READ_BYTES, + LAST_LEVEL_READ_COUNT, + NON_LAST_LEVEL_READ_BYTES, + NON_LAST_LEVEL_READ_COUNT, + + // Statistics on iterator Seek() (and variants) for each sorted run. I.e. a + // single user Seek() can result in many sorted run Seek()s. + // The stats are split between last level and non-last level. + // Filtered: a filter such as prefix Bloom filter indicate the Seek() would + // not find anything relevant, so avoided a likely access to data+index + // blocks. + LAST_LEVEL_SEEK_FILTERED, + // Filter match: a filter such as prefix Bloom filter was queried but did + // not filter out the seek. + LAST_LEVEL_SEEK_FILTER_MATCH, + // At least one data block was accessed for a Seek() (or variant) on a + // sorted run. + LAST_LEVEL_SEEK_DATA, + // At least one value() was accessed for the seek (suggesting it was useful), + // and no filter such as prefix Bloom was queried. + LAST_LEVEL_SEEK_DATA_USEFUL_NO_FILTER, + // At least one value() was accessed for the seek (suggesting it was useful), + // after querying a filter such as prefix Bloom. + LAST_LEVEL_SEEK_DATA_USEFUL_FILTER_MATCH, + // The same set of stats, but for non-last level seeks. + NON_LAST_LEVEL_SEEK_FILTERED, + NON_LAST_LEVEL_SEEK_FILTER_MATCH, + NON_LAST_LEVEL_SEEK_DATA, + NON_LAST_LEVEL_SEEK_DATA_USEFUL_NO_FILTER, + NON_LAST_LEVEL_SEEK_DATA_USEFUL_FILTER_MATCH, + + // Number of block checksum verifications + BLOCK_CHECKSUM_COMPUTE_COUNT, + // Number of times RocksDB detected a corruption while verifying a block + // checksum. RocksDB does not remember corruptions that happened during user + // reads so the same block corruption may be detected multiple times. + BLOCK_CHECKSUM_MISMATCH_COUNT, + + MULTIGET_COROUTINE_COUNT, + + // Integrated BlobDB specific stats + // # of times cache miss when accessing blob from blob cache. + BLOB_DB_CACHE_MISS, + // # of times cache hit when accessing blob from blob cache. + BLOB_DB_CACHE_HIT, + // # of data blocks added to blob cache. + BLOB_DB_CACHE_ADD, + // # of failures when adding blobs to blob cache. + BLOB_DB_CACHE_ADD_FAILURES, + // # of bytes read from blob cache. + BLOB_DB_CACHE_BYTES_READ, + // # of bytes written into blob cache. + BLOB_DB_CACHE_BYTES_WRITE, + + // Time spent in the ReadAsync file system call + READ_ASYNC_MICROS, + // Number of errors returned to the async read callback + ASYNC_READ_ERROR_COUNT, + + // Fine grained secondary cache stats + SECONDARY_CACHE_FILTER_HITS, + SECONDARY_CACHE_INDEX_HITS, + SECONDARY_CACHE_DATA_HITS, + + // Number of lookup into the prefetched tail (see + // `TABLE_OPEN_PREFETCH_TAIL_READ_BYTES`) + // that can't find its data for table open + TABLE_OPEN_PREFETCH_TAIL_MISS, + // Number of lookup into the prefetched tail (see + // `TABLE_OPEN_PREFETCH_TAIL_READ_BYTES`) + // that finds its data for table open + TABLE_OPEN_PREFETCH_TAIL_HIT, + + // Statistics on the filtering by user-defined timestamps + // # of times timestamps are checked on accessing the table + TIMESTAMP_FILTER_TABLE_CHECKED, + // # of times timestamps can successfully help skip the table access + TIMESTAMP_FILTER_TABLE_FILTERED, + + // Number of input bytes (uncompressed) to compression for SST blocks that + // are stored compressed. + BYTES_COMPRESSED_FROM, + // Number of output bytes (compressed) from compression for SST blocks that + // are stored compressed. + BYTES_COMPRESSED_TO, + // Number of uncompressed bytes for SST blocks that are stored uncompressed + // because compression type is kNoCompression, or some error case caused + // compression not to run or produce an output. Index blocks are only counted + // if enable_index_compression is true. + BYTES_COMPRESSION_BYPASSED, + // Number of input bytes (uncompressed) to compression for SST blocks that + // are stored uncompressed because the compression result was rejected, + // either because the ratio was not acceptable (see + // CompressionOptions::max_compressed_bytes_per_kb) or found invalid by the + // `verify_compression` option. + BYTES_COMPRESSION_REJECTED, + + // Like BYTES_COMPRESSION_BYPASSED but counting number of blocks + NUMBER_BLOCK_COMPRESSION_BYPASSED, + // Like BYTES_COMPRESSION_REJECTED but counting number of blocks + NUMBER_BLOCK_COMPRESSION_REJECTED, + + // Number of input bytes (compressed) to decompression in reading compressed + // SST blocks from storage. + BYTES_DECOMPRESSED_FROM, + // Number of output bytes (uncompressed) from decompression in reading + // compressed SST blocks from storage. + BYTES_DECOMPRESSED_TO, + + TICKER_ENUM_MAX +}; + +// The order of items listed in Tickers should be the same as +// the order listed in TickersNameMap +extern const std::vector> TickersNameMap; + +/** + * Keep adding histograms here. Note that the C++ enum values, unlike the values + * in the Java bindings, are not guaranteed to be stable; also, the C++ and Java + * values for any given histogram are not guaranteed to match. + * 1. Add the new histogram before HISTOGRAM_ENUM_MAX. + * 2. Add a readable string in HistogramsNameMap below for the newly added + * histogram. + * 3. Add a corresponding enum value to HistogramType.java in the Java API. + * 4. Add the enum conversions from/to Java/C++ to portal.h's + * toJavaHistogramsType and toCppHistograms. + */ +enum Histograms : uint32_t { + DB_GET = 0, + DB_WRITE, + COMPACTION_TIME, + COMPACTION_CPU_TIME, + SUBCOMPACTION_SETUP_TIME, + TABLE_SYNC_MICROS, + COMPACTION_OUTFILE_SYNC_MICROS, + WAL_FILE_SYNC_MICROS, + MANIFEST_FILE_SYNC_MICROS, + // TIME SPENT IN IO DURING TABLE OPEN + TABLE_OPEN_IO_MICROS, + DB_MULTIGET, + READ_BLOCK_COMPACTION_MICROS, + READ_BLOCK_GET_MICROS, + WRITE_RAW_BLOCK_MICROS, + NUM_FILES_IN_SINGLE_COMPACTION, + DB_SEEK, + WRITE_STALL, + // Time spent in reading block-based or plain SST table + SST_READ_MICROS, + // Time spent in reading SST table (currently only block-based table) or blob + // file corresponding to `Env::IOActivity` + FILE_READ_FLUSH_MICROS, + FILE_READ_COMPACTION_MICROS, + FILE_READ_DB_OPEN_MICROS, + + // The number of subcompactions actually scheduled during a compaction + NUM_SUBCOMPACTIONS_SCHEDULED, + // Value size distribution in each operation + BYTES_PER_READ, + BYTES_PER_WRITE, + BYTES_PER_MULTIGET, + + BYTES_COMPRESSED, // DEPRECATED / unused (see BYTES_COMPRESSED_{FROM,TO}) + BYTES_DECOMPRESSED, // DEPRECATED / unused (see BYTES_DECOMPRESSED_{FROM,TO}) + COMPRESSION_TIMES_NANOS, + DECOMPRESSION_TIMES_NANOS, + // Number of merge operands passed to the merge operator in user read + // requests. + READ_NUM_MERGE_OPERANDS, + + // BlobDB specific stats + // Size of keys written to BlobDB. Only applicable to legacy BlobDB. + BLOB_DB_KEY_SIZE, + // Size of values written to BlobDB. Only applicable to legacy BlobDB. + BLOB_DB_VALUE_SIZE, + // BlobDB Put/PutWithTTL/PutUntil/Write latency. Only applicable to legacy + // BlobDB. + BLOB_DB_WRITE_MICROS, + // BlobDB Get latency. Only applicable to legacy BlobDB. + BLOB_DB_GET_MICROS, + // BlobDB MultiGet latency. Only applicable to legacy BlobDB. + BLOB_DB_MULTIGET_MICROS, + // BlobDB Seek/SeekToFirst/SeekToLast/SeekForPrev latency. Only applicable to + // legacy BlobDB. + BLOB_DB_SEEK_MICROS, + // BlobDB Next latency. Only applicable to legacy BlobDB. + BLOB_DB_NEXT_MICROS, + // BlobDB Prev latency. Only applicable to legacy BlobDB. + BLOB_DB_PREV_MICROS, + // Blob file write latency. + BLOB_DB_BLOB_FILE_WRITE_MICROS, + // Blob file read latency. + BLOB_DB_BLOB_FILE_READ_MICROS, + // Blob file sync latency. + BLOB_DB_BLOB_FILE_SYNC_MICROS, + // BlobDB compression time. + BLOB_DB_COMPRESSION_MICROS, + // BlobDB decompression time. + BLOB_DB_DECOMPRESSION_MICROS, + // Time spent flushing memtable to disk + FLUSH_TIME, + SST_BATCH_SIZE, + + // MultiGet stats logged per level + // Num of index and filter blocks read from file system per level. + NUM_INDEX_AND_FILTER_BLOCKS_READ_PER_LEVEL, + // Num of sst files read from file system per level. + NUM_SST_READ_PER_LEVEL, + + // Error handler statistics + ERROR_HANDLER_AUTORESUME_RETRY_COUNT, + + // Stats related to asynchronous read requests. + ASYNC_READ_BYTES, + POLL_WAIT_MICROS, + + // Number of prefetched bytes discarded by RocksDB. + PREFETCHED_BYTES_DISCARDED, + + // Number of IOs issued in parallel in a MultiGet batch + MULTIGET_IO_BATCH_SIZE, + + // Number of levels requiring IO for MultiGet + NUM_LEVEL_READ_PER_MULTIGET, + + // Wait time for aborting async read in FilePrefetchBuffer destructor + ASYNC_PREFETCH_ABORT_MICROS, + + // Number of bytes read for RocksDB's prefetching contents (as opposed to file + // system's prefetch) from the end of SST table during block based table open + TABLE_OPEN_PREFETCH_TAIL_READ_BYTES, + + HISTOGRAM_ENUM_MAX +}; + +extern const std::vector> HistogramsNameMap; + +struct HistogramData { + double median; + double percentile95; + double percentile99; + double average; + double standard_deviation; + // zero-initialize new members since old Statistics::histogramData() + // implementations won't write them. + double max = 0.0; + uint64_t count = 0; + uint64_t sum = 0; + double min = 0.0; +}; + +// StatsLevel can be used to reduce statistics overhead by skipping certain +// types of stats in the stats collection process. +// Usage: +// options.statistics->set_stats_level(StatsLevel::kExceptTimeForMutex); +enum StatsLevel : uint8_t { + // Disable all metrics + kDisableAll, + // Disable tickers + kExceptTickers = kDisableAll, + // Disable timer stats, and skip histogram stats + kExceptHistogramOrTimers, + // Skip timer stats + kExceptTimers, + // Collect all stats except time inside mutex lock AND time spent on + // compression. + kExceptDetailedTimers, + // Collect all stats except the counters requiring to get time inside the + // mutex lock. + kExceptTimeForMutex, + // Collect all stats, including measuring duration of mutex operations. + // If getting time is expensive on the platform to run, it can + // reduce scalability to more threads, especially for writes. + kAll, +}; + +// Analyze the performance of a db by providing cumulative stats over time. +// Usage: +// Options options; +// options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); +// Status s = DB::Open(options, kDBPath, &db); +// ... +// options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED); +// HistogramData hist; +// options.statistics->histogramData(FLUSH_TIME, &hist); +// +// Exceptions MUST NOT propagate out of overridden functions into RocksDB, +// because RocksDB is not exception-safe. This could cause undefined behavior +// including data loss, unreported corruption, deadlocks, and more. +class Statistics : public Customizable { + public: + ~Statistics() override {} + static const char* Type() { return "Statistics"; } + static Status CreateFromString(const ConfigOptions& opts, + const std::string& value, + std::shared_ptr* result); + // Default name of empty, for backwards compatibility. Derived classes should + // override this method. + // This default implementation will likely be removed in a future release + const char* Name() const override { return ""; } + virtual uint64_t getTickerCount(uint32_t tickerType) const = 0; + virtual void histogramData(uint32_t type, + HistogramData* const data) const = 0; + virtual std::string getHistogramString(uint32_t /*type*/) const { return ""; } + virtual void recordTick(uint32_t tickerType, uint64_t count = 1) = 0; + virtual void setTickerCount(uint32_t tickerType, uint64_t count) = 0; + virtual uint64_t getAndResetTickerCount(uint32_t tickerType) = 0; + virtual void reportTimeToHistogram(uint32_t histogramType, uint64_t time) { + if (get_stats_level() <= StatsLevel::kExceptTimers) { + return; + } + recordInHistogram(histogramType, time); + } + // The function is here only for backward compatibility reason. + // Users implementing their own Statistics class should override + // recordInHistogram() instead and leave measureTime() as it is. + virtual void measureTime(uint32_t /*histogramType*/, uint64_t /*time*/) { + // This is not supposed to be called. + assert(false); + } + virtual void recordInHistogram(uint32_t histogramType, uint64_t time) { + // measureTime() is the old and inaccurate function name. + // To keep backward compatible. If users implement their own + // statistics, which overrides measureTime() but doesn't override + // this function. We forward to measureTime(). + measureTime(histogramType, time); + } + + // Resets all ticker and histogram stats + virtual Status Reset() { return Status::NotSupported("Not implemented"); } + + using Customizable::ToString; + // String representation of the statistic object. Must be thread-safe. + virtual std::string ToString() const { + // Do nothing by default + return std::string("ToString(): not implemented"); + } + + virtual bool getTickerMap(std::map*) const { + // Do nothing by default + return false; + } + + // Override this function to disable particular histogram collection + virtual bool HistEnabledForType(uint32_t type) const { + return type < HISTOGRAM_ENUM_MAX; + } + void set_stats_level(StatsLevel sl) { + stats_level_.store(sl, std::memory_order_relaxed); + } + StatsLevel get_stats_level() const { + return stats_level_.load(std::memory_order_relaxed); + } + + private: + std::atomic stats_level_{kExceptDetailedTimers}; +}; + +// Create a concrete DBStatistics object +std::shared_ptr CreateDBStatistics(); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/stats_history.h b/librocksdb-sys/rocksdb/include/rocksdb/stats_history.h new file mode 100644 index 0000000..57e4692 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/stats_history.h @@ -0,0 +1,70 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include +#include + +#include "rocksdb/statistics.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +class DBImpl; + +// StatsHistoryIterator is the main interface for users to programmatically +// access statistics snapshots that was automatically stored by RocksDB. +// Depending on options, the stats can be in memory or on disk. +// The stats snapshots are indexed by time that they were recorded, and each +// stats snapshot contains individual stat name and value at the time of +// recording. +// Example: +// std::unique_ptr stats_iter; +// Status s = db->GetStatsHistory(0 /* start_time */, +// env->NowMicros() /* end_time*/, +// &stats_iter); +// if (s.ok) { +// for (; stats_iter->Valid(); stats_iter->Next()) { +// uint64_t stats_time = stats_iter->GetStatsTime(); +// const std::map& stats_map = +// stats_iter->GetStatsMap(); +// process(stats_time, stats_map); +// } +// } +class StatsHistoryIterator { + public: + StatsHistoryIterator() {} + virtual ~StatsHistoryIterator() {} + + virtual bool Valid() const = 0; + + // Moves to the next stats history record. After this call, Valid() is + // true iff the iterator was not positioned at the last entry in the source. + // REQUIRES: Valid() + virtual void Next() = 0; + + // Return the time stamp (in seconds) when stats history is recorded. + // REQUIRES: Valid() + virtual uint64_t GetStatsTime() const = 0; + + // DEPRECATED (was never used) + virtual int GetFormatVersion() const { return -1; } + + // Return the current stats history as an std::map which specifies the + // mapping from stats name to stats value . The underlying storage + // for the returned map is valid only until the next modification of + // the iterator. + // REQUIRES: Valid() + virtual const std::map& GetStatsMap() const = 0; + + // If an error has occurred, return it. Else return an ok status. + virtual Status status() const = 0; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/status.h b/librocksdb-sys/rocksdb/include/rocksdb/status.h new file mode 100644 index 0000000..447c3b9 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/status.h @@ -0,0 +1,574 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// A Status encapsulates the result of an operation. It may indicate success, +// or it may indicate an error with an associated error message. +// +// Multiple threads can invoke const methods on a Status without +// external synchronization, but if any of the threads may call a +// non-const method, all threads accessing the same Status must use +// external synchronization. + +#pragma once + +#ifdef ROCKSDB_ASSERT_STATUS_CHECKED +#include +#include +#endif + +#include +#include + +#ifdef ROCKSDB_ASSERT_STATUS_CHECKED +#include "port/stack_trace.h" +#endif + +#include "rocksdb/slice.h" + +namespace ROCKSDB_NAMESPACE { + +class Status { + public: + // Create a success status. + Status() + : code_(kOk), + subcode_(kNone), + sev_(kNoError), + retryable_(false), + data_loss_(false), + scope_(0), + state_(nullptr) {} + ~Status() { +#ifdef ROCKSDB_ASSERT_STATUS_CHECKED + if (!checked_) { + fprintf(stderr, "Failed to check Status %p\n", this); + port::PrintStack(); + std::abort(); + } +#endif // ROCKSDB_ASSERT_STATUS_CHECKED + } + + // Copy the specified status. + Status(const Status& s); + Status& operator=(const Status& s); + Status(Status&& s) noexcept; + Status& operator=(Status&& s) noexcept; + bool operator==(const Status& rhs) const; + bool operator!=(const Status& rhs) const; + + // In case of intentionally swallowing an error, user must explicitly call + // this function. That way we are easily able to search the code to find where + // error swallowing occurs. + inline void PermitUncheckedError() const { MarkChecked(); } + + inline void MustCheck() const { +#ifdef ROCKSDB_ASSERT_STATUS_CHECKED + checked_ = false; +#endif // ROCKSDB_ASSERT_STATUS_CHECKED + } + + enum Code : unsigned char { + kOk = 0, + kNotFound = 1, + kCorruption = 2, + kNotSupported = 3, + kInvalidArgument = 4, + kIOError = 5, + kMergeInProgress = 6, + kIncomplete = 7, + kShutdownInProgress = 8, + kTimedOut = 9, + kAborted = 10, + kBusy = 11, + kExpired = 12, + kTryAgain = 13, + kCompactionTooLarge = 14, + kColumnFamilyDropped = 15, + kMaxCode + }; + + Code code() const { + MarkChecked(); + return code_; + } + + enum SubCode : unsigned char { + kNone = 0, + kMutexTimeout = 1, + kLockTimeout = 2, + kLockLimit = 3, + kNoSpace = 4, + kDeadlock = 5, + kStaleFile = 6, + kMemoryLimit = 7, + kSpaceLimit = 8, + kPathNotFound = 9, + KMergeOperandsInsufficientCapacity = 10, + kManualCompactionPaused = 11, + kOverwritten = 12, + kTxnNotPrepared = 13, + kIOFenced = 14, + kMergeOperatorFailed = 15, + kMaxSubCode + }; + + SubCode subcode() const { + MarkChecked(); + return subcode_; + } + + enum Severity : unsigned char { + kNoError = 0, + kSoftError = 1, + kHardError = 2, + kFatalError = 3, + kUnrecoverableError = 4, + kMaxSeverity + }; + + Status(const Status& s, Severity sev); + + Status(Code _code, SubCode _subcode, Severity _sev, const Slice& msg) + : Status(_code, _subcode, msg, "", _sev) {} + + static Status CopyAppendMessage(const Status& s, const Slice& delim, + const Slice& msg); + + Severity severity() const { + MarkChecked(); + return sev_; + } + + // Returns a C style string indicating the message of the Status + const char* getState() const { + MarkChecked(); + return state_.get(); + } + + // Return a success status. + static Status OK() { return Status(); } + + // Successful, though an existing something was overwritten + // Note: using variants of OK status for program logic is discouraged, + // but it can be useful for communicating statistical information without + // changing public APIs. + static Status OkOverwritten() { return Status(kOk, kOverwritten); } + + // Return error status of an appropriate type. + static Status NotFound(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kNotFound, msg, msg2); + } + + // Fast path for not found without malloc; + static Status NotFound(SubCode msg = kNone) { return Status(kNotFound, msg); } + + static Status NotFound(SubCode sc, const Slice& msg, + const Slice& msg2 = Slice()) { + return Status(kNotFound, sc, msg, msg2); + } + + static Status Corruption(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kCorruption, msg, msg2); + } + static Status Corruption(SubCode msg = kNone) { + return Status(kCorruption, msg); + } + + static Status NotSupported(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kNotSupported, msg, msg2); + } + static Status NotSupported(SubCode msg = kNone) { + return Status(kNotSupported, msg); + } + + static Status InvalidArgument(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kInvalidArgument, msg, msg2); + } + static Status InvalidArgument(SubCode msg = kNone) { + return Status(kInvalidArgument, msg); + } + + static Status IOError(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kIOError, msg, msg2); + } + static Status IOError(SubCode msg = kNone) { return Status(kIOError, msg); } + + static Status MergeInProgress(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kMergeInProgress, msg, msg2); + } + static Status MergeInProgress(SubCode msg = kNone) { + return Status(kMergeInProgress, msg); + } + + static Status Incomplete(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kIncomplete, msg, msg2); + } + static Status Incomplete(SubCode msg = kNone) { + return Status(kIncomplete, msg); + } + + static Status ShutdownInProgress(SubCode msg = kNone) { + return Status(kShutdownInProgress, msg); + } + static Status ShutdownInProgress(const Slice& msg, + const Slice& msg2 = Slice()) { + return Status(kShutdownInProgress, msg, msg2); + } + static Status Aborted(SubCode msg = kNone) { return Status(kAborted, msg); } + static Status Aborted(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kAborted, msg, msg2); + } + + static Status Busy(SubCode msg = kNone) { return Status(kBusy, msg); } + static Status Busy(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kBusy, msg, msg2); + } + + static Status TimedOut(SubCode msg = kNone) { return Status(kTimedOut, msg); } + static Status TimedOut(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kTimedOut, msg, msg2); + } + + static Status Expired(SubCode msg = kNone) { return Status(kExpired, msg); } + static Status Expired(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kExpired, msg, msg2); + } + + static Status TryAgain(SubCode msg = kNone) { return Status(kTryAgain, msg); } + static Status TryAgain(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kTryAgain, msg, msg2); + } + + static Status CompactionTooLarge(SubCode msg = kNone) { + return Status(kCompactionTooLarge, msg); + } + static Status CompactionTooLarge(const Slice& msg, + const Slice& msg2 = Slice()) { + return Status(kCompactionTooLarge, msg, msg2); + } + + static Status ColumnFamilyDropped(SubCode msg = kNone) { + return Status(kColumnFamilyDropped, msg); + } + + static Status ColumnFamilyDropped(const Slice& msg, + const Slice& msg2 = Slice()) { + return Status(kColumnFamilyDropped, msg, msg2); + } + + static Status NoSpace() { return Status(kIOError, kNoSpace); } + static Status NoSpace(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kIOError, kNoSpace, msg, msg2); + } + + static Status MemoryLimit() { return Status(kAborted, kMemoryLimit); } + static Status MemoryLimit(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kAborted, kMemoryLimit, msg, msg2); + } + + static Status SpaceLimit() { return Status(kIOError, kSpaceLimit); } + static Status SpaceLimit(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kIOError, kSpaceLimit, msg, msg2); + } + + static Status PathNotFound() { return Status(kIOError, kPathNotFound); } + static Status PathNotFound(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kIOError, kPathNotFound, msg, msg2); + } + + static Status TxnNotPrepared() { + return Status(kInvalidArgument, kTxnNotPrepared); + } + static Status TxnNotPrepared(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kInvalidArgument, kTxnNotPrepared, msg, msg2); + } + + // Returns true iff the status indicates success. + bool ok() const { + MarkChecked(); + return code() == kOk; + } + + // Returns true iff the status indicates success *with* something + // overwritten + bool IsOkOverwritten() const { + MarkChecked(); + return code() == kOk && subcode() == kOverwritten; + } + + // Returns true iff the status indicates a NotFound error. + bool IsNotFound() const { + MarkChecked(); + return code() == kNotFound; + } + + // Returns true iff the status indicates a Corruption error. + bool IsCorruption() const { + MarkChecked(); + return code() == kCorruption; + } + + // Returns true iff the status indicates a NotSupported error. + bool IsNotSupported() const { + MarkChecked(); + return code() == kNotSupported; + } + + // Returns true iff the status indicates an InvalidArgument error. + bool IsInvalidArgument() const { + MarkChecked(); + return code() == kInvalidArgument; + } + + // Returns true iff the status indicates an IOError. + bool IsIOError() const { + MarkChecked(); + return code() == kIOError; + } + + // Returns true iff the status indicates an MergeInProgress. + bool IsMergeInProgress() const { + MarkChecked(); + return code() == kMergeInProgress; + } + + // Returns true iff the status indicates Incomplete + bool IsIncomplete() const { + MarkChecked(); + return code() == kIncomplete; + } + + // Returns true iff the status indicates Shutdown In progress + bool IsShutdownInProgress() const { + MarkChecked(); + return code() == kShutdownInProgress; + } + + bool IsTimedOut() const { + MarkChecked(); + return code() == kTimedOut; + } + + bool IsAborted() const { + MarkChecked(); + return code() == kAborted; + } + + bool IsLockLimit() const { + MarkChecked(); + return code() == kAborted && subcode() == kLockLimit; + } + + // Returns true iff the status indicates that a resource is Busy and + // temporarily could not be acquired. + bool IsBusy() const { + MarkChecked(); + return code() == kBusy; + } + + bool IsDeadlock() const { + MarkChecked(); + return code() == kBusy && subcode() == kDeadlock; + } + + // Returns true iff the status indicated that the operation has Expired. + bool IsExpired() const { + MarkChecked(); + return code() == kExpired; + } + + // Returns true iff the status indicates a TryAgain error. + // This usually means that the operation failed, but may succeed if + // re-attempted. + bool IsTryAgain() const { + MarkChecked(); + return code() == kTryAgain; + } + + // Returns true iff the status indicates the proposed compaction is too large + bool IsCompactionTooLarge() const { + MarkChecked(); + return code() == kCompactionTooLarge; + } + + // Returns true iff the status indicates Column Family Dropped + bool IsColumnFamilyDropped() const { + MarkChecked(); + return code() == kColumnFamilyDropped; + } + + // Returns true iff the status indicates a NoSpace error + // This is caused by an I/O error returning the specific "out of space" + // error condition. Stricto sensu, an NoSpace error is an I/O error + // with a specific subcode, enabling users to take the appropriate action + // if needed + bool IsNoSpace() const { + MarkChecked(); + return (code() == kIOError) && (subcode() == kNoSpace); + } + + // Returns true iff the status indicates a memory limit error. There may be + // cases where we limit the memory used in certain operations (eg. the size + // of a write batch) in order to avoid out of memory exceptions. + bool IsMemoryLimit() const { + MarkChecked(); + return (code() == kAborted) && (subcode() == kMemoryLimit); + } + + // Returns true iff the status indicates a PathNotFound error + // This is caused by an I/O error returning the specific "no such file or + // directory" error condition. A PathNotFound error is an I/O error with + // a specific subcode, enabling users to take appropriate action if necessary + bool IsPathNotFound() const { + MarkChecked(); + return (code() == kIOError || code() == kNotFound) && + (subcode() == kPathNotFound); + } + + // Returns true iff the status indicates manual compaction paused. This + // is caused by a call to PauseManualCompaction + bool IsManualCompactionPaused() const { + MarkChecked(); + return (code() == kIncomplete) && (subcode() == kManualCompactionPaused); + } + + // Returns true iff the status indicates a TxnNotPrepared error. + bool IsTxnNotPrepared() const { + MarkChecked(); + return (code() == kInvalidArgument) && (subcode() == kTxnNotPrepared); + } + + // Returns true iff the status indicates a IOFenced error. + bool IsIOFenced() const { + MarkChecked(); + return (code() == kIOError) && (subcode() == kIOFenced); + } + + // Return a string representation of this status suitable for printing. + // Returns the string "OK" for success. + std::string ToString() const; + + protected: + Code code_; + SubCode subcode_; + Severity sev_; + bool retryable_; + bool data_loss_; + unsigned char scope_; + // A nullptr state_ (which is at least the case for OK) means the extra + // message is empty. + std::unique_ptr state_; +#ifdef ROCKSDB_ASSERT_STATUS_CHECKED + mutable bool checked_ = false; +#endif // ROCKSDB_ASSERT_STATUS_CHECKED + + explicit Status(Code _code, SubCode _subcode = kNone) + : code_(_code), + subcode_(_subcode), + sev_(kNoError), + retryable_(false), + data_loss_(false), + scope_(0) {} + + explicit Status(Code _code, SubCode _subcode, bool retryable, bool data_loss, + unsigned char scope) + : code_(_code), + subcode_(_subcode), + sev_(kNoError), + retryable_(retryable), + data_loss_(data_loss), + scope_(scope) {} + + Status(Code _code, SubCode _subcode, const Slice& msg, const Slice& msg2, + Severity sev = kNoError); + Status(Code _code, const Slice& msg, const Slice& msg2) + : Status(_code, kNone, msg, msg2) {} + + static std::unique_ptr CopyState(const char* s); + + inline void MarkChecked() const { +#ifdef ROCKSDB_ASSERT_STATUS_CHECKED + checked_ = true; +#endif // ROCKSDB_ASSERT_STATUS_CHECKED + } +}; + +inline Status::Status(const Status& s) + : code_(s.code_), + subcode_(s.subcode_), + sev_(s.sev_), + retryable_(s.retryable_), + data_loss_(s.data_loss_), + scope_(s.scope_) { + s.MarkChecked(); + state_ = (s.state_ == nullptr) ? nullptr : CopyState(s.state_.get()); +} +inline Status::Status(const Status& s, Severity sev) + : code_(s.code_), + subcode_(s.subcode_), + sev_(sev), + retryable_(s.retryable_), + data_loss_(s.data_loss_), + scope_(s.scope_) { + s.MarkChecked(); + state_ = (s.state_ == nullptr) ? nullptr : CopyState(s.state_.get()); +} +inline Status& Status::operator=(const Status& s) { + if (this != &s) { + s.MarkChecked(); + MustCheck(); + code_ = s.code_; + subcode_ = s.subcode_; + sev_ = s.sev_; + retryable_ = s.retryable_; + data_loss_ = s.data_loss_; + scope_ = s.scope_; + state_ = (s.state_ == nullptr) ? nullptr : CopyState(s.state_.get()); + } + return *this; +} + +inline Status::Status(Status&& s) noexcept : Status() { + s.MarkChecked(); + *this = std::move(s); +} + +inline Status& Status::operator=(Status&& s) noexcept { + if (this != &s) { + s.MarkChecked(); + MustCheck(); + code_ = std::move(s.code_); + s.code_ = kOk; + subcode_ = std::move(s.subcode_); + s.subcode_ = kNone; + sev_ = std::move(s.sev_); + s.sev_ = kNoError; + retryable_ = std::move(s.retryable_); + s.retryable_ = false; + data_loss_ = std::move(s.data_loss_); + s.data_loss_ = false; + scope_ = std::move(s.scope_); + s.scope_ = 0; + state_ = std::move(s.state_); + } + return *this; +} + +inline bool Status::operator==(const Status& rhs) const { + MarkChecked(); + rhs.MarkChecked(); + return (code_ == rhs.code_); +} + +inline bool Status::operator!=(const Status& rhs) const { + MarkChecked(); + rhs.MarkChecked(); + return !(*this == rhs); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/system_clock.h b/librocksdb-sys/rocksdb/include/rocksdb/system_clock.h new file mode 100644 index 0000000..7ca92e5 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/system_clock.h @@ -0,0 +1,114 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include + +#include + +#include "rocksdb/customizable.h" +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/status.h" + +#ifdef _WIN32 +// Windows API macro interference +#undef GetCurrentTime +#endif + +namespace ROCKSDB_NAMESPACE { +struct ConfigOptions; + +// A SystemClock is an interface used by the rocksdb implementation to access +// operating system time-related functionality. +class SystemClock : public Customizable { + public: + ~SystemClock() override {} + + static const char* Type() { return "SystemClock"; } + static Status CreateFromString(const ConfigOptions& options, + const std::string& value, + std::shared_ptr* result); + // The name of this system clock + virtual const char* Name() const override = 0; + + // The name/nickname for the Default SystemClock. This name can be used + // to determine if the clock is the default one. + static const char* kDefaultName() { return "DefaultClock"; } + + // Return a default SystemClock suitable for the current operating + // system. + static const std::shared_ptr& Default(); + + // Returns the number of micro-seconds since some fixed point in time. + // It is often used as system time such as in GenericRateLimiter + // and other places so a port needs to return system time in order to work. + virtual uint64_t NowMicros() = 0; + + // Returns the number of nano-seconds since some fixed point in time. Only + // useful for computing deltas of time in one run. + // Default implementation simply relies on NowMicros. + // In platform-specific implementations, NowNanos() should return time points + // that are MONOTONIC. + virtual uint64_t NowNanos() { return NowMicros() * 1000; } + + // Returns the number of micro-seconds of CPU time used by the current thread. + // 0 indicates not supported. + virtual uint64_t CPUMicros() { return 0; } + + // Returns the number of nano-seconds of CPU time used by the current thread. + // Default implementation simply relies on CPUMicros. + // 0 indicates not supported. + virtual uint64_t CPUNanos() { return CPUMicros() * 1000; } + + // Sleep/delay the thread for the prescribed number of micro-seconds. + virtual void SleepForMicroseconds(int micros) = 0; + + // Get the number of seconds since the Epoch, 1970-01-01 00:00:00 (UTC). + // Only overwrites *unix_time on success. + virtual Status GetCurrentTime(int64_t* unix_time) = 0; + + // Converts seconds-since-Jan-01-1970 to a printable string + virtual std::string TimeToString(uint64_t time) = 0; +}; + +// Wrapper class for a SystemClock. Redirects all methods (except Name) +// of the SystemClock interface to the target/wrapped class. +class SystemClockWrapper : public SystemClock { + public: + explicit SystemClockWrapper(const std::shared_ptr& t); + + uint64_t NowMicros() override { return target_->NowMicros(); } + + uint64_t NowNanos() override { return target_->NowNanos(); } + + uint64_t CPUMicros() override { return target_->CPUMicros(); } + + uint64_t CPUNanos() override { return target_->CPUNanos(); } + + virtual void SleepForMicroseconds(int micros) override { + return target_->SleepForMicroseconds(micros); + } + + Status GetCurrentTime(int64_t* unix_time) override { + return target_->GetCurrentTime(unix_time); + } + + std::string TimeToString(uint64_t time) override { + return target_->TimeToString(time); + } + + Status PrepareOptions(const ConfigOptions& options) override; + std::string SerializeOptions(const ConfigOptions& config_options, + const std::string& header) const override; + const Customizable* Inner() const override { return target_.get(); } + + protected: + std::shared_ptr target_; +}; + +} // end namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/table.h b/librocksdb-sys/rocksdb/include/rocksdb/table.h new file mode 100644 index 0000000..d19a95f --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/table.h @@ -0,0 +1,933 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Currently we support two types of tables: plain table and block-based table. +// 1. Block-based table: this is the default table type that we inherited from +// LevelDB, which was designed for storing data in hard disk or flash +// device. +// 2. Plain table: it is one of RocksDB's SST file format optimized +// for low query latency on pure-memory or really low-latency media. +// +// A tutorial of rocksdb table formats is available here: +// https://github.com/facebook/rocksdb/wiki/A-Tutorial-of-RocksDB-SST-formats +// +// Example code is also available +// https://github.com/facebook/rocksdb/wiki/A-Tutorial-of-RocksDB-SST-formats#wiki-examples + +#pragma once + +#include +#include +#include + +#include "rocksdb/cache.h" +#include "rocksdb/customizable.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +// -- Block-based Table +class Cache; +class FilterPolicy; +class FlushBlockPolicyFactory; +class PersistentCache; +class RandomAccessFile; +struct TableReaderOptions; +struct TableBuilderOptions; +class TableBuilder; +class TableFactory; +class TableReader; +class WritableFileWriter; +struct ConfigOptions; +struct EnvOptions; + +// Types of checksums to use for checking integrity of logical blocks within +// files. All checksums currently use 32 bits of checking power (1 in 4B +// chance of failing to detect random corruption). Traditionally, the actual +// checking power can be far from ideal if the corruption is due to misplaced +// data (e.g. physical blocks out of order in a file, or from another file), +// which is fixed in format_version=6 (see below). +enum ChecksumType : char { + kNoChecksum = 0x0, + kCRC32c = 0x1, + kxxHash = 0x2, + kxxHash64 = 0x3, + kXXH3 = 0x4, // Supported since RocksDB 6.27 +}; + +// `PinningTier` is used to specify which tier of block-based tables should +// be affected by a block cache pinning setting (see +// `MetadataCacheOptions` below). +enum class PinningTier { + // For compatibility, this value specifies to fallback to the behavior + // indicated by the deprecated options, + // `pin_l0_filter_and_index_blocks_in_cache` and + // `pin_top_level_index_and_filter`. + kFallback, + + // This tier contains no block-based tables. + kNone, + + // This tier contains block-based tables that may have originated from a + // memtable flush. In particular, it includes tables from L0 that are smaller + // than 1.5 times the current `write_buffer_size`. Note these criteria imply + // it can include intra-L0 compaction outputs and ingested files, as long as + // they are not abnormally large compared to flushed files in L0. + kFlushedAndSimilar, + + // This tier contains all block-based tables. + kAll, +}; + +// `MetadataCacheOptions` contains members indicating the desired caching +// behavior for the different categories of metadata blocks. +struct MetadataCacheOptions { + // The tier of block-based tables whose top-level index into metadata + // partitions will be pinned. Currently indexes and filters may be + // partitioned. + // + // Note `cache_index_and_filter_blocks` must be true for this option to have + // any effect. Otherwise any top-level index into metadata partitions would be + // held in table reader memory, outside the block cache. + PinningTier top_level_index_pinning = PinningTier::kFallback; + + // The tier of block-based tables whose metadata partitions will be pinned. + // Currently indexes and filters may be partitioned. + PinningTier partition_pinning = PinningTier::kFallback; + + // The tier of block-based tables whose unpartitioned metadata blocks will be + // pinned. + // + // Note `cache_index_and_filter_blocks` must be true for this option to have + // any effect. Otherwise the unpartitioned meta-blocks would be held in table + // reader memory, outside the block cache. + PinningTier unpartitioned_pinning = PinningTier::kFallback; +}; + +struct CacheEntryRoleOptions { + enum class Decision { + kEnabled, + kDisabled, + kFallback, + }; + Decision charged = Decision::kFallback; + bool operator==(const CacheEntryRoleOptions& other) const { + return charged == other.charged; + } +}; + +struct CacheUsageOptions { + CacheEntryRoleOptions options; + std::map options_overrides; +}; + +// For advanced user only +struct BlockBasedTableOptions { + static const char* kName() { return "BlockTableOptions"; }; + // @flush_block_policy_factory creates the instances of flush block policy. + // which provides a configurable way to determine when to flush a block in + // the block based tables. If not set, table builder will use the default + // block flush policy, which cut blocks by block size (please refer to + // `FlushBlockBySizePolicy`). + std::shared_ptr flush_block_policy_factory; + + // TODO(kailiu) Temporarily disable this feature by making the default value + // to be false. + // + // TODO(ajkr) we need to update names of variables controlling meta-block + // caching as they should now apply to range tombstone and compression + // dictionary meta-blocks, in addition to index and filter meta-blocks. + // + // Whether to put index/filter blocks in the block cache. When false, + // each "table reader" object will pre-load index/filter blocks during + // table initialization. Index and filter partition blocks always use + // block cache regardless of this option. + bool cache_index_and_filter_blocks = false; + + // If cache_index_and_filter_blocks is enabled, cache index and filter + // blocks with high priority. If set to true, depending on implementation of + // block cache, index, filter, and other metadata blocks may be less likely + // to be evicted than data blocks. + bool cache_index_and_filter_blocks_with_high_priority = true; + + // DEPRECATED: This option will be removed in a future version. For now, this + // option still takes effect by updating each of the following variables that + // has the default value, `PinningTier::kFallback`: + // + // - `MetadataCacheOptions::partition_pinning` + // - `MetadataCacheOptions::unpartitioned_pinning` + // + // The updated value is chosen as follows: + // + // - `pin_l0_filter_and_index_blocks_in_cache == false` -> + // `PinningTier::kNone` + // - `pin_l0_filter_and_index_blocks_in_cache == true` -> + // `PinningTier::kFlushedAndSimilar` + // + // To migrate away from this flag, explicitly configure + // `MetadataCacheOptions` as described above. + // + // if cache_index_and_filter_blocks is true and the below is true, then + // filter and index blocks are stored in the cache, but a reference is + // held in the "table reader" object so the blocks are pinned and only + // evicted from cache when the table reader is freed. + bool pin_l0_filter_and_index_blocks_in_cache = false; + + // DEPRECATED: This option will be removed in a future version. For now, this + // option still takes effect by updating + // `MetadataCacheOptions::top_level_index_pinning` when it has the + // default value, `PinningTier::kFallback`. + // + // The updated value is chosen as follows: + // + // - `pin_top_level_index_and_filter == false` -> + // `PinningTier::kNone` + // - `pin_top_level_index_and_filter == true` -> + // `PinningTier::kAll` + // + // To migrate away from this flag, explicitly configure + // `MetadataCacheOptions` as described above. + // + // If cache_index_and_filter_blocks is true and the below is true, then + // the top-level index of partitioned filter and index blocks are stored in + // the cache, but a reference is held in the "table reader" object so the + // blocks are pinned and only evicted from cache when the table reader is + // freed. This is not limited to l0 in LSM tree. + bool pin_top_level_index_and_filter = true; + + // The desired block cache pinning behavior for the different categories of + // metadata blocks. While pinning can reduce block cache contention, users + // must take care not to pin excessive amounts of data, which risks + // overflowing block cache. + MetadataCacheOptions metadata_cache_options; + + // The index type that will be used for this table. + enum IndexType : char { + // A space efficient index block that is optimized for + // binary-search-based index. + kBinarySearch = 0x00, + + // The hash index, if enabled, will do the hash lookup when + // `Options.prefix_extractor` is provided. + kHashSearch = 0x01, + + // A two-level index implementation. Both levels are binary search indexes. + // Second level index blocks ("partitions") use block cache even when + // cache_index_and_filter_blocks=false. + kTwoLevelIndexSearch = 0x02, + + // Like kBinarySearch, but index also contains first key of each block. + // This allows iterators to defer reading the block until it's actually + // needed. May significantly reduce read amplification of short range scans. + // Without it, iterator seek usually reads one block from each level-0 file + // and from each level, which may be expensive. + // Works best in combination with: + // - IndexShorteningMode::kNoShortening, + // - custom FlushBlockPolicy to cut blocks at some meaningful boundaries, + // e.g. when prefix changes. + // Makes the index significantly bigger (2x or more), especially when keys + // are long. + kBinarySearchWithFirstKey = 0x03, + }; + + IndexType index_type = kBinarySearch; + + // The index type that will be used for the data block. + enum DataBlockIndexType : char { + kDataBlockBinarySearch = 0, // traditional block type + kDataBlockBinaryAndHash = 1, // additional hash index + }; + + DataBlockIndexType data_block_index_type = kDataBlockBinarySearch; + + // #entries/#buckets. It is valid only when data_block_hash_index_type is + // kDataBlockBinaryAndHash. + double data_block_hash_table_util_ratio = 0.75; + + // Option hash_index_allow_collision is now deleted. + // It will behave as if hash_index_allow_collision=true. + + // Use the specified checksum type. Newly created table files will be + // protected with this checksum type. Old table files will still be readable, + // even though they have different checksum type. + ChecksumType checksum = kXXH3; + + // Disable block cache. If this is set to true, + // then no block cache should be used, and the block_cache should + // point to a nullptr object. + bool no_block_cache = false; + + // If non-NULL use the specified cache for blocks. + // If NULL, rocksdb will automatically create and use a 32MB internal cache. + std::shared_ptr block_cache = nullptr; + + // If non-NULL use the specified cache for pages read from device + // IF NULL, no page cache is used + std::shared_ptr persistent_cache = nullptr; + + // Approximate size of user data packed per block. Note that the + // block size specified here corresponds to uncompressed data. The + // actual size of the unit read from disk may be smaller if + // compression is enabled. This parameter can be changed dynamically. + uint64_t block_size = 4 * 1024; + + // This is used to close a block before it reaches the configured + // 'block_size'. If the percentage of free space in the current block is less + // than this specified number and adding a new record to the block will + // exceed the configured block size, then this block will be closed and the + // new record will be written to the next block. + int block_size_deviation = 10; + + // Number of keys between restart points for delta encoding of keys. + // This parameter can be changed dynamically. Most clients should + // leave this parameter alone. The minimum value allowed is 1. Any smaller + // value will be silently overwritten with 1. + int block_restart_interval = 16; + + // Same as block_restart_interval but used for the index block. + int index_block_restart_interval = 1; + + // Block size for partitioned metadata. Currently applied to indexes when + // kTwoLevelIndexSearch is used and to filters when partition_filters is used. + // Note: Since in the current implementation the filters and index partitions + // are aligned, an index/filter block is created when either index or filter + // block size reaches the specified limit. + // Note: this limit is currently applied to only index blocks; a filter + // partition is cut right after an index block is cut + // TODO(myabandeh): remove the note above when filter partitions are cut + // separately + uint64_t metadata_block_size = 4096; + + // `cache_usage_options` allows users to specify the default + // options (`cache_usage_options.options`) and the overriding + // options (`cache_usage_options.options_overrides`) + // for different `CacheEntryRole` under various features related to cache + // usage. + // + // For a certain `CacheEntryRole role` and a certain feature `f` of + // `CacheEntryRoleOptions`: + // 1. If `options_overrides` has an entry for `role` and + // `options_overrides[role].f != kFallback`, we use + // `options_overrides[role].f` + // 2. Otherwise, if `options[role].f != kFallback`, we use `options[role].f` + // 3. Otherwise, we follow the compatible existing behavior for `f` (see + // each feature's comment for more) + // + // `cache_usage_options` currently supports specifying options for the + // following features: + // + // 1. Memory charging to block cache (`CacheEntryRoleOptions::charged`) + // Memory charging is a feature of accounting memory usage of specific area + // (represented by `CacheEntryRole`) toward usage in block cache (if + // available), by updating a dynamical charge to the block cache loosely based + // on the actual memory usage of that area. + // + // (a) CacheEntryRole::kCompressionDictionaryBuildingBuffer + // (i) If kEnabled: + // Charge memory usage of the buffered data used as training samples for + // dictionary compression. + // If such memory usage exceeds the avaible space left in the block cache + // at some point (i.e, causing a cache full under + // `LRUCacheOptions::strict_capacity_limit` = true), the data will then be + // unbuffered. + // (ii) If kDisabled: + // Does not charge the memory usage mentioned above. + // (iii) Compatible existing behavior: + // Same as kEnabled. + // + // (b) CacheEntryRole::kFilterConstruction + // (i) If kEnabled: + // Charge memory usage of Bloom Filter + // (format_version >= 5) and Ribbon Filter construction. + // If additional temporary memory of Ribbon Filter exceeds the avaible + // space left in the block cache at some point (i.e, causing a cache full + // under `LRUCacheOptions::strict_capacity_limit` = true), + // construction will fall back to Bloom Filter. + // (ii) If kDisabled: + // Does not charge the memory usage mentioned above. + // (iii) Compatible existing behavior: + // Same as kDisabled. + // + // (c) CacheEntryRole::kBlockBasedTableReader + // (i) If kEnabled: + // Charge memory usage of table properties + + // index block/filter block/uncompression dictionary (when stored in table + // reader i.e, BlockBasedTableOptions::cache_index_and_filter_blocks == + // false) + some internal data structures during table reader creation. + // If such a table reader exceeds + // the avaible space left in the block cache at some point (i.e, causing + // a cache full under `LRUCacheOptions::strict_capacity_limit` = true), + // creation will fail with Status::MemoryLimit(). + // (ii) If kDisabled: + // Does not charge the memory usage mentioned above. + // (iii) Compatible existing behavior: + // Same as kDisabled. + // + // (d) CacheEntryRole::kFileMetadata + // (i) If kEnabled: + // Charge memory usage of file metadata. RocksDB holds one file metadata + // structure in-memory per on-disk table file. + // If such file metadata's + // memory exceeds the avaible space left in the block cache at some point + // (i.e, causing a cache full under `LRUCacheOptions::strict_capacity_limit` = + // true), creation will fail with Status::MemoryLimit(). + // (ii) If kDisabled: + // Does not charge the memory usage mentioned above. + // (iii) Compatible existing behavior: + // Same as kDisabled. + // + // (e) Other CacheEntryRole + // Not supported. + // `Status::kNotSupported` will be returned if + // `CacheEntryRoleOptions::charged` is set to {`kEnabled`, `kDisabled`}. + // + // + // 2. More to come ... + // + CacheUsageOptions cache_usage_options; + + // Note: currently this option requires kTwoLevelIndexSearch to be set as + // well. + // TODO(myabandeh): remove the note above once the limitation is lifted + // Use partitioned full filters for each SST file. This option is + // incompatible with block-based filters. Filter partition blocks use + // block cache even when cache_index_and_filter_blocks=false. + bool partition_filters = false; + + // Option to generate Bloom/Ribbon filters that minimize memory + // internal fragmentation. + // + // When false, malloc_usable_size is not available, or format_version < 5, + // filters are generated without regard to internal fragmentation when + // loaded into memory (historical behavior). When true (and + // malloc_usable_size is available and format_version >= 5), then + // filters are generated to "round up" and "round down" their sizes to + // minimize internal fragmentation when loaded into memory, assuming the + // reading DB has the same memory allocation characteristics as the + // generating DB. This option does not break forward or backward + // compatibility. + // + // While individual filters will vary in bits/key and false positive rate + // when setting is true, the implementation attempts to maintain a weighted + // average FP rate for filters consistent with this option set to false. + // + // With Jemalloc for example, this setting is expected to save about 10% of + // the memory footprint and block cache charge of filters, while increasing + // disk usage of filters by about 1-2% due to encoding efficiency losses + // with variance in bits/key. + // + // NOTE: Because some memory counted by block cache might be unmapped pages + // within internal fragmentation, this option can increase observed RSS + // memory usage. With cache_index_and_filter_blocks=true, this option makes + // the block cache better at using space it is allowed. (These issues + // should not arise with partitioned filters.) + // + // NOTE: Do not set to true if you do not trust malloc_usable_size. With + // this option, RocksDB might access an allocated memory object beyond its + // original size if malloc_usable_size says it is safe to do so. While this + // can be considered bad practice, it should not produce undefined behavior + // unless malloc_usable_size is buggy or broken. + bool optimize_filters_for_memory = false; + + // Use delta encoding to compress keys in blocks. + // ReadOptions::pin_data requires this option to be disabled. + // + // Default: true + bool use_delta_encoding = true; + + // If non-nullptr, use the specified filter policy to reduce disk reads. + // Many applications will benefit from passing the result of + // NewBloomFilterPolicy() here. + std::shared_ptr filter_policy = nullptr; + + // If true, place whole keys in the filter (not just prefixes). + // This must generally be true for gets to be efficient. + bool whole_key_filtering = true; + + // If true, detect corruption during Bloom Filter (format_version >= 5) + // and Ribbon Filter construction. + // + // This is an extra check that is only + // useful in detecting software bugs or CPU+memory malfunction. + // Turning on this feature increases filter construction time by 30%. + // + // This parameter can be changed dynamically by + // DB::SetOptions({{"block_based_table_factory", + // "{detect_filter_construct_corruption=true;}"}}); + // + // TODO: optimize this performance + bool detect_filter_construct_corruption = false; + + // Verify that decompressing the compressed block gives back the input. This + // is a verification mode that we use to detect bugs in compression + // algorithms. + bool verify_compression = false; + + // If used, For every data block we load into memory, we will create a bitmap + // of size ((block_size / `read_amp_bytes_per_bit`) / 8) bytes. This bitmap + // will be used to figure out the percentage we actually read of the blocks. + // + // When this feature is used Tickers::READ_AMP_ESTIMATE_USEFUL_BYTES and + // Tickers::READ_AMP_TOTAL_READ_BYTES can be used to calculate the + // read amplification using this formula + // (READ_AMP_TOTAL_READ_BYTES / READ_AMP_ESTIMATE_USEFUL_BYTES) + // + // value => memory usage (percentage of loaded blocks memory) + // 1 => 12.50 % + // 2 => 06.25 % + // 4 => 03.12 % + // 8 => 01.56 % + // 16 => 00.78 % + // + // Note: This number must be a power of 2, if not it will be sanitized + // to be the next lowest power of 2, for example a value of 7 will be + // treated as 4, a value of 19 will be treated as 16. + // + // Default: 0 (disabled) + uint32_t read_amp_bytes_per_bit = 0; + + // We currently have these versions: + // 0 -- This version can be read by really old RocksDB's. Doesn't support + // changing checksum type (default is CRC32). + // 1 -- Can be read by RocksDB's versions since 3.0. Supports non-default + // checksum, like xxHash. It is written by RocksDB when + // BlockBasedTableOptions::checksum is something other than kCRC32c. (version + // 0 is silently upconverted) + // 2 -- Can be read by RocksDB's versions since 3.10. Changes the way we + // encode compressed blocks with LZ4, BZip2 and Zlib compression. If you + // don't plan to run RocksDB before version 3.10, you should probably use + // this. + // 3 -- Can be read by RocksDB's versions since 5.15. Changes the way we + // encode the keys in index blocks. If you don't plan to run RocksDB before + // version 5.15, you should probably use this. + // This option only affects newly written tables. When reading existing + // tables, the information about version is read from the footer. + // 4 -- Can be read by RocksDB's versions since 5.16. Changes the way we + // encode the values in index blocks. If you don't plan to run RocksDB before + // version 5.16 and you are using index_block_restart_interval > 1, you should + // probably use this as it would reduce the index size. + // This option only affects newly written tables. When reading existing + // tables, the information about version is read from the footer. + // 5 -- Can be read by RocksDB's versions since 6.6.0. Full and partitioned + // filters use a generally faster and more accurate Bloom filter + // implementation, with a different schema. + // 6 -- Modified the file footer and checksum matching so that SST data + // misplaced within or between files is as likely to fail checksum + // verification as random corruption. Also checksum-protects SST footer. + uint32_t format_version = 5; + + // Store index blocks on disk in compressed format. Changing this option to + // false will avoid the overhead of decompression if index blocks are evicted + // and read back + bool enable_index_compression = true; + + // Align data blocks on lesser of page size and block size + bool block_align = false; + + // This enum allows trading off increased index size for improved iterator + // seek performance in some situations, particularly when block cache is + // disabled (ReadOptions::fill_cache = false) and direct IO is + // enabled (DBOptions::use_direct_reads = true). + // The default mode is the best tradeoff for most use cases. + // This option only affects newly written tables. + // + // The index contains a key separating each pair of consecutive blocks. + // Let A be the highest key in one block, B the lowest key in the next block, + // and I the index entry separating these two blocks: + // [ ... A] I [B ...] + // I is allowed to be anywhere in [A, B). + // If an iterator is seeked to a key in (A, I], we'll unnecessarily read the + // first block, then immediately fall through to the second block. + // However, if I=A, this can't happen, and we'll read only the second block. + // In kNoShortening mode, we use I=A. In other modes, we use the shortest + // key in [A, B), which usually significantly reduces index size. + // + // There's a similar story for the last index entry, which is an upper bound + // of the highest key in the file. If it's shortened and therefore + // overestimated, iterator is likely to unnecessarily read the last data block + // from each file on each seek. + enum class IndexShorteningMode : char { + // Use full keys. + kNoShortening, + // Shorten index keys between blocks, but use full key for the last index + // key, which is the upper bound of the whole file. + kShortenSeparators, + // Shorten both keys between blocks and key after last block. + kShortenSeparatorsAndSuccessor, + }; + + IndexShorteningMode index_shortening = + IndexShorteningMode::kShortenSeparators; + + // RocksDB does auto-readahead for iterators on noticing more than two reads + // for a table file if user doesn't provide readahead_size. The readahead + // starts at BlockBasedTableOptions.initial_auto_readahead_size (default: 8KB) + // and doubles on every additional read upto max_auto_readahead_size and + // max_auto_readahead_size can be configured. + // + // Special Value: 0 - If max_auto_readahead_size is set 0 then it will disable + // the implicit auto prefetching. + // If max_auto_readahead_size provided is less + // than initial_auto_readahead_size, then RocksDB will sanitize the + // initial_auto_readahead_size and set it to max_auto_readahead_size. + // + // Value should be provided along with KB i.e. 256 * 1024 as it will prefetch + // the blocks. + // + // Found that 256 KB readahead size provides the best performance, based on + // experiments, for auto readahead. Experiment data is in PR #3282. + // + // This parameter can be changed dynamically by + // DB::SetOptions({{"block_based_table_factory", + // "{max_auto_readahead_size=0;}"}})); + // + // Changing the value dynamically will only affect files opened after the + // change. + // + // Default: 256 KB (256 * 1024). + size_t max_auto_readahead_size = 256 * 1024; + + // If enabled, prepopulate warm/hot blocks (data, uncompressed dict, index and + // filter blocks) which are already in memory into block cache at the time of + // flush. On a flush, the block that is in memory (in memtables) get flushed + // to the device. If using Direct IO, additional IO is incurred to read this + // data back into memory again, which is avoided by enabling this option. This + // further helps if the workload exhibits high temporal locality, where most + // of the reads go to recently written data. This also helps in case of + // Distributed FileSystem. + // + // This parameter can be changed dynamically by + // DB::SetOptions({{"block_based_table_factory", + // "{prepopulate_block_cache=kFlushOnly;}"}})); + enum class PrepopulateBlockCache : char { + // Disable prepopulate block cache. + kDisable, + // Prepopulate blocks during flush only. + kFlushOnly, + }; + + PrepopulateBlockCache prepopulate_block_cache = + PrepopulateBlockCache::kDisable; + + // RocksDB does auto-readahead for iterators on noticing more than two reads + // for a table file if user doesn't provide readahead_size. The readahead size + // starts at initial_auto_readahead_size and doubles on every additional read + // upto BlockBasedTableOptions.max_auto_readahead_size. + // max_auto_readahead_size can also be configured. + // + // Scenarios: + // - If initial_auto_readahead_size is set 0 then it will disabled the + // implicit auto prefetching irrespective of max_auto_readahead_size. + // - If max_auto_readahead_size is set 0, it will disable the internal + // prefetching irrespective of initial_auto_readahead_size. + // - If initial_auto_readahead_size > max_auto_readahead_size, then RocksDB + // will sanitize the value of initial_auto_readahead_size to + // max_auto_readahead_size and readahead_size will be + // max_auto_readahead_size. + // + // Value should be provided along with KB i.e. 8 * 1024 as it will prefetch + // the blocks. + // + // This parameter can be changed dynamically by + // DB::SetOptions({{"block_based_table_factory", + // "{initial_auto_readahead_size=0;}"}})); + // + // Changing the value dynamically will only affect files opened after the + // change. + // + // Default: 8 KB (8 * 1024). + size_t initial_auto_readahead_size = 8 * 1024; + + // RocksDB does auto-readahead for iterators on noticing more than two reads + // for a table file if user doesn't provide readahead_size and reads are + // sequential. + // num_file_reads_for_auto_readahead indicates after how many + // sequential reads internal auto prefetching should be start. + // + // For example, if value is 2 then after reading 2 sequential data blocks on + // third data block prefetching will start. + // If set 0, it will start prefetching from the first read. + // + // This parameter can be changed dynamically by + // DB::SetOptions({{"block_based_table_factory", + // "{num_file_reads_for_auto_readahead=0;}"}})); + // + // Changing the value dynamically will only affect files opened after the + // change. + // + // Default: 2 + uint64_t num_file_reads_for_auto_readahead = 2; +}; + +// Table Properties that are specific to block-based table properties. +struct BlockBasedTablePropertyNames { + // value of this properties is a fixed int32 number. + static const std::string kIndexType; + // value is "1" for true and "0" for false. + static const std::string kWholeKeyFiltering; + // value is "1" for true and "0" for false. + static const std::string kPrefixFiltering; +}; + +// Create default block based table factory. +extern TableFactory* NewBlockBasedTableFactory( + const BlockBasedTableOptions& table_options = BlockBasedTableOptions()); + + +enum EncodingType : char { + // Always write full keys without any special encoding. + kPlain, + // Find opportunity to write the same prefix once for multiple rows. + // In some cases, when a key follows a previous key with the same prefix, + // instead of writing out the full key, it just writes out the size of the + // shared prefix, as well as other bytes, to save some bytes. + // + // When using this option, the user is required to use the same prefix + // extractor to make sure the same prefix will be extracted from the same key. + // The Name() value of the prefix extractor will be stored in the file. When + // reopening the file, the name of the options.prefix_extractor given will be + // bitwise compared to the prefix extractors stored in the file. An error + // will be returned if the two don't match. + kPrefix, +}; + +// Table Properties that are specific to plain table properties. +struct PlainTablePropertyNames { + static const std::string kEncodingType; + static const std::string kBloomVersion; + static const std::string kNumBloomBlocks; +}; + +const uint32_t kPlainTableVariableLength = 0; + +struct PlainTableOptions { + static const char* kName() { return "PlainTableOptions"; }; + // @user_key_len: plain table has optimization for fix-sized keys, which can + // be specified via user_key_len. Alternatively, you can pass + // `kPlainTableVariableLength` if your keys have variable + // lengths. + uint32_t user_key_len = kPlainTableVariableLength; + + // @bloom_bits_per_key: the number of bits used for bloom filer per prefix. + // You may disable it by passing a zero. + int bloom_bits_per_key = 10; + + // @hash_table_ratio: the desired utilization of the hash table used for + // prefix hashing. + // hash_table_ratio = number of prefixes / #buckets in the + // hash table + double hash_table_ratio = 0.75; + + // @index_sparseness: inside each prefix, need to build one index record for + // how many keys for binary search inside each hash bucket. + // For encoding type kPrefix, the value will be used when + // writing to determine an interval to rewrite the full + // key. It will also be used as a suggestion and satisfied + // when possible. + size_t index_sparseness = 16; + + // @huge_page_tlb_size: if <=0, allocate hash indexes and blooms from malloc. + // Otherwise from huge page TLB. The user needs to + // reserve huge pages for it to be allocated, like: + // sysctl -w vm.nr_hugepages=20 + // See linux doc Documentation/vm/hugetlbpage.txt + size_t huge_page_tlb_size = 0; + + // @encoding_type: how to encode the keys. See enum EncodingType above for + // the choices. The value will determine how to encode keys + // when writing to a new SST file. This value will be stored + // inside the SST file which will be used when reading from + // the file, which makes it possible for users to choose + // different encoding type when reopening a DB. Files with + // different encoding types can co-exist in the same DB and + // can be read. + EncodingType encoding_type = kPlain; + + // @full_scan_mode: mode for reading the whole file one record by one without + // using the index. + bool full_scan_mode = false; + + // @store_index_in_file: compute plain table index and bloom filter during + // file building and store it in file. When reading + // file, index will be mapped instead of recomputation. + bool store_index_in_file = false; +}; + +// -- Plain Table with prefix-only seek +// For this factory, you need to set Options.prefix_extractor properly to make +// it work. Look-up will starts with prefix hash lookup for key prefix. Inside +// the hash bucket found, a binary search is executed for hash conflicts. +// Finally, a linear search is used. + +extern TableFactory* NewPlainTableFactory( + const PlainTableOptions& options = PlainTableOptions()); + +struct CuckooTablePropertyNames { + // The key that is used to fill empty buckets. + static const std::string kEmptyKey; + // Fixed length of value. + static const std::string kValueLength; + // Number of hash functions used in Cuckoo Hash. + static const std::string kNumHashFunc; + // It denotes the number of buckets in a Cuckoo Block. Given a key and a + // particular hash function, a Cuckoo Block is a set of consecutive buckets, + // where starting bucket id is given by the hash function on the key. In case + // of a collision during inserting the key, the builder tries to insert the + // key in other locations of the cuckoo block before using the next hash + // function. This reduces cache miss during read operation in case of + // collision. + static const std::string kCuckooBlockSize; + // Size of the hash table. Use this number to compute the modulo of hash + // function. The actual number of buckets will be kMaxHashTableSize + + // kCuckooBlockSize - 1. The last kCuckooBlockSize-1 buckets are used to + // accommodate the Cuckoo Block from end of hash table, due to cache friendly + // implementation. + static const std::string kHashTableSize; + // Denotes if the key sorted in the file is Internal Key (if false) + // or User Key only (if true). + static const std::string kIsLastLevel; + // Indicate if using identity function for the first hash function. + static const std::string kIdentityAsFirstHash; + // Indicate if using module or bit and to calculate hash value + static const std::string kUseModuleHash; + // Fixed user key length + static const std::string kUserKeyLength; +}; + +struct CuckooTableOptions { + static const char* kName() { return "CuckooTableOptions"; }; + + // Determines the utilization of hash tables. Smaller values + // result in larger hash tables with fewer collisions. + double hash_table_ratio = 0.9; + // A property used by builder to determine the depth to go to + // to search for a path to displace elements in case of + // collision. See Builder.MakeSpaceForKey method. Higher + // values result in more efficient hash tables with fewer + // lookups but take more time to build. + uint32_t max_search_depth = 100; + // In case of collision while inserting, the builder + // attempts to insert in the next cuckoo_block_size + // locations before skipping over to the next Cuckoo hash + // function. This makes lookups more cache friendly in case + // of collisions. + uint32_t cuckoo_block_size = 5; + // If this option is enabled, user key is treated as uint64_t and its value + // is used as hash value directly. This option changes builder's behavior. + // Reader ignore this option and behave according to what specified in table + // property. + bool identity_as_first_hash = false; + // If this option is set to true, module is used during hash calculation. + // This often yields better space efficiency at the cost of performance. + // If this option is set to false, # of entries in table is constrained to be + // power of two, and bit and is used to calculate hash, which is faster in + // general. + bool use_module_hash = true; +}; + +// Cuckoo Table Factory for SST table format using Cache Friendly Cuckoo Hashing +extern TableFactory* NewCuckooTableFactory( + const CuckooTableOptions& table_options = CuckooTableOptions()); + + +class RandomAccessFileReader; + +// A base class for table factories. +class TableFactory : public Customizable { + public: + virtual ~TableFactory() override {} + + static const char* kBlockCacheOpts() { return "BlockCache"; }; + static const char* kBlockBasedTableName() { return "BlockBasedTable"; }; + static const char* kPlainTableName() { return "PlainTable"; } + static const char* kCuckooTableName() { return "CuckooTable"; }; + + // Creates and configures a new TableFactory from the input options and id. + static Status CreateFromString(const ConfigOptions& config_options, + const std::string& id, + std::shared_ptr* factory); + + static const char* Type() { return "TableFactory"; } + + // Returns a Table object table that can fetch data from file specified + // in parameter file. It's the caller's responsibility to make sure + // file is in the correct format. + // + // NewTableReader() is called in three places: + // (1) TableCache::FindTable() calls the function when table cache miss + // and cache the table object returned. + // (2) SstFileDumper (for SST Dump) opens the table and dump the table + // contents using the iterator of the table. + // (3) DBImpl::IngestExternalFile() calls this function to read the contents + // of the sst file it's attempting to add + // + // table_reader_options is a TableReaderOptions which contain all the + // needed parameters and configuration to open the table. + // file is a file handler to handle the file for the table. + // file_size is the physical file size of the file. + // table_reader is the output table reader. + virtual Status NewTableReader( + const TableReaderOptions& table_reader_options, + std::unique_ptr&& file, uint64_t file_size, + std::unique_ptr* table_reader, + bool prefetch_index_and_filter_in_cache = true) const { + ReadOptions ro; + return NewTableReader(ro, table_reader_options, std::move(file), file_size, + table_reader, prefetch_index_and_filter_in_cache); + } + + // Overload of the above function that allows the caller to pass in a + // ReadOptions + virtual Status NewTableReader( + const ReadOptions& ro, const TableReaderOptions& table_reader_options, + std::unique_ptr&& file, uint64_t file_size, + std::unique_ptr* table_reader, + bool prefetch_index_and_filter_in_cache) const = 0; + + // Return a table builder to write to a file for this table type. + // + // It is called in several places: + // (1) When flushing memtable to a level-0 output file, it creates a table + // builder (In DBImpl::WriteLevel0Table(), by calling BuildTable()) + // (2) During compaction, it gets the builder for writing compaction output + // files in DBImpl::OpenCompactionOutputFile(). + // (3) When recovering from transaction logs, it creates a table builder to + // write to a level-0 output file (In DBImpl::WriteLevel0TableForRecovery, + // by calling BuildTable()) + // (4) When running Repairer, it creates a table builder to convert logs to + // SST files (In Repairer::ConvertLogToTable() by calling BuildTable()) + // + // Multiple configured can be accessed from there, including and not limited + // to compression options. file is a handle of a writable file. + // It is the caller's responsibility to keep the file open and close the file + // after closing the table builder. compression_type is the compression type + // to use in this table. + virtual TableBuilder* NewTableBuilder( + const TableBuilderOptions& table_builder_options, + WritableFileWriter* file) const = 0; + + // Return is delete range supported + virtual bool IsDeleteRangeSupported() const { return false; } +}; + +// Create a special table factory that can open either of the supported +// table formats, based on setting inside the SST files. It should be used to +// convert a DB from one table format to another. +// @table_factory_to_write: the table factory used when writing to new files. +// @block_based_table_factory: block based table factory to use. If NULL, use +// a default one. +// @plain_table_factory: plain table factory to use. If NULL, use a default one. +// @cuckoo_table_factory: cuckoo table factory to use. If NULL, use a default +// one. +extern TableFactory* NewAdaptiveTableFactory( + std::shared_ptr table_factory_to_write = nullptr, + std::shared_ptr block_based_table_factory = nullptr, + std::shared_ptr plain_table_factory = nullptr, + std::shared_ptr cuckoo_table_factory = nullptr); + + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/table_properties.h b/librocksdb-sys/rocksdb/include/rocksdb/table_properties.h new file mode 100644 index 0000000..ebde339 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/table_properties.h @@ -0,0 +1,338 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +#pragma once + +#include + +#include +#include +#include + +#include "rocksdb/customizable.h" +#include "rocksdb/status.h" +#include "rocksdb/types.h" + +namespace ROCKSDB_NAMESPACE { + +// -- Table Properties +// Other than basic table properties, each table may also have the user +// collected properties. +// The value of the user-collected properties are encoded as raw bytes -- +// users have to interpret these values by themselves. +// Note: To do prefix seek/scan in `UserCollectedProperties`, you can do +// something similar to: +// +// UserCollectedProperties props = ...; +// for (auto pos = props.lower_bound(prefix); +// pos != props.end() && pos->first.compare(0, prefix.size(), prefix) == 0; +// ++pos) { +// ... +// } +using UserCollectedProperties = std::map; + +// table properties' human-readable names in the property block. +struct TablePropertiesNames { + static const std::string kDbId; + static const std::string kDbSessionId; + static const std::string kDbHostId; + static const std::string kOriginalFileNumber; + static const std::string kDataSize; + static const std::string kIndexSize; + static const std::string kIndexPartitions; + static const std::string kTopLevelIndexSize; + static const std::string kIndexKeyIsUserKey; + static const std::string kIndexValueIsDeltaEncoded; + static const std::string kFilterSize; + static const std::string kRawKeySize; + static const std::string kRawValueSize; + static const std::string kNumDataBlocks; + static const std::string kNumEntries; + static const std::string kNumFilterEntries; + static const std::string kDeletedKeys; + static const std::string kMergeOperands; + static const std::string kNumRangeDeletions; + static const std::string kFormatVersion; + static const std::string kFixedKeyLen; + static const std::string kFilterPolicy; + static const std::string kColumnFamilyName; + static const std::string kColumnFamilyId; + static const std::string kComparator; + static const std::string kMergeOperator; + static const std::string kPrefixExtractorName; + static const std::string kPropertyCollectors; + static const std::string kCompression; + static const std::string kCompressionOptions; + static const std::string kCreationTime; + static const std::string kOldestKeyTime; + static const std::string kFileCreationTime; + static const std::string kSlowCompressionEstimatedDataSize; + static const std::string kFastCompressionEstimatedDataSize; + static const std::string kSequenceNumberTimeMapping; + static const std::string kTailStartOffset; + static const std::string kUserDefinedTimestampsPersisted; +}; + +// `TablePropertiesCollector` provides the mechanism for users to collect +// their own properties that they are interested in. This class is essentially +// a collection of callback functions that will be invoked during table +// building. It is constructed with TablePropertiesCollectorFactory. The methods +// don't need to be thread-safe, as we will create exactly one +// TablePropertiesCollector object per table and then call it sequentially. +// +// Statuses from these callbacks are currently logged when not OK, but +// otherwise ignored by RocksDB. +// +// Exceptions MUST NOT propagate out of overridden functions into RocksDB, +// because RocksDB is not exception-safe. This could cause undefined behavior +// including data loss, unreported corruption, deadlocks, and more. +class TablePropertiesCollector { + public: + virtual ~TablePropertiesCollector() {} + + // DEPRECATE User defined collector should implement AddUserKey(), though + // this old function still works for backward compatible reason. + // Add() will be called when a new key/value pair is inserted into the table. + // @params key the user key that is inserted into the table. + // @params value the value that is inserted into the table. + virtual Status Add(const Slice& /*key*/, const Slice& /*value*/) { + return Status::InvalidArgument( + "TablePropertiesCollector::Add() deprecated."); + } + + // AddUserKey() will be called when a new key/value pair is inserted into the + // table. + // @params key the user key that is inserted into the table. + // @params value the value that is inserted into the table. + virtual Status AddUserKey(const Slice& key, const Slice& value, + EntryType /*type*/, SequenceNumber /*seq*/, + uint64_t /*file_size*/) { + // For backwards-compatibility. + return Add(key, value); + } + + // Called after each new block is cut + virtual void BlockAdd(uint64_t /* block_uncomp_bytes */, + uint64_t /* block_compressed_bytes_fast */, + uint64_t /* block_compressed_bytes_slow */) { + // Nothing to do here. Callback registers can override. + return; + } + + // Finish() will be called when a table has already been built and is ready + // for writing the properties block. + // @params properties User will add their collected statistics to + // `properties`. + virtual Status Finish(UserCollectedProperties* properties) = 0; + + // Return the human-readable properties, where the key is property name and + // the value is the human-readable form of value. + virtual UserCollectedProperties GetReadableProperties() const = 0; + + // The name of the properties collector can be used for debugging purpose. + virtual const char* Name() const = 0; + + // EXPERIMENTAL Return whether the output file should be further compacted + virtual bool NeedCompact() const { return false; } +}; + +// Constructs TablePropertiesCollector. Internals create a new +// TablePropertiesCollector for each new table +// +// Exceptions MUST NOT propagate out of overridden functions into RocksDB, +// because RocksDB is not exception-safe. This could cause undefined behavior +// including data loss, unreported corruption, deadlocks, and more. +class TablePropertiesCollectorFactory : public Customizable { + public: + struct Context { + uint32_t column_family_id; + // The level at creating the SST file (i.e, table), of which the + // properties are being collected. + int level_at_creation = kUnknownLevelAtCreation; + static const uint32_t kUnknownColumnFamily; + static const int kUnknownLevelAtCreation = -1; + }; + + ~TablePropertiesCollectorFactory() override {} + static const char* Type() { return "TablePropertiesCollectorFactory"; } + static Status CreateFromString( + const ConfigOptions& options, const std::string& value, + std::shared_ptr* result); + + // has to be thread-safe + virtual TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context context) = 0; + + // The name of the properties collector can be used for debugging purpose. + const char* Name() const override = 0; + + // Can be overridden by sub-classes to return the Name, followed by + // configuration info that will // be logged to the info log when the + // DB is opened + virtual std::string ToString() const { return Name(); } +}; + +// TableProperties contains a bunch of read-only properties of its associated +// table. +struct TableProperties { + public: + // the file number at creation time, or 0 for unknown. When known, + // combining with db_session_id must uniquely identify an SST file. + uint64_t orig_file_number = 0; + // the total size of all data blocks. + uint64_t data_size = 0; + // the size of index block. + uint64_t index_size = 0; + // Total number of index partitions if kTwoLevelIndexSearch is used + uint64_t index_partitions = 0; + // Size of the top-level index if kTwoLevelIndexSearch is used + uint64_t top_level_index_size = 0; + // Whether the index key is user key. Otherwise it includes 8 byte of sequence + // number added by internal key format. + uint64_t index_key_is_user_key = 0; + // Whether delta encoding is used to encode the index values. + uint64_t index_value_is_delta_encoded = 0; + // the size of filter block. + uint64_t filter_size = 0; + // total raw (uncompressed, undelineated) key size + uint64_t raw_key_size = 0; + // total raw (uncompressed, undelineated) value size + uint64_t raw_value_size = 0; + // the number of blocks in this table + uint64_t num_data_blocks = 0; + // the number of entries in this table + uint64_t num_entries = 0; + // the number of unique entries (keys or prefixes) added to filters + uint64_t num_filter_entries = 0; + // the number of deletions in the table + uint64_t num_deletions = 0; + // the number of merge operands in the table + uint64_t num_merge_operands = 0; + // the number of range deletions in this table + uint64_t num_range_deletions = 0; + // format version, reserved for backward compatibility + uint64_t format_version = 0; + // If 0, key is variable length. Otherwise number of bytes for each key. + uint64_t fixed_key_len = 0; + // ID of column family for this SST file, corresponding to the CF identified + // by column_family_name. + uint64_t column_family_id = ROCKSDB_NAMESPACE:: + TablePropertiesCollectorFactory::Context::kUnknownColumnFamily; + // Timestamp of the latest key. 0 means unknown. + // TODO(sagar0): Should be changed to latest_key_time ... but don't know the + // full implications of backward compatibility. Hence retaining for now. + uint64_t creation_time = 0; + + // Timestamp of the earliest key. 0 means unknown. + uint64_t oldest_key_time = 0; + // Actual SST file creation time. 0 means unknown. + uint64_t file_creation_time = 0; + // Estimated size of data blocks if compressed using a relatively slower + // compression algorithm (see `ColumnFamilyOptions::sample_for_compression`). + // 0 means unknown. + uint64_t slow_compression_estimated_data_size = 0; + // Estimated size of data blocks if compressed using a relatively faster + // compression algorithm (see `ColumnFamilyOptions::sample_for_compression`). + // 0 means unknown. + uint64_t fast_compression_estimated_data_size = 0; + // Offset of the value of the property "external sst file global seqno" in the + // file if the property exists. + // 0 means not exists. + uint64_t external_sst_file_global_seqno_offset = 0; + + // Offset where the "tail" part of SST file starts + // "Tail" refers to all blocks after data blocks till the end of the SST file + uint64_t tail_start_offset = 0; + + // Value of the `AdvancedColumnFamilyOptions.persist_user_defined_timestamps` + // when the file is created. Default to be true, only when this flag is false, + // it's explicitly written to meta properties block. + uint64_t user_defined_timestamps_persisted = 1; + + // DB identity + // db_id is an identifier generated the first time the DB is created + // If DB identity is unset or unassigned, `db_id` will be an empty string. + std::string db_id; + + // DB session identity + // db_session_id is an identifier that gets reset every time the DB is opened + // If DB session identity is unset or unassigned, `db_session_id` will be an + // empty string. + std::string db_session_id; + + // Location of the machine hosting the DB instance + // db_host_id identifies the location of the host in some form + // (hostname by default, but can also be any string of the user's choosing). + // It can potentially change whenever the DB is opened + std::string db_host_id; + + // Name of the column family with which this SST file is associated. + // If column family is unknown, `column_family_name` will be an empty string. + std::string column_family_name; + + // The name of the filter policy used in this table. + // If no filter policy is used, `filter_policy_name` will be an empty string. + std::string filter_policy_name; + + // The name of the comparator used in this table. + std::string comparator_name; + + // The name of the merge operator used in this table. + // If no merge operator is used, `merge_operator_name` will be "nullptr". + std::string merge_operator_name; + + // The name of the prefix extractor used in this table + // If no prefix extractor is used, `prefix_extractor_name` will be "nullptr". + std::string prefix_extractor_name; + + // The names of the property collectors factories used in this table + // separated by commas + // {collector_name[1]},{collector_name[2]},{collector_name[3]} .. + std::string property_collectors_names; + + // The compression algo used to compress the SST files. + std::string compression_name; + + // Compression options used to compress the SST files. + std::string compression_options; + + // Sequence number to time mapping, delta encoded. + std::string seqno_to_time_mapping; + + // user collected properties + UserCollectedProperties user_collected_properties; + UserCollectedProperties readable_properties; + + // convert this object to a human readable form + // @prop_delim: delimiter for each property. + std::string ToString(const std::string& prop_delim = "; ", + const std::string& kv_delim = "=") const; + + // Aggregate the numerical member variables of the specified + // TableProperties. + void Add(const TableProperties& tp); + + // Subset of properties that make sense when added together + // between tables. Keys match field names in this class instead + // of using full property names. + std::map GetAggregatablePropertiesAsMap() const; + + // Return the approximated memory usage of this TableProperties object, + // including memory used by the string properties and UserCollectedProperties + std::size_t ApproximateMemoryUsage() const; +}; + +// Extra properties +// Below is a list of non-basic properties that are collected by database +// itself. Especially some properties regarding to the internal keys (which +// is unknown to `table`). +// +// DEPRECATED: these properties now belong as TableProperties members. Please +// use TableProperties::num_deletions and TableProperties::num_merge_operands, +// respectively. +extern uint64_t GetDeletedKeys(const UserCollectedProperties& props); +extern uint64_t GetMergeOperands(const UserCollectedProperties& props, + bool* property_present); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/table_reader_caller.h b/librocksdb-sys/rocksdb/include/rocksdb/table_reader_caller.h new file mode 100644 index 0000000..10ec081 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/table_reader_caller.h @@ -0,0 +1,41 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { +// A list of callers for a table reader. It is used to trace the caller that +// accesses on a block. This is only used for block cache tracing and analysis. +// A user may use kUncategorized if the caller is not interesting for analysis +// or the table reader is called in the test environment, e.g., unit test, table +// reader benchmark, etc. +enum TableReaderCaller : char { + kUserGet = 1, + kUserMultiGet = 2, + kUserIterator = 3, + kUserApproximateSize = 4, + kUserVerifyChecksum = 5, + kSSTDumpTool = 6, + kExternalSSTIngestion = 7, + kRepair = 8, + kPrefetch = 9, + kCompaction = 10, + // A compaction job may refill the block cache with blocks in the new SST + // files if paranoid_file_checks is true. + kCompactionRefill = 11, + // After building a table, it may load all its blocks into the block cache if + // paranoid_file_checks is true. + kFlush = 12, + // sst_file_reader. + kSSTFileReader = 13, + // A list of callers that are either not interesting for analysis or are + // calling from a test environment, e.g., unit test, benchmark, etc. + kUncategorized = 14, + // All callers should be added before kMaxBlockCacheLookupCaller. + kMaxBlockCacheLookupCaller +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/thread_status.h b/librocksdb-sys/rocksdb/include/rocksdb/thread_status.h new file mode 100644 index 0000000..beecdfd --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/thread_status.h @@ -0,0 +1,190 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file defines the structures for exposing run-time status of any +// rocksdb-related thread. Such run-time status can be obtained via +// GetThreadList() API. +// +// Note that all thread-status features are still under-development, and +// thus APIs and class definitions might subject to change at this point. +// Will remove this comment once the APIs have been finalized. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "rocksdb/rocksdb_namespace.h" + +#if !defined(NROCKSDB_THREAD_STATUS) +#define ROCKSDB_USING_THREAD_STATUS +#endif + +namespace ROCKSDB_NAMESPACE { + +// TODO(yhchiang): remove this function once c++14 is available +// as std::max will be able to cover this. +// Current MS compiler does not support constexpr +template +struct constexpr_max { + static const int result = (A > B) ? A : B; +}; + +// A structure that describes the current status of a thread. +// The status of active threads can be fetched using +// ROCKSDB_NAMESPACE::GetThreadList(). +struct ThreadStatus { + // The type of a thread. + enum ThreadType : int { + HIGH_PRIORITY = 0, // RocksDB BG thread in high-pri thread pool + LOW_PRIORITY, // RocksDB BG thread in low-pri thread pool + USER, // User thread (Non-RocksDB BG thread) + BOTTOM_PRIORITY, // RocksDB BG thread in bottom-pri thread pool + NUM_THREAD_TYPES + }; + + // The type used to refer to a thread operation. + // A thread operation describes high-level action of a thread. + // Examples include compaction and flush. + enum OperationType : int { + OP_UNKNOWN = 0, + OP_COMPACTION, + OP_FLUSH, + OP_DBOPEN, + NUM_OP_TYPES + }; + + enum OperationStage : int { + STAGE_UNKNOWN = 0, + STAGE_FLUSH_RUN, + STAGE_FLUSH_WRITE_L0, + STAGE_COMPACTION_PREPARE, + STAGE_COMPACTION_RUN, + STAGE_COMPACTION_PROCESS_KV, + STAGE_COMPACTION_INSTALL, + STAGE_COMPACTION_SYNC_FILE, + STAGE_PICK_MEMTABLES_TO_FLUSH, + STAGE_MEMTABLE_ROLLBACK, + STAGE_MEMTABLE_INSTALL_FLUSH_RESULTS, + NUM_OP_STAGES + }; + + enum CompactionPropertyType : int { + COMPACTION_JOB_ID = 0, + COMPACTION_INPUT_OUTPUT_LEVEL, + COMPACTION_PROP_FLAGS, + COMPACTION_TOTAL_INPUT_BYTES, + COMPACTION_BYTES_READ, + COMPACTION_BYTES_WRITTEN, + NUM_COMPACTION_PROPERTIES + }; + + enum FlushPropertyType : int { + FLUSH_JOB_ID = 0, + FLUSH_BYTES_MEMTABLES, + FLUSH_BYTES_WRITTEN, + NUM_FLUSH_PROPERTIES + }; + + // The maximum number of properties of an operation. + // This number should be set to the biggest NUM_XXX_PROPERTIES. + static const int kNumOperationProperties = + constexpr_max::result; + + // The type used to refer to a thread state. + // A state describes lower-level action of a thread + // such as reading / writing a file or waiting for a mutex. + enum StateType : int { + STATE_UNKNOWN = 0, + STATE_MUTEX_WAIT = 1, + NUM_STATE_TYPES + }; + + ThreadStatus(const uint64_t _id, const ThreadType _thread_type, + const std::string& _db_name, const std::string& _cf_name, + const OperationType _operation_type, + const uint64_t _op_elapsed_micros, + const OperationStage _operation_stage, + const uint64_t _op_props[], const StateType _state_type) + : thread_id(_id), + thread_type(_thread_type), + db_name(_db_name), + cf_name(_cf_name), + operation_type(_operation_type), + op_elapsed_micros(_op_elapsed_micros), + operation_stage(_operation_stage), + state_type(_state_type) { + for (int i = 0; i < kNumOperationProperties; ++i) { + op_properties[i] = _op_props[i]; + } + } + + // An unique ID for the thread. + const uint64_t thread_id; + + // The type of the thread, it could be HIGH_PRIORITY, + // LOW_PRIORITY, and USER + const ThreadType thread_type; + + // The name of the DB instance where the thread is currently + // involved with. It would be set to empty string if the thread + // does not involve in any DB operation. + const std::string db_name; + + // The name of the column family where the thread is currently + // It would be set to empty string if the thread does not involve + // in any column family. + const std::string cf_name; + + // The operation (high-level action) that the current thread is involved. + const OperationType operation_type; + + // The elapsed time of the current thread operation in microseconds. + const uint64_t op_elapsed_micros; + + // An integer showing the current stage where the thread is involved + // in the current operation. + const OperationStage operation_stage; + + // A list of properties that describe some details about the current + // operation. Same field in op_properties[] might have different + // meanings for different operations. + uint64_t op_properties[kNumOperationProperties]; + + // The state (lower-level action) that the current thread is involved. + const StateType state_type; + + // The followings are a set of utility functions for interpreting + // the information of ThreadStatus + + static std::string GetThreadTypeName(ThreadType thread_type); + + // Obtain the name of an operation given its type. + static const std::string& GetOperationName(OperationType op_type); + + static const std::string MicrosToString(uint64_t op_elapsed_time); + + // Obtain a human-readable string describing the specified operation stage. + static const std::string& GetOperationStageName(OperationStage stage); + + // Obtain the name of the "i"th operation property of the + // specified operation. + static const std::string& GetOperationPropertyName(OperationType op_type, + int i); + + // Translate the "i"th property of the specified operation given + // a property value. + static std::map InterpretOperationProperties( + OperationType op_type, const uint64_t* op_properties); + + // Obtain the name of a state given its type. + static const std::string& GetStateName(StateType state_type); +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/threadpool.h b/librocksdb-sys/rocksdb/include/rocksdb/threadpool.h new file mode 100644 index 0000000..f1cc557 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/threadpool.h @@ -0,0 +1,67 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once + +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +/* + * ThreadPool is a component that will spawn N background threads that will + * be used to execute scheduled work, The number of background threads could + * be modified by calling SetBackgroundThreads(). + * */ +class ThreadPool { + public: + virtual ~ThreadPool() {} + + // Wait for all threads to finish. + // Discard those threads that did not start + // executing + virtual void JoinAllThreads() = 0; + + // Set the number of background threads that will be executing the + // scheduled jobs. + virtual void SetBackgroundThreads(int num) = 0; + virtual int GetBackgroundThreads() = 0; + + // Get the number of jobs scheduled in the ThreadPool queue. + virtual unsigned int GetQueueLen() const = 0; + + // Waits for all jobs to complete those + // that already started running and those that did not + // start yet. This ensures that everything that was thrown + // on the TP runs even though + // we may not have specified enough threads for the amount + // of jobs + virtual void WaitForJobsAndJoinAllThreads() = 0; + + // Submit a fire and forget jobs + // This allows to submit the same job multiple times + virtual void SubmitJob(const std::function&) = 0; + // This moves the function in for efficiency + virtual void SubmitJob(std::function&&) = 0; + + // Reserve available background threads. This function does not ensure + // so many threads can be reserved, instead it will return the number of + // threads that can be reserved against the desired one. In other words, + // the number of available threads could be less than the input. + virtual int ReserveThreads(int /*threads_to_be_reserved*/) { return 0; } + + // Release a specific number of reserved threads + virtual int ReleaseThreads(int /*threads_to_be_released*/) { return 0; } +}; + +// NewThreadPool() is a function that could be used to create a ThreadPool +// with `num_threads` background threads. +extern ThreadPool* NewThreadPool(int num_threads); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/trace_reader_writer.h b/librocksdb-sys/rocksdb/include/rocksdb/trace_reader_writer.h new file mode 100644 index 0000000..335e091 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/trace_reader_writer.h @@ -0,0 +1,52 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "rocksdb/env.h" + +namespace ROCKSDB_NAMESPACE { + +// Allow custom implementations of TraceWriter and TraceReader. +// By default, RocksDB provides a way to capture the traces to a file using the +// factory NewFileTraceWriter(). But users could also choose to export traces to +// any other system by providing custom implementations of TraceWriter and +// TraceReader. + +// TraceWriter allows exporting RocksDB traces to any system, one operation at +// a time. +class TraceWriter { + public: + virtual ~TraceWriter() = default; + + virtual Status Write(const Slice& data) = 0; + virtual Status Close() = 0; + virtual uint64_t GetFileSize() = 0; +}; + +// TraceReader allows reading RocksDB traces from any system, one operation at +// a time. A RocksDB Replayer could depend on this to replay operations. +class TraceReader { + public: + virtual ~TraceReader() = default; + + virtual Status Read(std::string* data) = 0; + virtual Status Close() = 0; + + // Seek back to the trace header. Replayer can call this method to restart + // replaying. Note this method may fail if the reader is already closed. + virtual Status Reset() = 0; +}; + +// Factory methods to write/read traces to/from a file. +// The implementations may not be thread-safe. +Status NewFileTraceWriter(Env* env, const EnvOptions& env_options, + const std::string& trace_filename, + std::unique_ptr* trace_writer); +Status NewFileTraceReader(Env* env, const EnvOptions& env_options, + const std::string& trace_filename, + std::unique_ptr* trace_reader); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/trace_record.h b/librocksdb-sys/rocksdb/include/rocksdb/trace_record.h new file mode 100644 index 0000000..c00f5ca --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/trace_record.h @@ -0,0 +1,248 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include + +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +class ColumnFamilyHandle; +class DB; + +// Supported trace record types. +enum TraceType : char { + kTraceNone = 0, + kTraceBegin = 1, + kTraceEnd = 2, + // Query level tracing related trace types. + kTraceWrite = 3, + kTraceGet = 4, + kTraceIteratorSeek = 5, + kTraceIteratorSeekForPrev = 6, + // Block cache tracing related trace types. + kBlockTraceIndexBlock = 7, + // TODO: split out kinds of filter blocks? + kBlockTraceFilterBlock = 8, + kBlockTraceDataBlock = 9, + kBlockTraceUncompressionDictBlock = 10, + kBlockTraceRangeDeletionBlock = 11, + // IO tracing related trace type. + kIOTracer = 12, + // Query level tracing related trace type. + kTraceMultiGet = 13, + // All trace types should be added before kTraceMax + kTraceMax, +}; + +class GetQueryTraceRecord; +class IteratorSeekQueryTraceRecord; +class MultiGetQueryTraceRecord; +class TraceRecordResult; +class WriteQueryTraceRecord; + +// Base class for all types of trace records. +class TraceRecord { + public: + explicit TraceRecord(uint64_t timestamp); + + virtual ~TraceRecord() = default; + + // Type of the trace record. + virtual TraceType GetTraceType() const = 0; + + // Timestamp (in microseconds) of this trace. + virtual uint64_t GetTimestamp() const; + + class Handler { + public: + virtual ~Handler() = default; + + virtual Status Handle(const WriteQueryTraceRecord& record, + std::unique_ptr* result) = 0; + + virtual Status Handle(const GetQueryTraceRecord& record, + std::unique_ptr* result) = 0; + + virtual Status Handle(const IteratorSeekQueryTraceRecord& record, + std::unique_ptr* result) = 0; + + virtual Status Handle(const MultiGetQueryTraceRecord& record, + std::unique_ptr* result) = 0; + }; + + // Accept the handler and report the corresponding result in `result`. + virtual Status Accept(Handler* handler, + std::unique_ptr* result) = 0; + + // Create a handler for the exeution of TraceRecord. + static Handler* NewExecutionHandler( + DB* db, const std::vector& handles); + + private: + uint64_t timestamp_; +}; + +// Base class for all query types of trace records. +class QueryTraceRecord : public TraceRecord { + public: + explicit QueryTraceRecord(uint64_t timestamp); +}; + +// Trace record for DB::Write() operation. +class WriteQueryTraceRecord : public QueryTraceRecord { + public: + WriteQueryTraceRecord(PinnableSlice&& write_batch_rep, uint64_t timestamp); + + WriteQueryTraceRecord(const std::string& write_batch_rep, uint64_t timestamp); + + virtual ~WriteQueryTraceRecord() override; + + TraceType GetTraceType() const override { return kTraceWrite; } + + // rep string for the WriteBatch. + virtual Slice GetWriteBatchRep() const; + + Status Accept(Handler* handler, + std::unique_ptr* result) override; + + private: + PinnableSlice rep_; +}; + +// Trace record for DB::Get() operation +class GetQueryTraceRecord : public QueryTraceRecord { + public: + GetQueryTraceRecord(uint32_t column_family_id, PinnableSlice&& key, + uint64_t timestamp); + + GetQueryTraceRecord(uint32_t column_family_id, const std::string& key, + uint64_t timestamp); + + virtual ~GetQueryTraceRecord() override; + + TraceType GetTraceType() const override { return kTraceGet; } + + // Column family ID. + virtual uint32_t GetColumnFamilyID() const; + + // Key to get. + virtual Slice GetKey() const; + + Status Accept(Handler* handler, + std::unique_ptr* result) override; + + private: + uint32_t cf_id_; + PinnableSlice key_; +}; + +// Base class for all Iterator related operations. +class IteratorQueryTraceRecord : public QueryTraceRecord { + public: + explicit IteratorQueryTraceRecord(uint64_t timestamp); + + IteratorQueryTraceRecord(PinnableSlice&& lower_bound, + PinnableSlice&& upper_bound, uint64_t timestamp); + + IteratorQueryTraceRecord(const std::string& lower_bound, + const std::string& upper_bound, uint64_t timestamp); + + virtual ~IteratorQueryTraceRecord() override; + + // Get the iterator's lower/upper bound. They may be used in ReadOptions to + // create an Iterator instance. + virtual Slice GetLowerBound() const; + virtual Slice GetUpperBound() const; + + private: + PinnableSlice lower_; + PinnableSlice upper_; +}; + +// Trace record for Iterator::Seek() and Iterator::SeekForPrev() operation. +class IteratorSeekQueryTraceRecord : public IteratorQueryTraceRecord { + public: + // Currently we only support Seek() and SeekForPrev(). + enum SeekType { + kSeek = kTraceIteratorSeek, + kSeekForPrev = kTraceIteratorSeekForPrev + }; + + IteratorSeekQueryTraceRecord(SeekType seekType, uint32_t column_family_id, + PinnableSlice&& key, uint64_t timestamp); + + IteratorSeekQueryTraceRecord(SeekType seekType, uint32_t column_family_id, + const std::string& key, uint64_t timestamp); + + IteratorSeekQueryTraceRecord(SeekType seekType, uint32_t column_family_id, + PinnableSlice&& key, PinnableSlice&& lower_bound, + PinnableSlice&& upper_bound, uint64_t timestamp); + + IteratorSeekQueryTraceRecord(SeekType seekType, uint32_t column_family_id, + const std::string& key, + const std::string& lower_bound, + const std::string& upper_bound, + uint64_t timestamp); + + virtual ~IteratorSeekQueryTraceRecord() override; + + // Trace type matches the seek type. + TraceType GetTraceType() const override; + + // Type of seek, Seek or SeekForPrev. + virtual SeekType GetSeekType() const; + + // Column family ID. + virtual uint32_t GetColumnFamilyID() const; + + // Key to seek to. + virtual Slice GetKey() const; + + Status Accept(Handler* handler, + std::unique_ptr* result) override; + + private: + SeekType type_; + uint32_t cf_id_; + PinnableSlice key_; +}; + +// Trace record for DB::MultiGet() operation. +class MultiGetQueryTraceRecord : public QueryTraceRecord { + public: + MultiGetQueryTraceRecord(std::vector column_family_ids, + std::vector&& keys, + uint64_t timestamp); + + MultiGetQueryTraceRecord(std::vector column_family_ids, + const std::vector& keys, + uint64_t timestamp); + + virtual ~MultiGetQueryTraceRecord() override; + + TraceType GetTraceType() const override { return kTraceMultiGet; } + + // Column familiy IDs. + virtual std::vector GetColumnFamilyIDs() const; + + // Keys to get. + virtual std::vector GetKeys() const; + + Status Accept(Handler* handler, + std::unique_ptr* result) override; + + private: + std::vector cf_ids_; + std::vector keys_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/trace_record_result.h b/librocksdb-sys/rocksdb/include/rocksdb/trace_record_result.h new file mode 100644 index 0000000..0cd0004 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/trace_record_result.h @@ -0,0 +1,187 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include + +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "rocksdb/trace_record.h" + +namespace ROCKSDB_NAMESPACE { + +class IteratorTraceExecutionResult; +class MultiValuesTraceExecutionResult; +class SingleValueTraceExecutionResult; +class StatusOnlyTraceExecutionResult; + +// Base class for the results of all types of trace records. +// Theses classes can be used to report the execution result of +// TraceRecord::Handler::Handle() or TraceRecord::Accept(). +class TraceRecordResult { + public: + explicit TraceRecordResult(TraceType trace_type); + + virtual ~TraceRecordResult() = default; + + // Trace type of the corresponding TraceRecord. + virtual TraceType GetTraceType() const; + + class Handler { + public: + virtual ~Handler() = default; + + virtual Status Handle(const StatusOnlyTraceExecutionResult& result) = 0; + + virtual Status Handle(const SingleValueTraceExecutionResult& result) = 0; + + virtual Status Handle(const MultiValuesTraceExecutionResult& result) = 0; + + virtual Status Handle(const IteratorTraceExecutionResult& result) = 0; + }; + + // Accept the handler. + virtual Status Accept(Handler* handler) = 0; + + private: + TraceType trace_type_; +}; + +// Base class for the results from the trace record execution handler (created +// by TraceRecord::NewExecutionHandler()). +// +// The actual execution status or returned values may be hidden from +// TraceRecord::Handler::Handle and TraceRecord::Accept. For example, a +// GetQueryTraceRecord's execution calls DB::Get() internally. DB::Get() may +// return Status::NotFound() but TraceRecord::Handler::Handle() or +// TraceRecord::Accept() will still return Status::OK(). The actual status from +// DB::Get() and the returned value string may be saved in a +// SingleValueTraceExecutionResult. +class TraceExecutionResult : public TraceRecordResult { + public: + TraceExecutionResult(uint64_t start_timestamp, uint64_t end_timestamp, + TraceType trace_type); + + // Execution start/end timestamps and request latency in microseconds. + virtual uint64_t GetStartTimestamp() const; + virtual uint64_t GetEndTimestamp() const; + inline uint64_t GetLatency() const { + return GetEndTimestamp() - GetStartTimestamp(); + } + + private: + uint64_t ts_start_; + uint64_t ts_end_; +}; + +// Result for operations that only return a single Status. +// Example operation: DB::Write() +class StatusOnlyTraceExecutionResult : public TraceExecutionResult { + public: + StatusOnlyTraceExecutionResult(Status status, uint64_t start_timestamp, + uint64_t end_timestamp, TraceType trace_type); + + virtual ~StatusOnlyTraceExecutionResult() override = default; + + // Return value of DB::Write(), etc. + virtual const Status& GetStatus() const; + + virtual Status Accept(Handler* handler) override; + + private: + Status status_; +}; + +// Result for operations that return a Status and a value. +// Example operation: DB::Get() +class SingleValueTraceExecutionResult : public TraceExecutionResult { + public: + SingleValueTraceExecutionResult(Status status, const std::string& value, + uint64_t start_timestamp, + uint64_t end_timestamp, TraceType trace_type); + + SingleValueTraceExecutionResult(Status status, std::string&& value, + uint64_t start_timestamp, + uint64_t end_timestamp, TraceType trace_type); + + virtual ~SingleValueTraceExecutionResult() override; + + // Return status of DB::Get(). + virtual const Status& GetStatus() const; + + // Value for the searched key. + virtual const std::string& GetValue() const; + + virtual Status Accept(Handler* handler) override; + + private: + Status status_; + std::string value_; +}; + +// Result for operations that return multiple Status(es) and values as vectors. +// Example operation: DB::MultiGet() +class MultiValuesTraceExecutionResult : public TraceExecutionResult { + public: + MultiValuesTraceExecutionResult(std::vector multi_status, + std::vector values, + uint64_t start_timestamp, + uint64_t end_timestamp, TraceType trace_type); + + virtual ~MultiValuesTraceExecutionResult() override; + + // Returned Status(es) of DB::MultiGet(). + virtual const std::vector& GetMultiStatus() const; + + // Returned values for the searched keys. + virtual const std::vector& GetValues() const; + + virtual Status Accept(Handler* handler) override; + + private: + std::vector multi_status_; + std::vector values_; +}; + +// Result for Iterator operations. +// Example operations: Iterator::Seek(), Iterator::SeekForPrev() +class IteratorTraceExecutionResult : public TraceExecutionResult { + public: + IteratorTraceExecutionResult(bool valid, Status status, PinnableSlice&& key, + PinnableSlice&& value, uint64_t start_timestamp, + uint64_t end_timestamp, TraceType trace_type); + + IteratorTraceExecutionResult(bool valid, Status status, + const std::string& key, const std::string& value, + uint64_t start_timestamp, uint64_t end_timestamp, + TraceType trace_type); + + virtual ~IteratorTraceExecutionResult() override; + + // Return if the Iterator is valid. + virtual bool GetValid() const; + + // Return the status of the Iterator. + virtual const Status& GetStatus() const; + + // Key of the current iterating entry, empty if GetValid() is false. + virtual Slice GetKey() const; + + // Value of the current iterating entry, empty if GetValid() is false. + virtual Slice GetValue() const; + + virtual Status Accept(Handler* handler) override; + + private: + bool valid_; + Status status_; + PinnableSlice key_; + PinnableSlice value_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/transaction_log.h b/librocksdb-sys/rocksdb/include/rocksdb/transaction_log.h new file mode 100644 index 0000000..e13ad8f --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/transaction_log.h @@ -0,0 +1,122 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include + +#include "rocksdb/status.h" +#include "rocksdb/types.h" +#include "rocksdb/write_batch.h" + +namespace ROCKSDB_NAMESPACE { + +class LogFile; +using VectorLogPtr = std::vector>; + +enum WalFileType { + /* Indicates that WAL file is in archive directory. WAL files are moved from + * the main db directory to archive directory once they are not live and stay + * there until cleaned up. Files are cleaned depending on archive size + * (Options::WAL_size_limit_MB) and time since last cleaning + * (Options::WAL_ttl_seconds). + */ + kArchivedLogFile = 0, + + /* Indicates that WAL file is live and resides in the main db directory */ + kAliveLogFile = 1 +}; + +class LogFile { + public: + LogFile() {} + virtual ~LogFile() {} + + // Returns log file's pathname relative to the main db dir + // Eg. For a live-log-file = /000003.log + // For an archived-log-file = /archive/000003.log + virtual std::string PathName() const = 0; + + // Primary identifier for log file. + // This is directly proportional to creation time of the log file + virtual uint64_t LogNumber() const = 0; + + // Log file can be either alive or archived + virtual WalFileType Type() const = 0; + + // Starting sequence number of writebatch written in this log file + virtual SequenceNumber StartSequence() const = 0; + + // Size of log file on disk in Bytes + virtual uint64_t SizeFileBytes() const = 0; +}; + +struct BatchResult { + SequenceNumber sequence = 0; + std::unique_ptr writeBatchPtr; + + // Add empty __ctor and __dtor for the rule of five + // However, preserve the original semantics and prohibit copying + // as the std::unique_ptr member does not copy. + BatchResult() {} + + ~BatchResult() {} + + BatchResult(const BatchResult&) = delete; + + BatchResult& operator=(const BatchResult&) = delete; + + BatchResult(BatchResult&& bResult) + : sequence(std::move(bResult.sequence)), + writeBatchPtr(std::move(bResult.writeBatchPtr)) {} + + BatchResult& operator=(BatchResult&& bResult) { + sequence = std::move(bResult.sequence); + writeBatchPtr = std::move(bResult.writeBatchPtr); + return *this; + } +}; + +// A TransactionLogIterator is used to iterate over the transactions in a db. +// One run of the iterator is continuous, i.e. the iterator will stop at the +// beginning of any gap in sequences +class TransactionLogIterator { + public: + TransactionLogIterator() {} + virtual ~TransactionLogIterator() {} + + // An iterator is either positioned at a WriteBatch or not valid. + // This method returns true if the iterator is valid. + // Can read data from a valid iterator. + virtual bool Valid() = 0; + + // Moves the iterator to the next WriteBatch. + // REQUIRES: Valid() to be true. + virtual void Next() = 0; + + // Returns ok if the iterator is valid. + // Returns the Error when something has gone wrong. + virtual Status status() = 0; + + // If valid return's the current write_batch and the sequence number of the + // earliest transaction contained in the batch. + // ONLY use if Valid() is true and status() is OK. + virtual BatchResult GetBatch() = 0; + + // The read options for TransactionLogIterator. + struct ReadOptions { + // If true, all data read from underlying storage will be + // verified against corresponding checksums. + // Default: true + bool verify_checksums_; + + ReadOptions() : verify_checksums_(true) {} + + explicit ReadOptions(bool verify_checksums) + : verify_checksums_(verify_checksums) {} + }; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/types.h b/librocksdb-sys/rocksdb/include/rocksdb/types.h new file mode 100644 index 0000000..3f8ce97 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/types.h @@ -0,0 +1,94 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "rocksdb/slice.h" + +namespace ROCKSDB_NAMESPACE { + +// Define all public custom types here. + +using ColumnFamilyId = uint32_t; + +// Represents a sequence number in a WAL file. +using SequenceNumber = uint64_t; + +const SequenceNumber kMinUnCommittedSeq = 1; // 0 is always committed + +enum class TableFileCreationReason { + kFlush, + kCompaction, + kRecovery, + kMisc, +}; + +enum class BlobFileCreationReason { + kFlush, + kCompaction, + kRecovery, +}; + +// The types of files RocksDB uses in a DB directory. (Available for +// advanced options.) +enum FileType { + kWalFile, + kDBLockFile, + kTableFile, + kDescriptorFile, + kCurrentFile, + kTempFile, + kInfoLogFile, // Either the current one, or an old one + kMetaDatabase, + kIdentityFile, + kOptionsFile, + kBlobFile +}; + +// User-oriented representation of internal key types. +// Ordering of this enum entries should not change. +enum EntryType { + kEntryPut, + kEntryDelete, + kEntrySingleDelete, + kEntryMerge, + kEntryRangeDeletion, + kEntryBlobIndex, + kEntryDeleteWithTimestamp, + kEntryWideColumnEntity, + kEntryOther, +}; + +enum class WriteStallCause { + // Beginning of CF-scope write stall causes + // + // Always keep `kMemtableLimit` as the first stat in this section + kMemtableLimit, + kL0FileCountLimit, + kPendingCompactionBytes, + kCFScopeWriteStallCauseEnumMax, + // End of CF-scope write stall causes + + // Beginning of DB-scope write stall causes + // + // Always keep `kWriteBufferManagerLimit` as the first stat in this section + kWriteBufferManagerLimit, + kDBScopeWriteStallCauseEnumMax, + // End of DB-scope write stall causes + + // Always add new WriteStallCause before `kNone` + kNone, +}; + +enum class WriteStallCondition { + kDelayed, + kStopped, + // Always add new WriteStallCondition before `kNormal` + kNormal, +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/unique_id.h b/librocksdb-sys/rocksdb/include/rocksdb/unique_id.h new file mode 100644 index 0000000..eb0c778 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/unique_id.h @@ -0,0 +1,55 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "rocksdb/table_properties.h" + +namespace ROCKSDB_NAMESPACE { + +// Computes a stable, universally unique 128-bit (16 binary char) identifier +// for an SST file from TableProperties. This is supported for table (SST) +// files created with RocksDB 6.24 and later. NotSupported will be returned +// for other cases. The first 16 bytes (128 bits) is of sufficient quality +// for almost all applications, and shorter prefixes are usable as a +// hash of the full unique id. +// +// Note: .c_str() is not compatible with binary char strings, so using +// .c_str() on the result will often result in information loss and very +// poor uniqueness probability. +// +// More detail: the value is *guaranteed* unique for SST files +// generated in the same process (even different DBs, RocksDB >= 6.26), +// and first 128 bits are guaranteed not "all zeros" (RocksDB >= 6.26) +// so that the "all zeros" value can be used reliably for a null ID. +// These IDs are more than sufficient for SST uniqueness within each of +// many DBs or hosts. For an extreme example assuming random IDs, consider +// 10^9 hosts each with 10^9 live SST files being replaced at 10^6/second. +// Such a service would need to run for 10 million years to see an ID +// collision among live SST files on any host. +// +// And assuming one generates many SST files in the lifetime of each process, +// the probability of ID collisions is much "better than random"; see +// https://github.com/pdillinger/unique_id +Status GetUniqueIdFromTableProperties(const TableProperties &props, + std::string *out_id); + +// Computes a 192-bit (24 binary char) stable, universally unique ID +// with an extra 64 bits of uniqueness compared to the standard ID. It is only +// appropriate to use this ID instead of the 128-bit ID if ID collisions +// between files among any hosts in a vast fleet is a problem, such as a shared +// global namespace for SST file backups. Under this criteria, the extreme +// example above would expect a global file ID collision every 4 days with +// 128-bit IDs (using some worst-case assumptions about process lifetime). +// It's 10^17 years with 192-bit IDs. +Status GetExtendedUniqueIdFromTableProperties(const TableProperties &props, + std::string *out_id); + +// Converts a binary string (unique id) to hexadecimal, with each 64 bits +// separated by '-', e.g. 6474DF650323BDF0-B48E64F3039308CA-17284B32E7F7444B +// Also works on unique id prefix. +std::string UniqueIdToHumanString(const std::string &id); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/universal_compaction.h b/librocksdb-sys/rocksdb/include/rocksdb/universal_compaction.h new file mode 100644 index 0000000..0b0a85e --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/universal_compaction.h @@ -0,0 +1,96 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +// +// Algorithm used to make a compaction request stop picking new files +// into a single compaction run +// +enum CompactionStopStyle { + kCompactionStopStyleSimilarSize, // pick files of similar size + kCompactionStopStyleTotalSize // total size of picked files > next file +}; + +class CompactionOptionsUniversal { + public: + // Percentage flexibility while comparing file size. If the candidate file(s) + // size is 1% smaller than the next file's size, then include next file into + // this candidate set. // Default: 1 + unsigned int size_ratio; + + // The minimum number of files in a single compaction run. Default: 2 + unsigned int min_merge_width; + + // The maximum number of files in a single compaction run. Default: UINT_MAX + unsigned int max_merge_width; + + // The size amplification is defined as the amount (in percentage) of + // additional storage needed to store a single byte of data in the database. + // For example, a size amplification of 2% means that a database that + // contains 100 bytes of user-data may occupy up to 102 bytes of + // physical storage. By this definition, a fully compacted database has + // a size amplification of 0%. Rocksdb uses the following heuristic + // to calculate size amplification: it assumes that all files excluding + // the earliest file contribute to the size amplification. + // Default: 200, which means that a 100 byte database could require up to + // 300 bytes of storage. + unsigned int max_size_amplification_percent; + + // If this option is set to be -1 (the default value), all the output files + // will follow compression type specified. + // + // If this option is not negative, we will try to make sure compressed + // size is just above this value. In normal cases, at least this percentage + // of data will be compressed. + // When we are compacting to a new file, here is the criteria whether + // it needs to be compressed: assuming here are the list of files sorted + // by generation time: + // A1...An B1...Bm C1...Ct + // where A1 is the newest and Ct is the oldest, and we are going to compact + // B1...Bm, we calculate the total size of all the files as total_size, as + // well as the total size of C1...Ct as total_C, the compaction output file + // will be compressed iff + // total_C / total_size < this percentage + // Default: -1 + int compression_size_percent; + + // The algorithm used to stop picking files into a single compaction run + // Default: kCompactionStopStyleTotalSize + CompactionStopStyle stop_style; + + // Option to optimize the universal multi level compaction by enabling + // trivial move for non overlapping files. + // Default: false + bool allow_trivial_move; + + // EXPERIMENTAL + // If true, try to limit compaction size under max_compaction_bytes. + // This might cause higher write amplification, but can prevent some + // problem caused by large compactions. + // Default: false + bool incremental; + + // Default set of parameters + CompactionOptionsUniversal() + : size_ratio(1), + min_merge_width(2), + max_merge_width(UINT_MAX), + max_size_amplification_percent(200), + compression_size_percent(-1), + stop_style(kCompactionStopStyleTotalSize), + allow_trivial_move(false), + incremental(false) {} +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/agg_merge.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/agg_merge.h new file mode 100644 index 0000000..4e21082 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/agg_merge.h @@ -0,0 +1,138 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include + +#include "rocksdb/merge_operator.h" +#include "rocksdb/slice.h" + +namespace ROCKSDB_NAMESPACE { +// The feature is still in development so the encoding format is subject +// to change. +// +// Aggregation Merge Operator is a merge operator that allows users to +// aggregate merge operands of different keys with different registered +// aggregation functions. The aggregation can also change for the same +// key if the functions store the data in the same format. +// The target application highly overlaps with merge operator in general +// but we try to provide a better interface so that users are more likely +// to use pre-implemented plug-in functions and connect with existing +// third-party aggregation functions (such as those from SQL engines). +// In this case, the need for users to write customized C++ plug-in code +// is reduced. +// If the idea proves to useful, we might consider to move it to be +// a core functionality of RocksDB, and reduce the support of merge +// operators. +// +// Users can implement aggregation functions by implementing abstract +// class Aggregator, and register it using AddAggregator(). +// The merge operator can be retrieved from GetAggMergeOperator() and +// it is a singleton. +// +// Users can push values to be updated with a merge operand encoded with +// registered function name and payload using EncodeAggFuncAndPayload(), +// and the merge operator will invoke the aggregation function. +// An example: +// +// // Assume class ExampleSumAggregator is implemented to do simple sum. +// AddAggregator("sum", std::make_unique()); +// std::shared_ptr mp_guard = CreateAggMergeOperator(); +// options.merge_operator = mp_guard.get(); +// ...... // Creating DB +// +// +// std::string encoded_value; +// s = EncodeAggFuncAndPayload(kUnamedFuncName, "200", encoded_value); +// assert(s.ok()); +// db->Put(WriteOptions(), "foo", encoded_value); +// s = EncodeAggFuncAndPayload("sum", "200", encoded_value); +// assert(s.ok()); +// db->Merge(WriteOptions(), "foo", encoded_value); +// s = EncodeAggFuncAndPayload("sum", "200", encoded_value); +// assert(s.ok()); +// db->Merge(WriteOptions(), "foo", encoded_value); +// +// std::string value; +// Status s = db->Get(ReadOptions, "foo", &value); +// assert(s.ok()); +// Slice func, aggregated_value; +// assert(ExtractAggFuncAndValue(value, func, aggregated_value)); +// assert(func == "sum"); +// assert(aggregated_value == "600"); +// +// +// DB::Put() can also be used to add a payloadin the same way as Merge(). +// +// kUnamedFuncName can be used as a placeholder function name. This will +// be aggregated with merge operands inserted later based on function +// name given there. +// +// If the aggregation function is not registered or there is an error +// returned by aggregation function, the result will be encoded with a fake +// aggregation function kErrorFuncName, with each merge operands to be encoded +// into a list that can be extracted using ExtractList(); +// +// If users add a merge operand using a different aggregation function from +// the previous one, the merge operands for the previous one is aggregated +// and the payload part of the result is treated as the first payload of +// the items for the new aggregation function. For example, users can +// Merge("plus, 1"), merge("plus 2"), merge("minus 3") and the aggregation +// result would be "minus 0". +// + +// A class used to aggregate data per key/value. The plug-in function is +// implemented and registered using AddAggregator(). And then use it +// with merge operator created using CreateAggMergeOperator(). +class Aggregator { + public: + virtual ~Aggregator() {} + // The input list is in reverse insertion order, with values[0] to be + // the one inserted last and values.back() to be the one inserted first. + // The oldest one might be from Get(). + // Return whether aggregation succeeded. False for aggregation error. + virtual bool Aggregate(const std::vector& values, + std::string& result) const = 0; + + // True if a partial aggregation should be invoked. Some aggregators + // might opt to skip partial aggregation if possible. + virtual bool DoPartialAggregate() const { return true; } +}; + +// The function adds aggregation plugin by function name. It is used +// by all the aggregation operator created using CreateAggMergeOperator(). +// It's currently not thread safe to run concurrently with the aggregation +// merge operator. It is recommended that all the aggregation function +// is added before calling CreateAggMergeOperator(). +Status AddAggregator(const std::string& function_name, + std::unique_ptr&& agg); + +// Get the singleton instance of merge operator for aggregation. +// Always the same one is returned with a shared_ptr is hold as a +// static variable by the function. +// This is done so because options.merge_operator is shared_ptr. +std::shared_ptr GetAggMergeOperator(); + +// Encode aggregation function and payload that can be consumed by aggregation +// merge operator. +Status EncodeAggFuncAndPayload(const Slice& function_name, const Slice& payload, + std::string& output); +// Helper function to extract aggregation function name and payload. +// Return false if it fails to decode. +bool ExtractAggFuncAndValue(const Slice& op, Slice& func, Slice& value); + +// Extract encoded list. This can be used to extract error merge operands when +// the returned function name is kErrorFuncName. +bool ExtractList(const Slice& encoded_list, std::vector& decoded_list); + +// Special function name that allows it to be merged to subsequent type. +extern const std::string kUnnamedFuncName; + +// Special error function name reserved for merging or aggregation error. +extern const std::string kErrorFuncName; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/backup_engine.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/backup_engine.h new file mode 100644 index 0000000..204d12d --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/backup_engine.h @@ -0,0 +1,689 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "rocksdb/env.h" +#include "rocksdb/io_status.h" +#include "rocksdb/metadata.h" +#include "rocksdb/options.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { +class BackupEngineReadOnlyBase; +class BackupEngine; + +// The default DB file checksum function name. +constexpr char kDbFileChecksumFuncName[] = "FileChecksumCrc32c"; +// The default BackupEngine file checksum function name. +constexpr char kBackupFileChecksumFuncName[] = "crc32c"; + +struct BackupEngineOptions { + // Where to keep the backup files. Has to be different than dbname_ + // Best to set this to dbname_ + "/backups" + // Required + std::string backup_dir; + + // Backup Env object. It will be used for backup file I/O. If it's + // nullptr, backups will be written out using DBs Env. If it's + // non-nullptr, backup's I/O will be performed using this object. + // Default: nullptr + Env* backup_env; + + // share_table_files supports table and blob files. + // + // If share_table_files == true, the backup directory will share table and + // blob files among backups, to save space among backups of the same DB and to + // enable incremental backups by only copying new files. + // If share_table_files == false, each backup will be on its own and will not + // share any data with other backups. + // + // default: true + bool share_table_files; + + // Backup info and error messages will be written to info_log + // if non-nullptr. + // Default: nullptr + Logger* info_log; + + // If sync == true, we can guarantee you'll get consistent backup and + // restore even on a machine crash/reboot. Backup and restore processes are + // slower with sync enabled. If sync == false, we can only guarantee that + // other previously synced backups and restores are not modified while + // creating a new one. + // Default: true + bool sync; + + // If true, it will delete whatever backups there are already + // Default: false + bool destroy_old_data; + + // If false, we won't backup log files. This option can be useful for backing + // up in-memory databases where log file are persisted, but table files are in + // memory. + // Default: true + bool backup_log_files; + + // Max bytes that can be transferred in a second during backup. + // If 0, go as fast as you can + // This limit only applies to writes. To also limit reads, + // a rate limiter able to also limit reads (e.g, its mode = kAllIo) + // have to be passed in through the option "backup_rate_limiter" + // Default: 0 + uint64_t backup_rate_limit; + + // Backup rate limiter. Used to control transfer speed for backup. If this is + // not null, backup_rate_limit is ignored. + // Default: nullptr + std::shared_ptr backup_rate_limiter{nullptr}; + + // Max bytes that can be transferred in a second during restore. + // If 0, go as fast as you can + // This limit only applies to writes. To also limit reads, + // a rate limiter able to also limit reads (e.g, its mode = kAllIo) + // have to be passed in through the option "restore_rate_limiter" + // Default: 0 + uint64_t restore_rate_limit; + + // Restore rate limiter. Used to control transfer speed during restore. If + // this is not null, restore_rate_limit is ignored. + // Default: nullptr + std::shared_ptr restore_rate_limiter{nullptr}; + + // share_files_with_checksum supports table and blob files. + // + // Only used if share_table_files is set to true. Setting to false is + // DEPRECATED and potentially dangerous because in that case BackupEngine + // can lose data if backing up databases with distinct or divergent + // history, for example if restoring from a backup other than the latest, + // writing to the DB, and creating another backup. Setting to true (default) + // prevents these issues by ensuring that different table files (SSTs) and + // blob files with the same number are treated as distinct. See + // share_files_with_checksum_naming and ShareFilesNaming. + // + // Default: true + bool share_files_with_checksum; + + // Up to this many background threads will copy files for CreateNewBackup() + // and RestoreDBFromBackup() + // Default: 1 + int max_background_operations; + + // During backup user can get callback every time next + // callback_trigger_interval_size bytes being copied. + // Default: 4194304 + uint64_t callback_trigger_interval_size; + + // For BackupEngineReadOnly, Open() will open at most this many of the + // latest non-corrupted backups. + // + // Note: this setting is ignored (behaves like INT_MAX) for any kind of + // writable BackupEngine because it would inhibit accounting for shared + // files for proper backup deletion, including purging any incompletely + // created backups on creation of a new backup. + // + // Default: INT_MAX + int max_valid_backups_to_open; + + // ShareFilesNaming describes possible naming schemes for backup + // table and blob file names when they are stored in the + // shared_checksum directory (i.e., both share_table_files and + // share_files_with_checksum are true). + enum ShareFilesNaming : uint32_t { + // Backup blob filenames are __.blob and + // backup SST filenames are __.sst + // where is an unsigned decimal integer. This is the + // original/legacy naming scheme for share_files_with_checksum, + // with two problems: + // * At massive scale, collisions on this triple with different file + // contents is plausible. + // * Determining the name to use requires computing the checksum, + // so generally requires reading the whole file even if the file + // is already backed up. + // + // ** ONLY RECOMMENDED FOR PRESERVING OLD BEHAVIOR ** + kLegacyCrc32cAndFileSize = 1U, + + // Backup SST filenames are _s.sst. This + // pair of values should be very strongly unique for a given SST file + // and easily determined before computing a checksum. The 's' indicates + // the value is a DB session id, not a checksum. + // + // Exceptions: + // * For blob files, kLegacyCrc32cAndFileSize is used as currently + // db_session_id is not supported by the blob file format. + // * For old SST files without a DB session id, kLegacyCrc32cAndFileSize + // will be used instead, matching the names assigned by RocksDB versions + // not supporting the newer naming scheme. + // * See also flags below. + kUseDbSessionId = 2U, + + kMaskNoNamingFlags = 0xffffU, + + // If not already part of the naming scheme, insert + // _ + // before .sst and .blob in the name. In case of user code actually parsing + // the last _ before the .sst and .blob as the file size, this + // preserves that feature of kLegacyCrc32cAndFileSize. In other words, this + // option makes official that unofficial feature of the backup metadata. + // + // We do not consider SST and blob file sizes to have sufficient entropy to + // contribute significantly to naming uniqueness. + kFlagIncludeFileSize = 1U << 31, + + kMaskNamingFlags = ~kMaskNoNamingFlags, + }; + + // Naming option for share_files_with_checksum table and blob files. See + // ShareFilesNaming for details. + // + // Modifying this option cannot introduce a downgrade compatibility issue + // because RocksDB can read, restore, and delete backups using different file + // names, and it's OK for a backup directory to use a mixture of table and + // blob files naming schemes. + // + // However, modifying this option and saving more backups to the same + // directory can lead to the same file getting saved again to that + // directory, under the new shared name in addition to the old shared + // name. + // + // Default: kUseDbSessionId | kFlagIncludeFileSize + // + // Note: This option comes into effect only if both share_files_with_checksum + // and share_table_files are true. + ShareFilesNaming share_files_with_checksum_naming; + + // Major schema version to use when writing backup meta files + // 1 (default) - compatible with very old versions of RocksDB. + // 2 - can be read by RocksDB versions >= 6.19.0. Minimum schema version for + // * (Experimental) saving and restoring file temperature metadata + int schema_version = 1; + + // (Experimental - subject to change or removal) When taking a backup and + // saving file temperature info (minimum schema_version is 2), there are + // two potential sources of truth for the placement of files into temperature + // tiers: (a) the current file temperature reported by the FileSystem or + // (b) the expected file temperature recorded in DB manifest. When this + // option is false (default), (b) overrides (a) if both are not UNKNOWN. + // When true, (a) overrides (b) if both are not UNKNOWN. Regardless of this + // setting, a known temperature overrides UNKNOWN. + bool current_temperatures_override_manifest = false; + + void Dump(Logger* logger) const; + + explicit BackupEngineOptions( + const std::string& _backup_dir, Env* _backup_env = nullptr, + bool _share_table_files = true, Logger* _info_log = nullptr, + bool _sync = true, bool _destroy_old_data = false, + bool _backup_log_files = true, uint64_t _backup_rate_limit = 0, + uint64_t _restore_rate_limit = 0, int _max_background_operations = 1, + uint64_t _callback_trigger_interval_size = 4 * 1024 * 1024, + int _max_valid_backups_to_open = INT_MAX, + ShareFilesNaming _share_files_with_checksum_naming = + static_cast(kUseDbSessionId | kFlagIncludeFileSize)) + : backup_dir(_backup_dir), + backup_env(_backup_env), + share_table_files(_share_table_files), + info_log(_info_log), + sync(_sync), + destroy_old_data(_destroy_old_data), + backup_log_files(_backup_log_files), + backup_rate_limit(_backup_rate_limit), + restore_rate_limit(_restore_rate_limit), + share_files_with_checksum(true), + max_background_operations(_max_background_operations), + callback_trigger_interval_size(_callback_trigger_interval_size), + max_valid_backups_to_open(_max_valid_backups_to_open), + share_files_with_checksum_naming(_share_files_with_checksum_naming) { + assert(share_table_files || !share_files_with_checksum); + assert((share_files_with_checksum_naming & kMaskNoNamingFlags) != 0); + } +}; + +inline BackupEngineOptions::ShareFilesNaming operator&( + BackupEngineOptions::ShareFilesNaming lhs, + BackupEngineOptions::ShareFilesNaming rhs) { + uint32_t l = static_cast(lhs); + uint32_t r = static_cast(rhs); + assert(r == BackupEngineOptions::kMaskNoNamingFlags || + (r & BackupEngineOptions::kMaskNoNamingFlags) == 0); + return static_cast(l & r); +} + +inline BackupEngineOptions::ShareFilesNaming operator|( + BackupEngineOptions::ShareFilesNaming lhs, + BackupEngineOptions::ShareFilesNaming rhs) { + uint32_t l = static_cast(lhs); + uint32_t r = static_cast(rhs); + assert((r & BackupEngineOptions::kMaskNoNamingFlags) == 0); + return static_cast(l | r); +} + +// Identifying information about a backup shared file that is (or might be) +// excluded from a backup using exclude_files_callback. +struct BackupExcludedFileInfo { + explicit BackupExcludedFileInfo(const std::string& _relative_file) + : relative_file(_relative_file) {} + + // File name and path relative to the backup dir. + std::string relative_file; +}; + +// An auxiliary structure for exclude_files_callback +struct MaybeExcludeBackupFile { + explicit MaybeExcludeBackupFile(BackupExcludedFileInfo&& _info) + : info(std::move(_info)) {} + + // Identifying information about a backup shared file that could be excluded + const BackupExcludedFileInfo info; + + // API user sets to true if the file should be excluded from this backup + bool exclude_decision = false; +}; + +struct CreateBackupOptions { + // Flush will always trigger if 2PC is enabled. + // If write-ahead logs are disabled, set flush_before_backup=true to + // avoid losing unflushed key/value pairs from the memtable. + bool flush_before_backup = false; + + // Callback for reporting progress, based on callback_trigger_interval_size. + // + // An exception thrown from the callback will result in Status::Aborted from + // the operation. + std::function progress_callback = {}; + + // A callback that allows the API user to select files for exclusion, such + // as if the files are known to exist in an alternate backup directory. + // Only "shared" files can be excluded from backups. This is an advanced + // feature because the BackupEngine user is trusted to keep track of files + // such that the DB can be restored. + // + // Input to the callback is a [begin,end) range of sharable files live in + // the DB being backed up, and the callback implementation sets + // exclude_decision=true for files to exclude. A callback offers maximum + // flexibility, e.g. if remote files are unavailable at backup time but + // whose existence has been recorded somewhere. In case of an empty or + // no-op callback, all files are included in the backup . + // + // To restore the DB, RestoreOptions::alternate_dirs must be used to provide + // the excluded files. + // + // An exception thrown from the callback will result in Status::Aborted from + // the operation. + std::function + exclude_files_callback = {}; + + // If false, background_thread_cpu_priority is ignored. + // Otherwise, the cpu priority can be decreased, + // if you try to increase the priority, the priority will not change. + // The initial priority of the threads is CpuPriority::kNormal, + // so you can decrease to priorities lower than kNormal. + bool decrease_background_thread_cpu_priority = false; + CpuPriority background_thread_cpu_priority = CpuPriority::kNormal; +}; + +struct RestoreOptions { + // If true, restore won't overwrite the existing log files in wal_dir. It will + // also move all log files from archive directory to wal_dir. Use this option + // in combination with BackupEngineOptions::backup_log_files = false for + // persisting in-memory databases. + // Default: false + bool keep_log_files; + + // For backups that were created using exclude_files_callback, this + // option enables restoring those backups by providing BackupEngines on + // directories known to contain the required files. + std::forward_list alternate_dirs; + + explicit RestoreOptions(bool _keep_log_files = false) + : keep_log_files(_keep_log_files) {} +}; + +using BackupID = uint32_t; + +using BackupFileInfo = FileStorageInfo; + +struct BackupInfo { + BackupID backup_id = 0U; + // Creation time, according to GetCurrentTime + int64_t timestamp = 0; + + // Total size in bytes (based on file payloads, not including filesystem + // overheads or backup meta file) + uint64_t size = 0U; + + // Number of backed up files, some of which might be shared with other + // backups. Does not include backup meta file. + uint32_t number_files = 0U; + + // Backup API user metadata + std::string app_metadata; + + // Backup file details, if requested with include_file_details=true. + // Does not include excluded_files. + std::vector file_details; + + // Identifying information about shared files that were excluded from the + // created backup. See exclude_files_callback and alternate_dirs. + // This information is only provided if include_file_details=true. + std::vector excluded_files; + + // DB "name" (a directory in the backup_env) for opening this backup as a + // read-only DB. This should also be used as the DBOptions::wal_dir, such + // as by default setting wal_dir="". See also env_for_open. + // This field is only set if include_file_details=true + std::string name_for_open; + + // An Env(+FileSystem) for opening this backup as a read-only DB, with + // DB::OpenForReadOnly or similar. This field is only set if + // include_file_details=true. (The FileSystem in this Env takes care + // of making shared backup files openable from the `name_for_open` DB + // directory.) See also name_for_open. + // + // This Env might or might not be shared with other backups. To work + // around DBOptions::env being a raw pointer, this is a shared_ptr so + // that keeping either this BackupInfo, the BackupEngine, or a copy of + // this shared_ptr alive is sufficient to keep the Env alive for use by + // a read-only DB. + std::shared_ptr env_for_open; + + BackupInfo() {} + + explicit BackupInfo(BackupID _backup_id, int64_t _timestamp, uint64_t _size, + uint32_t _number_files, const std::string& _app_metadata) + : backup_id(_backup_id), + timestamp(_timestamp), + size(_size), + number_files(_number_files), + app_metadata(_app_metadata) {} +}; + +class BackupStatistics { + public: + BackupStatistics() { + number_success_backup = 0; + number_fail_backup = 0; + } + + explicit BackupStatistics(uint32_t _number_success_backup, + uint32_t _number_fail_backup) + : number_success_backup(_number_success_backup), + number_fail_backup(_number_fail_backup) {} + + ~BackupStatistics() {} + + void IncrementNumberSuccessBackup(); + void IncrementNumberFailBackup(); + + uint32_t GetNumberSuccessBackup() const; + uint32_t GetNumberFailBackup() const; + + std::string ToString() const; + + private: + uint32_t number_success_backup; + uint32_t number_fail_backup; +}; + +// Read-only functions of a BackupEngine. (Restore writes to another directory +// not the backup directory.) See BackupEngine comments for details on +// safe concurrent operations. +class BackupEngineReadOnlyBase { + public: + virtual ~BackupEngineReadOnlyBase() {} + + // Returns info about the latest good backup in backup_info, or NotFound + // no good backup exists. + // Setting include_file_details=true provides information about each + // backed-up file in BackupInfo::file_details and more. + virtual Status GetLatestBackupInfo( + BackupInfo* backup_info, bool include_file_details = false) const = 0; + + // Returns info about a specific backup in backup_info, or NotFound + // or Corruption status if the requested backup id does not exist or is + // known corrupt. + // Setting include_file_details=true provides information about each + // backed-up file in BackupInfo::file_details and more. + virtual Status GetBackupInfo(BackupID backup_id, BackupInfo* backup_info, + bool include_file_details = false) const = 0; + + // Returns info about non-corrupt backups in backup_infos. + // Setting include_file_details=true provides information about each + // backed-up file in BackupInfo::file_details and more. + virtual void GetBackupInfo(std::vector* backup_infos, + bool include_file_details = false) const = 0; + + // Returns info about corrupt backups in corrupt_backups. + // WARNING: Any write to the BackupEngine could trigger automatic + // GarbageCollect(), which could delete files that would be needed to + // manually recover a corrupt backup or to preserve an unrecognized (e.g. + // incompatible future version) backup. + virtual void GetCorruptedBackups( + std::vector* corrupt_backup_ids) const = 0; + + // Restore to specified db_dir and wal_dir from backup_id. + virtual IOStatus RestoreDBFromBackup(const RestoreOptions& options, + BackupID backup_id, + const std::string& db_dir, + const std::string& wal_dir) const = 0; + + // keep for backward compatibility. + virtual IOStatus RestoreDBFromBackup( + BackupID backup_id, const std::string& db_dir, const std::string& wal_dir, + const RestoreOptions& options = RestoreOptions()) const { + return RestoreDBFromBackup(options, backup_id, db_dir, wal_dir); + } + + // Like RestoreDBFromBackup but restores from latest non-corrupt backup_id + virtual IOStatus RestoreDBFromLatestBackup( + const RestoreOptions& options, const std::string& db_dir, + const std::string& wal_dir) const = 0; + + // keep for backward compatibility. + virtual IOStatus RestoreDBFromLatestBackup( + const std::string& db_dir, const std::string& wal_dir, + const RestoreOptions& options = RestoreOptions()) const { + return RestoreDBFromLatestBackup(options, db_dir, wal_dir); + } + + // If verify_with_checksum is true, this function + // inspects the current checksums and file sizes of backup files to see if + // they match our expectation. + // + // If verify_with_checksum is false, this function + // checks that each file exists and that the size of the file matches our + // expectation. It does not check file checksum. + // + // If this BackupEngine created the backup, it compares the files' current + // sizes (and current checksum) against the number of bytes written to + // them (and the checksum calculated) during creation. + // Otherwise, it compares the files' current sizes (and checksums) against + // their sizes (and checksums) when the BackupEngine was opened. + // + // Returns Status::OK() if all checks are good + virtual IOStatus VerifyBackup(BackupID backup_id, + bool verify_with_checksum = false) const = 0; + + // Internal use only + virtual BackupEngine* AsBackupEngine() = 0; +}; + +// Append-only functions of a BackupEngine. See BackupEngine comment for +// details on distinction between Append and Write operations and safe +// concurrent operations. +class BackupEngineAppendOnlyBase { + public: + virtual ~BackupEngineAppendOnlyBase() {} + + // same as CreateNewBackup, but stores extra application metadata. + virtual IOStatus CreateNewBackupWithMetadata( + const CreateBackupOptions& options, DB* db, + const std::string& app_metadata, BackupID* new_backup_id = nullptr) = 0; + + // keep here for backward compatibility. + virtual IOStatus CreateNewBackupWithMetadata( + DB* db, const std::string& app_metadata, bool flush_before_backup = false, + std::function progress_callback = []() {}) { + CreateBackupOptions options; + options.flush_before_backup = flush_before_backup; + options.progress_callback = progress_callback; + return CreateNewBackupWithMetadata(options, db, app_metadata); + } + + // Captures the state of the database by creating a new (latest) backup. + // On success (OK status), the BackupID of the new backup is saved to + // *new_backup_id when not nullptr. + // NOTE: db_paths and cf_paths are not supported for creating backups, + // and NotSupported will be returned when the DB (without WALs) uses more + // than one directory. + virtual IOStatus CreateNewBackup(const CreateBackupOptions& options, DB* db, + BackupID* new_backup_id = nullptr) { + return CreateNewBackupWithMetadata(options, db, "", new_backup_id); + } + + // keep here for backward compatibility. + virtual IOStatus CreateNewBackup( + DB* db, bool flush_before_backup = false, + std::function progress_callback = []() {}) { + CreateBackupOptions options; + options.flush_before_backup = flush_before_backup; + options.progress_callback = progress_callback; + return CreateNewBackup(options, db); + } + + // Call this from another thread if you want to stop the backup + // that is currently happening. It will return immediately, will + // not wait for the backup to stop. + // The backup will stop ASAP and the call to CreateNewBackup will + // return Status::Incomplete(). It will not clean up after itself, but + // the state will remain consistent. The state will be cleaned up the + // next time you call CreateNewBackup or GarbageCollect. + virtual void StopBackup() = 0; + + // Will delete any files left over from incomplete creation or deletion of + // a backup. This is not normally needed as those operations also clean up + // after prior incomplete calls to the same kind of operation (create or + // delete). This does not delete corrupt backups but can delete files that + // would be needed to manually recover a corrupt backup or to preserve an + // unrecognized (e.g. incompatible future version) backup. + // NOTE: This is not designed to delete arbitrary files added to the backup + // directory outside of BackupEngine, and clean-up is always subject to + // permissions on and availability of the underlying filesystem. + // NOTE2: For concurrency and interference purposes (see BackupEngine + // comment), GarbageCollect (GC) is like other Append operations, even + // though it seems different. Although GC can delete physical data, it does + // not delete any logical data read by Read operations. GC can interfere + // with Append or Write operations in another BackupEngine on the same + // backup_dir, because temporary files will be treated as obsolete and + // deleted. + virtual IOStatus GarbageCollect() = 0; +}; + +// A backup engine for organizing and managing backups. +// This class is not user-extensible. +// +// This class declaration adds "Write" operations in addition to the +// operations from BackupEngineAppendOnlyBase and BackupEngineReadOnlyBase. +// +// # Concurrency between threads on the same BackupEngine* object +// +// As of version 6.20, BackupEngine* operations are generally thread-safe, +// using a read-write lock, though single-thread operation is still +// recommended to avoid TOCTOU bugs. Specifically, particular kinds of +// concurrent operations behave like this: +// +// op1\op2| Read | Append | Write +// -------|-------|--------|-------- +// Read | conc | block | block +// Append | block | block | block +// Write | block | block | block +// +// conc = operations safely proceed concurrently +// block = one of the operations safely blocks until the other completes. +// There is generally no guarantee as to which completes first. +// +// StopBackup is the only operation that affects an ongoing operation. +// +// # Interleaving operations between BackupEngine* objects open on the +// same backup_dir +// +// It is recommended only to have one BackupEngine* object open for a given +// backup_dir, but it is possible to mix / interleave some operations +// (regardless of whether they are concurrent) with these caveats: +// +// op1\op2| Open | Read | Append | Write +// -------|--------|--------|--------|-------- +// Open | conc | conc | atomic | unspec +// Read | conc | conc | old | unspec +// Append | atomic | old | unspec | unspec +// Write | unspec | unspec | unspec | unspec +// +// Special case: Open with destroy_old_data=true is really a Write +// +// conc = operations safely proceed, concurrently when applicable +// atomic = operations are effectively atomic; if a concurrent Append +// operation has not completed at some key point during Open, the +// opened BackupEngine* will never see the result of the Append op. +// old = Read operations do not include any state changes from other +// BackupEngine* objects; they return the state at their Open time. +// unspec = Behavior is unspecified, including possibly trashing the +// backup_dir, but is "memory safe" (no C++ undefined behavior) +// +class BackupEngine : public BackupEngineReadOnlyBase, + public BackupEngineAppendOnlyBase { + public: + virtual ~BackupEngine() {} + + // BackupEngineOptions have to be the same as the ones used in previous + // BackupEngines for the same backup directory. + static IOStatus Open(const BackupEngineOptions& options, Env* db_env, + BackupEngine** backup_engine_ptr); + + // keep for backward compatibility. + static IOStatus Open(Env* db_env, const BackupEngineOptions& options, + BackupEngine** backup_engine_ptr) { + return BackupEngine::Open(options, db_env, backup_engine_ptr); + } + + // Deletes old backups, keeping latest num_backups_to_keep alive. + // See also DeleteBackup. + virtual IOStatus PurgeOldBackups(uint32_t num_backups_to_keep) = 0; + + // Deletes a specific backup. If this operation (or PurgeOldBackups) + // is not completed due to crash, power failure, etc. the state + // will be cleaned up the next time you call DeleteBackup, + // PurgeOldBackups, or GarbageCollect. + virtual IOStatus DeleteBackup(BackupID backup_id) = 0; +}; + +// A variant of BackupEngine that only allows "Read" operations. See +// BackupEngine comment for details. This class is not user-extensible. +class BackupEngineReadOnly : public BackupEngineReadOnlyBase { + public: + virtual ~BackupEngineReadOnly() {} + + static IOStatus Open(const BackupEngineOptions& options, Env* db_env, + BackupEngineReadOnly** backup_engine_ptr); + // keep for backward compatibility. + static IOStatus Open(Env* db_env, const BackupEngineOptions& options, + BackupEngineReadOnly** backup_engine_ptr) { + return BackupEngineReadOnly::Open(options, db_env, backup_engine_ptr); + } +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/cache_dump_load.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/cache_dump_load.h new file mode 100644 index 0000000..8b91bb7 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/cache_dump_load.h @@ -0,0 +1,140 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "rocksdb/cache.h" +#include "rocksdb/env.h" +#include "rocksdb/file_system.h" +#include "rocksdb/io_status.h" +#include "rocksdb/secondary_cache.h" +#include "rocksdb/table.h" + +namespace ROCKSDB_NAMESPACE { + +// The classes and functions in this header file is used for dumping out the +// blocks in a block cache, storing or transfering the blocks to another +// destination host, and load these blocks to the secondary cache at destination +// host. +// NOTE that: The classes, functions, and data structures are EXPERIMENTAL! They +// my be changed in the future when the development continues. + +// The major and minor version number of the data format to be stored/trandfered +// via CacheDumpWriter and read out via CacheDumpReader +static const int kCacheDumpMajorVersion = 0; +static const int kCacheDumpMinorVersion = 1; + +// NOTE that: this class is EXPERIMENTAL! May be changed in the future! +// This is an abstract class to write or transfer the data that is created by +// CacheDumper. We pack one block with its block type, dump time, block key in +// the block cache, block len, block crc32c checksum and block itself as a unit +// and it is stored via WritePacket. Before we call WritePacket, we must call +// WriteMetadata once, which stores the sequence number, block unit checksum, +// and block unit size. +// We provide file based CacheDumpWriter to store the metadata and its package +// sequentially in a file as the defualt implementation. Users can implement +// their own CacheDumpWriter to store/transfer the data. For example, user can +// create a subclass which transfer the metadata and package on the fly. +class CacheDumpWriter { + public: + virtual ~CacheDumpWriter() = default; + + // Called ONCE before the calls to WritePacket + virtual IOStatus WriteMetadata(const Slice& metadata) = 0; + virtual IOStatus WritePacket(const Slice& data) = 0; + virtual IOStatus Close() = 0; +}; + +// NOTE that: this class is EXPERIMENTAL! May be changed in the future! +// This is an abstract class to read or receive the data that is stored +// or transfered by CacheDumpWriter. Note that, ReadMetadata must be called +// once before we call a ReadPacket. +class CacheDumpReader { + public: + virtual ~CacheDumpReader() = default; + // Called ONCE before the calls to ReadPacket + virtual IOStatus ReadMetadata(std::string* metadata) = 0; + // Sets data to empty string on EOF + virtual IOStatus ReadPacket(std::string* data) = 0; + // (Close not needed) +}; + +// CacheDumpOptions is the option for CacheDumper and CacheDumpedLoader. Any +// dump or load process related control variables can be added here. +struct CacheDumpOptions { + SystemClock* clock; +}; + +// NOTE that: this class is EXPERIMENTAL! May be changed in the future! +// This the class to dump out the block in the block cache, store/transfer them +// via CacheDumpWriter. In order to dump out the blocks belonging to a certain +// DB or a list of DB (block cache can be shared by many DB), user needs to call +// SetDumpFilter to specify a list of DB to filter out the blocks that do not +// belong to those DB. +// A typical use case is: when we migrate a DB instance from host A to host B. +// We need to reopen the DB at host B after all the files are copied to host B. +// At this moment, the block cache at host B does not have any block from this +// migrated DB. Therefore, the read performance can be low due to cache warm up. +// By using CacheDumper before we shut down the DB at host A and using +// CacheDumpedLoader at host B before we reopen the DB, we can warmup the cache +// ahead. This function can be used in other use cases also. +class CacheDumper { + public: + virtual ~CacheDumper() = default; + // Only dump the blocks in the block cache that belong to the DBs in this list + virtual Status SetDumpFilter(std::vector db_list) { + (void)db_list; + return Status::NotSupported("SetDumpFilter is not supported"); + } + // The main function to dump out all the blocks that satisfy the filter + // condition from block cache to a certain CacheDumpWriter in one shot. This + // process may take some time. + virtual IOStatus DumpCacheEntriesToWriter() { + return IOStatus::NotSupported("DumpCacheEntriesToWriter is not supported"); + } +}; + +// NOTE that: this class is EXPERIMENTAL! May be changed in the future! +// This is the class to load the dumped blocks to the destination cache. For now +// we only load the blocks to the SecondaryCache. In the future, we may plan to +// support loading to the block cache. +class CacheDumpedLoader { + public: + virtual ~CacheDumpedLoader() = default; + virtual IOStatus RestoreCacheEntriesToSecondaryCache() { + return IOStatus::NotSupported( + "RestoreCacheEntriesToSecondaryCache is not supported"); + } +}; + +// Get the writer which stores all the metadata and data sequentially to a file +IOStatus NewToFileCacheDumpWriter(const std::shared_ptr& fs, + const FileOptions& file_opts, + const std::string& file_name, + std::unique_ptr* writer); + +// Get the reader which read out the metadata and data sequentially from a file +IOStatus NewFromFileCacheDumpReader(const std::shared_ptr& fs, + const FileOptions& file_opts, + const std::string& file_name, + std::unique_ptr* reader); + +// Get the default cache dumper +Status NewDefaultCacheDumper(const CacheDumpOptions& dump_options, + const std::shared_ptr& cache, + std::unique_ptr&& writer, + std::unique_ptr* cache_dumper); + +// Get the default cache dump loader +Status NewDefaultCacheDumpedLoader( + const CacheDumpOptions& dump_options, + const BlockBasedTableOptions& toptions, + const std::shared_ptr& secondary_cache, + std::unique_ptr&& reader, + std::unique_ptr* cache_dump_loader); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/checkpoint.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/checkpoint.h new file mode 100644 index 0000000..6509f38 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/checkpoint.h @@ -0,0 +1,63 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// A checkpoint is an openable snapshot of a database at a point in time. + +#pragma once + +#include +#include + +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +class DB; +class ColumnFamilyHandle; +struct LiveFileMetaData; +struct ExportImportFilesMetaData; + +class Checkpoint { + public: + // Creates a Checkpoint object to be used for creating openable snapshots + static Status Create(DB* db, Checkpoint** checkpoint_ptr); + + // Builds an openable snapshot of RocksDB. checkpoint_dir should contain an + // absolute path. The specified directory should not exist, since it will be + // created by the API. + // When a checkpoint is created, + // (1) SST and blob files are hard linked if the output directory is on the + // same filesystem as the database, and copied otherwise. + // (2) other required files (like MANIFEST) are always copied. + // log_size_for_flush: if the total log file size is equal or larger than + // this value, then a flush is triggered for all the column families. The + // default value is 0, which means flush is always triggered. If you move + // away from the default, the checkpoint may not contain up-to-date data + // if WAL writing is not always enabled. + // Flush will always trigger if it is 2PC. + // sequence_number_ptr: if it is not nullptr, the value it points to will be + // set to a sequence number guaranteed to be part of the DB, not necessarily + // the latest. The default value of this parameter is nullptr. + // NOTE: db_paths and cf_paths are not supported for creating checkpoints + // and NotSupported will be returned when the DB (without WALs) uses more + // than one directory. + virtual Status CreateCheckpoint(const std::string& checkpoint_dir, + uint64_t log_size_for_flush = 0, + uint64_t* sequence_number_ptr = nullptr); + + // Exports all live SST files of a specified Column Family onto export_dir, + // returning SST files information in metadata. + // - SST files will be created as hard links when the directory specified + // is in the same partition as the db directory, copied otherwise. + // - export_dir should not already exist and will be created by this API. + // - Always triggers a flush. + virtual Status ExportColumnFamily(ColumnFamilyHandle* handle, + const std::string& export_dir, + ExportImportFilesMetaData** metadata); + + virtual ~Checkpoint() {} +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/convenience.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/convenience.h new file mode 100644 index 0000000..f61afd6 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/convenience.h @@ -0,0 +1,10 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +// This file was moved to rocksdb/convenience.h" + +#include "rocksdb/convenience.h" diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/customizable_util.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/customizable_util.h new file mode 100644 index 0000000..adf2540 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/customizable_util.h @@ -0,0 +1,322 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// The methods in this file are used to instantiate new Customizable +// instances of objects. These methods are most typically used by +// the "CreateFromString" method of a customizable class. +// If not developing a new Type of customizable class, you probably +// do not need the methods in this file. +// +// See https://github.com/facebook/rocksdb/wiki/RocksDB-Configurable-Objects +// for more information on how to develop and use customizable objects + +#pragma once +#include +#include + +#include "options/configurable_helper.h" +#include "rocksdb/convenience.h" +#include "rocksdb/customizable.h" +#include "rocksdb/status.h" +#include "rocksdb/utilities/object_registry.h" + +namespace ROCKSDB_NAMESPACE { +// Creates a new shared customizable instance object based on the +// input parameters using the object registry. +// +// The id parameter specifies the instance class of the object to create. +// The opt_map parameter specifies the configuration of the new instance. +// +// The config_options parameter controls the process and how errors are +// returned. If ignore_unknown_options=true, unknown values are ignored during +// the configuration. If ignore_unsupported_options=true, unknown instance types +// are ignored. If invoke_prepare_options=true, the resulting instance will be +// initialized (via PrepareOptions) +// +// @param config_options Controls how the instance is created and errors are +// handled +// @param id The identifier of the new object being created. This string +// will be used by the object registry to locate the appropriate object to +// create. +// @param opt_map Optional name-value pairs of properties to set for the newly +// created object +// @param result The newly created and configured instance. +template +static Status NewSharedObject( + const ConfigOptions& config_options, const std::string& id, + const std::unordered_map& opt_map, + std::shared_ptr* result) { + if (!id.empty()) { + Status status; + status = config_options.registry->NewSharedObject(id, result); + if (config_options.ignore_unsupported_options && status.IsNotSupported()) { + status = Status::OK(); + } else if (status.ok()) { + status = Customizable::ConfigureNewObject(config_options, result->get(), + opt_map); + } + return status; + } else if (opt_map.empty()) { + // There was no ID and no map (everything empty), so reset/clear the result + result->reset(); + return Status::OK(); + } else { + return Status::NotSupported("Cannot reset object "); + } +} + +// Creates a new managed customizable instance object based on the +// input parameters using the object registry. Unlike "shared" objects, +// managed objects are limited to a single instance per ID. +// +// The id parameter specifies the instance class of the object to create. +// If an object with this id exists in the registry, the existing object +// will be returned. If the object does not exist, a new one will be created. +// +// The opt_map parameter specifies the configuration of the new instance. +// If the object already exists, the existing object is returned "as is" and +// this parameter is ignored. +// +// The config_options parameter controls the process and how errors are +// returned. If ignore_unknown_options=true, unknown values are ignored during +// the configuration. If ignore_unsupported_options=true, unknown instance types +// are ignored. If invoke_prepare_options=true, the resulting instance will be +// initialized (via PrepareOptions) +// +// @param config_options Controls how the instance is created and errors are +// handled +// @param id The identifier of the object. This string +// will be used by the object registry to locate the appropriate object to +// create or return. +// @param opt_map Optional name-value pairs of properties to set for the newly +// created object +// @param result The managed instance. +template +static Status NewManagedObject( + const ConfigOptions& config_options, const std::string& id, + const std::unordered_map& opt_map, + std::shared_ptr* result) { + Status status; + if (!id.empty()) { + status = config_options.registry->GetOrCreateManagedObject( + id, result, [config_options, opt_map](T* object) { + return object->ConfigureFromMap(config_options, opt_map); + }); + if (config_options.ignore_unsupported_options && status.IsNotSupported()) { + return Status::OK(); + } + } else { + status = Status::NotSupported("Cannot reset object "); + } + return status; +} + +// Creates a new shared Customizable object based on the input parameters. +// This method parses the input value to determine the type of instance to +// create. If there is an existing instance (in result) and it is the same ID +// as the object being created, the existing configuration is stored and used as +// the default for the new object. +// +// The value parameter specified the instance class of the object to create. +// If it is a simple string (e.g. BlockBasedTable), then the instance will be +// created using the default settings. If the value is a set of name-value +// pairs, then the "id" value is used to determine the instance to create and +// the remaining parameters are used to configure the object. Id name-value +// pairs are specified, there should be an "id=value" pairing or an error may +// result. +// +// The config_options parameter controls the process and how errors are +// returned. If ignore_unknown_options=true, unknown values are ignored during +// the configuration. If ignore_unsupported_options=true, unknown instance types +// are ignored. If invoke_prepare_options=true, the resulting instance will be +// initialized (via PrepareOptions) +// +// @param config_options Controls how the instance is created and errors are +// handled +// @param value Either the simple name of the instance to create, or a set of +// name-value pairs to create and initailize the object +// @param result The newly created instance. +template +static Status LoadSharedObject(const ConfigOptions& config_options, + const std::string& value, + std::shared_ptr* result) { + std::string id; + std::unordered_map opt_map; + + Status status = Customizable::GetOptionsMap(config_options, result->get(), + value, &id, &opt_map); + if (!status.ok()) { // GetOptionsMap failed + return status; + } else { + return NewSharedObject(config_options, id, opt_map, result); + } +} + +// Creates a new shared Customizable object based on the input parameters. +// +// The value parameter specified the instance class of the object to create. +// If it is a simple string (e.g. BlockBasedTable), then the instance will be +// created using the default settings. If the value is a set of name-value +// pairs, then the "id" value is used to determine the instance to create and +// the remaining parameters are used to configure the object. Id name-value +// pairs are specified, there should be an "id=value" pairing or an error may +// result. +// +// The "id" field from the value (either the whole field or "id=XX") is used +// to determine the type/id of the object to return. For a given id, there +// the same instance of the object will be returned from this method (as opposed +// to LoadSharedObject which would create different objects for the same id. +// +// The config_options parameter controls the process and how errors are +// returned. If ignore_unknown_options=true, unknown values are ignored during +// the configuration. If ignore_unsupported_options=true, unknown instance types +// are ignored. If invoke_prepare_options=true, the resulting instance will be +// initialized (via PrepareOptions) +// +// @param config_options Controls how the instance is created and errors are +// handled +// @param value Either the simple name of the instance to create, or a set of +// name-value pairs to create and initailize the object +// @param result The newly created instance. +template +static Status LoadManagedObject(const ConfigOptions& config_options, + const std::string& value, + std::shared_ptr* result) { + std::string id; + std::unordered_map opt_map; + Status status = Customizable::GetOptionsMap(config_options, nullptr, value, + &id, &opt_map); + if (!status.ok()) { // GetOptionsMap failed + return status; + } else if (value.empty()) { // No Id and no options. Clear the object + *result = nullptr; + return Status::OK(); + } else { + return NewManagedObject(config_options, id, opt_map, result); + } +} + +// Creates a new unique pointer customizable instance object based on the +// input parameters using the object registry. +// @see NewSharedObject for more information on the inner workings of this +// method. +// +// @param config_options Controls how the instance is created and errors are +// handled +// @param id The identifier of the new object being created. This string +// will be used by the object registry to locate the appropriate object to +// create. +// @param opt_map Optional name-value pairs of properties to set for the newly +// created object +// @param result The newly created and configured instance. +template +static Status NewUniqueObject( + const ConfigOptions& config_options, const std::string& id, + const std::unordered_map& opt_map, + std::unique_ptr* result) { + if (!id.empty()) { + Status status; + status = config_options.registry->NewUniqueObject(id, result); + if (config_options.ignore_unsupported_options && status.IsNotSupported()) { + status = Status::OK(); + } else if (status.ok()) { + status = Customizable::ConfigureNewObject(config_options, result->get(), + opt_map); + } + return status; + } else if (opt_map.empty()) { + // There was no ID and no map (everything empty), so reset/clear the result + result->reset(); + return Status::OK(); + } else { + return Status::NotSupported("Cannot reset object "); + } +} + +// Creates a new unique customizable instance object based on the input +// parameters. +// @see LoadSharedObject for more information on the inner workings of this +// method. +// +// @param config_options Controls how the instance is created and errors are +// handled +// @param value Either the simple name of the instance to create, or a set of +// name-value pairs to create and initailize the object +// @param result The newly created instance. +template +static Status LoadUniqueObject(const ConfigOptions& config_options, + const std::string& value, + std::unique_ptr* result) { + std::string id; + std::unordered_map opt_map; + Status status = Customizable::GetOptionsMap(config_options, result->get(), + value, &id, &opt_map); + if (!status.ok()) { // GetOptionsMap failed + return status; + } else { + return NewUniqueObject(config_options, id, opt_map, result); + } +} + +// Creates a new static (raw pointer) customizable instance object based on the +// input parameters using the object registry. +// @see NewSharedObject for more information on the inner workings of this +// method. +// +// @param config_options Controls how the instance is created and errors are +// handled +// @param id The identifier of the new object being created. This string +// will be used by the object registry to locate the appropriate object to +// create. +// @param opt_map Optional name-value pairs of properties to set for the newly +// created object +// @param result The newly created and configured instance. +template +static Status NewStaticObject( + const ConfigOptions& config_options, const std::string& id, + const std::unordered_map& opt_map, T** result) { + if (!id.empty()) { + Status status; + status = config_options.registry->NewStaticObject(id, result); + if (config_options.ignore_unsupported_options && status.IsNotSupported()) { + status = Status::OK(); + } else if (status.ok()) { + status = + Customizable::ConfigureNewObject(config_options, *result, opt_map); + } + return status; + } else if (opt_map.empty()) { + // There was no ID and no map (everything empty), so reset/clear the result + *result = nullptr; + return Status::OK(); + } else { + return Status::NotSupported("Cannot reset object "); + } +} + +// Creates a new static (raw pointer) customizable instance object based on the +// input parameters. +// @see LoadSharedObject for more information on the inner workings of this +// method. +// +// @param config_options Controls how the instance is created and errors are +// handled +// @param value Either the simple name of the instance to create, or a set of +// name-value pairs to create and initailize the object +// @param result The newly created instance. +template +static Status LoadStaticObject(const ConfigOptions& config_options, + const std::string& value, T** result) { + std::string id; + std::unordered_map opt_map; + Status status = Customizable::GetOptionsMap(config_options, *result, value, + &id, &opt_map); + if (!status.ok()) { // GetOptionsMap failed + return status; + } else { + return NewStaticObject(config_options, id, opt_map, result); + } +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/db_ttl.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/db_ttl.h new file mode 100644 index 0000000..12f5cba --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/db_ttl.h @@ -0,0 +1,70 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include + +#include "rocksdb/db.h" +#include "rocksdb/utilities/stackable_db.h" + +namespace ROCKSDB_NAMESPACE { + +// Database with TTL support. +// +// USE-CASES: +// This API should be used to open the db when key-values inserted are +// meant to be removed from the db in a non-strict 'ttl' amount of time +// Therefore, this guarantees that key-values inserted will remain in the +// db for >= ttl amount of time and the db will make efforts to remove the +// key-values as soon as possible after ttl seconds of their insertion. +// +// BEHAVIOUR: +// TTL is accepted in seconds +// (int32_t)Timestamp(creation) is suffixed to values in Put internally +// Expired TTL values deleted in compaction only:(Timestamp+ttl=5 +// read_only=true opens in the usual read-only mode. Compactions will not be +// triggered(neither manual nor automatic), so no expired entries removed +// +// CONSTRAINTS: +// Not specifying/passing or non-positive TTL behaves like TTL = infinity +// +// !!!WARNING!!!: +// Calling DB::Open directly to re-open a db created by this API will get +// corrupt values(timestamp suffixed) and no ttl effect will be there +// during the second Open, so use this API consistently to open the db +// Be careful when passing ttl with a small positive value because the +// whole database may be deleted in a small amount of time + +class DBWithTTL : public StackableDB { + public: + virtual Status CreateColumnFamilyWithTtl( + const ColumnFamilyOptions& options, const std::string& column_family_name, + ColumnFamilyHandle** handle, int ttl) = 0; + + static Status Open(const Options& options, const std::string& dbname, + DBWithTTL** dbptr, int32_t ttl = 0, + bool read_only = false); + + static Status Open(const DBOptions& db_options, const std::string& dbname, + const std::vector& column_families, + std::vector* handles, + DBWithTTL** dbptr, const std::vector& ttls, + bool read_only = false); + + virtual void SetTtl(int32_t ttl) = 0; + + virtual void SetTtl(ColumnFamilyHandle* h, int32_t ttl) = 0; + + protected: + explicit DBWithTTL(DB* db) : StackableDB(db) {} +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/debug.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/debug.h new file mode 100644 index 0000000..e1fc76e --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/debug.h @@ -0,0 +1,46 @@ +// Copyright (c) 2017-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + + +#include "rocksdb/db.h" +#include "rocksdb/types.h" + +namespace ROCKSDB_NAMESPACE { + +// Data associated with a particular version of a key. A database may internally +// store multiple versions of a same user key due to snapshots, compaction not +// happening yet, etc. +struct KeyVersion { + KeyVersion() : user_key(""), value(""), sequence(0), type(0) {} + + KeyVersion(const std::string& _user_key, const std::string& _value, + SequenceNumber _sequence, int _type) + : user_key(_user_key), value(_value), sequence(_sequence), type(_type) {} + + std::string user_key; + std::string value; + SequenceNumber sequence; + int type; + std::string GetTypeName() const; +}; + +// Returns listing of all versions of keys in the provided user key range. +// The range is inclusive-inclusive, i.e., [`begin_key`, `end_key`], or +// `max_num_ikeys` has been reached. Since all those keys returned will be +// copied to memory, if the range covers too many keys, the memory usage +// may be huge. `max_num_ikeys` can be used to cap the memory usage. +// The result is inserted into the provided vector, `key_versions`. +Status GetAllKeyVersions(DB* db, Slice begin_key, Slice end_key, + size_t max_num_ikeys, + std::vector* key_versions); + +Status GetAllKeyVersions(DB* db, ColumnFamilyHandle* cfh, Slice begin_key, + Slice end_key, size_t max_num_ikeys, + std::vector* key_versions); + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/env_mirror.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/env_mirror.h new file mode 100644 index 0000000..2a12612 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/env_mirror.h @@ -0,0 +1,179 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// Copyright (c) 2015, Red Hat, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// MirrorEnv is an Env implementation that mirrors all file-related +// operations to two backing Env's (provided at construction time). +// Writes are mirrored. For read operations, we do the read from both +// backends and assert that the results match. +// +// This is useful when implementing a new Env and ensuring that the +// semantics and behavior are correct (in that they match that of an +// existing, stable Env, like the default POSIX one). + +#pragma once + + +#include +#include +#include + +#include "rocksdb/env.h" + +namespace ROCKSDB_NAMESPACE { + +class SequentialFileMirror; +class RandomAccessFileMirror; +class WritableFileMirror; + +class EnvMirror : public EnvWrapper { + Env *a_, *b_; + bool free_a_, free_b_; + + public: + EnvMirror(Env* a, Env* b, bool free_a = false, bool free_b = false) + : EnvWrapper(a), a_(a), b_(b), free_a_(free_a), free_b_(free_b) {} + ~EnvMirror() { + if (free_a_) delete a_; + if (free_b_) delete b_; + } + + Status NewSequentialFile(const std::string& f, + std::unique_ptr* r, + const EnvOptions& options) override; + Status NewRandomAccessFile(const std::string& f, + std::unique_ptr* r, + const EnvOptions& options) override; + Status NewWritableFile(const std::string& f, std::unique_ptr* r, + const EnvOptions& options) override; + Status ReuseWritableFile(const std::string& fname, + const std::string& old_fname, + std::unique_ptr* r, + const EnvOptions& options) override; + virtual Status NewDirectory(const std::string& name, + std::unique_ptr* result) override { + std::unique_ptr br; + Status as = a_->NewDirectory(name, result); + Status bs = b_->NewDirectory(name, &br); + assert(as == bs); + return as; + } + Status FileExists(const std::string& f) override { + Status as = a_->FileExists(f); + Status bs = b_->FileExists(f); + assert(as == bs); + return as; + } +#if defined(_MSC_VER) +#pragma warning(push) +// logical operation on address of string constant +#pragma warning(disable : 4130) +#endif + Status GetChildren(const std::string& dir, + std::vector* r) override { + std::vector ar, br; + Status as = a_->GetChildren(dir, &ar); + Status bs = b_->GetChildren(dir, &br); + assert(as == bs); + std::sort(ar.begin(), ar.end()); + std::sort(br.begin(), br.end()); + if (!as.ok() || ar != br) { + assert(0 == "getchildren results don't match"); + } + *r = ar; + return as; + } +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + Status DeleteFile(const std::string& f) override { + Status as = a_->DeleteFile(f); + Status bs = b_->DeleteFile(f); + assert(as == bs); + return as; + } + Status CreateDir(const std::string& d) override { + Status as = a_->CreateDir(d); + Status bs = b_->CreateDir(d); + assert(as == bs); + return as; + } + Status CreateDirIfMissing(const std::string& d) override { + Status as = a_->CreateDirIfMissing(d); + Status bs = b_->CreateDirIfMissing(d); + assert(as == bs); + return as; + } + Status DeleteDir(const std::string& d) override { + Status as = a_->DeleteDir(d); + Status bs = b_->DeleteDir(d); + assert(as == bs); + return as; + } + Status GetFileSize(const std::string& f, uint64_t* s) override { + uint64_t asize, bsize; + Status as = a_->GetFileSize(f, &asize); + Status bs = b_->GetFileSize(f, &bsize); + assert(as == bs); + assert(!as.ok() || asize == bsize); + *s = asize; + return as; + } + + Status GetFileModificationTime(const std::string& fname, + uint64_t* file_mtime) override { + uint64_t amtime, bmtime; + Status as = a_->GetFileModificationTime(fname, &amtime); + Status bs = b_->GetFileModificationTime(fname, &bmtime); + assert(as == bs); + assert(!as.ok() || amtime - bmtime < 10000 || bmtime - amtime < 10000); + *file_mtime = amtime; + return as; + } + + Status RenameFile(const std::string& s, const std::string& t) override { + Status as = a_->RenameFile(s, t); + Status bs = b_->RenameFile(s, t); + assert(as == bs); + return as; + } + + Status LinkFile(const std::string& s, const std::string& t) override { + Status as = a_->LinkFile(s, t); + Status bs = b_->LinkFile(s, t); + assert(as == bs); + return as; + } + + class FileLockMirror : public FileLock { + public: + FileLock *a_, *b_; + FileLockMirror(FileLock* a, FileLock* b) : a_(a), b_(b) {} + }; + + Status LockFile(const std::string& f, FileLock** l) override { + FileLock *al, *bl; + Status as = a_->LockFile(f, &al); + Status bs = b_->LockFile(f, &bl); + assert(as == bs); + if (as.ok()) *l = new FileLockMirror(al, bl); + return as; + } + + Status UnlockFile(FileLock* l) override { + FileLockMirror* ml = static_cast(l); + Status as = a_->UnlockFile(ml->a_); + Status bs = b_->UnlockFile(ml->b_); + assert(as == bs); + delete ml; + return as; + } +}; + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/info_log_finder.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/info_log_finder.h new file mode 100644 index 0000000..824f8a3 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/info_log_finder.h @@ -0,0 +1,19 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include + +#include "rocksdb/db.h" +#include "rocksdb/options.h" + +namespace ROCKSDB_NAMESPACE { + +// This function can be used to list the Information logs, +// given the db pointer. +Status GetInfoLogList(DB* db, std::vector* info_log_list); +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/ldb_cmd.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/ldb_cmd.h new file mode 100644 index 0000000..af5ee4b --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/ldb_cmd.h @@ -0,0 +1,316 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#pragma once + + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "rocksdb/convenience.h" +#include "rocksdb/env.h" +#include "rocksdb/iterator.h" +#include "rocksdb/ldb_tool.h" +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "rocksdb/utilities/db_ttl.h" +#include "rocksdb/utilities/ldb_cmd_execute_result.h" + +namespace ROCKSDB_NAMESPACE { + +class LDBCommand { + public: + // Command-line arguments + static const std::string ARG_ENV_URI; + static const std::string ARG_FS_URI; + static const std::string ARG_DB; + static const std::string ARG_PATH; + static const std::string ARG_SECONDARY_PATH; + static const std::string ARG_HEX; + static const std::string ARG_KEY_HEX; + static const std::string ARG_VALUE_HEX; + static const std::string ARG_CF_NAME; + static const std::string ARG_TTL; + static const std::string ARG_TTL_START; + static const std::string ARG_TTL_END; + static const std::string ARG_TIMESTAMP; + static const std::string ARG_TRY_LOAD_OPTIONS; + static const std::string ARG_IGNORE_UNKNOWN_OPTIONS; + static const std::string ARG_FROM; + static const std::string ARG_TO; + static const std::string ARG_MAX_KEYS; + static const std::string ARG_BLOOM_BITS; + static const std::string ARG_FIX_PREFIX_LEN; + static const std::string ARG_COMPRESSION_TYPE; + static const std::string ARG_COMPRESSION_MAX_DICT_BYTES; + static const std::string ARG_BLOCK_SIZE; + static const std::string ARG_AUTO_COMPACTION; + static const std::string ARG_DB_WRITE_BUFFER_SIZE; + static const std::string ARG_WRITE_BUFFER_SIZE; + static const std::string ARG_FILE_SIZE; + static const std::string ARG_CREATE_IF_MISSING; + static const std::string ARG_NO_VALUE; + static const std::string ARG_DISABLE_CONSISTENCY_CHECKS; + static const std::string ARG_ENABLE_BLOB_FILES; + static const std::string ARG_MIN_BLOB_SIZE; + static const std::string ARG_BLOB_FILE_SIZE; + static const std::string ARG_BLOB_COMPRESSION_TYPE; + static const std::string ARG_ENABLE_BLOB_GARBAGE_COLLECTION; + static const std::string ARG_BLOB_GARBAGE_COLLECTION_AGE_CUTOFF; + static const std::string ARG_BLOB_GARBAGE_COLLECTION_FORCE_THRESHOLD; + static const std::string ARG_BLOB_COMPACTION_READAHEAD_SIZE; + static const std::string ARG_BLOB_FILE_STARTING_LEVEL; + static const std::string ARG_PREPOPULATE_BLOB_CACHE; + static const std::string ARG_DECODE_BLOB_INDEX; + static const std::string ARG_DUMP_UNCOMPRESSED_BLOBS; + + struct ParsedParams { + std::string cmd; + std::vector cmd_params; + std::map option_map; + std::vector flags; + }; + + static LDBCommand* SelectCommand(const ParsedParams& parsed_parms); + + static LDBCommand* InitFromCmdLineArgs( + const std::vector& args, const Options& options, + const LDBOptions& ldb_options, + const std::vector* column_families, + const std::function& selector = + SelectCommand); + + static LDBCommand* InitFromCmdLineArgs( + int argc, char const* const* argv, const Options& options, + const LDBOptions& ldb_options, + const std::vector* column_families); + + bool ValidateCmdLineOptions(); + + virtual void PrepareOptions(); + + virtual void OverrideBaseOptions(); + + virtual void OverrideBaseCFOptions(ColumnFamilyOptions* cf_opts); + + virtual void SetDBOptions(Options options) { options_ = options; } + + virtual void SetColumnFamilies( + const std::vector* column_families) { + if (column_families != nullptr) { + column_families_ = *column_families; + } else { + column_families_.clear(); + } + } + + void SetLDBOptions(const LDBOptions& ldb_options) { + ldb_options_ = ldb_options; + } + + const std::map& TEST_GetOptionMap() { + return option_map_; + } + + const std::vector& TEST_GetFlags() { return flags_; } + + virtual bool NoDBOpen() { return false; } + + virtual ~LDBCommand() { CloseDB(); } + + /* Run the command, and return the execute result. */ + void Run(); + + virtual void DoCommand() = 0; + + LDBCommandExecuteResult GetExecuteState() { return exec_state_; } + + void ClearPreviousRunState() { exec_state_.Reset(); } + + // Consider using Slice::DecodeHex directly instead if you don't need the + // 0x prefix + static std::string HexToString(const std::string& str); + + // Consider using Slice::ToString(true) directly instead if + // you don't need the 0x prefix + static std::string StringToHex(const std::string& str); + + static const char* DELIM; + + protected: + LDBCommandExecuteResult exec_state_; + std::string env_uri_; + std::string fs_uri_; + std::string db_path_; + // If empty, open DB as primary. If non-empty, open the DB as secondary + // with this secondary path. When running against a database opened by + // another process, ldb wll leave the source directory completely intact. + std::string secondary_path_; + std::string column_family_name_; + DB* db_; + DBWithTTL* db_ttl_; + std::map cf_handles_; + + /** + * true implies that this command can work if the db is opened in read-only + * mode. + */ + bool is_read_only_; + + /** If true, the key is input/output as hex in get/put/scan/delete etc. */ + bool is_key_hex_; + + /** If true, the value is input/output as hex in get/put/scan/delete etc. */ + bool is_value_hex_; + + /** If true, the value is treated as timestamp suffixed */ + bool is_db_ttl_; + + // If true, the kvs are output with their insert/modify timestamp in a ttl db + bool timestamp_; + + // If true, try to construct options from DB's option files. + bool try_load_options_; + + // The value passed to options.force_consistency_checks. + bool force_consistency_checks_; + + bool enable_blob_files_; + + bool enable_blob_garbage_collection_; + + bool create_if_missing_; + + /** + * Map of options passed on the command-line. + */ + const std::map option_map_; + + /** + * Flags passed on the command-line. + */ + const std::vector flags_; + + /** List of command-line options valid for this command */ + const std::vector valid_cmd_line_options_; + + /** Shared pointer to underlying environment if applicable **/ + std::shared_ptr env_guard_; + + bool ParseKeyValue(const std::string& line, std::string* key, + std::string* value, bool is_key_hex, bool is_value_hex); + + LDBCommand(const std::map& options, + const std::vector& flags, bool is_read_only, + const std::vector& valid_cmd_line_options); + + void OpenDB(); + + void CloseDB(); + + ColumnFamilyHandle* GetCfHandle(); + + static std::string PrintKeyValue(const std::string& key, + const std::string& value, bool is_key_hex, + bool is_value_hex); + + static std::string PrintKeyValue(const std::string& key, + const std::string& value, bool is_hex); + + /** + * Return true if the specified flag is present in the specified flags vector + */ + static bool IsFlagPresent(const std::vector& flags, + const std::string& flag) { + return (std::find(flags.begin(), flags.end(), flag) != flags.end()); + } + + static std::string HelpRangeCmdArgs(); + + /** + * A helper function that returns a list of command line options + * used by this command. It includes the common options and the ones + * passed in. + */ + static std::vector BuildCmdLineOptions( + std::vector options); + + bool ParseIntOption(const std::map& options, + const std::string& option, int& value, + LDBCommandExecuteResult& exec_state); + + bool ParseDoubleOption(const std::map& options, + const std::string& option, double& value, + LDBCommandExecuteResult& exec_state); + + bool ParseStringOption(const std::map& options, + const std::string& option, std::string* value); + + bool ParseCompressionTypeOption( + const std::map& options, + const std::string& option, CompressionType& value, + LDBCommandExecuteResult& exec_state); + + /** + * Returns the value of the specified option as a boolean. + * default_val is used if the option is not found in options. + * Throws an exception if the value of the option is not + * "true" or "false" (case insensitive). + */ + bool ParseBooleanOption(const std::map& options, + const std::string& option, bool default_val); + + Options options_; + std::vector column_families_; + ConfigOptions config_options_; + LDBOptions ldb_options_; + + private: + /** + * Interpret command line options and flags to determine if the key + * should be input/output in hex. + */ + bool IsKeyHex(const std::map& options, + const std::vector& flags); + + /** + * Interpret command line options and flags to determine if the value + * should be input/output in hex. + */ + bool IsValueHex(const std::map& options, + const std::vector& flags); + + bool IsTryLoadOptions(const std::map& options, + const std::vector& flags); + + /** + * Converts val to a boolean. + * val must be either true or false (case insensitive). + * Otherwise an exception is thrown. + */ + bool StringToBool(std::string val); +}; + +class LDBCommandRunner { + public: + static void PrintHelp(const LDBOptions& ldb_options, const char* exec_name, + bool to_stderr = true); + + // Returns the status code to return. 0 is no error. + static int RunCommand( + int argc, char const* const* argv, Options options, + const LDBOptions& ldb_options, + const std::vector* column_families); +}; + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/ldb_cmd_execute_result.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/ldb_cmd_execute_result.h new file mode 100644 index 0000000..57bac33 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/ldb_cmd_execute_result.h @@ -0,0 +1,75 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#pragma once + +#include + +#include "rocksdb/rocksdb_namespace.h" + +#ifdef FAILED +#undef FAILED +#endif + +namespace ROCKSDB_NAMESPACE { + +class LDBCommandExecuteResult { + public: + enum State { + EXEC_NOT_STARTED = 0, + EXEC_SUCCEED = 1, + EXEC_FAILED = 2, + }; + + LDBCommandExecuteResult() : state_(EXEC_NOT_STARTED), message_("") {} + + LDBCommandExecuteResult(State state, std::string& msg) + : state_(state), message_(msg) {} + + std::string ToString() { + std::string ret; + switch (state_) { + case EXEC_SUCCEED: + break; + case EXEC_FAILED: + ret.append("Failed: "); + break; + case EXEC_NOT_STARTED: + ret.append("Not started: "); + } + if (!message_.empty()) { + ret.append(message_); + } + return ret; + } + + void Reset() { + state_ = EXEC_NOT_STARTED; + message_ = ""; + } + + bool IsSucceed() { return state_ == EXEC_SUCCEED; } + + bool IsNotStarted() { return state_ == EXEC_NOT_STARTED; } + + bool IsFailed() { return state_ == EXEC_FAILED; } + + static LDBCommandExecuteResult Succeed(std::string msg) { + return LDBCommandExecuteResult(EXEC_SUCCEED, msg); + } + + static LDBCommandExecuteResult Failed(std::string msg) { + return LDBCommandExecuteResult(EXEC_FAILED, msg); + } + + private: + State state_; + std::string message_; + + bool operator==(const LDBCommandExecuteResult&); + bool operator!=(const LDBCommandExecuteResult&); +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/leveldb_options.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/leveldb_options.h new file mode 100644 index 0000000..7e4a6fa --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/leveldb_options.h @@ -0,0 +1,145 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include + +#include "rocksdb/compression_type.h" +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +class Cache; +class Comparator; +class Env; +class FilterPolicy; +class Logger; +struct Options; +class Snapshot; + +// Options to control the behavior of a database (passed to +// DB::Open). A LevelDBOptions object can be initialized as though +// it were a LevelDB Options object, and then it can be converted into +// a RocksDB Options object. +struct LevelDBOptions { + // ------------------- + // Parameters that affect behavior + + // Comparator used to define the order of keys in the table. + // Default: a comparator that uses lexicographic byte-wise ordering + // + // REQUIRES: The client must ensure that the comparator supplied + // here has the same name and orders keys *exactly* the same as the + // comparator provided to previous open calls on the same DB. + const Comparator* comparator; + + // If true, the database will be created if it is missing. + // Default: false + bool create_if_missing; + + // If true, an error is raised if the database already exists. + // Default: false + bool error_if_exists; + + // If true, the implementation will do aggressive checking of the + // data it is processing and will stop early if it detects any + // errors. This may have unforeseen ramifications: for example, a + // corruption of one DB entry may cause a large number of entries to + // become unreadable or for the entire DB to become unopenable. + // Default: false + bool paranoid_checks; + + // Use the specified object to interact with the environment, + // e.g. to read/write files, schedule background work, etc. + // Default: Env::Default() + Env* env; + + // Any internal progress/error information generated by the db will + // be written to info_log if it is non-NULL, or to a file stored + // in the same directory as the DB contents if info_log is NULL. + // Default: NULL + Logger* info_log; + + // ------------------- + // Parameters that affect performance + + // Amount of data to build up in memory (backed by an unsorted log + // on disk) before converting to a sorted on-disk file. + // + // Larger values increase performance, especially during bulk loads. + // Up to two write buffers may be held in memory at the same time, + // so you may wish to adjust this parameter to control memory usage. + // Also, a larger write buffer will result in a longer recovery time + // the next time the database is opened. + // + // Default: 4MB + size_t write_buffer_size; + + // Number of open files that can be used by the DB. You may need to + // increase this if your database has a large working set (budget + // one open file per 2MB of working set). + // + // Default: 1000 + int max_open_files; + + // Control over blocks (user data is stored in a set of blocks, and + // a block is the unit of reading from disk). + + // If non-NULL, use the specified cache for blocks. + // If NULL, leveldb will automatically create and use an 8MB internal cache. + // Default: NULL + Cache* block_cache; + + // Approximate size of user data packed per block. Note that the + // block size specified here corresponds to uncompressed data. The + // actual size of the unit read from disk may be smaller if + // compression is enabled. This parameter can be changed dynamically. + // + // Default: 4K + size_t block_size; + + // Number of keys between restart points for delta encoding of keys. + // This parameter can be changed dynamically. Most clients should + // leave this parameter alone. + // + // Default: 16 + int block_restart_interval; + + // Compress blocks using the specified compression algorithm. This + // parameter can be changed dynamically. + // + // Default: kSnappyCompression, which gives lightweight but fast + // compression. + // + // Typical speeds of kSnappyCompression on an Intel(R) Core(TM)2 2.4GHz: + // ~200-500MB/s compression + // ~400-800MB/s decompression + // Note that these speeds are significantly faster than most + // persistent storage speeds, and therefore it is typically never + // worth switching to kNoCompression. Even if the input data is + // incompressible, the kSnappyCompression implementation will + // efficiently detect that and will switch to uncompressed mode. + CompressionType compression; + + // If non-NULL, use the specified filter policy to reduce disk reads. + // Many applications will benefit from passing the result of + // NewBloomFilterPolicy() here. + // + // Default: NULL + const FilterPolicy* filter_policy; + + // Create a LevelDBOptions object with default values for all fields. + LevelDBOptions(); +}; + +// Converts a LevelDBOptions object into a RocksDB Options object. +Options ConvertOptions(const LevelDBOptions& leveldb_options); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/lua/rocks_lua_custom_library.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/lua/rocks_lua_custom_library.h new file mode 100644 index 0000000..f617da0 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/lua/rocks_lua_custom_library.h @@ -0,0 +1,43 @@ +// Copyright (c) 2016, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once +#ifdef LUA + +// lua headers +extern "C" { +#include +#include +#include +} + +namespace ROCKSDB_NAMESPACE { +namespace lua { +// A class that used to define custom C Library that is callable +// from Lua script +class RocksLuaCustomLibrary { + public: + virtual ~RocksLuaCustomLibrary() {} + // The name of the C library. This name will also be used as the table + // (namespace) in Lua that contains the C library. + virtual const char* Name() const = 0; + + // Returns a "static const struct luaL_Reg[]", which includes a list of + // C functions. Note that the last entry of this static array must be + // {nullptr, nullptr} as required by Lua. + // + // More details about how to implement Lua C libraries can be found + // in the official Lua document http://www.lua.org/pil/26.2.html + virtual const struct luaL_Reg* Lib() const = 0; + + // A function that will be called right after the library has been created + // and pushed on the top of the lua_State. This custom setup function + // allows developers to put additional table or constant values inside + // the same table / namespace. + virtual void CustomSetup(lua_State* /*L*/) const {} +}; +} // namespace lua +} // namespace ROCKSDB_NAMESPACE +#endif // LUA diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/lua/rocks_lua_util.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/lua/rocks_lua_util.h new file mode 100644 index 0000000..3427b65 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/lua/rocks_lua_util.h @@ -0,0 +1,55 @@ +// Copyright (c) 2016, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once +// lua headers +extern "C" { +#include +#include +#include +} + +#ifdef LUA +#include +#include + +#include "rocksdb/utilities/lua/rocks_lua_custom_library.h" + +namespace ROCKSDB_NAMESPACE { +namespace lua { +class LuaStateWrapper { + public: + explicit LuaStateWrapper(const std::string& lua_script) { + lua_state_ = luaL_newstate(); + Init(lua_script, {}); + } + LuaStateWrapper( + const std::string& lua_script, + const std::vector>& libraries) { + lua_state_ = luaL_newstate(); + Init(lua_script, libraries); + } + lua_State* GetLuaState() const { return lua_state_; } + ~LuaStateWrapper() { lua_close(lua_state_); } + + private: + void Init( + const std::string& lua_script, + const std::vector>& libraries) { + if (lua_state_) { + luaL_openlibs(lua_state_); + for (const auto& library : libraries) { + luaL_openlib(lua_state_, library->Name(), library->Lib(), 0); + library->CustomSetup(lua_state_); + } + luaL_dostring(lua_state_, lua_script.c_str()); + } + } + + lua_State* lua_state_; +}; +} // namespace lua +} // namespace ROCKSDB_NAMESPACE +#endif // LUA diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/memory_util.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/memory_util.h new file mode 100644 index 0000000..b141b4e --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/memory_util.h @@ -0,0 +1,48 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + + +#pragma once + +#include +#include +#include +#include + +#include "rocksdb/cache.h" +#include "rocksdb/db.h" + +namespace ROCKSDB_NAMESPACE { + +// Returns the current memory usage of the specified DB instances. +class MemoryUtil { + public: + enum UsageType : int { + // Memory usage of all the mem-tables. + kMemTableTotal = 0, + // Memory usage of those un-flushed mem-tables. + kMemTableUnFlushed = 1, + // Memory usage of all the table readers. + kTableReadersTotal = 2, + // Memory usage by Cache. + kCacheTotal = 3, + kNumUsageTypes = 4 + }; + + // Returns the approximate memory usage of different types in the input + // list of DBs and Cache set. For instance, in the output map + // usage_by_type, usage_by_type[kMemTableTotal] will store the memory + // usage of all the mem-tables from all the input rocksdb instances. + // + // Note that for memory usage inside Cache class, we will + // only report the usage of the input "cache_set" without + // including those Cache usage inside the input list "dbs" + // of DBs. + static Status GetApproximateMemoryUsageByType( + const std::vector& dbs, + const std::unordered_set cache_set, + std::map* usage_by_type); +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/object_registry.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/object_registry.h new file mode 100644 index 0000000..613ef1c --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/object_registry.h @@ -0,0 +1,583 @@ +// Copyright (c) 2016-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { +class Customizable; +class Logger; +class ObjectLibrary; + +// Returns a new T when called with a string. Populates the std::unique_ptr +// argument if granting ownership to caller. +template +using FactoryFunc = + std::function*, std::string*)>; + +// The signature of the function for loading factories +// into an object library. This method is expected to register +// factory functions in the supplied ObjectLibrary. +// The ObjectLibrary is the library in which the factories will be loaded. +// The std::string is the argument passed to the loader function. +// The RegistrarFunc should return the number of objects loaded into this +// library +using RegistrarFunc = std::function; + +template +using ConfigureFunc = std::function; + +class ObjectLibrary { + private: + // Base class for an Entry in the Registry. + class Entry { + public: + virtual ~Entry() {} + virtual bool Matches(const std::string& target) const = 0; + virtual const char* Name() const = 0; + }; + + public: + // Class for matching target strings to a pattern. + // Entries consist of a name that starts the pattern and attributes + // The following attributes can be added to the entry: + // -Suffix: Comparable to name(suffix) + // -Separator: Comparable to name(separator).+ or name(separator).* + // -Number: Comparable to name(separator).[0-9]+ + // -AltName: Comparable to (name|alt) + // -Optional: Comparable to name(separator)? + // Multiple separators can be combined and cause multiple matches. + // For example, Pattern("A").AnotherName("B").AddSeparator("@").AddNumber("#") + // is roughly equivalent to "(A|B)@.+#.+" + // + // Note that though this class does provide some regex-style matching, + // it is not a full regex parser and has some key differences: + // - Separators are matched left-most. For example, an entry + // Name("Hello").AddSeparator(" ").AddSuffix("!") would match + // "Hello world!", but not "Hello world!!" + // - No backtracking is necessary, enabling reliably efficient matching + class PatternEntry : public Entry { + private: + enum Quantifier { + kMatchZeroOrMore, // [suffix].* + kMatchAtLeastOne, // [suffix].+ + kMatchExact, // [suffix] + kMatchInteger, // [suffix][0-9]+ + kMatchDecimal, // [suffix][0-9]+[.][0-9]+ + }; + + public: + // Short-cut for creating an entry that matches to a + // Customizable::IndividualId + static PatternEntry AsIndividualId(const std::string& name) { + PatternEntry entry(name, true); + entry.AddSeparator("@"); + entry.AddSeparator("#"); + return entry; + } + + // Creates a new PatternEntry for "name". If optional is true, + // Matches will also return true if name==target + explicit PatternEntry(const std::string& name, bool optional = true) + : name_(name), optional_(optional), slength_(0) { + nlength_ = name_.size(); + } + + // Adds a suffix (exact match of separator with no trailing characters) to + // the separator + PatternEntry& AddSuffix(const std::string& suffix) { + separators_.emplace_back(suffix, kMatchExact); + slength_ += suffix.size(); + return *this; + } + + // Adds a separator (exact match of separator with trailing characters) to + // the entry + // If at_least_one is true, the separator must be followed by at least + // one character (e.g. separator.+). + // If at_least_one is false, the separator may be followed by zero or + // more characters (e.g. separator.*). + PatternEntry& AddSeparator(const std::string& separator, + bool at_least_one = true) { + slength_ += separator.size(); + if (at_least_one) { + separators_.emplace_back(separator, kMatchAtLeastOne); + ++slength_; + } else { + separators_.emplace_back(separator, kMatchZeroOrMore); + } + return *this; + } + + // Adds a separator (exact match of separator with trailing numbers) to the + // entry + PatternEntry& AddNumber(const std::string& separator, bool is_int = true) { + separators_.emplace_back(separator, + (is_int) ? kMatchInteger : kMatchDecimal); + slength_ += separator.size() + 1; + return *this; + } + + // Sets another name that this entry will match, similar to (name|alt) + PatternEntry& AnotherName(const std::string& alt) { + names_.emplace_back(alt); + return *this; + } + + // Sets whether the separators are required -- similar to name(separator)? + // If optional is true, then name(separator)? would match + // If optional is false, then the separators must also match + PatternEntry& SetOptional(bool optional) { + optional_ = optional; + return *this; + } + + // Checks to see if the target matches this entry + bool Matches(const std::string& target) const override; + const char* Name() const override { return name_.c_str(); } + + private: + size_t MatchSeparatorAt(size_t start, Quantifier mode, + const std::string& target, size_t tlen, + const std::string& pattern) const; + + bool MatchesTarget(const std::string& name, size_t nlen, + const std::string& target, size_t ylen) const; + std::string name_; // The base name for this entry + size_t nlength_; // The length of name_ + std::vector names_; // Alternative names for this entry + bool optional_; // Whether matching of separators is required + size_t slength_; // The minimum required length to match the separators + std::vector> + separators_; // What to match + }; // End class Entry + + private: + // An Entry containing a FactoryFunc for creating new Objects + template + class FactoryEntry : public Entry { + public: + FactoryEntry(Entry* e, FactoryFunc f) + : entry_(e), factory_(std::move(f)) {} + bool Matches(const std::string& target) const override { + return entry_->Matches(target); + } + const char* Name() const override { return entry_->Name(); } + + // Creates a new T object. + T* NewFactoryObject(const std::string& target, std::unique_ptr* guard, + std::string* msg) const { + return factory_(target, guard, msg); + } + const FactoryFunc& GetFactory() const { return factory_; } + + private: + std::unique_ptr entry_; // What to match for this entry + FactoryFunc factory_; + }; // End class FactoryEntry + public: + explicit ObjectLibrary(const std::string& id) { id_ = id; } + + const std::string& GetID() const { return id_; } + + // Finds the factory function for the input target. + // @see PatternEntry for the matching rules to target + // @return If matched, the FactoryFunc for this target, else nullptr + template + FactoryFunc FindFactory(const std::string& target) const { + std::unique_lock lock(mu_); + auto factories = factories_.find(T::Type()); + if (factories != factories_.end()) { + for (const auto& e : factories->second) { + if (e->Matches(target)) { + const auto* fe = + static_cast*>(e.get()); + return fe->GetFactory(); + } + } + } + return nullptr; + } + + // Returns the total number of factories registered for this library. + // This method returns the sum of all factories registered for all types. + // @param num_types returns how many unique types are registered. + size_t GetFactoryCount(size_t* num_types) const; + + // Returns the number of factories registered for this library + // for the input type. + // @param num_types returns how many unique types are registered. + size_t GetFactoryCount(const std::string& type) const; + + // Returns the registered factory names for the input type + // names is updated to include the names for the type + void GetFactoryNames(const std::string& type, + std::vector* names) const; + + void GetFactoryTypes(std::unordered_set* types) const; + + void Dump(Logger* logger) const; + + // Registers the factory with the library for the name. + // If name==target, the factory may be used to create a new object. + template + const FactoryFunc& AddFactory(const std::string& name, + const FactoryFunc& func) { + std::unique_ptr entry( + new FactoryEntry(new PatternEntry(name), func)); + AddFactoryEntry(T::Type(), std::move(entry)); + return func; + } + + // Registers the factory with the library for the entry. + // If the entry matches the target, the factory may be used to create a new + // object. + // @see PatternEntry for the matching rules. + // NOTE: This function replaces the old ObjectLibrary::Register() + template + const FactoryFunc& AddFactory(const PatternEntry& entry, + const FactoryFunc& func) { + std::unique_ptr factory( + new FactoryEntry(new PatternEntry(entry), func)); + AddFactoryEntry(T::Type(), std::move(factory)); + return func; + } + + // Invokes the registrar function with the supplied arg for this library. + int Register(const RegistrarFunc& registrar, const std::string& arg) { + return registrar(*this, arg); + } + + // Returns the default ObjectLibrary + static std::shared_ptr& Default(); + + private: + void AddFactoryEntry(const char* type, std::unique_ptr&& entry) { + std::unique_lock lock(mu_); + auto& factories = factories_[type]; + factories.emplace_back(std::move(entry)); + } + + // Protects the entry map + mutable std::mutex mu_; + // ** FactoryFunctions for this loader, organized by type + std::unordered_map>> + factories_; + + // The name for this library + std::string id_; +}; + +// The ObjectRegistry is used to register objects that can be created by a +// name/pattern at run-time where the specific implementation of the object may +// not be known in advance. +class ObjectRegistry { + public: + static std::shared_ptr NewInstance(); + static std::shared_ptr NewInstance( + const std::shared_ptr& parent); + static std::shared_ptr Default(); + explicit ObjectRegistry(const std::shared_ptr& parent) + : parent_(parent) {} + explicit ObjectRegistry(const std::shared_ptr& library); + + std::shared_ptr AddLibrary(const std::string& id) { + auto library = std::make_shared(id); + AddLibrary(library); + return library; + } + + void AddLibrary(const std::shared_ptr& library) { + std::unique_lock lock(library_mutex_); + libraries_.push_back(library); + } + + void AddLibrary(const std::string& id, const RegistrarFunc& registrar, + const std::string& arg) { + auto library = AddLibrary(id); + library->Register(registrar, arg); + } + + // Finds the factory for target and instantiates a new T. + // Returns NotSupported if no factory is found + // Returns InvalidArgument if a factory is found but the factory failed. + template + Status NewObject(const std::string& target, T** object, + std::unique_ptr* guard) { + assert(guard != nullptr); + guard->reset(); + auto factory = FindFactory(target); + if (factory != nullptr) { + std::string errmsg; + *object = factory(target, guard, &errmsg); + if (*object != nullptr) { + return Status::OK(); + } else if (errmsg.empty()) { + return Status::InvalidArgument( + std::string("Could not load ") + T::Type(), target); + } else { + return Status::InvalidArgument(errmsg, target); + } + } else { + return Status::NotSupported(std::string("Could not load ") + T::Type(), + target); + } + } + // Creates a new unique T using the input factory functions. + // Returns OK if a new unique T was successfully created + // Returns NotSupported if the type/target could not be created + // Returns InvalidArgument if the factory return an unguarded object + // (meaning it cannot be managed by a unique ptr) + template + Status NewUniqueObject(const std::string& target, + std::unique_ptr* result) { + T* ptr = nullptr; + std::unique_ptr guard; + Status s = NewObject(target, &ptr, &guard); + if (!s.ok()) { + return s; + } else if (guard) { + result->reset(guard.release()); + return Status::OK(); + } else { + return Status::InvalidArgument(std::string("Cannot make a unique ") + + T::Type() + " from unguarded one ", + target); + } + } + + // Creates a new shared T using the input factory functions. + // Returns OK if a new shared T was successfully created + // Returns NotSupported if the type/target could not be created + // Returns InvalidArgument if the factory return an unguarded object + // (meaning it cannot be managed by a shared ptr) + template + Status NewSharedObject(const std::string& target, + std::shared_ptr* result) { + std::unique_ptr guard; + T* ptr = nullptr; + Status s = NewObject(target, &ptr, &guard); + if (!s.ok()) { + return s; + } else if (guard) { + result->reset(guard.release()); + return Status::OK(); + } else { + return Status::InvalidArgument(std::string("Cannot make a shared ") + + T::Type() + " from unguarded one ", + target); + } + } + + // Creates a new static T using the input factory functions. + // Returns OK if a new static T was successfully created + // Returns NotSupported if the type/target could not be created + // Returns InvalidArgument if the factory return a guarded object + // (meaning it is managed by a unique ptr) + template + Status NewStaticObject(const std::string& target, T** result) { + std::unique_ptr guard; + T* ptr = nullptr; + Status s = NewObject(target, &ptr, &guard); + if (!s.ok()) { + return s; + } else if (guard.get()) { + return Status::InvalidArgument(std::string("Cannot make a static ") + + T::Type() + " from a guarded one ", + target); + } else { + *result = ptr; + return Status::OK(); + } + } + + // Sets the object for the given id/type to be the input object + // If the registry does not contain this id/type, the object is added and OK + // is returned. If the registry contains a different object, an error is + // returned. If the registry contains the input object, OK is returned. + template + Status SetManagedObject(const std::shared_ptr& object) { + assert(object != nullptr); + return SetManagedObject(object->GetId(), object); + } + + template + Status SetManagedObject(const std::string& id, + const std::shared_ptr& object) { + const auto c = std::static_pointer_cast(object); + return SetManagedObject(T::Type(), id, c); + } + + // Returns the object for the given id, if one exists. + // If the object is not found in the registry, a nullptr is returned + template + std::shared_ptr GetManagedObject(const std::string& id) const { + auto c = GetManagedObject(T::Type(), id); + return std::static_pointer_cast(c); + } + + // Returns the set of managed objects found in the registry matching + // the input type and ID. + // If the input id is not empty, then only objects of that class + // (IsInstanceOf(id)) will be returned (for example, only return LRUCache + // objects) If the input id is empty, then all objects of that type (all Cache + // objects) + template + Status ListManagedObjects(const std::string& id, + std::vector>* results) const { + std::vector> customizables; + results->clear(); + Status s = ListManagedObjects(T::Type(), id, &customizables); + if (s.ok()) { + for (const auto& c : customizables) { + results->push_back(std::static_pointer_cast(c)); + } + } + return s; + } + + template + Status ListManagedObjects(std::vector>* results) const { + return ListManagedObjects("", results); + } + + // Creates a new ManagedObject in the registry for the id if one does not + // currently exist. If an object with that ID already exists, the current + // object is returned. + // + // The ID is the identifier of the object to be returned/created and returned + // in result + // If a new object is created (using the object factories), the cfunc + // parameter will be invoked to configure the new object. + template + Status GetOrCreateManagedObject(const std::string& id, + std::shared_ptr* result, + const ConfigureFunc& cfunc = nullptr) { + if (parent_ != nullptr) { + auto object = parent_->GetManagedObject(T::Type(), id); + if (object != nullptr) { + *result = std::static_pointer_cast(object); + return Status::OK(); + } + } + { + std::unique_lock lock(objects_mutex_); + auto key = ToManagedObjectKey(T::Type(), id); + auto iter = managed_objects_.find(key); + if (iter != managed_objects_.end()) { + auto object = iter->second.lock(); + if (object != nullptr) { + *result = std::static_pointer_cast(object); + return Status::OK(); + } + } + std::shared_ptr object; + Status s = NewSharedObject(id, &object); + if (s.ok() && cfunc != nullptr) { + s = cfunc(object.get()); + } + if (s.ok()) { + auto c = std::static_pointer_cast(object); + if (id != c->Name()) { + // If the ID is not the base name of the class, add the new + // object under the input ID + managed_objects_[key] = c; + } + if (id != c->GetId() && c->GetId() != c->Name()) { + // If the input and current ID do not match, and the + // current ID is not the base bame, add the new object under + // its new ID + key = ToManagedObjectKey(T::Type(), c->GetId()); + managed_objects_[key] = c; + } + *result = object; + } + return s; + } + } + + // Returns the number of factories registered for this library + // for the input type. + // @param num_types returns how many unique types are registered. + size_t GetFactoryCount(const std::string& type) const; + + // Returns the names of registered factories for the input type. + // names is updated to include the names for the type + void GetFactoryNames(const std::string& type, + std::vector* names) const; + + void GetFactoryTypes(std::unordered_set* types) const; + + // Dump the contents of the registry to the logger + void Dump(Logger* logger) const; + + // Invokes the input function to retrieve the properties for this plugin. + int RegisterPlugin(const std::string& name, const RegistrarFunc& func); + + private: + static std::string ToManagedObjectKey(const std::string& type, + const std::string& id) { + return type + "://" + id; + } + + // Returns the Customizable managed object associated with the key (Type/ID). + // If not found, nullptr is returned. + std::shared_ptr GetManagedObject(const std::string& type, + const std::string& id) const; + Status ListManagedObjects( + const std::string& type, const std::string& pattern, + std::vector>* results) const; + // Sets the managed object associated with the key (Type/ID) to c. + // If the named managed object does not exist, the object is added and OK is + // returned If the object exists and is the same as c, OK is returned + // Otherwise, an error status is returned. + Status SetManagedObject(const std::string& type, const std::string& id, + const std::shared_ptr& c); + + // Searches (from back to front) the libraries looking for the + // factory that matches this name. + // Returns the factory if it is found, and nullptr otherwise + template + const FactoryFunc FindFactory(const std::string& name) const { + { + std::unique_lock lock(library_mutex_); + for (auto iter = libraries_.crbegin(); iter != libraries_.crend(); + ++iter) { + const auto factory = iter->get()->FindFactory(name); + if (factory != nullptr) { + return factory; + } + } + } + if (parent_ == nullptr) { + return nullptr; + } else { + return parent_->FindFactory(name); + } + } + + // The set of libraries to search for factories for this registry. + // The libraries are searched in reverse order (back to front) when + // searching for entries. + std::vector> libraries_; + std::vector plugins_; + static std::unordered_map builtins_; + std::map> managed_objects_; + std::shared_ptr parent_; + mutable std::mutex objects_mutex_; // Mutex for managed objects + mutable std::mutex library_mutex_; // Mutex for managed libraries +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/optimistic_transaction_db.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/optimistic_transaction_db.h new file mode 100644 index 0000000..0925eaf --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/optimistic_transaction_db.h @@ -0,0 +1,129 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include + +#include "rocksdb/comparator.h" +#include "rocksdb/db.h" +#include "rocksdb/utilities/stackable_db.h" + +namespace ROCKSDB_NAMESPACE { + +class Transaction; + +// Database with Transaction support. +// +// See optimistic_transaction.h and examples/transaction_example.cc + +// Options to use when starting an Optimistic Transaction +struct OptimisticTransactionOptions { + // Setting set_snapshot=true is the same as calling SetSnapshot(). + bool set_snapshot = false; + + // Should be set if the DB has a non-default comparator. + // See comment in WriteBatchWithIndex constructor. + const Comparator* cmp = BytewiseComparator(); +}; + +enum class OccValidationPolicy { + // Validate serially at commit stage, AFTER entering the write-group. + // Isolation validation is processed single-threaded(since in the + // write-group). + // May suffer from high mutex contention, as per this link: + // https://github.com/facebook/rocksdb/issues/4402 + kValidateSerial = 0, + // Validate parallelly before commit stage, BEFORE entering the write-group to + // reduce mutex contention. Each txn acquires locks for its write-set + // records in some well-defined order. + kValidateParallel = 1 +}; + +class OccLockBuckets { + public: + // Most details in internal derived class. + // Users should not derive from this class. + virtual ~OccLockBuckets() {} + + virtual size_t ApproximateMemoryUsage() const = 0; + + private: + friend class OccLockBucketsImplBase; + OccLockBuckets() {} +}; + +// An object for sharing a pool of locks across DB instances. +// +// Making the locks cache-aligned avoids potential false sharing, at the +// potential cost of extra memory. The implementation has historically +// used cache_aligned = false. +std::shared_ptr MakeSharedOccLockBuckets( + size_t bucket_count, bool cache_aligned = false); + +struct OptimisticTransactionDBOptions { + OccValidationPolicy validate_policy = OccValidationPolicy::kValidateParallel; + + // Number of striped/bucketed mutex locks for validating transactions. + // Used on only if validate_policy == OccValidationPolicy::kValidateParallel + // and shared_lock_buckets (below) is empty. Larger number potentially + // reduces contention but uses more memory. + uint32_t occ_lock_buckets = (1 << 20); + + // A pool of mutex locks for validating transactions. Can be shared among + // DBs. Ignored if validate_policy != OccValidationPolicy::kValidateParallel. + // If empty and validate_policy == OccValidationPolicy::kValidateParallel, + // an OccLockBuckets will be created using the count in occ_lock_buckets. + // See MakeSharedOccLockBuckets() + std::shared_ptr shared_lock_buckets; +}; + +// Range deletions (including those in `WriteBatch`es passed to `Write()`) are +// incompatible with `OptimisticTransactionDB` and will return a non-OK `Status` +class OptimisticTransactionDB : public StackableDB { + public: + // Open an OptimisticTransactionDB similar to DB::Open(). + static Status Open(const Options& options, const std::string& dbname, + OptimisticTransactionDB** dbptr); + + static Status Open(const DBOptions& db_options, const std::string& dbname, + const std::vector& column_families, + std::vector* handles, + OptimisticTransactionDB** dbptr); + + static Status Open(const DBOptions& db_options, + const OptimisticTransactionDBOptions& occ_options, + const std::string& dbname, + const std::vector& column_families, + std::vector* handles, + OptimisticTransactionDB** dbptr); + + virtual ~OptimisticTransactionDB() {} + + // Starts a new Transaction. + // + // Caller is responsible for deleting the returned transaction when no + // longer needed. + // + // If old_txn is not null, BeginTransaction will reuse this Transaction + // handle instead of allocating a new one. This is an optimization to avoid + // extra allocations when repeatedly creating transactions. + virtual Transaction* BeginTransaction( + const WriteOptions& write_options, + const OptimisticTransactionOptions& txn_options = + OptimisticTransactionOptions(), + Transaction* old_txn = nullptr) = 0; + + OptimisticTransactionDB(const OptimisticTransactionDB&) = delete; + void operator=(const OptimisticTransactionDB&) = delete; + + protected: + // To Create an OptimisticTransactionDB, call Open() + explicit OptimisticTransactionDB(DB* db) : StackableDB(db) {} +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/option_change_migration.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/option_change_migration.h new file mode 100644 index 0000000..a73324a --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/option_change_migration.h @@ -0,0 +1,24 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "rocksdb/options.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { +// Try to migrate DB created with old_opts to be use new_opts. +// Multiple column families is not supported. +// It is best-effort. No guarantee to succeed. +// A full compaction may be executed. +// If the target options use FIFO compaction, the FIFO condition might be +// sacrificed: for data migrated, data inserted later might be dropped +// earlier. This is to gurantee FIFO compaction won't drop all the +// migrated data to fit max_table_files_size. +Status OptionChangeMigration(std::string dbname, const Options& old_opts, + const Options& new_opts); +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/options_type.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/options_type.h new file mode 100644 index 0000000..cd340ed --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/options_type.h @@ -0,0 +1,1221 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// The OptionTypeInfo and related classes provide a framework for +// configuring and validating RocksDB classes via the Options framework. +// This file is part of the public API to allow developers who wish to +// write their own extensions and plugins to take use the Options +// framework in their custom implementations. +// +// See https://github.com/facebook/rocksdb/wiki/RocksDB-Configurable-Objects +// for more information on how to develop and use custom extensions + +#pragma once + +#include +#include +#include + +#include "rocksdb/convenience.h" +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { +class OptionTypeInfo; +struct ColumnFamilyOptions; +struct DBOptions; + +// The underlying "class/type" of the option. +// This enum is used to determine how the option should +// be converted to/from strings and compared. +enum class OptionType { + kBoolean, + kInt, + kInt32T, + kInt64T, + kUInt, + kUInt8T, + kUInt32T, + kUInt64T, + kSizeT, + kString, + kDouble, + kCompactionStyle, + kCompactionPri, + kCompressionType, + kCompactionStopStyle, + kChecksumType, + kEncodingType, + kEnv, + kEnum, + kStruct, + kVector, + kConfigurable, + kCustomizable, + kEncodedString, + kTemperature, + kArray, + kUnknown, +}; + +enum class OptionVerificationType { + kNormal, + kByName, // The option is pointer typed so we can only verify + // based on it's name. + kByNameAllowNull, // Same as kByName, but it also allows the case + // where one of them is a nullptr. + kByNameAllowFromNull, // Same as kByName, but it also allows the case + // where the old option is nullptr. + kDeprecated, // The option is no longer used in rocksdb. The RocksDB + // OptionsParser will still accept this option if it + // happen to exists in some Options file. However, + // the parser will not include it in serialization + // and verification processes. + kAlias, // This option represents is a name/shortcut for + // another option and should not be written or verified + // independently +}; + +// A set of modifier flags used to alter how an option is evaluated or +// processed. These flags can be combined together (e.g. kMutable | kShared). +// The kCompare flags can be used to control if/when options are compared. +// If kCompareNever is set, two related options would never be compared (always +// equal) If kCompareExact is set, the options will only be compared if the +// sanity mode +// is exact +// kMutable means the option can be changed after it is prepared +// kShared means the option is contained in a std::shared_ptr +// kUnique means the option is contained in a std::uniqued_ptr +// kRawPointer means the option is a raw pointer value. +// kAllowNull means that an option is allowed to be null for verification +// purposes. +// kDontSerialize means this option should not be serialized and included in +// the string representation. +// kDontPrepare means do not call PrepareOptions for this pointer value. +enum class OptionTypeFlags : uint32_t { + kNone = 0x00, // No flags + kCompareDefault = 0x0, + kCompareNever = ConfigOptions::kSanityLevelNone, + kCompareLoose = ConfigOptions::kSanityLevelLooselyCompatible, + kCompareExact = ConfigOptions::kSanityLevelExactMatch, + + kMutable = 0x0100, // Option is mutable + kRawPointer = 0x0200, // The option is stored as a raw pointer + kShared = 0x0400, // The option is stored as a shared_ptr + kUnique = 0x0800, // The option is stored as a unique_ptr + kAllowNull = 0x1000, // The option can be null + kDontSerialize = 0x2000, // Don't serialize the option + kDontPrepare = 0x4000, // Don't prepare or sanitize this option + kStringNameOnly = 0x8000, // The option serializes to a name only +}; + +inline OptionTypeFlags operator|(const OptionTypeFlags& a, + const OptionTypeFlags& b) { + return static_cast(static_cast(a) | + static_cast(b)); +} + +inline OptionTypeFlags operator&(const OptionTypeFlags& a, + const OptionTypeFlags& b) { + return static_cast(static_cast(a) & + static_cast(b)); +} + +// Converts an string into its enumerated value. +// @param type_map Mapping between strings and enum values +// @param type The string representation of the enum +// @param value Returns the enum value represented by the string +// @return true if the string was found in the enum map, false otherwise. +template +bool ParseEnum(const std::unordered_map& type_map, + const std::string& type, T* value) { + auto iter = type_map.find(type); + if (iter != type_map.end()) { + *value = iter->second; + return true; + } + return false; +} + +// Converts an enum into its string representation. +// @param type_map Mapping between strings and enum values +// @param type The enum +// @param value Returned as the string representation of the enum +// @return true if the enum was found in the enum map, false otherwise. +template +bool SerializeEnum(const std::unordered_map& type_map, + const T& type, std::string* value) { + for (const auto& pair : type_map) { + if (pair.second == type) { + *value = pair.first; + return true; + } + } + return false; +} + +template +Status ParseArray(const ConfigOptions& config_options, + const OptionTypeInfo& elem_info, char separator, + const std::string& name, const std::string& value, + std::array* result); + +template +Status SerializeArray(const ConfigOptions& config_options, + const OptionTypeInfo& elem_info, char separator, + const std::string& name, const std::array& vec, + std::string* value); + +template +bool ArraysAreEqual(const ConfigOptions& config_options, + const OptionTypeInfo& elem_info, const std::string& name, + const std::array& array1, + const std::array& array2, std::string* mismatch); + +template +Status ParseVector(const ConfigOptions& config_options, + const OptionTypeInfo& elem_info, char separator, + const std::string& name, const std::string& value, + std::vector* result); + +template +Status SerializeVector(const ConfigOptions& config_options, + const OptionTypeInfo& elem_info, char separator, + const std::string& name, const std::vector& vec, + std::string* value); +template +bool VectorsAreEqual(const ConfigOptions& config_options, + const OptionTypeInfo& elem_info, const std::string& name, + const std::vector& vec1, const std::vector& vec2, + std::string* mismatch); + +// Function for converting a option string value into its underlying +// representation in "addr" +// On success, Status::OK is returned and addr is set to the parsed form +// On failure, a non-OK status is returned +// @param opts The ConfigOptions controlling how the value is parsed +// @param name The name of the options being parsed +// @param value The string representation of the option +// @param addr Pointer to the object +using ParseFunc = std::function; + +// Function for converting an option "addr" into its string representation. +// On success, Status::OK is returned and value is the serialized form. +// On failure, a non-OK status is returned +// @param opts The ConfigOptions controlling how the values are serialized +// @param name The name of the options being serialized +// @param addr Pointer to the value being serialized +// @param value The result of the serialization. +using SerializeFunc = std::function; + +// Function for comparing two option values +// If they are not equal, updates "mismatch" with the name of the bad option +// @param opts The ConfigOptions controlling how the values are compared +// @param name The name of the options being compared +// @param addr1 The first address to compare +// @param addr2 The address to compare to +// @param mismatch If the values are not equal, the name of the option that +// first differs +using EqualsFunc = std::function; + +// Function for preparing/initializing an option. +using PrepareFunc = + std::function; + +// Function for validating an option. +using ValidateFunc = std::function; + +// A struct for storing constant option information such as option name, +// option type, and offset. +class OptionTypeInfo { + public: + // A simple "normal", non-mutable Type "type" at offset + OptionTypeInfo(int offset, OptionType type) + : offset_(offset), + parse_func_(nullptr), + serialize_func_(nullptr), + equals_func_(nullptr), + type_(type), + verification_(OptionVerificationType::kNormal), + flags_(OptionTypeFlags::kNone) {} + + OptionTypeInfo(int offset, OptionType type, + OptionVerificationType verification, OptionTypeFlags flags) + : offset_(offset), + parse_func_(nullptr), + serialize_func_(nullptr), + equals_func_(nullptr), + type_(type), + verification_(verification), + flags_(flags) {} + + OptionTypeInfo(int offset, OptionType type, + OptionVerificationType verification, OptionTypeFlags flags, + const ParseFunc& parse_func) + : offset_(offset), + parse_func_(parse_func), + serialize_func_(nullptr), + equals_func_(nullptr), + type_(type), + verification_(verification), + flags_(flags) {} + + OptionTypeInfo(int offset, OptionType type, + OptionVerificationType verification, OptionTypeFlags flags, + const ParseFunc& parse_func, + const SerializeFunc& serialize_func, + const EqualsFunc& equals_func) + : offset_(offset), + parse_func_(parse_func), + serialize_func_(serialize_func), + equals_func_(equals_func), + type_(type), + verification_(verification), + flags_(flags) {} + + // Creates an OptionTypeInfo for an enum type. Enums use an additional + // map to convert the enums to/from their string representation. + // To create an OptionTypeInfo that is an Enum, one should: + // - Create a static map of string values to the corresponding enum value + // - Call this method passing the static map in as a parameter. + // Note that it is not necessary to add a new OptionType or make any + // other changes -- the returned object handles parsing, serialization, and + // comparisons. + // + // @param offset The offset in the option object for this enum + // @param map The string to enum mapping for this enum + template + static OptionTypeInfo Enum( + int offset, const std::unordered_map* const map, + OptionTypeFlags flags = OptionTypeFlags::kNone) { + OptionTypeInfo info(offset, OptionType::kEnum, + OptionVerificationType::kNormal, flags); + info.SetParseFunc( + // Uses the map argument to convert the input string into + // its corresponding enum value. If value is found in the map, + // addr is updated to the corresponding map entry. + // @return OK if the value is found in the map + // @return InvalidArgument if the value is not found in the map + [map](const ConfigOptions&, const std::string& name, + const std::string& value, void* addr) { + if (map == nullptr) { + return Status::NotSupported("No enum mapping ", name); + } else if (ParseEnum(*map, value, static_cast(addr))) { + return Status::OK(); + } else { + return Status::InvalidArgument("No mapping for enum ", name); + } + }); + info.SetSerializeFunc( + // Uses the map argument to convert the input enum into + // its corresponding string value. If enum value is found in the map, + // value is updated to the corresponding string value in the map. + // @return OK if the enum is found in the map + // @return InvalidArgument if the enum is not found in the map + [map](const ConfigOptions&, const std::string& name, const void* addr, + std::string* value) { + if (map == nullptr) { + return Status::NotSupported("No enum mapping ", name); + } else if (SerializeEnum(*map, (*static_cast(addr)), + value)) { + return Status::OK(); + } else { + return Status::InvalidArgument("No mapping for enum ", name); + } + }); + info.SetEqualsFunc( + // Casts addr1 and addr2 to the enum type and returns true if + // they are equal, false otherwise. + [](const ConfigOptions&, const std::string&, const void* addr1, + const void* addr2, std::string*) { + return (*static_cast(addr1) == + *static_cast(addr2)); + }); + return info; + } // End OptionTypeInfo::Enum + + // Creates an OptionTypeInfo for a Struct type. Structs have a + // map of string-OptionTypeInfo associated with them that describes how + // to process the object for parsing, serializing, and matching. + // Structs also have a struct_name, which is the name of the object + // as registered in the parent map. + // When processing a struct, the option name can be specified as: + // - Meaning to process the entire struct. + // - Meaning to process the single field + // - Process the single fields + // The CompactionOptionsFIFO, CompactionOptionsUniversal, and LRUCacheOptions + // are all examples of Struct options. + // + // To create an OptionTypeInfo that is a Struct, one should: + // - Create a static map of string-OptionTypeInfo corresponding to the + // properties of the object that can be set via the options. + // - Call this method passing the name and map in as parameters. + // Note that it is not necessary to add a new OptionType or make any + // other changes -- the returned object handles parsing, serialization, and + // comparisons. + // + // @param offset The offset in the option object for this enum + // @param map The string to enum mapping for this enum + static OptionTypeInfo Struct( + const std::string& struct_name, + const std::unordered_map* struct_map, + int offset, OptionVerificationType verification, OptionTypeFlags flags) { + OptionTypeInfo info(offset, OptionType::kStruct, verification, flags); + info.SetParseFunc( + // Parses the struct and updates the fields at addr + [struct_name, struct_map](const ConfigOptions& opts, + const std::string& name, + const std::string& value, void* addr) { + return ParseStruct(opts, struct_name, struct_map, name, value, addr); + }); + info.SetSerializeFunc( + // Serializes the struct options into value + [struct_name, struct_map](const ConfigOptions& opts, + const std::string& name, const void* addr, + std::string* value) { + return SerializeStruct(opts, struct_name, struct_map, name, addr, + value); + }); + info.SetEqualsFunc( + // Compares the struct fields of addr1 and addr2 for equality + [struct_name, struct_map](const ConfigOptions& opts, + const std::string& name, const void* addr1, + const void* addr2, std::string* mismatch) { + return StructsAreEqual(opts, struct_name, struct_map, name, addr1, + addr2, mismatch); + }); + return info; + } + static OptionTypeInfo Struct( + const std::string& struct_name, + const std::unordered_map* struct_map, + int offset, OptionVerificationType verification, OptionTypeFlags flags, + const ParseFunc& parse_func) { + OptionTypeInfo info( + Struct(struct_name, struct_map, offset, verification, flags)); + return info.SetParseFunc(parse_func); + } + + template + static OptionTypeInfo Array(int _offset, OptionVerificationType _verification, + OptionTypeFlags _flags, + const OptionTypeInfo& elem_info, + char separator = ':') { + OptionTypeInfo info(_offset, OptionType::kArray, _verification, _flags); + info.SetParseFunc([elem_info, separator]( + const ConfigOptions& opts, const std::string& name, + const std::string& value, void* addr) { + auto result = static_cast*>(addr); + return ParseArray(opts, elem_info, separator, name, value, + result); + }); + info.SetSerializeFunc([elem_info, separator](const ConfigOptions& opts, + const std::string& name, + const void* addr, + std::string* value) { + const auto& array = *(static_cast*>(addr)); + return SerializeArray(opts, elem_info, separator, name, array, + value); + }); + info.SetEqualsFunc([elem_info](const ConfigOptions& opts, + const std::string& name, const void* addr1, + const void* addr2, std::string* mismatch) { + const auto& array1 = *(static_cast*>(addr1)); + const auto& array2 = *(static_cast*>(addr2)); + return ArraysAreEqual(opts, elem_info, name, array1, array2, + mismatch); + }); + return info; + } + + template + static OptionTypeInfo Vector(int _offset, + OptionVerificationType _verification, + OptionTypeFlags _flags, + const OptionTypeInfo& elem_info, + char separator = ':') { + OptionTypeInfo info(_offset, OptionType::kVector, _verification, _flags); + info.SetParseFunc([elem_info, separator]( + const ConfigOptions& opts, const std::string& name, + const std::string& value, void* addr) { + auto result = static_cast*>(addr); + return ParseVector(opts, elem_info, separator, name, value, result); + }); + info.SetSerializeFunc([elem_info, separator](const ConfigOptions& opts, + const std::string& name, + const void* addr, + std::string* value) { + const auto& vec = *(static_cast*>(addr)); + return SerializeVector(opts, elem_info, separator, name, vec, value); + }); + info.SetEqualsFunc([elem_info](const ConfigOptions& opts, + const std::string& name, const void* addr1, + const void* addr2, std::string* mismatch) { + const auto& vec1 = *(static_cast*>(addr1)); + const auto& vec2 = *(static_cast*>(addr2)); + return VectorsAreEqual(opts, elem_info, name, vec1, vec2, mismatch); + }); + return info; + } + + // Create a new std::shared_ptr OptionTypeInfo + // This function will call the T::CreateFromString method to create a new + // std::shared_ptr object. + // + // @param offset The offset for the Customizable from the base pointer + // @param ovt How to verify this option + // @param flags, Extra flags specifying the behavior of this option + // @param _sfunc Optional function for serializing this option + // @param _efunc Optional function for comparing this option + template + static OptionTypeInfo AsCustomSharedPtr(int offset, + OptionVerificationType ovt, + OptionTypeFlags flags) { + OptionTypeInfo info(offset, OptionType::kCustomizable, ovt, + flags | OptionTypeFlags::kShared); + return info.SetParseFunc([](const ConfigOptions& opts, + const std::string& name, + const std::string& value, void* addr) { + auto* shared = static_cast*>(addr); + if (name == kIdPropName() && value.empty()) { + shared->reset(); + return Status::OK(); + } else { + return T::CreateFromString(opts, value, shared); + } + }); + } + + template + static OptionTypeInfo AsCustomSharedPtr(int offset, + OptionVerificationType ovt, + OptionTypeFlags flags, + const SerializeFunc& serialize_func, + const EqualsFunc& equals_func) { + OptionTypeInfo info(AsCustomSharedPtr(offset, ovt, flags)); + info.SetSerializeFunc(serialize_func); + info.SetEqualsFunc(equals_func); + return info; + } + + // Create a new std::unique_ptr OptionTypeInfo + // This function will call the T::CreateFromString method to create a new + // std::unique_ptr object. + // + // @param offset The offset for the Customizable from the base pointer + // @param ovt How to verify this option + // @param flags, Extra flags specifying the behavior of this option + // @param _sfunc Optional function for serializing this option + // @param _efunc Optional function for comparing this option + template + static OptionTypeInfo AsCustomUniquePtr(int offset, + OptionVerificationType ovt, + OptionTypeFlags flags) { + OptionTypeInfo info(offset, OptionType::kCustomizable, ovt, + flags | OptionTypeFlags::kUnique); + return info.SetParseFunc([](const ConfigOptions& opts, + const std::string& name, + const std::string& value, void* addr) { + auto* unique = static_cast*>(addr); + if (name == kIdPropName() && value.empty()) { + unique->reset(); + return Status::OK(); + } else { + return T::CreateFromString(opts, value, unique); + } + }); + } + + template + static OptionTypeInfo AsCustomUniquePtr(int offset, + OptionVerificationType ovt, + OptionTypeFlags flags, + const SerializeFunc& serialize_func, + const EqualsFunc& equals_func) { + OptionTypeInfo info(AsCustomUniquePtr(offset, ovt, flags)); + info.SetSerializeFunc(serialize_func); + info.SetEqualsFunc(equals_func); + return info; + } + + // Create a new Customizable* OptionTypeInfo + // This function will call the T::CreateFromString method to create a new + // T object. + // + // @param _offset The offset for the Customizable from the base pointer + // @param ovt How to verify this option + // @param flags, Extra flags specifying the behavior of this option + // @param _sfunc Optional function for serializing this option + // @param _efunc Optional function for comparing this option + template + static OptionTypeInfo AsCustomRawPtr(int offset, OptionVerificationType ovt, + OptionTypeFlags flags) { + OptionTypeInfo info(offset, OptionType::kCustomizable, ovt, + flags | OptionTypeFlags::kRawPointer); + return info.SetParseFunc([](const ConfigOptions& opts, + const std::string& name, + const std::string& value, void* addr) { + auto** pointer = static_cast(addr); + if (name == kIdPropName() && value.empty()) { + *pointer = nullptr; + return Status::OK(); + } else { + return T::CreateFromString(opts, value, pointer); + } + }); + } + + template + static OptionTypeInfo AsCustomRawPtr(int offset, OptionVerificationType ovt, + OptionTypeFlags flags, + const SerializeFunc& serialize_func, + const EqualsFunc& equals_func) { + OptionTypeInfo info(AsCustomRawPtr(offset, ovt, flags)); + info.SetSerializeFunc(serialize_func); + info.SetEqualsFunc(equals_func); + return info; + } + + OptionTypeInfo& SetParseFunc(const ParseFunc& f) { + parse_func_ = f; + return *this; + } + + OptionTypeInfo& SetSerializeFunc(const SerializeFunc& f) { + serialize_func_ = f; + return *this; + } + OptionTypeInfo& SetEqualsFunc(const EqualsFunc& f) { + equals_func_ = f; + return *this; + } + + OptionTypeInfo& SetPrepareFunc(const PrepareFunc& f) { + prepare_func_ = f; + return *this; + } + + OptionTypeInfo& SetValidateFunc(const ValidateFunc& f) { + validate_func_ = f; + return *this; + } + + bool IsEnabled(OptionTypeFlags otf) const { return (flags_ & otf) == otf; } + + bool IsEditable(const ConfigOptions& opts) const { + if (opts.mutable_options_only) { + return IsMutable(); + } else { + return true; + } + } + bool IsMutable() const { return IsEnabled(OptionTypeFlags::kMutable); } + + bool IsDeprecated() const { + return IsEnabled(OptionVerificationType::kDeprecated); + } + + // Returns true if the option is marked as an Alias. + // Aliases are valid options that are parsed but are not converted to strings + // or compared. + bool IsAlias() const { return IsEnabled(OptionVerificationType::kAlias); } + + bool IsEnabled(OptionVerificationType ovf) const { + return verification_ == ovf; + } + + // Returns the sanity level for comparing the option. + // If the options should not be compared, returns None + // If the option has a compare flag, returns it. + // Otherwise, returns "exact" + ConfigOptions::SanityLevel GetSanityLevel() const { + if (IsDeprecated() || IsAlias()) { + return ConfigOptions::SanityLevel::kSanityLevelNone; + } else { + auto match = (flags_ & OptionTypeFlags::kCompareExact); + if (match == OptionTypeFlags::kCompareDefault) { + return ConfigOptions::SanityLevel::kSanityLevelExactMatch; + } else { + return (ConfigOptions::SanityLevel)match; + } + } + } + + // Returns true if the option should be serialized. + // Options should be serialized if the are not deprecated, aliases, + // or marked as "Don't Serialize". + bool ShouldSerialize() const { + if (IsDeprecated() || IsAlias()) { + return false; + } else if (IsEnabled(OptionTypeFlags::kDontSerialize)) { + return false; + } else { + return true; + } + } + + bool ShouldPrepare() const { + if (IsDeprecated() || IsAlias()) { + return false; + } else if (IsEnabled(OptionTypeFlags::kDontPrepare)) { + return false; + } else { + return (prepare_func_ != nullptr || IsConfigurable()); + } + } + + bool ShouldValidate() const { + if (IsDeprecated() || IsAlias()) { + return false; + } else { + return (validate_func_ != nullptr || IsConfigurable()); + } + } + + // Returns true if the option is allowed to be null. + // Options can be null if the verification type is allow from null + // or if the flags specify allow null. + bool CanBeNull() const { + return (IsEnabled(OptionTypeFlags::kAllowNull) || + IsEnabled(OptionVerificationType::kByNameAllowNull) || + IsEnabled(OptionVerificationType::kByNameAllowFromNull)); + } + + bool IsSharedPtr() const { return IsEnabled(OptionTypeFlags::kShared); } + + bool IsUniquePtr() const { return IsEnabled(OptionTypeFlags::kUnique); } + + bool IsRawPtr() const { return IsEnabled(OptionTypeFlags::kRawPointer); } + + bool IsByName() const { + return (verification_ == OptionVerificationType::kByName || + verification_ == OptionVerificationType::kByNameAllowNull || + verification_ == OptionVerificationType::kByNameAllowFromNull); + } + + bool IsStruct() const { return (type_ == OptionType::kStruct); } + + bool IsConfigurable() const { + return (type_ == OptionType::kConfigurable || + type_ == OptionType::kCustomizable); + } + + bool IsCustomizable() const { return (type_ == OptionType::kCustomizable); } + + inline const void* GetOffset(const void* base) const { + return static_cast(base) + offset_; + } + + inline void* GetOffset(void* base) const { + return static_cast(base) + offset_; + } + + template + const T* GetOffsetAs(const void* base) const { + const void* addr = GetOffset(base); + return static_cast(addr); + } + + template + T* GetOffsetAs(void* base) const { + void* addr = GetOffset(base); + return static_cast(addr); + } + + // Returns the underlying pointer for the type at base_addr + // The value returned is the underlying "raw" pointer, offset from base. + template + const T* AsRawPointer(const void* const base_addr) const { + if (base_addr == nullptr) { + return nullptr; + } + if (IsUniquePtr()) { + const auto ptr = GetOffsetAs>(base_addr); + return ptr->get(); + } else if (IsSharedPtr()) { + const auto ptr = GetOffsetAs>(base_addr); + return ptr->get(); + } else if (IsRawPtr()) { + const T* const* ptr = GetOffsetAs(base_addr); + return *ptr; + } else { + return GetOffsetAs(base_addr); + } + } + + // Returns the underlying pointer for the type at base_addr + // The value returned is the underlying "raw" pointer, offset from base. + template + T* AsRawPointer(void* base_addr) const { + if (base_addr == nullptr) { + return nullptr; + } + if (IsUniquePtr()) { + auto ptr = GetOffsetAs>(base_addr); + return ptr->get(); + } else if (IsSharedPtr()) { + auto ptr = GetOffsetAs>(base_addr); + return ptr->get(); + } else if (IsRawPtr()) { + auto ptr = GetOffsetAs(base_addr); + return *ptr; + } else { + return GetOffsetAs(base_addr); + } + } + + // Parses the option in "opt_value" according to the rules of this class + // and updates the value at "opt_ptr". + // On success, Status::OK() is returned. On failure: + // NotFound means the opt_name is not valid for this option + // NotSupported means we do not know how to parse the value for this option + // InvalidArgument means the opt_value is not valid for this option. + Status Parse(const ConfigOptions& config_options, const std::string& opt_name, + const std::string& opt_value, void* const opt_ptr) const; + + // Serializes the option in "opt_addr" according to the rules of this class + // into the value at "opt_value". + Status Serialize(const ConfigOptions& config_options, + const std::string& opt_name, const void* const opt_ptr, + std::string* opt_value) const; + + // Compares the "addr1" and "addr2" values according to the rules of this + // class and returns true if they match. On a failed match, mismatch is the + // name of the option that failed to match. + bool AreEqual(const ConfigOptions& config_options, + const std::string& opt_name, const void* const addr1, + const void* const addr2, std::string* mismatch) const; + + // Used to override the match rules for "ByName" options. + bool AreEqualByName(const ConfigOptions& config_options, + const std::string& opt_name, const void* const this_ptr, + const void* const that_ptr) const; + bool AreEqualByName(const ConfigOptions& config_options, + const std::string& opt_name, const void* const this_ptr, + const std::string& that_value) const; + + Status Prepare(const ConfigOptions& config_options, const std::string& name, + void* opt_ptr) const; + Status Validate(const DBOptions& db_opts, const ColumnFamilyOptions& cf_opts, + const std::string& name, const void* opt_ptr) const; + + // Parses the input opts_map according to the type_map for the opt_addr + // For each name-value pair in opts_map, find the corresponding name in + // type_map If the name is found: + // - set the corresponding value in opt_addr, returning the status on + // failure; + // If the name is not found: + // - If unused is specified, add the name-value to unused and continue + // - If ingore_unknown_options is false, return NotFound + // Returns OK if all options were either: + // - Successfully set + // - options were not found and ignore_unknown_options=true + // - options were not found and unused was specified + // Note that this method is much less sophisticated than the comparable + // Configurable::Configure methods. For example, on error, there is no + // attempt to return opt_addr to the initial state. Additionally, there + // is no effort to initialize (Configurable::PrepareOptions) the object + // on success. This method should typically only be used for simpler, + // standalone structures and not those that contain shared and embedded + // objects. + static Status ParseType( + const ConfigOptions& config_options, const std::string& opts_str, + const std::unordered_map& type_map, + void* opt_addr, + std::unordered_map* unused = nullptr); + static Status ParseType( + const ConfigOptions& config_options, + const std::unordered_map& opts_map, + const std::unordered_map& type_map, + void* opt_addr, + std::unordered_map* unused = nullptr); + + // Parses the input value according to the map for the struct at opt_addr + // struct_name is the name of the struct option as registered + // opt_name is the name of the option being evaluated. This may + // be the whole struct or a sub-element of it, based on struct_name and + // opt_name. + static Status ParseStruct( + const ConfigOptions& config_options, const std::string& struct_name, + const std::unordered_map* map, + const std::string& opt_name, const std::string& value, void* opt_addr); + + // Serializes the values from opt_addr using the rules in type_map. + // Returns the serialized form in result. + // Returns OK on success or non-OK if some option could not be serialized. + static Status SerializeType( + const ConfigOptions& config_options, + const std::unordered_map& type_map, + const void* opt_addr, std::string* value); + + // Serializes the input addr according to the map for the struct to value. + // struct_name is the name of the struct option as registered + // opt_name is the name of the option being evaluated. This may + // be the whole struct or a sub-element of it + static Status SerializeStruct( + const ConfigOptions& config_options, const std::string& struct_name, + const std::unordered_map* map, + const std::string& opt_name, const void* opt_addr, std::string* value); + + // Compares the values in this_addr and that_addr using the rules in type_map. + // If the values are equal, returns true + // If the values are not equal, returns false and sets mismatch to the name + // of the first value that did not match. + static bool TypesAreEqual( + const ConfigOptions& config_options, + const std::unordered_map& map, + const void* this_addr, const void* that_addr, std::string* mismatch); + + // Compares the input offsets according to the map for the struct and returns + // true if they are equivalent, false otherwise. + // struct_name is the name of the struct option as registered + // opt_name is the name of the option being evaluated. This may + // be the whole struct or a sub-element of it + static bool StructsAreEqual( + const ConfigOptions& config_options, const std::string& struct_name, + const std::unordered_map* map, + const std::string& opt_name, const void* this_offset, + const void* that_offset, std::string* mismatch); + + // Finds the entry for the opt_name in the opt_map, returning + // nullptr if not found. + // If found, elem_name will be the name of option to find. + // This may be opt_name, or a substring of opt_name. + // For "simple" options, opt_name will be equal to elem_name. Given the + // opt_name "opt", elem_name will equal "opt". + // For "embedded" options (like structs), elem_name may be opt_name + // or a field within the opt_name. For example, given the struct "struct", + // and opt_name of "struct.field", elem_name will be "field" + static const OptionTypeInfo* Find( + const std::string& opt_name, + const std::unordered_map& opt_map, + std::string* elem_name); + + // Returns the next token marked by the delimiter from "opts" after start in + // token and updates end to point to where that token stops. Delimiters inside + // of braces are ignored. Returns OK if a token is found and an error if the + // input opts string is mis-formatted. + // Given "a=AA;b=BB;" start=2 and delimiter=";", token is "AA" and end points + // to "b" Given "{a=A;b=B}", the token would be "a=A;b=B" + // + // @param opts The string in which to find the next token + // @param delimiter The delimiter between tokens + // @param start The position in opts to start looking for the token + // @param ed Returns the end position in opts of the token + // @param token Returns the token + // @returns OK if a token was found + // @return InvalidArgument if the braces mismatch + // (e.g. "{a={b=c;}" ) -- missing closing brace + // @return InvalidArgument if an expected delimiter is not found + // e.g. "{a=b}c=d;" -- missing delimiter before "c" + static Status NextToken(const std::string& opts, char delimiter, size_t start, + size_t* end, std::string* token); + + constexpr static const char* kIdPropName() { return "id"; } + constexpr static const char* kIdPropSuffix() { return ".id"; } + + private: + int offset_; + + // The optional function to convert a string to its representation + ParseFunc parse_func_; + + // The optional function to convert a value to its string representation + SerializeFunc serialize_func_; + + // The optional function to match two option values + EqualsFunc equals_func_; + + PrepareFunc prepare_func_; + ValidateFunc validate_func_; + OptionType type_; + OptionVerificationType verification_; + OptionTypeFlags flags_; +}; + +// Parses the input value into elements of the result array, which has fixed +// array size. For example, if the value=1:2:3 and elem_info parses integers, +// the result array will be {1,2,3}. Array size is defined in the OptionTypeInfo +// the input value has to match with that. +// @param config_options Controls how the option value is parsed. +// @param elem_info Controls how individual tokens in value are parsed +// @param separator Character separating tokens in values (':' in the above +// example) +// @param name The name associated with this array option +// @param value The input string to parse into tokens +// @param result Returns the results of parsing value into its elements. +// @return OK if the value was successfully parse +// @return InvalidArgument if the value is improperly formed or element number +// doesn't match array size defined in OptionTypeInfo +// or if the token could not be parsed +// @return NotFound If the tokenized value contains unknown options for +// its type +template +Status ParseArray(const ConfigOptions& config_options, + const OptionTypeInfo& elem_info, char separator, + const std::string& name, const std::string& value, + std::array* result) { + Status status; + + ConfigOptions copy = config_options; + copy.ignore_unsupported_options = false; + size_t i = 0, start = 0, end = 0; + for (; status.ok() && i < kSize && start < value.size() && + end != std::string::npos; + i++, start = end + 1) { + std::string token; + status = OptionTypeInfo::NextToken(value, separator, start, &end, &token); + if (status.ok()) { + status = elem_info.Parse(copy, name, token, &((*result)[i])); + if (config_options.ignore_unsupported_options && + status.IsNotSupported()) { + // If we were ignoring unsupported options and this one should be + // ignored, ignore it by setting the status to OK + status = Status::OK(); + } + } + } + if (!status.ok()) { + return status; + } + // make sure the element number matches the array size + if (i < kSize) { + return Status::InvalidArgument( + "Serialized value has less elements than array size", name); + } + if (start < value.size() && end != std::string::npos) { + return Status::InvalidArgument( + "Serialized value has more elements than array size", name); + } + return status; +} + +// Serializes the fixed size input array into its output value. Elements are +// separated by the separator character. This element will convert all of the +// elements in array into their serialized form, using elem_info to perform the +// serialization. +// For example, if the array contains the integers 1,2,3 and elem_info +// serializes the output would be 1:2:3 for separator ":". +// @param config_options Controls how the option value is serialized. +// @param elem_info Controls how individual tokens in value are serialized +// @param separator Character separating tokens in value (':' in the above +// example) +// @param name The name associated with this array option +// @param array The input array to serialize +// @param value The output string of serialized options +// @return OK if the value was successfully parse +// @return InvalidArgument if the value is improperly formed or if the token +// could not be parsed +// @return NotFound If the tokenized value contains unknown options for +// its type +template +Status SerializeArray(const ConfigOptions& config_options, + const OptionTypeInfo& elem_info, char separator, + const std::string& name, + const std::array& array, std::string* value) { + std::string result; + ConfigOptions embedded = config_options; + embedded.delimiter = ";"; + int printed = 0; + for (const auto& elem : array) { + std::string elem_str; + Status s = elem_info.Serialize(embedded, name, &elem, &elem_str); + if (!s.ok()) { + return s; + } else if (!elem_str.empty()) { + if (printed++ > 0) { + result += separator; + } + // If the element contains embedded separators, put it inside of brackets + if (elem_str.find(separator) != std::string::npos) { + result += "{" + elem_str + "}"; + } else { + result += elem_str; + } + } + } + if (result.find("=") != std::string::npos) { + *value = "{" + result + "}"; + } else if (printed > 1 && result.at(0) == '{') { + *value = "{" + result + "}"; + } else { + *value = result; + } + return Status::OK(); +} + +// Compares the input arrays array1 and array2 for equality +// Elements of the array are compared one by one using elem_info to perform the +// comparison. +// +// @param config_options Controls how the arrays are compared. +// @param elem_info Controls how individual elements in the arrays are compared +// @param name The name associated with this array option +// @param array1,array2 The arrays to compare. +// @param mismatch If the arrays are not equivalent, mismatch will point to +// the first element of the comparison that did not match. +// @return true If vec1 and vec2 are "equal", false otherwise +template +bool ArraysAreEqual(const ConfigOptions& config_options, + const OptionTypeInfo& elem_info, const std::string& name, + const std::array& array1, + const std::array& array2, std::string* mismatch) { + assert(array1.size() == kSize); + assert(array2.size() == kSize); + for (size_t i = 0; i < kSize; ++i) { + if (!elem_info.AreEqual(config_options, name, &array1[i], &array2[i], + mismatch)) { + return false; + } + } + return true; +} + +// Parses the input value into elements of the result vector. This method +// will break the input value into the individual tokens (based on the +// separator), where each of those tokens will be parsed based on the rules of +// elem_info. The result vector will be populated with elements based on the +// input tokens. For example, if the value=1:2:3:4:5 and elem_info parses +// integers, the result vector will contain the integers 1,2,3,4,5 +// @param config_options Controls how the option value is parsed. +// @param elem_info Controls how individual tokens in value are parsed +// @param separator Character separating tokens in values (':' in the above +// example) +// @param name The name associated with this vector option +// @param value The input string to parse into tokens +// @param result Returns the results of parsing value into its elements. +// @return OK if the value was successfully parse +// @return InvalidArgument if the value is improperly formed or if the token +// could not be parsed +// @return NotFound If the tokenized value contains unknown options for +// its type +template +Status ParseVector(const ConfigOptions& config_options, + const OptionTypeInfo& elem_info, char separator, + const std::string& name, const std::string& value, + std::vector* result) { + result->clear(); + Status status; + + // Turn off ignore_unknown_objects so we can tell if the returned + // object is valid or not. + ConfigOptions copy = config_options; + copy.ignore_unsupported_options = false; + for (size_t start = 0, end = 0; + status.ok() && start < value.size() && end != std::string::npos; + start = end + 1) { + std::string token; + status = OptionTypeInfo::NextToken(value, separator, start, &end, &token); + if (status.ok()) { + T elem; + status = elem_info.Parse(copy, name, token, &elem); + if (status.ok()) { + result->emplace_back(elem); + } else if (config_options.ignore_unsupported_options && + status.IsNotSupported()) { + // If we were ignoring unsupported options and this one should be + // ignored, ignore it by setting the status to OK + status = Status::OK(); + } + } + } + return status; +} + +// Serializes the input vector into its output value. Elements are +// separated by the separator character. This element will convert all of the +// elements in vec into their serialized form, using elem_info to perform the +// serialization. +// For example, if the vec contains the integers 1,2,3,4,5 and elem_info +// serializes the output would be 1:2:3:4:5 for separator ":". +// @param config_options Controls how the option value is serialized. +// @param elem_info Controls how individual tokens in value are serialized +// @param separator Character separating tokens in value (':' in the above +// example) +// @param name The name associated with this vector option +// @param vec The input vector to serialize +// @param value The output string of serialized options +// @return OK if the value was successfully parse +// @return InvalidArgument if the value is improperly formed or if the token +// could not be parsed +// @return NotFound If the tokenized value contains unknown options for +// its type +template +Status SerializeVector(const ConfigOptions& config_options, + const OptionTypeInfo& elem_info, char separator, + const std::string& name, const std::vector& vec, + std::string* value) { + std::string result; + ConfigOptions embedded = config_options; + embedded.delimiter = ";"; + int printed = 0; + for (const auto& elem : vec) { + std::string elem_str; + Status s = elem_info.Serialize(embedded, name, &elem, &elem_str); + if (!s.ok()) { + return s; + } else if (!elem_str.empty()) { + if (printed++ > 0) { + result += separator; + } + // If the element contains embedded separators, put it inside of brackets + if (elem_str.find(separator) != std::string::npos) { + result += "{" + elem_str + "}"; + } else { + result += elem_str; + } + } + } + if (result.find("=") != std::string::npos) { + *value = "{" + result + "}"; + } else if (printed > 1 && result.at(0) == '{') { + *value = "{" + result + "}"; + } else { + *value = result; + } + return Status::OK(); +} + +// Compares the input vectors vec1 and vec2 for equality +// If the vectors are the same size, elements of the vectors are compared one by +// one using elem_info to perform the comparison. +// +// @param config_options Controls how the vectors are compared. +// @param elem_info Controls how individual elements in the vectors are compared +// @param name The name associated with this vector option +// @param vec1,vec2 The vectors to compare. +// @param mismatch If the vectors are not equivalent, mismatch will point to +// the first +// element of the comparison that did not match. +// @return true If vec1 and vec2 are "equal", false otherwise +template +bool VectorsAreEqual(const ConfigOptions& config_options, + const OptionTypeInfo& elem_info, const std::string& name, + const std::vector& vec1, const std::vector& vec2, + std::string* mismatch) { + if (vec1.size() != vec2.size()) { + *mismatch = name; + return false; + } else { + for (size_t i = 0; i < vec1.size(); ++i) { + if (!elem_info.AreEqual( + config_options, name, reinterpret_cast(&vec1[i]), + reinterpret_cast(&vec2[i]), mismatch)) { + return false; + } + } + return true; + } +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/options_util.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/options_util.h new file mode 100644 index 0000000..8d9488f --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/options_util.h @@ -0,0 +1,105 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +// This file contains utility functions for RocksDB Options. +#pragma once + + +#include +#include + +#include "rocksdb/convenience.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { +struct ConfigOptions; +// Constructs the DBOptions and ColumnFamilyDescriptors by loading the +// latest RocksDB options file stored in the specified rocksdb database. +// +// Note that the all the pointer options (except table_factory, which will +// be described in more details below) will be initialized with the default +// values. Developers can further initialize them after this function call. +// Below is an example list of pointer options which will be initialized +// +// * env +// * memtable_factory +// * compaction_filter_factory +// * prefix_extractor +// * comparator +// * merge_operator +// * compaction_filter +// +// User can also choose to load customized comparator, env, and/or +// merge_operator through object registry: +// * comparator needs to be registered through Registrar +// * env needs to be registered through Registrar +// * merge operator needs to be registered through +// Registrar>. +// +// For table_factory, this function further supports deserializing +// BlockBasedTableFactory and its BlockBasedTableOptions except the +// pointer options of BlockBasedTableOptions (flush_block_policy_factory, +// block_cache, and block_cache_compressed), which will be initialized with +// default values. Developers can further specify these three options by +// casting the return value of TableFactory::GetOptions() to +// BlockBasedTableOptions and making necessary changes. +// +// config_options contains a set of options that controls the processing +// of the options. +// +// config_options.ignore_unknown_options can be set to true if you want to +// ignore options that are from a newer version of the db, essentially for +// forward compatibility. +// +// examples/options_file_example.cc demonstrates how to use this function +// to open a RocksDB instance. +// +// @return the function returns an OK status when it went successfully. If +// the specified "dbpath" does not contain any option file, then a +// Status::NotFound will be returned. A return value other than +// Status::OK or Status::NotFound indicates there is some error related +// to the options file itself. +// +// @see LoadOptionsFromFile +Status LoadLatestOptions(const ConfigOptions& config_options, + const std::string& dbpath, DBOptions* db_options, + std::vector* cf_descs, + std::shared_ptr* cache = {}); + +// Similar to LoadLatestOptions, this function constructs the DBOptions +// and ColumnFamilyDescriptors based on the specified RocksDB Options file. +// +// @see LoadLatestOptions +Status LoadOptionsFromFile(const ConfigOptions& config_options, + const std::string& options_file_name, + DBOptions* db_options, + std::vector* cf_descs, + std::shared_ptr* cache = {}); + +// Returns the latest options file name under the specified db path. +Status GetLatestOptionsFileName(const std::string& dbpath, Env* env, + std::string* options_file_name); + +// Returns Status::OK if the input DBOptions and ColumnFamilyDescriptors +// are compatible with the latest options stored in the specified DB path. +// +// If the return status is non-ok, it means the specified RocksDB instance +// might not be correctly opened with the input set of options. Currently, +// changing one of the following options will fail the compatibility check: +// +// * comparator +// * prefix_extractor +// * table_factory +// * merge_operator +// * persist_user_defined_timestamps +Status CheckOptionsCompatibility( + const ConfigOptions& config_options, const std::string& dbpath, + const DBOptions& db_options, + const std::vector& cf_descs); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/replayer.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/replayer.h new file mode 100644 index 0000000..fc53199 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/replayer.h @@ -0,0 +1,85 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include + +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +class TraceRecord; +class TraceRecordResult; + +struct ReplayOptions { + // Number of threads used for replaying. If 0 or 1, replay using + // single thread. + uint32_t num_threads; + + // Enables fast forwarding a replay by increasing/reducing the delay between + // the ingested traces. + // If > 0.0 and < 1.0, slow down the replay by this amount. + // If 1.0, replay the operations at the same rate as in the trace stream. + // If > 1, speed up the replay by this amount. + double fast_forward; + + ReplayOptions() : num_threads(1), fast_forward(1.0) {} + + ReplayOptions(uint32_t num_of_threads, double fast_forward_ratio) + : num_threads(num_of_threads), fast_forward(fast_forward_ratio) {} +}; + +// Replayer helps to replay the captured RocksDB query level operations. +// The Replayer can either be created from DB::NewReplayer method, or be +// instantiated via db_bench today, on using "replay" benchmark. +class Replayer { + public: + virtual ~Replayer() = default; + + // Make some preparation before replaying the trace. This will also reset the + // replayer in order to restart replaying. + virtual Status Prepare() = 0; + + // Return the timestamp when the trace recording was started. + virtual uint64_t GetHeaderTimestamp() const = 0; + + // Atomically read one trace into a TraceRecord (excluding the header and + // footer traces). + // Return Status::OK() on success; + // Status::Incomplete() if Prepare() was not called or no more available + // trace; + // Status::NotSupported() if the read trace type is not supported. + virtual Status Next(std::unique_ptr* record) = 0; + + // Execute one TraceRecord. + // Return Status::OK() if the execution was successful. Get/MultiGet traces + // will still return Status::OK() even if they got Status::NotFound() + // from DB::Get() or DB::MultiGet(); + // Status::Incomplete() if Prepare() was not called or no more available + // trace; + // Status::NotSupported() if the operation is not supported; + // Otherwise, return the corresponding error status. + // + // The actual operation execution status and result(s) will be saved in + // result. For example, a GetQueryTraceRecord will have its DB::Get() status + // and the returned value saved in a SingleValueTraceExecutionResult. + virtual Status Execute(const std::unique_ptr& record, + std::unique_ptr* result) = 0; + + // Replay all the traces from the provided trace stream, taking the delay + // between the traces into consideration. + // + // result_callback reports the status of executing a trace record, and the + // actual operation execution result (See the description for Execute()). + virtual Status Replay( + const ReplayOptions& options, + const std::function&&)>& + result_callback) = 0; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/sim_cache.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/sim_cache.h new file mode 100644 index 0000000..6c52453 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/sim_cache.h @@ -0,0 +1,93 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include +#include + +#include "rocksdb/advanced_cache.h" +#include "rocksdb/env.h" +#include "rocksdb/slice.h" +#include "rocksdb/statistics.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +class SimCache; + +// For instrumentation purpose, use NewSimCache instead of NewLRUCache API +// NewSimCache is a wrapper function returning a SimCache instance that can +// have additional interface provided in Simcache class besides Cache interface +// to predict block cache hit rate without actually allocating the memory. It +// can help users tune their current block cache size, and determine how +// efficient they are using the memory. +// +// Since GetSimCapacity() returns the capacity for simulation, it differs from +// actual memory usage, which can be estimated as: +// sim_capacity * entry_size / (entry_size + block_size), +// where 76 <= entry_size <= 104, +// BlockBasedTableOptions.block_size = 4096 by default but is configurable, +// Therefore, generally the actual memory overhead of SimCache is Less than +// sim_capacity * 2% +extern std::shared_ptr NewSimCache(std::shared_ptr cache, + size_t sim_capacity, + int num_shard_bits); + +extern std::shared_ptr NewSimCache(std::shared_ptr sim_cache, + std::shared_ptr cache, + int num_shard_bits); + +// An abstract base class (public interface) to the SimCache implementation +class SimCache : public CacheWrapper { + public: + using CacheWrapper::CacheWrapper; + + // returns the maximum configured capacity of the simcache for simulation + virtual size_t GetSimCapacity() const = 0; + + // simcache doesn't provide internal handler reference to user, so always + // PinnedUsage = 0 and the behavior will be not exactly consistent the + // with real cache. + // returns the memory size for the entries residing in the simcache. + virtual size_t GetSimUsage() const = 0; + + // sets the maximum configured capacity of the simcache. When the new + // capacity is less than the old capacity and the existing usage is + // greater than new capacity, the implementation will purge old entries + // to fit new capacity. + virtual void SetSimCapacity(size_t capacity) = 0; + + // returns the lookup times of simcache + virtual uint64_t get_miss_counter() const = 0; + // returns the hit times of simcache + virtual uint64_t get_hit_counter() const = 0; + // reset the lookup and hit counters + virtual void reset_counter() = 0; + // String representation of the statistics of the simcache + virtual std::string ToString() const = 0; + + // Start storing logs of the cache activity (Add/Lookup) into + // a file located at activity_log_file, max_logging_size option can be used to + // stop logging to the file automatically after reaching a specific size in + // bytes, a values of 0 disable this feature + virtual Status StartActivityLogging(const std::string& activity_log_file, + Env* env, + uint64_t max_logging_size = 0) = 0; + + // Stop cache activity logging if any + virtual void StopActivityLogging() = 0; + + // Status of cache logging happening in background + virtual Status GetActivityLoggingStatus() = 0; + + private: + SimCache(const SimCache&); + SimCache& operator=(const SimCache&); +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/stackable_db.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/stackable_db.h new file mode 100644 index 0000000..8674f10 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/stackable_db.h @@ -0,0 +1,601 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include +#include +#include + +#include "rocksdb/db.h" + +#ifdef _WIN32 +// Windows API macro interference +#undef DeleteFile +#endif + +namespace ROCKSDB_NAMESPACE { + +// This class contains APIs to stack rocksdb wrappers.Eg. Stack TTL over base d +class StackableDB : public DB { + public: + // StackableDB take sole ownership of the underlying db. + explicit StackableDB(DB* db) : db_(db) {} + + // StackableDB take shared ownership of the underlying db. + explicit StackableDB(std::shared_ptr db) + : db_(db.get()), shared_db_ptr_(db) {} + + ~StackableDB() { + if (shared_db_ptr_ == nullptr) { + delete db_; + } else { + assert(shared_db_ptr_.get() == db_); + } + db_ = nullptr; + } + + virtual Status Close() override { return db_->Close(); } + + virtual DB* GetBaseDB() { return db_; } + + virtual DB* GetRootDB() override { return db_->GetRootDB(); } + + virtual Status CreateColumnFamily(const ColumnFamilyOptions& options, + const std::string& column_family_name, + ColumnFamilyHandle** handle) override { + return db_->CreateColumnFamily(options, column_family_name, handle); + } + + virtual Status CreateColumnFamilies( + const ColumnFamilyOptions& options, + const std::vector& column_family_names, + std::vector* handles) override { + return db_->CreateColumnFamilies(options, column_family_names, handles); + } + + virtual Status CreateColumnFamilies( + const std::vector& column_families, + std::vector* handles) override { + return db_->CreateColumnFamilies(column_families, handles); + } + + virtual Status DropColumnFamily(ColumnFamilyHandle* column_family) override { + return db_->DropColumnFamily(column_family); + } + + virtual Status DropColumnFamilies( + const std::vector& column_families) override { + return db_->DropColumnFamilies(column_families); + } + + virtual Status DestroyColumnFamilyHandle( + ColumnFamilyHandle* column_family) override { + return db_->DestroyColumnFamilyHandle(column_family); + } + + using DB::Put; + virtual Status Put(const WriteOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + const Slice& val) override { + return db_->Put(options, column_family, key, val); + } + Status Put(const WriteOptions& options, ColumnFamilyHandle* column_family, + const Slice& key, const Slice& ts, const Slice& val) override { + return db_->Put(options, column_family, key, ts, val); + } + + using DB::PutEntity; + Status PutEntity(const WriteOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + const WideColumns& columns) override { + return db_->PutEntity(options, column_family, key, columns); + } + + using DB::Get; + virtual Status Get(const ReadOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value) override { + return db_->Get(options, column_family, key, value); + } + + using DB::GetEntity; + Status GetEntity(const ReadOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableWideColumns* columns) override { + return db_->GetEntity(options, column_family, key, columns); + } + + using DB::GetMergeOperands; + virtual Status GetMergeOperands( + const ReadOptions& options, ColumnFamilyHandle* column_family, + const Slice& key, PinnableSlice* slice, + GetMergeOperandsOptions* get_merge_operands_options, + int* number_of_operands) override { + return db_->GetMergeOperands(options, column_family, key, slice, + get_merge_operands_options, + number_of_operands); + } + + using DB::MultiGet; + virtual std::vector MultiGet( + const ReadOptions& options, + const std::vector& column_family, + const std::vector& keys, + std::vector* values) override { + return db_->MultiGet(options, column_family, keys, values); + } + + virtual void MultiGet(const ReadOptions& options, + ColumnFamilyHandle* column_family, + const size_t num_keys, const Slice* keys, + PinnableSlice* values, Status* statuses, + const bool sorted_input = false) override { + return db_->MultiGet(options, column_family, num_keys, keys, values, + statuses, sorted_input); + } + + using DB::MultiGetEntity; + + void MultiGetEntity(const ReadOptions& options, + ColumnFamilyHandle* column_family, size_t num_keys, + const Slice* keys, PinnableWideColumns* results, + Status* statuses, bool sorted_input) override { + db_->MultiGetEntity(options, column_family, num_keys, keys, results, + statuses, sorted_input); + } + + void MultiGetEntity(const ReadOptions& options, size_t num_keys, + ColumnFamilyHandle** column_families, const Slice* keys, + PinnableWideColumns* results, Status* statuses, + bool sorted_input) override { + db_->MultiGetEntity(options, num_keys, column_families, keys, results, + statuses, sorted_input); + } + + using DB::IngestExternalFile; + virtual Status IngestExternalFile( + ColumnFamilyHandle* column_family, + const std::vector& external_files, + const IngestExternalFileOptions& options) override { + return db_->IngestExternalFile(column_family, external_files, options); + } + + using DB::IngestExternalFiles; + virtual Status IngestExternalFiles( + const std::vector& args) override { + return db_->IngestExternalFiles(args); + } + + using DB::CreateColumnFamilyWithImport; + virtual Status CreateColumnFamilyWithImport( + const ColumnFamilyOptions& options, const std::string& column_family_name, + const ImportColumnFamilyOptions& import_options, + const ExportImportFilesMetaData& metadata, + ColumnFamilyHandle** handle) override { + return db_->CreateColumnFamilyWithImport(options, column_family_name, + import_options, metadata, handle); + } + + virtual Status CreateColumnFamilyWithImport( + const ColumnFamilyOptions& options, const std::string& column_family_name, + const ImportColumnFamilyOptions& import_options, + const std::vector& metadatas, + ColumnFamilyHandle** handle) override { + return db_->CreateColumnFamilyWithImport(options, column_family_name, + import_options, metadatas, handle); + } + + using DB::ClipColumnFamily; + virtual Status ClipColumnFamily(ColumnFamilyHandle* column_family, + const Slice& begin_key, + const Slice& end_key) override { + return db_->ClipColumnFamily(column_family, begin_key, end_key); + } + + using DB::VerifyFileChecksums; + Status VerifyFileChecksums(const ReadOptions& read_opts) override { + return db_->VerifyFileChecksums(read_opts); + } + + virtual Status VerifyChecksum() override { return db_->VerifyChecksum(); } + + virtual Status VerifyChecksum(const ReadOptions& options) override { + return db_->VerifyChecksum(options); + } + + using DB::KeyMayExist; + virtual bool KeyMayExist(const ReadOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + std::string* value, + bool* value_found = nullptr) override { + return db_->KeyMayExist(options, column_family, key, value, value_found); + } + + using DB::Delete; + virtual Status Delete(const WriteOptions& wopts, + ColumnFamilyHandle* column_family, + const Slice& key) override { + return db_->Delete(wopts, column_family, key); + } + Status Delete(const WriteOptions& wopts, ColumnFamilyHandle* column_family, + const Slice& key, const Slice& ts) override { + return db_->Delete(wopts, column_family, key, ts); + } + + using DB::SingleDelete; + virtual Status SingleDelete(const WriteOptions& wopts, + ColumnFamilyHandle* column_family, + const Slice& key) override { + return db_->SingleDelete(wopts, column_family, key); + } + Status SingleDelete(const WriteOptions& wopts, + ColumnFamilyHandle* column_family, const Slice& key, + const Slice& ts) override { + return db_->SingleDelete(wopts, column_family, key, ts); + } + + using DB::DeleteRange; + Status DeleteRange(const WriteOptions& wopts, + ColumnFamilyHandle* column_family, const Slice& start_key, + const Slice& end_key) override { + return db_->DeleteRange(wopts, column_family, start_key, end_key); + } + + using DB::Merge; + virtual Status Merge(const WriteOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + const Slice& value) override { + return db_->Merge(options, column_family, key, value); + } + Status Merge(const WriteOptions& options, ColumnFamilyHandle* column_family, + const Slice& key, const Slice& ts, const Slice& value) override { + return db_->Merge(options, column_family, key, ts, value); + } + + virtual Status Write(const WriteOptions& opts, WriteBatch* updates) override { + return db_->Write(opts, updates); + } + + using DB::NewIterator; + virtual Iterator* NewIterator(const ReadOptions& opts, + ColumnFamilyHandle* column_family) override { + return db_->NewIterator(opts, column_family); + } + + virtual Status NewIterators( + const ReadOptions& options, + const std::vector& column_families, + std::vector* iterators) override { + return db_->NewIterators(options, column_families, iterators); + } + + virtual const Snapshot* GetSnapshot() override { return db_->GetSnapshot(); } + + virtual void ReleaseSnapshot(const Snapshot* snapshot) override { + return db_->ReleaseSnapshot(snapshot); + } + + using DB::GetMapProperty; + using DB::GetProperty; + virtual bool GetProperty(ColumnFamilyHandle* column_family, + const Slice& property, std::string* value) override { + return db_->GetProperty(column_family, property, value); + } + virtual bool GetMapProperty( + ColumnFamilyHandle* column_family, const Slice& property, + std::map* value) override { + return db_->GetMapProperty(column_family, property, value); + } + + using DB::GetIntProperty; + virtual bool GetIntProperty(ColumnFamilyHandle* column_family, + const Slice& property, uint64_t* value) override { + return db_->GetIntProperty(column_family, property, value); + } + + using DB::GetAggregatedIntProperty; + virtual bool GetAggregatedIntProperty(const Slice& property, + uint64_t* value) override { + return db_->GetAggregatedIntProperty(property, value); + } + + using DB::GetApproximateSizes; + virtual Status GetApproximateSizes(const SizeApproximationOptions& options, + ColumnFamilyHandle* column_family, + const Range* r, int n, + uint64_t* sizes) override { + return db_->GetApproximateSizes(options, column_family, r, n, sizes); + } + + using DB::GetApproximateMemTableStats; + virtual void GetApproximateMemTableStats(ColumnFamilyHandle* column_family, + const Range& range, + uint64_t* const count, + uint64_t* const size) override { + return db_->GetApproximateMemTableStats(column_family, range, count, size); + } + + using DB::CompactRange; + virtual Status CompactRange(const CompactRangeOptions& options, + ColumnFamilyHandle* column_family, + const Slice* begin, const Slice* end) override { + return db_->CompactRange(options, column_family, begin, end); + } + + using DB::CompactFiles; + virtual Status CompactFiles( + const CompactionOptions& compact_options, + ColumnFamilyHandle* column_family, + const std::vector& input_file_names, const int output_level, + const int output_path_id = -1, + std::vector* const output_file_names = nullptr, + CompactionJobInfo* compaction_job_info = nullptr) override { + return db_->CompactFiles(compact_options, column_family, input_file_names, + output_level, output_path_id, output_file_names, + compaction_job_info); + } + + virtual Status PauseBackgroundWork() override { + return db_->PauseBackgroundWork(); + } + virtual Status ContinueBackgroundWork() override { + return db_->ContinueBackgroundWork(); + } + + virtual Status EnableAutoCompaction( + const std::vector& column_family_handles) override { + return db_->EnableAutoCompaction(column_family_handles); + } + + virtual void EnableManualCompaction() override { + return db_->EnableManualCompaction(); + } + virtual void DisableManualCompaction() override { + return db_->DisableManualCompaction(); + } + + virtual Status WaitForCompact( + const WaitForCompactOptions& wait_for_compact_options) override { + return db_->WaitForCompact(wait_for_compact_options); + } + + using DB::NumberLevels; + virtual int NumberLevels(ColumnFamilyHandle* column_family) override { + return db_->NumberLevels(column_family); + } + + using DB::MaxMemCompactionLevel; + virtual int MaxMemCompactionLevel( + ColumnFamilyHandle* column_family) override { + return db_->MaxMemCompactionLevel(column_family); + } + + using DB::Level0StopWriteTrigger; + virtual int Level0StopWriteTrigger( + ColumnFamilyHandle* column_family) override { + return db_->Level0StopWriteTrigger(column_family); + } + + virtual const std::string& GetName() const override { return db_->GetName(); } + + virtual Env* GetEnv() const override { return db_->GetEnv(); } + + virtual FileSystem* GetFileSystem() const override { + return db_->GetFileSystem(); + } + + using DB::GetOptions; + virtual Options GetOptions(ColumnFamilyHandle* column_family) const override { + return db_->GetOptions(column_family); + } + + using DB::GetDBOptions; + virtual DBOptions GetDBOptions() const override { + return db_->GetDBOptions(); + } + + using DB::Flush; + virtual Status Flush(const FlushOptions& fopts, + ColumnFamilyHandle* column_family) override { + return db_->Flush(fopts, column_family); + } + virtual Status Flush( + const FlushOptions& fopts, + const std::vector& column_families) override { + return db_->Flush(fopts, column_families); + } + + virtual Status SyncWAL() override { return db_->SyncWAL(); } + + virtual Status FlushWAL(bool sync) override { return db_->FlushWAL(sync); } + + virtual Status LockWAL() override { return db_->LockWAL(); } + + virtual Status UnlockWAL() override { return db_->UnlockWAL(); } + + + virtual Status DisableFileDeletions() override { + return db_->DisableFileDeletions(); + } + + virtual Status EnableFileDeletions(bool force) override { + return db_->EnableFileDeletions(force); + } + + virtual void GetLiveFilesMetaData( + std::vector* metadata) override { + db_->GetLiveFilesMetaData(metadata); + } + + virtual Status GetLiveFilesChecksumInfo( + FileChecksumList* checksum_list) override { + return db_->GetLiveFilesChecksumInfo(checksum_list); + } + + virtual Status GetLiveFilesStorageInfo( + const LiveFilesStorageInfoOptions& opts, + std::vector* files) override { + return db_->GetLiveFilesStorageInfo(opts, files); + } + + virtual void GetColumnFamilyMetaData(ColumnFamilyHandle* column_family, + ColumnFamilyMetaData* cf_meta) override { + db_->GetColumnFamilyMetaData(column_family, cf_meta); + } + + using DB::StartBlockCacheTrace; + Status StartBlockCacheTrace( + const TraceOptions& trace_options, + std::unique_ptr&& trace_writer) override { + return db_->StartBlockCacheTrace(trace_options, std::move(trace_writer)); + } + + Status StartBlockCacheTrace( + const BlockCacheTraceOptions& options, + std::unique_ptr&& trace_writer) override { + return db_->StartBlockCacheTrace(options, std::move(trace_writer)); + } + + using DB::EndBlockCacheTrace; + Status EndBlockCacheTrace() override { return db_->EndBlockCacheTrace(); } + + using DB::StartIOTrace; + Status StartIOTrace(const TraceOptions& options, + std::unique_ptr&& trace_writer) override { + return db_->StartIOTrace(options, std::move(trace_writer)); + } + + using DB::EndIOTrace; + Status EndIOTrace() override { return db_->EndIOTrace(); } + + using DB::StartTrace; + Status StartTrace(const TraceOptions& options, + std::unique_ptr&& trace_writer) override { + return db_->StartTrace(options, std::move(trace_writer)); + } + + using DB::EndTrace; + Status EndTrace() override { return db_->EndTrace(); } + + using DB::NewDefaultReplayer; + Status NewDefaultReplayer(const std::vector& handles, + std::unique_ptr&& reader, + std::unique_ptr* replayer) override { + return db_->NewDefaultReplayer(handles, std::move(reader), replayer); + } + + + virtual Status GetLiveFiles(std::vector& vec, uint64_t* mfs, + bool flush_memtable = true) override { + return db_->GetLiveFiles(vec, mfs, flush_memtable); + } + + virtual SequenceNumber GetLatestSequenceNumber() const override { + return db_->GetLatestSequenceNumber(); + } + + Status IncreaseFullHistoryTsLow(ColumnFamilyHandle* column_family, + std::string ts_low) override { + return db_->IncreaseFullHistoryTsLow(column_family, ts_low); + } + + Status GetFullHistoryTsLow(ColumnFamilyHandle* column_family, + std::string* ts_low) override { + return db_->GetFullHistoryTsLow(column_family, ts_low); + } + + virtual Status GetSortedWalFiles(VectorLogPtr& files) override { + return db_->GetSortedWalFiles(files); + } + + virtual Status GetCurrentWalFile( + std::unique_ptr* current_log_file) override { + return db_->GetCurrentWalFile(current_log_file); + } + + virtual Status GetCreationTimeOfOldestFile(uint64_t* creation_time) override { + return db_->GetCreationTimeOfOldestFile(creation_time); + } + + // WARNING: This API is planned for removal in RocksDB 7.0 since it does not + // operate at the proper level of abstraction for a key-value store, and its + // contract/restrictions are poorly documented. For example, it returns non-OK + // `Status` for non-bottommost files and files undergoing compaction. Since we + // do not plan to maintain it, the contract will likely remain underspecified + // until its removal. Any user is encouraged to read the implementation + // carefully and migrate away from it when possible. + virtual Status DeleteFile(std::string name) override { + return db_->DeleteFile(name); + } + + virtual Status GetDbIdentity(std::string& identity) const override { + return db_->GetDbIdentity(identity); + } + + virtual Status GetDbSessionId(std::string& session_id) const override { + return db_->GetDbSessionId(session_id); + } + + using DB::SetOptions; + virtual Status SetOptions(ColumnFamilyHandle* column_family_handle, + const std::unordered_map& + new_options) override { + return db_->SetOptions(column_family_handle, new_options); + } + + virtual Status SetDBOptions( + const std::unordered_map& new_options) + override { + return db_->SetDBOptions(new_options); + } + + using DB::ResetStats; + virtual Status ResetStats() override { return db_->ResetStats(); } + + using DB::GetPropertiesOfAllTables; + virtual Status GetPropertiesOfAllTables( + ColumnFamilyHandle* column_family, + TablePropertiesCollection* props) override { + return db_->GetPropertiesOfAllTables(column_family, props); + } + + using DB::GetPropertiesOfTablesInRange; + virtual Status GetPropertiesOfTablesInRange( + ColumnFamilyHandle* column_family, const Range* range, std::size_t n, + TablePropertiesCollection* props) override { + return db_->GetPropertiesOfTablesInRange(column_family, range, n, props); + } + + virtual Status GetUpdatesSince( + SequenceNumber seq_number, std::unique_ptr* iter, + const TransactionLogIterator::ReadOptions& read_options) override { + return db_->GetUpdatesSince(seq_number, iter, read_options); + } + + virtual Status SuggestCompactRange(ColumnFamilyHandle* column_family, + const Slice* begin, + const Slice* end) override { + return db_->SuggestCompactRange(column_family, begin, end); + } + + virtual Status PromoteL0(ColumnFamilyHandle* column_family, + int target_level) override { + return db_->PromoteL0(column_family, target_level); + } + + virtual ColumnFamilyHandle* DefaultColumnFamily() const override { + return db_->DefaultColumnFamily(); + } + + Status TryCatchUpWithPrimary() override { + return db_->TryCatchUpWithPrimary(); + } + + protected: + DB* db_; + std::shared_ptr shared_db_ptr_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/table_properties_collectors.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/table_properties_collectors.h new file mode 100644 index 0000000..f9d8d5d --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/table_properties_collectors.h @@ -0,0 +1,88 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once +#include +#include + +#include "rocksdb/table_properties.h" + +namespace ROCKSDB_NAMESPACE { + +// A factory of a table property collector that marks a SST +// file as need-compaction when it observe at least "D" deletion +// entries in any "N" consecutive entries or the ratio of tombstone +// entries in the whole file >= the specified deletion ratio. +class CompactOnDeletionCollectorFactory + : public TablePropertiesCollectorFactory { + public: + // A factory of a table property collector that marks a SST + // file as need-compaction when it observe at least "D" deletion + // entries in any "N" consecutive entries, or the ratio of tombstone + // entries >= deletion_ratio. + // + // @param sliding_window_size "N" + // @param deletion_trigger "D" + // @param deletion_ratio, if <= 0 or > 1, disable triggering compaction + // based on deletion ratio. + CompactOnDeletionCollectorFactory(size_t sliding_window_size, + size_t deletion_trigger, + double deletion_ratio); + + ~CompactOnDeletionCollectorFactory() {} + + TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context context) override; + + // Change the value of sliding_window_size "N" + // Setting it to 0 disables the delete triggered compaction + void SetWindowSize(size_t sliding_window_size) { + sliding_window_size_.store(sliding_window_size); + } + size_t GetWindowSize() const { return sliding_window_size_.load(); } + + // Change the value of deletion_trigger "D" + void SetDeletionTrigger(size_t deletion_trigger) { + deletion_trigger_.store(deletion_trigger); + } + + size_t GetDeletionTrigger() const { return deletion_trigger_.load(); } + // Change deletion ratio. + // @param deletion_ratio, if <= 0 or > 1, disable triggering compaction + // based on deletion ratio. + void SetDeletionRatio(double deletion_ratio) { + deletion_ratio_.store(deletion_ratio); + } + + double GetDeletionRatio() const { return deletion_ratio_.load(); } + static const char* kClassName() { return "CompactOnDeletionCollector"; } + const char* Name() const override { return kClassName(); } + + std::string ToString() const override; + + private: + std::atomic sliding_window_size_; + std::atomic deletion_trigger_; + std::atomic deletion_ratio_; +}; + +// Creates a factory of a table property collector that marks a SST +// file as need-compaction when it observe at least "D" deletion +// entries in any "N" consecutive entries, or the ratio of tombstone +// entries >= deletion_ratio. +// +// @param sliding_window_size "N". Note that this number will be +// round up to the smallest multiple of 128 that is no less +// than the specified size. +// @param deletion_trigger "D". Note that even when "N" is changed, +// the specified number for "D" will not be changed. +// @param deletion_ratio, if <= 0 or > 1, disable triggering compaction +// based on deletion ratio. Disabled by default. +extern std::shared_ptr +NewCompactOnDeletionCollectorFactory(size_t sliding_window_size, + size_t deletion_trigger, + double deletion_ratio = 0); +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/transaction.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/transaction.h new file mode 100644 index 0000000..3cdcc9b --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/transaction.h @@ -0,0 +1,685 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + + +#include +#include +#include + +#include "rocksdb/comparator.h" +#include "rocksdb/db.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +class Iterator; +class TransactionDB; +class WriteBatchWithIndex; + +using TransactionName = std::string; + +using TransactionID = uint64_t; + +using TxnTimestamp = uint64_t; + +constexpr TxnTimestamp kMaxTxnTimestamp = + std::numeric_limits::max(); + +/* + class Endpoint allows to define prefix ranges. + + Prefix ranges are introduced below. + + == Basic Ranges == + Let's start from basic ranges. Key Comparator defines ordering of rowkeys. + Then, one can specify finite closed ranges by just providing rowkeys of their + endpoints: + + lower_endpoint <= X <= upper_endpoint + + However our goal is to provide a richer set of endpoints. Read on. + + == Lexicographic ordering == + A lexicographic (or dictionary) ordering satisfies these criteria: If there + are two keys in form + key_a = {prefix_a, suffix_a} + key_b = {prefix_b, suffix_b} + and + prefix_a < prefix_b + then + key_a < key_b. + + == Prefix ranges == + With lexicographic ordering, one may want to define ranges in form + + "prefix is $PREFIX" + + which translates to a range in form + + {$PREFIX, -infinity} < X < {$PREFIX, +infinity} + + where -infinity will compare less than any possible suffix, and +infinity + will compare as greater than any possible suffix. + + class Endpoint allows to define these kind of rangtes. + + == Notes == + BytewiseComparator and ReverseBytewiseComparator produce lexicographic + ordering. + + The row comparison function is able to compare key prefixes. If the data + domain includes keys A and B, then the comparison function is able to compare + equal-length prefixes: + + min_len= min(byte_length(A), byte_length(B)); + cmp(Slice(A, min_len), Slice(B, min_len)); // this call is valid + + == Other options == + As far as MyRocks is concerned, the alternative to prefix ranges would be to + support both open (non-inclusive) and closed (inclusive) range endpoints. +*/ + +class Endpoint { + public: + Slice slice; + + /* + true : the key has a "+infinity" suffix. A suffix that would compare as + greater than any other suffix + false : otherwise + */ + bool inf_suffix; + + explicit Endpoint(const Slice& slice_arg, bool inf_suffix_arg = false) + : slice(slice_arg), inf_suffix(inf_suffix_arg) {} + + explicit Endpoint(const char* s, bool inf_suffix_arg = false) + : slice(s), inf_suffix(inf_suffix_arg) {} + + Endpoint(const char* s, size_t size, bool inf_suffix_arg = false) + : slice(s, size), inf_suffix(inf_suffix_arg) {} + + Endpoint() : inf_suffix(false) {} +}; + +// Provides notification to the caller of SetSnapshotOnNextOperation when +// the actual snapshot gets created +class TransactionNotifier { + public: + virtual ~TransactionNotifier() {} + + // Implement this method to receive notification when a snapshot is + // requested via SetSnapshotOnNextOperation. + // Do not take exclusive ownership of `newSnapshot` because it is shared with + // the underlying transaction. + virtual void SnapshotCreated(const Snapshot* newSnapshot) = 0; +}; + +// Provides BEGIN/COMMIT/ROLLBACK transactions. +// +// To use transactions, you must first create either an OptimisticTransactionDB +// or a TransactionDB. See examples/[optimistic_]transaction_example.cc for +// more information. +// +// To create a transaction, use [Optimistic]TransactionDB::BeginTransaction(). +// +// It is up to the caller to synchronize access to this object. +// +// See examples/transaction_example.cc for some simple examples. +// +// TODO(agiardullo): Not yet implemented +// -PerfContext statistics +// -Support for using Transactions with DBWithTTL +class Transaction { + public: + // No copying allowed + Transaction(const Transaction&) = delete; + void operator=(const Transaction&) = delete; + + virtual ~Transaction() {} + + // If a transaction has a snapshot set, the transaction will ensure that + // any keys successfully written(or fetched via GetForUpdate()) have not + // been modified outside of this transaction since the time the snapshot was + // set. + // If a snapshot has not been set, the transaction guarantees that keys have + // not been modified since the time each key was first written (or fetched via + // GetForUpdate()). + // + // Using SetSnapshot() will provide stricter isolation guarantees at the + // expense of potentially more transaction failures due to conflicts with + // other writes. + // + // Calling SetSnapshot() has no effect on keys written before this function + // has been called. + // + // SetSnapshot() may be called multiple times if you would like to change + // the snapshot used for different operations in this transaction. + // + // Calling SetSnapshot will not affect the version of Data returned by Get() + // methods. See Transaction::Get() for more details. + virtual void SetSnapshot() = 0; + + // Similar to SetSnapshot(), but will not change the current snapshot + // until Put/Merge/Delete/GetForUpdate/MultigetForUpdate is called. + // By calling this function, the transaction will essentially call + // SetSnapshot() for you right before performing the next write/GetForUpdate. + // + // Calling SetSnapshotOnNextOperation() will not affect what snapshot is + // returned by GetSnapshot() until the next write/GetForUpdate is executed. + // + // When the snapshot is created the notifier's SnapshotCreated method will + // be called so that the caller can get access to the snapshot. + // + // This is an optimization to reduce the likelihood of conflicts that + // could occur in between the time SetSnapshot() is called and the first + // write/GetForUpdate operation. Eg, this prevents the following + // race-condition: + // + // txn1->SetSnapshot(); + // txn2->Put("A", ...); + // txn2->Commit(); + // txn1->GetForUpdate(opts, "A", ...); // FAIL! + // + // WriteCommittedTxn only: a new snapshot will be taken upon next operation, + // and next operation can be a Commit. + // TODO(yanqin) remove the "write-committed only" limitation. + virtual void SetSnapshotOnNextOperation( + std::shared_ptr notifier = nullptr) = 0; + + // Returns the Snapshot created by the last call to SetSnapshot(). + // + // REQUIRED: The returned Snapshot is only valid up until the next time + // SetSnapshot()/SetSnapshotOnNextSavePoint() is called, ClearSnapshot() + // is called, or the Transaction is deleted. + virtual const Snapshot* GetSnapshot() const = 0; + + // Returns the Snapshot created by the last call to SetSnapshot(). + // The returned snapshot can outlive the transaction. + virtual std::shared_ptr GetTimestampedSnapshot() const = 0; + + // Clears the current snapshot (i.e. no snapshot will be 'set') + // + // This removes any snapshot that currently exists or is set to be created + // on the next update operation (SetSnapshotOnNextOperation). + // + // Calling ClearSnapshot() has no effect on keys written before this function + // has been called. + // + // If a reference to a snapshot was retrieved via GetSnapshot(), it will no + // longer be valid and should be discarded after a call to ClearSnapshot(). + virtual void ClearSnapshot() = 0; + + // Prepare the current transaction for 2PC + virtual Status Prepare() = 0; + + // Write all batched keys to the db atomically. + // + // Returns OK on success. + // + // May return any error status that could be returned by DB:Write(). + // + // If this transaction was created by an OptimisticTransactionDB(), + // Status::Busy() may be returned if the transaction could not guarantee + // that there are no write conflicts. Status::TryAgain() may be returned + // if the memtable history size is not large enough + // (see max_write_buffer_size_to_maintain). In either case, a Rollback() + // or new transaction is required to expect a different result. + // + // If this transaction was created by a TransactionDB(), Status::Expired() + // may be returned if this transaction has lived for longer than + // TransactionOptions.expiration. Status::TxnNotPrepared() may be returned if + // TransactionOptions.skip_prepare is false and Prepare is not called on this + // transaction before Commit. + virtual Status Commit() = 0; + + // In addition to Commit(), also creates a snapshot of the db after all + // writes by this txn are visible to other readers. + // Caller is responsible for ensuring that + // snapshot1.seq < snapshot2.seq iff. snapshot1.ts < snapshot2.ts + // in which snapshot1 and snapshot2 are created by this API. + // + // Currently only supported by WriteCommittedTxn. Calling this method on + // other types of transactions will return non-ok Status resulting from + // Commit() or a `NotSupported` error. + // This method returns OK if and only if the transaction successfully + // commits. It is possible that transaction commits successfully but fails to + // create a timestamped snapshot. Therefore, the caller should check that the + // snapshot is created. + // notifier will be notified upon next snapshot creation. Nullable. + // ret non-null output argument storing a shared_ptr to the newly created + // snapshot. + Status CommitAndTryCreateSnapshot( + std::shared_ptr notifier = + std::shared_ptr(), + TxnTimestamp ts = kMaxTxnTimestamp, + std::shared_ptr* snapshot = nullptr); + + // Discard all batched writes in this transaction. + // FIXME: what happens if this isn't called before destruction? + virtual Status Rollback() = 0; + + // Records the state of the transaction for future calls to + // RollbackToSavePoint(). May be called multiple times to set multiple save + // points. + virtual void SetSavePoint() = 0; + + // Undo all operations in this transaction (Put, Merge, Delete, PutLogData) + // since the most recent call to SetSavePoint() and removes the most recent + // SetSavePoint(). + // If there is no previous call to SetSavePoint(), returns Status::NotFound() + virtual Status RollbackToSavePoint() = 0; + + // Pop the most recent save point. + // If there is no previous call to SetSavePoint(), Status::NotFound() + // will be returned. + // Otherwise returns Status::OK(). + virtual Status PopSavePoint() = 0; + + // This function is similar to DB::Get() except it will also read pending + // changes in this transaction. Currently, this function will return + // Status::MergeInProgress if the most recent write to the queried key in + // this batch is a Merge. + // + // If read_options.snapshot is not set, the current version of the key will + // be read. Calling SetSnapshot() does not affect the version of the data + // returned. + // + // Note that setting read_options.snapshot will affect what is read from the + // DB but will NOT change which keys are read from this transaction (the keys + // in this transaction do not yet belong to any snapshot and will be fetched + // regardless). + virtual Status Get(const ReadOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + std::string* value) = 0; + + // An overload of the above method that receives a PinnableSlice + // For backward compatibility a default implementation is provided + virtual Status Get(const ReadOptions& options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* pinnable_val) { + assert(pinnable_val != nullptr); + auto s = Get(options, column_family, key, pinnable_val->GetSelf()); + pinnable_val->PinSelf(); + return s; + } + + virtual Status Get(const ReadOptions& options, const Slice& key, + std::string* value) = 0; + virtual Status Get(const ReadOptions& options, const Slice& key, + PinnableSlice* pinnable_val) { + assert(pinnable_val != nullptr); + auto s = Get(options, key, pinnable_val->GetSelf()); + pinnable_val->PinSelf(); + return s; + } + + virtual std::vector MultiGet( + const ReadOptions& options, + const std::vector& column_family, + const std::vector& keys, std::vector* values) = 0; + + virtual std::vector MultiGet(const ReadOptions& options, + const std::vector& keys, + std::vector* values) = 0; + + // Batched version of MultiGet - see DBImpl::MultiGet(). Sub-classes are + // expected to override this with an implementation that calls + // DBImpl::MultiGet() + virtual void MultiGet(const ReadOptions& options, + ColumnFamilyHandle* column_family, + const size_t num_keys, const Slice* keys, + PinnableSlice* values, Status* statuses, + const bool /*sorted_input*/ = false) { + for (size_t i = 0; i < num_keys; ++i) { + statuses[i] = Get(options, column_family, keys[i], &values[i]); + } + } + + // Read this key and ensure that this transaction will only + // be able to be committed if this key is not written outside this + // transaction after it has first been read (or after the snapshot if a + // snapshot is set in this transaction and do_validate is true). If + // do_validate is false, ReadOptions::snapshot is expected to be nullptr so + // that GetForUpdate returns the latest committed value. The transaction + // behavior is the same regardless of whether the key exists or not. + // + // Note: Currently, this function will return Status::MergeInProgress + // if the most recent write to the queried key in this batch is a Merge. + // + // The values returned by this function are similar to Transaction::Get(). + // If value==nullptr, then this function will not read any data, but will + // still ensure that this key cannot be written to by outside of this + // transaction. + // + // If this transaction was created by an OptimisticTransaction, GetForUpdate() + // could cause commit() to fail. Otherwise, it could return any error + // that could be returned by DB::Get(). + // + // If this transaction was created by a TransactionDB, it can return + // Status::OK() on success, + // Status::Busy() if there is a write conflict, + // Status::TimedOut() if a lock could not be acquired, + // Status::TryAgain() if the memtable history size is not large enough + // (See max_write_buffer_size_to_maintain) + // Status::MergeInProgress() if merge operations cannot be resolved. + // or other errors if this key could not be read. + virtual Status GetForUpdate(const ReadOptions& options, + ColumnFamilyHandle* column_family, + const Slice& key, std::string* value, + bool exclusive = true, + const bool do_validate = true) = 0; + + // An overload of the above method that receives a PinnableSlice + // For backward compatibility a default implementation is provided + virtual Status GetForUpdate(const ReadOptions& options, + ColumnFamilyHandle* column_family, + const Slice& key, PinnableSlice* pinnable_val, + bool exclusive = true, + const bool do_validate = true) { + if (pinnable_val == nullptr) { + std::string* null_str = nullptr; + return GetForUpdate(options, column_family, key, null_str, exclusive, + do_validate); + } else { + auto s = GetForUpdate(options, column_family, key, + pinnable_val->GetSelf(), exclusive, do_validate); + pinnable_val->PinSelf(); + return s; + } + } + + // Get a range lock on [start_endpoint; end_endpoint]. + virtual Status GetRangeLock(ColumnFamilyHandle*, const Endpoint&, + const Endpoint&) { + return Status::NotSupported(); + } + + virtual Status GetForUpdate(const ReadOptions& options, const Slice& key, + std::string* value, bool exclusive = true, + const bool do_validate = true) = 0; + + virtual std::vector MultiGetForUpdate( + const ReadOptions& options, + const std::vector& column_family, + const std::vector& keys, std::vector* values) = 0; + + virtual std::vector MultiGetForUpdate( + const ReadOptions& options, const std::vector& keys, + std::vector* values) = 0; + + // Returns an iterator that will iterate on all keys in the default + // column family including both keys in the DB and uncommitted keys in this + // transaction. + // + // Setting read_options.snapshot will affect what is read from the + // DB but will NOT change which keys are read from this transaction (the keys + // in this transaction do not yet belong to any snapshot and will be fetched + // regardless). + // + // Caller is responsible for deleting the returned Iterator. + // + // The returned iterator is only valid until Commit(), Rollback(), or + // RollbackToSavePoint() is called. + virtual Iterator* GetIterator(const ReadOptions& read_options) = 0; + + virtual Iterator* GetIterator(const ReadOptions& read_options, + ColumnFamilyHandle* column_family) = 0; + + // Put, Merge, Delete, and SingleDelete behave similarly to the corresponding + // functions in WriteBatch, but will also do conflict checking on the + // keys being written. + // + // assume_tracked=true expects the key be already tracked. More + // specifically, it means the the key was previous tracked in the same + // savepoint, with the same exclusive flag, and at a lower sequence number. + // If valid then it skips ValidateSnapshot. Returns error otherwise. + // + // If this Transaction was created on an OptimisticTransactionDB, these + // functions should always return Status::OK(). + // + // If this Transaction was created on a TransactionDB, the status returned + // can be: + // Status::OK() on success, + // Status::Busy() if there is a write conflict, + // Status::TimedOut() if a lock could not be acquired, + // Status::TryAgain() if the memtable history size is not large enough + // (See max_write_buffer_size_to_maintain) + // or other errors on unexpected failures. + virtual Status Put(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& value, const bool assume_tracked = false) = 0; + virtual Status Put(const Slice& key, const Slice& value) = 0; + virtual Status Put(ColumnFamilyHandle* column_family, const SliceParts& key, + const SliceParts& value, + const bool assume_tracked = false) = 0; + virtual Status Put(const SliceParts& key, const SliceParts& value) = 0; + + virtual Status Merge(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& value, + const bool assume_tracked = false) = 0; + virtual Status Merge(const Slice& key, const Slice& value) = 0; + + virtual Status Delete(ColumnFamilyHandle* column_family, const Slice& key, + const bool assume_tracked = false) = 0; + virtual Status Delete(const Slice& key) = 0; + virtual Status Delete(ColumnFamilyHandle* column_family, + const SliceParts& key, + const bool assume_tracked = false) = 0; + virtual Status Delete(const SliceParts& key) = 0; + + virtual Status SingleDelete(ColumnFamilyHandle* column_family, + const Slice& key, + const bool assume_tracked = false) = 0; + virtual Status SingleDelete(const Slice& key) = 0; + virtual Status SingleDelete(ColumnFamilyHandle* column_family, + const SliceParts& key, + const bool assume_tracked = false) = 0; + virtual Status SingleDelete(const SliceParts& key) = 0; + + // PutUntracked() will write a Put to the batch of operations to be committed + // in this transaction. This write will only happen if this transaction + // gets committed successfully. But unlike Transaction::Put(), + // no conflict checking will be done for this key. + // + // If this Transaction was created on a PessimisticTransactionDB, this + // function will still acquire locks necessary to make sure this write doesn't + // cause conflicts in other transactions and may return Status::Busy(). + virtual Status PutUntracked(ColumnFamilyHandle* column_family, + const Slice& key, const Slice& value) = 0; + virtual Status PutUntracked(const Slice& key, const Slice& value) = 0; + virtual Status PutUntracked(ColumnFamilyHandle* column_family, + const SliceParts& key, + const SliceParts& value) = 0; + virtual Status PutUntracked(const SliceParts& key, + const SliceParts& value) = 0; + + virtual Status MergeUntracked(ColumnFamilyHandle* column_family, + const Slice& key, const Slice& value) = 0; + virtual Status MergeUntracked(const Slice& key, const Slice& value) = 0; + + virtual Status DeleteUntracked(ColumnFamilyHandle* column_family, + const Slice& key) = 0; + + virtual Status DeleteUntracked(const Slice& key) = 0; + virtual Status DeleteUntracked(ColumnFamilyHandle* column_family, + const SliceParts& key) = 0; + virtual Status DeleteUntracked(const SliceParts& key) = 0; + virtual Status SingleDeleteUntracked(ColumnFamilyHandle* column_family, + const Slice& key) = 0; + + virtual Status SingleDeleteUntracked(const Slice& key) = 0; + + // Similar to WriteBatch::PutLogData + virtual void PutLogData(const Slice& blob) = 0; + + // By default, all Put/Merge/Delete operations will be indexed in the + // transaction so that Get/GetForUpdate/GetIterator can search for these + // keys. + // + // If the caller does not want to fetch the keys about to be written, + // they may want to avoid indexing as a performance optimization. + // Calling DisableIndexing() will turn off indexing for all future + // Put/Merge/Delete operations until EnableIndexing() is called. + // + // If a key is Put/Merge/Deleted after DisableIndexing is called and then + // is fetched via Get/GetForUpdate/GetIterator, the result of the fetch is + // undefined. + virtual void DisableIndexing() = 0; + virtual void EnableIndexing() = 0; + + // Returns the number of distinct Keys being tracked by this transaction. + // If this transaction was created by a TransactionDB, this is the number of + // keys that are currently locked by this transaction. + // If this transaction was created by an OptimisticTransactionDB, this is the + // number of keys that need to be checked for conflicts at commit time. + virtual uint64_t GetNumKeys() const = 0; + + // Returns the number of Puts/Deletes/Merges that have been applied to this + // transaction so far. + virtual uint64_t GetNumPuts() const = 0; + virtual uint64_t GetNumDeletes() const = 0; + virtual uint64_t GetNumMerges() const = 0; + + // Returns the elapsed time in milliseconds since this Transaction began. + virtual uint64_t GetElapsedTime() const = 0; + + // Fetch the underlying write batch that contains all pending changes to be + // committed. + // + // Note: You should not write or delete anything from the batch directly and + // should only use the functions in the Transaction class to + // write to this transaction. + virtual WriteBatchWithIndex* GetWriteBatch() = 0; + + // Change the value of TransactionOptions.lock_timeout (in milliseconds) for + // this transaction. + // Has no effect on OptimisticTransactions. + virtual void SetLockTimeout(int64_t timeout) = 0; + + // Return the WriteOptions that will be used during Commit() + virtual WriteOptions* GetWriteOptions() = 0; + + // Reset the WriteOptions that will be used during Commit(). + virtual void SetWriteOptions(const WriteOptions& write_options) = 0; + + // If this key was previously fetched in this transaction using + // GetForUpdate/MultigetForUpdate(), calling UndoGetForUpdate will tell + // the transaction that it no longer needs to do any conflict checking + // for this key. + // + // If a key has been fetched N times via GetForUpdate/MultigetForUpdate(), + // then UndoGetForUpdate will only have an effect if it is also called N + // times. If this key has been written to in this transaction, + // UndoGetForUpdate() will have no effect. + // + // If SetSavePoint() has been called after the GetForUpdate(), + // UndoGetForUpdate() will not have any effect. + // + // If this Transaction was created by an OptimisticTransactionDB, + // calling UndoGetForUpdate can affect whether this key is conflict checked + // at commit time. + // If this Transaction was created by a TransactionDB, + // calling UndoGetForUpdate may release any held locks for this key. + virtual void UndoGetForUpdate(ColumnFamilyHandle* column_family, + const Slice& key) = 0; + virtual void UndoGetForUpdate(const Slice& key) = 0; + + virtual Status RebuildFromWriteBatch(WriteBatch* src_batch) = 0; + + // Note: data in the commit-time-write-batch bypasses concurrency control, + // thus should be used with great caution. + // For write-prepared/write-unprepared transactions, + // GetCommitTimeWriteBatch() can be used only if the transaction is started + // with + // `TransactionOptions::use_only_the_last_commit_time_batch_for_recovery` set + // to true. Otherwise, it is possible that two uncommitted versions of the + // same key exist in the database due to the current implementation (see the + // explanation in WritePreparedTxn::CommitInternal). + // During bottommost compaction, RocksDB may + // set the sequence numbers of both to zero once becoming committed, causing + // output SST file to have two identical internal keys. + virtual WriteBatch* GetCommitTimeWriteBatch() = 0; + + virtual void SetLogNumber(uint64_t log) { log_number_ = log; } + + virtual uint64_t GetLogNumber() const { return log_number_; } + + virtual Status SetName(const TransactionName& name) = 0; + + virtual TransactionName GetName() const { return name_; } + + virtual TransactionID GetID() const { return 0; } + + virtual bool IsDeadlockDetect() const { return false; } + + virtual std::vector GetWaitingTxns( + uint32_t* /*column_family_id*/, std::string* /*key*/) const { + assert(false); + return std::vector(); + } + + enum TransactionState { + STARTED = 0, + AWAITING_PREPARE = 1, + PREPARED = 2, + AWAITING_COMMIT = 3, + COMMITTED = 4, + COMMITED = COMMITTED, // old misspelled name + AWAITING_ROLLBACK = 5, + ROLLEDBACK = 6, + LOCKS_STOLEN = 7, + }; + + TransactionState GetState() const { return txn_state_; } + void SetState(TransactionState state) { txn_state_ = state; } + + // NOTE: Experimental feature + // The globally unique id with which the transaction is identified. This id + // might or might not be set depending on the implementation. Similarly the + // implementation decides the point in lifetime of a transaction at which it + // assigns the id. Although currently it is the case, the id is not guaranteed + // to remain the same across restarts. + uint64_t GetId() { return id_; } + + virtual Status SetReadTimestampForValidation(TxnTimestamp /*ts*/) { + return Status::NotSupported("timestamp not supported"); + } + + virtual Status SetCommitTimestamp(TxnTimestamp /*ts*/) { + return Status::NotSupported("timestamp not supported"); + } + + virtual TxnTimestamp GetCommitTimestamp() const { return kMaxTxnTimestamp; } + + protected: + explicit Transaction(const TransactionDB* /*db*/) {} + Transaction() : log_number_(0), txn_state_(STARTED) {} + + // the log in which the prepared section for this txn resides + // (for two phase commit) + uint64_t log_number_; + TransactionName name_; + + // Execution status of the transaction. + std::atomic txn_state_; + + uint64_t id_ = 0; + virtual void SetId(uint64_t id) { + assert(id_ == 0); + id_ = id; + } + + virtual uint64_t GetLastLogNumber() const { return log_number_; } + + private: + friend class PessimisticTransactionDB; + friend class WriteUnpreparedTxnDB; + friend class TransactionTest_TwoPhaseLogRollingTest_Test; + friend class TransactionTest_TwoPhaseLogRollingTest2_Test; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/transaction_db.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/transaction_db.h new file mode 100644 index 0000000..3c4b630 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/transaction_db.h @@ -0,0 +1,506 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include + +#include "rocksdb/comparator.h" +#include "rocksdb/db.h" +#include "rocksdb/utilities/stackable_db.h" +#include "rocksdb/utilities/transaction.h" + +// Database with Transaction support. +// +// See transaction.h and examples/transaction_example.cc + +namespace ROCKSDB_NAMESPACE { + +class TransactionDBMutexFactory; + +enum TxnDBWritePolicy { + WRITE_COMMITTED = 0, // write only the committed data + WRITE_PREPARED, // write data after the prepare phase of 2pc + WRITE_UNPREPARED // write data before the prepare phase of 2pc +}; + +constexpr uint32_t kInitialMaxDeadlocks = 5; + +class LockManager; +struct RangeLockInfo; + +// A lock manager handle +// The workflow is as follows: +// * Use a factory method (like NewRangeLockManager()) to create a lock +// manager and get its handle. +// * A Handle for a particular kind of lock manager will have extra +// methods and parameters to control the lock manager +// * Pass the handle to RocksDB in TransactionDBOptions::lock_mgr_handle. It +// will be used to perform locking. +class LockManagerHandle { + public: + // PessimisticTransactionDB will call this to get the Lock Manager it's going + // to use. + virtual LockManager* getLockManager() = 0; + + virtual ~LockManagerHandle() {} +}; + +// Same as class Endpoint, but use std::string to manage the buffer allocation +struct EndpointWithString { + std::string slice; + bool inf_suffix; +}; + +struct RangeDeadlockInfo { + TransactionID m_txn_id; + uint32_t m_cf_id; + bool m_exclusive; + + EndpointWithString m_start; + EndpointWithString m_end; +}; + +struct RangeDeadlockPath { + std::vector path; + bool limit_exceeded; + int64_t deadlock_time; + + explicit RangeDeadlockPath(std::vector path_entry, + const int64_t& dl_time) + : path(path_entry), limit_exceeded(false), deadlock_time(dl_time) {} + + // empty path, limit exceeded constructor and default constructor + explicit RangeDeadlockPath(const int64_t& dl_time = 0, bool limit = false) + : path(0), limit_exceeded(limit), deadlock_time(dl_time) {} + + bool empty() { return path.empty() && !limit_exceeded; } +}; + +// A handle to control RangeLockManager (Range-based lock manager) from outside +// RocksDB +class RangeLockManagerHandle : public LockManagerHandle { + public: + // Set total amount of lock memory to use. + // + // @return 0 Ok + // @return EDOM Failed to set because currently using more memory than + // specified + virtual int SetMaxLockMemory(size_t max_lock_memory) = 0; + virtual size_t GetMaxLockMemory() = 0; + + using RangeLockStatus = + std::unordered_multimap; + + // Lock Escalation barrier check function. + // It is called for a couple of endpoints A and B, such that A < B. + // If escalation_barrier_check_func(A, B)==true, then there's a lock + // escalation barrier between A and B, and lock escalation is not allowed + // to bridge the gap between A and B. + // + // The function may be called from any thread that acquires or releases + // locks. It should not throw exceptions. There is currently no way to return + // an error. + using EscalationBarrierFunc = + std::function; + + // Set the user-provided barrier check function + virtual void SetEscalationBarrierFunc(EscalationBarrierFunc func) = 0; + + virtual RangeLockStatus GetRangeLockStatusData() = 0; + + class Counters { + public: + // Number of times lock escalation was triggered (for all column families) + uint64_t escalation_count; + + // Number of times lock acquisition had to wait for a conflicting lock + // to be released. This counts both successful waits (where the desired + // lock was acquired) and waits that timed out or got other error. + uint64_t lock_wait_count; + + // How much memory is currently used for locks (total for all column + // families) + uint64_t current_lock_memory; + }; + + // Get the current counter values + virtual Counters GetStatus() = 0; + + // Functions for range-based Deadlock reporting. + virtual std::vector GetRangeDeadlockInfoBuffer() = 0; + virtual void SetRangeDeadlockInfoBufferSize(uint32_t target_size) = 0; + + virtual ~RangeLockManagerHandle() {} +}; + +// A factory function to create a Range Lock Manager. The created object should +// be: +// 1. Passed in TransactionDBOptions::lock_mgr_handle to open the database in +// range-locking mode +// 2. Used to control the lock manager when the DB is already open. +RangeLockManagerHandle* NewRangeLockManager( + std::shared_ptr mutex_factory); + +struct TransactionDBOptions { + // Specifies the maximum number of keys that can be locked at the same time + // per column family. + // If the number of locked keys is greater than max_num_locks, transaction + // writes (or GetForUpdate) will return an error. + // If this value is not positive, no limit will be enforced. + int64_t max_num_locks = -1; + + // Stores the number of latest deadlocks to track + uint32_t max_num_deadlocks = kInitialMaxDeadlocks; + + // Increasing this value will increase the concurrency by dividing the lock + // table (per column family) into more sub-tables, each with their own + // separate mutex. + size_t num_stripes = 16; + + // If positive, specifies the default wait timeout in milliseconds when + // a transaction attempts to lock a key if not specified by + // TransactionOptions::lock_timeout. + // + // If 0, no waiting is done if a lock cannot instantly be acquired. + // If negative, there is no timeout. Not using a timeout is not recommended + // as it can lead to deadlocks. Currently, there is no deadlock-detection to + // recover from a deadlock. + int64_t transaction_lock_timeout = 1000; // 1 second + + // If positive, specifies the wait timeout in milliseconds when writing a key + // OUTSIDE of a transaction (ie by calling DB::Put(),Merge(),Delete(),Write() + // directly). + // If 0, no waiting is done if a lock cannot instantly be acquired. + // If negative, there is no timeout and will block indefinitely when acquiring + // a lock. + // + // Not using a timeout can lead to deadlocks. Currently, there + // is no deadlock-detection to recover from a deadlock. While DB writes + // cannot deadlock with other DB writes, they can deadlock with a transaction. + // A negative timeout should only be used if all transactions have a small + // expiration set. + int64_t default_lock_timeout = 1000; // 1 second + + // If set, the TransactionDB will use this implementation of a mutex and + // condition variable for all transaction locking instead of the default + // mutex/condvar implementation. + std::shared_ptr custom_mutex_factory; + + // The policy for when to write the data into the DB. The default policy is to + // write only the committed data (WRITE_COMMITTED). The data could be written + // before the commit phase. The DB then needs to provide the mechanisms to + // tell apart committed from uncommitted data. + TxnDBWritePolicy write_policy = TxnDBWritePolicy::WRITE_COMMITTED; + + // TODO(myabandeh): remove this option + // Note: this is a temporary option as a hot fix in rollback of writeprepared + // txns in myrocks. MyRocks uses merge operands for autoinc column id without + // however obtaining locks. This breaks the assumption behind the rollback + // logic in myrocks. This hack of simply not rolling back merge operands works + // for the special way that myrocks uses this operands. + bool rollback_merge_operands = false; + + // nullptr means use default lock manager. + // Other value means the user provides a custom lock manager. + std::shared_ptr lock_mgr_handle; + + // If true, the TransactionDB implementation might skip concurrency control + // unless it is overridden by TransactionOptions or + // TransactionDBWriteOptimizations. This can be used in conjunction with + // DBOptions::unordered_write when the TransactionDB is used solely for write + // ordering rather than concurrency control. + bool skip_concurrency_control = false; + + // This option is only valid for write unprepared. If a write batch exceeds + // this threshold, then the transaction will implicitly flush the currently + // pending writes into the database. A value of 0 or less means no limit. + int64_t default_write_batch_flush_threshold = 0; + + // This option is valid only for write-prepared/write-unprepared. Transaction + // will rely on this callback to determine if a key should be rolled back + // with Delete or SingleDelete when necessary. If the callback returns true, + // then SingleDelete should be used. If the callback is not callable or the + // callback returns false, then a Delete is used. + // The application should ensure thread-safety of this callback. + // The callback should not throw because RocksDB is not exception-safe. + // The callback may be removed if we allow mixing Delete and SingleDelete in + // the future. + std::function + rollback_deletion_type_callback; + + private: + // 128 entries + // Should the default value change, please also update wp_snapshot_cache_bits + // in db_stress_gflags.cc + size_t wp_snapshot_cache_bits = static_cast(7); + // 8m entry, 64MB size + // Should the default value change, please also update wp_commit_cache_bits + // in db_stress_gflags.cc + size_t wp_commit_cache_bits = static_cast(23); + + // For testing, whether transaction name should be auto-generated or not. This + // is useful for write unprepared which requires named transactions. + bool autogenerate_name = false; + + friend class WritePreparedTxnDB; + friend class WriteUnpreparedTxn; + friend class WritePreparedTransactionTestBase; + friend class TransactionTestBase; + friend class MySQLStyleTransactionTest; + friend class StressTest; +}; + +struct TransactionOptions { + // Setting set_snapshot=true is the same as calling + // Transaction::SetSnapshot(). + bool set_snapshot = false; + + // Setting to true means that before acquiring locks, this transaction will + // check if doing so will cause a deadlock. If so, it will return with + // Status::Busy. The user should retry their transaction. + bool deadlock_detect = false; + + // If set, it states that the CommitTimeWriteBatch represents the latest state + // of the application, has only one sub-batch, i.e., no duplicate keys, and + // meant to be used later during recovery. It enables an optimization to + // postpone updating the memtable with CommitTimeWriteBatch to only + // SwitchMemtable or recovery. + // This option does not affect write-committed. Only + // write-prepared/write-unprepared transactions will be affected. + bool use_only_the_last_commit_time_batch_for_recovery = false; + + // TODO(agiardullo): TransactionDB does not yet support comparators that allow + // two non-equal keys to be equivalent. Ie, cmp->Compare(a,b) should only + // return 0 if + // a.compare(b) returns 0. + + // If positive, specifies the wait timeout in milliseconds when + // a transaction attempts to lock a key. + // + // If 0, no waiting is done if a lock cannot instantly be acquired. + // If negative, TransactionDBOptions::transaction_lock_timeout will be used. + int64_t lock_timeout = -1; + + // Expiration duration in milliseconds. If non-negative, transactions that + // last longer than this many milliseconds will fail to commit. If not set, + // a forgotten transaction that is never committed, rolled back, or deleted + // will never relinquish any locks it holds. This could prevent keys from + // being written by other writers. + int64_t expiration = -1; + + // The number of traversals to make during deadlock detection. + int64_t deadlock_detect_depth = 50; + + // The maximum number of bytes used for the write batch. 0 means no limit. + size_t max_write_batch_size = 0; + + // Skip Concurrency Control. This could be as an optimization if the + // application knows that the transaction would not have any conflict with + // concurrent transactions. It could also be used during recovery if (i) + // application guarantees no conflict between prepared transactions in the WAL + // (ii) application guarantees that recovered transactions will be rolled + // back/commit before new transactions start. + // Default: false + bool skip_concurrency_control = false; + + // In pessimistic transaction, if this is true, then you can skip Prepare + // before Commit, otherwise, you must Prepare before Commit. + bool skip_prepare = true; + + // See TransactionDBOptions::default_write_batch_flush_threshold for + // description. If a negative value is specified, then the default value from + // TransactionDBOptions is used. + int64_t write_batch_flush_threshold = -1; +}; + +// The per-write optimizations that do not involve transactions. TransactionDB +// implementation might or might not make use of the specified optimizations. +struct TransactionDBWriteOptimizations { + // If it is true it means that the application guarantees that the + // key-set in the write batch do not conflict with any concurrent transaction + // and hence the concurrency control mechanism could be skipped for this + // write. + bool skip_concurrency_control = false; + // If true, the application guarantees that there is no duplicate in the write batch and any employed mechanism to handle + // duplicate keys could be skipped. + bool skip_duplicate_key_check = false; +}; + +struct KeyLockInfo { + std::string key; + std::vector ids; + bool exclusive; +}; + +struct RangeLockInfo { + EndpointWithString start; + EndpointWithString end; + std::vector ids; + bool exclusive; +}; + +struct DeadlockInfo { + TransactionID m_txn_id; + uint32_t m_cf_id; + bool m_exclusive; + std::string m_waiting_key; +}; + +struct DeadlockPath { + std::vector path; + bool limit_exceeded; + int64_t deadlock_time; + + explicit DeadlockPath(std::vector path_entry, + const int64_t& dl_time) + : path(path_entry), limit_exceeded(false), deadlock_time(dl_time) {} + + // empty path, limit exceeded constructor and default constructor + explicit DeadlockPath(const int64_t& dl_time = 0, bool limit = false) + : path(0), limit_exceeded(limit), deadlock_time(dl_time) {} + + bool empty() { return path.empty() && !limit_exceeded; } +}; + +class TransactionDB : public StackableDB { + public: + // Optimized version of ::Write that receives more optimization request such + // as skip_concurrency_control. + using StackableDB::Write; + virtual Status Write(const WriteOptions& opts, + const TransactionDBWriteOptimizations&, + WriteBatch* updates) { + // The default implementation ignores TransactionDBWriteOptimizations and + // falls back to the un-optimized version of ::Write + return Write(opts, updates); + } + // Transactional `DeleteRange()` is not yet supported. + // However, users who know their deleted range does not conflict with + // anything can still use it via the `Write()` API. In all cases, the + // `Write()` overload specifying `TransactionDBWriteOptimizations` must be + // used and `skip_concurrency_control` must be set. When using either + // WRITE_PREPARED or WRITE_UNPREPARED , `skip_duplicate_key_check` must + // additionally be set. + using StackableDB::DeleteRange; + virtual Status DeleteRange(const WriteOptions&, ColumnFamilyHandle*, + const Slice&, const Slice&) override { + return Status::NotSupported(); + } + // Open a TransactionDB similar to DB::Open(). + // Internally call PrepareWrap() and WrapDB() + // If the return status is not ok, then dbptr is set to nullptr. + static Status Open(const Options& options, + const TransactionDBOptions& txn_db_options, + const std::string& dbname, TransactionDB** dbptr); + + static Status Open(const DBOptions& db_options, + const TransactionDBOptions& txn_db_options, + const std::string& dbname, + const std::vector& column_families, + std::vector* handles, + TransactionDB** dbptr); + // Note: PrepareWrap() may change parameters, make copies before the + // invocation if needed. + static void PrepareWrap(DBOptions* db_options, + std::vector* column_families, + std::vector* compaction_enabled_cf_indices); + // If the return status is not ok, then dbptr will bet set to nullptr. The + // input db parameter might or might not be deleted as a result of the + // failure. If it is properly deleted it will be set to nullptr. If the return + // status is ok, the ownership of db is transferred to dbptr. + static Status WrapDB(DB* db, const TransactionDBOptions& txn_db_options, + const std::vector& compaction_enabled_cf_indices, + const std::vector& handles, + TransactionDB** dbptr); + // If the return status is not ok, then dbptr will bet set to nullptr. The + // input db parameter might or might not be deleted as a result of the + // failure. If it is properly deleted it will be set to nullptr. If the return + // status is ok, the ownership of db is transferred to dbptr. + static Status WrapStackableDB( + StackableDB* db, const TransactionDBOptions& txn_db_options, + const std::vector& compaction_enabled_cf_indices, + const std::vector& handles, TransactionDB** dbptr); + // Since the destructor in StackableDB is virtual, this destructor is virtual + // too. The root db will be deleted by the base's destructor. + ~TransactionDB() override {} + + // Starts a new Transaction. + // + // Caller is responsible for deleting the returned transaction when no + // longer needed. + // + // If old_txn is not null, BeginTransaction will reuse this Transaction + // handle instead of allocating a new one. This is an optimization to avoid + // extra allocations when repeatedly creating transactions. + virtual Transaction* BeginTransaction( + const WriteOptions& write_options, + const TransactionOptions& txn_options = TransactionOptions(), + Transaction* old_txn = nullptr) = 0; + + virtual Transaction* GetTransactionByName(const TransactionName& name) = 0; + virtual void GetAllPreparedTransactions(std::vector* trans) = 0; + + // Returns set of all locks held. + // + // The mapping is column family id -> KeyLockInfo + virtual std::unordered_multimap + GetLockStatusData() = 0; + + virtual std::vector GetDeadlockInfoBuffer() = 0; + virtual void SetDeadlockInfoBufferSize(uint32_t target_size) = 0; + + // Create a snapshot and assign ts to it. Return the snapshot to caller. The + // snapshot-timestamp mapping is also tracked by the database. + // Caller must ensure there are no active writes when this API is called. + virtual std::pair> + CreateTimestampedSnapshot(TxnTimestamp ts) = 0; + + // Return the latest timestamped snapshot if present. + std::shared_ptr GetLatestTimestampedSnapshot() const { + return GetTimestampedSnapshot(kMaxTxnTimestamp); + } + // Return the snapshot correponding to given timestamp. If ts is + // kMaxTxnTimestamp, then we return the latest timestamped snapshot if + // present. Othersise, we return the snapshot whose timestamp is equal to + // `ts`. If no such snapshot exists, then we return null. + virtual std::shared_ptr GetTimestampedSnapshot( + TxnTimestamp ts) const = 0; + // Release timestamped snapshots whose timestamps are less than or equal to + // ts. + virtual void ReleaseTimestampedSnapshotsOlderThan(TxnTimestamp ts) = 0; + + // Get all timestamped snapshots which will be stored in + // timestamped_snapshots. + Status GetAllTimestampedSnapshots( + std::vector>& timestamped_snapshots) + const { + return GetTimestampedSnapshots(/*ts_lb=*/0, /*ts_ub=*/kMaxTxnTimestamp, + timestamped_snapshots); + } + + // Get all timestamped snapshots whose timestamps fall within [ts_lb, ts_ub). + // timestamped_snapshots will be cleared and contain returned snapshots. + virtual Status GetTimestampedSnapshots( + TxnTimestamp ts_lb, TxnTimestamp ts_ub, + std::vector>& timestamped_snapshots) + const = 0; + + protected: + // To Create an TransactionDB, call Open() + // The ownership of db is transferred to the base StackableDB + explicit TransactionDB(DB* db) : StackableDB(db) {} + // No copying allowed + TransactionDB(const TransactionDB&) = delete; + void operator=(const TransactionDB&) = delete; +}; + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/transaction_db_mutex.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/transaction_db_mutex.h new file mode 100644 index 0000000..4ef566d --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/transaction_db_mutex.h @@ -0,0 +1,89 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +// TransactionDBMutex and TransactionDBCondVar APIs allows applications to +// implement custom mutexes and condition variables to be used by a +// TransactionDB when locking keys. +// +// To open a TransactionDB with a custom TransactionDBMutexFactory, set +// TransactionDBOptions.custom_mutex_factory. +class TransactionDBMutex { + public: + virtual ~TransactionDBMutex() {} + + // Attempt to acquire lock. Return OK on success, or other Status on failure. + // If returned status is OK, TransactionDB will eventually call UnLock(). + virtual Status Lock() = 0; + + // Attempt to acquire lock. If timeout is non-negative, operation may be + // failed after this many microseconds. + // Returns OK on success, + // TimedOut if timed out, + // or other Status on failure. + // If returned status is OK, TransactionDB will eventually call UnLock(). + virtual Status TryLockFor(int64_t timeout_time) = 0; + + // Unlock Mutex that was successfully locked by Lock() or TryLockUntil() + virtual void UnLock() = 0; +}; + +class TransactionDBCondVar { + public: + virtual ~TransactionDBCondVar() {} + + // Block current thread until condition variable is notified by a call to + // Notify() or NotifyAll(). Wait() will be called with mutex locked. + // Returns OK if notified. + // Returns non-OK if TransactionDB should stop waiting and fail the operation. + // May return OK spuriously even if not notified. + virtual Status Wait(std::shared_ptr mutex) = 0; + + // Block current thread until condition variable is notified by a call to + // Notify() or NotifyAll(), or if the timeout is reached. + // Wait() will be called with mutex locked. + // + // If timeout is non-negative, operation should be failed after this many + // microseconds. + // If implementing a custom version of this class, the implementation may + // choose to ignore the timeout. + // + // Returns OK if notified. + // Returns TimedOut if timeout is reached. + // Returns other status if TransactionDB should otherwise stop waiting and + // fail the operation. + // May return OK spuriously even if not notified. + virtual Status WaitFor(std::shared_ptr mutex, + int64_t timeout_time) = 0; + + // If any threads are waiting on *this, unblock at least one of the + // waiting threads. + virtual void Notify() = 0; + + // Unblocks all threads waiting on *this. + virtual void NotifyAll() = 0; +}; + +// Factory class that can allocate mutexes and condition variables. +class TransactionDBMutexFactory { + public: + // Create a TransactionDBMutex object. + virtual std::shared_ptr AllocateMutex() = 0; + + // Create a TransactionDBCondVar object. + virtual std::shared_ptr AllocateCondVar() = 0; + + virtual ~TransactionDBMutexFactory() {} +}; + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/include/rocksdb/utilities/write_batch_with_index.h b/librocksdb-sys/rocksdb/include/rocksdb/utilities/write_batch_with_index.h new file mode 100644 index 0000000..ae1c088 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/utilities/write_batch_with_index.h @@ -0,0 +1,304 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// A WriteBatchWithIndex with a binary searchable index built for all the keys +// inserted. +#pragma once + + +#include +#include +#include + +#include "rocksdb/comparator.h" +#include "rocksdb/iterator.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "rocksdb/write_batch.h" +#include "rocksdb/write_batch_base.h" + +namespace ROCKSDB_NAMESPACE { + +class ColumnFamilyHandle; +class Comparator; +class DB; +class ReadCallback; +struct ReadOptions; +struct DBOptions; + +enum WriteType { + kPutRecord, + kMergeRecord, + kDeleteRecord, + kSingleDeleteRecord, + kDeleteRangeRecord, + kLogDataRecord, + kXIDRecord, + kUnknownRecord, +}; + +// an entry for Put, Merge, Delete, or SingleDelete entry for write batches. +// Used in WBWIIterator. +struct WriteEntry { + WriteType type = kUnknownRecord; + Slice key; + Slice value; +}; + +// Iterator of one column family out of a WriteBatchWithIndex. +class WBWIIterator { + public: + virtual ~WBWIIterator() {} + + virtual bool Valid() const = 0; + + virtual void SeekToFirst() = 0; + + virtual void SeekToLast() = 0; + + virtual void Seek(const Slice& key) = 0; + + virtual void SeekForPrev(const Slice& key) = 0; + + virtual void Next() = 0; + + virtual void Prev() = 0; + + // the return WriteEntry is only valid until the next mutation of + // WriteBatchWithIndex + virtual WriteEntry Entry() const = 0; + + virtual Status status() const = 0; +}; + +// A WriteBatchWithIndex with a binary searchable index built for all the keys +// inserted. +// In Put(), Merge() Delete(), or SingleDelete(), the same function of the +// wrapped will be called. At the same time, indexes will be built. +// By calling GetWriteBatch(), a user will get the WriteBatch for the data +// they inserted, which can be used for DB::Write(). +// A user can call NewIterator() to create an iterator. +class WriteBatchWithIndex : public WriteBatchBase { + public: + // backup_index_comparator: the backup comparator used to compare keys + // within the same column family, if column family is not given in the + // interface, or we can't find a column family from the column family handle + // passed in, backup_index_comparator will be used for the column family. + // reserved_bytes: reserved bytes in underlying WriteBatch + // max_bytes: maximum size of underlying WriteBatch in bytes + // overwrite_key: if true, overwrite the key in the index when inserting + // the same key as previously, so iterator will never + // show two entries with the same key. + explicit WriteBatchWithIndex( + const Comparator* backup_index_comparator = BytewiseComparator(), + size_t reserved_bytes = 0, bool overwrite_key = false, + size_t max_bytes = 0, size_t protection_bytes_per_key = 0); + + ~WriteBatchWithIndex() override; + WriteBatchWithIndex(WriteBatchWithIndex&&); + WriteBatchWithIndex& operator=(WriteBatchWithIndex&&); + + using WriteBatchBase::Put; + Status Put(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& value) override; + + Status Put(const Slice& key, const Slice& value) override; + + Status Put(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& ts, const Slice& value) override; + + Status PutEntity(ColumnFamilyHandle* column_family, const Slice& /* key */, + const WideColumns& /* columns */) override { + if (!column_family) { + return Status::InvalidArgument( + "Cannot call this method without a column family handle"); + } + + return Status::NotSupported( + "PutEntity not supported by WriteBatchWithIndex"); + } + + using WriteBatchBase::Merge; + Status Merge(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& value) override; + + Status Merge(const Slice& key, const Slice& value) override; + Status Merge(ColumnFamilyHandle* /*column_family*/, const Slice& /*key*/, + const Slice& /*ts*/, const Slice& /*value*/) override { + return Status::NotSupported( + "Merge does not support user-defined timestamp"); + } + + using WriteBatchBase::Delete; + Status Delete(ColumnFamilyHandle* column_family, const Slice& key) override; + Status Delete(const Slice& key) override; + Status Delete(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& ts) override; + + using WriteBatchBase::SingleDelete; + Status SingleDelete(ColumnFamilyHandle* column_family, + const Slice& key) override; + Status SingleDelete(const Slice& key) override; + Status SingleDelete(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& ts) override; + + using WriteBatchBase::DeleteRange; + Status DeleteRange(ColumnFamilyHandle* /* column_family */, + const Slice& /* begin_key */, + const Slice& /* end_key */) override { + return Status::NotSupported( + "DeleteRange unsupported in WriteBatchWithIndex"); + } + Status DeleteRange(const Slice& /* begin_key */, + const Slice& /* end_key */) override { + return Status::NotSupported( + "DeleteRange unsupported in WriteBatchWithIndex"); + } + Status DeleteRange(ColumnFamilyHandle* /*column_family*/, + const Slice& /*begin_key*/, const Slice& /*end_key*/, + const Slice& /*ts*/) override { + return Status::NotSupported( + "DeleteRange unsupported in WriteBatchWithIndex"); + } + + using WriteBatchBase::PutLogData; + Status PutLogData(const Slice& blob) override; + + using WriteBatchBase::Clear; + void Clear() override; + + using WriteBatchBase::GetWriteBatch; + WriteBatch* GetWriteBatch() override; + + // Create an iterator of a column family. User can call iterator.Seek() to + // search to the next entry of or after a key. Keys will be iterated in the + // order given by index_comparator. For multiple updates on the same key, + // each update will be returned as a separate entry, in the order of update + // time. + // + // The returned iterator should be deleted by the caller. + WBWIIterator* NewIterator(ColumnFamilyHandle* column_family); + // Create an iterator of the default column family. + WBWIIterator* NewIterator(); + + // Will create a new Iterator that will use WBWIIterator as a delta and + // base_iterator as base. + // + // The returned iterator should be deleted by the caller. + // The base_iterator is now 'owned' by the returned iterator. Deleting the + // returned iterator will also delete the base_iterator. + // + // Updating write batch with the current key of the iterator is not safe. + // We strongly recommend users not to do it. It will invalidate the current + // key() and value() of the iterator. This invalidation happens even before + // the write batch update finishes. The state may recover after Next() is + // called. + Iterator* NewIteratorWithBase(ColumnFamilyHandle* column_family, + Iterator* base_iterator, + const ReadOptions* opts = nullptr); + // default column family + Iterator* NewIteratorWithBase(Iterator* base_iterator); + + // Similar to DB::Get() but will only read the key from this batch. + // If the batch does not have enough data to resolve Merge operations, + // MergeInProgress status may be returned. + Status GetFromBatch(ColumnFamilyHandle* column_family, + const DBOptions& options, const Slice& key, + std::string* value); + + // Similar to previous function but does not require a column_family. + // Note: An InvalidArgument status will be returned if there are any Merge + // operators for this key. Use previous method instead. + Status GetFromBatch(const DBOptions& options, const Slice& key, + std::string* value) { + return GetFromBatch(nullptr, options, key, value); + } + + // Similar to DB::Get() but will also read writes from this batch. + // + // This function will query both this batch and the DB and then merge + // the results using the DB's merge operator (if the batch contains any + // merge requests). + // + // Setting read_options.snapshot will affect what is read from the DB + // but will NOT change which keys are read from the batch (the keys in + // this batch do not yet belong to any snapshot and will be fetched + // regardless). + Status GetFromBatchAndDB(DB* db, const ReadOptions& read_options, + const Slice& key, std::string* value); + + // An overload of the above method that receives a PinnableSlice + Status GetFromBatchAndDB(DB* db, const ReadOptions& read_options, + const Slice& key, PinnableSlice* value); + + Status GetFromBatchAndDB(DB* db, const ReadOptions& read_options, + ColumnFamilyHandle* column_family, const Slice& key, + std::string* value); + + // An overload of the above method that receives a PinnableSlice + Status GetFromBatchAndDB(DB* db, const ReadOptions& read_options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value); + + void MultiGetFromBatchAndDB(DB* db, const ReadOptions& read_options, + ColumnFamilyHandle* column_family, + const size_t num_keys, const Slice* keys, + PinnableSlice* values, Status* statuses, + bool sorted_input); + + // Records the state of the batch for future calls to RollbackToSavePoint(). + // May be called multiple times to set multiple save points. + void SetSavePoint() override; + + // Remove all entries in this batch (Put, Merge, Delete, SingleDelete, + // PutLogData) since the most recent call to SetSavePoint() and removes the + // most recent save point. + // If there is no previous call to SetSavePoint(), behaves the same as + // Clear(). + // + // Calling RollbackToSavePoint invalidates any open iterators on this batch. + // + // Returns Status::OK() on success, + // Status::NotFound() if no previous call to SetSavePoint(), + // or other Status on corruption. + Status RollbackToSavePoint() override; + + // Pop the most recent save point. + // If there is no previous call to SetSavePoint(), Status::NotFound() + // will be returned. + // Otherwise returns Status::OK(). + Status PopSavePoint() override; + + void SetMaxBytes(size_t max_bytes) override; + size_t GetDataSize() const; + + private: + friend class PessimisticTransactionDB; + friend class WritePreparedTxn; + friend class WriteUnpreparedTxn; + friend class WriteBatchWithIndex_SubBatchCnt_Test; + friend class WriteBatchWithIndexInternal; + // Returns the number of sub-batches inside the write batch. A sub-batch + // starts right before inserting a key that is a duplicate of a key in the + // last sub-batch. + size_t SubBatchCnt(); + + Status GetFromBatchAndDB(DB* db, const ReadOptions& read_options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value, ReadCallback* callback); + void MultiGetFromBatchAndDB(DB* db, const ReadOptions& read_options, + ColumnFamilyHandle* column_family, + const size_t num_keys, const Slice* keys, + PinnableSlice* values, Status* statuses, + bool sorted_input, ReadCallback* callback); + struct Rep; + std::unique_ptr rep; +}; + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/include/rocksdb/version.h b/librocksdb-sys/rocksdb/include/rocksdb/version.h new file mode 100644 index 0000000..de6629d --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/version.h @@ -0,0 +1,43 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include +#include + +#include "rocksdb/rocksdb_namespace.h" + +// NOTE: in 'main' development branch, this should be the *next* +// minor or major version number planned for release. +#define ROCKSDB_MAJOR 8 +#define ROCKSDB_MINOR 6 +#define ROCKSDB_PATCH 0 + +// Do not use these. We made the mistake of declaring macros starting with +// double underscore. Now we have to live with our choice. We'll deprecate these +// at some point +#define __ROCKSDB_MAJOR__ ROCKSDB_MAJOR +#define __ROCKSDB_MINOR__ ROCKSDB_MINOR +#define __ROCKSDB_PATCH__ ROCKSDB_PATCH + +namespace ROCKSDB_NAMESPACE { +// Returns a set of properties indicating how/when/where this version of RocksDB +// was created. +const std::unordered_map& GetRocksBuildProperties(); + +// Returns the current version of RocksDB as a string (e.g. "6.16.0"). +// If with_patch is true, the patch is included (6.16.x). +// Otherwise, only major and minor version is included (6.16) +std::string GetRocksVersionAsString(bool with_patch = true); + +// Gets the set of build properties (@see GetRocksBuildProperties) into a +// string. Properties are returned one-per-line, with the first line being: +// " from RocksDB . +// If verbose is true, the full set of properties is +// printed. If verbose is false, only the version information (@see +// GetRocksVersionString) is printed. +std::string GetRocksBuildInfoAsString(const std::string& program, + bool verbose = false); +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/wal_filter.h b/librocksdb-sys/rocksdb/include/rocksdb/wal_filter.h new file mode 100644 index 0000000..3e66c39 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/wal_filter.h @@ -0,0 +1,111 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include + +#include "rocksdb/customizable.h" +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +class WriteBatch; +struct ConfigOptions; + +// WALFilter allows an application to inspect write-ahead-log (WAL) +// records or modify their processing on recovery. +// Please see the details below. +// +// Exceptions MUST NOT propagate out of overridden functions into RocksDB, +// because RocksDB is not exception-safe. This could cause undefined behavior +// including data loss, unreported corruption, deadlocks, and more. +class WalFilter : public Customizable { + public: + static const char* Type() { return "WalFilter"; } + static Status CreateFromString(const ConfigOptions& options, + const std::string& value, WalFilter** result); + enum class WalProcessingOption { + // Continue processing as usual + kContinueProcessing = 0, + // Ignore the current record but continue processing of log(s) + kIgnoreCurrentRecord = 1, + // Stop replay of logs and discard logs + // Logs won't be replayed on subsequent recovery + kStopReplay = 2, + // Corrupted record detected by filter + kCorruptedRecord = 3, + // Marker for enum count + kWalProcessingOptionMax = 4 + }; + + virtual ~WalFilter() {} + + // Provide ColumnFamily->LogNumber map to filter + // so that filter can determine whether a log number applies to a given + // column family (i.e. that log hasn't been flushed to SST already for the + // column family). + // We also pass in name->id map as only name is known during + // recovery (as handles are opened post-recovery). + // while write batch callbacks happen in terms of column family id. + // + // @params cf_lognumber_map column_family_id to lognumber map + // @params cf_name_id_map column_family_name to column_family_id map + + virtual void ColumnFamilyLogNumberMap( + const std::map& /*cf_lognumber_map*/, + const std::map& /*cf_name_id_map*/) {} + + // LogRecord is invoked for each log record encountered for all the logs + // during replay on logs on recovery. This method can be used to: + // * inspect the record (using the batch parameter) + // * ignoring current record + // (by returning WalProcessingOption::kIgnoreCurrentRecord) + // * reporting corrupted record + // (by returning WalProcessingOption::kCorruptedRecord) + // * stop log replay + // (by returning kStop replay) - please note that this implies + // discarding the logs from current record onwards. + // + // @params log_number log_number of the current log. + // Filter might use this to determine if the log + // record is applicable to a certain column family. + // @params log_file_name log file name - only for informational purposes + // @params batch batch encountered in the log during recovery + // @params new_batch new_batch to populate if filter wants to change + // the batch (for example to filter some records out, + // or alter some records). + // Please note that the new batch MUST NOT contain + // more records than original, else recovery would + // be failed. + // @params batch_changed Whether batch was changed by the filter. + // It must be set to true if new_batch was populated, + // else new_batch has no effect. + // @returns Processing option for the current record. + // Please see WalProcessingOption enum above for + // details. + virtual WalProcessingOption LogRecordFound( + unsigned long long /*log_number*/, const std::string& /*log_file_name*/, + const WriteBatch& batch, WriteBatch* new_batch, bool* batch_changed) { + // Default implementation falls back to older function for compatibility + return LogRecord(batch, new_batch, batch_changed); + } + + // Please see the comments for LogRecord above. This function is for + // compatibility only and contains a subset of parameters. + // New code should use the function above. + virtual WalProcessingOption LogRecord(const WriteBatch& /*batch*/, + WriteBatch* /*new_batch*/, + bool* /*batch_changed*/) const { + return WalProcessingOption::kContinueProcessing; + } + + // Returns a name that identifies this WAL filter. + // The name will be printed to LOG file on start up for diagnosis. + virtual const char* Name() const override = 0; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/wide_columns.h b/librocksdb-sys/rocksdb/include/rocksdb/wide_columns.h new file mode 100644 index 0000000..5af3f51 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/wide_columns.h @@ -0,0 +1,210 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include +#include + +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +// Class representing a wide column, which is defined as a pair of column name +// and column value. +class WideColumn { + public: + WideColumn() = default; + + // Initializes a WideColumn object by forwarding the name and value + // arguments to the corresponding member Slices. This makes it possible to + // construct a WideColumn using combinations of const char*, const + // std::string&, const Slice& etc., for example: + // + // constexpr char foo[] = "foo"; + // const std::string bar("bar"); + // WideColumn column(foo, bar); + template + WideColumn(N&& name, V&& value) + : name_(std::forward(name)), value_(std::forward(value)) {} + + // Initializes a WideColumn object by forwarding the elements of + // name_tuple and value_tuple to the constructors of the corresponding member + // Slices. This makes it possible to initialize the Slices using the Slice + // constructors that take more than one argument, for example: + // + // constexpr char foo_name[] = "foo_name"; + // constexpr char bar_value[] = "bar_value"; + // WideColumn column(std::piecewise_construct, + // std::forward_as_tuple(foo_name, 3), + // std::forward_as_tuple(bar_value, 3)); + template + WideColumn(std::piecewise_construct_t, NTuple&& name_tuple, + VTuple&& value_tuple) + : name_(std::make_from_tuple(std::forward(name_tuple))), + value_(std::make_from_tuple(std::forward(value_tuple))) { + } + + const Slice& name() const { return name_; } + const Slice& value() const { return value_; } + + Slice& name() { return name_; } + Slice& value() { return value_; } + + private: + Slice name_; + Slice value_; +}; + +// Note: column names and values are compared bytewise. +inline bool operator==(const WideColumn& lhs, const WideColumn& rhs) { + return lhs.name() == rhs.name() && lhs.value() == rhs.value(); +} + +inline bool operator!=(const WideColumn& lhs, const WideColumn& rhs) { + return !(lhs == rhs); +} + +inline std::ostream& operator<<(std::ostream& os, const WideColumn& column) { + const bool hex = + (os.flags() & std::ios_base::basefield) == std::ios_base::hex; + os << column.name().ToString(hex) << ':' << column.value().ToString(hex); + + return os; +} + +// A collection of wide columns. +using WideColumns = std::vector; + +// The anonymous default wide column (an empty Slice). +extern const Slice kDefaultWideColumnName; + +// An empty set of wide columns. +extern const WideColumns kNoWideColumns; + +// A self-contained collection of wide columns. Used for the results of +// wide-column queries. +class PinnableWideColumns { + public: + const WideColumns& columns() const { return columns_; } + size_t serialized_size() const { return value_.size(); } + + void SetPlainValue(const Slice& value); + void SetPlainValue(const Slice& value, Cleanable* cleanable); + void SetPlainValue(PinnableSlice&& value); + void SetPlainValue(std::string&& value); + + Status SetWideColumnValue(const Slice& value); + Status SetWideColumnValue(const Slice& value, Cleanable* cleanable); + Status SetWideColumnValue(PinnableSlice&& value); + Status SetWideColumnValue(std::string&& value); + + void Reset(); + + private: + void CopyValue(const Slice& value); + void PinOrCopyValue(const Slice& value, Cleanable* cleanable); + void MoveValue(PinnableSlice&& value); + void MoveValue(std::string&& value); + + void CreateIndexForPlainValue(); + Status CreateIndexForWideColumns(); + + PinnableSlice value_; + WideColumns columns_; +}; + +inline void PinnableWideColumns::CopyValue(const Slice& value) { + value_.PinSelf(value); +} + +inline void PinnableWideColumns::PinOrCopyValue(const Slice& value, + Cleanable* cleanable) { + if (!cleanable) { + CopyValue(value); + return; + } + + value_.PinSlice(value, cleanable); +} + +inline void PinnableWideColumns::MoveValue(PinnableSlice&& value) { + value_ = std::move(value); +} + +inline void PinnableWideColumns::MoveValue(std::string&& value) { + std::string* const buf = value_.GetSelf(); + assert(buf); + + *buf = std::move(value); + value_.PinSelf(); +} + +inline void PinnableWideColumns::CreateIndexForPlainValue() { + columns_ = WideColumns{{kDefaultWideColumnName, value_}}; +} + +inline void PinnableWideColumns::SetPlainValue(const Slice& value) { + CopyValue(value); + CreateIndexForPlainValue(); +} + +inline void PinnableWideColumns::SetPlainValue(const Slice& value, + Cleanable* cleanable) { + PinOrCopyValue(value, cleanable); + CreateIndexForPlainValue(); +} + +inline void PinnableWideColumns::SetPlainValue(PinnableSlice&& value) { + MoveValue(std::move(value)); + CreateIndexForPlainValue(); +} + +inline void PinnableWideColumns::SetPlainValue(std::string&& value) { + MoveValue(std::move(value)); + CreateIndexForPlainValue(); +} + +inline Status PinnableWideColumns::SetWideColumnValue(const Slice& value) { + CopyValue(value); + return CreateIndexForWideColumns(); +} + +inline Status PinnableWideColumns::SetWideColumnValue(const Slice& value, + Cleanable* cleanable) { + PinOrCopyValue(value, cleanable); + return CreateIndexForWideColumns(); +} + +inline Status PinnableWideColumns::SetWideColumnValue(PinnableSlice&& value) { + MoveValue(std::move(value)); + return CreateIndexForWideColumns(); +} + +inline Status PinnableWideColumns::SetWideColumnValue(std::string&& value) { + MoveValue(std::move(value)); + return CreateIndexForWideColumns(); +} + +inline void PinnableWideColumns::Reset() { + value_.Reset(); + columns_.clear(); +} + +inline bool operator==(const PinnableWideColumns& lhs, + const PinnableWideColumns& rhs) { + return lhs.columns() == rhs.columns(); +} + +inline bool operator!=(const PinnableWideColumns& lhs, + const PinnableWideColumns& rhs) { + return !(lhs == rhs); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/write_batch.h b/librocksdb-sys/rocksdb/include/rocksdb/write_batch.h new file mode 100644 index 0000000..307cd75 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/write_batch.h @@ -0,0 +1,497 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// WriteBatch holds a collection of updates to apply atomically to a DB. +// +// The updates are applied in the order in which they are added +// to the WriteBatch. For example, the value of "key" will be "v3" +// after the following batch is written: +// +// batch.Put("key", "v1"); +// batch.Delete("key"); +// batch.Put("key", "v2"); +// batch.Put("key", "v3"); +// +// Multiple threads can invoke const methods on a WriteBatch without +// external synchronization, but if any of the threads may call a +// non-const method, all threads accessing the same WriteBatch must use +// external synchronization. + +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include "rocksdb/status.h" +#include "rocksdb/write_batch_base.h" + +namespace ROCKSDB_NAMESPACE { + +class Slice; +class ColumnFamilyHandle; +struct SavePoints; +struct SliceParts; + +struct SavePoint { + size_t size; // size of rep_ + int count; // count of elements in rep_ + uint32_t content_flags; + + SavePoint() : size(0), count(0), content_flags(0) {} + + SavePoint(size_t _size, int _count, uint32_t _flags) + : size(_size), count(_count), content_flags(_flags) {} + + void clear() { + size = 0; + count = 0; + content_flags = 0; + } + + bool is_cleared() const { return (size | count | content_flags) == 0; } +}; + +class WriteBatch : public WriteBatchBase { + public: + explicit WriteBatch(size_t reserved_bytes = 0, size_t max_bytes = 0) + : WriteBatch(reserved_bytes, max_bytes, 0, 0) {} + + // `protection_bytes_per_key` is the number of bytes used to store + // protection information for each key entry. Currently supported values are + // zero (disabled) and eight. + explicit WriteBatch(size_t reserved_bytes, size_t max_bytes, + size_t protection_bytes_per_key, size_t default_cf_ts_sz); + ~WriteBatch() override; + + using WriteBatchBase::Put; + // Store the mapping "key->value" in the database. + // The following Put(..., const Slice& key, ...) API can also be used when + // user-defined timestamp is enabled as long as `key` points to a contiguous + // buffer with timestamp appended after user key. The caller is responsible + // for setting up the memory buffer pointed to by `key`. + Status Put(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& value) override; + Status Put(const Slice& key, const Slice& value) override { + return Put(nullptr, key, value); + } + Status Put(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& ts, const Slice& value) override; + + // Variant of Put() that gathers output like writev(2). The key and value + // that will be written to the database are concatenations of arrays of + // slices. + // The following Put(..., const SliceParts& key, ...) API can be used when + // user-defined timestamp is enabled as long as the timestamp is the last + // Slice in `key`, a SliceParts (array of Slices). The caller is responsible + // for setting up the `key` SliceParts object. + Status Put(ColumnFamilyHandle* column_family, const SliceParts& key, + const SliceParts& value) override; + Status Put(const SliceParts& key, const SliceParts& value) override { + return Put(nullptr, key, value); + } + + // Store the mapping "key->{column1:value1, column2:value2, ...}" in the + // column family specified by "column_family". + using WriteBatchBase::PutEntity; + Status PutEntity(ColumnFamilyHandle* column_family, const Slice& key, + const WideColumns& columns) override; + + using WriteBatchBase::Delete; + // If the database contains a mapping for "key", erase it. Else do nothing. + // The following Delete(..., const Slice& key) can be used when user-defined + // timestamp is enabled as long as `key` points to a contiguous buffer with + // timestamp appended after user key. The caller is responsible for setting + // up the memory buffer pointed to by `key`. + Status Delete(ColumnFamilyHandle* column_family, const Slice& key) override; + Status Delete(const Slice& key) override { return Delete(nullptr, key); } + Status Delete(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& ts) override; + + // variant that takes SliceParts + // These two variants of Delete(..., const SliceParts& key) can be used when + // user-defined timestamp is enabled as long as the timestamp is the last + // Slice in `key`, a SliceParts (array of Slices). The caller is responsible + // for setting up the `key` SliceParts object. + Status Delete(ColumnFamilyHandle* column_family, + const SliceParts& key) override; + Status Delete(const SliceParts& key) override { return Delete(nullptr, key); } + + using WriteBatchBase::SingleDelete; + // WriteBatch implementation of DB::SingleDelete(). See db.h. + Status SingleDelete(ColumnFamilyHandle* column_family, + const Slice& key) override; + Status SingleDelete(const Slice& key) override { + return SingleDelete(nullptr, key); + } + Status SingleDelete(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& ts) override; + + // variant that takes SliceParts + Status SingleDelete(ColumnFamilyHandle* column_family, + const SliceParts& key) override; + Status SingleDelete(const SliceParts& key) override { + return SingleDelete(nullptr, key); + } + + using WriteBatchBase::DeleteRange; + // WriteBatch implementation of DB::DeleteRange(). See db.h. + Status DeleteRange(ColumnFamilyHandle* column_family, const Slice& begin_key, + const Slice& end_key) override; + Status DeleteRange(const Slice& begin_key, const Slice& end_key) override { + return DeleteRange(nullptr, begin_key, end_key); + } + // begin_key and end_key should be user keys without timestamp. + Status DeleteRange(ColumnFamilyHandle* column_family, const Slice& begin_key, + const Slice& end_key, const Slice& ts) override; + + // variant that takes SliceParts + Status DeleteRange(ColumnFamilyHandle* column_family, + const SliceParts& begin_key, + const SliceParts& end_key) override; + Status DeleteRange(const SliceParts& begin_key, + const SliceParts& end_key) override { + return DeleteRange(nullptr, begin_key, end_key); + } + + using WriteBatchBase::Merge; + // Merge "value" with the existing value of "key" in the database. + // "key->merge(existing, value)" + Status Merge(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& value) override; + Status Merge(const Slice& key, const Slice& value) override { + return Merge(nullptr, key, value); + } + Status Merge(ColumnFamilyHandle* /*column_family*/, const Slice& /*key*/, + const Slice& /*ts*/, const Slice& /*value*/) override; + + // variant that takes SliceParts + Status Merge(ColumnFamilyHandle* column_family, const SliceParts& key, + const SliceParts& value) override; + Status Merge(const SliceParts& key, const SliceParts& value) override { + return Merge(nullptr, key, value); + } + + using WriteBatchBase::PutLogData; + // Append a blob of arbitrary size to the records in this batch. The blob will + // be stored in the transaction log but not in any other file. In particular, + // it will not be persisted to the SST files. When iterating over this + // WriteBatch, WriteBatch::Handler::LogData will be called with the contents + // of the blob as it is encountered. Blobs, puts, deletes, and merges will be + // encountered in the same order in which they were inserted. The blob will + // NOT consume sequence number(s) and will NOT increase the count of the batch + // + // Example application: add timestamps to the transaction log for use in + // replication. + Status PutLogData(const Slice& blob) override; + + using WriteBatchBase::Clear; + // Clear all updates buffered in this batch. + void Clear() override; + + // Records the state of the batch for future calls to RollbackToSavePoint(). + // May be called multiple times to set multiple save points. + void SetSavePoint() override; + + // Remove all entries in this batch (Put, Merge, Delete, PutLogData) since the + // most recent call to SetSavePoint() and removes the most recent save point. + // If there is no previous call to SetSavePoint(), Status::NotFound() + // will be returned. + // Otherwise returns Status::OK(). + Status RollbackToSavePoint() override; + + // Pop the most recent save point. + // If there is no previous call to SetSavePoint(), Status::NotFound() + // will be returned. + // Otherwise returns Status::OK(). + Status PopSavePoint() override; + + // Support for iterating over the contents of a batch. + // Objects of subclasses of Handler will be used by WriteBatch::Iterate(). + class Handler { + public: + virtual ~Handler(); + // All handler functions in this class provide default implementations so + // we won't break existing clients of Handler on a source code level when + // adding a new member function. + + // default implementation will just call Put without column family for + // backwards compatibility. If the column family is not default, + // the function is noop + // If user-defined timestamp is enabled, then `key` includes timestamp. + virtual Status PutCF(uint32_t column_family_id, const Slice& key, + const Slice& value) { + if (column_family_id == 0) { + // Put() historically doesn't return status. We didn't want to be + // backwards incompatible so we didn't change the return status + // (this is a public API). We do an ordinary get and return Status::OK() + Put(key, value); + return Status::OK(); + } + return Status::InvalidArgument( + "non-default column family and PutCF not implemented"); + } + // If user-defined timestamp is enabled, then `key` includes timestamp. + virtual void Put(const Slice& /*key*/, const Slice& /*value*/) {} + + // If user-defined timestamp is enabled, then `key` includes timestamp. + virtual Status PutEntityCF(uint32_t /* column_family_id */, + const Slice& /* key */, + const Slice& /* entity */) { + return Status::NotSupported("PutEntityCF not implemented"); + } + + // If user-defined timestamp is enabled, then `key` includes timestamp. + virtual Status DeleteCF(uint32_t column_family_id, const Slice& key) { + if (column_family_id == 0) { + Delete(key); + return Status::OK(); + } + return Status::InvalidArgument( + "non-default column family and DeleteCF not implemented"); + } + // If user-defined timestamp is enabled, then `key` includes timestamp. + virtual void Delete(const Slice& /*key*/) {} + + // If user-defined timestamp is enabled, then `key` includes timestamp. + virtual Status SingleDeleteCF(uint32_t column_family_id, const Slice& key) { + if (column_family_id == 0) { + SingleDelete(key); + return Status::OK(); + } + return Status::InvalidArgument( + "non-default column family and SingleDeleteCF not implemented"); + } + // If user-defined timestamp is enabled, then `key` includes timestamp. + virtual void SingleDelete(const Slice& /*key*/) {} + + // If user-defined timestamp is enabled, then `begin_key` and `end_key` + // both include timestamp. + virtual Status DeleteRangeCF(uint32_t /*column_family_id*/, + const Slice& /*begin_key*/, + const Slice& /*end_key*/) { + return Status::InvalidArgument("DeleteRangeCF not implemented"); + } + + // If user-defined timestamp is enabled, then `key` includes timestamp. + virtual Status MergeCF(uint32_t column_family_id, const Slice& key, + const Slice& value) { + if (column_family_id == 0) { + Merge(key, value); + return Status::OK(); + } + return Status::InvalidArgument( + "non-default column family and MergeCF not implemented"); + } + // If user-defined timestamp is enabled, then `key` includes timestamp. + virtual void Merge(const Slice& /*key*/, const Slice& /*value*/) {} + + // If user-defined timestamp is enabled, then `key` includes timestamp. + virtual Status PutBlobIndexCF(uint32_t /*column_family_id*/, + const Slice& /*key*/, + const Slice& /*value*/) { + return Status::InvalidArgument("PutBlobIndexCF not implemented"); + } + + // The default implementation of LogData does nothing. + virtual void LogData(const Slice& blob); + + virtual Status MarkBeginPrepare(bool = false) { + return Status::InvalidArgument("MarkBeginPrepare() handler not defined."); + } + + virtual Status MarkEndPrepare(const Slice& /*xid*/) { + return Status::InvalidArgument("MarkEndPrepare() handler not defined."); + } + + virtual Status MarkNoop(bool /*empty_batch*/) { + return Status::InvalidArgument("MarkNoop() handler not defined."); + } + + virtual Status MarkRollback(const Slice& /*xid*/) { + return Status::InvalidArgument( + "MarkRollbackPrepare() handler not defined."); + } + + virtual Status MarkCommit(const Slice& /*xid*/) { + return Status::InvalidArgument("MarkCommit() handler not defined."); + } + + virtual Status MarkCommitWithTimestamp(const Slice& /*xid*/, + const Slice& /*commit_ts*/) { + return Status::InvalidArgument( + "MarkCommitWithTimestamp() handler not defined."); + } + + // Continue is called by WriteBatch::Iterate. If it returns false, + // iteration is halted. Otherwise, it continues iterating. The default + // implementation always returns true. + virtual bool Continue(); + + protected: + friend class WriteBatchInternal; + enum class OptionState { + kUnknown, + kDisabled, + kEnabled, + }; + virtual OptionState WriteAfterCommit() const { + return OptionState::kUnknown; + } + virtual OptionState WriteBeforePrepare() const { + return OptionState::kUnknown; + } + }; + Status Iterate(Handler* handler) const; + + // Retrieve the serialized version of this batch. + const std::string& Data() const { return rep_; } + + // Release the serialized data and clear this batch. + std::string Release(); + + // Retrieve data size of the batch. + size_t GetDataSize() const { return rep_.size(); } + + // Returns the number of updates in the batch + uint32_t Count() const; + + // Returns true if PutCF will be called during Iterate + bool HasPut() const; + + // Returns true if PutEntityCF will be called during Iterate + bool HasPutEntity() const; + + // Returns true if DeleteCF will be called during Iterate + bool HasDelete() const; + + // Returns true if SingleDeleteCF will be called during Iterate + bool HasSingleDelete() const; + + // Returns true if DeleteRangeCF will be called during Iterate + bool HasDeleteRange() const; + + // Returns true if MergeCF will be called during Iterate + bool HasMerge() const; + + // Returns true if MarkBeginPrepare will be called during Iterate + bool HasBeginPrepare() const; + + // Returns true if MarkEndPrepare will be called during Iterate + bool HasEndPrepare() const; + + // Returns true if MarkCommit will be called during Iterate + bool HasCommit() const; + + // Returns true if MarkRollback will be called during Iterate + bool HasRollback() const; + + // Experimental. + // + // Update timestamps of existing entries in the write batch if + // applicable. If a key is intended for a column family that disables + // timestamp, then this API won't set the timestamp for this key. + // This requires that all keys, if enable timestamp, (possibly from multiple + // column families) in the write batch have timestamps of the same format. + // + // ts_sz_func: callable object to obtain the timestamp sizes of column + // families. If ts_sz_func() accesses data structures, then the caller of this + // API must guarantee thread-safety. Like other parts of RocksDB, this API is + // not exception-safe. Therefore, ts_sz_func() must not throw. + // + // in: cf, the column family id. + // ret: timestamp size of the given column family. Return + // std::numeric_limits::max() indicating "don't know or column + // family info not found", this will cause UpdateTimestamps() to fail. + // size_t ts_sz_func(uint32_t cf); + Status UpdateTimestamps(const Slice& ts, + std::function ts_sz_func); + + // Verify the per-key-value checksums of this write batch. + // Corruption status will be returned if the verification fails. + // If this write batch does not have per-key-value checksum, + // OK status will be returned. + Status VerifyChecksum() const; + + using WriteBatchBase::GetWriteBatch; + WriteBatch* GetWriteBatch() override { return this; } + + // Constructor with a serialized string object + explicit WriteBatch(const std::string& rep); + explicit WriteBatch(std::string&& rep); + + WriteBatch(const WriteBatch& src); + WriteBatch(WriteBatch&& src) noexcept; + WriteBatch& operator=(const WriteBatch& src); + WriteBatch& operator=(WriteBatch&& src); + + // marks this point in the WriteBatch as the last record to + // be inserted into the WAL, provided the WAL is enabled + void MarkWalTerminationPoint(); + const SavePoint& GetWalTerminationPoint() const { return wal_term_point_; } + + void SetMaxBytes(size_t max_bytes) override { max_bytes_ = max_bytes; } + + struct ProtectionInfo; + size_t GetProtectionBytesPerKey() const; + + private: + friend class WriteBatchInternal; + friend class LocalSavePoint; + // TODO(myabandeh): this is needed for a hack to collapse the write batch and + // remove duplicate keys. Remove it when the hack is replaced with a proper + // solution. + friend class WriteBatchWithIndex; + std::unique_ptr save_points_; + + // When sending a WriteBatch through WriteImpl we might want to + // specify that only the first x records of the batch be written to + // the WAL. + SavePoint wal_term_point_; + + // Is the content of the batch the application's latest state that meant only + // to be used for recovery? Refer to + // TransactionOptions::use_only_the_last_commit_time_batch_for_recovery for + // more details. + bool is_latest_persistent_state_ = false; + + // False if all keys are from column families that disable user-defined + // timestamp OR UpdateTimestamps() has been called at least once. + // This flag will be set to true if any of the above Put(), Delete(), + // SingleDelete(), etc. APIs are called at least once. + // Calling Put(ts), Delete(ts), SingleDelete(ts), etc. will not set this flag + // to true because the assumption is that these APIs have already set the + // timestamps to desired values. + bool needs_in_place_update_ts_ = false; + + // True if the write batch contains at least one key from a column family + // that enables user-defined timestamp. + bool has_key_with_ts_ = false; + + // For HasXYZ. Mutable to allow lazy computation of results + mutable std::atomic content_flags_; + + // Performs deferred computation of content_flags if necessary + uint32_t ComputeContentFlags() const; + + // Maximum size of rep_. + size_t max_bytes_; + + std::unique_ptr prot_info_; + + size_t default_cf_ts_sz_ = 0; + + protected: + std::string rep_; // See comment in write_batch.cc for the format of rep_ +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/write_batch_base.h b/librocksdb-sys/rocksdb/include/rocksdb/write_batch_base.h new file mode 100644 index 0000000..f6f39ef --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/write_batch_base.h @@ -0,0 +1,144 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include + +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/wide_columns.h" + +namespace ROCKSDB_NAMESPACE { + +class Slice; +class Status; +class ColumnFamilyHandle; +class WriteBatch; +struct SliceParts; + +// Abstract base class that defines the basic interface for a write batch. +// See WriteBatch for a basic implementation and WrithBatchWithIndex for an +// indexed implementation. +class WriteBatchBase { + public: + virtual ~WriteBatchBase() {} + + // Store the mapping "key->value" in the database. + virtual Status Put(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& value) = 0; + virtual Status Put(const Slice& key, const Slice& value) = 0; + virtual Status Put(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& ts, const Slice& value) = 0; + + // Variant of Put() that gathers output like writev(2). The key and value + // that will be written to the database are concatenations of arrays of + // slices. + virtual Status Put(ColumnFamilyHandle* column_family, const SliceParts& key, + const SliceParts& value); + virtual Status Put(const SliceParts& key, const SliceParts& value); + + // Store the mapping "key->{column1:value1, column2:value2, ...}" in the + // column family specified by "column_family". + virtual Status PutEntity(ColumnFamilyHandle* column_family, const Slice& key, + const WideColumns& columns) = 0; + + // Merge "value" with the existing value of "key" in the database. + // "key->merge(existing, value)" + virtual Status Merge(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& value) = 0; + virtual Status Merge(const Slice& key, const Slice& value) = 0; + virtual Status Merge(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& ts, const Slice& value) = 0; + + // variant that takes SliceParts + virtual Status Merge(ColumnFamilyHandle* column_family, const SliceParts& key, + const SliceParts& value); + virtual Status Merge(const SliceParts& key, const SliceParts& value); + + // If the database contains a mapping for "key", erase it. Else do nothing. + virtual Status Delete(ColumnFamilyHandle* column_family, + const Slice& key) = 0; + virtual Status Delete(const Slice& key) = 0; + virtual Status Delete(ColumnFamilyHandle* column_family, const Slice& key, + const Slice& ts) = 0; + + // variant that takes SliceParts + virtual Status Delete(ColumnFamilyHandle* column_family, + const SliceParts& key); + virtual Status Delete(const SliceParts& key); + + // If the database contains a mapping for "key", erase it. Expects that the + // key was not overwritten. Else do nothing. + virtual Status SingleDelete(ColumnFamilyHandle* column_family, + const Slice& key) = 0; + virtual Status SingleDelete(const Slice& key) = 0; + virtual Status SingleDelete(ColumnFamilyHandle* column_family, + const Slice& key, const Slice& ts) = 0; + + // variant that takes SliceParts + virtual Status SingleDelete(ColumnFamilyHandle* column_family, + const SliceParts& key); + virtual Status SingleDelete(const SliceParts& key); + + // If the database contains mappings in the range ["begin_key", "end_key"), + // erase them. Else do nothing. + virtual Status DeleteRange(ColumnFamilyHandle* column_family, + const Slice& begin_key, const Slice& end_key) = 0; + virtual Status DeleteRange(const Slice& begin_key, const Slice& end_key) = 0; + virtual Status DeleteRange(ColumnFamilyHandle* column_family, + const Slice& begin_key, const Slice& end_key, + const Slice& ts) = 0; + + // variant that takes SliceParts + virtual Status DeleteRange(ColumnFamilyHandle* column_family, + const SliceParts& begin_key, + const SliceParts& end_key); + virtual Status DeleteRange(const SliceParts& begin_key, + const SliceParts& end_key); + + // Append a blob of arbitrary size to the records in this batch. The blob will + // be stored in the transaction log but not in any other file. In particular, + // it will not be persisted to the SST files. When iterating over this + // WriteBatch, WriteBatch::Handler::LogData will be called with the contents + // of the blob as it is encountered. Blobs, puts, deletes, and merges will be + // encountered in the same order in which they were inserted. The blob will + // NOT consume sequence number(s) and will NOT increase the count of the batch + // + // Example application: add timestamps to the transaction log for use in + // replication. + virtual Status PutLogData(const Slice& blob) = 0; + + // Clear all updates buffered in this batch. + virtual void Clear() = 0; + + // Covert this batch into a WriteBatch. This is an abstracted way of + // converting any WriteBatchBase(eg WriteBatchWithIndex) into a basic + // WriteBatch. + virtual WriteBatch* GetWriteBatch() = 0; + + // Records the state of the batch for future calls to RollbackToSavePoint(). + // May be called multiple times to set multiple save points. + virtual void SetSavePoint() = 0; + + // Remove all entries in this batch (Put, Merge, Delete, PutLogData) since the + // most recent call to SetSavePoint() and removes the most recent save point. + // If there is no previous call to SetSavePoint(), behaves the same as + // Clear(). + virtual Status RollbackToSavePoint() = 0; + + // Pop the most recent save point. + // If there is no previous call to SetSavePoint(), Status::NotFound() + // will be returned. + // Otherwise returns Status::OK(). + virtual Status PopSavePoint() = 0; + + // Sets the maximum size of the write batch in bytes. 0 means no limit. + virtual void SetMaxBytes(size_t max_bytes) = 0; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/include/rocksdb/write_buffer_manager.h b/librocksdb-sys/rocksdb/include/rocksdb/write_buffer_manager.h new file mode 100644 index 0000000..61e75c8 --- /dev/null +++ b/librocksdb-sys/rocksdb/include/rocksdb/write_buffer_manager.h @@ -0,0 +1,183 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// WriteBufferManager is for managing memory allocation for one or more +// MemTables. + +#pragma once + +#include +#include +#include +#include +#include + +#include "rocksdb/cache.h" + +namespace ROCKSDB_NAMESPACE { +class CacheReservationManager; + +// Interface to block and signal DB instances, intended for RocksDB +// internal use only. Each DB instance contains ptr to StallInterface. +class StallInterface { + public: + virtual ~StallInterface() {} + + virtual void Block() = 0; + + virtual void Signal() = 0; +}; + +class WriteBufferManager final { + public: + // Parameters: + // _buffer_size: _buffer_size = 0 indicates no limit. Memory won't be capped. + // memory_usage() won't be valid and ShouldFlush() will always return true. + // + // cache_: if `cache` is provided, we'll put dummy entries in the cache and + // cost the memory allocated to the cache. It can be used even if _buffer_size + // = 0. + // + // allow_stall: if set true, it will enable stalling of writes when + // memory_usage() exceeds buffer_size. It will wait for flush to complete and + // memory usage to drop down. + explicit WriteBufferManager(size_t _buffer_size, + std::shared_ptr cache = {}, + bool allow_stall = false); + // No copying allowed + WriteBufferManager(const WriteBufferManager&) = delete; + WriteBufferManager& operator=(const WriteBufferManager&) = delete; + + ~WriteBufferManager(); + + // Returns true if buffer_limit is passed to limit the total memory usage and + // is greater than 0. + bool enabled() const { return buffer_size() > 0; } + + // Returns true if pointer to cache is passed. + bool cost_to_cache() const { return cache_res_mgr_ != nullptr; } + + // Returns the total memory used by memtables. + // Only valid if enabled() + size_t memory_usage() const { + return memory_used_.load(std::memory_order_relaxed); + } + + // Returns the total memory used by active memtables. + size_t mutable_memtable_memory_usage() const { + return memory_active_.load(std::memory_order_relaxed); + } + + size_t dummy_entries_in_cache_usage() const; + + // Returns the buffer_size. + size_t buffer_size() const { + return buffer_size_.load(std::memory_order_relaxed); + } + + // REQUIRED: `new_size` > 0 + void SetBufferSize(size_t new_size) { + assert(new_size > 0); + buffer_size_.store(new_size, std::memory_order_relaxed); + mutable_limit_.store(new_size * 7 / 8, std::memory_order_relaxed); + // Check if stall is active and can be ended. + MaybeEndWriteStall(); + } + + void SetAllowStall(bool new_allow_stall) { + allow_stall_.store(new_allow_stall, std::memory_order_relaxed); + MaybeEndWriteStall(); + } + + // Below functions should be called by RocksDB internally. + + // Should only be called from write thread + bool ShouldFlush() const { + if (enabled()) { + if (mutable_memtable_memory_usage() > + mutable_limit_.load(std::memory_order_relaxed)) { + return true; + } + size_t local_size = buffer_size(); + if (memory_usage() >= local_size && + mutable_memtable_memory_usage() >= local_size / 2) { + // If the memory exceeds the buffer size, we trigger more aggressive + // flush. But if already more than half memory is being flushed, + // triggering more flush may not help. We will hold it instead. + return true; + } + } + return false; + } + + // Returns true if total memory usage exceeded buffer_size. + // We stall the writes untill memory_usage drops below buffer_size. When the + // function returns true, all writer threads (including one checking this + // condition) across all DBs will be stalled. Stall is allowed only if user + // pass allow_stall = true during WriteBufferManager instance creation. + // + // Should only be called by RocksDB internally . + bool ShouldStall() const { + if (!allow_stall_.load(std::memory_order_relaxed) || !enabled()) { + return false; + } + + return IsStallActive() || IsStallThresholdExceeded(); + } + + // Returns true if stall is active. + bool IsStallActive() const { + return stall_active_.load(std::memory_order_relaxed); + } + + // Returns true if stalling condition is met. + bool IsStallThresholdExceeded() const { + return memory_usage() >= buffer_size_; + } + + void ReserveMem(size_t mem); + + // We are in the process of freeing `mem` bytes, so it is not considered + // when checking the soft limit. + void ScheduleFreeMem(size_t mem); + + void FreeMem(size_t mem); + + // Add the DB instance to the queue and block the DB. + // Should only be called by RocksDB internally. + void BeginWriteStall(StallInterface* wbm_stall); + + // If stall conditions have resolved, remove DB instances from queue and + // signal them to continue. + void MaybeEndWriteStall(); + + void RemoveDBFromQueue(StallInterface* wbm_stall); + + private: + std::atomic buffer_size_; + std::atomic mutable_limit_; + std::atomic memory_used_; + // Memory that hasn't been scheduled to free. + std::atomic memory_active_; + std::shared_ptr cache_res_mgr_; + // Protects cache_res_mgr_ + std::mutex cache_res_mgr_mu_; + + std::list queue_; + // Protects the queue_ and stall_active_. + std::mutex mu_; + std::atomic allow_stall_; + // Value should only be changed by BeginWriteStall() and MaybeEndWriteStall() + // while holding mu_, but it can be read without a lock. + std::atomic stall_active_; + + void ReserveMemWithCache(size_t mem); + void FreeMemWithCache(size_t mem); +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/issue_template.md b/librocksdb-sys/rocksdb/issue_template.md new file mode 100644 index 0000000..ca52f5e --- /dev/null +++ b/librocksdb-sys/rocksdb/issue_template.md @@ -0,0 +1,7 @@ +> Note: Please use Issues only for bug reports. For questions, discussions, feature requests, etc. post to dev group: https://groups.google.com/forum/#!forum/rocksdb or https://www.facebook.com/groups/rocksdb.dev + +### Expected behavior + +### Actual behavior + +### Steps to reproduce the behavior diff --git a/librocksdb-sys/rocksdb/java/CMakeLists.txt b/librocksdb-sys/rocksdb/java/CMakeLists.txt new file mode 100644 index 0000000..ff1f05a --- /dev/null +++ b/librocksdb-sys/rocksdb/java/CMakeLists.txt @@ -0,0 +1,549 @@ +cmake_minimum_required(VERSION 3.4) + +if(${CMAKE_VERSION} VERSION_LESS "3.11.4") + message("Please consider switching to CMake 3.11.4 or newer") +endif() + +set(CMAKE_JAVA_COMPILE_FLAGS -source 8) + +set(JNI_NATIVE_SOURCES + rocksjni/backup_engine_options.cc + rocksjni/backupenginejni.cc + rocksjni/cassandra_compactionfilterjni.cc + rocksjni/cassandra_value_operator.cc + rocksjni/checkpoint.cc + rocksjni/clock_cache.cc + rocksjni/cache.cc + rocksjni/columnfamilyhandle.cc + rocksjni/compaction_filter.cc + rocksjni/compaction_filter_factory.cc + rocksjni/compaction_filter_factory_jnicallback.cc + rocksjni/compaction_job_info.cc + rocksjni/compaction_job_stats.cc + rocksjni/compaction_options.cc + rocksjni/compaction_options_fifo.cc + rocksjni/compaction_options_universal.cc + rocksjni/compact_range_options.cc + rocksjni/comparator.cc + rocksjni/comparatorjnicallback.cc + rocksjni/compression_options.cc + rocksjni/concurrent_task_limiter.cc + rocksjni/config_options.cc + rocksjni/env.cc + rocksjni/env_options.cc + rocksjni/event_listener.cc + rocksjni/event_listener_jnicallback.cc + rocksjni/filter.cc + rocksjni/ingest_external_file_options.cc + rocksjni/iterator.cc + rocksjni/jnicallback.cc + rocksjni/loggerjnicallback.cc + rocksjni/lru_cache.cc + rocksjni/memory_util.cc + rocksjni/memtablejni.cc + rocksjni/merge_operator.cc + rocksjni/native_comparator_wrapper_test.cc + rocksjni/optimistic_transaction_db.cc + rocksjni/optimistic_transaction_options.cc + rocksjni/options.cc + rocksjni/options_util.cc + rocksjni/persistent_cache.cc + rocksjni/ratelimiterjni.cc + rocksjni/remove_emptyvalue_compactionfilterjni.cc + rocksjni/restorejni.cc + rocksjni/rocks_callback_object.cc + rocksjni/rocksdb_exception_test.cc + rocksjni/rocksjni.cc + rocksjni/slice.cc + rocksjni/snapshot.cc + rocksjni/sst_file_manager.cc + rocksjni/sst_file_writerjni.cc + rocksjni/sst_file_readerjni.cc + rocksjni/sst_file_reader_iterator.cc + rocksjni/sst_partitioner.cc + rocksjni/statistics.cc + rocksjni/statisticsjni.cc + rocksjni/table.cc + rocksjni/table_filter.cc + rocksjni/table_filter_jnicallback.cc + rocksjni/testable_event_listener.cc + rocksjni/thread_status.cc + rocksjni/trace_writer.cc + rocksjni/trace_writer_jnicallback.cc + rocksjni/transaction.cc + rocksjni/transaction_db.cc + rocksjni/transaction_db_options.cc + rocksjni/transaction_log.cc + rocksjni/transaction_notifier.cc + rocksjni/transaction_notifier_jnicallback.cc + rocksjni/transaction_options.cc + rocksjni/ttl.cc + rocksjni/wal_filter.cc + rocksjni/wal_filter_jnicallback.cc + rocksjni/write_batch.cc + rocksjni/writebatchhandlerjnicallback.cc + rocksjni/write_batch_test.cc + rocksjni/write_batch_with_index.cc + rocksjni/write_buffer_manager.cc +) + +set(JAVA_MAIN_CLASSES + src/main/java/org/rocksdb/AbstractCompactionFilter.java + src/main/java/org/rocksdb/AbstractCompactionFilterFactory.java + src/main/java/org/rocksdb/AbstractComparator.java + src/main/java/org/rocksdb/AbstractEventListener.java + src/main/java/org/rocksdb/AbstractImmutableNativeReference.java + src/main/java/org/rocksdb/AbstractMutableOptions.java + src/main/java/org/rocksdb/AbstractNativeReference.java + src/main/java/org/rocksdb/AbstractRocksIterator.java + src/main/java/org/rocksdb/AbstractSlice.java + src/main/java/org/rocksdb/AbstractTableFilter.java + src/main/java/org/rocksdb/AbstractTraceWriter.java + src/main/java/org/rocksdb/AbstractTransactionNotifier.java + src/main/java/org/rocksdb/AbstractWalFilter.java + src/main/java/org/rocksdb/AbstractWriteBatch.java + src/main/java/org/rocksdb/AccessHint.java + src/main/java/org/rocksdb/AdvancedColumnFamilyOptionsInterface.java + src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java + src/main/java/org/rocksdb/BackgroundErrorReason.java + src/main/java/org/rocksdb/BackupEngineOptions.java + src/main/java/org/rocksdb/BackupEngine.java + src/main/java/org/rocksdb/BackupInfo.java + src/main/java/org/rocksdb/BlockBasedTableConfig.java + src/main/java/org/rocksdb/BloomFilter.java + src/main/java/org/rocksdb/BuiltinComparator.java + src/main/java/org/rocksdb/ByteBufferGetStatus.java + src/main/java/org/rocksdb/Cache.java + src/main/java/org/rocksdb/CassandraCompactionFilter.java + src/main/java/org/rocksdb/CassandraValueMergeOperator.java + src/main/java/org/rocksdb/Checkpoint.java + src/main/java/org/rocksdb/ChecksumType.java + src/main/java/org/rocksdb/ClockCache.java + src/main/java/org/rocksdb/ColumnFamilyDescriptor.java + src/main/java/org/rocksdb/ColumnFamilyHandle.java + src/main/java/org/rocksdb/ColumnFamilyMetaData.java + src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java + src/main/java/org/rocksdb/ColumnFamilyOptions.java + src/main/java/org/rocksdb/CompactionJobInfo.java + src/main/java/org/rocksdb/CompactionJobStats.java + src/main/java/org/rocksdb/CompactionOptions.java + src/main/java/org/rocksdb/CompactionOptionsFIFO.java + src/main/java/org/rocksdb/CompactionOptionsUniversal.java + src/main/java/org/rocksdb/CompactionPriority.java + src/main/java/org/rocksdb/CompactionReason.java + src/main/java/org/rocksdb/CompactRangeOptions.java + src/main/java/org/rocksdb/CompactionStopStyle.java + src/main/java/org/rocksdb/CompactionStyle.java + src/main/java/org/rocksdb/ComparatorOptions.java + src/main/java/org/rocksdb/ComparatorType.java + src/main/java/org/rocksdb/CompressionOptions.java + src/main/java/org/rocksdb/CompressionType.java + src/main/java/org/rocksdb/ConfigOptions.java + src/main/java/org/rocksdb/DataBlockIndexType.java + src/main/java/org/rocksdb/DBOptionsInterface.java + src/main/java/org/rocksdb/DBOptions.java + src/main/java/org/rocksdb/DbPath.java + src/main/java/org/rocksdb/DirectSlice.java + src/main/java/org/rocksdb/EncodingType.java + src/main/java/org/rocksdb/Env.java + src/main/java/org/rocksdb/EnvOptions.java + src/main/java/org/rocksdb/EventListener.java + src/main/java/org/rocksdb/Experimental.java + src/main/java/org/rocksdb/ExternalFileIngestionInfo.java + src/main/java/org/rocksdb/Filter.java + src/main/java/org/rocksdb/FileOperationInfo.java + src/main/java/org/rocksdb/FlushJobInfo.java + src/main/java/org/rocksdb/FlushReason.java + src/main/java/org/rocksdb/FlushOptions.java + src/main/java/org/rocksdb/HashLinkedListMemTableConfig.java + src/main/java/org/rocksdb/HashSkipListMemTableConfig.java + src/main/java/org/rocksdb/HistogramData.java + src/main/java/org/rocksdb/HistogramType.java + src/main/java/org/rocksdb/Holder.java + src/main/java/org/rocksdb/IndexShorteningMode.java + src/main/java/org/rocksdb/IndexType.java + src/main/java/org/rocksdb/InfoLogLevel.java + src/main/java/org/rocksdb/IngestExternalFileOptions.java + src/main/java/org/rocksdb/LevelMetaData.java + src/main/java/org/rocksdb/ConcurrentTaskLimiter.java + src/main/java/org/rocksdb/ConcurrentTaskLimiterImpl.java + src/main/java/org/rocksdb/KeyMayExist.java + src/main/java/org/rocksdb/LiveFileMetaData.java + src/main/java/org/rocksdb/LogFile.java + src/main/java/org/rocksdb/Logger.java + src/main/java/org/rocksdb/LRUCache.java + src/main/java/org/rocksdb/MemoryUsageType.java + src/main/java/org/rocksdb/MemoryUtil.java + src/main/java/org/rocksdb/MemTableConfig.java + src/main/java/org/rocksdb/MemTableInfo.java + src/main/java/org/rocksdb/MergeOperator.java + src/main/java/org/rocksdb/MutableColumnFamilyOptions.java + src/main/java/org/rocksdb/MutableColumnFamilyOptionsInterface.java + src/main/java/org/rocksdb/MutableDBOptions.java + src/main/java/org/rocksdb/MutableDBOptionsInterface.java + src/main/java/org/rocksdb/MutableOptionKey.java + src/main/java/org/rocksdb/MutableOptionValue.java + src/main/java/org/rocksdb/NativeComparatorWrapper.java + src/main/java/org/rocksdb/NativeLibraryLoader.java + src/main/java/org/rocksdb/OperationStage.java + src/main/java/org/rocksdb/OperationType.java + src/main/java/org/rocksdb/OptimisticTransactionDB.java + src/main/java/org/rocksdb/OptimisticTransactionOptions.java + src/main/java/org/rocksdb/Options.java + src/main/java/org/rocksdb/OptionString.java + src/main/java/org/rocksdb/OptionsUtil.java + src/main/java/org/rocksdb/PersistentCache.java + src/main/java/org/rocksdb/PlainTableConfig.java + src/main/java/org/rocksdb/PrepopulateBlobCache.java + src/main/java/org/rocksdb/Priority.java + src/main/java/org/rocksdb/Range.java + src/main/java/org/rocksdb/RateLimiter.java + src/main/java/org/rocksdb/RateLimiterMode.java + src/main/java/org/rocksdb/ReadOptions.java + src/main/java/org/rocksdb/ReadTier.java + src/main/java/org/rocksdb/RemoveEmptyValueCompactionFilter.java + src/main/java/org/rocksdb/RestoreOptions.java + src/main/java/org/rocksdb/ReusedSynchronisationType.java + src/main/java/org/rocksdb/RocksCallbackObject.java + src/main/java/org/rocksdb/RocksDBException.java + src/main/java/org/rocksdb/RocksDB.java + src/main/java/org/rocksdb/RocksEnv.java + src/main/java/org/rocksdb/RocksIteratorInterface.java + src/main/java/org/rocksdb/RocksIterator.java + src/main/java/org/rocksdb/RocksMemEnv.java + src/main/java/org/rocksdb/RocksMutableObject.java + src/main/java/org/rocksdb/RocksObject.java + src/main/java/org/rocksdb/SanityLevel.java + src/main/java/org/rocksdb/SizeApproximationFlag.java + src/main/java/org/rocksdb/SkipListMemTableConfig.java + src/main/java/org/rocksdb/Slice.java + src/main/java/org/rocksdb/Snapshot.java + src/main/java/org/rocksdb/SstFileManager.java + src/main/java/org/rocksdb/SstFileMetaData.java + src/main/java/org/rocksdb/SstFileReader.java + src/main/java/org/rocksdb/SstFileReaderIterator.java + src/main/java/org/rocksdb/SstFileWriter.java + src/main/java/org/rocksdb/SstPartitionerFactory.java + src/main/java/org/rocksdb/SstPartitionerFixedPrefixFactory.java + src/main/java/org/rocksdb/StateType.java + src/main/java/org/rocksdb/StatisticsCollectorCallback.java + src/main/java/org/rocksdb/StatisticsCollector.java + src/main/java/org/rocksdb/Statistics.java + src/main/java/org/rocksdb/StatsCollectorInput.java + src/main/java/org/rocksdb/StatsLevel.java + src/main/java/org/rocksdb/Status.java + src/main/java/org/rocksdb/StringAppendOperator.java + src/main/java/org/rocksdb/TableFileCreationBriefInfo.java + src/main/java/org/rocksdb/TableFileCreationInfo.java + src/main/java/org/rocksdb/TableFileCreationReason.java + src/main/java/org/rocksdb/TableFileDeletionInfo.java + src/main/java/org/rocksdb/TableFilter.java + src/main/java/org/rocksdb/TableProperties.java + src/main/java/org/rocksdb/TableFormatConfig.java + src/main/java/org/rocksdb/ThreadType.java + src/main/java/org/rocksdb/ThreadStatus.java + src/main/java/org/rocksdb/TickerType.java + src/main/java/org/rocksdb/TimedEnv.java + src/main/java/org/rocksdb/TraceOptions.java + src/main/java/org/rocksdb/TraceWriter.java + src/main/java/org/rocksdb/TransactionalDB.java + src/main/java/org/rocksdb/TransactionalOptions.java + src/main/java/org/rocksdb/TransactionDB.java + src/main/java/org/rocksdb/TransactionDBOptions.java + src/main/java/org/rocksdb/Transaction.java + src/main/java/org/rocksdb/TransactionLogIterator.java + src/main/java/org/rocksdb/TransactionOptions.java + src/main/java/org/rocksdb/TtlDB.java + src/main/java/org/rocksdb/TxnDBWritePolicy.java + src/main/java/org/rocksdb/VectorMemTableConfig.java + src/main/java/org/rocksdb/WalFileType.java + src/main/java/org/rocksdb/WalFilter.java + src/main/java/org/rocksdb/WalProcessingOption.java + src/main/java/org/rocksdb/WALRecoveryMode.java + src/main/java/org/rocksdb/WBWIRocksIterator.java + src/main/java/org/rocksdb/WriteBatch.java + src/main/java/org/rocksdb/WriteBatchInterface.java + src/main/java/org/rocksdb/WriteBatchWithIndex.java + src/main/java/org/rocksdb/WriteOptions.java + src/main/java/org/rocksdb/WriteBufferManager.java + src/main/java/org/rocksdb/WriteStallCondition.java + src/main/java/org/rocksdb/WriteStallInfo.java + src/main/java/org/rocksdb/util/ByteUtil.java + src/main/java/org/rocksdb/util/BytewiseComparator.java + src/main/java/org/rocksdb/util/Environment.java + src/main/java/org/rocksdb/util/IntComparator.java + src/main/java/org/rocksdb/util/ReverseBytewiseComparator.java + src/main/java/org/rocksdb/util/SizeUnit.java + src/main/java/org/rocksdb/UInt64AddOperator.java +) + +set(JAVA_TEST_CLASSES + src/test/java/org/rocksdb/BackupEngineTest.java + src/test/java/org/rocksdb/IngestExternalFileOptionsTest.java + src/test/java/org/rocksdb/NativeComparatorWrapperTest.java + src/test/java/org/rocksdb/PlatformRandomHelper.java + src/test/java/org/rocksdb/RocksDBExceptionTest.java + src/test/java/org/rocksdb/RocksNativeLibraryResource.java + src/test/java/org/rocksdb/SnapshotTest.java + src/test/java/org/rocksdb/WriteBatchTest.java + src/test/java/org/rocksdb/util/CapturingWriteBatchHandler.java + src/test/java/org/rocksdb/util/WriteBatchGetter.java + src/test/java/org/rocksdb/test/TestableEventListener.java +) + +include(FindJava) +include(UseJava) +find_package(JNI) + +include_directories(${JNI_INCLUDE_DIRS}) +include_directories(${PROJECT_SOURCE_DIR}/java) + +set(JAVA_TEST_LIBDIR ${PROJECT_SOURCE_DIR}/java/test-libs) +set(JAVA_TMP_JAR ${JAVA_TEST_LIBDIR}/tmp.jar) +set(JAVA_JUNIT_JAR ${JAVA_TEST_LIBDIR}/junit-4.12.jar) +set(JAVA_HAMCR_JAR ${JAVA_TEST_LIBDIR}/hamcrest-core-1.3.jar) +set(JAVA_MOCKITO_JAR ${JAVA_TEST_LIBDIR}/mockito-all-1.10.19.jar) +set(JAVA_CGLIB_JAR ${JAVA_TEST_LIBDIR}/cglib-2.2.2.jar) +set(JAVA_ASSERTJ_JAR ${JAVA_TEST_LIBDIR}/assertj-core-1.7.1.jar) +set(JAVA_TESTCLASSPATH ${JAVA_JUNIT_JAR} ${JAVA_HAMCR_JAR} ${JAVA_MOCKITO_JAR} ${JAVA_CGLIB_JAR} ${JAVA_ASSERTJ_JAR}) + +set(JNI_OUTPUT_DIR ${PROJECT_SOURCE_DIR}/java/include) +file(MAKE_DIRECTORY ${JNI_OUTPUT_DIR}) + +if(${Java_VERSION_MINOR} VERSION_LESS_EQUAL "7" AND ${Java_VERSION_MAJOR} STREQUAL "1") + message(FATAL_ERROR "Detected Java 7 or older (${Java_VERSION_STRING}), minimum required version in now Java 8") +endif() + +if(${Java_VERSION_MAJOR} VERSION_GREATER_EQUAL "10" AND ${CMAKE_VERSION} VERSION_LESS "3.11.4") + # Java 10 and newer don't have javah, but the alternative GENERATE_NATIVE_HEADERS requires CMake 3.11.4 or newer + message(FATAL_ERROR "Detected Java 10 or newer (${Java_VERSION_STRING}), to build with CMake please upgrade CMake to 3.11.4 or newer") + +elseif(${CMAKE_VERSION} VERSION_LESS "3.11.4") + # Old CMake + message("Using an old CMAKE (${CMAKE_VERSION}) - JNI headers generated in separate step") + add_jar( + rocksdbjni_classes + SOURCES + ${JAVA_MAIN_CLASSES} + ${JAVA_TEST_CLASSES} + INCLUDE_JARS ${JAVA_TESTCLASSPATH} + ) + +else () + # Java 1.8 or newer prepare the JAR... + message("Preparing Jar for JDK ${Java_VERSION_STRING}") + add_jar( + rocksdbjni_classes + SOURCES + ${JAVA_MAIN_CLASSES} + ${JAVA_TEST_CLASSES} + INCLUDE_JARS ${JAVA_TESTCLASSPATH} + GENERATE_NATIVE_HEADERS rocksdbjni_headers DESTINATION ${JNI_OUTPUT_DIR} + ) + +endif() + +if(NOT EXISTS ${PROJECT_SOURCE_DIR}/java/classes) + file(MAKE_DIRECTORY ${PROJECT_SOURCE_DIR}/java/classes) +endif() + +if(NOT EXISTS ${JAVA_TEST_LIBDIR}) + file(MAKE_DIRECTORY mkdir ${JAVA_TEST_LIBDIR}) +endif() + +if (DEFINED CUSTOM_DEPS_URL) + set(DEPS_URL ${CUSTOM_DEPS_URL}/) +else () + # Using a Facebook AWS account for S3 storage. (maven.org has a history + # of failing in Travis builds.) + set(DEPS_URL "https://rocksdb-deps.s3-us-west-2.amazonaws.com/jars") +endif() + +if(NOT EXISTS ${JAVA_JUNIT_JAR}) + message("Downloading ${JAVA_JUNIT_JAR}") + file(DOWNLOAD ${DEPS_URL}/junit-4.12.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + list(GET downloadStatus 0 error_code) + list(GET downloadStatus 1 error_message) + if(NOT error_code EQUAL 0) + message(FATAL_ERROR "Failed downloading ${JAVA_JUNIT_JAR}: ${error_message}") + endif() + file(RENAME ${JAVA_TMP_JAR} ${JAVA_JUNIT_JAR}) +endif() +if(NOT EXISTS ${JAVA_HAMCR_JAR}) + message("Downloading ${JAVA_HAMCR_JAR}") + file(DOWNLOAD ${DEPS_URL}/hamcrest-core-1.3.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + list(GET downloadStatus 0 error_code) + list(GET downloadStatus 1 error_message) + if(NOT error_code EQUAL 0) + message(FATAL_ERROR "Failed downloading ${JAVA_HAMCR_JAR}: ${error_message}") + endif() + file(RENAME ${JAVA_TMP_JAR} ${JAVA_HAMCR_JAR}) +endif() +if(NOT EXISTS ${JAVA_MOCKITO_JAR}) + message("Downloading ${JAVA_MOCKITO_JAR}") + file(DOWNLOAD ${DEPS_URL}/mockito-all-1.10.19.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + list(GET downloadStatus 0 error_code) + list(GET downloadStatus 1 error_message) + if(NOT error_code EQUAL 0) + message(FATAL_ERROR "Failed downloading ${JAVA_MOCKITO_JAR}: ${error_message}") + endif() + file(RENAME ${JAVA_TMP_JAR} ${JAVA_MOCKITO_JAR}) +endif() +if(NOT EXISTS ${JAVA_CGLIB_JAR}) + message("Downloading ${JAVA_CGLIB_JAR}") + file(DOWNLOAD ${DEPS_URL}/cglib-2.2.2.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + list(GET downloadStatus 0 error_code) + list(GET downloadStatus 1 error_message) + if(NOT error_code EQUAL 0) + message(FATAL_ERROR "Failed downloading ${JAVA_CGLIB_JAR}: ${error_message}") + endif() + file(RENAME ${JAVA_TMP_JAR} ${JAVA_CGLIB_JAR}) +endif() +if(NOT EXISTS ${JAVA_ASSERTJ_JAR}) + message("Downloading ${JAVA_ASSERTJ_JAR}") + file(DOWNLOAD ${DEPS_URL}/assertj-core-1.7.1.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + list(GET downloadStatus 0 error_code) + list(GET downloadStatus 1 error_message) + if(NOT error_code EQUAL 0) + message(FATAL_ERROR "Failed downloading ${JAVA_ASSERTJ_JAR}: ${error_message}") + endif() + file(RENAME ${JAVA_TMP_JAR} ${JAVA_ASSERTJ_JAR}) +endif() + +if(${CMAKE_VERSION} VERSION_LESS "3.11.4") + # Old CMake ONLY generate JNI headers, otherwise JNI is handled in add_jar step above + message("Preparing JNI headers for old CMake (${CMAKE_VERSION})") + set(NATIVE_JAVA_CLASSES + org.rocksdb.AbstractCompactionFilter + org.rocksdb.AbstractCompactionFilterFactory + org.rocksdb.AbstractComparator + org.rocksdb.AbstractEventListener + org.rocksdb.AbstractImmutableNativeReference + org.rocksdb.AbstractNativeReference + org.rocksdb.AbstractRocksIterator + org.rocksdb.AbstractSlice + org.rocksdb.AbstractTableFilter + org.rocksdb.AbstractTraceWriter + org.rocksdb.AbstractTransactionNotifier + org.rocksdb.AbstractWalFilter + org.rocksdb.BackupEngineOptions + org.rocksdb.BackupEngine + org.rocksdb.BlockBasedTableConfig + org.rocksdb.BloomFilter + org.rocksdb.CassandraCompactionFilter + org.rocksdb.CassandraValueMergeOperator + org.rocksdb.Checkpoint + org.rocksdb.ClockCache + org.rocksdb.Cache + org.rocksdb.ColumnFamilyHandle + org.rocksdb.ColumnFamilyOptions + org.rocksdb.CompactionJobInfo + org.rocksdb.CompactionJobStats + org.rocksdb.CompactionOptions + org.rocksdb.CompactionOptionsFIFO + org.rocksdb.CompactionOptionsUniversal + org.rocksdb.CompactRangeOptions + org.rocksdb.ComparatorOptions + org.rocksdb.CompressionOptions + org.rocksdb.ConcurrentTaskLimiterImpl + org.rocksdb.ConfigOptions + org.rocksdb.DBOptions + org.rocksdb.DirectSlice + org.rocksdb.Env + org.rocksdb.EnvOptions + org.rocksdb.Filter + org.rocksdb.FlushOptions + org.rocksdb.HashLinkedListMemTableConfig + org.rocksdb.HashSkipListMemTableConfig + org.rocksdb.IngestExternalFileOptions + org.rocksdb.Logger + org.rocksdb.LRUCache + org.rocksdb.MemoryUtil + org.rocksdb.MemTableConfig + org.rocksdb.NativeComparatorWrapper + org.rocksdb.NativeLibraryLoader + org.rocksdb.OptimisticTransactionDB + org.rocksdb.OptimisticTransactionOptions + org.rocksdb.Options + org.rocksdb.OptionsUtil + org.rocksdb.PersistentCache + org.rocksdb.PlainTableConfig + org.rocksdb.RateLimiter + org.rocksdb.ReadOptions + org.rocksdb.RemoveEmptyValueCompactionFilter + org.rocksdb.RestoreOptions + org.rocksdb.RocksCallbackObject + org.rocksdb.RocksDB + org.rocksdb.RocksEnv + org.rocksdb.RocksIterator + org.rocksdb.RocksIteratorInterface + org.rocksdb.RocksMemEnv + org.rocksdb.RocksMutableObject + org.rocksdb.RocksObject + org.rocksdb.SkipListMemTableConfig + org.rocksdb.Slice + org.rocksdb.Snapshot + org.rocksdb.SstFileManager + org.rocksdb.SstFileWriter + org.rocksdb.SstFileReader + org.rocksdb.SstFileReaderIterator + org.rocksdb.SstPartitionerFactory + org.rocksdb.SstPartitionerFixedPrefixFactory + org.rocksdb.Statistics + org.rocksdb.StringAppendOperator + org.rocksdb.TableFormatConfig + org.rocksdb.ThreadStatus + org.rocksdb.TimedEnv + org.rocksdb.Transaction + org.rocksdb.TransactionDB + org.rocksdb.TransactionDBOptions + org.rocksdb.TransactionLogIterator + org.rocksdb.TransactionOptions + org.rocksdb.TtlDB + org.rocksdb.UInt64AddOperator + org.rocksdb.VectorMemTableConfig + org.rocksdb.WBWIRocksIterator + org.rocksdb.WriteBatch + org.rocksdb.WriteBatch.Handler + org.rocksdb.WriteBatchInterface + org.rocksdb.WriteBatchWithIndex + org.rocksdb.WriteOptions + org.rocksdb.NativeComparatorWrapperTest + org.rocksdb.RocksDBExceptionTest + org.rocksdb.SnapshotTest + org.rocksdb.WriteBatchTest + org.rocksdb.WriteBatchTestInternalHelper + org.rocksdb.WriteBufferManager + org.rocksdb.test.TestableEventListener + ) + + create_javah( + TARGET rocksdbjni_headers + CLASSES ${NATIVE_JAVA_CLASSES} + CLASSPATH rocksdbjni_classes ${JAVA_TESTCLASSPATH} + OUTPUT_DIR ${JNI_OUTPUT_DIR} + ) +endif() + +if(NOT MSVC) + set_property(TARGET ${ROCKSDB_STATIC_LIB} PROPERTY POSITION_INDEPENDENT_CODE ON) +endif() + +set(ROCKSDBJNI_STATIC_LIB rocksdbjni${ARTIFACT_SUFFIX}) +add_library(${ROCKSDBJNI_STATIC_LIB} ${JNI_NATIVE_SOURCES}) +add_dependencies(${ROCKSDBJNI_STATIC_LIB} rocksdbjni_headers) +target_link_libraries(${ROCKSDBJNI_STATIC_LIB} ${ROCKSDB_STATIC_LIB} ${ROCKSDB_LIB}) + +if(NOT MINGW) + set(ROCKSDBJNI_SHARED_LIB rocksdbjni-shared${ARTIFACT_SUFFIX}) + add_library(${ROCKSDBJNI_SHARED_LIB} SHARED ${JNI_NATIVE_SOURCES}) + add_dependencies(${ROCKSDBJNI_SHARED_LIB} rocksdbjni_headers) + target_link_libraries(${ROCKSDBJNI_SHARED_LIB} ${ROCKSDB_STATIC_LIB} ${ROCKSDB_LIB}) + + set_target_properties( + ${ROCKSDBJNI_SHARED_LIB} + PROPERTIES + COMPILE_PDB_OUTPUT_DIRECTORY ${CMAKE_CFG_INTDIR} + COMPILE_PDB_NAME ${ROCKSDBJNI_STATIC_LIB}.pdb + ) +endif() diff --git a/librocksdb-sys/rocksdb/java/GetBenchmarks.md b/librocksdb-sys/rocksdb/java/GetBenchmarks.md new file mode 100644 index 0000000..b66a897 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/GetBenchmarks.md @@ -0,0 +1,161 @@ +# RocksDB Get Performance Benchmarks + +Results associated with [Improve Java API `get()` performance by reducing copies](https://github.com/facebook/rocksdb/pull/10970) + +## Build/Run + +Mac +``` +make clean jclean +DEBUG_LEVEL=0 make -j12 rocksdbjava +(cd java/target; cp rocksdbjni-7.9.0-osx.jar rocksdbjni-7.9.0-SNAPSHOT-osx.jar) +mvn install:install-file -Dfile=./java/target/rocksdbjni-7.9.0-SNAPSHOT-osx.jar -DgroupId=org.rocksdb -DartifactId=rocksdbjni -Dversion=7.9.0-SNAPSHOT -Dpackaging=jar +``` + +Linux +``` +make clean jclean +DEBUG_LEVEL=0 make -j12 rocksdbjava +(cd java/target; cp rocksdbjni-7.9.0-linux64.jar rocksdbjni-7.9.0-SNAPSHOT-linux64.jar) +mvn install:install-file -Dfile=./java/target/rocksdbjni-7.9.0-SNAPSHOT-linux64.jar -DgroupId=org.rocksdb -DartifactId=rocksdbjni -Dversion=7.9.0-SNAPSHOT -Dpackaging=jar +``` + +Build jmh test package, on either platform +``` +pushd java/jmh +mvn clean package +``` + +A quick test run, just as a sanity check, using a small number of keys, would be +``` +java -jar target/rocksdbjni-jmh-1.0-SNAPSHOT-benchmarks.jar -p keyCount=1000 -p keySize=128 -p valueSize=32768 -p columnFamilyTestType="no_column_family" GetBenchmarks +``` +The long performance run (as big as we can make it on our Ubuntu box without filling the disk) +``` +java -jar target/rocksdbjni-jmh-1.0-SNAPSHOT-benchmarks.jar -p keyCount=1000,50000 -p keySize=128 -p valueSize=1024,16384 -p columnFamilyTestType="1_column_family","20_column_families" GetBenchmarks.get GetBenchmarks.preallocatedByteBufferGet GetBenchmarks.preallocatedGet +``` + +## Results (small runs, Mac) + +These are run on a 10-core M1 with 64GB of memory and 2TB of SSD. +They probably reflect the absolute best case for this optimization, hitting in-memory buffers and completely eliminating a buffer copy. + +### Before +Benchmark (columnFamilyTestType) (keyCount) (keySize) (multiGetSize) (valueSize) Mode Cnt Score Error Units +GetBenchmarks.get no_column_family 1000 128 N/A 32768 thrpt 25 43496.578 ± 5743.090 ops/s +GetBenchmarks.preallocatedByteBufferGet no_column_family 1000 128 N/A 32768 thrpt 25 70765.578 ± 697.548 ops/s +GetBenchmarks.preallocatedGet no_column_family 1000 128 N/A 32768 thrpt 25 69883.554 ± 944.184 ops/s + +### After fixing byte[] (.get and .preallocatedGet) + +Benchmark (columnFamilyTestType) (keyCount) (keySize) (multiGetSize) (valueSize) Mode Cnt Score Error Units +GetBenchmarks.get no_column_family 1000 128 N/A 32768 thrpt 25 149207.681 ± 2261.671 ops/s +GetBenchmarks.preallocatedByteBufferGet no_column_family 1000 128 N/A 32768 thrpt 25 68920.489 ± 1574.664 ops/s +GetBenchmarks.preallocatedGet no_column_family 1000 128 N/A 32768 thrpt 25 177399.022 ± 2107.375 ops/s + +### After fixing ByteBuffer (.preallocatedByteBufferGet) + +Benchmark (columnFamilyTestType) (keyCount) (keySize) (multiGetSize) (valueSize) Mode Cnt Score Error Units +GetBenchmarks.get no_column_family 1000 128 N/A 32768 thrpt 25 150389.259 ± 1371.473 ops/s +GetBenchmarks.preallocatedByteBufferGet no_column_family 1000 128 N/A 32768 thrpt 25 179919.468 ± 1670.714 ops/s +GetBenchmarks.preallocatedGet no_column_family 1000 128 N/A 32768 thrpt 25 178261.938 ± 2630.571 ops/s +## Results (Ubuntu, big runs) +These take 3-4 hours +``` +java -jar target/rocksdbjni-jmh-1.0-SNAPSHOT-benchmarks.jar -p keyCount=1000,50000 -p keySize=128 -p valueSize=1024,16384 -p columnFamilyTestType="1_column_family","20_column_families" GetBenchmarks.get GetBenchmarks.preallocatedByteBufferGet GetBenchmarks.preallocatedGet +``` +It's clear that all `get()` variants have noticeably improved performance, though not the spectacular gains of the M1. +### With fixes for all of the `get()` instances + +Benchmark (columnFamilyTestType) (keyCount) (keySize) (valueSize) Mode Cnt Score Error Units +GetBenchmarks.get 1_column_family 1000 128 1024 thrpt 25 935648.793 ± 22879.910 ops/s +GetBenchmarks.get 1_column_family 1000 128 16384 thrpt 25 204366.301 ± 1326.570 ops/s +GetBenchmarks.get 1_column_family 50000 128 1024 thrpt 25 693451.990 ± 19822.720 ops/s +GetBenchmarks.get 1_column_family 50000 128 16384 thrpt 25 50473.768 ± 497.335 ops/s +GetBenchmarks.get 20_column_families 1000 128 1024 thrpt 25 550118.874 ± 14289.009 ops/s +GetBenchmarks.get 20_column_families 1000 128 16384 thrpt 25 120545.549 ± 648.280 ops/s +GetBenchmarks.get 20_column_families 50000 128 1024 thrpt 25 235671.353 ± 2231.195 ops/s +GetBenchmarks.get 20_column_families 50000 128 16384 thrpt 25 12463.887 ± 1950.746 ops/s +GetBenchmarks.preallocatedByteBufferGet 1_column_family 1000 128 1024 thrpt 25 1196026.040 ± 35435.729 ops/s +GetBenchmarks.preallocatedByteBufferGet 1_column_family 1000 128 16384 thrpt 25 403252.655 ± 3287.054 ops/s +GetBenchmarks.preallocatedByteBufferGet 1_column_family 50000 128 1024 thrpt 25 829965.448 ± 16945.452 ops/s +GetBenchmarks.preallocatedByteBufferGet 1_column_family 50000 128 16384 thrpt 25 63798.042 ± 1292.858 ops/s +GetBenchmarks.preallocatedByteBufferGet 20_column_families 1000 128 1024 thrpt 25 724557.253 ± 12710.828 ops/s +GetBenchmarks.preallocatedByteBufferGet 20_column_families 1000 128 16384 thrpt 25 176846.615 ± 1121.644 ops/s +GetBenchmarks.preallocatedByteBufferGet 20_column_families 50000 128 1024 thrpt 25 263553.764 ± 1304.243 ops/s +GetBenchmarks.preallocatedByteBufferGet 20_column_families 50000 128 16384 thrpt 25 14721.693 ± 2574.240 ops/s +GetBenchmarks.preallocatedGet 1_column_family 1000 128 1024 thrpt 25 1093947.765 ± 42846.276 ops/s +GetBenchmarks.preallocatedGet 1_column_family 1000 128 16384 thrpt 25 391629.913 ± 4039.965 ops/s +GetBenchmarks.preallocatedGet 1_column_family 50000 128 1024 thrpt 25 769332.958 ± 24180.749 ops/s +GetBenchmarks.preallocatedGet 1_column_family 50000 128 16384 thrpt 25 61712.038 ± 423.494 ops/s +GetBenchmarks.preallocatedGet 20_column_families 1000 128 1024 thrpt 25 694684.465 ± 5484.205 ops/s +GetBenchmarks.preallocatedGet 20_column_families 1000 128 16384 thrpt 25 172383.593 ± 841.679 ops/s +GetBenchmarks.preallocatedGet 20_column_families 50000 128 1024 thrpt 25 257447.351 ± 1388.667 ops/s +GetBenchmarks.preallocatedGet 20_column_families 50000 128 16384 thrpt 25 13418.522 ± 2418.619 ops/s + +### Baseline (no fixes) + +Benchmark (columnFamilyTestType) (keyCount) (keySize) (valueSize) Mode Cnt Score Error Units +GetBenchmarks.get 1_column_family 1000 128 1024 thrpt 25 866745.224 ± 8834.629 ops/s +GetBenchmarks.get 1_column_family 1000 128 16384 thrpt 25 184332.195 ± 2304.217 ops/s +GetBenchmarks.get 1_column_family 50000 128 1024 thrpt 25 666794.288 ± 16150.684 ops/s +GetBenchmarks.get 1_column_family 50000 128 16384 thrpt 25 47221.788 ± 433.165 ops/s +GetBenchmarks.get 20_column_families 1000 128 1024 thrpt 25 551513.636 ± 7763.681 ops/s +GetBenchmarks.get 20_column_families 1000 128 16384 thrpt 25 113117.720 ± 580.738 ops/s +GetBenchmarks.get 20_column_families 50000 128 1024 thrpt 25 238675.555 ± 1758.978 ops/s +GetBenchmarks.get 20_column_families 50000 128 16384 thrpt 25 11639.390 ± 1459.765 ops/s +GetBenchmarks.preallocatedByteBufferGet 1_column_family 1000 128 1024 thrpt 25 1153617.917 ± 26350.028 ops/s +GetBenchmarks.preallocatedByteBufferGet 1_column_family 1000 128 16384 thrpt 25 401710.334 ± 4324.539 ops/s +GetBenchmarks.preallocatedByteBufferGet 1_column_family 50000 128 1024 thrpt 25 809384.073 ± 13833.871 ops/s +GetBenchmarks.preallocatedByteBufferGet 1_column_family 50000 128 16384 thrpt 25 59279.005 ± 443.207 ops/s +GetBenchmarks.preallocatedByteBufferGet 20_column_families 1000 128 1024 thrpt 25 715466.403 ± 6591.375 ops/s +GetBenchmarks.preallocatedByteBufferGet 20_column_families 1000 128 16384 thrpt 25 175279.163 ± 910.923 ops/s +GetBenchmarks.preallocatedByteBufferGet 20_column_families 50000 128 1024 thrpt 25 263295.180 ± 856.456 ops/s +GetBenchmarks.preallocatedByteBufferGet 20_column_families 50000 128 16384 thrpt 25 14001.928 ± 2462.067 ops/s +GetBenchmarks.preallocatedGet 1_column_family 1000 128 1024 thrpt 25 1072866.854 ± 27030.592 ops/s +GetBenchmarks.preallocatedGet 1_column_family 1000 128 16384 thrpt 25 383950.853 ± 4510.654 ops/s +GetBenchmarks.preallocatedGet 1_column_family 50000 128 1024 thrpt 25 764395.469 ± 10097.417 ops/s +GetBenchmarks.preallocatedGet 1_column_family 50000 128 16384 thrpt 25 56851.330 ± 388.029 ops/s +GetBenchmarks.preallocatedGet 20_column_families 1000 128 1024 thrpt 25 668518.593 ± 9764.117 ops/s +GetBenchmarks.preallocatedGet 20_column_families 1000 128 16384 thrpt 25 171309.695 ± 875.895 ops/s +GetBenchmarks.preallocatedGet 20_column_families 50000 128 1024 thrpt 25 256057.801 ± 954.621 ops/s +GetBenchmarks.preallocatedGet 20_column_families 50000 128 16384 thrpt 25 13319.380 ± 2126.654 ops/s + +### Comparison + +It does at least look best when the data is cached. That is to say, smallest number of column families, and least keys. + +GetBenchmarks.get 1_column_family 1000 128 16384 thrpt 25 204366.301 ± 1326.570 ops/s +GetBenchmarks.get 1_column_family 1000 128 16384 thrpt 25 184332.195 ± 2304.217 ops/s + +GetBenchmarks.get 1_column_family 50000 128 16384 thrpt 25 50473.768 ± 497.335 ops/s +GetBenchmarks.get 1_column_family 50000 128 16384 thrpt 25 47221.788 ± 433.165 ops/s + +GetBenchmarks.get 20_column_families 1000 128 16384 thrpt 25 120545.549 ± 648.280 ops/s +GetBenchmarks.get 20_column_families 1000 128 16384 thrpt 25 113117.720 ± 580.738 ops/s + +GetBenchmarks.get 20_column_families 50000 128 16384 thrpt 25 12463.887 ± 1950.746 ops/s +GetBenchmarks.get 20_column_families 50000 128 16384 thrpt 25 11639.390 ± 1459.765 ops/s + +### Baseline +25 minute run, small number of keys +``` +java -jar target/rocksdbjni-jmh-1.0-SNAPSHOT-benchmarks.jar -p keyCount=1000 -p keySize=128 -p valueSize=32768 -p columnFamilyTestType="no_column_families" GetBenchmarks.get GetBenchmarks.preallocatedByteBufferGet GetBenchmarks.preallocatedGet +``` + +Benchmark (columnFamilyTestType) (keyCount) (keySize) (valueSize) Mode Cnt Score Error Units +GetBenchmarks.get no_column_families 1000 128 32768 thrpt 25 32344.908 ± 296.651 ops/s +GetBenchmarks.preallocatedByteBufferGet no_column_families 1000 128 32768 thrpt 25 45266.968 ± 424.514 ops/s +GetBenchmarks.preallocatedGet no_column_families 1000 128 32768 thrpt 25 43531.088 ± 291.785 ops/s + +### Optimized + +Benchmark (columnFamilyTestType) (keyCount) (keySize) (valueSize) Mode Cnt Score Error Units +GetBenchmarks.get no_column_families 1000 128 32768 thrpt 25 37463.716 ± 235.744 ops/s +GetBenchmarks.preallocatedByteBufferGet no_column_families 1000 128 32768 thrpt 25 48946.105 ± 466.463 ops/s +GetBenchmarks.preallocatedGet no_column_families 1000 128 32768 thrpt 25 47143.624 ± 576.763 ops/s + +## Conclusion + +The performance improvement is real. + diff --git a/librocksdb-sys/rocksdb/java/HISTORY-JAVA.md b/librocksdb-sys/rocksdb/java/HISTORY-JAVA.md new file mode 100644 index 0000000..731886a --- /dev/null +++ b/librocksdb-sys/rocksdb/java/HISTORY-JAVA.md @@ -0,0 +1,86 @@ +# RocksJava Change Log + +## 3.13 (8/4/2015) +### New Features +* Exposed BackupEngine API. +* Added CappedPrefixExtractor support. To use such extractor, simply call useCappedPrefixExtractor in either Options or ColumnFamilyOptions. +* Added RemoveEmptyValueCompactionFilter. + +## 3.10.0 (3/24/2015) +### New Features +* Added compression per level API. +* MemEnv is now available in RocksJava via RocksMemEnv class. +* lz4 compression is now included in rocksjava static library when running `make rocksdbjavastatic`. + +### Public API Changes +* Overflowing a size_t when setting rocksdb options now throws an IllegalArgumentException, which removes the necessity for a developer to catch these Exceptions explicitly. +* The set and get functions for tableCacheRemoveScanCountLimit are deprecated. + + +## By 01/31/2015 +### New Features +* WriteBatchWithIndex support. +* Iterator support for WriteBatch and WriteBatchWithIndex +* GetUpdatesSince support. +* Snapshots carry now information about the related sequence number. +* TTL DB support. + +## By 11/14/2014 +### New Features +* Full support for Column Family. +* Slice and Comparator support. +* Default merge operator support. +* RateLimiter support. + +## By 06/15/2014 +### New Features +* Added basic Java binding for rocksdb::Env such that multiple RocksDB can share the same thread pool and environment. +* Added RestoreBackupableDB + +## By 05/30/2014 +### Internal Framework Improvement +* Added disOwnNativeHandle to RocksObject, which allows a RocksObject to give-up the ownership of its native handle. This method is useful when sharing and transferring the ownership of RocksDB C++ resources. + +## By 05/15/2014 +### New Features +* Added RocksObject --- the base class of all RocksDB classes which holds some RocksDB resources in the C++ side. +* Use environmental variable JAVA_HOME in Makefile for RocksJava +### Public API changes +* Renamed org.rocksdb.Iterator to org.rocksdb.RocksIterator to avoid potential confliction with Java built-in Iterator. + +## By 04/30/2014 +### New Features +* Added Java binding for MultiGet. +* Added static method RocksDB.loadLibrary(), which loads necessary library files. +* Added Java bindings for 60+ rocksdb::Options. +* Added Java binding for BloomFilter. +* Added Java binding for ReadOptions. +* Added Java binding for memtables. +* Added Java binding for sst formats. +* Added Java binding for RocksDB Iterator which enables sequential scan operation. +* Added Java binding for Statistics +* Added Java binding for BackupableDB. + +### DB Benchmark +* Added filluniquerandom, readseq benchmark. +* 70+ command-line options. +* Enabled BloomFilter configuration. + +## By 04/15/2014 +### New Features +* Added Java binding for WriteOptions. +* Added Java binding for WriteBatch, which enables batch-write. +* Added Java binding for rocksdb::Options. +* Added Java binding for block cache. +* Added Java version DB Benchmark. + +### DB Benchmark +* Added readwhilewriting benchmark. + +### Internal Framework Improvement +* Avoid a potential byte-array-copy between c++ and Java in RocksDB.get. +* Added SizeUnit in org.rocksdb.util to store consts like KB and GB. + +### 03/28/2014 +* RocksJava project started. +* Added Java binding for RocksDB, which supports Open, Close, Get and Put. diff --git a/librocksdb-sys/rocksdb/java/Makefile b/librocksdb-sys/rocksdb/java/Makefile new file mode 100644 index 0000000..7d2695a --- /dev/null +++ b/librocksdb-sys/rocksdb/java/Makefile @@ -0,0 +1,453 @@ +NATIVE_JAVA_CLASSES = \ + org.rocksdb.AbstractCompactionFilter\ + org.rocksdb.AbstractCompactionFilterFactory\ + org.rocksdb.AbstractComparator\ + org.rocksdb.AbstractEventListener\ + org.rocksdb.AbstractSlice\ + org.rocksdb.AbstractTableFilter\ + org.rocksdb.AbstractTraceWriter\ + org.rocksdb.AbstractTransactionNotifier\ + org.rocksdb.AbstractWalFilter\ + org.rocksdb.BackupEngine\ + org.rocksdb.BackupEngineOptions\ + org.rocksdb.BlockBasedTableConfig\ + org.rocksdb.BloomFilter\ + org.rocksdb.Checkpoint\ + org.rocksdb.ClockCache\ + org.rocksdb.Cache\ + org.rocksdb.CassandraCompactionFilter\ + org.rocksdb.CassandraValueMergeOperator\ + org.rocksdb.ColumnFamilyHandle\ + org.rocksdb.ColumnFamilyOptions\ + org.rocksdb.CompactionJobInfo\ + org.rocksdb.CompactionJobStats\ + org.rocksdb.CompactionOptions\ + org.rocksdb.CompactionOptionsFIFO\ + org.rocksdb.CompactionOptionsUniversal\ + org.rocksdb.CompactRangeOptions\ + org.rocksdb.ComparatorOptions\ + org.rocksdb.CompressionOptions\ + org.rocksdb.ConfigOptions\ + org.rocksdb.DBOptions\ + org.rocksdb.DirectSlice\ + org.rocksdb.Env\ + org.rocksdb.EnvOptions\ + org.rocksdb.FlushOptions\ + org.rocksdb.Filter\ + org.rocksdb.IngestExternalFileOptions\ + org.rocksdb.HashLinkedListMemTableConfig\ + org.rocksdb.HashSkipListMemTableConfig\ + org.rocksdb.ConcurrentTaskLimiter\ + org.rocksdb.ConcurrentTaskLimiterImpl\ + org.rocksdb.KeyMayExist\ + org.rocksdb.Logger\ + org.rocksdb.LRUCache\ + org.rocksdb.MemoryUsageType\ + org.rocksdb.MemoryUtil\ + org.rocksdb.MergeOperator\ + org.rocksdb.NativeComparatorWrapper\ + org.rocksdb.OptimisticTransactionDB\ + org.rocksdb.OptimisticTransactionOptions\ + org.rocksdb.Options\ + org.rocksdb.OptionsUtil\ + org.rocksdb.PersistentCache\ + org.rocksdb.PlainTableConfig\ + org.rocksdb.RateLimiter\ + org.rocksdb.ReadOptions\ + org.rocksdb.RemoveEmptyValueCompactionFilter\ + org.rocksdb.RestoreOptions\ + org.rocksdb.RocksCallbackObject\ + org.rocksdb.RocksDB\ + org.rocksdb.RocksEnv\ + org.rocksdb.RocksIterator\ + org.rocksdb.RocksMemEnv\ + org.rocksdb.SkipListMemTableConfig\ + org.rocksdb.Slice\ + org.rocksdb.SstFileManager\ + org.rocksdb.SstFileWriter\ + org.rocksdb.SstFileReader\ + org.rocksdb.SstFileReaderIterator\ + org.rocksdb.SstPartitionerFactory\ + org.rocksdb.SstPartitionerFixedPrefixFactory\ + org.rocksdb.Statistics\ + org.rocksdb.ThreadStatus\ + org.rocksdb.TimedEnv\ + org.rocksdb.Transaction\ + org.rocksdb.TransactionDB\ + org.rocksdb.TransactionDBOptions\ + org.rocksdb.TransactionOptions\ + org.rocksdb.TransactionLogIterator\ + org.rocksdb.TtlDB\ + org.rocksdb.VectorMemTableConfig\ + org.rocksdb.Snapshot\ + org.rocksdb.StringAppendOperator\ + org.rocksdb.UInt64AddOperator\ + org.rocksdb.WriteBatch\ + org.rocksdb.WriteBatch.Handler\ + org.rocksdb.WriteOptions\ + org.rocksdb.WriteBatchWithIndex\ + org.rocksdb.WriteBufferManager\ + org.rocksdb.WBWIRocksIterator + +NATIVE_JAVA_TEST_CLASSES = \ + org.rocksdb.RocksDBExceptionTest\ + org.rocksdb.test.TestableEventListener\ + org.rocksdb.NativeComparatorWrapperTest.NativeStringComparatorWrapper\ + org.rocksdb.WriteBatchTest\ + org.rocksdb.WriteBatchTestInternalHelper + +ROCKSDB_MAJOR = $(shell grep -E "ROCKSDB_MAJOR.[0-9]" ../include/rocksdb/version.h | cut -d ' ' -f 3) +ROCKSDB_MINOR = $(shell grep -E "ROCKSDB_MINOR.[0-9]" ../include/rocksdb/version.h | cut -d ' ' -f 3) +ROCKSDB_PATCH = $(shell grep -E "ROCKSDB_PATCH.[0-9]" ../include/rocksdb/version.h | cut -d ' ' -f 3) + +NATIVE_INCLUDE = ./include +ARCH := $(shell getconf LONG_BIT) +SHA256_CMD ?= sha256sum + +JAVA_TESTS = \ + org.rocksdb.BackupEngineOptionsTest\ + org.rocksdb.BackupEngineTest\ + org.rocksdb.BlobOptionsTest\ + org.rocksdb.BlockBasedTableConfigTest\ + org.rocksdb.BuiltinComparatorTest\ + org.rocksdb.ByteBufferUnsupportedOperationTest\ + org.rocksdb.BytewiseComparatorRegressionTest\ + org.rocksdb.util.BytewiseComparatorTest\ + org.rocksdb.util.BytewiseComparatorIntTest\ + org.rocksdb.CheckPointTest\ + org.rocksdb.ClockCacheTest\ + org.rocksdb.ColumnFamilyOptionsTest\ + org.rocksdb.ColumnFamilyTest\ + org.rocksdb.CompactionFilterFactoryTest\ + org.rocksdb.CompactionJobInfoTest\ + org.rocksdb.CompactionJobStatsTest\ + org.rocksdb.CompactionOptionsTest\ + org.rocksdb.CompactionOptionsFIFOTest\ + org.rocksdb.CompactionOptionsUniversalTest\ + org.rocksdb.CompactionPriorityTest\ + org.rocksdb.CompactionStopStyleTest\ + org.rocksdb.ComparatorOptionsTest\ + org.rocksdb.CompressionOptionsTest\ + org.rocksdb.CompressionTypesTest\ + org.rocksdb.DBOptionsTest\ + org.rocksdb.DirectSliceTest\ + org.rocksdb.util.EnvironmentTest\ + org.rocksdb.EnvOptionsTest\ + org.rocksdb.EventListenerTest\ + org.rocksdb.IngestExternalFileOptionsTest\ + org.rocksdb.util.IntComparatorTest\ + org.rocksdb.util.JNIComparatorTest\ + org.rocksdb.FilterTest\ + org.rocksdb.FlushTest\ + org.rocksdb.InfoLogLevelTest\ + org.rocksdb.KeyMayExistTest\ + org.rocksdb.ConcurrentTaskLimiterTest\ + org.rocksdb.LoggerTest\ + org.rocksdb.LRUCacheTest\ + org.rocksdb.MemoryUtilTest\ + org.rocksdb.MemTableTest\ + org.rocksdb.MergeTest\ + org.rocksdb.MultiColumnRegressionTest \ + org.rocksdb.MultiGetManyKeysTest\ + org.rocksdb.MultiGetTest\ + org.rocksdb.MixedOptionsTest\ + org.rocksdb.MutableColumnFamilyOptionsTest\ + org.rocksdb.MutableDBOptionsTest\ + org.rocksdb.MutableOptionsGetSetTest \ + org.rocksdb.NativeComparatorWrapperTest\ + org.rocksdb.NativeLibraryLoaderTest\ + org.rocksdb.OptimisticTransactionTest\ + org.rocksdb.OptimisticTransactionDBTest\ + org.rocksdb.OptimisticTransactionOptionsTest\ + org.rocksdb.OptionsUtilTest\ + org.rocksdb.OptionsTest\ + org.rocksdb.PlainTableConfigTest\ + org.rocksdb.RateLimiterTest\ + org.rocksdb.ReadOnlyTest\ + org.rocksdb.ReadOptionsTest\ + org.rocksdb.util.ReverseBytewiseComparatorIntTest\ + org.rocksdb.RocksDBTest\ + org.rocksdb.RocksDBExceptionTest\ + org.rocksdb.DefaultEnvTest\ + org.rocksdb.RocksIteratorTest\ + org.rocksdb.RocksMemEnvTest\ + org.rocksdb.util.SizeUnitTest\ + org.rocksdb.SecondaryDBTest\ + org.rocksdb.SliceTest\ + org.rocksdb.SnapshotTest\ + org.rocksdb.SstFileManagerTest\ + org.rocksdb.SstFileWriterTest\ + org.rocksdb.SstFileReaderTest\ + org.rocksdb.SstPartitionerTest\ + org.rocksdb.TableFilterTest\ + org.rocksdb.TimedEnvTest\ + org.rocksdb.TransactionTest\ + org.rocksdb.TransactionDBTest\ + org.rocksdb.TransactionOptionsTest\ + org.rocksdb.TransactionDBOptionsTest\ + org.rocksdb.TransactionLogIteratorTest\ + org.rocksdb.TtlDBTest\ + org.rocksdb.StatisticsTest\ + org.rocksdb.StatisticsCollectorTest\ + org.rocksdb.VerifyChecksumsTest\ + org.rocksdb.WalFilterTest\ + org.rocksdb.WALRecoveryModeTest\ + org.rocksdb.WriteBatchHandlerTest\ + org.rocksdb.WriteBatchTest\ + org.rocksdb.WriteBatchThreadedTest\ + org.rocksdb.WriteOptionsTest\ + org.rocksdb.WriteBatchWithIndexTest + +MAIN_SRC = src/main/java +TEST_SRC = src/test/java +OUTPUT = target +MAIN_CLASSES = $(OUTPUT)/classes +TEST_CLASSES = $(OUTPUT)/test-classes +JAVADOC = $(OUTPUT)/apidocs + +BENCHMARK_MAIN_SRC = benchmark/src/main/java +BENCHMARK_OUTPUT = benchmark/target +BENCHMARK_MAIN_CLASSES = $(BENCHMARK_OUTPUT)/classes + +SAMPLES_MAIN_SRC = samples/src/main/java +SAMPLES_OUTPUT = samples/target +SAMPLES_MAIN_CLASSES = $(SAMPLES_OUTPUT)/classes + +JAVA_TEST_LIBDIR = test-libs +JAVA_JUNIT_VER = 4.13.1 +JAVA_JUNIT_SHA256 = c30719db974d6452793fe191b3638a5777005485bae145924044530ffa5f6122 +JAVA_JUNIT_JAR = junit-$(JAVA_JUNIT_VER).jar +JAVA_JUNIT_JAR_PATH = $(JAVA_TEST_LIBDIR)/$(JAVA_JUNIT_JAR) +JAVA_HAMCREST_VER = 2.2 +JAVA_HAMCREST_SHA256 = 5e62846a89f05cd78cd9c1a553f340d002458380c320455dd1f8fc5497a8a1c1 +JAVA_HAMCREST_JAR = hamcrest-$(JAVA_HAMCREST_VER).jar +JAVA_HAMCREST_JAR_PATH = $(JAVA_TEST_LIBDIR)/$(JAVA_HAMCREST_JAR) +JAVA_MOCKITO_VER = 1.10.19 +JAVA_MOCKITO_SHA256 = d1a7a7ef14b3db5c0fc3e0a63a81b374b510afe85add9f7984b97911f4c70605 +JAVA_MOCKITO_JAR = mockito-all-$(JAVA_MOCKITO_VER).jar +JAVA_MOCKITO_JAR_PATH = $(JAVA_TEST_LIBDIR)/$(JAVA_MOCKITO_JAR) +JAVA_CGLIB_VER = 3.3.0 +JAVA_CGLIB_SHA256 = 9fe0c26d7464140ccdfe019ac687be1fb906122b508ab54beb810db0f09a9212 +JAVA_CGLIB_JAR = cglib-$(JAVA_CGLIB_VER).jar +JAVA_CGLIB_JAR_PATH = $(JAVA_TEST_LIBDIR)/$(JAVA_CGLIB_JAR) +JAVA_ASSERTJ_VER = 2.9.0 +JAVA_ASSERTJ_SHA256 = 5e88ea3ecbe3c48aa1346fec76c84979fa9c8d22499f11479011691230e8babf +JAVA_ASSERTJ_JAR = assertj-core-$(JAVA_ASSERTJ_VER).jar +JAVA_ASSERTJ_JAR_PATH = $(JAVA_TEST_LIBDIR)/$(JAVA_ASSERTJ_JAR) +JAVA_TESTCLASSPATH = $(JAVA_JUNIT_JAR_PATH):$(JAVA_HAMCREST_JAR_PATH):$(JAVA_MOCKITO_JAR_PATH):$(JAVA_CGLIB_JAR_PATH):$(JAVA_ASSERTJ_JAR_PATH) + +MVN_LOCAL = ~/.m2/repository + +# Set the path of the java commands +ifeq ($(JAVA_CMD),) +ifneq ($(JAVA_HOME),) +JAVA_CMD := $(JAVA_HOME)/bin/java +else +JAVA_CMD := java +endif +endif + +ifeq ($(JAVAC_CMD),) +ifneq ($(JAVA_HOME),) +JAVAC_CMD := $(JAVA_HOME)/bin/javac +else +JAVAC_CMD := javac +endif +endif + +ifeq ($(JAVADOC_CMD),) +ifneq ($(JAVA_HOME),) +JAVADOC_CMD := $(JAVA_HOME)/bin/javadoc +else +JAVADOC_CMD := javadoc +endif +endif + +# Look for the Java version (1.6->6, 1.7->7, 1.8->8, 11.0->11, 13.0->13, 15.0->15 etc..) +JAVAC_VERSION := $(shell $(JAVAC_CMD) -version 2>&1) +JAVAC_MAJOR_VERSION := $(word 2,$(subst ., ,$(JAVAC_VERSION))) +ifeq ($(JAVAC_MAJOR_VERSION),1) +JAVAC_MAJOR_VERSION := $(word 3,$(subst ., ,$(JAVAC_VERSION))) +endif + +# Test whether the version we see meets our minimum +MIN_JAVAC_MAJOR_VERSION := 8 +JAVAC_VERSION_GE_MIN := $(shell [ $(JAVAC_MAJOR_VERSION) -ge $(MIN_JAVAC_MAJOR_VERSION) ] > /dev/null 2>&1 && echo true) + +# Set the default JAVA_ARGS to "" for DEBUG_LEVEL=0 +JAVA_ARGS ?= + +JAVAC_ARGS ?= + +# Read plugin configuration +PLUGIN_PATH = ../plugin +ROCKSDB_PLUGIN_MKS = $(foreach plugin, $(ROCKSDB_PLUGINS), $(PLUGIN_PATH)/$(plugin)/*.mk) +include $(ROCKSDB_PLUGIN_MKS) + +# Add paths to Java sources in plugins +ROCKSDB_PLUGIN_JAVA_ROOTS = $(foreach plugin, $(ROCKSDB_PLUGINS), $(PLUGIN_PATH)/$(plugin)/java) +PLUGIN_SOURCES = $(foreach root, $(ROCKSDB_PLUGIN_JAVA_ROOTS), $(foreach pkg, org/rocksdb/util org/rocksdb, $(root)/$(MAIN_SRC)/$(pkg)/*.java)) +CORE_SOURCES = $(foreach pkg, org/rocksdb/util org/rocksdb, $(MAIN_SRC)/$(pkg)/*.java) +SOURCES = $(wildcard $(CORE_SOURCES) $(PLUGIN_SOURCES)) +PLUGIN_TEST_SOURCES = $(foreach root, $(ROCKSDB_PLUGIN_JAVA_ROOTS), $(foreach pkg, org/rocksdb/test org/rocksdb/util org/rocksdb, $(root)/$(TEST_SRC)/$(pkg)/*.java)) +CORE_TEST_SOURCES = $(foreach pkg, org/rocksdb/test org/rocksdb/util org/rocksdb, $(TEST_SRC)/$(pkg)/*.java) +TEST_SOURCES = $(wildcard $(CORE_TEST_SOURCES) $(PLUGIN_TEST_SOURCES)) + +# Configure the plugin tests and java classes +ROCKSDB_PLUGIN_NATIVE_JAVA_CLASSES = $(foreach plugin, $(ROCKSDB_PLUGINS), $(foreach class, $($(plugin)_NATIVE_JAVA_CLASSES), $(class))) +NATIVE_JAVA_CLASSES = $(NATIVE_JAVA_CLASSES) $(ROCKSDB_PLUGIN_NATIVE_JAVA_CLASSES) +ROCKSDB_PLUGIN_JAVA_TESTS = $(foreach plugin, $(ROCKSDB_PLUGINS), $(foreach testclass, $($(plugin)_JAVA_TESTS), $(testclass))) +ALL_JAVA_TESTS = $(JAVA_TESTS) $(ROCKSDB_PLUGIN_JAVA_TESTS) + +# When debugging add -Xcheck:jni to the java args +ifneq ($(DEBUG_LEVEL),0) + JAVA_ARGS += -ea -Xcheck:jni + JAVAC_ARGS += -Xlint:deprecation -Xlint:unchecked +endif + +# Using a Facebook AWS account for S3 storage. (maven.org has a history +# of failing in Travis builds.) +DEPS_URL?=https://rocksdb-deps.s3-us-west-2.amazonaws.com/jars + +java-version: +ifneq ($(JAVAC_VERSION_GE_MIN),true) + echo 'Java version is $(JAVAC_VERSION), minimum required version is $(MIN_JAVAC_MAJOR_VERSION)' + exit 1 +endif + +clean: clean-not-downloaded clean-downloaded + +clean-not-downloaded: + $(AM_V_at)rm -rf $(NATIVE_INCLUDE) + $(AM_V_at)rm -rf $(OUTPUT) + $(AM_V_at)rm -rf $(BENCHMARK_OUTPUT) + $(AM_V_at)rm -rf $(SAMPLES_OUTPUT) + +clean-downloaded: + $(AM_V_at)rm -rf $(JAVA_TEST_LIBDIR) + + +javadocs: java + $(AM_V_GEN)mkdir -p $(JAVADOC) + $(AM_V_at)$(JAVADOC_CMD) -d $(JAVADOC) -sourcepath $(MAIN_SRC) -subpackages org + +javalib: java java_test javadocs + +java: java-version + $(AM_V_GEN)mkdir -p $(MAIN_CLASSES) + $(AM_V_at) $(JAVAC_CMD) $(JAVAC_ARGS) -h $(NATIVE_INCLUDE) -d $(MAIN_CLASSES) $(SOURCES) + $(AM_V_at)@cp ../HISTORY.md ./HISTORY-CPP.md + $(AM_V_at)@rm -f ./HISTORY-CPP.md + +sample: java + $(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES) + $(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/RocksDBSample.java + $(AM_V_at)@rm -rf /tmp/rocksdbjni + $(AM_V_at)@rm -rf /tmp/rocksdbjni_not_found + $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) RocksDBSample /tmp/rocksdbjni + $(AM_V_at)@rm -rf /tmp/rocksdbjni + $(AM_V_at)@rm -rf /tmp/rocksdbjni_not_found + +column_family_sample: java + $(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES) + $(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/RocksDBColumnFamilySample.java + $(AM_V_at)@rm -rf /tmp/rocksdbjni + $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) RocksDBColumnFamilySample /tmp/rocksdbjni + $(AM_V_at)@rm -rf /tmp/rocksdbjni + +transaction_sample: java + $(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES) + $(AM_V_at)$(JAVAC_CMD) -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/TransactionSample.java + $(AM_V_at)@rm -rf /tmp/rocksdbjni + $(JAVA_CMD) -ea -Xcheck:jni -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) TransactionSample /tmp/rocksdbjni + $(AM_V_at)@rm -rf /tmp/rocksdbjni + +optimistic_transaction_sample: java + $(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES) + $(AM_V_at)$(JAVAC_CMD) -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/OptimisticTransactionSample.java + $(AM_V_at)@rm -rf /tmp/rocksdbjni + $(JAVA_CMD) -ea -Xcheck:jni -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) OptimisticTransactionSample /tmp/rocksdbjni + $(AM_V_at)@rm -rf /tmp/rocksdbjni + +$(JAVA_TEST_LIBDIR): + mkdir -p "$(JAVA_TEST_LIBDIR)" + +$(JAVA_JUNIT_JAR_PATH): $(JAVA_TEST_LIBDIR) +ifneq (,$(wildcard $(MVN_LOCAL)/junit/junit/$(JAVA_JUNIT_VER)/$(JAVA_JUNIT_JAR))) + cp -v $(MVN_LOCAL)/junit/junit/$(JAVA_JUNIT_VER)/$(JAVA_JUNIT_JAR) $(JAVA_TEST_LIBDIR) +else + curl --fail --insecure --output $(JAVA_JUNIT_JAR_PATH) --location $(DEPS_URL)/$(JAVA_JUNIT_JAR) + JAVA_JUNIT_SHA256_ACTUAL=`$(SHA256_CMD) $(JAVA_JUNIT_JAR_PATH) | cut -d ' ' -f 1`; \ + if [ "$(JAVA_JUNIT_SHA256)" != "$$JAVA_JUNIT_SHA256_ACTUAL" ]; then \ + echo $(JAVA_JUNIT_JAR_PATH) checksum mismatch, expected=\"$(JAVA_JUNIT_SHA256)\" actual=\"$$JAVA_JUNIT_SHA256_ACTUAL\"; \ + exit 1; \ + fi +endif + +$(JAVA_HAMCREST_JAR_PATH): $(JAVA_TEST_LIBDIR) +ifneq (,$(wildcard $(MVN_LOCAL)/org/hamcrest/hamcrest/$(JAVA_HAMCREST_VER)/$(JAVA_HAMCREST_JAR))) + cp -v $(MVN_LOCAL)/org/hamcrest/hamcrest/$(JAVA_HAMCREST_VER)/$(JAVA_HAMCREST_JAR) $(JAVA_TEST_LIBDIR) +else + curl --fail --insecure --output $(JAVA_HAMCREST_JAR_PATH) --location $(DEPS_URL)/$(JAVA_HAMCREST_JAR) + JAVA_HAMCREST_SHA256_ACTUAL=`$(SHA256_CMD) $(JAVA_HAMCREST_JAR_PATH) | cut -d ' ' -f 1`; \ + if [ "$(JAVA_HAMCREST_SHA256)" != "$$JAVA_HAMCREST_SHA256_ACTUAL" ]; then \ + echo $(JAVA_HAMCREST_JAR_PATH) checksum mismatch, expected=\"$(JAVA_HAMCREST_SHA256)\" actual=\"$$JAVA_HAMCREST_SHA256_ACTUAL\"; \ + exit 1; \ + fi +endif + +$(JAVA_MOCKITO_JAR_PATH): $(JAVA_TEST_LIBDIR) +ifneq (,$(wildcard $(MVN_LOCAL)/org/mockito/mockito-all/$(JAVA_MOCKITO_VER)/$(JAVA_MOCKITO_JAR))) + cp -v $(MVN_LOCAL)/org/mockito/mockito-all/$(JAVA_MOCKITO_VER)/$(JAVA_MOCKITO_JAR) $(JAVA_TEST_LIBDIR) +else + curl --fail --insecure --output "$(JAVA_MOCKITO_JAR_PATH)" --location $(DEPS_URL)/$(JAVA_MOCKITO_JAR) + JAVA_MOCKITO_SHA256_ACTUAL=`$(SHA256_CMD) $(JAVA_MOCKITO_JAR_PATH) | cut -d ' ' -f 1`; \ + if [ "$(JAVA_MOCKITO_SHA256)" != "$$JAVA_MOCKITO_SHA256_ACTUAL" ]; then \ + echo $(JAVA_MOCKITO_JAR_PATH) checksum mismatch, expected=\"$(JAVA_MOCKITO_SHA256)\" actual=\"$$JAVA_MOCKITO_SHA256_ACTUAL\"; \ + exit 1; \ + fi +endif + +$(JAVA_CGLIB_JAR_PATH): $(JAVA_TEST_LIBDIR) +ifneq (,$(wildcard $(MVN_LOCAL)/cglib/cglib/$(JAVA_CGLIB_VER)/$(JAVA_CGLIB_JAR))) + cp -v $(MVN_LOCAL)/cglib/cglib/$(JAVA_CGLIB_VER)/$(JAVA_CGLIB_JAR) $(JAVA_TEST_LIBDIR) +else + curl --fail --insecure --output "$(JAVA_CGLIB_JAR_PATH)" --location $(DEPS_URL)/$(JAVA_CGLIB_JAR) + JAVA_CGLIB_SHA256_ACTUAL=`$(SHA256_CMD) $(JAVA_CGLIB_JAR_PATH) | cut -d ' ' -f 1`; \ + if [ "$(JAVA_CGLIB_SHA256)" != "$$JAVA_CGLIB_SHA256_ACTUAL" ]; then \ + echo $(JAVA_CGLIB_JAR_PATH) checksum mismatch, expected=\"$(JAVA_CGLIB_SHA256)\" actual=\"$$JAVA_CGLIB_SHA256_ACTUAL\"; \ + exit 1; \ + fi +endif + +$(JAVA_ASSERTJ_JAR_PATH): $(JAVA_TEST_LIBDIR) +ifneq (,$(wildcard $(MVN_LOCAL)/org/assertj/assertj-core/$(JAVA_ASSERTJ_VER)/$(JAVA_ASSERTJ_JAR))) + cp -v $(MVN_LOCAL)/org/assertj/assertj-core/$(JAVA_ASSERTJ_VER)/$(JAVA_ASSERTJ_JAR) $(JAVA_TEST_LIBDIR) +else + curl --fail --insecure --output "$(JAVA_ASSERTJ_JAR_PATH)" --location $(DEPS_URL)/$(JAVA_ASSERTJ_JAR) + JAVA_ASSERTJ_SHA256_ACTUAL=`$(SHA256_CMD) $(JAVA_ASSERTJ_JAR_PATH) | cut -d ' ' -f 1`; \ + if [ "$(JAVA_ASSERTJ_SHA256)" != "$$JAVA_ASSERTJ_SHA256_ACTUAL" ]; then \ + echo $(JAVA_ASSERTJ_JAR_PATH) checksum mismatch, expected=\"$(JAVA_ASSERTJ_SHA256)\" actual=\"$$JAVA_ASSERTJ_SHA256_ACTUAL\"; \ + exit 1; \ + fi +endif + +resolve_test_deps: $(JAVA_JUNIT_JAR_PATH) $(JAVA_HAMCREST_JAR_PATH) $(JAVA_MOCKITO_JAR_PATH) $(JAVA_CGLIB_JAR_PATH) $(JAVA_ASSERTJ_JAR_PATH) + +java_test: java resolve_test_deps + $(AM_V_GEN)mkdir -p $(TEST_CLASSES) + $(AM_V_at) $(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES):$(JAVA_TESTCLASSPATH) -h $(NATIVE_INCLUDE) -d $(TEST_CLASSES)\ + $(TEST_SOURCES) + +test: java java_test + $(MAKE) run_test + +run_test: + $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp "$(MAIN_CLASSES):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.rocksdb.test.RocksJunitRunner $(ALL_JAVA_TESTS) + +run_plugin_test: + $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp "$(MAIN_CLASSES):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.rocksdb.test.RocksJunitRunner $(ROCKSDB_PLUGIN_JAVA_TESTS) + +db_bench: java + $(AM_V_GEN)mkdir -p $(BENCHMARK_MAIN_CLASSES) + $(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES) -d $(BENCHMARK_MAIN_CLASSES) $(BENCHMARK_MAIN_SRC)/org/rocksdb/benchmark/*.java diff --git a/librocksdb-sys/rocksdb/java/RELEASE.md b/librocksdb-sys/rocksdb/java/RELEASE.md new file mode 100644 index 0000000..dda1945 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/RELEASE.md @@ -0,0 +1,59 @@ +## Cross-building + +RocksDB can be built as a single self contained cross-platform JAR. The cross-platform jar can be used on any 64-bit OSX system, 32-bit Linux system, or 64-bit Linux system. + +Building a cross-platform JAR requires: + + * [Docker](https://www.docker.com/docker-community) + * A Mac OSX machine that can compile RocksDB. + * Java 7 set as JAVA_HOME. + +Once you have these items, run this make command from RocksDB's root source directory: + + make jclean clean rocksdbjavastaticreleasedocker + +This command will build RocksDB natively on OSX, and will then spin up docker containers to build RocksDB for 32-bit and 64-bit Linux with glibc, and 32-bit and 64-bit Linux with musl libc. + +You can find all native binaries and JARs in the java/target directory upon completion: + + librocksdbjni-linux32.so + librocksdbjni-linux64.so + librocksdbjni-linux64-musl.so + librocksdbjni-linux32-musl.so + librocksdbjni-osx.jnilib + rocksdbjni-x.y.z-javadoc.jar + rocksdbjni-x.y.z-linux32.jar + rocksdbjni-x.y.z-linux64.jar + rocksdbjni-x.y.z-linux64-musl.jar + rocksdbjni-x.y.z-linux32-musl.jar + rocksdbjni-x.y.z-osx.jar + rocksdbjni-x.y.z-sources.jar + rocksdbjni-x.y.z.jar + +Where x.y.z is the built version number of RocksDB. + +## Maven publication + +Set ~/.m2/settings.xml to contain: + + + + + sonatype-nexus-staging + your-sonatype-jira-username + your-sonatype-jira-password + + + + +From RocksDB's root directory, first build the Java static JARs: + + make jclean clean rocksdbjavastaticpublish + +This command will [stage the JAR artifacts on the Sonatype staging repository](http://central.sonatype.org/pages/manual-staging-bundle-creation-and-deployment.html). To release the staged artifacts. + +1. Go to [https://oss.sonatype.org/#stagingRepositories](https://oss.sonatype.org/#stagingRepositories) and search for "rocksdb" in the upper right hand search box. +2. Select the rocksdb staging repository, and inspect its contents. +3. If all is well, follow [these steps](https://oss.sonatype.org/#stagingRepositories) to close the repository and release it. + +After the release has occurred, the artifacts will be synced to Maven central within 24-48 hours. diff --git a/librocksdb-sys/rocksdb/java/benchmark/src/main/java/org/rocksdb/benchmark/DbBenchmark.java b/librocksdb-sys/rocksdb/java/benchmark/src/main/java/org/rocksdb/benchmark/DbBenchmark.java new file mode 100644 index 0000000..070f0fe --- /dev/null +++ b/librocksdb-sys/rocksdb/java/benchmark/src/main/java/org/rocksdb/benchmark/DbBenchmark.java @@ -0,0 +1,1640 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +/** + * Copyright (C) 2011 the original author or authors. + * See the notice.md file distributed with this work for additional + * information regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.rocksdb.benchmark; + +import java.io.IOException; +import java.lang.Runnable; +import java.lang.Math; +import java.io.File; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.util.Collection; +import java.util.Date; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.concurrent.Callable; +import java.util.concurrent.Executors; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.rocksdb.*; +import org.rocksdb.RocksMemEnv; +import org.rocksdb.util.SizeUnit; + +class Stats { + int id_; + long start_; + long finish_; + double seconds_; + long done_; + long found_; + long lastOpTime_; + long nextReport_; + long bytes_; + StringBuilder message_; + boolean excludeFromMerge_; + + // TODO(yhchiang): use the following arguments: + // (Long)Flag.stats_interval + // (Integer)Flag.stats_per_interval + + Stats(int id) { + id_ = id; + nextReport_ = 100; + done_ = 0; + bytes_ = 0; + seconds_ = 0; + start_ = System.nanoTime(); + lastOpTime_ = start_; + finish_ = start_; + found_ = 0; + message_ = new StringBuilder(""); + excludeFromMerge_ = false; + } + + void merge(final Stats other) { + if (other.excludeFromMerge_) { + return; + } + + done_ += other.done_; + found_ += other.found_; + bytes_ += other.bytes_; + seconds_ += other.seconds_; + if (other.start_ < start_) start_ = other.start_; + if (other.finish_ > finish_) finish_ = other.finish_; + + // Just keep the messages from one thread + if (message_.length() == 0) { + message_ = other.message_; + } + } + + void stop() { + finish_ = System.nanoTime(); + seconds_ = (double) (finish_ - start_) * 1e-9; + } + + void addMessage(String msg) { + if (message_.length() > 0) { + message_.append(" "); + } + message_.append(msg); + } + + void setId(int id) { id_ = id; } + void setExcludeFromMerge() { excludeFromMerge_ = true; } + + void finishedSingleOp(int bytes) { + done_++; + lastOpTime_ = System.nanoTime(); + bytes_ += bytes; + if (done_ >= nextReport_) { + if (nextReport_ < 1000) { + nextReport_ += 100; + } else if (nextReport_ < 5000) { + nextReport_ += 500; + } else if (nextReport_ < 10000) { + nextReport_ += 1000; + } else if (nextReport_ < 50000) { + nextReport_ += 5000; + } else if (nextReport_ < 100000) { + nextReport_ += 10000; + } else if (nextReport_ < 500000) { + nextReport_ += 50000; + } else { + nextReport_ += 100000; + } + System.err.printf("... Task %s finished %d ops%30s\r", id_, done_, ""); + } + } + + void report(String name) { + // Pretend at least one op was done in case we are running a benchmark + // that does not call FinishedSingleOp(). + if (done_ < 1) done_ = 1; + + StringBuilder extra = new StringBuilder(""); + if (bytes_ > 0) { + // Rate is computed on actual elapsed time, not the sum of per-thread + // elapsed times. + double elapsed = (finish_ - start_) * 1e-9; + extra.append(String.format("%6.1f MB/s", (bytes_ / 1048576.0) / elapsed)); + } + extra.append(message_.toString()); + double elapsed = (finish_ - start_); + double throughput = (double) done_ / (elapsed * 1e-9); + + System.out.format("%-12s : %11.3f micros/op %d ops/sec;%s%s\n", + name, (elapsed * 1e-6) / done_, + (long) throughput, (extra.length() == 0 ? "" : " "), extra.toString()); + } +} + +public class DbBenchmark { + enum Order { + SEQUENTIAL, + RANDOM + } + + enum DBState { + FRESH, + EXISTING + } + + static { + RocksDB.loadLibrary(); + } + + abstract class BenchmarkTask implements Callable { + // TODO(yhchiang): use (Integer)Flag.perf_level. + public BenchmarkTask( + int tid, long randSeed, long numEntries, long keyRange) { + tid_ = tid; + rand_ = new Random(randSeed + tid * 1000); + numEntries_ = numEntries; + keyRange_ = keyRange; + stats_ = new Stats(tid); + } + + @Override public Stats call() throws RocksDBException { + stats_.start_ = System.nanoTime(); + runTask(); + stats_.finish_ = System.nanoTime(); + return stats_; + } + + abstract protected void runTask() throws RocksDBException; + + protected int tid_; + protected Random rand_; + protected long numEntries_; + protected long keyRange_; + protected Stats stats_; + + protected void getFixedKey(byte[] key, long sn) { + generateKeyFromLong(key, sn); + } + + protected void getRandomKey(byte[] key, long range) { + generateKeyFromLong(key, Math.abs(rand_.nextLong() % range)); + } + } + + abstract class WriteTask extends BenchmarkTask { + public WriteTask( + int tid, long randSeed, long numEntries, long keyRange, + WriteOptions writeOpt, long entriesPerBatch) { + super(tid, randSeed, numEntries, keyRange); + writeOpt_ = writeOpt; + entriesPerBatch_ = entriesPerBatch; + maxWritesPerSecond_ = -1; + } + + public WriteTask( + int tid, long randSeed, long numEntries, long keyRange, + WriteOptions writeOpt, long entriesPerBatch, long maxWritesPerSecond) { + super(tid, randSeed, numEntries, keyRange); + writeOpt_ = writeOpt; + entriesPerBatch_ = entriesPerBatch; + maxWritesPerSecond_ = maxWritesPerSecond; + } + + @Override public void runTask() throws RocksDBException { + if (numEntries_ != DbBenchmark.this.num_) { + stats_.message_.append(String.format(" (%d ops)", numEntries_)); + } + byte[] key = new byte[keySize_]; + byte[] value = new byte[valueSize_]; + + try { + if (entriesPerBatch_ == 1) { + for (long i = 0; i < numEntries_; ++i) { + getKey(key, i, keyRange_); + DbBenchmark.this.gen_.generate(value); + db_.put(writeOpt_, key, value); + stats_.finishedSingleOp(keySize_ + valueSize_); + writeRateControl(i); + if (isFinished()) { + return; + } + } + } else { + for (long i = 0; i < numEntries_; i += entriesPerBatch_) { + WriteBatch batch = new WriteBatch(); + for (long j = 0; j < entriesPerBatch_; j++) { + getKey(key, i + j, keyRange_); + DbBenchmark.this.gen_.generate(value); + batch.put(key, value); + stats_.finishedSingleOp(keySize_ + valueSize_); + } + db_.write(writeOpt_, batch); + batch.dispose(); + writeRateControl(i); + if (isFinished()) { + return; + } + } + } + } catch (InterruptedException e) { + // thread has been terminated. + } + } + + protected void writeRateControl(long writeCount) + throws InterruptedException { + if (maxWritesPerSecond_ <= 0) return; + long minInterval = + writeCount * TimeUnit.SECONDS.toNanos(1) / maxWritesPerSecond_; + long interval = System.nanoTime() - stats_.start_; + if (minInterval - interval > TimeUnit.MILLISECONDS.toNanos(1)) { + TimeUnit.NANOSECONDS.sleep(minInterval - interval); + } + } + + abstract protected void getKey(byte[] key, long id, long range); + protected WriteOptions writeOpt_; + protected long entriesPerBatch_; + protected long maxWritesPerSecond_; + } + + class WriteSequentialTask extends WriteTask { + public WriteSequentialTask( + int tid, long randSeed, long numEntries, long keyRange, + WriteOptions writeOpt, long entriesPerBatch) { + super(tid, randSeed, numEntries, keyRange, + writeOpt, entriesPerBatch); + } + public WriteSequentialTask( + int tid, long randSeed, long numEntries, long keyRange, + WriteOptions writeOpt, long entriesPerBatch, + long maxWritesPerSecond) { + super(tid, randSeed, numEntries, keyRange, + writeOpt, entriesPerBatch, + maxWritesPerSecond); + } + @Override protected void getKey(byte[] key, long id, long range) { + getFixedKey(key, id); + } + } + + class WriteRandomTask extends WriteTask { + public WriteRandomTask( + int tid, long randSeed, long numEntries, long keyRange, + WriteOptions writeOpt, long entriesPerBatch) { + super(tid, randSeed, numEntries, keyRange, + writeOpt, entriesPerBatch); + } + public WriteRandomTask( + int tid, long randSeed, long numEntries, long keyRange, + WriteOptions writeOpt, long entriesPerBatch, + long maxWritesPerSecond) { + super(tid, randSeed, numEntries, keyRange, + writeOpt, entriesPerBatch, + maxWritesPerSecond); + } + @Override protected void getKey(byte[] key, long id, long range) { + getRandomKey(key, range); + } + } + + class WriteUniqueRandomTask extends WriteTask { + static final int MAX_BUFFER_SIZE = 10000000; + public WriteUniqueRandomTask( + int tid, long randSeed, long numEntries, long keyRange, + WriteOptions writeOpt, long entriesPerBatch) { + super(tid, randSeed, numEntries, keyRange, + writeOpt, entriesPerBatch); + initRandomKeySequence(); + } + public WriteUniqueRandomTask( + int tid, long randSeed, long numEntries, long keyRange, + WriteOptions writeOpt, long entriesPerBatch, + long maxWritesPerSecond) { + super(tid, randSeed, numEntries, keyRange, + writeOpt, entriesPerBatch, + maxWritesPerSecond); + initRandomKeySequence(); + } + @Override protected void getKey(byte[] key, long id, long range) { + generateKeyFromLong(key, nextUniqueRandom()); + } + + protected void initRandomKeySequence() { + bufferSize_ = MAX_BUFFER_SIZE; + if (bufferSize_ > keyRange_) { + bufferSize_ = (int) keyRange_; + } + currentKeyCount_ = bufferSize_; + keyBuffer_ = new long[MAX_BUFFER_SIZE]; + for (int k = 0; k < bufferSize_; ++k) { + keyBuffer_[k] = k; + } + } + + /** + * Semi-randomly return the next unique key. It is guaranteed to be + * fully random if keyRange_ <= MAX_BUFFER_SIZE. + */ + long nextUniqueRandom() { + if (bufferSize_ == 0) { + System.err.println("bufferSize_ == 0."); + return 0; + } + int r = rand_.nextInt(bufferSize_); + // randomly pick one from the keyBuffer + long randKey = keyBuffer_[r]; + if (currentKeyCount_ < keyRange_) { + // if we have not yet inserted all keys, insert next new key to [r]. + keyBuffer_[r] = currentKeyCount_++; + } else { + // move the last element to [r] and decrease the size by 1. + keyBuffer_[r] = keyBuffer_[--bufferSize_]; + } + return randKey; + } + + int bufferSize_; + long currentKeyCount_; + long[] keyBuffer_; + } + + class ReadRandomTask extends BenchmarkTask { + public ReadRandomTask( + int tid, long randSeed, long numEntries, long keyRange) { + super(tid, randSeed, numEntries, keyRange); + } + @Override public void runTask() throws RocksDBException { + byte[] key = new byte[keySize_]; + byte[] value = new byte[valueSize_]; + for (long i = 0; i < numEntries_; i++) { + getRandomKey(key, keyRange_); + int len = db_.get(key, value); + if (len != RocksDB.NOT_FOUND) { + stats_.found_++; + stats_.finishedSingleOp(keySize_ + valueSize_); + } else { + stats_.finishedSingleOp(keySize_); + } + if (isFinished()) { + return; + } + } + } + } + + class ReadSequentialTask extends BenchmarkTask { + public ReadSequentialTask( + int tid, long randSeed, long numEntries, long keyRange) { + super(tid, randSeed, numEntries, keyRange); + } + @Override public void runTask() throws RocksDBException { + RocksIterator iter = db_.newIterator(); + long i; + for (iter.seekToFirst(), i = 0; + iter.isValid() && i < numEntries_; + iter.next(), ++i) { + stats_.found_++; + stats_.finishedSingleOp(iter.key().length + iter.value().length); + if (isFinished()) { + iter.dispose(); + return; + } + } + iter.dispose(); + } + } + + public DbBenchmark(Map flags) throws Exception { + benchmarks_ = (List) flags.get(Flag.benchmarks); + num_ = (Integer) flags.get(Flag.num); + threadNum_ = (Integer) flags.get(Flag.threads); + reads_ = (Integer) (flags.get(Flag.reads) == null ? + flags.get(Flag.num) : flags.get(Flag.reads)); + keySize_ = (Integer) flags.get(Flag.key_size); + valueSize_ = (Integer) flags.get(Flag.value_size); + compressionRatio_ = (Double) flags.get(Flag.compression_ratio); + useExisting_ = (Boolean) flags.get(Flag.use_existing_db); + randSeed_ = (Long) flags.get(Flag.seed); + databaseDir_ = (String) flags.get(Flag.db); + writesPerSeconds_ = (Integer) flags.get(Flag.writes_per_second); + memtable_ = (String) flags.get(Flag.memtablerep); + maxWriteBufferNumber_ = (Integer) flags.get(Flag.max_write_buffer_number); + prefixSize_ = (Integer) flags.get(Flag.prefix_size); + keysPerPrefix_ = (Integer) flags.get(Flag.keys_per_prefix); + hashBucketCount_ = (Long) flags.get(Flag.hash_bucket_count); + usePlainTable_ = (Boolean) flags.get(Flag.use_plain_table); + useMemenv_ = (Boolean) flags.get(Flag.use_mem_env); + flags_ = flags; + finishLock_ = new Object(); + // options.setPrefixSize((Integer)flags_.get(Flag.prefix_size)); + // options.setKeysPerPrefix((Long)flags_.get(Flag.keys_per_prefix)); + compressionType_ = (String) flags.get(Flag.compression_type); + compression_ = CompressionType.NO_COMPRESSION; + try { + if (compressionType_!=null) { + final CompressionType compressionType = + CompressionType.getCompressionType(compressionType_); + if (compressionType != null && + compressionType != CompressionType.NO_COMPRESSION) { + System.loadLibrary(compressionType.getLibraryName()); + } + + } + } catch (UnsatisfiedLinkError e) { + System.err.format("Unable to load %s library:%s%n" + + "No compression is used.%n", + compressionType_, e.toString()); + compressionType_ = "none"; + } + gen_ = new RandomGenerator(randSeed_, compressionRatio_); + } + + private void prepareReadOptions(ReadOptions options) { + options.setVerifyChecksums((Boolean)flags_.get(Flag.verify_checksum)); + options.setTailing((Boolean)flags_.get(Flag.use_tailing_iterator)); + } + + private void prepareWriteOptions(WriteOptions options) { + options.setSync((Boolean)flags_.get(Flag.sync)); + options.setDisableWAL((Boolean)flags_.get(Flag.disable_wal)); + } + + private void prepareOptions(Options options) throws RocksDBException { + if (!useExisting_) { + options.setCreateIfMissing(true); + } else { + options.setCreateIfMissing(false); + } + if (useMemenv_) { + options.setEnv(new RocksMemEnv(Env.getDefault())); + } + switch (memtable_) { + case "skip_list": + options.setMemTableConfig(new SkipListMemTableConfig()); + break; + case "vector": + options.setMemTableConfig(new VectorMemTableConfig()); + break; + case "hash_linkedlist": + options.setMemTableConfig( + new HashLinkedListMemTableConfig() + .setBucketCount(hashBucketCount_)); + options.useFixedLengthPrefixExtractor(prefixSize_); + break; + case "hash_skiplist": + case "prefix_hash": + options.setMemTableConfig( + new HashSkipListMemTableConfig() + .setBucketCount(hashBucketCount_)); + options.useFixedLengthPrefixExtractor(prefixSize_); + break; + default: + System.err.format( + "unable to detect the specified memtable, " + + "use the default memtable factory %s%n", + options.memTableFactoryName()); + break; + } + if (usePlainTable_) { + options.setTableFormatConfig( + new PlainTableConfig().setKeySize(keySize_)); + } else { + BlockBasedTableConfig table_options = new BlockBasedTableConfig(); + table_options.setBlockSize((Long)flags_.get(Flag.block_size)) + .setBlockCacheSize((Long)flags_.get(Flag.cache_size)) + .setCacheNumShardBits( + (Integer)flags_.get(Flag.cache_numshardbits)); + options.setTableFormatConfig(table_options); + } + options.setWriteBufferSize( + (Long)flags_.get(Flag.write_buffer_size)); + options.setMaxWriteBufferNumber( + (Integer)flags_.get(Flag.max_write_buffer_number)); + options.setMaxBackgroundCompactions( + (Integer)flags_.get(Flag.max_background_compactions)); + options.getEnv().setBackgroundThreads( + (Integer)flags_.get(Flag.max_background_compactions)); + options.setMaxBackgroundFlushes( + (Integer)flags_.get(Flag.max_background_flushes)); + options.setMaxBackgroundJobs((Integer) flags_.get(Flag.max_background_jobs)); + options.setMaxOpenFiles( + (Integer)flags_.get(Flag.open_files)); + options.setUseFsync( + (Boolean)flags_.get(Flag.use_fsync)); + options.setWalDir( + (String)flags_.get(Flag.wal_dir)); + options.setDeleteObsoleteFilesPeriodMicros( + (Integer)flags_.get(Flag.delete_obsolete_files_period_micros)); + options.setTableCacheNumshardbits( + (Integer)flags_.get(Flag.table_cache_numshardbits)); + options.setAllowMmapReads( + (Boolean)flags_.get(Flag.mmap_read)); + options.setAllowMmapWrites( + (Boolean)flags_.get(Flag.mmap_write)); + options.setAdviseRandomOnOpen( + (Boolean)flags_.get(Flag.advise_random_on_open)); + options.setUseAdaptiveMutex( + (Boolean)flags_.get(Flag.use_adaptive_mutex)); + options.setBytesPerSync( + (Long)flags_.get(Flag.bytes_per_sync)); + options.setBloomLocality( + (Integer)flags_.get(Flag.bloom_locality)); + options.setMinWriteBufferNumberToMerge( + (Integer)flags_.get(Flag.min_write_buffer_number_to_merge)); + options.setMemtablePrefixBloomSizeRatio((Double) flags_.get(Flag.memtable_bloom_size_ratio)); + options.setMemtableWholeKeyFiltering((Boolean) flags_.get(Flag.memtable_whole_key_filtering)); + options.setNumLevels( + (Integer)flags_.get(Flag.num_levels)); + options.setTargetFileSizeBase( + (Integer)flags_.get(Flag.target_file_size_base)); + options.setTargetFileSizeMultiplier((Integer)flags_.get(Flag.target_file_size_multiplier)); + options.setMaxBytesForLevelBase( + (Integer)flags_.get(Flag.max_bytes_for_level_base)); + options.setMaxBytesForLevelMultiplier((Double) flags_.get(Flag.max_bytes_for_level_multiplier)); + options.setLevelZeroStopWritesTrigger( + (Integer)flags_.get(Flag.level0_stop_writes_trigger)); + options.setLevelZeroSlowdownWritesTrigger( + (Integer)flags_.get(Flag.level0_slowdown_writes_trigger)); + options.setLevelZeroFileNumCompactionTrigger( + (Integer)flags_.get(Flag.level0_file_num_compaction_trigger)); + options.setMaxCompactionBytes( + (Long) flags_.get(Flag.max_compaction_bytes)); + options.setDisableAutoCompactions( + (Boolean)flags_.get(Flag.disable_auto_compactions)); + options.setMaxSuccessiveMerges( + (Integer)flags_.get(Flag.max_successive_merges)); + options.setWalTtlSeconds((Long)flags_.get(Flag.wal_ttl_seconds)); + options.setWalSizeLimitMB((Long)flags_.get(Flag.wal_size_limit_MB)); + if(flags_.get(Flag.java_comparator) != null) { + options.setComparator( + (AbstractComparator)flags_.get(Flag.java_comparator)); + } + + /* TODO(yhchiang): enable the following parameters + options.setCompressionType((String)flags_.get(Flag.compression_type)); + options.setCompressionLevel((Integer)flags_.get(Flag.compression_level)); + options.setMinLevelToCompress((Integer)flags_.get(Flag.min_level_to_compress)); + options.setStatistics((Boolean)flags_.get(Flag.statistics)); + options.setUniversalSizeRatio( + (Integer)flags_.get(Flag.universal_size_ratio)); + options.setUniversalMinMergeWidth( + (Integer)flags_.get(Flag.universal_min_merge_width)); + options.setUniversalMaxMergeWidth( + (Integer)flags_.get(Flag.universal_max_merge_width)); + options.setUniversalMaxSizeAmplificationPercent( + (Integer)flags_.get(Flag.universal_max_size_amplification_percent)); + options.setUniversalCompressionSizePercent( + (Integer)flags_.get(Flag.universal_compression_size_percent)); + // TODO(yhchiang): add RocksDB.openForReadOnly() to enable Flag.readonly + // TODO(yhchiang): enable Flag.merge_operator by switch + options.setAccessHintOnCompactionStart( + (String)flags_.get(Flag.compaction_fadvice)); + // available values of fadvice are "NONE", "NORMAL", "SEQUENTIAL", "WILLNEED" for fadvice + */ + } + + private void run() throws RocksDBException { + if (!useExisting_) { + destroyDb(); + } + Options options = new Options(); + prepareOptions(options); + open(options); + + printHeader(options); + + for (String benchmark : benchmarks_) { + List> tasks = new ArrayList>(); + List> bgTasks = new ArrayList>(); + WriteOptions writeOpt = new WriteOptions(); + prepareWriteOptions(writeOpt); + ReadOptions readOpt = new ReadOptions(); + prepareReadOptions(readOpt); + int currentTaskId = 0; + boolean known = true; + + switch (benchmark) { + case "fillseq": + tasks.add(new WriteSequentialTask( + currentTaskId++, randSeed_, num_, num_, writeOpt, 1)); + break; + case "fillbatch": + tasks.add( + new WriteSequentialTask(currentTaskId++, randSeed_, num_, num_, writeOpt, 1000)); + break; + case "fillrandom": + tasks.add(new WriteRandomTask( + currentTaskId++, randSeed_, num_, num_, writeOpt, 1)); + break; + case "filluniquerandom": + tasks.add(new WriteUniqueRandomTask( + currentTaskId++, randSeed_, num_, num_, writeOpt, 1)); + break; + case "fillsync": + writeOpt.setSync(true); + tasks.add(new WriteRandomTask( + currentTaskId++, randSeed_, num_ / 1000, num_ / 1000, + writeOpt, 1)); + break; + case "readseq": + for (int t = 0; t < threadNum_; ++t) { + tasks.add(new ReadSequentialTask( + currentTaskId++, randSeed_, reads_ / threadNum_, num_)); + } + break; + case "readrandom": + for (int t = 0; t < threadNum_; ++t) { + tasks.add(new ReadRandomTask( + currentTaskId++, randSeed_, reads_ / threadNum_, num_)); + } + break; + case "readwhilewriting": + WriteTask writeTask = new WriteRandomTask( + -1, randSeed_, Long.MAX_VALUE, num_, writeOpt, 1, writesPerSeconds_); + writeTask.stats_.setExcludeFromMerge(); + bgTasks.add(writeTask); + for (int t = 0; t < threadNum_; ++t) { + tasks.add(new ReadRandomTask( + currentTaskId++, randSeed_, reads_ / threadNum_, num_)); + } + break; + case "readhot": + for (int t = 0; t < threadNum_; ++t) { + tasks.add(new ReadRandomTask( + currentTaskId++, randSeed_, reads_ / threadNum_, num_ / 100)); + } + break; + case "delete": + destroyDb(); + open(options); + break; + default: + known = false; + System.err.println("Unknown benchmark: " + benchmark); + break; + } + if (known) { + ExecutorService executor = Executors.newCachedThreadPool(); + ExecutorService bgExecutor = Executors.newCachedThreadPool(); + try { + // measure only the main executor time + List> bgResults = new ArrayList>(); + for (Callable bgTask : bgTasks) { + bgResults.add(bgExecutor.submit(bgTask)); + } + start(); + List> results = executor.invokeAll(tasks); + executor.shutdown(); + boolean finished = executor.awaitTermination(10, TimeUnit.SECONDS); + if (!finished) { + System.out.format( + "Benchmark %s was not finished before timeout.", + benchmark); + executor.shutdownNow(); + } + setFinished(true); + bgExecutor.shutdown(); + finished = bgExecutor.awaitTermination(10, TimeUnit.SECONDS); + if (!finished) { + System.out.format( + "Benchmark %s was not finished before timeout.", + benchmark); + bgExecutor.shutdownNow(); + } + + stop(benchmark, results, currentTaskId); + } catch (InterruptedException e) { + System.err.println(e); + } + } + writeOpt.dispose(); + readOpt.dispose(); + } + options.dispose(); + db_.close(); + } + + private void printHeader(Options options) { + int kKeySize = 16; + System.out.printf("Keys: %d bytes each\n", kKeySize); + System.out.printf("Values: %d bytes each (%d bytes after compression)\n", + valueSize_, + (int) (valueSize_ * compressionRatio_ + 0.5)); + System.out.printf("Entries: %d\n", num_); + System.out.printf("RawSize: %.1f MB (estimated)\n", + ((double)(kKeySize + valueSize_) * num_) / SizeUnit.MB); + System.out.printf("FileSize: %.1f MB (estimated)\n", + (((kKeySize + valueSize_ * compressionRatio_) * num_) / SizeUnit.MB)); + System.out.format("Memtable Factory: %s%n", options.memTableFactoryName()); + System.out.format("Prefix: %d bytes%n", prefixSize_); + System.out.format("Compression: %s%n", compressionType_); + printWarnings(); + System.out.printf("------------------------------------------------\n"); + } + + void printWarnings() { + boolean assertsEnabled = false; + assert assertsEnabled = true; // Intentional side effect!!! + if (assertsEnabled) { + System.out.printf( + "WARNING: Assertions are enabled; benchmarks unnecessarily slow\n"); + } + } + + private void open(Options options) throws RocksDBException { + System.out.println("Using database directory: " + databaseDir_); + db_ = RocksDB.open(options, databaseDir_); + } + + private void start() { + setFinished(false); + startTime_ = System.nanoTime(); + } + + private void stop( + String benchmark, List> results, int concurrentThreads) { + long endTime = System.nanoTime(); + double elapsedSeconds = + 1.0d * (endTime - startTime_) / TimeUnit.SECONDS.toNanos(1); + + Stats stats = new Stats(-1); + int taskFinishedCount = 0; + for (Future result : results) { + if (result.isDone()) { + try { + Stats taskStats = result.get(3, TimeUnit.SECONDS); + if (!result.isCancelled()) { + taskFinishedCount++; + } + stats.merge(taskStats); + } catch (Exception e) { + // then it's not successful, the output will indicate this + } + } + } + String extra = ""; + if (benchmark.indexOf("read") >= 0) { + extra = String.format(" %d / %d found; ", stats.found_, stats.done_); + } else { + extra = String.format(" %d ops done; ", stats.done_); + } + + System.out.printf( + "%-16s : %11.5f micros/op; %6.1f MB/s;%s %d / %d task(s) finished.\n", + benchmark, elapsedSeconds / stats.done_ * 1e6, + (stats.bytes_ / 1048576.0) / elapsedSeconds, extra, + taskFinishedCount, concurrentThreads); + } + + public void generateKeyFromLong(byte[] slice, long n) { + assert(n >= 0); + int startPos = 0; + + if (keysPerPrefix_ > 0) { + long numPrefix = (num_ + keysPerPrefix_ - 1) / keysPerPrefix_; + long prefix = n % numPrefix; + int bytesToFill = Math.min(prefixSize_, 8); + for (int i = 0; i < bytesToFill; ++i) { + slice[i] = (byte) (prefix % 256); + prefix /= 256; + } + for (int i = 8; i < bytesToFill; ++i) { + slice[i] = '0'; + } + startPos = bytesToFill; + } + + for (int i = slice.length - 1; i >= startPos; --i) { + slice[i] = (byte) ('0' + (n % 10)); + n /= 10; + } + } + + private void destroyDb() { + if (db_ != null) { + db_.close(); + } + // TODO(yhchiang): develop our own FileUtil + // FileUtil.deleteDir(databaseDir_); + } + + private void printStats() { + } + + static void printHelp() { + System.out.println("usage:"); + for (Flag flag : Flag.values()) { + System.out.format(" --%s%n\t%s%n", + flag.name(), + flag.desc()); + if (flag.getDefaultValue() != null) { + System.out.format("\tDEFAULT: %s%n", + flag.getDefaultValue().toString()); + } + } + } + + public static void main(String[] args) throws Exception { + Map flags = new EnumMap(Flag.class); + for (Flag flag : Flag.values()) { + if (flag.getDefaultValue() != null) { + flags.put(flag, flag.getDefaultValue()); + } + } + for (String arg : args) { + boolean valid = false; + if (arg.equals("--help") || arg.equals("-h")) { + printHelp(); + System.exit(0); + } + if (arg.startsWith("--")) { + try { + String[] parts = arg.substring(2).split("="); + if (parts.length >= 1) { + Flag key = Flag.valueOf(parts[0]); + if (key != null) { + Object value = null; + if (parts.length >= 2) { + value = key.parseValue(parts[1]); + } + flags.put(key, value); + valid = true; + } + } + } + catch (Exception e) { + } + } + if (!valid) { + System.err.println("Invalid argument " + arg); + System.exit(1); + } + } + new DbBenchmark(flags).run(); + } + + private enum Flag { + benchmarks(Arrays.asList("fillseq", "readrandom", "fillrandom"), + "Comma-separated list of operations to run in the specified order\n" + + "\tActual benchmarks:\n" + + "\t\tfillseq -- write N values in sequential key order in async mode.\n" + + "\t\tfillrandom -- write N values in random key order in async mode.\n" + + "\t\tfillbatch -- write N/1000 batch where each batch has 1000 values\n" + + "\t\t in sequential key order in sync mode.\n" + + "\t\tfillsync -- write N/100 values in random key order in sync mode.\n" + + "\t\tfill100K -- write N/1000 100K values in random order in async mode.\n" + + "\t\treadseq -- read N times sequentially.\n" + + "\t\treadrandom -- read N times in random order.\n" + + "\t\treadhot -- read N times in random order from 1% section of DB.\n" + + "\t\treadwhilewriting -- measure the read performance of multiple readers\n" + + "\t\t with a bg single writer. The write rate of the bg\n" + + "\t\t is capped by --writes_per_second.\n" + + "\tMeta Operations:\n" + + "\t\tdelete -- delete DB") { + @Override public Object parseValue(String value) { + return new ArrayList(Arrays.asList(value.split(","))); + } + }, + compression_ratio(0.5d, + "Arrange to generate values that shrink to this fraction of\n" + + "\ttheir original size after compression.") { + @Override public Object parseValue(String value) { + return Double.parseDouble(value); + } + }, + use_existing_db(false, + "If true, do not destroy the existing database. If you set this\n" + + "\tflag and also specify a benchmark that wants a fresh database,\n" + + "\tthat benchmark will fail.") { + @Override public Object parseValue(String value) { + return parseBoolean(value); + } + }, + num(1000000, + "Number of key/values to place in database.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + threads(1, + "Number of concurrent threads to run.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + reads(null, + "Number of read operations to do. If negative, do --nums reads.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + key_size(16, + "The size of each key in bytes.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + value_size(100, + "The size of each value in bytes.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + write_buffer_size(4L * SizeUnit.MB, + "Number of bytes to buffer in memtable before compacting\n" + + "\t(initialized to default value by 'main'.)") { + @Override public Object parseValue(String value) { + return Long.parseLong(value); + } + }, + max_write_buffer_number(2, + "The number of in-memory memtables. Each memtable is of size\n" + + "\twrite_buffer_size.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + prefix_size(0, "Controls the prefix size for HashSkipList, HashLinkedList,\n" + + "\tand plain table.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + keys_per_prefix(0, "Controls the average number of keys generated\n" + + "\tper prefix, 0 means no special handling of the prefix,\n" + + "\ti.e. use the prefix comes with the generated random number.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + memtablerep("skip_list", + "The memtable format. Available options are\n" + + "\tskip_list,\n" + + "\tvector,\n" + + "\thash_linkedlist,\n" + + "\thash_skiplist (prefix_hash.)") { + @Override public Object parseValue(String value) { + return value; + } + }, + hash_bucket_count(SizeUnit.MB, + "The number of hash buckets used in the hash-bucket-based\n" + + "\tmemtables. Memtables that currently support this argument are\n" + + "\thash_linkedlist and hash_skiplist.") { + @Override public Object parseValue(String value) { + return Long.parseLong(value); + } + }, + writes_per_second(10000, + "The write-rate of the background writer used in the\n" + + "\t`readwhilewriting` benchmark. Non-positive number indicates\n" + + "\tusing an unbounded write-rate in `readwhilewriting` benchmark.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + use_plain_table(false, + "Use plain-table sst format.") { + @Override public Object parseValue(String value) { + return parseBoolean(value); + } + }, + cache_size(-1L, + "Number of bytes to use as a cache of uncompressed data.\n" + + "\tNegative means use default settings.") { + @Override public Object parseValue(String value) { + return Long.parseLong(value); + } + }, + seed(0L, + "Seed base for random number generators.") { + @Override public Object parseValue(String value) { + return Long.parseLong(value); + } + }, + num_levels(7, + "The total number of levels.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + numdistinct(1000L, + "Number of distinct keys to use. Used in RandomWithVerify to\n" + + "\tread/write on fewer keys so that gets are more likely to find the\n" + + "\tkey and puts are more likely to update the same key.") { + @Override public Object parseValue(String value) { + return Long.parseLong(value); + } + }, + merge_keys(-1L, + "Number of distinct keys to use for MergeRandom and\n" + + "\tReadRandomMergeRandom.\n" + + "\tIf negative, there will be FLAGS_num keys.") { + @Override public Object parseValue(String value) { + return Long.parseLong(value); + } + }, + bloom_locality(0,"Control bloom filter probes locality.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + duration(0,"Time in seconds for the random-ops tests to run.\n" + + "\tWhen 0 then num & reads determine the test duration.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + num_multi_db(0, + "Number of DBs used in the benchmark. 0 means single DB.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + histogram(false,"Print histogram of operation timings.") { + @Override public Object parseValue(String value) { + return parseBoolean(value); + } + }, + min_write_buffer_number_to_merge( + defaultOptions_.minWriteBufferNumberToMerge(), + "The minimum number of write buffers that will be merged together\n" + + "\tbefore writing to storage. This is cheap because it is an\n" + + "\tin-memory merge. If this feature is not enabled, then all these\n" + + "\twrite buffers are flushed to L0 as separate files and this\n" + + "\tincreases read amplification because a get request has to check\n" + + "\tin all of these files. Also, an in-memory merge may result in\n" + + "\twriting less data to storage if there are duplicate records\n" + + "\tin each of these individual write buffers.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + max_background_compactions( + defaultOptions_.maxBackgroundCompactions(), + "The maximum number of concurrent background compactions\n" + + "\tthat can occur in parallel.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + max_background_flushes( + defaultOptions_.maxBackgroundFlushes(), + "The maximum number of concurrent background flushes\n" + + "\tthat can occur in parallel.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + max_background_jobs(defaultOptions_.maxBackgroundJobs(), + "The maximum number of concurrent background jobs\n" + + "\tthat can occur in parallel.") { + @Override + public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + /* TODO(yhchiang): enable the following + compaction_style((int32_t) defaultOptions_.compactionStyle(), + "style of compaction: level-based vs universal.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + },*/ + universal_size_ratio(0, + "Percentage flexibility while comparing file size\n" + + "\t(for universal compaction only).") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + universal_min_merge_width(0,"The minimum number of files in a\n" + + "\tsingle compaction run (for universal compaction only).") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + universal_max_merge_width(0,"The max number of files to compact\n" + + "\tin universal style compaction.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + universal_max_size_amplification_percent(0, + "The max size amplification for universal style compaction.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + universal_compression_size_percent(-1, + "The percentage of the database to compress for universal\n" + + "\tcompaction. -1 means compress everything.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + block_size(defaultBlockBasedTableOptions_.blockSize(), + "Number of bytes in a block.") { + @Override public Object parseValue(String value) { + return Long.parseLong(value); + } + }, + compressed_cache_size(-1L, + "Number of bytes to use as a cache of compressed data.") { + @Override public Object parseValue(String value) { + return Long.parseLong(value); + } + }, + open_files(defaultOptions_.maxOpenFiles(), + "Maximum number of files to keep open at the same time\n" + + "\t(use default if == 0)") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + bloom_bits(-1,"Bloom filter bits per key. Negative means\n" + + "\tuse default settings.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + memtable_bloom_size_ratio(0.0d, "Ratio of memtable used by the bloom filter.\n" + + "\t0 means no bloom filter.") { + @Override public Object parseValue(String value) { + return Double.parseDouble(value); + } + }, + memtable_whole_key_filtering(false, "Enable whole key bloom filter in memtable.") { + @Override + public Object parseValue(String value) { + return parseBoolean(value); + } + }, + cache_numshardbits(-1,"Number of shards for the block cache\n" + + "\tis 2 ** cache_numshardbits. Negative means use default settings.\n" + + "\tThis is applied only if FLAGS_cache_size is non-negative.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + verify_checksum(false,"Verify checksum for every block read\n" + + "\tfrom storage.") { + @Override public Object parseValue(String value) { + return parseBoolean(value); + } + }, + statistics(false,"Database statistics.") { + @Override public Object parseValue(String value) { + return parseBoolean(value); + } + }, + writes(-1L, "Number of write operations to do. If negative, do\n" + + "\t--num reads.") { + @Override public Object parseValue(String value) { + return Long.parseLong(value); + } + }, + sync(false,"Sync all writes to disk.") { + @Override public Object parseValue(String value) { + return parseBoolean(value); + } + }, + use_fsync(false,"If true, issue fsync instead of fdatasync.") { + @Override public Object parseValue(String value) { + return parseBoolean(value); + } + }, + disable_wal(false,"If true, do not write WAL for write.") { + @Override public Object parseValue(String value) { + return parseBoolean(value); + } + }, + wal_dir("", "If not empty, use the given dir for WAL.") { + @Override public Object parseValue(String value) { + return value; + } + }, + target_file_size_base(2 * 1048576,"Target file size at level-1") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + target_file_size_multiplier(1, + "A multiplier to compute target level-N file size (N >= 2)") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + max_bytes_for_level_base(10 * 1048576, + "Max bytes for level-1") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + max_bytes_for_level_multiplier(10.0d, + "A multiplier to compute max bytes for level-N (N >= 2)") { + @Override public Object parseValue(String value) { + return Double.parseDouble(value); + } + }, + level0_stop_writes_trigger(12,"Number of files in level-0\n" + + "\tthat will trigger put stop.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + level0_slowdown_writes_trigger(8,"Number of files in level-0\n" + + "\tthat will slow down writes.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + level0_file_num_compaction_trigger(4,"Number of files in level-0\n" + + "\twhen compactions start.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + readwritepercent(90,"Ratio of reads to reads/writes (expressed\n" + + "\tas percentage) for the ReadRandomWriteRandom workload. The\n" + + "\tdefault value 90 means 90% operations out of all reads and writes\n" + + "\toperations are reads. In other words, 9 gets for every 1 put.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + mergereadpercent(70,"Ratio of merges to merges&reads (expressed\n" + + "\tas percentage) for the ReadRandomMergeRandom workload. The\n" + + "\tdefault value 70 means 70% out of all read and merge operations\n" + + "\tare merges. In other words, 7 merges for every 3 gets.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + deletepercent(2,"Percentage of deletes out of reads/writes/\n" + + "\tdeletes (used in RandomWithVerify only). RandomWithVerify\n" + + "\tcalculates writepercent as (100 - FLAGS_readwritepercent -\n" + + "\tdeletepercent), so deletepercent must be smaller than (100 -\n" + + "\tFLAGS_readwritepercent)") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + delete_obsolete_files_period_micros(0,"Option to delete\n" + + "\tobsolete files periodically. 0 means that obsolete files are\n" + + "\tdeleted after every compaction run.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + compression_type("snappy", + "Algorithm used to compress the database.") { + @Override public Object parseValue(String value) { + return value; + } + }, + compression_level(-1, + "Compression level. For zlib this should be -1 for the\n" + + "\tdefault level, or between 0 and 9.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + min_level_to_compress(-1,"If non-negative, compression starts\n" + + "\tfrom this level. Levels with number < min_level_to_compress are\n" + + "\tnot compressed. Otherwise, apply compression_type to\n" + + "\tall levels.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + table_cache_numshardbits(4,"") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + stats_interval(0L, "Stats are reported every N operations when\n" + + "\tthis is greater than zero. When 0 the interval grows over time.") { + @Override public Object parseValue(String value) { + return Long.parseLong(value); + } + }, + stats_per_interval(0,"Reports additional stats per interval when\n" + + "\tthis is greater than 0.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + perf_level(0,"Level of perf collection.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + max_compaction_bytes(0L, "Limit number of bytes in one compaction to be lower than this\n" + + "\threshold. But it's not guaranteed.") { + @Override public Object parseValue(String value) { + return Long.parseLong(value); + } + }, + readonly(false,"Run read only benchmarks.") { + @Override public Object parseValue(String value) { + return parseBoolean(value); + } + }, + disable_auto_compactions(false,"Do not auto trigger compactions.") { + @Override public Object parseValue(String value) { + return parseBoolean(value); + } + }, + wal_ttl_seconds(0L,"Set the TTL for the WAL Files in seconds.") { + @Override public Object parseValue(String value) { + return Long.parseLong(value); + } + }, + wal_size_limit_MB(0L,"Set the size limit for the WAL Files\n" + + "\tin MB.") { + @Override public Object parseValue(String value) { + return Long.parseLong(value); + } + }, + /* TODO(yhchiang): enable the following + direct_reads(rocksdb::EnvOptions().use_direct_reads, + "Allow direct I/O reads.") { + @Override public Object parseValue(String value) { + return parseBoolean(value); + } + }, + direct_writes(rocksdb::EnvOptions().use_direct_reads, + "Allow direct I/O reads.") { + @Override public Object parseValue(String value) { + return parseBoolean(value); + } + }, + */ + mmap_read(false, + "Allow reads to occur via mmap-ing files.") { + @Override public Object parseValue(String value) { + return parseBoolean(value); + } + }, + mmap_write(false, + "Allow writes to occur via mmap-ing files.") { + @Override public Object parseValue(String value) { + return parseBoolean(value); + } + }, + advise_random_on_open(defaultOptions_.adviseRandomOnOpen(), + "Advise random access on table file open.") { + @Override public Object parseValue(String value) { + return parseBoolean(value); + } + }, + compaction_fadvice("NORMAL", + "Access pattern advice when a file is compacted.") { + @Override public Object parseValue(String value) { + return value; + } + }, + use_tailing_iterator(false, + "Use tailing iterator to access a series of keys instead of get.") { + @Override public Object parseValue(String value) { + return parseBoolean(value); + } + }, + use_adaptive_mutex(defaultOptions_.useAdaptiveMutex(), + "Use adaptive mutex.") { + @Override public Object parseValue(String value) { + return parseBoolean(value); + } + }, + bytes_per_sync(defaultOptions_.bytesPerSync(), + "Allows OS to incrementally sync files to disk while they are\n" + + "\tbeing written, in the background. Issue one request for every\n" + + "\tbytes_per_sync written. 0 turns it off.") { + @Override public Object parseValue(String value) { + return Long.parseLong(value); + } + }, + filter_deletes(false," On true, deletes use bloom-filter and drop\n" + + "\tthe delete if key not present.") { + @Override public Object parseValue(String value) { + return parseBoolean(value); + } + }, + max_successive_merges(0,"Maximum number of successive merge\n" + + "\toperations on a key in the memtable.") { + @Override public Object parseValue(String value) { + return Integer.parseInt(value); + } + }, + db(getTempDir("rocksdb-jni"), + "Use the db with the following name.") { + @Override public Object parseValue(String value) { + return value; + } + }, + use_mem_env(false, "Use RocksMemEnv instead of default filesystem based\n" + + "environment.") { + @Override public Object parseValue(String value) { + return parseBoolean(value); + } + }, + java_comparator(null, "Class name of a Java Comparator to use instead\n" + + "\tof the default C++ ByteWiseComparatorImpl. Must be available on\n" + + "\tthe classpath") { + @Override + protected Object parseValue(final String value) { + try { + final ComparatorOptions copt = new ComparatorOptions(); + final Class clsComparator = + (Class)Class.forName(value); + final Constructor cstr = + clsComparator.getConstructor(ComparatorOptions.class); + return cstr.newInstance(copt); + } catch(final ClassNotFoundException cnfe) { + throw new IllegalArgumentException("Java Comparator '" + value + "'" + + " not found on the classpath", cnfe); + } catch(final NoSuchMethodException nsme) { + throw new IllegalArgumentException("Java Comparator '" + value + "'" + + " does not have a public ComparatorOptions constructor", nsme); + } catch(final IllegalAccessException | InstantiationException + | InvocationTargetException ie) { + throw new IllegalArgumentException("Unable to construct Java" + + " Comparator '" + value + "'", ie); + } + } + }; + + private Flag(Object defaultValue, String desc) { + defaultValue_ = defaultValue; + desc_ = desc; + } + + public Object getDefaultValue() { + return defaultValue_; + } + + public String desc() { + return desc_; + } + + public boolean parseBoolean(String value) { + if (value.equals("1")) { + return true; + } else if (value.equals("0")) { + return false; + } + return Boolean.parseBoolean(value); + } + + protected abstract Object parseValue(String value); + + private final Object defaultValue_; + private final String desc_; + } + + private final static String DEFAULT_TEMP_DIR = "/tmp"; + + private static String getTempDir(final String dirName) { + try { + return Files.createTempDirectory(dirName).toAbsolutePath().toString(); + } catch(final IOException ioe) { + System.err.println("Unable to create temp directory, defaulting to: " + + DEFAULT_TEMP_DIR); + return DEFAULT_TEMP_DIR + File.pathSeparator + dirName; + } + } + + private static class RandomGenerator { + private final byte[] data_; + private int dataLength_; + private int position_; + private double compressionRatio_; + Random rand_; + + private RandomGenerator(long seed, double compressionRatio) { + // We use a limited amount of data over and over again and ensure + // that it is larger than the compression window (32KB), and also + byte[] value = new byte[100]; + // large enough to serve all typical value sizes we want to write. + rand_ = new Random(seed); + dataLength_ = value.length * 10000; + data_ = new byte[dataLength_]; + compressionRatio_ = compressionRatio; + int pos = 0; + while (pos < dataLength_) { + compressibleBytes(value); + System.arraycopy(value, 0, data_, pos, + Math.min(value.length, dataLength_ - pos)); + pos += value.length; + } + } + + private void compressibleBytes(byte[] value) { + int baseLength = value.length; + if (compressionRatio_ < 1.0d) { + baseLength = (int) (compressionRatio_ * value.length + 0.5); + } + if (baseLength <= 0) { + baseLength = 1; + } + int pos; + for (pos = 0; pos < baseLength; ++pos) { + value[pos] = (byte) (' ' + rand_.nextInt(95)); // ' ' .. '~' + } + while (pos < value.length) { + System.arraycopy(value, 0, value, pos, + Math.min(baseLength, value.length - pos)); + pos += baseLength; + } + } + + private void generate(byte[] value) { + if (position_ + value.length > data_.length) { + position_ = 0; + assert(value.length <= data_.length); + } + position_ += value.length; + System.arraycopy(data_, position_ - value.length, + value, 0, value.length); + } + } + + boolean isFinished() { + synchronized(finishLock_) { + return isFinished_; + } + } + + void setFinished(boolean flag) { + synchronized(finishLock_) { + isFinished_ = flag; + } + } + + RocksDB db_; + final List benchmarks_; + final int num_; + final int reads_; + final int keySize_; + final int valueSize_; + final int threadNum_; + final int writesPerSeconds_; + final long randSeed_; + final boolean useExisting_; + final String databaseDir_; + double compressionRatio_; + RandomGenerator gen_; + long startTime_; + + // env + boolean useMemenv_; + + // memtable related + final int maxWriteBufferNumber_; + final int prefixSize_; + final int keysPerPrefix_; + final String memtable_; + final long hashBucketCount_; + + // sst format related + boolean usePlainTable_; + + Object finishLock_; + boolean isFinished_; + Map flags_; + // as the scope of a static member equals to the scope of the problem, + // we let its c++ pointer to be disposed in its finalizer. + static Options defaultOptions_ = new Options(); + static BlockBasedTableConfig defaultBlockBasedTableOptions_ = + new BlockBasedTableConfig(); + String compressionType_; + CompressionType compression_; +} diff --git a/librocksdb-sys/rocksdb/java/crossbuild/Vagrantfile b/librocksdb-sys/rocksdb/java/crossbuild/Vagrantfile new file mode 100644 index 0000000..0ee50de --- /dev/null +++ b/librocksdb-sys/rocksdb/java/crossbuild/Vagrantfile @@ -0,0 +1,51 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! +VAGRANTFILE_API_VERSION = "2" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + + config.vm.define "linux32" do |linux32| + linux32.vm.box = "bento/centos-6.10-i386" + linux32.vm.provision :shell, path: "build-linux-centos.sh" + end + + config.vm.define "linux64" do |linux64| + linux64.vm.box = "bento/centos-6.10" + linux64.vm.provision :shell, path: "build-linux-centos.sh" + end + + config.vm.define "linux32-musl" do |musl32| + musl32.vm.box = "alpine/alpine32" + musl32.vm.box_version = "3.6.0" + musl32.vm.provision :shell, path: "build-linux-alpine.sh" + end + + config.vm.define "linux64-musl" do |musl64| + musl64.vm.box = "generic/alpine36" + + ## Should use the alpine/alpine64 box, but this issue needs to be fixed first - https://github.com/hashicorp/vagrant/issues/11218 + # musl64.vm.box = "alpine/alpine64" + # musl64.vm.box_version = "3.6.0" + + musl64.vm.provision :shell, path: "build-linux-alpine.sh" + end + + config.vm.provider "virtualbox" do |v| + v.memory = 2048 + v.cpus = 4 + v.customize ["modifyvm", :id, "--nictype1", "virtio" ] + end + + if Vagrant.has_plugin?("vagrant-cachier") + config.cache.scope = :box + end + if Vagrant.has_plugin?("vagrant-vbguest") + config.vbguest.no_install = true + end + + config.vm.synced_folder "../target", "/rocksdb-build" + config.vm.synced_folder "../..", "/rocksdb", type: "rsync" + config.vm.boot_timeout = 1200 +end diff --git a/librocksdb-sys/rocksdb/java/crossbuild/build-linux-alpine.sh b/librocksdb-sys/rocksdb/java/crossbuild/build-linux-alpine.sh new file mode 100755 index 0000000..561d341 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/crossbuild/build-linux-alpine.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +set -e + +# update Alpine with latest versions +echo '@edge http://nl.alpinelinux.org/alpine/edge/main' >> /etc/apk/repositories +echo '@community http://nl.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories +apk update +apk upgrade + +# install CA certificates +apk add ca-certificates + +# install build tools +apk add \ + build-base \ + coreutils \ + file \ + git \ + perl \ + automake \ + autoconf \ + cmake + +# install tool dependencies for building RocksDB static library +apk add \ + curl \ + bash \ + wget \ + tar \ + openssl + +# install RocksDB dependencies +apk add \ + snappy snappy-dev \ + zlib zlib-dev \ + bzip2 bzip2-dev \ + lz4 lz4-dev \ + zstd zstd-dev \ + linux-headers \ + jemalloc jemalloc-dev + +# install OpenJDK7 +apk add openjdk7 \ + && apk add java-cacerts \ + && rm /usr/lib/jvm/java-1.7-openjdk/jre/lib/security/cacerts \ + && ln -s /etc/ssl/certs/java/cacerts /usr/lib/jvm/java-1.7-openjdk/jre/lib/security/cacerts + +# cleanup +rm -rf /var/cache/apk/* + +# puts javac in the PATH +export JAVA_HOME=/usr/lib/jvm/java-1.7-openjdk +export PATH=/usr/lib/jvm/java-1.7-openjdk/bin:$PATH + +# gflags from source +cd /tmp &&\ + git clone -b v2.0 --single-branch https://github.com/gflags/gflags.git &&\ + cd gflags &&\ + ./configure --prefix=/usr && make && make install &&\ + rm -rf /tmp/* + + +# build rocksdb +cd /rocksdb +make jclean clean +PORTABLE=1 make -j8 rocksdbjavastatic +cp /rocksdb/java/target/librocksdbjni-* /rocksdb-build +cp /rocksdb/java/target/rocksdbjni-* /rocksdb-build diff --git a/librocksdb-sys/rocksdb/java/crossbuild/build-linux-centos.sh b/librocksdb-sys/rocksdb/java/crossbuild/build-linux-centos.sh new file mode 100755 index 0000000..176e345 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/crossbuild/build-linux-centos.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +set -e + +# remove fixed relesever variable present in the hanscode boxes +sudo rm -f /etc/yum/vars/releasever + +# enable EPEL +sudo yum -y install epel-release + +# install all required packages for rocksdb that are available through yum +sudo yum -y install openssl java-1.7.0-openjdk-devel zlib-devel bzip2-devel lz4-devel snappy-devel libzstd-devel jemalloc-devel cmake3 + +# set up cmake3 as cmake binary +sudo alternatives --install /usr/local/bin/cmake cmake /usr/bin/cmake 10 --slave /usr/local/bin/ctest ctest /usr/bin/ctest --slave /usr/local/bin/cpack cpack /usr/bin/cpack --slave /usr/local/bin/ccmake ccmake /usr/bin/ccmake +sudo alternatives --install /usr/local/bin/cmake cmake /usr/bin/cmake3 20 --slave /usr/local/bin/ctest ctest /usr/bin/ctest3 --slave /usr/local/bin/cpack cpack /usr/bin/cpack3 --slave /usr/local/bin/ccmake ccmake /usr/bin/ccmake3 + +# install gcc/g++ 4.8.2 from tru/devtools-2 +sudo wget -O /etc/yum.repos.d/devtools-2.repo https://people.centos.org/tru/devtools-2/devtools-2.repo +sudo yum -y install devtoolset-2-binutils devtoolset-2-gcc devtoolset-2-gcc-c++ + +# install gflags +wget https://github.com/gflags/gflags/archive/v2.0.tar.gz -O gflags-2.0.tar.gz +tar xvfz gflags-2.0.tar.gz; cd gflags-2.0; scl enable devtoolset-2 ./configure; scl enable devtoolset-2 make; sudo make install +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib + +# set java home so we can build rocksdb jars +export JAVA_HOME=/usr/lib/jvm/java-1.7.0 + +export PATH=$JAVA_HOME:/usr/local/bin:$PATH + +# build rocksdb +cd /rocksdb +scl enable devtoolset-2 'make clean-not-downloaded' +scl enable devtoolset-2 'PORTABLE=1 make -j8 rocksdbjavastatic' +cp /rocksdb/java/target/librocksdbjni-* /rocksdb-build +cp /rocksdb/java/target/rocksdbjni-* /rocksdb-build diff --git a/librocksdb-sys/rocksdb/java/crossbuild/build-linux.sh b/librocksdb-sys/rocksdb/java/crossbuild/build-linux.sh new file mode 100755 index 0000000..74178ad --- /dev/null +++ b/librocksdb-sys/rocksdb/java/crossbuild/build-linux.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# install all required packages for rocksdb +sudo apt-get update +sudo apt-get -y install git make gcc g++ libgflags-dev libsnappy-dev zlib1g-dev libbz2-dev default-jdk + +# set java home so we can build rocksdb jars +export JAVA_HOME=$(echo /usr/lib/jvm/java-7-openjdk*) +cd /rocksdb +make jclean clean +make -j 4 rocksdbjavastatic +cp /rocksdb/java/target/librocksdbjni-* /rocksdb-build +cp /rocksdb/java/target/rocksdbjni-* /rocksdb-build +sudo shutdown -h now + diff --git a/librocksdb-sys/rocksdb/java/crossbuild/docker-build-linux-alpine.sh b/librocksdb-sys/rocksdb/java/crossbuild/docker-build-linux-alpine.sh new file mode 100755 index 0000000..e3e852e --- /dev/null +++ b/librocksdb-sys/rocksdb/java/crossbuild/docker-build-linux-alpine.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +set -e +#set -x + +# just in-case this is run outside Docker +mkdir -p /rocksdb-local-build + +rm -rf /rocksdb-local-build/* +cp -r /rocksdb-host/* /rocksdb-local-build +cd /rocksdb-local-build + +make clean-not-downloaded +PORTABLE=1 make -j2 rocksdbjavastatic + +cp java/target/librocksdbjni-linux*.so java/target/rocksdbjni-*-linux*.jar java/target/rocksdbjni-*-linux*.jar.sha1 /rocksdb-java-target diff --git a/librocksdb-sys/rocksdb/java/crossbuild/docker-build-linux-centos.sh b/librocksdb-sys/rocksdb/java/crossbuild/docker-build-linux-centos.sh new file mode 100755 index 0000000..16581de --- /dev/null +++ b/librocksdb-sys/rocksdb/java/crossbuild/docker-build-linux-centos.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +set -e +#set -x + +# just in-case this is run outside Docker +mkdir -p /rocksdb-local-build + +rm -rf /rocksdb-local-build/* +cp -r /rocksdb-host/* /rocksdb-local-build +cd /rocksdb-local-build + +# Use scl devtoolset if available +if hash scl 2>/dev/null; then + if scl --list | grep -q 'devtoolset-8'; then + # CentOS 6+ + scl enable devtoolset-8 'make clean-not-downloaded' + scl enable devtoolset-8 'PORTABLE=1 make -j2 rocksdbjavastatic' + elif scl --list | grep -q 'devtoolset-7'; then + # CentOS 6+ + scl enable devtoolset-7 'make clean-not-downloaded' + scl enable devtoolset-7 'PORTABLE=1 make -j2 rocksdbjavastatic' + elif scl --list | grep -q 'devtoolset-2'; then + # CentOS 5 or 6 + scl enable devtoolset-2 'make clean-not-downloaded' + scl enable devtoolset-2 'PORTABLE=1 make -j2 rocksdbjavastatic' + else + echo "Could not find devtoolset" + exit 1; + fi +else + make clean-not-downloaded + PORTABLE=1 make -j2 rocksdbjavastatic +fi + +cp java/target/librocksdbjni-linux*.so java/target/rocksdbjni-*-linux*.jar java/target/rocksdbjni-*-linux*.jar.sha1 /rocksdb-java-target + diff --git a/librocksdb-sys/rocksdb/java/jdb_bench.sh b/librocksdb-sys/rocksdb/java/jdb_bench.sh new file mode 100755 index 0000000..5dfc385 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/jdb_bench.sh @@ -0,0 +1,13 @@ +# shellcheck disable=SC2148 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +PLATFORM=64 +if [ `getconf LONG_BIT` != "64" ] +then + PLATFORM=32 +fi + +ROCKS_JAR=`find target -name rocksdbjni*.jar` + +echo "Running benchmark in $PLATFORM-Bit mode." +# shellcheck disable=SC2068 +java -server -d$PLATFORM -XX:NewSize=4m -XX:+AggressiveOpts -Djava.library.path=target -cp "${ROCKS_JAR}:benchmark/target/classes" org.rocksdb.benchmark.DbBenchmark $@ diff --git a/librocksdb-sys/rocksdb/java/jmh/LICENSE-HEADER.txt b/librocksdb-sys/rocksdb/java/jmh/LICENSE-HEADER.txt new file mode 100644 index 0000000..365ee65 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/jmh/LICENSE-HEADER.txt @@ -0,0 +1,5 @@ +Copyright (c) 2011-present, Facebook, Inc. All rights reserved. + This source code is licensed under both the GPLv2 (found in the + COPYING file in the root directory) and Apache 2.0 License + (found in the LICENSE.Apache file in the root directory). + diff --git a/librocksdb-sys/rocksdb/java/jmh/README.md b/librocksdb-sys/rocksdb/java/jmh/README.md new file mode 100644 index 0000000..1575ab5 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/jmh/README.md @@ -0,0 +1,24 @@ +# JMH Benchmarks for RocksJava + +These are micro-benchmarks for RocksJava functionality, using [JMH (Java Microbenchmark Harness)](https://openjdk.java.net/projects/code-tools/jmh/). + +## Compiling + +**Note**: This uses a specific build of RocksDB that is set in the `` element of the `dependencies` section of the `pom.xml` file. If you are testing local changes you should build and install a SNAPSHOT version of rocksdbjni, and update the `pom.xml` of rocksdbjni-jmh file to test with this. + +For instance, this is how to install the OSX jar you just built for 6.26.0 + +```bash +$ mvn install:install-file -Dfile=./java/target/rocksdbjni-6.26.0-SNAPSHOT-osx.jar -DgroupId=org.rocksdb -DartifactId=rocksdbjni -Dversion=6.26.0-SNAPSHOT -Dpackaging=jar +``` + +```bash +$ mvn package +``` + +## Running +```bash +$ java -jar target/rocksdbjni-jmh-1.0-SNAPSHOT-benchmarks.jar +``` + +NOTE: you can append `-help` to the command above to see all of the JMH runtime options. diff --git a/librocksdb-sys/rocksdb/java/jmh/pom.xml b/librocksdb-sys/rocksdb/java/jmh/pom.xml new file mode 100644 index 0000000..3016aef --- /dev/null +++ b/librocksdb-sys/rocksdb/java/jmh/pom.xml @@ -0,0 +1,138 @@ + + + 4.0.0 + + org.rocksdb + rocksdbjni-jmh + 1.0-SNAPSHOT + + http://rocksdb.org/ + + rocksdbjni-jmh + JMH Benchmarks for RocksDB Java API + + + Facebook, Inc. + https://www.facebook.com + + + + + Apache License 2.0 + http://www.apache.org/licenses/LICENSE-2.0.html + repo + + + GNU General Public License, version 2 + http://www.gnu.org/licenses/gpl-2.0.html + repo + + + + + scm:git:git://github.com/facebook/rocksdb.git + scm:git:git@github.com:facebook/rocksdb.git + http://github.com/facebook/rocksdb/ + + + + 1.7 + 1.7 + UTF-8 + + 1.22 + benchmarks + + + + + org.rocksdb + rocksdbjni + 7.9.0-SNAPSHOT + + + + org.openjdk.jmh + jmh-core + ${jmh.version} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + provided + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + ${project.build.source} + ${project.build.target} + ${project.build.sourceEncoding} + + + + + com.mycila + license-maven-plugin + 3.0 + true + +
    LICENSE-HEADER.txt
    + true + true + true + + pom.xml + + ${project.build.sourceEncoding} +     +     + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.1 + + + package + + shade + + + ${project.artifactId}-${project.version}-${uberjar.name} + + + org.openjdk.jmh.Main + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + +     +     + +     \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/java/jmh/src/main/java/org/rocksdb/jmh/ComparatorBenchmarks.java b/librocksdb-sys/rocksdb/java/jmh/src/main/java/org/rocksdb/jmh/ComparatorBenchmarks.java new file mode 100644 index 0000000..1973b54 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/jmh/src/main/java/org/rocksdb/jmh/ComparatorBenchmarks.java @@ -0,0 +1,139 @@ +/** + * Copyright (c) 2011-present, Facebook, Inc. All rights reserved. + * This source code is licensed under both the GPLv2 (found in the + * COPYING file in the root directory) and Apache 2.0 License + * (found in the LICENSE.Apache file in the root directory). + */ +package org.rocksdb.jmh; + +import org.openjdk.jmh.annotations.*; +import org.rocksdb.*; +import org.rocksdb.util.BytewiseComparator; +import org.rocksdb.util.FileUtils; +import org.rocksdb.util.ReverseBytewiseComparator; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.rocksdb.util.KVUtils.ba; + +@State(Scope.Benchmark) +public class ComparatorBenchmarks { + + @Param({ + "native_bytewise", + "native_reverse_bytewise", + + "java_bytewise_non-direct_reused-64_adaptive-mutex", + "java_bytewise_non-direct_reused-64_non-adaptive-mutex", + "java_bytewise_non-direct_reused-64_thread-local", + "java_bytewise_direct_reused-64_adaptive-mutex", + "java_bytewise_direct_reused-64_non-adaptive-mutex", + "java_bytewise_direct_reused-64_thread-local", + "java_bytewise_non-direct_no-reuse", + "java_bytewise_direct_no-reuse", + + "java_reverse_bytewise_non-direct_reused-64_adaptive-mutex", + "java_reverse_bytewise_non-direct_reused-64_non-adaptive-mutex", + "java_reverse_bytewise_non-direct_reused-64_thread-local", + "java_reverse_bytewise_direct_reused-64_adaptive-mutex", + "java_reverse_bytewise_direct_reused-64_non-adaptive-mutex", + "java_reverse_bytewise_direct_reused-64_thread-local", + "java_reverse_bytewise_non-direct_no-reuse", + "java_reverse_bytewise_direct_no-reuse" + }) + public String comparatorName; + + Path dbDir; + ComparatorOptions comparatorOptions; + AbstractComparator comparator; + Options options; + RocksDB db; + + @Setup(Level.Trial) + public void setup() throws IOException, RocksDBException { + RocksDB.loadLibrary(); + + dbDir = Files.createTempDirectory("rocksjava-comparator-benchmarks"); + + options = new Options() + .setCreateIfMissing(true); + + if ("native_bytewise".equals(comparatorName)) { + options.setComparator(BuiltinComparator.BYTEWISE_COMPARATOR); + + } else if ("native_reverse_bytewise".equals(comparatorName)) { + options.setComparator(BuiltinComparator.REVERSE_BYTEWISE_COMPARATOR); + + } else if (comparatorName.startsWith("java_")) { + comparatorOptions = new ComparatorOptions(); + + if (comparatorName.indexOf("non-direct") > -1) { + comparatorOptions.setUseDirectBuffer(false); + } else if (comparatorName.indexOf("direct") > -1) { + comparatorOptions.setUseDirectBuffer(true); + } + + if (comparatorName.indexOf("no-reuse") > -1) { + comparatorOptions.setMaxReusedBufferSize(-1); + } else if (comparatorName.indexOf("_reused-") > -1) { + final int idx = comparatorName.indexOf("_reused-"); + String s = comparatorName.substring(idx + 8); + s = s.substring(0, s.indexOf('_')); + comparatorOptions.setMaxReusedBufferSize(Integer.parseInt(s)); + } + + if (comparatorName.indexOf("non-adaptive-mutex") > -1) { + comparatorOptions.setReusedSynchronisationType(ReusedSynchronisationType.MUTEX); + } else if (comparatorName.indexOf("adaptive-mutex") > -1) { + comparatorOptions.setReusedSynchronisationType(ReusedSynchronisationType.ADAPTIVE_MUTEX); + } else if (comparatorName.indexOf("thread-local") > -1) { + comparatorOptions.setReusedSynchronisationType(ReusedSynchronisationType.THREAD_LOCAL); + } + + if (comparatorName.startsWith("java_bytewise")) { + comparator = new BytewiseComparator(comparatorOptions); + } else if (comparatorName.startsWith("java_reverse_bytewise")) { + comparator = new ReverseBytewiseComparator(comparatorOptions); + } + + options.setComparator(comparator); + + } else { + throw new IllegalArgumentException("Unknown comparatorName: " + comparatorName); + } + + db = RocksDB.open(options, dbDir.toAbsolutePath().toString()); + } + + @TearDown(Level.Trial) + public void cleanup() throws IOException { + db.close(); + if (comparator != null) { + comparator.close(); + } + if (comparatorOptions != null) { + comparatorOptions.close(); + } + options.close(); + FileUtils.delete(dbDir); + } + + @State(Scope.Benchmark) + public static class Counter { + private final AtomicInteger count = new AtomicInteger(); + + public int next() { + return count.getAndIncrement(); + } + } + + + @Benchmark + public void put(final Counter counter) throws RocksDBException { + final int i = counter.next(); + db.put(ba("key" + i), ba("value" + i)); + } +} diff --git a/librocksdb-sys/rocksdb/java/jmh/src/main/java/org/rocksdb/jmh/GetBenchmarks.java b/librocksdb-sys/rocksdb/java/jmh/src/main/java/org/rocksdb/jmh/GetBenchmarks.java new file mode 100644 index 0000000..1c4329b --- /dev/null +++ b/librocksdb-sys/rocksdb/java/jmh/src/main/java/org/rocksdb/jmh/GetBenchmarks.java @@ -0,0 +1,215 @@ +/** + * Copyright (c) 2011-present, Facebook, Inc. All rights reserved. + * This source code is licensed under both the GPLv2 (found in the + * COPYING file in the root directory) and Apache 2.0 License + * (found in the LICENSE.Apache file in the root directory). + */ +package org.rocksdb.jmh; + +import static org.rocksdb.util.KVUtils.ba; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.openjdk.jmh.annotations.*; +import org.rocksdb.*; +import org.rocksdb.util.FileUtils; + +@State(Scope.Benchmark) +public class GetBenchmarks { + + @Param({ + "no_column_family", + "1_column_family", + "20_column_families", + "100_column_families" + }) + String columnFamilyTestType; + + @Param({"1000", "100000"}) int keyCount; + + @Param({"12", "64", "128"}) int keySize; + + @Param({"64", "1024", "65536"}) int valueSize; + + Path dbDir; + DBOptions options; + ReadOptions readOptions; + int cfs = 0; // number of column families + private AtomicInteger cfHandlesIdx; + ColumnFamilyHandle[] cfHandles; + RocksDB db; + private final AtomicInteger keyIndex = new AtomicInteger(); + private ByteBuffer keyBuf; + private ByteBuffer valueBuf; + private byte[] keyArr; + private byte[] valueArr; + + @Setup(Level.Trial) + public void setup() throws IOException, RocksDBException { + RocksDB.loadLibrary(); + + dbDir = Files.createTempDirectory("rocksjava-get-benchmarks"); + + options = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + readOptions = new ReadOptions(); + + final List cfDescriptors = new ArrayList<>(); + cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY)); + + if ("1_column_family".equals(columnFamilyTestType)) { + cfs = 1; + } else if ("20_column_families".equals(columnFamilyTestType)) { + cfs = 20; + } else if ("100_column_families".equals(columnFamilyTestType)) { + cfs = 100; + } + + if (cfs > 0) { + cfHandlesIdx = new AtomicInteger(1); + for (int i = 1; i <= cfs; i++) { + cfDescriptors.add(new ColumnFamilyDescriptor(ba("cf" + i))); + } + } + + final List cfHandlesList = new ArrayList<>(cfDescriptors.size()); + db = RocksDB.open(options, dbDir.toAbsolutePath().toString(), cfDescriptors, cfHandlesList); + cfHandles = cfHandlesList.toArray(new ColumnFamilyHandle[0]); + + // store initial data for retrieving via get + keyArr = new byte[keySize]; + valueArr = new byte[valueSize]; + Arrays.fill(keyArr, (byte) 0x30); + Arrays.fill(valueArr, (byte) 0x30); + for (int i = 0; i <= cfs; i++) { + for (int j = 0; j < keyCount; j++) { + final byte[] keyPrefix = ba("key" + j); + final byte[] valuePrefix = ba("value" + j); + System.arraycopy(keyPrefix, 0, keyArr, 0, keyPrefix.length); + System.arraycopy(valuePrefix, 0, valueArr, 0, valuePrefix.length); + db.put(cfHandles[i], keyArr, valueArr); + } + } + + try (final FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + db.flush(flushOptions); + } + + keyBuf = ByteBuffer.allocateDirect(keySize); + valueBuf = ByteBuffer.allocateDirect(valueSize); + Arrays.fill(keyArr, (byte) 0x30); + Arrays.fill(valueArr, (byte) 0x30); + keyBuf.put(keyArr); + keyBuf.flip(); + valueBuf.put(valueArr); + valueBuf.flip(); + } + + @TearDown(Level.Trial) + public void cleanup() throws IOException { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + db.close(); + options.close(); + readOptions.close(); + FileUtils.delete(dbDir); + } + + private ColumnFamilyHandle getColumnFamily() { + if (cfs == 0) { + return cfHandles[0]; + } else if (cfs == 1) { + return cfHandles[1]; + } else { + int idx = cfHandlesIdx.getAndIncrement(); + if (idx > cfs) { + cfHandlesIdx.set(1); // doesn't ensure a perfect distribution, but it's ok + idx = 0; + } + return cfHandles[idx]; + } + } + + /** + * Takes the next position in the index. + */ + private int next() { + int idx; + int nextIdx; + while (true) { + idx = keyIndex.get(); + nextIdx = idx + 1; + if (nextIdx >= keyCount) { + nextIdx = 0; + } + + if (keyIndex.compareAndSet(idx, nextIdx)) { + break; + } + } + return idx; + } + + // String -> byte[] + private byte[] getKeyArr() { + final int MAX_LEN = 9; // key100000 + final int keyIdx = next(); + final byte[] keyPrefix = ba("key" + keyIdx); + System.arraycopy(keyPrefix, 0, keyArr, 0, keyPrefix.length); + Arrays.fill(keyArr, keyPrefix.length, MAX_LEN, (byte) 0x30); + return keyArr; + } + + // String -> ByteBuffer + private ByteBuffer getKeyBuf() { + final int MAX_LEN = 9; // key100000 + final int keyIdx = next(); + final String keyStr = "key" + keyIdx; + for (int i = 0; i < keyStr.length(); ++i) { + keyBuf.put(i, (byte) keyStr.charAt(i)); + } + for (int i = keyStr.length(); i < MAX_LEN; ++i) { + keyBuf.put(i, (byte) 0x30); + } + // Reset position for future reading + keyBuf.position(0); + return keyBuf; + } + + private byte[] getValueArr() { + return valueArr; + } + + private ByteBuffer getValueBuf() { + return valueBuf; + } + + @Benchmark + public void get() throws RocksDBException { + db.get(getColumnFamily(), getKeyArr()); + } + + @Benchmark + public void preallocatedGet() throws RocksDBException { + db.get(getColumnFamily(), getKeyArr(), getValueArr()); + } + + @Benchmark + public void preallocatedByteBufferGet() throws RocksDBException { + int res = db.get(getColumnFamily(), readOptions, getKeyBuf(), getValueBuf()); + // For testing correctness: + // assert res > 0; + // final byte[] ret = new byte[valueSize]; + // valueBuf.get(ret); + // System.out.println(str(ret)); + // valueBuf.flip(); + } +} \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/java/jmh/src/main/java/org/rocksdb/jmh/MultiGetBenchmarks.java b/librocksdb-sys/rocksdb/java/jmh/src/main/java/org/rocksdb/jmh/MultiGetBenchmarks.java new file mode 100644 index 0000000..d374477 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/jmh/src/main/java/org/rocksdb/jmh/MultiGetBenchmarks.java @@ -0,0 +1,214 @@ +/** + * Copyright (c) 2011-present, Facebook, Inc. All rights reserved. + * This source code is licensed under both the GPLv2 (found in the + * COPYING file in the root directory) and Apache 2.0 License + * (found in the LICENSE.Apache file in the root directory). + */ +package org.rocksdb.jmh; + +import static org.rocksdb.util.KVUtils.ba; +import static org.rocksdb.util.KVUtils.keys; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.rocksdb.*; +import org.rocksdb.util.FileUtils; + +@State(Scope.Thread) +public class MultiGetBenchmarks { + @Param({ + "no_column_family", + "1_column_family", + "20_column_families", + "100_column_families" + }) + String columnFamilyTestType; + + @Param({"10000", "25000", "100000"}) int keyCount; + + @Param({ + "10", + "100", + "1000", + "10000", + }) + int multiGetSize; + + @Param({"16", "64", "250", "1000", "4000", "16000"}) int valueSize; + @Param({"16"}) int keySize; // big enough + + Path dbDir; + DBOptions options; + int cfs = 0; // number of column families + private AtomicInteger cfHandlesIdx; + ColumnFamilyHandle[] cfHandles; + RocksDB db; + private final AtomicInteger keyIndex = new AtomicInteger(); + + @Setup(Level.Trial) + public void setup() throws IOException, RocksDBException { + RocksDB.loadLibrary(); + + dbDir = Files.createTempDirectory("rocksjava-multiget-benchmarks"); + + options = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + + final List cfDescriptors = new ArrayList<>(); + cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY)); + + if ("1_column_family".equals(columnFamilyTestType)) { + cfs = 1; + } else if ("20_column_families".equals(columnFamilyTestType)) { + cfs = 20; + } else if ("100_column_families".equals(columnFamilyTestType)) { + cfs = 100; + } + + if (cfs > 0) { + cfHandlesIdx = new AtomicInteger(1); + for (int i = 1; i <= cfs; i++) { + cfDescriptors.add(new ColumnFamilyDescriptor(ba("cf" + i))); + } + } + + final List cfHandlesList = new ArrayList<>(cfDescriptors.size()); + db = RocksDB.open(options, dbDir.toAbsolutePath().toString(), cfDescriptors, cfHandlesList); + cfHandles = cfHandlesList.toArray(new ColumnFamilyHandle[0]); + + // store initial data for retrieving via get + for (int i = 0; i < cfs; i++) { + for (int j = 0; j < keyCount; j++) { + final byte[] paddedValue = Arrays.copyOf(ba("value" + j), valueSize); + db.put(cfHandles[i], ba("key" + j), paddedValue); + } + } + + try (final FlushOptions flushOptions = new FlushOptions() + .setWaitForFlush(true)) { + db.flush(flushOptions); + } + } + + @TearDown(Level.Trial) + public void cleanup() throws IOException { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + db.close(); + options.close(); + FileUtils.delete(dbDir); + } + + private ColumnFamilyHandle getColumnFamily() { + if (cfs == 0) { + return cfHandles[0]; + } else if (cfs == 1) { + return cfHandles[1]; + } else { + int idx = cfHandlesIdx.getAndIncrement(); + if (idx > cfs) { + cfHandlesIdx.set(1); // doesn't ensure a perfect distribution, but it's ok + idx = 0; + } + return cfHandles[idx]; + } + } + + /** + * Reserves the next {@inc} positions in the index. + * + * @param inc the number by which to increment the index + * @param limit the limit for the index + * @return the index before {@code inc} is added + */ + private int next(final int inc, final int limit) { + int idx; + int nextIdx; + while (true) { + idx = keyIndex.get(); + nextIdx = idx + inc; + if (nextIdx >= limit) { + nextIdx = inc; + } + + if (keyIndex.compareAndSet(idx, nextIdx)) { + break; + } + } + + if (nextIdx >= limit) { + return -1; + } else { + return idx; + } + } + + ByteBuffer keysBuffer; + ByteBuffer valuesBuffer; + + List valueBuffersList; + List keyBuffersList; + + @Setup + public void allocateSliceBuffers() { + keysBuffer = ByteBuffer.allocateDirect(keyCount * valueSize); + valuesBuffer = ByteBuffer.allocateDirect(keyCount * valueSize); + valueBuffersList = new ArrayList<>(); + keyBuffersList = new ArrayList<>(); + for (int i = 0; i < keyCount; i++) { + valueBuffersList.add(valuesBuffer.slice()); + valuesBuffer.position(i * valueSize); + keyBuffersList.add(keysBuffer.slice()); + keysBuffer.position(i * keySize); + } + } + + @TearDown + public void freeSliceBuffers() { + valueBuffersList.clear(); + } + + @Benchmark + public List multiGet10() throws RocksDBException { + final int fromKeyIdx = next(multiGetSize, keyCount); + if (fromKeyIdx >= 0) { + final List keys = keys(fromKeyIdx, fromKeyIdx + multiGetSize); + final List valueResults = db.multiGetAsList(keys); + for (final byte[] result : valueResults) { + if (result.length != valueSize) + throw new RuntimeException("Test valueSize assumption wrong"); + } + } + return new ArrayList<>(); + } + + public static void main(final String[] args) throws RunnerException { + final org.openjdk.jmh.runner.options.Options opt = + new OptionsBuilder() + .include(MultiGetBenchmarks.class.getSimpleName()) + .forks(1) + .jvmArgs("-ea") + .warmupIterations(1) + .measurementIterations(2) + .forks(2) + .param("columnFamilyTestType=", "1_column_family") + .param("multiGetSize=", "10", "1000") + .param("keyCount=", "1000") + .output("jmh_output") + .build(); + + new Runner(opt).run(); + } +} diff --git a/librocksdb-sys/rocksdb/java/jmh/src/main/java/org/rocksdb/jmh/PutBenchmarks.java b/librocksdb-sys/rocksdb/java/jmh/src/main/java/org/rocksdb/jmh/PutBenchmarks.java new file mode 100644 index 0000000..5aae21c --- /dev/null +++ b/librocksdb-sys/rocksdb/java/jmh/src/main/java/org/rocksdb/jmh/PutBenchmarks.java @@ -0,0 +1,112 @@ +/** + * Copyright (c) 2011-present, Facebook, Inc. All rights reserved. + * This source code is licensed under both the GPLv2 (found in the + * COPYING file in the root directory) and Apache 2.0 License + * (found in the LICENSE.Apache file in the root directory). + */ +package org.rocksdb.jmh; + +import org.openjdk.jmh.annotations.*; +import org.rocksdb.*; +import org.rocksdb.util.FileUtils; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.rocksdb.util.KVUtils.ba; + +@State(Scope.Benchmark) +public class PutBenchmarks { + + @Param({ + "no_column_family", + "1_column_family", + "20_column_families", + "100_column_families" + }) + String columnFamilyTestType; + + Path dbDir; + DBOptions options; + int cfs = 0; // number of column families + private AtomicInteger cfHandlesIdx; + ColumnFamilyHandle[] cfHandles; + RocksDB db; + + @Setup(Level.Trial) + public void setup() throws IOException, RocksDBException { + RocksDB.loadLibrary(); + + dbDir = Files.createTempDirectory("rocksjava-put-benchmarks"); + + options = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + + final List cfDescriptors = new ArrayList<>(); + cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY)); + + if ("1_column_family".equals(columnFamilyTestType)) { + cfs = 1; + } else if ("20_column_families".equals(columnFamilyTestType)) { + cfs = 20; + } else if ("100_column_families".equals(columnFamilyTestType)) { + cfs = 100; + } + + if (cfs > 0) { + cfHandlesIdx = new AtomicInteger(1); + for (int i = 1; i <= cfs; i++) { + cfDescriptors.add(new ColumnFamilyDescriptor(ba("cf" + i))); + } + } + + final List cfHandlesList = new ArrayList<>(cfDescriptors.size()); + db = RocksDB.open(options, dbDir.toAbsolutePath().toString(), cfDescriptors, cfHandlesList); + cfHandles = cfHandlesList.toArray(new ColumnFamilyHandle[0]); + } + + @TearDown(Level.Trial) + public void cleanup() throws IOException { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + db.close(); + options.close(); + FileUtils.delete(dbDir); + } + + private ColumnFamilyHandle getColumnFamily() { + if (cfs == 0) { + return cfHandles[0]; + } else if (cfs == 1) { + return cfHandles[1]; + } else { + int idx = cfHandlesIdx.getAndIncrement(); + if (idx > cfs) { + cfHandlesIdx.set(1); // doesn't ensure a perfect distribution, but it's ok + idx = 0; + } + return cfHandles[idx]; + } + } + + @State(Scope.Benchmark) + public static class Counter { + private final AtomicInteger count = new AtomicInteger(); + + public int next() { + return count.getAndIncrement(); + } + } + + @Benchmark + public void put(final ComparatorBenchmarks.Counter counter) throws RocksDBException { + final int i = counter.next(); + db.put(getColumnFamily(), ba("key" + i), ba("value" + i)); + } +} diff --git a/librocksdb-sys/rocksdb/java/jmh/src/main/java/org/rocksdb/util/FileUtils.java b/librocksdb-sys/rocksdb/java/jmh/src/main/java/org/rocksdb/util/FileUtils.java new file mode 100644 index 0000000..63744a1 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/jmh/src/main/java/org/rocksdb/util/FileUtils.java @@ -0,0 +1,59 @@ +/** + * Copyright (c) 2011-present, Facebook, Inc. All rights reserved. + * This source code is licensed under both the GPLv2 (found in the + * COPYING file in the root directory) and Apache 2.0 License + * (found in the LICENSE.Apache file in the root directory). + */ +package org.rocksdb.util; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; + +public final class FileUtils { + private static final SimpleFileVisitor DELETE_DIR_VISITOR = new DeleteDirVisitor(); + + /** + * Deletes a path from the filesystem + * + * If the path is a directory its contents + * will be recursively deleted before it itself + * is deleted. + * + * Note that removal of a directory is not an atomic-operation + * and so if an error occurs during removal, some of the directories + * descendants may have already been removed + * + * @param path the path to delete. + * + * @throws IOException if an error occurs whilst removing a file or directory + */ + public static void delete(final Path path) throws IOException { + if (!Files.isDirectory(path)) { + Files.deleteIfExists(path); + } else { + Files.walkFileTree(path, DELETE_DIR_VISITOR); + } + } + + private static class DeleteDirVisitor extends SimpleFileVisitor { + @Override + public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { + Files.deleteIfExists(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException { + if (exc != null) { + throw exc; + } + + Files.deleteIfExists(dir); + return FileVisitResult.CONTINUE; + } + } +} diff --git a/librocksdb-sys/rocksdb/java/jmh/src/main/java/org/rocksdb/util/KVUtils.java b/librocksdb-sys/rocksdb/java/jmh/src/main/java/org/rocksdb/util/KVUtils.java new file mode 100644 index 0000000..5077291 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/jmh/src/main/java/org/rocksdb/util/KVUtils.java @@ -0,0 +1,72 @@ +/** + * Copyright (c) 2011-present, Facebook, Inc. All rights reserved. + * This source code is licensed under both the GPLv2 (found in the + * COPYING file in the root directory) and Apache 2.0 License + * (found in the LICENSE.Apache file in the root directory). + */ +package org.rocksdb.util; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +public final class KVUtils { + + /** + * Get a byte array from a string. + * + * Assumes UTF-8 encoding + * + * @param string the string + * + * @return the bytes. + */ + public static byte[] ba(final String string) { + return string.getBytes(UTF_8); + } + + /** + * Get a string from a byte array. + * + * Assumes UTF-8 encoding + * + * @param bytes the bytes + * + * @return the string. + */ + public static String str(final byte[] bytes) { + return new String(bytes, UTF_8); + } + + /** + * Get a list of keys where the keys are named key1..key1+N + * in the range of {@code from} to {@code to} i.e. keyFrom..keyTo. + * + * @param from the first key + * @param to the last key + * + * @return the array of keys + */ + public static List keys(final int from, final int to) { + final List keys = new ArrayList<>(to - from); + for (int i = from; i < to; i++) { + keys.add(ba("key" + i)); + } + return keys; + } + + public static List keys( + final List keyBuffers, final int from, final int to) { + final List keys = new ArrayList<>(to - from); + for (int i = from; i < to; i++) { + final ByteBuffer key = keyBuffers.get(i); + key.clear(); + key.put(ba("key" + i)); + key.flip(); + keys.add(key); + } + return keys; + } +} diff --git a/librocksdb-sys/rocksdb/java/pom.xml.template b/librocksdb-sys/rocksdb/java/pom.xml.template new file mode 100644 index 0000000..8a1981c --- /dev/null +++ b/librocksdb-sys/rocksdb/java/pom.xml.template @@ -0,0 +1,178 @@ + + + 4.0.0 + + org.rocksdb + rocksdbjni + ${ROCKSDB_JAVA_VERSION} + + RocksDB JNI + RocksDB fat jar that contains .so files for linux32 and linux64 (glibc and musl-libc), jnilib files + for Mac OSX, and a .dll for Windows x64. + + https://rocksdb.org + 2012 + + + + Apache License 2.0 + http://www.apache.org/licenses/LICENSE-2.0.html + repo + + + GNU General Public License, version 2 + http://www.gnu.org/licenses/gpl-2.0.html + repo + + + + + scm:git:https://github.com/facebook/rocksdb.git + scm:git:https://github.com/facebook/rocksdb.git + scm:git:https://github.com/facebook/rocksdb.git + + + + Facebook + https://www.facebook.com + + + + + Facebook + help@facebook.com + America/New_York + + architect + + + + + + + rocksdb - Google Groups + rocksdb-subscribe@googlegroups.com + rocksdb-unsubscribe@googlegroups.com + rocksdb@googlegroups.com + https://groups.google.com/forum/#!forum/rocksdb + + + + + 1.8 + 1.8 + UTF-8 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.2 + + ${project.build.source} + ${project.build.target} + ${project.build.sourceEncoding} + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.18.1 + + ${argLine} -ea -Xcheck:jni -Djava.library.path=${project.build.directory} + false + false + + ${project.build.directory}/* + + + + + org.jacoco + jacoco-maven-plugin + 0.7.2.201409121644 + + + + prepare-agent + + + + report + prepare-package + + report + + + + + + org.codehaus.gmaven + groovy-maven-plugin + 2.0 + + + process-classes + + execute + + + + Xenu + + + String fileContents = new File(project.basedir.absolutePath + '/../include/rocksdb/version.h').getText('UTF-8') + matcher = (fileContents =~ /(?s).*ROCKSDB_MAJOR ([0-9]+).*?/) + String major_version = matcher.getAt(0).getAt(1) + matcher = (fileContents =~ /(?s).*ROCKSDB_MINOR ([0-9]+).*?/) + String minor_version = matcher.getAt(0).getAt(1) + matcher = (fileContents =~ /(?s).*ROCKSDB_PATCH ([0-9]+).*?/) + String patch_version = matcher.getAt(0).getAt(1) + String version = String.format('%s.%s.%s', major_version, minor_version, patch_version) + // Set version to be used in pom.properties + project.version = version + // Set version to be set as jar name + project.build.finalName = project.artifactId + "-" + version + + + + + + + + + + + junit + junit + 4.13.1 + test + + + org.hamcrest + hamcrest + 2.2 + test + + + cglib + cglib + 3.3.0 + test + + + org.assertj + assertj-core + 2.9.0 + test + + + org.mockito + mockito-all + 1.10.19 + test + + + diff --git a/librocksdb-sys/rocksdb/java/rocksjni/backup_engine_options.cc b/librocksdb-sys/rocksdb/java/rocksjni/backup_engine_options.cc new file mode 100644 index 0000000..25bfb67 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/backup_engine_options.cc @@ -0,0 +1,365 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling c++ ROCKSDB_NAMESPACE::BackupEnginge and +// ROCKSDB_NAMESPACE::BackupEngineOptions methods from Java side. + +#include +#include +#include + +#include +#include + +#include "include/org_rocksdb_BackupEngineOptions.h" +#include "rocksdb/utilities/backup_engine.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +/////////////////////////////////////////////////////////////////////////// +// BackupDBOptions + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: newBackupEngineOptions + * Signature: (Ljava/lang/String;)J + */ +jlong Java_org_rocksdb_BackupEngineOptions_newBackupEngineOptions( + JNIEnv* env, jclass /*jcls*/, jstring jpath) { + const char* cpath = env->GetStringUTFChars(jpath, nullptr); + if (cpath == nullptr) { + // exception thrown: OutOfMemoryError + return 0; + } + auto* bopt = new ROCKSDB_NAMESPACE::BackupEngineOptions(cpath); + env->ReleaseStringUTFChars(jpath, cpath); + return GET_CPLUSPLUS_POINTER(bopt); +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: backupDir + * Signature: (J)Ljava/lang/String; + */ +jstring Java_org_rocksdb_BackupEngineOptions_backupDir(JNIEnv* env, + jobject /*jopt*/, + jlong jhandle) { + auto* bopt = + reinterpret_cast(jhandle); + return env->NewStringUTF(bopt->backup_dir.c_str()); +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: setBackupEnv + * Signature: (JJ)V + */ +void Java_org_rocksdb_BackupEngineOptions_setBackupEnv( + JNIEnv* /*env*/, jobject /*jopt*/, jlong jhandle, jlong jrocks_env_handle) { + auto* bopt = + reinterpret_cast(jhandle); + auto* rocks_env = + reinterpret_cast(jrocks_env_handle); + bopt->backup_env = rocks_env; +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: setShareTableFiles + * Signature: (JZ)V + */ +void Java_org_rocksdb_BackupEngineOptions_setShareTableFiles(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle, + jboolean flag) { + auto* bopt = + reinterpret_cast(jhandle); + bopt->share_table_files = flag; +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: shareTableFiles + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_BackupEngineOptions_shareTableFiles(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* bopt = + reinterpret_cast(jhandle); + return bopt->share_table_files; +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: setInfoLog + * Signature: (JJ)V + */ +void Java_org_rocksdb_BackupEngineOptions_setInfoLog(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle, + jlong /*jlogger_handle*/) { + auto* bopt = + reinterpret_cast(jhandle); + auto* sptr_logger = + reinterpret_cast*>( + jhandle); + bopt->info_log = sptr_logger->get(); +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: setSync + * Signature: (JZ)V + */ +void Java_org_rocksdb_BackupEngineOptions_setSync(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle, + jboolean flag) { + auto* bopt = + reinterpret_cast(jhandle); + bopt->sync = flag; +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: sync + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_BackupEngineOptions_sync(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* bopt = + reinterpret_cast(jhandle); + return bopt->sync; +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: setDestroyOldData + * Signature: (JZ)V + */ +void Java_org_rocksdb_BackupEngineOptions_setDestroyOldData(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle, + jboolean flag) { + auto* bopt = + reinterpret_cast(jhandle); + bopt->destroy_old_data = flag; +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: destroyOldData + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_BackupEngineOptions_destroyOldData(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* bopt = + reinterpret_cast(jhandle); + return bopt->destroy_old_data; +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: setBackupLogFiles + * Signature: (JZ)V + */ +void Java_org_rocksdb_BackupEngineOptions_setBackupLogFiles(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle, + jboolean flag) { + auto* bopt = + reinterpret_cast(jhandle); + bopt->backup_log_files = flag; +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: backupLogFiles + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_BackupEngineOptions_backupLogFiles(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* bopt = + reinterpret_cast(jhandle); + return bopt->backup_log_files; +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: setBackupRateLimit + * Signature: (JJ)V + */ +void Java_org_rocksdb_BackupEngineOptions_setBackupRateLimit( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, + jlong jbackup_rate_limit) { + auto* bopt = + reinterpret_cast(jhandle); + bopt->backup_rate_limit = jbackup_rate_limit; +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: backupRateLimit + * Signature: (J)J + */ +jlong Java_org_rocksdb_BackupEngineOptions_backupRateLimit(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* bopt = + reinterpret_cast(jhandle); + return bopt->backup_rate_limit; +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: setBackupRateLimiter + * Signature: (JJ)V + */ +void Java_org_rocksdb_BackupEngineOptions_setBackupRateLimiter( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, + jlong jrate_limiter_handle) { + auto* bopt = + reinterpret_cast(jhandle); + auto* sptr_rate_limiter = + reinterpret_cast*>( + jrate_limiter_handle); + bopt->backup_rate_limiter = *sptr_rate_limiter; +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: setRestoreRateLimit + * Signature: (JJ)V + */ +void Java_org_rocksdb_BackupEngineOptions_setRestoreRateLimit( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, + jlong jrestore_rate_limit) { + auto* bopt = + reinterpret_cast(jhandle); + bopt->restore_rate_limit = jrestore_rate_limit; +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: restoreRateLimit + * Signature: (J)J + */ +jlong Java_org_rocksdb_BackupEngineOptions_restoreRateLimit(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* bopt = + reinterpret_cast(jhandle); + return bopt->restore_rate_limit; +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: setRestoreRateLimiter + * Signature: (JJ)V + */ +void Java_org_rocksdb_BackupEngineOptions_setRestoreRateLimiter( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, + jlong jrate_limiter_handle) { + auto* bopt = + reinterpret_cast(jhandle); + auto* sptr_rate_limiter = + reinterpret_cast*>( + jrate_limiter_handle); + bopt->restore_rate_limiter = *sptr_rate_limiter; +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: setShareFilesWithChecksum + * Signature: (JZ)V + */ +void Java_org_rocksdb_BackupEngineOptions_setShareFilesWithChecksum( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jboolean flag) { + auto* bopt = + reinterpret_cast(jhandle); + bopt->share_files_with_checksum = flag; +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: shareFilesWithChecksum + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_BackupEngineOptions_shareFilesWithChecksum( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { + auto* bopt = + reinterpret_cast(jhandle); + return bopt->share_files_with_checksum; +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: setMaxBackgroundOperations + * Signature: (JI)V + */ +void Java_org_rocksdb_BackupEngineOptions_setMaxBackgroundOperations( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, + jint max_background_operations) { + auto* bopt = + reinterpret_cast(jhandle); + bopt->max_background_operations = static_cast(max_background_operations); +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: maxBackgroundOperations + * Signature: (J)I + */ +jint Java_org_rocksdb_BackupEngineOptions_maxBackgroundOperations( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { + auto* bopt = + reinterpret_cast(jhandle); + return static_cast(bopt->max_background_operations); +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: setCallbackTriggerIntervalSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_BackupEngineOptions_setCallbackTriggerIntervalSize( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, + jlong jcallback_trigger_interval_size) { + auto* bopt = + reinterpret_cast(jhandle); + bopt->callback_trigger_interval_size = + static_cast(jcallback_trigger_interval_size); +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: callbackTriggerIntervalSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_BackupEngineOptions_callbackTriggerIntervalSize( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { + auto* bopt = + reinterpret_cast(jhandle); + return static_cast(bopt->callback_trigger_interval_size); +} + +/* + * Class: org_rocksdb_BackupEngineOptions + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_BackupEngineOptions_disposeInternal(JNIEnv* /*env*/, + jobject /*jopt*/, + jlong jhandle) { + auto* bopt = + reinterpret_cast(jhandle); + assert(bopt != nullptr); + delete bopt; +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/backupenginejni.cc b/librocksdb-sys/rocksdb/java/rocksjni/backupenginejni.cc new file mode 100644 index 0000000..1ba7ea2 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/backupenginejni.cc @@ -0,0 +1,279 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling C++ ROCKSDB_NAMESPACE::BackupEngine methods from the Java side. + +#include + +#include + +#include "include/org_rocksdb_BackupEngine.h" +#include "rocksdb/utilities/backup_engine.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_BackupEngine + * Method: open + * Signature: (JJ)J + */ +jlong Java_org_rocksdb_BackupEngine_open(JNIEnv* env, jclass /*jcls*/, + jlong env_handle, + jlong backup_engine_options_handle) { + auto* rocks_env = reinterpret_cast(env_handle); + auto* backup_engine_options = + reinterpret_cast( + backup_engine_options_handle); + ROCKSDB_NAMESPACE::BackupEngine* backup_engine; + auto status = ROCKSDB_NAMESPACE::BackupEngine::Open( + rocks_env, *backup_engine_options, &backup_engine); + + if (status.ok()) { + return GET_CPLUSPLUS_POINTER(backup_engine); + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + return 0; + } +} + +/* + * Class: org_rocksdb_BackupEngine + * Method: createNewBackup + * Signature: (JJZ)V + */ +void Java_org_rocksdb_BackupEngine_createNewBackup( + JNIEnv* env, jobject /*jbe*/, jlong jbe_handle, jlong db_handle, + jboolean jflush_before_backup) { + auto* db = reinterpret_cast(db_handle); + auto* backup_engine = + reinterpret_cast(jbe_handle); + auto status = backup_engine->CreateNewBackup( + db, static_cast(jflush_before_backup)); + + if (status.ok()) { + return; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); +} + +/* + * Class: org_rocksdb_BackupEngine + * Method: createNewBackupWithMetadata + * Signature: (JJLjava/lang/String;Z)V + */ +void Java_org_rocksdb_BackupEngine_createNewBackupWithMetadata( + JNIEnv* env, jobject /*jbe*/, jlong jbe_handle, jlong db_handle, + jstring japp_metadata, jboolean jflush_before_backup) { + auto* db = reinterpret_cast(db_handle); + auto* backup_engine = + reinterpret_cast(jbe_handle); + + jboolean has_exception = JNI_FALSE; + std::string app_metadata = ROCKSDB_NAMESPACE::JniUtil::copyStdString( + env, japp_metadata, &has_exception); + if (has_exception == JNI_TRUE) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, "Could not copy jstring to std::string"); + return; + } + + auto status = backup_engine->CreateNewBackupWithMetadata( + db, app_metadata, static_cast(jflush_before_backup)); + + if (status.ok()) { + return; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); +} + +/* + * Class: org_rocksdb_BackupEngine + * Method: getBackupInfo + * Signature: (J)Ljava/util/List; + */ +jobject Java_org_rocksdb_BackupEngine_getBackupInfo(JNIEnv* env, + jobject /*jbe*/, + jlong jbe_handle) { + auto* backup_engine = + reinterpret_cast(jbe_handle); + std::vector backup_infos; + backup_engine->GetBackupInfo(&backup_infos); + return ROCKSDB_NAMESPACE::BackupInfoListJni::getBackupInfo(env, backup_infos); +} + +/* + * Class: org_rocksdb_BackupEngine + * Method: getCorruptedBackups + * Signature: (J)[I + */ +jintArray Java_org_rocksdb_BackupEngine_getCorruptedBackups(JNIEnv* env, + jobject /*jbe*/, + jlong jbe_handle) { + auto* backup_engine = + reinterpret_cast(jbe_handle); + std::vector backup_ids; + backup_engine->GetCorruptedBackups(&backup_ids); + // store backupids in int array + std::vector int_backup_ids(backup_ids.begin(), backup_ids.end()); + + // Store ints in java array + // Its ok to loose precision here (64->32) + jsize ret_backup_ids_size = static_cast(backup_ids.size()); + jintArray ret_backup_ids = env->NewIntArray(ret_backup_ids_size); + if (ret_backup_ids == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + env->SetIntArrayRegion(ret_backup_ids, 0, ret_backup_ids_size, + int_backup_ids.data()); + return ret_backup_ids; +} + +/* + * Class: org_rocksdb_BackupEngine + * Method: garbageCollect + * Signature: (J)V + */ +void Java_org_rocksdb_BackupEngine_garbageCollect(JNIEnv* env, jobject /*jbe*/, + jlong jbe_handle) { + auto* backup_engine = + reinterpret_cast(jbe_handle); + auto status = backup_engine->GarbageCollect(); + + if (status.ok()) { + return; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); +} + +/* + * Class: org_rocksdb_BackupEngine + * Method: purgeOldBackups + * Signature: (JI)V + */ +void Java_org_rocksdb_BackupEngine_purgeOldBackups(JNIEnv* env, jobject /*jbe*/, + jlong jbe_handle, + jint jnum_backups_to_keep) { + auto* backup_engine = + reinterpret_cast(jbe_handle); + auto status = backup_engine->PurgeOldBackups( + static_cast(jnum_backups_to_keep)); + + if (status.ok()) { + return; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); +} + +/* + * Class: org_rocksdb_BackupEngine + * Method: deleteBackup + * Signature: (JI)V + */ +void Java_org_rocksdb_BackupEngine_deleteBackup(JNIEnv* env, jobject /*jbe*/, + jlong jbe_handle, + jint jbackup_id) { + auto* backup_engine = + reinterpret_cast(jbe_handle); + auto status = backup_engine->DeleteBackup( + static_cast(jbackup_id)); + + if (status.ok()) { + return; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); +} + +/* + * Class: org_rocksdb_BackupEngine + * Method: restoreDbFromBackup + * Signature: (JILjava/lang/String;Ljava/lang/String;J)V + */ +void Java_org_rocksdb_BackupEngine_restoreDbFromBackup( + JNIEnv* env, jobject /*jbe*/, jlong jbe_handle, jint jbackup_id, + jstring jdb_dir, jstring jwal_dir, jlong jrestore_options_handle) { + auto* backup_engine = + reinterpret_cast(jbe_handle); + const char* db_dir = env->GetStringUTFChars(jdb_dir, nullptr); + if (db_dir == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + const char* wal_dir = env->GetStringUTFChars(jwal_dir, nullptr); + if (wal_dir == nullptr) { + // exception thrown: OutOfMemoryError + env->ReleaseStringUTFChars(jdb_dir, db_dir); + return; + } + auto* restore_options = reinterpret_cast( + jrestore_options_handle); + auto status = backup_engine->RestoreDBFromBackup( + static_cast(jbackup_id), db_dir, wal_dir, + *restore_options); + + env->ReleaseStringUTFChars(jwal_dir, wal_dir); + env->ReleaseStringUTFChars(jdb_dir, db_dir); + + if (status.ok()) { + return; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); +} + +/* + * Class: org_rocksdb_BackupEngine + * Method: restoreDbFromLatestBackup + * Signature: (JLjava/lang/String;Ljava/lang/String;J)V + */ +void Java_org_rocksdb_BackupEngine_restoreDbFromLatestBackup( + JNIEnv* env, jobject /*jbe*/, jlong jbe_handle, jstring jdb_dir, + jstring jwal_dir, jlong jrestore_options_handle) { + auto* backup_engine = + reinterpret_cast(jbe_handle); + const char* db_dir = env->GetStringUTFChars(jdb_dir, nullptr); + if (db_dir == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + const char* wal_dir = env->GetStringUTFChars(jwal_dir, nullptr); + if (wal_dir == nullptr) { + // exception thrown: OutOfMemoryError + env->ReleaseStringUTFChars(jdb_dir, db_dir); + return; + } + auto* restore_options = reinterpret_cast( + jrestore_options_handle); + auto status = backup_engine->RestoreDBFromLatestBackup(db_dir, wal_dir, + *restore_options); + + env->ReleaseStringUTFChars(jwal_dir, wal_dir); + env->ReleaseStringUTFChars(jdb_dir, db_dir); + + if (status.ok()) { + return; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); +} + +/* + * Class: org_rocksdb_BackupEngine + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_BackupEngine_disposeInternal(JNIEnv* /*env*/, + jobject /*jbe*/, + jlong jbe_handle) { + auto* be = reinterpret_cast(jbe_handle); + assert(be != nullptr); + delete be; +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/cache.cc b/librocksdb-sys/rocksdb/java/rocksjni/cache.cc new file mode 100644 index 0000000..5ca1d51 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/cache.cc @@ -0,0 +1,34 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::Cache. + +#include + +#include "include/org_rocksdb_Cache.h" +#include "rocksdb/advanced_cache.h" + +/* + * Class: org_rocksdb_Cache + * Method: getUsage + * Signature: (J)J + */ +jlong Java_org_rocksdb_Cache_getUsage(JNIEnv*, jclass, jlong jhandle) { + auto* sptr_cache = + reinterpret_cast*>(jhandle); + return static_cast(sptr_cache->get()->GetUsage()); +} + +/* + * Class: org_rocksdb_Cache + * Method: getPinnedUsage + * Signature: (J)J + */ +jlong Java_org_rocksdb_Cache_getPinnedUsage(JNIEnv*, jclass, jlong jhandle) { + auto* sptr_cache = + reinterpret_cast*>(jhandle); + return static_cast(sptr_cache->get()->GetPinnedUsage()); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/cassandra_compactionfilterjni.cc b/librocksdb-sys/rocksdb/java/rocksjni/cassandra_compactionfilterjni.cc new file mode 100644 index 0000000..25817ae --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/cassandra_compactionfilterjni.cc @@ -0,0 +1,25 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include + +#include "include/org_rocksdb_CassandraCompactionFilter.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "utilities/cassandra/cassandra_compaction_filter.h" + +/* + * Class: org_rocksdb_CassandraCompactionFilter + * Method: createNewCassandraCompactionFilter0 + * Signature: (ZI)J + */ +jlong Java_org_rocksdb_CassandraCompactionFilter_createNewCassandraCompactionFilter0( + JNIEnv* /*env*/, jclass /*jcls*/, jboolean purge_ttl_on_expiration, + jint gc_grace_period_in_seconds) { + auto* compaction_filter = + new ROCKSDB_NAMESPACE::cassandra::CassandraCompactionFilter( + purge_ttl_on_expiration, gc_grace_period_in_seconds); + // set the native handle to our native compaction filter + return GET_CPLUSPLUS_POINTER(compaction_filter); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/cassandra_value_operator.cc b/librocksdb-sys/rocksdb/java/rocksjni/cassandra_value_operator.cc new file mode 100644 index 0000000..6de28c1 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/cassandra_value_operator.cc @@ -0,0 +1,50 @@ +// Copyright (c) 2017-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include +#include +#include + +#include +#include + +#include "include/org_rocksdb_CassandraValueMergeOperator.h" +#include "rocksdb/db.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/options.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/statistics.h" +#include "rocksdb/table.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" +#include "utilities/cassandra/merge_operator.h" + +/* + * Class: org_rocksdb_CassandraValueMergeOperator + * Method: newSharedCassandraValueMergeOperator + * Signature: (II)J + */ +jlong Java_org_rocksdb_CassandraValueMergeOperator_newSharedCassandraValueMergeOperator( + JNIEnv* /*env*/, jclass /*jclazz*/, jint gcGracePeriodInSeconds, + jint operands_limit) { + auto* op = new std::shared_ptr( + new ROCKSDB_NAMESPACE::cassandra::CassandraValueMergeOperator( + gcGracePeriodInSeconds, operands_limit)); + return GET_CPLUSPLUS_POINTER(op); +} + +/* + * Class: org_rocksdb_CassandraValueMergeOperator + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_CassandraValueMergeOperator_disposeInternal( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { + auto* op = + reinterpret_cast*>( + jhandle); + delete op; +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/checkpoint.cc b/librocksdb-sys/rocksdb/java/rocksjni/checkpoint.cc new file mode 100644 index 0000000..d7cfd81 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/checkpoint.cc @@ -0,0 +1,71 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling c++ ROCKSDB_NAMESPACE::Checkpoint methods from Java side. + +#include "rocksdb/utilities/checkpoint.h" + +#include +#include +#include + +#include + +#include "include/org_rocksdb_Checkpoint.h" +#include "rocksdb/db.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" +/* + * Class: org_rocksdb_Checkpoint + * Method: newCheckpoint + * Signature: (J)J + */ +jlong Java_org_rocksdb_Checkpoint_newCheckpoint(JNIEnv* /*env*/, + jclass /*jclazz*/, + jlong jdb_handle) { + auto* db = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::Checkpoint* checkpoint; + ROCKSDB_NAMESPACE::Checkpoint::Create(db, &checkpoint); + return GET_CPLUSPLUS_POINTER(checkpoint); +} + +/* + * Class: org_rocksdb_Checkpoint + * Method: dispose + * Signature: (J)V + */ +void Java_org_rocksdb_Checkpoint_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* checkpoint = reinterpret_cast(jhandle); + assert(checkpoint != nullptr); + delete checkpoint; +} + +/* + * Class: org_rocksdb_Checkpoint + * Method: createCheckpoint + * Signature: (JLjava/lang/String;)V + */ +void Java_org_rocksdb_Checkpoint_createCheckpoint(JNIEnv* env, jobject /*jobj*/, + jlong jcheckpoint_handle, + jstring jcheckpoint_path) { + const char* checkpoint_path = env->GetStringUTFChars(jcheckpoint_path, 0); + if (checkpoint_path == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + auto* checkpoint = + reinterpret_cast(jcheckpoint_handle); + ROCKSDB_NAMESPACE::Status s = checkpoint->CreateCheckpoint(checkpoint_path); + + env->ReleaseStringUTFChars(jcheckpoint_path, checkpoint_path); + + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/clock_cache.cc b/librocksdb-sys/rocksdb/java/rocksjni/clock_cache.cc new file mode 100644 index 0000000..e04991a --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/clock_cache.cc @@ -0,0 +1,42 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::ClockCache. + +#include "cache/clock_cache.h" + +#include + +#include "include/org_rocksdb_ClockCache.h" +#include "rocksjni/cplusplus_to_java_convert.h" + +/* + * Class: org_rocksdb_ClockCache + * Method: newClockCache + * Signature: (JIZ)J + */ +jlong Java_org_rocksdb_ClockCache_newClockCache( + JNIEnv* /*env*/, jclass /*jcls*/, jlong jcapacity, jint jnum_shard_bits, + jboolean jstrict_capacity_limit) { + auto* sptr_clock_cache = new std::shared_ptr( + ROCKSDB_NAMESPACE::NewClockCache( + static_cast(jcapacity), static_cast(jnum_shard_bits), + static_cast(jstrict_capacity_limit))); + return GET_CPLUSPLUS_POINTER(sptr_clock_cache); +} + +/* + * Class: org_rocksdb_ClockCache + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_ClockCache_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* sptr_clock_cache = + reinterpret_cast*>(jhandle); + delete sptr_clock_cache; // delete std::shared_ptr +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/columnfamilyhandle.cc b/librocksdb-sys/rocksdb/java/rocksjni/columnfamilyhandle.cc new file mode 100644 index 0000000..4140580 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/columnfamilyhandle.cc @@ -0,0 +1,72 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::ColumnFamilyHandle. + +#include +#include +#include + +#include "include/org_rocksdb_ColumnFamilyHandle.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_ColumnFamilyHandle + * Method: getName + * Signature: (J)[B + */ +jbyteArray Java_org_rocksdb_ColumnFamilyHandle_getName(JNIEnv* env, + jobject /*jobj*/, + jlong jhandle) { + auto* cfh = reinterpret_cast(jhandle); + std::string cf_name = cfh->GetName(); + return ROCKSDB_NAMESPACE::JniUtil::copyBytes(env, cf_name); +} + +/* + * Class: org_rocksdb_ColumnFamilyHandle + * Method: getID + * Signature: (J)I + */ +jint Java_org_rocksdb_ColumnFamilyHandle_getID(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* cfh = reinterpret_cast(jhandle); + const int32_t id = cfh->GetID(); + return static_cast(id); +} + +/* + * Class: org_rocksdb_ColumnFamilyHandle + * Method: getDescriptor + * Signature: (J)Lorg/rocksdb/ColumnFamilyDescriptor; + */ +jobject Java_org_rocksdb_ColumnFamilyHandle_getDescriptor(JNIEnv* env, + jobject /*jobj*/, + jlong jhandle) { + auto* cfh = reinterpret_cast(jhandle); + ROCKSDB_NAMESPACE::ColumnFamilyDescriptor desc; + ROCKSDB_NAMESPACE::Status s = cfh->GetDescriptor(&desc); + if (s.ok()) { + return ROCKSDB_NAMESPACE::ColumnFamilyDescriptorJni::construct(env, &desc); + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return nullptr; + } +} + +/* + * Class: org_rocksdb_ColumnFamilyHandle + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_ColumnFamilyHandle_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* cfh = reinterpret_cast(jhandle); + assert(cfh != nullptr); + delete cfh; +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/compact_range_options.cc b/librocksdb-sys/rocksdb/java/rocksjni/compact_range_options.cc new file mode 100644 index 0000000..d07263a --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/compact_range_options.cc @@ -0,0 +1,338 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::CompactRangeOptions. + +#include + +#include "include/org_rocksdb_CompactRangeOptions.h" +#include "rocksdb/options.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" +#include "util/coding.h" + +/** + * @brief Class containing compact range options for Java API + * + * An object of this class is returned as the native handle for + * ROCKSDB_NAMESPACE::CompactRangeOptions It contains objects for various + * parameters which are passed by reference/pointer in CompactRangeOptions. We + * maintain the lifetime of these parameters (`full_history_ts_low`, `canceled`) + * by including their values in this class. + */ +class Java_org_rocksdb_CompactRangeOptions { + public: + ROCKSDB_NAMESPACE::CompactRangeOptions compactRangeOptions; + + private: + std::string full_history_ts_low; + std::atomic canceled; + + public: + void set_full_history_ts_low(uint64_t start, uint64_t range) { + full_history_ts_low = ""; + ROCKSDB_NAMESPACE::PutFixed64(&full_history_ts_low, start); + ROCKSDB_NAMESPACE::PutFixed64(&full_history_ts_low, range); + compactRangeOptions.full_history_ts_low = + new ROCKSDB_NAMESPACE::Slice(full_history_ts_low); + } + + bool read_full_history_ts_low(uint64_t* start, uint64_t* range) { + if (compactRangeOptions.full_history_ts_low == nullptr) return false; + ROCKSDB_NAMESPACE::Slice read_slice( + compactRangeOptions.full_history_ts_low->ToStringView()); + if (!ROCKSDB_NAMESPACE::GetFixed64(&read_slice, start)) return false; + return ROCKSDB_NAMESPACE::GetFixed64(&read_slice, range); + } + + void set_canceled(bool value) { + if (compactRangeOptions.canceled == nullptr) { + canceled.store(value, std::memory_order_seq_cst); + compactRangeOptions.canceled = &canceled; + } else { + compactRangeOptions.canceled->store(value, std::memory_order_seq_cst); + } + } + + bool get_canceled() { + return compactRangeOptions.canceled && + compactRangeOptions.canceled->load(std::memory_order_seq_cst); + } +}; + +/* + * Class: org_rocksdb_CompactRangeOptions + * Method: newCompactRangeOptions + * Signature: ()J + */ +jlong Java_org_rocksdb_CompactRangeOptions_newCompactRangeOptions( + JNIEnv* /*env*/, jclass /*jclazz*/) { + auto* options = new Java_org_rocksdb_CompactRangeOptions(); + return GET_CPLUSPLUS_POINTER(&options->compactRangeOptions); +} + +/* + * Class: org_rocksdb_CompactRangeOptions + * Method: exclusiveManualCompaction + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_CompactRangeOptions_exclusiveManualCompaction( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { + auto* options = + reinterpret_cast(jhandle); + return static_cast( + options->compactRangeOptions.exclusive_manual_compaction); +} + +/* + * Class: org_rocksdb_CompactRangeOptions + * Method: setExclusiveManualCompaction + * Signature: (JZ)V + */ +void Java_org_rocksdb_CompactRangeOptions_setExclusiveManualCompaction( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, + jboolean exclusive_manual_compaction) { + auto* options = + reinterpret_cast(jhandle); + options->compactRangeOptions.exclusive_manual_compaction = + static_cast(exclusive_manual_compaction); +} + +/* + * Class: org_rocksdb_CompactRangeOptions + * Method: bottommostLevelCompaction + * Signature: (J)I + */ +jint Java_org_rocksdb_CompactRangeOptions_bottommostLevelCompaction( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { + auto* options = + reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::BottommostLevelCompactionJni:: + toJavaBottommostLevelCompaction( + options->compactRangeOptions.bottommost_level_compaction); +} + +/* + * Class: org_rocksdb_CompactRangeOptions + * Method: setBottommostLevelCompaction + * Signature: (JI)V + */ +void Java_org_rocksdb_CompactRangeOptions_setBottommostLevelCompaction( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, + jint bottommost_level_compaction) { + auto* options = + reinterpret_cast(jhandle); + options->compactRangeOptions.bottommost_level_compaction = + ROCKSDB_NAMESPACE::BottommostLevelCompactionJni:: + toCppBottommostLevelCompaction(bottommost_level_compaction); +} + +/* + * Class: org_rocksdb_CompactRangeOptions + * Method: changeLevel + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_CompactRangeOptions_changeLevel(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* options = + reinterpret_cast(jhandle); + return static_cast(options->compactRangeOptions.change_level); +} + +/* + * Class: org_rocksdb_CompactRangeOptions + * Method: setChangeLevel + * Signature: (JZ)V + */ +void Java_org_rocksdb_CompactRangeOptions_setChangeLevel( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jboolean change_level) { + auto* options = + reinterpret_cast(jhandle); + options->compactRangeOptions.change_level = static_cast(change_level); +} + +/* + * Class: org_rocksdb_CompactRangeOptions + * Method: targetLevel + * Signature: (J)I + */ +jint Java_org_rocksdb_CompactRangeOptions_targetLevel(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* options = + reinterpret_cast(jhandle); + return static_cast(options->compactRangeOptions.target_level); +} + +/* + * Class: org_rocksdb_CompactRangeOptions + * Method: setTargetLevel + * Signature: (JI)V + */ +void Java_org_rocksdb_CompactRangeOptions_setTargetLevel(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle, + jint target_level) { + auto* options = + reinterpret_cast(jhandle); + options->compactRangeOptions.target_level = static_cast(target_level); +} + +/* + * Class: org_rocksdb_CompactRangeOptions + * Method: targetPathId + * Signature: (J)I + */ +jint Java_org_rocksdb_CompactRangeOptions_targetPathId(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* options = + reinterpret_cast(jhandle); + return static_cast(options->compactRangeOptions.target_path_id); +} + +/* + * Class: org_rocksdb_CompactRangeOptions + * Method: setTargetPathId + * Signature: (JI)V + */ +void Java_org_rocksdb_CompactRangeOptions_setTargetPathId(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle, + jint target_path_id) { + auto* options = + reinterpret_cast(jhandle); + options->compactRangeOptions.target_path_id = + static_cast(target_path_id); +} + +/* + * Class: org_rocksdb_CompactRangeOptions + * Method: allowWriteStall + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_CompactRangeOptions_allowWriteStall(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* options = + reinterpret_cast(jhandle); + return static_cast(options->compactRangeOptions.allow_write_stall); +} + +/* + * Class: org_rocksdb_CompactRangeOptions + * Method: setAllowWriteStall + * Signature: (JZ)V + */ +void Java_org_rocksdb_CompactRangeOptions_setAllowWriteStall( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, + jboolean allow_write_stall) { + auto* options = + reinterpret_cast(jhandle); + options->compactRangeOptions.allow_write_stall = + static_cast(allow_write_stall); +} + +/* + * Class: org_rocksdb_CompactRangeOptions + * Method: maxSubcompactions + * Signature: (J)I + */ +jint Java_org_rocksdb_CompactRangeOptions_maxSubcompactions(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* options = + reinterpret_cast(jhandle); + return static_cast(options->compactRangeOptions.max_subcompactions); +} + +/* + * Class: org_rocksdb_CompactRangeOptions + * Method: setMaxSubcompactions + * Signature: (JI)V + */ +void Java_org_rocksdb_CompactRangeOptions_setMaxSubcompactions( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jint max_subcompactions) { + auto* options = + reinterpret_cast(jhandle); + options->compactRangeOptions.max_subcompactions = + static_cast(max_subcompactions); +} + +/* + * Class: org_rocksdb_CompactRangeOptions + * Method: setFullHistoryTSLow + * Signature: (JJJ)V + */ +void Java_org_rocksdb_CompactRangeOptions_setFullHistoryTSLow(JNIEnv*, jobject, + jlong jhandle, + jlong start, + jlong range) { + auto* options = + reinterpret_cast(jhandle); + options->set_full_history_ts_low(start, range); +} + +/* + * Class: org_rocksdb_CompactRangeOptions + * Method: fullHistoryTSLow + * Signature: (J)Lorg/rocksdb/CompactRangeOptions/Timestamp; + */ +jobject Java_org_rocksdb_CompactRangeOptions_fullHistoryTSLow(JNIEnv* env, + jobject, + jlong jhandle) { + auto* options = + reinterpret_cast(jhandle); + uint64_t start; + uint64_t range; + jobject result = nullptr; + if (options->read_full_history_ts_low(&start, &range)) { + result = + ROCKSDB_NAMESPACE::CompactRangeOptionsTimestampJni::fromCppTimestamp( + env, start, range); + } + + return result; +} + +/* + * Class: org_rocksdb_CompactRangeOptions + * Method: setCanceled + * Signature: (JZ)V + */ +void Java_org_rocksdb_CompactRangeOptions_setCanceled(JNIEnv*, jobject, + jlong jhandle, + jboolean jcanceled) { + auto* options = + reinterpret_cast(jhandle); + options->set_canceled(jcanceled); +} + +/* + * Class: org_rocksdb_CompactRangeOptions + * Method: canceled + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_CompactRangeOptions_canceled(JNIEnv*, jobject, + jlong jhandle) { + auto* options = + reinterpret_cast(jhandle); + return options->get_canceled(); +} + +/* + * Class: org_rocksdb_CompactRangeOptions + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_CompactRangeOptions_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* options = + reinterpret_cast(jhandle); + delete options; +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/compaction_filter.cc b/librocksdb-sys/rocksdb/java/rocksjni/compaction_filter.cc new file mode 100644 index 0000000..ea04996 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/compaction_filter.cc @@ -0,0 +1,29 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::CompactionFilter. + +#include "rocksdb/compaction_filter.h" + +#include + +#include "include/org_rocksdb_AbstractCompactionFilter.h" + +// + +/* + * Class: org_rocksdb_AbstractCompactionFilter + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_AbstractCompactionFilter_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + auto* cf = reinterpret_cast(handle); + assert(cf != nullptr); + delete cf; +} +// diff --git a/librocksdb-sys/rocksdb/java/rocksjni/compaction_filter_factory.cc b/librocksdb-sys/rocksdb/java/rocksjni/compaction_filter_factory.cc new file mode 100644 index 0000000..16fbdbb --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/compaction_filter_factory.cc @@ -0,0 +1,42 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::CompactionFilterFactory. + +#include + +#include + +#include "include/org_rocksdb_AbstractCompactionFilterFactory.h" +#include "rocksjni/compaction_filter_factory_jnicallback.h" +#include "rocksjni/cplusplus_to_java_convert.h" + +/* + * Class: org_rocksdb_AbstractCompactionFilterFactory + * Method: createNewCompactionFilterFactory0 + * Signature: ()J + */ +jlong Java_org_rocksdb_AbstractCompactionFilterFactory_createNewCompactionFilterFactory0( + JNIEnv* env, jobject jobj) { + auto* cff = + new ROCKSDB_NAMESPACE::CompactionFilterFactoryJniCallback(env, jobj); + auto* ptr_sptr_cff = new std::shared_ptr< + ROCKSDB_NAMESPACE::CompactionFilterFactoryJniCallback>(cff); + return GET_CPLUSPLUS_POINTER(ptr_sptr_cff); +} + +/* + * Class: org_rocksdb_AbstractCompactionFilterFactory + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_AbstractCompactionFilterFactory_disposeInternal( + JNIEnv*, jobject, jlong jhandle) { + auto* ptr_sptr_cff = reinterpret_cast< + std::shared_ptr*>( + jhandle); + delete ptr_sptr_cff; +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/compaction_filter_factory_jnicallback.cc b/librocksdb-sys/rocksdb/java/rocksjni/compaction_filter_factory_jnicallback.cc new file mode 100644 index 0000000..1428552 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/compaction_filter_factory_jnicallback.cc @@ -0,0 +1,79 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the callback "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::CompactionFilterFactory. + +#include "rocksjni/compaction_filter_factory_jnicallback.h" + +#include "rocksjni/portal.h" + +namespace ROCKSDB_NAMESPACE { +CompactionFilterFactoryJniCallback::CompactionFilterFactoryJniCallback( + JNIEnv* env, jobject jcompaction_filter_factory) + : JniCallback(env, jcompaction_filter_factory) { + // Note: The name of a CompactionFilterFactory will not change during + // it's lifetime, so we cache it in a global var + jmethodID jname_method_id = + AbstractCompactionFilterFactoryJni::getNameMethodId(env); + if (jname_method_id == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return; + } + + jstring jname = + (jstring)env->CallObjectMethod(m_jcallback_obj, jname_method_id); + if (env->ExceptionCheck()) { + // exception thrown + return; + } + jboolean has_exception = JNI_FALSE; + m_name = + JniUtil::copyString(env, jname, &has_exception); // also releases jname + if (has_exception == JNI_TRUE) { + // exception thrown + return; + } + + m_jcreate_compaction_filter_methodid = + AbstractCompactionFilterFactoryJni::getCreateCompactionFilterMethodId( + env); + if (m_jcreate_compaction_filter_methodid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return; + } +} + +const char* CompactionFilterFactoryJniCallback::Name() const { + return m_name.get(); +} + +std::unique_ptr +CompactionFilterFactoryJniCallback::CreateCompactionFilter( + const CompactionFilter::Context& context) { + jboolean attached_thread = JNI_FALSE; + JNIEnv* env = getJniEnv(&attached_thread); + assert(env != nullptr); + + jlong addr_compaction_filter = + env->CallLongMethod(m_jcallback_obj, m_jcreate_compaction_filter_methodid, + static_cast(context.is_full_compaction), + static_cast(context.is_manual_compaction)); + + if (env->ExceptionCheck()) { + // exception thrown from CallLongMethod + env->ExceptionDescribe(); // print out exception to stderr + releaseJniEnv(attached_thread); + return nullptr; + } + + auto* cff = reinterpret_cast(addr_compaction_filter); + + releaseJniEnv(attached_thread); + + return std::unique_ptr(cff); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/java/rocksjni/compaction_filter_factory_jnicallback.h b/librocksdb-sys/rocksdb/java/rocksjni/compaction_filter_factory_jnicallback.h new file mode 100644 index 0000000..2f26f8d --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/compaction_filter_factory_jnicallback.h @@ -0,0 +1,37 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the callback "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::CompactionFilterFactory. + +#ifndef JAVA_ROCKSJNI_COMPACTION_FILTER_FACTORY_JNICALLBACK_H_ +#define JAVA_ROCKSJNI_COMPACTION_FILTER_FACTORY_JNICALLBACK_H_ + +#include + +#include + +#include "rocksdb/compaction_filter.h" +#include "rocksjni/jnicallback.h" + +namespace ROCKSDB_NAMESPACE { + +class CompactionFilterFactoryJniCallback : public JniCallback, + public CompactionFilterFactory { + public: + CompactionFilterFactoryJniCallback(JNIEnv* env, + jobject jcompaction_filter_factory); + virtual std::unique_ptr CreateCompactionFilter( + const CompactionFilter::Context& context); + virtual const char* Name() const; + + private: + std::unique_ptr m_name; + jmethodID m_jcreate_compaction_filter_methodid; +}; + +} // namespace ROCKSDB_NAMESPACE + +#endif // JAVA_ROCKSJNI_COMPACTION_FILTER_FACTORY_JNICALLBACK_H_ diff --git a/librocksdb-sys/rocksdb/java/rocksjni/compaction_job_info.cc b/librocksdb-sys/rocksdb/java/rocksjni/compaction_job_info.cc new file mode 100644 index 0000000..fb292f5 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/compaction_job_info.cc @@ -0,0 +1,230 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::CompactionJobInfo. + +#include + +#include "include/org_rocksdb_CompactionJobInfo.h" +#include "rocksdb/listener.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_CompactionJobInfo + * Method: newCompactionJobInfo + * Signature: ()J + */ +jlong Java_org_rocksdb_CompactionJobInfo_newCompactionJobInfo(JNIEnv*, jclass) { + auto* compact_job_info = new ROCKSDB_NAMESPACE::CompactionJobInfo(); + return GET_CPLUSPLUS_POINTER(compact_job_info); +} + +/* + * Class: org_rocksdb_CompactionJobInfo + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_CompactionJobInfo_disposeInternal(JNIEnv*, jobject, + jlong jhandle) { + auto* compact_job_info = + reinterpret_cast(jhandle); + delete compact_job_info; +} + +/* + * Class: org_rocksdb_CompactionJobInfo + * Method: columnFamilyName + * Signature: (J)[B + */ +jbyteArray Java_org_rocksdb_CompactionJobInfo_columnFamilyName(JNIEnv* env, + jclass, + jlong jhandle) { + auto* compact_job_info = + reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::JniUtil::copyBytes(env, compact_job_info->cf_name); +} + +/* + * Class: org_rocksdb_CompactionJobInfo + * Method: status + * Signature: (J)Lorg/rocksdb/Status; + */ +jobject Java_org_rocksdb_CompactionJobInfo_status(JNIEnv* env, jclass, + jlong jhandle) { + auto* compact_job_info = + reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::StatusJni::construct(env, compact_job_info->status); +} + +/* + * Class: org_rocksdb_CompactionJobInfo + * Method: threadId + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionJobInfo_threadId(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_job_info = + reinterpret_cast(jhandle); + return static_cast(compact_job_info->thread_id); +} + +/* + * Class: org_rocksdb_CompactionJobInfo + * Method: jobId + * Signature: (J)I + */ +jint Java_org_rocksdb_CompactionJobInfo_jobId(JNIEnv*, jclass, jlong jhandle) { + auto* compact_job_info = + reinterpret_cast(jhandle); + return static_cast(compact_job_info->job_id); +} + +/* + * Class: org_rocksdb_CompactionJobInfo + * Method: baseInputLevel + * Signature: (J)I + */ +jint Java_org_rocksdb_CompactionJobInfo_baseInputLevel(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_job_info = + reinterpret_cast(jhandle); + return static_cast(compact_job_info->base_input_level); +} + +/* + * Class: org_rocksdb_CompactionJobInfo + * Method: outputLevel + * Signature: (J)I + */ +jint Java_org_rocksdb_CompactionJobInfo_outputLevel(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_job_info = + reinterpret_cast(jhandle); + return static_cast(compact_job_info->output_level); +} + +/* + * Class: org_rocksdb_CompactionJobInfo + * Method: inputFiles + * Signature: (J)[Ljava/lang/String; + */ +jobjectArray Java_org_rocksdb_CompactionJobInfo_inputFiles(JNIEnv* env, jclass, + jlong jhandle) { + auto* compact_job_info = + reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::JniUtil::toJavaStrings( + env, &compact_job_info->input_files); +} + +/* + * Class: org_rocksdb_CompactionJobInfo + * Method: outputFiles + * Signature: (J)[Ljava/lang/String; + */ +jobjectArray Java_org_rocksdb_CompactionJobInfo_outputFiles(JNIEnv* env, jclass, + jlong jhandle) { + auto* compact_job_info = + reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::JniUtil::toJavaStrings( + env, &compact_job_info->output_files); +} + +/* + * Class: org_rocksdb_CompactionJobInfo + * Method: tableProperties + * Signature: (J)Ljava/util/Map; + */ +jobject Java_org_rocksdb_CompactionJobInfo_tableProperties(JNIEnv* env, jclass, + jlong jhandle) { + auto* compact_job_info = + reinterpret_cast(jhandle); + auto* map = &compact_job_info->table_properties; + + jobject jhash_map = ROCKSDB_NAMESPACE::HashMapJni::construct( + env, static_cast(map->size())); + if (jhash_map == nullptr) { + // exception occurred + return nullptr; + } + + const ROCKSDB_NAMESPACE::HashMapJni::FnMapKV< + const std::string, + std::shared_ptr, jobject, + jobject> + fn_map_kv = + [env](const std::pair< + const std::string, + std::shared_ptr>& + kv) { + jstring jkey = ROCKSDB_NAMESPACE::JniUtil::toJavaString( + env, &(kv.first), false); + if (env->ExceptionCheck()) { + // an error occurred + return std::unique_ptr>(nullptr); + } + + jobject jtable_properties = + ROCKSDB_NAMESPACE::TablePropertiesJni::fromCppTableProperties( + env, *(kv.second.get())); + if (env->ExceptionCheck()) { + // an error occurred + env->DeleteLocalRef(jkey); + return std::unique_ptr>(nullptr); + } + + return std::unique_ptr>( + new std::pair(static_cast(jkey), + jtable_properties)); + }; + + if (!ROCKSDB_NAMESPACE::HashMapJni::putAll(env, jhash_map, map->begin(), + map->end(), fn_map_kv)) { + // exception occurred + return nullptr; + } + + return jhash_map; +} + +/* + * Class: org_rocksdb_CompactionJobInfo + * Method: compactionReason + * Signature: (J)B + */ +jbyte Java_org_rocksdb_CompactionJobInfo_compactionReason(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_job_info = + reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::CompactionReasonJni::toJavaCompactionReason( + compact_job_info->compaction_reason); +} + +/* + * Class: org_rocksdb_CompactionJobInfo + * Method: compression + * Signature: (J)B + */ +jbyte Java_org_rocksdb_CompactionJobInfo_compression(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_job_info = + reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::CompressionTypeJni::toJavaCompressionType( + compact_job_info->compression); +} + +/* + * Class: org_rocksdb_CompactionJobInfo + * Method: stats + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionJobInfo_stats(JNIEnv*, jclass, jlong jhandle) { + auto* compact_job_info = + reinterpret_cast(jhandle); + auto* stats = new ROCKSDB_NAMESPACE::CompactionJobStats(); + stats->Add(compact_job_info->stats); + return GET_CPLUSPLUS_POINTER(stats); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/compaction_job_stats.cc b/librocksdb-sys/rocksdb/java/rocksjni/compaction_job_stats.cc new file mode 100644 index 0000000..a2599c1 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/compaction_job_stats.cc @@ -0,0 +1,345 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::CompactionJobStats. + +#include "rocksdb/compaction_job_stats.h" + +#include + +#include "include/org_rocksdb_CompactionJobStats.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: newCompactionJobStats + * Signature: ()J + */ +jlong Java_org_rocksdb_CompactionJobStats_newCompactionJobStats(JNIEnv*, + jclass) { + auto* compact_job_stats = new ROCKSDB_NAMESPACE::CompactionJobStats(); + return GET_CPLUSPLUS_POINTER(compact_job_stats); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_CompactionJobStats_disposeInternal(JNIEnv*, jobject, + jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + delete compact_job_stats; +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: reset + * Signature: (J)V + */ +void Java_org_rocksdb_CompactionJobStats_reset(JNIEnv*, jclass, jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + compact_job_stats->Reset(); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: add + * Signature: (JJ)V + */ +void Java_org_rocksdb_CompactionJobStats_add(JNIEnv*, jclass, jlong jhandle, + jlong jother_handle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + auto* other_compact_job_stats = + reinterpret_cast(jother_handle); + compact_job_stats->Add(*other_compact_job_stats); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: elapsedMicros + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionJobStats_elapsedMicros(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + return static_cast(compact_job_stats->elapsed_micros); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: numInputRecords + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionJobStats_numInputRecords(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + return static_cast(compact_job_stats->num_input_records); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: numInputFiles + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionJobStats_numInputFiles(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + return static_cast(compact_job_stats->num_input_files); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: numInputFilesAtOutputLevel + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionJobStats_numInputFilesAtOutputLevel( + JNIEnv*, jclass, jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + return static_cast(compact_job_stats->num_input_files_at_output_level); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: numOutputRecords + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionJobStats_numOutputRecords(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + return static_cast(compact_job_stats->num_output_records); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: numOutputFiles + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionJobStats_numOutputFiles(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + return static_cast(compact_job_stats->num_output_files); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: isManualCompaction + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_CompactionJobStats_isManualCompaction(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + if (compact_job_stats->is_manual_compaction) { + return JNI_TRUE; + } else { + return JNI_FALSE; + } +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: totalInputBytes + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionJobStats_totalInputBytes(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + return static_cast(compact_job_stats->total_input_bytes); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: totalOutputBytes + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionJobStats_totalOutputBytes(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + return static_cast(compact_job_stats->total_output_bytes); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: numRecordsReplaced + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionJobStats_numRecordsReplaced(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + return static_cast(compact_job_stats->num_records_replaced); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: totalInputRawKeyBytes + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionJobStats_totalInputRawKeyBytes(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + return static_cast(compact_job_stats->total_input_raw_key_bytes); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: totalInputRawValueBytes + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionJobStats_totalInputRawValueBytes( + JNIEnv*, jclass, jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + return static_cast(compact_job_stats->total_input_raw_value_bytes); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: numInputDeletionRecords + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionJobStats_numInputDeletionRecords( + JNIEnv*, jclass, jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + return static_cast(compact_job_stats->num_input_deletion_records); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: numExpiredDeletionRecords + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionJobStats_numExpiredDeletionRecords( + JNIEnv*, jclass, jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + return static_cast(compact_job_stats->num_expired_deletion_records); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: numCorruptKeys + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionJobStats_numCorruptKeys(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + return static_cast(compact_job_stats->num_corrupt_keys); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: fileWriteNanos + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionJobStats_fileWriteNanos(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + return static_cast(compact_job_stats->file_write_nanos); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: fileRangeSyncNanos + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionJobStats_fileRangeSyncNanos(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + return static_cast(compact_job_stats->file_range_sync_nanos); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: fileFsyncNanos + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionJobStats_fileFsyncNanos(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + return static_cast(compact_job_stats->file_fsync_nanos); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: filePrepareWriteNanos + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionJobStats_filePrepareWriteNanos(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + return static_cast(compact_job_stats->file_prepare_write_nanos); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: smallestOutputKeyPrefix + * Signature: (J)[B + */ +jbyteArray Java_org_rocksdb_CompactionJobStats_smallestOutputKeyPrefix( + JNIEnv* env, jclass, jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::JniUtil::copyBytes( + env, compact_job_stats->smallest_output_key_prefix); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: largestOutputKeyPrefix + * Signature: (J)[B + */ +jbyteArray Java_org_rocksdb_CompactionJobStats_largestOutputKeyPrefix( + JNIEnv* env, jclass, jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::JniUtil::copyBytes( + env, compact_job_stats->largest_output_key_prefix); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: numSingleDelFallthru + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionJobStats_numSingleDelFallthru(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + return static_cast(compact_job_stats->num_single_del_fallthru); +} + +/* + * Class: org_rocksdb_CompactionJobStats + * Method: numSingleDelMismatch + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionJobStats_numSingleDelMismatch(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_job_stats = + reinterpret_cast(jhandle); + return static_cast(compact_job_stats->num_single_del_mismatch); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/compaction_options.cc b/librocksdb-sys/rocksdb/java/rocksjni/compaction_options.cc new file mode 100644 index 0000000..bbbde03 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/compaction_options.cc @@ -0,0 +1,112 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::CompactionOptions. + +#include + +#include "include/org_rocksdb_CompactionOptions.h" +#include "rocksdb/options.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_CompactionOptions + * Method: newCompactionOptions + * Signature: ()J + */ +jlong Java_org_rocksdb_CompactionOptions_newCompactionOptions(JNIEnv*, jclass) { + auto* compact_opts = new ROCKSDB_NAMESPACE::CompactionOptions(); + return GET_CPLUSPLUS_POINTER(compact_opts); +} + +/* + * Class: org_rocksdb_CompactionOptions + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_CompactionOptions_disposeInternal(JNIEnv*, jobject, + jlong jhandle) { + auto* compact_opts = + reinterpret_cast(jhandle); + delete compact_opts; +} + +/* + * Class: org_rocksdb_CompactionOptions + * Method: compression + * Signature: (J)B + */ +jbyte Java_org_rocksdb_CompactionOptions_compression(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_opts = + reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::CompressionTypeJni::toJavaCompressionType( + compact_opts->compression); +} + +/* + * Class: org_rocksdb_CompactionOptions + * Method: setCompression + * Signature: (JB)V + */ +void Java_org_rocksdb_CompactionOptions_setCompression( + JNIEnv*, jclass, jlong jhandle, jbyte jcompression_type_value) { + auto* compact_opts = + reinterpret_cast(jhandle); + compact_opts->compression = + ROCKSDB_NAMESPACE::CompressionTypeJni::toCppCompressionType( + jcompression_type_value); +} + +/* + * Class: org_rocksdb_CompactionOptions + * Method: outputFileSizeLimit + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionOptions_outputFileSizeLimit(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_opts = + reinterpret_cast(jhandle); + return static_cast(compact_opts->output_file_size_limit); +} + +/* + * Class: org_rocksdb_CompactionOptions + * Method: setOutputFileSizeLimit + * Signature: (JJ)V + */ +void Java_org_rocksdb_CompactionOptions_setOutputFileSizeLimit( + JNIEnv*, jclass, jlong jhandle, jlong joutput_file_size_limit) { + auto* compact_opts = + reinterpret_cast(jhandle); + compact_opts->output_file_size_limit = + static_cast(joutput_file_size_limit); +} + +/* + * Class: org_rocksdb_CompactionOptions + * Method: maxSubcompactions + * Signature: (J)I + */ +jint Java_org_rocksdb_CompactionOptions_maxSubcompactions(JNIEnv*, jclass, + jlong jhandle) { + auto* compact_opts = + reinterpret_cast(jhandle); + return static_cast(compact_opts->max_subcompactions); +} + +/* + * Class: org_rocksdb_CompactionOptions + * Method: setMaxSubcompactions + * Signature: (JI)V + */ +void Java_org_rocksdb_CompactionOptions_setMaxSubcompactions( + JNIEnv*, jclass, jlong jhandle, jint jmax_subcompactions) { + auto* compact_opts = + reinterpret_cast(jhandle); + compact_opts->max_subcompactions = static_cast(jmax_subcompactions); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/compaction_options_fifo.cc b/librocksdb-sys/rocksdb/java/rocksjni/compaction_options_fifo.cc new file mode 100644 index 0000000..f6a47fe --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/compaction_options_fifo.cc @@ -0,0 +1,83 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::CompactionOptionsFIFO. + +#include + +#include "include/org_rocksdb_CompactionOptionsFIFO.h" +#include "rocksdb/advanced_options.h" +#include "rocksjni/cplusplus_to_java_convert.h" + +/* + * Class: org_rocksdb_CompactionOptionsFIFO + * Method: newCompactionOptionsFIFO + * Signature: ()J + */ +jlong Java_org_rocksdb_CompactionOptionsFIFO_newCompactionOptionsFIFO(JNIEnv*, + jclass) { + const auto* opt = new ROCKSDB_NAMESPACE::CompactionOptionsFIFO(); + return GET_CPLUSPLUS_POINTER(opt); +} + +/* + * Class: org_rocksdb_CompactionOptionsFIFO + * Method: setMaxTableFilesSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_CompactionOptionsFIFO_setMaxTableFilesSize( + JNIEnv*, jobject, jlong jhandle, jlong jmax_table_files_size) { + auto* opt = + reinterpret_cast(jhandle); + opt->max_table_files_size = static_cast(jmax_table_files_size); +} + +/* + * Class: org_rocksdb_CompactionOptionsFIFO + * Method: maxTableFilesSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompactionOptionsFIFO_maxTableFilesSize(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = + reinterpret_cast(jhandle); + return static_cast(opt->max_table_files_size); +} + +/* + * Class: org_rocksdb_CompactionOptionsFIFO + * Method: setAllowCompaction + * Signature: (JZ)V + */ +void Java_org_rocksdb_CompactionOptionsFIFO_setAllowCompaction( + JNIEnv*, jobject, jlong jhandle, jboolean allow_compaction) { + auto* opt = + reinterpret_cast(jhandle); + opt->allow_compaction = static_cast(allow_compaction); +} + +/* + * Class: org_rocksdb_CompactionOptionsFIFO + * Method: allowCompaction + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_CompactionOptionsFIFO_allowCompaction(JNIEnv*, + jobject, + jlong jhandle) { + auto* opt = + reinterpret_cast(jhandle); + return static_cast(opt->allow_compaction); +} + +/* + * Class: org_rocksdb_CompactionOptionsFIFO + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_CompactionOptionsFIFO_disposeInternal(JNIEnv*, jobject, + jlong jhandle) { + delete reinterpret_cast(jhandle); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/compaction_options_universal.cc b/librocksdb-sys/rocksdb/java/rocksjni/compaction_options_universal.cc new file mode 100644 index 0000000..9fc6f31 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/compaction_options_universal.cc @@ -0,0 +1,209 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::CompactionOptionsUniversal. + +#include + +#include "include/org_rocksdb_CompactionOptionsUniversal.h" +#include "rocksdb/advanced_options.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_CompactionOptionsUniversal + * Method: newCompactionOptionsUniversal + * Signature: ()J + */ +jlong Java_org_rocksdb_CompactionOptionsUniversal_newCompactionOptionsUniversal( + JNIEnv*, jclass) { + const auto* opt = new ROCKSDB_NAMESPACE::CompactionOptionsUniversal(); + return GET_CPLUSPLUS_POINTER(opt); +} + +/* + * Class: org_rocksdb_CompactionOptionsUniversal + * Method: setSizeRatio + * Signature: (JI)V + */ +void Java_org_rocksdb_CompactionOptionsUniversal_setSizeRatio( + JNIEnv*, jobject, jlong jhandle, jint jsize_ratio) { + auto* opt = + reinterpret_cast(jhandle); + opt->size_ratio = static_cast(jsize_ratio); +} + +/* + * Class: org_rocksdb_CompactionOptionsUniversal + * Method: sizeRatio + * Signature: (J)I + */ +jint Java_org_rocksdb_CompactionOptionsUniversal_sizeRatio(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = + reinterpret_cast(jhandle); + return static_cast(opt->size_ratio); +} + +/* + * Class: org_rocksdb_CompactionOptionsUniversal + * Method: setMinMergeWidth + * Signature: (JI)V + */ +void Java_org_rocksdb_CompactionOptionsUniversal_setMinMergeWidth( + JNIEnv*, jobject, jlong jhandle, jint jmin_merge_width) { + auto* opt = + reinterpret_cast(jhandle); + opt->min_merge_width = static_cast(jmin_merge_width); +} + +/* + * Class: org_rocksdb_CompactionOptionsUniversal + * Method: minMergeWidth + * Signature: (J)I + */ +jint Java_org_rocksdb_CompactionOptionsUniversal_minMergeWidth(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = + reinterpret_cast(jhandle); + return static_cast(opt->min_merge_width); +} + +/* + * Class: org_rocksdb_CompactionOptionsUniversal + * Method: setMaxMergeWidth + * Signature: (JI)V + */ +void Java_org_rocksdb_CompactionOptionsUniversal_setMaxMergeWidth( + JNIEnv*, jobject, jlong jhandle, jint jmax_merge_width) { + auto* opt = + reinterpret_cast(jhandle); + opt->max_merge_width = static_cast(jmax_merge_width); +} + +/* + * Class: org_rocksdb_CompactionOptionsUniversal + * Method: maxMergeWidth + * Signature: (J)I + */ +jint Java_org_rocksdb_CompactionOptionsUniversal_maxMergeWidth(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = + reinterpret_cast(jhandle); + return static_cast(opt->max_merge_width); +} + +/* + * Class: org_rocksdb_CompactionOptionsUniversal + * Method: setMaxSizeAmplificationPercent + * Signature: (JI)V + */ +void Java_org_rocksdb_CompactionOptionsUniversal_setMaxSizeAmplificationPercent( + JNIEnv*, jobject, jlong jhandle, jint jmax_size_amplification_percent) { + auto* opt = + reinterpret_cast(jhandle); + opt->max_size_amplification_percent = + static_cast(jmax_size_amplification_percent); +} + +/* + * Class: org_rocksdb_CompactionOptionsUniversal + * Method: maxSizeAmplificationPercent + * Signature: (J)I + */ +jint Java_org_rocksdb_CompactionOptionsUniversal_maxSizeAmplificationPercent( + JNIEnv*, jobject, jlong jhandle) { + auto* opt = + reinterpret_cast(jhandle); + return static_cast(opt->max_size_amplification_percent); +} + +/* + * Class: org_rocksdb_CompactionOptionsUniversal + * Method: setCompressionSizePercent + * Signature: (JI)V + */ +void Java_org_rocksdb_CompactionOptionsUniversal_setCompressionSizePercent( + JNIEnv*, jobject, jlong jhandle, jint jcompression_size_percent) { + auto* opt = + reinterpret_cast(jhandle); + opt->compression_size_percent = + static_cast(jcompression_size_percent); +} + +/* + * Class: org_rocksdb_CompactionOptionsUniversal + * Method: compressionSizePercent + * Signature: (J)I + */ +jint Java_org_rocksdb_CompactionOptionsUniversal_compressionSizePercent( + JNIEnv*, jobject, jlong jhandle) { + auto* opt = + reinterpret_cast(jhandle); + return static_cast(opt->compression_size_percent); +} + +/* + * Class: org_rocksdb_CompactionOptionsUniversal + * Method: setStopStyle + * Signature: (JB)V + */ +void Java_org_rocksdb_CompactionOptionsUniversal_setStopStyle( + JNIEnv*, jobject, jlong jhandle, jbyte jstop_style_value) { + auto* opt = + reinterpret_cast(jhandle); + opt->stop_style = + ROCKSDB_NAMESPACE::CompactionStopStyleJni::toCppCompactionStopStyle( + jstop_style_value); +} + +/* + * Class: org_rocksdb_CompactionOptionsUniversal + * Method: stopStyle + * Signature: (J)B + */ +jbyte Java_org_rocksdb_CompactionOptionsUniversal_stopStyle(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = + reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::CompactionStopStyleJni::toJavaCompactionStopStyle( + opt->stop_style); +} + +/* + * Class: org_rocksdb_CompactionOptionsUniversal + * Method: setAllowTrivialMove + * Signature: (JZ)V + */ +void Java_org_rocksdb_CompactionOptionsUniversal_setAllowTrivialMove( + JNIEnv*, jobject, jlong jhandle, jboolean jallow_trivial_move) { + auto* opt = + reinterpret_cast(jhandle); + opt->allow_trivial_move = static_cast(jallow_trivial_move); +} + +/* + * Class: org_rocksdb_CompactionOptionsUniversal + * Method: allowTrivialMove + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_CompactionOptionsUniversal_allowTrivialMove( + JNIEnv*, jobject, jlong jhandle) { + auto* opt = + reinterpret_cast(jhandle); + return opt->allow_trivial_move; +} + +/* + * Class: org_rocksdb_CompactionOptionsUniversal + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_CompactionOptionsUniversal_disposeInternal( + JNIEnv*, jobject, jlong jhandle) { + delete reinterpret_cast( + jhandle); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/comparator.cc b/librocksdb-sys/rocksdb/java/rocksjni/comparator.cc new file mode 100644 index 0000000..11279c4 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/comparator.cc @@ -0,0 +1,60 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::Comparator. + +#include +#include +#include + +#include +#include + +#include "include/org_rocksdb_AbstractComparator.h" +#include "include/org_rocksdb_NativeComparatorWrapper.h" +#include "rocksjni/comparatorjnicallback.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_AbstractComparator + * Method: createNewComparator + * Signature: (J)J + */ +jlong Java_org_rocksdb_AbstractComparator_createNewComparator( + JNIEnv* env, jobject jcomparator, jlong copt_handle) { + auto* copt = + reinterpret_cast( + copt_handle); + auto* c = + new ROCKSDB_NAMESPACE::ComparatorJniCallback(env, jcomparator, copt); + return GET_CPLUSPLUS_POINTER(c); +} + +/* + * Class: org_rocksdb_AbstractComparator + * Method: usingDirectBuffers + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_AbstractComparator_usingDirectBuffers(JNIEnv*, + jobject, + jlong jhandle) { + auto* c = + reinterpret_cast(jhandle); + return static_cast(c->m_options->direct_buffer); +} + +/* + * Class: org_rocksdb_NativeComparatorWrapper + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_NativeComparatorWrapper_disposeInternal( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jcomparator_handle) { + auto* comparator = + reinterpret_cast(jcomparator_handle); + delete comparator; +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/comparatorjnicallback.cc b/librocksdb-sys/rocksdb/java/rocksjni/comparatorjnicallback.cc new file mode 100644 index 0000000..d354b40 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/comparatorjnicallback.cc @@ -0,0 +1,647 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the callback "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::Comparator. + +#include "rocksjni/comparatorjnicallback.h" + +#include "rocksjni/portal.h" + +namespace ROCKSDB_NAMESPACE { +ComparatorJniCallback::ComparatorJniCallback( + JNIEnv* env, jobject jcomparator, + const ComparatorJniCallbackOptions* options) + : JniCallback(env, jcomparator), + m_options(std::make_unique(*options)) { + // cache the AbstractComparatorJniBridge class as we will reuse it many times + // for each callback + m_abstract_comparator_jni_bridge_clazz = static_cast( + env->NewGlobalRef(AbstractComparatorJniBridge::getJClass(env))); + + // Note: The name of a Comparator will not change during it's lifetime, + // so we cache it in a global var + jmethodID jname_mid = AbstractComparatorJni::getNameMethodId(env); + if (jname_mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return; + } + jstring js_name = (jstring)env->CallObjectMethod(m_jcallback_obj, jname_mid); + if (env->ExceptionCheck()) { + // exception thrown + return; + } + jboolean has_exception = JNI_FALSE; + m_name = JniUtil::copyString(env, js_name, + &has_exception); // also releases jsName + if (has_exception == JNI_TRUE) { + // exception thrown + return; + } + + // cache the ByteBuffer class as we will reuse it many times for each callback + m_jbytebuffer_clazz = + static_cast(env->NewGlobalRef(ByteBufferJni::getJClass(env))); + + m_jcompare_mid = AbstractComparatorJniBridge::getCompareInternalMethodId( + env, m_abstract_comparator_jni_bridge_clazz); + if (m_jcompare_mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return; + } + + m_jshortest_mid = + AbstractComparatorJniBridge::getFindShortestSeparatorInternalMethodId( + env, m_abstract_comparator_jni_bridge_clazz); + if (m_jshortest_mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return; + } + + m_jshort_mid = + AbstractComparatorJniBridge::getFindShortSuccessorInternalMethodId( + env, m_abstract_comparator_jni_bridge_clazz); + if (m_jshort_mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return; + } + + // do we need reusable buffers? + if (m_options->max_reused_buffer_size > -1) { + if (m_options->reused_synchronisation_type == + ReusedSynchronisationType::THREAD_LOCAL) { + // buffers reused per thread + UnrefHandler unref = [](void* ptr) { + ThreadLocalBuf* tlb = reinterpret_cast(ptr); + jboolean attached_thread = JNI_FALSE; + JNIEnv* _env = JniUtil::getJniEnv(tlb->jvm, &attached_thread); + if (_env != nullptr) { + if (tlb->direct_buffer) { + void* buf = _env->GetDirectBufferAddress(tlb->jbuf); + delete[] static_cast(buf); + } + _env->DeleteGlobalRef(tlb->jbuf); + JniUtil::releaseJniEnv(tlb->jvm, attached_thread); + } + }; + + m_tl_buf_a = new ThreadLocalPtr(unref); + m_tl_buf_b = new ThreadLocalPtr(unref); + + m_jcompare_buf_a = nullptr; + m_jcompare_buf_b = nullptr; + m_jshortest_buf_start = nullptr; + m_jshortest_buf_limit = nullptr; + m_jshort_buf_key = nullptr; + + } else { + // buffers reused and shared across threads + const bool adaptive = m_options->reused_synchronisation_type == + ReusedSynchronisationType::ADAPTIVE_MUTEX; + mtx_compare = std::unique_ptr(new port::Mutex(adaptive)); + mtx_shortest = std::unique_ptr(new port::Mutex(adaptive)); + mtx_short = std::unique_ptr(new port::Mutex(adaptive)); + + m_jcompare_buf_a = env->NewGlobalRef(ByteBufferJni::construct( + env, m_options->direct_buffer, m_options->max_reused_buffer_size, + m_jbytebuffer_clazz)); + if (m_jcompare_buf_a == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + m_jcompare_buf_b = env->NewGlobalRef(ByteBufferJni::construct( + env, m_options->direct_buffer, m_options->max_reused_buffer_size, + m_jbytebuffer_clazz)); + if (m_jcompare_buf_b == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + m_jshortest_buf_start = env->NewGlobalRef(ByteBufferJni::construct( + env, m_options->direct_buffer, m_options->max_reused_buffer_size, + m_jbytebuffer_clazz)); + if (m_jshortest_buf_start == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + m_jshortest_buf_limit = env->NewGlobalRef(ByteBufferJni::construct( + env, m_options->direct_buffer, m_options->max_reused_buffer_size, + m_jbytebuffer_clazz)); + if (m_jshortest_buf_limit == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + m_jshort_buf_key = env->NewGlobalRef(ByteBufferJni::construct( + env, m_options->direct_buffer, m_options->max_reused_buffer_size, + m_jbytebuffer_clazz)); + if (m_jshort_buf_key == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + m_tl_buf_a = nullptr; + m_tl_buf_b = nullptr; + } + + } else { + m_jcompare_buf_a = nullptr; + m_jcompare_buf_b = nullptr; + m_jshortest_buf_start = nullptr; + m_jshortest_buf_limit = nullptr; + m_jshort_buf_key = nullptr; + + m_tl_buf_a = nullptr; + m_tl_buf_b = nullptr; + } +} + +ComparatorJniCallback::~ComparatorJniCallback() { + jboolean attached_thread = JNI_FALSE; + JNIEnv* env = getJniEnv(&attached_thread); + assert(env != nullptr); + + env->DeleteGlobalRef(m_abstract_comparator_jni_bridge_clazz); + + env->DeleteGlobalRef(m_jbytebuffer_clazz); + + if (m_jcompare_buf_a != nullptr) { + if (m_options->direct_buffer) { + void* buf = env->GetDirectBufferAddress(m_jcompare_buf_a); + delete[] static_cast(buf); + } + env->DeleteGlobalRef(m_jcompare_buf_a); + } + + if (m_jcompare_buf_b != nullptr) { + if (m_options->direct_buffer) { + void* buf = env->GetDirectBufferAddress(m_jcompare_buf_b); + delete[] static_cast(buf); + } + env->DeleteGlobalRef(m_jcompare_buf_b); + } + + if (m_jshortest_buf_start != nullptr) { + if (m_options->direct_buffer) { + void* buf = env->GetDirectBufferAddress(m_jshortest_buf_start); + delete[] static_cast(buf); + } + env->DeleteGlobalRef(m_jshortest_buf_start); + } + + if (m_jshortest_buf_limit != nullptr) { + if (m_options->direct_buffer) { + void* buf = env->GetDirectBufferAddress(m_jshortest_buf_limit); + delete[] static_cast(buf); + } + env->DeleteGlobalRef(m_jshortest_buf_limit); + } + + if (m_jshort_buf_key != nullptr) { + if (m_options->direct_buffer) { + void* buf = env->GetDirectBufferAddress(m_jshort_buf_key); + delete[] static_cast(buf); + } + env->DeleteGlobalRef(m_jshort_buf_key); + } + + if (m_tl_buf_a != nullptr) { + delete m_tl_buf_a; + } + + if (m_tl_buf_b != nullptr) { + delete m_tl_buf_b; + } + + releaseJniEnv(attached_thread); +} + +const char* ComparatorJniCallback::Name() const { return m_name.get(); } + +int ComparatorJniCallback::Compare(const Slice& a, const Slice& b) const { + jboolean attached_thread = JNI_FALSE; + JNIEnv* env = getJniEnv(&attached_thread); + assert(env != nullptr); + + const bool reuse_jbuf_a = + static_cast(a.size()) <= m_options->max_reused_buffer_size; + const bool reuse_jbuf_b = + static_cast(b.size()) <= m_options->max_reused_buffer_size; + + MaybeLockForReuse(mtx_compare, reuse_jbuf_a || reuse_jbuf_b); + + jobject jcompare_buf_a = + GetBuffer(env, a, reuse_jbuf_a, m_tl_buf_a, m_jcompare_buf_a); + if (jcompare_buf_a == nullptr) { + // exception occurred + MaybeUnlockForReuse(mtx_compare, reuse_jbuf_a || reuse_jbuf_b); + env->ExceptionDescribe(); // print out exception to stderr + releaseJniEnv(attached_thread); + return 0; + } + + jobject jcompare_buf_b = + GetBuffer(env, b, reuse_jbuf_b, m_tl_buf_b, m_jcompare_buf_b); + if (jcompare_buf_b == nullptr) { + // exception occurred + if (!reuse_jbuf_a) { + DeleteBuffer(env, jcompare_buf_a); + } + MaybeUnlockForReuse(mtx_compare, reuse_jbuf_a || reuse_jbuf_b); + env->ExceptionDescribe(); // print out exception to stderr + releaseJniEnv(attached_thread); + return 0; + } + + jint result = env->CallStaticIntMethod( + m_abstract_comparator_jni_bridge_clazz, m_jcompare_mid, m_jcallback_obj, + jcompare_buf_a, reuse_jbuf_a ? a.size() : -1, jcompare_buf_b, + reuse_jbuf_b ? b.size() : -1); + + if (env->ExceptionCheck()) { + // exception thrown from CallIntMethod + env->ExceptionDescribe(); // print out exception to stderr + result = 0; // we could not get a result from java callback so use 0 + } + + if (!reuse_jbuf_a) { + DeleteBuffer(env, jcompare_buf_a); + } + if (!reuse_jbuf_b) { + DeleteBuffer(env, jcompare_buf_b); + } + + MaybeUnlockForReuse(mtx_compare, reuse_jbuf_a || reuse_jbuf_b); + + releaseJniEnv(attached_thread); + + return result; +} + +void ComparatorJniCallback::FindShortestSeparator(std::string* start, + const Slice& limit) const { + if (start == nullptr) { + return; + } + + jboolean attached_thread = JNI_FALSE; + JNIEnv* env = getJniEnv(&attached_thread); + assert(env != nullptr); + + const bool reuse_jbuf_start = static_cast(start->length()) <= + m_options->max_reused_buffer_size; + const bool reuse_jbuf_limit = + static_cast(limit.size()) <= m_options->max_reused_buffer_size; + + MaybeLockForReuse(mtx_shortest, reuse_jbuf_start || reuse_jbuf_limit); + + Slice sstart(start->data(), start->length()); + jobject j_start_buf = GetBuffer(env, sstart, reuse_jbuf_start, m_tl_buf_a, + m_jshortest_buf_start); + if (j_start_buf == nullptr) { + // exception occurred + MaybeUnlockForReuse(mtx_shortest, reuse_jbuf_start || reuse_jbuf_limit); + env->ExceptionDescribe(); // print out exception to stderr + releaseJniEnv(attached_thread); + return; + } + + jobject j_limit_buf = GetBuffer(env, limit, reuse_jbuf_limit, m_tl_buf_b, + m_jshortest_buf_limit); + if (j_limit_buf == nullptr) { + // exception occurred + if (!reuse_jbuf_start) { + DeleteBuffer(env, j_start_buf); + } + MaybeUnlockForReuse(mtx_shortest, reuse_jbuf_start || reuse_jbuf_limit); + env->ExceptionDescribe(); // print out exception to stderr + releaseJniEnv(attached_thread); + return; + } + + jint jstart_len = env->CallStaticIntMethod( + m_abstract_comparator_jni_bridge_clazz, m_jshortest_mid, m_jcallback_obj, + j_start_buf, reuse_jbuf_start ? start->length() : -1, j_limit_buf, + reuse_jbuf_limit ? limit.size() : -1); + + if (env->ExceptionCheck()) { + // exception thrown from CallIntMethod + env->ExceptionDescribe(); // print out exception to stderr + + } else if (static_cast(jstart_len) != start->length()) { + // start buffer has changed in Java, so update `start` with the result + bool copy_from_non_direct = false; + if (reuse_jbuf_start) { + // reused a buffer + if (m_options->direct_buffer) { + // reused direct buffer + void* start_buf = env->GetDirectBufferAddress(j_start_buf); + if (start_buf == nullptr) { + if (!reuse_jbuf_start) { + DeleteBuffer(env, j_start_buf); + } + if (!reuse_jbuf_limit) { + DeleteBuffer(env, j_limit_buf); + } + MaybeUnlockForReuse(mtx_shortest, + reuse_jbuf_start || reuse_jbuf_limit); + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, "Unable to get Direct Buffer Address"); + env->ExceptionDescribe(); // print out exception to stderr + releaseJniEnv(attached_thread); + return; + } + start->assign(static_cast(start_buf), jstart_len); + + } else { + // reused non-direct buffer + copy_from_non_direct = true; + } + } else { + // there was a new buffer + if (m_options->direct_buffer) { + // it was direct... don't forget to potentially truncate the `start` + // string + start->resize(jstart_len); + } else { + // it was non-direct + copy_from_non_direct = true; + } + } + + if (copy_from_non_direct) { + jbyteArray jarray = + ByteBufferJni::array(env, j_start_buf, m_jbytebuffer_clazz); + if (jarray == nullptr) { + if (!reuse_jbuf_start) { + DeleteBuffer(env, j_start_buf); + } + if (!reuse_jbuf_limit) { + DeleteBuffer(env, j_limit_buf); + } + MaybeUnlockForReuse(mtx_shortest, reuse_jbuf_start || reuse_jbuf_limit); + env->ExceptionDescribe(); // print out exception to stderr + releaseJniEnv(attached_thread); + return; + } + jboolean has_exception = JNI_FALSE; + JniUtil::byteString( + env, jarray, + [start, jstart_len](const char* data, const size_t) { + return start->assign(data, static_cast(jstart_len)); + }, + &has_exception); + env->DeleteLocalRef(jarray); + if (has_exception == JNI_TRUE) { + if (!reuse_jbuf_start) { + DeleteBuffer(env, j_start_buf); + } + if (!reuse_jbuf_limit) { + DeleteBuffer(env, j_limit_buf); + } + env->ExceptionDescribe(); // print out exception to stderr + MaybeUnlockForReuse(mtx_shortest, reuse_jbuf_start || reuse_jbuf_limit); + releaseJniEnv(attached_thread); + return; + } + } + } + + if (!reuse_jbuf_start) { + DeleteBuffer(env, j_start_buf); + } + if (!reuse_jbuf_limit) { + DeleteBuffer(env, j_limit_buf); + } + + MaybeUnlockForReuse(mtx_shortest, reuse_jbuf_start || reuse_jbuf_limit); + + releaseJniEnv(attached_thread); +} + +void ComparatorJniCallback::FindShortSuccessor(std::string* key) const { + if (key == nullptr) { + return; + } + + jboolean attached_thread = JNI_FALSE; + JNIEnv* env = getJniEnv(&attached_thread); + assert(env != nullptr); + + const bool reuse_jbuf_key = + static_cast(key->length()) <= m_options->max_reused_buffer_size; + + MaybeLockForReuse(mtx_short, reuse_jbuf_key); + + Slice skey(key->data(), key->length()); + jobject j_key_buf = + GetBuffer(env, skey, reuse_jbuf_key, m_tl_buf_a, m_jshort_buf_key); + if (j_key_buf == nullptr) { + // exception occurred + MaybeUnlockForReuse(mtx_short, reuse_jbuf_key); + env->ExceptionDescribe(); // print out exception to stderr + releaseJniEnv(attached_thread); + return; + } + + jint jkey_len = env->CallStaticIntMethod( + m_abstract_comparator_jni_bridge_clazz, m_jshort_mid, m_jcallback_obj, + j_key_buf, reuse_jbuf_key ? key->length() : -1); + + if (env->ExceptionCheck()) { + // exception thrown from CallObjectMethod + if (!reuse_jbuf_key) { + DeleteBuffer(env, j_key_buf); + } + MaybeUnlockForReuse(mtx_short, reuse_jbuf_key); + env->ExceptionDescribe(); // print out exception to stderr + releaseJniEnv(attached_thread); + return; + } + + if (static_cast(jkey_len) != key->length()) { + // key buffer has changed in Java, so update `key` with the result + bool copy_from_non_direct = false; + if (reuse_jbuf_key) { + // reused a buffer + if (m_options->direct_buffer) { + // reused direct buffer + void* key_buf = env->GetDirectBufferAddress(j_key_buf); + if (key_buf == nullptr) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, "Unable to get Direct Buffer Address"); + if (!reuse_jbuf_key) { + DeleteBuffer(env, j_key_buf); + } + MaybeUnlockForReuse(mtx_short, reuse_jbuf_key); + env->ExceptionDescribe(); // print out exception to stderr + releaseJniEnv(attached_thread); + return; + } + key->assign(static_cast(key_buf), jkey_len); + } else { + // reused non-direct buffer + copy_from_non_direct = true; + } + } else { + // there was a new buffer + if (m_options->direct_buffer) { + // it was direct... don't forget to potentially truncate the `key` + // string + key->resize(jkey_len); + } else { + // it was non-direct + copy_from_non_direct = true; + } + } + + if (copy_from_non_direct) { + jbyteArray jarray = + ByteBufferJni::array(env, j_key_buf, m_jbytebuffer_clazz); + if (jarray == nullptr) { + if (!reuse_jbuf_key) { + DeleteBuffer(env, j_key_buf); + } + MaybeUnlockForReuse(mtx_short, reuse_jbuf_key); + env->ExceptionDescribe(); // print out exception to stderr + releaseJniEnv(attached_thread); + return; + } + jboolean has_exception = JNI_FALSE; + JniUtil::byteString( + env, jarray, + [key, jkey_len](const char* data, const size_t) { + return key->assign(data, static_cast(jkey_len)); + }, + &has_exception); + env->DeleteLocalRef(jarray); + if (has_exception == JNI_TRUE) { + if (!reuse_jbuf_key) { + DeleteBuffer(env, j_key_buf); + } + MaybeUnlockForReuse(mtx_short, reuse_jbuf_key); + env->ExceptionDescribe(); // print out exception to stderr + releaseJniEnv(attached_thread); + return; + } + } + } + + if (!reuse_jbuf_key) { + DeleteBuffer(env, j_key_buf); + } + + MaybeUnlockForReuse(mtx_short, reuse_jbuf_key); + + releaseJniEnv(attached_thread); +} + +inline void ComparatorJniCallback::MaybeLockForReuse( + const std::unique_ptr& mutex, const bool cond) const { + // no need to lock if using thread_local + if (m_options->reused_synchronisation_type != + ReusedSynchronisationType::THREAD_LOCAL && + cond) { + mutex.get()->Lock(); + } +} + +inline void ComparatorJniCallback::MaybeUnlockForReuse( + const std::unique_ptr& mutex, const bool cond) const { + // no need to unlock if using thread_local + if (m_options->reused_synchronisation_type != + ReusedSynchronisationType::THREAD_LOCAL && + cond) { + mutex.get()->Unlock(); + } +} + +jobject ComparatorJniCallback::GetBuffer(JNIEnv* env, const Slice& src, + bool reuse_buffer, + ThreadLocalPtr* tl_buf, + jobject jreuse_buffer) const { + if (reuse_buffer) { + if (m_options->reused_synchronisation_type == + ReusedSynchronisationType::THREAD_LOCAL) { + // reuse thread-local bufffer + ThreadLocalBuf* tlb = reinterpret_cast(tl_buf->Get()); + if (tlb == nullptr) { + // thread-local buffer has not yet been created, so create it + jobject jtl_buf = env->NewGlobalRef(ByteBufferJni::construct( + env, m_options->direct_buffer, m_options->max_reused_buffer_size, + m_jbytebuffer_clazz)); + if (jtl_buf == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + tlb = new ThreadLocalBuf(m_jvm, m_options->direct_buffer, jtl_buf); + tl_buf->Reset(tlb); + } + return ReuseBuffer(env, src, tlb->jbuf); + } else { + // reuse class member buffer + return ReuseBuffer(env, src, jreuse_buffer); + } + } else { + // new buffer + return NewBuffer(env, src); + } +} + +jobject ComparatorJniCallback::ReuseBuffer(JNIEnv* env, const Slice& src, + jobject jreuse_buffer) const { + // we can reuse the buffer + if (m_options->direct_buffer) { + // copy into direct buffer + void* buf = env->GetDirectBufferAddress(jreuse_buffer); + if (buf == nullptr) { + // either memory region is undefined, given object is not a direct + // java.nio.Buffer, or JNI access to direct buffers is not supported by + // this virtual machine. + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, "Unable to get Direct Buffer Address"); + return nullptr; + } + memcpy(buf, src.data(), src.size()); + } else { + // copy into non-direct buffer + const jbyteArray jarray = + ByteBufferJni::array(env, jreuse_buffer, m_jbytebuffer_clazz); + if (jarray == nullptr) { + // exception occurred + return nullptr; + } + env->SetByteArrayRegion( + jarray, 0, static_cast(src.size()), + const_cast(reinterpret_cast(src.data()))); + if (env->ExceptionCheck()) { + // exception occurred + env->DeleteLocalRef(jarray); + return nullptr; + } + env->DeleteLocalRef(jarray); + } + return jreuse_buffer; +} + +jobject ComparatorJniCallback::NewBuffer(JNIEnv* env, const Slice& src) const { + // we need a new buffer + jobject jbuf = + ByteBufferJni::constructWith(env, m_options->direct_buffer, src.data(), + src.size(), m_jbytebuffer_clazz); + if (jbuf == nullptr) { + // exception occurred + return nullptr; + } + return jbuf; +} + +void ComparatorJniCallback::DeleteBuffer(JNIEnv* env, jobject jbuffer) const { + env->DeleteLocalRef(jbuffer); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/java/rocksjni/comparatorjnicallback.h b/librocksdb-sys/rocksdb/java/rocksjni/comparatorjnicallback.h new file mode 100644 index 0000000..034c0d5 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/comparatorjnicallback.h @@ -0,0 +1,137 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the callback "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::Comparator + +#ifndef JAVA_ROCKSJNI_COMPARATORJNICALLBACK_H_ +#define JAVA_ROCKSJNI_COMPARATORJNICALLBACK_H_ + +#include + +#include +#include + +#include "port/port.h" +#include "rocksdb/comparator.h" +#include "rocksdb/slice.h" +#include "rocksjni/jnicallback.h" +#include "util/thread_local.h" + +namespace ROCKSDB_NAMESPACE { + +enum ReusedSynchronisationType { + /** + * Standard mutex. + */ + MUTEX, + + /** + * Use adaptive mutex, which spins in the user space before resorting + * to kernel. This could reduce context switch when the mutex is not + * heavily contended. However, if the mutex is hot, we could end up + * wasting spin time. + */ + ADAPTIVE_MUTEX, + + /** + * There is a reused buffer per-thread. + */ + THREAD_LOCAL +}; + +struct ComparatorJniCallbackOptions { + // Set the synchronisation type used to guard the reused buffers. + // Only used if max_reused_buffer_size > 0. + ReusedSynchronisationType reused_synchronisation_type = ADAPTIVE_MUTEX; + + // Indicates if a direct byte buffer (i.e. outside of the normal + // garbage-collected heap) is used for the callbacks to Java, + // as opposed to a non-direct byte buffer which is a wrapper around + // an on-heap byte[]. + bool direct_buffer = true; + + // Maximum size of a buffer (in bytes) that will be reused. + // Comparators will use 5 of these buffers, + // so the retained memory size will be 5 * max_reused_buffer_size. + // When a buffer is needed for transferring data to a callback, + // if it requires less than max_reused_buffer_size, then an + // existing buffer will be reused, else a new buffer will be + // allocated just for that callback. -1 to disable. + int32_t max_reused_buffer_size = 64; +}; + +/** + * This class acts as a bridge between C++ + * and Java. The methods in this class will be + * called back from the RocksDB storage engine (C++) + * we then callback to the appropriate Java method + * this enables Comparators to be implemented in Java. + * + * The design of this Comparator caches the Java Slice + * objects that are used in the compare and findShortestSeparator + * method callbacks. Instead of creating new objects for each callback + * of those functions, by reuse via setHandle we are a lot + * faster; Unfortunately this means that we have to + * introduce independent locking in regions of each of those methods + * via the mutexs mtx_compare and mtx_findShortestSeparator respectively + */ +class ComparatorJniCallback : public JniCallback, public Comparator { + public: + ComparatorJniCallback(JNIEnv* env, jobject jcomparator, + const ComparatorJniCallbackOptions* options); + ~ComparatorJniCallback(); + virtual const char* Name() const; + virtual int Compare(const Slice& a, const Slice& b) const; + virtual void FindShortestSeparator(std::string* start, + const Slice& limit) const; + virtual void FindShortSuccessor(std::string* key) const; + const std::unique_ptr m_options; + + private: + struct ThreadLocalBuf { + ThreadLocalBuf(JavaVM* _jvm, bool _direct_buffer, jobject _jbuf) + : jvm(_jvm), direct_buffer(_direct_buffer), jbuf(_jbuf) {} + JavaVM* jvm; + bool direct_buffer; + jobject jbuf; + }; + inline void MaybeLockForReuse(const std::unique_ptr& mutex, + const bool cond) const; + inline void MaybeUnlockForReuse(const std::unique_ptr& mutex, + const bool cond) const; + jobject GetBuffer(JNIEnv* env, const Slice& src, bool reuse_buffer, + ThreadLocalPtr* tl_buf, jobject jreuse_buffer) const; + jobject ReuseBuffer(JNIEnv* env, const Slice& src, + jobject jreuse_buffer) const; + jobject NewBuffer(JNIEnv* env, const Slice& src) const; + void DeleteBuffer(JNIEnv* env, jobject jbuffer) const; + // used for synchronisation in compare method + std::unique_ptr mtx_compare; + // used for synchronisation in findShortestSeparator method + std::unique_ptr mtx_shortest; + // used for synchronisation in findShortSuccessor method + std::unique_ptr mtx_short; + std::unique_ptr m_name; + jclass m_abstract_comparator_jni_bridge_clazz; // TODO(AR) could we make this + // static somehow? + jclass m_jbytebuffer_clazz; // TODO(AR) we could cache this globally for the + // entire VM if we switch more APIs to use + // ByteBuffer // TODO(AR) could we make this + // static somehow? + jmethodID m_jcompare_mid; // TODO(AR) could we make this static somehow? + jmethodID m_jshortest_mid; // TODO(AR) could we make this static somehow? + jmethodID m_jshort_mid; // TODO(AR) could we make this static somehow? + jobject m_jcompare_buf_a; + jobject m_jcompare_buf_b; + jobject m_jshortest_buf_start; + jobject m_jshortest_buf_limit; + jobject m_jshort_buf_key; + ThreadLocalPtr* m_tl_buf_a; + ThreadLocalPtr* m_tl_buf_b; +}; +} // namespace ROCKSDB_NAMESPACE + +#endif // JAVA_ROCKSJNI_COMPARATORJNICALLBACK_H_ diff --git a/librocksdb-sys/rocksdb/java/rocksjni/compression_options.cc b/librocksdb-sys/rocksdb/java/rocksjni/compression_options.cc new file mode 100644 index 0000000..53f2405 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/compression_options.cc @@ -0,0 +1,214 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::CompressionOptions. + +#include + +#include "include/org_rocksdb_CompressionOptions.h" +#include "rocksdb/advanced_options.h" +#include "rocksjni/cplusplus_to_java_convert.h" + +/* + * Class: org_rocksdb_CompressionOptions + * Method: newCompressionOptions + * Signature: ()J + */ +jlong Java_org_rocksdb_CompressionOptions_newCompressionOptions(JNIEnv*, + jclass) { + const auto* opt = new ROCKSDB_NAMESPACE::CompressionOptions(); + return GET_CPLUSPLUS_POINTER(opt); +} + +/* + * Class: org_rocksdb_CompressionOptions + * Method: setWindowBits + * Signature: (JI)V + */ +void Java_org_rocksdb_CompressionOptions_setWindowBits(JNIEnv*, jobject, + jlong jhandle, + jint jwindow_bits) { + auto* opt = reinterpret_cast(jhandle); + opt->window_bits = static_cast(jwindow_bits); +} + +/* + * Class: org_rocksdb_CompressionOptions + * Method: windowBits + * Signature: (J)I + */ +jint Java_org_rocksdb_CompressionOptions_windowBits(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->window_bits); +} + +/* + * Class: org_rocksdb_CompressionOptions + * Method: setLevel + * Signature: (JI)V + */ +void Java_org_rocksdb_CompressionOptions_setLevel(JNIEnv*, jobject, + jlong jhandle, jint jlevel) { + auto* opt = reinterpret_cast(jhandle); + opt->level = static_cast(jlevel); +} + +/* + * Class: org_rocksdb_CompressionOptions + * Method: level + * Signature: (J)I + */ +jint Java_org_rocksdb_CompressionOptions_level(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->level); +} + +/* + * Class: org_rocksdb_CompressionOptions + * Method: setStrategy + * Signature: (JI)V + */ +void Java_org_rocksdb_CompressionOptions_setStrategy(JNIEnv*, jobject, + jlong jhandle, + jint jstrategy) { + auto* opt = reinterpret_cast(jhandle); + opt->strategy = static_cast(jstrategy); +} + +/* + * Class: org_rocksdb_CompressionOptions + * Method: strategy + * Signature: (J)I + */ +jint Java_org_rocksdb_CompressionOptions_strategy(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->strategy); +} + +/* + * Class: org_rocksdb_CompressionOptions + * Method: setMaxDictBytes + * Signature: (JI)V + */ +void Java_org_rocksdb_CompressionOptions_setMaxDictBytes(JNIEnv*, jobject, + jlong jhandle, + jint jmax_dict_bytes) { + auto* opt = reinterpret_cast(jhandle); + opt->max_dict_bytes = static_cast(jmax_dict_bytes); +} + +/* + * Class: org_rocksdb_CompressionOptions + * Method: maxDictBytes + * Signature: (J)I + */ +jint Java_org_rocksdb_CompressionOptions_maxDictBytes(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->max_dict_bytes); +} + +/* + * Class: org_rocksdb_CompressionOptions + * Method: setZstdMaxTrainBytes + * Signature: (JI)V + */ +void Java_org_rocksdb_CompressionOptions_setZstdMaxTrainBytes( + JNIEnv*, jobject, jlong jhandle, jint jzstd_max_train_bytes) { + auto* opt = reinterpret_cast(jhandle); + opt->zstd_max_train_bytes = static_cast(jzstd_max_train_bytes); +} + +/* + * Class: org_rocksdb_CompressionOptions + * Method: zstdMaxTrainBytes + * Signature: (J)I + */ +jint Java_org_rocksdb_CompressionOptions_zstdMaxTrainBytes(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->zstd_max_train_bytes); +} + +/* + * Class: org_rocksdb_CompressionOptions + * Method: setMaxDictBufferBytes + * Signature: (JJ)V + */ +void Java_org_rocksdb_CompressionOptions_setMaxDictBufferBytes( + JNIEnv*, jobject, jlong jhandle, jlong jmax_dict_buffer_bytes) { + auto* opt = reinterpret_cast(jhandle); + opt->max_dict_buffer_bytes = static_cast(jmax_dict_buffer_bytes); +} + +/* + * Class: org_rocksdb_CompressionOptions + * Method: maxDictBufferBytes + * Signature: (J)J + */ +jlong Java_org_rocksdb_CompressionOptions_maxDictBufferBytes(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->max_dict_buffer_bytes); +} + +/* + * Class: org_rocksdb_CompressionOptions + * Method: setZstdMaxTrainBytes + * Signature: (JZ)V + */ +void Java_org_rocksdb_CompressionOptions_setUseZstdDictTrainer( + JNIEnv*, jobject, jlong jhandle, jboolean juse_zstd_dict_trainer) { + auto* opt = reinterpret_cast(jhandle); + opt->use_zstd_dict_trainer = juse_zstd_dict_trainer == JNI_TRUE; +} + +/* + * Class: org_rocksdb_CompressionOptions + * Method: zstdMaxTrainBytes + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_CompressionOptions_useZstdDictTrainer(JNIEnv*, + jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->use_zstd_dict_trainer); +} + +/* + * Class: org_rocksdb_CompressionOptions + * Method: setEnabled + * Signature: (JZ)V + */ +void Java_org_rocksdb_CompressionOptions_setEnabled(JNIEnv*, jobject, + jlong jhandle, + jboolean jenabled) { + auto* opt = reinterpret_cast(jhandle); + opt->enabled = jenabled == JNI_TRUE; +} + +/* + * Class: org_rocksdb_CompressionOptions + * Method: enabled + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_CompressionOptions_enabled(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->enabled); +} +/* + * Class: org_rocksdb_CompressionOptions + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_CompressionOptions_disposeInternal(JNIEnv*, jobject, + jlong jhandle) { + delete reinterpret_cast(jhandle); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/concurrent_task_limiter.cc b/librocksdb-sys/rocksdb/java/rocksjni/concurrent_task_limiter.cc new file mode 100644 index 0000000..0b0b2d2 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/concurrent_task_limiter.cc @@ -0,0 +1,97 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "rocksdb/concurrent_task_limiter.h" + +#include + +#include +#include + +#include "include/org_rocksdb_ConcurrentTaskLimiterImpl.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_ConcurrentTaskLimiterImpl + * Method: newConcurrentTaskLimiterImpl0 + * Signature: (Ljava/lang/String;I)J + */ +jlong Java_org_rocksdb_ConcurrentTaskLimiterImpl_newConcurrentTaskLimiterImpl0( + JNIEnv* env, jclass, jstring jname, jint limit) { + jboolean has_exception = JNI_FALSE; + std::string name = + ROCKSDB_NAMESPACE::JniUtil::copyStdString(env, jname, &has_exception); + if (JNI_TRUE == has_exception) { + return 0; + } + + auto* ptr = new std::shared_ptr( + ROCKSDB_NAMESPACE::NewConcurrentTaskLimiter(name, limit)); + + return GET_CPLUSPLUS_POINTER(ptr); +} + +/* + * Class: org_rocksdb_ConcurrentTaskLimiterImpl + * Method: name + * Signature: (J)Ljava/lang/String; + */ +jstring Java_org_rocksdb_ConcurrentTaskLimiterImpl_name(JNIEnv* env, jclass, + jlong handle) { + const auto& limiter = *reinterpret_cast< + std::shared_ptr*>(handle); + return ROCKSDB_NAMESPACE::JniUtil::toJavaString(env, &limiter->GetName()); +} + +/* + * Class: org_rocksdb_ConcurrentTaskLimiterImpl + * Method: setMaxOutstandingTask + * Signature: (JI)V + */ +void Java_org_rocksdb_ConcurrentTaskLimiterImpl_setMaxOutstandingTask( + JNIEnv*, jclass, jlong handle, jint max_outstanding_task) { + const auto& limiter = *reinterpret_cast< + std::shared_ptr*>(handle); + limiter->SetMaxOutstandingTask(static_cast(max_outstanding_task)); +} + +/* + * Class: org_rocksdb_ConcurrentTaskLimiterImpl + * Method: resetMaxOutstandingTask + * Signature: (J)V + */ +void Java_org_rocksdb_ConcurrentTaskLimiterImpl_resetMaxOutstandingTask( + JNIEnv*, jclass, jlong handle) { + const auto& limiter = *reinterpret_cast< + std::shared_ptr*>(handle); + limiter->ResetMaxOutstandingTask(); +} + +/* + * Class: org_rocksdb_ConcurrentTaskLimiterImpl + * Method: outstandingTask + * Signature: (J)I + */ +jint Java_org_rocksdb_ConcurrentTaskLimiterImpl_outstandingTask(JNIEnv*, jclass, + jlong handle) { + const auto& limiter = *reinterpret_cast< + std::shared_ptr*>(handle); + return static_cast(limiter->GetOutstandingTask()); +} + +/* + * Class: org_rocksdb_ConcurrentTaskLimiterImpl + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_ConcurrentTaskLimiterImpl_disposeInternal(JNIEnv*, + jobject, + jlong jhandle) { + auto* ptr = reinterpret_cast< + std::shared_ptr*>(jhandle); + delete ptr; // delete std::shared_ptr +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/config_options.cc b/librocksdb-sys/rocksdb/java/rocksjni/config_options.cc new file mode 100644 index 0000000..55a9cbb --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/config_options.cc @@ -0,0 +1,103 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling C++ ROCKSDB_NAMESPACE::ConfigOptions methods +// from Java side. + +#include + +#include "include/org_rocksdb_ConfigOptions.h" +#include "rocksdb/convenience.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_ConfigOptions + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_ConfigOptions_disposeInternal(JNIEnv *, jobject, + jlong jhandle) { + auto *co = reinterpret_cast(jhandle); + assert(co != nullptr); + delete co; +} + +/* + * Class: org_rocksdb_ConfigOptions + * Method: newConfigOptions + * Signature: ()J + */ +jlong Java_org_rocksdb_ConfigOptions_newConfigOptions(JNIEnv *, jclass) { + auto *cfg_opt = new ROCKSDB_NAMESPACE::ConfigOptions(); + return GET_CPLUSPLUS_POINTER(cfg_opt); +} + +/* + * Class: org_rocksdb_ConfigOptions + * Method: setEnv + * Signature: (JJ;)V + */ +void Java_org_rocksdb_ConfigOptions_setEnv(JNIEnv *, jclass, jlong handle, + jlong rocksdb_env_handle) { + auto *cfg_opt = reinterpret_cast(handle); + auto *rocksdb_env = + reinterpret_cast(rocksdb_env_handle); + cfg_opt->env = rocksdb_env; +} + +/* + * Class: org_rocksdb_ConfigOptions + * Method: setDelimiter + * Signature: (JLjava/lang/String;)V + */ +void Java_org_rocksdb_ConfigOptions_setDelimiter(JNIEnv *env, jclass, + jlong handle, jstring s) { + auto *cfg_opt = reinterpret_cast(handle); + const char *delim = env->GetStringUTFChars(s, nullptr); + if (delim == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + cfg_opt->delimiter = delim; + env->ReleaseStringUTFChars(s, delim); +} + +/* + * Class: org_rocksdb_ConfigOptions + * Method: setIgnoreUnknownOptions + * Signature: (JZ)V + */ +void Java_org_rocksdb_ConfigOptions_setIgnoreUnknownOptions(JNIEnv *, jclass, + jlong handle, + jboolean b) { + auto *cfg_opt = reinterpret_cast(handle); + cfg_opt->ignore_unknown_options = static_cast(b); +} + +/* + * Class: org_rocksdb_ConfigOptions + * Method: setInputStringsEscaped + * Signature: (JZ)V + */ +void Java_org_rocksdb_ConfigOptions_setInputStringsEscaped(JNIEnv *, jclass, + jlong handle, + jboolean b) { + auto *cfg_opt = reinterpret_cast(handle); + cfg_opt->input_strings_escaped = static_cast(b); +} + +/* + * Class: org_rocksdb_ConfigOptions + * Method: setSanityLevel + * Signature: (JI)V + */ +void Java_org_rocksdb_ConfigOptions_setSanityLevel(JNIEnv *, jclass, + jlong handle, jbyte level) { + auto *cfg_opt = reinterpret_cast(handle); + cfg_opt->sanity_level = + ROCKSDB_NAMESPACE::SanityLevelJni::toCppSanityLevel(level); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/cplusplus_to_java_convert.h b/librocksdb-sys/rocksdb/java/rocksjni/cplusplus_to_java_convert.h new file mode 100644 index 0000000..0eea6fa --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/cplusplus_to_java_convert.h @@ -0,0 +1,37 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +/* + * This macro is used for 32 bit OS. In 32 bit OS, the result number is a + negative number if we use reinterpret_cast(pointer). + * For example, jlong ptr = reinterpret_cast(pointer), ptr is a negative + number in 32 bit OS. + * If we check ptr using ptr > 0, it fails. For example, the following code is + not correct. + * if (jblock_cache_handle > 0) { + std::shared_ptr *pCache = + reinterpret_cast *>( + jblock_cache_handle); + options.block_cache = *pCache; + } + * But the result number is positive number if we do + reinterpret_cast(pointer) first and then cast it to jlong. size_t is 4 + bytes long in 32 bit OS and 8 bytes long in 64 bit OS. + static_cast(reinterpret_cast(_pointer)) is also working in 64 + bit OS. + * + * We don't need an opposite cast because it works from jlong to c++ pointer in + both 32 bit and 64 bit OS. + * For example, the following code is working in both 32 bit and 64 bit OS. + jblock_cache_handle is jlong. + * std::shared_ptr *pCache = + reinterpret_cast *>( + jblock_cache_handle); +*/ + +#define GET_CPLUSPLUS_POINTER(_pointer) \ + static_cast(reinterpret_cast(_pointer)) diff --git a/librocksdb-sys/rocksdb/java/rocksjni/env.cc b/librocksdb-sys/rocksdb/java/rocksjni/env.cc new file mode 100644 index 0000000..bb739fe --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/env.cc @@ -0,0 +1,205 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling c++ ROCKSDB_NAMESPACE::Env methods from Java side. + +#include "rocksdb/env.h" + +#include + +#include + +#include "include/org_rocksdb_Env.h" +#include "include/org_rocksdb_RocksEnv.h" +#include "include/org_rocksdb_RocksMemEnv.h" +#include "include/org_rocksdb_TimedEnv.h" +#include "portal.h" +#include "rocksjni/cplusplus_to_java_convert.h" + +/* + * Class: org_rocksdb_Env + * Method: getDefaultEnvInternal + * Signature: ()J + */ +jlong Java_org_rocksdb_Env_getDefaultEnvInternal(JNIEnv*, jclass) { + return GET_CPLUSPLUS_POINTER(ROCKSDB_NAMESPACE::Env::Default()); +} + +/* + * Class: org_rocksdb_RocksEnv + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_RocksEnv_disposeInternal(JNIEnv*, jobject, + jlong jhandle) { + auto* e = reinterpret_cast(jhandle); + assert(e != nullptr); + delete e; +} + +/* + * Class: org_rocksdb_Env + * Method: setBackgroundThreads + * Signature: (JIB)V + */ +void Java_org_rocksdb_Env_setBackgroundThreads(JNIEnv*, jobject, jlong jhandle, + jint jnum, + jbyte jpriority_value) { + auto* rocks_env = reinterpret_cast(jhandle); + rocks_env->SetBackgroundThreads( + static_cast(jnum), + ROCKSDB_NAMESPACE::PriorityJni::toCppPriority(jpriority_value)); +} + +/* + * Class: org_rocksdb_Env + * Method: getBackgroundThreads + * Signature: (JB)I + */ +jint Java_org_rocksdb_Env_getBackgroundThreads(JNIEnv*, jobject, jlong jhandle, + jbyte jpriority_value) { + auto* rocks_env = reinterpret_cast(jhandle); + const int num = rocks_env->GetBackgroundThreads( + ROCKSDB_NAMESPACE::PriorityJni::toCppPriority(jpriority_value)); + return static_cast(num); +} + +/* + * Class: org_rocksdb_Env + * Method: getThreadPoolQueueLen + * Signature: (JB)I + */ +jint Java_org_rocksdb_Env_getThreadPoolQueueLen(JNIEnv*, jobject, jlong jhandle, + jbyte jpriority_value) { + auto* rocks_env = reinterpret_cast(jhandle); + const int queue_len = rocks_env->GetThreadPoolQueueLen( + ROCKSDB_NAMESPACE::PriorityJni::toCppPriority(jpriority_value)); + return static_cast(queue_len); +} + +/* + * Class: org_rocksdb_Env + * Method: incBackgroundThreadsIfNeeded + * Signature: (JIB)V + */ +void Java_org_rocksdb_Env_incBackgroundThreadsIfNeeded(JNIEnv*, jobject, + jlong jhandle, jint jnum, + jbyte jpriority_value) { + auto* rocks_env = reinterpret_cast(jhandle); + rocks_env->IncBackgroundThreadsIfNeeded( + static_cast(jnum), + ROCKSDB_NAMESPACE::PriorityJni::toCppPriority(jpriority_value)); +} + +/* + * Class: org_rocksdb_Env + * Method: lowerThreadPoolIOPriority + * Signature: (JB)V + */ +void Java_org_rocksdb_Env_lowerThreadPoolIOPriority(JNIEnv*, jobject, + jlong jhandle, + jbyte jpriority_value) { + auto* rocks_env = reinterpret_cast(jhandle); + rocks_env->LowerThreadPoolIOPriority( + ROCKSDB_NAMESPACE::PriorityJni::toCppPriority(jpriority_value)); +} + +/* + * Class: org_rocksdb_Env + * Method: lowerThreadPoolCPUPriority + * Signature: (JB)V + */ +void Java_org_rocksdb_Env_lowerThreadPoolCPUPriority(JNIEnv*, jobject, + jlong jhandle, + jbyte jpriority_value) { + auto* rocks_env = reinterpret_cast(jhandle); + rocks_env->LowerThreadPoolCPUPriority( + ROCKSDB_NAMESPACE::PriorityJni::toCppPriority(jpriority_value)); +} + +/* + * Class: org_rocksdb_Env + * Method: getThreadList + * Signature: (J)[Lorg/rocksdb/ThreadStatus; + */ +jobjectArray Java_org_rocksdb_Env_getThreadList(JNIEnv* env, jobject, + jlong jhandle) { + auto* rocks_env = reinterpret_cast(jhandle); + std::vector thread_status; + ROCKSDB_NAMESPACE::Status s = rocks_env->GetThreadList(&thread_status); + if (!s.ok()) { + // error, throw exception + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return nullptr; + } + + // object[] + const jsize len = static_cast(thread_status.size()); + jobjectArray jthread_status = env->NewObjectArray( + len, ROCKSDB_NAMESPACE::ThreadStatusJni::getJClass(env), nullptr); + if (jthread_status == nullptr) { + // an exception occurred + return nullptr; + } + for (jsize i = 0; i < len; ++i) { + jobject jts = + ROCKSDB_NAMESPACE::ThreadStatusJni::construct(env, &(thread_status[i])); + env->SetObjectArrayElement(jthread_status, i, jts); + if (env->ExceptionCheck()) { + // exception occurred + env->DeleteLocalRef(jthread_status); + return nullptr; + } + } + + return jthread_status; +} + +/* + * Class: org_rocksdb_RocksMemEnv + * Method: createMemEnv + * Signature: (J)J + */ +jlong Java_org_rocksdb_RocksMemEnv_createMemEnv(JNIEnv*, jclass, + jlong jbase_env_handle) { + auto* base_env = reinterpret_cast(jbase_env_handle); + return GET_CPLUSPLUS_POINTER(ROCKSDB_NAMESPACE::NewMemEnv(base_env)); +} + +/* + * Class: org_rocksdb_RocksMemEnv + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_RocksMemEnv_disposeInternal(JNIEnv*, jobject, + jlong jhandle) { + auto* e = reinterpret_cast(jhandle); + assert(e != nullptr); + delete e; +} + +/* + * Class: org_rocksdb_TimedEnv + * Method: createTimedEnv + * Signature: (J)J + */ +jlong Java_org_rocksdb_TimedEnv_createTimedEnv(JNIEnv*, jclass, + jlong jbase_env_handle) { + auto* base_env = reinterpret_cast(jbase_env_handle); + return GET_CPLUSPLUS_POINTER(ROCKSDB_NAMESPACE::NewTimedEnv(base_env)); +} + +/* + * Class: org_rocksdb_TimedEnv + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_TimedEnv_disposeInternal(JNIEnv*, jobject, + jlong jhandle) { + auto* e = reinterpret_cast(jhandle); + assert(e != nullptr); + delete e; +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/env_options.cc b/librocksdb-sys/rocksdb/java/rocksjni/env_options.cc new file mode 100644 index 0000000..3237e27 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/env_options.cc @@ -0,0 +1,305 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling C++ ROCKSDB_NAMESPACE::EnvOptions methods +// from Java side. + +#include + +#include "include/org_rocksdb_EnvOptions.h" +#include "rocksdb/env.h" +#include "rocksjni/cplusplus_to_java_convert.h" + +#define ENV_OPTIONS_SET_BOOL(_jhandle, _opt) \ + reinterpret_cast(_jhandle)->_opt = \ + static_cast(_opt) + +#define ENV_OPTIONS_SET_SIZE_T(_jhandle, _opt) \ + reinterpret_cast(_jhandle)->_opt = \ + static_cast(_opt) + +#define ENV_OPTIONS_SET_UINT64_T(_jhandle, _opt) \ + reinterpret_cast(_jhandle)->_opt = \ + static_cast(_opt) + +#define ENV_OPTIONS_GET(_jhandle, _opt) \ + reinterpret_cast(_jhandle)->_opt + +/* + * Class: org_rocksdb_EnvOptions + * Method: newEnvOptions + * Signature: ()J + */ +jlong Java_org_rocksdb_EnvOptions_newEnvOptions__(JNIEnv *, jclass) { + auto *env_opt = new ROCKSDB_NAMESPACE::EnvOptions(); + return GET_CPLUSPLUS_POINTER(env_opt); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: newEnvOptions + * Signature: (J)J + */ +jlong Java_org_rocksdb_EnvOptions_newEnvOptions__J(JNIEnv *, jclass, + jlong jdboptions_handle) { + auto *db_options = + reinterpret_cast(jdboptions_handle); + auto *env_opt = new ROCKSDB_NAMESPACE::EnvOptions(*db_options); + return GET_CPLUSPLUS_POINTER(env_opt); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_EnvOptions_disposeInternal(JNIEnv *, jobject, + jlong jhandle) { + auto *eo = reinterpret_cast(jhandle); + assert(eo != nullptr); + delete eo; +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: setUseMmapReads + * Signature: (JZ)V + */ +void Java_org_rocksdb_EnvOptions_setUseMmapReads(JNIEnv *, jobject, + jlong jhandle, + jboolean use_mmap_reads) { + ENV_OPTIONS_SET_BOOL(jhandle, use_mmap_reads); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: useMmapReads + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_EnvOptions_useMmapReads(JNIEnv *, jobject, + jlong jhandle) { + return ENV_OPTIONS_GET(jhandle, use_mmap_reads); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: setUseMmapWrites + * Signature: (JZ)V + */ +void Java_org_rocksdb_EnvOptions_setUseMmapWrites(JNIEnv *, jobject, + jlong jhandle, + jboolean use_mmap_writes) { + ENV_OPTIONS_SET_BOOL(jhandle, use_mmap_writes); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: useMmapWrites + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_EnvOptions_useMmapWrites(JNIEnv *, jobject, + jlong jhandle) { + return ENV_OPTIONS_GET(jhandle, use_mmap_writes); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: setUseDirectReads + * Signature: (JZ)V + */ +void Java_org_rocksdb_EnvOptions_setUseDirectReads(JNIEnv *, jobject, + jlong jhandle, + jboolean use_direct_reads) { + ENV_OPTIONS_SET_BOOL(jhandle, use_direct_reads); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: useDirectReads + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_EnvOptions_useDirectReads(JNIEnv *, jobject, + jlong jhandle) { + return ENV_OPTIONS_GET(jhandle, use_direct_reads); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: setUseDirectWrites + * Signature: (JZ)V + */ +void Java_org_rocksdb_EnvOptions_setUseDirectWrites( + JNIEnv *, jobject, jlong jhandle, jboolean use_direct_writes) { + ENV_OPTIONS_SET_BOOL(jhandle, use_direct_writes); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: useDirectWrites + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_EnvOptions_useDirectWrites(JNIEnv *, jobject, + jlong jhandle) { + return ENV_OPTIONS_GET(jhandle, use_direct_writes); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: setAllowFallocate + * Signature: (JZ)V + */ +void Java_org_rocksdb_EnvOptions_setAllowFallocate(JNIEnv *, jobject, + jlong jhandle, + jboolean allow_fallocate) { + ENV_OPTIONS_SET_BOOL(jhandle, allow_fallocate); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: allowFallocate + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_EnvOptions_allowFallocate(JNIEnv *, jobject, + jlong jhandle) { + return ENV_OPTIONS_GET(jhandle, allow_fallocate); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: setSetFdCloexec + * Signature: (JZ)V + */ +void Java_org_rocksdb_EnvOptions_setSetFdCloexec(JNIEnv *, jobject, + jlong jhandle, + jboolean set_fd_cloexec) { + ENV_OPTIONS_SET_BOOL(jhandle, set_fd_cloexec); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: setFdCloexec + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_EnvOptions_setFdCloexec(JNIEnv *, jobject, + jlong jhandle) { + return ENV_OPTIONS_GET(jhandle, set_fd_cloexec); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: setBytesPerSync + * Signature: (JJ)V + */ +void Java_org_rocksdb_EnvOptions_setBytesPerSync(JNIEnv *, jobject, + jlong jhandle, + jlong bytes_per_sync) { + ENV_OPTIONS_SET_UINT64_T(jhandle, bytes_per_sync); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: bytesPerSync + * Signature: (J)J + */ +jlong Java_org_rocksdb_EnvOptions_bytesPerSync(JNIEnv *, jobject, + jlong jhandle) { + return ENV_OPTIONS_GET(jhandle, bytes_per_sync); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: setFallocateWithKeepSize + * Signature: (JZ)V + */ +void Java_org_rocksdb_EnvOptions_setFallocateWithKeepSize( + JNIEnv *, jobject, jlong jhandle, jboolean fallocate_with_keep_size) { + ENV_OPTIONS_SET_BOOL(jhandle, fallocate_with_keep_size); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: fallocateWithKeepSize + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_EnvOptions_fallocateWithKeepSize(JNIEnv *, jobject, + jlong jhandle) { + return ENV_OPTIONS_GET(jhandle, fallocate_with_keep_size); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: setCompactionReadaheadSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_EnvOptions_setCompactionReadaheadSize( + JNIEnv *, jobject, jlong jhandle, jlong compaction_readahead_size) { + ENV_OPTIONS_SET_SIZE_T(jhandle, compaction_readahead_size); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: compactionReadaheadSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_EnvOptions_compactionReadaheadSize(JNIEnv *, jobject, + jlong jhandle) { + return ENV_OPTIONS_GET(jhandle, compaction_readahead_size); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: setRandomAccessMaxBufferSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_EnvOptions_setRandomAccessMaxBufferSize( + JNIEnv *, jobject, jlong jhandle, jlong random_access_max_buffer_size) { + ENV_OPTIONS_SET_SIZE_T(jhandle, random_access_max_buffer_size); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: randomAccessMaxBufferSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_EnvOptions_randomAccessMaxBufferSize(JNIEnv *, jobject, + jlong jhandle) { + return ENV_OPTIONS_GET(jhandle, random_access_max_buffer_size); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: setWritableFileMaxBufferSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_EnvOptions_setWritableFileMaxBufferSize( + JNIEnv *, jobject, jlong jhandle, jlong writable_file_max_buffer_size) { + ENV_OPTIONS_SET_SIZE_T(jhandle, writable_file_max_buffer_size); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: writableFileMaxBufferSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_EnvOptions_writableFileMaxBufferSize(JNIEnv *, jobject, + jlong jhandle) { + return ENV_OPTIONS_GET(jhandle, writable_file_max_buffer_size); +} + +/* + * Class: org_rocksdb_EnvOptions + * Method: setRateLimiter + * Signature: (JJ)V + */ +void Java_org_rocksdb_EnvOptions_setRateLimiter(JNIEnv *, jobject, + jlong jhandle, + jlong rl_handle) { + auto *sptr_rate_limiter = + reinterpret_cast *>( + rl_handle); + auto *env_opt = reinterpret_cast(jhandle); + env_opt->rate_limiter = sptr_rate_limiter->get(); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/event_listener.cc b/librocksdb-sys/rocksdb/java/rocksjni/event_listener.cc new file mode 100644 index 0000000..965932c --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/event_listener.cc @@ -0,0 +1,44 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::EventListener. + +#include + +#include + +#include "include/org_rocksdb_AbstractEventListener.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/event_listener_jnicallback.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_AbstractEventListener + * Method: createNewEventListener + * Signature: (J)J + */ +jlong Java_org_rocksdb_AbstractEventListener_createNewEventListener( + JNIEnv* env, jobject jobj, jlong jenabled_event_callback_values) { + auto enabled_event_callbacks = + ROCKSDB_NAMESPACE::EnabledEventCallbackJni::toCppEnabledEventCallbacks( + jenabled_event_callback_values); + auto* sptr_event_listener = + new std::shared_ptr( + new ROCKSDB_NAMESPACE::EventListenerJniCallback( + env, jobj, enabled_event_callbacks)); + return GET_CPLUSPLUS_POINTER(sptr_event_listener); +} + +/* + * Class: org_rocksdb_AbstractEventListener + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_AbstractEventListener_disposeInternal(JNIEnv*, jobject, + jlong jhandle) { + delete reinterpret_cast*>( + jhandle); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/event_listener_jnicallback.cc b/librocksdb-sys/rocksdb/java/rocksjni/event_listener_jnicallback.cc new file mode 100644 index 0000000..342d938 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/event_listener_jnicallback.cc @@ -0,0 +1,502 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the callback "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::EventListener. + +#include "rocksjni/event_listener_jnicallback.h" + +#include "rocksjni/portal.h" + +namespace ROCKSDB_NAMESPACE { +EventListenerJniCallback::EventListenerJniCallback( + JNIEnv* env, jobject jevent_listener, + const std::set& enabled_event_callbacks) + : JniCallback(env, jevent_listener), + m_enabled_event_callbacks(enabled_event_callbacks) { + InitCallbackMethodId( + m_on_flush_completed_proxy_mid, EnabledEventCallback::ON_FLUSH_COMPLETED, + env, AbstractEventListenerJni::getOnFlushCompletedProxyMethodId); + + InitCallbackMethodId(m_on_flush_begin_proxy_mid, + EnabledEventCallback::ON_FLUSH_BEGIN, env, + AbstractEventListenerJni::getOnFlushBeginProxyMethodId); + + InitCallbackMethodId(m_on_table_file_deleted_mid, + EnabledEventCallback::ON_TABLE_FILE_DELETED, env, + AbstractEventListenerJni::getOnTableFileDeletedMethodId); + + InitCallbackMethodId( + m_on_compaction_begin_proxy_mid, + EnabledEventCallback::ON_COMPACTION_BEGIN, env, + AbstractEventListenerJni::getOnCompactionBeginProxyMethodId); + + InitCallbackMethodId( + m_on_compaction_completed_proxy_mid, + EnabledEventCallback::ON_COMPACTION_COMPLETED, env, + AbstractEventListenerJni::getOnCompactionCompletedProxyMethodId); + + InitCallbackMethodId(m_on_table_file_created_mid, + EnabledEventCallback::ON_TABLE_FILE_CREATED, env, + AbstractEventListenerJni::getOnTableFileCreatedMethodId); + + InitCallbackMethodId( + m_on_table_file_creation_started_mid, + EnabledEventCallback::ON_TABLE_FILE_CREATION_STARTED, env, + AbstractEventListenerJni::getOnTableFileCreationStartedMethodId); + + InitCallbackMethodId(m_on_mem_table_sealed_mid, + EnabledEventCallback::ON_MEMTABLE_SEALED, env, + AbstractEventListenerJni::getOnMemTableSealedMethodId); + + InitCallbackMethodId( + m_on_column_family_handle_deletion_started_mid, + EnabledEventCallback::ON_COLUMN_FAMILY_HANDLE_DELETION_STARTED, env, + AbstractEventListenerJni::getOnColumnFamilyHandleDeletionStartedMethodId); + + InitCallbackMethodId( + m_on_external_file_ingested_proxy_mid, + EnabledEventCallback::ON_EXTERNAL_FILE_INGESTED, env, + AbstractEventListenerJni::getOnExternalFileIngestedProxyMethodId); + + InitCallbackMethodId( + m_on_background_error_proxy_mid, + EnabledEventCallback::ON_BACKGROUND_ERROR, env, + AbstractEventListenerJni::getOnBackgroundErrorProxyMethodId); + + InitCallbackMethodId( + m_on_stall_conditions_changed_mid, + EnabledEventCallback::ON_STALL_CONDITIONS_CHANGED, env, + AbstractEventListenerJni::getOnStallConditionsChangedMethodId); + + InitCallbackMethodId(m_on_file_read_finish_mid, + EnabledEventCallback::ON_FILE_READ_FINISH, env, + AbstractEventListenerJni::getOnFileReadFinishMethodId); + + InitCallbackMethodId(m_on_file_write_finish_mid, + EnabledEventCallback::ON_FILE_WRITE_FINISH, env, + AbstractEventListenerJni::getOnFileWriteFinishMethodId); + + InitCallbackMethodId(m_on_file_flush_finish_mid, + EnabledEventCallback::ON_FILE_FLUSH_FINISH, env, + AbstractEventListenerJni::getOnFileFlushFinishMethodId); + + InitCallbackMethodId(m_on_file_sync_finish_mid, + EnabledEventCallback::ON_FILE_SYNC_FINISH, env, + AbstractEventListenerJni::getOnFileSyncFinishMethodId); + + InitCallbackMethodId( + m_on_file_range_sync_finish_mid, + EnabledEventCallback::ON_FILE_RANGE_SYNC_FINISH, env, + AbstractEventListenerJni::getOnFileRangeSyncFinishMethodId); + + InitCallbackMethodId( + m_on_file_truncate_finish_mid, + EnabledEventCallback::ON_FILE_TRUNCATE_FINISH, env, + AbstractEventListenerJni::getOnFileTruncateFinishMethodId); + + InitCallbackMethodId(m_on_file_close_finish_mid, + EnabledEventCallback::ON_FILE_CLOSE_FINISH, env, + AbstractEventListenerJni::getOnFileCloseFinishMethodId); + + InitCallbackMethodId( + m_should_be_notified_on_file_io, + EnabledEventCallback::SHOULD_BE_NOTIFIED_ON_FILE_IO, env, + AbstractEventListenerJni::getShouldBeNotifiedOnFileIOMethodId); + + InitCallbackMethodId( + m_on_error_recovery_begin_proxy_mid, + EnabledEventCallback::ON_ERROR_RECOVERY_BEGIN, env, + AbstractEventListenerJni::getOnErrorRecoveryBeginProxyMethodId); + + InitCallbackMethodId( + m_on_error_recovery_completed_mid, + EnabledEventCallback::ON_ERROR_RECOVERY_COMPLETED, env, + AbstractEventListenerJni::getOnErrorRecoveryCompletedMethodId); +} + +EventListenerJniCallback::~EventListenerJniCallback() {} + +void EventListenerJniCallback::OnFlushCompleted( + DB* db, const FlushJobInfo& flush_job_info) { + if (m_on_flush_completed_proxy_mid == nullptr) { + return; + } + + JNIEnv* env; + jboolean attached_thread; + jobject jflush_job_info = SetupCallbackInvocation( + env, attached_thread, flush_job_info, + FlushJobInfoJni::fromCppFlushJobInfo); + + if (jflush_job_info != nullptr) { + env->CallVoidMethod(m_jcallback_obj, m_on_flush_completed_proxy_mid, + reinterpret_cast(db), jflush_job_info); + } + + CleanupCallbackInvocation(env, attached_thread, {&jflush_job_info}); +} + +void EventListenerJniCallback::OnFlushBegin( + DB* db, const FlushJobInfo& flush_job_info) { + if (m_on_flush_begin_proxy_mid == nullptr) { + return; + } + + JNIEnv* env; + jboolean attached_thread; + jobject jflush_job_info = SetupCallbackInvocation( + env, attached_thread, flush_job_info, + FlushJobInfoJni::fromCppFlushJobInfo); + + if (jflush_job_info != nullptr) { + env->CallVoidMethod(m_jcallback_obj, m_on_flush_begin_proxy_mid, + reinterpret_cast(db), jflush_job_info); + } + + CleanupCallbackInvocation(env, attached_thread, {&jflush_job_info}); +} + +void EventListenerJniCallback::OnTableFileDeleted( + const TableFileDeletionInfo& info) { + if (m_on_table_file_deleted_mid == nullptr) { + return; + } + + JNIEnv* env; + jboolean attached_thread; + jobject jdeletion_info = SetupCallbackInvocation( + env, attached_thread, info, + TableFileDeletionInfoJni::fromCppTableFileDeletionInfo); + + if (jdeletion_info != nullptr) { + env->CallVoidMethod(m_jcallback_obj, m_on_table_file_deleted_mid, + jdeletion_info); + } + + CleanupCallbackInvocation(env, attached_thread, {&jdeletion_info}); +} + +void EventListenerJniCallback::OnCompactionBegin(DB* db, + const CompactionJobInfo& ci) { + if (m_on_compaction_begin_proxy_mid == nullptr) { + return; + } + + JNIEnv* env; + jboolean attached_thread; + jobject jcompaction_job_info = SetupCallbackInvocation( + env, attached_thread, ci, CompactionJobInfoJni::fromCppCompactionJobInfo); + + if (jcompaction_job_info != nullptr) { + env->CallVoidMethod(m_jcallback_obj, m_on_compaction_begin_proxy_mid, + reinterpret_cast(db), jcompaction_job_info); + } + + CleanupCallbackInvocation(env, attached_thread, {&jcompaction_job_info}); +} + +void EventListenerJniCallback::OnCompactionCompleted( + DB* db, const CompactionJobInfo& ci) { + if (m_on_compaction_completed_proxy_mid == nullptr) { + return; + } + + JNIEnv* env; + jboolean attached_thread; + jobject jcompaction_job_info = SetupCallbackInvocation( + env, attached_thread, ci, CompactionJobInfoJni::fromCppCompactionJobInfo); + + if (jcompaction_job_info != nullptr) { + env->CallVoidMethod(m_jcallback_obj, m_on_compaction_completed_proxy_mid, + reinterpret_cast(db), jcompaction_job_info); + } + + CleanupCallbackInvocation(env, attached_thread, {&jcompaction_job_info}); +} + +void EventListenerJniCallback::OnTableFileCreated( + const TableFileCreationInfo& info) { + if (m_on_table_file_created_mid == nullptr) { + return; + } + + JNIEnv* env; + jboolean attached_thread; + jobject jfile_creation_info = SetupCallbackInvocation( + env, attached_thread, info, + TableFileCreationInfoJni::fromCppTableFileCreationInfo); + + if (jfile_creation_info != nullptr) { + env->CallVoidMethod(m_jcallback_obj, m_on_table_file_created_mid, + jfile_creation_info); + } + + CleanupCallbackInvocation(env, attached_thread, {&jfile_creation_info}); +} + +void EventListenerJniCallback::OnTableFileCreationStarted( + const TableFileCreationBriefInfo& info) { + if (m_on_table_file_creation_started_mid == nullptr) { + return; + } + + JNIEnv* env; + jboolean attached_thread; + jobject jcreation_brief_info = + SetupCallbackInvocation( + env, attached_thread, info, + TableFileCreationBriefInfoJni::fromCppTableFileCreationBriefInfo); + + if (jcreation_brief_info != nullptr) { + env->CallVoidMethod(m_jcallback_obj, m_on_table_file_creation_started_mid, + jcreation_brief_info); + } + + CleanupCallbackInvocation(env, attached_thread, {&jcreation_brief_info}); +} + +void EventListenerJniCallback::OnMemTableSealed(const MemTableInfo& info) { + if (m_on_mem_table_sealed_mid == nullptr) { + return; + } + + JNIEnv* env; + jboolean attached_thread; + jobject jmem_table_info = SetupCallbackInvocation( + env, attached_thread, info, MemTableInfoJni::fromCppMemTableInfo); + + if (jmem_table_info != nullptr) { + env->CallVoidMethod(m_jcallback_obj, m_on_mem_table_sealed_mid, + jmem_table_info); + } + + CleanupCallbackInvocation(env, attached_thread, {&jmem_table_info}); +} + +void EventListenerJniCallback::OnColumnFamilyHandleDeletionStarted( + ColumnFamilyHandle* handle) { + if (m_on_column_family_handle_deletion_started_mid == nullptr) { + return; + } + + JNIEnv* env; + jboolean attached_thread; + jobject jcf_handle = SetupCallbackInvocation( + env, attached_thread, *handle, + ColumnFamilyHandleJni::fromCppColumnFamilyHandle); + + if (jcf_handle != nullptr) { + env->CallVoidMethod(m_jcallback_obj, + m_on_column_family_handle_deletion_started_mid, + jcf_handle); + } + + CleanupCallbackInvocation(env, attached_thread, {&jcf_handle}); +} + +void EventListenerJniCallback::OnExternalFileIngested( + DB* db, const ExternalFileIngestionInfo& info) { + if (m_on_external_file_ingested_proxy_mid == nullptr) { + return; + } + + JNIEnv* env; + jboolean attached_thread; + jobject jingestion_info = SetupCallbackInvocation( + env, attached_thread, info, + ExternalFileIngestionInfoJni::fromCppExternalFileIngestionInfo); + + if (jingestion_info != nullptr) { + env->CallVoidMethod(m_jcallback_obj, m_on_external_file_ingested_proxy_mid, + reinterpret_cast(db), jingestion_info); + } + + CleanupCallbackInvocation(env, attached_thread, {&jingestion_info}); +} + +void EventListenerJniCallback::OnBackgroundError(BackgroundErrorReason reason, + Status* bg_error) { + if (m_on_background_error_proxy_mid == nullptr) { + return; + } + + JNIEnv* env; + jboolean attached_thread; + jobject jstatus = SetupCallbackInvocation( + env, attached_thread, *bg_error, StatusJni::construct); + + if (jstatus != nullptr) { + env->CallVoidMethod(m_jcallback_obj, m_on_background_error_proxy_mid, + static_cast(reason), jstatus); + } + + CleanupCallbackInvocation(env, attached_thread, {&jstatus}); +} + +void EventListenerJniCallback::OnStallConditionsChanged( + const WriteStallInfo& info) { + if (m_on_stall_conditions_changed_mid == nullptr) { + return; + } + + JNIEnv* env; + jboolean attached_thread; + jobject jwrite_stall_info = SetupCallbackInvocation( + env, attached_thread, info, WriteStallInfoJni::fromCppWriteStallInfo); + + if (jwrite_stall_info != nullptr) { + env->CallVoidMethod(m_jcallback_obj, m_on_stall_conditions_changed_mid, + jwrite_stall_info); + } + + CleanupCallbackInvocation(env, attached_thread, {&jwrite_stall_info}); +} + +void EventListenerJniCallback::OnFileReadFinish(const FileOperationInfo& info) { + OnFileOperation(m_on_file_read_finish_mid, info); +} + +void EventListenerJniCallback::OnFileWriteFinish( + const FileOperationInfo& info) { + OnFileOperation(m_on_file_write_finish_mid, info); +} + +void EventListenerJniCallback::OnFileFlushFinish( + const FileOperationInfo& info) { + OnFileOperation(m_on_file_flush_finish_mid, info); +} + +void EventListenerJniCallback::OnFileSyncFinish(const FileOperationInfo& info) { + OnFileOperation(m_on_file_sync_finish_mid, info); +} + +void EventListenerJniCallback::OnFileRangeSyncFinish( + const FileOperationInfo& info) { + OnFileOperation(m_on_file_range_sync_finish_mid, info); +} + +void EventListenerJniCallback::OnFileTruncateFinish( + const FileOperationInfo& info) { + OnFileOperation(m_on_file_truncate_finish_mid, info); +} + +void EventListenerJniCallback::OnFileCloseFinish( + const FileOperationInfo& info) { + OnFileOperation(m_on_file_close_finish_mid, info); +} + +bool EventListenerJniCallback::ShouldBeNotifiedOnFileIO() { + if (m_should_be_notified_on_file_io == nullptr) { + return false; + } + + jboolean attached_thread = JNI_FALSE; + JNIEnv* env = getJniEnv(&attached_thread); + assert(env != nullptr); + + jboolean jshould_be_notified = + env->CallBooleanMethod(m_jcallback_obj, m_should_be_notified_on_file_io); + + CleanupCallbackInvocation(env, attached_thread, {}); + + return static_cast(jshould_be_notified); +} + +void EventListenerJniCallback::OnErrorRecoveryBegin( + BackgroundErrorReason reason, Status bg_error, bool* auto_recovery) { + if (m_on_error_recovery_begin_proxy_mid == nullptr) { + return; + } + + JNIEnv* env; + jboolean attached_thread; + jobject jbg_error = SetupCallbackInvocation( + env, attached_thread, bg_error, StatusJni::construct); + + if (jbg_error != nullptr) { + jboolean jauto_recovery = env->CallBooleanMethod( + m_jcallback_obj, m_on_error_recovery_begin_proxy_mid, + static_cast(reason), jbg_error); + *auto_recovery = jauto_recovery == JNI_TRUE; + } + + CleanupCallbackInvocation(env, attached_thread, {&jbg_error}); +} + +void EventListenerJniCallback::OnErrorRecoveryCompleted(Status old_bg_error) { + if (m_on_error_recovery_completed_mid == nullptr) { + return; + } + + JNIEnv* env; + jboolean attached_thread; + jobject jold_bg_error = SetupCallbackInvocation( + env, attached_thread, old_bg_error, StatusJni::construct); + + if (jold_bg_error != nullptr) { + env->CallVoidMethod(m_jcallback_obj, m_on_error_recovery_completed_mid, + jold_bg_error); + } + + CleanupCallbackInvocation(env, attached_thread, {&jold_bg_error}); +} + +void EventListenerJniCallback::InitCallbackMethodId( + jmethodID& mid, EnabledEventCallback eec, JNIEnv* env, + jmethodID (*get_id)(JNIEnv* env)) { + if (m_enabled_event_callbacks.count(eec) == 1) { + mid = get_id(env); + } else { + mid = nullptr; + } +} + +template +jobject EventListenerJniCallback::SetupCallbackInvocation( + JNIEnv*& env, jboolean& attached_thread, const T& cpp_obj, + jobject (*convert)(JNIEnv* env, const T* cpp_obj)) { + attached_thread = JNI_FALSE; + env = getJniEnv(&attached_thread); + assert(env != nullptr); + + return convert(env, &cpp_obj); +} + +void EventListenerJniCallback::CleanupCallbackInvocation( + JNIEnv* env, jboolean attached_thread, + std::initializer_list refs) { + for (auto* ref : refs) { + if (*ref == nullptr) continue; + env->DeleteLocalRef(*ref); + } + + if (env->ExceptionCheck()) { + // exception thrown from CallVoidMethod + env->ExceptionDescribe(); // print out exception to stderr + } + + releaseJniEnv(attached_thread); +} + +void EventListenerJniCallback::OnFileOperation(const jmethodID& mid, + const FileOperationInfo& info) { + if (mid == nullptr) { + return; + } + + JNIEnv* env; + jboolean attached_thread; + jobject jop_info = SetupCallbackInvocation( + env, attached_thread, info, + FileOperationInfoJni::fromCppFileOperationInfo); + + if (jop_info != nullptr) { + env->CallVoidMethod(m_jcallback_obj, mid, jop_info); + } + + CleanupCallbackInvocation(env, attached_thread, {&jop_info}); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/java/rocksjni/event_listener_jnicallback.h b/librocksdb-sys/rocksdb/java/rocksjni/event_listener_jnicallback.h new file mode 100644 index 0000000..f4a235a --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/event_listener_jnicallback.h @@ -0,0 +1,122 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the callback "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::EventListener. + +#ifndef JAVA_ROCKSJNI_EVENT_LISTENER_JNICALLBACK_H_ +#define JAVA_ROCKSJNI_EVENT_LISTENER_JNICALLBACK_H_ + +#include + +#include +#include + +#include "rocksdb/listener.h" +#include "rocksjni/jnicallback.h" + +namespace ROCKSDB_NAMESPACE { + +enum EnabledEventCallback { + ON_FLUSH_COMPLETED = 0x0, + ON_FLUSH_BEGIN = 0x1, + ON_TABLE_FILE_DELETED = 0x2, + ON_COMPACTION_BEGIN = 0x3, + ON_COMPACTION_COMPLETED = 0x4, + ON_TABLE_FILE_CREATED = 0x5, + ON_TABLE_FILE_CREATION_STARTED = 0x6, + ON_MEMTABLE_SEALED = 0x7, + ON_COLUMN_FAMILY_HANDLE_DELETION_STARTED = 0x8, + ON_EXTERNAL_FILE_INGESTED = 0x9, + ON_BACKGROUND_ERROR = 0xA, + ON_STALL_CONDITIONS_CHANGED = 0xB, + ON_FILE_READ_FINISH = 0xC, + ON_FILE_WRITE_FINISH = 0xD, + ON_FILE_FLUSH_FINISH = 0xE, + ON_FILE_SYNC_FINISH = 0xF, + ON_FILE_RANGE_SYNC_FINISH = 0x10, + ON_FILE_TRUNCATE_FINISH = 0x11, + ON_FILE_CLOSE_FINISH = 0x12, + SHOULD_BE_NOTIFIED_ON_FILE_IO = 0x13, + ON_ERROR_RECOVERY_BEGIN = 0x14, + ON_ERROR_RECOVERY_COMPLETED = 0x15, + + NUM_ENABLED_EVENT_CALLBACK = 0x16, +}; + +class EventListenerJniCallback : public JniCallback, public EventListener { + public: + EventListenerJniCallback( + JNIEnv* env, jobject jevent_listener, + const std::set& enabled_event_callbacks); + virtual ~EventListenerJniCallback(); + virtual void OnFlushCompleted(DB* db, const FlushJobInfo& flush_job_info); + virtual void OnFlushBegin(DB* db, const FlushJobInfo& flush_job_info); + virtual void OnTableFileDeleted(const TableFileDeletionInfo& info); + virtual void OnCompactionBegin(DB* db, const CompactionJobInfo& ci); + virtual void OnCompactionCompleted(DB* db, const CompactionJobInfo& ci); + virtual void OnTableFileCreated(const TableFileCreationInfo& info); + virtual void OnTableFileCreationStarted( + const TableFileCreationBriefInfo& info); + virtual void OnMemTableSealed(const MemTableInfo& info); + virtual void OnColumnFamilyHandleDeletionStarted(ColumnFamilyHandle* handle); + virtual void OnExternalFileIngested(DB* db, + const ExternalFileIngestionInfo& info); + virtual void OnBackgroundError(BackgroundErrorReason reason, + Status* bg_error); + virtual void OnStallConditionsChanged(const WriteStallInfo& info); + virtual void OnFileReadFinish(const FileOperationInfo& info); + virtual void OnFileWriteFinish(const FileOperationInfo& info); + virtual void OnFileFlushFinish(const FileOperationInfo& info); + virtual void OnFileSyncFinish(const FileOperationInfo& info); + virtual void OnFileRangeSyncFinish(const FileOperationInfo& info); + virtual void OnFileTruncateFinish(const FileOperationInfo& info); + virtual void OnFileCloseFinish(const FileOperationInfo& info); + virtual bool ShouldBeNotifiedOnFileIO(); + virtual void OnErrorRecoveryBegin(BackgroundErrorReason reason, + Status bg_error, bool* auto_recovery); + virtual void OnErrorRecoveryCompleted(Status old_bg_error); + + private: + inline void InitCallbackMethodId(jmethodID& mid, EnabledEventCallback eec, + JNIEnv* env, + jmethodID (*get_id)(JNIEnv* env)); + template + inline jobject SetupCallbackInvocation( + JNIEnv*& env, jboolean& attached_thread, const T& cpp_obj, + jobject (*convert)(JNIEnv* env, const T* cpp_obj)); + inline void CleanupCallbackInvocation(JNIEnv* env, jboolean attached_thread, + std::initializer_list refs); + inline void OnFileOperation(const jmethodID& mid, + const FileOperationInfo& info); + + const std::set m_enabled_event_callbacks; + jmethodID m_on_flush_completed_proxy_mid; + jmethodID m_on_flush_begin_proxy_mid; + jmethodID m_on_table_file_deleted_mid; + jmethodID m_on_compaction_begin_proxy_mid; + jmethodID m_on_compaction_completed_proxy_mid; + jmethodID m_on_table_file_created_mid; + jmethodID m_on_table_file_creation_started_mid; + jmethodID m_on_mem_table_sealed_mid; + jmethodID m_on_column_family_handle_deletion_started_mid; + jmethodID m_on_external_file_ingested_proxy_mid; + jmethodID m_on_background_error_proxy_mid; + jmethodID m_on_stall_conditions_changed_mid; + jmethodID m_on_file_read_finish_mid; + jmethodID m_on_file_write_finish_mid; + jmethodID m_on_file_flush_finish_mid; + jmethodID m_on_file_sync_finish_mid; + jmethodID m_on_file_range_sync_finish_mid; + jmethodID m_on_file_truncate_finish_mid; + jmethodID m_on_file_close_finish_mid; + jmethodID m_should_be_notified_on_file_io; + jmethodID m_on_error_recovery_begin_proxy_mid; + jmethodID m_on_error_recovery_completed_mid; +}; + +} // namespace ROCKSDB_NAMESPACE + +#endif // JAVA_ROCKSJNI_EVENT_LISTENER_JNICALLBACK_H_ diff --git a/librocksdb-sys/rocksdb/java/rocksjni/filter.cc b/librocksdb-sys/rocksdb/java/rocksjni/filter.cc new file mode 100644 index 0000000..ed22016 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/filter.cc @@ -0,0 +1,46 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::FilterPolicy. + +#include +#include +#include + +#include + +#include "include/org_rocksdb_BloomFilter.h" +#include "include/org_rocksdb_Filter.h" +#include "rocksdb/filter_policy.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_BloomFilter + * Method: createBloomFilter + * Signature: (DZ)J + */ +jlong Java_org_rocksdb_BloomFilter_createNewBloomFilter(JNIEnv* /*env*/, + jclass /*jcls*/, + jdouble bits_per_key) { + auto* sptr_filter = + new std::shared_ptr( + ROCKSDB_NAMESPACE::NewBloomFilterPolicy(bits_per_key)); + return GET_CPLUSPLUS_POINTER(sptr_filter); +} + +/* + * Class: org_rocksdb_Filter + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_Filter_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, + jlong jhandle) { + auto* handle = + reinterpret_cast*>( + jhandle); + delete handle; // delete std::shared_ptr +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/ingest_external_file_options.cc b/librocksdb-sys/rocksdb/java/rocksjni/ingest_external_file_options.cc new file mode 100644 index 0000000..052cf33 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/ingest_external_file_options.cc @@ -0,0 +1,199 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::FilterPolicy. + +#include + +#include "include/org_rocksdb_IngestExternalFileOptions.h" +#include "rocksdb/options.h" +#include "rocksjni/cplusplus_to_java_convert.h" + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: newIngestExternalFileOptions + * Signature: ()J + */ +jlong Java_org_rocksdb_IngestExternalFileOptions_newIngestExternalFileOptions__( + JNIEnv*, jclass) { + auto* options = new ROCKSDB_NAMESPACE::IngestExternalFileOptions(); + return GET_CPLUSPLUS_POINTER(options); +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: newIngestExternalFileOptions + * Signature: (ZZZZ)J + */ +jlong Java_org_rocksdb_IngestExternalFileOptions_newIngestExternalFileOptions__ZZZZ( + JNIEnv*, jclass, jboolean jmove_files, jboolean jsnapshot_consistency, + jboolean jallow_global_seqno, jboolean jallow_blocking_flush) { + auto* options = new ROCKSDB_NAMESPACE::IngestExternalFileOptions(); + options->move_files = static_cast(jmove_files); + options->snapshot_consistency = static_cast(jsnapshot_consistency); + options->allow_global_seqno = static_cast(jallow_global_seqno); + options->allow_blocking_flush = static_cast(jallow_blocking_flush); + return GET_CPLUSPLUS_POINTER(options); +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: moveFiles + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_IngestExternalFileOptions_moveFiles(JNIEnv*, jobject, + jlong jhandle) { + auto* options = + reinterpret_cast(jhandle); + return static_cast(options->move_files); +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: setMoveFiles + * Signature: (JZ)V + */ +void Java_org_rocksdb_IngestExternalFileOptions_setMoveFiles( + JNIEnv*, jobject, jlong jhandle, jboolean jmove_files) { + auto* options = + reinterpret_cast(jhandle); + options->move_files = static_cast(jmove_files); +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: snapshotConsistency + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_IngestExternalFileOptions_snapshotConsistency( + JNIEnv*, jobject, jlong jhandle) { + auto* options = + reinterpret_cast(jhandle); + return static_cast(options->snapshot_consistency); +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: setSnapshotConsistency + * Signature: (JZ)V + */ +void Java_org_rocksdb_IngestExternalFileOptions_setSnapshotConsistency( + JNIEnv*, jobject, jlong jhandle, jboolean jsnapshot_consistency) { + auto* options = + reinterpret_cast(jhandle); + options->snapshot_consistency = static_cast(jsnapshot_consistency); +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: allowGlobalSeqNo + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_IngestExternalFileOptions_allowGlobalSeqNo( + JNIEnv*, jobject, jlong jhandle) { + auto* options = + reinterpret_cast(jhandle); + return static_cast(options->allow_global_seqno); +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: setAllowGlobalSeqNo + * Signature: (JZ)V + */ +void Java_org_rocksdb_IngestExternalFileOptions_setAllowGlobalSeqNo( + JNIEnv*, jobject, jlong jhandle, jboolean jallow_global_seqno) { + auto* options = + reinterpret_cast(jhandle); + options->allow_global_seqno = static_cast(jallow_global_seqno); +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: allowBlockingFlush + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_IngestExternalFileOptions_allowBlockingFlush( + JNIEnv*, jobject, jlong jhandle) { + auto* options = + reinterpret_cast(jhandle); + return static_cast(options->allow_blocking_flush); +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: setAllowBlockingFlush + * Signature: (JZ)V + */ +void Java_org_rocksdb_IngestExternalFileOptions_setAllowBlockingFlush( + JNIEnv*, jobject, jlong jhandle, jboolean jallow_blocking_flush) { + auto* options = + reinterpret_cast(jhandle); + options->allow_blocking_flush = static_cast(jallow_blocking_flush); +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: ingestBehind + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_IngestExternalFileOptions_ingestBehind( + JNIEnv*, jobject, jlong jhandle) { + auto* options = + reinterpret_cast(jhandle); + return options->ingest_behind == JNI_TRUE; +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: setIngestBehind + * Signature: (JZ)V + */ +void Java_org_rocksdb_IngestExternalFileOptions_setIngestBehind( + JNIEnv*, jobject, jlong jhandle, jboolean jingest_behind) { + auto* options = + reinterpret_cast(jhandle); + options->ingest_behind = jingest_behind == JNI_TRUE; +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: writeGlobalSeqno + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL +Java_org_rocksdb_IngestExternalFileOptions_writeGlobalSeqno(JNIEnv*, jobject, + jlong jhandle) { + auto* options = + reinterpret_cast(jhandle); + return options->write_global_seqno == JNI_TRUE; +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: setWriteGlobalSeqno + * Signature: (JZ)V + */ +JNIEXPORT void JNICALL +Java_org_rocksdb_IngestExternalFileOptions_setWriteGlobalSeqno( + JNIEnv*, jobject, jlong jhandle, jboolean jwrite_global_seqno) { + auto* options = + reinterpret_cast(jhandle); + options->write_global_seqno = jwrite_global_seqno == JNI_TRUE; +} + +/* + * Class: org_rocksdb_IngestExternalFileOptions + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_IngestExternalFileOptions_disposeInternal(JNIEnv*, + jobject, + jlong jhandle) { + auto* options = + reinterpret_cast(jhandle); + delete options; +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/iterator.cc b/librocksdb-sys/rocksdb/java/rocksjni/iterator.cc new file mode 100644 index 0000000..3ddb977 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/iterator.cc @@ -0,0 +1,340 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling c++ ROCKSDB_NAMESPACE::Iterator methods from Java side. + +#include "rocksdb/iterator.h" + +#include +#include +#include + +#include + +#include "include/org_rocksdb_RocksIterator.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_RocksIterator + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_RocksIterator_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + auto* it = reinterpret_cast(handle); + assert(it != nullptr); + delete it; +} + +/* + * Class: org_rocksdb_RocksIterator + * Method: isValid0 + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_RocksIterator_isValid0(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + return reinterpret_cast(handle)->Valid(); +} + +/* + * Class: org_rocksdb_RocksIterator + * Method: seekToFirst0 + * Signature: (J)V + */ +void Java_org_rocksdb_RocksIterator_seekToFirst0(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + reinterpret_cast(handle)->SeekToFirst(); +} + +/* + * Class: org_rocksdb_RocksIterator + * Method: seekToLast0 + * Signature: (J)V + */ +void Java_org_rocksdb_RocksIterator_seekToLast0(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + reinterpret_cast(handle)->SeekToLast(); +} + +/* + * Class: org_rocksdb_RocksIterator + * Method: next0 + * Signature: (J)V + */ +void Java_org_rocksdb_RocksIterator_next0(JNIEnv* /*env*/, jobject /*jobj*/, + jlong handle) { + reinterpret_cast(handle)->Next(); +} + +/* + * Class: org_rocksdb_RocksIterator + * Method: prev0 + * Signature: (J)V + */ +void Java_org_rocksdb_RocksIterator_prev0(JNIEnv* /*env*/, jobject /*jobj*/, + jlong handle) { + reinterpret_cast(handle)->Prev(); +} + +/* + * Class: org_rocksdb_RocksIterator + * Method: refresh0 + * Signature: (J)V + */ +void Java_org_rocksdb_RocksIterator_refresh0(JNIEnv* env, jobject /*jobj*/, + jlong handle) { + auto* it = reinterpret_cast(handle); + ROCKSDB_NAMESPACE::Status s = it->Refresh(); + + if (s.ok()) { + return; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); +} + +/* + * Class: org_rocksdb_RocksIterator + * Method: seek0 + * Signature: (J[BI)V + */ +void Java_org_rocksdb_RocksIterator_seek0(JNIEnv* env, jobject /*jobj*/, + jlong handle, jbyteArray jtarget, + jint jtarget_len) { + auto* it = reinterpret_cast(handle); + auto seek = [&it](ROCKSDB_NAMESPACE::Slice& target_slice) { + it->Seek(target_slice); + }; + ROCKSDB_NAMESPACE::JniUtil::k_op_region(seek, env, jtarget, 0, jtarget_len); +} + +/* + * This method supports fetching into indirect byte buffers; + * the Java wrapper extracts the byte[] and passes it here. + * In this case, the buffer offset of the key may be non-zero. + * + * Class: org_rocksdb_RocksIterator + * Method: seek0 + * Signature: (J[BII)V + */ +void Java_org_rocksdb_RocksIterator_seekByteArray0( + JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget, + jint jtarget_off, jint jtarget_len) { + auto* it = reinterpret_cast(handle); + auto seek = [&it](ROCKSDB_NAMESPACE::Slice& target_slice) { + it->Seek(target_slice); + }; + ROCKSDB_NAMESPACE::JniUtil::k_op_region(seek, env, jtarget, jtarget_off, + jtarget_len); +} + +/* + * Class: org_rocksdb_RocksIterator + * Method: seekDirect0 + * Signature: (JLjava/nio/ByteBuffer;II)V + */ +void Java_org_rocksdb_RocksIterator_seekDirect0(JNIEnv* env, jobject /*jobj*/, + jlong handle, jobject jtarget, + jint jtarget_off, + jint jtarget_len) { + auto* it = reinterpret_cast(handle); + auto seek = [&it](ROCKSDB_NAMESPACE::Slice& target_slice) { + it->Seek(target_slice); + }; + ROCKSDB_NAMESPACE::JniUtil::k_op_direct(seek, env, jtarget, jtarget_off, + jtarget_len); +} + +/* + * Class: org_rocksdb_RocksIterator + * Method: seekForPrevDirect0 + * Signature: (JLjava/nio/ByteBuffer;II)V + */ +void Java_org_rocksdb_RocksIterator_seekForPrevDirect0( + JNIEnv* env, jobject /*jobj*/, jlong handle, jobject jtarget, + jint jtarget_off, jint jtarget_len) { + auto* it = reinterpret_cast(handle); + auto seekPrev = [&it](ROCKSDB_NAMESPACE::Slice& target_slice) { + it->SeekForPrev(target_slice); + }; + ROCKSDB_NAMESPACE::JniUtil::k_op_direct(seekPrev, env, jtarget, jtarget_off, + jtarget_len); +} + +/* + * Class: org_rocksdb_RocksIterator + * Method: seekForPrev0 + * Signature: (J[BI)V + */ +void Java_org_rocksdb_RocksIterator_seekForPrev0(JNIEnv* env, jobject /*jobj*/, + jlong handle, + jbyteArray jtarget, + jint jtarget_len) { + auto* it = reinterpret_cast(handle); + auto seek = [&it](ROCKSDB_NAMESPACE::Slice& target_slice) { + it->SeekForPrev(target_slice); + }; + ROCKSDB_NAMESPACE::JniUtil::k_op_region(seek, env, jtarget, 0, jtarget_len); +} + +/* + * This method supports fetching into indirect byte buffers; + * the Java wrapper extracts the byte[] and passes it here. + * In this case, the buffer offset of the key may be non-zero. + * + * Class: org_rocksdb_RocksIterator + * Method: seek0 + * Signature: (J[BII)V + */ +void Java_org_rocksdb_RocksIterator_seekForPrevByteArray0( + JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget, + jint jtarget_off, jint jtarget_len) { + auto* it = reinterpret_cast(handle); + auto seek = [&it](ROCKSDB_NAMESPACE::Slice& target_slice) { + it->SeekForPrev(target_slice); + }; + ROCKSDB_NAMESPACE::JniUtil::k_op_region(seek, env, jtarget, jtarget_off, + jtarget_len); +} + +/* + * Class: org_rocksdb_RocksIterator + * Method: status0 + * Signature: (J)V + */ +void Java_org_rocksdb_RocksIterator_status0(JNIEnv* env, jobject /*jobj*/, + jlong handle) { + auto* it = reinterpret_cast(handle); + ROCKSDB_NAMESPACE::Status s = it->status(); + + if (s.ok()) { + return; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); +} + +/* + * Class: org_rocksdb_RocksIterator + * Method: key0 + * Signature: (J)[B + */ +jbyteArray Java_org_rocksdb_RocksIterator_key0(JNIEnv* env, jobject /*jobj*/, + jlong handle) { + auto* it = reinterpret_cast(handle); + ROCKSDB_NAMESPACE::Slice key_slice = it->key(); + + jbyteArray jkey = env->NewByteArray(static_cast(key_slice.size())); + if (jkey == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + env->SetByteArrayRegion( + jkey, 0, static_cast(key_slice.size()), + const_cast(reinterpret_cast(key_slice.data()))); + return jkey; +} + +/* + * Class: org_rocksdb_RocksIterator + * Method: keyDirect0 + * Signature: (JLjava/nio/ByteBuffer;II)I + */ +jint Java_org_rocksdb_RocksIterator_keyDirect0(JNIEnv* env, jobject /*jobj*/, + jlong handle, jobject jtarget, + jint jtarget_off, + jint jtarget_len) { + auto* it = reinterpret_cast(handle); + ROCKSDB_NAMESPACE::Slice key_slice = it->key(); + return ROCKSDB_NAMESPACE::JniUtil::copyToDirect(env, key_slice, jtarget, + jtarget_off, jtarget_len); +} + +/* + * This method supports fetching into indirect byte buffers; + * the Java wrapper extracts the byte[] and passes it here. + * + * Class: org_rocksdb_RocksIterator + * Method: keyByteArray0 + * Signature: (J[BII)I + */ +jint Java_org_rocksdb_RocksIterator_keyByteArray0(JNIEnv* env, jobject /*jobj*/, + jlong handle, jbyteArray jkey, + jint jkey_off, + jint jkey_len) { + auto* it = reinterpret_cast(handle); + ROCKSDB_NAMESPACE::Slice key_slice = it->key(); + jsize copy_size = std::min(static_cast(key_slice.size()), + static_cast(jkey_len)); + env->SetByteArrayRegion( + jkey, jkey_off, copy_size, + const_cast(reinterpret_cast(key_slice.data()))); + + return static_cast(key_slice.size()); +} + +/* + * Class: org_rocksdb_RocksIterator + * Method: value0 + * Signature: (J)[B + */ +jbyteArray Java_org_rocksdb_RocksIterator_value0(JNIEnv* env, jobject /*jobj*/, + jlong handle) { + auto* it = reinterpret_cast(handle); + ROCKSDB_NAMESPACE::Slice value_slice = it->value(); + + jbyteArray jkeyValue = + env->NewByteArray(static_cast(value_slice.size())); + if (jkeyValue == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + env->SetByteArrayRegion( + jkeyValue, 0, static_cast(value_slice.size()), + const_cast(reinterpret_cast(value_slice.data()))); + return jkeyValue; +} + +/* + * Class: org_rocksdb_RocksIterator + * Method: valueDirect0 + * Signature: (JLjava/nio/ByteBuffer;II)I + */ +jint Java_org_rocksdb_RocksIterator_valueDirect0(JNIEnv* env, jobject /*jobj*/, + jlong handle, jobject jtarget, + jint jtarget_off, + jint jtarget_len) { + auto* it = reinterpret_cast(handle); + ROCKSDB_NAMESPACE::Slice value_slice = it->value(); + return ROCKSDB_NAMESPACE::JniUtil::copyToDirect(env, value_slice, jtarget, + jtarget_off, jtarget_len); +} + +/* + * This method supports fetching into indirect byte buffers; + * the Java wrapper extracts the byte[] and passes it here. + * + * Class: org_rocksdb_RocksIterator + * Method: valueByteArray0 + * Signature: (J[BII)I + */ +jint Java_org_rocksdb_RocksIterator_valueByteArray0( + JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jvalue_target, + jint jvalue_off, jint jvalue_len) { + auto* it = reinterpret_cast(handle); + ROCKSDB_NAMESPACE::Slice value_slice = it->value(); + jsize copy_size = std::min(static_cast(value_slice.size()), + static_cast(jvalue_len)); + env->SetByteArrayRegion( + jvalue_target, jvalue_off, copy_size, + const_cast(reinterpret_cast(value_slice.data()))); + + return static_cast(value_slice.size()); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/jnicallback.cc b/librocksdb-sys/rocksdb/java/rocksjni/jnicallback.cc new file mode 100644 index 0000000..f2742cd --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/jnicallback.cc @@ -0,0 +1,54 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the callback "bridge" between Java and C++ for +// JNI Callbacks from C++ to sub-classes or org.rocksdb.RocksCallbackObject + +#include "rocksjni/jnicallback.h" + +#include + +#include "rocksjni/portal.h" + +namespace ROCKSDB_NAMESPACE { +JniCallback::JniCallback(JNIEnv* env, jobject jcallback_obj) { + // Note: jcallback_obj may be accessed by multiple threads, + // so we ref the jvm not the env + const jint rs = env->GetJavaVM(&m_jvm); + if (rs != JNI_OK) { + // exception thrown + return; + } + + // Note: we may want to access the Java callback object instance + // across multiple method calls, so we create a global ref + assert(jcallback_obj != nullptr); + m_jcallback_obj = env->NewGlobalRef(jcallback_obj); + if (jcallback_obj == nullptr) { + // exception thrown: OutOfMemoryError + return; + } +} + +JNIEnv* JniCallback::getJniEnv(jboolean* attached) const { + return JniUtil::getJniEnv(m_jvm, attached); +} + +void JniCallback::releaseJniEnv(jboolean& attached) const { + JniUtil::releaseJniEnv(m_jvm, attached); +} + +JniCallback::~JniCallback() { + jboolean attached_thread = JNI_FALSE; + JNIEnv* env = getJniEnv(&attached_thread); + assert(env != nullptr); + + if (m_jcallback_obj != nullptr) { + env->DeleteGlobalRef(m_jcallback_obj); + } + + releaseJniEnv(attached_thread); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/java/rocksjni/jnicallback.h b/librocksdb-sys/rocksdb/java/rocksjni/jnicallback.h new file mode 100644 index 0000000..a03a041 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/jnicallback.h @@ -0,0 +1,32 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the callback "bridge" between Java and C++ for +// JNI Callbacks from C++ to sub-classes or org.rocksdb.RocksCallbackObject + +#ifndef JAVA_ROCKSJNI_JNICALLBACK_H_ +#define JAVA_ROCKSJNI_JNICALLBACK_H_ + +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { +class JniCallback { + public: + JniCallback(JNIEnv* env, jobject jcallback_obj); + virtual ~JniCallback(); + + const jobject& GetJavaObject() const { return m_jcallback_obj; } + + protected: + JavaVM* m_jvm; + jobject m_jcallback_obj; + JNIEnv* getJniEnv(jboolean* attached) const; + void releaseJniEnv(jboolean& attached) const; +}; +} // namespace ROCKSDB_NAMESPACE + +#endif // JAVA_ROCKSJNI_JNICALLBACK_H_ diff --git a/librocksdb-sys/rocksdb/java/rocksjni/loggerjnicallback.cc b/librocksdb-sys/rocksdb/java/rocksjni/loggerjnicallback.cc new file mode 100644 index 0000000..aa9f95c --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/loggerjnicallback.cc @@ -0,0 +1,299 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the callback "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::Logger. + +#include "rocksjni/loggerjnicallback.h" + +#include +#include + +#include "include/org_rocksdb_Logger.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +namespace ROCKSDB_NAMESPACE { + +LoggerJniCallback::LoggerJniCallback(JNIEnv* env, jobject jlogger) + : JniCallback(env, jlogger) { + m_jLogMethodId = LoggerJni::getLogMethodId(env); + if (m_jLogMethodId == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return; + } + + jobject jdebug_level = InfoLogLevelJni::DEBUG_LEVEL(env); + if (jdebug_level == nullptr) { + // exception thrown: NoSuchFieldError, ExceptionInInitializerError + // or OutOfMemoryError + return; + } + m_jdebug_level = env->NewGlobalRef(jdebug_level); + if (m_jdebug_level == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + jobject jinfo_level = InfoLogLevelJni::INFO_LEVEL(env); + if (jinfo_level == nullptr) { + // exception thrown: NoSuchFieldError, ExceptionInInitializerError + // or OutOfMemoryError + return; + } + m_jinfo_level = env->NewGlobalRef(jinfo_level); + if (m_jinfo_level == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + jobject jwarn_level = InfoLogLevelJni::WARN_LEVEL(env); + if (jwarn_level == nullptr) { + // exception thrown: NoSuchFieldError, ExceptionInInitializerError + // or OutOfMemoryError + return; + } + m_jwarn_level = env->NewGlobalRef(jwarn_level); + if (m_jwarn_level == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + jobject jerror_level = InfoLogLevelJni::ERROR_LEVEL(env); + if (jerror_level == nullptr) { + // exception thrown: NoSuchFieldError, ExceptionInInitializerError + // or OutOfMemoryError + return; + } + m_jerror_level = env->NewGlobalRef(jerror_level); + if (m_jerror_level == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + jobject jfatal_level = InfoLogLevelJni::FATAL_LEVEL(env); + if (jfatal_level == nullptr) { + // exception thrown: NoSuchFieldError, ExceptionInInitializerError + // or OutOfMemoryError + return; + } + m_jfatal_level = env->NewGlobalRef(jfatal_level); + if (m_jfatal_level == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + jobject jheader_level = InfoLogLevelJni::HEADER_LEVEL(env); + if (jheader_level == nullptr) { + // exception thrown: NoSuchFieldError, ExceptionInInitializerError + // or OutOfMemoryError + return; + } + m_jheader_level = env->NewGlobalRef(jheader_level); + if (m_jheader_level == nullptr) { + // exception thrown: OutOfMemoryError + return; + } +} + +void LoggerJniCallback::Logv(const char* /*format*/, va_list /*ap*/) { + // We implement this method because it is virtual but we don't + // use it because we need to know about the log level. +} + +void LoggerJniCallback::Logv(const InfoLogLevel log_level, const char* format, + va_list ap) { + if (GetInfoLogLevel() <= log_level) { + // determine InfoLogLevel java enum instance + jobject jlog_level; + switch (log_level) { + case ROCKSDB_NAMESPACE::InfoLogLevel::DEBUG_LEVEL: + jlog_level = m_jdebug_level; + break; + case ROCKSDB_NAMESPACE::InfoLogLevel::INFO_LEVEL: + jlog_level = m_jinfo_level; + break; + case ROCKSDB_NAMESPACE::InfoLogLevel::WARN_LEVEL: + jlog_level = m_jwarn_level; + break; + case ROCKSDB_NAMESPACE::InfoLogLevel::ERROR_LEVEL: + jlog_level = m_jerror_level; + break; + case ROCKSDB_NAMESPACE::InfoLogLevel::FATAL_LEVEL: + jlog_level = m_jfatal_level; + break; + case ROCKSDB_NAMESPACE::InfoLogLevel::HEADER_LEVEL: + jlog_level = m_jheader_level; + break; + default: + jlog_level = m_jfatal_level; + break; + } + + assert(format != nullptr); + const std::unique_ptr msg = format_str(format, ap); + + // pass msg to java callback handler + jboolean attached_thread = JNI_FALSE; + JNIEnv* env = getJniEnv(&attached_thread); + assert(env != nullptr); + + jstring jmsg = env->NewStringUTF(msg.get()); + if (jmsg == nullptr) { + // unable to construct string + if (env->ExceptionCheck()) { + env->ExceptionDescribe(); // print out exception to stderr + } + releaseJniEnv(attached_thread); + return; + } + if (env->ExceptionCheck()) { + // exception thrown: OutOfMemoryError + env->ExceptionDescribe(); // print out exception to stderr + env->DeleteLocalRef(jmsg); + releaseJniEnv(attached_thread); + return; + } + + env->CallVoidMethod(m_jcallback_obj, m_jLogMethodId, jlog_level, jmsg); + if (env->ExceptionCheck()) { + // exception thrown + env->ExceptionDescribe(); // print out exception to stderr + env->DeleteLocalRef(jmsg); + releaseJniEnv(attached_thread); + return; + } + + env->DeleteLocalRef(jmsg); + releaseJniEnv(attached_thread); + } +} + +std::unique_ptr LoggerJniCallback::format_str(const char* format, + va_list ap) const { + va_list ap_copy; + + va_copy(ap_copy, ap); + const size_t required = + vsnprintf(nullptr, 0, format, ap_copy) + 1; // Extra space for '\0' + va_end(ap_copy); + + std::unique_ptr buf(new char[required]); + + va_copy(ap_copy, ap); + vsnprintf(buf.get(), required, format, ap_copy); + va_end(ap_copy); + + return buf; +} +LoggerJniCallback::~LoggerJniCallback() { + jboolean attached_thread = JNI_FALSE; + JNIEnv* env = getJniEnv(&attached_thread); + assert(env != nullptr); + + if (m_jdebug_level != nullptr) { + env->DeleteGlobalRef(m_jdebug_level); + } + + if (m_jinfo_level != nullptr) { + env->DeleteGlobalRef(m_jinfo_level); + } + + if (m_jwarn_level != nullptr) { + env->DeleteGlobalRef(m_jwarn_level); + } + + if (m_jerror_level != nullptr) { + env->DeleteGlobalRef(m_jerror_level); + } + + if (m_jfatal_level != nullptr) { + env->DeleteGlobalRef(m_jfatal_level); + } + + if (m_jheader_level != nullptr) { + env->DeleteGlobalRef(m_jheader_level); + } + + releaseJniEnv(attached_thread); +} + +} // namespace ROCKSDB_NAMESPACE + +/* + * Class: org_rocksdb_Logger + * Method: createNewLoggerOptions + * Signature: (J)J + */ +jlong Java_org_rocksdb_Logger_createNewLoggerOptions(JNIEnv* env, jobject jobj, + jlong joptions) { + auto* sptr_logger = new std::shared_ptr( + new ROCKSDB_NAMESPACE::LoggerJniCallback(env, jobj)); + + // set log level + auto* options = reinterpret_cast(joptions); + sptr_logger->get()->SetInfoLogLevel(options->info_log_level); + + return GET_CPLUSPLUS_POINTER(sptr_logger); +} + +/* + * Class: org_rocksdb_Logger + * Method: createNewLoggerDbOptions + * Signature: (J)J + */ +jlong Java_org_rocksdb_Logger_createNewLoggerDbOptions(JNIEnv* env, + jobject jobj, + jlong jdb_options) { + auto* sptr_logger = new std::shared_ptr( + new ROCKSDB_NAMESPACE::LoggerJniCallback(env, jobj)); + + // set log level + auto* db_options = + reinterpret_cast(jdb_options); + sptr_logger->get()->SetInfoLogLevel(db_options->info_log_level); + + return GET_CPLUSPLUS_POINTER(sptr_logger); +} + +/* + * Class: org_rocksdb_Logger + * Method: setInfoLogLevel + * Signature: (JB)V + */ +void Java_org_rocksdb_Logger_setInfoLogLevel(JNIEnv* /*env*/, jobject /*jobj*/, + jlong jhandle, jbyte jlog_level) { + auto* handle = + reinterpret_cast*>( + jhandle); + handle->get()->SetInfoLogLevel( + static_cast(jlog_level)); +} + +/* + * Class: org_rocksdb_Logger + * Method: infoLogLevel + * Signature: (J)B + */ +jbyte Java_org_rocksdb_Logger_infoLogLevel(JNIEnv* /*env*/, jobject /*jobj*/, + jlong jhandle) { + auto* handle = + reinterpret_cast*>( + jhandle); + return static_cast(handle->get()->GetInfoLogLevel()); +} + +/* + * Class: org_rocksdb_Logger + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_Logger_disposeInternal(JNIEnv* /*env*/, jobject /*jobj*/, + jlong jhandle) { + auto* handle = + reinterpret_cast*>( + jhandle); + delete handle; // delete std::shared_ptr +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/loggerjnicallback.h b/librocksdb-sys/rocksdb/java/rocksjni/loggerjnicallback.h new file mode 100644 index 0000000..5777498 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/loggerjnicallback.h @@ -0,0 +1,51 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the callback "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::Logger + +#ifndef JAVA_ROCKSJNI_LOGGERJNICALLBACK_H_ +#define JAVA_ROCKSJNI_LOGGERJNICALLBACK_H_ + +#include + +#include +#include + +#include "port/port.h" +#include "rocksdb/env.h" +#include "rocksjni/jnicallback.h" + +namespace ROCKSDB_NAMESPACE { + +class LoggerJniCallback : public JniCallback, public Logger { + public: + LoggerJniCallback(JNIEnv* env, jobject jLogger); + ~LoggerJniCallback(); + + using Logger::GetInfoLogLevel; + using Logger::SetInfoLogLevel; + // Write an entry to the log file with the specified format. + virtual void Logv(const char* format, va_list ap); + // Write an entry to the log file with the specified log level + // and format. Any log with level under the internal log level + // of *this (see @SetInfoLogLevel and @GetInfoLogLevel) will not be + // printed. + virtual void Logv(const InfoLogLevel log_level, const char* format, + va_list ap); + + private: + jmethodID m_jLogMethodId; + jobject m_jdebug_level; + jobject m_jinfo_level; + jobject m_jwarn_level; + jobject m_jerror_level; + jobject m_jfatal_level; + jobject m_jheader_level; + std::unique_ptr format_str(const char* format, va_list ap) const; +}; +} // namespace ROCKSDB_NAMESPACE + +#endif // JAVA_ROCKSJNI_LOGGERJNICALLBACK_H_ diff --git a/librocksdb-sys/rocksdb/java/rocksjni/lru_cache.cc b/librocksdb-sys/rocksdb/java/rocksjni/lru_cache.cc new file mode 100644 index 0000000..56dffa2 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/lru_cache.cc @@ -0,0 +1,49 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::LRUCache. + +#include "cache/lru_cache.h" + +#include + +#include "include/org_rocksdb_LRUCache.h" +#include "rocksjni/cplusplus_to_java_convert.h" + +/* + * Class: org_rocksdb_LRUCache + * Method: newLRUCache + * Signature: (JIZD)J + */ +jlong Java_org_rocksdb_LRUCache_newLRUCache(JNIEnv* /*env*/, jclass /*jcls*/, + jlong jcapacity, + jint jnum_shard_bits, + jboolean jstrict_capacity_limit, + jdouble jhigh_pri_pool_ratio, + jdouble jlow_pri_pool_ratio) { + auto* sptr_lru_cache = new std::shared_ptr( + ROCKSDB_NAMESPACE::NewLRUCache( + static_cast(jcapacity), static_cast(jnum_shard_bits), + static_cast(jstrict_capacity_limit), + static_cast(jhigh_pri_pool_ratio), + nullptr /* memory_allocator */, rocksdb::kDefaultToAdaptiveMutex, + rocksdb::kDefaultCacheMetadataChargePolicy, + static_cast(jlow_pri_pool_ratio))); + return GET_CPLUSPLUS_POINTER(sptr_lru_cache); +} + +/* + * Class: org_rocksdb_LRUCache + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_LRUCache_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* sptr_lru_cache = + reinterpret_cast*>(jhandle); + delete sptr_lru_cache; // delete std::shared_ptr +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/memory_util.cc b/librocksdb-sys/rocksdb/java/rocksjni/memory_util.cc new file mode 100644 index 0000000..c87c4f4 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/memory_util.cc @@ -0,0 +1,100 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "rocksdb/utilities/memory_util.h" + +#include + +#include +#include +#include +#include + +#include "include/org_rocksdb_MemoryUtil.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_MemoryUtil + * Method: getApproximateMemoryUsageByType + * Signature: ([J[J)Ljava/util/Map; + */ +jobject Java_org_rocksdb_MemoryUtil_getApproximateMemoryUsageByType( + JNIEnv *env, jclass, jlongArray jdb_handles, jlongArray jcache_handles) { + jboolean has_exception = JNI_FALSE; + std::vector dbs = + ROCKSDB_NAMESPACE::JniUtil::fromJPointers( + env, jdb_handles, &has_exception); + if (has_exception == JNI_TRUE) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + std::unordered_set cache_set; + jsize cache_handle_count = env->GetArrayLength(jcache_handles); + if (cache_handle_count > 0) { + jlong *ptr_jcache_handles = + env->GetLongArrayElements(jcache_handles, nullptr); + if (ptr_jcache_handles == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + for (jsize i = 0; i < cache_handle_count; i++) { + auto *cache_ptr = + reinterpret_cast *>( + ptr_jcache_handles[i]); + cache_set.insert(cache_ptr->get()); + } + env->ReleaseLongArrayElements(jcache_handles, ptr_jcache_handles, + JNI_ABORT); + } + + std::map usage_by_type; + if (ROCKSDB_NAMESPACE::MemoryUtil::GetApproximateMemoryUsageByType( + dbs, cache_set, &usage_by_type) != ROCKSDB_NAMESPACE::Status::OK()) { + // Non-OK status + return nullptr; + } + + jobject jusage_by_type = ROCKSDB_NAMESPACE::HashMapJni::construct( + env, static_cast(usage_by_type.size())); + if (jusage_by_type == nullptr) { + // exception occurred + return nullptr; + } + const ROCKSDB_NAMESPACE::HashMapJni::FnMapKV< + const ROCKSDB_NAMESPACE::MemoryUtil::UsageType, const uint64_t, jobject, + jobject> + fn_map_kv = [env]( + const std::pair &pair) { + // Construct key + const jobject jusage_type = ROCKSDB_NAMESPACE::ByteJni::valueOf( + env, ROCKSDB_NAMESPACE::MemoryUsageTypeJni::toJavaMemoryUsageType( + pair.first)); + if (jusage_type == nullptr) { + // an error occurred + return std::unique_ptr>(nullptr); + } + // Construct value + const jobject jusage_value = + ROCKSDB_NAMESPACE::LongJni::valueOf(env, pair.second); + if (jusage_value == nullptr) { + // an error occurred + return std::unique_ptr>(nullptr); + } + // Construct and return pointer to pair of jobjects + return std::unique_ptr>( + new std::pair(jusage_type, jusage_value)); + }; + + if (!ROCKSDB_NAMESPACE::HashMapJni::putAll(env, jusage_by_type, + usage_by_type.begin(), + usage_by_type.end(), fn_map_kv)) { + // exception occcurred + jusage_by_type = nullptr; + } + + return jusage_by_type; +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/memtablejni.cc b/librocksdb-sys/rocksdb/java/rocksjni/memtablejni.cc new file mode 100644 index 0000000..a4d02f3 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/memtablejni.cc @@ -0,0 +1,94 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for MemTables. + +#include "include/org_rocksdb_HashLinkedListMemTableConfig.h" +#include "include/org_rocksdb_HashSkipListMemTableConfig.h" +#include "include/org_rocksdb_SkipListMemTableConfig.h" +#include "include/org_rocksdb_VectorMemTableConfig.h" +#include "rocksdb/memtablerep.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_HashSkipListMemTableConfig + * Method: newMemTableFactoryHandle + * Signature: (JII)J + */ +jlong Java_org_rocksdb_HashSkipListMemTableConfig_newMemTableFactoryHandle( + JNIEnv* env, jobject /*jobj*/, jlong jbucket_count, jint jheight, + jint jbranching_factor) { + ROCKSDB_NAMESPACE::Status s = + ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t(jbucket_count); + if (s.ok()) { + return GET_CPLUSPLUS_POINTER(ROCKSDB_NAMESPACE::NewHashSkipListRepFactory( + static_cast(jbucket_count), static_cast(jheight), + static_cast(jbranching_factor))); + } + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + return 0; +} + +/* + * Class: org_rocksdb_HashLinkedListMemTableConfig + * Method: newMemTableFactoryHandle + * Signature: (JJIZI)J + */ +jlong Java_org_rocksdb_HashLinkedListMemTableConfig_newMemTableFactoryHandle( + JNIEnv* env, jobject /*jobj*/, jlong jbucket_count, + jlong jhuge_page_tlb_size, jint jbucket_entries_logging_threshold, + jboolean jif_log_bucket_dist_when_flash, jint jthreshold_use_skiplist) { + ROCKSDB_NAMESPACE::Status statusBucketCount = + ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t(jbucket_count); + ROCKSDB_NAMESPACE::Status statusHugePageTlb = + ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( + jhuge_page_tlb_size); + if (statusBucketCount.ok() && statusHugePageTlb.ok()) { + return GET_CPLUSPLUS_POINTER(ROCKSDB_NAMESPACE::NewHashLinkListRepFactory( + static_cast(jbucket_count), + static_cast(jhuge_page_tlb_size), + static_cast(jbucket_entries_logging_threshold), + static_cast(jif_log_bucket_dist_when_flash), + static_cast(jthreshold_use_skiplist))); + } + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew( + env, !statusBucketCount.ok() ? statusBucketCount : statusHugePageTlb); + return 0; +} + +/* + * Class: org_rocksdb_VectorMemTableConfig + * Method: newMemTableFactoryHandle + * Signature: (J)J + */ +jlong Java_org_rocksdb_VectorMemTableConfig_newMemTableFactoryHandle( + JNIEnv* env, jobject /*jobj*/, jlong jreserved_size) { + ROCKSDB_NAMESPACE::Status s = + ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t(jreserved_size); + if (s.ok()) { + return GET_CPLUSPLUS_POINTER(new ROCKSDB_NAMESPACE::VectorRepFactory( + static_cast(jreserved_size))); + } + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + return 0; +} + +/* + * Class: org_rocksdb_SkipListMemTableConfig + * Method: newMemTableFactoryHandle0 + * Signature: (J)J + */ +jlong Java_org_rocksdb_SkipListMemTableConfig_newMemTableFactoryHandle0( + JNIEnv* env, jobject /*jobj*/, jlong jlookahead) { + ROCKSDB_NAMESPACE::Status s = + ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t(jlookahead); + if (s.ok()) { + return GET_CPLUSPLUS_POINTER(new ROCKSDB_NAMESPACE::SkipListFactory( + static_cast(jlookahead))); + } + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + return 0; +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/merge_operator.cc b/librocksdb-sys/rocksdb/java/rocksjni/merge_operator.cc new file mode 100644 index 0000000..ce3c5df --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/merge_operator.cc @@ -0,0 +1,98 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// Copyright (c) 2014, Vlad Balan (vlad.gm@gmail.com). All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ +// for ROCKSDB_NAMESPACE::MergeOperator. + +#include "rocksdb/merge_operator.h" + +#include +#include +#include + +#include +#include + +#include "include/org_rocksdb_StringAppendOperator.h" +#include "include/org_rocksdb_UInt64AddOperator.h" +#include "rocksdb/db.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/options.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/statistics.h" +#include "rocksdb/table.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" +#include "utilities/merge_operators.h" + +/* + * Class: org_rocksdb_StringAppendOperator + * Method: newSharedStringAppendOperator + * Signature: (C)J + */ +jlong Java_org_rocksdb_StringAppendOperator_newSharedStringAppendOperator__C( + JNIEnv* /*env*/, jclass /*jclazz*/, jchar jdelim) { + auto* sptr_string_append_op = + new std::shared_ptr( + ROCKSDB_NAMESPACE::MergeOperators::CreateStringAppendOperator( + (char)jdelim)); + return GET_CPLUSPLUS_POINTER(sptr_string_append_op); +} + +jlong Java_org_rocksdb_StringAppendOperator_newSharedStringAppendOperator__Ljava_lang_String_2( + JNIEnv* env, jclass /*jclass*/, jstring jdelim) { + jboolean has_exception = JNI_FALSE; + auto delim = + ROCKSDB_NAMESPACE::JniUtil::copyStdString(env, jdelim, &has_exception); + if (has_exception == JNI_TRUE) { + return 0; + } + auto* sptr_string_append_op = + new std::shared_ptr( + ROCKSDB_NAMESPACE::MergeOperators::CreateStringAppendOperator(delim)); + return GET_CPLUSPLUS_POINTER(sptr_string_append_op); +} + +/* + * Class: org_rocksdb_StringAppendOperator + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_StringAppendOperator_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* sptr_string_append_op = + reinterpret_cast*>( + jhandle); + delete sptr_string_append_op; // delete std::shared_ptr +} + +/* + * Class: org_rocksdb_UInt64AddOperator + * Method: newSharedUInt64AddOperator + * Signature: ()J + */ +jlong Java_org_rocksdb_UInt64AddOperator_newSharedUInt64AddOperator( + JNIEnv* /*env*/, jclass /*jclazz*/) { + auto* sptr_uint64_add_op = + new std::shared_ptr( + ROCKSDB_NAMESPACE::MergeOperators::CreateUInt64AddOperator()); + return GET_CPLUSPLUS_POINTER(sptr_uint64_add_op); +} + +/* + * Class: org_rocksdb_UInt64AddOperator + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_UInt64AddOperator_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* sptr_uint64_add_op = + reinterpret_cast*>( + jhandle); + delete sptr_uint64_add_op; // delete std::shared_ptr +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/native_comparator_wrapper_test.cc b/librocksdb-sys/rocksdb/java/rocksjni/native_comparator_wrapper_test.cc new file mode 100644 index 0000000..ac33ca2 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/native_comparator_wrapper_test.cc @@ -0,0 +1,45 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include + +#include + +#include "include/org_rocksdb_NativeComparatorWrapperTest_NativeStringComparatorWrapper.h" +#include "rocksdb/comparator.h" +#include "rocksdb/slice.h" +#include "rocksjni/cplusplus_to_java_convert.h" + +namespace ROCKSDB_NAMESPACE { + +class NativeComparatorWrapperTestStringComparator : public Comparator { + const char* Name() const { + return "NativeComparatorWrapperTestStringComparator"; + } + + int Compare(const Slice& a, const Slice& b) const { + return a.ToString().compare(b.ToString()); + } + + void FindShortestSeparator(std::string* /*start*/, + const Slice& /*limit*/) const { + return; + } + + void FindShortSuccessor(std::string* /*key*/) const { return; } +}; +} // namespace ROCKSDB_NAMESPACE + +/* + * Class: org_rocksdb_NativeComparatorWrapperTest_NativeStringComparatorWrapper + * Method: newStringComparator + * Signature: ()J + */ +jlong Java_org_rocksdb_NativeComparatorWrapperTest_00024NativeStringComparatorWrapper_newStringComparator( + JNIEnv* /*env*/, jobject /*jobj*/) { + auto* comparator = + new ROCKSDB_NAMESPACE::NativeComparatorWrapperTestStringComparator(); + return GET_CPLUSPLUS_POINTER(comparator); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/optimistic_transaction_db.cc b/librocksdb-sys/rocksdb/java/rocksjni/optimistic_transaction_db.cc new file mode 100644 index 0000000..238224f --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/optimistic_transaction_db.cc @@ -0,0 +1,270 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ +// for ROCKSDB_NAMESPACE::TransactionDB. + +#include "rocksdb/utilities/optimistic_transaction_db.h" + +#include + +#include "include/org_rocksdb_OptimisticTransactionDB.h" +#include "rocksdb/options.h" +#include "rocksdb/utilities/transaction.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_OptimisticTransactionDB + * Method: open + * Signature: (JLjava/lang/String;)J + */ +jlong Java_org_rocksdb_OptimisticTransactionDB_open__JLjava_lang_String_2( + JNIEnv* env, jclass, jlong joptions_handle, jstring jdb_path) { + const char* db_path = env->GetStringUTFChars(jdb_path, nullptr); + if (db_path == nullptr) { + // exception thrown: OutOfMemoryError + return 0; + } + + auto* options = + reinterpret_cast(joptions_handle); + ROCKSDB_NAMESPACE::OptimisticTransactionDB* otdb = nullptr; + ROCKSDB_NAMESPACE::Status s = + ROCKSDB_NAMESPACE::OptimisticTransactionDB::Open(*options, db_path, + &otdb); + env->ReleaseStringUTFChars(jdb_path, db_path); + + if (s.ok()) { + return GET_CPLUSPLUS_POINTER(otdb); + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return 0; + } +} + +/* + * Class: org_rocksdb_OptimisticTransactionDB + * Method: open + * Signature: (JLjava/lang/String;[[B[J)[J + */ +jlongArray +Java_org_rocksdb_OptimisticTransactionDB_open__JLjava_lang_String_2_3_3B_3J( + JNIEnv* env, jclass, jlong jdb_options_handle, jstring jdb_path, + jobjectArray jcolumn_names, jlongArray jcolumn_options_handles) { + const char* db_path = env->GetStringUTFChars(jdb_path, nullptr); + if (db_path == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + std::vector column_families; + const jsize len_cols = env->GetArrayLength(jcolumn_names); + if (len_cols > 0) { + jlong* jco = env->GetLongArrayElements(jcolumn_options_handles, nullptr); + if (jco == nullptr) { + // exception thrown: OutOfMemoryError + env->ReleaseStringUTFChars(jdb_path, db_path); + return nullptr; + } + + for (int i = 0; i < len_cols; i++) { + const jobject jcn = env->GetObjectArrayElement(jcolumn_names, i); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->ReleaseLongArrayElements(jcolumn_options_handles, jco, JNI_ABORT); + env->ReleaseStringUTFChars(jdb_path, db_path); + return nullptr; + } + + const jbyteArray jcn_ba = reinterpret_cast(jcn); + const jsize jcf_name_len = env->GetArrayLength(jcn_ba); + jbyte* jcf_name = env->GetByteArrayElements(jcn_ba, nullptr); + if (jcf_name == nullptr) { + // exception thrown: OutOfMemoryError + env->DeleteLocalRef(jcn); + env->ReleaseLongArrayElements(jcolumn_options_handles, jco, JNI_ABORT); + env->ReleaseStringUTFChars(jdb_path, db_path); + return nullptr; + } + + const std::string cf_name(reinterpret_cast(jcf_name), + jcf_name_len); + const ROCKSDB_NAMESPACE::ColumnFamilyOptions* cf_options = + reinterpret_cast(jco[i]); + column_families.push_back( + ROCKSDB_NAMESPACE::ColumnFamilyDescriptor(cf_name, *cf_options)); + + env->ReleaseByteArrayElements(jcn_ba, jcf_name, JNI_ABORT); + env->DeleteLocalRef(jcn); + } + env->ReleaseLongArrayElements(jcolumn_options_handles, jco, JNI_ABORT); + } + + auto* db_options = + reinterpret_cast(jdb_options_handle); + std::vector handles; + ROCKSDB_NAMESPACE::OptimisticTransactionDB* otdb = nullptr; + const ROCKSDB_NAMESPACE::Status s = + ROCKSDB_NAMESPACE::OptimisticTransactionDB::Open( + *db_options, db_path, column_families, &handles, &otdb); + + env->ReleaseStringUTFChars(jdb_path, db_path); + + // check if open operation was successful + if (s.ok()) { + const jsize resultsLen = 1 + len_cols; // db handle + column family handles + std::unique_ptr results = + std::unique_ptr(new jlong[resultsLen]); + results[0] = reinterpret_cast(otdb); + for (int i = 1; i <= len_cols; i++) { + results[i] = reinterpret_cast(handles[i - 1]); + } + + jlongArray jresults = env->NewLongArray(resultsLen); + if (jresults == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + env->SetLongArrayRegion(jresults, 0, resultsLen, results.get()); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + return nullptr; + } + return jresults; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return nullptr; +} + +/* + * Class: org_rocksdb_OptimisticTransactionDB + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_OptimisticTransactionDB_disposeInternal(JNIEnv*, jobject, + jlong jhandle) { + auto* optimistic_txn_db = + reinterpret_cast(jhandle); + assert(optimistic_txn_db != nullptr); + delete optimistic_txn_db; +} + +/* + * Class: org_rocksdb_OptimisticTransactionDB + * Method: closeDatabase + * Signature: (J)V + */ +void Java_org_rocksdb_OptimisticTransactionDB_closeDatabase(JNIEnv* env, jclass, + jlong jhandle) { + auto* optimistic_txn_db = + reinterpret_cast(jhandle); + assert(optimistic_txn_db != nullptr); + ROCKSDB_NAMESPACE::Status s = optimistic_txn_db->Close(); + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); +} + +/* + * Class: org_rocksdb_OptimisticTransactionDB + * Method: beginTransaction + * Signature: (JJ)J + */ +jlong Java_org_rocksdb_OptimisticTransactionDB_beginTransaction__JJ( + JNIEnv*, jobject, jlong jhandle, jlong jwrite_options_handle) { + auto* optimistic_txn_db = + reinterpret_cast(jhandle); + auto* write_options = + reinterpret_cast(jwrite_options_handle); + ROCKSDB_NAMESPACE::Transaction* txn = + optimistic_txn_db->BeginTransaction(*write_options); + return GET_CPLUSPLUS_POINTER(txn); +} + +/* + * Class: org_rocksdb_OptimisticTransactionDB + * Method: beginTransaction + * Signature: (JJJ)J + */ +jlong Java_org_rocksdb_OptimisticTransactionDB_beginTransaction__JJJ( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, + jlong jwrite_options_handle, jlong joptimistic_txn_options_handle) { + auto* optimistic_txn_db = + reinterpret_cast(jhandle); + auto* write_options = + reinterpret_cast(jwrite_options_handle); + auto* optimistic_txn_options = + reinterpret_cast( + joptimistic_txn_options_handle); + ROCKSDB_NAMESPACE::Transaction* txn = optimistic_txn_db->BeginTransaction( + *write_options, *optimistic_txn_options); + return GET_CPLUSPLUS_POINTER(txn); +} + +/* + * Class: org_rocksdb_OptimisticTransactionDB + * Method: beginTransaction_withOld + * Signature: (JJJ)J + */ +jlong Java_org_rocksdb_OptimisticTransactionDB_beginTransaction_1withOld__JJJ( + JNIEnv*, jobject, jlong jhandle, jlong jwrite_options_handle, + jlong jold_txn_handle) { + auto* optimistic_txn_db = + reinterpret_cast(jhandle); + auto* write_options = + reinterpret_cast(jwrite_options_handle); + auto* old_txn = + reinterpret_cast(jold_txn_handle); + ROCKSDB_NAMESPACE::OptimisticTransactionOptions optimistic_txn_options; + ROCKSDB_NAMESPACE::Transaction* txn = optimistic_txn_db->BeginTransaction( + *write_options, optimistic_txn_options, old_txn); + + // RocksJava relies on the assumption that + // we do not allocate a new Transaction object + // when providing an old_optimistic_txn + assert(txn == old_txn); + + return GET_CPLUSPLUS_POINTER(txn); +} + +/* + * Class: org_rocksdb_OptimisticTransactionDB + * Method: beginTransaction_withOld + * Signature: (JJJJ)J + */ +jlong Java_org_rocksdb_OptimisticTransactionDB_beginTransaction_1withOld__JJJJ( + JNIEnv*, jobject, jlong jhandle, jlong jwrite_options_handle, + jlong joptimistic_txn_options_handle, jlong jold_txn_handle) { + auto* optimistic_txn_db = + reinterpret_cast(jhandle); + auto* write_options = + reinterpret_cast(jwrite_options_handle); + auto* optimistic_txn_options = + reinterpret_cast( + joptimistic_txn_options_handle); + auto* old_txn = + reinterpret_cast(jold_txn_handle); + ROCKSDB_NAMESPACE::Transaction* txn = optimistic_txn_db->BeginTransaction( + *write_options, *optimistic_txn_options, old_txn); + + // RocksJava relies on the assumption that + // we do not allocate a new Transaction object + // when providing an old_optimisic_txn + assert(txn == old_txn); + + return GET_CPLUSPLUS_POINTER(txn); +} + +/* + * Class: org_rocksdb_OptimisticTransactionDB + * Method: getBaseDB + * Signature: (J)J + */ +jlong Java_org_rocksdb_OptimisticTransactionDB_getBaseDB(JNIEnv*, jobject, + jlong jhandle) { + auto* optimistic_txn_db = + reinterpret_cast(jhandle); + return GET_CPLUSPLUS_POINTER(optimistic_txn_db->GetBaseDB()); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/optimistic_transaction_options.cc b/librocksdb-sys/rocksdb/java/rocksjni/optimistic_transaction_options.cc new file mode 100644 index 0000000..501c6c4 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/optimistic_transaction_options.cc @@ -0,0 +1,78 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ +// for ROCKSDB_NAMESPACE::OptimisticTransactionOptions. + +#include + +#include "include/org_rocksdb_OptimisticTransactionOptions.h" +#include "rocksdb/comparator.h" +#include "rocksdb/utilities/optimistic_transaction_db.h" +#include "rocksjni/cplusplus_to_java_convert.h" + +/* + * Class: org_rocksdb_OptimisticTransactionOptions + * Method: newOptimisticTransactionOptions + * Signature: ()J + */ +jlong Java_org_rocksdb_OptimisticTransactionOptions_newOptimisticTransactionOptions( + JNIEnv* /*env*/, jclass /*jcls*/) { + ROCKSDB_NAMESPACE::OptimisticTransactionOptions* opts = + new ROCKSDB_NAMESPACE::OptimisticTransactionOptions(); + return GET_CPLUSPLUS_POINTER(opts); +} + +/* + * Class: org_rocksdb_OptimisticTransactionOptions + * Method: isSetSnapshot + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_OptimisticTransactionOptions_isSetSnapshot( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { + auto* opts = + reinterpret_cast( + jhandle); + return opts->set_snapshot; +} + +/* + * Class: org_rocksdb_OptimisticTransactionOptions + * Method: setSetSnapshot + * Signature: (JZ)V + */ +void Java_org_rocksdb_OptimisticTransactionOptions_setSetSnapshot( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jboolean jset_snapshot) { + auto* opts = + reinterpret_cast( + jhandle); + opts->set_snapshot = jset_snapshot; +} + +/* + * Class: org_rocksdb_OptimisticTransactionOptions + * Method: setComparator + * Signature: (JJ)V + */ +void Java_org_rocksdb_OptimisticTransactionOptions_setComparator( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, + jlong jcomparator_handle) { + auto* opts = + reinterpret_cast( + jhandle); + opts->cmp = + reinterpret_cast(jcomparator_handle); +} + +/* + * Class: org_rocksdb_OptimisticTransactionOptions + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_OptimisticTransactionOptions_disposeInternal( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { + delete reinterpret_cast( + jhandle); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/options.cc b/librocksdb-sys/rocksdb/java/rocksjni/options.cc new file mode 100644 index 0000000..0d84901 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/options.cc @@ -0,0 +1,8742 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::Options. + +#include "rocksdb/options.h" + +#include +#include +#include + +#include +#include + +#include "include/org_rocksdb_ColumnFamilyOptions.h" +#include "include/org_rocksdb_ComparatorOptions.h" +#include "include/org_rocksdb_DBOptions.h" +#include "include/org_rocksdb_FlushOptions.h" +#include "include/org_rocksdb_Options.h" +#include "include/org_rocksdb_ReadOptions.h" +#include "include/org_rocksdb_WriteOptions.h" +#include "rocksdb/comparator.h" +#include "rocksdb/convenience.h" +#include "rocksdb/db.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/rate_limiter.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/sst_partitioner.h" +#include "rocksdb/statistics.h" +#include "rocksdb/table.h" +#include "rocksjni/comparatorjnicallback.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" +#include "rocksjni/statisticsjni.h" +#include "rocksjni/table_filter_jnicallback.h" +#include "utilities/merge_operators.h" + +/* + * Class: org_rocksdb_Options + * Method: newOptions + * Signature: ()J + */ +jlong Java_org_rocksdb_Options_newOptions__(JNIEnv*, jclass) { + auto* op = new ROCKSDB_NAMESPACE::Options(); + return GET_CPLUSPLUS_POINTER(op); +} + +/* + * Class: org_rocksdb_Options + * Method: newOptions + * Signature: (JJ)J + */ +jlong Java_org_rocksdb_Options_newOptions__JJ(JNIEnv*, jclass, jlong jdboptions, + jlong jcfoptions) { + auto* dbOpt = + reinterpret_cast(jdboptions); + auto* cfOpt = reinterpret_cast( + jcfoptions); + auto* op = new ROCKSDB_NAMESPACE::Options(*dbOpt, *cfOpt); + return GET_CPLUSPLUS_POINTER(op); +} + +/* + * Class: org_rocksdb_Options + * Method: copyOptions + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_copyOptions(JNIEnv*, jclass, jlong jhandle) { + auto new_opt = new ROCKSDB_NAMESPACE::Options( + *(reinterpret_cast(jhandle))); + return GET_CPLUSPLUS_POINTER(new_opt); +} + +/* + * Class: org_rocksdb_Options + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_Options_disposeInternal(JNIEnv*, jobject, jlong handle) { + auto* op = reinterpret_cast(handle); + assert(op != nullptr); + delete op; +} + +/* + * Class: org_rocksdb_Options + * Method: setIncreaseParallelism + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setIncreaseParallelism(JNIEnv*, jobject, + jlong jhandle, + jint totalThreads) { + reinterpret_cast(jhandle)->IncreaseParallelism( + static_cast(totalThreads)); +} + +/* + * Class: org_rocksdb_Options + * Method: setCreateIfMissing + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setCreateIfMissing(JNIEnv*, jobject, + jlong jhandle, jboolean flag) { + reinterpret_cast(jhandle)->create_if_missing = + flag; +} + +/* + * Class: org_rocksdb_Options + * Method: createIfMissing + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_createIfMissing(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->create_if_missing; +} + +/* + * Class: org_rocksdb_Options + * Method: setCreateMissingColumnFamilies + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setCreateMissingColumnFamilies(JNIEnv*, jobject, + jlong jhandle, + jboolean flag) { + reinterpret_cast(jhandle) + ->create_missing_column_families = flag; +} + +/* + * Class: org_rocksdb_Options + * Method: createMissingColumnFamilies + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_createMissingColumnFamilies(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->create_missing_column_families; +} + +/* + * Class: org_rocksdb_Options + * Method: setComparatorHandle + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setComparatorHandle__JI(JNIEnv*, jobject, + jlong jhandle, + jint builtinComparator) { + switch (builtinComparator) { + case 1: + reinterpret_cast(jhandle)->comparator = + ROCKSDB_NAMESPACE::ReverseBytewiseComparator(); + break; + default: + reinterpret_cast(jhandle)->comparator = + ROCKSDB_NAMESPACE::BytewiseComparator(); + break; + } +} + +/* + * Class: org_rocksdb_Options + * Method: setComparatorHandle + * Signature: (JJB)V + */ +void Java_org_rocksdb_Options_setComparatorHandle__JJB(JNIEnv*, jobject, + jlong jopt_handle, + jlong jcomparator_handle, + jbyte jcomparator_type) { + ROCKSDB_NAMESPACE::Comparator* comparator = nullptr; + switch (jcomparator_type) { + // JAVA_COMPARATOR + case 0x0: + comparator = reinterpret_cast( + jcomparator_handle); + break; + + // JAVA_NATIVE_COMPARATOR_WRAPPER + case 0x1: + comparator = + reinterpret_cast(jcomparator_handle); + break; + } + auto* opt = reinterpret_cast(jopt_handle); + opt->comparator = comparator; +} + +/* + * Class: org_rocksdb_Options + * Method: setMergeOperatorName + * Signature: (JJjava/lang/String)V + */ +void Java_org_rocksdb_Options_setMergeOperatorName(JNIEnv* env, jobject, + jlong jhandle, + jstring jop_name) { + const char* op_name = env->GetStringUTFChars(jop_name, nullptr); + if (op_name == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + auto* options = reinterpret_cast(jhandle); + options->merge_operator = + ROCKSDB_NAMESPACE::MergeOperators::CreateFromStringId(op_name); + + env->ReleaseStringUTFChars(jop_name, op_name); +} + +/* + * Class: org_rocksdb_Options + * Method: setMergeOperator + * Signature: (JJjava/lang/String)V + */ +void Java_org_rocksdb_Options_setMergeOperator(JNIEnv*, jobject, jlong jhandle, + jlong mergeOperatorHandle) { + reinterpret_cast(jhandle)->merge_operator = + *(reinterpret_cast*>( + mergeOperatorHandle)); +} + +/* + * Class: org_rocksdb_Options + * Method: setCompactionFilterHandle + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setCompactionFilterHandle( + JNIEnv*, jobject, jlong jopt_handle, jlong jcompactionfilter_handle) { + reinterpret_cast(jopt_handle) + ->compaction_filter = + reinterpret_cast( + jcompactionfilter_handle); +} + +/* + * Class: org_rocksdb_Options + * Method: setCompactionFilterFactoryHandle + * Signature: (JJ)V + */ +void JNICALL Java_org_rocksdb_Options_setCompactionFilterFactoryHandle( + JNIEnv*, jobject, jlong jopt_handle, + jlong jcompactionfilterfactory_handle) { + auto* cff_factory = reinterpret_cast< + std::shared_ptr*>( + jcompactionfilterfactory_handle); + reinterpret_cast(jopt_handle) + ->compaction_filter_factory = *cff_factory; +} + +/* + * Class: org_rocksdb_Options + * Method: setWriteBufferSize + * Signature: (JJ)I + */ +void Java_org_rocksdb_Options_setWriteBufferSize(JNIEnv* env, jobject, + jlong jhandle, + jlong jwrite_buffer_size) { + auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( + jwrite_buffer_size); + if (s.ok()) { + reinterpret_cast(jhandle)->write_buffer_size = + jwrite_buffer_size; + } else { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_Options + * Method: setWriteBufferManager + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setWriteBufferManager( + JNIEnv*, jobject, jlong joptions_handle, + jlong jwrite_buffer_manager_handle) { + auto* write_buffer_manager = + reinterpret_cast*>( + jwrite_buffer_manager_handle); + reinterpret_cast(joptions_handle) + ->write_buffer_manager = *write_buffer_manager; +} + +/* + * Class: org_rocksdb_Options + * Method: writeBufferSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_writeBufferSize(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->write_buffer_size; +} + +/* + * Class: org_rocksdb_Options + * Method: setMaxWriteBufferNumber + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setMaxWriteBufferNumber( + JNIEnv*, jobject, jlong jhandle, jint jmax_write_buffer_number) { + reinterpret_cast(jhandle) + ->max_write_buffer_number = jmax_write_buffer_number; +} + +/* + * Class: org_rocksdb_Options + * Method: setStatistics + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setStatistics(JNIEnv*, jobject, jlong jhandle, + jlong jstatistics_handle) { + auto* opt = reinterpret_cast(jhandle); + auto* pSptr = + reinterpret_cast*>( + jstatistics_handle); + opt->statistics = *pSptr; +} + +/* + * Class: org_rocksdb_Options + * Method: statistics + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_statistics(JNIEnv*, jobject, jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + std::shared_ptr sptr = opt->statistics; + if (sptr == nullptr) { + return 0; + } else { + std::shared_ptr* pSptr = + new std::shared_ptr(sptr); + return GET_CPLUSPLUS_POINTER(pSptr); + } +} + +/* + * Class: org_rocksdb_Options + * Method: maxWriteBufferNumber + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_maxWriteBufferNumber(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_write_buffer_number; +} + +/* + * Class: org_rocksdb_Options + * Method: errorIfExists + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_errorIfExists(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->error_if_exists; +} + +/* + * Class: org_rocksdb_Options + * Method: setErrorIfExists + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setErrorIfExists(JNIEnv*, jobject, jlong jhandle, + jboolean error_if_exists) { + reinterpret_cast(jhandle)->error_if_exists = + static_cast(error_if_exists); +} + +/* + * Class: org_rocksdb_Options + * Method: paranoidChecks + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_paranoidChecks(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->paranoid_checks; +} + +/* + * Class: org_rocksdb_Options + * Method: setParanoidChecks + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setParanoidChecks(JNIEnv*, jobject, jlong jhandle, + jboolean paranoid_checks) { + reinterpret_cast(jhandle)->paranoid_checks = + static_cast(paranoid_checks); +} + +/* + * Class: org_rocksdb_Options + * Method: setEnv + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setEnv(JNIEnv*, jobject, jlong jhandle, + jlong jenv) { + reinterpret_cast(jhandle)->env = + reinterpret_cast(jenv); +} + +/* + * Class: org_rocksdb_Options + * Method: setMaxTotalWalSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setMaxTotalWalSize(JNIEnv*, jobject, + jlong jhandle, + jlong jmax_total_wal_size) { + reinterpret_cast(jhandle)->max_total_wal_size = + static_cast(jmax_total_wal_size); +} + +/* + * Class: org_rocksdb_Options + * Method: maxTotalWalSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_maxTotalWalSize(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_total_wal_size; +} + +/* + * Class: org_rocksdb_Options + * Method: maxOpenFiles + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_maxOpenFiles(JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle)->max_open_files; +} + +/* + * Class: org_rocksdb_Options + * Method: setMaxOpenFiles + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setMaxOpenFiles(JNIEnv*, jobject, jlong jhandle, + jint max_open_files) { + reinterpret_cast(jhandle)->max_open_files = + static_cast(max_open_files); +} + +/* + * Class: org_rocksdb_Options + * Method: setMaxFileOpeningThreads + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setMaxFileOpeningThreads( + JNIEnv*, jobject, jlong jhandle, jint jmax_file_opening_threads) { + reinterpret_cast(jhandle) + ->max_file_opening_threads = static_cast(jmax_file_opening_threads); +} + +/* + * Class: org_rocksdb_Options + * Method: maxFileOpeningThreads + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_maxFileOpeningThreads(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->max_file_opening_threads); +} + +/* + * Class: org_rocksdb_Options + * Method: useFsync + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_useFsync(JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle)->use_fsync; +} + +/* + * Class: org_rocksdb_Options + * Method: setUseFsync + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setUseFsync(JNIEnv*, jobject, jlong jhandle, + jboolean use_fsync) { + reinterpret_cast(jhandle)->use_fsync = + static_cast(use_fsync); +} + +/* + * Class: org_rocksdb_Options + * Method: setDbPaths + * Signature: (J[Ljava/lang/String;[J)V + */ +void Java_org_rocksdb_Options_setDbPaths(JNIEnv* env, jobject, jlong jhandle, + jobjectArray jpaths, + jlongArray jtarget_sizes) { + std::vector db_paths; + jlong* ptr_jtarget_size = env->GetLongArrayElements(jtarget_sizes, nullptr); + if (ptr_jtarget_size == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + jboolean has_exception = JNI_FALSE; + const jsize len = env->GetArrayLength(jpaths); + for (jsize i = 0; i < len; i++) { + jobject jpath = + reinterpret_cast(env->GetObjectArrayElement(jpaths, i)); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->ReleaseLongArrayElements(jtarget_sizes, ptr_jtarget_size, JNI_ABORT); + return; + } + std::string path = ROCKSDB_NAMESPACE::JniUtil::copyStdString( + env, static_cast(jpath), &has_exception); + env->DeleteLocalRef(jpath); + + if (has_exception == JNI_TRUE) { + env->ReleaseLongArrayElements(jtarget_sizes, ptr_jtarget_size, JNI_ABORT); + return; + } + + jlong jtarget_size = ptr_jtarget_size[i]; + + db_paths.push_back( + ROCKSDB_NAMESPACE::DbPath(path, static_cast(jtarget_size))); + } + + env->ReleaseLongArrayElements(jtarget_sizes, ptr_jtarget_size, JNI_ABORT); + + auto* opt = reinterpret_cast(jhandle); + opt->db_paths = db_paths; +} + +/* + * Class: org_rocksdb_Options + * Method: dbPathsLen + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_dbPathsLen(JNIEnv*, jobject, jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->db_paths.size()); +} + +/* + * Class: org_rocksdb_Options + * Method: dbPaths + * Signature: (J[Ljava/lang/String;[J)V + */ +void Java_org_rocksdb_Options_dbPaths(JNIEnv* env, jobject, jlong jhandle, + jobjectArray jpaths, + jlongArray jtarget_sizes) { + jboolean is_copy; + jlong* ptr_jtarget_size = env->GetLongArrayElements(jtarget_sizes, &is_copy); + if (ptr_jtarget_size == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + auto* opt = reinterpret_cast(jhandle); + const jsize len = env->GetArrayLength(jpaths); + for (jsize i = 0; i < len; i++) { + ROCKSDB_NAMESPACE::DbPath db_path = opt->db_paths[i]; + + jstring jpath = env->NewStringUTF(db_path.path.c_str()); + if (jpath == nullptr) { + // exception thrown: OutOfMemoryError + env->ReleaseLongArrayElements(jtarget_sizes, ptr_jtarget_size, JNI_ABORT); + return; + } + env->SetObjectArrayElement(jpaths, i, jpath); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jpath); + env->ReleaseLongArrayElements(jtarget_sizes, ptr_jtarget_size, JNI_ABORT); + return; + } + + ptr_jtarget_size[i] = static_cast(db_path.target_size); + } + + env->ReleaseLongArrayElements(jtarget_sizes, ptr_jtarget_size, + is_copy == JNI_TRUE ? 0 : JNI_ABORT); +} + +/* + * Class: org_rocksdb_Options + * Method: dbLogDir + * Signature: (J)Ljava/lang/String + */ +jstring Java_org_rocksdb_Options_dbLogDir(JNIEnv* env, jobject, jlong jhandle) { + return env->NewStringUTF( + reinterpret_cast(jhandle) + ->db_log_dir.c_str()); +} + +/* + * Class: org_rocksdb_Options + * Method: setDbLogDir + * Signature: (JLjava/lang/String)V + */ +void Java_org_rocksdb_Options_setDbLogDir(JNIEnv* env, jobject, jlong jhandle, + jstring jdb_log_dir) { + const char* log_dir = env->GetStringUTFChars(jdb_log_dir, nullptr); + if (log_dir == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + reinterpret_cast(jhandle)->db_log_dir.assign( + log_dir); + env->ReleaseStringUTFChars(jdb_log_dir, log_dir); +} + +/* + * Class: org_rocksdb_Options + * Method: walDir + * Signature: (J)Ljava/lang/String + */ +jstring Java_org_rocksdb_Options_walDir(JNIEnv* env, jobject, jlong jhandle) { + return env->NewStringUTF( + reinterpret_cast(jhandle)->wal_dir.c_str()); +} + +/* + * Class: org_rocksdb_Options + * Method: setWalDir + * Signature: (JLjava/lang/String)V + */ +void Java_org_rocksdb_Options_setWalDir(JNIEnv* env, jobject, jlong jhandle, + jstring jwal_dir) { + const char* wal_dir = env->GetStringUTFChars(jwal_dir, nullptr); + if (wal_dir == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + reinterpret_cast(jhandle)->wal_dir.assign( + wal_dir); + env->ReleaseStringUTFChars(jwal_dir, wal_dir); +} + +/* + * Class: org_rocksdb_Options + * Method: deleteObsoleteFilesPeriodMicros + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_deleteObsoleteFilesPeriodMicros(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->delete_obsolete_files_period_micros; +} + +/* + * Class: org_rocksdb_Options + * Method: setDeleteObsoleteFilesPeriodMicros + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setDeleteObsoleteFilesPeriodMicros(JNIEnv*, + jobject, + jlong jhandle, + jlong micros) { + reinterpret_cast(jhandle) + ->delete_obsolete_files_period_micros = static_cast(micros); +} + +/* + * Class: org_rocksdb_Options + * Method: maxBackgroundCompactions + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_maxBackgroundCompactions(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_background_compactions; +} + +/* + * Class: org_rocksdb_Options + * Method: setMaxBackgroundCompactions + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setMaxBackgroundCompactions(JNIEnv*, jobject, + jlong jhandle, + jint max) { + reinterpret_cast(jhandle) + ->max_background_compactions = static_cast(max); +} + +/* + * Class: org_rocksdb_Options + * Method: setMaxSubcompactions + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setMaxSubcompactions(JNIEnv*, jobject, + jlong jhandle, jint max) { + reinterpret_cast(jhandle)->max_subcompactions = + static_cast(max); +} + +/* + * Class: org_rocksdb_Options + * Method: maxSubcompactions + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_maxSubcompactions(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_subcompactions; +} + +/* + * Class: org_rocksdb_Options + * Method: maxBackgroundFlushes + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_maxBackgroundFlushes(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_background_flushes; +} + +/* + * Class: org_rocksdb_Options + * Method: setMaxBackgroundFlushes + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setMaxBackgroundFlushes( + JNIEnv*, jobject, jlong jhandle, jint max_background_flushes) { + reinterpret_cast(jhandle) + ->max_background_flushes = static_cast(max_background_flushes); +} + +/* + * Class: org_rocksdb_Options + * Method: maxBackgroundJobs + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_maxBackgroundJobs(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_background_jobs; +} + +/* + * Class: org_rocksdb_Options + * Method: setMaxBackgroundJobs + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setMaxBackgroundJobs(JNIEnv*, jobject, + jlong jhandle, + jint max_background_jobs) { + reinterpret_cast(jhandle)->max_background_jobs = + static_cast(max_background_jobs); +} + +/* + * Class: org_rocksdb_Options + * Method: maxLogFileSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_maxLogFileSize(JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_log_file_size; +} + +/* + * Class: org_rocksdb_Options + * Method: setMaxLogFileSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setMaxLogFileSize(JNIEnv* env, jobject, + jlong jhandle, + jlong max_log_file_size) { + auto s = + ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t(max_log_file_size); + if (s.ok()) { + reinterpret_cast(jhandle)->max_log_file_size = + max_log_file_size; + } else { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_Options + * Method: logFileTimeToRoll + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_logFileTimeToRoll(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->log_file_time_to_roll; +} + +/* + * Class: org_rocksdb_Options + * Method: setLogFileTimeToRoll + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setLogFileTimeToRoll( + JNIEnv* env, jobject, jlong jhandle, jlong log_file_time_to_roll) { + auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( + log_file_time_to_roll); + if (s.ok()) { + reinterpret_cast(jhandle) + ->log_file_time_to_roll = log_file_time_to_roll; + } else { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_Options + * Method: keepLogFileNum + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_keepLogFileNum(JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->keep_log_file_num; +} + +/* + * Class: org_rocksdb_Options + * Method: setKeepLogFileNum + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setKeepLogFileNum(JNIEnv* env, jobject, + jlong jhandle, + jlong keep_log_file_num) { + auto s = + ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t(keep_log_file_num); + if (s.ok()) { + reinterpret_cast(jhandle)->keep_log_file_num = + keep_log_file_num; + } else { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_Options + * Method: recycleLogFileNum + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_recycleLogFileNum(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->recycle_log_file_num; +} + +/* + * Class: org_rocksdb_Options + * Method: setRecycleLogFileNum + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setRecycleLogFileNum(JNIEnv* env, jobject, + jlong jhandle, + jlong recycle_log_file_num) { + auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( + recycle_log_file_num); + if (s.ok()) { + reinterpret_cast(jhandle) + ->recycle_log_file_num = recycle_log_file_num; + } else { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_Options + * Method: maxManifestFileSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_maxManifestFileSize(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_manifest_file_size; +} + +/* + * Method: memTableFactoryName + * Signature: (J)Ljava/lang/String + */ +jstring Java_org_rocksdb_Options_memTableFactoryName(JNIEnv* env, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + ROCKSDB_NAMESPACE::MemTableRepFactory* tf = opt->memtable_factory.get(); + + // Should never be nullptr. + // Default memtable factory is SkipListFactory + assert(tf); + + // temporarly fix for the historical typo + if (strcmp(tf->Name(), "HashLinkListRepFactory") == 0) { + return env->NewStringUTF("HashLinkedListRepFactory"); + } + + return env->NewStringUTF(tf->Name()); +} + +static std::vector +rocksdb_convert_cf_paths_from_java_helper(JNIEnv* env, jobjectArray path_array, + jlongArray size_array, + jboolean* has_exception) { + jboolean copy_str_has_exception; + std::vector paths = ROCKSDB_NAMESPACE::JniUtil::copyStrings( + env, path_array, ©_str_has_exception); + if (JNI_TRUE == copy_str_has_exception) { + // Exception thrown + *has_exception = JNI_TRUE; + return {}; + } + + if (static_cast(env->GetArrayLength(size_array)) != paths.size()) { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew( + env, + ROCKSDB_NAMESPACE::Status::InvalidArgument( + ROCKSDB_NAMESPACE::Slice("There should be a corresponding target " + "size for every path and vice versa."))); + *has_exception = JNI_TRUE; + return {}; + } + + jlong* size_array_ptr = env->GetLongArrayElements(size_array, nullptr); + if (nullptr == size_array_ptr) { + // exception thrown: OutOfMemoryError + *has_exception = JNI_TRUE; + return {}; + } + std::vector cf_paths; + for (size_t i = 0; i < paths.size(); ++i) { + jlong target_size = size_array_ptr[i]; + if (target_size < 0) { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew( + env, + ROCKSDB_NAMESPACE::Status::InvalidArgument(ROCKSDB_NAMESPACE::Slice( + "Path target size has to be positive."))); + *has_exception = JNI_TRUE; + env->ReleaseLongArrayElements(size_array, size_array_ptr, JNI_ABORT); + return {}; + } + cf_paths.push_back(ROCKSDB_NAMESPACE::DbPath( + paths[i], static_cast(target_size))); + } + + env->ReleaseLongArrayElements(size_array, size_array_ptr, JNI_ABORT); + + return cf_paths; +} + +/* + * Class: org_rocksdb_Options + * Method: setCfPaths + * Signature: (J[Ljava/lang/String;[J)V + */ +void Java_org_rocksdb_Options_setCfPaths(JNIEnv* env, jclass, jlong jhandle, + jobjectArray path_array, + jlongArray size_array) { + auto* options = reinterpret_cast(jhandle); + jboolean has_exception = JNI_FALSE; + std::vector cf_paths = + rocksdb_convert_cf_paths_from_java_helper(env, path_array, size_array, + &has_exception); + if (JNI_FALSE == has_exception) { + options->cf_paths = std::move(cf_paths); + } +} + +/* + * Class: org_rocksdb_Options + * Method: cfPathsLen + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_cfPathsLen(JNIEnv*, jclass, jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->cf_paths.size()); +} + +template +static void rocksdb_convert_cf_paths_to_java_helper(JNIEnv* env, jlong jhandle, + jobjectArray jpaths, + jlongArray jtarget_sizes) { + jboolean is_copy; + jlong* ptr_jtarget_size = env->GetLongArrayElements(jtarget_sizes, &is_copy); + if (ptr_jtarget_size == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + auto* opt = reinterpret_cast(jhandle); + const jsize len = env->GetArrayLength(jpaths); + for (jsize i = 0; i < len; i++) { + ROCKSDB_NAMESPACE::DbPath cf_path = opt->cf_paths[i]; + + jstring jpath = env->NewStringUTF(cf_path.path.c_str()); + if (jpath == nullptr) { + // exception thrown: OutOfMemoryError + env->ReleaseLongArrayElements(jtarget_sizes, ptr_jtarget_size, JNI_ABORT); + return; + } + env->SetObjectArrayElement(jpaths, i, jpath); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jpath); + env->ReleaseLongArrayElements(jtarget_sizes, ptr_jtarget_size, JNI_ABORT); + return; + } + + ptr_jtarget_size[i] = static_cast(cf_path.target_size); + + env->DeleteLocalRef(jpath); + } + + env->ReleaseLongArrayElements(jtarget_sizes, ptr_jtarget_size, + is_copy ? 0 : JNI_ABORT); +} + +/* + * Class: org_rocksdb_Options + * Method: cfPaths + * Signature: (J[Ljava/lang/String;[J)V + */ +void Java_org_rocksdb_Options_cfPaths(JNIEnv* env, jclass, jlong jhandle, + jobjectArray jpaths, + jlongArray jtarget_sizes) { + rocksdb_convert_cf_paths_to_java_helper( + env, jhandle, jpaths, jtarget_sizes); +} + +/* + * Class: org_rocksdb_Options + * Method: setMaxManifestFileSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setMaxManifestFileSize( + JNIEnv*, jobject, jlong jhandle, jlong max_manifest_file_size) { + reinterpret_cast(jhandle) + ->max_manifest_file_size = static_cast(max_manifest_file_size); +} + +/* + * Method: setMemTableFactory + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setMemTableFactory(JNIEnv*, jobject, + jlong jhandle, + jlong jfactory_handle) { + reinterpret_cast(jhandle) + ->memtable_factory.reset( + reinterpret_cast( + jfactory_handle)); +} + +/* + * Class: org_rocksdb_Options + * Method: setRateLimiter + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setRateLimiter(JNIEnv*, jobject, jlong jhandle, + jlong jrate_limiter_handle) { + std::shared_ptr* pRateLimiter = + reinterpret_cast*>( + jrate_limiter_handle); + reinterpret_cast(jhandle)->rate_limiter = + *pRateLimiter; +} + +/* + * Class: org_rocksdb_Options + * Method: setSstFileManager + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setSstFileManager( + JNIEnv*, jobject, jlong jhandle, jlong jsst_file_manager_handle) { + auto* sptr_sst_file_manager = + reinterpret_cast*>( + jsst_file_manager_handle); + reinterpret_cast(jhandle)->sst_file_manager = + *sptr_sst_file_manager; +} + +/* + * Class: org_rocksdb_Options + * Method: setLogger + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setLogger(JNIEnv*, jobject, jlong jhandle, + jlong jlogger_handle) { + std::shared_ptr* pLogger = + reinterpret_cast*>( + jlogger_handle); + reinterpret_cast(jhandle)->info_log = *pLogger; +} + +/* + * Class: org_rocksdb_Options + * Method: setInfoLogLevel + * Signature: (JB)V + */ +void Java_org_rocksdb_Options_setInfoLogLevel(JNIEnv*, jobject, jlong jhandle, + jbyte jlog_level) { + reinterpret_cast(jhandle)->info_log_level = + static_cast(jlog_level); +} + +/* + * Class: org_rocksdb_Options + * Method: infoLogLevel + * Signature: (J)B + */ +jbyte Java_org_rocksdb_Options_infoLogLevel(JNIEnv*, jobject, jlong jhandle) { + return static_cast( + reinterpret_cast(jhandle)->info_log_level); +} + +/* + * Class: org_rocksdb_Options + * Method: tableCacheNumshardbits + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_tableCacheNumshardbits(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->table_cache_numshardbits; +} + +/* + * Class: org_rocksdb_Options + * Method: setTableCacheNumshardbits + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setTableCacheNumshardbits( + JNIEnv*, jobject, jlong jhandle, jint table_cache_numshardbits) { + reinterpret_cast(jhandle) + ->table_cache_numshardbits = static_cast(table_cache_numshardbits); +} + +/* + * Method: useFixedLengthPrefixExtractor + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_useFixedLengthPrefixExtractor( + JNIEnv*, jobject, jlong jhandle, jint jprefix_length) { + reinterpret_cast(jhandle) + ->prefix_extractor.reset(ROCKSDB_NAMESPACE::NewFixedPrefixTransform( + static_cast(jprefix_length))); +} + +/* + * Method: useCappedPrefixExtractor + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_useCappedPrefixExtractor(JNIEnv*, jobject, + jlong jhandle, + jint jprefix_length) { + reinterpret_cast(jhandle) + ->prefix_extractor.reset(ROCKSDB_NAMESPACE::NewCappedPrefixTransform( + static_cast(jprefix_length))); +} + +/* + * Class: org_rocksdb_Options + * Method: walTtlSeconds + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_walTtlSeconds(JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->WAL_ttl_seconds; +} + +/* + * Class: org_rocksdb_Options + * Method: setWalTtlSeconds + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setWalTtlSeconds(JNIEnv*, jobject, jlong jhandle, + jlong WAL_ttl_seconds) { + reinterpret_cast(jhandle)->WAL_ttl_seconds = + static_cast(WAL_ttl_seconds); +} + +/* + * Class: org_rocksdb_Options + * Method: walTtlSeconds + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_walSizeLimitMB(JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->WAL_size_limit_MB; +} + +/* + * Class: org_rocksdb_Options + * Method: setWalSizeLimitMB + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setWalSizeLimitMB(JNIEnv*, jobject, jlong jhandle, + jlong WAL_size_limit_MB) { + reinterpret_cast(jhandle)->WAL_size_limit_MB = + static_cast(WAL_size_limit_MB); +} + +/* + * Class: org_rocksdb_Options + * Method: setMaxWriteBatchGroupSizeBytes + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setMaxWriteBatchGroupSizeBytes( + JNIEnv*, jclass, jlong jhandle, jlong jmax_write_batch_group_size_bytes) { + auto* opt = reinterpret_cast(jhandle); + opt->max_write_batch_group_size_bytes = + static_cast(jmax_write_batch_group_size_bytes); +} + +/* + * Class: org_rocksdb_Options + * Method: maxWriteBatchGroupSizeBytes + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_maxWriteBatchGroupSizeBytes(JNIEnv*, jclass, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->max_write_batch_group_size_bytes); +} + +/* + * Class: org_rocksdb_Options + * Method: manifestPreallocationSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_manifestPreallocationSize(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->manifest_preallocation_size; +} + +/* + * Class: org_rocksdb_Options + * Method: setManifestPreallocationSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setManifestPreallocationSize( + JNIEnv* env, jobject, jlong jhandle, jlong preallocation_size) { + auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( + preallocation_size); + if (s.ok()) { + reinterpret_cast(jhandle) + ->manifest_preallocation_size = preallocation_size; + } else { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + } +} + +/* + * Method: setTableFactory + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setTableFactory(JNIEnv*, jobject, jlong jhandle, + jlong jtable_factory_handle) { + auto* options = reinterpret_cast(jhandle); + auto* table_factory = + reinterpret_cast(jtable_factory_handle); + options->table_factory.reset(table_factory); +} + +/* + * Method: setSstPartitionerFactory + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setSstPartitionerFactory(JNIEnv*, jobject, + jlong jhandle, + jlong factory_handle) { + auto* options = reinterpret_cast(jhandle); + auto factory = reinterpret_cast< + std::shared_ptr*>( + factory_handle); + options->sst_partitioner_factory = *factory; +} + +/* + * Class: org_rocksdb_Options + * Method: setCompactionThreadLimiter + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setCompactionThreadLimiter( + JNIEnv*, jclass, jlong jhandle, jlong jlimiter_handle) { + auto* options = reinterpret_cast(jhandle); + auto* limiter = reinterpret_cast< + std::shared_ptr*>( + jlimiter_handle); + options->compaction_thread_limiter = *limiter; +} + +/* + * Class: org_rocksdb_Options + * Method: allowMmapReads + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_allowMmapReads(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->allow_mmap_reads; +} + +/* + * Class: org_rocksdb_Options + * Method: setAllowMmapReads + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setAllowMmapReads(JNIEnv*, jobject, jlong jhandle, + jboolean allow_mmap_reads) { + reinterpret_cast(jhandle)->allow_mmap_reads = + static_cast(allow_mmap_reads); +} + +/* + * Class: org_rocksdb_Options + * Method: allowMmapWrites + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_allowMmapWrites(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->allow_mmap_writes; +} + +/* + * Class: org_rocksdb_Options + * Method: setAllowMmapWrites + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setAllowMmapWrites(JNIEnv*, jobject, + jlong jhandle, + jboolean allow_mmap_writes) { + reinterpret_cast(jhandle)->allow_mmap_writes = + static_cast(allow_mmap_writes); +} + +/* + * Class: org_rocksdb_Options + * Method: useDirectReads + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_useDirectReads(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->use_direct_reads; +} + +/* + * Class: org_rocksdb_Options + * Method: setUseDirectReads + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setUseDirectReads(JNIEnv*, jobject, jlong jhandle, + jboolean use_direct_reads) { + reinterpret_cast(jhandle)->use_direct_reads = + static_cast(use_direct_reads); +} + +/* + * Class: org_rocksdb_Options + * Method: useDirectIoForFlushAndCompaction + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_useDirectIoForFlushAndCompaction( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->use_direct_io_for_flush_and_compaction; +} + +/* + * Class: org_rocksdb_Options + * Method: setUseDirectIoForFlushAndCompaction + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setUseDirectIoForFlushAndCompaction( + JNIEnv*, jobject, jlong jhandle, + jboolean use_direct_io_for_flush_and_compaction) { + reinterpret_cast(jhandle) + ->use_direct_io_for_flush_and_compaction = + static_cast(use_direct_io_for_flush_and_compaction); +} + +/* + * Class: org_rocksdb_Options + * Method: setAllowFAllocate + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setAllowFAllocate(JNIEnv*, jobject, jlong jhandle, + jboolean jallow_fallocate) { + reinterpret_cast(jhandle)->allow_fallocate = + static_cast(jallow_fallocate); +} + +/* + * Class: org_rocksdb_Options + * Method: allowFAllocate + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_allowFAllocate(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->allow_fallocate); +} + +/* + * Class: org_rocksdb_Options + * Method: isFdCloseOnExec + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_isFdCloseOnExec(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->is_fd_close_on_exec; +} + +/* + * Class: org_rocksdb_Options + * Method: setIsFdCloseOnExec + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setIsFdCloseOnExec(JNIEnv*, jobject, + jlong jhandle, + jboolean is_fd_close_on_exec) { + reinterpret_cast(jhandle)->is_fd_close_on_exec = + static_cast(is_fd_close_on_exec); +} + +/* + * Class: org_rocksdb_Options + * Method: statsDumpPeriodSec + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_statsDumpPeriodSec(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->stats_dump_period_sec; +} + +/* + * Class: org_rocksdb_Options + * Method: setStatsDumpPeriodSec + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setStatsDumpPeriodSec( + JNIEnv*, jobject, jlong jhandle, jint jstats_dump_period_sec) { + reinterpret_cast(jhandle) + ->stats_dump_period_sec = + static_cast(jstats_dump_period_sec); +} + +/* + * Class: org_rocksdb_Options + * Method: statsPersistPeriodSec + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_statsPersistPeriodSec(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->stats_persist_period_sec; +} + +/* + * Class: org_rocksdb_Options + * Method: setStatsPersistPeriodSec + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setStatsPersistPeriodSec( + JNIEnv*, jobject, jlong jhandle, jint jstats_persist_period_sec) { + reinterpret_cast(jhandle) + ->stats_persist_period_sec = + static_cast(jstats_persist_period_sec); +} + +/* + * Class: org_rocksdb_Options + * Method: statsHistoryBufferSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_statsHistoryBufferSize(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->stats_history_buffer_size; +} + +/* + * Class: org_rocksdb_Options + * Method: setStatsHistoryBufferSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setStatsHistoryBufferSize( + JNIEnv*, jobject, jlong jhandle, jlong jstats_history_buffer_size) { + reinterpret_cast(jhandle) + ->stats_history_buffer_size = + static_cast(jstats_history_buffer_size); +} + +/* + * Class: org_rocksdb_Options + * Method: adviseRandomOnOpen + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_adviseRandomOnOpen(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->advise_random_on_open; +} + +/* + * Class: org_rocksdb_Options + * Method: setAdviseRandomOnOpen + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setAdviseRandomOnOpen( + JNIEnv*, jobject, jlong jhandle, jboolean advise_random_on_open) { + reinterpret_cast(jhandle) + ->advise_random_on_open = static_cast(advise_random_on_open); +} + +/* + * Class: org_rocksdb_Options + * Method: setDbWriteBufferSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setDbWriteBufferSize( + JNIEnv*, jobject, jlong jhandle, jlong jdb_write_buffer_size) { + auto* opt = reinterpret_cast(jhandle); + opt->db_write_buffer_size = static_cast(jdb_write_buffer_size); +} + +/* + * Class: org_rocksdb_Options + * Method: dbWriteBufferSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_dbWriteBufferSize(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->db_write_buffer_size); +} + +/* + * Class: org_rocksdb_Options + * Method: setAccessHintOnCompactionStart + * Signature: (JB)V + */ +void Java_org_rocksdb_Options_setAccessHintOnCompactionStart( + JNIEnv*, jobject, jlong jhandle, jbyte jaccess_hint_value) { + auto* opt = reinterpret_cast(jhandle); + opt->access_hint_on_compaction_start = + ROCKSDB_NAMESPACE::AccessHintJni::toCppAccessHint(jaccess_hint_value); +} + +/* + * Class: org_rocksdb_Options + * Method: accessHintOnCompactionStart + * Signature: (J)B + */ +jbyte Java_org_rocksdb_Options_accessHintOnCompactionStart(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::AccessHintJni::toJavaAccessHint( + opt->access_hint_on_compaction_start); +} + +/* + * Class: org_rocksdb_Options + * Method: setCompactionReadaheadSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setCompactionReadaheadSize( + JNIEnv*, jobject, jlong jhandle, jlong jcompaction_readahead_size) { + auto* opt = reinterpret_cast(jhandle); + opt->compaction_readahead_size = + static_cast(jcompaction_readahead_size); +} + +/* + * Class: org_rocksdb_Options + * Method: compactionReadaheadSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_compactionReadaheadSize(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->compaction_readahead_size); +} + +/* + * Class: org_rocksdb_Options + * Method: setRandomAccessMaxBufferSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setRandomAccessMaxBufferSize( + JNIEnv*, jobject, jlong jhandle, jlong jrandom_access_max_buffer_size) { + auto* opt = reinterpret_cast(jhandle); + opt->random_access_max_buffer_size = + static_cast(jrandom_access_max_buffer_size); +} + +/* + * Class: org_rocksdb_Options + * Method: randomAccessMaxBufferSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_randomAccessMaxBufferSize(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->random_access_max_buffer_size); +} + +/* + * Class: org_rocksdb_Options + * Method: setWritableFileMaxBufferSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setWritableFileMaxBufferSize( + JNIEnv*, jobject, jlong jhandle, jlong jwritable_file_max_buffer_size) { + auto* opt = reinterpret_cast(jhandle); + opt->writable_file_max_buffer_size = + static_cast(jwritable_file_max_buffer_size); +} + +/* + * Class: org_rocksdb_Options + * Method: writableFileMaxBufferSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_writableFileMaxBufferSize(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->writable_file_max_buffer_size); +} + +/* + * Class: org_rocksdb_Options + * Method: useAdaptiveMutex + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_useAdaptiveMutex(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->use_adaptive_mutex; +} + +/* + * Class: org_rocksdb_Options + * Method: setUseAdaptiveMutex + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setUseAdaptiveMutex(JNIEnv*, jobject, + jlong jhandle, + jboolean use_adaptive_mutex) { + reinterpret_cast(jhandle)->use_adaptive_mutex = + static_cast(use_adaptive_mutex); +} + +/* + * Class: org_rocksdb_Options + * Method: bytesPerSync + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_bytesPerSync(JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle)->bytes_per_sync; +} + +/* + * Class: org_rocksdb_Options + * Method: setBytesPerSync + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setBytesPerSync(JNIEnv*, jobject, jlong jhandle, + jlong bytes_per_sync) { + reinterpret_cast(jhandle)->bytes_per_sync = + static_cast(bytes_per_sync); +} + +/* + * Class: org_rocksdb_Options + * Method: setWalBytesPerSync + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setWalBytesPerSync(JNIEnv*, jobject, + jlong jhandle, + jlong jwal_bytes_per_sync) { + reinterpret_cast(jhandle)->wal_bytes_per_sync = + static_cast(jwal_bytes_per_sync); +} + +/* + * Class: org_rocksdb_Options + * Method: walBytesPerSync + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_walBytesPerSync(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->wal_bytes_per_sync); +} + +/* + * Class: org_rocksdb_Options + * Method: setStrictBytesPerSync + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setStrictBytesPerSync( + JNIEnv*, jobject, jlong jhandle, jboolean jstrict_bytes_per_sync) { + reinterpret_cast(jhandle) + ->strict_bytes_per_sync = jstrict_bytes_per_sync == JNI_TRUE; +} + +/* + * Class: org_rocksdb_Options + * Method: strictBytesPerSync + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_strictBytesPerSync(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->strict_bytes_per_sync); +} + +// Note: the RocksJava API currently only supports EventListeners implemented in +// Java. It could be extended in future to also support adding/removing +// EventListeners implemented in C++. +static void rocksdb_set_event_listeners_helper( + JNIEnv* env, jlongArray jlistener_array, + std::vector>& + listener_sptr_vec) { + jlong* ptr_jlistener_array = + env->GetLongArrayElements(jlistener_array, nullptr); + if (ptr_jlistener_array == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + const jsize array_size = env->GetArrayLength(jlistener_array); + listener_sptr_vec.clear(); + for (jsize i = 0; i < array_size; ++i) { + const auto& listener_sptr = + *reinterpret_cast*>( + ptr_jlistener_array[i]); + listener_sptr_vec.push_back(listener_sptr); + } +} + +/* + * Class: org_rocksdb_Options + * Method: setEventListeners + * Signature: (J[J)V + */ +void Java_org_rocksdb_Options_setEventListeners(JNIEnv* env, jclass, + jlong jhandle, + jlongArray jlistener_array) { + auto* opt = reinterpret_cast(jhandle); + rocksdb_set_event_listeners_helper(env, jlistener_array, opt->listeners); +} + +// Note: the RocksJava API currently only supports EventListeners implemented in +// Java. It could be extended in future to also support adding/removing +// EventListeners implemented in C++. +static jobjectArray rocksdb_get_event_listeners_helper( + JNIEnv* env, + const std::vector>& + listener_sptr_vec) { + jsize sz = static_cast(listener_sptr_vec.size()); + jclass jlistener_clazz = + ROCKSDB_NAMESPACE::AbstractEventListenerJni::getJClass(env); + jobjectArray jlisteners = env->NewObjectArray(sz, jlistener_clazz, nullptr); + if (jlisteners == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + for (jsize i = 0; i < sz; ++i) { + const auto* jni_cb = + static_cast( + listener_sptr_vec[i].get()); + env->SetObjectArrayElement(jlisteners, i, jni_cb->GetJavaObject()); + } + return jlisteners; +} + +/* + * Class: org_rocksdb_Options + * Method: eventListeners + * Signature: (J)[Lorg/rocksdb/AbstractEventListener; + */ +jobjectArray Java_org_rocksdb_Options_eventListeners(JNIEnv* env, jclass, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return rocksdb_get_event_listeners_helper(env, opt->listeners); +} + +/* + * Class: org_rocksdb_Options + * Method: setEnableThreadTracking + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setEnableThreadTracking( + JNIEnv*, jobject, jlong jhandle, jboolean jenable_thread_tracking) { + auto* opt = reinterpret_cast(jhandle); + opt->enable_thread_tracking = static_cast(jenable_thread_tracking); +} + +/* + * Class: org_rocksdb_Options + * Method: enableThreadTracking + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_enableThreadTracking(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->enable_thread_tracking); +} + +/* + * Class: org_rocksdb_Options + * Method: setDelayedWriteRate + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setDelayedWriteRate(JNIEnv*, jobject, + jlong jhandle, + jlong jdelayed_write_rate) { + auto* opt = reinterpret_cast(jhandle); + opt->delayed_write_rate = static_cast(jdelayed_write_rate); +} + +/* + * Class: org_rocksdb_Options + * Method: delayedWriteRate + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_delayedWriteRate(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->delayed_write_rate); +} + +/* + * Class: org_rocksdb_Options + * Method: setEnablePipelinedWrite + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setEnablePipelinedWrite( + JNIEnv*, jobject, jlong jhandle, jboolean jenable_pipelined_write) { + auto* opt = reinterpret_cast(jhandle); + opt->enable_pipelined_write = jenable_pipelined_write == JNI_TRUE; +} + +/* + * Class: org_rocksdb_Options + * Method: enablePipelinedWrite + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_enablePipelinedWrite(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->enable_pipelined_write); +} + +/* + * Class: org_rocksdb_Options + * Method: setUnorderedWrite + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setUnorderedWrite(JNIEnv*, jobject, jlong jhandle, + jboolean unordered_write) { + reinterpret_cast(jhandle)->unordered_write = + static_cast(unordered_write); +} + +/* + * Class: org_rocksdb_Options + * Method: unorderedWrite + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_unorderedWrite(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->unordered_write; +} + +/* + * Class: org_rocksdb_Options + * Method: setAllowConcurrentMemtableWrite + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setAllowConcurrentMemtableWrite(JNIEnv*, jobject, + jlong jhandle, + jboolean allow) { + reinterpret_cast(jhandle) + ->allow_concurrent_memtable_write = static_cast(allow); +} + +/* + * Class: org_rocksdb_Options + * Method: allowConcurrentMemtableWrite + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_allowConcurrentMemtableWrite(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->allow_concurrent_memtable_write; +} + +/* + * Class: org_rocksdb_Options + * Method: setEnableWriteThreadAdaptiveYield + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setEnableWriteThreadAdaptiveYield( + JNIEnv*, jobject, jlong jhandle, jboolean yield) { + reinterpret_cast(jhandle) + ->enable_write_thread_adaptive_yield = static_cast(yield); +} + +/* + * Class: org_rocksdb_Options + * Method: enableWriteThreadAdaptiveYield + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_enableWriteThreadAdaptiveYield( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->enable_write_thread_adaptive_yield; +} + +/* + * Class: org_rocksdb_Options + * Method: setWriteThreadMaxYieldUsec + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setWriteThreadMaxYieldUsec(JNIEnv*, jobject, + jlong jhandle, + jlong max) { + reinterpret_cast(jhandle) + ->write_thread_max_yield_usec = static_cast(max); +} + +/* + * Class: org_rocksdb_Options + * Method: writeThreadMaxYieldUsec + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_writeThreadMaxYieldUsec(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->write_thread_max_yield_usec; +} + +/* + * Class: org_rocksdb_Options + * Method: setWriteThreadSlowYieldUsec + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setWriteThreadSlowYieldUsec(JNIEnv*, jobject, + jlong jhandle, + jlong slow) { + reinterpret_cast(jhandle) + ->write_thread_slow_yield_usec = static_cast(slow); +} + +/* + * Class: org_rocksdb_Options + * Method: writeThreadSlowYieldUsec + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_writeThreadSlowYieldUsec(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->write_thread_slow_yield_usec; +} + +/* + * Class: org_rocksdb_Options + * Method: setSkipStatsUpdateOnDbOpen + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setSkipStatsUpdateOnDbOpen( + JNIEnv*, jobject, jlong jhandle, jboolean jskip_stats_update_on_db_open) { + auto* opt = reinterpret_cast(jhandle); + opt->skip_stats_update_on_db_open = + static_cast(jskip_stats_update_on_db_open); +} + +/* + * Class: org_rocksdb_Options + * Method: skipStatsUpdateOnDbOpen + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_skipStatsUpdateOnDbOpen(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->skip_stats_update_on_db_open); +} + +/* + * Class: org_rocksdb_Options + * Method: setSkipCheckingSstFileSizesOnDbOpen + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setSkipCheckingSstFileSizesOnDbOpen( + JNIEnv*, jclass, jlong jhandle, + jboolean jskip_checking_sst_file_sizes_on_db_open) { + auto* opt = reinterpret_cast(jhandle); + opt->skip_checking_sst_file_sizes_on_db_open = + static_cast(jskip_checking_sst_file_sizes_on_db_open); +} + +/* + * Class: org_rocksdb_Options + * Method: skipCheckingSstFileSizesOnDbOpen + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_skipCheckingSstFileSizesOnDbOpen( + JNIEnv*, jclass, jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->skip_checking_sst_file_sizes_on_db_open); +} + +/* + * Class: org_rocksdb_Options + * Method: setWalRecoveryMode + * Signature: (JB)V + */ +void Java_org_rocksdb_Options_setWalRecoveryMode( + JNIEnv*, jobject, jlong jhandle, jbyte jwal_recovery_mode_value) { + auto* opt = reinterpret_cast(jhandle); + opt->wal_recovery_mode = + ROCKSDB_NAMESPACE::WALRecoveryModeJni::toCppWALRecoveryMode( + jwal_recovery_mode_value); +} + +/* + * Class: org_rocksdb_Options + * Method: walRecoveryMode + * Signature: (J)B + */ +jbyte Java_org_rocksdb_Options_walRecoveryMode(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::WALRecoveryModeJni::toJavaWALRecoveryMode( + opt->wal_recovery_mode); +} + +/* + * Class: org_rocksdb_Options + * Method: setAllow2pc + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setAllow2pc(JNIEnv*, jobject, jlong jhandle, + jboolean jallow_2pc) { + auto* opt = reinterpret_cast(jhandle); + opt->allow_2pc = static_cast(jallow_2pc); +} + +/* + * Class: org_rocksdb_Options + * Method: allow2pc + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_allow2pc(JNIEnv*, jobject, jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->allow_2pc); +} + +/* + * Class: org_rocksdb_Options + * Method: setRowCache + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setRowCache(JNIEnv*, jobject, jlong jhandle, + jlong jrow_cache_handle) { + auto* opt = reinterpret_cast(jhandle); + auto* row_cache = + reinterpret_cast*>( + jrow_cache_handle); + opt->row_cache = *row_cache; +} + +/* + * Class: org_rocksdb_Options + * Method: setWalFilter + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setWalFilter(JNIEnv*, jobject, jlong jhandle, + jlong jwal_filter_handle) { + auto* opt = reinterpret_cast(jhandle); + auto* wal_filter = reinterpret_cast( + jwal_filter_handle); + opt->wal_filter = wal_filter; +} + +/* + * Class: org_rocksdb_Options + * Method: setFailIfOptionsFileError + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setFailIfOptionsFileError( + JNIEnv*, jobject, jlong jhandle, jboolean jfail_if_options_file_error) { + auto* opt = reinterpret_cast(jhandle); + opt->fail_if_options_file_error = + static_cast(jfail_if_options_file_error); +} + +/* + * Class: org_rocksdb_Options + * Method: failIfOptionsFileError + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_failIfOptionsFileError(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->fail_if_options_file_error); +} + +/* + * Class: org_rocksdb_Options + * Method: setDumpMallocStats + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setDumpMallocStats(JNIEnv*, jobject, + jlong jhandle, + jboolean jdump_malloc_stats) { + auto* opt = reinterpret_cast(jhandle); + opt->dump_malloc_stats = static_cast(jdump_malloc_stats); +} + +/* + * Class: org_rocksdb_Options + * Method: dumpMallocStats + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_dumpMallocStats(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->dump_malloc_stats); +} + +/* + * Class: org_rocksdb_Options + * Method: setAvoidFlushDuringRecovery + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setAvoidFlushDuringRecovery( + JNIEnv*, jobject, jlong jhandle, jboolean javoid_flush_during_recovery) { + auto* opt = reinterpret_cast(jhandle); + opt->avoid_flush_during_recovery = + static_cast(javoid_flush_during_recovery); +} + +/* + * Class: org_rocksdb_Options + * Method: avoidFlushDuringRecovery + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_avoidFlushDuringRecovery(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->avoid_flush_during_recovery); +} + +/* + * Class: org_rocksdb_Options + * Method: setAvoidUnnecessaryBlockingIO + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setAvoidUnnecessaryBlockingIO( + JNIEnv*, jclass, jlong jhandle, jboolean avoid_blocking_io) { + auto* opt = reinterpret_cast(jhandle); + opt->avoid_unnecessary_blocking_io = static_cast(avoid_blocking_io); +} + +/* + * Class: org_rocksdb_Options + * Method: avoidUnnecessaryBlockingIO + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_avoidUnnecessaryBlockingIO(JNIEnv*, jclass, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->avoid_unnecessary_blocking_io); +} + +/* + * Class: org_rocksdb_Options + * Method: setPersistStatsToDisk + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setPersistStatsToDisk( + JNIEnv*, jclass, jlong jhandle, jboolean persist_stats_to_disk) { + auto* opt = reinterpret_cast(jhandle); + opt->persist_stats_to_disk = static_cast(persist_stats_to_disk); +} + +/* + * Class: org_rocksdb_Options + * Method: persistStatsToDisk + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_persistStatsToDisk(JNIEnv*, jclass, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->persist_stats_to_disk); +} + +/* + * Class: org_rocksdb_Options + * Method: setWriteDbidToManifest + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setWriteDbidToManifest( + JNIEnv*, jclass, jlong jhandle, jboolean jwrite_dbid_to_manifest) { + auto* opt = reinterpret_cast(jhandle); + opt->write_dbid_to_manifest = static_cast(jwrite_dbid_to_manifest); +} + +/* + * Class: org_rocksdb_Options + * Method: writeDbidToManifest + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_writeDbidToManifest(JNIEnv*, jclass, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->write_dbid_to_manifest); +} + +/* + * Class: org_rocksdb_Options + * Method: setLogReadaheadSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setLogReadaheadSize(JNIEnv*, jclass, + jlong jhandle, + jlong jlog_readahead_size) { + auto* opt = reinterpret_cast(jhandle); + opt->log_readahead_size = static_cast(jlog_readahead_size); +} + +/* + * Class: org_rocksdb_Options + * Method: logReasaheadSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_logReadaheadSize(JNIEnv*, jclass, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->log_readahead_size); +} + +/* + * Class: org_rocksdb_Options + * Method: setBestEffortsRecovery + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setBestEffortsRecovery( + JNIEnv*, jclass, jlong jhandle, jboolean jbest_efforts_recovery) { + auto* opt = reinterpret_cast(jhandle); + opt->best_efforts_recovery = static_cast(jbest_efforts_recovery); +} + +/* + * Class: org_rocksdb_Options + * Method: bestEffortsRecovery + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_bestEffortsRecovery(JNIEnv*, jclass, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->best_efforts_recovery); +} + +/* + * Class: org_rocksdb_Options + * Method: setMaxBgErrorResumeCount + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setMaxBgErrorResumeCount( + JNIEnv*, jclass, jlong jhandle, jint jmax_bgerror_resume_count) { + auto* opt = reinterpret_cast(jhandle); + opt->max_bgerror_resume_count = static_cast(jmax_bgerror_resume_count); +} + +/* + * Class: org_rocksdb_Options + * Method: maxBgerrorResumeCount + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_maxBgerrorResumeCount(JNIEnv*, jclass, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->max_bgerror_resume_count); +} + +/* + * Class: org_rocksdb_Options + * Method: setBgerrorResumeRetryInterval + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setBgerrorResumeRetryInterval( + JNIEnv*, jclass, jlong jhandle, jlong jbgerror_resume_retry_interval) { + auto* opt = reinterpret_cast(jhandle); + opt->bgerror_resume_retry_interval = + static_cast(jbgerror_resume_retry_interval); +} + +/* + * Class: org_rocksdb_Options + * Method: bgerrorResumeRetryInterval + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_bgerrorResumeRetryInterval(JNIEnv*, jclass, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->bgerror_resume_retry_interval); +} + +/* + * Class: org_rocksdb_Options + * Method: setAvoidFlushDuringShutdown + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setAvoidFlushDuringShutdown( + JNIEnv*, jobject, jlong jhandle, jboolean javoid_flush_during_shutdown) { + auto* opt = reinterpret_cast(jhandle); + opt->avoid_flush_during_shutdown = + static_cast(javoid_flush_during_shutdown); +} + +/* + * Class: org_rocksdb_Options + * Method: avoidFlushDuringShutdown + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_avoidFlushDuringShutdown(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->avoid_flush_during_shutdown); +} + +/* + * Class: org_rocksdb_Options + * Method: setAllowIngestBehind + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setAllowIngestBehind( + JNIEnv*, jobject, jlong jhandle, jboolean jallow_ingest_behind) { + auto* opt = reinterpret_cast(jhandle); + opt->allow_ingest_behind = jallow_ingest_behind == JNI_TRUE; +} + +/* + * Class: org_rocksdb_Options + * Method: allowIngestBehind + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_allowIngestBehind(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->allow_ingest_behind); +} + +/* + * Class: org_rocksdb_Options + * Method: setTwoWriteQueues + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setTwoWriteQueues(JNIEnv*, jobject, jlong jhandle, + jboolean jtwo_write_queues) { + auto* opt = reinterpret_cast(jhandle); + opt->two_write_queues = jtwo_write_queues == JNI_TRUE; +} + +/* + * Class: org_rocksdb_Options + * Method: twoWriteQueues + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_twoWriteQueues(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->two_write_queues); +} + +/* + * Class: org_rocksdb_Options + * Method: setManualWalFlush + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setManualWalFlush(JNIEnv*, jobject, jlong jhandle, + jboolean jmanual_wal_flush) { + auto* opt = reinterpret_cast(jhandle); + opt->manual_wal_flush = jmanual_wal_flush == JNI_TRUE; +} + +/* + * Class: org_rocksdb_Options + * Method: manualWalFlush + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_manualWalFlush(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->manual_wal_flush); +} + +/* + * Class: org_rocksdb_Options + * Method: setAtomicFlush + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setAtomicFlush(JNIEnv*, jobject, jlong jhandle, + jboolean jatomic_flush) { + auto* opt = reinterpret_cast(jhandle); + opt->atomic_flush = jatomic_flush == JNI_TRUE; +} + +/* + * Class: org_rocksdb_Options + * Method: atomicFlush + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_atomicFlush(JNIEnv*, jobject, jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->atomic_flush); +} + +/* + * Method: tableFactoryName + * Signature: (J)Ljava/lang/String + */ +jstring Java_org_rocksdb_Options_tableFactoryName(JNIEnv* env, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + ROCKSDB_NAMESPACE::TableFactory* tf = opt->table_factory.get(); + + // Should never be nullptr. + // Default memtable factory is SkipListFactory + assert(tf); + + return env->NewStringUTF(tf->Name()); +} + +/* + * Class: org_rocksdb_Options + * Method: minWriteBufferNumberToMerge + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_minWriteBufferNumberToMerge(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->min_write_buffer_number_to_merge; +} + +/* + * Class: org_rocksdb_Options + * Method: setMinWriteBufferNumberToMerge + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setMinWriteBufferNumberToMerge( + JNIEnv*, jobject, jlong jhandle, jint jmin_write_buffer_number_to_merge) { + reinterpret_cast(jhandle) + ->min_write_buffer_number_to_merge = + static_cast(jmin_write_buffer_number_to_merge); +} +/* + * Class: org_rocksdb_Options + * Method: maxWriteBufferNumberToMaintain + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_maxWriteBufferNumberToMaintain(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_write_buffer_number_to_maintain; +} + +/* + * Class: org_rocksdb_Options + * Method: setMaxWriteBufferNumberToMaintain + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setMaxWriteBufferNumberToMaintain( + JNIEnv*, jobject, jlong jhandle, + jint jmax_write_buffer_number_to_maintain) { + reinterpret_cast(jhandle) + ->max_write_buffer_number_to_maintain = + static_cast(jmax_write_buffer_number_to_maintain); +} + +/* + * Class: org_rocksdb_Options + * Method: setCompressionType + * Signature: (JB)V + */ +void Java_org_rocksdb_Options_setCompressionType( + JNIEnv*, jobject, jlong jhandle, jbyte jcompression_type_value) { + auto* opts = reinterpret_cast(jhandle); + opts->compression = + ROCKSDB_NAMESPACE::CompressionTypeJni::toCppCompressionType( + jcompression_type_value); +} + +/* + * Class: org_rocksdb_Options + * Method: compressionType + * Signature: (J)B + */ +jbyte Java_org_rocksdb_Options_compressionType(JNIEnv*, jobject, + jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::CompressionTypeJni::toJavaCompressionType( + opts->compression); +} + +/** + * Helper method to convert a Java byte array of compression levels + * to a C++ vector of ROCKSDB_NAMESPACE::CompressionType + * + * @param env A pointer to the Java environment + * @param jcompression_levels A reference to a java byte array + * where each byte indicates a compression level + * + * @return A std::unique_ptr to the vector, or std::unique_ptr(nullptr) if a JNI + * exception occurs + */ +std::unique_ptr> +rocksdb_compression_vector_helper(JNIEnv* env, jbyteArray jcompression_levels) { + jsize len = env->GetArrayLength(jcompression_levels); + jbyte* jcompression_level = + env->GetByteArrayElements(jcompression_levels, nullptr); + if (jcompression_level == nullptr) { + // exception thrown: OutOfMemoryError + return std::unique_ptr>(); + } + + auto* compression_levels = + new std::vector(); + std::unique_ptr> + uptr_compression_levels(compression_levels); + + for (jsize i = 0; i < len; i++) { + jbyte jcl = jcompression_level[i]; + compression_levels->push_back( + static_cast(jcl)); + } + + env->ReleaseByteArrayElements(jcompression_levels, jcompression_level, + JNI_ABORT); + + return uptr_compression_levels; +} + +/** + * Helper method to convert a C++ vector of ROCKSDB_NAMESPACE::CompressionType + * to a Java byte array of compression levels + * + * @param env A pointer to the Java environment + * @param jcompression_levels A reference to a java byte array + * where each byte indicates a compression level + * + * @return A jbytearray or nullptr if an exception occurs + */ +jbyteArray rocksdb_compression_list_helper( + JNIEnv* env, + std::vector compression_levels) { + const size_t len = compression_levels.size(); + jbyte* jbuf = new jbyte[len]; + + for (size_t i = 0; i < len; i++) { + jbuf[i] = compression_levels[i]; + } + + // insert in java array + jbyteArray jcompression_levels = env->NewByteArray(static_cast(len)); + if (jcompression_levels == nullptr) { + // exception thrown: OutOfMemoryError + delete[] jbuf; + return nullptr; + } + env->SetByteArrayRegion(jcompression_levels, 0, static_cast(len), + jbuf); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jcompression_levels); + delete[] jbuf; + return nullptr; + } + + delete[] jbuf; + + return jcompression_levels; +} + +/* + * Class: org_rocksdb_Options + * Method: setCompressionPerLevel + * Signature: (J[B)V + */ +void Java_org_rocksdb_Options_setCompressionPerLevel( + JNIEnv* env, jobject, jlong jhandle, jbyteArray jcompressionLevels) { + auto uptr_compression_levels = + rocksdb_compression_vector_helper(env, jcompressionLevels); + if (!uptr_compression_levels) { + // exception occurred + return; + } + auto* options = reinterpret_cast(jhandle); + options->compression_per_level = *(uptr_compression_levels.get()); +} + +/* + * Class: org_rocksdb_Options + * Method: compressionPerLevel + * Signature: (J)[B + */ +jbyteArray Java_org_rocksdb_Options_compressionPerLevel(JNIEnv* env, jobject, + jlong jhandle) { + auto* options = reinterpret_cast(jhandle); + return rocksdb_compression_list_helper(env, options->compression_per_level); +} + +/* + * Class: org_rocksdb_Options + * Method: setBottommostCompressionType + * Signature: (JB)V + */ +void Java_org_rocksdb_Options_setBottommostCompressionType( + JNIEnv*, jobject, jlong jhandle, jbyte jcompression_type_value) { + auto* options = reinterpret_cast(jhandle); + options->bottommost_compression = + ROCKSDB_NAMESPACE::CompressionTypeJni::toCppCompressionType( + jcompression_type_value); +} + +/* + * Class: org_rocksdb_Options + * Method: bottommostCompressionType + * Signature: (J)B + */ +jbyte Java_org_rocksdb_Options_bottommostCompressionType(JNIEnv*, jobject, + jlong jhandle) { + auto* options = reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::CompressionTypeJni::toJavaCompressionType( + options->bottommost_compression); +} + +/* + * Class: org_rocksdb_Options + * Method: setBottommostCompressionOptions + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setBottommostCompressionOptions( + JNIEnv*, jobject, jlong jhandle, + jlong jbottommost_compression_options_handle) { + auto* options = reinterpret_cast(jhandle); + auto* bottommost_compression_options = + reinterpret_cast( + jbottommost_compression_options_handle); + options->bottommost_compression_opts = *bottommost_compression_options; +} + +/* + * Class: org_rocksdb_Options + * Method: setCompressionOptions + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setCompressionOptions( + JNIEnv*, jobject, jlong jhandle, jlong jcompression_options_handle) { + auto* options = reinterpret_cast(jhandle); + auto* compression_options = + reinterpret_cast( + jcompression_options_handle); + options->compression_opts = *compression_options; +} + +/* + * Class: org_rocksdb_Options + * Method: setCompactionStyle + * Signature: (JB)V + */ +void Java_org_rocksdb_Options_setCompactionStyle(JNIEnv*, jobject, + jlong jhandle, + jbyte jcompaction_style) { + auto* options = reinterpret_cast(jhandle); + options->compaction_style = + ROCKSDB_NAMESPACE::CompactionStyleJni::toCppCompactionStyle( + jcompaction_style); +} + +/* + * Class: org_rocksdb_Options + * Method: compactionStyle + * Signature: (J)B + */ +jbyte Java_org_rocksdb_Options_compactionStyle(JNIEnv*, jobject, + jlong jhandle) { + auto* options = reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::CompactionStyleJni::toJavaCompactionStyle( + options->compaction_style); +} + +/* + * Class: org_rocksdb_Options + * Method: setMaxTableFilesSizeFIFO + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setMaxTableFilesSizeFIFO( + JNIEnv*, jobject, jlong jhandle, jlong jmax_table_files_size) { + reinterpret_cast(jhandle) + ->compaction_options_fifo.max_table_files_size = + static_cast(jmax_table_files_size); +} + +/* + * Class: org_rocksdb_Options + * Method: maxTableFilesSizeFIFO + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_maxTableFilesSizeFIFO(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->compaction_options_fifo.max_table_files_size; +} + +/* + * Class: org_rocksdb_Options + * Method: numLevels + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_numLevels(JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle)->num_levels; +} + +/* + * Class: org_rocksdb_Options + * Method: setNumLevels + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setNumLevels(JNIEnv*, jobject, jlong jhandle, + jint jnum_levels) { + reinterpret_cast(jhandle)->num_levels = + static_cast(jnum_levels); +} + +/* + * Class: org_rocksdb_Options + * Method: levelZeroFileNumCompactionTrigger + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_levelZeroFileNumCompactionTrigger(JNIEnv*, + jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->level0_file_num_compaction_trigger; +} + +/* + * Class: org_rocksdb_Options + * Method: setLevelZeroFileNumCompactionTrigger + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setLevelZeroFileNumCompactionTrigger( + JNIEnv*, jobject, jlong jhandle, jint jlevel0_file_num_compaction_trigger) { + reinterpret_cast(jhandle) + ->level0_file_num_compaction_trigger = + static_cast(jlevel0_file_num_compaction_trigger); +} + +/* + * Class: org_rocksdb_Options + * Method: levelZeroSlowdownWritesTrigger + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_levelZeroSlowdownWritesTrigger(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->level0_slowdown_writes_trigger; +} + +/* + * Class: org_rocksdb_Options + * Method: setLevelSlowdownWritesTrigger + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setLevelZeroSlowdownWritesTrigger( + JNIEnv*, jobject, jlong jhandle, jint jlevel0_slowdown_writes_trigger) { + reinterpret_cast(jhandle) + ->level0_slowdown_writes_trigger = + static_cast(jlevel0_slowdown_writes_trigger); +} + +/* + * Class: org_rocksdb_Options + * Method: levelZeroStopWritesTrigger + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_levelZeroStopWritesTrigger(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->level0_stop_writes_trigger; +} + +/* + * Class: org_rocksdb_Options + * Method: setLevelStopWritesTrigger + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setLevelZeroStopWritesTrigger( + JNIEnv*, jobject, jlong jhandle, jint jlevel0_stop_writes_trigger) { + reinterpret_cast(jhandle) + ->level0_stop_writes_trigger = + static_cast(jlevel0_stop_writes_trigger); +} + +/* + * Class: org_rocksdb_Options + * Method: targetFileSizeBase + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_targetFileSizeBase(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->target_file_size_base; +} + +/* + * Class: org_rocksdb_Options + * Method: setTargetFileSizeBase + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setTargetFileSizeBase( + JNIEnv*, jobject, jlong jhandle, jlong jtarget_file_size_base) { + reinterpret_cast(jhandle) + ->target_file_size_base = static_cast(jtarget_file_size_base); +} + +/* + * Class: org_rocksdb_Options + * Method: targetFileSizeMultiplier + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_targetFileSizeMultiplier(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->target_file_size_multiplier; +} + +/* + * Class: org_rocksdb_Options + * Method: setTargetFileSizeMultiplier + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setTargetFileSizeMultiplier( + JNIEnv*, jobject, jlong jhandle, jint jtarget_file_size_multiplier) { + reinterpret_cast(jhandle) + ->target_file_size_multiplier = + static_cast(jtarget_file_size_multiplier); +} + +/* + * Class: org_rocksdb_Options + * Method: maxBytesForLevelBase + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_maxBytesForLevelBase(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_bytes_for_level_base; +} + +/* + * Class: org_rocksdb_Options + * Method: setMaxBytesForLevelBase + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setMaxBytesForLevelBase( + JNIEnv*, jobject, jlong jhandle, jlong jmax_bytes_for_level_base) { + reinterpret_cast(jhandle) + ->max_bytes_for_level_base = + static_cast(jmax_bytes_for_level_base); +} + +/* + * Class: org_rocksdb_Options + * Method: levelCompactionDynamicLevelBytes + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_levelCompactionDynamicLevelBytes( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->level_compaction_dynamic_level_bytes; +} + +/* + * Class: org_rocksdb_Options + * Method: setLevelCompactionDynamicLevelBytes + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setLevelCompactionDynamicLevelBytes( + JNIEnv*, jobject, jlong jhandle, jboolean jenable_dynamic_level_bytes) { + reinterpret_cast(jhandle) + ->level_compaction_dynamic_level_bytes = (jenable_dynamic_level_bytes); +} + +/* + * Class: org_rocksdb_Options + * Method: maxBytesForLevelMultiplier + * Signature: (J)D + */ +jdouble Java_org_rocksdb_Options_maxBytesForLevelMultiplier(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_bytes_for_level_multiplier; +} + +/* + * Class: org_rocksdb_Options + * Method: setMaxBytesForLevelMultiplier + * Signature: (JD)V + */ +void Java_org_rocksdb_Options_setMaxBytesForLevelMultiplier( + JNIEnv*, jobject, jlong jhandle, jdouble jmax_bytes_for_level_multiplier) { + reinterpret_cast(jhandle) + ->max_bytes_for_level_multiplier = + static_cast(jmax_bytes_for_level_multiplier); +} + +/* + * Class: org_rocksdb_Options + * Method: maxCompactionBytes + * Signature: (J)I + */ +jlong Java_org_rocksdb_Options_maxCompactionBytes(JNIEnv*, jobject, + jlong jhandle) { + return static_cast( + reinterpret_cast(jhandle) + ->max_compaction_bytes); +} + +/* + * Class: org_rocksdb_Options + * Method: setMaxCompactionBytes + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setMaxCompactionBytes( + JNIEnv*, jobject, jlong jhandle, jlong jmax_compaction_bytes) { + reinterpret_cast(jhandle)->max_compaction_bytes = + static_cast(jmax_compaction_bytes); +} + +/* + * Class: org_rocksdb_Options + * Method: arenaBlockSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_arenaBlockSize(JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->arena_block_size; +} + +/* + * Class: org_rocksdb_Options + * Method: setArenaBlockSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setArenaBlockSize(JNIEnv* env, jobject, + jlong jhandle, + jlong jarena_block_size) { + auto s = + ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t(jarena_block_size); + if (s.ok()) { + reinterpret_cast(jhandle)->arena_block_size = + jarena_block_size; + } else { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_Options + * Method: disableAutoCompactions + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_disableAutoCompactions(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->disable_auto_compactions; +} + +/* + * Class: org_rocksdb_Options + * Method: setDisableAutoCompactions + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setDisableAutoCompactions( + JNIEnv*, jobject, jlong jhandle, jboolean jdisable_auto_compactions) { + reinterpret_cast(jhandle) + ->disable_auto_compactions = static_cast(jdisable_auto_compactions); +} + +/* + * Class: org_rocksdb_Options + * Method: maxSequentialSkipInIterations + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_maxSequentialSkipInIterations(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_sequential_skip_in_iterations; +} + +/* + * Class: org_rocksdb_Options + * Method: setMaxSequentialSkipInIterations + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setMaxSequentialSkipInIterations( + JNIEnv*, jobject, jlong jhandle, jlong jmax_sequential_skip_in_iterations) { + reinterpret_cast(jhandle) + ->max_sequential_skip_in_iterations = + static_cast(jmax_sequential_skip_in_iterations); +} + +/* + * Class: org_rocksdb_Options + * Method: inplaceUpdateSupport + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_inplaceUpdateSupport(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->inplace_update_support; +} + +/* + * Class: org_rocksdb_Options + * Method: setInplaceUpdateSupport + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setInplaceUpdateSupport( + JNIEnv*, jobject, jlong jhandle, jboolean jinplace_update_support) { + reinterpret_cast(jhandle) + ->inplace_update_support = static_cast(jinplace_update_support); +} + +/* + * Class: org_rocksdb_Options + * Method: inplaceUpdateNumLocks + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_inplaceUpdateNumLocks(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->inplace_update_num_locks; +} + +/* + * Class: org_rocksdb_Options + * Method: setInplaceUpdateNumLocks + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setInplaceUpdateNumLocks( + JNIEnv* env, jobject, jlong jhandle, jlong jinplace_update_num_locks) { + auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( + jinplace_update_num_locks); + if (s.ok()) { + reinterpret_cast(jhandle) + ->inplace_update_num_locks = jinplace_update_num_locks; + } else { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_Options + * Method: memtablePrefixBloomSizeRatio + * Signature: (J)I + */ +jdouble Java_org_rocksdb_Options_memtablePrefixBloomSizeRatio(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->memtable_prefix_bloom_size_ratio; +} + +/* + * Class: org_rocksdb_Options + * Method: setMemtablePrefixBloomSizeRatio + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setMemtablePrefixBloomSizeRatio( + JNIEnv*, jobject, jlong jhandle, + jdouble jmemtable_prefix_bloom_size_ratio) { + reinterpret_cast(jhandle) + ->memtable_prefix_bloom_size_ratio = + static_cast(jmemtable_prefix_bloom_size_ratio); +} + +/* + * Class: org_rocksdb_Options + * Method: experimentalMempurgeThreshold + * Signature: (J)I + */ +jdouble Java_org_rocksdb_Options_experimentalMempurgeThreshold(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->experimental_mempurge_threshold; +} + +/* + * Class: org_rocksdb_Options + * Method: setExperimentalMempurgeThreshold + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setExperimentalMempurgeThreshold( + JNIEnv*, jobject, jlong jhandle, jdouble jexperimental_mempurge_threshold) { + reinterpret_cast(jhandle) + ->experimental_mempurge_threshold = + static_cast(jexperimental_mempurge_threshold); +} + +/* + * Class: org_rocksdb_Options + * Method: memtableWholeKeyFiltering + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_memtableWholeKeyFiltering(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->memtable_whole_key_filtering; +} + +/* + * Class: org_rocksdb_Options + * Method: setMemtableWholeKeyFiltering + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setMemtableWholeKeyFiltering( + JNIEnv*, jobject, jlong jhandle, jboolean jmemtable_whole_key_filtering) { + reinterpret_cast(jhandle) + ->memtable_whole_key_filtering = + static_cast(jmemtable_whole_key_filtering); +} + +/* + * Class: org_rocksdb_Options + * Method: bloomLocality + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_bloomLocality(JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle)->bloom_locality; +} + +/* + * Class: org_rocksdb_Options + * Method: setBloomLocality + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setBloomLocality(JNIEnv*, jobject, jlong jhandle, + jint jbloom_locality) { + reinterpret_cast(jhandle)->bloom_locality = + static_cast(jbloom_locality); +} + +/* + * Class: org_rocksdb_Options + * Method: maxSuccessiveMerges + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_maxSuccessiveMerges(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_successive_merges; +} + +/* + * Class: org_rocksdb_Options + * Method: setMaxSuccessiveMerges + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setMaxSuccessiveMerges( + JNIEnv* env, jobject, jlong jhandle, jlong jmax_successive_merges) { + auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( + jmax_successive_merges); + if (s.ok()) { + reinterpret_cast(jhandle) + ->max_successive_merges = jmax_successive_merges; + } else { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_Options + * Method: optimizeFiltersForHits + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_optimizeFiltersForHits(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->optimize_filters_for_hits; +} + +/* + * Class: org_rocksdb_Options + * Method: setOptimizeFiltersForHits + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setOptimizeFiltersForHits( + JNIEnv*, jobject, jlong jhandle, jboolean joptimize_filters_for_hits) { + reinterpret_cast(jhandle) + ->optimize_filters_for_hits = + static_cast(joptimize_filters_for_hits); +} + +/* + * Class: org_rocksdb_Options + * Method: oldDefaults + * Signature: (JII)V + */ +void Java_org_rocksdb_Options_oldDefaults(JNIEnv*, jclass, jlong jhandle, + jint major_version, + jint minor_version) { + reinterpret_cast(jhandle)->OldDefaults( + major_version, minor_version); +} + +/* + * Class: org_rocksdb_Options + * Method: optimizeForSmallDb + * Signature: (J)V + */ +void Java_org_rocksdb_Options_optimizeForSmallDb__J(JNIEnv*, jobject, + jlong jhandle) { + reinterpret_cast(jhandle)->OptimizeForSmallDb(); +} + +/* + * Class: org_rocksdb_Options + * Method: optimizeForSmallDb + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_optimizeForSmallDb__JJ(JNIEnv*, jclass, + jlong jhandle, + jlong cache_handle) { + auto* cache_sptr_ptr = + reinterpret_cast*>( + cache_handle); + auto* options_ptr = reinterpret_cast(jhandle); + auto* cf_options_ptr = + static_cast(options_ptr); + cf_options_ptr->OptimizeForSmallDb(cache_sptr_ptr); +} + +/* + * Class: org_rocksdb_Options + * Method: optimizeForPointLookup + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_optimizeForPointLookup( + JNIEnv*, jobject, jlong jhandle, jlong block_cache_size_mb) { + reinterpret_cast(jhandle) + ->OptimizeForPointLookup(block_cache_size_mb); +} + +/* + * Class: org_rocksdb_Options + * Method: optimizeLevelStyleCompaction + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_optimizeLevelStyleCompaction( + JNIEnv*, jobject, jlong jhandle, jlong memtable_memory_budget) { + reinterpret_cast(jhandle) + ->OptimizeLevelStyleCompaction(memtable_memory_budget); +} + +/* + * Class: org_rocksdb_Options + * Method: optimizeUniversalStyleCompaction + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_optimizeUniversalStyleCompaction( + JNIEnv*, jobject, jlong jhandle, jlong memtable_memory_budget) { + reinterpret_cast(jhandle) + ->OptimizeUniversalStyleCompaction(memtable_memory_budget); +} + +/* + * Class: org_rocksdb_Options + * Method: prepareForBulkLoad + * Signature: (J)V + */ +void Java_org_rocksdb_Options_prepareForBulkLoad(JNIEnv*, jobject, + jlong jhandle) { + reinterpret_cast(jhandle)->PrepareForBulkLoad(); +} + +/* + * Class: org_rocksdb_Options + * Method: memtableHugePageSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_memtableHugePageSize(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->memtable_huge_page_size; +} + +/* + * Class: org_rocksdb_Options + * Method: setMemtableHugePageSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setMemtableHugePageSize( + JNIEnv* env, jobject, jlong jhandle, jlong jmemtable_huge_page_size) { + auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( + jmemtable_huge_page_size); + if (s.ok()) { + reinterpret_cast(jhandle) + ->memtable_huge_page_size = jmemtable_huge_page_size; + } else { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_Options + * Method: softPendingCompactionBytesLimit + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_softPendingCompactionBytesLimit(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->soft_pending_compaction_bytes_limit; +} + +/* + * Class: org_rocksdb_Options + * Method: setSoftPendingCompactionBytesLimit + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setSoftPendingCompactionBytesLimit( + JNIEnv*, jobject, jlong jhandle, + jlong jsoft_pending_compaction_bytes_limit) { + reinterpret_cast(jhandle) + ->soft_pending_compaction_bytes_limit = + static_cast(jsoft_pending_compaction_bytes_limit); +} + +/* + * Class: org_rocksdb_Options + * Method: softHardCompactionBytesLimit + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_hardPendingCompactionBytesLimit(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->hard_pending_compaction_bytes_limit; +} + +/* + * Class: org_rocksdb_Options + * Method: setHardPendingCompactionBytesLimit + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setHardPendingCompactionBytesLimit( + JNIEnv*, jobject, jlong jhandle, + jlong jhard_pending_compaction_bytes_limit) { + reinterpret_cast(jhandle) + ->hard_pending_compaction_bytes_limit = + static_cast(jhard_pending_compaction_bytes_limit); +} + +/* + * Class: org_rocksdb_Options + * Method: level0FileNumCompactionTrigger + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_level0FileNumCompactionTrigger(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->level0_file_num_compaction_trigger; +} + +/* + * Class: org_rocksdb_Options + * Method: setLevel0FileNumCompactionTrigger + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setLevel0FileNumCompactionTrigger( + JNIEnv*, jobject, jlong jhandle, jint jlevel0_file_num_compaction_trigger) { + reinterpret_cast(jhandle) + ->level0_file_num_compaction_trigger = + static_cast(jlevel0_file_num_compaction_trigger); +} + +/* + * Class: org_rocksdb_Options + * Method: level0SlowdownWritesTrigger + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_level0SlowdownWritesTrigger(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->level0_slowdown_writes_trigger; +} + +/* + * Class: org_rocksdb_Options + * Method: setLevel0SlowdownWritesTrigger + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setLevel0SlowdownWritesTrigger( + JNIEnv*, jobject, jlong jhandle, jint jlevel0_slowdown_writes_trigger) { + reinterpret_cast(jhandle) + ->level0_slowdown_writes_trigger = + static_cast(jlevel0_slowdown_writes_trigger); +} + +/* + * Class: org_rocksdb_Options + * Method: level0StopWritesTrigger + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_level0StopWritesTrigger(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->level0_stop_writes_trigger; +} + +/* + * Class: org_rocksdb_Options + * Method: setLevel0StopWritesTrigger + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setLevel0StopWritesTrigger( + JNIEnv*, jobject, jlong jhandle, jint jlevel0_stop_writes_trigger) { + reinterpret_cast(jhandle) + ->level0_stop_writes_trigger = + static_cast(jlevel0_stop_writes_trigger); +} + +/* + * Class: org_rocksdb_Options + * Method: maxBytesForLevelMultiplierAdditional + * Signature: (J)[I + */ +jintArray Java_org_rocksdb_Options_maxBytesForLevelMultiplierAdditional( + JNIEnv* env, jobject, jlong jhandle) { + auto mbflma = reinterpret_cast(jhandle) + ->max_bytes_for_level_multiplier_additional; + + const size_t size = mbflma.size(); + + jint* additionals = new jint[size]; + for (size_t i = 0; i < size; i++) { + additionals[i] = static_cast(mbflma[i]); + } + + jsize jlen = static_cast(size); + jintArray result = env->NewIntArray(jlen); + if (result == nullptr) { + // exception thrown: OutOfMemoryError + delete[] additionals; + return nullptr; + } + + env->SetIntArrayRegion(result, 0, jlen, additionals); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(result); + delete[] additionals; + return nullptr; + } + + delete[] additionals; + + return result; +} + +/* + * Class: org_rocksdb_Options + * Method: setMaxBytesForLevelMultiplierAdditional + * Signature: (J[I)V + */ +void Java_org_rocksdb_Options_setMaxBytesForLevelMultiplierAdditional( + JNIEnv* env, jobject, jlong jhandle, + jintArray jmax_bytes_for_level_multiplier_additional) { + jsize len = env->GetArrayLength(jmax_bytes_for_level_multiplier_additional); + jint* additionals = env->GetIntArrayElements( + jmax_bytes_for_level_multiplier_additional, nullptr); + if (additionals == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + auto* opt = reinterpret_cast(jhandle); + opt->max_bytes_for_level_multiplier_additional.clear(); + for (jsize i = 0; i < len; i++) { + opt->max_bytes_for_level_multiplier_additional.push_back( + static_cast(additionals[i])); + } + + env->ReleaseIntArrayElements(jmax_bytes_for_level_multiplier_additional, + additionals, JNI_ABORT); +} + +/* + * Class: org_rocksdb_Options + * Method: paranoidFileChecks + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_paranoidFileChecks(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->paranoid_file_checks; +} + +/* + * Class: org_rocksdb_Options + * Method: setParanoidFileChecks + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setParanoidFileChecks( + JNIEnv*, jobject, jlong jhandle, jboolean jparanoid_file_checks) { + reinterpret_cast(jhandle)->paranoid_file_checks = + static_cast(jparanoid_file_checks); +} + +/* + * Class: org_rocksdb_Options + * Method: setCompactionPriority + * Signature: (JB)V + */ +void Java_org_rocksdb_Options_setCompactionPriority( + JNIEnv*, jobject, jlong jhandle, jbyte jcompaction_priority_value) { + auto* opts = reinterpret_cast(jhandle); + opts->compaction_pri = + ROCKSDB_NAMESPACE::CompactionPriorityJni::toCppCompactionPriority( + jcompaction_priority_value); +} + +/* + * Class: org_rocksdb_Options + * Method: compactionPriority + * Signature: (J)B + */ +jbyte Java_org_rocksdb_Options_compactionPriority(JNIEnv*, jobject, + jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::CompactionPriorityJni::toJavaCompactionPriority( + opts->compaction_pri); +} + +/* + * Class: org_rocksdb_Options + * Method: setReportBgIoStats + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setReportBgIoStats(JNIEnv*, jobject, + jlong jhandle, + jboolean jreport_bg_io_stats) { + auto* opts = reinterpret_cast(jhandle); + opts->report_bg_io_stats = static_cast(jreport_bg_io_stats); +} + +/* + * Class: org_rocksdb_Options + * Method: reportBgIoStats + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_reportBgIoStats(JNIEnv*, jobject, + jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + return static_cast(opts->report_bg_io_stats); +} + +/* + * Class: org_rocksdb_Options + * Method: setTtl + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setTtl(JNIEnv*, jobject, jlong jhandle, + jlong jttl) { + auto* opts = reinterpret_cast(jhandle); + opts->ttl = static_cast(jttl); +} + +/* + * Class: org_rocksdb_Options + * Method: ttl + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_ttl(JNIEnv*, jobject, jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + return static_cast(opts->ttl); +} + +/* + * Class: org_rocksdb_Options + * Method: setPeriodicCompactionSeconds + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setPeriodicCompactionSeconds( + JNIEnv*, jobject, jlong jhandle, jlong jperiodicCompactionSeconds) { + auto* opts = reinterpret_cast(jhandle); + opts->periodic_compaction_seconds = + static_cast(jperiodicCompactionSeconds); +} + +/* + * Class: org_rocksdb_Options + * Method: periodicCompactionSeconds + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_periodicCompactionSeconds(JNIEnv*, jobject, + jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + return static_cast(opts->periodic_compaction_seconds); +} + +/* + * Class: org_rocksdb_Options + * Method: setCompactionOptionsUniversal + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setCompactionOptionsUniversal( + JNIEnv*, jobject, jlong jhandle, + jlong jcompaction_options_universal_handle) { + auto* opts = reinterpret_cast(jhandle); + auto* opts_uni = + reinterpret_cast( + jcompaction_options_universal_handle); + opts->compaction_options_universal = *opts_uni; +} + +/* + * Class: org_rocksdb_Options + * Method: setCompactionOptionsFIFO + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setCompactionOptionsFIFO( + JNIEnv*, jobject, jlong jhandle, jlong jcompaction_options_fifo_handle) { + auto* opts = reinterpret_cast(jhandle); + auto* opts_fifo = reinterpret_cast( + jcompaction_options_fifo_handle); + opts->compaction_options_fifo = *opts_fifo; +} + +/* + * Class: org_rocksdb_Options + * Method: setForceConsistencyChecks + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setForceConsistencyChecks( + JNIEnv*, jobject, jlong jhandle, jboolean jforce_consistency_checks) { + auto* opts = reinterpret_cast(jhandle); + opts->force_consistency_checks = static_cast(jforce_consistency_checks); +} + +/* + * Class: org_rocksdb_Options + * Method: forceConsistencyChecks + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_forceConsistencyChecks(JNIEnv*, jobject, + jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + return static_cast(opts->force_consistency_checks); +} + +/// BLOB options + +/* + * Class: org_rocksdb_Options + * Method: setEnableBlobFiles + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setEnableBlobFiles(JNIEnv*, jobject, + jlong jhandle, + jboolean jenable_blob_files) { + auto* opts = reinterpret_cast(jhandle); + opts->enable_blob_files = static_cast(jenable_blob_files); +} + +/* + * Class: org_rocksdb_Options + * Method: enableBlobFiles + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_enableBlobFiles(JNIEnv*, jobject, + jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + return static_cast(opts->enable_blob_files); +} + +/* + * Class: org_rocksdb_Options + * Method: setMinBlobSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setMinBlobSize(JNIEnv*, jobject, jlong jhandle, + jlong jmin_blob_size) { + auto* opts = reinterpret_cast(jhandle); + opts->min_blob_size = static_cast(jmin_blob_size); +} + +/* + * Class: org_rocksdb_Options + * Method: minBlobSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_minBlobSize(JNIEnv*, jobject, jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + return static_cast(opts->min_blob_size); +} + +/* + * Class: org_rocksdb_Options + * Method: setBlobFileSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setBlobFileSize(JNIEnv*, jobject, jlong jhandle, + jlong jblob_file_size) { + auto* opts = reinterpret_cast(jhandle); + opts->blob_file_size = static_cast(jblob_file_size); +} + +/* + * Class: org_rocksdb_Options + * Method: blobFileSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_blobFileSize(JNIEnv*, jobject, jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + return static_cast(opts->blob_file_size); +} + +/* + * Class: org_rocksdb_Options + * Method: setBlobCompressionType + * Signature: (JB)V + */ +void Java_org_rocksdb_Options_setBlobCompressionType( + JNIEnv*, jobject, jlong jhandle, jbyte jblob_compression_type_value) { + auto* opts = reinterpret_cast(jhandle); + opts->blob_compression_type = + ROCKSDB_NAMESPACE::CompressionTypeJni::toCppCompressionType( + jblob_compression_type_value); +} + +/* + * Class: org_rocksdb_Options + * Method: blobCompressionType + * Signature: (J)B + */ +jbyte Java_org_rocksdb_Options_blobCompressionType(JNIEnv*, jobject, + jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::CompressionTypeJni::toJavaCompressionType( + opts->blob_compression_type); +} + +/* + * Class: org_rocksdb_Options + * Method: setEnableBlobGarbageCollection + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setEnableBlobGarbageCollection( + JNIEnv*, jobject, jlong jhandle, jboolean jenable_blob_garbage_collection) { + auto* opts = reinterpret_cast(jhandle); + opts->enable_blob_garbage_collection = + static_cast(jenable_blob_garbage_collection); +} + +/* + * Class: org_rocksdb_Options + * Method: enableBlobGarbageCollection + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_enableBlobGarbageCollection(JNIEnv*, jobject, + jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + return static_cast(opts->enable_blob_garbage_collection); +} + +/* + * Class: org_rocksdb_Options + * Method: setBlobGarbageCollectionAgeCutoff + * Signature: (JD)V + */ +void Java_org_rocksdb_Options_setBlobGarbageCollectionAgeCutoff( + JNIEnv*, jobject, jlong jhandle, + jdouble jblob_garbage_collection_age_cutoff) { + auto* opts = reinterpret_cast(jhandle); + opts->blob_garbage_collection_age_cutoff = + static_cast(jblob_garbage_collection_age_cutoff); +} + +/* + * Class: org_rocksdb_Options + * Method: blobGarbageCollectionAgeCutoff + * Signature: (J)D + */ +jdouble Java_org_rocksdb_Options_blobGarbageCollectionAgeCutoff(JNIEnv*, + jobject, + jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + return static_cast(opts->blob_garbage_collection_age_cutoff); +} + +/* + * Class: org_rocksdb_Options + * Method: setBlobGarbageCollectionForceThreshold + * Signature: (JD)V + */ +void Java_org_rocksdb_Options_setBlobGarbageCollectionForceThreshold( + JNIEnv*, jobject, jlong jhandle, + jdouble jblob_garbage_collection_force_threshold) { + auto* opts = reinterpret_cast(jhandle); + opts->blob_garbage_collection_force_threshold = + static_cast(jblob_garbage_collection_force_threshold); +} + +/* + * Class: org_rocksdb_Options + * Method: blobGarbageCollectionForceThreshold + * Signature: (J)D + */ +jdouble Java_org_rocksdb_Options_blobGarbageCollectionForceThreshold( + JNIEnv*, jobject, jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + return static_cast(opts->blob_garbage_collection_force_threshold); +} + +/* + * Class: org_rocksdb_Options + * Method: setBlobCompactionReadaheadSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setBlobCompactionReadaheadSize( + JNIEnv*, jobject, jlong jhandle, jlong jblob_compaction_readahead_size) { + auto* opts = reinterpret_cast(jhandle); + opts->blob_compaction_readahead_size = + static_cast(jblob_compaction_readahead_size); +} + +/* + * Class: org_rocksdb_Options + * Method: blobCompactionReadaheadSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_blobCompactionReadaheadSize(JNIEnv*, jobject, + jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + return static_cast(opts->blob_compaction_readahead_size); +} + +/* + * Class: org_rocksdb_Options + * Method: setBlobFileStartingLevel + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setBlobFileStartingLevel( + JNIEnv*, jobject, jlong jhandle, jint jblob_file_starting_level) { + auto* opts = reinterpret_cast(jhandle); + opts->blob_file_starting_level = jblob_file_starting_level; +} + +/* + * Class: org_rocksdb_Options + * Method: blobFileStartingLevel + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_blobFileStartingLevel(JNIEnv*, jobject, + jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + return static_cast(opts->blob_file_starting_level); +} + +/* + * Class: org_rocksdb_Options + * Method: setPrepopulateBlobCache + * Signature: (JB)V + */ +void Java_org_rocksdb_Options_setPrepopulateBlobCache( + JNIEnv*, jobject, jlong jhandle, jbyte jprepopulate_blob_cache_value) { + auto* opts = reinterpret_cast(jhandle); + opts->prepopulate_blob_cache = + ROCKSDB_NAMESPACE::PrepopulateBlobCacheJni::toCppPrepopulateBlobCache( + jprepopulate_blob_cache_value); +} + +/* + * Class: org_rocksdb_Options + * Method: prepopulateBlobCache + * Signature: (J)B + */ +jbyte Java_org_rocksdb_Options_prepopulateBlobCache(JNIEnv*, jobject, + jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::PrepopulateBlobCacheJni::toJavaPrepopulateBlobCache( + opts->prepopulate_blob_cache); +} + +/* + * Class: org_rocksdb_Options + * Method: setMemtableMaxRangeDeletions + * Signature: (JI)V + */ +void Java_org_rocksdb_Options_setMemtableMaxRangeDeletions( + JNIEnv*, jobject, jlong jhandle, jint jmemtable_max_range_deletions) { + auto* opts = reinterpret_cast(jhandle); + opts->memtable_max_range_deletions = + static_cast(jmemtable_max_range_deletions); +} + +/* + * Class: org_rocksdb_Options + * Method: memtableMaxRangeDeletions + * Signature: (J)I + */ +jint Java_org_rocksdb_Options_memtableMaxRangeDeletions(JNIEnv*, jobject, + jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + return static_cast(opts->memtable_max_range_deletions); +} + +////////////////////////////////////////////////////////////////////////////// +// ROCKSDB_NAMESPACE::ColumnFamilyOptions + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: newColumnFamilyOptions + * Signature: ()J + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_newColumnFamilyOptions(JNIEnv*, + jclass) { + auto* op = new ROCKSDB_NAMESPACE::ColumnFamilyOptions(); + return GET_CPLUSPLUS_POINTER(op); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: copyColumnFamilyOptions + * Signature: (J)J + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_copyColumnFamilyOptions( + JNIEnv*, jclass, jlong jhandle) { + auto new_opt = new ROCKSDB_NAMESPACE::ColumnFamilyOptions( + *(reinterpret_cast(jhandle))); + return GET_CPLUSPLUS_POINTER(new_opt); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: newColumnFamilyOptionsFromOptions + * Signature: (J)J + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_newColumnFamilyOptionsFromOptions( + JNIEnv*, jclass, jlong joptions_handle) { + auto new_opt = new ROCKSDB_NAMESPACE::ColumnFamilyOptions( + *reinterpret_cast(joptions_handle)); + return GET_CPLUSPLUS_POINTER(new_opt); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: getColumnFamilyOptionsFromProps + * Signature: (JLjava/lang/String;)J + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_getColumnFamilyOptionsFromProps__JLjava_lang_String_2( + JNIEnv* env, jclass, jlong cfg_handle, jstring jopt_string) { + const char* opt_string = env->GetStringUTFChars(jopt_string, nullptr); + if (opt_string == nullptr) { + // exception thrown: OutOfMemoryError + return 0; + } + auto* config_options = + reinterpret_cast(cfg_handle); + auto* cf_options = new ROCKSDB_NAMESPACE::ColumnFamilyOptions(); + ROCKSDB_NAMESPACE::Status status = + ROCKSDB_NAMESPACE::GetColumnFamilyOptionsFromString( + *config_options, ROCKSDB_NAMESPACE::ColumnFamilyOptions(), opt_string, + cf_options); + + env->ReleaseStringUTFChars(jopt_string, opt_string); + + // Check if ColumnFamilyOptions creation was possible. + jlong ret_value = 0; + if (status.ok()) { + ret_value = GET_CPLUSPLUS_POINTER(cf_options); + } else { + // if operation failed the ColumnFamilyOptions need to be deleted + // again to prevent a memory leak. + delete cf_options; + } + return ret_value; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: getColumnFamilyOptionsFromProps + * Signature: (Ljava/util/String;)J + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_getColumnFamilyOptionsFromProps__Ljava_lang_String_2( + JNIEnv* env, jclass, jstring jopt_string) { + const char* opt_string = env->GetStringUTFChars(jopt_string, nullptr); + if (opt_string == nullptr) { + // exception thrown: OutOfMemoryError + return 0; + } + + auto* cf_options = new ROCKSDB_NAMESPACE::ColumnFamilyOptions(); + ROCKSDB_NAMESPACE::ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + ROCKSDB_NAMESPACE::Status status = + ROCKSDB_NAMESPACE::GetColumnFamilyOptionsFromString( + config_options, ROCKSDB_NAMESPACE::ColumnFamilyOptions(), opt_string, + cf_options); + + env->ReleaseStringUTFChars(jopt_string, opt_string); + + // Check if ColumnFamilyOptions creation was possible. + jlong ret_value = 0; + if (status.ok()) { + ret_value = GET_CPLUSPLUS_POINTER(cf_options); + } else { + // if operation failed the ColumnFamilyOptions need to be deleted + // again to prevent a memory leak. + delete cf_options; + } + return ret_value; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_disposeInternal(JNIEnv*, jobject, + jlong handle) { + auto* cfo = reinterpret_cast(handle); + assert(cfo != nullptr); + delete cfo; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: oldDefaults + * Signature: (JII)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_oldDefaults(JNIEnv*, jclass, + jlong jhandle, + jint major_version, + jint minor_version) { + reinterpret_cast(jhandle) + ->OldDefaults(major_version, minor_version); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: optimizeForSmallDb + * Signature: (J)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_optimizeForSmallDb__J(JNIEnv*, + jobject, + jlong jhandle) { + reinterpret_cast(jhandle) + ->OptimizeForSmallDb(); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: optimizeForSmallDb + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_optimizeForSmallDb__JJ( + JNIEnv*, jclass, jlong jhandle, jlong cache_handle) { + auto* cache_sptr_ptr = + reinterpret_cast*>( + cache_handle); + reinterpret_cast(jhandle) + ->OptimizeForSmallDb(cache_sptr_ptr); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: optimizeForPointLookup + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_optimizeForPointLookup( + JNIEnv*, jobject, jlong jhandle, jlong block_cache_size_mb) { + reinterpret_cast(jhandle) + ->OptimizeForPointLookup(block_cache_size_mb); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: optimizeLevelStyleCompaction + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_optimizeLevelStyleCompaction( + JNIEnv*, jobject, jlong jhandle, jlong memtable_memory_budget) { + reinterpret_cast(jhandle) + ->OptimizeLevelStyleCompaction(memtable_memory_budget); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: optimizeUniversalStyleCompaction + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_optimizeUniversalStyleCompaction( + JNIEnv*, jobject, jlong jhandle, jlong memtable_memory_budget) { + reinterpret_cast(jhandle) + ->OptimizeUniversalStyleCompaction(memtable_memory_budget); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setComparatorHandle + * Signature: (JI)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setComparatorHandle__JI( + JNIEnv*, jobject, jlong jhandle, jint builtinComparator) { + switch (builtinComparator) { + case 1: + reinterpret_cast(jhandle) + ->comparator = ROCKSDB_NAMESPACE::ReverseBytewiseComparator(); + break; + default: + reinterpret_cast(jhandle) + ->comparator = ROCKSDB_NAMESPACE::BytewiseComparator(); + break; + } +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setComparatorHandle + * Signature: (JJB)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setComparatorHandle__JJB( + JNIEnv*, jobject, jlong jopt_handle, jlong jcomparator_handle, + jbyte jcomparator_type) { + ROCKSDB_NAMESPACE::Comparator* comparator = nullptr; + switch (jcomparator_type) { + // JAVA_COMPARATOR + case 0x0: + comparator = reinterpret_cast( + jcomparator_handle); + break; + + // JAVA_NATIVE_COMPARATOR_WRAPPER + case 0x1: + comparator = + reinterpret_cast(jcomparator_handle); + break; + } + auto* opt = + reinterpret_cast(jopt_handle); + opt->comparator = comparator; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setMergeOperatorName + * Signature: (JJjava/lang/String)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setMergeOperatorName( + JNIEnv* env, jobject, jlong jhandle, jstring jop_name) { + auto* options = + reinterpret_cast(jhandle); + const char* op_name = env->GetStringUTFChars(jop_name, nullptr); + if (op_name == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + options->merge_operator = + ROCKSDB_NAMESPACE::MergeOperators::CreateFromStringId(op_name); + env->ReleaseStringUTFChars(jop_name, op_name); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setMergeOperator + * Signature: (JJjava/lang/String)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setMergeOperator( + JNIEnv*, jobject, jlong jhandle, jlong mergeOperatorHandle) { + reinterpret_cast(jhandle) + ->merge_operator = + *(reinterpret_cast*>( + mergeOperatorHandle)); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setCompactionFilterHandle + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setCompactionFilterHandle( + JNIEnv*, jobject, jlong jopt_handle, jlong jcompactionfilter_handle) { + reinterpret_cast(jopt_handle) + ->compaction_filter = + reinterpret_cast( + jcompactionfilter_handle); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setCompactionFilterFactoryHandle + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setCompactionFilterFactoryHandle( + JNIEnv*, jobject, jlong jopt_handle, + jlong jcompactionfilterfactory_handle) { + auto* cff_factory = reinterpret_cast< + std::shared_ptr*>( + jcompactionfilterfactory_handle); + reinterpret_cast(jopt_handle) + ->compaction_filter_factory = *cff_factory; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setWriteBufferSize + * Signature: (JJ)I + */ +void Java_org_rocksdb_ColumnFamilyOptions_setWriteBufferSize( + JNIEnv* env, jobject, jlong jhandle, jlong jwrite_buffer_size) { + auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( + jwrite_buffer_size); + if (s.ok()) { + reinterpret_cast(jhandle) + ->write_buffer_size = jwrite_buffer_size; + } else { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: writeBufferSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_writeBufferSize(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->write_buffer_size; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setMaxWriteBufferNumber + * Signature: (JI)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setMaxWriteBufferNumber( + JNIEnv*, jobject, jlong jhandle, jint jmax_write_buffer_number) { + reinterpret_cast(jhandle) + ->max_write_buffer_number = jmax_write_buffer_number; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: maxWriteBufferNumber + * Signature: (J)I + */ +jint Java_org_rocksdb_ColumnFamilyOptions_maxWriteBufferNumber(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_write_buffer_number; +} + +/* + * Method: setMemTableFactory + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setMemTableFactory( + JNIEnv*, jobject, jlong jhandle, jlong jfactory_handle) { + reinterpret_cast(jhandle) + ->memtable_factory.reset( + reinterpret_cast( + jfactory_handle)); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: memTableFactoryName + * Signature: (J)Ljava/lang/String + */ +jstring Java_org_rocksdb_ColumnFamilyOptions_memTableFactoryName( + JNIEnv* env, jobject, jlong jhandle) { + auto* opt = + reinterpret_cast(jhandle); + ROCKSDB_NAMESPACE::MemTableRepFactory* tf = opt->memtable_factory.get(); + + // Should never be nullptr. + // Default memtable factory is SkipListFactory + assert(tf); + + // temporarly fix for the historical typo + if (strcmp(tf->Name(), "HashLinkListRepFactory") == 0) { + return env->NewStringUTF("HashLinkedListRepFactory"); + } + + return env->NewStringUTF(tf->Name()); +} + +/* + * Method: useFixedLengthPrefixExtractor + * Signature: (JI)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_useFixedLengthPrefixExtractor( + JNIEnv*, jobject, jlong jhandle, jint jprefix_length) { + reinterpret_cast(jhandle) + ->prefix_extractor.reset(ROCKSDB_NAMESPACE::NewFixedPrefixTransform( + static_cast(jprefix_length))); +} + +/* + * Method: useCappedPrefixExtractor + * Signature: (JI)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_useCappedPrefixExtractor( + JNIEnv*, jobject, jlong jhandle, jint jprefix_length) { + reinterpret_cast(jhandle) + ->prefix_extractor.reset(ROCKSDB_NAMESPACE::NewCappedPrefixTransform( + static_cast(jprefix_length))); +} + +/* + * Method: setTableFactory + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setTableFactory( + JNIEnv*, jobject, jlong jhandle, jlong jfactory_handle) { + reinterpret_cast(jhandle) + ->table_factory.reset( + reinterpret_cast(jfactory_handle)); +} + +/* + * Method: setSstPartitionerFactory + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setSstPartitionerFactory( + JNIEnv*, jobject, jlong jhandle, jlong factory_handle) { + auto* options = + reinterpret_cast(jhandle); + auto factory = reinterpret_cast< + std::shared_ptr*>( + factory_handle); + options->sst_partitioner_factory = *factory; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setCompactionThreadLimiter + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setCompactionThreadLimiter( + JNIEnv*, jclass, jlong jhandle, jlong jlimiter_handle) { + auto* options = + reinterpret_cast(jhandle); + auto* limiter = reinterpret_cast< + std::shared_ptr*>( + jlimiter_handle); + options->compaction_thread_limiter = *limiter; +} + +/* + * Method: tableFactoryName + * Signature: (J)Ljava/lang/String + */ +jstring Java_org_rocksdb_ColumnFamilyOptions_tableFactoryName(JNIEnv* env, + jobject, + jlong jhandle) { + auto* opt = + reinterpret_cast(jhandle); + ROCKSDB_NAMESPACE::TableFactory* tf = opt->table_factory.get(); + + // Should never be nullptr. + // Default memtable factory is SkipListFactory + assert(tf); + + return env->NewStringUTF(tf->Name()); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setCfPaths + * Signature: (J[Ljava/lang/String;[J)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setCfPaths(JNIEnv* env, jclass, + jlong jhandle, + jobjectArray path_array, + jlongArray size_array) { + auto* options = + reinterpret_cast(jhandle); + jboolean has_exception = JNI_FALSE; + std::vector cf_paths = + rocksdb_convert_cf_paths_from_java_helper(env, path_array, size_array, + &has_exception); + if (JNI_FALSE == has_exception) { + options->cf_paths = std::move(cf_paths); + } +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: cfPathsLen + * Signature: (J)J + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_cfPathsLen(JNIEnv*, jclass, + jlong jhandle) { + auto* opt = + reinterpret_cast(jhandle); + return static_cast(opt->cf_paths.size()); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: cfPaths + * Signature: (J[Ljava/lang/String;[J)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_cfPaths(JNIEnv* env, jclass, + jlong jhandle, + jobjectArray jpaths, + jlongArray jtarget_sizes) { + rocksdb_convert_cf_paths_to_java_helper< + ROCKSDB_NAMESPACE::ColumnFamilyOptions>(env, jhandle, jpaths, + jtarget_sizes); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: minWriteBufferNumberToMerge + * Signature: (J)I + */ +jint Java_org_rocksdb_ColumnFamilyOptions_minWriteBufferNumberToMerge( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->min_write_buffer_number_to_merge; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setMinWriteBufferNumberToMerge + * Signature: (JI)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setMinWriteBufferNumberToMerge( + JNIEnv*, jobject, jlong jhandle, jint jmin_write_buffer_number_to_merge) { + reinterpret_cast(jhandle) + ->min_write_buffer_number_to_merge = + static_cast(jmin_write_buffer_number_to_merge); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: maxWriteBufferNumberToMaintain + * Signature: (J)I + */ +jint Java_org_rocksdb_ColumnFamilyOptions_maxWriteBufferNumberToMaintain( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_write_buffer_number_to_maintain; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setMaxWriteBufferNumberToMaintain + * Signature: (JI)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setMaxWriteBufferNumberToMaintain( + JNIEnv*, jobject, jlong jhandle, + jint jmax_write_buffer_number_to_maintain) { + reinterpret_cast(jhandle) + ->max_write_buffer_number_to_maintain = + static_cast(jmax_write_buffer_number_to_maintain); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setCompressionType + * Signature: (JB)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setCompressionType( + JNIEnv*, jobject, jlong jhandle, jbyte jcompression_type_value) { + auto* cf_opts = + reinterpret_cast(jhandle); + cf_opts->compression = + ROCKSDB_NAMESPACE::CompressionTypeJni::toCppCompressionType( + jcompression_type_value); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: compressionType + * Signature: (J)B + */ +jbyte Java_org_rocksdb_ColumnFamilyOptions_compressionType(JNIEnv*, jobject, + jlong jhandle) { + auto* cf_opts = + reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::CompressionTypeJni::toJavaCompressionType( + cf_opts->compression); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setCompressionPerLevel + * Signature: (J[B)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setCompressionPerLevel( + JNIEnv* env, jobject, jlong jhandle, jbyteArray jcompressionLevels) { + auto* options = + reinterpret_cast(jhandle); + auto uptr_compression_levels = + rocksdb_compression_vector_helper(env, jcompressionLevels); + if (!uptr_compression_levels) { + // exception occurred + return; + } + options->compression_per_level = *(uptr_compression_levels.get()); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: compressionPerLevel + * Signature: (J)[B + */ +jbyteArray Java_org_rocksdb_ColumnFamilyOptions_compressionPerLevel( + JNIEnv* env, jobject, jlong jhandle) { + auto* cf_options = + reinterpret_cast(jhandle); + return rocksdb_compression_list_helper(env, + cf_options->compression_per_level); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setBottommostCompressionType + * Signature: (JB)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setBottommostCompressionType( + JNIEnv*, jobject, jlong jhandle, jbyte jcompression_type_value) { + auto* cf_options = + reinterpret_cast(jhandle); + cf_options->bottommost_compression = + ROCKSDB_NAMESPACE::CompressionTypeJni::toCppCompressionType( + jcompression_type_value); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: bottommostCompressionType + * Signature: (J)B + */ +jbyte Java_org_rocksdb_ColumnFamilyOptions_bottommostCompressionType( + JNIEnv*, jobject, jlong jhandle) { + auto* cf_options = + reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::CompressionTypeJni::toJavaCompressionType( + cf_options->bottommost_compression); +} +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setBottommostCompressionOptions + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setBottommostCompressionOptions( + JNIEnv*, jobject, jlong jhandle, + jlong jbottommost_compression_options_handle) { + auto* cf_options = + reinterpret_cast(jhandle); + auto* bottommost_compression_options = + reinterpret_cast( + jbottommost_compression_options_handle); + cf_options->bottommost_compression_opts = *bottommost_compression_options; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setCompressionOptions + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setCompressionOptions( + JNIEnv*, jobject, jlong jhandle, jlong jcompression_options_handle) { + auto* cf_options = + reinterpret_cast(jhandle); + auto* compression_options = + reinterpret_cast( + jcompression_options_handle); + cf_options->compression_opts = *compression_options; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setCompactionStyle + * Signature: (JB)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setCompactionStyle( + JNIEnv*, jobject, jlong jhandle, jbyte jcompaction_style) { + auto* cf_options = + reinterpret_cast(jhandle); + cf_options->compaction_style = + ROCKSDB_NAMESPACE::CompactionStyleJni::toCppCompactionStyle( + jcompaction_style); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: compactionStyle + * Signature: (J)B + */ +jbyte Java_org_rocksdb_ColumnFamilyOptions_compactionStyle(JNIEnv*, jobject, + jlong jhandle) { + auto* cf_options = + reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::CompactionStyleJni::toJavaCompactionStyle( + cf_options->compaction_style); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setMaxTableFilesSizeFIFO + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setMaxTableFilesSizeFIFO( + JNIEnv*, jobject, jlong jhandle, jlong jmax_table_files_size) { + reinterpret_cast(jhandle) + ->compaction_options_fifo.max_table_files_size = + static_cast(jmax_table_files_size); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: maxTableFilesSizeFIFO + * Signature: (J)J + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_maxTableFilesSizeFIFO( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->compaction_options_fifo.max_table_files_size; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: numLevels + * Signature: (J)I + */ +jint Java_org_rocksdb_ColumnFamilyOptions_numLevels(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->num_levels; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setNumLevels + * Signature: (JI)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setNumLevels(JNIEnv*, jobject, + jlong jhandle, + jint jnum_levels) { + reinterpret_cast(jhandle) + ->num_levels = static_cast(jnum_levels); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: levelZeroFileNumCompactionTrigger + * Signature: (J)I + */ +jint Java_org_rocksdb_ColumnFamilyOptions_levelZeroFileNumCompactionTrigger( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->level0_file_num_compaction_trigger; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setLevelZeroFileNumCompactionTrigger + * Signature: (JI)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setLevelZeroFileNumCompactionTrigger( + JNIEnv*, jobject, jlong jhandle, jint jlevel0_file_num_compaction_trigger) { + reinterpret_cast(jhandle) + ->level0_file_num_compaction_trigger = + static_cast(jlevel0_file_num_compaction_trigger); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: levelZeroSlowdownWritesTrigger + * Signature: (J)I + */ +jint Java_org_rocksdb_ColumnFamilyOptions_levelZeroSlowdownWritesTrigger( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->level0_slowdown_writes_trigger; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setLevelSlowdownWritesTrigger + * Signature: (JI)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setLevelZeroSlowdownWritesTrigger( + JNIEnv*, jobject, jlong jhandle, jint jlevel0_slowdown_writes_trigger) { + reinterpret_cast(jhandle) + ->level0_slowdown_writes_trigger = + static_cast(jlevel0_slowdown_writes_trigger); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: levelZeroStopWritesTrigger + * Signature: (J)I + */ +jint Java_org_rocksdb_ColumnFamilyOptions_levelZeroStopWritesTrigger( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->level0_stop_writes_trigger; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setLevelStopWritesTrigger + * Signature: (JI)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setLevelZeroStopWritesTrigger( + JNIEnv*, jobject, jlong jhandle, jint jlevel0_stop_writes_trigger) { + reinterpret_cast(jhandle) + ->level0_stop_writes_trigger = + static_cast(jlevel0_stop_writes_trigger); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: targetFileSizeBase + * Signature: (J)J + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_targetFileSizeBase(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->target_file_size_base; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setTargetFileSizeBase + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setTargetFileSizeBase( + JNIEnv*, jobject, jlong jhandle, jlong jtarget_file_size_base) { + reinterpret_cast(jhandle) + ->target_file_size_base = static_cast(jtarget_file_size_base); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: targetFileSizeMultiplier + * Signature: (J)I + */ +jint Java_org_rocksdb_ColumnFamilyOptions_targetFileSizeMultiplier( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->target_file_size_multiplier; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setTargetFileSizeMultiplier + * Signature: (JI)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setTargetFileSizeMultiplier( + JNIEnv*, jobject, jlong jhandle, jint jtarget_file_size_multiplier) { + reinterpret_cast(jhandle) + ->target_file_size_multiplier = + static_cast(jtarget_file_size_multiplier); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: maxBytesForLevelBase + * Signature: (J)J + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_maxBytesForLevelBase(JNIEnv*, + jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_bytes_for_level_base; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setMaxBytesForLevelBase + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setMaxBytesForLevelBase( + JNIEnv*, jobject, jlong jhandle, jlong jmax_bytes_for_level_base) { + reinterpret_cast(jhandle) + ->max_bytes_for_level_base = + static_cast(jmax_bytes_for_level_base); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: levelCompactionDynamicLevelBytes + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ColumnFamilyOptions_levelCompactionDynamicLevelBytes( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->level_compaction_dynamic_level_bytes; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setLevelCompactionDynamicLevelBytes + * Signature: (JZ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setLevelCompactionDynamicLevelBytes( + JNIEnv*, jobject, jlong jhandle, jboolean jenable_dynamic_level_bytes) { + reinterpret_cast(jhandle) + ->level_compaction_dynamic_level_bytes = (jenable_dynamic_level_bytes); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: maxBytesForLevelMultiplier + * Signature: (J)D + */ +jdouble Java_org_rocksdb_ColumnFamilyOptions_maxBytesForLevelMultiplier( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_bytes_for_level_multiplier; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setMaxBytesForLevelMultiplier + * Signature: (JD)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setMaxBytesForLevelMultiplier( + JNIEnv*, jobject, jlong jhandle, jdouble jmax_bytes_for_level_multiplier) { + reinterpret_cast(jhandle) + ->max_bytes_for_level_multiplier = + static_cast(jmax_bytes_for_level_multiplier); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: maxCompactionBytes + * Signature: (J)I + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_maxCompactionBytes(JNIEnv*, jobject, + jlong jhandle) { + return static_cast( + reinterpret_cast(jhandle) + ->max_compaction_bytes); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setMaxCompactionBytes + * Signature: (JI)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setMaxCompactionBytes( + JNIEnv*, jobject, jlong jhandle, jlong jmax_compaction_bytes) { + reinterpret_cast(jhandle) + ->max_compaction_bytes = static_cast(jmax_compaction_bytes); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: arenaBlockSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_arenaBlockSize(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->arena_block_size; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setArenaBlockSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setArenaBlockSize( + JNIEnv* env, jobject, jlong jhandle, jlong jarena_block_size) { + auto s = + ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t(jarena_block_size); + if (s.ok()) { + reinterpret_cast(jhandle) + ->arena_block_size = jarena_block_size; + } else { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: disableAutoCompactions + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ColumnFamilyOptions_disableAutoCompactions( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->disable_auto_compactions; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setDisableAutoCompactions + * Signature: (JZ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setDisableAutoCompactions( + JNIEnv*, jobject, jlong jhandle, jboolean jdisable_auto_compactions) { + reinterpret_cast(jhandle) + ->disable_auto_compactions = static_cast(jdisable_auto_compactions); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: maxSequentialSkipInIterations + * Signature: (J)J + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_maxSequentialSkipInIterations( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_sequential_skip_in_iterations; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setMaxSequentialSkipInIterations + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setMaxSequentialSkipInIterations( + JNIEnv*, jobject, jlong jhandle, jlong jmax_sequential_skip_in_iterations) { + reinterpret_cast(jhandle) + ->max_sequential_skip_in_iterations = + static_cast(jmax_sequential_skip_in_iterations); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: inplaceUpdateSupport + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ColumnFamilyOptions_inplaceUpdateSupport( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->inplace_update_support; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setInplaceUpdateSupport + * Signature: (JZ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setInplaceUpdateSupport( + JNIEnv*, jobject, jlong jhandle, jboolean jinplace_update_support) { + reinterpret_cast(jhandle) + ->inplace_update_support = static_cast(jinplace_update_support); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: inplaceUpdateNumLocks + * Signature: (J)J + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_inplaceUpdateNumLocks( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->inplace_update_num_locks; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setInplaceUpdateNumLocks + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setInplaceUpdateNumLocks( + JNIEnv* env, jobject, jlong jhandle, jlong jinplace_update_num_locks) { + auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( + jinplace_update_num_locks); + if (s.ok()) { + reinterpret_cast(jhandle) + ->inplace_update_num_locks = jinplace_update_num_locks; + } else { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: memtablePrefixBloomSizeRatio + * Signature: (J)I + */ +jdouble Java_org_rocksdb_ColumnFamilyOptions_memtablePrefixBloomSizeRatio( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->memtable_prefix_bloom_size_ratio; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setMemtablePrefixBloomSizeRatio + * Signature: (JI)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setMemtablePrefixBloomSizeRatio( + JNIEnv*, jobject, jlong jhandle, + jdouble jmemtable_prefix_bloom_size_ratio) { + reinterpret_cast(jhandle) + ->memtable_prefix_bloom_size_ratio = + static_cast(jmemtable_prefix_bloom_size_ratio); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: experimentalMempurgeThreshold + * Signature: (J)I + */ +jdouble Java_org_rocksdb_ColumnFamilyOptions_experimentalMempurgeThreshold( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->experimental_mempurge_threshold; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setExperimentalMempurgeThreshold + * Signature: (JI)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setExperimentalMempurgeThreshold( + JNIEnv*, jobject, jlong jhandle, jdouble jexperimental_mempurge_threshold) { + reinterpret_cast(jhandle) + ->experimental_mempurge_threshold = + static_cast(jexperimental_mempurge_threshold); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: memtableWholeKeyFiltering + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ColumnFamilyOptions_memtableWholeKeyFiltering( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->memtable_whole_key_filtering; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setMemtableWholeKeyFiltering + * Signature: (JZ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setMemtableWholeKeyFiltering( + JNIEnv*, jobject, jlong jhandle, jboolean jmemtable_whole_key_filtering) { + reinterpret_cast(jhandle) + ->memtable_whole_key_filtering = + static_cast(jmemtable_whole_key_filtering); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: bloomLocality + * Signature: (J)I + */ +jint Java_org_rocksdb_ColumnFamilyOptions_bloomLocality(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->bloom_locality; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setBloomLocality + * Signature: (JI)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setBloomLocality( + JNIEnv*, jobject, jlong jhandle, jint jbloom_locality) { + reinterpret_cast(jhandle) + ->bloom_locality = static_cast(jbloom_locality); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: maxSuccessiveMerges + * Signature: (J)J + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_maxSuccessiveMerges(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_successive_merges; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setMaxSuccessiveMerges + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setMaxSuccessiveMerges( + JNIEnv* env, jobject, jlong jhandle, jlong jmax_successive_merges) { + auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( + jmax_successive_merges); + if (s.ok()) { + reinterpret_cast(jhandle) + ->max_successive_merges = jmax_successive_merges; + } else { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: optimizeFiltersForHits + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ColumnFamilyOptions_optimizeFiltersForHits( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->optimize_filters_for_hits; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setOptimizeFiltersForHits + * Signature: (JZ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setOptimizeFiltersForHits( + JNIEnv*, jobject, jlong jhandle, jboolean joptimize_filters_for_hits) { + reinterpret_cast(jhandle) + ->optimize_filters_for_hits = + static_cast(joptimize_filters_for_hits); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: memtableHugePageSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_memtableHugePageSize(JNIEnv*, + jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->memtable_huge_page_size; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setMemtableHugePageSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setMemtableHugePageSize( + JNIEnv* env, jobject, jlong jhandle, jlong jmemtable_huge_page_size) { + auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( + jmemtable_huge_page_size); + if (s.ok()) { + reinterpret_cast(jhandle) + ->memtable_huge_page_size = jmemtable_huge_page_size; + } else { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: softPendingCompactionBytesLimit + * Signature: (J)J + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_softPendingCompactionBytesLimit( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->soft_pending_compaction_bytes_limit; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setSoftPendingCompactionBytesLimit + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setSoftPendingCompactionBytesLimit( + JNIEnv*, jobject, jlong jhandle, + jlong jsoft_pending_compaction_bytes_limit) { + reinterpret_cast(jhandle) + ->soft_pending_compaction_bytes_limit = + static_cast(jsoft_pending_compaction_bytes_limit); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: softHardCompactionBytesLimit + * Signature: (J)J + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_hardPendingCompactionBytesLimit( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->hard_pending_compaction_bytes_limit; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setHardPendingCompactionBytesLimit + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setHardPendingCompactionBytesLimit( + JNIEnv*, jobject, jlong jhandle, + jlong jhard_pending_compaction_bytes_limit) { + reinterpret_cast(jhandle) + ->hard_pending_compaction_bytes_limit = + static_cast(jhard_pending_compaction_bytes_limit); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: level0FileNumCompactionTrigger + * Signature: (J)I + */ +jint Java_org_rocksdb_ColumnFamilyOptions_level0FileNumCompactionTrigger( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->level0_file_num_compaction_trigger; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setLevel0FileNumCompactionTrigger + * Signature: (JI)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setLevel0FileNumCompactionTrigger( + JNIEnv*, jobject, jlong jhandle, jint jlevel0_file_num_compaction_trigger) { + reinterpret_cast(jhandle) + ->level0_file_num_compaction_trigger = + static_cast(jlevel0_file_num_compaction_trigger); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: level0SlowdownWritesTrigger + * Signature: (J)I + */ +jint Java_org_rocksdb_ColumnFamilyOptions_level0SlowdownWritesTrigger( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->level0_slowdown_writes_trigger; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setLevel0SlowdownWritesTrigger + * Signature: (JI)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setLevel0SlowdownWritesTrigger( + JNIEnv*, jobject, jlong jhandle, jint jlevel0_slowdown_writes_trigger) { + reinterpret_cast(jhandle) + ->level0_slowdown_writes_trigger = + static_cast(jlevel0_slowdown_writes_trigger); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: level0StopWritesTrigger + * Signature: (J)I + */ +jint Java_org_rocksdb_ColumnFamilyOptions_level0StopWritesTrigger( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->level0_stop_writes_trigger; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setLevel0StopWritesTrigger + * Signature: (JI)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setLevel0StopWritesTrigger( + JNIEnv*, jobject, jlong jhandle, jint jlevel0_stop_writes_trigger) { + reinterpret_cast(jhandle) + ->level0_stop_writes_trigger = + static_cast(jlevel0_stop_writes_trigger); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: maxBytesForLevelMultiplierAdditional + * Signature: (J)[I + */ +jintArray +Java_org_rocksdb_ColumnFamilyOptions_maxBytesForLevelMultiplierAdditional( + JNIEnv* env, jobject, jlong jhandle) { + auto mbflma = + reinterpret_cast(jhandle) + ->max_bytes_for_level_multiplier_additional; + + const size_t size = mbflma.size(); + + jint* additionals = new jint[size]; + for (size_t i = 0; i < size; i++) { + additionals[i] = static_cast(mbflma[i]); + } + + jsize jlen = static_cast(size); + jintArray result = env->NewIntArray(jlen); + if (result == nullptr) { + // exception thrown: OutOfMemoryError + delete[] additionals; + return nullptr; + } + env->SetIntArrayRegion(result, 0, jlen, additionals); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(result); + delete[] additionals; + return nullptr; + } + + delete[] additionals; + + return result; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setMaxBytesForLevelMultiplierAdditional + * Signature: (J[I)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setMaxBytesForLevelMultiplierAdditional( + JNIEnv* env, jobject, jlong jhandle, + jintArray jmax_bytes_for_level_multiplier_additional) { + jsize len = env->GetArrayLength(jmax_bytes_for_level_multiplier_additional); + jint* additionals = env->GetIntArrayElements( + jmax_bytes_for_level_multiplier_additional, nullptr); + if (additionals == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + auto* cf_opt = + reinterpret_cast(jhandle); + cf_opt->max_bytes_for_level_multiplier_additional.clear(); + for (jsize i = 0; i < len; i++) { + cf_opt->max_bytes_for_level_multiplier_additional.push_back( + static_cast(additionals[i])); + } + + env->ReleaseIntArrayElements(jmax_bytes_for_level_multiplier_additional, + additionals, JNI_ABORT); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: paranoidFileChecks + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ColumnFamilyOptions_paranoidFileChecks( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->paranoid_file_checks; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setParanoidFileChecks + * Signature: (JZ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setParanoidFileChecks( + JNIEnv*, jobject, jlong jhandle, jboolean jparanoid_file_checks) { + reinterpret_cast(jhandle) + ->paranoid_file_checks = static_cast(jparanoid_file_checks); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setCompactionPriority + * Signature: (JB)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setCompactionPriority( + JNIEnv*, jobject, jlong jhandle, jbyte jcompaction_priority_value) { + auto* cf_opts = + reinterpret_cast(jhandle); + cf_opts->compaction_pri = + ROCKSDB_NAMESPACE::CompactionPriorityJni::toCppCompactionPriority( + jcompaction_priority_value); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: compactionPriority + * Signature: (J)B + */ +jbyte Java_org_rocksdb_ColumnFamilyOptions_compactionPriority(JNIEnv*, jobject, + jlong jhandle) { + auto* cf_opts = + reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::CompactionPriorityJni::toJavaCompactionPriority( + cf_opts->compaction_pri); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setReportBgIoStats + * Signature: (JZ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setReportBgIoStats( + JNIEnv*, jobject, jlong jhandle, jboolean jreport_bg_io_stats) { + auto* cf_opts = + reinterpret_cast(jhandle); + cf_opts->report_bg_io_stats = static_cast(jreport_bg_io_stats); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: reportBgIoStats + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ColumnFamilyOptions_reportBgIoStats(JNIEnv*, jobject, + jlong jhandle) { + auto* cf_opts = + reinterpret_cast(jhandle); + return static_cast(cf_opts->report_bg_io_stats); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setTtl + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setTtl(JNIEnv*, jobject, + jlong jhandle, jlong jttl) { + auto* cf_opts = + reinterpret_cast(jhandle); + cf_opts->ttl = static_cast(jttl); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: ttl + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL +Java_org_rocksdb_ColumnFamilyOptions_ttl(JNIEnv*, jobject, jlong jhandle) { + auto* cf_opts = + reinterpret_cast(jhandle); + return static_cast(cf_opts->ttl); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setPeriodicCompactionSeconds + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setPeriodicCompactionSeconds( + JNIEnv*, jobject, jlong jhandle, jlong jperiodicCompactionSeconds) { + auto* cf_opts = + reinterpret_cast(jhandle); + cf_opts->periodic_compaction_seconds = + static_cast(jperiodicCompactionSeconds); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: periodicCompactionSeconds + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL +Java_org_rocksdb_ColumnFamilyOptions_periodicCompactionSeconds(JNIEnv*, jobject, + jlong jhandle) { + auto* cf_opts = + reinterpret_cast(jhandle); + return static_cast(cf_opts->periodic_compaction_seconds); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setCompactionOptionsUniversal + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setCompactionOptionsUniversal( + JNIEnv*, jobject, jlong jhandle, + jlong jcompaction_options_universal_handle) { + auto* cf_opts = + reinterpret_cast(jhandle); + auto* opts_uni = + reinterpret_cast( + jcompaction_options_universal_handle); + cf_opts->compaction_options_universal = *opts_uni; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setCompactionOptionsFIFO + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setCompactionOptionsFIFO( + JNIEnv*, jobject, jlong jhandle, jlong jcompaction_options_fifo_handle) { + auto* cf_opts = + reinterpret_cast(jhandle); + auto* opts_fifo = reinterpret_cast( + jcompaction_options_fifo_handle); + cf_opts->compaction_options_fifo = *opts_fifo; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setForceConsistencyChecks + * Signature: (JZ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setForceConsistencyChecks( + JNIEnv*, jobject, jlong jhandle, jboolean jforce_consistency_checks) { + auto* cf_opts = + reinterpret_cast(jhandle); + cf_opts->force_consistency_checks = + static_cast(jforce_consistency_checks); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: forceConsistencyChecks + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ColumnFamilyOptions_forceConsistencyChecks( + JNIEnv*, jobject, jlong jhandle) { + auto* cf_opts = + reinterpret_cast(jhandle); + return static_cast(cf_opts->force_consistency_checks); +} + +/// BLOB options + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setEnableBlobFiles + * Signature: (JZ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setEnableBlobFiles( + JNIEnv*, jobject, jlong jhandle, jboolean jenable_blob_files) { + auto* opts = + reinterpret_cast(jhandle); + opts->enable_blob_files = static_cast(jenable_blob_files); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: enableBlobFiles + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ColumnFamilyOptions_enableBlobFiles(JNIEnv*, jobject, + jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return static_cast(opts->enable_blob_files); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setMinBlobSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setMinBlobSize(JNIEnv*, jobject, + jlong jhandle, + jlong jmin_blob_size) { + auto* opts = + reinterpret_cast(jhandle); + opts->min_blob_size = static_cast(jmin_blob_size); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: minBlobSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_minBlobSize(JNIEnv*, jobject, + jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return static_cast(opts->min_blob_size); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setBlobFileSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setBlobFileSize( + JNIEnv*, jobject, jlong jhandle, jlong jblob_file_size) { + auto* opts = + reinterpret_cast(jhandle); + opts->blob_file_size = static_cast(jblob_file_size); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: blobFileSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_blobFileSize(JNIEnv*, jobject, + jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return static_cast(opts->blob_file_size); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setBlobCompressionType + * Signature: (JB)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setBlobCompressionType( + JNIEnv*, jobject, jlong jhandle, jbyte jblob_compression_type_value) { + auto* opts = + reinterpret_cast(jhandle); + opts->blob_compression_type = + ROCKSDB_NAMESPACE::CompressionTypeJni::toCppCompressionType( + jblob_compression_type_value); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: blobCompressionType + * Signature: (J)B + */ +jbyte Java_org_rocksdb_ColumnFamilyOptions_blobCompressionType(JNIEnv*, jobject, + jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::CompressionTypeJni::toJavaCompressionType( + opts->blob_compression_type); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setEnableBlobGarbageCollection + * Signature: (JZ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setEnableBlobGarbageCollection( + JNIEnv*, jobject, jlong jhandle, jboolean jenable_blob_garbage_collection) { + auto* opts = + reinterpret_cast(jhandle); + opts->enable_blob_garbage_collection = + static_cast(jenable_blob_garbage_collection); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: enableBlobGarbageCollection + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ColumnFamilyOptions_enableBlobGarbageCollection( + JNIEnv*, jobject, jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return static_cast(opts->enable_blob_garbage_collection); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setBlobGarbageCollectionAgeCutoff + * Signature: (JD)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setBlobGarbageCollectionAgeCutoff( + JNIEnv*, jobject, jlong jhandle, + jdouble jblob_garbage_collection_age_cutoff) { + auto* opts = + reinterpret_cast(jhandle); + opts->blob_garbage_collection_age_cutoff = + static_cast(jblob_garbage_collection_age_cutoff); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: blobGarbageCollectionAgeCutoff + * Signature: (J)D + */ +jdouble Java_org_rocksdb_ColumnFamilyOptions_blobGarbageCollectionAgeCutoff( + JNIEnv*, jobject, jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return static_cast(opts->blob_garbage_collection_age_cutoff); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setBlobGarbageCollectionForceThreshold + * Signature: (JD)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setBlobGarbageCollectionForceThreshold( + JNIEnv*, jobject, jlong jhandle, + jdouble jblob_garbage_collection_force_threshold) { + auto* opts = + reinterpret_cast(jhandle); + opts->blob_garbage_collection_force_threshold = + static_cast(jblob_garbage_collection_force_threshold); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: blobGarbageCollectionForceThreshold + * Signature: (J)D + */ +jdouble +Java_org_rocksdb_ColumnFamilyOptions_blobGarbageCollectionForceThreshold( + JNIEnv*, jobject, jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return static_cast(opts->blob_garbage_collection_force_threshold); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setBlobCompactionReadaheadSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setBlobCompactionReadaheadSize( + JNIEnv*, jobject, jlong jhandle, jlong jblob_compaction_readahead_size) { + auto* opts = + reinterpret_cast(jhandle); + opts->blob_compaction_readahead_size = + static_cast(jblob_compaction_readahead_size); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: blobCompactionReadaheadSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_blobCompactionReadaheadSize( + JNIEnv*, jobject, jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return static_cast(opts->blob_compaction_readahead_size); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setBlobFileStartingLevel + * Signature: (JI)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setBlobFileStartingLevel( + JNIEnv*, jobject, jlong jhandle, jint jblob_file_starting_level) { + auto* opts = + reinterpret_cast(jhandle); + opts->blob_file_starting_level = jblob_file_starting_level; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: blobFileStartingLevel + * Signature: (J)I + */ +jint Java_org_rocksdb_ColumnFamilyOptions_blobFileStartingLevel(JNIEnv*, + jobject, + jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return static_cast(opts->blob_file_starting_level); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setPrepopulateBlobCache + * Signature: (JB)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setPrepopulateBlobCache( + JNIEnv*, jobject, jlong jhandle, jbyte jprepopulate_blob_cache_value) { + auto* opts = + reinterpret_cast(jhandle); + opts->prepopulate_blob_cache = + ROCKSDB_NAMESPACE::PrepopulateBlobCacheJni::toCppPrepopulateBlobCache( + jprepopulate_blob_cache_value); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: prepopulateBlobCache + * Signature: (J)B + */ +jbyte Java_org_rocksdb_ColumnFamilyOptions_prepopulateBlobCache(JNIEnv*, + jobject, + jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::PrepopulateBlobCacheJni::toJavaPrepopulateBlobCache( + opts->prepopulate_blob_cache); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setMemtableMaxRangeDeletions + * Signature: (JI)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setMemtableMaxRangeDeletions( + JNIEnv*, jobject, jlong jhandle, jint jmemtable_max_range_deletions) { + auto* opts = + reinterpret_cast(jhandle); + opts->memtable_max_range_deletions = jmemtable_max_range_deletions; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: memtableMaxRangeDeletions + * Signature: (J)I + */ +jint Java_org_rocksdb_ColumnFamilyOptions_memtableMaxRangeDeletions( + JNIEnv*, jobject, jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return static_cast(opts->memtable_max_range_deletions); +} + +///////////////////////////////////////////////////////////////////// +// ROCKSDB_NAMESPACE::DBOptions + +/* + * Class: org_rocksdb_DBOptions + * Method: newDBOptions + * Signature: ()J + */ +jlong Java_org_rocksdb_DBOptions_newDBOptions(JNIEnv*, jclass) { + auto* dbop = new ROCKSDB_NAMESPACE::DBOptions(); + return GET_CPLUSPLUS_POINTER(dbop); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: copyDBOptions + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_copyDBOptions(JNIEnv*, jclass, jlong jhandle) { + auto new_opt = new ROCKSDB_NAMESPACE::DBOptions( + *(reinterpret_cast(jhandle))); + return GET_CPLUSPLUS_POINTER(new_opt); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: newDBOptionsFromOptions + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_newDBOptionsFromOptions( + JNIEnv*, jclass, jlong joptions_handle) { + auto new_opt = new ROCKSDB_NAMESPACE::DBOptions( + *reinterpret_cast(joptions_handle)); + return GET_CPLUSPLUS_POINTER(new_opt); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: getDBOptionsFromProps + * Signature: (JLjava/lang/String;)J + */ +jlong Java_org_rocksdb_DBOptions_getDBOptionsFromProps__JLjava_lang_String_2( + JNIEnv* env, jclass, jlong config_handle, jstring jopt_string) { + const char* opt_string = env->GetStringUTFChars(jopt_string, nullptr); + if (opt_string == nullptr) { + // exception thrown: OutOfMemoryError + return 0; + } + + auto* config_options = + reinterpret_cast(config_handle); + auto* db_options = new ROCKSDB_NAMESPACE::DBOptions(); + ROCKSDB_NAMESPACE::Status status = ROCKSDB_NAMESPACE::GetDBOptionsFromString( + *config_options, ROCKSDB_NAMESPACE::DBOptions(), opt_string, db_options); + + env->ReleaseStringUTFChars(jopt_string, opt_string); + + // Check if DBOptions creation was possible. + jlong ret_value = 0; + if (status.ok()) { + ret_value = GET_CPLUSPLUS_POINTER(db_options); + } else { + // if operation failed the DBOptions need to be deleted + // again to prevent a memory leak. + delete db_options; + } + return ret_value; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: getDBOptionsFromProps + * Signature: (Ljava/util/String;)J + */ +jlong Java_org_rocksdb_DBOptions_getDBOptionsFromProps__Ljava_lang_String_2( + JNIEnv* env, jclass, jstring jopt_string) { + const char* opt_string = env->GetStringUTFChars(jopt_string, nullptr); + if (opt_string == nullptr) { + // exception thrown: OutOfMemoryError + return 0; + } + + const ROCKSDB_NAMESPACE::DBOptions base_options; + auto* db_options = new ROCKSDB_NAMESPACE::DBOptions(); + ROCKSDB_NAMESPACE::ConfigOptions config_options(base_options); + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + ROCKSDB_NAMESPACE::Status status = ROCKSDB_NAMESPACE::GetDBOptionsFromString( + config_options, base_options, opt_string, db_options); + + env->ReleaseStringUTFChars(jopt_string, opt_string); + + // Check if DBOptions creation was possible. + jlong ret_value = 0; + if (status.ok()) { + ret_value = GET_CPLUSPLUS_POINTER(db_options); + } else { + // if operation failed the DBOptions need to be deleted + // again to prevent a memory leak. + delete db_options; + } + return ret_value; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_DBOptions_disposeInternal(JNIEnv*, jobject, + jlong handle) { + auto* dbo = reinterpret_cast(handle); + assert(dbo != nullptr); + delete dbo; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: optimizeForSmallDb + * Signature: (J)V + */ +void Java_org_rocksdb_DBOptions_optimizeForSmallDb(JNIEnv*, jobject, + jlong jhandle) { + reinterpret_cast(jhandle) + ->OptimizeForSmallDb(); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setEnv + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setEnv(JNIEnv*, jobject, jlong jhandle, + jlong jenv_handle) { + reinterpret_cast(jhandle)->env = + reinterpret_cast(jenv_handle); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setIncreaseParallelism + * Signature: (JI)V + */ +void Java_org_rocksdb_DBOptions_setIncreaseParallelism(JNIEnv*, jobject, + jlong jhandle, + jint totalThreads) { + reinterpret_cast(jhandle)->IncreaseParallelism( + static_cast(totalThreads)); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setCreateIfMissing + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setCreateIfMissing(JNIEnv*, jobject, + jlong jhandle, + jboolean flag) { + reinterpret_cast(jhandle)->create_if_missing = + flag; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: createIfMissing + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_createIfMissing(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->create_if_missing; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setCreateMissingColumnFamilies + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setCreateMissingColumnFamilies(JNIEnv*, jobject, + jlong jhandle, + jboolean flag) { + reinterpret_cast(jhandle) + ->create_missing_column_families = flag; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: createMissingColumnFamilies + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_createMissingColumnFamilies(JNIEnv*, + jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->create_missing_column_families; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setErrorIfExists + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setErrorIfExists(JNIEnv*, jobject, + jlong jhandle, + jboolean error_if_exists) { + reinterpret_cast(jhandle)->error_if_exists = + static_cast(error_if_exists); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: errorIfExists + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_errorIfExists(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->error_if_exists; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setParanoidChecks + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setParanoidChecks(JNIEnv*, jobject, + jlong jhandle, + jboolean paranoid_checks) { + reinterpret_cast(jhandle)->paranoid_checks = + static_cast(paranoid_checks); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: paranoidChecks + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_paranoidChecks(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->paranoid_checks; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setRateLimiter + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setRateLimiter(JNIEnv*, jobject, jlong jhandle, + jlong jrate_limiter_handle) { + std::shared_ptr* pRateLimiter = + reinterpret_cast*>( + jrate_limiter_handle); + reinterpret_cast(jhandle)->rate_limiter = + *pRateLimiter; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setSstFileManager + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setSstFileManager( + JNIEnv*, jobject, jlong jhandle, jlong jsst_file_manager_handle) { + auto* sptr_sst_file_manager = + reinterpret_cast*>( + jsst_file_manager_handle); + reinterpret_cast(jhandle)->sst_file_manager = + *sptr_sst_file_manager; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setLogger + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setLogger(JNIEnv*, jobject, jlong jhandle, + jlong jlogger_handle) { + std::shared_ptr* pLogger = + reinterpret_cast*>( + jlogger_handle); + reinterpret_cast(jhandle)->info_log = *pLogger; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setInfoLogLevel + * Signature: (JB)V + */ +void Java_org_rocksdb_DBOptions_setInfoLogLevel(JNIEnv*, jobject, jlong jhandle, + jbyte jlog_level) { + reinterpret_cast(jhandle)->info_log_level = + static_cast(jlog_level); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: infoLogLevel + * Signature: (J)B + */ +jbyte Java_org_rocksdb_DBOptions_infoLogLevel(JNIEnv*, jobject, jlong jhandle) { + return static_cast( + reinterpret_cast(jhandle)->info_log_level); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setMaxTotalWalSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setMaxTotalWalSize(JNIEnv*, jobject, + jlong jhandle, + jlong jmax_total_wal_size) { + reinterpret_cast(jhandle)->max_total_wal_size = + static_cast(jmax_total_wal_size); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: maxTotalWalSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_maxTotalWalSize(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_total_wal_size; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setMaxOpenFiles + * Signature: (JI)V + */ +void Java_org_rocksdb_DBOptions_setMaxOpenFiles(JNIEnv*, jobject, jlong jhandle, + jint max_open_files) { + reinterpret_cast(jhandle)->max_open_files = + static_cast(max_open_files); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: maxOpenFiles + * Signature: (J)I + */ +jint Java_org_rocksdb_DBOptions_maxOpenFiles(JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_open_files; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setMaxFileOpeningThreads + * Signature: (JI)V + */ +void Java_org_rocksdb_DBOptions_setMaxFileOpeningThreads( + JNIEnv*, jobject, jlong jhandle, jint jmax_file_opening_threads) { + reinterpret_cast(jhandle) + ->max_file_opening_threads = static_cast(jmax_file_opening_threads); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: maxFileOpeningThreads + * Signature: (J)I + */ +jint Java_org_rocksdb_DBOptions_maxFileOpeningThreads(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->max_file_opening_threads); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setStatistics + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setStatistics(JNIEnv*, jobject, jlong jhandle, + jlong jstatistics_handle) { + auto* opt = reinterpret_cast(jhandle); + auto* pSptr = + reinterpret_cast*>( + jstatistics_handle); + opt->statistics = *pSptr; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: statistics + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_statistics(JNIEnv*, jobject, jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + std::shared_ptr sptr = opt->statistics; + if (sptr == nullptr) { + return 0; + } else { + std::shared_ptr* pSptr = + new std::shared_ptr(sptr); + return GET_CPLUSPLUS_POINTER(pSptr); + } +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setUseFsync + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setUseFsync(JNIEnv*, jobject, jlong jhandle, + jboolean use_fsync) { + reinterpret_cast(jhandle)->use_fsync = + static_cast(use_fsync); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: useFsync + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_useFsync(JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle)->use_fsync; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setDbPaths + * Signature: (J[Ljava/lang/String;[J)V + */ +void Java_org_rocksdb_DBOptions_setDbPaths(JNIEnv* env, jobject, jlong jhandle, + jobjectArray jpaths, + jlongArray jtarget_sizes) { + std::vector db_paths; + jlong* ptr_jtarget_size = env->GetLongArrayElements(jtarget_sizes, nullptr); + if (ptr_jtarget_size == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + jboolean has_exception = JNI_FALSE; + const jsize len = env->GetArrayLength(jpaths); + for (jsize i = 0; i < len; i++) { + jobject jpath = + reinterpret_cast(env->GetObjectArrayElement(jpaths, i)); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->ReleaseLongArrayElements(jtarget_sizes, ptr_jtarget_size, JNI_ABORT); + return; + } + std::string path = ROCKSDB_NAMESPACE::JniUtil::copyStdString( + env, static_cast(jpath), &has_exception); + env->DeleteLocalRef(jpath); + + if (has_exception == JNI_TRUE) { + env->ReleaseLongArrayElements(jtarget_sizes, ptr_jtarget_size, JNI_ABORT); + return; + } + + jlong jtarget_size = ptr_jtarget_size[i]; + + db_paths.push_back( + ROCKSDB_NAMESPACE::DbPath(path, static_cast(jtarget_size))); + } + + env->ReleaseLongArrayElements(jtarget_sizes, ptr_jtarget_size, JNI_ABORT); + + auto* opt = reinterpret_cast(jhandle); + opt->db_paths = db_paths; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: dbPathsLen + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_dbPathsLen(JNIEnv*, jobject, jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->db_paths.size()); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: dbPaths + * Signature: (J[Ljava/lang/String;[J)V + */ +void Java_org_rocksdb_DBOptions_dbPaths(JNIEnv* env, jobject, jlong jhandle, + jobjectArray jpaths, + jlongArray jtarget_sizes) { + jboolean is_copy; + jlong* ptr_jtarget_size = env->GetLongArrayElements(jtarget_sizes, &is_copy); + if (ptr_jtarget_size == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + auto* opt = reinterpret_cast(jhandle); + const jsize len = env->GetArrayLength(jpaths); + for (jsize i = 0; i < len; i++) { + ROCKSDB_NAMESPACE::DbPath db_path = opt->db_paths[i]; + + jstring jpath = env->NewStringUTF(db_path.path.c_str()); + if (jpath == nullptr) { + // exception thrown: OutOfMemoryError + env->ReleaseLongArrayElements(jtarget_sizes, ptr_jtarget_size, JNI_ABORT); + return; + } + env->SetObjectArrayElement(jpaths, i, jpath); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jpath); + env->ReleaseLongArrayElements(jtarget_sizes, ptr_jtarget_size, JNI_ABORT); + return; + } + + ptr_jtarget_size[i] = static_cast(db_path.target_size); + } + + env->ReleaseLongArrayElements(jtarget_sizes, ptr_jtarget_size, + is_copy == JNI_TRUE ? 0 : JNI_ABORT); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setDbLogDir + * Signature: (JLjava/lang/String)V + */ +void Java_org_rocksdb_DBOptions_setDbLogDir(JNIEnv* env, jobject, jlong jhandle, + jstring jdb_log_dir) { + const char* log_dir = env->GetStringUTFChars(jdb_log_dir, nullptr); + if (log_dir == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + reinterpret_cast(jhandle)->db_log_dir.assign( + log_dir); + env->ReleaseStringUTFChars(jdb_log_dir, log_dir); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: dbLogDir + * Signature: (J)Ljava/lang/String + */ +jstring Java_org_rocksdb_DBOptions_dbLogDir(JNIEnv* env, jobject, + jlong jhandle) { + return env->NewStringUTF( + reinterpret_cast(jhandle) + ->db_log_dir.c_str()); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setWalDir + * Signature: (JLjava/lang/String)V + */ +void Java_org_rocksdb_DBOptions_setWalDir(JNIEnv* env, jobject, jlong jhandle, + jstring jwal_dir) { + const char* wal_dir = env->GetStringUTFChars(jwal_dir, 0); + reinterpret_cast(jhandle)->wal_dir.assign( + wal_dir); + env->ReleaseStringUTFChars(jwal_dir, wal_dir); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: walDir + * Signature: (J)Ljava/lang/String + */ +jstring Java_org_rocksdb_DBOptions_walDir(JNIEnv* env, jobject, jlong jhandle) { + return env->NewStringUTF( + reinterpret_cast(jhandle) + ->wal_dir.c_str()); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setDeleteObsoleteFilesPeriodMicros + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setDeleteObsoleteFilesPeriodMicros( + JNIEnv*, jobject, jlong jhandle, jlong micros) { + reinterpret_cast(jhandle) + ->delete_obsolete_files_period_micros = static_cast(micros); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: deleteObsoleteFilesPeriodMicros + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_deleteObsoleteFilesPeriodMicros( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->delete_obsolete_files_period_micros; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setMaxBackgroundCompactions + * Signature: (JI)V + */ +void Java_org_rocksdb_DBOptions_setMaxBackgroundCompactions(JNIEnv*, jobject, + jlong jhandle, + jint max) { + reinterpret_cast(jhandle) + ->max_background_compactions = static_cast(max); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: maxBackgroundCompactions + * Signature: (J)I + */ +jint Java_org_rocksdb_DBOptions_maxBackgroundCompactions(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_background_compactions; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setMaxSubcompactions + * Signature: (JI)V + */ +void Java_org_rocksdb_DBOptions_setMaxSubcompactions(JNIEnv*, jobject, + jlong jhandle, jint max) { + reinterpret_cast(jhandle)->max_subcompactions = + static_cast(max); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: maxSubcompactions + * Signature: (J)I + */ +jint Java_org_rocksdb_DBOptions_maxSubcompactions(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_subcompactions; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setMaxBackgroundFlushes + * Signature: (JI)V + */ +void Java_org_rocksdb_DBOptions_setMaxBackgroundFlushes( + JNIEnv*, jobject, jlong jhandle, jint max_background_flushes) { + reinterpret_cast(jhandle) + ->max_background_flushes = static_cast(max_background_flushes); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: maxBackgroundFlushes + * Signature: (J)I + */ +jint Java_org_rocksdb_DBOptions_maxBackgroundFlushes(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_background_flushes; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setMaxBackgroundJobs + * Signature: (JI)V + */ +void Java_org_rocksdb_DBOptions_setMaxBackgroundJobs(JNIEnv*, jobject, + jlong jhandle, + jint max_background_jobs) { + reinterpret_cast(jhandle) + ->max_background_jobs = static_cast(max_background_jobs); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: maxBackgroundJobs + * Signature: (J)I + */ +jint Java_org_rocksdb_DBOptions_maxBackgroundJobs(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_background_jobs; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setMaxLogFileSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setMaxLogFileSize(JNIEnv* env, jobject, + jlong jhandle, + jlong max_log_file_size) { + auto s = + ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t(max_log_file_size); + if (s.ok()) { + reinterpret_cast(jhandle) + ->max_log_file_size = max_log_file_size; + } else { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_DBOptions + * Method: maxLogFileSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_maxLogFileSize(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_log_file_size; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setLogFileTimeToRoll + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setLogFileTimeToRoll( + JNIEnv* env, jobject, jlong jhandle, jlong log_file_time_to_roll) { + auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( + log_file_time_to_roll); + if (s.ok()) { + reinterpret_cast(jhandle) + ->log_file_time_to_roll = log_file_time_to_roll; + } else { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_DBOptions + * Method: logFileTimeToRoll + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_logFileTimeToRoll(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->log_file_time_to_roll; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setKeepLogFileNum + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setKeepLogFileNum(JNIEnv* env, jobject, + jlong jhandle, + jlong keep_log_file_num) { + auto s = + ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t(keep_log_file_num); + if (s.ok()) { + reinterpret_cast(jhandle) + ->keep_log_file_num = keep_log_file_num; + } else { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_DBOptions + * Method: keepLogFileNum + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_keepLogFileNum(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->keep_log_file_num; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setRecycleLogFileNum + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setRecycleLogFileNum( + JNIEnv* env, jobject, jlong jhandle, jlong recycle_log_file_num) { + auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( + recycle_log_file_num); + if (s.ok()) { + reinterpret_cast(jhandle) + ->recycle_log_file_num = recycle_log_file_num; + } else { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_DBOptions + * Method: recycleLogFileNum + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_recycleLogFileNum(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->recycle_log_file_num; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setMaxManifestFileSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setMaxManifestFileSize( + JNIEnv*, jobject, jlong jhandle, jlong max_manifest_file_size) { + reinterpret_cast(jhandle) + ->max_manifest_file_size = static_cast(max_manifest_file_size); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: maxManifestFileSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_maxManifestFileSize(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->max_manifest_file_size; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setTableCacheNumshardbits + * Signature: (JI)V + */ +void Java_org_rocksdb_DBOptions_setTableCacheNumshardbits( + JNIEnv*, jobject, jlong jhandle, jint table_cache_numshardbits) { + reinterpret_cast(jhandle) + ->table_cache_numshardbits = static_cast(table_cache_numshardbits); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: tableCacheNumshardbits + * Signature: (J)I + */ +jint Java_org_rocksdb_DBOptions_tableCacheNumshardbits(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->table_cache_numshardbits; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setWalTtlSeconds + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setWalTtlSeconds(JNIEnv*, jobject, + jlong jhandle, + jlong WAL_ttl_seconds) { + reinterpret_cast(jhandle)->WAL_ttl_seconds = + static_cast(WAL_ttl_seconds); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: walTtlSeconds + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_walTtlSeconds(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->WAL_ttl_seconds; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setWalSizeLimitMB + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setWalSizeLimitMB(JNIEnv*, jobject, + jlong jhandle, + jlong WAL_size_limit_MB) { + reinterpret_cast(jhandle)->WAL_size_limit_MB = + static_cast(WAL_size_limit_MB); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: walTtlSeconds + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_walSizeLimitMB(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->WAL_size_limit_MB; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setMaxWriteBatchGroupSizeBytes + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setMaxWriteBatchGroupSizeBytes( + JNIEnv*, jclass, jlong jhandle, jlong jmax_write_batch_group_size_bytes) { + auto* opt = reinterpret_cast(jhandle); + opt->max_write_batch_group_size_bytes = + static_cast(jmax_write_batch_group_size_bytes); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: maxWriteBatchGroupSizeBytes + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_maxWriteBatchGroupSizeBytes(JNIEnv*, jclass, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->max_write_batch_group_size_bytes); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setManifestPreallocationSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setManifestPreallocationSize( + JNIEnv* env, jobject, jlong jhandle, jlong preallocation_size) { + auto s = ROCKSDB_NAMESPACE::JniUtil::check_if_jlong_fits_size_t( + preallocation_size); + if (s.ok()) { + reinterpret_cast(jhandle) + ->manifest_preallocation_size = preallocation_size; + } else { + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_DBOptions + * Method: manifestPreallocationSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_manifestPreallocationSize(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->manifest_preallocation_size; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: useDirectReads + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_useDirectReads(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->use_direct_reads; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setUseDirectReads + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setUseDirectReads(JNIEnv*, jobject, + jlong jhandle, + jboolean use_direct_reads) { + reinterpret_cast(jhandle)->use_direct_reads = + static_cast(use_direct_reads); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: useDirectIoForFlushAndCompaction + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_useDirectIoForFlushAndCompaction( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->use_direct_io_for_flush_and_compaction; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setUseDirectReads + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setUseDirectIoForFlushAndCompaction( + JNIEnv*, jobject, jlong jhandle, + jboolean use_direct_io_for_flush_and_compaction) { + reinterpret_cast(jhandle) + ->use_direct_io_for_flush_and_compaction = + static_cast(use_direct_io_for_flush_and_compaction); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setAllowFAllocate + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setAllowFAllocate(JNIEnv*, jobject, + jlong jhandle, + jboolean jallow_fallocate) { + reinterpret_cast(jhandle)->allow_fallocate = + static_cast(jallow_fallocate); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: allowFAllocate + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_allowFAllocate(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->allow_fallocate); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setAllowMmapReads + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setAllowMmapReads(JNIEnv*, jobject, + jlong jhandle, + jboolean allow_mmap_reads) { + reinterpret_cast(jhandle)->allow_mmap_reads = + static_cast(allow_mmap_reads); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: allowMmapReads + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_allowMmapReads(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->allow_mmap_reads; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setAllowMmapWrites + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setAllowMmapWrites(JNIEnv*, jobject, + jlong jhandle, + jboolean allow_mmap_writes) { + reinterpret_cast(jhandle)->allow_mmap_writes = + static_cast(allow_mmap_writes); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: allowMmapWrites + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_allowMmapWrites(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->allow_mmap_writes; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setIsFdCloseOnExec + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setIsFdCloseOnExec( + JNIEnv*, jobject, jlong jhandle, jboolean is_fd_close_on_exec) { + reinterpret_cast(jhandle) + ->is_fd_close_on_exec = static_cast(is_fd_close_on_exec); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: isFdCloseOnExec + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_isFdCloseOnExec(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->is_fd_close_on_exec; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setStatsDumpPeriodSec + * Signature: (JI)V + */ +void Java_org_rocksdb_DBOptions_setStatsDumpPeriodSec( + JNIEnv*, jobject, jlong jhandle, jint jstats_dump_period_sec) { + reinterpret_cast(jhandle) + ->stats_dump_period_sec = + static_cast(jstats_dump_period_sec); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: statsDumpPeriodSec + * Signature: (J)I + */ +jint Java_org_rocksdb_DBOptions_statsDumpPeriodSec(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->stats_dump_period_sec; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setStatsPersistPeriodSec + * Signature: (JI)V + */ +void Java_org_rocksdb_DBOptions_setStatsPersistPeriodSec( + JNIEnv*, jobject, jlong jhandle, jint jstats_persist_period_sec) { + reinterpret_cast(jhandle) + ->stats_persist_period_sec = + static_cast(jstats_persist_period_sec); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: statsPersistPeriodSec + * Signature: (J)I + */ +jint Java_org_rocksdb_DBOptions_statsPersistPeriodSec(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->stats_persist_period_sec; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setStatsHistoryBufferSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setStatsHistoryBufferSize( + JNIEnv*, jobject, jlong jhandle, jlong jstats_history_buffer_size) { + reinterpret_cast(jhandle) + ->stats_history_buffer_size = + static_cast(jstats_history_buffer_size); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: statsHistoryBufferSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_statsHistoryBufferSize(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->stats_history_buffer_size; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setAdviseRandomOnOpen + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setAdviseRandomOnOpen( + JNIEnv*, jobject, jlong jhandle, jboolean advise_random_on_open) { + reinterpret_cast(jhandle) + ->advise_random_on_open = static_cast(advise_random_on_open); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: adviseRandomOnOpen + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_adviseRandomOnOpen(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->advise_random_on_open; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setDbWriteBufferSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setDbWriteBufferSize( + JNIEnv*, jobject, jlong jhandle, jlong jdb_write_buffer_size) { + auto* opt = reinterpret_cast(jhandle); + opt->db_write_buffer_size = static_cast(jdb_write_buffer_size); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setWriteBufferManager + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setWriteBufferManager( + JNIEnv*, jobject, jlong jdb_options_handle, + jlong jwrite_buffer_manager_handle) { + auto* write_buffer_manager = + reinterpret_cast*>( + jwrite_buffer_manager_handle); + reinterpret_cast(jdb_options_handle) + ->write_buffer_manager = *write_buffer_manager; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: dbWriteBufferSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_dbWriteBufferSize(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->db_write_buffer_size); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setAccessHintOnCompactionStart + * Signature: (JB)V + */ +void Java_org_rocksdb_DBOptions_setAccessHintOnCompactionStart( + JNIEnv*, jobject, jlong jhandle, jbyte jaccess_hint_value) { + auto* opt = reinterpret_cast(jhandle); + opt->access_hint_on_compaction_start = + ROCKSDB_NAMESPACE::AccessHintJni::toCppAccessHint(jaccess_hint_value); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: accessHintOnCompactionStart + * Signature: (J)B + */ +jbyte Java_org_rocksdb_DBOptions_accessHintOnCompactionStart(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::AccessHintJni::toJavaAccessHint( + opt->access_hint_on_compaction_start); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setCompactionReadaheadSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setCompactionReadaheadSize( + JNIEnv*, jobject, jlong jhandle, jlong jcompaction_readahead_size) { + auto* opt = reinterpret_cast(jhandle); + opt->compaction_readahead_size = + static_cast(jcompaction_readahead_size); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: compactionReadaheadSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_compactionReadaheadSize(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->compaction_readahead_size); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setRandomAccessMaxBufferSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setRandomAccessMaxBufferSize( + JNIEnv*, jobject, jlong jhandle, jlong jrandom_access_max_buffer_size) { + auto* opt = reinterpret_cast(jhandle); + opt->random_access_max_buffer_size = + static_cast(jrandom_access_max_buffer_size); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: randomAccessMaxBufferSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_randomAccessMaxBufferSize(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->random_access_max_buffer_size); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setWritableFileMaxBufferSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setWritableFileMaxBufferSize( + JNIEnv*, jobject, jlong jhandle, jlong jwritable_file_max_buffer_size) { + auto* opt = reinterpret_cast(jhandle); + opt->writable_file_max_buffer_size = + static_cast(jwritable_file_max_buffer_size); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: writableFileMaxBufferSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_writableFileMaxBufferSize(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->writable_file_max_buffer_size); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setUseAdaptiveMutex + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setUseAdaptiveMutex( + JNIEnv*, jobject, jlong jhandle, jboolean use_adaptive_mutex) { + reinterpret_cast(jhandle)->use_adaptive_mutex = + static_cast(use_adaptive_mutex); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: useAdaptiveMutex + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_useAdaptiveMutex(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->use_adaptive_mutex; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setBytesPerSync + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setBytesPerSync(JNIEnv*, jobject, jlong jhandle, + jlong bytes_per_sync) { + reinterpret_cast(jhandle)->bytes_per_sync = + static_cast(bytes_per_sync); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: bytesPerSync + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_bytesPerSync(JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->bytes_per_sync; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setWalBytesPerSync + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setWalBytesPerSync(JNIEnv*, jobject, + jlong jhandle, + jlong jwal_bytes_per_sync) { + reinterpret_cast(jhandle)->wal_bytes_per_sync = + static_cast(jwal_bytes_per_sync); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: walBytesPerSync + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_walBytesPerSync(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->wal_bytes_per_sync); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setStrictBytesPerSync + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setStrictBytesPerSync( + JNIEnv*, jobject, jlong jhandle, jboolean jstrict_bytes_per_sync) { + reinterpret_cast(jhandle) + ->strict_bytes_per_sync = jstrict_bytes_per_sync == JNI_TRUE; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: strictBytesPerSync + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_strictBytesPerSync(JNIEnv*, jobject, + jlong jhandle) { + return static_cast( + reinterpret_cast(jhandle) + ->strict_bytes_per_sync); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setEventListeners + * Signature: (J[J)V + */ +void Java_org_rocksdb_DBOptions_setEventListeners(JNIEnv* env, jclass, + jlong jhandle, + jlongArray jlistener_array) { + auto* opt = reinterpret_cast(jhandle); + rocksdb_set_event_listeners_helper(env, jlistener_array, opt->listeners); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: eventListeners + * Signature: (J)[Lorg/rocksdb/AbstractEventListener; + */ +jobjectArray Java_org_rocksdb_DBOptions_eventListeners(JNIEnv* env, jclass, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return rocksdb_get_event_listeners_helper(env, opt->listeners); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setDelayedWriteRate + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setDelayedWriteRate(JNIEnv*, jobject, + jlong jhandle, + jlong jdelayed_write_rate) { + auto* opt = reinterpret_cast(jhandle); + opt->delayed_write_rate = static_cast(jdelayed_write_rate); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: delayedWriteRate + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_delayedWriteRate(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->delayed_write_rate); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setEnablePipelinedWrite + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setEnablePipelinedWrite( + JNIEnv*, jobject, jlong jhandle, jboolean jenable_pipelined_write) { + auto* opt = reinterpret_cast(jhandle); + opt->enable_pipelined_write = jenable_pipelined_write == JNI_TRUE; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: enablePipelinedWrite + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_enablePipelinedWrite(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->enable_pipelined_write); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setUnorderedWrite + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setUnorderedWrite(JNIEnv*, jobject, + jlong jhandle, + jboolean junordered_write) { + auto* opt = reinterpret_cast(jhandle); + opt->unordered_write = junordered_write == JNI_TRUE; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: unorderedWrite + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_unorderedWrite(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->unordered_write); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setEnableThreadTracking + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setEnableThreadTracking( + JNIEnv*, jobject, jlong jhandle, jboolean jenable_thread_tracking) { + auto* opt = reinterpret_cast(jhandle); + opt->enable_thread_tracking = jenable_thread_tracking == JNI_TRUE; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: enableThreadTracking + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_enableThreadTracking(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->enable_thread_tracking); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setAllowConcurrentMemtableWrite + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setAllowConcurrentMemtableWrite( + JNIEnv*, jobject, jlong jhandle, jboolean allow) { + reinterpret_cast(jhandle) + ->allow_concurrent_memtable_write = static_cast(allow); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: allowConcurrentMemtableWrite + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_allowConcurrentMemtableWrite( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->allow_concurrent_memtable_write; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setEnableWriteThreadAdaptiveYield + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setEnableWriteThreadAdaptiveYield( + JNIEnv*, jobject, jlong jhandle, jboolean yield) { + reinterpret_cast(jhandle) + ->enable_write_thread_adaptive_yield = static_cast(yield); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: enableWriteThreadAdaptiveYield + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_enableWriteThreadAdaptiveYield( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->enable_write_thread_adaptive_yield; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setWriteThreadMaxYieldUsec + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setWriteThreadMaxYieldUsec(JNIEnv*, jobject, + jlong jhandle, + jlong max) { + reinterpret_cast(jhandle) + ->write_thread_max_yield_usec = static_cast(max); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: writeThreadMaxYieldUsec + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_writeThreadMaxYieldUsec(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->write_thread_max_yield_usec; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setWriteThreadSlowYieldUsec + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setWriteThreadSlowYieldUsec(JNIEnv*, jobject, + jlong jhandle, + jlong slow) { + reinterpret_cast(jhandle) + ->write_thread_slow_yield_usec = static_cast(slow); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: writeThreadSlowYieldUsec + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_writeThreadSlowYieldUsec(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->write_thread_slow_yield_usec; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setSkipStatsUpdateOnDbOpen + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setSkipStatsUpdateOnDbOpen( + JNIEnv*, jobject, jlong jhandle, jboolean jskip_stats_update_on_db_open) { + auto* opt = reinterpret_cast(jhandle); + opt->skip_stats_update_on_db_open = + static_cast(jskip_stats_update_on_db_open); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: skipStatsUpdateOnDbOpen + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_skipStatsUpdateOnDbOpen(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->skip_stats_update_on_db_open); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setSkipCheckingSstFileSizesOnDbOpen + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setSkipCheckingSstFileSizesOnDbOpen( + JNIEnv*, jclass, jlong jhandle, + jboolean jskip_checking_sst_file_sizes_on_db_open) { + auto* opt = reinterpret_cast(jhandle); + opt->skip_checking_sst_file_sizes_on_db_open = + static_cast(jskip_checking_sst_file_sizes_on_db_open); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: skipCheckingSstFileSizesOnDbOpen + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_skipCheckingSstFileSizesOnDbOpen( + JNIEnv*, jclass, jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->skip_checking_sst_file_sizes_on_db_open); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setWalRecoveryMode + * Signature: (JB)V + */ +void Java_org_rocksdb_DBOptions_setWalRecoveryMode( + JNIEnv*, jobject, jlong jhandle, jbyte jwal_recovery_mode_value) { + auto* opt = reinterpret_cast(jhandle); + opt->wal_recovery_mode = + ROCKSDB_NAMESPACE::WALRecoveryModeJni::toCppWALRecoveryMode( + jwal_recovery_mode_value); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: walRecoveryMode + * Signature: (J)B + */ +jbyte Java_org_rocksdb_DBOptions_walRecoveryMode(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::WALRecoveryModeJni::toJavaWALRecoveryMode( + opt->wal_recovery_mode); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setAllow2pc + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setAllow2pc(JNIEnv*, jobject, jlong jhandle, + jboolean jallow_2pc) { + auto* opt = reinterpret_cast(jhandle); + opt->allow_2pc = static_cast(jallow_2pc); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: allow2pc + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_allow2pc(JNIEnv*, jobject, jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->allow_2pc); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setRowCache + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setRowCache(JNIEnv*, jobject, jlong jhandle, + jlong jrow_cache_handle) { + auto* opt = reinterpret_cast(jhandle); + auto* row_cache = + reinterpret_cast*>( + jrow_cache_handle); + opt->row_cache = *row_cache; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setWalFilter + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setWalFilter(JNIEnv*, jobject, jlong jhandle, + jlong jwal_filter_handle) { + auto* opt = reinterpret_cast(jhandle); + auto* wal_filter = reinterpret_cast( + jwal_filter_handle); + opt->wal_filter = wal_filter; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setFailIfOptionsFileError + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setFailIfOptionsFileError( + JNIEnv*, jobject, jlong jhandle, jboolean jfail_if_options_file_error) { + auto* opt = reinterpret_cast(jhandle); + opt->fail_if_options_file_error = + static_cast(jfail_if_options_file_error); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: failIfOptionsFileError + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_failIfOptionsFileError(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->fail_if_options_file_error); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setDumpMallocStats + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setDumpMallocStats( + JNIEnv*, jobject, jlong jhandle, jboolean jdump_malloc_stats) { + auto* opt = reinterpret_cast(jhandle); + opt->dump_malloc_stats = static_cast(jdump_malloc_stats); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: dumpMallocStats + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_dumpMallocStats(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->dump_malloc_stats); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setAvoidFlushDuringRecovery + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setAvoidFlushDuringRecovery( + JNIEnv*, jobject, jlong jhandle, jboolean javoid_flush_during_recovery) { + auto* opt = reinterpret_cast(jhandle); + opt->avoid_flush_during_recovery = + static_cast(javoid_flush_during_recovery); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: avoidFlushDuringRecovery + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_avoidFlushDuringRecovery(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->avoid_flush_during_recovery); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setAllowIngestBehind + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setAllowIngestBehind( + JNIEnv*, jobject, jlong jhandle, jboolean jallow_ingest_behind) { + auto* opt = reinterpret_cast(jhandle); + opt->allow_ingest_behind = jallow_ingest_behind == JNI_TRUE; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: allowIngestBehind + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_allowIngestBehind(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->allow_ingest_behind); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setTwoWriteQueues + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setTwoWriteQueues(JNIEnv*, jobject, + jlong jhandle, + jboolean jtwo_write_queues) { + auto* opt = reinterpret_cast(jhandle); + opt->two_write_queues = jtwo_write_queues == JNI_TRUE; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: twoWriteQueues + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_twoWriteQueues(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->two_write_queues); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setManualWalFlush + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setManualWalFlush(JNIEnv*, jobject, + jlong jhandle, + jboolean jmanual_wal_flush) { + auto* opt = reinterpret_cast(jhandle); + opt->manual_wal_flush = jmanual_wal_flush == JNI_TRUE; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: manualWalFlush + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_manualWalFlush(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->manual_wal_flush); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setAtomicFlush + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setAtomicFlush(JNIEnv*, jobject, jlong jhandle, + jboolean jatomic_flush) { + auto* opt = reinterpret_cast(jhandle); + opt->atomic_flush = jatomic_flush == JNI_TRUE; +} + +/* + * Class: org_rocksdb_DBOptions + * Method: atomicFlush + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_atomicFlush(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->atomic_flush); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setAvoidFlushDuringShutdown + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setAvoidFlushDuringShutdown( + JNIEnv*, jobject, jlong jhandle, jboolean javoid_flush_during_shutdown) { + auto* opt = reinterpret_cast(jhandle); + opt->avoid_flush_during_shutdown = + static_cast(javoid_flush_during_shutdown); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: avoidFlushDuringShutdown + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_avoidFlushDuringShutdown(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->avoid_flush_during_shutdown); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setAvoidUnnecessaryBlockingIO + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setAvoidUnnecessaryBlockingIO( + JNIEnv*, jclass, jlong jhandle, jboolean avoid_blocking_io) { + auto* opt = reinterpret_cast(jhandle); + opt->avoid_unnecessary_blocking_io = static_cast(avoid_blocking_io); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: avoidUnnecessaryBlockingIO + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_avoidUnnecessaryBlockingIO(JNIEnv*, jclass, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->avoid_unnecessary_blocking_io); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setPersistStatsToDisk + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setPersistStatsToDisk( + JNIEnv*, jclass, jlong jhandle, jboolean persist_stats_to_disk) { + auto* opt = reinterpret_cast(jhandle); + opt->persist_stats_to_disk = static_cast(persist_stats_to_disk); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: persistStatsToDisk + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_persistStatsToDisk(JNIEnv*, jclass, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->persist_stats_to_disk); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setWriteDbidToManifest + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setWriteDbidToManifest( + JNIEnv*, jclass, jlong jhandle, jboolean jwrite_dbid_to_manifest) { + auto* opt = reinterpret_cast(jhandle); + opt->write_dbid_to_manifest = static_cast(jwrite_dbid_to_manifest); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: writeDbidToManifest + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_writeDbidToManifest(JNIEnv*, jclass, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->write_dbid_to_manifest); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setLogReadaheadSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setLogReadaheadSize(JNIEnv*, jclass, + jlong jhandle, + jlong jlog_readahead_size) { + auto* opt = reinterpret_cast(jhandle); + opt->log_readahead_size = static_cast(jlog_readahead_size); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: logReasaheadSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_logReadaheadSize(JNIEnv*, jclass, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->log_readahead_size); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setBestEffortsRecovery + * Signature: (JZ)V + */ +void Java_org_rocksdb_DBOptions_setBestEffortsRecovery( + JNIEnv*, jclass, jlong jhandle, jboolean jbest_efforts_recovery) { + auto* opt = reinterpret_cast(jhandle); + opt->best_efforts_recovery = static_cast(jbest_efforts_recovery); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: bestEffortsRecovery + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_DBOptions_bestEffortsRecovery(JNIEnv*, jclass, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->best_efforts_recovery); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setMaxBgErrorResumeCount + * Signature: (JI)V + */ +void Java_org_rocksdb_DBOptions_setMaxBgErrorResumeCount( + JNIEnv*, jclass, jlong jhandle, jint jmax_bgerror_resume_count) { + auto* opt = reinterpret_cast(jhandle); + opt->max_bgerror_resume_count = static_cast(jmax_bgerror_resume_count); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: maxBgerrorResumeCount + * Signature: (J)I + */ +jint Java_org_rocksdb_DBOptions_maxBgerrorResumeCount(JNIEnv*, jclass, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->max_bgerror_resume_count); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: setBgerrorResumeRetryInterval + * Signature: (JJ)V + */ +void Java_org_rocksdb_DBOptions_setBgerrorResumeRetryInterval( + JNIEnv*, jclass, jlong jhandle, jlong jbgerror_resume_retry_interval) { + auto* opt = reinterpret_cast(jhandle); + opt->bgerror_resume_retry_interval = + static_cast(jbgerror_resume_retry_interval); +} + +/* + * Class: org_rocksdb_DBOptions + * Method: bgerrorResumeRetryInterval + * Signature: (J)J + */ +jlong Java_org_rocksdb_DBOptions_bgerrorResumeRetryInterval(JNIEnv*, jclass, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->bgerror_resume_retry_interval); +} + +////////////////////////////////////////////////////////////////////////////// +// ROCKSDB_NAMESPACE::WriteOptions + +/* + * Class: org_rocksdb_WriteOptions + * Method: newWriteOptions + * Signature: ()J + */ +jlong Java_org_rocksdb_WriteOptions_newWriteOptions(JNIEnv*, jclass) { + auto* op = new ROCKSDB_NAMESPACE::WriteOptions(); + return GET_CPLUSPLUS_POINTER(op); +} + +/* + * Class: org_rocksdb_WriteOptions + * Method: copyWriteOptions + * Signature: (J)J + */ +jlong Java_org_rocksdb_WriteOptions_copyWriteOptions(JNIEnv*, jclass, + jlong jhandle) { + auto new_opt = new ROCKSDB_NAMESPACE::WriteOptions( + *(reinterpret_cast(jhandle))); + return GET_CPLUSPLUS_POINTER(new_opt); +} + +/* + * Class: org_rocksdb_WriteOptions + * Method: disposeInternal + * Signature: ()V + */ +void Java_org_rocksdb_WriteOptions_disposeInternal(JNIEnv*, jobject, + jlong jhandle) { + auto* write_options = + reinterpret_cast(jhandle); + assert(write_options != nullptr); + delete write_options; +} + +/* + * Class: org_rocksdb_WriteOptions + * Method: setSync + * Signature: (JZ)V + */ +void Java_org_rocksdb_WriteOptions_setSync(JNIEnv*, jobject, jlong jhandle, + jboolean jflag) { + reinterpret_cast(jhandle)->sync = jflag; +} + +/* + * Class: org_rocksdb_WriteOptions + * Method: sync + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_WriteOptions_sync(JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle)->sync; +} + +/* + * Class: org_rocksdb_WriteOptions + * Method: setDisableWAL + * Signature: (JZ)V + */ +void Java_org_rocksdb_WriteOptions_setDisableWAL(JNIEnv*, jobject, + jlong jhandle, + jboolean jflag) { + reinterpret_cast(jhandle)->disableWAL = + jflag; +} + +/* + * Class: org_rocksdb_WriteOptions + * Method: disableWAL + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_WriteOptions_disableWAL(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->disableWAL; +} + +/* + * Class: org_rocksdb_WriteOptions + * Method: setIgnoreMissingColumnFamilies + * Signature: (JZ)V + */ +void Java_org_rocksdb_WriteOptions_setIgnoreMissingColumnFamilies( + JNIEnv*, jobject, jlong jhandle, jboolean jignore_missing_column_families) { + reinterpret_cast(jhandle) + ->ignore_missing_column_families = + static_cast(jignore_missing_column_families); +} + +/* + * Class: org_rocksdb_WriteOptions + * Method: ignoreMissingColumnFamilies + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_WriteOptions_ignoreMissingColumnFamilies( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->ignore_missing_column_families; +} + +/* + * Class: org_rocksdb_WriteOptions + * Method: setNoSlowdown + * Signature: (JZ)V + */ +void Java_org_rocksdb_WriteOptions_setNoSlowdown(JNIEnv*, jobject, + jlong jhandle, + jboolean jno_slowdown) { + reinterpret_cast(jhandle)->no_slowdown = + static_cast(jno_slowdown); +} + +/* + * Class: org_rocksdb_WriteOptions + * Method: noSlowdown + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_WriteOptions_noSlowdown(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->no_slowdown; +} + +/* + * Class: org_rocksdb_WriteOptions + * Method: setLowPri + * Signature: (JZ)V + */ +void Java_org_rocksdb_WriteOptions_setLowPri(JNIEnv*, jobject, jlong jhandle, + jboolean jlow_pri) { + reinterpret_cast(jhandle)->low_pri = + static_cast(jlow_pri); +} + +/* + * Class: org_rocksdb_WriteOptions + * Method: lowPri + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_WriteOptions_lowPri(JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle)->low_pri; +} + +/* + * Class: org_rocksdb_WriteOptions + * Method: memtableInsertHintPerBatch + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_WriteOptions_memtableInsertHintPerBatch( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->memtable_insert_hint_per_batch; +} + +/* + * Class: org_rocksdb_WriteOptions + * Method: setMemtableInsertHintPerBatch + * Signature: (JZ)V + */ +void Java_org_rocksdb_WriteOptions_setMemtableInsertHintPerBatch( + JNIEnv*, jobject, jlong jhandle, jboolean jmemtable_insert_hint_per_batch) { + reinterpret_cast(jhandle) + ->memtable_insert_hint_per_batch = + static_cast(jmemtable_insert_hint_per_batch); +} + +///////////////////////////////////////////////////////////////////// +// ROCKSDB_NAMESPACE::ReadOptions + +/* + * Class: org_rocksdb_ReadOptions + * Method: newReadOptions + * Signature: ()J + */ +jlong Java_org_rocksdb_ReadOptions_newReadOptions__(JNIEnv*, jclass) { + auto* read_options = new ROCKSDB_NAMESPACE::ReadOptions(); + return GET_CPLUSPLUS_POINTER(read_options); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: newReadOptions + * Signature: (ZZ)J + */ +jlong Java_org_rocksdb_ReadOptions_newReadOptions__ZZ( + JNIEnv*, jclass, jboolean jverify_checksums, jboolean jfill_cache) { + auto* read_options = new ROCKSDB_NAMESPACE::ReadOptions( + static_cast(jverify_checksums), static_cast(jfill_cache)); + return GET_CPLUSPLUS_POINTER(read_options); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: copyReadOptions + * Signature: (J)J + */ +jlong Java_org_rocksdb_ReadOptions_copyReadOptions(JNIEnv*, jclass, + jlong jhandle) { + auto new_opt = new ROCKSDB_NAMESPACE::ReadOptions( + *(reinterpret_cast(jhandle))); + return GET_CPLUSPLUS_POINTER(new_opt); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_ReadOptions_disposeInternal(JNIEnv*, jobject, + jlong jhandle) { + auto* read_options = + reinterpret_cast(jhandle); + assert(read_options != nullptr); + delete read_options; +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: setVerifyChecksums + * Signature: (JZ)V + */ +void Java_org_rocksdb_ReadOptions_setVerifyChecksums( + JNIEnv*, jobject, jlong jhandle, jboolean jverify_checksums) { + reinterpret_cast(jhandle)->verify_checksums = + static_cast(jverify_checksums); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: verifyChecksums + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ReadOptions_verifyChecksums(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->verify_checksums; +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: setFillCache + * Signature: (JZ)V + */ +void Java_org_rocksdb_ReadOptions_setFillCache(JNIEnv*, jobject, jlong jhandle, + jboolean jfill_cache) { + reinterpret_cast(jhandle)->fill_cache = + static_cast(jfill_cache); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: fillCache + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ReadOptions_fillCache(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle)->fill_cache; +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: setTailing + * Signature: (JZ)V + */ +void Java_org_rocksdb_ReadOptions_setTailing(JNIEnv*, jobject, jlong jhandle, + jboolean jtailing) { + reinterpret_cast(jhandle)->tailing = + static_cast(jtailing); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: tailing + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ReadOptions_tailing(JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle)->tailing; +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: managed + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ReadOptions_managed(JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle)->managed; +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: setManaged + * Signature: (JZ)V + */ +void Java_org_rocksdb_ReadOptions_setManaged(JNIEnv*, jobject, jlong jhandle, + jboolean jmanaged) { + reinterpret_cast(jhandle)->managed = + static_cast(jmanaged); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: totalOrderSeek + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ReadOptions_totalOrderSeek(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->total_order_seek; +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: setTotalOrderSeek + * Signature: (JZ)V + */ +void Java_org_rocksdb_ReadOptions_setTotalOrderSeek( + JNIEnv*, jobject, jlong jhandle, jboolean jtotal_order_seek) { + reinterpret_cast(jhandle)->total_order_seek = + static_cast(jtotal_order_seek); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: prefixSameAsStart + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ReadOptions_prefixSameAsStart(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->prefix_same_as_start; +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: setPrefixSameAsStart + * Signature: (JZ)V + */ +void Java_org_rocksdb_ReadOptions_setPrefixSameAsStart( + JNIEnv*, jobject, jlong jhandle, jboolean jprefix_same_as_start) { + reinterpret_cast(jhandle) + ->prefix_same_as_start = static_cast(jprefix_same_as_start); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: pinData + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ReadOptions_pinData(JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle)->pin_data; +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: setPinData + * Signature: (JZ)V + */ +void Java_org_rocksdb_ReadOptions_setPinData(JNIEnv*, jobject, jlong jhandle, + jboolean jpin_data) { + reinterpret_cast(jhandle)->pin_data = + static_cast(jpin_data); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: backgroundPurgeOnIteratorCleanup + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ReadOptions_backgroundPurgeOnIteratorCleanup( + JNIEnv*, jobject, jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->background_purge_on_iterator_cleanup); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: setBackgroundPurgeOnIteratorCleanup + * Signature: (JZ)V + */ +void Java_org_rocksdb_ReadOptions_setBackgroundPurgeOnIteratorCleanup( + JNIEnv*, jobject, jlong jhandle, + jboolean jbackground_purge_on_iterator_cleanup) { + auto* opt = reinterpret_cast(jhandle); + opt->background_purge_on_iterator_cleanup = + static_cast(jbackground_purge_on_iterator_cleanup); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: readaheadSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_ReadOptions_readaheadSize(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->readahead_size); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: setReadaheadSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_ReadOptions_setReadaheadSize(JNIEnv*, jobject, + jlong jhandle, + jlong jreadahead_size) { + auto* opt = reinterpret_cast(jhandle); + opt->readahead_size = static_cast(jreadahead_size); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: maxSkippableInternalKeys + * Signature: (J)J + */ +jlong Java_org_rocksdb_ReadOptions_maxSkippableInternalKeys(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->max_skippable_internal_keys); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: setMaxSkippableInternalKeys + * Signature: (JJ)V + */ +void Java_org_rocksdb_ReadOptions_setMaxSkippableInternalKeys( + JNIEnv*, jobject, jlong jhandle, jlong jmax_skippable_internal_keys) { + auto* opt = reinterpret_cast(jhandle); + opt->max_skippable_internal_keys = + static_cast(jmax_skippable_internal_keys); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: ignoreRangeDeletions + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ReadOptions_ignoreRangeDeletions(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->ignore_range_deletions); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: setIgnoreRangeDeletions + * Signature: (JZ)V + */ +void Java_org_rocksdb_ReadOptions_setIgnoreRangeDeletions( + JNIEnv*, jobject, jlong jhandle, jboolean jignore_range_deletions) { + auto* opt = reinterpret_cast(jhandle); + opt->ignore_range_deletions = static_cast(jignore_range_deletions); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: setSnapshot + * Signature: (JJ)V + */ +void Java_org_rocksdb_ReadOptions_setSnapshot(JNIEnv*, jobject, jlong jhandle, + jlong jsnapshot) { + reinterpret_cast(jhandle)->snapshot = + reinterpret_cast(jsnapshot); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: snapshot + * Signature: (J)J + */ +jlong Java_org_rocksdb_ReadOptions_snapshot(JNIEnv*, jobject, jlong jhandle) { + auto& snapshot = + reinterpret_cast(jhandle)->snapshot; + return GET_CPLUSPLUS_POINTER(snapshot); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: readTier + * Signature: (J)B + */ +jbyte Java_org_rocksdb_ReadOptions_readTier(JNIEnv*, jobject, jlong jhandle) { + return static_cast( + reinterpret_cast(jhandle)->read_tier); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: setReadTier + * Signature: (JB)V + */ +void Java_org_rocksdb_ReadOptions_setReadTier(JNIEnv*, jobject, jlong jhandle, + jbyte jread_tier) { + reinterpret_cast(jhandle)->read_tier = + static_cast(jread_tier); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: setIterateUpperBound + * Signature: (JJ)I + */ +void Java_org_rocksdb_ReadOptions_setIterateUpperBound( + JNIEnv*, jobject, jlong jhandle, jlong jupper_bound_slice_handle) { + reinterpret_cast(jhandle) + ->iterate_upper_bound = + reinterpret_cast(jupper_bound_slice_handle); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: iterateUpperBound + * Signature: (J)J + */ +jlong Java_org_rocksdb_ReadOptions_iterateUpperBound(JNIEnv*, jobject, + jlong jhandle) { + auto& upper_bound_slice_handle = + reinterpret_cast(jhandle) + ->iterate_upper_bound; + return GET_CPLUSPLUS_POINTER(upper_bound_slice_handle); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: setIterateLowerBound + * Signature: (JJ)I + */ +void Java_org_rocksdb_ReadOptions_setIterateLowerBound( + JNIEnv*, jobject, jlong jhandle, jlong jlower_bound_slice_handle) { + reinterpret_cast(jhandle) + ->iterate_lower_bound = + reinterpret_cast(jlower_bound_slice_handle); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: iterateLowerBound + * Signature: (J)J + */ +jlong Java_org_rocksdb_ReadOptions_iterateLowerBound(JNIEnv*, jobject, + jlong jhandle) { + auto& lower_bound_slice_handle = + reinterpret_cast(jhandle) + ->iterate_lower_bound; + return GET_CPLUSPLUS_POINTER(lower_bound_slice_handle); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: setTableFilter + * Signature: (JJ)V + */ +void Java_org_rocksdb_ReadOptions_setTableFilter( + JNIEnv*, jobject, jlong jhandle, jlong jjni_table_filter_handle) { + auto* opt = reinterpret_cast(jhandle); + auto* jni_table_filter = + reinterpret_cast( + jjni_table_filter_handle); + opt->table_filter = jni_table_filter->GetTableFilterFunction(); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: autoPrefixMode + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ReadOptions_autoPrefixMode(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->auto_prefix_mode); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: setAutoPrefixMode + * Signature: (JZ)V + */ +void Java_org_rocksdb_ReadOptions_setAutoPrefixMode( + JNIEnv*, jobject, jlong jhandle, jboolean jauto_prefix_mode) { + auto* opt = reinterpret_cast(jhandle); + opt->auto_prefix_mode = static_cast(jauto_prefix_mode); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: timestamp + * Signature: (J)J + */ +jlong Java_org_rocksdb_ReadOptions_timestamp(JNIEnv*, jobject, jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + auto& timestamp_slice_handle = opt->timestamp; + return reinterpret_cast(timestamp_slice_handle); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: setTimestamp + * Signature: (JJ)V + */ +void Java_org_rocksdb_ReadOptions_setTimestamp(JNIEnv*, jobject, jlong jhandle, + jlong jtimestamp_slice_handle) { + auto* opt = reinterpret_cast(jhandle); + opt->timestamp = + reinterpret_cast(jtimestamp_slice_handle); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: iterStartTs + * Signature: (J)J + */ +jlong Java_org_rocksdb_ReadOptions_iterStartTs(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + auto& iter_start_ts_handle = opt->iter_start_ts; + return reinterpret_cast(iter_start_ts_handle); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: setIterStartTs + * Signature: (JJ)V + */ +void Java_org_rocksdb_ReadOptions_setIterStartTs(JNIEnv*, jobject, + jlong jhandle, + jlong jiter_start_ts_handle) { + auto* opt = reinterpret_cast(jhandle); + opt->iter_start_ts = + reinterpret_cast(jiter_start_ts_handle); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: deadline + * Signature: (J)J + */ +jlong Java_org_rocksdb_ReadOptions_deadline(JNIEnv*, jobject, jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->deadline.count()); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: setDeadline + * Signature: (JJ)V + */ +void Java_org_rocksdb_ReadOptions_setDeadline(JNIEnv*, jobject, jlong jhandle, + jlong jdeadline) { + auto* opt = reinterpret_cast(jhandle); + opt->deadline = std::chrono::microseconds(static_cast(jdeadline)); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: ioTimeout + * Signature: (J)J + */ +jlong Java_org_rocksdb_ReadOptions_ioTimeout(JNIEnv*, jobject, jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->io_timeout.count()); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: setIoTimeout + * Signature: (JJ)V + */ +void Java_org_rocksdb_ReadOptions_setIoTimeout(JNIEnv*, jobject, jlong jhandle, + jlong jio_timeout) { + auto* opt = reinterpret_cast(jhandle); + opt->io_timeout = + std::chrono::microseconds(static_cast(jio_timeout)); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: valueSizeSofLimit + * Signature: (J)J + */ +jlong Java_org_rocksdb_ReadOptions_valueSizeSoftLimit(JNIEnv*, jobject, + jlong jhandle) { + auto* opt = reinterpret_cast(jhandle); + return static_cast(opt->value_size_soft_limit); +} + +/* + * Class: org_rocksdb_ReadOptions + * Method: setValueSizeSofLimit + * Signature: (JJ)V + */ +void Java_org_rocksdb_ReadOptions_setValueSizeSoftLimit( + JNIEnv*, jobject, jlong jhandle, jlong jvalue_size_soft_limit) { + auto* opt = reinterpret_cast(jhandle); + opt->value_size_soft_limit = static_cast(jvalue_size_soft_limit); +} + +///////////////////////////////////////////////////////////////////// +// ROCKSDB_NAMESPACE::ComparatorOptions + +/* + * Class: org_rocksdb_ComparatorOptions + * Method: newComparatorOptions + * Signature: ()J + */ +jlong Java_org_rocksdb_ComparatorOptions_newComparatorOptions(JNIEnv*, jclass) { + auto* comparator_opt = new ROCKSDB_NAMESPACE::ComparatorJniCallbackOptions(); + return GET_CPLUSPLUS_POINTER(comparator_opt); +} + +/* + * Class: org_rocksdb_ComparatorOptions + * Method: reusedSynchronisationType + * Signature: (J)B + */ +jbyte Java_org_rocksdb_ComparatorOptions_reusedSynchronisationType( + JNIEnv*, jobject, jlong jhandle) { + auto* comparator_opt = + reinterpret_cast( + jhandle); + return ROCKSDB_NAMESPACE::ReusedSynchronisationTypeJni:: + toJavaReusedSynchronisationType( + comparator_opt->reused_synchronisation_type); +} + +/* + * Class: org_rocksdb_ComparatorOptions + * Method: setReusedSynchronisationType + * Signature: (JB)V + */ +void Java_org_rocksdb_ComparatorOptions_setReusedSynchronisationType( + JNIEnv*, jobject, jlong jhandle, jbyte jreused_synhcronisation_type) { + auto* comparator_opt = + reinterpret_cast( + jhandle); + comparator_opt->reused_synchronisation_type = + ROCKSDB_NAMESPACE::ReusedSynchronisationTypeJni:: + toCppReusedSynchronisationType(jreused_synhcronisation_type); +} + +/* + * Class: org_rocksdb_ComparatorOptions + * Method: useDirectBuffer + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ComparatorOptions_useDirectBuffer(JNIEnv*, jobject, + jlong jhandle) { + return static_cast( + reinterpret_cast( + jhandle) + ->direct_buffer); +} + +/* + * Class: org_rocksdb_ComparatorOptions + * Method: setUseDirectBuffer + * Signature: (JZ)V + */ +void Java_org_rocksdb_ComparatorOptions_setUseDirectBuffer( + JNIEnv*, jobject, jlong jhandle, jboolean jdirect_buffer) { + reinterpret_cast(jhandle) + ->direct_buffer = jdirect_buffer == JNI_TRUE; +} + +/* + * Class: org_rocksdb_ComparatorOptions + * Method: maxReusedBufferSize + * Signature: (J)I + */ +jint Java_org_rocksdb_ComparatorOptions_maxReusedBufferSize(JNIEnv*, jobject, + jlong jhandle) { + return static_cast( + reinterpret_cast( + jhandle) + ->max_reused_buffer_size); +} + +/* + * Class: org_rocksdb_ComparatorOptions + * Method: setMaxReusedBufferSize + * Signature: (JI)V + */ +void Java_org_rocksdb_ComparatorOptions_setMaxReusedBufferSize( + JNIEnv*, jobject, jlong jhandle, jint jmax_reused_buffer_size) { + reinterpret_cast(jhandle) + ->max_reused_buffer_size = static_cast(jmax_reused_buffer_size); +} + +/* + * Class: org_rocksdb_ComparatorOptions + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_ComparatorOptions_disposeInternal(JNIEnv*, jobject, + jlong jhandle) { + auto* comparator_opt = + reinterpret_cast( + jhandle); + assert(comparator_opt != nullptr); + delete comparator_opt; +} + +///////////////////////////////////////////////////////////////////// +// ROCKSDB_NAMESPACE::FlushOptions + +/* + * Class: org_rocksdb_FlushOptions + * Method: newFlushOptions + * Signature: ()J + */ +jlong Java_org_rocksdb_FlushOptions_newFlushOptions(JNIEnv*, jclass) { + auto* flush_opt = new ROCKSDB_NAMESPACE::FlushOptions(); + return GET_CPLUSPLUS_POINTER(flush_opt); +} + +/* + * Class: org_rocksdb_FlushOptions + * Method: setWaitForFlush + * Signature: (JZ)V + */ +void Java_org_rocksdb_FlushOptions_setWaitForFlush(JNIEnv*, jobject, + jlong jhandle, + jboolean jwait) { + reinterpret_cast(jhandle)->wait = + static_cast(jwait); +} + +/* + * Class: org_rocksdb_FlushOptions + * Method: waitForFlush + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_FlushOptions_waitForFlush(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle)->wait; +} + +/* + * Class: org_rocksdb_FlushOptions + * Method: setAllowWriteStall + * Signature: (JZ)V + */ +void Java_org_rocksdb_FlushOptions_setAllowWriteStall( + JNIEnv*, jobject, jlong jhandle, jboolean jallow_write_stall) { + auto* flush_options = + reinterpret_cast(jhandle); + flush_options->allow_write_stall = jallow_write_stall == JNI_TRUE; +} + +/* + * Class: org_rocksdb_FlushOptions + * Method: allowWriteStall + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_FlushOptions_allowWriteStall(JNIEnv*, jobject, + jlong jhandle) { + auto* flush_options = + reinterpret_cast(jhandle); + return static_cast(flush_options->allow_write_stall); +} + +/* + * Class: org_rocksdb_FlushOptions + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_FlushOptions_disposeInternal(JNIEnv*, jobject, + jlong jhandle) { + auto* flush_opt = reinterpret_cast(jhandle); + assert(flush_opt != nullptr); + delete flush_opt; +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/options_util.cc b/librocksdb-sys/rocksdb/java/rocksjni/options_util.cc new file mode 100644 index 0000000..c3d7fce --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/options_util.cc @@ -0,0 +1,139 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling C++ ROCKSDB_NAMESPACE::OptionsUtil methods from Java side. + +#include "rocksdb/utilities/options_util.h" + +#include + +#include + +#include "include/org_rocksdb_OptionsUtil.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksjni/portal.h" + +void build_column_family_descriptor_list( + JNIEnv* env, jobject jcfds, + std::vector& cf_descs) { + jmethodID add_mid = ROCKSDB_NAMESPACE::ListJni::getListAddMethodId(env); + if (add_mid == nullptr) { + // exception occurred accessing method + return; + } + + // Column family descriptor + for (ROCKSDB_NAMESPACE::ColumnFamilyDescriptor& cfd : cf_descs) { + // Construct a ColumnFamilyDescriptor java object + jobject jcfd = + ROCKSDB_NAMESPACE::ColumnFamilyDescriptorJni::construct(env, &cfd); + if (env->ExceptionCheck()) { + // exception occurred constructing object + if (jcfd != nullptr) { + env->DeleteLocalRef(jcfd); + } + return; + } + + // Add the object to java list. + jboolean rs = env->CallBooleanMethod(jcfds, add_mid, jcfd); + if (env->ExceptionCheck() || rs == JNI_FALSE) { + // exception occurred calling method, or could not add + if (jcfd != nullptr) { + env->DeleteLocalRef(jcfd); + } + return; + } + } +} + +/* + * Class: org_rocksdb_OptionsUtil + * Method: loadLatestOptions + * Signature: (JLjava/lang/String;JLjava/util/List;)V + */ +void Java_org_rocksdb_OptionsUtil_loadLatestOptions( + JNIEnv* env, jclass /*jcls*/, jlong cfg_handle, jstring jdbpath, + jlong jdb_opts_handle, jobject jcfds) { + jboolean has_exception = JNI_FALSE; + auto db_path = + ROCKSDB_NAMESPACE::JniUtil::copyStdString(env, jdbpath, &has_exception); + if (has_exception == JNI_TRUE) { + // exception occurred + return; + } + std::vector cf_descs; + auto* config_options = + reinterpret_cast(cfg_handle); + auto* db_options = + reinterpret_cast(jdb_opts_handle); + ROCKSDB_NAMESPACE::Status s = ROCKSDB_NAMESPACE::LoadLatestOptions( + *config_options, db_path, db_options, &cf_descs); + if (!s.ok()) { + // error, raise an exception + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } else { + build_column_family_descriptor_list(env, jcfds, cf_descs); + } +} + +/* + * Class: org_rocksdb_OptionsUtil + * Method: loadOptionsFromFile + * Signature: (JLjava/lang/String;JLjava/util/List;)V + */ +void Java_org_rocksdb_OptionsUtil_loadOptionsFromFile( + JNIEnv* env, jclass /*jcls*/, jlong cfg_handle, jstring jopts_file_name, + jlong jdb_opts_handle, jobject jcfds) { + jboolean has_exception = JNI_FALSE; + auto opts_file_name = ROCKSDB_NAMESPACE::JniUtil::copyStdString( + env, jopts_file_name, &has_exception); + if (has_exception == JNI_TRUE) { + // exception occurred + return; + } + std::vector cf_descs; + auto* config_options = + reinterpret_cast(cfg_handle); + auto* db_options = + reinterpret_cast(jdb_opts_handle); + ROCKSDB_NAMESPACE::Status s = ROCKSDB_NAMESPACE::LoadOptionsFromFile( + *config_options, opts_file_name, db_options, &cf_descs); + if (!s.ok()) { + // error, raise an exception + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } else { + build_column_family_descriptor_list(env, jcfds, cf_descs); + } +} + +/* + * Class: org_rocksdb_OptionsUtil + * Method: getLatestOptionsFileName + * Signature: (Ljava/lang/String;J)Ljava/lang/String; + */ +jstring Java_org_rocksdb_OptionsUtil_getLatestOptionsFileName( + JNIEnv* env, jclass /*jcls*/, jstring jdbpath, jlong jenv_handle) { + jboolean has_exception = JNI_FALSE; + auto db_path = + ROCKSDB_NAMESPACE::JniUtil::copyStdString(env, jdbpath, &has_exception); + if (has_exception == JNI_TRUE) { + // exception occurred + return nullptr; + } + std::string options_file_name; + ROCKSDB_NAMESPACE::Status s = ROCKSDB_NAMESPACE::GetLatestOptionsFileName( + db_path, reinterpret_cast(jenv_handle), + &options_file_name); + if (!s.ok()) { + // error, raise an exception + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return nullptr; + } else { + return env->NewStringUTF(options_file_name.c_str()); + } +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/persistent_cache.cc b/librocksdb-sys/rocksdb/java/rocksjni/persistent_cache.cc new file mode 100644 index 0000000..295d917 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/persistent_cache.cc @@ -0,0 +1,60 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::PersistentCache. + +#include "rocksdb/persistent_cache.h" + +#include + +#include + +#include "include/org_rocksdb_PersistentCache.h" +#include "loggerjnicallback.h" +#include "portal.h" +#include "rocksjni/cplusplus_to_java_convert.h" + +/* + * Class: org_rocksdb_PersistentCache + * Method: newPersistentCache + * Signature: (JLjava/lang/String;JJZ)J + */ +jlong Java_org_rocksdb_PersistentCache_newPersistentCache( + JNIEnv* env, jclass, jlong jenv_handle, jstring jpath, jlong jsz, + jlong jlogger_handle, jboolean joptimized_for_nvm) { + auto* rocks_env = reinterpret_cast(jenv_handle); + jboolean has_exception = JNI_FALSE; + std::string path = + ROCKSDB_NAMESPACE::JniUtil::copyStdString(env, jpath, &has_exception); + if (has_exception == JNI_TRUE) { + return 0; + } + auto* logger = + reinterpret_cast*>( + jlogger_handle); + auto* cache = + new std::shared_ptr(nullptr); + ROCKSDB_NAMESPACE::Status s = ROCKSDB_NAMESPACE::NewPersistentCache( + rocks_env, path, static_cast(jsz), *logger, + static_cast(joptimized_for_nvm), cache); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } + return GET_CPLUSPLUS_POINTER(cache); +} + +/* + * Class: org_rocksdb_PersistentCache + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_PersistentCache_disposeInternal(JNIEnv*, jobject, + jlong jhandle) { + auto* cache = + reinterpret_cast*>( + jhandle); + delete cache; // delete std::shared_ptr +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/portal.h b/librocksdb-sys/rocksdb/java/rocksjni/portal.h new file mode 100644 index 0000000..f75e002 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/portal.h @@ -0,0 +1,8729 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +// This file is designed for caching those frequently used IDs and provide +// efficient portal (i.e, a set of static functions) to access java code +// from c++. + +#ifndef JAVA_ROCKSJNI_PORTAL_H_ +#define JAVA_ROCKSJNI_PORTAL_H_ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rocksdb/convenience.h" +#include "rocksdb/db.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/rate_limiter.h" +#include "rocksdb/status.h" +#include "rocksdb/table.h" +#include "rocksdb/utilities/backup_engine.h" +#include "rocksdb/utilities/memory_util.h" +#include "rocksdb/utilities/transaction_db.h" +#include "rocksdb/utilities/write_batch_with_index.h" +#include "rocksjni/compaction_filter_factory_jnicallback.h" +#include "rocksjni/comparatorjnicallback.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/event_listener_jnicallback.h" +#include "rocksjni/loggerjnicallback.h" +#include "rocksjni/table_filter_jnicallback.h" +#include "rocksjni/trace_writer_jnicallback.h" +#include "rocksjni/transaction_notifier_jnicallback.h" +#include "rocksjni/wal_filter_jnicallback.h" +#include "rocksjni/writebatchhandlerjnicallback.h" + +// Remove macro on windows +#ifdef DELETE +#undef DELETE +#endif + +namespace ROCKSDB_NAMESPACE { + +class JavaClass { + public: + /** + * Gets and initializes a Java Class + * + * @param env A pointer to the Java environment + * @param jclazz_name The fully qualified JNI name of the Java Class + * e.g. "java/lang/String" + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env, const char* jclazz_name) { + jclass jclazz = env->FindClass(jclazz_name); + assert(jclazz != nullptr); + return jclazz; + } +}; + +// Native class template +template +class RocksDBNativeClass : public JavaClass {}; + +// Native class template for sub-classes of RocksMutableObject +template +class NativeRocksMutableObject : public RocksDBNativeClass { + public: + /** + * Gets the Java Method ID for the + * RocksMutableObject#setNativeHandle(long, boolean) method + * + * @param env A pointer to the Java environment + * @return The Java Method ID or nullptr the RocksMutableObject class cannot + * be accessed, or if one of the NoSuchMethodError, + * ExceptionInInitializerError or OutOfMemoryError exceptions is thrown + */ + static jmethodID getSetNativeHandleMethod(JNIEnv* env) { + static jclass jclazz = DERIVED::getJClass(env); + if (jclazz == nullptr) { + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "setNativeHandle", "(JZ)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Sets the C++ object pointer handle in the Java object + * + * @param env A pointer to the Java environment + * @param jobj The Java object on which to set the pointer handle + * @param ptr The C++ object pointer + * @param java_owns_handle JNI_TRUE if ownership of the C++ object is + * managed by the Java object + * + * @return true if a Java exception is pending, false otherwise + */ + static bool setHandle(JNIEnv* env, jobject jobj, PTR ptr, + jboolean java_owns_handle) { + assert(jobj != nullptr); + static jmethodID mid = getSetNativeHandleMethod(env); + if (mid == nullptr) { + return true; // signal exception + } + + env->CallVoidMethod(jobj, mid, GET_CPLUSPLUS_POINTER(ptr), + java_owns_handle); + if (env->ExceptionCheck()) { + return true; // signal exception + } + + return false; + } +}; + +// Java Exception template +template +class JavaException : public JavaClass { + public: + /** + * Create and throw a java exception with the provided message + * + * @param env A pointer to the Java environment + * @param msg The message for the exception + * + * @return true if an exception was thrown, false otherwise + */ + static bool ThrowNew(JNIEnv* env, const std::string& msg) { + jclass jclazz = DERIVED::getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + std::cerr << "JavaException::ThrowNew - Error: unexpected exception!" + << std::endl; + return env->ExceptionCheck(); + } + + const jint rs = env->ThrowNew(jclazz, msg.c_str()); + if (rs != JNI_OK) { + // exception could not be thrown + std::cerr << "JavaException::ThrowNew - Fatal: could not throw exception!" + << std::endl; + return env->ExceptionCheck(); + } + + return true; + } +}; + +// The portal class for java.lang.IllegalArgumentException +class IllegalArgumentExceptionJni + : public JavaException { + public: + /** + * Get the Java Class java.lang.IllegalArgumentException + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaException::getJClass(env, "java/lang/IllegalArgumentException"); + } + + /** + * Create and throw a Java IllegalArgumentException with the provided status + * + * If s.ok() == true, then this function will not throw any exception. + * + * @param env A pointer to the Java environment + * @param s The status for the exception + * + * @return true if an exception was thrown, false otherwise + */ + static bool ThrowNew(JNIEnv* env, const Status& s) { + assert(!s.ok()); + if (s.ok()) { + return false; + } + + // get the IllegalArgumentException class + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + std::cerr << "IllegalArgumentExceptionJni::ThrowNew/class - Error: " + "unexpected exception!" + << std::endl; + return env->ExceptionCheck(); + } + + return JavaException::ThrowNew(env, s.ToString()); + } +}; + +// The portal class for org.rocksdb.Status.Code +class CodeJni : public JavaClass { + public: + /** + * Get the Java Class org.rocksdb.Status.Code + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/Status$Code"); + } + + /** + * Get the Java Method: Status.Code#getValue + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getValueMethod(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "getValue", "()b"); + assert(mid != nullptr); + return mid; + } +}; + +// The portal class for org.rocksdb.Status.SubCode +class SubCodeJni : public JavaClass { + public: + /** + * Get the Java Class org.rocksdb.Status.SubCode + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/Status$SubCode"); + } + + /** + * Get the Java Method: Status.SubCode#getValue + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getValueMethod(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "getValue", "()b"); + assert(mid != nullptr); + return mid; + } + + static ROCKSDB_NAMESPACE::Status::SubCode toCppSubCode( + const jbyte jsub_code) { + switch (jsub_code) { + case 0x0: + return ROCKSDB_NAMESPACE::Status::SubCode::kNone; + case 0x1: + return ROCKSDB_NAMESPACE::Status::SubCode::kMutexTimeout; + case 0x2: + return ROCKSDB_NAMESPACE::Status::SubCode::kLockTimeout; + case 0x3: + return ROCKSDB_NAMESPACE::Status::SubCode::kLockLimit; + case 0x4: + return ROCKSDB_NAMESPACE::Status::SubCode::kNoSpace; + case 0x5: + return ROCKSDB_NAMESPACE::Status::SubCode::kDeadlock; + case 0x6: + return ROCKSDB_NAMESPACE::Status::SubCode::kStaleFile; + case 0x7: + return ROCKSDB_NAMESPACE::Status::SubCode::kMemoryLimit; + + case 0x7F: + default: + return ROCKSDB_NAMESPACE::Status::SubCode::kNone; + } + } +}; + +// The portal class for org.rocksdb.Status +class StatusJni + : public RocksDBNativeClass { + public: + /** + * Get the Java Class org.rocksdb.Status + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, "org/rocksdb/Status"); + } + + /** + * Get the Java Method: Status#getCode + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getCodeMethod(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = + env->GetMethodID(jclazz, "getCode", "()Lorg/rocksdb/Status$Code;"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: Status#getSubCode + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getSubCodeMethod(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "getSubCode", + "()Lorg/rocksdb/Status$SubCode;"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: Status#getState + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getStateMethod(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = + env->GetMethodID(jclazz, "getState", "()Ljava/lang/String;"); + assert(mid != nullptr); + return mid; + } + + /** + * Create a new Java org.rocksdb.Status object with the same properties as + * the provided C++ ROCKSDB_NAMESPACE::Status object + * + * @param env A pointer to the Java environment + * @param status The ROCKSDB_NAMESPACE::Status object + * + * @return A reference to a Java org.rocksdb.Status object, or nullptr + * if an an exception occurs + */ + static jobject construct(JNIEnv* env, const Status& status) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID mid = + env->GetMethodID(jclazz, "", "(BBLjava/lang/String;)V"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return nullptr; + } + + // convert the Status state for Java + jstring jstate = nullptr; + if (status.getState() != nullptr) { + const char* const state = status.getState(); + jstate = env->NewStringUTF(state); + if (env->ExceptionCheck()) { + if (jstate != nullptr) { + env->DeleteLocalRef(jstate); + } + return nullptr; + } + } + + jobject jstatus = + env->NewObject(jclazz, mid, toJavaStatusCode(status.code()), + toJavaStatusSubCode(status.subcode()), jstate); + if (env->ExceptionCheck()) { + // exception occurred + if (jstate != nullptr) { + env->DeleteLocalRef(jstate); + } + return nullptr; + } + + if (jstate != nullptr) { + env->DeleteLocalRef(jstate); + } + + return jstatus; + } + + static jobject construct(JNIEnv* env, const Status* status) { + return construct(env, *status); + } + + // Returns the equivalent org.rocksdb.Status.Code for the provided + // C++ ROCKSDB_NAMESPACE::Status::Code enum + static jbyte toJavaStatusCode(const ROCKSDB_NAMESPACE::Status::Code& code) { + switch (code) { + case ROCKSDB_NAMESPACE::Status::Code::kOk: + return 0x0; + case ROCKSDB_NAMESPACE::Status::Code::kNotFound: + return 0x1; + case ROCKSDB_NAMESPACE::Status::Code::kCorruption: + return 0x2; + case ROCKSDB_NAMESPACE::Status::Code::kNotSupported: + return 0x3; + case ROCKSDB_NAMESPACE::Status::Code::kInvalidArgument: + return 0x4; + case ROCKSDB_NAMESPACE::Status::Code::kIOError: + return 0x5; + case ROCKSDB_NAMESPACE::Status::Code::kMergeInProgress: + return 0x6; + case ROCKSDB_NAMESPACE::Status::Code::kIncomplete: + return 0x7; + case ROCKSDB_NAMESPACE::Status::Code::kShutdownInProgress: + return 0x8; + case ROCKSDB_NAMESPACE::Status::Code::kTimedOut: + return 0x9; + case ROCKSDB_NAMESPACE::Status::Code::kAborted: + return 0xA; + case ROCKSDB_NAMESPACE::Status::Code::kBusy: + return 0xB; + case ROCKSDB_NAMESPACE::Status::Code::kExpired: + return 0xC; + case ROCKSDB_NAMESPACE::Status::Code::kTryAgain: + return 0xD; + case ROCKSDB_NAMESPACE::Status::Code::kColumnFamilyDropped: + return 0xE; + default: + return 0x7F; // undefined + } + } + + // Returns the equivalent org.rocksdb.Status.SubCode for the provided + // C++ ROCKSDB_NAMESPACE::Status::SubCode enum + static jbyte toJavaStatusSubCode( + const ROCKSDB_NAMESPACE::Status::SubCode& subCode) { + switch (subCode) { + case ROCKSDB_NAMESPACE::Status::SubCode::kNone: + return 0x0; + case ROCKSDB_NAMESPACE::Status::SubCode::kMutexTimeout: + return 0x1; + case ROCKSDB_NAMESPACE::Status::SubCode::kLockTimeout: + return 0x2; + case ROCKSDB_NAMESPACE::Status::SubCode::kLockLimit: + return 0x3; + case ROCKSDB_NAMESPACE::Status::SubCode::kNoSpace: + return 0x4; + case ROCKSDB_NAMESPACE::Status::SubCode::kDeadlock: + return 0x5; + case ROCKSDB_NAMESPACE::Status::SubCode::kStaleFile: + return 0x6; + case ROCKSDB_NAMESPACE::Status::SubCode::kMemoryLimit: + return 0x7; + default: + return 0x7F; // undefined + } + } + + static std::unique_ptr toCppStatus( + const jbyte jcode_value, const jbyte jsub_code_value) { + std::unique_ptr status; + switch (jcode_value) { + case 0x0: + // Ok + status = std::unique_ptr( + new ROCKSDB_NAMESPACE::Status(ROCKSDB_NAMESPACE::Status::OK())); + break; + case 0x1: + // NotFound + status = std::unique_ptr( + new ROCKSDB_NAMESPACE::Status(ROCKSDB_NAMESPACE::Status::NotFound( + ROCKSDB_NAMESPACE::SubCodeJni::toCppSubCode(jsub_code_value)))); + break; + case 0x2: + // Corruption + status = std::unique_ptr( + new ROCKSDB_NAMESPACE::Status(ROCKSDB_NAMESPACE::Status::Corruption( + ROCKSDB_NAMESPACE::SubCodeJni::toCppSubCode(jsub_code_value)))); + break; + case 0x3: + // NotSupported + status = std::unique_ptr( + new ROCKSDB_NAMESPACE::Status( + ROCKSDB_NAMESPACE::Status::NotSupported( + ROCKSDB_NAMESPACE::SubCodeJni::toCppSubCode( + jsub_code_value)))); + break; + case 0x4: + // InvalidArgument + status = std::unique_ptr( + new ROCKSDB_NAMESPACE::Status( + ROCKSDB_NAMESPACE::Status::InvalidArgument( + ROCKSDB_NAMESPACE::SubCodeJni::toCppSubCode( + jsub_code_value)))); + break; + case 0x5: + // IOError + status = std::unique_ptr( + new ROCKSDB_NAMESPACE::Status(ROCKSDB_NAMESPACE::Status::IOError( + ROCKSDB_NAMESPACE::SubCodeJni::toCppSubCode(jsub_code_value)))); + break; + case 0x6: + // MergeInProgress + status = std::unique_ptr( + new ROCKSDB_NAMESPACE::Status( + ROCKSDB_NAMESPACE::Status::MergeInProgress( + ROCKSDB_NAMESPACE::SubCodeJni::toCppSubCode( + jsub_code_value)))); + break; + case 0x7: + // Incomplete + status = std::unique_ptr( + new ROCKSDB_NAMESPACE::Status(ROCKSDB_NAMESPACE::Status::Incomplete( + ROCKSDB_NAMESPACE::SubCodeJni::toCppSubCode(jsub_code_value)))); + break; + case 0x8: + // ShutdownInProgress + status = std::unique_ptr( + new ROCKSDB_NAMESPACE::Status( + ROCKSDB_NAMESPACE::Status::ShutdownInProgress( + ROCKSDB_NAMESPACE::SubCodeJni::toCppSubCode( + jsub_code_value)))); + break; + case 0x9: + // TimedOut + status = std::unique_ptr( + new ROCKSDB_NAMESPACE::Status(ROCKSDB_NAMESPACE::Status::TimedOut( + ROCKSDB_NAMESPACE::SubCodeJni::toCppSubCode(jsub_code_value)))); + break; + case 0xA: + // Aborted + status = std::unique_ptr( + new ROCKSDB_NAMESPACE::Status(ROCKSDB_NAMESPACE::Status::Aborted( + ROCKSDB_NAMESPACE::SubCodeJni::toCppSubCode(jsub_code_value)))); + break; + case 0xB: + // Busy + status = std::unique_ptr( + new ROCKSDB_NAMESPACE::Status(ROCKSDB_NAMESPACE::Status::Busy( + ROCKSDB_NAMESPACE::SubCodeJni::toCppSubCode(jsub_code_value)))); + break; + case 0xC: + // Expired + status = std::unique_ptr( + new ROCKSDB_NAMESPACE::Status(ROCKSDB_NAMESPACE::Status::Expired( + ROCKSDB_NAMESPACE::SubCodeJni::toCppSubCode(jsub_code_value)))); + break; + case 0xD: + // TryAgain + status = std::unique_ptr( + new ROCKSDB_NAMESPACE::Status(ROCKSDB_NAMESPACE::Status::TryAgain( + ROCKSDB_NAMESPACE::SubCodeJni::toCppSubCode(jsub_code_value)))); + break; + case 0xE: + // ColumnFamilyDropped + status = std::unique_ptr( + new ROCKSDB_NAMESPACE::Status( + ROCKSDB_NAMESPACE::Status::ColumnFamilyDropped( + ROCKSDB_NAMESPACE::SubCodeJni::toCppSubCode( + jsub_code_value)))); + break; + case 0x7F: + default: + return nullptr; + } + return status; + } + + // Returns the equivalent ROCKSDB_NAMESPACE::Status for the Java + // org.rocksdb.Status + static std::unique_ptr toCppStatus( + JNIEnv* env, const jobject jstatus) { + jmethodID mid_code = getCodeMethod(env); + if (mid_code == nullptr) { + // exception occurred + return nullptr; + } + jobject jcode = env->CallObjectMethod(jstatus, mid_code); + if (env->ExceptionCheck()) { + // exception occurred + return nullptr; + } + + jmethodID mid_code_value = ROCKSDB_NAMESPACE::CodeJni::getValueMethod(env); + if (mid_code_value == nullptr) { + // exception occurred + return nullptr; + } + jbyte jcode_value = env->CallByteMethod(jcode, mid_code_value); + if (env->ExceptionCheck()) { + // exception occurred + if (jcode != nullptr) { + env->DeleteLocalRef(jcode); + } + return nullptr; + } + + jmethodID mid_subCode = getSubCodeMethod(env); + if (mid_subCode == nullptr) { + // exception occurred + return nullptr; + } + jobject jsubCode = env->CallObjectMethod(jstatus, mid_subCode); + if (env->ExceptionCheck()) { + // exception occurred + if (jcode != nullptr) { + env->DeleteLocalRef(jcode); + } + return nullptr; + } + + jbyte jsub_code_value = 0x0; // None + if (jsubCode != nullptr) { + jmethodID mid_subCode_value = + ROCKSDB_NAMESPACE::SubCodeJni::getValueMethod(env); + if (mid_subCode_value == nullptr) { + // exception occurred + return nullptr; + } + jsub_code_value = env->CallByteMethod(jsubCode, mid_subCode_value); + if (env->ExceptionCheck()) { + // exception occurred + if (jcode != nullptr) { + env->DeleteLocalRef(jcode); + } + return nullptr; + } + } + + jmethodID mid_state = getStateMethod(env); + if (mid_state == nullptr) { + // exception occurred + return nullptr; + } + jobject jstate = env->CallObjectMethod(jstatus, mid_state); + if (env->ExceptionCheck()) { + // exception occurred + if (jsubCode != nullptr) { + env->DeleteLocalRef(jsubCode); + } + if (jcode != nullptr) { + env->DeleteLocalRef(jcode); + } + return nullptr; + } + + std::unique_ptr status = + toCppStatus(jcode_value, jsub_code_value); + + // delete all local refs + if (jstate != nullptr) { + env->DeleteLocalRef(jstate); + } + if (jsubCode != nullptr) { + env->DeleteLocalRef(jsubCode); + } + if (jcode != nullptr) { + env->DeleteLocalRef(jcode); + } + + return status; + } +}; + +// The portal class for org.rocksdb.RocksDBException +class RocksDBExceptionJni : public JavaException { + public: + /** + * Get the Java Class org.rocksdb.RocksDBException + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaException::getJClass(env, "org/rocksdb/RocksDBException"); + } + + /** + * Create and throw a Java RocksDBException with the provided message + * + * @param env A pointer to the Java environment + * @param msg The message for the exception + * + * @return true if an exception was thrown, false otherwise + */ + static bool ThrowNew(JNIEnv* env, const std::string& msg) { + return JavaException::ThrowNew(env, msg); + } + + /** + * Create and throw a Java RocksDBException with the provided status + * + * If s->ok() == true, then this function will not throw any exception. + * + * @param env A pointer to the Java environment + * @param s The status for the exception + * + * @return true if an exception was thrown, false otherwise + */ + static bool ThrowNew(JNIEnv* env, std::unique_ptr& s) { + return ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, *(s.get())); + } + + /** + * Create and throw a Java RocksDBException with the provided status + * + * If s.ok() == true, then this function will not throw any exception. + * + * @param env A pointer to the Java environment + * @param s The status for the exception + * + * @return true if an exception was thrown, false otherwise + */ + static bool ThrowNew(JNIEnv* env, const Status& s) { + if (s.ok()) { + return false; + } + + // get the RocksDBException class + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + std::cerr << "RocksDBExceptionJni::ThrowNew/class - Error: unexpected " + "exception!" + << std::endl; + return env->ExceptionCheck(); + } + + // get the constructor of org.rocksdb.RocksDBException + jmethodID mid = + env->GetMethodID(jclazz, "", "(Lorg/rocksdb/Status;)V"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + std::cerr + << "RocksDBExceptionJni::ThrowNew/cstr - Error: unexpected exception!" + << std::endl; + return env->ExceptionCheck(); + } + + // get the Java status object + jobject jstatus = StatusJni::construct(env, s); + if (jstatus == nullptr) { + // exception occcurred + std::cerr << "RocksDBExceptionJni::ThrowNew/StatusJni - Error: " + "unexpected exception!" + << std::endl; + return env->ExceptionCheck(); + } + + // construct the RocksDBException + jthrowable rocksdb_exception = + reinterpret_cast(env->NewObject(jclazz, mid, jstatus)); + if (env->ExceptionCheck()) { + if (jstatus != nullptr) { + env->DeleteLocalRef(jstatus); + } + if (rocksdb_exception != nullptr) { + env->DeleteLocalRef(rocksdb_exception); + } + std::cerr << "RocksDBExceptionJni::ThrowNew/NewObject - Error: " + "unexpected exception!" + << std::endl; + return true; + } + + // throw the RocksDBException + const jint rs = env->Throw(rocksdb_exception); + if (rs != JNI_OK) { + // exception could not be thrown + std::cerr + << "RocksDBExceptionJni::ThrowNew - Fatal: could not throw exception!" + << std::endl; + if (jstatus != nullptr) { + env->DeleteLocalRef(jstatus); + } + if (rocksdb_exception != nullptr) { + env->DeleteLocalRef(rocksdb_exception); + } + return env->ExceptionCheck(); + } + + if (jstatus != nullptr) { + env->DeleteLocalRef(jstatus); + } + if (rocksdb_exception != nullptr) { + env->DeleteLocalRef(rocksdb_exception); + } + + return true; + } + + /** + * Create and throw a Java RocksDBException with the provided message + * and status + * + * If s.ok() == true, then this function will not throw any exception. + * + * @param env A pointer to the Java environment + * @param msg The message for the exception + * @param s The status for the exception + * + * @return true if an exception was thrown, false otherwise + */ + static bool ThrowNew(JNIEnv* env, const std::string& msg, const Status& s) { + assert(!s.ok()); + if (s.ok()) { + return false; + } + + // get the RocksDBException class + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + std::cerr << "RocksDBExceptionJni::ThrowNew/class - Error: unexpected " + "exception!" + << std::endl; + return env->ExceptionCheck(); + } + + // get the constructor of org.rocksdb.RocksDBException + jmethodID mid = env->GetMethodID( + jclazz, "", "(Ljava/lang/String;Lorg/rocksdb/Status;)V"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + std::cerr + << "RocksDBExceptionJni::ThrowNew/cstr - Error: unexpected exception!" + << std::endl; + return env->ExceptionCheck(); + } + + jstring jmsg = env->NewStringUTF(msg.c_str()); + if (jmsg == nullptr) { + // exception thrown: OutOfMemoryError + std::cerr + << "RocksDBExceptionJni::ThrowNew/msg - Error: unexpected exception!" + << std::endl; + return env->ExceptionCheck(); + } + + // get the Java status object + jobject jstatus = StatusJni::construct(env, s); + if (jstatus == nullptr) { + // exception occcurred + std::cerr << "RocksDBExceptionJni::ThrowNew/StatusJni - Error: " + "unexpected exception!" + << std::endl; + if (jmsg != nullptr) { + env->DeleteLocalRef(jmsg); + } + return env->ExceptionCheck(); + } + + // construct the RocksDBException + jthrowable rocksdb_exception = reinterpret_cast( + env->NewObject(jclazz, mid, jmsg, jstatus)); + if (env->ExceptionCheck()) { + if (jstatus != nullptr) { + env->DeleteLocalRef(jstatus); + } + if (jmsg != nullptr) { + env->DeleteLocalRef(jmsg); + } + if (rocksdb_exception != nullptr) { + env->DeleteLocalRef(rocksdb_exception); + } + std::cerr << "RocksDBExceptionJni::ThrowNew/NewObject - Error: " + "unexpected exception!" + << std::endl; + return true; + } + + // throw the RocksDBException + const jint rs = env->Throw(rocksdb_exception); + if (rs != JNI_OK) { + // exception could not be thrown + std::cerr + << "RocksDBExceptionJni::ThrowNew - Fatal: could not throw exception!" + << std::endl; + if (jstatus != nullptr) { + env->DeleteLocalRef(jstatus); + } + if (jmsg != nullptr) { + env->DeleteLocalRef(jmsg); + } + if (rocksdb_exception != nullptr) { + env->DeleteLocalRef(rocksdb_exception); + } + return env->ExceptionCheck(); + } + + if (jstatus != nullptr) { + env->DeleteLocalRef(jstatus); + } + if (jmsg != nullptr) { + env->DeleteLocalRef(jmsg); + } + if (rocksdb_exception != nullptr) { + env->DeleteLocalRef(rocksdb_exception); + } + + return true; + } + + /** + * Get the Java Method: RocksDBException#getStatus + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getStatusMethod(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = + env->GetMethodID(jclazz, "getStatus", "()Lorg/rocksdb/Status;"); + assert(mid != nullptr); + return mid; + } + + static std::unique_ptr toCppStatus( + JNIEnv* env, jthrowable jrocksdb_exception) { + if (!env->IsInstanceOf(jrocksdb_exception, getJClass(env))) { + // not an instance of RocksDBException + return nullptr; + } + + // get the java status object + jmethodID mid = getStatusMethod(env); + if (mid == nullptr) { + // exception occurred accessing class or method + return nullptr; + } + + jobject jstatus = env->CallObjectMethod(jrocksdb_exception, mid); + if (env->ExceptionCheck()) { + // exception occurred + return nullptr; + } + + if (jstatus == nullptr) { + return nullptr; // no status available + } + + return ROCKSDB_NAMESPACE::StatusJni::toCppStatus(env, jstatus); + } +}; + +// The portal class for java.util.List +class ListJni : public JavaClass { + public: + /** + * Get the Java Class java.util.List + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getListClass(JNIEnv* env) { + return JavaClass::getJClass(env, "java/util/List"); + } + + /** + * Get the Java Class java.util.ArrayList + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getArrayListClass(JNIEnv* env) { + return JavaClass::getJClass(env, "java/util/ArrayList"); + } + + /** + * Get the Java Class java.util.Iterator + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getIteratorClass(JNIEnv* env) { + return JavaClass::getJClass(env, "java/util/Iterator"); + } + + /** + * Get the Java Method: List#iterator + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getIteratorMethod(JNIEnv* env) { + jclass jlist_clazz = getListClass(env); + if (jlist_clazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = + env->GetMethodID(jlist_clazz, "iterator", "()Ljava/util/Iterator;"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: Iterator#hasNext + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getHasNextMethod(JNIEnv* env) { + jclass jiterator_clazz = getIteratorClass(env); + if (jiterator_clazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jiterator_clazz, "hasNext", "()Z"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: Iterator#next + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getNextMethod(JNIEnv* env) { + jclass jiterator_clazz = getIteratorClass(env); + if (jiterator_clazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = + env->GetMethodID(jiterator_clazz, "next", "()Ljava/lang/Object;"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: ArrayList constructor + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getArrayListConstructorMethodId(JNIEnv* env) { + jclass jarray_list_clazz = getArrayListClass(env); + if (jarray_list_clazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + static jmethodID mid = + env->GetMethodID(jarray_list_clazz, "", "(I)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: List#add + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getListAddMethodId(JNIEnv* env) { + jclass jlist_clazz = getListClass(env); + if (jlist_clazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = + env->GetMethodID(jlist_clazz, "add", "(Ljava/lang/Object;)Z"); + assert(mid != nullptr); + return mid; + } +}; + +// The portal class for java.lang.Byte +class ByteJni : public JavaClass { + public: + /** + * Get the Java Class java.lang.Byte + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "java/lang/Byte"); + } + + /** + * Get the Java Class byte[] + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getArrayJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "[B"); + } + + /** + * Creates a new 2-dimensional Java Byte Array byte[][] + * + * @param env A pointer to the Java environment + * @param len The size of the first dimension + * + * @return A reference to the Java byte[][] or nullptr if an exception occurs + */ + static jobjectArray new2dByteArray(JNIEnv* env, const jsize len) { + jclass clazz = getArrayJClass(env); + if (clazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + return env->NewObjectArray(len, clazz, nullptr); + } + + /** + * Get the Java Method: Byte#byteValue + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getByteValueMethod(JNIEnv* env) { + jclass clazz = getJClass(env); + if (clazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(clazz, "byteValue", "()B"); + assert(mid != nullptr); + return mid; + } + + /** + * Calls the Java Method: Byte#valueOf, returning a constructed Byte jobject + * + * @param env A pointer to the Java environment + * + * @return A constructing Byte object or nullptr if the class or method id + * could not be retrieved, or an exception occurred + */ + static jobject valueOf(JNIEnv* env, jbyte jprimitive_byte) { + jclass clazz = getJClass(env); + if (clazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = + env->GetStaticMethodID(clazz, "valueOf", "(B)Ljava/lang/Byte;"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return nullptr; + } + + const jobject jbyte_obj = + env->CallStaticObjectMethod(clazz, mid, jprimitive_byte); + if (env->ExceptionCheck()) { + // exception occurred + return nullptr; + } + + return jbyte_obj; + } +}; + +// The portal class for java.nio.ByteBuffer +class ByteBufferJni : public JavaClass { + public: + /** + * Get the Java Class java.nio.ByteBuffer + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "java/nio/ByteBuffer"); + } + + /** + * Get the Java Method: ByteBuffer#allocate + * + * @param env A pointer to the Java environment + * @param jbytebuffer_clazz if you have a reference to a ByteBuffer class, or + * nullptr + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getAllocateMethodId(JNIEnv* env, + jclass jbytebuffer_clazz = nullptr) { + const jclass jclazz = + jbytebuffer_clazz == nullptr ? getJClass(env) : jbytebuffer_clazz; + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = + env->GetStaticMethodID(jclazz, "allocate", "(I)Ljava/nio/ByteBuffer;"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: ByteBuffer#array + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getArrayMethodId(JNIEnv* env, + jclass jbytebuffer_clazz = nullptr) { + const jclass jclazz = + jbytebuffer_clazz == nullptr ? getJClass(env) : jbytebuffer_clazz; + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "array", "()[B"); + assert(mid != nullptr); + return mid; + } + + static jobject construct(JNIEnv* env, const bool direct, + const size_t capacity, + jclass jbytebuffer_clazz = nullptr) { + return constructWith(env, direct, nullptr, capacity, jbytebuffer_clazz); + } + + static jobject constructWith(JNIEnv* env, const bool direct, const char* buf, + const size_t capacity, + jclass jbytebuffer_clazz = nullptr) { + if (direct) { + bool allocated = false; + if (buf == nullptr) { + buf = new char[capacity]; + allocated = true; + } + jobject jbuf = env->NewDirectByteBuffer(const_cast(buf), + static_cast(capacity)); + if (jbuf == nullptr) { + // exception occurred + if (allocated) { + delete[] static_cast(buf); + } + return nullptr; + } + return jbuf; + } else { + const jclass jclazz = + jbytebuffer_clazz == nullptr ? getJClass(env) : jbytebuffer_clazz; + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + const jmethodID jmid_allocate = + getAllocateMethodId(env, jbytebuffer_clazz); + if (jmid_allocate == nullptr) { + // exception occurred accessing class, or NoSuchMethodException or + // OutOfMemoryError + return nullptr; + } + const jobject jbuf = env->CallStaticObjectMethod( + jclazz, jmid_allocate, static_cast(capacity)); + if (env->ExceptionCheck()) { + // exception occurred + return nullptr; + } + + // set buffer data? + if (buf != nullptr) { + jbyteArray jarray = array(env, jbuf, jbytebuffer_clazz); + if (jarray == nullptr) { + // exception occurred + env->DeleteLocalRef(jbuf); + return nullptr; + } + + jboolean is_copy = JNI_FALSE; + jbyte* ja = reinterpret_cast( + env->GetPrimitiveArrayCritical(jarray, &is_copy)); + if (ja == nullptr) { + // exception occurred + env->DeleteLocalRef(jarray); + env->DeleteLocalRef(jbuf); + return nullptr; + } + + memcpy(ja, const_cast(buf), capacity); + + env->ReleasePrimitiveArrayCritical(jarray, ja, 0); + + env->DeleteLocalRef(jarray); + } + + return jbuf; + } + } + + static jbyteArray array(JNIEnv* env, const jobject& jbyte_buffer, + jclass jbytebuffer_clazz = nullptr) { + const jmethodID mid = getArrayMethodId(env, jbytebuffer_clazz); + if (mid == nullptr) { + // exception occurred accessing class, or NoSuchMethodException or + // OutOfMemoryError + return nullptr; + } + const jobject jarray = env->CallObjectMethod(jbyte_buffer, mid); + if (env->ExceptionCheck()) { + // exception occurred + return nullptr; + } + return static_cast(jarray); + } +}; + +// The portal class for java.lang.Integer +class IntegerJni : public JavaClass { + public: + /** + * Get the Java Class java.lang.Integer + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "java/lang/Integer"); + } + + static jobject valueOf(JNIEnv* env, jint jprimitive_int) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID mid = + env->GetStaticMethodID(jclazz, "valueOf", "(I)Ljava/lang/Integer;"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return nullptr; + } + + const jobject jinteger_obj = + env->CallStaticObjectMethod(jclazz, mid, jprimitive_int); + if (env->ExceptionCheck()) { + // exception occurred + return nullptr; + } + + return jinteger_obj; + } +}; + +// The portal class for java.lang.Long +class LongJni : public JavaClass { + public: + /** + * Get the Java Class java.lang.Long + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "java/lang/Long"); + } + + static jobject valueOf(JNIEnv* env, jlong jprimitive_long) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID mid = + env->GetStaticMethodID(jclazz, "valueOf", "(J)Ljava/lang/Long;"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return nullptr; + } + + const jobject jlong_obj = + env->CallStaticObjectMethod(jclazz, mid, jprimitive_long); + if (env->ExceptionCheck()) { + // exception occurred + return nullptr; + } + + return jlong_obj; + } +}; + +// The portal class for java.lang.StringBuilder +class StringBuilderJni : public JavaClass { + public: + /** + * Get the Java Class java.lang.StringBuilder + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "java/lang/StringBuilder"); + } + + /** + * Get the Java Method: StringBuilder#append + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getListAddMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID( + jclazz, "append", "(Ljava/lang/String;)Ljava/lang/StringBuilder;"); + assert(mid != nullptr); + return mid; + } + + /** + * Appends a C-style string to a StringBuilder + * + * @param env A pointer to the Java environment + * @param jstring_builder Reference to a java.lang.StringBuilder + * @param c_str A C-style string to append to the StringBuilder + * + * @return A reference to the updated StringBuilder, or a nullptr if + * an exception occurs + */ + static jobject append(JNIEnv* env, jobject jstring_builder, + const char* c_str) { + jmethodID mid = getListAddMethodId(env); + if (mid == nullptr) { + // exception occurred accessing class or method + return nullptr; + } + + jstring new_value_str = env->NewStringUTF(c_str); + if (new_value_str == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + jobject jresult_string_builder = + env->CallObjectMethod(jstring_builder, mid, new_value_str); + if (env->ExceptionCheck()) { + // exception occurred + env->DeleteLocalRef(new_value_str); + return nullptr; + } + + return jresult_string_builder; + } +}; + +// various utility functions for working with RocksDB and JNI +class JniUtil { + public: + /** + * Detect if jlong overflows size_t + * + * @param jvalue the jlong value + * + * @return + */ + inline static Status check_if_jlong_fits_size_t(const jlong& jvalue) { + Status s = Status::OK(); + if (static_cast(jvalue) > std::numeric_limits::max()) { + s = Status::InvalidArgument(Slice("jlong overflows 32 bit value.")); + } + return s; + } + + /** + * Obtains a reference to the JNIEnv from + * the JVM + * + * If the current thread is not attached to the JavaVM + * then it will be attached so as to retrieve the JNIEnv + * + * If a thread is attached, it must later be manually + * released by calling JavaVM::DetachCurrentThread. + * This can be handled by always matching calls to this + * function with calls to {@link JniUtil::releaseJniEnv(JavaVM*, jboolean)} + * + * @param jvm (IN) A pointer to the JavaVM instance + * @param attached (OUT) A pointer to a boolean which + * will be set to JNI_TRUE if we had to attach the thread + * + * @return A pointer to the JNIEnv or nullptr if a fatal error + * occurs and the JNIEnv cannot be retrieved + */ + static JNIEnv* getJniEnv(JavaVM* jvm, jboolean* attached) { + assert(jvm != nullptr); + + JNIEnv* env; + const jint env_rs = + jvm->GetEnv(reinterpret_cast(&env), JNI_VERSION_1_6); + + if (env_rs == JNI_OK) { + // current thread is already attached, return the JNIEnv + *attached = JNI_FALSE; + return env; + } else if (env_rs == JNI_EDETACHED) { + // current thread is not attached, attempt to attach + const jint rs_attach = + jvm->AttachCurrentThread(reinterpret_cast(&env), NULL); + if (rs_attach == JNI_OK) { + *attached = JNI_TRUE; + return env; + } else { + // error, could not attach the thread + std::cerr << "JniUtil::getJniEnv - Fatal: could not attach current " + "thread to JVM!" + << std::endl; + return nullptr; + } + } else if (env_rs == JNI_EVERSION) { + // error, JDK does not support JNI_VERSION_1_6+ + std::cerr + << "JniUtil::getJniEnv - Fatal: JDK does not support JNI_VERSION_1_6" + << std::endl; + return nullptr; + } else { + std::cerr << "JniUtil::getJniEnv - Fatal: Unknown error: env_rs=" + << env_rs << std::endl; + return nullptr; + } + } + + /** + * Counterpart to {@link JniUtil::getJniEnv(JavaVM*, jboolean*)} + * + * Detachess the current thread from the JVM if it was previously + * attached + * + * @param jvm (IN) A pointer to the JavaVM instance + * @param attached (IN) JNI_TRUE if we previously had to attach the thread + * to the JavaVM to get the JNIEnv + */ + static void releaseJniEnv(JavaVM* jvm, jboolean& attached) { + assert(jvm != nullptr); + if (attached == JNI_TRUE) { + const jint rs_detach = jvm->DetachCurrentThread(); + assert(rs_detach == JNI_OK); + if (rs_detach != JNI_OK) { + std::cerr << "JniUtil::getJniEnv - Warn: Unable to detach current " + "thread from JVM!" + << std::endl; + } + } + } + + /** + * Copies a Java String[] to a C++ std::vector + * + * @param env (IN) A pointer to the java environment + * @param jss (IN) The Java String array to copy + * @param has_exception (OUT) will be set to JNI_TRUE + * if an OutOfMemoryError or ArrayIndexOutOfBoundsException + * exception occurs + * + * @return A std::vector containing copies of the Java strings + */ + static std::vector copyStrings(JNIEnv* env, jobjectArray jss, + jboolean* has_exception) { + return ROCKSDB_NAMESPACE::JniUtil::copyStrings( + env, jss, env->GetArrayLength(jss), has_exception); + } + + /** + * Copies a Java String[] to a C++ std::vector + * + * @param env (IN) A pointer to the java environment + * @param jss (IN) The Java String array to copy + * @param jss_len (IN) The length of the Java String array to copy + * @param has_exception (OUT) will be set to JNI_TRUE + * if an OutOfMemoryError or ArrayIndexOutOfBoundsException + * exception occurs + * + * @return A std::vector containing copies of the Java strings + */ + static std::vector copyStrings(JNIEnv* env, jobjectArray jss, + const jsize jss_len, + jboolean* has_exception) { + std::vector strs; + strs.reserve(jss_len); + for (jsize i = 0; i < jss_len; i++) { + jobject js = env->GetObjectArrayElement(jss, i); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + *has_exception = JNI_TRUE; + return strs; + } + + jstring jstr = static_cast(js); + const char* str = env->GetStringUTFChars(jstr, nullptr); + if (str == nullptr) { + // exception thrown: OutOfMemoryError + env->DeleteLocalRef(js); + *has_exception = JNI_TRUE; + return strs; + } + + strs.push_back(std::string(str)); + + env->ReleaseStringUTFChars(jstr, str); + env->DeleteLocalRef(js); + } + + *has_exception = JNI_FALSE; + return strs; + } + + /** + * Copies a jstring to a C-style null-terminated byte string + * and releases the original jstring + * + * The jstring is copied as UTF-8 + * + * If an exception occurs, then JNIEnv::ExceptionCheck() + * will have been called + * + * @param env (IN) A pointer to the java environment + * @param js (IN) The java string to copy + * @param has_exception (OUT) will be set to JNI_TRUE + * if an OutOfMemoryError exception occurs + * + * @return A pointer to the copied string, or a + * nullptr if has_exception == JNI_TRUE + */ + static std::unique_ptr copyString(JNIEnv* env, jstring js, + jboolean* has_exception) { + const char* utf = env->GetStringUTFChars(js, nullptr); + if (utf == nullptr) { + // exception thrown: OutOfMemoryError + env->ExceptionCheck(); + *has_exception = JNI_TRUE; + return nullptr; + } else if (env->ExceptionCheck()) { + // exception thrown + env->ReleaseStringUTFChars(js, utf); + *has_exception = JNI_TRUE; + return nullptr; + } + + const jsize utf_len = env->GetStringUTFLength(js); + std::unique_ptr str( + new char[utf_len + + 1]); // Note: + 1 is needed for the c_str null terminator + std::strcpy(str.get(), utf); + env->ReleaseStringUTFChars(js, utf); + *has_exception = JNI_FALSE; + return str; + } + + /** + * Copies a jstring to a std::string + * and releases the original jstring + * + * If an exception occurs, then JNIEnv::ExceptionCheck() + * will have been called + * + * @param env (IN) A pointer to the java environment + * @param js (IN) The java string to copy + * @param has_exception (OUT) will be set to JNI_TRUE + * if an OutOfMemoryError exception occurs + * + * @return A std:string copy of the jstring, or an + * empty std::string if has_exception == JNI_TRUE + */ + static std::string copyStdString(JNIEnv* env, jstring js, + jboolean* has_exception) { + const char* utf = env->GetStringUTFChars(js, nullptr); + if (utf == nullptr) { + // exception thrown: OutOfMemoryError + env->ExceptionCheck(); + *has_exception = JNI_TRUE; + return std::string(); + } else if (env->ExceptionCheck()) { + // exception thrown + env->ReleaseStringUTFChars(js, utf); + *has_exception = JNI_TRUE; + return std::string(); + } + + std::string name(utf); + env->ReleaseStringUTFChars(js, utf); + *has_exception = JNI_FALSE; + return name; + } + + /** + * Copies bytes from a std::string to a jByteArray + * + * @param env A pointer to the java environment + * @param bytes The bytes to copy + * + * @return the Java byte[], or nullptr if an exception occurs + * + * @throws RocksDBException thrown + * if memory size to copy exceeds general java specific array size + * limitation. + */ + static jbyteArray copyBytes(JNIEnv* env, std::string bytes) { + return createJavaByteArrayWithSizeCheck(env, bytes.c_str(), bytes.size()); + } + + /** + * Given a Java byte[][] which is an array of java.lang.Strings + * where each String is a byte[], the passed function `string_fn` + * will be called on each String, the result is the collected by + * calling the passed function `collector_fn` + * + * @param env (IN) A pointer to the java environment + * @param jbyte_strings (IN) A Java array of Strings expressed as bytes + * @param string_fn (IN) A transform function to call for each String + * @param collector_fn (IN) A collector which is called for the result + * of each `string_fn` + * @param has_exception (OUT) will be set to JNI_TRUE + * if an ArrayIndexOutOfBoundsException or OutOfMemoryError + * exception occurs + */ + template + static void byteStrings(JNIEnv* env, jobjectArray jbyte_strings, + std::function string_fn, + std::function collector_fn, + jboolean* has_exception) { + const jsize jlen = env->GetArrayLength(jbyte_strings); + + for (jsize i = 0; i < jlen; i++) { + jobject jbyte_string_obj = env->GetObjectArrayElement(jbyte_strings, i); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + *has_exception = JNI_TRUE; // signal error + return; + } + + jbyteArray jbyte_string_ary = + reinterpret_cast(jbyte_string_obj); + T result = byteString(env, jbyte_string_ary, string_fn, has_exception); + + env->DeleteLocalRef(jbyte_string_obj); + + if (*has_exception == JNI_TRUE) { + // exception thrown: OutOfMemoryError + return; + } + + collector_fn(i, result); + } + + *has_exception = JNI_FALSE; + } + + /** + * Given a Java String which is expressed as a Java Byte Array byte[], + * the passed function `string_fn` will be called on the String + * and the result returned + * + * @param env (IN) A pointer to the java environment + * @param jbyte_string_ary (IN) A Java String expressed in bytes + * @param string_fn (IN) A transform function to call on the String + * @param has_exception (OUT) will be set to JNI_TRUE + * if an OutOfMemoryError exception occurs + */ + template + static T byteString(JNIEnv* env, jbyteArray jbyte_string_ary, + std::function string_fn, + jboolean* has_exception) { + const jsize jbyte_string_len = env->GetArrayLength(jbyte_string_ary); + return byteString(env, jbyte_string_ary, jbyte_string_len, string_fn, + has_exception); + } + + /** + * Given a Java String which is expressed as a Java Byte Array byte[], + * the passed function `string_fn` will be called on the String + * and the result returned + * + * @param env (IN) A pointer to the java environment + * @param jbyte_string_ary (IN) A Java String expressed in bytes + * @param jbyte_string_len (IN) The length of the Java String + * expressed in bytes + * @param string_fn (IN) A transform function to call on the String + * @param has_exception (OUT) will be set to JNI_TRUE + * if an OutOfMemoryError exception occurs + */ + template + static T byteString(JNIEnv* env, jbyteArray jbyte_string_ary, + const jsize jbyte_string_len, + std::function string_fn, + jboolean* has_exception) { + jbyte* jbyte_string = env->GetByteArrayElements(jbyte_string_ary, nullptr); + if (jbyte_string == nullptr) { + // exception thrown: OutOfMemoryError + *has_exception = JNI_TRUE; + return nullptr; // signal error + } + + T result = + string_fn(reinterpret_cast(jbyte_string), jbyte_string_len); + + env->ReleaseByteArrayElements(jbyte_string_ary, jbyte_string, JNI_ABORT); + + *has_exception = JNI_FALSE; + return result; + } + + /** + * Converts a std::vector to a Java byte[][] where each Java String + * is expressed as a Java Byte Array byte[]. + * + * @param env A pointer to the java environment + * @param strings A vector of Strings + * + * @return A Java array of Strings expressed as bytes, + * or nullptr if an exception is thrown + */ + static jobjectArray stringsBytes(JNIEnv* env, + std::vector strings) { + jclass jcls_ba = ByteJni::getArrayJClass(env); + if (jcls_ba == nullptr) { + // exception occurred + return nullptr; + } + + const jsize len = static_cast(strings.size()); + + jobjectArray jbyte_strings = env->NewObjectArray(len, jcls_ba, nullptr); + if (jbyte_strings == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + for (jsize i = 0; i < len; i++) { + std::string* str = &strings[i]; + const jsize str_len = static_cast(str->size()); + + jbyteArray jbyte_string_ary = env->NewByteArray(str_len); + if (jbyte_string_ary == nullptr) { + // exception thrown: OutOfMemoryError + env->DeleteLocalRef(jbyte_strings); + return nullptr; + } + + env->SetByteArrayRegion( + jbyte_string_ary, 0, str_len, + const_cast(reinterpret_cast(str->c_str()))); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jbyte_string_ary); + env->DeleteLocalRef(jbyte_strings); + return nullptr; + } + + env->SetObjectArrayElement(jbyte_strings, i, jbyte_string_ary); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + // or ArrayStoreException + env->DeleteLocalRef(jbyte_string_ary); + env->DeleteLocalRef(jbyte_strings); + return nullptr; + } + + env->DeleteLocalRef(jbyte_string_ary); + } + + return jbyte_strings; + } + + /** + * Converts a std::vector to a Java String[]. + * + * @param env A pointer to the java environment + * @param strings A vector of Strings + * + * @return A Java array of Strings, + * or nullptr if an exception is thrown + */ + static jobjectArray toJavaStrings(JNIEnv* env, + const std::vector* strings) { + jclass jcls_str = env->FindClass("java/lang/String"); + if (jcls_str == nullptr) { + // exception occurred + return nullptr; + } + + const jsize len = static_cast(strings->size()); + + jobjectArray jstrings = env->NewObjectArray(len, jcls_str, nullptr); + if (jstrings == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + for (jsize i = 0; i < len; i++) { + const std::string* str = &((*strings)[i]); + jstring js = ROCKSDB_NAMESPACE::JniUtil::toJavaString(env, str); + if (js == nullptr) { + env->DeleteLocalRef(jstrings); + return nullptr; + } + + env->SetObjectArrayElement(jstrings, i, js); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + // or ArrayStoreException + env->DeleteLocalRef(js); + env->DeleteLocalRef(jstrings); + return nullptr; + } + } + + return jstrings; + } + + /** + * Creates a Java UTF String from a C++ std::string + * + * @param env A pointer to the java environment + * @param string the C++ std::string + * @param treat_empty_as_null true if empty strings should be treated as null + * + * @return the Java UTF string, or nullptr if the provided string + * is null (or empty and treat_empty_as_null is set), or if an + * exception occurs allocating the Java String. + */ + static jstring toJavaString(JNIEnv* env, const std::string* string, + const bool treat_empty_as_null = false) { + if (string == nullptr) { + return nullptr; + } + + if (treat_empty_as_null && string->empty()) { + return nullptr; + } + + return env->NewStringUTF(string->c_str()); + } + + /** + * Copies bytes to a new jByteArray with the check of java array size + * limitation. + * + * @param bytes pointer to memory to copy to a new jByteArray + * @param size number of bytes to copy + * + * @return the Java byte[], or nullptr if an exception occurs + * + * @throws RocksDBException thrown + * if memory size to copy exceeds general java array size limitation to + * avoid overflow. + */ + static jbyteArray createJavaByteArrayWithSizeCheck(JNIEnv* env, + const char* bytes, + const size_t size) { + // Limitation for java array size is vm specific + // In general it cannot exceed Integer.MAX_VALUE (2^31 - 1) + // Current HotSpot VM limitation for array size is Integer.MAX_VALUE - 5 + // (2^31 - 1 - 5) It means that the next call to env->NewByteArray can still + // end with OutOfMemoryError("Requested array size exceeds VM limit") coming + // from VM + static const size_t MAX_JARRAY_SIZE = (static_cast(1)) << 31; + if (size > MAX_JARRAY_SIZE) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, "Requested array size exceeds VM limit"); + return nullptr; + } + + const jsize jlen = static_cast(size); + jbyteArray jbytes = env->NewByteArray(jlen); + if (jbytes == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + env->SetByteArrayRegion( + jbytes, 0, jlen, + const_cast(reinterpret_cast(bytes))); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jbytes); + return nullptr; + } + + return jbytes; + } + + /** + * Copies bytes from a ROCKSDB_NAMESPACE::Slice to a jByteArray + * + * @param env A pointer to the java environment + * @param bytes The bytes to copy + * + * @return the Java byte[] or nullptr if an exception occurs + * + * @throws RocksDBException thrown + * if memory size to copy exceeds general java specific array size + * limitation. + */ + static jbyteArray copyBytes(JNIEnv* env, const Slice& bytes) { + return createJavaByteArrayWithSizeCheck(env, bytes.data(), bytes.size()); + } + + /* + * Helper for operations on a key and value + * for example WriteBatch->Put + * + * TODO(AR) could be used for RocksDB->Put etc. + */ + static std::unique_ptr kv_op( + std::function + op, + JNIEnv* env, jobject /*jobj*/, jbyteArray jkey, jint jkey_len, + jbyteArray jvalue, jint jvalue_len) { + jbyte* key = env->GetByteArrayElements(jkey, nullptr); + if (env->ExceptionCheck()) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + jbyte* value = env->GetByteArrayElements(jvalue, nullptr); + if (env->ExceptionCheck()) { + // exception thrown: OutOfMemoryError + if (key != nullptr) { + env->ReleaseByteArrayElements(jkey, key, JNI_ABORT); + } + return nullptr; + } + + ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), jkey_len); + ROCKSDB_NAMESPACE::Slice value_slice(reinterpret_cast(value), + jvalue_len); + + auto status = op(key_slice, value_slice); + + if (value != nullptr) { + env->ReleaseByteArrayElements(jvalue, value, JNI_ABORT); + } + if (key != nullptr) { + env->ReleaseByteArrayElements(jkey, key, JNI_ABORT); + } + + return std::unique_ptr( + new ROCKSDB_NAMESPACE::Status(status)); + } + + /* + * Helper for operations on a key + * for example WriteBatch->Delete + * + * TODO(AR) could be used for RocksDB->Delete etc. + */ + static std::unique_ptr k_op( + std::function op, + JNIEnv* env, jobject /*jobj*/, jbyteArray jkey, jint jkey_len) { + jbyte* key = env->GetByteArrayElements(jkey, nullptr); + if (env->ExceptionCheck()) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), jkey_len); + + auto status = op(key_slice); + + if (key != nullptr) { + env->ReleaseByteArrayElements(jkey, key, JNI_ABORT); + } + + return std::unique_ptr( + new ROCKSDB_NAMESPACE::Status(status)); + } + + /* + * Helper for operations on a key which is a region of an array + * Used to extract the common code from seek/seekForPrev. + * Possible that it can be generalised from that. + * + * We use GetByteArrayRegion to copy the key region of the whole array into + * a char[] We suspect this is not much slower than GetByteArrayElements, + * which probably copies anyway. + */ + static void k_op_region(std::function op, + JNIEnv* env, jbyteArray jkey, jint jkey_off, + jint jkey_len) { + const std::unique_ptr key(new char[jkey_len]); + if (key == nullptr) { + jclass oom_class = env->FindClass("/lang/java/OutOfMemoryError"); + env->ThrowNew(oom_class, + "Memory allocation failed in RocksDB JNI function"); + return; + } + env->GetByteArrayRegion(jkey, jkey_off, jkey_len, + reinterpret_cast(key.get())); + if (env->ExceptionCheck()) { + // exception thrown: OutOfMemoryError + return; + } + + ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key.get()), + jkey_len); + op(key_slice); + } + + /* + * Helper for operations on a value + * for example WriteBatchWithIndex->GetFromBatch + */ + static jbyteArray v_op(std::function + op, + JNIEnv* env, jbyteArray jkey, jint jkey_len) { + jbyte* key = env->GetByteArrayElements(jkey, nullptr); + if (env->ExceptionCheck()) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), jkey_len); + + std::string value; + ROCKSDB_NAMESPACE::Status s = op(key_slice, &value); + + if (key != nullptr) { + env->ReleaseByteArrayElements(jkey, key, JNI_ABORT); + } + + if (s.IsNotFound()) { + return nullptr; + } + + if (s.ok()) { + jbyteArray jret_value = + env->NewByteArray(static_cast(value.size())); + if (jret_value == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + env->SetByteArrayRegion( + jret_value, 0, static_cast(value.size()), + const_cast(reinterpret_cast(value.c_str()))); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + if (jret_value != nullptr) { + env->DeleteLocalRef(jret_value); + } + return nullptr; + } + + return jret_value; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return nullptr; + } + + /** + * Creates a vector of C++ pointers from + * a Java array of C++ pointer addresses. + * + * @param env (IN) A pointer to the java environment + * @param pointers (IN) A Java array of C++ pointer addresses + * @param has_exception (OUT) will be set to JNI_TRUE + * if an ArrayIndexOutOfBoundsException or OutOfMemoryError + * exception occurs. + * + * @return A vector of C++ pointers. + */ + template + static std::vector fromJPointers(JNIEnv* env, jlongArray jptrs, + jboolean* has_exception) { + const jsize jptrs_len = env->GetArrayLength(jptrs); + std::vector ptrs; + jlong* jptr = env->GetLongArrayElements(jptrs, nullptr); + if (jptr == nullptr) { + // exception thrown: OutOfMemoryError + *has_exception = JNI_TRUE; + return ptrs; + } + ptrs.reserve(jptrs_len); + for (jsize i = 0; i < jptrs_len; i++) { + ptrs.push_back(reinterpret_cast(jptr[i])); + } + env->ReleaseLongArrayElements(jptrs, jptr, JNI_ABORT); + return ptrs; + } + + /** + * Creates a Java array of C++ pointer addresses + * from a vector of C++ pointers. + * + * @param env (IN) A pointer to the java environment + * @param pointers (IN) A vector of C++ pointers + * @param has_exception (OUT) will be set to JNI_TRUE + * if an ArrayIndexOutOfBoundsException or OutOfMemoryError + * exception occurs + * + * @return Java array of C++ pointer addresses. + */ + template + static jlongArray toJPointers(JNIEnv* env, const std::vector& pointers, + jboolean* has_exception) { + const jsize len = static_cast(pointers.size()); + std::unique_ptr results(new jlong[len]); + std::transform( + pointers.begin(), pointers.end(), results.get(), + [](T* pointer) -> jlong { return GET_CPLUSPLUS_POINTER(pointer); }); + + jlongArray jpointers = env->NewLongArray(len); + if (jpointers == nullptr) { + // exception thrown: OutOfMemoryError + *has_exception = JNI_TRUE; + return nullptr; + } + + env->SetLongArrayRegion(jpointers, 0, len, results.get()); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + *has_exception = JNI_TRUE; + env->DeleteLocalRef(jpointers); + return nullptr; + } + + *has_exception = JNI_FALSE; + + return jpointers; + } + + /* + * Helper for operations on a key and value + * for example WriteBatch->Put + * + * TODO(AR) could be extended to cover returning ROCKSDB_NAMESPACE::Status + * from `op` and used for RocksDB->Put etc. + */ + static void kv_op_direct( + std::function + op, + JNIEnv* env, jobject jkey, jint jkey_off, jint jkey_len, jobject jval, + jint jval_off, jint jval_len) { + char* key = reinterpret_cast(env->GetDirectBufferAddress(jkey)); + if (key == nullptr || + env->GetDirectBufferCapacity(jkey) < (jkey_off + jkey_len)) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, + "Invalid key argument"); + return; + } + + char* value = reinterpret_cast(env->GetDirectBufferAddress(jval)); + if (value == nullptr || + env->GetDirectBufferCapacity(jval) < (jval_off + jval_len)) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, "Invalid value argument"); + return; + } + + key += jkey_off; + value += jval_off; + + ROCKSDB_NAMESPACE::Slice key_slice(key, jkey_len); + ROCKSDB_NAMESPACE::Slice value_slice(value, jval_len); + + op(key_slice, value_slice); + } + + /* + * Helper for operations on a key and value + * for example WriteBatch->Delete + * + * TODO(AR) could be extended to cover returning ROCKSDB_NAMESPACE::Status + * from `op` and used for RocksDB->Delete etc. + */ + static void k_op_direct(std::function op, + JNIEnv* env, jobject jkey, jint jkey_off, + jint jkey_len) { + char* key = reinterpret_cast(env->GetDirectBufferAddress(jkey)); + if (key == nullptr || + env->GetDirectBufferCapacity(jkey) < (jkey_off + jkey_len)) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, + "Invalid key argument"); + return; + } + + key += jkey_off; + + ROCKSDB_NAMESPACE::Slice key_slice(key, jkey_len); + + return op(key_slice); + } + + template + static jint copyToDirect(JNIEnv* env, T& source, jobject jtarget, + jint jtarget_off, jint jtarget_len) { + char* target = + reinterpret_cast(env->GetDirectBufferAddress(jtarget)); + if (target == nullptr || + env->GetDirectBufferCapacity(jtarget) < (jtarget_off + jtarget_len)) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, "Invalid target argument"); + return 0; + } + + target += jtarget_off; + + const jint cvalue_len = static_cast(source.size()); + const jint length = std::min(jtarget_len, cvalue_len); + + memcpy(target, source.data(), length); + + return cvalue_len; + } +}; + +class MapJni : public JavaClass { + public: + /** + * Get the Java Class java.util.Map + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "java/util/Map"); + } + + /** + * Get the Java Method: Map#put + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getMapPutMethodId(JNIEnv* env) { + jclass jlist_clazz = getJClass(env); + if (jlist_clazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID( + jlist_clazz, "put", + "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;"); + assert(mid != nullptr); + return mid; + } +}; + +class HashMapJni : public JavaClass { + public: + /** + * Get the Java Class java.util.HashMap + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "java/util/HashMap"); + } + + /** + * Create a new Java java.util.HashMap object. + * + * @param env A pointer to the Java environment + * + * @return A reference to a Java java.util.HashMap object, or + * nullptr if an an exception occurs + */ + static jobject construct(JNIEnv* env, const uint32_t initial_capacity = 16) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID mid = env->GetMethodID(jclazz, "", "(I)V"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return nullptr; + } + + jobject jhash_map = + env->NewObject(jclazz, mid, static_cast(initial_capacity)); + if (env->ExceptionCheck()) { + return nullptr; + } + + return jhash_map; + } + + /** + * A function which maps a std::pair to a std::pair + * + * @return Either a pointer to a std::pair, or nullptr + * if an error occurs during the mapping + */ + template + using FnMapKV = + std::function>(const std::pair&)>; + + // template ::value_type, std::pair>::value, + // int32_t>::type = 0> static void putAll(JNIEnv* env, const jobject + // jhash_map, I iterator, const FnMapKV &fn_map_kv) { + /** + * Returns true if it succeeds, false if an error occurs + */ + template + static bool putAll(JNIEnv* env, const jobject jhash_map, + iterator_type iterator, iterator_type end, + const FnMapKV& fn_map_kv) { + const jmethodID jmid_put = + ROCKSDB_NAMESPACE::MapJni::getMapPutMethodId(env); + if (jmid_put == nullptr) { + return false; + } + + for (auto it = iterator; it != end; ++it) { + const std::unique_ptr> result = + fn_map_kv(*it); + if (result == nullptr) { + // an error occurred during fn_map_kv + return false; + } + env->CallObjectMethod(jhash_map, jmid_put, result->first, result->second); + if (env->ExceptionCheck()) { + // exception occurred + env->DeleteLocalRef(result->second); + env->DeleteLocalRef(result->first); + return false; + } + + // release local references + env->DeleteLocalRef(result->second); + env->DeleteLocalRef(result->first); + } + + return true; + } + + /** + * Creates a java.util.Map from a std::map + * + * @param env A pointer to the Java environment + * @param map the Cpp map + * + * @return a reference to the Java java.util.Map object, or nullptr if an + * exception occcurred + */ + static jobject fromCppMap(JNIEnv* env, + const std::map* map) { + if (map == nullptr) { + return nullptr; + } + + jobject jhash_map = construct(env, static_cast(map->size())); + if (jhash_map == nullptr) { + // exception occurred + return nullptr; + } + + const ROCKSDB_NAMESPACE::HashMapJni::FnMapKV< + const std::string, const std::string, jobject, jobject> + fn_map_kv = + [env](const std::pair& kv) { + jstring jkey = ROCKSDB_NAMESPACE::JniUtil::toJavaString( + env, &(kv.first), false); + if (env->ExceptionCheck()) { + // an error occurred + return std::unique_ptr>(nullptr); + } + + jstring jvalue = ROCKSDB_NAMESPACE::JniUtil::toJavaString( + env, &(kv.second), true); + if (env->ExceptionCheck()) { + // an error occurred + env->DeleteLocalRef(jkey); + return std::unique_ptr>(nullptr); + } + + return std::unique_ptr>( + new std::pair( + static_cast(jkey), + static_cast(jvalue))); + }; + + if (!putAll(env, jhash_map, map->begin(), map->end(), fn_map_kv)) { + // exception occurred + return nullptr; + } + + return jhash_map; + } + + /** + * Creates a java.util.Map from a std::map + * + * @param env A pointer to the Java environment + * @param map the Cpp map + * + * @return a reference to the Java java.util.Map object, or nullptr if an + * exception occcurred + */ + static jobject fromCppMap(JNIEnv* env, + const std::map* map) { + if (map == nullptr) { + return nullptr; + } + + if (map == nullptr) { + return nullptr; + } + + jobject jhash_map = construct(env, static_cast(map->size())); + if (jhash_map == nullptr) { + // exception occurred + return nullptr; + } + + const ROCKSDB_NAMESPACE::HashMapJni::FnMapKV< + const std::string, const uint32_t, jobject, jobject> + fn_map_kv = + [env](const std::pair& kv) { + jstring jkey = ROCKSDB_NAMESPACE::JniUtil::toJavaString( + env, &(kv.first), false); + if (env->ExceptionCheck()) { + // an error occurred + return std::unique_ptr>(nullptr); + } + + jobject jvalue = ROCKSDB_NAMESPACE::IntegerJni::valueOf( + env, static_cast(kv.second)); + if (env->ExceptionCheck()) { + // an error occurred + env->DeleteLocalRef(jkey); + return std::unique_ptr>(nullptr); + } + + return std::unique_ptr>( + new std::pair(static_cast(jkey), + jvalue)); + }; + + if (!putAll(env, jhash_map, map->begin(), map->end(), fn_map_kv)) { + // exception occurred + return nullptr; + } + + return jhash_map; + } + + /** + * Creates a java.util.Map from a std::map + * + * @param env A pointer to the Java environment + * @param map the Cpp map + * + * @return a reference to the Java java.util.Map object, or nullptr if an + * exception occcurred + */ + static jobject fromCppMap(JNIEnv* env, + const std::map* map) { + if (map == nullptr) { + return nullptr; + } + + jobject jhash_map = construct(env, static_cast(map->size())); + if (jhash_map == nullptr) { + // exception occurred + return nullptr; + } + + const ROCKSDB_NAMESPACE::HashMapJni::FnMapKV< + const std::string, const uint64_t, jobject, jobject> + fn_map_kv = + [env](const std::pair& kv) { + jstring jkey = ROCKSDB_NAMESPACE::JniUtil::toJavaString( + env, &(kv.first), false); + if (env->ExceptionCheck()) { + // an error occurred + return std::unique_ptr>(nullptr); + } + + jobject jvalue = ROCKSDB_NAMESPACE::LongJni::valueOf( + env, static_cast(kv.second)); + if (env->ExceptionCheck()) { + // an error occurred + env->DeleteLocalRef(jkey); + return std::unique_ptr>(nullptr); + } + + return std::unique_ptr>( + new std::pair(static_cast(jkey), + jvalue)); + }; + + if (!putAll(env, jhash_map, map->begin(), map->end(), fn_map_kv)) { + // exception occurred + return nullptr; + } + + return jhash_map; + } + + /** + * Creates a java.util.Map from a std::map + * + * @param env A pointer to the Java environment + * @param map the Cpp map + * + * @return a reference to the Java java.util.Map object, or nullptr if an + * exception occcurred + */ + static jobject fromCppMap(JNIEnv* env, + const std::map* map) { + if (map == nullptr) { + return nullptr; + } + + jobject jhash_map = construct(env, static_cast(map->size())); + if (jhash_map == nullptr) { + // exception occurred + return nullptr; + } + + const ROCKSDB_NAMESPACE::HashMapJni::FnMapKV + fn_map_kv = [env](const std::pair& kv) { + jobject jkey = ROCKSDB_NAMESPACE::IntegerJni::valueOf( + env, static_cast(kv.first)); + if (env->ExceptionCheck()) { + // an error occurred + return std::unique_ptr>(nullptr); + } + + jobject jvalue = ROCKSDB_NAMESPACE::LongJni::valueOf( + env, static_cast(kv.second)); + if (env->ExceptionCheck()) { + // an error occurred + env->DeleteLocalRef(jkey); + return std::unique_ptr>(nullptr); + } + + return std::unique_ptr>( + new std::pair(static_cast(jkey), + jvalue)); + }; + + if (!putAll(env, jhash_map, map->begin(), map->end(), fn_map_kv)) { + // exception occurred + return nullptr; + } + + return jhash_map; + } +}; + +// The portal class for org.rocksdb.RocksDB +class RocksDBJni + : public RocksDBNativeClass { + public: + /** + * Get the Java Class org.rocksdb.RocksDB + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, "org/rocksdb/RocksDB"); + } +}; + +// The portal class for org.rocksdb.Options +class OptionsJni + : public RocksDBNativeClass { + public: + /** + * Get the Java Class org.rocksdb.Options + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, "org/rocksdb/Options"); + } +}; + +// The portal class for org.rocksdb.DBOptions +class DBOptionsJni + : public RocksDBNativeClass { + public: + /** + * Get the Java Class org.rocksdb.DBOptions + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, "org/rocksdb/DBOptions"); + } +}; + +// The portal class for org.rocksdb.ColumnFamilyOptions +class ColumnFamilyOptionsJni + : public RocksDBNativeClass { + public: + /** + * Get the Java Class org.rocksdb.ColumnFamilyOptions + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, + "org/rocksdb/ColumnFamilyOptions"); + } + + /** + * Create a new Java org.rocksdb.ColumnFamilyOptions object with the same + * properties as the provided C++ ROCKSDB_NAMESPACE::ColumnFamilyOptions + * object + * + * @param env A pointer to the Java environment + * @param cfoptions A pointer to ROCKSDB_NAMESPACE::ColumnFamilyOptions object + * + * @return A reference to a Java org.rocksdb.ColumnFamilyOptions object, or + * nullptr if an an exception occurs + */ + static jobject construct(JNIEnv* env, const ColumnFamilyOptions* cfoptions) { + auto* cfo = new ROCKSDB_NAMESPACE::ColumnFamilyOptions(*cfoptions); + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID mid = env->GetMethodID(jclazz, "", "(J)V"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return nullptr; + } + + jobject jcfd = env->NewObject(jclazz, mid, GET_CPLUSPLUS_POINTER(cfo)); + if (env->ExceptionCheck()) { + return nullptr; + } + + return jcfd; + } +}; + +// The portal class for org.rocksdb.WriteOptions +class WriteOptionsJni + : public RocksDBNativeClass { + public: + /** + * Get the Java Class org.rocksdb.WriteOptions + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, "org/rocksdb/WriteOptions"); + } +}; + +// The portal class for org.rocksdb.ReadOptions +class ReadOptionsJni + : public RocksDBNativeClass { + public: + /** + * Get the Java Class org.rocksdb.ReadOptions + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, "org/rocksdb/ReadOptions"); + } +}; + +// The portal class for org.rocksdb.WriteBatch +class WriteBatchJni + : public RocksDBNativeClass { + public: + /** + * Get the Java Class org.rocksdb.WriteBatch + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, "org/rocksdb/WriteBatch"); + } + + /** + * Create a new Java org.rocksdb.WriteBatch object + * + * @param env A pointer to the Java environment + * @param wb A pointer to ROCKSDB_NAMESPACE::WriteBatch object + * + * @return A reference to a Java org.rocksdb.WriteBatch object, or + * nullptr if an an exception occurs + */ + static jobject construct(JNIEnv* env, const WriteBatch* wb) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID mid = env->GetMethodID(jclazz, "", "(J)V"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return nullptr; + } + + jobject jwb = env->NewObject(jclazz, mid, GET_CPLUSPLUS_POINTER(wb)); + if (env->ExceptionCheck()) { + return nullptr; + } + + return jwb; + } +}; + +// The portal class for org.rocksdb.WriteBatch.Handler +class WriteBatchHandlerJni + : public RocksDBNativeClass< + const ROCKSDB_NAMESPACE::WriteBatchHandlerJniCallback*, + WriteBatchHandlerJni> { + public: + /** + * Get the Java Class org.rocksdb.WriteBatch.Handler + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, "org/rocksdb/WriteBatch$Handler"); + } + + /** + * Get the Java Method: WriteBatch.Handler#put + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getPutCfMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "put", "(I[B[B)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: WriteBatch.Handler#put + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getPutMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "put", "([B[B)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: WriteBatch.Handler#merge + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getMergeCfMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "merge", "(I[B[B)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: WriteBatch.Handler#merge + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getMergeMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "merge", "([B[B)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: WriteBatch.Handler#delete + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getDeleteCfMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "delete", "(I[B)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: WriteBatch.Handler#delete + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getDeleteMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "delete", "([B)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: WriteBatch.Handler#singleDelete + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getSingleDeleteCfMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "singleDelete", "(I[B)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: WriteBatch.Handler#singleDelete + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getSingleDeleteMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "singleDelete", "([B)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: WriteBatch.Handler#deleteRange + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getDeleteRangeCfMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "deleteRange", "(I[B[B)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: WriteBatch.Handler#deleteRange + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getDeleteRangeMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "deleteRange", "([B[B)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: WriteBatch.Handler#logData + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getLogDataMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "logData", "([B)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: WriteBatch.Handler#putBlobIndex + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getPutBlobIndexCfMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "putBlobIndex", "(I[B[B)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: WriteBatch.Handler#markBeginPrepare + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getMarkBeginPrepareMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "markBeginPrepare", "()V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: WriteBatch.Handler#markEndPrepare + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getMarkEndPrepareMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "markEndPrepare", "([B)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: WriteBatch.Handler#markNoop + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getMarkNoopMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "markNoop", "(Z)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: WriteBatch.Handler#markRollback + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getMarkRollbackMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "markRollback", "([B)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: WriteBatch.Handler#markCommit + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getMarkCommitMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "markCommit", "([B)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: WriteBatch.Handler#markCommitWithTimestamp + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getMarkCommitWithTimestampMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = + env->GetMethodID(jclazz, "markCommitWithTimestamp", "([B[B)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: WriteBatch.Handler#shouldContinue + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getContinueMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "shouldContinue", "()Z"); + assert(mid != nullptr); + return mid; + } +}; + +class WriteBatchSavePointJni : public JavaClass { + public: + /** + * Get the Java Class org.rocksdb.WriteBatch.SavePoint + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/WriteBatch$SavePoint"); + } + + /** + * Get the Java Method: HistogramData constructor + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getConstructorMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "", "(JJJ)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Create a new Java org.rocksdb.WriteBatch.SavePoint object + * + * @param env A pointer to the Java environment + * @param savePoint A pointer to ROCKSDB_NAMESPACE::WriteBatch::SavePoint + * object + * + * @return A reference to a Java org.rocksdb.WriteBatch.SavePoint object, or + * nullptr if an an exception occurs + */ + static jobject construct(JNIEnv* env, const SavePoint& save_point) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID mid = getConstructorMethodId(env); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return nullptr; + } + + jobject jsave_point = + env->NewObject(jclazz, mid, static_cast(save_point.size), + static_cast(save_point.count), + static_cast(save_point.content_flags)); + if (env->ExceptionCheck()) { + return nullptr; + } + + return jsave_point; + } +}; + +// The portal class for org.rocksdb.WriteBatchWithIndex +class WriteBatchWithIndexJni + : public RocksDBNativeClass { + public: + /** + * Get the Java Class org.rocksdb.WriteBatchWithIndex + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, + "org/rocksdb/WriteBatchWithIndex"); + } +}; + +// The portal class for org.rocksdb.HistogramData +class HistogramDataJni : public JavaClass { + public: + /** + * Get the Java Class org.rocksdb.HistogramData + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/HistogramData"); + } + + /** + * Get the Java Method: HistogramData constructor + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getConstructorMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "", "(DDDDDDJJD)V"); + assert(mid != nullptr); + return mid; + } +}; + +// The portal class for org.rocksdb.BackupEngineOptions +class BackupEngineOptionsJni + : public RocksDBNativeClass { + public: + /** + * Get the Java Class org.rocksdb.BackupEngineOptions + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, + "org/rocksdb/BackupEngineOptions"); + } +}; + +// The portal class for org.rocksdb.BackupEngine +class BackupEngineJni + : public RocksDBNativeClass { + public: + /** + * Get the Java Class org.rocksdb.BackupEngine + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, "org/rocksdb/BackupEngine"); + } +}; + +// The portal class for org.rocksdb.RocksIterator +class IteratorJni + : public RocksDBNativeClass { + public: + /** + * Get the Java Class org.rocksdb.RocksIterator + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, "org/rocksdb/RocksIterator"); + } +}; + +// The portal class for org.rocksdb.Filter +class FilterJni + : public RocksDBNativeClass< + std::shared_ptr*, FilterJni> { + public: + /** + * Get the Java Class org.rocksdb.Filter + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, "org/rocksdb/Filter"); + } +}; + +// The portal class for org.rocksdb.ColumnFamilyHandle +class ColumnFamilyHandleJni + : public RocksDBNativeClass { + public: + static jobject fromCppColumnFamilyHandle( + JNIEnv* env, const ROCKSDB_NAMESPACE::ColumnFamilyHandle* info) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID ctor = getConstructorMethodId(env, jclazz); + assert(ctor != nullptr); + return env->NewObject(jclazz, ctor, GET_CPLUSPLUS_POINTER(info)); + } + + static jmethodID getConstructorMethodId(JNIEnv* env, jclass clazz) { + return env->GetMethodID(clazz, "", "(J)V"); + } + + /** + * Get the Java Class org.rocksdb.ColumnFamilyHandle + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, "org/rocksdb/ColumnFamilyHandle"); + } +}; + +// The portal class for org.rocksdb.FlushOptions +class FlushOptionsJni + : public RocksDBNativeClass { + public: + /** + * Get the Java Class org.rocksdb.FlushOptions + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, "org/rocksdb/FlushOptions"); + } +}; + +// The portal class for org.rocksdb.ComparatorOptions +class ComparatorOptionsJni + : public RocksDBNativeClass< + ROCKSDB_NAMESPACE::ComparatorJniCallbackOptions*, + ComparatorOptionsJni> { + public: + /** + * Get the Java Class org.rocksdb.ComparatorOptions + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, "org/rocksdb/ComparatorOptions"); + } +}; + +// The portal class for org.rocksdb.AbstractCompactionFilterFactory +class AbstractCompactionFilterFactoryJni + : public RocksDBNativeClass< + const ROCKSDB_NAMESPACE::CompactionFilterFactoryJniCallback*, + AbstractCompactionFilterFactoryJni> { + public: + /** + * Get the Java Class org.rocksdb.AbstractCompactionFilterFactory + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass( + env, "org/rocksdb/AbstractCompactionFilterFactory"); + } + + /** + * Get the Java Method: AbstractCompactionFilterFactory#name + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getNameMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = + env->GetMethodID(jclazz, "name", "()Ljava/lang/String;"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractCompactionFilterFactory#createCompactionFilter + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getCreateCompactionFilterMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = + env->GetMethodID(jclazz, "createCompactionFilter", "(ZZ)J"); + assert(mid != nullptr); + return mid; + } +}; + +// The portal class for org.rocksdb.AbstractTransactionNotifier +class AbstractTransactionNotifierJni + : public RocksDBNativeClass< + const ROCKSDB_NAMESPACE::TransactionNotifierJniCallback*, + AbstractTransactionNotifierJni> { + public: + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass( + env, "org/rocksdb/AbstractTransactionNotifier"); + } + + // Get the java method `snapshotCreated` + // of org.rocksdb.AbstractTransactionNotifier. + static jmethodID getSnapshotCreatedMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "snapshotCreated", "(J)V"); + assert(mid != nullptr); + return mid; + } +}; + +// The portal class for org.rocksdb.AbstractComparatorJniBridge +class AbstractComparatorJniBridge : public JavaClass { + public: + /** + * Get the Java Class org.rocksdb.AbstractComparatorJniBridge + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/AbstractComparatorJniBridge"); + } + + /** + * Get the Java Method: Comparator#compareInternal + * + * @param env A pointer to the Java environment + * @param jclazz the AbstractComparatorJniBridge class + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getCompareInternalMethodId(JNIEnv* env, jclass jclazz) { + static jmethodID mid = + env->GetStaticMethodID(jclazz, "compareInternal", + "(Lorg/rocksdb/AbstractComparator;Ljava/nio/" + "ByteBuffer;ILjava/nio/ByteBuffer;I)I"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: Comparator#findShortestSeparatorInternal + * + * @param env A pointer to the Java environment + * @param jclazz the AbstractComparatorJniBridge class + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getFindShortestSeparatorInternalMethodId(JNIEnv* env, + jclass jclazz) { + static jmethodID mid = + env->GetStaticMethodID(jclazz, "findShortestSeparatorInternal", + "(Lorg/rocksdb/AbstractComparator;Ljava/nio/" + "ByteBuffer;ILjava/nio/ByteBuffer;I)I"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: Comparator#findShortSuccessorInternal + * + * @param env A pointer to the Java environment + * @param jclazz the AbstractComparatorJniBridge class + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getFindShortSuccessorInternalMethodId(JNIEnv* env, + jclass jclazz) { + static jmethodID mid = env->GetStaticMethodID( + jclazz, "findShortSuccessorInternal", + "(Lorg/rocksdb/AbstractComparator;Ljava/nio/ByteBuffer;I)I"); + assert(mid != nullptr); + return mid; + } +}; + +// The portal class for org.rocksdb.AbstractComparator +class AbstractComparatorJni + : public RocksDBNativeClass { + public: + /** + * Get the Java Class org.rocksdb.AbstractComparator + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, "org/rocksdb/AbstractComparator"); + } + + /** + * Get the Java Method: Comparator#name + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getNameMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = + env->GetMethodID(jclazz, "name", "()Ljava/lang/String;"); + assert(mid != nullptr); + return mid; + } +}; + +// The portal class for org.rocksdb.AbstractSlice +class AbstractSliceJni + : public NativeRocksMutableObject { + public: + /** + * Get the Java Class org.rocksdb.AbstractSlice + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, "org/rocksdb/AbstractSlice"); + } +}; + +// The portal class for org.rocksdb.Slice +class SliceJni + : public NativeRocksMutableObject { + public: + /** + * Get the Java Class org.rocksdb.Slice + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, "org/rocksdb/Slice"); + } + + /** + * Constructs a Slice object + * + * @param env A pointer to the Java environment + * + * @return A reference to a Java Slice object, or a nullptr if an + * exception occurs + */ + static jobject construct0(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "", "()V"); + if (mid == nullptr) { + // exception occurred accessing method + return nullptr; + } + + jobject jslice = env->NewObject(jclazz, mid); + if (env->ExceptionCheck()) { + return nullptr; + } + + return jslice; + } +}; + +// The portal class for org.rocksdb.DirectSlice +class DirectSliceJni + : public NativeRocksMutableObject { + public: + /** + * Get the Java Class org.rocksdb.DirectSlice + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, "org/rocksdb/DirectSlice"); + } + + /** + * Constructs a DirectSlice object + * + * @param env A pointer to the Java environment + * + * @return A reference to a Java DirectSlice object, or a nullptr if an + * exception occurs + */ + static jobject construct0(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "", "()V"); + if (mid == nullptr) { + // exception occurred accessing method + return nullptr; + } + + jobject jdirect_slice = env->NewObject(jclazz, mid); + if (env->ExceptionCheck()) { + return nullptr; + } + + return jdirect_slice; + } +}; + +// The portal class for org.rocksdb.BackupInfo +class BackupInfoJni : public JavaClass { + public: + /** + * Get the Java Class org.rocksdb.BackupInfo + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/BackupInfo"); + } + + /** + * Constructs a BackupInfo object + * + * @param env A pointer to the Java environment + * @param backup_id id of the backup + * @param timestamp timestamp of the backup + * @param size size of the backup + * @param number_files number of files related to the backup + * @param app_metadata application specific metadata + * + * @return A reference to a Java BackupInfo object, or a nullptr if an + * exception occurs + */ + static jobject construct0(JNIEnv* env, uint32_t backup_id, int64_t timestamp, + uint64_t size, uint32_t number_files, + const std::string& app_metadata) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = + env->GetMethodID(jclazz, "", "(IJJILjava/lang/String;)V"); + if (mid == nullptr) { + // exception occurred accessing method + return nullptr; + } + + jstring japp_metadata = nullptr; + if (app_metadata != nullptr) { + japp_metadata = env->NewStringUTF(app_metadata.c_str()); + if (japp_metadata == nullptr) { + // exception occurred creating java string + return nullptr; + } + } + + jobject jbackup_info = env->NewObject(jclazz, mid, backup_id, timestamp, + size, number_files, japp_metadata); + if (env->ExceptionCheck()) { + env->DeleteLocalRef(japp_metadata); + return nullptr; + } + + return jbackup_info; + } +}; + +class BackupInfoListJni { + public: + /** + * Converts a C++ std::vector object to + * a Java ArrayList object + * + * @param env A pointer to the Java environment + * @param backup_infos A vector of BackupInfo + * + * @return Either a reference to a Java ArrayList object, or a nullptr + * if an exception occurs + */ + static jobject getBackupInfo(JNIEnv* env, + std::vector backup_infos) { + jclass jarray_list_clazz = + ROCKSDB_NAMESPACE::ListJni::getArrayListClass(env); + if (jarray_list_clazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID cstr_mid = + ROCKSDB_NAMESPACE::ListJni::getArrayListConstructorMethodId(env); + if (cstr_mid == nullptr) { + // exception occurred accessing method + return nullptr; + } + + jmethodID add_mid = ROCKSDB_NAMESPACE::ListJni::getListAddMethodId(env); + if (add_mid == nullptr) { + // exception occurred accessing method + return nullptr; + } + + // create java list + jobject jbackup_info_handle_list = + env->NewObject(jarray_list_clazz, cstr_mid, backup_infos.size()); + if (env->ExceptionCheck()) { + // exception occurred constructing object + return nullptr; + } + + // insert in java list + auto end = backup_infos.end(); + for (auto it = backup_infos.begin(); it != end; ++it) { + auto backup_info = *it; + + jobject obj = ROCKSDB_NAMESPACE::BackupInfoJni::construct0( + env, backup_info.backup_id, backup_info.timestamp, backup_info.size, + backup_info.number_files, backup_info.app_metadata); + if (env->ExceptionCheck()) { + // exception occurred constructing object + if (obj != nullptr) { + env->DeleteLocalRef(obj); + } + if (jbackup_info_handle_list != nullptr) { + env->DeleteLocalRef(jbackup_info_handle_list); + } + return nullptr; + } + + jboolean rs = + env->CallBooleanMethod(jbackup_info_handle_list, add_mid, obj); + if (env->ExceptionCheck() || rs == JNI_FALSE) { + // exception occurred calling method, or could not add + if (obj != nullptr) { + env->DeleteLocalRef(obj); + } + if (jbackup_info_handle_list != nullptr) { + env->DeleteLocalRef(jbackup_info_handle_list); + } + return nullptr; + } + } + + return jbackup_info_handle_list; + } +}; + +// The portal class for org.rocksdb.WBWIRocksIterator +class WBWIRocksIteratorJni : public JavaClass { + public: + /** + * Get the Java Class org.rocksdb.WBWIRocksIterator + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/WBWIRocksIterator"); + } + + /** + * Get the Java Field: WBWIRocksIterator#entry + * + * @param env A pointer to the Java environment + * + * @return The Java Field ID or nullptr if the class or field id could not + * be retrieved + */ + static jfieldID getWriteEntryField(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jfieldID fid = env->GetFieldID( + jclazz, "entry", "Lorg/rocksdb/WBWIRocksIterator$WriteEntry;"); + assert(fid != nullptr); + return fid; + } + + /** + * Gets the value of the WBWIRocksIterator#entry + * + * @param env A pointer to the Java environment + * @param jwbwi_rocks_iterator A reference to a WBWIIterator + * + * @return A reference to a Java WBWIRocksIterator.WriteEntry object, or + * a nullptr if an exception occurs + */ + static jobject getWriteEntry(JNIEnv* env, jobject jwbwi_rocks_iterator) { + assert(jwbwi_rocks_iterator != nullptr); + + jfieldID jwrite_entry_field = getWriteEntryField(env); + if (jwrite_entry_field == nullptr) { + // exception occurred accessing the field + return nullptr; + } + + jobject jwe = env->GetObjectField(jwbwi_rocks_iterator, jwrite_entry_field); + assert(jwe != nullptr); + return jwe; + } +}; + +// The portal class for org.rocksdb.WBWIRocksIterator.WriteType +class WriteTypeJni : public JavaClass { + public: + /** + * Get the PUT enum field value of WBWIRocksIterator.WriteType + * + * @param env A pointer to the Java environment + * + * @return A reference to the enum field value or a nullptr if + * the enum field value could not be retrieved + */ + static jobject PUT(JNIEnv* env) { return getEnum(env, "PUT"); } + + /** + * Get the MERGE enum field value of WBWIRocksIterator.WriteType + * + * @param env A pointer to the Java environment + * + * @return A reference to the enum field value or a nullptr if + * the enum field value could not be retrieved + */ + static jobject MERGE(JNIEnv* env) { return getEnum(env, "MERGE"); } + + /** + * Get the DELETE enum field value of WBWIRocksIterator.WriteType + * + * @param env A pointer to the Java environment + * + * @return A reference to the enum field value or a nullptr if + * the enum field value could not be retrieved + */ + static jobject DELETE(JNIEnv* env) { return getEnum(env, "DELETE"); } + + /** + * Get the LOG enum field value of WBWIRocksIterator.WriteType + * + * @param env A pointer to the Java environment + * + * @return A reference to the enum field value or a nullptr if + * the enum field value could not be retrieved + */ + static jobject LOG(JNIEnv* env) { return getEnum(env, "LOG"); } + + // Returns the equivalent org.rocksdb.WBWIRocksIterator.WriteType for the + // provided C++ ROCKSDB_NAMESPACE::WriteType enum + static jbyte toJavaWriteType(const ROCKSDB_NAMESPACE::WriteType& writeType) { + switch (writeType) { + case ROCKSDB_NAMESPACE::WriteType::kPutRecord: + return 0x0; + case ROCKSDB_NAMESPACE::WriteType::kMergeRecord: + return 0x1; + case ROCKSDB_NAMESPACE::WriteType::kDeleteRecord: + return 0x2; + case ROCKSDB_NAMESPACE::WriteType::kSingleDeleteRecord: + return 0x3; + case ROCKSDB_NAMESPACE::WriteType::kDeleteRangeRecord: + return 0x4; + case ROCKSDB_NAMESPACE::WriteType::kLogDataRecord: + return 0x5; + case ROCKSDB_NAMESPACE::WriteType::kXIDRecord: + return 0x6; + default: + return 0x7F; // undefined + } + } + + private: + /** + * Get the Java Class org.rocksdb.WBWIRocksIterator.WriteType + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/WBWIRocksIterator$WriteType"); + } + + /** + * Get an enum field of org.rocksdb.WBWIRocksIterator.WriteType + * + * @param env A pointer to the Java environment + * @param name The name of the enum field + * + * @return A reference to the enum field value or a nullptr if + * the enum field value could not be retrieved + */ + static jobject getEnum(JNIEnv* env, const char name[]) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jfieldID jfid = env->GetStaticFieldID( + jclazz, name, "Lorg/rocksdb/WBWIRocksIterator$WriteType;"); + if (env->ExceptionCheck()) { + // exception occurred while getting field + return nullptr; + } else if (jfid == nullptr) { + return nullptr; + } + + jobject jwrite_type = env->GetStaticObjectField(jclazz, jfid); + assert(jwrite_type != nullptr); + return jwrite_type; + } +}; + +// The portal class for org.rocksdb.WBWIRocksIterator.WriteEntry +class WriteEntryJni : public JavaClass { + public: + /** + * Get the Java Class org.rocksdb.WBWIRocksIterator.WriteEntry + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, + "org/rocksdb/WBWIRocksIterator$WriteEntry"); + } +}; + +// The portal class for org.rocksdb.InfoLogLevel +class InfoLogLevelJni : public JavaClass { + public: + /** + * Get the DEBUG_LEVEL enum field value of InfoLogLevel + * + * @param env A pointer to the Java environment + * + * @return A reference to the enum field value or a nullptr if + * the enum field value could not be retrieved + */ + static jobject DEBUG_LEVEL(JNIEnv* env) { + return getEnum(env, "DEBUG_LEVEL"); + } + + /** + * Get the INFO_LEVEL enum field value of InfoLogLevel + * + * @param env A pointer to the Java environment + * + * @return A reference to the enum field value or a nullptr if + * the enum field value could not be retrieved + */ + static jobject INFO_LEVEL(JNIEnv* env) { return getEnum(env, "INFO_LEVEL"); } + + /** + * Get the WARN_LEVEL enum field value of InfoLogLevel + * + * @param env A pointer to the Java environment + * + * @return A reference to the enum field value or a nullptr if + * the enum field value could not be retrieved + */ + static jobject WARN_LEVEL(JNIEnv* env) { return getEnum(env, "WARN_LEVEL"); } + + /** + * Get the ERROR_LEVEL enum field value of InfoLogLevel + * + * @param env A pointer to the Java environment + * + * @return A reference to the enum field value or a nullptr if + * the enum field value could not be retrieved + */ + static jobject ERROR_LEVEL(JNIEnv* env) { + return getEnum(env, "ERROR_LEVEL"); + } + + /** + * Get the FATAL_LEVEL enum field value of InfoLogLevel + * + * @param env A pointer to the Java environment + * + * @return A reference to the enum field value or a nullptr if + * the enum field value could not be retrieved + */ + static jobject FATAL_LEVEL(JNIEnv* env) { + return getEnum(env, "FATAL_LEVEL"); + } + + /** + * Get the HEADER_LEVEL enum field value of InfoLogLevel + * + * @param env A pointer to the Java environment + * + * @return A reference to the enum field value or a nullptr if + * the enum field value could not be retrieved + */ + static jobject HEADER_LEVEL(JNIEnv* env) { + return getEnum(env, "HEADER_LEVEL"); + } + + private: + /** + * Get the Java Class org.rocksdb.InfoLogLevel + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/InfoLogLevel"); + } + + /** + * Get an enum field of org.rocksdb.InfoLogLevel + * + * @param env A pointer to the Java environment + * @param name The name of the enum field + * + * @return A reference to the enum field value or a nullptr if + * the enum field value could not be retrieved + */ + static jobject getEnum(JNIEnv* env, const char name[]) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jfieldID jfid = + env->GetStaticFieldID(jclazz, name, "Lorg/rocksdb/InfoLogLevel;"); + if (env->ExceptionCheck()) { + // exception occurred while getting field + return nullptr; + } else if (jfid == nullptr) { + return nullptr; + } + + jobject jinfo_log_level = env->GetStaticObjectField(jclazz, jfid); + assert(jinfo_log_level != nullptr); + return jinfo_log_level; + } +}; + +// The portal class for org.rocksdb.Logger +class LoggerJni + : public RocksDBNativeClass< + std::shared_ptr*, LoggerJni> { + public: + /** + * Get the Java Class org/rocksdb/Logger + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, "org/rocksdb/Logger"); + } + + /** + * Get the Java Method: Logger#log + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getLogMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID( + jclazz, "log", "(Lorg/rocksdb/InfoLogLevel;Ljava/lang/String;)V"); + assert(mid != nullptr); + return mid; + } +}; + +// The portal class for org.rocksdb.TransactionLogIterator.BatchResult +class BatchResultJni : public JavaClass { + public: + /** + * Get the Java Class org.rocksdb.TransactionLogIterator.BatchResult + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass( + env, "org/rocksdb/TransactionLogIterator$BatchResult"); + } + + /** + * Create a new Java org.rocksdb.TransactionLogIterator.BatchResult object + * with the same properties as the provided C++ ROCKSDB_NAMESPACE::BatchResult + * object + * + * @param env A pointer to the Java environment + * @param batch_result The ROCKSDB_NAMESPACE::BatchResult object + * + * @return A reference to a Java + * org.rocksdb.TransactionLogIterator.BatchResult object, + * or nullptr if an an exception occurs + */ + static jobject construct(JNIEnv* env, + ROCKSDB_NAMESPACE::BatchResult& batch_result) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID mid = env->GetMethodID(jclazz, "", "(JJ)V"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return nullptr; + } + + jobject jbatch_result = env->NewObject(jclazz, mid, batch_result.sequence, + batch_result.writeBatchPtr.get()); + if (jbatch_result == nullptr) { + // exception thrown: InstantiationException or OutOfMemoryError + return nullptr; + } + + batch_result.writeBatchPtr.release(); + return jbatch_result; + } +}; + +// The portal class for org.rocksdb.BottommostLevelCompaction +class BottommostLevelCompactionJni { + public: + // Returns the equivalent org.rocksdb.BottommostLevelCompaction for the + // provided C++ ROCKSDB_NAMESPACE::BottommostLevelCompaction enum + static jint toJavaBottommostLevelCompaction( + const ROCKSDB_NAMESPACE::BottommostLevelCompaction& + bottommost_level_compaction) { + switch (bottommost_level_compaction) { + case ROCKSDB_NAMESPACE::BottommostLevelCompaction::kSkip: + return 0x0; + case ROCKSDB_NAMESPACE::BottommostLevelCompaction:: + kIfHaveCompactionFilter: + return 0x1; + case ROCKSDB_NAMESPACE::BottommostLevelCompaction::kForce: + return 0x2; + case ROCKSDB_NAMESPACE::BottommostLevelCompaction::kForceOptimized: + return 0x3; + default: + return 0x7F; // undefined + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::BottommostLevelCompaction + // enum for the provided Java org.rocksdb.BottommostLevelCompaction + static ROCKSDB_NAMESPACE::BottommostLevelCompaction + toCppBottommostLevelCompaction(jint bottommost_level_compaction) { + switch (bottommost_level_compaction) { + case 0x0: + return ROCKSDB_NAMESPACE::BottommostLevelCompaction::kSkip; + case 0x1: + return ROCKSDB_NAMESPACE::BottommostLevelCompaction:: + kIfHaveCompactionFilter; + case 0x2: + return ROCKSDB_NAMESPACE::BottommostLevelCompaction::kForce; + case 0x3: + return ROCKSDB_NAMESPACE::BottommostLevelCompaction::kForceOptimized; + default: + // undefined/default + return ROCKSDB_NAMESPACE::BottommostLevelCompaction:: + kIfHaveCompactionFilter; + } + } +}; + +// The portal class for org.rocksdb.CompactionStopStyle +class CompactionStopStyleJni { + public: + // Returns the equivalent org.rocksdb.CompactionStopStyle for the provided + // C++ ROCKSDB_NAMESPACE::CompactionStopStyle enum + static jbyte toJavaCompactionStopStyle( + const ROCKSDB_NAMESPACE::CompactionStopStyle& compaction_stop_style) { + switch (compaction_stop_style) { + case ROCKSDB_NAMESPACE::CompactionStopStyle:: + kCompactionStopStyleSimilarSize: + return 0x0; + case ROCKSDB_NAMESPACE::CompactionStopStyle:: + kCompactionStopStyleTotalSize: + return 0x1; + default: + return 0x7F; // undefined + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::CompactionStopStyle enum for + // the provided Java org.rocksdb.CompactionStopStyle + static ROCKSDB_NAMESPACE::CompactionStopStyle toCppCompactionStopStyle( + jbyte jcompaction_stop_style) { + switch (jcompaction_stop_style) { + case 0x0: + return ROCKSDB_NAMESPACE::CompactionStopStyle:: + kCompactionStopStyleSimilarSize; + case 0x1: + return ROCKSDB_NAMESPACE::CompactionStopStyle:: + kCompactionStopStyleTotalSize; + default: + // undefined/default + return ROCKSDB_NAMESPACE::CompactionStopStyle:: + kCompactionStopStyleSimilarSize; + } + } +}; + +// The portal class for org.rocksdb.CompressionType +class CompressionTypeJni { + public: + // Returns the equivalent org.rocksdb.CompressionType for the provided + // C++ ROCKSDB_NAMESPACE::CompressionType enum + static jbyte toJavaCompressionType( + const ROCKSDB_NAMESPACE::CompressionType& compression_type) { + switch (compression_type) { + case ROCKSDB_NAMESPACE::CompressionType::kNoCompression: + return 0x0; + case ROCKSDB_NAMESPACE::CompressionType::kSnappyCompression: + return 0x1; + case ROCKSDB_NAMESPACE::CompressionType::kZlibCompression: + return 0x2; + case ROCKSDB_NAMESPACE::CompressionType::kBZip2Compression: + return 0x3; + case ROCKSDB_NAMESPACE::CompressionType::kLZ4Compression: + return 0x4; + case ROCKSDB_NAMESPACE::CompressionType::kLZ4HCCompression: + return 0x5; + case ROCKSDB_NAMESPACE::CompressionType::kXpressCompression: + return 0x6; + case ROCKSDB_NAMESPACE::CompressionType::kZSTD: + return 0x7; + case ROCKSDB_NAMESPACE::CompressionType::kDisableCompressionOption: + default: + return 0x7F; + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::CompressionType enum for the + // provided Java org.rocksdb.CompressionType + static ROCKSDB_NAMESPACE::CompressionType toCppCompressionType( + jbyte jcompression_type) { + switch (jcompression_type) { + case 0x0: + return ROCKSDB_NAMESPACE::CompressionType::kNoCompression; + case 0x1: + return ROCKSDB_NAMESPACE::CompressionType::kSnappyCompression; + case 0x2: + return ROCKSDB_NAMESPACE::CompressionType::kZlibCompression; + case 0x3: + return ROCKSDB_NAMESPACE::CompressionType::kBZip2Compression; + case 0x4: + return ROCKSDB_NAMESPACE::CompressionType::kLZ4Compression; + case 0x5: + return ROCKSDB_NAMESPACE::CompressionType::kLZ4HCCompression; + case 0x6: + return ROCKSDB_NAMESPACE::CompressionType::kXpressCompression; + case 0x7: + return ROCKSDB_NAMESPACE::CompressionType::kZSTD; + case 0x7F: + default: + return ROCKSDB_NAMESPACE::CompressionType::kDisableCompressionOption; + } + } +}; + +// The portal class for org.rocksdb.CompactionPriority +class CompactionPriorityJni { + public: + // Returns the equivalent org.rocksdb.CompactionPriority for the provided + // C++ ROCKSDB_NAMESPACE::CompactionPri enum + static jbyte toJavaCompactionPriority( + const ROCKSDB_NAMESPACE::CompactionPri& compaction_priority) { + switch (compaction_priority) { + case ROCKSDB_NAMESPACE::CompactionPri::kByCompensatedSize: + return 0x0; + case ROCKSDB_NAMESPACE::CompactionPri::kOldestLargestSeqFirst: + return 0x1; + case ROCKSDB_NAMESPACE::CompactionPri::kOldestSmallestSeqFirst: + return 0x2; + case ROCKSDB_NAMESPACE::CompactionPri::kMinOverlappingRatio: + return 0x3; + case ROCKSDB_NAMESPACE::CompactionPri::kRoundRobin: + return 0x4; + default: + return 0x0; // undefined + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::CompactionPri enum for the + // provided Java org.rocksdb.CompactionPriority + static ROCKSDB_NAMESPACE::CompactionPri toCppCompactionPriority( + jbyte jcompaction_priority) { + switch (jcompaction_priority) { + case 0x0: + return ROCKSDB_NAMESPACE::CompactionPri::kByCompensatedSize; + case 0x1: + return ROCKSDB_NAMESPACE::CompactionPri::kOldestLargestSeqFirst; + case 0x2: + return ROCKSDB_NAMESPACE::CompactionPri::kOldestSmallestSeqFirst; + case 0x3: + return ROCKSDB_NAMESPACE::CompactionPri::kMinOverlappingRatio; + case 0x4: + return ROCKSDB_NAMESPACE::CompactionPri::kRoundRobin; + default: + // undefined/default + return ROCKSDB_NAMESPACE::CompactionPri::kByCompensatedSize; + } + } +}; + +// The portal class for org.rocksdb.AccessHint +class AccessHintJni { + public: + // Returns the equivalent org.rocksdb.AccessHint for the provided + // C++ ROCKSDB_NAMESPACE::DBOptions::AccessHint enum + static jbyte toJavaAccessHint( + const ROCKSDB_NAMESPACE::DBOptions::AccessHint& access_hint) { + switch (access_hint) { + case ROCKSDB_NAMESPACE::DBOptions::AccessHint::NONE: + return 0x0; + case ROCKSDB_NAMESPACE::DBOptions::AccessHint::NORMAL: + return 0x1; + case ROCKSDB_NAMESPACE::DBOptions::AccessHint::SEQUENTIAL: + return 0x2; + case ROCKSDB_NAMESPACE::DBOptions::AccessHint::WILLNEED: + return 0x3; + default: + // undefined/default + return 0x1; + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::DBOptions::AccessHint enum + // for the provided Java org.rocksdb.AccessHint + static ROCKSDB_NAMESPACE::DBOptions::AccessHint toCppAccessHint( + jbyte jaccess_hint) { + switch (jaccess_hint) { + case 0x0: + return ROCKSDB_NAMESPACE::DBOptions::AccessHint::NONE; + case 0x1: + return ROCKSDB_NAMESPACE::DBOptions::AccessHint::NORMAL; + case 0x2: + return ROCKSDB_NAMESPACE::DBOptions::AccessHint::SEQUENTIAL; + case 0x3: + return ROCKSDB_NAMESPACE::DBOptions::AccessHint::WILLNEED; + default: + // undefined/default + return ROCKSDB_NAMESPACE::DBOptions::AccessHint::NORMAL; + } + } +}; + +// The portal class for org.rocksdb.WALRecoveryMode +class WALRecoveryModeJni { + public: + // Returns the equivalent org.rocksdb.WALRecoveryMode for the provided + // C++ ROCKSDB_NAMESPACE::WALRecoveryMode enum + static jbyte toJavaWALRecoveryMode( + const ROCKSDB_NAMESPACE::WALRecoveryMode& wal_recovery_mode) { + switch (wal_recovery_mode) { + case ROCKSDB_NAMESPACE::WALRecoveryMode::kTolerateCorruptedTailRecords: + return 0x0; + case ROCKSDB_NAMESPACE::WALRecoveryMode::kAbsoluteConsistency: + return 0x1; + case ROCKSDB_NAMESPACE::WALRecoveryMode::kPointInTimeRecovery: + return 0x2; + case ROCKSDB_NAMESPACE::WALRecoveryMode::kSkipAnyCorruptedRecords: + return 0x3; + default: + // undefined/default + return 0x2; + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::WALRecoveryMode enum for the + // provided Java org.rocksdb.WALRecoveryMode + static ROCKSDB_NAMESPACE::WALRecoveryMode toCppWALRecoveryMode( + jbyte jwal_recovery_mode) { + switch (jwal_recovery_mode) { + case 0x0: + return ROCKSDB_NAMESPACE::WALRecoveryMode:: + kTolerateCorruptedTailRecords; + case 0x1: + return ROCKSDB_NAMESPACE::WALRecoveryMode::kAbsoluteConsistency; + case 0x2: + return ROCKSDB_NAMESPACE::WALRecoveryMode::kPointInTimeRecovery; + case 0x3: + return ROCKSDB_NAMESPACE::WALRecoveryMode::kSkipAnyCorruptedRecords; + default: + // undefined/default + return ROCKSDB_NAMESPACE::WALRecoveryMode::kPointInTimeRecovery; + } + } +}; + +// The portal class for org.rocksdb.TickerType +class TickerTypeJni { + public: + // Returns the equivalent org.rocksdb.TickerType for the provided + // C++ ROCKSDB_NAMESPACE::Tickers enum + static jbyte toJavaTickerType(const ROCKSDB_NAMESPACE::Tickers& tickers) { + switch (tickers) { + case ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_MISS: + return 0x0; + case ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_HIT: + return 0x1; + case ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_ADD: + return 0x2; + case ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_ADD_FAILURES: + return 0x3; + case ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_INDEX_MISS: + return 0x4; + case ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_INDEX_HIT: + return 0x5; + case ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_INDEX_ADD: + return 0x6; + case ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_INDEX_BYTES_INSERT: + return 0x7; + case ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_FILTER_MISS: + return 0x9; + case ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_FILTER_HIT: + return 0xA; + case ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_FILTER_ADD: + return 0xB; + case ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_FILTER_BYTES_INSERT: + return 0xC; + case ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_DATA_MISS: + return 0xE; + case ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_DATA_HIT: + return 0xF; + case ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_DATA_ADD: + return 0x10; + case ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_DATA_BYTES_INSERT: + return 0x11; + case ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_BYTES_READ: + return 0x12; + case ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_BYTES_WRITE: + return 0x13; + case ROCKSDB_NAMESPACE::Tickers::BLOOM_FILTER_USEFUL: + return 0x14; + case ROCKSDB_NAMESPACE::Tickers::PERSISTENT_CACHE_HIT: + return 0x15; + case ROCKSDB_NAMESPACE::Tickers::PERSISTENT_CACHE_MISS: + return 0x16; + case ROCKSDB_NAMESPACE::Tickers::SIM_BLOCK_CACHE_HIT: + return 0x17; + case ROCKSDB_NAMESPACE::Tickers::SIM_BLOCK_CACHE_MISS: + return 0x18; + case ROCKSDB_NAMESPACE::Tickers::MEMTABLE_HIT: + return 0x19; + case ROCKSDB_NAMESPACE::Tickers::MEMTABLE_MISS: + return 0x1A; + case ROCKSDB_NAMESPACE::Tickers::GET_HIT_L0: + return 0x1B; + case ROCKSDB_NAMESPACE::Tickers::GET_HIT_L1: + return 0x1C; + case ROCKSDB_NAMESPACE::Tickers::GET_HIT_L2_AND_UP: + return 0x1D; + case ROCKSDB_NAMESPACE::Tickers::COMPACTION_KEY_DROP_NEWER_ENTRY: + return 0x1E; + case ROCKSDB_NAMESPACE::Tickers::COMPACTION_KEY_DROP_OBSOLETE: + return 0x1F; + case ROCKSDB_NAMESPACE::Tickers::COMPACTION_KEY_DROP_RANGE_DEL: + return 0x20; + case ROCKSDB_NAMESPACE::Tickers::COMPACTION_KEY_DROP_USER: + return 0x21; + case ROCKSDB_NAMESPACE::Tickers::COMPACTION_RANGE_DEL_DROP_OBSOLETE: + return 0x22; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_KEYS_WRITTEN: + return 0x23; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_KEYS_READ: + return 0x24; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_KEYS_UPDATED: + return 0x25; + case ROCKSDB_NAMESPACE::Tickers::BYTES_WRITTEN: + return 0x26; + case ROCKSDB_NAMESPACE::Tickers::BYTES_READ: + return 0x27; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_DB_SEEK: + return 0x28; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_DB_NEXT: + return 0x29; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_DB_PREV: + return 0x2A; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_DB_SEEK_FOUND: + return 0x2B; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_DB_NEXT_FOUND: + return 0x2C; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_DB_PREV_FOUND: + return 0x2D; + case ROCKSDB_NAMESPACE::Tickers::ITER_BYTES_READ: + return 0x2E; + case ROCKSDB_NAMESPACE::Tickers::NO_FILE_OPENS: + return 0x30; + case ROCKSDB_NAMESPACE::Tickers::NO_FILE_ERRORS: + return 0x31; + case ROCKSDB_NAMESPACE::Tickers::STALL_MICROS: + return 0x35; + case ROCKSDB_NAMESPACE::Tickers::DB_MUTEX_WAIT_MICROS: + return 0x36; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_MULTIGET_CALLS: + return 0x39; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_MULTIGET_KEYS_READ: + return 0x3A; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_MULTIGET_BYTES_READ: + return 0x3B; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_MERGE_FAILURES: + return 0x3D; + case ROCKSDB_NAMESPACE::Tickers::BLOOM_FILTER_PREFIX_CHECKED: + return 0x3E; + case ROCKSDB_NAMESPACE::Tickers::BLOOM_FILTER_PREFIX_USEFUL: + return 0x3F; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_OF_RESEEKS_IN_ITERATION: + return 0x40; + case ROCKSDB_NAMESPACE::Tickers::GET_UPDATES_SINCE_CALLS: + return 0x41; + case ROCKSDB_NAMESPACE::Tickers::WAL_FILE_SYNCED: + return 0x46; + case ROCKSDB_NAMESPACE::Tickers::WAL_FILE_BYTES: + return 0x47; + case ROCKSDB_NAMESPACE::Tickers::WRITE_DONE_BY_SELF: + return 0x48; + case ROCKSDB_NAMESPACE::Tickers::WRITE_DONE_BY_OTHER: + return 0x49; + case ROCKSDB_NAMESPACE::Tickers::WRITE_WITH_WAL: + return 0x4B; + case ROCKSDB_NAMESPACE::Tickers::COMPACT_READ_BYTES: + return 0x4C; + case ROCKSDB_NAMESPACE::Tickers::COMPACT_WRITE_BYTES: + return 0x4D; + case ROCKSDB_NAMESPACE::Tickers::FLUSH_WRITE_BYTES: + return 0x4E; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_DIRECT_LOAD_TABLE_PROPERTIES: + return 0x4F; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_SUPERVERSION_ACQUIRES: + return 0x50; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_SUPERVERSION_RELEASES: + return 0x51; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_SUPERVERSION_CLEANUPS: + return 0x52; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_BLOCK_COMPRESSED: + return 0x53; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_BLOCK_DECOMPRESSED: + return 0x54; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_BLOCK_NOT_COMPRESSED: + return 0x55; + case ROCKSDB_NAMESPACE::Tickers::MERGE_OPERATION_TOTAL_TIME: + return 0x56; + case ROCKSDB_NAMESPACE::Tickers::FILTER_OPERATION_TOTAL_TIME: + return 0x57; + case ROCKSDB_NAMESPACE::Tickers::ROW_CACHE_HIT: + return 0x58; + case ROCKSDB_NAMESPACE::Tickers::ROW_CACHE_MISS: + return 0x59; + case ROCKSDB_NAMESPACE::Tickers::READ_AMP_ESTIMATE_USEFUL_BYTES: + return 0x5A; + case ROCKSDB_NAMESPACE::Tickers::READ_AMP_TOTAL_READ_BYTES: + return 0x5B; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_RATE_LIMITER_DRAINS: + return 0x5C; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_ITER_SKIP: + return 0x5D; + case ROCKSDB_NAMESPACE::Tickers::NUMBER_MULTIGET_KEYS_FOUND: + return 0x5E; + case ROCKSDB_NAMESPACE::Tickers::NO_ITERATOR_CREATED: + // -0x01 so we can skip over the already taken 0x5F (TICKER_ENUM_MAX). + return -0x01; + case ROCKSDB_NAMESPACE::Tickers::NO_ITERATOR_DELETED: + return 0x60; + case ROCKSDB_NAMESPACE::Tickers::COMPACTION_OPTIMIZED_DEL_DROP_OBSOLETE: + return 0x61; + case ROCKSDB_NAMESPACE::Tickers::COMPACTION_CANCELLED: + return 0x62; + case ROCKSDB_NAMESPACE::Tickers::BLOOM_FILTER_FULL_POSITIVE: + return 0x63; + case ROCKSDB_NAMESPACE::Tickers::BLOOM_FILTER_FULL_TRUE_POSITIVE: + return 0x64; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_NUM_PUT: + return 0x65; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_NUM_WRITE: + return 0x66; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_NUM_GET: + return 0x67; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_NUM_MULTIGET: + return 0x68; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_NUM_SEEK: + return 0x69; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_NUM_NEXT: + return 0x6A; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_NUM_PREV: + return 0x6B; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_NUM_KEYS_WRITTEN: + return 0x6C; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_NUM_KEYS_READ: + return 0x6D; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_BYTES_WRITTEN: + return 0x6E; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_BYTES_READ: + return 0x6F; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_WRITE_INLINED: + return 0x70; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_WRITE_INLINED_TTL: + return 0x71; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_WRITE_BLOB: + return 0x72; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_WRITE_BLOB_TTL: + return 0x73; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_BLOB_FILE_BYTES_WRITTEN: + return 0x74; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_BLOB_FILE_BYTES_READ: + return 0x75; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_BLOB_FILE_SYNCED: + return 0x76; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_BLOB_INDEX_EXPIRED_COUNT: + return 0x77; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_BLOB_INDEX_EXPIRED_SIZE: + return 0x78; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_BLOB_INDEX_EVICTED_COUNT: + return 0x79; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_BLOB_INDEX_EVICTED_SIZE: + return 0x7A; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_GC_NUM_FILES: + return 0x7B; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_GC_NUM_NEW_FILES: + return 0x7C; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_GC_FAILURES: + return 0x7D; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_GC_NUM_KEYS_RELOCATED: + return -0x02; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_GC_BYTES_RELOCATED: + return -0x05; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_FIFO_NUM_FILES_EVICTED: + return -0x06; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_FIFO_NUM_KEYS_EVICTED: + return -0x07; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_FIFO_BYTES_EVICTED: + return -0x08; + case ROCKSDB_NAMESPACE::Tickers::TXN_PREPARE_MUTEX_OVERHEAD: + return -0x09; + case ROCKSDB_NAMESPACE::Tickers::TXN_OLD_COMMIT_MAP_MUTEX_OVERHEAD: + return -0x0A; + case ROCKSDB_NAMESPACE::Tickers::TXN_DUPLICATE_KEY_OVERHEAD: + return -0x0B; + case ROCKSDB_NAMESPACE::Tickers::TXN_SNAPSHOT_MUTEX_OVERHEAD: + return -0x0C; + case ROCKSDB_NAMESPACE::Tickers::TXN_GET_TRY_AGAIN: + return -0x0D; + case ROCKSDB_NAMESPACE::Tickers::FILES_MARKED_TRASH: + return -0x0E; + case ROCKSDB_NAMESPACE::Tickers::FILES_DELETED_IMMEDIATELY: + return -0X0F; + case ROCKSDB_NAMESPACE::Tickers::COMPACT_READ_BYTES_MARKED: + return -0x10; + case ROCKSDB_NAMESPACE::Tickers::COMPACT_READ_BYTES_PERIODIC: + return -0x11; + case ROCKSDB_NAMESPACE::Tickers::COMPACT_READ_BYTES_TTL: + return -0x12; + case ROCKSDB_NAMESPACE::Tickers::COMPACT_WRITE_BYTES_MARKED: + return -0x13; + case ROCKSDB_NAMESPACE::Tickers::COMPACT_WRITE_BYTES_PERIODIC: + return -0x14; + case ROCKSDB_NAMESPACE::Tickers::COMPACT_WRITE_BYTES_TTL: + return -0x15; + case ROCKSDB_NAMESPACE::Tickers::ERROR_HANDLER_BG_ERROR_COUNT: + return -0x16; + case ROCKSDB_NAMESPACE::Tickers::ERROR_HANDLER_BG_IO_ERROR_COUNT: + return -0x17; + case ROCKSDB_NAMESPACE::Tickers:: + ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT: + return -0x18; + case ROCKSDB_NAMESPACE::Tickers::ERROR_HANDLER_AUTORESUME_COUNT: + return -0x19; + case ROCKSDB_NAMESPACE::Tickers:: + ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT: + return -0x1A; + case ROCKSDB_NAMESPACE::Tickers::ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT: + return -0x1B; + case ROCKSDB_NAMESPACE::Tickers::MEMTABLE_PAYLOAD_BYTES_AT_FLUSH: + return -0x1C; + case ROCKSDB_NAMESPACE::Tickers::MEMTABLE_GARBAGE_BYTES_AT_FLUSH: + return -0x1D; + case ROCKSDB_NAMESPACE::Tickers::SECONDARY_CACHE_HITS: + return -0x1E; + case ROCKSDB_NAMESPACE::Tickers::VERIFY_CHECKSUM_READ_BYTES: + return -0x1F; + case ROCKSDB_NAMESPACE::Tickers::BACKUP_READ_BYTES: + return -0x20; + case ROCKSDB_NAMESPACE::Tickers::BACKUP_WRITE_BYTES: + return -0x21; + case ROCKSDB_NAMESPACE::Tickers::REMOTE_COMPACT_READ_BYTES: + return -0x22; + case ROCKSDB_NAMESPACE::Tickers::REMOTE_COMPACT_WRITE_BYTES: + return -0x23; + case ROCKSDB_NAMESPACE::Tickers::HOT_FILE_READ_BYTES: + return -0x24; + case ROCKSDB_NAMESPACE::Tickers::WARM_FILE_READ_BYTES: + return -0x25; + case ROCKSDB_NAMESPACE::Tickers::COLD_FILE_READ_BYTES: + return -0x26; + case ROCKSDB_NAMESPACE::Tickers::HOT_FILE_READ_COUNT: + return -0x27; + case ROCKSDB_NAMESPACE::Tickers::WARM_FILE_READ_COUNT: + return -0x28; + case ROCKSDB_NAMESPACE::Tickers::COLD_FILE_READ_COUNT: + return -0x29; + case ROCKSDB_NAMESPACE::Tickers::LAST_LEVEL_READ_BYTES: + return -0x2A; + case ROCKSDB_NAMESPACE::Tickers::LAST_LEVEL_READ_COUNT: + return -0x2B; + case ROCKSDB_NAMESPACE::Tickers::NON_LAST_LEVEL_READ_BYTES: + return -0x2C; + case ROCKSDB_NAMESPACE::Tickers::NON_LAST_LEVEL_READ_COUNT: + return -0x2D; + case ROCKSDB_NAMESPACE::Tickers::BLOCK_CHECKSUM_COMPUTE_COUNT: + return -0x2E; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_CACHE_MISS: + return -0x2F; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_CACHE_HIT: + return -0x30; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_CACHE_ADD: + return -0x31; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_CACHE_ADD_FAILURES: + return -0x32; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_CACHE_BYTES_READ: + return -0x33; + case ROCKSDB_NAMESPACE::Tickers::BLOB_DB_CACHE_BYTES_WRITE: + return -0x34; + case ROCKSDB_NAMESPACE::Tickers::READ_ASYNC_MICROS: + return -0x35; + case ROCKSDB_NAMESPACE::Tickers::ASYNC_READ_ERROR_COUNT: + return -0x36; + case ROCKSDB_NAMESPACE::Tickers::SECONDARY_CACHE_FILTER_HITS: + return -0x37; + case ROCKSDB_NAMESPACE::Tickers::SECONDARY_CACHE_INDEX_HITS: + return -0x38; + case ROCKSDB_NAMESPACE::Tickers::SECONDARY_CACHE_DATA_HITS: + return -0x39; + case ROCKSDB_NAMESPACE::Tickers::TABLE_OPEN_PREFETCH_TAIL_MISS: + return -0x3A; + case ROCKSDB_NAMESPACE::Tickers::TABLE_OPEN_PREFETCH_TAIL_HIT: + return -0x3B; + case ROCKSDB_NAMESPACE::Tickers::BLOCK_CHECKSUM_MISMATCH_COUNT: + return -0x3C; + case ROCKSDB_NAMESPACE::Tickers::TICKER_ENUM_MAX: + // 0x5F was the max value in the initial copy of tickers to Java. + // Since these values are exposed directly to Java clients, we keep + // the value the same forever. + // + // TODO: This particular case seems confusing and unnecessary to pin the + // value since it's meant to be the number of tickers, not an actual + // ticker value. But we aren't yet in a position to fix it since the + // number of tickers doesn't fit in the Java representation (jbyte). + return 0x5F; + default: + // undefined/default + return 0x0; + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::Tickers enum for the + // provided Java org.rocksdb.TickerType + static ROCKSDB_NAMESPACE::Tickers toCppTickers(jbyte jticker_type) { + switch (jticker_type) { + case 0x0: + return ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_MISS; + case 0x1: + return ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_HIT; + case 0x2: + return ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_ADD; + case 0x3: + return ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_ADD_FAILURES; + case 0x4: + return ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_INDEX_MISS; + case 0x5: + return ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_INDEX_HIT; + case 0x6: + return ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_INDEX_ADD; + case 0x7: + return ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_INDEX_BYTES_INSERT; + case 0x9: + return ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_FILTER_MISS; + case 0xA: + return ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_FILTER_HIT; + case 0xB: + return ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_FILTER_ADD; + case 0xC: + return ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_FILTER_BYTES_INSERT; + case 0xE: + return ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_DATA_MISS; + case 0xF: + return ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_DATA_HIT; + case 0x10: + return ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_DATA_ADD; + case 0x11: + return ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_DATA_BYTES_INSERT; + case 0x12: + return ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_BYTES_READ; + case 0x13: + return ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_BYTES_WRITE; + case 0x14: + return ROCKSDB_NAMESPACE::Tickers::BLOOM_FILTER_USEFUL; + case 0x15: + return ROCKSDB_NAMESPACE::Tickers::PERSISTENT_CACHE_HIT; + case 0x16: + return ROCKSDB_NAMESPACE::Tickers::PERSISTENT_CACHE_MISS; + case 0x17: + return ROCKSDB_NAMESPACE::Tickers::SIM_BLOCK_CACHE_HIT; + case 0x18: + return ROCKSDB_NAMESPACE::Tickers::SIM_BLOCK_CACHE_MISS; + case 0x19: + return ROCKSDB_NAMESPACE::Tickers::MEMTABLE_HIT; + case 0x1A: + return ROCKSDB_NAMESPACE::Tickers::MEMTABLE_MISS; + case 0x1B: + return ROCKSDB_NAMESPACE::Tickers::GET_HIT_L0; + case 0x1C: + return ROCKSDB_NAMESPACE::Tickers::GET_HIT_L1; + case 0x1D: + return ROCKSDB_NAMESPACE::Tickers::GET_HIT_L2_AND_UP; + case 0x1E: + return ROCKSDB_NAMESPACE::Tickers::COMPACTION_KEY_DROP_NEWER_ENTRY; + case 0x1F: + return ROCKSDB_NAMESPACE::Tickers::COMPACTION_KEY_DROP_OBSOLETE; + case 0x20: + return ROCKSDB_NAMESPACE::Tickers::COMPACTION_KEY_DROP_RANGE_DEL; + case 0x21: + return ROCKSDB_NAMESPACE::Tickers::COMPACTION_KEY_DROP_USER; + case 0x22: + return ROCKSDB_NAMESPACE::Tickers::COMPACTION_RANGE_DEL_DROP_OBSOLETE; + case 0x23: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_KEYS_WRITTEN; + case 0x24: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_KEYS_READ; + case 0x25: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_KEYS_UPDATED; + case 0x26: + return ROCKSDB_NAMESPACE::Tickers::BYTES_WRITTEN; + case 0x27: + return ROCKSDB_NAMESPACE::Tickers::BYTES_READ; + case 0x28: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_DB_SEEK; + case 0x29: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_DB_NEXT; + case 0x2A: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_DB_PREV; + case 0x2B: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_DB_SEEK_FOUND; + case 0x2C: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_DB_NEXT_FOUND; + case 0x2D: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_DB_PREV_FOUND; + case 0x2E: + return ROCKSDB_NAMESPACE::Tickers::ITER_BYTES_READ; + case 0x30: + return ROCKSDB_NAMESPACE::Tickers::NO_FILE_OPENS; + case 0x31: + return ROCKSDB_NAMESPACE::Tickers::NO_FILE_ERRORS; + case 0x35: + return ROCKSDB_NAMESPACE::Tickers::STALL_MICROS; + case 0x36: + return ROCKSDB_NAMESPACE::Tickers::DB_MUTEX_WAIT_MICROS; + case 0x39: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_MULTIGET_CALLS; + case 0x3A: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_MULTIGET_KEYS_READ; + case 0x3B: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_MULTIGET_BYTES_READ; + case 0x3D: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_MERGE_FAILURES; + case 0x3E: + return ROCKSDB_NAMESPACE::Tickers::BLOOM_FILTER_PREFIX_CHECKED; + case 0x3F: + return ROCKSDB_NAMESPACE::Tickers::BLOOM_FILTER_PREFIX_USEFUL; + case 0x40: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_OF_RESEEKS_IN_ITERATION; + case 0x41: + return ROCKSDB_NAMESPACE::Tickers::GET_UPDATES_SINCE_CALLS; + case 0x46: + return ROCKSDB_NAMESPACE::Tickers::WAL_FILE_SYNCED; + case 0x47: + return ROCKSDB_NAMESPACE::Tickers::WAL_FILE_BYTES; + case 0x48: + return ROCKSDB_NAMESPACE::Tickers::WRITE_DONE_BY_SELF; + case 0x49: + return ROCKSDB_NAMESPACE::Tickers::WRITE_DONE_BY_OTHER; + case 0x4B: + return ROCKSDB_NAMESPACE::Tickers::WRITE_WITH_WAL; + case 0x4C: + return ROCKSDB_NAMESPACE::Tickers::COMPACT_READ_BYTES; + case 0x4D: + return ROCKSDB_NAMESPACE::Tickers::COMPACT_WRITE_BYTES; + case 0x4E: + return ROCKSDB_NAMESPACE::Tickers::FLUSH_WRITE_BYTES; + case 0x4F: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_DIRECT_LOAD_TABLE_PROPERTIES; + case 0x50: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_SUPERVERSION_ACQUIRES; + case 0x51: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_SUPERVERSION_RELEASES; + case 0x52: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_SUPERVERSION_CLEANUPS; + case 0x53: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_BLOCK_COMPRESSED; + case 0x54: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_BLOCK_DECOMPRESSED; + case 0x55: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_BLOCK_NOT_COMPRESSED; + case 0x56: + return ROCKSDB_NAMESPACE::Tickers::MERGE_OPERATION_TOTAL_TIME; + case 0x57: + return ROCKSDB_NAMESPACE::Tickers::FILTER_OPERATION_TOTAL_TIME; + case 0x58: + return ROCKSDB_NAMESPACE::Tickers::ROW_CACHE_HIT; + case 0x59: + return ROCKSDB_NAMESPACE::Tickers::ROW_CACHE_MISS; + case 0x5A: + return ROCKSDB_NAMESPACE::Tickers::READ_AMP_ESTIMATE_USEFUL_BYTES; + case 0x5B: + return ROCKSDB_NAMESPACE::Tickers::READ_AMP_TOTAL_READ_BYTES; + case 0x5C: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_RATE_LIMITER_DRAINS; + case 0x5D: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_ITER_SKIP; + case 0x5E: + return ROCKSDB_NAMESPACE::Tickers::NUMBER_MULTIGET_KEYS_FOUND; + case -0x01: + // -0x01 so we can skip over the already taken 0x5F (TICKER_ENUM_MAX). + return ROCKSDB_NAMESPACE::Tickers::NO_ITERATOR_CREATED; + case 0x60: + return ROCKSDB_NAMESPACE::Tickers::NO_ITERATOR_DELETED; + case 0x61: + return ROCKSDB_NAMESPACE::Tickers:: + COMPACTION_OPTIMIZED_DEL_DROP_OBSOLETE; + case 0x62: + return ROCKSDB_NAMESPACE::Tickers::COMPACTION_CANCELLED; + case 0x63: + return ROCKSDB_NAMESPACE::Tickers::BLOOM_FILTER_FULL_POSITIVE; + case 0x64: + return ROCKSDB_NAMESPACE::Tickers::BLOOM_FILTER_FULL_TRUE_POSITIVE; + case 0x65: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_NUM_PUT; + case 0x66: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_NUM_WRITE; + case 0x67: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_NUM_GET; + case 0x68: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_NUM_MULTIGET; + case 0x69: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_NUM_SEEK; + case 0x6A: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_NUM_NEXT; + case 0x6B: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_NUM_PREV; + case 0x6C: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_NUM_KEYS_WRITTEN; + case 0x6D: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_NUM_KEYS_READ; + case 0x6E: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_BYTES_WRITTEN; + case 0x6F: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_BYTES_READ; + case 0x70: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_WRITE_INLINED; + case 0x71: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_WRITE_INLINED_TTL; + case 0x72: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_WRITE_BLOB; + case 0x73: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_WRITE_BLOB_TTL; + case 0x74: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_BLOB_FILE_BYTES_WRITTEN; + case 0x75: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_BLOB_FILE_BYTES_READ; + case 0x76: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_BLOB_FILE_SYNCED; + case 0x77: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_BLOB_INDEX_EXPIRED_COUNT; + case 0x78: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_BLOB_INDEX_EXPIRED_SIZE; + case 0x79: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_BLOB_INDEX_EVICTED_COUNT; + case 0x7A: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_BLOB_INDEX_EVICTED_SIZE; + case 0x7B: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_GC_NUM_FILES; + case 0x7C: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_GC_NUM_NEW_FILES; + case 0x7D: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_GC_FAILURES; + case -0x02: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_GC_NUM_KEYS_RELOCATED; + case -0x05: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_GC_BYTES_RELOCATED; + case -0x06: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_FIFO_NUM_FILES_EVICTED; + case -0x07: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_FIFO_NUM_KEYS_EVICTED; + case -0x08: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_FIFO_BYTES_EVICTED; + case -0x09: + return ROCKSDB_NAMESPACE::Tickers::TXN_PREPARE_MUTEX_OVERHEAD; + case -0x0A: + return ROCKSDB_NAMESPACE::Tickers::TXN_OLD_COMMIT_MAP_MUTEX_OVERHEAD; + case -0x0B: + return ROCKSDB_NAMESPACE::Tickers::TXN_DUPLICATE_KEY_OVERHEAD; + case -0x0C: + return ROCKSDB_NAMESPACE::Tickers::TXN_SNAPSHOT_MUTEX_OVERHEAD; + case -0x0D: + return ROCKSDB_NAMESPACE::Tickers::TXN_GET_TRY_AGAIN; + case -0x0E: + return ROCKSDB_NAMESPACE::Tickers::FILES_MARKED_TRASH; + case -0x0F: + return ROCKSDB_NAMESPACE::Tickers::FILES_DELETED_IMMEDIATELY; + case -0x10: + return ROCKSDB_NAMESPACE::Tickers::COMPACT_READ_BYTES_MARKED; + case -0x11: + return ROCKSDB_NAMESPACE::Tickers::COMPACT_READ_BYTES_PERIODIC; + case -0x12: + return ROCKSDB_NAMESPACE::Tickers::COMPACT_READ_BYTES_TTL; + case -0x13: + return ROCKSDB_NAMESPACE::Tickers::COMPACT_WRITE_BYTES_MARKED; + case -0x14: + return ROCKSDB_NAMESPACE::Tickers::COMPACT_WRITE_BYTES_PERIODIC; + case -0x15: + return ROCKSDB_NAMESPACE::Tickers::COMPACT_WRITE_BYTES_TTL; + case -0x16: + return ROCKSDB_NAMESPACE::Tickers::ERROR_HANDLER_BG_ERROR_COUNT; + case -0x17: + return ROCKSDB_NAMESPACE::Tickers::ERROR_HANDLER_BG_IO_ERROR_COUNT; + case -0x18: + return ROCKSDB_NAMESPACE::Tickers:: + ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT; + case -0x19: + return ROCKSDB_NAMESPACE::Tickers::ERROR_HANDLER_AUTORESUME_COUNT; + case -0x1A: + return ROCKSDB_NAMESPACE::Tickers:: + ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT; + case -0x1B: + return ROCKSDB_NAMESPACE::Tickers:: + ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT; + case -0x1C: + return ROCKSDB_NAMESPACE::Tickers::MEMTABLE_PAYLOAD_BYTES_AT_FLUSH; + case -0x1D: + return ROCKSDB_NAMESPACE::Tickers::MEMTABLE_GARBAGE_BYTES_AT_FLUSH; + case -0x1E: + return ROCKSDB_NAMESPACE::Tickers::SECONDARY_CACHE_HITS; + case -0x1F: + return ROCKSDB_NAMESPACE::Tickers::VERIFY_CHECKSUM_READ_BYTES; + case -0x20: + return ROCKSDB_NAMESPACE::Tickers::BACKUP_READ_BYTES; + case -0x21: + return ROCKSDB_NAMESPACE::Tickers::BACKUP_WRITE_BYTES; + case -0x22: + return ROCKSDB_NAMESPACE::Tickers::REMOTE_COMPACT_READ_BYTES; + case -0x23: + return ROCKSDB_NAMESPACE::Tickers::REMOTE_COMPACT_WRITE_BYTES; + case -0x24: + return ROCKSDB_NAMESPACE::Tickers::HOT_FILE_READ_BYTES; + case -0x25: + return ROCKSDB_NAMESPACE::Tickers::WARM_FILE_READ_BYTES; + case -0x26: + return ROCKSDB_NAMESPACE::Tickers::COLD_FILE_READ_BYTES; + case -0x27: + return ROCKSDB_NAMESPACE::Tickers::HOT_FILE_READ_COUNT; + case -0x28: + return ROCKSDB_NAMESPACE::Tickers::WARM_FILE_READ_COUNT; + case -0x29: + return ROCKSDB_NAMESPACE::Tickers::COLD_FILE_READ_COUNT; + case -0x2A: + return ROCKSDB_NAMESPACE::Tickers::LAST_LEVEL_READ_BYTES; + case -0x2B: + return ROCKSDB_NAMESPACE::Tickers::LAST_LEVEL_READ_COUNT; + case -0x2C: + return ROCKSDB_NAMESPACE::Tickers::NON_LAST_LEVEL_READ_BYTES; + case -0x2D: + return ROCKSDB_NAMESPACE::Tickers::NON_LAST_LEVEL_READ_COUNT; + case -0x2E: + return ROCKSDB_NAMESPACE::Tickers::BLOCK_CHECKSUM_COMPUTE_COUNT; + case -0x2F: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_CACHE_MISS; + case -0x30: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_CACHE_HIT; + case -0x31: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_CACHE_ADD; + case -0x32: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_CACHE_ADD_FAILURES; + case -0x33: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_CACHE_BYTES_READ; + case -0x34: + return ROCKSDB_NAMESPACE::Tickers::BLOB_DB_CACHE_BYTES_WRITE; + case -0x35: + return ROCKSDB_NAMESPACE::Tickers::READ_ASYNC_MICROS; + case -0x36: + return ROCKSDB_NAMESPACE::Tickers::ASYNC_READ_ERROR_COUNT; + case -0x37: + return ROCKSDB_NAMESPACE::Tickers::SECONDARY_CACHE_FILTER_HITS; + case -0x38: + return ROCKSDB_NAMESPACE::Tickers::SECONDARY_CACHE_INDEX_HITS; + case -0x39: + return ROCKSDB_NAMESPACE::Tickers::SECONDARY_CACHE_DATA_HITS; + case -0x3A: + return ROCKSDB_NAMESPACE::Tickers::TABLE_OPEN_PREFETCH_TAIL_MISS; + case -0x3B: + return ROCKSDB_NAMESPACE::Tickers::TABLE_OPEN_PREFETCH_TAIL_HIT; + case -0x3C: + return ROCKSDB_NAMESPACE::Tickers::BLOCK_CHECKSUM_MISMATCH_COUNT; + case 0x5F: + // 0x5F was the max value in the initial copy of tickers to Java. + // Since these values are exposed directly to Java clients, we keep + // the value the same forever. + // + // TODO: This particular case seems confusing and unnecessary to pin the + // value since it's meant to be the number of tickers, not an actual + // ticker value. But we aren't yet in a position to fix it since the + // number of tickers doesn't fit in the Java representation (jbyte). + return ROCKSDB_NAMESPACE::Tickers::TICKER_ENUM_MAX; + + default: + // undefined/default + return ROCKSDB_NAMESPACE::Tickers::BLOCK_CACHE_MISS; + } + } +}; + +// The portal class for org.rocksdb.HistogramType +class HistogramTypeJni { + public: + // Returns the equivalent org.rocksdb.HistogramType for the provided + // C++ ROCKSDB_NAMESPACE::Histograms enum + static jbyte toJavaHistogramsType( + const ROCKSDB_NAMESPACE::Histograms& histograms) { + switch (histograms) { + case ROCKSDB_NAMESPACE::Histograms::DB_GET: + return 0x0; + case ROCKSDB_NAMESPACE::Histograms::DB_WRITE: + return 0x1; + case ROCKSDB_NAMESPACE::Histograms::COMPACTION_TIME: + return 0x2; + case ROCKSDB_NAMESPACE::Histograms::SUBCOMPACTION_SETUP_TIME: + return 0x3; + case ROCKSDB_NAMESPACE::Histograms::TABLE_SYNC_MICROS: + return 0x4; + case ROCKSDB_NAMESPACE::Histograms::COMPACTION_OUTFILE_SYNC_MICROS: + return 0x5; + case ROCKSDB_NAMESPACE::Histograms::WAL_FILE_SYNC_MICROS: + return 0x6; + case ROCKSDB_NAMESPACE::Histograms::MANIFEST_FILE_SYNC_MICROS: + return 0x7; + case ROCKSDB_NAMESPACE::Histograms::TABLE_OPEN_IO_MICROS: + return 0x8; + case ROCKSDB_NAMESPACE::Histograms::DB_MULTIGET: + return 0x9; + case ROCKSDB_NAMESPACE::Histograms::READ_BLOCK_COMPACTION_MICROS: + return 0xA; + case ROCKSDB_NAMESPACE::Histograms::READ_BLOCK_GET_MICROS: + return 0xB; + case ROCKSDB_NAMESPACE::Histograms::WRITE_RAW_BLOCK_MICROS: + return 0xC; + case ROCKSDB_NAMESPACE::Histograms::NUM_FILES_IN_SINGLE_COMPACTION: + return 0x12; + case ROCKSDB_NAMESPACE::Histograms::DB_SEEK: + return 0x13; + case ROCKSDB_NAMESPACE::Histograms::WRITE_STALL: + return 0x14; + case ROCKSDB_NAMESPACE::Histograms::SST_READ_MICROS: + return 0x15; + case ROCKSDB_NAMESPACE::Histograms::NUM_SUBCOMPACTIONS_SCHEDULED: + return 0x16; + case ROCKSDB_NAMESPACE::Histograms::BYTES_PER_READ: + return 0x17; + case ROCKSDB_NAMESPACE::Histograms::BYTES_PER_WRITE: + return 0x18; + case ROCKSDB_NAMESPACE::Histograms::BYTES_PER_MULTIGET: + return 0x19; + case ROCKSDB_NAMESPACE::Histograms::BYTES_COMPRESSED: + return 0x1A; + case ROCKSDB_NAMESPACE::Histograms::BYTES_DECOMPRESSED: + return 0x1B; + case ROCKSDB_NAMESPACE::Histograms::COMPRESSION_TIMES_NANOS: + return 0x1C; + case ROCKSDB_NAMESPACE::Histograms::DECOMPRESSION_TIMES_NANOS: + return 0x1D; + case ROCKSDB_NAMESPACE::Histograms::READ_NUM_MERGE_OPERANDS: + return 0x1E; + // 0x20 to skip 0x1F so TICKER_ENUM_MAX remains unchanged for minor + // version compatibility. + case ROCKSDB_NAMESPACE::Histograms::FLUSH_TIME: + return 0x20; + case ROCKSDB_NAMESPACE::Histograms::BLOB_DB_KEY_SIZE: + return 0x21; + case ROCKSDB_NAMESPACE::Histograms::BLOB_DB_VALUE_SIZE: + return 0x22; + case ROCKSDB_NAMESPACE::Histograms::BLOB_DB_WRITE_MICROS: + return 0x23; + case ROCKSDB_NAMESPACE::Histograms::BLOB_DB_GET_MICROS: + return 0x24; + case ROCKSDB_NAMESPACE::Histograms::BLOB_DB_MULTIGET_MICROS: + return 0x25; + case ROCKSDB_NAMESPACE::Histograms::BLOB_DB_SEEK_MICROS: + return 0x26; + case ROCKSDB_NAMESPACE::Histograms::BLOB_DB_NEXT_MICROS: + return 0x27; + case ROCKSDB_NAMESPACE::Histograms::BLOB_DB_PREV_MICROS: + return 0x28; + case ROCKSDB_NAMESPACE::Histograms::BLOB_DB_BLOB_FILE_WRITE_MICROS: + return 0x29; + case ROCKSDB_NAMESPACE::Histograms::BLOB_DB_BLOB_FILE_READ_MICROS: + return 0x2A; + case ROCKSDB_NAMESPACE::Histograms::BLOB_DB_BLOB_FILE_SYNC_MICROS: + return 0x2B; + case ROCKSDB_NAMESPACE::Histograms::BLOB_DB_COMPRESSION_MICROS: + return 0x2D; + case ROCKSDB_NAMESPACE::Histograms::BLOB_DB_DECOMPRESSION_MICROS: + return 0x2E; + case ROCKSDB_NAMESPACE::Histograms:: + NUM_INDEX_AND_FILTER_BLOCKS_READ_PER_LEVEL: + return 0x2F; + case ROCKSDB_NAMESPACE::Histograms::NUM_SST_READ_PER_LEVEL: + return 0x31; + case ROCKSDB_NAMESPACE::Histograms::ERROR_HANDLER_AUTORESUME_RETRY_COUNT: + return 0x32; + case ROCKSDB_NAMESPACE::Histograms::ASYNC_READ_BYTES: + return 0x33; + case ROCKSDB_NAMESPACE::Histograms::POLL_WAIT_MICROS: + return 0x34; + case ROCKSDB_NAMESPACE::Histograms::PREFETCHED_BYTES_DISCARDED: + return 0x35; + case ROCKSDB_NAMESPACE::Histograms::MULTIGET_IO_BATCH_SIZE: + return 0x36; + case NUM_LEVEL_READ_PER_MULTIGET: + return 0x37; + case ASYNC_PREFETCH_ABORT_MICROS: + return 0x38; + case ROCKSDB_NAMESPACE::Histograms::TABLE_OPEN_PREFETCH_TAIL_READ_BYTES: + return 0x39; + case ROCKSDB_NAMESPACE::Histograms::FILE_READ_FLUSH_MICROS: + return 0x3A; + case ROCKSDB_NAMESPACE::Histograms::FILE_READ_COMPACTION_MICROS: + return 0x3B; + case ROCKSDB_NAMESPACE::Histograms::FILE_READ_DB_OPEN_MICROS: + return 0x3C; + case ROCKSDB_NAMESPACE::Histograms::HISTOGRAM_ENUM_MAX: + // 0x1F for backwards compatibility on current minor version. + return 0x1F; + + default: + // undefined/default + return 0x0; + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::Histograms enum for the + // provided Java org.rocksdb.HistogramsType + static ROCKSDB_NAMESPACE::Histograms toCppHistograms(jbyte jhistograms_type) { + switch (jhistograms_type) { + case 0x0: + return ROCKSDB_NAMESPACE::Histograms::DB_GET; + case 0x1: + return ROCKSDB_NAMESPACE::Histograms::DB_WRITE; + case 0x2: + return ROCKSDB_NAMESPACE::Histograms::COMPACTION_TIME; + case 0x3: + return ROCKSDB_NAMESPACE::Histograms::SUBCOMPACTION_SETUP_TIME; + case 0x4: + return ROCKSDB_NAMESPACE::Histograms::TABLE_SYNC_MICROS; + case 0x5: + return ROCKSDB_NAMESPACE::Histograms::COMPACTION_OUTFILE_SYNC_MICROS; + case 0x6: + return ROCKSDB_NAMESPACE::Histograms::WAL_FILE_SYNC_MICROS; + case 0x7: + return ROCKSDB_NAMESPACE::Histograms::MANIFEST_FILE_SYNC_MICROS; + case 0x8: + return ROCKSDB_NAMESPACE::Histograms::TABLE_OPEN_IO_MICROS; + case 0x9: + return ROCKSDB_NAMESPACE::Histograms::DB_MULTIGET; + case 0xA: + return ROCKSDB_NAMESPACE::Histograms::READ_BLOCK_COMPACTION_MICROS; + case 0xB: + return ROCKSDB_NAMESPACE::Histograms::READ_BLOCK_GET_MICROS; + case 0xC: + return ROCKSDB_NAMESPACE::Histograms::WRITE_RAW_BLOCK_MICROS; + case 0x12: + return ROCKSDB_NAMESPACE::Histograms::NUM_FILES_IN_SINGLE_COMPACTION; + case 0x13: + return ROCKSDB_NAMESPACE::Histograms::DB_SEEK; + case 0x14: + return ROCKSDB_NAMESPACE::Histograms::WRITE_STALL; + case 0x15: + return ROCKSDB_NAMESPACE::Histograms::SST_READ_MICROS; + case 0x16: + return ROCKSDB_NAMESPACE::Histograms::NUM_SUBCOMPACTIONS_SCHEDULED; + case 0x17: + return ROCKSDB_NAMESPACE::Histograms::BYTES_PER_READ; + case 0x18: + return ROCKSDB_NAMESPACE::Histograms::BYTES_PER_WRITE; + case 0x19: + return ROCKSDB_NAMESPACE::Histograms::BYTES_PER_MULTIGET; + case 0x1A: + return ROCKSDB_NAMESPACE::Histograms::BYTES_COMPRESSED; + case 0x1B: + return ROCKSDB_NAMESPACE::Histograms::BYTES_DECOMPRESSED; + case 0x1C: + return ROCKSDB_NAMESPACE::Histograms::COMPRESSION_TIMES_NANOS; + case 0x1D: + return ROCKSDB_NAMESPACE::Histograms::DECOMPRESSION_TIMES_NANOS; + case 0x1E: + return ROCKSDB_NAMESPACE::Histograms::READ_NUM_MERGE_OPERANDS; + // 0x20 to skip 0x1F so TICKER_ENUM_MAX remains unchanged for minor + // version compatibility. + case 0x20: + return ROCKSDB_NAMESPACE::Histograms::FLUSH_TIME; + case 0x21: + return ROCKSDB_NAMESPACE::Histograms::BLOB_DB_KEY_SIZE; + case 0x22: + return ROCKSDB_NAMESPACE::Histograms::BLOB_DB_VALUE_SIZE; + case 0x23: + return ROCKSDB_NAMESPACE::Histograms::BLOB_DB_WRITE_MICROS; + case 0x24: + return ROCKSDB_NAMESPACE::Histograms::BLOB_DB_GET_MICROS; + case 0x25: + return ROCKSDB_NAMESPACE::Histograms::BLOB_DB_MULTIGET_MICROS; + case 0x26: + return ROCKSDB_NAMESPACE::Histograms::BLOB_DB_SEEK_MICROS; + case 0x27: + return ROCKSDB_NAMESPACE::Histograms::BLOB_DB_NEXT_MICROS; + case 0x28: + return ROCKSDB_NAMESPACE::Histograms::BLOB_DB_PREV_MICROS; + case 0x29: + return ROCKSDB_NAMESPACE::Histograms::BLOB_DB_BLOB_FILE_WRITE_MICROS; + case 0x2A: + return ROCKSDB_NAMESPACE::Histograms::BLOB_DB_BLOB_FILE_READ_MICROS; + case 0x2B: + return ROCKSDB_NAMESPACE::Histograms::BLOB_DB_BLOB_FILE_SYNC_MICROS; + case 0x2D: + return ROCKSDB_NAMESPACE::Histograms::BLOB_DB_COMPRESSION_MICROS; + case 0x2E: + return ROCKSDB_NAMESPACE::Histograms::BLOB_DB_DECOMPRESSION_MICROS; + case 0x2F: + return ROCKSDB_NAMESPACE::Histograms:: + NUM_INDEX_AND_FILTER_BLOCKS_READ_PER_LEVEL; + case 0x31: + return ROCKSDB_NAMESPACE::Histograms::NUM_SST_READ_PER_LEVEL; + case 0x32: + return ROCKSDB_NAMESPACE::Histograms:: + ERROR_HANDLER_AUTORESUME_RETRY_COUNT; + case 0x33: + return ROCKSDB_NAMESPACE::Histograms::ASYNC_READ_BYTES; + case 0x34: + return ROCKSDB_NAMESPACE::Histograms::POLL_WAIT_MICROS; + case 0x35: + return ROCKSDB_NAMESPACE::Histograms::PREFETCHED_BYTES_DISCARDED; + case 0x36: + return ROCKSDB_NAMESPACE::Histograms::MULTIGET_IO_BATCH_SIZE; + case 0x37: + return ROCKSDB_NAMESPACE::Histograms::NUM_LEVEL_READ_PER_MULTIGET; + case 0x38: + return ROCKSDB_NAMESPACE::Histograms::ASYNC_PREFETCH_ABORT_MICROS; + case 0x39: + return ROCKSDB_NAMESPACE::Histograms:: + TABLE_OPEN_PREFETCH_TAIL_READ_BYTES; + case 0x3A: + return ROCKSDB_NAMESPACE::Histograms::FILE_READ_FLUSH_MICROS; + case 0x3B: + return ROCKSDB_NAMESPACE::Histograms::FILE_READ_COMPACTION_MICROS; + case 0x3C: + return ROCKSDB_NAMESPACE::Histograms::FILE_READ_DB_OPEN_MICROS; + case 0x1F: + // 0x1F for backwards compatibility on current minor version. + return ROCKSDB_NAMESPACE::Histograms::HISTOGRAM_ENUM_MAX; + + default: + // undefined/default + return ROCKSDB_NAMESPACE::Histograms::DB_GET; + } + } +}; + +// The portal class for org.rocksdb.StatsLevel +class StatsLevelJni { + public: + // Returns the equivalent org.rocksdb.StatsLevel for the provided + // C++ ROCKSDB_NAMESPACE::StatsLevel enum + static jbyte toJavaStatsLevel( + const ROCKSDB_NAMESPACE::StatsLevel& stats_level) { + switch (stats_level) { + case ROCKSDB_NAMESPACE::StatsLevel::kExceptDetailedTimers: + return 0x0; + case ROCKSDB_NAMESPACE::StatsLevel::kExceptTimeForMutex: + return 0x1; + case ROCKSDB_NAMESPACE::StatsLevel::kAll: + return 0x2; + + default: + // undefined/default + return 0x0; + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::StatsLevel enum for the + // provided Java org.rocksdb.StatsLevel + static ROCKSDB_NAMESPACE::StatsLevel toCppStatsLevel(jbyte jstats_level) { + switch (jstats_level) { + case 0x0: + return ROCKSDB_NAMESPACE::StatsLevel::kExceptDetailedTimers; + case 0x1: + return ROCKSDB_NAMESPACE::StatsLevel::kExceptTimeForMutex; + case 0x2: + return ROCKSDB_NAMESPACE::StatsLevel::kAll; + + default: + // undefined/default + return ROCKSDB_NAMESPACE::StatsLevel::kExceptDetailedTimers; + } + } +}; + +// The portal class for org.rocksdb.RateLimiterMode +class RateLimiterModeJni { + public: + // Returns the equivalent org.rocksdb.RateLimiterMode for the provided + // C++ ROCKSDB_NAMESPACE::RateLimiter::Mode enum + static jbyte toJavaRateLimiterMode( + const ROCKSDB_NAMESPACE::RateLimiter::Mode& rate_limiter_mode) { + switch (rate_limiter_mode) { + case ROCKSDB_NAMESPACE::RateLimiter::Mode::kReadsOnly: + return 0x0; + case ROCKSDB_NAMESPACE::RateLimiter::Mode::kWritesOnly: + return 0x1; + case ROCKSDB_NAMESPACE::RateLimiter::Mode::kAllIo: + return 0x2; + + default: + // undefined/default + return 0x1; + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::RateLimiter::Mode enum for + // the provided Java org.rocksdb.RateLimiterMode + static ROCKSDB_NAMESPACE::RateLimiter::Mode toCppRateLimiterMode( + jbyte jrate_limiter_mode) { + switch (jrate_limiter_mode) { + case 0x0: + return ROCKSDB_NAMESPACE::RateLimiter::Mode::kReadsOnly; + case 0x1: + return ROCKSDB_NAMESPACE::RateLimiter::Mode::kWritesOnly; + case 0x2: + return ROCKSDB_NAMESPACE::RateLimiter::Mode::kAllIo; + + default: + // undefined/default + return ROCKSDB_NAMESPACE::RateLimiter::Mode::kWritesOnly; + } + } +}; + +// The portal class for org.rocksdb.MemoryUsageType +class MemoryUsageTypeJni { + public: + // Returns the equivalent org.rocksdb.MemoryUsageType for the provided + // C++ ROCKSDB_NAMESPACE::MemoryUtil::UsageType enum + static jbyte toJavaMemoryUsageType( + const ROCKSDB_NAMESPACE::MemoryUtil::UsageType& usage_type) { + switch (usage_type) { + case ROCKSDB_NAMESPACE::MemoryUtil::UsageType::kMemTableTotal: + return 0x0; + case ROCKSDB_NAMESPACE::MemoryUtil::UsageType::kMemTableUnFlushed: + return 0x1; + case ROCKSDB_NAMESPACE::MemoryUtil::UsageType::kTableReadersTotal: + return 0x2; + case ROCKSDB_NAMESPACE::MemoryUtil::UsageType::kCacheTotal: + return 0x3; + default: + // undefined: use kNumUsageTypes + return 0x4; + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::MemoryUtil::UsageType enum + // for the provided Java org.rocksdb.MemoryUsageType + static ROCKSDB_NAMESPACE::MemoryUtil::UsageType toCppMemoryUsageType( + jbyte usage_type) { + switch (usage_type) { + case 0x0: + return ROCKSDB_NAMESPACE::MemoryUtil::UsageType::kMemTableTotal; + case 0x1: + return ROCKSDB_NAMESPACE::MemoryUtil::UsageType::kMemTableUnFlushed; + case 0x2: + return ROCKSDB_NAMESPACE::MemoryUtil::UsageType::kTableReadersTotal; + case 0x3: + return ROCKSDB_NAMESPACE::MemoryUtil::UsageType::kCacheTotal; + default: + // undefined/default: use kNumUsageTypes + return ROCKSDB_NAMESPACE::MemoryUtil::UsageType::kNumUsageTypes; + } + } +}; + +// The portal class for org.rocksdb.Transaction +class TransactionJni : public JavaClass { + public: + /** + * Get the Java Class org.rocksdb.Transaction + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/Transaction"); + } + + /** + * Create a new Java org.rocksdb.Transaction.WaitingTransactions object + * + * @param env A pointer to the Java environment + * @param jtransaction A Java org.rocksdb.Transaction object + * @param column_family_id The id of the column family + * @param key The key + * @param transaction_ids The transaction ids + * + * @return A reference to a Java + * org.rocksdb.Transaction.WaitingTransactions object, + * or nullptr if an an exception occurs + */ + static jobject newWaitingTransactions( + JNIEnv* env, jobject jtransaction, const uint32_t column_family_id, + const std::string& key, + const std::vector& transaction_ids) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID mid = env->GetMethodID( + jclazz, "newWaitingTransactions", + "(JLjava/lang/String;[J)Lorg/rocksdb/Transaction$WaitingTransactions;"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return nullptr; + } + + jstring jkey = env->NewStringUTF(key.c_str()); + if (jkey == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + const size_t len = transaction_ids.size(); + jlongArray jtransaction_ids = env->NewLongArray(static_cast(len)); + if (jtransaction_ids == nullptr) { + // exception thrown: OutOfMemoryError + env->DeleteLocalRef(jkey); + return nullptr; + } + + jboolean is_copy; + jlong* body = env->GetLongArrayElements(jtransaction_ids, &is_copy); + if (body == nullptr) { + // exception thrown: OutOfMemoryError + env->DeleteLocalRef(jkey); + env->DeleteLocalRef(jtransaction_ids); + return nullptr; + } + for (size_t i = 0; i < len; ++i) { + body[i] = static_cast(transaction_ids[i]); + } + env->ReleaseLongArrayElements(jtransaction_ids, body, + is_copy == JNI_TRUE ? 0 : JNI_ABORT); + + jobject jwaiting_transactions = env->CallObjectMethod( + jtransaction, mid, static_cast(column_family_id), jkey, + jtransaction_ids); + if (env->ExceptionCheck()) { + // exception thrown: InstantiationException or OutOfMemoryError + env->DeleteLocalRef(jkey); + env->DeleteLocalRef(jtransaction_ids); + return nullptr; + } + + return jwaiting_transactions; + } +}; + +// The portal class for org.rocksdb.TransactionDB +class TransactionDBJni : public JavaClass { + public: + /** + * Get the Java Class org.rocksdb.TransactionDB + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/TransactionDB"); + } + + /** + * Create a new Java org.rocksdb.TransactionDB.DeadlockInfo object + * + * @param env A pointer to the Java environment + * @param jtransaction A Java org.rocksdb.Transaction object + * @param column_family_id The id of the column family + * @param key The key + * @param transaction_ids The transaction ids + * + * @return A reference to a Java + * org.rocksdb.Transaction.WaitingTransactions object, + * or nullptr if an an exception occurs + */ + static jobject newDeadlockInfo( + JNIEnv* env, jobject jtransaction_db, + const ROCKSDB_NAMESPACE::TransactionID transaction_id, + const uint32_t column_family_id, const std::string& waiting_key, + const bool exclusive) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID mid = env->GetMethodID( + jclazz, "newDeadlockInfo", + "(JJLjava/lang/String;Z)Lorg/rocksdb/TransactionDB$DeadlockInfo;"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return nullptr; + } + + jstring jwaiting_key = env->NewStringUTF(waiting_key.c_str()); + if (jwaiting_key == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + // resolve the column family id to a ColumnFamilyHandle + jobject jdeadlock_info = env->CallObjectMethod( + jtransaction_db, mid, transaction_id, + static_cast(column_family_id), jwaiting_key, exclusive); + if (env->ExceptionCheck()) { + // exception thrown: InstantiationException or OutOfMemoryError + env->DeleteLocalRef(jwaiting_key); + return nullptr; + } + + return jdeadlock_info; + } +}; + +// The portal class for org.rocksdb.TxnDBWritePolicy +class TxnDBWritePolicyJni { + public: + // Returns the equivalent org.rocksdb.TxnDBWritePolicy for the provided + // C++ ROCKSDB_NAMESPACE::TxnDBWritePolicy enum + static jbyte toJavaTxnDBWritePolicy( + const ROCKSDB_NAMESPACE::TxnDBWritePolicy& txndb_write_policy) { + switch (txndb_write_policy) { + case ROCKSDB_NAMESPACE::TxnDBWritePolicy::WRITE_COMMITTED: + return 0x0; + case ROCKSDB_NAMESPACE::TxnDBWritePolicy::WRITE_PREPARED: + return 0x1; + case ROCKSDB_NAMESPACE::TxnDBWritePolicy::WRITE_UNPREPARED: + return 0x2; + default: + return 0x7F; // undefined + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::TxnDBWritePolicy enum for the + // provided Java org.rocksdb.TxnDBWritePolicy + static ROCKSDB_NAMESPACE::TxnDBWritePolicy toCppTxnDBWritePolicy( + jbyte jtxndb_write_policy) { + switch (jtxndb_write_policy) { + case 0x0: + return ROCKSDB_NAMESPACE::TxnDBWritePolicy::WRITE_COMMITTED; + case 0x1: + return ROCKSDB_NAMESPACE::TxnDBWritePolicy::WRITE_PREPARED; + case 0x2: + return ROCKSDB_NAMESPACE::TxnDBWritePolicy::WRITE_UNPREPARED; + default: + // undefined/default + return ROCKSDB_NAMESPACE::TxnDBWritePolicy::WRITE_COMMITTED; + } + } +}; + +// The portal class for org.rocksdb.TransactionDB.KeyLockInfo +class KeyLockInfoJni : public JavaClass { + public: + /** + * Get the Java Class org.rocksdb.TransactionDB.KeyLockInfo + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/TransactionDB$KeyLockInfo"); + } + + /** + * Create a new Java org.rocksdb.TransactionDB.KeyLockInfo object + * with the same properties as the provided C++ ROCKSDB_NAMESPACE::KeyLockInfo + * object + * + * @param env A pointer to the Java environment + * @param key_lock_info The ROCKSDB_NAMESPACE::KeyLockInfo object + * + * @return A reference to a Java + * org.rocksdb.TransactionDB.KeyLockInfo object, + * or nullptr if an an exception occurs + */ + static jobject construct( + JNIEnv* env, const ROCKSDB_NAMESPACE::KeyLockInfo& key_lock_info) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID mid = + env->GetMethodID(jclazz, "", "(Ljava/lang/String;[JZ)V"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return nullptr; + } + + jstring jkey = env->NewStringUTF(key_lock_info.key.c_str()); + if (jkey == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + const jsize jtransaction_ids_len = + static_cast(key_lock_info.ids.size()); + jlongArray jtransactions_ids = env->NewLongArray(jtransaction_ids_len); + if (jtransactions_ids == nullptr) { + // exception thrown: OutOfMemoryError + env->DeleteLocalRef(jkey); + return nullptr; + } + + const jobject jkey_lock_info = env->NewObject( + jclazz, mid, jkey, jtransactions_ids, key_lock_info.exclusive); + if (jkey_lock_info == nullptr) { + // exception thrown: InstantiationException or OutOfMemoryError + env->DeleteLocalRef(jtransactions_ids); + env->DeleteLocalRef(jkey); + return nullptr; + } + + return jkey_lock_info; + } +}; + +// The portal class for org.rocksdb.TransactionDB.DeadlockInfo +class DeadlockInfoJni : public JavaClass { + public: + /** + * Get the Java Class org.rocksdb.TransactionDB.DeadlockInfo + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/TransactionDB$DeadlockInfo"); + } +}; + +// The portal class for org.rocksdb.TransactionDB.DeadlockPath +class DeadlockPathJni : public JavaClass { + public: + /** + * Get the Java Class org.rocksdb.TransactionDB.DeadlockPath + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/TransactionDB$DeadlockPath"); + } + + /** + * Create a new Java org.rocksdb.TransactionDB.DeadlockPath object + * + * @param env A pointer to the Java environment + * + * @return A reference to a Java + * org.rocksdb.TransactionDB.DeadlockPath object, + * or nullptr if an an exception occurs + */ + static jobject construct(JNIEnv* env, const jobjectArray jdeadlock_infos, + const bool limit_exceeded) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID mid = env->GetMethodID(jclazz, "", "([LDeadlockInfo;Z)V"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return nullptr; + } + + const jobject jdeadlock_path = + env->NewObject(jclazz, mid, jdeadlock_infos, limit_exceeded); + if (jdeadlock_path == nullptr) { + // exception thrown: InstantiationException or OutOfMemoryError + return nullptr; + } + + return jdeadlock_path; + } +}; + +class AbstractTableFilterJni + : public RocksDBNativeClass< + const ROCKSDB_NAMESPACE::TableFilterJniCallback*, + AbstractTableFilterJni> { + public: + /** + * Get the Java Method: TableFilter#filter(TableProperties) + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getFilterMethod(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = + env->GetMethodID(jclazz, "filter", "(Lorg/rocksdb/TableProperties;)Z"); + assert(mid != nullptr); + return mid; + } + + private: + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/TableFilter"); + } +}; + +class TablePropertiesJni : public JavaClass { + public: + /** + * Create a new Java org.rocksdb.TableProperties object. + * + * @param env A pointer to the Java environment + * @param table_properties A Cpp table properties object + * + * @return A reference to a Java org.rocksdb.TableProperties object, or + * nullptr if an an exception occurs + */ + static jobject fromCppTableProperties( + JNIEnv* env, const ROCKSDB_NAMESPACE::TableProperties& table_properties) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID mid = env->GetMethodID( + jclazz, "", + "(JJJJJJJJJJJJJJJJJJJJJJ[BLjava/lang/String;Ljava/lang/String;Ljava/" + "lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/" + "String;Ljava/util/Map;Ljava/util/Map;)V"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return nullptr; + } + + jbyteArray jcolumn_family_name = ROCKSDB_NAMESPACE::JniUtil::copyBytes( + env, table_properties.column_family_name); + if (jcolumn_family_name == nullptr) { + // exception occurred creating java string + return nullptr; + } + + jstring jfilter_policy_name = ROCKSDB_NAMESPACE::JniUtil::toJavaString( + env, &table_properties.filter_policy_name, true); + if (env->ExceptionCheck()) { + // exception occurred creating java string + env->DeleteLocalRef(jcolumn_family_name); + return nullptr; + } + + jstring jcomparator_name = ROCKSDB_NAMESPACE::JniUtil::toJavaString( + env, &table_properties.comparator_name, true); + if (env->ExceptionCheck()) { + // exception occurred creating java string + env->DeleteLocalRef(jcolumn_family_name); + env->DeleteLocalRef(jfilter_policy_name); + return nullptr; + } + + jstring jmerge_operator_name = ROCKSDB_NAMESPACE::JniUtil::toJavaString( + env, &table_properties.merge_operator_name, true); + if (env->ExceptionCheck()) { + // exception occurred creating java string + env->DeleteLocalRef(jcolumn_family_name); + env->DeleteLocalRef(jfilter_policy_name); + env->DeleteLocalRef(jcomparator_name); + return nullptr; + } + + jstring jprefix_extractor_name = ROCKSDB_NAMESPACE::JniUtil::toJavaString( + env, &table_properties.prefix_extractor_name, true); + if (env->ExceptionCheck()) { + // exception occurred creating java string + env->DeleteLocalRef(jcolumn_family_name); + env->DeleteLocalRef(jfilter_policy_name); + env->DeleteLocalRef(jcomparator_name); + env->DeleteLocalRef(jmerge_operator_name); + return nullptr; + } + + jstring jproperty_collectors_names = + ROCKSDB_NAMESPACE::JniUtil::toJavaString( + env, &table_properties.property_collectors_names, true); + if (env->ExceptionCheck()) { + // exception occurred creating java string + env->DeleteLocalRef(jcolumn_family_name); + env->DeleteLocalRef(jfilter_policy_name); + env->DeleteLocalRef(jcomparator_name); + env->DeleteLocalRef(jmerge_operator_name); + env->DeleteLocalRef(jprefix_extractor_name); + return nullptr; + } + + jstring jcompression_name = ROCKSDB_NAMESPACE::JniUtil::toJavaString( + env, &table_properties.compression_name, true); + if (env->ExceptionCheck()) { + // exception occurred creating java string + env->DeleteLocalRef(jcolumn_family_name); + env->DeleteLocalRef(jfilter_policy_name); + env->DeleteLocalRef(jcomparator_name); + env->DeleteLocalRef(jmerge_operator_name); + env->DeleteLocalRef(jprefix_extractor_name); + env->DeleteLocalRef(jproperty_collectors_names); + return nullptr; + } + + // Map + jobject juser_collected_properties = + ROCKSDB_NAMESPACE::HashMapJni::fromCppMap( + env, &table_properties.user_collected_properties); + if (env->ExceptionCheck()) { + // exception occurred creating java map + env->DeleteLocalRef(jcolumn_family_name); + env->DeleteLocalRef(jfilter_policy_name); + env->DeleteLocalRef(jcomparator_name); + env->DeleteLocalRef(jmerge_operator_name); + env->DeleteLocalRef(jprefix_extractor_name); + env->DeleteLocalRef(jproperty_collectors_names); + env->DeleteLocalRef(jcompression_name); + return nullptr; + } + + // Map + jobject jreadable_properties = ROCKSDB_NAMESPACE::HashMapJni::fromCppMap( + env, &table_properties.readable_properties); + if (env->ExceptionCheck()) { + // exception occurred creating java map + env->DeleteLocalRef(jcolumn_family_name); + env->DeleteLocalRef(jfilter_policy_name); + env->DeleteLocalRef(jcomparator_name); + env->DeleteLocalRef(jmerge_operator_name); + env->DeleteLocalRef(jprefix_extractor_name); + env->DeleteLocalRef(jproperty_collectors_names); + env->DeleteLocalRef(jcompression_name); + env->DeleteLocalRef(juser_collected_properties); + return nullptr; + } + + jobject jtable_properties = env->NewObject( + jclazz, mid, static_cast(table_properties.data_size), + static_cast(table_properties.index_size), + static_cast(table_properties.index_partitions), + static_cast(table_properties.top_level_index_size), + static_cast(table_properties.index_key_is_user_key), + static_cast(table_properties.index_value_is_delta_encoded), + static_cast(table_properties.filter_size), + static_cast(table_properties.raw_key_size), + static_cast(table_properties.raw_value_size), + static_cast(table_properties.num_data_blocks), + static_cast(table_properties.num_entries), + static_cast(table_properties.num_deletions), + static_cast(table_properties.num_merge_operands), + static_cast(table_properties.num_range_deletions), + static_cast(table_properties.format_version), + static_cast(table_properties.fixed_key_len), + static_cast(table_properties.column_family_id), + static_cast(table_properties.creation_time), + static_cast(table_properties.oldest_key_time), + static_cast( + table_properties.slow_compression_estimated_data_size), + static_cast( + table_properties.fast_compression_estimated_data_size), + static_cast( + table_properties.external_sst_file_global_seqno_offset), + jcolumn_family_name, jfilter_policy_name, jcomparator_name, + jmerge_operator_name, jprefix_extractor_name, + jproperty_collectors_names, jcompression_name, + juser_collected_properties, jreadable_properties); + + if (env->ExceptionCheck()) { + return nullptr; + } + + return jtable_properties; + } + + private: + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/TableProperties"); + } +}; + +class ColumnFamilyDescriptorJni : public JavaClass { + public: + /** + * Get the Java Class org.rocksdb.ColumnFamilyDescriptor + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/ColumnFamilyDescriptor"); + } + + /** + * Create a new Java org.rocksdb.ColumnFamilyDescriptor object with the same + * properties as the provided C++ ROCKSDB_NAMESPACE::ColumnFamilyDescriptor + * object + * + * @param env A pointer to the Java environment + * @param cfd A pointer to ROCKSDB_NAMESPACE::ColumnFamilyDescriptor object + * + * @return A reference to a Java org.rocksdb.ColumnFamilyDescriptor object, or + * nullptr if an an exception occurs + */ + static jobject construct(JNIEnv* env, ColumnFamilyDescriptor* cfd) { + jbyteArray jcf_name = JniUtil::copyBytes(env, cfd->name); + jobject cfopts = ColumnFamilyOptionsJni::construct(env, &(cfd->options)); + + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID mid = env->GetMethodID(jclazz, "", + "([BLorg/rocksdb/ColumnFamilyOptions;)V"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + env->DeleteLocalRef(jcf_name); + return nullptr; + } + + jobject jcfd = env->NewObject(jclazz, mid, jcf_name, cfopts); + if (env->ExceptionCheck()) { + env->DeleteLocalRef(jcf_name); + return nullptr; + } + + return jcfd; + } + + /** + * Get the Java Method: ColumnFamilyDescriptor#columnFamilyName + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getColumnFamilyNameMethod(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "columnFamilyName", "()[B"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: ColumnFamilyDescriptor#columnFamilyOptions + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getColumnFamilyOptionsMethod(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID( + jclazz, "columnFamilyOptions", "()Lorg/rocksdb/ColumnFamilyOptions;"); + assert(mid != nullptr); + return mid; + } +}; + +// The portal class for org.rocksdb.IndexType +class IndexTypeJni { + public: + // Returns the equivalent org.rocksdb.IndexType for the provided + // C++ ROCKSDB_NAMESPACE::IndexType enum + static jbyte toJavaIndexType( + const ROCKSDB_NAMESPACE::BlockBasedTableOptions::IndexType& index_type) { + switch (index_type) { + case ROCKSDB_NAMESPACE::BlockBasedTableOptions::IndexType::kBinarySearch: + return 0x0; + case ROCKSDB_NAMESPACE::BlockBasedTableOptions::IndexType::kHashSearch: + return 0x1; + case ROCKSDB_NAMESPACE::BlockBasedTableOptions::IndexType:: + kTwoLevelIndexSearch: + return 0x2; + case ROCKSDB_NAMESPACE::BlockBasedTableOptions::IndexType:: + kBinarySearchWithFirstKey: + return 0x3; + default: + return 0x7F; // undefined + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::IndexType enum for the + // provided Java org.rocksdb.IndexType + static ROCKSDB_NAMESPACE::BlockBasedTableOptions::IndexType toCppIndexType( + jbyte jindex_type) { + switch (jindex_type) { + case 0x0: + return ROCKSDB_NAMESPACE::BlockBasedTableOptions::IndexType:: + kBinarySearch; + case 0x1: + return ROCKSDB_NAMESPACE::BlockBasedTableOptions::IndexType:: + kHashSearch; + case 0x2: + return ROCKSDB_NAMESPACE::BlockBasedTableOptions::IndexType:: + kTwoLevelIndexSearch; + case 0x3: + return ROCKSDB_NAMESPACE::BlockBasedTableOptions::IndexType:: + kBinarySearchWithFirstKey; + default: + // undefined/default + return ROCKSDB_NAMESPACE::BlockBasedTableOptions::IndexType:: + kBinarySearch; + } + } +}; + +// The portal class for org.rocksdb.DataBlockIndexType +class DataBlockIndexTypeJni { + public: + // Returns the equivalent org.rocksdb.DataBlockIndexType for the provided + // C++ ROCKSDB_NAMESPACE::DataBlockIndexType enum + static jbyte toJavaDataBlockIndexType( + const ROCKSDB_NAMESPACE::BlockBasedTableOptions::DataBlockIndexType& + index_type) { + switch (index_type) { + case ROCKSDB_NAMESPACE::BlockBasedTableOptions::DataBlockIndexType:: + kDataBlockBinarySearch: + return 0x0; + case ROCKSDB_NAMESPACE::BlockBasedTableOptions::DataBlockIndexType:: + kDataBlockBinaryAndHash: + return 0x1; + default: + return 0x7F; // undefined + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::DataBlockIndexType enum for + // the provided Java org.rocksdb.DataBlockIndexType + static ROCKSDB_NAMESPACE::BlockBasedTableOptions::DataBlockIndexType + toCppDataBlockIndexType(jbyte jindex_type) { + switch (jindex_type) { + case 0x0: + return ROCKSDB_NAMESPACE::BlockBasedTableOptions::DataBlockIndexType:: + kDataBlockBinarySearch; + case 0x1: + return ROCKSDB_NAMESPACE::BlockBasedTableOptions::DataBlockIndexType:: + kDataBlockBinaryAndHash; + default: + // undefined/default + return ROCKSDB_NAMESPACE::BlockBasedTableOptions::DataBlockIndexType:: + kDataBlockBinarySearch; + } + } +}; + +// The portal class for org.rocksdb.ChecksumType +class ChecksumTypeJni { + public: + // Returns the equivalent org.rocksdb.ChecksumType for the provided + // C++ ROCKSDB_NAMESPACE::ChecksumType enum + static jbyte toJavaChecksumType( + const ROCKSDB_NAMESPACE::ChecksumType& checksum_type) { + switch (checksum_type) { + case ROCKSDB_NAMESPACE::ChecksumType::kNoChecksum: + return 0x0; + case ROCKSDB_NAMESPACE::ChecksumType::kCRC32c: + return 0x1; + case ROCKSDB_NAMESPACE::ChecksumType::kxxHash: + return 0x2; + case ROCKSDB_NAMESPACE::ChecksumType::kxxHash64: + return 0x3; + case ROCKSDB_NAMESPACE::ChecksumType::kXXH3: + return 0x4; + default: + return 0x7F; // undefined + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::ChecksumType enum for the + // provided Java org.rocksdb.ChecksumType + static ROCKSDB_NAMESPACE::ChecksumType toCppChecksumType( + jbyte jchecksum_type) { + switch (jchecksum_type) { + case 0x0: + return ROCKSDB_NAMESPACE::ChecksumType::kNoChecksum; + case 0x1: + return ROCKSDB_NAMESPACE::ChecksumType::kCRC32c; + case 0x2: + return ROCKSDB_NAMESPACE::ChecksumType::kxxHash; + case 0x3: + return ROCKSDB_NAMESPACE::ChecksumType::kxxHash64; + case 0x4: + return ROCKSDB_NAMESPACE::ChecksumType::kXXH3; + default: + // undefined/default + return ROCKSDB_NAMESPACE::ChecksumType::kCRC32c; + } + } +}; + +// The portal class for org.rocksdb.IndexShorteningMode +class IndexShorteningModeJni { + public: + // Returns the equivalent org.rocksdb.IndexShorteningMode for the provided + // C++ ROCKSDB_NAMESPACE::IndexShorteningMode enum + static jbyte toJavaIndexShorteningMode( + const ROCKSDB_NAMESPACE::BlockBasedTableOptions::IndexShorteningMode& + index_shortening_mode) { + switch (index_shortening_mode) { + case ROCKSDB_NAMESPACE::BlockBasedTableOptions::IndexShorteningMode:: + kNoShortening: + return 0x0; + case ROCKSDB_NAMESPACE::BlockBasedTableOptions::IndexShorteningMode:: + kShortenSeparators: + return 0x1; + case ROCKSDB_NAMESPACE::BlockBasedTableOptions::IndexShorteningMode:: + kShortenSeparatorsAndSuccessor: + return 0x2; + default: + return 0x7F; // undefined + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::IndexShorteningMode enum for + // the provided Java org.rocksdb.IndexShorteningMode + static ROCKSDB_NAMESPACE::BlockBasedTableOptions::IndexShorteningMode + toCppIndexShorteningMode(jbyte jindex_shortening_mode) { + switch (jindex_shortening_mode) { + case 0x0: + return ROCKSDB_NAMESPACE::BlockBasedTableOptions::IndexShorteningMode:: + kNoShortening; + case 0x1: + return ROCKSDB_NAMESPACE::BlockBasedTableOptions::IndexShorteningMode:: + kShortenSeparators; + case 0x2: + return ROCKSDB_NAMESPACE::BlockBasedTableOptions::IndexShorteningMode:: + kShortenSeparatorsAndSuccessor; + default: + // undefined/default + return ROCKSDB_NAMESPACE::BlockBasedTableOptions::IndexShorteningMode:: + kShortenSeparators; + } + } +}; + +// The portal class for org.rocksdb.Priority +class PriorityJni { + public: + // Returns the equivalent org.rocksdb.Priority for the provided + // C++ ROCKSDB_NAMESPACE::Env::Priority enum + static jbyte toJavaPriority( + const ROCKSDB_NAMESPACE::Env::Priority& priority) { + switch (priority) { + case ROCKSDB_NAMESPACE::Env::Priority::BOTTOM: + return 0x0; + case ROCKSDB_NAMESPACE::Env::Priority::LOW: + return 0x1; + case ROCKSDB_NAMESPACE::Env::Priority::HIGH: + return 0x2; + case ROCKSDB_NAMESPACE::Env::Priority::TOTAL: + return 0x3; + default: + return 0x7F; // undefined + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::env::Priority enum for the + // provided Java org.rocksdb.Priority + static ROCKSDB_NAMESPACE::Env::Priority toCppPriority(jbyte jpriority) { + switch (jpriority) { + case 0x0: + return ROCKSDB_NAMESPACE::Env::Priority::BOTTOM; + case 0x1: + return ROCKSDB_NAMESPACE::Env::Priority::LOW; + case 0x2: + return ROCKSDB_NAMESPACE::Env::Priority::HIGH; + case 0x3: + return ROCKSDB_NAMESPACE::Env::Priority::TOTAL; + default: + // undefined/default + return ROCKSDB_NAMESPACE::Env::Priority::LOW; + } + } +}; + +// The portal class for org.rocksdb.ThreadType +class ThreadTypeJni { + public: + // Returns the equivalent org.rocksdb.ThreadType for the provided + // C++ ROCKSDB_NAMESPACE::ThreadStatus::ThreadType enum + static jbyte toJavaThreadType( + const ROCKSDB_NAMESPACE::ThreadStatus::ThreadType& thread_type) { + switch (thread_type) { + case ROCKSDB_NAMESPACE::ThreadStatus::ThreadType::HIGH_PRIORITY: + return 0x0; + case ROCKSDB_NAMESPACE::ThreadStatus::ThreadType::LOW_PRIORITY: + return 0x1; + case ROCKSDB_NAMESPACE::ThreadStatus::ThreadType::USER: + return 0x2; + case ROCKSDB_NAMESPACE::ThreadStatus::ThreadType::BOTTOM_PRIORITY: + return 0x3; + default: + return 0x7F; // undefined + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::ThreadStatus::ThreadType enum + // for the provided Java org.rocksdb.ThreadType + static ROCKSDB_NAMESPACE::ThreadStatus::ThreadType toCppThreadType( + jbyte jthread_type) { + switch (jthread_type) { + case 0x0: + return ROCKSDB_NAMESPACE::ThreadStatus::ThreadType::HIGH_PRIORITY; + case 0x1: + return ROCKSDB_NAMESPACE::ThreadStatus::ThreadType::LOW_PRIORITY; + case 0x2: + return ThreadStatus::ThreadType::USER; + case 0x3: + return ROCKSDB_NAMESPACE::ThreadStatus::ThreadType::BOTTOM_PRIORITY; + default: + // undefined/default + return ROCKSDB_NAMESPACE::ThreadStatus::ThreadType::LOW_PRIORITY; + } + } +}; + +// The portal class for org.rocksdb.OperationType +class OperationTypeJni { + public: + // Returns the equivalent org.rocksdb.OperationType for the provided + // C++ ROCKSDB_NAMESPACE::ThreadStatus::OperationType enum + static jbyte toJavaOperationType( + const ROCKSDB_NAMESPACE::ThreadStatus::OperationType& operation_type) { + switch (operation_type) { + case ROCKSDB_NAMESPACE::ThreadStatus::OperationType::OP_UNKNOWN: + return 0x0; + case ROCKSDB_NAMESPACE::ThreadStatus::OperationType::OP_COMPACTION: + return 0x1; + case ROCKSDB_NAMESPACE::ThreadStatus::OperationType::OP_FLUSH: + return 0x2; + case ROCKSDB_NAMESPACE::ThreadStatus::OperationType::OP_DBOPEN: + return 0x3; + default: + return 0x7F; // undefined + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::ThreadStatus::OperationType + // enum for the provided Java org.rocksdb.OperationType + static ROCKSDB_NAMESPACE::ThreadStatus::OperationType toCppOperationType( + jbyte joperation_type) { + switch (joperation_type) { + case 0x0: + return ROCKSDB_NAMESPACE::ThreadStatus::OperationType::OP_UNKNOWN; + case 0x1: + return ROCKSDB_NAMESPACE::ThreadStatus::OperationType::OP_COMPACTION; + case 0x2: + return ROCKSDB_NAMESPACE::ThreadStatus::OperationType::OP_FLUSH; + case 0x3: + return ROCKSDB_NAMESPACE::ThreadStatus::OperationType::OP_DBOPEN; + default: + // undefined/default + return ROCKSDB_NAMESPACE::ThreadStatus::OperationType::OP_UNKNOWN; + } + } +}; + +// The portal class for org.rocksdb.OperationStage +class OperationStageJni { + public: + // Returns the equivalent org.rocksdb.OperationStage for the provided + // C++ ROCKSDB_NAMESPACE::ThreadStatus::OperationStage enum + static jbyte toJavaOperationStage( + const ROCKSDB_NAMESPACE::ThreadStatus::OperationStage& operation_stage) { + switch (operation_stage) { + case ROCKSDB_NAMESPACE::ThreadStatus::OperationStage::STAGE_UNKNOWN: + return 0x0; + case ROCKSDB_NAMESPACE::ThreadStatus::OperationStage::STAGE_FLUSH_RUN: + return 0x1; + case ROCKSDB_NAMESPACE::ThreadStatus::OperationStage:: + STAGE_FLUSH_WRITE_L0: + return 0x2; + case ROCKSDB_NAMESPACE::ThreadStatus::OperationStage:: + STAGE_COMPACTION_PREPARE: + return 0x3; + case ROCKSDB_NAMESPACE::ThreadStatus::OperationStage:: + STAGE_COMPACTION_RUN: + return 0x4; + case ROCKSDB_NAMESPACE::ThreadStatus::OperationStage:: + STAGE_COMPACTION_PROCESS_KV: + return 0x5; + case ROCKSDB_NAMESPACE::ThreadStatus::OperationStage:: + STAGE_COMPACTION_INSTALL: + return 0x6; + case ROCKSDB_NAMESPACE::ThreadStatus::OperationStage:: + STAGE_COMPACTION_SYNC_FILE: + return 0x7; + case ROCKSDB_NAMESPACE::ThreadStatus::OperationStage:: + STAGE_PICK_MEMTABLES_TO_FLUSH: + return 0x8; + case ROCKSDB_NAMESPACE::ThreadStatus::OperationStage:: + STAGE_MEMTABLE_ROLLBACK: + return 0x9; + case ROCKSDB_NAMESPACE::ThreadStatus::OperationStage:: + STAGE_MEMTABLE_INSTALL_FLUSH_RESULTS: + return 0xA; + default: + return 0x7F; // undefined + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::ThreadStatus::OperationStage + // enum for the provided Java org.rocksdb.OperationStage + static ROCKSDB_NAMESPACE::ThreadStatus::OperationStage toCppOperationStage( + jbyte joperation_stage) { + switch (joperation_stage) { + case 0x0: + return ROCKSDB_NAMESPACE::ThreadStatus::OperationStage::STAGE_UNKNOWN; + case 0x1: + return ROCKSDB_NAMESPACE::ThreadStatus::OperationStage::STAGE_FLUSH_RUN; + case 0x2: + return ROCKSDB_NAMESPACE::ThreadStatus::OperationStage:: + STAGE_FLUSH_WRITE_L0; + case 0x3: + return ROCKSDB_NAMESPACE::ThreadStatus::OperationStage:: + STAGE_COMPACTION_PREPARE; + case 0x4: + return ROCKSDB_NAMESPACE::ThreadStatus::OperationStage:: + STAGE_COMPACTION_RUN; + case 0x5: + return ROCKSDB_NAMESPACE::ThreadStatus::OperationStage:: + STAGE_COMPACTION_PROCESS_KV; + case 0x6: + return ROCKSDB_NAMESPACE::ThreadStatus::OperationStage:: + STAGE_COMPACTION_INSTALL; + case 0x7: + return ROCKSDB_NAMESPACE::ThreadStatus::OperationStage:: + STAGE_COMPACTION_SYNC_FILE; + case 0x8: + return ROCKSDB_NAMESPACE::ThreadStatus::OperationStage:: + STAGE_PICK_MEMTABLES_TO_FLUSH; + case 0x9: + return ROCKSDB_NAMESPACE::ThreadStatus::OperationStage:: + STAGE_MEMTABLE_ROLLBACK; + case 0xA: + return ROCKSDB_NAMESPACE::ThreadStatus::OperationStage:: + STAGE_MEMTABLE_INSTALL_FLUSH_RESULTS; + default: + // undefined/default + return ROCKSDB_NAMESPACE::ThreadStatus::OperationStage::STAGE_UNKNOWN; + } + } +}; + +// The portal class for org.rocksdb.StateType +class StateTypeJni { + public: + // Returns the equivalent org.rocksdb.StateType for the provided + // C++ ROCKSDB_NAMESPACE::ThreadStatus::StateType enum + static jbyte toJavaStateType( + const ROCKSDB_NAMESPACE::ThreadStatus::StateType& state_type) { + switch (state_type) { + case ROCKSDB_NAMESPACE::ThreadStatus::StateType::STATE_UNKNOWN: + return 0x0; + case ROCKSDB_NAMESPACE::ThreadStatus::StateType::STATE_MUTEX_WAIT: + return 0x1; + default: + return 0x7F; // undefined + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::ThreadStatus::StateType enum + // for the provided Java org.rocksdb.StateType + static ROCKSDB_NAMESPACE::ThreadStatus::StateType toCppStateType( + jbyte jstate_type) { + switch (jstate_type) { + case 0x0: + return ROCKSDB_NAMESPACE::ThreadStatus::StateType::STATE_UNKNOWN; + case 0x1: + return ROCKSDB_NAMESPACE::ThreadStatus::StateType::STATE_MUTEX_WAIT; + default: + // undefined/default + return ROCKSDB_NAMESPACE::ThreadStatus::StateType::STATE_UNKNOWN; + } + } +}; + +// The portal class for org.rocksdb.ThreadStatus +class ThreadStatusJni : public JavaClass { + public: + /** + * Get the Java Class org.rocksdb.ThreadStatus + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/ThreadStatus"); + } + + /** + * Create a new Java org.rocksdb.ThreadStatus object with the same + * properties as the provided C++ ROCKSDB_NAMESPACE::ThreadStatus object + * + * @param env A pointer to the Java environment + * @param thread_status A pointer to ROCKSDB_NAMESPACE::ThreadStatus object + * + * @return A reference to a Java org.rocksdb.ColumnFamilyOptions object, or + * nullptr if an an exception occurs + */ + static jobject construct( + JNIEnv* env, const ROCKSDB_NAMESPACE::ThreadStatus* thread_status) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID mid = env->GetMethodID( + jclazz, "", "(JBLjava/lang/String;Ljava/lang/String;BJB[JB)V"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return nullptr; + } + + jstring jdb_name = + JniUtil::toJavaString(env, &(thread_status->db_name), true); + if (env->ExceptionCheck()) { + // an error occurred + return nullptr; + } + + jstring jcf_name = + JniUtil::toJavaString(env, &(thread_status->cf_name), true); + if (env->ExceptionCheck()) { + // an error occurred + env->DeleteLocalRef(jdb_name); + return nullptr; + } + + // long[] + const jsize len = static_cast( + ROCKSDB_NAMESPACE::ThreadStatus::kNumOperationProperties); + jlongArray joperation_properties = env->NewLongArray(len); + if (joperation_properties == nullptr) { + // an exception occurred + env->DeleteLocalRef(jdb_name); + env->DeleteLocalRef(jcf_name); + return nullptr; + } + jboolean is_copy; + jlong* body = env->GetLongArrayElements(joperation_properties, &is_copy); + if (body == nullptr) { + // exception thrown: OutOfMemoryError + env->DeleteLocalRef(jdb_name); + env->DeleteLocalRef(jcf_name); + env->DeleteLocalRef(joperation_properties); + return nullptr; + } + for (size_t i = 0; i < len; ++i) { + body[i] = static_cast(thread_status->op_properties[i]); + } + env->ReleaseLongArrayElements(joperation_properties, body, + is_copy == JNI_TRUE ? 0 : JNI_ABORT); + + jobject jcfd = env->NewObject( + jclazz, mid, static_cast(thread_status->thread_id), + ThreadTypeJni::toJavaThreadType(thread_status->thread_type), jdb_name, + jcf_name, + OperationTypeJni::toJavaOperationType(thread_status->operation_type), + static_cast(thread_status->op_elapsed_micros), + OperationStageJni::toJavaOperationStage(thread_status->operation_stage), + joperation_properties, + StateTypeJni::toJavaStateType(thread_status->state_type)); + if (env->ExceptionCheck()) { + // exception occurred + env->DeleteLocalRef(jdb_name); + env->DeleteLocalRef(jcf_name); + env->DeleteLocalRef(joperation_properties); + return nullptr; + } + + // cleanup + env->DeleteLocalRef(jdb_name); + env->DeleteLocalRef(jcf_name); + env->DeleteLocalRef(joperation_properties); + + return jcfd; + } +}; + +// The portal class for org.rocksdb.CompactionStyle +class CompactionStyleJni { + public: + // Returns the equivalent org.rocksdb.CompactionStyle for the provided + // C++ ROCKSDB_NAMESPACE::CompactionStyle enum + static jbyte toJavaCompactionStyle( + const ROCKSDB_NAMESPACE::CompactionStyle& compaction_style) { + switch (compaction_style) { + case ROCKSDB_NAMESPACE::CompactionStyle::kCompactionStyleLevel: + return 0x0; + case ROCKSDB_NAMESPACE::CompactionStyle::kCompactionStyleUniversal: + return 0x1; + case ROCKSDB_NAMESPACE::CompactionStyle::kCompactionStyleFIFO: + return 0x2; + case ROCKSDB_NAMESPACE::CompactionStyle::kCompactionStyleNone: + return 0x3; + default: + return 0x7F; // undefined + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::CompactionStyle enum for the + // provided Java org.rocksdb.CompactionStyle + static ROCKSDB_NAMESPACE::CompactionStyle toCppCompactionStyle( + jbyte jcompaction_style) { + switch (jcompaction_style) { + case 0x0: + return ROCKSDB_NAMESPACE::CompactionStyle::kCompactionStyleLevel; + case 0x1: + return ROCKSDB_NAMESPACE::CompactionStyle::kCompactionStyleUniversal; + case 0x2: + return ROCKSDB_NAMESPACE::CompactionStyle::kCompactionStyleFIFO; + case 0x3: + return ROCKSDB_NAMESPACE::CompactionStyle::kCompactionStyleNone; + default: + // undefined/default + return ROCKSDB_NAMESPACE::CompactionStyle::kCompactionStyleLevel; + } + } +}; + +// The portal class for org.rocksdb.CompactionReason +class CompactionReasonJni { + public: + // Returns the equivalent org.rocksdb.CompactionReason for the provided + // C++ ROCKSDB_NAMESPACE::CompactionReason enum + static jbyte toJavaCompactionReason( + const ROCKSDB_NAMESPACE::CompactionReason& compaction_reason) { + switch (compaction_reason) { + case ROCKSDB_NAMESPACE::CompactionReason::kUnknown: + return 0x0; + case ROCKSDB_NAMESPACE::CompactionReason::kLevelL0FilesNum: + return 0x1; + case ROCKSDB_NAMESPACE::CompactionReason::kLevelMaxLevelSize: + return 0x2; + case ROCKSDB_NAMESPACE::CompactionReason::kUniversalSizeAmplification: + return 0x3; + case ROCKSDB_NAMESPACE::CompactionReason::kUniversalSizeRatio: + return 0x4; + case ROCKSDB_NAMESPACE::CompactionReason::kUniversalSortedRunNum: + return 0x5; + case ROCKSDB_NAMESPACE::CompactionReason::kFIFOMaxSize: + return 0x6; + case ROCKSDB_NAMESPACE::CompactionReason::kFIFOReduceNumFiles: + return 0x7; + case ROCKSDB_NAMESPACE::CompactionReason::kFIFOTtl: + return 0x8; + case ROCKSDB_NAMESPACE::CompactionReason::kManualCompaction: + return 0x9; + case ROCKSDB_NAMESPACE::CompactionReason::kFilesMarkedForCompaction: + return 0x10; + case ROCKSDB_NAMESPACE::CompactionReason::kBottommostFiles: + return 0x0A; + case ROCKSDB_NAMESPACE::CompactionReason::kTtl: + return 0x0B; + case ROCKSDB_NAMESPACE::CompactionReason::kFlush: + return 0x0C; + case ROCKSDB_NAMESPACE::CompactionReason::kExternalSstIngestion: + return 0x0D; + case ROCKSDB_NAMESPACE::CompactionReason::kPeriodicCompaction: + return 0x0E; + case ROCKSDB_NAMESPACE::CompactionReason::kChangeTemperature: + return 0x0F; + case ROCKSDB_NAMESPACE::CompactionReason::kForcedBlobGC: + return 0x11; + case ROCKSDB_NAMESPACE::CompactionReason::kRoundRobinTtl: + return 0x12; + case ROCKSDB_NAMESPACE::CompactionReason::kRefitLevel: + return 0x13; + default: + return 0x7F; // undefined + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::CompactionReason enum for the + // provided Java org.rocksdb.CompactionReason + static ROCKSDB_NAMESPACE::CompactionReason toCppCompactionReason( + jbyte jcompaction_reason) { + switch (jcompaction_reason) { + case 0x0: + return ROCKSDB_NAMESPACE::CompactionReason::kUnknown; + case 0x1: + return ROCKSDB_NAMESPACE::CompactionReason::kLevelL0FilesNum; + case 0x2: + return ROCKSDB_NAMESPACE::CompactionReason::kLevelMaxLevelSize; + case 0x3: + return ROCKSDB_NAMESPACE::CompactionReason::kUniversalSizeAmplification; + case 0x4: + return ROCKSDB_NAMESPACE::CompactionReason::kUniversalSizeRatio; + case 0x5: + return ROCKSDB_NAMESPACE::CompactionReason::kUniversalSortedRunNum; + case 0x6: + return ROCKSDB_NAMESPACE::CompactionReason::kFIFOMaxSize; + case 0x7: + return ROCKSDB_NAMESPACE::CompactionReason::kFIFOReduceNumFiles; + case 0x8: + return ROCKSDB_NAMESPACE::CompactionReason::kFIFOTtl; + case 0x9: + return ROCKSDB_NAMESPACE::CompactionReason::kManualCompaction; + case 0x10: + return ROCKSDB_NAMESPACE::CompactionReason::kFilesMarkedForCompaction; + case 0x0A: + return ROCKSDB_NAMESPACE::CompactionReason::kBottommostFiles; + case 0x0B: + return ROCKSDB_NAMESPACE::CompactionReason::kTtl; + case 0x0C: + return ROCKSDB_NAMESPACE::CompactionReason::kFlush; + case 0x0D: + return ROCKSDB_NAMESPACE::CompactionReason::kExternalSstIngestion; + case 0x0E: + return ROCKSDB_NAMESPACE::CompactionReason::kPeriodicCompaction; + case 0x0F: + return ROCKSDB_NAMESPACE::CompactionReason::kChangeTemperature; + case 0x11: + return ROCKSDB_NAMESPACE::CompactionReason::kForcedBlobGC; + case 0x12: + return ROCKSDB_NAMESPACE::CompactionReason::kRoundRobinTtl; + case 0x13: + return ROCKSDB_NAMESPACE::CompactionReason::kRefitLevel; + default: + // undefined/default + return ROCKSDB_NAMESPACE::CompactionReason::kUnknown; + } + } +}; + +// The portal class for org.rocksdb.WalFileType +class WalFileTypeJni { + public: + // Returns the equivalent org.rocksdb.WalFileType for the provided + // C++ ROCKSDB_NAMESPACE::WalFileType enum + static jbyte toJavaWalFileType( + const ROCKSDB_NAMESPACE::WalFileType& wal_file_type) { + switch (wal_file_type) { + case ROCKSDB_NAMESPACE::WalFileType::kArchivedLogFile: + return 0x0; + case ROCKSDB_NAMESPACE::WalFileType::kAliveLogFile: + return 0x1; + default: + return 0x7F; // undefined + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::WalFileType enum for the + // provided Java org.rocksdb.WalFileType + static ROCKSDB_NAMESPACE::WalFileType toCppWalFileType(jbyte jwal_file_type) { + switch (jwal_file_type) { + case 0x0: + return ROCKSDB_NAMESPACE::WalFileType::kArchivedLogFile; + case 0x1: + return ROCKSDB_NAMESPACE::WalFileType::kAliveLogFile; + default: + // undefined/default + return ROCKSDB_NAMESPACE::WalFileType::kAliveLogFile; + } + } +}; + +class LogFileJni : public JavaClass { + public: + /** + * Create a new Java org.rocksdb.LogFile object. + * + * @param env A pointer to the Java environment + * @param log_file A Cpp log file object + * + * @return A reference to a Java org.rocksdb.LogFile object, or + * nullptr if an an exception occurs + */ + static jobject fromCppLogFile(JNIEnv* env, + ROCKSDB_NAMESPACE::LogFile* log_file) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID mid = + env->GetMethodID(jclazz, "", "(Ljava/lang/String;JBJJ)V"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return nullptr; + } + + std::string path_name = log_file->PathName(); + jstring jpath_name = + ROCKSDB_NAMESPACE::JniUtil::toJavaString(env, &path_name, true); + if (env->ExceptionCheck()) { + // exception occurred creating java string + return nullptr; + } + + jobject jlog_file = env->NewObject( + jclazz, mid, jpath_name, static_cast(log_file->LogNumber()), + ROCKSDB_NAMESPACE::WalFileTypeJni::toJavaWalFileType(log_file->Type()), + static_cast(log_file->StartSequence()), + static_cast(log_file->SizeFileBytes())); + + if (env->ExceptionCheck()) { + env->DeleteLocalRef(jpath_name); + return nullptr; + } + + // cleanup + env->DeleteLocalRef(jpath_name); + + return jlog_file; + } + + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/LogFile"); + } +}; + +class LiveFileMetaDataJni : public JavaClass { + public: + /** + * Create a new Java org.rocksdb.LiveFileMetaData object. + * + * @param env A pointer to the Java environment + * @param live_file_meta_data A Cpp live file meta data object + * + * @return A reference to a Java org.rocksdb.LiveFileMetaData object, or + * nullptr if an an exception occurs + */ + static jobject fromCppLiveFileMetaData( + JNIEnv* env, ROCKSDB_NAMESPACE::LiveFileMetaData* live_file_meta_data) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID mid = env->GetMethodID( + jclazz, "", + "([BILjava/lang/String;Ljava/lang/String;JJJ[B[BJZJJ)V"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return nullptr; + } + + jbyteArray jcolumn_family_name = ROCKSDB_NAMESPACE::JniUtil::copyBytes( + env, live_file_meta_data->column_family_name); + if (jcolumn_family_name == nullptr) { + // exception occurred creating java byte array + return nullptr; + } + + jstring jfile_name = ROCKSDB_NAMESPACE::JniUtil::toJavaString( + env, &live_file_meta_data->name, true); + if (env->ExceptionCheck()) { + // exception occurred creating java string + env->DeleteLocalRef(jcolumn_family_name); + return nullptr; + } + + jstring jpath = ROCKSDB_NAMESPACE::JniUtil::toJavaString( + env, &live_file_meta_data->db_path, true); + if (env->ExceptionCheck()) { + // exception occurred creating java string + env->DeleteLocalRef(jcolumn_family_name); + env->DeleteLocalRef(jfile_name); + return nullptr; + } + + jbyteArray jsmallest_key = ROCKSDB_NAMESPACE::JniUtil::copyBytes( + env, live_file_meta_data->smallestkey); + if (jsmallest_key == nullptr) { + // exception occurred creating java byte array + env->DeleteLocalRef(jcolumn_family_name); + env->DeleteLocalRef(jfile_name); + env->DeleteLocalRef(jpath); + return nullptr; + } + + jbyteArray jlargest_key = ROCKSDB_NAMESPACE::JniUtil::copyBytes( + env, live_file_meta_data->largestkey); + if (jlargest_key == nullptr) { + // exception occurred creating java byte array + env->DeleteLocalRef(jcolumn_family_name); + env->DeleteLocalRef(jfile_name); + env->DeleteLocalRef(jpath); + env->DeleteLocalRef(jsmallest_key); + return nullptr; + } + + jobject jlive_file_meta_data = env->NewObject( + jclazz, mid, jcolumn_family_name, + static_cast(live_file_meta_data->level), jfile_name, jpath, + static_cast(live_file_meta_data->size), + static_cast(live_file_meta_data->smallest_seqno), + static_cast(live_file_meta_data->largest_seqno), jsmallest_key, + jlargest_key, + static_cast(live_file_meta_data->num_reads_sampled), + static_cast(live_file_meta_data->being_compacted), + static_cast(live_file_meta_data->num_entries), + static_cast(live_file_meta_data->num_deletions)); + + if (env->ExceptionCheck()) { + env->DeleteLocalRef(jcolumn_family_name); + env->DeleteLocalRef(jfile_name); + env->DeleteLocalRef(jpath); + env->DeleteLocalRef(jsmallest_key); + env->DeleteLocalRef(jlargest_key); + return nullptr; + } + + // cleanup + env->DeleteLocalRef(jcolumn_family_name); + env->DeleteLocalRef(jfile_name); + env->DeleteLocalRef(jpath); + env->DeleteLocalRef(jsmallest_key); + env->DeleteLocalRef(jlargest_key); + + return jlive_file_meta_data; + } + + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/LiveFileMetaData"); + } +}; + +class SstFileMetaDataJni : public JavaClass { + public: + /** + * Create a new Java org.rocksdb.SstFileMetaData object. + * + * @param env A pointer to the Java environment + * @param sst_file_meta_data A Cpp sst file meta data object + * + * @return A reference to a Java org.rocksdb.SstFileMetaData object, or + * nullptr if an an exception occurs + */ + static jobject fromCppSstFileMetaData( + JNIEnv* env, + const ROCKSDB_NAMESPACE::SstFileMetaData* sst_file_meta_data) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID mid = env->GetMethodID( + jclazz, "", "(Ljava/lang/String;Ljava/lang/String;JJJ[B[BJZJJ)V"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return nullptr; + } + + jstring jfile_name = ROCKSDB_NAMESPACE::JniUtil::toJavaString( + env, &sst_file_meta_data->name, true); + if (jfile_name == nullptr) { + // exception occurred creating java byte array + return nullptr; + } + + jstring jpath = ROCKSDB_NAMESPACE::JniUtil::toJavaString( + env, &sst_file_meta_data->db_path, true); + if (jpath == nullptr) { + // exception occurred creating java byte array + env->DeleteLocalRef(jfile_name); + return nullptr; + } + + jbyteArray jsmallest_key = ROCKSDB_NAMESPACE::JniUtil::copyBytes( + env, sst_file_meta_data->smallestkey); + if (jsmallest_key == nullptr) { + // exception occurred creating java byte array + env->DeleteLocalRef(jfile_name); + env->DeleteLocalRef(jpath); + return nullptr; + } + + jbyteArray jlargest_key = ROCKSDB_NAMESPACE::JniUtil::copyBytes( + env, sst_file_meta_data->largestkey); + if (jlargest_key == nullptr) { + // exception occurred creating java byte array + env->DeleteLocalRef(jfile_name); + env->DeleteLocalRef(jpath); + env->DeleteLocalRef(jsmallest_key); + return nullptr; + } + + jobject jsst_file_meta_data = env->NewObject( + jclazz, mid, jfile_name, jpath, + static_cast(sst_file_meta_data->size), + static_cast(sst_file_meta_data->smallest_seqno), + static_cast(sst_file_meta_data->largest_seqno), jsmallest_key, + jlargest_key, static_cast(sst_file_meta_data->num_reads_sampled), + static_cast(sst_file_meta_data->being_compacted), + static_cast(sst_file_meta_data->num_entries), + static_cast(sst_file_meta_data->num_deletions)); + + if (env->ExceptionCheck()) { + env->DeleteLocalRef(jfile_name); + env->DeleteLocalRef(jpath); + env->DeleteLocalRef(jsmallest_key); + env->DeleteLocalRef(jlargest_key); + return nullptr; + } + + // cleanup + env->DeleteLocalRef(jfile_name); + env->DeleteLocalRef(jpath); + env->DeleteLocalRef(jsmallest_key); + env->DeleteLocalRef(jlargest_key); + + return jsst_file_meta_data; + } + + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/SstFileMetaData"); + } +}; + +class LevelMetaDataJni : public JavaClass { + public: + /** + * Create a new Java org.rocksdb.LevelMetaData object. + * + * @param env A pointer to the Java environment + * @param level_meta_data A Cpp level meta data object + * + * @return A reference to a Java org.rocksdb.LevelMetaData object, or + * nullptr if an an exception occurs + */ + static jobject fromCppLevelMetaData( + JNIEnv* env, const ROCKSDB_NAMESPACE::LevelMetaData* level_meta_data) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID mid = env->GetMethodID(jclazz, "", + "(IJ[Lorg/rocksdb/SstFileMetaData;)V"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return nullptr; + } + + const jsize jlen = static_cast(level_meta_data->files.size()); + jobjectArray jfiles = + env->NewObjectArray(jlen, SstFileMetaDataJni::getJClass(env), nullptr); + if (jfiles == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + jsize i = 0; + for (auto it = level_meta_data->files.begin(); + it != level_meta_data->files.end(); ++it) { + jobject jfile = SstFileMetaDataJni::fromCppSstFileMetaData(env, &(*it)); + if (jfile == nullptr) { + // exception occurred + env->DeleteLocalRef(jfiles); + return nullptr; + } + env->SetObjectArrayElement(jfiles, i++, jfile); + } + + jobject jlevel_meta_data = + env->NewObject(jclazz, mid, static_cast(level_meta_data->level), + static_cast(level_meta_data->size), jfiles); + + if (env->ExceptionCheck()) { + env->DeleteLocalRef(jfiles); + return nullptr; + } + + // cleanup + env->DeleteLocalRef(jfiles); + + return jlevel_meta_data; + } + + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/LevelMetaData"); + } +}; + +class ColumnFamilyMetaDataJni : public JavaClass { + public: + /** + * Create a new Java org.rocksdb.ColumnFamilyMetaData object. + * + * @param env A pointer to the Java environment + * @param column_famly_meta_data A Cpp live file meta data object + * + * @return A reference to a Java org.rocksdb.ColumnFamilyMetaData object, or + * nullptr if an an exception occurs + */ + static jobject fromCppColumnFamilyMetaData( + JNIEnv* env, + const ROCKSDB_NAMESPACE::ColumnFamilyMetaData* column_famly_meta_data) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID mid = env->GetMethodID(jclazz, "", + "(JJ[B[Lorg/rocksdb/LevelMetaData;)V"); + if (mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return nullptr; + } + + jbyteArray jname = ROCKSDB_NAMESPACE::JniUtil::copyBytes( + env, column_famly_meta_data->name); + if (jname == nullptr) { + // exception occurred creating java byte array + return nullptr; + } + + const jsize jlen = + static_cast(column_famly_meta_data->levels.size()); + jobjectArray jlevels = + env->NewObjectArray(jlen, LevelMetaDataJni::getJClass(env), nullptr); + if (jlevels == nullptr) { + // exception thrown: OutOfMemoryError + env->DeleteLocalRef(jname); + return nullptr; + } + + jsize i = 0; + for (auto it = column_famly_meta_data->levels.begin(); + it != column_famly_meta_data->levels.end(); ++it) { + jobject jlevel = LevelMetaDataJni::fromCppLevelMetaData(env, &(*it)); + if (jlevel == nullptr) { + // exception occurred + env->DeleteLocalRef(jname); + env->DeleteLocalRef(jlevels); + return nullptr; + } + env->SetObjectArrayElement(jlevels, i++, jlevel); + } + + jobject jcolumn_family_meta_data = env->NewObject( + jclazz, mid, static_cast(column_famly_meta_data->size), + static_cast(column_famly_meta_data->file_count), jname, jlevels); + + if (env->ExceptionCheck()) { + env->DeleteLocalRef(jname); + env->DeleteLocalRef(jlevels); + return nullptr; + } + + // cleanup + env->DeleteLocalRef(jname); + env->DeleteLocalRef(jlevels); + + return jcolumn_family_meta_data; + } + + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/ColumnFamilyMetaData"); + } +}; + +// The portal class for org.rocksdb.AbstractTraceWriter +class AbstractTraceWriterJni + : public RocksDBNativeClass< + const ROCKSDB_NAMESPACE::TraceWriterJniCallback*, + AbstractTraceWriterJni> { + public: + /** + * Get the Java Class org.rocksdb.AbstractTraceWriter + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, + "org/rocksdb/AbstractTraceWriter"); + } + + /** + * Get the Java Method: AbstractTraceWriter#write + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getWriteProxyMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "writeProxy", "(J)S"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractTraceWriter#closeWriter + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getCloseWriterProxyMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "closeWriterProxy", "()S"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractTraceWriter#getFileSize + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getGetFileSizeMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "getFileSize", "()J"); + assert(mid != nullptr); + return mid; + } +}; + +// The portal class for org.rocksdb.AbstractWalFilter +class AbstractWalFilterJni + : public RocksDBNativeClass { + public: + /** + * Get the Java Class org.rocksdb.AbstractWalFilter + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, "org/rocksdb/AbstractWalFilter"); + } + + /** + * Get the Java Method: AbstractWalFilter#columnFamilyLogNumberMap + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getColumnFamilyLogNumberMapMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = + env->GetMethodID(jclazz, "columnFamilyLogNumberMap", + "(Ljava/util/Map;Ljava/util/Map;)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractTraceWriter#logRecordFoundProxy + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getLogRecordFoundProxyMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = env->GetMethodID(jclazz, "logRecordFoundProxy", + "(JLjava/lang/String;JJ)S"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractTraceWriter#name + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID or nullptr if the class or method id could not + * be retrieved + */ + static jmethodID getNameMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + static jmethodID mid = + env->GetMethodID(jclazz, "name", "()Ljava/lang/String;"); + assert(mid != nullptr); + return mid; + } +}; + +// The portal class for org.rocksdb.WalProcessingOption +class WalProcessingOptionJni { + public: + // Returns the equivalent org.rocksdb.WalProcessingOption for the provided + // C++ ROCKSDB_NAMESPACE::WalFilter::WalProcessingOption enum + static jbyte toJavaWalProcessingOption( + const ROCKSDB_NAMESPACE::WalFilter::WalProcessingOption& + wal_processing_option) { + switch (wal_processing_option) { + case ROCKSDB_NAMESPACE::WalFilter::WalProcessingOption:: + kContinueProcessing: + return 0x0; + case ROCKSDB_NAMESPACE::WalFilter::WalProcessingOption:: + kIgnoreCurrentRecord: + return 0x1; + case ROCKSDB_NAMESPACE::WalFilter::WalProcessingOption::kStopReplay: + return 0x2; + case ROCKSDB_NAMESPACE::WalFilter::WalProcessingOption::kCorruptedRecord: + return 0x3; + default: + return 0x7F; // undefined + } + } + + // Returns the equivalent C++ + // ROCKSDB_NAMESPACE::WalFilter::WalProcessingOption enum for the provided + // Java org.rocksdb.WalProcessingOption + static ROCKSDB_NAMESPACE::WalFilter::WalProcessingOption + toCppWalProcessingOption(jbyte jwal_processing_option) { + switch (jwal_processing_option) { + case 0x0: + return ROCKSDB_NAMESPACE::WalFilter::WalProcessingOption:: + kContinueProcessing; + case 0x1: + return ROCKSDB_NAMESPACE::WalFilter::WalProcessingOption:: + kIgnoreCurrentRecord; + case 0x2: + return ROCKSDB_NAMESPACE::WalFilter::WalProcessingOption::kStopReplay; + case 0x3: + return ROCKSDB_NAMESPACE::WalFilter::WalProcessingOption:: + kCorruptedRecord; + default: + // undefined/default + return ROCKSDB_NAMESPACE::WalFilter::WalProcessingOption:: + kCorruptedRecord; + } + } +}; + +// The portal class for org.rocksdb.ReusedSynchronisationType +class ReusedSynchronisationTypeJni { + public: + // Returns the equivalent org.rocksdb.ReusedSynchronisationType for the + // provided C++ ROCKSDB_NAMESPACE::ReusedSynchronisationType enum + static jbyte toJavaReusedSynchronisationType( + const ROCKSDB_NAMESPACE::ReusedSynchronisationType& + reused_synchronisation_type) { + switch (reused_synchronisation_type) { + case ROCKSDB_NAMESPACE::ReusedSynchronisationType::MUTEX: + return 0x0; + case ROCKSDB_NAMESPACE::ReusedSynchronisationType::ADAPTIVE_MUTEX: + return 0x1; + case ROCKSDB_NAMESPACE::ReusedSynchronisationType::THREAD_LOCAL: + return 0x2; + default: + return 0x7F; // undefined + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::ReusedSynchronisationType + // enum for the provided Java org.rocksdb.ReusedSynchronisationType + static ROCKSDB_NAMESPACE::ReusedSynchronisationType + toCppReusedSynchronisationType(jbyte reused_synchronisation_type) { + switch (reused_synchronisation_type) { + case 0x0: + return ROCKSDB_NAMESPACE::ReusedSynchronisationType::MUTEX; + case 0x1: + return ROCKSDB_NAMESPACE::ReusedSynchronisationType::ADAPTIVE_MUTEX; + case 0x2: + return ROCKSDB_NAMESPACE::ReusedSynchronisationType::THREAD_LOCAL; + default: + // undefined/default + return ROCKSDB_NAMESPACE::ReusedSynchronisationType::ADAPTIVE_MUTEX; + } + } +}; +// The portal class for org.rocksdb.SanityLevel +class SanityLevelJni { + public: + // Returns the equivalent org.rocksdb.SanityLevel for the provided + // C++ ROCKSDB_NAMESPACE::ConfigOptions::SanityLevel enum + static jbyte toJavaSanityLevel( + const ROCKSDB_NAMESPACE::ConfigOptions::SanityLevel& sanity_level) { + switch (sanity_level) { + case ROCKSDB_NAMESPACE::ConfigOptions::SanityLevel::kSanityLevelNone: + return 0x0; + case ROCKSDB_NAMESPACE::ConfigOptions::SanityLevel:: + kSanityLevelLooselyCompatible: + return 0x1; + case ROCKSDB_NAMESPACE::ConfigOptions::SanityLevel:: + kSanityLevelExactMatch: + return -0x01; + default: + return -0x01; // undefined + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::ConfigOptions::SanityLevel + // enum for the provided Java org.rocksdb.SanityLevel + static ROCKSDB_NAMESPACE::ConfigOptions::SanityLevel toCppSanityLevel( + jbyte sanity_level) { + switch (sanity_level) { + case 0x0: + return ROCKSDB_NAMESPACE::ConfigOptions::kSanityLevelNone; + case 0x1: + return ROCKSDB_NAMESPACE::ConfigOptions::kSanityLevelLooselyCompatible; + default: + // undefined/default + return ROCKSDB_NAMESPACE::ConfigOptions::kSanityLevelExactMatch; + } + } +}; + +// The portal class for org.rocksdb.PrepopulateBlobCache +class PrepopulateBlobCacheJni { + public: + // Returns the equivalent org.rocksdb.PrepopulateBlobCache for the provided + // C++ ROCKSDB_NAMESPACE::PrepopulateBlobCache enum + static jbyte toJavaPrepopulateBlobCache( + ROCKSDB_NAMESPACE::PrepopulateBlobCache prepopulate_blob_cache) { + switch (prepopulate_blob_cache) { + case ROCKSDB_NAMESPACE::PrepopulateBlobCache::kDisable: + return 0x0; + case ROCKSDB_NAMESPACE::PrepopulateBlobCache::kFlushOnly: + return 0x1; + default: + return 0x7f; // undefined + } + } + + // Returns the equivalent C++ ROCKSDB_NAMESPACE::PrepopulateBlobCache enum for + // the provided Java org.rocksdb.PrepopulateBlobCache + static ROCKSDB_NAMESPACE::PrepopulateBlobCache toCppPrepopulateBlobCache( + jbyte jprepopulate_blob_cache) { + switch (jprepopulate_blob_cache) { + case 0x0: + return ROCKSDB_NAMESPACE::PrepopulateBlobCache::kDisable; + case 0x1: + return ROCKSDB_NAMESPACE::PrepopulateBlobCache::kFlushOnly; + case 0x7F: + default: + // undefined/default + return ROCKSDB_NAMESPACE::PrepopulateBlobCache::kDisable; + } + } +}; + +// The portal class for org.rocksdb.AbstractListener.EnabledEventCallback +class EnabledEventCallbackJni { + public: + // Returns the set of equivalent C++ + // ROCKSDB_NAMESPACE::EnabledEventCallbackJni::EnabledEventCallback enums for + // the provided Java jenabled_event_callback_values + static std::set toCppEnabledEventCallbacks( + jlong jenabled_event_callback_values) { + std::set enabled_event_callbacks; + for (size_t i = 0; i < EnabledEventCallback::NUM_ENABLED_EVENT_CALLBACK; + ++i) { + if (((1ULL << i) & jenabled_event_callback_values) > 0) { + enabled_event_callbacks.emplace(static_cast(i)); + } + } + return enabled_event_callbacks; + } +}; + +// The portal class for org.rocksdb.AbstractEventListener +class AbstractEventListenerJni + : public RocksDBNativeClass< + const ROCKSDB_NAMESPACE::EventListenerJniCallback*, + AbstractEventListenerJni> { + public: + /** + * Get the Java Class org.rocksdb.AbstractEventListener + * + * @param env A pointer to the Java environment + * + * @return The Java Class or nullptr if one of the + * ClassFormatError, ClassCircularityError, NoClassDefFoundError, + * OutOfMemoryError or ExceptionInInitializerError exceptions is thrown + */ + static jclass getJClass(JNIEnv* env) { + return RocksDBNativeClass::getJClass(env, + "org/rocksdb/AbstractEventListener"); + } + + /** + * Get the Java Method: AbstractEventListener#onFlushCompletedProxy + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID + */ + static jmethodID getOnFlushCompletedProxyMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID mid = env->GetMethodID(jclazz, "onFlushCompletedProxy", + "(JLorg/rocksdb/FlushJobInfo;)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractEventListener#onFlushBeginProxy + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID + */ + static jmethodID getOnFlushBeginProxyMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID mid = env->GetMethodID(jclazz, "onFlushBeginProxy", + "(JLorg/rocksdb/FlushJobInfo;)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractEventListener#onTableFileDeleted + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID + */ + static jmethodID getOnTableFileDeletedMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID mid = env->GetMethodID( + jclazz, "onTableFileDeleted", "(Lorg/rocksdb/TableFileDeletionInfo;)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractEventListener#onCompactionBeginProxy + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID + */ + static jmethodID getOnCompactionBeginProxyMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID mid = + env->GetMethodID(jclazz, "onCompactionBeginProxy", + "(JLorg/rocksdb/CompactionJobInfo;)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractEventListener#onCompactionCompletedProxy + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID + */ + static jmethodID getOnCompactionCompletedProxyMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID mid = + env->GetMethodID(jclazz, "onCompactionCompletedProxy", + "(JLorg/rocksdb/CompactionJobInfo;)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractEventListener#onTableFileCreated + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID + */ + static jmethodID getOnTableFileCreatedMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID mid = env->GetMethodID( + jclazz, "onTableFileCreated", "(Lorg/rocksdb/TableFileCreationInfo;)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractEventListener#onTableFileCreationStarted + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID + */ + static jmethodID getOnTableFileCreationStartedMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID mid = + env->GetMethodID(jclazz, "onTableFileCreationStarted", + "(Lorg/rocksdb/TableFileCreationBriefInfo;)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractEventListener#onMemTableSealed + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID + */ + static jmethodID getOnMemTableSealedMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID mid = env->GetMethodID(jclazz, "onMemTableSealed", + "(Lorg/rocksdb/MemTableInfo;)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: + * AbstractEventListener#onColumnFamilyHandleDeletionStarted + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID + */ + static jmethodID getOnColumnFamilyHandleDeletionStartedMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID mid = + env->GetMethodID(jclazz, "onColumnFamilyHandleDeletionStarted", + "(Lorg/rocksdb/ColumnFamilyHandle;)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractEventListener#onExternalFileIngestedProxy + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID + */ + static jmethodID getOnExternalFileIngestedProxyMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID mid = + env->GetMethodID(jclazz, "onExternalFileIngestedProxy", + "(JLorg/rocksdb/ExternalFileIngestionInfo;)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractEventListener#onBackgroundError + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID + */ + static jmethodID getOnBackgroundErrorProxyMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID mid = env->GetMethodID(jclazz, "onBackgroundErrorProxy", + "(BLorg/rocksdb/Status;)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractEventListener#onStallConditionsChanged + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID + */ + static jmethodID getOnStallConditionsChangedMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID mid = env->GetMethodID(jclazz, "onStallConditionsChanged", + "(Lorg/rocksdb/WriteStallInfo;)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractEventListener#onFileReadFinish + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID + */ + static jmethodID getOnFileReadFinishMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID mid = env->GetMethodID( + jclazz, "onFileReadFinish", "(Lorg/rocksdb/FileOperationInfo;)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractEventListener#onFileWriteFinish + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID + */ + static jmethodID getOnFileWriteFinishMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID mid = env->GetMethodID( + jclazz, "onFileWriteFinish", "(Lorg/rocksdb/FileOperationInfo;)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractEventListener#onFileFlushFinish + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID + */ + static jmethodID getOnFileFlushFinishMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID mid = env->GetMethodID( + jclazz, "onFileFlushFinish", "(Lorg/rocksdb/FileOperationInfo;)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractEventListener#onFileSyncFinish + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID + */ + static jmethodID getOnFileSyncFinishMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID mid = env->GetMethodID( + jclazz, "onFileSyncFinish", "(Lorg/rocksdb/FileOperationInfo;)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractEventListener#onFileRangeSyncFinish + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID + */ + static jmethodID getOnFileRangeSyncFinishMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID mid = env->GetMethodID( + jclazz, "onFileRangeSyncFinish", "(Lorg/rocksdb/FileOperationInfo;)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractEventListener#onFileTruncateFinish + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID + */ + static jmethodID getOnFileTruncateFinishMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID mid = env->GetMethodID( + jclazz, "onFileTruncateFinish", "(Lorg/rocksdb/FileOperationInfo;)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractEventListener#onFileCloseFinish + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID + */ + static jmethodID getOnFileCloseFinishMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID mid = env->GetMethodID( + jclazz, "onFileCloseFinish", "(Lorg/rocksdb/FileOperationInfo;)V"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractEventListener#shouldBeNotifiedOnFileIO + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID + */ + static jmethodID getShouldBeNotifiedOnFileIOMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID mid = + env->GetMethodID(jclazz, "shouldBeNotifiedOnFileIO", "()Z"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractEventListener#onErrorRecoveryBeginProxy + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID + */ + static jmethodID getOnErrorRecoveryBeginProxyMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID mid = env->GetMethodID(jclazz, "onErrorRecoveryBeginProxy", + "(BLorg/rocksdb/Status;)Z"); + assert(mid != nullptr); + return mid; + } + + /** + * Get the Java Method: AbstractEventListener#onErrorRecoveryCompleted + * + * @param env A pointer to the Java environment + * + * @return The Java Method ID + */ + static jmethodID getOnErrorRecoveryCompletedMethodId(JNIEnv* env) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID mid = env->GetMethodID(jclazz, "onErrorRecoveryCompleted", + "(Lorg/rocksdb/Status;)V"); + assert(mid != nullptr); + return mid; + } +}; + +class FlushJobInfoJni : public JavaClass { + public: + /** + * Create a new Java org.rocksdb.FlushJobInfo object. + * + * @param env A pointer to the Java environment + * @param flush_job_info A Cpp flush job info object + * + * @return A reference to a Java org.rocksdb.FlushJobInfo object, or + * nullptr if an an exception occurs + */ + static jobject fromCppFlushJobInfo( + JNIEnv* env, const ROCKSDB_NAMESPACE::FlushJobInfo* flush_job_info) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + static jmethodID ctor = getConstructorMethodId(env, jclazz); + assert(ctor != nullptr); + jstring jcf_name = JniUtil::toJavaString(env, &flush_job_info->cf_name); + if (env->ExceptionCheck()) { + return nullptr; + } + jstring jfile_path = JniUtil::toJavaString(env, &flush_job_info->file_path); + if (env->ExceptionCheck()) { + env->DeleteLocalRef(jfile_path); + return nullptr; + } + jobject jtable_properties = TablePropertiesJni::fromCppTableProperties( + env, flush_job_info->table_properties); + if (jtable_properties == nullptr) { + env->DeleteLocalRef(jcf_name); + env->DeleteLocalRef(jfile_path); + return nullptr; + } + return env->NewObject( + jclazz, ctor, static_cast(flush_job_info->cf_id), jcf_name, + jfile_path, static_cast(flush_job_info->thread_id), + static_cast(flush_job_info->job_id), + static_cast(flush_job_info->triggered_writes_slowdown), + static_cast(flush_job_info->triggered_writes_stop), + static_cast(flush_job_info->smallest_seqno), + static_cast(flush_job_info->largest_seqno), jtable_properties, + static_cast(flush_job_info->flush_reason)); + } + + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/FlushJobInfo"); + } + + static jmethodID getConstructorMethodId(JNIEnv* env, jclass clazz) { + return env->GetMethodID(clazz, "", + "(JLjava/lang/String;Ljava/lang/String;JIZZJJLorg/" + "rocksdb/TableProperties;B)V"); + } +}; + +class TableFileDeletionInfoJni : public JavaClass { + public: + /** + * Create a new Java org.rocksdb.TableFileDeletionInfo object. + * + * @param env A pointer to the Java environment + * @param file_del_info A Cpp table file deletion info object + * + * @return A reference to a Java org.rocksdb.TableFileDeletionInfo object, or + * nullptr if an an exception occurs + */ + static jobject fromCppTableFileDeletionInfo( + JNIEnv* env, + const ROCKSDB_NAMESPACE::TableFileDeletionInfo* file_del_info) { + jclass jclazz = getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + static jmethodID ctor = getConstructorMethodId(env, jclazz); + assert(ctor != nullptr); + jstring jdb_name = JniUtil::toJavaString(env, &file_del_info->db_name); + if (env->ExceptionCheck()) { + return nullptr; + } + jobject jstatus = StatusJni::construct(env, file_del_info->status); + if (jstatus == nullptr) { + env->DeleteLocalRef(jdb_name); + return nullptr; + } + return env->NewObject(jclazz, ctor, jdb_name, + JniUtil::toJavaString(env, &file_del_info->file_path), + static_cast(file_del_info->job_id), jstatus); + } + + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/TableFileDeletionInfo"); + } + + static jmethodID getConstructorMethodId(JNIEnv* env, jclass clazz) { + return env->GetMethodID( + clazz, "", + "(Ljava/lang/String;Ljava/lang/String;ILorg/rocksdb/Status;)V"); + } +}; + +class CompactionJobInfoJni : public JavaClass { + public: + static jobject fromCppCompactionJobInfo( + JNIEnv* env, + const ROCKSDB_NAMESPACE::CompactionJobInfo* compaction_job_info) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID ctor = getConstructorMethodId(env, jclazz); + assert(ctor != nullptr); + return env->NewObject(jclazz, ctor, + GET_CPLUSPLUS_POINTER(compaction_job_info)); + } + + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/CompactionJobInfo"); + } + + static jmethodID getConstructorMethodId(JNIEnv* env, jclass clazz) { + return env->GetMethodID(clazz, "", "(J)V"); + } +}; + +class TableFileCreationInfoJni : public JavaClass { + public: + static jobject fromCppTableFileCreationInfo( + JNIEnv* env, const ROCKSDB_NAMESPACE::TableFileCreationInfo* info) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID ctor = getConstructorMethodId(env, jclazz); + assert(ctor != nullptr); + jstring jdb_name = JniUtil::toJavaString(env, &info->db_name); + if (env->ExceptionCheck()) { + return nullptr; + } + jstring jcf_name = JniUtil::toJavaString(env, &info->cf_name); + if (env->ExceptionCheck()) { + env->DeleteLocalRef(jdb_name); + return nullptr; + } + jstring jfile_path = JniUtil::toJavaString(env, &info->file_path); + if (env->ExceptionCheck()) { + env->DeleteLocalRef(jdb_name); + env->DeleteLocalRef(jcf_name); + return nullptr; + } + jobject jtable_properties = + TablePropertiesJni::fromCppTableProperties(env, info->table_properties); + if (jtable_properties == nullptr) { + env->DeleteLocalRef(jdb_name); + env->DeleteLocalRef(jcf_name); + return nullptr; + } + jobject jstatus = StatusJni::construct(env, info->status); + if (jstatus == nullptr) { + env->DeleteLocalRef(jdb_name); + env->DeleteLocalRef(jcf_name); + env->DeleteLocalRef(jtable_properties); + return nullptr; + } + return env->NewObject(jclazz, ctor, static_cast(info->file_size), + jtable_properties, jstatus, jdb_name, jcf_name, + jfile_path, static_cast(info->job_id), + static_cast(info->reason)); + } + + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/TableFileCreationInfo"); + } + + static jmethodID getConstructorMethodId(JNIEnv* env, jclass clazz) { + return env->GetMethodID( + clazz, "", + "(JLorg/rocksdb/TableProperties;Lorg/rocksdb/Status;Ljava/lang/" + "String;Ljava/lang/String;Ljava/lang/String;IB)V"); + } +}; + +class TableFileCreationBriefInfoJni : public JavaClass { + public: + static jobject fromCppTableFileCreationBriefInfo( + JNIEnv* env, const ROCKSDB_NAMESPACE::TableFileCreationBriefInfo* info) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID ctor = getConstructorMethodId(env, jclazz); + assert(ctor != nullptr); + jstring jdb_name = JniUtil::toJavaString(env, &info->db_name); + if (env->ExceptionCheck()) { + return nullptr; + } + jstring jcf_name = JniUtil::toJavaString(env, &info->cf_name); + if (env->ExceptionCheck()) { + env->DeleteLocalRef(jdb_name); + return nullptr; + } + jstring jfile_path = JniUtil::toJavaString(env, &info->file_path); + if (env->ExceptionCheck()) { + env->DeleteLocalRef(jdb_name); + env->DeleteLocalRef(jcf_name); + return nullptr; + } + return env->NewObject(jclazz, ctor, jdb_name, jcf_name, jfile_path, + static_cast(info->job_id), + static_cast(info->reason)); + } + + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/TableFileCreationBriefInfo"); + } + + static jmethodID getConstructorMethodId(JNIEnv* env, jclass clazz) { + return env->GetMethodID( + clazz, "", + "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;IB)V"); + } +}; + +class MemTableInfoJni : public JavaClass { + public: + static jobject fromCppMemTableInfo( + JNIEnv* env, const ROCKSDB_NAMESPACE::MemTableInfo* info) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID ctor = getConstructorMethodId(env, jclazz); + assert(ctor != nullptr); + jstring jcf_name = JniUtil::toJavaString(env, &info->cf_name); + if (env->ExceptionCheck()) { + return nullptr; + } + return env->NewObject(jclazz, ctor, jcf_name, + static_cast(info->first_seqno), + static_cast(info->earliest_seqno), + static_cast(info->num_entries), + static_cast(info->num_deletes)); + } + + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/MemTableInfo"); + } + + static jmethodID getConstructorMethodId(JNIEnv* env, jclass clazz) { + return env->GetMethodID(clazz, "", "(Ljava/lang/String;JJJJ)V"); + } +}; + +class ExternalFileIngestionInfoJni : public JavaClass { + public: + static jobject fromCppExternalFileIngestionInfo( + JNIEnv* env, const ROCKSDB_NAMESPACE::ExternalFileIngestionInfo* info) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID ctor = getConstructorMethodId(env, jclazz); + assert(ctor != nullptr); + jstring jcf_name = JniUtil::toJavaString(env, &info->cf_name); + if (env->ExceptionCheck()) { + return nullptr; + } + jstring jexternal_file_path = + JniUtil::toJavaString(env, &info->external_file_path); + if (env->ExceptionCheck()) { + env->DeleteLocalRef(jcf_name); + return nullptr; + } + jstring jinternal_file_path = + JniUtil::toJavaString(env, &info->internal_file_path); + if (env->ExceptionCheck()) { + env->DeleteLocalRef(jcf_name); + env->DeleteLocalRef(jexternal_file_path); + return nullptr; + } + jobject jtable_properties = + TablePropertiesJni::fromCppTableProperties(env, info->table_properties); + if (jtable_properties == nullptr) { + env->DeleteLocalRef(jcf_name); + env->DeleteLocalRef(jexternal_file_path); + env->DeleteLocalRef(jinternal_file_path); + return nullptr; + } + return env->NewObject( + jclazz, ctor, jcf_name, jexternal_file_path, jinternal_file_path, + static_cast(info->global_seqno), jtable_properties); + } + + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/ExternalFileIngestionInfo"); + } + + static jmethodID getConstructorMethodId(JNIEnv* env, jclass clazz) { + return env->GetMethodID(clazz, "", + "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/" + "String;JLorg/rocksdb/TableProperties;)V"); + } +}; + +class WriteStallInfoJni : public JavaClass { + public: + static jobject fromCppWriteStallInfo( + JNIEnv* env, const ROCKSDB_NAMESPACE::WriteStallInfo* info) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID ctor = getConstructorMethodId(env, jclazz); + assert(ctor != nullptr); + jstring jcf_name = JniUtil::toJavaString(env, &info->cf_name); + if (env->ExceptionCheck()) { + return nullptr; + } + return env->NewObject(jclazz, ctor, jcf_name, + static_cast(info->condition.cur), + static_cast(info->condition.prev)); + } + + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/WriteStallInfo"); + } + + static jmethodID getConstructorMethodId(JNIEnv* env, jclass clazz) { + return env->GetMethodID(clazz, "", "(Ljava/lang/String;BB)V"); + } +}; + +class FileOperationInfoJni : public JavaClass { + public: + static jobject fromCppFileOperationInfo( + JNIEnv* env, const ROCKSDB_NAMESPACE::FileOperationInfo* info) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID ctor = getConstructorMethodId(env, jclazz); + assert(ctor != nullptr); + jstring jpath = JniUtil::toJavaString(env, &info->path); + if (env->ExceptionCheck()) { + return nullptr; + } + jobject jstatus = StatusJni::construct(env, info->status); + if (jstatus == nullptr) { + env->DeleteLocalRef(jpath); + return nullptr; + } + return env->NewObject( + jclazz, ctor, jpath, static_cast(info->offset), + static_cast(info->length), + static_cast(info->start_ts.time_since_epoch().count()), + static_cast(info->duration.count()), jstatus); + } + + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, "org/rocksdb/FileOperationInfo"); + } + + static jmethodID getConstructorMethodId(JNIEnv* env, jclass clazz) { + return env->GetMethodID(clazz, "", + "(Ljava/lang/String;JJJJLorg/rocksdb/Status;)V"); + } +}; + +class CompactRangeOptionsTimestampJni : public JavaClass { + public: + static jobject fromCppTimestamp(JNIEnv* env, const uint64_t start, + const uint64_t range) { + jclass jclazz = getJClass(env); + assert(jclazz != nullptr); + static jmethodID ctor = getConstructorMethodId(env, jclazz); + assert(ctor != nullptr); + return env->NewObject(jclazz, ctor, static_cast(start), + static_cast(range)); + } + + static jclass getJClass(JNIEnv* env) { + return JavaClass::getJClass(env, + "org/rocksdb/CompactRangeOptions$Timestamp"); + } + + static jmethodID getConstructorMethodId(JNIEnv* env, jclass clazz) { + return env->GetMethodID(clazz, "", "(JJ)V"); + } +}; + +} // namespace ROCKSDB_NAMESPACE +#endif // JAVA_ROCKSJNI_PORTAL_H_ diff --git a/librocksdb-sys/rocksdb/java/rocksjni/ratelimiterjni.cc b/librocksdb-sys/rocksdb/java/rocksjni/ratelimiterjni.cc new file mode 100644 index 0000000..7a17f36 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/ratelimiterjni.cc @@ -0,0 +1,128 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for RateLimiter. + +#include "include/org_rocksdb_RateLimiter.h" +#include "rocksdb/rate_limiter.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_RateLimiter + * Method: newRateLimiterHandle + * Signature: (JJIBZ)J + */ +jlong Java_org_rocksdb_RateLimiter_newRateLimiterHandle( + JNIEnv* /*env*/, jclass /*jclazz*/, jlong jrate_bytes_per_second, + jlong jrefill_period_micros, jint jfairness, jbyte jrate_limiter_mode, + jboolean jauto_tune) { + auto rate_limiter_mode = + ROCKSDB_NAMESPACE::RateLimiterModeJni::toCppRateLimiterMode( + jrate_limiter_mode); + auto* sptr_rate_limiter = new std::shared_ptr( + ROCKSDB_NAMESPACE::NewGenericRateLimiter( + static_cast(jrate_bytes_per_second), + static_cast(jrefill_period_micros), + static_cast(jfairness), rate_limiter_mode, jauto_tune)); + + return GET_CPLUSPLUS_POINTER(sptr_rate_limiter); +} + +/* + * Class: org_rocksdb_RateLimiter + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_RateLimiter_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* handle = + reinterpret_cast*>( + jhandle); + delete handle; // delete std::shared_ptr +} + +/* + * Class: org_rocksdb_RateLimiter + * Method: setBytesPerSecond + * Signature: (JJ)V + */ +void Java_org_rocksdb_RateLimiter_setBytesPerSecond(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle, + jlong jbytes_per_second) { + reinterpret_cast*>(handle) + ->get() + ->SetBytesPerSecond(jbytes_per_second); +} + +/* + * Class: org_rocksdb_RateLimiter + * Method: getBytesPerSecond + * Signature: (J)J + */ +jlong Java_org_rocksdb_RateLimiter_getBytesPerSecond(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + return reinterpret_cast*>( + handle) + ->get() + ->GetBytesPerSecond(); +} + +/* + * Class: org_rocksdb_RateLimiter + * Method: request + * Signature: (JJ)V + */ +void Java_org_rocksdb_RateLimiter_request(JNIEnv* /*env*/, jobject /*jobj*/, + jlong handle, jlong jbytes) { + reinterpret_cast*>(handle) + ->get() + ->Request(jbytes, ROCKSDB_NAMESPACE::Env::IO_TOTAL); +} + +/* + * Class: org_rocksdb_RateLimiter + * Method: getSingleBurstBytes + * Signature: (J)J + */ +jlong Java_org_rocksdb_RateLimiter_getSingleBurstBytes(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + return reinterpret_cast*>( + handle) + ->get() + ->GetSingleBurstBytes(); +} + +/* + * Class: org_rocksdb_RateLimiter + * Method: getTotalBytesThrough + * Signature: (J)J + */ +jlong Java_org_rocksdb_RateLimiter_getTotalBytesThrough(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + return reinterpret_cast*>( + handle) + ->get() + ->GetTotalBytesThrough(); +} + +/* + * Class: org_rocksdb_RateLimiter + * Method: getTotalRequests + * Signature: (J)J + */ +jlong Java_org_rocksdb_RateLimiter_getTotalRequests(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + return reinterpret_cast*>( + handle) + ->get() + ->GetTotalRequests(); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/remove_emptyvalue_compactionfilterjni.cc b/librocksdb-sys/rocksdb/java/rocksjni/remove_emptyvalue_compactionfilterjni.cc new file mode 100644 index 0000000..c0b09e1 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/remove_emptyvalue_compactionfilterjni.cc @@ -0,0 +1,24 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include + +#include "include/org_rocksdb_RemoveEmptyValueCompactionFilter.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "utilities/compaction_filters/remove_emptyvalue_compactionfilter.h" + +/* + * Class: org_rocksdb_RemoveEmptyValueCompactionFilter + * Method: createNewRemoveEmptyValueCompactionFilter0 + * Signature: ()J + */ +jlong Java_org_rocksdb_RemoveEmptyValueCompactionFilter_createNewRemoveEmptyValueCompactionFilter0( + JNIEnv* /*env*/, jclass /*jcls*/) { + auto* compaction_filter = + new ROCKSDB_NAMESPACE::RemoveEmptyValueCompactionFilter(); + + // set the native handle to our native compaction filter + return GET_CPLUSPLUS_POINTER(compaction_filter); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/restorejni.cc b/librocksdb-sys/rocksdb/java/rocksjni/restorejni.cc new file mode 100644 index 0000000..aadc861 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/restorejni.cc @@ -0,0 +1,42 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling C++ ROCKSDB_NAMESPACE::RestoreOptions methods +// from Java side. + +#include +#include +#include + +#include + +#include "include/org_rocksdb_RestoreOptions.h" +#include "rocksdb/utilities/backup_engine.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" +/* + * Class: org_rocksdb_RestoreOptions + * Method: newRestoreOptions + * Signature: (Z)J + */ +jlong Java_org_rocksdb_RestoreOptions_newRestoreOptions( + JNIEnv* /*env*/, jclass /*jcls*/, jboolean keep_log_files) { + auto* ropt = new ROCKSDB_NAMESPACE::RestoreOptions(keep_log_files); + return GET_CPLUSPLUS_POINTER(ropt); +} + +/* + * Class: org_rocksdb_RestoreOptions + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_RestoreOptions_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* ropt = reinterpret_cast(jhandle); + assert(ropt); + delete ropt; +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/rocks_callback_object.cc b/librocksdb-sys/rocksdb/java/rocksjni/rocks_callback_object.cc new file mode 100644 index 0000000..35513e1 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/rocks_callback_object.cc @@ -0,0 +1,30 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// JNI Callbacks from C++ to sub-classes or org.rocksdb.RocksCallbackObject + +#include + +#include "include/org_rocksdb_RocksCallbackObject.h" +#include "jnicallback.h" + +/* + * Class: org_rocksdb_RocksCallbackObject + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_RocksCallbackObject_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + // TODO(AR) is deleting from the super class JniCallback OK, or must we delete + // the subclass? Example hierarchies: + // 1) Comparator -> BaseComparatorJniCallback + JniCallback -> + // DirectComparatorJniCallback 2) Comparator -> BaseComparatorJniCallback + + // JniCallback -> ComparatorJniCallback + // I think this is okay, as Comparator and JniCallback both have virtual + // destructors... + delete reinterpret_cast(handle); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/rocksdb_exception_test.cc b/librocksdb-sys/rocksdb/java/rocksjni/rocksdb_exception_test.cc new file mode 100644 index 0000000..67e62f7 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/rocksdb_exception_test.cc @@ -0,0 +1,81 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include + +#include "include/org_rocksdb_RocksDBExceptionTest.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_RocksDBExceptionTest + * Method: raiseException + * Signature: ()V + */ +void Java_org_rocksdb_RocksDBExceptionTest_raiseException(JNIEnv* env, + jobject /*jobj*/) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, + std::string("test message")); +} + +/* + * Class: org_rocksdb_RocksDBExceptionTest + * Method: raiseExceptionWithStatusCode + * Signature: ()V + */ +void Java_org_rocksdb_RocksDBExceptionTest_raiseExceptionWithStatusCode( + JNIEnv* env, jobject /*jobj*/) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, "test message", ROCKSDB_NAMESPACE::Status::NotSupported()); +} + +/* + * Class: org_rocksdb_RocksDBExceptionTest + * Method: raiseExceptionNoMsgWithStatusCode + * Signature: ()V + */ +void Java_org_rocksdb_RocksDBExceptionTest_raiseExceptionNoMsgWithStatusCode( + JNIEnv* env, jobject /*jobj*/) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::NotSupported()); +} + +/* + * Class: org_rocksdb_RocksDBExceptionTest + * Method: raiseExceptionWithStatusCodeSubCode + * Signature: ()V + */ +void Java_org_rocksdb_RocksDBExceptionTest_raiseExceptionWithStatusCodeSubCode( + JNIEnv* env, jobject /*jobj*/) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, "test message", + ROCKSDB_NAMESPACE::Status::TimedOut( + ROCKSDB_NAMESPACE::Status::SubCode::kLockTimeout)); +} + +/* + * Class: org_rocksdb_RocksDBExceptionTest + * Method: raiseExceptionNoMsgWithStatusCodeSubCode + * Signature: ()V + */ +void Java_org_rocksdb_RocksDBExceptionTest_raiseExceptionNoMsgWithStatusCodeSubCode( + JNIEnv* env, jobject /*jobj*/) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::TimedOut( + ROCKSDB_NAMESPACE::Status::SubCode::kLockTimeout)); +} + +/* + * Class: org_rocksdb_RocksDBExceptionTest + * Method: raiseExceptionWithStatusCodeState + * Signature: ()V + */ +void Java_org_rocksdb_RocksDBExceptionTest_raiseExceptionWithStatusCodeState( + JNIEnv* env, jobject /*jobj*/) { + ROCKSDB_NAMESPACE::Slice state("test state"); + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, "test message", ROCKSDB_NAMESPACE::Status::NotSupported(state)); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/rocksjni.cc b/librocksdb-sys/rocksdb/java/rocksjni/rocksjni.cc new file mode 100644 index 0000000..ced72e8 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/rocksjni.cc @@ -0,0 +1,3957 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling c++ ROCKSDB_NAMESPACE::DB methods from Java side. + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "include/org_rocksdb_RocksDB.h" +#include "rocksdb/cache.h" +#include "rocksdb/convenience.h" +#include "rocksdb/db.h" +#include "rocksdb/options.h" +#include "rocksdb/types.h" +#include "rocksdb/version.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +#ifdef min +#undef min +#endif + +jlong rocksdb_open_helper(JNIEnv* env, jlong jopt_handle, jstring jdb_path, + std::function + open_fn) { + const char* db_path = env->GetStringUTFChars(jdb_path, nullptr); + if (db_path == nullptr) { + // exception thrown: OutOfMemoryError + return 0; + } + + auto* opt = reinterpret_cast(jopt_handle); + ROCKSDB_NAMESPACE::DB* db = nullptr; + ROCKSDB_NAMESPACE::Status s = open_fn(*opt, db_path, &db); + + env->ReleaseStringUTFChars(jdb_path, db_path); + + if (s.ok()) { + return GET_CPLUSPLUS_POINTER(db); + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return 0; + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: open + * Signature: (JLjava/lang/String;)J + */ +jlong Java_org_rocksdb_RocksDB_open__JLjava_lang_String_2(JNIEnv* env, jclass, + jlong jopt_handle, + jstring jdb_path) { + return rocksdb_open_helper(env, jopt_handle, jdb_path, + (ROCKSDB_NAMESPACE::Status(*)( + const ROCKSDB_NAMESPACE::Options&, + const std::string&, ROCKSDB_NAMESPACE::DB**)) & + ROCKSDB_NAMESPACE::DB::Open); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: openROnly + * Signature: (JLjava/lang/String;Z)J + */ +jlong Java_org_rocksdb_RocksDB_openROnly__JLjava_lang_String_2Z( + JNIEnv* env, jclass, jlong jopt_handle, jstring jdb_path, + jboolean jerror_if_wal_file_exists) { + const bool error_if_wal_file_exists = jerror_if_wal_file_exists == JNI_TRUE; + return rocksdb_open_helper( + env, jopt_handle, jdb_path, + [error_if_wal_file_exists](const ROCKSDB_NAMESPACE::Options& options, + const std::string& db_path, + ROCKSDB_NAMESPACE::DB** db) { + return ROCKSDB_NAMESPACE::DB::OpenForReadOnly(options, db_path, db, + error_if_wal_file_exists); + }); +} + +jlongArray rocksdb_open_helper( + JNIEnv* env, jlong jopt_handle, jstring jdb_path, + jobjectArray jcolumn_names, jlongArray jcolumn_options, + std::function&, + std::vector*, + ROCKSDB_NAMESPACE::DB**)> + open_fn) { + const char* db_path = env->GetStringUTFChars(jdb_path, nullptr); + if (db_path == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + const jsize len_cols = env->GetArrayLength(jcolumn_names); + jlong* jco = env->GetLongArrayElements(jcolumn_options, nullptr); + if (jco == nullptr) { + // exception thrown: OutOfMemoryError + env->ReleaseStringUTFChars(jdb_path, db_path); + return nullptr; + } + + std::vector column_families; + jboolean has_exception = JNI_FALSE; + ROCKSDB_NAMESPACE::JniUtil::byteStrings( + env, jcolumn_names, + [](const char* str_data, const size_t str_len) { + return std::string(str_data, str_len); + }, + [&jco, &column_families](size_t idx, std::string cf_name) { + ROCKSDB_NAMESPACE::ColumnFamilyOptions* cf_options = + reinterpret_cast(jco[idx]); + column_families.push_back( + ROCKSDB_NAMESPACE::ColumnFamilyDescriptor(cf_name, *cf_options)); + }, + &has_exception); + + env->ReleaseLongArrayElements(jcolumn_options, jco, JNI_ABORT); + + if (has_exception == JNI_TRUE) { + // exception occurred + env->ReleaseStringUTFChars(jdb_path, db_path); + return nullptr; + } + + auto* opt = reinterpret_cast(jopt_handle); + std::vector cf_handles; + ROCKSDB_NAMESPACE::DB* db = nullptr; + ROCKSDB_NAMESPACE::Status s = + open_fn(*opt, db_path, column_families, &cf_handles, &db); + + // we have now finished with db_path + env->ReleaseStringUTFChars(jdb_path, db_path); + + // check if open operation was successful + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return nullptr; + } + + const jsize resultsLen = 1 + len_cols; // db handle + column family handles + std::unique_ptr results = + std::unique_ptr(new jlong[resultsLen]); + results[0] = GET_CPLUSPLUS_POINTER(db); + for (int i = 1; i <= len_cols; i++) { + results[i] = GET_CPLUSPLUS_POINTER(cf_handles[i - 1]); + } + + jlongArray jresults = env->NewLongArray(resultsLen); + if (jresults == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + env->SetLongArrayRegion(jresults, 0, resultsLen, results.get()); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jresults); + return nullptr; + } + + return jresults; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: openROnly + * Signature: (JLjava/lang/String;[[B[JZ)[J + */ +jlongArray Java_org_rocksdb_RocksDB_openROnly__JLjava_lang_String_2_3_3B_3JZ( + JNIEnv* env, jclass, jlong jopt_handle, jstring jdb_path, + jobjectArray jcolumn_names, jlongArray jcolumn_options, + jboolean jerror_if_wal_file_exists) { + const bool error_if_wal_file_exists = jerror_if_wal_file_exists == JNI_TRUE; + return rocksdb_open_helper( + env, jopt_handle, jdb_path, jcolumn_names, jcolumn_options, + [error_if_wal_file_exists]( + const ROCKSDB_NAMESPACE::DBOptions& options, + const std::string& db_path, + const std::vector& + column_families, + std::vector* handles, + ROCKSDB_NAMESPACE::DB** db) { + return ROCKSDB_NAMESPACE::DB::OpenForReadOnly( + options, db_path, column_families, handles, db, + error_if_wal_file_exists); + }); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: open + * Signature: (JLjava/lang/String;[[B[J)[J + */ +jlongArray Java_org_rocksdb_RocksDB_open__JLjava_lang_String_2_3_3B_3J( + JNIEnv* env, jclass, jlong jopt_handle, jstring jdb_path, + jobjectArray jcolumn_names, jlongArray jcolumn_options) { + return rocksdb_open_helper( + env, jopt_handle, jdb_path, jcolumn_names, jcolumn_options, + (ROCKSDB_NAMESPACE::Status(*)( + const ROCKSDB_NAMESPACE::DBOptions&, const std::string&, + const std::vector&, + std::vector*, + ROCKSDB_NAMESPACE::DB**)) & + ROCKSDB_NAMESPACE::DB::Open); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: openAsSecondary + * Signature: (JLjava/lang/String;Ljava/lang/String;)J + */ +jlong Java_org_rocksdb_RocksDB_openAsSecondary__JLjava_lang_String_2Ljava_lang_String_2( + JNIEnv* env, jclass, jlong jopt_handle, jstring jdb_path, + jstring jsecondary_db_path) { + const char* secondary_db_path = + env->GetStringUTFChars(jsecondary_db_path, nullptr); + if (secondary_db_path == nullptr) { + // exception thrown: OutOfMemoryError + return 0; + } + + jlong db_handle = rocksdb_open_helper( + env, jopt_handle, jdb_path, + [secondary_db_path](const ROCKSDB_NAMESPACE::Options& options, + const std::string& db_path, + ROCKSDB_NAMESPACE::DB** db) { + return ROCKSDB_NAMESPACE::DB::OpenAsSecondary(options, db_path, + secondary_db_path, db); + }); + + // we have now finished with secondary_db_path + env->ReleaseStringUTFChars(jsecondary_db_path, secondary_db_path); + + return db_handle; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: openAsSecondary + * Signature: (JLjava/lang/String;Ljava/lang/String;[[B[J)[J + */ +jlongArray +Java_org_rocksdb_RocksDB_openAsSecondary__JLjava_lang_String_2Ljava_lang_String_2_3_3B_3J( + JNIEnv* env, jclass, jlong jopt_handle, jstring jdb_path, + jstring jsecondary_db_path, jobjectArray jcolumn_names, + jlongArray jcolumn_options) { + const char* secondary_db_path = + env->GetStringUTFChars(jsecondary_db_path, nullptr); + if (secondary_db_path == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + jlongArray jhandles = rocksdb_open_helper( + env, jopt_handle, jdb_path, jcolumn_names, jcolumn_options, + [secondary_db_path]( + const ROCKSDB_NAMESPACE::DBOptions& options, + const std::string& db_path, + const std::vector& + column_families, + std::vector* handles, + ROCKSDB_NAMESPACE::DB** db) { + return ROCKSDB_NAMESPACE::DB::OpenAsSecondary( + options, db_path, secondary_db_path, column_families, handles, db); + }); + + // we have now finished with secondary_db_path + env->ReleaseStringUTFChars(jsecondary_db_path, secondary_db_path); + + return jhandles; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_RocksDB_disposeInternal(JNIEnv*, jobject, jlong jhandle) { + auto* db = reinterpret_cast(jhandle); + assert(db != nullptr); + delete db; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: closeDatabase + * Signature: (J)V + */ +void Java_org_rocksdb_RocksDB_closeDatabase(JNIEnv* env, jclass, + jlong jhandle) { + auto* db = reinterpret_cast(jhandle); + assert(db != nullptr); + ROCKSDB_NAMESPACE::Status s = db->Close(); + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: listColumnFamilies + * Signature: (JLjava/lang/String;)[[B + */ +jobjectArray Java_org_rocksdb_RocksDB_listColumnFamilies(JNIEnv* env, jclass, + jlong jopt_handle, + jstring jdb_path) { + std::vector column_family_names; + const char* db_path = env->GetStringUTFChars(jdb_path, nullptr); + if (db_path == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + auto* opt = reinterpret_cast(jopt_handle); + ROCKSDB_NAMESPACE::Status s = ROCKSDB_NAMESPACE::DB::ListColumnFamilies( + *opt, db_path, &column_family_names); + + env->ReleaseStringUTFChars(jdb_path, db_path); + + jobjectArray jcolumn_family_names = + ROCKSDB_NAMESPACE::JniUtil::stringsBytes(env, column_family_names); + + return jcolumn_family_names; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: createColumnFamily + * Signature: (J[BIJ)J + */ +jlong Java_org_rocksdb_RocksDB_createColumnFamily(JNIEnv* env, jobject, + jlong jhandle, + jbyteArray jcf_name, + jint jcf_name_len, + jlong jcf_options_handle) { + auto* db = reinterpret_cast(jhandle); + jboolean has_exception = JNI_FALSE; + const std::string cf_name = + ROCKSDB_NAMESPACE::JniUtil::byteString( + env, jcf_name, jcf_name_len, + [](const char* str, const size_t len) { + return std::string(str, len); + }, + &has_exception); + if (has_exception == JNI_TRUE) { + // exception occurred + return 0; + } + auto* cf_options = reinterpret_cast( + jcf_options_handle); + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle; + ROCKSDB_NAMESPACE::Status s = + db->CreateColumnFamily(*cf_options, cf_name, &cf_handle); + if (!s.ok()) { + // error occurred + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return 0; + } + return GET_CPLUSPLUS_POINTER(cf_handle); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: createColumnFamilies + * Signature: (JJ[[B)[J + */ +jlongArray Java_org_rocksdb_RocksDB_createColumnFamilies__JJ_3_3B( + JNIEnv* env, jobject, jlong jhandle, jlong jcf_options_handle, + jobjectArray jcf_names) { + auto* db = reinterpret_cast(jhandle); + auto* cf_options = reinterpret_cast( + jcf_options_handle); + jboolean has_exception = JNI_FALSE; + std::vector cf_names; + ROCKSDB_NAMESPACE::JniUtil::byteStrings( + env, jcf_names, + [](const char* str, const size_t len) { return std::string(str, len); }, + [&cf_names](const size_t, std::string str) { cf_names.push_back(str); }, + &has_exception); + if (has_exception == JNI_TRUE) { + // exception occurred + return nullptr; + } + + std::vector cf_handles; + ROCKSDB_NAMESPACE::Status s = + db->CreateColumnFamilies(*cf_options, cf_names, &cf_handles); + if (!s.ok()) { + // error occurred + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return nullptr; + } + + jlongArray jcf_handles = ROCKSDB_NAMESPACE::JniUtil::toJPointers< + ROCKSDB_NAMESPACE::ColumnFamilyHandle>(env, cf_handles, &has_exception); + if (has_exception == JNI_TRUE) { + // exception occurred + return nullptr; + } + return jcf_handles; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: createColumnFamilies + * Signature: (J[J[[B)[J + */ +jlongArray Java_org_rocksdb_RocksDB_createColumnFamilies__J_3J_3_3B( + JNIEnv* env, jobject, jlong jhandle, jlongArray jcf_options_handles, + jobjectArray jcf_names) { + auto* db = reinterpret_cast(jhandle); + const jsize jlen = env->GetArrayLength(jcf_options_handles); + std::vector cf_descriptors; + cf_descriptors.reserve(jlen); + + jlong* jcf_options_handles_elems = + env->GetLongArrayElements(jcf_options_handles, nullptr); + if (jcf_options_handles_elems == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + // extract the column family descriptors + jboolean has_exception = JNI_FALSE; + for (jsize i = 0; i < jlen; i++) { + auto* cf_options = + reinterpret_cast( + jcf_options_handles_elems[i]); + jbyteArray jcf_name = + static_cast(env->GetObjectArrayElement(jcf_names, i)); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->ReleaseLongArrayElements(jcf_options_handles, + jcf_options_handles_elems, JNI_ABORT); + return nullptr; + } + const std::string cf_name = + ROCKSDB_NAMESPACE::JniUtil::byteString( + env, jcf_name, + [](const char* str, const size_t len) { + return std::string(str, len); + }, + &has_exception); + if (has_exception == JNI_TRUE) { + // exception occurred + env->DeleteLocalRef(jcf_name); + env->ReleaseLongArrayElements(jcf_options_handles, + jcf_options_handles_elems, JNI_ABORT); + return nullptr; + } + + cf_descriptors.push_back( + ROCKSDB_NAMESPACE::ColumnFamilyDescriptor(cf_name, *cf_options)); + + env->DeleteLocalRef(jcf_name); + } + + std::vector cf_handles; + ROCKSDB_NAMESPACE::Status s = + db->CreateColumnFamilies(cf_descriptors, &cf_handles); + + env->ReleaseLongArrayElements(jcf_options_handles, jcf_options_handles_elems, + JNI_ABORT); + + if (!s.ok()) { + // error occurred + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return nullptr; + } + + jlongArray jcf_handles = ROCKSDB_NAMESPACE::JniUtil::toJPointers< + ROCKSDB_NAMESPACE::ColumnFamilyHandle>(env, cf_handles, &has_exception); + if (has_exception == JNI_TRUE) { + // exception occurred + return nullptr; + } + return jcf_handles; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: dropColumnFamily + * Signature: (JJ)V; + */ +void Java_org_rocksdb_RocksDB_dropColumnFamily(JNIEnv* env, jobject, + jlong jdb_handle, + jlong jcf_handle) { + auto* db_handle = reinterpret_cast(jdb_handle); + auto* cf_handle = + reinterpret_cast(jcf_handle); + ROCKSDB_NAMESPACE::Status s = db_handle->DropColumnFamily(cf_handle); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: dropColumnFamilies + * Signature: (J[J)V + */ +void Java_org_rocksdb_RocksDB_dropColumnFamilies( + JNIEnv* env, jobject, jlong jdb_handle, jlongArray jcolumn_family_handles) { + auto* db_handle = reinterpret_cast(jdb_handle); + + std::vector cf_handles; + if (jcolumn_family_handles != nullptr) { + const jsize len_cols = env->GetArrayLength(jcolumn_family_handles); + + jlong* jcfh = env->GetLongArrayElements(jcolumn_family_handles, nullptr); + if (jcfh == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + for (jsize i = 0; i < len_cols; i++) { + auto* cf_handle = + reinterpret_cast(jcfh[i]); + cf_handles.push_back(cf_handle); + } + env->ReleaseLongArrayElements(jcolumn_family_handles, jcfh, JNI_ABORT); + } + + ROCKSDB_NAMESPACE::Status s = db_handle->DropColumnFamilies(cf_handles); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +////////////////////////////////////////////////////////////////////////////// +// ROCKSDB_NAMESPACE::DB::Put + +/** + * @return true if the put succeeded, false if a Java Exception was thrown + */ +bool rocksdb_put_helper(JNIEnv* env, ROCKSDB_NAMESPACE::DB* db, + const ROCKSDB_NAMESPACE::WriteOptions& write_options, + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle, + jbyteArray jkey, jint jkey_off, jint jkey_len, + jbyteArray jval, jint jval_off, jint jval_len) { + jbyte* key = new jbyte[jkey_len]; + env->GetByteArrayRegion(jkey, jkey_off, jkey_len, key); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key; + return false; + } + + jbyte* value = new jbyte[jval_len]; + env->GetByteArrayRegion(jval, jval_off, jval_len, value); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] value; + delete[] key; + return false; + } + + ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), jkey_len); + ROCKSDB_NAMESPACE::Slice value_slice(reinterpret_cast(value), + jval_len); + + ROCKSDB_NAMESPACE::Status s; + if (cf_handle != nullptr) { + s = db->Put(write_options, cf_handle, key_slice, value_slice); + } else { + // backwards compatibility + s = db->Put(write_options, key_slice, value_slice); + } + + // cleanup + delete[] value; + delete[] key; + + if (s.ok()) { + return true; + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return false; + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: put + * Signature: (J[BII[BII)V + */ +void Java_org_rocksdb_RocksDB_put__J_3BII_3BII(JNIEnv* env, jobject, + jlong jdb_handle, + jbyteArray jkey, jint jkey_off, + jint jkey_len, jbyteArray jval, + jint jval_off, jint jval_len) { + auto* db = reinterpret_cast(jdb_handle); + static const ROCKSDB_NAMESPACE::WriteOptions default_write_options = + ROCKSDB_NAMESPACE::WriteOptions(); + rocksdb_put_helper(env, db, default_write_options, nullptr, jkey, jkey_off, + jkey_len, jval, jval_off, jval_len); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: put + * Signature: (J[BII[BIIJ)V + */ +void Java_org_rocksdb_RocksDB_put__J_3BII_3BIIJ(JNIEnv* env, jobject, + jlong jdb_handle, + jbyteArray jkey, jint jkey_off, + jint jkey_len, jbyteArray jval, + jint jval_off, jint jval_len, + jlong jcf_handle) { + auto* db = reinterpret_cast(jdb_handle); + static const ROCKSDB_NAMESPACE::WriteOptions default_write_options = + ROCKSDB_NAMESPACE::WriteOptions(); + auto* cf_handle = + reinterpret_cast(jcf_handle); + if (cf_handle != nullptr) { + rocksdb_put_helper(env, db, default_write_options, cf_handle, jkey, + jkey_off, jkey_len, jval, jval_off, jval_len); + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::InvalidArgument( + "Invalid ColumnFamilyHandle.")); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: put + * Signature: (JJ[BII[BII)V + */ +void Java_org_rocksdb_RocksDB_put__JJ_3BII_3BII(JNIEnv* env, jobject, + jlong jdb_handle, + jlong jwrite_options_handle, + jbyteArray jkey, jint jkey_off, + jint jkey_len, jbyteArray jval, + jint jval_off, jint jval_len) { + auto* db = reinterpret_cast(jdb_handle); + auto* write_options = + reinterpret_cast(jwrite_options_handle); + rocksdb_put_helper(env, db, *write_options, nullptr, jkey, jkey_off, jkey_len, + jval, jval_off, jval_len); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: put + * Signature: (JJ[BII[BIIJ)V + */ +void Java_org_rocksdb_RocksDB_put__JJ_3BII_3BIIJ( + JNIEnv* env, jobject, jlong jdb_handle, jlong jwrite_options_handle, + jbyteArray jkey, jint jkey_off, jint jkey_len, jbyteArray jval, + jint jval_off, jint jval_len, jlong jcf_handle) { + auto* db = reinterpret_cast(jdb_handle); + auto* write_options = + reinterpret_cast(jwrite_options_handle); + auto* cf_handle = + reinterpret_cast(jcf_handle); + if (cf_handle != nullptr) { + rocksdb_put_helper(env, db, *write_options, cf_handle, jkey, jkey_off, + jkey_len, jval, jval_off, jval_len); + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::InvalidArgument( + "Invalid ColumnFamilyHandle.")); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: putDirect + * Signature: (JJLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJ)V + */ +void Java_org_rocksdb_RocksDB_putDirect( + JNIEnv* env, jobject /*jdb*/, jlong jdb_handle, jlong jwrite_options_handle, + jobject jkey, jint jkey_off, jint jkey_len, jobject jval, jint jval_off, + jint jval_len, jlong jcf_handle) { + auto* db = reinterpret_cast(jdb_handle); + auto* write_options = + reinterpret_cast(jwrite_options_handle); + auto* cf_handle = + reinterpret_cast(jcf_handle); + auto put = [&env, &db, &cf_handle, &write_options]( + ROCKSDB_NAMESPACE::Slice& key, + ROCKSDB_NAMESPACE::Slice& value) { + ROCKSDB_NAMESPACE::Status s; + if (cf_handle == nullptr) { + s = db->Put(*write_options, key, value); + } else { + s = db->Put(*write_options, cf_handle, key, value); + } + if (s.ok()) { + return; + } + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + }; + ROCKSDB_NAMESPACE::JniUtil::kv_op_direct(put, env, jkey, jkey_off, jkey_len, + jval, jval_off, jval_len); +} + +////////////////////////////////////////////////////////////////////////////// +// ROCKSDB_NAMESPACE::DB::Delete() + +/** + * @return true if the delete succeeded, false if a Java Exception was thrown + */ +bool rocksdb_delete_helper(JNIEnv* env, ROCKSDB_NAMESPACE::DB* db, + const ROCKSDB_NAMESPACE::WriteOptions& write_options, + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle, + jbyteArray jkey, jint jkey_off, jint jkey_len) { + jbyte* key = new jbyte[jkey_len]; + env->GetByteArrayRegion(jkey, jkey_off, jkey_len, key); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key; + return false; + } + ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), jkey_len); + + ROCKSDB_NAMESPACE::Status s; + if (cf_handle != nullptr) { + s = db->Delete(write_options, cf_handle, key_slice); + } else { + // backwards compatibility + s = db->Delete(write_options, key_slice); + } + + // cleanup + delete[] key; + + if (s.ok()) { + return true; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return false; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: delete + * Signature: (J[BII)V + */ +void Java_org_rocksdb_RocksDB_delete__J_3BII(JNIEnv* env, jobject, + jlong jdb_handle, jbyteArray jkey, + jint jkey_off, jint jkey_len) { + auto* db = reinterpret_cast(jdb_handle); + static const ROCKSDB_NAMESPACE::WriteOptions default_write_options = + ROCKSDB_NAMESPACE::WriteOptions(); + rocksdb_delete_helper(env, db, default_write_options, nullptr, jkey, jkey_off, + jkey_len); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: delete + * Signature: (J[BIIJ)V + */ +void Java_org_rocksdb_RocksDB_delete__J_3BIIJ(JNIEnv* env, jobject, + jlong jdb_handle, jbyteArray jkey, + jint jkey_off, jint jkey_len, + jlong jcf_handle) { + auto* db = reinterpret_cast(jdb_handle); + static const ROCKSDB_NAMESPACE::WriteOptions default_write_options = + ROCKSDB_NAMESPACE::WriteOptions(); + auto* cf_handle = + reinterpret_cast(jcf_handle); + if (cf_handle != nullptr) { + rocksdb_delete_helper(env, db, default_write_options, cf_handle, jkey, + jkey_off, jkey_len); + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::InvalidArgument( + "Invalid ColumnFamilyHandle.")); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: delete + * Signature: (JJ[BII)V + */ +void Java_org_rocksdb_RocksDB_delete__JJ_3BII(JNIEnv* env, jobject, + jlong jdb_handle, + jlong jwrite_options, + jbyteArray jkey, jint jkey_off, + jint jkey_len) { + auto* db = reinterpret_cast(jdb_handle); + auto* write_options = + reinterpret_cast(jwrite_options); + rocksdb_delete_helper(env, db, *write_options, nullptr, jkey, jkey_off, + jkey_len); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: delete + * Signature: (JJ[BIIJ)V + */ +void Java_org_rocksdb_RocksDB_delete__JJ_3BIIJ( + JNIEnv* env, jobject, jlong jdb_handle, jlong jwrite_options, + jbyteArray jkey, jint jkey_off, jint jkey_len, jlong jcf_handle) { + auto* db = reinterpret_cast(jdb_handle); + auto* write_options = + reinterpret_cast(jwrite_options); + auto* cf_handle = + reinterpret_cast(jcf_handle); + if (cf_handle != nullptr) { + rocksdb_delete_helper(env, db, *write_options, cf_handle, jkey, jkey_off, + jkey_len); + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::InvalidArgument( + "Invalid ColumnFamilyHandle.")); + } +} + +////////////////////////////////////////////////////////////////////////////// +// ROCKSDB_NAMESPACE::DB::SingleDelete() +/** + * @return true if the single delete succeeded, false if a Java Exception + * was thrown + */ +bool rocksdb_single_delete_helper( + JNIEnv* env, ROCKSDB_NAMESPACE::DB* db, + const ROCKSDB_NAMESPACE::WriteOptions& write_options, + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle, jbyteArray jkey, + jint jkey_len) { + jbyte* key = new jbyte[jkey_len]; + env->GetByteArrayRegion(jkey, 0, jkey_len, key); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key; + return false; + } + ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), jkey_len); + + ROCKSDB_NAMESPACE::Status s; + if (cf_handle != nullptr) { + s = db->SingleDelete(write_options, cf_handle, key_slice); + } else { + // backwards compatibility + s = db->SingleDelete(write_options, key_slice); + } + + delete[] key; + + if (s.ok()) { + return true; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return false; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: singleDelete + * Signature: (J[BI)V + */ +void Java_org_rocksdb_RocksDB_singleDelete__J_3BI(JNIEnv* env, jobject, + jlong jdb_handle, + jbyteArray jkey, + jint jkey_len) { + auto* db = reinterpret_cast(jdb_handle); + static const ROCKSDB_NAMESPACE::WriteOptions default_write_options = + ROCKSDB_NAMESPACE::WriteOptions(); + rocksdb_single_delete_helper(env, db, default_write_options, nullptr, jkey, + jkey_len); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: singleDelete + * Signature: (J[BIJ)V + */ +void Java_org_rocksdb_RocksDB_singleDelete__J_3BIJ(JNIEnv* env, jobject, + jlong jdb_handle, + jbyteArray jkey, + jint jkey_len, + jlong jcf_handle) { + auto* db = reinterpret_cast(jdb_handle); + static const ROCKSDB_NAMESPACE::WriteOptions default_write_options = + ROCKSDB_NAMESPACE::WriteOptions(); + auto* cf_handle = + reinterpret_cast(jcf_handle); + if (cf_handle != nullptr) { + rocksdb_single_delete_helper(env, db, default_write_options, cf_handle, + jkey, jkey_len); + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::InvalidArgument( + "Invalid ColumnFamilyHandle.")); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: singleDelete + * Signature: (JJ[BIJ)V + */ +void Java_org_rocksdb_RocksDB_singleDelete__JJ_3BI(JNIEnv* env, jobject, + jlong jdb_handle, + jlong jwrite_options, + jbyteArray jkey, + jint jkey_len) { + auto* db = reinterpret_cast(jdb_handle); + auto* write_options = + reinterpret_cast(jwrite_options); + rocksdb_single_delete_helper(env, db, *write_options, nullptr, jkey, + jkey_len); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: singleDelete + * Signature: (JJ[BIJ)V + */ +void Java_org_rocksdb_RocksDB_singleDelete__JJ_3BIJ( + JNIEnv* env, jobject, jlong jdb_handle, jlong jwrite_options, + jbyteArray jkey, jint jkey_len, jlong jcf_handle) { + auto* db = reinterpret_cast(jdb_handle); + auto* write_options = + reinterpret_cast(jwrite_options); + auto* cf_handle = + reinterpret_cast(jcf_handle); + if (cf_handle != nullptr) { + rocksdb_single_delete_helper(env, db, *write_options, cf_handle, jkey, + jkey_len); + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::InvalidArgument( + "Invalid ColumnFamilyHandle.")); + } +} + +////////////////////////////////////////////////////////////////////////////// +// ROCKSDB_NAMESPACE::DB::DeleteRange() +/** + * @return true if the delete range succeeded, false if a Java Exception + * was thrown + */ +bool rocksdb_delete_range_helper( + JNIEnv* env, ROCKSDB_NAMESPACE::DB* db, + const ROCKSDB_NAMESPACE::WriteOptions& write_options, + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle, jbyteArray jbegin_key, + jint jbegin_key_off, jint jbegin_key_len, jbyteArray jend_key, + jint jend_key_off, jint jend_key_len) { + jbyte* begin_key = new jbyte[jbegin_key_len]; + env->GetByteArrayRegion(jbegin_key, jbegin_key_off, jbegin_key_len, + begin_key); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] begin_key; + return false; + } + ROCKSDB_NAMESPACE::Slice begin_key_slice(reinterpret_cast(begin_key), + jbegin_key_len); + + jbyte* end_key = new jbyte[jend_key_len]; + env->GetByteArrayRegion(jend_key, jend_key_off, jend_key_len, end_key); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] begin_key; + delete[] end_key; + return false; + } + ROCKSDB_NAMESPACE::Slice end_key_slice(reinterpret_cast(end_key), + jend_key_len); + + ROCKSDB_NAMESPACE::Status s = + db->DeleteRange(write_options, cf_handle, begin_key_slice, end_key_slice); + + // cleanup + delete[] begin_key; + delete[] end_key; + + if (s.ok()) { + return true; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return false; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: deleteRange + * Signature: (J[BII[BII)V + */ +void Java_org_rocksdb_RocksDB_deleteRange__J_3BII_3BII( + JNIEnv* env, jobject, jlong jdb_handle, jbyteArray jbegin_key, + jint jbegin_key_off, jint jbegin_key_len, jbyteArray jend_key, + jint jend_key_off, jint jend_key_len) { + auto* db = reinterpret_cast(jdb_handle); + static const ROCKSDB_NAMESPACE::WriteOptions default_write_options = + ROCKSDB_NAMESPACE::WriteOptions(); + rocksdb_delete_range_helper(env, db, default_write_options, nullptr, + jbegin_key, jbegin_key_off, jbegin_key_len, + jend_key, jend_key_off, jend_key_len); +} + +jint rocksdb_get_helper_direct( + JNIEnv* env, ROCKSDB_NAMESPACE::DB* db, + const ROCKSDB_NAMESPACE::ReadOptions& read_options, + ROCKSDB_NAMESPACE::ColumnFamilyHandle* column_family_handle, jobject jkey, + jint jkey_off, jint jkey_len, jobject jval, jint jval_off, jint jval_len, + bool* has_exception) { + static const int kNotFound = -1; + static const int kStatusError = -2; + static const int kArgumentError = -3; + + char* key = reinterpret_cast(env->GetDirectBufferAddress(jkey)); + if (key == nullptr) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, + "Invalid key argument (argument is not a valid direct ByteBuffer)"); + *has_exception = true; + return kArgumentError; + } + if (env->GetDirectBufferCapacity(jkey) < (jkey_off + jkey_len)) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, + "Invalid key argument. Capacity is less than requested region (offset " + "+ length)."); + *has_exception = true; + return kArgumentError; + } + + char* value = reinterpret_cast(env->GetDirectBufferAddress(jval)); + if (value == nullptr) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, + "Invalid value argument (argument is not a valid direct ByteBuffer)"); + *has_exception = true; + return kArgumentError; + } + + if (env->GetDirectBufferCapacity(jval) < (jval_off + jval_len)) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, + "Invalid value argument. Capacity is less than requested region " + "(offset + length)."); + *has_exception = true; + return kArgumentError; + } + + key += jkey_off; + value += jval_off; + + ROCKSDB_NAMESPACE::Slice key_slice(key, jkey_len); + + ROCKSDB_NAMESPACE::PinnableSlice pinnable_value; + ROCKSDB_NAMESPACE::Status s; + if (column_family_handle != nullptr) { + s = db->Get(read_options, column_family_handle, key_slice, &pinnable_value); + } else { + // backwards compatibility + s = db->Get(read_options, db->DefaultColumnFamily(), key_slice, + &pinnable_value); + } + + if (s.IsNotFound()) { + *has_exception = false; + return kNotFound; + } else if (!s.ok()) { + *has_exception = true; + // Here since we are throwing a Java exception from c++ side. + // As a result, c++ does not know calling this function will in fact + // throwing an exception. As a result, the execution flow will + // not stop here, and codes after this throw will still be + // executed. + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + + // Return a dummy const value to avoid compilation error, although + // java side might not have a chance to get the return value :) + return kStatusError; + } + + const jint pinnable_value_len = static_cast(pinnable_value.size()); + const jint length = std::min(jval_len, pinnable_value_len); + + memcpy(value, pinnable_value.data(), length); + pinnable_value.Reset(); + + *has_exception = false; + return pinnable_value_len; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: deleteRange + * Signature: (J[BII[BIIJ)V + */ +void Java_org_rocksdb_RocksDB_deleteRange__J_3BII_3BIIJ( + JNIEnv* env, jobject, jlong jdb_handle, jbyteArray jbegin_key, + jint jbegin_key_off, jint jbegin_key_len, jbyteArray jend_key, + jint jend_key_off, jint jend_key_len, jlong jcf_handle) { + auto* db = reinterpret_cast(jdb_handle); + static const ROCKSDB_NAMESPACE::WriteOptions default_write_options = + ROCKSDB_NAMESPACE::WriteOptions(); + auto* cf_handle = + reinterpret_cast(jcf_handle); + if (cf_handle != nullptr) { + rocksdb_delete_range_helper(env, db, default_write_options, cf_handle, + jbegin_key, jbegin_key_off, jbegin_key_len, + jend_key, jend_key_off, jend_key_len); + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::InvalidArgument( + "Invalid ColumnFamilyHandle.")); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: deleteRange + * Signature: (JJ[BII[BII)V + */ +void Java_org_rocksdb_RocksDB_deleteRange__JJ_3BII_3BII( + JNIEnv* env, jobject, jlong jdb_handle, jlong jwrite_options, + jbyteArray jbegin_key, jint jbegin_key_off, jint jbegin_key_len, + jbyteArray jend_key, jint jend_key_off, jint jend_key_len) { + auto* db = reinterpret_cast(jdb_handle); + auto* write_options = + reinterpret_cast(jwrite_options); + rocksdb_delete_range_helper(env, db, *write_options, nullptr, jbegin_key, + jbegin_key_off, jbegin_key_len, jend_key, + jend_key_off, jend_key_len); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: deleteRange + * Signature: (JJ[BII[BIIJ)V + */ +void Java_org_rocksdb_RocksDB_deleteRange__JJ_3BII_3BIIJ( + JNIEnv* env, jobject, jlong jdb_handle, jlong jwrite_options, + jbyteArray jbegin_key, jint jbegin_key_off, jint jbegin_key_len, + jbyteArray jend_key, jint jend_key_off, jint jend_key_len, + jlong jcf_handle) { + auto* db = reinterpret_cast(jdb_handle); + auto* write_options = + reinterpret_cast(jwrite_options); + auto* cf_handle = + reinterpret_cast(jcf_handle); + if (cf_handle != nullptr) { + rocksdb_delete_range_helper(env, db, *write_options, cf_handle, jbegin_key, + jbegin_key_off, jbegin_key_len, jend_key, + jend_key_off, jend_key_len); + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::InvalidArgument( + "Invalid ColumnFamilyHandle.")); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: getDirect + * Signature: (JJLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJ)I + */ +jint Java_org_rocksdb_RocksDB_getDirect(JNIEnv* env, jobject /*jdb*/, + jlong jdb_handle, jlong jropt_handle, + jobject jkey, jint jkey_off, + jint jkey_len, jobject jval, + jint jval_off, jint jval_len, + jlong jcf_handle) { + auto* db_handle = reinterpret_cast(jdb_handle); + auto* ro_opt = + reinterpret_cast(jropt_handle); + auto* cf_handle = + reinterpret_cast(jcf_handle); + bool has_exception = false; + return rocksdb_get_helper_direct( + env, db_handle, + ro_opt == nullptr ? ROCKSDB_NAMESPACE::ReadOptions() : *ro_opt, cf_handle, + jkey, jkey_off, jkey_len, jval, jval_off, jval_len, &has_exception); +} + +////////////////////////////////////////////////////////////////////////////// +// ROCKSDB_NAMESPACE::DB::Merge + +/** + * @return true if the merge succeeded, false if a Java Exception was thrown + */ +bool rocksdb_merge_helper(JNIEnv* env, ROCKSDB_NAMESPACE::DB* db, + const ROCKSDB_NAMESPACE::WriteOptions& write_options, + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle, + jbyteArray jkey, jint jkey_off, jint jkey_len, + jbyteArray jval, jint jval_off, jint jval_len) { + jbyte* key = new jbyte[jkey_len]; + env->GetByteArrayRegion(jkey, jkey_off, jkey_len, key); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key; + return false; + } + ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), jkey_len); + + jbyte* value = new jbyte[jval_len]; + env->GetByteArrayRegion(jval, jval_off, jval_len, value); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] value; + delete[] key; + return false; + } + ROCKSDB_NAMESPACE::Slice value_slice(reinterpret_cast(value), + jval_len); + + ROCKSDB_NAMESPACE::Status s; + if (cf_handle != nullptr) { + s = db->Merge(write_options, cf_handle, key_slice, value_slice); + } else { + s = db->Merge(write_options, key_slice, value_slice); + } + + // cleanup + delete[] value; + delete[] key; + + if (s.ok()) { + return true; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return false; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: merge + * Signature: (J[BII[BII)V + */ +void Java_org_rocksdb_RocksDB_merge__J_3BII_3BII(JNIEnv* env, jobject, + jlong jdb_handle, + jbyteArray jkey, jint jkey_off, + jint jkey_len, jbyteArray jval, + jint jval_off, jint jval_len) { + auto* db = reinterpret_cast(jdb_handle); + static const ROCKSDB_NAMESPACE::WriteOptions default_write_options = + ROCKSDB_NAMESPACE::WriteOptions(); + rocksdb_merge_helper(env, db, default_write_options, nullptr, jkey, jkey_off, + jkey_len, jval, jval_off, jval_len); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: merge + * Signature: (J[BII[BIIJ)V + */ +void Java_org_rocksdb_RocksDB_merge__J_3BII_3BIIJ( + JNIEnv* env, jobject, jlong jdb_handle, jbyteArray jkey, jint jkey_off, + jint jkey_len, jbyteArray jval, jint jval_off, jint jval_len, + jlong jcf_handle) { + auto* db = reinterpret_cast(jdb_handle); + static const ROCKSDB_NAMESPACE::WriteOptions default_write_options = + ROCKSDB_NAMESPACE::WriteOptions(); + auto* cf_handle = + reinterpret_cast(jcf_handle); + if (cf_handle != nullptr) { + rocksdb_merge_helper(env, db, default_write_options, cf_handle, jkey, + jkey_off, jkey_len, jval, jval_off, jval_len); + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::InvalidArgument( + "Invalid ColumnFamilyHandle.")); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: merge + * Signature: (JJ[BII[BII)V + */ +void Java_org_rocksdb_RocksDB_merge__JJ_3BII_3BII( + JNIEnv* env, jobject, jlong jdb_handle, jlong jwrite_options_handle, + jbyteArray jkey, jint jkey_off, jint jkey_len, jbyteArray jval, + jint jval_off, jint jval_len) { + auto* db = reinterpret_cast(jdb_handle); + auto* write_options = + reinterpret_cast(jwrite_options_handle); + rocksdb_merge_helper(env, db, *write_options, nullptr, jkey, jkey_off, + jkey_len, jval, jval_off, jval_len); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: merge + * Signature: (JJ[BII[BIIJ)V + */ +void Java_org_rocksdb_RocksDB_merge__JJ_3BII_3BIIJ( + JNIEnv* env, jobject, jlong jdb_handle, jlong jwrite_options_handle, + jbyteArray jkey, jint jkey_off, jint jkey_len, jbyteArray jval, + jint jval_off, jint jval_len, jlong jcf_handle) { + auto* db = reinterpret_cast(jdb_handle); + auto* write_options = + reinterpret_cast(jwrite_options_handle); + auto* cf_handle = + reinterpret_cast(jcf_handle); + if (cf_handle != nullptr) { + rocksdb_merge_helper(env, db, *write_options, cf_handle, jkey, jkey_off, + jkey_len, jval, jval_off, jval_len); + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::InvalidArgument( + "Invalid ColumnFamilyHandle.")); + } +} + +jlong rocksdb_iterator_helper( + ROCKSDB_NAMESPACE::DB* db, ROCKSDB_NAMESPACE::ReadOptions read_options, + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle) { + ROCKSDB_NAMESPACE::Iterator* iterator = nullptr; + if (cf_handle != nullptr) { + iterator = db->NewIterator(read_options, cf_handle); + } else { + iterator = db->NewIterator(read_options); + } + return GET_CPLUSPLUS_POINTER(iterator); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: deleteDirect + * Signature: (JJLjava/nio/ByteBuffer;IIJ)V + */ +void Java_org_rocksdb_RocksDB_deleteDirect(JNIEnv* env, jobject /*jdb*/, + jlong jdb_handle, + jlong jwrite_options, jobject jkey, + jint jkey_offset, jint jkey_len, + jlong jcf_handle) { + auto* db = reinterpret_cast(jdb_handle); + auto* write_options = + reinterpret_cast(jwrite_options); + auto* cf_handle = + reinterpret_cast(jcf_handle); + auto remove = [&env, &db, &write_options, + &cf_handle](ROCKSDB_NAMESPACE::Slice& key) { + ROCKSDB_NAMESPACE::Status s; + if (cf_handle == nullptr) { + s = db->Delete(*write_options, key); + } else { + s = db->Delete(*write_options, cf_handle, key); + } + if (s.ok()) { + return; + } + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + }; + ROCKSDB_NAMESPACE::JniUtil::k_op_direct(remove, env, jkey, jkey_offset, + jkey_len); +} + +////////////////////////////////////////////////////////////////////////////// +// ROCKSDB_NAMESPACE::DB::Write +/* + * Class: org_rocksdb_RocksDB + * Method: write0 + * Signature: (JJJ)V + */ +void Java_org_rocksdb_RocksDB_write0(JNIEnv* env, jobject, jlong jdb_handle, + jlong jwrite_options_handle, + jlong jwb_handle) { + auto* db = reinterpret_cast(jdb_handle); + auto* write_options = + reinterpret_cast(jwrite_options_handle); + auto* wb = reinterpret_cast(jwb_handle); + + ROCKSDB_NAMESPACE::Status s = db->Write(*write_options, wb); + + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: write1 + * Signature: (JJJ)V + */ +void Java_org_rocksdb_RocksDB_write1(JNIEnv* env, jobject, jlong jdb_handle, + jlong jwrite_options_handle, + jlong jwbwi_handle) { + auto* db = reinterpret_cast(jdb_handle); + auto* write_options = + reinterpret_cast(jwrite_options_handle); + auto* wbwi = + reinterpret_cast(jwbwi_handle); + auto* wb = wbwi->GetWriteBatch(); + + ROCKSDB_NAMESPACE::Status s = db->Write(*write_options, wb); + + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +////////////////////////////////////////////////////////////////////////////// +// ROCKSDB_NAMESPACE::DB::Get + +jbyteArray rocksdb_get_helper( + JNIEnv* env, ROCKSDB_NAMESPACE::DB* db, + const ROCKSDB_NAMESPACE::ReadOptions& read_opt, + ROCKSDB_NAMESPACE::ColumnFamilyHandle* column_family_handle, + jbyteArray jkey, jint jkey_off, jint jkey_len) { + jbyte* key = new jbyte[jkey_len]; + env->GetByteArrayRegion(jkey, jkey_off, jkey_len, key); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key; + return nullptr; + } + + ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), jkey_len); + + ROCKSDB_NAMESPACE::PinnableSlice pinnable_value; + ROCKSDB_NAMESPACE::Status s; + if (column_family_handle != nullptr) { + s = db->Get(read_opt, column_family_handle, key_slice, &pinnable_value); + } else { + s = db->Get(read_opt, db->DefaultColumnFamily(), key_slice, + &pinnable_value); + } + + // cleanup + delete[] key; + + if (s.IsNotFound()) { + return nullptr; + } + + if (s.ok()) { + jbyteArray jret_value = + ROCKSDB_NAMESPACE::JniUtil::copyBytes(env, pinnable_value); + pinnable_value.Reset(); + if (jret_value == nullptr) { + // exception occurred + return nullptr; + } + return jret_value; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return nullptr; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: get + * Signature: (J[BII)[B + */ +jbyteArray Java_org_rocksdb_RocksDB_get__J_3BII(JNIEnv* env, jobject, + jlong jdb_handle, + jbyteArray jkey, jint jkey_off, + jint jkey_len) { + return rocksdb_get_helper( + env, reinterpret_cast(jdb_handle), + ROCKSDB_NAMESPACE::ReadOptions(), nullptr, jkey, jkey_off, jkey_len); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: get + * Signature: (J[BIIJ)[B + */ +jbyteArray Java_org_rocksdb_RocksDB_get__J_3BIIJ(JNIEnv* env, jobject, + jlong jdb_handle, + jbyteArray jkey, jint jkey_off, + jint jkey_len, + jlong jcf_handle) { + auto db_handle = reinterpret_cast(jdb_handle); + auto cf_handle = + reinterpret_cast(jcf_handle); + if (cf_handle != nullptr) { + return rocksdb_get_helper(env, db_handle, ROCKSDB_NAMESPACE::ReadOptions(), + cf_handle, jkey, jkey_off, jkey_len); + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::InvalidArgument( + "Invalid ColumnFamilyHandle.")); + return nullptr; + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: get + * Signature: (JJ[BII)[B + */ +jbyteArray Java_org_rocksdb_RocksDB_get__JJ_3BII(JNIEnv* env, jobject, + jlong jdb_handle, + jlong jropt_handle, + jbyteArray jkey, jint jkey_off, + jint jkey_len) { + return rocksdb_get_helper( + env, reinterpret_cast(jdb_handle), + *reinterpret_cast(jropt_handle), nullptr, + jkey, jkey_off, jkey_len); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: get + * Signature: (JJ[BIIJ)[B + */ +jbyteArray Java_org_rocksdb_RocksDB_get__JJ_3BIIJ( + JNIEnv* env, jobject, jlong jdb_handle, jlong jropt_handle, jbyteArray jkey, + jint jkey_off, jint jkey_len, jlong jcf_handle) { + auto* db_handle = reinterpret_cast(jdb_handle); + auto& ro_opt = + *reinterpret_cast(jropt_handle); + auto* cf_handle = + reinterpret_cast(jcf_handle); + if (cf_handle != nullptr) { + return rocksdb_get_helper(env, db_handle, ro_opt, cf_handle, jkey, jkey_off, + jkey_len); + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::InvalidArgument( + "Invalid ColumnFamilyHandle.")); + return nullptr; + } +} + +jint rocksdb_get_helper( + JNIEnv* env, ROCKSDB_NAMESPACE::DB* db, + const ROCKSDB_NAMESPACE::ReadOptions& read_options, + ROCKSDB_NAMESPACE::ColumnFamilyHandle* column_family_handle, + jbyteArray jkey, jint jkey_off, jint jkey_len, jbyteArray jval, + jint jval_off, jint jval_len, bool* has_exception) { + static const int kNotFound = -1; + static const int kStatusError = -2; + + jbyte* key = new jbyte[jkey_len]; + env->GetByteArrayRegion(jkey, jkey_off, jkey_len, key); + if (env->ExceptionCheck()) { + // exception thrown: OutOfMemoryError + delete[] key; + *has_exception = true; + return kStatusError; + } + ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), jkey_len); + + ROCKSDB_NAMESPACE::PinnableSlice pinnable_value; + ROCKSDB_NAMESPACE::Status s; + if (column_family_handle != nullptr) { + s = db->Get(read_options, column_family_handle, key_slice, &pinnable_value); + } else { + s = db->Get(read_options, db->DefaultColumnFamily(), key_slice, + &pinnable_value); + } + + // cleanup + delete[] key; + + if (s.IsNotFound()) { + *has_exception = false; + return kNotFound; + } else if (!s.ok()) { + *has_exception = true; + // Here since we are throwing a Java exception from c++ side. + // As a result, c++ does not know calling this function will in fact + // throwing an exception. As a result, the execution flow will + // not stop here, and codes after this throw will still be + // executed. + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + + // Return a dummy const value to avoid compilation error, although + // java side might not have a chance to get the return value :) + return kStatusError; + } + + const jint pinnable_value_len = static_cast(pinnable_value.size()); + const jint length = std::min(jval_len, pinnable_value_len); + + env->SetByteArrayRegion(jval, jval_off, length, + const_cast(reinterpret_cast( + pinnable_value.data()))); + pinnable_value.Reset(); + if (env->ExceptionCheck()) { + // exception thrown: OutOfMemoryError + *has_exception = true; + return kStatusError; + } + + *has_exception = false; + return pinnable_value_len; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: get + * Signature: (J[BII[BII)I + */ +jint Java_org_rocksdb_RocksDB_get__J_3BII_3BII(JNIEnv* env, jobject, + jlong jdb_handle, + jbyteArray jkey, jint jkey_off, + jint jkey_len, jbyteArray jval, + jint jval_off, jint jval_len) { + bool has_exception = false; + return rocksdb_get_helper( + env, reinterpret_cast(jdb_handle), + ROCKSDB_NAMESPACE::ReadOptions(), nullptr, jkey, jkey_off, jkey_len, jval, + jval_off, jval_len, &has_exception); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: get + * Signature: (J[BII[BIIJ)I + */ +jint Java_org_rocksdb_RocksDB_get__J_3BII_3BIIJ(JNIEnv* env, jobject, + jlong jdb_handle, + jbyteArray jkey, jint jkey_off, + jint jkey_len, jbyteArray jval, + jint jval_off, jint jval_len, + jlong jcf_handle) { + auto* db_handle = reinterpret_cast(jdb_handle); + auto* cf_handle = + reinterpret_cast(jcf_handle); + if (cf_handle != nullptr) { + bool has_exception = false; + return rocksdb_get_helper(env, db_handle, ROCKSDB_NAMESPACE::ReadOptions(), + cf_handle, jkey, jkey_off, jkey_len, jval, + jval_off, jval_len, &has_exception); + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::InvalidArgument( + "Invalid ColumnFamilyHandle.")); + // will never be evaluated + return 0; + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: get + * Signature: (JJ[BII[BII)I + */ +jint Java_org_rocksdb_RocksDB_get__JJ_3BII_3BII(JNIEnv* env, jobject, + jlong jdb_handle, + jlong jropt_handle, + jbyteArray jkey, jint jkey_off, + jint jkey_len, jbyteArray jval, + jint jval_off, jint jval_len) { + bool has_exception = false; + return rocksdb_get_helper( + env, reinterpret_cast(jdb_handle), + *reinterpret_cast(jropt_handle), nullptr, + jkey, jkey_off, jkey_len, jval, jval_off, jval_len, &has_exception); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: get + * Signature: (JJ[BII[BIIJ)I + */ +jint Java_org_rocksdb_RocksDB_get__JJ_3BII_3BIIJ( + JNIEnv* env, jobject, jlong jdb_handle, jlong jropt_handle, jbyteArray jkey, + jint jkey_off, jint jkey_len, jbyteArray jval, jint jval_off, jint jval_len, + jlong jcf_handle) { + auto* db_handle = reinterpret_cast(jdb_handle); + auto& ro_opt = + *reinterpret_cast(jropt_handle); + auto* cf_handle = + reinterpret_cast(jcf_handle); + if (cf_handle != nullptr) { + bool has_exception = false; + return rocksdb_get_helper(env, db_handle, ro_opt, cf_handle, jkey, jkey_off, + jkey_len, jval, jval_off, jval_len, + &has_exception); + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::InvalidArgument( + "Invalid ColumnFamilyHandle.")); + // will never be evaluated + return 0; + } +} + +inline void multi_get_helper_release_keys(std::vector& keys_to_free) { + auto end = keys_to_free.end(); + for (auto it = keys_to_free.begin(); it != end; ++it) { + delete[] * it; + } + keys_to_free.clear(); +} + +/** + * @brief fill a native array of cf handles from java handles + * + * @param env + * @param cf_handles to fill from the java variants + * @param jcolumn_family_handles + * @return true if the copy succeeds + * @return false if a JNI exception is generated + */ +inline bool cf_handles_from_jcf_handles( + JNIEnv* env, + std::vector& cf_handles, + jlongArray jcolumn_family_handles) { + if (jcolumn_family_handles != nullptr) { + const jsize len_cols = env->GetArrayLength(jcolumn_family_handles); + + jlong* jcfh = env->GetLongArrayElements(jcolumn_family_handles, nullptr); + if (jcfh == nullptr) { + // exception thrown: OutOfMemoryError + jclass exception_cls = (env)->FindClass("java/lang/OutOfMemoryError"); + (env)->ThrowNew(exception_cls, + "Insufficient Memory for CF handle array."); + return false; + } + + for (jsize i = 0; i < len_cols; i++) { + auto* cf_handle = + reinterpret_cast(jcfh[i]); + cf_handles.push_back(cf_handle); + } + env->ReleaseLongArrayElements(jcolumn_family_handles, jcfh, JNI_ABORT); + } + return true; +} + +/** + * @brief copy keys from JNI into vector of slices for Rocks API + * + * @param keys to instantiate + * @param jkeys + * @param jkey_offs + * @param jkey_lens + * @return true if the copy succeeds + * @return false if a JNI exception is raised + */ +inline bool keys_from_jkeys(JNIEnv* env, + std::vector& keys, + std::vector& keys_to_free, + jobjectArray jkeys, jintArray jkey_offs, + jintArray jkey_lens) { + jint* jkey_off = env->GetIntArrayElements(jkey_offs, nullptr); + if (jkey_off == nullptr) { + // exception thrown: OutOfMemoryError + jclass exception_cls = (env)->FindClass("java/lang/OutOfMemoryError"); + (env)->ThrowNew(exception_cls, "Insufficient Memory for key offset array."); + return false; + } + + jint* jkey_len = env->GetIntArrayElements(jkey_lens, nullptr); + if (jkey_len == nullptr) { + // exception thrown: OutOfMemoryError + env->ReleaseIntArrayElements(jkey_offs, jkey_off, JNI_ABORT); + jclass exception_cls = (env)->FindClass("java/lang/OutOfMemoryError"); + (env)->ThrowNew(exception_cls, "Insufficient Memory for key length array."); + return false; + } + + const jsize len_keys = env->GetArrayLength(jkeys); + for (jsize i = 0; i < len_keys; i++) { + jobject jkey = env->GetObjectArrayElement(jkeys, i); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->ReleaseIntArrayElements(jkey_lens, jkey_len, JNI_ABORT); + env->ReleaseIntArrayElements(jkey_offs, jkey_off, JNI_ABORT); + multi_get_helper_release_keys(keys_to_free); + jclass exception_cls = (env)->FindClass("java/lang/OutOfMemoryError"); + (env)->ThrowNew(exception_cls, + "Insufficient Memory for key object array."); + return false; + } + + jbyteArray jkey_ba = reinterpret_cast(jkey); + + const jint len_key = jkey_len[i]; + jbyte* key = new jbyte[len_key]; + env->GetByteArrayRegion(jkey_ba, jkey_off[i], len_key, key); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key; + env->DeleteLocalRef(jkey); + env->ReleaseIntArrayElements(jkey_lens, jkey_len, JNI_ABORT); + env->ReleaseIntArrayElements(jkey_offs, jkey_off, JNI_ABORT); + multi_get_helper_release_keys(keys_to_free); + jclass exception_cls = + (env)->FindClass("java/lang/ArrayIndexOutOfBoundsException"); + (env)->ThrowNew(exception_cls, "Invalid byte array region index."); + return false; + } + + ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), len_key); + keys.push_back(key_slice); + + env->DeleteLocalRef(jkey); + keys_to_free.push_back(key); + } + + // cleanup jkey_off and jken_len + env->ReleaseIntArrayElements(jkey_lens, jkey_len, JNI_ABORT); + env->ReleaseIntArrayElements(jkey_offs, jkey_off, JNI_ABORT); + + return true; +} + +inline bool keys_from_bytebuffers(JNIEnv* env, + std::vector& keys, + jobjectArray jkeys, jintArray jkey_offs, + jintArray jkey_lens) { + jint* jkey_off = env->GetIntArrayElements(jkey_offs, nullptr); + if (jkey_off == nullptr) { + // exception thrown: OutOfMemoryError + jclass exception_cls = (env)->FindClass("java/lang/OutOfMemoryError"); + (env)->ThrowNew(exception_cls, "Insufficient Memory for key offset array."); + return false; + } + + jint* jkey_len = env->GetIntArrayElements(jkey_lens, nullptr); + if (jkey_len == nullptr) { + // exception thrown: OutOfMemoryError + env->ReleaseIntArrayElements(jkey_offs, jkey_off, JNI_ABORT); + jclass exception_cls = (env)->FindClass("java/lang/OutOfMemoryError"); + (env)->ThrowNew(exception_cls, "Insufficient Memory for key length array."); + return false; + } + + const jsize len_keys = env->GetArrayLength(jkeys); + for (jsize i = 0; i < len_keys; i++) { + jobject jkey = env->GetObjectArrayElement(jkeys, i); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + // cleanup jkey_off and jkey_len + env->ReleaseIntArrayElements(jkey_lens, jkey_len, JNI_ABORT); + env->ReleaseIntArrayElements(jkey_offs, jkey_off, JNI_ABORT); + + return false; + } + char* key = reinterpret_cast(env->GetDirectBufferAddress(jkey)); + ROCKSDB_NAMESPACE::Slice key_slice(key + jkey_off[i], jkey_len[i]); + keys.push_back(key_slice); + + env->DeleteLocalRef(jkey); + } + + // cleanup jkey_off and jkey_len + env->ReleaseIntArrayElements(jkey_lens, jkey_len, JNI_ABORT); + env->ReleaseIntArrayElements(jkey_offs, jkey_off, JNI_ABORT); + + return true; +} + +/** + * cf multi get + * + * @return byte[][] of values or nullptr if an + * exception occurs + */ +jobjectArray multi_get_helper(JNIEnv* env, jobject, ROCKSDB_NAMESPACE::DB* db, + const ROCKSDB_NAMESPACE::ReadOptions& rOpt, + jobjectArray jkeys, jintArray jkey_offs, + jintArray jkey_lens, + jlongArray jcolumn_family_handles) { + std::vector cf_handles; + if (!cf_handles_from_jcf_handles(env, cf_handles, jcolumn_family_handles)) { + return nullptr; + } + + std::vector keys; + std::vector keys_to_free; + if (!keys_from_jkeys(env, keys, keys_to_free, jkeys, jkey_offs, jkey_lens)) { + return nullptr; + } + + std::vector values; + std::vector s; + if (cf_handles.size() == 0) { + s = db->MultiGet(rOpt, keys, &values); + } else { + s = db->MultiGet(rOpt, cf_handles, keys, &values); + } + + // free up allocated byte arrays + multi_get_helper_release_keys(keys_to_free); + + // prepare the results + jobjectArray jresults = ROCKSDB_NAMESPACE::ByteJni::new2dByteArray( + env, static_cast(s.size())); + if (jresults == nullptr) { + // exception occurred + jclass exception_cls = (env)->FindClass("java/lang/OutOfMemoryError"); + (env)->ThrowNew(exception_cls, "Insufficient Memory for results."); + return nullptr; + } + + // add to the jresults + for (std::vector::size_type i = 0; i != s.size(); + i++) { + if (s[i].ok()) { + std::string* value = &values[i]; + const jsize jvalue_len = static_cast(value->size()); + jbyteArray jentry_value = env->NewByteArray(jvalue_len); + if (jentry_value == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + env->SetByteArrayRegion( + jentry_value, 0, static_cast(jvalue_len), + const_cast(reinterpret_cast(value->c_str()))); + if (env->ExceptionCheck()) { + // exception thrown: + // ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jentry_value); + return nullptr; + } + + env->SetObjectArrayElement(jresults, static_cast(i), jentry_value); + if (env->ExceptionCheck()) { + // exception thrown: + // ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jentry_value); + return nullptr; + } + + env->DeleteLocalRef(jentry_value); + } + } + + return jresults; +} + +/** + * cf multi get + * + * fill supplied native buffers, or raise JNI + * exception on a problem + */ + +/** + * @brief multi_get_helper_direct for fast-path multiget (io_uring) on Linux + * + * @param env + * @param db + * @param rOpt read options + * @param jcolumn_family_handles 0, 1, or n column family handles + * @param jkeys + * @param jkey_offsets + * @param jkey_lengths + * @param jvalues byte buffers to receive values + * @param jvalue_sizes returned actual sizes of data values for keys + * @param jstatuses returned java RocksDB status values for per key + */ +void multi_get_helper_direct(JNIEnv* env, jobject, ROCKSDB_NAMESPACE::DB* db, + const ROCKSDB_NAMESPACE::ReadOptions& rOpt, + jlongArray jcolumn_family_handles, + jobjectArray jkeys, jintArray jkey_offsets, + jintArray jkey_lengths, jobjectArray jvalues, + jintArray jvalue_sizes, jobjectArray jstatuses) { + const jsize num_keys = env->GetArrayLength(jkeys); + + std::vector keys; + if (!keys_from_bytebuffers(env, keys, jkeys, jkey_offsets, jkey_lengths)) { + return; + } + + std::vector values(num_keys); + + std::vector cf_handles; + if (!cf_handles_from_jcf_handles(env, cf_handles, jcolumn_family_handles)) { + return; + } + + std::vector s(num_keys); + if (cf_handles.size() == 0) { + // we can use the more efficient call here + auto cf_handle = db->DefaultColumnFamily(); + db->MultiGet(rOpt, cf_handle, num_keys, keys.data(), values.data(), + s.data()); + } else if (cf_handles.size() == 1) { + // we can use the more efficient call here + auto cf_handle = cf_handles[0]; + db->MultiGet(rOpt, cf_handle, num_keys, keys.data(), values.data(), + s.data()); + } else { + // multiple CFs version + db->MultiGet(rOpt, num_keys, cf_handles.data(), keys.data(), values.data(), + s.data()); + } + + // prepare the results + jobjectArray jresults = ROCKSDB_NAMESPACE::ByteJni::new2dByteArray( + env, static_cast(s.size())); + if (jresults == nullptr) { + // exception occurred + jclass exception_cls = (env)->FindClass("java/lang/OutOfMemoryError"); + (env)->ThrowNew(exception_cls, "Insufficient Memory for results."); + return; + } + + std::vector value_size; + for (int i = 0; i < num_keys; i++) { + auto jstatus = ROCKSDB_NAMESPACE::StatusJni::construct(env, s[i]); + if (jstatus == nullptr) { + // exception in context + return; + } + env->SetObjectArrayElement(jstatuses, i, jstatus); + + if (s[i].ok()) { + jobject jvalue_bytebuf = env->GetObjectArrayElement(jvalues, i); + if (env->ExceptionCheck()) { + // ArrayIndexOutOfBoundsException is thrown + return; + } + jlong jvalue_capacity = env->GetDirectBufferCapacity(jvalue_bytebuf); + if (jvalue_capacity == -1) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, + "Invalid value(s) argument (argument is not a valid direct " + "ByteBuffer)"); + return; + } + void* jvalue_address = env->GetDirectBufferAddress(jvalue_bytebuf); + if (jvalue_address == nullptr) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, + "Invalid value(s) argument (argument is not a valid direct " + "ByteBuffer)"); + return; + } + + // record num returned, push back that number, which may be bigger then + // the ByteBuffer supplied. then copy as much as fits in the ByteBuffer. + value_size.push_back(static_cast(values[i].size())); + auto copy_bytes = + std::min(static_cast(values[i].size()), jvalue_capacity); + memcpy(jvalue_address, values[i].data(), copy_bytes); + } else { + // bad status for this + value_size.push_back(0); + } + } + + env->SetIntArrayRegion(jvalue_sizes, 0, num_keys, value_size.data()); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: multiGet + * Signature: (J[[B[I[I)[[B + */ +jobjectArray Java_org_rocksdb_RocksDB_multiGet__J_3_3B_3I_3I( + JNIEnv* env, jobject jdb, jlong jdb_handle, jobjectArray jkeys, + jintArray jkey_offs, jintArray jkey_lens) { + return multi_get_helper( + env, jdb, reinterpret_cast(jdb_handle), + ROCKSDB_NAMESPACE::ReadOptions(), jkeys, jkey_offs, jkey_lens, nullptr); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: multiGet + * Signature: (J[[B[I[I[J)[[B + */ +jobjectArray Java_org_rocksdb_RocksDB_multiGet__J_3_3B_3I_3I_3J( + JNIEnv* env, jobject jdb, jlong jdb_handle, jobjectArray jkeys, + jintArray jkey_offs, jintArray jkey_lens, + jlongArray jcolumn_family_handles) { + return multi_get_helper(env, jdb, + reinterpret_cast(jdb_handle), + ROCKSDB_NAMESPACE::ReadOptions(), jkeys, jkey_offs, + jkey_lens, jcolumn_family_handles); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: multiGet + * Signature: (JJ[[B[I[I)[[B + */ +jobjectArray Java_org_rocksdb_RocksDB_multiGet__JJ_3_3B_3I_3I( + JNIEnv* env, jobject jdb, jlong jdb_handle, jlong jropt_handle, + jobjectArray jkeys, jintArray jkey_offs, jintArray jkey_lens) { + return multi_get_helper( + env, jdb, reinterpret_cast(jdb_handle), + *reinterpret_cast(jropt_handle), jkeys, + jkey_offs, jkey_lens, nullptr); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: multiGet + * Signature: (JJ[[B[I[I[J)[[B + */ +jobjectArray Java_org_rocksdb_RocksDB_multiGet__JJ_3_3B_3I_3I_3J( + JNIEnv* env, jobject jdb, jlong jdb_handle, jlong jropt_handle, + jobjectArray jkeys, jintArray jkey_offs, jintArray jkey_lens, + jlongArray jcolumn_family_handles) { + return multi_get_helper( + env, jdb, reinterpret_cast(jdb_handle), + *reinterpret_cast(jropt_handle), jkeys, + jkey_offs, jkey_lens, jcolumn_family_handles); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: multiGet + * Signature: + * (JJ[J[Ljava/nio/ByteBuffer;[I[I[Ljava/nio/ByteBuffer;[I[Lorg/rocksdb/Status;)V + */ +void Java_org_rocksdb_RocksDB_multiGet__JJ_3J_3Ljava_nio_ByteBuffer_2_3I_3I_3Ljava_nio_ByteBuffer_2_3I_3Lorg_rocksdb_Status_2( + JNIEnv* env, jobject jdb, jlong jdb_handle, jlong jropt_handle, + jlongArray jcolumn_family_handles, jobjectArray jkeys, + jintArray jkey_offsets, jintArray jkey_lengths, jobjectArray jvalues, + jintArray jvalues_sizes, jobjectArray jstatus_objects) { + return multi_get_helper_direct( + env, jdb, reinterpret_cast(jdb_handle), + *reinterpret_cast(jropt_handle), + jcolumn_family_handles, jkeys, jkey_offsets, jkey_lengths, jvalues, + jvalues_sizes, jstatus_objects); +} +// private native void +// multiGet(final long dbHandle, final long rOptHandle, +// final long[] columnFamilyHandles, final ByteBuffer[] keysArray, +// final ByteBuffer[] valuesArray); + +////////////////////////////////////////////////////////////////////////////// +// ROCKSDB_NAMESPACE::DB::KeyMayExist +bool key_may_exist_helper(JNIEnv* env, jlong jdb_handle, jlong jcf_handle, + jlong jread_opts_handle, jbyteArray jkey, + jint jkey_offset, jint jkey_len, bool* has_exception, + std::string* value, bool* value_found) { + auto* db = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle; + if (jcf_handle == 0) { + cf_handle = db->DefaultColumnFamily(); + } else { + cf_handle = + reinterpret_cast(jcf_handle); + } + ROCKSDB_NAMESPACE::ReadOptions read_opts = + jread_opts_handle == 0 + ? ROCKSDB_NAMESPACE::ReadOptions() + : *(reinterpret_cast( + jread_opts_handle)); + + jbyte* key = new jbyte[jkey_len]; + env->GetByteArrayRegion(jkey, jkey_offset, jkey_len, key); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] key; + *has_exception = true; + return false; + } + ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), jkey_len); + + const bool exists = + db->KeyMayExist(read_opts, cf_handle, key_slice, value, value_found); + + // cleanup + delete[] key; + + return exists; +} + +bool key_may_exist_direct_helper(JNIEnv* env, jlong jdb_handle, + jlong jcf_handle, jlong jread_opts_handle, + jobject jkey, jint jkey_offset, jint jkey_len, + bool* has_exception, std::string* value, + bool* value_found) { + auto* db = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle; + if (jcf_handle == 0) { + cf_handle = db->DefaultColumnFamily(); + } else { + cf_handle = + reinterpret_cast(jcf_handle); + } + ROCKSDB_NAMESPACE::ReadOptions read_opts = + jread_opts_handle == 0 + ? ROCKSDB_NAMESPACE::ReadOptions() + : *(reinterpret_cast( + jread_opts_handle)); + + char* key = reinterpret_cast(env->GetDirectBufferAddress(jkey)); + if (key == nullptr) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, + "Invalid key argument (argument is not a valid direct ByteBuffer)"); + *has_exception = true; + return false; + } + if (env->GetDirectBufferCapacity(jkey) < (jkey_offset + jkey_len)) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, + "Invalid key argument. Capacity is less than requested region (offset " + "+ length)."); + *has_exception = true; + return false; + } + + ROCKSDB_NAMESPACE::Slice key_slice(key, jkey_len); + + const bool exists = + db->KeyMayExist(read_opts, cf_handle, key_slice, value, value_found); + + return exists; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: keyMayExist + * Signature: (JJJ[BII)Z + */ +jboolean Java_org_rocksdb_RocksDB_keyMayExist( + JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle, + jlong jread_opts_handle, jbyteArray jkey, jint jkey_offset, jint jkey_len) { + bool has_exception = false; + std::string value; + bool value_found = false; + + const bool exists = key_may_exist_helper( + env, jdb_handle, jcf_handle, jread_opts_handle, jkey, jkey_offset, + jkey_len, &has_exception, &value, &value_found); + + if (has_exception) { + // java exception already raised + return false; + } + + return static_cast(exists); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: keyMayExistDirect + * Signature: (JJJLjava/nio/ByteBuffer;II)Z + */ +jboolean Java_org_rocksdb_RocksDB_keyMayExistDirect( + JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle, + jlong jread_opts_handle, jobject jkey, jint jkey_offset, jint jkey_len) { + bool has_exception = false; + std::string value; + bool value_found = false; + + const bool exists = key_may_exist_direct_helper( + env, jdb_handle, jcf_handle, jread_opts_handle, jkey, jkey_offset, + jkey_len, &has_exception, &value, &value_found); + if (has_exception) { + // java exception already raised + return false; + } + + return static_cast(exists); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: keyMayExistDirectFoundValue + * Signature: + * (JJJLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;II)[J + */ +jintArray Java_org_rocksdb_RocksDB_keyMayExistDirectFoundValue( + JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle, + jlong jread_opts_handle, jobject jkey, jint jkey_offset, jint jkey_len, + jobject jval, jint jval_offset, jint jval_len) { + char* val_buffer = reinterpret_cast(env->GetDirectBufferAddress(jval)); + if (val_buffer == nullptr) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, + "Invalid value argument (argument is not a valid direct ByteBuffer)"); + return nullptr; + } + + if (env->GetDirectBufferCapacity(jval) < (jval_offset + jval_len)) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, + "Invalid value argument. Capacity is less than requested region " + "(offset + length)."); + return nullptr; + } + + bool has_exception = false; + std::string cvalue; + bool value_found = false; + + const bool exists = key_may_exist_direct_helper( + env, jdb_handle, jcf_handle, jread_opts_handle, jkey, jkey_offset, + jkey_len, &has_exception, &cvalue, &value_found); + + if (has_exception) { + // java exception already raised + return nullptr; + } + + const jint cvalue_len = static_cast(cvalue.size()); + const jint length = std::min(jval_len, cvalue_len); + memcpy(val_buffer + jval_offset, cvalue.c_str(), length); + + // keep consistent with java KeyMayExistEnum.values() + const int kNotExist = 0; + const int kExistsWithoutValue = 1; + const int kExistsWithValue = 2; + + // TODO fix return value/type + // exists/value_found/neither + // cvalue_len + jintArray jresult = env->NewIntArray(2); + const jint jexists = + exists ? (value_found ? kExistsWithValue : kExistsWithoutValue) + : kNotExist; + + env->SetIntArrayRegion(jresult, 0, 1, &jexists); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jresult); + return nullptr; + } + env->SetIntArrayRegion(jresult, 1, 1, &cvalue_len); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jresult); + return nullptr; + } + + return jresult; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: keyMayExistFoundValue + * Signature: (JJJ[BII)[[B + */ +jobjectArray Java_org_rocksdb_RocksDB_keyMayExistFoundValue( + JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle, + jlong jread_opts_handle, jbyteArray jkey, jint jkey_offset, jint jkey_len) { + bool has_exception = false; + std::string value; + bool value_found = false; + + const bool exists = key_may_exist_helper( + env, jdb_handle, jcf_handle, jread_opts_handle, jkey, jkey_offset, + jkey_len, &has_exception, &value, &value_found); + + if (has_exception) { + // java exception already raised + return nullptr; + } + + jbyte result_flags[1]; + if (!exists) { + result_flags[0] = 0; + } else if (!value_found) { + result_flags[0] = 1; + } else { + // found + result_flags[0] = 2; + } + + jobjectArray jresults = ROCKSDB_NAMESPACE::ByteJni::new2dByteArray(env, 2); + if (jresults == nullptr) { + // exception occurred + return nullptr; + } + + // prepare the result flag + jbyteArray jresult_flags = env->NewByteArray(1); + if (jresult_flags == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + env->SetByteArrayRegion(jresult_flags, 0, 1, result_flags); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jresult_flags); + return nullptr; + } + + env->SetObjectArrayElement(jresults, 0, jresult_flags); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jresult_flags); + return nullptr; + } + + env->DeleteLocalRef(jresult_flags); + + if (result_flags[0] == 2) { + // set the value + const jsize jvalue_len = static_cast(value.size()); + jbyteArray jresult_value = env->NewByteArray(jvalue_len); + if (jresult_value == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + env->SetByteArrayRegion( + jresult_value, 0, jvalue_len, + const_cast(reinterpret_cast(value.data()))); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jresult_value); + return nullptr; + } + env->SetObjectArrayElement(jresults, 1, jresult_value); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jresult_value); + return nullptr; + } + + env->DeleteLocalRef(jresult_value); + } + + return jresults; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: iterator + * Signature: (J)J + */ +jlong Java_org_rocksdb_RocksDB_iterator__J(JNIEnv*, jobject, jlong db_handle) { + auto* db = reinterpret_cast(db_handle); + return rocksdb_iterator_helper(db, ROCKSDB_NAMESPACE::ReadOptions(), nullptr); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: iterator + * Signature: (JJ)J + */ +jlong Java_org_rocksdb_RocksDB_iterator__JJ(JNIEnv*, jobject, jlong db_handle, + jlong jread_options_handle) { + auto* db = reinterpret_cast(db_handle); + auto& read_options = + *reinterpret_cast(jread_options_handle); + return rocksdb_iterator_helper(db, read_options, nullptr); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: iteratorCF + * Signature: (JJ)J + */ +jlong Java_org_rocksdb_RocksDB_iteratorCF__JJ(JNIEnv*, jobject, jlong db_handle, + jlong jcf_handle) { + auto* db = reinterpret_cast(db_handle); + auto* cf_handle = + reinterpret_cast(jcf_handle); + return rocksdb_iterator_helper(db, ROCKSDB_NAMESPACE::ReadOptions(), + cf_handle); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: iteratorCF + * Signature: (JJJ)J + */ +jlong Java_org_rocksdb_RocksDB_iteratorCF__JJJ(JNIEnv*, jobject, + jlong db_handle, + jlong jcf_handle, + jlong jread_options_handle) { + auto* db = reinterpret_cast(db_handle); + auto* cf_handle = + reinterpret_cast(jcf_handle); + auto& read_options = + *reinterpret_cast(jread_options_handle); + return rocksdb_iterator_helper(db, read_options, cf_handle); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: iterators + * Signature: (J[JJ)[J + */ +jlongArray Java_org_rocksdb_RocksDB_iterators(JNIEnv* env, jobject, + jlong db_handle, + jlongArray jcolumn_family_handles, + jlong jread_options_handle) { + auto* db = reinterpret_cast(db_handle); + auto& read_options = + *reinterpret_cast(jread_options_handle); + + std::vector cf_handles; + if (jcolumn_family_handles != nullptr) { + const jsize len_cols = env->GetArrayLength(jcolumn_family_handles); + jlong* jcfh = env->GetLongArrayElements(jcolumn_family_handles, nullptr); + if (jcfh == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + for (jsize i = 0; i < len_cols; i++) { + auto* cf_handle = + reinterpret_cast(jcfh[i]); + cf_handles.push_back(cf_handle); + } + + env->ReleaseLongArrayElements(jcolumn_family_handles, jcfh, JNI_ABORT); + } + + std::vector iterators; + ROCKSDB_NAMESPACE::Status s = + db->NewIterators(read_options, cf_handles, &iterators); + if (s.ok()) { + jlongArray jLongArray = + env->NewLongArray(static_cast(iterators.size())); + if (jLongArray == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + for (std::vector::size_type i = 0; + i < iterators.size(); i++) { + env->SetLongArrayRegion( + jLongArray, static_cast(i), 1, + const_cast(reinterpret_cast(&iterators[i]))); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jLongArray); + return nullptr; + } + } + + return jLongArray; + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return nullptr; + } +} + +/* + * Method: getSnapshot + * Signature: (J)J + */ +jlong Java_org_rocksdb_RocksDB_getSnapshot(JNIEnv*, jobject, jlong db_handle) { + auto* db = reinterpret_cast(db_handle); + const ROCKSDB_NAMESPACE::Snapshot* snapshot = db->GetSnapshot(); + return GET_CPLUSPLUS_POINTER(snapshot); +} + +/* + * Method: releaseSnapshot + * Signature: (JJ)V + */ +void Java_org_rocksdb_RocksDB_releaseSnapshot(JNIEnv*, jobject, jlong db_handle, + jlong snapshot_handle) { + auto* db = reinterpret_cast(db_handle); + auto* snapshot = + reinterpret_cast(snapshot_handle); + db->ReleaseSnapshot(snapshot); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: getProperty + * Signature: (JJLjava/lang/String;I)Ljava/lang/String; + */ +jstring Java_org_rocksdb_RocksDB_getProperty(JNIEnv* env, jobject, + jlong jdb_handle, jlong jcf_handle, + jstring jproperty, + jint jproperty_len) { + const char* property = env->GetStringUTFChars(jproperty, nullptr); + if (property == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + ROCKSDB_NAMESPACE::Slice property_name(property, jproperty_len); + + auto* db = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle; + if (jcf_handle == 0) { + cf_handle = db->DefaultColumnFamily(); + } else { + cf_handle = + reinterpret_cast(jcf_handle); + } + + std::string property_value; + bool retCode = db->GetProperty(cf_handle, property_name, &property_value); + env->ReleaseStringUTFChars(jproperty, property); + + if (retCode) { + return env->NewStringUTF(property_value.c_str()); + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::NotFound()); + return nullptr; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: getMapProperty + * Signature: (JJLjava/lang/String;I)Ljava/util/Map; + */ +jobject Java_org_rocksdb_RocksDB_getMapProperty(JNIEnv* env, jobject, + jlong jdb_handle, + jlong jcf_handle, + jstring jproperty, + jint jproperty_len) { + const char* property = env->GetStringUTFChars(jproperty, nullptr); + if (property == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + ROCKSDB_NAMESPACE::Slice property_name(property, jproperty_len); + + auto* db = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle; + if (jcf_handle == 0) { + cf_handle = db->DefaultColumnFamily(); + } else { + cf_handle = + reinterpret_cast(jcf_handle); + } + + std::map property_value; + bool retCode = db->GetMapProperty(cf_handle, property_name, &property_value); + env->ReleaseStringUTFChars(jproperty, property); + + if (retCode) { + return ROCKSDB_NAMESPACE::HashMapJni::fromCppMap(env, &property_value); + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::NotFound()); + return nullptr; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: getLongProperty + * Signature: (JJLjava/lang/String;I)J + */ +jlong Java_org_rocksdb_RocksDB_getLongProperty(JNIEnv* env, jobject, + jlong jdb_handle, + jlong jcf_handle, + jstring jproperty, + jint jproperty_len) { + const char* property = env->GetStringUTFChars(jproperty, nullptr); + if (property == nullptr) { + // exception thrown: OutOfMemoryError + return 0; + } + ROCKSDB_NAMESPACE::Slice property_name(property, jproperty_len); + + auto* db = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle; + if (jcf_handle == 0) { + cf_handle = db->DefaultColumnFamily(); + } else { + cf_handle = + reinterpret_cast(jcf_handle); + } + + uint64_t property_value; + bool retCode = db->GetIntProperty(cf_handle, property_name, &property_value); + env->ReleaseStringUTFChars(jproperty, property); + + if (retCode) { + return property_value; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::NotFound()); + return 0; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: resetStats + * Signature: (J)V + */ +void Java_org_rocksdb_RocksDB_resetStats(JNIEnv*, jobject, jlong jdb_handle) { + auto* db = reinterpret_cast(jdb_handle); + db->ResetStats(); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: getAggregatedLongProperty + * Signature: (JLjava/lang/String;I)J + */ +jlong Java_org_rocksdb_RocksDB_getAggregatedLongProperty(JNIEnv* env, jobject, + jlong db_handle, + jstring jproperty, + jint jproperty_len) { + const char* property = env->GetStringUTFChars(jproperty, nullptr); + if (property == nullptr) { + return 0; + } + ROCKSDB_NAMESPACE::Slice property_name(property, jproperty_len); + auto* db = reinterpret_cast(db_handle); + uint64_t property_value = 0; + bool retCode = db->GetAggregatedIntProperty(property_name, &property_value); + env->ReleaseStringUTFChars(jproperty, property); + + if (retCode) { + return property_value; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::NotFound()); + return 0; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: getApproximateSizes + * Signature: (JJ[JB)[J + */ +jlongArray Java_org_rocksdb_RocksDB_getApproximateSizes( + JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle, + jlongArray jrange_slice_handles, jbyte jinclude_flags) { + const jsize jlen = env->GetArrayLength(jrange_slice_handles); + const size_t range_count = jlen / 2; + + jlong* jranges = env->GetLongArrayElements(jrange_slice_handles, nullptr); + if (jranges == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + auto ranges = std::unique_ptr( + new ROCKSDB_NAMESPACE::Range[range_count]); + size_t range_offset = 0; + for (jsize i = 0; i < jlen; ++i) { + auto* start = reinterpret_cast(jranges[i]); + auto* limit = reinterpret_cast(jranges[++i]); + ranges.get()[range_offset++] = ROCKSDB_NAMESPACE::Range(*start, *limit); + } + + auto* db = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle; + if (jcf_handle == 0) { + cf_handle = db->DefaultColumnFamily(); + } else { + cf_handle = + reinterpret_cast(jcf_handle); + } + + auto sizes = std::unique_ptr(new uint64_t[range_count]); + + ROCKSDB_NAMESPACE::DB::SizeApproximationFlags include_flags = + ROCKSDB_NAMESPACE::DB::SizeApproximationFlags::NONE; + if (jinclude_flags & 1) { + include_flags = + ROCKSDB_NAMESPACE::DB::SizeApproximationFlags::INCLUDE_MEMTABLES; + } + if (jinclude_flags & 2) { + include_flags = + (include_flags | + ROCKSDB_NAMESPACE::DB::SizeApproximationFlags::INCLUDE_FILES); + } + + db->GetApproximateSizes(cf_handle, ranges.get(), + static_cast(range_count), sizes.get(), + include_flags); + + // release LongArrayElements + env->ReleaseLongArrayElements(jrange_slice_handles, jranges, JNI_ABORT); + + // prepare results + auto results = std::unique_ptr(new jlong[range_count]); + for (size_t i = 0; i < range_count; ++i) { + results.get()[i] = static_cast(sizes.get()[i]); + } + + const jsize jrange_count = jlen / 2; + jlongArray jresults = env->NewLongArray(jrange_count); + if (jresults == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + env->SetLongArrayRegion(jresults, 0, jrange_count, results.get()); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jresults); + return nullptr; + } + + return jresults; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: getApproximateMemTableStats + * Signature: (JJJJ)[J + */ +jlongArray Java_org_rocksdb_RocksDB_getApproximateMemTableStats( + JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle, + jlong jstartHandle, jlong jlimitHandle) { + auto* start = reinterpret_cast(jstartHandle); + auto* limit = reinterpret_cast(jlimitHandle); + const ROCKSDB_NAMESPACE::Range range(*start, *limit); + + auto* db = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle; + if (jcf_handle == 0) { + cf_handle = db->DefaultColumnFamily(); + } else { + cf_handle = + reinterpret_cast(jcf_handle); + } + + uint64_t count = 0; + uint64_t sizes = 0; + db->GetApproximateMemTableStats(cf_handle, range, &count, &sizes); + + // prepare results + jlong results[2] = {static_cast(count), static_cast(sizes)}; + + jlongArray jsizes = env->NewLongArray(2); + if (jsizes == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + env->SetLongArrayRegion(jsizes, 0, 2, results); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jsizes); + return nullptr; + } + + return jsizes; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: compactRange + * Signature: (J[BI[BIJJ)V + */ +void Java_org_rocksdb_RocksDB_compactRange(JNIEnv* env, jobject, + jlong jdb_handle, jbyteArray jbegin, + jint jbegin_len, jbyteArray jend, + jint jend_len, + jlong jcompact_range_opts_handle, + jlong jcf_handle) { + jboolean has_exception = JNI_FALSE; + + std::string str_begin; + if (jbegin_len > 0) { + str_begin = ROCKSDB_NAMESPACE::JniUtil::byteString( + env, jbegin, jbegin_len, + [](const char* str, const size_t len) { return std::string(str, len); }, + &has_exception); + if (has_exception == JNI_TRUE) { + // exception occurred + return; + } + } + + std::string str_end; + if (jend_len > 0) { + str_end = ROCKSDB_NAMESPACE::JniUtil::byteString( + env, jend, jend_len, + [](const char* str, const size_t len) { return std::string(str, len); }, + &has_exception); + if (has_exception == JNI_TRUE) { + // exception occurred + return; + } + } + + ROCKSDB_NAMESPACE::CompactRangeOptions* compact_range_opts = nullptr; + if (jcompact_range_opts_handle == 0) { + // NOTE: we DO own the pointer! + compact_range_opts = new ROCKSDB_NAMESPACE::CompactRangeOptions(); + } else { + // NOTE: we do NOT own the pointer! + compact_range_opts = + reinterpret_cast( + jcompact_range_opts_handle); + } + + auto* db = reinterpret_cast(jdb_handle); + + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle; + if (jcf_handle == 0) { + cf_handle = db->DefaultColumnFamily(); + } else { + cf_handle = + reinterpret_cast(jcf_handle); + } + + ROCKSDB_NAMESPACE::Status s; + if (jbegin_len > 0 || jend_len > 0) { + const ROCKSDB_NAMESPACE::Slice begin(str_begin); + const ROCKSDB_NAMESPACE::Slice end(str_end); + s = db->CompactRange(*compact_range_opts, cf_handle, &begin, &end); + } else { + s = db->CompactRange(*compact_range_opts, cf_handle, nullptr, nullptr); + } + + if (jcompact_range_opts_handle == 0) { + delete compact_range_opts; + } + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: setOptions + * Signature: (JJ[Ljava/lang/String;[Ljava/lang/String;)V + */ +void Java_org_rocksdb_RocksDB_setOptions(JNIEnv* env, jobject, jlong jdb_handle, + jlong jcf_handle, jobjectArray jkeys, + jobjectArray jvalues) { + const jsize len = env->GetArrayLength(jkeys); + assert(len == env->GetArrayLength(jvalues)); + + std::unordered_map options_map; + for (jsize i = 0; i < len; i++) { + jobject jobj_key = env->GetObjectArrayElement(jkeys, i); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + return; + } + + jobject jobj_value = env->GetObjectArrayElement(jvalues, i); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jobj_key); + return; + } + + jboolean has_exception = JNI_FALSE; + std::string s_key = ROCKSDB_NAMESPACE::JniUtil::copyStdString( + env, reinterpret_cast(jobj_key), &has_exception); + if (has_exception == JNI_TRUE) { + // exception occurred + env->DeleteLocalRef(jobj_value); + env->DeleteLocalRef(jobj_key); + return; + } + + std::string s_value = ROCKSDB_NAMESPACE::JniUtil::copyStdString( + env, reinterpret_cast(jobj_value), &has_exception); + if (has_exception == JNI_TRUE) { + // exception occurred + env->DeleteLocalRef(jobj_value); + env->DeleteLocalRef(jobj_key); + return; + } + + options_map[s_key] = s_value; + + env->DeleteLocalRef(jobj_key); + env->DeleteLocalRef(jobj_value); + } + + auto* db = reinterpret_cast(jdb_handle); + auto* cf_handle = + reinterpret_cast(jcf_handle); + if (cf_handle == nullptr) { + cf_handle = db->DefaultColumnFamily(); + } + auto s = db->SetOptions(cf_handle, options_map); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: setDBOptions + * Signature: (J[Ljava/lang/String;[Ljava/lang/String;)V + */ +void Java_org_rocksdb_RocksDB_setDBOptions(JNIEnv* env, jobject, + jlong jdb_handle, jobjectArray jkeys, + jobjectArray jvalues) { + const jsize len = env->GetArrayLength(jkeys); + assert(len == env->GetArrayLength(jvalues)); + + std::unordered_map options_map; + for (jsize i = 0; i < len; i++) { + jobject jobj_key = env->GetObjectArrayElement(jkeys, i); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + return; + } + + jobject jobj_value = env->GetObjectArrayElement(jvalues, i); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jobj_key); + return; + } + + jboolean has_exception = JNI_FALSE; + std::string s_key = ROCKSDB_NAMESPACE::JniUtil::copyStdString( + env, reinterpret_cast(jobj_key), &has_exception); + if (has_exception == JNI_TRUE) { + // exception occurred + env->DeleteLocalRef(jobj_value); + env->DeleteLocalRef(jobj_key); + return; + } + + std::string s_value = ROCKSDB_NAMESPACE::JniUtil::copyStdString( + env, reinterpret_cast(jobj_value), &has_exception); + if (has_exception == JNI_TRUE) { + // exception occurred + env->DeleteLocalRef(jobj_value); + env->DeleteLocalRef(jobj_key); + return; + } + + options_map[s_key] = s_value; + + env->DeleteLocalRef(jobj_key); + env->DeleteLocalRef(jobj_value); + } + + auto* db = reinterpret_cast(jdb_handle); + auto s = db->SetDBOptions(options_map); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: getOptions + * Signature: (JJ)Ljava/lang/String; + */ +jstring Java_org_rocksdb_RocksDB_getOptions(JNIEnv* env, jobject, + jlong jdb_handle, + jlong jcf_handle) { + auto* db = reinterpret_cast(jdb_handle); + + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle; + if (jcf_handle == 0) { + cf_handle = db->DefaultColumnFamily(); + } else { + cf_handle = + reinterpret_cast(jcf_handle); + } + + auto options = db->GetOptions(cf_handle); + std::string options_as_string; + ROCKSDB_NAMESPACE::Status s = + GetStringFromColumnFamilyOptions(&options_as_string, options); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return nullptr; + } + return env->NewStringUTF(options_as_string.c_str()); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: getDBOptions + * Signature: (J)Ljava/lang/String; + */ +jstring Java_org_rocksdb_RocksDB_getDBOptions(JNIEnv* env, jobject, + jlong jdb_handle) { + auto* db = reinterpret_cast(jdb_handle); + + auto options = db->GetDBOptions(); + std::string options_as_string; + ROCKSDB_NAMESPACE::Status s = + GetStringFromDBOptions(&options_as_string, options); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return nullptr; + } + return env->NewStringUTF(options_as_string.c_str()); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: compactFiles + * Signature: (JJJ[Ljava/lang/String;IIJ)[Ljava/lang/String; + */ +jobjectArray Java_org_rocksdb_RocksDB_compactFiles( + JNIEnv* env, jobject, jlong jdb_handle, jlong jcompaction_opts_handle, + jlong jcf_handle, jobjectArray jinput_file_names, jint joutput_level, + jint joutput_path_id, jlong jcompaction_job_info_handle) { + jboolean has_exception = JNI_FALSE; + const std::vector input_file_names = + ROCKSDB_NAMESPACE::JniUtil::copyStrings(env, jinput_file_names, + &has_exception); + if (has_exception == JNI_TRUE) { + // exception occurred + return nullptr; + } + + auto* compaction_opts = + reinterpret_cast( + jcompaction_opts_handle); + auto* db = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle; + if (jcf_handle == 0) { + cf_handle = db->DefaultColumnFamily(); + } else { + cf_handle = + reinterpret_cast(jcf_handle); + } + + ROCKSDB_NAMESPACE::CompactionJobInfo* compaction_job_info = nullptr; + if (jcompaction_job_info_handle != 0) { + compaction_job_info = + reinterpret_cast( + jcompaction_job_info_handle); + } + + std::vector output_file_names; + auto s = db->CompactFiles(*compaction_opts, cf_handle, input_file_names, + static_cast(joutput_level), + static_cast(joutput_path_id), + &output_file_names, compaction_job_info); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return nullptr; + } + + return ROCKSDB_NAMESPACE::JniUtil::toJavaStrings(env, &output_file_names); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: cancelAllBackgroundWork + * Signature: (JZ)V + */ +void Java_org_rocksdb_RocksDB_cancelAllBackgroundWork(JNIEnv*, jobject, + jlong jdb_handle, + jboolean jwait) { + auto* db = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::CancelAllBackgroundWork(db, jwait); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: pauseBackgroundWork + * Signature: (J)V + */ +void Java_org_rocksdb_RocksDB_pauseBackgroundWork(JNIEnv* env, jobject, + jlong jdb_handle) { + auto* db = reinterpret_cast(jdb_handle); + auto s = db->PauseBackgroundWork(); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: continueBackgroundWork + * Signature: (J)V + */ +void Java_org_rocksdb_RocksDB_continueBackgroundWork(JNIEnv* env, jobject, + jlong jdb_handle) { + auto* db = reinterpret_cast(jdb_handle); + auto s = db->ContinueBackgroundWork(); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: enableAutoCompaction + * Signature: (J[J)V + */ +void Java_org_rocksdb_RocksDB_enableAutoCompaction(JNIEnv* env, jobject, + jlong jdb_handle, + jlongArray jcf_handles) { + auto* db = reinterpret_cast(jdb_handle); + jboolean has_exception = JNI_FALSE; + const std::vector cf_handles = + ROCKSDB_NAMESPACE::JniUtil::fromJPointers< + ROCKSDB_NAMESPACE::ColumnFamilyHandle>(env, jcf_handles, + &has_exception); + if (has_exception == JNI_TRUE) { + // exception occurred + return; + } + db->EnableAutoCompaction(cf_handles); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: numberLevels + * Signature: (JJ)I + */ +jint Java_org_rocksdb_RocksDB_numberLevels(JNIEnv*, jobject, jlong jdb_handle, + jlong jcf_handle) { + auto* db = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle; + if (jcf_handle == 0) { + cf_handle = db->DefaultColumnFamily(); + } else { + cf_handle = + reinterpret_cast(jcf_handle); + } + return static_cast(db->NumberLevels(cf_handle)); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: maxMemCompactionLevel + * Signature: (JJ)I + */ +jint Java_org_rocksdb_RocksDB_maxMemCompactionLevel(JNIEnv*, jobject, + jlong jdb_handle, + jlong jcf_handle) { + auto* db = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle; + if (jcf_handle == 0) { + cf_handle = db->DefaultColumnFamily(); + } else { + cf_handle = + reinterpret_cast(jcf_handle); + } + return static_cast(db->MaxMemCompactionLevel(cf_handle)); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: level0StopWriteTrigger + * Signature: (JJ)I + */ +jint Java_org_rocksdb_RocksDB_level0StopWriteTrigger(JNIEnv*, jobject, + jlong jdb_handle, + jlong jcf_handle) { + auto* db = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle; + if (jcf_handle == 0) { + cf_handle = db->DefaultColumnFamily(); + } else { + cf_handle = + reinterpret_cast(jcf_handle); + } + return static_cast(db->Level0StopWriteTrigger(cf_handle)); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: getName + * Signature: (J)Ljava/lang/String; + */ +jstring Java_org_rocksdb_RocksDB_getName(JNIEnv* env, jobject, + jlong jdb_handle) { + auto* db = reinterpret_cast(jdb_handle); + std::string name = db->GetName(); + return ROCKSDB_NAMESPACE::JniUtil::toJavaString(env, &name, false); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: getEnv + * Signature: (J)J + */ +jlong Java_org_rocksdb_RocksDB_getEnv(JNIEnv*, jobject, jlong jdb_handle) { + auto* db = reinterpret_cast(jdb_handle); + return GET_CPLUSPLUS_POINTER(db->GetEnv()); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: flush + * Signature: (JJ[J)V + */ +void Java_org_rocksdb_RocksDB_flush(JNIEnv* env, jobject, jlong jdb_handle, + jlong jflush_opts_handle, + jlongArray jcf_handles) { + auto* db = reinterpret_cast(jdb_handle); + auto* flush_opts = + reinterpret_cast(jflush_opts_handle); + std::vector cf_handles; + if (jcf_handles == nullptr) { + cf_handles.push_back(db->DefaultColumnFamily()); + } else { + jboolean has_exception = JNI_FALSE; + cf_handles = ROCKSDB_NAMESPACE::JniUtil::fromJPointers< + ROCKSDB_NAMESPACE::ColumnFamilyHandle>(env, jcf_handles, + &has_exception); + if (has_exception) { + // exception occurred + return; + } + } + auto s = db->Flush(*flush_opts, cf_handles); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: flushWal + * Signature: (JZ)V + */ +void Java_org_rocksdb_RocksDB_flushWal(JNIEnv* env, jobject, jlong jdb_handle, + jboolean jsync) { + auto* db = reinterpret_cast(jdb_handle); + auto s = db->FlushWAL(jsync == JNI_TRUE); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: syncWal + * Signature: (J)V + */ +void Java_org_rocksdb_RocksDB_syncWal(JNIEnv* env, jobject, jlong jdb_handle) { + auto* db = reinterpret_cast(jdb_handle); + auto s = db->SyncWAL(); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: getLatestSequenceNumber + * Signature: (J)V + */ +jlong Java_org_rocksdb_RocksDB_getLatestSequenceNumber(JNIEnv*, jobject, + jlong jdb_handle) { + auto* db = reinterpret_cast(jdb_handle); + return db->GetLatestSequenceNumber(); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: disableFileDeletions + * Signature: (J)V + */ +void Java_org_rocksdb_RocksDB_disableFileDeletions(JNIEnv* env, jobject, + jlong jdb_handle) { + auto* db = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::Status s = db->DisableFileDeletions(); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: enableFileDeletions + * Signature: (JZ)V + */ +void Java_org_rocksdb_RocksDB_enableFileDeletions(JNIEnv* env, jobject, + jlong jdb_handle, + jboolean jforce) { + auto* db = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::Status s = db->EnableFileDeletions(jforce); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: getLiveFiles + * Signature: (JZ)[Ljava/lang/String; + */ +jobjectArray Java_org_rocksdb_RocksDB_getLiveFiles(JNIEnv* env, jobject, + jlong jdb_handle, + jboolean jflush_memtable) { + auto* db = reinterpret_cast(jdb_handle); + std::vector live_files; + uint64_t manifest_file_size = 0; + auto s = db->GetLiveFiles(live_files, &manifest_file_size, + jflush_memtable == JNI_TRUE); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return nullptr; + } + + // append the manifest_file_size to the vector + // for passing back to java + live_files.push_back(std::to_string(manifest_file_size)); + + return ROCKSDB_NAMESPACE::JniUtil::toJavaStrings(env, &live_files); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: getSortedWalFiles + * Signature: (J)[Lorg/rocksdb/LogFile; + */ +jobjectArray Java_org_rocksdb_RocksDB_getSortedWalFiles(JNIEnv* env, jobject, + jlong jdb_handle) { + auto* db = reinterpret_cast(jdb_handle); + std::vector> sorted_wal_files; + auto s = db->GetSortedWalFiles(sorted_wal_files); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return nullptr; + } + + // convert to Java type + const jsize jlen = static_cast(sorted_wal_files.size()); + jobjectArray jsorted_wal_files = env->NewObjectArray( + jlen, ROCKSDB_NAMESPACE::LogFileJni::getJClass(env), nullptr); + if (jsorted_wal_files == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + jsize i = 0; + for (auto it = sorted_wal_files.begin(); it != sorted_wal_files.end(); ++it) { + jobject jlog_file = + ROCKSDB_NAMESPACE::LogFileJni::fromCppLogFile(env, it->get()); + if (jlog_file == nullptr) { + // exception occurred + env->DeleteLocalRef(jsorted_wal_files); + return nullptr; + } + + env->SetObjectArrayElement(jsorted_wal_files, i++, jlog_file); + if (env->ExceptionCheck()) { + // exception occurred + env->DeleteLocalRef(jlog_file); + env->DeleteLocalRef(jsorted_wal_files); + return nullptr; + } + + env->DeleteLocalRef(jlog_file); + } + + return jsorted_wal_files; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: getUpdatesSince + * Signature: (JJ)J + */ +jlong Java_org_rocksdb_RocksDB_getUpdatesSince(JNIEnv* env, jobject, + jlong jdb_handle, + jlong jsequence_number) { + auto* db = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::SequenceNumber sequence_number = + static_cast(jsequence_number); + std::unique_ptr iter; + ROCKSDB_NAMESPACE::Status s = db->GetUpdatesSince(sequence_number, &iter); + if (s.ok()) { + return GET_CPLUSPLUS_POINTER(iter.release()); + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return 0; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: deleteFile + * Signature: (JLjava/lang/String;)V + */ +void Java_org_rocksdb_RocksDB_deleteFile(JNIEnv* env, jobject, jlong jdb_handle, + jstring jname) { + auto* db = reinterpret_cast(jdb_handle); + jboolean has_exception = JNI_FALSE; + std::string name = + ROCKSDB_NAMESPACE::JniUtil::copyStdString(env, jname, &has_exception); + if (has_exception == JNI_TRUE) { + // exception occurred + return; + } + db->DeleteFile(name); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: getLiveFilesMetaData + * Signature: (J)[Lorg/rocksdb/LiveFileMetaData; + */ +jobjectArray Java_org_rocksdb_RocksDB_getLiveFilesMetaData(JNIEnv* env, jobject, + jlong jdb_handle) { + auto* db = reinterpret_cast(jdb_handle); + std::vector live_files_meta_data; + db->GetLiveFilesMetaData(&live_files_meta_data); + + // convert to Java type + const jsize jlen = static_cast(live_files_meta_data.size()); + jobjectArray jlive_files_meta_data = env->NewObjectArray( + jlen, ROCKSDB_NAMESPACE::LiveFileMetaDataJni::getJClass(env), nullptr); + if (jlive_files_meta_data == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + jsize i = 0; + for (auto it = live_files_meta_data.begin(); it != live_files_meta_data.end(); + ++it) { + jobject jlive_file_meta_data = + ROCKSDB_NAMESPACE::LiveFileMetaDataJni::fromCppLiveFileMetaData(env, + &(*it)); + if (jlive_file_meta_data == nullptr) { + // exception occurred + env->DeleteLocalRef(jlive_files_meta_data); + return nullptr; + } + + env->SetObjectArrayElement(jlive_files_meta_data, i++, + jlive_file_meta_data); + if (env->ExceptionCheck()) { + // exception occurred + env->DeleteLocalRef(jlive_file_meta_data); + env->DeleteLocalRef(jlive_files_meta_data); + return nullptr; + } + + env->DeleteLocalRef(jlive_file_meta_data); + } + + return jlive_files_meta_data; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: getColumnFamilyMetaData + * Signature: (JJ)Lorg/rocksdb/ColumnFamilyMetaData; + */ +jobject Java_org_rocksdb_RocksDB_getColumnFamilyMetaData(JNIEnv* env, jobject, + jlong jdb_handle, + jlong jcf_handle) { + auto* db = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle; + if (jcf_handle == 0) { + cf_handle = db->DefaultColumnFamily(); + } else { + cf_handle = + reinterpret_cast(jcf_handle); + } + ROCKSDB_NAMESPACE::ColumnFamilyMetaData cf_metadata; + db->GetColumnFamilyMetaData(cf_handle, &cf_metadata); + return ROCKSDB_NAMESPACE::ColumnFamilyMetaDataJni:: + fromCppColumnFamilyMetaData(env, &cf_metadata); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: ingestExternalFile + * Signature: (JJ[Ljava/lang/String;IJ)V + */ +void Java_org_rocksdb_RocksDB_ingestExternalFile( + JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle, + jobjectArray jfile_path_list, jint jfile_path_list_len, + jlong jingest_external_file_options_handle) { + jboolean has_exception = JNI_FALSE; + std::vector file_path_list = + ROCKSDB_NAMESPACE::JniUtil::copyStrings( + env, jfile_path_list, jfile_path_list_len, &has_exception); + if (has_exception == JNI_TRUE) { + // exception occurred + return; + } + + auto* db = reinterpret_cast(jdb_handle); + auto* column_family = + reinterpret_cast(jcf_handle); + auto* ifo = reinterpret_cast( + jingest_external_file_options_handle); + ROCKSDB_NAMESPACE::Status s = + db->IngestExternalFile(column_family, file_path_list, *ifo); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: verifyChecksum + * Signature: (J)V + */ +void Java_org_rocksdb_RocksDB_verifyChecksum(JNIEnv* env, jobject, + jlong jdb_handle) { + auto* db = reinterpret_cast(jdb_handle); + auto s = db->VerifyChecksum(); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: getDefaultColumnFamily + * Signature: (J)J + */ +jlong Java_org_rocksdb_RocksDB_getDefaultColumnFamily(JNIEnv*, jobject, + jlong jdb_handle) { + auto* db_handle = reinterpret_cast(jdb_handle); + auto* cf_handle = db_handle->DefaultColumnFamily(); + return GET_CPLUSPLUS_POINTER(cf_handle); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: getPropertiesOfAllTables + * Signature: (JJ)Ljava/util/Map; + */ +jobject Java_org_rocksdb_RocksDB_getPropertiesOfAllTables(JNIEnv* env, jobject, + jlong jdb_handle, + jlong jcf_handle) { + auto* db = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle; + if (jcf_handle == 0) { + cf_handle = db->DefaultColumnFamily(); + } else { + cf_handle = + reinterpret_cast(jcf_handle); + } + ROCKSDB_NAMESPACE::TablePropertiesCollection table_properties_collection; + auto s = + db->GetPropertiesOfAllTables(cf_handle, &table_properties_collection); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } + + // convert to Java type + jobject jhash_map = ROCKSDB_NAMESPACE::HashMapJni::construct( + env, static_cast(table_properties_collection.size())); + if (jhash_map == nullptr) { + // exception occurred + return nullptr; + } + + const ROCKSDB_NAMESPACE::HashMapJni::FnMapKV< + const std::string, + const std::shared_ptr, jobject, + jobject> + fn_map_kv = + [env](const std::pair>& + kv) { + jstring jkey = ROCKSDB_NAMESPACE::JniUtil::toJavaString( + env, &(kv.first), false); + if (env->ExceptionCheck()) { + // an error occurred + return std::unique_ptr>(nullptr); + } + + jobject jtable_properties = + ROCKSDB_NAMESPACE::TablePropertiesJni::fromCppTableProperties( + env, *(kv.second.get())); + if (jtable_properties == nullptr) { + // an error occurred + env->DeleteLocalRef(jkey); + return std::unique_ptr>(nullptr); + } + + return std::unique_ptr>( + new std::pair( + static_cast(jkey), + static_cast(jtable_properties))); + }; + + if (!ROCKSDB_NAMESPACE::HashMapJni::putAll( + env, jhash_map, table_properties_collection.begin(), + table_properties_collection.end(), fn_map_kv)) { + // exception occurred + return nullptr; + } + + return jhash_map; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: getPropertiesOfTablesInRange + * Signature: (JJ[J)Ljava/util/Map; + */ +jobject Java_org_rocksdb_RocksDB_getPropertiesOfTablesInRange( + JNIEnv* env, jobject, jlong jdb_handle, jlong jcf_handle, + jlongArray jrange_slice_handles) { + auto* db = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle; + if (jcf_handle == 0) { + cf_handle = db->DefaultColumnFamily(); + } else { + cf_handle = + reinterpret_cast(jcf_handle); + } + const jsize jlen = env->GetArrayLength(jrange_slice_handles); + jlong* jrange_slice_handle = + env->GetLongArrayElements(jrange_slice_handles, nullptr); + if (jrange_slice_handle == nullptr) { + // exception occurred + return nullptr; + } + + const size_t ranges_len = static_cast(jlen / 2); + auto ranges = std::unique_ptr( + new ROCKSDB_NAMESPACE::Range[ranges_len]); + for (jsize i = 0, j = 0; i < jlen; ++i) { + auto* start = + reinterpret_cast(jrange_slice_handle[i]); + auto* limit = + reinterpret_cast(jrange_slice_handle[++i]); + ranges[j++] = ROCKSDB_NAMESPACE::Range(*start, *limit); + } + + ROCKSDB_NAMESPACE::TablePropertiesCollection table_properties_collection; + auto s = db->GetPropertiesOfTablesInRange(cf_handle, ranges.get(), ranges_len, + &table_properties_collection); + if (!s.ok()) { + // error occurred + env->ReleaseLongArrayElements(jrange_slice_handles, jrange_slice_handle, + JNI_ABORT); + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return nullptr; + } + + // cleanup + env->ReleaseLongArrayElements(jrange_slice_handles, jrange_slice_handle, + JNI_ABORT); + + return jrange_slice_handles; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: suggestCompactRange + * Signature: (JJ)[J + */ +jlongArray Java_org_rocksdb_RocksDB_suggestCompactRange(JNIEnv* env, jobject, + jlong jdb_handle, + jlong jcf_handle) { + auto* db = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle; + if (jcf_handle == 0) { + cf_handle = db->DefaultColumnFamily(); + } else { + cf_handle = + reinterpret_cast(jcf_handle); + } + auto* begin = new ROCKSDB_NAMESPACE::Slice(); + auto* end = new ROCKSDB_NAMESPACE::Slice(); + auto s = db->SuggestCompactRange(cf_handle, begin, end); + if (!s.ok()) { + // error occurred + delete begin; + delete end; + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return nullptr; + } + + jlongArray jslice_handles = env->NewLongArray(2); + if (jslice_handles == nullptr) { + // exception thrown: OutOfMemoryError + delete begin; + delete end; + return nullptr; + } + + jlong slice_handles[2]; + slice_handles[0] = GET_CPLUSPLUS_POINTER(begin); + slice_handles[1] = GET_CPLUSPLUS_POINTER(end); + env->SetLongArrayRegion(jslice_handles, 0, 2, slice_handles); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete begin; + delete end; + env->DeleteLocalRef(jslice_handles); + return nullptr; + } + + return jslice_handles; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: promoteL0 + * Signature: (JJI)V + */ +void Java_org_rocksdb_RocksDB_promoteL0(JNIEnv*, jobject, jlong jdb_handle, + jlong jcf_handle, jint jtarget_level) { + auto* db = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle; + if (jcf_handle == 0) { + cf_handle = db->DefaultColumnFamily(); + } else { + cf_handle = + reinterpret_cast(jcf_handle); + } + db->PromoteL0(cf_handle, static_cast(jtarget_level)); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: startTrace + * Signature: (JJJ)V + */ +void Java_org_rocksdb_RocksDB_startTrace( + JNIEnv* env, jobject, jlong jdb_handle, jlong jmax_trace_file_size, + jlong jtrace_writer_jnicallback_handle) { + auto* db = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::TraceOptions trace_options; + trace_options.max_trace_file_size = + static_cast(jmax_trace_file_size); + // transfer ownership of trace writer from Java to C++ + auto trace_writer = + std::unique_ptr( + reinterpret_cast( + jtrace_writer_jnicallback_handle)); + auto s = db->StartTrace(trace_options, std::move(trace_writer)); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: endTrace + * Signature: (J)V + */ +void Java_org_rocksdb_RocksDB_endTrace(JNIEnv* env, jobject, jlong jdb_handle) { + auto* db = reinterpret_cast(jdb_handle); + auto s = db->EndTrace(); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: tryCatchUpWithPrimary + * Signature: (J)V + */ +void Java_org_rocksdb_RocksDB_tryCatchUpWithPrimary(JNIEnv* env, jobject, + jlong jdb_handle) { + auto* db = reinterpret_cast(jdb_handle); + auto s = db->TryCatchUpWithPrimary(); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: destroyDB + * Signature: (Ljava/lang/String;J)V + */ +void Java_org_rocksdb_RocksDB_destroyDB(JNIEnv* env, jclass, jstring jdb_path, + jlong joptions_handle) { + const char* db_path = env->GetStringUTFChars(jdb_path, nullptr); + if (db_path == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + auto* options = + reinterpret_cast(joptions_handle); + if (options == nullptr) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::InvalidArgument("Invalid Options.")); + } + + ROCKSDB_NAMESPACE::Status s = ROCKSDB_NAMESPACE::DestroyDB(db_path, *options); + env->ReleaseStringUTFChars(jdb_path, db_path); + + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +bool get_slice_helper(JNIEnv* env, jobjectArray ranges, jsize index, + std::unique_ptr& slice, + std::vector>& ranges_to_free) { + jobject jArray = env->GetObjectArrayElement(ranges, index); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + return false; + } + + if (jArray == nullptr) { + return true; + } + + jbyteArray jba = reinterpret_cast(jArray); + jsize len_ba = env->GetArrayLength(jba); + ranges_to_free.push_back(std::unique_ptr(new jbyte[len_ba])); + env->GetByteArrayRegion(jba, 0, len_ba, ranges_to_free.back().get()); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jArray); + return false; + } + env->DeleteLocalRef(jArray); + slice.reset(new ROCKSDB_NAMESPACE::Slice( + reinterpret_cast(ranges_to_free.back().get()), len_ba)); + return true; +} +/* + * Class: org_rocksdb_RocksDB + * Method: deleteFilesInRanges + * Signature: (JJLjava/util/List;Z)V + */ +void Java_org_rocksdb_RocksDB_deleteFilesInRanges(JNIEnv* env, jobject /*jdb*/, + jlong jdb_handle, + jlong jcf_handle, + jobjectArray ranges, + jboolean include_end) { + jsize length = env->GetArrayLength(ranges); + + std::vector rangesVector; + std::vector> slices; + std::vector> ranges_to_free; + for (jsize i = 0; (i + 1) < length; i += 2) { + slices.push_back(std::unique_ptr()); + if (!get_slice_helper(env, ranges, i, slices.back(), ranges_to_free)) { + // exception thrown + return; + } + + slices.push_back(std::unique_ptr()); + if (!get_slice_helper(env, ranges, i + 1, slices.back(), ranges_to_free)) { + // exception thrown + return; + } + + rangesVector.push_back(ROCKSDB_NAMESPACE::RangePtr( + slices[slices.size() - 2].get(), slices[slices.size() - 1].get())); + } + + auto* db = reinterpret_cast(jdb_handle); + auto* column_family = + reinterpret_cast(jcf_handle); + + ROCKSDB_NAMESPACE::Status s = ROCKSDB_NAMESPACE::DeleteFilesInRanges( + db, column_family == nullptr ? db->DefaultColumnFamily() : column_family, + rangesVector.data(), rangesVector.size(), include_end); + + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RocksDB + * Method: version + * Signature: ()I + */ +jint Java_org_rocksdb_RocksDB_version(JNIEnv*, jclass) { + uint32_t encodedVersion = (ROCKSDB_MAJOR & 0xff) << 16; + encodedVersion |= (ROCKSDB_MINOR & 0xff) << 8; + encodedVersion |= (ROCKSDB_PATCH & 0xff); + return static_cast(encodedVersion); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/slice.cc b/librocksdb-sys/rocksdb/java/rocksjni/slice.cc new file mode 100644 index 0000000..63c6b1b --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/slice.cc @@ -0,0 +1,374 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::Slice. + +#include "rocksdb/slice.h" + +#include +#include +#include + +#include + +#include "include/org_rocksdb_AbstractSlice.h" +#include "include/org_rocksdb_DirectSlice.h" +#include "include/org_rocksdb_Slice.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +// + +/* + * Class: org_rocksdb_Slice + * Method: createNewSlice0 + * Signature: ([BI)J + */ +jlong Java_org_rocksdb_Slice_createNewSlice0(JNIEnv* env, jclass /*jcls*/, + jbyteArray data, jint offset) { + const jsize dataSize = env->GetArrayLength(data); + const int len = dataSize - offset; + + // NOTE: buf will be deleted in the Java_org_rocksdb_Slice_disposeInternalBuf + // method + jbyte* buf = new jbyte[len]; + env->GetByteArrayRegion(data, offset, len, buf); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + return 0; + } + + const auto* slice = new ROCKSDB_NAMESPACE::Slice((const char*)buf, len); + return GET_CPLUSPLUS_POINTER(slice); +} + +/* + * Class: org_rocksdb_Slice + * Method: createNewSlice1 + * Signature: ([B)J + */ +jlong Java_org_rocksdb_Slice_createNewSlice1(JNIEnv* env, jclass /*jcls*/, + jbyteArray data) { + jbyte* ptrData = env->GetByteArrayElements(data, nullptr); + if (ptrData == nullptr) { + // exception thrown: OutOfMemoryError + return 0; + } + const int len = env->GetArrayLength(data) + 1; + + // NOTE: buf will be deleted in the Java_org_rocksdb_Slice_disposeInternalBuf + // method + char* buf = new char[len]; + memcpy(buf, ptrData, len - 1); + buf[len - 1] = '\0'; + + const auto* slice = new ROCKSDB_NAMESPACE::Slice(buf, len - 1); + + env->ReleaseByteArrayElements(data, ptrData, JNI_ABORT); + + return GET_CPLUSPLUS_POINTER(slice); +} + +/* + * Class: org_rocksdb_Slice + * Method: data0 + * Signature: (J)[B + */ +jbyteArray Java_org_rocksdb_Slice_data0(JNIEnv* env, jobject /*jobj*/, + jlong handle) { + const auto* slice = reinterpret_cast(handle); + const jsize len = static_cast(slice->size()); + const jbyteArray data = env->NewByteArray(len); + if (data == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + env->SetByteArrayRegion( + data, 0, len, + const_cast(reinterpret_cast(slice->data()))); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(data); + return nullptr; + } + + return data; +} + +/* + * Class: org_rocksdb_Slice + * Method: clear0 + * Signature: (JZJ)V + */ +void Java_org_rocksdb_Slice_clear0(JNIEnv* /*env*/, jobject /*jobj*/, + jlong handle, jboolean shouldRelease, + jlong internalBufferOffset) { + auto* slice = reinterpret_cast(handle); + if (shouldRelease == JNI_TRUE) { + const char* buf = slice->data_ - internalBufferOffset; + delete[] buf; + } + slice->clear(); +} + +/* + * Class: org_rocksdb_Slice + * Method: removePrefix0 + * Signature: (JI)V + */ +void Java_org_rocksdb_Slice_removePrefix0(JNIEnv* /*env*/, jobject /*jobj*/, + jlong handle, jint length) { + auto* slice = reinterpret_cast(handle); + slice->remove_prefix(length); +} + +/* + * Class: org_rocksdb_DirectSlice + * Method: setLength0 + * Signature: (JI)V + */ +void Java_org_rocksdb_DirectSlice_setLength0(JNIEnv* /*env*/, jobject /*jobj*/, + jlong handle, jint length) { + auto* slice = reinterpret_cast(handle); + slice->size_ = length; +} + +/* + * Class: org_rocksdb_Slice + * Method: disposeInternalBuf + * Signature: (JJ)V + */ +void Java_org_rocksdb_Slice_disposeInternalBuf(JNIEnv* /*env*/, + jobject /*jobj*/, jlong handle, + jlong internalBufferOffset) { + const auto* slice = reinterpret_cast(handle); + const char* buf = slice->data_ - internalBufferOffset; + delete[] buf; +} + +// + +// (data_addr); + const auto* slice = new ROCKSDB_NAMESPACE::Slice(ptrData, length); + return GET_CPLUSPLUS_POINTER(slice); +} + +/* + * Class: org_rocksdb_DirectSlice + * Method: createNewDirectSlice1 + * Signature: (Ljava/nio/ByteBuffer;)J + */ +jlong Java_org_rocksdb_DirectSlice_createNewDirectSlice1(JNIEnv* env, + jclass /*jcls*/, + jobject data) { + void* data_addr = env->GetDirectBufferAddress(data); + if (data_addr == nullptr) { + // error: memory region is undefined, given object is not a direct + // java.nio.Buffer, or JNI access to direct buffers is not supported by JVM + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::InvalidArgument( + "Could not access DirectBuffer")); + return 0; + } + + const auto* ptrData = reinterpret_cast(data_addr); + const auto* slice = new ROCKSDB_NAMESPACE::Slice(ptrData); + return GET_CPLUSPLUS_POINTER(slice); +} + +/* + * Class: org_rocksdb_DirectSlice + * Method: data0 + * Signature: (J)Ljava/lang/Object; + */ +jobject Java_org_rocksdb_DirectSlice_data0(JNIEnv* env, jobject /*jobj*/, + jlong handle) { + const auto* slice = reinterpret_cast(handle); + return env->NewDirectByteBuffer(const_cast(slice->data()), + slice->size()); +} + +/* + * Class: org_rocksdb_DirectSlice + * Method: get0 + * Signature: (JI)B + */ +jbyte Java_org_rocksdb_DirectSlice_get0(JNIEnv* /*env*/, jobject /*jobj*/, + jlong handle, jint offset) { + const auto* slice = reinterpret_cast(handle); + return (*slice)[offset]; +} + +/* + * Class: org_rocksdb_DirectSlice + * Method: clear0 + * Signature: (JZJ)V + */ +void Java_org_rocksdb_DirectSlice_clear0(JNIEnv* /*env*/, jobject /*jobj*/, + jlong handle, jboolean shouldRelease, + jlong internalBufferOffset) { + auto* slice = reinterpret_cast(handle); + if (shouldRelease == JNI_TRUE) { + const char* buf = slice->data_ - internalBufferOffset; + delete[] buf; + } + slice->clear(); +} + +/* + * Class: org_rocksdb_DirectSlice + * Method: removePrefix0 + * Signature: (JI)V + */ +void Java_org_rocksdb_DirectSlice_removePrefix0(JNIEnv* /*env*/, + jobject /*jobj*/, jlong handle, + jint length) { + auto* slice = reinterpret_cast(handle); + slice->remove_prefix(length); +} + +/* + * Class: org_rocksdb_DirectSlice + * Method: disposeInternalBuf + * Signature: (JJ)V + */ +void Java_org_rocksdb_DirectSlice_disposeInternalBuf( + JNIEnv* /*env*/, jobject /*jobj*/, jlong handle, + jlong internalBufferOffset) { + const auto* slice = reinterpret_cast(handle); + const char* buf = slice->data_ - internalBufferOffset; + delete[] buf; +} + +// diff --git a/librocksdb-sys/rocksdb/java/rocksjni/snapshot.cc b/librocksdb-sys/rocksdb/java/rocksjni/snapshot.cc new file mode 100644 index 0000000..2a1265a --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/snapshot.cc @@ -0,0 +1,27 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++. + +#include +#include +#include + +#include "include/org_rocksdb_Snapshot.h" +#include "rocksdb/db.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_Snapshot + * Method: getSequenceNumber + * Signature: (J)J + */ +jlong Java_org_rocksdb_Snapshot_getSequenceNumber(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jsnapshot_handle) { + auto* snapshot = + reinterpret_cast(jsnapshot_handle); + return snapshot->GetSequenceNumber(); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/sst_file_manager.cc b/librocksdb-sys/rocksdb/java/rocksjni/sst_file_manager.cc new file mode 100644 index 0000000..c514368 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/sst_file_manager.cc @@ -0,0 +1,250 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling C++ ROCKSDB_NAMESPACE::SstFileManager methods +// from Java side. + +#include "rocksdb/sst_file_manager.h" + +#include + +#include + +#include "include/org_rocksdb_SstFileManager.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_SstFileManager + * Method: newSstFileManager + * Signature: (JJJDJ)J + */ +jlong Java_org_rocksdb_SstFileManager_newSstFileManager( + JNIEnv* jnienv, jclass /*jcls*/, jlong jenv_handle, jlong jlogger_handle, + jlong jrate_bytes, jdouble jmax_trash_db_ratio, + jlong jmax_delete_chunk_bytes) { + auto* env = reinterpret_cast(jenv_handle); + ROCKSDB_NAMESPACE::Status s; + ROCKSDB_NAMESPACE::SstFileManager* sst_file_manager = nullptr; + + if (jlogger_handle != 0) { + auto* sptr_logger = + reinterpret_cast*>( + jlogger_handle); + sst_file_manager = ROCKSDB_NAMESPACE::NewSstFileManager( + env, *sptr_logger, "", jrate_bytes, true, &s, jmax_trash_db_ratio, + jmax_delete_chunk_bytes); + } else { + sst_file_manager = ROCKSDB_NAMESPACE::NewSstFileManager( + env, nullptr, "", jrate_bytes, true, &s, jmax_trash_db_ratio, + jmax_delete_chunk_bytes); + } + + if (!s.ok()) { + if (sst_file_manager != nullptr) { + delete sst_file_manager; + } + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(jnienv, s); + } + auto* sptr_sst_file_manager = + new std::shared_ptr(sst_file_manager); + + return GET_CPLUSPLUS_POINTER(sptr_sst_file_manager); +} + +/* + * Class: org_rocksdb_SstFileManager + * Method: setMaxAllowedSpaceUsage + * Signature: (JJ)V + */ +void Java_org_rocksdb_SstFileManager_setMaxAllowedSpaceUsage( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, + jlong jmax_allowed_space) { + auto* sptr_sst_file_manager = + reinterpret_cast*>( + jhandle); + sptr_sst_file_manager->get()->SetMaxAllowedSpaceUsage(jmax_allowed_space); +} + +/* + * Class: org_rocksdb_SstFileManager + * Method: setCompactionBufferSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_SstFileManager_setCompactionBufferSize( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, + jlong jcompaction_buffer_size) { + auto* sptr_sst_file_manager = + reinterpret_cast*>( + jhandle); + sptr_sst_file_manager->get()->SetCompactionBufferSize( + jcompaction_buffer_size); +} + +/* + * Class: org_rocksdb_SstFileManager + * Method: isMaxAllowedSpaceReached + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_SstFileManager_isMaxAllowedSpaceReached( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { + auto* sptr_sst_file_manager = + reinterpret_cast*>( + jhandle); + return sptr_sst_file_manager->get()->IsMaxAllowedSpaceReached(); +} + +/* + * Class: org_rocksdb_SstFileManager + * Method: isMaxAllowedSpaceReachedIncludingCompactions + * Signature: (J)Z + */ +jboolean +Java_org_rocksdb_SstFileManager_isMaxAllowedSpaceReachedIncludingCompactions( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { + auto* sptr_sst_file_manager = + reinterpret_cast*>( + jhandle); + return sptr_sst_file_manager->get() + ->IsMaxAllowedSpaceReachedIncludingCompactions(); +} + +/* + * Class: org_rocksdb_SstFileManager + * Method: getTotalSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_SstFileManager_getTotalSize(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* sptr_sst_file_manager = + reinterpret_cast*>( + jhandle); + return sptr_sst_file_manager->get()->GetTotalSize(); +} + +/* + * Class: org_rocksdb_SstFileManager + * Method: getTrackedFiles + * Signature: (J)Ljava/util/Map; + */ +jobject Java_org_rocksdb_SstFileManager_getTrackedFiles(JNIEnv* env, + jobject /*jobj*/, + jlong jhandle) { + auto* sptr_sst_file_manager = + reinterpret_cast*>( + jhandle); + auto tracked_files = sptr_sst_file_manager->get()->GetTrackedFiles(); + + // TODO(AR) could refactor to share code with + // ROCKSDB_NAMESPACE::HashMapJni::fromCppMap(env, tracked_files); + + const jobject jtracked_files = ROCKSDB_NAMESPACE::HashMapJni::construct( + env, static_cast(tracked_files.size())); + if (jtracked_files == nullptr) { + // exception occurred + return nullptr; + } + + const ROCKSDB_NAMESPACE::HashMapJni::FnMapKV + fn_map_kv = + [env](const std::pair& pair) { + const jstring jtracked_file_path = + env->NewStringUTF(pair.first.c_str()); + if (jtracked_file_path == nullptr) { + // an error occurred + return std::unique_ptr>(nullptr); + } + const jobject jtracked_file_size = + ROCKSDB_NAMESPACE::LongJni::valueOf(env, pair.second); + if (jtracked_file_size == nullptr) { + // an error occurred + return std::unique_ptr>(nullptr); + } + return std::unique_ptr>( + new std::pair(jtracked_file_path, + jtracked_file_size)); + }; + + if (!ROCKSDB_NAMESPACE::HashMapJni::putAll(env, jtracked_files, + tracked_files.begin(), + tracked_files.end(), fn_map_kv)) { + // exception occcurred + return nullptr; + } + + return jtracked_files; +} + +/* + * Class: org_rocksdb_SstFileManager + * Method: getDeleteRateBytesPerSecond + * Signature: (J)J + */ +jlong Java_org_rocksdb_SstFileManager_getDeleteRateBytesPerSecond( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { + auto* sptr_sst_file_manager = + reinterpret_cast*>( + jhandle); + return sptr_sst_file_manager->get()->GetDeleteRateBytesPerSecond(); +} + +/* + * Class: org_rocksdb_SstFileManager + * Method: setDeleteRateBytesPerSecond + * Signature: (JJ)V + */ +void Java_org_rocksdb_SstFileManager_setDeleteRateBytesPerSecond( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jdelete_rate) { + auto* sptr_sst_file_manager = + reinterpret_cast*>( + jhandle); + sptr_sst_file_manager->get()->SetDeleteRateBytesPerSecond(jdelete_rate); +} + +/* + * Class: org_rocksdb_SstFileManager + * Method: getMaxTrashDBRatio + * Signature: (J)D + */ +jdouble Java_org_rocksdb_SstFileManager_getMaxTrashDBRatio(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* sptr_sst_file_manager = + reinterpret_cast*>( + jhandle); + return sptr_sst_file_manager->get()->GetMaxTrashDBRatio(); +} + +/* + * Class: org_rocksdb_SstFileManager + * Method: setMaxTrashDBRatio + * Signature: (JD)V + */ +void Java_org_rocksdb_SstFileManager_setMaxTrashDBRatio(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle, + jdouble jratio) { + auto* sptr_sst_file_manager = + reinterpret_cast*>( + jhandle); + sptr_sst_file_manager->get()->SetMaxTrashDBRatio(jratio); +} + +/* + * Class: org_rocksdb_SstFileManager + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_SstFileManager_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* sptr_sst_file_manager = + reinterpret_cast*>( + jhandle); + delete sptr_sst_file_manager; +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/sst_file_reader_iterator.cc b/librocksdb-sys/rocksdb/java/rocksjni/sst_file_reader_iterator.cc new file mode 100644 index 0000000..68fa4c3 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/sst_file_reader_iterator.cc @@ -0,0 +1,373 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling c++ ROCKSDB_NAMESPACE::Iterator methods from Java side. + +#include +#include +#include + +#include "include/org_rocksdb_SstFileReaderIterator.h" +#include "rocksdb/iterator.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_SstFileReaderIterator_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + auto* it = reinterpret_cast(handle); + assert(it != nullptr); + delete it; +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: isValid0 + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_SstFileReaderIterator_isValid0(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + return reinterpret_cast(handle)->Valid(); +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: seekToFirst0 + * Signature: (J)V + */ +void Java_org_rocksdb_SstFileReaderIterator_seekToFirst0(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + reinterpret_cast(handle)->SeekToFirst(); +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: seekToLast0 + * Signature: (J)V + */ +void Java_org_rocksdb_SstFileReaderIterator_seekToLast0(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + reinterpret_cast(handle)->SeekToLast(); +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: next0 + * Signature: (J)V + */ +void Java_org_rocksdb_SstFileReaderIterator_next0(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + reinterpret_cast(handle)->Next(); +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: prev0 + * Signature: (J)V + */ +void Java_org_rocksdb_SstFileReaderIterator_prev0(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + reinterpret_cast(handle)->Prev(); +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: seek0 + * Signature: (J[BI)V + */ +void Java_org_rocksdb_SstFileReaderIterator_seek0(JNIEnv* env, jobject /*jobj*/, + jlong handle, + jbyteArray jtarget, + jint jtarget_len) { + jbyte* target = env->GetByteArrayElements(jtarget, nullptr); + if (target == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + ROCKSDB_NAMESPACE::Slice target_slice(reinterpret_cast(target), + jtarget_len); + + auto* it = reinterpret_cast(handle); + it->Seek(target_slice); + + env->ReleaseByteArrayElements(jtarget, target, JNI_ABORT); +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: seekForPrev0 + * Signature: (J[BI)V + */ +void Java_org_rocksdb_SstFileReaderIterator_seekForPrev0(JNIEnv* env, + jobject /*jobj*/, + jlong handle, + jbyteArray jtarget, + jint jtarget_len) { + jbyte* target = env->GetByteArrayElements(jtarget, nullptr); + if (target == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + ROCKSDB_NAMESPACE::Slice target_slice(reinterpret_cast(target), + jtarget_len); + + auto* it = reinterpret_cast(handle); + it->SeekForPrev(target_slice); + + env->ReleaseByteArrayElements(jtarget, target, JNI_ABORT); +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: status0 + * Signature: (J)V + */ +void Java_org_rocksdb_SstFileReaderIterator_status0(JNIEnv* env, + jobject /*jobj*/, + jlong handle) { + auto* it = reinterpret_cast(handle); + ROCKSDB_NAMESPACE::Status s = it->status(); + + if (s.ok()) { + return; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: key0 + * Signature: (J)[B + */ +jbyteArray Java_org_rocksdb_SstFileReaderIterator_key0(JNIEnv* env, + jobject /*jobj*/, + jlong handle) { + auto* it = reinterpret_cast(handle); + ROCKSDB_NAMESPACE::Slice key_slice = it->key(); + + jbyteArray jkey = env->NewByteArray(static_cast(key_slice.size())); + if (jkey == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + env->SetByteArrayRegion( + jkey, 0, static_cast(key_slice.size()), + const_cast(reinterpret_cast(key_slice.data()))); + return jkey; +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: value0 + * Signature: (J)[B + */ +jbyteArray Java_org_rocksdb_SstFileReaderIterator_value0(JNIEnv* env, + jobject /*jobj*/, + jlong handle) { + auto* it = reinterpret_cast(handle); + ROCKSDB_NAMESPACE::Slice value_slice = it->value(); + + jbyteArray jkeyValue = + env->NewByteArray(static_cast(value_slice.size())); + if (jkeyValue == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + env->SetByteArrayRegion( + jkeyValue, 0, static_cast(value_slice.size()), + const_cast(reinterpret_cast(value_slice.data()))); + return jkeyValue; +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: keyDirect0 + * Signature: (JLjava/nio/ByteBuffer;II)I + */ +jint Java_org_rocksdb_SstFileReaderIterator_keyDirect0( + JNIEnv* env, jobject /*jobj*/, jlong handle, jobject jtarget, + jint jtarget_off, jint jtarget_len) { + auto* it = reinterpret_cast(handle); + ROCKSDB_NAMESPACE::Slice key_slice = it->key(); + return ROCKSDB_NAMESPACE::JniUtil::copyToDirect(env, key_slice, jtarget, + jtarget_off, jtarget_len); +} + +/* + * This method supports fetching into indirect byte buffers; + * the Java wrapper extracts the byte[] and passes it here. + * + * Class: org_rocksdb_SstFileReaderIterator + * Method: keyByteArray0 + * Signature: (J[BII)I + */ +jint Java_org_rocksdb_SstFileReaderIterator_keyByteArray0( + JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jkey, jint jkey_off, + jint jkey_len) { + auto* it = reinterpret_cast(handle); + ROCKSDB_NAMESPACE::Slice key_slice = it->key(); + auto slice_size = key_slice.size(); + jsize copy_size = std::min(static_cast(slice_size), + static_cast(jkey_len)); + env->SetByteArrayRegion( + jkey, jkey_off, copy_size, + const_cast(reinterpret_cast(key_slice.data()))); + + return static_cast(slice_size); +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: valueDirect0 + * Signature: (JLjava/nio/ByteBuffer;II)I + */ +jint Java_org_rocksdb_SstFileReaderIterator_valueDirect0( + JNIEnv* env, jobject /*jobj*/, jlong handle, jobject jtarget, + jint jtarget_off, jint jtarget_len) { + auto* it = reinterpret_cast(handle); + ROCKSDB_NAMESPACE::Slice value_slice = it->value(); + return ROCKSDB_NAMESPACE::JniUtil::copyToDirect(env, value_slice, jtarget, + jtarget_off, jtarget_len); +} + +/* + * This method supports fetching into indirect byte buffers; + * the Java wrapper extracts the byte[] and passes it here. + * + * Class: org_rocksdb_SstFileReaderIterator + * Method: valueByteArray0 + * Signature: (J[BII)I + */ +jint Java_org_rocksdb_SstFileReaderIterator_valueByteArray0( + JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jvalue_target, + jint jvalue_off, jint jvalue_len) { + auto* it = reinterpret_cast(handle); + ROCKSDB_NAMESPACE::Slice value_slice = it->value(); + auto slice_size = value_slice.size(); + jsize copy_size = std::min(static_cast(slice_size), + static_cast(jvalue_len)); + env->SetByteArrayRegion( + jvalue_target, jvalue_off, copy_size, + const_cast(reinterpret_cast(value_slice.data()))); + + return static_cast(slice_size); +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: seekDirect0 + * Signature: (JLjava/nio/ByteBuffer;II)V + */ +void Java_org_rocksdb_SstFileReaderIterator_seekDirect0( + JNIEnv* env, jobject /*jobj*/, jlong handle, jobject jtarget, + jint jtarget_off, jint jtarget_len) { + auto* it = reinterpret_cast(handle); + auto seek = [&it](ROCKSDB_NAMESPACE::Slice& target_slice) { + it->Seek(target_slice); + }; + ROCKSDB_NAMESPACE::JniUtil::k_op_direct(seek, env, jtarget, jtarget_off, + jtarget_len); +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: seekForPrevDirect0 + * Signature: (JLjava/nio/ByteBuffer;II)V + */ +void Java_org_rocksdb_SstFileReaderIterator_seekForPrevDirect0( + JNIEnv* env, jobject /*jobj*/, jlong handle, jobject jtarget, + jint jtarget_off, jint jtarget_len) { + auto* it = reinterpret_cast(handle); + auto seekPrev = [&it](ROCKSDB_NAMESPACE::Slice& target_slice) { + it->SeekForPrev(target_slice); + }; + ROCKSDB_NAMESPACE::JniUtil::k_op_direct(seekPrev, env, jtarget, jtarget_off, + jtarget_len); +} + +/* + * This method supports fetching into indirect byte buffers; + * the Java wrapper extracts the byte[] and passes it here. + * + * Class: org_rocksdb_SstFileReaderIterator + * Method: seekByteArray0 + * Signature: (J[BII)V + */ +void Java_org_rocksdb_SstFileReaderIterator_seekByteArray0( + JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget, + jint jtarget_off, jint jtarget_len) { + const std::unique_ptr target(new char[jtarget_len]); + if (target == nullptr) { + jclass oom_class = env->FindClass("/lang/java/OutOfMemoryError"); + env->ThrowNew(oom_class, + "Memory allocation failed in RocksDB JNI function"); + return; + } + env->GetByteArrayRegion(jtarget, jtarget_off, jtarget_len, + reinterpret_cast(target.get())); + + ROCKSDB_NAMESPACE::Slice target_slice(target.get(), jtarget_len); + + auto* it = reinterpret_cast(handle); + it->Seek(target_slice); +} + +/* + * This method supports fetching into indirect byte buffers; + * the Java wrapper extracts the byte[] and passes it here. + * + * Class: org_rocksdb_SstFileReaderIterator + * Method: seekForPrevByteArray0 + * Signature: (J[BII)V + */ +void Java_org_rocksdb_SstFileReaderIterator_seekForPrevByteArray0( + JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget, + jint jtarget_off, jint jtarget_len) { + const std::unique_ptr target(new char[jtarget_len]); + if (target == nullptr) { + jclass oom_class = env->FindClass("/lang/java/OutOfMemoryError"); + env->ThrowNew(oom_class, + "Memory allocation failed in RocksDB JNI function"); + return; + } + env->GetByteArrayRegion(jtarget, jtarget_off, jtarget_len, + reinterpret_cast(target.get())); + + ROCKSDB_NAMESPACE::Slice target_slice(target.get(), jtarget_len); + + auto* it = reinterpret_cast(handle); + it->SeekForPrev(target_slice); +} + +/* + * Class: org_rocksdb_SstFileReaderIterator + * Method: refresh0 + * Signature: (J)V + */ +void Java_org_rocksdb_SstFileReaderIterator_refresh0(JNIEnv* env, + jobject /*jobj*/, + jlong handle) { + auto* it = reinterpret_cast(handle); + ROCKSDB_NAMESPACE::Status s = it->Refresh(); + + if (s.ok()) { + return; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/sst_file_readerjni.cc b/librocksdb-sys/rocksdb/java/rocksjni/sst_file_readerjni.cc new file mode 100644 index 0000000..7ef7118 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/sst_file_readerjni.cc @@ -0,0 +1,118 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling C++ ROCKSDB_NAMESPACE::SstFileReader methods +// from Java side. + +#include + +#include + +#include "include/org_rocksdb_SstFileReader.h" +#include "rocksdb/comparator.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "rocksdb/sst_file_reader.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_SstFileReader + * Method: newSstFileReader + * Signature: (J)J + */ +jlong Java_org_rocksdb_SstFileReader_newSstFileReader(JNIEnv * /*env*/, + jclass /*jcls*/, + jlong joptions) { + auto *options = + reinterpret_cast(joptions); + ROCKSDB_NAMESPACE::SstFileReader *sst_file_reader = + new ROCKSDB_NAMESPACE::SstFileReader(*options); + return GET_CPLUSPLUS_POINTER(sst_file_reader); +} + +/* + * Class: org_rocksdb_SstFileReader + * Method: open + * Signature: (JLjava/lang/String;)V + */ +void Java_org_rocksdb_SstFileReader_open(JNIEnv *env, jobject /*jobj*/, + jlong jhandle, jstring jfile_path) { + const char *file_path = env->GetStringUTFChars(jfile_path, nullptr); + if (file_path == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + ROCKSDB_NAMESPACE::Status s = + reinterpret_cast(jhandle)->Open( + file_path); + env->ReleaseStringUTFChars(jfile_path, file_path); + + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_SstFileReader + * Method: newIterator + * Signature: (JJ)J + */ +jlong Java_org_rocksdb_SstFileReader_newIterator(JNIEnv * /*env*/, + jobject /*jobj*/, + jlong jhandle, + jlong jread_options_handle) { + auto *sst_file_reader = + reinterpret_cast(jhandle); + auto *read_options = + reinterpret_cast(jread_options_handle); + return GET_CPLUSPLUS_POINTER(sst_file_reader->NewIterator(*read_options)); +} + +/* + * Class: org_rocksdb_SstFileReader + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_SstFileReader_disposeInternal(JNIEnv * /*env*/, + jobject /*jobj*/, + jlong jhandle) { + delete reinterpret_cast(jhandle); +} + +/* + * Class: org_rocksdb_SstFileReader + * Method: verifyChecksum + * Signature: (J)V + */ +void Java_org_rocksdb_SstFileReader_verifyChecksum(JNIEnv *env, + jobject /*jobj*/, + jlong jhandle) { + auto *sst_file_reader = + reinterpret_cast(jhandle); + auto s = sst_file_reader->VerifyChecksum(); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_SstFileReader + * Method: getTableProperties + * Signature: (J)J + */ +jobject Java_org_rocksdb_SstFileReader_getTableProperties(JNIEnv *env, + jobject /*jobj*/, + jlong jhandle) { + auto *sst_file_reader = + reinterpret_cast(jhandle); + std::shared_ptr tp = + sst_file_reader->GetTableProperties(); + jobject jtable_properties = + ROCKSDB_NAMESPACE::TablePropertiesJni::fromCppTableProperties( + env, *(tp.get())); + return jtable_properties; +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/sst_file_writerjni.cc b/librocksdb-sys/rocksdb/java/rocksjni/sst_file_writerjni.cc new file mode 100644 index 0000000..1898c3c --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/sst_file_writerjni.cc @@ -0,0 +1,310 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling C++ ROCKSDB_NAMESPACE::SstFileWriter methods +// from Java side. + +#include + +#include + +#include "include/org_rocksdb_SstFileWriter.h" +#include "rocksdb/comparator.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "rocksdb/sst_file_writer.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_SstFileWriter + * Method: newSstFileWriter + * Signature: (JJJB)J + */ +jlong Java_org_rocksdb_SstFileWriter_newSstFileWriter__JJJB( + JNIEnv * /*env*/, jclass /*jcls*/, jlong jenvoptions, jlong joptions, + jlong jcomparator_handle, jbyte jcomparator_type) { + ROCKSDB_NAMESPACE::Comparator *comparator = nullptr; + switch (jcomparator_type) { + // JAVA_COMPARATOR + case 0x0: + comparator = reinterpret_cast( + jcomparator_handle); + break; + + // JAVA_NATIVE_COMPARATOR_WRAPPER + case 0x1: + comparator = + reinterpret_cast(jcomparator_handle); + break; + } + auto *env_options = + reinterpret_cast(jenvoptions); + auto *options = + reinterpret_cast(joptions); + ROCKSDB_NAMESPACE::SstFileWriter *sst_file_writer = + new ROCKSDB_NAMESPACE::SstFileWriter(*env_options, *options, comparator); + return GET_CPLUSPLUS_POINTER(sst_file_writer); +} + +/* + * Class: org_rocksdb_SstFileWriter + * Method: newSstFileWriter + * Signature: (JJ)J + */ +jlong Java_org_rocksdb_SstFileWriter_newSstFileWriter__JJ(JNIEnv * /*env*/, + jclass /*jcls*/, + jlong jenvoptions, + jlong joptions) { + auto *env_options = + reinterpret_cast(jenvoptions); + auto *options = + reinterpret_cast(joptions); + ROCKSDB_NAMESPACE::SstFileWriter *sst_file_writer = + new ROCKSDB_NAMESPACE::SstFileWriter(*env_options, *options); + return GET_CPLUSPLUS_POINTER(sst_file_writer); +} + +/* + * Class: org_rocksdb_SstFileWriter + * Method: open + * Signature: (JLjava/lang/String;)V + */ +void Java_org_rocksdb_SstFileWriter_open(JNIEnv *env, jobject /*jobj*/, + jlong jhandle, jstring jfile_path) { + const char *file_path = env->GetStringUTFChars(jfile_path, nullptr); + if (file_path == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + ROCKSDB_NAMESPACE::Status s = + reinterpret_cast(jhandle)->Open( + file_path); + env->ReleaseStringUTFChars(jfile_path, file_path); + + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_SstFileWriter + * Method: put + * Signature: (JJJ)V + */ +void Java_org_rocksdb_SstFileWriter_put__JJJ(JNIEnv *env, jobject /*jobj*/, + jlong jhandle, jlong jkey_handle, + jlong jvalue_handle) { + auto *key_slice = reinterpret_cast(jkey_handle); + auto *value_slice = + reinterpret_cast(jvalue_handle); + ROCKSDB_NAMESPACE::Status s = + reinterpret_cast(jhandle)->Put( + *key_slice, *value_slice); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_SstFileWriter + * Method: put + * Signature: (JJJ)V + */ +void Java_org_rocksdb_SstFileWriter_put__J_3B_3B(JNIEnv *env, jobject /*jobj*/, + jlong jhandle, jbyteArray jkey, + jbyteArray jval) { + jbyte *key = env->GetByteArrayElements(jkey, nullptr); + if (key == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), + env->GetArrayLength(jkey)); + + jbyte *value = env->GetByteArrayElements(jval, nullptr); + if (value == nullptr) { + // exception thrown: OutOfMemoryError + env->ReleaseByteArrayElements(jkey, key, JNI_ABORT); + return; + } + ROCKSDB_NAMESPACE::Slice value_slice(reinterpret_cast(value), + env->GetArrayLength(jval)); + + ROCKSDB_NAMESPACE::Status s = + reinterpret_cast(jhandle)->Put( + key_slice, value_slice); + + env->ReleaseByteArrayElements(jkey, key, JNI_ABORT); + env->ReleaseByteArrayElements(jval, value, JNI_ABORT); + + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_SstFileWriter + * Method: putDirect + * Signature: (JLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;II)V + */ +void Java_org_rocksdb_SstFileWriter_putDirect(JNIEnv *env, jobject /*jdb*/, + jlong jdb_handle, jobject jkey, + jint jkey_off, jint jkey_len, + jobject jval, jint jval_off, + jint jval_len) { + auto *writer = + reinterpret_cast(jdb_handle); + auto put = [&env, &writer](ROCKSDB_NAMESPACE::Slice &key, + ROCKSDB_NAMESPACE::Slice &value) { + ROCKSDB_NAMESPACE::Status s = writer->Put(key, value); + if (s.ok()) { + return; + } + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + }; + ROCKSDB_NAMESPACE::JniUtil::kv_op_direct(put, env, jkey, jkey_off, jkey_len, + jval, jval_off, jval_len); +} + +/* + * Class: org_rocksdb_SstFileWriter + * Method: fileSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_SstFileWriter_fileSize(JNIEnv * /*env*/, jobject /*jdb*/, + jlong jdb_handle) { + auto *writer = + reinterpret_cast(jdb_handle); + return static_cast(writer->FileSize()); +} + +/* + * Class: org_rocksdb_SstFileWriter + * Method: merge + * Signature: (JJJ)V + */ +void Java_org_rocksdb_SstFileWriter_merge__JJJ(JNIEnv *env, jobject /*jobj*/, + jlong jhandle, jlong jkey_handle, + jlong jvalue_handle) { + auto *key_slice = reinterpret_cast(jkey_handle); + auto *value_slice = + reinterpret_cast(jvalue_handle); + ROCKSDB_NAMESPACE::Status s = + reinterpret_cast(jhandle)->Merge( + *key_slice, *value_slice); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_SstFileWriter + * Method: merge + * Signature: (J[B[B)V + */ +void Java_org_rocksdb_SstFileWriter_merge__J_3B_3B(JNIEnv *env, + jobject /*jobj*/, + jlong jhandle, + jbyteArray jkey, + jbyteArray jval) { + jbyte *key = env->GetByteArrayElements(jkey, nullptr); + if (key == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), + env->GetArrayLength(jkey)); + + jbyte *value = env->GetByteArrayElements(jval, nullptr); + if (value == nullptr) { + // exception thrown: OutOfMemoryError + env->ReleaseByteArrayElements(jkey, key, JNI_ABORT); + return; + } + ROCKSDB_NAMESPACE::Slice value_slice(reinterpret_cast(value), + env->GetArrayLength(jval)); + + ROCKSDB_NAMESPACE::Status s = + reinterpret_cast(jhandle)->Merge( + key_slice, value_slice); + + env->ReleaseByteArrayElements(jkey, key, JNI_ABORT); + env->ReleaseByteArrayElements(jval, value, JNI_ABORT); + + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_SstFileWriter + * Method: delete + * Signature: (JJJ)V + */ +void Java_org_rocksdb_SstFileWriter_delete__J_3B(JNIEnv *env, jobject /*jobj*/, + jlong jhandle, + jbyteArray jkey) { + jbyte *key = env->GetByteArrayElements(jkey, nullptr); + if (key == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), + env->GetArrayLength(jkey)); + + ROCKSDB_NAMESPACE::Status s = + reinterpret_cast(jhandle)->Delete( + key_slice); + + env->ReleaseByteArrayElements(jkey, key, JNI_ABORT); + + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_SstFileWriter + * Method: delete + * Signature: (JJJ)V + */ +void Java_org_rocksdb_SstFileWriter_delete__JJ(JNIEnv *env, jobject /*jobj*/, + jlong jhandle, + jlong jkey_handle) { + auto *key_slice = reinterpret_cast(jkey_handle); + ROCKSDB_NAMESPACE::Status s = + reinterpret_cast(jhandle)->Delete( + *key_slice); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_SstFileWriter + * Method: finish + * Signature: (J)V + */ +void Java_org_rocksdb_SstFileWriter_finish(JNIEnv *env, jobject /*jobj*/, + jlong jhandle) { + ROCKSDB_NAMESPACE::Status s = + reinterpret_cast(jhandle)->Finish(); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_SstFileWriter + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_SstFileWriter_disposeInternal(JNIEnv * /*env*/, + jobject /*jobj*/, + jlong jhandle) { + delete reinterpret_cast(jhandle); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/sst_partitioner.cc b/librocksdb-sys/rocksdb/java/rocksjni/sst_partitioner.cc new file mode 100644 index 0000000..1cea3b0 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/sst_partitioner.cc @@ -0,0 +1,43 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling C++ ROCKSDB_NAMESPACE::SstFileManager methods +// from Java side. + +#include "rocksdb/sst_partitioner.h" + +#include + +#include + +#include "include/org_rocksdb_SstPartitionerFixedPrefixFactory.h" +#include "rocksdb/sst_file_manager.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_SstPartitionerFixedPrefixFactory + * Method: newSstPartitionerFixedPrefixFactory0 + * Signature: (J)J + */ +jlong Java_org_rocksdb_SstPartitionerFixedPrefixFactory_newSstPartitionerFixedPrefixFactory0( + JNIEnv*, jclass, jlong prefix_len) { + auto* ptr = new std::shared_ptr( + ROCKSDB_NAMESPACE::NewSstPartitionerFixedPrefixFactory(prefix_len)); + return GET_CPLUSPLUS_POINTER(ptr); +} + +/* + * Class: org_rocksdb_SstPartitionerFixedPrefixFactory + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_SstPartitionerFixedPrefixFactory_disposeInternal( + JNIEnv*, jobject, jlong jhandle) { + auto* ptr = reinterpret_cast< + std::shared_ptr*>(jhandle); + delete ptr; // delete std::shared_ptr +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/statistics.cc b/librocksdb-sys/rocksdb/java/rocksjni/statistics.cc new file mode 100644 index 0000000..bd405af --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/statistics.cc @@ -0,0 +1,268 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling c++ ROCKSDB_NAMESPACE::Statistics methods from Java side. + +#include "rocksdb/statistics.h" + +#include + +#include +#include + +#include "include/org_rocksdb_Statistics.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" +#include "rocksjni/statisticsjni.h" + +/* + * Class: org_rocksdb_Statistics + * Method: newStatistics + * Signature: ()J + */ +jlong Java_org_rocksdb_Statistics_newStatistics__(JNIEnv* env, jclass jcls) { + return Java_org_rocksdb_Statistics_newStatistics___3BJ(env, jcls, nullptr, 0); +} + +/* + * Class: org_rocksdb_Statistics + * Method: newStatistics + * Signature: (J)J + */ +jlong Java_org_rocksdb_Statistics_newStatistics__J( + JNIEnv* env, jclass jcls, jlong jother_statistics_handle) { + return Java_org_rocksdb_Statistics_newStatistics___3BJ( + env, jcls, nullptr, jother_statistics_handle); +} + +/* + * Class: org_rocksdb_Statistics + * Method: newStatistics + * Signature: ([B)J + */ +jlong Java_org_rocksdb_Statistics_newStatistics___3B(JNIEnv* env, jclass jcls, + jbyteArray jhistograms) { + return Java_org_rocksdb_Statistics_newStatistics___3BJ(env, jcls, jhistograms, + 0); +} + +/* + * Class: org_rocksdb_Statistics + * Method: newStatistics + * Signature: ([BJ)J + */ +jlong Java_org_rocksdb_Statistics_newStatistics___3BJ( + JNIEnv* env, jclass, jbyteArray jhistograms, + jlong jother_statistics_handle) { + std::shared_ptr* pSptr_other_statistics = + nullptr; + if (jother_statistics_handle > 0) { + pSptr_other_statistics = + reinterpret_cast*>( + jother_statistics_handle); + } + + std::set histograms; + if (jhistograms != nullptr) { + const jsize len = env->GetArrayLength(jhistograms); + if (len > 0) { + jbyte* jhistogram = env->GetByteArrayElements(jhistograms, nullptr); + if (jhistogram == nullptr) { + // exception thrown: OutOfMemoryError + return 0; + } + + for (jsize i = 0; i < len; i++) { + const ROCKSDB_NAMESPACE::Histograms histogram = + ROCKSDB_NAMESPACE::HistogramTypeJni::toCppHistograms(jhistogram[i]); + histograms.emplace(histogram); + } + + env->ReleaseByteArrayElements(jhistograms, jhistogram, JNI_ABORT); + } + } + + std::shared_ptr sptr_other_statistics = + nullptr; + if (pSptr_other_statistics != nullptr) { + sptr_other_statistics = *pSptr_other_statistics; + } + + auto* pSptr_statistics = + new std::shared_ptr( + new ROCKSDB_NAMESPACE::StatisticsJni(sptr_other_statistics, + histograms)); + + return GET_CPLUSPLUS_POINTER(pSptr_statistics); +} + +/* + * Class: org_rocksdb_Statistics + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_Statistics_disposeInternal(JNIEnv*, jobject, + jlong jhandle) { + if (jhandle > 0) { + auto* pSptr_statistics = + reinterpret_cast*>( + jhandle); + delete pSptr_statistics; + } +} + +/* + * Class: org_rocksdb_Statistics + * Method: statsLevel + * Signature: (J)B + */ +jbyte Java_org_rocksdb_Statistics_statsLevel(JNIEnv*, jobject, jlong jhandle) { + auto* pSptr_statistics = + reinterpret_cast*>( + jhandle); + assert(pSptr_statistics != nullptr); + return ROCKSDB_NAMESPACE::StatsLevelJni::toJavaStatsLevel( + pSptr_statistics->get()->get_stats_level()); +} + +/* + * Class: org_rocksdb_Statistics + * Method: setStatsLevel + * Signature: (JB)V + */ +void Java_org_rocksdb_Statistics_setStatsLevel(JNIEnv*, jobject, jlong jhandle, + jbyte jstats_level) { + auto* pSptr_statistics = + reinterpret_cast*>( + jhandle); + assert(pSptr_statistics != nullptr); + auto stats_level = + ROCKSDB_NAMESPACE::StatsLevelJni::toCppStatsLevel(jstats_level); + pSptr_statistics->get()->set_stats_level(stats_level); +} + +/* + * Class: org_rocksdb_Statistics + * Method: getTickerCount + * Signature: (JB)J + */ +jlong Java_org_rocksdb_Statistics_getTickerCount(JNIEnv*, jobject, + jlong jhandle, + jbyte jticker_type) { + auto* pSptr_statistics = + reinterpret_cast*>( + jhandle); + assert(pSptr_statistics != nullptr); + auto ticker = ROCKSDB_NAMESPACE::TickerTypeJni::toCppTickers(jticker_type); + uint64_t count = pSptr_statistics->get()->getTickerCount(ticker); + return static_cast(count); +} + +/* + * Class: org_rocksdb_Statistics + * Method: getAndResetTickerCount + * Signature: (JB)J + */ +jlong Java_org_rocksdb_Statistics_getAndResetTickerCount(JNIEnv*, jobject, + jlong jhandle, + jbyte jticker_type) { + auto* pSptr_statistics = + reinterpret_cast*>( + jhandle); + assert(pSptr_statistics != nullptr); + auto ticker = ROCKSDB_NAMESPACE::TickerTypeJni::toCppTickers(jticker_type); + return pSptr_statistics->get()->getAndResetTickerCount(ticker); +} + +/* + * Class: org_rocksdb_Statistics + * Method: getHistogramData + * Signature: (JB)Lorg/rocksdb/HistogramData; + */ +jobject Java_org_rocksdb_Statistics_getHistogramData(JNIEnv* env, jobject, + jlong jhandle, + jbyte jhistogram_type) { + auto* pSptr_statistics = + reinterpret_cast*>( + jhandle); + assert(pSptr_statistics != nullptr); + + // TODO(AR) perhaps better to construct a Java Object Wrapper that + // uses ptr to C++ `new HistogramData` + ROCKSDB_NAMESPACE::HistogramData data; + + auto histogram = + ROCKSDB_NAMESPACE::HistogramTypeJni::toCppHistograms(jhistogram_type); + pSptr_statistics->get()->histogramData( + static_cast(histogram), &data); + + jclass jclazz = ROCKSDB_NAMESPACE::HistogramDataJni::getJClass(env); + if (jclazz == nullptr) { + // exception occurred accessing class + return nullptr; + } + + jmethodID mid = + ROCKSDB_NAMESPACE::HistogramDataJni::getConstructorMethodId(env); + if (mid == nullptr) { + // exception occurred accessing method + return nullptr; + } + + return env->NewObject(jclazz, mid, data.median, data.percentile95, + data.percentile99, data.average, + data.standard_deviation, data.max, data.count, data.sum, + data.min); +} + +/* + * Class: org_rocksdb_Statistics + * Method: getHistogramString + * Signature: (JB)Ljava/lang/String; + */ +jstring Java_org_rocksdb_Statistics_getHistogramString(JNIEnv* env, jobject, + jlong jhandle, + jbyte jhistogram_type) { + auto* pSptr_statistics = + reinterpret_cast*>( + jhandle); + assert(pSptr_statistics != nullptr); + auto histogram = + ROCKSDB_NAMESPACE::HistogramTypeJni::toCppHistograms(jhistogram_type); + auto str = pSptr_statistics->get()->getHistogramString(histogram); + return env->NewStringUTF(str.c_str()); +} + +/* + * Class: org_rocksdb_Statistics + * Method: reset + * Signature: (J)V + */ +void Java_org_rocksdb_Statistics_reset(JNIEnv* env, jobject, jlong jhandle) { + auto* pSptr_statistics = + reinterpret_cast*>( + jhandle); + assert(pSptr_statistics != nullptr); + ROCKSDB_NAMESPACE::Status s = pSptr_statistics->get()->Reset(); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_Statistics + * Method: toString + * Signature: (J)Ljava/lang/String; + */ +jstring Java_org_rocksdb_Statistics_toString(JNIEnv* env, jobject, + jlong jhandle) { + auto* pSptr_statistics = + reinterpret_cast*>( + jhandle); + assert(pSptr_statistics != nullptr); + auto str = pSptr_statistics->get()->ToString(); + return env->NewStringUTF(str.c_str()); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/statisticsjni.cc b/librocksdb-sys/rocksdb/java/rocksjni/statisticsjni.cc new file mode 100644 index 0000000..f463378 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/statisticsjni.cc @@ -0,0 +1,31 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the callback "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::Statistics + +#include "rocksjni/statisticsjni.h" + +namespace ROCKSDB_NAMESPACE { + +StatisticsJni::StatisticsJni(std::shared_ptr stats) + : StatisticsImpl(stats), m_ignore_histograms() {} + +StatisticsJni::StatisticsJni(std::shared_ptr stats, + const std::set ignore_histograms) + : StatisticsImpl(stats), m_ignore_histograms(ignore_histograms) {} + +bool StatisticsJni::HistEnabledForType(uint32_t type) const { + if (type >= HISTOGRAM_ENUM_MAX) { + return false; + } + + if (m_ignore_histograms.count(type) > 0) { + return false; + } + + return true; +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/java/rocksjni/statisticsjni.h b/librocksdb-sys/rocksdb/java/rocksjni/statisticsjni.h new file mode 100644 index 0000000..3262b29 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/statisticsjni.h @@ -0,0 +1,34 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the callback "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::Statistics + +#ifndef JAVA_ROCKSJNI_STATISTICSJNI_H_ +#define JAVA_ROCKSJNI_STATISTICSJNI_H_ + +#include +#include +#include + +#include "monitoring/statistics_impl.h" +#include "rocksdb/statistics.h" + +namespace ROCKSDB_NAMESPACE { + +class StatisticsJni : public StatisticsImpl { + public: + StatisticsJni(std::shared_ptr stats); + StatisticsJni(std::shared_ptr stats, + const std::set ignore_histograms); + virtual bool HistEnabledForType(uint32_t type) const override; + + private: + const std::set m_ignore_histograms; +}; + +} // namespace ROCKSDB_NAMESPACE + +#endif // JAVA_ROCKSJNI_STATISTICSJNI_H_ diff --git a/librocksdb-sys/rocksdb/java/rocksjni/table.cc b/librocksdb-sys/rocksdb/java/rocksjni/table.cc new file mode 100644 index 0000000..7f99900 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/table.cc @@ -0,0 +1,145 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::Options. + +#include "rocksdb/table.h" + +#include + +#include "include/org_rocksdb_BlockBasedTableConfig.h" +#include "include/org_rocksdb_PlainTableConfig.h" +#include "portal.h" +#include "rocksdb/cache.h" +#include "rocksdb/filter_policy.h" +#include "rocksjni/cplusplus_to_java_convert.h" + +/* + * Class: org_rocksdb_PlainTableConfig + * Method: newTableFactoryHandle + * Signature: (IIDIIBZZ)J + */ +jlong Java_org_rocksdb_PlainTableConfig_newTableFactoryHandle( + JNIEnv * /*env*/, jobject /*jobj*/, jint jkey_size, + jint jbloom_bits_per_key, jdouble jhash_table_ratio, jint jindex_sparseness, + jint jhuge_page_tlb_size, jbyte jencoding_type, jboolean jfull_scan_mode, + jboolean jstore_index_in_file) { + ROCKSDB_NAMESPACE::PlainTableOptions options = + ROCKSDB_NAMESPACE::PlainTableOptions(); + options.user_key_len = jkey_size; + options.bloom_bits_per_key = jbloom_bits_per_key; + options.hash_table_ratio = jhash_table_ratio; + options.index_sparseness = jindex_sparseness; + options.huge_page_tlb_size = jhuge_page_tlb_size; + options.encoding_type = + static_cast(jencoding_type); + options.full_scan_mode = jfull_scan_mode; + options.store_index_in_file = jstore_index_in_file; + return GET_CPLUSPLUS_POINTER( + ROCKSDB_NAMESPACE::NewPlainTableFactory(options)); +} + +/* + * Class: org_rocksdb_BlockBasedTableConfig + * Method: newTableFactoryHandle + * Signature: (ZZZZBBDBZJJJJIIIJZZZJZZIIZZBJIJI)J + */ +jlong Java_org_rocksdb_BlockBasedTableConfig_newTableFactoryHandle( + JNIEnv *, jobject, jboolean jcache_index_and_filter_blocks, + jboolean jcache_index_and_filter_blocks_with_high_priority, + jboolean jpin_l0_filter_and_index_blocks_in_cache, + jboolean jpin_top_level_index_and_filter, jbyte jindex_type_value, + jbyte jdata_block_index_type_value, + jdouble jdata_block_hash_table_util_ratio, jbyte jchecksum_type_value, + jboolean jno_block_cache, jlong jblock_cache_handle, + jlong jpersistent_cache_handle, jlong jblock_size, + jint jblock_size_deviation, jint jblock_restart_interval, + jint jindex_block_restart_interval, jlong jmetadata_block_size, + jboolean jpartition_filters, jboolean joptimize_filters_for_memory, + jboolean juse_delta_encoding, jlong jfilter_policy_handle, + jboolean jwhole_key_filtering, jboolean jverify_compression, + jint jread_amp_bytes_per_bit, jint jformat_version, + jboolean jenable_index_compression, jboolean jblock_align, + jbyte jindex_shortening, jlong jblock_cache_size, + jint jblock_cache_num_shard_bits) { + ROCKSDB_NAMESPACE::BlockBasedTableOptions options; + options.cache_index_and_filter_blocks = + static_cast(jcache_index_and_filter_blocks); + options.cache_index_and_filter_blocks_with_high_priority = + static_cast(jcache_index_and_filter_blocks_with_high_priority); + options.pin_l0_filter_and_index_blocks_in_cache = + static_cast(jpin_l0_filter_and_index_blocks_in_cache); + options.pin_top_level_index_and_filter = + static_cast(jpin_top_level_index_and_filter); + options.index_type = + ROCKSDB_NAMESPACE::IndexTypeJni::toCppIndexType(jindex_type_value); + options.data_block_index_type = + ROCKSDB_NAMESPACE::DataBlockIndexTypeJni::toCppDataBlockIndexType( + jdata_block_index_type_value); + options.data_block_hash_table_util_ratio = + static_cast(jdata_block_hash_table_util_ratio); + options.checksum = ROCKSDB_NAMESPACE::ChecksumTypeJni::toCppChecksumType( + jchecksum_type_value); + options.no_block_cache = static_cast(jno_block_cache); + if (options.no_block_cache) { + options.block_cache = nullptr; + } else { + if (jblock_cache_handle > 0) { + std::shared_ptr *pCache = + reinterpret_cast *>( + jblock_cache_handle); + options.block_cache = *pCache; + } else if (jblock_cache_size >= 0) { + if (jblock_cache_num_shard_bits > 0) { + options.block_cache = ROCKSDB_NAMESPACE::NewLRUCache( + static_cast(jblock_cache_size), + static_cast(jblock_cache_num_shard_bits)); + } else { + options.block_cache = ROCKSDB_NAMESPACE::NewLRUCache( + static_cast(jblock_cache_size)); + } + } else { + options.no_block_cache = true; + options.block_cache = nullptr; + } + } + if (jpersistent_cache_handle > 0) { + std::shared_ptr *pCache = + reinterpret_cast *>( + jpersistent_cache_handle); + options.persistent_cache = *pCache; + } + options.block_size = static_cast(jblock_size); + options.block_size_deviation = static_cast(jblock_size_deviation); + options.block_restart_interval = static_cast(jblock_restart_interval); + options.index_block_restart_interval = + static_cast(jindex_block_restart_interval); + options.metadata_block_size = static_cast(jmetadata_block_size); + options.partition_filters = static_cast(jpartition_filters); + options.optimize_filters_for_memory = + static_cast(joptimize_filters_for_memory); + options.use_delta_encoding = static_cast(juse_delta_encoding); + if (jfilter_policy_handle > 0) { + std::shared_ptr *pFilterPolicy = + reinterpret_cast *>( + jfilter_policy_handle); + options.filter_policy = *pFilterPolicy; + } + options.whole_key_filtering = static_cast(jwhole_key_filtering); + options.verify_compression = static_cast(jverify_compression); + options.read_amp_bytes_per_bit = + static_cast(jread_amp_bytes_per_bit); + options.format_version = static_cast(jformat_version); + options.enable_index_compression = + static_cast(jenable_index_compression); + options.block_align = static_cast(jblock_align); + options.index_shortening = + ROCKSDB_NAMESPACE::IndexShorteningModeJni::toCppIndexShorteningMode( + jindex_shortening); + + return GET_CPLUSPLUS_POINTER( + ROCKSDB_NAMESPACE::NewBlockBasedTableFactory(options)); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/table_filter.cc b/librocksdb-sys/rocksdb/java/rocksjni/table_filter.cc new file mode 100644 index 0000000..1400fa1 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/table_filter.cc @@ -0,0 +1,27 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// org.rocksdb.AbstractTableFilter. + +#include + +#include + +#include "include/org_rocksdb_AbstractTableFilter.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/table_filter_jnicallback.h" + +/* + * Class: org_rocksdb_AbstractTableFilter + * Method: createNewTableFilter + * Signature: ()J + */ +jlong Java_org_rocksdb_AbstractTableFilter_createNewTableFilter( + JNIEnv* env, jobject jtable_filter) { + auto* table_filter_jnicallback = + new ROCKSDB_NAMESPACE::TableFilterJniCallback(env, jtable_filter); + return GET_CPLUSPLUS_POINTER(table_filter_jnicallback); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/table_filter_jnicallback.cc b/librocksdb-sys/rocksdb/java/rocksjni/table_filter_jnicallback.cc new file mode 100644 index 0000000..5350c5c --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/table_filter_jnicallback.cc @@ -0,0 +1,66 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the callback "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::TableFilter. + +#include "rocksjni/table_filter_jnicallback.h" + +#include "rocksjni/portal.h" + +namespace ROCKSDB_NAMESPACE { +TableFilterJniCallback::TableFilterJniCallback(JNIEnv* env, + jobject jtable_filter) + : JniCallback(env, jtable_filter) { + m_jfilter_methodid = AbstractTableFilterJni::getFilterMethod(env); + if (m_jfilter_methodid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return; + } + + // create the function reference + /* + Note the JNI ENV must be obtained/release + on each call to the function itself as + it may be called from multiple threads + */ + m_table_filter_function = + [this](const ROCKSDB_NAMESPACE::TableProperties& table_properties) { + jboolean attached_thread = JNI_FALSE; + JNIEnv* thread_env = getJniEnv(&attached_thread); + assert(thread_env != nullptr); + + // create a Java TableProperties object + jobject jtable_properties = TablePropertiesJni::fromCppTableProperties( + thread_env, table_properties); + if (jtable_properties == nullptr) { + // exception thrown from fromCppTableProperties + thread_env->ExceptionDescribe(); // print out exception to stderr + releaseJniEnv(attached_thread); + return false; + } + + jboolean result = thread_env->CallBooleanMethod( + m_jcallback_obj, m_jfilter_methodid, jtable_properties); + if (thread_env->ExceptionCheck()) { + // exception thrown from CallBooleanMethod + thread_env->DeleteLocalRef(jtable_properties); + thread_env->ExceptionDescribe(); // print out exception to stderr + releaseJniEnv(attached_thread); + return false; + } + + // ok... cleanup and then return + releaseJniEnv(attached_thread); + return static_cast(result); + }; +} + +std::function +TableFilterJniCallback::GetTableFilterFunction() { + return m_table_filter_function; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/java/rocksjni/table_filter_jnicallback.h b/librocksdb-sys/rocksdb/java/rocksjni/table_filter_jnicallback.h new file mode 100644 index 0000000..0ef404c --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/table_filter_jnicallback.h @@ -0,0 +1,36 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the callback "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::TableFilter. + +#ifndef JAVA_ROCKSJNI_TABLE_FILTER_JNICALLBACK_H_ +#define JAVA_ROCKSJNI_TABLE_FILTER_JNICALLBACK_H_ + +#include + +#include +#include + +#include "rocksdb/table_properties.h" +#include "rocksjni/jnicallback.h" + +namespace ROCKSDB_NAMESPACE { + +class TableFilterJniCallback : public JniCallback { + public: + TableFilterJniCallback(JNIEnv* env, jobject jtable_filter); + std::function + GetTableFilterFunction(); + + private: + jmethodID m_jfilter_methodid; + std::function + m_table_filter_function; +}; + +} // namespace ROCKSDB_NAMESPACE + +#endif // JAVA_ROCKSJNI_TABLE_FILTER_JNICALLBACK_H_ diff --git a/librocksdb-sys/rocksdb/java/rocksjni/testable_event_listener.cc b/librocksdb-sys/rocksdb/java/rocksjni/testable_event_listener.cc new file mode 100644 index 0000000..71188bc --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/testable_event_listener.cc @@ -0,0 +1,219 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#include +#include +#include +#include + +#include "include/org_rocksdb_test_TestableEventListener.h" +#include "rocksdb/listener.h" +#include "rocksdb/status.h" +#include "rocksdb/table_properties.h" + +using ROCKSDB_NAMESPACE::BackgroundErrorReason; +using ROCKSDB_NAMESPACE::CompactionJobInfo; +using ROCKSDB_NAMESPACE::CompactionJobStats; +using ROCKSDB_NAMESPACE::CompactionReason; +using ROCKSDB_NAMESPACE::CompressionType; +using ROCKSDB_NAMESPACE::ExternalFileIngestionInfo; +using ROCKSDB_NAMESPACE::FileOperationInfo; +using ROCKSDB_NAMESPACE::FileOperationType; +using ROCKSDB_NAMESPACE::FlushJobInfo; +using ROCKSDB_NAMESPACE::FlushReason; +using ROCKSDB_NAMESPACE::MemTableInfo; +using ROCKSDB_NAMESPACE::Status; +using ROCKSDB_NAMESPACE::TableFileCreationBriefInfo; +using ROCKSDB_NAMESPACE::TableFileCreationInfo; +using ROCKSDB_NAMESPACE::TableFileCreationReason; +using ROCKSDB_NAMESPACE::TableFileDeletionInfo; +using ROCKSDB_NAMESPACE::TableProperties; +using ROCKSDB_NAMESPACE::WriteStallCondition; +using ROCKSDB_NAMESPACE::WriteStallInfo; + +static TableProperties newTablePropertiesForTest() { + TableProperties table_properties; + table_properties.data_size = UINT64_MAX; + table_properties.index_size = UINT64_MAX; + table_properties.index_partitions = UINT64_MAX; + table_properties.top_level_index_size = UINT64_MAX; + table_properties.index_key_is_user_key = UINT64_MAX; + table_properties.index_value_is_delta_encoded = UINT64_MAX; + table_properties.filter_size = UINT64_MAX; + table_properties.raw_key_size = UINT64_MAX; + table_properties.raw_value_size = UINT64_MAX; + table_properties.num_data_blocks = UINT64_MAX; + table_properties.num_entries = UINT64_MAX; + table_properties.num_deletions = UINT64_MAX; + table_properties.num_merge_operands = UINT64_MAX; + table_properties.num_range_deletions = UINT64_MAX; + table_properties.format_version = UINT64_MAX; + table_properties.fixed_key_len = UINT64_MAX; + table_properties.column_family_id = UINT64_MAX; + table_properties.creation_time = UINT64_MAX; + table_properties.oldest_key_time = UINT64_MAX; + table_properties.file_creation_time = UINT64_MAX; + table_properties.slow_compression_estimated_data_size = UINT64_MAX; + table_properties.fast_compression_estimated_data_size = UINT64_MAX; + table_properties.external_sst_file_global_seqno_offset = UINT64_MAX; + table_properties.db_id = "dbId"; + table_properties.db_session_id = "sessionId"; + table_properties.column_family_name = "columnFamilyName"; + table_properties.filter_policy_name = "filterPolicyName"; + table_properties.comparator_name = "comparatorName"; + table_properties.merge_operator_name = "mergeOperatorName"; + table_properties.prefix_extractor_name = "prefixExtractorName"; + table_properties.property_collectors_names = "propertyCollectorsNames"; + table_properties.compression_name = "compressionName"; + table_properties.compression_options = "compressionOptions"; + table_properties.user_collected_properties = {{"key", "value"}}; + table_properties.readable_properties = {{"key", "value"}}; + return table_properties; +} + +/* + * Class: org_rocksdb_test_TestableEventListener + * Method: invokeAllCallbacks + * Signature: (J)V + */ +void Java_org_rocksdb_test_TestableEventListener_invokeAllCallbacks( + JNIEnv *, jclass, jlong jhandle) { + const auto &el = + *reinterpret_cast *>( + jhandle); + + TableProperties table_properties = newTablePropertiesForTest(); + + FlushJobInfo flush_job_info; + flush_job_info.cf_id = INT_MAX; + flush_job_info.cf_name = "testColumnFamily"; + flush_job_info.file_path = "/file/path"; + flush_job_info.file_number = UINT64_MAX; + flush_job_info.oldest_blob_file_number = UINT64_MAX; + flush_job_info.thread_id = UINT64_MAX; + flush_job_info.job_id = INT_MAX; + flush_job_info.triggered_writes_slowdown = true; + flush_job_info.triggered_writes_stop = true; + flush_job_info.smallest_seqno = UINT64_MAX; + flush_job_info.largest_seqno = UINT64_MAX; + flush_job_info.table_properties = table_properties; + flush_job_info.flush_reason = FlushReason::kManualFlush; + + el->OnFlushCompleted(nullptr, flush_job_info); + el->OnFlushBegin(nullptr, flush_job_info); + + Status status = Status::Incomplete(Status::SubCode::kNoSpace); + + TableFileDeletionInfo file_deletion_info; + file_deletion_info.db_name = "dbName"; + file_deletion_info.file_path = "/file/path"; + file_deletion_info.job_id = INT_MAX; + file_deletion_info.status = status; + + el->OnTableFileDeleted(file_deletion_info); + + CompactionJobInfo compaction_job_info; + compaction_job_info.cf_id = UINT32_MAX; + compaction_job_info.cf_name = "compactionColumnFamily"; + compaction_job_info.status = status; + compaction_job_info.thread_id = UINT64_MAX; + compaction_job_info.job_id = INT_MAX; + compaction_job_info.base_input_level = INT_MAX; + compaction_job_info.output_level = INT_MAX; + compaction_job_info.input_files = {"inputFile.sst"}; + compaction_job_info.input_file_infos = {}; + compaction_job_info.output_files = {"outputFile.sst"}; + compaction_job_info.output_file_infos = {}; + compaction_job_info.table_properties = { + {"tableProperties", std::shared_ptr( + &table_properties, [](TableProperties *) {})}}; + compaction_job_info.compaction_reason = CompactionReason::kFlush; + compaction_job_info.compression = CompressionType::kSnappyCompression; + + compaction_job_info.stats = CompactionJobStats(); + + el->OnCompactionBegin(nullptr, compaction_job_info); + el->OnCompactionCompleted(nullptr, compaction_job_info); + + TableFileCreationInfo file_creation_info; + file_creation_info.file_size = UINT64_MAX; + file_creation_info.table_properties = table_properties; + file_creation_info.status = status; + file_creation_info.file_checksum = "fileChecksum"; + file_creation_info.file_checksum_func_name = "fileChecksumFuncName"; + file_creation_info.db_name = "dbName"; + file_creation_info.cf_name = "columnFamilyName"; + file_creation_info.file_path = "/file/path"; + file_creation_info.job_id = INT_MAX; + file_creation_info.reason = TableFileCreationReason::kMisc; + + el->OnTableFileCreated(file_creation_info); + + TableFileCreationBriefInfo file_creation_brief_info; + file_creation_brief_info.db_name = "dbName"; + file_creation_brief_info.cf_name = "columnFamilyName"; + file_creation_brief_info.file_path = "/file/path"; + file_creation_brief_info.job_id = INT_MAX; + file_creation_brief_info.reason = TableFileCreationReason::kMisc; + + el->OnTableFileCreationStarted(file_creation_brief_info); + + MemTableInfo mem_table_info; + mem_table_info.cf_name = "columnFamilyName"; + mem_table_info.first_seqno = UINT64_MAX; + mem_table_info.earliest_seqno = UINT64_MAX; + mem_table_info.num_entries = UINT64_MAX; + mem_table_info.num_deletes = UINT64_MAX; + + el->OnMemTableSealed(mem_table_info); + el->OnColumnFamilyHandleDeletionStarted(nullptr); + + ExternalFileIngestionInfo file_ingestion_info; + file_ingestion_info.cf_name = "columnFamilyName"; + file_ingestion_info.external_file_path = "/external/file/path"; + file_ingestion_info.internal_file_path = "/internal/file/path"; + file_ingestion_info.global_seqno = UINT64_MAX; + file_ingestion_info.table_properties = table_properties; + el->OnExternalFileIngested(nullptr, file_ingestion_info); + + el->OnBackgroundError(BackgroundErrorReason::kFlush, &status); + + WriteStallInfo write_stall_info; + write_stall_info.cf_name = "columnFamilyName"; + write_stall_info.condition.cur = WriteStallCondition::kDelayed; + write_stall_info.condition.prev = WriteStallCondition::kStopped; + el->OnStallConditionsChanged(write_stall_info); + + const std::string file_path = "/file/path"; + const auto start_timestamp = + std::make_pair(std::chrono::time_point( + std::chrono::nanoseconds(1600699420000000000ll)), + std::chrono::time_point( + std::chrono::nanoseconds(1600699420000000000ll))); + const auto finish_timestamp = + std::chrono::time_point( + std::chrono::nanoseconds(1600699425000000000ll)); + FileOperationInfo op_info = + FileOperationInfo(FileOperationType::kRead, file_path, start_timestamp, + finish_timestamp, status); + op_info.offset = UINT64_MAX; + op_info.length = SIZE_MAX; + + el->OnFileReadFinish(op_info); + el->OnFileWriteFinish(op_info); + el->OnFileFlushFinish(op_info); + el->OnFileSyncFinish(op_info); + el->OnFileRangeSyncFinish(op_info); + el->OnFileTruncateFinish(op_info); + el->OnFileCloseFinish(op_info); + el->ShouldBeNotifiedOnFileIO(); + + bool auto_recovery; + el->OnErrorRecoveryBegin(BackgroundErrorReason::kFlush, status, + &auto_recovery); + el->OnErrorRecoveryCompleted(status); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/thread_status.cc b/librocksdb-sys/rocksdb/java/rocksjni/thread_status.cc new file mode 100644 index 0000000..c600f6c --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/thread_status.cc @@ -0,0 +1,125 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling c++ ROCKSDB_NAMESPACE::ThreadStatus methods from Java side. + +#include "rocksdb/thread_status.h" + +#include + +#include "include/org_rocksdb_ThreadStatus.h" +#include "portal.h" + +/* + * Class: org_rocksdb_ThreadStatus + * Method: getThreadTypeName + * Signature: (B)Ljava/lang/String; + */ +jstring Java_org_rocksdb_ThreadStatus_getThreadTypeName( + JNIEnv* env, jclass, jbyte jthread_type_value) { + auto name = ROCKSDB_NAMESPACE::ThreadStatus::GetThreadTypeName( + ROCKSDB_NAMESPACE::ThreadTypeJni::toCppThreadType(jthread_type_value)); + return ROCKSDB_NAMESPACE::JniUtil::toJavaString(env, &name, true); +} + +/* + * Class: org_rocksdb_ThreadStatus + * Method: getOperationName + * Signature: (B)Ljava/lang/String; + */ +jstring Java_org_rocksdb_ThreadStatus_getOperationName( + JNIEnv* env, jclass, jbyte joperation_type_value) { + auto name = ROCKSDB_NAMESPACE::ThreadStatus::GetOperationName( + ROCKSDB_NAMESPACE::OperationTypeJni::toCppOperationType( + joperation_type_value)); + return ROCKSDB_NAMESPACE::JniUtil::toJavaString(env, &name, true); +} + +/* + * Class: org_rocksdb_ThreadStatus + * Method: microsToStringNative + * Signature: (J)Ljava/lang/String; + */ +jstring Java_org_rocksdb_ThreadStatus_microsToStringNative(JNIEnv* env, jclass, + jlong jmicros) { + auto str = ROCKSDB_NAMESPACE::ThreadStatus::MicrosToString( + static_cast(jmicros)); + return ROCKSDB_NAMESPACE::JniUtil::toJavaString(env, &str, true); +} + +/* + * Class: org_rocksdb_ThreadStatus + * Method: getOperationStageName + * Signature: (B)Ljava/lang/String; + */ +jstring Java_org_rocksdb_ThreadStatus_getOperationStageName( + JNIEnv* env, jclass, jbyte joperation_stage_value) { + auto name = ROCKSDB_NAMESPACE::ThreadStatus::GetOperationStageName( + ROCKSDB_NAMESPACE::OperationStageJni::toCppOperationStage( + joperation_stage_value)); + return ROCKSDB_NAMESPACE::JniUtil::toJavaString(env, &name, true); +} + +/* + * Class: org_rocksdb_ThreadStatus + * Method: getOperationPropertyName + * Signature: (BI)Ljava/lang/String; + */ +jstring Java_org_rocksdb_ThreadStatus_getOperationPropertyName( + JNIEnv* env, jclass, jbyte joperation_type_value, jint jindex) { + auto name = ROCKSDB_NAMESPACE::ThreadStatus::GetOperationPropertyName( + ROCKSDB_NAMESPACE::OperationTypeJni::toCppOperationType( + joperation_type_value), + static_cast(jindex)); + return ROCKSDB_NAMESPACE::JniUtil::toJavaString(env, &name, true); +} + +/* + * Class: org_rocksdb_ThreadStatus + * Method: interpretOperationProperties + * Signature: (B[J)Ljava/util/Map; + */ +jobject Java_org_rocksdb_ThreadStatus_interpretOperationProperties( + JNIEnv* env, jclass, jbyte joperation_type_value, + jlongArray joperation_properties) { + // convert joperation_properties + const jsize len = env->GetArrayLength(joperation_properties); + const std::unique_ptr op_properties(new uint64_t[len]); + jlong* jop = env->GetLongArrayElements(joperation_properties, nullptr); + if (jop == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + for (jsize i = 0; i < len; i++) { + op_properties[i] = static_cast(jop[i]); + } + env->ReleaseLongArrayElements(joperation_properties, jop, JNI_ABORT); + + // call the function + auto result = ROCKSDB_NAMESPACE::ThreadStatus::InterpretOperationProperties( + ROCKSDB_NAMESPACE::OperationTypeJni::toCppOperationType( + joperation_type_value), + op_properties.get()); + jobject jresult = ROCKSDB_NAMESPACE::HashMapJni::fromCppMap(env, &result); + if (env->ExceptionCheck()) { + // exception occurred + return nullptr; + } + + return jresult; +} + +/* + * Class: org_rocksdb_ThreadStatus + * Method: getStateName + * Signature: (B)Ljava/lang/String; + */ +jstring Java_org_rocksdb_ThreadStatus_getStateName(JNIEnv* env, jclass, + jbyte jstate_type_value) { + auto name = ROCKSDB_NAMESPACE::ThreadStatus::GetStateName( + ROCKSDB_NAMESPACE::StateTypeJni::toCppStateType(jstate_type_value)); + return ROCKSDB_NAMESPACE::JniUtil::toJavaString(env, &name, true); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/trace_writer.cc b/librocksdb-sys/rocksdb/java/rocksjni/trace_writer.cc new file mode 100644 index 0000000..d582763 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/trace_writer.cc @@ -0,0 +1,24 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::CompactionFilterFactory. + +#include + +#include "include/org_rocksdb_AbstractTraceWriter.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/trace_writer_jnicallback.h" + +/* + * Class: org_rocksdb_AbstractTraceWriter + * Method: createNewTraceWriter + * Signature: ()J + */ +jlong Java_org_rocksdb_AbstractTraceWriter_createNewTraceWriter(JNIEnv* env, + jobject jobj) { + auto* trace_writer = new ROCKSDB_NAMESPACE::TraceWriterJniCallback(env, jobj); + return GET_CPLUSPLUS_POINTER(trace_writer); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/trace_writer_jnicallback.cc b/librocksdb-sys/rocksdb/java/rocksjni/trace_writer_jnicallback.cc new file mode 100644 index 0000000..d1ed320 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/trace_writer_jnicallback.cc @@ -0,0 +1,118 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the callback "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::TraceWriter. + +#include "rocksjni/trace_writer_jnicallback.h" + +#include "rocksjni/portal.h" + +namespace ROCKSDB_NAMESPACE { +TraceWriterJniCallback::TraceWriterJniCallback(JNIEnv* env, + jobject jtrace_writer) + : JniCallback(env, jtrace_writer) { + m_jwrite_proxy_methodid = AbstractTraceWriterJni::getWriteProxyMethodId(env); + if (m_jwrite_proxy_methodid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return; + } + + m_jclose_writer_proxy_methodid = + AbstractTraceWriterJni::getCloseWriterProxyMethodId(env); + if (m_jclose_writer_proxy_methodid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return; + } + + m_jget_file_size_methodid = + AbstractTraceWriterJni::getGetFileSizeMethodId(env); + if (m_jget_file_size_methodid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return; + } +} + +Status TraceWriterJniCallback::Write(const Slice& data) { + jboolean attached_thread = JNI_FALSE; + JNIEnv* env = getJniEnv(&attached_thread); + if (env == nullptr) { + return Status::IOError("Unable to attach JNI Environment"); + } + + jshort jstatus = + env->CallShortMethod(m_jcallback_obj, m_jwrite_proxy_methodid, &data); + + if (env->ExceptionCheck()) { + // exception thrown from CallShortMethod + env->ExceptionDescribe(); // print out exception to stderr + releaseJniEnv(attached_thread); + return Status::IOError( + "Unable to call AbstractTraceWriter#writeProxy(long)"); + } + + // unpack status code and status sub-code from jstatus + jbyte jcode_value = (jstatus >> 8) & 0xFF; + jbyte jsub_code_value = jstatus & 0xFF; + std::unique_ptr s = + StatusJni::toCppStatus(jcode_value, jsub_code_value); + + releaseJniEnv(attached_thread); + + return Status(*s); +} + +Status TraceWriterJniCallback::Close() { + jboolean attached_thread = JNI_FALSE; + JNIEnv* env = getJniEnv(&attached_thread); + if (env == nullptr) { + return Status::IOError("Unable to attach JNI Environment"); + } + + jshort jstatus = + env->CallShortMethod(m_jcallback_obj, m_jclose_writer_proxy_methodid); + + if (env->ExceptionCheck()) { + // exception thrown from CallShortMethod + env->ExceptionDescribe(); // print out exception to stderr + releaseJniEnv(attached_thread); + return Status::IOError( + "Unable to call AbstractTraceWriter#closeWriterProxy()"); + } + + // unpack status code and status sub-code from jstatus + jbyte code_value = (jstatus >> 8) & 0xFF; + jbyte sub_code_value = jstatus & 0xFF; + std::unique_ptr s = + StatusJni::toCppStatus(code_value, sub_code_value); + + releaseJniEnv(attached_thread); + + return Status(*s); +} + +uint64_t TraceWriterJniCallback::GetFileSize() { + jboolean attached_thread = JNI_FALSE; + JNIEnv* env = getJniEnv(&attached_thread); + if (env == nullptr) { + return 0; + } + + jlong jfile_size = + env->CallLongMethod(m_jcallback_obj, m_jget_file_size_methodid); + + if (env->ExceptionCheck()) { + // exception thrown from CallLongMethod + env->ExceptionDescribe(); // print out exception to stderr + releaseJniEnv(attached_thread); + return 0; + } + + releaseJniEnv(attached_thread); + + return static_cast(jfile_size); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/java/rocksjni/trace_writer_jnicallback.h b/librocksdb-sys/rocksdb/java/rocksjni/trace_writer_jnicallback.h new file mode 100644 index 0000000..c82a3a7 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/trace_writer_jnicallback.h @@ -0,0 +1,36 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the callback "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::TraceWriter. + +#ifndef JAVA_ROCKSJNI_TRACE_WRITER_JNICALLBACK_H_ +#define JAVA_ROCKSJNI_TRACE_WRITER_JNICALLBACK_H_ + +#include + +#include + +#include "rocksdb/trace_reader_writer.h" +#include "rocksjni/jnicallback.h" + +namespace ROCKSDB_NAMESPACE { + +class TraceWriterJniCallback : public JniCallback, public TraceWriter { + public: + TraceWriterJniCallback(JNIEnv* env, jobject jtrace_writer); + virtual Status Write(const Slice& data); + virtual Status Close(); + virtual uint64_t GetFileSize(); + + private: + jmethodID m_jwrite_proxy_methodid; + jmethodID m_jclose_writer_proxy_methodid; + jmethodID m_jget_file_size_methodid; +}; + +} // namespace ROCKSDB_NAMESPACE + +#endif // JAVA_ROCKSJNI_TRACE_WRITER_JNICALLBACK_H_ diff --git a/librocksdb-sys/rocksdb/java/rocksjni/transaction.cc b/librocksdb-sys/rocksdb/java/rocksjni/transaction.cc new file mode 100644 index 0000000..1a0a64f --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/transaction.cc @@ -0,0 +1,1655 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ +// for ROCKSDB_NAMESPACE::Transaction. + +#include "rocksdb/utilities/transaction.h" + +#include + +#include + +#include "include/org_rocksdb_Transaction.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4503) // identifier' : decorated name length + // exceeded, name was truncated +#endif + +/* + * Class: org_rocksdb_Transaction + * Method: setSnapshot + * Signature: (J)V + */ +void Java_org_rocksdb_Transaction_setSnapshot(JNIEnv* /*env*/, jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + txn->SetSnapshot(); +} + +/* + * Class: org_rocksdb_Transaction + * Method: setSnapshotOnNextOperation + * Signature: (J)V + */ +void Java_org_rocksdb_Transaction_setSnapshotOnNextOperation__J( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + txn->SetSnapshotOnNextOperation(nullptr); +} + +/* + * Class: org_rocksdb_Transaction + * Method: setSnapshotOnNextOperation + * Signature: (JJ)V + */ +void Java_org_rocksdb_Transaction_setSnapshotOnNextOperation__JJ( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, + jlong jtxn_notifier_handle) { + auto* txn = reinterpret_cast(jhandle); + auto* txn_notifier = reinterpret_cast< + std::shared_ptr*>( + jtxn_notifier_handle); + txn->SetSnapshotOnNextOperation(*txn_notifier); +} + +/* + * Class: org_rocksdb_Transaction + * Method: getSnapshot + * Signature: (J)J + */ +jlong Java_org_rocksdb_Transaction_getSnapshot(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + const ROCKSDB_NAMESPACE::Snapshot* snapshot = txn->GetSnapshot(); + return GET_CPLUSPLUS_POINTER(snapshot); +} + +/* + * Class: org_rocksdb_Transaction + * Method: clearSnapshot + * Signature: (J)V + */ +void Java_org_rocksdb_Transaction_clearSnapshot(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + txn->ClearSnapshot(); +} + +/* + * Class: org_rocksdb_Transaction + * Method: prepare + * Signature: (J)V + */ +void Java_org_rocksdb_Transaction_prepare(JNIEnv* env, jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + ROCKSDB_NAMESPACE::Status s = txn->Prepare(); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_Transaction + * Method: commit + * Signature: (J)V + */ +void Java_org_rocksdb_Transaction_commit(JNIEnv* env, jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + ROCKSDB_NAMESPACE::Status s = txn->Commit(); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_Transaction + * Method: rollback + * Signature: (J)V + */ +void Java_org_rocksdb_Transaction_rollback(JNIEnv* env, jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + ROCKSDB_NAMESPACE::Status s = txn->Rollback(); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_Transaction + * Method: setSavePoint + * Signature: (J)V + */ +void Java_org_rocksdb_Transaction_setSavePoint(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + txn->SetSavePoint(); +} + +/* + * Class: org_rocksdb_Transaction + * Method: rollbackToSavePoint + * Signature: (J)V + */ +void Java_org_rocksdb_Transaction_rollbackToSavePoint(JNIEnv* env, + jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + ROCKSDB_NAMESPACE::Status s = txn->RollbackToSavePoint(); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +typedef std::function + FnGet; + +// TODO(AR) consider refactoring to share this between here and rocksjni.cc +jbyteArray txn_get_helper(JNIEnv* env, const FnGet& fn_get, + const jlong& jread_options_handle, + const jbyteArray& jkey, const jint& jkey_part_len) { + jbyte* key = env->GetByteArrayElements(jkey, nullptr); + if (key == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), + jkey_part_len); + + auto* read_options = + reinterpret_cast(jread_options_handle); + std::string value; + ROCKSDB_NAMESPACE::Status s = fn_get(*read_options, key_slice, &value); + + // trigger java unref on key. + // by passing JNI_ABORT, it will simply release the reference without + // copying the result back to the java byte array. + env->ReleaseByteArrayElements(jkey, key, JNI_ABORT); + + if (s.IsNotFound()) { + return nullptr; + } + + if (s.ok()) { + jbyteArray jret_value = env->NewByteArray(static_cast(value.size())); + if (jret_value == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + env->SetByteArrayRegion( + jret_value, 0, static_cast(value.size()), + const_cast(reinterpret_cast(value.c_str()))); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + return nullptr; + } + return jret_value; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return nullptr; +} + +/* + * Class: org_rocksdb_Transaction + * Method: get + * Signature: (JJ[BIJ)[B + */ +jbyteArray Java_org_rocksdb_Transaction_get__JJ_3BIJ( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jlong jread_options_handle, + jbyteArray jkey, jint jkey_part_len, jlong jcolumn_family_handle) { + auto* txn = reinterpret_cast(jhandle); + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + FnGet fn_get = + std::bind( + &ROCKSDB_NAMESPACE::Transaction::Get, txn, std::placeholders::_1, + column_family_handle, std::placeholders::_2, std::placeholders::_3); + return txn_get_helper(env, fn_get, jread_options_handle, jkey, jkey_part_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: get + * Signature: (JJ[BI)[B + */ +jbyteArray Java_org_rocksdb_Transaction_get__JJ_3BI( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jlong jread_options_handle, + jbyteArray jkey, jint jkey_part_len) { + auto* txn = reinterpret_cast(jhandle); + FnGet fn_get = + std::bind( + &ROCKSDB_NAMESPACE::Transaction::Get, txn, std::placeholders::_1, + std::placeholders::_2, std::placeholders::_3); + return txn_get_helper(env, fn_get, jread_options_handle, jkey, jkey_part_len); +} + +// TODO(AR) consider refactoring to share this between here and rocksjni.cc +// used by txn_multi_get_helper below +std::vector txn_column_families_helper( + JNIEnv* env, jlongArray jcolumn_family_handles, bool* has_exception) { + std::vector cf_handles; + if (jcolumn_family_handles != nullptr) { + const jsize len_cols = env->GetArrayLength(jcolumn_family_handles); + if (len_cols > 0) { + jlong* jcfh = env->GetLongArrayElements(jcolumn_family_handles, nullptr); + if (jcfh == nullptr) { + // exception thrown: OutOfMemoryError + *has_exception = JNI_TRUE; + return std::vector(); + } + for (int i = 0; i < len_cols; i++) { + auto* cf_handle = + reinterpret_cast(jcfh[i]); + cf_handles.push_back(cf_handle); + } + env->ReleaseLongArrayElements(jcolumn_family_handles, jcfh, JNI_ABORT); + } + } + return cf_handles; +} + +typedef std::function( + const ROCKSDB_NAMESPACE::ReadOptions&, + const std::vector&, std::vector*)> + FnMultiGet; + +void free_parts( + JNIEnv* env, + std::vector>& parts_to_free) { + for (auto& value : parts_to_free) { + jobject jk; + jbyteArray jk_ba; + jbyte* jk_val; + std::tie(jk_ba, jk_val, jk) = value; + env->ReleaseByteArrayElements(jk_ba, jk_val, JNI_ABORT); + env->DeleteLocalRef(jk); + } +} + +void free_key_values(std::vector& keys_to_free) { + for (auto& key : keys_to_free) { + delete[] key; + } +} + +// TODO(AR) consider refactoring to share this between here and rocksjni.cc +// cf multi get +jobjectArray txn_multi_get_helper(JNIEnv* env, const FnMultiGet& fn_multi_get, + const jlong& jread_options_handle, + const jobjectArray& jkey_parts) { + const jsize len_key_parts = env->GetArrayLength(jkey_parts); + + std::vector key_parts; + std::vector keys_to_free; + for (int i = 0; i < len_key_parts; i++) { + const jobject jk = env->GetObjectArrayElement(jkey_parts, i); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + free_key_values(keys_to_free); + return nullptr; + } + jbyteArray jk_ba = reinterpret_cast(jk); + const jsize len_key = env->GetArrayLength(jk_ba); + jbyte* jk_val = new jbyte[len_key]; + if (jk_val == nullptr) { + // exception thrown: OutOfMemoryError + env->DeleteLocalRef(jk); + free_key_values(keys_to_free); + + jclass exception_cls = (env)->FindClass("java/lang/OutOfMemoryError"); + (env)->ThrowNew(exception_cls, + "Insufficient Memory for CF handle array."); + return nullptr; + } + env->GetByteArrayRegion(jk_ba, 0, len_key, jk_val); + + ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(jk_val), + len_key); + key_parts.push_back(key_slice); + keys_to_free.push_back(jk_val); + env->DeleteLocalRef(jk); + } + + auto* read_options = + reinterpret_cast(jread_options_handle); + std::vector value_parts; + std::vector s = + fn_multi_get(*read_options, key_parts, &value_parts); + + // free up allocated byte arrays + free_key_values(keys_to_free); + + // prepare the results + const jclass jcls_ba = env->FindClass("[B"); + jobjectArray jresults = + env->NewObjectArray(static_cast(s.size()), jcls_ba, nullptr); + if (jresults == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + // add to the jresults + for (std::vector::size_type i = 0; i != s.size(); + i++) { + if (s[i].ok()) { + jbyteArray jentry_value = + env->NewByteArray(static_cast(value_parts[i].size())); + if (jentry_value == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + env->SetByteArrayRegion( + jentry_value, 0, static_cast(value_parts[i].size()), + const_cast( + reinterpret_cast(value_parts[i].c_str()))); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jentry_value); + return nullptr; + } + + env->SetObjectArrayElement(jresults, static_cast(i), jentry_value); + env->DeleteLocalRef(jentry_value); + } + } + + return jresults; +} + +/* + * Class: org_rocksdb_Transaction + * Method: multiGet + * Signature: (JJ[[B[J)[[B + */ +jobjectArray Java_org_rocksdb_Transaction_multiGet__JJ_3_3B_3J( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jlong jread_options_handle, + jobjectArray jkey_parts, jlongArray jcolumn_family_handles) { + bool has_exception = false; + const std::vector + column_family_handles = txn_column_families_helper( + env, jcolumn_family_handles, &has_exception); + if (has_exception) { + // exception thrown: OutOfMemoryError + return nullptr; + } + auto* txn = reinterpret_cast(jhandle); + FnMultiGet fn_multi_get = std::bind ( + ROCKSDB_NAMESPACE::Transaction::*)( + const ROCKSDB_NAMESPACE::ReadOptions&, + const std::vector&, + const std::vector&, std::vector*)>( + &ROCKSDB_NAMESPACE::Transaction::MultiGet, txn, std::placeholders::_1, + column_family_handles, std::placeholders::_2, std::placeholders::_3); + return txn_multi_get_helper(env, fn_multi_get, jread_options_handle, + jkey_parts); +} + +/* + * Class: org_rocksdb_Transaction + * Method: multiGet + * Signature: (JJ[[B)[[B + */ +jobjectArray Java_org_rocksdb_Transaction_multiGet__JJ_3_3B( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jlong jread_options_handle, + jobjectArray jkey_parts) { + auto* txn = reinterpret_cast(jhandle); + FnMultiGet fn_multi_get = std::bind ( + ROCKSDB_NAMESPACE::Transaction::*)( + const ROCKSDB_NAMESPACE::ReadOptions&, + const std::vector&, std::vector*)>( + &ROCKSDB_NAMESPACE::Transaction::MultiGet, txn, std::placeholders::_1, + std::placeholders::_2, std::placeholders::_3); + return txn_multi_get_helper(env, fn_multi_get, jread_options_handle, + jkey_parts); +} + +/* + * Class: org_rocksdb_Transaction + * Method: getForUpdate + * Signature: (JJ[BIJZZ)[B + */ +jbyteArray Java_org_rocksdb_Transaction_getForUpdate__JJ_3BIJZZ( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jlong jread_options_handle, + jbyteArray jkey, jint jkey_part_len, jlong jcolumn_family_handle, + jboolean jexclusive, jboolean jdo_validate) { + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + auto* txn = reinterpret_cast(jhandle); + FnGet fn_get_for_update = + std::bind( + &ROCKSDB_NAMESPACE::Transaction::GetForUpdate, txn, + std::placeholders::_1, column_family_handle, std::placeholders::_2, + std::placeholders::_3, jexclusive, jdo_validate); + return txn_get_helper(env, fn_get_for_update, jread_options_handle, jkey, + jkey_part_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: getForUpdate + * Signature: (JJ[BIZZ)[B + */ +jbyteArray Java_org_rocksdb_Transaction_getForUpdate__JJ_3BIZZ( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jlong jread_options_handle, + jbyteArray jkey, jint jkey_part_len, jboolean jexclusive, + jboolean jdo_validate) { + auto* txn = reinterpret_cast(jhandle); + FnGet fn_get_for_update = + std::bind( + &ROCKSDB_NAMESPACE::Transaction::GetForUpdate, txn, + std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, + jexclusive, jdo_validate); + return txn_get_helper(env, fn_get_for_update, jread_options_handle, jkey, + jkey_part_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: multiGetForUpdate + * Signature: (JJ[[B[J)[[B + */ +jobjectArray Java_org_rocksdb_Transaction_multiGetForUpdate__JJ_3_3B_3J( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jlong jread_options_handle, + jobjectArray jkey_parts, jlongArray jcolumn_family_handles) { + bool has_exception = false; + const std::vector + column_family_handles = txn_column_families_helper( + env, jcolumn_family_handles, &has_exception); + if (has_exception) { + // exception thrown: OutOfMemoryError + return nullptr; + } + auto* txn = reinterpret_cast(jhandle); + FnMultiGet fn_multi_get_for_update = std::bind (ROCKSDB_NAMESPACE::Transaction::*)( + const ROCKSDB_NAMESPACE::ReadOptions&, + const std::vector&, + const std::vector&, std::vector*)>( + &ROCKSDB_NAMESPACE::Transaction::MultiGetForUpdate, txn, + std::placeholders::_1, column_family_handles, std::placeholders::_2, + std::placeholders::_3); + return txn_multi_get_helper(env, fn_multi_get_for_update, + jread_options_handle, jkey_parts); +} + +/* + * Class: org_rocksdb_Transaction + * Method: multiGetForUpdate + * Signature: (JJ[[B)[[B + */ +jobjectArray Java_org_rocksdb_Transaction_multiGetForUpdate__JJ_3_3B( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jlong jread_options_handle, + jobjectArray jkey_parts) { + auto* txn = reinterpret_cast(jhandle); + FnMultiGet fn_multi_get_for_update = std::bind (ROCKSDB_NAMESPACE::Transaction::*)( + const ROCKSDB_NAMESPACE::ReadOptions&, + const std::vector&, std::vector*)>( + &ROCKSDB_NAMESPACE::Transaction::MultiGetForUpdate, txn, + std::placeholders::_1, std::placeholders::_2, std::placeholders::_3); + return txn_multi_get_helper(env, fn_multi_get_for_update, + jread_options_handle, jkey_parts); +} + +/* + * Class: org_rocksdb_Transaction + * Method: getIterator + * Signature: (JJ)J + */ +jlong Java_org_rocksdb_Transaction_getIterator__JJ(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle, + jlong jread_options_handle) { + auto* txn = reinterpret_cast(jhandle); + auto* read_options = + reinterpret_cast(jread_options_handle); + return GET_CPLUSPLUS_POINTER(txn->GetIterator(*read_options)); +} + +/* + * Class: org_rocksdb_Transaction + * Method: getIterator + * Signature: (JJJ)J + */ +jlong Java_org_rocksdb_Transaction_getIterator__JJJ( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, + jlong jread_options_handle, jlong jcolumn_family_handle) { + auto* txn = reinterpret_cast(jhandle); + auto* read_options = + reinterpret_cast(jread_options_handle); + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + return GET_CPLUSPLUS_POINTER( + txn->GetIterator(*read_options, column_family_handle)); +} + +typedef std::function + FnWriteKV; + +// TODO(AR) consider refactoring to share this between here and rocksjni.cc +void txn_write_kv_helper(JNIEnv* env, const FnWriteKV& fn_write_kv, + const jbyteArray& jkey, const jint& jkey_part_len, + const jbyteArray& jval, const jint& jval_len) { + jbyte* key = env->GetByteArrayElements(jkey, nullptr); + if (key == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + jbyte* value = env->GetByteArrayElements(jval, nullptr); + if (value == nullptr) { + // exception thrown: OutOfMemoryError + env->ReleaseByteArrayElements(jkey, key, JNI_ABORT); + return; + } + ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), + jkey_part_len); + ROCKSDB_NAMESPACE::Slice value_slice(reinterpret_cast(value), + jval_len); + + ROCKSDB_NAMESPACE::Status s = fn_write_kv(key_slice, value_slice); + + // trigger java unref on key. + // by passing JNI_ABORT, it will simply release the reference without + // copying the result back to the java byte array. + env->ReleaseByteArrayElements(jval, value, JNI_ABORT); + env->ReleaseByteArrayElements(jkey, key, JNI_ABORT); + + if (s.ok()) { + return; + } + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); +} + +/* + * Class: org_rocksdb_Transaction + * Method: put + * Signature: (J[BI[BIJZ)V + */ +void Java_org_rocksdb_Transaction_put__J_3BI_3BIJZ( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, + jint jkey_part_len, jbyteArray jval, jint jval_len, + jlong jcolumn_family_handle, jboolean jassume_tracked) { + auto* txn = reinterpret_cast(jhandle); + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + FnWriteKV fn_put = + std::bind(&ROCKSDB_NAMESPACE::Transaction::Put, txn, + column_family_handle, std::placeholders::_1, + std::placeholders::_2, jassume_tracked); + txn_write_kv_helper(env, fn_put, jkey, jkey_part_len, jval, jval_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: put + * Signature: (J[BI[BI)V + */ +void Java_org_rocksdb_Transaction_put__J_3BI_3BI(JNIEnv* env, jobject /*jobj*/, + jlong jhandle, jbyteArray jkey, + jint jkey_part_len, + jbyteArray jval, + jint jval_len) { + auto* txn = reinterpret_cast(jhandle); + FnWriteKV fn_put = + std::bind( + &ROCKSDB_NAMESPACE::Transaction::Put, txn, std::placeholders::_1, + std::placeholders::_2); + txn_write_kv_helper(env, fn_put, jkey, jkey_part_len, jval, jval_len); +} + +typedef std::function + FnWriteKVParts; + +// TODO(AR) consider refactoring to share this between here and rocksjni.cc +void txn_write_kv_parts_helper(JNIEnv* env, + const FnWriteKVParts& fn_write_kv_parts, + const jobjectArray& jkey_parts, + const jint& jkey_parts_len, + const jobjectArray& jvalue_parts, + const jint& jvalue_parts_len) { +#ifndef DEBUG + (void)jvalue_parts_len; +#else + assert(jkey_parts_len == jvalue_parts_len); +#endif + + auto key_parts = std::vector(); + auto value_parts = std::vector(); + auto jparts_to_free = std::vector>(); + + // Since this is fundamentally a gather write at the RocksDB level, + // it seems wrong to refactor it by copying (gathering) keys and data here, + // in order to avoid the local reference limit. + // The user needs to be a aware that there is a limit to the number of parts + // which can be gathered. + if (env->EnsureLocalCapacity(jkey_parts_len + jvalue_parts_len) != 0) { + // no space for all the jobjects we store up + env->ExceptionClear(); + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, "Insufficient JNI local references for " + + std::to_string(jkey_parts_len) + " key/value parts"); + return; + } + + // convert java key_parts/value_parts byte[][] to Slice(s) + for (jsize i = 0; i < jkey_parts_len; ++i) { + const jobject jobj_key_part = env->GetObjectArrayElement(jkey_parts, i); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + free_parts(env, jparts_to_free); + return; + } + const jobject jobj_value_part = env->GetObjectArrayElement(jvalue_parts, i); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jobj_key_part); + free_parts(env, jparts_to_free); + return; + } + + const jbyteArray jba_key_part = reinterpret_cast(jobj_key_part); + const jsize jkey_part_len = env->GetArrayLength(jba_key_part); + jbyte* jkey_part = env->GetByteArrayElements(jba_key_part, nullptr); + if (jkey_part == nullptr) { + // exception thrown: OutOfMemoryError + env->DeleteLocalRef(jobj_value_part); + env->DeleteLocalRef(jobj_key_part); + free_parts(env, jparts_to_free); + return; + } + + const jbyteArray jba_value_part = + reinterpret_cast(jobj_value_part); + const jsize jvalue_part_len = env->GetArrayLength(jba_value_part); + jbyte* jvalue_part = env->GetByteArrayElements(jba_value_part, nullptr); + if (jvalue_part == nullptr) { + // exception thrown: OutOfMemoryError + env->DeleteLocalRef(jobj_value_part); + env->DeleteLocalRef(jobj_key_part); + env->ReleaseByteArrayElements(jba_key_part, jkey_part, JNI_ABORT); + free_parts(env, jparts_to_free); + return; + } + + jparts_to_free.push_back( + std::make_tuple(jba_key_part, jkey_part, jobj_key_part)); + jparts_to_free.push_back( + std::make_tuple(jba_value_part, jvalue_part, jobj_value_part)); + + key_parts.push_back(ROCKSDB_NAMESPACE::Slice( + reinterpret_cast(jkey_part), jkey_part_len)); + value_parts.push_back(ROCKSDB_NAMESPACE::Slice( + reinterpret_cast(jvalue_part), jvalue_part_len)); + } + + // call the write_multi function + ROCKSDB_NAMESPACE::Status s = fn_write_kv_parts( + ROCKSDB_NAMESPACE::SliceParts(key_parts.data(), (int)key_parts.size()), + ROCKSDB_NAMESPACE::SliceParts(value_parts.data(), + (int)value_parts.size())); + + // cleanup temporary memory + free_parts(env, jparts_to_free); + + // return + if (s.ok()) { + return; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); +} + +/* + * Class: org_rocksdb_Transaction + * Method: put + * Signature: (J[[BI[[BIJZ)V + */ +void Java_org_rocksdb_Transaction_put__J_3_3BI_3_3BIJZ( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jobjectArray jkey_parts, + jint jkey_parts_len, jobjectArray jvalue_parts, jint jvalue_parts_len, + jlong jcolumn_family_handle, jboolean jassume_tracked) { + auto* txn = reinterpret_cast(jhandle); + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + FnWriteKVParts fn_put_parts = + std::bind( + &ROCKSDB_NAMESPACE::Transaction::Put, txn, column_family_handle, + std::placeholders::_1, std::placeholders::_2, jassume_tracked); + txn_write_kv_parts_helper(env, fn_put_parts, jkey_parts, jkey_parts_len, + jvalue_parts, jvalue_parts_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: put + * Signature: (J[[BI[[BI)V + */ +void Java_org_rocksdb_Transaction_put__J_3_3BI_3_3BI( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jobjectArray jkey_parts, + jint jkey_parts_len, jobjectArray jvalue_parts, jint jvalue_parts_len) { + auto* txn = reinterpret_cast(jhandle); + FnWriteKVParts fn_put_parts = std::bind( + &ROCKSDB_NAMESPACE::Transaction::Put, txn, std::placeholders::_1, + std::placeholders::_2); + txn_write_kv_parts_helper(env, fn_put_parts, jkey_parts, jkey_parts_len, + jvalue_parts, jvalue_parts_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: merge + * Signature: (J[BI[BIJZ)V + */ +void Java_org_rocksdb_Transaction_merge__J_3BI_3BIJZ( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, + jint jkey_part_len, jbyteArray jval, jint jval_len, + jlong jcolumn_family_handle, jboolean jassume_tracked) { + auto* txn = reinterpret_cast(jhandle); + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + FnWriteKV fn_merge = + std::bind(&ROCKSDB_NAMESPACE::Transaction::Merge, txn, + column_family_handle, std::placeholders::_1, + std::placeholders::_2, jassume_tracked); + txn_write_kv_helper(env, fn_merge, jkey, jkey_part_len, jval, jval_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: merge + * Signature: (J[BI[BI)V + */ +void Java_org_rocksdb_Transaction_merge__J_3BI_3BI( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, + jint jkey_part_len, jbyteArray jval, jint jval_len) { + auto* txn = reinterpret_cast(jhandle); + FnWriteKV fn_merge = + std::bind( + &ROCKSDB_NAMESPACE::Transaction::Merge, txn, std::placeholders::_1, + std::placeholders::_2); + txn_write_kv_helper(env, fn_merge, jkey, jkey_part_len, jval, jval_len); +} + +typedef std::function + FnWriteK; + +// TODO(AR) consider refactoring to share this between here and rocksjni.cc +void txn_write_k_helper(JNIEnv* env, const FnWriteK& fn_write_k, + const jbyteArray& jkey, const jint& jkey_part_len) { + jbyte* key = env->GetByteArrayElements(jkey, nullptr); + if (key == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), + jkey_part_len); + + ROCKSDB_NAMESPACE::Status s = fn_write_k(key_slice); + + // trigger java unref on key. + // by passing JNI_ABORT, it will simply release the reference without + // copying the result back to the java byte array. + env->ReleaseByteArrayElements(jkey, key, JNI_ABORT); + + if (s.ok()) { + return; + } + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); +} + +/* + * Class: org_rocksdb_Transaction + * Method: delete + * Signature: (J[BIJZ)V + */ +void Java_org_rocksdb_Transaction_delete__J_3BIJZ( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, + jint jkey_part_len, jlong jcolumn_family_handle, jboolean jassume_tracked) { + auto* txn = reinterpret_cast(jhandle); + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + FnWriteK fn_delete = + std::bind( + &ROCKSDB_NAMESPACE::Transaction::Delete, txn, column_family_handle, + std::placeholders::_1, jassume_tracked); + txn_write_k_helper(env, fn_delete, jkey, jkey_part_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: delete + * Signature: (J[BI)V + */ +void Java_org_rocksdb_Transaction_delete__J_3BI(JNIEnv* env, jobject /*jobj*/, + jlong jhandle, jbyteArray jkey, + jint jkey_part_len) { + auto* txn = reinterpret_cast(jhandle); + FnWriteK fn_delete = std::bind( + &ROCKSDB_NAMESPACE::Transaction::Delete, txn, std::placeholders::_1); + txn_write_k_helper(env, fn_delete, jkey, jkey_part_len); +} + +typedef std::function + FnWriteKParts; + +// TODO(AR) consider refactoring to share this between here and rocksjni.cc +void txn_write_k_parts_helper(JNIEnv* env, + const FnWriteKParts& fn_write_k_parts, + const jobjectArray& jkey_parts, + const jint& jkey_parts_len) { + std::vector key_parts; + std::vector> jkey_parts_to_free; + + // convert java key_parts byte[][] to Slice(s) + for (jint i = 0; i < jkey_parts_len; ++i) { + const jobject jobj_key_part = env->GetObjectArrayElement(jkey_parts, i); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + free_parts(env, jkey_parts_to_free); + return; + } + + const jbyteArray jba_key_part = reinterpret_cast(jobj_key_part); + const jsize jkey_part_len = env->GetArrayLength(jba_key_part); + jbyte* jkey_part = env->GetByteArrayElements(jba_key_part, nullptr); + if (jkey_part == nullptr) { + // exception thrown: OutOfMemoryError + env->DeleteLocalRef(jobj_key_part); + free_parts(env, jkey_parts_to_free); + return; + } + + jkey_parts_to_free.push_back(std::tuple( + jba_key_part, jkey_part, jobj_key_part)); + + key_parts.push_back(ROCKSDB_NAMESPACE::Slice( + reinterpret_cast(jkey_part), jkey_part_len)); + } + + // call the write_multi function + ROCKSDB_NAMESPACE::Status s = fn_write_k_parts( + ROCKSDB_NAMESPACE::SliceParts(key_parts.data(), (int)key_parts.size())); + + // cleanup temporary memory + free_parts(env, jkey_parts_to_free); + + // return + if (s.ok()) { + return; + } + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); +} + +/* + * Class: org_rocksdb_Transaction + * Method: delete + * Signature: (J[[BIJZ)V + */ +void Java_org_rocksdb_Transaction_delete__J_3_3BIJZ( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jobjectArray jkey_parts, + jint jkey_parts_len, jlong jcolumn_family_handle, + jboolean jassume_tracked) { + auto* txn = reinterpret_cast(jhandle); + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + FnWriteKParts fn_delete_parts = + std::bind( + &ROCKSDB_NAMESPACE::Transaction::Delete, txn, column_family_handle, + std::placeholders::_1, jassume_tracked); + txn_write_k_parts_helper(env, fn_delete_parts, jkey_parts, jkey_parts_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: delete + * Signature: (J[[BI)V + */ +void Java_org_rocksdb_Transaction_delete__J_3_3BI(JNIEnv* env, jobject /*jobj*/, + jlong jhandle, + jobjectArray jkey_parts, + jint jkey_parts_len) { + auto* txn = reinterpret_cast(jhandle); + FnWriteKParts fn_delete_parts = std::bind( + &ROCKSDB_NAMESPACE::Transaction::Delete, txn, std::placeholders::_1); + txn_write_k_parts_helper(env, fn_delete_parts, jkey_parts, jkey_parts_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: singleDelete + * Signature: (J[BIJZ)V + */ +void Java_org_rocksdb_Transaction_singleDelete__J_3BIJZ( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, + jint jkey_part_len, jlong jcolumn_family_handle, jboolean jassume_tracked) { + auto* txn = reinterpret_cast(jhandle); + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + FnWriteK fn_single_delete = + std::bind( + &ROCKSDB_NAMESPACE::Transaction::SingleDelete, txn, + column_family_handle, std::placeholders::_1, jassume_tracked); + txn_write_k_helper(env, fn_single_delete, jkey, jkey_part_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: singleDelete + * Signature: (J[BI)V + */ +void Java_org_rocksdb_Transaction_singleDelete__J_3BI(JNIEnv* env, + jobject /*jobj*/, + jlong jhandle, + jbyteArray jkey, + jint jkey_part_len) { + auto* txn = reinterpret_cast(jhandle); + FnWriteK fn_single_delete = std::bind( + &ROCKSDB_NAMESPACE::Transaction::SingleDelete, txn, + std::placeholders::_1); + txn_write_k_helper(env, fn_single_delete, jkey, jkey_part_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: singleDelete + * Signature: (J[[BIJZ)V + */ +void Java_org_rocksdb_Transaction_singleDelete__J_3_3BIJZ( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jobjectArray jkey_parts, + jint jkey_parts_len, jlong jcolumn_family_handle, + jboolean jassume_tracked) { + auto* txn = reinterpret_cast(jhandle); + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + FnWriteKParts fn_single_delete_parts = + std::bind( + &ROCKSDB_NAMESPACE::Transaction::SingleDelete, txn, + column_family_handle, std::placeholders::_1, jassume_tracked); + txn_write_k_parts_helper(env, fn_single_delete_parts, jkey_parts, + jkey_parts_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: singleDelete + * Signature: (J[[BI)V + */ +void Java_org_rocksdb_Transaction_singleDelete__J_3_3BI(JNIEnv* env, + jobject /*jobj*/, + jlong jhandle, + jobjectArray jkey_parts, + jint jkey_parts_len) { + auto* txn = reinterpret_cast(jhandle); + FnWriteKParts fn_single_delete_parts = std::bind( + &ROCKSDB_NAMESPACE::Transaction::SingleDelete, txn, + std::placeholders::_1); + txn_write_k_parts_helper(env, fn_single_delete_parts, jkey_parts, + jkey_parts_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: putUntracked + * Signature: (J[BI[BIJ)V + */ +void Java_org_rocksdb_Transaction_putUntracked__J_3BI_3BIJ( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, + jint jkey_part_len, jbyteArray jval, jint jval_len, + jlong jcolumn_family_handle) { + auto* txn = reinterpret_cast(jhandle); + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + FnWriteKV fn_put_untracked = + std::bind( + &ROCKSDB_NAMESPACE::Transaction::PutUntracked, txn, + column_family_handle, std::placeholders::_1, std::placeholders::_2); + txn_write_kv_helper(env, fn_put_untracked, jkey, jkey_part_len, jval, + jval_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: putUntracked + * Signature: (J[BI[BI)V + */ +void Java_org_rocksdb_Transaction_putUntracked__J_3BI_3BI( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, + jint jkey_part_len, jbyteArray jval, jint jval_len) { + auto* txn = reinterpret_cast(jhandle); + FnWriteKV fn_put_untracked = + std::bind( + &ROCKSDB_NAMESPACE::Transaction::PutUntracked, txn, + std::placeholders::_1, std::placeholders::_2); + txn_write_kv_helper(env, fn_put_untracked, jkey, jkey_part_len, jval, + jval_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: putUntracked + * Signature: (J[[BI[[BIJ)V + */ +void Java_org_rocksdb_Transaction_putUntracked__J_3_3BI_3_3BIJ( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jobjectArray jkey_parts, + jint jkey_parts_len, jobjectArray jvalue_parts, jint jvalue_parts_len, + jlong jcolumn_family_handle) { + auto* txn = reinterpret_cast(jhandle); + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + FnWriteKVParts fn_put_parts_untracked = std::bind( + &ROCKSDB_NAMESPACE::Transaction::PutUntracked, txn, column_family_handle, + std::placeholders::_1, std::placeholders::_2); + txn_write_kv_parts_helper(env, fn_put_parts_untracked, jkey_parts, + jkey_parts_len, jvalue_parts, jvalue_parts_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: putUntracked + * Signature: (J[[BI[[BI)V + */ +void Java_org_rocksdb_Transaction_putUntracked__J_3_3BI_3_3BI( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jobjectArray jkey_parts, + jint jkey_parts_len, jobjectArray jvalue_parts, jint jvalue_parts_len) { + auto* txn = reinterpret_cast(jhandle); + FnWriteKVParts fn_put_parts_untracked = std::bind( + &ROCKSDB_NAMESPACE::Transaction::PutUntracked, txn, std::placeholders::_1, + std::placeholders::_2); + txn_write_kv_parts_helper(env, fn_put_parts_untracked, jkey_parts, + jkey_parts_len, jvalue_parts, jvalue_parts_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: mergeUntracked + * Signature: (J[BI[BIJ)V + */ +void Java_org_rocksdb_Transaction_mergeUntracked__J_3BI_3BIJ( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, + jint jkey_part_len, jbyteArray jval, jint jval_len, + jlong jcolumn_family_handle) { + auto* txn = reinterpret_cast(jhandle); + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + FnWriteKV fn_merge_untracked = + std::bind( + &ROCKSDB_NAMESPACE::Transaction::MergeUntracked, txn, + column_family_handle, std::placeholders::_1, std::placeholders::_2); + txn_write_kv_helper(env, fn_merge_untracked, jkey, jkey_part_len, jval, + jval_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: mergeUntracked + * Signature: (J[BI[BI)V + */ +void Java_org_rocksdb_Transaction_mergeUntracked__J_3BI_3BI( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, + jint jkey_part_len, jbyteArray jval, jint jval_len) { + auto* txn = reinterpret_cast(jhandle); + FnWriteKV fn_merge_untracked = + std::bind( + &ROCKSDB_NAMESPACE::Transaction::MergeUntracked, txn, + std::placeholders::_1, std::placeholders::_2); + txn_write_kv_helper(env, fn_merge_untracked, jkey, jkey_part_len, jval, + jval_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: deleteUntracked + * Signature: (J[BIJ)V + */ +void Java_org_rocksdb_Transaction_deleteUntracked__J_3BIJ( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, + jint jkey_part_len, jlong jcolumn_family_handle) { + auto* txn = reinterpret_cast(jhandle); + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + FnWriteK fn_delete_untracked = std::bind( + &ROCKSDB_NAMESPACE::Transaction::DeleteUntracked, txn, + column_family_handle, std::placeholders::_1); + txn_write_k_helper(env, fn_delete_untracked, jkey, jkey_part_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: deleteUntracked + * Signature: (J[BI)V + */ +void Java_org_rocksdb_Transaction_deleteUntracked__J_3BI(JNIEnv* env, + jobject /*jobj*/, + jlong jhandle, + jbyteArray jkey, + jint jkey_part_len) { + auto* txn = reinterpret_cast(jhandle); + FnWriteK fn_delete_untracked = std::bind( + &ROCKSDB_NAMESPACE::Transaction::DeleteUntracked, txn, + std::placeholders::_1); + txn_write_k_helper(env, fn_delete_untracked, jkey, jkey_part_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: deleteUntracked + * Signature: (J[[BIJ)V + */ +void Java_org_rocksdb_Transaction_deleteUntracked__J_3_3BIJ( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jobjectArray jkey_parts, + jint jkey_parts_len, jlong jcolumn_family_handle) { + auto* txn = reinterpret_cast(jhandle); + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + FnWriteKParts fn_delete_untracked_parts = + std::bind( + &ROCKSDB_NAMESPACE::Transaction::DeleteUntracked, txn, + column_family_handle, std::placeholders::_1); + txn_write_k_parts_helper(env, fn_delete_untracked_parts, jkey_parts, + jkey_parts_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: deleteUntracked + * Signature: (J[[BI)V + */ +void Java_org_rocksdb_Transaction_deleteUntracked__J_3_3BI( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jobjectArray jkey_parts, + jint jkey_parts_len) { + auto* txn = reinterpret_cast(jhandle); + FnWriteKParts fn_delete_untracked_parts = + std::bind( + &ROCKSDB_NAMESPACE::Transaction::DeleteUntracked, txn, + std::placeholders::_1); + txn_write_k_parts_helper(env, fn_delete_untracked_parts, jkey_parts, + jkey_parts_len); +} + +/* + * Class: org_rocksdb_Transaction + * Method: putLogData + * Signature: (J[BI)V + */ +void Java_org_rocksdb_Transaction_putLogData(JNIEnv* env, jobject /*jobj*/, + jlong jhandle, jbyteArray jkey, + jint jkey_part_len) { + auto* txn = reinterpret_cast(jhandle); + + jbyte* key = env->GetByteArrayElements(jkey, nullptr); + if (key == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), + jkey_part_len); + txn->PutLogData(key_slice); + + // trigger java unref on key. + // by passing JNI_ABORT, it will simply release the reference without + // copying the result back to the java byte array. + env->ReleaseByteArrayElements(jkey, key, JNI_ABORT); +} + +/* + * Class: org_rocksdb_Transaction + * Method: disableIndexing + * Signature: (J)V + */ +void Java_org_rocksdb_Transaction_disableIndexing(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + txn->DisableIndexing(); +} + +/* + * Class: org_rocksdb_Transaction + * Method: enableIndexing + * Signature: (J)V + */ +void Java_org_rocksdb_Transaction_enableIndexing(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + txn->EnableIndexing(); +} + +/* + * Class: org_rocksdb_Transaction + * Method: getNumKeys + * Signature: (J)J + */ +jlong Java_org_rocksdb_Transaction_getNumKeys(JNIEnv* /*env*/, jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + return txn->GetNumKeys(); +} + +/* + * Class: org_rocksdb_Transaction + * Method: getNumPuts + * Signature: (J)J + */ +jlong Java_org_rocksdb_Transaction_getNumPuts(JNIEnv* /*env*/, jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + return txn->GetNumPuts(); +} + +/* + * Class: org_rocksdb_Transaction + * Method: getNumDeletes + * Signature: (J)J + */ +jlong Java_org_rocksdb_Transaction_getNumDeletes(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + return txn->GetNumDeletes(); +} + +/* + * Class: org_rocksdb_Transaction + * Method: getNumMerges + * Signature: (J)J + */ +jlong Java_org_rocksdb_Transaction_getNumMerges(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + return txn->GetNumMerges(); +} + +/* + * Class: org_rocksdb_Transaction + * Method: getElapsedTime + * Signature: (J)J + */ +jlong Java_org_rocksdb_Transaction_getElapsedTime(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + return txn->GetElapsedTime(); +} + +/* + * Class: org_rocksdb_Transaction + * Method: getWriteBatch + * Signature: (J)J + */ +jlong Java_org_rocksdb_Transaction_getWriteBatch(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + return GET_CPLUSPLUS_POINTER(txn->GetWriteBatch()); +} + +/* + * Class: org_rocksdb_Transaction + * Method: setLockTimeout + * Signature: (JJ)V + */ +void Java_org_rocksdb_Transaction_setLockTimeout(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle, + jlong jlock_timeout) { + auto* txn = reinterpret_cast(jhandle); + txn->SetLockTimeout(jlock_timeout); +} + +/* + * Class: org_rocksdb_Transaction + * Method: getWriteOptions + * Signature: (J)J + */ +jlong Java_org_rocksdb_Transaction_getWriteOptions(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + return GET_CPLUSPLUS_POINTER(txn->GetWriteOptions()); +} + +/* + * Class: org_rocksdb_Transaction + * Method: setWriteOptions + * Signature: (JJ)V + */ +void Java_org_rocksdb_Transaction_setWriteOptions(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle, + jlong jwrite_options_handle) { + auto* txn = reinterpret_cast(jhandle); + auto* write_options = + reinterpret_cast(jwrite_options_handle); + txn->SetWriteOptions(*write_options); +} + +/* + * Class: org_rocksdb_Transaction + * Method: undo + * Signature: (J[BIJ)V + */ +void Java_org_rocksdb_Transaction_undoGetForUpdate__J_3BIJ( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jbyteArray jkey, + jint jkey_part_len, jlong jcolumn_family_handle) { + auto* txn = reinterpret_cast(jhandle); + auto* column_family_handle = + reinterpret_cast( + jcolumn_family_handle); + jbyte* key = env->GetByteArrayElements(jkey, nullptr); + if (key == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), + jkey_part_len); + txn->UndoGetForUpdate(column_family_handle, key_slice); + + env->ReleaseByteArrayElements(jkey, key, JNI_ABORT); +} + +/* + * Class: org_rocksdb_Transaction + * Method: undoGetForUpdate + * Signature: (J[BI)V + */ +void Java_org_rocksdb_Transaction_undoGetForUpdate__J_3BI(JNIEnv* env, + jobject /*jobj*/, + jlong jhandle, + jbyteArray jkey, + jint jkey_part_len) { + auto* txn = reinterpret_cast(jhandle); + jbyte* key = env->GetByteArrayElements(jkey, nullptr); + if (key == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast(key), + jkey_part_len); + txn->UndoGetForUpdate(key_slice); + + env->ReleaseByteArrayElements(jkey, key, JNI_ABORT); +} + +/* + * Class: org_rocksdb_Transaction + * Method: rebuildFromWriteBatch + * Signature: (JJ)V + */ +void Java_org_rocksdb_Transaction_rebuildFromWriteBatch( + JNIEnv* env, jobject /*jobj*/, jlong jhandle, jlong jwrite_batch_handle) { + auto* txn = reinterpret_cast(jhandle); + auto* write_batch = + reinterpret_cast(jwrite_batch_handle); + ROCKSDB_NAMESPACE::Status s = txn->RebuildFromWriteBatch(write_batch); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_Transaction + * Method: getCommitTimeWriteBatch + * Signature: (J)J + */ +jlong Java_org_rocksdb_Transaction_getCommitTimeWriteBatch(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + return GET_CPLUSPLUS_POINTER(txn->GetCommitTimeWriteBatch()); +} + +/* + * Class: org_rocksdb_Transaction + * Method: setLogNumber + * Signature: (JJ)V + */ +void Java_org_rocksdb_Transaction_setLogNumber(JNIEnv* /*env*/, + jobject /*jobj*/, jlong jhandle, + jlong jlog_number) { + auto* txn = reinterpret_cast(jhandle); + txn->SetLogNumber(jlog_number); +} + +/* + * Class: org_rocksdb_Transaction + * Method: getLogNumber + * Signature: (J)J + */ +jlong Java_org_rocksdb_Transaction_getLogNumber(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + return txn->GetLogNumber(); +} + +/* + * Class: org_rocksdb_Transaction + * Method: setName + * Signature: (JLjava/lang/String;)V + */ +void Java_org_rocksdb_Transaction_setName(JNIEnv* env, jobject /*jobj*/, + jlong jhandle, jstring jname) { + auto* txn = reinterpret_cast(jhandle); + const char* name = env->GetStringUTFChars(jname, nullptr); + if (name == nullptr) { + // exception thrown: OutOfMemoryError + return; + } + + ROCKSDB_NAMESPACE::Status s = txn->SetName(name); + + env->ReleaseStringUTFChars(jname, name); + + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_Transaction + * Method: getName + * Signature: (J)Ljava/lang/String; + */ +jstring Java_org_rocksdb_Transaction_getName(JNIEnv* env, jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + ROCKSDB_NAMESPACE::TransactionName name = txn->GetName(); + return env->NewStringUTF(name.data()); +} + +/* + * Class: org_rocksdb_Transaction + * Method: getID + * Signature: (J)J + */ +jlong Java_org_rocksdb_Transaction_getID(JNIEnv* /*env*/, jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + ROCKSDB_NAMESPACE::TransactionID id = txn->GetID(); + return static_cast(id); +} + +/* + * Class: org_rocksdb_Transaction + * Method: isDeadlockDetect + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Transaction_isDeadlockDetect(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + return static_cast(txn->IsDeadlockDetect()); +} + +/* + * Class: org_rocksdb_Transaction + * Method: getWaitingTxns + * Signature: (J)Lorg/rocksdb/Transaction/WaitingTransactions; + */ +jobject Java_org_rocksdb_Transaction_getWaitingTxns(JNIEnv* env, + jobject jtransaction_obj, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + uint32_t column_family_id; + std::string key; + std::vector waiting_txns = + txn->GetWaitingTxns(&column_family_id, &key); + jobject jwaiting_txns = + ROCKSDB_NAMESPACE::TransactionJni::newWaitingTransactions( + env, jtransaction_obj, column_family_id, key, waiting_txns); + return jwaiting_txns; +} + +/* + * Class: org_rocksdb_Transaction + * Method: getState + * Signature: (J)B + */ +jbyte Java_org_rocksdb_Transaction_getState(JNIEnv* /*env*/, jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + ROCKSDB_NAMESPACE::Transaction::TransactionState txn_status = txn->GetState(); + switch (txn_status) { + case ROCKSDB_NAMESPACE::Transaction::TransactionState::STARTED: + return 0x0; + + case ROCKSDB_NAMESPACE::Transaction::TransactionState::AWAITING_PREPARE: + return 0x1; + + case ROCKSDB_NAMESPACE::Transaction::TransactionState::PREPARED: + return 0x2; + + case ROCKSDB_NAMESPACE::Transaction::TransactionState::AWAITING_COMMIT: + return 0x3; + + case ROCKSDB_NAMESPACE::Transaction::TransactionState::COMMITTED: + return 0x4; + + case ROCKSDB_NAMESPACE::Transaction::TransactionState::AWAITING_ROLLBACK: + return 0x5; + + case ROCKSDB_NAMESPACE::Transaction::TransactionState::ROLLEDBACK: + return 0x6; + + case ROCKSDB_NAMESPACE::Transaction::TransactionState::LOCKS_STOLEN: + return 0x7; + } + + assert(false); + return static_cast(-1); +} + +/* + * Class: org_rocksdb_Transaction + * Method: getId + * Signature: (J)J + */ +jlong Java_org_rocksdb_Transaction_getId(JNIEnv* /*env*/, jobject /*jobj*/, + jlong jhandle) { + auto* txn = reinterpret_cast(jhandle); + uint64_t id = txn->GetId(); + return static_cast(id); +} + +/* + * Class: org_rocksdb_Transaction + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_Transaction_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + delete reinterpret_cast(jhandle); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/transaction_db.cc b/librocksdb-sys/rocksdb/java/rocksjni/transaction_db.cc new file mode 100644 index 0000000..0adf856 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/transaction_db.cc @@ -0,0 +1,451 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ +// for ROCKSDB_NAMESPACE::TransactionDB. + +#include "rocksdb/utilities/transaction_db.h" + +#include + +#include +#include +#include + +#include "include/org_rocksdb_TransactionDB.h" +#include "rocksdb/options.h" +#include "rocksdb/utilities/transaction.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_TransactionDB + * Method: open + * Signature: (JJLjava/lang/String;)J + */ +jlong Java_org_rocksdb_TransactionDB_open__JJLjava_lang_String_2( + JNIEnv* env, jclass, jlong joptions_handle, jlong jtxn_db_options_handle, + jstring jdb_path) { + auto* options = + reinterpret_cast(joptions_handle); + auto* txn_db_options = + reinterpret_cast( + jtxn_db_options_handle); + ROCKSDB_NAMESPACE::TransactionDB* tdb = nullptr; + const char* db_path = env->GetStringUTFChars(jdb_path, nullptr); + if (db_path == nullptr) { + // exception thrown: OutOfMemoryError + return 0; + } + ROCKSDB_NAMESPACE::Status s = ROCKSDB_NAMESPACE::TransactionDB::Open( + *options, *txn_db_options, db_path, &tdb); + env->ReleaseStringUTFChars(jdb_path, db_path); + + if (s.ok()) { + return GET_CPLUSPLUS_POINTER(tdb); + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return 0; + } +} + +/* + * Class: org_rocksdb_TransactionDB + * Method: open + * Signature: (JJLjava/lang/String;[[B[J)[J + */ +jlongArray Java_org_rocksdb_TransactionDB_open__JJLjava_lang_String_2_3_3B_3J( + JNIEnv* env, jclass, jlong jdb_options_handle, jlong jtxn_db_options_handle, + jstring jdb_path, jobjectArray jcolumn_names, + jlongArray jcolumn_options_handles) { + const char* db_path = env->GetStringUTFChars(jdb_path, nullptr); + if (db_path == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + const jsize len_cols = env->GetArrayLength(jcolumn_names); + jlong* jco = env->GetLongArrayElements(jcolumn_options_handles, nullptr); + if (jco == nullptr) { + // exception thrown: OutOfMemoryError + env->ReleaseStringUTFChars(jdb_path, db_path); + return nullptr; + } + std::vector column_families; + for (int i = 0; i < len_cols; i++) { + const jobject jcn = env->GetObjectArrayElement(jcolumn_names, i); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->ReleaseLongArrayElements(jcolumn_options_handles, jco, JNI_ABORT); + env->ReleaseStringUTFChars(jdb_path, db_path); + return nullptr; + } + const jbyteArray jcn_ba = reinterpret_cast(jcn); + jbyte* jcf_name = env->GetByteArrayElements(jcn_ba, nullptr); + if (jcf_name == nullptr) { + // exception thrown: OutOfMemoryError + env->DeleteLocalRef(jcn); + env->ReleaseLongArrayElements(jcolumn_options_handles, jco, JNI_ABORT); + env->ReleaseStringUTFChars(jdb_path, db_path); + return nullptr; + } + + const int jcf_name_len = env->GetArrayLength(jcn_ba); + const std::string cf_name(reinterpret_cast(jcf_name), jcf_name_len); + const ROCKSDB_NAMESPACE::ColumnFamilyOptions* cf_options = + reinterpret_cast(jco[i]); + column_families.push_back( + ROCKSDB_NAMESPACE::ColumnFamilyDescriptor(cf_name, *cf_options)); + + env->ReleaseByteArrayElements(jcn_ba, jcf_name, JNI_ABORT); + env->DeleteLocalRef(jcn); + } + env->ReleaseLongArrayElements(jcolumn_options_handles, jco, JNI_ABORT); + + auto* db_options = + reinterpret_cast(jdb_options_handle); + auto* txn_db_options = + reinterpret_cast( + jtxn_db_options_handle); + std::vector handles; + ROCKSDB_NAMESPACE::TransactionDB* tdb = nullptr; + const ROCKSDB_NAMESPACE::Status s = ROCKSDB_NAMESPACE::TransactionDB::Open( + *db_options, *txn_db_options, db_path, column_families, &handles, &tdb); + + // check if open operation was successful + if (s.ok()) { + const jsize resultsLen = 1 + len_cols; // db handle + column family handles + std::unique_ptr results = + std::unique_ptr(new jlong[resultsLen]); + results[0] = GET_CPLUSPLUS_POINTER(tdb); + for (int i = 1; i <= len_cols; i++) { + results[i] = GET_CPLUSPLUS_POINTER(handles[i - 1]); + } + + jlongArray jresults = env->NewLongArray(resultsLen); + if (jresults == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + env->SetLongArrayRegion(jresults, 0, resultsLen, results.get()); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jresults); + return nullptr; + } + return jresults; + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return nullptr; + } +} + +/* + * Class: org_rocksdb_TransactionDB + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_TransactionDB_disposeInternal(JNIEnv*, jobject, + jlong jhandle) { + auto* txn_db = reinterpret_cast(jhandle); + assert(txn_db != nullptr); + delete txn_db; +} + +/* + * Class: org_rocksdb_TransactionDB + * Method: closeDatabase + * Signature: (J)V + */ +void Java_org_rocksdb_TransactionDB_closeDatabase(JNIEnv* env, jclass, + jlong jhandle) { + auto* txn_db = reinterpret_cast(jhandle); + assert(txn_db != nullptr); + ROCKSDB_NAMESPACE::Status s = txn_db->Close(); + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); +} + +/* + * Class: org_rocksdb_TransactionDB + * Method: beginTransaction + * Signature: (JJ)J + */ +jlong Java_org_rocksdb_TransactionDB_beginTransaction__JJ( + JNIEnv*, jobject, jlong jhandle, jlong jwrite_options_handle) { + auto* txn_db = reinterpret_cast(jhandle); + auto* write_options = + reinterpret_cast(jwrite_options_handle); + ROCKSDB_NAMESPACE::Transaction* txn = + txn_db->BeginTransaction(*write_options); + return GET_CPLUSPLUS_POINTER(txn); +} + +/* + * Class: org_rocksdb_TransactionDB + * Method: beginTransaction + * Signature: (JJJ)J + */ +jlong Java_org_rocksdb_TransactionDB_beginTransaction__JJJ( + JNIEnv*, jobject, jlong jhandle, jlong jwrite_options_handle, + jlong jtxn_options_handle) { + auto* txn_db = reinterpret_cast(jhandle); + auto* write_options = + reinterpret_cast(jwrite_options_handle); + auto* txn_options = reinterpret_cast( + jtxn_options_handle); + ROCKSDB_NAMESPACE::Transaction* txn = + txn_db->BeginTransaction(*write_options, *txn_options); + return GET_CPLUSPLUS_POINTER(txn); +} + +/* + * Class: org_rocksdb_TransactionDB + * Method: beginTransaction_withOld + * Signature: (JJJ)J + */ +jlong Java_org_rocksdb_TransactionDB_beginTransaction_1withOld__JJJ( + JNIEnv*, jobject, jlong jhandle, jlong jwrite_options_handle, + jlong jold_txn_handle) { + auto* txn_db = reinterpret_cast(jhandle); + auto* write_options = + reinterpret_cast(jwrite_options_handle); + auto* old_txn = + reinterpret_cast(jold_txn_handle); + ROCKSDB_NAMESPACE::TransactionOptions txn_options; + ROCKSDB_NAMESPACE::Transaction* txn = + txn_db->BeginTransaction(*write_options, txn_options, old_txn); + + // RocksJava relies on the assumption that + // we do not allocate a new Transaction object + // when providing an old_txn + assert(txn == old_txn); + + return GET_CPLUSPLUS_POINTER(txn); +} + +/* + * Class: org_rocksdb_TransactionDB + * Method: beginTransaction_withOld + * Signature: (JJJJ)J + */ +jlong Java_org_rocksdb_TransactionDB_beginTransaction_1withOld__JJJJ( + JNIEnv*, jobject, jlong jhandle, jlong jwrite_options_handle, + jlong jtxn_options_handle, jlong jold_txn_handle) { + auto* txn_db = reinterpret_cast(jhandle); + auto* write_options = + reinterpret_cast(jwrite_options_handle); + auto* txn_options = reinterpret_cast( + jtxn_options_handle); + auto* old_txn = + reinterpret_cast(jold_txn_handle); + ROCKSDB_NAMESPACE::Transaction* txn = + txn_db->BeginTransaction(*write_options, *txn_options, old_txn); + + // RocksJava relies on the assumption that + // we do not allocate a new Transaction object + // when providing an old_txn + assert(txn == old_txn); + + return GET_CPLUSPLUS_POINTER(txn); +} + +/* + * Class: org_rocksdb_TransactionDB + * Method: getTransactionByName + * Signature: (JLjava/lang/String;)J + */ +jlong Java_org_rocksdb_TransactionDB_getTransactionByName(JNIEnv* env, jobject, + jlong jhandle, + jstring jname) { + auto* txn_db = reinterpret_cast(jhandle); + const char* name = env->GetStringUTFChars(jname, nullptr); + if (name == nullptr) { + // exception thrown: OutOfMemoryError + return 0; + } + ROCKSDB_NAMESPACE::Transaction* txn = txn_db->GetTransactionByName(name); + env->ReleaseStringUTFChars(jname, name); + return GET_CPLUSPLUS_POINTER(txn); +} + +/* + * Class: org_rocksdb_TransactionDB + * Method: getAllPreparedTransactions + * Signature: (J)[J + */ +jlongArray Java_org_rocksdb_TransactionDB_getAllPreparedTransactions( + JNIEnv* env, jobject, jlong jhandle) { + auto* txn_db = reinterpret_cast(jhandle); + std::vector txns; + txn_db->GetAllPreparedTransactions(&txns); + + const size_t size = txns.size(); + assert(size < UINT32_MAX); // does it fit in a jint? + + const jsize len = static_cast(size); + std::vector tmp(len); + for (jsize i = 0; i < len; ++i) { + tmp[i] = GET_CPLUSPLUS_POINTER(txns[i]); + } + + jlongArray jtxns = env->NewLongArray(len); + if (jtxns == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + env->SetLongArrayRegion(jtxns, 0, len, tmp.data()); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jtxns); + return nullptr; + } + + return jtxns; +} + +/* + * Class: org_rocksdb_TransactionDB + * Method: getLockStatusData + * Signature: (J)Ljava/util/Map; + */ +jobject Java_org_rocksdb_TransactionDB_getLockStatusData(JNIEnv* env, jobject, + jlong jhandle) { + auto* txn_db = reinterpret_cast(jhandle); + const std::unordered_multimap + lock_status_data = txn_db->GetLockStatusData(); + const jobject jlock_status_data = ROCKSDB_NAMESPACE::HashMapJni::construct( + env, static_cast(lock_status_data.size())); + if (jlock_status_data == nullptr) { + // exception occurred + return nullptr; + } + + const ROCKSDB_NAMESPACE::HashMapJni::FnMapKV< + const int32_t, const ROCKSDB_NAMESPACE::KeyLockInfo, jobject, jobject> + fn_map_kv = + [env](const std::pair& pair) { + const jobject jlong_column_family_id = + ROCKSDB_NAMESPACE::LongJni::valueOf(env, pair.first); + if (jlong_column_family_id == nullptr) { + // an error occurred + return std::unique_ptr>(nullptr); + } + const jobject jkey_lock_info = + ROCKSDB_NAMESPACE::KeyLockInfoJni::construct(env, pair.second); + if (jkey_lock_info == nullptr) { + // an error occurred + return std::unique_ptr>(nullptr); + } + return std::unique_ptr>( + new std::pair(jlong_column_family_id, + jkey_lock_info)); + }; + + if (!ROCKSDB_NAMESPACE::HashMapJni::putAll( + env, jlock_status_data, lock_status_data.begin(), + lock_status_data.end(), fn_map_kv)) { + // exception occcurred + return nullptr; + } + + return jlock_status_data; +} + +/* + * Class: org_rocksdb_TransactionDB + * Method: getDeadlockInfoBuffer + * Signature: (J)[Lorg/rocksdb/TransactionDB/DeadlockPath; + */ +jobjectArray Java_org_rocksdb_TransactionDB_getDeadlockInfoBuffer( + JNIEnv* env, jobject jobj, jlong jhandle) { + auto* txn_db = reinterpret_cast(jhandle); + const std::vector deadlock_info_buffer = + txn_db->GetDeadlockInfoBuffer(); + + const jsize deadlock_info_buffer_len = + static_cast(deadlock_info_buffer.size()); + jobjectArray jdeadlock_info_buffer = env->NewObjectArray( + deadlock_info_buffer_len, + ROCKSDB_NAMESPACE::DeadlockPathJni::getJClass(env), nullptr); + if (jdeadlock_info_buffer == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + jsize jdeadlock_info_buffer_offset = 0; + + auto buf_end = deadlock_info_buffer.end(); + for (auto buf_it = deadlock_info_buffer.begin(); buf_it != buf_end; + ++buf_it) { + const ROCKSDB_NAMESPACE::DeadlockPath deadlock_path = *buf_it; + const std::vector deadlock_infos = + deadlock_path.path; + const jsize deadlock_infos_len = + static_cast(deadlock_info_buffer.size()); + jobjectArray jdeadlock_infos = env->NewObjectArray( + deadlock_infos_len, ROCKSDB_NAMESPACE::DeadlockInfoJni::getJClass(env), + nullptr); + if (jdeadlock_infos == nullptr) { + // exception thrown: OutOfMemoryError + env->DeleteLocalRef(jdeadlock_info_buffer); + return nullptr; + } + jsize jdeadlock_infos_offset = 0; + + auto infos_end = deadlock_infos.end(); + for (auto infos_it = deadlock_infos.begin(); infos_it != infos_end; + ++infos_it) { + const ROCKSDB_NAMESPACE::DeadlockInfo deadlock_info = *infos_it; + const jobject jdeadlock_info = + ROCKSDB_NAMESPACE::TransactionDBJni::newDeadlockInfo( + env, jobj, deadlock_info.m_txn_id, deadlock_info.m_cf_id, + deadlock_info.m_waiting_key, deadlock_info.m_exclusive); + if (jdeadlock_info == nullptr) { + // exception occcurred + env->DeleteLocalRef(jdeadlock_info_buffer); + return nullptr; + } + env->SetObjectArrayElement(jdeadlock_infos, jdeadlock_infos_offset++, + jdeadlock_info); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException or + // ArrayStoreException + env->DeleteLocalRef(jdeadlock_info); + env->DeleteLocalRef(jdeadlock_info_buffer); + return nullptr; + } + } + + const jobject jdeadlock_path = + ROCKSDB_NAMESPACE::DeadlockPathJni::construct( + env, jdeadlock_infos, deadlock_path.limit_exceeded); + if (jdeadlock_path == nullptr) { + // exception occcurred + env->DeleteLocalRef(jdeadlock_info_buffer); + return nullptr; + } + env->SetObjectArrayElement(jdeadlock_info_buffer, + jdeadlock_info_buffer_offset++, jdeadlock_path); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException or ArrayStoreException + env->DeleteLocalRef(jdeadlock_path); + env->DeleteLocalRef(jdeadlock_info_buffer); + return nullptr; + } + } + + return jdeadlock_info_buffer; +} + +/* + * Class: org_rocksdb_TransactionDB + * Method: setDeadlockInfoBufferSize + * Signature: (JI)V + */ +void Java_org_rocksdb_TransactionDB_setDeadlockInfoBufferSize( + JNIEnv*, jobject, jlong jhandle, jint jdeadlock_info_buffer_size) { + auto* txn_db = reinterpret_cast(jhandle); + txn_db->SetDeadlockInfoBufferSize(jdeadlock_info_buffer_size); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/transaction_db_options.cc b/librocksdb-sys/rocksdb/java/rocksjni/transaction_db_options.cc new file mode 100644 index 0000000..4cf2712 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/transaction_db_options.cc @@ -0,0 +1,169 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ +// for ROCKSDB_NAMESPACE::TransactionDBOptions. + +#include + +#include "include/org_rocksdb_TransactionDBOptions.h" +#include "rocksdb/utilities/transaction_db.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_TransactionDBOptions + * Method: newTransactionDBOptions + * Signature: ()J + */ +jlong Java_org_rocksdb_TransactionDBOptions_newTransactionDBOptions( + JNIEnv* /*env*/, jclass /*jcls*/) { + ROCKSDB_NAMESPACE::TransactionDBOptions* opts = + new ROCKSDB_NAMESPACE::TransactionDBOptions(); + return GET_CPLUSPLUS_POINTER(opts); +} + +/* + * Class: org_rocksdb_TransactionDBOptions + * Method: getMaxNumLocks + * Signature: (J)J + */ +jlong Java_org_rocksdb_TransactionDBOptions_getMaxNumLocks(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return opts->max_num_locks; +} + +/* + * Class: org_rocksdb_TransactionDBOptions + * Method: setMaxNumLocks + * Signature: (JJ)V + */ +void Java_org_rocksdb_TransactionDBOptions_setMaxNumLocks( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jlong jmax_num_locks) { + auto* opts = + reinterpret_cast(jhandle); + opts->max_num_locks = jmax_num_locks; +} + +/* + * Class: org_rocksdb_TransactionDBOptions + * Method: getNumStripes + * Signature: (J)J + */ +jlong Java_org_rocksdb_TransactionDBOptions_getNumStripes(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return opts->num_stripes; +} + +/* + * Class: org_rocksdb_TransactionDBOptions + * Method: setNumStripes + * Signature: (JJ)V + */ +void Java_org_rocksdb_TransactionDBOptions_setNumStripes(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle, + jlong jnum_stripes) { + auto* opts = + reinterpret_cast(jhandle); + opts->num_stripes = jnum_stripes; +} + +/* + * Class: org_rocksdb_TransactionDBOptions + * Method: getTransactionLockTimeout + * Signature: (J)J + */ +jlong Java_org_rocksdb_TransactionDBOptions_getTransactionLockTimeout( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return opts->transaction_lock_timeout; +} + +/* + * Class: org_rocksdb_TransactionDBOptions + * Method: setTransactionLockTimeout + * Signature: (JJ)V + */ +void Java_org_rocksdb_TransactionDBOptions_setTransactionLockTimeout( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, + jlong jtransaction_lock_timeout) { + auto* opts = + reinterpret_cast(jhandle); + opts->transaction_lock_timeout = jtransaction_lock_timeout; +} + +/* + * Class: org_rocksdb_TransactionDBOptions + * Method: getDefaultLockTimeout + * Signature: (J)J + */ +jlong Java_org_rocksdb_TransactionDBOptions_getDefaultLockTimeout( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return opts->default_lock_timeout; +} + +/* + * Class: org_rocksdb_TransactionDBOptions + * Method: setDefaultLockTimeout + * Signature: (JJ)V + */ +void Java_org_rocksdb_TransactionDBOptions_setDefaultLockTimeout( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, + jlong jdefault_lock_timeout) { + auto* opts = + reinterpret_cast(jhandle); + opts->default_lock_timeout = jdefault_lock_timeout; +} + +/* + * Class: org_rocksdb_TransactionDBOptions + * Method: getWritePolicy + * Signature: (J)B + */ +jbyte Java_org_rocksdb_TransactionDBOptions_getWritePolicy(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return ROCKSDB_NAMESPACE::TxnDBWritePolicyJni::toJavaTxnDBWritePolicy( + opts->write_policy); +} + +/* + * Class: org_rocksdb_TransactionDBOptions + * Method: setWritePolicy + * Signature: (JB)V + */ +void Java_org_rocksdb_TransactionDBOptions_setWritePolicy(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle, + jbyte jwrite_policy) { + auto* opts = + reinterpret_cast(jhandle); + opts->write_policy = + ROCKSDB_NAMESPACE::TxnDBWritePolicyJni::toCppTxnDBWritePolicy( + jwrite_policy); +} + +/* + * Class: org_rocksdb_TransactionDBOptions + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_TransactionDBOptions_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + delete reinterpret_cast(jhandle); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/transaction_log.cc b/librocksdb-sys/rocksdb/java/rocksjni/transaction_log.cc new file mode 100644 index 0000000..97c3bb3 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/transaction_log.cc @@ -0,0 +1,80 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling c++ ROCKSDB_NAMESPACE::Iterator methods from Java side. + +#include "rocksdb/transaction_log.h" + +#include +#include +#include + +#include "include/org_rocksdb_TransactionLogIterator.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_TransactionLogIterator + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_TransactionLogIterator_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + delete reinterpret_cast(handle); +} + +/* + * Class: org_rocksdb_TransactionLogIterator + * Method: isValid + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_TransactionLogIterator_isValid(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + return reinterpret_cast(handle) + ->Valid(); +} + +/* + * Class: org_rocksdb_TransactionLogIterator + * Method: next + * Signature: (J)V + */ +void Java_org_rocksdb_TransactionLogIterator_next(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + reinterpret_cast(handle)->Next(); +} + +/* + * Class: org_rocksdb_TransactionLogIterator + * Method: status + * Signature: (J)V + */ +void Java_org_rocksdb_TransactionLogIterator_status(JNIEnv* env, + jobject /*jobj*/, + jlong handle) { + ROCKSDB_NAMESPACE::Status s = + reinterpret_cast(handle) + ->status(); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_TransactionLogIterator + * Method: getBatch + * Signature: (J)Lorg/rocksdb/TransactionLogIterator$BatchResult + */ +jobject Java_org_rocksdb_TransactionLogIterator_getBatch(JNIEnv* env, + jobject /*jobj*/, + jlong handle) { + ROCKSDB_NAMESPACE::BatchResult batch_result = + reinterpret_cast(handle) + ->GetBatch(); + return ROCKSDB_NAMESPACE::BatchResultJni::construct(env, batch_result); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/transaction_notifier.cc b/librocksdb-sys/rocksdb/java/rocksjni/transaction_notifier.cc new file mode 100644 index 0000000..cefeb64 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/transaction_notifier.cc @@ -0,0 +1,44 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ +// for ROCKSDB_NAMESPACE::TransactionNotifier. + +#include + +#include "include/org_rocksdb_AbstractTransactionNotifier.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/transaction_notifier_jnicallback.h" + +/* + * Class: org_rocksdb_AbstractTransactionNotifier + * Method: createNewTransactionNotifier + * Signature: ()J + */ +jlong Java_org_rocksdb_AbstractTransactionNotifier_createNewTransactionNotifier( + JNIEnv* env, jobject jobj) { + auto* transaction_notifier = + new ROCKSDB_NAMESPACE::TransactionNotifierJniCallback(env, jobj); + auto* sptr_transaction_notifier = + new std::shared_ptr( + transaction_notifier); + return GET_CPLUSPLUS_POINTER(sptr_transaction_notifier); +} + +/* + * Class: org_rocksdb_AbstractTransactionNotifier + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_AbstractTransactionNotifier_disposeInternal( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { + // TODO(AR) refactor to use JniCallback::JniCallback + // when https://github.com/facebook/rocksdb/pull/1241/ is merged + std::shared_ptr* handle = + reinterpret_cast< + std::shared_ptr*>( + jhandle); + delete handle; +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/transaction_notifier_jnicallback.cc b/librocksdb-sys/rocksdb/java/rocksjni/transaction_notifier_jnicallback.cc new file mode 100644 index 0000000..26761ca --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/transaction_notifier_jnicallback.cc @@ -0,0 +1,42 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the callback "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::TransactionNotifier. + +#include "rocksjni/transaction_notifier_jnicallback.h" + +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +namespace ROCKSDB_NAMESPACE { + +TransactionNotifierJniCallback::TransactionNotifierJniCallback( + JNIEnv* env, jobject jtransaction_notifier) + : JniCallback(env, jtransaction_notifier) { + // we cache the method id for the JNI callback + m_jsnapshot_created_methodID = + AbstractTransactionNotifierJni::getSnapshotCreatedMethodId(env); +} + +void TransactionNotifierJniCallback::SnapshotCreated( + const Snapshot* newSnapshot) { + jboolean attached_thread = JNI_FALSE; + JNIEnv* env = getJniEnv(&attached_thread); + assert(env != nullptr); + + env->CallVoidMethod(m_jcallback_obj, m_jsnapshot_created_methodID, + GET_CPLUSPLUS_POINTER(newSnapshot)); + + if (env->ExceptionCheck()) { + // exception thrown from CallVoidMethod + env->ExceptionDescribe(); // print out exception to stderr + releaseJniEnv(attached_thread); + return; + } + + releaseJniEnv(attached_thread); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/java/rocksjni/transaction_notifier_jnicallback.h b/librocksdb-sys/rocksdb/java/rocksjni/transaction_notifier_jnicallback.h new file mode 100644 index 0000000..089a5ee --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/transaction_notifier_jnicallback.h @@ -0,0 +1,42 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the callback "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::TransactionNotifier. + +#ifndef JAVA_ROCKSJNI_TRANSACTION_NOTIFIER_JNICALLBACK_H_ +#define JAVA_ROCKSJNI_TRANSACTION_NOTIFIER_JNICALLBACK_H_ + +#include + +#include "rocksdb/utilities/transaction.h" +#include "rocksjni/jnicallback.h" + +namespace ROCKSDB_NAMESPACE { + +/** + * This class acts as a bridge between C++ + * and Java. The methods in this class will be + * called back from the RocksDB TransactionDB or OptimisticTransactionDB (C++), + * we then callback to the appropriate Java method + * this enables TransactionNotifier to be implemented in Java. + * + * Unlike RocksJava's Comparator JNI Callback, we do not attempt + * to reduce Java object allocations by caching the Snapshot object + * presented to the callback. This could be revisited in future + * if performance is lacking. + */ +class TransactionNotifierJniCallback : public JniCallback, + public TransactionNotifier { + public: + TransactionNotifierJniCallback(JNIEnv* env, jobject jtransaction_notifier); + virtual void SnapshotCreated(const Snapshot* newSnapshot); + + private: + jmethodID m_jsnapshot_created_methodID; +}; +} // namespace ROCKSDB_NAMESPACE + +#endif // JAVA_ROCKSJNI_TRANSACTION_NOTIFIER_JNICALLBACK_H_ diff --git a/librocksdb-sys/rocksdb/java/rocksjni/transaction_options.cc b/librocksdb-sys/rocksdb/java/rocksjni/transaction_options.cc new file mode 100644 index 0000000..dcf363e --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/transaction_options.cc @@ -0,0 +1,191 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ +// for ROCKSDB_NAMESPACE::TransactionOptions. + +#include + +#include "include/org_rocksdb_TransactionOptions.h" +#include "rocksdb/utilities/transaction_db.h" +#include "rocksjni/cplusplus_to_java_convert.h" + +/* + * Class: org_rocksdb_TransactionOptions + * Method: newTransactionOptions + * Signature: ()J + */ +jlong Java_org_rocksdb_TransactionOptions_newTransactionOptions( + JNIEnv* /*env*/, jclass /*jcls*/) { + auto* opts = new ROCKSDB_NAMESPACE::TransactionOptions(); + return GET_CPLUSPLUS_POINTER(opts); +} + +/* + * Class: org_rocksdb_TransactionOptions + * Method: isSetSnapshot + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_TransactionOptions_isSetSnapshot(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return opts->set_snapshot; +} + +/* + * Class: org_rocksdb_TransactionOptions + * Method: setSetSnapshot + * Signature: (JZ)V + */ +void Java_org_rocksdb_TransactionOptions_setSetSnapshot( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, jboolean jset_snapshot) { + auto* opts = + reinterpret_cast(jhandle); + opts->set_snapshot = jset_snapshot; +} + +/* + * Class: org_rocksdb_TransactionOptions + * Method: isDeadlockDetect + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_TransactionOptions_isDeadlockDetect(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return opts->deadlock_detect; +} + +/* + * Class: org_rocksdb_TransactionOptions + * Method: setDeadlockDetect + * Signature: (JZ)V + */ +void Java_org_rocksdb_TransactionOptions_setDeadlockDetect( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, + jboolean jdeadlock_detect) { + auto* opts = + reinterpret_cast(jhandle); + opts->deadlock_detect = jdeadlock_detect; +} + +/* + * Class: org_rocksdb_TransactionOptions + * Method: getLockTimeout + * Signature: (J)J + */ +jlong Java_org_rocksdb_TransactionOptions_getLockTimeout(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return opts->lock_timeout; +} + +/* + * Class: org_rocksdb_TransactionOptions + * Method: setLockTimeout + * Signature: (JJ)V + */ +void Java_org_rocksdb_TransactionOptions_setLockTimeout(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle, + jlong jlock_timeout) { + auto* opts = + reinterpret_cast(jhandle); + opts->lock_timeout = jlock_timeout; +} + +/* + * Class: org_rocksdb_TransactionOptions + * Method: getExpiration + * Signature: (J)J + */ +jlong Java_org_rocksdb_TransactionOptions_getExpiration(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return opts->expiration; +} + +/* + * Class: org_rocksdb_TransactionOptions + * Method: setExpiration + * Signature: (JJ)V + */ +void Java_org_rocksdb_TransactionOptions_setExpiration(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle, + jlong jexpiration) { + auto* opts = + reinterpret_cast(jhandle); + opts->expiration = jexpiration; +} + +/* + * Class: org_rocksdb_TransactionOptions + * Method: getDeadlockDetectDepth + * Signature: (J)J + */ +jlong Java_org_rocksdb_TransactionOptions_getDeadlockDetectDepth( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return opts->deadlock_detect_depth; +} + +/* + * Class: org_rocksdb_TransactionOptions + * Method: setDeadlockDetectDepth + * Signature: (JJ)V + */ +void Java_org_rocksdb_TransactionOptions_setDeadlockDetectDepth( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, + jlong jdeadlock_detect_depth) { + auto* opts = + reinterpret_cast(jhandle); + opts->deadlock_detect_depth = jdeadlock_detect_depth; +} + +/* + * Class: org_rocksdb_TransactionOptions + * Method: getMaxWriteBatchSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_TransactionOptions_getMaxWriteBatchSize(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return opts->max_write_batch_size; +} + +/* + * Class: org_rocksdb_TransactionOptions + * Method: setMaxWriteBatchSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_TransactionOptions_setMaxWriteBatchSize( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jhandle, + jlong jmax_write_batch_size) { + auto* opts = + reinterpret_cast(jhandle); + opts->max_write_batch_size = jmax_write_batch_size; +} + +/* + * Class: org_rocksdb_TransactionOptions + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_TransactionOptions_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + delete reinterpret_cast(jhandle); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/ttl.cc b/librocksdb-sys/rocksdb/java/rocksjni/ttl.cc new file mode 100644 index 0000000..1fe2083 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/ttl.cc @@ -0,0 +1,212 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling c++ ROCKSDB_NAMESPACE::TtlDB methods. +// from Java side. + +#include +#include +#include + +#include +#include +#include + +#include "include/org_rocksdb_TtlDB.h" +#include "rocksdb/utilities/db_ttl.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_TtlDB + * Method: open + * Signature: (JLjava/lang/String;IZ)J + */ +jlong Java_org_rocksdb_TtlDB_open(JNIEnv* env, jclass, jlong joptions_handle, + jstring jdb_path, jint jttl, + jboolean jread_only) { + const char* db_path = env->GetStringUTFChars(jdb_path, nullptr); + if (db_path == nullptr) { + // exception thrown: OutOfMemoryError + return 0; + } + + auto* opt = reinterpret_cast(joptions_handle); + ROCKSDB_NAMESPACE::DBWithTTL* db = nullptr; + ROCKSDB_NAMESPACE::Status s = + ROCKSDB_NAMESPACE::DBWithTTL::Open(*opt, db_path, &db, jttl, jread_only); + env->ReleaseStringUTFChars(jdb_path, db_path); + + // as TTLDB extends RocksDB on the java side, we can reuse + // the RocksDB portal here. + if (s.ok()) { + return GET_CPLUSPLUS_POINTER(db); + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return 0; + } +} + +/* + * Class: org_rocksdb_TtlDB + * Method: openCF + * Signature: (JLjava/lang/String;[[B[J[IZ)[J + */ +jlongArray Java_org_rocksdb_TtlDB_openCF(JNIEnv* env, jclass, jlong jopt_handle, + jstring jdb_path, + jobjectArray jcolumn_names, + jlongArray jcolumn_options, + jintArray jttls, jboolean jread_only) { + const char* db_path = env->GetStringUTFChars(jdb_path, nullptr); + if (db_path == nullptr) { + // exception thrown: OutOfMemoryError + return 0; + } + + const jsize len_cols = env->GetArrayLength(jcolumn_names); + jlong* jco = env->GetLongArrayElements(jcolumn_options, nullptr); + if (jco == nullptr) { + // exception thrown: OutOfMemoryError + env->ReleaseStringUTFChars(jdb_path, db_path); + return nullptr; + } + + std::vector column_families; + jboolean has_exception = JNI_FALSE; + ROCKSDB_NAMESPACE::JniUtil::byteStrings( + env, jcolumn_names, + [](const char* str_data, const size_t str_len) { + return std::string(str_data, str_len); + }, + [&jco, &column_families](size_t idx, std::string cf_name) { + ROCKSDB_NAMESPACE::ColumnFamilyOptions* cf_options = + reinterpret_cast(jco[idx]); + column_families.push_back( + ROCKSDB_NAMESPACE::ColumnFamilyDescriptor(cf_name, *cf_options)); + }, + &has_exception); + + env->ReleaseLongArrayElements(jcolumn_options, jco, JNI_ABORT); + + if (has_exception == JNI_TRUE) { + // exception occurred + env->ReleaseStringUTFChars(jdb_path, db_path); + return nullptr; + } + + std::vector ttl_values; + jint* jttlv = env->GetIntArrayElements(jttls, nullptr); + if (jttlv == nullptr) { + // exception thrown: OutOfMemoryError + env->ReleaseStringUTFChars(jdb_path, db_path); + return nullptr; + } + const jsize len_ttls = env->GetArrayLength(jttls); + for (jsize i = 0; i < len_ttls; i++) { + ttl_values.push_back(jttlv[i]); + } + env->ReleaseIntArrayElements(jttls, jttlv, JNI_ABORT); + + auto* opt = reinterpret_cast(jopt_handle); + std::vector handles; + ROCKSDB_NAMESPACE::DBWithTTL* db = nullptr; + ROCKSDB_NAMESPACE::Status s = ROCKSDB_NAMESPACE::DBWithTTL::Open( + *opt, db_path, column_families, &handles, &db, ttl_values, jread_only); + + // we have now finished with db_path + env->ReleaseStringUTFChars(jdb_path, db_path); + + // check if open operation was successful + if (s.ok()) { + const jsize resultsLen = 1 + len_cols; // db handle + column family handles + std::unique_ptr results = + std::unique_ptr(new jlong[resultsLen]); + results[0] = GET_CPLUSPLUS_POINTER(db); + for (int i = 1; i <= len_cols; i++) { + results[i] = GET_CPLUSPLUS_POINTER(handles[i - 1]); + } + + jlongArray jresults = env->NewLongArray(resultsLen); + if (jresults == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + env->SetLongArrayRegion(jresults, 0, resultsLen, results.get()); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jresults); + return nullptr; + } + + return jresults; + } else { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return NULL; + } +} + +/* + * Class: org_rocksdb_TtlDB + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_TtlDB_disposeInternal(JNIEnv*, jobject, jlong jhandle) { + auto* ttl_db = reinterpret_cast(jhandle); + assert(ttl_db != nullptr); + delete ttl_db; +} + +/* + * Class: org_rocksdb_TtlDB + * Method: closeDatabase + * Signature: (J)V + */ +void Java_org_rocksdb_TtlDB_closeDatabase(JNIEnv* /* env */, jclass, + jlong /* jhandle */) { + // auto* ttl_db = reinterpret_cast(jhandle); + // assert(ttl_db != nullptr); + // ROCKSDB_NAMESPACE::Status s = ttl_db->Close(); + // ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + + // TODO(AR) this is disabled until + // https://github.com/facebook/rocksdb/issues/4818 is resolved! +} + +/* + * Class: org_rocksdb_TtlDB + * Method: createColumnFamilyWithTtl + * Signature: (JLorg/rocksdb/ColumnFamilyDescriptor;[BJI)J; + */ +jlong Java_org_rocksdb_TtlDB_createColumnFamilyWithTtl(JNIEnv* env, jobject, + jlong jdb_handle, + jbyteArray jcolumn_name, + jlong jcolumn_options, + jint jttl) { + jbyte* cfname = env->GetByteArrayElements(jcolumn_name, nullptr); + if (cfname == nullptr) { + // exception thrown: OutOfMemoryError + return 0; + } + const jsize len = env->GetArrayLength(jcolumn_name); + + auto* cfOptions = reinterpret_cast( + jcolumn_options); + + auto* db_handle = reinterpret_cast(jdb_handle); + ROCKSDB_NAMESPACE::ColumnFamilyHandle* handle; + ROCKSDB_NAMESPACE::Status s = db_handle->CreateColumnFamilyWithTtl( + *cfOptions, std::string(reinterpret_cast(cfname), len), &handle, + jttl); + + env->ReleaseByteArrayElements(jcolumn_name, cfname, JNI_ABORT); + + if (s.ok()) { + return GET_CPLUSPLUS_POINTER(handle); + } + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return 0; +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/wal_filter.cc b/librocksdb-sys/rocksdb/java/rocksjni/wal_filter.cc new file mode 100644 index 0000000..24b88af --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/wal_filter.cc @@ -0,0 +1,24 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::WalFilter. + +#include + +#include "include/org_rocksdb_AbstractWalFilter.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/wal_filter_jnicallback.h" + +/* + * Class: org_rocksdb_AbstractWalFilter + * Method: createNewWalFilter + * Signature: ()J + */ +jlong Java_org_rocksdb_AbstractWalFilter_createNewWalFilter(JNIEnv* env, + jobject jobj) { + auto* wal_filter = new ROCKSDB_NAMESPACE::WalFilterJniCallback(env, jobj); + return GET_CPLUSPLUS_POINTER(wal_filter); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/wal_filter_jnicallback.cc b/librocksdb-sys/rocksdb/java/rocksjni/wal_filter_jnicallback.cc new file mode 100644 index 0000000..d2e3c90 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/wal_filter_jnicallback.cc @@ -0,0 +1,139 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the callback "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::WalFilter. + +#include "rocksjni/wal_filter_jnicallback.h" + +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +namespace ROCKSDB_NAMESPACE { +WalFilterJniCallback::WalFilterJniCallback(JNIEnv* env, jobject jwal_filter) + : JniCallback(env, jwal_filter) { + // Note: The name of a WalFilter will not change during it's lifetime, + // so we cache it in a global var + jmethodID jname_mid = AbstractWalFilterJni::getNameMethodId(env); + if (jname_mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return; + } + jstring jname = (jstring)env->CallObjectMethod(m_jcallback_obj, jname_mid); + if (env->ExceptionCheck()) { + // exception thrown + return; + } + jboolean has_exception = JNI_FALSE; + m_name = JniUtil::copyString(env, jname, + &has_exception); // also releases jname + if (has_exception == JNI_TRUE) { + // exception thrown + return; + } + + m_column_family_log_number_map_mid = + AbstractWalFilterJni::getColumnFamilyLogNumberMapMethodId(env); + if (m_column_family_log_number_map_mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return; + } + + m_log_record_found_proxy_mid = + AbstractWalFilterJni::getLogRecordFoundProxyMethodId(env); + if (m_log_record_found_proxy_mid == nullptr) { + // exception thrown: NoSuchMethodException or OutOfMemoryError + return; + } +} + +void WalFilterJniCallback::ColumnFamilyLogNumberMap( + const std::map& cf_lognumber_map, + const std::map& cf_name_id_map) { + jboolean attached_thread = JNI_FALSE; + JNIEnv* env = getJniEnv(&attached_thread); + if (env == nullptr) { + return; + } + + jobject jcf_lognumber_map = + ROCKSDB_NAMESPACE::HashMapJni::fromCppMap(env, &cf_lognumber_map); + if (jcf_lognumber_map == nullptr) { + // exception occurred + env->ExceptionDescribe(); // print out exception to stderr + releaseJniEnv(attached_thread); + return; + } + + jobject jcf_name_id_map = + ROCKSDB_NAMESPACE::HashMapJni::fromCppMap(env, &cf_name_id_map); + if (jcf_name_id_map == nullptr) { + // exception occurred + env->ExceptionDescribe(); // print out exception to stderr + env->DeleteLocalRef(jcf_lognumber_map); + releaseJniEnv(attached_thread); + return; + } + + env->CallVoidMethod(m_jcallback_obj, m_column_family_log_number_map_mid, + jcf_lognumber_map, jcf_name_id_map); + + env->DeleteLocalRef(jcf_lognumber_map); + env->DeleteLocalRef(jcf_name_id_map); + + if (env->ExceptionCheck()) { + // exception thrown from CallVoidMethod + env->ExceptionDescribe(); // print out exception to stderr + } + + releaseJniEnv(attached_thread); +} + +WalFilter::WalProcessingOption WalFilterJniCallback::LogRecordFound( + unsigned long long log_number, const std::string& log_file_name, + const WriteBatch& batch, WriteBatch* new_batch, bool* batch_changed) { + jboolean attached_thread = JNI_FALSE; + JNIEnv* env = getJniEnv(&attached_thread); + if (env == nullptr) { + return WalFilter::WalProcessingOption::kCorruptedRecord; + } + + jstring jlog_file_name = JniUtil::toJavaString(env, &log_file_name); + if (jlog_file_name == nullptr) { + // exception occcurred + env->ExceptionDescribe(); // print out exception to stderr + releaseJniEnv(attached_thread); + return WalFilter::WalProcessingOption::kCorruptedRecord; + } + + jshort jlog_record_found_result = env->CallShortMethod( + m_jcallback_obj, m_log_record_found_proxy_mid, + static_cast(log_number), jlog_file_name, + GET_CPLUSPLUS_POINTER(&batch), GET_CPLUSPLUS_POINTER(new_batch)); + + env->DeleteLocalRef(jlog_file_name); + + if (env->ExceptionCheck()) { + // exception thrown from CallShortMethod + env->ExceptionDescribe(); // print out exception to stderr + releaseJniEnv(attached_thread); + return WalFilter::WalProcessingOption::kCorruptedRecord; + } + + // unpack WalProcessingOption and batch_changed from jlog_record_found_result + jbyte jwal_processing_option_value = (jlog_record_found_result >> 8) & 0xFF; + jbyte jbatch_changed_value = jlog_record_found_result & 0xFF; + + releaseJniEnv(attached_thread); + + *batch_changed = jbatch_changed_value == JNI_TRUE; + + return WalProcessingOptionJni::toCppWalProcessingOption( + jwal_processing_option_value); +} + +const char* WalFilterJniCallback::Name() const { return m_name.get(); } + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/java/rocksjni/wal_filter_jnicallback.h b/librocksdb-sys/rocksdb/java/rocksjni/wal_filter_jnicallback.h new file mode 100644 index 0000000..5cdc659 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/wal_filter_jnicallback.h @@ -0,0 +1,42 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the callback "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::WalFilter. + +#ifndef JAVA_ROCKSJNI_WAL_FILTER_JNICALLBACK_H_ +#define JAVA_ROCKSJNI_WAL_FILTER_JNICALLBACK_H_ + +#include + +#include +#include +#include + +#include "rocksdb/wal_filter.h" +#include "rocksjni/jnicallback.h" + +namespace ROCKSDB_NAMESPACE { + +class WalFilterJniCallback : public JniCallback, public WalFilter { + public: + WalFilterJniCallback(JNIEnv* env, jobject jwal_filter); + virtual void ColumnFamilyLogNumberMap( + const std::map& cf_lognumber_map, + const std::map& cf_name_id_map); + virtual WalFilter::WalProcessingOption LogRecordFound( + unsigned long long log_number, const std::string& log_file_name, + const WriteBatch& batch, WriteBatch* new_batch, bool* batch_changed); + virtual const char* Name() const; + + private: + std::unique_ptr m_name; + jmethodID m_column_family_log_number_map_mid; + jmethodID m_log_record_found_proxy_mid; +}; + +} // namespace ROCKSDB_NAMESPACE + +#endif // JAVA_ROCKSJNI_WAL_FILTER_JNICALLBACK_H_ diff --git a/librocksdb-sys/rocksdb/java/rocksjni/write_batch.cc b/librocksdb-sys/rocksdb/java/rocksjni/write_batch.cc new file mode 100644 index 0000000..6704e4a --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/write_batch.cc @@ -0,0 +1,676 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling c++ ROCKSDB_NAMESPACE::WriteBatch methods from Java side. +#include "rocksdb/write_batch.h" + +#include + +#include "db/memtable.h" +#include "db/write_batch_internal.h" +#include "include/org_rocksdb_WriteBatch.h" +#include "include/org_rocksdb_WriteBatch_Handler.h" +#include "logging/logging.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/status.h" +#include "rocksdb/write_buffer_manager.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" +#include "rocksjni/writebatchhandlerjnicallback.h" +#include "table/scoped_arena_iterator.h" + +/* + * Class: org_rocksdb_WriteBatch + * Method: newWriteBatch + * Signature: (I)J + */ +jlong Java_org_rocksdb_WriteBatch_newWriteBatch__I(JNIEnv* /*env*/, + jclass /*jcls*/, + jint jreserved_bytes) { + auto* wb = + new ROCKSDB_NAMESPACE::WriteBatch(static_cast(jreserved_bytes)); + return GET_CPLUSPLUS_POINTER(wb); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: newWriteBatch + * Signature: ([BI)J + */ +jlong Java_org_rocksdb_WriteBatch_newWriteBatch___3BI(JNIEnv* env, + jclass /*jcls*/, + jbyteArray jserialized, + jint jserialized_length) { + jboolean has_exception = JNI_FALSE; + std::string serialized = ROCKSDB_NAMESPACE::JniUtil::byteString( + env, jserialized, jserialized_length, + [](const char* str, const size_t len) { return std::string(str, len); }, + &has_exception); + if (has_exception == JNI_TRUE) { + // exception occurred + return 0; + } + + auto* wb = new ROCKSDB_NAMESPACE::WriteBatch(serialized); + return GET_CPLUSPLUS_POINTER(wb); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: count0 + * Signature: (J)I + */ +jint Java_org_rocksdb_WriteBatch_count0(JNIEnv* /*env*/, jobject /*jobj*/, + jlong jwb_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + + return static_cast(wb->Count()); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: clear0 + * Signature: (J)V + */ +void Java_org_rocksdb_WriteBatch_clear0(JNIEnv* /*env*/, jobject /*jobj*/, + jlong jwb_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + + wb->Clear(); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: setSavePoint0 + * Signature: (J)V + */ +void Java_org_rocksdb_WriteBatch_setSavePoint0(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jwb_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + + wb->SetSavePoint(); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: rollbackToSavePoint0 + * Signature: (J)V + */ +void Java_org_rocksdb_WriteBatch_rollbackToSavePoint0(JNIEnv* env, + jobject /*jobj*/, + jlong jwb_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + + auto s = wb->RollbackToSavePoint(); + + if (s.ok()) { + return; + } + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: popSavePoint + * Signature: (J)V + */ +void Java_org_rocksdb_WriteBatch_popSavePoint(JNIEnv* env, jobject /*jobj*/, + jlong jwb_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + + auto s = wb->PopSavePoint(); + + if (s.ok()) { + return; + } + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: setMaxBytes + * Signature: (JJ)V + */ +void Java_org_rocksdb_WriteBatch_setMaxBytes(JNIEnv* /*env*/, jobject /*jobj*/, + jlong jwb_handle, + jlong jmax_bytes) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + + wb->SetMaxBytes(static_cast(jmax_bytes)); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: put + * Signature: (J[BI[BI)V + */ +void Java_org_rocksdb_WriteBatch_put__J_3BI_3BI(JNIEnv* env, jobject jobj, + jlong jwb_handle, + jbyteArray jkey, jint jkey_len, + jbyteArray jentry_value, + jint jentry_value_len) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + auto put = [&wb](ROCKSDB_NAMESPACE::Slice key, + ROCKSDB_NAMESPACE::Slice value) { + return wb->Put(key, value); + }; + std::unique_ptr status = + ROCKSDB_NAMESPACE::JniUtil::kv_op(put, env, jobj, jkey, jkey_len, + jentry_value, jentry_value_len); + if (status != nullptr && !status->ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + } +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: put + * Signature: (J[BI[BIJ)V + */ +void Java_org_rocksdb_WriteBatch_put__J_3BI_3BIJ( + JNIEnv* env, jobject jobj, jlong jwb_handle, jbyteArray jkey, jint jkey_len, + jbyteArray jentry_value, jint jentry_value_len, jlong jcf_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + auto* cf_handle = + reinterpret_cast(jcf_handle); + assert(cf_handle != nullptr); + auto put = [&wb, &cf_handle](ROCKSDB_NAMESPACE::Slice key, + ROCKSDB_NAMESPACE::Slice value) { + return wb->Put(cf_handle, key, value); + }; + std::unique_ptr status = + ROCKSDB_NAMESPACE::JniUtil::kv_op(put, env, jobj, jkey, jkey_len, + jentry_value, jentry_value_len); + if (status != nullptr && !status->ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + } +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: putDirect + * Signature: (JLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJ)V + */ +void Java_org_rocksdb_WriteBatch_putDirect(JNIEnv* env, jobject /*jobj*/, + jlong jwb_handle, jobject jkey, + jint jkey_offset, jint jkey_len, + jobject jval, jint jval_offset, + jint jval_len, jlong jcf_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + auto* cf_handle = + reinterpret_cast(jcf_handle); + auto put = [&wb, &cf_handle](ROCKSDB_NAMESPACE::Slice& key, + ROCKSDB_NAMESPACE::Slice& value) { + if (cf_handle == nullptr) { + wb->Put(key, value); + } else { + wb->Put(cf_handle, key, value); + } + }; + ROCKSDB_NAMESPACE::JniUtil::kv_op_direct( + put, env, jkey, jkey_offset, jkey_len, jval, jval_offset, jval_len); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: merge + * Signature: (J[BI[BI)V + */ +void Java_org_rocksdb_WriteBatch_merge__J_3BI_3BI( + JNIEnv* env, jobject jobj, jlong jwb_handle, jbyteArray jkey, jint jkey_len, + jbyteArray jentry_value, jint jentry_value_len) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + auto merge = [&wb](ROCKSDB_NAMESPACE::Slice key, + ROCKSDB_NAMESPACE::Slice value) { + return wb->Merge(key, value); + }; + std::unique_ptr status = + ROCKSDB_NAMESPACE::JniUtil::kv_op(merge, env, jobj, jkey, jkey_len, + jentry_value, jentry_value_len); + if (status != nullptr && !status->ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + } +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: merge + * Signature: (J[BI[BIJ)V + */ +void Java_org_rocksdb_WriteBatch_merge__J_3BI_3BIJ( + JNIEnv* env, jobject jobj, jlong jwb_handle, jbyteArray jkey, jint jkey_len, + jbyteArray jentry_value, jint jentry_value_len, jlong jcf_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + auto* cf_handle = + reinterpret_cast(jcf_handle); + assert(cf_handle != nullptr); + auto merge = [&wb, &cf_handle](ROCKSDB_NAMESPACE::Slice key, + ROCKSDB_NAMESPACE::Slice value) { + return wb->Merge(cf_handle, key, value); + }; + std::unique_ptr status = + ROCKSDB_NAMESPACE::JniUtil::kv_op(merge, env, jobj, jkey, jkey_len, + jentry_value, jentry_value_len); + if (status != nullptr && !status->ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + } +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: delete + * Signature: (J[BI)V + */ +void Java_org_rocksdb_WriteBatch_delete__J_3BI(JNIEnv* env, jobject jobj, + jlong jwb_handle, + jbyteArray jkey, jint jkey_len) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + auto remove = [&wb](ROCKSDB_NAMESPACE::Slice key) { return wb->Delete(key); }; + std::unique_ptr status = + ROCKSDB_NAMESPACE::JniUtil::k_op(remove, env, jobj, jkey, jkey_len); + if (status != nullptr && !status->ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + } +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: delete + * Signature: (J[BIJ)V + */ +void Java_org_rocksdb_WriteBatch_delete__J_3BIJ(JNIEnv* env, jobject jobj, + jlong jwb_handle, + jbyteArray jkey, jint jkey_len, + jlong jcf_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + auto* cf_handle = + reinterpret_cast(jcf_handle); + assert(cf_handle != nullptr); + auto remove = [&wb, &cf_handle](ROCKSDB_NAMESPACE::Slice key) { + return wb->Delete(cf_handle, key); + }; + std::unique_ptr status = + ROCKSDB_NAMESPACE::JniUtil::k_op(remove, env, jobj, jkey, jkey_len); + if (status != nullptr && !status->ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + } +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: singleDelete + * Signature: (J[BI)V + */ +void Java_org_rocksdb_WriteBatch_singleDelete__J_3BI(JNIEnv* env, jobject jobj, + jlong jwb_handle, + jbyteArray jkey, + jint jkey_len) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + auto single_delete = [&wb](ROCKSDB_NAMESPACE::Slice key) { + return wb->SingleDelete(key); + }; + std::unique_ptr status = + ROCKSDB_NAMESPACE::JniUtil::k_op(single_delete, env, jobj, jkey, + jkey_len); + if (status != nullptr && !status->ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + } +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: singleDelete + * Signature: (J[BIJ)V + */ +void Java_org_rocksdb_WriteBatch_singleDelete__J_3BIJ(JNIEnv* env, jobject jobj, + jlong jwb_handle, + jbyteArray jkey, + jint jkey_len, + jlong jcf_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + auto* cf_handle = + reinterpret_cast(jcf_handle); + assert(cf_handle != nullptr); + auto single_delete = [&wb, &cf_handle](ROCKSDB_NAMESPACE::Slice key) { + return wb->SingleDelete(cf_handle, key); + }; + std::unique_ptr status = + ROCKSDB_NAMESPACE::JniUtil::k_op(single_delete, env, jobj, jkey, + jkey_len); + if (status != nullptr && !status->ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + } +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: deleteDirect + * Signature: (JLjava/nio/ByteBuffer;IIJ)V + */ +void Java_org_rocksdb_WriteBatch_deleteDirect(JNIEnv* env, jobject /*jobj*/, + jlong jwb_handle, jobject jkey, + jint jkey_offset, jint jkey_len, + jlong jcf_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + auto* cf_handle = + reinterpret_cast(jcf_handle); + auto remove = [&wb, &cf_handle](ROCKSDB_NAMESPACE::Slice& key) { + if (cf_handle == nullptr) { + wb->Delete(key); + } else { + wb->Delete(cf_handle, key); + } + }; + ROCKSDB_NAMESPACE::JniUtil::k_op_direct(remove, env, jkey, jkey_offset, + jkey_len); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: deleteRange + * Signature: (J[BI[BI)V + */ +void Java_org_rocksdb_WriteBatch_deleteRange__J_3BI_3BI( + JNIEnv* env, jobject jobj, jlong jwb_handle, jbyteArray jbegin_key, + jint jbegin_key_len, jbyteArray jend_key, jint jend_key_len) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + auto deleteRange = [&wb](ROCKSDB_NAMESPACE::Slice beginKey, + ROCKSDB_NAMESPACE::Slice endKey) { + return wb->DeleteRange(beginKey, endKey); + }; + std::unique_ptr status = + ROCKSDB_NAMESPACE::JniUtil::kv_op(deleteRange, env, jobj, jbegin_key, + jbegin_key_len, jend_key, jend_key_len); + if (status != nullptr && !status->ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + } +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: deleteRange + * Signature: (J[BI[BIJ)V + */ +void Java_org_rocksdb_WriteBatch_deleteRange__J_3BI_3BIJ( + JNIEnv* env, jobject jobj, jlong jwb_handle, jbyteArray jbegin_key, + jint jbegin_key_len, jbyteArray jend_key, jint jend_key_len, + jlong jcf_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + auto* cf_handle = + reinterpret_cast(jcf_handle); + assert(cf_handle != nullptr); + auto deleteRange = [&wb, &cf_handle](ROCKSDB_NAMESPACE::Slice beginKey, + ROCKSDB_NAMESPACE::Slice endKey) { + return wb->DeleteRange(cf_handle, beginKey, endKey); + }; + std::unique_ptr status = + ROCKSDB_NAMESPACE::JniUtil::kv_op(deleteRange, env, jobj, jbegin_key, + jbegin_key_len, jend_key, jend_key_len); + if (status != nullptr && !status->ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + } +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: putLogData + * Signature: (J[BI)V + */ +void Java_org_rocksdb_WriteBatch_putLogData(JNIEnv* env, jobject jobj, + jlong jwb_handle, jbyteArray jblob, + jint jblob_len) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + auto putLogData = [&wb](ROCKSDB_NAMESPACE::Slice blob) { + return wb->PutLogData(blob); + }; + std::unique_ptr status = + ROCKSDB_NAMESPACE::JniUtil::k_op(putLogData, env, jobj, jblob, jblob_len); + if (status != nullptr && !status->ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + } +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: iterate + * Signature: (JJ)V + */ +void Java_org_rocksdb_WriteBatch_iterate(JNIEnv* env, jobject /*jobj*/, + jlong jwb_handle, + jlong handlerHandle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + + ROCKSDB_NAMESPACE::Status s = wb->Iterate( + reinterpret_cast( + handlerHandle)); + + if (s.ok()) { + return; + } + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: data + * Signature: (J)[B + */ +jbyteArray Java_org_rocksdb_WriteBatch_data(JNIEnv* env, jobject /*jobj*/, + jlong jwb_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + + auto data = wb->Data(); + return ROCKSDB_NAMESPACE::JniUtil::copyBytes(env, data); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: getDataSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_WriteBatch_getDataSize(JNIEnv* /*env*/, jobject /*jobj*/, + jlong jwb_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + + auto data_size = wb->GetDataSize(); + return static_cast(data_size); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: hasPut + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_WriteBatch_hasPut(JNIEnv* /*env*/, jobject /*jobj*/, + jlong jwb_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + + return wb->HasPut(); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: hasDelete + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_WriteBatch_hasDelete(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jwb_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + + return wb->HasDelete(); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: hasSingleDelete + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_rocksdb_WriteBatch_hasSingleDelete( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jwb_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + + return wb->HasSingleDelete(); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: hasDeleteRange + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_rocksdb_WriteBatch_hasDeleteRange( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jwb_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + + return wb->HasDeleteRange(); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: hasMerge + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_rocksdb_WriteBatch_hasMerge( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jwb_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + + return wb->HasMerge(); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: hasBeginPrepare + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_rocksdb_WriteBatch_hasBeginPrepare( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jwb_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + + return wb->HasBeginPrepare(); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: hasEndPrepare + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_rocksdb_WriteBatch_hasEndPrepare( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jwb_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + + return wb->HasEndPrepare(); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: hasCommit + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_rocksdb_WriteBatch_hasCommit( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jwb_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + + return wb->HasCommit(); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: hasRollback + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_rocksdb_WriteBatch_hasRollback( + JNIEnv* /*env*/, jobject /*jobj*/, jlong jwb_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + + return wb->HasRollback(); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: markWalTerminationPoint + * Signature: (J)V + */ +void Java_org_rocksdb_WriteBatch_markWalTerminationPoint(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jwb_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + + wb->MarkWalTerminationPoint(); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: getWalTerminationPoint + * Signature: (J)Lorg/rocksdb/WriteBatch/SavePoint; + */ +jobject Java_org_rocksdb_WriteBatch_getWalTerminationPoint(JNIEnv* env, + jobject /*jobj*/, + jlong jwb_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + + auto save_point = wb->GetWalTerminationPoint(); + return ROCKSDB_NAMESPACE::WriteBatchSavePointJni::construct(env, save_point); +} + +/* + * Class: org_rocksdb_WriteBatch + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_WriteBatch_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + auto* wb = reinterpret_cast(handle); + assert(wb != nullptr); + delete wb; +} + +/* + * Class: org_rocksdb_WriteBatch_Handler + * Method: createNewHandler0 + * Signature: ()J + */ +jlong Java_org_rocksdb_WriteBatch_00024Handler_createNewHandler0(JNIEnv* env, + jobject jobj) { + auto* wbjnic = new ROCKSDB_NAMESPACE::WriteBatchHandlerJniCallback(env, jobj); + return GET_CPLUSPLUS_POINTER(wbjnic); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/write_batch_test.cc b/librocksdb-sys/rocksdb/java/rocksjni/write_batch_test.cc new file mode 100644 index 0000000..30b9a72 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/write_batch_test.cc @@ -0,0 +1,199 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling c++ ROCKSDB_NAMESPACE::WriteBatch methods testing from Java side. +#include "rocksdb/write_batch.h" + +#include + +#include "db/memtable.h" +#include "db/write_batch_internal.h" +#include "include/org_rocksdb_WriteBatch.h" +#include "include/org_rocksdb_WriteBatchTest.h" +#include "include/org_rocksdb_WriteBatchTestInternalHelper.h" +#include "include/org_rocksdb_WriteBatch_Handler.h" +#include "options/cf_options.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/status.h" +#include "rocksdb/write_buffer_manager.h" +#include "rocksjni/portal.h" +#include "table/scoped_arena_iterator.h" +#include "test_util/testharness.h" +#include "util/string_util.h" + +/* + * Class: org_rocksdb_WriteBatchTest + * Method: getContents + * Signature: (J)[B + */ +jbyteArray Java_org_rocksdb_WriteBatchTest_getContents(JNIEnv* env, + jclass /*jclazz*/, + jlong jwb_handle) { + auto* b = reinterpret_cast(jwb_handle); + assert(b != nullptr); + + // todo: Currently the following code is directly copied from + // db/write_bench_test.cc. It could be implemented in java once + // all the necessary components can be accessed via jni api. + + ROCKSDB_NAMESPACE::InternalKeyComparator cmp( + ROCKSDB_NAMESPACE::BytewiseComparator()); + auto factory = std::make_shared(); + ROCKSDB_NAMESPACE::Options options; + ROCKSDB_NAMESPACE::WriteBufferManager wb(options.db_write_buffer_size); + options.memtable_factory = factory; + ROCKSDB_NAMESPACE::MemTable* mem = new ROCKSDB_NAMESPACE::MemTable( + cmp, ROCKSDB_NAMESPACE::ImmutableOptions(options), + ROCKSDB_NAMESPACE::MutableCFOptions(options), &wb, + ROCKSDB_NAMESPACE::kMaxSequenceNumber, 0 /* column_family_id */); + mem->Ref(); + std::string state; + ROCKSDB_NAMESPACE::ColumnFamilyMemTablesDefault cf_mems_default(mem); + ROCKSDB_NAMESPACE::Status s = + ROCKSDB_NAMESPACE::WriteBatchInternal::InsertInto(b, &cf_mems_default, + nullptr, nullptr); + unsigned int count = 0; + ROCKSDB_NAMESPACE::Arena arena; + ROCKSDB_NAMESPACE::ScopedArenaIterator iter( + mem->NewIterator(ROCKSDB_NAMESPACE::ReadOptions(), &arena)); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ROCKSDB_NAMESPACE::ParsedInternalKey ikey; + ikey.clear(); + ROCKSDB_NAMESPACE::Status pik_status = ROCKSDB_NAMESPACE::ParseInternalKey( + iter->key(), &ikey, true /* log_err_key */); + pik_status.PermitUncheckedError(); + assert(pik_status.ok()); + switch (ikey.type) { + case ROCKSDB_NAMESPACE::kTypeValue: + state.append("Put("); + state.append(ikey.user_key.ToString()); + state.append(", "); + state.append(iter->value().ToString()); + state.append(")"); + count++; + break; + case ROCKSDB_NAMESPACE::kTypeMerge: + state.append("Merge("); + state.append(ikey.user_key.ToString()); + state.append(", "); + state.append(iter->value().ToString()); + state.append(")"); + count++; + break; + case ROCKSDB_NAMESPACE::kTypeDeletion: + state.append("Delete("); + state.append(ikey.user_key.ToString()); + state.append(")"); + count++; + break; + case ROCKSDB_NAMESPACE::kTypeSingleDeletion: + state.append("SingleDelete("); + state.append(ikey.user_key.ToString()); + state.append(")"); + count++; + break; + case ROCKSDB_NAMESPACE::kTypeRangeDeletion: + state.append("DeleteRange("); + state.append(ikey.user_key.ToString()); + state.append(", "); + state.append(iter->value().ToString()); + state.append(")"); + count++; + break; + case ROCKSDB_NAMESPACE::kTypeLogData: + state.append("LogData("); + state.append(ikey.user_key.ToString()); + state.append(")"); + count++; + break; + default: + assert(false); + state.append("Err:Expected("); + state.append(std::to_string(ikey.type)); + state.append(")"); + count++; + break; + } + state.append("@"); + state.append(std::to_string(ikey.sequence)); + } + if (!s.ok()) { + state.append(s.ToString()); + } else if (ROCKSDB_NAMESPACE::WriteBatchInternal::Count(b) != count) { + state.append("Err:CountMismatch(expected="); + state.append( + std::to_string(ROCKSDB_NAMESPACE::WriteBatchInternal::Count(b))); + state.append(", actual="); + state.append(std::to_string(count)); + state.append(")"); + } + delete mem->Unref(); + + jbyteArray jstate = env->NewByteArray(static_cast(state.size())); + if (jstate == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + + env->SetByteArrayRegion( + jstate, 0, static_cast(state.size()), + const_cast(reinterpret_cast(state.c_str()))); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jstate); + return nullptr; + } + + return jstate; +} + +/* + * Class: org_rocksdb_WriteBatchTestInternalHelper + * Method: setSequence + * Signature: (JJ)V + */ +void Java_org_rocksdb_WriteBatchTestInternalHelper_setSequence( + JNIEnv* /*env*/, jclass /*jclazz*/, jlong jwb_handle, jlong jsn) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + + ROCKSDB_NAMESPACE::WriteBatchInternal::SetSequence( + wb, static_cast(jsn)); +} + +/* + * Class: org_rocksdb_WriteBatchTestInternalHelper + * Method: sequence + * Signature: (J)J + */ +jlong Java_org_rocksdb_WriteBatchTestInternalHelper_sequence(JNIEnv* /*env*/, + jclass /*jclazz*/, + jlong jwb_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + + return static_cast( + ROCKSDB_NAMESPACE::WriteBatchInternal::Sequence(wb)); +} + +/* + * Class: org_rocksdb_WriteBatchTestInternalHelper + * Method: append + * Signature: (JJ)V + */ +void Java_org_rocksdb_WriteBatchTestInternalHelper_append(JNIEnv* /*env*/, + jclass /*jclazz*/, + jlong jwb_handle_1, + jlong jwb_handle_2) { + auto* wb1 = reinterpret_cast(jwb_handle_1); + assert(wb1 != nullptr); + auto* wb2 = reinterpret_cast(jwb_handle_2); + assert(wb2 != nullptr); + + ROCKSDB_NAMESPACE::WriteBatchInternal::Append(wb1, wb2); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/write_batch_with_index.cc b/librocksdb-sys/rocksdb/java/rocksjni/write_batch_with_index.cc new file mode 100644 index 0000000..a5c3216 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/write_batch_with_index.cc @@ -0,0 +1,953 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the "bridge" between Java and C++ and enables +// calling c++ ROCKSDB_NAMESPACE::WriteBatchWithIndex methods from Java side. + +#include "rocksdb/utilities/write_batch_with_index.h" + +#include "include/org_rocksdb_WBWIRocksIterator.h" +#include "include/org_rocksdb_WriteBatchWithIndex.h" +#include "rocksdb/comparator.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: newWriteBatchWithIndex + * Signature: ()J + */ +jlong Java_org_rocksdb_WriteBatchWithIndex_newWriteBatchWithIndex__( + JNIEnv* /*env*/, jclass /*jcls*/) { + auto* wbwi = new ROCKSDB_NAMESPACE::WriteBatchWithIndex(); + return GET_CPLUSPLUS_POINTER(wbwi); +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: newWriteBatchWithIndex + * Signature: (Z)J + */ +jlong Java_org_rocksdb_WriteBatchWithIndex_newWriteBatchWithIndex__Z( + JNIEnv* /*env*/, jclass /*jcls*/, jboolean joverwrite_key) { + auto* wbwi = new ROCKSDB_NAMESPACE::WriteBatchWithIndex( + ROCKSDB_NAMESPACE::BytewiseComparator(), 0, + static_cast(joverwrite_key)); + return GET_CPLUSPLUS_POINTER(wbwi); +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: newWriteBatchWithIndex + * Signature: (JBIZ)J + */ +jlong Java_org_rocksdb_WriteBatchWithIndex_newWriteBatchWithIndex__JBIZ( + JNIEnv* /*env*/, jclass /*jcls*/, jlong jfallback_index_comparator_handle, + jbyte jcomparator_type, jint jreserved_bytes, jboolean joverwrite_key) { + ROCKSDB_NAMESPACE::Comparator* fallback_comparator = nullptr; + switch (jcomparator_type) { + // JAVA_COMPARATOR + case 0x0: + fallback_comparator = + reinterpret_cast( + jfallback_index_comparator_handle); + break; + + // JAVA_NATIVE_COMPARATOR_WRAPPER + case 0x1: + fallback_comparator = reinterpret_cast( + jfallback_index_comparator_handle); + break; + } + auto* wbwi = new ROCKSDB_NAMESPACE::WriteBatchWithIndex( + fallback_comparator, static_cast(jreserved_bytes), + static_cast(joverwrite_key)); + return GET_CPLUSPLUS_POINTER(wbwi); +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: count0 + * Signature: (J)I + */ +jint Java_org_rocksdb_WriteBatchWithIndex_count0(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jwbwi_handle) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + assert(wbwi != nullptr); + + return static_cast(wbwi->GetWriteBatch()->Count()); +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: put + * Signature: (J[BI[BI)V + */ +void Java_org_rocksdb_WriteBatchWithIndex_put__J_3BI_3BI( + JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey, + jint jkey_len, jbyteArray jentry_value, jint jentry_value_len) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + assert(wbwi != nullptr); + auto put = [&wbwi](ROCKSDB_NAMESPACE::Slice key, + ROCKSDB_NAMESPACE::Slice value) { + return wbwi->Put(key, value); + }; + std::unique_ptr status = + ROCKSDB_NAMESPACE::JniUtil::kv_op(put, env, jobj, jkey, jkey_len, + jentry_value, jentry_value_len); + if (status != nullptr && !status->ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + } +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: put + * Signature: (J[BI[BIJ)V + */ +void Java_org_rocksdb_WriteBatchWithIndex_put__J_3BI_3BIJ( + JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey, + jint jkey_len, jbyteArray jentry_value, jint jentry_value_len, + jlong jcf_handle) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + assert(wbwi != nullptr); + auto* cf_handle = + reinterpret_cast(jcf_handle); + assert(cf_handle != nullptr); + auto put = [&wbwi, &cf_handle](ROCKSDB_NAMESPACE::Slice key, + ROCKSDB_NAMESPACE::Slice value) { + return wbwi->Put(cf_handle, key, value); + }; + std::unique_ptr status = + ROCKSDB_NAMESPACE::JniUtil::kv_op(put, env, jobj, jkey, jkey_len, + jentry_value, jentry_value_len); + if (status != nullptr && !status->ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + } +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: putDirect + * Signature: (JLjava/nio/ByteBuffer;IILjava/nio/ByteBuffer;IIJ)V + */ +void Java_org_rocksdb_WriteBatchWithIndex_putDirect( + JNIEnv* env, jobject /*jobj*/, jlong jwb_handle, jobject jkey, + jint jkey_offset, jint jkey_len, jobject jval, jint jval_offset, + jint jval_len, jlong jcf_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + auto* cf_handle = + reinterpret_cast(jcf_handle); + auto put = [&wb, &cf_handle](ROCKSDB_NAMESPACE::Slice& key, + ROCKSDB_NAMESPACE::Slice& value) { + if (cf_handle == nullptr) { + wb->Put(key, value); + } else { + wb->Put(cf_handle, key, value); + } + }; + ROCKSDB_NAMESPACE::JniUtil::kv_op_direct( + put, env, jkey, jkey_offset, jkey_len, jval, jval_offset, jval_len); +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: merge + * Signature: (J[BI[BI)V + */ +void Java_org_rocksdb_WriteBatchWithIndex_merge__J_3BI_3BI( + JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey, + jint jkey_len, jbyteArray jentry_value, jint jentry_value_len) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + assert(wbwi != nullptr); + auto merge = [&wbwi](ROCKSDB_NAMESPACE::Slice key, + ROCKSDB_NAMESPACE::Slice value) { + return wbwi->Merge(key, value); + }; + std::unique_ptr status = + ROCKSDB_NAMESPACE::JniUtil::kv_op(merge, env, jobj, jkey, jkey_len, + jentry_value, jentry_value_len); + if (status != nullptr && !status->ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + } +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: merge + * Signature: (J[BI[BIJ)V + */ +void Java_org_rocksdb_WriteBatchWithIndex_merge__J_3BI_3BIJ( + JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey, + jint jkey_len, jbyteArray jentry_value, jint jentry_value_len, + jlong jcf_handle) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + assert(wbwi != nullptr); + auto* cf_handle = + reinterpret_cast(jcf_handle); + assert(cf_handle != nullptr); + auto merge = [&wbwi, &cf_handle](ROCKSDB_NAMESPACE::Slice key, + ROCKSDB_NAMESPACE::Slice value) { + return wbwi->Merge(cf_handle, key, value); + }; + std::unique_ptr status = + ROCKSDB_NAMESPACE::JniUtil::kv_op(merge, env, jobj, jkey, jkey_len, + jentry_value, jentry_value_len); + if (status != nullptr && !status->ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + } +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: delete + * Signature: (J[BI)V + */ +void Java_org_rocksdb_WriteBatchWithIndex_delete__J_3BI(JNIEnv* env, + jobject jobj, + jlong jwbwi_handle, + jbyteArray jkey, + jint jkey_len) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + assert(wbwi != nullptr); + auto remove = [&wbwi](ROCKSDB_NAMESPACE::Slice key) { + return wbwi->Delete(key); + }; + std::unique_ptr status = + ROCKSDB_NAMESPACE::JniUtil::k_op(remove, env, jobj, jkey, jkey_len); + if (status != nullptr && !status->ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + } +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: delete + * Signature: (J[BIJ)V + */ +void Java_org_rocksdb_WriteBatchWithIndex_delete__J_3BIJ( + JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey, + jint jkey_len, jlong jcf_handle) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + assert(wbwi != nullptr); + auto* cf_handle = + reinterpret_cast(jcf_handle); + assert(cf_handle != nullptr); + auto remove = [&wbwi, &cf_handle](ROCKSDB_NAMESPACE::Slice key) { + return wbwi->Delete(cf_handle, key); + }; + std::unique_ptr status = + ROCKSDB_NAMESPACE::JniUtil::k_op(remove, env, jobj, jkey, jkey_len); + if (status != nullptr && !status->ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + } +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: singleDelete + * Signature: (J[BI)V + */ +void Java_org_rocksdb_WriteBatchWithIndex_singleDelete__J_3BI( + JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey, + jint jkey_len) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + assert(wbwi != nullptr); + auto single_delete = [&wbwi](ROCKSDB_NAMESPACE::Slice key) { + return wbwi->SingleDelete(key); + }; + std::unique_ptr status = + ROCKSDB_NAMESPACE::JniUtil::k_op(single_delete, env, jobj, jkey, + jkey_len); + if (status != nullptr && !status->ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + } +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: singleDelete + * Signature: (J[BIJ)V + */ +void Java_org_rocksdb_WriteBatchWithIndex_singleDelete__J_3BIJ( + JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey, + jint jkey_len, jlong jcf_handle) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + assert(wbwi != nullptr); + auto* cf_handle = + reinterpret_cast(jcf_handle); + assert(cf_handle != nullptr); + auto single_delete = [&wbwi, &cf_handle](ROCKSDB_NAMESPACE::Slice key) { + return wbwi->SingleDelete(cf_handle, key); + }; + std::unique_ptr status = + ROCKSDB_NAMESPACE::JniUtil::k_op(single_delete, env, jobj, jkey, + jkey_len); + if (status != nullptr && !status->ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + } +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: deleteDirect + * Signature: (JLjava/nio/ByteBuffer;IIJ)V + */ +void Java_org_rocksdb_WriteBatchWithIndex_deleteDirect( + JNIEnv* env, jobject /*jobj*/, jlong jwb_handle, jobject jkey, + jint jkey_offset, jint jkey_len, jlong jcf_handle) { + auto* wb = reinterpret_cast(jwb_handle); + assert(wb != nullptr); + auto* cf_handle = + reinterpret_cast(jcf_handle); + auto remove = [&wb, &cf_handle](ROCKSDB_NAMESPACE::Slice& key) { + if (cf_handle == nullptr) { + wb->Delete(key); + } else { + wb->Delete(cf_handle, key); + } + }; + ROCKSDB_NAMESPACE::JniUtil::k_op_direct(remove, env, jkey, jkey_offset, + jkey_len); +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: deleteRange + * Signature: (J[BI[BI)V + */ +void Java_org_rocksdb_WriteBatchWithIndex_deleteRange__J_3BI_3BI( + JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jbegin_key, + jint jbegin_key_len, jbyteArray jend_key, jint jend_key_len) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + assert(wbwi != nullptr); + auto deleteRange = [&wbwi](ROCKSDB_NAMESPACE::Slice beginKey, + ROCKSDB_NAMESPACE::Slice endKey) { + return wbwi->DeleteRange(beginKey, endKey); + }; + std::unique_ptr status = + ROCKSDB_NAMESPACE::JniUtil::kv_op(deleteRange, env, jobj, jbegin_key, + jbegin_key_len, jend_key, jend_key_len); + if (status != nullptr && !status->ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + } +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: deleteRange + * Signature: (J[BI[BIJ)V + */ +void Java_org_rocksdb_WriteBatchWithIndex_deleteRange__J_3BI_3BIJ( + JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jbegin_key, + jint jbegin_key_len, jbyteArray jend_key, jint jend_key_len, + jlong jcf_handle) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + assert(wbwi != nullptr); + auto* cf_handle = + reinterpret_cast(jcf_handle); + assert(cf_handle != nullptr); + auto deleteRange = [&wbwi, &cf_handle](ROCKSDB_NAMESPACE::Slice beginKey, + ROCKSDB_NAMESPACE::Slice endKey) { + return wbwi->DeleteRange(cf_handle, beginKey, endKey); + }; + std::unique_ptr status = + ROCKSDB_NAMESPACE::JniUtil::kv_op(deleteRange, env, jobj, jbegin_key, + jbegin_key_len, jend_key, jend_key_len); + if (status != nullptr && !status->ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + } +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: putLogData + * Signature: (J[BI)V + */ +void Java_org_rocksdb_WriteBatchWithIndex_putLogData(JNIEnv* env, jobject jobj, + jlong jwbwi_handle, + jbyteArray jblob, + jint jblob_len) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + assert(wbwi != nullptr); + auto putLogData = [&wbwi](ROCKSDB_NAMESPACE::Slice blob) { + return wbwi->PutLogData(blob); + }; + std::unique_ptr status = + ROCKSDB_NAMESPACE::JniUtil::k_op(putLogData, env, jobj, jblob, jblob_len); + if (status != nullptr && !status->ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status); + } +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: clear + * Signature: (J)V + */ +void Java_org_rocksdb_WriteBatchWithIndex_clear0(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jwbwi_handle) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + assert(wbwi != nullptr); + + wbwi->Clear(); +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: setSavePoint0 + * Signature: (J)V + */ +void Java_org_rocksdb_WriteBatchWithIndex_setSavePoint0(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jwbwi_handle) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + assert(wbwi != nullptr); + + wbwi->SetSavePoint(); +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: rollbackToSavePoint0 + * Signature: (J)V + */ +void Java_org_rocksdb_WriteBatchWithIndex_rollbackToSavePoint0( + JNIEnv* env, jobject /*jobj*/, jlong jwbwi_handle) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + assert(wbwi != nullptr); + + auto s = wbwi->RollbackToSavePoint(); + + if (s.ok()) { + return; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: popSavePoint + * Signature: (J)V + */ +void Java_org_rocksdb_WriteBatchWithIndex_popSavePoint(JNIEnv* env, + jobject /*jobj*/, + jlong jwbwi_handle) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + assert(wbwi != nullptr); + + auto s = wbwi->PopSavePoint(); + + if (s.ok()) { + return; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: setMaxBytes + * Signature: (JJ)V + */ +void Java_org_rocksdb_WriteBatchWithIndex_setMaxBytes(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jwbwi_handle, + jlong jmax_bytes) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + assert(wbwi != nullptr); + + wbwi->SetMaxBytes(static_cast(jmax_bytes)); +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: getWriteBatch + * Signature: (J)Lorg/rocksdb/WriteBatch; + */ +jobject Java_org_rocksdb_WriteBatchWithIndex_getWriteBatch(JNIEnv* env, + jobject /*jobj*/, + jlong jwbwi_handle) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + assert(wbwi != nullptr); + + auto* wb = wbwi->GetWriteBatch(); + + // TODO(AR) is the `wb` object owned by us? + return ROCKSDB_NAMESPACE::WriteBatchJni::construct(env, wb); +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: iterator0 + * Signature: (J)J + */ +jlong Java_org_rocksdb_WriteBatchWithIndex_iterator0(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jwbwi_handle) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + auto* wbwi_iterator = wbwi->NewIterator(); + return GET_CPLUSPLUS_POINTER(wbwi_iterator); +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: iterator1 + * Signature: (JJ)J + */ +jlong Java_org_rocksdb_WriteBatchWithIndex_iterator1(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jwbwi_handle, + jlong jcf_handle) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + auto* cf_handle = + reinterpret_cast(jcf_handle); + auto* wbwi_iterator = wbwi->NewIterator(cf_handle); + return GET_CPLUSPLUS_POINTER(wbwi_iterator); +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: iteratorWithBase + * Signature: (JJJJ)J + */ +jlong Java_org_rocksdb_WriteBatchWithIndex_iteratorWithBase( + JNIEnv*, jobject, jlong jwbwi_handle, jlong jcf_handle, + jlong jbase_iterator_handle, jlong jread_opts_handle) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + auto* cf_handle = + reinterpret_cast(jcf_handle); + auto* base_iterator = + reinterpret_cast(jbase_iterator_handle); + ROCKSDB_NAMESPACE::ReadOptions* read_opts = + jread_opts_handle == 0 + ? nullptr + : reinterpret_cast( + jread_opts_handle); + auto* iterator = + wbwi->NewIteratorWithBase(cf_handle, base_iterator, read_opts); + return GET_CPLUSPLUS_POINTER(iterator); +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: getFromBatch + * Signature: (JJ[BI)[B + */ +jbyteArray JNICALL Java_org_rocksdb_WriteBatchWithIndex_getFromBatch__JJ_3BI( + JNIEnv* env, jobject /*jobj*/, jlong jwbwi_handle, jlong jdbopt_handle, + jbyteArray jkey, jint jkey_len) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + auto* dbopt = reinterpret_cast(jdbopt_handle); + + auto getter = [&wbwi, &dbopt](const ROCKSDB_NAMESPACE::Slice& key, + std::string* value) { + return wbwi->GetFromBatch(*dbopt, key, value); + }; + + return ROCKSDB_NAMESPACE::JniUtil::v_op(getter, env, jkey, jkey_len); +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: getFromBatch + * Signature: (JJ[BIJ)[B + */ +jbyteArray Java_org_rocksdb_WriteBatchWithIndex_getFromBatch__JJ_3BIJ( + JNIEnv* env, jobject /*jobj*/, jlong jwbwi_handle, jlong jdbopt_handle, + jbyteArray jkey, jint jkey_len, jlong jcf_handle) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + auto* dbopt = reinterpret_cast(jdbopt_handle); + auto* cf_handle = + reinterpret_cast(jcf_handle); + + auto getter = [&wbwi, &cf_handle, &dbopt](const ROCKSDB_NAMESPACE::Slice& key, + std::string* value) { + return wbwi->GetFromBatch(cf_handle, *dbopt, key, value); + }; + + return ROCKSDB_NAMESPACE::JniUtil::v_op(getter, env, jkey, jkey_len); +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: getFromBatchAndDB + * Signature: (JJJ[BI)[B + */ +jbyteArray Java_org_rocksdb_WriteBatchWithIndex_getFromBatchAndDB__JJJ_3BI( + JNIEnv* env, jobject /*jobj*/, jlong jwbwi_handle, jlong jdb_handle, + jlong jreadopt_handle, jbyteArray jkey, jint jkey_len) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + auto* db = reinterpret_cast(jdb_handle); + auto* readopt = + reinterpret_cast(jreadopt_handle); + + auto getter = [&wbwi, &db, &readopt](const ROCKSDB_NAMESPACE::Slice& key, + std::string* value) { + return wbwi->GetFromBatchAndDB(db, *readopt, key, value); + }; + + return ROCKSDB_NAMESPACE::JniUtil::v_op(getter, env, jkey, jkey_len); +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: getFromBatchAndDB + * Signature: (JJJ[BIJ)[B + */ +jbyteArray Java_org_rocksdb_WriteBatchWithIndex_getFromBatchAndDB__JJJ_3BIJ( + JNIEnv* env, jobject /*jobj*/, jlong jwbwi_handle, jlong jdb_handle, + jlong jreadopt_handle, jbyteArray jkey, jint jkey_len, jlong jcf_handle) { + auto* wbwi = + reinterpret_cast(jwbwi_handle); + auto* db = reinterpret_cast(jdb_handle); + auto* readopt = + reinterpret_cast(jreadopt_handle); + auto* cf_handle = + reinterpret_cast(jcf_handle); + + auto getter = [&wbwi, &db, &cf_handle, &readopt]( + const ROCKSDB_NAMESPACE::Slice& key, std::string* value) { + return wbwi->GetFromBatchAndDB(db, *readopt, cf_handle, key, value); + }; + + return ROCKSDB_NAMESPACE::JniUtil::v_op(getter, env, jkey, jkey_len); +} + +/* + * Class: org_rocksdb_WriteBatchWithIndex + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_WriteBatchWithIndex_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + auto* wbwi = + reinterpret_cast(handle); + assert(wbwi != nullptr); + delete wbwi; +} + +/* WBWIRocksIterator below */ + +/* + * Class: org_rocksdb_WBWIRocksIterator + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_WBWIRocksIterator_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + auto* it = reinterpret_cast(handle); + assert(it != nullptr); + delete it; +} + +/* + * Class: org_rocksdb_WBWIRocksIterator + * Method: isValid0 + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_WBWIRocksIterator_isValid0(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + return reinterpret_cast(handle)->Valid(); +} + +/* + * Class: org_rocksdb_WBWIRocksIterator + * Method: seekToFirst0 + * Signature: (J)V + */ +void Java_org_rocksdb_WBWIRocksIterator_seekToFirst0(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + reinterpret_cast(handle)->SeekToFirst(); +} + +/* + * Class: org_rocksdb_WBWIRocksIterator + * Method: seekToLast0 + * Signature: (J)V + */ +void Java_org_rocksdb_WBWIRocksIterator_seekToLast0(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong handle) { + reinterpret_cast(handle)->SeekToLast(); +} + +/* + * Class: org_rocksdb_WBWIRocksIterator + * Method: next0 + * Signature: (J)V + */ +void Java_org_rocksdb_WBWIRocksIterator_next0(JNIEnv* /*env*/, jobject /*jobj*/, + jlong handle) { + reinterpret_cast(handle)->Next(); +} + +/* + * Class: org_rocksdb_WBWIRocksIterator + * Method: prev0 + * Signature: (J)V + */ +void Java_org_rocksdb_WBWIRocksIterator_prev0(JNIEnv* /*env*/, jobject /*jobj*/, + jlong handle) { + reinterpret_cast(handle)->Prev(); +} + +/* + * Class: org_rocksdb_WBWIRocksIterator + * Method: seek0 + * Signature: (J[BI)V + */ +void Java_org_rocksdb_WBWIRocksIterator_seek0(JNIEnv* env, jobject /*jobj*/, + jlong handle, jbyteArray jtarget, + jint jtarget_len) { + auto* it = reinterpret_cast(handle); + jbyte* target = new jbyte[jtarget_len]; + env->GetByteArrayRegion(jtarget, 0, jtarget_len, target); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] target; + return; + } + + ROCKSDB_NAMESPACE::Slice target_slice(reinterpret_cast(target), + jtarget_len); + + it->Seek(target_slice); + + delete[] target; +} + +/* + * Class: org_rocksdb_WBWIRocksIterator + * Method: seekDirect0 + * Signature: (JLjava/nio/ByteBuffer;II)V + */ +void Java_org_rocksdb_WBWIRocksIterator_seekDirect0( + JNIEnv* env, jobject /*jobj*/, jlong handle, jobject jtarget, + jint jtarget_off, jint jtarget_len) { + auto* it = reinterpret_cast(handle); + auto seek = [&it](ROCKSDB_NAMESPACE::Slice& target_slice) { + it->Seek(target_slice); + }; + ROCKSDB_NAMESPACE::JniUtil::k_op_direct(seek, env, jtarget, jtarget_off, + jtarget_len); +} + +/* + * This method supports fetching into indirect byte buffers; + * the Java wrapper extracts the byte[] and passes it here. + * + * Class: org_rocksdb_WBWIRocksIterator + * Method: seekByteArray0 + * Signature: (J[BII)V + */ +void Java_org_rocksdb_WBWIRocksIterator_seekByteArray0( + JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget, + jint jtarget_off, jint jtarget_len) { + const std::unique_ptr target(new char[jtarget_len]); + if (target == nullptr) { + jclass oom_class = env->FindClass("/lang/java/OutOfMemoryError"); + env->ThrowNew(oom_class, + "Memory allocation failed in RocksDB JNI function"); + return; + } + env->GetByteArrayRegion(jtarget, jtarget_off, jtarget_len, + reinterpret_cast(target.get())); + + ROCKSDB_NAMESPACE::Slice target_slice(target.get(), jtarget_len); + + auto* it = reinterpret_cast(handle); + it->Seek(target_slice); +} + +/* + * Class: org_rocksdb_WBWIRocksIterator + * Method: seekForPrev0 + * Signature: (J[BI)V + */ +void Java_org_rocksdb_WBWIRocksIterator_seekForPrev0(JNIEnv* env, + jobject /*jobj*/, + jlong handle, + jbyteArray jtarget, + jint jtarget_len) { + auto* it = reinterpret_cast(handle); + jbyte* target = new jbyte[jtarget_len]; + env->GetByteArrayRegion(jtarget, 0, jtarget_len, target); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + delete[] target; + return; + } + + ROCKSDB_NAMESPACE::Slice target_slice(reinterpret_cast(target), + jtarget_len); + + it->SeekForPrev(target_slice); + + delete[] target; +} + +/* + * Class: org_rocksdb_WBWIRocksIterator + * Method: seekForPrevDirect0 + * Signature: (JLjava/nio/ByteBuffer;II)V + */ +void Java_org_rocksdb_WBWIRocksIterator_seekForPrevDirect0( + JNIEnv* env, jobject /*jobj*/, jlong handle, jobject jtarget, + jint jtarget_off, jint jtarget_len) { + auto* it = reinterpret_cast(handle); + auto seek_for_prev = [&it](ROCKSDB_NAMESPACE::Slice& target_slice) { + it->SeekForPrev(target_slice); + }; + ROCKSDB_NAMESPACE::JniUtil::k_op_direct(seek_for_prev, env, jtarget, + jtarget_off, jtarget_len); +} + +/* + * This method supports fetching into indirect byte buffers; + * the Java wrapper extracts the byte[] and passes it here. + * + * Class: org_rocksdb_WBWIRocksIterator + * Method: seekForPrevByteArray0 + * Signature: (J[BII)V + */ +void Java_org_rocksdb_WBWIRocksIterator_seekForPrevByteArray0( + JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget, + jint jtarget_off, jint jtarget_len) { + const std::unique_ptr target(new char[jtarget_len]); + if (target == nullptr) { + jclass oom_class = env->FindClass("/lang/java/OutOfMemoryError"); + env->ThrowNew(oom_class, + "Memory allocation failed in RocksDB JNI function"); + return; + } + env->GetByteArrayRegion(jtarget, jtarget_off, jtarget_len, + reinterpret_cast(target.get())); + + ROCKSDB_NAMESPACE::Slice target_slice(target.get(), jtarget_len); + + auto* it = reinterpret_cast(handle); + it->SeekForPrev(target_slice); +} + +/* + * Class: org_rocksdb_WBWIRocksIterator + * Method: status0 + * Signature: (J)V + */ +void Java_org_rocksdb_WBWIRocksIterator_status0(JNIEnv* env, jobject /*jobj*/, + jlong handle) { + auto* it = reinterpret_cast(handle); + ROCKSDB_NAMESPACE::Status s = it->status(); + + if (s.ok()) { + return; + } + + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); +} + +/* + * Class: org_rocksdb_WBWIRocksIterator + * Method: entry1 + * Signature: (J)[J + */ +jlongArray Java_org_rocksdb_WBWIRocksIterator_entry1(JNIEnv* env, + jobject /*jobj*/, + jlong handle) { + auto* it = reinterpret_cast(handle); + const ROCKSDB_NAMESPACE::WriteEntry& we = it->Entry(); + + jlong results[3]; + + // set the type of the write entry + results[0] = ROCKSDB_NAMESPACE::WriteTypeJni::toJavaWriteType(we.type); + + // NOTE: key_slice and value_slice will be freed by + // org.rocksdb.DirectSlice#close + + auto* key_slice = new ROCKSDB_NAMESPACE::Slice(we.key.data(), we.key.size()); + results[1] = GET_CPLUSPLUS_POINTER(key_slice); + if (we.type == ROCKSDB_NAMESPACE::kDeleteRecord || + we.type == ROCKSDB_NAMESPACE::kSingleDeleteRecord || + we.type == ROCKSDB_NAMESPACE::kLogDataRecord) { + // set native handle of value slice to null if no value available + results[2] = 0; + } else { + auto* value_slice = + new ROCKSDB_NAMESPACE::Slice(we.value.data(), we.value.size()); + results[2] = GET_CPLUSPLUS_POINTER(value_slice); + } + + jlongArray jresults = env->NewLongArray(3); + if (jresults == nullptr) { + // exception thrown: OutOfMemoryError + if (results[2] != 0) { + auto* value_slice = + reinterpret_cast(results[2]); + delete value_slice; + } + delete key_slice; + return nullptr; + } + + env->SetLongArrayRegion(jresults, 0, 3, results); + if (env->ExceptionCheck()) { + // exception thrown: ArrayIndexOutOfBoundsException + env->DeleteLocalRef(jresults); + if (results[2] != 0) { + auto* value_slice = + reinterpret_cast(results[2]); + delete value_slice; + } + delete key_slice; + return nullptr; + } + + return jresults; +} + +/* + * Class: org_rocksdb_WBWIRocksIterator + * Method: refresh0 + * Signature: (J)V + */ +void Java_org_rocksdb_WBWIRocksIterator_refresh0(JNIEnv* env) { + ROCKSDB_NAMESPACE::Status s = + ROCKSDB_NAMESPACE::Status::NotSupported("Refresh() is not supported"); + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/write_buffer_manager.cc b/librocksdb-sys/rocksdb/java/rocksjni/write_buffer_manager.cc new file mode 100644 index 0000000..9ce697e --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/write_buffer_manager.cc @@ -0,0 +1,47 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "rocksdb/write_buffer_manager.h" + +#include + +#include + +#include "include/org_rocksdb_WriteBufferManager.h" +#include "rocksdb/cache.h" +#include "rocksjni/cplusplus_to_java_convert.h" + +/* + * Class: org_rocksdb_WriteBufferManager + * Method: newWriteBufferManager + * Signature: (JJ)J + */ +jlong Java_org_rocksdb_WriteBufferManager_newWriteBufferManager( + JNIEnv* /*env*/, jclass /*jclazz*/, jlong jbuffer_size, jlong jcache_handle, + jboolean allow_stall) { + auto* cache_ptr = + reinterpret_cast*>( + jcache_handle); + auto* write_buffer_manager = + new std::shared_ptr( + std::make_shared( + jbuffer_size, *cache_ptr, allow_stall)); + return GET_CPLUSPLUS_POINTER(write_buffer_manager); +} + +/* + * Class: org_rocksdb_WriteBufferManager + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_WriteBufferManager_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* write_buffer_manager = + reinterpret_cast*>( + jhandle); + assert(write_buffer_manager != nullptr); + delete write_buffer_manager; +} diff --git a/librocksdb-sys/rocksdb/java/rocksjni/writebatchhandlerjnicallback.cc b/librocksdb-sys/rocksdb/java/rocksjni/writebatchhandlerjnicallback.cc new file mode 100644 index 0000000..66ceabe --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/writebatchhandlerjnicallback.cc @@ -0,0 +1,519 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the callback "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::Comparator. + +#include "rocksjni/writebatchhandlerjnicallback.h" + +#include "rocksjni/portal.h" + +namespace ROCKSDB_NAMESPACE { +WriteBatchHandlerJniCallback::WriteBatchHandlerJniCallback( + JNIEnv* env, jobject jWriteBatchHandler) + : JniCallback(env, jWriteBatchHandler), m_env(env) { + m_jPutCfMethodId = WriteBatchHandlerJni::getPutCfMethodId(env); + if (m_jPutCfMethodId == nullptr) { + // exception thrown + return; + } + + m_jPutMethodId = WriteBatchHandlerJni::getPutMethodId(env); + if (m_jPutMethodId == nullptr) { + // exception thrown + return; + } + + m_jMergeCfMethodId = WriteBatchHandlerJni::getMergeCfMethodId(env); + if (m_jMergeCfMethodId == nullptr) { + // exception thrown + return; + } + + m_jMergeMethodId = WriteBatchHandlerJni::getMergeMethodId(env); + if (m_jMergeMethodId == nullptr) { + // exception thrown + return; + } + + m_jDeleteCfMethodId = WriteBatchHandlerJni::getDeleteCfMethodId(env); + if (m_jDeleteCfMethodId == nullptr) { + // exception thrown + return; + } + + m_jDeleteMethodId = WriteBatchHandlerJni::getDeleteMethodId(env); + if (m_jDeleteMethodId == nullptr) { + // exception thrown + return; + } + + m_jSingleDeleteCfMethodId = + WriteBatchHandlerJni::getSingleDeleteCfMethodId(env); + if (m_jSingleDeleteCfMethodId == nullptr) { + // exception thrown + return; + } + + m_jSingleDeleteMethodId = WriteBatchHandlerJni::getSingleDeleteMethodId(env); + if (m_jSingleDeleteMethodId == nullptr) { + // exception thrown + return; + } + + m_jDeleteRangeCfMethodId = + WriteBatchHandlerJni::getDeleteRangeCfMethodId(env); + if (m_jDeleteRangeCfMethodId == nullptr) { + // exception thrown + return; + } + + m_jDeleteRangeMethodId = WriteBatchHandlerJni::getDeleteRangeMethodId(env); + if (m_jDeleteRangeMethodId == nullptr) { + // exception thrown + return; + } + + m_jLogDataMethodId = WriteBatchHandlerJni::getLogDataMethodId(env); + if (m_jLogDataMethodId == nullptr) { + // exception thrown + return; + } + + m_jPutBlobIndexCfMethodId = + WriteBatchHandlerJni::getPutBlobIndexCfMethodId(env); + if (m_jPutBlobIndexCfMethodId == nullptr) { + // exception thrown + return; + } + + m_jMarkBeginPrepareMethodId = + WriteBatchHandlerJni::getMarkBeginPrepareMethodId(env); + if (m_jMarkBeginPrepareMethodId == nullptr) { + // exception thrown + return; + } + + m_jMarkEndPrepareMethodId = + WriteBatchHandlerJni::getMarkEndPrepareMethodId(env); + if (m_jMarkEndPrepareMethodId == nullptr) { + // exception thrown + return; + } + + m_jMarkNoopMethodId = WriteBatchHandlerJni::getMarkNoopMethodId(env); + if (m_jMarkNoopMethodId == nullptr) { + // exception thrown + return; + } + + m_jMarkRollbackMethodId = WriteBatchHandlerJni::getMarkRollbackMethodId(env); + if (m_jMarkRollbackMethodId == nullptr) { + // exception thrown + return; + } + + m_jMarkCommitMethodId = WriteBatchHandlerJni::getMarkCommitMethodId(env); + if (m_jMarkCommitMethodId == nullptr) { + // exception thrown + return; + } + + m_jMarkCommitWithTimestampMethodId = + WriteBatchHandlerJni::getMarkCommitWithTimestampMethodId(env); + if (m_jMarkCommitWithTimestampMethodId == nullptr) { + // exception thrown + return; + } + + m_jContinueMethodId = WriteBatchHandlerJni::getContinueMethodId(env); + if (m_jContinueMethodId == nullptr) { + // exception thrown + return; + } +} + +ROCKSDB_NAMESPACE::Status WriteBatchHandlerJniCallback::PutCF( + uint32_t column_family_id, const Slice& key, const Slice& value) { + auto put = [this, column_family_id](jbyteArray j_key, jbyteArray j_value) { + m_env->CallVoidMethod(m_jcallback_obj, m_jPutCfMethodId, + static_cast(column_family_id), j_key, j_value); + }; + auto status = WriteBatchHandlerJniCallback::kv_op(key, value, put); + if (status == nullptr) { + return ROCKSDB_NAMESPACE::Status::OK(); // TODO(AR) what to do if there is + // an Exception but we don't know + // the ROCKSDB_NAMESPACE::Status? + } else { + return ROCKSDB_NAMESPACE::Status(*status); + } +} + +void WriteBatchHandlerJniCallback::Put(const Slice& key, const Slice& value) { + auto put = [this](jbyteArray j_key, jbyteArray j_value) { + m_env->CallVoidMethod(m_jcallback_obj, m_jPutMethodId, j_key, j_value); + }; + WriteBatchHandlerJniCallback::kv_op(key, value, put); +} + +ROCKSDB_NAMESPACE::Status WriteBatchHandlerJniCallback::MergeCF( + uint32_t column_family_id, const Slice& key, const Slice& value) { + auto merge = [this, column_family_id](jbyteArray j_key, jbyteArray j_value) { + m_env->CallVoidMethod(m_jcallback_obj, m_jMergeCfMethodId, + static_cast(column_family_id), j_key, j_value); + }; + auto status = WriteBatchHandlerJniCallback::kv_op(key, value, merge); + if (status == nullptr) { + return ROCKSDB_NAMESPACE::Status::OK(); // TODO(AR) what to do if there is + // an Exception but we don't know + // the ROCKSDB_NAMESPACE::Status? + } else { + return ROCKSDB_NAMESPACE::Status(*status); + } +} + +void WriteBatchHandlerJniCallback::Merge(const Slice& key, const Slice& value) { + auto merge = [this](jbyteArray j_key, jbyteArray j_value) { + m_env->CallVoidMethod(m_jcallback_obj, m_jMergeMethodId, j_key, j_value); + }; + WriteBatchHandlerJniCallback::kv_op(key, value, merge); +} + +ROCKSDB_NAMESPACE::Status WriteBatchHandlerJniCallback::DeleteCF( + uint32_t column_family_id, const Slice& key) { + auto remove = [this, column_family_id](jbyteArray j_key) { + m_env->CallVoidMethod(m_jcallback_obj, m_jDeleteCfMethodId, + static_cast(column_family_id), j_key); + }; + auto status = WriteBatchHandlerJniCallback::k_op(key, remove); + if (status == nullptr) { + return ROCKSDB_NAMESPACE::Status::OK(); // TODO(AR) what to do if there is + // an Exception but we don't know + // the ROCKSDB_NAMESPACE::Status? + } else { + return ROCKSDB_NAMESPACE::Status(*status); + } +} + +void WriteBatchHandlerJniCallback::Delete(const Slice& key) { + auto remove = [this](jbyteArray j_key) { + m_env->CallVoidMethod(m_jcallback_obj, m_jDeleteMethodId, j_key); + }; + WriteBatchHandlerJniCallback::k_op(key, remove); +} + +ROCKSDB_NAMESPACE::Status WriteBatchHandlerJniCallback::SingleDeleteCF( + uint32_t column_family_id, const Slice& key) { + auto singleDelete = [this, column_family_id](jbyteArray j_key) { + m_env->CallVoidMethod(m_jcallback_obj, m_jSingleDeleteCfMethodId, + static_cast(column_family_id), j_key); + }; + auto status = WriteBatchHandlerJniCallback::k_op(key, singleDelete); + if (status == nullptr) { + return ROCKSDB_NAMESPACE::Status::OK(); // TODO(AR) what to do if there is + // an Exception but we don't know + // the ROCKSDB_NAMESPACE::Status? + } else { + return ROCKSDB_NAMESPACE::Status(*status); + } +} + +void WriteBatchHandlerJniCallback::SingleDelete(const Slice& key) { + auto singleDelete = [this](jbyteArray j_key) { + m_env->CallVoidMethod(m_jcallback_obj, m_jSingleDeleteMethodId, j_key); + }; + WriteBatchHandlerJniCallback::k_op(key, singleDelete); +} + +ROCKSDB_NAMESPACE::Status WriteBatchHandlerJniCallback::DeleteRangeCF( + uint32_t column_family_id, const Slice& beginKey, const Slice& endKey) { + auto deleteRange = [this, column_family_id](jbyteArray j_beginKey, + jbyteArray j_endKey) { + m_env->CallVoidMethod(m_jcallback_obj, m_jDeleteRangeCfMethodId, + static_cast(column_family_id), j_beginKey, + j_endKey); + }; + auto status = + WriteBatchHandlerJniCallback::kv_op(beginKey, endKey, deleteRange); + if (status == nullptr) { + return ROCKSDB_NAMESPACE::Status::OK(); // TODO(AR) what to do if there is + // an Exception but we don't know + // the ROCKSDB_NAMESPACE::Status? + } else { + return ROCKSDB_NAMESPACE::Status(*status); + } +} + +void WriteBatchHandlerJniCallback::DeleteRange(const Slice& beginKey, + const Slice& endKey) { + auto deleteRange = [this](jbyteArray j_beginKey, jbyteArray j_endKey) { + m_env->CallVoidMethod(m_jcallback_obj, m_jDeleteRangeMethodId, j_beginKey, + j_endKey); + }; + WriteBatchHandlerJniCallback::kv_op(beginKey, endKey, deleteRange); +} + +void WriteBatchHandlerJniCallback::LogData(const Slice& blob) { + auto logData = [this](jbyteArray j_blob) { + m_env->CallVoidMethod(m_jcallback_obj, m_jLogDataMethodId, j_blob); + }; + WriteBatchHandlerJniCallback::k_op(blob, logData); +} + +ROCKSDB_NAMESPACE::Status WriteBatchHandlerJniCallback::PutBlobIndexCF( + uint32_t column_family_id, const Slice& key, const Slice& value) { + auto putBlobIndex = [this, column_family_id](jbyteArray j_key, + jbyteArray j_value) { + m_env->CallVoidMethod(m_jcallback_obj, m_jPutBlobIndexCfMethodId, + static_cast(column_family_id), j_key, j_value); + }; + auto status = WriteBatchHandlerJniCallback::kv_op(key, value, putBlobIndex); + if (status == nullptr) { + return ROCKSDB_NAMESPACE::Status::OK(); // TODO(AR) what to do if there is + // an Exception but we don't know + // the ROCKSDB_NAMESPACE::Status? + } else { + return ROCKSDB_NAMESPACE::Status(*status); + } +} + +ROCKSDB_NAMESPACE::Status WriteBatchHandlerJniCallback::MarkBeginPrepare( + bool unprepare) { +#ifndef DEBUG + (void)unprepare; +#else + assert(!unprepare); +#endif + m_env->CallVoidMethod(m_jcallback_obj, m_jMarkBeginPrepareMethodId); + + // check for Exception, in-particular RocksDBException + if (m_env->ExceptionCheck()) { + // exception thrown + jthrowable exception = m_env->ExceptionOccurred(); + std::unique_ptr status = + ROCKSDB_NAMESPACE::RocksDBExceptionJni::toCppStatus(m_env, exception); + if (status == nullptr) { + // unkown status or exception occurred extracting status + m_env->ExceptionDescribe(); + return ROCKSDB_NAMESPACE::Status::OK(); // TODO(AR) probably need a + // better error code here + + } else { + m_env->ExceptionClear(); // clear the exception, as we have extracted the + // status + return ROCKSDB_NAMESPACE::Status(*status); + } + } + + return ROCKSDB_NAMESPACE::Status::OK(); +} + +ROCKSDB_NAMESPACE::Status WriteBatchHandlerJniCallback::MarkEndPrepare( + const Slice& xid) { + auto markEndPrepare = [this](jbyteArray j_xid) { + m_env->CallVoidMethod(m_jcallback_obj, m_jMarkEndPrepareMethodId, j_xid); + }; + auto status = WriteBatchHandlerJniCallback::k_op(xid, markEndPrepare); + if (status == nullptr) { + return ROCKSDB_NAMESPACE::Status::OK(); // TODO(AR) what to do if there is + // an Exception but we don't know + // the ROCKSDB_NAMESPACE::Status? + } else { + return ROCKSDB_NAMESPACE::Status(*status); + } +} + +ROCKSDB_NAMESPACE::Status WriteBatchHandlerJniCallback::MarkNoop( + bool empty_batch) { + m_env->CallVoidMethod(m_jcallback_obj, m_jMarkNoopMethodId, + static_cast(empty_batch)); + + // check for Exception, in-particular RocksDBException + if (m_env->ExceptionCheck()) { + // exception thrown + jthrowable exception = m_env->ExceptionOccurred(); + std::unique_ptr status = + ROCKSDB_NAMESPACE::RocksDBExceptionJni::toCppStatus(m_env, exception); + if (status == nullptr) { + // unkown status or exception occurred extracting status + m_env->ExceptionDescribe(); + return ROCKSDB_NAMESPACE::Status::OK(); // TODO(AR) probably need a + // better error code here + + } else { + m_env->ExceptionClear(); // clear the exception, as we have extracted the + // status + return ROCKSDB_NAMESPACE::Status(*status); + } + } + + return ROCKSDB_NAMESPACE::Status::OK(); +} + +ROCKSDB_NAMESPACE::Status WriteBatchHandlerJniCallback::MarkRollback( + const Slice& xid) { + auto markRollback = [this](jbyteArray j_xid) { + m_env->CallVoidMethod(m_jcallback_obj, m_jMarkRollbackMethodId, j_xid); + }; + auto status = WriteBatchHandlerJniCallback::k_op(xid, markRollback); + if (status == nullptr) { + return ROCKSDB_NAMESPACE::Status::OK(); // TODO(AR) what to do if there is + // an Exception but we don't know + // the ROCKSDB_NAMESPACE::Status? + } else { + return ROCKSDB_NAMESPACE::Status(*status); + } +} + +ROCKSDB_NAMESPACE::Status WriteBatchHandlerJniCallback::MarkCommit( + const Slice& xid) { + auto markCommit = [this](jbyteArray j_xid) { + m_env->CallVoidMethod(m_jcallback_obj, m_jMarkCommitMethodId, j_xid); + }; + auto status = WriteBatchHandlerJniCallback::k_op(xid, markCommit); + if (status == nullptr) { + return ROCKSDB_NAMESPACE::Status::OK(); // TODO(AR) what to do if there is + // an Exception but we don't know + // the ROCKSDB_NAMESPACE::Status? + } else { + return ROCKSDB_NAMESPACE::Status(*status); + } +} + +ROCKSDB_NAMESPACE::Status WriteBatchHandlerJniCallback::MarkCommitWithTimestamp( + const Slice& xid, const Slice& ts) { + auto markCommitWithTimestamp = [this](jbyteArray j_xid, jbyteArray j_ts) { + m_env->CallVoidMethod(m_jcallback_obj, m_jMarkCommitWithTimestampMethodId, + j_xid, j_ts); + }; + auto status = + WriteBatchHandlerJniCallback::kv_op(xid, ts, markCommitWithTimestamp); + if (status == nullptr) { + return ROCKSDB_NAMESPACE::Status::OK(); // TODO(AR) what to do if there is + // an Exception but we don't know + // the ROCKSDB_NAMESPACE::Status? + } else { + return ROCKSDB_NAMESPACE::Status(*status); + } +} + +bool WriteBatchHandlerJniCallback::Continue() { + jboolean jContinue = + m_env->CallBooleanMethod(m_jcallback_obj, m_jContinueMethodId); + if (m_env->ExceptionCheck()) { + // exception thrown + m_env->ExceptionDescribe(); + } + + return static_cast(jContinue == JNI_TRUE); +} + +std::unique_ptr WriteBatchHandlerJniCallback::kv_op( + const Slice& key, const Slice& value, + std::function kvFn) { + const jbyteArray j_key = JniUtil::copyBytes(m_env, key); + if (j_key == nullptr) { + // exception thrown + if (m_env->ExceptionCheck()) { + m_env->ExceptionDescribe(); + } + return nullptr; + } + + const jbyteArray j_value = JniUtil::copyBytes(m_env, value); + if (j_value == nullptr) { + // exception thrown + if (m_env->ExceptionCheck()) { + m_env->ExceptionDescribe(); + } + if (j_key != nullptr) { + m_env->DeleteLocalRef(j_key); + } + return nullptr; + } + + kvFn(j_key, j_value); + + // check for Exception, in-particular RocksDBException + if (m_env->ExceptionCheck()) { + if (j_value != nullptr) { + m_env->DeleteLocalRef(j_value); + } + if (j_key != nullptr) { + m_env->DeleteLocalRef(j_key); + } + + // exception thrown + jthrowable exception = m_env->ExceptionOccurred(); + std::unique_ptr status = + ROCKSDB_NAMESPACE::RocksDBExceptionJni::toCppStatus(m_env, exception); + if (status == nullptr) { + // unkown status or exception occurred extracting status + m_env->ExceptionDescribe(); + return nullptr; + + } else { + m_env->ExceptionClear(); // clear the exception, as we have extracted the + // status + return status; + } + } + + if (j_value != nullptr) { + m_env->DeleteLocalRef(j_value); + } + if (j_key != nullptr) { + m_env->DeleteLocalRef(j_key); + } + + // all OK + return std::unique_ptr( + new ROCKSDB_NAMESPACE::Status(ROCKSDB_NAMESPACE::Status::OK())); +} + +std::unique_ptr WriteBatchHandlerJniCallback::k_op( + const Slice& key, std::function kFn) { + const jbyteArray j_key = JniUtil::copyBytes(m_env, key); + if (j_key == nullptr) { + // exception thrown + if (m_env->ExceptionCheck()) { + m_env->ExceptionDescribe(); + } + return nullptr; + } + + kFn(j_key); + + // check for Exception, in-particular RocksDBException + if (m_env->ExceptionCheck()) { + if (j_key != nullptr) { + m_env->DeleteLocalRef(j_key); + } + + // exception thrown + jthrowable exception = m_env->ExceptionOccurred(); + std::unique_ptr status = + ROCKSDB_NAMESPACE::RocksDBExceptionJni::toCppStatus(m_env, exception); + if (status == nullptr) { + // unkown status or exception occurred extracting status + m_env->ExceptionDescribe(); + return nullptr; + + } else { + m_env->ExceptionClear(); // clear the exception, as we have extracted the + // status + return status; + } + } + + if (j_key != nullptr) { + m_env->DeleteLocalRef(j_key); + } + + // all OK + return std::unique_ptr( + new ROCKSDB_NAMESPACE::Status(ROCKSDB_NAMESPACE::Status::OK())); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/java/rocksjni/writebatchhandlerjnicallback.h b/librocksdb-sys/rocksdb/java/rocksjni/writebatchhandlerjnicallback.h new file mode 100644 index 0000000..9629797 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/rocksjni/writebatchhandlerjnicallback.h @@ -0,0 +1,92 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// This file implements the callback "bridge" between Java and C++ for +// ROCKSDB_NAMESPACE::WriteBatch::Handler. + +#ifndef JAVA_ROCKSJNI_WRITEBATCHHANDLERJNICALLBACK_H_ +#define JAVA_ROCKSJNI_WRITEBATCHHANDLERJNICALLBACK_H_ + +#include + +#include +#include + +#include "rocksdb/write_batch.h" +#include "rocksjni/jnicallback.h" + +namespace ROCKSDB_NAMESPACE { +/** + * This class acts as a bridge between C++ + * and Java. The methods in this class will be + * called back from the RocksDB storage engine (C++) + * which calls the appropriate Java method. + * This enables Write Batch Handlers to be implemented in Java. + */ +class WriteBatchHandlerJniCallback : public JniCallback, + public WriteBatch::Handler { + public: + WriteBatchHandlerJniCallback(JNIEnv* env, jobject jWriteBackHandler); + Status PutCF(uint32_t column_family_id, const Slice& key, const Slice& value); + void Put(const Slice& key, const Slice& value); + Status MergeCF(uint32_t column_family_id, const Slice& key, + const Slice& value); + void Merge(const Slice& key, const Slice& value); + Status DeleteCF(uint32_t column_family_id, const Slice& key); + void Delete(const Slice& key); + Status SingleDeleteCF(uint32_t column_family_id, const Slice& key); + void SingleDelete(const Slice& key); + Status DeleteRangeCF(uint32_t column_family_id, const Slice& beginKey, + const Slice& endKey); + void DeleteRange(const Slice& beginKey, const Slice& endKey); + void LogData(const Slice& blob); + Status PutBlobIndexCF(uint32_t column_family_id, const Slice& key, + const Slice& value); + Status MarkBeginPrepare(bool); + Status MarkEndPrepare(const Slice& xid); + Status MarkNoop(bool empty_batch); + Status MarkRollback(const Slice& xid); + Status MarkCommit(const Slice& xid); + Status MarkCommitWithTimestamp(const Slice& xid, const Slice& commit_ts); + bool Continue(); + + private: + JNIEnv* m_env; + jmethodID m_jPutCfMethodId; + jmethodID m_jPutMethodId; + jmethodID m_jMergeCfMethodId; + jmethodID m_jMergeMethodId; + jmethodID m_jDeleteCfMethodId; + jmethodID m_jDeleteMethodId; + jmethodID m_jSingleDeleteCfMethodId; + jmethodID m_jSingleDeleteMethodId; + jmethodID m_jDeleteRangeCfMethodId; + jmethodID m_jDeleteRangeMethodId; + jmethodID m_jLogDataMethodId; + jmethodID m_jPutBlobIndexCfMethodId; + jmethodID m_jMarkBeginPrepareMethodId; + jmethodID m_jMarkEndPrepareMethodId; + jmethodID m_jMarkNoopMethodId; + jmethodID m_jMarkRollbackMethodId; + jmethodID m_jMarkCommitMethodId; + jmethodID m_jMarkCommitWithTimestampMethodId; + jmethodID m_jContinueMethodId; + /** + * @return A pointer to a ROCKSDB_NAMESPACE::Status or nullptr if an + * unexpected exception occurred + */ + std::unique_ptr kv_op( + const Slice& key, const Slice& value, + std::function kvFn); + /** + * @return A pointer to a ROCKSDB_NAMESPACE::Status or nullptr if an + * unexpected exception occurred + */ + std::unique_ptr k_op( + const Slice& key, std::function kFn); +}; +} // namespace ROCKSDB_NAMESPACE + +#endif // JAVA_ROCKSJNI_WRITEBATCHHANDLERJNICALLBACK_H_ diff --git a/librocksdb-sys/rocksdb/java/samples/src/main/java/OptimisticTransactionSample.java b/librocksdb-sys/rocksdb/java/samples/src/main/java/OptimisticTransactionSample.java new file mode 100644 index 0000000..7e7a22e --- /dev/null +++ b/librocksdb-sys/rocksdb/java/samples/src/main/java/OptimisticTransactionSample.java @@ -0,0 +1,184 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +import org.rocksdb.*; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Demonstrates using Transactions on an OptimisticTransactionDB with + * varying isolation guarantees + */ +public class OptimisticTransactionSample { + private static final String dbPath = "/tmp/rocksdb_optimistic_transaction_example"; + + public static final void main(final String args[]) throws RocksDBException { + + try(final Options options = new Options() + .setCreateIfMissing(true); + final OptimisticTransactionDB txnDb = + OptimisticTransactionDB.open(options, dbPath)) { + + try (final WriteOptions writeOptions = new WriteOptions(); + final ReadOptions readOptions = new ReadOptions()) { + + //////////////////////////////////////////////////////// + // + // Simple OptimisticTransaction Example ("Read Committed") + // + //////////////////////////////////////////////////////// + readCommitted(txnDb, writeOptions, readOptions); + + + //////////////////////////////////////////////////////// + // + // "Repeatable Read" (Snapshot Isolation) Example + // -- Using a single Snapshot + // + //////////////////////////////////////////////////////// + repeatableRead(txnDb, writeOptions, readOptions); + + + //////////////////////////////////////////////////////// + // + // "Read Committed" (Monotonic Atomic Views) Example + // --Using multiple Snapshots + // + //////////////////////////////////////////////////////// + readCommitted_monotonicAtomicViews(txnDb, writeOptions, readOptions); + } + } + } + + /** + * Demonstrates "Read Committed" isolation + */ + private static void readCommitted(final OptimisticTransactionDB txnDb, + final WriteOptions writeOptions, final ReadOptions readOptions) + throws RocksDBException { + final byte key1[] = "abc".getBytes(UTF_8); + final byte value1[] = "def".getBytes(UTF_8); + + final byte key2[] = "xyz".getBytes(UTF_8); + final byte value2[] = "zzz".getBytes(UTF_8); + + // Start a transaction + try(final Transaction txn = txnDb.beginTransaction(writeOptions)) { + // Read a key in this transaction + byte[] value = txn.get(readOptions, key1); + assert(value == null); + + // Write a key in this transaction + txn.put(key1, value1); + + // Read a key OUTSIDE this transaction. Does not affect txn. + value = txnDb.get(readOptions, key1); + assert(value == null); + + // Write a key OUTSIDE of this transaction. + // Does not affect txn since this is an unrelated key. + // If we wrote key 'abc' here, the transaction would fail to commit. + txnDb.put(writeOptions, key2, value2); + + // Commit transaction + txn.commit(); + } + } + + /** + * Demonstrates "Repeatable Read" (Snapshot Isolation) isolation + */ + private static void repeatableRead(final OptimisticTransactionDB txnDb, + final WriteOptions writeOptions, final ReadOptions readOptions) + throws RocksDBException { + + final byte key1[] = "ghi".getBytes(UTF_8); + final byte value1[] = "jkl".getBytes(UTF_8); + + // Set a snapshot at start of transaction by setting setSnapshot(true) + try(final OptimisticTransactionOptions txnOptions = + new OptimisticTransactionOptions().setSetSnapshot(true); + final Transaction txn = + txnDb.beginTransaction(writeOptions, txnOptions)) { + + final Snapshot snapshot = txn.getSnapshot(); + + // Write a key OUTSIDE of transaction + txnDb.put(writeOptions, key1, value1); + + // Read a key using the snapshot. + readOptions.setSnapshot(snapshot); + final byte[] value = txn.getForUpdate(readOptions, key1, true); + assert (value == null); + + try { + // Attempt to commit transaction + txn.commit(); + throw new IllegalStateException(); + } catch(final RocksDBException e) { + // Transaction could not commit since the write outside of the txn + // conflicted with the read! + assert(e.getStatus().getCode() == Status.Code.Busy); + } + + txn.rollback(); + } finally { + // Clear snapshot from read options since it is no longer valid + readOptions.setSnapshot(null); + } + } + + /** + * Demonstrates "Read Committed" (Monotonic Atomic Views) isolation + * + * In this example, we set the snapshot multiple times. This is probably + * only necessary if you have very strict isolation requirements to + * implement. + */ + private static void readCommitted_monotonicAtomicViews( + final OptimisticTransactionDB txnDb, final WriteOptions writeOptions, + final ReadOptions readOptions) throws RocksDBException { + + final byte keyX[] = "x".getBytes(UTF_8); + final byte valueX[] = "x".getBytes(UTF_8); + + final byte keyY[] = "y".getBytes(UTF_8); + final byte valueY[] = "y".getBytes(UTF_8); + + try (final OptimisticTransactionOptions txnOptions = + new OptimisticTransactionOptions().setSetSnapshot(true); + final Transaction txn = + txnDb.beginTransaction(writeOptions, txnOptions)) { + + // Do some reads and writes to key "x" + Snapshot snapshot = txnDb.getSnapshot(); + readOptions.setSnapshot(snapshot); + byte[] value = txn.get(readOptions, keyX); + txn.put(valueX, valueX); + + // Do a write outside of the transaction to key "y" + txnDb.put(writeOptions, keyY, valueY); + + // Set a new snapshot in the transaction + txn.setSnapshot(); + snapshot = txnDb.getSnapshot(); + readOptions.setSnapshot(snapshot); + + // Do some reads and writes to key "y" + // Since the snapshot was advanced, the write done outside of the + // transaction does not conflict. + value = txn.getForUpdate(readOptions, keyY, true); + txn.put(keyY, valueY); + + // Commit. Since the snapshot was advanced, the write done outside of the + // transaction does not prevent this transaction from Committing. + txn.commit(); + + } finally { + // Clear snapshot from read options since it is no longer valid + readOptions.setSnapshot(null); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/samples/src/main/java/RocksDBColumnFamilySample.java b/librocksdb-sys/rocksdb/java/samples/src/main/java/RocksDBColumnFamilySample.java new file mode 100644 index 0000000..72f5731 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/samples/src/main/java/RocksDBColumnFamilySample.java @@ -0,0 +1,78 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +import org.rocksdb.*; + +import java.util.ArrayList; +import java.util.List; + +public class RocksDBColumnFamilySample { + static { + RocksDB.loadLibrary(); + } + + public static void main(final String[] args) throws RocksDBException { + if (args.length < 1) { + System.out.println( + "usage: RocksDBColumnFamilySample db_path"); + System.exit(-1); + } + + final String db_path = args[0]; + + System.out.println("RocksDBColumnFamilySample"); + try(final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, db_path)) { + + assert(db != null); + + // create column family + try(final ColumnFamilyHandle columnFamilyHandle = db.createColumnFamily( + new ColumnFamilyDescriptor("new_cf".getBytes(), + new ColumnFamilyOptions()))) { + assert (columnFamilyHandle != null); + } + } + + // open DB with two column families + final List columnFamilyDescriptors = + new ArrayList<>(); + // have to open default column family + columnFamilyDescriptors.add(new ColumnFamilyDescriptor( + RocksDB.DEFAULT_COLUMN_FAMILY, new ColumnFamilyOptions())); + // open the new one, too + columnFamilyDescriptors.add(new ColumnFamilyDescriptor( + "new_cf".getBytes(), new ColumnFamilyOptions())); + final List columnFamilyHandles = new ArrayList<>(); + try(final DBOptions options = new DBOptions(); + final RocksDB db = RocksDB.open(options, db_path, + columnFamilyDescriptors, columnFamilyHandles)) { + assert(db != null); + + try { + // put and get from non-default column family + db.put( + columnFamilyHandles.get(1), new WriteOptions(), "key".getBytes(), "value".getBytes()); + + // atomic write + try (final WriteBatch wb = new WriteBatch()) { + wb.put(columnFamilyHandles.get(0), "key2".getBytes(), + "value2".getBytes()); + wb.put(columnFamilyHandles.get(1), "key3".getBytes(), + "value3".getBytes()); + wb.delete(columnFamilyHandles.get(1), "key".getBytes()); + db.write(new WriteOptions(), wb); + } + + // drop column family + db.dropColumnFamily(columnFamilyHandles.get(1)); + } finally { + for (final ColumnFamilyHandle handle : columnFamilyHandles) { + handle.close(); + } + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/samples/src/main/java/RocksDBSample.java b/librocksdb-sys/rocksdb/java/samples/src/main/java/RocksDBSample.java new file mode 100644 index 0000000..8ab9b2d --- /dev/null +++ b/librocksdb-sys/rocksdb/java/samples/src/main/java/RocksDBSample.java @@ -0,0 +1,295 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +import java.lang.IllegalArgumentException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.ArrayList; + +import org.rocksdb.*; +import org.rocksdb.util.SizeUnit; + +public class RocksDBSample { + static { + RocksDB.loadLibrary(); + } + + public static void main(final String[] args) { + if (args.length < 1) { + System.out.println("usage: RocksDBSample db_path"); + System.exit(-1); + } + + final String db_path = args[0]; + final String db_path_not_found = db_path + "_not_found"; + + System.out.println("RocksDBSample"); + try (final Options options = new Options(); + final Filter bloomFilter = new BloomFilter(10); + final ReadOptions readOptions = new ReadOptions() + .setFillCache(false); + final Statistics stats = new Statistics(); + final RateLimiter rateLimiter = new RateLimiter(10000000,10000, 10)) { + + try (final RocksDB db = RocksDB.open(options, db_path_not_found)) { + assert (false); + } catch (final RocksDBException e) { + System.out.format("Caught the expected exception -- %s\n", e); + } + + try { + options.setCreateIfMissing(true) + .setStatistics(stats) + .setWriteBufferSize(8 * SizeUnit.KB) + .setMaxWriteBufferNumber(3) + .setMaxBackgroundJobs(10) + .setCompressionType(CompressionType.ZLIB_COMPRESSION) + .setCompactionStyle(CompactionStyle.UNIVERSAL); + } catch (final IllegalArgumentException e) { + assert (false); + } + + assert (options.createIfMissing() == true); + assert (options.writeBufferSize() == 8 * SizeUnit.KB); + assert (options.maxWriteBufferNumber() == 3); + assert (options.maxBackgroundJobs() == 10); + assert (options.compressionType() == CompressionType.ZLIB_COMPRESSION); + assert (options.compactionStyle() == CompactionStyle.UNIVERSAL); + + assert (options.memTableFactoryName().equals("SkipListFactory")); + options.setMemTableConfig( + new HashSkipListMemTableConfig() + .setHeight(4) + .setBranchingFactor(4) + .setBucketCount(2000000)); + assert (options.memTableFactoryName().equals("HashSkipListRepFactory")); + + options.setMemTableConfig( + new HashLinkedListMemTableConfig() + .setBucketCount(100000)); + assert (options.memTableFactoryName().equals("HashLinkedListRepFactory")); + + options.setMemTableConfig( + new VectorMemTableConfig().setReservedSize(10000)); + assert (options.memTableFactoryName().equals("VectorRepFactory")); + + options.setMemTableConfig(new SkipListMemTableConfig()); + assert (options.memTableFactoryName().equals("SkipListFactory")); + + options.setTableFormatConfig(new PlainTableConfig()); + // Plain-Table requires mmap read + options.setAllowMmapReads(true); + assert (options.tableFactoryName().equals("PlainTable")); + + options.setRateLimiter(rateLimiter); + + final BlockBasedTableConfig table_options = new BlockBasedTableConfig(); + Cache cache = new LRUCache(64 * 1024, 6); + table_options.setBlockCache(cache) + .setFilterPolicy(bloomFilter) + .setBlockSizeDeviation(5) + .setBlockRestartInterval(10) + .setCacheIndexAndFilterBlocks(true); + + assert (table_options.blockSizeDeviation() == 5); + assert (table_options.blockRestartInterval() == 10); + assert (table_options.cacheIndexAndFilterBlocks() == true); + + options.setTableFormatConfig(table_options); + assert (options.tableFactoryName().equals("BlockBasedTable")); + + try (final RocksDB db = RocksDB.open(options, db_path)) { + db.put("hello".getBytes(), "world".getBytes()); + + final byte[] value = db.get("hello".getBytes()); + assert ("world".equals(new String(value))); + + final String str = db.getProperty("rocksdb.stats"); + assert (str != null && !str.equals("")); + } catch (final RocksDBException e) { + System.out.format("[ERROR] caught the unexpected exception -- %s\n", e); + assert (false); + } + + try (final RocksDB db = RocksDB.open(options, db_path)) { + db.put("hello".getBytes(), "world".getBytes()); + byte[] value = db.get("hello".getBytes()); + System.out.format("Get('hello') = %s\n", + new String(value)); + + for (int i = 1; i <= 9; ++i) { + for (int j = 1; j <= 9; ++j) { + db.put(String.format("%dx%d", i, j).getBytes(), + String.format("%d", i * j).getBytes()); + } + } + + for (int i = 1; i <= 9; ++i) { + for (int j = 1; j <= 9; ++j) { + System.out.format("%s ", new String(db.get( + String.format("%dx%d", i, j).getBytes()))); + } + System.out.println(""); + } + + // write batch test + try (final WriteOptions writeOpt = new WriteOptions()) { + for (int i = 10; i <= 19; ++i) { + try (final WriteBatch batch = new WriteBatch()) { + for (int j = 10; j <= 19; ++j) { + batch.put(String.format("%dx%d", i, j).getBytes(), + String.format("%d", i * j).getBytes()); + } + db.write(writeOpt, batch); + } + } + } + for (int i = 10; i <= 19; ++i) { + for (int j = 10; j <= 19; ++j) { + assert (new String( + db.get(String.format("%dx%d", i, j).getBytes())).equals( + String.format("%d", i * j))); + System.out.format("%s ", new String(db.get( + String.format("%dx%d", i, j).getBytes()))); + } + System.out.println(""); + } + + value = db.get("1x1".getBytes()); + assert (value != null); + value = db.get("world".getBytes()); + assert (value == null); + value = db.get(readOptions, "world".getBytes()); + assert (value == null); + + final byte[] testKey = "asdf".getBytes(); + final byte[] testValue = + "asdfghjkl;'?> insufficientArray.length); + len = db.get("asdfjkl;".getBytes(), enoughArray); + assert (len == RocksDB.NOT_FOUND); + len = db.get(testKey, enoughArray); + assert (len == testValue.length); + + len = db.get(readOptions, testKey, insufficientArray); + assert (len > insufficientArray.length); + len = db.get(readOptions, "asdfjkl;".getBytes(), enoughArray); + assert (len == RocksDB.NOT_FOUND); + len = db.get(readOptions, testKey, enoughArray); + assert (len == testValue.length); + + db.delete(testKey); + len = db.get(testKey, enoughArray); + assert (len == RocksDB.NOT_FOUND); + + // repeat the test with WriteOptions + try (final WriteOptions writeOpts = new WriteOptions()) { + writeOpts.setSync(true); + writeOpts.setDisableWAL(false); + db.put(writeOpts, testKey, testValue); + len = db.get(testKey, enoughArray); + assert (len == testValue.length); + assert (new String(testValue).equals( + new String(enoughArray, 0, len))); + } + + try { + for (final TickerType statsType : TickerType.values()) { + if (statsType != TickerType.TICKER_ENUM_MAX) { + stats.getTickerCount(statsType); + } + } + System.out.println("getTickerCount() passed."); + } catch (final Exception e) { + System.out.println("Failed in call to getTickerCount()"); + assert (false); //Should never reach here. + } + + try { + for (final HistogramType histogramType : HistogramType.values()) { + if (histogramType != HistogramType.HISTOGRAM_ENUM_MAX) { + HistogramData data = stats.getHistogramData(histogramType); + } + } + System.out.println("getHistogramData() passed."); + } catch (final Exception e) { + System.out.println("Failed in call to getHistogramData()"); + assert (false); //Should never reach here. + } + + try (final RocksIterator iterator = db.newIterator()) { + + boolean seekToFirstPassed = false; + for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { + iterator.status(); + assert (iterator.key() != null); + assert (iterator.value() != null); + seekToFirstPassed = true; + } + if (seekToFirstPassed) { + System.out.println("iterator seekToFirst tests passed."); + } + + boolean seekToLastPassed = false; + for (iterator.seekToLast(); iterator.isValid(); iterator.prev()) { + iterator.status(); + assert (iterator.key() != null); + assert (iterator.value() != null); + seekToLastPassed = true; + } + + if (seekToLastPassed) { + System.out.println("iterator seekToLastPassed tests passed."); + } + + iterator.seekToFirst(); + iterator.seek(iterator.key()); + assert (iterator.key() != null); + assert (iterator.value() != null); + + System.out.println("iterator seek test passed."); + + } + System.out.println("iterator tests passed."); + + final List keys = new ArrayList<>(); + try (final RocksIterator iterator = db.newIterator()) { + for (iterator.seekToLast(); iterator.isValid(); iterator.prev()) { + keys.add(iterator.key()); + } + } + + List values = db.multiGetAsList(keys); + assert (values.size() == keys.size()); + for (final byte[] value1 : values) { + assert (value1 != null); + } + + values = db.multiGetAsList(new ReadOptions(), keys); + assert (values.size() == keys.size()); + for (final byte[] value1 : values) { + assert (value1 != null); + } + } catch (final RocksDBException e) { + System.err.println(e); + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/samples/src/main/java/TransactionSample.java b/librocksdb-sys/rocksdb/java/samples/src/main/java/TransactionSample.java new file mode 100644 index 0000000..b88a68f --- /dev/null +++ b/librocksdb-sys/rocksdb/java/samples/src/main/java/TransactionSample.java @@ -0,0 +1,183 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +import org.rocksdb.*; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Demonstrates using Transactions on a TransactionDB with + * varying isolation guarantees + */ +public class TransactionSample { + private static final String dbPath = "/tmp/rocksdb_transaction_example"; + + public static final void main(final String args[]) throws RocksDBException { + + try(final Options options = new Options() + .setCreateIfMissing(true); + final TransactionDBOptions txnDbOptions = new TransactionDBOptions(); + final TransactionDB txnDb = + TransactionDB.open(options, txnDbOptions, dbPath)) { + + try (final WriteOptions writeOptions = new WriteOptions(); + final ReadOptions readOptions = new ReadOptions()) { + + //////////////////////////////////////////////////////// + // + // Simple Transaction Example ("Read Committed") + // + //////////////////////////////////////////////////////// + readCommitted(txnDb, writeOptions, readOptions); + + + //////////////////////////////////////////////////////// + // + // "Repeatable Read" (Snapshot Isolation) Example + // -- Using a single Snapshot + // + //////////////////////////////////////////////////////// + repeatableRead(txnDb, writeOptions, readOptions); + + + //////////////////////////////////////////////////////// + // + // "Read Committed" (Monotonic Atomic Views) Example + // --Using multiple Snapshots + // + //////////////////////////////////////////////////////// + readCommitted_monotonicAtomicViews(txnDb, writeOptions, readOptions); + } + } + } + + /** + * Demonstrates "Read Committed" isolation + */ + private static void readCommitted(final TransactionDB txnDb, + final WriteOptions writeOptions, final ReadOptions readOptions) + throws RocksDBException { + final byte key1[] = "abc".getBytes(UTF_8); + final byte value1[] = "def".getBytes(UTF_8); + + final byte key2[] = "xyz".getBytes(UTF_8); + final byte value2[] = "zzz".getBytes(UTF_8); + + // Start a transaction + try(final Transaction txn = txnDb.beginTransaction(writeOptions)) { + // Read a key in this transaction + byte[] value = txn.get(readOptions, key1); + assert(value == null); + + // Write a key in this transaction + txn.put(key1, value1); + + // Read a key OUTSIDE this transaction. Does not affect txn. + value = txnDb.get(readOptions, key1); + assert(value == null); + + // Write a key OUTSIDE of this transaction. + // Does not affect txn since this is an unrelated key. + // If we wrote key 'abc' here, the transaction would fail to commit. + txnDb.put(writeOptions, key2, value2); + + // Commit transaction + txn.commit(); + } + } + + /** + * Demonstrates "Repeatable Read" (Snapshot Isolation) isolation + */ + private static void repeatableRead(final TransactionDB txnDb, + final WriteOptions writeOptions, final ReadOptions readOptions) + throws RocksDBException { + + final byte key1[] = "ghi".getBytes(UTF_8); + final byte value1[] = "jkl".getBytes(UTF_8); + + // Set a snapshot at start of transaction by setting setSnapshot(true) + try(final TransactionOptions txnOptions = new TransactionOptions() + .setSetSnapshot(true); + final Transaction txn = + txnDb.beginTransaction(writeOptions, txnOptions)) { + + final Snapshot snapshot = txn.getSnapshot(); + + // Write a key OUTSIDE of transaction + txnDb.put(writeOptions, key1, value1); + + // Attempt to read a key using the snapshot. This will fail since + // the previous write outside this txn conflicts with this read. + readOptions.setSnapshot(snapshot); + + try { + final byte[] value = txn.getForUpdate(readOptions, key1, true); + throw new IllegalStateException(); + } catch(final RocksDBException e) { + assert(e.getStatus().getCode() == Status.Code.Busy); + } + + txn.rollback(); + } finally { + // Clear snapshot from read options since it is no longer valid + readOptions.setSnapshot(null); + } + } + + /** + * Demonstrates "Read Committed" (Monotonic Atomic Views) isolation + * + * In this example, we set the snapshot multiple times. This is probably + * only necessary if you have very strict isolation requirements to + * implement. + */ + private static void readCommitted_monotonicAtomicViews( + final TransactionDB txnDb, final WriteOptions writeOptions, + final ReadOptions readOptions) throws RocksDBException { + + final byte keyX[] = "x".getBytes(UTF_8); + final byte valueX[] = "x".getBytes(UTF_8); + + final byte keyY[] = "y".getBytes(UTF_8); + final byte valueY[] = "y".getBytes(UTF_8); + + try (final TransactionOptions txnOptions = new TransactionOptions() + .setSetSnapshot(true); + final Transaction txn = + txnDb.beginTransaction(writeOptions, txnOptions)) { + + // Do some reads and writes to key "x" + Snapshot snapshot = txnDb.getSnapshot(); + readOptions.setSnapshot(snapshot); + byte[] value = txn.get(readOptions, keyX); + txn.put(valueX, valueX); + + // Do a write outside of the transaction to key "y" + txnDb.put(writeOptions, keyY, valueY); + + // Set a new snapshot in the transaction + txn.setSnapshot(); + txn.setSavePoint(); + snapshot = txnDb.getSnapshot(); + readOptions.setSnapshot(snapshot); + + // Do some reads and writes to key "y" + // Since the snapshot was advanced, the write done outside of the + // transaction does not conflict. + value = txn.getForUpdate(readOptions, keyY, true); + txn.put(keyY, valueY); + + // Decide we want to revert the last write from this transaction. + txn.rollbackToSavePoint(); + + // Commit. + txn.commit(); + } finally { + // Clear snapshot from read options since it is no longer valid + readOptions.setSnapshot(null); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractCompactionFilter.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractCompactionFilter.java new file mode 100644 index 0000000..fd7eef4 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractCompactionFilter.java @@ -0,0 +1,59 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +/** + * A CompactionFilter allows an application to modify/delete a key-value at + * the time of compaction. + *

    + * At present, we just permit an overriding Java class to wrap a C++ + * implementation + */ +public abstract class AbstractCompactionFilter> + extends RocksObject { + + public static class Context { + private final boolean fullCompaction; + private final boolean manualCompaction; + + public Context(final boolean fullCompaction, final boolean manualCompaction) { + this.fullCompaction = fullCompaction; + this.manualCompaction = manualCompaction; + } + + /** + * Does this compaction run include all data files + * + * @return true if this is a full compaction run + */ + public boolean isFullCompaction() { + return fullCompaction; + } + + /** + * Is this compaction requested by the client, + * or is it occurring as an automatic compaction process + * + * @return true if the compaction was initiated by the client + */ + public boolean isManualCompaction() { + return manualCompaction; + } + } + + protected AbstractCompactionFilter(final long nativeHandle) { + super(nativeHandle); + } + + /** + * Deletes underlying C++ compaction pointer. + *

    + * Note that this function should be called only after all + * RocksDB instances referencing the compaction filter are closed. + * Otherwise, an undefined behavior will occur. + */ + @Override + protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractCompactionFilterFactory.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractCompactionFilterFactory.java new file mode 100644 index 0000000..4bb985a --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractCompactionFilterFactory.java @@ -0,0 +1,77 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Each compaction will create a new {@link AbstractCompactionFilter} + * allowing the application to know about different compactions + * + * @param The concrete type of the compaction filter + */ +public abstract class AbstractCompactionFilterFactory> + extends RocksCallbackObject { + + public AbstractCompactionFilterFactory() { + super(0L); + } + + @Override + protected long initializeNative(final long... nativeParameterHandles) { + return createNewCompactionFilterFactory0(); + } + + /** + * Called from JNI, see compaction_filter_factory_jnicallback.cc + * + * @param fullCompaction {@link AbstractCompactionFilter.Context#fullCompaction} + * @param manualCompaction {@link AbstractCompactionFilter.Context#manualCompaction} + * + * @return native handle of the CompactionFilter + */ + private long createCompactionFilter(final boolean fullCompaction, + final boolean manualCompaction) { + final T filter = createCompactionFilter( + new AbstractCompactionFilter.Context(fullCompaction, manualCompaction)); + + // CompactionFilterFactory::CreateCompactionFilter returns a std::unique_ptr + // which therefore has ownership of the underlying native object + filter.disOwnNativeHandle(); + + return filter.nativeHandle_; + } + + /** + * Create a new compaction filter + * + * @param context The context describing the need for a new compaction filter + * + * @return A new instance of {@link AbstractCompactionFilter} + */ + public abstract T createCompactionFilter( + final AbstractCompactionFilter.Context context); + + /** + * A name which identifies this compaction filter + *

    + * The name will be printed to the LOG file on start up for diagnosis + * + * @return name which identifies this compaction filter. + */ + public abstract String name(); + + /** + * We override {@link RocksCallbackObject#disposeInternal()} + * as disposing of a rocksdb::AbstractCompactionFilterFactory requires + * a slightly different approach as it is a std::shared_ptr + */ + @Override + protected void disposeInternal() { + disposeInternal(nativeHandle_); + } + + private native long createNewCompactionFilterFactory0(); + private native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractComparator.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractComparator.java new file mode 100644 index 0000000..83e0f06 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractComparator.java @@ -0,0 +1,124 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.nio.ByteBuffer; + +/** + * Comparators are used by RocksDB to determine + * the ordering of keys. + * + * Implementations of Comparators in Java should extend this class. + */ +public abstract class AbstractComparator + extends RocksCallbackObject { + + AbstractComparator() { + super(); + } + + protected AbstractComparator(final ComparatorOptions comparatorOptions) { + super(comparatorOptions.nativeHandle_); + } + + @Override + protected long initializeNative(final long... nativeParameterHandles) { + return createNewComparator(nativeParameterHandles[0]); + } + + /** + * Get the type of this comparator. + *

    + * Used for determining the correct C++ cast in native code. + * + * @return The type of the comparator. + */ + ComparatorType getComparatorType() { + return ComparatorType.JAVA_COMPARATOR; + } + + /** + * The name of the comparator. Used to check for comparator + * mismatches (i.e., a DB created with one comparator is + * accessed using a different comparator). + *

    + * A new name should be used whenever + * the comparator implementation changes in a way that will cause + * the relative ordering of any two keys to change. + *

    + * Names starting with "rocksdb." are reserved and should not be used. + * + * @return The name of this comparator implementation + */ + public abstract String name(); + + /** + * Three-way key comparison. Implementations should provide a + * total order + * on keys that might be passed to it. + * + * The implementation may modify the {@code ByteBuffer}s passed in, though + * it would be unconventional to modify the "limit" or any of the + * underlying bytes. As a callback, RocksJava will ensure that {@code a} + * is a different instance from {@code b}. + * + * @param a buffer containing the first key in its "remaining" elements + * @param b buffer containing the second key in its "remaining" elements + * + * @return Should return either: + * 1) < 0 if "a" < "b" + * 2) == 0 if "a" == "b" + * 3) > 0 if "a" > "b" + */ + public abstract int compare(final ByteBuffer a, final ByteBuffer b); + + /** + *

    Used to reduce the space requirements + * for internal data structures like index blocks.

    + * + *

    If start < limit, you may modify start which is a + * shorter string in [start, limit).

    + * + * If you modify start, it is expected that you set the byte buffer so that + * a subsequent read of start.remaining() bytes from start.position() + * to start.limit() will obtain the new start value. + * + *

    Simple comparator implementations may return with start unchanged. + * i.e., an implementation of this method that does nothing is correct.

    + * + * @param start the start + * @param limit the limit + */ + public void findShortestSeparator(final ByteBuffer start, + final ByteBuffer limit) { + // no-op + } + + /** + *

    Used to reduce the space requirements + * for internal data structures like index blocks.

    + * + *

    You may change key to a shorter key (key1) where + * key1 ≥ key.

    + * + *

    Simple comparator implementations may return the key unchanged. + * i.e., an implementation of + * this method that does nothing is correct.

    + * + * @param key the key + */ + public void findShortSuccessor(final ByteBuffer key) { + // no-op + } + + public final boolean usingDirectBuffers() { + return usingDirectBuffers(nativeHandle_); + } + + private native boolean usingDirectBuffers(final long nativeHandle); + + private native long createNewComparator(final long comparatorOptionsHandle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractComparatorJniBridge.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractComparatorJniBridge.java new file mode 100644 index 0000000..2d1bf70 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractComparatorJniBridge.java @@ -0,0 +1,119 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.nio.ByteBuffer; + +/** + * This class is intentionally private, + * it holds methods which are called + * from C++ to interact with a Comparator + * written in Java. + * + * Placing these bridge methods in this + * class keeps the API of the + * {@link org.rocksdb.AbstractComparator} clean. + */ +class AbstractComparatorJniBridge { + /** + * Only called from JNI. + *

    + * Simply a bridge to calling + * {@link AbstractComparator#compare(ByteBuffer, ByteBuffer)}, + * which ensures that the byte buffer lengths are correct + * before and after the call. + * + * @param comparator the comparator object on which to + * call {@link AbstractComparator#compare(ByteBuffer, ByteBuffer)} + * @param a buffer access to first key + * @param aLen the length of the a key, + * may be smaller than the buffer {@code a} + * @param b buffer access to second key + * @param bLen the length of the b key, + * may be smaller than the buffer {@code b} + * + * @return the result of the comparison + */ + private static int compareInternal(final AbstractComparator comparator, final ByteBuffer a, + final int aLen, final ByteBuffer b, final int bLen) { + if (aLen != -1) { + a.mark(); + a.limit(aLen); + } + if (bLen != -1) { + b.mark(); + b.limit(bLen); + } + + final int c = comparator.compare(a, b); + + if (aLen != -1) { + a.reset(); + } + if (bLen != -1) { + b.reset(); + } + + return c; + } + + /** + * Only called from JNI. + *

    + * Simply a bridge to calling + * {@link AbstractComparator#findShortestSeparator(ByteBuffer, ByteBuffer)}, + * which ensures that the byte buffer lengths are correct + * before the call. + * + * @param comparator the comparator object on which to + * call {@link AbstractComparator#findShortestSeparator(ByteBuffer, ByteBuffer)} + * @param start buffer access to the start key + * @param startLen the length of the start key, + * may be smaller than the buffer {@code start} + * @param limit buffer access to the limit key + * @param limitLen the length of the limit key, + * may be smaller than the buffer {@code limit} + * + * @return either {@code startLen} if the start key is unchanged, otherwise + * the new length of the start key + */ + private static int findShortestSeparatorInternal(final AbstractComparator comparator, + final ByteBuffer start, final int startLen, final ByteBuffer limit, final int limitLen) { + if (startLen != -1) { + start.limit(startLen); + } + if (limitLen != -1) { + limit.limit(limitLen); + } + comparator.findShortestSeparator(start, limit); + return start.remaining(); + } + + /** + * Only called from JNI. + *

    + * Simply a bridge to calling + * {@link AbstractComparator#findShortestSeparator(ByteBuffer, ByteBuffer)}, + * which ensures that the byte buffer length is correct + * before the call. + * + * @param comparator the comparator object on which to + * call {@link AbstractComparator#findShortSuccessor(ByteBuffer)} + * @param key buffer access to the key + * @param keyLen the length of the key, + * may be smaller than the buffer {@code key} + * + * @return either keyLen if the key is unchanged, otherwise the new length of the key + */ + private static int findShortSuccessorInternal( + final AbstractComparator comparator, final ByteBuffer key, final int keyLen) { + if (keyLen != -1) { + key.limit(keyLen); + } + comparator.findShortSuccessor(key); + return key.remaining(); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractEventListener.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractEventListener.java new file mode 100644 index 0000000..d640d34 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractEventListener.java @@ -0,0 +1,334 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static org.rocksdb.AbstractEventListener.EnabledEventCallback.*; + +/** + * Base class for Event Listeners. + */ +public abstract class AbstractEventListener extends RocksCallbackObject implements EventListener { + public enum EnabledEventCallback { + ON_FLUSH_COMPLETED((byte) 0x0), + ON_FLUSH_BEGIN((byte) 0x1), + ON_TABLE_FILE_DELETED((byte) 0x2), + ON_COMPACTION_BEGIN((byte) 0x3), + ON_COMPACTION_COMPLETED((byte) 0x4), + ON_TABLE_FILE_CREATED((byte) 0x5), + ON_TABLE_FILE_CREATION_STARTED((byte) 0x6), + ON_MEMTABLE_SEALED((byte) 0x7), + ON_COLUMN_FAMILY_HANDLE_DELETION_STARTED((byte) 0x8), + ON_EXTERNAL_FILE_INGESTED((byte) 0x9), + ON_BACKGROUND_ERROR((byte) 0xA), + ON_STALL_CONDITIONS_CHANGED((byte) 0xB), + ON_FILE_READ_FINISH((byte) 0xC), + ON_FILE_WRITE_FINISH((byte) 0xD), + ON_FILE_FLUSH_FINISH((byte) 0xE), + ON_FILE_SYNC_FINISH((byte) 0xF), + ON_FILE_RANGE_SYNC_FINISH((byte) 0x10), + ON_FILE_TRUNCATE_FINISH((byte) 0x11), + ON_FILE_CLOSE_FINISH((byte) 0x12), + SHOULD_BE_NOTIFIED_ON_FILE_IO((byte) 0x13), + ON_ERROR_RECOVERY_BEGIN((byte) 0x14), + ON_ERROR_RECOVERY_COMPLETED((byte) 0x15); + + private final byte value; + + EnabledEventCallback(final byte value) { + this.value = value; + } + + /** + * Get the internal representation value. + * + * @return the internal representation value + */ + byte getValue() { + return value; + } + + /** + * Get the EnabledEventCallbacks from the internal representation value. + * + * @return the enabled event callback. + * + * @throws IllegalArgumentException if the value is unknown. + */ + static EnabledEventCallback fromValue(final byte value) { + for (final EnabledEventCallback enabledEventCallback : EnabledEventCallback.values()) { + if (enabledEventCallback.value == value) { + return enabledEventCallback; + } + } + + throw new IllegalArgumentException( + "Illegal value provided for EnabledEventCallback: " + value); + } + } + + /** + * Creates an Event Listener that will + * receive all callbacks from C++. + *

    + * If you don't need all callbacks, it is much more efficient to + * just register for the ones you need by calling + * {@link #AbstractEventListener(EnabledEventCallback...)} instead. + */ + protected AbstractEventListener() { + this(ON_FLUSH_COMPLETED, ON_FLUSH_BEGIN, ON_TABLE_FILE_DELETED, ON_COMPACTION_BEGIN, + ON_COMPACTION_COMPLETED, ON_TABLE_FILE_CREATED, ON_TABLE_FILE_CREATION_STARTED, + ON_MEMTABLE_SEALED, ON_COLUMN_FAMILY_HANDLE_DELETION_STARTED, ON_EXTERNAL_FILE_INGESTED, + ON_BACKGROUND_ERROR, ON_STALL_CONDITIONS_CHANGED, ON_FILE_READ_FINISH, ON_FILE_WRITE_FINISH, + ON_FILE_FLUSH_FINISH, ON_FILE_SYNC_FINISH, ON_FILE_RANGE_SYNC_FINISH, + ON_FILE_TRUNCATE_FINISH, ON_FILE_CLOSE_FINISH, SHOULD_BE_NOTIFIED_ON_FILE_IO, + ON_ERROR_RECOVERY_BEGIN, ON_ERROR_RECOVERY_COMPLETED); + } + + /** + * Creates an Event Listener that will + * receive only certain callbacks from C++. + * + * @param enabledEventCallbacks callbacks to enable in Java. + */ + protected AbstractEventListener(final EnabledEventCallback... enabledEventCallbacks) { + super(packToLong(enabledEventCallbacks)); + } + + /** + * Pack EnabledEventCallbacks to a long. + * + * @param enabledEventCallbacks the flags + * + * @return a long + */ + private static long packToLong(final EnabledEventCallback... enabledEventCallbacks) { + long l = 0; + for (final EnabledEventCallback enabledEventCallback : enabledEventCallbacks) { + l |= 1L << enabledEventCallback.getValue(); + } + return l; + } + + @Override + public void onFlushCompleted(final RocksDB db, final FlushJobInfo flushJobInfo) { + // no-op + } + + /** + * Called from JNI, proxy for + * {@link #onFlushCompleted(RocksDB, FlushJobInfo)}. + * + * @param dbHandle native handle of the database + * @param flushJobInfo the flush job info + */ + private void onFlushCompletedProxy(final long dbHandle, final FlushJobInfo flushJobInfo) { + final RocksDB db = new RocksDB(dbHandle); + db.disOwnNativeHandle(); // we don't own this! + onFlushCompleted(db, flushJobInfo); + } + + @Override + public void onFlushBegin(final RocksDB db, final FlushJobInfo flushJobInfo) { + // no-op + } + + /** + * Called from JNI, proxy for + * {@link #onFlushBegin(RocksDB, FlushJobInfo)}. + * + * @param dbHandle native handle of the database + * @param flushJobInfo the flush job info + */ + private void onFlushBeginProxy(final long dbHandle, final FlushJobInfo flushJobInfo) { + final RocksDB db = new RocksDB(dbHandle); + db.disOwnNativeHandle(); // we don't own this! + onFlushBegin(db, flushJobInfo); + } + + @Override + public void onTableFileDeleted(final TableFileDeletionInfo tableFileDeletionInfo) { + // no-op + } + + @Override + public void onCompactionBegin(final RocksDB db, final CompactionJobInfo compactionJobInfo) { + // no-op + } + + /** + * Called from JNI, proxy for + * {@link #onCompactionBegin(RocksDB, CompactionJobInfo)}. + * + * @param dbHandle native handle of the database + * @param compactionJobInfo the flush job info + */ + private void onCompactionBeginProxy( + final long dbHandle, final CompactionJobInfo compactionJobInfo) { + final RocksDB db = new RocksDB(dbHandle); + db.disOwnNativeHandle(); // we don't own this! + onCompactionBegin(db, compactionJobInfo); + } + + @Override + public void onCompactionCompleted(final RocksDB db, final CompactionJobInfo compactionJobInfo) { + // no-op + } + + /** + * Called from JNI, proxy for + * {@link #onCompactionCompleted(RocksDB, CompactionJobInfo)}. + * + * @param dbHandle native handle of the database + * @param compactionJobInfo the flush job info + */ + private void onCompactionCompletedProxy( + final long dbHandle, final CompactionJobInfo compactionJobInfo) { + final RocksDB db = new RocksDB(dbHandle); + db.disOwnNativeHandle(); // we don't own this! + onCompactionCompleted(db, compactionJobInfo); + } + + @Override + public void onTableFileCreated(final TableFileCreationInfo tableFileCreationInfo) { + // no-op + } + + @Override + public void onTableFileCreationStarted( + final TableFileCreationBriefInfo tableFileCreationBriefInfo) { + // no-op + } + + @Override + public void onMemTableSealed(final MemTableInfo memTableInfo) { + // no-op + } + + @Override + public void onColumnFamilyHandleDeletionStarted(final ColumnFamilyHandle columnFamilyHandle) { + // no-op + } + + @Override + public void onExternalFileIngested( + final RocksDB db, final ExternalFileIngestionInfo externalFileIngestionInfo) { + // no-op + } + + /** + * Called from JNI, proxy for + * {@link #onExternalFileIngested(RocksDB, ExternalFileIngestionInfo)}. + * + * @param dbHandle native handle of the database + * @param externalFileIngestionInfo the flush job info + */ + private void onExternalFileIngestedProxy( + final long dbHandle, final ExternalFileIngestionInfo externalFileIngestionInfo) { + final RocksDB db = new RocksDB(dbHandle); + db.disOwnNativeHandle(); // we don't own this! + onExternalFileIngested(db, externalFileIngestionInfo); + } + + @Override + public void onBackgroundError( + final BackgroundErrorReason backgroundErrorReason, final Status backgroundError) { + // no-op + } + + /** + * Called from JNI, proxy for + * {@link #onBackgroundError(BackgroundErrorReason, Status)}. + * + * @param reasonByte byte value representing error reason + * @param backgroundError status with error code + */ + private void onBackgroundErrorProxy(final byte reasonByte, final Status backgroundError) { + onBackgroundError(BackgroundErrorReason.fromValue(reasonByte), backgroundError); + } + + @Override + public void onStallConditionsChanged(final WriteStallInfo writeStallInfo) { + // no-op + } + + @Override + public void onFileReadFinish(final FileOperationInfo fileOperationInfo) { + // no-op + } + + @Override + public void onFileWriteFinish(final FileOperationInfo fileOperationInfo) { + // no-op + } + + @Override + public void onFileFlushFinish(final FileOperationInfo fileOperationInfo) { + // no-op + } + + @Override + public void onFileSyncFinish(final FileOperationInfo fileOperationInfo) { + // no-op + } + + @Override + public void onFileRangeSyncFinish(final FileOperationInfo fileOperationInfo) { + // no-op + } + + @Override + public void onFileTruncateFinish(final FileOperationInfo fileOperationInfo) { + // no-op + } + + @Override + public void onFileCloseFinish(final FileOperationInfo fileOperationInfo) { + // no-op + } + + @Override + public boolean shouldBeNotifiedOnFileIO() { + return false; + } + + @Override + public boolean onErrorRecoveryBegin( + final BackgroundErrorReason backgroundErrorReason, final Status backgroundError) { + return true; + } + + /** + * Called from JNI, proxy for + * {@link #onErrorRecoveryBegin(BackgroundErrorReason, Status)}. + * + * @param reasonByte byte value representing error reason + * @param backgroundError status with error code + */ + private boolean onErrorRecoveryBeginProxy(final byte reasonByte, final Status backgroundError) { + return onErrorRecoveryBegin(BackgroundErrorReason.fromValue(reasonByte), backgroundError); + } + + @Override + public void onErrorRecoveryCompleted(final Status oldBackgroundError) { + // no-op + } + + @Override + protected long initializeNative(final long... nativeParameterHandles) { + return createNewEventListener(nativeParameterHandles[0]); + } + + /** + * Deletes underlying C++ native callback object pointer + */ + @Override + protected void disposeInternal() { + disposeInternal(nativeHandle_); + } + + private native long createNewEventListener(final long enabledEventCallbackValues); + private native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractImmutableNativeReference.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractImmutableNativeReference.java new file mode 100644 index 0000000..173d63e --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractImmutableNativeReference.java @@ -0,0 +1,65 @@ +// Copyright (c) 2016, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Offers functionality for implementations of + * {@link AbstractNativeReference} which have an immutable reference to the + * underlying native C++ object + */ +//@ThreadSafe +public abstract class AbstractImmutableNativeReference + extends AbstractNativeReference { + + /** + * A flag indicating whether the current {@code AbstractNativeReference} is + * responsible to free the underlying C++ object + */ + protected final AtomicBoolean owningHandle_; + + protected AbstractImmutableNativeReference(final boolean owningHandle) { + this.owningHandle_ = new AtomicBoolean(owningHandle); + } + + @Override + public boolean isOwningHandle() { + return owningHandle_.get(); + } + + /** + * Releases this {@code AbstractNativeReference} from the responsibility of + * freeing the underlying native C++ object + *

    + * This will prevent the object from attempting to delete the underlying + * native object in {@code close()}. This must be used when another object + * takes over ownership of the native object or both will attempt to delete + * the underlying object when closed. + *

    + * When {@code disOwnNativeHandle()} is called, {@code close()} will + * subsequently take no action. As a result, incorrect use of this function + * may cause a memory leak. + *

    + */ + protected final void disOwnNativeHandle() { + owningHandle_.set(false); + } + + @Override + public void close() { + if (owningHandle_.compareAndSet(true, false)) { + disposeInternal(); + } + } + + /** + * The helper function of {@link AbstractImmutableNativeReference#close()} + * which all subclasses of {@code AbstractImmutableNativeReference} must + * implement to release their underlying native C++ objects. + */ + protected abstract void disposeInternal(); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractMutableOptions.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractMutableOptions.java new file mode 100644 index 0000000..1a6251b --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractMutableOptions.java @@ -0,0 +1,368 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb; + +import java.util.*; + +public abstract class AbstractMutableOptions { + + protected static final String KEY_VALUE_PAIR_SEPARATOR = ";"; + protected static final char KEY_VALUE_SEPARATOR = '='; + static final String INT_ARRAY_INT_SEPARATOR = ":"; + + protected final String[] keys; + private final String[] values; + + /** + * User must use builder pattern, or parser. + * + * @param keys the keys + * @param values the values + */ + protected AbstractMutableOptions(final String[] keys, final String[] values) { + this.keys = keys; + this.values = values; + } + + String[] getKeys() { + return keys; + } + + String[] getValues() { + return values; + } + + /** + * Returns a string representation of MutableOptions which + * is suitable for consumption by {@code #parse(String)}. + * + * @return String representation of MutableOptions + */ + @Override + public String toString() { + final StringBuilder buffer = new StringBuilder(); + for(int i = 0; i < keys.length; i++) { + buffer + .append(keys[i]) + .append(KEY_VALUE_SEPARATOR) + .append(values[i]); + + if(i + 1 < keys.length) { + buffer.append(KEY_VALUE_PAIR_SEPARATOR); + } + } + return buffer.toString(); + } + + public abstract static class AbstractMutableOptionsBuilder< + T extends AbstractMutableOptions, U extends AbstractMutableOptionsBuilder, K + extends MutableOptionKey> { + private final Map> options = new LinkedHashMap<>(); + private final List unknown = new ArrayList<>(); + + protected abstract U self(); + + /** + * Get all the possible keys + * + * @return A map of all keys, indexed by name. + */ + protected abstract Map allKeys(); + + /** + * Construct a subclass instance of {@link AbstractMutableOptions}. + * + * @param keys the keys + * @param values the values + * + * @return an instance of the options. + */ + protected abstract T build(final String[] keys, final String[] values); + + public T build() { + final String[] keys = new String[options.size()]; + final String[] values = new String[options.size()]; + + int i = 0; + for (final Map.Entry> option : options.entrySet()) { + keys[i] = option.getKey().name(); + values[i] = option.getValue().asString(); + i++; + } + + return build(keys, values); + } + + protected U setDouble( + final K key, final double value) { + if (key.getValueType() != MutableOptionKey.ValueType.DOUBLE) { + throw new IllegalArgumentException( + key + " does not accept a double value"); + } + options.put(key, MutableOptionValue.fromDouble(value)); + return self(); + } + + protected double getDouble(final K key) + throws NoSuchElementException, NumberFormatException { + final MutableOptionValue value = options.get(key); + if(value == null) { + throw new NoSuchElementException(key.name() + " has not been set"); + } + return value.asDouble(); + } + + protected U setLong( + final K key, final long value) { + if(key.getValueType() != MutableOptionKey.ValueType.LONG) { + throw new IllegalArgumentException( + key + " does not accept a long value"); + } + options.put(key, MutableOptionValue.fromLong(value)); + return self(); + } + + protected long getLong(final K key) + throws NoSuchElementException, NumberFormatException { + final MutableOptionValue value = options.get(key); + if(value == null) { + throw new NoSuchElementException(key.name() + " has not been set"); + } + return value.asLong(); + } + + protected U setInt( + final K key, final int value) { + if(key.getValueType() != MutableOptionKey.ValueType.INT) { + throw new IllegalArgumentException( + key + " does not accept an integer value"); + } + options.put(key, MutableOptionValue.fromInt(value)); + return self(); + } + + protected int getInt(final K key) + throws NoSuchElementException, NumberFormatException { + final MutableOptionValue value = options.get(key); + if(value == null) { + throw new NoSuchElementException(key.name() + " has not been set"); + } + return value.asInt(); + } + + protected U setBoolean( + final K key, final boolean value) { + if(key.getValueType() != MutableOptionKey.ValueType.BOOLEAN) { + throw new IllegalArgumentException( + key + " does not accept a boolean value"); + } + options.put(key, MutableOptionValue.fromBoolean(value)); + return self(); + } + + protected boolean getBoolean(final K key) + throws NoSuchElementException, NumberFormatException { + final MutableOptionValue value = options.get(key); + if(value == null) { + throw new NoSuchElementException(key.name() + " has not been set"); + } + return value.asBoolean(); + } + + protected U setIntArray( + final K key, final int[] value) { + if(key.getValueType() != MutableOptionKey.ValueType.INT_ARRAY) { + throw new IllegalArgumentException( + key + " does not accept an int array value"); + } + options.put(key, MutableOptionValue.fromIntArray(value)); + return self(); + } + + protected int[] getIntArray(final K key) + throws NoSuchElementException, NumberFormatException { + final MutableOptionValue value = options.get(key); + if(value == null) { + throw new NoSuchElementException(key.name() + " has not been set"); + } + return value.asIntArray(); + } + + protected > U setEnum( + final K key, final N value) { + if(key.getValueType() != MutableOptionKey.ValueType.ENUM) { + throw new IllegalArgumentException( + key + " does not accept a Enum value"); + } + options.put(key, MutableOptionValue.fromEnum(value)); + return self(); + } + + @SuppressWarnings("unchecked") + protected > N getEnum(final K key) + throws NoSuchElementException, NumberFormatException { + final MutableOptionValue value = options.get(key); + if (value == null) { + throw new NoSuchElementException(key.name() + " has not been set"); + } + + if (!(value instanceof MutableOptionValue.MutableOptionEnumValue)) { + throw new NoSuchElementException(key.name() + " is not of Enum type"); + } + + return ((MutableOptionValue.MutableOptionEnumValue) value).asObject(); + } + + /** + * Parse a string into a long value, accepting values expressed as a double (such as 9.00) which + * are meant to be a long, not a double + * + * @param value the string containing a value which represents a long + * @return the long value of the parsed string + */ + private long parseAsLong(final String value) { + try { + return Long.parseLong(value); + } catch (final NumberFormatException nfe) { + final double doubleValue = Double.parseDouble(value); + if (doubleValue != Math.round(doubleValue)) + throw new IllegalArgumentException("Unable to parse or round " + value + " to long"); + return Math.round(doubleValue); + } + } + + /** + * Parse a string into an int value, accepting values expressed as a double (such as 9.00) which + * are meant to be an int, not a double + * + * @param value the string containing a value which represents an int + * @return the int value of the parsed string + */ + private int parseAsInt(final String value) { + try { + return Integer.parseInt(value); + } catch (final NumberFormatException nfe) { + final double doubleValue = Double.parseDouble(value); + if (doubleValue != Math.round(doubleValue)) + throw new IllegalArgumentException("Unable to parse or round " + value + " to int"); + return (int) Math.round(doubleValue); + } + } + + /** + * Constructs a builder for mutable column family options from a hierarchical parsed options + * string representation. The {@link OptionString.Parser} class output has been used to create a + * (name,value)-list; each value may be either a simple string or a (name, value)-list in turn. + * + * @param options a list of parsed option string objects + * @param ignoreUnknown what to do if the key is not one of the keys we expect + * + * @return a builder with the values from the parsed input set + * + * @throws IllegalArgumentException if an option value is of the wrong type, or a key is empty + */ + protected U fromParsed(final List options, final boolean ignoreUnknown) { + Objects.requireNonNull(options); + + for (final OptionString.Entry option : options) { + try { + if (option.key.isEmpty()) { + throw new IllegalArgumentException("options string is invalid: " + option); + } + fromOptionString(option, ignoreUnknown); + } catch (final NumberFormatException nfe) { + throw new IllegalArgumentException( + "" + option.key + "=" + option.value + " - not a valid value for its type", nfe); + } + } + + return self(); + } + + /** + * Set a value in the builder from the supplied option string + * + * @param option the option key/value to add to this builder + * @param ignoreUnknown if this is not set, throw an exception when a key is not in the known + * set + * @return the same object, after adding options + * @throws IllegalArgumentException if the key is unknown, or a value has the wrong type/form + */ + private U fromOptionString(final OptionString.Entry option, final boolean ignoreUnknown) + throws IllegalArgumentException { + Objects.requireNonNull(option.key); + Objects.requireNonNull(option.value); + + final K key = allKeys().get(option.key); + if (key == null && ignoreUnknown) { + unknown.add(option); + return self(); + } else if (key == null) { + throw new IllegalArgumentException("Key: " + null + " is not a known option key"); + } + + if (!option.value.isList()) { + throw new IllegalArgumentException( + "Option: " + key + " is not a simple value or list, don't know how to parse it"); + } + + // Check that simple values are the single item in the array + if (key.getValueType() != MutableOptionKey.ValueType.INT_ARRAY) { + { + if (option.value.list.size() != 1) { + throw new IllegalArgumentException( + "Simple value does not have exactly 1 item: " + option.value.list); + } + } + } + + final List valueStrs = option.value.list; + final String valueStr = valueStrs.get(0); + + switch (key.getValueType()) { + case DOUBLE: + return setDouble(key, Double.parseDouble(valueStr)); + + case LONG: + return setLong(key, parseAsLong(valueStr)); + + case INT: + return setInt(key, parseAsInt(valueStr)); + + case BOOLEAN: + return setBoolean(key, Boolean.parseBoolean(valueStr)); + + case INT_ARRAY: + final int[] value = new int[valueStrs.size()]; + for (int i = 0; i < valueStrs.size(); i++) { + value[i] = Integer.parseInt(valueStrs.get(i)); + } + return setIntArray(key, value); + + case ENUM: + final String optionName = key.name(); + if (optionName.equals("prepopulate_blob_cache")) { + final PrepopulateBlobCache prepopulateBlobCache = + PrepopulateBlobCache.getFromInternal(valueStr); + return setEnum(key, prepopulateBlobCache); + } else if (optionName.equals("compression") + || optionName.equals("blob_compression_type")) { + final CompressionType compressionType = CompressionType.getFromInternal(valueStr); + return setEnum(key, compressionType); + } else { + throw new IllegalArgumentException("Unknown enum type: " + key.name()); + } + + default: + throw new IllegalStateException(key + " has unknown value type: " + key.getValueType()); + } + } + + /** + * + * @return the list of keys encountered which were not known to the type being generated + */ + public List getUnknown() { + return new ArrayList<>(unknown); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractNativeReference.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractNativeReference.java new file mode 100644 index 0000000..1ce54fc --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractNativeReference.java @@ -0,0 +1,49 @@ +// Copyright (c) 2016, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * AbstractNativeReference is the base-class of all RocksDB classes that have + * a pointer to a native C++ {@code rocksdb} object. + *

    + * AbstractNativeReference has the {@link AbstractNativeReference#close()} + * method, which frees its associated C++ object.

    + *

    + * This function should be called manually, or even better, called implicitly using a + * try-with-resources + * statement, when you are finished with the object. It is no longer + * called automatically during the regular Java GC process via finalization + * {@link AbstractNativeReference#finalize()}.

    + * which is deprecated from Java 9. + *

    + * Explanatory note - When or if the Garbage Collector calls {@link Object#finalize()} + * depends on the JVM implementation and system conditions, which the programmer + * cannot control. In addition, the GC cannot see through the native reference + * long member variable (which is the C++ pointer value to the native object), + * and cannot know what other resources depend on it. + *

    + */ +public abstract class AbstractNativeReference implements AutoCloseable { + /** + * Returns true if we are responsible for freeing the underlying C++ object + * + * @return true if we are responsible to free the C++ object + */ + protected abstract boolean isOwningHandle(); + + /** + * Frees the underlying C++ object + *

    + * It is strong recommended that the developer calls this after they + * have finished using the object.

    + *

    + * Note, that once an instance of {@link AbstractNativeReference} has been + * closed, calling any of its functions will lead to undefined + * behavior.

    + */ + @Override public abstract void close(); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractRocksIterator.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractRocksIterator.java new file mode 100644 index 0000000..1aade1b --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractRocksIterator.java @@ -0,0 +1,146 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.nio.ByteBuffer; + +/** + * Base class implementation for Rocks Iterators + * in the Java API + * + *

    Multiple threads can invoke const methods on an RocksIterator without + * external synchronization, but if any of the threads may call a + * non-const method, all threads accessing the same RocksIterator must use + * external synchronization.

    + * + * @param

    The type of the Parent Object from which the Rocks Iterator was + * created. This is used by disposeInternal to avoid double-free + * issues with the underlying C++ object. + * @see org.rocksdb.RocksObject + */ +public abstract class AbstractRocksIterator

    + extends RocksObject implements RocksIteratorInterface { + final P parent_; + + protected AbstractRocksIterator(final P parent, + final long nativeHandle) { + super(nativeHandle); + // parent must point to a valid RocksDB instance. + assert (parent != null); + // RocksIterator must hold a reference to the related parent instance + // to guarantee that while a GC cycle starts RocksIterator instances + // are freed prior to parent instances. + parent_ = parent; + } + + @Override + public boolean isValid() { + assert (isOwningHandle()); + return isValid0(nativeHandle_); + } + + @Override + public void seekToFirst() { + assert (isOwningHandle()); + seekToFirst0(nativeHandle_); + } + + @Override + public void seekToLast() { + assert (isOwningHandle()); + seekToLast0(nativeHandle_); + } + + @Override + public void seek(final byte[] target) { + assert (isOwningHandle()); + seek0(nativeHandle_, target, target.length); + } + + @Override + public void seekForPrev(final byte[] target) { + assert (isOwningHandle()); + seekForPrev0(nativeHandle_, target, target.length); + } + + @Override + public void seek(final ByteBuffer target) { + assert (isOwningHandle()); + if (target.isDirect()) { + seekDirect0(nativeHandle_, target, target.position(), target.remaining()); + } else { + seekByteArray0(nativeHandle_, target.array(), target.arrayOffset() + target.position(), + target.remaining()); + } + target.position(target.limit()); + } + + @Override + public void seekForPrev(final ByteBuffer target) { + assert (isOwningHandle()); + if (target.isDirect()) { + seekForPrevDirect0(nativeHandle_, target, target.position(), target.remaining()); + } else { + seekForPrevByteArray0(nativeHandle_, target.array(), target.arrayOffset() + target.position(), + target.remaining()); + } + target.position(target.limit()); + } + + @Override + public void next() { + assert (isOwningHandle()); + next0(nativeHandle_); + } + + @Override + public void prev() { + assert (isOwningHandle()); + prev0(nativeHandle_); + } + + @Override + public void refresh() throws RocksDBException { + assert (isOwningHandle()); + refresh0(nativeHandle_); + } + + @Override + public void status() throws RocksDBException { + assert (isOwningHandle()); + status0(nativeHandle_); + } + + /** + *

    Deletes underlying C++ iterator pointer.

    + * + *

    Note: the underlying handle can only be safely deleted if the parent + * instance related to a certain RocksIterator is still valid and initialized. + * Therefore {@code disposeInternal()} checks if the parent is initialized + * before freeing the native handle.

    + */ + @Override + protected void disposeInternal() { + if (parent_.isOwningHandle()) { + disposeInternal(nativeHandle_); + } + } + + abstract boolean isValid0(long handle); + abstract void seekToFirst0(long handle); + abstract void seekToLast0(long handle); + abstract void next0(long handle); + abstract void prev0(long handle); + abstract void refresh0(long handle) throws RocksDBException; + abstract void seek0(long handle, byte[] target, int targetLen); + abstract void seekForPrev0(long handle, byte[] target, int targetLen); + abstract void seekDirect0(long handle, ByteBuffer target, int targetOffset, int targetLen); + abstract void seekForPrevDirect0(long handle, ByteBuffer target, int targetOffset, int targetLen); + abstract void seekByteArray0(long handle, byte[] target, int targetOffset, int targetLen); + abstract void seekForPrevByteArray0(long handle, byte[] target, int targetOffset, int targetLen); + + abstract void status0(long handle) throws RocksDBException; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractSlice.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractSlice.java new file mode 100644 index 0000000..0681b67 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractSlice.java @@ -0,0 +1,191 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Slices are used by RocksDB to provide + * efficient access to keys and values. + *

    + * This class is package private, implementers + * should extend either of the public abstract classes: + * @see org.rocksdb.Slice + * @see org.rocksdb.DirectSlice + * + * Regards the lifecycle of Java Slices in RocksDB: + * At present when you configure a Comparator from Java, it creates an + * instance of a C++ BaseComparatorJniCallback subclass and + * passes that to RocksDB as the comparator. That subclass of + * BaseComparatorJniCallback creates the Java + * @see org.rocksdb.AbstractSlice subclass Objects. When you dispose + * the Java @see org.rocksdb.AbstractComparator subclass, it disposes the + * C++ BaseComparatorJniCallback subclass, which in turn destroys the + * Java @see org.rocksdb.AbstractSlice subclass Objects. + */ +public abstract class AbstractSlice extends RocksMutableObject { + + protected AbstractSlice() { + super(); + } + + protected AbstractSlice(final long nativeHandle) { + super(nativeHandle); + } + + /** + * Returns the data of the slice. + * + * @return The slice data. Note, the type of access is + * determined by the subclass + * @see org.rocksdb.AbstractSlice#data0(long) + */ + public T data() { + return data0(getNativeHandle()); + } + + /** + * Access to the data is provided by the + * subtype as it needs to handle the + * generic typing. + * + * @param handle The address of the underlying + * native object. + * + * @return Java typed access to the data. + */ + protected abstract T data0(long handle); + + /** + * Drops the specified {@code n} + * number of bytes from the start + * of the backing slice + * + * @param n The number of bytes to drop + */ + public abstract void removePrefix(final int n); + + /** + * Clears the backing slice + */ + public abstract void clear(); + + /** + * Return the length (in bytes) of the data. + * + * @return The length in bytes. + */ + public int size() { + return size0(getNativeHandle()); + } + + /** + * Return true if the length of the + * data is zero. + * + * @return true if there is no data, false otherwise. + */ + public boolean empty() { + return empty0(getNativeHandle()); + } + + /** + * Creates a string representation of the data + * + * @param hex When true, the representation + * will be encoded in hexadecimal. + * + * @return The string representation of the data. + */ + public String toString(final boolean hex) { + return toString0(getNativeHandle(), hex); + } + + @Override + public String toString() { + return toString(false); + } + + /** + * Three-way key comparison + * + * @param other A slice to compare against + * + * @return Should return either: + * 1) < 0 if this < other + * 2) == 0 if this == other + * 3) > 0 if this > other + */ + public int compare(final AbstractSlice other) { + assert (other != null); + if(!isOwningHandle()) { + return other.isOwningHandle() ? -1 : 0; + } else { + if(!other.isOwningHandle()) { + return 1; + } else { + return compare0(getNativeHandle(), other.getNativeHandle()); + } + } + } + + @Override + public int hashCode() { + return toString().hashCode(); + } + + /** + * If other is a slice object, then + * we defer to {@link #compare(AbstractSlice) compare} + * to check equality, otherwise we return false. + * + * @param other Object to test for equality + * + * @return true when {@code this.compare(other) == 0}, + * false otherwise. + */ + @Override + public boolean equals(final Object other) { + if (other instanceof AbstractSlice) { + return compare((AbstractSlice)other) == 0; + } else { + return false; + } + } + + /** + * Determines whether this slice starts with + * another slice + * + * @param prefix Another slice which may of may not + * be a prefix of this slice. + * + * @return true when this slice starts with the + * {@code prefix} slice + */ + public boolean startsWith(final AbstractSlice prefix) { + if (prefix != null) { + return startsWith0(getNativeHandle(), prefix.getNativeHandle()); + } else { + return false; + } + } + + protected static native long createNewSliceFromString(final String str); + private native int size0(long handle); + private native boolean empty0(long handle); + private native String toString0(long handle, boolean hex); + private native int compare0(long handle, long otherHandle); + private native boolean startsWith0(long handle, long otherHandle); + + /** + * Deletes underlying C++ slice pointer. + * Note that this function should be called only after all + * RocksDB instances referencing the slice are closed. + * Otherwise, an undefined behavior will occur. + */ + @Override + protected final native void disposeInternal(final long handle); + +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractTableFilter.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractTableFilter.java new file mode 100644 index 0000000..c696c3e --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractTableFilter.java @@ -0,0 +1,20 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb; + +/** + * Base class for Table Filters. + */ +public abstract class AbstractTableFilter + extends RocksCallbackObject implements TableFilter { + + protected AbstractTableFilter() { + super(); + } + + @Override + protected long initializeNative(final long... nativeParameterHandles) { + return createNewTableFilter(); + } + + private native long createNewTableFilter(); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractTraceWriter.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractTraceWriter.java new file mode 100644 index 0000000..13edfbd --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractTraceWriter.java @@ -0,0 +1,70 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Base class for TraceWriters. + */ +public abstract class AbstractTraceWriter + extends RocksCallbackObject implements TraceWriter { + + @Override + protected long initializeNative(final long... nativeParameterHandles) { + return createNewTraceWriter(); + } + + /** + * Called from JNI, proxy for {@link TraceWriter#write(Slice)}. + * + * @param sliceHandle the native handle of the slice (which we do not own) + * + * @return short (2 bytes) where the first byte is the + * {@link Status.Code#getValue()} and the second byte is the + * {@link Status.SubCode#getValue()}. + */ + private short writeProxy(final long sliceHandle) { + try { + write(new Slice(sliceHandle)); + return statusToShort(Status.Code.Ok, Status.SubCode.None); + } catch (final RocksDBException e) { + return statusToShort(e.getStatus()); + } + } + + /** + * Called from JNI, proxy for {@link TraceWriter#closeWriter()}. + * + * @return short (2 bytes) where the first byte is the + * {@link Status.Code#getValue()} and the second byte is the + * {@link Status.SubCode#getValue()}. + */ + private short closeWriterProxy() { + try { + closeWriter(); + return statusToShort(Status.Code.Ok, Status.SubCode.None); + } catch (final RocksDBException e) { + return statusToShort(e.getStatus()); + } + } + + private static short statusToShort(/*@Nullable*/ final Status status) { + final Status.Code code = status != null && status.getCode() != null + ? status.getCode() + : Status.Code.IOError; + final Status.SubCode subCode = status != null && status.getSubCode() != null + ? status.getSubCode() + : Status.SubCode.None; + return statusToShort(code, subCode); + } + + private static short statusToShort(final Status.Code code, + final Status.SubCode subCode) { + final short result = (short) (code.getValue() << 8); + return (short)(result | subCode.getValue()); + } + + private native long createNewTraceWriter(); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractTransactionNotifier.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractTransactionNotifier.java new file mode 100644 index 0000000..b117e5c --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractTransactionNotifier.java @@ -0,0 +1,54 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Provides notification to the caller of SetSnapshotOnNextOperation when + * the actual snapshot gets created + */ +public abstract class AbstractTransactionNotifier + extends RocksCallbackObject { + + protected AbstractTransactionNotifier() { + super(); + } + + /** + * Implement this method to receive notification when a snapshot is + * requested via {@link Transaction#setSnapshotOnNextOperation()}. + * + * @param newSnapshot the snapshot that has been created. + */ + public abstract void snapshotCreated(final Snapshot newSnapshot); + + /** + * This is intentionally private as it is the callback hook + * from JNI + */ + private void snapshotCreated(final long snapshotHandle) { + snapshotCreated(new Snapshot(snapshotHandle)); + } + + @Override + protected long initializeNative(final long... nativeParameterHandles) { + return createNewTransactionNotifier(); + } + + private native long createNewTransactionNotifier(); + + /** + * Deletes underlying C++ TransactionNotifier pointer. + *

    + * Note that this function should be called only after all + * Transactions referencing the comparator are closed. + * Otherwise, an undefined behavior will occur. + */ + @Override + protected void disposeInternal() { + disposeInternal(nativeHandle_); + } + protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractWalFilter.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractWalFilter.java new file mode 100644 index 0000000..fc77eab --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractWalFilter.java @@ -0,0 +1,49 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Base class for WAL Filters. + */ +public abstract class AbstractWalFilter + extends RocksCallbackObject implements WalFilter { + + @Override + protected long initializeNative(final long... nativeParameterHandles) { + return createNewWalFilter(); + } + + /** + * Called from JNI, proxy for + * {@link WalFilter#logRecordFound(long, String, WriteBatch, WriteBatch)}. + * + * @param logNumber the log handle. + * @param logFileName the log file name + * @param batchHandle the native handle of a WriteBatch (which we do not own) + * @param newBatchHandle the native handle of a + * new WriteBatch (which we do not own) + * + * @return short (2 bytes) where the first byte is the + * {@link WalFilter.LogRecordFoundResult#walProcessingOption} + * {@link WalFilter.LogRecordFoundResult#batchChanged}. + */ + private short logRecordFoundProxy(final long logNumber, + final String logFileName, final long batchHandle, + final long newBatchHandle) { + final LogRecordFoundResult logRecordFoundResult = logRecordFound( + logNumber, logFileName, new WriteBatch(batchHandle), + new WriteBatch(newBatchHandle)); + return logRecordFoundResultToShort(logRecordFoundResult); + } + + private static short logRecordFoundResultToShort( + final LogRecordFoundResult logRecordFoundResult) { + final short result = (short) (logRecordFoundResult.walProcessingOption.getValue() << 8); + return (short)(result | (logRecordFoundResult.batchChanged ? 1 : 0)); + } + + private native long createNewWalFilter(); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractWriteBatch.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractWriteBatch.java new file mode 100644 index 0000000..41d967f --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AbstractWriteBatch.java @@ -0,0 +1,203 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.nio.ByteBuffer; + +public abstract class AbstractWriteBatch extends RocksObject + implements WriteBatchInterface { + + protected AbstractWriteBatch(final long nativeHandle) { + super(nativeHandle); + } + + @Override + public int count() { + return count0(nativeHandle_); + } + + @Override + public void put(final byte[] key, final byte[] value) throws RocksDBException { + put(nativeHandle_, key, key.length, value, value.length); + } + + @Override + public void put(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, final byte[] value) + throws RocksDBException { + put(nativeHandle_, key, key.length, value, value.length, + columnFamilyHandle.nativeHandle_); + } + + @Override + public void merge(final byte[] key, final byte[] value) throws RocksDBException { + merge(nativeHandle_, key, key.length, value, value.length); + } + + @Override + public void merge(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, + final byte[] value) throws RocksDBException { + merge(nativeHandle_, key, key.length, value, value.length, + columnFamilyHandle.nativeHandle_); + } + + @Override + public void put(final ByteBuffer key, final ByteBuffer value) throws RocksDBException { + assert key.isDirect() && value.isDirect(); + putDirect(nativeHandle_, key, key.position(), key.remaining(), value, value.position(), + value.remaining(), 0); + key.position(key.limit()); + value.position(value.limit()); + } + + @Override + public void put(final ColumnFamilyHandle columnFamilyHandle, final ByteBuffer key, + final ByteBuffer value) throws RocksDBException { + assert key.isDirect() && value.isDirect(); + putDirect(nativeHandle_, key, key.position(), key.remaining(), value, value.position(), + value.remaining(), columnFamilyHandle.nativeHandle_); + key.position(key.limit()); + value.position(value.limit()); + } + + @Override + public void delete(final byte[] key) throws RocksDBException { + delete(nativeHandle_, key, key.length); + } + + @Override + public void delete(final ColumnFamilyHandle columnFamilyHandle, final byte[] key) + throws RocksDBException { + delete(nativeHandle_, key, key.length, columnFamilyHandle.nativeHandle_); + } + + @Override + public void delete(final ByteBuffer key) throws RocksDBException { + deleteDirect(nativeHandle_, key, key.position(), key.remaining(), 0); + key.position(key.limit()); + } + + @Override + public void delete(final ColumnFamilyHandle columnFamilyHandle, final ByteBuffer key) + throws RocksDBException { + deleteDirect( + nativeHandle_, key, key.position(), key.remaining(), columnFamilyHandle.nativeHandle_); + key.position(key.limit()); + } + + @Override + public void singleDelete(final byte[] key) throws RocksDBException { + singleDelete(nativeHandle_, key, key.length); + } + + @Override + public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, final byte[] key) + throws RocksDBException { + singleDelete(nativeHandle_, key, key.length, columnFamilyHandle.nativeHandle_); + } + + @Override + public void deleteRange(final byte[] beginKey, final byte[] endKey) throws RocksDBException { + deleteRange(nativeHandle_, beginKey, beginKey.length, endKey, endKey.length); + } + + @Override + public void deleteRange(final ColumnFamilyHandle columnFamilyHandle, final byte[] beginKey, + final byte[] endKey) throws RocksDBException { + deleteRange(nativeHandle_, beginKey, beginKey.length, endKey, endKey.length, + columnFamilyHandle.nativeHandle_); + } + + @Override + public void putLogData(final byte[] blob) throws RocksDBException { + putLogData(nativeHandle_, blob, blob.length); + } + + @Override + public void clear() { + clear0(nativeHandle_); + } + + @Override + public void setSavePoint() { + setSavePoint0(nativeHandle_); + } + + @Override + public void rollbackToSavePoint() throws RocksDBException { + rollbackToSavePoint0(nativeHandle_); + } + + @Override + public void popSavePoint() throws RocksDBException { + popSavePoint(nativeHandle_); + } + + @Override + public void setMaxBytes(final long maxBytes) { + setMaxBytes(nativeHandle_, maxBytes); + } + + @Override + public WriteBatch getWriteBatch() { + return getWriteBatch(nativeHandle_); + } + + abstract int count0(final long handle); + + abstract void put(final long handle, final byte[] key, final int keyLen, + final byte[] value, final int valueLen) throws RocksDBException; + + abstract void put(final long handle, final byte[] key, final int keyLen, + final byte[] value, final int valueLen, final long cfHandle) + throws RocksDBException; + + abstract void putDirect(final long handle, final ByteBuffer key, final int keyOffset, + final int keyLength, final ByteBuffer value, final int valueOffset, final int valueLength, + final long cfHandle) throws RocksDBException; + + abstract void merge(final long handle, final byte[] key, final int keyLen, + final byte[] value, final int valueLen) throws RocksDBException; + + abstract void merge(final long handle, final byte[] key, final int keyLen, + final byte[] value, final int valueLen, final long cfHandle) + throws RocksDBException; + + abstract void delete(final long handle, final byte[] key, + final int keyLen) throws RocksDBException; + + abstract void delete(final long handle, final byte[] key, + final int keyLen, final long cfHandle) throws RocksDBException; + + abstract void singleDelete(final long handle, final byte[] key, final int keyLen) + throws RocksDBException; + + abstract void singleDelete(final long handle, final byte[] key, final int keyLen, + final long cfHandle) throws RocksDBException; + + abstract void deleteDirect(final long handle, final ByteBuffer key, final int keyOffset, + final int keyLength, final long cfHandle) throws RocksDBException; + + abstract void deleteRange(final long handle, final byte[] beginKey, final int beginKeyLen, + final byte[] endKey, final int endKeyLen) throws RocksDBException; + + abstract void deleteRange(final long handle, final byte[] beginKey, final int beginKeyLen, + final byte[] endKey, final int endKeyLen, final long cfHandle) throws RocksDBException; + + abstract void putLogData(final long handle, final byte[] blob, + final int blobLen) throws RocksDBException; + + abstract void clear0(final long handle); + + abstract void setSavePoint0(final long handle); + + abstract void rollbackToSavePoint0(final long handle); + + abstract void popSavePoint(final long handle) throws RocksDBException; + + abstract void setMaxBytes(final long handle, long maxBytes); + + abstract WriteBatch getWriteBatch(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AccessHint.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AccessHint.java new file mode 100644 index 0000000..b7ccadd --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AccessHint.java @@ -0,0 +1,54 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * File access pattern once a compaction has started + */ +@Deprecated +public enum AccessHint { + NONE((byte)0x0), + NORMAL((byte)0x1), + SEQUENTIAL((byte)0x2), + WILLNEED((byte)0x3); + + private final byte value; + + AccessHint(final byte value) { + this.value = value; + } + + /** + *

    Returns the byte value of the enumerations value.

    + * + * @return byte representation + */ + public byte getValue() { + return value; + } + + /** + *

    Get the AccessHint enumeration value by + * passing the byte identifier to this method.

    + * + * @param byteIdentifier of AccessHint. + * + * @return AccessHint instance. + * + * @throws IllegalArgumentException if the access hint for the byteIdentifier + * cannot be found + */ + public static AccessHint getAccessHint(final byte byteIdentifier) { + for (final AccessHint accessHint : AccessHint.values()) { + if (accessHint.getValue() == byteIdentifier) { + return accessHint; + } + } + + throw new IllegalArgumentException( + "Illegal value provided for AccessHint."); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AdvancedColumnFamilyOptionsInterface.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AdvancedColumnFamilyOptionsInterface.java new file mode 100644 index 0000000..d1d1123 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AdvancedColumnFamilyOptionsInterface.java @@ -0,0 +1,463 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.List; + +/** + * Advanced Column Family Options which are not + * mutable (i.e. present in {@link AdvancedMutableColumnFamilyOptionsInterface}) + *

    + * Taken from include/rocksdb/advanced_options.h + */ +public interface AdvancedColumnFamilyOptionsInterface< + T extends AdvancedColumnFamilyOptionsInterface & ColumnFamilyOptionsInterface> { + /** + * The minimum number of write buffers that will be merged together + * before writing to storage. If set to 1, then + * all write buffers are flushed to L0 as individual files and this increases + * read amplification because a get request has to check in all of these + * files. Also, an in-memory merge may result in writing lesser + * data to storage if there are duplicate records in each of these + * individual write buffers. Default: 1 + * + * @param minWriteBufferNumberToMerge the minimum number of write buffers + * that will be merged together. + * @return the reference to the current options. + */ + T setMinWriteBufferNumberToMerge( + int minWriteBufferNumberToMerge); + + /** + * The minimum number of write buffers that will be merged together + * before writing to storage. If set to 1, then + * all write buffers are flushed to L0 as individual files and this increases + * read amplification because a get request has to check in all of these + * files. Also, an in-memory merge may result in writing lesser + * data to storage if there are duplicate records in each of these + * individual write buffers. Default: 1 + * + * @return the minimum number of write buffers that will be merged together. + */ + int minWriteBufferNumberToMerge(); + + /** + * The total maximum number of write buffers to maintain in memory including + * copies of buffers that have already been flushed. Unlike + * {@link AdvancedMutableColumnFamilyOptionsInterface#maxWriteBufferNumber()}, + * this parameter does not affect flushing. + * This controls the minimum amount of write history that will be available + * in memory for conflict checking when Transactions are used. + *

    + * When using an OptimisticTransactionDB: + * If this value is too low, some transactions may fail at commit time due + * to not being able to determine whether there were any write conflicts. + *

    + * When using a TransactionDB: + * If Transaction::SetSnapshot is used, TransactionDB will read either + * in-memory write buffers or SST files to do write-conflict checking. + * Increasing this value can reduce the number of reads to SST files + * done for conflict detection. + *

    + * Setting this value to 0 will cause write buffers to be freed immediately + * after they are flushed. + * If this value is set to -1, + * {@link AdvancedMutableColumnFamilyOptionsInterface#maxWriteBufferNumber()} + * will be used. + *

    + * Default: + * If using a TransactionDB/OptimisticTransactionDB, the default value will + * be set to the value of + * {@link AdvancedMutableColumnFamilyOptionsInterface#maxWriteBufferNumber()} + * if it is not explicitly set by the user. Otherwise, the default is 0. + * + * @param maxWriteBufferNumberToMaintain The maximum number of write + * buffers to maintain + * + * @return the reference to the current options. + */ + T setMaxWriteBufferNumberToMaintain( + int maxWriteBufferNumberToMaintain); + + /** + * The total maximum number of write buffers to maintain in memory including + * copies of buffers that have already been flushed. + * + * @return maxWriteBufferNumberToMaintain The maximum number of write buffers + * to maintain + */ + int maxWriteBufferNumberToMaintain(); + + /** + * Allows thread-safe inplace updates. + * If inplace_callback function is not set, + * Put(key, new_value) will update inplace the existing_value iff + * * key exists in current memtable + * * new sizeof(new_value) ≤ sizeof(existing_value) + * * existing_value for that key is a put i.e. kTypeValue + * If inplace_callback function is set, check doc for inplace_callback. + * Default: false. + * + * @param inplaceUpdateSupport true if thread-safe inplace updates + * are allowed. + * @return the reference to the current options. + */ + T setInplaceUpdateSupport( + boolean inplaceUpdateSupport); + + /** + * Allows thread-safe inplace updates. + * If inplace_callback function is not set, + * Put(key, new_value) will update inplace the existing_value iff + * * key exists in current memtable + * * new sizeof(new_value) ≤ sizeof(existing_value) + * * existing_value for that key is a put i.e. kTypeValue + * If inplace_callback function is set, check doc for inplace_callback. + * Default: false. + * + * @return true if thread-safe inplace updates are allowed. + */ + boolean inplaceUpdateSupport(); + + /** + * Control locality of bloom filter probes to improve cache miss rate. + * This option only applies to memtable prefix bloom and plaintable + * prefix bloom. It essentially limits the max number of cache lines each + * bloom filter check can touch. + * This optimization is turned off when set to 0. The number should never + * be greater than number of probes. This option can boost performance + * for in-memory workload but should use with care since it can cause + * higher false positive rate. + * Default: 0 + * + * @param bloomLocality the level of locality of bloom-filter probes. + * @return the reference to the current options. + */ + T setBloomLocality(int bloomLocality); + + /** + * Control locality of bloom filter probes to improve cache miss rate. + * This option only applies to memtable prefix bloom and plaintable + * prefix bloom. It essentially limits the max number of cache lines each + * bloom filter check can touch. + * This optimization is turned off when set to 0. The number should never + * be greater than number of probes. This option can boost performance + * for in-memory workload but should use with care since it can cause + * higher false positive rate. + * Default: 0 + * + * @return the level of locality of bloom-filter probes. + * @see #setBloomLocality(int) + */ + int bloomLocality(); + + /** + *

    Different levels can have different compression + * policies. There are cases where most lower levels + * would like to use quick compression algorithms while + * the higher levels (which have more data) use + * compression algorithms that have better compression + * but could be slower. This array, if non-empty, should + * have an entry for each level of the database; + * these override the value specified in the previous + * field 'compression'.

    + * + * NOTICE + *

    If {@code level_compaction_dynamic_level_bytes=true}, + * {@code compression_per_level[0]} still determines {@code L0}, + * but other elements of the array are based on base level + * (the level {@code L0} files are merged to), and may not + * match the level users see from info log for metadata. + *

    + *

    If {@code L0} files are merged to {@code level - n}, + * then, for {@code i>0}, {@code compression_per_level[i]} + * determines compaction type for level {@code n+i-1}.

    + * + * Example + *

    For example, if we have 5 levels, and we determine to + * merge {@code L0} data to {@code L4} (which means {@code L1..L3} + * will be empty), then the new files go to {@code L4} uses + * compression type {@code compression_per_level[1]}.

    + * + *

    If now {@code L0} is merged to {@code L2}. Data goes to + * {@code L2} will be compressed according to + * {@code compression_per_level[1]}, {@code L3} using + * {@code compression_per_level[2]}and {@code L4} using + * {@code compression_per_level[3]}. Compaction for each + * level can change when data grows.

    + * + *

    Default: empty

    + * + * @param compressionLevels list of + * {@link org.rocksdb.CompressionType} instances. + * + * @return the reference to the current options. + */ + T setCompressionPerLevel( + List compressionLevels); + + /** + *

    Return the currently set {@link org.rocksdb.CompressionType} + * per instances.

    + * + *

    See: {@link #setCompressionPerLevel(java.util.List)}

    + * + * @return list of {@link org.rocksdb.CompressionType} + * instances. + */ + List compressionPerLevel(); + + /** + * Set the number of levels for this database + * If level-styled compaction is used, then this number determines + * the total number of levels. + * + * @param numLevels the number of levels. + * @return the reference to the current options. + */ + T setNumLevels(int numLevels); + + /** + * If level-styled compaction is used, then this number determines + * the total number of levels. + * + * @return the number of levels. + */ + int numLevels(); + + /** + *

    If {@code true}, RocksDB will pick target size of each level + * dynamically. We will pick a base level b >= 1. L0 will be + * directly merged into level b, instead of always into level 1. + * Level 1 to b-1 need to be empty. We try to pick b and its target + * size so that

    + * + *
      + *
    1. target size is in the range of + * (max_bytes_for_level_base / max_bytes_for_level_multiplier, + * max_bytes_for_level_base]
      2. + *
    2. target size of the last level (level num_levels-1) equals to extra size + * of the level.
      3. + *
    + * + *

    At the same time max_bytes_for_level_multiplier and + * max_bytes_for_level_multiplier_additional are still satisfied.

    + * + *

    With this option on, from an empty DB, we make last level the base + * level, which means merging L0 data into the last level, until it exceeds + * max_bytes_for_level_base. And then we make the second last level to be + * base level, to start to merge L0 data to second last level, with its + * target size to be {@code 1/max_bytes_for_level_multiplier} of the last + * levels extra size. After the data accumulates more so that we need to + * move the base level to the third last one, and so on.

    + * + *

    Example

    + * + *

    For example, assume {@code max_bytes_for_level_multiplier=10}, + * {@code num_levels=6}, and {@code max_bytes_for_level_base=10MB}.

    + * + *

    Target sizes of level 1 to 5 starts with:

    + * {@code [- - - - 10MB]} + *

    with base level is level. Target sizes of level 1 to 4 are not applicable + * because they will not be used. + * Until the size of Level 5 grows to more than 10MB, say 11MB, we make + * base target to level 4 and now the targets looks like:

    + * {@code [- - - 1.1MB 11MB]} + *

    While data are accumulated, size targets are tuned based on actual data + * of level 5. When level 5 has 50MB of data, the target is like:

    + * {@code [- - - 5MB 50MB]} + *

    Until level 5's actual size is more than 100MB, say 101MB. Now if we + * keep level 4 to be the base level, its target size needs to be 10.1MB, + * which doesn't satisfy the target size range. So now we make level 3 + * the target size and the target sizes of the levels look like:

    + * {@code [- - 1.01MB 10.1MB 101MB]} + *

    In the same way, while level 5 further grows, all levels' targets grow, + * like

    + * {@code [- - 5MB 50MB 500MB]} + *

    Until level 5 exceeds 1000MB and becomes 1001MB, we make level 2 the + * base level and make levels' target sizes like this:

    + * {@code [- 1.001MB 10.01MB 100.1MB 1001MB]} + *

    and go on...

    + * + *

    By doing it, we give {@code max_bytes_for_level_multiplier} a priority + * against {@code max_bytes_for_level_base}, for a more predictable LSM tree + * shape. It is useful to limit worse case space amplification.

    + * + *

    {@code max_bytes_for_level_multiplier_additional} is ignored with + * this flag on.

    + * + *

    Turning this feature on or off for an existing DB can cause unexpected + * LSM tree structure so it's not recommended.

    + * + *

    Caution: this option is experimental

    + * + *

    Default: false

    + * + * @param enableLevelCompactionDynamicLevelBytes boolean value indicating + * if {@code LevelCompactionDynamicLevelBytes} shall be enabled. + * @return the reference to the current options. + */ + @Experimental("Turning this feature on or off for an existing DB can cause" + + " unexpected LSM tree structure so it's not recommended") + T setLevelCompactionDynamicLevelBytes( + boolean enableLevelCompactionDynamicLevelBytes); + + /** + *

    Return if {@code LevelCompactionDynamicLevelBytes} is enabled. + *

    + * + *

    For further information see + * {@link #setLevelCompactionDynamicLevelBytes(boolean)}

    + * + * @return boolean value indicating if + * {@code levelCompactionDynamicLevelBytes} is enabled. + */ + @Experimental("Caution: this option is experimental") + boolean levelCompactionDynamicLevelBytes(); + + /** + * Maximum size of each compaction (not guarantee) + * + * @param maxCompactionBytes the compaction size limit + * @return the reference to the current options. + */ + T setMaxCompactionBytes( + long maxCompactionBytes); + + /** + * Control maximum size of each compaction (not guaranteed) + * + * @return compaction size threshold + */ + long maxCompactionBytes(); + + /** + * Set compaction style for DB. + *

    + * Default: LEVEL. + * + * @param compactionStyle Compaction style. + * @return the reference to the current options. + */ + ColumnFamilyOptionsInterface setCompactionStyle(CompactionStyle compactionStyle); + + /** + * Compaction style for DB. + * + * @return Compaction style. + */ + CompactionStyle compactionStyle(); + + /** + * If level {@link #compactionStyle()} == {@link CompactionStyle#LEVEL}, + * for each level, which files are prioritized to be picked to compact. + *

    + * Default: {@link CompactionPriority#ByCompensatedSize} + * + * @param compactionPriority The compaction priority + * + * @return the reference to the current options. + */ + T setCompactionPriority( + CompactionPriority compactionPriority); + + /** + * Get the Compaction priority if level compaction + * is used for all levels + * + * @return The compaction priority + */ + CompactionPriority compactionPriority(); + + /** + * Set the options needed to support Universal Style compactions + * + * @param compactionOptionsUniversal The Universal Style compaction options + * + * @return the reference to the current options. + */ + T setCompactionOptionsUniversal( + CompactionOptionsUniversal compactionOptionsUniversal); + + /** + * The options needed to support Universal Style compactions + * + * @return The Universal Style compaction options + */ + CompactionOptionsUniversal compactionOptionsUniversal(); + + /** + * The options for FIFO compaction style + * + * @param compactionOptionsFIFO The FIFO compaction options + * + * @return the reference to the current options. + */ + T setCompactionOptionsFIFO( + CompactionOptionsFIFO compactionOptionsFIFO); + + /** + * The options for FIFO compaction style + * + * @return The FIFO compaction options + */ + CompactionOptionsFIFO compactionOptionsFIFO(); + + /** + *

    This flag specifies that the implementation should optimize the filters + * mainly for cases where keys are found rather than also optimize for keys + * missed. This would be used in cases where the application knows that + * there are very few misses or the performance in the case of misses is not + * important.

    + * + *

    For now, this flag allows us to not store filters for the last level i.e + * the largest level which contains data of the LSM store. For keys which + * are hits, the filters in this level are not useful because we will search + * for the data anyway.

    + * + *

    NOTE: the filters in other levels are still useful + * even for key hit because they tell us whether to look in that level or go + * to the higher level.

    + * + *

    Default: false

    + * + * @param optimizeFiltersForHits boolean value indicating if this flag is set. + * @return the reference to the current options. + */ + T setOptimizeFiltersForHits( + boolean optimizeFiltersForHits); + + /** + *

    Returns the current state of the {@code optimize_filters_for_hits} + * setting.

    + * + * @return boolean value indicating if the flag + * {@code optimize_filters_for_hits} was set. + */ + boolean optimizeFiltersForHits(); + + /** + * By default, RocksDB runs consistency checks on the LSM every time the LSM + * changes (Flush, Compaction, AddFile). Use this option if you need to + * disable them. + *

    + * Default: true + * + * @param forceConsistencyChecks false to disable consistency checks + * + * @return the reference to the current options. + */ + T setForceConsistencyChecks( + boolean forceConsistencyChecks); + + /** + * By default, RocksDB runs consistency checks on the LSM every time the LSM + * changes (Flush, Compaction, AddFile). + * + * @return true if consistency checks are enforced + */ + boolean forceConsistencyChecks(); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java new file mode 100644 index 0000000..c8fc841 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java @@ -0,0 +1,830 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Advanced Column Family Options which are mutable + *

    + * Taken from include/rocksdb/advanced_options.h + * and MutableCFOptions in util/cf_options.h + */ +public interface AdvancedMutableColumnFamilyOptionsInterface< + T extends AdvancedMutableColumnFamilyOptionsInterface> { + /** + * The maximum number of write buffers that are built up in memory. + * The default is 2, so that when 1 write buffer is being flushed to + * storage, new writes can continue to the other write buffer. + * Default: 2 + * + * @param maxWriteBufferNumber maximum number of write buffers. + * @return the instance of the current options. + */ + T setMaxWriteBufferNumber( + int maxWriteBufferNumber); + + /** + * Returns maximum number of write buffers. + * + * @return maximum number of write buffers. + * @see #setMaxWriteBufferNumber(int) + */ + int maxWriteBufferNumber(); + + /** + * Number of locks used for inplace update + * Default: 10000, if inplace_update_support = true, else 0. + * + * @param inplaceUpdateNumLocks the number of locks used for + * inplace updates. + * @return the reference to the current options. + * @throws java.lang.IllegalArgumentException thrown on 32-Bit platforms + * while overflowing the underlying platform specific value. + */ + T setInplaceUpdateNumLocks( + long inplaceUpdateNumLocks); + + /** + * Number of locks used for inplace update + * Default: 10000, if inplace_update_support = true, else 0. + * + * @return the number of locks used for inplace update. + */ + long inplaceUpdateNumLocks(); + + /** + * if prefix_extractor is set and memtable_prefix_bloom_size_ratio is not 0, + * create prefix bloom for memtable with the size of + * write_buffer_size * memtable_prefix_bloom_size_ratio. + * If it is larger than 0.25, it is sanitized to 0.25. + *

    + * Default: 0 (disabled) + * + * @param memtablePrefixBloomSizeRatio the ratio of memtable used by the + * bloom filter, 0 means no bloom filter + * @return the reference to the current options. + */ + T setMemtablePrefixBloomSizeRatio( + double memtablePrefixBloomSizeRatio); + + /** + * if prefix_extractor is set and memtable_prefix_bloom_size_ratio is not 0, + * create prefix bloom for memtable with the size of + * write_buffer_size * memtable_prefix_bloom_size_ratio. + * If it is larger than 0.25, it is sanitized to 0.25. + *

    + * Default: 0 (disabled) + * + * @return the ratio of memtable used by the bloom filter + */ + double memtablePrefixBloomSizeRatio(); + + /** + * Threshold used in the MemPurge (memtable garbage collection) + * feature. A value of 0.0 corresponds to no MemPurge, + * a value of 1.0 will trigger a MemPurge as often as possible. + *

    + * Default: 0.0 (disabled) + * + * @param experimentalMempurgeThreshold the threshold used by + * the MemPurge decider. + * @return the reference to the current options. + */ + T setExperimentalMempurgeThreshold(double experimentalMempurgeThreshold); + + /** + * Threshold used in the MemPurge (memtable garbage collection) + * feature. A value of 0.0 corresponds to no MemPurge, + * a value of 1.0 will trigger a MemPurge as often as possible. + *

    + * Default: 0 (disabled) + * + * @return the threshold used by the MemPurge decider + */ + double experimentalMempurgeThreshold(); + + /** + * Enable whole key bloom filter in memtable. Note this will only take effect + * if memtable_prefix_bloom_size_ratio is not 0. Enabling whole key filtering + * can potentially reduce CPU usage for point-look-ups. + *

    + * Default: false (disabled) + * + * @param memtableWholeKeyFiltering true if whole key bloom filter is enabled + * in memtable + * @return the reference to the current options. + */ + T setMemtableWholeKeyFiltering(boolean memtableWholeKeyFiltering); + + /** + * Returns whether whole key bloom filter is enabled in memtable + * + * @return true if whole key bloom filter is enabled in memtable + */ + boolean memtableWholeKeyFiltering(); + + /** + * Page size for huge page TLB for bloom in memtable. If ≤ 0, not allocate + * from huge page TLB but from malloc. + * Need to reserve huge pages for it to be allocated. For example: + * sysctl -w vm.nr_hugepages=20 + * See linux doc Documentation/vm/hugetlbpage.txt + * + * @param memtableHugePageSize The page size of the huge + * page tlb + * @return the reference to the current options. + */ + T setMemtableHugePageSize( + long memtableHugePageSize); + + /** + * Page size for huge page TLB for bloom in memtable. If ≤ 0, not allocate + * from huge page TLB but from malloc. + * Need to reserve huge pages for it to be allocated. For example: + * sysctl -w vm.nr_hugepages=20 + * See linux doc Documentation/vm/hugetlbpage.txt + * + * @return The page size of the huge page tlb + */ + long memtableHugePageSize(); + + /** + * The size of one block in arena memory allocation. + * If ≤ 0, a proper value is automatically calculated (usually 1/10 of + * writer_buffer_size). + *

    + * There are two additional restriction of the specified size: + * (1) size should be in the range of [4096, 2 << 30] and + * (2) be the multiple of the CPU word (which helps with the memory + * alignment). + *

    + * We'll automatically check and adjust the size number to make sure it + * conforms to the restrictions. + * Default: 0 + * + * @param arenaBlockSize the size of an arena block + * @return the reference to the current options. + * @throws java.lang.IllegalArgumentException thrown on 32-Bit platforms + * while overflowing the underlying platform specific value. + */ + T setArenaBlockSize(long arenaBlockSize); + + /** + * The size of one block in arena memory allocation. + * If ≤ 0, a proper value is automatically calculated (usually 1/10 of + * writer_buffer_size). + *

    + * There are two additional restriction of the specified size: + * (1) size should be in the range of [4096, 2 << 30] and + * (2) be the multiple of the CPU word (which helps with the memory + * alignment). + *

    + * We'll automatically check and adjust the size number to make sure it + * conforms to the restrictions. + * Default: 0 + * + * @return the size of an arena block + */ + long arenaBlockSize(); + + /** + * Soft limit on number of level-0 files. We start slowing down writes at this + * point. A value < 0 means that no writing slow down will be triggered by + * number of files in level-0. + * + * @param level0SlowdownWritesTrigger The soft limit on the number of + * level-0 files + * @return the reference to the current options. + */ + T setLevel0SlowdownWritesTrigger( + int level0SlowdownWritesTrigger); + + /** + * Soft limit on number of level-0 files. We start slowing down writes at this + * point. A value < 0 means that no writing slow down will be triggered by + * number of files in level-0. + * + * @return The soft limit on the number of + * level-0 files + */ + int level0SlowdownWritesTrigger(); + + /** + * Maximum number of level-0 files. We stop writes at this point. + * + * @param level0StopWritesTrigger The maximum number of level-0 files + * @return the reference to the current options. + */ + T setLevel0StopWritesTrigger( + int level0StopWritesTrigger); + + /** + * Maximum number of level-0 files. We stop writes at this point. + * + * @return The maximum number of level-0 files + */ + int level0StopWritesTrigger(); + + /** + * The target file size for compaction. + * This targetFileSizeBase determines a level-1 file size. + * Target file size for level L can be calculated by + * targetFileSizeBase * (targetFileSizeMultiplier ^ (L-1)) + * For example, if targetFileSizeBase is 2MB and + * target_file_size_multiplier is 10, then each file on level-1 will + * be 2MB, and each file on level 2 will be 20MB, + * and each file on level-3 will be 200MB. + * by default targetFileSizeBase is 64MB. + * + * @param targetFileSizeBase the target size of a level-0 file. + * @return the reference to the current options. + * + * @see #setTargetFileSizeMultiplier(int) + */ + T setTargetFileSizeBase( + long targetFileSizeBase); + + /** + * The target file size for compaction. + * This targetFileSizeBase determines a level-1 file size. + * Target file size for level L can be calculated by + * targetFileSizeBase * (targetFileSizeMultiplier ^ (L-1)) + * For example, if targetFileSizeBase is 2MB and + * target_file_size_multiplier is 10, then each file on level-1 will + * be 2MB, and each file on level 2 will be 20MB, + * and each file on level-3 will be 200MB. + * by default targetFileSizeBase is 64MB. + * + * @return the target size of a level-0 file. + * + * @see #targetFileSizeMultiplier() + */ + long targetFileSizeBase(); + + /** + * targetFileSizeMultiplier defines the size ratio between a + * level-L file and level-(L+1) file. + * By default target_file_size_multiplier is 1, meaning + * files in different levels have the same target. + * + * @param multiplier the size ratio between a level-(L+1) file + * and level-L file. + * @return the reference to the current options. + */ + T setTargetFileSizeMultiplier( + int multiplier); + + /** + * targetFileSizeMultiplier defines the size ratio between a + * level-(L+1) file and level-L file. + * By default targetFileSizeMultiplier is 1, meaning + * files in different levels have the same target. + * + * @return the size ratio between a level-(L+1) file and level-L file. + */ + int targetFileSizeMultiplier(); + + /** + * The ratio between the total size of level-(L+1) files and the total + * size of level-L files for all L. + * DEFAULT: 10 + * + * @param multiplier the ratio between the total size of level-(L+1) + * files and the total size of level-L files for all L. + * @return the reference to the current options. + *

    + * See {@link MutableColumnFamilyOptionsInterface#setMaxBytesForLevelBase(long)} + */ + T setMaxBytesForLevelMultiplier(double multiplier); + + /** + * The ratio between the total size of level-(L+1) files and the total + * size of level-L files for all L. + * DEFAULT: 10 + * + * @return the ratio between the total size of level-(L+1) files and + * the total size of level-L files for all L. + *

    + * See {@link MutableColumnFamilyOptionsInterface#maxBytesForLevelBase()} + */ + double maxBytesForLevelMultiplier(); + + /** + * Different max-size multipliers for different levels. + * These are multiplied by max_bytes_for_level_multiplier to arrive + * at the max-size of each level. + *

    + * Default: 1 + * + * @param maxBytesForLevelMultiplierAdditional The max-size multipliers + * for each level + * @return the reference to the current options. + */ + T setMaxBytesForLevelMultiplierAdditional( + int[] maxBytesForLevelMultiplierAdditional); + + /** + * Different max-size multipliers for different levels. + * These are multiplied by max_bytes_for_level_multiplier to arrive + * at the max-size of each level. + *

    + * Default: 1 + * + * @return The max-size multipliers for each level + */ + int[] maxBytesForLevelMultiplierAdditional(); + + /** + * All writes will be slowed down to at least delayed_write_rate if estimated + * bytes needed to be compaction exceed this threshold. + *

    + * Default: 64GB + * + * @param softPendingCompactionBytesLimit The soft limit to impose on + * compaction + * @return the reference to the current options. + */ + T setSoftPendingCompactionBytesLimit( + long softPendingCompactionBytesLimit); + + /** + * All writes will be slowed down to at least delayed_write_rate if estimated + * bytes needed to be compaction exceed this threshold. + *

    + * Default: 64GB + * + * @return The soft limit to impose on compaction + */ + long softPendingCompactionBytesLimit(); + + /** + * All writes are stopped if estimated bytes needed to be compaction exceed + * this threshold. + *

    + * Default: 256GB + * + * @param hardPendingCompactionBytesLimit The hard limit to impose on + * compaction + * @return the reference to the current options. + */ + T setHardPendingCompactionBytesLimit( + long hardPendingCompactionBytesLimit); + + /** + * All writes are stopped if estimated bytes needed to be compaction exceed + * this threshold. + *

    + * Default: 256GB + * + * @return The hard limit to impose on compaction + */ + long hardPendingCompactionBytesLimit(); + + /** + * An iteration->Next() sequentially skips over keys with the same + * user-key unless this option is set. This number specifies the number + * of keys (with the same userkey) that will be sequentially + * skipped before a reseek is issued. + * Default: 8 + * + * @param maxSequentialSkipInIterations the number of keys could + * be skipped in an iteration. + * @return the reference to the current options. + */ + T setMaxSequentialSkipInIterations( + long maxSequentialSkipInIterations); + + /** + * An iteration->Next() sequentially skips over keys with the same + * user-key unless this option is set. This number specifies the number + * of keys (with the same userkey) that will be sequentially + * skipped before a reseek is issued. + * Default: 8 + * + * @return the number of keys could be skipped in an iteration. + */ + long maxSequentialSkipInIterations(); + + /** + * Maximum number of successive merge operations on a key in the memtable. + *

    + * When a merge operation is added to the memtable and the maximum number of + * successive merges is reached, the value of the key will be calculated and + * inserted into the memtable instead of the merge operation. This will + * ensure that there are never more than max_successive_merges merge + * operations in the memtable. + *

    + * Default: 0 (disabled) + * + * @param maxSuccessiveMerges the maximum number of successive merges. + * @return the reference to the current options. + * @throws java.lang.IllegalArgumentException thrown on 32-Bit platforms + * while overflowing the underlying platform specific value. + */ + T setMaxSuccessiveMerges( + long maxSuccessiveMerges); + + /** + * Maximum number of successive merge operations on a key in the memtable. + *

    + * When a merge operation is added to the memtable and the maximum number of + * successive merges is reached, the value of the key will be calculated and + * inserted into the memtable instead of the merge operation. This will + * ensure that there are never more than max_successive_merges merge + * operations in the memtable. + *

    + * Default: 0 (disabled) + * + * @return the maximum number of successive merges. + */ + long maxSuccessiveMerges(); + + /** + * After writing every SST file, reopen it and read all the keys. + *

    + * Default: false + * + * @param paranoidFileChecks true to enable paranoid file checks + * @return the reference to the current options. + */ + T setParanoidFileChecks( + boolean paranoidFileChecks); + + /** + * After writing every SST file, reopen it and read all the keys. + *

    + * Default: false + * + * @return true if paranoid file checks are enabled + */ + boolean paranoidFileChecks(); + + /** + * Measure IO stats in compactions and flushes, if true. + *

    + * Default: false + * + * @param reportBgIoStats true to enable reporting + * @return the reference to the current options. + */ + T setReportBgIoStats( + boolean reportBgIoStats); + + /** + * Determine whether IO stats in compactions and flushes are being measured + * + * @return true if reporting is enabled + */ + boolean reportBgIoStats(); + + /** + * Non-bottom-level files older than TTL will go through the compaction + * process. This needs {@link MutableDBOptionsInterface#maxOpenFiles()} to be + * set to -1. + *

    + * Enabled only for level compaction for now. + *

    + * Default: 0 (disabled) + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @param ttl the time-to-live. + * + * @return the reference to the current options. + */ + T setTtl(final long ttl); + + /** + * Get the TTL for Non-bottom-level files that will go through the compaction + * process. + *

    + * See {@link #setTtl(long)}. + * + * @return the time-to-live. + */ + long ttl(); + + /** + * Files older than this value will be picked up for compaction, and + * re-written to the same level as they were before. + * One main use of the feature is to make sure a file goes through compaction + * filters periodically. Users can also use the feature to clear up SST + * files using old format. + *

    + * A file's age is computed by looking at file_creation_time or creation_time + * table properties in order, if they have valid non-zero values; if not, the + * age is based on the file's last modified time (given by the underlying + * Env). + *

    + * Supported in Level and FIFO compaction. + * In FIFO compaction, this option has the same meaning as TTL and whichever + * stricter will be used. + * Pre-req: max_open_file == -1. + * unit: seconds. Ex: 7 days = 7 * 24 * 60 * 60 + *

    + * Values: + * 0: Turn off Periodic compactions. + * UINT64_MAX - 1 (i.e 0xfffffffffffffffe): Let RocksDB control this feature + * as needed. For now, RocksDB will change this value to 30 days + * (i.e 30 * 24 * 60 * 60) so that every file goes through the compaction + * process at least once every 30 days if not compacted sooner. + * In FIFO compaction, since the option has the same meaning as ttl, + * when this value is left default, and ttl is left to 0, 30 days will be + * used. Otherwise, min(ttl, periodic_compaction_seconds) will be used. + *

    + * Default: 0xfffffffffffffffe (allow RocksDB to auto-tune) + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @param periodicCompactionSeconds the periodic compaction in seconds. + * + * @return the reference to the current options. + */ + T setPeriodicCompactionSeconds(final long periodicCompactionSeconds); + + /** + * Get the periodicCompactionSeconds. + *

    + * See {@link #setPeriodicCompactionSeconds(long)}. + * + * @return the periodic compaction in seconds. + */ + long periodicCompactionSeconds(); + + // + // BEGIN options for blobs (integrated BlobDB) + // + + /** + * When set, large values (blobs) are written to separate blob files, and only + * pointers to them are stored in SST files. This can reduce write amplification + * for large-value use cases at the cost of introducing a level of indirection + * for reads. See also the options min_blob_size, blob_file_size, + * blob_compression_type, enable_blob_garbage_collection, and + * blob_garbage_collection_age_cutoff below. + *

    + * Default: false + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @param enableBlobFiles true iff blob files should be enabled + * + * @return the reference to the current options. + */ + T setEnableBlobFiles(final boolean enableBlobFiles); + + /** + * When set, large values (blobs) are written to separate blob files, and only + * pointers to them are stored in SST files. This can reduce write amplification + * for large-value use cases at the cost of introducing a level of indirection + * for reads. See also the options min_blob_size, blob_file_size, + * blob_compression_type, enable_blob_garbage_collection, and + * blob_garbage_collection_age_cutoff below. + *

    + * Default: false + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @return true if blob files are enabled + */ + boolean enableBlobFiles(); + + /** + * Set the size of the smallest value to be stored separately in a blob file. Values + * which have an uncompressed size smaller than this threshold are stored + * alongside the keys in SST files in the usual fashion. A value of zero for + * this option means that all values are stored in blob files. Note that + * enable_blob_files has to be set in order for this option to have any effect. + *

    + * Default: 0 + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @param minBlobSize the size of the smallest value to be stored separately in a blob file + * @return the reference to the current options. + */ + T setMinBlobSize(final long minBlobSize); + + /** + * Get the size of the smallest value to be stored separately in a blob file. Values + * which have an uncompressed size smaller than this threshold are stored + * alongside the keys in SST files in the usual fashion. A value of zero for + * this option means that all values are stored in blob files. Note that + * enable_blob_files has to be set in order for this option to have any effect. + *

    + * Default: 0 + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @return the current minimum size of value which is stored separately in a blob + */ + long minBlobSize(); + + /** + * Set the size limit for blob files. When writing blob files, a new file is opened + * once this limit is reached. Note that enable_blob_files has to be set in + * order for this option to have any effect. + *

    + * Default: 256 MB + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @param blobFileSize the size limit for blob files + * + * @return the reference to the current options. + */ + T setBlobFileSize(final long blobFileSize); + + /** + * The size limit for blob files. When writing blob files, a new file is opened + * once this limit is reached. + * + * @return the current size limit for blob files + */ + long blobFileSize(); + + /** + * Set the compression algorithm to use for large values stored in blob files. Note + * that enable_blob_files has to be set in order for this option to have any + * effect. + *

    + * Default: no compression + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @param compressionType the compression algorithm to use. + * + * @return the reference to the current options. + */ + T setBlobCompressionType(CompressionType compressionType); + + /** + * Get the compression algorithm in use for large values stored in blob files. + * Note that enable_blob_files has to be set in order for this option to have any + * effect. + * + * @return the current compression algorithm + */ + CompressionType blobCompressionType(); + + /** + * Enable/disable garbage collection of blobs. Blob GC is performed as part of + * compaction. Valid blobs residing in blob files older than a cutoff get + * relocated to new files as they are encountered during compaction, which makes + * it possible to clean up blob files once they contain nothing but + * obsolete/garbage blobs. See also blob_garbage_collection_age_cutoff below. + *

    + * Default: false + * + * @param enableBlobGarbageCollection the new enabled/disabled state of blob garbage collection + * + * @return the reference to the current options. + */ + T setEnableBlobGarbageCollection(final boolean enableBlobGarbageCollection); + + /** + * Query whether garbage collection of blobs is enabled.Blob GC is performed as part of + * compaction. Valid blobs residing in blob files older than a cutoff get + * relocated to new files as they are encountered during compaction, which makes + * it possible to clean up blob files once they contain nothing but + * obsolete/garbage blobs. See also blob_garbage_collection_age_cutoff below. + *

    + * Default: false + * + * @return true if blob garbage collection is currently enabled. + */ + boolean enableBlobGarbageCollection(); + + /** + * Set cutoff in terms of blob file age for garbage collection. Blobs in the + * oldest N blob files will be relocated when encountered during compaction, + * where N = garbage_collection_cutoff * number_of_blob_files. Note that + * enable_blob_garbage_collection has to be set in order for this option to have + * any effect. + *

    + * Default: 0.25 + * + * @param blobGarbageCollectionAgeCutoff the new age cutoff + * + * @return the reference to the current options. + */ + T setBlobGarbageCollectionAgeCutoff(double blobGarbageCollectionAgeCutoff); + /** + * Get cutoff in terms of blob file age for garbage collection. Blobs in the + * oldest N blob files will be relocated when encountered during compaction, + * where N = garbage_collection_cutoff * number_of_blob_files. Note that + * enable_blob_garbage_collection has to be set in order for this option to have + * any effect. + *

    + * Default: 0.25 + * + * @return the current age cutoff for garbage collection + */ + double blobGarbageCollectionAgeCutoff(); + + /** + * If the ratio of garbage in the oldest blob files exceeds this threshold, + * targeted compactions are scheduled in order to force garbage collecting + * the blob files in question, assuming they are all eligible based on the + * value of {@link #blobGarbageCollectionAgeCutoff} above. This option is + * currently only supported with leveled compactions. + *

    + * Note that {@link #enableBlobGarbageCollection} has to be set in order for this + * option to have any effect. + *

    + * Default: 1.0 + *

    + * Dynamically changeable through the SetOptions() API + * + * @param blobGarbageCollectionForceThreshold new value for the threshold + * @return the reference to the current options + */ + T setBlobGarbageCollectionForceThreshold(double blobGarbageCollectionForceThreshold); + + /** + * Get the current value for the {@code #blobGarbageCollectionForceThreshold} + * @return the current threshold at which garbage collection of blobs is forced + */ + double blobGarbageCollectionForceThreshold(); + + /** + * Set compaction readahead for blob files. + *

    + * Default: 0 + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @param blobCompactionReadaheadSize the compaction readahead for blob files + * + * @return the reference to the current options. + */ + T setBlobCompactionReadaheadSize(final long blobCompactionReadaheadSize); + + /** + * Get compaction readahead for blob files. + * + * @return the current compaction readahead for blob files + */ + long blobCompactionReadaheadSize(); + + /** + * Set a certain LSM tree level to enable blob files. + *

    + * Default: 0 + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @param blobFileStartingLevel the starting level to enable blob files + * + * @return the reference to the current options. + */ + T setBlobFileStartingLevel(final int blobFileStartingLevel); + + /** + * Get the starting LSM tree level to enable blob files. + *

    + * Default: 0 + * + * @return the current LSM tree level to enable blob files. + */ + int blobFileStartingLevel(); + + /** + * Set a certain prepopulate blob cache option. + *

    + * Default: 0 + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @param prepopulateBlobCache prepopulate the blob cache option + * + * @return the reference to the current options. + */ + T setPrepopulateBlobCache(final PrepopulateBlobCache prepopulateBlobCache); + + /** + * Get the prepopulate blob cache option. + *

    + * Default: 0 + * + * @return the current prepopulate blob cache option. + */ + PrepopulateBlobCache prepopulateBlobCache(); + + // + // END options for blobs (integrated BlobDB) + // +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/BackgroundErrorReason.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/BackgroundErrorReason.java new file mode 100644 index 0000000..eec593d --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/BackgroundErrorReason.java @@ -0,0 +1,46 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +public enum BackgroundErrorReason { + FLUSH((byte) 0x0), + COMPACTION((byte) 0x1), + WRITE_CALLBACK((byte) 0x2), + MEMTABLE((byte) 0x3); + + private final byte value; + + BackgroundErrorReason(final byte value) { + this.value = value; + } + + /** + * Get the internal representation. + * + * @return the internal representation + */ + byte getValue() { + return value; + } + + /** + * Get the BackgroundErrorReason from the internal representation value. + * + * @return the background error reason. + * + * @throws IllegalArgumentException if the value is unknown. + */ + static BackgroundErrorReason fromValue(final byte value) { + for (final BackgroundErrorReason backgroundErrorReason : BackgroundErrorReason.values()) { + if (backgroundErrorReason.value == value) { + return backgroundErrorReason; + } + } + + throw new IllegalArgumentException( + "Illegal value provided for BackgroundErrorReason: " + value); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/BackupEngine.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/BackupEngine.java new file mode 100644 index 0000000..3ab2206 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/BackupEngine.java @@ -0,0 +1,259 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +import java.util.List; + +/** + * BackupEngine allows you to backup + * and restore the database + *

    + * Be aware, that `new BackupEngine` takes time proportional to the amount + * of backups. So if you have a slow filesystem to backup + * and you have a lot of backups then restoring can take some time. + * That's why we recommend to limit the number of backups. + * Also we recommend to keep BackupEngine alive and not to recreate it every + * time you need to do a backup. + */ +public class BackupEngine extends RocksObject implements AutoCloseable { + + protected BackupEngine(final long nativeHandle) { + super(nativeHandle); + } + + /** + * Opens a new Backup Engine + * + * @param env The environment that the backup engine should operate within + * @param options Any options for the backup engine + * + * @return A new BackupEngine instance + * @throws RocksDBException thrown if the backup engine could not be opened + */ + public static BackupEngine open(final Env env, final BackupEngineOptions options) + throws RocksDBException { + return new BackupEngine(open(env.nativeHandle_, options.nativeHandle_)); + } + + /** + * Captures the state of the database in the latest backup + *

    + * Just a convenience for {@link #createNewBackup(RocksDB, boolean)} with + * the flushBeforeBackup parameter set to false + * + * @param db The database to backup + *

    + * Note - This method is not thread safe + * + * @throws RocksDBException thrown if a new backup could not be created + */ + public void createNewBackup(final RocksDB db) throws RocksDBException { + createNewBackup(db, false); + } + + /** + * Captures the state of the database in the latest backup + * + * @param db The database to backup + * @param flushBeforeBackup When true, the Backup Engine will first issue a + * memtable flush and only then copy the DB files to + * the backup directory. Doing so will prevent log + * files from being copied to the backup directory + * (since flush will delete them). + * When false, the Backup Engine will not issue a + * flush before starting the backup. In that case, + * the backup will also include log files + * corresponding to live memtables. If writes have + * been performed with the write ahead log disabled, + * set flushBeforeBackup to true to prevent those + * writes from being lost. Otherwise, the backup will + * always be consistent with the current state of the + * database regardless of the flushBeforeBackup + * parameter. + *

    + * Note - This method is not thread safe + * + * @throws RocksDBException thrown if a new backup could not be created + */ + public void createNewBackup( + final RocksDB db, final boolean flushBeforeBackup) + throws RocksDBException { + assert (isOwningHandle()); + createNewBackup(nativeHandle_, db.nativeHandle_, flushBeforeBackup); + } + + /** + * Captures the state of the database in the latest backup along with + * application specific metadata. + * + * @param db The database to backup + * @param metadata Application metadata + * @param flushBeforeBackup When true, the Backup Engine will first issue a + * memtable flush and only then copy the DB files to + * the backup directory. Doing so will prevent log + * files from being copied to the backup directory + * (since flush will delete them). + * When false, the Backup Engine will not issue a + * flush before starting the backup. In that case, + * the backup will also include log files + * corresponding to live memtables. If writes have + * been performed with the write ahead log disabled, + * set flushBeforeBackup to true to prevent those + * writes from being lost. Otherwise, the backup will + * always be consistent with the current state of the + * database regardless of the flushBeforeBackup + * parameter. + *

    + * Note - This method is not thread safe + * + * @throws RocksDBException thrown if a new backup could not be created + */ + public void createNewBackupWithMetadata(final RocksDB db, final String metadata, + final boolean flushBeforeBackup) throws RocksDBException { + assert (isOwningHandle()); + createNewBackupWithMetadata(nativeHandle_, db.nativeHandle_, metadata, flushBeforeBackup); + } + + /** + * Gets information about the available + * backups + * + * @return A list of information about each available backup + */ + public List getBackupInfo() { + assert (isOwningHandle()); + return getBackupInfo(nativeHandle_); + } + + /** + *

    Returns a list of corrupted backup ids. If there + * is no corrupted backup the method will return an + * empty list.

    + * + * @return array of backup ids as int ids. + */ + public int[] getCorruptedBackups() { + assert(isOwningHandle()); + return getCorruptedBackups(nativeHandle_); + } + + /** + *

    Will delete all the files we don't need anymore. It will + * do the full scan of the files/ directory and delete all the + * files that are not referenced.

    + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void garbageCollect() throws RocksDBException { + assert(isOwningHandle()); + garbageCollect(nativeHandle_); + } + + /** + * Deletes old backups, keeping just the latest numBackupsToKeep + * + * @param numBackupsToKeep The latest n backups to keep + * + * @throws RocksDBException thrown if the old backups could not be deleted + */ + public void purgeOldBackups( + final int numBackupsToKeep) throws RocksDBException { + assert (isOwningHandle()); + purgeOldBackups(nativeHandle_, numBackupsToKeep); + } + + /** + * Deletes a backup + * + * @param backupId The id of the backup to delete + * + * @throws RocksDBException thrown if the backup could not be deleted + */ + public void deleteBackup(final int backupId) throws RocksDBException { + assert (isOwningHandle()); + deleteBackup(nativeHandle_, backupId); + } + + /** + * Restore the database from a backup + *

    + * IMPORTANT: if options.share_table_files == true and you restore the DB + * from some backup that is not the latest, and you start creating new + * backups from the new DB, they will probably fail! + *

    + * Example: Let's say you have backups 1, 2, 3, 4, 5 and you restore 3. + * If you add new data to the DB and try creating a new backup now, the + * database will diverge from backups 4 and 5 and the new backup will fail. + * If you want to create new backup, you will first have to delete backups 4 + * and 5. + * + * @param backupId The id of the backup to restore + * @param dbDir The directory to restore the backup to, i.e. where your + * database is + * @param walDir The location of the log files for your database, + * often the same as dbDir + * @param restoreOptions Options for controlling the restore + * + * @throws RocksDBException thrown if the database could not be restored + */ + public void restoreDbFromBackup( + final int backupId, final String dbDir, final String walDir, + final RestoreOptions restoreOptions) throws RocksDBException { + assert (isOwningHandle()); + restoreDbFromBackup(nativeHandle_, backupId, dbDir, walDir, + restoreOptions.nativeHandle_); + } + + /** + * Restore the database from the latest backup + * + * @param dbDir The directory to restore the backup to, i.e. where your + * database is + * @param walDir The location of the log files for your database, often the + * same as dbDir + * @param restoreOptions Options for controlling the restore + * + * @throws RocksDBException thrown if the database could not be restored + */ + public void restoreDbFromLatestBackup( + final String dbDir, final String walDir, + final RestoreOptions restoreOptions) throws RocksDBException { + assert (isOwningHandle()); + restoreDbFromLatestBackup(nativeHandle_, dbDir, walDir, + restoreOptions.nativeHandle_); + } + + private static native long open(final long env, final long backupEngineOptions) + throws RocksDBException; + + private native void createNewBackup(final long handle, final long dbHandle, + final boolean flushBeforeBackup) throws RocksDBException; + + private native void createNewBackupWithMetadata(final long handle, final long dbHandle, + final String metadata, final boolean flushBeforeBackup) throws RocksDBException; + + private native List getBackupInfo(final long handle); + + private native int[] getCorruptedBackups(final long handle); + + private native void garbageCollect(final long handle) throws RocksDBException; + + private native void purgeOldBackups(final long handle, + final int numBackupsToKeep) throws RocksDBException; + + private native void deleteBackup(final long handle, final int backupId) + throws RocksDBException; + + private native void restoreDbFromBackup(final long handle, final int backupId, + final String dbDir, final String walDir, final long restoreOptionsHandle) + throws RocksDBException; + + private native void restoreDbFromLatestBackup(final long handle, + final String dbDir, final String walDir, final long restoreOptionsHandle) + throws RocksDBException; + + @Override protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/BackupEngineOptions.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/BackupEngineOptions.java new file mode 100644 index 0000000..2a358fa --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/BackupEngineOptions.java @@ -0,0 +1,458 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.io.File; + +/** + *

    BackupEngineOptions controls the behavior of a + * {@link org.rocksdb.BackupEngine}. + *

    + *

    Note that dispose() must be called before an Options instance + * become out-of-scope to release the allocated memory in c++.

    + * + * @see org.rocksdb.BackupEngine + */ +public class BackupEngineOptions extends RocksObject { + private Env backupEnv = null; + private Logger infoLog = null; + private RateLimiter backupRateLimiter = null; + private RateLimiter restoreRateLimiter = null; + + /** + *

    BackupEngineOptions constructor.

    + * + * @param path Where to keep the backup files. Has to be different from db + * name. Best to set this to {@code db name_ + "/backups"} + * @throws java.lang.IllegalArgumentException if illegal path is used. + */ + public BackupEngineOptions(final String path) { + super(newBackupEngineOptions(ensureWritableFile(path))); + } + + private static String ensureWritableFile(final String path) { + final File backupPath = path == null ? null : new File(path); + if (backupPath == null || !backupPath.isDirectory() || + !backupPath.canWrite()) { + throw new IllegalArgumentException("Illegal path provided."); + } else { + return path; + } + } + + /** + *

    Returns the path to the BackupEngine directory.

    + * + * @return the path to the BackupEngine directory. + */ + public String backupDir() { + assert(isOwningHandle()); + return backupDir(nativeHandle_); + } + + /** + * Backup Env object. It will be used for backup file I/O. If it's + * null, backups will be written out using DBs Env. Otherwise, + * backup's I/O will be performed using this object. + *

    + * Default: null + * + * @param env The environment to use + * @return instance of current BackupEngineOptions. + */ + public BackupEngineOptions setBackupEnv(final Env env) { + assert(isOwningHandle()); + setBackupEnv(nativeHandle_, env.nativeHandle_); + this.backupEnv = env; + return this; + } + + /** + * Backup Env object. It will be used for backup file I/O. If it's + * null, backups will be written out using DBs Env. Otherwise, + * backup's I/O will be performed using this object. + *

    + * Default: null + * + * @return The environment in use + */ + public Env backupEnv() { + return this.backupEnv; + } + + /** + *

    Share table files between backups.

    + * + * @param shareTableFiles If {@code share_table_files == true}, backup will + * assume that table files with same name have the same contents. This + * enables incremental backups and avoids unnecessary data copies. If + * {@code share_table_files == false}, each backup will be on its own and + * will not share any data with other backups. + * + *

    Default: true

    + * + * @return instance of current BackupEngineOptions. + */ + public BackupEngineOptions setShareTableFiles(final boolean shareTableFiles) { + assert(isOwningHandle()); + setShareTableFiles(nativeHandle_, shareTableFiles); + return this; + } + + /** + *

    Share table files between backups.

    + * + * @return boolean value indicating if SST files will be shared between + * backups. + */ + public boolean shareTableFiles() { + assert(isOwningHandle()); + return shareTableFiles(nativeHandle_); + } + + /** + * Set the logger to use for Backup info and error messages + * + * @param logger The logger to use for the backup + * @return instance of current BackupEngineOptions. + */ + public BackupEngineOptions setInfoLog(final Logger logger) { + assert(isOwningHandle()); + setInfoLog(nativeHandle_, logger.nativeHandle_); + this.infoLog = logger; + return this; + } + + /** + * Set the logger to use for Backup info and error messages + *

    + * Default: null + * + * @return The logger in use for the backup + */ + public Logger infoLog() { + return this.infoLog; + } + + /** + *

    Set synchronous backups.

    + * + * @param sync If {@code sync == true}, we can guarantee you'll get consistent + * backup even on a machine crash/reboot. Backup process is slower with sync + * enabled. If {@code sync == false}, we don't guarantee anything on machine + * reboot. However, chances are some backups are consistent. + * + *

    Default: true

    + * + * @return instance of current BackupEngineOptions. + */ + public BackupEngineOptions setSync(final boolean sync) { + assert(isOwningHandle()); + setSync(nativeHandle_, sync); + return this; + } + + /** + *

    Are synchronous backups activated.

    + * + * @return boolean value if synchronous backups are configured. + */ + public boolean sync() { + assert(isOwningHandle()); + return sync(nativeHandle_); + } + + /** + *

    Set if old data will be destroyed.

    + * + * @param destroyOldData If true, it will delete whatever backups there are + * already. + * + *

    Default: false

    + * + * @return instance of current BackupEngineOptions. + */ + public BackupEngineOptions setDestroyOldData(final boolean destroyOldData) { + assert(isOwningHandle()); + setDestroyOldData(nativeHandle_, destroyOldData); + return this; + } + + /** + *

    Returns if old data will be destroyed will performing new backups.

    + * + * @return boolean value indicating if old data will be destroyed. + */ + public boolean destroyOldData() { + assert(isOwningHandle()); + return destroyOldData(nativeHandle_); + } + + /** + *

    Set if log files shall be persisted.

    + * + * @param backupLogFiles If false, we won't back up log files. This option can + * be useful for backing up in-memory databases where log file are + * persisted, but table files are in memory. + * + *

    Default: true

    + * + * @return instance of current BackupEngineOptions. + */ + public BackupEngineOptions setBackupLogFiles(final boolean backupLogFiles) { + assert(isOwningHandle()); + setBackupLogFiles(nativeHandle_, backupLogFiles); + return this; + } + + /** + *

    Return information if log files shall be persisted.

    + * + * @return boolean value indicating if log files will be persisted. + */ + public boolean backupLogFiles() { + assert(isOwningHandle()); + return backupLogFiles(nativeHandle_); + } + + /** + *

    Set backup rate limit.

    + * + * @param backupRateLimit Max bytes that can be transferred in a second during + * backup. If 0 or negative, then go as fast as you can. + * + *

    Default: 0

    + * + * @return instance of current BackupEngineOptions. + */ + public BackupEngineOptions setBackupRateLimit(long backupRateLimit) { + assert(isOwningHandle()); + backupRateLimit = (backupRateLimit <= 0) ? 0 : backupRateLimit; + setBackupRateLimit(nativeHandle_, backupRateLimit); + return this; + } + + /** + *

    Return backup rate limit which described the max bytes that can be + * transferred in a second during backup.

    + * + * @return numerical value describing the backup transfer limit in bytes per + * second. + */ + public long backupRateLimit() { + assert(isOwningHandle()); + return backupRateLimit(nativeHandle_); + } + + /** + * Backup rate limiter. Used to control transfer speed for backup. If this is + * not null, {@link #backupRateLimit()} is ignored. + *

    + * Default: null + * + * @param backupRateLimiter The rate limiter to use for the backup + * @return instance of current BackupEngineOptions. + */ + public BackupEngineOptions setBackupRateLimiter(final RateLimiter backupRateLimiter) { + assert(isOwningHandle()); + setBackupRateLimiter(nativeHandle_, backupRateLimiter.nativeHandle_); + this.backupRateLimiter = backupRateLimiter; + return this; + } + + /** + * Backup rate limiter. Used to control transfer speed for backup. If this is + * not null, {@link #backupRateLimit()} is ignored. + *

    + * Default: null + * + * @return The rate limiter in use for the backup + */ + public RateLimiter backupRateLimiter() { + assert(isOwningHandle()); + return this.backupRateLimiter; + } + + /** + *

    Set restore rate limit.

    + * + * @param restoreRateLimit Max bytes that can be transferred in a second + * during restore. If 0 or negative, then go as fast as you can. + * + *

    Default: 0

    + * + * @return instance of current BackupEngineOptions. + */ + public BackupEngineOptions setRestoreRateLimit(long restoreRateLimit) { + assert(isOwningHandle()); + restoreRateLimit = (restoreRateLimit <= 0) ? 0 : restoreRateLimit; + setRestoreRateLimit(nativeHandle_, restoreRateLimit); + return this; + } + + /** + *

    Return restore rate limit which described the max bytes that can be + * transferred in a second during restore.

    + * + * @return numerical value describing the restore transfer limit in bytes per + * second. + */ + public long restoreRateLimit() { + assert(isOwningHandle()); + return restoreRateLimit(nativeHandle_); + } + + /** + * Restore rate limiter. Used to control transfer speed during restore. If + * this is not null, {@link #restoreRateLimit()} is ignored. + *

    + * Default: null + * + * @param restoreRateLimiter The rate limiter to use during restore + * @return instance of current BackupEngineOptions. + */ + public BackupEngineOptions setRestoreRateLimiter(final RateLimiter restoreRateLimiter) { + assert(isOwningHandle()); + setRestoreRateLimiter(nativeHandle_, restoreRateLimiter.nativeHandle_); + this.restoreRateLimiter = restoreRateLimiter; + return this; + } + + /** + * Restore rate limiter. Used to control transfer speed during restore. If + * this is not null, {@link #restoreRateLimit()} is ignored. + *

    + * Default: null + * + * @return The rate limiter in use during restore + */ + public RateLimiter restoreRateLimiter() { + assert(isOwningHandle()); + return this.restoreRateLimiter; + } + + /** + *

    Only used if share_table_files is set to true. If true, will consider + * that backups can come from different databases, hence a sst is not uniquely + * identified by its name, but by the triple (file name, crc32, file length) + *

    + * + * @param shareFilesWithChecksum boolean value indicating if SST files are + * stored using the triple (file name, crc32, file length) and not its name. + * + *

    Note: this is an experimental option, and you'll need to set it manually + * turn it on only if you know what you're doing*

    + * + *

    Default: false

    + * + * @return instance of current BackupEngineOptions. + */ + public BackupEngineOptions setShareFilesWithChecksum(final boolean shareFilesWithChecksum) { + assert(isOwningHandle()); + setShareFilesWithChecksum(nativeHandle_, shareFilesWithChecksum); + return this; + } + + /** + *

    Return of share files with checksum is active.

    + * + * @return boolean value indicating if share files with checksum + * is active. + */ + public boolean shareFilesWithChecksum() { + assert(isOwningHandle()); + return shareFilesWithChecksum(nativeHandle_); + } + + /** + * Up to this many background threads will copy files for + * {@link BackupEngine#createNewBackup(RocksDB, boolean)} and + * {@link BackupEngine#restoreDbFromBackup(int, String, String, RestoreOptions)} + * + * Default: 1 + * + * @param maxBackgroundOperations The maximum number of background threads + * @return instance of current BackupEngineOptions. + */ + public BackupEngineOptions setMaxBackgroundOperations(final int maxBackgroundOperations) { + assert(isOwningHandle()); + setMaxBackgroundOperations(nativeHandle_, maxBackgroundOperations); + return this; + } + + /** + * Up to this many background threads will copy files for + * {@link BackupEngine#createNewBackup(RocksDB, boolean)} and + * {@link BackupEngine#restoreDbFromBackup(int, String, String, RestoreOptions)} + * + * Default: 1 + * + * @return The maximum number of background threads + */ + public int maxBackgroundOperations() { + assert(isOwningHandle()); + return maxBackgroundOperations(nativeHandle_); + } + + /** + * During backup user can get callback every time next + * {@link #callbackTriggerIntervalSize()} bytes being copied. + *

    + * Default: 4194304 + * + * @param callbackTriggerIntervalSize The interval size for the + * callback trigger + * @return instance of current BackupEngineOptions. + */ + public BackupEngineOptions setCallbackTriggerIntervalSize( + final long callbackTriggerIntervalSize) { + assert(isOwningHandle()); + setCallbackTriggerIntervalSize(nativeHandle_, callbackTriggerIntervalSize); + return this; + } + + /** + * During backup user can get callback every time next + * {@code #callbackTriggerIntervalSize()} bytes being copied. + *

    + * Default: 4194304 + * + * @return The interval size for the callback trigger + */ + public long callbackTriggerIntervalSize() { + assert(isOwningHandle()); + return callbackTriggerIntervalSize(nativeHandle_); + } + + private static native long newBackupEngineOptions(final String path); + private native String backupDir(long handle); + private native void setBackupEnv(final long handle, final long envHandle); + private native void setShareTableFiles(long handle, boolean flag); + private native boolean shareTableFiles(long handle); + private native void setInfoLog(final long handle, final long infoLogHandle); + private native void setSync(long handle, boolean flag); + private native boolean sync(long handle); + private native void setDestroyOldData(long handle, boolean flag); + private native boolean destroyOldData(long handle); + private native void setBackupLogFiles(long handle, boolean flag); + private native boolean backupLogFiles(long handle); + private native void setBackupRateLimit(long handle, long rateLimit); + private native long backupRateLimit(long handle); + private native void setBackupRateLimiter(long handle, long rateLimiterHandle); + private native void setRestoreRateLimit(long handle, long rateLimit); + private native long restoreRateLimit(long handle); + private native void setRestoreRateLimiter(final long handle, + final long rateLimiterHandle); + private native void setShareFilesWithChecksum(long handle, boolean flag); + private native boolean shareFilesWithChecksum(long handle); + private native void setMaxBackgroundOperations(final long handle, + final int maxBackgroundOperations); + private native int maxBackgroundOperations(final long handle); + private native void setCallbackTriggerIntervalSize(final long handle, + long callbackTriggerIntervalSize); + private native long callbackTriggerIntervalSize(final long handle); + @Override protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/BackupInfo.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/BackupInfo.java new file mode 100644 index 0000000..9581b09 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/BackupInfo.java @@ -0,0 +1,76 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +/** + * Instances of this class describe a Backup made by + * {@link org.rocksdb.BackupEngine}. + */ +public class BackupInfo { + + /** + * Package private constructor used to create instances + * of BackupInfo by {@link org.rocksdb.BackupEngine} + * + * @param backupId id of backup + * @param timestamp timestamp of backup + * @param size size of backup + * @param numberFiles number of files related to this backup. + */ + BackupInfo(final int backupId, final long timestamp, final long size, final int numberFiles, + final String app_metadata) { + backupId_ = backupId; + timestamp_ = timestamp; + size_ = size; + numberFiles_ = numberFiles; + app_metadata_ = app_metadata; + } + + /** + * + * @return the backup id. + */ + public int backupId() { + return backupId_; + } + + /** + * + * @return the timestamp of the backup. + */ + public long timestamp() { + return timestamp_; + } + + /** + * + * @return the size of the backup + */ + public long size() { + return size_; + } + + /** + * + * @return the number of files of this backup. + */ + public int numberFiles() { + return numberFiles_; + } + + /** + * + * @return the associated application metadata, or null + */ + public String appMetadata() { + return app_metadata_; + } + + private final int backupId_; + private final long timestamp_; + private final long size_; + private final int numberFiles_; + private final String app_metadata_; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/BlockBasedTableConfig.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/BlockBasedTableConfig.java new file mode 100644 index 0000000..70dee3d --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/BlockBasedTableConfig.java @@ -0,0 +1,951 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +/** + * The config for plain table sst format. + *

    + * BlockBasedTable is a RocksDB's default SST file format. + */ +// TODO(AR) should be renamed BlockBasedTableOptions +public class BlockBasedTableConfig extends TableFormatConfig { + + public BlockBasedTableConfig() { + //TODO(AR) flushBlockPolicyFactory + cacheIndexAndFilterBlocks = false; + cacheIndexAndFilterBlocksWithHighPriority = true; + pinL0FilterAndIndexBlocksInCache = false; + pinTopLevelIndexAndFilter = true; + indexType = IndexType.kBinarySearch; + dataBlockIndexType = DataBlockIndexType.kDataBlockBinarySearch; + dataBlockHashTableUtilRatio = 0.75; + checksumType = ChecksumType.kCRC32c; + noBlockCache = false; + blockCache = null; + persistentCache = null; + blockSize = 4 * 1024; + blockSizeDeviation = 10; + blockRestartInterval = 16; + indexBlockRestartInterval = 1; + metadataBlockSize = 4096; + partitionFilters = false; + optimizeFiltersForMemory = false; + useDeltaEncoding = true; + filterPolicy = null; + wholeKeyFiltering = true; + verifyCompression = false; + readAmpBytesPerBit = 0; + formatVersion = 5; + enableIndexCompression = true; + blockAlign = false; + indexShortening = IndexShorteningMode.kShortenSeparators; + + // NOTE: ONLY used if blockCache == null + blockCacheSize = 8 * 1024 * 1024; + blockCacheNumShardBits = 0; + } + + /** + * Indicating if we'd put index/filter blocks to the block cache. + * If not specified, each "table reader" object will pre-load index/filter + * block during table initialization. + * + * @return if index and filter blocks should be put in block cache. + */ + public boolean cacheIndexAndFilterBlocks() { + return cacheIndexAndFilterBlocks; + } + + /** + * Indicating if we'd put index/filter blocks to the block cache. + * If not specified, each "table reader" object will pre-load index/filter + * block during table initialization. + * + * @param cacheIndexAndFilterBlocks and filter blocks should be put in block cache. + * @return the reference to the current config. + */ + public BlockBasedTableConfig setCacheIndexAndFilterBlocks( + final boolean cacheIndexAndFilterBlocks) { + this.cacheIndexAndFilterBlocks = cacheIndexAndFilterBlocks; + return this; + } + + /** + * Indicates if index and filter blocks will be treated as high-priority in the block cache. + * See note below about applicability. If not specified, defaults to true. + * + * @return if index and filter blocks will be treated as high-priority. + */ + public boolean cacheIndexAndFilterBlocksWithHighPriority() { + return cacheIndexAndFilterBlocksWithHighPriority; + } + + /** + * If true, cache index and filter blocks with high priority. If set to true, + * depending on implementation of block cache, index and filter blocks may be + * less likely to be evicted than data blocks. + * + * @param cacheIndexAndFilterBlocksWithHighPriority if index and filter blocks + * will be treated as high-priority. + * @return the reference to the current config. + */ + public BlockBasedTableConfig setCacheIndexAndFilterBlocksWithHighPriority( + final boolean cacheIndexAndFilterBlocksWithHighPriority) { + this.cacheIndexAndFilterBlocksWithHighPriority = cacheIndexAndFilterBlocksWithHighPriority; + return this; + } + + /** + * Indicating if we'd like to pin L0 index/filter blocks to the block cache. + If not specified, defaults to false. + * + * @return if L0 index and filter blocks should be pinned to the block cache. + */ + public boolean pinL0FilterAndIndexBlocksInCache() { + return pinL0FilterAndIndexBlocksInCache; + } + + /** + * Indicating if we'd like to pin L0 index/filter blocks to the block cache. + If not specified, defaults to false. + * + * @param pinL0FilterAndIndexBlocksInCache pin blocks in block cache + * @return the reference to the current config. + */ + public BlockBasedTableConfig setPinL0FilterAndIndexBlocksInCache( + final boolean pinL0FilterAndIndexBlocksInCache) { + this.pinL0FilterAndIndexBlocksInCache = pinL0FilterAndIndexBlocksInCache; + return this; + } + + /** + * Indicates if top-level index and filter blocks should be pinned. + * + * @return if top-level index and filter blocks should be pinned. + */ + public boolean pinTopLevelIndexAndFilter() { + return pinTopLevelIndexAndFilter; + } + + /** + * If cacheIndexAndFilterBlocks is true and the below is true, then + * the top-level index of partitioned filter and index blocks are stored in + * the cache, but a reference is held in the "table reader" object so the + * blocks are pinned and only evicted from cache when the table reader is + * freed. This is not limited to l0 in LSM tree. + * + * @param pinTopLevelIndexAndFilter if top-level index and filter blocks should be pinned. + * @return the reference to the current config. + */ + public BlockBasedTableConfig setPinTopLevelIndexAndFilter(final boolean pinTopLevelIndexAndFilter) { + this.pinTopLevelIndexAndFilter = pinTopLevelIndexAndFilter; + return this; + } + + /** + * Get the index type. + * + * @return the currently set index type + */ + public IndexType indexType() { + return indexType; + } + + /** + * Sets the index type to used with this table. + * + * @param indexType {@link org.rocksdb.IndexType} value + * @return the reference to the current option. + */ + public BlockBasedTableConfig setIndexType( + final IndexType indexType) { + this.indexType = indexType; + return this; + } + + /** + * Get the data block index type. + * + * @return the currently set data block index type + */ + public DataBlockIndexType dataBlockIndexType() { + return dataBlockIndexType; + } + + /** + * Sets the data block index type to used with this table. + * + * @param dataBlockIndexType {@link org.rocksdb.DataBlockIndexType} value + * @return the reference to the current option. + */ + public BlockBasedTableConfig setDataBlockIndexType( + final DataBlockIndexType dataBlockIndexType) { + this.dataBlockIndexType = dataBlockIndexType; + return this; + } + + /** + * Get the #entries/#buckets. It is valid only when {@link #dataBlockIndexType()} is + * {@link DataBlockIndexType#kDataBlockBinaryAndHash}. + * + * @return the #entries/#buckets. + */ + public double dataBlockHashTableUtilRatio() { + return dataBlockHashTableUtilRatio; + } + + /** + * Set the #entries/#buckets. It is valid only when {@link #dataBlockIndexType()} is + * {@link DataBlockIndexType#kDataBlockBinaryAndHash}. + * + * @param dataBlockHashTableUtilRatio #entries/#buckets + * @return the reference to the current option. + */ + public BlockBasedTableConfig setDataBlockHashTableUtilRatio( + final double dataBlockHashTableUtilRatio) { + this.dataBlockHashTableUtilRatio = dataBlockHashTableUtilRatio; + return this; + } + + /** + * Get the checksum type to be used with this table. + * + * @return the currently set checksum type + */ + public ChecksumType checksumType() { + return checksumType; + } + + /** + * Sets + * + * @param checksumType {@link org.rocksdb.ChecksumType} value. + * @return the reference to the current option. + */ + public BlockBasedTableConfig setChecksumType( + final ChecksumType checksumType) { + this.checksumType = checksumType; + return this; + } + + /** + * Determine if the block cache is disabled. + * + * @return if block cache is disabled + */ + public boolean noBlockCache() { + return noBlockCache; + } + + /** + * Disable block cache. If this is set to true, + * then no block cache should be used, and the {@link #setBlockCache(Cache)} + * should point to a {@code null} object. + *

    + * Default: false + * + * @param noBlockCache if use block cache + * @return the reference to the current config. + */ + public BlockBasedTableConfig setNoBlockCache(final boolean noBlockCache) { + this.noBlockCache = noBlockCache; + return this; + } + + /** + * Use the specified cache for blocks. + * When not null this take precedence even if the user sets a block cache size. + *

    + * {@link org.rocksdb.Cache} should not be disposed before options instances + * using this cache is disposed. + *

    + * {@link org.rocksdb.Cache} instance can be re-used in multiple options + * instances. + * + * @param blockCache {@link org.rocksdb.Cache} Cache java instance + * (e.g. LRUCache). + * + * @return the reference to the current config. + */ + public BlockBasedTableConfig setBlockCache(final Cache blockCache) { + this.blockCache = blockCache; + return this; + } + + /** + * Use the specified persistent cache. + *

    + * If {@code !null} use the specified cache for pages read from device, + * otherwise no page cache is used. + * + * @param persistentCache the persistent cache + * + * @return the reference to the current config. + */ + public BlockBasedTableConfig setPersistentCache( + final PersistentCache persistentCache) { + this.persistentCache = persistentCache; + return this; + } + + /** + * Get the approximate size of user data packed per block. + * + * @return block size in bytes + */ + public long blockSize() { + return blockSize; + } + + /** + * Approximate size of user data packed per block. Note that the + * block size specified here corresponds to uncompressed data. The + * actual size of the unit read from disk may be smaller if + * compression is enabled. This parameter can be changed dynamically. + * Default: 4K + * + * @param blockSize block size in bytes + * @return the reference to the current config. + */ + public BlockBasedTableConfig setBlockSize(final long blockSize) { + this.blockSize = blockSize; + return this; + } + + /** + * @return the hash table ratio. + */ + public int blockSizeDeviation() { + return blockSizeDeviation; + } + + /** + * This is used to close a block before it reaches the configured + * {@link #blockSize()}. If the percentage of free space in the current block + * is less than this specified number and adding a new record to the block + * will exceed the configured block size, then this block will be closed and + * the new record will be written to the next block. + *

    + * Default is 10. + * + * @param blockSizeDeviation the deviation to block size allowed + * @return the reference to the current config. + */ + public BlockBasedTableConfig setBlockSizeDeviation( + final int blockSizeDeviation) { + this.blockSizeDeviation = blockSizeDeviation; + return this; + } + + /** + * Get the block restart interval. + * + * @return block restart interval + */ + public int blockRestartInterval() { + return blockRestartInterval; + } + + /** + * Set the block restart interval. + * + * @param restartInterval block restart interval. + * @return the reference to the current config. + */ + public BlockBasedTableConfig setBlockRestartInterval( + final int restartInterval) { + blockRestartInterval = restartInterval; + return this; + } + + /** + * Get the index block restart interval. + * + * @return index block restart interval + */ + public int indexBlockRestartInterval() { + return indexBlockRestartInterval; + } + + /** + * Set the index block restart interval + * + * @param restartInterval index block restart interval. + * @return the reference to the current config. + */ + public BlockBasedTableConfig setIndexBlockRestartInterval( + final int restartInterval) { + indexBlockRestartInterval = restartInterval; + return this; + } + + /** + * Get the block size for partitioned metadata. + * + * @return block size for partitioned metadata. + */ + public long metadataBlockSize() { + return metadataBlockSize; + } + + /** + * Set block size for partitioned metadata. + * + * @param metadataBlockSize Partitioned metadata block size. + * @return the reference to the current config. + */ + public BlockBasedTableConfig setMetadataBlockSize( + final long metadataBlockSize) { + this.metadataBlockSize = metadataBlockSize; + return this; + } + + /** + * Indicates if we're using partitioned filters. + * + * @return if we're using partition filters. + */ + public boolean partitionFilters() { + return partitionFilters; + } + + /** + * Use partitioned full filters for each SST file. This option is incompatible + * with block-based filters. + *

    + * Defaults to false. + * + * @param partitionFilters use partition filters. + * @return the reference to the current config. + */ + public BlockBasedTableConfig setPartitionFilters(final boolean partitionFilters) { + this.partitionFilters = partitionFilters; + return this; + } + + /*** + * Option to generate Bloom filters that minimize memory + * internal fragmentation. + *

    + * See {@link #setOptimizeFiltersForMemory(boolean)}. + * + * @return true if bloom filters are used to minimize memory internal + * fragmentation + */ + @Experimental("Option to generate Bloom filters that minimize memory internal fragmentation") + public boolean optimizeFiltersForMemory() { + return optimizeFiltersForMemory; + } + + /** + * Option to generate Bloom filters that minimize memory + * internal fragmentation. + *

    + * When false, malloc_usable_size is not available, or format_version < 5, + * filters are generated without regard to internal fragmentation when + * loaded into memory (historical behavior). When true (and + * malloc_usable_size is available and {@link #formatVersion()} >= 5), + * then Bloom filters are generated to "round up" and "round down" their + * sizes to minimize internal fragmentation when loaded into memory, assuming + * the reading DB has the same memory allocation characteristics as the + * generating DB. This option does not break forward or backward + * compatibility. + *

    + * While individual filters will vary in bits/key and false positive rate + * when setting is true, the implementation attempts to maintain a weighted + * average FP rate for filters consistent with this option set to false. + *

    + * With Jemalloc for example, this setting is expected to save about 10% of + * the memory footprint and block cache charge of filters, while increasing + * disk usage of filters by about 1-2% due to encoding efficiency losses + * with variance in bits/key. + *

    + * NOTE: Because some memory counted by block cache might be unmapped pages + * within internal fragmentation, this option can increase observed RSS + * memory usage. With {@link #cacheIndexAndFilterBlocks()} == true, + * this option makes the block cache better at using space it is allowed. + *

    + * NOTE: Do not set to true if you do not trust malloc_usable_size. With + * this option, RocksDB might access an allocated memory object beyond its + * original size if malloc_usable_size says it is safe to do so. While this + * can be considered bad practice, it should not produce undefined behavior + * unless malloc_usable_size is buggy or broken. + * + * @param optimizeFiltersForMemory true to enable Bloom filters that minimize + * memory internal fragmentation, or false to disable. + * + * @return the reference to the current config. + */ + @Experimental("Option to generate Bloom filters that minimize memory internal fragmentation") + public BlockBasedTableConfig setOptimizeFiltersForMemory(final boolean optimizeFiltersForMemory) { + this.optimizeFiltersForMemory = optimizeFiltersForMemory; + return this; + } + + /** + * Determine if delta encoding is being used to compress block keys. + * + * @return true if delta encoding is enabled, false otherwise. + */ + public boolean useDeltaEncoding() { + return useDeltaEncoding; + } + + /** + * Use delta encoding to compress keys in blocks. + *

    + * NOTE: {@link ReadOptions#pinData()} requires this option to be disabled. + *

    + * Default: true + * + * @param useDeltaEncoding true to enable delta encoding + * + * @return the reference to the current config. + */ + public BlockBasedTableConfig setUseDeltaEncoding( + final boolean useDeltaEncoding) { + this.useDeltaEncoding = useDeltaEncoding; + return this; + } + + /** + * Get the filter policy. + * + * @return the current filter policy. + */ + public Filter filterPolicy() { + return filterPolicy; + } + + /** + * Use the specified filter policy to reduce disk reads. + *

    + * {@link org.rocksdb.Filter} should not be closed before options instances + * using this filter are closed. + *

    + * {@link org.rocksdb.Filter} instance can be re-used in multiple options + * instances. + * + * @param filterPolicy {@link org.rocksdb.Filter} Filter Policy java instance. + * @return the reference to the current config. + */ + public BlockBasedTableConfig setFilterPolicy( + final Filter filterPolicy) { + this.filterPolicy = filterPolicy; + return this; + } + + /** + * Set the filter. + * + * @param filter the filter + * @return the reference to the current config. + * + * @deprecated Use {@link #setFilterPolicy(Filter)} + */ + @Deprecated + public BlockBasedTableConfig setFilter( + final Filter filter) { + return setFilterPolicy(filter); + } + + /** + * Determine if whole keys as opposed to prefixes are placed in the filter. + * + * @return if whole key filtering is enabled + */ + public boolean wholeKeyFiltering() { + return wholeKeyFiltering; + } + + /** + * If true, place whole keys in the filter (not just prefixes). + * This must generally be true for gets to be efficient. + * Default: true + * + * @param wholeKeyFiltering if enable whole key filtering + * @return the reference to the current config. + */ + public BlockBasedTableConfig setWholeKeyFiltering( + final boolean wholeKeyFiltering) { + this.wholeKeyFiltering = wholeKeyFiltering; + return this; + } + + /** + * Returns true when compression verification is enabled. + *

    + * See {@link #setVerifyCompression(boolean)}. + * + * @return true if compression verification is enabled. + */ + public boolean verifyCompression() { + return verifyCompression; + } + + /** + * Verify that decompressing the compressed block gives back the input. This + * is a verification mode that we use to detect bugs in compression + * algorithms. + * + * @param verifyCompression true to enable compression verification. + * + * @return the reference to the current config. + */ + public BlockBasedTableConfig setVerifyCompression( + final boolean verifyCompression) { + this.verifyCompression = verifyCompression; + return this; + } + + /** + * Get the Read amplification bytes per-bit. + *

    + * See {@link #setReadAmpBytesPerBit(int)}. + * + * @return the bytes per-bit. + */ + public int readAmpBytesPerBit() { + return readAmpBytesPerBit; + } + + /** + * Set the Read amplification bytes per-bit. + *

    + * If used, For every data block we load into memory, we will create a bitmap + * of size ((block_size / `read_amp_bytes_per_bit`) / 8) bytes. This bitmap + * will be used to figure out the percentage we actually read of the blocks. + *

    + * When this feature is used Tickers::READ_AMP_ESTIMATE_USEFUL_BYTES and + * Tickers::READ_AMP_TOTAL_READ_BYTES can be used to calculate the + * read amplification using this formula + * (READ_AMP_TOTAL_READ_BYTES / READ_AMP_ESTIMATE_USEFUL_BYTES) + *

    + * value => memory usage (percentage of loaded blocks memory) + * 1 => 12.50 % + * 2 => 06.25 % + * 4 => 03.12 % + * 8 => 01.56 % + * 16 => 00.78 % + *

    + * Note: This number must be a power of 2, if not it will be sanitized + * to be the next lowest power of 2, for example a value of 7 will be + * treated as 4, a value of 19 will be treated as 16. + *

    + * Default: 0 (disabled) + * + * @param readAmpBytesPerBit the bytes per-bit + * + * @return the reference to the current config. + */ + public BlockBasedTableConfig setReadAmpBytesPerBit(final int readAmpBytesPerBit) { + this.readAmpBytesPerBit = readAmpBytesPerBit; + return this; + } + + /** + * Get the format version. + * See {@link #setFormatVersion(int)}. + * + * @return the currently configured format version. + */ + public int formatVersion() { + return formatVersion; + } + + /** + *

    We currently have five versions:

    + * + *
      + *
    • 0 - This version is currently written + * out by all RocksDB's versions by default. Can be read by really old + * RocksDB's. Doesn't support changing checksum (default is CRC32).
      • + *
    • 1 - Can be read by RocksDB's versions since 3.0. + * Supports non-default checksum, like xxHash. It is written by RocksDB when + * BlockBasedTableOptions::checksum is something other than kCRC32c. (version + * 0 is silently upconverted)
      • + *
    • 2 - Can be read by RocksDB's versions since 3.10. + * Changes the way we encode compressed blocks with LZ4, BZip2 and Zlib + * compression. If you don't plan to run RocksDB before version 3.10, + * you should probably use this.
      • + *
    • 3 - Can be read by RocksDB's versions since 5.15. Changes the way we + * encode the keys in index blocks. If you don't plan to run RocksDB before + * version 5.15, you should probably use this. + * This option only affects newly written tables. When reading existing + * tables, the information about version is read from the footer.
      • + *
    • 4 - Can be read by RocksDB's versions since 5.16. Changes the way we + * encode the values in index blocks. If you don't plan to run RocksDB before + * version 5.16 and you are using index_block_restart_interval > 1, you should + * probably use this as it would reduce the index size. + * This option only affects newly written tables. When reading existing + * tables, the information about version is read from the footer.
      • + *
    • 5 - Can be read by RocksDB's versions since 6.6.0. + * Full and partitioned filters use a generally faster and more accurate + * Bloom filter implementation, with a different schema.
      • + *
    + * + * @param formatVersion integer representing the version to be used. + * + * @return the reference to the current option. + */ + public BlockBasedTableConfig setFormatVersion( + final int formatVersion) { + assert (formatVersion >= 0); + this.formatVersion = formatVersion; + return this; + } + + /** + * Determine if index compression is enabled. + *

    + * See {@link #setEnableIndexCompression(boolean)}. + * + * @return true if index compression is enabled, false otherwise + */ + public boolean enableIndexCompression() { + return enableIndexCompression; + } + + /** + * Store index blocks on disk in compressed format. + *

    + * Changing this option to false will avoid the overhead of decompression + * if index blocks are evicted and read back. + * + * @param enableIndexCompression true to enable index compression, + * false to disable + * + * @return the reference to the current option. + */ + public BlockBasedTableConfig setEnableIndexCompression( + final boolean enableIndexCompression) { + this.enableIndexCompression = enableIndexCompression; + return this; + } + + /** + * Determines whether data blocks are aligned on the lesser of page size + * and block size. + * + * @return true if data blocks are aligned on the lesser of page size + * and block size. + */ + public boolean blockAlign() { + return blockAlign; + } + + /** + * Set whether data blocks should be aligned on the lesser of page size + * and block size. + * + * @param blockAlign true to align data blocks on the lesser of page size + * and block size. + * + * @return the reference to the current option. + */ + public BlockBasedTableConfig setBlockAlign(final boolean blockAlign) { + this.blockAlign = blockAlign; + return this; + } + + /** + * Get the index shortening mode. + * + * @return the index shortening mode. + */ + public IndexShorteningMode indexShortening() { + return indexShortening; + } + + /** + * Set the index shortening mode. + * + * See {@link IndexShorteningMode}. + * + * @param indexShortening the index shortening mode. + * + * @return the reference to the current option. + */ + public BlockBasedTableConfig setIndexShortening(final IndexShorteningMode indexShortening) { + this.indexShortening = indexShortening; + return this; + } + + /** + * Get the size of the cache in bytes that will be used by RocksDB. + * + * @return block cache size in bytes + */ + @Deprecated + public long blockCacheSize() { + return blockCacheSize; + } + + /** + * Set the size of the cache in bytes that will be used by RocksDB. + * If cacheSize is negative, then cache will not be used. + * DEFAULT: 8M + * + * @param blockCacheSize block cache size in bytes + * @return the reference to the current config. + * + * @deprecated Use {@link #setBlockCache(Cache)}. + */ + @Deprecated + public BlockBasedTableConfig setBlockCacheSize(final long blockCacheSize) { + this.blockCacheSize = blockCacheSize; + return this; + } + + /** + * Returns the number of shard bits used in the block cache. + * The resulting number of shards would be 2 ^ (returned value). + * Any negative number means use default settings. + * + * @return the number of shard bits used in the block cache. + */ + @Deprecated + public int cacheNumShardBits() { + return blockCacheNumShardBits; + } + + /** + * Controls the number of shards for the block cache. + * This is applied only if cacheSize is set to non-negative. + * + * @param blockCacheNumShardBits the number of shard bits. The resulting + * number of shards would be 2 ^ numShardBits. Any negative + * number means use default settings." + * @return the reference to the current option. + * + * @deprecated Use {@link #setBlockCache(Cache)}. + */ + @Deprecated + public BlockBasedTableConfig setCacheNumShardBits( + final int blockCacheNumShardBits) { + this.blockCacheNumShardBits = blockCacheNumShardBits; + return this; + } + + /** + * Influence the behavior when kHashSearch is used. + * if false, stores a precise prefix to block range mapping + * if true, does not store prefix and allows prefix hash collision + * (less memory consumption) + * + * @return if hash collisions should be allowed. + * + * @deprecated This option is now deprecated. No matter what value it + * is set to, it will behave as + * if {@link #hashIndexAllowCollision()} == true. + */ + @Deprecated + public boolean hashIndexAllowCollision() { + return true; + } + + /** + * Influence the behavior when kHashSearch is used. + * if false, stores a precise prefix to block range mapping + * if true, does not store prefix and allows prefix hash collision + * (less memory consumption) + * + * @param hashIndexAllowCollision points out if hash collisions should be allowed. + * + * @return the reference to the current config. + * + * @deprecated This option is now deprecated. No matter what value it + * is set to, it will behave as + * if {@link #hashIndexAllowCollision()} == true. + */ + @Deprecated + public BlockBasedTableConfig setHashIndexAllowCollision( + final boolean hashIndexAllowCollision) { + // no-op + return this; + } + + @Override protected long newTableFactoryHandle() { + final long filterPolicyHandle; + if (filterPolicy != null) { + filterPolicyHandle = filterPolicy.nativeHandle_; + } else { + filterPolicyHandle = 0; + } + + final long blockCacheHandle; + if (blockCache != null) { + blockCacheHandle = blockCache.nativeHandle_; + } else { + blockCacheHandle = 0; + } + + final long persistentCacheHandle; + if (persistentCache != null) { + persistentCacheHandle = persistentCache.nativeHandle_; + } else { + persistentCacheHandle = 0; + } + + return newTableFactoryHandle(cacheIndexAndFilterBlocks, + cacheIndexAndFilterBlocksWithHighPriority, pinL0FilterAndIndexBlocksInCache, + pinTopLevelIndexAndFilter, indexType.getValue(), dataBlockIndexType.getValue(), + dataBlockHashTableUtilRatio, checksumType.getValue(), noBlockCache, blockCacheHandle, + persistentCacheHandle, blockSize, blockSizeDeviation, blockRestartInterval, + indexBlockRestartInterval, metadataBlockSize, partitionFilters, optimizeFiltersForMemory, + useDeltaEncoding, filterPolicyHandle, wholeKeyFiltering, verifyCompression, + readAmpBytesPerBit, formatVersion, enableIndexCompression, blockAlign, + indexShortening.getValue(), blockCacheSize, blockCacheNumShardBits); + } + + private native long newTableFactoryHandle(final boolean cacheIndexAndFilterBlocks, + final boolean cacheIndexAndFilterBlocksWithHighPriority, + final boolean pinL0FilterAndIndexBlocksInCache, final boolean pinTopLevelIndexAndFilter, + final byte indexTypeValue, final byte dataBlockIndexTypeValue, + final double dataBlockHashTableUtilRatio, final byte checksumTypeValue, + final boolean noBlockCache, final long blockCacheHandle, final long persistentCacheHandle, + final long blockSize, final int blockSizeDeviation, final int blockRestartInterval, + final int indexBlockRestartInterval, final long metadataBlockSize, + final boolean partitionFilters, final boolean optimizeFiltersForMemory, + final boolean useDeltaEncoding, final long filterPolicyHandle, + final boolean wholeKeyFiltering, final boolean verifyCompression, + final int readAmpBytesPerBit, final int formatVersion, final boolean enableIndexCompression, + final boolean blockAlign, final byte indexShortening, + + @Deprecated final long blockCacheSize, @Deprecated final int blockCacheNumShardBits); + + //TODO(AR) flushBlockPolicyFactory + private boolean cacheIndexAndFilterBlocks; + private boolean cacheIndexAndFilterBlocksWithHighPriority; + private boolean pinL0FilterAndIndexBlocksInCache; + private boolean pinTopLevelIndexAndFilter; + private IndexType indexType; + private DataBlockIndexType dataBlockIndexType; + private double dataBlockHashTableUtilRatio; + private ChecksumType checksumType; + private boolean noBlockCache; + private Cache blockCache; + private PersistentCache persistentCache; + private long blockSize; + private int blockSizeDeviation; + private int blockRestartInterval; + private int indexBlockRestartInterval; + private long metadataBlockSize; + private boolean partitionFilters; + private boolean optimizeFiltersForMemory; + private boolean useDeltaEncoding; + private Filter filterPolicy; + private boolean wholeKeyFiltering; + private boolean verifyCompression; + private int readAmpBytesPerBit; + private int formatVersion; + private boolean enableIndexCompression; + private boolean blockAlign; + private IndexShorteningMode indexShortening; + + // NOTE: ONLY used if blockCache == null + @Deprecated private long blockCacheSize; + @Deprecated private int blockCacheNumShardBits; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/BloomFilter.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/BloomFilter.java new file mode 100644 index 0000000..0b4e932 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/BloomFilter.java @@ -0,0 +1,73 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Bloom filter policy that uses a bloom filter with approximately + * the specified number of bits per key. + * + *

    + * Note: if you are using a custom comparator that ignores some parts + * of the keys being compared, you must not use this {@code BloomFilter} + * and must provide your own FilterPolicy that also ignores the + * corresponding parts of the keys. For example, if the comparator + * ignores trailing spaces, it would be incorrect to use a + * FilterPolicy (like {@code BloomFilter}) that does not ignore + * trailing spaces in keys.

    + */ +public class BloomFilter extends Filter { + + private static final double DEFAULT_BITS_PER_KEY = 10.0; + + /** + * BloomFilter constructor + * + *

    + * Callers must delete the result after any database that is using the + * result has been closed.

    + */ + public BloomFilter() { + this(DEFAULT_BITS_PER_KEY); + } + + /** + * BloomFilter constructor + * + *

    + * bits_per_key: bits per key in bloom filter. A good value for bits_per_key + * is 9.9, which yields a filter with ~ 1% false positive rate. + *

    + *

    + * Callers must delete the result after any database that is using the + * result has been closed.

    + * + * @param bitsPerKey number of bits to use + */ + public BloomFilter(final double bitsPerKey) { + super(createNewBloomFilter(bitsPerKey)); + } + + /** + * BloomFilter constructor + * + *

    + * bits_per_key: bits per key in bloom filter. A good value for bits_per_key + * is 10, which yields a filter with ~ 1% false positive rate. + *

    default bits_per_key: 10

    + * + *

    + * Callers must delete the result after any database that is using the + * result has been closed.

    + * + * @param bitsPerKey number of bits to use + * @param IGNORED_useBlockBasedMode obsolete, ignored parameter + */ + public BloomFilter(final double bitsPerKey, final boolean IGNORED_useBlockBasedMode) { + this(bitsPerKey); + } + + private static native long createNewBloomFilter(final double bitsKeyKey); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/BuiltinComparator.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/BuiltinComparator.java new file mode 100644 index 0000000..2c89bf2 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/BuiltinComparator.java @@ -0,0 +1,20 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Builtin RocksDB comparators + * + *
      + *
    1. BYTEWISE_COMPARATOR - Sorts all keys in ascending bytewise + * order.
      2. + *
    2. REVERSE_BYTEWISE_COMPARATOR - Sorts all keys in descending bytewise + * order
      3. + *
    + */ +public enum BuiltinComparator { + BYTEWISE_COMPARATOR, REVERSE_BYTEWISE_COMPARATOR +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ByteBufferGetStatus.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ByteBufferGetStatus.java new file mode 100644 index 0000000..f918a8d --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ByteBufferGetStatus.java @@ -0,0 +1,50 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.nio.ByteBuffer; +import java.util.List; + +/** + * A ByteBuffer containing fetched data, together with a result for the fetch + * and the total size of the object fetched. + *

    + * Used for the individual results of + * {@link RocksDB#multiGetByteBuffers(List, List)} + * {@link RocksDB#multiGetByteBuffers(List, List, List)} + * {@link RocksDB#multiGetByteBuffers(ReadOptions, List, List)} + * {@link RocksDB#multiGetByteBuffers(ReadOptions, List, List, List)} + */ +public class ByteBufferGetStatus { + public final Status status; + public final int requiredSize; + public final ByteBuffer value; + + /** + * Constructor used for success status, when the value is contained in the buffer + * + * @param status the status of the request to fetch into the buffer + * @param requiredSize the size of the data, which may be bigger than the buffer + * @param value the buffer containing as much of the value as fits + */ + ByteBufferGetStatus(final Status status, final int requiredSize, final ByteBuffer value) { + this.status = status; + this.requiredSize = requiredSize; + this.value = value; + } + + /** + * Constructor used for a failure status, when no value is filled in + * + * @param status the status of the request to fetch into the buffer + */ + ByteBufferGetStatus(final Status status) { + this.status = status; + this.requiredSize = 0; + this.value = null; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Cache.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Cache.java new file mode 100644 index 0000000..04bd3fc --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Cache.java @@ -0,0 +1,40 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + + +public abstract class Cache extends RocksObject { + protected Cache(final long nativeHandle) { + super(nativeHandle); + } + + /** + * Returns the memory size for the entries + * residing in cache. + * + * @return cache usage size. + * + */ + public long getUsage() { + assert (isOwningHandle()); + return getUsage(this.nativeHandle_); + } + + /** + * Returns the memory size for the entries + * being pinned in cache. + * + * @return cache pinned usage size. + * + */ + public long getPinnedUsage() { + assert (isOwningHandle()); + return getPinnedUsage(this.nativeHandle_); + } + + private static native long getUsage(final long handle); + private static native long getPinnedUsage(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CassandraCompactionFilter.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CassandraCompactionFilter.java new file mode 100644 index 0000000..12854c5 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CassandraCompactionFilter.java @@ -0,0 +1,20 @@ +// Copyright (c) 2017-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Just a Java wrapper around CassandraCompactionFilter implemented in C++ + */ +public class CassandraCompactionFilter + extends AbstractCompactionFilter { + public CassandraCompactionFilter( + final boolean purgeTtlOnExpiration, final int gcGracePeriodInSeconds) { + super(createNewCassandraCompactionFilter0(purgeTtlOnExpiration, gcGracePeriodInSeconds)); + } + + private static native long createNewCassandraCompactionFilter0( + boolean purgeTtlOnExpiration, int gcGracePeriodInSeconds); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CassandraValueMergeOperator.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CassandraValueMergeOperator.java new file mode 100644 index 0000000..732faee --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CassandraValueMergeOperator.java @@ -0,0 +1,25 @@ +// Copyright (c) 2017-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * CassandraValueMergeOperator is a merge operator that merges two cassandra wide column + * values. + */ +public class CassandraValueMergeOperator extends MergeOperator { + public CassandraValueMergeOperator(final int gcGracePeriodInSeconds) { + super(newSharedCassandraValueMergeOperator(gcGracePeriodInSeconds, 0)); + } + + public CassandraValueMergeOperator(final int gcGracePeriodInSeconds, final int operandsLimit) { + super(newSharedCassandraValueMergeOperator(gcGracePeriodInSeconds, operandsLimit)); + } + + private static native long newSharedCassandraValueMergeOperator( + int gcGracePeriodInSeconds, int limit); + + @Override protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Checkpoint.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Checkpoint.java new file mode 100644 index 0000000..c9b3886 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Checkpoint.java @@ -0,0 +1,62 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Provides Checkpoint functionality. Checkpoints + * provide persistent snapshots of RocksDB databases. + */ +public class Checkpoint extends RocksObject { + + /** + * Creates a Checkpoint object to be used for creating open-able + * snapshots. + * + * @param db {@link RocksDB} instance. + * @return a Checkpoint instance. + * + * @throws java.lang.IllegalArgumentException if {@link RocksDB} + * instance is null. + * @throws java.lang.IllegalStateException if {@link RocksDB} + * instance is not initialized. + */ + public static Checkpoint create(final RocksDB db) { + if (db == null) { + throw new IllegalArgumentException( + "RocksDB instance shall not be null."); + } else if (!db.isOwningHandle()) { + throw new IllegalStateException( + "RocksDB instance must be initialized."); + } + return new Checkpoint(db); + } + + /** + *

    Builds an open-able snapshot of RocksDB on the same disk, which + * accepts an output directory on the same disk, and under the directory + * (1) hard-linked SST files pointing to existing live SST files + * (2) a copied manifest files and other files

    + * + * @param checkpointPath path to the folder where the snapshot is going + * to be stored. + * @throws RocksDBException thrown if an error occurs within the native + * part of the library. + */ + public void createCheckpoint(final String checkpointPath) + throws RocksDBException { + createCheckpoint(nativeHandle_, checkpointPath); + } + + private Checkpoint(final RocksDB db) { + super(newCheckpoint(db.nativeHandle_)); + } + + private static native long newCheckpoint(long dbHandle); + @Override protected final native void disposeInternal(final long handle); + + private native void createCheckpoint(long handle, String checkpointPath) + throws RocksDBException; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ChecksumType.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ChecksumType.java new file mode 100644 index 0000000..e03fa14 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ChecksumType.java @@ -0,0 +1,45 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Checksum types used in conjunction with BlockBasedTable. + */ +public enum ChecksumType { + /** + * Not implemented yet. + */ + kNoChecksum((byte) 0), + /** + * CRC32 Checksum + */ + kCRC32c((byte) 1), + /** + * XX Hash + */ + kxxHash((byte) 2), + /** + * XX Hash 64 + */ + kxxHash64((byte) 3), + + kXXH3((byte) 4); + + /** + * Returns the byte value of the enumerations value + * + * @return byte representation + */ + public byte getValue() { + return value_; + } + + private ChecksumType(final byte value) { + value_ = value; + } + + private final byte value_; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ClockCache.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ClockCache.java new file mode 100644 index 0000000..e4251db --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ClockCache.java @@ -0,0 +1,59 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Similar to {@link LRUCache}, but based on the CLOCK algorithm with + * better concurrent performance in some cases + */ +public class ClockCache extends Cache { + + /** + * Create a new cache with a fixed size capacity. + * + * @param capacity The fixed size capacity of the cache + */ + public ClockCache(final long capacity) { + super(newClockCache(capacity, -1, false)); + } + + /** + * Create a new cache with a fixed size capacity. The cache is sharded + * to 2^numShardBits shards, by hash of the key. The total capacity + * is divided and evenly assigned to each shard. + * numShardBits = -1 means it is automatically determined: every shard + * will be at least 512KB and number of shard bits will not exceed 6. + * + * @param capacity The fixed size capacity of the cache + * @param numShardBits The cache is sharded to 2^numShardBits shards, + * by hash of the key + */ + public ClockCache(final long capacity, final int numShardBits) { + super(newClockCache(capacity, numShardBits, false)); + } + + /** + * Create a new cache with a fixed size capacity. The cache is sharded + * to 2^numShardBits shards, by hash of the key. The total capacity + * is divided and evenly assigned to each shard. If strictCapacityLimit + * is set, insert to the cache will fail when cache is full. + * numShardBits = -1 means it is automatically determined: every shard + * will be at least 512KB and number of shard bits will not exceed 6. + * + * @param capacity The fixed size capacity of the cache + * @param numShardBits The cache is sharded to 2^numShardBits shards, + * by hash of the key + * @param strictCapacityLimit insert to the cache will fail when cache is full + */ + public ClockCache(final long capacity, final int numShardBits, + final boolean strictCapacityLimit) { + super(newClockCache(capacity, numShardBits, strictCapacityLimit)); + } + + private static native long newClockCache( + final long capacity, final int numShardBits, final boolean strictCapacityLimit); + @Override protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ColumnFamilyDescriptor.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ColumnFamilyDescriptor.java new file mode 100644 index 0000000..125a8dc --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ColumnFamilyDescriptor.java @@ -0,0 +1,84 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Arrays; + +/** + *

    Describes a column family with a + * name and respective Options.

    + */ +public class ColumnFamilyDescriptor { + + /** + *

    Creates a new Column Family using a name and default + * options,

    + * + * @param columnFamilyName name of column family. + * @since 3.10.0 + */ + public ColumnFamilyDescriptor(final byte[] columnFamilyName) { + this(columnFamilyName, new ColumnFamilyOptions()); + } + + /** + *

    Creates a new Column Family using a name and custom + * options.

    + * + * @param columnFamilyName name of column family. + * @param columnFamilyOptions options to be used with + * column family. + * @since 3.10.0 + */ + public ColumnFamilyDescriptor(final byte[] columnFamilyName, + final ColumnFamilyOptions columnFamilyOptions) { + columnFamilyName_ = columnFamilyName; + columnFamilyOptions_ = columnFamilyOptions; + } + + /** + * Retrieve name of column family. + * + * @return column family name. + * @since 3.10.0 + */ + public byte[] getName() { + return columnFamilyName_; + } + + /** + * Retrieve assigned options instance. + * + * @return Options instance assigned to this instance. + */ + public ColumnFamilyOptions getOptions() { + return columnFamilyOptions_; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + final ColumnFamilyDescriptor that = (ColumnFamilyDescriptor) o; + return Arrays.equals(columnFamilyName_, that.columnFamilyName_) + && columnFamilyOptions_.nativeHandle_ == that.columnFamilyOptions_.nativeHandle_; + } + + @Override + public int hashCode() { + int result = (int) (columnFamilyOptions_.nativeHandle_ ^ (columnFamilyOptions_.nativeHandle_ >>> 32)); + result = 31 * result + Arrays.hashCode(columnFamilyName_); + return result; + } + + private final byte[] columnFamilyName_; + private final ColumnFamilyOptions columnFamilyOptions_; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ColumnFamilyHandle.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ColumnFamilyHandle.java new file mode 100644 index 0000000..32ea4b0 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ColumnFamilyHandle.java @@ -0,0 +1,151 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Arrays; +import java.util.Objects; + +/** + * ColumnFamilyHandle class to hold handles to underlying rocksdb + * ColumnFamily Pointers. + */ +public class ColumnFamilyHandle extends RocksObject { + /** + * Constructs column family Java object, which operates on underlying native object. + * + * @param rocksDB db instance associated with this column family + * @param nativeHandle native handle to underlying native ColumnFamily object + */ + ColumnFamilyHandle(final RocksDB rocksDB, + final long nativeHandle) { + super(nativeHandle); + // rocksDB must point to a valid RocksDB instance; + assert(rocksDB != null); + // ColumnFamilyHandle must hold a reference to the related RocksDB instance + // to guarantee that while a GC cycle starts ColumnFamilyHandle instances + // are freed prior to RocksDB instances. + this.rocksDB_ = rocksDB; + } + + /** + * Constructor called only from JNI. + *

    + * NOTE: we are producing an additional Java Object here to represent the underlying native C++ + * ColumnFamilyHandle object. The underlying object is not owned by ourselves. The Java API user + * likely already had a ColumnFamilyHandle Java object which owns the underlying C++ object, as + * they will have been presented it when they opened the database or added a Column Family. + *

    + * + * TODO(AR) - Potentially a better design would be to cache the active Java Column Family Objects + * in RocksDB, and return the same Java Object instead of instantiating a new one here. This could + * also help us to improve the Java API semantics for Java users. See for example + * .... + * + * @param nativeHandle native handle to the column family. + */ + ColumnFamilyHandle(final long nativeHandle) { + super(nativeHandle); + rocksDB_ = null; + disOwnNativeHandle(); + } + + /** + * Gets the name of the Column Family. + * + * @return The name of the Column Family. + * + * @throws RocksDBException if an error occurs whilst retrieving the name. + */ + public byte[] getName() throws RocksDBException { + assert(isOwningHandle() || isDefaultColumnFamily()); + return getName(nativeHandle_); + } + + /** + * Gets the ID of the Column Family. + * + * @return the ID of the Column Family. + */ + public int getID() { + assert(isOwningHandle() || isDefaultColumnFamily()); + return getID(nativeHandle_); + } + + /** + * Gets the up-to-date descriptor of the column family + * associated with this handle. Since it fills "*desc" with the up-to-date + * information, this call might internally lock and release DB mutex to + * access the up-to-date CF options. In addition, all the pointer-typed + * options cannot be referenced any longer than the original options exist. + *

    + * Note that this function is not supported in RocksDBLite. + * + * @return the up-to-date descriptor. + * + * @throws RocksDBException if an error occurs whilst retrieving the + * descriptor. + */ + public ColumnFamilyDescriptor getDescriptor() throws RocksDBException { + assert(isOwningHandle() || isDefaultColumnFamily()); + return getDescriptor(nativeHandle_); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + final ColumnFamilyHandle that = (ColumnFamilyHandle) o; + try { + return rocksDB_.nativeHandle_ == that.rocksDB_.nativeHandle_ && + getID() == that.getID() && + Arrays.equals(getName(), that.getName()); + } catch (final RocksDBException e) { + throw new RuntimeException("Cannot compare column family handles", e); + } + } + + @Override + public int hashCode() { + try { + int result = Objects.hash(getID(), rocksDB_.nativeHandle_); + result = 31 * result + Arrays.hashCode(getName()); + return result; + } catch (final RocksDBException e) { + throw new RuntimeException("Cannot calculate hash code of column family handle", e); + } + } + + protected boolean isDefaultColumnFamily() { + return nativeHandle_ == rocksDB_.getDefaultColumnFamily().nativeHandle_; + } + + /** + *

    Deletes underlying C++ iterator pointer.

    + * + *

    Note: the underlying handle can only be safely deleted if the RocksDB + * instance related to a certain ColumnFamilyHandle is still valid and + * initialized. Therefore {@code disposeInternal()} checks if the RocksDB is + * initialized before freeing the native handle.

    + */ + @Override + protected void disposeInternal() { + if(rocksDB_.isOwningHandle()) { + disposeInternal(nativeHandle_); + } + } + + private native byte[] getName(final long handle) throws RocksDBException; + private native int getID(final long handle); + private native ColumnFamilyDescriptor getDescriptor(final long handle) throws RocksDBException; + @Override protected final native void disposeInternal(final long handle); + + private final RocksDB rocksDB_; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ColumnFamilyMetaData.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ColumnFamilyMetaData.java new file mode 100644 index 0000000..1919040 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ColumnFamilyMetaData.java @@ -0,0 +1,70 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Arrays; +import java.util.List; + +/** + * The metadata that describes a column family. + */ +public class ColumnFamilyMetaData { + private final long size; + private final long fileCount; + private final byte[] name; + private final LevelMetaData[] levels; + + /** + * Called from JNI C++ + */ + private ColumnFamilyMetaData( + final long size, + final long fileCount, + final byte[] name, + final LevelMetaData[] levels) { + this.size = size; + this.fileCount = fileCount; + this.name = name; + this.levels = levels; + } + + /** + * The size of this column family in bytes, which is equal to the sum of + * the file size of its {@link #levels()}. + * + * @return the size of this column family + */ + public long size() { + return size; + } + + /** + * The number of files in this column family. + * + * @return the number of files + */ + public long fileCount() { + return fileCount; + } + + /** + * The name of the column family. + * + * @return the name + */ + public byte[] name() { + return name; + } + + /** + * The metadata of all levels in this column family. + * + * @return the levels metadata + */ + public List levels() { + return Arrays.asList(levels); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java new file mode 100644 index 0000000..8274ebe --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java @@ -0,0 +1,1556 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.nio.file.Paths; +import java.util.*; + +/** + * ColumnFamilyOptions to control the behavior of a database. It will be used + * during the creation of a {@link org.rocksdb.RocksDB} (i.e., RocksDB.open()). + *

    + * As a descendant of {@link AbstractNativeReference}, this class is {@link AutoCloseable} + * and will be automatically released if opened in the preamble of a try with resources block. + */ +public class ColumnFamilyOptions extends RocksObject + implements ColumnFamilyOptionsInterface, + MutableColumnFamilyOptionsInterface { + static { + RocksDB.loadLibrary(); + } + + /** + * Construct ColumnFamilyOptions. + *

    + * This constructor will create (by allocating a block of memory) + * an {@code rocksdb::ColumnFamilyOptions} in the c++ side. + */ + public ColumnFamilyOptions() { + super(newColumnFamilyOptions()); + } + + /** + * Copy constructor for ColumnFamilyOptions. + *

    + * NOTE: This does a shallow copy, which means comparator, merge_operator, compaction_filter, + * compaction_filter_factory and other pointers will be cloned! + * + * @param other The ColumnFamilyOptions to copy. + */ + public ColumnFamilyOptions(final ColumnFamilyOptions other) { + super(copyColumnFamilyOptions(other.nativeHandle_)); + this.memTableConfig_ = other.memTableConfig_; + this.tableFormatConfig_ = other.tableFormatConfig_; + this.comparator_ = other.comparator_; + this.compactionFilter_ = other.compactionFilter_; + this.compactionFilterFactory_ = other.compactionFilterFactory_; + this.compactionOptionsUniversal_ = other.compactionOptionsUniversal_; + this.compactionOptionsFIFO_ = other.compactionOptionsFIFO_; + this.bottommostCompressionOptions_ = other.bottommostCompressionOptions_; + this.compressionOptions_ = other.compressionOptions_; + this.compactionThreadLimiter_ = other.compactionThreadLimiter_; + this.sstPartitionerFactory_ = other.sstPartitionerFactory_; + } + + /** + * Constructor from Options + * + * @param options The options. + */ + public ColumnFamilyOptions(final Options options) { + super(newColumnFamilyOptionsFromOptions(options.nativeHandle_)); + } + + /** + *

    Constructor to be used by + * {@link #getColumnFamilyOptionsFromProps(java.util.Properties)}, + * {@link ColumnFamilyDescriptor#getOptions()} + * and also called via JNI.

    + * + * @param handle native handle to ColumnFamilyOptions instance. + */ + ColumnFamilyOptions(final long handle) { + super(handle); + } + + /** + *

    Method to get a options instance by using pre-configured + * property values. If one or many values are undefined in + * the context of RocksDB the method will return a null + * value.

    + * + *

    Note: Property keys can be derived from + * getter methods within the options class. Example: the method + * {@code writeBufferSize()} has a property key: + * {@code write_buffer_size}.

    + * + * @param properties {@link java.util.Properties} instance. + * + * @return {@link org.rocksdb.ColumnFamilyOptions instance} + * or null. + * + * @throws java.lang.IllegalArgumentException if null or empty + * {@link Properties} instance is passed to the method call. + */ + public static ColumnFamilyOptions getColumnFamilyOptionsFromProps( + final Properties properties) { + ColumnFamilyOptions columnFamilyOptions = null; + final long handle = + getColumnFamilyOptionsFromProps(Options.getOptionStringFromProps(properties)); + if (handle != 0) { + columnFamilyOptions = new ColumnFamilyOptions(handle); + } + return columnFamilyOptions; + } + + /** + *

    Method to get a options instance by using pre-configured + * property values. If one or many values are undefined in + * the context of RocksDB the method will return a null + * value.

    + * + *

    Note: Property keys can be derived from + * getter methods within the options class. Example: the method + * {@code writeBufferSize()} has a property key: + * {@code write_buffer_size}.

    + * + * @param cfgOpts ConfigOptions controlling how the properties are parsed. + * @param properties {@link java.util.Properties} instance. + * + * @return {@link org.rocksdb.ColumnFamilyOptions instance} + * or null. + * + * @throws java.lang.IllegalArgumentException if null or empty + * {@link Properties} instance is passed to the method call. + */ + public static ColumnFamilyOptions getColumnFamilyOptionsFromProps( + final ConfigOptions cfgOpts, final Properties properties) { + ColumnFamilyOptions columnFamilyOptions = null; + final long handle = getColumnFamilyOptionsFromProps( + cfgOpts.nativeHandle_, Options.getOptionStringFromProps(properties)); + if (handle != 0){ + columnFamilyOptions = new ColumnFamilyOptions(handle); + } + return columnFamilyOptions; + } + + @Override + public ColumnFamilyOptions oldDefaults(final int majorVersion, final int minorVersion) { + oldDefaults(nativeHandle_, majorVersion, minorVersion); + return this; + } + + @Override + public ColumnFamilyOptions optimizeForSmallDb() { + optimizeForSmallDb(nativeHandle_); + return this; + } + + @Override + public ColumnFamilyOptions optimizeForSmallDb(final Cache cache) { + optimizeForSmallDb(nativeHandle_, cache.getNativeHandle()); + return this; + } + + @Override + public ColumnFamilyOptions optimizeForPointLookup( + final long blockCacheSizeMb) { + optimizeForPointLookup(nativeHandle_, + blockCacheSizeMb); + return this; + } + + @Override + public ColumnFamilyOptions optimizeLevelStyleCompaction() { + optimizeLevelStyleCompaction(nativeHandle_, + DEFAULT_COMPACTION_MEMTABLE_MEMORY_BUDGET); + return this; + } + + @Override + public ColumnFamilyOptions optimizeLevelStyleCompaction( + final long memtableMemoryBudget) { + optimizeLevelStyleCompaction(nativeHandle_, + memtableMemoryBudget); + return this; + } + + @Override + public ColumnFamilyOptions optimizeUniversalStyleCompaction() { + optimizeUniversalStyleCompaction(nativeHandle_, + DEFAULT_COMPACTION_MEMTABLE_MEMORY_BUDGET); + return this; + } + + @Override + public ColumnFamilyOptions optimizeUniversalStyleCompaction( + final long memtableMemoryBudget) { + optimizeUniversalStyleCompaction(nativeHandle_, + memtableMemoryBudget); + return this; + } + + @Override + public ColumnFamilyOptions setComparator( + final BuiltinComparator builtinComparator) { + assert(isOwningHandle()); + setComparatorHandle(nativeHandle_, builtinComparator.ordinal()); + return this; + } + + @Override + public ColumnFamilyOptions setComparator( + final AbstractComparator comparator) { + assert (isOwningHandle()); + setComparatorHandle(nativeHandle_, comparator.nativeHandle_, + comparator.getComparatorType().getValue()); + comparator_ = comparator; + return this; + } + + @Override + public ColumnFamilyOptions setMergeOperatorName(final String name) { + assert (isOwningHandle()); + if (name == null) { + throw new IllegalArgumentException( + "Merge operator name must not be null."); + } + setMergeOperatorName(nativeHandle_, name); + return this; + } + + @Override + public ColumnFamilyOptions setMergeOperator( + final MergeOperator mergeOperator) { + setMergeOperator(nativeHandle_, mergeOperator.nativeHandle_); + return this; + } + + @Override + public ColumnFamilyOptions setCompactionFilter( + final AbstractCompactionFilter> + compactionFilter) { + setCompactionFilterHandle(nativeHandle_, compactionFilter.nativeHandle_); + compactionFilter_ = compactionFilter; + return this; + } + + @Override + public AbstractCompactionFilter> compactionFilter() { + assert (isOwningHandle()); + return compactionFilter_; + } + + @Override + public ColumnFamilyOptions setCompactionFilterFactory(final AbstractCompactionFilterFactory> compactionFilterFactory) { + assert (isOwningHandle()); + setCompactionFilterFactoryHandle(nativeHandle_, compactionFilterFactory.nativeHandle_); + compactionFilterFactory_ = compactionFilterFactory; + return this; + } + + @Override + public AbstractCompactionFilterFactory> compactionFilterFactory() { + assert (isOwningHandle()); + return compactionFilterFactory_; + } + + @Override + public ColumnFamilyOptions setWriteBufferSize(final long writeBufferSize) { + assert(isOwningHandle()); + setWriteBufferSize(nativeHandle_, writeBufferSize); + return this; + } + + @Override + public long writeBufferSize() { + assert(isOwningHandle()); + return writeBufferSize(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setMaxWriteBufferNumber( + final int maxWriteBufferNumber) { + assert(isOwningHandle()); + setMaxWriteBufferNumber(nativeHandle_, maxWriteBufferNumber); + return this; + } + + @Override + public int maxWriteBufferNumber() { + assert(isOwningHandle()); + return maxWriteBufferNumber(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setMinWriteBufferNumberToMerge( + final int minWriteBufferNumberToMerge) { + setMinWriteBufferNumberToMerge(nativeHandle_, minWriteBufferNumberToMerge); + return this; + } + + @Override + public int minWriteBufferNumberToMerge() { + return minWriteBufferNumberToMerge(nativeHandle_); + } + + @Override + public ColumnFamilyOptions useFixedLengthPrefixExtractor(final int n) { + assert(isOwningHandle()); + useFixedLengthPrefixExtractor(nativeHandle_, n); + return this; + } + + @Override + public ColumnFamilyOptions useCappedPrefixExtractor(final int n) { + assert(isOwningHandle()); + useCappedPrefixExtractor(nativeHandle_, n); + return this; + } + + @Override + public ColumnFamilyOptions setCompressionType( + final CompressionType compressionType) { + setCompressionType(nativeHandle_, compressionType.getValue()); + return this; + } + + @Override + public CompressionType compressionType() { + return CompressionType.getCompressionType(compressionType(nativeHandle_)); + } + + @Override + public ColumnFamilyOptions setCompressionPerLevel( + final List compressionLevels) { + final byte[] byteCompressionTypes = new byte[ + compressionLevels.size()]; + for (int i = 0; i < compressionLevels.size(); i++) { + byteCompressionTypes[i] = compressionLevels.get(i).getValue(); + } + setCompressionPerLevel(nativeHandle_, byteCompressionTypes); + return this; + } + + @Override + public List compressionPerLevel() { + final byte[] byteCompressionTypes = + compressionPerLevel(nativeHandle_); + final List compressionLevels = new ArrayList<>(); + for (final byte byteCompressionType : byteCompressionTypes) { + compressionLevels.add(CompressionType.getCompressionType( + byteCompressionType)); + } + return compressionLevels; + } + + @Override + public ColumnFamilyOptions setBottommostCompressionType( + final CompressionType bottommostCompressionType) { + setBottommostCompressionType(nativeHandle_, + bottommostCompressionType.getValue()); + return this; + } + + @Override + public CompressionType bottommostCompressionType() { + return CompressionType.getCompressionType( + bottommostCompressionType(nativeHandle_)); + } + + @Override + public ColumnFamilyOptions setBottommostCompressionOptions( + final CompressionOptions bottommostCompressionOptions) { + setBottommostCompressionOptions(nativeHandle_, + bottommostCompressionOptions.nativeHandle_); + this.bottommostCompressionOptions_ = bottommostCompressionOptions; + return this; + } + + @Override + public CompressionOptions bottommostCompressionOptions() { + return this.bottommostCompressionOptions_; + } + + @Override + public ColumnFamilyOptions setCompressionOptions( + final CompressionOptions compressionOptions) { + setCompressionOptions(nativeHandle_, compressionOptions.nativeHandle_); + this.compressionOptions_ = compressionOptions; + return this; + } + + @Override + public CompressionOptions compressionOptions() { + return this.compressionOptions_; + } + + @Override + public ColumnFamilyOptions setNumLevels(final int numLevels) { + setNumLevels(nativeHandle_, numLevels); + return this; + } + + @Override + public int numLevels() { + return numLevels(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setLevelZeroFileNumCompactionTrigger( + final int numFiles) { + setLevelZeroFileNumCompactionTrigger( + nativeHandle_, numFiles); + return this; + } + + @Override + public int levelZeroFileNumCompactionTrigger() { + return levelZeroFileNumCompactionTrigger(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setLevelZeroSlowdownWritesTrigger( + final int numFiles) { + setLevelZeroSlowdownWritesTrigger(nativeHandle_, numFiles); + return this; + } + + @Override + public int levelZeroSlowdownWritesTrigger() { + return levelZeroSlowdownWritesTrigger(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setLevelZeroStopWritesTrigger(final int numFiles) { + setLevelZeroStopWritesTrigger(nativeHandle_, numFiles); + return this; + } + + @Override + public int levelZeroStopWritesTrigger() { + return levelZeroStopWritesTrigger(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setTargetFileSizeBase( + final long targetFileSizeBase) { + setTargetFileSizeBase(nativeHandle_, targetFileSizeBase); + return this; + } + + @Override + public long targetFileSizeBase() { + return targetFileSizeBase(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setTargetFileSizeMultiplier( + final int multiplier) { + setTargetFileSizeMultiplier(nativeHandle_, multiplier); + return this; + } + + @Override + public int targetFileSizeMultiplier() { + return targetFileSizeMultiplier(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setMaxBytesForLevelBase( + final long maxBytesForLevelBase) { + setMaxBytesForLevelBase(nativeHandle_, maxBytesForLevelBase); + return this; + } + + @Override + public long maxBytesForLevelBase() { + return maxBytesForLevelBase(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setLevelCompactionDynamicLevelBytes( + final boolean enableLevelCompactionDynamicLevelBytes) { + setLevelCompactionDynamicLevelBytes(nativeHandle_, + enableLevelCompactionDynamicLevelBytes); + return this; + } + + @Override + public boolean levelCompactionDynamicLevelBytes() { + return levelCompactionDynamicLevelBytes(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setMaxBytesForLevelMultiplier(final double multiplier) { + setMaxBytesForLevelMultiplier(nativeHandle_, multiplier); + return this; + } + + @Override + public double maxBytesForLevelMultiplier() { + return maxBytesForLevelMultiplier(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setMaxCompactionBytes(final long maxCompactionBytes) { + setMaxCompactionBytes(nativeHandle_, maxCompactionBytes); + return this; + } + + @Override + public long maxCompactionBytes() { + return maxCompactionBytes(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setArenaBlockSize( + final long arenaBlockSize) { + setArenaBlockSize(nativeHandle_, arenaBlockSize); + return this; + } + + @Override + public long arenaBlockSize() { + return arenaBlockSize(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setDisableAutoCompactions( + final boolean disableAutoCompactions) { + setDisableAutoCompactions(nativeHandle_, disableAutoCompactions); + return this; + } + + @Override + public boolean disableAutoCompactions() { + return disableAutoCompactions(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setCompactionStyle( + final CompactionStyle compactionStyle) { + setCompactionStyle(nativeHandle_, compactionStyle.getValue()); + return this; + } + + @Override + public CompactionStyle compactionStyle() { + return CompactionStyle.fromValue(compactionStyle(nativeHandle_)); + } + + @Override + public ColumnFamilyOptions setMaxTableFilesSizeFIFO( + final long maxTableFilesSize) { + assert(maxTableFilesSize > 0); // unsigned native type + assert(isOwningHandle()); + setMaxTableFilesSizeFIFO(nativeHandle_, maxTableFilesSize); + return this; + } + + @Override + public long maxTableFilesSizeFIFO() { + return maxTableFilesSizeFIFO(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setMaxSequentialSkipInIterations( + final long maxSequentialSkipInIterations) { + setMaxSequentialSkipInIterations(nativeHandle_, + maxSequentialSkipInIterations); + return this; + } + + @Override + public long maxSequentialSkipInIterations() { + return maxSequentialSkipInIterations(nativeHandle_); + } + + @Override + public MemTableConfig memTableConfig() { + return this.memTableConfig_; + } + + @Override + public ColumnFamilyOptions setMemTableConfig( + final MemTableConfig memTableConfig) { + setMemTableFactory( + nativeHandle_, memTableConfig.newMemTableFactoryHandle()); + this.memTableConfig_ = memTableConfig; + return this; + } + + @Override + public String memTableFactoryName() { + assert(isOwningHandle()); + return memTableFactoryName(nativeHandle_); + } + + @Override + public TableFormatConfig tableFormatConfig() { + return this.tableFormatConfig_; + } + + @Override + public ColumnFamilyOptions setTableFormatConfig( + final TableFormatConfig tableFormatConfig) { + setTableFactory(nativeHandle_, tableFormatConfig.newTableFactoryHandle()); + this.tableFormatConfig_ = tableFormatConfig; + return this; + } + + @Override + public String tableFactoryName() { + assert(isOwningHandle()); + return tableFactoryName(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setCfPaths(final Collection cfPaths) { + assert (isOwningHandle()); + + final int len = cfPaths.size(); + final String[] paths = new String[len]; + final long[] targetSizes = new long[len]; + + int i = 0; + for (final DbPath dbPath : cfPaths) { + paths[i] = dbPath.path.toString(); + targetSizes[i] = dbPath.targetSize; + i++; + } + setCfPaths(nativeHandle_, paths, targetSizes); + return this; + } + + @Override + public List cfPaths() { + final int len = (int) cfPathsLen(nativeHandle_); + + if (len == 0) { + return Collections.emptyList(); + } + + final String[] paths = new String[len]; + final long[] targetSizes = new long[len]; + + cfPaths(nativeHandle_, paths, targetSizes); + + final List cfPaths = new ArrayList<>(); + for (int i = 0; i < len; i++) { + cfPaths.add(new DbPath(Paths.get(paths[i]), targetSizes[i])); + } + + return cfPaths; + } + + @Override + public ColumnFamilyOptions setInplaceUpdateSupport( + final boolean inplaceUpdateSupport) { + setInplaceUpdateSupport(nativeHandle_, inplaceUpdateSupport); + return this; + } + + @Override + public boolean inplaceUpdateSupport() { + return inplaceUpdateSupport(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setInplaceUpdateNumLocks( + final long inplaceUpdateNumLocks) { + setInplaceUpdateNumLocks(nativeHandle_, inplaceUpdateNumLocks); + return this; + } + + @Override + public long inplaceUpdateNumLocks() { + return inplaceUpdateNumLocks(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setMemtablePrefixBloomSizeRatio( + final double memtablePrefixBloomSizeRatio) { + setMemtablePrefixBloomSizeRatio(nativeHandle_, memtablePrefixBloomSizeRatio); + return this; + } + + @Override + public double memtablePrefixBloomSizeRatio() { + return memtablePrefixBloomSizeRatio(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setExperimentalMempurgeThreshold( + final double experimentalMempurgeThreshold) { + setExperimentalMempurgeThreshold(nativeHandle_, experimentalMempurgeThreshold); + return this; + } + + @Override + public double experimentalMempurgeThreshold() { + return experimentalMempurgeThreshold(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setMemtableWholeKeyFiltering(final boolean memtableWholeKeyFiltering) { + setMemtableWholeKeyFiltering(nativeHandle_, memtableWholeKeyFiltering); + return this; + } + + @Override + public boolean memtableWholeKeyFiltering() { + return memtableWholeKeyFiltering(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setBloomLocality(final int bloomLocality) { + setBloomLocality(nativeHandle_, bloomLocality); + return this; + } + + @Override + public int bloomLocality() { + return bloomLocality(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setMaxSuccessiveMerges( + final long maxSuccessiveMerges) { + setMaxSuccessiveMerges(nativeHandle_, maxSuccessiveMerges); + return this; + } + + @Override + public long maxSuccessiveMerges() { + return maxSuccessiveMerges(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setOptimizeFiltersForHits( + final boolean optimizeFiltersForHits) { + setOptimizeFiltersForHits(nativeHandle_, optimizeFiltersForHits); + return this; + } + + @Override + public boolean optimizeFiltersForHits() { + return optimizeFiltersForHits(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setMemtableHugePageSize(final long memtableHugePageSize) { + setMemtableHugePageSize(nativeHandle_, + memtableHugePageSize); + return this; + } + + @Override + public long memtableHugePageSize() { + return memtableHugePageSize(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setSoftPendingCompactionBytesLimit( + final long softPendingCompactionBytesLimit) { + setSoftPendingCompactionBytesLimit(nativeHandle_, + softPendingCompactionBytesLimit); + return this; + } + + @Override + public long softPendingCompactionBytesLimit() { + return softPendingCompactionBytesLimit(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setHardPendingCompactionBytesLimit( + final long hardPendingCompactionBytesLimit) { + setHardPendingCompactionBytesLimit(nativeHandle_, hardPendingCompactionBytesLimit); + return this; + } + + @Override + public long hardPendingCompactionBytesLimit() { + return hardPendingCompactionBytesLimit(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setLevel0FileNumCompactionTrigger( + final int level0FileNumCompactionTrigger) { + setLevel0FileNumCompactionTrigger(nativeHandle_, level0FileNumCompactionTrigger); + return this; + } + + @Override + public int level0FileNumCompactionTrigger() { + return level0FileNumCompactionTrigger(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setLevel0SlowdownWritesTrigger(final int level0SlowdownWritesTrigger) { + setLevel0SlowdownWritesTrigger(nativeHandle_, level0SlowdownWritesTrigger); + return this; + } + + @Override + public int level0SlowdownWritesTrigger() { + return level0SlowdownWritesTrigger(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setLevel0StopWritesTrigger(final int level0StopWritesTrigger) { + setLevel0StopWritesTrigger(nativeHandle_, level0StopWritesTrigger); + return this; + } + + @Override + public int level0StopWritesTrigger() { + return level0StopWritesTrigger(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setMaxBytesForLevelMultiplierAdditional( + final int[] maxBytesForLevelMultiplierAdditional) { + setMaxBytesForLevelMultiplierAdditional(nativeHandle_, maxBytesForLevelMultiplierAdditional); + return this; + } + + @Override + public int[] maxBytesForLevelMultiplierAdditional() { + return maxBytesForLevelMultiplierAdditional(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setParanoidFileChecks(final boolean paranoidFileChecks) { + setParanoidFileChecks(nativeHandle_, paranoidFileChecks); + return this; + } + + @Override + public boolean paranoidFileChecks() { + return paranoidFileChecks(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setMaxWriteBufferNumberToMaintain( + final int maxWriteBufferNumberToMaintain) { + setMaxWriteBufferNumberToMaintain( + nativeHandle_, maxWriteBufferNumberToMaintain); + return this; + } + + @Override + public int maxWriteBufferNumberToMaintain() { + return maxWriteBufferNumberToMaintain(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setCompactionPriority( + final CompactionPriority compactionPriority) { + setCompactionPriority(nativeHandle_, compactionPriority.getValue()); + return this; + } + + @Override + public CompactionPriority compactionPriority() { + return CompactionPriority.getCompactionPriority( + compactionPriority(nativeHandle_)); + } + + @Override + public ColumnFamilyOptions setReportBgIoStats(final boolean reportBgIoStats) { + setReportBgIoStats(nativeHandle_, reportBgIoStats); + return this; + } + + @Override + public boolean reportBgIoStats() { + return reportBgIoStats(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setTtl(final long ttl) { + setTtl(nativeHandle_, ttl); + return this; + } + + @Override + public long ttl() { + return ttl(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setPeriodicCompactionSeconds(final long periodicCompactionSeconds) { + setPeriodicCompactionSeconds(nativeHandle_, periodicCompactionSeconds); + return this; + } + + @Override + public long periodicCompactionSeconds() { + return periodicCompactionSeconds(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setCompactionOptionsUniversal( + final CompactionOptionsUniversal compactionOptionsUniversal) { + setCompactionOptionsUniversal(nativeHandle_, + compactionOptionsUniversal.nativeHandle_); + this.compactionOptionsUniversal_ = compactionOptionsUniversal; + return this; + } + + @Override + public CompactionOptionsUniversal compactionOptionsUniversal() { + return this.compactionOptionsUniversal_; + } + + @Override + public ColumnFamilyOptions setCompactionOptionsFIFO(final CompactionOptionsFIFO compactionOptionsFIFO) { + setCompactionOptionsFIFO(nativeHandle_, + compactionOptionsFIFO.nativeHandle_); + this.compactionOptionsFIFO_ = compactionOptionsFIFO; + return this; + } + + @Override + public CompactionOptionsFIFO compactionOptionsFIFO() { + return this.compactionOptionsFIFO_; + } + + @Override + public ColumnFamilyOptions setForceConsistencyChecks(final boolean forceConsistencyChecks) { + setForceConsistencyChecks(nativeHandle_, forceConsistencyChecks); + return this; + } + + @Override + public boolean forceConsistencyChecks() { + return forceConsistencyChecks(nativeHandle_); + } + + @Override + public ColumnFamilyOptions setSstPartitionerFactory( + final SstPartitionerFactory sstPartitionerFactory) { + setSstPartitionerFactory(nativeHandle_, sstPartitionerFactory.nativeHandle_); + this.sstPartitionerFactory_ = sstPartitionerFactory; + return this; + } + + @Override + public ColumnFamilyOptions setCompactionThreadLimiter( + final ConcurrentTaskLimiter compactionThreadLimiter) { + setCompactionThreadLimiter(nativeHandle_, compactionThreadLimiter.nativeHandle_); + this.compactionThreadLimiter_ = compactionThreadLimiter; + return this; + } + + @Override + public ConcurrentTaskLimiter compactionThreadLimiter() { + assert (isOwningHandle()); + return this.compactionThreadLimiter_; + } + + @Override + public SstPartitionerFactory sstPartitionerFactory() { + return sstPartitionerFactory_; + } + + @Override + public ColumnFamilyOptions setMemtableMaxRangeDeletions(final int count) { + setMemtableMaxRangeDeletions(nativeHandle_, count); + return this; + } + + @Override + public int memtableMaxRangeDeletions() { + return memtableMaxRangeDeletions(nativeHandle_); + } + + // + // BEGIN options for blobs (integrated BlobDB) + // + + /** + * When set, large values (blobs) are written to separate blob files, and only + * pointers to them are stored in SST files. This can reduce write amplification + * for large-value use cases at the cost of introducing a level of indirection + * for reads. See also the options min_blob_size, blob_file_size, + * blob_compression_type, enable_blob_garbage_collection, and + * blob_garbage_collection_age_cutoff below. + *

    + * Default: false + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @param enableBlobFiles true iff blob files should be enabled + * + * @return the reference to the current options. + */ + @Override + public ColumnFamilyOptions setEnableBlobFiles(final boolean enableBlobFiles) { + setEnableBlobFiles(nativeHandle_, enableBlobFiles); + return this; + } + + /** + * When set, large values (blobs) are written to separate blob files, and only + * pointers to them are stored in SST files. This can reduce write amplification + * for large-value use cases at the cost of introducing a level of indirection + * for reads. See also the options min_blob_size, blob_file_size, + * blob_compression_type, enable_blob_garbage_collection, and + * blob_garbage_collection_age_cutoff below. + *

    + * Default: false + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @return true iff blob files are currently enabled + */ + public boolean enableBlobFiles() { + return enableBlobFiles(nativeHandle_); + } + + /** + * Set the size of the smallest value to be stored separately in a blob file. Values + * which have an uncompressed size smaller than this threshold are stored + * alongside the keys in SST files in the usual fashion. A value of zero for + * this option means that all values are stored in blob files. Note that + * enable_blob_files has to be set in order for this option to have any effect. + *

    + * Default: 0 + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @param minBlobSize the size of the smallest value to be stored separately in a blob file + * @return these options, updated with the supplied minimum blob size value + */ + @Override + public ColumnFamilyOptions setMinBlobSize(final long minBlobSize) { + setMinBlobSize(nativeHandle_, minBlobSize); + return this; + } + + /** + * Get the size of the smallest value to be stored separately in a blob file. Values + * which have an uncompressed size smaller than this threshold are stored + * alongside the keys in SST files in the usual fashion. A value of zero for + * this option means that all values are stored in blob files. Note that + * enable_blob_files has to be set in order for this option to have any effect. + *

    + * Default: 0 + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @return the current minimum blob size + */ + @Override + public long minBlobSize() { + return minBlobSize(nativeHandle_); + } + + /** + * Set the size limit for blob files. When writing blob files, a new file is opened + * once this limit is reached. Note that enable_blob_files has to be set in + * order for this option to have any effect. + *

    + * Default: 256 MB + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @param blobFileSize the new size limit for blob files + * + * @return the reference to the current options. + */ + @Override + public ColumnFamilyOptions setBlobFileSize(final long blobFileSize) { + setBlobFileSize(nativeHandle_, blobFileSize); + return this; + } + + /** + * Get the size limit for blob files. When writing blob files, a new file is opened + * once this limit is reached. Note that enable_blob_files has to be set in + * order for this option to have any effect. + *

    + * Default: 256 MB + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @return the size limit for blob files + */ + @Override + public long blobFileSize() { + return blobFileSize(nativeHandle_); + } + + /** + * Set the compression algorithm to use for large values stored in blob files. Note + * that enable_blob_files has to be set in order for this option to have any + * effect. + *

    + * Default: no compression + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @param compressionType the compression algorithm to use + * + * @return the reference to the current options. + */ + @Override + public ColumnFamilyOptions setBlobCompressionType(final CompressionType compressionType) { + setBlobCompressionType(nativeHandle_, compressionType.getValue()); + return this; + } + + /** + * Get the compression algorithm to use for large values stored in blob files. Note + * that enable_blob_files has to be set in order for this option to have any + * effect. + *

    + * Default: no compression + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @return the compression algorithm currently in use for blobs + */ + @Override + public CompressionType blobCompressionType() { + return CompressionType.values()[blobCompressionType(nativeHandle_)]; + } + + /** + * Enable/disable garbage collection of blobs. Blob GC is performed as part of + * compaction. Valid blobs residing in blob files older than a cutoff get + * relocated to new files as they are encountered during compaction, which makes + * it possible to clean up blob files once they contain nothing but + * obsolete/garbage blobs. See also blob_garbage_collection_age_cutoff below. + *

    + * Default: false + * + * @param enableBlobGarbageCollection true iff blob garbage collection is to be enabled + * + * @return the reference to the current options. + */ + @Override + public ColumnFamilyOptions setEnableBlobGarbageCollection( + final boolean enableBlobGarbageCollection) { + setEnableBlobGarbageCollection(nativeHandle_, enableBlobGarbageCollection); + return this; + } + + /** + * Get enabled/disables state for garbage collection of blobs. Blob GC is performed as part of + * compaction. Valid blobs residing in blob files older than a cutoff get + * relocated to new files as they are encountered during compaction, which makes + * it possible to clean up blob files once they contain nothing but + * obsolete/garbage blobs. See also blob_garbage_collection_age_cutoff below. + *

    + * Default: false + * + * @return true iff blob garbage collection is currently enabled + */ + @Override + public boolean enableBlobGarbageCollection() { + return enableBlobGarbageCollection(nativeHandle_); + } + + /** + * Set the cutoff in terms of blob file age for garbage collection. Blobs in the + * oldest N blob files will be relocated when encountered during compaction, + * where N = garbage_collection_cutoff * number_of_blob_files. Note that + * enable_blob_garbage_collection has to be set in order for this option to have + * any effect. + *

    + * Default: 0.25 + * + * @param blobGarbageCollectionAgeCutoff the new blob garbage collection age cutoff + * + * @return the reference to the current options. + */ + @Override + public ColumnFamilyOptions setBlobGarbageCollectionAgeCutoff( + final double blobGarbageCollectionAgeCutoff) { + setBlobGarbageCollectionAgeCutoff(nativeHandle_, blobGarbageCollectionAgeCutoff); + return this; + } + + /** + * Get the cutoff in terms of blob file age for garbage collection. Blobs in the + * oldest N blob files will be relocated when encountered during compaction, + * where N = garbage_collection_cutoff * number_of_blob_files. Note that + * enable_blob_garbage_collection has to be set in order for this option to have + * any effect. + *

    + * Default: 0.25 + * + * @return the current blob garbage collection age cutoff + */ + @Override + public double blobGarbageCollectionAgeCutoff() { + return blobGarbageCollectionAgeCutoff(nativeHandle_); + } + + /** + * If the ratio of garbage in the oldest blob files exceeds this threshold, + * targeted compactions are scheduled in order to force garbage collecting + * the blob files in question, assuming they are all eligible based on the + * value of {@link #blobGarbageCollectionAgeCutoff} above. This option is + * currently only supported with leveled compactions. + *

    + * Note that {@link #enableBlobGarbageCollection} has to be set in order for this + * option to have any effect. + *

    + * Default: 1.0 + *

    + * Dynamically changeable through the SetOptions() API + * + * @param blobGarbageCollectionForceThreshold new value for the threshold + * @return the reference to the current options + */ + @Override + public ColumnFamilyOptions setBlobGarbageCollectionForceThreshold( + final double blobGarbageCollectionForceThreshold) { + setBlobGarbageCollectionForceThreshold(nativeHandle_, blobGarbageCollectionForceThreshold); + return this; + } + + /** + * Get the current value for the {@link #blobGarbageCollectionForceThreshold} + * @return the current threshold at which garbage collection of blobs is forced + */ + @Override + public double blobGarbageCollectionForceThreshold() { + return blobGarbageCollectionForceThreshold(nativeHandle_); + } + + /** + * Set compaction readahead for blob files. + *

    + * Default: 0 + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @param blobCompactionReadaheadSize the compaction readahead for blob files + * + * @return the reference to the current options. + */ + @Override + public ColumnFamilyOptions setBlobCompactionReadaheadSize( + final long blobCompactionReadaheadSize) { + setBlobCompactionReadaheadSize(nativeHandle_, blobCompactionReadaheadSize); + return this; + } + + /** + * Get compaction readahead for blob files. + * + * @return the current compaction readahead for blob files + */ + @Override + public long blobCompactionReadaheadSize() { + return blobCompactionReadaheadSize(nativeHandle_); + } + + /** + * Set a certain LSM tree level to enable blob files. + *

    + * Default: 0 + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @param blobFileStartingLevel the starting level to enable blob files + * + * @return the reference to the current options. + */ + @Override + public ColumnFamilyOptions setBlobFileStartingLevel(final int blobFileStartingLevel) { + setBlobFileStartingLevel(nativeHandle_, blobFileStartingLevel); + return this; + } + + /** + * Get the starting LSM tree level to enable blob files. + *

    + * Default: 0 + * + * @return the current LSM tree level to enable blob files. + */ + @Override + public int blobFileStartingLevel() { + return blobFileStartingLevel(nativeHandle_); + } + + /** + * Set a certain prepopulate blob cache option. + *

    + * Default: 0 + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @param prepopulateBlobCache prepopulate the blob cache option + * + * @return the reference to the current options. + */ + @Override + public ColumnFamilyOptions setPrepopulateBlobCache( + final PrepopulateBlobCache prepopulateBlobCache) { + setPrepopulateBlobCache(nativeHandle_, prepopulateBlobCache.getValue()); + return this; + } + + /** + * Get the prepopulate blob cache option. + *

    + * Default: 0 + * + * @return the current prepopulate blob cache option. + */ + @Override + public PrepopulateBlobCache prepopulateBlobCache() { + return PrepopulateBlobCache.getPrepopulateBlobCache(prepopulateBlobCache(nativeHandle_)); + } + + // + // END options for blobs (integrated BlobDB) + // + + private static native long getColumnFamilyOptionsFromProps( + final long cfgHandle, String optString); + private static native long getColumnFamilyOptionsFromProps(final String optString); + + private static native long newColumnFamilyOptions(); + private static native long copyColumnFamilyOptions(final long handle); + private static native long newColumnFamilyOptionsFromOptions( + final long optionsHandle); + @Override protected final native void disposeInternal(final long handle); + + private static native void oldDefaults( + final long handle, final int majorVersion, final int minorVersion); + private native void optimizeForSmallDb(final long handle); + private static native void optimizeForSmallDb(final long handle, final long cacheHandle); + private native void optimizeForPointLookup(long handle, + long blockCacheSizeMb); + private native void optimizeLevelStyleCompaction(long handle, + long memtableMemoryBudget); + private native void optimizeUniversalStyleCompaction(long handle, + long memtableMemoryBudget); + private native void setComparatorHandle(long handle, int builtinComparator); + private native void setComparatorHandle(long optHandle, + long comparatorHandle, byte comparatorType); + private native void setMergeOperatorName(long handle, String name); + private native void setMergeOperator(long handle, long mergeOperatorHandle); + private native void setCompactionFilterHandle(long handle, + long compactionFilterHandle); + private native void setCompactionFilterFactoryHandle(long handle, + long compactionFilterFactoryHandle); + private native void setWriteBufferSize(long handle, long writeBufferSize) + throws IllegalArgumentException; + private native long writeBufferSize(long handle); + private native void setMaxWriteBufferNumber( + long handle, int maxWriteBufferNumber); + private native int maxWriteBufferNumber(long handle); + private native void setMinWriteBufferNumberToMerge( + long handle, int minWriteBufferNumberToMerge); + private native int minWriteBufferNumberToMerge(long handle); + private native void setCompressionType(long handle, byte compressionType); + private native byte compressionType(long handle); + private native void setCompressionPerLevel(long handle, + byte[] compressionLevels); + private native byte[] compressionPerLevel(long handle); + private native void setBottommostCompressionType(long handle, + byte bottommostCompressionType); + private native byte bottommostCompressionType(long handle); + private native void setBottommostCompressionOptions(final long handle, + final long bottommostCompressionOptionsHandle); + private native void setCompressionOptions(long handle, + long compressionOptionsHandle); + private native void useFixedLengthPrefixExtractor( + long handle, int prefixLength); + private native void useCappedPrefixExtractor( + long handle, int prefixLength); + private native void setNumLevels( + long handle, int numLevels); + private native int numLevels(long handle); + private native void setLevelZeroFileNumCompactionTrigger( + long handle, int numFiles); + private native int levelZeroFileNumCompactionTrigger(long handle); + private native void setLevelZeroSlowdownWritesTrigger( + long handle, int numFiles); + private native int levelZeroSlowdownWritesTrigger(long handle); + private native void setLevelZeroStopWritesTrigger( + long handle, int numFiles); + private native int levelZeroStopWritesTrigger(long handle); + private native void setTargetFileSizeBase( + long handle, long targetFileSizeBase); + private native long targetFileSizeBase(long handle); + private native void setTargetFileSizeMultiplier( + long handle, int multiplier); + private native int targetFileSizeMultiplier(long handle); + private native void setMaxBytesForLevelBase( + long handle, long maxBytesForLevelBase); + private native long maxBytesForLevelBase(long handle); + private native void setLevelCompactionDynamicLevelBytes( + long handle, boolean enableLevelCompactionDynamicLevelBytes); + private native boolean levelCompactionDynamicLevelBytes( + long handle); + private native void setMaxBytesForLevelMultiplier(long handle, double multiplier); + private native double maxBytesForLevelMultiplier(long handle); + private native void setMaxCompactionBytes(long handle, long maxCompactionBytes); + private native long maxCompactionBytes(long handle); + private native void setArenaBlockSize( + long handle, long arenaBlockSize) + throws IllegalArgumentException; + private native long arenaBlockSize(long handle); + private native void setDisableAutoCompactions( + long handle, boolean disableAutoCompactions); + private native boolean disableAutoCompactions(long handle); + private native void setCompactionStyle(long handle, byte compactionStyle); + private native byte compactionStyle(long handle); + private native void setMaxTableFilesSizeFIFO( + long handle, long max_table_files_size); + private native long maxTableFilesSizeFIFO(long handle); + private native void setMaxSequentialSkipInIterations( + long handle, long maxSequentialSkipInIterations); + private native long maxSequentialSkipInIterations(long handle); + private native void setMemTableFactory(long handle, long factoryHandle); + private native String memTableFactoryName(long handle); + private native void setTableFactory(long handle, long factoryHandle); + private native String tableFactoryName(long handle); + private static native void setCfPaths( + final long handle, final String[] paths, final long[] targetSizes); + private static native long cfPathsLen(final long handle); + private static native void cfPaths( + final long handle, final String[] paths, final long[] targetSizes); + private native void setInplaceUpdateSupport( + long handle, boolean inplaceUpdateSupport); + private native boolean inplaceUpdateSupport(long handle); + private native void setInplaceUpdateNumLocks( + long handle, long inplaceUpdateNumLocks) + throws IllegalArgumentException; + private native long inplaceUpdateNumLocks(long handle); + private native void setMemtablePrefixBloomSizeRatio( + long handle, double memtablePrefixBloomSizeRatio); + private native double memtablePrefixBloomSizeRatio(long handle); + private native void setExperimentalMempurgeThreshold( + long handle, double experimentalMempurgeThreshold); + private native double experimentalMempurgeThreshold(long handle); + private native void setMemtableWholeKeyFiltering(long handle, boolean memtableWholeKeyFiltering); + private native boolean memtableWholeKeyFiltering(long handle); + private native void setBloomLocality( + long handle, int bloomLocality); + private native int bloomLocality(long handle); + private native void setMaxSuccessiveMerges( + long handle, long maxSuccessiveMerges) + throws IllegalArgumentException; + private native long maxSuccessiveMerges(long handle); + private native void setOptimizeFiltersForHits(long handle, + boolean optimizeFiltersForHits); + private native boolean optimizeFiltersForHits(long handle); + private native void setMemtableHugePageSize(long handle, + long memtableHugePageSize); + private native long memtableHugePageSize(long handle); + private native void setSoftPendingCompactionBytesLimit(long handle, + long softPendingCompactionBytesLimit); + private native long softPendingCompactionBytesLimit(long handle); + private native void setHardPendingCompactionBytesLimit(long handle, + long hardPendingCompactionBytesLimit); + private native long hardPendingCompactionBytesLimit(long handle); + private native void setLevel0FileNumCompactionTrigger(long handle, + int level0FileNumCompactionTrigger); + private native int level0FileNumCompactionTrigger(long handle); + private native void setLevel0SlowdownWritesTrigger(long handle, + int level0SlowdownWritesTrigger); + private native int level0SlowdownWritesTrigger(long handle); + private native void setLevel0StopWritesTrigger(long handle, + int level0StopWritesTrigger); + private native int level0StopWritesTrigger(long handle); + private native void setMaxBytesForLevelMultiplierAdditional(long handle, + int[] maxBytesForLevelMultiplierAdditional); + private native int[] maxBytesForLevelMultiplierAdditional(long handle); + private native void setParanoidFileChecks(long handle, + boolean paranoidFileChecks); + private native boolean paranoidFileChecks(long handle); + private native void setMaxWriteBufferNumberToMaintain(final long handle, + final int maxWriteBufferNumberToMaintain); + private native int maxWriteBufferNumberToMaintain(final long handle); + private native void setCompactionPriority(final long handle, + final byte compactionPriority); + private native byte compactionPriority(final long handle); + private native void setReportBgIoStats(final long handle, + final boolean reportBgIoStats); + private native boolean reportBgIoStats(final long handle); + private native void setTtl(final long handle, final long ttl); + private native long ttl(final long handle); + private native void setPeriodicCompactionSeconds( + final long handle, final long periodicCompactionSeconds); + private native long periodicCompactionSeconds(final long handle); + private native void setCompactionOptionsUniversal(final long handle, + final long compactionOptionsUniversalHandle); + private native void setCompactionOptionsFIFO(final long handle, + final long compactionOptionsFIFOHandle); + private native void setForceConsistencyChecks(final long handle, + final boolean forceConsistencyChecks); + private native boolean forceConsistencyChecks(final long handle); + private native void setSstPartitionerFactory(long nativeHandle_, long newFactoryHandle); + private static native void setCompactionThreadLimiter( + final long nativeHandle_, final long compactionThreadLimiterHandle); + private native void setMemtableMaxRangeDeletions(final long handle, final int count); + private native int memtableMaxRangeDeletions(final long handle); + + private native void setEnableBlobFiles(final long nativeHandle_, final boolean enableBlobFiles); + private native boolean enableBlobFiles(final long nativeHandle_); + private native void setMinBlobSize(final long nativeHandle_, final long minBlobSize); + private native long minBlobSize(final long nativeHandle_); + private native void setBlobFileSize(final long nativeHandle_, final long blobFileSize); + private native long blobFileSize(final long nativeHandle_); + private native void setBlobCompressionType(final long nativeHandle_, final byte compressionType); + private native byte blobCompressionType(final long nativeHandle_); + private native void setEnableBlobGarbageCollection( + final long nativeHandle_, final boolean enableBlobGarbageCollection); + private native boolean enableBlobGarbageCollection(final long nativeHandle_); + private native void setBlobGarbageCollectionAgeCutoff( + final long nativeHandle_, final double blobGarbageCollectionAgeCutoff); + private native double blobGarbageCollectionAgeCutoff(final long nativeHandle_); + private native void setBlobGarbageCollectionForceThreshold( + final long nativeHandle_, final double blobGarbageCollectionForceThreshold); + private native double blobGarbageCollectionForceThreshold(final long nativeHandle_); + private native void setBlobCompactionReadaheadSize( + final long nativeHandle_, final long blobCompactionReadaheadSize); + private native long blobCompactionReadaheadSize(final long nativeHandle_); + private native void setBlobFileStartingLevel( + final long nativeHandle_, final int blobFileStartingLevel); + private native int blobFileStartingLevel(final long nativeHandle_); + private native void setPrepopulateBlobCache( + final long nativeHandle_, final byte prepopulateBlobCache); + private native byte prepopulateBlobCache(final long nativeHandle_); + + // instance variables + // NOTE: If you add new member variables, please update the copy constructor above! + private MemTableConfig memTableConfig_; + private TableFormatConfig tableFormatConfig_; + private AbstractComparator comparator_; + private AbstractCompactionFilter> compactionFilter_; + private AbstractCompactionFilterFactory> + compactionFilterFactory_; + private CompactionOptionsUniversal compactionOptionsUniversal_; + private CompactionOptionsFIFO compactionOptionsFIFO_; + private CompressionOptions bottommostCompressionOptions_; + private CompressionOptions compressionOptions_; + private SstPartitionerFactory sstPartitionerFactory_; + private ConcurrentTaskLimiter compactionThreadLimiter_; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java new file mode 100644 index 0000000..4776773 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java @@ -0,0 +1,553 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Collection; +import java.util.List; + +public interface ColumnFamilyOptionsInterface> + extends AdvancedColumnFamilyOptionsInterface { + /** + * The function recovers options to a previous version. Only 4.6 or later + * versions are supported. + * + * @param majorVersion The major version to recover default values of options + * @param minorVersion The minor version to recover default values of options + * @return the instance of the current object. + */ + T oldDefaults(int majorVersion, int minorVersion); + + /** + * Use this if your DB is very small (like under 1GB) and you don't want to + * spend lots of memory for memtables. + * + * @return the instance of the current object. + */ + T optimizeForSmallDb(); + + /** + * Some functions that make it easier to optimize RocksDB + * Use this if your DB is very small (like under 1GB) and you don't want to + * spend lots of memory for memtables. + * + * @param cache An optional cache object is passed in to be used as the block cache + * @return the instance of the current object. + */ + T optimizeForSmallDb(Cache cache); + + /** + * Use this if you don't need to keep the data sorted, i.e. you'll never use + * an iterator, only Put() and Get() API calls + * + * @param blockCacheSizeMb Block cache size in MB + * @return the instance of the current object. + */ + T optimizeForPointLookup(long blockCacheSizeMb); + + /** + *

    Default values for some parameters in ColumnFamilyOptions are not + * optimized for heavy workloads and big datasets, which means you might + * observe write stalls under some conditions. As a starting point for tuning + * RocksDB options, use the following for level style compaction.

    + * + *

    Make sure to also call IncreaseParallelism(), which will provide the + * biggest performance gains.

    + *

    Note: we might use more memory than memtable_memory_budget during high + * write rate period

    + * + * @return the instance of the current object. + */ + T optimizeLevelStyleCompaction(); + + /** + *

    Default values for some parameters in ColumnFamilyOptions are not + * optimized for heavy workloads and big datasets, which means you might + * observe write stalls under some conditions. As a starting point for tuning + * RocksDB options, use the following for level style compaction.

    + * + *

    Make sure to also call IncreaseParallelism(), which will provide the + * biggest performance gains.

    + *

    Note: we might use more memory than memtable_memory_budget during high + * write rate period

    + * + * @param memtableMemoryBudget memory budget in bytes + * @return the instance of the current object. + */ + T optimizeLevelStyleCompaction( + long memtableMemoryBudget); + + /** + *

    Default values for some parameters in ColumnFamilyOptions are not + * optimized for heavy workloads and big datasets, which means you might + * observe write stalls under some conditions. As a starting point for tuning + * RocksDB options, use the following for universal style compaction.

    + * + *

    Universal style compaction is focused on reducing Write Amplification + * Factor for big data sets, but increases Space Amplification.

    + * + *

    Make sure to also call IncreaseParallelism(), which will provide the + * biggest performance gains.

    + * + *

    Note: we might use more memory than memtable_memory_budget during high + * write rate period

    + * + * @return the instance of the current object. + */ + T optimizeUniversalStyleCompaction(); + + /** + *

    Default values for some parameters in ColumnFamilyOptions are not + * optimized for heavy workloads and big datasets, which means you might + * observe write stalls under some conditions. As a starting point for tuning + * RocksDB options, use the following for universal style compaction.

    + * + *

    Universal style compaction is focused on reducing Write Amplification + * Factor for big data sets, but increases Space Amplification.

    + * + *

    Make sure to also call IncreaseParallelism(), which will provide the + * biggest performance gains.

    + * + *

    Note: we might use more memory than memtable_memory_budget during high + * write rate period

    + * + * @param memtableMemoryBudget memory budget in bytes + * @return the instance of the current object. + */ + T optimizeUniversalStyleCompaction( + long memtableMemoryBudget); + + /** + * Set {@link BuiltinComparator} to be used with RocksDB. + *

    + * Note: Comparator can be set once upon database creation. + *

    + * Default: BytewiseComparator. + * @param builtinComparator a {@link BuiltinComparator} type. + * @return the instance of the current object. + */ + T setComparator( + BuiltinComparator builtinComparator); + + /** + * Use the specified comparator for key ordering. + *

    + * Comparator should not be disposed before options instances using this comparator is + * disposed. If dispose() function is not called, then comparator object will be + * GC'd automatically. + *

    + * Comparator instance can be re-used in multiple options instances. + * + * @param comparator java instance. + * @return the instance of the current object. + */ + T setComparator( + AbstractComparator comparator); + + /** + *

    Set the merge operator to be used for merging two merge operands + * of the same key. The merge function is invoked during + * compaction and at lookup time, if multiple key/value pairs belonging + * to the same key are found in the database.

    + * + * @param name the name of the merge function, as defined by + * the MergeOperators factory (see utilities/MergeOperators.h) + * The merge function is specified by name and must be one of the + * standard merge operators provided by RocksDB. The available + * operators are "put", "uint64add", "stringappend" and "stringappendtest". + * @return the instance of the current object. + */ + T setMergeOperatorName(String name); + + /** + *

    Set the merge operator to be used for merging two different key/value + * pairs that share the same key. The merge function is invoked during + * compaction and at lookup time, if multiple key/value pairs belonging + * to the same key are found in the database.

    + * + * @param mergeOperator {@link MergeOperator} instance. + * @return the instance of the current object. + */ + T setMergeOperator(MergeOperator mergeOperator); + + /** + * A single CompactionFilter instance to call into during compaction. + * Allows an application to modify/delete a key-value during background + * compaction. + *

    + * If the client requires a new compaction filter to be used for different + * compaction runs, it can specify call + * {@link #setCompactionFilterFactory(AbstractCompactionFilterFactory)} + * instead. + *

    + * The client should specify only set one of the two. + * {#setCompactionFilter(AbstractCompactionFilter)} takes precedence + * over {@link #setCompactionFilterFactory(AbstractCompactionFilterFactory)} + * if the client specifies both. + *

    + * If multithreaded compaction is being used, the supplied CompactionFilter + * instance may be used from different threads concurrently and so should be thread-safe. + * + * @param compactionFilter {@link AbstractCompactionFilter} instance. + * @return the instance of the current object. + */ + T setCompactionFilter( + final AbstractCompactionFilter> compactionFilter); + + /** + * Accessor for the CompactionFilter instance in use. + * + * @return Reference to the CompactionFilter, or null if one hasn't been set. + */ + AbstractCompactionFilter> compactionFilter(); + + /** + * This is a factory that provides {@link AbstractCompactionFilter} objects + * which allow an application to modify/delete a key-value during background + * compaction. + *

    + * A new filter will be created on each compaction run. If multithreaded + * compaction is being used, each created CompactionFilter will only be used + * from a single thread and so does not need to be thread-safe. + * + * @param compactionFilterFactory {@link AbstractCompactionFilterFactory} instance. + * @return the instance of the current object. + */ + T setCompactionFilterFactory( + final AbstractCompactionFilterFactory> + compactionFilterFactory); + + /** + * Accessor for the CompactionFilterFactory instance in use. + * + * @return Reference to the CompactionFilterFactory, or null if one hasn't been set. + */ + AbstractCompactionFilterFactory> compactionFilterFactory(); + + /** + * This prefix-extractor uses the first n bytes of a key as its prefix. + *

    + * In some hash-based memtable representation such as HashLinkedList + * and HashSkipList, prefixes are used to partition the keys into + * several buckets. Prefix extractor is used to specify how to + * extract the prefix given a key. + * + * @param n use the first n bytes of a key as its prefix. + * @return the reference to the current option. + */ + T useFixedLengthPrefixExtractor(int n); + + /** + * Same as fixed length prefix extractor, except that when slice is + * shorter than the fixed length, it will use the full key. + * + * @param n use the first n bytes of a key as its prefix. + * @return the reference to the current option. + */ + T useCappedPrefixExtractor(int n); + + /** + * Number of files to trigger level-0 compaction. A value < 0 means that + * level-0 compaction will not be triggered by number of files at all. + * Default: 4 + * + * @param numFiles the number of files in level-0 to trigger compaction. + * @return the reference to the current option. + */ + T setLevelZeroFileNumCompactionTrigger( + int numFiles); + + /** + * The number of files in level 0 to trigger compaction from level-0 to + * level-1. A value < 0 means that level-0 compaction will not be + * triggered by number of files at all. + * Default: 4 + * + * @return the number of files in level 0 to trigger compaction. + */ + int levelZeroFileNumCompactionTrigger(); + + /** + * Soft limit on number of level-0 files. We start slowing down writes at this + * point. A value < 0 means that no writing slow down will be triggered by + * number of files in level-0. + * + * @param numFiles soft limit on number of level-0 files. + * @return the reference to the current option. + */ + T setLevelZeroSlowdownWritesTrigger( + int numFiles); + + /** + * Soft limit on the number of level-0 files. We start slowing down writes + * at this point. A value < 0 means that no writing slow down will be + * triggered by number of files in level-0. + * + * @return the soft limit on the number of level-0 files. + */ + int levelZeroSlowdownWritesTrigger(); + + /** + * Maximum number of level-0 files. We stop writes at this point. + * + * @param numFiles the hard limit of the number of level-0 files. + * @return the reference to the current option. + */ + T setLevelZeroStopWritesTrigger(int numFiles); + + /** + * Maximum number of level-0 files. We stop writes at this point. + * + * @return the hard limit of the number of level-0 file. + */ + int levelZeroStopWritesTrigger(); + + /** + * The ratio between the total size of level-(L+1) files and the total + * size of level-L files for all L. + * DEFAULT: 10 + * + * @param multiplier the ratio between the total size of level-(L+1) + * files and the total size of level-L files for all L. + * @return the reference to the current option. + */ + T setMaxBytesForLevelMultiplier( + double multiplier); + + /** + * The ratio between the total size of level-(L+1) files and the total + * size of level-L files for all L. + * DEFAULT: 10 + * + * @return the ratio between the total size of level-(L+1) files and + * the total size of level-L files for all L. + */ + double maxBytesForLevelMultiplier(); + + /** + * FIFO compaction option. + * The oldest table file will be deleted + * once the sum of table files reaches this size. + * The default value is 1GB (1 * 1024 * 1024 * 1024). + * + * @param maxTableFilesSize the size limit of the total sum of table files. + * @return the instance of the current object. + */ + T setMaxTableFilesSizeFIFO( + long maxTableFilesSize); + + /** + * FIFO compaction option. + * The oldest table file will be deleted + * once the sum of table files reaches this size. + * The default value is 1GB (1 * 1024 * 1024 * 1024). + * + * @return the size limit of the total sum of table files. + */ + long maxTableFilesSizeFIFO(); + + /** + * Get the config for mem-table. + * + * @return the mem-table config. + */ + MemTableConfig memTableConfig(); + + /** + * Set the config for mem-table. + * + * @param memTableConfig the mem-table config. + * @return the instance of the current object. + * @throws java.lang.IllegalArgumentException thrown on 32-Bit platforms + * while overflowing the underlying platform specific value. + */ + T setMemTableConfig(MemTableConfig memTableConfig); + + /** + * Returns the name of the current mem table representation. + * Memtable format can be set using setTableFormatConfig. + * + * @return the name of the currently-used memtable factory. + * @see #setTableFormatConfig(org.rocksdb.TableFormatConfig) + */ + String memTableFactoryName(); + + /** + * Get the config for table format. + * + * @return the table format config. + */ + TableFormatConfig tableFormatConfig(); + + /** + * Set the config for table format. + * + * @param config the table format config. + * @return the reference of the current options. + */ + T setTableFormatConfig(TableFormatConfig config); + + /** + * @return the name of the currently used table factory. + */ + String tableFactoryName(); + + /** + * A list of paths where SST files for this column family + * can be put into, with its target size. Similar to db_paths, + * newer data is placed into paths specified earlier in the + * vector while older data gradually moves to paths specified + * later in the vector. + * Note that, if a path is supplied to multiple column + * families, it would have files and total size from all + * the column families combined. User should provision for the + * total size(from all the column families) in such cases. + *

    + * If left empty, db_paths will be used. + * Default: empty + * + * @param paths collection of paths for SST files. + * @return the reference of the current options. + */ + T setCfPaths(final Collection paths); + + /** + * @return collection of paths for SST files. + */ + List cfPaths(); + + /** + * Compression algorithm that will be used for the bottommost level that + * contain files. If level-compaction is used, this option will only affect + * levels after base level. + *

    + * Default: {@link CompressionType#DISABLE_COMPRESSION_OPTION} + * + * @param bottommostCompressionType The compression type to use for the + * bottommost level + * + * @return the reference of the current options. + */ + T setBottommostCompressionType( + final CompressionType bottommostCompressionType); + + /** + * Compression algorithm that will be used for the bottommost level that + * contain files. If level-compaction is used, this option will only affect + * levels after base level. + *

    + * Default: {@link CompressionType#DISABLE_COMPRESSION_OPTION} + * + * @return The compression type used for the bottommost level + */ + CompressionType bottommostCompressionType(); + + /** + * Set the options for compression algorithms used by + * {@link #bottommostCompressionType()} if it is enabled. + *

    + * To enable it, please see the definition of + * {@link CompressionOptions}. + * + * @param compressionOptions the bottom most compression options. + * + * @return the reference of the current options. + */ + T setBottommostCompressionOptions( + final CompressionOptions compressionOptions); + + /** + * Get the bottom most compression options. + *

    + * See {@link #setBottommostCompressionOptions(CompressionOptions)}. + * + * @return the bottom most compression options. + */ + CompressionOptions bottommostCompressionOptions(); + + /** + * Set the different options for compression algorithms + * + * @param compressionOptions The compression options + * + * @return the reference of the current options. + */ + T setCompressionOptions( + CompressionOptions compressionOptions); + + /** + * Get the different options for compression algorithms + * + * @return The compression options + */ + CompressionOptions compressionOptions(); + + /** + * If non-nullptr, use the specified factory for a function to determine the + * partitioning of sst files. This helps compaction to split the files + * on interesting boundaries (key prefixes) to make propagation of sst + * files less write amplifying (covering the whole key space). + *

    + * Default: nullptr + * + * @param factory The factory reference + * @return the reference of the current options. + */ + @Experimental("Caution: this option is experimental") + T setSstPartitionerFactory(SstPartitionerFactory factory); + + /** + * Get SST partitioner factory + * + * @return SST partitioner factory + */ + @Experimental("Caution: this option is experimental") + SstPartitionerFactory sstPartitionerFactory(); + + /** + * Sets the maximum range delete calls, after which memtable is flushed. + * This applies to the mutable memtable. + * + * @param count a positive integer, 0 (default) to disable the feature. + * @return the reference of the current options. + */ + T setMemtableMaxRangeDeletions(final int count); + + /** + * Gets the current setting of maximum range deletes allowed + * 0(default) indicates that feature is disabled. + * + * @return current value of memtable_max_range_deletions + */ + int memtableMaxRangeDeletions(); + + /** + * Compaction concurrent thread limiter for the column family. + * If non-nullptr, use given concurrent thread limiter to control + * the max outstanding compaction tasks. Limiter can be shared with + * multiple column families across db instances. + * + * @param concurrentTaskLimiter The compaction thread limiter. + * @return the reference of the current options. + */ + T setCompactionThreadLimiter(ConcurrentTaskLimiter concurrentTaskLimiter); + + /** + * Get compaction thread limiter + * + * @return Compaction thread limiter + */ + ConcurrentTaskLimiter compactionThreadLimiter(); + + /** + * Default memtable memory budget used with the following methods: + * + *

      + *
    1. {@link #optimizeLevelStyleCompaction()}
      2. + *
    2. {@link #optimizeUniversalStyleCompaction()}
      3. + *
    + */ + long DEFAULT_COMPACTION_MEMTABLE_MEMORY_BUDGET = 512 * 1024 * 1024; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactRangeOptions.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactRangeOptions.java new file mode 100644 index 0000000..616a775 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactRangeOptions.java @@ -0,0 +1,304 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Objects; + +/** + * CompactRangeOptions is used by CompactRange() call. In the documentation of the methods "the compaction" refers to + * any compaction that is using this CompactRangeOptions. + */ +public class CompactRangeOptions extends RocksObject { + private static final byte VALUE_kSkip = 0; + private static final byte VALUE_kIfHaveCompactionFilter = 1; + private static final byte VALUE_kForce = 2; + private static final byte VALUE_kForceOptimized = 3; + + // For level based compaction, we can configure if we want to skip/force bottommost level + // compaction. The order of this enum MUST follow the C++ layer. See BottommostLevelCompaction in + // db/options.h + public enum BottommostLevelCompaction { + /** + * Skip bottommost level compaction + */ + kSkip(VALUE_kSkip), + /** + * Only compact bottommost level if there is a compaction filter. This is the default option + */ + kIfHaveCompactionFilter(VALUE_kIfHaveCompactionFilter), + /** + * Always compact bottommost level + */ + kForce(VALUE_kForce), + /** + * Always compact bottommost level but in bottommost level avoid + * double-compacting files created in the same compaction + */ + kForceOptimized(VALUE_kForceOptimized); + + private final byte value; + + BottommostLevelCompaction(final byte value) { + this.value = value; + } + + /** + *

    Returns the byte value of the enumerations value.

    + * + * @return byte representation + */ + public byte getValue() { + return value; + } + + /** + * Returns the BottommostLevelCompaction for the given C++ rocks enum value. + * @param bottommostLevelCompaction The value of the BottommostLevelCompaction + * @return BottommostLevelCompaction instance, or null if none matches + */ + public static BottommostLevelCompaction fromRocksId(final int bottommostLevelCompaction) { + switch (bottommostLevelCompaction) { + case VALUE_kSkip: return kSkip; + case VALUE_kIfHaveCompactionFilter: return kIfHaveCompactionFilter; + case VALUE_kForce: return kForce; + case VALUE_kForceOptimized: + return kForceOptimized; + default: return null; + } + } + } + + public static class Timestamp { + public final long start; + public final long range; + + public Timestamp(final long start, final long duration) { + this.start = start; + this.range = duration; + } + + public Timestamp() { + this.start = 0; + this.range = 0; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + Timestamp timestamp = (Timestamp) o; + return start == timestamp.start && range == timestamp.range; + } + + @Override + public int hashCode() { + return Objects.hash(start, range); + } + } + + /** + * Construct CompactRangeOptions. + */ + public CompactRangeOptions() { + super(newCompactRangeOptions()); + } + + /** + * Returns whether the compaction is exclusive or other compactions may run concurrently at the same time. + * + * @return true if exclusive, false if concurrent + */ + public boolean exclusiveManualCompaction() { + return exclusiveManualCompaction(nativeHandle_); + } + + /** + * Sets whether the compaction is exclusive or other compaction are allowed run concurrently at the same time. + * + * @param exclusiveCompaction true if compaction should be exclusive + * @return This CompactRangeOptions + */ + public CompactRangeOptions setExclusiveManualCompaction(final boolean exclusiveCompaction) { + setExclusiveManualCompaction(nativeHandle_, exclusiveCompaction); + return this; + } + + /** + * Returns whether compacted files will be moved to the minimum level capable of holding the data or given level + * (specified non-negative target_level). + * @return true, if compacted files will be moved to the minimum level + */ + public boolean changeLevel() { + return changeLevel(nativeHandle_); + } + + /** + * Whether compacted files will be moved to the minimum level capable of holding the data or given level + * (specified non-negative target_level). + * + * @param changeLevel If true, compacted files will be moved to the minimum level + * @return This CompactRangeOptions + */ + public CompactRangeOptions setChangeLevel(final boolean changeLevel) { + setChangeLevel(nativeHandle_, changeLevel); + return this; + } + + /** + * If change_level is true and target_level have non-negative value, compacted files will be moved to target_level. + * @return The target level for the compacted files + */ + public int targetLevel() { + return targetLevel(nativeHandle_); + } + + + /** + * If change_level is true and target_level have non-negative value, compacted files will be moved to target_level. + * + * @param targetLevel target level for the compacted files + * @return This CompactRangeOptions + */ + public CompactRangeOptions setTargetLevel(final int targetLevel) { + setTargetLevel(nativeHandle_, targetLevel); + return this; + } + + /** + * target_path_id for compaction output. Compaction outputs will be placed in options.db_paths[target_path_id]. + * + * @return target_path_id + */ + public int targetPathId() { + return targetPathId(nativeHandle_); + } + + /** + * Compaction outputs will be placed in options.db_paths[target_path_id]. Behavior is undefined if target_path_id is + * out of range. + * + * @param targetPathId target path id + * @return This CompactRangeOptions + */ + public CompactRangeOptions setTargetPathId(final int targetPathId) { + setTargetPathId(nativeHandle_, targetPathId); + return this; + } + + /** + * Returns the policy for compacting the bottommost level + * @return The BottommostLevelCompaction policy + */ + public BottommostLevelCompaction bottommostLevelCompaction() { + return BottommostLevelCompaction.fromRocksId(bottommostLevelCompaction(nativeHandle_)); + } + + /** + * Sets the policy for compacting the bottommost level + * + * @param bottommostLevelCompaction The policy for compacting the bottommost level + * @return This CompactRangeOptions + */ + public CompactRangeOptions setBottommostLevelCompaction(final BottommostLevelCompaction bottommostLevelCompaction) { + setBottommostLevelCompaction(nativeHandle_, bottommostLevelCompaction.getValue()); + return this; + } + + /** + * If true, compaction will execute immediately even if doing so would cause the DB to + * enter write stall mode. Otherwise, it'll sleep until load is low enough. + * @return true if compaction will execute immediately + */ + public boolean allowWriteStall() { + return allowWriteStall(nativeHandle_); + } + + + /** + * If true, compaction will execute immediately even if doing so would cause the DB to + * enter write stall mode. Otherwise, it'll sleep until load is low enough. + * + * @return This CompactRangeOptions + * @param allowWriteStall true if compaction should execute immediately + */ + public CompactRangeOptions setAllowWriteStall(final boolean allowWriteStall) { + setAllowWriteStall(nativeHandle_, allowWriteStall); + return this; + } + + /** + * If > 0, it will replace the option in the DBOptions for this compaction + * @return number of subcompactions + */ + public int maxSubcompactions() { + return maxSubcompactions(nativeHandle_); + } + + /** + * If > 0, it will replace the option in the DBOptions for this compaction + * + * @param maxSubcompactions number of subcompactions + * @return This CompactRangeOptions + */ + public CompactRangeOptions setMaxSubcompactions(final int maxSubcompactions) { + setMaxSubcompactions(nativeHandle_, maxSubcompactions); + return this; + } + + public CompactRangeOptions setFullHistoryTSLow(final Timestamp tsLow) { + setFullHistoryTSLow(nativeHandle_, tsLow.start, tsLow.range); + return this; + } + + public Timestamp fullHistoryTSLow() { + return fullHistoryTSLow(nativeHandle_); + } + + public CompactRangeOptions setCanceled(final boolean canceled) { + setCanceled(nativeHandle_, canceled); + return this; + } + + public boolean canceled() { + return canceled(nativeHandle_); + } + + private static native long newCompactRangeOptions(); + @Override protected final native void disposeInternal(final long handle); + + private native boolean exclusiveManualCompaction(final long handle); + private native void setExclusiveManualCompaction(final long handle, + final boolean exclusive_manual_compaction); + private native boolean changeLevel(final long handle); + private native void setChangeLevel(final long handle, + final boolean changeLevel); + private native int targetLevel(final long handle); + private native void setTargetLevel(final long handle, + final int targetLevel); + private native int targetPathId(final long handle); + private native void setTargetPathId(final long handle, + final int targetPathId); + private native int bottommostLevelCompaction(final long handle); + private native void setBottommostLevelCompaction(final long handle, + final int bottommostLevelCompaction); + private native boolean allowWriteStall(final long handle); + private native void setAllowWriteStall(final long handle, + final boolean allowWriteStall); + private native void setMaxSubcompactions(final long handle, + final int maxSubcompactions); + private native int maxSubcompactions(final long handle); + + private native void setFullHistoryTSLow( + final long handle, final long timestampStart, final long timestampRange); + + private native Timestamp fullHistoryTSLow(final long handle); + + private native void setCanceled(final long handle, final boolean canceled); + + private native boolean canceled(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionJobInfo.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionJobInfo.java new file mode 100644 index 0000000..cf04bde --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionJobInfo.java @@ -0,0 +1,161 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +public class CompactionJobInfo extends RocksObject { + + public CompactionJobInfo() { + super(newCompactionJobInfo()); + } + + /** + * Private as called from JNI C++ + */ + private CompactionJobInfo(final long nativeHandle) { + super(nativeHandle); + // We do not own the native object! + disOwnNativeHandle(); + } + + /** + * Get the name of the column family where the compaction happened. + * + * @return the name of the column family + */ + public byte[] columnFamilyName() { + return columnFamilyName(nativeHandle_); + } + + /** + * Get the status indicating whether the compaction was successful or not. + * + * @return the status + */ + public Status status() { + return status(nativeHandle_); + } + + /** + * Get the id of the thread that completed this compaction job. + * + * @return the id of the thread + */ + public long threadId() { + return threadId(nativeHandle_); + } + + /** + * Get the job id, which is unique in the same thread. + * + * @return the id of the thread + */ + public int jobId() { + return jobId(nativeHandle_); + } + + /** + * Get the smallest input level of the compaction. + * + * @return the input level + */ + public int baseInputLevel() { + return baseInputLevel(nativeHandle_); + } + + /** + * Get the output level of the compaction. + * + * @return the output level + */ + public int outputLevel() { + return outputLevel(nativeHandle_); + } + + /** + * Get the names of the compaction input files. + * + * @return the names of the input files. + */ + public List inputFiles() { + return Arrays.asList(inputFiles(nativeHandle_)); + } + + /** + * Get the names of the compaction output files. + * + * @return the names of the output files. + */ + public List outputFiles() { + return Arrays.asList(outputFiles(nativeHandle_)); + } + + /** + * Get the table properties for the input and output tables. + *

    + * The map is keyed by values from {@link #inputFiles()} and + * {@link #outputFiles()}. + * + * @return the table properties + */ + public Map tableProperties() { + return tableProperties(nativeHandle_); + } + + /** + * Get the Reason for running the compaction. + * + * @return the reason. + */ + public CompactionReason compactionReason() { + return CompactionReason.fromValue(compactionReason(nativeHandle_)); + } + + // + /** + * Get the compression algorithm used for output files. + * + * @return the compression algorithm + */ + public CompressionType compression() { + return CompressionType.getCompressionType(compression(nativeHandle_)); + } + + /** + * Get detailed information about this compaction. + * + * @return the detailed information, or null if not available. + */ + public /* @Nullable */ CompactionJobStats stats() { + final long statsHandle = stats(nativeHandle_); + if (statsHandle == 0) { + return null; + } + + return new CompactionJobStats(statsHandle); + } + + + private static native long newCompactionJobInfo(); + @Override protected native void disposeInternal(final long handle); + + private static native byte[] columnFamilyName(final long handle); + private static native Status status(final long handle); + private static native long threadId(final long handle); + private static native int jobId(final long handle); + private static native int baseInputLevel(final long handle); + private static native int outputLevel(final long handle); + private static native String[] inputFiles(final long handle); + private static native String[] outputFiles(final long handle); + private static native Map tableProperties( + final long handle); + private static native byte compactionReason(final long handle); + private static native byte compression(final long handle); + private static native long stats(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionJobStats.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionJobStats.java new file mode 100644 index 0000000..3d53b55 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionJobStats.java @@ -0,0 +1,295 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +public class CompactionJobStats extends RocksObject { + + public CompactionJobStats() { + super(newCompactionJobStats()); + } + + /** + * Private as called from JNI C++ + */ + CompactionJobStats(final long nativeHandle) { + super(nativeHandle); + } + + /** + * Reset the stats. + */ + public void reset() { + reset(nativeHandle_); + } + + /** + * Aggregate the CompactionJobStats from another instance with this one. + * + * @param compactionJobStats another instance of stats. + */ + public void add(final CompactionJobStats compactionJobStats) { + add(nativeHandle_, compactionJobStats.nativeHandle_); + } + + /** + * Get the elapsed time in micro of this compaction. + * + * @return the elapsed time in micro of this compaction. + */ + public long elapsedMicros() { + return elapsedMicros(nativeHandle_); + } + + /** + * Get the number of compaction input records. + * + * @return the number of compaction input records. + */ + public long numInputRecords() { + return numInputRecords(nativeHandle_); + } + + /** + * Get the number of compaction input files. + * + * @return the number of compaction input files. + */ + public long numInputFiles() { + return numInputFiles(nativeHandle_); + } + + /** + * Get the number of compaction input files at the output level. + * + * @return the number of compaction input files at the output level. + */ + public long numInputFilesAtOutputLevel() { + return numInputFilesAtOutputLevel(nativeHandle_); + } + + /** + * Get the number of compaction output records. + * + * @return the number of compaction output records. + */ + public long numOutputRecords() { + return numOutputRecords(nativeHandle_); + } + + /** + * Get the number of compaction output files. + * + * @return the number of compaction output files. + */ + public long numOutputFiles() { + return numOutputFiles(nativeHandle_); + } + + /** + * Determine if the compaction is a manual compaction. + * + * @return true if the compaction is a manual compaction, false otherwise. + */ + public boolean isManualCompaction() { + return isManualCompaction(nativeHandle_); + } + + /** + * Get the size of the compaction input in bytes. + * + * @return the size of the compaction input in bytes. + */ + public long totalInputBytes() { + return totalInputBytes(nativeHandle_); + } + + /** + * Get the size of the compaction output in bytes. + * + * @return the size of the compaction output in bytes. + */ + public long totalOutputBytes() { + return totalOutputBytes(nativeHandle_); + } + + /** + * Get the number of records being replaced by newer record associated + * with same key. + * + * This could be a new value or a deletion entry for that key so this field + * sums up all updated and deleted keys. + * + * @return the number of records being replaced by newer record associated + * with same key. + */ + public long numRecordsReplaced() { + return numRecordsReplaced(nativeHandle_); + } + + /** + * Get the sum of the uncompressed input keys in bytes. + * + * @return the sum of the uncompressed input keys in bytes. + */ + public long totalInputRawKeyBytes() { + return totalInputRawKeyBytes(nativeHandle_); + } + + /** + * Get the sum of the uncompressed input values in bytes. + * + * @return the sum of the uncompressed input values in bytes. + */ + public long totalInputRawValueBytes() { + return totalInputRawValueBytes(nativeHandle_); + } + + /** + * Get the number of deletion entries before compaction. + * + * Deletion entries can disappear after compaction because they expired. + * + * @return the number of deletion entries before compaction. + */ + public long numInputDeletionRecords() { + return numInputDeletionRecords(nativeHandle_); + } + + /** + * Get the number of deletion records that were found obsolete and discarded + * because it is not possible to delete any more keys with this entry. + * (i.e. all possible deletions resulting from it have been completed) + * + * @return the number of deletion records that were found obsolete and + * discarded. + */ + public long numExpiredDeletionRecords() { + return numExpiredDeletionRecords(nativeHandle_); + } + + /** + * Get the number of corrupt keys (ParseInternalKey returned false when + * applied to the key) encountered and written out. + * + * @return the number of corrupt keys. + */ + public long numCorruptKeys() { + return numCorruptKeys(nativeHandle_); + } + + /** + * Get the Time spent on file's Append() call. + * + * Only populated if {@link ColumnFamilyOptions#reportBgIoStats()} is set. + * + * @return the Time spent on file's Append() call. + */ + public long fileWriteNanos() { + return fileWriteNanos(nativeHandle_); + } + + /** + * Get the Time spent on sync file range. + * + * Only populated if {@link ColumnFamilyOptions#reportBgIoStats()} is set. + * + * @return the Time spent on sync file range. + */ + public long fileRangeSyncNanos() { + return fileRangeSyncNanos(nativeHandle_); + } + + /** + * Get the Time spent on file fsync. + * + * Only populated if {@link ColumnFamilyOptions#reportBgIoStats()} is set. + * + * @return the Time spent on file fsync. + */ + public long fileFsyncNanos() { + return fileFsyncNanos(nativeHandle_); + } + + /** + * Get the Time spent on preparing file write (falocate, etc) + * + * Only populated if {@link ColumnFamilyOptions#reportBgIoStats()} is set. + * + * @return the Time spent on preparing file write (falocate, etc). + */ + public long filePrepareWriteNanos() { + return filePrepareWriteNanos(nativeHandle_); + } + + /** + * Get the smallest output key prefix. + * + * @return the smallest output key prefix. + */ + public byte[] smallestOutputKeyPrefix() { + return smallestOutputKeyPrefix(nativeHandle_); + } + + /** + * Get the largest output key prefix. + * + * @return the smallest output key prefix. + */ + public byte[] largestOutputKeyPrefix() { + return largestOutputKeyPrefix(nativeHandle_); + } + + /** + * Get the number of single-deletes which do not meet a put. + * + * @return number of single-deletes which do not meet a put. + */ + @Experimental("Performance optimization for a very specific workload") + public long numSingleDelFallthru() { + return numSingleDelFallthru(nativeHandle_); + } + + /** + * Get the number of single-deletes which meet something other than a put. + * + * @return the number of single-deletes which meet something other than a put. + */ + @Experimental("Performance optimization for a very specific workload") + public long numSingleDelMismatch() { + return numSingleDelMismatch(nativeHandle_); + } + + private static native long newCompactionJobStats(); + @Override protected native void disposeInternal(final long handle); + + + private static native void reset(final long handle); + private static native void add(final long handle, + final long compactionJobStatsHandle); + private static native long elapsedMicros(final long handle); + private static native long numInputRecords(final long handle); + private static native long numInputFiles(final long handle); + private static native long numInputFilesAtOutputLevel(final long handle); + private static native long numOutputRecords(final long handle); + private static native long numOutputFiles(final long handle); + private static native boolean isManualCompaction(final long handle); + private static native long totalInputBytes(final long handle); + private static native long totalOutputBytes(final long handle); + private static native long numRecordsReplaced(final long handle); + private static native long totalInputRawKeyBytes(final long handle); + private static native long totalInputRawValueBytes(final long handle); + private static native long numInputDeletionRecords(final long handle); + private static native long numExpiredDeletionRecords(final long handle); + private static native long numCorruptKeys(final long handle); + private static native long fileWriteNanos(final long handle); + private static native long fileRangeSyncNanos(final long handle); + private static native long fileFsyncNanos(final long handle); + private static native long filePrepareWriteNanos(final long handle); + private static native byte[] smallestOutputKeyPrefix(final long handle); + private static native byte[] largestOutputKeyPrefix(final long handle); + private static native long numSingleDelFallthru(final long handle); + private static native long numSingleDelMismatch(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionOptions.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionOptions.java new file mode 100644 index 0000000..2c7e391 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionOptions.java @@ -0,0 +1,121 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.List; + +/** + * CompactionOptions are used in + * {@link RocksDB#compactFiles(CompactionOptions, ColumnFamilyHandle, List, int, int, CompactionJobInfo)} + * calls. + */ +public class CompactionOptions extends RocksObject { + + public CompactionOptions() { + super(newCompactionOptions()); + } + + /** + * Get the compaction output compression type. + * + * See {@link #setCompression(CompressionType)}. + * + * @return the compression type. + */ + public CompressionType compression() { + return CompressionType.getCompressionType( + compression(nativeHandle_)); + } + + /** + * Set the compaction output compression type. + * + * Default: snappy + * + * If set to {@link CompressionType#DISABLE_COMPRESSION_OPTION}, + * RocksDB will choose compression type according to the + * {@link ColumnFamilyOptions#compressionType()}, taking into account + * the output level if {@link ColumnFamilyOptions#compressionPerLevel()} + * is specified. + * + * @param compression the compression type to use for compaction output. + * + * @return the instance of the current Options. + */ + public CompactionOptions setCompression(final CompressionType compression) { + setCompression(nativeHandle_, compression.getValue()); + return this; + } + + /** + * Get the compaction output file size limit. + * + * See {@link #setOutputFileSizeLimit(long)}. + * + * @return the file size limit. + */ + public long outputFileSizeLimit() { + return outputFileSizeLimit(nativeHandle_); + } + + /** + * Compaction will create files of size {@link #outputFileSizeLimit()}. + * + * Default: 2^64-1, which means that compaction will create a single file + * + * @param outputFileSizeLimit the size limit + * + * @return the instance of the current Options. + */ + public CompactionOptions setOutputFileSizeLimit( + final long outputFileSizeLimit) { + setOutputFileSizeLimit(nativeHandle_, outputFileSizeLimit); + return this; + } + + /** + * Get the maximum number of threads that will concurrently perform a + * compaction job. + * + * @return the maximum number of threads. + */ + public int maxSubcompactions() { + return maxSubcompactions(nativeHandle_); + } + + /** + * This value represents the maximum number of threads that will + * concurrently perform a compaction job by breaking it into multiple, + * smaller ones that are run simultaneously. + * + * Default: 0 (i.e. no subcompactions) + * + * If > 0, it will replace the option in + * {@link DBOptions#maxSubcompactions()} for this compaction. + * + * @param maxSubcompactions The maximum number of threads that will + * concurrently perform a compaction job + * + * @return the instance of the current Options. + */ + public CompactionOptions setMaxSubcompactions(final int maxSubcompactions) { + setMaxSubcompactions(nativeHandle_, maxSubcompactions); + return this; + } + + private static native long newCompactionOptions(); + @Override protected final native void disposeInternal(final long handle); + + private static native byte compression(final long handle); + private static native void setCompression(final long handle, + final byte compressionTypeValue); + private static native long outputFileSizeLimit(final long handle); + private static native void setOutputFileSizeLimit(final long handle, + final long outputFileSizeLimit); + private static native int maxSubcompactions(final long handle); + private static native void setMaxSubcompactions(final long handle, + final int maxSubcompactions); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionOptionsFIFO.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionOptionsFIFO.java new file mode 100644 index 0000000..92b21fc --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionOptionsFIFO.java @@ -0,0 +1,87 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Options for FIFO Compaction + */ +public class CompactionOptionsFIFO extends RocksObject { + + public CompactionOptionsFIFO() { + super(newCompactionOptionsFIFO()); + } + + /** + * Once the total sum of table files reaches this, we will delete the oldest + * table file + *

    + * Default: 1GB + * + * @param maxTableFilesSize The maximum size of the table files + * + * @return the reference to the current options. + */ + public CompactionOptionsFIFO setMaxTableFilesSize( + final long maxTableFilesSize) { + setMaxTableFilesSize(nativeHandle_, maxTableFilesSize); + return this; + } + + /** + * Once the total sum of table files reaches this, we will delete the oldest + * table file + *

    + * Default: 1GB + * + * @return max table file size in bytes + */ + public long maxTableFilesSize() { + return maxTableFilesSize(nativeHandle_); + } + + /** + * If true, try to do compaction to compact smaller files into larger ones. + * Minimum files to compact follows options.level0_file_num_compaction_trigger + * and compaction won't trigger if average compact bytes per del file is + * larger than options.write_buffer_size. This is to protect large files + * from being compacted again. + *

    + * Default: false + * + * @param allowCompaction true to allow intra-L0 compaction + * + * @return the reference to the current options. + */ + public CompactionOptionsFIFO setAllowCompaction( + final boolean allowCompaction) { + setAllowCompaction(nativeHandle_, allowCompaction); + return this; + } + + /** + * Check if intra-L0 compaction is enabled. + * When enabled, we try to compact smaller files into larger ones. + *

    + * See {@link #setAllowCompaction(boolean)}. + *

    + * Default: false + * + * @return true if intra-L0 compaction is enabled, false otherwise. + */ + public boolean allowCompaction() { + return allowCompaction(nativeHandle_); + } + + private static native long newCompactionOptionsFIFO(); + @Override protected final native void disposeInternal(final long handle); + + private native void setMaxTableFilesSize(final long handle, + final long maxTableFilesSize); + private native long maxTableFilesSize(final long handle); + private native void setAllowCompaction(final long handle, + final boolean allowCompaction); + private native boolean allowCompaction(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionOptionsUniversal.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionOptionsUniversal.java new file mode 100644 index 0000000..4d2ebdb --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionOptionsUniversal.java @@ -0,0 +1,273 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Options for Universal Compaction + */ +public class CompactionOptionsUniversal extends RocksObject { + + public CompactionOptionsUniversal() { + super(newCompactionOptionsUniversal()); + } + + /** + * Percentage flexibility while comparing file size. If the candidate file(s) + * size is 1% smaller than the next file's size, then include next file into + * this candidate set. + *

    + * Default: 1 + * + * @param sizeRatio The size ratio to use + * + * @return the reference to the current options. + */ + public CompactionOptionsUniversal setSizeRatio(final int sizeRatio) { + setSizeRatio(nativeHandle_, sizeRatio); + return this; + } + + /** + * Percentage flexibility while comparing file size. If the candidate file(s) + * size is 1% smaller than the next file's size, then include next file into + * this candidate set. + *

    + * Default: 1 + * + * @return The size ratio in use + */ + public int sizeRatio() { + return sizeRatio(nativeHandle_); + } + + /** + * The minimum number of files in a single compaction run. + *

    + * Default: 2 + * + * @param minMergeWidth minimum number of files in a single compaction run + * + * @return the reference to the current options. + */ + public CompactionOptionsUniversal setMinMergeWidth(final int minMergeWidth) { + setMinMergeWidth(nativeHandle_, minMergeWidth); + return this; + } + + /** + * The minimum number of files in a single compaction run. + *

    + * Default: 2 + * + * @return minimum number of files in a single compaction run + */ + public int minMergeWidth() { + return minMergeWidth(nativeHandle_); + } + + /** + * The maximum number of files in a single compaction run. + *

    + * Default: {@link Long#MAX_VALUE} + * + * @param maxMergeWidth maximum number of files in a single compaction run + * + * @return the reference to the current options. + */ + public CompactionOptionsUniversal setMaxMergeWidth(final int maxMergeWidth) { + setMaxMergeWidth(nativeHandle_, maxMergeWidth); + return this; + } + + /** + * The maximum number of files in a single compaction run. + *

    + * Default: {@link Long#MAX_VALUE} + * + * @return maximum number of files in a single compaction run + */ + public int maxMergeWidth() { + return maxMergeWidth(nativeHandle_); + } + + /** + * The size amplification is defined as the amount (in percentage) of + * additional storage needed to store a single byte of data in the database. + * For example, a size amplification of 2% means that a database that + * contains 100 bytes of user-data may occupy upto 102 bytes of + * physical storage. By this definition, a fully compacted database has + * a size amplification of 0%. Rocksdb uses the following heuristic + * to calculate size amplification: it assumes that all files excluding + * the earliest file contribute to the size amplification. + *

    + * Default: 200, which means that a 100 byte database could require upto + * 300 bytes of storage. + * + * @param maxSizeAmplificationPercent the amount of additional storage needed + * (as a percentage) to store a single byte in the database + * + * @return the reference to the current options. + */ + public CompactionOptionsUniversal setMaxSizeAmplificationPercent( + final int maxSizeAmplificationPercent) { + setMaxSizeAmplificationPercent(nativeHandle_, maxSizeAmplificationPercent); + return this; + } + + /** + * The size amplification is defined as the amount (in percentage) of + * additional storage needed to store a single byte of data in the database. + * For example, a size amplification of 2% means that a database that + * contains 100 bytes of user-data may occupy upto 102 bytes of + * physical storage. By this definition, a fully compacted database has + * a size amplification of 0%. Rocksdb uses the following heuristic + * to calculate size amplification: it assumes that all files excluding + * the earliest file contribute to the size amplification. + *

    + * Default: 200, which means that a 100 byte database could require upto + * 300 bytes of storage. + * + * @return the amount of additional storage needed (as a percentage) to store + * a single byte in the database + */ + public int maxSizeAmplificationPercent() { + return maxSizeAmplificationPercent(nativeHandle_); + } + + /** + * If this option is set to be -1 (the default value), all the output files + * will follow compression type specified. + *

    + * If this option is not negative, we will try to make sure compressed + * size is just above this value. In normal cases, at least this percentage + * of data will be compressed. + *

    + * When we are compacting to a new file, here is the criteria whether + * it needs to be compressed: assuming here are the list of files sorted + * by generation time: + * A1...An B1...Bm C1...Ct + * where A1 is the newest and Ct is the oldest, and we are going to compact + * B1...Bm, we calculate the total size of all the files as total_size, as + * well as the total size of C1...Ct as total_C, the compaction output file + * will be compressed iff + * total_C / total_size < this percentage + *

    + * Default: -1 + * + * @param compressionSizePercent percentage of size for compression + * + * @return the reference to the current options. + */ + public CompactionOptionsUniversal setCompressionSizePercent( + final int compressionSizePercent) { + setCompressionSizePercent(nativeHandle_, compressionSizePercent); + return this; + } + + /** + * If this option is set to be -1 (the default value), all the output files + * will follow compression type specified. + *

    + * If this option is not negative, we will try to make sure compressed + * size is just above this value. In normal cases, at least this percentage + * of data will be compressed. + *

    + * When we are compacting to a new file, here is the criteria whether + * it needs to be compressed: assuming here are the list of files sorted + * by generation time: + * A1...An B1...Bm C1...Ct + * where A1 is the newest and Ct is the oldest, and we are going to compact + * B1...Bm, we calculate the total size of all the files as total_size, as + * well as the total size of C1...Ct as total_C, the compaction output file + * will be compressed iff + * total_C / total_size < this percentage + *

    + * Default: -1 + * + * @return percentage of size for compression + */ + public int compressionSizePercent() { + return compressionSizePercent(nativeHandle_); + } + + /** + * The algorithm used to stop picking files into a single compaction run + *

    + * Default: {@link CompactionStopStyle#CompactionStopStyleTotalSize} + * + * @param compactionStopStyle The compaction algorithm + * + * @return the reference to the current options. + */ + public CompactionOptionsUniversal setStopStyle( + final CompactionStopStyle compactionStopStyle) { + setStopStyle(nativeHandle_, compactionStopStyle.getValue()); + return this; + } + + /** + * The algorithm used to stop picking files into a single compaction run + *

    + * Default: {@link CompactionStopStyle#CompactionStopStyleTotalSize} + * + * @return The compaction algorithm + */ + public CompactionStopStyle stopStyle() { + return CompactionStopStyle.getCompactionStopStyle(stopStyle(nativeHandle_)); + } + + /** + * Option to optimize the universal multi level compaction by enabling + * trivial move for non overlapping files. + *

    + * Default: false + * + * @param allowTrivialMove true if trivial move is allowed + * + * @return the reference to the current options. + */ + public CompactionOptionsUniversal setAllowTrivialMove( + final boolean allowTrivialMove) { + setAllowTrivialMove(nativeHandle_, allowTrivialMove); + return this; + } + + /** + * Option to optimize the universal multi level compaction by enabling + * trivial move for non overlapping files. + *

    + * Default: false + * + * @return true if trivial move is allowed + */ + public boolean allowTrivialMove() { + return allowTrivialMove(nativeHandle_); + } + + private static native long newCompactionOptionsUniversal(); + @Override protected final native void disposeInternal(final long handle); + + private native void setSizeRatio(final long handle, final int sizeRatio); + private native int sizeRatio(final long handle); + private native void setMinMergeWidth( + final long handle, final int minMergeWidth); + private native int minMergeWidth(final long handle); + private native void setMaxMergeWidth( + final long handle, final int maxMergeWidth); + private native int maxMergeWidth(final long handle); + private native void setMaxSizeAmplificationPercent( + final long handle, final int maxSizeAmplificationPercent); + private native int maxSizeAmplificationPercent(final long handle); + private native void setCompressionSizePercent( + final long handle, final int compressionSizePercent); + private native int compressionSizePercent(final long handle); + private native void setStopStyle( + final long handle, final byte stopStyle); + private native byte stopStyle(final long handle); + private native void setAllowTrivialMove( + final long handle, final boolean allowTrivialMove); + private native boolean allowTrivialMove(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionPriority.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionPriority.java new file mode 100644 index 0000000..eda0594 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionPriority.java @@ -0,0 +1,81 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Compaction Priorities + */ +public enum CompactionPriority { + + /** + * Slightly Prioritize larger files by size compensated by #deletes + */ + ByCompensatedSize((byte)0x0), + + /** + * First compact files whose data's latest update time is oldest. + * Try this if you only update some hot keys in small ranges. + */ + OldestLargestSeqFirst((byte)0x1), + + /** + * First compact files whose range hasn't been compacted to the next level + * for the longest. If your updates are random across the key space, + * write amplification is slightly better with this option. + */ + OldestSmallestSeqFirst((byte)0x2), + + /** + * First compact files whose ratio between overlapping size in next level + * and its size is the smallest. It in many cases can optimize write + * amplification. + */ + MinOverlappingRatio((byte)0x3), + + /** + * Keeps a cursor(s) of the successor of the file (key range) was/were + * compacted before, and always picks the next files (key range) in that + * level. The file picking process will cycle through all the files in a + * round-robin manner. + */ + RoundRobin((byte)0x4); + + + private final byte value; + + CompactionPriority(final byte value) { + this.value = value; + } + + /** + * Returns the byte value of the enumerations value + * + * @return byte representation + */ + public byte getValue() { + return value; + } + + /** + * Get CompactionPriority by byte value. + * + * @param value byte representation of CompactionPriority. + * + * @return {@link org.rocksdb.CompactionPriority} instance or null. + * @throws java.lang.IllegalArgumentException if an invalid + * value is provided. + */ + public static CompactionPriority getCompactionPriority(final byte value) { + for (final CompactionPriority compactionPriority : + CompactionPriority.values()) { + if (compactionPriority.getValue() == value){ + return compactionPriority; + } + } + throw new IllegalArgumentException( + "Illegal value provided for CompactionPriority."); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionReason.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionReason.java new file mode 100644 index 0000000..46ec33f --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionReason.java @@ -0,0 +1,141 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +public enum CompactionReason { + kUnknown((byte)0x0), + + /** + * [Level] number of L0 files > level0_file_num_compaction_trigger + */ + kLevelL0FilesNum((byte)0x1), + + /** + * [Level] total size of level > MaxBytesForLevel() + */ + kLevelMaxLevelSize((byte)0x2), + + /** + * [Universal] Compacting for size amplification + */ + kUniversalSizeAmplification((byte)0x3), + + /** + * [Universal] Compacting for size ratio + */ + kUniversalSizeRatio((byte)0x4), + + /** + * [Universal] number of sorted runs > level0_file_num_compaction_trigger + */ + kUniversalSortedRunNum((byte)0x5), + + /** + * [FIFO] total size > max_table_files_size + */ + kFIFOMaxSize((byte)0x6), + + /** + * [FIFO] reduce number of files. + */ + kFIFOReduceNumFiles((byte)0x7), + + /** + * [FIFO] files with creation time < (current_time - interval) + */ + kFIFOTtl((byte)0x8), + + /** + * Manual compaction + */ + kManualCompaction((byte)0x9), + + /** + * DB::SuggestCompactRange() marked files for compaction + */ + kFilesMarkedForCompaction((byte)0x10), + + /** + * [Level] Automatic compaction within bottommost level to cleanup duplicate + * versions of same user key, usually due to a released snapshot. + */ + kBottommostFiles((byte)0x0A), + + /** + * Compaction based on TTL + */ + kTtl((byte)0x0B), + + /** + * According to the comments in flush_job.cc, RocksDB treats flush as + * a level 0 compaction in internal stats. + */ + kFlush((byte)0x0C), + + /** + * Compaction caused by external sst file ingestion + */ + kExternalSstIngestion((byte) 0x0D), + + /** + * Compaction due to SST file being too old + */ + kPeriodicCompaction((byte) 0x0E), + + /** + * Compaction in order to move files to temperature + */ + kChangeTemperature((byte) 0x0F), + + /** + * Compaction scheduled to force garbage collection of blob files + */ + kForcedBlobGC((byte) 0x11), + + /** + * A special TTL compaction for RoundRobin policy, which basically the same as + * kLevelMaxLevelSize, but the goal is to compact TTLed files. + */ + kRoundRobinTtl((byte) 0x12), + + /** + * Compaction by calling DBImpl::ReFitLevel + */ + kRefitLevel((byte) 0x13); + + private final byte value; + + CompactionReason(final byte value) { + this.value = value; + } + + /** + * Get the internal representation value. + * + * @return the internal representation value + */ + byte getValue() { + return value; + } + + /** + * Get the CompactionReason from the internal representation value. + * + * @return the compaction reason. + * + * @throws IllegalArgumentException if the value is unknown. + */ + static CompactionReason fromValue(final byte value) { + for (final CompactionReason compactionReason : CompactionReason.values()) { + if(compactionReason.value == value) { + return compactionReason; + } + } + + throw new IllegalArgumentException( + "Illegal value provided for CompactionReason: " + value); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionStopStyle.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionStopStyle.java new file mode 100644 index 0000000..f6e6320 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionStopStyle.java @@ -0,0 +1,55 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb; + +/** + * Algorithm used to make a compaction request stop picking new files + * into a single compaction run + */ +public enum CompactionStopStyle { + + /** + * Pick files of similar size + */ + CompactionStopStyleSimilarSize((byte)0x0), + + /** + * Total size of picked files > next file + */ + CompactionStopStyleTotalSize((byte)0x1); + + + private final byte value; + + CompactionStopStyle(final byte value) { + this.value = value; + } + + /** + * Returns the byte value of the enumerations value + * + * @return byte representation + */ + public byte getValue() { + return value; + } + + /** + * Get CompactionStopStyle by byte value. + * + * @param value byte representation of CompactionStopStyle. + * + * @return {@link org.rocksdb.CompactionStopStyle} instance or null. + * @throws java.lang.IllegalArgumentException if an invalid + * value is provided. + */ + public static CompactionStopStyle getCompactionStopStyle(final byte value) { + for (final CompactionStopStyle compactionStopStyle : + CompactionStopStyle.values()) { + if (compactionStopStyle.getValue() == value){ + return compactionStopStyle; + } + } + throw new IllegalArgumentException( + "Illegal value provided for CompactionStopStyle."); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionStyle.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionStyle.java new file mode 100644 index 0000000..794074d --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompactionStyle.java @@ -0,0 +1,81 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.List; + +/** + * Enum CompactionStyle + *

    + * RocksDB supports different styles of compaction. Available + * compaction styles can be chosen using this enumeration. + * + *

      + *
    1. LEVEL - Level based Compaction style
      2. + *
    2. UNIVERSAL - Universal Compaction Style is a + * compaction style, targeting the use cases requiring lower write + * amplification, trading off read amplification and space + * amplification.
      3. + *
    3. FIFO - FIFO compaction style is the simplest + * compaction strategy. It is suited for keeping event log data with + * very low overhead (query log for example). It periodically deletes + * the old data, so it's basically a TTL compaction style.
      4. + *
    4. NONE - Disable background compaction. + * Compaction jobs are submitted + * {@link RocksDB#compactFiles(CompactionOptions, ColumnFamilyHandle, List, int, int, + * CompactionJobInfo)} ()}.
      5. + *
    + * + * @see + * Universal Compaction + * @see + * FIFO Compaction + */ +public enum CompactionStyle { + LEVEL((byte) 0x0), + UNIVERSAL((byte) 0x1), + FIFO((byte) 0x2), + NONE((byte) 0x3); + + private final byte value; + + CompactionStyle(final byte value) { + this.value = value; + } + + /** + * Get the internal representation value. + * + * @return the internal representation value. + */ + //TODO(AR) should be made package-private + public byte getValue() { + return value; + } + + /** + * Get the Compaction style from the internal representation value. + * + * @param value the internal representation value. + * + * @return the Compaction style + * + * @throws IllegalArgumentException if the value does not match a + * CompactionStyle + */ + static CompactionStyle fromValue(final byte value) + throws IllegalArgumentException { + for (final CompactionStyle compactionStyle : CompactionStyle.values()) { + if (compactionStyle.value == value) { + return compactionStyle; + } + } + throw new IllegalArgumentException("Unknown value for CompactionStyle: " + + value); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ComparatorOptions.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ComparatorOptions.java new file mode 100644 index 0000000..ee5beb8 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ComparatorOptions.java @@ -0,0 +1,133 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +/** + * This class controls the behaviour + * of Java implementations of + * AbstractComparator + *

    + * Note that dispose() must be called before a ComparatorOptions + * instance becomes out-of-scope to release the allocated memory in C++. + */ +public class ComparatorOptions extends RocksObject { + public ComparatorOptions() { + super(newComparatorOptions()); + } + + /** + * Get the synchronisation type used to guard the reused buffers. + * Only used if {@link #maxReusedBufferSize()} > 0 + * Default: {@link ReusedSynchronisationType#ADAPTIVE_MUTEX} + * + * @return the synchronisation type + */ + public ReusedSynchronisationType reusedSynchronisationType() { + assert(isOwningHandle()); + return ReusedSynchronisationType.getReusedSynchronisationType( + reusedSynchronisationType(nativeHandle_)); + } + + /** + * Set the synchronisation type used to guard the reused buffers. + * Only used if {@link #maxReusedBufferSize()} > 0 + * Default: {@link ReusedSynchronisationType#ADAPTIVE_MUTEX} + * + * @param reusedSynchronisationType the synchronisation type + * + * @return the reference to the current comparator options. + */ + public ComparatorOptions setReusedSynchronisationType( + final ReusedSynchronisationType reusedSynchronisationType) { + assert (isOwningHandle()); + setReusedSynchronisationType(nativeHandle_, + reusedSynchronisationType.getValue()); + return this; + } + + /** + * Indicates if a direct byte buffer (i.e. outside the normal + * garbage-collected heap) is used, as opposed to a non-direct byte buffer + * which is a wrapper around an on-heap byte[]. + *

    + * Default: true + * + * @return true if a direct byte buffer will be used, false otherwise + */ + public boolean useDirectBuffer() { + assert(isOwningHandle()); + return useDirectBuffer(nativeHandle_); + } + + /** + * Controls whether a direct byte buffer (i.e. outside the normal + * garbage-collected heap) is used, as opposed to a non-direct byte buffer + * which is a wrapper around an on-heap byte[]. + *

    + * Default: true + * + * @param useDirectBuffer true if a direct byte buffer should be used, + * false otherwise + * @return the reference to the current comparator options. + */ + public ComparatorOptions setUseDirectBuffer(final boolean useDirectBuffer) { + assert(isOwningHandle()); + setUseDirectBuffer(nativeHandle_, useDirectBuffer); + return this; + } + + /** + * Maximum size of a buffer (in bytes) that will be reused. + * Comparators will use 5 of these buffers, + * so the retained memory size will be 5 * max_reused_buffer_size. + * When a buffer is needed for transferring data to a callback, + * if it requires less than {@code maxReuseBufferSize}, then an + * existing buffer will be reused, else a new buffer will be + * allocated just for that callback. + *

    + * Default: 64 bytes + * + * @return the maximum size of a buffer which is reused, + * or 0 if reuse is disabled + */ + public int maxReusedBufferSize() { + assert(isOwningHandle()); + return maxReusedBufferSize(nativeHandle_); + } + + /** + * Sets the maximum size of a buffer (in bytes) that will be reused. + * Comparators will use 5 of these buffers, + * so the retained memory size will be 5 * max_reused_buffer_size. + * When a buffer is needed for transferring data to a callback, + * if it requires less than {@code maxReuseBufferSize}, then an + * existing buffer will be reused, else a new buffer will be + * allocated just for that callback. + *

    + * Default: 64 bytes + * + * @param maxReusedBufferSize the maximum size for a buffer to reuse, or 0 to + * disable reuse + * + * @return the maximum size of a buffer which is reused + */ + public ComparatorOptions setMaxReusedBufferSize(final int maxReusedBufferSize) { + assert(isOwningHandle()); + setMaxReusedBufferSize(nativeHandle_, maxReusedBufferSize); + return this; + } + + private static native long newComparatorOptions(); + private native byte reusedSynchronisationType(final long handle); + private native void setReusedSynchronisationType(final long handle, + final byte reusedSynchronisationType); + private native boolean useDirectBuffer(final long handle); + private native void setUseDirectBuffer(final long handle, + final boolean useDirectBuffer); + private native int maxReusedBufferSize(final long handle); + private native void setMaxReusedBufferSize(final long handle, + final int maxReuseBufferSize); + @Override protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ComparatorType.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ComparatorType.java new file mode 100644 index 0000000..199980b --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ComparatorType.java @@ -0,0 +1,48 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +enum ComparatorType { + JAVA_COMPARATOR((byte)0x0), + JAVA_NATIVE_COMPARATOR_WRAPPER((byte)0x1); + + private final byte value; + + ComparatorType(final byte value) { + this.value = value; + } + + /** + *

    Returns the byte value of the enumerations value.

    + * + * @return byte representation + */ + byte getValue() { + return value; + } + + /** + *

    Get the ComparatorType enumeration value by + * passing the byte identifier to this method.

    + * + * @param byteIdentifier of ComparatorType. + * + * @return ComparatorType instance. + * + * @throws IllegalArgumentException if the comparator type for the byteIdentifier + * cannot be found + */ + static ComparatorType getComparatorType(final byte byteIdentifier) { + for (final ComparatorType comparatorType : ComparatorType.values()) { + if (comparatorType.getValue() == byteIdentifier) { + return comparatorType; + } + } + + throw new IllegalArgumentException( + "Illegal value provided for ComparatorType."); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompressionOptions.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompressionOptions.java new file mode 100644 index 0000000..2e1ee57 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompressionOptions.java @@ -0,0 +1,150 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Options for Compression + */ +public class CompressionOptions extends RocksObject { + + public CompressionOptions() { + super(newCompressionOptions()); + } + + public CompressionOptions setWindowBits(final int windowBits) { + setWindowBits(nativeHandle_, windowBits); + return this; + } + + public int windowBits() { + return windowBits(nativeHandle_); + } + + public CompressionOptions setLevel(final int level) { + setLevel(nativeHandle_, level); + return this; + } + + public int level() { + return level(nativeHandle_); + } + + public CompressionOptions setStrategy(final int strategy) { + setStrategy(nativeHandle_, strategy); + return this; + } + + public int strategy() { + return strategy(nativeHandle_); + } + + /** + * Maximum size of dictionary used to prime the compression library. Currently + * this dictionary will be constructed by sampling the first output file in a + * subcompaction when the target level is bottommost. This dictionary will be + * loaded into the compression library before compressing/uncompressing each + * data block of subsequent files in the subcompaction. Effectively, this + * improves compression ratios when there are repetitions across data blocks. + *

    + * A value of 0 indicates the feature is disabled. + *

    + * Default: 0. + * + * @param maxDictBytes Maximum bytes to use for the dictionary + * + * @return the reference to the current options + */ + public CompressionOptions setMaxDictBytes(final int maxDictBytes) { + setMaxDictBytes(nativeHandle_, maxDictBytes); + return this; + } + + /** + * Maximum size of dictionary used to prime the compression library. + * + * @return The maximum bytes to use for the dictionary + */ + public int maxDictBytes() { + return maxDictBytes(nativeHandle_); + } + + /** + * Maximum size of training data passed to zstd's dictionary trainer. Using + * zstd's dictionary trainer can achieve even better compression ratio + * improvements than using {@link #setMaxDictBytes(int)} alone. + *

    + * The training data will be used to generate a dictionary + * of {@link #maxDictBytes()}. + *

    + * Default: 0. + * + * @param zstdMaxTrainBytes Maximum bytes to use for training ZStd. + * + * @return the reference to the current options + */ + public CompressionOptions setZStdMaxTrainBytes(final int zstdMaxTrainBytes) { + setZstdMaxTrainBytes(nativeHandle_, zstdMaxTrainBytes); + return this; + } + + /** + * Maximum size of training data passed to zstd's dictionary trainer. + * + * @return Maximum bytes to use for training ZStd + */ + public int zstdMaxTrainBytes() { + return zstdMaxTrainBytes(nativeHandle_); + } + + /** + * When the compression options are set by the user, it will be set to "true". + * For bottommost_compression_opts, to enable it, user must set enabled=true. + * Otherwise, bottommost compression will use compression_opts as default + * compression options. + *

    + * For compression_opts, if compression_opts.enabled=false, it is still + * used as compression options for compression process. + *

    + * Default: false. + * + * @param enabled true to use these compression options + * for the bottommost_compression_opts, false otherwise + * + * @return the reference to the current options + */ + public CompressionOptions setEnabled(final boolean enabled) { + setEnabled(nativeHandle_, enabled); + return this; + } + + /** + * Determine whether these compression options + * are used for the bottommost_compression_opts. + * + * @return true if these compression options are used + * for the bottommost_compression_opts, false otherwise + */ + public boolean enabled() { + return enabled(nativeHandle_); + } + + private static native long newCompressionOptions(); + @Override protected final native void disposeInternal(final long handle); + + private native void setWindowBits(final long handle, final int windowBits); + private native int windowBits(final long handle); + private native void setLevel(final long handle, final int level); + private native int level(final long handle); + private native void setStrategy(final long handle, final int strategy); + private native int strategy(final long handle); + private native void setMaxDictBytes(final long handle, final int maxDictBytes); + private native int maxDictBytes(final long handle); + private native void setZstdMaxTrainBytes(final long handle, + final int zstdMaxTrainBytes); + private native int zstdMaxTrainBytes(final long handle); + private native void setEnabled(final long handle, final boolean enabled); + private native boolean enabled(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompressionType.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompressionType.java new file mode 100644 index 0000000..d1ecf0a --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/CompressionType.java @@ -0,0 +1,121 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Enum CompressionType + * + *

    DB contents are stored in a set of blocks, each of which holds a + * sequence of key,value pairs. Each block may be compressed before + * being stored in a file. The following enum describes which + * compression method (if any) is used to compress a block.

    + */ +public enum CompressionType { + NO_COMPRESSION((byte) 0x0, null, "kNoCompression"), + SNAPPY_COMPRESSION((byte) 0x1, "snappy", "kSnappyCompression"), + ZLIB_COMPRESSION((byte) 0x2, "z", "kZlibCompression"), + BZLIB2_COMPRESSION((byte) 0x3, "bzip2", "kBZip2Compression"), + LZ4_COMPRESSION((byte) 0x4, "lz4", "kLZ4Compression"), + LZ4HC_COMPRESSION((byte) 0x5, "lz4hc", "kLZ4HCCompression"), + XPRESS_COMPRESSION((byte) 0x6, "xpress", "kXpressCompression"), + ZSTD_COMPRESSION((byte) 0x7, "zstd", "kZSTD"), + DISABLE_COMPRESSION_OPTION((byte) 0x7F, null, "kDisableCompressionOption"); + + /** + *

    Get the CompressionType enumeration value by + * passing the library name to this method.

    + * + *

    If library cannot be found the enumeration + * value {@code NO_COMPRESSION} will be returned.

    + * + * @param libraryName compression library name. + * + * @return CompressionType instance. + */ + public static CompressionType getCompressionType(final String libraryName) { + if (libraryName != null) { + for (final CompressionType compressionType : CompressionType.values()) { + if (compressionType.getLibraryName() != null && + compressionType.getLibraryName().equals(libraryName)) { + return compressionType; + } + } + } + return CompressionType.NO_COMPRESSION; + } + + /** + *

    Get the CompressionType enumeration value by + * passing the byte identifier to this method.

    + * + * @param byteIdentifier of CompressionType. + * + * @return CompressionType instance. + * + * @throws IllegalArgumentException If CompressionType cannot be found for the + * provided byteIdentifier + */ + public static CompressionType getCompressionType(final byte byteIdentifier) { + for (final CompressionType compressionType : CompressionType.values()) { + if (compressionType.getValue() == byteIdentifier) { + return compressionType; + } + } + + throw new IllegalArgumentException( + "Illegal value provided for CompressionType."); + } + + /** + *

    Get a CompressionType value based on the string key in the C++ options output. + * This gets used in support of getting options into Java from an options string, + * which is generated at the C++ level. + *

    + * + * @param internalName the internal (C++) name by which the option is known. + * + * @return CompressionType instance (optional) + */ + static CompressionType getFromInternal(final String internalName) { + for (final CompressionType compressionType : CompressionType.values()) { + if (compressionType.internalName_.equals(internalName)) { + return compressionType; + } + } + + throw new IllegalArgumentException( + "Illegal internalName '" + internalName + " ' provided for CompressionType."); + } + + /** + *

    Returns the byte value of the enumerations value.

    + * + * @return byte representation + */ + public byte getValue() { + return value_; + } + + /** + *

    Returns the library name of the compression type + * identified by the enumeration value.

    + * + * @return library name + */ + public String getLibraryName() { + return libraryName_; + } + + CompressionType(final byte value, final String libraryName, final String internalName) { + value_ = value; + libraryName_ = libraryName; + internalName_ = internalName; + } + + private final byte value_; + private final String libraryName_; + private final String internalName_; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ConcurrentTaskLimiter.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ConcurrentTaskLimiter.java new file mode 100644 index 0000000..b4e3430 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ConcurrentTaskLimiter.java @@ -0,0 +1,44 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +public abstract class ConcurrentTaskLimiter extends RocksObject { + protected ConcurrentTaskLimiter(final long nativeHandle) { + super(nativeHandle); + } + + /** + * Returns a name that identifies this concurrent task limiter. + * + * @return Concurrent task limiter name. + */ + public abstract String name(); + + /** + * Set max concurrent tasks.
    + * limit = 0 means no new task allowed.
    + * limit < 0 means no limitation. + * + * @param maxOutstandinsTask max concurrent tasks. + * @return the reference to the current instance of ConcurrentTaskLimiter. + */ + public abstract ConcurrentTaskLimiter setMaxOutstandingTask(final int maxOutstandinsTask); + + /** + * Reset to unlimited max concurrent task. + * + * @return the reference to the current instance of ConcurrentTaskLimiter. + */ + public abstract ConcurrentTaskLimiter resetMaxOutstandingTask(); + + /** + * Returns current outstanding task count. + * + * @return current outstanding task count. + */ + public abstract int outstandingTask(); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ConcurrentTaskLimiterImpl.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ConcurrentTaskLimiterImpl.java new file mode 100644 index 0000000..d28b906 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ConcurrentTaskLimiterImpl.java @@ -0,0 +1,48 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +public class ConcurrentTaskLimiterImpl extends ConcurrentTaskLimiter { + public ConcurrentTaskLimiterImpl(final String name, final int maxOutstandingTask) { + super(newConcurrentTaskLimiterImpl0(name, maxOutstandingTask)); + } + + @Override + public String name() { + assert (isOwningHandle()); + return name(nativeHandle_); + } + + @Override + public ConcurrentTaskLimiter setMaxOutstandingTask(final int maxOutstandingTask) { + assert (isOwningHandle()); + setMaxOutstandingTask(nativeHandle_, maxOutstandingTask); + return this; + } + + @Override + public ConcurrentTaskLimiter resetMaxOutstandingTask() { + assert (isOwningHandle()); + resetMaxOutstandingTask(nativeHandle_); + return this; + } + + @Override + public int outstandingTask() { + assert (isOwningHandle()); + return outstandingTask(nativeHandle_); + } + + private static native long newConcurrentTaskLimiterImpl0( + final String name, final int maxOutstandingTask); + private static native String name(final long handle); + private static native void setMaxOutstandingTask(final long handle, final int limit); + private static native void resetMaxOutstandingTask(final long handle); + private static native int outstandingTask(final long handle); + + @Override protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ConfigOptions.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ConfigOptions.java new file mode 100644 index 0000000..026f8b0 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ConfigOptions.java @@ -0,0 +1,53 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +public class ConfigOptions extends RocksObject { + static { + RocksDB.loadLibrary(); + } + + /** + * Construct with default Options + */ + public ConfigOptions() { + super(newConfigOptions()); + } + + public ConfigOptions setDelimiter(final String delimiter) { + setDelimiter(nativeHandle_, delimiter); + return this; + } + public ConfigOptions setIgnoreUnknownOptions(final boolean ignore) { + setIgnoreUnknownOptions(nativeHandle_, ignore); + return this; + } + + public ConfigOptions setEnv(final Env env) { + setEnv(nativeHandle_, env.nativeHandle_); + return this; + } + + public ConfigOptions setInputStringsEscaped(final boolean escaped) { + setInputStringsEscaped(nativeHandle_, escaped); + return this; + } + + public ConfigOptions setSanityLevel(final SanityLevel level) { + setSanityLevel(nativeHandle_, level.getValue()); + return this; + } + + @Override protected final native void disposeInternal(final long handle); + + private static native long newConfigOptions(); + private static native void setEnv(final long handle, final long envHandle); + private static native void setDelimiter(final long handle, final String delimiter); + private static native void setIgnoreUnknownOptions(final long handle, final boolean ignore); + private static native void setInputStringsEscaped(final long handle, final boolean escaped); + private static native void setSanityLevel(final long handle, final byte level); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/DBOptions.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/DBOptions.java new file mode 100644 index 0000000..62ad137 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/DBOptions.java @@ -0,0 +1,1498 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.nio.file.Paths; +import java.util.*; + +/** + * DBOptions to control the behavior of a database. It will be used + * during the creation of a {@link org.rocksdb.RocksDB} (i.e., RocksDB.open()). + *

    + * As a descendent of {@link AbstractNativeReference}, this class is {@link AutoCloseable} + * and will be automatically released if opened in the preamble of a try with resources block. + */ +public class DBOptions extends RocksObject + implements DBOptionsInterface, + MutableDBOptionsInterface { + static { + RocksDB.loadLibrary(); + } + + /** + * Construct DBOptions. + *

    + * This constructor will create (by allocating a block of memory) + * an {@code rocksdb::DBOptions} in the c++ side. + */ + public DBOptions() { + super(newDBOptions()); + numShardBits_ = DEFAULT_NUM_SHARD_BITS; + env_ = Env.getDefault(); + } + + /** + * Copy constructor for DBOptions. + *

    + * NOTE: This does a shallow copy, which means env, rate_limiter, sst_file_manager, + * info_log and other pointers will be cloned! + * + * @param other The DBOptions to copy. + */ + public DBOptions(final DBOptions other) { + super(copyDBOptions(other.nativeHandle_)); + this.env_ = other.env_; + this.numShardBits_ = other.numShardBits_; + this.rateLimiter_ = other.rateLimiter_; + this.rowCache_ = other.rowCache_; + this.walFilter_ = other.walFilter_; + this.writeBufferManager_ = other.writeBufferManager_; + } + + /** + * Constructor from Options + * + * @param options The options. + */ + public DBOptions(final Options options) { + super(newDBOptionsFromOptions(options.nativeHandle_)); + } + + /** + *

    Method to get a options instance by using pre-configured + * property values. If one or many values are undefined in + * the context of RocksDB the method will return a null + * value.

    + * + *

    Note: Property keys can be derived from + * getter methods within the options class. Example: the method + * {@code allowMmapReads()} has a property key: + * {@code allow_mmap_reads}.

    + * + * @param cfgOpts The ConfigOptions to control how the string is processed. + * @param properties {@link java.util.Properties} instance. + * + * @return {@link org.rocksdb.DBOptions instance} + * or null. + * + * @throws java.lang.IllegalArgumentException if null or empty + * {@link java.util.Properties} instance is passed to the method call. + */ + public static DBOptions getDBOptionsFromProps( + final ConfigOptions cfgOpts, final Properties properties) { + DBOptions dbOptions = null; + final String optionsString = Options.getOptionStringFromProps(properties); + final long handle = getDBOptionsFromProps(cfgOpts.nativeHandle_, optionsString); + if (handle != 0) { + dbOptions = new DBOptions(handle); + } + return dbOptions; + } + + /** + *

    Method to get a options instance by using pre-configured + * property values. If one or many values are undefined in + * the context of RocksDB the method will return a null + * value.

    + * + *

    Note: Property keys can be derived from + * getter methods within the options class. Example: the method + * {@code allowMmapReads()} has a property key: + * {@code allow_mmap_reads}.

    + * + * @param properties {@link java.util.Properties} instance. + * + * @return {@link org.rocksdb.DBOptions instance} + * or null. + * + * @throws java.lang.IllegalArgumentException if null or empty + * {@link java.util.Properties} instance is passed to the method call. + */ + public static DBOptions getDBOptionsFromProps(final Properties properties) { + DBOptions dbOptions = null; + final String optionsString = Options.getOptionStringFromProps(properties); + final long handle = getDBOptionsFromProps(optionsString); + if (handle != 0) { + dbOptions = new DBOptions(handle); + } + return dbOptions; + } + + @Override + public DBOptions optimizeForSmallDb() { + optimizeForSmallDb(nativeHandle_); + return this; + } + + @Override + public DBOptions setIncreaseParallelism( + final int totalThreads) { + assert(isOwningHandle()); + setIncreaseParallelism(nativeHandle_, totalThreads); + return this; + } + + @Override + public DBOptions setCreateIfMissing(final boolean flag) { + assert(isOwningHandle()); + setCreateIfMissing(nativeHandle_, flag); + return this; + } + + @Override + public boolean createIfMissing() { + assert(isOwningHandle()); + return createIfMissing(nativeHandle_); + } + + @Override + public DBOptions setCreateMissingColumnFamilies( + final boolean flag) { + assert(isOwningHandle()); + setCreateMissingColumnFamilies(nativeHandle_, flag); + return this; + } + + @Override + public boolean createMissingColumnFamilies() { + assert(isOwningHandle()); + return createMissingColumnFamilies(nativeHandle_); + } + + @Override + public DBOptions setErrorIfExists( + final boolean errorIfExists) { + assert(isOwningHandle()); + setErrorIfExists(nativeHandle_, errorIfExists); + return this; + } + + @Override + public boolean errorIfExists() { + assert(isOwningHandle()); + return errorIfExists(nativeHandle_); + } + + @Override + public DBOptions setParanoidChecks( + final boolean paranoidChecks) { + assert(isOwningHandle()); + setParanoidChecks(nativeHandle_, paranoidChecks); + return this; + } + + @Override + public boolean paranoidChecks() { + assert(isOwningHandle()); + return paranoidChecks(nativeHandle_); + } + + @Override + public DBOptions setEnv(final Env env) { + setEnv(nativeHandle_, env.nativeHandle_); + this.env_ = env; + return this; + } + + @Override + public Env getEnv() { + return env_; + } + + @Override + public DBOptions setRateLimiter(final RateLimiter rateLimiter) { + assert(isOwningHandle()); + rateLimiter_ = rateLimiter; + setRateLimiter(nativeHandle_, rateLimiter.nativeHandle_); + return this; + } + + @Override + public DBOptions setSstFileManager(final SstFileManager sstFileManager) { + assert(isOwningHandle()); + setSstFileManager(nativeHandle_, sstFileManager.nativeHandle_); + return this; + } + + @Override + public DBOptions setLogger(final Logger logger) { + assert(isOwningHandle()); + setLogger(nativeHandle_, logger.nativeHandle_); + return this; + } + + @Override + public DBOptions setInfoLogLevel( + final InfoLogLevel infoLogLevel) { + assert(isOwningHandle()); + setInfoLogLevel(nativeHandle_, infoLogLevel.getValue()); + return this; + } + + @Override + public InfoLogLevel infoLogLevel() { + assert(isOwningHandle()); + return InfoLogLevel.getInfoLogLevel( + infoLogLevel(nativeHandle_)); + } + + @Override + public DBOptions setMaxOpenFiles( + final int maxOpenFiles) { + assert(isOwningHandle()); + setMaxOpenFiles(nativeHandle_, maxOpenFiles); + return this; + } + + @Override + public int maxOpenFiles() { + assert(isOwningHandle()); + return maxOpenFiles(nativeHandle_); + } + + @Override + public DBOptions setMaxFileOpeningThreads(final int maxFileOpeningThreads) { + assert(isOwningHandle()); + setMaxFileOpeningThreads(nativeHandle_, maxFileOpeningThreads); + return this; + } + + @Override + public int maxFileOpeningThreads() { + assert(isOwningHandle()); + return maxFileOpeningThreads(nativeHandle_); + } + + @Override + public DBOptions setMaxTotalWalSize( + final long maxTotalWalSize) { + assert(isOwningHandle()); + setMaxTotalWalSize(nativeHandle_, maxTotalWalSize); + return this; + } + + @Override + public long maxTotalWalSize() { + assert(isOwningHandle()); + return maxTotalWalSize(nativeHandle_); + } + + @Override + public DBOptions setStatistics(final Statistics statistics) { + assert(isOwningHandle()); + setStatistics(nativeHandle_, statistics.nativeHandle_); + return this; + } + + @Override + public Statistics statistics() { + assert(isOwningHandle()); + final long statisticsNativeHandle = statistics(nativeHandle_); + if(statisticsNativeHandle == 0) { + return null; + } else { + return new Statistics(statisticsNativeHandle); + } + } + + @Override + public DBOptions setUseFsync( + final boolean useFsync) { + assert(isOwningHandle()); + setUseFsync(nativeHandle_, useFsync); + return this; + } + + @Override + public boolean useFsync() { + assert(isOwningHandle()); + return useFsync(nativeHandle_); + } + + @Override + public DBOptions setDbPaths(final Collection dbPaths) { + assert(isOwningHandle()); + + final int len = dbPaths.size(); + final String[] paths = new String[len]; + final long[] targetSizes = new long[len]; + + int i = 0; + for(final DbPath dbPath : dbPaths) { + paths[i] = dbPath.path.toString(); + targetSizes[i] = dbPath.targetSize; + i++; + } + setDbPaths(nativeHandle_, paths, targetSizes); + return this; + } + + @Override + public List dbPaths() { + final int len = (int)dbPathsLen(nativeHandle_); + if(len == 0) { + return Collections.emptyList(); + } else { + final String[] paths = new String[len]; + final long[] targetSizes = new long[len]; + + dbPaths(nativeHandle_, paths, targetSizes); + + final List dbPaths = new ArrayList<>(); + for(int i = 0; i < len; i++) { + dbPaths.add(new DbPath(Paths.get(paths[i]), targetSizes[i])); + } + return dbPaths; + } + } + + @Override + public DBOptions setDbLogDir( + final String dbLogDir) { + assert(isOwningHandle()); + setDbLogDir(nativeHandle_, dbLogDir); + return this; + } + + @Override + public String dbLogDir() { + assert(isOwningHandle()); + return dbLogDir(nativeHandle_); + } + + @Override + public DBOptions setWalDir( + final String walDir) { + assert(isOwningHandle()); + setWalDir(nativeHandle_, walDir); + return this; + } + + @Override + public String walDir() { + assert(isOwningHandle()); + return walDir(nativeHandle_); + } + + @Override + public DBOptions setDeleteObsoleteFilesPeriodMicros( + final long micros) { + assert(isOwningHandle()); + setDeleteObsoleteFilesPeriodMicros(nativeHandle_, micros); + return this; + } + + @Override + public long deleteObsoleteFilesPeriodMicros() { + assert(isOwningHandle()); + return deleteObsoleteFilesPeriodMicros(nativeHandle_); + } + + @Override + public DBOptions setMaxBackgroundJobs(final int maxBackgroundJobs) { + assert(isOwningHandle()); + setMaxBackgroundJobs(nativeHandle_, maxBackgroundJobs); + return this; + } + + @Override + public int maxBackgroundJobs() { + assert(isOwningHandle()); + return maxBackgroundJobs(nativeHandle_); + } + + @Override + @Deprecated + public DBOptions setMaxBackgroundCompactions( + final int maxBackgroundCompactions) { + assert(isOwningHandle()); + setMaxBackgroundCompactions(nativeHandle_, maxBackgroundCompactions); + return this; + } + + @Override + @Deprecated + public int maxBackgroundCompactions() { + assert(isOwningHandle()); + return maxBackgroundCompactions(nativeHandle_); + } + + @Override + public DBOptions setMaxSubcompactions(final int maxSubcompactions) { + assert(isOwningHandle()); + setMaxSubcompactions(nativeHandle_, maxSubcompactions); + return this; + } + + @Override + public int maxSubcompactions() { + assert(isOwningHandle()); + return maxSubcompactions(nativeHandle_); + } + + @Override + @Deprecated + public DBOptions setMaxBackgroundFlushes( + final int maxBackgroundFlushes) { + assert(isOwningHandle()); + setMaxBackgroundFlushes(nativeHandle_, maxBackgroundFlushes); + return this; + } + + @Override + @Deprecated + public int maxBackgroundFlushes() { + assert(isOwningHandle()); + return maxBackgroundFlushes(nativeHandle_); + } + + @Override + public DBOptions setMaxLogFileSize(final long maxLogFileSize) { + assert(isOwningHandle()); + setMaxLogFileSize(nativeHandle_, maxLogFileSize); + return this; + } + + @Override + public long maxLogFileSize() { + assert(isOwningHandle()); + return maxLogFileSize(nativeHandle_); + } + + @Override + public DBOptions setLogFileTimeToRoll( + final long logFileTimeToRoll) { + assert(isOwningHandle()); + setLogFileTimeToRoll(nativeHandle_, logFileTimeToRoll); + return this; + } + + @Override + public long logFileTimeToRoll() { + assert(isOwningHandle()); + return logFileTimeToRoll(nativeHandle_); + } + + @Override + public DBOptions setKeepLogFileNum( + final long keepLogFileNum) { + assert(isOwningHandle()); + setKeepLogFileNum(nativeHandle_, keepLogFileNum); + return this; + } + + @Override + public long keepLogFileNum() { + assert(isOwningHandle()); + return keepLogFileNum(nativeHandle_); + } + + @Override + public DBOptions setRecycleLogFileNum(final long recycleLogFileNum) { + assert(isOwningHandle()); + setRecycleLogFileNum(nativeHandle_, recycleLogFileNum); + return this; + } + + @Override + public long recycleLogFileNum() { + assert(isOwningHandle()); + return recycleLogFileNum(nativeHandle_); + } + + @Override + public DBOptions setMaxManifestFileSize( + final long maxManifestFileSize) { + assert(isOwningHandle()); + setMaxManifestFileSize(nativeHandle_, maxManifestFileSize); + return this; + } + + @Override + public long maxManifestFileSize() { + assert(isOwningHandle()); + return maxManifestFileSize(nativeHandle_); + } + + @Override + public DBOptions setTableCacheNumshardbits( + final int tableCacheNumshardbits) { + assert(isOwningHandle()); + setTableCacheNumshardbits(nativeHandle_, tableCacheNumshardbits); + return this; + } + + @Override + public int tableCacheNumshardbits() { + assert(isOwningHandle()); + return tableCacheNumshardbits(nativeHandle_); + } + + @Override + public DBOptions setWalTtlSeconds( + final long walTtlSeconds) { + assert(isOwningHandle()); + setWalTtlSeconds(nativeHandle_, walTtlSeconds); + return this; + } + + @Override + public long walTtlSeconds() { + assert(isOwningHandle()); + return walTtlSeconds(nativeHandle_); + } + + @Override + public DBOptions setWalSizeLimitMB( + final long sizeLimitMB) { + assert(isOwningHandle()); + setWalSizeLimitMB(nativeHandle_, sizeLimitMB); + return this; + } + + @Override + public long walSizeLimitMB() { + assert(isOwningHandle()); + return walSizeLimitMB(nativeHandle_); + } + + @Override + public DBOptions setMaxWriteBatchGroupSizeBytes(final long maxWriteBatchGroupSizeBytes) { + setMaxWriteBatchGroupSizeBytes(nativeHandle_, maxWriteBatchGroupSizeBytes); + return this; + } + + @Override + public long maxWriteBatchGroupSizeBytes() { + assert (isOwningHandle()); + return maxWriteBatchGroupSizeBytes(nativeHandle_); + } + + @Override + public DBOptions setManifestPreallocationSize( + final long size) { + assert(isOwningHandle()); + setManifestPreallocationSize(nativeHandle_, size); + return this; + } + + @Override + public long manifestPreallocationSize() { + assert(isOwningHandle()); + return manifestPreallocationSize(nativeHandle_); + } + + @Override + public DBOptions setAllowMmapReads( + final boolean allowMmapReads) { + assert(isOwningHandle()); + setAllowMmapReads(nativeHandle_, allowMmapReads); + return this; + } + + @Override + public boolean allowMmapReads() { + assert(isOwningHandle()); + return allowMmapReads(nativeHandle_); + } + + @Override + public DBOptions setAllowMmapWrites( + final boolean allowMmapWrites) { + assert(isOwningHandle()); + setAllowMmapWrites(nativeHandle_, allowMmapWrites); + return this; + } + + @Override + public boolean allowMmapWrites() { + assert(isOwningHandle()); + return allowMmapWrites(nativeHandle_); + } + + @Override + public DBOptions setUseDirectReads( + final boolean useDirectReads) { + assert(isOwningHandle()); + setUseDirectReads(nativeHandle_, useDirectReads); + return this; + } + + @Override + public boolean useDirectReads() { + assert(isOwningHandle()); + return useDirectReads(nativeHandle_); + } + + @Override + public DBOptions setUseDirectIoForFlushAndCompaction( + final boolean useDirectIoForFlushAndCompaction) { + assert(isOwningHandle()); + setUseDirectIoForFlushAndCompaction(nativeHandle_, + useDirectIoForFlushAndCompaction); + return this; + } + + @Override + public boolean useDirectIoForFlushAndCompaction() { + assert(isOwningHandle()); + return useDirectIoForFlushAndCompaction(nativeHandle_); + } + + @Override + public DBOptions setAllowFAllocate(final boolean allowFAllocate) { + assert(isOwningHandle()); + setAllowFAllocate(nativeHandle_, allowFAllocate); + return this; + } + + @Override + public boolean allowFAllocate() { + assert(isOwningHandle()); + return allowFAllocate(nativeHandle_); + } + + @Override + public DBOptions setIsFdCloseOnExec( + final boolean isFdCloseOnExec) { + assert(isOwningHandle()); + setIsFdCloseOnExec(nativeHandle_, isFdCloseOnExec); + return this; + } + + @Override + public boolean isFdCloseOnExec() { + assert(isOwningHandle()); + return isFdCloseOnExec(nativeHandle_); + } + + @Override + public DBOptions setStatsDumpPeriodSec( + final int statsDumpPeriodSec) { + assert(isOwningHandle()); + setStatsDumpPeriodSec(nativeHandle_, statsDumpPeriodSec); + return this; + } + + @Override + public int statsDumpPeriodSec() { + assert(isOwningHandle()); + return statsDumpPeriodSec(nativeHandle_); + } + + @Override + public DBOptions setStatsPersistPeriodSec( + final int statsPersistPeriodSec) { + assert(isOwningHandle()); + setStatsPersistPeriodSec(nativeHandle_, statsPersistPeriodSec); + return this; + } + + @Override + public int statsPersistPeriodSec() { + assert(isOwningHandle()); + return statsPersistPeriodSec(nativeHandle_); + } + + @Override + public DBOptions setStatsHistoryBufferSize( + final long statsHistoryBufferSize) { + assert(isOwningHandle()); + setStatsHistoryBufferSize(nativeHandle_, statsHistoryBufferSize); + return this; + } + + @Override + public long statsHistoryBufferSize() { + assert(isOwningHandle()); + return statsHistoryBufferSize(nativeHandle_); + } + + @Override + public DBOptions setAdviseRandomOnOpen( + final boolean adviseRandomOnOpen) { + assert(isOwningHandle()); + setAdviseRandomOnOpen(nativeHandle_, adviseRandomOnOpen); + return this; + } + + @Override + public boolean adviseRandomOnOpen() { + return adviseRandomOnOpen(nativeHandle_); + } + + @Override + public DBOptions setDbWriteBufferSize(final long dbWriteBufferSize) { + assert(isOwningHandle()); + setDbWriteBufferSize(nativeHandle_, dbWriteBufferSize); + return this; + } + + @Override + public DBOptions setWriteBufferManager(final WriteBufferManager writeBufferManager) { + assert(isOwningHandle()); + setWriteBufferManager(nativeHandle_, writeBufferManager.nativeHandle_); + this.writeBufferManager_ = writeBufferManager; + return this; + } + + @Override + public WriteBufferManager writeBufferManager() { + assert(isOwningHandle()); + return this.writeBufferManager_; + } + + @Override + public long dbWriteBufferSize() { + assert(isOwningHandle()); + return dbWriteBufferSize(nativeHandle_); + } + + @Override + @Deprecated + public DBOptions setAccessHintOnCompactionStart(final AccessHint accessHint) { + assert(isOwningHandle()); + setAccessHintOnCompactionStart(nativeHandle_, accessHint.getValue()); + return this; + } + + @Override + @Deprecated + public AccessHint accessHintOnCompactionStart() { + assert(isOwningHandle()); + return AccessHint.getAccessHint(accessHintOnCompactionStart(nativeHandle_)); + } + + @Override + public DBOptions setCompactionReadaheadSize(final long compactionReadaheadSize) { + assert(isOwningHandle()); + setCompactionReadaheadSize(nativeHandle_, compactionReadaheadSize); + return this; + } + + @Override + public long compactionReadaheadSize() { + assert(isOwningHandle()); + return compactionReadaheadSize(nativeHandle_); + } + + @Override + public DBOptions setRandomAccessMaxBufferSize(final long randomAccessMaxBufferSize) { + assert(isOwningHandle()); + setRandomAccessMaxBufferSize(nativeHandle_, randomAccessMaxBufferSize); + return this; + } + + @Override + public long randomAccessMaxBufferSize() { + assert(isOwningHandle()); + return randomAccessMaxBufferSize(nativeHandle_); + } + + @Override + public DBOptions setWritableFileMaxBufferSize(final long writableFileMaxBufferSize) { + assert(isOwningHandle()); + setWritableFileMaxBufferSize(nativeHandle_, writableFileMaxBufferSize); + return this; + } + + @Override + public long writableFileMaxBufferSize() { + assert(isOwningHandle()); + return writableFileMaxBufferSize(nativeHandle_); + } + + @Override + public DBOptions setUseAdaptiveMutex( + final boolean useAdaptiveMutex) { + assert(isOwningHandle()); + setUseAdaptiveMutex(nativeHandle_, useAdaptiveMutex); + return this; + } + + @Override + public boolean useAdaptiveMutex() { + assert(isOwningHandle()); + return useAdaptiveMutex(nativeHandle_); + } + + @Override + public DBOptions setBytesPerSync( + final long bytesPerSync) { + assert(isOwningHandle()); + setBytesPerSync(nativeHandle_, bytesPerSync); + return this; + } + + @Override + public long bytesPerSync() { + return bytesPerSync(nativeHandle_); + } + + @Override + public DBOptions setWalBytesPerSync(final long walBytesPerSync) { + assert(isOwningHandle()); + setWalBytesPerSync(nativeHandle_, walBytesPerSync); + return this; + } + + @Override + public long walBytesPerSync() { + assert(isOwningHandle()); + return walBytesPerSync(nativeHandle_); + } + + @Override + public DBOptions setStrictBytesPerSync(final boolean strictBytesPerSync) { + assert(isOwningHandle()); + setStrictBytesPerSync(nativeHandle_, strictBytesPerSync); + return this; + } + + @Override + public boolean strictBytesPerSync() { + assert(isOwningHandle()); + return strictBytesPerSync(nativeHandle_); + } + + @Override + public DBOptions setListeners(final List listeners) { + assert (isOwningHandle()); + setEventListeners(nativeHandle_, RocksCallbackObject.toNativeHandleList(listeners)); + return this; + } + + @Override + public List listeners() { + assert (isOwningHandle()); + return Arrays.asList(eventListeners(nativeHandle_)); + } + + @Override + public DBOptions setEnableThreadTracking(final boolean enableThreadTracking) { + assert(isOwningHandle()); + setEnableThreadTracking(nativeHandle_, enableThreadTracking); + return this; + } + + @Override + public boolean enableThreadTracking() { + assert(isOwningHandle()); + return enableThreadTracking(nativeHandle_); + } + + @Override + public DBOptions setDelayedWriteRate(final long delayedWriteRate) { + assert(isOwningHandle()); + setDelayedWriteRate(nativeHandle_, delayedWriteRate); + return this; + } + + @Override + public long delayedWriteRate(){ + return delayedWriteRate(nativeHandle_); + } + + @Override + public DBOptions setEnablePipelinedWrite(final boolean enablePipelinedWrite) { + assert(isOwningHandle()); + setEnablePipelinedWrite(nativeHandle_, enablePipelinedWrite); + return this; + } + + @Override + public boolean enablePipelinedWrite() { + assert(isOwningHandle()); + return enablePipelinedWrite(nativeHandle_); + } + + @Override + public DBOptions setUnorderedWrite(final boolean unorderedWrite) { + setUnorderedWrite(nativeHandle_, unorderedWrite); + return this; + } + + @Override + public boolean unorderedWrite() { + return unorderedWrite(nativeHandle_); + } + + + @Override + public DBOptions setAllowConcurrentMemtableWrite( + final boolean allowConcurrentMemtableWrite) { + setAllowConcurrentMemtableWrite(nativeHandle_, + allowConcurrentMemtableWrite); + return this; + } + + @Override + public boolean allowConcurrentMemtableWrite() { + return allowConcurrentMemtableWrite(nativeHandle_); + } + + @Override + public DBOptions setEnableWriteThreadAdaptiveYield( + final boolean enableWriteThreadAdaptiveYield) { + setEnableWriteThreadAdaptiveYield(nativeHandle_, + enableWriteThreadAdaptiveYield); + return this; + } + + @Override + public boolean enableWriteThreadAdaptiveYield() { + return enableWriteThreadAdaptiveYield(nativeHandle_); + } + + @Override + public DBOptions setWriteThreadMaxYieldUsec(final long writeThreadMaxYieldUsec) { + setWriteThreadMaxYieldUsec(nativeHandle_, writeThreadMaxYieldUsec); + return this; + } + + @Override + public long writeThreadMaxYieldUsec() { + return writeThreadMaxYieldUsec(nativeHandle_); + } + + @Override + public DBOptions setWriteThreadSlowYieldUsec(final long writeThreadSlowYieldUsec) { + setWriteThreadSlowYieldUsec(nativeHandle_, writeThreadSlowYieldUsec); + return this; + } + + @Override + public long writeThreadSlowYieldUsec() { + return writeThreadSlowYieldUsec(nativeHandle_); + } + + @Override + public DBOptions setSkipStatsUpdateOnDbOpen(final boolean skipStatsUpdateOnDbOpen) { + assert(isOwningHandle()); + setSkipStatsUpdateOnDbOpen(nativeHandle_, skipStatsUpdateOnDbOpen); + return this; + } + + @Override + public boolean skipStatsUpdateOnDbOpen() { + assert(isOwningHandle()); + return skipStatsUpdateOnDbOpen(nativeHandle_); + } + + @Override + public DBOptions setSkipCheckingSstFileSizesOnDbOpen( + final boolean skipCheckingSstFileSizesOnDbOpen) { + setSkipCheckingSstFileSizesOnDbOpen(nativeHandle_, skipCheckingSstFileSizesOnDbOpen); + return this; + } + + @Override + public boolean skipCheckingSstFileSizesOnDbOpen() { + assert (isOwningHandle()); + return skipCheckingSstFileSizesOnDbOpen(nativeHandle_); + } + + @Override + public DBOptions setWalRecoveryMode(final WALRecoveryMode walRecoveryMode) { + assert(isOwningHandle()); + setWalRecoveryMode(nativeHandle_, walRecoveryMode.getValue()); + return this; + } + + @Override + public WALRecoveryMode walRecoveryMode() { + assert(isOwningHandle()); + return WALRecoveryMode.getWALRecoveryMode(walRecoveryMode(nativeHandle_)); + } + + @Override + public DBOptions setAllow2pc(final boolean allow2pc) { + assert(isOwningHandle()); + setAllow2pc(nativeHandle_, allow2pc); + return this; + } + + @Override + public boolean allow2pc() { + assert(isOwningHandle()); + return allow2pc(nativeHandle_); + } + + @Override + public DBOptions setRowCache(final Cache rowCache) { + assert(isOwningHandle()); + setRowCache(nativeHandle_, rowCache.nativeHandle_); + this.rowCache_ = rowCache; + return this; + } + + @Override + public Cache rowCache() { + assert(isOwningHandle()); + return this.rowCache_; + } + + @Override + public DBOptions setWalFilter(final AbstractWalFilter walFilter) { + assert(isOwningHandle()); + setWalFilter(nativeHandle_, walFilter.nativeHandle_); + this.walFilter_ = walFilter; + return this; + } + + @Override + public WalFilter walFilter() { + assert(isOwningHandle()); + return this.walFilter_; + } + + @Override + public DBOptions setFailIfOptionsFileError(final boolean failIfOptionsFileError) { + assert(isOwningHandle()); + setFailIfOptionsFileError(nativeHandle_, failIfOptionsFileError); + return this; + } + + @Override + public boolean failIfOptionsFileError() { + assert(isOwningHandle()); + return failIfOptionsFileError(nativeHandle_); + } + + @Override + public DBOptions setDumpMallocStats(final boolean dumpMallocStats) { + assert(isOwningHandle()); + setDumpMallocStats(nativeHandle_, dumpMallocStats); + return this; + } + + @Override + public boolean dumpMallocStats() { + assert(isOwningHandle()); + return dumpMallocStats(nativeHandle_); + } + + @Override + public DBOptions setAvoidFlushDuringRecovery(final boolean avoidFlushDuringRecovery) { + assert(isOwningHandle()); + setAvoidFlushDuringRecovery(nativeHandle_, avoidFlushDuringRecovery); + return this; + } + + @Override + public boolean avoidFlushDuringRecovery() { + assert(isOwningHandle()); + return avoidFlushDuringRecovery(nativeHandle_); + } + + @Override + public DBOptions setAvoidFlushDuringShutdown(final boolean avoidFlushDuringShutdown) { + assert(isOwningHandle()); + setAvoidFlushDuringShutdown(nativeHandle_, avoidFlushDuringShutdown); + return this; + } + + @Override + public boolean avoidFlushDuringShutdown() { + assert(isOwningHandle()); + return avoidFlushDuringShutdown(nativeHandle_); + } + + @Override + public DBOptions setAllowIngestBehind(final boolean allowIngestBehind) { + assert(isOwningHandle()); + setAllowIngestBehind(nativeHandle_, allowIngestBehind); + return this; + } + + @Override + public boolean allowIngestBehind() { + assert(isOwningHandle()); + return allowIngestBehind(nativeHandle_); + } + + @Override + public DBOptions setTwoWriteQueues(final boolean twoWriteQueues) { + assert(isOwningHandle()); + setTwoWriteQueues(nativeHandle_, twoWriteQueues); + return this; + } + + @Override + public boolean twoWriteQueues() { + assert(isOwningHandle()); + return twoWriteQueues(nativeHandle_); + } + + @Override + public DBOptions setManualWalFlush(final boolean manualWalFlush) { + assert(isOwningHandle()); + setManualWalFlush(nativeHandle_, manualWalFlush); + return this; + } + + @Override + public boolean manualWalFlush() { + assert(isOwningHandle()); + return manualWalFlush(nativeHandle_); + } + + @Override + public DBOptions setAtomicFlush(final boolean atomicFlush) { + setAtomicFlush(nativeHandle_, atomicFlush); + return this; + } + + @Override + public boolean atomicFlush() { + return atomicFlush(nativeHandle_); + } + + @Override + public DBOptions setAvoidUnnecessaryBlockingIO(final boolean avoidUnnecessaryBlockingIO) { + setAvoidUnnecessaryBlockingIO(nativeHandle_, avoidUnnecessaryBlockingIO); + return this; + } + + @Override + public boolean avoidUnnecessaryBlockingIO() { + assert (isOwningHandle()); + return avoidUnnecessaryBlockingIO(nativeHandle_); + } + + @Override + public DBOptions setPersistStatsToDisk(final boolean persistStatsToDisk) { + setPersistStatsToDisk(nativeHandle_, persistStatsToDisk); + return this; + } + + @Override + public boolean persistStatsToDisk() { + assert (isOwningHandle()); + return persistStatsToDisk(nativeHandle_); + } + + @Override + public DBOptions setWriteDbidToManifest(final boolean writeDbidToManifest) { + setWriteDbidToManifest(nativeHandle_, writeDbidToManifest); + return this; + } + + @Override + public boolean writeDbidToManifest() { + assert (isOwningHandle()); + return writeDbidToManifest(nativeHandle_); + } + + @Override + public DBOptions setLogReadaheadSize(final long logReadaheadSize) { + setLogReadaheadSize(nativeHandle_, logReadaheadSize); + return this; + } + + @Override + public long logReadaheadSize() { + assert (isOwningHandle()); + return logReadaheadSize(nativeHandle_); + } + + @Override + public DBOptions setBestEffortsRecovery(final boolean bestEffortsRecovery) { + setBestEffortsRecovery(nativeHandle_, bestEffortsRecovery); + return this; + } + + @Override + public boolean bestEffortsRecovery() { + assert (isOwningHandle()); + return bestEffortsRecovery(nativeHandle_); + } + + @Override + public DBOptions setMaxBgErrorResumeCount(final int maxBgerrorResumeCount) { + setMaxBgErrorResumeCount(nativeHandle_, maxBgerrorResumeCount); + return this; + } + + @Override + public int maxBgerrorResumeCount() { + assert (isOwningHandle()); + return maxBgerrorResumeCount(nativeHandle_); + } + + @Override + public DBOptions setBgerrorResumeRetryInterval(final long bgerrorResumeRetryInterval) { + setBgerrorResumeRetryInterval(nativeHandle_, bgerrorResumeRetryInterval); + return this; + } + + @Override + public long bgerrorResumeRetryInterval() { + assert (isOwningHandle()); + return bgerrorResumeRetryInterval(nativeHandle_); + } + + static final int DEFAULT_NUM_SHARD_BITS = -1; + + + + + /** + *

    Private constructor to be used by + * {@link #getDBOptionsFromProps(java.util.Properties)}

    + * + * @param nativeHandle native handle to DBOptions instance. + */ + private DBOptions(final long nativeHandle) { + super(nativeHandle); + } + + private static native long getDBOptionsFromProps(long cfgHandle, String optString); + private static native long getDBOptionsFromProps(String optString); + + private static native long newDBOptions(); + private static native long copyDBOptions(final long handle); + private static native long newDBOptionsFromOptions(final long optionsHandle); + @Override protected final native void disposeInternal(final long handle); + + private native void optimizeForSmallDb(final long handle); + private native void setIncreaseParallelism(long handle, int totalThreads); + private native void setCreateIfMissing(long handle, boolean flag); + private native boolean createIfMissing(long handle); + private native void setCreateMissingColumnFamilies( + long handle, boolean flag); + private native boolean createMissingColumnFamilies(long handle); + private native void setEnv(long handle, long envHandle); + private native void setErrorIfExists(long handle, boolean errorIfExists); + private native boolean errorIfExists(long handle); + private native void setParanoidChecks( + long handle, boolean paranoidChecks); + private native boolean paranoidChecks(long handle); + private native void setRateLimiter(long handle, + long rateLimiterHandle); + private native void setSstFileManager(final long handle, + final long sstFileManagerHandle); + private native void setLogger(long handle, + long loggerHandle); + private native void setInfoLogLevel(long handle, byte logLevel); + private native byte infoLogLevel(long handle); + private native void setMaxOpenFiles(long handle, int maxOpenFiles); + private native int maxOpenFiles(long handle); + private native void setMaxFileOpeningThreads(final long handle, + final int maxFileOpeningThreads); + private native int maxFileOpeningThreads(final long handle); + private native void setMaxTotalWalSize(long handle, + long maxTotalWalSize); + private native long maxTotalWalSize(long handle); + private native void setStatistics(final long handle, final long statisticsHandle); + private native long statistics(final long handle); + private native boolean useFsync(long handle); + private native void setUseFsync(long handle, boolean useFsync); + private native void setDbPaths(final long handle, final String[] paths, + final long[] targetSizes); + private native long dbPathsLen(final long handle); + private native void dbPaths(final long handle, final String[] paths, + final long[] targetSizes); + private native void setDbLogDir(long handle, String dbLogDir); + private native String dbLogDir(long handle); + private native void setWalDir(long handle, String walDir); + private native String walDir(long handle); + private native void setDeleteObsoleteFilesPeriodMicros( + long handle, long micros); + private native long deleteObsoleteFilesPeriodMicros(long handle); + private native void setMaxBackgroundCompactions( + long handle, int maxBackgroundCompactions); + private native int maxBackgroundCompactions(long handle); + private native void setMaxSubcompactions(long handle, int maxSubcompactions); + private native int maxSubcompactions(long handle); + private native void setMaxBackgroundFlushes( + long handle, int maxBackgroundFlushes); + private native int maxBackgroundFlushes(long handle); + private native void setMaxBackgroundJobs(long handle, int maxBackgroundJobs); + private native int maxBackgroundJobs(long handle); + private native void setMaxLogFileSize(long handle, long maxLogFileSize) + throws IllegalArgumentException; + private native long maxLogFileSize(long handle); + private native void setLogFileTimeToRoll( + long handle, long logFileTimeToRoll) throws IllegalArgumentException; + private native long logFileTimeToRoll(long handle); + private native void setKeepLogFileNum(long handle, long keepLogFileNum) + throws IllegalArgumentException; + private native long keepLogFileNum(long handle); + private native void setRecycleLogFileNum(long handle, long recycleLogFileNum); + private native long recycleLogFileNum(long handle); + private native void setMaxManifestFileSize( + long handle, long maxManifestFileSize); + private native long maxManifestFileSize(long handle); + private native void setTableCacheNumshardbits( + long handle, int tableCacheNumshardbits); + private native int tableCacheNumshardbits(long handle); + private native void setWalTtlSeconds(long handle, long walTtlSeconds); + private native long walTtlSeconds(long handle); + private native void setWalSizeLimitMB(long handle, long sizeLimitMB); + private native long walSizeLimitMB(long handle); + private static native void setMaxWriteBatchGroupSizeBytes( + final long handle, final long maxWriteBatchGroupSizeBytes); + private static native long maxWriteBatchGroupSizeBytes(final long handle); + private native void setManifestPreallocationSize( + long handle, long size) throws IllegalArgumentException; + private native long manifestPreallocationSize(long handle); + private native void setUseDirectReads(long handle, boolean useDirectReads); + private native boolean useDirectReads(long handle); + private native void setUseDirectIoForFlushAndCompaction( + long handle, boolean useDirectIoForFlushAndCompaction); + private native boolean useDirectIoForFlushAndCompaction(long handle); + private native void setAllowFAllocate(final long handle, + final boolean allowFAllocate); + private native boolean allowFAllocate(final long handle); + private native void setAllowMmapReads( + long handle, boolean allowMmapReads); + private native boolean allowMmapReads(long handle); + private native void setAllowMmapWrites( + long handle, boolean allowMmapWrites); + private native boolean allowMmapWrites(long handle); + private native void setIsFdCloseOnExec( + long handle, boolean isFdCloseOnExec); + private native boolean isFdCloseOnExec(long handle); + private native void setStatsDumpPeriodSec( + long handle, int statsDumpPeriodSec); + private native int statsDumpPeriodSec(long handle); + private native void setStatsPersistPeriodSec( + final long handle, final int statsPersistPeriodSec); + private native int statsPersistPeriodSec( + final long handle); + private native void setStatsHistoryBufferSize( + final long handle, final long statsHistoryBufferSize); + private native long statsHistoryBufferSize( + final long handle); + private native void setAdviseRandomOnOpen( + long handle, boolean adviseRandomOnOpen); + private native boolean adviseRandomOnOpen(long handle); + private native void setDbWriteBufferSize(final long handle, + final long dbWriteBufferSize); + private native void setWriteBufferManager(final long dbOptionsHandle, + final long writeBufferManagerHandle); + private native long dbWriteBufferSize(final long handle); + private native void setAccessHintOnCompactionStart(final long handle, + final byte accessHintOnCompactionStart); + private native byte accessHintOnCompactionStart(final long handle); + private native void setCompactionReadaheadSize(final long handle, + final long compactionReadaheadSize); + private native long compactionReadaheadSize(final long handle); + private native void setRandomAccessMaxBufferSize(final long handle, + final long randomAccessMaxBufferSize); + private native long randomAccessMaxBufferSize(final long handle); + private native void setWritableFileMaxBufferSize(final long handle, + final long writableFileMaxBufferSize); + private native long writableFileMaxBufferSize(final long handle); + private native void setUseAdaptiveMutex( + long handle, boolean useAdaptiveMutex); + private native boolean useAdaptiveMutex(long handle); + private native void setBytesPerSync( + long handle, long bytesPerSync); + private native long bytesPerSync(long handle); + private native void setWalBytesPerSync(long handle, long walBytesPerSync); + private native long walBytesPerSync(long handle); + private native void setStrictBytesPerSync( + final long handle, final boolean strictBytesPerSync); + private native boolean strictBytesPerSync( + final long handle); + private static native void setEventListeners( + final long handle, final long[] eventListenerHandles); + private static native AbstractEventListener[] eventListeners(final long handle); + private native void setEnableThreadTracking(long handle, + boolean enableThreadTracking); + private native boolean enableThreadTracking(long handle); + private native void setDelayedWriteRate(long handle, long delayedWriteRate); + private native long delayedWriteRate(long handle); + private native void setEnablePipelinedWrite(final long handle, + final boolean enablePipelinedWrite); + private native boolean enablePipelinedWrite(final long handle); + private native void setUnorderedWrite(final long handle, + final boolean unorderedWrite); + private native boolean unorderedWrite(final long handle); + private native void setAllowConcurrentMemtableWrite(long handle, + boolean allowConcurrentMemtableWrite); + private native boolean allowConcurrentMemtableWrite(long handle); + private native void setEnableWriteThreadAdaptiveYield(long handle, + boolean enableWriteThreadAdaptiveYield); + private native boolean enableWriteThreadAdaptiveYield(long handle); + private native void setWriteThreadMaxYieldUsec(long handle, + long writeThreadMaxYieldUsec); + private native long writeThreadMaxYieldUsec(long handle); + private native void setWriteThreadSlowYieldUsec(long handle, + long writeThreadSlowYieldUsec); + private native long writeThreadSlowYieldUsec(long handle); + private native void setSkipStatsUpdateOnDbOpen(final long handle, + final boolean skipStatsUpdateOnDbOpen); + private native boolean skipStatsUpdateOnDbOpen(final long handle); + private static native void setSkipCheckingSstFileSizesOnDbOpen( + final long handle, final boolean skipChecking); + private static native boolean skipCheckingSstFileSizesOnDbOpen(final long handle); + private native void setWalRecoveryMode(final long handle, + final byte walRecoveryMode); + private native byte walRecoveryMode(final long handle); + private native void setAllow2pc(final long handle, + final boolean allow2pc); + private native boolean allow2pc(final long handle); + private native void setRowCache(final long handle, + final long rowCacheHandle); + private native void setWalFilter(final long handle, + final long walFilterHandle); + private native void setFailIfOptionsFileError(final long handle, + final boolean failIfOptionsFileError); + private native boolean failIfOptionsFileError(final long handle); + private native void setDumpMallocStats(final long handle, + final boolean dumpMallocStats); + private native boolean dumpMallocStats(final long handle); + private native void setAvoidFlushDuringRecovery(final long handle, + final boolean avoidFlushDuringRecovery); + private native boolean avoidFlushDuringRecovery(final long handle); + private native void setAvoidFlushDuringShutdown(final long handle, + final boolean avoidFlushDuringShutdown); + private native boolean avoidFlushDuringShutdown(final long handle); + private native void setAllowIngestBehind(final long handle, + final boolean allowIngestBehind); + private native boolean allowIngestBehind(final long handle); + private native void setTwoWriteQueues(final long handle, + final boolean twoWriteQueues); + private native boolean twoWriteQueues(final long handle); + private native void setManualWalFlush(final long handle, + final boolean manualWalFlush); + private native boolean manualWalFlush(final long handle); + private native void setAtomicFlush(final long handle, + final boolean atomicFlush); + private native boolean atomicFlush(final long handle); + private static native void setAvoidUnnecessaryBlockingIO( + final long handle, final boolean avoidBlockingIO); + private static native boolean avoidUnnecessaryBlockingIO(final long handle); + private static native void setPersistStatsToDisk( + final long handle, final boolean persistStatsToDisk); + private static native boolean persistStatsToDisk(final long handle); + private static native void setWriteDbidToManifest( + final long handle, final boolean writeDbidToManifest); + private static native boolean writeDbidToManifest(final long handle); + private static native void setLogReadaheadSize(final long handle, final long logReadaheadSize); + private static native long logReadaheadSize(final long handle); + private static native void setBestEffortsRecovery( + final long handle, final boolean bestEffortsRecovery); + private static native boolean bestEffortsRecovery(final long handle); + private static native void setMaxBgErrorResumeCount( + final long handle, final int maxBgerrorRecumeCount); + private static native int maxBgerrorResumeCount(final long handle); + private static native void setBgerrorResumeRetryInterval( + final long handle, final long bgerrorResumeRetryInterval); + private static native long bgerrorResumeRetryInterval(final long handle); + + // instance variables + // NOTE: If you add new member variables, please update the copy constructor above! + private Env env_; + private int numShardBits_; + private RateLimiter rateLimiter_; + private Cache rowCache_; + private WalFilter walFilter_; + private WriteBufferManager writeBufferManager_; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/DBOptionsInterface.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/DBOptionsInterface.java new file mode 100644 index 0000000..326da98 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/DBOptionsInterface.java @@ -0,0 +1,1756 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Collection; +import java.util.List; + +public interface DBOptionsInterface> { + /** + * Use this if your DB is very small (like under 1GB) and you don't want to + * spend lots of memory for memtables. + * + * @return the instance of the current object. + */ + T optimizeForSmallDb(); + + /** + * Use the specified object to interact with the environment, + * e.g. to read/write files, schedule background work, etc. + * Default: {@link Env#getDefault()} + * + * @param env {@link Env} instance. + * @return the instance of the current Options. + */ + T setEnv(final Env env); + + /** + * Returns the set RocksEnv instance. + * + * @return {@link RocksEnv} instance set in the options. + */ + Env getEnv(); + + /** + *

    By default, RocksDB uses only one background thread for flush and + * compaction. Calling this function will set it up such that total of + * `total_threads` is used.

    + * + *

    You almost definitely want to call this function if your system is + * bottlenecked by RocksDB.

    + * + * @param totalThreads The total number of threads to be used by RocksDB. + * A good value is the number of cores. + * + * @return the instance of the current Options + */ + T setIncreaseParallelism(int totalThreads); + + /** + * If this value is set to true, then the database will be created + * if it is missing during {@code RocksDB.open()}. + * Default: false + * + * @param flag a flag indicating whether to create a database the + * specified database in {@link RocksDB#open(org.rocksdb.Options, String)} operation + * is missing. + * @return the instance of the current Options + * @see RocksDB#open(org.rocksdb.Options, String) + */ + T setCreateIfMissing(boolean flag); + + /** + * Return true if the create_if_missing flag is set to true. + * If true, the database will be created if it is missing. + * + * @return true if the createIfMissing option is set to true. + * @see #setCreateIfMissing(boolean) + */ + boolean createIfMissing(); + + /** + *

    If true, missing column families will be automatically created

    + * + *

    Default: false

    + * + * @param flag a flag indicating if missing column families shall be + * created automatically. + * @return true if missing column families shall be created automatically + * on open. + */ + T setCreateMissingColumnFamilies(boolean flag); + + /** + * Return true if the create_missing_column_families flag is set + * to true. If true column families be created if missing. + * + * @return true if the createMissingColumnFamilies is set to + * true. + * @see #setCreateMissingColumnFamilies(boolean) + */ + boolean createMissingColumnFamilies(); + + /** + * If true, an error will be thrown during RocksDB.open() if the + * database already exists. + * Default: false + * + * @param errorIfExists if true, an exception will be thrown + * during {@code RocksDB.open()} if the database already exists. + * @return the reference to the current option. + * @see RocksDB#open(org.rocksdb.Options, String) + */ + T setErrorIfExists(boolean errorIfExists); + + /** + * If true, an error will be thrown during RocksDB.open() if the + * database already exists. + * + * @return if true, an error is raised when the specified database + * already exists before open. + */ + boolean errorIfExists(); + + /** + * If true, the implementation will do aggressive checking of the + * data it is processing and will stop early if it detects any + * errors. This may have unforeseen ramifications: for example, a + * corruption of one DB entry may cause a large number of entries to + * become unreadable or for the entire DB to become unopenable. + * If any of the writes to the database fails (Put, Delete, Merge, Write), + * the database will switch to read-only mode and fail all other + * Write operations. + * Default: true + * + * @param paranoidChecks a flag to indicate whether paranoid-check + * is on. + * @return the reference to the current option. + */ + T setParanoidChecks(boolean paranoidChecks); + + /** + * If true, the implementation will do aggressive checking of the + * data it is processing and will stop early if it detects any + * errors. This may have unforeseen ramifications: for example, a + * corruption of one DB entry may cause a large number of entries to + * become unreadable or for the entire DB to become unopenable. + * If any of the writes to the database fails (Put, Delete, Merge, Write), + * the database will switch to read-only mode and fail all other + * Write operations. + * + * @return a boolean indicating whether paranoid-check is on. + */ + boolean paranoidChecks(); + + /** + * Use to control write rate of flush and compaction. Flush has higher + * priority than compaction. Rate limiting is disabled if nullptr. + * Default: nullptr + * + * @param rateLimiter {@link org.rocksdb.RateLimiter} instance. + * @return the instance of the current object. + * + * @since 3.10.0 + */ + T setRateLimiter(RateLimiter rateLimiter); + + /** + * Use to track SST files and control their file deletion rate. + * + * Features: + * - Throttle the deletion rate of the SST files. + * - Keep track the total size of all SST files. + * - Set a maximum allowed space limit for SST files that when reached + * the DB wont do any further flushes or compactions and will set the + * background error. + * - Can be shared between multiple dbs. + * + * Limitations: + * - Only track and throttle deletes of SST files in + * first db_path (db_name if db_paths is empty). + * + * @param sstFileManager The SST File Manager for the db. + * @return the instance of the current object. + */ + T setSstFileManager(SstFileManager sstFileManager); + + /** + *

    Any internal progress/error information generated by + * the db will be written to the Logger if it is non-nullptr, + * or to a file stored in the same directory as the DB + * contents if info_log is nullptr.

    + * + *

    Default: nullptr

    + * + * @param logger {@link Logger} instance. + * @return the instance of the current object. + */ + T setLogger(Logger logger); + + /** + *

    Sets the RocksDB log level. Default level is INFO

    + * + * @param infoLogLevel log level to set. + * @return the instance of the current object. + */ + T setInfoLogLevel(InfoLogLevel infoLogLevel); + + /** + *

    Returns currently set log level.

    + * @return {@link org.rocksdb.InfoLogLevel} instance. + */ + InfoLogLevel infoLogLevel(); + + /** + * If {@link MutableDBOptionsInterface#maxOpenFiles()} is -1, DB will open + * all files on DB::Open(). You can use this option to increase the number + * of threads used to open the files. + * + * Default: 16 + * + * @param maxFileOpeningThreads the maximum number of threads to use to + * open files + * + * @return the reference to the current options. + */ + T setMaxFileOpeningThreads(int maxFileOpeningThreads); + + /** + * If {@link MutableDBOptionsInterface#maxOpenFiles()} is -1, DB will open all + * files on DB::Open(). You can use this option to increase the number of + * threads used to open the files. + * + * Default: 16 + * + * @return the maximum number of threads to use to open files + */ + int maxFileOpeningThreads(); + + /** + *

    Sets the statistics object which collects metrics about database operations. + * Statistics objects should not be shared between DB instances as + * it does not use any locks to prevent concurrent updates.

    + * + * @param statistics The statistics to set + * + * @return the instance of the current object. + * + * @see RocksDB#open(org.rocksdb.Options, String) + */ + T setStatistics(final Statistics statistics); + + /** + *

    Returns statistics object.

    + * + * @return the instance of the statistics object or null if there is no + * statistics object. + * + * @see #setStatistics(Statistics) + */ + Statistics statistics(); + + /** + *

    If true, then every store to stable storage will issue a fsync.

    + *

    If false, then every store to stable storage will issue a fdatasync. + * This parameter should be set to true while storing data to + * filesystem like ext3 that can lose files after a reboot.

    + *

    Default: false

    + * + * @param useFsync a boolean flag to specify whether to use fsync + * @return the instance of the current object. + */ + T setUseFsync(boolean useFsync); + + /** + *

    If true, then every store to stable storage will issue a fsync.

    + *

    If false, then every store to stable storage will issue a fdatasync. + * This parameter should be set to true while storing data to + * filesystem like ext3 that can lose files after a reboot.

    + * + * @return boolean value indicating if fsync is used. + */ + boolean useFsync(); + + /** + * A list of paths where SST files can be put into, with its target size. + * Newer data is placed into paths specified earlier in the vector while + * older data gradually moves to paths specified later in the vector. + * + * For example, you have a flash device with 10GB allocated for the DB, + * as well as a hard drive of 2TB, you should config it to be: + * [{"/flash_path", 10GB}, {"/hard_drive", 2TB}] + * + * The system will try to guarantee data under each path is close to but + * not larger than the target size. But current and future file sizes used + * by determining where to place a file are based on best-effort estimation, + * which means there is a chance that the actual size under the directory + * is slightly more than target size under some workloads. User should give + * some buffer room for those cases. + * + * If none of the paths has sufficient room to place a file, the file will + * be placed to the last path anyway, despite to the target size. + * + * Placing newer data to earlier paths is also best-efforts. User should + * expect user files to be placed in higher levels in some extreme cases. + * + * If left empty, only one path will be used, which is db_name passed when + * opening the DB. + * + * Default: empty + * + * @param dbPaths the paths and target sizes + * + * @return the reference to the current options + */ + T setDbPaths(final Collection dbPaths); + + /** + * A list of paths where SST files can be put into, with its target size. + * Newer data is placed into paths specified earlier in the vector while + * older data gradually moves to paths specified later in the vector. + * + * For example, you have a flash device with 10GB allocated for the DB, + * as well as a hard drive of 2TB, you should config it to be: + * [{"/flash_path", 10GB}, {"/hard_drive", 2TB}] + * + * The system will try to guarantee data under each path is close to but + * not larger than the target size. But current and future file sizes used + * by determining where to place a file are based on best-effort estimation, + * which means there is a chance that the actual size under the directory + * is slightly more than target size under some workloads. User should give + * some buffer room for those cases. + * + * If none of the paths has sufficient room to place a file, the file will + * be placed to the last path anyway, despite to the target size. + * + * Placing newer data to earlier paths is also best-efforts. User should + * expect user files to be placed in higher levels in some extreme cases. + * + * If left empty, only one path will be used, which is db_name passed when + * opening the DB. + * + * Default: {@link java.util.Collections#emptyList()} + * + * @return dbPaths the paths and target sizes + */ + List dbPaths(); + + /** + * This specifies the info LOG dir. + * If it is empty, the log files will be in the same dir as data. + * If it is non empty, the log files will be in the specified dir, + * and the db data dir's absolute path will be used as the log file + * name's prefix. + * + * @param dbLogDir the path to the info log directory + * @return the instance of the current object. + */ + T setDbLogDir(String dbLogDir); + + /** + * Returns the directory of info log. + * + * If it is empty, the log files will be in the same dir as data. + * If it is non empty, the log files will be in the specified dir, + * and the db data dir's absolute path will be used as the log file + * name's prefix. + * + * @return the path to the info log directory + */ + String dbLogDir(); + + /** + * This specifies the absolute dir path for write-ahead logs (WAL). + * If it is empty, the log files will be in the same dir as data, + * dbname is used as the data dir by default + * If it is non empty, the log files will be in kept the specified dir. + * When destroying the db, + * all log files in wal_dir and the dir itself is deleted + * + * @param walDir the path to the write-ahead-log directory. + * @return the instance of the current object. + */ + T setWalDir(String walDir); + + /** + * Returns the path to the write-ahead-logs (WAL) directory. + * + * If it is empty, the log files will be in the same dir as data, + * dbname is used as the data dir by default + * If it is non empty, the log files will be in kept the specified dir. + * When destroying the db, + * all log files in wal_dir and the dir itself is deleted + * + * @return the path to the write-ahead-logs (WAL) directory. + */ + String walDir(); + + /** + * The periodicity when obsolete files get deleted. The default + * value is 6 hours. The files that get out of scope by compaction + * process will still get automatically delete on every compaction, + * regardless of this setting + * + * @param micros the time interval in micros + * @return the instance of the current object. + */ + T setDeleteObsoleteFilesPeriodMicros(long micros); + + /** + * The periodicity when obsolete files get deleted. The default + * value is 6 hours. The files that get out of scope by compaction + * process will still get automatically delete on every compaction, + * regardless of this setting + * + * @return the time interval in micros when obsolete files will be deleted. + */ + long deleteObsoleteFilesPeriodMicros(); + + /** + * This value represents the maximum number of threads that will + * concurrently perform a compaction job by breaking it into multiple, + * smaller ones that are run simultaneously. + * Default: 1 (i.e. no subcompactions) + * + * @param maxSubcompactions The maximum number of threads that will + * concurrently perform a compaction job + * + * @return the instance of the current object. + */ + T setMaxSubcompactions(int maxSubcompactions); + + /** + * This value represents the maximum number of threads that will + * concurrently perform a compaction job by breaking it into multiple, + * smaller ones that are run simultaneously. + * Default: 1 (i.e. no subcompactions) + * + * @return The maximum number of threads that will concurrently perform a + * compaction job + */ + int maxSubcompactions(); + + /** + * NOT SUPPORTED ANYMORE: RocksDB automatically decides this based on the + * value of max_background_jobs. For backwards compatibility we will set + * `max_background_jobs = max_background_compactions + max_background_flushes` + * in the case where user sets at least one of `max_background_compactions` or + * `max_background_flushes`. + * + * Specifies the maximum number of concurrent background flush jobs. + * If you're increasing this, also consider increasing number of threads in + * HIGH priority thread pool. For more information, see + * Default: -1 + * + * @param maxBackgroundFlushes number of max concurrent flush jobs + * @return the instance of the current object. + * + * @see RocksEnv#setBackgroundThreads(int) + * @see RocksEnv#setBackgroundThreads(int, Priority) + * @see MutableDBOptionsInterface#maxBackgroundCompactions() + * + * @deprecated Use {@link MutableDBOptionsInterface#setMaxBackgroundJobs(int)} + */ + @Deprecated + T setMaxBackgroundFlushes(int maxBackgroundFlushes); + + /** + * NOT SUPPORTED ANYMORE: RocksDB automatically decides this based on the + * value of max_background_jobs. For backwards compatibility we will set + * `max_background_jobs = max_background_compactions + max_background_flushes` + * in the case where user sets at least one of `max_background_compactions` or + * `max_background_flushes`. + * + * Returns the maximum number of concurrent background flush jobs. + * If you're increasing this, also consider increasing number of threads in + * HIGH priority thread pool. For more information, see + * Default: -1 + * + * @return the maximum number of concurrent background flush jobs. + * @see RocksEnv#setBackgroundThreads(int) + * @see RocksEnv#setBackgroundThreads(int, Priority) + */ + @Deprecated + int maxBackgroundFlushes(); + + /** + * Specifies the maximum size of a info log file. If the current log file + * is larger than `max_log_file_size`, a new info log file will + * be created. + * If 0, all logs will be written to one log file. + * + * @param maxLogFileSize the maximum size of a info log file. + * @return the instance of the current object. + * @throws java.lang.IllegalArgumentException thrown on 32-Bit platforms + * while overflowing the underlying platform specific value. + */ + T setMaxLogFileSize(long maxLogFileSize); + + /** + * Returns the maximum size of a info log file. If the current log file + * is larger than this size, a new info log file will be created. + * If 0, all logs will be written to one log file. + * + * @return the maximum size of the info log file. + */ + long maxLogFileSize(); + + /** + * Specifies the time interval for the info log file to roll (in seconds). + * If specified with non-zero value, log file will be rolled + * if it has been active longer than `log_file_time_to_roll`. + * Default: 0 (disabled) + * + * @param logFileTimeToRoll the time interval in seconds. + * @return the instance of the current object. + * @throws java.lang.IllegalArgumentException thrown on 32-Bit platforms + * while overflowing the underlying platform specific value. + */ + T setLogFileTimeToRoll(long logFileTimeToRoll); + + /** + * Returns the time interval for the info log file to roll (in seconds). + * If specified with non-zero value, log file will be rolled + * if it has been active longer than `log_file_time_to_roll`. + * Default: 0 (disabled) + * + * @return the time interval in seconds. + */ + long logFileTimeToRoll(); + + /** + * Specifies the maximum number of info log files to be kept. + * Default: 1000 + * + * @param keepLogFileNum the maximum number of info log files to be kept. + * @return the instance of the current object. + * @throws java.lang.IllegalArgumentException thrown on 32-Bit platforms + * while overflowing the underlying platform specific value. + */ + T setKeepLogFileNum(long keepLogFileNum); + + /** + * Returns the maximum number of info log files to be kept. + * Default: 1000 + * + * @return the maximum number of info log files to be kept. + */ + long keepLogFileNum(); + + /** + * Recycle log files. + * + * If non-zero, we will reuse previously written log files for new + * logs, overwriting the old data. The value indicates how many + * such files we will keep around at any point in time for later + * use. + * + * This is more efficient because the blocks are already + * allocated and fdatasync does not need to update the inode after + * each write. + * + * Default: 0 + * + * @param recycleLogFileNum the number of log files to keep for recycling + * + * @return the reference to the current options + */ + T setRecycleLogFileNum(long recycleLogFileNum); + + /** + * Recycle log files. + * + * If non-zero, we will reuse previously written log files for new + * logs, overwriting the old data. The value indicates how many + * such files we will keep around at any point in time for later + * use. + * + * This is more efficient because the blocks are already + * allocated and fdatasync does not need to update the inode after + * each write. + * + * Default: 0 + * + * @return the number of log files kept for recycling + */ + long recycleLogFileNum(); + + /** + * Manifest file is rolled over on reaching this limit. + * The older manifest file be deleted. + * The default value is 1GB so that the manifest file can grow, but not + * reach the limit of storage capacity. + * + * @param maxManifestFileSize the size limit of a manifest file. + * @return the instance of the current object. + */ + T setMaxManifestFileSize(long maxManifestFileSize); + + /** + * Manifest file is rolled over on reaching this limit. + * The older manifest file be deleted. + * The default value is 1GB so that the manifest file can grow, but not + * reach the limit of storage capacity. + * + * @return the size limit of a manifest file. + */ + long maxManifestFileSize(); + + /** + * Number of shards used for table cache. + * + * @param tableCacheNumshardbits the number of chards + * @return the instance of the current object. + */ + T setTableCacheNumshardbits(int tableCacheNumshardbits); + + /** + * Number of shards used for table cache. + * + * @return the number of shards used for table cache. + */ + int tableCacheNumshardbits(); + + /** + * {@link #walTtlSeconds()} and {@link #walSizeLimitMB()} affect how archived logs + * will be deleted. + *
      + *
    1. If both set to 0, logs will be deleted asap and will not get into + * the archive.
      2. + *
    2. If WAL_ttl_seconds is 0 and WAL_size_limit_MB is not 0, + * WAL files will be checked every 10 min and if total size is greater + * then WAL_size_limit_MB, they will be deleted starting with the + * earliest until size_limit is met. All empty files will be deleted.
      3. + *
    3. If WAL_ttl_seconds is not 0 and WAL_size_limit_MB is 0, then + * WAL files will be checked every WAL_ttl_seconds / 2 and those that + * are older than WAL_ttl_seconds will be deleted.
      4. + *
    4. If both are not 0, WAL files will be checked every 10 min and both + * checks will be performed with ttl being first.
      5. + *
    + * + * @param walTtlSeconds the ttl seconds + * @return the instance of the current object. + * @see #setWalSizeLimitMB(long) + */ + T setWalTtlSeconds(long walTtlSeconds); + + /** + * WalTtlSeconds() and walSizeLimitMB() affect how archived logs + * will be deleted. + *
      + *
    1. If both set to 0, logs will be deleted asap and will not get into + * the archive.
      2. + *
    2. If WAL_ttl_seconds is 0 and WAL_size_limit_MB is not 0, + * WAL files will be checked every 10 min and if total size is greater + * then WAL_size_limit_MB, they will be deleted starting with the + * earliest until size_limit is met. All empty files will be deleted.
      3. + *
    3. If WAL_ttl_seconds is not 0 and WAL_size_limit_MB is 0, then + * WAL files will be checked every WAL_ttl_seconds / 2 and those that + * are older than WAL_ttl_seconds will be deleted.
      4. + *
    4. If both are not 0, WAL files will be checked every 10 min and both + * checks will be performed with ttl being first.
      5. + *
    + * + * @return the wal-ttl seconds + * @see #walSizeLimitMB() + */ + long walTtlSeconds(); + + /** + * WalTtlSeconds() and walSizeLimitMB() affect how archived logs + * will be deleted. + *
      + *
    1. If both set to 0, logs will be deleted asap and will not get into + * the archive.
      2. + *
    2. If WAL_ttl_seconds is 0 and WAL_size_limit_MB is not 0, + * WAL files will be checked every 10 min and if total size is greater + * then WAL_size_limit_MB, they will be deleted starting with the + * earliest until size_limit is met. All empty files will be deleted.
      3. + *
    3. If WAL_ttl_seconds is not 0 and WAL_size_limit_MB is 0, then + * WAL files will be checked every WAL_ttl_secondsi / 2 and those that + * are older than WAL_ttl_seconds will be deleted.
      4. + *
    4. If both are not 0, WAL files will be checked every 10 min and both + * checks will be performed with ttl being first.
      5. + *
    + * + * @param sizeLimitMB size limit in mega-bytes. + * @return the instance of the current object. + * @see #setWalSizeLimitMB(long) + */ + T setWalSizeLimitMB(long sizeLimitMB); + + /** + * {@link #walTtlSeconds()} and {@code #walSizeLimitMB()} affect how archived logs + * will be deleted. + *
      + *
    1. If both set to 0, logs will be deleted asap and will not get into + * the archive.
      2. + *
    2. If WAL_ttl_seconds is 0 and WAL_size_limit_MB is not 0, + * WAL files will be checked every 10 min and if total size is greater + * then WAL_size_limit_MB, they will be deleted starting with the + * earliest until size_limit is met. All empty files will be deleted.
      3. + *
    3. If WAL_ttl_seconds is not 0 and WAL_size_limit_MB is 0, then + * WAL files will be checked every WAL_ttl_seconds i / 2 and those that + * are older than WAL_ttl_seconds will be deleted.
      4. + *
    4. If both are not 0, WAL files will be checked every 10 min and both + * checks will be performed with ttl being first.
      5. + *
    + * @return size limit in mega-bytes. + * @see #walSizeLimitMB() + */ + long walSizeLimitMB(); + + /** + * The maximum limit of number of bytes that are written in a single batch + * of WAL or memtable write. It is followed when the leader write size + * is larger than 1/8 of this limit. + * + * Default: 1 MB + * + * @param maxWriteBatchGroupSizeBytes the maximum limit of number of bytes, see description. + * @return the instance of the current object. + */ + T setMaxWriteBatchGroupSizeBytes(final long maxWriteBatchGroupSizeBytes); + + /** + * The maximum limit of number of bytes that are written in a single batch + * of WAL or memtable write. It is followed when the leader write size + * is larger than 1/8 of this limit. + * + * Default: 1 MB + * + * @return the maximum limit of number of bytes, see description. + */ + long maxWriteBatchGroupSizeBytes(); + + /** + * Number of bytes to preallocate (via fallocate) the manifest + * files. Default is 4mb, which is reasonable to reduce random IO + * as well as prevent overallocation for mounts that preallocate + * large amounts of data (such as xfs's allocsize option). + * + * @param size the size in byte + * @return the instance of the current object. + * @throws java.lang.IllegalArgumentException thrown on 32-Bit platforms + * while overflowing the underlying platform specific value. + */ + T setManifestPreallocationSize(long size); + + /** + * Number of bytes to preallocate (via fallocate) the manifest + * files. Default is 4mb, which is reasonable to reduce random IO + * as well as prevent overallocation for mounts that preallocate + * large amounts of data (such as xfs's allocsize option). + * + * @return size in bytes. + */ + long manifestPreallocationSize(); + + /** + * Enable the OS to use direct I/O for reading sst tables. + * Default: false + * + * @param useDirectReads if true, then direct read is enabled + * @return the instance of the current object. + */ + T setUseDirectReads(boolean useDirectReads); + + /** + * Enable the OS to use direct I/O for reading sst tables. + * Default: false + * + * @return if true, then direct reads are enabled + */ + boolean useDirectReads(); + + /** + * Enable the OS to use direct reads and writes in flush and + * compaction + * Default: false + * + * @param useDirectIoForFlushAndCompaction if true, then direct + * I/O will be enabled for background flush and compactions + * @return the instance of the current object. + */ + T setUseDirectIoForFlushAndCompaction(boolean useDirectIoForFlushAndCompaction); + + /** + * Enable the OS to use direct reads and writes in flush and + * compaction + * + * @return if true, then direct I/O is enabled for flush and + * compaction + */ + boolean useDirectIoForFlushAndCompaction(); + + /** + * Whether fallocate calls are allowed + * + * @param allowFAllocate false if fallocate() calls are bypassed + * + * @return the reference to the current options. + */ + T setAllowFAllocate(boolean allowFAllocate); + + /** + * Whether fallocate calls are allowed + * + * @return false if fallocate() calls are bypassed + */ + boolean allowFAllocate(); + + /** + * Allow the OS to mmap file for reading sst tables. + * Default: false + * + * @param allowMmapReads true if mmap reads are allowed. + * @return the instance of the current object. + */ + T setAllowMmapReads(boolean allowMmapReads); + + /** + * Allow the OS to mmap file for reading sst tables. + * Default: false + * + * @return true if mmap reads are allowed. + */ + boolean allowMmapReads(); + + /** + * Allow the OS to mmap file for writing. Default: false + * + * @param allowMmapWrites true if mmap writes are allowd. + * @return the instance of the current object. + */ + T setAllowMmapWrites(boolean allowMmapWrites); + + /** + * Allow the OS to mmap file for writing. Default: false + * + * @return true if mmap writes are allowed. + */ + boolean allowMmapWrites(); + + /** + * Disable child process inherit open files. Default: true + * + * @param isFdCloseOnExec true if child process inheriting open + * files is disabled. + * @return the instance of the current object. + */ + T setIsFdCloseOnExec(boolean isFdCloseOnExec); + + /** + * Disable child process inherit open files. Default: true + * + * @return true if child process inheriting open files is disabled. + */ + boolean isFdCloseOnExec(); + + /** + * If set true, will hint the underlying file system that the file + * access pattern is random, when a sst file is opened. + * Default: true + * + * @param adviseRandomOnOpen true if hinting random access is on. + * @return the instance of the current object. + */ + T setAdviseRandomOnOpen(boolean adviseRandomOnOpen); + + /** + * If set true, will hint the underlying file system that the file + * access pattern is random, when a sst file is opened. + * Default: true + * + * @return true if hinting random access is on. + */ + boolean adviseRandomOnOpen(); + + /** + * Amount of data to build up in memtables across all column + * families before writing to disk. + * + * This is distinct from {@link ColumnFamilyOptions#writeBufferSize()}, + * which enforces a limit for a single memtable. + * + * This feature is disabled by default. Specify a non-zero value + * to enable it. + * + * Default: 0 (disabled) + * + * @param dbWriteBufferSize the size of the write buffer + * + * @return the reference to the current options. + */ + T setDbWriteBufferSize(long dbWriteBufferSize); + + /** + * Use passed {@link WriteBufferManager} to control memory usage across + * multiple column families and/or DB instances. + * + * Check + * https://github.com/facebook/rocksdb/wiki/Write-Buffer-Manager + * for more details on when to use it + * + * @param writeBufferManager The WriteBufferManager to use + * @return the reference of the current options. + */ + T setWriteBufferManager(final WriteBufferManager writeBufferManager); + + /** + * Reference to {@link WriteBufferManager} used by it.
    + * + * Default: null (Disabled) + * + * @return a reference to WriteBufferManager + */ + WriteBufferManager writeBufferManager(); + + /** + * Amount of data to build up in memtables across all column + * families before writing to disk. + * + * This is distinct from {@link ColumnFamilyOptions#writeBufferSize()}, + * which enforces a limit for a single memtable. + * + * This feature is disabled by default. Specify a non-zero value + * to enable it. + * + * Default: 0 (disabled) + * + * @return the size of the write buffer + */ + long dbWriteBufferSize(); + + /** + * Specify the file access pattern once a compaction is started. + * It will be applied to all input files of a compaction. + * + * Default: {@link AccessHint#NORMAL} + * + * @param accessHint The access hint + * + * @return the reference to the current options. + */ + @Deprecated T setAccessHintOnCompactionStart(final AccessHint accessHint); + + /** + * Specify the file access pattern once a compaction is started. + * It will be applied to all input files of a compaction. + * + * Default: {@link AccessHint#NORMAL} + * + * @return The access hint + */ + @Deprecated AccessHint accessHintOnCompactionStart(); + + /** + * This is a maximum buffer size that is used by WinMmapReadableFile in + * unbuffered disk I/O mode. We need to maintain an aligned buffer for + * reads. We allow the buffer to grow until the specified value and then + * for bigger requests allocate one shot buffers. In unbuffered mode we + * always bypass read-ahead buffer at ReadaheadRandomAccessFile + * When read-ahead is required we then make use of + * {@link MutableDBOptionsInterface#compactionReadaheadSize()} value and + * always try to read ahead. + * With read-ahead we always pre-allocate buffer to the size instead of + * growing it up to a limit. + * + * This option is currently honored only on Windows + * + * Default: 1 Mb + * + * Special value: 0 - means do not maintain per instance buffer. Allocate + * per request buffer and avoid locking. + * + * @param randomAccessMaxBufferSize the maximum size of the random access + * buffer + * + * @return the reference to the current options. + */ + T setRandomAccessMaxBufferSize(long randomAccessMaxBufferSize); + + /** + * This is a maximum buffer size that is used by WinMmapReadableFile in + * unbuffered disk I/O mode. We need to maintain an aligned buffer for + * reads. We allow the buffer to grow until the specified value and then + * for bigger requests allocate one shot buffers. In unbuffered mode we + * always bypass read-ahead buffer at ReadaheadRandomAccessFile + * When read-ahead is required we then make use of + * {@link MutableDBOptionsInterface#compactionReadaheadSize()} value and + * always try to read ahead. With read-ahead we always pre-allocate buffer + * to the size instead of growing it up to a limit. + * + * This option is currently honored only on Windows + * + * Default: 1 Mb + * + * Special value: 0 - means do not maintain per instance buffer. Allocate + * per request buffer and avoid locking. + * + * @return the maximum size of the random access buffer + */ + long randomAccessMaxBufferSize(); + + /** + * Use adaptive mutex, which spins in the user space before resorting + * to kernel. This could reduce context switch when the mutex is not + * heavily contended. However, if the mutex is hot, we could end up + * wasting spin time. + * Default: false + * + * @param useAdaptiveMutex true if adaptive mutex is used. + * @return the instance of the current object. + */ + T setUseAdaptiveMutex(boolean useAdaptiveMutex); + + /** + * Use adaptive mutex, which spins in the user space before resorting + * to kernel. This could reduce context switch when the mutex is not + * heavily contended. However, if the mutex is hot, we could end up + * wasting spin time. + * Default: false + * + * @return true if adaptive mutex is used. + */ + boolean useAdaptiveMutex(); + + /** + * Sets the {@link EventListener}s whose callback functions + * will be called when specific RocksDB event happens. + * + * Note: the RocksJava API currently only supports EventListeners implemented in Java. + * It could be extended in future to also support adding/removing EventListeners implemented in + * C++. + * + * @param listeners the listeners who should be notified on various events. + * + * @return the instance of the current object. + */ + T setListeners(final List listeners); + + /** + * Sets the {@link EventListener}s whose callback functions + * will be called when specific RocksDB event happens. + * + * Note: the RocksJava API currently only supports EventListeners implemented in Java. + * It could be extended in future to also support adding/removing EventListeners implemented in + * C++. + * + * @return the instance of the current object. + */ + List listeners(); + + /** + * If true, then the status of the threads involved in this DB will + * be tracked and available via GetThreadList() API. + * + * Default: false + * + * @param enableThreadTracking true to enable tracking + * + * @return the reference to the current options. + */ + T setEnableThreadTracking(boolean enableThreadTracking); + + /** + * If true, then the status of the threads involved in this DB will + * be tracked and available via GetThreadList() API. + * + * Default: false + * + * @return true if tracking is enabled + */ + boolean enableThreadTracking(); + + /** + * By default, a single write thread queue is maintained. The thread gets + * to the head of the queue becomes write batch group leader and responsible + * for writing to WAL and memtable for the batch group. + * + * If {@link #enablePipelinedWrite()} is true, separate write thread queue is + * maintained for WAL write and memtable write. A write thread first enter WAL + * writer queue and then memtable writer queue. Pending thread on the WAL + * writer queue thus only have to wait for previous writers to finish their + * WAL writing but not the memtable writing. Enabling the feature may improve + * write throughput and reduce latency of the prepare phase of two-phase + * commit. + * + * Default: false + * + * @param enablePipelinedWrite true to enabled pipelined writes + * + * @return the reference to the current options. + */ + T setEnablePipelinedWrite(final boolean enablePipelinedWrite); + + /** + * Returns true if pipelined writes are enabled. + * See {@link #setEnablePipelinedWrite(boolean)}. + * + * @return true if pipelined writes are enabled, false otherwise. + */ + boolean enablePipelinedWrite(); + + /** + * Setting {@link #unorderedWrite()} to true trades higher write throughput with + * relaxing the immutability guarantee of snapshots. This violates the + * repeatability one expects from ::Get from a snapshot, as well as + * ::MultiGet and Iterator's consistent-point-in-time view property. + * If the application cannot tolerate the relaxed guarantees, it can implement + * its own mechanisms to work around that and yet benefit from the higher + * throughput. Using TransactionDB with WRITE_PREPARED write policy and + * {@link #twoWriteQueues()} true is one way to achieve immutable snapshots despite + * unordered_write. + * + * By default, i.e., when it is false, rocksdb does not advance the sequence + * number for new snapshots unless all the writes with lower sequence numbers + * are already finished. This provides the immutability that we except from + * snapshots. Moreover, since Iterator and MultiGet internally depend on + * snapshots, the snapshot immutability results into Iterator and MultiGet + * offering consistent-point-in-time view. If set to true, although + * Read-Your-Own-Write property is still provided, the snapshot immutability + * property is relaxed: the writes issued after the snapshot is obtained (with + * larger sequence numbers) will be still not visible to the reads from that + * snapshot, however, there still might be pending writes (with lower sequence + * number) that will change the state visible to the snapshot after they are + * landed to the memtable. + * + * @param unorderedWrite true to enabled unordered write + * + * @return the reference to the current options. + */ + T setUnorderedWrite(final boolean unorderedWrite); + + /** + * Returns true if unordered write are enabled. + * See {@link #setUnorderedWrite(boolean)}. + * + * @return true if unordered write are enabled, false otherwise. + */ + boolean unorderedWrite(); + + /** + * If true, allow multi-writers to update mem tables in parallel. + * Only some memtable factorys support concurrent writes; currently it + * is implemented only for SkipListFactory. Concurrent memtable writes + * are not compatible with inplace_update_support or filter_deletes. + * It is strongly recommended to set + * {@link #setEnableWriteThreadAdaptiveYield(boolean)} if you are going to use + * this feature. + * Default: true + * + * @param allowConcurrentMemtableWrite true to enable concurrent writes + * for the memtable + * + * @return the reference to the current options. + */ + T setAllowConcurrentMemtableWrite(boolean allowConcurrentMemtableWrite); + + /** + * If true, allow multi-writers to update mem tables in parallel. + * Only some memtable factorys support concurrent writes; currently it + * is implemented only for SkipListFactory. Concurrent memtable writes + * are not compatible with inplace_update_support or filter_deletes. + * It is strongly recommended to set + * {@link #setEnableWriteThreadAdaptiveYield(boolean)} if you are going to use + * this feature. + * Default: true + * + * @return true if concurrent writes are enabled for the memtable + */ + boolean allowConcurrentMemtableWrite(); + + /** + * If true, threads synchronizing with the write batch group leader will + * wait for up to {@link #writeThreadMaxYieldUsec()} before blocking on a + * mutex. This can substantially improve throughput for concurrent workloads, + * regardless of whether {@link #allowConcurrentMemtableWrite()} is enabled. + * Default: true + * + * @param enableWriteThreadAdaptiveYield true to enable adaptive yield for the + * write threads + * + * @return the reference to the current options. + */ + T setEnableWriteThreadAdaptiveYield( + boolean enableWriteThreadAdaptiveYield); + + /** + * If true, threads synchronizing with the write batch group leader will + * wait for up to {@link #writeThreadMaxYieldUsec()} before blocking on a + * mutex. This can substantially improve throughput for concurrent workloads, + * regardless of whether {@link #allowConcurrentMemtableWrite()} is enabled. + * Default: true + * + * @return true if adaptive yield is enabled + * for the writing threads + */ + boolean enableWriteThreadAdaptiveYield(); + + /** + * The maximum number of microseconds that a write operation will use + * a yielding spin loop to coordinate with other write threads before + * blocking on a mutex. (Assuming {@link #writeThreadSlowYieldUsec()} is + * set properly) increasing this value is likely to increase RocksDB + * throughput at the expense of increased CPU usage. + * Default: 100 + * + * @param writeThreadMaxYieldUsec maximum number of microseconds + * + * @return the reference to the current options. + */ + T setWriteThreadMaxYieldUsec(long writeThreadMaxYieldUsec); + + /** + * The maximum number of microseconds that a write operation will use + * a yielding spin loop to coordinate with other write threads before + * blocking on a mutex. (Assuming {@link #writeThreadSlowYieldUsec()} is + * set properly) increasing this value is likely to increase RocksDB + * throughput at the expense of increased CPU usage. + * Default: 100 + * + * @return the maximum number of microseconds + */ + long writeThreadMaxYieldUsec(); + + /** + * The latency in microseconds after which a std::this_thread::yield + * call (sched_yield on Linux) is considered to be a signal that + * other processes or threads would like to use the current core. + * Increasing this makes writer threads more likely to take CPU + * by spinning, which will show up as an increase in the number of + * involuntary context switches. + * Default: 3 + * + * @param writeThreadSlowYieldUsec the latency in microseconds + * + * @return the reference to the current options. + */ + T setWriteThreadSlowYieldUsec(long writeThreadSlowYieldUsec); + + /** + * The latency in microseconds after which a std::this_thread::yield + * call (sched_yield on Linux) is considered to be a signal that + * other processes or threads would like to use the current core. + * Increasing this makes writer threads more likely to take CPU + * by spinning, which will show up as an increase in the number of + * involuntary context switches. + * Default: 3 + * + * @return writeThreadSlowYieldUsec the latency in microseconds + */ + long writeThreadSlowYieldUsec(); + + /** + * If true, then DB::Open() will not update the statistics used to optimize + * compaction decision by loading table properties from many files. + * Turning off this feature will improve DBOpen time especially in + * disk environment. + * + * Default: false + * + * @param skipStatsUpdateOnDbOpen true if updating stats will be skipped + * + * @return the reference to the current options. + */ + T setSkipStatsUpdateOnDbOpen(boolean skipStatsUpdateOnDbOpen); + + /** + * If true, then DB::Open() will not update the statistics used to optimize + * compaction decision by loading table properties from many files. + * Turning off this feature will improve DBOpen time especially in + * disk environment. + * + * Default: false + * + * @return true if updating stats will be skipped + */ + boolean skipStatsUpdateOnDbOpen(); + + /** + * If true, then {@link RocksDB#open(String)} will not fetch and check sizes of all sst files. + * This may significantly speed up startup if there are many sst files, + * especially when using non-default Env with expensive GetFileSize(). + * We'll still check that all required sst files exist. + * If {@code paranoid_checks} is false, this option is ignored, and sst files are + * not checked at all. + * + * Default: false + * + * @param skipCheckingSstFileSizesOnDbOpen if true, then SST file sizes will not be checked + * when calling {@link RocksDB#open(String)}. + * @return the reference to the current options. + */ + T setSkipCheckingSstFileSizesOnDbOpen(final boolean skipCheckingSstFileSizesOnDbOpen); + + /** + * If true, then {@link RocksDB#open(String)} will not fetch and check sizes of all sst files. + * This may significantly speed up startup if there are many sst files, + * especially when using non-default Env with expensive GetFileSize(). + * We'll still check that all required sst files exist. + * If {@code paranoid_checks} is false, this option is ignored, and sst files are + * not checked at all. + * + * Default: false + * + * @return true, if file sizes will not be checked when calling {@link RocksDB#open(String)}. + */ + boolean skipCheckingSstFileSizesOnDbOpen(); + + /** + * Recovery mode to control the consistency while replaying WAL + * + * Default: {@link WALRecoveryMode#PointInTimeRecovery} + * + * @param walRecoveryMode The WAL recover mode + * + * @return the reference to the current options. + */ + T setWalRecoveryMode(WALRecoveryMode walRecoveryMode); + + /** + * Recovery mode to control the consistency while replaying WAL + * + * Default: {@link WALRecoveryMode#PointInTimeRecovery} + * + * @return The WAL recover mode + */ + WALRecoveryMode walRecoveryMode(); + + /** + * if set to false then recovery will fail when a prepared + * transaction is encountered in the WAL + * + * Default: false + * + * @param allow2pc true if two-phase-commit is enabled + * + * @return the reference to the current options. + */ + T setAllow2pc(boolean allow2pc); + + /** + * if set to false then recovery will fail when a prepared + * transaction is encountered in the WAL + * + * Default: false + * + * @return true if two-phase-commit is enabled + */ + boolean allow2pc(); + + /** + * A global cache for table-level rows. + * + * Default: null (disabled) + * + * @param rowCache The global row cache + * + * @return the reference to the current options. + */ + T setRowCache(final Cache rowCache); + + /** + * A global cache for table-level rows. + * + * Default: null (disabled) + * + * @return The global row cache + */ + Cache rowCache(); + + /** + * A filter object supplied to be invoked while processing write-ahead-logs + * (WALs) during recovery. The filter provides a way to inspect log + * records, ignoring a particular record or skipping replay. + * The filter is invoked at startup and is invoked from a single-thread + * currently. + * + * @param walFilter the filter for processing WALs during recovery. + * + * @return the reference to the current options. + */ + T setWalFilter(final AbstractWalFilter walFilter); + + /** + * Get's the filter for processing WALs during recovery. + * See {@link #setWalFilter(AbstractWalFilter)}. + * + * @return the filter used for processing WALs during recovery. + */ + WalFilter walFilter(); + + /** + * If true, then DB::Open / CreateColumnFamily / DropColumnFamily + * / SetOptions will fail if options file is not detected or properly + * persisted. + * + * DEFAULT: false + * + * @param failIfOptionsFileError true if we should fail if there is an error + * in the options file + * + * @return the reference to the current options. + */ + T setFailIfOptionsFileError(boolean failIfOptionsFileError); + + /** + * If true, then DB::Open / CreateColumnFamily / DropColumnFamily + * / SetOptions will fail if options file is not detected or properly + * persisted. + * + * DEFAULT: false + * + * @return true if we should fail if there is an error in the options file + */ + boolean failIfOptionsFileError(); + + /** + * If true, then print malloc stats together with rocksdb.stats + * when printing to LOG. + * + * DEFAULT: false + * + * @param dumpMallocStats true if malloc stats should be printed to LOG + * + * @return the reference to the current options. + */ + T setDumpMallocStats(boolean dumpMallocStats); + + /** + * If true, then print malloc stats together with rocksdb.stats + * when printing to LOG. + * + * DEFAULT: false + * + * @return true if malloc stats should be printed to LOG + */ + boolean dumpMallocStats(); + + /** + * By default RocksDB replay WAL logs and flush them on DB open, which may + * create very small SST files. If this option is enabled, RocksDB will try + * to avoid (but not guarantee not to) flush during recovery. Also, existing + * WAL logs will be kept, so that if crash happened before flush, we still + * have logs to recover from. + * + * DEFAULT: false + * + * @param avoidFlushDuringRecovery true to try to avoid (but not guarantee + * not to) flush during recovery + * + * @return the reference to the current options. + */ + T setAvoidFlushDuringRecovery(boolean avoidFlushDuringRecovery); + + /** + * By default RocksDB replay WAL logs and flush them on DB open, which may + * create very small SST files. If this option is enabled, RocksDB will try + * to avoid (but not guarantee not to) flush during recovery. Also, existing + * WAL logs will be kept, so that if crash happened before flush, we still + * have logs to recover from. + * + * DEFAULT: false + * + * @return true to try to avoid (but not guarantee not to) flush during + * recovery + */ + boolean avoidFlushDuringRecovery(); + + /** + * Set this option to true during creation of database if you want + * to be able to ingest behind (call IngestExternalFile() skipping keys + * that already exist, rather than overwriting matching keys). + * Setting this option to true will affect 2 things: + * 1) Disable some internal optimizations around SST file compression + * 2) Reserve bottom-most level for ingested files only. + * 3) Note that num_levels should be >= 3 if this option is turned on. + * + * DEFAULT: false + * + * @param allowIngestBehind true to allow ingest behind, false to disallow. + * + * @return the reference to the current options. + */ + T setAllowIngestBehind(final boolean allowIngestBehind); + + /** + * Returns true if ingest behind is allowed. + * See {@link #setAllowIngestBehind(boolean)}. + * + * @return true if ingest behind is allowed, false otherwise. + */ + boolean allowIngestBehind(); + + /** + * If enabled it uses two queues for writes, one for the ones with + * disable_memtable and one for the ones that also write to memtable. This + * allows the memtable writes not to lag behind other writes. It can be used + * to optimize MySQL 2PC in which only the commits, which are serial, write to + * memtable. + * + * DEFAULT: false + * + * @param twoWriteQueues true to enable two write queues, false otherwise. + * + * @return the reference to the current options. + */ + T setTwoWriteQueues(final boolean twoWriteQueues); + + /** + * Returns true if two write queues are enabled. + * + * @return true if two write queues are enabled, false otherwise. + */ + boolean twoWriteQueues(); + + /** + * If true WAL is not flushed automatically after each write. Instead it + * relies on manual invocation of FlushWAL to write the WAL buffer to its + * file. + * + * DEFAULT: false + * + * @param manualWalFlush true to set disable automatic WAL flushing, + * false otherwise. + * + * @return the reference to the current options. + */ + T setManualWalFlush(final boolean manualWalFlush); + + /** + * Returns true if automatic WAL flushing is disabled. + * See {@link #setManualWalFlush(boolean)}. + * + * @return true if automatic WAL flushing is disabled, false otherwise. + */ + boolean manualWalFlush(); + + /** + * If true, RocksDB supports flushing multiple column families and committing + * their results atomically to MANIFEST. Note that it is not + * necessary to set atomic_flush to true if WAL is always enabled since WAL + * allows the database to be restored to the last persistent state in WAL. + * This option is useful when there are column families with writes NOT + * protected by WAL. + * For manual flush, application has to specify which column families to + * flush atomically in {@link RocksDB#flush(FlushOptions, List)}. + * For auto-triggered flush, RocksDB atomically flushes ALL column families. + * + * Currently, any WAL-enabled writes after atomic flush may be replayed + * independently if the process crashes later and tries to recover. + * + * @param atomicFlush true to enable atomic flush of multiple column families. + * + * @return the reference to the current options. + */ + T setAtomicFlush(final boolean atomicFlush); + + /** + * Determine if atomic flush of multiple column families is enabled. + * + * See {@link #setAtomicFlush(boolean)}. + * + * @return true if atomic flush is enabled. + */ + boolean atomicFlush(); + + /** + * If true, working thread may avoid doing unnecessary and long-latency + * operation (such as deleting obsolete files directly or deleting memtable) + * and will instead schedule a background job to do it. + * Use it if you're latency-sensitive. + * If set to true, takes precedence over + * {@link ReadOptions#setBackgroundPurgeOnIteratorCleanup(boolean)}. + * + * @param avoidUnnecessaryBlockingIO If true, working thread may avoid doing unnecessary + * operation. + * @return the reference to the current options. + */ + T setAvoidUnnecessaryBlockingIO(final boolean avoidUnnecessaryBlockingIO); + + /** + * If true, working thread may avoid doing unnecessary and long-latency + * operation (such as deleting obsolete files directly or deleting memtable) + * and will instead schedule a background job to do it. + * Use it if you're latency-sensitive. + * If set to true, takes precedence over + * {@link ReadOptions#setBackgroundPurgeOnIteratorCleanup(boolean)}. + * + * @return true, if working thread may avoid doing unnecessary operation. + */ + boolean avoidUnnecessaryBlockingIO(); + + /** + * If true, automatically persist stats to a hidden column family (column + * family name: ___rocksdb_stats_history___) every + * stats_persist_period_sec seconds; otherwise, write to an in-memory + * struct. User can query through `GetStatsHistory` API. + * If user attempts to create a column family with the same name on a DB + * which have previously set persist_stats_to_disk to true, the column family + * creation will fail, but the hidden column family will survive, as well as + * the previously persisted statistics. + * When peristing stats to disk, the stat name will be limited at 100 bytes. + * Default: false + * + * @param persistStatsToDisk true if stats should be persisted to hidden column family. + * @return the instance of the current object. + */ + T setPersistStatsToDisk(final boolean persistStatsToDisk); + + /** + * If true, automatically persist stats to a hidden column family (column + * family name: ___rocksdb_stats_history___) every + * stats_persist_period_sec seconds; otherwise, write to an in-memory + * struct. User can query through `GetStatsHistory` API. + * If user attempts to create a column family with the same name on a DB + * which have previously set persist_stats_to_disk to true, the column family + * creation will fail, but the hidden column family will survive, as well as + * the previously persisted statistics. + * When peristing stats to disk, the stat name will be limited at 100 bytes. + * Default: false + * + * @return true if stats should be persisted to hidden column family. + */ + boolean persistStatsToDisk(); + + /** + * Historically DB ID has always been stored in Identity File in DB folder. + * If this flag is true, the DB ID is written to Manifest file in addition + * to the Identity file. By doing this 2 problems are solved + * 1. We don't checksum the Identity file where as Manifest file is. + * 2. Since the source of truth for DB is Manifest file DB ID will sit with + * the source of truth. Previously the Identity file could be copied + * independent of Manifest and that can result in wrong DB ID. + * We recommend setting this flag to true. + * Default: false + * + * @param writeDbidToManifest if true, then DB ID will be written to Manifest file. + * @return the instance of the current object. + */ + T setWriteDbidToManifest(final boolean writeDbidToManifest); + + /** + * Historically DB ID has always been stored in Identity File in DB folder. + * If this flag is true, the DB ID is written to Manifest file in addition + * to the Identity file. By doing this 2 problems are solved + * 1. We don't checksum the Identity file where as Manifest file is. + * 2. Since the source of truth for DB is Manifest file DB ID will sit with + * the source of truth. Previously the Identity file could be copied + * independent of Manifest and that can result in wrong DB ID. + * We recommend setting this flag to true. + * Default: false + * + * @return true, if DB ID will be written to Manifest file. + */ + boolean writeDbidToManifest(); + + /** + * The number of bytes to prefetch when reading the log. This is mostly useful + * for reading a remotely located log, as it can save the number of + * round-trips. If 0, then the prefetching is disabled. + * + * Default: 0 + * + * @param logReadaheadSize the number of bytes to prefetch when reading the log. + * @return the instance of the current object. + */ + T setLogReadaheadSize(final long logReadaheadSize); + + /** + * The number of bytes to prefetch when reading the log. This is mostly useful + * for reading a remotely located log, as it can save the number of + * round-trips. If 0, then the prefetching is disabled. + * + * Default: 0 + * + * @return the number of bytes to prefetch when reading the log. + */ + long logReadaheadSize(); + + /** + * By default, RocksDB recovery fails if any table file referenced in + * MANIFEST are missing after scanning the MANIFEST. + * Best-efforts recovery is another recovery mode that + * tries to restore the database to the most recent point in time without + * missing file. + * Currently not compatible with atomic flush. Furthermore, WAL files will + * not be used for recovery if best_efforts_recovery is true. + * Default: false + * + * @param bestEffortsRecovery if true, RocksDB will use best-efforts mode when recovering. + * @return the instance of the current object. + */ + T setBestEffortsRecovery(final boolean bestEffortsRecovery); + + /** + * By default, RocksDB recovery fails if any table file referenced in + * MANIFEST are missing after scanning the MANIFEST. + * Best-efforts recovery is another recovery mode that + * tries to restore the database to the most recent point in time without + * missing file. + * Currently not compatible with atomic flush. Furthermore, WAL files will + * not be used for recovery if best_efforts_recovery is true. + * Default: false + * + * @return true, if RocksDB uses best-efforts mode when recovering. + */ + boolean bestEffortsRecovery(); + + /** + * It defines how many times db resume is called by a separate thread when + * background retryable IO Error happens. When background retryable IO + * Error happens, SetBGError is called to deal with the error. If the error + * can be auto-recovered (e.g., retryable IO Error during Flush or WAL write), + * then db resume is called in background to recover from the error. If this + * value is 0 or negative, db resume will not be called. + * + * Default: INT_MAX + * + * @param maxBgerrorResumeCount maximum number of times db resume should be called when IO Error + * happens. + * @return the instance of the current object. + */ + T setMaxBgErrorResumeCount(final int maxBgerrorResumeCount); + + /** + * It defines how many times db resume is called by a separate thread when + * background retryable IO Error happens. When background retryable IO + * Error happens, SetBGError is called to deal with the error. If the error + * can be auto-recovered (e.g., retryable IO Error during Flush or WAL write), + * then db resume is called in background to recover from the error. If this + * value is 0 or negative, db resume will not be called. + * + * Default: INT_MAX + * + * @return maximum number of times db resume should be called when IO Error happens. + */ + int maxBgerrorResumeCount(); + + /** + * If max_bgerror_resume_count is ≥ 2, db resume is called multiple times. + * This option decides how long to wait to retry the next resume if the + * previous resume fails and satisfy redo resume conditions. + * + * Default: 1000000 (microseconds). + * + * @param bgerrorResumeRetryInterval how many microseconds to wait between DB resume attempts. + * @return the instance of the current object. + */ + T setBgerrorResumeRetryInterval(final long bgerrorResumeRetryInterval); + + /** + * If max_bgerror_resume_count is ≥ 2, db resume is called multiple times. + * This option decides how long to wait to retry the next resume if the + * previous resume fails and satisfy redo resume conditions. + * + * Default: 1000000 (microseconds). + * + * @return the instance of the current object. + */ + long bgerrorResumeRetryInterval(); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/DataBlockIndexType.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/DataBlockIndexType.java new file mode 100644 index 0000000..513e5b4 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/DataBlockIndexType.java @@ -0,0 +1,32 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + + +/** + * DataBlockIndexType used in conjunction with BlockBasedTable. + */ +public enum DataBlockIndexType { + /** + * traditional block type + */ + kDataBlockBinarySearch((byte)0x0), + + /** + * additional hash index + */ + kDataBlockBinaryAndHash((byte)0x1); + + private final byte value; + + DataBlockIndexType(final byte value) { + this.value = value; + } + + byte getValue() { + return value; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/DbPath.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/DbPath.java new file mode 100644 index 0000000..3f0b675 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/DbPath.java @@ -0,0 +1,47 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.nio.file.Path; + +/** + * Tuple of database path and target size + */ +public class DbPath { + final Path path; + final long targetSize; + + public DbPath(final Path path, final long targetSize) { + this.path = path; + this.targetSize = targetSize; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + final DbPath dbPath = (DbPath) o; + + if (targetSize != dbPath.targetSize) { + return false; + } + + return path != null ? path.equals(dbPath.path) : dbPath.path == null; + } + + @Override + public int hashCode() { + int result = path != null ? path.hashCode() : 0; + result = 31 * result + (int) (targetSize ^ (targetSize >>> 32)); + return result; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/DirectSlice.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/DirectSlice.java new file mode 100644 index 0000000..5aa0866 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/DirectSlice.java @@ -0,0 +1,136 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.nio.ByteBuffer; + +/** + * Base class for slices which will receive direct + * ByteBuffer based access to the underlying data. + *

    + * ByteBuffer backed slices typically perform better with + * larger keys and values. When using smaller keys and + * values consider using @see org.rocksdb.Slice + */ +public class DirectSlice extends AbstractSlice { + public static final DirectSlice NONE = new DirectSlice(); + + /** + * Indicates whether we have to free the memory pointed to by the Slice + */ + private final boolean internalBuffer; + private volatile boolean cleared = false; + private volatile long internalBufferOffset = 0; + + /** + * Called from JNI to construct a new Java DirectSlice + * without an underlying C++ object set + * at creation time. + *

    + * Note: You should be aware that it is intentionally marked as + * package-private. This is so that developers cannot construct their own + * default DirectSlice objects (at present). As developers cannot construct + * their own DirectSlice objects through this, they are not creating + * underlying C++ DirectSlice objects, and so there is nothing to free + * (dispose) from Java. + */ + DirectSlice() { + super(); + this.internalBuffer = false; + } + + /** + * Constructs a slice + * where the data is taken from + * a String. + * + * @param str The string + */ + public DirectSlice(final String str) { + super(createNewSliceFromString(str)); + this.internalBuffer = true; + } + + /** + * Constructs a slice where the data is + * read from the provided + * ByteBuffer up to a certain length + * + * @param data The buffer containing the data + * @param length The length of the data to use for the slice + */ + public DirectSlice(final ByteBuffer data, final int length) { + super(createNewDirectSlice0(ensureDirect(data), length)); + this.internalBuffer = false; + } + + /** + * Constructs a slice where the data is + * read from the provided + * ByteBuffer + * + * @param data The bugger containing the data + */ + public DirectSlice(final ByteBuffer data) { + super(createNewDirectSlice1(ensureDirect(data))); + this.internalBuffer = false; + } + + private static ByteBuffer ensureDirect(final ByteBuffer data) { + if(!data.isDirect()) { + throw new IllegalArgumentException("The ByteBuffer must be direct"); + } + return data; + } + + /** + * Retrieves the byte at a specific offset + * from the underlying data + * + * @param offset The (zero-based) offset of the byte to retrieve + * + * @return the requested byte + */ + public byte get(final int offset) { + return get0(getNativeHandle(), offset); + } + + @Override + public void clear() { + clear0(getNativeHandle(), !cleared && internalBuffer, internalBufferOffset); + cleared = true; + } + + @Override + public void removePrefix(final int n) { + removePrefix0(getNativeHandle(), n); + this.internalBufferOffset += n; + } + + public void setLength(final int n) { + setLength0(getNativeHandle(), n); + } + + @Override + protected void disposeInternal() { + final long nativeHandle = getNativeHandle(); + if(!cleared && internalBuffer) { + disposeInternalBuf(nativeHandle, internalBufferOffset); + } + disposeInternal(nativeHandle); + } + + private static native long createNewDirectSlice0(final ByteBuffer data, final int length); + private static native long createNewDirectSlice1(final ByteBuffer data); + @Override protected final native ByteBuffer data0(long handle); + private native byte get0(long handle, int offset); + private native void clear0(long handle, boolean internalBuffer, + long internalBufferOffset); + private native void removePrefix0(long handle, int length); + private native void setLength0(long handle, int length); + private native void disposeInternalBuf(final long handle, + long internalBufferOffset); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/EncodingType.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/EncodingType.java new file mode 100644 index 0000000..c2790c1 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/EncodingType.java @@ -0,0 +1,55 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * EncodingType + * + *

    The value will determine how to encode keys + * when writing to a new SST file.

    + * + *

    This value will be stored + * inside the SST file which will be used when reading from + * the file, which makes it possible for users to choose + * different encoding type when reopening a DB. Files with + * different encoding types can co-exist in the same DB and + * can be read.

    + */ +public enum EncodingType { + /** + * Always write full keys without any special encoding. + */ + kPlain((byte) 0), + /** + *

    Find opportunity to write the same prefix once for multiple rows. + * In some cases, when a key follows a previous key with the same prefix, + * instead of writing out the full key, it just writes out the size of the + * shared prefix, as well as other bytes, to save some bytes.

    + * + *

    When using this option, the user is required to use the same prefix + * extractor to make sure the same prefix will be extracted from the same key. + * The Name() value of the prefix extractor will be stored in the file. When + * reopening the file, the name of the options.prefix_extractor given will be + * bitwise compared to the prefix extractors stored in the file. An error + * will be returned if the two don't match.

    + */ + kPrefix((byte) 1); + + /** + * Returns the byte value of the enumerations value + * + * @return byte representation + */ + public byte getValue() { + return value_; + } + + private EncodingType(final byte value) { + value_ = value; + } + + private final byte value_; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Env.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Env.java new file mode 100644 index 0000000..db4c6fd --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Env.java @@ -0,0 +1,167 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Arrays; +import java.util.List; + +/** + * Base class for all Env implementations in RocksDB. + */ +public abstract class Env extends RocksObject { + + static { + RocksDB.loadLibrary(); + } + + private static final Env DEFAULT_ENV = new RocksEnv(getDefaultEnvInternal()); + static { + /* + * The Ownership of the Default Env belongs to C++ + * and so we disown the native handle here so that + * we cannot accidentally free it from Java. + */ + DEFAULT_ENV.disOwnNativeHandle(); + } + + /** + *

    Returns the default environment suitable for the current operating + * system.

    + * + *

    The result of {@code getDefault()} is a singleton whose ownership + * belongs to rocksdb c++. As a result, the returned RocksEnv will not + * have the ownership of its c++ resource, and calling its dispose()/close() + * will be no-op.

    + * + * @return the default {@link org.rocksdb.RocksEnv} instance. + */ + public static Env getDefault() { + return DEFAULT_ENV; + } + + /** + *

    Sets the number of background worker threads of the low priority + * pool for this environment.

    + *

    Default number: 1

    + * + * @param number the number of threads + * + * @return current {@link RocksEnv} instance. + */ + public Env setBackgroundThreads(final int number) { + return setBackgroundThreads(number, Priority.LOW); + } + + /** + *

    Gets the number of background worker threads of the pool + * for this environment.

    + * + * @param priority the priority id of a specified thread pool. + * + * @return the number of threads. + */ + public int getBackgroundThreads(final Priority priority) { + return getBackgroundThreads(nativeHandle_, priority.getValue()); + } + + /** + *

    Sets the number of background worker threads of the specified thread + * pool for this environment.

    + * + * @param number the number of threads + * @param priority the priority id of a specified thread pool. + * + *

    Default number: 1

    + * @return current {@link RocksEnv} instance. + */ + public Env setBackgroundThreads(final int number, final Priority priority) { + setBackgroundThreads(nativeHandle_, number, priority.getValue()); + return this; + } + + /** + *

    Returns the length of the queue associated with the specified + * thread pool.

    + * + * @param priority the priority id of a specified thread pool. + * + * @return the thread pool queue length. + */ + public int getThreadPoolQueueLen(final Priority priority) { + return getThreadPoolQueueLen(nativeHandle_, priority.getValue()); + } + + /** + * Enlarge number of background worker threads of a specific thread pool + * for this environment if it is smaller than specified. 'LOW' is the default + * pool. + * + * @param number the number of threads. + * @param priority the priority id of a specified thread pool. + * + * @return current {@link RocksEnv} instance. + */ + public Env incBackgroundThreadsIfNeeded(final int number, + final Priority priority) { + incBackgroundThreadsIfNeeded(nativeHandle_, number, priority.getValue()); + return this; + } + + /** + * Lower IO priority for threads from the specified pool. + * + * @param priority the priority id of a specified thread pool. + * + * @return current {@link RocksEnv} instance. + */ + public Env lowerThreadPoolIOPriority(final Priority priority) { + lowerThreadPoolIOPriority(nativeHandle_, priority.getValue()); + return this; + } + + /** + * Lower CPU priority for threads from the specified pool. + * + * @param priority the priority id of a specified thread pool. + * + * @return current {@link RocksEnv} instance. + */ + public Env lowerThreadPoolCPUPriority(final Priority priority) { + lowerThreadPoolCPUPriority(nativeHandle_, priority.getValue()); + return this; + } + + /** + * Returns the status of all threads that belong to the current Env. + * + * @return the status of all threads belong to this env. + * + * @throws RocksDBException if the thread list cannot be acquired. + */ + public List getThreadList() throws RocksDBException { + return Arrays.asList(getThreadList(nativeHandle_)); + } + + Env(final long nativeHandle) { + super(nativeHandle); + } + + private static native long getDefaultEnvInternal(); + private native void setBackgroundThreads( + final long handle, final int number, final byte priority); + private native int getBackgroundThreads(final long handle, + final byte priority); + private native int getThreadPoolQueueLen(final long handle, + final byte priority); + private native void incBackgroundThreadsIfNeeded(final long handle, + final int number, final byte priority); + private native void lowerThreadPoolIOPriority(final long handle, + final byte priority); + private native void lowerThreadPoolCPUPriority(final long handle, + final byte priority); + private native ThreadStatus[] getThreadList(final long handle) + throws RocksDBException; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/EnvOptions.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/EnvOptions.java new file mode 100644 index 0000000..5cb193a --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/EnvOptions.java @@ -0,0 +1,366 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Options while opening a file to read/write + */ +public class EnvOptions extends RocksObject { + static { + RocksDB.loadLibrary(); + } + + /** + * Construct with default Options + */ + public EnvOptions() { + super(newEnvOptions()); + } + + /** + * Construct from {@link DBOptions}. + * + * @param dbOptions the database options. + */ + public EnvOptions(final DBOptions dbOptions) { + super(newEnvOptions(dbOptions.nativeHandle_)); + } + + /** + * Enable/Disable memory mapped reads. + *

    + * Default: false + * + * @param useMmapReads true to enable memory mapped reads, false to disable. + * + * @return the reference to these options. + */ + public EnvOptions setUseMmapReads(final boolean useMmapReads) { + setUseMmapReads(nativeHandle_, useMmapReads); + return this; + } + + /** + * Determine if memory mapped reads are in-use. + * + * @return true if memory mapped reads are in-use, false otherwise. + */ + public boolean useMmapReads() { + assert(isOwningHandle()); + return useMmapReads(nativeHandle_); + } + + /** + * Enable/Disable memory mapped Writes. + *

    + * Default: true + * + * @param useMmapWrites true to enable memory mapped writes, false to disable. + * + * @return the reference to these options. + */ + public EnvOptions setUseMmapWrites(final boolean useMmapWrites) { + setUseMmapWrites(nativeHandle_, useMmapWrites); + return this; + } + + /** + * Determine if memory mapped writes are in-use. + * + * @return true if memory mapped writes are in-use, false otherwise. + */ + public boolean useMmapWrites() { + assert(isOwningHandle()); + return useMmapWrites(nativeHandle_); + } + + /** + * Enable/Disable direct reads, i.e. {@code O_DIRECT}. + *

    + * Default: false + * + * @param useDirectReads true to enable direct reads, false to disable. + * + * @return the reference to these options. + */ + public EnvOptions setUseDirectReads(final boolean useDirectReads) { + setUseDirectReads(nativeHandle_, useDirectReads); + return this; + } + + /** + * Determine if direct reads are in-use. + * + * @return true if direct reads are in-use, false otherwise. + */ + public boolean useDirectReads() { + assert(isOwningHandle()); + return useDirectReads(nativeHandle_); + } + + /** + * Enable/Disable direct writes, i.e. {@code O_DIRECT}. + *

    + * Default: false + * + * @param useDirectWrites true to enable direct writes, false to disable. + * + * @return the reference to these options. + */ + public EnvOptions setUseDirectWrites(final boolean useDirectWrites) { + setUseDirectWrites(nativeHandle_, useDirectWrites); + return this; + } + + /** + * Determine if direct writes are in-use. + * + * @return true if direct writes are in-use, false otherwise. + */ + public boolean useDirectWrites() { + assert(isOwningHandle()); + return useDirectWrites(nativeHandle_); + } + + /** + * Enable/Disable fallocate calls. + *

    + * Default: true + *

    + * If false, {@code fallocate()} calls are bypassed. + * + * @param allowFallocate true to enable fallocate calls, false to disable. + * + * @return the reference to these options. + */ + public EnvOptions setAllowFallocate(final boolean allowFallocate) { + setAllowFallocate(nativeHandle_, allowFallocate); + return this; + } + + /** + * Determine if fallocate calls are used. + * + * @return true if fallocate calls are used, false otherwise. + */ + public boolean allowFallocate() { + assert(isOwningHandle()); + return allowFallocate(nativeHandle_); + } + + /** + * Enable/Disable the {@code FD_CLOEXEC} bit when opening file descriptors. + *

    + * Default: true + * + * @param setFdCloexec true to enable the {@code FB_CLOEXEC} bit, + * false to disable. + * + * @return the reference to these options. + */ + public EnvOptions setSetFdCloexec(final boolean setFdCloexec) { + setSetFdCloexec(nativeHandle_, setFdCloexec); + return this; + } + + /** + * Determine i fthe {@code FD_CLOEXEC} bit is set when opening file + * descriptors. + * + * @return true if the {@code FB_CLOEXEC} bit is enabled, false otherwise. + */ + public boolean setFdCloexec() { + assert(isOwningHandle()); + return setFdCloexec(nativeHandle_); + } + + /** + * Allows OS to incrementally sync files to disk while they are being + * written, in the background. Issue one request for every + * {@code bytesPerSync} written. + *

    + * Default: 0 + * + * @param bytesPerSync 0 to disable, otherwise the number of bytes. + * + * @return the reference to these options. + */ + public EnvOptions setBytesPerSync(final long bytesPerSync) { + setBytesPerSync(nativeHandle_, bytesPerSync); + return this; + } + + /** + * Get the number of incremental bytes per sync written in the background. + * + * @return 0 if disabled, otherwise the number of bytes. + */ + public long bytesPerSync() { + assert(isOwningHandle()); + return bytesPerSync(nativeHandle_); + } + + /** + * If true, we will preallocate the file with {@code FALLOC_FL_KEEP_SIZE} + * flag, which means that file size won't change as part of preallocation. + * If false, preallocation will also change the file size. This option will + * improve the performance in workloads where you sync the data on every + * write. By default, we set it to true for MANIFEST writes and false for + * WAL writes + * + * @param fallocateWithKeepSize true to preallocate, false otherwise. + * + * @return the reference to these options. + */ + public EnvOptions setFallocateWithKeepSize( + final boolean fallocateWithKeepSize) { + setFallocateWithKeepSize(nativeHandle_, fallocateWithKeepSize); + return this; + } + + /** + * Determine if file is preallocated. + * + * @return true if the file is preallocated, false otherwise. + */ + public boolean fallocateWithKeepSize() { + assert(isOwningHandle()); + return fallocateWithKeepSize(nativeHandle_); + } + + /** + * See {@link DBOptions#setCompactionReadaheadSize(long)}. + * + * @param compactionReadaheadSize the compaction read-ahead size. + * + * @return the reference to these options. + */ + public EnvOptions setCompactionReadaheadSize( + final long compactionReadaheadSize) { + setCompactionReadaheadSize(nativeHandle_, compactionReadaheadSize); + return this; + } + + /** + * See {@link DBOptions#compactionReadaheadSize()}. + * + * @return the compaction read-ahead size. + */ + public long compactionReadaheadSize() { + assert(isOwningHandle()); + return compactionReadaheadSize(nativeHandle_); + } + + /** + * See {@link DBOptions#setRandomAccessMaxBufferSize(long)}. + * + * @param randomAccessMaxBufferSize the max buffer size for random access. + * + * @return the reference to these options. + */ + public EnvOptions setRandomAccessMaxBufferSize( + final long randomAccessMaxBufferSize) { + setRandomAccessMaxBufferSize(nativeHandle_, randomAccessMaxBufferSize); + return this; + } + + /** + * See {@link DBOptions#randomAccessMaxBufferSize()}. + * + * @return the max buffer size for random access. + */ + public long randomAccessMaxBufferSize() { + assert(isOwningHandle()); + return randomAccessMaxBufferSize(nativeHandle_); + } + + /** + * See {@link DBOptions#setWritableFileMaxBufferSize(long)}. + * + * @param writableFileMaxBufferSize the max buffer size. + * + * @return the reference to these options. + */ + public EnvOptions setWritableFileMaxBufferSize( + final long writableFileMaxBufferSize) { + setWritableFileMaxBufferSize(nativeHandle_, writableFileMaxBufferSize); + return this; + } + + /** + * See {@link DBOptions#writableFileMaxBufferSize()}. + * + * @return the max buffer size. + */ + public long writableFileMaxBufferSize() { + assert(isOwningHandle()); + return writableFileMaxBufferSize(nativeHandle_); + } + + /** + * Set the write rate limiter for flush and compaction. + * + * @param rateLimiter the rate limiter. + * + * @return the reference to these options. + */ + public EnvOptions setRateLimiter(final RateLimiter rateLimiter) { + this.rateLimiter = rateLimiter; + setRateLimiter(nativeHandle_, rateLimiter.nativeHandle_); + return this; + } + + /** + * Get the write rate limiter for flush and compaction. + * + * @return the rate limiter. + */ + public RateLimiter rateLimiter() { + assert(isOwningHandle()); + return rateLimiter; + } + + private static native long newEnvOptions(); + private static native long newEnvOptions(final long dboptions_handle); + @Override protected final native void disposeInternal(final long handle); + + private native void setUseMmapReads(final long handle, + final boolean useMmapReads); + private native boolean useMmapReads(final long handle); + private native void setUseMmapWrites(final long handle, + final boolean useMmapWrites); + private native boolean useMmapWrites(final long handle); + private native void setUseDirectReads(final long handle, + final boolean useDirectReads); + private native boolean useDirectReads(final long handle); + private native void setUseDirectWrites(final long handle, + final boolean useDirectWrites); + private native boolean useDirectWrites(final long handle); + private native void setAllowFallocate(final long handle, + final boolean allowFallocate); + private native boolean allowFallocate(final long handle); + private native void setSetFdCloexec(final long handle, + final boolean setFdCloexec); + private native boolean setFdCloexec(final long handle); + private native void setBytesPerSync(final long handle, + final long bytesPerSync); + private native long bytesPerSync(final long handle); + private native void setFallocateWithKeepSize( + final long handle, final boolean fallocateWithKeepSize); + private native boolean fallocateWithKeepSize(final long handle); + private native void setCompactionReadaheadSize( + final long handle, final long compactionReadaheadSize); + private native long compactionReadaheadSize(final long handle); + private native void setRandomAccessMaxBufferSize( + final long handle, final long randomAccessMaxBufferSize); + private native long randomAccessMaxBufferSize(final long handle); + private native void setWritableFileMaxBufferSize( + final long handle, final long writableFileMaxBufferSize); + private native long writableFileMaxBufferSize(final long handle); + private native void setRateLimiter(final long handle, + final long rateLimiterHandle); + private RateLimiter rateLimiter; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/EventListener.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/EventListener.java new file mode 100644 index 0000000..27652ea --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/EventListener.java @@ -0,0 +1,335 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.List; + +/** + * EventListener class contains a set of callback functions that will + * be called when specific RocksDB event happens such as flush. It can + * be used as a building block for developing custom features such as + * stats-collector or external compaction algorithm. + *

    + * Note that callback functions should not run for an extended period of + * time before the function returns, otherwise RocksDB may be blocked. + * For example, it is not suggested to do + * {@link RocksDB#compactFiles(CompactionOptions, ColumnFamilyHandle, List, int, int, + * CompactionJobInfo)} (as it may run for a long while) or issue many of + * {@link RocksDB#put(ColumnFamilyHandle, WriteOptions, byte[], byte[])} + * (as Put may be blocked in certain cases) in the same thread in the + * EventListener callback. + *

    + * However, doing + * {@link RocksDB#compactFiles(CompactionOptions, ColumnFamilyHandle, List, int, int, + * CompactionJobInfo)} and {@link RocksDB#put(ColumnFamilyHandle, WriteOptions, byte[], byte[])} in + * another thread is considered safe. + *

    + * [Threading] All EventListener callback will be called using the + * actual thread that involves in that specific event. For example, it + * is the RocksDB background flush thread that does the actual flush to + * call {@link #onFlushCompleted(RocksDB, FlushJobInfo)}. + *

    + * [Locking] All EventListener callbacks are designed to be called without + * the current thread holding any DB mutex. This is to prevent potential + * deadlock and performance issue when using EventListener callback + * in a complex way. + */ +public interface EventListener { + /** + * A callback function to RocksDB which will be called before a + * RocksDB starts to flush memtables. + *

    + * Note that the this function must be implemented in a way such that + * it should not run for an extended period of time before the function + * returns. Otherwise, RocksDB may be blocked. + * + * @param db the database + * @param flushJobInfo the flush job info, contains data copied from + * respective native structure. + */ + void onFlushBegin(final RocksDB db, final FlushJobInfo flushJobInfo); + + /** + * callback function to RocksDB which will be called whenever a + * registered RocksDB flushes a file. + *

    + * Note that the this function must be implemented in a way such that + * it should not run for an extended period of time before the function + * returns. Otherwise, RocksDB may be blocked. + * + * @param db the database + * @param flushJobInfo the flush job info, contains data copied from + * respective native structure. + */ + void onFlushCompleted(final RocksDB db, final FlushJobInfo flushJobInfo); + + /** + * A callback function for RocksDB which will be called whenever + * a SST file is deleted. Different from + * {@link #onCompactionCompleted(RocksDB, CompactionJobInfo)} and + * {@link #onFlushCompleted(RocksDB, FlushJobInfo)}, + * this callback is designed for external logging + * service and thus only provide string parameters instead + * of a pointer to DB. Applications that build logic basic based + * on file creations and deletions is suggested to implement + * {@link #onFlushCompleted(RocksDB, FlushJobInfo)} and + * {@link #onCompactionCompleted(RocksDB, CompactionJobInfo)}. + *

    + * Note that if applications would like to use the passed reference + * outside this function call, they should make copies from the + * returned value. + * + * @param tableFileDeletionInfo the table file deletion info, + * contains data copied from respective native structure. + */ + void onTableFileDeleted(final TableFileDeletionInfo tableFileDeletionInfo); + + /** + * A callback function to RocksDB which will be called before a + * RocksDB starts to compact. The default implementation is + * no-op. + *

    + * Note that the this function must be implemented in a way such that + * it should not run for an extended period of time before the function + * returns. Otherwise, RocksDB may be blocked. + * + * @param db a pointer to the rocksdb instance which just compacted + * a file. + * @param compactionJobInfo a reference to a native CompactionJobInfo struct, + * which is released after this function is returned, and must be copied + * if it is needed outside of this function. + */ + void onCompactionBegin(final RocksDB db, final CompactionJobInfo compactionJobInfo); + + /** + * A callback function for RocksDB which will be called whenever + * a registered RocksDB compacts a file. The default implementation + * is a no-op. + *

    + * Note that this function must be implemented in a way such that + * it should not run for an extended period of time before the function + * returns. Otherwise, RocksDB may be blocked. + * + * @param db a pointer to the rocksdb instance which just compacted + * a file. + * @param compactionJobInfo a reference to a native CompactionJobInfo struct, + * which is released after this function is returned, and must be copied + * if it is needed outside of this function. + */ + void onCompactionCompleted(final RocksDB db, final CompactionJobInfo compactionJobInfo); + + /** + * A callback function for RocksDB which will be called whenever + * a SST file is created. Different from OnCompactionCompleted and + * OnFlushCompleted, this callback is designed for external logging + * service and thus only provide string parameters instead + * of a pointer to DB. Applications that build logic basic based + * on file creations and deletions is suggested to implement + * OnFlushCompleted and OnCompactionCompleted. + *

    + * Historically it will only be called if the file is successfully created. + * Now it will also be called on failure case. User can check info.status + * to see if it succeeded or not. + *

    + * Note that if applications would like to use the passed reference + * outside this function call, they should make copies from these + * returned value. + * + * @param tableFileCreationInfo the table file creation info, + * contains data copied from respective native structure. + */ + void onTableFileCreated(final TableFileCreationInfo tableFileCreationInfo); + + /** + * A callback function for RocksDB which will be called before + * a SST file is being created. It will follow by OnTableFileCreated after + * the creation finishes. + *

    + * Note that if applications would like to use the passed reference + * outside this function call, they should make copies from these + * returned value. + * + * @param tableFileCreationBriefInfo the table file creation brief info, + * contains data copied from respective native structure. + */ + void onTableFileCreationStarted(final TableFileCreationBriefInfo tableFileCreationBriefInfo); + + /** + * A callback function for RocksDB which will be called before + * a memtable is made immutable. + *

    + * Note that the this function must be implemented in a way such that + * it should not run for an extended period of time before the function + * returns. Otherwise, RocksDB may be blocked. + *

    + * Note that if applications would like to use the passed reference + * outside this function call, they should make copies from these + * returned value. + * + * @param memTableInfo the mem table info, contains data + * copied from respective native structure. + */ + void onMemTableSealed(final MemTableInfo memTableInfo); + + /** + * A callback function for RocksDB which will be called before + * a column family handle is deleted. + *

    + * Note that the this function must be implemented in a way such that + * it should not run for an extended period of time before the function + * returns. Otherwise, RocksDB may be blocked. + * + * @param columnFamilyHandle is a pointer to the column family handle to be + * deleted which will become a dangling pointer after the deletion. + */ + void onColumnFamilyHandleDeletionStarted(final ColumnFamilyHandle columnFamilyHandle); + + /** + * A callback function for RocksDB which will be called after an external + * file is ingested using IngestExternalFile. + *

    + * Note that the this function will run on the same thread as + * IngestExternalFile(), if this function is blocked, IngestExternalFile() + * will be blocked from finishing. + * + * @param db the database + * @param externalFileIngestionInfo the external file ingestion info, + * contains data copied from respective native structure. + */ + void onExternalFileIngested( + final RocksDB db, final ExternalFileIngestionInfo externalFileIngestionInfo); + + /** + * A callback function for RocksDB which will be called before setting the + * background error status to a non-OK value. The new background error status + * is provided in `bg_error` and can be modified by the callback. E.g., a + * callback can suppress errors by resetting it to Status::OK(), thus + * preventing the database from entering read-only mode. We do not provide any + * guarantee when failed flushes/compactions will be rescheduled if the user + * suppresses an error. + *

    + * Note that this function can run on the same threads as flush, compaction, + * and user writes. So, it is extremely important not to perform heavy + * computations or blocking calls in this function. + * + * @param backgroundErrorReason background error reason code + * @param backgroundError background error codes + */ + void onBackgroundError( + final BackgroundErrorReason backgroundErrorReason, final Status backgroundError); + + /** + * A callback function for RocksDB which will be called whenever a change + * of superversion triggers a change of the stall conditions. + *

    + * Note that the this function must be implemented in a way such that + * it should not run for an extended period of time before the function + * returns. Otherwise, RocksDB may be blocked. + * + * @param writeStallInfo write stall info, + * contains data copied from respective native structure. + */ + void onStallConditionsChanged(final WriteStallInfo writeStallInfo); + + /** + * A callback function for RocksDB which will be called whenever a file read + * operation finishes. + * + * @param fileOperationInfo file operation info, + * contains data copied from respective native structure. + */ + void onFileReadFinish(final FileOperationInfo fileOperationInfo); + + /** + * A callback function for RocksDB which will be called whenever a file write + * operation finishes. + * + * @param fileOperationInfo file operation info, + * contains data copied from respective native structure. + */ + void onFileWriteFinish(final FileOperationInfo fileOperationInfo); + + /** + * A callback function for RocksDB which will be called whenever a file flush + * operation finishes. + * + * @param fileOperationInfo file operation info, + * contains data copied from respective native structure. + */ + void onFileFlushFinish(final FileOperationInfo fileOperationInfo); + + /** + * A callback function for RocksDB which will be called whenever a file sync + * operation finishes. + * + * @param fileOperationInfo file operation info, + * contains data copied from respective native structure. + */ + void onFileSyncFinish(final FileOperationInfo fileOperationInfo); + + /** + * A callback function for RocksDB which will be called whenever a file + * rangeSync operation finishes. + * + * @param fileOperationInfo file operation info, + * contains data copied from respective native structure. + */ + void onFileRangeSyncFinish(final FileOperationInfo fileOperationInfo); + + /** + * A callback function for RocksDB which will be called whenever a file + * truncate operation finishes. + * + * @param fileOperationInfo file operation info, + * contains data copied from respective native structure. + */ + void onFileTruncateFinish(final FileOperationInfo fileOperationInfo); + + /** + * A callback function for RocksDB which will be called whenever a file close + * operation finishes. + * + * @param fileOperationInfo file operation info, + * contains data copied from respective native structure. + */ + void onFileCloseFinish(final FileOperationInfo fileOperationInfo); + + /** + * If true, the {@link #onFileReadFinish(FileOperationInfo)} + * and {@link #onFileWriteFinish(FileOperationInfo)} will be called. If + * false, then they won't be called. + *

    + * Default: false + * + * @return whether to callback when file read/write is finished + */ + boolean shouldBeNotifiedOnFileIO(); + + /** + * A callback function for RocksDB which will be called just before + * starting the automatic recovery process for recoverable background + * errors, such as NoSpace(). The callback can suppress the automatic + * recovery by setting returning false. The database will then + * have to be transitioned out of read-only mode by calling + * RocksDB#resume(). + * + * @param backgroundErrorReason background error reason code + * @param backgroundError background error codes + * @return return {@code false} if the automatic recovery should be suppressed + */ + boolean onErrorRecoveryBegin( + final BackgroundErrorReason backgroundErrorReason, final Status backgroundError); + + /** + * A callback function for RocksDB which will be called once the database + * is recovered from read-only mode after an error. When this is called, it + * means normal writes to the database can be issued and the user can + * initiate any further recovery actions needed + * + * @param oldBackgroundError old background error codes + */ + void onErrorRecoveryCompleted(final Status oldBackgroundError); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Experimental.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Experimental.java new file mode 100644 index 0000000..64b404d --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Experimental.java @@ -0,0 +1,23 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Marks a feature as experimental, meaning that it is likely + * to change or even be removed/re-engineered in the future + */ +@Documented +@Retention(RetentionPolicy.SOURCE) +@Target({ElementType.TYPE, ElementType.METHOD}) +public @interface Experimental { + String value(); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ExternalFileIngestionInfo.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ExternalFileIngestionInfo.java new file mode 100644 index 0000000..7a99dd6 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ExternalFileIngestionInfo.java @@ -0,0 +1,103 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Objects; + +public class ExternalFileIngestionInfo { + private final String columnFamilyName; + private final String externalFilePath; + private final String internalFilePath; + private final long globalSeqno; + private final TableProperties tableProperties; + + /** + * Access is package private as this will only be constructed from + * C++ via JNI and for testing. + */ + ExternalFileIngestionInfo(final String columnFamilyName, final String externalFilePath, + final String internalFilePath, final long globalSeqno, + final TableProperties tableProperties) { + this.columnFamilyName = columnFamilyName; + this.externalFilePath = externalFilePath; + this.internalFilePath = internalFilePath; + this.globalSeqno = globalSeqno; + this.tableProperties = tableProperties; + } + + /** + * Get the name of the column family. + * + * @return the name of the column family. + */ + public String getColumnFamilyName() { + return columnFamilyName; + } + + /** + * Get the path of the file outside the DB. + * + * @return the path of the file outside the DB. + */ + public String getExternalFilePath() { + return externalFilePath; + } + + /** + * Get the path of the file inside the DB. + * + * @return the path of the file inside the DB. + */ + public String getInternalFilePath() { + return internalFilePath; + } + + /** + * Get the global sequence number assigned to keys in this file. + * + * @return the global sequence number. + */ + public long getGlobalSeqno() { + return globalSeqno; + } + + /** + * Get the Table properties of the table being flushed. + * + * @return the table properties. + */ + public TableProperties getTableProperties() { + return tableProperties; + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final ExternalFileIngestionInfo that = (ExternalFileIngestionInfo) o; + return globalSeqno == that.globalSeqno + && Objects.equals(columnFamilyName, that.columnFamilyName) + && Objects.equals(externalFilePath, that.externalFilePath) + && Objects.equals(internalFilePath, that.internalFilePath) + && Objects.equals(tableProperties, that.tableProperties); + } + + @Override + public int hashCode() { + return Objects.hash( + columnFamilyName, externalFilePath, internalFilePath, globalSeqno, tableProperties); + } + + @Override + public String toString() { + return "ExternalFileIngestionInfo{" + + "columnFamilyName='" + columnFamilyName + '\'' + ", externalFilePath='" + externalFilePath + + '\'' + ", internalFilePath='" + internalFilePath + '\'' + ", globalSeqno=" + globalSeqno + + ", tableProperties=" + tableProperties + '}'; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/FileOperationInfo.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/FileOperationInfo.java new file mode 100644 index 0000000..fae9cd5 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/FileOperationInfo.java @@ -0,0 +1,112 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Objects; + +/** + * Java representation of FileOperationInfo struct from include/rocksdb/listener.h + */ +public class FileOperationInfo { + private final String path; + private final long offset; + private final long length; + private final long startTimestamp; + private final long duration; + private final Status status; + + /** + * Access is private as this will only be constructed from + * C++ via JNI. + */ + FileOperationInfo(final String path, final long offset, final long length, + final long startTimestamp, final long duration, final Status status) { + this.path = path; + this.offset = offset; + this.length = length; + this.startTimestamp = startTimestamp; + this.duration = duration; + this.status = status; + } + + /** + * Get the file path. + * + * @return the file path. + */ + public String getPath() { + return path; + } + + /** + * Get the offset. + * + * @return the offset. + */ + public long getOffset() { + return offset; + } + + /** + * Get the length. + * + * @return the length. + */ + public long getLength() { + return length; + } + + /** + * Get the start timestamp (in nanoseconds). + * + * @return the start timestamp. + */ + public long getStartTimestamp() { + return startTimestamp; + } + + /** + * Get the operation duration (in nanoseconds). + * + * @return the operation duration. + */ + public long getDuration() { + return duration; + } + + /** + * Get the status. + * + * @return the status. + */ + public Status getStatus() { + return status; + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + FileOperationInfo that = (FileOperationInfo) o; + return offset == that.offset && length == that.length && startTimestamp == that.startTimestamp + && duration == that.duration && Objects.equals(path, that.path) + && Objects.equals(status, that.status); + } + + @Override + public int hashCode() { + return Objects.hash(path, offset, length, startTimestamp, duration, status); + } + + @Override + public String toString() { + return "FileOperationInfo{" + + "path='" + path + '\'' + ", offset=" + offset + ", length=" + length + ", startTimestamp=" + + startTimestamp + ", duration=" + duration + ", status=" + status + '}'; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Filter.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Filter.java new file mode 100644 index 0000000..7f490cf --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Filter.java @@ -0,0 +1,36 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Filters are stored in rocksdb and are consulted automatically + * by rocksdb to decide whether or not to read some + * information from disk. In many cases, a filter can cut down the + * number of disk seeks form a handful to a single disk seek per + * DB::Get() call. + */ +//TODO(AR) should be renamed FilterPolicy +public abstract class Filter extends RocksObject { + + protected Filter(final long nativeHandle) { + super(nativeHandle); + } + + /** + * Deletes underlying C++ filter pointer. + * + * Note that this function should be called only after all + * RocksDB instances referencing the filter are closed. + * Otherwise an undefined behavior will occur. + */ + @Override + protected void disposeInternal() { + disposeInternal(nativeHandle_); + } + + @Override + protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/FlushJobInfo.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/FlushJobInfo.java new file mode 100644 index 0000000..414d3a2 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/FlushJobInfo.java @@ -0,0 +1,186 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Objects; + +public class FlushJobInfo { + private final long columnFamilyId; + private final String columnFamilyName; + private final String filePath; + private final long threadId; + private final int jobId; + private final boolean triggeredWritesSlowdown; + private final boolean triggeredWritesStop; + private final long smallestSeqno; + private final long largestSeqno; + private final TableProperties tableProperties; + private final FlushReason flushReason; + + /** + * Access is package private as this will only be constructed from + * C++ via JNI and for testing. + */ + FlushJobInfo(final long columnFamilyId, final String columnFamilyName, final String filePath, + final long threadId, final int jobId, final boolean triggeredWritesSlowdown, + final boolean triggeredWritesStop, final long smallestSeqno, final long largestSeqno, + final TableProperties tableProperties, final byte flushReasonValue) { + this.columnFamilyId = columnFamilyId; + this.columnFamilyName = columnFamilyName; + this.filePath = filePath; + this.threadId = threadId; + this.jobId = jobId; + this.triggeredWritesSlowdown = triggeredWritesSlowdown; + this.triggeredWritesStop = triggeredWritesStop; + this.smallestSeqno = smallestSeqno; + this.largestSeqno = largestSeqno; + this.tableProperties = tableProperties; + this.flushReason = FlushReason.fromValue(flushReasonValue); + } + + /** + * Get the id of the column family. + * + * @return the id of the column family + */ + public long getColumnFamilyId() { + return columnFamilyId; + } + + /** + * Get the name of the column family. + * + * @return the name of the column family + */ + public String getColumnFamilyName() { + return columnFamilyName; + } + + /** + * Get the path to the newly created file. + * + * @return the path to the newly created file + */ + public String getFilePath() { + return filePath; + } + + /** + * Get the id of the thread that completed this flush job. + * + * @return the id of the thread that completed this flush job + */ + public long getThreadId() { + return threadId; + } + + /** + * Get the job id, which is unique in the same thread. + * + * @return the job id + */ + public int getJobId() { + return jobId; + } + + /** + * Determine if rocksdb is currently slowing-down all writes to prevent + * creating too many Level 0 files as compaction seems not able to + * catch up the write request speed. + *

    + * This indicates that there are too many files in Level 0. + * + * @return true if rocksdb is currently slowing-down all writes, + * false otherwise + */ + public boolean isTriggeredWritesSlowdown() { + return triggeredWritesSlowdown; + } + + /** + * Determine if rocksdb is currently blocking any writes to prevent + * creating more L0 files. + *

    + * This indicates that there are too many files in level 0. + * Compactions should try to compact L0 files down to lower levels as soon + * as possible. + * + * @return true if rocksdb is currently blocking any writes, false otherwise + */ + public boolean isTriggeredWritesStop() { + return triggeredWritesStop; + } + + /** + * Get the smallest sequence number in the newly created file. + * + * @return the smallest sequence number + */ + public long getSmallestSeqno() { + return smallestSeqno; + } + + /** + * Get the largest sequence number in the newly created file. + * + * @return the largest sequence number + */ + public long getLargestSeqno() { + return largestSeqno; + } + + /** + * Get the Table properties of the table being flushed. + * + * @return the Table properties of the table being flushed + */ + public TableProperties getTableProperties() { + return tableProperties; + } + + /** + * Get the reason for initiating the flush. + * + * @return the reason for initiating the flush. + */ + public FlushReason getFlushReason() { + return flushReason; + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final FlushJobInfo that = (FlushJobInfo) o; + return columnFamilyId == that.columnFamilyId && threadId == that.threadId && jobId == that.jobId + && triggeredWritesSlowdown == that.triggeredWritesSlowdown + && triggeredWritesStop == that.triggeredWritesStop && smallestSeqno == that.smallestSeqno + && largestSeqno == that.largestSeqno + && Objects.equals(columnFamilyName, that.columnFamilyName) + && Objects.equals(filePath, that.filePath) + && Objects.equals(tableProperties, that.tableProperties) && flushReason == that.flushReason; + } + + @Override + public int hashCode() { + return Objects.hash(columnFamilyId, columnFamilyName, filePath, threadId, jobId, + triggeredWritesSlowdown, triggeredWritesStop, smallestSeqno, largestSeqno, tableProperties, + flushReason); + } + + @Override + public String toString() { + return "FlushJobInfo{" + + "columnFamilyId=" + columnFamilyId + ", columnFamilyName='" + columnFamilyName + '\'' + + ", filePath='" + filePath + '\'' + ", threadId=" + threadId + ", jobId=" + jobId + + ", triggeredWritesSlowdown=" + triggeredWritesSlowdown + + ", triggeredWritesStop=" + triggeredWritesStop + ", smallestSeqno=" + smallestSeqno + + ", largestSeqno=" + largestSeqno + ", tableProperties=" + tableProperties + + ", flushReason=" + flushReason + '}'; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/FlushOptions.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/FlushOptions.java new file mode 100644 index 0000000..0ec8350 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/FlushOptions.java @@ -0,0 +1,90 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * FlushOptions to be passed to flush operations of + * {@link org.rocksdb.RocksDB}. + */ +public class FlushOptions extends RocksObject { + static { + RocksDB.loadLibrary(); + } + + /** + * Construct a new instance of FlushOptions. + */ + public FlushOptions(){ + super(newFlushOptions()); + } + + /** + * Set if the flush operation shall block until it terminates. + * + * @param waitForFlush boolean value indicating if the flush + * operations waits for termination of the flush process. + * + * @return instance of current FlushOptions. + */ + public FlushOptions setWaitForFlush(final boolean waitForFlush) { + assert(isOwningHandle()); + setWaitForFlush(nativeHandle_, waitForFlush); + return this; + } + + /** + * Wait for flush to finished. + * + * @return boolean value indicating if the flush operation + * waits for termination of the flush process. + */ + public boolean waitForFlush() { + assert(isOwningHandle()); + return waitForFlush(nativeHandle_); + } + + /** + * Set to true so that flush would proceed immediately even if it means + * writes will stall for the duration of the flush. + *

    + * Set to false so that the operation will wait until it's possible to do + * the flush without causing stall or until required flush is performed by + * someone else (foreground call or background thread). + *

    + * Default: false + * + * @param allowWriteStall true to allow writes to stall for flush, false + * otherwise. + * + * @return instance of current FlushOptions. + */ + public FlushOptions setAllowWriteStall(final boolean allowWriteStall) { + assert(isOwningHandle()); + setAllowWriteStall(nativeHandle_, allowWriteStall); + return this; + } + + /** + * Returns true if writes are allowed to stall for flushes to complete, false + * otherwise. + * + * @return true if writes are allowed to stall for flushes + */ + public boolean allowWriteStall() { + assert(isOwningHandle()); + return allowWriteStall(nativeHandle_); + } + + private static native long newFlushOptions(); + @Override protected final native void disposeInternal(final long handle); + + private native void setWaitForFlush(final long handle, + final boolean wait); + private native boolean waitForFlush(final long handle); + private native void setAllowWriteStall(final long handle, + final boolean allowWriteStall); + private native boolean allowWriteStall(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/FlushReason.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/FlushReason.java new file mode 100644 index 0000000..9d486cd --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/FlushReason.java @@ -0,0 +1,53 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +public enum FlushReason { + OTHERS((byte) 0x00), + GET_LIVE_FILES((byte) 0x01), + SHUTDOWN((byte) 0x02), + EXTERNAL_FILE_INGESTION((byte) 0x03), + MANUAL_COMPACTION((byte) 0x04), + WRITE_BUFFER_MANAGER((byte) 0x05), + WRITE_BUFFER_FULL((byte) 0x06), + TEST((byte) 0x07), + DELETE_FILES((byte) 0x08), + AUTO_COMPACTION((byte) 0x09), + MANUAL_FLUSH((byte) 0x0a), + ERROR_RECOVERY((byte) 0xb); + + private final byte value; + + FlushReason(final byte value) { + this.value = value; + } + + /** + * Get the internal representation. + * + * @return the internal representation + */ + byte getValue() { + return value; + } + + /** + * Get the FlushReason from the internal representation value. + * + * @return the flush reason. + * + * @throws IllegalArgumentException if the value is unknown. + */ + static FlushReason fromValue(final byte value) { + for (final FlushReason flushReason : FlushReason.values()) { + if (flushReason.value == value) { + return flushReason; + } + } + + throw new IllegalArgumentException("Illegal value provided for FlushReason: " + value); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/HashLinkedListMemTableConfig.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/HashLinkedListMemTableConfig.java new file mode 100644 index 0000000..4bc860d --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/HashLinkedListMemTableConfig.java @@ -0,0 +1,174 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb; + +/** + * The config for hash linked list memtable representation + * Such memtable contains a fix-sized array of buckets, where + * each bucket points to a sorted singly-linked + * list (or null if the bucket is empty). + *

    + * Note that since this mem-table representation relies on the + * key prefix, it is required to invoke one of the usePrefixExtractor + * functions to specify how to extract key prefix given a key. + * If proper prefix-extractor is not set, then RocksDB will + * use the default memtable representation (SkipList) instead + * and post a warning in the LOG. + */ +public class HashLinkedListMemTableConfig extends MemTableConfig { + public static final long DEFAULT_BUCKET_COUNT = 50000; + public static final long DEFAULT_HUGE_PAGE_TLB_SIZE = 0; + public static final int DEFAULT_BUCKET_ENTRIES_LOG_THRES = 4096; + public static final boolean + DEFAULT_IF_LOG_BUCKET_DIST_WHEN_FLUSH = true; + public static final int DEFAUL_THRESHOLD_USE_SKIPLIST = 256; + + /** + * HashLinkedListMemTableConfig constructor + */ + public HashLinkedListMemTableConfig() { + bucketCount_ = DEFAULT_BUCKET_COUNT; + hugePageTlbSize_ = DEFAULT_HUGE_PAGE_TLB_SIZE; + bucketEntriesLoggingThreshold_ = DEFAULT_BUCKET_ENTRIES_LOG_THRES; + ifLogBucketDistWhenFlush_ = DEFAULT_IF_LOG_BUCKET_DIST_WHEN_FLUSH; + thresholdUseSkiplist_ = DEFAUL_THRESHOLD_USE_SKIPLIST; + } + + /** + * Set the number of buckets in the fixed-size array used + * in the hash linked-list mem-table. + * + * @param count the number of hash buckets. + * @return the reference to the current HashLinkedListMemTableConfig. + */ + public HashLinkedListMemTableConfig setBucketCount( + final long count) { + bucketCount_ = count; + return this; + } + + /** + * Returns the number of buckets that will be used in the memtable + * created based on this config. + * + * @return the number of buckets + */ + public long bucketCount() { + return bucketCount_; + } + + /** + *

    Set the size of huge tlb or allocate the hashtable bytes from + * malloc if {@code size <= 0}.

    + * + *

    The user needs to reserve huge pages for it to be allocated, + * like: {@code sysctl -w vm.nr_hugepages=20}

    + * + *

    See linux documentation/vm/hugetlbpage.txt

    + * + * @param size if set to {@code <= 0} hashtable bytes from malloc + * @return the reference to the current HashLinkedListMemTableConfig. + */ + public HashLinkedListMemTableConfig setHugePageTlbSize( + final long size) { + hugePageTlbSize_ = size; + return this; + } + + /** + * Returns the size value of hugePageTlbSize. + * + * @return the hugePageTlbSize. + */ + public long hugePageTlbSize() { + return hugePageTlbSize_; + } + + /** + * If number of entries in one bucket exceeds that setting, log + * about it. + * + * @param threshold - number of entries in a single bucket before + * logging starts. + * @return the reference to the current HashLinkedListMemTableConfig. + */ + public HashLinkedListMemTableConfig + setBucketEntriesLoggingThreshold(final int threshold) { + bucketEntriesLoggingThreshold_ = threshold; + return this; + } + + /** + * Returns the maximum number of entries in one bucket before + * logging starts. + * + * @return maximum number of entries in one bucket before logging + * starts. + */ + public int bucketEntriesLoggingThreshold() { + return bucketEntriesLoggingThreshold_; + } + + /** + * If true the distrubition of number of entries will be logged. + * + * @param logDistribution - boolean parameter indicating if number + * of entry distribution shall be logged. + * @return the reference to the current HashLinkedListMemTableConfig. + */ + public HashLinkedListMemTableConfig + setIfLogBucketDistWhenFlush(final boolean logDistribution) { + ifLogBucketDistWhenFlush_ = logDistribution; + return this; + } + + /** + * Returns information about logging the distribution of + * number of entries on flush. + * + * @return if distribution of number of entries shall be logged. + */ + public boolean ifLogBucketDistWhenFlush() { + return ifLogBucketDistWhenFlush_; + } + + /** + * Set maximum number of entries in one bucket. Exceeding this val + * leads to a switch from LinkedList to SkipList. + * + * @param threshold maximum number of entries before SkipList is + * used. + * @return the reference to the current HashLinkedListMemTableConfig. + */ + public HashLinkedListMemTableConfig + setThresholdUseSkiplist(final int threshold) { + thresholdUseSkiplist_ = threshold; + return this; + } + + /** + * Returns entries per bucket threshold before LinkedList is + * replaced by SkipList usage for that bucket. + * + * @return entries per bucket threshold before SkipList is used. + */ + public int thresholdUseSkiplist() { + return thresholdUseSkiplist_; + } + + @Override protected long newMemTableFactoryHandle() { + return newMemTableFactoryHandle(bucketCount_, hugePageTlbSize_, + bucketEntriesLoggingThreshold_, ifLogBucketDistWhenFlush_, + thresholdUseSkiplist_); + } + + private native long newMemTableFactoryHandle(long bucketCount, + long hugePageTlbSize, int bucketEntriesLoggingThreshold, + boolean ifLogBucketDistWhenFlush, int thresholdUseSkiplist) + throws IllegalArgumentException; + + private long bucketCount_; + private long hugePageTlbSize_; + private int bucketEntriesLoggingThreshold_; + private boolean ifLogBucketDistWhenFlush_; + private int thresholdUseSkiplist_; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/HashSkipListMemTableConfig.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/HashSkipListMemTableConfig.java new file mode 100644 index 0000000..7cfa1c0 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/HashSkipListMemTableConfig.java @@ -0,0 +1,106 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb; + +/** + * The config for hash skip-list mem-table representation. + * Such mem-table representation contains a fix-sized array of + * buckets, where each bucket points to a skiplist (or null if the + * bucket is empty). + *

    + * Note that since this mem-table representation relies on the + * key prefix, it is required to invoke one of the usePrefixExtractor + * functions to specify how to extract key prefix given a key. + * If proper prefix-extractor is not set, then RocksDB will + * use the default memtable representation (SkipList) instead + * and post a warning in the LOG. + */ +public class HashSkipListMemTableConfig extends MemTableConfig { + public static final int DEFAULT_BUCKET_COUNT = 1000000; + public static final int DEFAULT_BRANCHING_FACTOR = 4; + public static final int DEFAULT_HEIGHT = 4; + + /** + * HashSkipListMemTableConfig constructor + */ + public HashSkipListMemTableConfig() { + bucketCount_ = DEFAULT_BUCKET_COUNT; + branchingFactor_ = DEFAULT_BRANCHING_FACTOR; + height_ = DEFAULT_HEIGHT; + } + + /** + * Set the number of hash buckets used in the hash skiplist memtable. + * Default = 1000000. + * + * @param count the number of hash buckets used in the hash + * skiplist memtable. + * @return the reference to the current HashSkipListMemTableConfig. + */ + public HashSkipListMemTableConfig setBucketCount( + final long count) { + bucketCount_ = count; + return this; + } + + /** + * @return the number of hash buckets + */ + public long bucketCount() { + return bucketCount_; + } + + /** + * Set the height of the skip list. Default = 4. + * + * @param height height to set. + * + * @return the reference to the current HashSkipListMemTableConfig. + */ + public HashSkipListMemTableConfig setHeight(final int height) { + height_ = height; + return this; + } + + /** + * @return the height of the skip list. + */ + public int height() { + return height_; + } + + /** + * Set the branching factor used in the hash skip-list memtable. + * This factor controls the probabilistic size ratio between adjacent + * links in the skip list. + * + * @param bf the probabilistic size ratio between adjacent link + * lists in the skip list. + * @return the reference to the current HashSkipListMemTableConfig. + */ + public HashSkipListMemTableConfig setBranchingFactor( + final int bf) { + branchingFactor_ = bf; + return this; + } + + /** + * @return branching factor, the probabilistic size ratio between + * adjacent links in the skip list. + */ + public int branchingFactor() { + return branchingFactor_; + } + + @Override protected long newMemTableFactoryHandle() { + return newMemTableFactoryHandle( + bucketCount_, height_, branchingFactor_); + } + + private native long newMemTableFactoryHandle( + long bucketCount, int height, int branchingFactor) + throws IllegalArgumentException; + + private long bucketCount_; + private int branchingFactor_; + private int height_; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/HistogramData.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/HistogramData.java new file mode 100644 index 0000000..81d8908 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/HistogramData.java @@ -0,0 +1,75 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +public class HistogramData { + private final double median_; + private final double percentile95_; + private final double percentile99_; + private final double average_; + private final double standardDeviation_; + private final double max_; + private final long count_; + private final long sum_; + private final double min_; + + public HistogramData(final double median, final double percentile95, + final double percentile99, final double average, + final double standardDeviation) { + this(median, percentile95, percentile99, average, standardDeviation, 0.0, 0, 0, 0.0); + } + + public HistogramData(final double median, final double percentile95, + final double percentile99, final double average, + final double standardDeviation, final double max, final long count, + final long sum, final double min) { + median_ = median; + percentile95_ = percentile95; + percentile99_ = percentile99; + average_ = average; + standardDeviation_ = standardDeviation; + min_ = min; + max_ = max; + count_ = count; + sum_ = sum; + } + + public double getMedian() { + return median_; + } + + public double getPercentile95() { + return percentile95_; + } + + public double getPercentile99() { + return percentile99_; + } + + public double getAverage() { + return average_; + } + + public double getStandardDeviation() { + return standardDeviation_; + } + + public double getMax() { + return max_; + } + + public long getCount() { + return count_; + } + + public long getSum() { + return sum_; + } + + public double getMin() { + return min_; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/HistogramType.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/HistogramType.java new file mode 100644 index 0000000..aad0d95 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/HistogramType.java @@ -0,0 +1,214 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +public enum HistogramType { + + DB_GET((byte) 0x0), + + DB_WRITE((byte) 0x1), + + COMPACTION_TIME((byte) 0x2), + + SUBCOMPACTION_SETUP_TIME((byte) 0x3), + + TABLE_SYNC_MICROS((byte) 0x4), + + COMPACTION_OUTFILE_SYNC_MICROS((byte) 0x5), + + WAL_FILE_SYNC_MICROS((byte) 0x6), + + MANIFEST_FILE_SYNC_MICROS((byte) 0x7), + + /** + * TIME SPENT IN IO DURING TABLE OPEN. + */ + TABLE_OPEN_IO_MICROS((byte) 0x8), + + DB_MULTIGET((byte) 0x9), + + READ_BLOCK_COMPACTION_MICROS((byte) 0xA), + + READ_BLOCK_GET_MICROS((byte) 0xB), + + WRITE_RAW_BLOCK_MICROS((byte) 0xC), + + NUM_FILES_IN_SINGLE_COMPACTION((byte) 0x12), + + DB_SEEK((byte) 0x13), + + WRITE_STALL((byte) 0x14), + + SST_READ_MICROS((byte) 0x15), + + /** + * The number of subcompactions actually scheduled during a compaction. + */ + NUM_SUBCOMPACTIONS_SCHEDULED((byte) 0x16), + + /** + * Value size distribution in each operation. + */ + BYTES_PER_READ((byte) 0x17), + BYTES_PER_WRITE((byte) 0x18), + BYTES_PER_MULTIGET((byte) 0x19), + + /** + * number of bytes compressed. + */ + BYTES_COMPRESSED((byte) 0x1A), + + /** + * number of bytes decompressed. + *

    + * number of bytes is when uncompressed; i.e. before/after respectively + */ + BYTES_DECOMPRESSED((byte) 0x1B), + + COMPRESSION_TIMES_NANOS((byte) 0x1C), + + DECOMPRESSION_TIMES_NANOS((byte) 0x1D), + + READ_NUM_MERGE_OPERANDS((byte) 0x1E), + + /** + * Time spent flushing memtable to disk. + */ + FLUSH_TIME((byte) 0x20), + + /** + * Size of keys written to BlobDB. + */ + BLOB_DB_KEY_SIZE((byte) 0x21), + + /** + * Size of values written to BlobDB. + */ + BLOB_DB_VALUE_SIZE((byte) 0x22), + + /** + * BlobDB Put/PutWithTTL/PutUntil/Write latency. + */ + BLOB_DB_WRITE_MICROS((byte) 0x23), + + /** + * BlobDB Get lagency. + */ + BLOB_DB_GET_MICROS((byte) 0x24), + + /** + * BlobDB MultiGet latency. + */ + BLOB_DB_MULTIGET_MICROS((byte) 0x25), + + /** + * BlobDB Seek/SeekToFirst/SeekToLast/SeekForPrev latency. + */ + BLOB_DB_SEEK_MICROS((byte) 0x26), + + /** + * BlobDB Next latency. + */ + BLOB_DB_NEXT_MICROS((byte) 0x27), + + /** + * BlobDB Prev latency. + */ + BLOB_DB_PREV_MICROS((byte) 0x28), + + /** + * Blob file write latency. + */ + BLOB_DB_BLOB_FILE_WRITE_MICROS((byte) 0x29), + + /** + * Blob file read latency. + */ + BLOB_DB_BLOB_FILE_READ_MICROS((byte) 0x2A), + + /** + * Blob file sync latency. + */ + BLOB_DB_BLOB_FILE_SYNC_MICROS((byte) 0x2B), + + /** + * BlobDB compression time. + */ + BLOB_DB_COMPRESSION_MICROS((byte) 0x2D), + + /** + * BlobDB decompression time. + */ + BLOB_DB_DECOMPRESSION_MICROS((byte) 0x2E), + + /** + * Num of Index and Filter blocks read from file system per level in MultiGet + * request + */ + NUM_INDEX_AND_FILTER_BLOCKS_READ_PER_LEVEL((byte) 0x2F), + + /** + * Num of SST files read from file system per level in MultiGet request. + */ + NUM_SST_READ_PER_LEVEL((byte) 0x31), + + /** + * The number of retry in auto resume + */ + ERROR_HANDLER_AUTORESUME_RETRY_COUNT((byte) 0x32), + + ASYNC_READ_BYTES((byte) 0x33), + + /** + * Number of bytes read for RocksDB's prefetching contents + * (as opposed to file system's prefetch) + * from the end of SST table during block based table open + */ + TABLE_OPEN_PREFETCH_TAIL_READ_BYTES((byte) 0x39), + + FILE_READ_FLUSH_MICROS((byte) 0x3A), + + FILE_READ_COMPACTION_MICROS((byte) 0x3B), + + FILE_READ_DB_OPEN_MICROS((byte) 0x3C), + + // 0x1F for backwards compatibility on current minor version. + HISTOGRAM_ENUM_MAX((byte) 0x1F); + + private final byte value; + + HistogramType(final byte value) { + this.value = value; + } + + /** + * Returns the byte value of the enumerations value + * + * @return byte representation + */ + public byte getValue() { + return value; + } + + /** + * Get Histogram type by byte value. + * + * @param value byte representation of HistogramType. + * + * @return {@link org.rocksdb.HistogramType} instance. + * @throws java.lang.IllegalArgumentException if an invalid + * value is provided. + */ + public static HistogramType getHistogramType(final byte value) { + for (final HistogramType histogramType : HistogramType.values()) { + if (histogramType.getValue() == value) { + return histogramType; + } + } + throw new IllegalArgumentException( + "Illegal value provided for HistogramType."); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Holder.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Holder.java new file mode 100644 index 0000000..716a0bd --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Holder.java @@ -0,0 +1,46 @@ +// Copyright (c) 2016, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Simple instance reference wrapper. + */ +public class Holder { + private /* @Nullable */ T value; + + /** + * Constructs a new Holder with null instance. + */ + public Holder() { + } + + /** + * Constructs a new Holder. + * + * @param value the instance or null + */ + public Holder(/* @Nullable */ final T value) { + this.value = value; + } + + /** + * Get the instance reference. + * + * @return value the instance reference or null + */ + public /* @Nullable */ T getValue() { + return value; + } + + /** + * Set the instance reference. + * + * @param value the instance reference or null + */ + public void setValue(/* @Nullable */ final T value) { + this.value = value; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/IndexShorteningMode.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/IndexShorteningMode.java new file mode 100644 index 0000000..a68346c --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/IndexShorteningMode.java @@ -0,0 +1,60 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +/** + * This enum allows trading off increased index size for improved iterator + * seek performance in some situations, particularly when block cache is + * disabled ({@link ReadOptions#fillCache()} == false and direct IO is + * enabled ({@link DBOptions#useDirectReads()} == true). + * The default mode is the best tradeoff for most use cases. + * This option only affects newly written tables. + * + * The index contains a key separating each pair of consecutive blocks. + * Let A be the highest key in one block, B the lowest key in the next block, + * and I the index entry separating these two blocks: + * [ ... A] I [B ...] + * I is allowed to be anywhere in [A, B). + * If an iterator is seeked to a key in (A, I], we'll unnecessarily read the + * first block, then immediately fall through to the second block. + * However, if I=A, this can't happen, and we'll read only the second block. + * In kNoShortening mode, we use I=A. In other modes, we use the shortest + * key in [A, B), which usually significantly reduces index size. + * + * There's a similar story for the last index entry, which is an upper bound + * of the highest key in the file. If it's shortened and therefore + * overestimated, iterator is likely to unnecessarily read the last data block + * from each file on each seek. + */ +public enum IndexShorteningMode { + /** + * Use full keys. + */ + kNoShortening((byte) 0), + /** + * Shorten index keys between blocks, but use full key for the last index + * key, which is the upper bound of the whole file. + */ + kShortenSeparators((byte) 1), + /** + * Shorten both keys between blocks and key after last block. + */ + kShortenSeparatorsAndSuccessor((byte) 2); + + private final byte value; + + IndexShorteningMode(final byte value) { + this.value = value; + } + + /** + * Returns the byte value of the enumerations value. + * + * @return byte representation + */ + byte getValue() { + return value; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/IndexType.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/IndexType.java new file mode 100644 index 0000000..5615e92 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/IndexType.java @@ -0,0 +1,55 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * IndexType used in conjunction with BlockBasedTable. + */ +public enum IndexType { + /** + * A space efficient index block that is optimized for + * binary-search-based index. + */ + kBinarySearch((byte) 0), + /** + * The hash index, if enabled, will do the hash lookup when + * {@code Options.prefix_extractor} is provided. + */ + kHashSearch((byte) 1), + /** + * A two-level index implementation. Both levels are binary search indexes. + */ + kTwoLevelIndexSearch((byte) 2), + /** + * Like {@link #kBinarySearch}, but index also contains first key of each block. + * This allows iterators to defer reading the block until it's actually + * needed. May significantly reduce read amplification of short range scans. + * Without it, iterator seek usually reads one block from each level-0 file + * and from each level, which may be expensive. + * Works best in combination with: + * - IndexShorteningMode::kNoShortening, + * - custom FlushBlockPolicy to cut blocks at some meaningful boundaries, + * e.g. when prefix changes. + * Makes the index significantly bigger (2x or more), especially when keys + * are long. + */ + kBinarySearchWithFirstKey((byte) 3); + + /** + * Returns the byte value of the enumerations value + * + * @return byte representation + */ + public byte getValue() { + return value_; + } + + IndexType(final byte value) { + value_ = value; + } + + private final byte value_; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/InfoLogLevel.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/InfoLogLevel.java new file mode 100644 index 0000000..197bd89 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/InfoLogLevel.java @@ -0,0 +1,49 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb; + +/** + * RocksDB log levels. + */ +public enum InfoLogLevel { + DEBUG_LEVEL((byte)0), + INFO_LEVEL((byte)1), + WARN_LEVEL((byte)2), + ERROR_LEVEL((byte)3), + FATAL_LEVEL((byte)4), + HEADER_LEVEL((byte)5), + NUM_INFO_LOG_LEVELS((byte)6); + + private final byte value_; + + InfoLogLevel(final byte value) { + value_ = value; + } + + /** + * Returns the byte value of the enumerations value + * + * @return byte representation + */ + public byte getValue() { + return value_; + } + + /** + * Get InfoLogLevel by byte value. + * + * @param value byte representation of InfoLogLevel. + * + * @return {@link org.rocksdb.InfoLogLevel} instance. + * @throws java.lang.IllegalArgumentException if an invalid + * value is provided. + */ + public static InfoLogLevel getInfoLogLevel(final byte value) { + for (final InfoLogLevel infoLogLevel : InfoLogLevel.values()) { + if (infoLogLevel.getValue() == value) { + return infoLogLevel; + } + } + throw new IllegalArgumentException( + "Illegal value provided for InfoLogLevel."); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/IngestExternalFileOptions.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/IngestExternalFileOptions.java new file mode 100644 index 0000000..1a6a5fc --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/IngestExternalFileOptions.java @@ -0,0 +1,227 @@ +package org.rocksdb; +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +import java.util.List; + +/** + * IngestExternalFileOptions is used by + * {@link RocksDB#ingestExternalFile(ColumnFamilyHandle, List, IngestExternalFileOptions)}. + */ +public class IngestExternalFileOptions extends RocksObject { + + public IngestExternalFileOptions() { + super(newIngestExternalFileOptions()); + } + + /** + * @param moveFiles {@link #setMoveFiles(boolean)} + * @param snapshotConsistency {@link #setSnapshotConsistency(boolean)} + * @param allowGlobalSeqNo {@link #setAllowGlobalSeqNo(boolean)} + * @param allowBlockingFlush {@link #setAllowBlockingFlush(boolean)} + */ + public IngestExternalFileOptions(final boolean moveFiles, + final boolean snapshotConsistency, final boolean allowGlobalSeqNo, + final boolean allowBlockingFlush) { + super(newIngestExternalFileOptions(moveFiles, snapshotConsistency, + allowGlobalSeqNo, allowBlockingFlush)); + } + + /** + * Can be set to true to move the files instead of copying them. + * + * @return true if files will be moved + */ + public boolean moveFiles() { + return moveFiles(nativeHandle_); + } + + /** + * Can be set to true to move the files instead of copying them. + * + * @param moveFiles true if files should be moved instead of copied + * + * @return the reference to the current IngestExternalFileOptions. + */ + public IngestExternalFileOptions setMoveFiles(final boolean moveFiles) { + setMoveFiles(nativeHandle_, moveFiles); + return this; + } + + /** + * If set to false, an ingested file keys could appear in existing snapshots + * that where created before the file was ingested. + * + * @return true if snapshot consistency is assured + */ + public boolean snapshotConsistency() { + return snapshotConsistency(nativeHandle_); + } + + /** + * If set to false, an ingested file keys could appear in existing snapshots + * that where created before the file was ingested. + * + * @param snapshotConsistency true if snapshot consistency is required + * + * @return the reference to the current IngestExternalFileOptions. + */ + public IngestExternalFileOptions setSnapshotConsistency( + final boolean snapshotConsistency) { + setSnapshotConsistency(nativeHandle_, snapshotConsistency); + return this; + } + + /** + * If set to false, {@link RocksDB#ingestExternalFile(ColumnFamilyHandle, List, IngestExternalFileOptions)} + * will fail if the file key range overlaps with existing keys or tombstones in the DB. + * + * @return true if global seq numbers are assured + */ + public boolean allowGlobalSeqNo() { + return allowGlobalSeqNo(nativeHandle_); + } + + /** + * If set to false, {@link RocksDB#ingestExternalFile(ColumnFamilyHandle, List, IngestExternalFileOptions)} + * will fail if the file key range overlaps with existing keys or tombstones in the DB. + * + * @param allowGlobalSeqNo true if global seq numbers are required + * + * @return the reference to the current IngestExternalFileOptions. + */ + public IngestExternalFileOptions setAllowGlobalSeqNo( + final boolean allowGlobalSeqNo) { + setAllowGlobalSeqNo(nativeHandle_, allowGlobalSeqNo); + return this; + } + + /** + * If set to false and the file key range overlaps with the memtable key range + * (memtable flush required), IngestExternalFile will fail. + * + * @return true if blocking flushes may occur + */ + public boolean allowBlockingFlush() { + return allowBlockingFlush(nativeHandle_); + } + + /** + * If set to false and the file key range overlaps with the memtable key range + * (memtable flush required), IngestExternalFile will fail. + * + * @param allowBlockingFlush true if blocking flushes are allowed + * + * @return the reference to the current IngestExternalFileOptions. + */ + public IngestExternalFileOptions setAllowBlockingFlush( + final boolean allowBlockingFlush) { + setAllowBlockingFlush(nativeHandle_, allowBlockingFlush); + return this; + } + + /** + * Returns true if duplicate keys in the file being ingested are + * to be skipped rather than overwriting existing data under that key. + * + * @return true if duplicate keys in the file being ingested are to be + * skipped, false otherwise. + */ + public boolean ingestBehind() { + return ingestBehind(nativeHandle_); + } + + /** + * Set to true if you would like duplicate keys in the file being ingested + * to be skipped rather than overwriting existing data under that key. + *

    + * Usecase: back-fill of some historical data in the database without + * over-writing existing newer version of data. + *

    + * This option could only be used if the DB has been running + * with DBOptions#allowIngestBehind() == true since the dawn of time. + *

    + * All files will be ingested at the bottommost level with seqno=0. + *

    + * Default: false + * + * @param ingestBehind true if you would like duplicate keys in the file being + * ingested to be skipped. + * + * @return the reference to the current IngestExternalFileOptions. + */ + public IngestExternalFileOptions setIngestBehind(final boolean ingestBehind) { + setIngestBehind(nativeHandle_, ingestBehind); + return this; + } + + /** + * Returns true write if the global_seqno is written to a given offset + * in the external SST file for backward compatibility. + *

    + * See {@link #setWriteGlobalSeqno(boolean)}. + * + * @return true if the global_seqno is written to a given offset, + * false otherwise. + */ + public boolean writeGlobalSeqno() { + return writeGlobalSeqno(nativeHandle_); + } + + /** + * Set to true if you would like to write the global_seqno to a given offset + * in the external SST file for backward compatibility. + *

    + * Older versions of RocksDB write the global_seqno to a given offset within + * the ingested SST files, and new versions of RocksDB do not. + *

    + * If you ingest an external SST using new version of RocksDB and would like + * to be able to downgrade to an older version of RocksDB, you should set + * {@link #writeGlobalSeqno()} to true. + *

    + * If your service is just starting to use the new RocksDB, we recommend that + * you set this option to false, which brings two benefits: + * 1. No extra random write for global_seqno during ingestion. + * 2. Without writing external SST file, it's possible to do checksum. + *

    + * We have a plan to set this option to false by default in the future. + *

    + * Default: true + * + * @param writeGlobalSeqno true to write the gloal_seqno to a given offset, + * false otherwise + * + * @return the reference to the current IngestExternalFileOptions. + */ + public IngestExternalFileOptions setWriteGlobalSeqno( + final boolean writeGlobalSeqno) { + setWriteGlobalSeqno(nativeHandle_, writeGlobalSeqno); + return this; + } + + private static native long newIngestExternalFileOptions(); + private static native long newIngestExternalFileOptions(final boolean moveFiles, + final boolean snapshotConsistency, final boolean allowGlobalSeqNo, + final boolean allowBlockingFlush); + @Override protected final native void disposeInternal(final long handle); + + private native boolean moveFiles(final long handle); + private native void setMoveFiles(final long handle, final boolean move_files); + private native boolean snapshotConsistency(final long handle); + private native void setSnapshotConsistency(final long handle, + final boolean snapshotConsistency); + private native boolean allowGlobalSeqNo(final long handle); + private native void setAllowGlobalSeqNo(final long handle, + final boolean allowGloablSeqNo); + private native boolean allowBlockingFlush(final long handle); + private native void setAllowBlockingFlush(final long handle, + final boolean allowBlockingFlush); + private native boolean ingestBehind(final long handle); + private native void setIngestBehind(final long handle, + final boolean ingestBehind); + private native boolean writeGlobalSeqno(final long handle); + private native void setWriteGlobalSeqno(final long handle, + final boolean writeGlobalSeqNo); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/KeyMayExist.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/KeyMayExist.java new file mode 100644 index 0000000..6149b85 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/KeyMayExist.java @@ -0,0 +1,35 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Objects; + +public class KeyMayExist { + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final KeyMayExist that = (KeyMayExist) o; + return (valueLength == that.valueLength && exists == that.exists); + } + + @Override + public int hashCode() { + return Objects.hash(exists, valueLength); + } + + public enum KeyMayExistEnum { kNotExist, kExistsWithoutValue, kExistsWithValue } + + public KeyMayExist(final KeyMayExistEnum exists, final int valueLength) { + this.exists = exists; + this.valueLength = valueLength; + } + + public final KeyMayExistEnum exists; + public final int valueLength; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/LRUCache.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/LRUCache.java new file mode 100644 index 0000000..0a9d02e --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/LRUCache.java @@ -0,0 +1,106 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Least Recently Used Cache + */ +public class LRUCache extends Cache { + + /** + * Create a new cache with a fixed size capacity + * + * @param capacity The fixed size capacity of the cache + */ + public LRUCache(final long capacity) { + this(capacity, -1, false, 0.0, 0.0); + } + + /** + * Create a new cache with a fixed size capacity. The cache is sharded + * to 2^numShardBits shards, by hash of the key. The total capacity + * is divided and evenly assigned to each shard. + * numShardBits = -1 means it is automatically determined: every shard + * will be at least 512KB and number of shard bits will not exceed 6. + * + * @param capacity The fixed size capacity of the cache + * @param numShardBits The cache is sharded to 2^numShardBits shards, + * by hash of the key + */ + public LRUCache(final long capacity, final int numShardBits) { + super(newLRUCache(capacity, numShardBits, false, 0.0, 0.0)); + } + + /** + * Create a new cache with a fixed size capacity. The cache is sharded + * to 2^numShardBits shards, by hash of the key. The total capacity + * is divided and evenly assigned to each shard. If strictCapacityLimit + * is set, insert to the cache will fail when cache is full. + * numShardBits = -1 means it is automatically determined: every shard + * will be at least 512KB and number of shard bits will not exceed 6. + * + * @param capacity The fixed size capacity of the cache + * @param numShardBits The cache is sharded to 2^numShardBits shards, + * by hash of the key + * @param strictCapacityLimit insert to the cache will fail when cache is full + */ + public LRUCache(final long capacity, final int numShardBits, + final boolean strictCapacityLimit) { + super(newLRUCache(capacity, numShardBits, strictCapacityLimit, 0.0, 0.0)); + } + + /** + * Create a new cache with a fixed size capacity. The cache is sharded + * to 2^numShardBits shards, by hash of the key. The total capacity + * is divided and evenly assigned to each shard. If strictCapacityLimit + * is set, insert to the cache will fail when cache is full. User can also + * set percentage of the cache reserves for high priority entries via + * highPriPoolRatio. + * numShardBits = -1 means it is automatically determined: every shard + * will be at least 512KB and number of shard bits will not exceed 6. + * + * @param capacity The fixed size capacity of the cache + * @param numShardBits The cache is sharded to 2^numShardBits shards, + * by hash of the key + * @param strictCapacityLimit insert to the cache will fail when cache is full + * @param highPriPoolRatio percentage of the cache reserves for high priority + * entries + */ + public LRUCache(final long capacity, final int numShardBits, final boolean strictCapacityLimit, + final double highPriPoolRatio) { + super(newLRUCache(capacity, numShardBits, strictCapacityLimit, highPriPoolRatio, 0.0)); + } + + /** + * Create a new cache with a fixed size capacity. The cache is sharded + * to 2^numShardBits shards, by hash of the key. The total capacity + * is divided and evenly assigned to each shard. If strictCapacityLimit + * is set, insert to the cache will fail when cache is full. User can also + * set percentage of the cache reserves for high priority entries and low + * priority entries via highPriPoolRatio and lowPriPoolRatio. + * numShardBits = -1 means it is automatically determined: every shard + * will be at least 512KB and number of shard bits will not exceed 6. + * + * @param capacity The fixed size capacity of the cache + * @param numShardBits The cache is sharded to 2^numShardBits shards, + * by hash of the key + * @param strictCapacityLimit insert to the cache will fail when cache is full + * @param highPriPoolRatio percentage of the cache reserves for high priority + * entries + * @param lowPriPoolRatio percentage of the cache reserves for low priority + * entries + */ + public LRUCache(final long capacity, final int numShardBits, final boolean strictCapacityLimit, + final double highPriPoolRatio, final double lowPriPoolRatio) { + super(newLRUCache( + capacity, numShardBits, strictCapacityLimit, highPriPoolRatio, lowPriPoolRatio)); + } + + private static native long newLRUCache(final long capacity, final int numShardBits, + final boolean strictCapacityLimit, final double highPriPoolRatio, + final double lowPriPoolRatio); + @Override protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/LevelMetaData.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/LevelMetaData.java new file mode 100644 index 0000000..c568509 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/LevelMetaData.java @@ -0,0 +1,56 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Arrays; +import java.util.List; + +/** + * The metadata that describes a level. + */ +public class LevelMetaData { + private final int level; + private final long size; + private final SstFileMetaData[] files; + + /** + * Called from JNI C++ + */ + private LevelMetaData(final int level, final long size, + final SstFileMetaData[] files) { + this.level = level; + this.size = size; + this.files = files; + } + + /** + * The level which this meta data describes. + * + * @return the level + */ + public int level() { + return level; + } + + /** + * The size of this level in bytes, which is equal to the sum of + * the file size of its {@link #files()}. + * + * @return the size + */ + public long size() { + return size; + } + + /** + * The metadata of all sst files in this level. + * + * @return the metadata of the files + */ + public List files() { + return Arrays.asList(files); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/LiveFileMetaData.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/LiveFileMetaData.java new file mode 100644 index 0000000..35d883e --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/LiveFileMetaData.java @@ -0,0 +1,55 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * The full set of metadata associated with each SST file. + */ +public class LiveFileMetaData extends SstFileMetaData { + private final byte[] columnFamilyName; + private final int level; + + /** + * Called from JNI C++ + */ + private LiveFileMetaData( + final byte[] columnFamilyName, + final int level, + final String fileName, + final String path, + final long size, + final long smallestSeqno, + final long largestSeqno, + final byte[] smallestKey, + final byte[] largestKey, + final long numReadsSampled, + final boolean beingCompacted, + final long numEntries, + final long numDeletions) { + super(fileName, path, size, smallestSeqno, largestSeqno, smallestKey, + largestKey, numReadsSampled, beingCompacted, numEntries, numDeletions); + this.columnFamilyName = columnFamilyName; + this.level = level; + } + + /** + * Get the name of the column family. + * + * @return the name of the column family + */ + public byte[] columnFamilyName() { + return columnFamilyName; + } + + /** + * Get the level at which this file resides. + * + * @return the level at which the file resides. + */ + public int level() { + return level; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/LogFile.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/LogFile.java new file mode 100644 index 0000000..ef24a64 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/LogFile.java @@ -0,0 +1,75 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +public class LogFile { + private final String pathName; + private final long logNumber; + private final WalFileType type; + private final long startSequence; + private final long sizeFileBytes; + + /** + * Called from JNI C++ + */ + private LogFile(final String pathName, final long logNumber, + final byte walFileTypeValue, final long startSequence, + final long sizeFileBytes) { + this.pathName = pathName; + this.logNumber = logNumber; + this.type = WalFileType.fromValue(walFileTypeValue); + this.startSequence = startSequence; + this.sizeFileBytes = sizeFileBytes; + } + + /** + * Returns log file's pathname relative to the main db dir + * Eg. For a live-log-file = /000003.log + * For an archived-log-file = /archive/000003.log + * + * @return log file's pathname + */ + public String pathName() { + return pathName; + } + + /** + * Primary identifier for log file. + * This is directly proportional to creation time of the log file + * + * @return the log number + */ + public long logNumber() { + return logNumber; + } + + /** + * Log file can be either alive or archived. + * + * @return the type of the log file. + */ + public WalFileType type() { + return type; + } + + /** + * Starting sequence number of writebatch written in this log file. + * + * @return the stating sequence number + */ + public long startSequence() { + return startSequence; + } + + /** + * Size of log file on disk in Bytes. + * + * @return size of log file + */ + public long sizeFileBytes() { + return sizeFileBytes; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Logger.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Logger.java new file mode 100644 index 0000000..614a7fa --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Logger.java @@ -0,0 +1,121 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + *

    This class provides a custom logger functionality + * in Java which wraps {@code RocksDB} logging facilities. + *

    + * + *

    Using this class RocksDB can log with common + * Java logging APIs like Log4j or Slf4j without keeping + * database logs in the filesystem.

    + * + * Performance + *

    There are certain performance penalties using a Java + * {@code Logger} implementation within production code. + *

    + * + *

    + * A log level can be set using {@link org.rocksdb.Options} or + * {@link Logger#setInfoLogLevel(InfoLogLevel)}. The set log level + * influences the underlying native code. Each log message is + * checked against the set log level and if the log level is more + * verbose as the set log level, native allocations will be made + * and data structures are allocated. + *

    + * + *

    Every log message which will be emitted by native code will + * trigger expensive native to Java transitions. So the preferred + * setting for production use is either + * {@link org.rocksdb.InfoLogLevel#ERROR_LEVEL} or + * {@link org.rocksdb.InfoLogLevel#FATAL_LEVEL}. + *

    + */ +public abstract class Logger extends RocksCallbackObject { + private static final long WITH_OPTIONS = 0; + private static final long WITH_DBOPTIONS = 1; + + /** + *

    AbstractLogger constructor.

    + * + *

    Important: the log level set within + * the {@link org.rocksdb.Options} instance will be used as + * maximum log level of RocksDB.

    + * + * @param options {@link org.rocksdb.Options} instance. + */ + public Logger(final Options options) { + super(options.nativeHandle_, WITH_OPTIONS); + + } + + /** + *

    AbstractLogger constructor.

    + * + *

    Important: the log level set within + * the {@link org.rocksdb.DBOptions} instance will be used + * as maximum log level of RocksDB.

    + * + * @param dboptions {@link org.rocksdb.DBOptions} instance. + */ + public Logger(final DBOptions dboptions) { + super(dboptions.nativeHandle_, WITH_DBOPTIONS); + } + + @Override + protected long initializeNative(final long... nativeParameterHandles) { + if(nativeParameterHandles[1] == WITH_OPTIONS) { + return createNewLoggerOptions(nativeParameterHandles[0]); + } else if(nativeParameterHandles[1] == WITH_DBOPTIONS) { + return createNewLoggerDbOptions(nativeParameterHandles[0]); + } else { + throw new IllegalArgumentException(); + } + } + + /** + * Set {@link org.rocksdb.InfoLogLevel} to AbstractLogger. + * + * @param infoLogLevel {@link org.rocksdb.InfoLogLevel} instance. + */ + public void setInfoLogLevel(final InfoLogLevel infoLogLevel) { + setInfoLogLevel(nativeHandle_, infoLogLevel.getValue()); + } + + /** + * Return the loggers log level. + * + * @return {@link org.rocksdb.InfoLogLevel} instance. + */ + public InfoLogLevel infoLogLevel() { + return InfoLogLevel.getInfoLogLevel( + infoLogLevel(nativeHandle_)); + } + + protected abstract void log(InfoLogLevel infoLogLevel, + String logMsg); + + protected native long createNewLoggerOptions( + long options); + protected native long createNewLoggerDbOptions( + long dbOptions); + protected native void setInfoLogLevel(long handle, + byte infoLogLevel); + protected native byte infoLogLevel(long handle); + + /** + * We override {@link RocksCallbackObject#disposeInternal()} + * as disposing of a rocksdb::LoggerJniCallback requires + * a slightly different approach as it is a std::shared_ptr + */ + @Override + protected void disposeInternal() { + disposeInternal(nativeHandle_); + } + + private native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MemTableConfig.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MemTableConfig.java new file mode 100644 index 0000000..17033d2 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MemTableConfig.java @@ -0,0 +1,29 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +/** + * MemTableConfig is used to config the internal mem-table of a RocksDB. + * It is required for each memtable to have one such sub-class to allow + * Java developers to use it. + *

    + * To make a RocksDB to use a specific MemTable format, its associated + * MemTableConfig should be properly set and passed into Options + * via Options.setMemTableFactory() and open the db using that Options. + * + * @see Options + */ +public abstract class MemTableConfig { + /** + * This function should only be called by Options.setMemTableConfig(), + * which will create a c++ shared-pointer to the c++ MemTableRepFactory + * that associated with the Java MemTableConfig. + * + * @see Options#setMemTableConfig(MemTableConfig) + * + * @return native handle address to native memory table instance. + */ + protected abstract long newMemTableFactoryHandle(); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MemTableInfo.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MemTableInfo.java new file mode 100644 index 0000000..3d42903 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MemTableInfo.java @@ -0,0 +1,103 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Objects; + +public class MemTableInfo { + private final String columnFamilyName; + private final long firstSeqno; + private final long earliestSeqno; + private final long numEntries; + private final long numDeletes; + + /** + * Access is package private as this will only be constructed from + * C++ via JNI and for testing. + */ + MemTableInfo(final String columnFamilyName, final long firstSeqno, final long earliestSeqno, + final long numEntries, final long numDeletes) { + this.columnFamilyName = columnFamilyName; + this.firstSeqno = firstSeqno; + this.earliestSeqno = earliestSeqno; + this.numEntries = numEntries; + this.numDeletes = numDeletes; + } + + /** + * Get the name of the column family to which memtable belongs. + * + * @return the name of the column family. + */ + public String getColumnFamilyName() { + return columnFamilyName; + } + + /** + * Get the Sequence number of the first element that was inserted into the + * memtable. + * + * @return the sequence number of the first inserted element. + */ + public long getFirstSeqno() { + return firstSeqno; + } + + /** + * Get the Sequence number that is guaranteed to be smaller than or equal + * to the sequence number of any key that could be inserted into this + * memtable. It can then be assumed that any write with a larger(or equal) + * sequence number will be present in this memtable or a later memtable. + * + * @return the earliest sequence number. + */ + public long getEarliestSeqno() { + return earliestSeqno; + } + + /** + * Get the total number of entries in memtable. + * + * @return the total number of entries. + */ + public long getNumEntries() { + return numEntries; + } + + /** + * Get the total number of deletes in memtable. + * + * @return the total number of deletes. + */ + public long getNumDeletes() { + return numDeletes; + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final MemTableInfo that = (MemTableInfo) o; + return firstSeqno == that.firstSeqno && earliestSeqno == that.earliestSeqno + && numEntries == that.numEntries && numDeletes == that.numDeletes + && Objects.equals(columnFamilyName, that.columnFamilyName); + } + + @Override + public int hashCode() { + return Objects.hash(columnFamilyName, firstSeqno, earliestSeqno, numEntries, numDeletes); + } + + @Override + public String toString() { + return "MemTableInfo{" + + "columnFamilyName='" + columnFamilyName + '\'' + ", firstSeqno=" + firstSeqno + + ", earliestSeqno=" + earliestSeqno + ", numEntries=" + numEntries + + ", numDeletes=" + numDeletes + '}'; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MemoryUsageType.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MemoryUsageType.java new file mode 100644 index 0000000..40e6d17 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MemoryUsageType.java @@ -0,0 +1,72 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * MemoryUsageType + * + *

    The value will be used as a key to indicate the type of memory usage + * described

    + */ +public enum MemoryUsageType { + /** + * Memory usage of all the mem-tables. + */ + kMemTableTotal((byte) 0), + /** + * Memory usage of those un-flushed mem-tables. + */ + kMemTableUnFlushed((byte) 1), + /** + * Memory usage of all the table readers. + */ + kTableReadersTotal((byte) 2), + /** + * Memory usage by Cache. + */ + kCacheTotal((byte) 3), + /** + * Max usage types - copied to keep 1:1 with native. + */ + kNumUsageTypes((byte) 4); + + /** + * Returns the byte value of the enumerations value + * + * @return byte representation + */ + public byte getValue() { + return value_; + } + + /** + *

    Get the MemoryUsageType enumeration value by + * passing the byte identifier to this method.

    + * + * @param byteIdentifier of MemoryUsageType. + * + * @return MemoryUsageType instance. + * + * @throws IllegalArgumentException if the usage type for the byteIdentifier + * cannot be found + */ + public static MemoryUsageType getMemoryUsageType(final byte byteIdentifier) { + for (final MemoryUsageType memoryUsageType : MemoryUsageType.values()) { + if (memoryUsageType.getValue() == byteIdentifier) { + return memoryUsageType; + } + } + + throw new IllegalArgumentException( + "Illegal value provided for MemoryUsageType."); + } + + MemoryUsageType(final byte value) { + value_ = value; + } + + private final byte value_; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MemoryUtil.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MemoryUtil.java new file mode 100644 index 0000000..15b9f00 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MemoryUtil.java @@ -0,0 +1,60 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.*; + +/** + * JNI passthrough for MemoryUtil. + */ +public class MemoryUtil { + + /** + *

    Returns the approximate memory usage of different types in the input + * list of DBs and Cache set. For instance, in the output map the key + * kMemTableTotal will be associated with the memory + * usage of all the mem-tables from all the input rocksdb instances.

    + * + *

    Note that for memory usage inside Cache class, we will + * only report the usage of the input "cache_set" without + * including those Cache usage inside the input list "dbs" + * of DBs.

    + * + * @param dbs List of dbs to collect memory usage for. + * @param caches Set of caches to collect memory usage for. + * @return Map from {@link MemoryUsageType} to memory usage as a {@link Long}. + */ + public static Map getApproximateMemoryUsageByType(final List dbs, final Set caches) { + final int dbCount = (dbs == null) ? 0 : dbs.size(); + final int cacheCount = (caches == null) ? 0 : caches.size(); + final long[] dbHandles = new long[dbCount]; + final long[] cacheHandles = new long[cacheCount]; + if (dbCount > 0) { + final ListIterator dbIter = dbs.listIterator(); + while (dbIter.hasNext()) { + dbHandles[dbIter.nextIndex()] = dbIter.next().nativeHandle_; + } + } + if (cacheCount > 0) { + // NOTE: This index handling is super ugly but I couldn't get a clean way to track both the + // index and the iterator simultaneously within a Set. + int i = 0; + for (final Cache cache : caches) { + cacheHandles[i] = cache.nativeHandle_; + i++; + } + } + final Map byteOutput = getApproximateMemoryUsageByType(dbHandles, cacheHandles); + final Map output = new HashMap<>(); + for (final Map.Entry longEntry : byteOutput.entrySet()) { + output.put(MemoryUsageType.getMemoryUsageType(longEntry.getKey()), longEntry.getValue()); + } + return output; + } + + private static native Map getApproximateMemoryUsageByType( + final long[] dbHandles, final long[] cacheHandles); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MergeOperator.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MergeOperator.java new file mode 100644 index 0000000..c299f62 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MergeOperator.java @@ -0,0 +1,18 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// Copyright (c) 2014, Vlad Balan (vlad.gm@gmail.com). All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * MergeOperator holds an operator to be applied when compacting + * two merge operands held under the same key in order to obtain a single + * value. + */ +public abstract class MergeOperator extends RocksObject { + protected MergeOperator(final long nativeHandle) { + super(nativeHandle); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java new file mode 100644 index 0000000..e54db71 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java @@ -0,0 +1,621 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.*; + +public class MutableColumnFamilyOptions extends AbstractMutableOptions { + /** + * User must use builder pattern, or parser. + * + * @param keys the keys + * @param values the values + *

    + * See {@link #builder()} and {@link #parse(String)}. + */ + private MutableColumnFamilyOptions(final String[] keys, + final String[] values) { + super(keys, values); + } + + /** + * Creates a builder which allows you + * to set MutableColumnFamilyOptions in a fluent + * manner + * + * @return A builder for MutableColumnFamilyOptions + */ + public static MutableColumnFamilyOptionsBuilder builder() { + return new MutableColumnFamilyOptionsBuilder(); + } + + /** + * Parses a String representation of MutableColumnFamilyOptions + *

    + * The format is: key1=value1;key2=value2;key3=value3 etc + *

    + * For int[] values, each int should be separated by a colon, e.g. + *

    + * key1=value1;intArrayKey1=1:2:3 + * + * @param str The string representation of the mutable column family options + * @param ignoreUnknown what to do if the key is not one of the keys we expect + * + * @return A builder for the mutable column family options + */ + public static MutableColumnFamilyOptionsBuilder parse( + final String str, final boolean ignoreUnknown) { + Objects.requireNonNull(str); + + final List parsedOptions = OptionString.Parser.parse(str); + return new MutableColumnFamilyOptionsBuilder().fromParsed(parsedOptions, ignoreUnknown); + } + + public static MutableColumnFamilyOptionsBuilder parse(final String str) { + return parse(str, false); + } + + private interface MutableColumnFamilyOptionKey extends MutableOptionKey {} + + public enum MemtableOption implements MutableColumnFamilyOptionKey { + write_buffer_size(ValueType.LONG), + arena_block_size(ValueType.LONG), + memtable_prefix_bloom_size_ratio(ValueType.DOUBLE), + memtable_whole_key_filtering(ValueType.BOOLEAN), + @Deprecated memtable_prefix_bloom_bits(ValueType.INT), + @Deprecated memtable_prefix_bloom_probes(ValueType.INT), + memtable_huge_page_size(ValueType.LONG), + max_successive_merges(ValueType.LONG), + @Deprecated filter_deletes(ValueType.BOOLEAN), + max_write_buffer_number(ValueType.INT), + inplace_update_num_locks(ValueType.LONG), + experimental_mempurge_threshold(ValueType.DOUBLE); + + private final ValueType valueType; + MemtableOption(final ValueType valueType) { + this.valueType = valueType; + } + + @Override + public ValueType getValueType() { + return valueType; + } + } + + public enum CompactionOption implements MutableColumnFamilyOptionKey { + disable_auto_compactions(ValueType.BOOLEAN), + soft_pending_compaction_bytes_limit(ValueType.LONG), + hard_pending_compaction_bytes_limit(ValueType.LONG), + level0_file_num_compaction_trigger(ValueType.INT), + level0_slowdown_writes_trigger(ValueType.INT), + level0_stop_writes_trigger(ValueType.INT), + max_compaction_bytes(ValueType.LONG), + target_file_size_base(ValueType.LONG), + target_file_size_multiplier(ValueType.INT), + max_bytes_for_level_base(ValueType.LONG), + max_bytes_for_level_multiplier(ValueType.INT), + max_bytes_for_level_multiplier_additional(ValueType.INT_ARRAY), + ttl(ValueType.LONG), + periodic_compaction_seconds(ValueType.LONG); + + private final ValueType valueType; + CompactionOption(final ValueType valueType) { + this.valueType = valueType; + } + + @Override + public ValueType getValueType() { + return valueType; + } + } + + public enum BlobOption implements MutableColumnFamilyOptionKey { + enable_blob_files(ValueType.BOOLEAN), + min_blob_size(ValueType.LONG), + blob_file_size(ValueType.LONG), + blob_compression_type(ValueType.ENUM), + enable_blob_garbage_collection(ValueType.BOOLEAN), + blob_garbage_collection_age_cutoff(ValueType.DOUBLE), + blob_garbage_collection_force_threshold(ValueType.DOUBLE), + blob_compaction_readahead_size(ValueType.LONG), + blob_file_starting_level(ValueType.INT), + prepopulate_blob_cache(ValueType.ENUM); + + private final ValueType valueType; + BlobOption(final ValueType valueType) { + this.valueType = valueType; + } + + @Override + public ValueType getValueType() { + return valueType; + } + } + + public enum MiscOption implements MutableColumnFamilyOptionKey { + max_sequential_skip_in_iterations(ValueType.LONG), + paranoid_file_checks(ValueType.BOOLEAN), + report_bg_io_stats(ValueType.BOOLEAN), + compression(ValueType.ENUM); + + private final ValueType valueType; + MiscOption(final ValueType valueType) { + this.valueType = valueType; + } + + @Override + public ValueType getValueType() { + return valueType; + } + } + + public static class MutableColumnFamilyOptionsBuilder + extends AbstractMutableOptionsBuilder + implements MutableColumnFamilyOptionsInterface { + private static final Map ALL_KEYS_LOOKUP = + new HashMap<>(); + static { + for(final MutableColumnFamilyOptionKey key : MemtableOption.values()) { + ALL_KEYS_LOOKUP.put(key.name(), key); + } + + for(final MutableColumnFamilyOptionKey key : CompactionOption.values()) { + ALL_KEYS_LOOKUP.put(key.name(), key); + } + + for(final MutableColumnFamilyOptionKey key : MiscOption.values()) { + ALL_KEYS_LOOKUP.put(key.name(), key); + } + + for (final MutableColumnFamilyOptionKey key : BlobOption.values()) { + ALL_KEYS_LOOKUP.put(key.name(), key); + } + } + + private MutableColumnFamilyOptionsBuilder() { + super(); + } + + @Override + protected MutableColumnFamilyOptionsBuilder self() { + return this; + } + + @Override + protected Map allKeys() { + return ALL_KEYS_LOOKUP; + } + + @Override + protected MutableColumnFamilyOptions build(final String[] keys, + final String[] values) { + return new MutableColumnFamilyOptions(keys, values); + } + + @Override + public MutableColumnFamilyOptionsBuilder setWriteBufferSize( + final long writeBufferSize) { + return setLong(MemtableOption.write_buffer_size, writeBufferSize); + } + + @Override + public long writeBufferSize() { + return getLong(MemtableOption.write_buffer_size); + } + + @Override + public MutableColumnFamilyOptionsBuilder setArenaBlockSize( + final long arenaBlockSize) { + return setLong(MemtableOption.arena_block_size, arenaBlockSize); + } + + @Override + public long arenaBlockSize() { + return getLong(MemtableOption.arena_block_size); + } + + @Override + public MutableColumnFamilyOptionsBuilder setMemtablePrefixBloomSizeRatio( + final double memtablePrefixBloomSizeRatio) { + return setDouble(MemtableOption.memtable_prefix_bloom_size_ratio, + memtablePrefixBloomSizeRatio); + } + + @Override + public double memtablePrefixBloomSizeRatio() { + return getDouble(MemtableOption.memtable_prefix_bloom_size_ratio); + } + + @Override + public MutableColumnFamilyOptionsBuilder setMemtableWholeKeyFiltering( + final boolean memtableWholeKeyFiltering) { + return setBoolean(MemtableOption.memtable_whole_key_filtering, memtableWholeKeyFiltering); + } + + @Override + public boolean memtableWholeKeyFiltering() { + return getBoolean(MemtableOption.memtable_whole_key_filtering); + } + + @Override + public MutableColumnFamilyOptionsBuilder setMemtableHugePageSize( + final long memtableHugePageSize) { + return setLong(MemtableOption.memtable_huge_page_size, + memtableHugePageSize); + } + + @Override + public long memtableHugePageSize() { + return getLong(MemtableOption.memtable_huge_page_size); + } + + @Override + public MutableColumnFamilyOptionsBuilder setMaxSuccessiveMerges( + final long maxSuccessiveMerges) { + return setLong(MemtableOption.max_successive_merges, maxSuccessiveMerges); + } + + @Override + public long maxSuccessiveMerges() { + return getLong(MemtableOption.max_successive_merges); + } + + @Override + public MutableColumnFamilyOptionsBuilder setMaxWriteBufferNumber( + final int maxWriteBufferNumber) { + return setInt(MemtableOption.max_write_buffer_number, + maxWriteBufferNumber); + } + + @Override + public int maxWriteBufferNumber() { + return getInt(MemtableOption.max_write_buffer_number); + } + + @Override + public MutableColumnFamilyOptionsBuilder setInplaceUpdateNumLocks( + final long inplaceUpdateNumLocks) { + return setLong(MemtableOption.inplace_update_num_locks, + inplaceUpdateNumLocks); + } + + @Override + public long inplaceUpdateNumLocks() { + return getLong(MemtableOption.inplace_update_num_locks); + } + + @Override + public MutableColumnFamilyOptionsBuilder setExperimentalMempurgeThreshold( + final double experimentalMempurgeThreshold) { + return setDouble( + MemtableOption.experimental_mempurge_threshold, experimentalMempurgeThreshold); + } + + @Override + public double experimentalMempurgeThreshold() { + return getDouble(MemtableOption.experimental_mempurge_threshold); + } + + @Override + public MutableColumnFamilyOptionsBuilder setDisableAutoCompactions( + final boolean disableAutoCompactions) { + return setBoolean(CompactionOption.disable_auto_compactions, + disableAutoCompactions); + } + + @Override + public boolean disableAutoCompactions() { + return getBoolean(CompactionOption.disable_auto_compactions); + } + + @Override + public MutableColumnFamilyOptionsBuilder setSoftPendingCompactionBytesLimit( + final long softPendingCompactionBytesLimit) { + return setLong(CompactionOption.soft_pending_compaction_bytes_limit, + softPendingCompactionBytesLimit); + } + + @Override + public long softPendingCompactionBytesLimit() { + return getLong(CompactionOption.soft_pending_compaction_bytes_limit); + } + + @Override + public MutableColumnFamilyOptionsBuilder setHardPendingCompactionBytesLimit( + final long hardPendingCompactionBytesLimit) { + return setLong(CompactionOption.hard_pending_compaction_bytes_limit, + hardPendingCompactionBytesLimit); + } + + @Override + public long hardPendingCompactionBytesLimit() { + return getLong(CompactionOption.hard_pending_compaction_bytes_limit); + } + + @Override + public MutableColumnFamilyOptionsBuilder setLevel0FileNumCompactionTrigger( + final int level0FileNumCompactionTrigger) { + return setInt(CompactionOption.level0_file_num_compaction_trigger, + level0FileNumCompactionTrigger); + } + + @Override + public int level0FileNumCompactionTrigger() { + return getInt(CompactionOption.level0_file_num_compaction_trigger); + } + + @Override + public MutableColumnFamilyOptionsBuilder setLevel0SlowdownWritesTrigger( + final int level0SlowdownWritesTrigger) { + return setInt(CompactionOption.level0_slowdown_writes_trigger, + level0SlowdownWritesTrigger); + } + + @Override + public int level0SlowdownWritesTrigger() { + return getInt(CompactionOption.level0_slowdown_writes_trigger); + } + + @Override + public MutableColumnFamilyOptionsBuilder setLevel0StopWritesTrigger( + final int level0StopWritesTrigger) { + return setInt(CompactionOption.level0_stop_writes_trigger, + level0StopWritesTrigger); + } + + @Override + public int level0StopWritesTrigger() { + return getInt(CompactionOption.level0_stop_writes_trigger); + } + + @Override + public MutableColumnFamilyOptionsBuilder setMaxCompactionBytes(final long maxCompactionBytes) { + return setLong(CompactionOption.max_compaction_bytes, maxCompactionBytes); + } + + @Override + public long maxCompactionBytes() { + return getLong(CompactionOption.max_compaction_bytes); + } + + + @Override + public MutableColumnFamilyOptionsBuilder setTargetFileSizeBase( + final long targetFileSizeBase) { + return setLong(CompactionOption.target_file_size_base, + targetFileSizeBase); + } + + @Override + public long targetFileSizeBase() { + return getLong(CompactionOption.target_file_size_base); + } + + @Override + public MutableColumnFamilyOptionsBuilder setTargetFileSizeMultiplier( + final int targetFileSizeMultiplier) { + return setInt(CompactionOption.target_file_size_multiplier, + targetFileSizeMultiplier); + } + + @Override + public int targetFileSizeMultiplier() { + return getInt(CompactionOption.target_file_size_multiplier); + } + + @Override + public MutableColumnFamilyOptionsBuilder setMaxBytesForLevelBase( + final long maxBytesForLevelBase) { + return setLong(CompactionOption.max_bytes_for_level_base, + maxBytesForLevelBase); + } + + @Override + public long maxBytesForLevelBase() { + return getLong(CompactionOption.max_bytes_for_level_base); + } + + @Override + public MutableColumnFamilyOptionsBuilder setMaxBytesForLevelMultiplier( + final double maxBytesForLevelMultiplier) { + return setDouble(CompactionOption.max_bytes_for_level_multiplier, maxBytesForLevelMultiplier); + } + + @Override + public double maxBytesForLevelMultiplier() { + return getDouble(CompactionOption.max_bytes_for_level_multiplier); + } + + @Override + public MutableColumnFamilyOptionsBuilder setMaxBytesForLevelMultiplierAdditional( + final int[] maxBytesForLevelMultiplierAdditional) { + return setIntArray( + CompactionOption.max_bytes_for_level_multiplier_additional, + maxBytesForLevelMultiplierAdditional); + } + + @Override + public int[] maxBytesForLevelMultiplierAdditional() { + return getIntArray( + CompactionOption.max_bytes_for_level_multiplier_additional); + } + + @Override + public MutableColumnFamilyOptionsBuilder setMaxSequentialSkipInIterations( + final long maxSequentialSkipInIterations) { + return setLong(MiscOption.max_sequential_skip_in_iterations, + maxSequentialSkipInIterations); + } + + @Override + public long maxSequentialSkipInIterations() { + return getLong(MiscOption.max_sequential_skip_in_iterations); + } + + @Override + public MutableColumnFamilyOptionsBuilder setParanoidFileChecks( + final boolean paranoidFileChecks) { + return setBoolean(MiscOption.paranoid_file_checks, paranoidFileChecks); + } + + @Override + public boolean paranoidFileChecks() { + return getBoolean(MiscOption.paranoid_file_checks); + } + + @Override + public MutableColumnFamilyOptionsBuilder setCompressionType( + final CompressionType compressionType) { + return setEnum(MiscOption.compression, compressionType); + } + + @Override + public CompressionType compressionType() { + return getEnum(MiscOption.compression); + } + + @Override + public MutableColumnFamilyOptionsBuilder setReportBgIoStats( + final boolean reportBgIoStats) { + return setBoolean(MiscOption.report_bg_io_stats, reportBgIoStats); + } + + @Override + public boolean reportBgIoStats() { + return getBoolean(MiscOption.report_bg_io_stats); + } + + @Override + public MutableColumnFamilyOptionsBuilder setTtl(final long ttl) { + return setLong(CompactionOption.ttl, ttl); + } + + @Override + public long ttl() { + return getLong(CompactionOption.ttl); + } + + @Override + public MutableColumnFamilyOptionsBuilder setPeriodicCompactionSeconds( + final long periodicCompactionSeconds) { + return setLong(CompactionOption.periodic_compaction_seconds, periodicCompactionSeconds); + } + + @Override + public long periodicCompactionSeconds() { + return getLong(CompactionOption.periodic_compaction_seconds); + } + + @Override + public MutableColumnFamilyOptionsBuilder setEnableBlobFiles(final boolean enableBlobFiles) { + return setBoolean(BlobOption.enable_blob_files, enableBlobFiles); + } + + @Override + public boolean enableBlobFiles() { + return getBoolean(BlobOption.enable_blob_files); + } + + @Override + public MutableColumnFamilyOptionsBuilder setMinBlobSize(final long minBlobSize) { + return setLong(BlobOption.min_blob_size, minBlobSize); + } + + @Override + public long minBlobSize() { + return getLong(BlobOption.min_blob_size); + } + + @Override + public MutableColumnFamilyOptionsBuilder setBlobFileSize(final long blobFileSize) { + return setLong(BlobOption.blob_file_size, blobFileSize); + } + + @Override + public long blobFileSize() { + return getLong(BlobOption.blob_file_size); + } + + @Override + public MutableColumnFamilyOptionsBuilder setBlobCompressionType( + final CompressionType compressionType) { + return setEnum(BlobOption.blob_compression_type, compressionType); + } + + @Override + public CompressionType blobCompressionType() { + return getEnum(BlobOption.blob_compression_type); + } + + @Override + public MutableColumnFamilyOptionsBuilder setEnableBlobGarbageCollection( + final boolean enableBlobGarbageCollection) { + return setBoolean(BlobOption.enable_blob_garbage_collection, enableBlobGarbageCollection); + } + + @Override + public boolean enableBlobGarbageCollection() { + return getBoolean(BlobOption.enable_blob_garbage_collection); + } + + @Override + public MutableColumnFamilyOptionsBuilder setBlobGarbageCollectionAgeCutoff( + final double blobGarbageCollectionAgeCutoff) { + return setDouble( + BlobOption.blob_garbage_collection_age_cutoff, blobGarbageCollectionAgeCutoff); + } + + @Override + public double blobGarbageCollectionAgeCutoff() { + return getDouble(BlobOption.blob_garbage_collection_age_cutoff); + } + + @Override + public MutableColumnFamilyOptionsBuilder setBlobGarbageCollectionForceThreshold( + final double blobGarbageCollectionForceThreshold) { + return setDouble( + BlobOption.blob_garbage_collection_force_threshold, blobGarbageCollectionForceThreshold); + } + + @Override + public double blobGarbageCollectionForceThreshold() { + return getDouble(BlobOption.blob_garbage_collection_force_threshold); + } + + @Override + public MutableColumnFamilyOptionsBuilder setBlobCompactionReadaheadSize( + final long blobCompactionReadaheadSize) { + return setLong(BlobOption.blob_compaction_readahead_size, blobCompactionReadaheadSize); + } + + @Override + public long blobCompactionReadaheadSize() { + return getLong(BlobOption.blob_compaction_readahead_size); + } + + @Override + public MutableColumnFamilyOptionsBuilder setBlobFileStartingLevel( + final int blobFileStartingLevel) { + return setInt(BlobOption.blob_file_starting_level, blobFileStartingLevel); + } + + @Override + public int blobFileStartingLevel() { + return getInt(BlobOption.blob_file_starting_level); + } + + @Override + public MutableColumnFamilyOptionsBuilder setPrepopulateBlobCache( + final PrepopulateBlobCache prepopulateBlobCache) { + return setEnum(BlobOption.prepopulate_blob_cache, prepopulateBlobCache); + } + + @Override + public PrepopulateBlobCache prepopulateBlobCache() { + return getEnum(BlobOption.prepopulate_blob_cache); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MutableColumnFamilyOptionsInterface.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MutableColumnFamilyOptionsInterface.java new file mode 100644 index 0000000..729b0e8 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MutableColumnFamilyOptionsInterface.java @@ -0,0 +1,156 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +public interface MutableColumnFamilyOptionsInterface< + T extends MutableColumnFamilyOptionsInterface> + extends AdvancedMutableColumnFamilyOptionsInterface { + /** + * Amount of data to build up in memory (backed by an unsorted log + * on disk) before converting to a sorted on-disk file. + *

    + * Larger values increase performance, especially during bulk loads. + * Up to {@code max_write_buffer_number} write buffers may be held in memory + * at the same time, so you may wish to adjust this parameter + * to control memory usage. + *

    + * Also, a larger write buffer will result in a longer recovery time + * the next time the database is opened. + *

    + * Default: 64MB + * @param writeBufferSize the size of write buffer. + * @return the instance of the current object. + * @throws java.lang.IllegalArgumentException thrown on 32-Bit platforms + * while overflowing the underlying platform specific value. + */ + T setWriteBufferSize(long writeBufferSize); + + /** + * Return size of write buffer size. + * + * @return size of write buffer. + * @see #setWriteBufferSize(long) + */ + long writeBufferSize(); + + /** + * Disable automatic compactions. Manual compactions can still + * be issued on this column family + * + * @param disableAutoCompactions true if auto-compactions are disabled. + * @return the reference to the current option. + */ + T setDisableAutoCompactions(boolean disableAutoCompactions); + + /** + * Disable automatic compactions. Manual compactions can still + * be issued on this column family + * + * @return true if auto-compactions are disabled. + */ + boolean disableAutoCompactions(); + + /** + * Number of files to trigger level-0 compaction. A value < 0 means that + * level-0 compaction will not be triggered by number of files at all. + *

    + * Default: 4 + * + * @param level0FileNumCompactionTrigger The number of files to trigger + * level-0 compaction + * @return the reference to the current option. + */ + T setLevel0FileNumCompactionTrigger(int level0FileNumCompactionTrigger); + + /** + * Number of files to trigger level-0 compaction. A value < 0 means that + * level-0 compaction will not be triggered by number of files at all. + *

    + * Default: 4 + * + * @return The number of files to trigger + */ + int level0FileNumCompactionTrigger(); + + /** + * We try to limit number of bytes in one compaction to be lower than this + * threshold. But it's not guaranteed. + * Value 0 will be sanitized. + * + * @param maxCompactionBytes max bytes in a compaction + * @return the reference to the current option. + * @see #maxCompactionBytes() + */ + T setMaxCompactionBytes(final long maxCompactionBytes); + + /** + * We try to limit number of bytes in one compaction to be lower than this + * threshold. But it's not guaranteed. + * Value 0 will be sanitized. + * + * @return the maximum number of bytes in for a compaction. + * @see #setMaxCompactionBytes(long) + */ + long maxCompactionBytes(); + + /** + * The upper-bound of the total size of level-1 files in bytes. + * Maximum number of bytes for level L can be calculated as + * (maxBytesForLevelBase) * (maxBytesForLevelMultiplier ^ (L-1)) + * For example, if maxBytesForLevelBase is 20MB, and if + * max_bytes_for_level_multiplier is 10, total data size for level-1 + * will be 200MB, total file size for level-2 will be 2GB, + * and total file size for level-3 will be 20GB. + * by default 'maxBytesForLevelBase' is 256MB. + * + * @param maxBytesForLevelBase maximum bytes for level base. + * + * @return the reference to the current option. + *

    + * See {@link AdvancedMutableColumnFamilyOptionsInterface#setMaxBytesForLevelMultiplier(double)} + */ + T setMaxBytesForLevelBase( + long maxBytesForLevelBase); + + /** + * The upper-bound of the total size of level-1 files in bytes. + * Maximum number of bytes for level L can be calculated as + * (maxBytesForLevelBase) * (maxBytesForLevelMultiplier ^ (L-1)) + * For example, if maxBytesForLevelBase is 20MB, and if + * max_bytes_for_level_multiplier is 10, total data size for level-1 + * will be 200MB, total file size for level-2 will be 2GB, + * and total file size for level-3 will be 20GB. + * by default 'maxBytesForLevelBase' is 256MB. + * + * @return the upper-bound of the total size of level-1 files + * in bytes. + *

    + * See {@link AdvancedMutableColumnFamilyOptionsInterface#maxBytesForLevelMultiplier()} + */ + long maxBytesForLevelBase(); + + /** + * Compress blocks using the specified compression algorithm. This + * parameter can be changed dynamically. + *

    + * Default: SNAPPY_COMPRESSION, which gives lightweight but fast compression. + * + * @param compressionType Compression Type. + * @return the reference to the current option. + */ + T setCompressionType( + CompressionType compressionType); + + /** + * Compress blocks using the specified compression algorithm. This + * parameter can be changed dynamically. + *

    + * Default: SNAPPY_COMPRESSION, which gives lightweight but fast compression. + * + * @return Compression type. + */ + CompressionType compressionType(); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MutableDBOptions.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MutableDBOptions.java new file mode 100644 index 0000000..927e805 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MutableDBOptions.java @@ -0,0 +1,292 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class MutableDBOptions extends AbstractMutableOptions { + /** + * User must use builder pattern, or parser. + * + * @param keys the keys + * @param values the values + *

    + * See {@link #builder()} and {@link #parse(String)}. + */ + private MutableDBOptions(final String[] keys, final String[] values) { + super(keys, values); + } + + /** + * Creates a builder which allows you + * to set MutableDBOptions in a fluent + * manner + * + * @return A builder for MutableDBOptions + */ + public static MutableDBOptionsBuilder builder() { + return new MutableDBOptionsBuilder(); + } + + /** + * Parses a String representation of MutableDBOptions + *

    + * The format is: key1=value1;key2=value2;key3=value3 etc + *

    + * For int[] values, each int should be separated by a comma, e.g. + *

    + * key1=value1;intArrayKey1=1:2:3 + * + * @param str The string representation of the mutable db options + * @param ignoreUnknown what to do if the key is not one of the keys we expect + * + * @return A builder for the mutable db options + */ + public static MutableDBOptionsBuilder parse(final String str, final boolean ignoreUnknown) { + Objects.requireNonNull(str); + + final List parsedOptions = OptionString.Parser.parse(str); + return new MutableDBOptions.MutableDBOptionsBuilder().fromParsed(parsedOptions, ignoreUnknown); + } + + public static MutableDBOptionsBuilder parse(final String str) { + return parse(str, false); + } + + private interface MutableDBOptionKey extends MutableOptionKey {} + + public enum DBOption implements MutableDBOptionKey { + max_background_jobs(ValueType.INT), + max_background_compactions(ValueType.INT), + avoid_flush_during_shutdown(ValueType.BOOLEAN), + writable_file_max_buffer_size(ValueType.LONG), + delayed_write_rate(ValueType.LONG), + max_total_wal_size(ValueType.LONG), + delete_obsolete_files_period_micros(ValueType.LONG), + stats_dump_period_sec(ValueType.INT), + stats_persist_period_sec(ValueType.INT), + stats_history_buffer_size(ValueType.LONG), + max_open_files(ValueType.INT), + bytes_per_sync(ValueType.LONG), + wal_bytes_per_sync(ValueType.LONG), + strict_bytes_per_sync(ValueType.BOOLEAN), + compaction_readahead_size(ValueType.LONG); + + private final ValueType valueType; + DBOption(final ValueType valueType) { + this.valueType = valueType; + } + + @Override + public ValueType getValueType() { + return valueType; + } + } + + public static class MutableDBOptionsBuilder + extends AbstractMutableOptionsBuilder + implements MutableDBOptionsInterface { + private static final Map ALL_KEYS_LOOKUP = new HashMap<>(); + static { + for(final MutableDBOptionKey key : DBOption.values()) { + ALL_KEYS_LOOKUP.put(key.name(), key); + } + } + + private MutableDBOptionsBuilder() { + super(); + } + + @Override + protected MutableDBOptionsBuilder self() { + return this; + } + + @Override + protected Map allKeys() { + return ALL_KEYS_LOOKUP; + } + + @Override + protected MutableDBOptions build(final String[] keys, + final String[] values) { + return new MutableDBOptions(keys, values); + } + + @Override + public MutableDBOptionsBuilder setMaxBackgroundJobs( + final int maxBackgroundJobs) { + return setInt(DBOption.max_background_jobs, maxBackgroundJobs); + } + + @Override + public int maxBackgroundJobs() { + return getInt(DBOption.max_background_jobs); + } + + @Override + @Deprecated + public MutableDBOptionsBuilder setMaxBackgroundCompactions( + final int maxBackgroundCompactions) { + return setInt(DBOption.max_background_compactions, + maxBackgroundCompactions); + } + + @Override + @Deprecated + public int maxBackgroundCompactions() { + return getInt(DBOption.max_background_compactions); + } + + @Override + public MutableDBOptionsBuilder setAvoidFlushDuringShutdown( + final boolean avoidFlushDuringShutdown) { + return setBoolean(DBOption.avoid_flush_during_shutdown, + avoidFlushDuringShutdown); + } + + @Override + public boolean avoidFlushDuringShutdown() { + return getBoolean(DBOption.avoid_flush_during_shutdown); + } + + @Override + public MutableDBOptionsBuilder setWritableFileMaxBufferSize( + final long writableFileMaxBufferSize) { + return setLong(DBOption.writable_file_max_buffer_size, + writableFileMaxBufferSize); + } + + @Override + public long writableFileMaxBufferSize() { + return getLong(DBOption.writable_file_max_buffer_size); + } + + @Override + public MutableDBOptionsBuilder setDelayedWriteRate( + final long delayedWriteRate) { + return setLong(DBOption.delayed_write_rate, + delayedWriteRate); + } + + @Override + public long delayedWriteRate() { + return getLong(DBOption.delayed_write_rate); + } + + @Override + public MutableDBOptionsBuilder setMaxTotalWalSize( + final long maxTotalWalSize) { + return setLong(DBOption.max_total_wal_size, maxTotalWalSize); + } + + @Override + public long maxTotalWalSize() { + return getLong(DBOption.max_total_wal_size); + } + + @Override + public MutableDBOptionsBuilder setDeleteObsoleteFilesPeriodMicros( + final long micros) { + return setLong(DBOption.delete_obsolete_files_period_micros, micros); + } + + @Override + public long deleteObsoleteFilesPeriodMicros() { + return getLong(DBOption.delete_obsolete_files_period_micros); + } + + @Override + public MutableDBOptionsBuilder setStatsDumpPeriodSec( + final int statsDumpPeriodSec) { + return setInt(DBOption.stats_dump_period_sec, statsDumpPeriodSec); + } + + @Override + public int statsDumpPeriodSec() { + return getInt(DBOption.stats_dump_period_sec); + } + + @Override + public MutableDBOptionsBuilder setStatsPersistPeriodSec( + final int statsPersistPeriodSec) { + return setInt(DBOption.stats_persist_period_sec, statsPersistPeriodSec); + } + + @Override + public int statsPersistPeriodSec() { + return getInt(DBOption.stats_persist_period_sec); + } + + @Override + public MutableDBOptionsBuilder setStatsHistoryBufferSize( + final long statsHistoryBufferSize) { + return setLong(DBOption.stats_history_buffer_size, statsHistoryBufferSize); + } + + @Override + public long statsHistoryBufferSize() { + return getLong(DBOption.stats_history_buffer_size); + } + + @Override + public MutableDBOptionsBuilder setMaxOpenFiles(final int maxOpenFiles) { + return setInt(DBOption.max_open_files, maxOpenFiles); + } + + @Override + public int maxOpenFiles() { + return getInt(DBOption.max_open_files); + } + + @Override + public MutableDBOptionsBuilder setBytesPerSync(final long bytesPerSync) { + return setLong(DBOption.bytes_per_sync, bytesPerSync); + } + + @Override + public long bytesPerSync() { + return getLong(DBOption.bytes_per_sync); + } + + @Override + public MutableDBOptionsBuilder setWalBytesPerSync( + final long walBytesPerSync) { + return setLong(DBOption.wal_bytes_per_sync, walBytesPerSync); + } + + @Override + public long walBytesPerSync() { + return getLong(DBOption.wal_bytes_per_sync); + } + + @Override + public MutableDBOptionsBuilder setStrictBytesPerSync( + final boolean strictBytesPerSync) { + return setBoolean(DBOption.strict_bytes_per_sync, strictBytesPerSync); + } + + @Override + public boolean strictBytesPerSync() { + return getBoolean(DBOption.strict_bytes_per_sync); + } + + @Override + public MutableDBOptionsBuilder setCompactionReadaheadSize( + final long compactionReadaheadSize) { + return setLong(DBOption.compaction_readahead_size, + compactionReadaheadSize); + } + + @Override + public long compactionReadaheadSize() { + return getLong(DBOption.compaction_readahead_size); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MutableDBOptionsInterface.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MutableDBOptionsInterface.java new file mode 100644 index 0000000..8bf7b0d --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MutableDBOptionsInterface.java @@ -0,0 +1,440 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb; + +public interface MutableDBOptionsInterface> { + /** + * Specifies the maximum number of concurrent background jobs (both flushes + * and compactions combined). + * Default: 2 + * + * @param maxBackgroundJobs number of max concurrent background jobs + * @return the instance of the current object. + */ + T setMaxBackgroundJobs(int maxBackgroundJobs); + + /** + * Returns the maximum number of concurrent background jobs (both flushes + * and compactions combined). + * Default: 2 + * + * @return the maximum number of concurrent background jobs. + */ + int maxBackgroundJobs(); + + /** + * NOT SUPPORTED ANYMORE: RocksDB automatically decides this based on the + * value of max_background_jobs. For backwards compatibility we will set + * `max_background_jobs = max_background_compactions + max_background_flushes` + * in the case where user sets at least one of `max_background_compactions` or + * `max_background_flushes` (we replace -1 by 1 in case one option is unset). + *

    + * Specifies the maximum number of concurrent background compaction jobs, + * submitted to the default LOW priority thread pool. + * If you're increasing this, also consider increasing number of threads in + * LOW priority thread pool. For more information, see + * Default: -1 + * + * @param maxBackgroundCompactions the maximum number of background + * compaction jobs. + * @return the instance of the current object. + * + * @see RocksEnv#setBackgroundThreads(int) + * @see RocksEnv#setBackgroundThreads(int, Priority) + * @see DBOptionsInterface#maxBackgroundFlushes() + * @deprecated Use {@link #setMaxBackgroundJobs(int)} + */ + @Deprecated + T setMaxBackgroundCompactions(int maxBackgroundCompactions); + + /** + * NOT SUPPORTED ANYMORE: RocksDB automatically decides this based on the + * value of max_background_jobs. For backwards compatibility we will set + * `max_background_jobs = max_background_compactions + max_background_flushes` + * in the case where user sets at least one of `max_background_compactions` or + * `max_background_flushes` (we replace -1 by 1 in case one option is unset). + *

    + * Returns the maximum number of concurrent background compaction jobs, + * submitted to the default LOW priority thread pool. + * When increasing this number, we may also want to consider increasing + * number of threads in LOW priority thread pool. + * Default: -1 + * + * @return the maximum number of concurrent background compaction jobs. + * @see RocksEnv#setBackgroundThreads(int) + * @see RocksEnv#setBackgroundThreads(int, Priority) + * + * @deprecated Use {@link #setMaxBackgroundJobs(int)} + */ + @Deprecated + int maxBackgroundCompactions(); + + /** + * By default RocksDB will flush all memtables on DB close if there are + * unpersisted data (i.e. with WAL disabled) The flush can be skip to speedup + * DB close. Unpersisted data WILL BE LOST. + *

    + * DEFAULT: false + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)} + * API. + * + * @param avoidFlushDuringShutdown true if we should avoid flush during + * shutdown + * + * @return the reference to the current options. + */ + T setAvoidFlushDuringShutdown(boolean avoidFlushDuringShutdown); + + /** + * By default RocksDB will flush all memtables on DB close if there are + * unpersisted data (i.e. with WAL disabled) The flush can be skip to speedup + * DB close. Unpersisted data WILL BE LOST. + *

    + * DEFAULT: false + *

    + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)} + * API. + * + * @return true if we should avoid flush during shutdown + */ + boolean avoidFlushDuringShutdown(); + + /** + * This is the maximum buffer size that is used by WritableFileWriter. + * On Windows, we need to maintain an aligned buffer for writes. + * We allow the buffer to grow until it's size hits the limit. + *

    + * Default: 1024 * 1024 (1 MB) + * + * @param writableFileMaxBufferSize the maximum buffer size + * + * @return the reference to the current options. + */ + T setWritableFileMaxBufferSize(long writableFileMaxBufferSize); + + /** + * This is the maximum buffer size that is used by WritableFileWriter. + * On Windows, we need to maintain an aligned buffer for writes. + * We allow the buffer to grow until it's size hits the limit. + *

    + * Default: 1024 * 1024 (1 MB) + * + * @return the maximum buffer size + */ + long writableFileMaxBufferSize(); + + /** + * The limited write rate to DB if + * {@link ColumnFamilyOptions#softPendingCompactionBytesLimit()} or + * {@link ColumnFamilyOptions#level0SlowdownWritesTrigger()} is triggered, + * or we are writing to the last mem table allowed and we allow more than 3 + * mem tables. It is calculated using size of user write requests before + * compression. RocksDB may decide to slow down more if the compaction still + * gets behind further. + * If the value is 0, we will infer a value from `rater_limiter` value + * if it is not empty, or 16MB if `rater_limiter` is empty. Note that + * if users change the rate in `rate_limiter` after DB is opened, + * `delayed_write_rate` won't be adjusted. + *

    + * Unit: bytes per second. + *

    + * Default: 0 + *

    + * Dynamically changeable through {@link RocksDB#setDBOptions(MutableDBOptions)}. + * + * @param delayedWriteRate the rate in bytes per second + * + * @return the reference to the current options. + */ + T setDelayedWriteRate(long delayedWriteRate); + + /** + * The limited write rate to DB if + * {@link ColumnFamilyOptions#softPendingCompactionBytesLimit()} or + * {@link ColumnFamilyOptions#level0SlowdownWritesTrigger()} is triggered, + * or we are writing to the last mem table allowed and we allow more than 3 + * mem tables. It is calculated using size of user write requests before + * compression. RocksDB may decide to slow down more if the compaction still + * gets behind further. + * If the value is 0, we will infer a value from `rater_limiter` value + * if it is not empty, or 16MB if `rater_limiter` is empty. Note that + * if users change the rate in `rate_limiter` after DB is opened, + * `delayed_write_rate` won't be adjusted. + *

    + * Unit: bytes per second. + *

    + * Default: 0 + *

    + * Dynamically changeable through {@link RocksDB#setDBOptions(MutableDBOptions)}. + * + * @return the rate in bytes per second + */ + long delayedWriteRate(); + + /** + *

    Set the max total write-ahead log size. Once write-ahead logs exceed this size, we will + * start forcing the flush of column families whose memtables are backed by the oldest live WAL + * file + *

    + *

    The oldest WAL files are the ones that are causing all the space amplification. + *

    + * For example, with 15 column families, each with + * write_buffer_size = 128 MB + * max_write_buffer_number = 6 + * max_total_wal_size will be calculated to be [15 * 128MB * 6] * 4 = + * 45GB + *

    + * The RocksDB wiki has some discussion about how the WAL interacts + * with memtables and flushing of column families, at + * ... + *

    + *

    If set to 0 (default), we will dynamically choose the WAL size limit to + * be [sum of all write_buffer_size * max_write_buffer_number] * 4

    + *

    This option takes effect only when there are more than one column family as + * otherwise the wal size is dictated by the write_buffer_size.

    + *

    Default: 0

    + * + * @param maxTotalWalSize max total wal size. + * @return the instance of the current object. + */ + T setMaxTotalWalSize(long maxTotalWalSize); + + /** + *

    Returns the max total write-ahead log size. Once write-ahead logs exceed this size, + * we will start forcing the flush of column families whose memtables are + * backed by the oldest live WAL file.

    + *

    The oldest WAL files are the ones that are causing all the space amplification. + *

    + * For example, with 15 column families, each with + * write_buffer_size = 128 MB + * max_write_buffer_number = 6 + * max_total_wal_size will be calculated to be [15 * 128MB * 6] * 4 = + * 45GB + *

    + * The RocksDB wiki has some discussion about how the WAL interacts + * with memtables and flushing of column families, at + * ... + *

    + *

    If set to 0 (default), we will dynamically choose the WAL size limit to + * be [sum of all write_buffer_size * max_write_buffer_number] * 4

    + *

    This option takes effect only when there are more than one column family as + * otherwise the wal size is dictated by the write_buffer_size.

    + *

    Default: 0

    + * + * + *

    If set to 0 (default), we will dynamically choose the WAL size limit + * to be [sum of all write_buffer_size * max_write_buffer_number] * 4 + *

    + * + * @return max total wal size + */ + long maxTotalWalSize(); + + /** + * The periodicity when obsolete files get deleted. The default + * value is 6 hours. The files that get out of scope by compaction + * process will still get automatically delete on every compaction, + * regardless of this setting + * + * @param micros the time interval in micros + * @return the instance of the current object. + */ + T setDeleteObsoleteFilesPeriodMicros(long micros); + + /** + * The periodicity when obsolete files get deleted. The default + * value is 6 hours. The files that get out of scope by compaction + * process will still get automatically delete on every compaction, + * regardless of this setting + * + * @return the time interval in micros when obsolete files will be deleted. + */ + long deleteObsoleteFilesPeriodMicros(); + + /** + * if not zero, dump rocksdb.stats to LOG every stats_dump_period_sec + * Default: 600 (10 minutes) + * + * @param statsDumpPeriodSec time interval in seconds. + * @return the instance of the current object. + */ + T setStatsDumpPeriodSec(int statsDumpPeriodSec); + + /** + * If not zero, dump rocksdb.stats to LOG every stats_dump_period_sec + * Default: 600 (10 minutes) + * + * @return time interval in seconds. + */ + int statsDumpPeriodSec(); + + /** + * If not zero, dump rocksdb.stats to RocksDB every + * {@code statsPersistPeriodSec} + * + * Default: 600 + * + * @param statsPersistPeriodSec time interval in seconds. + * @return the instance of the current object. + */ + T setStatsPersistPeriodSec(int statsPersistPeriodSec); + + /** + * If not zero, dump rocksdb.stats to RocksDB every + * {@code statsPersistPeriodSec} + * + * @return time interval in seconds. + */ + int statsPersistPeriodSec(); + + /** + * If not zero, periodically take stats snapshots and store in memory, the + * memory size for stats snapshots is capped at {@code statsHistoryBufferSize} + * + * Default: 1MB + * + * @param statsHistoryBufferSize the size of the buffer. + * @return the instance of the current object. + */ + T setStatsHistoryBufferSize(long statsHistoryBufferSize); + + /** + * If not zero, periodically take stats snapshots and store in memory, the + * memory size for stats snapshots is capped at {@code statsHistoryBufferSize} + * + * @return the size of the buffer. + */ + long statsHistoryBufferSize(); + + /** + * Number of open files that can be used by the DB. You may need to + * increase this if your database has a large working set. Value -1 means + * files opened are always kept open. You can estimate number of files based + * on {@code target_file_size_base} and {@code target_file_size_multiplier} + * for level-based compaction. For universal-style compaction, you can usually + * set it to -1. + * Default: -1 + * + * @param maxOpenFiles the maximum number of open files. + * @return the instance of the current object. + */ + T setMaxOpenFiles(int maxOpenFiles); + + /** + * Number of open files that can be used by the DB. You may need to + * increase this if your database has a large working set. Value -1 means + * files opened are always kept open. You can estimate number of files based + * on {@code target_file_size_base} and {@code target_file_size_multiplier} + * for level-based compaction. For universal-style compaction, you can usually + * set it to -1. + * Default: -1 + * + * @return the maximum number of open files. + */ + int maxOpenFiles(); + + /** + * Allows OS to incrementally sync files to disk while they are being + * written, asynchronously, in the background. + * Issue one request for every bytes_per_sync written. 0 turns it off. + * Default: 0 + * + * @param bytesPerSync size in bytes + * @return the instance of the current object. + */ + T setBytesPerSync(long bytesPerSync); + + /** + * Allows OS to incrementally sync files to disk while they are being + * written, asynchronously, in the background. + * Issue one request for every bytes_per_sync written. 0 turns it off. + * Default: 0 + * + * @return size in bytes + */ + long bytesPerSync(); + + /** + * Same as {@link #setBytesPerSync(long)} , but applies to WAL files + *

    + * Default: 0, turned off + * + * @param walBytesPerSync size in bytes + * @return the instance of the current object. + */ + T setWalBytesPerSync(long walBytesPerSync); + + /** + * Same as {@link #bytesPerSync()} , but applies to WAL files + *

    + * Default: 0, turned off + * + * @return size in bytes + */ + long walBytesPerSync(); + + /** + * When true, guarantees WAL files have at most {@link #walBytesPerSync()} + * bytes submitted for writeback at any given time, and SST files have at most + * {@link #bytesPerSync()} bytes pending writeback at any given time. This + * can be used to handle cases where processing speed exceeds I/O speed + * during file generation, which can lead to a huge sync when the file is + * finished, even with {@link #bytesPerSync()} / {@link #walBytesPerSync()} + * properly configured. + *

    + * - If `sync_file_range` is supported it achieves this by waiting for any + * prior `sync_file_range`s to finish before proceeding. In this way, + * processing (compression, etc.) can proceed uninhibited in the gap + * between `sync_file_range`s, and we block only when I/O falls + * behind. + * - Otherwise the `WritableFile::Sync` method is used. Note this mechanism + * always blocks, thus preventing the interleaving of I/O and processing. + *

    + * Note: Enabling this option does not provide any additional persistence + * guarantees, as it may use `sync_file_range`, which does not write out + * metadata. + *

    + * Default: false + * + * @param strictBytesPerSync the bytes per sync + * @return the instance of the current object. + */ + T setStrictBytesPerSync(boolean strictBytesPerSync); + + /** + * Return the strict byte limit per sync. + *

    + * See {@link #setStrictBytesPerSync(boolean)} + * + * @return the limit in bytes. + */ + boolean strictBytesPerSync(); + + /** + * If non-zero, we perform bigger reads when doing compaction. If you're + * running RocksDB on spinning disks, you should set this to at least 2MB. + *

    + * That way RocksDB's compaction is doing sequential instead of random reads. + *

    + * Default: 0 + * + * @param compactionReadaheadSize The compaction read-ahead size + * + * @return the reference to the current options. + */ + T setCompactionReadaheadSize(final long compactionReadaheadSize); + + /** + * If non-zero, we perform bigger reads when doing compaction. If you're + * running RocksDB on spinning disks, you should set this to at least 2MB. + *

    + * That way RocksDB's compaction is doing sequential instead of random reads. + *

    + * Default: 0 + * + * @return The compaction read-ahead size + */ + long compactionReadaheadSize(); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MutableOptionKey.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MutableOptionKey.java new file mode 100644 index 0000000..ec1b9ff --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MutableOptionKey.java @@ -0,0 +1,16 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb; + +public interface MutableOptionKey { + enum ValueType { + DOUBLE, + LONG, + INT, + BOOLEAN, + INT_ARRAY, + ENUM + } + + String name(); + ValueType getValueType(); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MutableOptionValue.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MutableOptionValue.java new file mode 100644 index 0000000..fe689b5 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/MutableOptionValue.java @@ -0,0 +1,368 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb; + +import static org.rocksdb.AbstractMutableOptions.INT_ARRAY_INT_SEPARATOR; + +public abstract class MutableOptionValue { + + abstract double asDouble() throws NumberFormatException; + abstract long asLong() throws NumberFormatException; + abstract int asInt() throws NumberFormatException; + abstract boolean asBoolean() throws IllegalStateException; + abstract int[] asIntArray() throws IllegalStateException; + abstract String asString(); + abstract T asObject(); + + private abstract static class MutableOptionValueObject extends MutableOptionValue { + protected final T value; + + protected MutableOptionValueObject(final T value) { + this.value = value; + } + + @Override T asObject() { + return value; + } + } + + static MutableOptionValue fromString(final String s) { + return new MutableOptionStringValue(s); + } + + static MutableOptionValue fromDouble(final double d) { + return new MutableOptionDoubleValue(d); + } + + static MutableOptionValue fromLong(final long d) { + return new MutableOptionLongValue(d); + } + + static MutableOptionValue fromInt(final int i) { + return new MutableOptionIntValue(i); + } + + static MutableOptionValue fromBoolean(final boolean b) { + return new MutableOptionBooleanValue(b); + } + + static MutableOptionValue fromIntArray(final int[] ix) { + return new MutableOptionIntArrayValue(ix); + } + + static > MutableOptionValue fromEnum(final N value) { + return new MutableOptionEnumValue<>(value); + } + + static class MutableOptionStringValue + extends MutableOptionValueObject { + MutableOptionStringValue(final String value) { + super(value); + } + + @Override + double asDouble() throws NumberFormatException { + return Double.parseDouble(value); + } + + @Override + long asLong() throws NumberFormatException { + return Long.parseLong(value); + } + + @Override + int asInt() throws NumberFormatException { + return Integer.parseInt(value); + } + + @Override + boolean asBoolean() throws IllegalStateException { + return Boolean.parseBoolean(value); + } + + @Override + int[] asIntArray() throws IllegalStateException { + throw new IllegalStateException("String is not applicable as int[]"); + } + + @Override + String asString() { + return value; + } + } + + static class MutableOptionDoubleValue + extends MutableOptionValue { + private final double value; + MutableOptionDoubleValue(final double value) { + this.value = value; + } + + @Override + double asDouble() { + return value; + } + + @Override + long asLong() throws NumberFormatException { + return Double.valueOf(value).longValue(); + } + + @Override + int asInt() throws NumberFormatException { + if(value > Integer.MAX_VALUE || value < Integer.MIN_VALUE) { + throw new NumberFormatException( + "double value lies outside the bounds of int"); + } + return Double.valueOf(value).intValue(); + } + + @Override + boolean asBoolean() throws IllegalStateException { + throw new IllegalStateException( + "double is not applicable as boolean"); + } + + @Override + int[] asIntArray() throws IllegalStateException { + if(value > Integer.MAX_VALUE || value < Integer.MIN_VALUE) { + throw new NumberFormatException( + "double value lies outside the bounds of int"); + } + return new int[] { Double.valueOf(value).intValue() }; + } + + @Override + String asString() { + return String.valueOf(value); + } + + @Override + Double asObject() { + return value; + } + } + + static class MutableOptionLongValue + extends MutableOptionValue { + private final long value; + + MutableOptionLongValue(final long value) { + this.value = value; + } + + @Override + double asDouble() { + return Long.valueOf(value).doubleValue(); + } + + @Override + long asLong() throws NumberFormatException { + return value; + } + + @Override + int asInt() throws NumberFormatException { + if(value > Integer.MAX_VALUE || value < Integer.MIN_VALUE) { + throw new NumberFormatException( + "long value lies outside the bounds of int"); + } + return Long.valueOf(value).intValue(); + } + + @Override + boolean asBoolean() throws IllegalStateException { + throw new IllegalStateException( + "long is not applicable as boolean"); + } + + @Override + int[] asIntArray() throws IllegalStateException { + if(value > Integer.MAX_VALUE || value < Integer.MIN_VALUE) { + throw new NumberFormatException( + "long value lies outside the bounds of int"); + } + return new int[] { Long.valueOf(value).intValue() }; + } + + @Override + String asString() { + return String.valueOf(value); + } + + @Override + Long asObject() { + return value; + } + } + + static class MutableOptionIntValue + extends MutableOptionValue { + private final int value; + + MutableOptionIntValue(final int value) { + this.value = value; + } + + @Override + double asDouble() { + return Integer.valueOf(value).doubleValue(); + } + + @Override + long asLong() throws NumberFormatException { + return value; + } + + @Override + int asInt() throws NumberFormatException { + return value; + } + + @Override + boolean asBoolean() throws IllegalStateException { + throw new IllegalStateException("int is not applicable as boolean"); + } + + @Override + int[] asIntArray() throws IllegalStateException { + return new int[] { value }; + } + + @Override + String asString() { + return String.valueOf(value); + } + + @Override + Integer asObject() { + return value; + } + } + + static class MutableOptionBooleanValue + extends MutableOptionValue { + private final boolean value; + + MutableOptionBooleanValue(final boolean value) { + this.value = value; + } + + @Override + double asDouble() { + throw new NumberFormatException("boolean is not applicable as double"); + } + + @Override + long asLong() throws NumberFormatException { + throw new NumberFormatException("boolean is not applicable as Long"); + } + + @Override + int asInt() throws NumberFormatException { + throw new NumberFormatException("boolean is not applicable as int"); + } + + @Override + boolean asBoolean() { + return value; + } + + @Override + int[] asIntArray() throws IllegalStateException { + throw new IllegalStateException("boolean is not applicable as int[]"); + } + + @Override + String asString() { + return String.valueOf(value); + } + + @Override + Boolean asObject() { + return value; + } + } + + static class MutableOptionIntArrayValue + extends MutableOptionValueObject { + MutableOptionIntArrayValue(final int[] value) { + super(value); + } + + @Override + double asDouble() { + throw new NumberFormatException("int[] is not applicable as double"); + } + + @Override + long asLong() throws NumberFormatException { + throw new NumberFormatException("int[] is not applicable as Long"); + } + + @Override + int asInt() throws NumberFormatException { + throw new NumberFormatException("int[] is not applicable as int"); + } + + @Override + boolean asBoolean() { + throw new NumberFormatException("int[] is not applicable as boolean"); + } + + @Override + int[] asIntArray() throws IllegalStateException { + return value; + } + + @Override + String asString() { + final StringBuilder builder = new StringBuilder(); + for(int i = 0; i < value.length; i++) { + builder.append(value[i]); + if(i + 1 < value.length) { + builder.append(INT_ARRAY_INT_SEPARATOR); + } + } + return builder.toString(); + } + } + + static class MutableOptionEnumValue> + extends MutableOptionValueObject { + + MutableOptionEnumValue(final T value) { + super(value); + } + + @Override + double asDouble() throws NumberFormatException { + throw new NumberFormatException("Enum is not applicable as double"); + } + + @Override + long asLong() throws NumberFormatException { + throw new NumberFormatException("Enum is not applicable as long"); + } + + @Override + int asInt() throws NumberFormatException { + throw new NumberFormatException("Enum is not applicable as int"); + } + + @Override + boolean asBoolean() throws IllegalStateException { + throw new NumberFormatException("Enum is not applicable as boolean"); + } + + @Override + int[] asIntArray() throws IllegalStateException { + throw new NumberFormatException("Enum is not applicable as int[]"); + } + + @Override + String asString() { + return value.name(); + } + } + +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/NativeComparatorWrapper.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/NativeComparatorWrapper.java new file mode 100644 index 0000000..5ee042a --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/NativeComparatorWrapper.java @@ -0,0 +1,59 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.nio.ByteBuffer; + +/** + * A simple abstraction to allow a Java class to wrap a custom comparator + * implemented in C++. + *

    + * The native comparator must directly extend rocksdb::Comparator. + */ +public abstract class NativeComparatorWrapper + extends AbstractComparator { + + @Override + final ComparatorType getComparatorType() { + return ComparatorType.JAVA_NATIVE_COMPARATOR_WRAPPER; + } + + @Override + public final String name() { + throw new IllegalStateException("This should not be called. " + + "Implementation is in Native code"); + } + + @Override + public final int compare(final ByteBuffer s1, final ByteBuffer s2) { + throw new IllegalStateException("This should not be called. " + + "Implementation is in Native code"); + } + + @Override + public final void findShortestSeparator(final ByteBuffer start, final ByteBuffer limit) { + throw new IllegalStateException("This should not be called. " + + "Implementation is in Native code"); + } + + @Override + public final void findShortSuccessor(final ByteBuffer key) { + throw new IllegalStateException("This should not be called. " + + "Implementation is in Native code"); + } + + /** + * We override {@link RocksCallbackObject#disposeInternal()} + * as disposing of a native rocksdb::Comparator extension requires + * a slightly different approach as it is not really a RocksCallbackObject + */ + @Override + protected void disposeInternal() { + disposeInternal(nativeHandle_); + } + + private native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/NativeLibraryLoader.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/NativeLibraryLoader.java new file mode 100644 index 0000000..b97cf28 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/NativeLibraryLoader.java @@ -0,0 +1,172 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; + +import org.rocksdb.util.Environment; + +/** + * This class is used to load the RocksDB shared library from within the jar. + * The shared library is extracted to a temp folder and loaded from there. + */ +public class NativeLibraryLoader { + //singleton + private static final NativeLibraryLoader instance = new NativeLibraryLoader(); + private static boolean initialized = false; + + private static final String sharedLibraryName = Environment.getSharedLibraryName("rocksdb"); + private static final String jniLibraryName = Environment.getJniLibraryName("rocksdb"); + private static final /* @Nullable */ String fallbackJniLibraryName = + Environment.getFallbackJniLibraryName("rocksdb"); + private static final String jniLibraryFileName = Environment.getJniLibraryFileName("rocksdb"); + private static final /* @Nullable */ String fallbackJniLibraryFileName = + Environment.getFallbackJniLibraryFileName("rocksdb"); + private static final String tempFilePrefix = "librocksdbjni"; + private static final String tempFileSuffix = Environment.getJniLibraryExtension(); + + /** + * Get a reference to the NativeLibraryLoader + * + * @return The NativeLibraryLoader + */ + public static NativeLibraryLoader getInstance() { + return instance; + } + + /** + * Firstly attempts to load the library from java.library.path, + * if that fails then it falls back to extracting + * the library from the classpath + * {@link org.rocksdb.NativeLibraryLoader#loadLibraryFromJar(java.lang.String)} + * + * @param tmpDir A temporary directory to use + * to copy the native library to when loading from the classpath. + * If null, or the empty string, we rely on Java's + * {@link java.io.File#createTempFile(String, String)} + * function to provide a temporary location. + * The temporary file will be registered for deletion + * on exit. + * + * @throws java.io.IOException if a filesystem operation fails. + */ + public synchronized void loadLibrary(final String tmpDir) throws IOException { + try { + // try dynamic library + System.loadLibrary(sharedLibraryName); + return; + } catch (final UnsatisfiedLinkError ule) { + // ignore - try from static library + } + + try { + // try static library + System.loadLibrary(jniLibraryName); + return; + } catch (final UnsatisfiedLinkError ule) { + // ignore - then try static library fallback or from jar + } + + if (fallbackJniLibraryName != null) { + try { + // try static library fallback + System.loadLibrary(fallbackJniLibraryName); + return; + } catch (final UnsatisfiedLinkError ule) { + // ignore - then try from jar + } + } + + // try jar + loadLibraryFromJar(tmpDir); + } + + /** + * Attempts to extract the native RocksDB library + * from the classpath and load it + * + * @param tmpDir A temporary directory to use + * to copy the native library to. If null, + * or the empty string, we rely on Java's + * {@link java.io.File#createTempFile(String, String)} + * function to provide a temporary location. + * The temporary file will be registered for deletion + * on exit. + * + * @throws java.io.IOException if a filesystem operation fails. + */ + void loadLibraryFromJar(final String tmpDir) + throws IOException { + if (!initialized) { + System.load(loadLibraryFromJarToTemp(tmpDir).getAbsolutePath()); + initialized = true; + } + } + + File loadLibraryFromJarToTemp(final String tmpDir) + throws IOException { + InputStream is = null; + try { + // attempt to look up the static library in the jar file + String libraryFileName = jniLibraryFileName; + is = getClass().getClassLoader().getResourceAsStream(libraryFileName); + + if (is == null) { + // is there a fallback we can try + if (fallbackJniLibraryFileName == null) { + throw new RuntimeException(libraryFileName + " was not found inside JAR."); + } + + // attempt to look up the fallback static library in the jar file + libraryFileName = fallbackJniLibraryFileName; + is = getClass().getClassLoader().getResourceAsStream(libraryFileName); + if (is == null) { + throw new RuntimeException(libraryFileName + " was not found inside JAR."); + } + } + + // create a temporary file to copy the library to + final File temp; + if (tmpDir == null || tmpDir.isEmpty()) { + temp = File.createTempFile(tempFilePrefix, tempFileSuffix); + } else { + final File parentDir = new File(tmpDir); + if (!parentDir.exists()) { + throw new RuntimeException( + "Directory: " + parentDir.getAbsolutePath() + " does not exist!"); + } + temp = new File(parentDir, libraryFileName); + if (temp.exists() && !temp.delete()) { + throw new RuntimeException( + "File: " + temp.getAbsolutePath() + " already exists and cannot be removed."); + } + if (!temp.createNewFile()) { + throw new RuntimeException("File: " + temp.getAbsolutePath() + " could not be created."); + } + } + if (!temp.exists()) { + throw new RuntimeException("File " + temp.getAbsolutePath() + " does not exist."); + } else { + temp.deleteOnExit(); + } + + // copy the library from the Jar file to the temp destination + Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING); + + // return the temporary library file + return temp; + + } finally { + if (is != null) { + is.close(); + } + } + } + + /** + * Private constructor to disallow instantiation + */ + private NativeLibraryLoader() { + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/OperationStage.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/OperationStage.java new file mode 100644 index 0000000..6ac0a15 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/OperationStage.java @@ -0,0 +1,59 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * The operation stage. + */ +public enum OperationStage { + STAGE_UNKNOWN((byte)0x0), + STAGE_FLUSH_RUN((byte)0x1), + STAGE_FLUSH_WRITE_L0((byte)0x2), + STAGE_COMPACTION_PREPARE((byte)0x3), + STAGE_COMPACTION_RUN((byte)0x4), + STAGE_COMPACTION_PROCESS_KV((byte)0x5), + STAGE_COMPACTION_INSTALL((byte)0x6), + STAGE_COMPACTION_SYNC_FILE((byte)0x7), + STAGE_PICK_MEMTABLES_TO_FLUSH((byte)0x8), + STAGE_MEMTABLE_ROLLBACK((byte)0x9), + STAGE_MEMTABLE_INSTALL_FLUSH_RESULTS((byte)0xA); + + private final byte value; + + OperationStage(final byte value) { + this.value = value; + } + + /** + * Get the internal representation value. + * + * @return the internal representation value. + */ + byte getValue() { + return value; + } + + /** + * Get the Operation stage from the internal representation value. + * + * @param value the internal representation value. + * + * @return the operation stage + * + * @throws IllegalArgumentException if the value does not match + * an OperationStage + */ + static OperationStage fromValue(final byte value) + throws IllegalArgumentException { + for (final OperationStage threadType : OperationStage.values()) { + if (threadType.value == value) { + return threadType; + } + } + throw new IllegalArgumentException( + "Unknown value for OperationStage: " + value); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/OperationType.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/OperationType.java new file mode 100644 index 0000000..bf73534 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/OperationType.java @@ -0,0 +1,55 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * The type used to refer to a thread operation. + *

    + * A thread operation describes high-level action of a thread, + * examples include compaction and flush. + */ +public enum OperationType { + OP_UNKNOWN((byte)0x0), + OP_COMPACTION((byte)0x1), + OP_FLUSH((byte) 0x2), + OP_DBOPEN((byte) 0x3); + + private final byte value; + + OperationType(final byte value) { + this.value = value; + } + + /** + * Get the internal representation value. + * + * @return the internal representation value. + */ + byte getValue() { + return value; + } + + /** + * Get the Operation type from the internal representation value. + * + * @param value the internal representation value. + * + * @return the operation type + * + * @throws IllegalArgumentException if the value does not match + * an OperationType + */ + static OperationType fromValue(final byte value) + throws IllegalArgumentException { + for (final OperationType threadType : OperationType.values()) { + if (threadType.value == value) { + return threadType; + } + } + throw new IllegalArgumentException( + "Unknown value for OperationType: " + value); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/OptimisticTransactionDB.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/OptimisticTransactionDB.java new file mode 100644 index 0000000..ac3cdc2 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/OptimisticTransactionDB.java @@ -0,0 +1,224 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.List; + +/** + * Database with Transaction support. + */ +public class OptimisticTransactionDB extends RocksDB + implements TransactionalDB { + + /** + * Private constructor. + * + * @param nativeHandle The native handle of the C++ OptimisticTransactionDB + * object + */ + private OptimisticTransactionDB(final long nativeHandle) { + super(nativeHandle); + } + + /** + * Open an OptimisticTransactionDB similar to + * {@link RocksDB#open(Options, String)}. + * + * @param options {@link org.rocksdb.Options} instance. + * @param path the path to the rocksdb. + * + * @return a {@link OptimisticTransactionDB} instance on success, null if the + * specified {@link OptimisticTransactionDB} can not be opened. + * + * @throws RocksDBException if an error occurs whilst opening the database. + */ + public static OptimisticTransactionDB open(final Options options, + final String path) throws RocksDBException { + final OptimisticTransactionDB otdb = new OptimisticTransactionDB(open( + options.nativeHandle_, path)); + + // when non-default Options is used, keeping an Options reference + // in RocksDB can prevent Java to GC during the life-time of + // the currently-created RocksDB. + otdb.storeOptionsInstance(options); + + return otdb; + } + + /** + * Open an OptimisticTransactionDB similar to + * {@link RocksDB#open(DBOptions, String, List, List)}. + * + * @param dbOptions {@link org.rocksdb.DBOptions} instance. + * @param path the path to the rocksdb. + * @param columnFamilyDescriptors list of column family descriptors + * @param columnFamilyHandles will be filled with ColumnFamilyHandle instances + * + * @return a {@link OptimisticTransactionDB} instance on success, null if the + * specified {@link OptimisticTransactionDB} can not be opened. + * + * @throws RocksDBException if an error occurs whilst opening the database. + */ + public static OptimisticTransactionDB open(final DBOptions dbOptions, + final String path, + final List columnFamilyDescriptors, + final List columnFamilyHandles) + throws RocksDBException { + + final byte[][] cfNames = new byte[columnFamilyDescriptors.size()][]; + final long[] cfOptionHandles = new long[columnFamilyDescriptors.size()]; + for (int i = 0; i < columnFamilyDescriptors.size(); i++) { + final ColumnFamilyDescriptor cfDescriptor = columnFamilyDescriptors + .get(i); + cfNames[i] = cfDescriptor.getName(); + cfOptionHandles[i] = cfDescriptor.getOptions().nativeHandle_; + } + + final long[] handles = open(dbOptions.nativeHandle_, path, cfNames, + cfOptionHandles); + final OptimisticTransactionDB otdb = + new OptimisticTransactionDB(handles[0]); + + // when non-default Options is used, keeping an Options reference + // in RocksDB can prevent Java to GC during the life-time of + // the currently-created RocksDB. + otdb.storeOptionsInstance(dbOptions); + + for (int i = 1; i < handles.length; i++) { + columnFamilyHandles.add(new ColumnFamilyHandle(otdb, handles[i])); + } + + return otdb; + } + + /** + * This is similar to {@link #close()} except that it + * throws an exception if any error occurs. + *

    + * This will not fsync the WAL files. + * If syncing is required, the caller must first call {@link #syncWal()} + * or {@link #write(WriteOptions, WriteBatch)} using an empty write batch + * with {@link WriteOptions#setSync(boolean)} set to true. + *

    + * See also {@link #close()}. + * + * @throws RocksDBException if an error occurs whilst closing. + */ + public void closeE() throws RocksDBException { + if (owningHandle_.compareAndSet(true, false)) { + try { + closeDatabase(nativeHandle_); + } finally { + disposeInternal(); + } + } + } + + /** + * This is similar to {@link #closeE()} except that it + * silently ignores any errors. + *

    + * This will not fsync the WAL files. + * If syncing is required, the caller must first call {@link #syncWal()} + * or {@link #write(WriteOptions, WriteBatch)} using an empty write batch + * with {@link WriteOptions#setSync(boolean)} set to true. + *

    + * See also {@link #close()}. + */ + @Override + public void close() { + if (owningHandle_.compareAndSet(true, false)) { + try { + closeDatabase(nativeHandle_); + } catch (final RocksDBException e) { + // silently ignore the error report + } finally { + disposeInternal(); + } + } + } + + @Override + public Transaction beginTransaction(final WriteOptions writeOptions) { + return new Transaction(this, beginTransaction(nativeHandle_, + writeOptions.nativeHandle_)); + } + + @Override + public Transaction beginTransaction(final WriteOptions writeOptions, + final OptimisticTransactionOptions optimisticTransactionOptions) { + return new Transaction(this, beginTransaction(nativeHandle_, + writeOptions.nativeHandle_, + optimisticTransactionOptions.nativeHandle_)); + } + + // TODO(AR) consider having beingTransaction(... oldTransaction) set a + // reference count inside Transaction, so that we can always call + // Transaction#close but the object is only disposed when there are as many + // closes as beginTransaction. Makes the try-with-resources paradigm easier for + // java developers + + @Override + public Transaction beginTransaction(final WriteOptions writeOptions, + final Transaction oldTransaction) { + final long jtxn_handle = beginTransaction_withOld(nativeHandle_, + writeOptions.nativeHandle_, oldTransaction.nativeHandle_); + + // RocksJava relies on the assumption that + // we do not allocate a new Transaction object + // when providing an old_txn + assert(jtxn_handle == oldTransaction.nativeHandle_); + + return oldTransaction; + } + + @Override + public Transaction beginTransaction(final WriteOptions writeOptions, + final OptimisticTransactionOptions optimisticTransactionOptions, + final Transaction oldTransaction) { + final long jtxn_handle = beginTransaction_withOld(nativeHandle_, + writeOptions.nativeHandle_, optimisticTransactionOptions.nativeHandle_, + oldTransaction.nativeHandle_); + + // RocksJava relies on the assumption that + // we do not allocate a new Transaction object + // when providing an old_txn + assert(jtxn_handle == oldTransaction.nativeHandle_); + + return oldTransaction; + } + + /** + * Get the underlying database that was opened. + * + * @return The underlying database that was opened. + */ + public RocksDB getBaseDB() { + final RocksDB db = new RocksDB(getBaseDB(nativeHandle_)); + db.disOwnNativeHandle(); + return db; + } + + @Override protected final native void disposeInternal(final long handle); + + protected static native long open(final long optionsHandle, + final String path) throws RocksDBException; + protected static native long[] open(final long handle, final String path, + final byte[][] columnFamilyNames, final long[] columnFamilyOptions); + private static native void closeDatabase(final long handle) throws RocksDBException; + private native long beginTransaction(final long handle, + final long writeOptionsHandle); + private native long beginTransaction(final long handle, + final long writeOptionsHandle, + final long optimisticTransactionOptionsHandle); + private native long beginTransaction_withOld(final long handle, + final long writeOptionsHandle, final long oldTransactionHandle); + private native long beginTransaction_withOld(final long handle, + final long writeOptionsHandle, + final long optimisticTransactionOptionsHandle, + final long oldTransactionHandle); + private native long getBaseDB(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/OptimisticTransactionOptions.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/OptimisticTransactionOptions.java new file mode 100644 index 0000000..a2f5d85 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/OptimisticTransactionOptions.java @@ -0,0 +1,53 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +public class OptimisticTransactionOptions extends RocksObject + implements TransactionalOptions { + + public OptimisticTransactionOptions() { + super(newOptimisticTransactionOptions()); + } + + @Override + public boolean isSetSnapshot() { + assert(isOwningHandle()); + return isSetSnapshot(nativeHandle_); + } + + @Override + public OptimisticTransactionOptions setSetSnapshot( + final boolean setSnapshot) { + assert(isOwningHandle()); + setSetSnapshot(nativeHandle_, setSnapshot); + return this; + } + + /** + * Should be set if the DB has a non-default comparator. + * See comment in + * {@link WriteBatchWithIndex#WriteBatchWithIndex(AbstractComparator, int, boolean)} + * constructor. + * + * @param comparator The comparator to use for the transaction. + * + * @return this OptimisticTransactionOptions instance + */ + public OptimisticTransactionOptions setComparator( + final AbstractComparator comparator) { + assert(isOwningHandle()); + setComparator(nativeHandle_, comparator.nativeHandle_); + return this; + } + + private static native long newOptimisticTransactionOptions(); + private native boolean isSetSnapshot(final long handle); + private native void setSetSnapshot(final long handle, + final boolean setSnapshot); + private native void setComparator(final long handle, + final long comparatorHandle); + @Override protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/OptionString.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/OptionString.java new file mode 100644 index 0000000..61d2a94 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/OptionString.java @@ -0,0 +1,261 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public class OptionString { + private static final char kvPairSeparator = ';'; + private static final char kvSeparator = '='; + private static final char complexValueBegin = '{'; + private static final char complexValueEnd = '}'; + private static final char wrappedValueBegin = '{'; + private static final char wrappedValueEnd = '}'; + private static final char arrayValueSeparator = ':'; + + static class Value { + final List list; + final List complex; + + public Value(final List list, final List complex) { + this.list = list; + this.complex = complex; + } + + public boolean isList() { + return (this.list != null && this.complex == null); + } + + public static Value fromList(final List list) { + return new Value(list, null); + } + + public static Value fromComplex(final List complex) { + return new Value(null, complex); + } + + public String toString() { + final StringBuilder sb = new StringBuilder(); + if (isList()) { + for (final String item : list) { + sb.append(item).append(arrayValueSeparator); + } + // remove the final separator + if (sb.length() > 0) + sb.delete(sb.length() - 1, sb.length()); + } else { + sb.append('['); + for (final Entry entry : complex) { + sb.append(entry.toString()).append(';'); + } + sb.append(']'); + } + return sb.toString(); + } + } + + static class Entry { + public final String key; + public final Value value; + + private Entry(final String key, final Value value) { + this.key = key; + this.value = value; + } + + public String toString() { + return "" + key + "=" + value; + } + } + + static class Parser { + static class Exception extends RuntimeException { + public Exception(final String s) { + super(s); + } + } + + final String str; + final StringBuilder sb; + + private Parser(final String str) { + this.str = str; + this.sb = new StringBuilder(str); + } + + private void exception(final String message) { + final int pos = str.length() - sb.length(); + final int before = Math.min(pos, 64); + final int after = Math.min(64, str.length() - pos); + final String here = + str.substring(pos - before, pos) + "__*HERE*__" + str.substring(pos, pos + after); + + throw new Parser.Exception(message + " at [" + here + "]"); + } + + private void skipWhite() { + while (sb.length() > 0 && Character.isWhitespace(sb.charAt(0))) { + sb.delete(0, 1); + } + } + + private char first() { + if (sb.length() == 0) + exception("Unexpected end of input"); + return sb.charAt(0); + } + + private char next() { + if (sb.length() == 0) + exception("Unexpected end of input"); + final char c = sb.charAt(0); + sb.delete(0, 1); + return c; + } + + private boolean hasNext() { + return (sb.length() > 0); + } + + private boolean is(final char c) { + return (sb.length() > 0 && sb.charAt(0) == c); + } + + private boolean isKeyChar() { + if (!hasNext()) + return false; + final char c = first(); + return (Character.isAlphabetic(c) || Character.isDigit(c) || "_".indexOf(c) != -1); + } + + private boolean isValueChar() { + if (!hasNext()) + return false; + final char c = first(); + return (Character.isAlphabetic(c) || Character.isDigit(c) || "_-+.[]".indexOf(c) != -1); + } + + private String parseKey() { + final StringBuilder sbKey = new StringBuilder(); + sbKey.append(next()); + while (isKeyChar()) { + sbKey.append(next()); + } + + return sbKey.toString(); + } + + private String parseSimpleValue() { + if (is(wrappedValueBegin)) { + next(); + final String result = parseSimpleValue(); + if (!is(wrappedValueEnd)) { + exception("Expected to end a wrapped value with " + wrappedValueEnd); + } + next(); + + return result; + } else { + final StringBuilder sbValue = new StringBuilder(); + while (isValueChar()) sbValue.append(next()); + + return sbValue.toString(); + } + } + + private List parseList() { + final List list = new ArrayList<>(1); + while (true) { + list.add(parseSimpleValue()); + if (!is(arrayValueSeparator)) + break; + + next(); + } + + return list; + } + + private Entry parseOption() { + skipWhite(); + if (!isKeyChar()) { + exception("No valid key character(s) for key in key=value "); + } + final String key = parseKey(); + skipWhite(); + if (is(kvSeparator)) { + next(); + } else { + exception("Expected = separating key and value"); + } + skipWhite(); + final Value value = parseValue(); + return new Entry(key, value); + } + + private Value parseValue() { + skipWhite(); + if (is(complexValueBegin)) { + next(); + skipWhite(); + final Value value = Value.fromComplex(parseComplex()); + skipWhite(); + if (is(complexValueEnd)) { + next(); + skipWhite(); + } else { + exception("Expected } ending complex value"); + } + return value; + } else if (isValueChar()) { + return Value.fromList(parseList()); + } else if (is(kvPairSeparator)) { + // e.g. empty vector embedded in a struct option looks like + // struct_opt = {vector_opt=;...} + final List entries = new ArrayList<>(); + return Value.fromList(entries); + } + + exception("No valid value character(s) for value in key=value"); + return null; + } + + private List parseComplex() { + final List entries = new ArrayList<>(); + + skipWhite(); + if (hasNext()) { + entries.add(parseOption()); + skipWhite(); + while (is(kvPairSeparator)) { + next(); + skipWhite(); + if (!isKeyChar()) { + // the separator was a terminator + break; + } + entries.add(parseOption()); + skipWhite(); + } + } + return entries; + } + + public static List parse(final String str) { + Objects.requireNonNull(str); + + final Parser parser = new Parser(str); + final List result = parser.parseComplex(); + if (parser.hasNext()) { + parser.exception("Unexpected end of parsing "); + } + + return result; + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Options.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Options.java new file mode 100644 index 0000000..d00b489 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Options.java @@ -0,0 +1,2589 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.nio.file.Paths; +import java.util.*; + +/** + * Options to control the behavior of a database. It will be used + * during the creation of a {@link org.rocksdb.RocksDB} (i.e., RocksDB.open()). + *

    + * As a descendent of {@link AbstractNativeReference}, this class is {@link AutoCloseable} + * and will be automatically released if opened in the preamble of a try with resources block. + */ +public class Options extends RocksObject + implements DBOptionsInterface, + MutableDBOptionsInterface, + ColumnFamilyOptionsInterface, + MutableColumnFamilyOptionsInterface { + static { + RocksDB.loadLibrary(); + } + + /** + * Converts the input properties into a Options-style formatted string + * @param properties The set of properties to convert + * @return The Options-style representation of those properties. + */ + public static String getOptionStringFromProps(final Properties properties) { + if (properties == null || properties.size() == 0) { + throw new IllegalArgumentException("Properties value must contain at least one value."); + } + final StringBuilder stringBuilder = new StringBuilder(); + for (final String name : properties.stringPropertyNames()) { + stringBuilder.append(name); + stringBuilder.append("="); + stringBuilder.append(properties.getProperty(name)); + stringBuilder.append(";"); + } + return stringBuilder.toString(); + } + + /** + * Construct options for opening a RocksDB. + *

    + * This constructor will create (by allocating a block of memory) + * an {@code rocksdb::Options} in the c++ side. + */ + public Options() { + super(newOptions()); + env_ = Env.getDefault(); + } + + /** + * Construct options for opening a RocksDB. Reusing database options + * and column family options. + * + * @param dbOptions {@link org.rocksdb.DBOptions} instance + * @param columnFamilyOptions {@link org.rocksdb.ColumnFamilyOptions} + * instance + */ + public Options(final DBOptions dbOptions, + final ColumnFamilyOptions columnFamilyOptions) { + super(newOptions(dbOptions.nativeHandle_, + columnFamilyOptions.nativeHandle_)); + env_ = dbOptions.getEnv() != null ? dbOptions.getEnv() : Env.getDefault(); + } + + /** + * Copy constructor for ColumnFamilyOptions. + *

    + * NOTE: This does a shallow copy, which means comparator, merge_operator + * and other pointers will be cloned! + * + * @param other The Options to copy. + */ + public Options(final Options other) { + super(copyOptions(other.nativeHandle_)); + this.env_ = other.env_; + this.memTableConfig_ = other.memTableConfig_; + this.tableFormatConfig_ = other.tableFormatConfig_; + this.rateLimiter_ = other.rateLimiter_; + this.comparator_ = other.comparator_; + this.compactionFilter_ = other.compactionFilter_; + this.compactionFilterFactory_ = other.compactionFilterFactory_; + this.compactionOptionsUniversal_ = other.compactionOptionsUniversal_; + this.compactionOptionsFIFO_ = other.compactionOptionsFIFO_; + this.compressionOptions_ = other.compressionOptions_; + this.rowCache_ = other.rowCache_; + this.writeBufferManager_ = other.writeBufferManager_; + this.compactionThreadLimiter_ = other.compactionThreadLimiter_; + this.bottommostCompressionOptions_ = other.bottommostCompressionOptions_; + this.walFilter_ = other.walFilter_; + this.sstPartitionerFactory_ = other.sstPartitionerFactory_; + } + + @Override + public Options setIncreaseParallelism(final int totalThreads) { + assert(isOwningHandle()); + setIncreaseParallelism(nativeHandle_, totalThreads); + return this; + } + + @Override + public Options setCreateIfMissing(final boolean flag) { + assert(isOwningHandle()); + setCreateIfMissing(nativeHandle_, flag); + return this; + } + + @Override + public Options setCreateMissingColumnFamilies(final boolean flag) { + assert(isOwningHandle()); + setCreateMissingColumnFamilies(nativeHandle_, flag); + return this; + } + + @Override + public Options setEnv(final Env env) { + assert(isOwningHandle()); + setEnv(nativeHandle_, env.nativeHandle_); + env_ = env; + return this; + } + + @Override + public Env getEnv() { + return env_; + } + + /** + *

    Set appropriate parameters for bulk loading. + * The reason that this is a function that returns "this" instead of a + * constructor is to enable chaining of multiple similar calls in the future. + *

    + * + *

    All data will be in level 0 without any automatic compaction. + * It's recommended to manually call CompactRange(NULL, NULL) before reading + * from the database, because otherwise the read can be very slow.

    + * + * @return the instance of the current Options. + */ + public Options prepareForBulkLoad() { + prepareForBulkLoad(nativeHandle_); + return this; + } + + @Override + public boolean createIfMissing() { + assert(isOwningHandle()); + return createIfMissing(nativeHandle_); + } + + @Override + public boolean createMissingColumnFamilies() { + assert(isOwningHandle()); + return createMissingColumnFamilies(nativeHandle_); + } + + @Override + public Options oldDefaults(final int majorVersion, final int minorVersion) { + oldDefaults(nativeHandle_, majorVersion, minorVersion); + return this; + } + + @Override + public Options optimizeForSmallDb() { + optimizeForSmallDb(nativeHandle_); + return this; + } + + @Override + public Options optimizeForSmallDb(final Cache cache) { + optimizeForSmallDb(nativeHandle_, cache.getNativeHandle()); + return this; + } + + @Override + public Options optimizeForPointLookup(final long blockCacheSizeMb) { + optimizeForPointLookup(nativeHandle_, + blockCacheSizeMb); + return this; + } + + @Override + public Options optimizeLevelStyleCompaction() { + optimizeLevelStyleCompaction(nativeHandle_, + DEFAULT_COMPACTION_MEMTABLE_MEMORY_BUDGET); + return this; + } + + @Override + public Options optimizeLevelStyleCompaction(final long memtableMemoryBudget) { + optimizeLevelStyleCompaction(nativeHandle_, + memtableMemoryBudget); + return this; + } + + @Override + public Options optimizeUniversalStyleCompaction() { + optimizeUniversalStyleCompaction(nativeHandle_, + DEFAULT_COMPACTION_MEMTABLE_MEMORY_BUDGET); + return this; + } + + @Override + public Options optimizeUniversalStyleCompaction( + final long memtableMemoryBudget) { + optimizeUniversalStyleCompaction(nativeHandle_, + memtableMemoryBudget); + return this; + } + + @Override + public Options setComparator(final BuiltinComparator builtinComparator) { + assert(isOwningHandle()); + setComparatorHandle(nativeHandle_, builtinComparator.ordinal()); + return this; + } + + @Override + public Options setComparator( + final AbstractComparator comparator) { + assert(isOwningHandle()); + setComparatorHandle(nativeHandle_, comparator.nativeHandle_, + comparator.getComparatorType().getValue()); + comparator_ = comparator; + return this; + } + + @Override + public Options setMergeOperatorName(final String name) { + assert(isOwningHandle()); + if (name == null) { + throw new IllegalArgumentException( + "Merge operator name must not be null."); + } + setMergeOperatorName(nativeHandle_, name); + return this; + } + + @Override + public Options setMergeOperator(final MergeOperator mergeOperator) { + setMergeOperator(nativeHandle_, mergeOperator.nativeHandle_); + return this; + } + + @Override + public Options setCompactionFilter( + final AbstractCompactionFilter> + compactionFilter) { + setCompactionFilterHandle(nativeHandle_, compactionFilter.nativeHandle_); + compactionFilter_ = compactionFilter; + return this; + } + + @Override + public AbstractCompactionFilter> compactionFilter() { + assert (isOwningHandle()); + return compactionFilter_; + } + + @Override + public Options setCompactionFilterFactory(final AbstractCompactionFilterFactory> compactionFilterFactory) { + assert (isOwningHandle()); + setCompactionFilterFactoryHandle(nativeHandle_, compactionFilterFactory.nativeHandle_); + compactionFilterFactory_ = compactionFilterFactory; + return this; + } + + @Override + public AbstractCompactionFilterFactory> compactionFilterFactory() { + assert (isOwningHandle()); + return compactionFilterFactory_; + } + + @Override + public Options setWriteBufferSize(final long writeBufferSize) { + assert(isOwningHandle()); + setWriteBufferSize(nativeHandle_, writeBufferSize); + return this; + } + + @Override + public long writeBufferSize() { + assert(isOwningHandle()); + return writeBufferSize(nativeHandle_); + } + + @Override + public Options setMaxWriteBufferNumber(final int maxWriteBufferNumber) { + assert(isOwningHandle()); + setMaxWriteBufferNumber(nativeHandle_, maxWriteBufferNumber); + return this; + } + + @Override + public int maxWriteBufferNumber() { + assert(isOwningHandle()); + return maxWriteBufferNumber(nativeHandle_); + } + + @Override + public boolean errorIfExists() { + assert(isOwningHandle()); + return errorIfExists(nativeHandle_); + } + + @Override + public Options setErrorIfExists(final boolean errorIfExists) { + assert(isOwningHandle()); + setErrorIfExists(nativeHandle_, errorIfExists); + return this; + } + + @Override + public boolean paranoidChecks() { + assert(isOwningHandle()); + return paranoidChecks(nativeHandle_); + } + + @Override + public Options setParanoidChecks(final boolean paranoidChecks) { + assert(isOwningHandle()); + setParanoidChecks(nativeHandle_, paranoidChecks); + return this; + } + + @Override + public int maxOpenFiles() { + assert(isOwningHandle()); + return maxOpenFiles(nativeHandle_); + } + + @Override + public Options setMaxFileOpeningThreads(final int maxFileOpeningThreads) { + assert(isOwningHandle()); + setMaxFileOpeningThreads(nativeHandle_, maxFileOpeningThreads); + return this; + } + + @Override + public int maxFileOpeningThreads() { + assert(isOwningHandle()); + return maxFileOpeningThreads(nativeHandle_); + } + + @Override + public Options setMaxTotalWalSize(final long maxTotalWalSize) { + assert(isOwningHandle()); + setMaxTotalWalSize(nativeHandle_, maxTotalWalSize); + return this; + } + + @Override + public long maxTotalWalSize() { + assert(isOwningHandle()); + return maxTotalWalSize(nativeHandle_); + } + + @Override + public Options setMaxOpenFiles(final int maxOpenFiles) { + assert(isOwningHandle()); + setMaxOpenFiles(nativeHandle_, maxOpenFiles); + return this; + } + + @Override + public boolean useFsync() { + assert(isOwningHandle()); + return useFsync(nativeHandle_); + } + + @Override + public Options setUseFsync(final boolean useFsync) { + assert(isOwningHandle()); + setUseFsync(nativeHandle_, useFsync); + return this; + } + + @Override + public Options setDbPaths(final Collection dbPaths) { + assert(isOwningHandle()); + + final int len = dbPaths.size(); + final String[] paths = new String[len]; + final long[] targetSizes = new long[len]; + + int i = 0; + for(final DbPath dbPath : dbPaths) { + paths[i] = dbPath.path.toString(); + targetSizes[i] = dbPath.targetSize; + i++; + } + setDbPaths(nativeHandle_, paths, targetSizes); + return this; + } + + @Override + public List dbPaths() { + final int len = (int)dbPathsLen(nativeHandle_); + if(len == 0) { + return Collections.emptyList(); + } else { + final String[] paths = new String[len]; + final long[] targetSizes = new long[len]; + + dbPaths(nativeHandle_, paths, targetSizes); + + final List dbPaths = new ArrayList<>(); + for(int i = 0; i < len; i++) { + dbPaths.add(new DbPath(Paths.get(paths[i]), targetSizes[i])); + } + return dbPaths; + } + } + + @Override + public String dbLogDir() { + assert(isOwningHandle()); + return dbLogDir(nativeHandle_); + } + + @Override + public Options setDbLogDir(final String dbLogDir) { + assert(isOwningHandle()); + setDbLogDir(nativeHandle_, dbLogDir); + return this; + } + + @Override + public String walDir() { + assert(isOwningHandle()); + return walDir(nativeHandle_); + } + + @Override + public Options setWalDir(final String walDir) { + assert(isOwningHandle()); + setWalDir(nativeHandle_, walDir); + return this; + } + + @Override + public long deleteObsoleteFilesPeriodMicros() { + assert(isOwningHandle()); + return deleteObsoleteFilesPeriodMicros(nativeHandle_); + } + + @Override + public Options setDeleteObsoleteFilesPeriodMicros( + final long micros) { + assert(isOwningHandle()); + setDeleteObsoleteFilesPeriodMicros(nativeHandle_, micros); + return this; + } + + @Override + @Deprecated + public int maxBackgroundCompactions() { + assert(isOwningHandle()); + return maxBackgroundCompactions(nativeHandle_); + } + + @Override + public Options setStatistics(final Statistics statistics) { + assert(isOwningHandle()); + setStatistics(nativeHandle_, statistics.nativeHandle_); + return this; + } + + @Override + public Statistics statistics() { + assert(isOwningHandle()); + final long statisticsNativeHandle = statistics(nativeHandle_); + if(statisticsNativeHandle == 0) { + return null; + } else { + return new Statistics(statisticsNativeHandle); + } + } + + @Override + @Deprecated + public Options setMaxBackgroundCompactions( + final int maxBackgroundCompactions) { + assert(isOwningHandle()); + setMaxBackgroundCompactions(nativeHandle_, maxBackgroundCompactions); + return this; + } + + @Override + public Options setMaxSubcompactions(final int maxSubcompactions) { + assert(isOwningHandle()); + setMaxSubcompactions(nativeHandle_, maxSubcompactions); + return this; + } + + @Override + public int maxSubcompactions() { + assert(isOwningHandle()); + return maxSubcompactions(nativeHandle_); + } + + @Override + @Deprecated + public int maxBackgroundFlushes() { + assert(isOwningHandle()); + return maxBackgroundFlushes(nativeHandle_); + } + + @Override + @Deprecated + public Options setMaxBackgroundFlushes( + final int maxBackgroundFlushes) { + assert(isOwningHandle()); + setMaxBackgroundFlushes(nativeHandle_, maxBackgroundFlushes); + return this; + } + + @Override + public int maxBackgroundJobs() { + assert(isOwningHandle()); + return maxBackgroundJobs(nativeHandle_); + } + + @Override + public Options setMaxBackgroundJobs(final int maxBackgroundJobs) { + assert(isOwningHandle()); + setMaxBackgroundJobs(nativeHandle_, maxBackgroundJobs); + return this; + } + + @Override + public long maxLogFileSize() { + assert(isOwningHandle()); + return maxLogFileSize(nativeHandle_); + } + + @Override + public Options setMaxLogFileSize(final long maxLogFileSize) { + assert(isOwningHandle()); + setMaxLogFileSize(nativeHandle_, maxLogFileSize); + return this; + } + + @Override + public long logFileTimeToRoll() { + assert(isOwningHandle()); + return logFileTimeToRoll(nativeHandle_); + } + + @Override + public Options setLogFileTimeToRoll(final long logFileTimeToRoll) { + assert(isOwningHandle()); + setLogFileTimeToRoll(nativeHandle_, logFileTimeToRoll); + return this; + } + + @Override + public long keepLogFileNum() { + assert(isOwningHandle()); + return keepLogFileNum(nativeHandle_); + } + + @Override + public Options setKeepLogFileNum(final long keepLogFileNum) { + assert(isOwningHandle()); + setKeepLogFileNum(nativeHandle_, keepLogFileNum); + return this; + } + + + @Override + public Options setRecycleLogFileNum(final long recycleLogFileNum) { + assert(isOwningHandle()); + setRecycleLogFileNum(nativeHandle_, recycleLogFileNum); + return this; + } + + @Override + public long recycleLogFileNum() { + assert(isOwningHandle()); + return recycleLogFileNum(nativeHandle_); + } + + @Override + public long maxManifestFileSize() { + assert(isOwningHandle()); + return maxManifestFileSize(nativeHandle_); + } + + @Override + public Options setMaxManifestFileSize( + final long maxManifestFileSize) { + assert(isOwningHandle()); + setMaxManifestFileSize(nativeHandle_, maxManifestFileSize); + return this; + } + + @Override + public Options setMaxTableFilesSizeFIFO( + final long maxTableFilesSize) { + assert(maxTableFilesSize > 0); // unsigned native type + assert(isOwningHandle()); + setMaxTableFilesSizeFIFO(nativeHandle_, maxTableFilesSize); + return this; + } + + @Override + public long maxTableFilesSizeFIFO() { + return maxTableFilesSizeFIFO(nativeHandle_); + } + + @Override + public int tableCacheNumshardbits() { + assert(isOwningHandle()); + return tableCacheNumshardbits(nativeHandle_); + } + + @Override + public Options setTableCacheNumshardbits( + final int tableCacheNumshardbits) { + assert(isOwningHandle()); + setTableCacheNumshardbits(nativeHandle_, tableCacheNumshardbits); + return this; + } + + @Override + public long walTtlSeconds() { + assert(isOwningHandle()); + return walTtlSeconds(nativeHandle_); + } + + @Override + public Options setWalTtlSeconds(final long walTtlSeconds) { + assert(isOwningHandle()); + setWalTtlSeconds(nativeHandle_, walTtlSeconds); + return this; + } + + @Override + public long walSizeLimitMB() { + assert(isOwningHandle()); + return walSizeLimitMB(nativeHandle_); + } + + @Override + public Options setMaxWriteBatchGroupSizeBytes(final long maxWriteBatchGroupSizeBytes) { + setMaxWriteBatchGroupSizeBytes(nativeHandle_, maxWriteBatchGroupSizeBytes); + return this; + } + + @Override + public long maxWriteBatchGroupSizeBytes() { + assert (isOwningHandle()); + return maxWriteBatchGroupSizeBytes(nativeHandle_); + } + + @Override + public Options setWalSizeLimitMB(final long sizeLimitMB) { + assert(isOwningHandle()); + setWalSizeLimitMB(nativeHandle_, sizeLimitMB); + return this; + } + + @Override + public long manifestPreallocationSize() { + assert(isOwningHandle()); + return manifestPreallocationSize(nativeHandle_); + } + + @Override + public Options setManifestPreallocationSize(final long size) { + assert(isOwningHandle()); + setManifestPreallocationSize(nativeHandle_, size); + return this; + } + + @Override + public Options setUseDirectReads(final boolean useDirectReads) { + assert(isOwningHandle()); + setUseDirectReads(nativeHandle_, useDirectReads); + return this; + } + + @Override + public boolean useDirectReads() { + assert(isOwningHandle()); + return useDirectReads(nativeHandle_); + } + + @Override + public Options setUseDirectIoForFlushAndCompaction( + final boolean useDirectIoForFlushAndCompaction) { + assert(isOwningHandle()); + setUseDirectIoForFlushAndCompaction(nativeHandle_, useDirectIoForFlushAndCompaction); + return this; + } + + @Override + public boolean useDirectIoForFlushAndCompaction() { + assert(isOwningHandle()); + return useDirectIoForFlushAndCompaction(nativeHandle_); + } + + @Override + public Options setAllowFAllocate(final boolean allowFAllocate) { + assert(isOwningHandle()); + setAllowFAllocate(nativeHandle_, allowFAllocate); + return this; + } + + @Override + public boolean allowFAllocate() { + assert(isOwningHandle()); + return allowFAllocate(nativeHandle_); + } + + @Override + public boolean allowMmapReads() { + assert(isOwningHandle()); + return allowMmapReads(nativeHandle_); + } + + @Override + public Options setAllowMmapReads(final boolean allowMmapReads) { + assert(isOwningHandle()); + setAllowMmapReads(nativeHandle_, allowMmapReads); + return this; + } + + @Override + public boolean allowMmapWrites() { + assert(isOwningHandle()); + return allowMmapWrites(nativeHandle_); + } + + @Override + public Options setAllowMmapWrites(final boolean allowMmapWrites) { + assert(isOwningHandle()); + setAllowMmapWrites(nativeHandle_, allowMmapWrites); + return this; + } + + @Override + public boolean isFdCloseOnExec() { + assert(isOwningHandle()); + return isFdCloseOnExec(nativeHandle_); + } + + @Override + public Options setIsFdCloseOnExec(final boolean isFdCloseOnExec) { + assert(isOwningHandle()); + setIsFdCloseOnExec(nativeHandle_, isFdCloseOnExec); + return this; + } + + @Override + public int statsDumpPeriodSec() { + assert(isOwningHandle()); + return statsDumpPeriodSec(nativeHandle_); + } + + @Override + public Options setStatsDumpPeriodSec(final int statsDumpPeriodSec) { + assert(isOwningHandle()); + setStatsDumpPeriodSec(nativeHandle_, statsDumpPeriodSec); + return this; + } + + @Override + public Options setStatsPersistPeriodSec( + final int statsPersistPeriodSec) { + assert(isOwningHandle()); + setStatsPersistPeriodSec(nativeHandle_, statsPersistPeriodSec); + return this; + } + + @Override + public int statsPersistPeriodSec() { + assert(isOwningHandle()); + return statsPersistPeriodSec(nativeHandle_); + } + + @Override + public Options setStatsHistoryBufferSize( + final long statsHistoryBufferSize) { + assert(isOwningHandle()); + setStatsHistoryBufferSize(nativeHandle_, statsHistoryBufferSize); + return this; + } + + @Override + public long statsHistoryBufferSize() { + assert(isOwningHandle()); + return statsHistoryBufferSize(nativeHandle_); + } + + @Override + public boolean adviseRandomOnOpen() { + return adviseRandomOnOpen(nativeHandle_); + } + + @Override + public Options setAdviseRandomOnOpen(final boolean adviseRandomOnOpen) { + assert(isOwningHandle()); + setAdviseRandomOnOpen(nativeHandle_, adviseRandomOnOpen); + return this; + } + + @Override + public Options setDbWriteBufferSize(final long dbWriteBufferSize) { + assert(isOwningHandle()); + setDbWriteBufferSize(nativeHandle_, dbWriteBufferSize); + return this; + } + + @Override + public Options setWriteBufferManager(final WriteBufferManager writeBufferManager) { + assert(isOwningHandle()); + setWriteBufferManager(nativeHandle_, writeBufferManager.nativeHandle_); + this.writeBufferManager_ = writeBufferManager; + return this; + } + + @Override + public WriteBufferManager writeBufferManager() { + assert(isOwningHandle()); + return this.writeBufferManager_; + } + + @Override + public long dbWriteBufferSize() { + assert(isOwningHandle()); + return dbWriteBufferSize(nativeHandle_); + } + + @Override + @Deprecated + public Options setAccessHintOnCompactionStart(final AccessHint accessHint) { + assert(isOwningHandle()); + setAccessHintOnCompactionStart(nativeHandle_, accessHint.getValue()); + return this; + } + + @Override + @Deprecated + public AccessHint accessHintOnCompactionStart() { + assert(isOwningHandle()); + return AccessHint.getAccessHint(accessHintOnCompactionStart(nativeHandle_)); + } + + @Override + public Options setCompactionReadaheadSize(final long compactionReadaheadSize) { + assert(isOwningHandle()); + setCompactionReadaheadSize(nativeHandle_, compactionReadaheadSize); + return this; + } + + @Override + public long compactionReadaheadSize() { + assert(isOwningHandle()); + return compactionReadaheadSize(nativeHandle_); + } + + @Override + public Options setRandomAccessMaxBufferSize(final long randomAccessMaxBufferSize) { + assert(isOwningHandle()); + setRandomAccessMaxBufferSize(nativeHandle_, randomAccessMaxBufferSize); + return this; + } + + @Override + public long randomAccessMaxBufferSize() { + assert(isOwningHandle()); + return randomAccessMaxBufferSize(nativeHandle_); + } + + @Override + public Options setWritableFileMaxBufferSize(final long writableFileMaxBufferSize) { + assert(isOwningHandle()); + setWritableFileMaxBufferSize(nativeHandle_, writableFileMaxBufferSize); + return this; + } + + @Override + public long writableFileMaxBufferSize() { + assert(isOwningHandle()); + return writableFileMaxBufferSize(nativeHandle_); + } + + @Override + public boolean useAdaptiveMutex() { + assert(isOwningHandle()); + return useAdaptiveMutex(nativeHandle_); + } + + @Override + public Options setUseAdaptiveMutex(final boolean useAdaptiveMutex) { + assert(isOwningHandle()); + setUseAdaptiveMutex(nativeHandle_, useAdaptiveMutex); + return this; + } + + @Override + public long bytesPerSync() { + return bytesPerSync(nativeHandle_); + } + + @Override + public Options setBytesPerSync(final long bytesPerSync) { + assert(isOwningHandle()); + setBytesPerSync(nativeHandle_, bytesPerSync); + return this; + } + + @Override + public Options setWalBytesPerSync(final long walBytesPerSync) { + assert(isOwningHandle()); + setWalBytesPerSync(nativeHandle_, walBytesPerSync); + return this; + } + + @Override + public long walBytesPerSync() { + assert(isOwningHandle()); + return walBytesPerSync(nativeHandle_); + } + + @Override + public Options setStrictBytesPerSync(final boolean strictBytesPerSync) { + assert(isOwningHandle()); + setStrictBytesPerSync(nativeHandle_, strictBytesPerSync); + return this; + } + + @Override + public boolean strictBytesPerSync() { + assert(isOwningHandle()); + return strictBytesPerSync(nativeHandle_); + } + + @Override + public Options setListeners(final List listeners) { + assert (isOwningHandle()); + setEventListeners(nativeHandle_, RocksCallbackObject.toNativeHandleList(listeners)); + return this; + } + + @Override + public List listeners() { + assert (isOwningHandle()); + return Arrays.asList(eventListeners(nativeHandle_)); + } + + @Override + public Options setEnableThreadTracking(final boolean enableThreadTracking) { + assert(isOwningHandle()); + setEnableThreadTracking(nativeHandle_, enableThreadTracking); + return this; + } + + @Override + public boolean enableThreadTracking() { + assert(isOwningHandle()); + return enableThreadTracking(nativeHandle_); + } + + @Override + public Options setDelayedWriteRate(final long delayedWriteRate) { + assert(isOwningHandle()); + setDelayedWriteRate(nativeHandle_, delayedWriteRate); + return this; + } + + @Override + public long delayedWriteRate(){ + return delayedWriteRate(nativeHandle_); + } + + @Override + public Options setEnablePipelinedWrite(final boolean enablePipelinedWrite) { + setEnablePipelinedWrite(nativeHandle_, enablePipelinedWrite); + return this; + } + + @Override + public boolean enablePipelinedWrite() { + return enablePipelinedWrite(nativeHandle_); + } + + @Override + public Options setUnorderedWrite(final boolean unorderedWrite) { + setUnorderedWrite(nativeHandle_, unorderedWrite); + return this; + } + + @Override + public boolean unorderedWrite() { + return unorderedWrite(nativeHandle_); + } + + @Override + public Options setAllowConcurrentMemtableWrite( + final boolean allowConcurrentMemtableWrite) { + setAllowConcurrentMemtableWrite(nativeHandle_, + allowConcurrentMemtableWrite); + return this; + } + + @Override + public boolean allowConcurrentMemtableWrite() { + return allowConcurrentMemtableWrite(nativeHandle_); + } + + @Override + public Options setEnableWriteThreadAdaptiveYield( + final boolean enableWriteThreadAdaptiveYield) { + setEnableWriteThreadAdaptiveYield(nativeHandle_, + enableWriteThreadAdaptiveYield); + return this; + } + + @Override + public boolean enableWriteThreadAdaptiveYield() { + return enableWriteThreadAdaptiveYield(nativeHandle_); + } + + @Override + public Options setWriteThreadMaxYieldUsec(final long writeThreadMaxYieldUsec) { + setWriteThreadMaxYieldUsec(nativeHandle_, writeThreadMaxYieldUsec); + return this; + } + + @Override + public long writeThreadMaxYieldUsec() { + return writeThreadMaxYieldUsec(nativeHandle_); + } + + @Override + public Options setWriteThreadSlowYieldUsec(final long writeThreadSlowYieldUsec) { + setWriteThreadSlowYieldUsec(nativeHandle_, writeThreadSlowYieldUsec); + return this; + } + + @Override + public long writeThreadSlowYieldUsec() { + return writeThreadSlowYieldUsec(nativeHandle_); + } + + @Override + public Options setSkipStatsUpdateOnDbOpen(final boolean skipStatsUpdateOnDbOpen) { + assert(isOwningHandle()); + setSkipStatsUpdateOnDbOpen(nativeHandle_, skipStatsUpdateOnDbOpen); + return this; + } + + @Override + public boolean skipStatsUpdateOnDbOpen() { + assert(isOwningHandle()); + return skipStatsUpdateOnDbOpen(nativeHandle_); + } + + @Override + public Options setSkipCheckingSstFileSizesOnDbOpen( + final boolean skipCheckingSstFileSizesOnDbOpen) { + setSkipCheckingSstFileSizesOnDbOpen(nativeHandle_, skipCheckingSstFileSizesOnDbOpen); + return this; + } + + @Override + public boolean skipCheckingSstFileSizesOnDbOpen() { + assert (isOwningHandle()); + return skipCheckingSstFileSizesOnDbOpen(nativeHandle_); + } + + @Override + public Options setWalRecoveryMode(final WALRecoveryMode walRecoveryMode) { + assert(isOwningHandle()); + setWalRecoveryMode(nativeHandle_, walRecoveryMode.getValue()); + return this; + } + + @Override + public WALRecoveryMode walRecoveryMode() { + assert(isOwningHandle()); + return WALRecoveryMode.getWALRecoveryMode(walRecoveryMode(nativeHandle_)); + } + + @Override + public Options setAllow2pc(final boolean allow2pc) { + assert(isOwningHandle()); + setAllow2pc(nativeHandle_, allow2pc); + return this; + } + + @Override + public boolean allow2pc() { + assert(isOwningHandle()); + return allow2pc(nativeHandle_); + } + + @Override + public Options setRowCache(final Cache rowCache) { + assert(isOwningHandle()); + setRowCache(nativeHandle_, rowCache.nativeHandle_); + this.rowCache_ = rowCache; + return this; + } + + @Override + public Cache rowCache() { + assert(isOwningHandle()); + return this.rowCache_; + } + + @Override + public Options setWalFilter(final AbstractWalFilter walFilter) { + assert(isOwningHandle()); + setWalFilter(nativeHandle_, walFilter.nativeHandle_); + this.walFilter_ = walFilter; + return this; + } + + @Override + public WalFilter walFilter() { + assert(isOwningHandle()); + return this.walFilter_; + } + + @Override + public Options setFailIfOptionsFileError(final boolean failIfOptionsFileError) { + assert(isOwningHandle()); + setFailIfOptionsFileError(nativeHandle_, failIfOptionsFileError); + return this; + } + + @Override + public boolean failIfOptionsFileError() { + assert(isOwningHandle()); + return failIfOptionsFileError(nativeHandle_); + } + + @Override + public Options setDumpMallocStats(final boolean dumpMallocStats) { + assert(isOwningHandle()); + setDumpMallocStats(nativeHandle_, dumpMallocStats); + return this; + } + + @Override + public boolean dumpMallocStats() { + assert(isOwningHandle()); + return dumpMallocStats(nativeHandle_); + } + + @Override + public Options setAvoidFlushDuringRecovery(final boolean avoidFlushDuringRecovery) { + assert(isOwningHandle()); + setAvoidFlushDuringRecovery(nativeHandle_, avoidFlushDuringRecovery); + return this; + } + + @Override + public boolean avoidFlushDuringRecovery() { + assert(isOwningHandle()); + return avoidFlushDuringRecovery(nativeHandle_); + } + + @Override + public Options setAvoidFlushDuringShutdown(final boolean avoidFlushDuringShutdown) { + assert(isOwningHandle()); + setAvoidFlushDuringShutdown(nativeHandle_, avoidFlushDuringShutdown); + return this; + } + + @Override + public boolean avoidFlushDuringShutdown() { + assert(isOwningHandle()); + return avoidFlushDuringShutdown(nativeHandle_); + } + + @Override + public Options setAllowIngestBehind(final boolean allowIngestBehind) { + assert(isOwningHandle()); + setAllowIngestBehind(nativeHandle_, allowIngestBehind); + return this; + } + + @Override + public boolean allowIngestBehind() { + assert(isOwningHandle()); + return allowIngestBehind(nativeHandle_); + } + + @Override + public Options setTwoWriteQueues(final boolean twoWriteQueues) { + assert(isOwningHandle()); + setTwoWriteQueues(nativeHandle_, twoWriteQueues); + return this; + } + + @Override + public boolean twoWriteQueues() { + assert(isOwningHandle()); + return twoWriteQueues(nativeHandle_); + } + + @Override + public Options setManualWalFlush(final boolean manualWalFlush) { + assert(isOwningHandle()); + setManualWalFlush(nativeHandle_, manualWalFlush); + return this; + } + + @Override + public boolean manualWalFlush() { + assert(isOwningHandle()); + return manualWalFlush(nativeHandle_); + } + + @Override + public MemTableConfig memTableConfig() { + return this.memTableConfig_; + } + + @Override + public Options setMemTableConfig(final MemTableConfig config) { + memTableConfig_ = config; + setMemTableFactory(nativeHandle_, config.newMemTableFactoryHandle()); + return this; + } + + @Override + public Options setRateLimiter(final RateLimiter rateLimiter) { + assert(isOwningHandle()); + rateLimiter_ = rateLimiter; + setRateLimiter(nativeHandle_, rateLimiter.nativeHandle_); + return this; + } + + @Override + public Options setSstFileManager(final SstFileManager sstFileManager) { + assert(isOwningHandle()); + setSstFileManager(nativeHandle_, sstFileManager.nativeHandle_); + return this; + } + + @Override + public Options setLogger(final Logger logger) { + assert(isOwningHandle()); + setLogger(nativeHandle_, logger.nativeHandle_); + return this; + } + + @Override + public Options setInfoLogLevel(final InfoLogLevel infoLogLevel) { + assert(isOwningHandle()); + setInfoLogLevel(nativeHandle_, infoLogLevel.getValue()); + return this; + } + + @Override + public InfoLogLevel infoLogLevel() { + assert(isOwningHandle()); + return InfoLogLevel.getInfoLogLevel( + infoLogLevel(nativeHandle_)); + } + + @Override + public String memTableFactoryName() { + assert(isOwningHandle()); + return memTableFactoryName(nativeHandle_); + } + + @Override + public TableFormatConfig tableFormatConfig() { + return this.tableFormatConfig_; + } + + @Override + public Options setTableFormatConfig(final TableFormatConfig config) { + tableFormatConfig_ = config; + setTableFactory(nativeHandle_, config.newTableFactoryHandle()); + return this; + } + + @Override + public String tableFactoryName() { + assert(isOwningHandle()); + return tableFactoryName(nativeHandle_); + } + + @Override + public Options setCfPaths(final Collection cfPaths) { + assert (isOwningHandle()); + + final int len = cfPaths.size(); + final String[] paths = new String[len]; + final long[] targetSizes = new long[len]; + + int i = 0; + for (final DbPath dbPath : cfPaths) { + paths[i] = dbPath.path.toString(); + targetSizes[i] = dbPath.targetSize; + i++; + } + setCfPaths(nativeHandle_, paths, targetSizes); + return this; + } + + @Override + public List cfPaths() { + final int len = (int) cfPathsLen(nativeHandle_); + + if (len == 0) { + return Collections.emptyList(); + } + + final String[] paths = new String[len]; + final long[] targetSizes = new long[len]; + + cfPaths(nativeHandle_, paths, targetSizes); + + final List cfPaths = new ArrayList<>(); + for (int i = 0; i < len; i++) { + cfPaths.add(new DbPath(Paths.get(paths[i]), targetSizes[i])); + } + + return cfPaths; + } + + @Override + public Options useFixedLengthPrefixExtractor(final int n) { + assert(isOwningHandle()); + useFixedLengthPrefixExtractor(nativeHandle_, n); + return this; + } + + @Override + public Options useCappedPrefixExtractor(final int n) { + assert(isOwningHandle()); + useCappedPrefixExtractor(nativeHandle_, n); + return this; + } + + @Override + public CompressionType compressionType() { + return CompressionType.getCompressionType(compressionType(nativeHandle_)); + } + + @Override + public Options setCompressionPerLevel( + final List compressionLevels) { + final byte[] byteCompressionTypes = new byte[ + compressionLevels.size()]; + for (int i = 0; i < compressionLevels.size(); i++) { + byteCompressionTypes[i] = compressionLevels.get(i).getValue(); + } + setCompressionPerLevel(nativeHandle_, byteCompressionTypes); + return this; + } + + @Override + public List compressionPerLevel() { + final byte[] byteCompressionTypes = + compressionPerLevel(nativeHandle_); + final List compressionLevels = new ArrayList<>(); + for (final byte byteCompressionType : byteCompressionTypes) { + compressionLevels.add(CompressionType.getCompressionType( + byteCompressionType)); + } + return compressionLevels; + } + + @Override + public Options setCompressionType(final CompressionType compressionType) { + setCompressionType(nativeHandle_, compressionType.getValue()); + return this; + } + + @Override + public Options setBottommostCompressionType( + final CompressionType bottommostCompressionType) { + setBottommostCompressionType(nativeHandle_, + bottommostCompressionType.getValue()); + return this; + } + + @Override + public CompressionType bottommostCompressionType() { + return CompressionType.getCompressionType( + bottommostCompressionType(nativeHandle_)); + } + + @Override + public Options setBottommostCompressionOptions( + final CompressionOptions bottommostCompressionOptions) { + setBottommostCompressionOptions(nativeHandle_, + bottommostCompressionOptions.nativeHandle_); + this.bottommostCompressionOptions_ = bottommostCompressionOptions; + return this; + } + + @Override + public CompressionOptions bottommostCompressionOptions() { + return this.bottommostCompressionOptions_; + } + + @Override + public Options setCompressionOptions( + final CompressionOptions compressionOptions) { + setCompressionOptions(nativeHandle_, compressionOptions.nativeHandle_); + this.compressionOptions_ = compressionOptions; + return this; + } + + @Override + public CompressionOptions compressionOptions() { + return this.compressionOptions_; + } + + @Override + public CompactionStyle compactionStyle() { + return CompactionStyle.fromValue(compactionStyle(nativeHandle_)); + } + + @Override + public Options setCompactionStyle( + final CompactionStyle compactionStyle) { + setCompactionStyle(nativeHandle_, compactionStyle.getValue()); + return this; + } + + @Override + public int numLevels() { + return numLevels(nativeHandle_); + } + + @Override + public Options setNumLevels(final int numLevels) { + setNumLevels(nativeHandle_, numLevels); + return this; + } + + @Override + public int levelZeroFileNumCompactionTrigger() { + return levelZeroFileNumCompactionTrigger(nativeHandle_); + } + + @Override + public Options setLevelZeroFileNumCompactionTrigger( + final int numFiles) { + setLevelZeroFileNumCompactionTrigger( + nativeHandle_, numFiles); + return this; + } + + @Override + public int levelZeroSlowdownWritesTrigger() { + return levelZeroSlowdownWritesTrigger(nativeHandle_); + } + + @Override + public Options setLevelZeroSlowdownWritesTrigger( + final int numFiles) { + setLevelZeroSlowdownWritesTrigger(nativeHandle_, numFiles); + return this; + } + + @Override + public int levelZeroStopWritesTrigger() { + return levelZeroStopWritesTrigger(nativeHandle_); + } + + @Override + public Options setLevelZeroStopWritesTrigger( + final int numFiles) { + setLevelZeroStopWritesTrigger(nativeHandle_, numFiles); + return this; + } + + @Override + public long targetFileSizeBase() { + return targetFileSizeBase(nativeHandle_); + } + + @Override + public Options setTargetFileSizeBase(final long targetFileSizeBase) { + setTargetFileSizeBase(nativeHandle_, targetFileSizeBase); + return this; + } + + @Override + public int targetFileSizeMultiplier() { + return targetFileSizeMultiplier(nativeHandle_); + } + + @Override + public Options setTargetFileSizeMultiplier(final int multiplier) { + setTargetFileSizeMultiplier(nativeHandle_, multiplier); + return this; + } + + @Override + public Options setMaxBytesForLevelBase(final long maxBytesForLevelBase) { + setMaxBytesForLevelBase(nativeHandle_, maxBytesForLevelBase); + return this; + } + + @Override + public long maxBytesForLevelBase() { + return maxBytesForLevelBase(nativeHandle_); + } + + @Override + public Options setLevelCompactionDynamicLevelBytes( + final boolean enableLevelCompactionDynamicLevelBytes) { + setLevelCompactionDynamicLevelBytes(nativeHandle_, + enableLevelCompactionDynamicLevelBytes); + return this; + } + + @Override + public boolean levelCompactionDynamicLevelBytes() { + return levelCompactionDynamicLevelBytes(nativeHandle_); + } + + @Override + public double maxBytesForLevelMultiplier() { + return maxBytesForLevelMultiplier(nativeHandle_); + } + + @Override + public Options setMaxBytesForLevelMultiplier(final double multiplier) { + setMaxBytesForLevelMultiplier(nativeHandle_, multiplier); + return this; + } + + @Override + public long maxCompactionBytes() { + return maxCompactionBytes(nativeHandle_); + } + + @Override + public Options setMaxCompactionBytes(final long maxCompactionBytes) { + setMaxCompactionBytes(nativeHandle_, maxCompactionBytes); + return this; + } + + @Override + public long arenaBlockSize() { + return arenaBlockSize(nativeHandle_); + } + + @Override + public Options setArenaBlockSize(final long arenaBlockSize) { + setArenaBlockSize(nativeHandle_, arenaBlockSize); + return this; + } + + @Override + public boolean disableAutoCompactions() { + return disableAutoCompactions(nativeHandle_); + } + + @Override + public Options setDisableAutoCompactions( + final boolean disableAutoCompactions) { + setDisableAutoCompactions(nativeHandle_, disableAutoCompactions); + return this; + } + + @Override + public long maxSequentialSkipInIterations() { + return maxSequentialSkipInIterations(nativeHandle_); + } + + @Override + public Options setMaxSequentialSkipInIterations( + final long maxSequentialSkipInIterations) { + setMaxSequentialSkipInIterations(nativeHandle_, + maxSequentialSkipInIterations); + return this; + } + + @Override + public boolean inplaceUpdateSupport() { + return inplaceUpdateSupport(nativeHandle_); + } + + @Override + public Options setInplaceUpdateSupport( + final boolean inplaceUpdateSupport) { + setInplaceUpdateSupport(nativeHandle_, inplaceUpdateSupport); + return this; + } + + @Override + public long inplaceUpdateNumLocks() { + return inplaceUpdateNumLocks(nativeHandle_); + } + + @Override + public Options setInplaceUpdateNumLocks( + final long inplaceUpdateNumLocks) { + setInplaceUpdateNumLocks(nativeHandle_, inplaceUpdateNumLocks); + return this; + } + + @Override + public double memtablePrefixBloomSizeRatio() { + return memtablePrefixBloomSizeRatio(nativeHandle_); + } + + @Override + public Options setMemtablePrefixBloomSizeRatio(final double memtablePrefixBloomSizeRatio) { + setMemtablePrefixBloomSizeRatio(nativeHandle_, memtablePrefixBloomSizeRatio); + return this; + } + + @Override + public double experimentalMempurgeThreshold() { + return experimentalMempurgeThreshold(nativeHandle_); + } + + @Override + public Options setExperimentalMempurgeThreshold(final double experimentalMempurgeThreshold) { + setExperimentalMempurgeThreshold(nativeHandle_, experimentalMempurgeThreshold); + return this; + } + + @Override + public boolean memtableWholeKeyFiltering() { + return memtableWholeKeyFiltering(nativeHandle_); + } + + @Override + public Options setMemtableWholeKeyFiltering(final boolean memtableWholeKeyFiltering) { + setMemtableWholeKeyFiltering(nativeHandle_, memtableWholeKeyFiltering); + return this; + } + + @Override + public int bloomLocality() { + return bloomLocality(nativeHandle_); + } + + @Override + public Options setBloomLocality(final int bloomLocality) { + setBloomLocality(nativeHandle_, bloomLocality); + return this; + } + + @Override + public long maxSuccessiveMerges() { + return maxSuccessiveMerges(nativeHandle_); + } + + @Override + public Options setMaxSuccessiveMerges(final long maxSuccessiveMerges) { + setMaxSuccessiveMerges(nativeHandle_, maxSuccessiveMerges); + return this; + } + + @Override + public int minWriteBufferNumberToMerge() { + return minWriteBufferNumberToMerge(nativeHandle_); + } + + @Override + public Options setMinWriteBufferNumberToMerge( + final int minWriteBufferNumberToMerge) { + setMinWriteBufferNumberToMerge(nativeHandle_, minWriteBufferNumberToMerge); + return this; + } + + @Override + public Options setOptimizeFiltersForHits( + final boolean optimizeFiltersForHits) { + setOptimizeFiltersForHits(nativeHandle_, optimizeFiltersForHits); + return this; + } + + @Override + public boolean optimizeFiltersForHits() { + return optimizeFiltersForHits(nativeHandle_); + } + + @Override + public Options setMemtableHugePageSize(final long memtableHugePageSize) { + setMemtableHugePageSize(nativeHandle_, + memtableHugePageSize); + return this; + } + + @Override + public long memtableHugePageSize() { + return memtableHugePageSize(nativeHandle_); + } + + @Override + public Options setSoftPendingCompactionBytesLimit(final long softPendingCompactionBytesLimit) { + setSoftPendingCompactionBytesLimit(nativeHandle_, + softPendingCompactionBytesLimit); + return this; + } + + @Override + public long softPendingCompactionBytesLimit() { + return softPendingCompactionBytesLimit(nativeHandle_); + } + + @Override + public Options setHardPendingCompactionBytesLimit(final long hardPendingCompactionBytesLimit) { + setHardPendingCompactionBytesLimit(nativeHandle_, hardPendingCompactionBytesLimit); + return this; + } + + @Override + public long hardPendingCompactionBytesLimit() { + return hardPendingCompactionBytesLimit(nativeHandle_); + } + + @Override + public Options setLevel0FileNumCompactionTrigger(final int level0FileNumCompactionTrigger) { + setLevel0FileNumCompactionTrigger(nativeHandle_, level0FileNumCompactionTrigger); + return this; + } + + @Override + public int level0FileNumCompactionTrigger() { + return level0FileNumCompactionTrigger(nativeHandle_); + } + + @Override + public Options setLevel0SlowdownWritesTrigger(final int level0SlowdownWritesTrigger) { + setLevel0SlowdownWritesTrigger(nativeHandle_, level0SlowdownWritesTrigger); + return this; + } + + @Override + public int level0SlowdownWritesTrigger() { + return level0SlowdownWritesTrigger(nativeHandle_); + } + + @Override + public Options setLevel0StopWritesTrigger(final int level0StopWritesTrigger) { + setLevel0StopWritesTrigger(nativeHandle_, level0StopWritesTrigger); + return this; + } + + @Override + public int level0StopWritesTrigger() { + return level0StopWritesTrigger(nativeHandle_); + } + + @Override + public Options setMaxBytesForLevelMultiplierAdditional( + final int[] maxBytesForLevelMultiplierAdditional) { + setMaxBytesForLevelMultiplierAdditional(nativeHandle_, maxBytesForLevelMultiplierAdditional); + return this; + } + + @Override + public int[] maxBytesForLevelMultiplierAdditional() { + return maxBytesForLevelMultiplierAdditional(nativeHandle_); + } + + @Override + public Options setParanoidFileChecks(final boolean paranoidFileChecks) { + setParanoidFileChecks(nativeHandle_, paranoidFileChecks); + return this; + } + + @Override + public boolean paranoidFileChecks() { + return paranoidFileChecks(nativeHandle_); + } + + @Override + public Options setMaxWriteBufferNumberToMaintain( + final int maxWriteBufferNumberToMaintain) { + setMaxWriteBufferNumberToMaintain( + nativeHandle_, maxWriteBufferNumberToMaintain); + return this; + } + + @Override + public int maxWriteBufferNumberToMaintain() { + return maxWriteBufferNumberToMaintain(nativeHandle_); + } + + @Override + public Options setCompactionPriority( + final CompactionPriority compactionPriority) { + setCompactionPriority(nativeHandle_, compactionPriority.getValue()); + return this; + } + + @Override + public CompactionPriority compactionPriority() { + return CompactionPriority.getCompactionPriority( + compactionPriority(nativeHandle_)); + } + + @Override + public Options setReportBgIoStats(final boolean reportBgIoStats) { + setReportBgIoStats(nativeHandle_, reportBgIoStats); + return this; + } + + @Override + public boolean reportBgIoStats() { + return reportBgIoStats(nativeHandle_); + } + + @Override + public Options setTtl(final long ttl) { + setTtl(nativeHandle_, ttl); + return this; + } + + @Override + public long ttl() { + return ttl(nativeHandle_); + } + + @Override + public Options setPeriodicCompactionSeconds(final long periodicCompactionSeconds) { + setPeriodicCompactionSeconds(nativeHandle_, periodicCompactionSeconds); + return this; + } + + @Override + public long periodicCompactionSeconds() { + return periodicCompactionSeconds(nativeHandle_); + } + + @Override + public Options setCompactionOptionsUniversal( + final CompactionOptionsUniversal compactionOptionsUniversal) { + setCompactionOptionsUniversal(nativeHandle_, + compactionOptionsUniversal.nativeHandle_); + this.compactionOptionsUniversal_ = compactionOptionsUniversal; + return this; + } + + @Override + public CompactionOptionsUniversal compactionOptionsUniversal() { + return this.compactionOptionsUniversal_; + } + + @Override + public Options setCompactionOptionsFIFO(final CompactionOptionsFIFO compactionOptionsFIFO) { + setCompactionOptionsFIFO(nativeHandle_, + compactionOptionsFIFO.nativeHandle_); + this.compactionOptionsFIFO_ = compactionOptionsFIFO; + return this; + } + + @Override + public CompactionOptionsFIFO compactionOptionsFIFO() { + return this.compactionOptionsFIFO_; + } + + @Override + public Options setForceConsistencyChecks(final boolean forceConsistencyChecks) { + setForceConsistencyChecks(nativeHandle_, forceConsistencyChecks); + return this; + } + + @Override + public boolean forceConsistencyChecks() { + return forceConsistencyChecks(nativeHandle_); + } + + @Override + public Options setAtomicFlush(final boolean atomicFlush) { + setAtomicFlush(nativeHandle_, atomicFlush); + return this; + } + + @Override + public boolean atomicFlush() { + return atomicFlush(nativeHandle_); + } + + @Override + public Options setAvoidUnnecessaryBlockingIO(final boolean avoidUnnecessaryBlockingIO) { + setAvoidUnnecessaryBlockingIO(nativeHandle_, avoidUnnecessaryBlockingIO); + return this; + } + + @Override + public boolean avoidUnnecessaryBlockingIO() { + assert (isOwningHandle()); + return avoidUnnecessaryBlockingIO(nativeHandle_); + } + + @Override + public Options setPersistStatsToDisk(final boolean persistStatsToDisk) { + setPersistStatsToDisk(nativeHandle_, persistStatsToDisk); + return this; + } + + @Override + public boolean persistStatsToDisk() { + assert (isOwningHandle()); + return persistStatsToDisk(nativeHandle_); + } + + @Override + public Options setWriteDbidToManifest(final boolean writeDbidToManifest) { + setWriteDbidToManifest(nativeHandle_, writeDbidToManifest); + return this; + } + + @Override + public boolean writeDbidToManifest() { + assert (isOwningHandle()); + return writeDbidToManifest(nativeHandle_); + } + + @Override + public Options setLogReadaheadSize(final long logReadaheadSize) { + setLogReadaheadSize(nativeHandle_, logReadaheadSize); + return this; + } + + @Override + public long logReadaheadSize() { + assert (isOwningHandle()); + return logReadaheadSize(nativeHandle_); + } + + @Override + public Options setBestEffortsRecovery(final boolean bestEffortsRecovery) { + setBestEffortsRecovery(nativeHandle_, bestEffortsRecovery); + return this; + } + + @Override + public boolean bestEffortsRecovery() { + assert (isOwningHandle()); + return bestEffortsRecovery(nativeHandle_); + } + + @Override + public Options setMaxBgErrorResumeCount(final int maxBgerrorResumeCount) { + setMaxBgErrorResumeCount(nativeHandle_, maxBgerrorResumeCount); + return this; + } + + @Override + public int maxBgerrorResumeCount() { + assert (isOwningHandle()); + return maxBgerrorResumeCount(nativeHandle_); + } + + @Override + public Options setBgerrorResumeRetryInterval(final long bgerrorResumeRetryInterval) { + setBgerrorResumeRetryInterval(nativeHandle_, bgerrorResumeRetryInterval); + return this; + } + + @Override + public long bgerrorResumeRetryInterval() { + assert (isOwningHandle()); + return bgerrorResumeRetryInterval(nativeHandle_); + } + + @Override + public Options setSstPartitionerFactory(final SstPartitionerFactory sstPartitionerFactory) { + setSstPartitionerFactory(nativeHandle_, sstPartitionerFactory.nativeHandle_); + this.sstPartitionerFactory_ = sstPartitionerFactory; + return this; + } + + @Override + public SstPartitionerFactory sstPartitionerFactory() { + return sstPartitionerFactory_; + } + + @Override + public Options setMemtableMaxRangeDeletions(final int count) { + setMemtableMaxRangeDeletions(nativeHandle_, count); + return this; + } + + @Override + public int memtableMaxRangeDeletions() { + return memtableMaxRangeDeletions(nativeHandle_); + } + + @Override + public Options setCompactionThreadLimiter(final ConcurrentTaskLimiter compactionThreadLimiter) { + setCompactionThreadLimiter(nativeHandle_, compactionThreadLimiter.nativeHandle_); + this.compactionThreadLimiter_ = compactionThreadLimiter; + return this; + } + + @Override + public ConcurrentTaskLimiter compactionThreadLimiter() { + assert (isOwningHandle()); + return this.compactionThreadLimiter_; + } + + // + // BEGIN options for blobs (integrated BlobDB) + // + + @Override + public Options setEnableBlobFiles(final boolean enableBlobFiles) { + setEnableBlobFiles(nativeHandle_, enableBlobFiles); + return this; + } + + @Override + public boolean enableBlobFiles() { + return enableBlobFiles(nativeHandle_); + } + + @Override + public Options setMinBlobSize(final long minBlobSize) { + setMinBlobSize(nativeHandle_, minBlobSize); + return this; + } + + @Override + public long minBlobSize() { + return minBlobSize(nativeHandle_); + } + + @Override + public Options setBlobFileSize(final long blobFileSize) { + setBlobFileSize(nativeHandle_, blobFileSize); + return this; + } + + @Override + public long blobFileSize() { + return blobFileSize(nativeHandle_); + } + + @Override + public Options setBlobCompressionType(final CompressionType compressionType) { + setBlobCompressionType(nativeHandle_, compressionType.getValue()); + return this; + } + + @Override + public CompressionType blobCompressionType() { + return CompressionType.values()[blobCompressionType(nativeHandle_)]; + } + + @Override + public Options setEnableBlobGarbageCollection(final boolean enableBlobGarbageCollection) { + setEnableBlobGarbageCollection(nativeHandle_, enableBlobGarbageCollection); + return this; + } + + @Override + public boolean enableBlobGarbageCollection() { + return enableBlobGarbageCollection(nativeHandle_); + } + + @Override + public Options setBlobGarbageCollectionAgeCutoff(final double blobGarbageCollectionAgeCutoff) { + setBlobGarbageCollectionAgeCutoff(nativeHandle_, blobGarbageCollectionAgeCutoff); + return this; + } + + @Override + public double blobGarbageCollectionAgeCutoff() { + return blobGarbageCollectionAgeCutoff(nativeHandle_); + } + + @Override + public Options setBlobGarbageCollectionForceThreshold( + final double blobGarbageCollectionForceThreshold) { + setBlobGarbageCollectionForceThreshold(nativeHandle_, blobGarbageCollectionForceThreshold); + return this; + } + + @Override + public double blobGarbageCollectionForceThreshold() { + return blobGarbageCollectionForceThreshold(nativeHandle_); + } + + @Override + public Options setBlobCompactionReadaheadSize(final long blobCompactionReadaheadSize) { + setBlobCompactionReadaheadSize(nativeHandle_, blobCompactionReadaheadSize); + return this; + } + + @Override + public long blobCompactionReadaheadSize() { + return blobCompactionReadaheadSize(nativeHandle_); + } + + @Override + public Options setBlobFileStartingLevel(final int blobFileStartingLevel) { + setBlobFileStartingLevel(nativeHandle_, blobFileStartingLevel); + return this; + } + + @Override + public int blobFileStartingLevel() { + return blobFileStartingLevel(nativeHandle_); + } + + @Override + public Options setPrepopulateBlobCache(final PrepopulateBlobCache prepopulateBlobCache) { + setPrepopulateBlobCache(nativeHandle_, prepopulateBlobCache.getValue()); + return this; + } + + @Override + public PrepopulateBlobCache prepopulateBlobCache() { + return PrepopulateBlobCache.getPrepopulateBlobCache(prepopulateBlobCache(nativeHandle_)); + } + + // + // END options for blobs (integrated BlobDB) + // + + private static native long newOptions(); + private static native long newOptions(long dbOptHandle, long cfOptHandle); + private static native long copyOptions(long handle); + @Override protected final native void disposeInternal(final long handle); + private native void setEnv(long optHandle, long envHandle); + private native void prepareForBulkLoad(long handle); + + // DB native handles + private native void setIncreaseParallelism(long handle, int totalThreads); + private native void setCreateIfMissing(long handle, boolean flag); + private native boolean createIfMissing(long handle); + private native void setCreateMissingColumnFamilies( + long handle, boolean flag); + private native boolean createMissingColumnFamilies(long handle); + private native void setErrorIfExists(long handle, boolean errorIfExists); + private native boolean errorIfExists(long handle); + private native void setParanoidChecks( + long handle, boolean paranoidChecks); + private native boolean paranoidChecks(long handle); + private native void setRateLimiter(long handle, + long rateLimiterHandle); + private native void setSstFileManager(final long handle, + final long sstFileManagerHandle); + private native void setLogger(long handle, + long loggerHandle); + private native void setInfoLogLevel(long handle, byte logLevel); + private native byte infoLogLevel(long handle); + private native void setMaxOpenFiles(long handle, int maxOpenFiles); + private native int maxOpenFiles(long handle); + private native void setMaxTotalWalSize(long handle, + long maxTotalWalSize); + private native void setMaxFileOpeningThreads(final long handle, + final int maxFileOpeningThreads); + private native int maxFileOpeningThreads(final long handle); + private native long maxTotalWalSize(long handle); + private native void setStatistics(final long handle, final long statisticsHandle); + private native long statistics(final long handle); + private native boolean useFsync(long handle); + private native void setUseFsync(long handle, boolean useFsync); + private native void setDbPaths(final long handle, final String[] paths, + final long[] targetSizes); + private native long dbPathsLen(final long handle); + private native void dbPaths(final long handle, final String[] paths, + final long[] targetSizes); + private native void setDbLogDir(long handle, String dbLogDir); + private native String dbLogDir(long handle); + private native void setWalDir(long handle, String walDir); + private native String walDir(long handle); + private native void setDeleteObsoleteFilesPeriodMicros( + long handle, long micros); + private native long deleteObsoleteFilesPeriodMicros(long handle); + private native void setMaxBackgroundCompactions( + long handle, int maxBackgroundCompactions); + private native int maxBackgroundCompactions(long handle); + private native void setMaxSubcompactions(long handle, int maxSubcompactions); + private native int maxSubcompactions(long handle); + private native void setMaxBackgroundFlushes( + long handle, int maxBackgroundFlushes); + private native int maxBackgroundFlushes(long handle); + private native void setMaxBackgroundJobs(long handle, int maxMaxBackgroundJobs); + private native int maxBackgroundJobs(long handle); + private native void setMaxLogFileSize(long handle, long maxLogFileSize) + throws IllegalArgumentException; + private native long maxLogFileSize(long handle); + private native void setLogFileTimeToRoll( + long handle, long logFileTimeToRoll) throws IllegalArgumentException; + private native long logFileTimeToRoll(long handle); + private native void setKeepLogFileNum(long handle, long keepLogFileNum) + throws IllegalArgumentException; + private native long keepLogFileNum(long handle); + private native void setRecycleLogFileNum(long handle, long recycleLogFileNum); + private native long recycleLogFileNum(long handle); + private native void setMaxManifestFileSize( + long handle, long maxManifestFileSize); + private native long maxManifestFileSize(long handle); + private native void setMaxTableFilesSizeFIFO( + long handle, long maxTableFilesSize); + private native long maxTableFilesSizeFIFO(long handle); + private native void setTableCacheNumshardbits( + long handle, int tableCacheNumshardbits); + private native int tableCacheNumshardbits(long handle); + private native void setWalTtlSeconds(long handle, long walTtlSeconds); + private native long walTtlSeconds(long handle); + private native void setWalSizeLimitMB(long handle, long sizeLimitMB); + private native long walSizeLimitMB(long handle); + private static native void setMaxWriteBatchGroupSizeBytes( + final long handle, final long maxWriteBatchGroupSizeBytes); + private static native long maxWriteBatchGroupSizeBytes(final long handle); + private native void setManifestPreallocationSize( + long handle, long size) throws IllegalArgumentException; + private native long manifestPreallocationSize(long handle); + private native void setUseDirectReads(long handle, boolean useDirectReads); + private native boolean useDirectReads(long handle); + private native void setUseDirectIoForFlushAndCompaction( + long handle, boolean useDirectIoForFlushAndCompaction); + private native boolean useDirectIoForFlushAndCompaction(long handle); + private native void setAllowFAllocate(final long handle, + final boolean allowFAllocate); + private native boolean allowFAllocate(final long handle); + private native void setAllowMmapReads( + long handle, boolean allowMmapReads); + private native boolean allowMmapReads(long handle); + private native void setAllowMmapWrites( + long handle, boolean allowMmapWrites); + private native boolean allowMmapWrites(long handle); + private native void setIsFdCloseOnExec( + long handle, boolean isFdCloseOnExec); + private native boolean isFdCloseOnExec(long handle); + private native void setStatsDumpPeriodSec( + long handle, int statsDumpPeriodSec); + private native int statsDumpPeriodSec(long handle); + private native void setStatsPersistPeriodSec( + final long handle, final int statsPersistPeriodSec); + private native int statsPersistPeriodSec( + final long handle); + private native void setStatsHistoryBufferSize( + final long handle, final long statsHistoryBufferSize); + private native long statsHistoryBufferSize( + final long handle); + private native void setAdviseRandomOnOpen( + long handle, boolean adviseRandomOnOpen); + private native boolean adviseRandomOnOpen(long handle); + private native void setDbWriteBufferSize(final long handle, + final long dbWriteBufferSize); + private native void setWriteBufferManager(final long handle, + final long writeBufferManagerHandle); + private native long dbWriteBufferSize(final long handle); + private native void setAccessHintOnCompactionStart(final long handle, + final byte accessHintOnCompactionStart); + private native byte accessHintOnCompactionStart(final long handle); + private native void setCompactionReadaheadSize(final long handle, + final long compactionReadaheadSize); + private native long compactionReadaheadSize(final long handle); + private native void setRandomAccessMaxBufferSize(final long handle, + final long randomAccessMaxBufferSize); + private native long randomAccessMaxBufferSize(final long handle); + private native void setWritableFileMaxBufferSize(final long handle, + final long writableFileMaxBufferSize); + private native long writableFileMaxBufferSize(final long handle); + private native void setUseAdaptiveMutex( + long handle, boolean useAdaptiveMutex); + private native boolean useAdaptiveMutex(long handle); + private native void setBytesPerSync( + long handle, long bytesPerSync); + private native long bytesPerSync(long handle); + private native void setWalBytesPerSync(long handle, long walBytesPerSync); + private native long walBytesPerSync(long handle); + private native void setStrictBytesPerSync( + final long handle, final boolean strictBytesPerSync); + private native boolean strictBytesPerSync( + final long handle); + private static native void setEventListeners( + final long handle, final long[] eventListenerHandles); + private static native AbstractEventListener[] eventListeners(final long handle); + private native void setEnableThreadTracking(long handle, + boolean enableThreadTracking); + private native boolean enableThreadTracking(long handle); + private native void setDelayedWriteRate(long handle, long delayedWriteRate); + private native long delayedWriteRate(long handle); + private native void setEnablePipelinedWrite(final long handle, + final boolean pipelinedWrite); + private native boolean enablePipelinedWrite(final long handle); + private native void setUnorderedWrite(final long handle, + final boolean unorderedWrite); + private native boolean unorderedWrite(final long handle); + private native void setAllowConcurrentMemtableWrite(long handle, + boolean allowConcurrentMemtableWrite); + private native boolean allowConcurrentMemtableWrite(long handle); + private native void setEnableWriteThreadAdaptiveYield(long handle, + boolean enableWriteThreadAdaptiveYield); + private native boolean enableWriteThreadAdaptiveYield(long handle); + private native void setWriteThreadMaxYieldUsec(long handle, + long writeThreadMaxYieldUsec); + private native long writeThreadMaxYieldUsec(long handle); + private native void setWriteThreadSlowYieldUsec(long handle, + long writeThreadSlowYieldUsec); + private native long writeThreadSlowYieldUsec(long handle); + private native void setSkipStatsUpdateOnDbOpen(final long handle, + final boolean skipStatsUpdateOnDbOpen); + private native boolean skipStatsUpdateOnDbOpen(final long handle); + private static native void setSkipCheckingSstFileSizesOnDbOpen( + final long handle, final boolean skipChecking); + private static native boolean skipCheckingSstFileSizesOnDbOpen(final long handle); + private native void setWalRecoveryMode(final long handle, + final byte walRecoveryMode); + private native byte walRecoveryMode(final long handle); + private native void setAllow2pc(final long handle, + final boolean allow2pc); + private native boolean allow2pc(final long handle); + private native void setRowCache(final long handle, + final long rowCacheHandle); + private native void setWalFilter(final long handle, + final long walFilterHandle); + private native void setFailIfOptionsFileError(final long handle, + final boolean failIfOptionsFileError); + private native boolean failIfOptionsFileError(final long handle); + private native void setDumpMallocStats(final long handle, + final boolean dumpMallocStats); + private native boolean dumpMallocStats(final long handle); + private native void setAvoidFlushDuringRecovery(final long handle, + final boolean avoidFlushDuringRecovery); + private native boolean avoidFlushDuringRecovery(final long handle); + private native void setAvoidFlushDuringShutdown(final long handle, + final boolean avoidFlushDuringShutdown); + private native boolean avoidFlushDuringShutdown(final long handle); + private native void setAllowIngestBehind(final long handle, + final boolean allowIngestBehind); + private native boolean allowIngestBehind(final long handle); + private native void setTwoWriteQueues(final long handle, + final boolean twoWriteQueues); + private native boolean twoWriteQueues(final long handle); + private native void setManualWalFlush(final long handle, + final boolean manualWalFlush); + private native boolean manualWalFlush(final long handle); + + + // CF native handles + private static native void oldDefaults( + final long handle, final int majorVersion, final int minorVersion); + private native void optimizeForSmallDb(final long handle); + private static native void optimizeForSmallDb(final long handle, final long cacheHandle); + private native void optimizeForPointLookup(long handle, + long blockCacheSizeMb); + private native void optimizeLevelStyleCompaction(long handle, + long memtableMemoryBudget); + private native void optimizeUniversalStyleCompaction(long handle, + long memtableMemoryBudget); + private native void setComparatorHandle(long handle, int builtinComparator); + private native void setComparatorHandle(long optHandle, + long comparatorHandle, byte comparatorType); + private native void setMergeOperatorName( + long handle, String name); + private native void setMergeOperator( + long handle, long mergeOperatorHandle); + private native void setCompactionFilterHandle( + long handle, long compactionFilterHandle); + private native void setCompactionFilterFactoryHandle( + long handle, long compactionFilterFactoryHandle); + private native void setWriteBufferSize(long handle, long writeBufferSize) + throws IllegalArgumentException; + private native long writeBufferSize(long handle); + private native void setMaxWriteBufferNumber( + long handle, int maxWriteBufferNumber); + private native int maxWriteBufferNumber(long handle); + private native void setMinWriteBufferNumberToMerge( + long handle, int minWriteBufferNumberToMerge); + private native int minWriteBufferNumberToMerge(long handle); + private native void setCompressionType(long handle, byte compressionType); + private native byte compressionType(long handle); + private native void setCompressionPerLevel(long handle, + byte[] compressionLevels); + private native byte[] compressionPerLevel(long handle); + private native void setBottommostCompressionType(long handle, + byte bottommostCompressionType); + private native byte bottommostCompressionType(long handle); + private native void setBottommostCompressionOptions(final long handle, + final long bottommostCompressionOptionsHandle); + private native void setCompressionOptions(long handle, + long compressionOptionsHandle); + private native void useFixedLengthPrefixExtractor( + long handle, int prefixLength); + private native void useCappedPrefixExtractor( + long handle, int prefixLength); + private native void setNumLevels( + long handle, int numLevels); + private native int numLevels(long handle); + private native void setLevelZeroFileNumCompactionTrigger( + long handle, int numFiles); + private native int levelZeroFileNumCompactionTrigger(long handle); + private native void setLevelZeroSlowdownWritesTrigger( + long handle, int numFiles); + private native int levelZeroSlowdownWritesTrigger(long handle); + private native void setLevelZeroStopWritesTrigger( + long handle, int numFiles); + private native int levelZeroStopWritesTrigger(long handle); + private native void setTargetFileSizeBase( + long handle, long targetFileSizeBase); + private native long targetFileSizeBase(long handle); + private native void setTargetFileSizeMultiplier( + long handle, int multiplier); + private native int targetFileSizeMultiplier(long handle); + private native void setMaxBytesForLevelBase( + long handle, long maxBytesForLevelBase); + private native long maxBytesForLevelBase(long handle); + private native void setLevelCompactionDynamicLevelBytes( + long handle, boolean enableLevelCompactionDynamicLevelBytes); + private native boolean levelCompactionDynamicLevelBytes( + long handle); + private native void setMaxBytesForLevelMultiplier(long handle, double multiplier); + private native double maxBytesForLevelMultiplier(long handle); + private native void setMaxCompactionBytes(long handle, long maxCompactionBytes); + private native long maxCompactionBytes(long handle); + private native void setArenaBlockSize( + long handle, long arenaBlockSize) throws IllegalArgumentException; + private native long arenaBlockSize(long handle); + private native void setDisableAutoCompactions( + long handle, boolean disableAutoCompactions); + private native boolean disableAutoCompactions(long handle); + private native void setCompactionStyle(long handle, byte compactionStyle); + private native byte compactionStyle(long handle); + private native void setMaxSequentialSkipInIterations( + long handle, long maxSequentialSkipInIterations); + private native long maxSequentialSkipInIterations(long handle); + private native void setMemTableFactory(long handle, long factoryHandle); + private native String memTableFactoryName(long handle); + private native void setTableFactory(long handle, long factoryHandle); + private native String tableFactoryName(long handle); + private static native void setCfPaths( + final long handle, final String[] paths, final long[] targetSizes); + private static native long cfPathsLen(final long handle); + private static native void cfPaths( + final long handle, final String[] paths, final long[] targetSizes); + private native void setInplaceUpdateSupport( + long handle, boolean inplaceUpdateSupport); + private native boolean inplaceUpdateSupport(long handle); + private native void setInplaceUpdateNumLocks( + long handle, long inplaceUpdateNumLocks) + throws IllegalArgumentException; + private native long inplaceUpdateNumLocks(long handle); + private native void setMemtablePrefixBloomSizeRatio( + long handle, double memtablePrefixBloomSizeRatio); + private native double memtablePrefixBloomSizeRatio(long handle); + private native void setExperimentalMempurgeThreshold( + long handle, double experimentalMempurgeThreshold); + private native double experimentalMempurgeThreshold(long handle); + private native void setMemtableWholeKeyFiltering(long handle, boolean memtableWholeKeyFiltering); + private native boolean memtableWholeKeyFiltering(long handle); + private native void setBloomLocality( + long handle, int bloomLocality); + private native int bloomLocality(long handle); + private native void setMaxSuccessiveMerges( + long handle, long maxSuccessiveMerges) + throws IllegalArgumentException; + private native long maxSuccessiveMerges(long handle); + private native void setOptimizeFiltersForHits(long handle, + boolean optimizeFiltersForHits); + private native boolean optimizeFiltersForHits(long handle); + private native void setMemtableHugePageSize(long handle, + long memtableHugePageSize); + private native long memtableHugePageSize(long handle); + private native void setSoftPendingCompactionBytesLimit(long handle, + long softPendingCompactionBytesLimit); + private native long softPendingCompactionBytesLimit(long handle); + private native void setHardPendingCompactionBytesLimit(long handle, + long hardPendingCompactionBytesLimit); + private native long hardPendingCompactionBytesLimit(long handle); + private native void setLevel0FileNumCompactionTrigger(long handle, + int level0FileNumCompactionTrigger); + private native int level0FileNumCompactionTrigger(long handle); + private native void setLevel0SlowdownWritesTrigger(long handle, + int level0SlowdownWritesTrigger); + private native int level0SlowdownWritesTrigger(long handle); + private native void setLevel0StopWritesTrigger(long handle, + int level0StopWritesTrigger); + private native int level0StopWritesTrigger(long handle); + private native void setMaxBytesForLevelMultiplierAdditional(long handle, + int[] maxBytesForLevelMultiplierAdditional); + private native int[] maxBytesForLevelMultiplierAdditional(long handle); + private native void setParanoidFileChecks(long handle, + boolean paranoidFileChecks); + private native boolean paranoidFileChecks(long handle); + private native void setMaxWriteBufferNumberToMaintain(final long handle, + final int maxWriteBufferNumberToMaintain); + private native int maxWriteBufferNumberToMaintain(final long handle); + private native void setCompactionPriority(final long handle, + final byte compactionPriority); + private native byte compactionPriority(final long handle); + private native void setReportBgIoStats(final long handle, + final boolean reportBgIoStats); + private native boolean reportBgIoStats(final long handle); + private native void setTtl(final long handle, final long ttl); + private native long ttl(final long handle); + private native void setPeriodicCompactionSeconds( + final long handle, final long periodicCompactionSeconds); + private native long periodicCompactionSeconds(final long handle); + private native void setCompactionOptionsUniversal(final long handle, + final long compactionOptionsUniversalHandle); + private native void setCompactionOptionsFIFO(final long handle, + final long compactionOptionsFIFOHandle); + private native void setForceConsistencyChecks(final long handle, + final boolean forceConsistencyChecks); + private native boolean forceConsistencyChecks(final long handle); + private native void setAtomicFlush(final long handle, + final boolean atomicFlush); + private native boolean atomicFlush(final long handle); + private native void setSstPartitionerFactory(long nativeHandle_, long newFactoryHandle); + private native void setMemtableMaxRangeDeletions(final long handle, final int count); + private native int memtableMaxRangeDeletions(final long handle); + private static native void setCompactionThreadLimiter( + final long nativeHandle_, final long newLimiterHandle); + private static native void setAvoidUnnecessaryBlockingIO( + final long handle, final boolean avoidBlockingIO); + private static native boolean avoidUnnecessaryBlockingIO(final long handle); + private static native void setPersistStatsToDisk( + final long handle, final boolean persistStatsToDisk); + private static native boolean persistStatsToDisk(final long handle); + private static native void setWriteDbidToManifest( + final long handle, final boolean writeDbidToManifest); + private static native boolean writeDbidToManifest(final long handle); + private static native void setLogReadaheadSize(final long handle, final long logReadaheadSize); + private static native long logReadaheadSize(final long handle); + private static native void setBestEffortsRecovery( + final long handle, final boolean bestEffortsRecovery); + private static native boolean bestEffortsRecovery(final long handle); + private static native void setMaxBgErrorResumeCount( + final long handle, final int maxBgerrorRecumeCount); + private static native int maxBgerrorResumeCount(final long handle); + private static native void setBgerrorResumeRetryInterval( + final long handle, final long bgerrorResumeRetryInterval); + private static native long bgerrorResumeRetryInterval(final long handle); + + private native void setEnableBlobFiles(final long nativeHandle_, final boolean enableBlobFiles); + private native boolean enableBlobFiles(final long nativeHandle_); + private native void setMinBlobSize(final long nativeHandle_, final long minBlobSize); + private native long minBlobSize(final long nativeHandle_); + private native void setBlobFileSize(final long nativeHandle_, final long blobFileSize); + private native long blobFileSize(final long nativeHandle_); + private native void setBlobCompressionType(final long nativeHandle_, final byte compressionType); + private native byte blobCompressionType(final long nativeHandle_); + private native void setEnableBlobGarbageCollection( + final long nativeHandle_, final boolean enableBlobGarbageCollection); + private native boolean enableBlobGarbageCollection(final long nativeHandle_); + private native void setBlobGarbageCollectionAgeCutoff( + final long nativeHandle_, final double blobGarbageCollectionAgeCutoff); + private native double blobGarbageCollectionAgeCutoff(final long nativeHandle_); + private native void setBlobGarbageCollectionForceThreshold( + final long nativeHandle_, final double blobGarbageCollectionForceThreshold); + private native double blobGarbageCollectionForceThreshold(final long nativeHandle_); + private native void setBlobCompactionReadaheadSize( + final long nativeHandle_, final long blobCompactionReadaheadSize); + private native long blobCompactionReadaheadSize(final long nativeHandle_); + private native void setBlobFileStartingLevel( + final long nativeHandle_, final int blobFileStartingLevel); + private native int blobFileStartingLevel(final long nativeHandle_); + private native void setPrepopulateBlobCache( + final long nativeHandle_, final byte prepopulateBlobCache); + private native byte prepopulateBlobCache(final long nativeHandle_); + + // instance variables + // NOTE: If you add new member variables, please update the copy constructor above! + private Env env_; + private MemTableConfig memTableConfig_; + private TableFormatConfig tableFormatConfig_; + private RateLimiter rateLimiter_; + private AbstractComparator comparator_; + private AbstractCompactionFilter> compactionFilter_; + private AbstractCompactionFilterFactory> + compactionFilterFactory_; + private CompactionOptionsUniversal compactionOptionsUniversal_; + private CompactionOptionsFIFO compactionOptionsFIFO_; + private CompressionOptions bottommostCompressionOptions_; + private CompressionOptions compressionOptions_; + private Cache rowCache_; + private WalFilter walFilter_; + private WriteBufferManager writeBufferManager_; + private SstPartitionerFactory sstPartitionerFactory_; + private ConcurrentTaskLimiter compactionThreadLimiter_; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/OptionsUtil.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/OptionsUtil.java new file mode 100644 index 0000000..612023d --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/OptionsUtil.java @@ -0,0 +1,101 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.List; + +public class OptionsUtil { + /** + * A static method to construct the DBOptions and ColumnFamilyDescriptors by + * loading the latest RocksDB options file stored in the specified rocksdb + * database. + *

    + * Note that the all the pointer options (except table_factory, which will + * be described in more details below) will be initialized with the default + * values. Developers can further initialize them after this function call. + * Below is an example list of pointer options which will be initialized. + *

    + * - env + * - memtable_factory + * - compaction_filter_factory + * - prefix_extractor + * - comparator + * - merge_operator + * - compaction_filter + *

    + * For table_factory, this function further supports deserializing + * BlockBasedTableFactory and its BlockBasedTableOptions except the + * pointer options of BlockBasedTableOptions (flush_block_policy_factory, + * and block_cache), which will be initialized with + * default values. Developers can further specify these three options by + * casting the return value of TableFactoroy::GetOptions() to + * BlockBasedTableOptions and making necessary changes. + * + * @param dbPath the path to the RocksDB. + * @param configOptions {@link org.rocksdb.ConfigOptions} instance. + * @param dbOptions {@link org.rocksdb.DBOptions} instance. This will be + * filled and returned. + * @param cfDescs A list of {@link org.rocksdb.ColumnFamilyDescriptor}'s be + * returned. + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public static void loadLatestOptions(final ConfigOptions configOptions, final String dbPath, + final DBOptions dbOptions, final List cfDescs) + throws RocksDBException { + loadLatestOptions(configOptions.nativeHandle_, dbPath, dbOptions.nativeHandle_, cfDescs); + } + + /** + * Similar to LoadLatestOptions, this function constructs the DBOptions + * and ColumnFamilyDescriptors based on the specified RocksDB Options file. + * See LoadLatestOptions above. + * + * @param optionsFileName the RocksDB options file path. + * @param configOptions {@link org.rocksdb.ConfigOptions} instance. + * @param dbOptions {@link org.rocksdb.DBOptions} instance. This will be + * filled and returned. + * @param cfDescs A list of {@link org.rocksdb.ColumnFamilyDescriptor}'s be + * returned. + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public static void loadOptionsFromFile(final ConfigOptions configOptions, + final String optionsFileName, final DBOptions dbOptions, + final List cfDescs) throws RocksDBException { + loadOptionsFromFile( + configOptions.nativeHandle_, optionsFileName, dbOptions.nativeHandle_, cfDescs); + } + + /** + * Returns the latest options file name under the specified RocksDB path. + * + * @param dbPath the path to the RocksDB. + * @param env {@link org.rocksdb.Env} instance. + * @return the latest options file name under the db path. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public static String getLatestOptionsFileName(final String dbPath, final Env env) + throws RocksDBException { + return getLatestOptionsFileName(dbPath, env.nativeHandle_); + } + + /** + * Private constructor. + * This class has only static methods and shouldn't be instantiated. + */ + private OptionsUtil() {} + + // native methods + private static native void loadLatestOptions(long cfgHandle, String dbPath, long dbOptionsHandle, + List cfDescs) throws RocksDBException; + private static native void loadOptionsFromFile(long cfgHandle, String optionsFileName, + long dbOptionsHandle, List cfDescs) throws RocksDBException; + private static native String getLatestOptionsFileName(String dbPath, long envHandle) + throws RocksDBException; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/PersistentCache.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/PersistentCache.java new file mode 100644 index 0000000..5297111 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/PersistentCache.java @@ -0,0 +1,26 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Persistent cache for caching IO pages on a persistent medium. The + * cache is specifically designed for persistent read cache. + */ +public class PersistentCache extends RocksObject { + + public PersistentCache(final Env env, final String path, final long size, + final Logger logger, final boolean optimizedForNvm) + throws RocksDBException { + super(newPersistentCache(env.nativeHandle_, path, size, + logger.nativeHandle_, optimizedForNvm)); + } + + private static native long newPersistentCache(final long envHandle, final String path, + final long size, final long loggerHandle, final boolean optimizedForNvm) + throws RocksDBException; + + @Override protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/PlainTableConfig.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/PlainTableConfig.java new file mode 100644 index 0000000..46077ba --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/PlainTableConfig.java @@ -0,0 +1,251 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +/** + * The config for plain table sst format. + * + *

    PlainTable is a RocksDB's SST file format optimized for low query + * latency on pure-memory or really low-latency media.

    + * + *

    It also support prefix hash feature.

    + */ +public class PlainTableConfig extends TableFormatConfig { + public static final int VARIABLE_LENGTH = 0; + public static final int DEFAULT_BLOOM_BITS_PER_KEY = 10; + public static final double DEFAULT_HASH_TABLE_RATIO = 0.75; + public static final int DEFAULT_INDEX_SPARSENESS = 16; + public static final int DEFAULT_HUGE_TLB_SIZE = 0; + public static final EncodingType DEFAULT_ENCODING_TYPE = + EncodingType.kPlain; + public static final boolean DEFAULT_FULL_SCAN_MODE = false; + public static final boolean DEFAULT_STORE_INDEX_IN_FILE + = false; + + public PlainTableConfig() { + keySize_ = VARIABLE_LENGTH; + bloomBitsPerKey_ = DEFAULT_BLOOM_BITS_PER_KEY; + hashTableRatio_ = DEFAULT_HASH_TABLE_RATIO; + indexSparseness_ = DEFAULT_INDEX_SPARSENESS; + hugePageTlbSize_ = DEFAULT_HUGE_TLB_SIZE; + encodingType_ = DEFAULT_ENCODING_TYPE; + fullScanMode_ = DEFAULT_FULL_SCAN_MODE; + storeIndexInFile_ = DEFAULT_STORE_INDEX_IN_FILE; + } + + /** + *

    Set the length of the user key. If it is set to be + * VARIABLE_LENGTH, then it indicates the user keys are + * of variable length.

    + * + *

    Otherwise,all the keys need to have the same length + * in byte.

    + * + *

    DEFAULT: VARIABLE_LENGTH

    + * + * @param keySize the length of the user key. + * @return the reference to the current config. + */ + public PlainTableConfig setKeySize(final int keySize) { + keySize_ = keySize; + return this; + } + + /** + * @return the specified size of the user key. If VARIABLE_LENGTH, + * then it indicates variable-length key. + */ + public int keySize() { + return keySize_; + } + + /** + * Set the number of bits per key used by the internal bloom filter + * in the plain table sst format. + * + * @param bitsPerKey the number of bits per key for bloom filer. + * @return the reference to the current config. + */ + public PlainTableConfig setBloomBitsPerKey(final int bitsPerKey) { + bloomBitsPerKey_ = bitsPerKey; + return this; + } + + /** + * @return the number of bits per key used for the bloom filter. + */ + public int bloomBitsPerKey() { + return bloomBitsPerKey_; + } + + /** + * hashTableRatio is the desired utilization of the hash table used + * for prefix hashing. The ideal ratio would be the number of + * prefixes / the number of hash buckets. If this value is set to + * zero, then hash table will not be used. + * + * @param ratio the hash table ratio. + * @return the reference to the current config. + */ + public PlainTableConfig setHashTableRatio(final double ratio) { + hashTableRatio_ = ratio; + return this; + } + + /** + * @return the hash table ratio. + */ + public double hashTableRatio() { + return hashTableRatio_; + } + + /** + * Index sparseness determines the index interval for keys inside the + * same prefix. This number is equal to the maximum number of linear + * search required after hash and binary search. If it's set to 0, + * then each key will be indexed. + * + * @param sparseness the index sparseness. + * @return the reference to the current config. + */ + public PlainTableConfig setIndexSparseness(final int sparseness) { + indexSparseness_ = sparseness; + return this; + } + + /** + * @return the index sparseness. + */ + public long indexSparseness() { + return indexSparseness_; + } + + /** + *

    huge_page_tlb_size: if ≤0, allocate hash indexes and blooms + * from malloc otherwise from huge page TLB.

    + * + *

    The user needs to reserve huge pages for it to be allocated, + * like: {@code sysctl -w vm.nr_hugepages=20}

    + * + *

    See linux doc Documentation/vm/hugetlbpage.txt

    + * + * @param hugePageTlbSize huge page tlb size + * @return the reference to the current config. + */ + public PlainTableConfig setHugePageTlbSize(final int hugePageTlbSize) { + this.hugePageTlbSize_ = hugePageTlbSize; + return this; + } + + /** + * Returns the value for huge page tlb size + * + * @return hugePageTlbSize + */ + public int hugePageTlbSize() { + return hugePageTlbSize_; + } + + /** + * Sets the encoding type. + * + *

    This setting determines how to encode + * the keys. See enum {@link EncodingType} for + * the choices.

    + * + *

    The value will determine how to encode keys + * when writing to a new SST file. This value will be stored + * inside the SST file which will be used when reading from + * the file, which makes it possible for users to choose + * different encoding type when reopening a DB. Files with + * different encoding types can co-exist in the same DB and + * can be read.

    + * + * @param encodingType {@link org.rocksdb.EncodingType} value. + * @return the reference to the current config. + */ + public PlainTableConfig setEncodingType(final EncodingType encodingType) { + this.encodingType_ = encodingType; + return this; + } + + /** + * Returns the active EncodingType + * + * @return currently set encoding type + */ + public EncodingType encodingType() { + return encodingType_; + } + + /** + * Set full scan mode, if true the whole file will be read + * one record by one without using the index. + * + * @param fullScanMode boolean value indicating if full + * scan mode shall be enabled. + * @return the reference to the current config. + */ + public PlainTableConfig setFullScanMode(final boolean fullScanMode) { + this.fullScanMode_ = fullScanMode; + return this; + } + + /** + * Return if full scan mode is active + * @return boolean value indicating if the full scan mode is + * enabled. + */ + public boolean fullScanMode() { + return fullScanMode_; + } + + /** + *

    If set to true: compute plain table index and bloom + * filter during file building and store it in file. + * When reading file, index will be mmaped instead + * of doing recomputation.

    + * + * @param storeIndexInFile value indicating if index shall + * be stored in a file + * @return the reference to the current config. + */ + public PlainTableConfig setStoreIndexInFile(final boolean storeIndexInFile) { + this.storeIndexInFile_ = storeIndexInFile; + return this; + } + + /** + * Return a boolean value indicating if index shall be stored + * in a file. + * + * @return currently set value for store index in file. + */ + public boolean storeIndexInFile() { + return storeIndexInFile_; + } + + @Override protected long newTableFactoryHandle() { + return newTableFactoryHandle(keySize_, bloomBitsPerKey_, + hashTableRatio_, indexSparseness_, hugePageTlbSize_, + encodingType_.getValue(), fullScanMode_, + storeIndexInFile_); + } + + private native long newTableFactoryHandle( + int keySize, int bloomBitsPerKey, + double hashTableRatio, int indexSparseness, + int hugePageTlbSize, byte encodingType, + boolean fullScanMode, boolean storeIndexInFile); + + private int keySize_; + private int bloomBitsPerKey_; + private double hashTableRatio_; + private int indexSparseness_; + private int hugePageTlbSize_; + private EncodingType encodingType_; + private boolean fullScanMode_; + private boolean storeIndexInFile_; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/PrepopulateBlobCache.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/PrepopulateBlobCache.java new file mode 100644 index 0000000..f1237aa --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/PrepopulateBlobCache.java @@ -0,0 +1,117 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Enum PrepopulateBlobCache + * + *

    Prepopulate warm/hot blobs which are already in memory into blob + * cache at the time of flush. On a flush, the blob that is in memory + * (in memtables) get flushed to the device. If using Direct IO, + * additional IO is incurred to read this blob back into memory again, + * which is avoided by enabling this option. This further helps if the + * workload exhibits high temporal locality, where most of the reads go + * to recently written data. This also helps in case of the remote file + * system since it involves network traffic and higher latencies.

    + */ +public enum PrepopulateBlobCache { + PREPOPULATE_BLOB_DISABLE((byte) 0x0, "prepopulate_blob_disable", "kDisable"), + PREPOPULATE_BLOB_FLUSH_ONLY((byte) 0x1, "prepopulate_blob_flush_only", "kFlushOnly"); + + /** + *

    Get the PrepopulateBlobCache enumeration value by + * passing the library name to this method.

    + * + *

    If library cannot be found the enumeration + * value {@code PREPOPULATE_BLOB_DISABLE} will be returned.

    + * + * @param libraryName prepopulate blob cache library name. + * + * @return PrepopulateBlobCache instance. + */ + public static PrepopulateBlobCache getPrepopulateBlobCache(String libraryName) { + if (libraryName != null) { + for (PrepopulateBlobCache prepopulateBlobCache : PrepopulateBlobCache.values()) { + if (prepopulateBlobCache.getLibraryName() != null + && prepopulateBlobCache.getLibraryName().equals(libraryName)) { + return prepopulateBlobCache; + } + } + } + return PrepopulateBlobCache.PREPOPULATE_BLOB_DISABLE; + } + + /** + *

    Get the PrepopulateBlobCache enumeration value by + * passing the byte identifier to this method.

    + * + * @param byteIdentifier of PrepopulateBlobCache. + * + * @return PrepopulateBlobCache instance. + * + * @throws IllegalArgumentException If PrepopulateBlobCache cannot be found for the + * provided byteIdentifier + */ + public static PrepopulateBlobCache getPrepopulateBlobCache(byte byteIdentifier) { + for (final PrepopulateBlobCache prepopulateBlobCache : PrepopulateBlobCache.values()) { + if (prepopulateBlobCache.getValue() == byteIdentifier) { + return prepopulateBlobCache; + } + } + + throw new IllegalArgumentException("Illegal value provided for PrepopulateBlobCache."); + } + + /** + *

    Get a PrepopulateBlobCache value based on the string key in the C++ options output. + * This gets used in support of getting options into Java from an options string, + * which is generated at the C++ level. + *

    + * + * @param internalName the internal (C++) name by which the option is known. + * + * @return PrepopulateBlobCache instance (optional) + */ + static PrepopulateBlobCache getFromInternal(final String internalName) { + for (final PrepopulateBlobCache prepopulateBlobCache : PrepopulateBlobCache.values()) { + if (prepopulateBlobCache.internalName_.equals(internalName)) { + return prepopulateBlobCache; + } + } + + throw new IllegalArgumentException( + "Illegal internalName '" + internalName + " ' provided for PrepopulateBlobCache."); + } + + /** + *

    Returns the byte value of the enumerations value.

    + * + * @return byte representation + */ + public byte getValue() { + return value_; + } + + /** + *

    Returns the library name of the prepopulate blob cache mode + * identified by the enumeration value.

    + * + * @return library name + */ + public String getLibraryName() { + return libraryName_; + } + + PrepopulateBlobCache(final byte value, final String libraryName, final String internalName) { + value_ = value; + libraryName_ = libraryName; + internalName_ = internalName; + } + + private final byte value_; + private final String libraryName_; + private final String internalName_; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Priority.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Priority.java new file mode 100644 index 0000000..34a56ed --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Priority.java @@ -0,0 +1,49 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * The Thread Pool priority. + */ +public enum Priority { + BOTTOM((byte) 0x0), + LOW((byte) 0x1), + HIGH((byte)0x2), + TOTAL((byte)0x3); + + private final byte value; + + Priority(final byte value) { + this.value = value; + } + + /** + *

    Returns the byte value of the enumerations value.

    + * + * @return byte representation + */ + byte getValue() { + return value; + } + + /** + * Get Priority by byte value. + * + * @param value byte representation of Priority. + * + * @return {@link org.rocksdb.Priority} instance. + * @throws java.lang.IllegalArgumentException if an invalid + * value is provided. + */ + static Priority getPriority(final byte value) { + for (final Priority priority : Priority.values()) { + if (priority.getValue() == value){ + return priority; + } + } + throw new IllegalArgumentException("Illegal value provided for Priority."); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Range.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Range.java new file mode 100644 index 0000000..74c85e5 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Range.java @@ -0,0 +1,19 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Range from start to limit. + */ +public class Range { + final Slice start; + final Slice limit; + + public Range(final Slice start, final Slice limit) { + this.start = start; + this.limit = limit; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RateLimiter.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RateLimiter.java new file mode 100644 index 0000000..c2b8a0f --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RateLimiter.java @@ -0,0 +1,227 @@ +// Copyright (c) 2015, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * RateLimiter, which is used to control write rate of flush and + * compaction. + * + * @since 3.10.0 + */ +public class RateLimiter extends RocksObject { + public static final long DEFAULT_REFILL_PERIOD_MICROS = 100 * 1000; + public static final int DEFAULT_FAIRNESS = 10; + public static final RateLimiterMode DEFAULT_MODE = + RateLimiterMode.WRITES_ONLY; + public static final boolean DEFAULT_AUTOTUNE = false; + + /** + * RateLimiter constructor + * + * @param rateBytesPerSecond this is the only parameter you want to set + * most of the time. It controls the total write rate of compaction + * and flush in bytes per second. Currently, RocksDB does not enforce + * rate limit for anything other than flush and compaction, e.g. write to + * WAL. + */ + public RateLimiter(final long rateBytesPerSecond) { + this(rateBytesPerSecond, DEFAULT_REFILL_PERIOD_MICROS, DEFAULT_FAIRNESS, + DEFAULT_MODE, DEFAULT_AUTOTUNE); + } + + /** + * RateLimiter constructor + * + * @param rateBytesPerSecond this is the only parameter you want to set + * most of the time. It controls the total write rate of compaction + * and flush in bytes per second. Currently, RocksDB does not enforce + * rate limit for anything other than flush and compaction, e.g. write to + * WAL. + * @param refillPeriodMicros this controls how often tokens are refilled. For + * example, + * when rate_bytes_per_sec is set to 10MB/s and refill_period_us is set to + * 100ms, then 1MB is refilled every 100ms internally. Larger value can + * lead to burstier writes while smaller value introduces more CPU + * overhead. The default of 100,000ms should work for most cases. + */ + public RateLimiter(final long rateBytesPerSecond, + final long refillPeriodMicros) { + this(rateBytesPerSecond, refillPeriodMicros, DEFAULT_FAIRNESS, DEFAULT_MODE, + DEFAULT_AUTOTUNE); + } + + /** + * RateLimiter constructor + * + * @param rateBytesPerSecond this is the only parameter you want to set + * most of the time. It controls the total write rate of compaction + * and flush in bytes per second. Currently, RocksDB does not enforce + * rate limit for anything other than flush and compaction, e.g. write to + * WAL. + * @param refillPeriodMicros this controls how often tokens are refilled. For + * example, + * when rate_bytes_per_sec is set to 10MB/s and refill_period_us is set to + * 100ms, then 1MB is refilled every 100ms internally. Larger value can + * lead to burstier writes while smaller value introduces more CPU + * overhead. The default of 100,000ms should work for most cases. + * @param fairness RateLimiter accepts high-pri requests and low-pri requests. + * A low-pri request is usually blocked in favor of hi-pri request. + * Currently, RocksDB assigns low-pri to request from compaction and + * high-pri to request from flush. Low-pri requests can get blocked if + * flush requests come in continuously. This fairness parameter grants + * low-pri requests permission by fairness chance even though high-pri + * requests exist to avoid starvation. + * You should be good by leaving it at default 10. + */ + public RateLimiter(final long rateBytesPerSecond, + final long refillPeriodMicros, final int fairness) { + this(rateBytesPerSecond, refillPeriodMicros, fairness, DEFAULT_MODE, + DEFAULT_AUTOTUNE); + } + + /** + * RateLimiter constructor + * + * @param rateBytesPerSecond this is the only parameter you want to set + * most of the time. It controls the total write rate of compaction + * and flush in bytes per second. Currently, RocksDB does not enforce + * rate limit for anything other than flush and compaction, e.g. write to + * WAL. + * @param refillPeriodMicros this controls how often tokens are refilled. For + * example, + * when rate_bytes_per_sec is set to 10MB/s and refill_period_us is set to + * 100ms, then 1MB is refilled every 100ms internally. Larger value can + * lead to burstier writes while smaller value introduces more CPU + * overhead. The default of 100,000ms should work for most cases. + * @param fairness RateLimiter accepts high-pri requests and low-pri requests. + * A low-pri request is usually blocked in favor of hi-pri request. + * Currently, RocksDB assigns low-pri to request from compaction and + * high-pri to request from flush. Low-pri requests can get blocked if + * flush requests come in continuously. This fairness parameter grants + * low-pri requests permission by fairness chance even though high-pri + * requests exist to avoid starvation. + * You should be good by leaving it at default 10. + * @param rateLimiterMode indicates which types of operations count against + * the limit. + */ + public RateLimiter(final long rateBytesPerSecond, + final long refillPeriodMicros, final int fairness, + final RateLimiterMode rateLimiterMode) { + this(rateBytesPerSecond, refillPeriodMicros, fairness, rateLimiterMode, + DEFAULT_AUTOTUNE); + } + + /** + * RateLimiter constructor + * + * @param rateBytesPerSecond this is the only parameter you want to set + * most of the time. It controls the total write rate of compaction + * and flush in bytes per second. Currently, RocksDB does not enforce + * rate limit for anything other than flush and compaction, e.g. write to + * WAL. + * @param refillPeriodMicros this controls how often tokens are refilled. For + * example, + * when rate_bytes_per_sec is set to 10MB/s and refill_period_us is set to + * 100ms, then 1MB is refilled every 100ms internally. Larger value can + * lead to burstier writes while smaller value introduces more CPU + * overhead. The default of 100,000ms should work for most cases. + * @param fairness RateLimiter accepts high-pri requests and low-pri requests. + * A low-pri request is usually blocked in favor of hi-pri request. + * Currently, RocksDB assigns low-pri to request from compaction and + * high-pri to request from flush. Low-pri requests can get blocked if + * flush requests come in continuously. This fairness parameter grants + * low-pri requests permission by fairness chance even though high-pri + * requests exist to avoid starvation. + * You should be good by leaving it at default 10. + * @param rateLimiterMode indicates which types of operations count against + * the limit. + * @param autoTune Enables dynamic adjustment of rate limit within the range + * {@code [rate_bytes_per_sec / 20, rate_bytes_per_sec]}, according to + * the recent demand for background I/O. + */ + public RateLimiter(final long rateBytesPerSecond, + final long refillPeriodMicros, final int fairness, + final RateLimiterMode rateLimiterMode, final boolean autoTune) { + super(newRateLimiterHandle(rateBytesPerSecond, + refillPeriodMicros, fairness, rateLimiterMode.getValue(), autoTune)); + } + + /** + *

    This API allows user to dynamically change rate limiter's bytes per second. + * REQUIRED: bytes_per_second > 0

    + * + * @param bytesPerSecond bytes per second. + */ + public void setBytesPerSecond(final long bytesPerSecond) { + assert(isOwningHandle()); + setBytesPerSecond(nativeHandle_, bytesPerSecond); + } + + /** + * Returns the bytes per second. + * + * @return bytes per second. + */ + public long getBytesPerSecond() { + assert(isOwningHandle()); + return getBytesPerSecond(nativeHandle_); + } + + /** + *

    Request for token to write bytes. If this request can not be satisfied, + * the call is blocked. Caller is responsible to make sure + * {@code bytes < GetSingleBurstBytes()}.

    + * + * @param bytes requested bytes. + */ + public void request(final long bytes) { + assert(isOwningHandle()); + request(nativeHandle_, bytes); + } + + /** + *

    Max bytes can be granted in a single burst.

    + * + * @return max bytes can be granted in a single burst. + */ + public long getSingleBurstBytes() { + assert(isOwningHandle()); + return getSingleBurstBytes(nativeHandle_); + } + + /** + *

    Total bytes that go through rate limiter.

    + * + * @return total bytes that go through rate limiter. + */ + public long getTotalBytesThrough() { + assert(isOwningHandle()); + return getTotalBytesThrough(nativeHandle_); + } + + /** + *

    Total # of requests that go through rate limiter.

    + * + * @return total # of requests that go through rate limiter. + */ + public long getTotalRequests() { + assert(isOwningHandle()); + return getTotalRequests(nativeHandle_); + } + + private static native long newRateLimiterHandle(final long rateBytesPerSecond, + final long refillPeriodMicros, final int fairness, + final byte rateLimiterMode, final boolean autoTune); + @Override protected final native void disposeInternal(final long handle); + + private native void setBytesPerSecond(final long handle, + final long bytesPerSecond); + private native long getBytesPerSecond(final long handle); + private native void request(final long handle, final long bytes); + private native long getSingleBurstBytes(final long handle); + private native long getTotalBytesThrough(final long handle); + private native long getTotalRequests(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RateLimiterMode.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RateLimiterMode.java new file mode 100644 index 0000000..4b029d8 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RateLimiterMode.java @@ -0,0 +1,52 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Mode for {@link RateLimiter#RateLimiter(long, long, int, RateLimiterMode)}. + */ +public enum RateLimiterMode { + READS_ONLY((byte)0x0), + WRITES_ONLY((byte)0x1), + ALL_IO((byte)0x2); + + private final byte value; + + RateLimiterMode(final byte value) { + this.value = value; + } + + /** + *

    Returns the byte value of the enumerations value.

    + * + * @return byte representation + */ + public byte getValue() { + return value; + } + + /** + *

    Get the RateLimiterMode enumeration value by + * passing the byte identifier to this method.

    + * + * @param byteIdentifier of RateLimiterMode. + * + * @return AccessHint instance. + * + * @throws IllegalArgumentException if the access hint for the byteIdentifier + * cannot be found + */ + public static RateLimiterMode getRateLimiterMode(final byte byteIdentifier) { + for (final RateLimiterMode rateLimiterMode : RateLimiterMode.values()) { + if (rateLimiterMode.getValue() == byteIdentifier) { + return rateLimiterMode; + } + } + + throw new IllegalArgumentException( + "Illegal value provided for RateLimiterMode."); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ReadOptions.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ReadOptions.java new file mode 100644 index 0000000..65b781d --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ReadOptions.java @@ -0,0 +1,822 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * The class that controls the get behavior. + *

    + * Note that dispose() must be called before an Options instance + * become out-of-scope to release the allocated memory in c++. + */ +public class ReadOptions extends RocksObject { + public ReadOptions() { + super(newReadOptions()); + } + + /** + * @param verifyChecksums verification will be performed on every read + * when set to true + * @param fillCache if true, then fill-cache behavior will be performed. + */ + public ReadOptions(final boolean verifyChecksums, final boolean fillCache) { + super(newReadOptions(verifyChecksums, fillCache)); + } + + /** + * Copy constructor. + *

    + * NOTE: This does a shallow copy, which means snapshot, iterate_upper_bound + * and other pointers will be cloned! + * + * @param other The ReadOptions to copy. + */ + public ReadOptions(final ReadOptions other) { + super(copyReadOptions(other.nativeHandle_)); + this.iterateLowerBoundSlice_ = other.iterateLowerBoundSlice_; + this.iterateUpperBoundSlice_ = other.iterateUpperBoundSlice_; + this.timestampSlice_ = other.timestampSlice_; + this.iterStartTs_ = other.iterStartTs_; + } + + /** + * If true, all data read from underlying storage will be + * verified against corresponding checksums. + * Default: true + * + * @return true if checksum verification is on. + */ + public boolean verifyChecksums() { + assert(isOwningHandle()); + return verifyChecksums(nativeHandle_); + } + + /** + * If true, all data read from underlying storage will be + * verified against corresponding checksums. + * Default: true + * + * @param verifyChecksums if true, then checksum verification + * will be performed on every read. + * @return the reference to the current ReadOptions. + */ + public ReadOptions setVerifyChecksums( + final boolean verifyChecksums) { + assert(isOwningHandle()); + setVerifyChecksums(nativeHandle_, verifyChecksums); + return this; + } + + // TODO(yhchiang): this option seems to be block-based table only. + // move this to a better place? + /** + * Fill the cache when loading the block-based sst formated db. + * Callers may wish to set this field to false for bulk scans. + * Default: true + * + * @return true if the fill-cache behavior is on. + */ + public boolean fillCache() { + assert(isOwningHandle()); + return fillCache(nativeHandle_); + } + + /** + * Fill the cache when loading the block-based sst formatted db. + * Callers may wish to set this field to false for bulk scans. + * Default: true + * + * @param fillCache if true, then fill-cache behavior will be + * performed. + * @return the reference to the current ReadOptions. + */ + public ReadOptions setFillCache(final boolean fillCache) { + assert(isOwningHandle()); + setFillCache(nativeHandle_, fillCache); + return this; + } + + /** + * Returns the currently assigned Snapshot instance. + * + * @return the Snapshot assigned to this instance. If no Snapshot + * is assigned null. + */ + public Snapshot snapshot() { + assert(isOwningHandle()); + final long snapshotHandle = snapshot(nativeHandle_); + if (snapshotHandle != 0) { + return new Snapshot(snapshotHandle); + } + return null; + } + + /** + *

    If "snapshot" is non-nullptr, read as of the supplied snapshot + * (which must belong to the DB that is being read and which must + * not have been released). If "snapshot" is nullptr, use an implicit + * snapshot of the state at the beginning of this read operation.

    + *

    Default: null

    + * + * @param snapshot {@link Snapshot} instance + * @return the reference to the current ReadOptions. + */ + public ReadOptions setSnapshot(final Snapshot snapshot) { + assert(isOwningHandle()); + if (snapshot != null) { + setSnapshot(nativeHandle_, snapshot.nativeHandle_); + } else { + setSnapshot(nativeHandle_, 0L); + } + return this; + } + + /** + * Returns the current read tier. + * + * @return the read tier in use, by default {@link ReadTier#READ_ALL_TIER} + */ + public ReadTier readTier() { + assert(isOwningHandle()); + return ReadTier.getReadTier(readTier(nativeHandle_)); + } + + /** + * Specify if this read request should process data that ALREADY + * resides on a particular cache. If the required data is not + * found at the specified cache, then {@link RocksDBException} is thrown. + * + * @param readTier {@link ReadTier} instance + * @return the reference to the current ReadOptions. + */ + public ReadOptions setReadTier(final ReadTier readTier) { + assert(isOwningHandle()); + setReadTier(nativeHandle_, readTier.getValue()); + return this; + } + + /** + * Specify to create a tailing iterator -- a special iterator that has a + * view of the complete database (i.e. it can also be used to read newly + * added data) and is optimized for sequential reads. It will return records + * that were inserted into the database after the creation of the iterator. + * Default: false + * @return true if tailing iterator is enabled. + */ + public boolean tailing() { + assert(isOwningHandle()); + return tailing(nativeHandle_); + } + + /** + * Specify to create a tailing iterator -- a special iterator that has a + * view of the complete database (i.e. it can also be used to read newly + * added data) and is optimized for sequential reads. It will return records + * that were inserted into the database after the creation of the iterator. + * Default: false + * + * @param tailing if true, then tailing iterator will be enabled. + * @return the reference to the current ReadOptions. + */ + public ReadOptions setTailing(final boolean tailing) { + assert(isOwningHandle()); + setTailing(nativeHandle_, tailing); + return this; + } + + /** + * Returns whether managed iterators will be used. + * + * @return the setting of whether managed iterators will be used, + * by default false + * + * @deprecated This options is not used anymore. + */ + @Deprecated + public boolean managed() { + assert(isOwningHandle()); + return managed(nativeHandle_); + } + + /** + * Specify to create a managed iterator -- a special iterator that + * uses less resources by having the ability to free its underlying + * resources on request. + * + * @param managed if true, then managed iterators will be enabled. + * @return the reference to the current ReadOptions. + * + * @deprecated This options is not used anymore. + */ + @Deprecated + public ReadOptions setManaged(final boolean managed) { + assert(isOwningHandle()); + setManaged(nativeHandle_, managed); + return this; + } + + /** + * Returns whether a total seek order will be used + * + * @return the setting of whether a total seek order will be used + */ + public boolean totalOrderSeek() { + assert(isOwningHandle()); + return totalOrderSeek(nativeHandle_); + } + + /** + * Enable a total order seek regardless of index format (e.g. hash index) + * used in the table. Some table format (e.g. plain table) may not support + * this option. + * + * @param totalOrderSeek if true, then total order seek will be enabled. + * @return the reference to the current ReadOptions. + */ + public ReadOptions setTotalOrderSeek(final boolean totalOrderSeek) { + assert(isOwningHandle()); + setTotalOrderSeek(nativeHandle_, totalOrderSeek); + return this; + } + + /** + * Returns whether the iterator only iterates over the same prefix as the seek + * + * @return the setting of whether the iterator only iterates over the same + * prefix as the seek, default is false + */ + public boolean prefixSameAsStart() { + assert(isOwningHandle()); + return prefixSameAsStart(nativeHandle_); + } + + /** + * Enforce that the iterator only iterates over the same prefix as the seek. + * This option is effective only for prefix seeks, i.e. prefix_extractor is + * non-null for the column family and {@link #totalOrderSeek()} is false. + * Unlike iterate_upper_bound, {@code #setPrefixSameAsStart(boolean)} only + * works within a prefix but in both directions. + * + * @param prefixSameAsStart if true, then the iterator only iterates over the + * same prefix as the seek + * @return the reference to the current ReadOptions. + */ + public ReadOptions setPrefixSameAsStart(final boolean prefixSameAsStart) { + assert(isOwningHandle()); + setPrefixSameAsStart(nativeHandle_, prefixSameAsStart); + return this; + } + + /** + * Returns whether the blocks loaded by the iterator will be pinned in memory + * + * @return the setting of whether the blocks loaded by the iterator will be + * pinned in memory + */ + public boolean pinData() { + assert(isOwningHandle()); + return pinData(nativeHandle_); + } + + /** + * Keep the blocks loaded by the iterator pinned in memory as long as the + * iterator is not deleted, If used when reading from tables created with + * BlockBasedTableOptions::use_delta_encoding = false, + * Iterator's property "rocksdb.iterator.is-key-pinned" is guaranteed to + * return 1. + * + * @param pinData if true, the blocks loaded by the iterator will be pinned + * @return the reference to the current ReadOptions. + */ + public ReadOptions setPinData(final boolean pinData) { + assert(isOwningHandle()); + setPinData(nativeHandle_, pinData); + return this; + } + + /** + * If true, when PurgeObsoleteFile is called in CleanupIteratorState, we + * schedule a background job in the flush job queue and delete obsolete files + * in background. + *

    + * Default: false + * + * @return true when PurgeObsoleteFile is called in CleanupIteratorState + */ + public boolean backgroundPurgeOnIteratorCleanup() { + assert(isOwningHandle()); + return backgroundPurgeOnIteratorCleanup(nativeHandle_); + } + + /** + * If true, when PurgeObsoleteFile is called in CleanupIteratorState, we + * schedule a background job in the flush job queue and delete obsolete files + * in background. + *

    + * Default: false + * + * @param backgroundPurgeOnIteratorCleanup true when PurgeObsoleteFile is + * called in CleanupIteratorState + * @return the reference to the current ReadOptions. + */ + public ReadOptions setBackgroundPurgeOnIteratorCleanup( + final boolean backgroundPurgeOnIteratorCleanup) { + assert(isOwningHandle()); + setBackgroundPurgeOnIteratorCleanup(nativeHandle_, + backgroundPurgeOnIteratorCleanup); + return this; + } + + /** + * If non-zero, NewIterator will create a new table reader which + * performs reads of the given size. Using a large size (> 2MB) can + * improve the performance of forward iteration on spinning disks. + *

    + * Default: 0 + * + * @return The readahead size is bytes + */ + public long readaheadSize() { + assert(isOwningHandle()); + return readaheadSize(nativeHandle_); + } + + /** + * If non-zero, NewIterator will create a new table reader which + * performs reads of the given size. Using a large size (> 2MB) can + * improve the performance of forward iteration on spinning disks. + *

    + * Default: 0 + * + * @param readaheadSize The readahead size is bytes + * @return the reference to the current ReadOptions. + */ + public ReadOptions setReadaheadSize(final long readaheadSize) { + assert(isOwningHandle()); + setReadaheadSize(nativeHandle_, readaheadSize); + return this; + } + + /** + * A threshold for the number of keys that can be skipped before failing an + * iterator seek as incomplete. + * + * @return the number of keys that can be skipped + * before failing an iterator seek as incomplete. + */ + public long maxSkippableInternalKeys() { + assert(isOwningHandle()); + return maxSkippableInternalKeys(nativeHandle_); + } + + /** + * A threshold for the number of keys that can be skipped before failing an + * iterator seek as incomplete. The default value of 0 should be used to + * never fail a request as incomplete, even on skipping too many keys. + *

    + * Default: 0 + * + * @param maxSkippableInternalKeys the number of keys that can be skipped + * before failing an iterator seek as incomplete. + * + * @return the reference to the current ReadOptions. + */ + public ReadOptions setMaxSkippableInternalKeys( + final long maxSkippableInternalKeys) { + assert(isOwningHandle()); + setMaxSkippableInternalKeys(nativeHandle_, maxSkippableInternalKeys); + return this; + } + + /** + * If true, keys deleted using the DeleteRange() API will be visible to + * readers until they are naturally deleted during compaction. This improves + * read performance in DBs with many range deletions. + *

    + * Default: false + * + * @return true if keys deleted using the DeleteRange() API will be visible + */ + public boolean ignoreRangeDeletions() { + assert(isOwningHandle()); + return ignoreRangeDeletions(nativeHandle_); + } + + /** + * If true, keys deleted using the DeleteRange() API will be visible to + * readers until they are naturally deleted during compaction. This improves + * read performance in DBs with many range deletions. + *

    + * Default: false + * + * @param ignoreRangeDeletions true if keys deleted using the DeleteRange() + * API should be visible + * @return the reference to the current ReadOptions. + */ + public ReadOptions setIgnoreRangeDeletions(final boolean ignoreRangeDeletions) { + assert(isOwningHandle()); + setIgnoreRangeDeletions(nativeHandle_, ignoreRangeDeletions); + return this; + } + + /** + * Defines the smallest key at which the backward + * iterator can return an entry. Once the bound is passed, + * {@link RocksIterator#isValid()} will be false. + *

    + * The lower bound is inclusive i.e. the bound value is a valid + * entry. + *

    + * If prefix_extractor is not null, the Seek target and `iterate_lower_bound` + * need to have the same prefix. This is because ordering is not guaranteed + * outside of prefix domain. + *

    + * Default: null + * + * @param iterateLowerBound Slice representing the lower bound + * @return the reference to the current ReadOptions. + */ + public ReadOptions setIterateLowerBound(final AbstractSlice iterateLowerBound) { + assert(isOwningHandle()); + setIterateLowerBound( + nativeHandle_, iterateLowerBound == null ? 0 : iterateLowerBound.getNativeHandle()); + // Hold onto a reference so it doesn't get garbage collected out from under us. + iterateLowerBoundSlice_ = iterateLowerBound; + return this; + } + + /** + * Returns the smallest key at which the backward + * iterator can return an entry. + *

    + * The lower bound is inclusive i.e. the bound value is a valid entry. + * + * @return the smallest key, or null if there is no lower bound defined. + */ + public Slice iterateLowerBound() { + assert(isOwningHandle()); + final long lowerBoundSliceHandle = iterateLowerBound(nativeHandle_); + if (lowerBoundSliceHandle != 0) { + // Disown the new slice - it's owned by the C++ side of the JNI boundary + // from the perspective of this method. + return new Slice(lowerBoundSliceHandle, false); + } + return null; + } + + /** + * Defines the extent up to which the forward iterator + * can return entries. Once the bound is reached, + * {@link RocksIterator#isValid()} will be false. + *

    + * The upper bound is exclusive i.e. the bound value is not a valid entry. + *

    + * If prefix_extractor is not null, the Seek target and iterate_upper_bound + * need to have the same prefix. This is because ordering is not guaranteed + * outside of prefix domain. + *

    + * Default: null + * + * @param iterateUpperBound Slice representing the upper bound + * @return the reference to the current ReadOptions. + */ + public ReadOptions setIterateUpperBound(final AbstractSlice iterateUpperBound) { + assert(isOwningHandle()); + setIterateUpperBound( + nativeHandle_, iterateUpperBound == null ? 0 : iterateUpperBound.getNativeHandle()); + // Hold onto a reference so it doesn't get garbage collected out from under us. + iterateUpperBoundSlice_ = iterateUpperBound; + return this; + } + + /** + * Returns the largest key at which the forward + * iterator can return an entry. + *

    + * The upper bound is exclusive i.e. the bound value is not a valid entry. + * + * @return the largest key, or null if there is no upper bound defined. + */ + public Slice iterateUpperBound() { + assert(isOwningHandle()); + final long upperBoundSliceHandle = iterateUpperBound(nativeHandle_); + if (upperBoundSliceHandle != 0) { + // Disown the new slice - it's owned by the C++ side of the JNI boundary + // from the perspective of this method. + return new Slice(upperBoundSliceHandle, false); + } + return null; + } + + /** + * A callback to determine whether relevant keys for this scan exist in a + * given table based on the table's properties. The callback is passed the + * properties of each table during iteration. If the callback returns false, + * the table will not be scanned. This option only affects Iterators and has + * no impact on point lookups. + *

    + * Default: null (every table will be scanned) + * + * @param tableFilter the table filter for the callback. + * + * @return the reference to the current ReadOptions. + */ + public ReadOptions setTableFilter(final AbstractTableFilter tableFilter) { + assert(isOwningHandle()); + setTableFilter(nativeHandle_, tableFilter.nativeHandle_); + return this; + } + + /** + * When true, by default use total_order_seek = true, and RocksDB can + * selectively enable prefix seek mode if won't generate a different result + * from total_order_seek, based on seek key, and iterator upper bound. + * Default: false + * + * @return true if auto prefix mode is set. + * + */ + public boolean autoPrefixMode() { + assert (isOwningHandle()); + return autoPrefixMode(nativeHandle_); + } + + /** + * When true, by default use total_order_seek = true, and RocksDB can + * selectively enable prefix seek mode if won't generate a different result + * from total_order_seek, based on seek key, and iterator upper bound. + * Default: false + * @param mode auto prefix mode + * @return the reference to the current ReadOptions. + */ + public ReadOptions setAutoPrefixMode(final boolean mode) { + assert (isOwningHandle()); + setAutoPrefixMode(nativeHandle_, mode); + return this; + } + + /** + * Timestamp of operation. Read should return the latest data visible to the + * specified timestamp. All timestamps of the same database must be of the + * same length and format. The user is responsible for providing a customized + * compare function via Comparator to order >key, timestamp> tuples. + * For iterator, iter_start_ts is the lower bound (older) and timestamp + * serves as the upper bound. Versions of the same record that fall in + * the timestamp range will be returned. If iter_start_ts is nullptr, + * only the most recent version visible to timestamp is returned. + * The user-specified timestamp feature is still under active development, + * and the API is subject to change. + *

    + * Default: null + * @see #iterStartTs() + * @return Reference to timestamp or null if there is no timestamp defined. + */ + public Slice timestamp() { + assert (isOwningHandle()); + final long timestampSliceHandle = timestamp(nativeHandle_); + if (timestampSliceHandle != 0) { + return new Slice(timestampSliceHandle); + } else { + return null; + } + } + + /** + * Timestamp of operation. Read should return the latest data visible to the + * specified timestamp. All timestamps of the same database must be of the + * same length and format. The user is responsible for providing a customized + * compare function via Comparator to order {@code } tuples. + * For iterator, {@code iter_start_ts} is the lower bound (older) and timestamp + * serves as the upper bound. Versions of the same record that fall in + * the timestamp range will be returned. If iter_start_ts is nullptr, + * only the most recent version visible to timestamp is returned. + * The user-specified timestamp feature is still under active development, + * and the API is subject to change. + *

    + * Default: null + * @see #setIterStartTs(AbstractSlice) + * @param timestamp Slice representing the timestamp + * @return the reference to the current ReadOptions. + */ + public ReadOptions setTimestamp(final AbstractSlice timestamp) { + assert (isOwningHandle()); + setTimestamp(nativeHandle_, timestamp == null ? 0 : timestamp.getNativeHandle()); + timestampSlice_ = timestamp; + return this; + } + + /** + * Timestamp of operation. Read should return the latest data visible to the + * specified timestamp. All timestamps of the same database must be of the + * same length and format. The user is responsible for providing a customized + * compare function via Comparator to order {@code } tuples. + * For iterator, {@code iter_start_ts} is the lower bound (older) and timestamp + * serves as the upper bound. Versions of the same record that fall in + * the timestamp range will be returned. If iter_start_ts is nullptr, + * only the most recent version visible to timestamp is returned. + * The user-specified timestamp feature is still under active development, + * and the API is subject to change. + *

    + * Default: null + * @return Reference to lower bound timestamp or null if there is no lower bound timestamp + * defined. + */ + public Slice iterStartTs() { + assert (isOwningHandle()); + final long iterStartTsHandle = iterStartTs(nativeHandle_); + if (iterStartTsHandle != 0) { + return new Slice(iterStartTsHandle); + } else { + return null; + } + } + + /** + * Timestamp of operation. Read should return the latest data visible to the + * specified timestamp. All timestamps of the same database must be of the + * same length and format. The user is responsible for providing a customized + * compare function via Comparator to order {@code } tuples. + * For iterator, {@code iter_start_ts} is the lower bound (older) and timestamp + * serves as the upper bound. Versions of the same record that fall in + * the timestamp range will be returned. If iter_start_ts is nullptr, + * only the most recent version visible to timestamp is returned. + * The user-specified timestamp feature is still under active development, + * and the API is subject to change. + *

    + * Default: null + * + * @param iterStartTs Reference to lower bound timestamp or null if there is no lower bound + * timestamp defined + * @return the reference to the current ReadOptions. + */ + public ReadOptions setIterStartTs(final AbstractSlice iterStartTs) { + assert (isOwningHandle()); + setIterStartTs(nativeHandle_, iterStartTs == null ? 0 : iterStartTs.getNativeHandle()); + iterStartTs_ = iterStartTs; + return this; + } + + /** + * Deadline for completing an API call (Get/MultiGet/Seek/Next for now) + * in microseconds. + * It should be set to microseconds since epoch, i.e, {@code gettimeofday} or + * equivalent plus allowed duration in microseconds. The best way is to use + * {@code env->NowMicros() + some timeout}. + * This is best efforts. The call may exceed the deadline if there is IO + * involved and the file system doesn't support deadlines, or due to + * checking for deadline periodically rather than for every key if + * processing a batch + * + * @return deadline time in microseconds + */ + public long deadline() { + assert (isOwningHandle()); + return deadline(nativeHandle_); + } + + /** + * Deadline for completing an API call (Get/MultiGet/Seek/Next for now) + * in microseconds. + * It should be set to microseconds since epoch, i.e, {@code gettimeofday} or + * equivalent plus allowed duration in microseconds. The best way is to use + * {@code env->NowMicros() + some timeout}. + * This is best efforts. The call may exceed the deadline if there is IO + * involved and the file system doesn't support deadlines, or due to + * checking for deadline periodically rather than for every key if + * processing a batch + * + * @param deadlineTime deadline time in microseconds. + * @return the reference to the current ReadOptions. + */ + public ReadOptions setDeadline(final long deadlineTime) { + assert (isOwningHandle()); + setDeadline(nativeHandle_, deadlineTime); + return this; + } + + /** + * A timeout in microseconds to be passed to the underlying FileSystem for + * reads. As opposed to deadline, this determines the timeout for each + * individual file read request. If a MultiGet/Get/Seek/Next etc call + * results in multiple reads, each read can last up to io_timeout us. + * @return ioTimeout time in microseconds + */ + public long ioTimeout() { + assert (isOwningHandle()); + return ioTimeout(nativeHandle_); + } + + /** + * A timeout in microseconds to be passed to the underlying FileSystem for + * reads. As opposed to deadline, this determines the timeout for each + * individual file read request. If a MultiGet/Get/Seek/Next etc call + * results in multiple reads, each read can last up to io_timeout us. + * + * @param ioTimeout time in microseconds. + * @return the reference to the current ReadOptions. + */ + public ReadOptions setIoTimeout(final long ioTimeout) { + assert (isOwningHandle()); + setIoTimeout(nativeHandle_, ioTimeout); + return this; + } + + /** + * It limits the maximum cumulative value size of the keys in batch while + * reading through MultiGet. Once the cumulative value size exceeds this + * soft limit then all the remaining keys are returned with status Aborted. + *

    + * Default: {@code std::numeric_limits::max()} + * @return actual valueSizeSofLimit + */ + public long valueSizeSoftLimit() { + assert (isOwningHandle()); + return valueSizeSoftLimit(nativeHandle_); + } + + /** + * It limits the maximum cumulative value size of the keys in batch while + * reading through MultiGet. Once the cumulative value size exceeds this + * soft limit then all the remaining keys are returned with status Aborted. + *

    + * Default: {@code std::numeric_limits::max()} + * + * @param valueSizeSoftLimit the maximum cumulative value size of the keys + * @return the reference to the current ReadOptions + */ + public ReadOptions setValueSizeSoftLimit(final long valueSizeSoftLimit) { + assert (isOwningHandle()); + setValueSizeSoftLimit(nativeHandle_, valueSizeSoftLimit); + return this; + } + + // instance variables + // NOTE: If you add new member variables, please update the copy constructor above! + // + // Hold a reference to any iterate lower or upper bound that was set on this + // object until we're destroyed or it's overwritten. That way the caller can + // freely leave scope without us losing the Java Slice object, which during + // close() would also reap its associated rocksdb::Slice native object since + // it's possibly (likely) to be an owning handle. + private AbstractSlice iterateLowerBoundSlice_; + private AbstractSlice iterateUpperBoundSlice_; + private AbstractSlice timestampSlice_; + private AbstractSlice iterStartTs_; + + private static native long newReadOptions(); + private static native long newReadOptions(final boolean verifyChecksums, final boolean fillCache); + private static native long copyReadOptions(long handle); + @Override protected final native void disposeInternal(final long handle); + + private native boolean verifyChecksums(long handle); + private native void setVerifyChecksums(long handle, boolean verifyChecksums); + private native boolean fillCache(long handle); + private native void setFillCache(long handle, boolean fillCache); + private native long snapshot(long handle); + private native void setSnapshot(long handle, long snapshotHandle); + private native byte readTier(long handle); + private native void setReadTier(long handle, byte readTierValue); + private native boolean tailing(long handle); + private native void setTailing(long handle, boolean tailing); + private native boolean managed(long handle); + private native void setManaged(long handle, boolean managed); + private native boolean totalOrderSeek(long handle); + private native void setTotalOrderSeek(long handle, boolean totalOrderSeek); + private native boolean prefixSameAsStart(long handle); + private native void setPrefixSameAsStart(long handle, boolean prefixSameAsStart); + private native boolean pinData(long handle); + private native void setPinData(long handle, boolean pinData); + private native boolean backgroundPurgeOnIteratorCleanup(final long handle); + private native void setBackgroundPurgeOnIteratorCleanup(final long handle, + final boolean backgroundPurgeOnIteratorCleanup); + private native long readaheadSize(final long handle); + private native void setReadaheadSize(final long handle, + final long readaheadSize); + private native long maxSkippableInternalKeys(final long handle); + private native void setMaxSkippableInternalKeys(final long handle, + final long maxSkippableInternalKeys); + private native boolean ignoreRangeDeletions(final long handle); + private native void setIgnoreRangeDeletions(final long handle, + final boolean ignoreRangeDeletions); + private native void setIterateUpperBound(final long handle, + final long upperBoundSliceHandle); + private native long iterateUpperBound(final long handle); + private native void setIterateLowerBound(final long handle, + final long lowerBoundSliceHandle); + private native long iterateLowerBound(final long handle); + private native void setTableFilter(final long handle, final long tableFilterHandle); + private native boolean autoPrefixMode(final long handle); + private native void setAutoPrefixMode(final long handle, final boolean autoPrefixMode); + private native long timestamp(final long handle); + private native void setTimestamp(final long handle, final long timestampSliceHandle); + private native long iterStartTs(final long handle); + private native void setIterStartTs(final long handle, final long iterStartTsHandle); + private native long deadline(final long handle); + private native void setDeadline(final long handle, final long deadlineTime); + private native long ioTimeout(final long handle); + private native void setIoTimeout(final long handle, final long ioTimeout); + private native long valueSizeSoftLimit(final long handle); + private native void setValueSizeSoftLimit(final long handle, final long softLimit); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ReadTier.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ReadTier.java new file mode 100644 index 0000000..78f83f6 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ReadTier.java @@ -0,0 +1,49 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * RocksDB {@link ReadOptions} read tiers. + */ +public enum ReadTier { + READ_ALL_TIER((byte)0), + BLOCK_CACHE_TIER((byte)1), + PERSISTED_TIER((byte)2), + MEMTABLE_TIER((byte)3); + + private final byte value; + + ReadTier(final byte value) { + this.value = value; + } + + /** + * Returns the byte value of the enumerations value + * + * @return byte representation + */ + public byte getValue() { + return value; + } + + /** + * Get ReadTier by byte value. + * + * @param value byte representation of ReadTier. + * + * @return {@link org.rocksdb.ReadTier} instance or null. + * @throws java.lang.IllegalArgumentException if an invalid + * value is provided. + */ + public static ReadTier getReadTier(final byte value) { + for (final ReadTier readTier : ReadTier.values()) { + if (readTier.getValue() == value){ + return readTier; + } + } + throw new IllegalArgumentException("Illegal value provided for ReadTier."); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RemoveEmptyValueCompactionFilter.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RemoveEmptyValueCompactionFilter.java new file mode 100644 index 0000000..e966943 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RemoveEmptyValueCompactionFilter.java @@ -0,0 +1,18 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Just a Java wrapper around EmptyValueCompactionFilter implemented in C++ + */ +public class RemoveEmptyValueCompactionFilter + extends AbstractCompactionFilter { + public RemoveEmptyValueCompactionFilter() { + super(createNewRemoveEmptyValueCompactionFilter0()); + } + + private static native long createNewRemoveEmptyValueCompactionFilter0(); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RestoreOptions.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RestoreOptions.java new file mode 100644 index 0000000..a6b43d4 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RestoreOptions.java @@ -0,0 +1,32 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * RestoreOptions to control the behavior of restore. + *

    + * Note that dispose() must be called before this instance become out-of-scope + * to release the allocated memory in c++. + * + */ +public class RestoreOptions extends RocksObject { + /** + * Constructor + * + * @param keepLogFiles If true, restore won't overwrite the existing log files + * in wal_dir. It will also move all log files from archive directory to + * wal_dir. Use this option in combination with + * BackupEngineOptions::backup_log_files = false for persisting in-memory + * databases. + * Default: false + */ + public RestoreOptions(final boolean keepLogFiles) { + super(newRestoreOptions(keepLogFiles)); + } + + private static native long newRestoreOptions(boolean keepLogFiles); + @Override protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ReusedSynchronisationType.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ReusedSynchronisationType.java new file mode 100644 index 0000000..2709a5d --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ReusedSynchronisationType.java @@ -0,0 +1,65 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +/** + * Determines the type of synchronisation primitive used + * in native code. + */ +public enum ReusedSynchronisationType { + /** + * Standard mutex. + */ + MUTEX((byte)0x0), + + /** + * Use adaptive mutex, which spins in the user space before resorting + * to kernel. This could reduce context switch when the mutex is not + * heavily contended. However, if the mutex is hot, we could end up + * wasting spin time. + */ + ADAPTIVE_MUTEX((byte)0x1), + + /** + * There is a reused buffer per-thread. + */ + THREAD_LOCAL((byte)0x2); + + private final byte value; + + ReusedSynchronisationType(final byte value) { + this.value = value; + } + + /** + * Returns the byte value of the enumerations value + * + * @return byte representation + */ + public byte getValue() { + return value; + } + + /** + * Get ReusedSynchronisationType by byte value. + * + * @param value byte representation of ReusedSynchronisationType. + * + * @return {@link org.rocksdb.ReusedSynchronisationType} instance. + * @throws java.lang.IllegalArgumentException if an invalid + * value is provided. + */ + public static ReusedSynchronisationType getReusedSynchronisationType( + final byte value) { + for (final ReusedSynchronisationType reusedSynchronisationType + : ReusedSynchronisationType.values()) { + if (reusedSynchronisationType.getValue() == value) { + return reusedSynchronisationType; + } + } + throw new IllegalArgumentException( + "Illegal value provided for ReusedSynchronisationType."); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksCallbackObject.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksCallbackObject.java new file mode 100644 index 0000000..2b9de4b --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksCallbackObject.java @@ -0,0 +1,73 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.List; + +/** + * RocksCallbackObject is similar to {@link RocksObject} but varies + * in its construction as it is designed for Java objects which have functions + * which are called from C++ via JNI. + *

    + * RocksCallbackObject is the base-class any RocksDB classes that acts as a + * callback from some underlying underlying native C++ {@code rocksdb} object. + *

    + * The use of {@code RocksObject} should always be preferred over + * {@link RocksCallbackObject} if callbacks are not required. + */ +public abstract class RocksCallbackObject extends + AbstractImmutableNativeReference { + + protected final long nativeHandle_; + + protected RocksCallbackObject(final long... nativeParameterHandles) { + super(true); + this.nativeHandle_ = initializeNative(nativeParameterHandles); + } + + /** + * Given a list of RocksCallbackObjects, it returns a list + * of the native handles of the underlying objects. + * + * @param objectList the rocks callback objects + * + * @return the native handles + */ + static /* @Nullable */ long[] toNativeHandleList( + /* @Nullable */ final List objectList) { + if (objectList == null) { + return null; + } + final int len = objectList.size(); + final long[] handleList = new long[len]; + for (int i = 0; i < len; i++) { + handleList[i] = objectList.get(i).nativeHandle_; + } + return handleList; + } + + /** + * Construct the Native C++ object which will callback + * to our object methods + * + * @param nativeParameterHandles An array of native handles for any parameter + * objects that are needed during construction + * + * @return The native handle of the C++ object which will callback to us + */ + protected abstract long initializeNative( + final long... nativeParameterHandles); + + /** + * Deletes underlying C++ native callback object pointer + */ + @Override + protected void disposeInternal() { + disposeInternal(nativeHandle_); + } + + private native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksDB.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksDB.java new file mode 100644 index 0000000..fb35208 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksDB.java @@ -0,0 +1,4688 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.atomic.AtomicReference; +import org.rocksdb.util.Environment; + +/** + * A RocksDB is a persistent ordered map from keys to values. It is safe for + * concurrent access from multiple threads without any external synchronization. + * All methods of this class could potentially throw RocksDBException, which + * indicates sth wrong at the RocksDB library side and the call failed. + */ +public class RocksDB extends RocksObject { + public static final byte[] DEFAULT_COLUMN_FAMILY = "default".getBytes(UTF_8); + public static final int NOT_FOUND = -1; + + private enum LibraryState { + NOT_LOADED, + LOADING, + LOADED + } + + private static final AtomicReference libraryLoaded = + new AtomicReference<>(LibraryState.NOT_LOADED); + + static { + RocksDB.loadLibrary(); + } + + private final List ownedColumnFamilyHandles = new ArrayList<>(); + + /** + * Loads the necessary library files. + * Calling this method twice will have no effect. + * By default the method extracts the shared library for loading at + * java.io.tmpdir, however, you can override this temporary location by + * setting the environment variable ROCKSDB_SHAREDLIB_DIR. + */ + public static void loadLibrary() { + if (libraryLoaded.get() == LibraryState.LOADED) { + return; + } + + if (libraryLoaded.compareAndSet(LibraryState.NOT_LOADED, + LibraryState.LOADING)) { + final String tmpDir = System.getenv("ROCKSDB_SHAREDLIB_DIR"); + // loading possibly necessary libraries. + for (final CompressionType compressionType : CompressionType.values()) { + try { + if (compressionType.getLibraryName() != null) { + System.loadLibrary(compressionType.getLibraryName()); + } + } catch (final UnsatisfiedLinkError e) { + // since it may be optional, we ignore its loading failure here. + } + } + try { + NativeLibraryLoader.getInstance().loadLibrary(tmpDir); + } catch (final IOException e) { + libraryLoaded.set(LibraryState.NOT_LOADED); + throw new RuntimeException("Unable to load the RocksDB shared library", + e); + } + + final int encodedVersion = version(); + version = Version.fromEncodedVersion(encodedVersion); + + libraryLoaded.set(LibraryState.LOADED); + return; + } + + while (libraryLoaded.get() == LibraryState.LOADING) { + try { + Thread.sleep(10); + } catch(final InterruptedException e) { + //ignore + } + } + } + + /** + * Tries to load the necessary library files from the given list of + * directories. + * + * @param paths a list of strings where each describes a directory + * of a library. + */ + public static void loadLibrary(final List paths) { + if (libraryLoaded.get() == LibraryState.LOADED) { + return; + } + + if (libraryLoaded.compareAndSet(LibraryState.NOT_LOADED, + LibraryState.LOADING)) { + for (final CompressionType compressionType : CompressionType.values()) { + if (compressionType.equals(CompressionType.NO_COMPRESSION)) { + continue; + } + for (final String path : paths) { + try { + System.load(path + "/" + Environment.getSharedLibraryFileName( + compressionType.getLibraryName())); + break; + } catch (final UnsatisfiedLinkError e) { + // since they are optional, we ignore loading fails. + } + } + } + boolean success = false; + UnsatisfiedLinkError err = null; + for (final String path : paths) { + try { + System.load(path + "/" + + Environment.getJniLibraryFileName("rocksdbjni")); + success = true; + break; + } catch (final UnsatisfiedLinkError e) { + err = e; + } + } + if (!success) { + libraryLoaded.set(LibraryState.NOT_LOADED); + throw err; + } + + final int encodedVersion = version(); + version = Version.fromEncodedVersion(encodedVersion); + + libraryLoaded.set(LibraryState.LOADED); + return; + } + + while (libraryLoaded.get() == LibraryState.LOADING) { + try { + Thread.sleep(10); + } catch(final InterruptedException e) { + //ignore + } + } + } + + public static Version rocksdbVersion() { + return version; + } + + /** + * Private constructor. + * + * @param nativeHandle The native handle of the C++ RocksDB object + */ + protected RocksDB(final long nativeHandle) { + super(nativeHandle); + } + + /** + * The factory constructor of RocksDB that opens a RocksDB instance given + * the path to the database using the default options w/ createIfMissing + * set to true. + * + * @param path the path to the rocksdb. + * @return a {@link RocksDB} instance on success, null if the specified + * {@link RocksDB} can not be opened. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + * @see Options#setCreateIfMissing(boolean) + */ + public static RocksDB open(final String path) throws RocksDBException { + final Options options = new Options(); + options.setCreateIfMissing(true); + return open(options, path); + } + + /** + * The factory constructor of RocksDB that opens a RocksDB instance given + * the path to the database using the specified options and db path and a list + * of column family names. + *

    + * If opened in read write mode every existing column family name must be + * passed within the list to this method.

    + *

    + * If opened in read-only mode only a subset of existing column families must + * be passed to this method.

    + *

    + * Options instance *should* not be disposed before all DBs using this options + * instance have been closed. If user doesn't call options dispose explicitly, + * then this options instance will be GC'd automatically

    + *

    + * ColumnFamily handles are disposed when the RocksDB instance is disposed. + *

    + * + * @param path the path to the rocksdb. + * @param columnFamilyDescriptors list of column family descriptors + * @param columnFamilyHandles will be filled with ColumnFamilyHandle instances + * on open. + * @return a {@link RocksDB} instance on success, null if the specified + * {@link RocksDB} can not be opened. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + * @see DBOptions#setCreateIfMissing(boolean) + */ + public static RocksDB open(final String path, + final List columnFamilyDescriptors, + final List columnFamilyHandles) + throws RocksDBException { + final DBOptions options = new DBOptions(); + return open(options, path, columnFamilyDescriptors, columnFamilyHandles); + } + + /** + * The factory constructor of RocksDB that opens a RocksDB instance given + * the path to the database using the specified options and db path. + * + *

    + * Options instance *should* not be disposed before all DBs using this options + * instance have been closed. If user doesn't call options dispose explicitly, + * then this options instance will be GC'd automatically.

    + *

    + * Options instance can be re-used to open multiple DBs if DB statistics is + * not used. If DB statistics are required, then its recommended to open DB + * with new Options instance as underlying native statistics instance does not + * use any locks to prevent concurrent updates.

    + * + * @param options {@link org.rocksdb.Options} instance. + * @param path the path to the rocksdb. + * @return a {@link RocksDB} instance on success, null if the specified + * {@link RocksDB} can not be opened. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + * + * @see Options#setCreateIfMissing(boolean) + */ + public static RocksDB open(final Options options, final String path) + throws RocksDBException { + // when non-default Options is used, keeping an Options reference + // in RocksDB can prevent Java to GC during the life-time of + // the currently-created RocksDB. + final RocksDB db = new RocksDB(open(options.nativeHandle_, path)); + db.storeOptionsInstance(options); + return db; + } + + /** + * The factory constructor of RocksDB that opens a RocksDB instance given + * the path to the database using the specified options and db path and a list + * of column family names. + *

    + * If opened in read write mode every existing column family name must be + * passed within the list to this method.

    + *

    + * If opened in read-only mode only a subset of existing column families must + * be passed to this method.

    + *

    + * Options instance *should* not be disposed before all DBs using this options + * instance have been closed. If user doesn't call options dispose explicitly, + * then this options instance will be GC'd automatically.

    + *

    + * Options instance can be re-used to open multiple DBs if DB statistics is + * not used. If DB statistics are required, then its recommended to open DB + * with new Options instance as underlying native statistics instance does not + * use any locks to prevent concurrent updates.

    + *

    + * ColumnFamily handles are disposed when the RocksDB instance is disposed. + *

    + * + * @param options {@link org.rocksdb.DBOptions} instance. + * @param path the path to the rocksdb. + * @param columnFamilyDescriptors list of column family descriptors + * @param columnFamilyHandles will be filled with ColumnFamilyHandle instances + * on open. + * @return a {@link RocksDB} instance on success, null if the specified + * {@link RocksDB} can not be opened. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + * + * @see DBOptions#setCreateIfMissing(boolean) + */ + public static RocksDB open(final DBOptions options, final String path, + final List columnFamilyDescriptors, + final List columnFamilyHandles) + throws RocksDBException { + + final byte[][] cfNames = new byte[columnFamilyDescriptors.size()][]; + final long[] cfOptionHandles = new long[columnFamilyDescriptors.size()]; + for (int i = 0; i < columnFamilyDescriptors.size(); i++) { + final ColumnFamilyDescriptor cfDescriptor = columnFamilyDescriptors + .get(i); + cfNames[i] = cfDescriptor.getName(); + cfOptionHandles[i] = cfDescriptor.getOptions().nativeHandle_; + } + + final long[] handles = open(options.nativeHandle_, path, cfNames, + cfOptionHandles); + final RocksDB db = new RocksDB(handles[0]); + db.storeOptionsInstance(options); + + for (int i = 1; i < handles.length; i++) { + final ColumnFamilyHandle columnFamilyHandle = new ColumnFamilyHandle(db, handles[i]); + columnFamilyHandles.add(columnFamilyHandle); + } + + db.ownedColumnFamilyHandles.addAll(columnFamilyHandles); + + return db; + } + + /** + * The factory constructor of RocksDB that opens a RocksDB instance in + * Read-Only mode given the path to the database using the default + * options. + * + * @param path the path to the RocksDB. + * @return a {@link RocksDB} instance on success, null if the specified + * {@link RocksDB} can not be opened. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public static RocksDB openReadOnly(final String path) + throws RocksDBException { + // This allows to use the rocksjni default Options instead of + // the c++ one. + final Options options = new Options(); + return openReadOnly(options, path); + } + + /** + * The factory constructor of RocksDB that opens a RocksDB instance in + * Read-Only mode given the path to the database using the specified + * options and db path. + *

    + * Options instance *should* not be disposed before all DBs using this options + * instance have been closed. If user doesn't call options dispose explicitly, + * then this options instance will be GC'd automatically. + * + * @param options {@link Options} instance. + * @param path the path to the RocksDB. + * @return a {@link RocksDB} instance on success, null if the specified + * {@link RocksDB} can not be opened. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public static RocksDB openReadOnly(final Options options, final String path) + throws RocksDBException { + return openReadOnly(options, path, false); + } + + /** + * The factory constructor of RocksDB that opens a RocksDB instance in + * Read-Only mode given the path to the database using the specified + * options and db path. + *

    + * Options instance *should* not be disposed before all DBs using this options + * instance have been closed. If user doesn't call options dispose explicitly, + * then this options instance will be GC'd automatically. + * + * @param options {@link Options} instance. + * @param path the path to the RocksDB. + * @param errorIfWalFileExists true to raise an error when opening the db + * if a Write Ahead Log file exists, false otherwise. + * @return a {@link RocksDB} instance on success, null if the specified + * {@link RocksDB} can not be opened. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public static RocksDB openReadOnly(final Options options, final String path, + final boolean errorIfWalFileExists) throws RocksDBException { + // when non-default Options is used, keeping an Options reference + // in RocksDB can prevent Java to GC during the life-time of + // the currently-created RocksDB. + final RocksDB db = new RocksDB(openROnly(options.nativeHandle_, path, errorIfWalFileExists)); + db.storeOptionsInstance(options); + return db; + } + + /** + * The factory constructor of RocksDB that opens a RocksDB instance in + * Read-Only mode given the path to the database using the default + * options. + * + * @param path the path to the RocksDB. + * @param columnFamilyDescriptors list of column family descriptors + * @param columnFamilyHandles will be filled with ColumnFamilyHandle instances + * on open. + * @return a {@link RocksDB} instance on success, null if the specified + * {@link RocksDB} can not be opened. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public static RocksDB openReadOnly(final String path, + final List columnFamilyDescriptors, + final List columnFamilyHandles) + throws RocksDBException { + // This allows to use the rocksjni default Options instead of + // the c++ one. + final DBOptions options = new DBOptions(); + return openReadOnly(options, path, columnFamilyDescriptors, columnFamilyHandles, false); + } + + /** + * The factory constructor of RocksDB that opens a RocksDB instance in + * Read-Only mode given the path to the database using the specified + * options and db path. + * + *

    This open method allows to open RocksDB using a subset of available + * column families

    + *

    Options instance *should* not be disposed before all DBs using this + * options instance have been closed. If user doesn't call options dispose + * explicitly,then this options instance will be GC'd automatically.

    + * + * @param options {@link DBOptions} instance. + * @param path the path to the RocksDB. + * @param columnFamilyDescriptors list of column family descriptors + * @param columnFamilyHandles will be filled with ColumnFamilyHandle instances + * on open. + * @return a {@link RocksDB} instance on success, null if the specified + * {@link RocksDB} can not be opened. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public static RocksDB openReadOnly(final DBOptions options, final String path, + final List columnFamilyDescriptors, + final List columnFamilyHandles) throws RocksDBException { + return openReadOnly(options, path, columnFamilyDescriptors, columnFamilyHandles, false); + } + + /** + * The factory constructor of RocksDB that opens a RocksDB instance in + * Read-Only mode given the path to the database using the specified + * options and db path. + * + *

    This open method allows to open RocksDB using a subset of available + * column families

    + *

    Options instance *should* not be disposed before all DBs using this + * options instance have been closed. If user doesn't call options dispose + * explicitly,then this options instance will be GC'd automatically.

    + * + * @param options {@link DBOptions} instance. + * @param path the path to the RocksDB. + * @param columnFamilyDescriptors list of column family descriptors + * @param columnFamilyHandles will be filled with ColumnFamilyHandle instances + * on open. + * @param errorIfWalFileExists true to raise an error when opening the db + * if a Write Ahead Log file exists, false otherwise. + * @return a {@link RocksDB} instance on success, null if the specified + * {@link RocksDB} can not be opened. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public static RocksDB openReadOnly(final DBOptions options, final String path, + final List columnFamilyDescriptors, + final List columnFamilyHandles, final boolean errorIfWalFileExists) + throws RocksDBException { + // when non-default Options is used, keeping an Options reference + // in RocksDB can prevent Java to GC during the life-time of + // the currently-created RocksDB. + + final byte[][] cfNames = new byte[columnFamilyDescriptors.size()][]; + final long[] cfOptionHandles = new long[columnFamilyDescriptors.size()]; + for (int i = 0; i < columnFamilyDescriptors.size(); i++) { + final ColumnFamilyDescriptor cfDescriptor = columnFamilyDescriptors + .get(i); + cfNames[i] = cfDescriptor.getName(); + cfOptionHandles[i] = cfDescriptor.getOptions().nativeHandle_; + } + + final long[] handles = + openROnly(options.nativeHandle_, path, cfNames, cfOptionHandles, errorIfWalFileExists); + final RocksDB db = new RocksDB(handles[0]); + db.storeOptionsInstance(options); + + for (int i = 1; i < handles.length; i++) { + final ColumnFamilyHandle columnFamilyHandle = new ColumnFamilyHandle(db, handles[i]); + columnFamilyHandles.add(columnFamilyHandle); + } + + db.ownedColumnFamilyHandles.addAll(columnFamilyHandles); + + return db; + } + + /** + * Open DB as secondary instance with only the default column family. + *

    + * The secondary instance can dynamically tail the MANIFEST of + * a primary that must have already been created. User can call + * {@link #tryCatchUpWithPrimary()} to make the secondary instance catch up + * with primary (WAL tailing is NOT supported now) whenever the user feels + * necessary. Column families created by the primary after the secondary + * instance starts are currently ignored by the secondary instance. + * Column families opened by secondary and dropped by the primary will be + * dropped by secondary as well. However the user of the secondary instance + * can still access the data of such dropped column family as long as they + * do not destroy the corresponding column family handle. + * WAL tailing is not supported at present, but will arrive soon. + * + * @param options the options to open the secondary instance. + * @param path the path to the primary RocksDB instance. + * @param secondaryPath points to a directory where the secondary instance + * stores its info log + * + * @return a {@link RocksDB} instance on success, null if the specified + * {@link RocksDB} can not be opened. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public static RocksDB openAsSecondary(final Options options, final String path, + final String secondaryPath) throws RocksDBException { + // when non-default Options is used, keeping an Options reference + // in RocksDB can prevent Java to GC during the life-time of + // the currently-created RocksDB. + final RocksDB db = new RocksDB(openAsSecondary(options.nativeHandle_, path, secondaryPath)); + db.storeOptionsInstance(options); + return db; + } + + /** + * Open DB as secondary instance with column families. + * You can open a subset of column families in secondary mode. + *

    + * The secondary instance can dynamically tail the MANIFEST of + * a primary that must have already been created. User can call + * {@link #tryCatchUpWithPrimary()} to make the secondary instance catch up + * with primary (WAL tailing is NOT supported now) whenever the user feels + * necessary. Column families created by the primary after the secondary + * instance starts are currently ignored by the secondary instance. + * Column families opened by secondary and dropped by the primary will be + * dropped by secondary as well. However the user of the secondary instance + * can still access the data of such dropped column family as long as they + * do not destroy the corresponding column family handle. + * WAL tailing is not supported at present, but will arrive soon. + * + * @param options the options to open the secondary instance. + * @param path the path to the primary RocksDB instance. + * @param secondaryPath points to a directory where the secondary instance + * stores its info log. + * @param columnFamilyDescriptors list of column family descriptors + * @param columnFamilyHandles will be filled with ColumnFamilyHandle instances + * on open. + * + * @return a {@link RocksDB} instance on success, null if the specified + * {@link RocksDB} can not be opened. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public static RocksDB openAsSecondary(final DBOptions options, final String path, + final String secondaryPath, final List columnFamilyDescriptors, + final List columnFamilyHandles) throws RocksDBException { + // when non-default Options is used, keeping an Options reference + // in RocksDB can prevent Java to GC during the life-time of + // the currently-created RocksDB. + + final byte[][] cfNames = new byte[columnFamilyDescriptors.size()][]; + final long[] cfOptionHandles = new long[columnFamilyDescriptors.size()]; + for (int i = 0; i < columnFamilyDescriptors.size(); i++) { + final ColumnFamilyDescriptor cfDescriptor = columnFamilyDescriptors.get(i); + cfNames[i] = cfDescriptor.getName(); + cfOptionHandles[i] = cfDescriptor.getOptions().nativeHandle_; + } + + final long[] handles = + openAsSecondary(options.nativeHandle_, path, secondaryPath, cfNames, cfOptionHandles); + final RocksDB db = new RocksDB(handles[0]); + db.storeOptionsInstance(options); + + for (int i = 1; i < handles.length; i++) { + final ColumnFamilyHandle columnFamilyHandle = new ColumnFamilyHandle(db, handles[i]); + columnFamilyHandles.add(columnFamilyHandle); + } + + db.ownedColumnFamilyHandles.addAll(columnFamilyHandles); + + return db; + } + + /** + * This is similar to {@link #close()} except that it + * throws an exception if any error occurs. + *

    + * This will not fsync the WAL files. + * If syncing is required, the caller must first call {@link #syncWal()} + * or {@link #write(WriteOptions, WriteBatch)} using an empty write batch + * with {@link WriteOptions#setSync(boolean)} set to true. + *

    + * See also {@link #close()}. + * + * @throws RocksDBException if an error occurs whilst closing. + */ + public void closeE() throws RocksDBException { + for (final ColumnFamilyHandle columnFamilyHandle : ownedColumnFamilyHandles) { + columnFamilyHandle.close(); + } + ownedColumnFamilyHandles.clear(); + + if (owningHandle_.compareAndSet(true, false)) { + try { + closeDatabase(nativeHandle_); + } finally { + disposeInternal(); + } + } + } + + /** + * This is similar to {@link #closeE()} except that it + * silently ignores any errors. + *

    + * This will not fsync the WAL files. + * If syncing is required, the caller must first call {@link #syncWal()} + * or {@link #write(WriteOptions, WriteBatch)} using an empty write batch + * with {@link WriteOptions#setSync(boolean)} set to true. + *

    + * See also {@link #close()}. + */ + @Override + public void close() { + for (final ColumnFamilyHandle columnFamilyHandle : ownedColumnFamilyHandles) { + columnFamilyHandle.close(); + } + ownedColumnFamilyHandles.clear(); + + if (owningHandle_.compareAndSet(true, false)) { + try { + closeDatabase(nativeHandle_); + } catch (final RocksDBException e) { + // silently ignore the error report + } finally { + disposeInternal(); + } + } + } + + /** + * Static method to determine all available column families for a + * rocksdb database identified by path + * + * @param options Options for opening the database + * @param path Absolute path to rocksdb database + * @return List<byte[]> List containing the column family names + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public static List listColumnFamilies(final Options options, + final String path) throws RocksDBException { + return Arrays.asList(RocksDB.listColumnFamilies(options.nativeHandle_, + path)); + } + + /** + * Creates a new column family with the name columnFamilyName and + * allocates a ColumnFamilyHandle within an internal structure. + * The ColumnFamilyHandle is automatically disposed with DB disposal. + * + * @param columnFamilyDescriptor column family to be created. + * @return {@link org.rocksdb.ColumnFamilyHandle} instance. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public ColumnFamilyHandle createColumnFamily( + final ColumnFamilyDescriptor columnFamilyDescriptor) + throws RocksDBException { + final ColumnFamilyHandle columnFamilyHandle = new ColumnFamilyHandle(this, + createColumnFamily(nativeHandle_, columnFamilyDescriptor.getName(), + columnFamilyDescriptor.getName().length, + columnFamilyDescriptor.getOptions().nativeHandle_)); + ownedColumnFamilyHandles.add(columnFamilyHandle); + return columnFamilyHandle; + } + + /** + * Bulk create column families with the same column family options. + * + * @param columnFamilyOptions the options for the column families. + * @param columnFamilyNames the names of the column families. + * + * @return the handles to the newly created column families. + * + * @throws RocksDBException if an error occurs whilst creating + * the column families + */ + public List createColumnFamilies( + final ColumnFamilyOptions columnFamilyOptions, + final List columnFamilyNames) throws RocksDBException { + final byte[][] cfNames = columnFamilyNames.toArray( + new byte[0][]); + final long[] cfHandles = createColumnFamilies(nativeHandle_, + columnFamilyOptions.nativeHandle_, cfNames); + final List columnFamilyHandles = + new ArrayList<>(cfHandles.length); + for (final long cfHandle : cfHandles) { + final ColumnFamilyHandle columnFamilyHandle = new ColumnFamilyHandle(this, cfHandle); + columnFamilyHandles.add(columnFamilyHandle); + } + ownedColumnFamilyHandles.addAll(columnFamilyHandles); + return columnFamilyHandles; + } + + /** + * Bulk create column families with the same column family options. + * + * @param columnFamilyDescriptors the descriptions of the column families. + * + * @return the handles to the newly created column families. + * + * @throws RocksDBException if an error occurs whilst creating + * the column families + */ + public List createColumnFamilies( + final List columnFamilyDescriptors) + throws RocksDBException { + final long[] cfOptsHandles = new long[columnFamilyDescriptors.size()]; + final byte[][] cfNames = new byte[columnFamilyDescriptors.size()][]; + for (int i = 0; i < columnFamilyDescriptors.size(); i++) { + final ColumnFamilyDescriptor columnFamilyDescriptor + = columnFamilyDescriptors.get(i); + cfOptsHandles[i] = columnFamilyDescriptor.getOptions().nativeHandle_; + cfNames[i] = columnFamilyDescriptor.getName(); + } + final long[] cfHandles = createColumnFamilies(nativeHandle_, + cfOptsHandles, cfNames); + final List columnFamilyHandles = + new ArrayList<>(cfHandles.length); + for (final long cfHandle : cfHandles) { + final ColumnFamilyHandle columnFamilyHandle = new ColumnFamilyHandle(this, cfHandle); + columnFamilyHandles.add(columnFamilyHandle); + } + ownedColumnFamilyHandles.addAll(columnFamilyHandles); + return columnFamilyHandles; + } + + /** + * Drops the column family specified by {@code columnFamilyHandle}. This call + * only records a drop record in the manifest and prevents the column + * family from flushing and compacting. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void dropColumnFamily(final ColumnFamilyHandle columnFamilyHandle) + throws RocksDBException { + dropColumnFamily(nativeHandle_, columnFamilyHandle.nativeHandle_); + } + + // Bulk drop column families. This call only records drop records in the + // manifest and prevents the column families from flushing and compacting. + // In case of error, the request may succeed partially. User may call + // ListColumnFamilies to check the result. + public void dropColumnFamilies( + final List columnFamilies) throws RocksDBException { + final long[] cfHandles = new long[columnFamilies.size()]; + for (int i = 0; i < columnFamilies.size(); i++) { + cfHandles[i] = columnFamilies.get(i).nativeHandle_; + } + dropColumnFamilies(nativeHandle_, cfHandles); + } + + /** + * Deletes native column family handle of given {@link ColumnFamilyHandle} Java object + * and removes reference from {@link RocksDB#ownedColumnFamilyHandles}. + * + * @param columnFamilyHandle column family handle object. + */ + public void destroyColumnFamilyHandle(final ColumnFamilyHandle columnFamilyHandle) { + for (int i = 0; i < ownedColumnFamilyHandles.size(); ++i) { + final ColumnFamilyHandle ownedHandle = ownedColumnFamilyHandles.get(i); + if (ownedHandle.equals(columnFamilyHandle)) { + columnFamilyHandle.close(); + ownedColumnFamilyHandles.remove(i); + return; + } + } + } + + /** + * Set the database entry for "key" to "value". + * + * @param key the specified key to be inserted. + * @param value the value associated with the specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void put(final byte[] key, final byte[] value) + throws RocksDBException { + put(nativeHandle_, key, 0, key.length, value, 0, value.length); + } + + /** + * Set the database entry for "key" to "value". + * + * @param key The specified key to be inserted + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than ("key".length - offset) + * @param value the value associated with the specified key + * @param vOffset the offset of the "value" array to be used, must be + * non-negative and no longer than "key".length + * @param vLen the length of the "value" array to be used, must be + * non-negative and no larger than ("value".length - offset) + * + * @throws RocksDBException thrown if errors happens in underlying native + * library. + * @throws IndexOutOfBoundsException if an offset or length is out of bounds + */ + public void put(final byte[] key, final int offset, final int len, + final byte[] value, final int vOffset, final int vLen) + throws RocksDBException { + checkBounds(offset, len, key.length); + checkBounds(vOffset, vLen, value.length); + put(nativeHandle_, key, offset, len, value, vOffset, vLen); + } + + /** + * Set the database entry for "key" to "value" in the specified + * column family. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param key the specified key to be inserted. + * @param value the value associated with the specified key. + *

    + * throws IllegalArgumentException if column family is not present + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void put(final ColumnFamilyHandle columnFamilyHandle, + final byte[] key, final byte[] value) throws RocksDBException { + put(nativeHandle_, key, 0, key.length, value, 0, value.length, + columnFamilyHandle.nativeHandle_); + } + + /** + * Set the database entry for "key" to "value" in the specified + * column family. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param key The specified key to be inserted + * @param offset the offset of the "key" array to be used, must + * be non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than ("key".length - offset) + * @param value the value associated with the specified key + * @param vOffset the offset of the "value" array to be used, must be + * non-negative and no longer than "key".length + * @param vLen the length of the "value" array to be used, must be + * non-negative and no larger than ("value".length - offset) + * + * @throws RocksDBException thrown if errors happens in underlying native + * library. + * @throws IndexOutOfBoundsException if an offset or length is out of bounds + */ + public void put(final ColumnFamilyHandle columnFamilyHandle, + final byte[] key, final int offset, final int len, + final byte[] value, final int vOffset, final int vLen) + throws RocksDBException { + checkBounds(offset, len, key.length); + checkBounds(vOffset, vLen, value.length); + put(nativeHandle_, key, offset, len, value, vOffset, vLen, + columnFamilyHandle.nativeHandle_); + } + + /** + * Set the database entry for "key" to "value". + * + * @param writeOpts {@link org.rocksdb.WriteOptions} instance. + * @param key the specified key to be inserted. + * @param value the value associated with the specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void put(final WriteOptions writeOpts, final byte[] key, + final byte[] value) throws RocksDBException { + put(nativeHandle_, writeOpts.nativeHandle_, + key, 0, key.length, value, 0, value.length); + } + + /** + * Set the database entry for "key" to "value". + * + * @param writeOpts {@link org.rocksdb.WriteOptions} instance. + * @param key The specified key to be inserted + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than ("key".length - offset) + * @param value the value associated with the specified key + * @param vOffset the offset of the "value" array to be used, must be + * non-negative and no longer than "key".length + * @param vLen the length of the "value" array to be used, must be + * non-negative and no larger than ("value".length - offset) + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + * @throws IndexOutOfBoundsException if an offset or length is out of bounds + */ + public void put(final WriteOptions writeOpts, + final byte[] key, final int offset, final int len, + final byte[] value, final int vOffset, final int vLen) + throws RocksDBException { + checkBounds(offset, len, key.length); + checkBounds(vOffset, vLen, value.length); + put(nativeHandle_, writeOpts.nativeHandle_, + key, offset, len, value, vOffset, vLen); + } + + /** + * Set the database entry for "key" to "value" for the specified + * column family. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param writeOpts {@link org.rocksdb.WriteOptions} instance. + * @param key the specified key to be inserted. + * @param value the value associated with the specified key. + *

    + * throws IllegalArgumentException if column family is not present + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + * @see IllegalArgumentException + */ + public void put(final ColumnFamilyHandle columnFamilyHandle, + final WriteOptions writeOpts, final byte[] key, + final byte[] value) throws RocksDBException { + put(nativeHandle_, writeOpts.nativeHandle_, key, 0, key.length, value, + 0, value.length, columnFamilyHandle.nativeHandle_); + } + + /** + * Set the database entry for "key" to "value" for the specified + * column family. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param writeOpts {@link org.rocksdb.WriteOptions} instance. + * @param key the specified key to be inserted. Position and limit is used. + * Supports direct buffer only. + * @param value the value associated with the specified key. Position and limit is used. + * Supports direct buffer only. + *

    + * throws IllegalArgumentException if column family is not present + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + * @see IllegalArgumentException + */ + public void put(final ColumnFamilyHandle columnFamilyHandle, final WriteOptions writeOpts, + final ByteBuffer key, final ByteBuffer value) throws RocksDBException { + assert key.isDirect() && value.isDirect(); + putDirect(nativeHandle_, writeOpts.nativeHandle_, key, key.position(), key.remaining(), value, + value.position(), value.remaining(), columnFamilyHandle.nativeHandle_); + key.position(key.limit()); + value.position(value.limit()); + } + + /** + * Set the database entry for "key" to "value". + * + * @param writeOpts {@link org.rocksdb.WriteOptions} instance. + * @param key the specified key to be inserted. Position and limit is used. + * Supports direct buffer only. + * @param value the value associated with the specified key. Position and limit is used. + * Supports direct buffer only. + *

    + * throws IllegalArgumentException if column family is not present + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + * @see IllegalArgumentException + */ + public void put(final WriteOptions writeOpts, final ByteBuffer key, final ByteBuffer value) + throws RocksDBException { + assert key.isDirect() && value.isDirect(); + putDirect(nativeHandle_, writeOpts.nativeHandle_, key, key.position(), key.remaining(), value, + value.position(), value.remaining(), 0); + key.position(key.limit()); + value.position(value.limit()); + } + + /** + * Set the database entry for "key" to "value" for the specified + * column family. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param writeOpts {@link org.rocksdb.WriteOptions} instance. + * @param key The specified key to be inserted + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than ("key".length - offset) + * @param value the value associated with the specified key + * @param vOffset the offset of the "value" array to be used, must be + * non-negative and no longer than "key".length + * @param vLen the length of the "value" array to be used, must be + * non-negative and no larger than ("value".length - offset) + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + * @throws IndexOutOfBoundsException if an offset or length is out of bounds + */ + public void put(final ColumnFamilyHandle columnFamilyHandle, + final WriteOptions writeOpts, + final byte[] key, final int offset, final int len, + final byte[] value, final int vOffset, final int vLen) + throws RocksDBException { + checkBounds(offset, len, key.length); + checkBounds(vOffset, vLen, value.length); + put(nativeHandle_, writeOpts.nativeHandle_, key, offset, len, value, + vOffset, vLen, columnFamilyHandle.nativeHandle_); + } + + /** + * Delete the database entry (if any) for "key". Returns OK on + * success, and a non-OK status on error. It is not an error if "key" + * did not exist in the database. + * + * @param key Key to delete within database + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void delete(final byte[] key) throws RocksDBException { + delete(nativeHandle_, key, 0, key.length); + } + + /** + * Delete the database entry (if any) for "key". Returns OK on + * success, and a non-OK status on error. It is not an error if "key" + * did not exist in the database. + * + * @param key Key to delete within database + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be + * non-negative and no larger than ("key".length - offset) + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void delete(final byte[] key, final int offset, final int len) + throws RocksDBException { + delete(nativeHandle_, key, offset, len); + } + + /** + * Delete the database entry (if any) for "key". Returns OK on + * success, and a non-OK status on error. It is not an error if "key" + * did not exist in the database. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param key Key to delete within database + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void delete(final ColumnFamilyHandle columnFamilyHandle, + final byte[] key) throws RocksDBException { + delete(nativeHandle_, key, 0, key.length, columnFamilyHandle.nativeHandle_); + } + + /** + * Delete the database entry (if any) for "key". Returns OK on + * success, and a non-OK status on error. It is not an error if "key" + * did not exist in the database. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param key Key to delete within database + * @param offset the offset of the "key" array to be used, + * must be non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than ("value".length - offset) + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void delete(final ColumnFamilyHandle columnFamilyHandle, + final byte[] key, final int offset, final int len) + throws RocksDBException { + delete(nativeHandle_, key, offset, len, columnFamilyHandle.nativeHandle_); + } + + /** + * Delete the database entry (if any) for "key". Returns OK on + * success, and a non-OK status on error. It is not an error if "key" + * did not exist in the database. + * + * @param writeOpt WriteOptions to be used with delete operation + * @param key Key to delete within database + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void delete(final WriteOptions writeOpt, final byte[] key) + throws RocksDBException { + delete(nativeHandle_, writeOpt.nativeHandle_, key, 0, key.length); + } + + /** + * Delete the database entry (if any) for "key". Returns OK on + * success, and a non-OK status on error. It is not an error if "key" + * did not exist in the database. + * + * @param writeOpt WriteOptions to be used with delete operation + * @param key Key to delete within database + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be + * non-negative and no larger than ("key".length - offset) + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void delete(final WriteOptions writeOpt, final byte[] key, + final int offset, final int len) throws RocksDBException { + delete(nativeHandle_, writeOpt.nativeHandle_, key, offset, len); + } + + /** + * Delete the database entry (if any) for "key". Returns OK on + * success, and a non-OK status on error. It is not an error if "key" + * did not exist in the database. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param writeOpt WriteOptions to be used with delete operation + * @param key Key to delete within database + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void delete(final ColumnFamilyHandle columnFamilyHandle, + final WriteOptions writeOpt, final byte[] key) + throws RocksDBException { + delete(nativeHandle_, writeOpt.nativeHandle_, key, 0, key.length, + columnFamilyHandle.nativeHandle_); + } + + /** + * Delete the database entry (if any) for "key". Returns OK on + * success, and a non-OK status on error. It is not an error if "key" + * did not exist in the database. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param writeOpt WriteOptions to be used with delete operation + * @param key Key to delete within database + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be + * non-negative and no larger than ("key".length - offset) + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void delete(final ColumnFamilyHandle columnFamilyHandle, + final WriteOptions writeOpt, final byte[] key, final int offset, + final int len) throws RocksDBException { + delete(nativeHandle_, writeOpt.nativeHandle_, key, offset, len, + columnFamilyHandle.nativeHandle_); + } + + /** + * Get the value associated with the specified key within column family. + * + * @param opt {@link org.rocksdb.ReadOptions} instance. + * @param key the key to retrieve the value. It is using position and limit. + * Supports direct buffer only. + * @param value the out-value to receive the retrieved value. + * It is using position and limit. Limit is set according to value size. + * Supports direct buffer only. + * @return The size of the actual value that matches the specified + * {@code key} in byte. If the return value is greater than the + * length of {@code value}, then it indicates that the size of the + * input buffer {@code value} is insufficient and partial result will + * be returned. RocksDB.NOT_FOUND will be returned if the value not + * found. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public int get(final ReadOptions opt, final ByteBuffer key, final ByteBuffer value) + throws RocksDBException { + assert key.isDirect() && value.isDirect(); + final int result = getDirect(nativeHandle_, opt.nativeHandle_, key, key.position(), + key.remaining(), value, value.position(), value.remaining(), 0); + if (result != NOT_FOUND) { + value.limit(Math.min(value.limit(), value.position() + result)); + } + key.position(key.limit()); + return result; + } + + /** + * Get the value associated with the specified key within column family. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param opt {@link org.rocksdb.ReadOptions} instance. + * @param key the key to retrieve the value. It is using position and limit. + * Supports direct buffer only. + * @param value the out-value to receive the retrieved value. + * It is using position and limit. Limit is set according to value size. + * Supports direct buffer only. + * @return The size of the actual value that matches the specified + * {@code key} in byte. If the return value is greater than the + * length of {@code value}, then it indicates that the size of the + * input buffer {@code value} is insufficient and partial result will + * be returned. RocksDB.NOT_FOUND will be returned if the value not + * found. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public int get(final ColumnFamilyHandle columnFamilyHandle, final ReadOptions opt, + final ByteBuffer key, final ByteBuffer value) throws RocksDBException { + assert key.isDirect() && value.isDirect(); + final int result = + getDirect(nativeHandle_, opt.nativeHandle_, key, key.position(), key.remaining(), value, + value.position(), value.remaining(), columnFamilyHandle.nativeHandle_); + if (result != NOT_FOUND) { + value.limit(Math.min(value.limit(), value.position() + result)); + } + key.position(key.limit()); + return result; + } + + /** + * Remove the database entry for {@code key}. Requires that the key exists + * and was not overwritten. It is not an error if the key did not exist + * in the database. + *

    + * If a key is overwritten (by calling {@link #put(byte[], byte[])} multiple + * times), then the result of calling SingleDelete() on this key is undefined. + * SingleDelete() only behaves correctly if there has been only one Put() + * for this key since the previous call to SingleDelete() for this key. + *

    + * This feature is currently an experimental performance optimization + * for a very specific workload. It is up to the caller to ensure that + * SingleDelete is only used for a key that is not deleted using Delete() or + * written using Merge(). Mixing SingleDelete operations with Deletes and + * Merges can result in undefined behavior. + * + * @param key Key to delete within database + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + @Experimental("Performance optimization for a very specific workload") + public void singleDelete(final byte[] key) throws RocksDBException { + singleDelete(nativeHandle_, key, key.length); + } + + /** + * Remove the database entry for {@code key}. Requires that the key exists + * and was not overwritten. It is not an error if the key did not exist + * in the database. + *

    + * If a key is overwritten (by calling {@link #put(byte[], byte[])} multiple + * times), then the result of calling SingleDelete() on this key is undefined. + * SingleDelete() only behaves correctly if there has been only one Put() + * for this key since the previous call to SingleDelete() for this key. + *

    + * This feature is currently an experimental performance optimization + * for a very specific workload. It is up to the caller to ensure that + * SingleDelete is only used for a key that is not deleted using Delete() or + * written using Merge(). Mixing SingleDelete operations with Deletes and + * Merges can result in undefined behavior. + * + * @param columnFamilyHandle The column family to delete the key from + * @param key Key to delete within database + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + @Experimental("Performance optimization for a very specific workload") + public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, + final byte[] key) throws RocksDBException { + singleDelete(nativeHandle_, key, key.length, + columnFamilyHandle.nativeHandle_); + } + + /** + * Remove the database entry for {@code key}. Requires that the key exists + * and was not overwritten. It is not an error if the key did not exist + * in the database. + *

    + * If a key is overwritten (by calling {@link #put(byte[], byte[])} multiple + * times), then the result of calling SingleDelete() on this key is undefined. + * SingleDelete() only behaves correctly if there has been only one Put() + * for this key since the previous call to SingleDelete() for this key. + *

    + * This feature is currently an experimental performance optimization + * for a very specific workload. It is up to the caller to ensure that + * SingleDelete is only used for a key that is not deleted using Delete() or + * written using Merge(). Mixing SingleDelete operations with Deletes and + * Merges can result in undefined behavior. + *

    + * Note: consider setting {@link WriteOptions#setSync(boolean)} true. + * + * @param writeOpt Write options for the delete + * @param key Key to delete within database + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + @Experimental("Performance optimization for a very specific workload") + public void singleDelete(final WriteOptions writeOpt, final byte[] key) + throws RocksDBException { + singleDelete(nativeHandle_, writeOpt.nativeHandle_, key, key.length); + } + + /** + * Remove the database entry for {@code key}. Requires that the key exists + * and was not overwritten. It is not an error if the key did not exist + * in the database. + *

    + * If a key is overwritten (by calling {@link #put(byte[], byte[])} multiple + * times), then the result of calling SingleDelete() on this key is undefined. + * SingleDelete() only behaves correctly if there has been only one Put() + * for this key since the previous call to SingleDelete() for this key. + *

    + * This feature is currently an experimental performance optimization + * for a very specific workload. It is up to the caller to ensure that + * SingleDelete is only used for a key that is not deleted using Delete() or + * written using Merge(). Mixing SingleDelete operations with Deletes and + * Merges can result in undefined behavior. + *

    + * Note: consider setting {@link WriteOptions#setSync(boolean)} true. + * + * @param columnFamilyHandle The column family to delete the key from + * @param writeOpt Write options for the delete + * @param key Key to delete within database + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + @Experimental("Performance optimization for a very specific workload") + public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, + final WriteOptions writeOpt, final byte[] key) throws RocksDBException { + singleDelete(nativeHandle_, writeOpt.nativeHandle_, key, key.length, + columnFamilyHandle.nativeHandle_); + } + + /** + * Removes the database entries in the range ["beginKey", "endKey"), i.e., + * including "beginKey" and excluding "endKey". a non-OK status on error. It + * is not an error if no keys exist in the range ["beginKey", "endKey"). + *

    + * Delete the database entry (if any) for "key". Returns OK on success, and a + * non-OK status on error. It is not an error if "key" did not exist in the + * database. + * + * @param beginKey First key to delete within database (inclusive) + * @param endKey Last key to delete within database (exclusive) + * + * @throws RocksDBException thrown if error happens in underlying native + * library. + */ + public void deleteRange(final byte[] beginKey, final byte[] endKey) + throws RocksDBException { + deleteRange(nativeHandle_, beginKey, 0, beginKey.length, endKey, 0, + endKey.length); + } + + /** + * Removes the database entries in the range ["beginKey", "endKey"), i.e., + * including "beginKey" and excluding "endKey". a non-OK status on error. It + * is not an error if no keys exist in the range ["beginKey", "endKey"). + *

    + * Delete the database entry (if any) for "key". Returns OK on success, and a + * non-OK status on error. It is not an error if "key" did not exist in the + * database. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} instance + * @param beginKey First key to delete within database (inclusive) + * @param endKey Last key to delete within database (exclusive) + * + * @throws RocksDBException thrown if error happens in underlying native + * library. + */ + public void deleteRange(final ColumnFamilyHandle columnFamilyHandle, + final byte[] beginKey, final byte[] endKey) throws RocksDBException { + deleteRange(nativeHandle_, beginKey, 0, beginKey.length, endKey, 0, + endKey.length, columnFamilyHandle.nativeHandle_); + } + + /** + * Removes the database entries in the range ["beginKey", "endKey"), i.e., + * including "beginKey" and excluding "endKey". a non-OK status on error. It + * is not an error if no keys exist in the range ["beginKey", "endKey"). + *

    + * Delete the database entry (if any) for "key". Returns OK on success, and a + * non-OK status on error. It is not an error if "key" did not exist in the + * database. + * + * @param writeOpt WriteOptions to be used with delete operation + * @param beginKey First key to delete within database (inclusive) + * @param endKey Last key to delete within database (exclusive) + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void deleteRange(final WriteOptions writeOpt, final byte[] beginKey, + final byte[] endKey) throws RocksDBException { + deleteRange(nativeHandle_, writeOpt.nativeHandle_, beginKey, 0, + beginKey.length, endKey, 0, endKey.length); + } + + /** + * Removes the database entries in the range ["beginKey", "endKey"), i.e., + * including "beginKey" and excluding "endKey". a non-OK status on error. It + * is not an error if no keys exist in the range ["beginKey", "endKey"). + *

    + * Delete the database entry (if any) for "key". Returns OK on success, and a + * non-OK status on error. It is not an error if "key" did not exist in the + * database. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} instance + * @param writeOpt WriteOptions to be used with delete operation + * @param beginKey First key to delete within database (included) + * @param endKey Last key to delete within database (excluded) + * + * @throws RocksDBException thrown if error happens in underlying native + * library. + */ + public void deleteRange(final ColumnFamilyHandle columnFamilyHandle, + final WriteOptions writeOpt, final byte[] beginKey, final byte[] endKey) + throws RocksDBException { + deleteRange(nativeHandle_, writeOpt.nativeHandle_, beginKey, 0, + beginKey.length, endKey, 0, endKey.length, + columnFamilyHandle.nativeHandle_); + } + + + /** + * Add merge operand for key/value pair. + * + * @param key the specified key to be merged. + * @param value the value to be merged with the current value for the + * specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void merge(final byte[] key, final byte[] value) + throws RocksDBException { + merge(nativeHandle_, key, 0, key.length, value, 0, value.length); + } + + /** + * Add merge operand for key/value pair. + * + * @param key the specified key to be merged. + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than ("key".length - offset) + * @param value the value to be merged with the current value for the + * specified key. + * @param vOffset the offset of the "value" array to be used, must be + * non-negative and no longer than "key".length + * @param vLen the length of the "value" array to be used, must be + * non-negative and must be non-negative and no larger than + * ("value".length - offset) + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + * @throws IndexOutOfBoundsException if an offset or length is out of bounds + */ + public void merge(final byte[] key, final int offset, final int len, final byte[] value, + final int vOffset, final int vLen) throws RocksDBException { + checkBounds(offset, len, key.length); + checkBounds(vOffset, vLen, value.length); + merge(nativeHandle_, key, offset, len, value, vOffset, vLen); + } + + /** + * Add merge operand for key/value pair in a ColumnFamily. + * + * @param columnFamilyHandle {@link ColumnFamilyHandle} instance + * @param key the specified key to be merged. + * @param value the value to be merged with the current value for + * the specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void merge(final ColumnFamilyHandle columnFamilyHandle, + final byte[] key, final byte[] value) throws RocksDBException { + merge(nativeHandle_, key, 0, key.length, value, 0, value.length, + columnFamilyHandle.nativeHandle_); + } + + /** + * Add merge operand for key/value pair in a ColumnFamily. + * + * @param columnFamilyHandle {@link ColumnFamilyHandle} instance + * @param key the specified key to be merged. + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than ("key".length - offset) + * @param value the value to be merged with the current value for + * the specified key. + * @param vOffset the offset of the "value" array to be used, must be + * non-negative and no longer than "key".length + * @param vLen the length of the "value" array to be used, must be + * must be non-negative and no larger than ("value".length - offset) + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + * @throws IndexOutOfBoundsException if an offset or length is out of bounds + */ + public void merge(final ColumnFamilyHandle columnFamilyHandle, + final byte[] key, final int offset, final int len, final byte[] value, + final int vOffset, final int vLen) throws RocksDBException { + checkBounds(offset, len, key.length); + checkBounds(vOffset, vLen, value.length); + merge(nativeHandle_, key, offset, len, value, vOffset, vLen, + columnFamilyHandle.nativeHandle_); + } + + /** + * Add merge operand for key/value pair. + * + * @param writeOpts {@link WriteOptions} for this write. + * @param key the specified key to be merged. + * @param value the value to be merged with the current value for + * the specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void merge(final WriteOptions writeOpts, final byte[] key, + final byte[] value) throws RocksDBException { + merge(nativeHandle_, writeOpts.nativeHandle_, + key, 0, key.length, value, 0, value.length); + } + + /** + * Add merge operand for key/value pair. + * + * @param writeOpts {@link WriteOptions} for this write. + * @param key the specified key to be merged. + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than ("value".length - offset) + * @param value the value to be merged with the current value for + * the specified key. + * @param vOffset the offset of the "value" array to be used, must be + * non-negative and no longer than "key".length + * @param vLen the length of the "value" array to be used, must be + * non-negative and no larger than ("value".length - offset) + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + * @throws IndexOutOfBoundsException if an offset or length is out of bounds + */ + public void merge(final WriteOptions writeOpts, + final byte[] key, final int offset, final int len, + final byte[] value, final int vOffset, final int vLen) + throws RocksDBException { + checkBounds(offset, len, key.length); + checkBounds(vOffset, vLen, value.length); + merge(nativeHandle_, writeOpts.nativeHandle_, + key, offset, len, value, vOffset, vLen); + } + + /** + * Delete the database entry (if any) for "key". Returns OK on + * success, and a non-OK status on error. It is not an error if "key" + * did not exist in the database. + * + * @param writeOpt WriteOptions to be used with delete operation + * @param key Key to delete within database. It is using position and limit. + * Supports direct buffer only. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void delete(final WriteOptions writeOpt, final ByteBuffer key) throws RocksDBException { + assert key.isDirect(); + deleteDirect(nativeHandle_, writeOpt.nativeHandle_, key, key.position(), key.remaining(), 0); + key.position(key.limit()); + } + + /** + * Delete the database entry (if any) for "key". Returns OK on + * success, and a non-OK status on error. It is not an error if "key" + * did not exist in the database. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param writeOpt WriteOptions to be used with delete operation + * @param key Key to delete within database. It is using position and limit. + * Supports direct buffer only. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void delete(final ColumnFamilyHandle columnFamilyHandle, final WriteOptions writeOpt, + final ByteBuffer key) throws RocksDBException { + assert key.isDirect(); + deleteDirect(nativeHandle_, writeOpt.nativeHandle_, key, key.position(), key.remaining(), + columnFamilyHandle.nativeHandle_); + key.position(key.limit()); + } + + /** + * Add merge operand for key/value pair. + * + * @param columnFamilyHandle {@link ColumnFamilyHandle} instance + * @param writeOpts {@link WriteOptions} for this write. + * @param key the specified key to be merged. + * @param value the value to be merged with the current value for the + * specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void merge(final ColumnFamilyHandle columnFamilyHandle, + final WriteOptions writeOpts, final byte[] key, final byte[] value) + throws RocksDBException { + merge(nativeHandle_, writeOpts.nativeHandle_, + key, 0, key.length, value, 0, value.length, + columnFamilyHandle.nativeHandle_); + } + + /** + * Add merge operand for key/value pair. + * + * @param columnFamilyHandle {@link ColumnFamilyHandle} instance + * @param writeOpts {@link WriteOptions} for this write. + * @param key the specified key to be merged. + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than ("key".length - offset) + * @param value the value to be merged with the current value for + * the specified key. + * @param vOffset the offset of the "value" array to be used, must be + * non-negative and no longer than "key".length + * @param vLen the length of the "value" array to be used, must be + * non-negative and no larger than ("value".length - offset) + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + * @throws IndexOutOfBoundsException if an offset or length is out of bounds + */ + public void merge( + final ColumnFamilyHandle columnFamilyHandle, final WriteOptions writeOpts, + final byte[] key, final int offset, final int len, + final byte[] value, final int vOffset, final int vLen) + throws RocksDBException { + checkBounds(offset, len, key.length); + checkBounds(vOffset, vLen, value.length); + merge(nativeHandle_, writeOpts.nativeHandle_, + key, offset, len, value, vOffset, vLen, + columnFamilyHandle.nativeHandle_); + } + + /** + * Apply the specified updates to the database. + * + * @param writeOpts WriteOptions instance + * @param updates WriteBatch instance + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void write(final WriteOptions writeOpts, final WriteBatch updates) + throws RocksDBException { + write0(nativeHandle_, writeOpts.nativeHandle_, updates.nativeHandle_); + } + + /** + * Apply the specified updates to the database. + * + * @param writeOpts WriteOptions instance + * @param updates WriteBatchWithIndex instance + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void write(final WriteOptions writeOpts, + final WriteBatchWithIndex updates) throws RocksDBException { + write1(nativeHandle_, writeOpts.nativeHandle_, updates.nativeHandle_); + } + + // TODO(AR) we should improve the #get() API, returning -1 (RocksDB.NOT_FOUND) is not very nice + // when we could communicate better status into, also the C++ code show that -2 could be returned + + /** + * Get the value associated with the specified key within column family* + * + * @param key the key to retrieve the value. + * @param value the out-value to receive the retrieved value. + * + * @return The size of the actual value that matches the specified + * {@code key} in byte. If the return value is greater than the + * length of {@code value}, then it indicates that the size of the + * input buffer {@code value} is insufficient and partial result will + * be returned. RocksDB.NOT_FOUND will be returned if the value not + * found. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public int get(final byte[] key, final byte[] value) throws RocksDBException { + return get(nativeHandle_, key, 0, key.length, value, 0, value.length); + } + + /** + * Get the value associated with the specified key within column family* + * + * @param key the key to retrieve the value. + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than ("key".length - offset) + * @param value the out-value to receive the retrieved value. + * @param vOffset the offset of the "value" array to be used, must be + * non-negative and no longer than "value".length + * @param vLen the length of the "value" array to be used, must be + * non-negative and and no larger than ("value".length - offset) + * + * @return The size of the actual value that matches the specified + * {@code key} in byte. If the return value is greater than the + * length of {@code value}, then it indicates that the size of the + * input buffer {@code value} is insufficient and partial result will + * be returned. RocksDB.NOT_FOUND will be returned if the value not + * found. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public int get(final byte[] key, final int offset, final int len, + final byte[] value, final int vOffset, final int vLen) + throws RocksDBException { + checkBounds(offset, len, key.length); + checkBounds(vOffset, vLen, value.length); + return get(nativeHandle_, key, offset, len, value, vOffset, vLen); + } + + /** + * Get the value associated with the specified key within column family. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param key the key to retrieve the value. + * @param value the out-value to receive the retrieved value. + * @return The size of the actual value that matches the specified + * {@code key} in byte. If the return value is greater than the + * length of {@code value}, then it indicates that the size of the + * input buffer {@code value} is insufficient and partial result will + * be returned. RocksDB.NOT_FOUND will be returned if the value not + * found. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public int get(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, + final byte[] value) throws RocksDBException, IllegalArgumentException { + return get(nativeHandle_, key, 0, key.length, value, 0, value.length, + columnFamilyHandle.nativeHandle_); + } + + /** + * Get the value associated with the specified key within column family. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param key the key to retrieve the value. + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * an no larger than ("key".length - offset) + * @param value the out-value to receive the retrieved value. + * @param vOffset the offset of the "value" array to be used, must be + * non-negative and no longer than "key".length + * @param vLen the length of the "value" array to be used, must be + * non-negative and no larger than ("value".length - offset) + * + * @return The size of the actual value that matches the specified + * {@code key} in byte. If the return value is greater than the + * length of {@code value}, then it indicates that the size of the + * input buffer {@code value} is insufficient and partial result will + * be returned. RocksDB.NOT_FOUND will be returned if the value not + * found. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public int get(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, + final int offset, final int len, final byte[] value, final int vOffset, + final int vLen) throws RocksDBException, IllegalArgumentException { + checkBounds(offset, len, key.length); + checkBounds(vOffset, vLen, value.length); + return get(nativeHandle_, key, offset, len, value, vOffset, vLen, + columnFamilyHandle.nativeHandle_); + } + + /** + * Get the value associated with the specified key. + * + * @param opt {@link org.rocksdb.ReadOptions} instance. + * @param key the key to retrieve the value. + * @param value the out-value to receive the retrieved value. + * @return The size of the actual value that matches the specified + * {@code key} in byte. If the return value is greater than the + * length of {@code value}, then it indicates that the size of the + * input buffer {@code value} is insufficient and partial result will + * be returned. RocksDB.NOT_FOUND will be returned if the value not + * found. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public int get(final ReadOptions opt, final byte[] key, + final byte[] value) throws RocksDBException { + return get(nativeHandle_, opt.nativeHandle_, + key, 0, key.length, value, 0, value.length); + } + + /** + * Get the value associated with the specified key. + * + * @param opt {@link org.rocksdb.ReadOptions} instance. + * @param key the key to retrieve the value. + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than ("key".length - offset) + * @param value the out-value to receive the retrieved value. + * @param vOffset the offset of the "value" array to be used, must be + * non-negative and no longer than "key".length + * @param vLen the length of the "value" array to be used, must be + * non-negative and no larger than ("value".length - offset) + * @return The size of the actual value that matches the specified + * {@code key} in byte. If the return value is greater than the + * length of {@code value}, then it indicates that the size of the + * input buffer {@code value} is insufficient and partial result will + * be returned. RocksDB.NOT_FOUND will be returned if the value not + * found. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public int get(final ReadOptions opt, final byte[] key, final int offset, + final int len, final byte[] value, final int vOffset, final int vLen) + throws RocksDBException { + checkBounds(offset, len, key.length); + checkBounds(vOffset, vLen, value.length); + return get(nativeHandle_, opt.nativeHandle_, + key, offset, len, value, vOffset, vLen); + } + + /** + * Get the value associated with the specified key within column family. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param opt {@link org.rocksdb.ReadOptions} instance. + * @param key the key to retrieve the value. + * @param value the out-value to receive the retrieved value. + * @return The size of the actual value that matches the specified + * {@code key} in byte. If the return value is greater than the + * length of {@code value}, then it indicates that the size of the + * input buffer {@code value} is insufficient and partial result will + * be returned. RocksDB.NOT_FOUND will be returned if the value not + * found. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public int get(final ColumnFamilyHandle columnFamilyHandle, + final ReadOptions opt, final byte[] key, final byte[] value) + throws RocksDBException { + return get(nativeHandle_, opt.nativeHandle_, key, 0, key.length, value, + 0, value.length, columnFamilyHandle.nativeHandle_); + } + + /** + * Get the value associated with the specified key within column family. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param opt {@link org.rocksdb.ReadOptions} instance. + * @param key the key to retrieve the value. + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be + * non-negative and and no larger than ("key".length - offset) + * @param value the out-value to receive the retrieved value. + * @param vOffset the offset of the "value" array to be used, must be + * non-negative and no longer than "key".length + * @param vLen the length of the "value" array to be used, and must be + * non-negative and no larger than ("value".length - offset) + * @return The size of the actual value that matches the specified + * {@code key} in byte. If the return value is greater than the + * length of {@code value}, then it indicates that the size of the + * input buffer {@code value} is insufficient and partial result will + * be returned. RocksDB.NOT_FOUND will be returned if the value not + * found. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public int get(final ColumnFamilyHandle columnFamilyHandle, + final ReadOptions opt, final byte[] key, final int offset, final int len, + final byte[] value, final int vOffset, final int vLen) + throws RocksDBException { + checkBounds(offset, len, key.length); + checkBounds(vOffset, vLen, value.length); + return get(nativeHandle_, opt.nativeHandle_, key, offset, len, value, + vOffset, vLen, columnFamilyHandle.nativeHandle_); + } + + /** + * The simplified version of get which returns a new byte array storing + * the value associated with the specified input key if any. null will be + * returned if the specified key is not found. + * + * @param key the key retrieve the value. + * @return a byte array storing the value associated with the input key if + * any. null if it does not find the specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public byte[] get(final byte[] key) throws RocksDBException { + return get(nativeHandle_, key, 0, key.length); + } + + /** + * The simplified version of get which returns a new byte array storing + * the value associated with the specified input key if any. null will be + * returned if the specified key is not found. + * + * @param key the key retrieve the value. + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than ("key".length - offset) + * @return a byte array storing the value associated with the input key if + * any. null if it does not find the specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public byte[] get(final byte[] key, final int offset, + final int len) throws RocksDBException { + checkBounds(offset, len, key.length); + return get(nativeHandle_, key, offset, len); + } + + /** + * The simplified version of get which returns a new byte array storing + * the value associated with the specified input key if any. null will be + * returned if the specified key is not found. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param key the key retrieve the value. + * @return a byte array storing the value associated with the input key if + * any. null if it does not find the specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public byte[] get(final ColumnFamilyHandle columnFamilyHandle, + final byte[] key) throws RocksDBException { + return get(nativeHandle_, key, 0, key.length, + columnFamilyHandle.nativeHandle_); + } + + /** + * The simplified version of get which returns a new byte array storing + * the value associated with the specified input key if any. null will be + * returned if the specified key is not found. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param key the key retrieve the value. + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than ("key".length - offset) + * @return a byte array storing the value associated with the input key if + * any. null if it does not find the specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public byte[] get(final ColumnFamilyHandle columnFamilyHandle, + final byte[] key, final int offset, final int len) + throws RocksDBException { + checkBounds(offset, len, key.length); + return get(nativeHandle_, key, offset, len, + columnFamilyHandle.nativeHandle_); + } + + /** + * The simplified version of get which returns a new byte array storing + * the value associated with the specified input key if any. null will be + * returned if the specified key is not found. + * + * @param key the key retrieve the value. + * @param opt Read options. + * @return a byte array storing the value associated with the input key if + * any. null if it does not find the specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public byte[] get(final ReadOptions opt, final byte[] key) + throws RocksDBException { + return get(nativeHandle_, opt.nativeHandle_, key, 0, key.length); + } + + /** + * The simplified version of get which returns a new byte array storing + * the value associated with the specified input key if any. null will be + * returned if the specified key is not found. + * + * @param key the key retrieve the value. + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than ("key".length - offset) + * @param opt Read options. + * @return a byte array storing the value associated with the input key if + * any. null if it does not find the specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public byte[] get(final ReadOptions opt, final byte[] key, final int offset, + final int len) throws RocksDBException { + checkBounds(offset, len, key.length); + return get(nativeHandle_, opt.nativeHandle_, key, offset, len); + } + + /** + * The simplified version of get which returns a new byte array storing + * the value associated with the specified input key if any. null will be + * returned if the specified key is not found. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param key the key retrieve the value. + * @param opt Read options. + * @return a byte array storing the value associated with the input key if + * any. null if it does not find the specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public byte[] get(final ColumnFamilyHandle columnFamilyHandle, + final ReadOptions opt, final byte[] key) throws RocksDBException { + return get(nativeHandle_, opt.nativeHandle_, key, 0, key.length, + columnFamilyHandle.nativeHandle_); + } + + /** + * The simplified version of get which returns a new byte array storing + * the value associated with the specified input key if any. null will be + * returned if the specified key is not found. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param key the key retrieve the value. + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than ("key".length - offset) + * @param opt Read options. + * @return a byte array storing the value associated with the input key if + * any. null if it does not find the specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public byte[] get(final ColumnFamilyHandle columnFamilyHandle, + final ReadOptions opt, final byte[] key, final int offset, final int len) + throws RocksDBException { + checkBounds(offset, len, key.length); + return get(nativeHandle_, opt.nativeHandle_, key, offset, len, + columnFamilyHandle.nativeHandle_); + } + + /** + * Takes a list of keys, and returns a list of values for the given list of + * keys. List will contain null for keys which could not be found. + * + * @param keys List of keys for which values need to be retrieved. + * @return List of values for the given list of keys. List will contain + * null for keys which could not be found. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public List multiGetAsList(final List keys) + throws RocksDBException { + assert(keys.size() != 0); + + final byte[][] keysArray = keys.toArray(new byte[keys.size()][]); + final int[] keyOffsets = new int[keysArray.length]; + final int[] keyLengths = new int[keysArray.length]; + for(int i = 0; i < keyLengths.length; i++) { + keyLengths[i] = keysArray[i].length; + } + + return Arrays.asList(multiGet(nativeHandle_, keysArray, keyOffsets, + keyLengths)); + } + + /** + * Returns a list of values for the given list of keys. List will contain + * null for keys which could not be found. + *

    + * Note: Every key needs to have a related column family name in + * {@code columnFamilyHandleList}. + *

    + * + * @param columnFamilyHandleList {@link java.util.List} containing + * {@link org.rocksdb.ColumnFamilyHandle} instances. + * @param keys List of keys for which values need to be retrieved. + * @return List of values for the given list of keys. List will contain + * null for keys which could not be found. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + * @throws IllegalArgumentException thrown if the size of passed keys is not + * equal to the amount of passed column family handles. + */ + public List multiGetAsList( + final List columnFamilyHandleList, + final List keys) throws RocksDBException, + IllegalArgumentException { + assert(keys.size() != 0); + // Check if key size equals cfList size. If not a exception must be + // thrown. If not a Segmentation fault happens. + if (keys.size() != columnFamilyHandleList.size()) { + throw new IllegalArgumentException( + "For each key there must be a ColumnFamilyHandle."); + } + final long[] cfHandles = new long[columnFamilyHandleList.size()]; + for (int i = 0; i < columnFamilyHandleList.size(); i++) { + cfHandles[i] = columnFamilyHandleList.get(i).nativeHandle_; + } + + final byte[][] keysArray = keys.toArray(new byte[keys.size()][]); + final int[] keyOffsets = new int[keysArray.length]; + final int[] keyLengths = new int[keysArray.length]; + for(int i = 0; i < keyLengths.length; i++) { + keyLengths[i] = keysArray[i].length; + } + + return Arrays.asList(multiGet(nativeHandle_, keysArray, keyOffsets, + keyLengths, cfHandles)); + } + + /** + * Returns a list of values for the given list of keys. List will contain + * null for keys which could not be found. + * + * @param opt Read options. + * @param keys of keys for which values need to be retrieved. + * @return List of values for the given list of keys. List will contain + * null for keys which could not be found. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public List multiGetAsList(final ReadOptions opt, + final List keys) throws RocksDBException { + assert(keys.size() != 0); + + final byte[][] keysArray = keys.toArray(new byte[keys.size()][]); + final int[] keyOffsets = new int[keysArray.length]; + final int[] keyLengths = new int[keysArray.length]; + for(int i = 0; i < keyLengths.length; i++) { + keyLengths[i] = keysArray[i].length; + } + + return Arrays.asList(multiGet(nativeHandle_, opt.nativeHandle_, + keysArray, keyOffsets, keyLengths)); + } + + /** + * Returns a list of values for the given list of keys. List will contain + * null for keys which could not be found. + *

    + * Note: Every key needs to have a related column family name in + * {@code columnFamilyHandleList}. + *

    + * + * @param opt Read options. + * @param columnFamilyHandleList {@link java.util.List} containing + * {@link org.rocksdb.ColumnFamilyHandle} instances. + * @param keys of keys for which values need to be retrieved. + * @return List of values for the given list of keys. List will contain + * null for keys which could not be found. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + * @throws IllegalArgumentException thrown if the size of passed keys is not + * equal to the amount of passed column family handles. + */ + public List multiGetAsList(final ReadOptions opt, + final List columnFamilyHandleList, + final List keys) throws RocksDBException { + assert(keys.size() != 0); + // Check if key size equals cfList size. If not a exception must be + // thrown. If not a Segmentation fault happens. + if (keys.size()!=columnFamilyHandleList.size()){ + throw new IllegalArgumentException( + "For each key there must be a ColumnFamilyHandle."); + } + final long[] cfHandles = new long[columnFamilyHandleList.size()]; + for (int i = 0; i < columnFamilyHandleList.size(); i++) { + cfHandles[i] = columnFamilyHandleList.get(i).nativeHandle_; + } + + final byte[][] keysArray = keys.toArray(new byte[keys.size()][]); + final int[] keyOffsets = new int[keysArray.length]; + final int[] keyLengths = new int[keysArray.length]; + for(int i = 0; i < keyLengths.length; i++) { + keyLengths[i] = keysArray[i].length; + } + + return Arrays.asList(multiGet(nativeHandle_, opt.nativeHandle_, + keysArray, keyOffsets, keyLengths, cfHandles)); + } + + /** + * Fetches a list of values for the given list of keys, all from the default column family. + * + * @param keys list of keys for which values need to be retrieved. + * @param values list of buffers to return retrieved values in + * @return list of number of bytes in DB for each requested key + * this can be more than the size of the corresponding buffer; then the buffer will be filled + * with the appropriate truncation of the database value. + * @throws RocksDBException if error happens in underlying native library. + * @throws IllegalArgumentException thrown if the number of passed keys and passed values + * do not match. + */ + public List multiGetByteBuffers( + final List keys, final List values) throws RocksDBException { + final ReadOptions readOptions = new ReadOptions(); + final List columnFamilyHandleList = new ArrayList<>(1); + columnFamilyHandleList.add(getDefaultColumnFamily()); + return multiGetByteBuffers(readOptions, columnFamilyHandleList, keys, values); + } + + /** + * Fetches a list of values for the given list of keys, all from the default column family. + * + * @param readOptions Read options + * @param keys list of keys for which values need to be retrieved. + * @param values list of buffers to return retrieved values in + * @throws RocksDBException if error happens in underlying native library. + * @throws IllegalArgumentException thrown if the number of passed keys and passed values + * do not match. + * @return the list of values for the given list of keys + */ + public List multiGetByteBuffers(final ReadOptions readOptions, + final List keys, final List values) throws RocksDBException { + final List columnFamilyHandleList = new ArrayList<>(1); + columnFamilyHandleList.add(getDefaultColumnFamily()); + return multiGetByteBuffers(readOptions, columnFamilyHandleList, keys, values); + } + + /** + * Fetches a list of values for the given list of keys. + *

    + * Note: Every key needs to have a related column family name in + * {@code columnFamilyHandleList}. + *

    + * + * @param columnFamilyHandleList {@link java.util.List} containing + * {@link org.rocksdb.ColumnFamilyHandle} instances. + * @param keys list of keys for which values need to be retrieved. + * @param values list of buffers to return retrieved values in + * @throws RocksDBException if error happens in underlying native library. + * @throws IllegalArgumentException thrown if the number of passed keys, passed values and + * passed column family handles do not match. + * @return the list of values for the given list of keys + */ + public List multiGetByteBuffers( + final List columnFamilyHandleList, final List keys, + final List values) throws RocksDBException { + final ReadOptions readOptions = new ReadOptions(); + return multiGetByteBuffers(readOptions, columnFamilyHandleList, keys, values); + } + + /** + * Fetches a list of values for the given list of keys. + *

    + * Note: Every key needs to have a related column family name in + * {@code columnFamilyHandleList}. + *

    + * + * @param readOptions Read options + * @param columnFamilyHandleList {@link java.util.List} containing + * {@link org.rocksdb.ColumnFamilyHandle} instances. + * @param keys list of keys for which values need to be retrieved. + * @param values list of buffers to return retrieved values in + * @throws RocksDBException if error happens in underlying native library. + * @throws IllegalArgumentException thrown if the number of passed keys, passed values and + * passed column family handles do not match. + * @return the list of values for the given list of keys + */ + public List multiGetByteBuffers(final ReadOptions readOptions, + final List columnFamilyHandleList, final List keys, + final List values) throws RocksDBException { + assert (keys.size() != 0); + + // Check if key size equals cfList size. If not a exception must be + // thrown. If not a Segmentation fault happens. + if (keys.size() != columnFamilyHandleList.size() && columnFamilyHandleList.size() > 1) { + throw new IllegalArgumentException( + "Wrong number of ColumnFamilyHandle(s) supplied. Provide 0, 1, or as many as there are key/value(s)"); + } + + // Check if key size equals cfList size. If not a exception must be + // thrown. If not a Segmentation fault happens. + if (values.size() != keys.size()) { + throw new IllegalArgumentException("For each key there must be a corresponding value."); + } + + // TODO (AP) support indirect buffers + for (final ByteBuffer key : keys) { + if (!key.isDirect()) { + throw new IllegalArgumentException("All key buffers must be direct byte buffers"); + } + } + + // TODO (AP) support indirect buffers, though probably via a less efficient code path + for (final ByteBuffer value : values) { + if (!value.isDirect()) { + throw new IllegalArgumentException("All value buffers must be direct byte buffers"); + } + } + + final int numCFHandles = columnFamilyHandleList.size(); + final long[] cfHandles = new long[numCFHandles]; + for (int i = 0; i < numCFHandles; i++) { + cfHandles[i] = columnFamilyHandleList.get(i).nativeHandle_; + } + + final int numValues = keys.size(); + + final ByteBuffer[] keysArray = keys.toArray(new ByteBuffer[0]); + final int[] keyOffsets = new int[numValues]; + final int[] keyLengths = new int[numValues]; + for (int i = 0; i < numValues; i++) { + // TODO (AP) add keysArray[i].arrayOffset() if the buffer is indirect + // TODO (AP) because in that case we have to pass the array directly, + // so that the JNI C++ code will not know to compensate for the array offset + keyOffsets[i] = keysArray[i].position(); + keyLengths[i] = keysArray[i].limit(); + } + final ByteBuffer[] valuesArray = values.toArray(new ByteBuffer[0]); + final int[] valuesSizeArray = new int[numValues]; + final Status[] statusArray = new Status[numValues]; + + multiGet(nativeHandle_, readOptions.nativeHandle_, cfHandles, keysArray, keyOffsets, keyLengths, + valuesArray, valuesSizeArray, statusArray); + + final List results = new ArrayList<>(); + for (int i = 0; i < numValues; i++) { + final Status status = statusArray[i]; + if (status.getCode() == Status.Code.Ok) { + final ByteBuffer value = valuesArray[i]; + value.position(Math.min(valuesSizeArray[i], value.capacity())); + value.flip(); // prepare for read out + results.add(new ByteBufferGetStatus(status, valuesSizeArray[i], value)); + } else { + results.add(new ByteBufferGetStatus(status)); + } + } + + return results; + } + + /** + * If the key definitely does not exist in the database, then this method + * returns false, otherwise it returns true if the key might exist. + * That is to say that this method is probabilistic and may return false + * positives, but never a false negative. + *

    + * If the caller wants to obtain value when the key + * is found in memory, then {@code valueHolder} must be set. + *

    + * This check is potentially lighter-weight than invoking + * {@link #get(byte[])}. One way to make this lighter weight is to avoid + * doing any IOs. + * + * @param key byte array of a key to search for + * @param valueHolder non-null to retrieve the value if it is found, or null + * if the value is not needed. If non-null, upon return of the function, + * the {@code value} will be set if it could be retrieved. + * + * @return false if the key definitely does not exist in the database, + * otherwise true. + */ + public boolean keyMayExist(final byte[] key, + /* @Nullable */ final Holder valueHolder) { + return keyMayExist(key, 0, key.length, valueHolder); + } + + /** + * If the key definitely does not exist in the database, then this method + * returns false, otherwise it returns true if the key might exist. + * That is to say that this method is probabilistic and may return false + * positives, but never a false negative. + *

    + * If the caller wants to obtain value when the key + * is found in memory, then {@code valueHolder} must be set. + *

    + * This check is potentially lighter-weight than invoking + * {@link #get(byte[], int, int)}. One way to make this lighter weight is to + * avoid doing any IOs. + * + * @param key byte array of a key to search for + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than "key".length + * @param valueHolder non-null to retrieve the value if it is found, or null + * if the value is not needed. If non-null, upon return of the function, + * the {@code value} will be set if it could be retrieved. + * + * @return false if the key definitely does not exist in the database, + * otherwise true. + */ + public boolean keyMayExist(final byte[] key, + final int offset, final int len, + /* @Nullable */ final Holder valueHolder) { + return keyMayExist((ColumnFamilyHandle)null, key, offset, len, valueHolder); + } + + /** + * If the key definitely does not exist in the database, then this method + * returns false, otherwise it returns true if the key might exist. + * That is to say that this method is probabilistic and may return false + * positives, but never a false negative. + *

    + * If the caller wants to obtain value when the key + * is found in memory, then {@code valueHolder} must be set. + *

    + * This check is potentially lighter-weight than invoking + * {@link #get(ColumnFamilyHandle,byte[])}. One way to make this lighter + * weight is to avoid doing any IOs. + * + * @param columnFamilyHandle {@link ColumnFamilyHandle} instance + * @param key byte array of a key to search for + * @param valueHolder non-null to retrieve the value if it is found, or null + * if the value is not needed. If non-null, upon return of the function, + * the {@code value} will be set if it could be retrieved. + * + * @return false if the key definitely does not exist in the database, + * otherwise true. + */ + public boolean keyMayExist( + final ColumnFamilyHandle columnFamilyHandle, final byte[] key, + /* @Nullable */ final Holder valueHolder) { + return keyMayExist(columnFamilyHandle, key, 0, key.length, + valueHolder); + } + + /** + * If the key definitely does not exist in the database, then this method + * returns false, otherwise it returns true if the key might exist. + * That is to say that this method is probabilistic and may return false + * positives, but never a false negative. + *

    + * If the caller wants to obtain value when the key + * is found in memory, then {@code valueHolder} must be set. + *

    + * This check is potentially lighter-weight than invoking + * {@link #get(ColumnFamilyHandle, byte[], int, int)}. One way to make this + * lighter weight is to avoid doing any IOs. + * + * @param columnFamilyHandle {@link ColumnFamilyHandle} instance + * @param key byte array of a key to search for + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than "key".length + * @param valueHolder non-null to retrieve the value if it is found, or null + * if the value is not needed. If non-null, upon return of the function, + * the {@code value} will be set if it could be retrieved. + * + * @return false if the key definitely does not exist in the database, + * otherwise true. + */ + public boolean keyMayExist(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, + final int offset, final int len, + /* @Nullable */ final Holder valueHolder) { + return keyMayExist(columnFamilyHandle, null, key, offset, len, + valueHolder); + } + + /** + * If the key definitely does not exist in the database, then this method + * returns false, otherwise it returns true if the key might exist. + * That is to say that this method is probabilistic and may return false + * positives, but never a true negative. + *

    + * If the caller wants to obtain value when the key + * is found in memory, then {@code valueHolder} must be set. + *

    + * This check is potentially lighter-weight than invoking + * {@link #get(ReadOptions, byte[])}. One way to make this + * lighter weight is to avoid doing any IOs. + * + * @param readOptions {@link ReadOptions} instance + * @param key byte array of a key to search for + * @param valueHolder non-null to retrieve the value if it is found, or null + * if the value is not needed. If non-null, upon return of the function, + * the {@code value} will be set if it could be retrieved. + * + * @return false if the key definitely does not exist in the database, + * otherwise true. + */ + public boolean keyMayExist( + final ReadOptions readOptions, final byte[] key, + /* @Nullable */ final Holder valueHolder) { + return keyMayExist(readOptions, key, 0, key.length, + valueHolder); + } + + /** + * If the key definitely does not exist in the database, then this method + * returns false, otherwise it returns true if the key might exist. + * That is to say that this method is probabilistic and may return false + * positives, but never a true negative. + *

    + * If the caller wants to obtain value when the key + * is found in memory, then {@code valueHolder} must be set. + *

    + * This check is potentially lighter-weight than invoking + * {@link #get(ReadOptions, byte[], int, int)}. One way to make this + * lighter weight is to avoid doing any IOs. + * + * @param readOptions {@link ReadOptions} instance + * @param key byte array of a key to search for + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than "key".length + * @param valueHolder non-null to retrieve the value if it is found, or null + * if the value is not needed. If non-null, upon return of the function, + * the {@code value} will be set if it could be retrieved. + * + * @return false if the key definitely does not exist in the database, + * otherwise true. + */ + public boolean keyMayExist( + final ReadOptions readOptions, + final byte[] key, final int offset, final int len, + /* @Nullable */ final Holder valueHolder) { + return keyMayExist(null, readOptions, + key, offset, len, valueHolder); + } + + /** + * If the key definitely does not exist in the database, then this method + * returns false, otherwise it returns true if the key might exist. + * That is to say that this method is probabilistic and may return false + * positives, but never a true negative. + *

    + * If the caller wants to obtain value when the key + * is found in memory, then {@code valueHolder} must be set. + *

    + * This check is potentially lighter-weight than invoking + * {@link #get(ColumnFamilyHandle, ReadOptions, byte[])}. One way to make this + * lighter weight is to avoid doing any IOs. + * + * @param columnFamilyHandle {@link ColumnFamilyHandle} instance + * @param readOptions {@link ReadOptions} instance + * @param key byte array of a key to search for + * @param valueHolder non-null to retrieve the value if it is found, or null + * if the value is not needed. If non-null, upon return of the function, + * the {@code value} will be set if it could be retrieved. + * + * @return false if the key definitely does not exist in the database, + * otherwise true. + */ + public boolean keyMayExist( + final ColumnFamilyHandle columnFamilyHandle, + final ReadOptions readOptions, final byte[] key, + /* @Nullable */ final Holder valueHolder) { + return keyMayExist(columnFamilyHandle, readOptions, + key, 0, key.length, valueHolder); + } + + /** + * If the key definitely does not exist in the database, then this method + * returns false, otherwise it returns true if the key might exist. + * That is to say that this method is probabilistic and may return false + * positives, but never a false negative. + *

    + * If the caller wants to obtain value when the key + * is found in memory, then {@code valueHolder} must be set. + *

    + * This check is potentially lighter-weight than invoking + * {@link #get(ColumnFamilyHandle, ReadOptions, byte[], int, int)}. + * One way to make this lighter weight is to avoid doing any IOs. + * + * @param columnFamilyHandle {@link ColumnFamilyHandle} instance + * @param readOptions {@link ReadOptions} instance + * @param key byte array of a key to search for + * @param offset the offset of the "key" array to be used, must be + * non-negative and no larger than "key".length + * @param len the length of the "key" array to be used, must be non-negative + * and no larger than "key".length + * @param valueHolder non-null to retrieve the value if it is found, or null + * if the value is not needed. If non-null, upon return of the function, + * the {@code value} will be set if it could be retrieved. + * + * @return false if the key definitely does not exist in the database, + * otherwise true. + */ + public boolean keyMayExist( + final ColumnFamilyHandle columnFamilyHandle, + final ReadOptions readOptions, + final byte[] key, final int offset, final int len, + /* @Nullable */ final Holder valueHolder) { + checkBounds(offset, len, key.length); + if (valueHolder == null) { + return keyMayExist(nativeHandle_, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_, + readOptions == null ? 0 : readOptions.nativeHandle_, + key, offset, len); + } else { + final byte[][] result = keyMayExistFoundValue( + nativeHandle_, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_, + readOptions == null ? 0 : readOptions.nativeHandle_, + key, offset, len); + if (result[0][0] == 0x0) { + valueHolder.setValue(null); + return false; + } else if (result[0][0] == 0x1) { + valueHolder.setValue(null); + return true; + } else { + valueHolder.setValue(result[1]); + return true; + } + } + } + + /** + * If the key definitely does not exist in the database, then this method + * returns false, otherwise it returns true if the key might exist. + * That is to say that this method is probabilistic and may return false + * positives, but never a false negative. + * + * @param key bytebuffer containing the value of the key + * @return false if the key definitely does not exist in the database, + * otherwise true. + */ + public boolean keyMayExist(final ByteBuffer key) { + return keyMayExist(null, (ReadOptions) null, key); + } + + /** + * If the key definitely does not exist in the database, then this method + * returns false, otherwise it returns true if the key might exist. + * That is to say that this method is probabilistic and may return false + * positives, but never a false negative. + * + * @param columnFamilyHandle the {@link ColumnFamilyHandle} to look for the key in + * @param key bytebuffer containing the value of the key + * @return false if the key definitely does not exist in the database, + * otherwise true. + */ + public boolean keyMayExist(final ColumnFamilyHandle columnFamilyHandle, final ByteBuffer key) { + return keyMayExist(columnFamilyHandle, (ReadOptions) null, key); + } + + /** + * If the key definitely does not exist in the database, then this method + * returns false, otherwise it returns true if the key might exist. + * That is to say that this method is probabilistic and may return false + * positives, but never a false negative. + * + * @param readOptions the {@link ReadOptions} to use when reading the key/value + * @param key bytebuffer containing the value of the key + * @return false if the key definitely does not exist in the database, + * otherwise true. + */ + public boolean keyMayExist(final ReadOptions readOptions, final ByteBuffer key) { + return keyMayExist(null, readOptions, key); + } + + /** + * If the key definitely does not exist in the database, then this method + * returns {@link KeyMayExist.KeyMayExistEnum#kNotExist}, + * otherwise if it can with best effort retreive the value, it returns {@link + * KeyMayExist.KeyMayExistEnum#kExistsWithValue} otherwise it returns {@link + * KeyMayExist.KeyMayExistEnum#kExistsWithoutValue}. The choice not to return a value which might + * exist is at the discretion of the implementation; the only guarantee is that {@link + * KeyMayExist.KeyMayExistEnum#kNotExist} is an assurance that the key does not exist. + * + * @param key bytebuffer containing the value of the key + * @param value bytebuffer which will receive a value if the key exists and a value is known + * @return a {@link KeyMayExist} object reporting if key may exist and if a value is provided + */ + public KeyMayExist keyMayExist(final ByteBuffer key, final ByteBuffer value) { + return keyMayExist(null, null, key, value); + } + + /** + * If the key definitely does not exist in the database, then this method + * returns {@link KeyMayExist.KeyMayExistEnum#kNotExist}, + * otherwise if it can with best effort retreive the value, it returns {@link + * KeyMayExist.KeyMayExistEnum#kExistsWithValue} otherwise it returns {@link + * KeyMayExist.KeyMayExistEnum#kExistsWithoutValue}. The choice not to return a value which might + * exist is at the discretion of the implementation; the only guarantee is that {@link + * KeyMayExist.KeyMayExistEnum#kNotExist} is an assurance that the key does not exist. + * + * @param columnFamilyHandle the {@link ColumnFamilyHandle} to look for the key in + * @param key bytebuffer containing the value of the key + * @param value bytebuffer which will receive a value if the key exists and a value is known + * @return a {@link KeyMayExist} object reporting if key may exist and if a value is provided + */ + public KeyMayExist keyMayExist( + final ColumnFamilyHandle columnFamilyHandle, final ByteBuffer key, final ByteBuffer value) { + return keyMayExist(columnFamilyHandle, null, key, value); + } + + /** + * If the key definitely does not exist in the database, then this method + * returns {@link KeyMayExist.KeyMayExistEnum#kNotExist}, + * otherwise if it can with best effort retreive the value, it returns {@link + * KeyMayExist.KeyMayExistEnum#kExistsWithValue} otherwise it returns {@link + * KeyMayExist.KeyMayExistEnum#kExistsWithoutValue}. The choice not to return a value which might + * exist is at the discretion of the implementation; the only guarantee is that {@link + * KeyMayExist.KeyMayExistEnum#kNotExist} is an assurance that the key does not exist. + * + * @param readOptions the {@link ReadOptions} to use when reading the key/value + * @param key bytebuffer containing the value of the key + * @param value bytebuffer which will receive a value if the key exists and a value is known + * @return a {@link KeyMayExist} object reporting if key may exist and if a value is provided + */ + public KeyMayExist keyMayExist( + final ReadOptions readOptions, final ByteBuffer key, final ByteBuffer value) { + return keyMayExist(null, readOptions, key, value); + } + + /** + * If the key definitely does not exist in the database, then this method + * returns false, otherwise it returns true if the key might exist. + * That is to say that this method is probabilistic and may return false + * positives, but never a false negative. + * + * @param columnFamilyHandle the {@link ColumnFamilyHandle} to look for the key in + * @param readOptions the {@link ReadOptions} to use when reading the key/value + * @param key bytebuffer containing the value of the key + * @return false if the key definitely does not exist in the database, + * otherwise true. + */ + public boolean keyMayExist(final ColumnFamilyHandle columnFamilyHandle, + final ReadOptions readOptions, final ByteBuffer key) { + assert key != null : "key ByteBuffer parameter cannot be null"; + assert key.isDirect() : "key parameter must be a direct ByteBuffer"; + return keyMayExistDirect(nativeHandle_, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_, + readOptions == null ? 0 : readOptions.nativeHandle_, key, key.position(), key.limit()); + } + + /** + * If the key definitely does not exist in the database, then this method + * returns {@link KeyMayExist.KeyMayExistEnum#kNotExist}, + * otherwise if it can with best effort retreive the value, it returns {@link + * KeyMayExist.KeyMayExistEnum#kExistsWithValue} otherwise it returns {@link + * KeyMayExist.KeyMayExistEnum#kExistsWithoutValue}. The choice not to return a value which might + * exist is at the discretion of the implementation; the only guarantee is that {@link + * KeyMayExist.KeyMayExistEnum#kNotExist} is an assurance that the key does not exist. + * + * @param columnFamilyHandle the {@link ColumnFamilyHandle} to look for the key in + * @param readOptions the {@link ReadOptions} to use when reading the key/value + * @param key bytebuffer containing the value of the key + * @param value bytebuffer which will receive a value if the key exists and a value is known + * @return a {@link KeyMayExist} object reporting if key may exist and if a value is provided + */ + public KeyMayExist keyMayExist(final ColumnFamilyHandle columnFamilyHandle, + final ReadOptions readOptions, final ByteBuffer key, final ByteBuffer value) { + assert key != null : "key ByteBuffer parameter cannot be null"; + assert key.isDirect() : "key parameter must be a direct ByteBuffer"; + assert value + != null + : "value ByteBuffer parameter cannot be null. If you do not need the value, use a different version of the method"; + assert value.isDirect() : "value parameter must be a direct ByteBuffer"; + + final int[] result = keyMayExistDirectFoundValue(nativeHandle_, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_, + readOptions == null ? 0 : readOptions.nativeHandle_, key, key.position(), key.remaining(), + value, value.position(), value.remaining()); + final int valueLength = result[1]; + value.limit(value.position() + Math.min(valueLength, value.remaining())); + return new KeyMayExist(KeyMayExist.KeyMayExistEnum.values()[result[0]], valueLength); + } + + /** + *

    Return a heap-allocated iterator over the contents of the + * database. The result of newIterator() is initially invalid + * (caller must call one of the Seek methods on the iterator + * before using it).

    + * + *

    Caller should close the iterator when it is no longer needed. + * The returned iterator should be closed before this db is closed. + *

    + * + * @return instance of iterator object. + */ + public RocksIterator newIterator() { + return new RocksIterator(this, iterator(nativeHandle_)); + } + + /** + *

    Return a heap-allocated iterator over the contents of the + * database. The result of newIterator() is initially invalid + * (caller must call one of the Seek methods on the iterator + * before using it).

    + * + *

    Caller should close the iterator when it is no longer needed. + * The returned iterator should be closed before this db is closed. + *

    + * + * @param readOptions {@link ReadOptions} instance. + * @return instance of iterator object. + */ + public RocksIterator newIterator(final ReadOptions readOptions) { + return new RocksIterator(this, iterator(nativeHandle_, + readOptions.nativeHandle_)); + } + + /** + *

    Return a heap-allocated iterator over the contents of a + * ColumnFamily. The result of newIterator() is initially invalid + * (caller must call one of the Seek methods on the iterator + * before using it).

    + * + *

    Caller should close the iterator when it is no longer needed. + * The returned iterator should be closed before this db is closed. + *

    + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @return instance of iterator object. + */ + public RocksIterator newIterator( + final ColumnFamilyHandle columnFamilyHandle) { + return new RocksIterator(this, iteratorCF(nativeHandle_, + columnFamilyHandle.nativeHandle_)); + } + + /** + *

    Return a heap-allocated iterator over the contents of a + * ColumnFamily. The result of newIterator() is initially invalid + * (caller must call one of the Seek methods on the iterator + * before using it).

    + * + *

    Caller should close the iterator when it is no longer needed. + * The returned iterator should be closed before this db is closed. + *

    + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param readOptions {@link ReadOptions} instance. + * @return instance of iterator object. + */ + public RocksIterator newIterator(final ColumnFamilyHandle columnFamilyHandle, + final ReadOptions readOptions) { + return new RocksIterator(this, iteratorCF(nativeHandle_, + columnFamilyHandle.nativeHandle_, readOptions.nativeHandle_)); + } + + /** + * Returns iterators from a consistent database state across multiple + * column families. Iterators are heap allocated and need to be deleted + * before the db is deleted + * + * @param columnFamilyHandleList {@link java.util.List} containing + * {@link org.rocksdb.ColumnFamilyHandle} instances. + * @return {@link java.util.List} containing {@link org.rocksdb.RocksIterator} + * instances + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public List newIterators( + final List columnFamilyHandleList) + throws RocksDBException { + return newIterators(columnFamilyHandleList, new ReadOptions()); + } + + /** + * Returns iterators from a consistent database state across multiple + * column families. Iterators are heap allocated and need to be deleted + * before the db is deleted + * + * @param columnFamilyHandleList {@link java.util.List} containing + * {@link org.rocksdb.ColumnFamilyHandle} instances. + * @param readOptions {@link ReadOptions} instance. + * @return {@link java.util.List} containing {@link org.rocksdb.RocksIterator} + * instances + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public List newIterators( + final List columnFamilyHandleList, + final ReadOptions readOptions) throws RocksDBException { + + final long[] columnFamilyHandles = new long[columnFamilyHandleList.size()]; + for (int i = 0; i < columnFamilyHandleList.size(); i++) { + columnFamilyHandles[i] = columnFamilyHandleList.get(i).nativeHandle_; + } + + final long[] iteratorRefs = iterators(nativeHandle_, columnFamilyHandles, + readOptions.nativeHandle_); + + final List iterators = new ArrayList<>( + columnFamilyHandleList.size()); + for (int i=0; iReturn a handle to the current DB state. Iterators created with + * this handle will all observe a stable snapshot of the current DB + * state. The caller must call ReleaseSnapshot(result) when the + * snapshot is no longer needed.

    + * + *

    nullptr will be returned if the DB fails to take a snapshot or does + * not support snapshot.

    + * + * @return Snapshot {@link Snapshot} instance + */ + public Snapshot getSnapshot() { + final long snapshotHandle = getSnapshot(nativeHandle_); + if (snapshotHandle != 0) { + return new Snapshot(snapshotHandle); + } + return null; + } + + /** + * Release a previously acquired snapshot. + *

    + * The caller must not use "snapshot" after this call. + * + * @param snapshot {@link Snapshot} instance + */ + public void releaseSnapshot(final Snapshot snapshot) { + if (snapshot != null) { + releaseSnapshot(nativeHandle_, snapshot.nativeHandle_); + } + } + + /** + * DB implements can export properties about their state + * via this method on a per column family level. + * + *

    If {@code property} is a valid property understood by this DB + * implementation, fills {@code value} with its current value and + * returns true. Otherwise returns false.

    + * + *

    Valid property names include: + *

      + *
    • "rocksdb.num-files-at-level<N>" - return the number of files at + * level <N>, where <N> is an ASCII representation of a level + * number (e.g. "0").
      • + *
    • "rocksdb.stats" - returns a multi-line string that describes statistics + * about the internal operation of the DB.
      • + *
    • "rocksdb.sstables" - returns a multi-line string that describes all + * of the sstables that make up the db contents.
      • + *
    + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance, or null for the default column family. + * @param property to be fetched. See above for examples + * @return property value + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public String getProperty( + /* @Nullable */ final ColumnFamilyHandle columnFamilyHandle, + final String property) throws RocksDBException { + return getProperty(nativeHandle_, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_, + property, property.length()); + } + + /** + * DB implementations can export properties about their state + * via this method. If "property" is a valid property understood by this + * DB implementation, fills "*value" with its current value and returns + * true. Otherwise returns false. + * + *

    Valid property names include: + *

      + *
    • "rocksdb.num-files-at-level<N>" - return the number of files at + * level <N>, where <N> is an ASCII representation of a level + * number (e.g. "0").
      • + *
    • "rocksdb.stats" - returns a multi-line string that describes statistics + * about the internal operation of the DB.
      • + *
    • "rocksdb.sstables" - returns a multi-line string that describes all + * of the sstables that make up the db contents.
      • + *
    + * + * @param property to be fetched. See above for examples + * @return property value + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public String getProperty(final String property) throws RocksDBException { + return getProperty(null, property); + } + + + /** + * Gets a property map. + * + * @param property to be fetched. + * + * @return the property map + * + * @throws RocksDBException if an error happens in the underlying native code. + */ + public Map getMapProperty(final String property) + throws RocksDBException { + return getMapProperty(null, property); + } + + /** + * Gets a property map. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance, or null for the default column family. + * @param property to be fetched. + * + * @return the property map + * + * @throws RocksDBException if an error happens in the underlying native code. + */ + public Map getMapProperty( + /* @Nullable */ final ColumnFamilyHandle columnFamilyHandle, + final String property) throws RocksDBException { + return getMapProperty(nativeHandle_, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_, + property, property.length()); + } + + /** + *

    Similar to GetProperty(), but only works for a subset of properties + * whose return value is a numerical value. Return the value as long.

    + * + *

    Note: As the returned property is of type + * {@code uint64_t} on C++ side the returning value can be negative + * because Java supports in Java 7 only signed long values.

    + * + *

    Java 7: To mitigate the problem of the non + * existent unsigned long tpye, values should be encapsulated using + * {@link java.math.BigInteger} to reflect the correct value. The correct + * behavior is guaranteed if {@code 2^64} is added to negative values.

    + * + *

    Java 8: In Java 8 the value should be treated as + * unsigned long using provided methods of type {@link Long}.

    + * + * @param property to be fetched. + * + * @return numerical property value. + * + * @throws RocksDBException if an error happens in the underlying native code. + */ + public long getLongProperty(final String property) throws RocksDBException { + return getLongProperty(null, property); + } + + /** + *

    Similar to GetProperty(), but only works for a subset of properties + * whose return value is a numerical value. Return the value as long.

    + * + *

    Note: As the returned property is of type + * {@code uint64_t} on C++ side the returning value can be negative + * because Java supports in Java 7 only signed long values.

    + * + *

    Java 7: To mitigate the problem of the non + * existent unsigned long tpye, values should be encapsulated using + * {@link java.math.BigInteger} to reflect the correct value. The correct + * behavior is guaranteed if {@code 2^64} is added to negative values.

    + * + *

    Java 8: In Java 8 the value should be treated as + * unsigned long using provided methods of type {@link Long}.

    + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance, or null for the default column family + * @param property to be fetched. + * + * @return numerical property value + * + * @throws RocksDBException if an error happens in the underlying native code. + */ + public long getLongProperty( + /* @Nullable */ final ColumnFamilyHandle columnFamilyHandle, + final String property) throws RocksDBException { + return getLongProperty(nativeHandle_, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_, + property, property.length()); + } + + /** + * Reset internal stats for DB and all column families. + *

    + * Note this doesn't reset {@link Options#statistics()} as it is not + * owned by DB. + * + * @throws RocksDBException if an error occurs whilst reseting the stats + */ + public void resetStats() throws RocksDBException { + resetStats(nativeHandle_); + } + + /** + *

    Return sum of the getLongProperty of all the column families

    + * + *

    Note: As the returned property is of type + * {@code uint64_t} on C++ side the returning value can be negative + * because Java supports in Java 7 only signed long values.

    + * + *

    Java 7: To mitigate the problem of the non + * existent unsigned long tpye, values should be encapsulated using + * {@link java.math.BigInteger} to reflect the correct value. The correct + * behavior is guaranteed if {@code 2^64} is added to negative values.

    + * + *

    Java 8: In Java 8 the value should be treated as + * unsigned long using provided methods of type {@link Long}.

    + * + * @param property to be fetched. + * + * @return numerical property value + * + * @throws RocksDBException if an error happens in the underlying native code. + */ + public long getAggregatedLongProperty(final String property) + throws RocksDBException { + return getAggregatedLongProperty(nativeHandle_, property, + property.length()); + } + + /** + * Get the approximate file system space used by keys in each range. + *

    + * Note that the returned sizes measure file system space usage, so + * if the user data compresses by a factor of ten, the returned + * sizes will be one-tenth the size of the corresponding user data size. + *

    + * If {@code sizeApproximationFlags} defines whether the returned size + * should include the recently written data in the mem-tables (if + * the mem-table type supports it), data serialized to disk, or both. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance, or null for the default column family + * @param ranges the ranges over which to approximate sizes + * @param sizeApproximationFlags flags to determine what to include in the + * approximation. + * + * @return the sizes + */ + public long[] getApproximateSizes( + /*@Nullable*/ final ColumnFamilyHandle columnFamilyHandle, + final List ranges, + final SizeApproximationFlag... sizeApproximationFlags) { + + byte flags = 0x0; + for (final SizeApproximationFlag sizeApproximationFlag + : sizeApproximationFlags) { + flags |= sizeApproximationFlag.getValue(); + } + + return getApproximateSizes(nativeHandle_, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_, + toRangeSliceHandles(ranges), flags); + } + + /** + * Get the approximate file system space used by keys in each range for + * the default column family. + *

    + * Note that the returned sizes measure file system space usage, so + * if the user data compresses by a factor of ten, the returned + * sizes will be one-tenth the size of the corresponding user data size. + *

    + * If {@code sizeApproximationFlags} defines whether the returned size + * should include the recently written data in the mem-tables (if + * the mem-table type supports it), data serialized to disk, or both. + * + * @param ranges the ranges over which to approximate sizes + * @param sizeApproximationFlags flags to determine what to include in the + * approximation. + * + * @return the sizes. + */ + public long[] getApproximateSizes(final List ranges, + final SizeApproximationFlag... sizeApproximationFlags) { + return getApproximateSizes(null, ranges, sizeApproximationFlags); + } + + public static class CountAndSize { + public final long count; + public final long size; + + public CountAndSize(final long count, final long size) { + this.count = count; + this.size = size; + } + } + + /** + * This method is similar to + * {@link #getApproximateSizes(ColumnFamilyHandle, List, SizeApproximationFlag...)}, + * except that it returns approximate number of records and size in memtables. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance, or null for the default column family + * @param range the ranges over which to get the memtable stats + * + * @return the count and size for the range + */ + public CountAndSize getApproximateMemTableStats( + /*@Nullable*/ final ColumnFamilyHandle columnFamilyHandle, + final Range range) { + final long[] result = getApproximateMemTableStats(nativeHandle_, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_, + range.start.getNativeHandle(), + range.limit.getNativeHandle()); + return new CountAndSize(result[0], result[1]); + } + + /** + * This method is similar to + * {@link #getApproximateSizes(ColumnFamilyHandle, List, SizeApproximationFlag...)}, + * except that it returns approximate number of records and size in memtables. + * + * @param range the ranges over which to get the memtable stats + * + * @return the count and size for the range + */ + public CountAndSize getApproximateMemTableStats( + final Range range) { + return getApproximateMemTableStats(null, range); + } + + /** + *

    Range compaction of database.

    + *

    Note: After the database has been compacted, + * all data will have been pushed down to the last level containing + * any data.

    + * + *

    See also

    + *
      + *
    • {@link #compactRange(byte[], byte[])}
      • + *
    + * + * @throws RocksDBException thrown if an error occurs within the native + * part of the library. + */ + public void compactRange() throws RocksDBException { + compactRange(null); + } + + /** + *

    Range compaction of column family.

    + *

    Note: After the database has been compacted, + * all data will have been pushed down to the last level containing + * any data.

    + * + *

    See also

    + *
      + *
    • + * {@link #compactRange(ColumnFamilyHandle, byte[], byte[])} + *
      • + *
    + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance, or null for the default column family. + * + * @throws RocksDBException thrown if an error occurs within the native + * part of the library. + */ + public void compactRange( + /* @Nullable */ final ColumnFamilyHandle columnFamilyHandle) + throws RocksDBException { + compactRange(nativeHandle_, null, -1, null, -1, 0, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_); + } + + /** + *

    Range compaction of database.

    + *

    Note: After the database has been compacted, + * all data will have been pushed down to the last level containing + * any data.

    + * + *

    See also

    + *
      + *
    • {@link #compactRange()}
      • + *
    + * + * @param begin start of key range (included in range) + * @param end end of key range (excluded from range) + * + * @throws RocksDBException thrown if an error occurs within the native + * part of the library. + */ + public void compactRange(final byte[] begin, final byte[] end) + throws RocksDBException { + compactRange(null, begin, end); + } + + /** + *

    Range compaction of column family.

    + *

    Note: After the database has been compacted, + * all data will have been pushed down to the last level containing + * any data.

    + * + *

    See also

    + *
      + *
    • {@link #compactRange(ColumnFamilyHandle)}
      • + *
    + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance, or null for the default column family. + * @param begin start of key range (included in range) + * @param end end of key range (excluded from range) + * + * @throws RocksDBException thrown if an error occurs within the native + * part of the library. + */ + public void compactRange( + /* @Nullable */ final ColumnFamilyHandle columnFamilyHandle, + final byte[] begin, final byte[] end) throws RocksDBException { + compactRange(nativeHandle_, + begin, begin == null ? -1 : begin.length, + end, end == null ? -1 : end.length, + 0, columnFamilyHandle == null ? 0: columnFamilyHandle.nativeHandle_); + } + + /** + *

    Range compaction of column family.

    + *

    Note: After the database has been compacted, + * all data will have been pushed down to the last level containing + * any data.

    + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} instance. + * @param begin start of key range (included in range) + * @param end end of key range (excluded from range) + * @param compactRangeOptions options for the compaction + * + * @throws RocksDBException thrown if an error occurs within the native + * part of the library. + */ + public void compactRange( + /* @Nullable */ final ColumnFamilyHandle columnFamilyHandle, + final byte[] begin, final byte[] end, + final CompactRangeOptions compactRangeOptions) throws RocksDBException { + compactRange(nativeHandle_, + begin, begin == null ? -1 : begin.length, + end, end == null ? -1 : end.length, + compactRangeOptions.nativeHandle_, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_); + } + + /** + * Change the options for the column family handle. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance, or null for the default column family. + * @param mutableColumnFamilyOptions the options. + * + * @throws RocksDBException if an error occurs whilst setting the options + */ + public void setOptions( + /* @Nullable */final ColumnFamilyHandle columnFamilyHandle, + final MutableColumnFamilyOptions mutableColumnFamilyOptions) + throws RocksDBException { + setOptions(nativeHandle_, columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_, + mutableColumnFamilyOptions.getKeys(), mutableColumnFamilyOptions.getValues()); + } + + /** + * Get the options for the column family handle + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance, or null for the default column family. + * + * @return the options parsed from the options string return by RocksDB + * + * @throws RocksDBException if an error occurs while getting the options string, or parsing the + * resulting options string into options + */ + public MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder getOptions( + /* @Nullable */ final ColumnFamilyHandle columnFamilyHandle) throws RocksDBException { + final String optionsString = getOptions( + nativeHandle_, columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_); + return MutableColumnFamilyOptions.parse(optionsString, true); + } + + /** + * Default column family options + * + * @return the options parsed from the options string return by RocksDB + * + * @throws RocksDBException if an error occurs while getting the options string, or parsing the + * resulting options string into options + */ + public MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder getOptions() + throws RocksDBException { + return getOptions(null); + } + + /** + * Get the database options + * + * @return the DB options parsed from the options string return by RocksDB + * + * @throws RocksDBException if an error occurs while getting the options string, or parsing the + * resulting options string into options + */ + public MutableDBOptions.MutableDBOptionsBuilder getDBOptions() throws RocksDBException { + final String optionsString = getDBOptions(nativeHandle_); + return MutableDBOptions.parse(optionsString, true); + } + + /** + * Change the options for the default column family handle. + * + * @param mutableColumnFamilyOptions the options. + * + * @throws RocksDBException if an error occurs whilst setting the options + */ + public void setOptions( + final MutableColumnFamilyOptions mutableColumnFamilyOptions) + throws RocksDBException { + setOptions(null, mutableColumnFamilyOptions); + } + + /** + * Set the options for the column family handle. + * + * @param mutableDBoptions the options. + * + * @throws RocksDBException if an error occurs whilst setting the options + */ + public void setDBOptions(final MutableDBOptions mutableDBoptions) + throws RocksDBException { + setDBOptions(nativeHandle_, + mutableDBoptions.getKeys(), + mutableDBoptions.getValues()); + } + + /** + * Takes a list of files specified by file names and + * compacts them to the specified level. + *

    + * Note that the behavior is different from + * {@link #compactRange(ColumnFamilyHandle, byte[], byte[])} + * in that CompactFiles() performs the compaction job using the CURRENT + * thread. + * + * @param compactionOptions compaction options + * @param inputFileNames the name of the files to compact + * @param outputLevel the level to which they should be compacted + * @param outputPathId the id of the output path, or -1 + * @param compactionJobInfo the compaction job info, this parameter + * will be updated with the info from compacting the files, + * can just be null if you don't need it. + * + * @return the list of compacted files + * + * @throws RocksDBException if an error occurs during compaction + */ + public List compactFiles( + final CompactionOptions compactionOptions, + final List inputFileNames, + final int outputLevel, + final int outputPathId, + /* @Nullable */ final CompactionJobInfo compactionJobInfo) + throws RocksDBException { + return compactFiles(compactionOptions, null, inputFileNames, outputLevel, + outputPathId, compactionJobInfo); + } + + /** + * Takes a list of files specified by file names and + * compacts them to the specified level. + *

    + * Note that the behavior is different from + * {@link #compactRange(ColumnFamilyHandle, byte[], byte[])} + * in that CompactFiles() performs the compaction job using the CURRENT + * thread. + * + * @param compactionOptions compaction options + * @param columnFamilyHandle columnFamilyHandle, or null for the + * default column family + * @param inputFileNames the name of the files to compact + * @param outputLevel the level to which they should be compacted + * @param outputPathId the id of the output path, or -1 + * @param compactionJobInfo the compaction job info, this parameter + * will be updated with the info from compacting the files, + * can just be null if you don't need it. + * + * @return the list of compacted files + * + * @throws RocksDBException if an error occurs during compaction + */ + public List compactFiles( + final CompactionOptions compactionOptions, + /* @Nullable */ final ColumnFamilyHandle columnFamilyHandle, + final List inputFileNames, + final int outputLevel, + final int outputPathId, + /* @Nullable */ final CompactionJobInfo compactionJobInfo) + throws RocksDBException { + return Arrays.asList(compactFiles(nativeHandle_, compactionOptions.nativeHandle_, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_, + inputFileNames.toArray(new String[0]), + outputLevel, + outputPathId, + compactionJobInfo == null ? 0 : compactionJobInfo.nativeHandle_)); + } + + /** + * This function will cancel all currently running background processes. + * + * @param wait if true, wait for all background work to be cancelled before + * returning. + * + */ + public void cancelAllBackgroundWork(final boolean wait) { + cancelAllBackgroundWork(nativeHandle_, wait); + } + + /** + * This function will wait until all currently running background processes + * finish. After it returns, no background process will be run until + * {@link #continueBackgroundWork()} is called + * + * @throws RocksDBException if an error occurs when pausing background work + */ + public void pauseBackgroundWork() throws RocksDBException { + pauseBackgroundWork(nativeHandle_); + } + + /** + * Resumes background work which was suspended by + * previously calling {@link #pauseBackgroundWork()} + * + * @throws RocksDBException if an error occurs when resuming background work + */ + public void continueBackgroundWork() throws RocksDBException { + continueBackgroundWork(nativeHandle_); + } + + /** + * Enable automatic compactions for the given column + * families if they were previously disabled. + *

    + * The function will first set the + * {@link ColumnFamilyOptions#disableAutoCompactions()} option for each + * column family to false, after which it will schedule a flush/compaction. + *

    + * NOTE: Setting disableAutoCompactions to 'false' through + * {@link #setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)} + * does NOT schedule a flush/compaction afterwards, and only changes the + * parameter itself within the column family option. + * + * @param columnFamilyHandles the column family handles + * + * @throws RocksDBException if an error occurs whilst enabling auto-compaction + */ + public void enableAutoCompaction( + final List columnFamilyHandles) + throws RocksDBException { + enableAutoCompaction(nativeHandle_, + toNativeHandleList(columnFamilyHandles)); + } + + /** + * Number of levels used for this DB. + * + * @return the number of levels + */ + public int numberLevels() { + return numberLevels(null); + } + + /** + * Number of levels used for a column family in this DB. + * + * @param columnFamilyHandle the column family handle, or null + * for the default column family + * + * @return the number of levels + */ + public int numberLevels(/* @Nullable */final ColumnFamilyHandle columnFamilyHandle) { + return numberLevels(nativeHandle_, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_); + } + + /** + * Maximum level to which a new compacted memtable is pushed if it + * does not create overlap. + * + * @return the maximum level + */ + public int maxMemCompactionLevel() { + return maxMemCompactionLevel(null); + } + + /** + * Maximum level to which a new compacted memtable is pushed if it + * does not create overlap. + * + * @param columnFamilyHandle the column family handle + * + * @return the maximum level + */ + public int maxMemCompactionLevel( + /* @Nullable */ final ColumnFamilyHandle columnFamilyHandle) { + return maxMemCompactionLevel(nativeHandle_, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_); + } + + /** + * Number of files in level-0 that would stop writes. + * + * @return the number of files + */ + public int level0StopWriteTrigger() { + return level0StopWriteTrigger(null); + } + + /** + * Number of files in level-0 that would stop writes. + * + * @param columnFamilyHandle the column family handle + * + * @return the number of files + */ + public int level0StopWriteTrigger( + /* @Nullable */final ColumnFamilyHandle columnFamilyHandle) { + return level0StopWriteTrigger(nativeHandle_, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_); + } + + /** + * Get DB name -- the exact same name that was provided as an argument to + * as path to {@link #open(Options, String)}. + * + * @return the DB name + */ + public String getName() { + return getName(nativeHandle_); + } + + /** + * Get the Env object from the DB + * + * @return the env + */ + public Env getEnv() { + final long envHandle = getEnv(nativeHandle_); + if (envHandle == Env.getDefault().nativeHandle_) { + return Env.getDefault(); + } else { + final Env env = new RocksEnv(envHandle); + env.disOwnNativeHandle(); // we do not own the Env! + return env; + } + } + + /** + *

    Flush all memory table data.

    + * + *

    Note: it must be ensured that the FlushOptions instance + * is not GC'ed before this method finishes. If the wait parameter is + * set to false, flush processing is asynchronous.

    + * + * @param flushOptions {@link org.rocksdb.FlushOptions} instance. + * @throws RocksDBException thrown if an error occurs within the native + * part of the library. + */ + public void flush(final FlushOptions flushOptions) + throws RocksDBException { + flush(flushOptions, (List) null); + } + + /** + *

    Flush all memory table data.

    + * + *

    Note: it must be ensured that the FlushOptions instance + * is not GC'ed before this method finishes. If the wait parameter is + * set to false, flush processing is asynchronous.

    + * + * @param flushOptions {@link org.rocksdb.FlushOptions} instance. + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} instance. + * @throws RocksDBException thrown if an error occurs within the native + * part of the library. + */ + public void flush(final FlushOptions flushOptions, + /* @Nullable */ final ColumnFamilyHandle columnFamilyHandle) + throws RocksDBException { + flush(flushOptions, + columnFamilyHandle == null ? null : Collections.singletonList(columnFamilyHandle)); + } + + /** + * Flushes multiple column families. + *

    + * If atomic flush is not enabled, this is equivalent to calling + * {@link #flush(FlushOptions, ColumnFamilyHandle)} multiple times. + *

    + * If atomic flush is enabled, this will flush all column families + * specified up to the latest sequence number at the time when flush is + * requested. + * + * @param flushOptions {@link org.rocksdb.FlushOptions} instance. + * @param columnFamilyHandles column family handles. + * @throws RocksDBException thrown if an error occurs within the native + * part of the library. + */ + public void flush(final FlushOptions flushOptions, + /* @Nullable */ final List columnFamilyHandles) + throws RocksDBException { + flush(nativeHandle_, flushOptions.nativeHandle_, + toNativeHandleList(columnFamilyHandles)); + } + + /** + * Flush the WAL memory buffer to the file. If {@code sync} is true, + * it calls {@link #syncWal()} afterwards. + * + * @param sync true to also fsync to disk. + * + * @throws RocksDBException if an error occurs whilst flushing + */ + public void flushWal(final boolean sync) throws RocksDBException { + flushWal(nativeHandle_, sync); + } + + /** + * Sync the WAL. + *

    + * Note that {@link #write(WriteOptions, WriteBatch)} followed by + * {@code #syncWal()} is not exactly the same as + * {@link #write(WriteOptions, WriteBatch)} with + * {@link WriteOptions#sync()} set to true; In the latter case the changes + * won't be visible until the sync is done. + *

    + * Currently only works if {@link Options#allowMmapWrites()} is set to false. + * + * @throws RocksDBException if an error occurs whilst syncing + */ + public void syncWal() throws RocksDBException { + syncWal(nativeHandle_); + } + + /** + *

    The sequence number of the most recent transaction.

    + * + * @return sequence number of the most + * recent transaction. + */ + public long getLatestSequenceNumber() { + return getLatestSequenceNumber(nativeHandle_); + } + + /** + *

    Prevent file deletions. Compactions will continue to occur, + * but no obsolete files will be deleted. Calling this multiple + * times have the same effect as calling it once.

    + * + * @throws RocksDBException thrown if operation was not performed + * successfully. + */ + public void disableFileDeletions() throws RocksDBException { + disableFileDeletions(nativeHandle_); + } + + /** + *

    Allow compactions to delete obsolete files. + * If force == true, the call to EnableFileDeletions() + * will guarantee that file deletions are enabled after + * the call, even if DisableFileDeletions() was called + * multiple times before.

    + * + *

    If force == false, EnableFileDeletions will only + * enable file deletion after it's been called at least + * as many times as DisableFileDeletions(), enabling + * the two methods to be called by two threads + * concurrently without synchronization + * -- i.e., file deletions will be enabled only after both + * threads call EnableFileDeletions()

    + * + * @param force boolean value described above. + * + * @throws RocksDBException thrown if operation was not performed + * successfully. + */ + public void enableFileDeletions(final boolean force) + throws RocksDBException { + enableFileDeletions(nativeHandle_, force); + } + + public static class LiveFiles { + /** + * The valid size of the manifest file. The manifest file is an ever growing + * file, but only the portion specified here is valid for this snapshot. + */ + public final long manifestFileSize; + + /** + * The files are relative to the {@link #getName()} and are not + * absolute paths. Despite being relative paths, the file names begin + * with "/". + */ + public final List files; + + LiveFiles(final long manifestFileSize, final List files) { + this.manifestFileSize = manifestFileSize; + this.files = files; + } + } + + /** + * Retrieve the list of all files in the database after flushing the memtable. + *

    + * See {@link #getLiveFiles(boolean)}. + * + * @return the live files + * + * @throws RocksDBException if an error occurs whilst retrieving the list + * of live files + */ + public LiveFiles getLiveFiles() throws RocksDBException { + return getLiveFiles(true); + } + + /** + * Retrieve the list of all files in the database. + *

    + * In case you have multiple column families, even if {@code flushMemtable} + * is true, you still need to call {@link #getSortedWalFiles()} + * after {@code #getLiveFiles(boolean)} to compensate for new data that + * arrived to already-flushed column families while other column families + * were flushing. + *

    + * NOTE: Calling {@code #getLiveFiles(boolean)} followed by + * {@link #getSortedWalFiles()} can generate a lossless backup. + * + * @param flushMemtable set to true to flush before recoding the live + * files. Setting to false is useful when we don't want to wait for flush + * which may have to wait for compaction to complete taking an + * indeterminate time. + * + * @return the live files + * + * @throws RocksDBException if an error occurs whilst retrieving the list + * of live files + */ + public LiveFiles getLiveFiles(final boolean flushMemtable) + throws RocksDBException { + final String[] result = getLiveFiles(nativeHandle_, flushMemtable); + if (result == null) { + return null; + } + final String[] files = Arrays.copyOf(result, result.length - 1); + final long manifestFileSize = Long.parseLong(result[result.length - 1]); + + return new LiveFiles(manifestFileSize, Arrays.asList(files)); + } + + /** + * Retrieve the sorted list of all wal files with earliest file first. + * + * @return the log files + * + * @throws RocksDBException if an error occurs whilst retrieving the list + * of sorted WAL files + */ + public List getSortedWalFiles() throws RocksDBException { + final LogFile[] logFiles = getSortedWalFiles(nativeHandle_); + return Arrays.asList(logFiles); + } + + /** + *

    Returns an iterator that is positioned at a write-batch containing + * seq_number. If the sequence number is non existent, it returns an iterator + * at the first available seq_no after the requested seq_no.

    + * + *

    Must set WAL_ttl_seconds or WAL_size_limit_MB to large values to + * use this api, else the WAL files will get + * cleared aggressively and the iterator might keep getting invalid before + * an update is read.

    + * + * @param sequenceNumber sequence number offset + * + * @return {@link org.rocksdb.TransactionLogIterator} instance. + * + * @throws org.rocksdb.RocksDBException if iterator cannot be retrieved + * from native-side. + */ + public TransactionLogIterator getUpdatesSince(final long sequenceNumber) + throws RocksDBException { + return new TransactionLogIterator( + getUpdatesSince(nativeHandle_, sequenceNumber)); + } + + /** + * Delete the file name from the db directory and update the internal state to + * reflect that. Supports deletion of sst and log files only. 'name' must be + * path relative to the db directory. eg. 000001.sst, /archive/000003.log + * + * @param name the file name + * + * @throws RocksDBException if an error occurs whilst deleting the file + */ + public void deleteFile(final String name) throws RocksDBException { + deleteFile(nativeHandle_, name); + } + + /** + * Gets a list of all table files metadata. + * + * @return table files metadata. + */ + public List getLiveFilesMetaData() { + return Arrays.asList(getLiveFilesMetaData(nativeHandle_)); + } + + /** + * Obtains the meta data of the specified column family of the DB. + * + * @param columnFamilyHandle the column family + * + * @return the column family metadata + */ + public ColumnFamilyMetaData getColumnFamilyMetaData( + /* @Nullable */ final ColumnFamilyHandle columnFamilyHandle) { + return getColumnFamilyMetaData(nativeHandle_, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_); + } + + /** + * Obtains the meta data of the default column family of the DB. + * + * @return the column family metadata + */ + public ColumnFamilyMetaData getColumnFamilyMetaData() { + return getColumnFamilyMetaData(null); + } + + /** + * ingestExternalFile will load a list of external SST files (1) into the DB + * We will try to find the lowest possible level that the file can fit in, and + * ingest the file into this level (2). A file that have a key range that + * overlap with the memtable key range will require us to Flush the memtable + * first before ingesting the file. + *

    + * (1) External SST files can be created using {@link SstFileWriter} + * (2) We will try to ingest the files to the lowest possible level + * even if the file compression doesn't match the level compression + * + * @param filePathList The list of files to ingest + * @param ingestExternalFileOptions the options for the ingestion + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void ingestExternalFile(final List filePathList, + final IngestExternalFileOptions ingestExternalFileOptions) + throws RocksDBException { + ingestExternalFile(nativeHandle_, getDefaultColumnFamily().nativeHandle_, + filePathList.toArray(new String[0]), + filePathList.size(), ingestExternalFileOptions.nativeHandle_); + } + + /** + * ingestExternalFile will load a list of external SST files (1) into the DB + * We will try to find the lowest possible level that the file can fit in, and + * ingest the file into this level (2). A file that have a key range that + * overlap with the memtable key range will require us to Flush the memtable + * first before ingesting the file. + *

    + * (1) External SST files can be created using {@link SstFileWriter} + * (2) We will try to ingest the files to the lowest possible level + * even if the file compression doesn't match the level compression + * + * @param columnFamilyHandle The column family for the ingested files + * @param filePathList The list of files to ingest + * @param ingestExternalFileOptions the options for the ingestion + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void ingestExternalFile(final ColumnFamilyHandle columnFamilyHandle, + final List filePathList, + final IngestExternalFileOptions ingestExternalFileOptions) + throws RocksDBException { + ingestExternalFile(nativeHandle_, columnFamilyHandle.nativeHandle_, + filePathList.toArray(new String[0]), + filePathList.size(), ingestExternalFileOptions.nativeHandle_); + } + + /** + * Verify checksum + * + * @throws RocksDBException if the checksum is not valid + */ + public void verifyChecksum() throws RocksDBException { + verifyChecksum(nativeHandle_); + } + + /** + * Gets the handle for the default column family + * + * @return The handle of the default column family + */ + public ColumnFamilyHandle getDefaultColumnFamily() { + final ColumnFamilyHandle cfHandle = new ColumnFamilyHandle(this, + getDefaultColumnFamily(nativeHandle_)); + cfHandle.disOwnNativeHandle(); + return cfHandle; + } + + /** + * Get the properties of all tables. + * + * @param columnFamilyHandle the column family handle, or null for the default + * column family. + * + * @return the properties + * + * @throws RocksDBException if an error occurs whilst getting the properties + */ + public Map getPropertiesOfAllTables( + /* @Nullable */final ColumnFamilyHandle columnFamilyHandle) + throws RocksDBException { + return getPropertiesOfAllTables(nativeHandle_, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_); + } + + /** + * Get the properties of all tables in the default column family. + * + * @return the properties + * + * @throws RocksDBException if an error occurs whilst getting the properties + */ + public Map getPropertiesOfAllTables() + throws RocksDBException { + return getPropertiesOfAllTables(null); + } + + /** + * Get the properties of tables in range. + * + * @param columnFamilyHandle the column family handle, or null for the default + * column family. + * @param ranges the ranges over which to get the table properties + * + * @return the properties + * + * @throws RocksDBException if an error occurs whilst getting the properties + */ + public Map getPropertiesOfTablesInRange( + /* @Nullable */final ColumnFamilyHandle columnFamilyHandle, + final List ranges) throws RocksDBException { + return getPropertiesOfTablesInRange(nativeHandle_, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_, + toRangeSliceHandles(ranges)); + } + + /** + * Get the properties of tables in range for the default column family. + * + * @param ranges the ranges over which to get the table properties + * + * @return the properties + * + * @throws RocksDBException if an error occurs whilst getting the properties + */ + public Map getPropertiesOfTablesInRange( + final List ranges) throws RocksDBException { + return getPropertiesOfTablesInRange(null, ranges); + } + + /** + * Suggest the range to compact. + * + * @param columnFamilyHandle the column family handle, or null for the default + * column family. + * + * @return the suggested range. + * + * @throws RocksDBException if an error occurs whilst suggesting the range + */ + public Range suggestCompactRange( + /* @Nullable */final ColumnFamilyHandle columnFamilyHandle) + throws RocksDBException { + final long[] rangeSliceHandles = suggestCompactRange(nativeHandle_, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_); + return new Range(new Slice(rangeSliceHandles[0]), + new Slice(rangeSliceHandles[1])); + } + + /** + * Suggest the range to compact for the default column family. + * + * @return the suggested range. + * + * @throws RocksDBException if an error occurs whilst suggesting the range + */ + public Range suggestCompactRange() + throws RocksDBException { + return suggestCompactRange(null); + } + + /** + * Promote L0. + * + * @param columnFamilyHandle the column family handle, + * or null for the default column family. + * @param targetLevel the target level for L0 + * + * @throws RocksDBException if an error occurs whilst promoting L0 + */ + public void promoteL0( + /* @Nullable */final ColumnFamilyHandle columnFamilyHandle, + final int targetLevel) throws RocksDBException { + promoteL0(nativeHandle_, + columnFamilyHandle == null ? 0 : columnFamilyHandle.nativeHandle_, + targetLevel); + } + + /** + * Promote L0 for the default column family. + * + * @param targetLevel the target level for L0 + * + * @throws RocksDBException if an error occurs whilst promoting L0 + */ + public void promoteL0(final int targetLevel) + throws RocksDBException { + promoteL0(null, targetLevel); + } + + /** + * Trace DB operations. + *

    + * Use {@link #endTrace()} to stop tracing. + * + * @param traceOptions the options + * @param traceWriter the trace writer + * + * @throws RocksDBException if an error occurs whilst starting the trace + */ + public void startTrace(final TraceOptions traceOptions, + final AbstractTraceWriter traceWriter) throws RocksDBException { + startTrace(nativeHandle_, traceOptions.getMaxTraceFileSize(), + traceWriter.nativeHandle_); + /* + * NOTE: {@link #startTrace(long, long, long) transfers the ownership + * from Java to C++, so we must disown the native handle here. + */ + traceWriter.disOwnNativeHandle(); + } + + /** + * Stop tracing DB operations. + *

    + * See {@link #startTrace(TraceOptions, AbstractTraceWriter)} + * + * @throws RocksDBException if an error occurs whilst ending the trace + */ + public void endTrace() throws RocksDBException { + endTrace(nativeHandle_); + } + + /** + * Make the secondary instance catch up with the primary by tailing and + * replaying the MANIFEST and WAL of the primary. + * Column families created by the primary after the secondary instance starts + * will be ignored unless the secondary instance closes and restarts with the + * newly created column families. + * Column families that exist before secondary instance starts and dropped by + * the primary afterwards will be marked as dropped. However, as long as the + * secondary instance does not delete the corresponding column family + * handles, the data of the column family is still accessible to the + * secondary. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void tryCatchUpWithPrimary() throws RocksDBException { + tryCatchUpWithPrimary(nativeHandle_); + } + + /** + * Delete files in multiple ranges at once. + * Delete files in a lot of ranges one at a time can be slow, use this API for + * better performance in that case. + * + * @param columnFamily - The column family for operation (null for default) + * @param includeEnd - Whether ranges should include end + * @param ranges - pairs of ranges (from1, to1, from2, to2, ...) + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void deleteFilesInRanges(final ColumnFamilyHandle columnFamily, + final List ranges, final boolean includeEnd) + throws RocksDBException { + if (ranges.size() == 0) { + return; + } + if ((ranges.size() % 2) != 0) { + throw new IllegalArgumentException("Ranges size needs to be multiple of 2 " + + "(from1, to1, from2, to2, ...), but is " + ranges.size()); + } + + final byte[][] rangesArray = ranges.toArray(new byte[ranges.size()][]); + + deleteFilesInRanges(nativeHandle_, columnFamily == null ? 0 : columnFamily.nativeHandle_, + rangesArray, includeEnd); + } + + /** + * Static method to destroy the contents of the specified database. + * Be very careful using this method. + * + * @param path the path to the Rocksdb database. + * @param options {@link org.rocksdb.Options} instance. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public static void destroyDB(final String path, final Options options) + throws RocksDBException { + destroyDB(path, options.nativeHandle_); + } + + private /* @Nullable */ long[] toNativeHandleList( + /* @Nullable */ final List objectList) { + if (objectList == null) { + return null; + } + final int len = objectList.size(); + final long[] handleList = new long[len]; + for (int i = 0; i < len; i++) { + handleList[i] = objectList.get(i).nativeHandle_; + } + return handleList; + } + + private static long[] toRangeSliceHandles(final List ranges) { + final long[] rangeSliceHandles = new long[ranges.size() * 2]; + for (int i = 0, j = 0; i < ranges.size(); i++) { + final Range range = ranges.get(i); + rangeSliceHandles[j++] = range.start.getNativeHandle(); + rangeSliceHandles[j++] = range.limit.getNativeHandle(); + } + return rangeSliceHandles; + } + + protected void storeOptionsInstance(final DBOptionsInterface options) { + options_ = options; + } + + private static void checkBounds(final int offset, final int len, final int size) { + if ((offset | len | (offset + len) | (size - (offset + len))) < 0) { + throw new IndexOutOfBoundsException(String.format("offset(%d), len(%d), size(%d)", offset, len, size)); + } + } + + private static int computeCapacityHint(final int estimatedNumberOfItems) { + // Default load factor for HashMap is 0.75, so N * 1.5 will be at the load + // limit. We add +1 for a buffer. + return (int)Math.ceil(estimatedNumberOfItems * 1.5 + 1.0); + } + + // native methods + private static native long open(final long optionsHandle, final String path) + throws RocksDBException; + + /** + * @param optionsHandle Native handle pointing to an Options object + * @param path The directory path for the database files + * @param columnFamilyNames An array of column family names + * @param columnFamilyOptions An array of native handles pointing to + * ColumnFamilyOptions objects + * + * @return An array of native handles, [0] is the handle of the RocksDB object + * [1..1+n] are handles of the ColumnFamilyReferences + * + * @throws RocksDBException thrown if the database could not be opened + */ + private static native long[] open(final long optionsHandle, final String path, + final byte[][] columnFamilyNames, final long[] columnFamilyOptions) throws RocksDBException; + + private static native long openROnly(final long optionsHandle, final String path, + final boolean errorIfWalFileExists) throws RocksDBException; + + /** + * @param optionsHandle Native handle pointing to an Options object + * @param path The directory path for the database files + * @param columnFamilyNames An array of column family names + * @param columnFamilyOptions An array of native handles pointing to + * ColumnFamilyOptions objects + * + * @return An array of native handles, [0] is the handle of the RocksDB object + * [1..1+n] are handles of the ColumnFamilyReferences + * + * @throws RocksDBException thrown if the database could not be opened + */ + private static native long[] openROnly(final long optionsHandle, final String path, + final byte[][] columnFamilyNames, final long[] columnFamilyOptions, + final boolean errorIfWalFileExists) throws RocksDBException; + + private static native long openAsSecondary(final long optionsHandle, final String path, + final String secondaryPath) throws RocksDBException; + + private static native long[] openAsSecondary(final long optionsHandle, final String path, + final String secondaryPath, final byte[][] columnFamilyNames, + final long[] columnFamilyOptions) throws RocksDBException; + + @Override protected native void disposeInternal(final long handle); + + private static native void closeDatabase(final long handle) throws RocksDBException; + private static native byte[][] listColumnFamilies(final long optionsHandle, final String path) + throws RocksDBException; + private native long createColumnFamily(final long handle, + final byte[] columnFamilyName, final int columnFamilyNamelen, + final long columnFamilyOptions) throws RocksDBException; + private native long[] createColumnFamilies(final long handle, + final long columnFamilyOptionsHandle, final byte[][] columnFamilyNames) + throws RocksDBException; + private native long[] createColumnFamilies( + final long handle, final long[] columnFamilyOptionsHandles, final byte[][] columnFamilyNames) + throws RocksDBException; + private native void dropColumnFamily( + final long handle, final long cfHandle) throws RocksDBException; + private native void dropColumnFamilies(final long handle, + final long[] cfHandles) throws RocksDBException; + private native void put(final long handle, final byte[] key, + final int keyOffset, final int keyLength, final byte[] value, + final int valueOffset, int valueLength) throws RocksDBException; + private native void put(final long handle, final byte[] key, final int keyOffset, + final int keyLength, final byte[] value, final int valueOffset, + final int valueLength, final long cfHandle) throws RocksDBException; + private native void put(final long handle, final long writeOptHandle, + final byte[] key, final int keyOffset, final int keyLength, + final byte[] value, final int valueOffset, final int valueLength) + throws RocksDBException; + private native void put(final long handle, final long writeOptHandle, + final byte[] key, final int keyOffset, final int keyLength, + final byte[] value, final int valueOffset, final int valueLength, + final long cfHandle) throws RocksDBException; + private native void delete(final long handle, final byte[] key, + final int keyOffset, final int keyLength) throws RocksDBException; + private native void delete(final long handle, final byte[] key, + final int keyOffset, final int keyLength, final long cfHandle) + throws RocksDBException; + private native void delete(final long handle, final long writeOptHandle, + final byte[] key, final int keyOffset, final int keyLength) + throws RocksDBException; + private native void delete(final long handle, final long writeOptHandle, + final byte[] key, final int keyOffset, final int keyLength, + final long cfHandle) throws RocksDBException; + private native void singleDelete( + final long handle, final byte[] key, final int keyLen) + throws RocksDBException; + private native void singleDelete( + final long handle, final byte[] key, final int keyLen, + final long cfHandle) throws RocksDBException; + private native void singleDelete( + final long handle, final long writeOptHandle, final byte[] key, + final int keyLen) throws RocksDBException; + private native void singleDelete( + final long handle, final long writeOptHandle, + final byte[] key, final int keyLen, final long cfHandle) + throws RocksDBException; + private native void deleteRange(final long handle, final byte[] beginKey, + final int beginKeyOffset, final int beginKeyLength, final byte[] endKey, + final int endKeyOffset, final int endKeyLength) throws RocksDBException; + private native void deleteRange(final long handle, final byte[] beginKey, + final int beginKeyOffset, final int beginKeyLength, final byte[] endKey, + final int endKeyOffset, final int endKeyLength, final long cfHandle) + throws RocksDBException; + private native void deleteRange(final long handle, final long writeOptHandle, + final byte[] beginKey, final int beginKeyOffset, final int beginKeyLength, + final byte[] endKey, final int endKeyOffset, final int endKeyLength) + throws RocksDBException; + private native void deleteRange( + final long handle, final long writeOptHandle, final byte[] beginKey, + final int beginKeyOffset, final int beginKeyLength, final byte[] endKey, + final int endKeyOffset, final int endKeyLength, final long cfHandle) + throws RocksDBException; + private native void merge(final long handle, final byte[] key, + final int keyOffset, final int keyLength, final byte[] value, + final int valueOffset, final int valueLength) throws RocksDBException; + private native void merge(final long handle, final byte[] key, + final int keyOffset, final int keyLength, final byte[] value, + final int valueOffset, final int valueLength, final long cfHandle) + throws RocksDBException; + private native void merge(final long handle, final long writeOptHandle, + final byte[] key, final int keyOffset, final int keyLength, + final byte[] value, final int valueOffset, final int valueLength) + throws RocksDBException; + private native void merge(final long handle, final long writeOptHandle, + final byte[] key, final int keyOffset, final int keyLength, + final byte[] value, final int valueOffset, final int valueLength, + final long cfHandle) throws RocksDBException; + private native void write0(final long handle, final long writeOptHandle, + final long wbHandle) throws RocksDBException; + private native void write1(final long handle, final long writeOptHandle, + final long wbwiHandle) throws RocksDBException; + private native int get(final long handle, final byte[] key, + final int keyOffset, final int keyLength, final byte[] value, + final int valueOffset, final int valueLength) throws RocksDBException; + private native int get(final long handle, final byte[] key, + final int keyOffset, final int keyLength, byte[] value, + final int valueOffset, final int valueLength, final long cfHandle) + throws RocksDBException; + private native int get(final long handle, final long readOptHandle, + final byte[] key, final int keyOffset, final int keyLength, + final byte[] value, final int valueOffset, final int valueLength) + throws RocksDBException; + private native int get(final long handle, final long readOptHandle, + final byte[] key, final int keyOffset, final int keyLength, + final byte[] value, final int valueOffset, final int valueLength, + final long cfHandle) throws RocksDBException; + private native byte[] get(final long handle, byte[] key, final int keyOffset, + final int keyLength) throws RocksDBException; + private native byte[] get(final long handle, final byte[] key, + final int keyOffset, final int keyLength, final long cfHandle) + throws RocksDBException; + private native byte[] get(final long handle, final long readOptHandle, + final byte[] key, final int keyOffset, final int keyLength) + throws RocksDBException; + private native byte[] get(final long handle, + final long readOptHandle, final byte[] key, final int keyOffset, + final int keyLength, final long cfHandle) throws RocksDBException; + private native byte[][] multiGet(final long dbHandle, final byte[][] keys, + final int[] keyOffsets, final int[] keyLengths); + private native byte[][] multiGet(final long dbHandle, final byte[][] keys, + final int[] keyOffsets, final int[] keyLengths, + final long[] columnFamilyHandles); + private native byte[][] multiGet(final long dbHandle, final long rOptHandle, + final byte[][] keys, final int[] keyOffsets, final int[] keyLengths); + private native byte[][] multiGet(final long dbHandle, final long rOptHandle, + final byte[][] keys, final int[] keyOffsets, final int[] keyLengths, + final long[] columnFamilyHandles); + + private native void multiGet(final long dbHandle, final long rOptHandle, + final long[] columnFamilyHandles, final ByteBuffer[] keysArray, final int[] keyOffsets, + final int[] keyLengths, final ByteBuffer[] valuesArray, final int[] valuesSizeArray, + final Status[] statusArray); + + private native boolean keyMayExist( + final long handle, final long cfHandle, final long readOptHandle, + final byte[] key, final int keyOffset, final int keyLength); + private native byte[][] keyMayExistFoundValue( + final long handle, final long cfHandle, final long readOptHandle, + final byte[] key, final int keyOffset, final int keyLength); + private native void putDirect(long handle, long writeOptHandle, ByteBuffer key, int keyOffset, + int keyLength, ByteBuffer value, int valueOffset, int valueLength, long cfHandle) + throws RocksDBException; + private native long iterator(final long handle); + private native long iterator(final long handle, final long readOptHandle); + private native long iteratorCF(final long handle, final long cfHandle); + private native long iteratorCF(final long handle, final long cfHandle, + final long readOptHandle); + private native long[] iterators(final long handle, + final long[] columnFamilyHandles, final long readOptHandle) + throws RocksDBException; + private native long getSnapshot(final long nativeHandle); + private native void releaseSnapshot( + final long nativeHandle, final long snapshotHandle); + private native String getProperty(final long nativeHandle, + final long cfHandle, final String property, final int propertyLength) + throws RocksDBException; + private native Map getMapProperty(final long nativeHandle, + final long cfHandle, final String property, final int propertyLength) + throws RocksDBException; + private native int getDirect(long handle, long readOptHandle, ByteBuffer key, int keyOffset, + int keyLength, ByteBuffer value, int valueOffset, int valueLength, long cfHandle) + throws RocksDBException; + private native boolean keyMayExistDirect(final long handle, final long cfHhandle, + final long readOptHandle, final ByteBuffer key, final int keyOffset, final int keyLength); + private native int[] keyMayExistDirectFoundValue(final long handle, final long cfHhandle, + final long readOptHandle, final ByteBuffer key, final int keyOffset, final int keyLength, + final ByteBuffer value, final int valueOffset, final int valueLength); + private native void deleteDirect(long handle, long optHandle, ByteBuffer key, int keyOffset, + int keyLength, long cfHandle) throws RocksDBException; + private native long getLongProperty(final long nativeHandle, + final long cfHandle, final String property, final int propertyLength) + throws RocksDBException; + private native void resetStats(final long nativeHandle) + throws RocksDBException; + private native long getAggregatedLongProperty(final long nativeHandle, + final String property, int propertyLength) throws RocksDBException; + private native long[] getApproximateSizes(final long nativeHandle, + final long columnFamilyHandle, final long[] rangeSliceHandles, + final byte includeFlags); + private native long[] getApproximateMemTableStats(final long nativeHandle, + final long columnFamilyHandle, final long rangeStartSliceHandle, + final long rangeLimitSliceHandle); + private native void compactRange(final long handle, + /* @Nullable */ final byte[] begin, final int beginLen, + /* @Nullable */ final byte[] end, final int endLen, + final long compactRangeOptHandle, final long cfHandle) + throws RocksDBException; + private native void setOptions(final long handle, final long cfHandle, + final String[] keys, final String[] values) throws RocksDBException; + private native String getOptions(final long handle, final long cfHandle); + private native void setDBOptions(final long handle, + final String[] keys, final String[] values) throws RocksDBException; + private native String getDBOptions(final long handle); + private native String[] compactFiles(final long handle, + final long compactionOptionsHandle, + final long columnFamilyHandle, + final String[] inputFileNames, + final int outputLevel, + final int outputPathId, + final long compactionJobInfoHandle) throws RocksDBException; + private native void cancelAllBackgroundWork(final long handle, + final boolean wait); + private native void pauseBackgroundWork(final long handle) + throws RocksDBException; + private native void continueBackgroundWork(final long handle) + throws RocksDBException; + private native void enableAutoCompaction(final long handle, + final long[] columnFamilyHandles) throws RocksDBException; + private native int numberLevels(final long handle, + final long columnFamilyHandle); + private native int maxMemCompactionLevel(final long handle, + final long columnFamilyHandle); + private native int level0StopWriteTrigger(final long handle, + final long columnFamilyHandle); + private native String getName(final long handle); + private native long getEnv(final long handle); + private native void flush(final long handle, final long flushOptHandle, + /* @Nullable */ final long[] cfHandles) throws RocksDBException; + private native void flushWal(final long handle, final boolean sync) + throws RocksDBException; + private native void syncWal(final long handle) throws RocksDBException; + private native long getLatestSequenceNumber(final long handle); + private native void disableFileDeletions(long handle) throws RocksDBException; + private native void enableFileDeletions(long handle, boolean force) + throws RocksDBException; + private native String[] getLiveFiles(final long handle, + final boolean flushMemtable) throws RocksDBException; + private native LogFile[] getSortedWalFiles(final long handle) + throws RocksDBException; + private native long getUpdatesSince(final long handle, + final long sequenceNumber) throws RocksDBException; + private native void deleteFile(final long handle, final String name) + throws RocksDBException; + private native LiveFileMetaData[] getLiveFilesMetaData(final long handle); + private native ColumnFamilyMetaData getColumnFamilyMetaData( + final long handle, final long columnFamilyHandle); + private native void ingestExternalFile(final long handle, + final long columnFamilyHandle, final String[] filePathList, + final int filePathListLen, final long ingestExternalFileOptionsHandle) + throws RocksDBException; + private native void verifyChecksum(final long handle) throws RocksDBException; + private native long getDefaultColumnFamily(final long handle); + private native Map getPropertiesOfAllTables( + final long handle, final long columnFamilyHandle) throws RocksDBException; + private native Map getPropertiesOfTablesInRange( + final long handle, final long columnFamilyHandle, + final long[] rangeSliceHandles); + private native long[] suggestCompactRange(final long handle, + final long columnFamilyHandle) throws RocksDBException; + private native void promoteL0(final long handle, + final long columnFamilyHandle, final int tragetLevel) + throws RocksDBException; + private native void startTrace(final long handle, final long maxTraceFileSize, + final long traceWriterHandle) throws RocksDBException; + private native void endTrace(final long handle) throws RocksDBException; + private native void tryCatchUpWithPrimary(final long handle) throws RocksDBException; + private native void deleteFilesInRanges(long handle, long cfHandle, final byte[][] ranges, + boolean include_end) throws RocksDBException; + + private static native void destroyDB(final String path, final long optionsHandle) + throws RocksDBException; + + private static native int version(); + + protected DBOptionsInterface options_; + private static Version version; + + public static class Version { + private final byte major; + private final byte minor; + private final byte patch; + + public Version(final byte major, final byte minor, final byte patch) { + this.major = major; + this.minor = minor; + this.patch = patch; + } + + public int getMajor() { + return major; + } + + public int getMinor() { + return minor; + } + + public int getPatch() { + return patch; + } + + @Override + public String toString() { + return getMajor() + "." + getMinor() + "." + getPatch(); + } + + private static Version fromEncodedVersion(int encodedVersion) { + final byte patch = (byte) (encodedVersion & 0xff); + encodedVersion >>= 8; + final byte minor = (byte) (encodedVersion & 0xff); + encodedVersion >>= 8; + final byte major = (byte) (encodedVersion & 0xff); + + return new Version(major, minor, patch); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksDBException.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksDBException.java new file mode 100644 index 0000000..8b035f4 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksDBException.java @@ -0,0 +1,44 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * A RocksDBException encapsulates the error of an operation. This exception + * type is used to describe an internal error from the c++ rocksdb library. + */ +public class RocksDBException extends Exception { + + /* @Nullable */ private final Status status; + + /** + * The private construct used by a set of public static factory method. + * + * @param msg the specified error message. + */ + public RocksDBException(final String msg) { + this(msg, null); + } + + public RocksDBException(final String msg, final Status status) { + super(msg); + this.status = status; + } + + public RocksDBException(final Status status) { + super(status.getState() != null ? status.getState() + : status.getCodeString()); + this.status = status; + } + + /** + * Get the status returned from RocksDB + * + * @return The status reported by RocksDB, or null if no status is available + */ + public Status getStatus() { + return status; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksEnv.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksEnv.java new file mode 100644 index 0000000..ca010c9 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksEnv.java @@ -0,0 +1,31 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + *

    A RocksEnv is an interface used by the rocksdb implementation to access + * operating system functionality like the filesystem etc.

    + * + *

    All Env implementations are safe for concurrent access from + * multiple threads without any external synchronization.

    + */ +public class RocksEnv extends Env { + + /** + *

    Package-private constructor that uses the specified native handle + * to construct a RocksEnv.

    + * + *

    Note that the ownership of the input handle + * belongs to the caller, and the newly created RocksEnv will not take + * the ownership of the input handle. As a result, calling + * {@code dispose()} of the created RocksEnv will be no-op.

    + */ + RocksEnv(final long handle) { + super(handle); + } + + @Override protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksIterator.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksIterator.java new file mode 100644 index 0000000..20e56d2 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksIterator.java @@ -0,0 +1,140 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.nio.ByteBuffer; + +/** + *

    An iterator that yields a sequence of key/value pairs from a source. + * Multiple implementations are provided by this library. + * In particular, iterators are provided + * to access the contents of a Table or a DB.

    + * + *

    Multiple threads can invoke const methods on an RocksIterator without + * external synchronization, but if any of the threads may call a + * non-const method, all threads accessing the same RocksIterator must use + * external synchronization.

    + * + * @see org.rocksdb.RocksObject + */ +public class RocksIterator extends AbstractRocksIterator { + protected RocksIterator(final RocksDB rocksDB, final long nativeHandle) { + super(rocksDB, nativeHandle); + } + + /** + *

    Return the key for the current entry. The underlying storage for + * the returned slice is valid only until the next modification of + * the iterator.

    + * + *

    REQUIRES: {@link #isValid()}

    + * + * @return key for the current entry. + */ + public byte[] key() { + assert(isOwningHandle()); + return key0(nativeHandle_); + } + + /** + *

    Return the key for the current entry. The underlying storage for + * the returned slice is valid only until the next modification of + * the iterator.

    + * + *

    REQUIRES: {@link #isValid()}

    + * + * @param key the out-value to receive the retrieved key. + * It is using position and limit. Limit is set according to key size. + * Supports direct buffer only. + * @return The size of the actual key. If the return key is greater than the + * length of {@code key}, then it indicates that the size of the + * input buffer {@code key} is insufficient and partial result will + * be returned. + */ + public int key(final ByteBuffer key) { + assert isOwningHandle(); + final int result; + if (key.isDirect()) { + result = keyDirect0(nativeHandle_, key, key.position(), key.remaining()); + } else { + assert key.hasArray(); + result = keyByteArray0( + nativeHandle_, key.array(), key.arrayOffset() + key.position(), key.remaining()); + } + key.limit(Math.min(key.position() + result, key.limit())); + return result; + } + + /** + *

    Return the value for the current entry. The underlying storage for + * the returned slice is valid only until the next modification of + * the iterator.

    + * + *

    REQUIRES: !AtEnd() && !AtStart()

    + * @return value for the current entry. + */ + public byte[] value() { + assert(isOwningHandle()); + return value0(nativeHandle_); + } + + /** + *

    Return the value for the current entry. The underlying storage for + * the returned slice is valid only until the next modification of + * the iterator.

    + * + *

    REQUIRES: {@link #isValid()}

    + * + * @param value the out-value to receive the retrieved value. + * It is using position and limit. Limit is set according to value size. + * Supports direct buffer only. + * @return The size of the actual value. If the return value is greater than the + * length of {@code value}, then it indicates that the size of the + * input buffer {@code value} is insufficient and partial result will + * be returned. + */ + public int value(final ByteBuffer value) { + assert isOwningHandle(); + final int result; + if (value.isDirect()) { + result = valueDirect0(nativeHandle_, value, value.position(), value.remaining()); + } else { + assert value.hasArray(); + result = valueByteArray0( + nativeHandle_, value.array(), value.arrayOffset() + value.position(), value.remaining()); + } + value.limit(Math.min(value.position() + result, value.limit())); + return result; + } + + @Override protected final native void disposeInternal(final long handle); + @Override final native boolean isValid0(long handle); + @Override final native void seekToFirst0(long handle); + @Override final native void seekToLast0(long handle); + @Override final native void next0(long handle); + @Override final native void prev0(long handle); + @Override final native void refresh0(long handle); + @Override final native void seek0(long handle, byte[] target, int targetLen); + @Override final native void seekForPrev0(long handle, byte[] target, int targetLen); + @Override + final native void seekDirect0(long handle, ByteBuffer target, int targetOffset, int targetLen); + @Override + final native void seekByteArray0(long handle, byte[] target, int targetOffset, int targetLen); + @Override + final native void seekForPrevDirect0( + long handle, ByteBuffer target, int targetOffset, int targetLen); + @Override + final native void seekForPrevByteArray0( + long handle, byte[] target, int targetOffset, int targetLen); + @Override final native void status0(long handle) throws RocksDBException; + + private native byte[] key0(long handle); + private native byte[] value0(long handle); + private native int keyDirect0(long handle, ByteBuffer buffer, int bufferOffset, int bufferLen); + private native int keyByteArray0(long handle, byte[] array, int arrayOffset, int arrayLen); + private native int valueDirect0(long handle, ByteBuffer buffer, int bufferOffset, int bufferLen); + private native int valueByteArray0(long handle, byte[] array, int arrayOffset, int arrayLen); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksIteratorInterface.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksIteratorInterface.java new file mode 100644 index 0000000..819c21c --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksIteratorInterface.java @@ -0,0 +1,127 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.nio.ByteBuffer; + +/** + *

    Defines the interface for an Iterator which provides + * access to data one entry at a time. Multiple implementations + * are provided by this library. In particular, iterators are provided + * to access the contents of a DB and Write Batch.

    + * + *

    Multiple threads can invoke const methods on an RocksIterator without + * external synchronization, but if any of the threads may call a + * non-const method, all threads accessing the same RocksIterator must use + * external synchronization.

    + * + * @see org.rocksdb.RocksObject + */ +public interface RocksIteratorInterface { + + /** + *

    An iterator is either positioned at an entry, or + * not valid. This method returns true if the iterator is valid.

    + * + * @return true if iterator is valid. + */ + boolean isValid(); + + /** + *

    Position at the first entry in the source. The iterator is Valid() + * after this call if the source is not empty.

    + */ + void seekToFirst(); + + /** + *

    Position at the last entry in the source. The iterator is + * valid after this call if the source is not empty.

    + */ + void seekToLast(); + + /** + *

    Position at the first entry in the source whose key is at or + * past target.

    + * + *

    The iterator is valid after this call if the source contains + * a key that comes at or past target.

    + * + * @param target byte array describing a key or a + * key prefix to seek for. + */ + void seek(byte[] target); + + /** + *

    Position at the first entry in the source whose key is that or + * before target.

    + * + *

    The iterator is valid after this call if the source contains + * a key that comes at or before target.

    + * + * @param target byte array describing a key or a + * key prefix to seek for. + */ + void seekForPrev(byte[] target); + + /** + *

    Position at the first entry in the source whose key is that or + * past target.

    + * + *

    The iterator is valid after this call if the source contains + * a key that comes at or past target.

    + * + * @param target byte array describing a key or a + * key prefix to seek for. Supports direct buffer only. + */ + void seek(ByteBuffer target); + + /** + *

    Position at the last key that is less than or equal to the target key.

    + * + *

    The iterator is valid after this call if the source contains + * a key that comes at or past target.

    + * + * @param target byte array describing a key or a + * key prefix to seek for. Supports direct buffer only. + */ + void seekForPrev(ByteBuffer target); + + /** + *

    Moves to the next entry in the source. After this call, Valid() is + * true if the iterator was not positioned at the last entry in the source.

    + * + *

    REQUIRES: {@link #isValid()}

    + */ + void next(); + + /** + *

    Moves to the previous entry in the source. After this call, Valid() is + * true if the iterator was not positioned at the first entry in source.

    + * + *

    REQUIRES: {@link #isValid()}

    + */ + void prev(); + + /** + *

    If an error has occurred, return it. Else return an ok status. + * If non-blocking IO is requested and this operation cannot be + * satisfied without doing some IO, then this returns Status::Incomplete().

    + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + void status() throws RocksDBException; + + /** + *

    If supported, renew the iterator to represent the latest state. The iterator will be + * invalidated after the call. Not supported if {@link ReadOptions#setSnapshot(Snapshot)} was + * specified when creating the iterator.

    + * + * @throws RocksDBException thrown if the operation is not supported or an error happens in the + * underlying native library + */ + void refresh() throws RocksDBException; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksMemEnv.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksMemEnv.java new file mode 100644 index 0000000..39a6f6e --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksMemEnv.java @@ -0,0 +1,31 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Memory environment. + */ +//TODO(AR) rename to MemEnv +public class RocksMemEnv extends Env { + + /** + *

    Creates a new environment that stores its data + * in memory and delegates all non-file-storage tasks to + * {@code baseEnv}.

    + * + *

    The caller must delete the result when it is + * no longer needed.

    + * + * @param baseEnv the base environment, + * must remain live while the result is in use. + */ + public RocksMemEnv(final Env baseEnv) { + super(createMemEnv(baseEnv.nativeHandle_)); + } + + private static native long createMemEnv(final long baseEnvHandle); + @Override protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksMutableObject.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksMutableObject.java new file mode 100644 index 0000000..eb32152 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksMutableObject.java @@ -0,0 +1,87 @@ +// Copyright (c) 2016, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * RocksMutableObject is an implementation of {@link AbstractNativeReference} + * whose reference to the underlying native C++ object can change. + * + *

    The use of {@code RocksMutableObject} should be kept to a minimum, as it + * has synchronization overheads and introduces complexity. Instead it is + * recommended to use {@link RocksObject} where possible.

    + */ +public abstract class RocksMutableObject extends AbstractNativeReference { + + /** + * An mutable reference to the value of the C++ pointer pointing to some + * underlying native RocksDB C++ object. + */ + private long nativeHandle_; + private boolean owningHandle_; + + protected RocksMutableObject() { + } + + protected RocksMutableObject(final long nativeHandle) { + this.nativeHandle_ = nativeHandle; + this.owningHandle_ = true; + } + + /** + * Closes the existing handle, and changes the handle to the new handle + * + * @param newNativeHandle The C++ pointer to the new native object + * @param owningNativeHandle true if we own the new native object + */ + public synchronized void resetNativeHandle(final long newNativeHandle, + final boolean owningNativeHandle) { + close(); + setNativeHandle(newNativeHandle, owningNativeHandle); + } + + /** + * Sets the handle (C++ pointer) of the underlying C++ native object + * + * @param nativeHandle The C++ pointer to the native object + * @param owningNativeHandle true if we own the native object + */ + public synchronized void setNativeHandle(final long nativeHandle, + final boolean owningNativeHandle) { + this.nativeHandle_ = nativeHandle; + this.owningHandle_ = owningNativeHandle; + } + + @Override + protected synchronized boolean isOwningHandle() { + return this.owningHandle_; + } + + /** + * Gets the value of the C++ pointer pointing to the underlying + * native C++ object + * + * @return the pointer value for the native object + */ + protected synchronized long getNativeHandle() { + assert (this.nativeHandle_ != 0); + return this.nativeHandle_; + } + + @Override + public final synchronized void close() { + if (isOwningHandle()) { + disposeInternal(); + this.owningHandle_ = false; + this.nativeHandle_ = 0; + } + } + + protected void disposeInternal() { + disposeInternal(nativeHandle_); + } + + protected abstract void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksObject.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksObject.java new file mode 100644 index 0000000..f07e101 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/RocksObject.java @@ -0,0 +1,45 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * RocksObject is an implementation of {@link AbstractNativeReference} which + * has an immutable and therefore thread-safe reference to the underlying + * native C++ RocksDB object. + *

    + * RocksObject is the base-class of almost all RocksDB classes that have a + * pointer to some underlying native C++ {@code rocksdb} object.

    + *

    + * The use of {@code RocksObject} should always be preferred over + * {@link RocksMutableObject}.

    + */ +public abstract class RocksObject extends AbstractImmutableNativeReference { + + /** + * An immutable reference to the value of the C++ pointer pointing to some + * underlying native RocksDB C++ object. + */ + protected final long nativeHandle_; + + protected RocksObject(final long nativeHandle) { + super(true); + this.nativeHandle_ = nativeHandle; + } + + /** + * Deletes underlying C++ object pointer. + */ + @Override + protected void disposeInternal() { + disposeInternal(nativeHandle_); + } + + protected abstract void disposeInternal(final long handle); + + public long getNativeHandle() { + return nativeHandle_; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SanityLevel.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SanityLevel.java new file mode 100644 index 0000000..30568c3 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SanityLevel.java @@ -0,0 +1,47 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +public enum SanityLevel { + NONE((byte) 0x0), + LOOSELY_COMPATIBLE((byte) 0x1), + EXACT_MATCH((byte) 0xFF); + + private final byte value; + + SanityLevel(final byte value) { + this.value = value; + } + + /** + * Get the internal representation value. + * + * @return the internal representation value. + */ + byte getValue() { + return value; + } + + /** + * Get the SanityLevel from the internal representation value. + * + * @param value the internal representation value. + * + * @return the SanityLevel + * + * @throws IllegalArgumentException if the value does not match a + * SanityLevel + */ + static SanityLevel fromValue(final byte value) throws IllegalArgumentException { + for (final SanityLevel level : SanityLevel.values()) { + if (level.value == value) { + return level; + } + } + throw new IllegalArgumentException("Unknown value for SanityLevel: " + value); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SizeApproximationFlag.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SizeApproximationFlag.java new file mode 100644 index 0000000..fe3c2dd --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SizeApproximationFlag.java @@ -0,0 +1,31 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb; + +import java.util.List; + +/** + * Flags for + * {@link RocksDB#getApproximateSizes(ColumnFamilyHandle, List, SizeApproximationFlag...)} + * that specify whether memtable stats should be included, + * or file stats approximation or both. + */ +public enum SizeApproximationFlag { + NONE((byte)0x0), + INCLUDE_MEMTABLES((byte)0x1), + INCLUDE_FILES((byte)0x2); + + private final byte value; + + SizeApproximationFlag(final byte value) { + this.value = value; + } + + /** + * Get the internal byte representation. + * + * @return the internal representation. + */ + byte getValue() { + return value; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SkipListMemTableConfig.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SkipListMemTableConfig.java new file mode 100644 index 0000000..e2c1b97 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SkipListMemTableConfig.java @@ -0,0 +1,51 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb; + +/** + * The config for skip-list memtable representation. + */ +public class SkipListMemTableConfig extends MemTableConfig { + + public static final long DEFAULT_LOOKAHEAD = 0; + + /** + * SkipListMemTableConfig constructor + */ + public SkipListMemTableConfig() { + lookahead_ = DEFAULT_LOOKAHEAD; + } + + /** + * Sets lookahead for SkipList + * + * @param lookahead If non-zero, each iterator's seek operation + * will start the search from the previously visited record + * (doing at most 'lookahead' steps). This is an + * optimization for the access pattern including many + * seeks with consecutive keys. + * @return the current instance of SkipListMemTableConfig + */ + public SkipListMemTableConfig setLookahead(final long lookahead) { + lookahead_ = lookahead; + return this; + } + + /** + * Returns the currently set lookahead value. + * + * @return lookahead value + */ + public long lookahead() { + return lookahead_; + } + + + @Override protected long newMemTableFactoryHandle() { + return newMemTableFactoryHandle0(lookahead_); + } + + private native long newMemTableFactoryHandle0(long lookahead) + throws IllegalArgumentException; + + private long lookahead_; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Slice.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Slice.java new file mode 100644 index 0000000..6a01374 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Slice.java @@ -0,0 +1,135 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + *

    Base class for slices which will receive + * byte[] based access to the underlying data.

    + * + *

    byte[] backed slices typically perform better with + * small keys and values. When using larger keys and + * values consider using {@link org.rocksdb.DirectSlice}

    + */ +public class Slice extends AbstractSlice { + + /** + * Indicates whether we have to free the memory pointed to by the Slice + */ + private volatile boolean cleared; + private volatile long internalBufferOffset = 0; + + /** + *

    Called from JNI to construct a new Java Slice + * without an underlying C++ object set + * at creation time.

    + * + *

    Note: You should be aware that + * {@see org.rocksdb.RocksObject#disOwnNativeHandle()} is intentionally + * called from the default Slice constructor, and that it is marked as + * private. This is so that developers cannot construct their own default + * Slice objects (at present). As developers cannot construct their own + * Slice objects through this, they are not creating underlying C++ Slice + * objects, and so there is nothing to free (dispose) from Java.

    + */ + @SuppressWarnings("unused") + private Slice() { + super(); + } + + /** + *

    Package-private Slice constructor which is used to construct + * Slice instances from C++ side. As the reference to this + * object is also managed from C++ side the handle will be disowned.

    + * + * @param nativeHandle address of native instance. + */ + Slice(final long nativeHandle) { + this(nativeHandle, false); + } + + /** + *

    Package-private Slice constructor which is used to construct + * Slice instances using a handle.

    + * + * @param nativeHandle address of native instance. + * @param owningNativeHandle true if the Java side owns the memory pointed to + * by this reference, false if ownership belongs to the C++ side + */ + Slice(final long nativeHandle, final boolean owningNativeHandle) { + super(); + setNativeHandle(nativeHandle, owningNativeHandle); + } + + /** + *

    Constructs a slice where the data is taken from + * a String.

    + * + * @param str String value. + */ + public Slice(final String str) { + super(createNewSliceFromString(str)); + } + + /** + *

    Constructs a slice where the data is a copy of + * the byte array from a specific offset.

    + * + * @param data byte array. + * @param offset offset within the byte array. + */ + public Slice(final byte[] data, final int offset) { + super(createNewSlice0(data, offset)); + } + + /** + *

    Constructs a slice where the data is a copy of + * the byte array.

    + * + * @param data byte array. + */ + public Slice(final byte[] data) { + super(createNewSlice1(data)); + } + + @Override + public void clear() { + clear0(getNativeHandle(), !cleared, internalBufferOffset); + cleared = true; + } + + @Override + public void removePrefix(final int n) { + removePrefix0(getNativeHandle(), n); + this.internalBufferOffset += n; + } + + /** + *

    Deletes underlying C++ slice pointer + * and any buffered data.

    + * + *

    + * Note that this function should be called only after all + * RocksDB instances referencing the slice are closed. + * Otherwise an undefined behavior will occur.

    + */ + @Override + protected void disposeInternal() { + final long nativeHandle = getNativeHandle(); + if(!cleared) { + disposeInternalBuf(nativeHandle, internalBufferOffset); + } + super.disposeInternal(nativeHandle); + } + + @Override protected final native byte[] data0(long handle); + private static native long createNewSlice0(final byte[] data, final int length); + private static native long createNewSlice1(final byte[] data); + private native void clear0(long handle, boolean internalBuffer, + long internalBufferOffset); + private native void removePrefix0(long handle, int length); + private native void disposeInternalBuf(final long handle, + long internalBufferOffset); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Snapshot.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Snapshot.java new file mode 100644 index 0000000..1f471bd --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Snapshot.java @@ -0,0 +1,41 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Snapshot of database + */ +public class Snapshot extends RocksObject { + Snapshot(final long nativeHandle) { + super(nativeHandle); + + // The pointer to the snapshot is always released + // by the database instance. + disOwnNativeHandle(); + } + + /** + * Return the associated sequence number; + * + * @return the associated sequence number of + * this snapshot. + */ + public long getSequenceNumber() { + return getSequenceNumber(nativeHandle_); + } + + @Override + protected final void disposeInternal(final long handle) { + /* + * Nothing to release, we never own the pointer for a + * Snapshot. The pointer + * to the snapshot is released by the database + * instance. + */ + } + + private native long getSequenceNumber(long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SstFileManager.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SstFileManager.java new file mode 100644 index 0000000..0b9a600 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SstFileManager.java @@ -0,0 +1,249 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Map; + +/** + * SstFileManager is used to track SST files in the DB and control their + * deletion rate. + *

    + * All SstFileManager public functions are thread-safe. + *

    + * SstFileManager is not extensible. + */ +//@ThreadSafe +public final class SstFileManager extends RocksObject { + + public static final long RATE_BYTES_PER_SEC_DEFAULT = 0; + public static final boolean DELETE_EXISTING_TRASH_DEFAULT = true; + public static final double MAX_TRASH_DB_RATION_DEFAULT = 0.25; + public static final long BYTES_MAX_DELETE_CHUNK_DEFAULT = 64 * 1024 * 1024; + + /** + * Create a new SstFileManager that can be shared among multiple RocksDB + * instances to track SST file and control there deletion rate. + * + * @param env the environment. + * + * @throws RocksDBException thrown if error happens in underlying native library. + */ + public SstFileManager(final Env env) throws RocksDBException { + this(env, null); + } + + /** + * Create a new SstFileManager that can be shared among multiple RocksDB + * instances to track SST file and control there deletion rate. + * + * @param env the environment. + * @param logger if not null, the logger will be used to log errors. + * + * @throws RocksDBException thrown if error happens in underlying native library. + */ + public SstFileManager(final Env env, /*@Nullable*/ final Logger logger) + throws RocksDBException { + this(env, logger, RATE_BYTES_PER_SEC_DEFAULT); + } + + /** + * Create a new SstFileManager that can be shared among multiple RocksDB + * instances to track SST file and control there deletion rate. + * + * @param env the environment. + * @param logger if not null, the logger will be used to log errors. + *

    + * == Deletion rate limiting specific arguments == + * @param rateBytesPerSec how many bytes should be deleted per second, If + * this value is set to 1024 (1 Kb / sec) and we deleted a file of size + * 4 Kb in 1 second, we will wait for another 3 seconds before we delete + * other files, Set to 0 to disable deletion rate limiting. + * + * @throws RocksDBException thrown if error happens in underlying native library. + */ + public SstFileManager(final Env env, /*@Nullable*/ final Logger logger, + final long rateBytesPerSec) throws RocksDBException { + this(env, logger, rateBytesPerSec, MAX_TRASH_DB_RATION_DEFAULT); + } + + /** + * Create a new SstFileManager that can be shared among multiple RocksDB + * instances to track SST file and control there deletion rate. + * + * @param env the environment. + * @param logger if not null, the logger will be used to log errors. + *

    + * == Deletion rate limiting specific arguments == + * @param rateBytesPerSec how many bytes should be deleted per second, If + * this value is set to 1024 (1 Kb / sec) and we deleted a file of size + * 4 Kb in 1 second, we will wait for another 3 seconds before we delete + * other files, Set to 0 to disable deletion rate limiting. + * @param maxTrashDbRatio if the trash size constitutes for more than this + * fraction of the total DB size we will start deleting new files passed + * to DeleteScheduler immediately. + * + * @throws RocksDBException thrown if error happens in underlying native library. + */ + public SstFileManager(final Env env, /*@Nullable*/ final Logger logger, + final long rateBytesPerSec, final double maxTrashDbRatio) + throws RocksDBException { + this(env, logger, rateBytesPerSec, maxTrashDbRatio, + BYTES_MAX_DELETE_CHUNK_DEFAULT); + } + + /** + * Create a new SstFileManager that can be shared among multiple RocksDB + * instances to track SST file and control there deletion rate. + * + * @param env the environment. + * @param logger if not null, the logger will be used to log errors. + *

    + * == Deletion rate limiting specific arguments == + * @param rateBytesPerSec how many bytes should be deleted per second, If + * this value is set to 1024 (1 Kb / sec) and we deleted a file of size + * 4 Kb in 1 second, we will wait for another 3 seconds before we delete + * other files, Set to 0 to disable deletion rate limiting. + * @param maxTrashDbRatio if the trash size constitutes for more than this + * fraction of the total DB size we will start deleting new files passed + * to DeleteScheduler immediately. + * @param bytesMaxDeleteChunk if a single file is larger than delete chunk, + * ftruncate the file by this size each time, rather than dropping the whole + * file. 0 means to always delete the whole file. + * + * @throws RocksDBException thrown if error happens in underlying native library. + */ + public SstFileManager(final Env env, /*@Nullable*/final Logger logger, + final long rateBytesPerSec, final double maxTrashDbRatio, + final long bytesMaxDeleteChunk) throws RocksDBException { + super(newSstFileManager(env.nativeHandle_, + logger != null ? logger.nativeHandle_ : 0, + rateBytesPerSec, maxTrashDbRatio, bytesMaxDeleteChunk)); + } + + /** + * Update the maximum allowed space that should be used by RocksDB, if + * the total size of the SST files exceeds {@code maxAllowedSpace}, writes to + * RocksDB will fail. + *

    + * Setting {@code maxAllowedSpace} to 0 will disable this feature; + * maximum allowed space will be infinite (Default value). + * + * @param maxAllowedSpace the maximum allowed space that should be used by + * RocksDB. + */ + public void setMaxAllowedSpaceUsage(final long maxAllowedSpace) { + setMaxAllowedSpaceUsage(nativeHandle_, maxAllowedSpace); + } + + /** + * Set the amount of buffer room each compaction should be able to leave. + * In other words, at its maximum disk space consumption, the compaction + * should still leave {@code compactionBufferSize} available on the disk so + * that other background functions may continue, such as logging and flushing. + * + * @param compactionBufferSize the amount of buffer room each compaction + * should be able to leave. + */ + public void setCompactionBufferSize(final long compactionBufferSize) { + setCompactionBufferSize(nativeHandle_, compactionBufferSize); + } + + /** + * Determines if the total size of SST files exceeded the maximum allowed + * space usage. + * + * @return true when the maximum allows space usage has been exceeded. + */ + public boolean isMaxAllowedSpaceReached() { + return isMaxAllowedSpaceReached(nativeHandle_); + } + + /** + * Determines if the total size of SST files as well as estimated size + * of ongoing compactions exceeds the maximums allowed space usage. + * + * @return true when the total size of SST files as well as estimated size + * of ongoing compactions exceeds the maximums allowed space usage. + */ + public boolean isMaxAllowedSpaceReachedIncludingCompactions() { + return isMaxAllowedSpaceReachedIncludingCompactions(nativeHandle_); + } + + /** + * Get the total size of all tracked files. + * + * @return the total size of all tracked files. + */ + public long getTotalSize() { + return getTotalSize(nativeHandle_); + } + + /** + * Gets all tracked files and their corresponding sizes. + * + * @return a map containing all tracked files and there corresponding sizes. + */ + public Map getTrackedFiles() { + return getTrackedFiles(nativeHandle_); + } + + /** + * Gets the delete rate limit. + * + * @return the delete rate limit (in bytes per second). + */ + public long getDeleteRateBytesPerSecond() { + return getDeleteRateBytesPerSecond(nativeHandle_); + } + + /** + * Set the delete rate limit. + *

    + * Zero means disable delete rate limiting and delete files immediately. + * + * @param deleteRate the delete rate limit (in bytes per second). + */ + public void setDeleteRateBytesPerSecond(final long deleteRate) { + setDeleteRateBytesPerSecond(nativeHandle_, deleteRate); + } + + /** + * Get the trash/DB size ratio where new files will be deleted immediately. + * + * @return the trash/DB size ratio. + */ + public double getMaxTrashDBRatio() { + return getMaxTrashDBRatio(nativeHandle_); + } + + /** + * Set the trash/DB size ratio where new files will be deleted immediately. + * + * @param ratio the trash/DB size ratio. + */ + public void setMaxTrashDBRatio(final double ratio) { + setMaxTrashDBRatio(nativeHandle_, ratio); + } + + private static native long newSstFileManager(final long handle, final long logger_handle, + final long rateBytesPerSec, final double maxTrashDbRatio, final long bytesMaxDeleteChunk) + throws RocksDBException; + private native void setMaxAllowedSpaceUsage(final long handle, + final long maxAllowedSpace); + private native void setCompactionBufferSize(final long handle, + final long compactionBufferSize); + private native boolean isMaxAllowedSpaceReached(final long handle); + private native boolean isMaxAllowedSpaceReachedIncludingCompactions( + final long handle); + private native long getTotalSize(final long handle); + private native Map getTrackedFiles(final long handle); + private native long getDeleteRateBytesPerSecond(final long handle); + private native void setDeleteRateBytesPerSecond(final long handle, + final long deleteRate); + private native double getMaxTrashDBRatio(final long handle); + private native void setMaxTrashDBRatio(final long handle, final double ratio); + @Override protected native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SstFileMetaData.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SstFileMetaData.java new file mode 100644 index 0000000..a04d05c --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SstFileMetaData.java @@ -0,0 +1,162 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * The metadata that describes a SST file. + */ +public class SstFileMetaData { + private final String fileName; + private final String path; + private final long size; + private final long smallestSeqno; + private final long largestSeqno; + private final byte[] smallestKey; + private final byte[] largestKey; + private final long numReadsSampled; + private final boolean beingCompacted; + private final long numEntries; + private final long numDeletions; + + /** + * Called from JNI C++ + * + * @param fileName the file name + * @param path the file path + * @param size the size of the file + * @param smallestSeqno the smallest sequence number + * @param largestSeqno the largest sequence number + * @param smallestKey the smallest key + * @param largestKey the largest key + * @param numReadsSampled the number of reads sampled + * @param beingCompacted true if the file is being compacted, false otherwise + * @param numEntries the number of entries + * @param numDeletions the number of deletions + */ + protected SstFileMetaData( + final String fileName, + final String path, + final long size, + final long smallestSeqno, + final long largestSeqno, + final byte[] smallestKey, + final byte[] largestKey, + final long numReadsSampled, + final boolean beingCompacted, + final long numEntries, + final long numDeletions) { + this.fileName = fileName; + this.path = path; + this.size = size; + this.smallestSeqno = smallestSeqno; + this.largestSeqno = largestSeqno; + this.smallestKey = smallestKey; + this.largestKey = largestKey; + this.numReadsSampled = numReadsSampled; + this.beingCompacted = beingCompacted; + this.numEntries = numEntries; + this.numDeletions = numDeletions; + } + + /** + * Get the name of the file. + * + * @return the name of the file. + */ + public String fileName() { + return fileName; + } + + /** + * Get the full path where the file locates. + * + * @return the full path + */ + public String path() { + return path; + } + + /** + * Get the file size in bytes. + * + * @return file size + */ + public long size() { + return size; + } + + /** + * Get the smallest sequence number in file. + * + * @return the smallest sequence number + */ + public long smallestSeqno() { + return smallestSeqno; + } + + /** + * Get the largest sequence number in file. + * + * @return the largest sequence number + */ + public long largestSeqno() { + return largestSeqno; + } + + /** + * Get the smallest user defined key in the file. + * + * @return the smallest user defined key + */ + public byte[] smallestKey() { + return smallestKey; + } + + /** + * Get the largest user defined key in the file. + * + * @return the largest user defined key + */ + public byte[] largestKey() { + return largestKey; + } + + /** + * Get the number of times the file has been read. + * + * @return the number of times the file has been read + */ + public long numReadsSampled() { + return numReadsSampled; + } + + /** + * Returns true if the file is currently being compacted. + * + * @return true if the file is currently being compacted, false otherwise. + */ + public boolean beingCompacted() { + return beingCompacted; + } + + /** + * Get the number of entries. + * + * @return the number of entries. + */ + public long numEntries() { + return numEntries; + } + + /** + * Get the number of deletions. + * + * @return the number of deletions. + */ + public long numDeletions() { + return numDeletions; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SstFileReader.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SstFileReader.java new file mode 100644 index 0000000..678c351 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SstFileReader.java @@ -0,0 +1,82 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +public class SstFileReader extends RocksObject { + static { + RocksDB.loadLibrary(); + } + + public SstFileReader(final Options options) { + super(newSstFileReader(options.nativeHandle_)); + } + + /** + * Returns an iterator that will iterate on all keys in the default + * column family including both keys in the DB and uncommitted keys in this + * transaction. + *

    + * Setting {@link ReadOptions#setSnapshot(Snapshot)} will affect what is read + * from the DB but will NOT change which keys are read from this transaction + * (the keys in this transaction do not yet belong to any snapshot and will be + * fetched regardless). + *

    + * Caller is responsible for deleting the returned Iterator. + * + * @param readOptions Read options. + * + * @return instance of iterator object. + */ + public SstFileReaderIterator newIterator(final ReadOptions readOptions) { + assert (isOwningHandle()); + final long iter = newIterator(nativeHandle_, readOptions.nativeHandle_); + return new SstFileReaderIterator(this, iter); + } + + /** + * Prepare SstFileReader to read a file. + * + * @param filePath the location of file + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void open(final String filePath) throws RocksDBException { + open(nativeHandle_, filePath); + } + + /** + * Verify checksum + * + * @throws RocksDBException if the checksum is not valid + */ + public void verifyChecksum() throws RocksDBException { + verifyChecksum(nativeHandle_); + } + + /** + * Get the properties of the table. + * + * @return the properties + * + * @throws RocksDBException if an error occurs whilst getting the table + * properties + */ + public TableProperties getTableProperties() throws RocksDBException { + return getTableProperties(nativeHandle_); + } + + @Override protected final native void disposeInternal(final long handle); + private native long newIterator(final long handle, final long readOptionsHandle); + + private native void open(final long handle, final String filePath) + throws RocksDBException; + + private static native long newSstFileReader(final long optionsHandle); + private native void verifyChecksum(final long handle) throws RocksDBException; + private native TableProperties getTableProperties(final long handle) + throws RocksDBException; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SstFileReaderIterator.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SstFileReaderIterator.java new file mode 100644 index 0000000..a4a0816 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SstFileReaderIterator.java @@ -0,0 +1,140 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.nio.ByteBuffer; + +/** + *

    An iterator that yields a sequence of key/value pairs from a source. + * Multiple implementations are provided by this library. + * In particular, iterators are provided + * to access the contents of a Table or a DB.

    + * + *

    Multiple threads can invoke const methods on an RocksIterator without + * external synchronization, but if any of the threads may call a + * non-const method, all threads accessing the same RocksIterator must use + * external synchronization.

    + * + * @see RocksObject + */ +public class SstFileReaderIterator extends AbstractRocksIterator { + protected SstFileReaderIterator(final SstFileReader reader, final long nativeHandle) { + super(reader, nativeHandle); + } + + /** + *

    Return the key for the current entry. The underlying storage for + * the returned slice is valid only until the next modification of + * the iterator.

    + * + *

    REQUIRES: {@link #isValid()}

    + * + * @return key for the current entry. + */ + public byte[] key() { + assert (isOwningHandle()); + return key0(nativeHandle_); + } + + /** + *

    Return the key for the current entry. The underlying storage for + * the returned slice is valid only until the next modification of + * the iterator.

    + * + *

    REQUIRES: {@link #isValid()}

    + * + * @param key the out-value to receive the retrieved key. + * It is using position and limit. Limit is set according to key size. + * Supports direct buffer only. + * @return The size of the actual key. If the return key is greater than the + * length of {@code key}, then it indicates that the size of the + * input buffer {@code key} is insufficient and partial result will + * be returned. + */ + public int key(final ByteBuffer key) { + assert (isOwningHandle()); + final int result; + if (key.isDirect()) { + result = keyDirect0(nativeHandle_, key, key.position(), key.remaining()); + } else { + result = keyByteArray0( + nativeHandle_, key.array(), key.arrayOffset() + key.position(), key.remaining()); + } + key.limit(Math.min(key.position() + result, key.limit())); + return result; + } + + /** + *

    Return the value for the current entry. The underlying storage for + * the returned slice is valid only until the next modification of + * the iterator.

    + * + *

    REQUIRES: !AtEnd() && !AtStart()

    + * @return value for the current entry. + */ + public byte[] value() { + assert (isOwningHandle()); + return value0(nativeHandle_); + } + + /** + *

    Return the value for the current entry. The underlying storage for + * the returned slice is valid only until the next modification of + * the iterator.

    + * + *

    REQUIRES: {@link #isValid()}

    + * + * @param value the out-value to receive the retrieved value. + * It is using position and limit. Limit is set according to value size. + * Supports direct buffer only. + * @return The size of the actual value. If the return value is greater than the + * length of {@code value}, then it indicates that the size of the + * input buffer {@code value} is insufficient and partial result will + * be returned. + */ + public int value(final ByteBuffer value) { + assert (isOwningHandle()); + final int result; + if (value.isDirect()) { + result = valueDirect0(nativeHandle_, value, value.position(), value.remaining()); + } else { + result = valueByteArray0( + nativeHandle_, value.array(), value.arrayOffset() + value.position(), value.remaining()); + } + value.limit(Math.min(value.position() + result, value.limit())); + return result; + } + + @Override protected final native void disposeInternal(final long handle); + @Override final native boolean isValid0(long handle); + @Override final native void seekToFirst0(long handle); + @Override final native void seekToLast0(long handle); + @Override final native void next0(long handle); + @Override final native void prev0(long handle); + @Override final native void refresh0(long handle) throws RocksDBException; + @Override final native void seek0(long handle, byte[] target, int targetLen); + @Override final native void seekForPrev0(long handle, byte[] target, int targetLen); + @Override final native void status0(long handle) throws RocksDBException; + @Override + final native void seekDirect0(long handle, ByteBuffer target, int targetOffset, int targetLen); + @Override + final native void seekForPrevDirect0( + long handle, ByteBuffer target, int targetOffset, int targetLen); + @Override + final native void seekByteArray0( + final long handle, final byte[] target, final int targetOffset, final int targetLen); + @Override + final native void seekForPrevByteArray0( + final long handle, final byte[] target, final int targetOffset, final int targetLen); + + private native byte[] key0(long handle); + private native byte[] value0(long handle); + + private native int keyDirect0(long handle, ByteBuffer buffer, int bufferOffset, int bufferLen); + private native int keyByteArray0(long handle, byte[] buffer, int bufferOffset, int bufferLen); + private native int valueDirect0(long handle, ByteBuffer buffer, int bufferOffset, int bufferLen); + private native int valueByteArray0(long handle, byte[] buffer, int bufferOffset, int bufferLen); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SstFileWriter.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SstFileWriter.java new file mode 100644 index 0000000..5dd0b6d --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SstFileWriter.java @@ -0,0 +1,237 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.nio.ByteBuffer; + +/** + * SstFileWriter is used to create sst files that can be added to the + * database later. All keys in files generated by SstFileWriter will have + * sequence number = 0. + */ +public class SstFileWriter extends RocksObject { + static { + RocksDB.loadLibrary(); + } + + /** + * SstFileWriter Constructor. + * + * @param envOptions {@link org.rocksdb.EnvOptions} instance. + * @param options {@link org.rocksdb.Options} instance. + */ + public SstFileWriter(final EnvOptions envOptions, final Options options) { + super(newSstFileWriter( + envOptions.nativeHandle_, options.nativeHandle_)); + } + + /** + * Prepare SstFileWriter to write to a file. + * + * @param filePath the location of file + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void open(final String filePath) throws RocksDBException { + open(nativeHandle_, filePath); + } + + /** + * Add a Put key with value to currently opened file. + * + * @param key the specified key to be inserted. + * @param value the value associated with the specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void put(final Slice key, final Slice value) throws RocksDBException { + put(nativeHandle_, key.getNativeHandle(), value.getNativeHandle()); + } + + /** + * Add a Put key with value to currently opened file. + * + * @param key the specified key to be inserted. + * @param value the value associated with the specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void put(final DirectSlice key, final DirectSlice value) + throws RocksDBException { + put(nativeHandle_, key.getNativeHandle(), value.getNativeHandle()); + } + + /** + * Add a Put key with value to currently opened file. + * + * @param key the specified key to be inserted. + * @param value the value associated with the specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void put(final ByteBuffer key, final ByteBuffer value) throws RocksDBException { + assert key.isDirect() && value.isDirect(); + putDirect(nativeHandle_, key, key.position(), key.remaining(), value, value.position(), + value.remaining()); + key.position(key.limit()); + value.position(value.limit()); + } + + /** + * Add a Put key with value to currently opened file. + * + * @param key the specified key to be inserted. + * @param value the value associated with the specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void put(final byte[] key, final byte[] value) throws RocksDBException { + put(nativeHandle_, key, value); + } + + /** + * Add a Merge key with value to currently opened file. + * + * @param key the specified key to be merged. + * @param value the value to be merged with the current value for + * the specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void merge(final Slice key, final Slice value) + throws RocksDBException { + merge(nativeHandle_, key.getNativeHandle(), value.getNativeHandle()); + } + + /** + * Add a Merge key with value to currently opened file. + * + * @param key the specified key to be merged. + * @param value the value to be merged with the current value for + * the specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void merge(final byte[] key, final byte[] value) + throws RocksDBException { + merge(nativeHandle_, key, value); + } + + /** + * Add a Merge key with value to currently opened file. + * + * @param key the specified key to be merged. + * @param value the value to be merged with the current value for + * the specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void merge(final DirectSlice key, final DirectSlice value) + throws RocksDBException { + merge(nativeHandle_, key.getNativeHandle(), value.getNativeHandle()); + } + + /** + * Add a deletion key to currently opened file. + * + * @param key the specified key to be deleted. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void delete(final Slice key) throws RocksDBException { + delete(nativeHandle_, key.getNativeHandle()); + } + + /** + * Add a deletion key to currently opened file. + * + * @param key the specified key to be deleted. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void delete(final DirectSlice key) throws RocksDBException { + delete(nativeHandle_, key.getNativeHandle()); + } + + /** + * Add a deletion key to currently opened file. + * + * @param key the specified key to be deleted. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void delete(final byte[] key) throws RocksDBException { + delete(nativeHandle_, key); + } + + /** + * Finish the process and close the sst file. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public void finish() throws RocksDBException { + finish(nativeHandle_); + } + + /** + * Return the current file size. + * + * @return the current file size. + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public long fileSize() throws RocksDBException { + return fileSize(nativeHandle_); + } + + private static native long newSstFileWriter(final long envOptionsHandle, final long optionsHandle, + final long userComparatorHandle, final byte comparatorType); + + private static native long newSstFileWriter( + final long envOptionsHandle, final long optionsHandle); + + private native void open(final long handle, final String filePath) + throws RocksDBException; + + private native void put(final long handle, final long keyHandle, + final long valueHandle) throws RocksDBException; + + private native void put(final long handle, final byte[] key, + final byte[] value) throws RocksDBException; + + private native void putDirect(long handle, ByteBuffer key, int keyOffset, int keyLength, + ByteBuffer value, int valueOffset, int valueLength) throws RocksDBException; + + private native long fileSize(long handle) throws RocksDBException; + + private native void merge(final long handle, final long keyHandle, + final long valueHandle) throws RocksDBException; + + private native void merge(final long handle, final byte[] key, + final byte[] value) throws RocksDBException; + + private native void delete(final long handle, final long keyHandle) + throws RocksDBException; + + private native void delete(final long handle, final byte[] key) + throws RocksDBException; + + private native void finish(final long handle) throws RocksDBException; + + @Override protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SstPartitionerFactory.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SstPartitionerFactory.java new file mode 100644 index 0000000..ea6f135 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SstPartitionerFactory.java @@ -0,0 +1,15 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Handle to factory for SstPartitioner. It is used in {@link ColumnFamilyOptions} + */ +public abstract class SstPartitionerFactory extends RocksObject { + protected SstPartitionerFactory(final long nativeHandle) { + super(nativeHandle); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SstPartitionerFixedPrefixFactory.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SstPartitionerFixedPrefixFactory.java new file mode 100644 index 0000000..b1ccf08 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/SstPartitionerFixedPrefixFactory.java @@ -0,0 +1,19 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Fixed prefix factory. It partitions SST files using fixed prefix of the key. + */ +public class SstPartitionerFixedPrefixFactory extends SstPartitionerFactory { + public SstPartitionerFixedPrefixFactory(final long prefixLength) { + super(newSstPartitionerFixedPrefixFactory0(prefixLength)); + } + + private static native long newSstPartitionerFixedPrefixFactory0(long prefixLength); + + @Override protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/StateType.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/StateType.java new file mode 100644 index 0000000..803fa37 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/StateType.java @@ -0,0 +1,53 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * The type used to refer to a thread state. + *

    + * A state describes lower-level action of a thread + * such as reading / writing a file or waiting for a mutex. + */ +public enum StateType { + STATE_UNKNOWN((byte)0x0), + STATE_MUTEX_WAIT((byte)0x1); + + private final byte value; + + StateType(final byte value) { + this.value = value; + } + + /** + * Get the internal representation value. + * + * @return the internal representation value. + */ + byte getValue() { + return value; + } + + /** + * Get the State type from the internal representation value. + * + * @param value the internal representation value. + * + * @return the state type + * + * @throws IllegalArgumentException if the value does not match + * a StateType + */ + static StateType fromValue(final byte value) + throws IllegalArgumentException { + for (final StateType threadType : StateType.values()) { + if (threadType.value == value) { + return threadType; + } + } + throw new IllegalArgumentException( + "Unknown value for StateType: " + value); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Statistics.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Statistics.java new file mode 100644 index 0000000..9f3c9a6 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Statistics.java @@ -0,0 +1,153 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.EnumSet; + +/** + * Statistics to analyze the performance of a db. Pointer for statistics object + * is managed by Options class. + */ +public class Statistics extends RocksObject { + + public Statistics() { + super(newStatistics()); + } + + public Statistics(final Statistics otherStatistics) { + super(newStatistics(otherStatistics.nativeHandle_)); + } + + public Statistics(final EnumSet ignoreHistograms) { + super(newStatistics(toArrayValues(ignoreHistograms))); + } + + public Statistics(final EnumSet ignoreHistograms, final Statistics otherStatistics) { + super(newStatistics(toArrayValues(ignoreHistograms), otherStatistics.nativeHandle_)); + } + + /** + * Intentionally package-private. + *

    + * Used from {@link DBOptions#statistics()} + * + * @param existingStatisticsHandle The C++ pointer to an existing statistics object + */ + Statistics(final long existingStatisticsHandle) { + super(existingStatisticsHandle); + } + + private static byte[] toArrayValues(final EnumSet histogramTypes) { + final byte[] values = new byte[histogramTypes.size()]; + int i = 0; + for(final HistogramType histogramType : histogramTypes) { + values[i++] = histogramType.getValue(); + } + return values; + } + + /** + * Gets the current stats level. + * + * @return The stats level. + */ + public StatsLevel statsLevel() { + return StatsLevel.getStatsLevel(statsLevel(nativeHandle_)); + } + + /** + * Sets the stats level. + * + * @param statsLevel The stats level to set. + */ + public void setStatsLevel(final StatsLevel statsLevel) { + setStatsLevel(nativeHandle_, statsLevel.getValue()); + } + + /** + * Get the count for a ticker. + * + * @param tickerType The ticker to get the count for + * + * @return The count for the ticker + */ + public long getTickerCount(final TickerType tickerType) { + assert(isOwningHandle()); + return getTickerCount(nativeHandle_, tickerType.getValue()); + } + + /** + * Get the count for a ticker and reset the tickers count. + * + * @param tickerType The ticker to get the count for + * + * @return The count for the ticker + */ + public long getAndResetTickerCount(final TickerType tickerType) { + assert(isOwningHandle()); + return getAndResetTickerCount(nativeHandle_, tickerType.getValue()); + } + + /** + * Gets the histogram data for a particular histogram. + * + * @param histogramType The histogram to retrieve the data for + * + * @return The histogram data + */ + public HistogramData getHistogramData(final HistogramType histogramType) { + assert(isOwningHandle()); + return getHistogramData(nativeHandle_, histogramType.getValue()); + } + + /** + * Gets a string representation of a particular histogram. + * + * @param histogramType The histogram to retrieve the data for + * + * @return A string representation of the histogram data + */ + public String getHistogramString(final HistogramType histogramType) { + assert(isOwningHandle()); + return getHistogramString(nativeHandle_, histogramType.getValue()); + } + + /** + * Resets all ticker and histogram stats. + * + * @throws RocksDBException if an error occurs when resetting the statistics. + */ + public void reset() throws RocksDBException { + assert(isOwningHandle()); + reset(nativeHandle_); + } + + /** + * String representation of the statistic object. + */ + @Override + public String toString() { + assert(isOwningHandle()); + return toString(nativeHandle_); + } + + private static native long newStatistics(); + private static native long newStatistics(final long otherStatisticsHandle); + private static native long newStatistics(final byte[] ignoreHistograms); + private static native long newStatistics( + final byte[] ignoreHistograms, final long otherStatisticsHandle); + + @Override protected final native void disposeInternal(final long handle); + + private native byte statsLevel(final long handle); + private native void setStatsLevel(final long handle, final byte statsLevel); + private native long getTickerCount(final long handle, final byte tickerType); + private native long getAndResetTickerCount(final long handle, final byte tickerType); + private native HistogramData getHistogramData(final long handle, final byte histogramType); + private native String getHistogramString(final long handle, final byte histogramType); + private native void reset(final long nativeHandle) throws RocksDBException; + private native String toString(final long nativeHandle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/StatisticsCollector.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/StatisticsCollector.java new file mode 100644 index 0000000..fd00f85 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/StatisticsCollector.java @@ -0,0 +1,102 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + +/** + *

    Helper class to collect DB statistics periodically at a period specified in + * constructor. Callback function (provided in constructor) is called with + * every statistics collection.

    + * + *

    Caller should call start() to start statistics collection. Shutdown() should + * be called to stop stats collection and should be called before statistics ( + * provided in constructor) reference has been disposed.

    + */ +public class StatisticsCollector { + private final List _statsCollectorInputList; + private final ExecutorService _executorService; + private final int _statsCollectionInterval; + private volatile boolean _isRunning = true; + + /** + * Constructor for statistics collector. + * + * @param statsCollectorInputList List of statistics collector input. + * @param statsCollectionIntervalInMilliSeconds Statistics collection time + * period (specified in milliseconds). + */ + public StatisticsCollector( + final List statsCollectorInputList, + final int statsCollectionIntervalInMilliSeconds) { + _statsCollectorInputList = statsCollectorInputList; + _statsCollectionInterval = statsCollectionIntervalInMilliSeconds; + + _executorService = Executors.newSingleThreadExecutor(); + } + + public void start() { + _executorService.submit(collectStatistics()); + } + + /** + * Shuts down statistics collector. + * + * @param shutdownTimeout Time in milli-seconds to wait for shutdown before + * killing the collection process. + * @throws java.lang.InterruptedException thrown if Threads are interrupted. + */ + public void shutDown(final int shutdownTimeout) throws InterruptedException { + _isRunning = false; + + _executorService.shutdownNow(); + // Wait for collectStatistics runnable to finish so that disposal of + // statistics does not cause any exceptions to be thrown. + _executorService.awaitTermination(shutdownTimeout, TimeUnit.MILLISECONDS); + } + + private Runnable collectStatistics() { + return () -> { + while (_isRunning) { + try { + if (Thread.currentThread().isInterrupted()) { + break; + } + for (final StatsCollectorInput statsCollectorInput : _statsCollectorInputList) { + final Statistics statistics = statsCollectorInput.getStatistics(); + final StatisticsCollectorCallback statsCallback = statsCollectorInput.getCallback(); + + // Collect ticker data + for (final TickerType ticker : TickerType.values()) { + if (ticker != TickerType.TICKER_ENUM_MAX) { + final long tickerValue = statistics.getTickerCount(ticker); + statsCallback.tickerCallback(ticker, tickerValue); + } + } + + // Collect histogram data + for (final HistogramType histogramType : HistogramType.values()) { + if (histogramType != HistogramType.HISTOGRAM_ENUM_MAX) { + final HistogramData histogramData = statistics.getHistogramData(histogramType); + statsCallback.histogramCallback(histogramType, histogramData); + } + } + } + + Thread.sleep(_statsCollectionInterval); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } catch (final Exception e) { + throw new RuntimeException("Error while calculating statistics", e); + } + } + }; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/StatisticsCollectorCallback.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/StatisticsCollectorCallback.java new file mode 100644 index 0000000..bed7828 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/StatisticsCollectorCallback.java @@ -0,0 +1,32 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Callback interface provided to StatisticsCollector. + *

    + * Thread safety: + * StatisticsCollector doesn't make any guarantees about thread safety. + * If the same reference of StatisticsCollectorCallback is passed to multiple + * StatisticsCollector references, then its the responsibility of the + * user to make StatisticsCollectorCallback's implementation thread-safe. + * + */ +public interface StatisticsCollectorCallback { + /** + * Callback function to get ticker values. + * @param tickerType Ticker type. + * @param tickerCount Value of ticker type. + */ + void tickerCallback(TickerType tickerType, long tickerCount); + + /** + * Callback function to get histogram values. + * @param histType Histogram type. + * @param histData Histogram data. + */ + void histogramCallback(HistogramType histType, HistogramData histData); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/StatsCollectorInput.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/StatsCollectorInput.java new file mode 100644 index 0000000..5bf43ad --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/StatsCollectorInput.java @@ -0,0 +1,35 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Contains all information necessary to collect statistics from one instance + * of DB statistics. + */ +public class StatsCollectorInput { + private final Statistics _statistics; + private final StatisticsCollectorCallback _statsCallback; + + /** + * Constructor for StatsCollectorInput. + * + * @param statistics Reference of DB statistics. + * @param statsCallback Reference of statistics callback interface. + */ + public StatsCollectorInput(final Statistics statistics, + final StatisticsCollectorCallback statsCallback) { + _statistics = statistics; + _statsCallback = statsCallback; + } + + public Statistics getStatistics() { + return _statistics; + } + + public StatisticsCollectorCallback getCallback() { + return _statsCallback; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/StatsLevel.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/StatsLevel.java new file mode 100644 index 0000000..8190e50 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/StatsLevel.java @@ -0,0 +1,65 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * The level of Statistics to report. + */ +public enum StatsLevel { + /** + * Collect all stats except time inside mutex lock AND time spent on + * compression. + */ + EXCEPT_DETAILED_TIMERS((byte) 0x0), + + /** + * Collect all stats except the counters requiring to get time inside the + * mutex lock. + */ + EXCEPT_TIME_FOR_MUTEX((byte) 0x1), + + /** + * Collect all stats, including measuring duration of mutex operations. + *

    + * If getting time is expensive on the platform to run, it can + * reduce scalability to more threads, especially for writes. + */ + ALL((byte) 0x2); + + private final byte value; + + StatsLevel(final byte value) { + this.value = value; + } + + /** + *

    Returns the byte value of the enumerations value.

    + * + * @return byte representation + */ + public byte getValue() { + return value; + } + + /** + * Get StatsLevel by byte value. + * + * @param value byte representation of StatsLevel. + * + * @return {@link org.rocksdb.StatsLevel} instance. + * @throws java.lang.IllegalArgumentException if an invalid + * value is provided. + */ + public static StatsLevel getStatsLevel(final byte value) { + for (final StatsLevel statsLevel : StatsLevel.values()) { + if (statsLevel.getValue() == value){ + return statsLevel; + } + } + throw new IllegalArgumentException( + "Illegal value provided for StatsLevel."); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Status.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Status.java new file mode 100644 index 0000000..5c50e70 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Status.java @@ -0,0 +1,155 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Objects; + +/** + * Represents the status returned by a function call in RocksDB. + *

    + * Currently only used with {@link RocksDBException} when the + * status is not {@link Code#Ok} + */ +public class Status { + private final Code code; + /* @Nullable */ private final SubCode subCode; + /* @Nullable */ private final String state; + + public Status(final Code code, final SubCode subCode, final String state) { + this.code = code; + this.subCode = subCode; + this.state = state; + } + + /** + * Intentionally private as this will be called from JNI + */ + private Status(final byte code, final byte subCode, final String state) { + this.code = Code.getCode(code); + this.subCode = SubCode.getSubCode(subCode); + this.state = state; + } + + public Code getCode() { + return code; + } + + public SubCode getSubCode() { + return subCode; + } + + public String getState() { + return state; + } + + public String getCodeString() { + final StringBuilder builder = new StringBuilder() + .append(code.name()); + if(subCode != null && subCode != SubCode.None) { + builder.append("(") + .append(subCode.name()) + .append(")"); + } + return builder.toString(); + } + + // should stay in sync with /include/rocksdb/status.h:Code and /java/rocksjni/portal.h:toJavaStatusCode + public enum Code { + Ok( (byte)0x0), + NotFound( (byte)0x1), + Corruption( (byte)0x2), + NotSupported( (byte)0x3), + InvalidArgument( (byte)0x4), + IOError( (byte)0x5), + MergeInProgress( (byte)0x6), + Incomplete( (byte)0x7), + ShutdownInProgress( (byte)0x8), + TimedOut( (byte)0x9), + Aborted( (byte)0xA), + Busy( (byte)0xB), + Expired( (byte)0xC), + TryAgain( (byte)0xD), + Undefined( (byte)0x7F); + + private final byte value; + + Code(final byte value) { + this.value = value; + } + + public static Code getCode(final byte value) { + for (final Code code : Code.values()) { + if (code.value == value){ + return code; + } + } + throw new IllegalArgumentException( + "Illegal value provided for Code (" + value + ")."); + } + + /** + * Returns the byte value of the enumerations value. + * + * @return byte representation + */ + public byte getValue() { + return value; + } + } + + // should stay in sync with /include/rocksdb/status.h:SubCode and /java/rocksjni/portal.h:toJavaStatusSubCode + public enum SubCode { + None( (byte)0x0), + MutexTimeout( (byte)0x1), + LockTimeout( (byte)0x2), + LockLimit( (byte)0x3), + NoSpace( (byte)0x4), + Deadlock( (byte)0x5), + StaleFile( (byte)0x6), + MemoryLimit( (byte)0x7), + Undefined( (byte)0x7F); + + private final byte value; + + SubCode(final byte value) { + this.value = value; + } + + public static SubCode getSubCode(final byte value) { + for (final SubCode subCode : SubCode.values()) { + if (subCode.value == value){ + return subCode; + } + } + throw new IllegalArgumentException( + "Illegal value provided for SubCode (" + value + ")."); + } + + /** + * Returns the byte value of the enumerations value. + * + * @return byte representation + */ + public byte getValue() { + return value; + } + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final Status status = (Status) o; + return code == status.code && subCode == status.subCode && Objects.equals(state, status.state); + } + + @Override + public int hashCode() { + return Objects.hash(code, subCode, state); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/StringAppendOperator.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/StringAppendOperator.java new file mode 100644 index 0000000..547371e --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/StringAppendOperator.java @@ -0,0 +1,29 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// Copyright (c) 2014, Vlad Balan (vlad.gm@gmail.com). All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * StringAppendOperator is a merge operator that concatenates + * two strings. + */ +public class StringAppendOperator extends MergeOperator { + public StringAppendOperator() { + this(','); + } + + public StringAppendOperator(final char delim) { + super(newSharedStringAppendOperator(delim)); + } + + public StringAppendOperator(final String delim) { + super(newSharedStringAppendOperator(delim)); + } + + private static native long newSharedStringAppendOperator(final char delim); + private static native long newSharedStringAppendOperator(final String delim); + @Override protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TableFileCreationBriefInfo.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TableFileCreationBriefInfo.java new file mode 100644 index 0000000..8dc5679 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TableFileCreationBriefInfo.java @@ -0,0 +1,107 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Objects; + +public class TableFileCreationBriefInfo { + private final String dbName; + private final String columnFamilyName; + private final String filePath; + private final int jobId; + private final TableFileCreationReason reason; + + /** + * Access is private as this will only be constructed from + * C++ via JNI, either directly of via + * {@link TableFileCreationInfo#TableFileCreationInfo(long, TableProperties, Status, String, + * String, String, int, byte)}. + * + * @param dbName the database name + * @param columnFamilyName the column family name + * @param filePath the path to the table file + * @param jobId the job identifier + * @param tableFileCreationReasonValue the reason for creation of the table file + */ + protected TableFileCreationBriefInfo(final String dbName, final String columnFamilyName, + final String filePath, final int jobId, final byte tableFileCreationReasonValue) { + this.dbName = dbName; + this.columnFamilyName = columnFamilyName; + this.filePath = filePath; + this.jobId = jobId; + this.reason = TableFileCreationReason.fromValue(tableFileCreationReasonValue); + } + + /** + * Get the name of the database where the file was created. + * + * @return the name of the database. + */ + public String getDbName() { + return dbName; + } + + /** + * Get the name of the column family where the file was created. + * + * @return the name of the column family. + */ + public String getColumnFamilyName() { + return columnFamilyName; + } + + /** + * Get the path to the created file. + * + * @return the path. + */ + public String getFilePath() { + return filePath; + } + + /** + * Get the id of the job (which could be flush or compaction) that + * created the file. + * + * @return the id of the job. + */ + public int getJobId() { + return jobId; + } + + /** + * Get the reason for creating the table. + * + * @return the reason for creating the table. + */ + public TableFileCreationReason getReason() { + return reason; + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final TableFileCreationBriefInfo that = (TableFileCreationBriefInfo) o; + return jobId == that.jobId && Objects.equals(dbName, that.dbName) + && Objects.equals(columnFamilyName, that.columnFamilyName) + && Objects.equals(filePath, that.filePath) && reason == that.reason; + } + + @Override + public int hashCode() { + return Objects.hash(dbName, columnFamilyName, filePath, jobId, reason); + } + + @Override + public String toString() { + return "TableFileCreationBriefInfo{" + + "dbName='" + dbName + '\'' + ", columnFamilyName='" + columnFamilyName + '\'' + + ", filePath='" + filePath + '\'' + ", jobId=" + jobId + ", reason=" + reason + '}'; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TableFileCreationInfo.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TableFileCreationInfo.java new file mode 100644 index 0000000..5654603 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TableFileCreationInfo.java @@ -0,0 +1,86 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Objects; + +public class TableFileCreationInfo extends TableFileCreationBriefInfo { + private final long fileSize; + private final TableProperties tableProperties; + private final Status status; + + /** + * Access is protected as this will only be constructed from + * C++ via JNI. + * + * @param fileSize the size of the table file + * @param tableProperties the properties of the table file + * @param status the status of the creation operation + * @param dbName the database name + * @param columnFamilyName the column family name + * @param filePath the path to the table file + * @param jobId the job identifier + * @param tableFileCreationReasonValue the reason for creation of the table file + */ + protected TableFileCreationInfo(final long fileSize, final TableProperties tableProperties, + final Status status, final String dbName, final String columnFamilyName, + final String filePath, final int jobId, final byte tableFileCreationReasonValue) { + super(dbName, columnFamilyName, filePath, jobId, tableFileCreationReasonValue); + this.fileSize = fileSize; + this.tableProperties = tableProperties; + this.status = status; + } + + /** + * Get the size of the file. + * + * @return the size. + */ + public long getFileSize() { + return fileSize; + } + + /** + * Get the detailed properties of the created file. + * + * @return the properties. + */ + public TableProperties getTableProperties() { + return tableProperties; + } + + /** + * Get the status indicating whether the creation was successful or not. + * + * @return the status. + */ + public Status getStatus() { + return status; + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final TableFileCreationInfo that = (TableFileCreationInfo) o; + return fileSize == that.fileSize && Objects.equals(tableProperties, that.tableProperties) + && Objects.equals(status, that.status); + } + + @Override + public int hashCode() { + return Objects.hash(fileSize, tableProperties, status); + } + + @Override + public String toString() { + return "TableFileCreationInfo{" + + "fileSize=" + fileSize + ", tableProperties=" + tableProperties + ", status=" + status + + '}'; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TableFileCreationReason.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TableFileCreationReason.java new file mode 100644 index 0000000..d398466 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TableFileCreationReason.java @@ -0,0 +1,46 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +public enum TableFileCreationReason { + FLUSH((byte) 0x00), + COMPACTION((byte) 0x01), + RECOVERY((byte) 0x02), + MISC((byte) 0x03); + + private final byte value; + + TableFileCreationReason(final byte value) { + this.value = value; + } + + /** + * Get the internal representation. + * + * @return the internal representation + */ + byte getValue() { + return value; + } + + /** + * Get the TableFileCreationReason from the internal representation value. + * + * @return the table file creation reason. + * + * @throws IllegalArgumentException if the value is unknown. + */ + static TableFileCreationReason fromValue(final byte value) { + for (final TableFileCreationReason tableFileCreationReason : TableFileCreationReason.values()) { + if (tableFileCreationReason.value == value) { + return tableFileCreationReason; + } + } + + throw new IllegalArgumentException( + "Illegal value provided for TableFileCreationReason: " + value); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TableFileDeletionInfo.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TableFileDeletionInfo.java new file mode 100644 index 0000000..9a777e3 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TableFileDeletionInfo.java @@ -0,0 +1,86 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Objects; + +public class TableFileDeletionInfo { + private final String dbName; + private final String filePath; + private final int jobId; + private final Status status; + + /** + * Access is package private as this will only be constructed from + * C++ via JNI and for testing. + */ + TableFileDeletionInfo( + final String dbName, final String filePath, final int jobId, final Status status) { + this.dbName = dbName; + this.filePath = filePath; + this.jobId = jobId; + this.status = status; + } + + /** + * Get the name of the database where the file was deleted. + * + * @return the name of the database. + */ + public String getDbName() { + return dbName; + } + + /** + * Get the path to the deleted file. + * + * @return the path. + */ + public String getFilePath() { + return filePath; + } + + /** + * Get the id of the job which deleted the file. + * + * @return the id of the job. + */ + public int getJobId() { + return jobId; + } + + /** + * Get the status indicating whether the deletion was successful or not. + * + * @return the status + */ + public Status getStatus() { + return status; + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final TableFileDeletionInfo that = (TableFileDeletionInfo) o; + return jobId == that.jobId && Objects.equals(dbName, that.dbName) + && Objects.equals(filePath, that.filePath) && Objects.equals(status, that.status); + } + + @Override + public int hashCode() { + return Objects.hash(dbName, filePath, jobId, status); + } + + @Override + public String toString() { + return "TableFileDeletionInfo{" + + "dbName='" + dbName + '\'' + ", filePath='" + filePath + '\'' + ", jobId=" + jobId + + ", status=" + status + '}'; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TableFilter.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TableFilter.java new file mode 100644 index 0000000..a39a329 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TableFilter.java @@ -0,0 +1,21 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb; + +/** + * Filter for iterating a table. + */ +public interface TableFilter { + + /** + * A callback to determine whether relevant keys for this scan exist in a + * given table based on the table's properties. The callback is passed the + * properties of each table during iteration. If the callback returns false, + * the table will not be scanned. This option only affects Iterators and has + * no impact on point lookups. + * + * @param tableProperties the table properties. + * + * @return true if the table should be scanned, false otherwise. + */ + boolean filter(final TableProperties tableProperties); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TableFormatConfig.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TableFormatConfig.java new file mode 100644 index 0000000..726c6f1 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TableFormatConfig.java @@ -0,0 +1,22 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +/** + * TableFormatConfig is used to config the internal Table format of a RocksDB. + * To make a RocksDB to use a specific Table format, its associated + * TableFormatConfig should be properly set and passed into Options via + * Options.setTableFormatConfig() and open the db using that Options. + */ +public abstract class TableFormatConfig { + /** + *

    This function should only be called by Options.setTableFormatConfig(), + * which will create a c++ shared-pointer to the c++ TableFactory + * that associated with the Java TableFormatConfig.

    + * + * @return native handle address to native table instance. + */ + protected abstract long newTableFactoryHandle(); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TableProperties.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TableProperties.java new file mode 100644 index 0000000..02b9560 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TableProperties.java @@ -0,0 +1,426 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb; + +import java.util.Arrays; +import java.util.Map; +import java.util.Objects; + +/** + * TableProperties contains read-only properties of its associated + * table. + */ +public class TableProperties { + private final long dataSize; + private final long indexSize; + private final long indexPartitions; + private final long topLevelIndexSize; + private final long indexKeyIsUserKey; + private final long indexValueIsDeltaEncoded; + private final long filterSize; + private final long rawKeySize; + private final long rawValueSize; + private final long numDataBlocks; + private final long numEntries; + private final long numDeletions; + private final long numMergeOperands; + private final long numRangeDeletions; + private final long formatVersion; + private final long fixedKeyLen; + private final long columnFamilyId; + private final long creationTime; + private final long oldestKeyTime; + private final long slowCompressionEstimatedDataSize; + private final long fastCompressionEstimatedDataSize; + private final long externalSstFileGlobalSeqnoOffset; + private final byte[] columnFamilyName; + private final String filterPolicyName; + private final String comparatorName; + private final String mergeOperatorName; + private final String prefixExtractorName; + private final String propertyCollectorsNames; + private final String compressionName; + private final Map userCollectedProperties; + private final Map readableProperties; + + /** + * Access is package private as this will only be constructed from + * C++ via JNI and for testing. + */ + TableProperties(final long dataSize, final long indexSize, final long indexPartitions, + final long topLevelIndexSize, final long indexKeyIsUserKey, + final long indexValueIsDeltaEncoded, final long filterSize, final long rawKeySize, + final long rawValueSize, final long numDataBlocks, final long numEntries, + final long numDeletions, final long numMergeOperands, final long numRangeDeletions, + final long formatVersion, final long fixedKeyLen, final long columnFamilyId, + final long creationTime, final long oldestKeyTime, + final long slowCompressionEstimatedDataSize, final long fastCompressionEstimatedDataSize, + final long externalSstFileGlobalSeqnoOffset, final byte[] columnFamilyName, + final String filterPolicyName, final String comparatorName, final String mergeOperatorName, + final String prefixExtractorName, final String propertyCollectorsNames, + final String compressionName, final Map userCollectedProperties, + final Map readableProperties) { + this.dataSize = dataSize; + this.indexSize = indexSize; + this.indexPartitions = indexPartitions; + this.topLevelIndexSize = topLevelIndexSize; + this.indexKeyIsUserKey = indexKeyIsUserKey; + this.indexValueIsDeltaEncoded = indexValueIsDeltaEncoded; + this.filterSize = filterSize; + this.rawKeySize = rawKeySize; + this.rawValueSize = rawValueSize; + this.numDataBlocks = numDataBlocks; + this.numEntries = numEntries; + this.numDeletions = numDeletions; + this.numMergeOperands = numMergeOperands; + this.numRangeDeletions = numRangeDeletions; + this.formatVersion = formatVersion; + this.fixedKeyLen = fixedKeyLen; + this.columnFamilyId = columnFamilyId; + this.creationTime = creationTime; + this.oldestKeyTime = oldestKeyTime; + this.slowCompressionEstimatedDataSize = slowCompressionEstimatedDataSize; + this.fastCompressionEstimatedDataSize = fastCompressionEstimatedDataSize; + this.externalSstFileGlobalSeqnoOffset = externalSstFileGlobalSeqnoOffset; + this.columnFamilyName = columnFamilyName; + this.filterPolicyName = filterPolicyName; + this.comparatorName = comparatorName; + this.mergeOperatorName = mergeOperatorName; + this.prefixExtractorName = prefixExtractorName; + this.propertyCollectorsNames = propertyCollectorsNames; + this.compressionName = compressionName; + this.userCollectedProperties = userCollectedProperties; + this.readableProperties = readableProperties; + } + + /** + * Get the total size of all data blocks. + * + * @return the total size of all data blocks. + */ + public long getDataSize() { + return dataSize; + } + + /** + * Get the size of index block. + * + * @return the size of index block. + */ + public long getIndexSize() { + return indexSize; + } + + /** + * Get the total number of index partitions + * if {@link IndexType#kTwoLevelIndexSearch} is used. + * + * @return the total number of index partitions. + */ + public long getIndexPartitions() { + return indexPartitions; + } + + /** + * Size of the top-level index + * if {@link IndexType#kTwoLevelIndexSearch} is used. + * + * @return the size of the top-level index. + */ + public long getTopLevelIndexSize() { + return topLevelIndexSize; + } + + /** + * Whether the index key is user key. + * Otherwise it includes 8 byte of sequence + * number added by internal key format. + * + * @return the index key + */ + public long getIndexKeyIsUserKey() { + return indexKeyIsUserKey; + } + + /** + * Whether delta encoding is used to encode the index values. + * + * @return whether delta encoding is used to encode the index values. + */ + public long getIndexValueIsDeltaEncoded() { + return indexValueIsDeltaEncoded; + } + + /** + * Get the size of filter block. + * + * @return the size of filter block. + */ + public long getFilterSize() { + return filterSize; + } + + /** + * Get the total raw key size. + * + * @return the total raw key size. + */ + public long getRawKeySize() { + return rawKeySize; + } + + /** + * Get the total raw value size. + * + * @return the total raw value size. + */ + public long getRawValueSize() { + return rawValueSize; + } + + /** + * Get the number of blocks in this table. + * + * @return the number of blocks in this table. + */ + public long getNumDataBlocks() { + return numDataBlocks; + } + + /** + * Get the number of entries in this table. + * + * @return the number of entries in this table. + */ + public long getNumEntries() { + return numEntries; + } + + /** + * Get the number of deletions in the table. + * + * @return the number of deletions in the table. + */ + public long getNumDeletions() { + return numDeletions; + } + + /** + * Get the number of merge operands in the table. + * + * @return the number of merge operands in the table. + */ + public long getNumMergeOperands() { + return numMergeOperands; + } + + /** + * Get the number of range deletions in this table. + * + * @return the number of range deletions in this table. + */ + public long getNumRangeDeletions() { + return numRangeDeletions; + } + + /** + * Get the format version, reserved for backward compatibility. + * + * @return the format version. + */ + public long getFormatVersion() { + return formatVersion; + } + + /** + * Get the length of the keys. + * + * @return 0 when the key is variable length, otherwise number of + * bytes for each key. + */ + public long getFixedKeyLen() { + return fixedKeyLen; + } + + /** + * Get the ID of column family for this SST file, + * corresponding to the column family identified by + * {@link #getColumnFamilyName()}. + * + * @return the id of the column family. + */ + public long getColumnFamilyId() { + return columnFamilyId; + } + + /** + * The time when the SST file was created. + * Since SST files are immutable, this is equivalent + * to last modified time. + * + * @return the created time. + */ + public long getCreationTime() { + return creationTime; + } + + /** + * Get the timestamp of the earliest key. + * + * @return 0 means unknown, otherwise the timestamp. + */ + public long getOldestKeyTime() { + return oldestKeyTime; + } + + /** + * Get the estimated size of data blocks compressed with a relatively slower + * compression algorithm. + * + * @return 0 means unknown, otherwise the timestamp. + */ + public long getSlowCompressionEstimatedDataSize() { + return slowCompressionEstimatedDataSize; + } + + /** + * Get the estimated size of data blocks compressed with a relatively faster + * compression algorithm. + * + * @return 0 means unknown, otherwise the timestamp. + */ + public long getFastCompressionEstimatedDataSize() { + return fastCompressionEstimatedDataSize; + } + + /** + * Get the name of the column family with which this + * SST file is associated. + * + * @return the name of the column family, or null if the + * column family is unknown. + */ + /*@Nullable*/ public byte[] getColumnFamilyName() { + return columnFamilyName; + } + + /** + * Get the name of the filter policy used in this table. + * + * @return the name of the filter policy, or null if + * no filter policy is used. + */ + /*@Nullable*/ public String getFilterPolicyName() { + return filterPolicyName; + } + + /** + * Get the name of the comparator used in this table. + * + * @return the name of the comparator. + */ + public String getComparatorName() { + return comparatorName; + } + + /** + * Get the name of the merge operator used in this table. + * + * @return the name of the merge operator, or null if no merge operator + * is used. + */ + /*@Nullable*/ public String getMergeOperatorName() { + return mergeOperatorName; + } + + /** + * Get the name of the prefix extractor used in this table. + * + * @return the name of the prefix extractor, or null if no prefix + * extractor is used. + */ + /*@Nullable*/ public String getPrefixExtractorName() { + return prefixExtractorName; + } + + /** + * Get the names of the property collectors factories used in this table. + * + * @return the names of the property collector factories separated + * by commas, e.g. {collector_name[1]},{collector_name[2]},... + */ + public String getPropertyCollectorsNames() { + return propertyCollectorsNames; + } + + /** + * Get the name of the compression algorithm used to compress the SST files. + * + * @return the name of the compression algorithm. + */ + public String getCompressionName() { + return compressionName; + } + + /** + * Get the user collected properties. + * + * @return the user collected properties. + */ + public Map getUserCollectedProperties() { + return userCollectedProperties; + } + + /** + * Get the readable properties. + * + * @return the readable properties. + */ + public Map getReadableProperties() { + return readableProperties; + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final TableProperties that = (TableProperties) o; + return dataSize == that.dataSize && indexSize == that.indexSize + && indexPartitions == that.indexPartitions && topLevelIndexSize == that.topLevelIndexSize + && indexKeyIsUserKey == that.indexKeyIsUserKey + && indexValueIsDeltaEncoded == that.indexValueIsDeltaEncoded + && filterSize == that.filterSize && rawKeySize == that.rawKeySize + && rawValueSize == that.rawValueSize && numDataBlocks == that.numDataBlocks + && numEntries == that.numEntries && numDeletions == that.numDeletions + && numMergeOperands == that.numMergeOperands && numRangeDeletions == that.numRangeDeletions + && formatVersion == that.formatVersion && fixedKeyLen == that.fixedKeyLen + && columnFamilyId == that.columnFamilyId && creationTime == that.creationTime + && oldestKeyTime == that.oldestKeyTime + && slowCompressionEstimatedDataSize == that.slowCompressionEstimatedDataSize + && fastCompressionEstimatedDataSize == that.fastCompressionEstimatedDataSize + && externalSstFileGlobalSeqnoOffset == that.externalSstFileGlobalSeqnoOffset + && Arrays.equals(columnFamilyName, that.columnFamilyName) + && Objects.equals(filterPolicyName, that.filterPolicyName) + && Objects.equals(comparatorName, that.comparatorName) + && Objects.equals(mergeOperatorName, that.mergeOperatorName) + && Objects.equals(prefixExtractorName, that.prefixExtractorName) + && Objects.equals(propertyCollectorsNames, that.propertyCollectorsNames) + && Objects.equals(compressionName, that.compressionName) + && Objects.equals(userCollectedProperties, that.userCollectedProperties) + && Objects.equals(readableProperties, that.readableProperties); + } + + @Override + public int hashCode() { + int result = Objects.hash(dataSize, indexSize, indexPartitions, topLevelIndexSize, + indexKeyIsUserKey, indexValueIsDeltaEncoded, filterSize, rawKeySize, rawValueSize, + numDataBlocks, numEntries, numDeletions, numMergeOperands, numRangeDeletions, formatVersion, + fixedKeyLen, columnFamilyId, creationTime, oldestKeyTime, slowCompressionEstimatedDataSize, + fastCompressionEstimatedDataSize, externalSstFileGlobalSeqnoOffset, filterPolicyName, + comparatorName, mergeOperatorName, prefixExtractorName, propertyCollectorsNames, + compressionName, userCollectedProperties, readableProperties); + result = 31 * result + Arrays.hashCode(columnFamilyName); + return result; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ThreadStatus.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ThreadStatus.java new file mode 100644 index 0000000..38e7fad --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ThreadStatus.java @@ -0,0 +1,224 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Map; + +public class ThreadStatus { + private final long threadId; + private final ThreadType threadType; + private final String dbName; + private final String cfName; + private final OperationType operationType; + private final long operationElapsedTime; // microseconds + private final OperationStage operationStage; + private final long[] operationProperties; + private final StateType stateType; + + /** + * Invoked from C++ via JNI + */ + private ThreadStatus(final long threadId, + final byte threadTypeValue, + final String dbName, + final String cfName, + final byte operationTypeValue, + final long operationElapsedTime, + final byte operationStageValue, + final long[] operationProperties, + final byte stateTypeValue) { + this.threadId = threadId; + this.threadType = ThreadType.fromValue(threadTypeValue); + this.dbName = dbName; + this.cfName = cfName; + this.operationType = OperationType.fromValue(operationTypeValue); + this.operationElapsedTime = operationElapsedTime; + this.operationStage = OperationStage.fromValue(operationStageValue); + this.operationProperties = operationProperties; + this.stateType = StateType.fromValue(stateTypeValue); + } + + /** + * Get the unique ID of the thread. + * + * @return the thread id + */ + public long getThreadId() { + return threadId; + } + + /** + * Get the type of the thread. + * + * @return the type of the thread. + */ + public ThreadType getThreadType() { + return threadType; + } + + /** + * The name of the DB instance that the thread is currently + * involved with. + * + * @return the name of the db, or null if the thread is not involved + * in any DB operation. + */ + /* @Nullable */ public String getDbName() { + return dbName; + } + + /** + * The name of the Column Family that the thread is currently + * involved with. + * + * @return the name of the db, or null if the thread is not involved + * in any column Family operation. + */ + /* @Nullable */ public String getCfName() { + return cfName; + } + + /** + * Get the operation (high-level action) that the current thread is involved + * with. + * + * @return the operation + */ + public OperationType getOperationType() { + return operationType; + } + + /** + * Get the elapsed time of the current thread operation in microseconds. + * + * @return the elapsed time + */ + public long getOperationElapsedTime() { + return operationElapsedTime; + } + + /** + * Get the current stage where the thread is involved in the current + * operation. + * + * @return the current stage of the current operation + */ + public OperationStage getOperationStage() { + return operationStage; + } + + /** + * Get the list of properties that describe some details about the current + * operation. + *

    + * Each field in might have different meanings for different operations. + * + * @return the properties + */ + public long[] getOperationProperties() { + return operationProperties; + } + + /** + * Get the state (lower-level action) that the current thread is involved + * with. + * + * @return the state + */ + public StateType getStateType() { + return stateType; + } + + /** + * Get the name of the thread type. + * + * @param threadType the thread type + * + * @return the name of the thread type. + */ + public static String getThreadTypeName(final ThreadType threadType) { + return getThreadTypeName(threadType.getValue()); + } + + /** + * Get the name of an operation given its type. + * + * @param operationType the type of operation. + * + * @return the name of the operation. + */ + public static String getOperationName(final OperationType operationType) { + return getOperationName(operationType.getValue()); + } + + public static String microsToString(final long operationElapsedTime) { + return microsToStringNative(operationElapsedTime); + } + + /** + * Obtain a human-readable string describing the specified operation stage. + * + * @param operationStage the stage of the operation. + * + * @return the description of the operation stage. + */ + public static String getOperationStageName( + final OperationStage operationStage) { + return getOperationStageName(operationStage.getValue()); + } + + /** + * Obtain the name of the "i"th operation property of the + * specified operation. + * + * @param operationType the operation type. + * @param i the index of the operation property. + * + * @return the name of the operation property + */ + public static String getOperationPropertyName( + final OperationType operationType, final int i) { + return getOperationPropertyName(operationType.getValue(), i); + } + + /** + * Translate the "i"th property of the specified operation given + * a property value. + * + * @param operationType the operation type. + * @param operationProperties the operation properties. + * + * @return the property values. + */ + public static Map interpretOperationProperties( + final OperationType operationType, final long[] operationProperties) { + return interpretOperationProperties(operationType.getValue(), + operationProperties); + } + + /** + * Obtain the name of a state given its type. + * + * @param stateType the state type. + * + * @return the name of the state. + */ + public static String getStateName(final StateType stateType) { + return getStateName(stateType.getValue()); + } + + private static native String getThreadTypeName(final byte threadTypeValue); + private static native String getOperationName(final byte operationTypeValue); + private static native String microsToStringNative( + final long operationElapsedTime); + private static native String getOperationStageName( + final byte operationStageTypeValue); + private static native String getOperationPropertyName( + final byte operationTypeValue, final int i); + private static native MapinterpretOperationProperties( + final byte operationTypeValue, final long[] operationProperties); + private static native String getStateName(final byte stateTypeValue); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ThreadType.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ThreadType.java new file mode 100644 index 0000000..cc329f4 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/ThreadType.java @@ -0,0 +1,65 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * The type of a thread. + */ +public enum ThreadType { + /** + * RocksDB BG thread in high-pri thread pool. + */ + HIGH_PRIORITY((byte)0x0), + + /** + * RocksDB BG thread in low-pri thread pool. + */ + LOW_PRIORITY((byte)0x1), + + /** + * User thread (Non-RocksDB BG thread). + */ + USER((byte)0x2), + + /** + * RocksDB BG thread in bottom-pri thread pool + */ + BOTTOM_PRIORITY((byte)0x3); + + private final byte value; + + ThreadType(final byte value) { + this.value = value; + } + + /** + * Get the internal representation value. + * + * @return the internal representation value. + */ + byte getValue() { + return value; + } + + /** + * Get the Thread type from the internal representation value. + * + * @param value the internal representation value. + * + * @return the thread type + * + * @throws IllegalArgumentException if the value does not match a ThreadType + */ + static ThreadType fromValue(final byte value) + throws IllegalArgumentException { + for (final ThreadType threadType : ThreadType.values()) { + if (threadType.value == value) { + return threadType; + } + } + throw new IllegalArgumentException("Unknown value for ThreadType: " + value); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TickerType.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TickerType.java new file mode 100644 index 0000000..c167f74 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TickerType.java @@ -0,0 +1,802 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * The logical mapping of tickers defined in rocksdb::Tickers. + *

    + * Java byte value mappings don't align 1:1 to the c++ values. c++ rocksdb::Tickers enumeration type + * is uint32_t and java org.rocksdb.TickerType is byte, this causes mapping issues when + * rocksdb::Tickers value is greater then 127 (0x7F) for jbyte jni interface as range greater is not + * available. Without breaking interface in minor versions, value mappings for + * org.rocksdb.TickerType leverage full byte range [-128 (-0x80), (0x7F)]. Newer tickers added + * should descend into negative values until TICKER_ENUM_MAX reaches -128 (-0x80). + */ +public enum TickerType { + + /** + * total block cache misses + * + * REQUIRES: BLOCK_CACHE_MISS == BLOCK_CACHE_INDEX_MISS + + * BLOCK_CACHE_FILTER_MISS + + * BLOCK_CACHE_DATA_MISS; + */ + BLOCK_CACHE_MISS((byte) 0x0), + + /** + * total block cache hit + * + * REQUIRES: BLOCK_CACHE_HIT == BLOCK_CACHE_INDEX_HIT + + * BLOCK_CACHE_FILTER_HIT + + * BLOCK_CACHE_DATA_HIT; + */ + BLOCK_CACHE_HIT((byte) 0x1), + + BLOCK_CACHE_ADD((byte) 0x2), + + /** + * # of failures when adding blocks to block cache. + */ + BLOCK_CACHE_ADD_FAILURES((byte) 0x3), + + /** + * # of times cache miss when accessing index block from block cache. + */ + BLOCK_CACHE_INDEX_MISS((byte) 0x4), + + /** + * # of times cache hit when accessing index block from block cache. + */ + BLOCK_CACHE_INDEX_HIT((byte) 0x5), + + /** + * # of index blocks added to block cache. + */ + BLOCK_CACHE_INDEX_ADD((byte) 0x6), + + /** + * # of bytes of index blocks inserted into cache + */ + BLOCK_CACHE_INDEX_BYTES_INSERT((byte) 0x7), + + /** + * # of times cache miss when accessing filter block from block cache. + */ + BLOCK_CACHE_FILTER_MISS((byte) 0x9), + + /** + * # of times cache hit when accessing filter block from block cache. + */ + BLOCK_CACHE_FILTER_HIT((byte) 0xA), + + /** + * # of filter blocks added to block cache. + */ + BLOCK_CACHE_FILTER_ADD((byte) 0xB), + + /** + * # of bytes of bloom filter blocks inserted into cache + */ + BLOCK_CACHE_FILTER_BYTES_INSERT((byte) 0xC), + + /** + * # of times cache miss when accessing data block from block cache. + */ + BLOCK_CACHE_DATA_MISS((byte) 0xE), + + /** + * # of times cache hit when accessing data block from block cache. + */ + BLOCK_CACHE_DATA_HIT((byte) 0xF), + + /** + * # of data blocks added to block cache. + */ + BLOCK_CACHE_DATA_ADD((byte) 0x10), + + /** + * # of bytes of data blocks inserted into cache + */ + BLOCK_CACHE_DATA_BYTES_INSERT((byte) 0x11), + + /** + * # of bytes read from cache. + */ + BLOCK_CACHE_BYTES_READ((byte) 0x12), + + /** + * # of bytes written into cache. + */ + BLOCK_CACHE_BYTES_WRITE((byte) 0x13), + + /** + * # of times bloom filter has avoided file reads. + */ + BLOOM_FILTER_USEFUL((byte) 0x14), + + /** + * # persistent cache hit + */ + PERSISTENT_CACHE_HIT((byte) 0x15), + + /** + * # persistent cache miss + */ + PERSISTENT_CACHE_MISS((byte) 0x16), + + /** + * # total simulation block cache hits + */ + SIM_BLOCK_CACHE_HIT((byte) 0x17), + + /** + * # total simulation block cache misses + */ + SIM_BLOCK_CACHE_MISS((byte) 0x18), + + /** + * # of memtable hits. + */ + MEMTABLE_HIT((byte) 0x19), + + /** + * # of memtable misses. + */ + MEMTABLE_MISS((byte) 0x1A), + + /** + * # of Get() queries served by L0 + */ + GET_HIT_L0((byte) 0x1B), + + /** + * # of Get() queries served by L1 + */ + GET_HIT_L1((byte) 0x1C), + + /** + * # of Get() queries served by L2 and up + */ + GET_HIT_L2_AND_UP((byte) 0x1D), + + /** + * COMPACTION_KEY_DROP_* count the reasons for key drop during compaction + * There are 4 reasons currently. + */ + + /** + * key was written with a newer value. + */ + COMPACTION_KEY_DROP_NEWER_ENTRY((byte) 0x1E), + + /** + * Also includes keys dropped for range del. + * The key is obsolete. + */ + COMPACTION_KEY_DROP_OBSOLETE((byte) 0x1F), + + /** + * key was covered by a range tombstone. + */ + COMPACTION_KEY_DROP_RANGE_DEL((byte) 0x20), + + /** + * User compaction function has dropped the key. + */ + COMPACTION_KEY_DROP_USER((byte) 0x21), + + /** + * all keys in range were deleted. + */ + COMPACTION_RANGE_DEL_DROP_OBSOLETE((byte) 0x22), + + /** + * Number of keys written to the database via the Put and Write call's. + */ + NUMBER_KEYS_WRITTEN((byte) 0x23), + + /** + * Number of Keys read. + */ + NUMBER_KEYS_READ((byte) 0x24), + + /** + * Number keys updated, if inplace update is enabled + */ + NUMBER_KEYS_UPDATED((byte) 0x25), + + /** + * The number of uncompressed bytes issued by DB::Put(), DB::Delete(),\ + * DB::Merge(), and DB::Write(). + */ + BYTES_WRITTEN((byte) 0x26), + + /** + * The number of uncompressed bytes read from DB::Get(). It could be + * either from memtables, cache, or table files. + * + * For the number of logical bytes read from DB::MultiGet(), + * please use {@link #NUMBER_MULTIGET_BYTES_READ}. + */ + BYTES_READ((byte) 0x27), + + /** + * The number of calls to seek. + */ + NUMBER_DB_SEEK((byte) 0x28), + + /** + * The number of calls to next. + */ + NUMBER_DB_NEXT((byte) 0x29), + + /** + * The number of calls to prev. + */ + NUMBER_DB_PREV((byte) 0x2A), + + /** + * The number of calls to seek that returned data. + */ + NUMBER_DB_SEEK_FOUND((byte) 0x2B), + + /** + * The number of calls to next that returned data. + */ + NUMBER_DB_NEXT_FOUND((byte) 0x2C), + + /** + * The number of calls to prev that returned data. + */ + NUMBER_DB_PREV_FOUND((byte) 0x2D), + + /** + * The number of uncompressed bytes read from an iterator. + * Includes size of key and value. + */ + ITER_BYTES_READ((byte) 0x2E), + + NO_FILE_OPENS((byte) 0x30), + + NO_FILE_ERRORS((byte) 0x31), + + /** + * Writer has to wait for compaction or flush to finish. + */ + STALL_MICROS((byte) 0x35), + + /** + * The wait time for db mutex. + * + * Disabled by default. To enable it set stats level to {@link StatsLevel#ALL} + */ + DB_MUTEX_WAIT_MICROS((byte) 0x36), + + /** + * Number of MultiGet calls. + */ + NUMBER_MULTIGET_CALLS((byte) 0x39), + + /** + * Number of MultiGet keys read. + */ + NUMBER_MULTIGET_KEYS_READ((byte) 0x3A), + + /** + * Number of MultiGet bytes read. + */ + NUMBER_MULTIGET_BYTES_READ((byte) 0x3B), + + NUMBER_MERGE_FAILURES((byte) 0x3D), + + /** + * Number of times bloom was checked before creating iterator on a + * file, and the number of times the check was useful in avoiding + * iterator creation (and thus likely IOPs). + */ + BLOOM_FILTER_PREFIX_CHECKED((byte) 0x3E), + BLOOM_FILTER_PREFIX_USEFUL((byte) 0x3F), + + /** + * Number of times we had to reseek inside an iteration to skip + * over large number of keys with same userkey. + */ + NUMBER_OF_RESEEKS_IN_ITERATION((byte) 0x40), + + /** + * Record the number of calls to {@link RocksDB#getUpdatesSince(long)}. Useful to keep track of + * transaction log iterator refreshes. + */ + GET_UPDATES_SINCE_CALLS((byte) 0x41), + + /** + * Number of times WAL sync is done. + */ + WAL_FILE_SYNCED((byte) 0x46), + + /** + * Number of bytes written to WAL. + */ + WAL_FILE_BYTES((byte) 0x47), + + /** + * Writes can be processed by requesting thread or by the thread at the + * head of the writers queue. + */ + WRITE_DONE_BY_SELF((byte) 0x48), + + /** + * Equivalent to writes done for others. + */ + WRITE_DONE_BY_OTHER((byte) 0x49), + + /** + * Number of Write calls that request WAL. + */ + WRITE_WITH_WAL((byte) 0x4B), + + /** + * Bytes read during compaction. + */ + COMPACT_READ_BYTES((byte) 0x4C), + + /** + * Bytes written during compaction. + */ + COMPACT_WRITE_BYTES((byte) 0x4D), + + /** + * Bytes written during flush. + */ + FLUSH_WRITE_BYTES((byte) 0x4E), + + /** + * Number of table's properties loaded directly from file, without creating + * table reader object. + */ + NUMBER_DIRECT_LOAD_TABLE_PROPERTIES((byte) 0x4F), + NUMBER_SUPERVERSION_ACQUIRES((byte) 0x50), + NUMBER_SUPERVERSION_RELEASES((byte) 0x51), + NUMBER_SUPERVERSION_CLEANUPS((byte) 0x52), + + /** + * # of compressions/decompressions executed + */ + NUMBER_BLOCK_COMPRESSED((byte) 0x53), + NUMBER_BLOCK_DECOMPRESSED((byte) 0x54), + + NUMBER_BLOCK_NOT_COMPRESSED((byte) 0x55), + MERGE_OPERATION_TOTAL_TIME((byte) 0x56), + FILTER_OPERATION_TOTAL_TIME((byte) 0x57), + + /** + * Row cache. + */ + ROW_CACHE_HIT((byte) 0x58), + ROW_CACHE_MISS((byte) 0x59), + + /** + * Read amplification statistics. + * + * Read amplification can be calculated using this formula + * (READ_AMP_TOTAL_READ_BYTES / READ_AMP_ESTIMATE_USEFUL_BYTES) + * + * REQUIRES: ReadOptions::read_amp_bytes_per_bit to be enabled + */ + + /** + * Estimate of total bytes actually used. + */ + READ_AMP_ESTIMATE_USEFUL_BYTES((byte) 0x5A), + + /** + * Total size of loaded data blocks. + */ + READ_AMP_TOTAL_READ_BYTES((byte) 0x5B), + + /** + * Number of refill intervals where rate limiter's bytes are fully consumed. + */ + NUMBER_RATE_LIMITER_DRAINS((byte) 0x5C), + + /** + * Number of internal skipped during iteration + */ + NUMBER_ITER_SKIP((byte) 0x5D), + + /** + * Number of MultiGet keys found (vs number requested) + */ + NUMBER_MULTIGET_KEYS_FOUND((byte) 0x5E), + + // -0x01 to fixate the new value that incorrectly changed TICKER_ENUM_MAX + /** + * Number of iterators created. + */ + NO_ITERATOR_CREATED((byte) -0x01), + + /** + * Number of iterators deleted. + */ + NO_ITERATOR_DELETED((byte) 0x60), + + /** + * Deletions obsoleted before bottom level due to file gap optimization. + */ + COMPACTION_OPTIMIZED_DEL_DROP_OBSOLETE((byte) 0x61), + + /** + * If a compaction was cancelled in sfm to prevent ENOSPC + */ + COMPACTION_CANCELLED((byte) 0x62), + + /** + * # of times bloom FullFilter has not avoided the reads. + */ + BLOOM_FILTER_FULL_POSITIVE((byte) 0x63), + + /** + * # of times bloom FullFilter has not avoided the reads and data actually + * exist. + */ + BLOOM_FILTER_FULL_TRUE_POSITIVE((byte) 0x64), + + /** + * BlobDB specific stats + * # of Put/PutTTL/PutUntil to BlobDB. + */ + BLOB_DB_NUM_PUT((byte) 0x65), + + /** + * # of Write to BlobDB. + */ + BLOB_DB_NUM_WRITE((byte) 0x66), + + /** + * # of Get to BlobDB. + */ + BLOB_DB_NUM_GET((byte) 0x67), + + /** + * # of MultiGet to BlobDB. + */ + BLOB_DB_NUM_MULTIGET((byte) 0x68), + + /** + * # of Seek/SeekToFirst/SeekToLast/SeekForPrev to BlobDB iterator. + */ + BLOB_DB_NUM_SEEK((byte) 0x69), + + /** + * # of Next to BlobDB iterator. + */ + BLOB_DB_NUM_NEXT((byte) 0x6A), + + /** + * # of Prev to BlobDB iterator. + */ + BLOB_DB_NUM_PREV((byte) 0x6B), + + /** + * # of keys written to BlobDB. + */ + BLOB_DB_NUM_KEYS_WRITTEN((byte) 0x6C), + + /** + * # of keys read from BlobDB. + */ + BLOB_DB_NUM_KEYS_READ((byte) 0x6D), + + /** + * # of bytes (key + value) written to BlobDB. + */ + BLOB_DB_BYTES_WRITTEN((byte) 0x6E), + + /** + * # of bytes (keys + value) read from BlobDB. + */ + BLOB_DB_BYTES_READ((byte) 0x6F), + + /** + * # of keys written by BlobDB as non-TTL inlined value. + */ + BLOB_DB_WRITE_INLINED((byte) 0x70), + + /** + * # of keys written by BlobDB as TTL inlined value. + */ + BLOB_DB_WRITE_INLINED_TTL((byte) 0x71), + + /** + * # of keys written by BlobDB as non-TTL blob value. + */ + BLOB_DB_WRITE_BLOB((byte) 0x72), + + /** + * # of keys written by BlobDB as TTL blob value. + */ + BLOB_DB_WRITE_BLOB_TTL((byte) 0x73), + + /** + * # of bytes written to blob file. + */ + BLOB_DB_BLOB_FILE_BYTES_WRITTEN((byte) 0x74), + + /** + * # of bytes read from blob file. + */ + BLOB_DB_BLOB_FILE_BYTES_READ((byte) 0x75), + + /** + * # of times a blob files being synced. + */ + BLOB_DB_BLOB_FILE_SYNCED((byte) 0x76), + + /** + * # of blob index evicted from base DB by BlobDB compaction filter because + * of expiration. + */ + BLOB_DB_BLOB_INDEX_EXPIRED_COUNT((byte) 0x77), + + /** + * Size of blob index evicted from base DB by BlobDB compaction filter + * because of expiration. + */ + BLOB_DB_BLOB_INDEX_EXPIRED_SIZE((byte) 0x78), + + /** + * # of blob index evicted from base DB by BlobDB compaction filter because + * of corresponding file deleted. + */ + BLOB_DB_BLOB_INDEX_EVICTED_COUNT((byte) 0x79), + + /** + * Size of blob index evicted from base DB by BlobDB compaction filter + * because of corresponding file deleted. + */ + BLOB_DB_BLOB_INDEX_EVICTED_SIZE((byte) 0x7A), + + /** + * # of blob files being garbage collected. + */ + BLOB_DB_GC_NUM_FILES((byte) 0x7B), + + /** + * # of blob files generated by garbage collection. + */ + BLOB_DB_GC_NUM_NEW_FILES((byte) 0x7C), + + /** + * # of BlobDB garbage collection failures. + */ + BLOB_DB_GC_FAILURES((byte) 0x7D), + + /** + * # of keys relocated to new blob file by garbage collection. + */ + BLOB_DB_GC_NUM_KEYS_RELOCATED((byte) -0x02), + + /** + * # of bytes relocated to new blob file by garbage collection. + */ + BLOB_DB_GC_BYTES_RELOCATED((byte) -0x05), + + /** + * # of blob files evicted because of BlobDB is full. + */ + BLOB_DB_FIFO_NUM_FILES_EVICTED((byte) -0x06), + + /** + * # of keys in the blob files evicted because of BlobDB is full. + */ + BLOB_DB_FIFO_NUM_KEYS_EVICTED((byte) -0x07), + + /** + * # of bytes in the blob files evicted because of BlobDB is full. + */ + BLOB_DB_FIFO_BYTES_EVICTED((byte) -0x08), + + /** + * These counters indicate a performance issue in WritePrepared transactions. + * We should not seem them ticking them much. + * # of times prepare_mutex_ is acquired in the fast path. + */ + TXN_PREPARE_MUTEX_OVERHEAD((byte) -0x09), + + /** + * # of times old_commit_map_mutex_ is acquired in the fast path. + */ + TXN_OLD_COMMIT_MAP_MUTEX_OVERHEAD((byte) -0x0A), + + /** + * # of times we checked a batch for duplicate keys. + */ + TXN_DUPLICATE_KEY_OVERHEAD((byte) -0x0B), + + /** + * # of times snapshot_mutex_ is acquired in the fast path. + */ + TXN_SNAPSHOT_MUTEX_OVERHEAD((byte) -0x0C), + + /** + * # of times ::Get returned TryAgain due to expired snapshot seq + */ + TXN_GET_TRY_AGAIN((byte) -0x0D), + + /** + * # of files marked as trash by delete scheduler + */ + FILES_MARKED_TRASH((byte) -0x0E), + + /** + * # of files deleted immediately by delete scheduler + */ + FILES_DELETED_IMMEDIATELY((byte) -0x0f), + + /** + * Compaction read and write statistics broken down by CompactionReason + */ + COMPACT_READ_BYTES_MARKED((byte) -0x10), + COMPACT_READ_BYTES_PERIODIC((byte) -0x11), + COMPACT_READ_BYTES_TTL((byte) -0x12), + COMPACT_WRITE_BYTES_MARKED((byte) -0x13), + COMPACT_WRITE_BYTES_PERIODIC((byte) -0x14), + COMPACT_WRITE_BYTES_TTL((byte) -0x15), + + /** + * DB error handler statistics + */ + ERROR_HANDLER_BG_ERROR_COUNT((byte) -0x16), + ERROR_HANDLER_BG_IO_ERROR_COUNT((byte) -0x17), + ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT((byte) -0x18), + ERROR_HANDLER_AUTORESUME_COUNT((byte) -0x19), + ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT((byte) -0x1A), + ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT((byte) -0x1B), + + /** + * Bytes of raw data (payload) found on memtable at flush time. + * Contains the sum of garbage payload (bytes that are discarded + * at flush time) and useful payload (bytes of data that will + * eventually be written to SSTable). + */ + MEMTABLE_PAYLOAD_BYTES_AT_FLUSH((byte) -0x1C), + /** + * Outdated bytes of data present on memtable at flush time. + */ + MEMTABLE_GARBAGE_BYTES_AT_FLUSH((byte) -0x1D), + + /** + * Number of secondary cache hits + */ + SECONDARY_CACHE_HITS((byte) -0x1E), + + /** + * Bytes read by `VerifyChecksum()` and `VerifyFileChecksums()` APIs. + */ + VERIFY_CHECKSUM_READ_BYTES((byte) -0x1F), + + /** + * Bytes read/written while creating backups + */ + BACKUP_READ_BYTES((byte) -0x20), + BACKUP_WRITE_BYTES((byte) -0x21), + + /** + * Remote compaction read/write statistics + */ + REMOTE_COMPACT_READ_BYTES((byte) -0x22), + REMOTE_COMPACT_WRITE_BYTES((byte) -0x23), + + /** + * Tiered storage related statistics + */ + HOT_FILE_READ_BYTES((byte) -0x24), + WARM_FILE_READ_BYTES((byte) -0x25), + COLD_FILE_READ_BYTES((byte) -0x26), + HOT_FILE_READ_COUNT((byte) -0x27), + WARM_FILE_READ_COUNT((byte) -0x28), + COLD_FILE_READ_COUNT((byte) -0x29), + + /** + * (non-)last level read statistics + */ + LAST_LEVEL_READ_BYTES((byte) -0x2A), + LAST_LEVEL_READ_COUNT((byte) -0x2B), + NON_LAST_LEVEL_READ_BYTES((byte) -0x2C), + NON_LAST_LEVEL_READ_COUNT((byte) -0x2D), + + /** + * Number of block checksum verifications + */ + BLOCK_CHECKSUM_COMPUTE_COUNT((byte) -0x2E), + + /** + * # of times cache miss when accessing blob from blob cache. + */ + BLOB_DB_CACHE_MISS((byte) -0x2F), + + /** + * # of times cache hit when accessing blob from blob cache. + */ + BLOB_DB_CACHE_HIT((byte) -0x30), + + /** + * # of data blocks added to blob cache. + */ + BLOB_DB_CACHE_ADD((byte) -0x31), + + /** + * # # of failures when adding blobs to blob cache. + */ + BLOB_DB_CACHE_ADD_FAILURES((byte) -0x32), + + /** + * # of bytes read from blob cache. + */ + BLOB_DB_CACHE_BYTES_READ((byte) -0x33), + + /** + * # of bytes written into blob cache. + */ + BLOB_DB_CACHE_BYTES_WRITE((byte) -0x34), + + /** + * Number of lookup into the prefetched tail (see + * `TABLE_OPEN_PREFETCH_TAIL_READ_BYTES`) + * that can't find its data for table open + */ + TABLE_OPEN_PREFETCH_TAIL_MISS((byte) -0x3A), + + /** + * Number of lookup into the prefetched tail (see + * `TABLE_OPEN_PREFETCH_TAIL_READ_BYTES`) + * that finds its data for table open + */ + TABLE_OPEN_PREFETCH_TAIL_HIT((byte) -0x3B), + + /** + * Number of times RocksDB detected a corruption while verifying a block + * checksum. RocksDB does not remember corruptions that happened during user + * reads so the same block corruption may be detected multiple times. + */ + BLOCK_CHECKSUM_MISMATCH_COUNT((byte) -0x3C), + + TICKER_ENUM_MAX((byte) 0x5F); + + private final byte value; + + TickerType(final byte value) { + this.value = value; + } + + /** + * Returns the byte value of the enumerations value + * + * @return byte representation + */ + public byte getValue() { + return value; + } + + /** + * Get Ticker type by byte value. + * + * @param value byte representation of TickerType. + * + * @return {@link org.rocksdb.TickerType} instance. + * @throws java.lang.IllegalArgumentException if an invalid + * value is provided. + */ + public static TickerType getTickerType(final byte value) { + for (final TickerType tickerType : TickerType.values()) { + if (tickerType.getValue() == value) { + return tickerType; + } + } + throw new IllegalArgumentException( + "Illegal value provided for TickerType."); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TimedEnv.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TimedEnv.java new file mode 100644 index 0000000..dc8b5d6 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TimedEnv.java @@ -0,0 +1,30 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Timed environment. + */ +public class TimedEnv extends Env { + + /** + *

    Creates a new environment that measures function call times for + * filesystem operations, reporting results to variables in PerfContext.

    + * + * + *

    The caller must delete the result when it is + * no longer needed.

    + * + * @param baseEnv the base environment, + * must remain live while the result is in use. + */ + public TimedEnv(final Env baseEnv) { + super(createTimedEnv(baseEnv.nativeHandle_)); + } + + private static native long createTimedEnv(final long baseEnvHandle); + @Override protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TraceOptions.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TraceOptions.java new file mode 100644 index 0000000..cf5f7bb --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TraceOptions.java @@ -0,0 +1,32 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * TraceOptions is used for + * {@link RocksDB#startTrace(TraceOptions, AbstractTraceWriter)}. + */ +public class TraceOptions { + private final long maxTraceFileSize; + + public TraceOptions() { + this.maxTraceFileSize = 64L * 1024L * 1024L * 1024L; // 64 GB + } + + public TraceOptions(final long maxTraceFileSize) { + this.maxTraceFileSize = maxTraceFileSize; + } + + /** + * To avoid the trace file size grows larger than the storage space, + * user can set the max trace file size in Bytes. Default is 64 GB. + * + * @return the max trace size + */ + public long getMaxTraceFileSize() { + return maxTraceFileSize; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TraceWriter.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TraceWriter.java new file mode 100644 index 0000000..cb0234e --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TraceWriter.java @@ -0,0 +1,36 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * TraceWriter allows exporting RocksDB traces to any system, + * one operation at a time. + */ +public interface TraceWriter { + + /** + * Write the data. + * + * @param data the data + * + * @throws RocksDBException if an error occurs whilst writing. + */ + void write(final Slice data) throws RocksDBException; + + /** + * Close the writer. + * + * @throws RocksDBException if an error occurs whilst closing the writer. + */ + void closeWriter() throws RocksDBException; + + /** + * Get the size of the file that this writer is writing to. + * + * @return the file size + */ + long getFileSize(); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Transaction.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Transaction.java new file mode 100644 index 0000000..7d61a20 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/Transaction.java @@ -0,0 +1,2162 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Provides BEGIN/COMMIT/ROLLBACK transactions. + *

    + * To use transactions, you must first create either an + * {@link OptimisticTransactionDB} or a {@link TransactionDB} + * + * To create a transaction, use + * {@link OptimisticTransactionDB#beginTransaction(org.rocksdb.WriteOptions)} or + * {@link TransactionDB#beginTransaction(org.rocksdb.WriteOptions)} + * + * It is up to the caller to synchronize access to this object. + *

    + * See samples/src/main/java/OptimisticTransactionSample.java and + * samples/src/main/java/TransactionSample.java for some simple + * examples. + */ +public class Transaction extends RocksObject { + + private final RocksDB parent; + + /** + * Intentionally package private + * as this is called from + * {@link OptimisticTransactionDB#beginTransaction(org.rocksdb.WriteOptions)} + * or {@link TransactionDB#beginTransaction(org.rocksdb.WriteOptions)} + * + * @param parent This must be either {@link TransactionDB} or + * {@link OptimisticTransactionDB} + * @param transactionHandle The native handle to the underlying C++ + * transaction object + */ + Transaction(final RocksDB parent, final long transactionHandle) { + super(transactionHandle); + this.parent = parent; + } + + /** + * If a transaction has a snapshot set, the transaction will ensure that + * any keys successfully written (or fetched via {@link #getForUpdate}) have + * not been modified outside of this transaction since the time the snapshot + * was set. + *

    + * If a snapshot has not been set, the transaction guarantees that keys have + * not been modified since the time each key was first written (or fetched via + * {@link #getForUpdate}). + *

    + * Using {@code #setSnapshot()} will provide stricter isolation guarantees + * at the expense of potentially more transaction failures due to conflicts + * with other writes. + *

    + * Calling {@code #setSnapshot()} has no effect on keys written before this + * function has been called. + *

    + * {@code #setSnapshot()} may be called multiple times if you would like to + * change the snapshot used for different operations in this transaction. + *

    + * Calling {@code #setSnapshot()} will not affect the version of Data returned + * by get(...) methods. See {@link #get} for more details. + */ + public void setSnapshot() { + assert(isOwningHandle()); + setSnapshot(nativeHandle_); + } + + /** + * Similar to {@link #setSnapshot()}, but will not change the current snapshot + * until put/merge/delete/getForUpdate/multiGetForUpdate is called. + * By calling this function, the transaction will essentially call + * {@link #setSnapshot()} for you right before performing the next + * write/getForUpdate. + *

    + * Calling {@code #setSnapshotOnNextOperation()} will not affect what + * snapshot is returned by {@link #getSnapshot} until the next + * write/getForUpdate is executed. + *

    + * When the snapshot is created the notifier's snapshotCreated method will + * be called so that the caller can get access to the snapshot. + *

    + * This is an optimization to reduce the likelihood of conflicts that + * could occur in between the time {@link #setSnapshot()} is called and the + * first write/getForUpdate operation. i.e. this prevents the following + * race-condition: + *

    + * txn1->setSnapshot(); + * txn2->put("A", ...); + * txn2->commit(); + * txn1->getForUpdate(opts, "A", ...); * FAIL! + */ + public void setSnapshotOnNextOperation() { + assert(isOwningHandle()); + setSnapshotOnNextOperation(nativeHandle_); + } + + /** + * Similar to {@link #setSnapshot()}, but will not change the current snapshot + * until put/merge/delete/getForUpdate/multiGetForUpdate is called. + * By calling this function, the transaction will essentially call + * {@link #setSnapshot()} for you right before performing the next + * write/getForUpdate. + *

    + * Calling {@link #setSnapshotOnNextOperation()} will not affect what + * snapshot is returned by {@link #getSnapshot} until the next + * write/getForUpdate is executed. + *

    + * When the snapshot is created the + * {@link AbstractTransactionNotifier#snapshotCreated(Snapshot)} method will + * be called so that the caller can get access to the snapshot. + *

    + * This is an optimization to reduce the likelihood of conflicts that + * could occur in between the time {@link #setSnapshot()} is called and the + * first write/getForUpdate operation. i.e. this prevents the following + * race-condition: + *

    + * txn1->setSnapshot(); + * txn2->put("A", ...); + * txn2->commit(); + * txn1->getForUpdate(opts, "A", ...); * FAIL! + * + * @param transactionNotifier A handler for receiving snapshot notifications + * for the transaction + * + */ + public void setSnapshotOnNextOperation( + final AbstractTransactionNotifier transactionNotifier) { + assert(isOwningHandle()); + setSnapshotOnNextOperation(nativeHandle_, transactionNotifier.nativeHandle_); + } + + /** + * Returns the Snapshot created by the last call to {@link #setSnapshot()}. + *

    + * REQUIRED: The returned Snapshot is only valid up until the next time + * {@link #setSnapshot()}/{@link #setSnapshotOnNextOperation()} is called, + * {@link #clearSnapshot()} is called, or the Transaction is deleted. + * + * @return The snapshot or null if there is no snapshot + */ + public Snapshot getSnapshot() { + assert(isOwningHandle()); + final long snapshotNativeHandle = getSnapshot(nativeHandle_); + if(snapshotNativeHandle == 0) { + return null; + } else { + return new Snapshot(snapshotNativeHandle); + } + } + + /** + * Clears the current snapshot (i.e. no snapshot will be 'set') + *

    + * This removes any snapshot that currently exists or is set to be created + * on the next update operation ({@link #setSnapshotOnNextOperation()}). + *

    + * Calling {@code #clearSnapshot()} has no effect on keys written before this + * function has been called. + *

    + * If a reference to a snapshot was retrieved via {@link #getSnapshot()}, it + * will no longer be valid and should be discarded after a call to + * {@code #clearSnapshot()}. + */ + public void clearSnapshot() { + assert(isOwningHandle()); + clearSnapshot(nativeHandle_); + } + + /** + * Prepare the current transaction for 2PC + */ + public void prepare() throws RocksDBException { + //TODO(AR) consider a Java'ish version of this function, which returns an AutoCloseable (commit) + assert(isOwningHandle()); + prepare(nativeHandle_); + } + + /** + * Write all batched keys to the db atomically. + *

    + * Returns OK on success. + *

    + * May return any error status that could be returned by DB:Write(). + *

    + * If this transaction was created by an {@link OptimisticTransactionDB} + * Status::Busy() may be returned if the transaction could not guarantee + * that there are no write conflicts. Status::TryAgain() may be returned + * if the memtable history size is not large enough + * (See max_write_buffer_number_to_maintain). + *

    + * If this transaction was created by a {@link TransactionDB}, + * Status::Expired() may be returned if this transaction has lived for + * longer than {@link TransactionOptions#getExpiration()}. + * + * @throws RocksDBException if an error occurs when committing the transaction + */ + public void commit() throws RocksDBException { + assert(isOwningHandle()); + commit(nativeHandle_); + } + + /** + * Discard all batched writes in this transaction. + * + * @throws RocksDBException if an error occurs when rolling back the transaction + */ + public void rollback() throws RocksDBException { + assert(isOwningHandle()); + rollback(nativeHandle_); + } + + /** + * Records the state of the transaction for future calls to + * {@link #rollbackToSavePoint()}. + *

    + * May be called multiple times to set multiple save points. + * + * @throws RocksDBException if an error occurs whilst setting a save point + */ + public void setSavePoint() throws RocksDBException { + assert(isOwningHandle()); + setSavePoint(nativeHandle_); + } + + /** + * Undo all operations in this transaction (put, merge, delete, putLogData) + * since the most recent call to {@link #setSavePoint()} and removes the most + * recent {@link #setSavePoint()}. + *

    + * If there is no previous call to {@link #setSavePoint()}, + * returns Status::NotFound() + * + * @throws RocksDBException if an error occurs when rolling back to a save point + */ + public void rollbackToSavePoint() throws RocksDBException { + assert(isOwningHandle()); + rollbackToSavePoint(nativeHandle_); + } + + /** + * This function is similar to + * {@link RocksDB#get(ColumnFamilyHandle, ReadOptions, byte[])} except it will + * also read pending changes in this transaction. + * Currently, this function will return Status::MergeInProgress if the most + * recent write to the queried key in this batch is a Merge. + *

    + * If {@link ReadOptions#snapshot()} is not set, the current version of the + * key will be read. Calling {@link #setSnapshot()} does not affect the + * version of the data returned. + *

    + * Note that setting {@link ReadOptions#setSnapshot(Snapshot)} will affect + * what is read from the DB but will NOT change which keys are read from this + * transaction (the keys in this transaction do not yet belong to any snapshot + * and will be fetched regardless). + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} instance + * @param readOptions Read options. + * @param key the key to retrieve the value for. + * + * @return a byte array storing the value associated with the input key if + * any. null if it does not find the specified key. + * + * @throws RocksDBException thrown if error happens in underlying native + * library. + */ + public byte[] get(final ColumnFamilyHandle columnFamilyHandle, + final ReadOptions readOptions, final byte[] key) throws RocksDBException { + assert(isOwningHandle()); + return get(nativeHandle_, readOptions.nativeHandle_, key, key.length, + columnFamilyHandle.nativeHandle_); + } + + /** + * This function is similar to + * {@link RocksDB#get(ReadOptions, byte[])} except it will + * also read pending changes in this transaction. + * Currently, this function will return Status::MergeInProgress if the most + * recent write to the queried key in this batch is a Merge. + *

    + * If {@link ReadOptions#snapshot()} is not set, the current version of the + * key will be read. Calling {@link #setSnapshot()} does not affect the + * version of the data returned. + *

    + * Note that setting {@link ReadOptions#setSnapshot(Snapshot)} will affect + * what is read from the DB but will NOT change which keys are read from this + * transaction (the keys in this transaction do not yet belong to any snapshot + * and will be fetched regardless). + * + * @param readOptions Read options. + * @param key the key to retrieve the value for. + * + * @return a byte array storing the value associated with the input key if + * any. null if it does not find the specified key. + * + * @throws RocksDBException thrown if error happens in underlying native + * library. + */ + public byte[] get(final ReadOptions readOptions, final byte[] key) + throws RocksDBException { + assert(isOwningHandle()); + return get(nativeHandle_, readOptions.nativeHandle_, key, key.length); + } + + /** + * This function is similar to + * {@link RocksDB#multiGetAsList} except it will + * also read pending changes in this transaction. + * Currently, this function will return Status::MergeInProgress if the most + * recent write to the queried key in this batch is a Merge. + *

    + * If {@link ReadOptions#snapshot()} is not set, the current version of the + * key will be read. Calling {@link #setSnapshot()} does not affect the + * version of the data returned. + *

    + * Note that setting {@link ReadOptions#setSnapshot(Snapshot)} will affect + * what is read from the DB but will NOT change which keys are read from this + * transaction (the keys in this transaction do not yet belong to any snapshot + * and will be fetched regardless). + * + * @param readOptions Read options. + * @param columnFamilyHandles {@link java.util.List} containing + * {@link org.rocksdb.ColumnFamilyHandle} instances. + * @param keys of keys for which values need to be retrieved. + * + * @return Array of values, one for each key + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + * @throws IllegalArgumentException thrown if the size of passed keys is not + * equal to the amount of passed column family handles. + */ + @Deprecated + public byte[][] multiGet(final ReadOptions readOptions, + final List columnFamilyHandles, final byte[][] keys) + throws RocksDBException { + assert(isOwningHandle()); + // Check if key size equals cfList size. If not a exception must be + // thrown. If not a Segmentation fault happens. + if (keys.length != columnFamilyHandles.size()) { + throw new IllegalArgumentException( + "For each key there must be a ColumnFamilyHandle."); + } + if(keys.length == 0) { + return new byte[0][0]; + } + final long[] cfHandles = new long[columnFamilyHandles.size()]; + for (int i = 0; i < columnFamilyHandles.size(); i++) { + cfHandles[i] = columnFamilyHandles.get(i).nativeHandle_; + } + + return multiGet(nativeHandle_, readOptions.nativeHandle_, + keys, cfHandles); + } + + /** + * This function is similar to + * {@link RocksDB#multiGetAsList(ReadOptions, List, List)} except it will + * also read pending changes in this transaction. + * Currently, this function will return Status::MergeInProgress if the most + * recent write to the queried key in this batch is a Merge. + *

    + * If {@link ReadOptions#snapshot()} is not set, the current version of the + * key will be read. Calling {@link #setSnapshot()} does not affect the + * version of the data returned. + *

    + * Note that setting {@link ReadOptions#setSnapshot(Snapshot)} will affect + * what is read from the DB but will NOT change which keys are read from this + * transaction (the keys in this transaction do not yet belong to any snapshot + * and will be fetched regardless). + * + * @param readOptions Read options. + * @param columnFamilyHandles {@link java.util.List} containing + * {@link org.rocksdb.ColumnFamilyHandle} instances. + * @param keys of keys for which values need to be retrieved. + * + * @return Array of values, one for each key + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + * @throws IllegalArgumentException thrown if the size of passed keys is not + * equal to the amount of passed column family handles. + */ + + public List multiGetAsList(final ReadOptions readOptions, + final List columnFamilyHandles, final List keys) + throws RocksDBException { + assert (isOwningHandle()); + // Check if key size equals cfList size. If not a exception must be + // thrown. If not a Segmentation fault happens. + if (keys.size() != columnFamilyHandles.size()) { + throw new IllegalArgumentException("For each key there must be a ColumnFamilyHandle."); + } + if (keys.size() == 0) { + return new ArrayList<>(0); + } + final byte[][] keysArray = keys.toArray(new byte[keys.size()][]); + final long[] cfHandles = new long[columnFamilyHandles.size()]; + for (int i = 0; i < columnFamilyHandles.size(); i++) { + cfHandles[i] = columnFamilyHandles.get(i).nativeHandle_; + } + + return Arrays.asList(multiGet(nativeHandle_, readOptions.nativeHandle_, keysArray, cfHandles)); + } + + /** + * This function is similar to + * {@link RocksDB#multiGetAsList} except it will + * also read pending changes in this transaction. + * Currently, this function will return Status::MergeInProgress if the most + * recent write to the queried key in this batch is a Merge. + *

    + * If {@link ReadOptions#snapshot()} is not set, the current version of the + * key will be read. Calling {@link #setSnapshot()} does not affect the + * version of the data returned. + *

    + * Note that setting {@link ReadOptions#setSnapshot(Snapshot)} will affect + * what is read from the DB but will NOT change which keys are read from this + * transaction (the keys in this transaction do not yet belong to any snapshot + * and will be fetched regardless). + * + * @param readOptions Read options.= + * {@link org.rocksdb.ColumnFamilyHandle} instances. + * @param keys of keys for which values need to be retrieved. + * + * @return Array of values, one for each key + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + @Deprecated + public byte[][] multiGet(final ReadOptions readOptions, final byte[][] keys) + throws RocksDBException { + assert(isOwningHandle()); + if(keys.length == 0) { + return new byte[0][0]; + } + + return multiGet(nativeHandle_, readOptions.nativeHandle_, + keys); + } + + /** + * This function is similar to + * {@link RocksDB#multiGetAsList} except it will + * also read pending changes in this transaction. + * Currently, this function will return Status::MergeInProgress if the most + * recent write to the queried key in this batch is a Merge. + *

    + * If {@link ReadOptions#snapshot()} is not set, the current version of the + * key will be read. Calling {@link #setSnapshot()} does not affect the + * version of the data returned. + *

    + * Note that setting {@link ReadOptions#setSnapshot(Snapshot)} will affect + * what is read from the DB but will NOT change which keys are read from this + * transaction (the keys in this transaction do not yet belong to any snapshot + * and will be fetched regardless). + * + * @param readOptions Read options.= + * {@link org.rocksdb.ColumnFamilyHandle} instances. + * @param keys of keys for which values need to be retrieved. + * + * @return Array of values, one for each key + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public List multiGetAsList(final ReadOptions readOptions, final List keys) + throws RocksDBException { + if (keys.size() == 0) { + return new ArrayList<>(0); + } + final byte[][] keysArray = keys.toArray(new byte[keys.size()][]); + + return Arrays.asList(multiGet(nativeHandle_, readOptions.nativeHandle_, keysArray)); + } + + /** + * Read this key and ensure that this transaction will only + * be able to be committed if this key is not written outside this + * transaction after it has first been read (or after the snapshot if a + * snapshot is set in this transaction). The transaction behavior is the + * same regardless of whether the key exists or not. + *

    + * Note: Currently, this function will return Status::MergeInProgress + * if the most recent write to the queried key in this batch is a Merge. + *

    + * The values returned by this function are similar to + * {@link RocksDB#get(ColumnFamilyHandle, ReadOptions, byte[])}. + * If value==nullptr, then this function will not read any data, but will + * still ensure that this key cannot be written to by outside of this + * transaction. + *

    + * If this transaction was created by an {@link OptimisticTransactionDB}, + * {@link #getForUpdate(ReadOptions, ColumnFamilyHandle, byte[], boolean)} + * could cause {@link #commit()} to fail. Otherwise, it could return any error + * that could be returned by + * {@link RocksDB#get(ColumnFamilyHandle, ReadOptions, byte[])}. + *

    + * If this transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * {@link Status.Code#MergeInProgress} if merge operations cannot be + * resolved. + * + * @param readOptions Read options. + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param key the key to retrieve the value for. + * @param exclusive true if the transaction should have exclusive access to + * the key, otherwise false for shared access. + * @param doValidate true if it should validate the snapshot before doing the read + * + * @return a byte array storing the value associated with the input key if + * any. null if it does not find the specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public byte[] getForUpdate(final ReadOptions readOptions, + final ColumnFamilyHandle columnFamilyHandle, final byte[] key, final boolean exclusive, + final boolean doValidate) throws RocksDBException { + assert (isOwningHandle()); + return getForUpdate(nativeHandle_, readOptions.nativeHandle_, key, key.length, + columnFamilyHandle.nativeHandle_, exclusive, doValidate); + } + + /** + * Same as + * {@link #getForUpdate(ReadOptions, ColumnFamilyHandle, byte[], boolean, boolean)} + * with doValidate=true. + * + * @param readOptions Read options. + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param key the key to retrieve the value for. + * @param exclusive true if the transaction should have exclusive access to + * the key, otherwise false for shared access. + * + * @return a byte array storing the value associated with the input key if + * any. null if it does not find the specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public byte[] getForUpdate(final ReadOptions readOptions, + final ColumnFamilyHandle columnFamilyHandle, final byte[] key, + final boolean exclusive) throws RocksDBException { + assert(isOwningHandle()); + return getForUpdate(nativeHandle_, readOptions.nativeHandle_, key, key.length, + columnFamilyHandle.nativeHandle_, exclusive, true /*doValidate*/); + } + + /** + * Read this key and ensure that this transaction will only + * be able to be committed if this key is not written outside this + * transaction after it has first been read (or after the snapshot if a + * snapshot is set in this transaction). The transaction behavior is the + * same regardless of whether the key exists or not. + *

    + * Note: Currently, this function will return Status::MergeInProgress + * if the most recent write to the queried key in this batch is a Merge. + *

    + * The values returned by this function are similar to + * {@link RocksDB#get(ReadOptions, byte[])}. + * If value==nullptr, then this function will not read any data, but will + * still ensure that this key cannot be written to by outside of this + * transaction. + *

    + * If this transaction was created on an {@link OptimisticTransactionDB}, + * {@link #getForUpdate(ReadOptions, ColumnFamilyHandle, byte[], boolean)} + * could cause {@link #commit()} to fail. Otherwise, it could return any error + * that could be returned by + * {@link RocksDB#get(ReadOptions, byte[])}. + *

    + * If this transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * {@link Status.Code#MergeInProgress} if merge operations cannot be + * resolved. + * + * @param readOptions Read options. + * @param key the key to retrieve the value for. + * @param exclusive true if the transaction should have exclusive access to + * the key, otherwise false for shared access. + * + * @return a byte array storing the value associated with the input key if + * any. null if it does not find the specified key. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public byte[] getForUpdate(final ReadOptions readOptions, final byte[] key, + final boolean exclusive) throws RocksDBException { + assert(isOwningHandle()); + return getForUpdate( + nativeHandle_, readOptions.nativeHandle_, key, key.length, exclusive, true /*doValidate*/); + } + + /** + * A multi-key version of + * {@link #getForUpdate(ReadOptions, ColumnFamilyHandle, byte[], boolean)}. + *

    + * + * @param readOptions Read options. + * @param columnFamilyHandles {@link org.rocksdb.ColumnFamilyHandle} + * instances + * @param keys the keys to retrieve the values for. + * + * @return Array of values, one for each key + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + @Deprecated + public byte[][] multiGetForUpdate(final ReadOptions readOptions, + final List columnFamilyHandles, final byte[][] keys) + throws RocksDBException { + assert(isOwningHandle()); + // Check if key size equals cfList size. If not a exception must be + // thrown. If not a Segmentation fault happens. + if (keys.length != columnFamilyHandles.size()){ + throw new IllegalArgumentException( + "For each key there must be a ColumnFamilyHandle."); + } + if(keys.length == 0) { + return new byte[0][0]; + } + final long[] cfHandles = new long[columnFamilyHandles.size()]; + for (int i = 0; i < columnFamilyHandles.size(); i++) { + cfHandles[i] = columnFamilyHandles.get(i).nativeHandle_; + } + return multiGetForUpdate(nativeHandle_, readOptions.nativeHandle_, + keys, cfHandles); + } + + /** + * A multi-key version of + * {@link #getForUpdate(ReadOptions, ColumnFamilyHandle, byte[], boolean)}. + *

    + * + * @param readOptions Read options. + * @param columnFamilyHandles {@link org.rocksdb.ColumnFamilyHandle} + * instances + * @param keys the keys to retrieve the values for. + * + * @return Array of values, one for each key + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public List multiGetForUpdateAsList(final ReadOptions readOptions, + final List columnFamilyHandles, final List keys) + throws RocksDBException { + assert (isOwningHandle()); + // Check if key size equals cfList size. If not a exception must be + // thrown. If not a Segmentation fault happens. + if (keys.size() != columnFamilyHandles.size()) { + throw new IllegalArgumentException("For each key there must be a ColumnFamilyHandle."); + } + if (keys.size() == 0) { + return new ArrayList<>(); + } + final byte[][] keysArray = keys.toArray(new byte[keys.size()][]); + + final long[] cfHandles = new long[columnFamilyHandles.size()]; + for (int i = 0; i < columnFamilyHandles.size(); i++) { + cfHandles[i] = columnFamilyHandles.get(i).nativeHandle_; + } + return Arrays.asList( + multiGetForUpdate(nativeHandle_, readOptions.nativeHandle_, keysArray, cfHandles)); + } + + /** + * A multi-key version of {@link #getForUpdate(ReadOptions, byte[], boolean)}. + *

    + * + * @param readOptions Read options. + * @param keys the keys to retrieve the values for. + * + * @return Array of values, one for each key + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + @Deprecated + public byte[][] multiGetForUpdate(final ReadOptions readOptions, final byte[][] keys) + throws RocksDBException { + assert(isOwningHandle()); + if(keys.length == 0) { + return new byte[0][0]; + } + + return multiGetForUpdate(nativeHandle_, + readOptions.nativeHandle_, keys); + } + + /** + * A multi-key version of {@link #getForUpdate(ReadOptions, byte[], boolean)}. + *

    + * + * @param readOptions Read options. + * @param keys the keys to retrieve the values for. + * + * @return List of values, one for each key + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public List multiGetForUpdateAsList( + final ReadOptions readOptions, final List keys) throws RocksDBException { + assert (isOwningHandle()); + if (keys.size() == 0) { + return new ArrayList<>(0); + } + + final byte[][] keysArray = keys.toArray(new byte[keys.size()][]); + + return Arrays.asList(multiGetForUpdate(nativeHandle_, readOptions.nativeHandle_, keysArray)); + } + + /** + * Returns an iterator that will iterate on all keys in the default + * column family including both keys in the DB and uncommitted keys in this + * transaction. + *

    + * Setting {@link ReadOptions#setSnapshot(Snapshot)} will affect what is read + * from the DB but will NOT change which keys are read from this transaction + * (the keys in this transaction do not yet belong to any snapshot and will be + * fetched regardless). + *

    + * Caller is responsible for deleting the returned Iterator. + *

    + * The returned iterator is only valid until {@link #commit()}, + * {@link #rollback()}, or {@link #rollbackToSavePoint()} is called. + * + * @param readOptions Read options. + * + * @return instance of iterator object. + */ + public RocksIterator getIterator(final ReadOptions readOptions) { + assert(isOwningHandle()); + return new RocksIterator(parent, getIterator(nativeHandle_, + readOptions.nativeHandle_)); + } + + /** + * Returns an iterator that will iterate on all keys in the column family + * specified by {@code columnFamilyHandle} including both keys in the DB + * and uncommitted keys in this transaction. + *

    + * Setting {@link ReadOptions#setSnapshot(Snapshot)} will affect what is read + * from the DB but will NOT change which keys are read from this transaction + * (the keys in this transaction do not yet belong to any snapshot and will be + * fetched regardless). + *

    + * Caller is responsible for calling {@link RocksIterator#close()} on + * the returned Iterator. + *

    + * The returned iterator is only valid until {@link #commit()}, + * {@link #rollback()}, or {@link #rollbackToSavePoint()} is called. + * + * @param readOptions Read options. + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * + * @return instance of iterator object. + */ + public RocksIterator getIterator(final ReadOptions readOptions, + final ColumnFamilyHandle columnFamilyHandle) { + assert(isOwningHandle()); + return new RocksIterator(parent, getIterator(nativeHandle_, + readOptions.nativeHandle_, columnFamilyHandle.nativeHandle_)); + } + + /** + * Similar to {@link RocksDB#put(ColumnFamilyHandle, byte[], byte[])}, but + * will also perform conflict checking on the keys be written. + *

    + * If this Transaction was created on an {@link OptimisticTransactionDB}, + * these functions should always succeed. + *

    + * If this Transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * + * @param columnFamilyHandle The column family to put the key/value into + * @param key the specified key to be inserted. + * @param value the value associated with the specified key. + * @param assumeTracked true when it is expected that the key is already + * tracked. More specifically, it means the the key was previous tracked + * in the same savepoint, with the same exclusive flag, and at a lower + * sequence number. If valid then it skips ValidateSnapshot, + * throws an error otherwise. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void put(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, + final byte[] value, final boolean assumeTracked) throws RocksDBException { + assert (isOwningHandle()); + put(nativeHandle_, key, key.length, value, value.length, + columnFamilyHandle.nativeHandle_, assumeTracked); + } + + /** + * Similar to {@link #put(ColumnFamilyHandle, byte[], byte[], boolean)} + * but with {@code assumeTracked = false}. + *

    + * Will also perform conflict checking on the keys be written. + *

    + * If this Transaction was created on an {@link OptimisticTransactionDB}, + * these functions should always succeed. + *

    + * If this Transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * + * @param columnFamilyHandle The column family to put the key/value into + * @param key the specified key to be inserted. + * @param value the value associated with the specified key. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void put(final ColumnFamilyHandle columnFamilyHandle, final byte[] key, + final byte[] value) throws RocksDBException { + assert(isOwningHandle()); + put(nativeHandle_, key, key.length, value, value.length, + columnFamilyHandle.nativeHandle_, false); + } + + /** + * Similar to {@link RocksDB#put(byte[], byte[])}, but + * will also perform conflict checking on the keys be written. + *

    + * If this Transaction was created on an {@link OptimisticTransactionDB}, + * these functions should always succeed. + *

    + * If this Transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * + * @param key the specified key to be inserted. + * @param value the value associated with the specified key. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void put(final byte[] key, final byte[] value) + throws RocksDBException { + assert(isOwningHandle()); + put(nativeHandle_, key, key.length, value, value.length); + } + + //TODO(AR) refactor if we implement org.rocksdb.SliceParts in future + /** + * Similar to {@link #put(ColumnFamilyHandle, byte[], byte[])} but allows + * you to specify the key and value in several parts that will be + * concatenated together. + * + * @param columnFamilyHandle The column family to put the key/value into + * @param keyParts the specified key to be inserted. + * @param valueParts the value associated with the specified key. + * @param assumeTracked true when it is expected that the key is already + * tracked. More specifically, it means the the key was previous tracked + * in the same savepoint, with the same exclusive flag, and at a lower + * sequence number. If valid then it skips ValidateSnapshot, + * throws an error otherwise. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void put(final ColumnFamilyHandle columnFamilyHandle, + final byte[][] keyParts, final byte[][] valueParts, + final boolean assumeTracked) throws RocksDBException { + assert (isOwningHandle()); + put(nativeHandle_, keyParts, keyParts.length, valueParts, valueParts.length, + columnFamilyHandle.nativeHandle_, assumeTracked); + } + + /** + * Similar to {@link #put(ColumnFamilyHandle, byte[][], byte[][], boolean)} + * but with with {@code assumeTracked = false}. + *

    + * Allows you to specify the key and value in several parts that will be + * concatenated together. + * + * @param columnFamilyHandle The column family to put the key/value into + * @param keyParts the specified key to be inserted. + * @param valueParts the value associated with the specified key. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void put(final ColumnFamilyHandle columnFamilyHandle, + final byte[][] keyParts, final byte[][] valueParts) + throws RocksDBException { + assert(isOwningHandle()); + put(nativeHandle_, keyParts, keyParts.length, valueParts, valueParts.length, + columnFamilyHandle.nativeHandle_, false); + } + + //TODO(AR) refactor if we implement org.rocksdb.SliceParts in future + /** + * Similar to {@link #put(byte[], byte[])} but allows + * you to specify the key and value in several parts that will be + * concatenated together + * + * @param keyParts the specified key to be inserted. + * @param valueParts the value associated with the specified key. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void put(final byte[][] keyParts, final byte[][] valueParts) + throws RocksDBException { + assert(isOwningHandle()); + put(nativeHandle_, keyParts, keyParts.length, valueParts, + valueParts.length); + } + + /** + * Similar to {@link RocksDB#merge(ColumnFamilyHandle, byte[], byte[])}, but + * will also perform conflict checking on the keys be written. + *

    + * If this Transaction was created on an {@link OptimisticTransactionDB}, + * these functions should always succeed. + *

    + * If this Transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * + * @param columnFamilyHandle The column family to merge the key/value into + * @param key the specified key to be merged. + * @param value the value associated with the specified key. + * @param assumeTracked true when it is expected that the key is already + * tracked. More specifically, it means the the key was previous tracked + * in the same savepoint, with the same exclusive flag, and at a lower + * sequence number. If valid then it skips ValidateSnapshot, + * throws an error otherwise. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void merge(final ColumnFamilyHandle columnFamilyHandle, + final byte[] key, final byte[] value, final boolean assumeTracked) + throws RocksDBException { + assert (isOwningHandle()); + merge(nativeHandle_, key, key.length, value, value.length, + columnFamilyHandle.nativeHandle_, assumeTracked); + } + + /** + * Similar to {@link #merge(ColumnFamilyHandle, byte[], byte[], boolean)} + * but with {@code assumeTracked = false}. + *

    + * Will also perform conflict checking on the keys be written. + *

    + * If this Transaction was created on an {@link OptimisticTransactionDB}, + * these functions should always succeed. + *

    + * If this Transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * + * @param columnFamilyHandle The column family to merge the key/value into + * @param key the specified key to be merged. + * @param value the value associated with the specified key. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void merge(final ColumnFamilyHandle columnFamilyHandle, + final byte[] key, final byte[] value) throws RocksDBException { + assert(isOwningHandle()); + merge(nativeHandle_, key, key.length, value, value.length, + columnFamilyHandle.nativeHandle_, false); + } + + /** + * Similar to {@link RocksDB#merge(byte[], byte[])}, but + * will also perform conflict checking on the keys be written. + *

    + * If this Transaction was created on an {@link OptimisticTransactionDB}, + * these functions should always succeed. + *

    + * If this Transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * + * @param key the specified key to be merged. + * @param value the value associated with the specified key. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void merge(final byte[] key, final byte[] value) + throws RocksDBException { + assert(isOwningHandle()); + merge(nativeHandle_, key, key.length, value, value.length); + } + + /** + * Similar to {@link RocksDB#delete(ColumnFamilyHandle, byte[])}, but + * will also perform conflict checking on the keys be written. + *

    + * If this Transaction was created on an {@link OptimisticTransactionDB}, + * these functions should always succeed. + *

    + * If this Transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * + * @param columnFamilyHandle The column family to delete the key/value from + * @param key the specified key to be deleted. + * @param assumeTracked true when it is expected that the key is already + * tracked. More specifically, it means the the key was previous tracked + * in the same savepoint, with the same exclusive flag, and at a lower + * sequence number. If valid then it skips ValidateSnapshot, + * throws an error otherwise. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void delete(final ColumnFamilyHandle columnFamilyHandle, + final byte[] key, final boolean assumeTracked) throws RocksDBException { + assert (isOwningHandle()); + delete(nativeHandle_, key, key.length, columnFamilyHandle.nativeHandle_, + assumeTracked); + } + + /** + * Similar to {@link #delete(ColumnFamilyHandle, byte[], boolean)} + * but with {@code assumeTracked = false}. + *

    + * Will also perform conflict checking on the keys be written. + *

    + * If this Transaction was created on an {@link OptimisticTransactionDB}, + * these functions should always succeed. + *

    + * If this Transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * + * @param columnFamilyHandle The column family to delete the key/value from + * @param key the specified key to be deleted. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void delete(final ColumnFamilyHandle columnFamilyHandle, + final byte[] key) throws RocksDBException { + assert(isOwningHandle()); + delete(nativeHandle_, key, key.length, columnFamilyHandle.nativeHandle_, + /*assumeTracked*/ false); + } + + /** + * Similar to {@link RocksDB#delete(byte[])}, but + * will also perform conflict checking on the keys be written. + *

    + * If this Transaction was created on an {@link OptimisticTransactionDB}, + * these functions should always succeed. + *

    + * If this Transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * + * @param key the specified key to be deleted. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void delete(final byte[] key) throws RocksDBException { + assert(isOwningHandle()); + delete(nativeHandle_, key, key.length); + } + + //TODO(AR) refactor if we implement org.rocksdb.SliceParts in future + /** + * Similar to {@link #delete(ColumnFamilyHandle, byte[])} but allows + * you to specify the key in several parts that will be + * concatenated together. + * + * @param columnFamilyHandle The column family to delete the key/value from + * @param keyParts the specified key to be deleted. + * @param assumeTracked true when it is expected that the key is already + * tracked. More specifically, it means the the key was previous tracked + * in the same savepoint, with the same exclusive flag, and at a lower + * sequence number. If valid then it skips ValidateSnapshot, + * throws an error otherwise. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void delete(final ColumnFamilyHandle columnFamilyHandle, + final byte[][] keyParts, final boolean assumeTracked) + throws RocksDBException { + assert (isOwningHandle()); + delete(nativeHandle_, keyParts, keyParts.length, + columnFamilyHandle.nativeHandle_, assumeTracked); + } + + /** + * Similar to{@link #delete(ColumnFamilyHandle, byte[][], boolean)} + * but with {@code assumeTracked = false}. + *

    + * Allows you to specify the key in several parts that will be + * concatenated together. + * + * @param columnFamilyHandle The column family to delete the key/value from + * @param keyParts the specified key to be deleted. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void delete(final ColumnFamilyHandle columnFamilyHandle, + final byte[][] keyParts) throws RocksDBException { + assert(isOwningHandle()); + delete(nativeHandle_, keyParts, keyParts.length, + columnFamilyHandle.nativeHandle_, false); + } + + //TODO(AR) refactor if we implement org.rocksdb.SliceParts in future + /** + * Similar to {@link #delete(byte[])} but allows + * you to specify key the in several parts that will be + * concatenated together. + * + * @param keyParts the specified key to be deleted + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void delete(final byte[][] keyParts) throws RocksDBException { + assert(isOwningHandle()); + delete(nativeHandle_, keyParts, keyParts.length); + } + + /** + * Similar to {@link RocksDB#singleDelete(ColumnFamilyHandle, byte[])}, but + * will also perform conflict checking on the keys be written. + *

    + * If this Transaction was created on an {@link OptimisticTransactionDB}, + * these functions should always succeed. + *

    + * If this Transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * + * @param columnFamilyHandle The column family to delete the key/value from + * @param key the specified key to be deleted. + * @param assumeTracked true when it is expected that the key is already + * tracked. More specifically, it means the key was previously tracked + * in the same savepoint, with the same exclusive flag, and at a lower + * sequence number. If valid then it skips ValidateSnapshot, + * throws an error otherwise. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + @Experimental("Performance optimization for a very specific workload") + public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, + final byte[] key, final boolean assumeTracked) throws RocksDBException { + assert (isOwningHandle()); + singleDelete(nativeHandle_, key, key.length, + columnFamilyHandle.nativeHandle_, assumeTracked); + } + + /** + * Similar to {@link #singleDelete(ColumnFamilyHandle, byte[], boolean)} + * but with {@code assumeTracked = false}. + *

    + * will also perform conflict checking on the keys be written. + *

    + * If this Transaction was created on an {@link OptimisticTransactionDB}, + * these functions should always succeed. + *

    + * If this Transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * + * @param columnFamilyHandle The column family to delete the key/value from + * @param key the specified key to be deleted. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + @Experimental("Performance optimization for a very specific workload") + public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, + final byte[] key) throws RocksDBException { + assert(isOwningHandle()); + singleDelete(nativeHandle_, key, key.length, + columnFamilyHandle.nativeHandle_, false); + } + + /** + * Similar to {@link RocksDB#singleDelete(byte[])}, but + * will also perform conflict checking on the keys be written. + *

    + * If this Transaction was created on an {@link OptimisticTransactionDB}, + * these functions should always succeed. + *

    + * If this Transaction was created on a {@link TransactionDB}, an + * {@link RocksDBException} may be thrown with an accompanying {@link Status} + * when: + * {@link Status.Code#Busy} if there is a write conflict, + * {@link Status.Code#TimedOut} if a lock could not be acquired, + * {@link Status.Code#TryAgain} if the memtable history size is not large + * enough. See + * {@link ColumnFamilyOptions#maxWriteBufferNumberToMaintain()} + * + * @param key the specified key to be deleted. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + @Experimental("Performance optimization for a very specific workload") + public void singleDelete(final byte[] key) throws RocksDBException { + assert(isOwningHandle()); + singleDelete(nativeHandle_, key, key.length); + } + + //TODO(AR) refactor if we implement org.rocksdb.SliceParts in future + /** + * Similar to {@link #singleDelete(ColumnFamilyHandle, byte[])} but allows + * you to specify the key in several parts that will be + * concatenated together. + * + * @param columnFamilyHandle The column family to delete the key/value from + * @param keyParts the specified key to be deleted. + * @param assumeTracked true when it is expected that the key is already + * tracked. More specifically, it means the key was previously tracked + * in the same savepoint, with the same exclusive flag, and at a lower + * sequence number. If valid then it skips ValidateSnapshot, + * throws an error otherwise. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + @Experimental("Performance optimization for a very specific workload") + public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, + final byte[][] keyParts, final boolean assumeTracked) + throws RocksDBException { + assert (isOwningHandle()); + singleDelete(nativeHandle_, keyParts, keyParts.length, + columnFamilyHandle.nativeHandle_, assumeTracked); + } + + /** + * Similar to{@link #singleDelete(ColumnFamilyHandle, byte[][], boolean)} + * but with {@code assumeTracked = false}. + *

    + * Allows you to specify the key in several parts that will be + * concatenated together. + * + * @param columnFamilyHandle The column family to delete the key/value from + * @param keyParts the specified key to be deleted. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + @Experimental("Performance optimization for a very specific workload") + public void singleDelete(final ColumnFamilyHandle columnFamilyHandle, + final byte[][] keyParts) throws RocksDBException { + assert(isOwningHandle()); + singleDelete(nativeHandle_, keyParts, keyParts.length, + columnFamilyHandle.nativeHandle_, false); + } + + //TODO(AR) refactor if we implement org.rocksdb.SliceParts in future + /** + * Similar to {@link #singleDelete(byte[])} but allows + * you to specify the key in several parts that will be + * concatenated together. + * + * @param keyParts the specified key to be deleted. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + @Experimental("Performance optimization for a very specific workload") + public void singleDelete(final byte[][] keyParts) throws RocksDBException { + assert(isOwningHandle()); + singleDelete(nativeHandle_, keyParts, keyParts.length); + } + + /** + * Similar to {@link RocksDB#put(ColumnFamilyHandle, byte[], byte[])}, + * but operates on the transactions write batch. This write will only happen + * if this transaction gets committed successfully. + *

    + * Unlike {@link #put(ColumnFamilyHandle, byte[], byte[])} no conflict + * checking will be performed for this key. + *

    + * If this Transaction was created on a {@link TransactionDB}, this function + * will still acquire locks necessary to make sure this write doesn't cause + * conflicts in other transactions; This may cause a {@link RocksDBException} + * with associated {@link Status.Code#Busy}. + * + * @param columnFamilyHandle The column family to put the key/value into + * @param key the specified key to be inserted. + * @param value the value associated with the specified key. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void putUntracked(final ColumnFamilyHandle columnFamilyHandle, + final byte[] key, final byte[] value) throws RocksDBException { + assert(isOwningHandle()); + putUntracked(nativeHandle_, key, key.length, value, value.length, + columnFamilyHandle.nativeHandle_); + } + + /** + * Similar to {@link RocksDB#put(byte[], byte[])}, + * but operates on the transactions write batch. This write will only happen + * if this transaction gets committed successfully. + *

    + * Unlike {@link #put(byte[], byte[])} no conflict + * checking will be performed for this key. + *

    + * If this Transaction was created on a {@link TransactionDB}, this function + * will still acquire locks necessary to make sure this write doesn't cause + * conflicts in other transactions; This may cause a {@link RocksDBException} + * with associated {@link Status.Code#Busy}. + * + * @param key the specified key to be inserted. + * @param value the value associated with the specified key. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void putUntracked(final byte[] key, final byte[] value) + throws RocksDBException { + assert(isOwningHandle()); + putUntracked(nativeHandle_, key, key.length, value, value.length); + } + + //TODO(AR) refactor if we implement org.rocksdb.SliceParts in future + /** + * Similar to {@link #putUntracked(ColumnFamilyHandle, byte[], byte[])} but + * allows you to specify the key and value in several parts that will be + * concatenated together. + * + * @param columnFamilyHandle The column family to put the key/value into + * @param keyParts the specified key to be inserted. + * @param valueParts the value associated with the specified key. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void putUntracked(final ColumnFamilyHandle columnFamilyHandle, + final byte[][] keyParts, final byte[][] valueParts) + throws RocksDBException { + assert(isOwningHandle()); + putUntracked(nativeHandle_, keyParts, keyParts.length, valueParts, + valueParts.length, columnFamilyHandle.nativeHandle_); + } + + //TODO(AR) refactor if we implement org.rocksdb.SliceParts in future + /** + * Similar to {@link #putUntracked(byte[], byte[])} but + * allows you to specify the key and value in several parts that will be + * concatenated together. + * + * @param keyParts the specified key to be inserted. + * @param valueParts the value associated with the specified key. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void putUntracked(final byte[][] keyParts, final byte[][] valueParts) + throws RocksDBException { + assert(isOwningHandle()); + putUntracked(nativeHandle_, keyParts, keyParts.length, valueParts, + valueParts.length); + } + + /** + * Similar to {@link RocksDB#merge(ColumnFamilyHandle, byte[], byte[])}, + * but operates on the transactions write batch. This write will only happen + * if this transaction gets committed successfully. + *

    + * Unlike {@link #merge(ColumnFamilyHandle, byte[], byte[])} no conflict + * checking will be performed for this key. + *

    + * If this Transaction was created on a {@link TransactionDB}, this function + * will still acquire locks necessary to make sure this write doesn't cause + * conflicts in other transactions; This may cause a {@link RocksDBException} + * with associated {@link Status.Code#Busy}. + * + * @param columnFamilyHandle The column family to merge the key/value into + * @param key the specified key to be merged. + * @param value the value associated with the specified key. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void mergeUntracked(final ColumnFamilyHandle columnFamilyHandle, + final byte[] key, final byte[] value) throws RocksDBException { + mergeUntracked(nativeHandle_, key, key.length, value, value.length, + columnFamilyHandle.nativeHandle_); + } + + /** + * Similar to {@link RocksDB#merge(byte[], byte[])}, + * but operates on the transactions write batch. This write will only happen + * if this transaction gets committed successfully. + *

    + * Unlike {@link #merge(byte[], byte[])} no conflict + * checking will be performed for this key. + *

    + * If this Transaction was created on a {@link TransactionDB}, this function + * will still acquire locks necessary to make sure this write doesn't cause + * conflicts in other transactions; This may cause a {@link RocksDBException} + * with associated {@link Status.Code#Busy}. + * + * @param key the specified key to be merged. + * @param value the value associated with the specified key. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void mergeUntracked(final byte[] key, final byte[] value) + throws RocksDBException { + assert(isOwningHandle()); + mergeUntracked(nativeHandle_, key, key.length, value, value.length); + } + + /** + * Similar to {@link RocksDB#delete(ColumnFamilyHandle, byte[])}, + * but operates on the transactions write batch. This write will only happen + * if this transaction gets committed successfully. + *

    + * Unlike {@link #delete(ColumnFamilyHandle, byte[])} no conflict + * checking will be performed for this key. + *

    + * If this Transaction was created on a {@link TransactionDB}, this function + * will still acquire locks necessary to make sure this write doesn't cause + * conflicts in other transactions; This may cause a {@link RocksDBException} + * with associated {@link Status.Code#Busy}. + * + * @param columnFamilyHandle The column family to delete the key/value from + * @param key the specified key to be deleted. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void deleteUntracked(final ColumnFamilyHandle columnFamilyHandle, + final byte[] key) throws RocksDBException { + assert(isOwningHandle()); + deleteUntracked(nativeHandle_, key, key.length, + columnFamilyHandle.nativeHandle_); + } + + /** + * Similar to {@link RocksDB#delete(byte[])}, + * but operates on the transactions write batch. This write will only happen + * if this transaction gets committed successfully. + *

    + * Unlike {@link #delete(byte[])} no conflict + * checking will be performed for this key. + *

    + * If this Transaction was created on a {@link TransactionDB}, this function + * will still acquire locks necessary to make sure this write doesn't cause + * conflicts in other transactions; This may cause a {@link RocksDBException} + * with associated {@link Status.Code#Busy}. + * + * @param key the specified key to be deleted. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void deleteUntracked(final byte[] key) throws RocksDBException { + assert(isOwningHandle()); + deleteUntracked(nativeHandle_, key, key.length); + } + + //TODO(AR) refactor if we implement org.rocksdb.SliceParts in future + /** + * Similar to {@link #deleteUntracked(ColumnFamilyHandle, byte[])} but allows + * you to specify the key in several parts that will be + * concatenated together. + * + * @param columnFamilyHandle The column family to delete the key/value from + * @param keyParts the specified key to be deleted. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void deleteUntracked(final ColumnFamilyHandle columnFamilyHandle, + final byte[][] keyParts) throws RocksDBException { + assert(isOwningHandle()); + deleteUntracked(nativeHandle_, keyParts, keyParts.length, + columnFamilyHandle.nativeHandle_); + } + + //TODO(AR) refactor if we implement org.rocksdb.SliceParts in future + /** + * Similar to {@link #deleteUntracked(byte[])} but allows + * you to specify the key in several parts that will be + * concatenated together. + * + * @param keyParts the specified key to be deleted. + * + * @throws RocksDBException when one of the TransactionalDB conditions + * described above occurs, or in the case of an unexpected error + */ + public void deleteUntracked(final byte[][] keyParts) throws RocksDBException { + assert(isOwningHandle()); + deleteUntracked(nativeHandle_, keyParts, keyParts.length); + } + + /** + * Similar to {@link WriteBatch#putLogData(byte[])} + * + * @param blob binary object to be inserted + */ + public void putLogData(final byte[] blob) { + assert(isOwningHandle()); + putLogData(nativeHandle_, blob, blob.length); + } + + /** + * By default, all put/merge/delete operations will be indexed in the + * transaction so that get/getForUpdate/getIterator can search for these + * keys. + *

    + * If the caller does not want to fetch the keys about to be written, + * they may want to avoid indexing as a performance optimization. + * Calling {@code #disableIndexing()} will turn off indexing for all future + * put/merge/delete operations until {@link #enableIndexing()} is called. + *

    + * If a key is put/merge/deleted after {@code #disableIndexing()} is called + * and then is fetched via get/getForUpdate/getIterator, the result of the + * fetch is undefined. + */ + public void disableIndexing() { + assert(isOwningHandle()); + disableIndexing(nativeHandle_); + } + + /** + * Re-enables indexing after a previous call to {@link #disableIndexing()} + */ + public void enableIndexing() { + assert(isOwningHandle()); + enableIndexing(nativeHandle_); + } + + /** + * Returns the number of distinct Keys being tracked by this transaction. + * If this transaction was created by a {@link TransactionDB}, this is the + * number of keys that are currently locked by this transaction. + * If this transaction was created by an {@link OptimisticTransactionDB}, + * this is the number of keys that need to be checked for conflicts at commit + * time. + * + * @return the number of distinct Keys being tracked by this transaction + */ + public long getNumKeys() { + assert(isOwningHandle()); + return getNumKeys(nativeHandle_); + } + + /** + * Returns the number of puts that have been applied to this + * transaction so far. + * + * @return the number of puts that have been applied to this transaction + */ + public long getNumPuts() { + assert(isOwningHandle()); + return getNumPuts(nativeHandle_); + } + + /** + * Returns the number of deletes that have been applied to this + * transaction so far. + * + * @return the number of deletes that have been applied to this transaction + */ + public long getNumDeletes() { + assert(isOwningHandle()); + return getNumDeletes(nativeHandle_); + } + + /** + * Returns the number of merges that have been applied to this + * transaction so far. + * + * @return the number of merges that have been applied to this transaction + */ + public long getNumMerges() { + assert(isOwningHandle()); + return getNumMerges(nativeHandle_); + } + + /** + * Returns the elapsed time in milliseconds since this Transaction began. + * + * @return the elapsed time in milliseconds since this transaction began. + */ + public long getElapsedTime() { + assert(isOwningHandle()); + return getElapsedTime(nativeHandle_); + } + + /** + * Fetch the underlying write batch that contains all pending changes to be + * committed. + *

    + * Note: You should not write or delete anything from the batch directly and + * should only use the functions in the {@link Transaction} class to + * write to this transaction. + * + * @return The write batch + */ + public WriteBatchWithIndex getWriteBatch() { + assert(isOwningHandle()); + return new WriteBatchWithIndex(getWriteBatch(nativeHandle_)); + } + + /** + * Change the value of {@link TransactionOptions#getLockTimeout()} + * (in milliseconds) for this transaction. + *

    + * Has no effect on OptimisticTransactions. + * + * @param lockTimeout the timeout (in milliseconds) for locks used by this + * transaction. + */ + public void setLockTimeout(final long lockTimeout) { + assert(isOwningHandle()); + setLockTimeout(nativeHandle_, lockTimeout); + } + + /** + * Return the WriteOptions that will be used during {@link #commit()}. + * + * @return the WriteOptions that will be used + */ + public WriteOptions getWriteOptions() { + assert(isOwningHandle()); + return new WriteOptions(getWriteOptions(nativeHandle_)); + } + + /** + * Reset the WriteOptions that will be used during {@link #commit()}. + * + * @param writeOptions The new WriteOptions + */ + public void setWriteOptions(final WriteOptions writeOptions) { + assert(isOwningHandle()); + setWriteOptions(nativeHandle_, writeOptions.nativeHandle_); + } + + /** + * If this key was previously fetched in this transaction using + * {@link #getForUpdate(ReadOptions, ColumnFamilyHandle, byte[], boolean)}/ + * {@link #multiGetForUpdate(ReadOptions, List, byte[][])}, calling + * {@code #undoGetForUpdate(ColumnFamilyHandle, byte[])} will tell + * the transaction that it no longer needs to do any conflict checking + * for this key. + *

    + * If a key has been fetched N times via + * {@link #getForUpdate(ReadOptions, ColumnFamilyHandle, byte[], boolean)}/ + * {@link #multiGetForUpdate(ReadOptions, List, byte[][])}, then + * {@code #undoGetForUpdate(ColumnFamilyHandle, byte[])} will only have an + * effect if it is also called N times. If this key has been written to in + * this transaction, {@code #undoGetForUpdate(ColumnFamilyHandle, byte[])} + * will have no effect. + *

    + * If {@link #setSavePoint()} has been called after the + * {@link #getForUpdate(ReadOptions, ColumnFamilyHandle, byte[], boolean)}, + * {@code #undoGetForUpdate(ColumnFamilyHandle, byte[])} will not have any + * effect. + *

    + * If this Transaction was created by an {@link OptimisticTransactionDB}, + * calling {@code #undoGetForUpdate(ColumnFamilyHandle, byte[])} can affect + * whether this key is conflict checked at commit time. + * If this Transaction was created by a {@link TransactionDB}, + * calling {@code #undoGetForUpdate(ColumnFamilyHandle, byte[])} may release + * any held locks for this key. + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param key the key to retrieve the value for. + */ + public void undoGetForUpdate(final ColumnFamilyHandle columnFamilyHandle, + final byte[] key) { + assert(isOwningHandle()); + undoGetForUpdate(nativeHandle_, key, key.length, columnFamilyHandle.nativeHandle_); + } + + /** + * If this key was previously fetched in this transaction using + * {@link #getForUpdate(ReadOptions, byte[], boolean)}/ + * {@link #multiGetForUpdate(ReadOptions, List, byte[][])}, calling + * {@code #undoGetForUpdate(byte[])} will tell + * the transaction that it no longer needs to do any conflict checking + * for this key. + *

    + * If a key has been fetched N times via + * {@link #getForUpdate(ReadOptions, byte[], boolean)}/ + * {@link #multiGetForUpdate(ReadOptions, List, byte[][])}, then + * {@code #undoGetForUpdate(byte[])} will only have an + * effect if it is also called N times. If this key has been written to in + * this transaction, {@code #undoGetForUpdate(byte[])} + * will have no effect. + *

    + * If {@link #setSavePoint()} has been called after the + * {@link #getForUpdate(ReadOptions, byte[], boolean)}, + * {@code #undoGetForUpdate(byte[])} will not have any + * effect. + *

    + * If this Transaction was created by an {@link OptimisticTransactionDB}, + * calling {@code #undoGetForUpdate(byte[])} can affect + * whether this key is conflict checked at commit time. + * If this Transaction was created by a {@link TransactionDB}, + * calling {@code #undoGetForUpdate(byte[])} may release + * any held locks for this key. + * + * @param key the key to retrieve the value for. + */ + public void undoGetForUpdate(final byte[] key) { + assert(isOwningHandle()); + undoGetForUpdate(nativeHandle_, key, key.length); + } + + /** + * Adds the keys from the WriteBatch to the transaction + * + * @param writeBatch The write batch to read from + * + * @throws RocksDBException if an error occurs whilst rebuilding from the + * write batch. + */ + public void rebuildFromWriteBatch(final WriteBatch writeBatch) + throws RocksDBException { + assert(isOwningHandle()); + rebuildFromWriteBatch(nativeHandle_, writeBatch.nativeHandle_); + } + + /** + * Get the Commit time Write Batch. + * + * @return the commit time write batch. + */ + public WriteBatch getCommitTimeWriteBatch() { + assert(isOwningHandle()); + return new WriteBatch(getCommitTimeWriteBatch(nativeHandle_)); + } + + /** + * Set the log number. + * + * @param logNumber the log number + */ + public void setLogNumber(final long logNumber) { + assert(isOwningHandle()); + setLogNumber(nativeHandle_, logNumber); + } + + /** + * Get the log number. + * + * @return the log number + */ + public long getLogNumber() { + assert(isOwningHandle()); + return getLogNumber(nativeHandle_); + } + + /** + * Set the name of the transaction. + * + * @param transactionName the name of the transaction + * + * @throws RocksDBException if an error occurs when setting the transaction + * name. + */ + public void setName(final String transactionName) throws RocksDBException { + assert(isOwningHandle()); + setName(nativeHandle_, transactionName); + } + + /** + * Get the name of the transaction. + * + * @return the name of the transaction + */ + public String getName() { + assert(isOwningHandle()); + return getName(nativeHandle_); + } + + /** + * Get the ID of the transaction. + * + * @return the ID of the transaction. + */ + public long getID() { + assert(isOwningHandle()); + return getID(nativeHandle_); + } + + /** + * Determine if a deadlock has been detected. + * + * @return true if a deadlock has been detected. + */ + public boolean isDeadlockDetect() { + assert(isOwningHandle()); + return isDeadlockDetect(nativeHandle_); + } + + /** + * Get the list of waiting transactions. + * + * @return The list of waiting transactions. + */ + public WaitingTransactions getWaitingTxns() { + assert(isOwningHandle()); + return getWaitingTxns(nativeHandle_); + } + + /** + * Get the execution status of the transaction. + *

    + * NOTE: The execution status of an Optimistic Transaction + * never changes. This is only useful for non-optimistic transactions! + * + * @return The execution status of the transaction + */ + public TransactionState getState() { + assert(isOwningHandle()); + return TransactionState.getTransactionState( + getState(nativeHandle_)); + } + + /** + * The globally unique id with which the transaction is identified. This id + * might or might not be set depending on the implementation. Similarly the + * implementation decides the point in lifetime of a transaction at which it + * assigns the id. Although currently it is the case, the id is not guaranteed + * to remain the same across restarts. + * + * @return the transaction id. + */ + @Experimental("NOTE: Experimental feature") + public long getId() { + assert(isOwningHandle()); + return getId(nativeHandle_); + } + + public enum TransactionState { + STARTED((byte)0), + AWAITING_PREPARE((byte)1), + PREPARED((byte)2), + AWAITING_COMMIT((byte)3), + COMMITTED((byte)4), + AWAITING_ROLLBACK((byte)5), + ROLLEDBACK((byte)6), + LOCKS_STOLEN((byte)7); + + /* + * Keep old misspelled variable as alias + * Tip from https://stackoverflow.com/a/37092410/454544 + */ + public static final TransactionState COMMITED = COMMITTED; + + private final byte value; + + TransactionState(final byte value) { + this.value = value; + } + + /** + * Get TransactionState by byte value. + * + * @param value byte representation of TransactionState. + * + * @return {@link org.rocksdb.Transaction.TransactionState} instance or null. + * @throws java.lang.IllegalArgumentException if an invalid + * value is provided. + */ + public static TransactionState getTransactionState(final byte value) { + for (final TransactionState transactionState : TransactionState.values()) { + if (transactionState.value == value){ + return transactionState; + } + } + throw new IllegalArgumentException( + "Illegal value provided for TransactionState."); + } + } + + /** + * Called from C++ native method {@link #getWaitingTxns(long)} + * to construct a WaitingTransactions object. + * + * @param columnFamilyId The id of the {@link ColumnFamilyHandle} + * @param key The key + * @param transactionIds The transaction ids + * + * @return The waiting transactions + */ + private WaitingTransactions newWaitingTransactions( + final long columnFamilyId, final String key, + final long[] transactionIds) { + return new WaitingTransactions(columnFamilyId, key, transactionIds); + } + + public static class WaitingTransactions { + private final long columnFamilyId; + private final String key; + private final long[] transactionIds; + + private WaitingTransactions(final long columnFamilyId, final String key, + final long[] transactionIds) { + this.columnFamilyId = columnFamilyId; + this.key = key; + this.transactionIds = transactionIds; + } + + /** + * Get the Column Family ID. + * + * @return The column family ID + */ + public long getColumnFamilyId() { + return columnFamilyId; + } + + /** + * Get the key on which the transactions are waiting. + * + * @return The key + */ + public String getKey() { + return key; + } + + /** + * Get the IDs of the waiting transactions. + * + * @return The IDs of the waiting transactions + */ + public long[] getTransactionIds() { + return transactionIds; + } + } + + private native void setSnapshot(final long handle); + private native void setSnapshotOnNextOperation(final long handle); + private native void setSnapshotOnNextOperation(final long handle, + final long transactionNotifierHandle); + private native long getSnapshot(final long handle); + private native void clearSnapshot(final long handle); + private native void prepare(final long handle) throws RocksDBException; + private native void commit(final long handle) throws RocksDBException; + private native void rollback(final long handle) throws RocksDBException; + private native void setSavePoint(final long handle) throws RocksDBException; + private native void rollbackToSavePoint(final long handle) + throws RocksDBException; + private native byte[] get(final long handle, final long readOptionsHandle, final byte[] key, + final int keyLength, final long columnFamilyHandle) throws RocksDBException; + private native byte[] get(final long handle, final long readOptionsHandle, final byte[] key, + final int keyLen) throws RocksDBException; + private native byte[][] multiGet(final long handle, + final long readOptionsHandle, final byte[][] keys, + final long[] columnFamilyHandles) throws RocksDBException; + private native byte[][] multiGet(final long handle, + final long readOptionsHandle, final byte[][] keys) + throws RocksDBException; + private native byte[] getForUpdate(final long handle, final long readOptionsHandle, + final byte[] key, final int keyLength, final long columnFamilyHandle, final boolean exclusive, + final boolean doValidate) throws RocksDBException; + private native byte[] getForUpdate(final long handle, final long readOptionsHandle, + final byte[] key, final int keyLen, final boolean exclusive, final boolean doValidate) + throws RocksDBException; + private native byte[][] multiGetForUpdate(final long handle, + final long readOptionsHandle, final byte[][] keys, + final long[] columnFamilyHandles) throws RocksDBException; + private native byte[][] multiGetForUpdate(final long handle, + final long readOptionsHandle, final byte[][] keys) + throws RocksDBException; + private native long getIterator(final long handle, + final long readOptionsHandle); + private native long getIterator(final long handle, + final long readOptionsHandle, final long columnFamilyHandle); + private native void put(final long handle, final byte[] key, final int keyLength, + final byte[] value, final int valueLength, final long columnFamilyHandle, + final boolean assumeTracked) throws RocksDBException; + private native void put(final long handle, final byte[] key, + final int keyLength, final byte[] value, final int valueLength) + throws RocksDBException; + private native void put(final long handle, final byte[][] keys, final int keysLength, + final byte[][] values, final int valuesLength, final long columnFamilyHandle, + final boolean assumeTracked) throws RocksDBException; + private native void put(final long handle, final byte[][] keys, + final int keysLength, final byte[][] values, final int valuesLength) + throws RocksDBException; + private native void merge(final long handle, final byte[] key, final int keyLength, + final byte[] value, final int valueLength, final long columnFamilyHandle, + final boolean assumeTracked) throws RocksDBException; + private native void merge(final long handle, final byte[] key, + final int keyLength, final byte[] value, final int valueLength) + throws RocksDBException; + private native void delete(final long handle, final byte[] key, final int keyLength, + final long columnFamilyHandle, final boolean assumeTracked) throws RocksDBException; + private native void delete(final long handle, final byte[] key, + final int keyLength) throws RocksDBException; + private native void delete(final long handle, final byte[][] keys, final int keysLength, + final long columnFamilyHandle, final boolean assumeTracked) throws RocksDBException; + private native void delete(final long handle, final byte[][] keys, + final int keysLength) throws RocksDBException; + private native void singleDelete(final long handle, final byte[] key, final int keyLength, + final long columnFamilyHandle, final boolean assumeTracked) throws RocksDBException; + private native void singleDelete(final long handle, final byte[] key, + final int keyLength) throws RocksDBException; + private native void singleDelete(final long handle, final byte[][] keys, final int keysLength, + final long columnFamilyHandle, final boolean assumeTracked) throws RocksDBException; + private native void singleDelete(final long handle, final byte[][] keys, + final int keysLength) throws RocksDBException; + private native void putUntracked(final long handle, final byte[] key, + final int keyLength, final byte[] value, final int valueLength, + final long columnFamilyHandle) throws RocksDBException; + private native void putUntracked(final long handle, final byte[] key, + final int keyLength, final byte[] value, final int valueLength) + throws RocksDBException; + private native void putUntracked(final long handle, final byte[][] keys, + final int keysLength, final byte[][] values, final int valuesLength, + final long columnFamilyHandle) throws RocksDBException; + private native void putUntracked(final long handle, final byte[][] keys, + final int keysLength, final byte[][] values, final int valuesLength) + throws RocksDBException; + private native void mergeUntracked(final long handle, final byte[] key, + final int keyLength, final byte[] value, final int valueLength, + final long columnFamilyHandle) throws RocksDBException; + private native void mergeUntracked(final long handle, final byte[] key, + final int keyLength, final byte[] value, final int valueLength) + throws RocksDBException; + private native void deleteUntracked(final long handle, final byte[] key, + final int keyLength, final long columnFamilyHandle) + throws RocksDBException; + private native void deleteUntracked(final long handle, final byte[] key, + final int keyLength) throws RocksDBException; + private native void deleteUntracked(final long handle, final byte[][] keys, + final int keysLength, final long columnFamilyHandle) + throws RocksDBException; + private native void deleteUntracked(final long handle, final byte[][] keys, + final int keysLength) throws RocksDBException; + private native void putLogData(final long handle, final byte[] blob, + final int blobLength); + private native void disableIndexing(final long handle); + private native void enableIndexing(final long handle); + private native long getNumKeys(final long handle); + private native long getNumPuts(final long handle); + private native long getNumDeletes(final long handle); + private native long getNumMerges(final long handle); + private native long getElapsedTime(final long handle); + private native long getWriteBatch(final long handle); + private native void setLockTimeout(final long handle, final long lockTimeout); + private native long getWriteOptions(final long handle); + private native void setWriteOptions(final long handle, + final long writeOptionsHandle); + private native void undoGetForUpdate(final long handle, final byte[] key, + final int keyLength, final long columnFamilyHandle); + private native void undoGetForUpdate(final long handle, final byte[] key, + final int keyLength); + private native void rebuildFromWriteBatch(final long handle, + final long writeBatchHandle) throws RocksDBException; + private native long getCommitTimeWriteBatch(final long handle); + private native void setLogNumber(final long handle, final long logNumber); + private native long getLogNumber(final long handle); + private native void setName(final long handle, final String name) + throws RocksDBException; + private native String getName(final long handle); + private native long getID(final long handle); + private native boolean isDeadlockDetect(final long handle); + private native WaitingTransactions getWaitingTxns(final long handle); + private native byte getState(final long handle); + private native long getId(final long handle); + + @Override protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TransactionDB.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TransactionDB.java new file mode 100644 index 0000000..105f4ef --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TransactionDB.java @@ -0,0 +1,401 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Database with Transaction support + */ +public class TransactionDB extends RocksDB + implements TransactionalDB { + + private TransactionDBOptions transactionDbOptions_; + + /** + * Private constructor. + * + * @param nativeHandle The native handle of the C++ TransactionDB object + */ + private TransactionDB(final long nativeHandle) { + super(nativeHandle); + } + + /** + * Open a TransactionDB, similar to {@link RocksDB#open(Options, String)}. + * + * @param options {@link org.rocksdb.Options} instance. + * @param transactionDbOptions {@link org.rocksdb.TransactionDBOptions} + * instance. + * @param path the path to the rocksdb. + * + * @return a {@link TransactionDB} instance on success, null if the specified + * {@link TransactionDB} can not be opened. + * + * @throws RocksDBException if an error occurs whilst opening the database. + */ + public static TransactionDB open(final Options options, + final TransactionDBOptions transactionDbOptions, final String path) + throws RocksDBException { + final TransactionDB tdb = new TransactionDB(open(options.nativeHandle_, + transactionDbOptions.nativeHandle_, path)); + + // when non-default Options is used, keeping an Options reference + // in RocksDB can prevent Java to GC during the life-time of + // the currently-created RocksDB. + tdb.storeOptionsInstance(options); + tdb.storeTransactionDbOptions(transactionDbOptions); + + return tdb; + } + + /** + * Open a TransactionDB, similar to + * {@link RocksDB#open(DBOptions, String, List, List)}. + * + * @param dbOptions {@link org.rocksdb.DBOptions} instance. + * @param transactionDbOptions {@link org.rocksdb.TransactionDBOptions} + * instance. + * @param path the path to the rocksdb. + * @param columnFamilyDescriptors list of column family descriptors + * @param columnFamilyHandles will be filled with ColumnFamilyHandle instances + * + * @return a {@link TransactionDB} instance on success, null if the specified + * {@link TransactionDB} can not be opened. + * + * @throws RocksDBException if an error occurs whilst opening the database. + */ + public static TransactionDB open(final DBOptions dbOptions, + final TransactionDBOptions transactionDbOptions, + final String path, + final List columnFamilyDescriptors, + final List columnFamilyHandles) + throws RocksDBException { + + final byte[][] cfNames = new byte[columnFamilyDescriptors.size()][]; + final long[] cfOptionHandles = new long[columnFamilyDescriptors.size()]; + for (int i = 0; i < columnFamilyDescriptors.size(); i++) { + final ColumnFamilyDescriptor cfDescriptor = columnFamilyDescriptors + .get(i); + cfNames[i] = cfDescriptor.getName(); + cfOptionHandles[i] = cfDescriptor.getOptions().nativeHandle_; + } + + final long[] handles = open(dbOptions.nativeHandle_, + transactionDbOptions.nativeHandle_, path, cfNames, cfOptionHandles); + final TransactionDB tdb = new TransactionDB(handles[0]); + + // when non-default Options is used, keeping an Options reference + // in RocksDB can prevent Java to GC during the life-time of + // the currently-created RocksDB. + tdb.storeOptionsInstance(dbOptions); + tdb.storeTransactionDbOptions(transactionDbOptions); + + for (int i = 1; i < handles.length; i++) { + columnFamilyHandles.add(new ColumnFamilyHandle(tdb, handles[i])); + } + + return tdb; + } + + /** + * This is similar to {@link #close()} except that it + * throws an exception if any error occurs. + *

    + * This will not fsync the WAL files. + * If syncing is required, the caller must first call {@link #syncWal()} + * or {@link #write(WriteOptions, WriteBatch)} using an empty write batch + * with {@link WriteOptions#setSync(boolean)} set to true. + *

    + * See also {@link #close()}. + * + * @throws RocksDBException if an error occurs whilst closing. + */ + public void closeE() throws RocksDBException { + if (owningHandle_.compareAndSet(true, false)) { + try { + closeDatabase(nativeHandle_); + } finally { + disposeInternal(); + } + } + } + + /** + * This is similar to {@link #closeE()} except that it + * silently ignores any errors. + *

    + * This will not fsync the WAL files. + * If syncing is required, the caller must first call {@link #syncWal()} + * or {@link #write(WriteOptions, WriteBatch)} using an empty write batch + * with {@link WriteOptions#setSync(boolean)} set to true. + *

    + * See also {@link #close()}. + */ + @Override + public void close() { + if (owningHandle_.compareAndSet(true, false)) { + try { + closeDatabase(nativeHandle_); + } catch (final RocksDBException e) { + // silently ignore the error report + } finally { + disposeInternal(); + } + } + } + + @Override + public Transaction beginTransaction(final WriteOptions writeOptions) { + return new Transaction(this, beginTransaction(nativeHandle_, + writeOptions.nativeHandle_)); + } + + @Override + public Transaction beginTransaction(final WriteOptions writeOptions, + final TransactionOptions transactionOptions) { + return new Transaction(this, beginTransaction(nativeHandle_, + writeOptions.nativeHandle_, transactionOptions.nativeHandle_)); + } + + // TODO(AR) consider having beingTransaction(... oldTransaction) set a + // reference count inside Transaction, so that we can always call + // Transaction#close but the object is only disposed when there are as many + // closes as beginTransaction. Makes the try-with-resources paradigm easier for + // java developers + + @Override + public Transaction beginTransaction(final WriteOptions writeOptions, + final Transaction oldTransaction) { + final long jtxnHandle = beginTransaction_withOld(nativeHandle_, + writeOptions.nativeHandle_, oldTransaction.nativeHandle_); + + // RocksJava relies on the assumption that + // we do not allocate a new Transaction object + // when providing an old_txn + assert(jtxnHandle == oldTransaction.nativeHandle_); + + return oldTransaction; + } + + @Override + public Transaction beginTransaction(final WriteOptions writeOptions, + final TransactionOptions transactionOptions, + final Transaction oldTransaction) { + final long jtxn_handle = beginTransaction_withOld(nativeHandle_, + writeOptions.nativeHandle_, transactionOptions.nativeHandle_, + oldTransaction.nativeHandle_); + + // RocksJava relies on the assumption that + // we do not allocate a new Transaction object + // when providing an old_txn + assert(jtxn_handle == oldTransaction.nativeHandle_); + + return oldTransaction; + } + + public Transaction getTransactionByName(final String transactionName) { + final long jtxnHandle = getTransactionByName(nativeHandle_, transactionName); + if(jtxnHandle == 0) { + return null; + } + + final Transaction txn = new Transaction(this, jtxnHandle); + + // this instance doesn't own the underlying C++ object + txn.disOwnNativeHandle(); + + return txn; + } + + public List getAllPreparedTransactions() { + final long[] jtxnHandles = getAllPreparedTransactions(nativeHandle_); + + final List txns = new ArrayList<>(); + for(final long jtxnHandle : jtxnHandles) { + final Transaction txn = new Transaction(this, jtxnHandle); + + // this instance doesn't own the underlying C++ object + txn.disOwnNativeHandle(); + + txns.add(txn); + } + return txns; + } + + public static class KeyLockInfo { + private final String key; + private final long[] transactionIDs; + private final boolean exclusive; + + public KeyLockInfo(final String key, final long[] transactionIDs, final boolean exclusive) { + this.key = key; + this.transactionIDs = transactionIDs; + this.exclusive = exclusive; + } + + /** + * Get the key. + * + * @return the key + */ + public String getKey() { + return key; + } + + /** + * Get the Transaction IDs. + * + * @return the Transaction IDs. + */ + public long[] getTransactionIDs() { + return transactionIDs; + } + + /** + * Get the Lock status. + * + * @return true if the lock is exclusive, false if the lock is shared. + */ + public boolean isExclusive() { + return exclusive; + } + } + + /** + * Returns map of all locks held. + * + * @return a map of all the locks held. + */ + public Map getLockStatusData() { + return getLockStatusData(nativeHandle_); + } + + /** + * Called from C++ native method {@link #getDeadlockInfoBuffer(long)} + * to construct a DeadlockInfo object. + * + * @param transactionID The transaction id + * @param columnFamilyId The id of the {@link ColumnFamilyHandle} + * @param waitingKey the key that we are waiting on + * @param exclusive true if the lock is exclusive, false if the lock is shared + * + * @return The waiting transactions + */ + private DeadlockInfo newDeadlockInfo( + final long transactionID, final long columnFamilyId, + final String waitingKey, final boolean exclusive) { + return new DeadlockInfo(transactionID, columnFamilyId, + waitingKey, exclusive); + } + + public static class DeadlockInfo { + private final long transactionID; + private final long columnFamilyId; + private final String waitingKey; + private final boolean exclusive; + + private DeadlockInfo(final long transactionID, final long columnFamilyId, + final String waitingKey, final boolean exclusive) { + this.transactionID = transactionID; + this.columnFamilyId = columnFamilyId; + this.waitingKey = waitingKey; + this.exclusive = exclusive; + } + + /** + * Get the Transaction ID. + * + * @return the transaction ID + */ + public long getTransactionID() { + return transactionID; + } + + /** + * Get the Column Family ID. + * + * @return The column family ID + */ + public long getColumnFamilyId() { + return columnFamilyId; + } + + /** + * Get the key that we are waiting on. + * + * @return the key that we are waiting on + */ + public String getWaitingKey() { + return waitingKey; + } + + /** + * Get the Lock status. + * + * @return true if the lock is exclusive, false if the lock is shared. + */ + public boolean isExclusive() { + return exclusive; + } + } + + public static class DeadlockPath { + final DeadlockInfo[] path; + final boolean limitExceeded; + + public DeadlockPath(final DeadlockInfo[] path, final boolean limitExceeded) { + this.path = path; + this.limitExceeded = limitExceeded; + } + + public boolean isEmpty() { + return path.length == 0 && !limitExceeded; + } + } + + public DeadlockPath[] getDeadlockInfoBuffer() { + return getDeadlockInfoBuffer(nativeHandle_); + } + + public void setDeadlockInfoBufferSize(final int targetSize) { + setDeadlockInfoBufferSize(nativeHandle_, targetSize); + } + + private void storeTransactionDbOptions( + final TransactionDBOptions transactionDbOptions) { + this.transactionDbOptions_ = transactionDbOptions; + } + + @Override protected final native void disposeInternal(final long handle); + + private static native long open(final long optionsHandle, + final long transactionDbOptionsHandle, final String path) + throws RocksDBException; + private static native long[] open(final long dbOptionsHandle, + final long transactionDbOptionsHandle, final String path, + final byte[][] columnFamilyNames, final long[] columnFamilyOptions); + private static native void closeDatabase(final long handle) throws RocksDBException; + private native long beginTransaction(final long handle, + final long writeOptionsHandle); + private native long beginTransaction(final long handle, + final long writeOptionsHandle, final long transactionOptionsHandle); + private native long beginTransaction_withOld(final long handle, + final long writeOptionsHandle, final long oldTransactionHandle); + private native long beginTransaction_withOld(final long handle, + final long writeOptionsHandle, final long transactionOptionsHandle, + final long oldTransactionHandle); + private native long getTransactionByName(final long handle, + final String name); + private native long[] getAllPreparedTransactions(final long handle); + private native Map getLockStatusData( + final long handle); + private native DeadlockPath[] getDeadlockInfoBuffer(final long handle); + private native void setDeadlockInfoBufferSize(final long handle, + final int targetSize); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TransactionDBOptions.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TransactionDBOptions.java new file mode 100644 index 0000000..391025d --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TransactionDBOptions.java @@ -0,0 +1,216 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +public class TransactionDBOptions extends RocksObject { + + public TransactionDBOptions() { + super(newTransactionDBOptions()); + } + + /** + * Specifies the maximum number of keys that can be locked at the same time + * per column family. + *

    + * If the number of locked keys is greater than {@code #getMaxNumLocks()}, + * transaction writes (or GetForUpdate) will return an error. + * + * @return The maximum number of keys that can be locked + */ + public long getMaxNumLocks() { + assert(isOwningHandle()); + return getMaxNumLocks(nativeHandle_); + } + + /** + * Specifies the maximum number of keys that can be locked at the same time + * per column family. + *

    + * If the number of locked keys is greater than {@link #getMaxNumLocks()}, + * transaction writes (or GetForUpdate) will return an error. + * + * @param maxNumLocks The maximum number of keys that can be locked; + * If this value is not positive, no limit will be enforced. + * + * @return this TransactionDBOptions instance + */ + public TransactionDBOptions setMaxNumLocks(final long maxNumLocks) { + assert(isOwningHandle()); + setMaxNumLocks(nativeHandle_, maxNumLocks); + return this; + } + + /** + * The number of sub-tables per lock table (per column family) + * + * @return The number of sub-tables + */ + public long getNumStripes() { + assert(isOwningHandle()); + return getNumStripes(nativeHandle_); + } + + /** + * Increasing this value will increase the concurrency by dividing the lock + * table (per column family) into more sub-tables, each with their own + * separate mutex. + *

    + * Default: 16 + * + * @param numStripes The number of sub-tables + * + * @return this TransactionDBOptions instance + */ + public TransactionDBOptions setNumStripes(final long numStripes) { + assert(isOwningHandle()); + setNumStripes(nativeHandle_, numStripes); + return this; + } + + /** + * The default wait timeout in milliseconds when + * a transaction attempts to lock a key if not specified by + * {@link TransactionOptions#setLockTimeout(long)} + * + * If 0, no waiting is done if a lock cannot instantly be acquired. + * If negative, there is no timeout. + * + * @return the default wait timeout in milliseconds + */ + public long getTransactionLockTimeout() { + assert(isOwningHandle()); + return getTransactionLockTimeout(nativeHandle_); + } + + /** + * If positive, specifies the default wait timeout in milliseconds when + * a transaction attempts to lock a key if not specified by + * {@link TransactionOptions#setLockTimeout(long)} + * + * If 0, no waiting is done if a lock cannot instantly be acquired. + * If negative, there is no timeout. Not using a timeout is not recommended + * as it can lead to deadlocks. Currently, there is no deadlock-detection to + * recover from a deadlock. + *

    + * Default: 1000 + * + * @param transactionLockTimeout the default wait timeout in milliseconds + * + * @return this TransactionDBOptions instance + */ + public TransactionDBOptions setTransactionLockTimeout( + final long transactionLockTimeout) { + assert(isOwningHandle()); + setTransactionLockTimeout(nativeHandle_, transactionLockTimeout); + return this; + } + + /** + * The wait timeout in milliseconds when writing a key + * OUTSIDE of a transaction (ie by calling {@link RocksDB#put}, + * {@link RocksDB#merge}, {@link RocksDB#delete} or {@link RocksDB#write} + * directly). + *

    + * If 0, no waiting is done if a lock cannot instantly be acquired. + * If negative, there is no timeout and will block indefinitely when acquiring + * a lock. + * + * @return the timeout in milliseconds when writing a key OUTSIDE of a + * transaction + */ + public long getDefaultLockTimeout() { + assert(isOwningHandle()); + return getDefaultLockTimeout(nativeHandle_); + } + + /** + * If positive, specifies the wait timeout in milliseconds when writing a key + * OUTSIDE of a transaction (ie by calling {@link RocksDB#put}, + * {@link RocksDB#merge}, {@link RocksDB#delete} or {@link RocksDB#write} + * directly). + *

    + * If 0, no waiting is done if a lock cannot instantly be acquired. + * If negative, there is no timeout and will block indefinitely when acquiring + * a lock. + *

    + * Not using a timeout can lead to deadlocks. Currently, there + * is no deadlock-detection to recover from a deadlock. While DB writes + * cannot deadlock with other DB writes, they can deadlock with a transaction. + * A negative timeout should only be used if all transactions have a small + * expiration set. + *

    + * Default: 1000 + * + * @param defaultLockTimeout the timeout in milliseconds when writing a key + * OUTSIDE of a transaction + * @return this TransactionDBOptions instance + */ + public TransactionDBOptions setDefaultLockTimeout(final long defaultLockTimeout) { + assert (isOwningHandle()); + setDefaultLockTimeout(nativeHandle_, defaultLockTimeout); + return this; + } + +// /** +// * If set, the {@link TransactionDB} will use this implementation of a mutex +// * and condition variable for all transaction locking instead of the default +// * mutex/condvar implementation. +// * +// * @param transactionDbMutexFactory the mutex factory for the transactions +// * +// * @return this TransactionDBOptions instance +// */ +// public TransactionDBOptions setCustomMutexFactory( +// final TransactionDBMutexFactory transactionDbMutexFactory) { +// +// } + + /** + * The policy for when to write the data into the DB. The default policy is to + * write only the committed data {@link TxnDBWritePolicy#WRITE_COMMITTED}. + * The data could be written before the commit phase. The DB then needs to + * provide the mechanisms to tell apart committed from uncommitted data. + * + * @return The write policy. + */ + public TxnDBWritePolicy getWritePolicy() { + assert(isOwningHandle()); + return TxnDBWritePolicy.getTxnDBWritePolicy(getWritePolicy(nativeHandle_)); + } + + /** + * The policy for when to write the data into the DB. The default policy is to + * write only the committed data {@link TxnDBWritePolicy#WRITE_COMMITTED}. + * The data could be written before the commit phase. The DB then needs to + * provide the mechanisms to tell apart committed from uncommitted data. + * + * @param writePolicy The write policy. + * + * @return this TransactionDBOptions instance + */ + public TransactionDBOptions setWritePolicy( + final TxnDBWritePolicy writePolicy) { + assert(isOwningHandle()); + setWritePolicy(nativeHandle_, writePolicy.getValue()); + return this; + } + + private static native long newTransactionDBOptions(); + private native long getMaxNumLocks(final long handle); + private native void setMaxNumLocks(final long handle, + final long maxNumLocks); + private native long getNumStripes(final long handle); + private native void setNumStripes(final long handle, final long numStripes); + private native long getTransactionLockTimeout(final long handle); + private native void setTransactionLockTimeout(final long handle, + final long transactionLockTimeout); + private native long getDefaultLockTimeout(final long handle); + private native void setDefaultLockTimeout(final long handle, + final long transactionLockTimeout); + private native byte getWritePolicy(final long handle); + private native void setWritePolicy(final long handle, final byte writePolicy); + @Override protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TransactionLogIterator.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TransactionLogIterator.java new file mode 100644 index 0000000..5d9ec58 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TransactionLogIterator.java @@ -0,0 +1,112 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb; + +/** + *

    A TransactionLogIterator is used to iterate over the transactions in a db. + * One run of the iterator is continuous, i.e. the iterator will stop at the + * beginning of any gap in sequences.

    + */ +public class TransactionLogIterator extends RocksObject { + + /** + *

    An iterator is either positioned at a WriteBatch + * or not valid. This method returns true if the iterator + * is valid. Can read data from a valid iterator.

    + * + * @return true if iterator position is valid. + */ + public boolean isValid() { + return isValid(nativeHandle_); + } + + /** + *

    Moves the iterator to the next WriteBatch. + * REQUIRES: Valid() to be true.

    + */ + public void next() { + next(nativeHandle_); + } + + /** + *

    Throws RocksDBException if something went wrong.

    + * + * @throws org.rocksdb.RocksDBException if something went + * wrong in the underlying C++ code. + */ + public void status() throws RocksDBException { + status(nativeHandle_); + } + + /** + *

    If iterator position is valid, return the current + * write_batch and the sequence number of the earliest + * transaction contained in the batch.

    + * + *

    ONLY use if Valid() is true and status() is OK.

    + * + * @return {@link org.rocksdb.TransactionLogIterator.BatchResult} + * instance. + */ + public BatchResult getBatch() { + assert(isValid()); + return getBatch(nativeHandle_); + } + + /** + *

    TransactionLogIterator constructor.

    + * + * @param nativeHandle address to native address. + */ + TransactionLogIterator(final long nativeHandle) { + super(nativeHandle); + } + + /** + *

    BatchResult represents a data structure returned + * by a TransactionLogIterator containing a sequence + * number and a {@link WriteBatch} instance.

    + */ + public static final class BatchResult { + /** + *

    Constructor of BatchResult class.

    + * + * @param sequenceNumber related to this BatchResult instance. + * @param nativeHandle to {@link org.rocksdb.WriteBatch} + * native instance. + */ + public BatchResult(final long sequenceNumber, + final long nativeHandle) { + sequenceNumber_ = sequenceNumber; + writeBatch_ = new WriteBatch(nativeHandle, true); + } + + /** + *

    Return sequence number related to this BatchResult.

    + * + * @return Sequence number. + */ + public long sequenceNumber() { + return sequenceNumber_; + } + + /** + *

    Return contained {@link org.rocksdb.WriteBatch} + * instance

    + * + * @return {@link org.rocksdb.WriteBatch} instance. + */ + public WriteBatch writeBatch() { + return writeBatch_; + } + + private final long sequenceNumber_; + private final WriteBatch writeBatch_; + } + + @Override protected final native void disposeInternal(final long handle); + private native boolean isValid(long handle); + private native void next(long handle); + private native void status(long handle) + throws RocksDBException; + private native BatchResult getBatch(long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TransactionOptions.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TransactionOptions.java new file mode 100644 index 0000000..f93d3cb --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TransactionOptions.java @@ -0,0 +1,189 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +public class TransactionOptions extends RocksObject + implements TransactionalOptions { + + public TransactionOptions() { + super(newTransactionOptions()); + } + + @Override + public boolean isSetSnapshot() { + assert(isOwningHandle()); + return isSetSnapshot(nativeHandle_); + } + + @Override + public TransactionOptions setSetSnapshot(final boolean setSnapshot) { + assert(isOwningHandle()); + setSetSnapshot(nativeHandle_, setSnapshot); + return this; + } + + /** + * True means that before acquiring locks, this transaction will + * check if doing so will cause a deadlock. If so, it will return with + * {@link Status.Code#Busy}. The user should retry their transaction. + * + * @return true if a deadlock is detected. + */ + public boolean isDeadlockDetect() { + assert(isOwningHandle()); + return isDeadlockDetect(nativeHandle_); + } + + /** + * Setting to true means that before acquiring locks, this transaction will + * check if doing so will cause a deadlock. If so, it will return with + * {@link Status.Code#Busy}. The user should retry their transaction. + * + * @param deadlockDetect true if we should detect deadlocks. + * + * @return this TransactionOptions instance + */ + public TransactionOptions setDeadlockDetect(final boolean deadlockDetect) { + assert(isOwningHandle()); + setDeadlockDetect(nativeHandle_, deadlockDetect); + return this; + } + + /** + * The wait timeout in milliseconds when a transaction attempts to lock a key. + *

    + * If 0, no waiting is done if a lock cannot instantly be acquired. + * If negative, {@link TransactionDBOptions#getTransactionLockTimeout(long)} + * will be used + * + * @return the lock timeout in milliseconds + */ + public long getLockTimeout() { + assert(isOwningHandle()); + return getLockTimeout(nativeHandle_); + } + + /** + * If positive, specifies the wait timeout in milliseconds when + * a transaction attempts to lock a key. + *

    + * If 0, no waiting is done if a lock cannot instantly be acquired. + * If negative, {@link TransactionDBOptions#getTransactionLockTimeout(long)} + * will be used + *

    + * Default: -1 + * + * @param lockTimeout the lock timeout in milliseconds + * + * @return this TransactionOptions instance + */ + public TransactionOptions setLockTimeout(final long lockTimeout) { + assert(isOwningHandle()); + setLockTimeout(nativeHandle_, lockTimeout); + return this; + } + + /** + * Expiration duration in milliseconds. + *

    + * If non-negative, transactions that last longer than this many milliseconds + * will fail to commit. If not set, a forgotten transaction that is never + * committed, rolled back, or deleted will never relinquish any locks it + * holds. This could prevent keys from being written by other writers. + * + * @return expiration the expiration duration in milliseconds + */ + public long getExpiration() { + assert(isOwningHandle()); + return getExpiration(nativeHandle_); + } + + /** + * Expiration duration in milliseconds. + *

    + * If non-negative, transactions that last longer than this many milliseconds + * will fail to commit. If not set, a forgotten transaction that is never + * committed, rolled back, or deleted will never relinquish any locks it + * holds. This could prevent keys from being written by other writers. + *

    + * Default: -1 + * + * @param expiration the expiration duration in milliseconds + * + * @return this TransactionOptions instance + */ + public TransactionOptions setExpiration(final long expiration) { + assert(isOwningHandle()); + setExpiration(nativeHandle_, expiration); + return this; + } + + /** + * Gets the number of traversals to make during deadlock detection. + * + * @return the number of traversals to make during + * deadlock detection + */ + public long getDeadlockDetectDepth() { + return getDeadlockDetectDepth(nativeHandle_); + } + + /** + * Sets the number of traversals to make during deadlock detection. + *

    + * Default: 50 + * + * @param deadlockDetectDepth the number of traversals to make during + * deadlock detection + * + * @return this TransactionOptions instance + */ + public TransactionOptions setDeadlockDetectDepth( + final long deadlockDetectDepth) { + setDeadlockDetectDepth(nativeHandle_, deadlockDetectDepth); + return this; + } + + /** + * Get the maximum number of bytes that may be used for the write batch. + * + * @return the maximum number of bytes, 0 means no limit. + */ + public long getMaxWriteBatchSize() { + return getMaxWriteBatchSize(nativeHandle_); + } + + /** + * Set the maximum number of bytes that may be used for the write batch. + * + * @param maxWriteBatchSize the maximum number of bytes, 0 means no limit. + * + * @return this TransactionOptions instance + */ + public TransactionOptions setMaxWriteBatchSize(final long maxWriteBatchSize) { + setMaxWriteBatchSize(nativeHandle_, maxWriteBatchSize); + return this; + } + + private static native long newTransactionOptions(); + private native boolean isSetSnapshot(final long handle); + private native void setSetSnapshot(final long handle, + final boolean setSnapshot); + private native boolean isDeadlockDetect(final long handle); + private native void setDeadlockDetect(final long handle, + final boolean deadlockDetect); + private native long getLockTimeout(final long handle); + private native void setLockTimeout(final long handle, final long lockTimeout); + private native long getExpiration(final long handle); + private native void setExpiration(final long handle, final long expiration); + private native long getDeadlockDetectDepth(final long handle); + private native void setDeadlockDetectDepth(final long handle, + final long deadlockDetectDepth); + private native long getMaxWriteBatchSize(final long handle); + private native void setMaxWriteBatchSize(final long handle, + final long maxWriteBatchSize); + @Override protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TransactionalDB.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TransactionalDB.java new file mode 100644 index 0000000..1ba9554 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TransactionalDB.java @@ -0,0 +1,65 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +interface TransactionalDB> extends AutoCloseable { + /** + * Starts a new Transaction. + *

    + * Caller is responsible for calling {@link #close()} on the returned + * transaction when it is no longer needed. + * + * @param writeOptions Any write options for the transaction + * @return a new transaction + */ + Transaction beginTransaction(final WriteOptions writeOptions); + + /** + * Starts a new Transaction. + *

    + * Caller is responsible for calling {@link #close()} on the returned + * transaction when it is no longer needed. + * + * @param writeOptions Any write options for the transaction + * @param transactionOptions Any options for the transaction + * @return a new transaction + */ + Transaction beginTransaction(final WriteOptions writeOptions, + final T transactionOptions); + + /** + * Starts a new Transaction. + *

    + * Caller is responsible for calling {@link #close()} on the returned + * transaction when it is no longer needed. + * + * @param writeOptions Any write options for the transaction + * @param oldTransaction this Transaction will be reused instead of allocating + * a new one. This is an optimization to avoid extra allocations + * when repeatedly creating transactions. + * @return The oldTransaction which has been reinitialized as a new + * transaction + */ + Transaction beginTransaction(final WriteOptions writeOptions, + final Transaction oldTransaction); + + /** + * Starts a new Transaction. + *

    + * Caller is responsible for calling {@link #close()} on the returned + * transaction when it is no longer needed. + * + * @param writeOptions Any write options for the transaction + * @param transactionOptions Any options for the transaction + * @param oldTransaction this Transaction will be reused instead of allocating + * a new one. This is an optimization to avoid extra allocations + * when repeatedly creating transactions. + * @return The oldTransaction which has been reinitialized as a new + * transaction + */ + Transaction beginTransaction(final WriteOptions writeOptions, + final T transactionOptions, final Transaction oldTransaction); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TransactionalOptions.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TransactionalOptions.java new file mode 100644 index 0000000..2175693 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TransactionalOptions.java @@ -0,0 +1,31 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + + +interface TransactionalOptions> + extends AutoCloseable { + + /** + * True indicates snapshots will be set, just like if + * {@link Transaction#setSnapshot()} had been called + * + * @return whether a snapshot will be set + */ + boolean isSetSnapshot(); + + /** + * Setting the setSnapshot to true is the same as calling + * {@link Transaction#setSnapshot()}. + *

    + * Default: false + * + * @param setSnapshot Whether to set a snapshot + * + * @return this TransactionalOptions instance + */ + T setSetSnapshot(final boolean setSnapshot); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TtlDB.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TtlDB.java new file mode 100644 index 0000000..2bb0c43 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TtlDB.java @@ -0,0 +1,242 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.List; + +/** + * Database with TTL support. + * + *

    Use case

    + *

    This API should be used to open the db when key-values inserted are + * meant to be removed from the db in a non-strict 'ttl' amount of time + * Therefore, this guarantees that key-values inserted will remain in the + * db for >= ttl amount of time and the db will make efforts to remove the + * key-values as soon as possible after ttl seconds of their insertion. + *

    + * + *

    Behaviour

    + *

    TTL is accepted in seconds + * (int32_t)Timestamp(creation) is suffixed to values in Put internally + * Expired TTL values deleted in compaction only:(Timestamp+ttl<time_now) + * Get/Iterator may return expired entries(compaction not run on them yet) + * Different TTL may be used during different Opens + *

    + * + *

    Example

    + *
      + *
    • Open1 at t=0 with ttl=4 and insert k1,k2, close at t=2
      • + *
    • Open2 at t=3 with ttl=5. Now k1,k2 should be deleted at t>=5
      • + *
    + * + *

    + * read_only=true opens in the usual read-only mode. Compactions will not be + * triggered(neither manual nor automatic), so no expired entries removed + *

    + * + *

    Constraints

    + *

    Not specifying/passing or non-positive TTL behaves + * like TTL = infinity

    + * + *

    !!!WARNING!!!

    + *

    Calling DB::Open directly to re-open a db created by this API will get + * corrupt values(timestamp suffixed) and no ttl effect will be there + * during the second Open, so use this API consistently to open the db + * Be careful when passing ttl with a small positive value because the + * whole database may be deleted in a small amount of time.

    + */ +public class TtlDB extends RocksDB { + + /** + *

    Opens a TtlDB.

    + * + *

    Database is opened in read-write mode without default TTL.

    + * + * @param options {@link org.rocksdb.Options} instance. + * @param db_path path to database. + * + * @return TtlDB instance. + * + * @throws RocksDBException thrown if an error occurs within the native + * part of the library. + */ + public static TtlDB open(final Options options, final String db_path) + throws RocksDBException { + return open(options, db_path, 0, false); + } + + /** + *

    Opens a TtlDB.

    + * + * @param options {@link org.rocksdb.Options} instance. + * @param db_path path to database. + * @param ttl time to live for new entries. + * @param readOnly boolean value indicating if database if db is + * opened read-only. + * + * @return TtlDB instance. + * + * @throws RocksDBException thrown if an error occurs within the native + * part of the library. + */ + public static TtlDB open(final Options options, final String db_path, + final int ttl, final boolean readOnly) throws RocksDBException { + return new TtlDB(open(options.nativeHandle_, db_path, ttl, readOnly)); + } + + /** + *

    Opens a TtlDB.

    + * + * @param options {@link org.rocksdb.Options} instance. + * @param db_path path to database. + * @param columnFamilyDescriptors list of column family descriptors + * @param columnFamilyHandles will be filled with ColumnFamilyHandle instances + * on open. + * @param ttlValues time to live values per column family handle + * @param readOnly boolean value indicating if database if db is + * opened read-only. + * + * @return TtlDB instance. + * + * @throws RocksDBException thrown if an error occurs within the native + * part of the library. + * @throws java.lang.IllegalArgumentException when there is not a ttl value + * per given column family handle. + */ + public static TtlDB open(final DBOptions options, final String db_path, + final List columnFamilyDescriptors, + final List columnFamilyHandles, + final List ttlValues, final boolean readOnly) + throws RocksDBException { + if (columnFamilyDescriptors.size() != ttlValues.size()) { + throw new IllegalArgumentException("There must be a ttl value per column" + + " family handle."); + } + + final byte[][] cfNames = new byte[columnFamilyDescriptors.size()][]; + final long[] cfOptionHandles = new long[columnFamilyDescriptors.size()]; + for (int i = 0; i < columnFamilyDescriptors.size(); i++) { + final ColumnFamilyDescriptor cfDescriptor = + columnFamilyDescriptors.get(i); + cfNames[i] = cfDescriptor.getName(); + cfOptionHandles[i] = cfDescriptor.getOptions().nativeHandle_; + } + + final int[] ttlVals = new int[ttlValues.size()]; + for(int i = 0; i < ttlValues.size(); i++) { + ttlVals[i] = ttlValues.get(i); + } + final long[] handles = openCF(options.nativeHandle_, db_path, + cfNames, cfOptionHandles, ttlVals, readOnly); + + final TtlDB ttlDB = new TtlDB(handles[0]); + for (int i = 1; i < handles.length; i++) { + columnFamilyHandles.add(new ColumnFamilyHandle(ttlDB, handles[i])); + } + return ttlDB; + } + + /** + *

    Close the TtlDB instance and release resource.

    + * + * This is similar to {@link #close()} except that it + * throws an exception if any error occurs. + *

    + * This will not fsync the WAL files. + * If syncing is required, the caller must first call {@link #syncWal()} + * or {@link #write(WriteOptions, WriteBatch)} using an empty write batch + * with {@link WriteOptions#setSync(boolean)} set to true. + *

    + * See also {@link #close()}. + * + * @throws RocksDBException if an error occurs whilst closing. + */ + public void closeE() throws RocksDBException { + if (owningHandle_.compareAndSet(true, false)) { + try { + closeDatabase(nativeHandle_); + } finally { + disposeInternal(); + } + } + } + + /** + *

    Close the TtlDB instance and release resource.

    + * + * + * This will not fsync the WAL files. + * If syncing is required, the caller must first call {@link #syncWal()} + * or {@link #write(WriteOptions, WriteBatch)} using an empty write batch + * with {@link WriteOptions#setSync(boolean)} set to true. + *

    + * See also {@link #close()}. + */ + @Override + public void close() { + if (owningHandle_.compareAndSet(true, false)) { + try { + closeDatabase(nativeHandle_); + } catch (final RocksDBException e) { + // silently ignore the error report + } finally { + disposeInternal(); + } + } + } + + /** + *

    Creates a new ttl based column family with a name defined + * in given ColumnFamilyDescriptor and allocates a + * ColumnFamilyHandle within an internal structure.

    + * + *

    The ColumnFamilyHandle is automatically disposed with DB + * disposal.

    + * + * @param columnFamilyDescriptor column family to be created. + * @param ttl TTL to set for this column family. + * + * @return {@link org.rocksdb.ColumnFamilyHandle} instance. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public ColumnFamilyHandle createColumnFamilyWithTtl( + final ColumnFamilyDescriptor columnFamilyDescriptor, + final int ttl) throws RocksDBException { + return new ColumnFamilyHandle(this, + createColumnFamilyWithTtl(nativeHandle_, + columnFamilyDescriptor.getName(), + columnFamilyDescriptor.getOptions().nativeHandle_, ttl)); + } + + /** + *

    A protected constructor that will be used in the static + * factory method + * {@link #open(Options, String, int, boolean)} + * and + * {@link #open(DBOptions, String, java.util.List, java.util.List, + * java.util.List, boolean)}. + *

    + * + * @param nativeHandle The native handle of the C++ TtlDB object + */ + protected TtlDB(final long nativeHandle) { + super(nativeHandle); + } + + @Override protected native void disposeInternal(final long handle); + + private static native long open(final long optionsHandle, final String db_path, final int ttl, + final boolean readOnly) throws RocksDBException; + private static native long[] openCF(final long optionsHandle, final String db_path, + final byte[][] columnFamilyNames, final long[] columnFamilyOptions, final int[] ttlValues, + final boolean readOnly) throws RocksDBException; + private native long createColumnFamilyWithTtl(final long handle, + final byte[] columnFamilyName, final long columnFamilyOptions, int ttl) + throws RocksDBException; + private static native void closeDatabase(final long handle) throws RocksDBException; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TxnDBWritePolicy.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TxnDBWritePolicy.java new file mode 100644 index 0000000..28cb855 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/TxnDBWritePolicy.java @@ -0,0 +1,62 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +/** + * The transaction db write policy. + */ +public enum TxnDBWritePolicy { + /** + * Write only the committed data. + */ + WRITE_COMMITTED((byte)0x00), + + /** + * Write data after the prepare phase of 2pc. + */ + WRITE_PREPARED((byte)0x1), + + /** + * Write data before the prepare phase of 2pc. + */ + WRITE_UNPREPARED((byte)0x2); + + private final byte value; + + TxnDBWritePolicy(final byte value) { + this.value = value; + } + + /** + *

    Returns the byte value of the enumerations value.

    + * + * @return byte representation + */ + public byte getValue() { + return value; + } + + /** + *

    Get the TxnDBWritePolicy enumeration value by + * passing the byte identifier to this method.

    + * + * @param byteIdentifier of TxnDBWritePolicy. + * + * @return TxnDBWritePolicy instance. + * + * @throws IllegalArgumentException If TxnDBWritePolicy cannot be found for + * the provided byteIdentifier + */ + public static TxnDBWritePolicy getTxnDBWritePolicy(final byte byteIdentifier) { + for (final TxnDBWritePolicy txnDBWritePolicy : TxnDBWritePolicy.values()) { + if (txnDBWritePolicy.getValue() == byteIdentifier) { + return txnDBWritePolicy; + } + } + + throw new IllegalArgumentException( + "Illegal value provided for TxnDBWritePolicy."); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/UInt64AddOperator.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/UInt64AddOperator.java new file mode 100644 index 0000000..0cffdce --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/UInt64AddOperator.java @@ -0,0 +1,19 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Uint64AddOperator is a merge operator that accumlates a long + * integer value. + */ +public class UInt64AddOperator extends MergeOperator { + public UInt64AddOperator() { + super(newSharedUInt64AddOperator()); + } + + private static native long newSharedUInt64AddOperator(); + @Override protected final native void disposeInternal(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/VectorMemTableConfig.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/VectorMemTableConfig.java new file mode 100644 index 0000000..fb1e7a9 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/VectorMemTableConfig.java @@ -0,0 +1,46 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb; + +/** + * The config for vector memtable representation. + */ +public class VectorMemTableConfig extends MemTableConfig { + public static final int DEFAULT_RESERVED_SIZE = 0; + + /** + * VectorMemTableConfig constructor + */ + public VectorMemTableConfig() { + reservedSize_ = DEFAULT_RESERVED_SIZE; + } + + /** + * Set the initial size of the vector that will be used + * by the memtable created based on this config. + * + * @param size the initial size of the vector. + * @return the reference to the current config. + */ + public VectorMemTableConfig setReservedSize(final int size) { + reservedSize_ = size; + return this; + } + + /** + * Returns the initial size of the vector used by the memtable + * created based on this config. + * + * @return the initial size of the vector. + */ + public int reservedSize() { + return reservedSize_; + } + + @Override protected long newMemTableFactoryHandle() { + return newMemTableFactoryHandle(reservedSize_); + } + + private native long newMemTableFactoryHandle(long reservedSize) + throws IllegalArgumentException; + private int reservedSize_; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WALRecoveryMode.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WALRecoveryMode.java new file mode 100644 index 0000000..b8c098f --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WALRecoveryMode.java @@ -0,0 +1,82 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * The WAL Recover Mode + */ +public enum WALRecoveryMode { + /** + * Original levelDB recovery + *

    + * We tolerate incomplete record in trailing data on all logs + * Use case : This is legacy behavior (default) + */ + TolerateCorruptedTailRecords((byte)0x00), + + /** + * Recover from clean shutdown + *

    + * We don't expect to find any corruption in the WAL + * Use case : This is ideal for unit tests and rare applications that + * can require high consistency guarantee + */ + AbsoluteConsistency((byte)0x01), + + /** + * Recover to point-in-time consistency + * We stop the WAL playback on discovering WAL inconsistency + * Use case : Ideal for systems that have disk controller cache like + * hard disk, SSD without super capacitor that store related data + */ + PointInTimeRecovery((byte)0x02), + + /** + * Recovery after a disaster + * We ignore any corruption in the WAL and try to salvage as much data as + * possible + * Use case : Ideal for last ditch effort to recover data or systems that + * operate with low grade unrelated data + */ + SkipAnyCorruptedRecords((byte)0x03); + + private final byte value; + + WALRecoveryMode(final byte value) { + this.value = value; + } + + /** + *

    Returns the byte value of the enumerations value.

    + * + * @return byte representation + */ + public byte getValue() { + return value; + } + + /** + *

    Get the WALRecoveryMode enumeration value by + * passing the byte identifier to this method.

    + * + * @param byteIdentifier of WALRecoveryMode. + * + * @return WALRecoveryMode instance. + * + * @throws IllegalArgumentException If WALRecoveryMode cannot be found for the + * provided byteIdentifier + */ + public static WALRecoveryMode getWALRecoveryMode(final byte byteIdentifier) { + for (final WALRecoveryMode walRecoveryMode : WALRecoveryMode.values()) { + if (walRecoveryMode.getValue() == byteIdentifier) { + return walRecoveryMode; + } + } + + throw new IllegalArgumentException( + "Illegal value provided for WALRecoveryMode."); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WBWIRocksIterator.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WBWIRocksIterator.java new file mode 100644 index 0000000..e0b99b1 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WBWIRocksIterator.java @@ -0,0 +1,203 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.nio.ByteBuffer; + +public class WBWIRocksIterator + extends AbstractRocksIterator { + private final WriteEntry entry = new WriteEntry(); + + protected WBWIRocksIterator(final WriteBatchWithIndex wbwi, + final long nativeHandle) { + super(wbwi, nativeHandle); + } + + /** + * Get the current entry + *

    + * The WriteEntry is only valid + * until the iterator is repositioned. + * If you want to keep the WriteEntry across iterator + * movements, you must make a copy of its data! + *

    + * Note - This method is not thread-safe with respect to the WriteEntry + * as it performs a non-atomic update across the fields of the WriteEntry + * + * @return The WriteEntry of the current entry + */ + public WriteEntry entry() { + assert(isOwningHandle()); + final long[] ptrs = entry1(nativeHandle_); + + entry.type = WriteType.fromId((byte)ptrs[0]); + entry.key.resetNativeHandle(ptrs[1], ptrs[1] != 0); + entry.value.resetNativeHandle(ptrs[2], ptrs[2] != 0); + + return entry; + } + + @Override protected final native void disposeInternal(final long handle); + @Override final native boolean isValid0(long handle); + @Override final native void seekToFirst0(long handle); + @Override final native void seekToLast0(long handle); + @Override final native void next0(long handle); + @Override final native void prev0(long handle); + @Override final native void refresh0(final long handle) throws RocksDBException; + @Override final native void seek0(long handle, byte[] target, int targetLen); + @Override final native void seekForPrev0(long handle, byte[] target, int targetLen); + @Override final native void status0(long handle) throws RocksDBException; + @Override + final native void seekDirect0( + final long handle, final ByteBuffer target, final int targetOffset, final int targetLen); + @Override + final native void seekForPrevDirect0( + final long handle, final ByteBuffer target, final int targetOffset, final int targetLen); + @Override + final native void seekByteArray0( + final long handle, final byte[] target, final int targetOffset, final int targetLen); + @Override + final native void seekForPrevByteArray0( + final long handle, final byte[] target, final int targetOffset, final int targetLen); + + private native long[] entry1(final long handle); + + /** + * Enumeration of the Write operation + * that created the record in the Write Batch + */ + public enum WriteType { + PUT((byte)0x0), + MERGE((byte)0x1), + DELETE((byte)0x2), + SINGLE_DELETE((byte)0x3), + DELETE_RANGE((byte)0x4), + LOG((byte)0x5), + XID((byte)0x6); + + final byte id; + WriteType(final byte id) { + this.id = id; + } + + public static WriteType fromId(final byte id) { + for(final WriteType wt : WriteType.values()) { + if(id == wt.id) { + return wt; + } + } + throw new IllegalArgumentException("No WriteType with id=" + id); + } + } + + @Override + public void close() { + entry.close(); + super.close(); + } + + /** + * Represents an entry returned by + * {@link org.rocksdb.WBWIRocksIterator#entry()} + * + * It is worth noting that a WriteEntry with + * the type {@link org.rocksdb.WBWIRocksIterator.WriteType#DELETE} + * or {@link org.rocksdb.WBWIRocksIterator.WriteType#LOG} + * will not have a value. + */ + public static class WriteEntry implements AutoCloseable { + WriteType type = null; + final DirectSlice key; + final DirectSlice value; + + /** + * Intentionally private as this + * should only be instantiated in + * this manner by the outer WBWIRocksIterator + * class; The class members are then modified + * by calling {@link org.rocksdb.WBWIRocksIterator#entry()} + */ + private WriteEntry() { + key = new DirectSlice(); + value = new DirectSlice(); + } + + public WriteEntry(final WriteType type, final DirectSlice key, + final DirectSlice value) { + this.type = type; + this.key = key; + this.value = value; + } + + /** + * Returns the type of the Write Entry + * + * @return the WriteType of the WriteEntry + */ + public WriteType getType() { + return type; + } + + /** + * Returns the key of the Write Entry + * + * @return The slice containing the key + * of the WriteEntry + */ + public DirectSlice getKey() { + return key; + } + + /** + * Returns the value of the Write Entry + * + * @return The slice containing the value of + * the WriteEntry or null if the WriteEntry has + * no value + */ + public DirectSlice getValue() { + if(!value.isOwningHandle()) { + return null; //TODO(AR) migrate to JDK8 java.util.Optional#empty() + } else { + return value; + } + } + + /** + * Generates a hash code for the Write Entry. NOTE: The hash code is based + * on the string representation of the key, so it may not work correctly + * with exotic custom comparators. + * + * @return The hash code for the Write Entry + */ + @Override + public int hashCode() { + return (key == null) ? 0 : key.hashCode(); + } + + @Override + public boolean equals(final Object other) { + if(other == null) { + return false; + } else if (this == other) { + return true; + } else if(other instanceof WriteEntry) { + final WriteEntry otherWriteEntry = (WriteEntry)other; + return type.equals(otherWriteEntry.type) + && key.equals(otherWriteEntry.key) + && value.equals(otherWriteEntry.value); + } else { + return false; + } + } + + @Override + public void close() { + value.close(); + key.close(); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WalFileType.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WalFileType.java new file mode 100644 index 0000000..fed27ed --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WalFileType.java @@ -0,0 +1,55 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +public enum WalFileType { + /** + * Indicates that WAL file is in archive directory. WAL files are moved from + * the main db directory to archive directory once they are not live and stay + * there until cleaned up. Files are cleaned depending on archive size + * (Options::WAL_size_limit_MB) and time since last cleaning + * (Options::WAL_ttl_seconds). + */ + kArchivedLogFile((byte)0x0), + + /** + * Indicates that WAL file is live and resides in the main db directory + */ + kAliveLogFile((byte)0x1); + + private final byte value; + + WalFileType(final byte value) { + this.value = value; + } + + /** + * Get the internal representation value. + * + * @return the internal representation value + */ + byte getValue() { + return value; + } + + /** + * Get the WalFileType from the internal representation value. + * + * @return the wal file type. + * + * @throws IllegalArgumentException if the value is unknown. + */ + static WalFileType fromValue(final byte value) { + for (final WalFileType walFileType : WalFileType.values()) { + if(walFileType.value == value) { + return walFileType; + } + } + + throw new IllegalArgumentException( + "Illegal value provided for WalFileType: " + value); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WalFilter.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WalFilter.java new file mode 100644 index 0000000..a283663 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WalFilter.java @@ -0,0 +1,86 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Map; + +/** + * WALFilter allows an application to inspect write-ahead-log (WAL) + * records or modify their processing on recovery. + */ +public interface WalFilter { + /** + * Provide ColumnFamily->LogNumber map to filter + * so that filter can determine whether a log number applies to a given + * column family (i.e. that log hasn't been flushed to SST already for the + * column family). + *

    + * We also pass in name>id map as only name is known during + * recovery (as handles are opened post-recovery). + * while write batch callbacks happen in terms of column family id. + * + * @param cfLognumber column_family_id to lognumber map + * @param cfNameId column_family_name to column_family_id map + */ + void columnFamilyLogNumberMap(final Map cfLognumber, + final Map cfNameId); + + /** + * LogRecord is invoked for each log record encountered for all the logs + * during replay on logs on recovery. This method can be used to: + * * inspect the record (using the batch parameter) + * * ignoring current record + * (by returning WalProcessingOption::kIgnoreCurrentRecord) + * * reporting corrupted record + * (by returning WalProcessingOption::kCorruptedRecord) + * * stop log replay + * (by returning kStop replay) - please note that this implies + * discarding the logs from current record onwards. + * + * @param logNumber log number of the current log. + * Filter might use this to determine if the log + * record is applicable to a certain column family. + * @param logFileName log file name - only for informational purposes + * @param batch batch encountered in the log during recovery + * @param newBatch new batch to populate if filter wants to change + * the batch (for example to filter some records out, or alter some + * records). Please note that the new batch MUST NOT contain + * more records than original, else recovery would be failed. + * + * @return Processing option for the current record. + */ + LogRecordFoundResult logRecordFound(final long logNumber, + final String logFileName, final WriteBatch batch, + final WriteBatch newBatch); + + class LogRecordFoundResult { + public static LogRecordFoundResult CONTINUE_UNCHANGED = + new LogRecordFoundResult(WalProcessingOption.CONTINUE_PROCESSING, false); + + final WalProcessingOption walProcessingOption; + final boolean batchChanged; + + /** + * @param walProcessingOption the processing option + * @param batchChanged Whether batch was changed by the filter. + * It must be set to true if newBatch was populated, + * else newBatch has no effect. + */ + public LogRecordFoundResult(final WalProcessingOption walProcessingOption, + final boolean batchChanged) { + this.walProcessingOption = walProcessingOption; + this.batchChanged = batchChanged; + } + } + + /** + * Returns a name that identifies this WAL filter. + * The name will be printed to LOG file on start up for diagnosis. + * + * @return the name + */ + String name(); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WalProcessingOption.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WalProcessingOption.java new file mode 100644 index 0000000..3a9c2be --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WalProcessingOption.java @@ -0,0 +1,54 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +public enum WalProcessingOption { + /* + * Continue processing as usual. + */ + CONTINUE_PROCESSING((byte)0x0), + + /** + * Ignore the current record but continue processing of log(s). + */ + IGNORE_CURRENT_RECORD((byte)0x1), + + /** + * Stop replay of logs and discard logs. + * Logs won't be replayed on subsequent recovery. + */ + STOP_REPLAY((byte)0x2), + + /** + * Corrupted record detected by filter. + */ + CORRUPTED_RECORD((byte)0x3); + + private final byte value; + + WalProcessingOption(final byte value) { + this.value = value; + } + + /** + * Get the internal representation. + * + * @return the internal representation. + */ + byte getValue() { + return value; + } + + public static WalProcessingOption fromValue(final byte value) { + for (final WalProcessingOption walProcessingOption : WalProcessingOption.values()) { + if (walProcessingOption.value == value) { + return walProcessingOption; + } + } + throw new IllegalArgumentException( + "Illegal value provided for WalProcessingOption: " + value); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WriteBatch.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WriteBatch.java new file mode 100644 index 0000000..49e1f7f --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WriteBatch.java @@ -0,0 +1,394 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.nio.ByteBuffer; + +/** + * WriteBatch holds a collection of updates to apply atomically to a DB. + *

    + * The updates are applied in the order in which they are added + * to the WriteBatch. For example, the value of "key" will be "v3" + * after the following batch is written: + *

    + * batch.put("key", "v1"); + * batch.remove("key"); + * batch.put("key", "v2"); + * batch.put("key", "v3"); + *

    + * Multiple threads can invoke const methods on a WriteBatch without + * external synchronization, but if any of the threads may call a + * non-const method, all threads accessing the same WriteBatch must use + * external synchronization. + */ +public class WriteBatch extends AbstractWriteBatch { + /** + * Constructs a WriteBatch instance. + */ + public WriteBatch() { + this(0); + } + + /** + * Constructs a WriteBatch instance with a given size. + * + * @param reserved_bytes reserved size for WriteBatch + */ + public WriteBatch(final int reserved_bytes) { + super(newWriteBatch(reserved_bytes)); + } + + /** + * Constructs a WriteBatch instance from a serialized representation + * as returned by {@link #data()}. + * + * @param serialized the serialized representation. + */ + public WriteBatch(final byte[] serialized) { + super(newWriteBatch(serialized, serialized.length)); + } + + /** + * Support for iterating over the contents of a batch. + * + * @param handler A handler that is called back for each + * update present in the batch + * + * @throws RocksDBException If we cannot iterate over the batch + */ + public void iterate(final Handler handler) throws RocksDBException { + iterate(nativeHandle_, handler.nativeHandle_); + } + + /** + * Retrieve the serialized version of this batch. + * + * @return the serialized representation of this write batch. + * + * @throws RocksDBException if an error occurs whilst retrieving + * the serialized batch data. + */ + public byte[] data() throws RocksDBException { + return data(nativeHandle_); + } + + /** + * Retrieve data size of the batch. + * + * @return the serialized data size of the batch. + */ + public long getDataSize() { + return getDataSize(nativeHandle_); + } + + /** + * Returns true if Put will be called during Iterate. + * + * @return true if Put will be called during Iterate. + */ + public boolean hasPut() { + return hasPut(nativeHandle_); + } + + /** + * Returns true if Delete will be called during Iterate. + * + * @return true if Delete will be called during Iterate. + */ + public boolean hasDelete() { + return hasDelete(nativeHandle_); + } + + /** + * Returns true if SingleDelete will be called during Iterate. + * + * @return true if SingleDelete will be called during Iterate. + */ + public boolean hasSingleDelete() { + return hasSingleDelete(nativeHandle_); + } + + /** + * Returns true if DeleteRange will be called during Iterate. + * + * @return true if DeleteRange will be called during Iterate. + */ + public boolean hasDeleteRange() { + return hasDeleteRange(nativeHandle_); + } + + /** + * Returns true if Merge will be called during Iterate. + * + * @return true if Merge will be called during Iterate. + */ + public boolean hasMerge() { + return hasMerge(nativeHandle_); + } + + /** + * Returns true if MarkBeginPrepare will be called during Iterate. + * + * @return true if MarkBeginPrepare will be called during Iterate. + */ + public boolean hasBeginPrepare() { + return hasBeginPrepare(nativeHandle_); + } + + /** + * Returns true if MarkEndPrepare will be called during Iterate. + * + * @return true if MarkEndPrepare will be called during Iterate. + */ + public boolean hasEndPrepare() { + return hasEndPrepare(nativeHandle_); + } + + /** + * Returns true if MarkCommit will be called during Iterate. + * + * @return true if MarkCommit will be called during Iterate. + */ + public boolean hasCommit() { + return hasCommit(nativeHandle_); + } + + /** + * Returns true if MarkRollback will be called during Iterate. + * + * @return true if MarkRollback will be called during Iterate. + */ + public boolean hasRollback() { + return hasRollback(nativeHandle_); + } + + @Override + public WriteBatch getWriteBatch() { + return this; + } + + /** + * Marks this point in the WriteBatch as the last record to + * be inserted into the WAL, provided the WAL is enabled. + */ + public void markWalTerminationPoint() { + markWalTerminationPoint(nativeHandle_); + } + + /** + * Gets the WAL termination point. + *

    + * See {@link #markWalTerminationPoint()} + * + * @return the WAL termination point + */ + public SavePoint getWalTerminationPoint() { + return getWalTerminationPoint(nativeHandle_); + } + + @Override + WriteBatch getWriteBatch(final long handle) { + return this; + } + + /** + *

    Private WriteBatch constructor which is used to construct + * WriteBatch instances from C++ side. As the reference to this + * object is also managed from C++ side the handle will be disowned.

    + * + * @param nativeHandle address of native instance. + */ + WriteBatch(final long nativeHandle) { + this(nativeHandle, false); + } + + /** + *

    Private WriteBatch constructor which is used to construct + * WriteBatch instances.

    + * + * @param nativeHandle address of native instance. + * @param owningNativeHandle whether to own this reference from the C++ side or not + */ + WriteBatch(final long nativeHandle, final boolean owningNativeHandle) { + super(nativeHandle); + if(!owningNativeHandle) + disOwnNativeHandle(); + } + + @Override protected final native void disposeInternal(final long handle); + @Override final native int count0(final long handle); + @Override final native void put(final long handle, final byte[] key, + final int keyLen, final byte[] value, final int valueLen); + @Override final native void put(final long handle, final byte[] key, + final int keyLen, final byte[] value, final int valueLen, + final long cfHandle); + @Override + final native void putDirect(final long handle, final ByteBuffer key, final int keyOffset, + final int keyLength, final ByteBuffer value, final int valueOffset, final int valueLength, + final long cfHandle); + @Override final native void merge(final long handle, final byte[] key, + final int keyLen, final byte[] value, final int valueLen); + @Override final native void merge(final long handle, final byte[] key, + final int keyLen, final byte[] value, final int valueLen, + final long cfHandle); + @Override final native void delete(final long handle, final byte[] key, + final int keyLen) throws RocksDBException; + @Override final native void delete(final long handle, final byte[] key, + final int keyLen, final long cfHandle) throws RocksDBException; + @Override final native void singleDelete(final long handle, final byte[] key, + final int keyLen) throws RocksDBException; + @Override final native void singleDelete(final long handle, final byte[] key, + final int keyLen, final long cfHandle) throws RocksDBException; + @Override + final native void deleteDirect(final long handle, final ByteBuffer key, final int keyOffset, + final int keyLength, final long cfHandle) throws RocksDBException; + @Override + final native void deleteRange(final long handle, final byte[] beginKey, final int beginKeyLen, + final byte[] endKey, final int endKeyLen); + @Override + final native void deleteRange(final long handle, final byte[] beginKey, final int beginKeyLen, + final byte[] endKey, final int endKeyLen, final long cfHandle); + @Override final native void putLogData(final long handle, + final byte[] blob, final int blobLen) throws RocksDBException; + @Override final native void clear0(final long handle); + @Override final native void setSavePoint0(final long handle); + @Override final native void rollbackToSavePoint0(final long handle); + @Override final native void popSavePoint(final long handle) throws RocksDBException; + @Override final native void setMaxBytes(final long nativeHandle, + final long maxBytes); + + private static native long newWriteBatch(final int reserved_bytes); + private static native long newWriteBatch(final byte[] serialized, final int serializedLength); + private native void iterate(final long handle, final long handlerHandle) + throws RocksDBException; + private native byte[] data(final long nativeHandle) throws RocksDBException; + private native long getDataSize(final long nativeHandle); + private native boolean hasPut(final long nativeHandle); + private native boolean hasDelete(final long nativeHandle); + private native boolean hasSingleDelete(final long nativeHandle); + private native boolean hasDeleteRange(final long nativeHandle); + private native boolean hasMerge(final long nativeHandle); + private native boolean hasBeginPrepare(final long nativeHandle); + private native boolean hasEndPrepare(final long nativeHandle); + private native boolean hasCommit(final long nativeHandle); + private native boolean hasRollback(final long nativeHandle); + private native void markWalTerminationPoint(final long nativeHandle); + private native SavePoint getWalTerminationPoint(final long nativeHandle); + + /** + * Handler callback for iterating over the contents of a batch. + */ + public abstract static class Handler extends RocksCallbackObject { + public Handler() { + super(0L); + } + + @Override + protected long initializeNative(final long... nativeParameterHandles) { + return createNewHandler0(); + } + + public abstract void put(final int columnFamilyId, final byte[] key, + final byte[] value) throws RocksDBException; + public abstract void put(final byte[] key, final byte[] value); + public abstract void merge(final int columnFamilyId, final byte[] key, + final byte[] value) throws RocksDBException; + public abstract void merge(final byte[] key, final byte[] value); + public abstract void delete(final int columnFamilyId, final byte[] key) + throws RocksDBException; + public abstract void delete(final byte[] key); + public abstract void singleDelete(final int columnFamilyId, + final byte[] key) throws RocksDBException; + public abstract void singleDelete(final byte[] key); + public abstract void deleteRange(final int columnFamilyId, + final byte[] beginKey, final byte[] endKey) throws RocksDBException; + public abstract void deleteRange(final byte[] beginKey, + final byte[] endKey); + public abstract void logData(final byte[] blob); + public abstract void putBlobIndex(final int columnFamilyId, + final byte[] key, final byte[] value) throws RocksDBException; + public abstract void markBeginPrepare() throws RocksDBException; + public abstract void markEndPrepare(final byte[] xid) + throws RocksDBException; + public abstract void markNoop(final boolean emptyBatch) + throws RocksDBException; + public abstract void markRollback(final byte[] xid) + throws RocksDBException; + public abstract void markCommit(final byte[] xid) + throws RocksDBException; + public abstract void markCommitWithTimestamp(final byte[] xid, final byte[] ts) + throws RocksDBException; + + /** + * shouldContinue is called by the underlying iterator + * {@link WriteBatch#iterate(Handler)}. If it returns false, + * iteration is halted. Otherwise, it continues + * iterating. The default implementation always + * returns true. + * + * @return boolean value indicating if the + * iteration is halted. + */ + public boolean shouldContinue() { + return true; + } + + private native long createNewHandler0(); + } + + /** + * A structure for describing the save point in the Write Batch. + */ + public static class SavePoint { + private long size; + private long count; + private long contentFlags; + + public SavePoint(final long size, final long count, + final long contentFlags) { + this.size = size; + this.count = count; + this.contentFlags = contentFlags; + } + + public void clear() { + this.size = 0; + this.count = 0; + this.contentFlags = 0; + } + + /** + * Get the size of the serialized representation. + * + * @return the size of the serialized representation. + */ + public long getSize() { + return size; + } + + /** + * Get the number of elements. + * + * @return the number of elements. + */ + public long getCount() { + return count; + } + + /** + * Get the content flags. + * + * @return the content flags. + */ + public long getContentFlags() { + return contentFlags; + } + + public boolean isCleared() { + return (size | count | contentFlags) == 0; + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WriteBatchInterface.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WriteBatchInterface.java new file mode 100644 index 0000000..32cd8d1 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WriteBatchInterface.java @@ -0,0 +1,283 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.nio.ByteBuffer; + +/** + *

    Defines the interface for a Write Batch which + * holds a collection of updates to apply atomically to a DB.

    + */ +public interface WriteBatchInterface { + + /** + * Returns the number of updates in the batch. + * + * @return number of items in WriteBatch + */ + int count(); + + /** + *

    Store the mapping "key->value" in the database.

    + * + * @param key the specified key to be inserted. + * @param value the value associated with the specified key. + * @throws RocksDBException thrown if error happens in underlying native library. + */ + void put(byte[] key, byte[] value) throws RocksDBException; + + /** + *

    Store the mapping "key->value" within given column + * family.

    + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param key the specified key to be inserted. + * @param value the value associated with the specified key. + * @throws RocksDBException thrown if error happens in underlying native library. + */ + void put(ColumnFamilyHandle columnFamilyHandle, byte[] key, byte[] value) + throws RocksDBException; + + /** + *

    Store the mapping "key->value" within given column + * family.

    + * + * @param key the specified key to be inserted. It is using position and limit. + * Supports direct buffer only. + * @param value the value associated with the specified key. It is using position and limit. + * Supports direct buffer only. + * @throws RocksDBException thrown if error happens in underlying native library. + */ + void put(final ByteBuffer key, final ByteBuffer value) throws RocksDBException; + + /** + *

    Store the mapping "key->value" within given column + * family.

    + * + * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} + * instance + * @param key the specified key to be inserted. It is using position and limit. + * Supports direct buffer only. + * @param value the value associated with the specified key. It is using position and limit. + * Supports direct buffer only. + * @throws RocksDBException thrown if error happens in underlying native library. + */ + void put(ColumnFamilyHandle columnFamilyHandle, final ByteBuffer key, final ByteBuffer value) + throws RocksDBException; + + /** + *

    Merge "value" with the existing value of "key" in the database. + * "key->merge(existing, value)"

    + * + * @param key the specified key to be merged. + * @param value the value to be merged with the current value for + * the specified key. + * @throws RocksDBException thrown if error happens in underlying native library. + */ + void merge(byte[] key, byte[] value) throws RocksDBException; + + /** + *

    Merge "value" with the existing value of "key" in given column family. + * "key->merge(existing, value)"

    + * + * @param columnFamilyHandle {@link ColumnFamilyHandle} instance + * @param key the specified key to be merged. + * @param value the value to be merged with the current value for + * the specified key. + * @throws RocksDBException thrown if error happens in underlying native library. + */ + void merge(ColumnFamilyHandle columnFamilyHandle, byte[] key, byte[] value) + throws RocksDBException; + + /** + *

    If the database contains a mapping for "key", erase it. Else do nothing.

    + * + * @param key Key to delete within database + * @throws RocksDBException thrown if error happens in underlying native library. + */ + void delete(byte[] key) throws RocksDBException; + + /** + *

    If column family contains a mapping for "key", erase it. Else do nothing.

    + * + * @param columnFamilyHandle {@link ColumnFamilyHandle} instance + * @param key Key to delete within database + * @throws RocksDBException thrown if error happens in underlying native library. + */ + void delete(ColumnFamilyHandle columnFamilyHandle, byte[] key) throws RocksDBException; + + /** + *

    If column family contains a mapping for "key", erase it. Else do nothing.

    + * + * @param key Key to delete within database. It is using position and limit. + * Supports direct buffer only. + * + * @throws RocksDBException thrown if error happens in underlying native library. + */ + void delete(final ByteBuffer key) throws RocksDBException; + + /** + *

    If column family contains a mapping for "key", erase it. Else do nothing.

    + * + * @param columnFamilyHandle {@link ColumnFamilyHandle} instance + * @param key Key to delete within database. It is using position and limit. + * Supports direct buffer only. + * + * @throws RocksDBException thrown if error happens in underlying native library. + */ + void delete(ColumnFamilyHandle columnFamilyHandle, final ByteBuffer key) + throws RocksDBException; + + /** + * Remove the database entry for {@code key}. Requires that the key exists + * and was not overwritten. It is not an error if the key did not exist + * in the database. + *

    + * If a key is overwritten (by calling {@link #put(byte[], byte[])} multiple + * times), then the result of calling SingleDelete() on this key is undefined. + * SingleDelete() only behaves correctly if there has been only one Put() + * for this key since the previous call to SingleDelete() for this key. + *

    + * This feature is currently an experimental performance optimization + * for a very specific workload. It is up to the caller to ensure that + * SingleDelete is only used for a key that is not deleted using Delete() or + * written using Merge(). Mixing SingleDelete operations with Deletes and + * Merges can result in undefined behavior. + * + * @param key Key to delete within database + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + @Experimental("Performance optimization for a very specific workload") + void singleDelete(final byte[] key) throws RocksDBException; + + /** + * Remove the database entry for {@code key}. Requires that the key exists + * and was not overwritten. It is not an error if the key did not exist + * in the database. + *

    + * If a key is overwritten (by calling {@link #put(byte[], byte[])} multiple + * times), then the result of calling SingleDelete() on this key is undefined. + * SingleDelete() only behaves correctly if there has been only one Put() + * for this key since the previous call to SingleDelete() for this key. + *

    + * This feature is currently an experimental performance optimization + * for a very specific workload. It is up to the caller to ensure that + * SingleDelete is only used for a key that is not deleted using Delete() or + * written using Merge(). Mixing SingleDelete operations with Deletes and + * Merges can result in undefined behavior. + * + * @param columnFamilyHandle The column family to delete the key from + * @param key Key to delete within database + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + @Experimental("Performance optimization for a very specific workload") + void singleDelete(final ColumnFamilyHandle columnFamilyHandle, final byte[] key) + throws RocksDBException; + + /** + * Removes the database entries in the range ["beginKey", "endKey"), i.e., + * including "beginKey" and excluding "endKey". a non-OK status on error. It + * is not an error if no keys exist in the range ["beginKey", "endKey"). + *

    + * Delete the database entry (if any) for "key". Returns OK on success, and a + * non-OK status on error. It is not an error if "key" did not exist in the + * database. + * + * @param beginKey + * First key to delete within database (included) + * @param endKey + * Last key to delete within database (excluded) + * @throws RocksDBException thrown if error happens in underlying native library. + */ + void deleteRange(byte[] beginKey, byte[] endKey) throws RocksDBException; + + /** + * Removes the database entries in the range ["beginKey", "endKey"), i.e., + * including "beginKey" and excluding "endKey". a non-OK status on error. It + * is not an error if no keys exist in the range ["beginKey", "endKey"). + *

    + * Delete the database entry (if any) for "key". Returns OK on success, and a + * non-OK status on error. It is not an error if "key" did not exist in the + * database. + * + * @param columnFamilyHandle {@link ColumnFamilyHandle} instance + * @param beginKey + * First key to delete within database (included) + * @param endKey + * Last key to delete within database (excluded) + * @throws RocksDBException thrown if error happens in underlying native library. + */ + void deleteRange(ColumnFamilyHandle columnFamilyHandle, byte[] beginKey, byte[] endKey) + throws RocksDBException; + + /** + * Append a blob of arbitrary size to the records in this batch. The blob will + * be stored in the transaction log but not in any other file. In particular, + * it will not be persisted to the SST files. When iterating over this + * WriteBatch, WriteBatch::Handler::LogData will be called with the contents + * of the blob as it is encountered. Blobs, puts, deletes, and merges will be + * encountered in the same order in which they were inserted. The blob will + * NOT consume sequence number(s) and will NOT increase the count of the batch + *

    + * Example application: add timestamps to the transaction log for use in + * replication. + * + * @param blob binary object to be inserted + * @throws RocksDBException thrown if error happens in underlying native library. + */ + void putLogData(byte[] blob) throws RocksDBException; + + /** + * Clear all updates buffered in this batch + */ + void clear(); + + /** + * Records the state of the batch for future calls to RollbackToSavePoint(). + * May be called multiple times to set multiple save points. + */ + void setSavePoint(); + + /** + * Remove all entries in this batch (Put, Merge, Delete, PutLogData) since + * the most recent call to SetSavePoint() and removes the most recent save + * point. + * + * @throws RocksDBException if there is no previous call to SetSavePoint() + */ + void rollbackToSavePoint() throws RocksDBException; + + /** + * Pop the most recent save point. + *

    + * That is to say that it removes the last save point, + * which was set by {@link #setSavePoint()}. + * + * @throws RocksDBException If there is no previous call to + * {@link #setSavePoint()}, an exception with + * {@link Status.Code#NotFound} will be thrown. + */ + void popSavePoint() throws RocksDBException; + + /** + * Set the maximum size of the write batch. + * + * @param maxBytes the maximum size in bytes. + */ + void setMaxBytes(long maxBytes); + + /** + * Get the underlying Write Batch. + * + * @return the underlying WriteBatch. + */ + WriteBatch getWriteBatch(); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WriteBatchWithIndex.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WriteBatchWithIndex.java new file mode 100644 index 0000000..d41be58 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WriteBatchWithIndex.java @@ -0,0 +1,358 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.nio.ByteBuffer; + +/** + * Similar to {@link org.rocksdb.WriteBatch} but with a binary searchable + * index built for all the keys inserted. + *

    + * Calling put, merge, remove or putLogData calls the same function + * as with {@link org.rocksdb.WriteBatch} whilst also building an index. + *

    + * A user can call {@link org.rocksdb.WriteBatchWithIndex#newIterator()} to + * create an iterator over the write batch or + * {@link org.rocksdb.WriteBatchWithIndex#newIteratorWithBase(org.rocksdb.RocksIterator)} + * to get an iterator for the database with Read-Your-Own-Writes like capability + */ +public class WriteBatchWithIndex extends AbstractWriteBatch { + /** + * Creates a WriteBatchWithIndex where no bytes + * are reserved up-front, byte wise comparison is + * used for fallback key comparisons, + * and duplicate keys operations are retained + */ + public WriteBatchWithIndex() { + super(newWriteBatchWithIndex()); + } + + /** + * Creates a WriteBatchWithIndex where no bytes + * are reserved up-front, byte wise comparison is + * used for fallback key comparisons, and duplicate key + * assignment is determined by the constructor argument + * + * @param overwriteKey if true, overwrite the key in the index when + * inserting a duplicate key, in this way an iterator will never + * show two entries with the same key. + */ + public WriteBatchWithIndex(final boolean overwriteKey) { + super(newWriteBatchWithIndex(overwriteKey)); + } + + /** + * Creates a WriteBatchWithIndex + * + * @param fallbackIndexComparator We fall back to this comparator + * to compare keys within a column family if we cannot determine + * the column family and so look up its comparator. + * + * @param reservedBytes reserved bytes in underlying WriteBatch + * + * @param overwriteKey if true, overwrite the key in the index when + * inserting a duplicate key, in this way an iterator will never + * show two entries with the same key. + */ + public WriteBatchWithIndex( + final AbstractComparator + fallbackIndexComparator, final int reservedBytes, + final boolean overwriteKey) { + super(newWriteBatchWithIndex(fallbackIndexComparator.nativeHandle_, + fallbackIndexComparator.getComparatorType().getValue(), reservedBytes, + overwriteKey)); + } + + /** + *

    Private WriteBatchWithIndex constructor which is used to construct + * WriteBatchWithIndex instances from C++ side. As the reference to this + * object is also managed from C++ side the handle will be disowned.

    + * + * @param nativeHandle address of native instance. + */ + WriteBatchWithIndex(final long nativeHandle) { + super(nativeHandle); + disOwnNativeHandle(); + } + + /** + * Create an iterator of a column family. User can call + * {@link org.rocksdb.RocksIteratorInterface#seek(byte[])} to + * search to the next entry of or after a key. Keys will be iterated in the + * order given by index_comparator. For multiple updates on the same key, + * each update will be returned as a separate entry, in the order of update + * time. + * + * @param columnFamilyHandle The column family to iterate over + * @return An iterator for the Write Batch contents, restricted to the column + * family + */ + public WBWIRocksIterator newIterator( + final ColumnFamilyHandle columnFamilyHandle) { + return new WBWIRocksIterator(this, iterator1(nativeHandle_, + columnFamilyHandle.nativeHandle_)); + } + + /** + * Create an iterator of the default column family. User can call + * {@link org.rocksdb.RocksIteratorInterface#seek(byte[])} to + * search to the next entry of or after a key. Keys will be iterated in the + * order given by index_comparator. For multiple updates on the same key, + * each update will be returned as a separate entry, in the order of update + * time. + * + * @return An iterator for the Write Batch contents + */ + public WBWIRocksIterator newIterator() { + return new WBWIRocksIterator(this, iterator0(nativeHandle_)); + } + + /** + * Provides Read-Your-Own-Writes like functionality by + * creating a new Iterator that will use {@link org.rocksdb.WBWIRocksIterator} + * as a delta and baseIterator as a base + *

    + * Updating write batch with the current key of the iterator is not safe. + * We strongly recommend users not to do it. It will invalidate the current + * key() and value() of the iterator. This invalidation happens even before + * the write batch update finishes. The state may recover after Next() is + * called. + * + * @param columnFamilyHandle The column family to iterate over + * @param baseIterator The base iterator, + * e.g. {@link org.rocksdb.RocksDB#newIterator()} + * @return An iterator which shows a view comprised of both the database + * point-in-time from baseIterator and modifications made in this write batch. + */ + public RocksIterator newIteratorWithBase( + final ColumnFamilyHandle columnFamilyHandle, + final RocksIterator baseIterator) { + return newIteratorWithBase(columnFamilyHandle, baseIterator, null); + } + + /** + * Provides Read-Your-Own-Writes like functionality by + * creating a new Iterator that will use {@link org.rocksdb.WBWIRocksIterator} + * as a delta and baseIterator as a base + *

    + * Updating write batch with the current key of the iterator is not safe. + * We strongly recommend users not to do it. It will invalidate the current + * key() and value() of the iterator. This invalidation happens even before + * the write batch update finishes. The state may recover after Next() is + * called. + * + * @param columnFamilyHandle The column family to iterate over + * @param baseIterator The base iterator, + * e.g. {@link org.rocksdb.RocksDB#newIterator()} + * @param readOptions the read options, or null + * @return An iterator which shows a view comprised of both the database + * point-in-time from baseIterator and modifications made in this write batch. + */ + public RocksIterator newIteratorWithBase(final ColumnFamilyHandle columnFamilyHandle, + final RocksIterator baseIterator, /* @Nullable */ final ReadOptions readOptions) { + final RocksIterator iterator = new RocksIterator(baseIterator.parent_, + iteratorWithBase(nativeHandle_, columnFamilyHandle.nativeHandle_, + baseIterator.nativeHandle_, readOptions == null ? 0 : readOptions.nativeHandle_)); + + // when the iterator is deleted it will also delete the baseIterator + baseIterator.disOwnNativeHandle(); + + return iterator; + } + + /** + * Provides Read-Your-Own-Writes like functionality by + * creating a new Iterator that will use {@link org.rocksdb.WBWIRocksIterator} + * as a delta and baseIterator as a base. Operates on the default column + * family. + * + * @param baseIterator The base iterator, + * e.g. {@link org.rocksdb.RocksDB#newIterator()} + * @return An iterator which shows a view comprised of both the database + * point-in-time from baseIterator and modifications made in this write batch. + */ + public RocksIterator newIteratorWithBase(final RocksIterator baseIterator) { + return newIteratorWithBase(baseIterator.parent_.getDefaultColumnFamily(), baseIterator, null); + } + + /** + * Provides Read-Your-Own-Writes like functionality by + * creating a new Iterator that will use {@link org.rocksdb.WBWIRocksIterator} + * as a delta and baseIterator as a base. Operates on the default column + * family. + * + * @param baseIterator The base iterator, + * e.g. {@link org.rocksdb.RocksDB#newIterator()} + * @param readOptions the read options, or null + * @return An iterator which shows a view comprised of both the database + * point-in-time from baseIterator and modifications made in this write batch. + */ + public RocksIterator newIteratorWithBase(final RocksIterator baseIterator, + /* @Nullable */ final ReadOptions readOptions) { + return newIteratorWithBase( + baseIterator.parent_.getDefaultColumnFamily(), baseIterator, readOptions); + } + + /** + * Similar to {@link RocksDB#get(ColumnFamilyHandle, byte[])} but will only + * read the key from this batch. + * + * @param columnFamilyHandle The column family to retrieve the value from + * @param options The database options to use + * @param key The key to read the value for + * + * @return a byte array storing the value associated with the input key if + * any. null if it does not find the specified key. + * + * @throws RocksDBException if the batch does not have enough data to resolve + * Merge operations, MergeInProgress status may be returned. + */ + public byte[] getFromBatch(final ColumnFamilyHandle columnFamilyHandle, + final DBOptions options, final byte[] key) throws RocksDBException { + return getFromBatch(nativeHandle_, options.nativeHandle_, + key, key.length, columnFamilyHandle.nativeHandle_); + } + + /** + * Similar to {@link RocksDB#get(byte[])} but will only + * read the key from this batch. + * + * @param options The database options to use + * @param key The key to read the value for + * + * @return a byte array storing the value associated with the input key if + * any. null if it does not find the specified key. + * + * @throws RocksDBException if the batch does not have enough data to resolve + * Merge operations, MergeInProgress status may be returned. + */ + public byte[] getFromBatch(final DBOptions options, final byte[] key) + throws RocksDBException { + return getFromBatch(nativeHandle_, options.nativeHandle_, key, key.length); + } + + /** + * Similar to {@link RocksDB#get(ColumnFamilyHandle, byte[])} but will also + * read writes from this batch. + *

    + * This function will query both this batch and the DB and then merge + * the results using the DB's merge operator (if the batch contains any + * merge requests). + *

    + * Setting {@link ReadOptions#setSnapshot(Snapshot)} will affect what is + * read from the DB but will NOT change which keys are read from the batch + * (the keys in this batch do not yet belong to any snapshot and will be + * fetched regardless). + * + * @param db The Rocks database + * @param columnFamilyHandle The column family to retrieve the value from + * @param options The read options to use + * @param key The key to read the value for + * + * @return a byte array storing the value associated with the input key if + * any. null if it does not find the specified key. + * + * @throws RocksDBException if the value for the key cannot be read + */ + public byte[] getFromBatchAndDB(final RocksDB db, final ColumnFamilyHandle columnFamilyHandle, + final ReadOptions options, final byte[] key) throws RocksDBException { + return getFromBatchAndDB(nativeHandle_, db.nativeHandle_, + options.nativeHandle_, key, key.length, + columnFamilyHandle.nativeHandle_); + } + + /** + * Similar to {@link RocksDB#get(byte[])} but will also + * read writes from this batch. + *

    + * This function will query both this batch and the DB and then merge + * the results using the DB's merge operator (if the batch contains any + * merge requests). + *

    + * Setting {@link ReadOptions#setSnapshot(Snapshot)} will affect what is + * read from the DB but will NOT change which keys are read from the batch + * (the keys in this batch do not yet belong to any snapshot and will be + * fetched regardless). + * + * @param db The Rocks database + * @param options The read options to use + * @param key The key to read the value for + * + * @return a byte array storing the value associated with the input key if + * any. null if it does not find the specified key. + * + * @throws RocksDBException if the value for the key cannot be read + */ + public byte[] getFromBatchAndDB(final RocksDB db, final ReadOptions options, + final byte[] key) throws RocksDBException { + return getFromBatchAndDB(nativeHandle_, db.nativeHandle_, + options.nativeHandle_, key, key.length); + } + + @Override protected final native void disposeInternal(final long handle); + @Override final native int count0(final long handle); + @Override final native void put(final long handle, final byte[] key, + final int keyLen, final byte[] value, final int valueLen); + @Override final native void put(final long handle, final byte[] key, + final int keyLen, final byte[] value, final int valueLen, + final long cfHandle); + @Override + final native void putDirect(final long handle, final ByteBuffer key, final int keyOffset, + final int keyLength, final ByteBuffer value, final int valueOffset, final int valueLength, + final long cfHandle); + @Override final native void merge(final long handle, final byte[] key, + final int keyLen, final byte[] value, final int valueLen); + @Override final native void merge(final long handle, final byte[] key, + final int keyLen, final byte[] value, final int valueLen, + final long cfHandle); + @Override final native void delete(final long handle, final byte[] key, + final int keyLen) throws RocksDBException; + @Override final native void delete(final long handle, final byte[] key, + final int keyLen, final long cfHandle) throws RocksDBException; + @Override final native void singleDelete(final long handle, final byte[] key, + final int keyLen) throws RocksDBException; + @Override final native void singleDelete(final long handle, final byte[] key, + final int keyLen, final long cfHandle) throws RocksDBException; + @Override + final native void deleteDirect(final long handle, final ByteBuffer key, final int keyOffset, + final int keyLength, final long cfHandle) throws RocksDBException; + // DO NOT USE - `WriteBatchWithIndex::deleteRange` is not yet supported + @Override + final native void deleteRange(final long handle, final byte[] beginKey, final int beginKeyLen, + final byte[] endKey, final int endKeyLen); + // DO NOT USE - `WriteBatchWithIndex::deleteRange` is not yet supported + @Override + final native void deleteRange(final long handle, final byte[] beginKey, final int beginKeyLen, + final byte[] endKey, final int endKeyLen, final long cfHandle); + @Override final native void putLogData(final long handle, final byte[] blob, + final int blobLen) throws RocksDBException; + @Override final native void clear0(final long handle); + @Override final native void setSavePoint0(final long handle); + @Override final native void rollbackToSavePoint0(final long handle); + @Override final native void popSavePoint(final long handle) throws RocksDBException; + @Override final native void setMaxBytes(final long nativeHandle, + final long maxBytes); + @Override final native WriteBatch getWriteBatch(final long handle); + + private static native long newWriteBatchWithIndex(); + private static native long newWriteBatchWithIndex(final boolean overwriteKey); + private static native long newWriteBatchWithIndex(final long fallbackIndexComparatorHandle, + final byte comparatorType, final int reservedBytes, final boolean overwriteKey); + private native long iterator0(final long handle); + private native long iterator1(final long handle, final long cfHandle); + private native long iteratorWithBase(final long handle, final long cfHandle, + final long baseIteratorHandle, final long readOptionsHandle); + private native byte[] getFromBatch(final long handle, final long optHandle, + final byte[] key, final int keyLen); + private native byte[] getFromBatch(final long handle, final long optHandle, + final byte[] key, final int keyLen, final long cfHandle); + private native byte[] getFromBatchAndDB(final long handle, + final long dbHandle, final long readOptHandle, final byte[] key, + final int keyLen); + private native byte[] getFromBatchAndDB(final long handle, + final long dbHandle, final long readOptHandle, final byte[] key, + final int keyLen, final long cfHandle); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WriteBufferManager.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WriteBufferManager.java new file mode 100644 index 0000000..3364d6e --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WriteBufferManager.java @@ -0,0 +1,50 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Java wrapper over native write_buffer_manager class + */ +public class WriteBufferManager extends RocksObject { + static { + RocksDB.loadLibrary(); + } + + /** + * Construct a new instance of WriteBufferManager. + *

    + * Check + * https://github.com/facebook/rocksdb/wiki/Write-Buffer-Manager + * for more details on when to use it + * + * @param bufferSizeBytes buffer size(in bytes) to use for native write_buffer_manager + * @param cache cache whose memory should be bounded by this write buffer manager + * @param allowStall if set true, it will enable stalling of writes when memory_usage() exceeds + * buffer_size. + * It will wait for flush to complete and memory usage to drop down. + */ + public WriteBufferManager( + final long bufferSizeBytes, final Cache cache, final boolean allowStall) { + super(newWriteBufferManager(bufferSizeBytes, cache.nativeHandle_, allowStall)); + this.allowStall_ = allowStall; + } + + public WriteBufferManager(final long bufferSizeBytes, final Cache cache){ + this(bufferSizeBytes, cache, false); + } + + public boolean allowStall() { + return allowStall_; + } + + private static native long newWriteBufferManager( + final long bufferSizeBytes, final long cacheHandle, final boolean allowStall); + + @Override + protected native void disposeInternal(final long handle); + + private final boolean allowStall_; +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WriteOptions.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WriteOptions.java new file mode 100644 index 0000000..7c184b0 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WriteOptions.java @@ -0,0 +1,255 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Options that control write operations. + *

    + * Note that developers should call WriteOptions.dispose() to release the + * c++ side memory before a WriteOptions instance runs out of scope. + */ +public class WriteOptions extends RocksObject { + /** + * Construct WriteOptions instance. + */ + public WriteOptions() { + super(newWriteOptions()); + + } + + // TODO(AR) consider ownership + WriteOptions(final long nativeHandle) { + super(nativeHandle); + disOwnNativeHandle(); + } + + /** + * Copy constructor for WriteOptions. + *

    + * NOTE: This does a shallow copy, which means comparator, merge_operator, compaction_filter, + * compaction_filter_factory and other pointers will be cloned! + * + * @param other The ColumnFamilyOptions to copy. + */ + public WriteOptions(final WriteOptions other) { + super(copyWriteOptions(other.nativeHandle_)); + } + + /** + * If true, the write will be flushed from the operating system + * buffer cache (by calling WritableFile::Sync()) before the write + * is considered complete. If this flag is true, writes will be + * slower. + *

    + * If this flag is false, and the machine crashes, some recent + * writes may be lost. Note that if it is just the process that + * crashes (i.e., the machine does not reboot), no writes will be + * lost even if sync==false. + *

    + * In other words, a DB write with sync==false has similar + * crash semantics as the "write()" system call. A DB write + * with sync==true has similar crash semantics to a "write()" + * system call followed by "fdatasync()". + *

    + * Default: false + * + * @param flag a boolean flag to indicate whether a write + * should be synchronized. + * @return the instance of the current WriteOptions. + */ + public WriteOptions setSync(final boolean flag) { + setSync(nativeHandle_, flag); + return this; + } + + /** + * If true, the write will be flushed from the operating system + * buffer cache (by calling WritableFile::Sync()) before the write + * is considered complete. If this flag is true, writes will be + * slower. + *

    + * If this flag is false, and the machine crashes, some recent + * writes may be lost. Note that if it is just the process that + * crashes (i.e., the machine does not reboot), no writes will be + * lost even if sync==false. + *

    + * In other words, a DB write with sync==false has similar + * crash semantics as the "write()" system call. A DB write + * with sync==true has similar crash semantics to a "write()" + * system call followed by "fdatasync()". + * + * @return boolean value indicating if sync is active. + */ + public boolean sync() { + return sync(nativeHandle_); + } + + /** + * If true, writes will not first go to the write ahead log, + * and the write may got lost after a crash. The backup engine + * relies on write-ahead logs to back up the memtable, so if + * you disable write-ahead logs, you must create backups with + * flush_before_backup=true to avoid losing unflushed memtable data. + * + * @param flag a boolean flag to specify whether to disable + * write-ahead-log on writes. + * @return the instance of the current WriteOptions. + */ + public WriteOptions setDisableWAL(final boolean flag) { + setDisableWAL(nativeHandle_, flag); + return this; + } + + /** + * If true, writes will not first go to the write ahead log, + * and the write may got lost after a crash. The backup engine + * relies on write-ahead logs to back up the memtable, so if + * you disable write-ahead logs, you must create backups with + * flush_before_backup=true to avoid losing unflushed memtable data. + * + * @return boolean value indicating if WAL is disabled. + */ + public boolean disableWAL() { + return disableWAL(nativeHandle_); + } + + /** + * If true and if user is trying to write to column families that don't exist + * (they were dropped), ignore the write (don't return an error). If there + * are multiple writes in a WriteBatch, other writes will succeed. + *

    + * Default: false + * + * @param ignoreMissingColumnFamilies true to ignore writes to column families + * which don't exist + * @return the instance of the current WriteOptions. + */ + public WriteOptions setIgnoreMissingColumnFamilies( + final boolean ignoreMissingColumnFamilies) { + setIgnoreMissingColumnFamilies(nativeHandle_, ignoreMissingColumnFamilies); + return this; + } + + /** + * If true and if user is trying to write to column families that don't exist + * (they were dropped), ignore the write (don't return an error). If there + * are multiple writes in a WriteBatch, other writes will succeed. + *

    + * Default: false + * + * @return true if writes to column families which don't exist are ignored + */ + public boolean ignoreMissingColumnFamilies() { + return ignoreMissingColumnFamilies(nativeHandle_); + } + + /** + * If true and we need to wait or sleep for the write request, fails + * immediately with {@link Status.Code#Incomplete}. + * + * @param noSlowdown true to fail write requests if we need to wait or sleep + * @return the instance of the current WriteOptions. + */ + public WriteOptions setNoSlowdown(final boolean noSlowdown) { + setNoSlowdown(nativeHandle_, noSlowdown); + return this; + } + + /** + * If true and we need to wait or sleep for the write request, fails + * immediately with {@link Status.Code#Incomplete}. + * + * @return true when write requests are failed if we need to wait or sleep + */ + public boolean noSlowdown() { + return noSlowdown(nativeHandle_); + } + + /** + * If true, this write request is of lower priority if compaction is + * behind. In the case that, {@link #noSlowdown()} == true, the request + * will be cancelled immediately with {@link Status.Code#Incomplete} returned. + * Otherwise, it will be slowed down. The slowdown value is determined by + * RocksDB to guarantee it introduces minimum impacts to high priority writes. + *

    + * Default: false + * + * @param lowPri true if the write request should be of lower priority than + * compactions which are behind. + * + * @return the instance of the current WriteOptions. + */ + public WriteOptions setLowPri(final boolean lowPri) { + setLowPri(nativeHandle_, lowPri); + return this; + } + + /** + * Returns true if this write request is of lower priority if compaction is + * behind. + *

    + * See {@link #setLowPri(boolean)}. + * + * @return true if this write request is of lower priority, false otherwise. + */ + public boolean lowPri() { + return lowPri(nativeHandle_); + } + + /** + * If true, this writebatch will maintain the last insert positions of each + * memtable as hints in concurrent write. It can improve write performance + * in concurrent writes if keys in one writebatch are sequential. In + * non-concurrent writes (when {@code concurrent_memtable_writes} is false) this + * option will be ignored. + *

    + * Default: false + * + * @return true if writebatch will maintain the last insert positions of each memtable as hints in + * concurrent write. + */ + public boolean memtableInsertHintPerBatch() { + return memtableInsertHintPerBatch(nativeHandle_); + } + + /** + * If true, this writebatch will maintain the last insert positions of each + * memtable as hints in concurrent write. It can improve write performance + * in concurrent writes if keys in one writebatch are sequential. In + * non-concurrent writes (when {@code concurrent_memtable_writes} is false) this + * option will be ignored. + *

    + * Default: false + * + * @param memtableInsertHintPerBatch true if writebatch should maintain the last insert positions + * of each memtable as hints in concurrent write. + * @return the instance of the current WriteOptions. + */ + public WriteOptions setMemtableInsertHintPerBatch(final boolean memtableInsertHintPerBatch) { + setMemtableInsertHintPerBatch(nativeHandle_, memtableInsertHintPerBatch); + return this; + } + + private static native long newWriteOptions(); + private static native long copyWriteOptions(long handle); + @Override protected final native void disposeInternal(final long handle); + + private native void setSync(long handle, boolean flag); + private native boolean sync(long handle); + private native void setDisableWAL(long handle, boolean flag); + private native boolean disableWAL(long handle); + private native void setIgnoreMissingColumnFamilies(final long handle, + final boolean ignoreMissingColumnFamilies); + private native boolean ignoreMissingColumnFamilies(final long handle); + private native void setNoSlowdown(final long handle, + final boolean noSlowdown); + private native boolean noSlowdown(final long handle); + private native void setLowPri(final long handle, final boolean lowPri); + private native boolean lowPri(final long handle); + private native boolean memtableInsertHintPerBatch(final long handle); + private native void setMemtableInsertHintPerBatch( + final long handle, final boolean memtableInsertHintPerBatch); +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WriteStallCondition.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WriteStallCondition.java new file mode 100644 index 0000000..98d9e2c --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WriteStallCondition.java @@ -0,0 +1,44 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +public enum WriteStallCondition { + DELAYED((byte) 0x0), + STOPPED((byte) 0x1), + NORMAL((byte) 0x2); + + private final byte value; + + WriteStallCondition(final byte value) { + this.value = value; + } + + /** + * Get the internal representation. + * + * @return the internal representation + */ + byte getValue() { + return value; + } + + /** + * Get the WriteStallCondition from the internal representation value. + * + * @return the flush reason. + * + * @throws IllegalArgumentException if the value is unknown. + */ + static WriteStallCondition fromValue(final byte value) { + for (final WriteStallCondition writeStallCondition : WriteStallCondition.values()) { + if (writeStallCondition.value == value) { + return writeStallCondition; + } + } + + throw new IllegalArgumentException("Illegal value provided for WriteStallCondition: " + value); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WriteStallInfo.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WriteStallInfo.java new file mode 100644 index 0000000..1cade0a --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/WriteStallInfo.java @@ -0,0 +1,75 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Objects; + +public class WriteStallInfo { + private final String columnFamilyName; + private final WriteStallCondition currentCondition; + private final WriteStallCondition previousCondition; + + /** + * Access is package private as this will only be constructed from + * C++ via JNI and for testing. + */ + WriteStallInfo(final String columnFamilyName, final byte currentConditionValue, + final byte previousConditionValue) { + this.columnFamilyName = columnFamilyName; + this.currentCondition = WriteStallCondition.fromValue(currentConditionValue); + this.previousCondition = WriteStallCondition.fromValue(previousConditionValue); + } + + /** + * Get the name of the column family. + * + * @return the name of the column family. + */ + public String getColumnFamilyName() { + return columnFamilyName; + } + + /** + * Get the current state of the write controller. + * + * @return the current state. + */ + public WriteStallCondition getCurrentCondition() { + return currentCondition; + } + + /** + * Get the previous state of the write controller. + * + * @return the previous state. + */ + public WriteStallCondition getPreviousCondition() { + return previousCondition; + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final WriteStallInfo that = (WriteStallInfo) o; + return Objects.equals(columnFamilyName, that.columnFamilyName) + && currentCondition == that.currentCondition && previousCondition == that.previousCondition; + } + + @Override + public int hashCode() { + return Objects.hash(columnFamilyName, currentCondition, previousCondition); + } + + @Override + public String toString() { + return "WriteStallInfo{" + + "columnFamilyName='" + columnFamilyName + '\'' + ", currentCondition=" + currentCondition + + ", previousCondition=" + previousCondition + '}'; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/util/ByteUtil.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/util/ByteUtil.java new file mode 100644 index 0000000..5d64d5d --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/util/ByteUtil.java @@ -0,0 +1,52 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb.util; + +import java.nio.ByteBuffer; + +import static java.nio.charset.StandardCharsets.UTF_8; + +public class ByteUtil { + + /** + * Convert a String to a UTF-8 byte array. + * + * @param str the string + * + * @return the byte array. + */ + public static byte[] bytes(final String str) { + return str.getBytes(UTF_8); + } + + /** + * Compares the first {@code count} bytes of two areas of memory. Returns + * zero if they are the same, a value less than zero if {@code x} is + * lexically less than {@code y}, or a value greater than zero if {@code x} + * is lexically greater than {@code y}. Note that lexical order is determined + * as if comparing unsigned char arrays. + * + * Similar to memcmp.c. + * + * @param x the first value to compare with + * @param y the second value to compare against + * @param count the number of bytes to compare + * + * @return the result of the comparison + */ + public static int memcmp(final ByteBuffer x, final ByteBuffer y, + final int count) { + for (int idx = 0; idx < count; idx++) { + final int aa = x.get(idx) & 0xff; + final int bb = y.get(idx) & 0xff; + if (aa != bb) { + return aa - bb; + } + } + return 0; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/util/BytewiseComparator.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/util/BytewiseComparator.java new file mode 100644 index 0000000..9561b0a --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/util/BytewiseComparator.java @@ -0,0 +1,121 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb.util; + +import org.rocksdb.*; + +import java.nio.ByteBuffer; + +import static org.rocksdb.util.ByteUtil.memcmp; + +/** + * This is a Java Native implementation of the C++ + * equivalent BytewiseComparatorImpl using {@link Slice} + * + * The performance of Comparators implemented in Java is always + * less than their C++ counterparts due to the bridging overhead, + * as such you likely don't want to use this apart from benchmarking + * and you most likely instead wanted + * {@link org.rocksdb.BuiltinComparator#BYTEWISE_COMPARATOR} + */ +public final class BytewiseComparator extends AbstractComparator { + + public BytewiseComparator(final ComparatorOptions copt) { + super(copt); + } + + @Override + public String name() { + return "rocksdb.java.BytewiseComparator"; + } + + @Override + public int compare(final ByteBuffer a, final ByteBuffer b) { + return _compare(a, b); + } + + static int _compare(final ByteBuffer a, final ByteBuffer b) { + assert(a != null && b != null); + final int minLen = a.remaining() < b.remaining() ? + a.remaining() : b.remaining(); + int r = memcmp(a, b, minLen); + if (r == 0) { + if (a.remaining() < b.remaining()) { + r = -1; + } else if (a.remaining() > b.remaining()) { + r = +1; + } + } + return r; + } + + @Override + public void findShortestSeparator(final ByteBuffer start, + final ByteBuffer limit) { + // Find length of common prefix + final int minLength = Math.min(start.remaining(), limit.remaining()); + int diffIndex = 0; + while (diffIndex < minLength && + start.get(diffIndex) == limit.get(diffIndex)) { + diffIndex++; + } + + if (diffIndex >= minLength) { + // Do not shorten if one string is a prefix of the other + } else { + final int startByte = start.get(diffIndex) & 0xff; + final int limitByte = limit.get(diffIndex) & 0xff; + if (startByte >= limitByte) { + // Cannot shorten since limit is smaller than start or start is + // already the shortest possible. + return; + } + assert(startByte < limitByte); + + if (diffIndex < limit.remaining() - 1 || startByte + 1 < limitByte) { + start.put(diffIndex, (byte)((start.get(diffIndex) & 0xff) + 1)); + start.limit(diffIndex + 1); + } else { + // v + // A A 1 A A A + // A A 2 + // + // Incrementing the current byte will make start bigger than limit, we + // will skip this byte, and find the first non 0xFF byte in start and + // increment it. + diffIndex++; + + while (diffIndex < start.remaining()) { + // Keep moving until we find the first non 0xFF byte to + // increment it + if ((start.get(diffIndex) & 0xff) < + 0xff) { + start.put(diffIndex, (byte)((start.get(diffIndex) & 0xff) + 1)); + start.limit(diffIndex + 1); + break; + } + diffIndex++; + } + } + assert(compare(start.duplicate(), limit.duplicate()) < 0); + } + } + + @Override + public void findShortSuccessor(final ByteBuffer key) { + // Find first character that can be incremented + final int n = key.remaining(); + for (int i = 0; i < n; i++) { + final int byt = key.get(i) & 0xff; + if (byt != 0xff) { + key.put(i, (byte)(byt + 1)); + key.limit(i+1); + return; + } + } + // *key is a run of 0xffs. Leave it alone. + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/util/Environment.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/util/Environment.java new file mode 100644 index 0000000..9ad51c7 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/util/Environment.java @@ -0,0 +1,245 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb.util; + +import java.io.File; +import java.io.IOException; + +public class Environment { + private static String OS = System.getProperty("os.name").toLowerCase(); + private static String ARCH = System.getProperty("os.arch").toLowerCase(); + private static String MUSL_ENVIRONMENT = System.getenv("ROCKSDB_MUSL_LIBC"); + + /** + * Will be lazily initialised by {@link #isMuslLibc()} instead of the previous static + * initialisation. The lazy initialisation prevents Windows from reporting suspicious behaviour of + * the JVM attempting IO on Unix paths. + */ + private static Boolean MUSL_LIBC = null; + + public static boolean isAarch64() { + return ARCH.contains("aarch64"); + } + + public static boolean isPowerPC() { + return ARCH.contains("ppc"); + } + + public static boolean isS390x() { + return ARCH.contains("s390x"); + } + + public static boolean isWindows() { + return (OS.contains("win")); + } + + public static boolean isFreeBSD() { + return (OS.contains("freebsd")); + } + + public static boolean isMac() { + return (OS.contains("mac")); + } + + public static boolean isAix() { + return OS.contains("aix"); + } + + public static boolean isUnix() { + return OS.contains("nix") || + OS.contains("nux"); + } + + /** + * Determine if the environment has a musl libc. + * + * @return true if the environment has a musl libc, false otherwise. + */ + public static boolean isMuslLibc() { + if (MUSL_LIBC == null) { + MUSL_LIBC = initIsMuslLibc(); + } + return MUSL_LIBC; + } + + /** + * Determine if the environment has a musl libc. + * + * The initialisation counterpart of {@link #isMuslLibc()}. + * + * Intentionally package-private for testing. + * + * @return true if the environment has a musl libc, false otherwise. + */ + static boolean initIsMuslLibc() { + // consider explicit user setting from environment first + if ("true".equalsIgnoreCase(MUSL_ENVIRONMENT)) { + return true; + } + if ("false".equalsIgnoreCase(MUSL_ENVIRONMENT)) { + return false; + } + + // check if ldd indicates a muslc lib + try { + final Process p = + new ProcessBuilder("/usr/bin/env", "sh", "-c", "ldd /usr/bin/env | grep -q musl").start(); + if (p.waitFor() == 0) { + return true; + } + } catch (final IOException | InterruptedException e) { + // do nothing, and move on to the next check + } + + final File lib = new File("/lib"); + if (lib.exists() && lib.isDirectory() && lib.canRead()) { + // attempt the most likely musl libc name first + final String possibleMuslcLibName; + if (isPowerPC()) { + possibleMuslcLibName = "libc.musl-ppc64le.so.1"; + } else if (isAarch64()) { + possibleMuslcLibName = "libc.musl-aarch64.so.1"; + } else if (isS390x()) { + possibleMuslcLibName = "libc.musl-s390x.so.1"; + } else { + possibleMuslcLibName = "libc.musl-x86_64.so.1"; + } + final File possibleMuslcLib = new File(lib, possibleMuslcLibName); + if (possibleMuslcLib.exists() && possibleMuslcLib.canRead()) { + return true; + } + + // fallback to scanning for a musl libc + final File[] libFiles = lib.listFiles(); + if (libFiles == null) { + return false; + } + for (final File f : libFiles) { + if (f.getName().startsWith("libc.musl")) { + return true; + } + } + } + + return false; + } + + public static boolean isSolaris() { + return OS.contains("sunos"); + } + + public static boolean isOpenBSD() { + return (OS.contains("openbsd")); + } + + public static boolean is64Bit() { + if (ARCH.indexOf("sparcv9") >= 0) { + return true; + } + return (ARCH.indexOf("64") > 0); + } + + public static String getSharedLibraryName(final String name) { + return name + "jni"; + } + + public static String getSharedLibraryFileName(final String name) { + return appendLibOsSuffix("lib" + getSharedLibraryName(name), true); + } + + /** + * Get the name of the libc implementation + * + * @return the name of the implementation, + * or null if the default for that platform (e.g. glibc on Linux). + */ + public static /* @Nullable */ String getLibcName() { + if (isMuslLibc()) { + return "musl"; + } else { + return null; + } + } + + private static String getLibcPostfix() { + final String libcName = getLibcName(); + if (libcName == null) { + return ""; + } + return "-" + libcName; + } + + public static String getJniLibraryName(final String name) { + if (isUnix()) { + final String arch = is64Bit() ? "64" : "32"; + if (isPowerPC() || isAarch64()) { + return String.format("%sjni-linux-%s%s", name, ARCH, getLibcPostfix()); + } else if (isS390x()) { + return String.format("%sjni-linux-%s", name, ARCH); + } else { + return String.format("%sjni-linux%s%s", name, arch, getLibcPostfix()); + } + } else if (isMac()) { + if (is64Bit()) { + final String arch; + if (isAarch64()) { + arch = "arm64"; + } else { + arch = "x86_64"; + } + return String.format("%sjni-osx-%s", name, arch); + } else { + return String.format("%sjni-osx", name); + } + } else if (isFreeBSD()) { + return String.format("%sjni-freebsd%s", name, is64Bit() ? "64" : "32"); + } else if (isAix() && is64Bit()) { + return String.format("%sjni-aix64", name); + } else if (isSolaris()) { + final String arch = is64Bit() ? "64" : "32"; + return String.format("%sjni-solaris%s", name, arch); + } else if (isWindows() && is64Bit()) { + return String.format("%sjni-win64", name); + } else if (isOpenBSD()) { + return String.format("%sjni-openbsd%s", name, is64Bit() ? "64" : "32"); + } + + throw new UnsupportedOperationException(String.format("Cannot determine JNI library name for ARCH='%s' OS='%s' name='%s'", ARCH, OS, name)); + } + + public static /*@Nullable*/ String getFallbackJniLibraryName(final String name) { + if (isMac() && is64Bit()) { + return String.format("%sjni-osx", name); + } + return null; + } + + public static String getJniLibraryFileName(final String name) { + return appendLibOsSuffix("lib" + getJniLibraryName(name), false); + } + + public static /*@Nullable*/ String getFallbackJniLibraryFileName(final String name) { + final String fallbackJniLibraryName = getFallbackJniLibraryName(name); + if (fallbackJniLibraryName == null) { + return null; + } + return appendLibOsSuffix("lib" + fallbackJniLibraryName, false); + } + + private static String appendLibOsSuffix(final String libraryFileName, final boolean shared) { + if (isUnix() || isAix() || isSolaris() || isFreeBSD() || isOpenBSD()) { + return libraryFileName + ".so"; + } else if (isMac()) { + return libraryFileName + (shared ? ".dylib" : ".jnilib"); + } else if (isWindows()) { + return libraryFileName + ".dll"; + } + throw new UnsupportedOperationException(); + } + + public static String getJniLibraryExtension() { + if (isWindows()) { + return ".dll"; + } + return (isMac()) ? ".jnilib" : ".so"; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/util/IntComparator.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/util/IntComparator.java new file mode 100644 index 0000000..cc096cd --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/util/IntComparator.java @@ -0,0 +1,67 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb.util; + +import org.rocksdb.AbstractComparator; +import org.rocksdb.ComparatorOptions; + +import java.nio.ByteBuffer; + +/** + * This is a Java implementation of a Comparator for Java int + * keys. + * + * This comparator assumes keys are (at least) four bytes, so + * the caller must guarantee that in accessing other APIs in + * combination with this comparator. + * + * The performance of Comparators implemented in Java is always + * less than their C++ counterparts due to the bridging overhead, + * as such you likely don't want to use this apart from benchmarking + * or testing. + */ +public final class IntComparator extends AbstractComparator { + + public IntComparator(final ComparatorOptions copt) { + super(copt); + } + + @Override + public String name() { + return "rocksdb.java.IntComparator"; + } + + @Override + public int compare(final ByteBuffer a, final ByteBuffer b) { + return compareIntKeys(a, b); + } + + /** + * Compares integer keys + * so that they are in ascending order + * + * @param a 4-bytes representing an integer key + * @param b 4-bytes representing an integer key + * + * @return negative if a < b, 0 if a == b, positive otherwise + */ + private final int compareIntKeys(final ByteBuffer a, final ByteBuffer b) { + final int iA = a.getInt(); + final int iB = b.getInt(); + + // protect against int key calculation overflow + final long diff = (long)iA - iB; + final int result; + if (diff < Integer.MIN_VALUE) { + result = Integer.MIN_VALUE; + } else if(diff > Integer.MAX_VALUE) { + result = Integer.MAX_VALUE; + } else { + result = (int)diff; + } + return result; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/util/ReverseBytewiseComparator.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/util/ReverseBytewiseComparator.java new file mode 100644 index 0000000..4c06f80 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/util/ReverseBytewiseComparator.java @@ -0,0 +1,88 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb.util; + +import org.rocksdb.AbstractComparator; +import org.rocksdb.BuiltinComparator; +import org.rocksdb.ComparatorOptions; +import org.rocksdb.Slice; + +import java.nio.ByteBuffer; + +/** + * This is a Java Native implementation of the C++ + * equivalent ReverseBytewiseComparatorImpl using {@link Slice} + * + * The performance of Comparators implemented in Java is always + * less than their C++ counterparts due to the bridging overhead, + * as such you likely don't want to use this apart from benchmarking + * and you most likely instead wanted + * {@link BuiltinComparator#REVERSE_BYTEWISE_COMPARATOR} + */ +public final class ReverseBytewiseComparator extends AbstractComparator { + + public ReverseBytewiseComparator(final ComparatorOptions copt) { + super(copt); + } + + @Override + public String name() { + return "rocksdb.java.ReverseBytewiseComparator"; + } + + @Override + public int compare(final ByteBuffer a, final ByteBuffer b) { + return -BytewiseComparator._compare(a, b); + } + + @Override + public void findShortestSeparator(final ByteBuffer start, + final ByteBuffer limit) { + // Find length of common prefix + final int minLength = Math.min(start.remaining(), limit.remaining()); + int diffIndex = 0; + while (diffIndex < minLength && + start.get(diffIndex) == limit.get(diffIndex)) { + diffIndex++; + } + + assert(diffIndex <= minLength); + if (diffIndex == minLength) { + // Do not shorten if one string is a prefix of the other + // + // We could handle cases like: + // V + // A A 2 X Y + // A A 2 + // in a similar way as BytewiseComparator::FindShortestSeparator(). + // We keep it simple by not implementing it. We can come back to it + // later when needed. + } else { + final int startByte = start.get(diffIndex) & 0xff; + final int limitByte = limit.get(diffIndex) & 0xff; + if (startByte > limitByte && diffIndex < start.remaining() - 1) { + // Case like + // V + // A A 3 A A + // A A 1 B B + // + // or + // v + // A A 2 A A + // A A 1 B B + // In this case "AA2" will be good. +//#ifndef NDEBUG +// std::string old_start = *start; +//#endif + start.limit(diffIndex + 1); +//#ifndef NDEBUG +// assert(old_start >= *start); +//#endif + assert(BytewiseComparator._compare(start.duplicate(), limit.duplicate()) > 0); + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/util/SizeUnit.java b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/util/SizeUnit.java new file mode 100644 index 0000000..0f717e8 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/main/java/org/rocksdb/util/SizeUnit.java @@ -0,0 +1,16 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb.util; + +public class SizeUnit { + public static final long KB = 1024L; + public static final long MB = KB * KB; + public static final long GB = KB * MB; + public static final long TB = KB * GB; + public static final long PB = KB * TB; + + private SizeUnit() {} +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/AbstractTransactionTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/AbstractTransactionTest.java new file mode 100644 index 0000000..d572580 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/AbstractTransactionTest.java @@ -0,0 +1,979 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** + * Base class of {@link TransactionTest} and {@link OptimisticTransactionTest} + */ +public abstract class AbstractTransactionTest { + protected static final byte[] TXN_TEST_COLUMN_FAMILY = "txn_test_cf".getBytes(); + + protected static final Random rand = PlatformRandomHelper. + getPlatformSpecificRandomFactory(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + public abstract DBContainer startDb() + throws RocksDBException; + + @Test + public void setSnapshot() throws RocksDBException { + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + txn.setSnapshot(); + } + } + + @Test + public void setSnapshotOnNextOperation() throws RocksDBException { + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + txn.setSnapshotOnNextOperation(); + txn.put("key1".getBytes(), "value1".getBytes()); + } + } + + @Test + public void setSnapshotOnNextOperation_transactionNotifier() throws RocksDBException { + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + + try(final TestTransactionNotifier notifier = new TestTransactionNotifier()) { + txn.setSnapshotOnNextOperation(notifier); + txn.put("key1".getBytes(), "value1".getBytes()); + + txn.setSnapshotOnNextOperation(notifier); + txn.put("key2".getBytes(), "value2".getBytes()); + + assertThat(notifier.getCreatedSnapshots().size()).isEqualTo(2); + } + } + } + + @Test + public void getSnapshot() throws RocksDBException { + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + txn.setSnapshot(); + final Snapshot snapshot = txn.getSnapshot(); + assertThat(snapshot.isOwningHandle()).isFalse(); + } + } + + @Test + public void getSnapshot_null() throws RocksDBException { + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + final Snapshot snapshot = txn.getSnapshot(); + assertThat(snapshot).isNull(); + } + } + + @Test + public void clearSnapshot() throws RocksDBException { + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + txn.setSnapshot(); + txn.clearSnapshot(); + } + } + + @Test + public void clearSnapshot_none() throws RocksDBException { + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + txn.clearSnapshot(); + } + } + + @Test + public void commit() throws RocksDBException { + final byte[] k1 = "rollback-key1".getBytes(UTF_8); + final byte[] v1 = "rollback-value1".getBytes(UTF_8); + try(final DBContainer dbContainer = startDb()) { + try(final Transaction txn = dbContainer.beginTransaction()) { + txn.put(k1, v1); + txn.commit(); + } + + try(final ReadOptions readOptions = new ReadOptions(); + final Transaction txn2 = dbContainer.beginTransaction()) { + assertThat(txn2.get(readOptions, k1)).isEqualTo(v1); + } + } + } + + @Test + public void rollback() throws RocksDBException { + final byte[] k1 = "rollback-key1".getBytes(UTF_8); + final byte[] v1 = "rollback-value1".getBytes(UTF_8); + try(final DBContainer dbContainer = startDb()) { + try(final Transaction txn = dbContainer.beginTransaction()) { + txn.put(k1, v1); + txn.rollback(); + } + + try(final ReadOptions readOptions = new ReadOptions(); + final Transaction txn2 = dbContainer.beginTransaction()) { + assertThat(txn2.get(readOptions, k1)).isNull(); + } + } + } + + @Test + public void savePoint() throws RocksDBException { + final byte[] k1 = "savePoint-key1".getBytes(UTF_8); + final byte[] v1 = "savePoint-value1".getBytes(UTF_8); + final byte[] k2 = "savePoint-key2".getBytes(UTF_8); + final byte[] v2 = "savePoint-value2".getBytes(UTF_8); + + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions()) { + + + try(final Transaction txn = dbContainer.beginTransaction()) { + txn.put(k1, v1); + + assertThat(txn.get(readOptions, k1)).isEqualTo(v1); + + txn.setSavePoint(); + + txn.put(k2, v2); + + assertThat(txn.get(readOptions, k1)).isEqualTo(v1); + assertThat(txn.get(readOptions, k2)).isEqualTo(v2); + + txn.rollbackToSavePoint(); + + assertThat(txn.get(readOptions, k1)).isEqualTo(v1); + assertThat(txn.get(readOptions, k2)).isNull(); + + txn.commit(); + } + + try(final Transaction txn2 = dbContainer.beginTransaction()) { + assertThat(txn2.get(readOptions, k1)).isEqualTo(v1); + assertThat(txn2.get(readOptions, k2)).isNull(); + } + } + } + + @Test + public void getPut_cf() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + assertThat(txn.get(testCf, readOptions, k1)).isNull(); + txn.put(testCf, k1, v1); + assertThat(txn.get(testCf, readOptions, k1)).isEqualTo(v1); + } + } + + @Test + public void getPut() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.get(readOptions, k1)).isNull(); + txn.put(k1, v1); + assertThat(txn.get(readOptions, k1)).isEqualTo(v1); + } + } + + @Test + public void multiGetPut_cf() throws RocksDBException { + final byte[][] keys = new byte[][] {"key1".getBytes(UTF_8), "key2".getBytes(UTF_8)}; + final byte[][] values = new byte[][] {"value1".getBytes(UTF_8), "value2".getBytes(UTF_8)}; + + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + final List cfList = Arrays.asList(testCf, testCf); + + assertThat(txn.multiGet(readOptions, cfList, keys)).isEqualTo(new byte[][] { null, null }); + + txn.put(testCf, keys[0], values[0]); + txn.put(testCf, keys[1], values[1]); + assertThat(txn.multiGet(readOptions, cfList, keys)).isEqualTo(values); + } + } + + @Test + public void multiGetPutAsList_cf() throws RocksDBException { + final byte[][] keys = new byte[][] {"key1".getBytes(UTF_8), "key2".getBytes(UTF_8)}; + final byte[][] values = new byte[][] {"value1".getBytes(UTF_8), "value2".getBytes(UTF_8)}; + + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + final List cfList = Arrays.asList(testCf, testCf); + + assertThat(txn.multiGetAsList(readOptions, cfList, Arrays.asList(keys))) + .containsExactly(null, null); + + txn.put(testCf, keys[0], values[0]); + txn.put(testCf, keys[1], values[1]); + assertThat(txn.multiGetAsList(readOptions, cfList, Arrays.asList(keys))) + .containsExactly(values); + } + } + + @Test + public void multiGetPut() throws RocksDBException { + final byte[][] keys = new byte[][] {"key1".getBytes(UTF_8), "key2".getBytes(UTF_8)}; + final byte[][] values = new byte[][] {"value1".getBytes(UTF_8), "value2".getBytes(UTF_8)}; + + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + + assertThat(txn.multiGet(readOptions, keys)).isEqualTo(new byte[][] { null, null }); + + txn.put(keys[0], values[0]); + txn.put(keys[1], values[1]); + assertThat(txn.multiGet(readOptions, keys)).isEqualTo(values); + } + } + + @Test + public void multiGetPutAsList() throws RocksDBException { + final byte[][] keys = new byte[][] {"key1".getBytes(UTF_8), "key2".getBytes(UTF_8)}; + final byte[][] values = new byte[][] {"value1".getBytes(UTF_8), "value2".getBytes(UTF_8)}; + + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.multiGetAsList(readOptions, Arrays.asList(keys))).containsExactly(null, null); + + txn.put(keys[0], values[0]); + txn.put(keys[1], values[1]); + assertThat(txn.multiGetAsList(readOptions, Arrays.asList(keys))).containsExactly(values); + } + } + + @Test + public void getForUpdate_cf() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + assertThat(txn.getForUpdate(readOptions, testCf, k1, true)).isNull(); + txn.put(testCf, k1, v1); + assertThat(txn.getForUpdate(readOptions, testCf, k1, true)).isEqualTo(v1); + } + } + + @Test + public void getForUpdate() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.getForUpdate(readOptions, k1, true)).isNull(); + txn.put(k1, v1); + assertThat(txn.getForUpdate(readOptions, k1, true)).isEqualTo(v1); + } + } + + @Test + public void multiGetForUpdate_cf() throws RocksDBException { + final byte[][] keys = new byte[][] {"key1".getBytes(UTF_8), "key2".getBytes(UTF_8)}; + final byte[][] values = new byte[][] {"value1".getBytes(UTF_8), "value2".getBytes(UTF_8)}; + + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + final List cfList = Arrays.asList(testCf, testCf); + + assertThat(txn.multiGetForUpdate(readOptions, cfList, keys)) + .isEqualTo(new byte[][] { null, null }); + + txn.put(testCf, keys[0], values[0]); + txn.put(testCf, keys[1], values[1]); + assertThat(txn.multiGetForUpdate(readOptions, cfList, keys)) + .isEqualTo(values); + } + } + + @Test + public void multiGetForUpdate() throws RocksDBException { + final byte[][] keys = new byte[][] {"key1".getBytes(UTF_8), "key2".getBytes(UTF_8)}; + final byte[][] values = new byte[][] {"value1".getBytes(UTF_8), "value2".getBytes(UTF_8)}; + + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.multiGetForUpdate(readOptions, keys)).isEqualTo(new byte[][]{null, null}); + + txn.put(keys[0], values[0]); + txn.put(keys[1], values[1]); + assertThat(txn.multiGetForUpdate(readOptions, keys)).isEqualTo(values); + } + } + + @Test + public void multiGetForUpdateAsList_cf() throws RocksDBException { + final List keys = Arrays.asList("key1".getBytes(UTF_8), "key2".getBytes(UTF_8)); + final List values = Arrays.asList("value1".getBytes(UTF_8), "value2".getBytes(UTF_8)); + + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + final List cfList = Arrays.asList(testCf, testCf); + + assertThat(txn.multiGetForUpdateAsList(readOptions, cfList, keys)) + .isEqualTo(Arrays.asList(null, null)); + + txn.put(testCf, keys.get(0), values.get(0)); + txn.put(testCf, keys.get(1), values.get(1)); + final List result = txn.multiGetForUpdateAsList(readOptions, cfList, keys); + assertThat(result.size()).isEqualTo(values.size()); + for (int i = 0; i < result.size(); i++) { + assertThat(result.get(i)).isEqualTo(values.get(i)); + } + } + } + + @Test + public void multiGetForUpdateAsList() throws RocksDBException { + final List keys = Arrays.asList("key1".getBytes(UTF_8), "key2".getBytes(UTF_8)); + final List values = Arrays.asList("value1".getBytes(UTF_8), "value2".getBytes(UTF_8)); + + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final List nulls = new ArrayList<>(); + nulls.add(null); + nulls.add(null); + assertThat(txn.multiGetForUpdateAsList(readOptions, keys)).isEqualTo(nulls); + + txn.put(keys.get(0), values.get(0)); + txn.put(keys.get(1), values.get(1)); + final List result = txn.multiGetForUpdateAsList(readOptions, keys); + assertThat(result.size()).isEqualTo(values.size()); + for (int i = 0; i < result.size(); i++) { + assertThat(result.get(i)).isEqualTo(values.get(i)); + } + } + } + + @Test + public void getIterator() throws RocksDBException { + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + + txn.put(k1, v1); + + try(final RocksIterator iterator = txn.getIterator(readOptions)) { + iterator.seek(k1); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo(k1); + assertThat(iterator.value()).isEqualTo(v1); + } + } + } + + @Test + public void getIterator_cf() throws RocksDBException { + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + + txn.put(testCf, k1, v1); + + try(final RocksIterator iterator = txn.getIterator(readOptions, testCf)) { + iterator.seek(k1); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo(k1); + assertThat(iterator.value()).isEqualTo(v1); + } + } + } + + @Test + public void merge_cf() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + txn.merge(testCf, k1, v1); + } + } + + @Test + public void merge() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + txn.merge(k1, v1); + } + } + + + @Test + public void delete_cf() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + txn.put(testCf, k1, v1); + assertThat(txn.get(testCf, readOptions, k1)).isEqualTo(v1); + + txn.delete(testCf, k1); + assertThat(txn.get(testCf, readOptions, k1)).isNull(); + } + } + + @Test + public void delete() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + txn.put(k1, v1); + assertThat(txn.get(readOptions, k1)).isEqualTo(v1); + + txn.delete(k1); + assertThat(txn.get(readOptions, k1)).isNull(); + } + } + + @Test + public void delete_parts_cf() throws RocksDBException { + final byte[][] keyParts = new byte[][] {"ke".getBytes(UTF_8), "y1".getBytes(UTF_8)}; + final byte[][] valueParts = new byte[][] {"val".getBytes(UTF_8), "ue1".getBytes(UTF_8)}; + final byte[] key = concat(keyParts); + final byte[] value = concat(valueParts); + + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + txn.put(testCf, keyParts, valueParts); + assertThat(txn.get(testCf, readOptions, key)).isEqualTo(value); + + txn.delete(testCf, keyParts); + + assertThat(txn.get(testCf, readOptions, key)) + .isNull(); + } + } + + @Test + public void delete_parts() throws RocksDBException { + final byte[][] keyParts = new byte[][] {"ke".getBytes(UTF_8), "y1".getBytes(UTF_8)}; + final byte[][] valueParts = new byte[][] {"val".getBytes(UTF_8), "ue1".getBytes(UTF_8)}; + final byte[] key = concat(keyParts); + final byte[] value = concat(valueParts); + + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + + txn.put(keyParts, valueParts); + + assertThat(txn.get(readOptions, key)).isEqualTo(value); + + txn.delete(keyParts); + + assertThat(txn.get(readOptions, key)).isNull(); + } + } + + @Test + public void getPutUntracked_cf() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + assertThat(txn.get(testCf, readOptions, k1)).isNull(); + txn.putUntracked(testCf, k1, v1); + assertThat(txn.get(testCf, readOptions, k1)).isEqualTo(v1); + } + } + + @Test + public void getPutUntracked() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.get(readOptions, k1)).isNull(); + txn.putUntracked(k1, v1); + assertThat(txn.get(readOptions, k1)).isEqualTo(v1); + } + } + + @Deprecated + @Test + public void multiGetPutUntracked_cf() throws RocksDBException { + final byte[][] keys = new byte[][] {"key1".getBytes(UTF_8), "key2".getBytes(UTF_8)}; + final byte[][] values = new byte[][] {"value1".getBytes(UTF_8), "value2".getBytes(UTF_8)}; + + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + + final List cfList = Arrays.asList(testCf, testCf); + + assertThat(txn.multiGet(readOptions, cfList, keys)).isEqualTo(new byte[][] { null, null }); + txn.putUntracked(testCf, keys[0], values[0]); + txn.putUntracked(testCf, keys[1], values[1]); + assertThat(txn.multiGet(readOptions, cfList, keys)).isEqualTo(values); + } + } + + @Test + public void multiGetPutUntrackedAsList_cf() throws RocksDBException { + final byte[][] keys = new byte[][] {"key1".getBytes(UTF_8), "key2".getBytes(UTF_8)}; + final byte[][] values = new byte[][] {"value1".getBytes(UTF_8), "value2".getBytes(UTF_8)}; + + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + + final List cfList = Arrays.asList(testCf, testCf); + + assertThat(txn.multiGetAsList(readOptions, cfList, Arrays.asList(keys))) + .containsExactly(null, null); + txn.putUntracked(testCf, keys[0], values[0]); + txn.putUntracked(testCf, keys[1], values[1]); + assertThat(txn.multiGetAsList(readOptions, cfList, Arrays.asList(keys))) + .containsExactly(values); + } + } + + @Deprecated + @Test + public void multiGetPutUntracked() throws RocksDBException { + final byte[][] keys = new byte[][] {"key1".getBytes(UTF_8), "key2".getBytes(UTF_8)}; + final byte[][] values = new byte[][] {"value1".getBytes(UTF_8), "value2".getBytes(UTF_8)}; + + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + + assertThat(txn.multiGet(readOptions, keys)).isEqualTo(new byte[][] { null, null }); + txn.putUntracked(keys[0], values[0]); + txn.putUntracked(keys[1], values[1]); + assertThat(txn.multiGet(readOptions, keys)).isEqualTo(values); + } + } + + @Test + public void multiGetPutAsListUntracked() throws RocksDBException { + final byte[][] keys = new byte[][] {"key1".getBytes(UTF_8), "key2".getBytes(UTF_8)}; + final byte[][] values = new byte[][] {"value1".getBytes(UTF_8), "value2".getBytes(UTF_8)}; + + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.multiGetAsList(readOptions, Arrays.asList(keys))).containsExactly(null, null); + txn.putUntracked(keys[0], values[0]); + txn.putUntracked(keys[1], values[1]); + assertThat(txn.multiGetAsList(readOptions, Arrays.asList(keys))).containsExactly(values); + } + } + + @Test + public void mergeUntracked_cf() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + txn.mergeUntracked(testCf, k1, v1); + } + } + + @Test + public void mergeUntracked() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + txn.mergeUntracked(k1, v1); + } + } + + @Test + public void deleteUntracked_cf() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + txn.put(testCf, k1, v1); + assertThat(txn.get(testCf, readOptions, k1)).isEqualTo(v1); + + txn.deleteUntracked(testCf, k1); + assertThat(txn.get(testCf, readOptions, k1)).isNull(); + } + } + + @Test + public void deleteUntracked() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + txn.put(k1, v1); + assertThat(txn.get(readOptions, k1)).isEqualTo(v1); + + txn.deleteUntracked(k1); + assertThat(txn.get(readOptions, k1)).isNull(); + } + } + + @Test + public void deleteUntracked_parts_cf() throws RocksDBException { + final byte[][] keyParts = new byte[][] {"ke".getBytes(UTF_8), "y1".getBytes(UTF_8)}; + final byte[][] valueParts = new byte[][] {"val".getBytes(UTF_8), "ue1".getBytes(UTF_8)}; + final byte[] key = concat(keyParts); + final byte[] value = concat(valueParts); + + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + txn.put(testCf, keyParts, valueParts); + assertThat(txn.get(testCf, readOptions, key)).isEqualTo(value); + + txn.deleteUntracked(testCf, keyParts); + assertThat(txn.get(testCf, readOptions, key)).isNull(); + } + } + + @Test + public void deleteUntracked_parts() throws RocksDBException { + final byte[][] keyParts = new byte[][] {"ke".getBytes(UTF_8), "y1".getBytes(UTF_8)}; + final byte[][] valueParts = new byte[][] {"val".getBytes(UTF_8), "ue1".getBytes(UTF_8)}; + final byte[] key = concat(keyParts); + final byte[] value = concat(valueParts); + + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + txn.put(keyParts, valueParts); + assertThat(txn.get(readOptions, key)).isEqualTo(value); + + txn.deleteUntracked(keyParts); + assertThat(txn.get(readOptions, key)).isNull(); + } + } + + @Test + public void putLogData() throws RocksDBException { + final byte[] blob = "blobby".getBytes(UTF_8); + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + txn.putLogData(blob); + } + } + + @Test + public void enabledDisableIndexing() throws RocksDBException { + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + txn.disableIndexing(); + txn.enableIndexing(); + txn.disableIndexing(); + txn.enableIndexing(); + } + } + + @Test + public void numKeys() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + final byte[] k2 = "key2".getBytes(UTF_8); + final byte[] v2 = "value2".getBytes(UTF_8); + final byte[] k3 = "key3".getBytes(UTF_8); + final byte[] v3 = "value3".getBytes(UTF_8); + + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + txn.put(k1, v1); + txn.put(testCf, k2, v2); + txn.merge(k3, v3); + txn.delete(testCf, k2); + + assertThat(txn.getNumKeys()).isEqualTo(3); + assertThat(txn.getNumPuts()).isEqualTo(2); + assertThat(txn.getNumMerges()).isEqualTo(1); + assertThat(txn.getNumDeletes()).isEqualTo(1); + } + } + + @Test + public void elapsedTime() throws RocksDBException, InterruptedException { + final long preStartTxnTime = System.currentTimeMillis(); + try (final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + Thread.sleep(2); + + final long txnElapsedTime = txn.getElapsedTime(); + assertThat(txnElapsedTime).isLessThan(System.currentTimeMillis() - preStartTxnTime); + assertThat(txnElapsedTime).isGreaterThan(0); + } + } + + @Test + public void getWriteBatch() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + + txn.put(k1, v1); + + final WriteBatchWithIndex writeBatch = txn.getWriteBatch(); + assertThat(writeBatch).isNotNull(); + assertThat(writeBatch.isOwningHandle()).isFalse(); + assertThat(writeBatch.count()).isEqualTo(1); + } + } + + @Test + public void setLockTimeout() throws RocksDBException { + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + txn.setLockTimeout(1000); + } + } + + @Test + public void writeOptions() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + + try(final DBContainer dbContainer = startDb(); + final WriteOptions writeOptions = new WriteOptions() + .setDisableWAL(true) + .setSync(true); + final Transaction txn = dbContainer.beginTransaction(writeOptions)) { + + txn.put(k1, v1); + + WriteOptions txnWriteOptions = txn.getWriteOptions(); + assertThat(txnWriteOptions).isNotNull(); + assertThat(txnWriteOptions.isOwningHandle()).isFalse(); + assertThat(txnWriteOptions).isNotSameAs(writeOptions); + assertThat(txnWriteOptions.disableWAL()).isTrue(); + assertThat(txnWriteOptions.sync()).isTrue(); + + txn.setWriteOptions(txnWriteOptions.setSync(false)); + txnWriteOptions = txn.getWriteOptions(); + assertThat(txnWriteOptions).isNotNull(); + assertThat(txnWriteOptions.isOwningHandle()).isFalse(); + assertThat(txnWriteOptions).isNotSameAs(writeOptions); + assertThat(txnWriteOptions.disableWAL()).isTrue(); + assertThat(txnWriteOptions.sync()).isFalse(); + } + } + + @Test + public void undoGetForUpdate_cf() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + assertThat(txn.getForUpdate(readOptions, testCf, k1, true)).isNull(); + txn.put(testCf, k1, v1); + assertThat(txn.getForUpdate(readOptions, testCf, k1, true)).isEqualTo(v1); + txn.undoGetForUpdate(testCf, k1); + } + } + + @Test + public void undoGetForUpdate() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.getForUpdate(readOptions, k1, true)).isNull(); + txn.put(k1, v1); + assertThat(txn.getForUpdate(readOptions, k1, true)).isEqualTo(v1); + txn.undoGetForUpdate(k1); + } + } + + @Test + public void rebuildFromWriteBatch() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + final byte[] k2 = "key2".getBytes(UTF_8); + final byte[] v2 = "value2".getBytes(UTF_8); + final byte[] k3 = "key3".getBytes(UTF_8); + final byte[] v3 = "value3".getBytes(UTF_8); + + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions(); + final Transaction txn = dbContainer.beginTransaction()) { + + txn.put(k1, v1); + + assertThat(txn.get(readOptions, k1)).isEqualTo(v1); + assertThat(txn.getNumKeys()).isEqualTo(1); + + try(final WriteBatch writeBatch = new WriteBatch()) { + writeBatch.put(k2, v2); + writeBatch.put(k3, v3); + txn.rebuildFromWriteBatch(writeBatch); + + assertThat(txn.get(readOptions, k1)).isEqualTo(v1); + assertThat(txn.get(readOptions, k2)).isEqualTo(v2); + assertThat(txn.get(readOptions, k3)).isEqualTo(v3); + assertThat(txn.getNumKeys()).isEqualTo(3); + } + } + } + + @Test + public void getCommitTimeWriteBatch() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + + txn.put(k1, v1); + final WriteBatch writeBatch = txn.getCommitTimeWriteBatch(); + + assertThat(writeBatch).isNotNull(); + assertThat(writeBatch.isOwningHandle()).isFalse(); + assertThat(writeBatch.count()).isEqualTo(0); + } + } + + @Test + public void logNumber() throws RocksDBException { + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.getLogNumber()).isEqualTo(0); + final long logNumber = rand.nextLong(); + txn.setLogNumber(logNumber); + assertThat(txn.getLogNumber()).isEqualTo(logNumber); + } + } + + private static byte[] concat(final byte[][] bufs) { + int resultLength = 0; + for(final byte[] buf : bufs) { + resultLength += buf.length; + } + + final byte[] result = new byte[resultLength]; + int resultOffset = 0; + for(final byte[] buf : bufs) { + final int srcLength = buf.length; + System.arraycopy(buf, 0, result, resultOffset, srcLength); + resultOffset += srcLength; + } + + return result; + } + + private static class TestTransactionNotifier + extends AbstractTransactionNotifier { + private final List createdSnapshots = new ArrayList<>(); + + @Override + public void snapshotCreated(final Snapshot newSnapshot) { + createdSnapshots.add(newSnapshot); + } + + public List getCreatedSnapshots() { + return createdSnapshots; + } + } + + protected abstract static class DBContainer implements AutoCloseable { + protected final WriteOptions writeOptions; + protected final List columnFamilyHandles; + protected final ColumnFamilyOptions columnFamilyOptions; + protected final DBOptions options; + + public DBContainer(final WriteOptions writeOptions, + final List columnFamilyHandles, + final ColumnFamilyOptions columnFamilyOptions, + final DBOptions options) { + this.writeOptions = writeOptions; + this.columnFamilyHandles = columnFamilyHandles; + this.columnFamilyOptions = columnFamilyOptions; + this.options = options; + } + + public abstract Transaction beginTransaction(); + + public abstract Transaction beginTransaction( + final WriteOptions writeOptions); + + public ColumnFamilyHandle getTestColumnFamily() { + return columnFamilyHandles.get(1); + } + + @Override + public abstract void close(); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/BackupEngineOptionsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/BackupEngineOptionsTest.java new file mode 100644 index 0000000..b07f8d3 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/BackupEngineOptionsTest.java @@ -0,0 +1,297 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Random; + +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +public class BackupEngineOptionsTest { + private static final String ARBITRARY_PATH = System.getProperty("java.io.tmpdir"); + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public ExpectedException exception = ExpectedException.none(); + + public static final Random rand = PlatformRandomHelper. + getPlatformSpecificRandomFactory(); + + @Test + public void backupDir() { + try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) { + assertThat(backupEngineOptions.backupDir()).isEqualTo(ARBITRARY_PATH); + } + } + + @Test + public void env() { + try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) { + assertThat(backupEngineOptions.backupEnv()).isNull(); + + try(final Env env = new RocksMemEnv(Env.getDefault())) { + backupEngineOptions.setBackupEnv(env); + assertThat(backupEngineOptions.backupEnv()).isEqualTo(env); + } + } + } + + @Test + public void shareTableFiles() { + try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) { + final boolean value = rand.nextBoolean(); + backupEngineOptions.setShareTableFiles(value); + assertThat(backupEngineOptions.shareTableFiles()).isEqualTo(value); + } + } + + @Test + public void infoLog() { + try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) { + assertThat(backupEngineOptions.infoLog()).isNull(); + + try (final Options options = new Options(); final Logger logger = new Logger(options) { + @Override + protected void log(final InfoLogLevel infoLogLevel, final String logMsg) {} + }) { + backupEngineOptions.setInfoLog(logger); + assertThat(backupEngineOptions.infoLog()).isEqualTo(logger); + } + } + } + + @Test + public void sync() { + try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) { + final boolean value = rand.nextBoolean(); + backupEngineOptions.setSync(value); + assertThat(backupEngineOptions.sync()).isEqualTo(value); + } + } + + @Test + public void destroyOldData() { + try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) { + final boolean value = rand.nextBoolean(); + backupEngineOptions.setDestroyOldData(value); + assertThat(backupEngineOptions.destroyOldData()).isEqualTo(value); + } + } + + @Test + public void backupLogFiles() { + try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) { + final boolean value = rand.nextBoolean(); + backupEngineOptions.setBackupLogFiles(value); + assertThat(backupEngineOptions.backupLogFiles()).isEqualTo(value); + } + } + + @Test + public void backupRateLimit() { + try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) { + final long value = Math.abs(rand.nextLong()); + backupEngineOptions.setBackupRateLimit(value); + assertThat(backupEngineOptions.backupRateLimit()).isEqualTo(value); + // negative will be mapped to 0 + backupEngineOptions.setBackupRateLimit(-1); + assertThat(backupEngineOptions.backupRateLimit()).isEqualTo(0); + } + } + + @Test + public void backupRateLimiter() { + try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) { + assertThat(backupEngineOptions.backupEnv()).isNull(); + + try(final RateLimiter backupRateLimiter = + new RateLimiter(999)) { + backupEngineOptions.setBackupRateLimiter(backupRateLimiter); + assertThat(backupEngineOptions.backupRateLimiter()).isEqualTo(backupRateLimiter); + } + } + } + + @Test + public void restoreRateLimit() { + try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) { + final long value = Math.abs(rand.nextLong()); + backupEngineOptions.setRestoreRateLimit(value); + assertThat(backupEngineOptions.restoreRateLimit()).isEqualTo(value); + // negative will be mapped to 0 + backupEngineOptions.setRestoreRateLimit(-1); + assertThat(backupEngineOptions.restoreRateLimit()).isEqualTo(0); + } + } + + @Test + public void restoreRateLimiter() { + try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) { + assertThat(backupEngineOptions.backupEnv()).isNull(); + + try(final RateLimiter restoreRateLimiter = + new RateLimiter(911)) { + backupEngineOptions.setRestoreRateLimiter(restoreRateLimiter); + assertThat(backupEngineOptions.restoreRateLimiter()).isEqualTo(restoreRateLimiter); + } + } + } + + @Test + public void shareFilesWithChecksum() { + try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) { + final boolean value = rand.nextBoolean(); + backupEngineOptions.setShareFilesWithChecksum(value); + assertThat(backupEngineOptions.shareFilesWithChecksum()).isEqualTo(value); + } + } + + @Test + public void maxBackgroundOperations() { + try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) { + final int value = rand.nextInt(); + backupEngineOptions.setMaxBackgroundOperations(value); + assertThat(backupEngineOptions.maxBackgroundOperations()).isEqualTo(value); + } + } + + @Test + public void callbackTriggerIntervalSize() { + try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) { + final long value = rand.nextLong(); + backupEngineOptions.setCallbackTriggerIntervalSize(value); + assertThat(backupEngineOptions.callbackTriggerIntervalSize()).isEqualTo(value); + } + } + + @Test + public void failBackupDirIsNull() { + exception.expect(IllegalArgumentException.class); + try (final BackupEngineOptions ignored = new BackupEngineOptions(null)) { + //no-op + } + } + + @Test + public void failBackupDirIfDisposed() { + try (final BackupEngineOptions options = setupUninitializedBackupEngineOptions(exception)) { + options.backupDir(); + } + } + + @Test + public void failSetShareTableFilesIfDisposed() { + try (final BackupEngineOptions options = setupUninitializedBackupEngineOptions(exception)) { + options.setShareTableFiles(true); + } + } + + @Test + public void failShareTableFilesIfDisposed() { + try (final BackupEngineOptions options = setupUninitializedBackupEngineOptions(exception)) { + options.shareTableFiles(); + } + } + + @Test + public void failSetSyncIfDisposed() { + try (final BackupEngineOptions options = setupUninitializedBackupEngineOptions(exception)) { + options.setSync(true); + } + } + + @Test + public void failSyncIfDisposed() { + try (final BackupEngineOptions options = setupUninitializedBackupEngineOptions(exception)) { + options.sync(); + } + } + + @Test + public void failSetDestroyOldDataIfDisposed() { + try (final BackupEngineOptions options = setupUninitializedBackupEngineOptions(exception)) { + options.setDestroyOldData(true); + } + } + + @Test + public void failDestroyOldDataIfDisposed() { + try (final BackupEngineOptions options = setupUninitializedBackupEngineOptions(exception)) { + options.destroyOldData(); + } + } + + @Test + public void failSetBackupLogFilesIfDisposed() { + try (final BackupEngineOptions options = setupUninitializedBackupEngineOptions(exception)) { + options.setBackupLogFiles(true); + } + } + + @Test + public void failBackupLogFilesIfDisposed() { + try (final BackupEngineOptions options = setupUninitializedBackupEngineOptions(exception)) { + options.backupLogFiles(); + } + } + + @Test + public void failSetBackupRateLimitIfDisposed() { + try (final BackupEngineOptions options = setupUninitializedBackupEngineOptions(exception)) { + options.setBackupRateLimit(1); + } + } + + @Test + public void failBackupRateLimitIfDisposed() { + try (final BackupEngineOptions options = setupUninitializedBackupEngineOptions(exception)) { + options.backupRateLimit(); + } + } + + @Test + public void failSetRestoreRateLimitIfDisposed() { + try (final BackupEngineOptions options = setupUninitializedBackupEngineOptions(exception)) { + options.setRestoreRateLimit(1); + } + } + + @Test + public void failRestoreRateLimitIfDisposed() { + try (final BackupEngineOptions options = setupUninitializedBackupEngineOptions(exception)) { + options.restoreRateLimit(); + } + } + + @Test + public void failSetShareFilesWithChecksumIfDisposed() { + try (final BackupEngineOptions options = setupUninitializedBackupEngineOptions(exception)) { + options.setShareFilesWithChecksum(true); + } + } + + @Test + public void failShareFilesWithChecksumIfDisposed() { + try (final BackupEngineOptions options = setupUninitializedBackupEngineOptions(exception)) { + options.shareFilesWithChecksum(); + } + } + + private BackupEngineOptions setupUninitializedBackupEngineOptions( + final ExpectedException exception) { + final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH); + backupEngineOptions.close(); + exception.expect(AssertionError.class); + return backupEngineOptions; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/BackupEngineTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/BackupEngineTest.java new file mode 100644 index 0000000..67145f8 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/BackupEngineTest.java @@ -0,0 +1,261 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; + +import static org.assertj.core.api.Assertions.assertThat; + +public class BackupEngineTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Rule + public TemporaryFolder backupFolder = new TemporaryFolder(); + + @Test + public void backupDb() throws RocksDBException { + // Open empty database. + try(final Options opt = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + + // Fill database with some test values + prepareDatabase(db); + + // Create two backups + try (final BackupEngineOptions bopt = + new BackupEngineOptions(backupFolder.getRoot().getAbsolutePath()); + final BackupEngine be = BackupEngine.open(opt.getEnv(), bopt)) { + be.createNewBackup(db, false); + be.createNewBackup(db, true); + verifyNumberOfValidBackups(be, 2); + } + } + } + + @Test + public void deleteBackup() throws RocksDBException { + // Open empty database. + try(final Options opt = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + // Fill database with some test values + prepareDatabase(db); + // Create two backups + try (final BackupEngineOptions bopt = + new BackupEngineOptions(backupFolder.getRoot().getAbsolutePath()); + final BackupEngine be = BackupEngine.open(opt.getEnv(), bopt)) { + be.createNewBackup(db, false); + be.createNewBackup(db, true); + final List backupInfo = + verifyNumberOfValidBackups(be, 2); + // Delete the first backup + be.deleteBackup(backupInfo.get(0).backupId()); + final List newBackupInfo = + verifyNumberOfValidBackups(be, 1); + + // The second backup must remain. + assertThat(newBackupInfo.get(0).backupId()). + isEqualTo(backupInfo.get(1).backupId()); + } + } + } + + @Test + public void purgeOldBackups() throws RocksDBException { + // Open empty database. + try(final Options opt = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + // Fill database with some test values + prepareDatabase(db); + // Create four backups + try (final BackupEngineOptions bopt = + new BackupEngineOptions(backupFolder.getRoot().getAbsolutePath()); + final BackupEngine be = BackupEngine.open(opt.getEnv(), bopt)) { + be.createNewBackup(db, false); + be.createNewBackup(db, true); + be.createNewBackup(db, true); + be.createNewBackup(db, true); + final List backupInfo = + verifyNumberOfValidBackups(be, 4); + // Delete everything except the latest backup + be.purgeOldBackups(1); + final List newBackupInfo = + verifyNumberOfValidBackups(be, 1); + // The latest backup must remain. + assertThat(newBackupInfo.get(0).backupId()). + isEqualTo(backupInfo.get(3).backupId()); + } + } + } + + @Test + public void restoreLatestBackup() throws RocksDBException { + try(final Options opt = new Options().setCreateIfMissing(true)) { + // Open empty database. + RocksDB db = null; + try { + db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath()); + // Fill database with some test values + prepareDatabase(db); + + try (final BackupEngineOptions bopt = + new BackupEngineOptions(backupFolder.getRoot().getAbsolutePath()); + final BackupEngine be = BackupEngine.open(opt.getEnv(), bopt)) { + be.createNewBackup(db, true); + verifyNumberOfValidBackups(be, 1); + db.put("key1".getBytes(), "valueV2".getBytes()); + db.put("key2".getBytes(), "valueV2".getBytes()); + be.createNewBackup(db, true); + verifyNumberOfValidBackups(be, 2); + db.put("key1".getBytes(), "valueV3".getBytes()); + db.put("key2".getBytes(), "valueV3".getBytes()); + assertThat(new String(db.get("key1".getBytes()))).endsWith("V3"); + assertThat(new String(db.get("key2".getBytes()))).endsWith("V3"); + + db.close(); + db = null; + + verifyNumberOfValidBackups(be, 2); + // restore db from latest backup + try(final RestoreOptions ropts = new RestoreOptions(false)) { + be.restoreDbFromLatestBackup(dbFolder.getRoot().getAbsolutePath(), + dbFolder.getRoot().getAbsolutePath(), ropts); + } + + // Open database again. + db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath()); + + // Values must have suffix V2 because of restoring latest backup. + assertThat(new String(db.get("key1".getBytes()))).endsWith("V2"); + assertThat(new String(db.get("key2".getBytes()))).endsWith("V2"); + } + } finally { + if(db != null) { + db.close(); + } + } + } + } + + @Test + public void restoreFromBackup() + throws RocksDBException { + try(final Options opt = new Options().setCreateIfMissing(true)) { + RocksDB db = null; + try { + // Open empty database. + db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath()); + // Fill database with some test values + prepareDatabase(db); + try (final BackupEngineOptions bopt = + new BackupEngineOptions(backupFolder.getRoot().getAbsolutePath()); + final BackupEngine be = BackupEngine.open(opt.getEnv(), bopt)) { + be.createNewBackup(db, true); + verifyNumberOfValidBackups(be, 1); + db.put("key1".getBytes(), "valueV2".getBytes()); + db.put("key2".getBytes(), "valueV2".getBytes()); + be.createNewBackup(db, true); + verifyNumberOfValidBackups(be, 2); + db.put("key1".getBytes(), "valueV3".getBytes()); + db.put("key2".getBytes(), "valueV3".getBytes()); + assertThat(new String(db.get("key1".getBytes()))).endsWith("V3"); + assertThat(new String(db.get("key2".getBytes()))).endsWith("V3"); + + //close the database + db.close(); + db = null; + + //restore the backup + final List backupInfo = verifyNumberOfValidBackups(be, 2); + // restore db from first backup + be.restoreDbFromBackup(backupInfo.get(0).backupId(), + dbFolder.getRoot().getAbsolutePath(), + dbFolder.getRoot().getAbsolutePath(), + new RestoreOptions(false)); + // Open database again. + db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath()); + // Values must have suffix V2 because of restoring latest backup. + assertThat(new String(db.get("key1".getBytes()))).endsWith("V1"); + assertThat(new String(db.get("key2".getBytes()))).endsWith("V1"); + } + } finally { + if(db != null) { + db.close(); + } + } + } + } + + @Test + public void backupDbWithMetadata() throws RocksDBException { + // Open empty database. + try (final Options opt = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + // Fill database with some test values + prepareDatabase(db); + + // Create two backups + try (final BackupEngineOptions bopt = + new BackupEngineOptions(backupFolder.getRoot().getAbsolutePath()); + final BackupEngine be = BackupEngine.open(opt.getEnv(), bopt)) { + final String metadata = String.valueOf(ThreadLocalRandom.current().nextInt()); + be.createNewBackupWithMetadata(db, metadata, true); + final List backupInfoList = verifyNumberOfValidBackups(be, 1); + assertThat(backupInfoList.get(0).appMetadata()).isEqualTo(metadata); + } + } + } + + /** + * Verify backups. + * + * @param be {@link BackupEngine} instance. + * @param expectedNumberOfBackups numerical value + * @throws RocksDBException thrown if an error occurs within the native + * part of the library. + */ + private List verifyNumberOfValidBackups(final BackupEngine be, + final int expectedNumberOfBackups) throws RocksDBException { + // Verify that backups exist + assertThat(be.getCorruptedBackups().length). + isEqualTo(0); + be.garbageCollect(); + final List backupInfo = be.getBackupInfo(); + assertThat(backupInfo.size()). + isEqualTo(expectedNumberOfBackups); + return backupInfo; + } + + /** + * Fill database with some test values. + * + * @param db {@link RocksDB} instance. + * @throws RocksDBException thrown if an error occurs within the native + * part of the library. + */ + private void prepareDatabase(final RocksDB db) + throws RocksDBException { + db.put("key1".getBytes(), "valueV1".getBytes()); + db.put("key2".getBytes(), "valueV1".getBytes()); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/BlobOptionsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/BlobOptionsTest.java new file mode 100644 index 0000000..a0a2af8 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/BlobOptionsTest.java @@ -0,0 +1,353 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.*; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class BlobOptionsTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + final int minBlobSize = 65536; + final int largeBlobSize = 65536 * 2; + + /** + * Count the files in the temporary folder which end with a particular suffix + * Used to query the state of a test database to check if it is as the test expects + * + * @param endsWith the suffix to match + * @return the number of files with a matching suffix + */ + @SuppressWarnings("CallToStringConcatCanBeReplacedByOperator") + private int countDBFiles(final String endsWith) { + return Objects.requireNonNull(dbFolder.getRoot().list((dir, name) -> name.endsWith(endsWith))) + .length; + } + + @SuppressWarnings("SameParameterValue") + private byte[] small_key(final String suffix) { + return ("small_key_" + suffix).getBytes(UTF_8); + } + + @SuppressWarnings("SameParameterValue") + private byte[] small_value(final String suffix) { + return ("small_value_" + suffix).getBytes(UTF_8); + } + + private byte[] large_key(final String suffix) { + return ("large_key_" + suffix).getBytes(UTF_8); + } + + private byte[] large_value(final String repeat) { + final byte[] large_value = ("" + repeat + "_" + largeBlobSize + "b").getBytes(UTF_8); + final byte[] large_buffer = new byte[largeBlobSize]; + for (int pos = 0; pos < largeBlobSize; pos += large_value.length) { + final int numBytes = Math.min(large_value.length, large_buffer.length - pos); + System.arraycopy(large_value, 0, large_buffer, pos, numBytes); + } + return large_buffer; + } + + @Test + public void blobOptions() { + try (final Options options = new Options()) { + assertThat(options.enableBlobFiles()).isEqualTo(false); + assertThat(options.minBlobSize()).isEqualTo(0); + assertThat(options.blobCompressionType()).isEqualTo(CompressionType.NO_COMPRESSION); + assertThat(options.enableBlobGarbageCollection()).isEqualTo(false); + assertThat(options.blobFileSize()).isEqualTo(268435456L); + assertThat(options.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25); + assertThat(options.blobGarbageCollectionForceThreshold()).isEqualTo(1.0); + assertThat(options.blobCompactionReadaheadSize()).isEqualTo(0); + assertThat(options.prepopulateBlobCache()) + .isEqualTo(PrepopulateBlobCache.PREPOPULATE_BLOB_DISABLE); + + assertThat(options.setEnableBlobFiles(true)).isEqualTo(options); + assertThat(options.setMinBlobSize(132768L)).isEqualTo(options); + assertThat(options.setBlobCompressionType(CompressionType.BZLIB2_COMPRESSION)) + .isEqualTo(options); + assertThat(options.setEnableBlobGarbageCollection(true)).isEqualTo(options); + assertThat(options.setBlobFileSize(132768L)).isEqualTo(options); + assertThat(options.setBlobGarbageCollectionAgeCutoff(0.89)).isEqualTo(options); + assertThat(options.setBlobGarbageCollectionForceThreshold(0.80)).isEqualTo(options); + assertThat(options.setBlobCompactionReadaheadSize(262144L)).isEqualTo(options); + assertThat(options.setBlobFileStartingLevel(0)).isEqualTo(options); + assertThat(options.setPrepopulateBlobCache(PrepopulateBlobCache.PREPOPULATE_BLOB_FLUSH_ONLY)) + .isEqualTo(options); + + assertThat(options.enableBlobFiles()).isEqualTo(true); + assertThat(options.minBlobSize()).isEqualTo(132768L); + assertThat(options.blobCompressionType()).isEqualTo(CompressionType.BZLIB2_COMPRESSION); + assertThat(options.enableBlobGarbageCollection()).isEqualTo(true); + assertThat(options.blobFileSize()).isEqualTo(132768L); + assertThat(options.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89); + assertThat(options.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); + assertThat(options.blobCompactionReadaheadSize()).isEqualTo(262144L); + assertThat(options.blobFileStartingLevel()).isEqualTo(0); + assertThat(options.prepopulateBlobCache()) + .isEqualTo(PrepopulateBlobCache.PREPOPULATE_BLOB_FLUSH_ONLY); + } + } + + @Test + public void blobColumnFamilyOptions() { + try (final ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions()) { + assertThat(columnFamilyOptions.enableBlobFiles()).isEqualTo(false); + assertThat(columnFamilyOptions.minBlobSize()).isEqualTo(0); + assertThat(columnFamilyOptions.blobCompressionType()) + .isEqualTo(CompressionType.NO_COMPRESSION); + assertThat(columnFamilyOptions.enableBlobGarbageCollection()).isEqualTo(false); + assertThat(columnFamilyOptions.blobFileSize()).isEqualTo(268435456L); + assertThat(columnFamilyOptions.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25); + assertThat(columnFamilyOptions.blobGarbageCollectionForceThreshold()).isEqualTo(1.0); + assertThat(columnFamilyOptions.blobCompactionReadaheadSize()).isEqualTo(0); + + assertThat(columnFamilyOptions.setEnableBlobFiles(true)).isEqualTo(columnFamilyOptions); + assertThat(columnFamilyOptions.setMinBlobSize(132768L)).isEqualTo(columnFamilyOptions); + assertThat(columnFamilyOptions.setBlobCompressionType(CompressionType.BZLIB2_COMPRESSION)) + .isEqualTo(columnFamilyOptions); + assertThat(columnFamilyOptions.setEnableBlobGarbageCollection(true)) + .isEqualTo(columnFamilyOptions); + assertThat(columnFamilyOptions.setBlobFileSize(132768L)).isEqualTo(columnFamilyOptions); + assertThat(columnFamilyOptions.setBlobGarbageCollectionAgeCutoff(0.89)) + .isEqualTo(columnFamilyOptions); + assertThat(columnFamilyOptions.setBlobGarbageCollectionForceThreshold(0.80)) + .isEqualTo(columnFamilyOptions); + assertThat(columnFamilyOptions.setBlobCompactionReadaheadSize(262144L)) + .isEqualTo(columnFamilyOptions); + assertThat(columnFamilyOptions.setBlobFileStartingLevel(0)).isEqualTo(columnFamilyOptions); + assertThat(columnFamilyOptions.setPrepopulateBlobCache( + PrepopulateBlobCache.PREPOPULATE_BLOB_DISABLE)) + .isEqualTo(columnFamilyOptions); + + assertThat(columnFamilyOptions.enableBlobFiles()).isEqualTo(true); + assertThat(columnFamilyOptions.minBlobSize()).isEqualTo(132768L); + assertThat(columnFamilyOptions.blobCompressionType()) + .isEqualTo(CompressionType.BZLIB2_COMPRESSION); + assertThat(columnFamilyOptions.enableBlobGarbageCollection()).isEqualTo(true); + assertThat(columnFamilyOptions.blobFileSize()).isEqualTo(132768L); + assertThat(columnFamilyOptions.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89); + assertThat(columnFamilyOptions.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); + assertThat(columnFamilyOptions.blobCompactionReadaheadSize()).isEqualTo(262144L); + assertThat(columnFamilyOptions.blobFileStartingLevel()).isEqualTo(0); + assertThat(columnFamilyOptions.prepopulateBlobCache()) + .isEqualTo(PrepopulateBlobCache.PREPOPULATE_BLOB_DISABLE); + } + } + + @Test + public void blobMutableColumnFamilyOptionsBuilder() { + final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder = + MutableColumnFamilyOptions.builder(); + builder.setEnableBlobFiles(true) + .setMinBlobSize(1024) + .setBlobFileSize(132768) + .setBlobCompressionType(CompressionType.BZLIB2_COMPRESSION) + .setEnableBlobGarbageCollection(true) + .setBlobGarbageCollectionAgeCutoff(0.89) + .setBlobGarbageCollectionForceThreshold(0.80) + .setBlobCompactionReadaheadSize(262144) + .setBlobFileStartingLevel(1) + .setPrepopulateBlobCache(PrepopulateBlobCache.PREPOPULATE_BLOB_FLUSH_ONLY); + + assertThat(builder.enableBlobFiles()).isEqualTo(true); + assertThat(builder.minBlobSize()).isEqualTo(1024); + assertThat(builder.blobFileSize()).isEqualTo(132768); + assertThat(builder.blobCompressionType()).isEqualTo(CompressionType.BZLIB2_COMPRESSION); + assertThat(builder.enableBlobGarbageCollection()).isEqualTo(true); + assertThat(builder.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89); + assertThat(builder.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); + assertThat(builder.blobCompactionReadaheadSize()).isEqualTo(262144); + assertThat(builder.blobFileStartingLevel()).isEqualTo(1); + assertThat(builder.prepopulateBlobCache()) + .isEqualTo(PrepopulateBlobCache.PREPOPULATE_BLOB_FLUSH_ONLY); + + builder.setEnableBlobFiles(false) + .setMinBlobSize(4096) + .setBlobFileSize(2048) + .setBlobCompressionType(CompressionType.LZ4_COMPRESSION) + .setEnableBlobGarbageCollection(false) + .setBlobGarbageCollectionAgeCutoff(0.91) + .setBlobGarbageCollectionForceThreshold(0.96) + .setBlobCompactionReadaheadSize(1024) + .setBlobFileStartingLevel(0) + .setPrepopulateBlobCache(PrepopulateBlobCache.PREPOPULATE_BLOB_DISABLE); + + assertThat(builder.enableBlobFiles()).isEqualTo(false); + assertThat(builder.minBlobSize()).isEqualTo(4096); + assertThat(builder.blobFileSize()).isEqualTo(2048); + assertThat(builder.blobCompressionType()).isEqualTo(CompressionType.LZ4_COMPRESSION); + assertThat(builder.enableBlobGarbageCollection()).isEqualTo(false); + assertThat(builder.blobGarbageCollectionAgeCutoff()).isEqualTo(0.91); + assertThat(builder.blobGarbageCollectionForceThreshold()).isEqualTo(0.96); + assertThat(builder.blobCompactionReadaheadSize()).isEqualTo(1024); + assertThat(builder.blobFileStartingLevel()).isEqualTo(0); + assertThat(builder.prepopulateBlobCache()) + .isEqualTo(PrepopulateBlobCache.PREPOPULATE_BLOB_DISABLE); + + final MutableColumnFamilyOptions options = builder.build(); + assertThat(options.getKeys()) + .isEqualTo(new String[] {"enable_blob_files", "min_blob_size", "blob_file_size", + "blob_compression_type", "enable_blob_garbage_collection", + "blob_garbage_collection_age_cutoff", "blob_garbage_collection_force_threshold", + "blob_compaction_readahead_size", "blob_file_starting_level", + "prepopulate_blob_cache"}); + assertThat(options.getValues()) + .isEqualTo(new String[] {"false", "4096", "2048", "LZ4_COMPRESSION", "false", "0.91", + "0.96", "1024", "0", "PREPOPULATE_BLOB_DISABLE"}); + } + + /** + * Configure the default column family with BLOBs. + * Confirm that BLOBs are generated when appropriately-sized writes are flushed. + * + * @throws RocksDBException if a db access throws an exception + */ + @Test + public void testBlobWriteAboveThreshold() throws RocksDBException { + try (final Options options = new Options() + .setCreateIfMissing(true) + .setMinBlobSize(minBlobSize) + .setEnableBlobFiles(true); + + final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + db.put(small_key("default"), small_value("default")); + try (final FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + db.flush(flushOptions); + } + + // check there are no blobs in the database + assertThat(countDBFiles(".sst")).isEqualTo(1); + assertThat(countDBFiles(".blob")).isEqualTo(0); + + db.put(large_key("default"), large_value("default")); + try (final FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + db.flush(flushOptions); + } + + // wrote and flushed a value larger than the blobbing threshold + // check there is a single blob in the database + assertThat(countDBFiles(".sst")).isEqualTo(2); + assertThat(countDBFiles(".blob")).isEqualTo(1); + + assertThat(db.get(small_key("default"))).isEqualTo(small_value("default")); + assertThat(db.get(large_key("default"))).isEqualTo(large_value("default")); + + final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder fetchOptions = + db.getOptions(null); + assertThat(fetchOptions.minBlobSize()).isEqualTo(minBlobSize); + assertThat(fetchOptions.enableBlobFiles()).isEqualTo(true); + assertThat(fetchOptions.writeBufferSize()).isEqualTo(64 << 20); + } + } + + /** + * Configure 2 column families respectively with and without BLOBs. + * Confirm that BLOB files are generated (once the DB is flushed) only for the appropriate column + * family. + * + * @throws RocksDBException if a db access throws an exception + */ + @Test + public void testBlobWriteAboveThresholdCF() throws RocksDBException { + final ColumnFamilyOptions columnFamilyOptions0 = new ColumnFamilyOptions(); + final ColumnFamilyDescriptor columnFamilyDescriptor0 = + new ColumnFamilyDescriptor("default".getBytes(UTF_8), columnFamilyOptions0); + List columnFamilyDescriptors = + Collections.singletonList(columnFamilyDescriptor0); + List columnFamilyHandles = new ArrayList<>(); + + try (final DBOptions dbOptions = new DBOptions().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(dbOptions, dbFolder.getRoot().getAbsolutePath(), + columnFamilyDescriptors, columnFamilyHandles)) { + db.put(columnFamilyHandles.get(0), small_key("default"), small_value("default")); + try (final FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + db.flush(flushOptions); + } + + assertThat(countDBFiles(".blob")).isEqualTo(0); + + try (final ColumnFamilyOptions columnFamilyOptions1 = + new ColumnFamilyOptions().setMinBlobSize(minBlobSize).setEnableBlobFiles(true); + + final ColumnFamilyOptions columnFamilyOptions2 = + new ColumnFamilyOptions().setMinBlobSize(minBlobSize).setEnableBlobFiles(false)) { + final ColumnFamilyDescriptor columnFamilyDescriptor1 = + new ColumnFamilyDescriptor("column_family_1".getBytes(UTF_8), columnFamilyOptions1); + final ColumnFamilyDescriptor columnFamilyDescriptor2 = + new ColumnFamilyDescriptor("column_family_2".getBytes(UTF_8), columnFamilyOptions2); + + // Create the first column family with blob options + db.createColumnFamily(columnFamilyDescriptor1); + + // Create the second column family with not-blob options + db.createColumnFamily(columnFamilyDescriptor2); + } + } + + // Now re-open after auto-close - at this point the CF options we use are recognized. + try (final ColumnFamilyOptions columnFamilyOptions1 = + new ColumnFamilyOptions().setMinBlobSize(minBlobSize).setEnableBlobFiles(true); + + final ColumnFamilyOptions columnFamilyOptions2 = + new ColumnFamilyOptions().setMinBlobSize(minBlobSize).setEnableBlobFiles(false)) { + assertThat(columnFamilyOptions1.enableBlobFiles()).isEqualTo(true); + assertThat(columnFamilyOptions1.minBlobSize()).isEqualTo(minBlobSize); + assertThat(columnFamilyOptions2.enableBlobFiles()).isEqualTo(false); + assertThat(columnFamilyOptions1.minBlobSize()).isEqualTo(minBlobSize); + + final ColumnFamilyDescriptor columnFamilyDescriptor1 = + new ColumnFamilyDescriptor("column_family_1".getBytes(UTF_8), columnFamilyOptions1); + final ColumnFamilyDescriptor columnFamilyDescriptor2 = + new ColumnFamilyDescriptor("column_family_2".getBytes(UTF_8), columnFamilyOptions2); + columnFamilyDescriptors = new ArrayList<>(); + columnFamilyDescriptors.add(columnFamilyDescriptor0); + columnFamilyDescriptors.add(columnFamilyDescriptor1); + columnFamilyDescriptors.add(columnFamilyDescriptor2); + columnFamilyHandles = new ArrayList<>(); + + assertThat(columnFamilyDescriptor1.getOptions().enableBlobFiles()).isEqualTo(true); + assertThat(columnFamilyDescriptor2.getOptions().enableBlobFiles()).isEqualTo(false); + + try (final DBOptions dbOptions = new DBOptions(); + final RocksDB db = RocksDB.open(dbOptions, dbFolder.getRoot().getAbsolutePath(), + columnFamilyDescriptors, columnFamilyHandles)) { + final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder1 = + db.getOptions(columnFamilyHandles.get(1)); + assertThat(builder1.enableBlobFiles()).isEqualTo(true); + assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize); + + final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder2 = + db.getOptions(columnFamilyHandles.get(2)); + assertThat(builder2.enableBlobFiles()).isEqualTo(false); + assertThat(builder2.minBlobSize()).isEqualTo(minBlobSize); + + db.put(columnFamilyHandles.get(1), large_key("column_family_1_k2"), + large_value("column_family_1_k2")); + try (final FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + db.flush(flushOptions, columnFamilyHandles.get(1)); + } + assertThat(countDBFiles(".blob")).isEqualTo(1); + + db.put(columnFamilyHandles.get(2), large_key("column_family_2_k2"), + large_value("column_family_2_k2")); + try (final FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + db.flush(flushOptions, columnFamilyHandles.get(2)); + } + assertThat(countDBFiles(".blob")).isEqualTo(1); + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/BlockBasedTableConfigTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/BlockBasedTableConfigTest.java new file mode 100644 index 0000000..13247d1 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/BlockBasedTableConfigTest.java @@ -0,0 +1,415 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.fail; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.stream.Stream; +import org.junit.ClassRule; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class BlockBasedTableConfigTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void cacheIndexAndFilterBlocks() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setCacheIndexAndFilterBlocks(true); + assertThat(blockBasedTableConfig.cacheIndexAndFilterBlocks()). + isTrue(); + } + + @Test + public void cacheIndexAndFilterBlocksWithHighPriority() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + assertThat(blockBasedTableConfig.cacheIndexAndFilterBlocksWithHighPriority()). + isTrue(); + blockBasedTableConfig.setCacheIndexAndFilterBlocksWithHighPriority(false); + assertThat(blockBasedTableConfig.cacheIndexAndFilterBlocksWithHighPriority()).isFalse(); + } + + @Test + public void pinL0FilterAndIndexBlocksInCache() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setPinL0FilterAndIndexBlocksInCache(true); + assertThat(blockBasedTableConfig.pinL0FilterAndIndexBlocksInCache()). + isTrue(); + } + + @Test + public void pinTopLevelIndexAndFilter() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setPinTopLevelIndexAndFilter(false); + assertThat(blockBasedTableConfig.pinTopLevelIndexAndFilter()). + isFalse(); + } + + @Test + public void indexType() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + assertThat(IndexType.values().length).isEqualTo(4); + blockBasedTableConfig.setIndexType(IndexType.kHashSearch); + assertThat(blockBasedTableConfig.indexType()).isEqualTo(IndexType.kHashSearch); + assertThat(IndexType.valueOf("kBinarySearch")).isNotNull(); + blockBasedTableConfig.setIndexType(IndexType.valueOf("kBinarySearch")); + assertThat(blockBasedTableConfig.indexType()).isEqualTo(IndexType.kBinarySearch); + } + + @Test + public void dataBlockIndexType() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setDataBlockIndexType(DataBlockIndexType.kDataBlockBinaryAndHash); + assertThat(blockBasedTableConfig.dataBlockIndexType()) + .isEqualTo(DataBlockIndexType.kDataBlockBinaryAndHash); + blockBasedTableConfig.setDataBlockIndexType(DataBlockIndexType.kDataBlockBinarySearch); + assertThat(blockBasedTableConfig.dataBlockIndexType()) + .isEqualTo(DataBlockIndexType.kDataBlockBinarySearch); + } + + @Test + public void checksumType() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + assertThat(ChecksumType.values().length).isEqualTo(5); + assertThat(ChecksumType.valueOf("kxxHash")). + isEqualTo(ChecksumType.kxxHash); + blockBasedTableConfig.setChecksumType(ChecksumType.kNoChecksum); + assertThat(blockBasedTableConfig.checksumType()).isEqualTo(ChecksumType.kNoChecksum); + blockBasedTableConfig.setChecksumType(ChecksumType.kxxHash); + assertThat(blockBasedTableConfig.checksumType()).isEqualTo(ChecksumType.kxxHash); + blockBasedTableConfig.setChecksumType(ChecksumType.kxxHash64); + assertThat(blockBasedTableConfig.checksumType()).isEqualTo(ChecksumType.kxxHash64); + blockBasedTableConfig.setChecksumType(ChecksumType.kXXH3); + assertThat(blockBasedTableConfig.checksumType()).isEqualTo(ChecksumType.kXXH3); + } + + @Test + public void jniPortal() throws Exception { + // Verifies that the JNI layer is correctly translating options. + // Since introspecting the options requires creating a database, the checks + // cover multiple options at the same time. + + final BlockBasedTableConfig tableConfig = new BlockBasedTableConfig(); + + tableConfig.setIndexType(IndexType.kBinarySearch); + tableConfig.setDataBlockIndexType(DataBlockIndexType.kDataBlockBinarySearch); + tableConfig.setChecksumType(ChecksumType.kNoChecksum); + try (final Options options = new Options().setTableFormatConfig(tableConfig)) { + final String opts = getOptionAsString(options); + assertThat(opts).contains("index_type=kBinarySearch"); + assertThat(opts).contains("data_block_index_type=kDataBlockBinarySearch"); + assertThat(opts).contains("checksum=kNoChecksum"); + } + + tableConfig.setIndexType(IndexType.kHashSearch); + tableConfig.setDataBlockIndexType(DataBlockIndexType.kDataBlockBinaryAndHash); + tableConfig.setChecksumType(ChecksumType.kCRC32c); + try (final Options options = new Options().setTableFormatConfig(tableConfig)) { + options.useCappedPrefixExtractor(1); // Needed to use kHashSearch + final String opts = getOptionAsString(options); + assertThat(opts).contains("index_type=kHashSearch"); + assertThat(opts).contains("data_block_index_type=kDataBlockBinaryAndHash"); + assertThat(opts).contains("checksum=kCRC32c"); + } + + tableConfig.setIndexType(IndexType.kTwoLevelIndexSearch); + tableConfig.setChecksumType(ChecksumType.kxxHash); + try (final Options options = new Options().setTableFormatConfig(tableConfig)) { + final String opts = getOptionAsString(options); + assertThat(opts).contains("index_type=kTwoLevelIndexSearch"); + assertThat(opts).contains("checksum=kxxHash"); + } + + tableConfig.setIndexType(IndexType.kBinarySearchWithFirstKey); + tableConfig.setChecksumType(ChecksumType.kxxHash64); + try (final Options options = new Options().setTableFormatConfig(tableConfig)) { + final String opts = getOptionAsString(options); + assertThat(opts).contains("index_type=kBinarySearchWithFirstKey"); + assertThat(opts).contains("checksum=kxxHash64"); + } + + tableConfig.setChecksumType(ChecksumType.kXXH3); + try (final Options options = new Options().setTableFormatConfig(tableConfig)) { + final String opts = getOptionAsString(options); + assertThat(opts).contains("checksum=kXXH3"); + } + } + + private String getOptionAsString(final Options options) throws Exception { + options.setCreateIfMissing(true); + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + final String result; + try (final RocksDB ignored = RocksDB.open(options, dbPath); + final Stream pathStream = Files.walk(Paths.get(dbPath))) { + final Path optionsPath = + pathStream.filter(p -> p.getFileName().toString().startsWith("OPTIONS")) + .findAny() + .orElseThrow(() -> new AssertionError("Missing options file")); + final byte[] optionsData = Files.readAllBytes(optionsPath); + result = new String(optionsData, StandardCharsets.UTF_8); + } + RocksDB.destroyDB(dbPath, options); + return result; + } + + @Test + public void noBlockCache() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setNoBlockCache(true); + assertThat(blockBasedTableConfig.noBlockCache()).isTrue(); + } + + @Test + public void blockCache() { + try ( + final Cache cache = new LRUCache(17 * 1024 * 1024); + final Options options = new Options().setTableFormatConfig( + new BlockBasedTableConfig().setBlockCache(cache))) { + assertThat(options.tableFactoryName()).isEqualTo("BlockBasedTable"); + } + } + + @Test + public void blockCacheIntegration() throws RocksDBException { + try (final Cache cache = new LRUCache(8 * 1024 * 1024); + final Statistics statistics = new Statistics()) { + for (int shard = 0; shard < 8; shard++) { + try (final Options options = + new Options() + .setCreateIfMissing(true) + .setStatistics(statistics) + .setTableFormatConfig(new BlockBasedTableConfig().setBlockCache(cache)); + final RocksDB db = + RocksDB.open(options, dbFolder.getRoot().getAbsolutePath() + "/" + shard)) { + final byte[] key = "some-key".getBytes(StandardCharsets.UTF_8); + final byte[] value = "some-value".getBytes(StandardCharsets.UTF_8); + + db.put(key, value); + db.flush(new FlushOptions()); + db.get(key); + + assertThat(statistics.getTickerCount(TickerType.BLOCK_CACHE_ADD)).isEqualTo(shard + 1); + } + } + } + } + + @Test + public void persistentCache() throws RocksDBException { + try (final DBOptions dbOptions = new DBOptions(). + setInfoLogLevel(InfoLogLevel.INFO_LEVEL). + setCreateIfMissing(true); + final Logger logger = new Logger(dbOptions) { + @Override + protected void log(final InfoLogLevel infoLogLevel, final String logMsg) { + System.out.println(infoLogLevel.name() + ": " + logMsg); + } + }) { + try (final PersistentCache persistentCache = + new PersistentCache(Env.getDefault(), dbFolder.getRoot().getPath(), 1024 * 1024 * 100, logger, false); + final Options options = new Options().setTableFormatConfig( + new BlockBasedTableConfig().setPersistentCache(persistentCache))) { + assertThat(options.tableFactoryName()).isEqualTo("BlockBasedTable"); + } + } + } + + @Test + public void blockSize() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setBlockSize(10); + assertThat(blockBasedTableConfig.blockSize()).isEqualTo(10); + } + + @Test + public void blockSizeDeviation() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setBlockSizeDeviation(12); + assertThat(blockBasedTableConfig.blockSizeDeviation()). + isEqualTo(12); + } + + @Test + public void blockRestartInterval() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setBlockRestartInterval(15); + assertThat(blockBasedTableConfig.blockRestartInterval()). + isEqualTo(15); + } + + @Test + public void indexBlockRestartInterval() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setIndexBlockRestartInterval(15); + assertThat(blockBasedTableConfig.indexBlockRestartInterval()). + isEqualTo(15); + } + + @Test + public void metadataBlockSize() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setMetadataBlockSize(1024); + assertThat(blockBasedTableConfig.metadataBlockSize()). + isEqualTo(1024); + } + + @Test + public void partitionFilters() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setPartitionFilters(true); + assertThat(blockBasedTableConfig.partitionFilters()). + isTrue(); + } + + @Test + public void optimizeFiltersForMemory() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setOptimizeFiltersForMemory(true); + assertThat(blockBasedTableConfig.optimizeFiltersForMemory()).isTrue(); + } + + @Test + public void useDeltaEncoding() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setUseDeltaEncoding(false); + assertThat(blockBasedTableConfig.useDeltaEncoding()). + isFalse(); + } + + @Test + public void blockBasedTableWithFilterPolicy() { + try(final Options options = new Options() + .setTableFormatConfig(new BlockBasedTableConfig() + .setFilterPolicy(new BloomFilter(10)))) { + assertThat(options.tableFactoryName()). + isEqualTo("BlockBasedTable"); + } + } + + @Test + public void blockBasedTableWithoutFilterPolicy() { + try(final Options options = new Options().setTableFormatConfig( + new BlockBasedTableConfig().setFilterPolicy(null))) { + assertThat(options.tableFactoryName()). + isEqualTo("BlockBasedTable"); + } + } + + @Test + public void wholeKeyFiltering() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setWholeKeyFiltering(false); + assertThat(blockBasedTableConfig.wholeKeyFiltering()). + isFalse(); + } + + @Test + public void verifyCompression() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + assertThat(blockBasedTableConfig.verifyCompression()).isFalse(); + blockBasedTableConfig.setVerifyCompression(true); + assertThat(blockBasedTableConfig.verifyCompression()). + isTrue(); + } + + @Test + public void readAmpBytesPerBit() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setReadAmpBytesPerBit(2); + assertThat(blockBasedTableConfig.readAmpBytesPerBit()). + isEqualTo(2); + } + + @Test + public void formatVersion() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + for (int version = 0; version <= 5; version++) { + blockBasedTableConfig.setFormatVersion(version); + assertThat(blockBasedTableConfig.formatVersion()).isEqualTo(version); + } + } + + @Test(expected = AssertionError.class) + public void formatVersionFailNegative() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setFormatVersion(-1); + } + + @Test(expected = RocksDBException.class) + public void invalidFormatVersion() throws RocksDBException { + final BlockBasedTableConfig blockBasedTableConfig = + new BlockBasedTableConfig().setFormatVersion(99999); + + try (final Options options = new Options().setTableFormatConfig(blockBasedTableConfig); + final RocksDB ignored = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + fail("Opening the database with an invalid format_version should have raised an exception"); + } + } + + @Test + public void enableIndexCompression() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setEnableIndexCompression(false); + assertThat(blockBasedTableConfig.enableIndexCompression()). + isFalse(); + } + + @Test + public void blockAlign() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setBlockAlign(true); + assertThat(blockBasedTableConfig.blockAlign()). + isTrue(); + } + + @Test + public void indexShortening() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setIndexShortening(IndexShorteningMode.kShortenSeparatorsAndSuccessor); + assertThat(blockBasedTableConfig.indexShortening()) + .isEqualTo(IndexShorteningMode.kShortenSeparatorsAndSuccessor); + } + + @Deprecated + @Test + public void hashIndexAllowCollision() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setHashIndexAllowCollision(false); + assertThat(blockBasedTableConfig.hashIndexAllowCollision()). + isTrue(); // NOTE: setHashIndexAllowCollision should do nothing! + } + + @Deprecated + @Test + public void blockCacheSize() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setBlockCacheSize(8 * 1024); + assertThat(blockBasedTableConfig.blockCacheSize()). + isEqualTo(8 * 1024); + } + + @Deprecated + @Test + public void blockCacheNumShardBits() { + final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setCacheNumShardBits(5); + assertThat(blockBasedTableConfig.cacheNumShardBits()). + isEqualTo(5); + } + +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/BuiltinComparatorTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/BuiltinComparatorTest.java new file mode 100644 index 0000000..e238ae0 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/BuiltinComparatorTest.java @@ -0,0 +1,145 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import static org.assertj.core.api.Assertions.assertThat; + +public class BuiltinComparatorTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void builtinForwardComparator() + throws RocksDBException { + try (final Options options = new Options() + .setCreateIfMissing(true) + .setComparator(BuiltinComparator.BYTEWISE_COMPARATOR); + final RocksDB rocksDb = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath()) + ) { + rocksDb.put("abc1".getBytes(), "abc1".getBytes()); + rocksDb.put("abc2".getBytes(), "abc2".getBytes()); + rocksDb.put("abc3".getBytes(), "abc3".getBytes()); + + try(final RocksIterator rocksIterator = rocksDb.newIterator()) { + // Iterate over keys using a iterator + rocksIterator.seekToFirst(); + assertThat(rocksIterator.isValid()).isTrue(); + assertThat(rocksIterator.key()).isEqualTo( + "abc1".getBytes()); + assertThat(rocksIterator.value()).isEqualTo( + "abc1".getBytes()); + rocksIterator.next(); + assertThat(rocksIterator.isValid()).isTrue(); + assertThat(rocksIterator.key()).isEqualTo( + "abc2".getBytes()); + assertThat(rocksIterator.value()).isEqualTo( + "abc2".getBytes()); + rocksIterator.next(); + assertThat(rocksIterator.isValid()).isTrue(); + assertThat(rocksIterator.key()).isEqualTo( + "abc3".getBytes()); + assertThat(rocksIterator.value()).isEqualTo( + "abc3".getBytes()); + rocksIterator.next(); + assertThat(rocksIterator.isValid()).isFalse(); + // Get last one + rocksIterator.seekToLast(); + assertThat(rocksIterator.isValid()).isTrue(); + assertThat(rocksIterator.key()).isEqualTo( + "abc3".getBytes()); + assertThat(rocksIterator.value()).isEqualTo( + "abc3".getBytes()); + // Seek for abc + rocksIterator.seek("abc".getBytes()); + assertThat(rocksIterator.isValid()).isTrue(); + assertThat(rocksIterator.key()).isEqualTo( + "abc1".getBytes()); + assertThat(rocksIterator.value()).isEqualTo( + "abc1".getBytes()); + } + } + } + + @Test + public void builtinReverseComparator() + throws RocksDBException { + try (final Options options = new Options() + .setCreateIfMissing(true) + .setComparator(BuiltinComparator.REVERSE_BYTEWISE_COMPARATOR); + final RocksDB rocksDb = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath()) + ) { + + rocksDb.put("abc1".getBytes(), "abc1".getBytes()); + rocksDb.put("abc2".getBytes(), "abc2".getBytes()); + rocksDb.put("abc3".getBytes(), "abc3".getBytes()); + + try (final RocksIterator rocksIterator = rocksDb.newIterator()) { + // Iterate over keys using a iterator + rocksIterator.seekToFirst(); + assertThat(rocksIterator.isValid()).isTrue(); + assertThat(rocksIterator.key()).isEqualTo( + "abc3".getBytes()); + assertThat(rocksIterator.value()).isEqualTo( + "abc3".getBytes()); + rocksIterator.next(); + assertThat(rocksIterator.isValid()).isTrue(); + assertThat(rocksIterator.key()).isEqualTo( + "abc2".getBytes()); + assertThat(rocksIterator.value()).isEqualTo( + "abc2".getBytes()); + rocksIterator.next(); + assertThat(rocksIterator.isValid()).isTrue(); + assertThat(rocksIterator.key()).isEqualTo( + "abc1".getBytes()); + assertThat(rocksIterator.value()).isEqualTo( + "abc1".getBytes()); + rocksIterator.next(); + assertThat(rocksIterator.isValid()).isFalse(); + // Get last one + rocksIterator.seekToLast(); + assertThat(rocksIterator.isValid()).isTrue(); + assertThat(rocksIterator.key()).isEqualTo( + "abc1".getBytes()); + assertThat(rocksIterator.value()).isEqualTo( + "abc1".getBytes()); + // Will be invalid because abc is after abc1 + rocksIterator.seek("abc".getBytes()); + assertThat(rocksIterator.isValid()).isFalse(); + // Will be abc3 because the next one after abc999 + // is abc3 + rocksIterator.seek("abc999".getBytes()); + assertThat(rocksIterator.key()).isEqualTo( + "abc3".getBytes()); + assertThat(rocksIterator.value()).isEqualTo( + "abc3".getBytes()); + } + } + } + + @Test + public void builtinComparatorEnum(){ + assertThat(BuiltinComparator.BYTEWISE_COMPARATOR.ordinal()) + .isEqualTo(0); + assertThat( + BuiltinComparator.REVERSE_BYTEWISE_COMPARATOR.ordinal()) + .isEqualTo(1); + assertThat(BuiltinComparator.values().length).isEqualTo(2); + assertThat(BuiltinComparator.valueOf("BYTEWISE_COMPARATOR")). + isEqualTo(BuiltinComparator.BYTEWISE_COMPARATOR); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ByteBufferUnsupportedOperationTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ByteBufferUnsupportedOperationTest.java new file mode 100644 index 0000000..f596f57 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ByteBufferUnsupportedOperationTest.java @@ -0,0 +1,132 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.util.ReverseBytewiseComparator; + +public class ByteBufferUnsupportedOperationTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + public static class Handler { + private final RocksDB database; + private final Map columnFamilies; + + public Handler(final String path, final Options options) throws RocksDBException { + RocksDB.destroyDB(path, options); + this.database = RocksDB.open(options, path); + this.columnFamilies = new ConcurrentHashMap<>(); + } + + public void addTable(final UUID streamID) throws RocksDBException { + final ColumnFamilyOptions tableOptions = new ColumnFamilyOptions(); + tableOptions.optimizeUniversalStyleCompaction(); + try (final ComparatorOptions comparatorOptions = new ComparatorOptions()) { + // comparatorOptions.setReusedSynchronisationType(ReusedSynchronisationType.ADAPTIVE_MUTEX); + tableOptions.setComparator(new ReverseBytewiseComparator(comparatorOptions)); + final ColumnFamilyDescriptor tableDescriptor = new ColumnFamilyDescriptor( + streamID.toString().getBytes(StandardCharsets.UTF_8), tableOptions); + final ColumnFamilyHandle tableHandle = database.createColumnFamily(tableDescriptor); + columnFamilies.put(streamID, tableHandle); + } + } + + public void updateAll(final List keyValuePairs, final UUID streamID) + throws RocksDBException { + final ColumnFamilyHandle currTable = columnFamilies.get(streamID); + try (final WriteBatch batchedWrite = new WriteBatch(); + final WriteOptions writeOptions = new WriteOptions()) { + for (final byte[][] pair : keyValuePairs) { + final byte[] keyBytes = pair[0]; + final byte[] valueBytes = pair[1]; + batchedWrite.put(currTable, keyBytes, valueBytes); + } + database.write(writeOptions, batchedWrite); + } + } + public boolean containsValue(final byte[] encodedValue, final UUID streamID) { + try (final RocksIterator iter = database.newIterator(columnFamilies.get(streamID))) { + iter.seekToFirst(); + while (iter.isValid()) { + final byte[] val = iter.value(); + if (Arrays.equals(val, encodedValue)) { + return true; + } + iter.next(); + } + } + return false; + } + + public void close() { + for (final ColumnFamilyHandle handle : columnFamilies.values()) { + handle.close(); + } + database.close(); + } + } + + private void inner(final int numRepeats) throws RocksDBException { + final Options opts = new Options(); + opts.setCreateIfMissing(true); + final Handler handler = new Handler("testDB", opts); + final UUID stream1 = UUID.randomUUID(); + + final List entries = new ArrayList<>(); + for (int i = 0; i < numRepeats; i++) { + final byte[] value = value(i); + final byte[] key = key(i); + entries.add(new byte[][] {key, value}); + } + handler.addTable(stream1); + handler.updateAll(entries, stream1); + + for (int i = 0; i < numRepeats; i++) { + final byte[] val = value(i); + final boolean hasValue = handler.containsValue(val, stream1); + if (!hasValue) { + throw new IllegalStateException("not has value " + i); + } + } + + handler.close(); + } + + private static byte[] key(final int i) { + return ("key" + i).getBytes(StandardCharsets.UTF_8); + } + + private static byte[] value(final int i) { + return ("value" + i).getBytes(StandardCharsets.UTF_8); + } + + @Test + public void unsupportedOperation() throws RocksDBException { + final int numRepeats = 1000; + final int repeatTest = 10; + + // the error is not always reproducible... let's try to increase the odds by repeating the main + // test body + for (int i = 0; i < repeatTest; i++) { + try { + inner(numRepeats); + } catch (final RuntimeException runtimeException) { + System.out.println("Exception on repeat " + i); + throw runtimeException; + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/BytewiseComparatorRegressionTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/BytewiseComparatorRegressionTest.java new file mode 100644 index 0000000..13aa6c2 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/BytewiseComparatorRegressionTest.java @@ -0,0 +1,132 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static org.junit.Assert.assertArrayEquals; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.util.BytewiseComparator; + +/** + * This test confirms that the following issues were in fact resolved + * by a change made between 6.2.2 and 6.22.1, + * to wit {@link ...} + * which as part of its effect, changed the Java bytewise comparators. + *

    + * {@link ...} + * {@link ...} + */ +public class BytewiseComparatorRegressionTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Rule public TemporaryFolder temporarySSTFolder = new TemporaryFolder(); + + private static final byte[][] testData = {{10, -11, 13}, {10, 11, 12}, {10, 11, 14}}; + private static final byte[][] orderedData = {{10, 11, 12}, {10, 11, 14}, {10, -11, 13}}; + + /** + * {@link ...} + */ + @Test + public void testJavaComparator() throws RocksDBException { + final BytewiseComparator comparator = new BytewiseComparator(new ComparatorOptions()); + try (final Options options = new Options().setCreateIfMissing(true).setComparator(comparator)) { + performTest(options); + } + } + + @Test + public void testDefaultComparator() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true)) { + performTest(options); + } + } + + /** + * {@link ...} + */ + @Test + public void testCppComparator() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true).setComparator( + BuiltinComparator.BYTEWISE_COMPARATOR)) { + performTest(options); + } + } + + private void performTest(final Options options) throws RocksDBException { + try (final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + for (final byte[] item : testData) { + db.put(item, item); + } + try (final RocksIterator iterator = db.newIterator()) { + iterator.seekToFirst(); + final ArrayList result = new ArrayList<>(); + while (iterator.isValid()) { + result.add(iterator.key()); + iterator.next(); + } + assertArrayEquals(orderedData, result.toArray()); + } + } + } + + private byte[] hexToByte(final String hexString) { + final byte[] bytes = new byte[hexString.length() / 2]; + if (bytes.length * 2 < hexString.length()) { + throw new RuntimeException("Hex string has odd length: " + hexString); + } + + for (int i = 0; i < bytes.length; i++) { + final int firstDigit = toDigit(hexString.charAt(i + i)); + final int secondDigit = toDigit(hexString.charAt(i + i + 1)); + bytes[i] = (byte) ((firstDigit << 4) + secondDigit); + } + + return bytes; + } + + private int toDigit(final char hexChar) { + final int digit = Character.digit(hexChar, 16); + if (digit == -1) { + throw new IllegalArgumentException("Invalid Hexadecimal Character: " + hexChar); + } + return digit; + } + + /** + * {@link ...} + * + * @throws RocksDBException if something goes wrong, or if the regression occurs + * @throws IOException if we can't make the temporary file + */ + @Test + public void testSST() throws RocksDBException, IOException { + final File tempSSTFile = temporarySSTFolder.newFile("test_file_with_weird_keys.sst"); + + final EnvOptions envOpts = new EnvOptions(); + final Options opts = new Options(); + opts.setComparator(new BytewiseComparator(new ComparatorOptions())); + final SstFileWriter writer = new SstFileWriter(envOpts, opts); + writer.open(tempSSTFile.getAbsolutePath()); + final byte[] gKey = + hexToByte("000000293030303030303030303030303030303030303032303736343730696E666F33"); + final byte[] wKey = + hexToByte("0000008d3030303030303030303030303030303030303030303437363433696e666f34"); + writer.put(new Slice(gKey), new Slice("dummyV1")); + writer.put(new Slice(wKey), new Slice("dummyV2")); + writer.finish(); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CheckPointTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CheckPointTest.java new file mode 100644 index 0000000..2b3cc7a --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CheckPointTest.java @@ -0,0 +1,82 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb; + + +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import static org.assertj.core.api.Assertions.assertThat; + +public class CheckPointTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Rule + public TemporaryFolder checkpointFolder = new TemporaryFolder(); + + @Test + public void checkPoint() throws RocksDBException { + try (final Options options = new Options(). + setCreateIfMissing(true)) { + + try (final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath())) { + db.put("key".getBytes(), "value".getBytes()); + try (final Checkpoint checkpoint = Checkpoint.create(db)) { + checkpoint.createCheckpoint(checkpointFolder. + getRoot().getAbsolutePath() + "/snapshot1"); + db.put("key2".getBytes(), "value2".getBytes()); + checkpoint.createCheckpoint(checkpointFolder. + getRoot().getAbsolutePath() + "/snapshot2"); + } + } + + try (final RocksDB db = RocksDB.open(options, + checkpointFolder.getRoot().getAbsolutePath() + + "/snapshot1")) { + assertThat(new String(db.get("key".getBytes()))). + isEqualTo("value"); + assertThat(db.get("key2".getBytes())).isNull(); + } + + try (final RocksDB db = RocksDB.open(options, + checkpointFolder.getRoot().getAbsolutePath() + + "/snapshot2")) { + assertThat(new String(db.get("key".getBytes()))). + isEqualTo("value"); + assertThat(new String(db.get("key2".getBytes()))). + isEqualTo("value2"); + } + } + } + + @Test(expected = IllegalArgumentException.class) + public void failIfDbIsNull() { + try (final Checkpoint ignored = Checkpoint.create(null)) { + } + } + + @Test(expected = IllegalStateException.class) + public void failIfDbNotInitialized() throws RocksDBException { + try (final RocksDB db = RocksDB.open( + dbFolder.getRoot().getAbsolutePath())) { + db.close(); + Checkpoint.create(db); + } + } + + @Test(expected = RocksDBException.class) + public void failWithIllegalPath() throws RocksDBException { + try (final RocksDB db = RocksDB.open(dbFolder.getRoot().getAbsolutePath()); + final Checkpoint checkpoint = Checkpoint.create(db)) { + checkpoint.createCheckpoint("/Z:///:\\C:\\TZ/-"); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ClockCacheTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ClockCacheTest.java new file mode 100644 index 0000000..718c24f --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ClockCacheTest.java @@ -0,0 +1,25 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.Test; + +public class ClockCacheTest { + + static { + RocksDB.loadLibrary(); + } + + @Test + public void newClockCache() { + final long capacity = 1000; + final int numShardBits = 16; + final boolean strictCapacityLimit = true; + try (final Cache ignored = new ClockCache(capacity, numShardBits, strictCapacityLimit)) { + //no op + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java new file mode 100644 index 0000000..35a04a6 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java @@ -0,0 +1,722 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.nio.file.Paths; +import java.util.*; +import org.junit.ClassRule; +import org.junit.Test; +import org.rocksdb.test.RemoveEmptyValueCompactionFilterFactory; + +public class ColumnFamilyOptionsTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + public static final Random rand = PlatformRandomHelper. + getPlatformSpecificRandomFactory(); + + @Test + public void copyConstructor() { + final ColumnFamilyOptions origOpts = new ColumnFamilyOptions(); + origOpts.setNumLevels(rand.nextInt(8)); + origOpts.setTargetFileSizeMultiplier(rand.nextInt(100)); + origOpts.setLevel0StopWritesTrigger(rand.nextInt(50)); + final ColumnFamilyOptions copyOpts = new ColumnFamilyOptions(origOpts); + assertThat(origOpts.numLevels()).isEqualTo(copyOpts.numLevels()); + assertThat(origOpts.targetFileSizeMultiplier()).isEqualTo(copyOpts.targetFileSizeMultiplier()); + assertThat(origOpts.level0StopWritesTrigger()).isEqualTo(copyOpts.level0StopWritesTrigger()); + } + + @Test + public void getColumnFamilyOptionsFromProps() { + final Properties properties = new Properties(); + properties.put("write_buffer_size", "112"); + properties.put("max_write_buffer_number", "13"); + + try (final ColumnFamilyOptions opt = ColumnFamilyOptions. + getColumnFamilyOptionsFromProps(properties)) { + // setup sample properties + assertThat(opt).isNotNull(); + assertThat(String.valueOf(opt.writeBufferSize())). + isEqualTo(properties.get("write_buffer_size")); + assertThat(String.valueOf(opt.maxWriteBufferNumber())). + isEqualTo(properties.get("max_write_buffer_number")); + } + } + + @Test + public void getColumnFamilyOptionsFromPropsWithIgnoreIllegalValue() { + // setup sample properties + final Properties properties = new Properties(); + properties.put("tomato", "1024"); + properties.put("burger", "2"); + properties.put("write_buffer_size", "112"); + properties.put("max_write_buffer_number", "13"); + + try (final ConfigOptions cfgOpts = new ConfigOptions().setIgnoreUnknownOptions(true); + final ColumnFamilyOptions opt = + ColumnFamilyOptions.getColumnFamilyOptionsFromProps(cfgOpts, properties)) { + // setup sample properties + assertThat(opt).isNotNull(); + assertThat(String.valueOf(opt.writeBufferSize())) + .isEqualTo(properties.get("write_buffer_size")); + assertThat(String.valueOf(opt.maxWriteBufferNumber())) + .isEqualTo(properties.get("max_write_buffer_number")); + } + } + + @Test + public void failColumnFamilyOptionsFromPropsWithIllegalValue() { + // setup sample properties + final Properties properties = new Properties(); + properties.put("tomato", "1024"); + properties.put("burger", "2"); + + try (final ColumnFamilyOptions opt = + ColumnFamilyOptions.getColumnFamilyOptionsFromProps(properties)) { + assertThat(opt).isNull(); + } + } + + @Test(expected = IllegalArgumentException.class) + public void failColumnFamilyOptionsFromPropsWithNullValue() { + try (final ColumnFamilyOptions ignored = + ColumnFamilyOptions.getColumnFamilyOptionsFromProps(null)) { + } + } + + @Test(expected = IllegalArgumentException.class) + public void failColumnFamilyOptionsFromPropsWithEmptyProps() { + try (final ColumnFamilyOptions ignored = + ColumnFamilyOptions.getColumnFamilyOptionsFromProps(new Properties())) { + } + } + + @Test + public void writeBufferSize() throws RocksDBException { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final long longValue = rand.nextLong(); + opt.setWriteBufferSize(longValue); + assertThat(opt.writeBufferSize()).isEqualTo(longValue); + } + } + + @Test + public void maxWriteBufferNumber() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final int intValue = rand.nextInt(); + opt.setMaxWriteBufferNumber(intValue); + assertThat(opt.maxWriteBufferNumber()).isEqualTo(intValue); + } + } + + @Test + public void minWriteBufferNumberToMerge() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final int intValue = rand.nextInt(); + opt.setMinWriteBufferNumberToMerge(intValue); + assertThat(opt.minWriteBufferNumberToMerge()).isEqualTo(intValue); + } + } + + @Test + public void numLevels() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final int intValue = rand.nextInt(); + opt.setNumLevels(intValue); + assertThat(opt.numLevels()).isEqualTo(intValue); + } + } + + @Test + public void levelZeroFileNumCompactionTrigger() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final int intValue = rand.nextInt(); + opt.setLevelZeroFileNumCompactionTrigger(intValue); + assertThat(opt.levelZeroFileNumCompactionTrigger()).isEqualTo(intValue); + } + } + + @Test + public void levelZeroSlowdownWritesTrigger() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final int intValue = rand.nextInt(); + opt.setLevelZeroSlowdownWritesTrigger(intValue); + assertThat(opt.levelZeroSlowdownWritesTrigger()).isEqualTo(intValue); + } + } + + @Test + public void levelZeroStopWritesTrigger() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final int intValue = rand.nextInt(); + opt.setLevelZeroStopWritesTrigger(intValue); + assertThat(opt.levelZeroStopWritesTrigger()).isEqualTo(intValue); + } + } + + @Test + public void targetFileSizeBase() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final long longValue = rand.nextLong(); + opt.setTargetFileSizeBase(longValue); + assertThat(opt.targetFileSizeBase()).isEqualTo(longValue); + } + } + + @Test + public void targetFileSizeMultiplier() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final int intValue = rand.nextInt(); + opt.setTargetFileSizeMultiplier(intValue); + assertThat(opt.targetFileSizeMultiplier()).isEqualTo(intValue); + } + } + + @Test + public void maxBytesForLevelBase() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final long longValue = rand.nextLong(); + opt.setMaxBytesForLevelBase(longValue); + assertThat(opt.maxBytesForLevelBase()).isEqualTo(longValue); + } + } + + @Test + public void levelCompactionDynamicLevelBytes() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setLevelCompactionDynamicLevelBytes(boolValue); + assertThat(opt.levelCompactionDynamicLevelBytes()) + .isEqualTo(boolValue); + } + } + + @Test + public void maxBytesForLevelMultiplier() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final double doubleValue = rand.nextDouble(); + opt.setMaxBytesForLevelMultiplier(doubleValue); + assertThat(opt.maxBytesForLevelMultiplier()).isEqualTo(doubleValue); + } + } + + @Test + public void maxBytesForLevelMultiplierAdditional() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final int intValue1 = rand.nextInt(); + final int intValue2 = rand.nextInt(); + final int[] ints = new int[]{intValue1, intValue2}; + opt.setMaxBytesForLevelMultiplierAdditional(ints); + assertThat(opt.maxBytesForLevelMultiplierAdditional()).isEqualTo(ints); + } + } + + @Test + public void maxCompactionBytes() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final long longValue = rand.nextLong(); + opt.setMaxCompactionBytes(longValue); + assertThat(opt.maxCompactionBytes()).isEqualTo(longValue); + } + } + + @Test + public void softPendingCompactionBytesLimit() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final long longValue = rand.nextLong(); + opt.setSoftPendingCompactionBytesLimit(longValue); + assertThat(opt.softPendingCompactionBytesLimit()).isEqualTo(longValue); + } + } + + @Test + public void hardPendingCompactionBytesLimit() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final long longValue = rand.nextLong(); + opt.setHardPendingCompactionBytesLimit(longValue); + assertThat(opt.hardPendingCompactionBytesLimit()).isEqualTo(longValue); + } + } + + @Test + public void level0FileNumCompactionTrigger() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final int intValue = rand.nextInt(); + opt.setLevel0FileNumCompactionTrigger(intValue); + assertThat(opt.level0FileNumCompactionTrigger()).isEqualTo(intValue); + } + } + + @Test + public void level0SlowdownWritesTrigger() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final int intValue = rand.nextInt(); + opt.setLevel0SlowdownWritesTrigger(intValue); + assertThat(opt.level0SlowdownWritesTrigger()).isEqualTo(intValue); + } + } + + @Test + public void level0StopWritesTrigger() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final int intValue = rand.nextInt(); + opt.setLevel0StopWritesTrigger(intValue); + assertThat(opt.level0StopWritesTrigger()).isEqualTo(intValue); + } + } + + @Test + public void arenaBlockSize() throws RocksDBException { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final long longValue = rand.nextLong(); + opt.setArenaBlockSize(longValue); + assertThat(opt.arenaBlockSize()).isEqualTo(longValue); + } + } + + @Test + public void disableAutoCompactions() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setDisableAutoCompactions(boolValue); + assertThat(opt.disableAutoCompactions()).isEqualTo(boolValue); + } + } + + @Test + public void maxSequentialSkipInIterations() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final long longValue = rand.nextLong(); + opt.setMaxSequentialSkipInIterations(longValue); + assertThat(opt.maxSequentialSkipInIterations()).isEqualTo(longValue); + } + } + + @Test + public void inplaceUpdateSupport() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setInplaceUpdateSupport(boolValue); + assertThat(opt.inplaceUpdateSupport()).isEqualTo(boolValue); + } + } + + @Test + public void inplaceUpdateNumLocks() throws RocksDBException { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final long longValue = rand.nextLong(); + opt.setInplaceUpdateNumLocks(longValue); + assertThat(opt.inplaceUpdateNumLocks()).isEqualTo(longValue); + } + } + + @Test + public void memtablePrefixBloomSizeRatio() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final double doubleValue = rand.nextDouble(); + opt.setMemtablePrefixBloomSizeRatio(doubleValue); + assertThat(opt.memtablePrefixBloomSizeRatio()).isEqualTo(doubleValue); + } + } + + @Test + public void experimentalMempurgeThreshold() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final double doubleValue = rand.nextDouble(); + opt.setExperimentalMempurgeThreshold(doubleValue); + assertThat(opt.experimentalMempurgeThreshold()).isEqualTo(doubleValue); + } + } + + @Test + public void memtableWholeKeyFiltering() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final boolean booleanValue = rand.nextBoolean(); + opt.setMemtableWholeKeyFiltering(booleanValue); + assertThat(opt.memtableWholeKeyFiltering()).isEqualTo(booleanValue); + } + } + + @Test + public void memtableHugePageSize() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final long longValue = rand.nextLong(); + opt.setMemtableHugePageSize(longValue); + assertThat(opt.memtableHugePageSize()).isEqualTo(longValue); + } + } + + @Test + public void bloomLocality() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final int intValue = rand.nextInt(); + opt.setBloomLocality(intValue); + assertThat(opt.bloomLocality()).isEqualTo(intValue); + } + } + + @Test + public void maxSuccessiveMerges() throws RocksDBException { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final long longValue = rand.nextLong(); + opt.setMaxSuccessiveMerges(longValue); + assertThat(opt.maxSuccessiveMerges()).isEqualTo(longValue); + } + } + + @Test + public void optimizeFiltersForHits() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final boolean aBoolean = rand.nextBoolean(); + opt.setOptimizeFiltersForHits(aBoolean); + assertThat(opt.optimizeFiltersForHits()).isEqualTo(aBoolean); + } + } + + @Test + public void memTable() throws RocksDBException { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + opt.setMemTableConfig(new HashLinkedListMemTableConfig()); + assertThat(opt.memTableFactoryName()). + isEqualTo("HashLinkedListRepFactory"); + } + } + + @Test + public void comparator() throws RocksDBException { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + opt.setComparator(BuiltinComparator.BYTEWISE_COMPARATOR); + } + } + + @Test + public void linkageOfPrepMethods() { + try (final ColumnFamilyOptions options = new ColumnFamilyOptions()) { + options.optimizeUniversalStyleCompaction(); + options.optimizeUniversalStyleCompaction(4000); + options.optimizeLevelStyleCompaction(); + options.optimizeLevelStyleCompaction(3000); + options.optimizeForPointLookup(10); + options.optimizeForSmallDb(); + } + } + + @Test + public void shouldSetTestPrefixExtractor() { + try (final ColumnFamilyOptions options = new ColumnFamilyOptions()) { + options.useFixedLengthPrefixExtractor(100); + options.useFixedLengthPrefixExtractor(10); + } + } + + @Test + public void shouldSetTestCappedPrefixExtractor() { + try (final ColumnFamilyOptions options = new ColumnFamilyOptions()) { + options.useCappedPrefixExtractor(100); + options.useCappedPrefixExtractor(10); + } + } + + @Test + public void compressionTypes() { + try (final ColumnFamilyOptions columnFamilyOptions + = new ColumnFamilyOptions()) { + for (final CompressionType compressionType : + CompressionType.values()) { + columnFamilyOptions.setCompressionType(compressionType); + assertThat(columnFamilyOptions.compressionType()). + isEqualTo(compressionType); + assertThat(CompressionType.valueOf("NO_COMPRESSION")). + isEqualTo(CompressionType.NO_COMPRESSION); + } + } + } + + @Test + public void compressionPerLevel() { + try (final ColumnFamilyOptions columnFamilyOptions + = new ColumnFamilyOptions()) { + assertThat(columnFamilyOptions.compressionPerLevel()).isEmpty(); + List compressionTypeList = new ArrayList<>(); + for (int i = 0; i < columnFamilyOptions.numLevels(); i++) { + compressionTypeList.add(CompressionType.NO_COMPRESSION); + } + columnFamilyOptions.setCompressionPerLevel(compressionTypeList); + compressionTypeList = columnFamilyOptions.compressionPerLevel(); + for (final CompressionType compressionType : compressionTypeList) { + assertThat(compressionType).isEqualTo( + CompressionType.NO_COMPRESSION); + } + } + } + + @Test + public void differentCompressionsPerLevel() { + try (final ColumnFamilyOptions columnFamilyOptions + = new ColumnFamilyOptions()) { + columnFamilyOptions.setNumLevels(3); + + assertThat(columnFamilyOptions.compressionPerLevel()).isEmpty(); + List compressionTypeList = new ArrayList<>(); + + compressionTypeList.add(CompressionType.BZLIB2_COMPRESSION); + compressionTypeList.add(CompressionType.SNAPPY_COMPRESSION); + compressionTypeList.add(CompressionType.LZ4_COMPRESSION); + + columnFamilyOptions.setCompressionPerLevel(compressionTypeList); + compressionTypeList = columnFamilyOptions.compressionPerLevel(); + + assertThat(compressionTypeList.size()).isEqualTo(3); + assertThat(compressionTypeList). + containsExactly( + CompressionType.BZLIB2_COMPRESSION, + CompressionType.SNAPPY_COMPRESSION, + CompressionType.LZ4_COMPRESSION); + + } + } + + @Test + public void bottommostCompressionType() { + try (final ColumnFamilyOptions columnFamilyOptions + = new ColumnFamilyOptions()) { + assertThat(columnFamilyOptions.bottommostCompressionType()) + .isEqualTo(CompressionType.DISABLE_COMPRESSION_OPTION); + + for (final CompressionType compressionType : CompressionType.values()) { + columnFamilyOptions.setBottommostCompressionType(compressionType); + assertThat(columnFamilyOptions.bottommostCompressionType()) + .isEqualTo(compressionType); + } + } + } + + @Test + public void bottommostCompressionOptions() { + try (final ColumnFamilyOptions columnFamilyOptions = + new ColumnFamilyOptions(); + final CompressionOptions bottommostCompressionOptions = + new CompressionOptions() + .setMaxDictBytes(123)) { + + columnFamilyOptions.setBottommostCompressionOptions( + bottommostCompressionOptions); + assertThat(columnFamilyOptions.bottommostCompressionOptions()) + .isEqualTo(bottommostCompressionOptions); + assertThat(columnFamilyOptions.bottommostCompressionOptions() + .maxDictBytes()).isEqualTo(123); + } + } + + @Test + public void compressionOptions() { + try (final ColumnFamilyOptions columnFamilyOptions + = new ColumnFamilyOptions(); + final CompressionOptions compressionOptions = new CompressionOptions() + .setMaxDictBytes(123)) { + + columnFamilyOptions.setCompressionOptions(compressionOptions); + assertThat(columnFamilyOptions.compressionOptions()) + .isEqualTo(compressionOptions); + assertThat(columnFamilyOptions.compressionOptions().maxDictBytes()) + .isEqualTo(123); + } + } + + @Test + public void compactionStyles() { + try (final ColumnFamilyOptions columnFamilyOptions + = new ColumnFamilyOptions()) { + for (final CompactionStyle compactionStyle : + CompactionStyle.values()) { + columnFamilyOptions.setCompactionStyle(compactionStyle); + assertThat(columnFamilyOptions.compactionStyle()). + isEqualTo(compactionStyle); + assertThat(CompactionStyle.valueOf("FIFO")). + isEqualTo(CompactionStyle.FIFO); + } + } + } + + @Test + public void maxTableFilesSizeFIFO() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + long longValue = rand.nextLong(); + // Size has to be positive + longValue = (longValue < 0) ? -longValue : longValue; + longValue = (longValue == 0) ? longValue + 1 : longValue; + opt.setMaxTableFilesSizeFIFO(longValue); + assertThat(opt.maxTableFilesSizeFIFO()). + isEqualTo(longValue); + } + } + + @Test + public void maxWriteBufferNumberToMaintain() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + int intValue = rand.nextInt(); + // Size has to be positive + intValue = (intValue < 0) ? -intValue : intValue; + intValue = (intValue == 0) ? intValue + 1 : intValue; + opt.setMaxWriteBufferNumberToMaintain(intValue); + assertThat(opt.maxWriteBufferNumberToMaintain()). + isEqualTo(intValue); + } + } + + @Test + public void compactionPriorities() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + for (final CompactionPriority compactionPriority : + CompactionPriority.values()) { + opt.setCompactionPriority(compactionPriority); + assertThat(opt.compactionPriority()). + isEqualTo(compactionPriority); + } + } + } + + @Test + public void reportBgIoStats() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final boolean booleanValue = true; + opt.setReportBgIoStats(booleanValue); + assertThat(opt.reportBgIoStats()). + isEqualTo(booleanValue); + } + } + + @Test + public void ttl() { + try (final ColumnFamilyOptions options = new ColumnFamilyOptions()) { + options.setTtl(1000 * 60); + assertThat(options.ttl()). + isEqualTo(1000 * 60); + } + } + + @Test + public void periodicCompactionSeconds() { + try (final ColumnFamilyOptions options = new ColumnFamilyOptions()) { + options.setPeriodicCompactionSeconds(1000 * 60); + assertThat(options.periodicCompactionSeconds()).isEqualTo(1000 * 60); + } + } + + @Test + public void compactionOptionsUniversal() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions(); + final CompactionOptionsUniversal optUni = new CompactionOptionsUniversal() + .setCompressionSizePercent(7)) { + opt.setCompactionOptionsUniversal(optUni); + assertThat(opt.compactionOptionsUniversal()). + isEqualTo(optUni); + assertThat(opt.compactionOptionsUniversal().compressionSizePercent()) + .isEqualTo(7); + } + } + + @Test + public void compactionOptionsFIFO() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions(); + final CompactionOptionsFIFO optFifo = new CompactionOptionsFIFO() + .setMaxTableFilesSize(2000)) { + opt.setCompactionOptionsFIFO(optFifo); + assertThat(opt.compactionOptionsFIFO()). + isEqualTo(optFifo); + assertThat(opt.compactionOptionsFIFO().maxTableFilesSize()) + .isEqualTo(2000); + } + } + + @Test + public void forceConsistencyChecks() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final boolean booleanValue = true; + opt.setForceConsistencyChecks(booleanValue); + assertThat(opt.forceConsistencyChecks()). + isEqualTo(booleanValue); + } + } + + @Test + public void compactionFilter() { + try(final ColumnFamilyOptions options = new ColumnFamilyOptions(); + final RemoveEmptyValueCompactionFilter cf = new RemoveEmptyValueCompactionFilter()) { + options.setCompactionFilter(cf); + assertThat(options.compactionFilter()).isEqualTo(cf); + } + } + + @Test + public void compactionFilterFactory() { + try(final ColumnFamilyOptions options = new ColumnFamilyOptions(); + final RemoveEmptyValueCompactionFilterFactory cff = new RemoveEmptyValueCompactionFilterFactory()) { + options.setCompactionFilterFactory(cff); + assertThat(options.compactionFilterFactory()).isEqualTo(cff); + } + } + + @Test + public void compactionThreadLimiter() { + try (final ColumnFamilyOptions options = new ColumnFamilyOptions(); + final ConcurrentTaskLimiter compactionThreadLimiter = + new ConcurrentTaskLimiterImpl("name", 3)) { + options.setCompactionThreadLimiter(compactionThreadLimiter); + assertThat(options.compactionThreadLimiter()).isEqualTo(compactionThreadLimiter); + } + } + + @Test + public void oldDefaults() { + try (final ColumnFamilyOptions options = new ColumnFamilyOptions()) { + options.oldDefaults(4, 6); + assertEquals(4 << 20, options.writeBufferSize()); + assertThat(options.compactionPriority()).isEqualTo(CompactionPriority.ByCompensatedSize); + assertThat(options.targetFileSizeBase()).isEqualTo(2 * 1048576); + assertThat(options.maxBytesForLevelBase()).isEqualTo(10 * 1048576); + assertThat(options.softPendingCompactionBytesLimit()).isEqualTo(0); + assertThat(options.hardPendingCompactionBytesLimit()).isEqualTo(0); + assertThat(options.level0StopWritesTrigger()).isEqualTo(24); + } + } + + @Test + public void optimizeForSmallDbWithCache() { + try (final ColumnFamilyOptions options = new ColumnFamilyOptions(); + final Cache cache = new LRUCache(1024)) { + assertThat(options.optimizeForSmallDb(cache)).isEqualTo(options); + } + } + + @Test + public void cfPaths() throws IOException { + try (final ColumnFamilyOptions options = new ColumnFamilyOptions()) { + final List paths = Arrays.asList( + new DbPath(Paths.get("test1"), 2 << 25), new DbPath(Paths.get("/test2/path"), 2 << 25)); + assertThat(options.cfPaths()).isEqualTo(Collections.emptyList()); + assertThat(options.setCfPaths(paths)).isEqualTo(options); + assertThat(options.cfPaths()).isEqualTo(paths); + } + } + + @Test + public void memtableMaxRangeDeletions() { + try (final ColumnFamilyOptions options = new ColumnFamilyOptions()) { + assertThat(options.memtableMaxRangeDeletions()).isEqualTo(0); + final int val = 32; + assertThat(options.setMemtableMaxRangeDeletions(val)).isEqualTo(options); + assertThat(options.memtableMaxRangeDeletions()).isEqualTo(val); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ColumnFamilyTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ColumnFamilyTest.java new file mode 100644 index 0000000..fb8a450 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ColumnFamilyTest.java @@ -0,0 +1,562 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.*; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class ColumnFamilyTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void columnFamilyDescriptorName() throws RocksDBException { + final byte[] cfName = "some_name".getBytes(UTF_8); + + try(final ColumnFamilyOptions cfOptions = new ColumnFamilyOptions()) { + final ColumnFamilyDescriptor cfDescriptor = new ColumnFamilyDescriptor(cfName, cfOptions); + assertThat(cfDescriptor.getName()).isEqualTo(cfName); + } + } + + @Test + public void columnFamilyDescriptorOptions() throws RocksDBException { + final byte[] cfName = "some_name".getBytes(UTF_8); + + try (final ColumnFamilyOptions cfOptions = + new ColumnFamilyOptions().setCompressionType(CompressionType.BZLIB2_COMPRESSION)) { + final ColumnFamilyDescriptor cfDescriptor = + new ColumnFamilyDescriptor(cfName, cfOptions); + + assertThat(cfDescriptor.getOptions().compressionType()) + .isEqualTo(CompressionType.BZLIB2_COMPRESSION); + } + } + + @Test + public void listColumnFamilies() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB ignored = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + // Test listColumnFamilies + final List columnFamilyNames = + RocksDB.listColumnFamilies(options, dbFolder.getRoot().getAbsolutePath()); + assertThat(columnFamilyNames).isNotNull(); + assertThat(columnFamilyNames.size()).isGreaterThan(0); + assertThat(columnFamilyNames.size()).isEqualTo(1); + assertThat(new String(columnFamilyNames.get(0))).isEqualTo("default"); + } + } + + @Test + public void defaultColumnFamily() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath())) { + try (final ColumnFamilyHandle cfh = db.getDefaultColumnFamily()) { + assertThat(cfh).isNotNull(); + + assertThat(cfh.getName()).isEqualTo("default".getBytes(UTF_8)); + assertThat(cfh.getID()).isEqualTo(0); + assertThat(cfh.getDescriptor().getName()).isEqualTo("default".getBytes(UTF_8)); + + final byte[] key = "key".getBytes(); + final byte[] value = "value".getBytes(); + + db.put(cfh, key, value); + + final byte[] actualValue = db.get(cfh, key); + + assertThat(cfh).isNotNull(); + assertThat(actualValue).isEqualTo(value); + } + } + } + + @Test + public void createColumnFamily() throws RocksDBException { + final byte[] cfName = "new_cf".getBytes(UTF_8); + final ColumnFamilyDescriptor cfDescriptor = + new ColumnFamilyDescriptor(cfName, new ColumnFamilyOptions()); + + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + try (final ColumnFamilyHandle columnFamilyHandle = db.createColumnFamily(cfDescriptor)) { + assertThat(columnFamilyHandle.getName()).isEqualTo(cfName); + assertThat(columnFamilyHandle.getID()).isEqualTo(1); + + final ColumnFamilyDescriptor latestDescriptor = columnFamilyHandle.getDescriptor(); + assertThat(latestDescriptor.getName()).isEqualTo(cfName); + + final List columnFamilyNames = + RocksDB.listColumnFamilies(options, dbFolder.getRoot().getAbsolutePath()); + assertThat(columnFamilyNames).isNotNull(); + assertThat(columnFamilyNames.size()).isGreaterThan(0); + assertThat(columnFamilyNames.size()).isEqualTo(2); + assertThat(new String(columnFamilyNames.get(0))).isEqualTo("default"); + assertThat(new String(columnFamilyNames.get(1))).isEqualTo("new_cf"); + } + } + } + + @Test + public void openWithColumnFamilies() throws RocksDBException { + final List cfNames = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes()) + ); + + final List columnFamilyHandleList = + new ArrayList<>(); + + // Test open database with column family names + try (final DBOptions options = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath(), cfNames, + columnFamilyHandleList)) { + assertThat(columnFamilyHandleList.size()).isEqualTo(2); + db.put("dfkey1".getBytes(), "dfvalue".getBytes()); + db.put(columnFamilyHandleList.get(0), "dfkey2".getBytes(), "dfvalue".getBytes()); + db.put(columnFamilyHandleList.get(1), "newcfkey1".getBytes(), "newcfvalue".getBytes()); + + final String retVal = + new String(db.get(columnFamilyHandleList.get(1), "newcfkey1".getBytes())); + assertThat(retVal).isEqualTo("newcfvalue"); + assertThat((db.get(columnFamilyHandleList.get(1), "dfkey1".getBytes()))).isNull(); + db.delete(columnFamilyHandleList.get(1), "newcfkey1".getBytes()); + assertThat((db.get(columnFamilyHandleList.get(1), "newcfkey1".getBytes()))).isNull(); + db.delete(columnFamilyHandleList.get(0), new WriteOptions(), "dfkey2".getBytes()); + assertThat(db.get(columnFamilyHandleList.get(0), new ReadOptions(), "dfkey2".getBytes())) + .isNull(); + } + } + + @Test + public void getWithOutValueAndCf() throws RocksDBException { + final List cfDescriptors = + Collections.singletonList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY)); + final List columnFamilyHandleList = new ArrayList<>(); + + // Test open database with column family names + try (final DBOptions options = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath(), cfDescriptors, + columnFamilyHandleList)) { + db.put( + columnFamilyHandleList.get(0), new WriteOptions(), "key1".getBytes(), "value".getBytes()); + db.put("key2".getBytes(), "12345678".getBytes()); + final byte[] outValue = new byte[5]; + // not found value + int getResult = db.get("keyNotFound".getBytes(), outValue); + assertThat(getResult).isEqualTo(RocksDB.NOT_FOUND); + // found value which fits in outValue + getResult = db.get(columnFamilyHandleList.get(0), "key1".getBytes(), outValue); + assertThat(getResult).isNotEqualTo(RocksDB.NOT_FOUND); + assertThat(outValue).isEqualTo("value".getBytes()); + // found value which fits partially + getResult = + db.get(columnFamilyHandleList.get(0), new ReadOptions(), "key2".getBytes(), outValue); + assertThat(getResult).isNotEqualTo(RocksDB.NOT_FOUND); + assertThat(outValue).isEqualTo("12345".getBytes()); + } + } + + @Test + public void createWriteDropColumnFamily() throws RocksDBException { + final List cfDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes())); + final List columnFamilyHandleList = new ArrayList<>(); + try (final DBOptions options = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath(), cfDescriptors, + columnFamilyHandleList)) { + final ColumnFamilyHandle tmpColumnFamilyHandle; + tmpColumnFamilyHandle = db.createColumnFamily( + new ColumnFamilyDescriptor("tmpCF".getBytes(), new ColumnFamilyOptions())); + db.put(tmpColumnFamilyHandle, "key".getBytes(), "value".getBytes()); + db.dropColumnFamily(tmpColumnFamilyHandle); + assertThat(tmpColumnFamilyHandle.isOwningHandle()).isTrue(); + } + } + + @Test + public void createWriteDropColumnFamilies() throws RocksDBException { + final List cfDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes())); + final List columnFamilyHandleList = new ArrayList<>(); + try (final DBOptions options = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath(), cfDescriptors, + columnFamilyHandleList)) { + final ColumnFamilyHandle tmpColumnFamilyHandle; + final ColumnFamilyHandle tmpColumnFamilyHandle2; + tmpColumnFamilyHandle = db.createColumnFamily( + new ColumnFamilyDescriptor("tmpCF".getBytes(), new ColumnFamilyOptions())); + tmpColumnFamilyHandle2 = db.createColumnFamily( + new ColumnFamilyDescriptor("tmpCF2".getBytes(), new ColumnFamilyOptions())); + db.put(tmpColumnFamilyHandle, "key".getBytes(), "value".getBytes()); + db.put(tmpColumnFamilyHandle2, "key".getBytes(), "value".getBytes()); + db.dropColumnFamilies(Arrays.asList(tmpColumnFamilyHandle, tmpColumnFamilyHandle2)); + assertThat(tmpColumnFamilyHandle.isOwningHandle()).isTrue(); + assertThat(tmpColumnFamilyHandle2.isOwningHandle()).isTrue(); + } + } + + @Test + public void writeBatch() throws RocksDBException { + try (final StringAppendOperator stringAppendOperator = new StringAppendOperator(); + final ColumnFamilyOptions defaultCfOptions = new ColumnFamilyOptions() + .setMergeOperator(stringAppendOperator)) { + final List cfDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, + defaultCfOptions), + new ColumnFamilyDescriptor("new_cf".getBytes())); + final List columnFamilyHandleList = new ArrayList<>(); + try (final DBOptions options = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath(), + cfDescriptors, columnFamilyHandleList); + final WriteBatch writeBatch = new WriteBatch(); + final WriteOptions writeOpt = new WriteOptions()) { + writeBatch.put("key".getBytes(), "value".getBytes()); + writeBatch.put(db.getDefaultColumnFamily(), "mergeKey".getBytes(), "merge".getBytes()); + writeBatch.merge(db.getDefaultColumnFamily(), "mergeKey".getBytes(), "merge".getBytes()); + writeBatch.put(columnFamilyHandleList.get(1), "newcfkey".getBytes(), "value".getBytes()); + writeBatch.put(columnFamilyHandleList.get(1), "newcfkey2".getBytes(), "value2".getBytes()); + writeBatch.delete("xyz".getBytes()); + writeBatch.delete(columnFamilyHandleList.get(1), "xyz".getBytes()); + db.write(writeOpt, writeBatch); + + assertThat(db.get(columnFamilyHandleList.get(1), "xyz".getBytes())).isNull(); + assertThat(new String(db.get(columnFamilyHandleList.get(1), "newcfkey".getBytes()))) + .isEqualTo("value"); + assertThat(new String(db.get(columnFamilyHandleList.get(1), "newcfkey2".getBytes()))) + .isEqualTo("value2"); + assertThat(new String(db.get("key".getBytes()))).isEqualTo("value"); + // check if key is merged + assertThat(new String(db.get(db.getDefaultColumnFamily(), "mergeKey".getBytes()))) + .isEqualTo("merge,merge"); + } + } + } + + @Test + public void iteratorOnColumnFamily() throws RocksDBException { + final List cfDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes())); + final List columnFamilyHandleList = new ArrayList<>(); + try (final DBOptions options = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath(), + cfDescriptors, columnFamilyHandleList)) { + db.put(columnFamilyHandleList.get(1), "newcfkey".getBytes(), "value".getBytes()); + db.put(columnFamilyHandleList.get(1), "newcfkey2".getBytes(), "value2".getBytes()); + try (final RocksIterator rocksIterator = db.newIterator(columnFamilyHandleList.get(1))) { + rocksIterator.seekToFirst(); + final Map refMap = new HashMap<>(); + refMap.put("newcfkey", "value"); + refMap.put("newcfkey2", "value2"); + int i = 0; + while (rocksIterator.isValid()) { + i++; + assertThat(refMap.get(new String(rocksIterator.key()))) + .isEqualTo(new String(rocksIterator.value())); + rocksIterator.next(); + } + assertThat(i).isEqualTo(2); + } + } + } + + @Test + public void multiGet() throws RocksDBException { + final List cfDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes())); + final List columnFamilyHandleList = new ArrayList<>(); + try (final DBOptions options = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath(), + cfDescriptors, columnFamilyHandleList)) { + db.put(columnFamilyHandleList.get(0), "key".getBytes(), "value".getBytes()); + db.put(columnFamilyHandleList.get(1), "newcfkey".getBytes(), "value".getBytes()); + + final List keys = Arrays.asList("key".getBytes(), "newcfkey".getBytes()); + + List retValues = db.multiGetAsList(columnFamilyHandleList, keys); + assertThat(retValues.size()).isEqualTo(2); + assertThat(new String(retValues.get(0))).isEqualTo("value"); + assertThat(new String(retValues.get(1))).isEqualTo("value"); + retValues = db.multiGetAsList(new ReadOptions(), columnFamilyHandleList, keys); + assertThat(retValues.size()).isEqualTo(2); + assertThat(new String(retValues.get(0))).isEqualTo("value"); + assertThat(new String(retValues.get(1))).isEqualTo("value"); + } + } + + @Test + public void multiGetAsList() throws RocksDBException { + final List cfDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes())); + final List columnFamilyHandleList = new ArrayList<>(); + try (final DBOptions options = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath(), + cfDescriptors, columnFamilyHandleList)) { + db.put(columnFamilyHandleList.get(0), "key".getBytes(), "value".getBytes()); + db.put(columnFamilyHandleList.get(1), "newcfkey".getBytes(), "value".getBytes()); + + final List keys = Arrays.asList("key".getBytes(), "newcfkey".getBytes()); + List retValues = db.multiGetAsList(columnFamilyHandleList, keys); + assertThat(retValues.size()).isEqualTo(2); + assertThat(new String(retValues.get(0))).isEqualTo("value"); + assertThat(new String(retValues.get(1))).isEqualTo("value"); + retValues = db.multiGetAsList(new ReadOptions(), columnFamilyHandleList, keys); + assertThat(retValues.size()).isEqualTo(2); + assertThat(new String(retValues.get(0))).isEqualTo("value"); + assertThat(new String(retValues.get(1))).isEqualTo("value"); + } + } + + @Test + public void properties() throws RocksDBException { + final List cfDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes())); + final List columnFamilyHandleList = new ArrayList<>(); + try (final DBOptions options = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath(), + cfDescriptors, columnFamilyHandleList)) { + assertThat(db.getProperty("rocksdb.estimate-num-keys")).isNotNull(); + assertThat(db.getLongProperty(columnFamilyHandleList.get(0), "rocksdb.estimate-num-keys")) + .isGreaterThanOrEqualTo(0); + assertThat(db.getProperty("rocksdb.stats")).isNotNull(); + assertThat(db.getProperty(columnFamilyHandleList.get(0), "rocksdb.sstables")).isNotNull(); + assertThat(db.getProperty(columnFamilyHandleList.get(1), "rocksdb.estimate-num-keys")) + .isNotNull(); + assertThat(db.getProperty(columnFamilyHandleList.get(1), "rocksdb.stats")).isNotNull(); + assertThat(db.getProperty(columnFamilyHandleList.get(1), "rocksdb.sstables")).isNotNull(); + assertThat(db.getAggregatedLongProperty("rocksdb.estimate-num-keys")).isNotNull(); + assertThat(db.getAggregatedLongProperty("rocksdb.estimate-num-keys")) + .isGreaterThanOrEqualTo(0); + } + } + + + @Test + public void iterators() throws RocksDBException { + final List cfDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes())); + final List columnFamilyHandleList = new ArrayList<>(); + try (final DBOptions options = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath(), cfDescriptors, + columnFamilyHandleList)) { + List iterators = null; + try { + iterators = db.newIterators(columnFamilyHandleList); + assertThat(iterators.size()).isEqualTo(2); + RocksIterator iter = iterators.get(0); + iter.seekToFirst(); + final Map defRefMap = new HashMap<>(); + defRefMap.put("dfkey1", "dfvalue"); + defRefMap.put("key", "value"); + while (iter.isValid()) { + assertThat(defRefMap.get(new String(iter.key()))). + isEqualTo(new String(iter.value())); + iter.next(); + } + // iterate over new_cf key/value pairs + final Map cfRefMap = new HashMap<>(); + cfRefMap.put("newcfkey", "value"); + cfRefMap.put("newcfkey2", "value2"); + iter = iterators.get(1); + iter.seekToFirst(); + while (iter.isValid()) { + assertThat(cfRefMap.get(new String(iter.key()))). + isEqualTo(new String(iter.value())); + iter.next(); + } + } finally { + if (iterators != null) { + for (final RocksIterator rocksIterator : iterators) { + rocksIterator.close(); + } + } + } + } + } + + @Test(expected = RocksDBException.class) + public void failPutDisposedCF() throws RocksDBException { + final List cfDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes())); + final List columnFamilyHandleList = new ArrayList<>(); + try (final DBOptions options = new DBOptions() + .setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath(), + cfDescriptors, columnFamilyHandleList)) { + db.dropColumnFamily(columnFamilyHandleList.get(1)); + db.put(columnFamilyHandleList.get(1), "key".getBytes(), "value".getBytes()); + } + } + + @Test(expected = RocksDBException.class) + public void failRemoveDisposedCF() throws RocksDBException { + final List cfDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes())); + final List columnFamilyHandleList = new ArrayList<>(); + try (final DBOptions options = new DBOptions() + .setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath(), + cfDescriptors, columnFamilyHandleList)) { + db.dropColumnFamily(columnFamilyHandleList.get(1)); + db.delete(columnFamilyHandleList.get(1), "key".getBytes()); + } + } + + @Test(expected = RocksDBException.class) + public void failGetDisposedCF() throws RocksDBException { + final List cfDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes())); + final List columnFamilyHandleList = new ArrayList<>(); + try (final DBOptions options = new DBOptions() + .setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath(), cfDescriptors, + columnFamilyHandleList)) { + db.dropColumnFamily(columnFamilyHandleList.get(1)); + db.get(columnFamilyHandleList.get(1), "key".getBytes()); + } + } + + @Test(expected = RocksDBException.class) + public void failMultiGetWithoutCorrectNumberOfCF() throws RocksDBException { + final List cfDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes())); + final List columnFamilyHandleList = new ArrayList<>(); + try (final DBOptions options = new DBOptions() + .setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath(), cfDescriptors, + columnFamilyHandleList)) { + final List keys = new ArrayList<>(); + keys.add("key".getBytes()); + keys.add("newcfkey".getBytes()); + final List cfCustomList = new ArrayList<>(); + db.multiGetAsList(cfCustomList, keys); + } + } + + @Test + public void testByteCreateFolumnFamily() throws RocksDBException { + + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath()) + ) { + final byte[] b0 = new byte[]{(byte) 0x00}; + final byte[] b1 = new byte[]{(byte) 0x01}; + final byte[] b2 = new byte[]{(byte) 0x02}; + db.createColumnFamily(new ColumnFamilyDescriptor(b0)); + db.createColumnFamily(new ColumnFamilyDescriptor(b1)); + final List families = + RocksDB.listColumnFamilies(options, dbFolder.getRoot().getAbsolutePath()); + assertThat(families).contains("default".getBytes(), b0, b1); + db.createColumnFamily(new ColumnFamilyDescriptor(b2)); + } + } + + @Test + public void testCFNamesWithZeroBytes() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + final byte[] b0 = new byte[] {0, 0}; + final byte[] b1 = new byte[] {0, 1}; + db.createColumnFamily(new ColumnFamilyDescriptor(b0)); + db.createColumnFamily(new ColumnFamilyDescriptor(b1)); + final List families = + RocksDB.listColumnFamilies(options, dbFolder.getRoot().getAbsolutePath()); + assertThat(families).contains("default".getBytes(), b0, b1); + } + } + + @Test + public void testCFNameSimplifiedChinese() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + final String simplifiedChinese = "\u7b80\u4f53\u5b57"; + db.createColumnFamily(new ColumnFamilyDescriptor(simplifiedChinese.getBytes())); + + final List families = + RocksDB.listColumnFamilies(options, dbFolder.getRoot().getAbsolutePath()); + assertThat(families).contains("default".getBytes(), simplifiedChinese.getBytes()); + } + } + + @Test + public void testDestroyColumnFamilyHandle() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + final byte[] name1 = "cf1".getBytes(); + final byte[] name2 = "cf2".getBytes(); + final ColumnFamilyDescriptor desc1 = new ColumnFamilyDescriptor(name1); + final ColumnFamilyDescriptor desc2 = new ColumnFamilyDescriptor(name2); + final ColumnFamilyHandle cf1 = db.createColumnFamily(desc1); + final ColumnFamilyHandle cf2 = db.createColumnFamily(desc2); + assertTrue(cf1.isOwningHandle()); + assertTrue(cf2.isOwningHandle()); + assertFalse(cf1.isDefaultColumnFamily()); + db.destroyColumnFamilyHandle(cf1); + // At this point cf1 should not be used! + assertFalse(cf1.isOwningHandle()); + assertTrue(cf2.isOwningHandle()); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactRangeOptionsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactRangeOptionsTest.java new file mode 100644 index 0000000..549b74b --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactRangeOptionsTest.java @@ -0,0 +1,138 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.Test; +import org.rocksdb.CompactRangeOptions.BottommostLevelCompaction; + +import static org.assertj.core.api.Assertions.assertThat; + +public class CompactRangeOptionsTest { + + static { + RocksDB.loadLibrary(); + } + + @Test + public void exclusiveManualCompaction() { + try (final CompactRangeOptions opt = new CompactRangeOptions()) { + opt.setExclusiveManualCompaction(false); + assertThat(opt.exclusiveManualCompaction()).isEqualTo(false); + opt.setExclusiveManualCompaction(true); + assertThat(opt.exclusiveManualCompaction()).isEqualTo(true); + } + } + + @Test + public void bottommostLevelCompaction() { + try (final CompactRangeOptions opt = new CompactRangeOptions()) { + BottommostLevelCompaction value = BottommostLevelCompaction.kSkip; + opt.setBottommostLevelCompaction(value); + assertThat(opt.bottommostLevelCompaction()).isEqualTo(value); + value = BottommostLevelCompaction.kForce; + opt.setBottommostLevelCompaction(value); + assertThat(opt.bottommostLevelCompaction()).isEqualTo(value); + value = BottommostLevelCompaction.kIfHaveCompactionFilter; + opt.setBottommostLevelCompaction(value); + assertThat(opt.bottommostLevelCompaction()).isEqualTo(value); + value = BottommostLevelCompaction.kForceOptimized; + opt.setBottommostLevelCompaction(value); + assertThat(opt.bottommostLevelCompaction()).isEqualTo(value); + } + } + + @Test + public void changeLevel() { + try (final CompactRangeOptions opt = new CompactRangeOptions()) { + opt.setChangeLevel(false); + assertThat(opt.changeLevel()).isEqualTo(false); + opt.setChangeLevel(true); + assertThat(opt.changeLevel()).isEqualTo(true); + } + } + + @Test + public void targetLevel() { + try (final CompactRangeOptions opt = new CompactRangeOptions()) { + int value = 2; + opt.setTargetLevel(value); + assertThat(opt.targetLevel()).isEqualTo(value); + value = 3; + opt.setTargetLevel(value); + assertThat(opt.targetLevel()).isEqualTo(value); + } + } + + @Test + public void targetPathId() { + try (final CompactRangeOptions opt = new CompactRangeOptions()) { + int value = 2; + opt.setTargetPathId(value); + assertThat(opt.targetPathId()).isEqualTo(value); + value = 3; + opt.setTargetPathId(value); + assertThat(opt.targetPathId()).isEqualTo(value); + } + } + + @Test + public void allowWriteStall() { + try (final CompactRangeOptions opt = new CompactRangeOptions()) { + opt.setAllowWriteStall(false); + assertThat(opt.allowWriteStall()).isEqualTo(false); + opt.setAllowWriteStall(true); + assertThat(opt.allowWriteStall()).isEqualTo(true); + } + } + + @Test + public void maxSubcompactions() { + try (final CompactRangeOptions opt = new CompactRangeOptions()) { + int value = 2; + opt.setMaxSubcompactions(value); + assertThat(opt.maxSubcompactions()).isEqualTo(value); + value = 3; + opt.setMaxSubcompactions(value); + assertThat(opt.maxSubcompactions()).isEqualTo(value); + } + } + + @Test + public void fullHistoryTSLow() { + CompactRangeOptions opt = new CompactRangeOptions(); + CompactRangeOptions.Timestamp timestamp = new CompactRangeOptions.Timestamp(18, 1); + opt.setFullHistoryTSLow(timestamp); + + for (int times = 1; times <= 2; times++) { + // worried slightly about destructive reads, so read it twice + CompactRangeOptions.Timestamp timestampResult = opt.fullHistoryTSLow(); + assertThat(timestamp.start).isEqualTo(timestampResult.start); + assertThat(timestamp.range).isEqualTo(timestampResult.range); + assertThat(timestamp).isEqualTo(timestampResult); + } + } + + @Test + public void fullHistoryTSLowDefault() { + CompactRangeOptions opt = new CompactRangeOptions(); + CompactRangeOptions.Timestamp timestampResult = opt.fullHistoryTSLow(); + assertThat(timestampResult).isNull(); + } + + @Test + public void canceled() { + CompactRangeOptions opt = new CompactRangeOptions(); + assertThat(opt.canceled()).isEqualTo(false); + opt.setCanceled(true); + assertThat(opt.canceled()).isEqualTo(true); + opt.setCanceled(false); + assertThat(opt.canceled()).isEqualTo(false); + opt.setCanceled(true); + assertThat(opt.canceled()).isEqualTo(true); + opt.setCanceled(true); + assertThat(opt.canceled()).isEqualTo(true); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionFilterFactoryTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionFilterFactoryTest.java new file mode 100644 index 0000000..35a14eb --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionFilterFactoryTest.java @@ -0,0 +1,61 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.test.RemoveEmptyValueCompactionFilterFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +public class CompactionFilterFactoryTest { + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void columnFamilyOptions_setCompactionFilterFactory() + throws RocksDBException { + try(final DBOptions options = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + final RemoveEmptyValueCompactionFilterFactory compactionFilterFactory + = new RemoveEmptyValueCompactionFilterFactory(); + final ColumnFamilyOptions new_cf_opts + = new ColumnFamilyOptions() + .setCompactionFilterFactory(compactionFilterFactory)) { + + final List cfNames = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes(), new_cf_opts)); + + final List cfHandles = new ArrayList<>(); + + try (final RocksDB rocksDb = + RocksDB.open(options, dbFolder.getRoot().getAbsolutePath(), cfNames, cfHandles)) { + final byte[] key1 = "key1".getBytes(); + final byte[] key2 = "key2".getBytes(); + + final byte[] value1 = "value1".getBytes(); + final byte[] value2 = new byte[0]; + + rocksDb.put(cfHandles.get(1), key1, value1); + rocksDb.put(cfHandles.get(1), key2, value2); + + rocksDb.compactRange(cfHandles.get(1)); + + assertThat(rocksDb.get(cfHandles.get(1), key1)).isEqualTo(value1); + final boolean exists = rocksDb.keyMayExist(cfHandles.get(1), key2, null); + assertThat(exists).isFalse(); + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionJobInfoTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionJobInfoTest.java new file mode 100644 index 0000000..c71b0da --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionJobInfoTest.java @@ -0,0 +1,114 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class CompactionJobInfoTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Test + public void columnFamilyName() { + try (final CompactionJobInfo compactionJobInfo = new CompactionJobInfo()) { + assertThat(compactionJobInfo.columnFamilyName()) + .isEmpty(); + } + } + + @Test + public void status() { + try (final CompactionJobInfo compactionJobInfo = new CompactionJobInfo()) { + assertThat(compactionJobInfo.status().getCode()) + .isEqualTo(Status.Code.Ok); + } + } + + @Test + public void threadId() { + try (final CompactionJobInfo compactionJobInfo = new CompactionJobInfo()) { + assertThat(compactionJobInfo.threadId()) + .isEqualTo(0); + } + } + + @Test + public void jobId() { + try (final CompactionJobInfo compactionJobInfo = new CompactionJobInfo()) { + assertThat(compactionJobInfo.jobId()) + .isEqualTo(0); + } + } + + @Test + public void baseInputLevel() { + try (final CompactionJobInfo compactionJobInfo = new CompactionJobInfo()) { + assertThat(compactionJobInfo.baseInputLevel()) + .isEqualTo(0); + } + } + + @Test + public void outputLevel() { + try (final CompactionJobInfo compactionJobInfo = new CompactionJobInfo()) { + assertThat(compactionJobInfo.outputLevel()) + .isEqualTo(0); + } + } + + @Test + public void inputFiles() { + try (final CompactionJobInfo compactionJobInfo = new CompactionJobInfo()) { + assertThat(compactionJobInfo.inputFiles()) + .isEmpty(); + } + } + + @Test + public void outputFiles() { + try (final CompactionJobInfo compactionJobInfo = new CompactionJobInfo()) { + assertThat(compactionJobInfo.outputFiles()) + .isEmpty(); + } + } + + @Test + public void tableProperties() { + try (final CompactionJobInfo compactionJobInfo = new CompactionJobInfo()) { + assertThat(compactionJobInfo.tableProperties()) + .isEmpty(); + } + } + + @Test + public void compactionReason() { + try (final CompactionJobInfo compactionJobInfo = new CompactionJobInfo()) { + assertThat(compactionJobInfo.compactionReason()) + .isEqualTo(CompactionReason.kUnknown); + } + } + + @Test + public void compression() { + try (final CompactionJobInfo compactionJobInfo = new CompactionJobInfo()) { + assertThat(compactionJobInfo.compression()) + .isEqualTo(CompressionType.NO_COMPRESSION); + } + } + + @Test + public void stats() { + try (final CompactionJobInfo compactionJobInfo = new CompactionJobInfo()) { + assertThat(compactionJobInfo.stats()) + .isNotNull(); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionJobStatsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionJobStatsTest.java new file mode 100644 index 0000000..5c1eb2a --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionJobStatsTest.java @@ -0,0 +1,196 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class CompactionJobStatsTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Test + public void reset() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + compactionJobStats.reset(); + assertThat(compactionJobStats.elapsedMicros()).isEqualTo(0); + } + } + + @Test + public void add() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats(); + final CompactionJobStats otherCompactionJobStats = new CompactionJobStats()) { + compactionJobStats.add(otherCompactionJobStats); + } + } + + @Test + public void elapsedMicros() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.elapsedMicros()).isEqualTo(0); + } + } + + @Test + public void numInputRecords() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.numInputRecords()).isEqualTo(0); + } + } + + @Test + public void numInputFiles() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.numInputFiles()).isEqualTo(0); + } + } + + @Test + public void numInputFilesAtOutputLevel() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.numInputFilesAtOutputLevel()).isEqualTo(0); + } + } + + @Test + public void numOutputRecords() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.numOutputRecords()).isEqualTo(0); + } + } + + @Test + public void numOutputFiles() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.numOutputFiles()).isEqualTo(0); + } + } + + @Test + public void isManualCompaction() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.isManualCompaction()).isFalse(); + } + } + + @Test + public void totalInputBytes() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.totalInputBytes()).isEqualTo(0); + } + } + + @Test + public void totalOutputBytes() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.totalOutputBytes()).isEqualTo(0); + } + } + + + @Test + public void numRecordsReplaced() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.numRecordsReplaced()).isEqualTo(0); + } + } + + @Test + public void totalInputRawKeyBytes() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.totalInputRawKeyBytes()).isEqualTo(0); + } + } + + @Test + public void totalInputRawValueBytes() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.totalInputRawValueBytes()).isEqualTo(0); + } + } + + @Test + public void numInputDeletionRecords() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.numInputDeletionRecords()).isEqualTo(0); + } + } + + @Test + public void numExpiredDeletionRecords() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.numExpiredDeletionRecords()).isEqualTo(0); + } + } + + @Test + public void numCorruptKeys() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.numCorruptKeys()).isEqualTo(0); + } + } + + @Test + public void fileWriteNanos() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.fileWriteNanos()).isEqualTo(0); + } + } + + @Test + public void fileRangeSyncNanos() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.fileRangeSyncNanos()).isEqualTo(0); + } + } + + @Test + public void fileFsyncNanos() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.fileFsyncNanos()).isEqualTo(0); + } + } + + @Test + public void filePrepareWriteNanos() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.filePrepareWriteNanos()).isEqualTo(0); + } + } + + @Test + public void smallestOutputKeyPrefix() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.smallestOutputKeyPrefix()).isEmpty(); + } + } + + @Test + public void largestOutputKeyPrefix() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.largestOutputKeyPrefix()).isEmpty(); + } + } + + @Test + public void numSingleDelFallthru() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.numSingleDelFallthru()).isEqualTo(0); + } + } + + @Test + public void numSingleDelMismatch() { + try (final CompactionJobStats compactionJobStats = new CompactionJobStats()) { + assertThat(compactionJobStats.numSingleDelMismatch()).isEqualTo(0); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionOptionsFIFOTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionOptionsFIFOTest.java new file mode 100644 index 0000000..841615e --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionOptionsFIFOTest.java @@ -0,0 +1,35 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class CompactionOptionsFIFOTest { + + static { + RocksDB.loadLibrary(); + } + + @Test + public void maxTableFilesSize() { + final long size = 500 * 1024 * 1026; + try (final CompactionOptionsFIFO opt = new CompactionOptionsFIFO()) { + opt.setMaxTableFilesSize(size); + assertThat(opt.maxTableFilesSize()).isEqualTo(size); + } + } + + @Test + public void allowCompaction() { + final boolean allowCompaction = true; + try (final CompactionOptionsFIFO opt = new CompactionOptionsFIFO()) { + opt.setAllowCompaction(allowCompaction); + assertThat(opt.allowCompaction()).isEqualTo(allowCompaction); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionOptionsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionOptionsTest.java new file mode 100644 index 0000000..9b7d796 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionOptionsTest.java @@ -0,0 +1,52 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class CompactionOptionsTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Test + public void compression() { + try (final CompactionOptions compactionOptions = new CompactionOptions()) { + assertThat(compactionOptions.compression()) + .isEqualTo(CompressionType.SNAPPY_COMPRESSION); + compactionOptions.setCompression(CompressionType.NO_COMPRESSION); + assertThat(compactionOptions.compression()) + .isEqualTo(CompressionType.NO_COMPRESSION); + } + } + + @Test + public void outputFileSizeLimit() { + final long mb250 = 1024 * 1024 * 250; + try (final CompactionOptions compactionOptions = new CompactionOptions()) { + assertThat(compactionOptions.outputFileSizeLimit()) + .isEqualTo(-1); + compactionOptions.setOutputFileSizeLimit(mb250); + assertThat(compactionOptions.outputFileSizeLimit()) + .isEqualTo(mb250); + } + } + + @Test + public void maxSubcompactions() { + try (final CompactionOptions compactionOptions = new CompactionOptions()) { + assertThat(compactionOptions.maxSubcompactions()) + .isEqualTo(0); + compactionOptions.setMaxSubcompactions(9); + assertThat(compactionOptions.maxSubcompactions()) + .isEqualTo(9); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionOptionsUniversalTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionOptionsUniversalTest.java new file mode 100644 index 0000000..5e2d195 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionOptionsUniversalTest.java @@ -0,0 +1,80 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class CompactionOptionsUniversalTest { + + static { + RocksDB.loadLibrary(); + } + + @Test + public void sizeRatio() { + final int sizeRatio = 4; + try(final CompactionOptionsUniversal opt = new CompactionOptionsUniversal()) { + opt.setSizeRatio(sizeRatio); + assertThat(opt.sizeRatio()).isEqualTo(sizeRatio); + } + } + + @Test + public void minMergeWidth() { + final int minMergeWidth = 3; + try(final CompactionOptionsUniversal opt = new CompactionOptionsUniversal()) { + opt.setMinMergeWidth(minMergeWidth); + assertThat(opt.minMergeWidth()).isEqualTo(minMergeWidth); + } + } + + @Test + public void maxMergeWidth() { + final int maxMergeWidth = Integer.MAX_VALUE - 1234; + try(final CompactionOptionsUniversal opt = new CompactionOptionsUniversal()) { + opt.setMaxMergeWidth(maxMergeWidth); + assertThat(opt.maxMergeWidth()).isEqualTo(maxMergeWidth); + } + } + + @Test + public void maxSizeAmplificationPercent() { + final int maxSizeAmplificationPercent = 150; + try(final CompactionOptionsUniversal opt = new CompactionOptionsUniversal()) { + opt.setMaxSizeAmplificationPercent(maxSizeAmplificationPercent); + assertThat(opt.maxSizeAmplificationPercent()).isEqualTo(maxSizeAmplificationPercent); + } + } + + @Test + public void compressionSizePercent() { + final int compressionSizePercent = 500; + try(final CompactionOptionsUniversal opt = new CompactionOptionsUniversal()) { + opt.setCompressionSizePercent(compressionSizePercent); + assertThat(opt.compressionSizePercent()).isEqualTo(compressionSizePercent); + } + } + + @Test + public void stopStyle() { + final CompactionStopStyle stopStyle = CompactionStopStyle.CompactionStopStyleSimilarSize; + try(final CompactionOptionsUniversal opt = new CompactionOptionsUniversal()) { + opt.setStopStyle(stopStyle); + assertThat(opt.stopStyle()).isEqualTo(stopStyle); + } + } + + @Test + public void allowTrivialMove() { + final boolean allowTrivialMove = true; + try(final CompactionOptionsUniversal opt = new CompactionOptionsUniversal()) { + opt.setAllowTrivialMove(allowTrivialMove); + assertThat(opt.allowTrivialMove()).isEqualTo(allowTrivialMove); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionPriorityTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionPriorityTest.java new file mode 100644 index 0000000..b078e13 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionPriorityTest.java @@ -0,0 +1,31 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class CompactionPriorityTest { + + @Test(expected = IllegalArgumentException.class) + public void failIfIllegalByteValueProvided() { + CompactionPriority.getCompactionPriority((byte) -1); + } + + @Test + public void getCompactionPriority() { + assertThat(CompactionPriority.getCompactionPriority( + CompactionPriority.OldestLargestSeqFirst.getValue())) + .isEqualTo(CompactionPriority.OldestLargestSeqFirst); + } + + @Test + public void valueOf() { + assertThat(CompactionPriority.valueOf("OldestSmallestSeqFirst")). + isEqualTo(CompactionPriority.OldestSmallestSeqFirst); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionStopStyleTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionStopStyleTest.java new file mode 100644 index 0000000..4c8a209 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompactionStopStyleTest.java @@ -0,0 +1,31 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class CompactionStopStyleTest { + + @Test(expected = IllegalArgumentException.class) + public void failIfIllegalByteValueProvided() { + CompactionStopStyle.getCompactionStopStyle((byte) -1); + } + + @Test + public void getCompactionStopStyle() { + assertThat(CompactionStopStyle.getCompactionStopStyle( + CompactionStopStyle.CompactionStopStyleTotalSize.getValue())) + .isEqualTo(CompactionStopStyle.CompactionStopStyleTotalSize); + } + + @Test + public void valueOf() { + assertThat(CompactionStopStyle.valueOf("CompactionStopStyleSimilarSize")). + isEqualTo(CompactionStopStyle.CompactionStopStyleSimilarSize); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ComparatorOptionsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ComparatorOptionsTest.java new file mode 100644 index 0000000..3e90b9f --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ComparatorOptionsTest.java @@ -0,0 +1,58 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ComparatorOptionsTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Test + public void reusedSynchronisationType() { + try(final ComparatorOptions copt = new ComparatorOptions()) { + + copt.setReusedSynchronisationType(ReusedSynchronisationType.MUTEX); + assertThat(copt.reusedSynchronisationType()) + .isEqualTo(ReusedSynchronisationType.MUTEX); + + copt.setReusedSynchronisationType(ReusedSynchronisationType.ADAPTIVE_MUTEX); + assertThat(copt.reusedSynchronisationType()) + .isEqualTo(ReusedSynchronisationType.ADAPTIVE_MUTEX); + + copt.setReusedSynchronisationType(ReusedSynchronisationType.THREAD_LOCAL); + assertThat(copt.reusedSynchronisationType()) + .isEqualTo(ReusedSynchronisationType.THREAD_LOCAL); + } + } + + @Test + public void useDirectBuffer() { + try(final ComparatorOptions copt = new ComparatorOptions()) { + copt.setUseDirectBuffer(true); + assertThat(copt.useDirectBuffer()).isTrue(); + + copt.setUseDirectBuffer(false); + assertThat(copt.useDirectBuffer()).isFalse(); + } + } + + @Test + public void maxReusedBufferSize() { + try(final ComparatorOptions copt = new ComparatorOptions()) { + copt.setMaxReusedBufferSize(12345); + assertThat(copt.maxReusedBufferSize()).isEqualTo(12345); + + copt.setMaxReusedBufferSize(-1); + assertThat(copt.maxReusedBufferSize()).isEqualTo(-1); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompressionOptionsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompressionOptionsTest.java new file mode 100644 index 0000000..116552c --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompressionOptionsTest.java @@ -0,0 +1,71 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class CompressionOptionsTest { + + static { + RocksDB.loadLibrary(); + } + + @Test + public void windowBits() { + final int windowBits = 7; + try(final CompressionOptions opt = new CompressionOptions()) { + opt.setWindowBits(windowBits); + assertThat(opt.windowBits()).isEqualTo(windowBits); + } + } + + @Test + public void level() { + final int level = 6; + try(final CompressionOptions opt = new CompressionOptions()) { + opt.setLevel(level); + assertThat(opt.level()).isEqualTo(level); + } + } + + @Test + public void strategy() { + final int strategy = 2; + try(final CompressionOptions opt = new CompressionOptions()) { + opt.setStrategy(strategy); + assertThat(opt.strategy()).isEqualTo(strategy); + } + } + + @Test + public void maxDictBytes() { + final int maxDictBytes = 999; + try(final CompressionOptions opt = new CompressionOptions()) { + opt.setMaxDictBytes(maxDictBytes); + assertThat(opt.maxDictBytes()).isEqualTo(maxDictBytes); + } + } + + @Test + public void zstdMaxTrainBytes() { + final int zstdMaxTrainBytes = 999; + try(final CompressionOptions opt = new CompressionOptions()) { + opt.setZStdMaxTrainBytes(zstdMaxTrainBytes); + assertThat(opt.zstdMaxTrainBytes()).isEqualTo(zstdMaxTrainBytes); + } + } + + @Test + public void enabled() { + try(final CompressionOptions opt = new CompressionOptions()) { + assertThat(opt.enabled()).isFalse(); + opt.setEnabled(true); + assertThat(opt.enabled()).isTrue(); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompressionTypesTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompressionTypesTest.java new file mode 100644 index 0000000..a983f47 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/CompressionTypesTest.java @@ -0,0 +1,25 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.Test; + +public class CompressionTypesTest { + @Test + public void getCompressionType() { + for (final CompressionType compressionType : CompressionType.values()) { + final String libraryName = compressionType.getLibraryName(); + if (compressionType == CompressionType.DISABLE_COMPRESSION_OPTION) { + assertThat(CompressionType.getCompressionType(libraryName)) + .isEqualTo(CompressionType.NO_COMPRESSION); + } else { + assertThat(CompressionType.getCompressionType(libraryName)).isEqualTo(compressionType); + } + } + } +} \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ConcurrentTaskLimiterTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ConcurrentTaskLimiterTest.java new file mode 100644 index 0000000..165f4f2 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ConcurrentTaskLimiterTest.java @@ -0,0 +1,56 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static org.junit.Assert.assertEquals; + +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; + +public class ConcurrentTaskLimiterTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + private static final String NAME = "name"; + + private ConcurrentTaskLimiter concurrentTaskLimiter; + + @Before + public void beforeTest() { + concurrentTaskLimiter = new ConcurrentTaskLimiterImpl(NAME, 3); + } + + @Test + public void name() { + assertEquals(NAME, concurrentTaskLimiter.name()); + } + + @Test + public void outstandingTask() { + assertEquals(0, concurrentTaskLimiter.outstandingTask()); + } + + @Test + public void setMaxOutstandingTask() { + assertEquals(concurrentTaskLimiter, concurrentTaskLimiter.setMaxOutstandingTask(4)); + assertEquals(0, concurrentTaskLimiter.outstandingTask()); + } + + @Test + public void resetMaxOutstandingTask() { + assertEquals(concurrentTaskLimiter, concurrentTaskLimiter.resetMaxOutstandingTask()); + assertEquals(0, concurrentTaskLimiter.outstandingTask()); + } + + @After + public void afterTest() { + concurrentTaskLimiter.close(); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/DBOptionsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/DBOptionsTest.java new file mode 100644 index 0000000..d79f78d --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/DBOptionsTest.java @@ -0,0 +1,903 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.nio.file.Paths; +import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.ClassRule; +import org.junit.Test; + +public class DBOptionsTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + public static final Random rand = PlatformRandomHelper. + getPlatformSpecificRandomFactory(); + + @Test + public void copyConstructor() { + final DBOptions origOpts = new DBOptions(); + origOpts.setCreateIfMissing(rand.nextBoolean()); + origOpts.setAllow2pc(rand.nextBoolean()); + origOpts.setMaxBackgroundJobs(rand.nextInt(10)); + final DBOptions copyOpts = new DBOptions(origOpts); + assertThat(origOpts.createIfMissing()).isEqualTo(copyOpts.createIfMissing()); + assertThat(origOpts.allow2pc()).isEqualTo(copyOpts.allow2pc()); + } + + @Test + public void getDBOptionsFromProps() { + // setup sample properties + final Properties properties = new Properties(); + properties.put("allow_mmap_reads", "true"); + properties.put("bytes_per_sync", "13"); + try(final DBOptions opt = DBOptions.getDBOptionsFromProps(properties)) { + assertThat(opt).isNotNull(); + assertThat(String.valueOf(opt.allowMmapReads())). + isEqualTo(properties.get("allow_mmap_reads")); + assertThat(String.valueOf(opt.bytesPerSync())). + isEqualTo(properties.get("bytes_per_sync")); + } + } + + @Test + public void failDBOptionsFromPropsWithIllegalValue() { + // setup sample properties + final Properties properties = new Properties(); + properties.put("tomato", "1024"); + properties.put("burger", "2"); + try(final DBOptions opt = DBOptions.getDBOptionsFromProps(properties)) { + assertThat(opt).isNull(); + } + } + + @Test(expected = IllegalArgumentException.class) + public void failDBOptionsFromPropsWithNullValue() { + try(final DBOptions opt = DBOptions.getDBOptionsFromProps(null)) { + //no-op + } + } + + @Test(expected = IllegalArgumentException.class) + public void failDBOptionsFromPropsWithEmptyProps() { + try(final DBOptions opt = DBOptions.getDBOptionsFromProps( + new Properties())) { + //no-op + } + } + + @Test + public void linkageOfPrepMethods() { + try (final DBOptions opt = new DBOptions()) { + opt.optimizeForSmallDb(); + } + } + + @Test + public void env() { + try (final DBOptions opt = new DBOptions(); + final Env env = Env.getDefault()) { + opt.setEnv(env); + assertThat(opt.getEnv()).isSameAs(env); + } + } + + @Test + public void setIncreaseParallelism() { + try(final DBOptions opt = new DBOptions()) { + final int threads = Runtime.getRuntime().availableProcessors() * 2; + opt.setIncreaseParallelism(threads); + } + } + + @Test + public void createIfMissing() { + try(final DBOptions opt = new DBOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setCreateIfMissing(boolValue); + assertThat(opt.createIfMissing()).isEqualTo(boolValue); + } + } + + @Test + public void createMissingColumnFamilies() { + try(final DBOptions opt = new DBOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setCreateMissingColumnFamilies(boolValue); + assertThat(opt.createMissingColumnFamilies()).isEqualTo(boolValue); + } + } + + @Test + public void errorIfExists() { + try(final DBOptions opt = new DBOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setErrorIfExists(boolValue); + assertThat(opt.errorIfExists()).isEqualTo(boolValue); + } + } + + @Test + public void paranoidChecks() { + try(final DBOptions opt = new DBOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setParanoidChecks(boolValue); + assertThat(opt.paranoidChecks()).isEqualTo(boolValue); + } + } + + @Test + public void maxTotalWalSize() { + try(final DBOptions opt = new DBOptions()) { + final long longValue = rand.nextLong(); + opt.setMaxTotalWalSize(longValue); + assertThat(opt.maxTotalWalSize()).isEqualTo(longValue); + } + } + + @Test + public void maxOpenFiles() { + try(final DBOptions opt = new DBOptions()) { + final int intValue = rand.nextInt(); + opt.setMaxOpenFiles(intValue); + assertThat(opt.maxOpenFiles()).isEqualTo(intValue); + } + } + + @Test + public void maxFileOpeningThreads() { + try(final DBOptions opt = new DBOptions()) { + final int intValue = rand.nextInt(); + opt.setMaxFileOpeningThreads(intValue); + assertThat(opt.maxFileOpeningThreads()).isEqualTo(intValue); + } + } + + @Test + public void useFsync() { + try(final DBOptions opt = new DBOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setUseFsync(boolValue); + assertThat(opt.useFsync()).isEqualTo(boolValue); + } + } + + @Test + public void dbPaths() { + final List dbPaths = new ArrayList<>(); + dbPaths.add(new DbPath(Paths.get("/a"), 10)); + dbPaths.add(new DbPath(Paths.get("/b"), 100)); + dbPaths.add(new DbPath(Paths.get("/c"), 1000)); + + try(final DBOptions opt = new DBOptions()) { + assertThat(opt.dbPaths()).isEqualTo(Collections.emptyList()); + + opt.setDbPaths(dbPaths); + + assertThat(opt.dbPaths()).isEqualTo(dbPaths); + } + } + + @Test + public void dbLogDir() { + try(final DBOptions opt = new DBOptions()) { + final String str = "path/to/DbLogDir"; + opt.setDbLogDir(str); + assertThat(opt.dbLogDir()).isEqualTo(str); + } + } + + @Test + public void walDir() { + try(final DBOptions opt = new DBOptions()) { + final String str = "path/to/WalDir"; + opt.setWalDir(str); + assertThat(opt.walDir()).isEqualTo(str); + } + } + + @Test + public void deleteObsoleteFilesPeriodMicros() { + try(final DBOptions opt = new DBOptions()) { + final long longValue = rand.nextLong(); + opt.setDeleteObsoleteFilesPeriodMicros(longValue); + assertThat(opt.deleteObsoleteFilesPeriodMicros()).isEqualTo(longValue); + } + } + + @SuppressWarnings("deprecated") + @Test + public void maxBackgroundCompactions() { + try(final DBOptions opt = new DBOptions()) { + final int intValue = rand.nextInt(); + opt.setMaxBackgroundCompactions(intValue); + assertThat(opt.maxBackgroundCompactions()).isEqualTo(intValue); + } + } + + @Test + public void maxSubcompactions() { + try (final DBOptions opt = new DBOptions()) { + final int intValue = rand.nextInt(); + opt.setMaxSubcompactions(intValue); + assertThat(opt.maxSubcompactions()). + isEqualTo(intValue); + } + } + + @SuppressWarnings("deprecated") + @Test + public void maxBackgroundFlushes() { + try(final DBOptions opt = new DBOptions()) { + final int intValue = rand.nextInt(); + opt.setMaxBackgroundFlushes(intValue); + assertThat(opt.maxBackgroundFlushes()).isEqualTo(intValue); + } + } + + @Test + public void maxBackgroundJobs() { + try (final DBOptions opt = new DBOptions()) { + final int intValue = rand.nextInt(); + opt.setMaxBackgroundJobs(intValue); + assertThat(opt.maxBackgroundJobs()).isEqualTo(intValue); + } + } + + @Test + public void maxLogFileSize() throws RocksDBException { + try(final DBOptions opt = new DBOptions()) { + final long longValue = rand.nextLong(); + opt.setMaxLogFileSize(longValue); + assertThat(opt.maxLogFileSize()).isEqualTo(longValue); + } + } + + @Test + public void logFileTimeToRoll() throws RocksDBException { + try(final DBOptions opt = new DBOptions()) { + final long longValue = rand.nextLong(); + opt.setLogFileTimeToRoll(longValue); + assertThat(opt.logFileTimeToRoll()).isEqualTo(longValue); + } + } + + @Test + public void keepLogFileNum() throws RocksDBException { + try(final DBOptions opt = new DBOptions()) { + final long longValue = rand.nextLong(); + opt.setKeepLogFileNum(longValue); + assertThat(opt.keepLogFileNum()).isEqualTo(longValue); + } + } + + @Test + public void recycleLogFileNum() throws RocksDBException { + try(final DBOptions opt = new DBOptions()) { + final long longValue = rand.nextLong(); + opt.setRecycleLogFileNum(longValue); + assertThat(opt.recycleLogFileNum()).isEqualTo(longValue); + } + } + + @Test + public void maxManifestFileSize() { + try(final DBOptions opt = new DBOptions()) { + final long longValue = rand.nextLong(); + opt.setMaxManifestFileSize(longValue); + assertThat(opt.maxManifestFileSize()).isEqualTo(longValue); + } + } + + @Test + public void tableCacheNumshardbits() { + try(final DBOptions opt = new DBOptions()) { + final int intValue = rand.nextInt(); + opt.setTableCacheNumshardbits(intValue); + assertThat(opt.tableCacheNumshardbits()).isEqualTo(intValue); + } + } + + @Test + public void walSizeLimitMB() { + try(final DBOptions opt = new DBOptions()) { + final long longValue = rand.nextLong(); + opt.setWalSizeLimitMB(longValue); + assertThat(opt.walSizeLimitMB()).isEqualTo(longValue); + } + } + + @Test + public void walTtlSeconds() { + try(final DBOptions opt = new DBOptions()) { + final long longValue = rand.nextLong(); + opt.setWalTtlSeconds(longValue); + assertThat(opt.walTtlSeconds()).isEqualTo(longValue); + } + } + + @Test + public void manifestPreallocationSize() throws RocksDBException { + try(final DBOptions opt = new DBOptions()) { + final long longValue = rand.nextLong(); + opt.setManifestPreallocationSize(longValue); + assertThat(opt.manifestPreallocationSize()).isEqualTo(longValue); + } + } + + @Test + public void useDirectReads() { + try(final DBOptions opt = new DBOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setUseDirectReads(boolValue); + assertThat(opt.useDirectReads()).isEqualTo(boolValue); + } + } + + @Test + public void useDirectIoForFlushAndCompaction() { + try(final DBOptions opt = new DBOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setUseDirectIoForFlushAndCompaction(boolValue); + assertThat(opt.useDirectIoForFlushAndCompaction()).isEqualTo(boolValue); + } + } + + @Test + public void allowFAllocate() { + try(final DBOptions opt = new DBOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setAllowFAllocate(boolValue); + assertThat(opt.allowFAllocate()).isEqualTo(boolValue); + } + } + + @Test + public void allowMmapReads() { + try(final DBOptions opt = new DBOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setAllowMmapReads(boolValue); + assertThat(opt.allowMmapReads()).isEqualTo(boolValue); + } + } + + @Test + public void allowMmapWrites() { + try(final DBOptions opt = new DBOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setAllowMmapWrites(boolValue); + assertThat(opt.allowMmapWrites()).isEqualTo(boolValue); + } + } + + @Test + public void isFdCloseOnExec() { + try(final DBOptions opt = new DBOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setIsFdCloseOnExec(boolValue); + assertThat(opt.isFdCloseOnExec()).isEqualTo(boolValue); + } + } + + @Test + public void statsDumpPeriodSec() { + try(final DBOptions opt = new DBOptions()) { + final int intValue = rand.nextInt(); + opt.setStatsDumpPeriodSec(intValue); + assertThat(opt.statsDumpPeriodSec()).isEqualTo(intValue); + } + } + + @Test + public void statsPersistPeriodSec() { + try (final DBOptions opt = new DBOptions()) { + final int intValue = rand.nextInt(); + opt.setStatsPersistPeriodSec(intValue); + assertThat(opt.statsPersistPeriodSec()).isEqualTo(intValue); + } + } + + @Test + public void statsHistoryBufferSize() { + try (final DBOptions opt = new DBOptions()) { + final long longValue = rand.nextLong(); + opt.setStatsHistoryBufferSize(longValue); + assertThat(opt.statsHistoryBufferSize()).isEqualTo(longValue); + } + } + + @Test + public void adviseRandomOnOpen() { + try(final DBOptions opt = new DBOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setAdviseRandomOnOpen(boolValue); + assertThat(opt.adviseRandomOnOpen()).isEqualTo(boolValue); + } + } + + @Test + public void dbWriteBufferSize() { + try(final DBOptions opt = new DBOptions()) { + final long longValue = rand.nextLong(); + opt.setDbWriteBufferSize(longValue); + assertThat(opt.dbWriteBufferSize()).isEqualTo(longValue); + } + } + + @Test + public void setWriteBufferManager() throws RocksDBException { + try (final DBOptions opt = new DBOptions(); final Cache cache = new LRUCache(1024 * 1024); + final WriteBufferManager writeBufferManager = new WriteBufferManager(2000L, cache)) { + opt.setWriteBufferManager(writeBufferManager); + assertThat(opt.writeBufferManager()).isEqualTo(writeBufferManager); + } + } + + @Test + public void setWriteBufferManagerWithZeroBufferSize() throws RocksDBException { + try (final DBOptions opt = new DBOptions(); final Cache cache = new LRUCache(1024 * 1024); + final WriteBufferManager writeBufferManager = new WriteBufferManager(0L, cache)) { + opt.setWriteBufferManager(writeBufferManager); + assertThat(opt.writeBufferManager()).isEqualTo(writeBufferManager); + } + } + + @SuppressWarnings("deprecated") + @Test + public void accessHintOnCompactionStart() { + try(final DBOptions opt = new DBOptions()) { + final AccessHint accessHint = AccessHint.SEQUENTIAL; + opt.setAccessHintOnCompactionStart(accessHint); + assertThat(opt.accessHintOnCompactionStart()).isEqualTo(accessHint); + } + } + + @Test + public void compactionReadaheadSize() { + try(final DBOptions opt = new DBOptions()) { + final long longValue = rand.nextLong(); + opt.setCompactionReadaheadSize(longValue); + assertThat(opt.compactionReadaheadSize()).isEqualTo(longValue); + } + } + + @Test + public void randomAccessMaxBufferSize() { + try(final DBOptions opt = new DBOptions()) { + final long longValue = rand.nextLong(); + opt.setRandomAccessMaxBufferSize(longValue); + assertThat(opt.randomAccessMaxBufferSize()).isEqualTo(longValue); + } + } + + @Test + public void writableFileMaxBufferSize() { + try(final DBOptions opt = new DBOptions()) { + final long longValue = rand.nextLong(); + opt.setWritableFileMaxBufferSize(longValue); + assertThat(opt.writableFileMaxBufferSize()).isEqualTo(longValue); + } + } + + @Test + public void useAdaptiveMutex() { + try(final DBOptions opt = new DBOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setUseAdaptiveMutex(boolValue); + assertThat(opt.useAdaptiveMutex()).isEqualTo(boolValue); + } + } + + @Test + public void bytesPerSync() { + try(final DBOptions opt = new DBOptions()) { + final long longValue = rand.nextLong(); + opt.setBytesPerSync(longValue); + assertThat(opt.bytesPerSync()).isEqualTo(longValue); + } + } + + @Test + public void walBytesPerSync() { + try(final DBOptions opt = new DBOptions()) { + final long longValue = rand.nextLong(); + opt.setWalBytesPerSync(longValue); + assertThat(opt.walBytesPerSync()).isEqualTo(longValue); + } + } + + @Test + public void strictBytesPerSync() { + try (final DBOptions opt = new DBOptions()) { + assertThat(opt.strictBytesPerSync()).isFalse(); + opt.setStrictBytesPerSync(true); + assertThat(opt.strictBytesPerSync()).isTrue(); + } + } + + @Test + public void enableThreadTracking() { + try (final DBOptions opt = new DBOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setEnableThreadTracking(boolValue); + assertThat(opt.enableThreadTracking()).isEqualTo(boolValue); + } + } + + @Test + public void delayedWriteRate() { + try(final DBOptions opt = new DBOptions()) { + final long longValue = rand.nextLong(); + opt.setDelayedWriteRate(longValue); + assertThat(opt.delayedWriteRate()).isEqualTo(longValue); + } + } + + @Test + public void enablePipelinedWrite() { + try(final DBOptions opt = new DBOptions()) { + assertThat(opt.enablePipelinedWrite()).isFalse(); + opt.setEnablePipelinedWrite(true); + assertThat(opt.enablePipelinedWrite()).isTrue(); + } + } + + @Test + public void unordredWrite() { + try(final DBOptions opt = new DBOptions()) { + assertThat(opt.unorderedWrite()).isFalse(); + opt.setUnorderedWrite(true); + assertThat(opt.unorderedWrite()).isTrue(); + } + } + + @Test + public void allowConcurrentMemtableWrite() { + try (final DBOptions opt = new DBOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setAllowConcurrentMemtableWrite(boolValue); + assertThat(opt.allowConcurrentMemtableWrite()).isEqualTo(boolValue); + } + } + + @Test + public void enableWriteThreadAdaptiveYield() { + try (final DBOptions opt = new DBOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setEnableWriteThreadAdaptiveYield(boolValue); + assertThat(opt.enableWriteThreadAdaptiveYield()).isEqualTo(boolValue); + } + } + + @Test + public void writeThreadMaxYieldUsec() { + try (final DBOptions opt = new DBOptions()) { + final long longValue = rand.nextLong(); + opt.setWriteThreadMaxYieldUsec(longValue); + assertThat(opt.writeThreadMaxYieldUsec()).isEqualTo(longValue); + } + } + + @Test + public void writeThreadSlowYieldUsec() { + try (final DBOptions opt = new DBOptions()) { + final long longValue = rand.nextLong(); + opt.setWriteThreadSlowYieldUsec(longValue); + assertThat(opt.writeThreadSlowYieldUsec()).isEqualTo(longValue); + } + } + + @Test + public void skipStatsUpdateOnDbOpen() { + try (final DBOptions opt = new DBOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setSkipStatsUpdateOnDbOpen(boolValue); + assertThat(opt.skipStatsUpdateOnDbOpen()).isEqualTo(boolValue); + } + } + + @Test + public void walRecoveryMode() { + try (final DBOptions opt = new DBOptions()) { + for (final WALRecoveryMode walRecoveryMode : WALRecoveryMode.values()) { + opt.setWalRecoveryMode(walRecoveryMode); + assertThat(opt.walRecoveryMode()).isEqualTo(walRecoveryMode); + } + } + } + + @Test + public void allow2pc() { + try (final DBOptions opt = new DBOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setAllow2pc(boolValue); + assertThat(opt.allow2pc()).isEqualTo(boolValue); + } + } + + @Test + public void rowCache() { + try (final DBOptions opt = new DBOptions()) { + assertThat(opt.rowCache()).isNull(); + + try(final Cache lruCache = new LRUCache(1000)) { + opt.setRowCache(lruCache); + assertThat(opt.rowCache()).isEqualTo(lruCache); + } + + try(final Cache clockCache = new ClockCache(1000)) { + opt.setRowCache(clockCache); + assertThat(opt.rowCache()).isEqualTo(clockCache); + } + } + } + + @Test + public void walFilter() { + try (final DBOptions opt = new DBOptions()) { + assertThat(opt.walFilter()).isNull(); + + try (final AbstractWalFilter walFilter = new AbstractWalFilter() { + @Override + public void columnFamilyLogNumberMap( + final Map cfLognumber, + final Map cfNameId) { + // no-op + } + + @Override + public LogRecordFoundResult logRecordFound(final long logNumber, + final String logFileName, final WriteBatch batch, + final WriteBatch newBatch) { + return new LogRecordFoundResult( + WalProcessingOption.CONTINUE_PROCESSING, false); + } + + @Override + public String name() { + return "test-wal-filter"; + } + }) { + opt.setWalFilter(walFilter); + assertThat(opt.walFilter()).isEqualTo(walFilter); + } + } + } + + @Test + public void failIfOptionsFileError() { + try (final DBOptions opt = new DBOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setFailIfOptionsFileError(boolValue); + assertThat(opt.failIfOptionsFileError()).isEqualTo(boolValue); + } + } + + @Test + public void dumpMallocStats() { + try (final DBOptions opt = new DBOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setDumpMallocStats(boolValue); + assertThat(opt.dumpMallocStats()).isEqualTo(boolValue); + } + } + + @Test + public void avoidFlushDuringRecovery() { + try (final DBOptions opt = new DBOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setAvoidFlushDuringRecovery(boolValue); + assertThat(opt.avoidFlushDuringRecovery()).isEqualTo(boolValue); + } + } + + @Test + public void avoidFlushDuringShutdown() { + try (final DBOptions opt = new DBOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setAvoidFlushDuringShutdown(boolValue); + assertThat(opt.avoidFlushDuringShutdown()).isEqualTo(boolValue); + } + } + + @Test + public void allowIngestBehind() { + try (final DBOptions opt = new DBOptions()) { + assertThat(opt.allowIngestBehind()).isFalse(); + opt.setAllowIngestBehind(true); + assertThat(opt.allowIngestBehind()).isTrue(); + } + } + + @Test + public void twoWriteQueues() { + try (final DBOptions opt = new DBOptions()) { + assertThat(opt.twoWriteQueues()).isFalse(); + opt.setTwoWriteQueues(true); + assertThat(opt.twoWriteQueues()).isTrue(); + } + } + + @Test + public void manualWalFlush() { + try (final DBOptions opt = new DBOptions()) { + assertThat(opt.manualWalFlush()).isFalse(); + opt.setManualWalFlush(true); + assertThat(opt.manualWalFlush()).isTrue(); + } + } + + @Test + public void atomicFlush() { + try (final DBOptions opt = new DBOptions()) { + assertThat(opt.atomicFlush()).isFalse(); + opt.setAtomicFlush(true); + assertThat(opt.atomicFlush()).isTrue(); + } + } + + @Test + public void rateLimiter() { + try(final DBOptions options = new DBOptions(); + final DBOptions anotherOptions = new DBOptions(); + final RateLimiter rateLimiter = new RateLimiter(1000, 100 * 1000, 1)) { + options.setRateLimiter(rateLimiter); + // Test with parameter initialization + anotherOptions.setRateLimiter( + new RateLimiter(1000)); + } + } + + @Test + public void sstFileManager() throws RocksDBException { + try (final DBOptions options = new DBOptions(); + final SstFileManager sstFileManager = + new SstFileManager(Env.getDefault())) { + options.setSstFileManager(sstFileManager); + } + } + + @Test + public void statistics() { + try(final DBOptions options = new DBOptions()) { + final Statistics statistics = options.statistics(); + assertThat(statistics).isNull(); + } + + try(final Statistics statistics = new Statistics(); + final DBOptions options = new DBOptions().setStatistics(statistics); + final Statistics stats = options.statistics()) { + assertThat(stats).isNotNull(); + } + } + + @Test + public void avoidUnnecessaryBlockingIO() { + try (final DBOptions options = new DBOptions()) { + assertThat(options.avoidUnnecessaryBlockingIO()).isEqualTo(false); + assertThat(options.setAvoidUnnecessaryBlockingIO(true)).isEqualTo(options); + assertThat(options.avoidUnnecessaryBlockingIO()).isEqualTo(true); + } + } + + @Test + public void persistStatsToDisk() { + try (final DBOptions options = new DBOptions()) { + assertThat(options.persistStatsToDisk()).isEqualTo(false); + assertThat(options.setPersistStatsToDisk(true)).isEqualTo(options); + assertThat(options.persistStatsToDisk()).isEqualTo(true); + } + } + + @Test + public void writeDbidToManifest() { + try (final DBOptions options = new DBOptions()) { + assertThat(options.writeDbidToManifest()).isEqualTo(false); + assertThat(options.setWriteDbidToManifest(true)).isEqualTo(options); + assertThat(options.writeDbidToManifest()).isEqualTo(true); + } + } + + @Test + public void logReadaheadSize() { + try (final DBOptions options = new DBOptions()) { + assertThat(options.logReadaheadSize()).isEqualTo(0); + final int size = 1024 * 1024 * 100; + assertThat(options.setLogReadaheadSize(size)).isEqualTo(options); + assertThat(options.logReadaheadSize()).isEqualTo(size); + } + } + + @Test + public void bestEffortsRecovery() { + try (final DBOptions options = new DBOptions()) { + assertThat(options.bestEffortsRecovery()).isEqualTo(false); + assertThat(options.setBestEffortsRecovery(true)).isEqualTo(options); + assertThat(options.bestEffortsRecovery()).isEqualTo(true); + } + } + + @Test + public void maxBgerrorResumeCount() { + try (final DBOptions options = new DBOptions()) { + final int INT_MAX = 2147483647; + assertThat(options.maxBgerrorResumeCount()).isEqualTo(INT_MAX); + assertThat(options.setMaxBgErrorResumeCount(-1)).isEqualTo(options); + assertThat(options.maxBgerrorResumeCount()).isEqualTo(-1); + } + } + + @Test + public void bgerrorResumeRetryInterval() { + try (final DBOptions options = new DBOptions()) { + assertThat(options.bgerrorResumeRetryInterval()).isEqualTo(1000000); + final long newRetryInterval = 24 * 3600 * 1000000L; + assertThat(options.setBgerrorResumeRetryInterval(newRetryInterval)).isEqualTo(options); + assertThat(options.bgerrorResumeRetryInterval()).isEqualTo(newRetryInterval); + } + } + + @Test + public void maxWriteBatchGroupSizeBytes() { + try (final DBOptions options = new DBOptions()) { + assertThat(options.maxWriteBatchGroupSizeBytes()).isEqualTo(1024 * 1024); + final long size = 1024 * 1024 * 1024 * 10L; + assertThat(options.setMaxWriteBatchGroupSizeBytes(size)).isEqualTo(options); + assertThat(options.maxWriteBatchGroupSizeBytes()).isEqualTo(size); + } + } + + @Test + public void skipCheckingSstFileSizesOnDbOpen() { + try (final DBOptions options = new DBOptions()) { + assertThat(options.skipCheckingSstFileSizesOnDbOpen()).isEqualTo(false); + assertThat(options.setSkipCheckingSstFileSizesOnDbOpen(true)).isEqualTo(options); + assertThat(options.skipCheckingSstFileSizesOnDbOpen()).isEqualTo(true); + } + } + + @Test + public void eventListeners() { + final AtomicBoolean wasCalled1 = new AtomicBoolean(); + final AtomicBoolean wasCalled2 = new AtomicBoolean(); + try (final DBOptions options = new DBOptions(); + final AbstractEventListener el1 = + new AbstractEventListener() { + @Override + public void onTableFileDeleted(final TableFileDeletionInfo tableFileDeletionInfo) { + wasCalled1.set(true); + } + }; + final AbstractEventListener el2 = + new AbstractEventListener() { + @Override + public void onMemTableSealed(final MemTableInfo memTableInfo) { + wasCalled2.set(true); + } + }) { + assertThat(options.setListeners(Arrays.asList(el1, el2))).isEqualTo(options); + final List listeners = options.listeners(); + assertEquals(el1, listeners.get(0)); + assertEquals(el2, listeners.get(1)); + options.setListeners(Collections.emptyList()); + listeners.get(0).onTableFileDeleted(null); + assertTrue(wasCalled1.get()); + listeners.get(1).onMemTableSealed(null); + assertTrue(wasCalled2.get()); + final List listeners2 = options.listeners(); + assertNotNull(listeners2); + assertEquals(0, listeners2.size()); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/DefaultEnvTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/DefaultEnvTest.java new file mode 100644 index 0000000..3fb563e --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/DefaultEnvTest.java @@ -0,0 +1,113 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.util.Collection; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +public class DefaultEnvTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void backgroundThreads() { + try (final Env defaultEnv = RocksEnv.getDefault()) { + defaultEnv.setBackgroundThreads(5, Priority.BOTTOM); + assertThat(defaultEnv.getBackgroundThreads(Priority.BOTTOM)).isEqualTo(5); + + defaultEnv.setBackgroundThreads(5); + assertThat(defaultEnv.getBackgroundThreads(Priority.LOW)).isEqualTo(5); + + defaultEnv.setBackgroundThreads(5, Priority.LOW); + assertThat(defaultEnv.getBackgroundThreads(Priority.LOW)).isEqualTo(5); + + defaultEnv.setBackgroundThreads(5, Priority.HIGH); + assertThat(defaultEnv.getBackgroundThreads(Priority.HIGH)).isEqualTo(5); + } + } + + @Test + public void threadPoolQueueLen() { + try (final Env defaultEnv = RocksEnv.getDefault()) { + assertThat(defaultEnv.getThreadPoolQueueLen(Priority.BOTTOM)).isEqualTo(0); + assertThat(defaultEnv.getThreadPoolQueueLen(Priority.LOW)).isEqualTo(0); + assertThat(defaultEnv.getThreadPoolQueueLen(Priority.HIGH)).isEqualTo(0); + } + } + + @Test + public void incBackgroundThreadsIfNeeded() { + try (final Env defaultEnv = RocksEnv.getDefault()) { + defaultEnv.incBackgroundThreadsIfNeeded(20, Priority.BOTTOM); + assertThat(defaultEnv.getBackgroundThreads(Priority.BOTTOM)).isGreaterThanOrEqualTo(20); + + defaultEnv.incBackgroundThreadsIfNeeded(20, Priority.LOW); + assertThat(defaultEnv.getBackgroundThreads(Priority.LOW)).isGreaterThanOrEqualTo(20); + + defaultEnv.incBackgroundThreadsIfNeeded(20, Priority.HIGH); + assertThat(defaultEnv.getBackgroundThreads(Priority.HIGH)).isGreaterThanOrEqualTo(20); + } + } + + @Test + public void lowerThreadPoolIOPriority() { + try (final Env defaultEnv = RocksEnv.getDefault()) { + defaultEnv.lowerThreadPoolIOPriority(Priority.BOTTOM); + + defaultEnv.lowerThreadPoolIOPriority(Priority.LOW); + + defaultEnv.lowerThreadPoolIOPriority(Priority.HIGH); + } + } + + @Test + public void lowerThreadPoolCPUPriority() { + try (final Env defaultEnv = RocksEnv.getDefault()) { + defaultEnv.lowerThreadPoolCPUPriority(Priority.BOTTOM); + + defaultEnv.lowerThreadPoolCPUPriority(Priority.LOW); + + defaultEnv.lowerThreadPoolCPUPriority(Priority.HIGH); + } + } + + @Test + public void threadList() throws RocksDBException { + try (final Env defaultEnv = RocksEnv.getDefault()) { + final Collection threadList = defaultEnv.getThreadList(); + assertThat(threadList.size()).isGreaterThan(0); + } + } + + @Test + public void threadList_integration() throws RocksDBException { + try (final Env env = RocksEnv.getDefault(); + final Options opt = new Options() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true) + .setEnv(env)) { + // open database + try (final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + + final List threadList = env.getThreadList(); + assertThat(threadList.size()).isGreaterThan(0); + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/DirectSliceTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/DirectSliceTest.java new file mode 100644 index 0000000..6738534 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/DirectSliceTest.java @@ -0,0 +1,93 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Test; + +import java.nio.ByteBuffer; + +import static org.assertj.core.api.Assertions.assertThat; + +public class DirectSliceTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Test + public void directSlice() { + try(final DirectSlice directSlice = new DirectSlice("abc"); + final DirectSlice otherSlice = new DirectSlice("abc")) { + assertThat(directSlice.toString()).isEqualTo("abc"); + // clear first slice + directSlice.clear(); + assertThat(directSlice.toString()).isEmpty(); + // get first char in otherslice + assertThat(otherSlice.get(0)).isEqualTo("a".getBytes()[0]); + // remove prefix + otherSlice.removePrefix(1); + assertThat(otherSlice.toString()).isEqualTo("bc"); + } + } + + @Test + public void directSliceWithByteBuffer() { + final byte[] data = "Some text".getBytes(); + final ByteBuffer buffer = ByteBuffer.allocateDirect(data.length + 1); + buffer.put(data); + buffer.put(data.length, (byte)0); + + try(final DirectSlice directSlice = new DirectSlice(buffer)) { + assertThat(directSlice.toString()).isEqualTo("Some text"); + } + } + + @Test + public void directSliceWithByteBufferAndLength() { + final byte[] data = "Some text".getBytes(); + final ByteBuffer buffer = ByteBuffer.allocateDirect(data.length); + buffer.put(data); + try(final DirectSlice directSlice = new DirectSlice(buffer, 4)) { + assertThat(directSlice.toString()).isEqualTo("Some"); + } + } + + @Test(expected = IllegalArgumentException.class) + public void directSliceInitWithoutDirectAllocation() { + final byte[] data = "Some text".getBytes(); + final ByteBuffer buffer = ByteBuffer.wrap(data); + try(final DirectSlice directSlice = new DirectSlice(buffer)) { + //no-op + } + } + + @Test(expected = IllegalArgumentException.class) + public void directSlicePrefixInitWithoutDirectAllocation() { + final byte[] data = "Some text".getBytes(); + final ByteBuffer buffer = ByteBuffer.wrap(data); + try(final DirectSlice directSlice = new DirectSlice(buffer, 4)) { + //no-op + } + } + + @Test + public void directSliceClear() { + try(final DirectSlice directSlice = new DirectSlice("abc")) { + assertThat(directSlice.toString()).isEqualTo("abc"); + directSlice.clear(); + assertThat(directSlice.toString()).isEmpty(); + directSlice.clear(); // make sure we don't double-free + } + } + + @Test + public void directSliceRemovePrefix() { + try(final DirectSlice directSlice = new DirectSlice("abc")) { + assertThat(directSlice.toString()).isEqualTo("abc"); + directSlice.removePrefix(1); + assertThat(directSlice.toString()).isEqualTo("bc"); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/EnvOptionsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/EnvOptionsTest.java new file mode 100644 index 0000000..0f3d8e2 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/EnvOptionsTest.java @@ -0,0 +1,145 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Test; + +import java.util.Random; + +import static org.assertj.core.api.Assertions.assertThat; + +public class EnvOptionsTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = new RocksNativeLibraryResource(); + + public static final Random rand = PlatformRandomHelper.getPlatformSpecificRandomFactory(); + + @Test + public void dbOptionsConstructor() { + final long compactionReadaheadSize = 4 * 1024 * 1024; + try (final DBOptions dbOptions = new DBOptions() + .setCompactionReadaheadSize(compactionReadaheadSize)) { + try (final EnvOptions envOptions = new EnvOptions(dbOptions)) { + assertThat(envOptions.compactionReadaheadSize()) + .isEqualTo(compactionReadaheadSize); + } + } + } + + @Test + public void useMmapReads() { + try (final EnvOptions envOptions = new EnvOptions()) { + final boolean boolValue = rand.nextBoolean(); + envOptions.setUseMmapReads(boolValue); + assertThat(envOptions.useMmapReads()).isEqualTo(boolValue); + } + } + + @Test + public void useMmapWrites() { + try (final EnvOptions envOptions = new EnvOptions()) { + final boolean boolValue = rand.nextBoolean(); + envOptions.setUseMmapWrites(boolValue); + assertThat(envOptions.useMmapWrites()).isEqualTo(boolValue); + } + } + + @Test + public void useDirectReads() { + try (final EnvOptions envOptions = new EnvOptions()) { + final boolean boolValue = rand.nextBoolean(); + envOptions.setUseDirectReads(boolValue); + assertThat(envOptions.useDirectReads()).isEqualTo(boolValue); + } + } + + @Test + public void useDirectWrites() { + try (final EnvOptions envOptions = new EnvOptions()) { + final boolean boolValue = rand.nextBoolean(); + envOptions.setUseDirectWrites(boolValue); + assertThat(envOptions.useDirectWrites()).isEqualTo(boolValue); + } + } + + @Test + public void allowFallocate() { + try (final EnvOptions envOptions = new EnvOptions()) { + final boolean boolValue = rand.nextBoolean(); + envOptions.setAllowFallocate(boolValue); + assertThat(envOptions.allowFallocate()).isEqualTo(boolValue); + } + } + + @Test + public void setFdCloexecs() { + try (final EnvOptions envOptions = new EnvOptions()) { + final boolean boolValue = rand.nextBoolean(); + envOptions.setSetFdCloexec(boolValue); + assertThat(envOptions.setFdCloexec()).isEqualTo(boolValue); + } + } + + @Test + public void bytesPerSync() { + try (final EnvOptions envOptions = new EnvOptions()) { + final long longValue = rand.nextLong(); + envOptions.setBytesPerSync(longValue); + assertThat(envOptions.bytesPerSync()).isEqualTo(longValue); + } + } + + @Test + public void fallocateWithKeepSize() { + try (final EnvOptions envOptions = new EnvOptions()) { + final boolean boolValue = rand.nextBoolean(); + envOptions.setFallocateWithKeepSize(boolValue); + assertThat(envOptions.fallocateWithKeepSize()).isEqualTo(boolValue); + } + } + + @Test + public void compactionReadaheadSize() { + try (final EnvOptions envOptions = new EnvOptions()) { + final int intValue = rand.nextInt(2147483647); + envOptions.setCompactionReadaheadSize(intValue); + assertThat(envOptions.compactionReadaheadSize()).isEqualTo(intValue); + } + } + + @Test + public void randomAccessMaxBufferSize() { + try (final EnvOptions envOptions = new EnvOptions()) { + final int intValue = rand.nextInt(2147483647); + envOptions.setRandomAccessMaxBufferSize(intValue); + assertThat(envOptions.randomAccessMaxBufferSize()).isEqualTo(intValue); + } + } + + @Test + public void writableFileMaxBufferSize() { + try (final EnvOptions envOptions = new EnvOptions()) { + final int intValue = rand.nextInt(2147483647); + envOptions.setWritableFileMaxBufferSize(intValue); + assertThat(envOptions.writableFileMaxBufferSize()).isEqualTo(intValue); + } + } + + @Test + public void rateLimiter() { + try (final EnvOptions envOptions = new EnvOptions(); + final RateLimiter rateLimiter1 = new RateLimiter(1000, 100 * 1000, 1)) { + envOptions.setRateLimiter(rateLimiter1); + assertThat(envOptions.rateLimiter()).isEqualTo(rateLimiter1); + + try(final RateLimiter rateLimiter2 = new RateLimiter(1000)) { + envOptions.setRateLimiter(rateLimiter2); + assertThat(envOptions.rateLimiter()).isEqualTo(rateLimiter2); + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/EventListenerTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/EventListenerTest.java new file mode 100644 index 0000000..84be232 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/EventListenerTest.java @@ -0,0 +1,725 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; +import org.assertj.core.api.AbstractObjectAssert; +import org.assertj.core.api.ObjectAssert; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.AbstractEventListener.EnabledEventCallback; +import org.rocksdb.test.TestableEventListener; + +public class EventListenerTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + public static final Random rand = PlatformRandomHelper.getPlatformSpecificRandomFactory(); + + void flushDb(final AbstractEventListener el, final AtomicBoolean wasCbCalled) + throws RocksDBException { + try (final Options opt = + new Options().setCreateIfMissing(true).setListeners(Collections.singletonList(el)); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + assertThat(db).isNotNull(); + final byte[] value = new byte[24]; + rand.nextBytes(value); + db.put("testKey".getBytes(), value); + db.flush(new FlushOptions()); + assertThat(wasCbCalled.get()).isTrue(); + } + } + + @Test + public void onFlushCompleted() throws RocksDBException { + final AtomicBoolean wasCbCalled = new AtomicBoolean(); + final AbstractEventListener onFlushCompletedListener = new AbstractEventListener() { + @Override + public void onFlushCompleted(final RocksDB rocksDb, final FlushJobInfo flushJobInfo) { + assertThat(flushJobInfo.getColumnFamilyName()).isNotNull(); + assertThat(flushJobInfo.getFlushReason()).isEqualTo(FlushReason.MANUAL_FLUSH); + wasCbCalled.set(true); + } + }; + flushDb(onFlushCompletedListener, wasCbCalled); + } + + @Test + public void onFlushBegin() throws RocksDBException { + final AtomicBoolean wasCbCalled = new AtomicBoolean(); + final AbstractEventListener onFlushBeginListener = new AbstractEventListener() { + @Override + public void onFlushBegin(final RocksDB rocksDb, final FlushJobInfo flushJobInfo) { + assertThat(flushJobInfo.getColumnFamilyName()).isNotNull(); + assertThat(flushJobInfo.getFlushReason()).isEqualTo(FlushReason.MANUAL_FLUSH); + wasCbCalled.set(true); + } + }; + flushDb(onFlushBeginListener, wasCbCalled); + } + + void deleteTableFile(final AbstractEventListener el, final AtomicBoolean wasCbCalled) + throws RocksDBException { + try (final Options opt = + new Options().setCreateIfMissing(true).setListeners(Collections.singletonList(el)); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + assertThat(db).isNotNull(); + final byte[] value = new byte[24]; + rand.nextBytes(value); + db.put("testKey".getBytes(), value); + final RocksDB.LiveFiles liveFiles = db.getLiveFiles(); + assertThat(liveFiles).isNotNull(); + assertThat(liveFiles.files).isNotNull(); + assertThat(liveFiles.files.isEmpty()).isFalse(); + db.deleteFile(liveFiles.files.get(0)); + assertThat(wasCbCalled.get()).isTrue(); + } + } + + @Test + public void onTableFileDeleted() throws RocksDBException { + final AtomicBoolean wasCbCalled = new AtomicBoolean(); + final AbstractEventListener onTableFileDeletedListener = new AbstractEventListener() { + @Override + public void onTableFileDeleted(final TableFileDeletionInfo tableFileDeletionInfo) { + assertThat(tableFileDeletionInfo.getDbName()).isNotNull(); + wasCbCalled.set(true); + } + }; + deleteTableFile(onTableFileDeletedListener, wasCbCalled); + } + + void compactRange(final AbstractEventListener el, final AtomicBoolean wasCbCalled) + throws RocksDBException { + try (final Options opt = + new Options().setCreateIfMissing(true).setListeners(Collections.singletonList(el)); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + assertThat(db).isNotNull(); + final byte[] value = new byte[24]; + rand.nextBytes(value); + db.put("testKey".getBytes(), value); + db.compactRange(); + assertThat(wasCbCalled.get()).isTrue(); + } + } + + @Test + public void onCompactionBegin() throws RocksDBException { + final AtomicBoolean wasCbCalled = new AtomicBoolean(); + final AbstractEventListener onCompactionBeginListener = new AbstractEventListener() { + @Override + public void onCompactionBegin(final RocksDB db, final CompactionJobInfo compactionJobInfo) { + assertThat(compactionJobInfo.compactionReason()) + .isEqualTo(CompactionReason.kManualCompaction); + wasCbCalled.set(true); + } + }; + compactRange(onCompactionBeginListener, wasCbCalled); + } + + @Test + public void onCompactionCompleted() throws RocksDBException { + final AtomicBoolean wasCbCalled = new AtomicBoolean(); + final AbstractEventListener onCompactionCompletedListener = new AbstractEventListener() { + @Override + public void onCompactionCompleted( + final RocksDB db, final CompactionJobInfo compactionJobInfo) { + assertThat(compactionJobInfo.compactionReason()) + .isEqualTo(CompactionReason.kManualCompaction); + wasCbCalled.set(true); + } + }; + compactRange(onCompactionCompletedListener, wasCbCalled); + } + + @Test + public void onTableFileCreated() throws RocksDBException { + final AtomicBoolean wasCbCalled = new AtomicBoolean(); + final AbstractEventListener onTableFileCreatedListener = new AbstractEventListener() { + @Override + public void onTableFileCreated(final TableFileCreationInfo tableFileCreationInfo) { + assertThat(tableFileCreationInfo.getReason()).isEqualTo(TableFileCreationReason.FLUSH); + wasCbCalled.set(true); + } + }; + flushDb(onTableFileCreatedListener, wasCbCalled); + } + + @Test + public void onTableFileCreationStarted() throws RocksDBException { + final AtomicBoolean wasCbCalled = new AtomicBoolean(); + final AbstractEventListener onTableFileCreationStartedListener = new AbstractEventListener() { + @Override + public void onTableFileCreationStarted( + final TableFileCreationBriefInfo tableFileCreationBriefInfo) { + assertThat(tableFileCreationBriefInfo.getReason()).isEqualTo(TableFileCreationReason.FLUSH); + wasCbCalled.set(true); + } + }; + flushDb(onTableFileCreationStartedListener, wasCbCalled); + } + + void deleteColumnFamilyHandle(final AbstractEventListener el, final AtomicBoolean wasCbCalled) + throws RocksDBException { + try (final Options opt = + new Options().setCreateIfMissing(true).setListeners(Collections.singletonList(el)); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + assertThat(db).isNotNull(); + final byte[] value = new byte[24]; + rand.nextBytes(value); + db.put("testKey".getBytes(), value); + final ColumnFamilyHandle columnFamilyHandle = db.getDefaultColumnFamily(); + columnFamilyHandle.close(); + assertThat(wasCbCalled.get()).isTrue(); + } + } + + @Test + public void onColumnFamilyHandleDeletionStarted() throws RocksDBException { + final AtomicBoolean wasCbCalled = new AtomicBoolean(); + final AbstractEventListener onColumnFamilyHandleDeletionStartedListener = + new AbstractEventListener() { + @Override + public void onColumnFamilyHandleDeletionStarted( + final ColumnFamilyHandle columnFamilyHandle) { + assertThat(columnFamilyHandle).isNotNull(); + wasCbCalled.set(true); + } + }; + deleteColumnFamilyHandle(onColumnFamilyHandleDeletionStartedListener, wasCbCalled); + } + + void ingestExternalFile(final AbstractEventListener el, final AtomicBoolean wasCbCalled) + throws RocksDBException { + try (final Options opt = + new Options().setCreateIfMissing(true).setListeners(Collections.singletonList(el)); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + assertThat(db).isNotNull(); + final String uuid = UUID.randomUUID().toString(); + final SstFileWriter sstFileWriter = new SstFileWriter(new EnvOptions(), opt); + final Path externalFilePath = Paths.get(db.getName(), uuid); + sstFileWriter.open(externalFilePath.toString()); + sstFileWriter.put("testKey".getBytes(), uuid.getBytes()); + sstFileWriter.finish(); + db.ingestExternalFile( + Collections.singletonList(externalFilePath.toString()), new IngestExternalFileOptions()); + assertThat(wasCbCalled.get()).isTrue(); + } + } + + @Test + public void onExternalFileIngested() throws RocksDBException { + final AtomicBoolean wasCbCalled = new AtomicBoolean(); + final AbstractEventListener onExternalFileIngestedListener = new AbstractEventListener() { + @Override + public void onExternalFileIngested( + final RocksDB db, final ExternalFileIngestionInfo externalFileIngestionInfo) { + assertThat(db).isNotNull(); + wasCbCalled.set(true); + } + }; + ingestExternalFile(onExternalFileIngestedListener, wasCbCalled); + } + + @Test + public void testAllCallbacksInvocation() { + final long TEST_LONG_VAL = -1; + // Expected test data objects + final Map userCollectedPropertiesTestData = + Collections.singletonMap("key", "value"); + final Map readablePropertiesTestData = Collections.singletonMap("key", "value"); + final TableProperties tablePropertiesTestData = new TableProperties(TEST_LONG_VAL, + TEST_LONG_VAL, TEST_LONG_VAL, TEST_LONG_VAL, TEST_LONG_VAL, TEST_LONG_VAL, TEST_LONG_VAL, + TEST_LONG_VAL, TEST_LONG_VAL, TEST_LONG_VAL, TEST_LONG_VAL, TEST_LONG_VAL, TEST_LONG_VAL, + TEST_LONG_VAL, TEST_LONG_VAL, TEST_LONG_VAL, TEST_LONG_VAL, TEST_LONG_VAL, TEST_LONG_VAL, + TEST_LONG_VAL, TEST_LONG_VAL, TEST_LONG_VAL, "columnFamilyName".getBytes(), + "filterPolicyName", "comparatorName", "mergeOperatorName", "prefixExtractorName", + "propertyCollectorsNames", "compressionName", userCollectedPropertiesTestData, + readablePropertiesTestData); + final FlushJobInfo flushJobInfoTestData = new FlushJobInfo(Integer.MAX_VALUE, + "testColumnFamily", "/file/path", TEST_LONG_VAL, Integer.MAX_VALUE, true, true, + TEST_LONG_VAL, TEST_LONG_VAL, tablePropertiesTestData, (byte) 0x0a); + final Status statusTestData = new Status(Status.Code.Incomplete, Status.SubCode.NoSpace, null); + final TableFileDeletionInfo tableFileDeletionInfoTestData = + new TableFileDeletionInfo("dbName", "/file/path", Integer.MAX_VALUE, statusTestData); + final TableFileCreationInfo tableFileCreationInfoTestData = + new TableFileCreationInfo(TEST_LONG_VAL, tablePropertiesTestData, statusTestData, "dbName", + "columnFamilyName", "/file/path", Integer.MAX_VALUE, (byte) 0x03); + final TableFileCreationBriefInfo tableFileCreationBriefInfoTestData = + new TableFileCreationBriefInfo( + "dbName", "columnFamilyName", "/file/path", Integer.MAX_VALUE, (byte) 0x03); + final MemTableInfo memTableInfoTestData = new MemTableInfo( + "columnFamilyName", TEST_LONG_VAL, TEST_LONG_VAL, TEST_LONG_VAL, TEST_LONG_VAL); + final FileOperationInfo fileOperationInfoTestData = new FileOperationInfo("/file/path", + TEST_LONG_VAL, TEST_LONG_VAL, 1_600_699_420_000_000_000L, 5_000_000_000L, statusTestData); + final WriteStallInfo writeStallInfoTestData = + new WriteStallInfo("columnFamilyName", (byte) 0x0, (byte) 0x1); + final ExternalFileIngestionInfo externalFileIngestionInfoTestData = + new ExternalFileIngestionInfo("columnFamilyName", "/external/file/path", + "/internal/file/path", TEST_LONG_VAL, tablePropertiesTestData); + + final CapturingTestableEventListener listener = new CapturingTestableEventListener() { + @Override + public void onFlushCompleted(final RocksDB db, final FlushJobInfo flushJobInfo) { + super.onFlushCompleted(db, flushJobInfo); + assertThat(flushJobInfo).isEqualTo(flushJobInfoTestData); + } + + @Override + public void onFlushBegin(final RocksDB db, final FlushJobInfo flushJobInfo) { + super.onFlushBegin(db, flushJobInfo); + assertThat(flushJobInfo).isEqualTo(flushJobInfoTestData); + } + + @Override + public void onTableFileDeleted(final TableFileDeletionInfo tableFileDeletionInfo) { + super.onTableFileDeleted(tableFileDeletionInfo); + assertThat(tableFileDeletionInfo).isEqualTo(tableFileDeletionInfoTestData); + } + + @Override + public void onCompactionBegin(final RocksDB db, final CompactionJobInfo compactionJobInfo) { + super.onCompactionBegin(db, compactionJobInfo); + assertThat(new String(compactionJobInfo.columnFamilyName(), StandardCharsets.UTF_8)) + .isEqualTo("compactionColumnFamily"); + assertThat(compactionJobInfo.status()).isEqualTo(statusTestData); + assertThat(compactionJobInfo.threadId()).isEqualTo(TEST_LONG_VAL); + assertThat(compactionJobInfo.jobId()).isEqualTo(Integer.MAX_VALUE); + assertThat(compactionJobInfo.baseInputLevel()).isEqualTo(Integer.MAX_VALUE); + assertThat(compactionJobInfo.outputLevel()).isEqualTo(Integer.MAX_VALUE); + assertThat(compactionJobInfo.inputFiles()) + .isEqualTo(Collections.singletonList("inputFile.sst")); + assertThat(compactionJobInfo.outputFiles()) + .isEqualTo(Collections.singletonList("outputFile.sst")); + assertThat(compactionJobInfo.tableProperties()) + .isEqualTo(Collections.singletonMap("tableProperties", tablePropertiesTestData)); + assertThat(compactionJobInfo.compactionReason()).isEqualTo(CompactionReason.kFlush); + assertThat(compactionJobInfo.compression()).isEqualTo(CompressionType.SNAPPY_COMPRESSION); + } + + @Override + public void onCompactionCompleted( + final RocksDB db, final CompactionJobInfo compactionJobInfo) { + super.onCompactionCompleted(db, compactionJobInfo); + assertThat(new String(compactionJobInfo.columnFamilyName())) + .isEqualTo("compactionColumnFamily"); + assertThat(compactionJobInfo.status()).isEqualTo(statusTestData); + assertThat(compactionJobInfo.threadId()).isEqualTo(TEST_LONG_VAL); + assertThat(compactionJobInfo.jobId()).isEqualTo(Integer.MAX_VALUE); + assertThat(compactionJobInfo.baseInputLevel()).isEqualTo(Integer.MAX_VALUE); + assertThat(compactionJobInfo.outputLevel()).isEqualTo(Integer.MAX_VALUE); + assertThat(compactionJobInfo.inputFiles()) + .isEqualTo(Collections.singletonList("inputFile.sst")); + assertThat(compactionJobInfo.outputFiles()) + .isEqualTo(Collections.singletonList("outputFile.sst")); + assertThat(compactionJobInfo.tableProperties()) + .isEqualTo(Collections.singletonMap("tableProperties", tablePropertiesTestData)); + assertThat(compactionJobInfo.compactionReason()).isEqualTo(CompactionReason.kFlush); + assertThat(compactionJobInfo.compression()).isEqualTo(CompressionType.SNAPPY_COMPRESSION); + } + + @Override + public void onTableFileCreated(final TableFileCreationInfo tableFileCreationInfo) { + super.onTableFileCreated(tableFileCreationInfo); + assertThat(tableFileCreationInfo).isEqualTo(tableFileCreationInfoTestData); + } + + @Override + public void onTableFileCreationStarted( + final TableFileCreationBriefInfo tableFileCreationBriefInfo) { + super.onTableFileCreationStarted(tableFileCreationBriefInfo); + assertThat(tableFileCreationBriefInfo).isEqualTo(tableFileCreationBriefInfoTestData); + } + + @Override + public void onMemTableSealed(final MemTableInfo memTableInfo) { + super.onMemTableSealed(memTableInfo); + assertThat(memTableInfo).isEqualTo(memTableInfoTestData); + } + + @Override + public void onColumnFamilyHandleDeletionStarted(final ColumnFamilyHandle columnFamilyHandle) { + super.onColumnFamilyHandleDeletionStarted(columnFamilyHandle); + } + + @Override + public void onExternalFileIngested( + final RocksDB db, final ExternalFileIngestionInfo externalFileIngestionInfo) { + super.onExternalFileIngested(db, externalFileIngestionInfo); + assertThat(externalFileIngestionInfo).isEqualTo(externalFileIngestionInfoTestData); + } + + @Override + public void onBackgroundError( + final BackgroundErrorReason backgroundErrorReason, final Status backgroundError) { + super.onBackgroundError(backgroundErrorReason, backgroundError); + } + + @Override + public void onStallConditionsChanged(final WriteStallInfo writeStallInfo) { + super.onStallConditionsChanged(writeStallInfo); + assertThat(writeStallInfo).isEqualTo(writeStallInfoTestData); + } + + @Override + public void onFileReadFinish(final FileOperationInfo fileOperationInfo) { + super.onFileReadFinish(fileOperationInfo); + assertThat(fileOperationInfo).isEqualTo(fileOperationInfoTestData); + } + + @Override + public void onFileWriteFinish(final FileOperationInfo fileOperationInfo) { + super.onFileWriteFinish(fileOperationInfo); + assertThat(fileOperationInfo).isEqualTo(fileOperationInfoTestData); + } + + @Override + public void onFileFlushFinish(final FileOperationInfo fileOperationInfo) { + super.onFileFlushFinish(fileOperationInfo); + assertThat(fileOperationInfo).isEqualTo(fileOperationInfoTestData); + } + + @Override + public void onFileSyncFinish(final FileOperationInfo fileOperationInfo) { + super.onFileSyncFinish(fileOperationInfo); + assertThat(fileOperationInfo).isEqualTo(fileOperationInfoTestData); + } + + @Override + public void onFileRangeSyncFinish(final FileOperationInfo fileOperationInfo) { + super.onFileRangeSyncFinish(fileOperationInfo); + assertThat(fileOperationInfo).isEqualTo(fileOperationInfoTestData); + } + + @Override + public void onFileTruncateFinish(final FileOperationInfo fileOperationInfo) { + super.onFileTruncateFinish(fileOperationInfo); + assertThat(fileOperationInfo).isEqualTo(fileOperationInfoTestData); + } + + @Override + public void onFileCloseFinish(final FileOperationInfo fileOperationInfo) { + super.onFileCloseFinish(fileOperationInfo); + assertThat(fileOperationInfo).isEqualTo(fileOperationInfoTestData); + } + + @Override + public boolean shouldBeNotifiedOnFileIO() { + super.shouldBeNotifiedOnFileIO(); + return false; + } + + @Override + public boolean onErrorRecoveryBegin( + final BackgroundErrorReason backgroundErrorReason, final Status backgroundError) { + super.onErrorRecoveryBegin(backgroundErrorReason, backgroundError); + assertThat(backgroundErrorReason).isEqualTo(BackgroundErrorReason.FLUSH); + assertThat(backgroundError).isEqualTo(statusTestData); + return true; + } + + @Override + public void onErrorRecoveryCompleted(final Status oldBackgroundError) { + super.onErrorRecoveryCompleted(oldBackgroundError); + assertThat(oldBackgroundError).isEqualTo(statusTestData); + } + }; + + // test action + listener.invokeAllCallbacks(); + + // assert + assertAllEventsCalled(listener); + + assertNoCallbackErrors(listener); + } + + @Test + public void testEnabledCallbacks() { + final EnabledEventCallback[] enabledEvents = { + EnabledEventCallback.ON_MEMTABLE_SEALED, EnabledEventCallback.ON_ERROR_RECOVERY_COMPLETED}; + + final CapturingTestableEventListener listener = + new CapturingTestableEventListener(enabledEvents); + + // test action + listener.invokeAllCallbacks(); + + // assert + assertEventsCalled(listener, enabledEvents); + } + + private static void assertAllEventsCalled( + final CapturingTestableEventListener capturingTestableEventListener) { + assertEventsCalled(capturingTestableEventListener, EnumSet.allOf(EnabledEventCallback.class)); + } + + private static void assertEventsCalled( + final CapturingTestableEventListener capturingTestableEventListener, + final EnabledEventCallback[] expected) { + assertEventsCalled(capturingTestableEventListener, EnumSet.copyOf(Arrays.asList(expected))); + } + + private static void assertNoCallbackErrors( + final CapturingTestableEventListener capturingTestableEventListener) { + for (final AssertionError error : capturingTestableEventListener.capturedAssertionErrors) { + throw new Error("An assertion failed in callback", error); + } + } + + private static void assertEventsCalled( + final CapturingTestableEventListener capturingTestableEventListener, + final EnumSet expected) { + final ListenerEvents capturedEvents = capturingTestableEventListener.capturedListenerEvents; + + assertThat(capturedEvents.flushCompleted) + .isEqualTo(expected.contains(EnabledEventCallback.ON_FLUSH_COMPLETED)); + assertThat(capturedEvents.flushBegin) + .isEqualTo(expected.contains(EnabledEventCallback.ON_FLUSH_BEGIN)); + assertThat(capturedEvents.tableFileDeleted) + .isEqualTo(expected.contains(EnabledEventCallback.ON_TABLE_FILE_DELETED)); + assertThat(capturedEvents.compactionBegin) + .isEqualTo(expected.contains(EnabledEventCallback.ON_COMPACTION_BEGIN)); + assertThat(capturedEvents.compactionCompleted) + .isEqualTo(expected.contains(EnabledEventCallback.ON_COMPACTION_COMPLETED)); + assertThat(capturedEvents.tableFileCreated) + .isEqualTo(expected.contains(EnabledEventCallback.ON_TABLE_FILE_CREATED)); + assertThat(capturedEvents.tableFileCreationStarted) + .isEqualTo(expected.contains(EnabledEventCallback.ON_TABLE_FILE_CREATION_STARTED)); + assertThat(capturedEvents.memTableSealed) + .isEqualTo(expected.contains(EnabledEventCallback.ON_MEMTABLE_SEALED)); + assertThat(capturedEvents.columnFamilyHandleDeletionStarted) + .isEqualTo( + expected.contains(EnabledEventCallback.ON_COLUMN_FAMILY_HANDLE_DELETION_STARTED)); + assertThat(capturedEvents.externalFileIngested) + .isEqualTo(expected.contains(EnabledEventCallback.ON_EXTERNAL_FILE_INGESTED)); + assertThat(capturedEvents.backgroundError) + .isEqualTo(expected.contains(EnabledEventCallback.ON_BACKGROUND_ERROR)); + assertThat(capturedEvents.stallConditionsChanged) + .isEqualTo(expected.contains(EnabledEventCallback.ON_STALL_CONDITIONS_CHANGED)); + assertThat(capturedEvents.fileReadFinish) + .isEqualTo(expected.contains(EnabledEventCallback.ON_FILE_READ_FINISH)); + assertThat(capturedEvents.fileWriteFinish) + .isEqualTo(expected.contains(EnabledEventCallback.ON_FILE_WRITE_FINISH)); + assertThat(capturedEvents.fileFlushFinish) + .isEqualTo(expected.contains(EnabledEventCallback.ON_FILE_FLUSH_FINISH)); + assertThat(capturedEvents.fileSyncFinish) + .isEqualTo(expected.contains(EnabledEventCallback.ON_FILE_SYNC_FINISH)); + assertThat(capturedEvents.fileRangeSyncFinish) + .isEqualTo(expected.contains(EnabledEventCallback.ON_FILE_RANGE_SYNC_FINISH)); + assertThat(capturedEvents.fileTruncateFinish) + .isEqualTo(expected.contains(EnabledEventCallback.ON_FILE_TRUNCATE_FINISH)); + assertThat(capturedEvents.fileCloseFinish) + .isEqualTo(expected.contains(EnabledEventCallback.ON_FILE_CLOSE_FINISH)); + assertThat(capturedEvents.shouldBeNotifiedOnFileIO) + .isEqualTo(expected.contains(EnabledEventCallback.SHOULD_BE_NOTIFIED_ON_FILE_IO)); + assertThat(capturedEvents.errorRecoveryBegin) + .isEqualTo(expected.contains(EnabledEventCallback.ON_ERROR_RECOVERY_BEGIN)); + assertThat(capturedEvents.errorRecoveryCompleted) + .isEqualTo(expected.contains(EnabledEventCallback.ON_ERROR_RECOVERY_COMPLETED)); + assertThat(capturedEvents.errorRecoveryCompleted) + .isEqualTo(expected.contains(EnabledEventCallback.ON_ERROR_RECOVERY_COMPLETED)); + } + + /** + * Members are volatile as they may be written + * and read by different threads. + */ + private static class ListenerEvents { + volatile boolean flushCompleted; + volatile boolean flushBegin; + volatile boolean tableFileDeleted; + volatile boolean compactionBegin; + volatile boolean compactionCompleted; + volatile boolean tableFileCreated; + volatile boolean tableFileCreationStarted; + volatile boolean memTableSealed; + volatile boolean columnFamilyHandleDeletionStarted; + volatile boolean externalFileIngested; + volatile boolean backgroundError; + volatile boolean stallConditionsChanged; + volatile boolean fileReadFinish; + volatile boolean fileWriteFinish; + volatile boolean fileFlushFinish; + volatile boolean fileSyncFinish; + volatile boolean fileRangeSyncFinish; + volatile boolean fileTruncateFinish; + volatile boolean fileCloseFinish; + volatile boolean shouldBeNotifiedOnFileIO; + volatile boolean errorRecoveryBegin; + volatile boolean errorRecoveryCompleted; + } + + private static class CapturingObjectAssert extends ObjectAssert { + private final List assertionErrors; + public CapturingObjectAssert(final T t, final List assertionErrors) { + super(t); + this.assertionErrors = assertionErrors; + } + + @Override + public ObjectAssert isEqualTo(final Object other) { + try { + return super.isEqualTo(other); + } catch (final AssertionError error) { + assertionErrors.add(error); + throw error; + } + } + + @Override + public ObjectAssert isNotNull() { + try { + return super.isNotNull(); + } catch (final AssertionError error) { + assertionErrors.add(error); + throw error; + } + } + } + + private static class CapturingTestableEventListener extends TestableEventListener { + final ListenerEvents capturedListenerEvents = new ListenerEvents(); + + final List capturedAssertionErrors = new ArrayList<>(); + + protected AbstractObjectAssert assertThat(final T actual) { + return new CapturingObjectAssert<>(actual, capturedAssertionErrors); + } + + public CapturingTestableEventListener() {} + + public CapturingTestableEventListener(final EnabledEventCallback... enabledEventCallbacks) { + super(enabledEventCallbacks); + } + + @Override + public void onFlushCompleted(final RocksDB db, final FlushJobInfo flushJobInfo) { + capturedListenerEvents.flushCompleted = true; + } + + @Override + public void onFlushBegin(final RocksDB db, final FlushJobInfo flushJobInfo) { + capturedListenerEvents.flushBegin = true; + } + + @Override + public void onTableFileDeleted(final TableFileDeletionInfo tableFileDeletionInfo) { + capturedListenerEvents.tableFileDeleted = true; + } + + @Override + public void onCompactionBegin(final RocksDB db, final CompactionJobInfo compactionJobInfo) { + capturedListenerEvents.compactionBegin = true; + } + + @Override + public void onCompactionCompleted(final RocksDB db, final CompactionJobInfo compactionJobInfo) { + capturedListenerEvents.compactionCompleted = true; + } + + @Override + public void onTableFileCreated(final TableFileCreationInfo tableFileCreationInfo) { + capturedListenerEvents.tableFileCreated = true; + } + + @Override + public void onTableFileCreationStarted( + final TableFileCreationBriefInfo tableFileCreationBriefInfo) { + capturedListenerEvents.tableFileCreationStarted = true; + } + + @Override + public void onMemTableSealed(final MemTableInfo memTableInfo) { + capturedListenerEvents.memTableSealed = true; + } + + @Override + public void onColumnFamilyHandleDeletionStarted(final ColumnFamilyHandle columnFamilyHandle) { + capturedListenerEvents.columnFamilyHandleDeletionStarted = true; + } + + @Override + public void onExternalFileIngested( + final RocksDB db, final ExternalFileIngestionInfo externalFileIngestionInfo) { + capturedListenerEvents.externalFileIngested = true; + } + + @Override + public void onBackgroundError( + final BackgroundErrorReason backgroundErrorReason, final Status backgroundError) { + capturedListenerEvents.backgroundError = true; + } + + @Override + public void onStallConditionsChanged(final WriteStallInfo writeStallInfo) { + capturedListenerEvents.stallConditionsChanged = true; + } + + @Override + public void onFileReadFinish(final FileOperationInfo fileOperationInfo) { + capturedListenerEvents.fileReadFinish = true; + } + + @Override + public void onFileWriteFinish(final FileOperationInfo fileOperationInfo) { + capturedListenerEvents.fileWriteFinish = true; + } + + @Override + public void onFileFlushFinish(final FileOperationInfo fileOperationInfo) { + capturedListenerEvents.fileFlushFinish = true; + } + + @Override + public void onFileSyncFinish(final FileOperationInfo fileOperationInfo) { + capturedListenerEvents.fileSyncFinish = true; + } + + @Override + public void onFileRangeSyncFinish(final FileOperationInfo fileOperationInfo) { + capturedListenerEvents.fileRangeSyncFinish = true; + } + + @Override + public void onFileTruncateFinish(final FileOperationInfo fileOperationInfo) { + capturedListenerEvents.fileTruncateFinish = true; + } + + @Override + public void onFileCloseFinish(final FileOperationInfo fileOperationInfo) { + capturedListenerEvents.fileCloseFinish = true; + } + + @Override + public boolean shouldBeNotifiedOnFileIO() { + capturedListenerEvents.shouldBeNotifiedOnFileIO = true; + return false; + } + + @Override + public boolean onErrorRecoveryBegin( + final BackgroundErrorReason backgroundErrorReason, final Status backgroundError) { + capturedListenerEvents.errorRecoveryBegin = true; + return true; + } + + @Override + public void onErrorRecoveryCompleted(final Status oldBackgroundError) { + capturedListenerEvents.errorRecoveryCompleted = true; + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/FilterTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/FilterTest.java new file mode 100644 index 0000000..dc5c19f --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/FilterTest.java @@ -0,0 +1,39 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Test; + +public class FilterTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Test + public void filter() { + // new Bloom filter + final BlockBasedTableConfig blockConfig = new BlockBasedTableConfig(); + try(final Options options = new Options()) { + + try(final Filter bloomFilter = new BloomFilter()) { + blockConfig.setFilterPolicy(bloomFilter); + options.setTableFormatConfig(blockConfig); + } + + try(final Filter bloomFilter = new BloomFilter(10)) { + blockConfig.setFilterPolicy(bloomFilter); + options.setTableFormatConfig(blockConfig); + } + + try(final Filter bloomFilter = new BloomFilter(10, false)) { + blockConfig.setFilterPolicy(bloomFilter); + options.setTableFormatConfig(blockConfig); + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/FlushOptionsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/FlushOptionsTest.java new file mode 100644 index 0000000..f90ae91 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/FlushOptionsTest.java @@ -0,0 +1,31 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class FlushOptionsTest { + + @Test + public void waitForFlush() { + try (final FlushOptions flushOptions = new FlushOptions()) { + assertThat(flushOptions.waitForFlush()).isTrue(); + flushOptions.setWaitForFlush(false); + assertThat(flushOptions.waitForFlush()).isFalse(); + } + } + + @Test + public void allowWriteStall() { + try (final FlushOptions flushOptions = new FlushOptions()) { + assertThat(flushOptions.allowWriteStall()).isFalse(); + flushOptions.setAllowWriteStall(true); + assertThat(flushOptions.allowWriteStall()).isTrue(); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/FlushTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/FlushTest.java new file mode 100644 index 0000000..1a354f4 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/FlushTest.java @@ -0,0 +1,49 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import static org.assertj.core.api.Assertions.assertThat; + +public class FlushTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void flush() throws RocksDBException { + try(final Options options = new Options() + .setCreateIfMissing(true) + .setMaxWriteBufferNumber(10) + .setMinWriteBufferNumberToMerge(10); + final WriteOptions wOpt = new WriteOptions() + .setDisableWAL(true); + final FlushOptions flushOptions = new FlushOptions() + .setWaitForFlush(true)) { + assertThat(flushOptions.waitForFlush()).isTrue(); + + try(final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath())) { + db.put(wOpt, "key1".getBytes(), "value1".getBytes()); + db.put(wOpt, "key2".getBytes(), "value2".getBytes()); + db.put(wOpt, "key3".getBytes(), "value3".getBytes()); + db.put(wOpt, "key4".getBytes(), "value4".getBytes()); + assertThat(db.getProperty("rocksdb.num-entries-active-mem-table")) + .isEqualTo("4"); + db.flush(flushOptions); + assertThat(db.getProperty("rocksdb.num-entries-active-mem-table")) + .isEqualTo("0"); + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/InfoLogLevelTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/InfoLogLevelTest.java new file mode 100644 index 0000000..90b0b4e --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/InfoLogLevelTest.java @@ -0,0 +1,109 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.util.Environment; + +import java.io.IOException; + +import static java.nio.file.Files.readAllBytes; +import static java.nio.file.Paths.get; +import static org.assertj.core.api.Assertions.assertThat; + +public class InfoLogLevelTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void testInfoLogLevel() throws RocksDBException, + IOException { + try (final RocksDB db = + RocksDB.open(dbFolder.getRoot().getAbsolutePath())) { + db.put("key".getBytes(), "value".getBytes()); + db.flush(new FlushOptions().setWaitForFlush(true)); + assertThat(getLogContentsWithoutHeader()).isNotEmpty(); + } + } + + @Test + public void testFatalLogLevel() throws RocksDBException, + IOException { + try (final Options options = new Options(). + setCreateIfMissing(true). + setInfoLogLevel(InfoLogLevel.FATAL_LEVEL); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath())) { + assertThat(options.infoLogLevel()). + isEqualTo(InfoLogLevel.FATAL_LEVEL); + db.put("key".getBytes(), "value".getBytes()); + // As InfoLogLevel is set to FATAL_LEVEL, here we expect the log + // content to be empty. + assertThat(getLogContentsWithoutHeader()).isEmpty(); + } + } + + @Test + public void testFatalLogLevelWithDBOptions() + throws RocksDBException, IOException { + try (final DBOptions dbOptions = new DBOptions(). + setInfoLogLevel(InfoLogLevel.FATAL_LEVEL); + final Options options = new Options(dbOptions, + new ColumnFamilyOptions()). + setCreateIfMissing(true); + final RocksDB db = + RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + assertThat(dbOptions.infoLogLevel()). + isEqualTo(InfoLogLevel.FATAL_LEVEL); + assertThat(options.infoLogLevel()). + isEqualTo(InfoLogLevel.FATAL_LEVEL); + db.put("key".getBytes(), "value".getBytes()); + assertThat(getLogContentsWithoutHeader()).isEmpty(); + } + } + + @Test(expected = IllegalArgumentException.class) + public void failIfIllegalByteValueProvided() { + InfoLogLevel.getInfoLogLevel((byte) -1); + } + + @Test + public void valueOf() { + assertThat(InfoLogLevel.valueOf("DEBUG_LEVEL")). + isEqualTo(InfoLogLevel.DEBUG_LEVEL); + } + + /** + * Read LOG file contents into String. + * + * @return LOG file contents as String. + * @throws IOException if file is not found. + */ + private String getLogContentsWithoutHeader() throws IOException { + final String separator = Environment.isWindows() ? + "\n" : System.getProperty("line.separator"); + final String[] lines = new String(readAllBytes(get( + dbFolder.getRoot().getAbsolutePath() + "/LOG"))).split(separator); + + int first_non_header = lines.length; + // Identify the last line of the header + for (int i = lines.length - 1; i >= 0; --i) { + if (lines[i].contains("DB pointer")) { + first_non_header = i + 1; + break; + } + } + final StringBuilder builder = new StringBuilder(); + for (int i = first_non_header; i < lines.length; ++i) { + builder.append(lines[i]).append(separator); + } + return builder.toString(); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/IngestExternalFileOptionsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/IngestExternalFileOptionsTest.java new file mode 100644 index 0000000..2306946 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/IngestExternalFileOptionsTest.java @@ -0,0 +1,107 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Test; + +import java.util.Random; + +import static org.assertj.core.api.Assertions.assertThat; + +public class IngestExternalFileOptionsTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE + = new RocksNativeLibraryResource(); + + public static final Random rand = + PlatformRandomHelper.getPlatformSpecificRandomFactory(); + + @Test + public void createExternalSstFileInfoWithoutParameters() { + try (final IngestExternalFileOptions options = + new IngestExternalFileOptions()) { + assertThat(options).isNotNull(); + } + } + + @Test + public void createExternalSstFileInfoWithParameters() { + final boolean moveFiles = rand.nextBoolean(); + final boolean snapshotConsistency = rand.nextBoolean(); + final boolean allowGlobalSeqNo = rand.nextBoolean(); + final boolean allowBlockingFlush = rand.nextBoolean(); + try (final IngestExternalFileOptions options = + new IngestExternalFileOptions(moveFiles, snapshotConsistency, + allowGlobalSeqNo, allowBlockingFlush)) { + assertThat(options).isNotNull(); + assertThat(options.moveFiles()).isEqualTo(moveFiles); + assertThat(options.snapshotConsistency()).isEqualTo(snapshotConsistency); + assertThat(options.allowGlobalSeqNo()).isEqualTo(allowGlobalSeqNo); + assertThat(options.allowBlockingFlush()).isEqualTo(allowBlockingFlush); + } + } + + @Test + public void moveFiles() { + try (final IngestExternalFileOptions options = + new IngestExternalFileOptions()) { + final boolean moveFiles = rand.nextBoolean(); + options.setMoveFiles(moveFiles); + assertThat(options.moveFiles()).isEqualTo(moveFiles); + } + } + + @Test + public void snapshotConsistency() { + try (final IngestExternalFileOptions options = + new IngestExternalFileOptions()) { + final boolean snapshotConsistency = rand.nextBoolean(); + options.setSnapshotConsistency(snapshotConsistency); + assertThat(options.snapshotConsistency()).isEqualTo(snapshotConsistency); + } + } + + @Test + public void allowGlobalSeqNo() { + try (final IngestExternalFileOptions options = + new IngestExternalFileOptions()) { + final boolean allowGlobalSeqNo = rand.nextBoolean(); + options.setAllowGlobalSeqNo(allowGlobalSeqNo); + assertThat(options.allowGlobalSeqNo()).isEqualTo(allowGlobalSeqNo); + } + } + + @Test + public void allowBlockingFlush() { + try (final IngestExternalFileOptions options = + new IngestExternalFileOptions()) { + final boolean allowBlockingFlush = rand.nextBoolean(); + options.setAllowBlockingFlush(allowBlockingFlush); + assertThat(options.allowBlockingFlush()).isEqualTo(allowBlockingFlush); + } + } + + @Test + public void ingestBehind() { + try (final IngestExternalFileOptions options = + new IngestExternalFileOptions()) { + assertThat(options.ingestBehind()).isFalse(); + options.setIngestBehind(true); + assertThat(options.ingestBehind()).isTrue(); + } + } + + @Test + public void writeGlobalSeqno() { + try (final IngestExternalFileOptions options = + new IngestExternalFileOptions()) { + assertThat(options.writeGlobalSeqno()).isFalse(); + options.setWriteGlobalSeqno(true); + assertThat(options.writeGlobalSeqno()).isTrue(); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/KeyMayExistTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/KeyMayExistTest.java new file mode 100644 index 0000000..3f3bec6 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/KeyMayExistTest.java @@ -0,0 +1,528 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.*; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; + +public class KeyMayExistTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Rule public ExpectedException exceptionRule = ExpectedException.none(); + + List cfDescriptors; + List columnFamilyHandleList = new ArrayList<>(); + RocksDB db; + + // Slice key + int offset; + int len; + + byte[] sliceKey; + byte[] sliceValue; + + @Before + public void before() throws RocksDBException { + cfDescriptors = Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes())); + final DBOptions options = + new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); + + db = RocksDB.open( + options, dbFolder.getRoot().getAbsolutePath(), cfDescriptors, columnFamilyHandleList); + + // Build the slice key + final StringBuilder builder = new StringBuilder("prefix"); + offset = builder.toString().length(); + builder.append("slice key 0"); + len = builder.toString().length() - offset; + builder.append("suffix"); + sliceKey = builder.toString().getBytes(UTF_8); + sliceValue = "slice value 0".getBytes(UTF_8); + } + + @After + public void after() { + for (final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandleList) { + columnFamilyHandle.close(); + } + db.close(); + } + + @Test + public void keyMayExist() throws RocksDBException { + assertThat(columnFamilyHandleList.size()).isEqualTo(2); + + // Standard key + db.put("key".getBytes(UTF_8), "value".getBytes(UTF_8)); + + // Test without column family + final Holder holder = new Holder<>(); + boolean exists = db.keyMayExist("key".getBytes(UTF_8), holder); + assertThat(exists).isTrue(); + assertThat(holder.getValue()).isNotNull(); + assertThat(new String(holder.getValue(), UTF_8)).isEqualTo("value"); + + exists = db.keyMayExist("key".getBytes(UTF_8), null); + assertThat(exists).isTrue(); + } + + @Test + public void keyMayExistReadOptions() throws RocksDBException { + // Test without column family but with readOptions + try (final ReadOptions readOptions = new ReadOptions()) { + // Standard key + db.put("key".getBytes(UTF_8), "value".getBytes(UTF_8)); + + // Slice key + db.put(sliceKey, offset, len, sliceValue, 0, sliceValue.length); + + final Holder holder = new Holder<>(); + boolean exists = db.keyMayExist(readOptions, "key".getBytes(UTF_8), holder); + assertThat(exists).isTrue(); + assertThat(holder.getValue()).isNotNull(); + assertThat(new String(holder.getValue(), UTF_8)).isEqualTo("value"); + + exists = db.keyMayExist(readOptions, "key".getBytes(UTF_8), null); + assertThat(exists).isTrue(); + + exists = db.keyMayExist(readOptions, sliceKey, offset, len, holder); + assertThat(exists).isTrue(); + assertThat(holder.getValue()).isNotNull(); + assertThat(holder.getValue()).isEqualTo(sliceValue); + + exists = db.keyMayExist(readOptions, sliceKey, offset, len, null); + assertThat(exists).isTrue(); + } + } + + @Test + public void keyMayExistColumnFamily() throws RocksDBException { + // Standard key + db.put("key".getBytes(UTF_8), "value".getBytes(UTF_8)); + + // Slice key + db.put(sliceKey, offset, len, sliceValue, 0, sliceValue.length); + + // Test slice key with column family + final Holder holder = new Holder<>(); + boolean exists = db.keyMayExist(columnFamilyHandleList.get(0), sliceKey, offset, len, holder); + assertThat(exists).isTrue(); + assertThat(holder.getValue()).isNotNull(); + assertThat(holder.getValue()).isEqualTo(sliceValue); + + exists = db.keyMayExist(columnFamilyHandleList.get(0), sliceKey, offset, len, null); + assertThat(exists).isTrue(); + } + + @Test + public void keyMayExistColumnFamilyReadOptions() throws RocksDBException { + // Standard key + db.put("key".getBytes(UTF_8), "value".getBytes(UTF_8)); + + // Slice key + db.put(sliceKey, offset, len, sliceValue, 0, sliceValue.length); + + // Test slice key with column family and read options + final Holder holder = new Holder<>(); + try (final ReadOptions readOptions = new ReadOptions()) { + boolean exists = + db.keyMayExist(columnFamilyHandleList.get(0), readOptions, "key".getBytes(UTF_8), holder); + assertThat(exists).isTrue(); + assertThat(holder.getValue()).isNotNull(); + assertThat(new String(holder.getValue(), UTF_8)).isEqualTo("value"); + + exists = + db.keyMayExist(columnFamilyHandleList.get(0), readOptions, "key".getBytes(UTF_8), null); + assertThat(exists).isTrue(); + + // Test slice key with column family and read options + exists = + db.keyMayExist(columnFamilyHandleList.get(0), readOptions, sliceKey, offset, len, holder); + assertThat(exists).isTrue(); + assertThat(holder.getValue()).isNotNull(); + assertThat(holder.getValue()).isEqualTo(sliceValue); + + exists = + db.keyMayExist(columnFamilyHandleList.get(0), readOptions, sliceKey, offset, len, null); + assertThat(exists).isTrue(); + } + } + + @Test + public void keyMayExistSliceKey() throws RocksDBException { + assertThat(columnFamilyHandleList.size()).isEqualTo(2); + + // Standard key + db.put("key".getBytes(UTF_8), "value".getBytes(UTF_8)); + + // Slice key + db.put(sliceKey, offset, len, sliceValue, 0, sliceValue.length); + + final Holder holder = new Holder<>(); + boolean exists = db.keyMayExist(sliceKey, offset, len, holder); + assertThat(exists).isTrue(); + assertThat(holder.getValue()).isNotNull(); + assertThat(holder.getValue()).isEqualTo(sliceValue); + + exists = db.keyMayExist(sliceKey, offset, len, null); + assertThat(exists).isTrue(); + + exists = db.keyMayExist("slice key".getBytes(UTF_8), null); + assertThat(exists).isFalse(); + + exists = db.keyMayExist("slice key 0".getBytes(UTF_8), null); + assertThat(exists).isTrue(); + + // Test with column family + exists = db.keyMayExist(columnFamilyHandleList.get(0), "key".getBytes(UTF_8), holder); + assertThat(exists).isTrue(); + assertThat(holder.getValue()).isNotNull(); + assertThat(new String(holder.getValue(), UTF_8)).isEqualTo("value"); + + exists = db.keyMayExist(columnFamilyHandleList.get(0), "key".getBytes(UTF_8), null); + assertThat(exists).isTrue(); + + // KeyMayExist in CF1 must return null value + exists = db.keyMayExist(columnFamilyHandleList.get(1), "key".getBytes(UTF_8), holder); + assertThat(exists).isFalse(); + assertThat(holder.getValue()).isNull(); + exists = db.keyMayExist(columnFamilyHandleList.get(1), "key".getBytes(UTF_8), null); + assertThat(exists).isFalse(); + + // slice key + exists = db.keyMayExist(columnFamilyHandleList.get(1), sliceKey, 1, 3, holder); + assertThat(exists).isFalse(); + assertThat(holder.getValue()).isNull(); + exists = db.keyMayExist(columnFamilyHandleList.get(1), sliceKey, 1, 3, null); + assertThat(exists).isFalse(); + } + + @Test + public void keyMayExistCF1() throws RocksDBException { + // Standard key + db.put("key".getBytes(UTF_8), "value".getBytes(UTF_8)); + + // Slice key + db.put(sliceKey, offset, len, sliceValue, 0, sliceValue.length); + + // KeyMayExist in CF1 must return null value + final Holder holder = new Holder<>(); + boolean exists = db.keyMayExist(columnFamilyHandleList.get(1), "key".getBytes(UTF_8), holder); + assertThat(exists).isFalse(); + assertThat(holder.getValue()).isNull(); + exists = db.keyMayExist(columnFamilyHandleList.get(1), "key".getBytes(UTF_8), null); + assertThat(exists).isFalse(); + } + + @Test + public void keyMayExistCF1Slice() throws RocksDBException { + // Standard key + db.put("key".getBytes(UTF_8), "value".getBytes(UTF_8)); + + // Slice key + db.put(sliceKey, offset, len, sliceValue, 0, sliceValue.length); + + // slice key + final Holder holder = new Holder<>(); + boolean exists = db.keyMayExist(columnFamilyHandleList.get(1), sliceKey, 1, 3, holder); + assertThat(exists).isFalse(); + assertThat(holder.getValue()).isNull(); + exists = db.keyMayExist(columnFamilyHandleList.get(1), sliceKey, 1, 3, null); + assertThat(exists).isFalse(); + } + + @Test + public void keyMayExistBB() throws RocksDBException { + // Standard key + db.put("keyBB".getBytes(UTF_8), "valueBB".getBytes(UTF_8)); + + final byte[] key = "keyBB".getBytes(UTF_8); + final byte[] value = "valueBB".getBytes(UTF_8); + + final ByteBuffer keyBuffer = ByteBuffer.allocateDirect(key.length); + keyBuffer.put(key, 0, key.length); + keyBuffer.flip(); + + assertThat(db.keyMayExist(keyBuffer)).isEqualTo(true); + + final ByteBuffer valueBuffer = ByteBuffer.allocateDirect(value.length + 24); + valueBuffer.position(12); + KeyMayExist keyMayExist = db.keyMayExist(keyBuffer, valueBuffer); + assertThat(keyMayExist.exists).isEqualTo(KeyMayExist.KeyMayExistEnum.kExistsWithValue); + assertThat(keyMayExist.valueLength).isEqualTo(value.length); + assertThat(valueBuffer.position()).isEqualTo(12); + assertThat(valueBuffer.limit()).isEqualTo(12 + value.length); + byte[] valueGet = new byte[value.length]; + valueBuffer.get(valueGet); + assertThat(valueGet).isEqualTo(value); + + valueBuffer.limit(value.length + 24); + valueBuffer.position(25); + keyMayExist = db.keyMayExist(keyBuffer, valueBuffer); + assertThat(keyMayExist.exists).isEqualTo(KeyMayExist.KeyMayExistEnum.kExistsWithValue); + assertThat(keyMayExist.valueLength).isEqualTo(value.length); + assertThat(valueBuffer.position()).isEqualTo(25); + assertThat(valueBuffer.limit()).isEqualTo(24 + value.length); + valueGet = new byte[value.length - 1]; + valueBuffer.get(valueGet); + assertThat(valueGet).isEqualTo(Arrays.copyOfRange(value, 0, value.length - 1)); + + exceptionRule.expect(BufferUnderflowException.class); + valueGet = new byte[value.length]; + valueBuffer.get(valueGet); + } + + @Test + public void keyMayExistBBReadOptions() throws RocksDBException { + // Standard key + db.put("keyBB".getBytes(UTF_8), "valueBB".getBytes(UTF_8)); + + final byte[] key = "keyBB".getBytes(UTF_8); + final byte[] value = "valueBB".getBytes(UTF_8); + + final ByteBuffer keyBuffer = ByteBuffer.allocateDirect(key.length); + keyBuffer.put(key, 0, key.length); + keyBuffer.flip(); + + try (final ReadOptions readOptions = new ReadOptions()) { + assertThat(db.keyMayExist(readOptions, keyBuffer)).isEqualTo(true); + + final ByteBuffer valueBuffer = ByteBuffer.allocateDirect(value.length + 24); + valueBuffer.position(12); + KeyMayExist keyMayExist = db.keyMayExist(readOptions, keyBuffer, valueBuffer); + assertThat(keyMayExist.exists).isEqualTo(KeyMayExist.KeyMayExistEnum.kExistsWithValue); + assertThat(keyMayExist.valueLength).isEqualTo(value.length); + assertThat(valueBuffer.position()).isEqualTo(12); + assertThat(valueBuffer.limit()).isEqualTo(12 + value.length); + byte[] valueGet = new byte[value.length]; + valueBuffer.get(valueGet); + assertThat(valueGet).isEqualTo(value); + + valueBuffer.limit(value.length + 24); + valueBuffer.position(25); + keyMayExist = db.keyMayExist(readOptions, keyBuffer, valueBuffer); + assertThat(keyMayExist.exists).isEqualTo(KeyMayExist.KeyMayExistEnum.kExistsWithValue); + assertThat(keyMayExist.valueLength).isEqualTo(value.length); + assertThat(valueBuffer.position()).isEqualTo(25); + assertThat(valueBuffer.limit()).isEqualTo(24 + value.length); + valueGet = new byte[value.length - 1]; + valueBuffer.get(valueGet); + assertThat(valueGet).isEqualTo(Arrays.copyOfRange(value, 0, value.length - 1)); + + exceptionRule.expect(BufferUnderflowException.class); + valueGet = new byte[value.length]; + valueBuffer.get(valueGet); + } + } + + @Test + public void keyMayExistBBNullValue() throws RocksDBException { + // Standard key + db.put("keyBB".getBytes(UTF_8), "valueBB".getBytes(UTF_8)); + + final byte[] key = "keyBB".getBytes(UTF_8); + + final ByteBuffer keyBuffer = ByteBuffer.allocateDirect(key.length); + keyBuffer.put(key, 0, key.length); + keyBuffer.flip(); + + exceptionRule.expect(AssertionError.class); + exceptionRule.expectMessage( + "value ByteBuffer parameter cannot be null. If you do not need the value, use a different version of the method"); + final KeyMayExist keyMayExist = db.keyMayExist(keyBuffer, null); + } + + @Test + public void keyMayExistBBCF() throws RocksDBException { + // Standard key + db.put(columnFamilyHandleList.get(0), "keyBBCF0".getBytes(UTF_8), "valueBBCF0".getBytes(UTF_8)); + db.put(columnFamilyHandleList.get(1), "keyBBCF1".getBytes(UTF_8), "valueBBCF1".getBytes(UTF_8)); + + // 0 is the default CF + byte[] key = "keyBBCF0".getBytes(UTF_8); + ByteBuffer keyBuffer = ByteBuffer.allocateDirect(key.length); + keyBuffer.put(key, 0, key.length); + keyBuffer.flip(); + + assertThat(db.keyMayExist(keyBuffer)).isEqualTo(true); + assertThat(db.keyMayExist(columnFamilyHandleList.get(1), keyBuffer)).isEqualTo(false); + assertThat(db.keyMayExist(columnFamilyHandleList.get(0), keyBuffer)).isEqualTo(true); + + // 1 is just a CF + key = "keyBBCF1".getBytes(UTF_8); + keyBuffer = ByteBuffer.allocateDirect(key.length); + keyBuffer.put(key, 0, key.length); + keyBuffer.flip(); + + assertThat(db.keyMayExist(keyBuffer)).isEqualTo(false); + assertThat(db.keyMayExist(columnFamilyHandleList.get(1), keyBuffer)).isEqualTo(true); + assertThat(db.keyMayExist(columnFamilyHandleList.get(0), keyBuffer)).isEqualTo(false); + + exceptionRule.expect(AssertionError.class); + exceptionRule.expectMessage( + "value ByteBuffer parameter cannot be null. If you do not need the value, use a different version of the method"); + final KeyMayExist keyMayExist = db.keyMayExist(columnFamilyHandleList.get(0), keyBuffer, null); + } + + @Test + public void keyMayExistBBCFReadOptions() throws RocksDBException { + // Standard key + db.put(columnFamilyHandleList.get(0), "keyBBCF0".getBytes(UTF_8), "valueBBCF0".getBytes(UTF_8)); + db.put(columnFamilyHandleList.get(1), "keyBBCF1".getBytes(UTF_8), "valueBBCF1".getBytes(UTF_8)); + + // 0 is the default CF + byte[] key = "keyBBCF0".getBytes(UTF_8); + ByteBuffer keyBuffer = ByteBuffer.allocateDirect(key.length); + keyBuffer.put(key, 0, key.length); + keyBuffer.flip(); + + try (final ReadOptions readOptions = new ReadOptions()) { + assertThat(db.keyMayExist(keyBuffer)).isEqualTo(true); + assertThat(db.keyMayExist(columnFamilyHandleList.get(1), readOptions, keyBuffer)) + .isEqualTo(false); + assertThat(db.keyMayExist(columnFamilyHandleList.get(0), readOptions, keyBuffer)) + .isEqualTo(true); + + // 1 is just a CF + key = "keyBBCF1".getBytes(UTF_8); + keyBuffer = ByteBuffer.allocateDirect(key.length); + keyBuffer.put(key, 0, key.length); + keyBuffer.flip(); + + assertThat(db.keyMayExist(readOptions, keyBuffer)).isEqualTo(false); + assertThat(db.keyMayExist(columnFamilyHandleList.get(1), readOptions, keyBuffer)) + .isEqualTo(true); + assertThat(db.keyMayExist(columnFamilyHandleList.get(0), readOptions, keyBuffer)) + .isEqualTo(false); + + exceptionRule.expect(AssertionError.class); + exceptionRule.expectMessage( + "value ByteBuffer parameter cannot be null. If you do not need the value, use a different version of the method"); + final KeyMayExist keyMayExist = + db.keyMayExist(columnFamilyHandleList.get(0), readOptions, keyBuffer, null); + } + } + + @Test + public void keyMayExistBBCFOffset() throws RocksDBException { + db.put(columnFamilyHandleList.get(1), "keyBBCF1".getBytes(UTF_8), "valueBBCF1".getBytes(UTF_8)); + + final byte[] key = "keyBBCF1".getBytes(UTF_8); + final byte[] value = "valueBBCF1".getBytes(UTF_8); + + final ByteBuffer keyBuffer = ByteBuffer.allocateDirect(key.length); + keyBuffer.put(key, 0, key.length); + keyBuffer.flip(); + + assertThat(db.keyMayExist(columnFamilyHandleList.get(1), keyBuffer)).isEqualTo(true); + + final ByteBuffer valueBuffer = ByteBuffer.allocateDirect(value.length + 24); + valueBuffer.position(12); + KeyMayExist keyMayExist = db.keyMayExist(columnFamilyHandleList.get(1), keyBuffer, valueBuffer); + assertThat(keyMayExist.exists).isEqualTo(KeyMayExist.KeyMayExistEnum.kExistsWithValue); + assertThat(keyMayExist.valueLength).isEqualTo(value.length); + assertThat(valueBuffer.position()).isEqualTo(12); + assertThat(valueBuffer.limit()).isEqualTo(12 + value.length); + byte[] valueGet = new byte[value.length]; + valueBuffer.get(valueGet); + assertThat(valueGet).isEqualTo(value); + + valueBuffer.limit(value.length + 24); + valueBuffer.position(25); + keyMayExist = db.keyMayExist(columnFamilyHandleList.get(1), keyBuffer, valueBuffer); + assertThat(keyMayExist.exists).isEqualTo(KeyMayExist.KeyMayExistEnum.kExistsWithValue); + assertThat(keyMayExist.valueLength).isEqualTo(value.length); + assertThat(valueBuffer.position()).isEqualTo(25); + assertThat(valueBuffer.limit()).isEqualTo(24 + value.length); + valueGet = new byte[value.length - 1]; + valueBuffer.get(valueGet); + assertThat(valueGet).isEqualTo(Arrays.copyOfRange(value, 0, value.length - 1)); + + exceptionRule.expect(BufferUnderflowException.class); + valueGet = new byte[value.length]; + valueBuffer.get(valueGet); + } + + @Test + public void keyMayExistBBCFOffsetReadOptions() throws RocksDBException { + db.put(columnFamilyHandleList.get(1), "keyBBCF1".getBytes(UTF_8), "valueBBCF1".getBytes(UTF_8)); + + final byte[] key = "keyBBCF1".getBytes(UTF_8); + final byte[] value = "valueBBCF1".getBytes(UTF_8); + + final ByteBuffer keyBuffer = ByteBuffer.allocateDirect(key.length); + keyBuffer.put(key, 0, key.length); + keyBuffer.flip(); + + try (final ReadOptions readOptions = new ReadOptions()) { + assertThat(db.keyMayExist(columnFamilyHandleList.get(1), readOptions, keyBuffer)) + .isEqualTo(true); + + final ByteBuffer valueBuffer = ByteBuffer.allocateDirect(value.length + 24); + valueBuffer.position(12); + KeyMayExist keyMayExist = + db.keyMayExist(columnFamilyHandleList.get(1), readOptions, keyBuffer, valueBuffer); + assertThat(keyMayExist.exists).isEqualTo(KeyMayExist.KeyMayExistEnum.kExistsWithValue); + assertThat(keyMayExist.valueLength).isEqualTo(value.length); + assertThat(valueBuffer.position()).isEqualTo(12); + assertThat(valueBuffer.limit()).isEqualTo(12 + value.length); + byte[] valueGet = new byte[value.length]; + valueBuffer.get(valueGet); + assertThat(valueGet).isEqualTo(value); + + valueBuffer.limit(value.length + 24); + valueBuffer.position(25); + keyMayExist = + db.keyMayExist(columnFamilyHandleList.get(1), readOptions, keyBuffer, valueBuffer); + assertThat(keyMayExist.exists).isEqualTo(KeyMayExist.KeyMayExistEnum.kExistsWithValue); + assertThat(keyMayExist.valueLength).isEqualTo(value.length); + assertThat(valueBuffer.position()).isEqualTo(25); + assertThat(valueBuffer.limit()).isEqualTo(24 + value.length); + valueGet = new byte[value.length - 1]; + valueBuffer.get(valueGet); + assertThat(valueGet).isEqualTo(Arrays.copyOfRange(value, 0, value.length - 1)); + + exceptionRule.expect(BufferUnderflowException.class); + valueGet = new byte[value.length]; + valueBuffer.get(valueGet); + } + } + + @Test + public void keyMayExistNonUnicodeString() throws RocksDBException { + final byte[] key = "key".getBytes(UTF_8); + final byte[] value = {(byte) 0x80}; // invalid unicode code-point + db.put(key, value); + + final byte[] buf = new byte[10]; + final int read = db.get(key, buf); + assertThat(read).isEqualTo(1); + assertThat(buf).startsWith(value); + + final Holder holder = new Holder<>(); + boolean exists = db.keyMayExist("key".getBytes(UTF_8), holder); + assertThat(exists).isTrue(); + assertThat(holder.getValue()).isNotNull(); + assertThat(holder.getValue()).isEqualTo(value); + + exists = db.keyMayExist("key".getBytes(UTF_8), null); + assertThat(exists).isTrue(); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/LRUCacheTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/LRUCacheTest.java new file mode 100644 index 0000000..4d194e7 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/LRUCacheTest.java @@ -0,0 +1,32 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.ClassRule; +import org.junit.Test; + +public class LRUCacheTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Test + public void newLRUCache() { + final long capacity = 80000000; + final int numShardBits = 16; + final boolean strictCapacityLimit = true; + final double highPriPoolRatio = 0.5; + final double lowPriPoolRatio = 0.5; + try (final Cache lruCache = new LRUCache( + capacity, numShardBits, strictCapacityLimit, highPriPoolRatio, lowPriPoolRatio)) { + //no op + assertThat(lruCache.getUsage()).isGreaterThanOrEqualTo(0); + assertThat(lruCache.getPinnedUsage()).isGreaterThanOrEqualTo(0); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/LoggerTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/LoggerTest.java new file mode 100644 index 0000000..b6a7be5 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/LoggerTest.java @@ -0,0 +1,235 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class LoggerTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void customLogger() throws RocksDBException { + final AtomicInteger logMessageCounter = new AtomicInteger(); + try (final Options options = new Options(). + setInfoLogLevel(InfoLogLevel.DEBUG_LEVEL). + setCreateIfMissing(true); + final Logger logger = new Logger(options) { + // Create new logger with max log level passed by options + @Override + protected void log(final InfoLogLevel infoLogLevel, final String logMsg) { + assertThat(logMsg).isNotNull(); + assertThat(logMsg.length()).isGreaterThan(0); + logMessageCounter.incrementAndGet(); + } + } + ) { + // Set custom logger to options + options.setLogger(logger); + + try (final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath())) { + // there should be more than zero received log messages in + // debug level. + assertThat(logMessageCounter.get()).isGreaterThan(0); + } + } + } + + @Test + public void warnLogger() throws RocksDBException { + final AtomicInteger logMessageCounter = new AtomicInteger(); + try (final Options options = new Options(). + setInfoLogLevel(InfoLogLevel.WARN_LEVEL). + setCreateIfMissing(true); + + final Logger logger = new Logger(options) { + // Create new logger with max log level passed by options + @Override + protected void log(final InfoLogLevel infoLogLevel, final String logMsg) { + assertThat(logMsg).isNotNull(); + assertThat(logMsg.length()).isGreaterThan(0); + logMessageCounter.incrementAndGet(); + } + } + ) { + + // Set custom logger to options + options.setLogger(logger); + + try (final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath())) { + // there should be zero messages + // using warn level as log level. + assertThat(logMessageCounter.get()).isEqualTo(0); + } + } + } + + + @Test + public void fatalLogger() throws RocksDBException { + final AtomicInteger logMessageCounter = new AtomicInteger(); + try (final Options options = new Options(). + setInfoLogLevel(InfoLogLevel.FATAL_LEVEL). + setCreateIfMissing(true); + + final Logger logger = new Logger(options) { + // Create new logger with max log level passed by options + @Override + protected void log(final InfoLogLevel infoLogLevel, final String logMsg) { + assertThat(logMsg).isNotNull(); + assertThat(logMsg.length()).isGreaterThan(0); + logMessageCounter.incrementAndGet(); + } + } + ) { + + // Set custom logger to options + options.setLogger(logger); + + try (final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath())) { + // there should be zero messages + // using fatal level as log level. + assertThat(logMessageCounter.get()).isEqualTo(0); + } + } + } + + @Test + public void dbOptionsLogger() throws RocksDBException { + final AtomicInteger logMessageCounter = new AtomicInteger(); + try (final DBOptions options = new DBOptions(). + setInfoLogLevel(InfoLogLevel.FATAL_LEVEL). + setCreateIfMissing(true); + final Logger logger = new Logger(options) { + // Create new logger with max log level passed by options + @Override + protected void log(final InfoLogLevel infoLogLevel, final String logMsg) { + assertThat(logMsg).isNotNull(); + assertThat(logMsg.length()).isGreaterThan(0); + logMessageCounter.incrementAndGet(); + } + } + ) { + // Set custom logger to options + options.setLogger(logger); + + final List cfDescriptors = + Collections.singletonList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY)); + final List cfHandles = new ArrayList<>(); + + try (final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath(), + cfDescriptors, cfHandles)) { + try { + // there should be zero messages + // using fatal level as log level. + assertThat(logMessageCounter.get()).isEqualTo(0); + } finally { + for (final ColumnFamilyHandle columnFamilyHandle : cfHandles) { + columnFamilyHandle.close(); + } + } + } + } + } + + @Test + public void setWarnLogLevel() { + final AtomicInteger logMessageCounter = new AtomicInteger(); + try (final Options options = new Options(). + setInfoLogLevel(InfoLogLevel.FATAL_LEVEL). + setCreateIfMissing(true); + final Logger logger = new Logger(options) { + // Create new logger with max log level passed by options + @Override + protected void log(final InfoLogLevel infoLogLevel, final String logMsg) { + assertThat(logMsg).isNotNull(); + assertThat(logMsg.length()).isGreaterThan(0); + logMessageCounter.incrementAndGet(); + } + } + ) { + assertThat(logger.infoLogLevel()). + isEqualTo(InfoLogLevel.FATAL_LEVEL); + logger.setInfoLogLevel(InfoLogLevel.WARN_LEVEL); + assertThat(logger.infoLogLevel()). + isEqualTo(InfoLogLevel.WARN_LEVEL); + } + } + + @Test + public void setInfoLogLevel() { + final AtomicInteger logMessageCounter = new AtomicInteger(); + try (final Options options = new Options(). + setInfoLogLevel(InfoLogLevel.FATAL_LEVEL). + setCreateIfMissing(true); + final Logger logger = new Logger(options) { + // Create new logger with max log level passed by options + @Override + protected void log(final InfoLogLevel infoLogLevel, final String logMsg) { + assertThat(logMsg).isNotNull(); + assertThat(logMsg.length()).isGreaterThan(0); + logMessageCounter.incrementAndGet(); + } + } + ) { + assertThat(logger.infoLogLevel()). + isEqualTo(InfoLogLevel.FATAL_LEVEL); + logger.setInfoLogLevel(InfoLogLevel.DEBUG_LEVEL); + assertThat(logger.infoLogLevel()). + isEqualTo(InfoLogLevel.DEBUG_LEVEL); + } + } + + @Test + public void changeLogLevelAtRuntime() throws RocksDBException { + final AtomicInteger logMessageCounter = new AtomicInteger(); + try (final Options options = + new Options().setInfoLogLevel(InfoLogLevel.FATAL_LEVEL).setCreateIfMissing(true); + + // Create new logger with max log level passed by options + final Logger logger = new Logger(options) { + @Override + protected void log(final InfoLogLevel infoLogLevel, final String logMsg) { + assertThat(logMsg).isNotNull(); + assertThat(logMsg.length()).isGreaterThan(0); + logMessageCounter.incrementAndGet(); + } + }) { + // Set custom logger to options + options.setLogger(logger); + + try (final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath())) { + + // there should be zero messages + // using fatal level as log level. + assertThat(logMessageCounter.get()).isEqualTo(0); + + // change log level to debug level + logger.setInfoLogLevel(InfoLogLevel.DEBUG_LEVEL); + + db.put("key".getBytes(), "value".getBytes()); + db.flush(new FlushOptions().setWaitForFlush(true)); + + // messages shall be received due to previous actions. + assertThat(logMessageCounter.get()).isNotEqualTo(0); + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MemTableTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MemTableTest.java new file mode 100644 index 0000000..6ebf9ef --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MemTableTest.java @@ -0,0 +1,108 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class MemTableTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Test + public void hashSkipListMemTable() throws RocksDBException { + try(final Options options = new Options()) { + // Test HashSkipListMemTableConfig + final HashSkipListMemTableConfig memTableConfig = new HashSkipListMemTableConfig(); + assertThat(memTableConfig.bucketCount()). + isEqualTo(1000000); + memTableConfig.setBucketCount(2000000); + assertThat(memTableConfig.bucketCount()). + isEqualTo(2000000); + assertThat(memTableConfig.height()). + isEqualTo(4); + memTableConfig.setHeight(5); + assertThat(memTableConfig.height()). + isEqualTo(5); + assertThat(memTableConfig.branchingFactor()). + isEqualTo(4); + memTableConfig.setBranchingFactor(6); + assertThat(memTableConfig.branchingFactor()). + isEqualTo(6); + options.setMemTableConfig(memTableConfig); + } + } + + @Test + public void skipListMemTable() throws RocksDBException { + try(final Options options = new Options()) { + final SkipListMemTableConfig skipMemTableConfig = new SkipListMemTableConfig(); + assertThat(skipMemTableConfig.lookahead()). + isEqualTo(0); + skipMemTableConfig.setLookahead(20); + assertThat(skipMemTableConfig.lookahead()). + isEqualTo(20); + options.setMemTableConfig(skipMemTableConfig); + } + } + + @Test + public void hashLinkedListMemTable() throws RocksDBException { + try(final Options options = new Options()) { + final HashLinkedListMemTableConfig hashLinkedListMemTableConfig = + new HashLinkedListMemTableConfig(); + assertThat(hashLinkedListMemTableConfig.bucketCount()). + isEqualTo(50000); + hashLinkedListMemTableConfig.setBucketCount(100000); + assertThat(hashLinkedListMemTableConfig.bucketCount()). + isEqualTo(100000); + assertThat(hashLinkedListMemTableConfig.hugePageTlbSize()). + isEqualTo(0); + hashLinkedListMemTableConfig.setHugePageTlbSize(1); + assertThat(hashLinkedListMemTableConfig.hugePageTlbSize()). + isEqualTo(1); + assertThat(hashLinkedListMemTableConfig. + bucketEntriesLoggingThreshold()). + isEqualTo(4096); + hashLinkedListMemTableConfig. + setBucketEntriesLoggingThreshold(200); + assertThat(hashLinkedListMemTableConfig. + bucketEntriesLoggingThreshold()). + isEqualTo(200); + assertThat(hashLinkedListMemTableConfig. + ifLogBucketDistWhenFlush()).isTrue(); + hashLinkedListMemTableConfig. + setIfLogBucketDistWhenFlush(false); + assertThat(hashLinkedListMemTableConfig. + ifLogBucketDistWhenFlush()).isFalse(); + assertThat(hashLinkedListMemTableConfig. + thresholdUseSkiplist()). + isEqualTo(256); + hashLinkedListMemTableConfig.setThresholdUseSkiplist(29); + assertThat(hashLinkedListMemTableConfig. + thresholdUseSkiplist()). + isEqualTo(29); + options.setMemTableConfig(hashLinkedListMemTableConfig); + } + } + + @Test + public void vectorMemTable() throws RocksDBException { + try(final Options options = new Options()) { + final VectorMemTableConfig vectorMemTableConfig = new VectorMemTableConfig(); + assertThat(vectorMemTableConfig.reservedSize()). + isEqualTo(0); + vectorMemTableConfig.setReservedSize(123); + assertThat(vectorMemTableConfig.reservedSize()). + isEqualTo(123); + options.setMemTableConfig(vectorMemTableConfig); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MemoryUtilTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MemoryUtilTest.java new file mode 100644 index 0000000..bfdcb9f --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MemoryUtilTest.java @@ -0,0 +1,136 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.nio.charset.StandardCharsets; +import java.util.*; + +import static org.assertj.core.api.Assertions.assertThat; + +public class MemoryUtilTest { + + private static final String MEMTABLE_SIZE = "rocksdb.size-all-mem-tables"; + private static final String UNFLUSHED_MEMTABLE_SIZE = "rocksdb.cur-size-all-mem-tables"; + private static final String TABLE_READERS = "rocksdb.estimate-table-readers-mem"; + + private final byte[] key = "some-key".getBytes(StandardCharsets.UTF_8); + private final byte[] value = "some-value".getBytes(StandardCharsets.UTF_8); + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule public TemporaryFolder dbFolder1 = new TemporaryFolder(); + @Rule public TemporaryFolder dbFolder2 = new TemporaryFolder(); + + /** + * Test MemoryUtil.getApproximateMemoryUsageByType before and after a put + get + */ + @Test + public void getApproximateMemoryUsageByType() throws RocksDBException { + try (final Cache cache = new LRUCache(8 * 1024 * 1024); + final Options options = + new Options() + .setCreateIfMissing(true) + .setTableFormatConfig(new BlockBasedTableConfig().setBlockCache(cache)); + final FlushOptions flushOptions = + new FlushOptions().setWaitForFlush(true); + final RocksDB db = + RocksDB.open(options, dbFolder1.getRoot().getAbsolutePath())) { + final List dbs = new ArrayList<>(1); + dbs.add(db); + final Set caches = new HashSet<>(1); + caches.add(cache); + Map usage = MemoryUtil.getApproximateMemoryUsageByType(dbs, caches); + + assertThat(usage.get(MemoryUsageType.kMemTableTotal)).isEqualTo( + db.getAggregatedLongProperty(MEMTABLE_SIZE)); + assertThat(usage.get(MemoryUsageType.kMemTableUnFlushed)).isEqualTo( + db.getAggregatedLongProperty(UNFLUSHED_MEMTABLE_SIZE)); + assertThat(usage.get(MemoryUsageType.kTableReadersTotal)).isEqualTo( + db.getAggregatedLongProperty(TABLE_READERS)); + // TODO(peterd): disable block cache entry stats and check for 0 + assertThat(usage.get(MemoryUsageType.kCacheTotal)).isLessThan(1024); + + db.put(key, value); + db.flush(flushOptions); + db.get(key); + + usage = MemoryUtil.getApproximateMemoryUsageByType(dbs, caches); + assertThat(usage.get(MemoryUsageType.kMemTableTotal)).isGreaterThan(0); + assertThat(usage.get(MemoryUsageType.kMemTableTotal)).isEqualTo( + db.getAggregatedLongProperty(MEMTABLE_SIZE)); + assertThat(usage.get(MemoryUsageType.kMemTableUnFlushed)).isGreaterThan(0); + assertThat(usage.get(MemoryUsageType.kMemTableUnFlushed)).isEqualTo( + db.getAggregatedLongProperty(UNFLUSHED_MEMTABLE_SIZE)); + assertThat(usage.get(MemoryUsageType.kTableReadersTotal)).isGreaterThan(0); + assertThat(usage.get(MemoryUsageType.kTableReadersTotal)).isEqualTo( + db.getAggregatedLongProperty(TABLE_READERS)); + assertThat(usage.get(MemoryUsageType.kCacheTotal)).isGreaterThan(0); + + } + } + + /** + * Test MemoryUtil.getApproximateMemoryUsageByType with null inputs + */ + @Test + public void getApproximateMemoryUsageByTypeNulls() throws RocksDBException { + final Map usage = MemoryUtil.getApproximateMemoryUsageByType(null, null); + + assertThat(usage.get(MemoryUsageType.kMemTableTotal)).isEqualTo(null); + assertThat(usage.get(MemoryUsageType.kMemTableUnFlushed)).isEqualTo(null); + assertThat(usage.get(MemoryUsageType.kTableReadersTotal)).isEqualTo(null); + assertThat(usage.get(MemoryUsageType.kCacheTotal)).isEqualTo(null); + } + + /** + * Test MemoryUtil.getApproximateMemoryUsageByType with two DBs and two caches + */ + @Test + public void getApproximateMemoryUsageByTypeMultiple() throws RocksDBException { + try (final Cache cache1 = new LRUCache(1024 * 1024); + final Options options1 = new Options().setCreateIfMissing(true).setTableFormatConfig( + new BlockBasedTableConfig().setBlockCache(cache1)); + final RocksDB db1 = RocksDB.open(options1, dbFolder1.getRoot().getAbsolutePath()); + final Cache cache2 = new LRUCache(1024 * 1024); + final Options options2 = new Options().setCreateIfMissing(true).setTableFormatConfig( + new BlockBasedTableConfig().setBlockCache(cache2)); + final RocksDB db2 = RocksDB.open(options2, dbFolder2.getRoot().getAbsolutePath()); + final FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true) + + ) { + final List dbs = new ArrayList<>(1); + dbs.add(db1); + dbs.add(db2); + final Set caches = new HashSet<>(1); + caches.add(cache1); + caches.add(cache2); + + for (final RocksDB db : dbs) { + db.put(key, value); + db.flush(flushOptions); + db.get(key); + } + + final Map usage = + MemoryUtil.getApproximateMemoryUsageByType(dbs, caches); + assertThat(usage.get(MemoryUsageType.kMemTableTotal)).isEqualTo( + db1.getAggregatedLongProperty(MEMTABLE_SIZE) + db2.getAggregatedLongProperty(MEMTABLE_SIZE)); + assertThat(usage.get(MemoryUsageType.kMemTableUnFlushed)).isEqualTo( + db1.getAggregatedLongProperty(UNFLUSHED_MEMTABLE_SIZE) + db2.getAggregatedLongProperty(UNFLUSHED_MEMTABLE_SIZE)); + assertThat(usage.get(MemoryUsageType.kTableReadersTotal)).isEqualTo( + db1.getAggregatedLongProperty(TABLE_READERS) + db2.getAggregatedLongProperty(TABLE_READERS)); + assertThat(usage.get(MemoryUsageType.kCacheTotal)).isGreaterThan(0); + } + } + +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MergeTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MergeTest.java new file mode 100644 index 0000000..f99ac49 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MergeTest.java @@ -0,0 +1,465 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class MergeTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void stringOption() + throws InterruptedException, RocksDBException { + try (final Options opt = new Options() + .setCreateIfMissing(true) + .setMergeOperatorName("stringappend"); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + // writing aa under key + db.put("key".getBytes(), "aa".getBytes()); + // merge bb under key + db.merge("key".getBytes(), "bb".getBytes()); + + final byte[] value = db.get("key".getBytes()); + final String strValue = new String(value); + assertThat(strValue).isEqualTo("aa,bb"); + } + } + + private byte[] longToByteArray(final long l) { + final ByteBuffer buf = + ByteBuffer.allocate(Long.SIZE / Byte.SIZE).order(ByteOrder.LITTLE_ENDIAN); + buf.putLong(l); + return buf.array(); + } + + private long longFromByteArray(final byte[] a) { + final ByteBuffer buf = + ByteBuffer.allocate(Long.SIZE / Byte.SIZE).order(ByteOrder.LITTLE_ENDIAN); + buf.put(a); + buf.flip(); + return buf.getLong(); + } + + @Test + public void uint64AddOption() + throws InterruptedException, RocksDBException { + try (final Options opt = new Options() + .setCreateIfMissing(true) + .setMergeOperatorName("uint64add"); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + // writing (long)100 under key + db.put("key".getBytes(), longToByteArray(100)); + // merge (long)1 under key + db.merge("key".getBytes(), longToByteArray(1)); + + final byte[] value = db.get("key".getBytes()); + final long longValue = longFromByteArray(value); + assertThat(longValue).isEqualTo(101); + } + } + + @Test + public void cFStringOption() + throws InterruptedException, RocksDBException { + + try (final ColumnFamilyOptions cfOpt1 = new ColumnFamilyOptions() + .setMergeOperatorName("stringappend"); + final ColumnFamilyOptions cfOpt2 = new ColumnFamilyOptions() + .setMergeOperatorName("stringappend") + ) { + final List cfDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpt1), + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpt2) + ); + + final List columnFamilyHandleList = new ArrayList<>(); + try (final DBOptions opt = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath(), cfDescriptors, + columnFamilyHandleList)) { + try { + // writing aa under key + db.put(columnFamilyHandleList.get(1), + "cfkey".getBytes(), "aa".getBytes()); + // merge bb under key + db.merge(columnFamilyHandleList.get(1), + "cfkey".getBytes(), "bb".getBytes()); + + final byte[] value = db.get(columnFamilyHandleList.get(1), "cfkey".getBytes()); + final String strValue = new String(value); + assertThat(strValue).isEqualTo("aa,bb"); + } finally { + for (final ColumnFamilyHandle handle : columnFamilyHandleList) { + handle.close(); + } + } + } + } + } + + @Test + public void cFUInt64AddOption() + throws InterruptedException, RocksDBException { + + try (final ColumnFamilyOptions cfOpt1 = new ColumnFamilyOptions() + .setMergeOperatorName("uint64add"); + final ColumnFamilyOptions cfOpt2 = new ColumnFamilyOptions() + .setMergeOperatorName("uint64add") + ) { + final List cfDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpt1), + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpt2) + ); + + final List columnFamilyHandleList = new ArrayList<>(); + try (final DBOptions opt = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath(), cfDescriptors, + columnFamilyHandleList)) { + try { + // writing (long)100 under key + db.put(columnFamilyHandleList.get(1), + "cfkey".getBytes(), longToByteArray(100)); + // merge (long)157 under key + db.merge(columnFamilyHandleList.get(1), "cfkey".getBytes(), longToByteArray(157)); + + final byte[] value = db.get(columnFamilyHandleList.get(1), "cfkey".getBytes()); + final long longValue = longFromByteArray(value); + assertThat(longValue).isEqualTo(257); + } finally { + for (final ColumnFamilyHandle handle : columnFamilyHandleList) { + handle.close(); + } + } + } + } + } + + @Test + public void operatorOption() + throws InterruptedException, RocksDBException { + try (final StringAppendOperator stringAppendOperator = new StringAppendOperator(); + final Options opt = new Options() + .setCreateIfMissing(true) + .setMergeOperator(stringAppendOperator); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + // Writing aa under key + db.put("key".getBytes(), "aa".getBytes()); + + // Writing bb under key + db.merge("key".getBytes(), "bb".getBytes()); + + final byte[] value = db.get("key".getBytes()); + final String strValue = new String(value); + + assertThat(strValue).isEqualTo("aa,bb"); + } + } + + @Test + public void uint64AddOperatorOption() + throws InterruptedException, RocksDBException { + try (final UInt64AddOperator uint64AddOperator = new UInt64AddOperator(); + final Options opt = new Options() + .setCreateIfMissing(true) + .setMergeOperator(uint64AddOperator); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + // Writing (long)100 under key + db.put("key".getBytes(), longToByteArray(100)); + + // Writing (long)1 under key + db.merge("key".getBytes(), longToByteArray(1)); + + final byte[] value = db.get("key".getBytes()); + final long longValue = longFromByteArray(value); + + assertThat(longValue).isEqualTo(101); + } + } + + @Test + public void cFOperatorOption() + throws InterruptedException, RocksDBException { + try (final StringAppendOperator stringAppendOperator = new StringAppendOperator(); + final ColumnFamilyOptions cfOpt1 = new ColumnFamilyOptions() + .setMergeOperator(stringAppendOperator); + final ColumnFamilyOptions cfOpt2 = new ColumnFamilyOptions() + .setMergeOperator(stringAppendOperator) + ) { + final List cfDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpt1), + new ColumnFamilyDescriptor("new_cf".getBytes(), cfOpt2) + ); + final List columnFamilyHandleList = new ArrayList<>(); + try (final DBOptions opt = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath(), cfDescriptors, + columnFamilyHandleList) + ) { + try { + // writing aa under key + db.put(columnFamilyHandleList.get(1), + "cfkey".getBytes(), "aa".getBytes()); + // merge bb under key + db.merge(columnFamilyHandleList.get(1), + "cfkey".getBytes(), "bb".getBytes()); + byte[] value = db.get(columnFamilyHandleList.get(1), + "cfkey".getBytes()); + final String strValue = new String(value); + + // Test also with createColumnFamily + try (final ColumnFamilyOptions cfHandleOpts = + new ColumnFamilyOptions() + .setMergeOperator(stringAppendOperator); + final ColumnFamilyHandle cfHandle = + db.createColumnFamily( + new ColumnFamilyDescriptor("new_cf2".getBytes(), + cfHandleOpts)) + ) { + // writing xx under cfkey2 + db.put(cfHandle, "cfkey2".getBytes(), "xx".getBytes()); + // merge yy under cfkey2 + db.merge(cfHandle, new WriteOptions(), "cfkey2".getBytes(), + "yy".getBytes()); + value = db.get(cfHandle, "cfkey2".getBytes()); + final String strValueTmpCf = new String(value); + + assertThat(strValue).isEqualTo("aa,bb"); + assertThat(strValueTmpCf).isEqualTo("xx,yy"); + } + } finally { + for (final ColumnFamilyHandle columnFamilyHandle : + columnFamilyHandleList) { + columnFamilyHandle.close(); + } + } + } + } + } + + @Test + public void cFUInt64AddOperatorOption() + throws InterruptedException, RocksDBException { + try (final UInt64AddOperator uint64AddOperator = new UInt64AddOperator(); + final ColumnFamilyOptions cfOpt1 = new ColumnFamilyOptions() + .setMergeOperator(uint64AddOperator); + final ColumnFamilyOptions cfOpt2 = new ColumnFamilyOptions() + .setMergeOperator(uint64AddOperator) + ) { + final List cfDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpt1), + new ColumnFamilyDescriptor("new_cf".getBytes(), cfOpt2) + ); + final List columnFamilyHandleList = new ArrayList<>(); + try (final DBOptions opt = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath(), cfDescriptors, + columnFamilyHandleList) + ) { + try { + // writing (long)100 under key + db.put(columnFamilyHandleList.get(1), + "cfkey".getBytes(), longToByteArray(100)); + // merge (long)1 under key + db.merge(columnFamilyHandleList.get(1), + "cfkey".getBytes(), longToByteArray(1)); + byte[] value = db.get(columnFamilyHandleList.get(1), + "cfkey".getBytes()); + final long longValue = longFromByteArray(value); + + // Test also with createColumnFamily + try (final ColumnFamilyOptions cfHandleOpts = + new ColumnFamilyOptions() + .setMergeOperator(uint64AddOperator); + final ColumnFamilyHandle cfHandle = + db.createColumnFamily( + new ColumnFamilyDescriptor("new_cf2".getBytes(), + cfHandleOpts)) + ) { + // writing (long)200 under cfkey2 + db.put(cfHandle, "cfkey2".getBytes(), longToByteArray(200)); + // merge (long)50 under cfkey2 + db.merge(cfHandle, new WriteOptions(), "cfkey2".getBytes(), + longToByteArray(50)); + value = db.get(cfHandle, "cfkey2".getBytes()); + final long longValueTmpCf = longFromByteArray(value); + + assertThat(longValue).isEqualTo(101); + assertThat(longValueTmpCf).isEqualTo(250); + } + } finally { + for (final ColumnFamilyHandle columnFamilyHandle : + columnFamilyHandleList) { + columnFamilyHandle.close(); + } + } + } + } + } + + @Test + public void operatorGcBehaviour() + throws RocksDBException { + try (final StringAppendOperator stringAppendOperator = new StringAppendOperator()) { + try (final Options opt = new Options() + .setCreateIfMissing(true) + .setMergeOperator(stringAppendOperator); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + //no-op + } + + // test reuse + try (final Options opt = new Options() + .setMergeOperator(stringAppendOperator); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + //no-op + } + + // test param init + try (final StringAppendOperator stringAppendOperator2 = new StringAppendOperator(); + final Options opt = new Options() + .setMergeOperator(stringAppendOperator2); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + //no-op + } + + // test replace one with another merge operator instance + try (final Options opt = new Options() + .setMergeOperator(stringAppendOperator); + final StringAppendOperator newStringAppendOperator = new StringAppendOperator()) { + opt.setMergeOperator(newStringAppendOperator); + try (final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + //no-op + } + } + } + } + + @Test + public void uint64AddOperatorGcBehaviour() + throws RocksDBException { + try (final UInt64AddOperator uint64AddOperator = new UInt64AddOperator()) { + try (final Options opt = new Options() + .setCreateIfMissing(true) + .setMergeOperator(uint64AddOperator); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + //no-op + } + + // test reuse + try (final Options opt = new Options() + .setMergeOperator(uint64AddOperator); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + //no-op + } + + // test param init + try (final UInt64AddOperator uint64AddOperator2 = new UInt64AddOperator(); + final Options opt = new Options() + .setMergeOperator(uint64AddOperator2); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + //no-op + } + + // test replace one with another merge operator instance + try (final Options opt = new Options() + .setMergeOperator(uint64AddOperator); + final UInt64AddOperator newUInt64AddOperator = new UInt64AddOperator()) { + opt.setMergeOperator(newUInt64AddOperator); + try (final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + //no-op + } + } + } + } + + @Test + public void emptyStringAsStringAppendDelimiter() throws RocksDBException { + try (final StringAppendOperator stringAppendOperator = new StringAppendOperator(""); + final Options opt = + new Options().setCreateIfMissing(true).setMergeOperator(stringAppendOperator); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + db.put("key".getBytes(), "aa".getBytes()); + db.merge("key".getBytes(), "bb".getBytes()); + final byte[] value = db.get("key".getBytes()); + assertThat(new String(value)).isEqualTo("aabb"); + } + } + + @Test + public void multiCharStringAsStringAppendDelimiter() throws RocksDBException { + try (final StringAppendOperator stringAppendOperator = new StringAppendOperator("<>"); + final Options opt = + new Options().setCreateIfMissing(true).setMergeOperator(stringAppendOperator); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + db.put("key".getBytes(), "aa".getBytes()); + db.merge("key".getBytes(), "bb".getBytes()); + final byte[] value = db.get("key".getBytes()); + assertThat(new String(value)).isEqualTo("aa<>bb"); + } + } + + @Test + public void emptyStringInSetMergeOperatorByName() { + try (final Options opt = new Options() + .setMergeOperatorName(""); + final ColumnFamilyOptions cOpt = new ColumnFamilyOptions() + .setMergeOperatorName("")) { + //no-op + } + } + + @Test(expected = IllegalArgumentException.class) + public void nullStringInSetMergeOperatorByNameOptions() { + try (final Options opt = new Options()) { + opt.setMergeOperatorName(null); + } + } + + @Test(expected = IllegalArgumentException.class) + public void + nullStringInSetMergeOperatorByNameColumnFamilyOptions() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + opt.setMergeOperatorName(null); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MixedOptionsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MixedOptionsTest.java new file mode 100644 index 0000000..4e17d04 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MixedOptionsTest.java @@ -0,0 +1,85 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class MixedOptionsTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Test + public void mixedOptionsTest(){ + // Set a table factory and check the names + try(final Filter bloomFilter = new BloomFilter(); + final ColumnFamilyOptions cfOptions = new ColumnFamilyOptions() + .setTableFormatConfig( + new BlockBasedTableConfig().setFilterPolicy(bloomFilter)) + ) { + assertThat(cfOptions.tableFactoryName()).isEqualTo( + "BlockBasedTable"); + cfOptions.setTableFormatConfig(new PlainTableConfig()); + assertThat(cfOptions.tableFactoryName()).isEqualTo("PlainTable"); + // Initialize a dbOptions object from cf options and + // db options + try (final DBOptions dbOptions = new DBOptions(); + final Options options = new Options(dbOptions, cfOptions)) { + assertThat(options.tableFactoryName()).isEqualTo("PlainTable"); + // Free instances + } + } + + // Test Optimize for statements + try(final ColumnFamilyOptions cfOptions = new ColumnFamilyOptions()) { + cfOptions.optimizeUniversalStyleCompaction(); + cfOptions.optimizeLevelStyleCompaction(); + cfOptions.optimizeForPointLookup(1024); + try(final Options options = new Options()) { + options.optimizeLevelStyleCompaction(); + options.optimizeLevelStyleCompaction(400); + options.optimizeUniversalStyleCompaction(); + options.optimizeUniversalStyleCompaction(400); + options.optimizeForPointLookup(1024); + options.prepareForBulkLoad(); + } + } + } + + @Test + public void mixedOptionsEnvTest() { + try (final ColumnFamilyOptions cfOptions = new ColumnFamilyOptions(); + final DBOptions dbOptions = new DBOptions()) { + assertThat(dbOptions.getEnv()).isNotNull(); + assertThat(dbOptions.getEnv()).isSameAs(Env.getDefault()); + final Env memEnv = new RocksMemEnv(Env.getDefault()); + + try (final Options options = new Options(dbOptions, cfOptions)) { + assertThat(options.getEnv()).isSameAs(Env.getDefault()); + } + + dbOptions.setEnv(memEnv); + memEnv.setBackgroundThreads(4, Priority.LOW); + Env.getDefault().setBackgroundThreads(2, Priority.HIGH); + assertThat(dbOptions.getEnv().getBackgroundThreads(Priority.LOW)).isEqualTo(4); + assertThat(dbOptions.getEnv().getBackgroundThreads(Priority.HIGH)).isEqualTo(2); + assertThat(Env.getDefault().getBackgroundThreads(Priority.LOW)).isEqualTo(4); + assertThat(Env.getDefault().getBackgroundThreads(Priority.HIGH)).isEqualTo(2); + + try (final Options options = new Options(dbOptions, cfOptions)) { + assertThat(options.getEnv().getBackgroundThreads(Priority.LOW)).isEqualTo(4); + assertThat(options.getEnv().getBackgroundThreads(Priority.HIGH)).isEqualTo(2); + + assertThat(options.getEnv()).isNotSameAs(Env.getDefault()); + assertThat(options.getEnv()).isSameAs(memEnv); + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MultiColumnRegressionTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MultiColumnRegressionTest.java new file mode 100644 index 0000000..6087b02 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MultiColumnRegressionTest.java @@ -0,0 +1,145 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +/** + * Test for changes made by + * transactional multiGet problem + * the tests here were previously broken by the nonsense removed by that change. + */ +@RunWith(Parameterized.class) +public class MultiColumnRegressionTest { + @Parameterized.Parameters + public static List data() { + return Arrays.asList(new Params(3, 100), new Params(3, 1000000)); + } + + public static class Params { + final int numColumns; + final int keySize; + + public Params(final int numColumns, final int keySize) { + this.numColumns = numColumns; + this.keySize = keySize; + } + } + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + private final Params params; + + public MultiColumnRegressionTest(final Params params) { + this.params = params; + } + + @Test + public void transactionDB() throws RocksDBException { + final List columnFamilyDescriptors = new ArrayList<>(); + for (int i = 0; i < params.numColumns; i++) { + final StringBuilder sb = new StringBuilder(); + sb.append("cf" + i); + for (int j = 0; j < params.keySize; j++) sb.append("_cf"); + columnFamilyDescriptors.add(new ColumnFamilyDescriptor(sb.toString().getBytes())); + } + try (final Options opt = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + db.createColumnFamilies(columnFamilyDescriptors); + } + + columnFamilyDescriptors.add(new ColumnFamilyDescriptor("default".getBytes())); + final List columnFamilyHandles = new ArrayList<>(); + try (final TransactionDB tdb = TransactionDB.open(new DBOptions().setCreateIfMissing(true), + new TransactionDBOptions(), dbFolder.getRoot().getAbsolutePath(), + columnFamilyDescriptors, columnFamilyHandles)) { + final WriteOptions writeOptions = new WriteOptions(); + try (final Transaction transaction = tdb.beginTransaction(writeOptions)) { + for (int i = 0; i < params.numColumns; i++) { + transaction.put( + columnFamilyHandles.get(i), ("key" + i).getBytes(), ("value" + (i - 7)).getBytes()); + } + transaction.put("key".getBytes(), "value".getBytes()); + transaction.commit(); + } + for (final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandles) { + columnFamilyHandle.close(); + } + } + + final List columnFamilyHandles2 = new ArrayList<>(); + try (final TransactionDB tdb = TransactionDB.open(new DBOptions().setCreateIfMissing(true), + new TransactionDBOptions(), dbFolder.getRoot().getAbsolutePath(), + columnFamilyDescriptors, columnFamilyHandles2)) { + try (final Transaction transaction = tdb.beginTransaction(new WriteOptions())) { + final ReadOptions readOptions = new ReadOptions(); + for (int i = 0; i < params.numColumns; i++) { + final byte[] value = + transaction.get(columnFamilyHandles2.get(i), readOptions, ("key" + i).getBytes()); + assertThat(value).isEqualTo(("value" + (i - 7)).getBytes()); + } + transaction.commit(); + } + for (final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandles2) { + columnFamilyHandle.close(); + } + } + } + + @Test + public void optimisticDB() throws RocksDBException { + final List columnFamilyDescriptors = new ArrayList<>(); + for (int i = 0; i < params.numColumns; i++) { + columnFamilyDescriptors.add(new ColumnFamilyDescriptor("default".getBytes())); + } + + columnFamilyDescriptors.add(new ColumnFamilyDescriptor("default".getBytes())); + final List columnFamilyHandles = new ArrayList<>(); + try (final OptimisticTransactionDB otdb = OptimisticTransactionDB.open( + new DBOptions().setCreateIfMissing(true), dbFolder.getRoot().getAbsolutePath(), + columnFamilyDescriptors, columnFamilyHandles)) { + try (final Transaction transaction = otdb.beginTransaction(new WriteOptions())) { + for (int i = 0; i < params.numColumns; i++) { + transaction.put( + columnFamilyHandles.get(i), ("key" + i).getBytes(), ("value" + (i - 7)).getBytes()); + } + transaction.put("key".getBytes(), "value".getBytes()); + transaction.commit(); + } + for (final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandles) { + columnFamilyHandle.close(); + } + } + + final List columnFamilyHandles2 = new ArrayList<>(); + try (final OptimisticTransactionDB otdb = OptimisticTransactionDB.open( + new DBOptions().setCreateIfMissing(true), dbFolder.getRoot().getAbsolutePath(), + columnFamilyDescriptors, columnFamilyHandles2)) { + try (final Transaction transaction = otdb.beginTransaction(new WriteOptions())) { + final ReadOptions readOptions = new ReadOptions(); + for (int i = 0; i < params.numColumns; i++) { + final byte[] value = + transaction.get(columnFamilyHandles2.get(i), readOptions, ("key" + i).getBytes()); + assertThat(value).isEqualTo(("value" + (i - 7)).getBytes()); + } + transaction.commit(); + } + for (final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandles2) { + columnFamilyHandle.close(); + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MultiGetManyKeysTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MultiGetManyKeysTest.java new file mode 100644 index 0000000..e66eef6 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MultiGetManyKeysTest.java @@ -0,0 +1,241 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.*; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class MultiGetManyKeysTest { + @Parameterized.Parameters + public static List data() { + return Arrays.asList(2, 3, 250, 60000, 70000, 150000, 750000); + } + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + private final int numKeys; + + public MultiGetManyKeysTest(final Integer numKeys) { + this.numKeys = numKeys; + } + + /** + * Test for multiGet problem + */ + @Test + public void multiGetAsListLarge() throws RocksDBException { + final List keys = generateRandomKeys(numKeys); + final Map keyValues = generateRandomKeyValues(keys, 10); + putKeysAndValues(keyValues); + + try (final Options opt = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + final List values = db.multiGetAsList(keys); + assertKeysAndValues(keys, keyValues, values); + } + } + + /** + * Test for transactional multiGet + * problem + */ + @Test + public void multiGetAsListLargeTransactional() throws RocksDBException { + final List keys = generateRandomKeys(numKeys); + final Map keyValues = generateRandomKeyValues(keys, 10); + putKeysAndValues(keyValues); + + try (final Options options = new Options().setCreateIfMissing(true); + final TransactionDBOptions txnDbOptions = new TransactionDBOptions(); + final TransactionDB txnDB = + TransactionDB.open(options, txnDbOptions, dbFolder.getRoot().getAbsolutePath())) { + try (final Transaction transaction = txnDB.beginTransaction(new WriteOptions())) { + final List values = transaction.multiGetAsList(new ReadOptions(), keys); + assertKeysAndValues(keys, keyValues, values); + } + } + } + + /** + * Test for transactional multiGet + * problem + */ + @Test + public void multiGetForUpdateAsListLargeTransactional() throws RocksDBException { + final List keys = generateRandomKeys(numKeys); + final Map keyValues = generateRandomKeyValues(keys, 10); + putKeysAndValues(keyValues); + + try (final Options options = new Options().setCreateIfMissing(true); + final TransactionDBOptions txnDbOptions = new TransactionDBOptions(); + final TransactionDB txnDB = + TransactionDB.open(options, txnDbOptions, dbFolder.getRoot().getAbsolutePath())) { + try (final Transaction transaction = txnDB.beginTransaction(new WriteOptions())) { + final List values = transaction.multiGetForUpdateAsList(new ReadOptions(), keys); + assertKeysAndValues(keys, keyValues, values); + } + } + } + + /** + * Test for transactional multiGet + * problem + */ + @Test + public void multiGetAsListLargeTransactionalCF() throws RocksDBException { + final List keys = generateRandomKeys(numKeys); + final Map keyValues = generateRandomKeyValues(keys, 10); + final ColumnFamilyDescriptor columnFamilyDescriptor = + new ColumnFamilyDescriptor("cfTest".getBytes()); + putKeysAndValues(columnFamilyDescriptor, keyValues); + + final List columnFamilyDescriptors = new ArrayList<>(); + columnFamilyDescriptors.add(columnFamilyDescriptor); + columnFamilyDescriptors.add(new ColumnFamilyDescriptor("default".getBytes())); + final List columnFamilyHandles = new ArrayList<>(); + try (final Options options = new Options().setCreateIfMissing(true); + final TransactionDBOptions txnDbOptions = new TransactionDBOptions(); + final TransactionDB txnDB = TransactionDB.open(new DBOptions(options), txnDbOptions, + dbFolder.getRoot().getAbsolutePath(), columnFamilyDescriptors, columnFamilyHandles)) { + final List columnFamilyHandlesForMultiGet = new ArrayList<>(numKeys); + for (int i = 0; i < numKeys; i++) + columnFamilyHandlesForMultiGet.add(columnFamilyHandles.get(0)); + try (final Transaction transaction = txnDB.beginTransaction(new WriteOptions())) { + final List values = + transaction.multiGetAsList(new ReadOptions(), columnFamilyHandlesForMultiGet, keys); + assertKeysAndValues(keys, keyValues, values); + } + for (final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandles) { + columnFamilyHandle.close(); + } + } + } + + /** + * Test for transactional multiGet + * problem + */ + @Test + public void multiGetForUpdateAsListLargeTransactionalCF() throws RocksDBException { + final List keys = generateRandomKeys(numKeys); + final Map keyValues = generateRandomKeyValues(keys, 10); + final ColumnFamilyDescriptor columnFamilyDescriptor = + new ColumnFamilyDescriptor("cfTest".getBytes()); + putKeysAndValues(columnFamilyDescriptor, keyValues); + + final List columnFamilyDescriptors = new ArrayList<>(); + columnFamilyDescriptors.add(columnFamilyDescriptor); + columnFamilyDescriptors.add(new ColumnFamilyDescriptor("default".getBytes())); + final List columnFamilyHandles = new ArrayList<>(); + try (final Options options = new Options().setCreateIfMissing(true); + final TransactionDBOptions txnDbOptions = new TransactionDBOptions(); + final TransactionDB txnDB = TransactionDB.open(new DBOptions(options), txnDbOptions, + dbFolder.getRoot().getAbsolutePath(), columnFamilyDescriptors, columnFamilyHandles)) { + final List columnFamilyHandlesForMultiGet = new ArrayList<>(numKeys); + for (int i = 0; i < numKeys; i++) + columnFamilyHandlesForMultiGet.add(columnFamilyHandles.get(0)); + try (final Transaction transaction = txnDB.beginTransaction(new WriteOptions())) { + final List values = transaction.multiGetForUpdateAsList( + new ReadOptions(), columnFamilyHandlesForMultiGet, keys); + assertKeysAndValues(keys, keyValues, values); + } + for (final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandles) { + columnFamilyHandle.close(); + } + } + } + + private List generateRandomKeys(final int numKeys) { + final Random rand = new Random(); + final List keys = new ArrayList<>(); + for (int i = 0; i < numKeys; i++) { + final byte[] key = new byte[4]; + rand.nextBytes(key); + keys.add(key); + } + return keys; + } + + private Map generateRandomKeyValues(final List keys, final int percent) { + final Random rand = new Random(); + final Map keyValues = new HashMap<>(); + for (int i = 0; i < numKeys; i++) { + if (rand.nextInt(100) < percent) { + final byte[] value = new byte[1024]; + rand.nextBytes(value); + keyValues.put(new Key(keys.get(i)), value); + } + } + return keyValues; + } + + private void putKeysAndValues(final Map keyValues) throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + for (final Map.Entry keyValue : keyValues.entrySet()) { + db.put(keyValue.getKey().get(), keyValue.getValue()); + } + } + } + + private void putKeysAndValues(final ColumnFamilyDescriptor columnFamilyDescriptor, + final Map keyValues) throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath()); + final ColumnFamilyHandle columnFamilyHandle = + db.createColumnFamily(columnFamilyDescriptor)) { + for (final Map.Entry keyValue : keyValues.entrySet()) { + db.put(columnFamilyHandle, keyValue.getKey().get(), keyValue.getValue()); + } + } + } + + private void assertKeysAndValues( + final List keys, final Map keyValues, final List values) { + assertThat(values.size()).isEqualTo(keys.size()); + for (int i = 0; i < numKeys; i++) { + final Key key = new Key(keys.get(i)); + final byte[] value = values.get(i); + if (keyValues.containsKey(key)) { + assertThat(value).isEqualTo(keyValues.get(key)); + } else { + assertThat(value).isNull(); + } + } + } + + private static class Key { + private final byte[] bytes; + public Key(final byte[] bytes) { + this.bytes = bytes; + } + + public byte[] get() { + return this.bytes; + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final Key key = (Key) o; + return Arrays.equals(bytes, key.bytes); + } + + @Override + public int hashCode() { + return Arrays.hashCode(bytes); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MultiGetTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MultiGetTest.java new file mode 100644 index 0000000..c391d81 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MultiGetTest.java @@ -0,0 +1,530 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.util.TestUtil; + +public class MultiGetTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void putNThenMultiGet() throws RocksDBException { + try (final Options opt = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + db.put("key1".getBytes(), "value1ForKey1".getBytes()); + db.put("key2".getBytes(), "value2ForKey2".getBytes()); + db.put("key3".getBytes(), "value3ForKey3".getBytes()); + final List keys = + Arrays.asList("key1".getBytes(), "key2".getBytes(), "key3".getBytes()); + final List values = db.multiGetAsList(keys); + assertThat(values.size()).isEqualTo(keys.size()); + assertThat(values.get(0)).isEqualTo("value1ForKey1".getBytes()); + assertThat(values.get(1)).isEqualTo("value2ForKey2".getBytes()); + assertThat(values.get(2)).isEqualTo("value3ForKey3".getBytes()); + } + } + + @Test + public void putNThenMultiGetDirect() throws RocksDBException { + try (final Options opt = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + db.put("key1".getBytes(), "value1ForKey1".getBytes()); + db.put("key2".getBytes(), "value2ForKey2".getBytes()); + db.put("key3".getBytes(), "value3ForKey3".getBytes()); + + final List keys = new ArrayList<>(); + keys.add(ByteBuffer.allocateDirect(12).put("key1".getBytes())); + keys.add(ByteBuffer.allocateDirect(12).put("key2".getBytes())); + keys.add(ByteBuffer.allocateDirect(12).put("key3".getBytes())); + // Java8 and lower flip() returns Buffer not ByteBuffer, so can't chain above /\/\ + for (final ByteBuffer key : keys) { + key.flip(); + } + final List values = new ArrayList<>(); + for (int i = 0; i < keys.size(); i++) { + values.add(ByteBuffer.allocateDirect(24)); + } + + { + final List results = db.multiGetByteBuffers(keys, values); + + assertThat(results.get(0).status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(results.get(1).status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(results.get(2).status.getCode()).isEqualTo(Status.Code.Ok); + + assertThat(results.get(0).requiredSize).isEqualTo("value1ForKey1".getBytes().length); + assertThat(results.get(1).requiredSize).isEqualTo("value2ForKey2".getBytes().length); + assertThat(results.get(2).requiredSize).isEqualTo("value3ForKey3".getBytes().length); + + assertThat(TestUtil.bufferBytes(results.get(0).value)) + .isEqualTo("value1ForKey1".getBytes()); + assertThat(TestUtil.bufferBytes(results.get(1).value)) + .isEqualTo("value2ForKey2".getBytes()); + assertThat(TestUtil.bufferBytes(results.get(2).value)) + .isEqualTo("value3ForKey3".getBytes()); + } + + { + final List results = + db.multiGetByteBuffers(new ReadOptions(), keys, values); + + assertThat(results.get(0).status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(results.get(1).status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(results.get(2).status.getCode()).isEqualTo(Status.Code.Ok); + + assertThat(results.get(0).requiredSize).isEqualTo("value1ForKey1".getBytes().length); + assertThat(results.get(1).requiredSize).isEqualTo("value2ForKey2".getBytes().length); + assertThat(results.get(2).requiredSize).isEqualTo("value3ForKey3".getBytes().length); + + assertThat(TestUtil.bufferBytes(results.get(0).value)) + .isEqualTo("value1ForKey1".getBytes()); + assertThat(TestUtil.bufferBytes(results.get(1).value)) + .isEqualTo("value2ForKey2".getBytes()); + assertThat(TestUtil.bufferBytes(results.get(2).value)) + .isEqualTo("value3ForKey3".getBytes()); + } + } + } + + @Test + public void putNThenMultiGetDirectSliced() throws RocksDBException { + try (final Options opt = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + db.put("key1".getBytes(), "value1ForKey1".getBytes()); + db.put("key2".getBytes(), "value2ForKey2".getBytes()); + db.put("key3".getBytes(), "value3ForKey3".getBytes()); + + final List keys = new ArrayList<>(); + keys.add(ByteBuffer.allocateDirect(12).put("key2".getBytes())); + keys.add(ByteBuffer.allocateDirect(12).put("key3".getBytes())); + keys.add( + ByteBuffer.allocateDirect(12).put("prefix1".getBytes()).slice().put("key1".getBytes())); + // Java8 and lower flip() returns Buffer not ByteBuffer, so can't chain above /\/\ + for (final ByteBuffer key : keys) { + key.flip(); + } + final List values = new ArrayList<>(); + for (int i = 0; i < keys.size(); i++) { + values.add(ByteBuffer.allocateDirect(24)); + } + + { + final List results = db.multiGetByteBuffers(keys, values); + + assertThat(results.get(0).status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(results.get(1).status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(results.get(2).status.getCode()).isEqualTo(Status.Code.Ok); + + assertThat(results.get(1).requiredSize).isEqualTo("value2ForKey2".getBytes().length); + assertThat(results.get(2).requiredSize).isEqualTo("value3ForKey3".getBytes().length); + assertThat(results.get(0).requiredSize).isEqualTo("value1ForKey1".getBytes().length); + + assertThat(TestUtil.bufferBytes(results.get(0).value)) + .isEqualTo("value2ForKey2".getBytes()); + assertThat(TestUtil.bufferBytes(results.get(1).value)) + .isEqualTo("value3ForKey3".getBytes()); + assertThat(TestUtil.bufferBytes(results.get(2).value)) + .isEqualTo("value1ForKey1".getBytes()); + } + } + } + + @Test + public void putNThenMultiGetDirectBadValuesArray() throws RocksDBException { + try (final Options opt = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + db.put("key1".getBytes(), "value1ForKey1".getBytes()); + db.put("key2".getBytes(), "value2ForKey2".getBytes()); + db.put("key3".getBytes(), "value3ForKey3".getBytes()); + + final List keys = new ArrayList<>(); + keys.add(ByteBuffer.allocateDirect(12).put("key1".getBytes())); + keys.add(ByteBuffer.allocateDirect(12).put("key2".getBytes())); + keys.add(ByteBuffer.allocateDirect(12).put("key3".getBytes())); + // Java8 and lower flip() returns Buffer not ByteBuffer, so can't chain above /\/\ + for (final ByteBuffer key : keys) { + key.flip(); + } + + { + final List values = new ArrayList<>(); + for (int i = 0; i < keys.size(); i++) { + values.add(ByteBuffer.allocateDirect(24)); + } + + values.remove(0); + + try { + db.multiGetByteBuffers(keys, values); + fail("Expected exception when not enough value ByteBuffers supplied"); + } catch (final IllegalArgumentException e) { + assertThat(e.getMessage()).contains("For each key there must be a corresponding value"); + } + } + + { + final List values = new ArrayList<>(); + for (int i = 0; i < keys.size(); i++) { + values.add(ByteBuffer.allocateDirect(24)); + } + + values.add(ByteBuffer.allocateDirect(24)); + + try { + db.multiGetByteBuffers(keys, values); + fail("Expected exception when too many value ByteBuffers supplied"); + } catch (final IllegalArgumentException e) { + assertThat(e.getMessage()).contains("For each key there must be a corresponding value"); + } + } + } + } + + @Test + public void putNThenMultiGetDirectShortValueBuffers() throws RocksDBException { + try (final Options opt = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + db.put("key1".getBytes(), "value1ForKey1".getBytes()); + db.put("key2".getBytes(), "value2ForKey2".getBytes()); + db.put("key3".getBytes(), "value3ForKey3".getBytes()); + + final List keys = new ArrayList<>(); + keys.add(ByteBuffer.allocateDirect(12).put("key1".getBytes())); + keys.add(ByteBuffer.allocateDirect(12).put("key2".getBytes())); + keys.add(ByteBuffer.allocateDirect(12).put("key3".getBytes())); + // Java8 and lower flip() returns Buffer not ByteBuffer, so can't chain above /\/\ + for (final ByteBuffer key : keys) { + key.flip(); + } + + { + final List values = new ArrayList<>(); + for (int i = 0; i < keys.size(); i++) { + values.add(ByteBuffer.allocateDirect(4)); + } + + final List statii = db.multiGetByteBuffers(keys, values); + assertThat(statii.size()).isEqualTo(values.size()); + for (final ByteBufferGetStatus status : statii) { + assertThat(status.status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(status.requiredSize).isEqualTo("value3ForKey3".getBytes().length); + final ByteBuffer expected = + ByteBuffer.allocateDirect(24).put(Arrays.copyOf("valueX".getBytes(), 4)); + expected.flip(); + assertThat(status.value).isEqualTo(expected); + } + } + } + } + + @Test + public void putNThenMultiGetDirectNondefaultCF() throws RocksDBException { + try (final Options opt = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + final List cfDescriptors = new ArrayList<>(0); + cfDescriptors.add(new ColumnFamilyDescriptor("cf0".getBytes())); + cfDescriptors.add(new ColumnFamilyDescriptor("cf1".getBytes())); + cfDescriptors.add(new ColumnFamilyDescriptor("cf2".getBytes())); + + final List cf = db.createColumnFamilies(cfDescriptors); + + db.put(cf.get(0), "key1".getBytes(), "value1ForKey1".getBytes()); + db.put(cf.get(0), "key2".getBytes(), "value2ForKey2".getBytes()); + db.put(cf.get(0), "key3".getBytes(), "value3ForKey3".getBytes()); + + final List keys = new ArrayList<>(); + keys.add(ByteBuffer.allocateDirect(12).put("key1".getBytes())); + keys.add(ByteBuffer.allocateDirect(12).put("key2".getBytes())); + keys.add(ByteBuffer.allocateDirect(12).put("key3".getBytes())); + // Java8 and lower flip() returns Buffer not ByteBuffer, so can't chain above /\/\ + for (final ByteBuffer key : keys) { + key.flip(); + } + final List values = new ArrayList<>(); + for (int i = 0; i < keys.size(); i++) { + values.add(ByteBuffer.allocateDirect(24)); + } + + { + final List results = db.multiGetByteBuffers(keys, values); + + assertThat(results.get(0).status.getCode()).isEqualTo(Status.Code.NotFound); + assertThat(results.get(1).status.getCode()).isEqualTo(Status.Code.NotFound); + assertThat(results.get(2).status.getCode()).isEqualTo(Status.Code.NotFound); + } + + { + final List columnFamilyHandles = new ArrayList<>(); + columnFamilyHandles.add(cf.get(0)); + final List results = + db.multiGetByteBuffers(columnFamilyHandles, keys, values); + + assertThat(results.get(0).status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(results.get(1).status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(results.get(2).status.getCode()).isEqualTo(Status.Code.Ok); + + assertThat(results.get(0).requiredSize).isEqualTo("value1ForKey1".getBytes().length); + assertThat(results.get(1).requiredSize).isEqualTo("value2ForKey2".getBytes().length); + assertThat(results.get(2).requiredSize).isEqualTo("value3ForKey3".getBytes().length); + + assertThat(TestUtil.bufferBytes(results.get(0).value)) + .isEqualTo("value1ForKey1".getBytes()); + assertThat(TestUtil.bufferBytes(results.get(1).value)) + .isEqualTo("value2ForKey2".getBytes()); + assertThat(TestUtil.bufferBytes(results.get(2).value)) + .isEqualTo("value3ForKey3".getBytes()); + } + + { + final List columnFamilyHandles = new ArrayList<>(); + columnFamilyHandles.add(cf.get(0)); + columnFamilyHandles.add(cf.get(0)); + columnFamilyHandles.add(cf.get(0)); + final List results = + db.multiGetByteBuffers(columnFamilyHandles, keys, values); + + assertThat(results.get(0).status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(results.get(1).status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(results.get(2).status.getCode()).isEqualTo(Status.Code.Ok); + + assertThat(results.get(0).requiredSize).isEqualTo("value1ForKey1".getBytes().length); + assertThat(results.get(1).requiredSize).isEqualTo("value2ForKey2".getBytes().length); + assertThat(results.get(2).requiredSize).isEqualTo("value3ForKey3".getBytes().length); + + assertThat(TestUtil.bufferBytes(results.get(0).value)) + .isEqualTo("value1ForKey1".getBytes()); + assertThat(TestUtil.bufferBytes(results.get(1).value)) + .isEqualTo("value2ForKey2".getBytes()); + assertThat(TestUtil.bufferBytes(results.get(2).value)) + .isEqualTo("value3ForKey3".getBytes()); + } + } + } + + @Test + public void putNThenMultiGetDirectCFParams() throws RocksDBException { + try (final Options opt = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + db.put("key1".getBytes(), "value1ForKey1".getBytes()); + db.put("key2".getBytes(), "value2ForKey2".getBytes()); + db.put("key3".getBytes(), "value3ForKey3".getBytes()); + + final List columnFamilyHandles = new ArrayList<>(); + columnFamilyHandles.add(db.getDefaultColumnFamily()); + columnFamilyHandles.add(db.getDefaultColumnFamily()); + + final List keys = new ArrayList<>(); + keys.add(ByteBuffer.allocateDirect(12).put("key1".getBytes())); + keys.add(ByteBuffer.allocateDirect(12).put("key2".getBytes())); + keys.add(ByteBuffer.allocateDirect(12).put("key3".getBytes())); + // Java8 and lower flip() returns Buffer not ByteBuffer, so can't chain above /\/\ + for (final ByteBuffer key : keys) { + key.flip(); + } + final List values = new ArrayList<>(); + for (int i = 0; i < keys.size(); i++) { + values.add(ByteBuffer.allocateDirect(24)); + } + try { + db.multiGetByteBuffers(columnFamilyHandles, keys, values); + fail("Expected exception when 2 column families supplied"); + } catch (final IllegalArgumentException e) { + assertThat(e.getMessage()).contains("Wrong number of ColumnFamilyHandle(s) supplied"); + } + + columnFamilyHandles.clear(); + columnFamilyHandles.add(db.getDefaultColumnFamily()); + final List results = + db.multiGetByteBuffers(columnFamilyHandles, keys, values); + + assertThat(results.get(0).status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(results.get(1).status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(results.get(2).status.getCode()).isEqualTo(Status.Code.Ok); + + assertThat(results.get(0).requiredSize).isEqualTo("value1ForKey1".getBytes().length); + assertThat(results.get(1).requiredSize).isEqualTo("value2ForKey2".getBytes().length); + assertThat(results.get(2).requiredSize).isEqualTo("value3ForKey3".getBytes().length); + + assertThat(TestUtil.bufferBytes(results.get(0).value)).isEqualTo("value1ForKey1".getBytes()); + assertThat(TestUtil.bufferBytes(results.get(1).value)).isEqualTo("value2ForKey2".getBytes()); + assertThat(TestUtil.bufferBytes(results.get(2).value)).isEqualTo("value3ForKey3".getBytes()); + } + } + + @Test + public void putNThenMultiGetDirectMixedCF() throws RocksDBException { + try (final Options opt = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + final List cfDescriptors = new ArrayList<>(); + cfDescriptors.add(new ColumnFamilyDescriptor("cf0".getBytes())); + cfDescriptors.add(new ColumnFamilyDescriptor("cf1".getBytes())); + cfDescriptors.add(new ColumnFamilyDescriptor("cf2".getBytes())); + cfDescriptors.add(new ColumnFamilyDescriptor("cf3".getBytes())); + + final List cf = db.createColumnFamilies(cfDescriptors); + + db.put(cf.get(1), "key1".getBytes(), "value1ForKey1".getBytes()); + db.put("key2".getBytes(), "value2ForKey2".getBytes()); + db.put(cf.get(3), "key3".getBytes(), "value3ForKey3".getBytes()); + + final List keys = new ArrayList<>(); + keys.add(ByteBuffer.allocateDirect(12).put("key1".getBytes())); + keys.add(ByteBuffer.allocateDirect(12).put("key2".getBytes())); + keys.add(ByteBuffer.allocateDirect(12).put("key3".getBytes())); + // Java8 and lower flip() returns Buffer not ByteBuffer, so can't chain above /\/\ + for (final ByteBuffer key : keys) { + key.flip(); + } + final List values = new ArrayList<>(); + for (int i = 0; i < keys.size(); i++) { + values.add(ByteBuffer.allocateDirect(24)); + } + + { + final List columnFamilyHandles = new ArrayList<>(); + columnFamilyHandles.add(db.getDefaultColumnFamily()); + + final List results = + db.multiGetByteBuffers(columnFamilyHandles, keys, values); + + assertThat(results.get(0).status.getCode()).isEqualTo(Status.Code.NotFound); + assertThat(results.get(1).status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(results.get(2).status.getCode()).isEqualTo(Status.Code.NotFound); + + assertThat(results.get(1).requiredSize).isEqualTo("value2ForKey2".getBytes().length); + + assertThat(TestUtil.bufferBytes(results.get(1).value)) + .isEqualTo("value2ForKey2".getBytes()); + } + + { + final List columnFamilyHandles = new ArrayList<>(); + columnFamilyHandles.add(cf.get(1)); + + final List results = + db.multiGetByteBuffers(columnFamilyHandles, keys, values); + + assertThat(results.get(0).status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(results.get(1).status.getCode()).isEqualTo(Status.Code.NotFound); + assertThat(results.get(2).status.getCode()).isEqualTo(Status.Code.NotFound); + + assertThat(results.get(0).requiredSize).isEqualTo("value2ForKey2".getBytes().length); + + assertThat(TestUtil.bufferBytes(results.get(0).value)) + .isEqualTo("value1ForKey1".getBytes()); + } + + { + final List columnFamilyHandles = new ArrayList<>(); + columnFamilyHandles.add(cf.get(1)); + columnFamilyHandles.add(db.getDefaultColumnFamily()); + columnFamilyHandles.add(cf.get(3)); + + final List results = + db.multiGetByteBuffers(columnFamilyHandles, keys, values); + + assertThat(results.get(0).status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(results.get(1).status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(results.get(2).status.getCode()).isEqualTo(Status.Code.Ok); + + assertThat(results.get(0).requiredSize).isEqualTo("value1ForKey1".getBytes().length); + assertThat(results.get(1).requiredSize).isEqualTo("value2ForKey2".getBytes().length); + assertThat(results.get(2).requiredSize).isEqualTo("value3ForKey3".getBytes().length); + + assertThat(TestUtil.bufferBytes(results.get(0).value)) + .isEqualTo("value1ForKey1".getBytes()); + assertThat(TestUtil.bufferBytes(results.get(1).value)) + .isEqualTo("value2ForKey2".getBytes()); + assertThat(TestUtil.bufferBytes(results.get(2).value)) + .isEqualTo("value3ForKey3".getBytes()); + } + + { + final List columnFamilyHandles = new ArrayList<>(); + columnFamilyHandles.add(db.getDefaultColumnFamily()); + columnFamilyHandles.add(cf.get(1)); + columnFamilyHandles.add(cf.get(3)); + + final List results = + db.multiGetByteBuffers(columnFamilyHandles, keys, values); + + assertThat(results.get(0).status.getCode()).isEqualTo(Status.Code.NotFound); + assertThat(results.get(1).status.getCode()).isEqualTo(Status.Code.NotFound); + assertThat(results.get(2).status.getCode()).isEqualTo(Status.Code.Ok); + + assertThat(results.get(2).requiredSize).isEqualTo("value3ForKey3".getBytes().length); + + assertThat(TestUtil.bufferBytes(results.get(2).value)) + .isEqualTo("value3ForKey3".getBytes()); + } + } + } + + @Test + public void putNThenMultiGetDirectTruncateCF() throws RocksDBException { + try (final Options opt = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + final List cfDescriptors = new ArrayList<>(); + cfDescriptors.add(new ColumnFamilyDescriptor("cf0".getBytes())); + + final List cf = db.createColumnFamilies(cfDescriptors); + + db.put(cf.get(0), "key1".getBytes(), "value1ForKey1".getBytes()); + db.put(cf.get(0), "key2".getBytes(), "value2ForKey2WithLotsOfTrailingGarbage".getBytes()); + db.put(cf.get(0), "key3".getBytes(), "value3ForKey3".getBytes()); + + final List keys = new ArrayList<>(); + keys.add(ByteBuffer.allocateDirect(12).put("key1".getBytes())); + keys.add(ByteBuffer.allocateDirect(12).put("key2".getBytes())); + keys.add(ByteBuffer.allocateDirect(12).put("key3".getBytes())); + // Java8 and lower flip() returns Buffer not ByteBuffer, so can't chain above /\/\ + for (final ByteBuffer key : keys) { + key.flip(); + } + final List values = new ArrayList<>(); + for (int i = 0; i < keys.size(); i++) { + values.add(ByteBuffer.allocateDirect(24)); + } + + { + final List columnFamilyHandles = new ArrayList<>(); + columnFamilyHandles.add(cf.get(0)); + final List results = + db.multiGetByteBuffers(columnFamilyHandles, keys, values); + + assertThat(results.get(0).status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(results.get(1).status.getCode()).isEqualTo(Status.Code.Ok); + assertThat(results.get(2).status.getCode()).isEqualTo(Status.Code.Ok); + + assertThat(results.get(0).requiredSize).isEqualTo("value1ForKey1".getBytes().length); + assertThat(results.get(1).requiredSize) + .isEqualTo("value2ForKey2WithLotsOfTrailingGarbage".getBytes().length); + assertThat(results.get(2).requiredSize).isEqualTo("value3ForKey3".getBytes().length); + + assertThat(TestUtil.bufferBytes(results.get(0).value)) + .isEqualTo("value1ForKey1".getBytes()); + assertThat(TestUtil.bufferBytes(results.get(1).value)) + .isEqualTo("valu e2Fo rKey 2Wit hLot sOfT".replace(" ", "").getBytes()); + assertThat(TestUtil.bufferBytes(results.get(2).value)) + .isEqualTo("value3ForKey3".getBytes()); + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MutableColumnFamilyOptionsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MutableColumnFamilyOptionsTest.java new file mode 100644 index 0000000..d858a15 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MutableColumnFamilyOptionsTest.java @@ -0,0 +1,167 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +import org.junit.Test; +import org.rocksdb.MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder; + +import java.util.NoSuchElementException; + +import static org.assertj.core.api.Assertions.assertThat; + +public class MutableColumnFamilyOptionsTest { + + @Test + public void builder() { + final MutableColumnFamilyOptionsBuilder builder = + MutableColumnFamilyOptions.builder(); + builder + .setWriteBufferSize(10) + .setInplaceUpdateNumLocks(5) + .setDisableAutoCompactions(true) + .setParanoidFileChecks(true); + + assertThat(builder.writeBufferSize()).isEqualTo(10); + assertThat(builder.inplaceUpdateNumLocks()).isEqualTo(5); + assertThat(builder.disableAutoCompactions()).isEqualTo(true); + assertThat(builder.paranoidFileChecks()).isEqualTo(true); + } + + @Test(expected = NoSuchElementException.class) + public void builder_getWhenNotSet() { + final MutableColumnFamilyOptionsBuilder builder = + MutableColumnFamilyOptions.builder(); + + builder.writeBufferSize(); + } + + @Test + public void builder_build() { + final MutableColumnFamilyOptions options = MutableColumnFamilyOptions + .builder() + .setWriteBufferSize(10) + .setParanoidFileChecks(true) + .build(); + + assertThat(options.getKeys().length).isEqualTo(2); + assertThat(options.getValues().length).isEqualTo(2); + assertThat(options.getKeys()[0]) + .isEqualTo( + MutableColumnFamilyOptions.MemtableOption.write_buffer_size.name()); + assertThat(options.getValues()[0]).isEqualTo("10"); + assertThat(options.getKeys()[1]) + .isEqualTo( + MutableColumnFamilyOptions.MiscOption.paranoid_file_checks.name()); + assertThat(options.getValues()[1]).isEqualTo("true"); + } + + @Test + public void mutableColumnFamilyOptions_toString() { + final String str = MutableColumnFamilyOptions.builder() + .setWriteBufferSize(10) + .setInplaceUpdateNumLocks(5) + .setDisableAutoCompactions(true) + .setParanoidFileChecks(true) + .setMaxBytesForLevelMultiplierAdditional(new int[] {2, 3, 5, 7, 11, 13}) + .build() + .toString(); + + assertThat(str).isEqualTo("write_buffer_size=10;inplace_update_num_locks=5;" + + "disable_auto_compactions=true;paranoid_file_checks=true;max_bytes_for_level_multiplier_additional=2:3:5:7:11:13"); + } + + @Test + public void mutableColumnFamilyOptions_parse() { + final String str = "write_buffer_size=10;inplace_update_num_locks=5;" + + "disable_auto_compactions=true;paranoid_file_checks=true;max_bytes_for_level_multiplier_additional=2:{3}:{5}:{7}:{11}:{13}"; + + final MutableColumnFamilyOptionsBuilder builder = + MutableColumnFamilyOptions.parse(str); + + assertThat(builder.writeBufferSize()).isEqualTo(10); + assertThat(builder.inplaceUpdateNumLocks()).isEqualTo(5); + assertThat(builder.disableAutoCompactions()).isEqualTo(true); + assertThat(builder.paranoidFileChecks()).isEqualTo(true); + assertThat(builder.maxBytesForLevelMultiplierAdditional()) + .isEqualTo(new int[] {2, 3, 5, 7, 11, 13}); + } + + /** + * Extended parsing test to deal with all the options which C++ may return. + * We have canned a set of options returned by {RocksDB#getOptions} + */ + @Test + public void mutableColumnFamilyOptions_parse_getOptions_output() { + final String optionsString = + "bottommost_compression=kDisableCompressionOption; sample_for_compression=0; " + + "blob_garbage_collection_age_cutoff=0.250000; blob_garbage_collection_force_threshold=0.800000;" + + "arena_block_size=1048576; enable_blob_garbage_collection=false; level0_stop_writes_trigger=36; min_blob_size=65536;" + + "blob_compaction_readahead_size=262144; blob_file_starting_level=5; prepopulate_blob_cache=kDisable;" + + "compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;" + + "compression_size_percent=-1;max_size_amplification_percent=200;max_merge_width=4294967295;size_ratio=1;}; " + + "target_file_size_base=67108864; max_bytes_for_level_base=268435456; memtable_whole_key_filtering=false; " + + "soft_pending_compaction_bytes_limit=68719476736; blob_compression_type=kNoCompression; max_write_buffer_number=2; " + + "ttl=2592000; compaction_options_fifo={allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;}; " + + "check_flush_compaction_key_order=true; max_successive_merges=0; inplace_update_num_locks=10000; " + + "bottommost_compression_opts={enabled=false;parallel_threads=1;zstd_max_train_bytes=0;max_dict_bytes=0;" + + "strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;}; " + + "target_file_size_multiplier=1; max_bytes_for_level_multiplier_additional=5:{7}:{9}:{11}:{13}:{15}:{17}; " + + "enable_blob_files=true; level0_slowdown_writes_trigger=20; compression=kLZ4HCCompression; level0_file_num_compaction_trigger=4; " + + "blob_file_size=268435456; prefix_extractor=nullptr; max_bytes_for_level_multiplier=10.000000; write_buffer_size=67108864; " + + "disable_auto_compactions=false; max_compaction_bytes=1677721600; memtable_huge_page_size=0; " + + "compression_opts={enabled=false;parallel_threads=1;zstd_max_train_bytes=0;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;" + + "level=32767;window_bits=-14;}; " + + "hard_pending_compaction_bytes_limit=274877906944; periodic_compaction_seconds=0; paranoid_file_checks=true; " + + "memtable_prefix_bloom_size_ratio=7.500000; max_sequential_skip_in_iterations=8; report_bg_io_stats=true; " + + "compaction_pri=kMinOverlappingRatio; compaction_style=kCompactionStyleLevel; memtable_factory=SkipListFactory; " + + "comparator=leveldb.BytewiseComparator; bloom_locality=0; compaction_filter_factory=nullptr; " + + "min_write_buffer_number_to_merge=1; max_write_buffer_number_to_maintain=0; compaction_filter=nullptr; merge_operator=nullptr; " + + "num_levels=7; optimize_filters_for_hits=false; force_consistency_checks=true; table_factory=BlockBasedTable; " + + "max_write_buffer_size_to_maintain=0; memtable_insert_with_hint_prefix_extractor=nullptr; level_compaction_dynamic_level_bytes=false; " + + "inplace_update_support=false; experimental_mempurge_threshold=0.003"; + + final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder cf = + MutableColumnFamilyOptions.parse(optionsString, true); + + // Check the values from the parsed string which are column family options + assertThat(cf.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25); + assertThat(cf.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); + assertThat(cf.arenaBlockSize()).isEqualTo(1048576); + assertThat(cf.enableBlobGarbageCollection()).isEqualTo(false); + assertThat(cf.level0StopWritesTrigger()).isEqualTo(36); + assertThat(cf.minBlobSize()).isEqualTo(65536); + assertThat(cf.blobCompactionReadaheadSize()).isEqualTo(262144); + assertThat(cf.blobFileStartingLevel()).isEqualTo(5); + assertThat(cf.prepopulateBlobCache()).isEqualTo(PrepopulateBlobCache.PREPOPULATE_BLOB_DISABLE); + assertThat(cf.targetFileSizeBase()).isEqualTo(67108864); + assertThat(cf.maxBytesForLevelBase()).isEqualTo(268435456); + assertThat(cf.softPendingCompactionBytesLimit()).isEqualTo(68719476736L); + assertThat(cf.blobCompressionType()).isEqualTo(CompressionType.NO_COMPRESSION); + assertThat(cf.maxWriteBufferNumber()).isEqualTo(2); + assertThat(cf.ttl()).isEqualTo(2592000); + assertThat(cf.maxSuccessiveMerges()).isEqualTo(0); + assertThat(cf.inplaceUpdateNumLocks()).isEqualTo(10000); + assertThat(cf.targetFileSizeMultiplier()).isEqualTo(1); + assertThat(cf.maxBytesForLevelMultiplierAdditional()) + .isEqualTo(new int[] {5, 7, 9, 11, 13, 15, 17}); + assertThat(cf.enableBlobFiles()).isEqualTo(true); + assertThat(cf.level0SlowdownWritesTrigger()).isEqualTo(20); + assertThat(cf.compressionType()).isEqualTo(CompressionType.LZ4HC_COMPRESSION); + assertThat(cf.level0FileNumCompactionTrigger()).isEqualTo(4); + assertThat(cf.blobFileSize()).isEqualTo(268435456); + assertThat(cf.maxBytesForLevelMultiplier()).isEqualTo(10.0); + assertThat(cf.writeBufferSize()).isEqualTo(67108864); + assertThat(cf.disableAutoCompactions()).isEqualTo(false); + assertThat(cf.maxCompactionBytes()).isEqualTo(1677721600); + assertThat(cf.memtableHugePageSize()).isEqualTo(0); + assertThat(cf.hardPendingCompactionBytesLimit()).isEqualTo(274877906944L); + assertThat(cf.periodicCompactionSeconds()).isEqualTo(0); + assertThat(cf.paranoidFileChecks()).isEqualTo(true); + assertThat(cf.memtablePrefixBloomSizeRatio()).isEqualTo(7.5); + assertThat(cf.experimentalMempurgeThreshold()).isEqualTo(0.003); + assertThat(cf.maxSequentialSkipInIterations()).isEqualTo(8); + assertThat(cf.reportBgIoStats()).isEqualTo(true); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MutableDBOptionsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MutableDBOptionsTest.java new file mode 100644 index 0000000..063a8de --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MutableDBOptionsTest.java @@ -0,0 +1,85 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +import org.junit.Test; +import org.rocksdb.MutableDBOptions.MutableDBOptionsBuilder; + +import java.util.NoSuchElementException; + +import static org.assertj.core.api.Assertions.assertThat; + +public class MutableDBOptionsTest { + + @Test + public void builder() { + final MutableDBOptionsBuilder builder = + MutableDBOptions.builder(); + builder + .setBytesPerSync(1024 * 1024 * 7) + .setMaxBackgroundJobs(5) + .setAvoidFlushDuringShutdown(false); + + assertThat(builder.bytesPerSync()).isEqualTo(1024 * 1024 * 7); + assertThat(builder.maxBackgroundJobs()).isEqualTo(5); + assertThat(builder.avoidFlushDuringShutdown()).isEqualTo(false); + } + + @Test(expected = NoSuchElementException.class) + public void builder_getWhenNotSet() { + final MutableDBOptionsBuilder builder = + MutableDBOptions.builder(); + + builder.bytesPerSync(); + } + + @Test + public void builder_build() { + final MutableDBOptions options = MutableDBOptions + .builder() + .setBytesPerSync(1024 * 1024 * 7) + .setMaxBackgroundJobs(5) + .build(); + + assertThat(options.getKeys().length).isEqualTo(2); + assertThat(options.getValues().length).isEqualTo(2); + assertThat(options.getKeys()[0]) + .isEqualTo( + MutableDBOptions.DBOption.bytes_per_sync.name()); + assertThat(options.getValues()[0]).isEqualTo("7340032"); + assertThat(options.getKeys()[1]) + .isEqualTo( + MutableDBOptions.DBOption.max_background_jobs.name()); + assertThat(options.getValues()[1]).isEqualTo("5"); + } + + @Test + public void mutableDBOptions_toString() { + final String str = MutableDBOptions + .builder() + .setMaxOpenFiles(99) + .setDelayedWriteRate(789) + .setAvoidFlushDuringShutdown(true) + .setStrictBytesPerSync(true) + .build() + .toString(); + + assertThat(str).isEqualTo("max_open_files=99;delayed_write_rate=789;" + + "avoid_flush_during_shutdown=true;strict_bytes_per_sync=true"); + } + + @Test + public void mutableDBOptions_parse() { + final String str = "max_open_files=99;delayed_write_rate=789;" + + "avoid_flush_during_shutdown=true"; + + final MutableDBOptionsBuilder builder = + MutableDBOptions.parse(str); + + assertThat(builder.maxOpenFiles()).isEqualTo(99); + assertThat(builder.delayedWriteRate()).isEqualTo(789); + assertThat(builder.avoidFlushDuringShutdown()).isEqualTo(true); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java new file mode 100644 index 0000000..6db9406 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java @@ -0,0 +1,429 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class MutableOptionsGetSetTest { + final int minBlobSize = 65536; + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + /** + * Validate the round-trip of blob options into and out of the C++ core of RocksDB + * From CF options on CF Creation to {RocksDB#getOptions} + * Uses 2x column families with different values for their options. + * NOTE that some constraints are applied to the options in the C++ core, + * e.g. on {ColumnFamilyOptions#setMemtablePrefixBloomSizeRatio} + * + * @throws RocksDBException if the database throws an exception + */ + @Test + public void testGetMutableBlobOptionsAfterCreate() throws RocksDBException { + final ColumnFamilyOptions columnFamilyOptions0 = new ColumnFamilyOptions(); + final ColumnFamilyDescriptor columnFamilyDescriptor0 = + new ColumnFamilyDescriptor("default".getBytes(UTF_8), columnFamilyOptions0); + final List columnFamilyDescriptors = + Collections.singletonList(columnFamilyDescriptor0); + final List columnFamilyHandles = new ArrayList<>(); + + try (final DBOptions dbOptions = new DBOptions().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(dbOptions, dbFolder.getRoot().getAbsolutePath(), + columnFamilyDescriptors, columnFamilyHandles)) { + try (final ColumnFamilyOptions columnFamilyOptions1 = + new ColumnFamilyOptions() + .setMinBlobSize(minBlobSize) + .setEnableBlobFiles(true) + .setBlobGarbageCollectionAgeCutoff(0.25) + .setBlobGarbageCollectionForceThreshold(0.80) + .setBlobCompactionReadaheadSize(262144) + .setBlobFileStartingLevel(2) + .setArenaBlockSize(42) + .setMemtablePrefixBloomSizeRatio(0.17) + .setExperimentalMempurgeThreshold(0.005) + .setMemtableWholeKeyFiltering(false) + .setMemtableHugePageSize(3) + .setMaxSuccessiveMerges(4) + .setMaxWriteBufferNumber(12) + .setInplaceUpdateNumLocks(16) + .setDisableAutoCompactions(false) + .setSoftPendingCompactionBytesLimit(112) + .setHardPendingCompactionBytesLimit(280) + .setLevel0FileNumCompactionTrigger(200) + .setLevel0SlowdownWritesTrigger(312) + .setLevel0StopWritesTrigger(584) + .setMaxCompactionBytes(12) + .setTargetFileSizeBase(99) + .setTargetFileSizeMultiplier(112) + .setMaxSequentialSkipInIterations(50) + .setReportBgIoStats(true); + + final ColumnFamilyOptions columnFamilyOptions2 = + new ColumnFamilyOptions() + .setMinBlobSize(minBlobSize) + .setEnableBlobFiles(false) + .setArenaBlockSize(42) + .setMemtablePrefixBloomSizeRatio(0.236) + .setExperimentalMempurgeThreshold(0.247) + .setMemtableWholeKeyFiltering(true) + .setMemtableHugePageSize(8) + .setMaxSuccessiveMerges(12) + .setMaxWriteBufferNumber(22) + .setInplaceUpdateNumLocks(160) + .setDisableAutoCompactions(true) + .setSoftPendingCompactionBytesLimit(1124) + .setHardPendingCompactionBytesLimit(2800) + .setLevel0FileNumCompactionTrigger(2000) + .setLevel0SlowdownWritesTrigger(5840) + .setLevel0StopWritesTrigger(31200) + .setMaxCompactionBytes(112) + .setTargetFileSizeBase(999) + .setTargetFileSizeMultiplier(1120) + .setMaxSequentialSkipInIterations(24) + .setReportBgIoStats(true)) { + final ColumnFamilyDescriptor columnFamilyDescriptor1 = + new ColumnFamilyDescriptor("column_family_1".getBytes(UTF_8), columnFamilyOptions1); + final ColumnFamilyDescriptor columnFamilyDescriptor2 = + new ColumnFamilyDescriptor("column_family_2".getBytes(UTF_8), columnFamilyOptions2); + + // Create the column family with blob options + final ColumnFamilyHandle columnFamilyHandle1 = + db.createColumnFamily(columnFamilyDescriptor1); + final ColumnFamilyHandle columnFamilyHandle2 = + db.createColumnFamily(columnFamilyDescriptor2); + + // Check the getOptions() brings back the creation options for CF1 + final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder1 = + db.getOptions(columnFamilyHandle1); + assertThat(builder1.enableBlobFiles()).isEqualTo(true); + assertThat(builder1.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25); + assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); + assertThat(builder1.blobCompactionReadaheadSize()).isEqualTo(262144); + assertThat(builder1.blobFileStartingLevel()).isEqualTo(2); + assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize); + assertThat(builder1.arenaBlockSize()).isEqualTo(42); + assertThat(builder1.memtablePrefixBloomSizeRatio()).isEqualTo(0.17); + assertThat(builder1.experimentalMempurgeThreshold()).isEqualTo(0.005); + assertThat(builder1.memtableWholeKeyFiltering()).isEqualTo(false); + assertThat(builder1.memtableHugePageSize()).isEqualTo(3); + assertThat(builder1.maxSuccessiveMerges()).isEqualTo(4); + assertThat(builder1.maxWriteBufferNumber()).isEqualTo(12); + assertThat(builder1.inplaceUpdateNumLocks()).isEqualTo(16); + assertThat(builder1.disableAutoCompactions()).isEqualTo(false); + assertThat(builder1.softPendingCompactionBytesLimit()).isEqualTo(112); + assertThat(builder1.hardPendingCompactionBytesLimit()).isEqualTo(280); + assertThat(builder1.level0FileNumCompactionTrigger()).isEqualTo(200); + assertThat(builder1.level0SlowdownWritesTrigger()).isEqualTo(312); + assertThat(builder1.level0StopWritesTrigger()).isEqualTo(584); + assertThat(builder1.maxCompactionBytes()).isEqualTo(12); + assertThat(builder1.targetFileSizeBase()).isEqualTo(99); + assertThat(builder1.targetFileSizeMultiplier()).isEqualTo(112); + assertThat(builder1.maxSequentialSkipInIterations()).isEqualTo(50); + assertThat(builder1.reportBgIoStats()).isEqualTo(true); + + // Check the getOptions() brings back the creation options for CF2 + final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder2 = + db.getOptions(columnFamilyHandle2); + assertThat(builder2.enableBlobFiles()).isEqualTo(false); + assertThat(builder2.minBlobSize()).isEqualTo(minBlobSize); + assertThat(builder2.arenaBlockSize()).isEqualTo(42); + assertThat(builder2.memtablePrefixBloomSizeRatio()).isEqualTo(0.236); + assertThat(builder2.experimentalMempurgeThreshold()).isEqualTo(0.247); + assertThat(builder2.memtableWholeKeyFiltering()).isEqualTo(true); + assertThat(builder2.memtableHugePageSize()).isEqualTo(8); + assertThat(builder2.maxSuccessiveMerges()).isEqualTo(12); + assertThat(builder2.maxWriteBufferNumber()).isEqualTo(22); + assertThat(builder2.inplaceUpdateNumLocks()).isEqualTo(160); + assertThat(builder2.disableAutoCompactions()).isEqualTo(true); + assertThat(builder2.softPendingCompactionBytesLimit()).isEqualTo(1124); + assertThat(builder2.hardPendingCompactionBytesLimit()).isEqualTo(2800); + assertThat(builder2.level0FileNumCompactionTrigger()).isEqualTo(2000); + assertThat(builder2.level0SlowdownWritesTrigger()).isEqualTo(5840); + assertThat(builder2.level0StopWritesTrigger()).isEqualTo(31200); + assertThat(builder2.maxCompactionBytes()).isEqualTo(112); + assertThat(builder2.targetFileSizeBase()).isEqualTo(999); + assertThat(builder2.targetFileSizeMultiplier()).isEqualTo(1120); + assertThat(builder2.maxSequentialSkipInIterations()).isEqualTo(24); + assertThat(builder2.reportBgIoStats()).isEqualTo(true); + } + } + } + + /** + * Validate the round-trip of blob options into and out of the C++ core of RocksDB + * From {RocksDB#setOptions} to {RocksDB#getOptions} + * Uses 2x column families with different values for their options. + * NOTE that some constraints are applied to the options in the C++ core, + * e.g. on {ColumnFamilyOptions#setMemtablePrefixBloomSizeRatio} + * + * @throws RocksDBException if a database access has an error + */ + @Test + public void testGetMutableBlobOptionsAfterSetCF() throws RocksDBException { + final ColumnFamilyOptions columnFamilyOptions0 = new ColumnFamilyOptions(); + final ColumnFamilyDescriptor columnFamilyDescriptor0 = + new ColumnFamilyDescriptor("default".getBytes(UTF_8), columnFamilyOptions0); + final List columnFamilyDescriptors = + Collections.singletonList(columnFamilyDescriptor0); + final List columnFamilyHandles = new ArrayList<>(); + + try (final DBOptions dbOptions = new DBOptions().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(dbOptions, dbFolder.getRoot().getAbsolutePath(), + columnFamilyDescriptors, columnFamilyHandles)) { + try (final ColumnFamilyOptions columnFamilyOptions1 = new ColumnFamilyOptions(); + + final ColumnFamilyOptions columnFamilyOptions2 = new ColumnFamilyOptions()) { + final ColumnFamilyDescriptor columnFamilyDescriptor1 = + new ColumnFamilyDescriptor("column_family_1".getBytes(UTF_8), columnFamilyOptions1); + final ColumnFamilyDescriptor columnFamilyDescriptor2 = + new ColumnFamilyDescriptor("column_family_2".getBytes(UTF_8), columnFamilyOptions2); + + // Create the column family with blob options + final ColumnFamilyHandle columnFamilyHandle1 = + db.createColumnFamily(columnFamilyDescriptor1); + final ColumnFamilyHandle columnFamilyHandle2 = + db.createColumnFamily(columnFamilyDescriptor2); + db.flush(new FlushOptions().setWaitForFlush(true)); + + final MutableColumnFamilyOptions + .MutableColumnFamilyOptionsBuilder mutableColumnFamilyOptions1 = + MutableColumnFamilyOptions.builder() + .setMinBlobSize(minBlobSize) + .setEnableBlobFiles(true) + .setBlobGarbageCollectionAgeCutoff(0.25) + .setBlobGarbageCollectionForceThreshold(0.80) + .setBlobCompactionReadaheadSize(262144) + .setBlobFileStartingLevel(3) + .setArenaBlockSize(42) + .setMemtablePrefixBloomSizeRatio(0.17) + .setExperimentalMempurgeThreshold(0.005) + .setMemtableWholeKeyFiltering(false) + .setMemtableHugePageSize(3) + .setMaxSuccessiveMerges(4) + .setMaxWriteBufferNumber(12) + .setInplaceUpdateNumLocks(16) + .setDisableAutoCompactions(false) + .setSoftPendingCompactionBytesLimit(112) + .setHardPendingCompactionBytesLimit(280) + .setLevel0FileNumCompactionTrigger(200) + .setLevel0SlowdownWritesTrigger(312) + .setLevel0StopWritesTrigger(584) + .setMaxCompactionBytes(12) + .setTargetFileSizeBase(99) + .setTargetFileSizeMultiplier(112); + db.setOptions(columnFamilyHandle1, mutableColumnFamilyOptions1.build()); + + // Check the getOptions() brings back the creation options for CF1 + final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder1 = + db.getOptions(columnFamilyHandle1); + assertThat(builder1.enableBlobFiles()).isEqualTo(true); + assertThat(builder1.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25); + assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); + assertThat(builder1.blobCompactionReadaheadSize()).isEqualTo(262144); + assertThat(builder1.blobFileStartingLevel()).isEqualTo(3); + assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize); + assertThat(builder1.arenaBlockSize()).isEqualTo(42); + assertThat(builder1.memtablePrefixBloomSizeRatio()).isEqualTo(0.17); + assertThat(builder1.experimentalMempurgeThreshold()).isEqualTo(0.005); + assertThat(builder1.memtableWholeKeyFiltering()).isEqualTo(false); + assertThat(builder1.memtableHugePageSize()).isEqualTo(3); + assertThat(builder1.maxSuccessiveMerges()).isEqualTo(4); + assertThat(builder1.maxWriteBufferNumber()).isEqualTo(12); + assertThat(builder1.inplaceUpdateNumLocks()).isEqualTo(16); + assertThat(builder1.disableAutoCompactions()).isEqualTo(false); + assertThat(builder1.softPendingCompactionBytesLimit()).isEqualTo(112); + assertThat(builder1.hardPendingCompactionBytesLimit()).isEqualTo(280); + assertThat(builder1.level0FileNumCompactionTrigger()).isEqualTo(200); + assertThat(builder1.level0SlowdownWritesTrigger()).isEqualTo(312); + assertThat(builder1.level0StopWritesTrigger()).isEqualTo(584); + assertThat(builder1.maxCompactionBytes()).isEqualTo(12); + assertThat(builder1.targetFileSizeBase()).isEqualTo(99); + assertThat(builder1.targetFileSizeMultiplier()).isEqualTo(112); + + final MutableColumnFamilyOptions + .MutableColumnFamilyOptionsBuilder mutableColumnFamilyOptions2 = + MutableColumnFamilyOptions.builder() + .setMinBlobSize(minBlobSize) + .setEnableBlobFiles(false) + .setArenaBlockSize(42) + .setMemtablePrefixBloomSizeRatio(0.236) + .setExperimentalMempurgeThreshold(0.247) + .setMemtableWholeKeyFiltering(true) + .setMemtableHugePageSize(8) + .setMaxSuccessiveMerges(12) + .setMaxWriteBufferNumber(22) + .setInplaceUpdateNumLocks(160) + .setDisableAutoCompactions(true) + .setSoftPendingCompactionBytesLimit(1124) + .setHardPendingCompactionBytesLimit(2800) + .setLevel0FileNumCompactionTrigger(2000) + .setLevel0SlowdownWritesTrigger(5840) + .setLevel0StopWritesTrigger(31200) + .setMaxCompactionBytes(112) + .setTargetFileSizeBase(999) + .setTargetFileSizeMultiplier(1120); + db.setOptions(columnFamilyHandle2, mutableColumnFamilyOptions2.build()); + + // Check the getOptions() brings back the creation options for CF2 + final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder2 = + db.getOptions(columnFamilyHandle2); + assertThat(builder2.enableBlobFiles()).isEqualTo(false); + assertThat(builder2.minBlobSize()).isEqualTo(minBlobSize); + assertThat(builder2.arenaBlockSize()).isEqualTo(42); + assertThat(builder2.memtablePrefixBloomSizeRatio()).isEqualTo(0.236); + assertThat(builder2.experimentalMempurgeThreshold()).isEqualTo(0.247); + assertThat(builder2.memtableWholeKeyFiltering()).isEqualTo(true); + assertThat(builder2.memtableHugePageSize()).isEqualTo(8); + assertThat(builder2.maxSuccessiveMerges()).isEqualTo(12); + assertThat(builder2.maxWriteBufferNumber()).isEqualTo(22); + assertThat(builder2.inplaceUpdateNumLocks()).isEqualTo(160); + assertThat(builder2.disableAutoCompactions()).isEqualTo(true); + assertThat(builder2.softPendingCompactionBytesLimit()).isEqualTo(1124); + assertThat(builder2.hardPendingCompactionBytesLimit()).isEqualTo(2800); + assertThat(builder2.level0FileNumCompactionTrigger()).isEqualTo(2000); + assertThat(builder2.level0SlowdownWritesTrigger()).isEqualTo(5840); + assertThat(builder2.level0StopWritesTrigger()).isEqualTo(31200); + assertThat(builder2.maxCompactionBytes()).isEqualTo(112); + assertThat(builder2.targetFileSizeBase()).isEqualTo(999); + assertThat(builder2.targetFileSizeMultiplier()).isEqualTo(1120); + } + } + } + + /** + * Validate the round-trip of blob options into and out of the C++ core of RocksDB + * From {RocksDB#setOptions} to {RocksDB#getOptions} + * Uses 2x column families with different values for their options. + * NOTE that some constraints are applied to the options in the C++ core, + * e.g. on {ColumnFamilyOptions#setMemtablePrefixBloomSizeRatio} + * + * @throws RocksDBException if a database access has an error + */ + @Test + public void testGetMutableBlobOptionsAfterSet() throws RocksDBException { + final ColumnFamilyOptions columnFamilyOptions0 = new ColumnFamilyOptions(); + final ColumnFamilyDescriptor columnFamilyDescriptor0 = + new ColumnFamilyDescriptor("default".getBytes(UTF_8), columnFamilyOptions0); + final List columnFamilyDescriptors = + Collections.singletonList(columnFamilyDescriptor0); + final List columnFamilyHandles = new ArrayList<>(); + + try (final DBOptions dbOptions = new DBOptions().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(dbOptions, dbFolder.getRoot().getAbsolutePath(), + columnFamilyDescriptors, columnFamilyHandles)) { + final MutableColumnFamilyOptions + .MutableColumnFamilyOptionsBuilder mutableColumnFamilyOptions = + MutableColumnFamilyOptions.builder() + .setMinBlobSize(minBlobSize) + .setEnableBlobFiles(true) + .setBlobGarbageCollectionAgeCutoff(0.25) + .setBlobGarbageCollectionForceThreshold(0.80) + .setBlobCompactionReadaheadSize(131072) + .setBlobFileStartingLevel(4) + .setArenaBlockSize(42) + .setMemtablePrefixBloomSizeRatio(0.17) + .setExperimentalMempurgeThreshold(0.005) + .setMemtableWholeKeyFiltering(false) + .setMemtableHugePageSize(3) + .setMaxSuccessiveMerges(4) + .setMaxWriteBufferNumber(12) + .setInplaceUpdateNumLocks(16) + .setDisableAutoCompactions(false) + .setSoftPendingCompactionBytesLimit(112) + .setHardPendingCompactionBytesLimit(280) + .setLevel0FileNumCompactionTrigger(200) + .setLevel0SlowdownWritesTrigger(312) + .setLevel0StopWritesTrigger(584) + .setMaxCompactionBytes(12) + .setTargetFileSizeBase(99) + .setTargetFileSizeMultiplier(112); + db.setOptions(mutableColumnFamilyOptions.build()); + + // Check the getOptions() brings back the creation options for CF1 + final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder1 = db.getOptions(); + assertThat(builder1.enableBlobFiles()).isEqualTo(true); + assertThat(builder1.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25); + assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); + assertThat(builder1.blobCompactionReadaheadSize()).isEqualTo(131072); + assertThat(builder1.blobFileStartingLevel()).isEqualTo(4); + assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize); + assertThat(builder1.arenaBlockSize()).isEqualTo(42); + assertThat(builder1.memtablePrefixBloomSizeRatio()).isEqualTo(0.17); + assertThat(builder1.experimentalMempurgeThreshold()).isEqualTo(0.005); + assertThat(builder1.memtableWholeKeyFiltering()).isEqualTo(false); + assertThat(builder1.memtableHugePageSize()).isEqualTo(3); + assertThat(builder1.maxSuccessiveMerges()).isEqualTo(4); + assertThat(builder1.maxWriteBufferNumber()).isEqualTo(12); + assertThat(builder1.inplaceUpdateNumLocks()).isEqualTo(16); + assertThat(builder1.disableAutoCompactions()).isEqualTo(false); + assertThat(builder1.softPendingCompactionBytesLimit()).isEqualTo(112); + assertThat(builder1.hardPendingCompactionBytesLimit()).isEqualTo(280); + assertThat(builder1.level0FileNumCompactionTrigger()).isEqualTo(200); + assertThat(builder1.level0SlowdownWritesTrigger()).isEqualTo(312); + assertThat(builder1.level0StopWritesTrigger()).isEqualTo(584); + assertThat(builder1.maxCompactionBytes()).isEqualTo(12); + assertThat(builder1.targetFileSizeBase()).isEqualTo(99); + assertThat(builder1.targetFileSizeMultiplier()).isEqualTo(112); + } + } + + @Test + public void testGetMutableDBOptionsAfterSet() throws RocksDBException { + final ColumnFamilyOptions columnFamilyOptions0 = new ColumnFamilyOptions(); + final ColumnFamilyDescriptor columnFamilyDescriptor0 = + new ColumnFamilyDescriptor("default".getBytes(UTF_8), columnFamilyOptions0); + final List columnFamilyDescriptors = + Collections.singletonList(columnFamilyDescriptor0); + final List columnFamilyHandles = new ArrayList<>(); + + try (final DBOptions dbOptions = new DBOptions().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(dbOptions, dbFolder.getRoot().getAbsolutePath(), + columnFamilyDescriptors, columnFamilyHandles)) { + final MutableDBOptions.MutableDBOptionsBuilder mutableDBOptions = + MutableDBOptions.builder() + .setMaxBackgroundJobs(16) + .setAvoidFlushDuringShutdown(true) + .setWritableFileMaxBufferSize(2097152) + .setDelayedWriteRate(67108864) + .setMaxTotalWalSize(16777216) + .setDeleteObsoleteFilesPeriodMicros(86400000000L) + .setStatsDumpPeriodSec(1200) + .setStatsPersistPeriodSec(7200) + .setStatsHistoryBufferSize(6291456) + .setMaxOpenFiles(8) + .setBytesPerSync(4194304) + .setWalBytesPerSync(1048576) + .setStrictBytesPerSync(true) + .setCompactionReadaheadSize(1024); + + db.setDBOptions(mutableDBOptions.build()); + + final MutableDBOptions.MutableDBOptionsBuilder getBuilder = db.getDBOptions(); + assertThat(getBuilder.maxBackgroundJobs()).isEqualTo(16); // 4 + assertThat(getBuilder.avoidFlushDuringShutdown()).isEqualTo(true); // false + assertThat(getBuilder.writableFileMaxBufferSize()).isEqualTo(2097152); // 1048576 + assertThat(getBuilder.delayedWriteRate()).isEqualTo(67108864); // 16777216 + assertThat(getBuilder.maxTotalWalSize()).isEqualTo(16777216); + assertThat(getBuilder.deleteObsoleteFilesPeriodMicros()) + .isEqualTo(86400000000L); // 21600000000 + assertThat(getBuilder.statsDumpPeriodSec()).isEqualTo(1200); // 600 + assertThat(getBuilder.statsPersistPeriodSec()).isEqualTo(7200); // 600 + assertThat(getBuilder.statsHistoryBufferSize()).isEqualTo(6291456); // 1048576 + assertThat(getBuilder.maxOpenFiles()).isEqualTo(8); //-1 + assertThat(getBuilder.bytesPerSync()).isEqualTo(4194304); // 1048576 + assertThat(getBuilder.walBytesPerSync()).isEqualTo(1048576); // 0 + assertThat(getBuilder.strictBytesPerSync()).isEqualTo(true); // false + assertThat(getBuilder.compactionReadaheadSize()).isEqualTo(1024); // 0 + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/NativeComparatorWrapperTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/NativeComparatorWrapperTest.java new file mode 100644 index 0000000..1e0ded8 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/NativeComparatorWrapperTest.java @@ -0,0 +1,90 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static org.junit.Assert.assertEquals; + +import java.util.Arrays; +import java.util.Comparator; +import java.util.Random; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class NativeComparatorWrapperTest { + static { + RocksDB.loadLibrary(); + } + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + private static final Random random = new Random(); + + @Test + public void rountrip() throws RocksDBException { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + final int ITERATIONS = 1_000; + + final String[] storedKeys = new String[ITERATIONS]; + try (final NativeStringComparatorWrapper comparator = new NativeStringComparatorWrapper(); + final Options opt = new Options() + .setCreateIfMissing(true) + .setComparator(comparator)) { + + // store random integer keys + try (final RocksDB db = RocksDB.open(opt, dbPath)) { + for (int i = 0; i < ITERATIONS; i++) { + final String strKey = randomString(); + final byte[] key = strKey.getBytes(); + // does key already exist (avoid duplicates) + if (i > 0 && db.get(key) != null) { + i--; // generate a different key + } else { + db.put(key, "value".getBytes()); + storedKeys[i] = strKey; + } + } + } + + // sort the stored keys into ascending alpha-numeric order + Arrays.sort(storedKeys, Comparator.naturalOrder()); + + // re-open db and read from start to end + // string keys should be in ascending + // order + try (final RocksDB db = RocksDB.open(opt, dbPath); + final RocksIterator it = db.newIterator()) { + int count = 0; + for (it.seekToFirst(); it.isValid(); it.next()) { + final String strKey = new String(it.key()); + assertEquals(storedKeys[count++], strKey); + } + } + } + } + + private String randomString() { + final char[] chars = new char[12]; + for(int i = 0; i < 12; i++) { + final int letterCode = random.nextInt(24); + final char letter = (char) (((int) 'a') + letterCode); + chars[i] = letter; + } + return String.copyValueOf(chars); + } + + public static class NativeStringComparatorWrapper + extends NativeComparatorWrapper { + + @Override + protected long initializeNative(final long... nativeParameterHandles) { + return newStringComparator(); + } + + private native long newStringComparator(); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/NativeLibraryLoaderTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/NativeLibraryLoaderTest.java new file mode 100644 index 0000000..6b954f6 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/NativeLibraryLoaderTest.java @@ -0,0 +1,41 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.util.Environment; + +import java.io.File; +import java.io.IOException; +import java.nio.file.*; + +import static org.assertj.core.api.Assertions.assertThat; + +public class NativeLibraryLoaderTest { + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void tempFolder() throws IOException { + NativeLibraryLoader.getInstance().loadLibraryFromJarToTemp( + temporaryFolder.getRoot().getAbsolutePath()); + final Path path = Paths.get(temporaryFolder.getRoot().getAbsolutePath(), + Environment.getJniLibraryFileName("rocksdb")); + assertThat(Files.exists(path)).isTrue(); + assertThat(Files.isReadable(path)).isTrue(); + } + + @Test + public void overridesExistingLibrary() throws IOException { + final File first = NativeLibraryLoader.getInstance().loadLibraryFromJarToTemp( + temporaryFolder.getRoot().getAbsolutePath()); + NativeLibraryLoader.getInstance().loadLibraryFromJarToTemp( + temporaryFolder.getRoot().getAbsolutePath()); + assertThat(first.exists()).isTrue(); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/OptimisticTransactionDBTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/OptimisticTransactionDBTest.java new file mode 100644 index 0000000..519b70b --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/OptimisticTransactionDBTest.java @@ -0,0 +1,131 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +public class OptimisticTransactionDBTest { + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void open() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final OptimisticTransactionDB otdb = OptimisticTransactionDB.open(options, + dbFolder.getRoot().getAbsolutePath())) { + assertThat(otdb).isNotNull(); + } + } + + @Test + public void open_columnFamilies() throws RocksDBException { + try(final DBOptions dbOptions = new DBOptions().setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + final ColumnFamilyOptions myCfOpts = new ColumnFamilyOptions()) { + + final List columnFamilyDescriptors = + Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("myCf".getBytes(), myCfOpts)); + + final List columnFamilyHandles = new ArrayList<>(); + + try (final OptimisticTransactionDB otdb = OptimisticTransactionDB.open(dbOptions, + dbFolder.getRoot().getAbsolutePath(), + columnFamilyDescriptors, columnFamilyHandles)) { + try { + assertThat(otdb).isNotNull(); + } finally { + for (final ColumnFamilyHandle handle : columnFamilyHandles) { + handle.close(); + } + } + } + } + } + + @Test + public void beginTransaction() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final OptimisticTransactionDB otdb = OptimisticTransactionDB.open( + options, dbFolder.getRoot().getAbsolutePath()); + final WriteOptions writeOptions = new WriteOptions()) { + + try(final Transaction txn = otdb.beginTransaction(writeOptions)) { + assertThat(txn).isNotNull(); + } + } + } + + @Test + public void beginTransaction_transactionOptions() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final OptimisticTransactionDB otdb = OptimisticTransactionDB.open( + options, dbFolder.getRoot().getAbsolutePath()); + final WriteOptions writeOptions = new WriteOptions(); + final OptimisticTransactionOptions optimisticTxnOptions = + new OptimisticTransactionOptions()) { + + try(final Transaction txn = otdb.beginTransaction(writeOptions, + optimisticTxnOptions)) { + assertThat(txn).isNotNull(); + } + } + } + + @Test + public void beginTransaction_withOld() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final OptimisticTransactionDB otdb = OptimisticTransactionDB.open( + options, dbFolder.getRoot().getAbsolutePath()); + final WriteOptions writeOptions = new WriteOptions()) { + + try(final Transaction txn = otdb.beginTransaction(writeOptions)) { + final Transaction txnReused = otdb.beginTransaction(writeOptions, txn); + assertThat(txnReused).isSameAs(txn); + } + } + } + + @Test + public void beginTransaction_withOld_transactionOptions() + throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final OptimisticTransactionDB otdb = OptimisticTransactionDB.open( + options, dbFolder.getRoot().getAbsolutePath()); + final WriteOptions writeOptions = new WriteOptions(); + final OptimisticTransactionOptions optimisticTxnOptions = + new OptimisticTransactionOptions()) { + + try(final Transaction txn = otdb.beginTransaction(writeOptions)) { + final Transaction txnReused = otdb.beginTransaction(writeOptions, + optimisticTxnOptions, txn); + assertThat(txnReused).isSameAs(txn); + } + } + } + + @Test + public void baseDB() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final OptimisticTransactionDB otdb = OptimisticTransactionDB.open(options, + dbFolder.getRoot().getAbsolutePath())) { + assertThat(otdb).isNotNull(); + final RocksDB db = otdb.getBaseDB(); + assertThat(db).isNotNull(); + assertThat(db.isOwningHandle()).isFalse(); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/OptimisticTransactionOptionsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/OptimisticTransactionOptionsTest.java new file mode 100644 index 0000000..ef656b9 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/OptimisticTransactionOptionsTest.java @@ -0,0 +1,38 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.Test; +import org.rocksdb.util.BytewiseComparator; + +import java.util.Random; + +import static org.assertj.core.api.Assertions.assertThat; + +public class OptimisticTransactionOptionsTest { + + private static final Random rand = PlatformRandomHelper. + getPlatformSpecificRandomFactory(); + + @Test + public void setSnapshot() { + try (final OptimisticTransactionOptions opt = new OptimisticTransactionOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setSetSnapshot(boolValue); + assertThat(opt.isSetSnapshot()).isEqualTo(boolValue); + } + } + + @Test + public void comparator() { + try (final OptimisticTransactionOptions opt = new OptimisticTransactionOptions(); + final ComparatorOptions copt = new ComparatorOptions() + .setUseDirectBuffer(true); + final AbstractComparator comparator = new BytewiseComparator(copt)) { + opt.setComparator(comparator); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/OptimisticTransactionTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/OptimisticTransactionTest.java new file mode 100644 index 0000000..d2f92e1 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/OptimisticTransactionTest.java @@ -0,0 +1,446 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.*; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.Test; + +public class OptimisticTransactionTest extends AbstractTransactionTest { + @Test + public void prepare_commit() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + final byte[] v12 = "value12".getBytes(UTF_8); + + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions()) { + try (final Transaction txn = dbContainer.beginTransaction()) { + txn.put(k1, v1); + txn.commit(); + } + + try (final Transaction txn = dbContainer.beginTransaction()) { + txn.put(k1, v12); + txn.prepare(); + + failBecauseExceptionWasNotThrown(RocksDBException.class); + } catch (final RocksDBException e) { + assertThat(e.getMessage()) + .contains("Two phase commit not supported for optimistic transactions"); + } + } + } + + @Test + public void getForUpdate_cf_conflict() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + final byte[] v12 = "value12".getBytes(UTF_8); + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + + try(final Transaction txn = dbContainer.beginTransaction()) { + txn.put(testCf, k1, v1); + assertThat(txn.get(testCf, readOptions, k1)).isEqualTo(v1); + txn.commit(); + } + + try(final Transaction txn2 = dbContainer.beginTransaction()) { + try(final Transaction txn3 = dbContainer.beginTransaction()) { + assertThat(txn3.getForUpdate(readOptions, testCf, k1, true)).isEqualTo(v1); + + // NOTE: txn2 updates k1, during txn3 + txn2.put(testCf, k1, v12); + assertThat(txn2.get(testCf, readOptions, k1)).isEqualTo(v12); + txn2.commit(); + + try { + txn3.commit(); // should cause an exception! + } catch(final RocksDBException e) { + assertThat(e.getStatus().getCode()).isSameAs(Status.Code.Busy); + return; + } + } + } + + fail("Expected an exception for put after getForUpdate from conflicting" + + "transactions"); + } + } + + @Test + public void getForUpdate_conflict() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + final byte[] v12 = "value12".getBytes(UTF_8); + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions()) { + + try(final Transaction txn = dbContainer.beginTransaction()) { + txn.put(k1, v1); + assertThat(txn.get(readOptions, k1)).isEqualTo(v1); + txn.commit(); + } + + try(final Transaction txn2 = dbContainer.beginTransaction()) { + try(final Transaction txn3 = dbContainer.beginTransaction()) { + assertThat(txn3.getForUpdate(readOptions, k1, true)).isEqualTo(v1); + + // NOTE: txn2 updates k1, during txn3 + txn2.put(k1, v12); + assertThat(txn2.get(readOptions, k1)).isEqualTo(v12); + txn2.commit(); + + try { + txn3.commit(); // should cause an exception! + } catch(final RocksDBException e) { + assertThat(e.getStatus().getCode()).isSameAs(Status.Code.Busy); + return; + } + } + } + + fail("Expected an exception for put after getForUpdate from conflicting" + + "transactions"); + } + } + + @Deprecated + @Test + public void multiGetForUpdate_cf_conflict() throws RocksDBException { + final byte[][] keys = new byte[][] {"key1".getBytes(UTF_8), "key2".getBytes(UTF_8)}; + final byte[][] values = new byte[][] {"value1".getBytes(UTF_8), "value2".getBytes(UTF_8)}; + final byte[] otherValue = "otherValue".getBytes(UTF_8); + + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + final List cfList = Arrays.asList(testCf, testCf); + + try(final Transaction txn = dbContainer.beginTransaction()) { + txn.put(testCf, keys[0], values[0]); + txn.put(testCf, keys[1], values[1]); + assertThat(txn.multiGet(readOptions, cfList, keys)).isEqualTo(values); + txn.commit(); + } + + try(final Transaction txn2 = dbContainer.beginTransaction()) { + try(final Transaction txn3 = dbContainer.beginTransaction()) { + assertThat(txn3.multiGetForUpdate(readOptions, cfList, keys)) + .isEqualTo(values); + + // NOTE: txn2 updates k1, during txn3 + txn2.put(testCf, keys[0], otherValue); + assertThat(txn2.get(testCf, readOptions, keys[0])) + .isEqualTo(otherValue); + txn2.commit(); + + try { + txn3.commit(); // should cause an exception! + } catch(final RocksDBException e) { + assertThat(e.getStatus().getCode()).isSameAs(Status.Code.Busy); + return; + } + } + } + + fail("Expected an exception for put after getForUpdate from conflicting" + + "transactions"); + } + } + + @Test + public void multiGetAsListForUpdate_cf_conflict() throws RocksDBException { + final byte[][] keys = new byte[][] {"key1".getBytes(UTF_8), "key2".getBytes(UTF_8)}; + final byte[][] values = new byte[][] {"value1".getBytes(UTF_8), "value2".getBytes(UTF_8)}; + final byte[] otherValue = "otherValue".getBytes(UTF_8); + + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + final List cfList = Arrays.asList(testCf, testCf); + + try (final Transaction txn = dbContainer.beginTransaction()) { + txn.put(testCf, keys[0], values[0]); + txn.put(testCf, keys[1], values[1]); + assertThat(txn.multiGetAsList(readOptions, cfList, Arrays.asList(keys))) + .containsExactly(values); + txn.commit(); + } + + try (final Transaction txn2 = dbContainer.beginTransaction()) { + try (final Transaction txn3 = dbContainer.beginTransaction()) { + assertThat(txn3.multiGetForUpdateAsList(readOptions, cfList, Arrays.asList(keys))) + .containsExactly(values); + + // NOTE: txn2 updates k1, during txn3 + txn2.put(testCf, keys[0], otherValue); + assertThat(txn2.get(testCf, readOptions, keys[0])).isEqualTo(otherValue); + txn2.commit(); + + try { + txn3.commit(); // should cause an exception! + } catch (final RocksDBException e) { + assertThat(e.getStatus().getCode()).isSameAs(Status.Code.Busy); + return; + } + } + } + + fail("Expected an exception for put after getForUpdate from conflicting" + + "transactions"); + } + } + + @Deprecated + @Test + public void multiGetForUpdate_conflict() throws RocksDBException { + final byte[][] keys = new byte[][] {"key1".getBytes(UTF_8), "key2".getBytes(UTF_8)}; + final byte[][] values = new byte[][] {"value1".getBytes(UTF_8), "value2".getBytes(UTF_8)}; + final byte[] otherValue = "otherValue".getBytes(UTF_8); + + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions()) { + try(final Transaction txn = dbContainer.beginTransaction()) { + txn.put(keys[0], values[0]); + txn.put(keys[1], values[1]); + assertThat(txn.multiGet(readOptions, keys)).isEqualTo(values); + txn.commit(); + } + + try(final Transaction txn2 = dbContainer.beginTransaction()) { + try(final Transaction txn3 = dbContainer.beginTransaction()) { + assertThat(txn3.multiGetForUpdate(readOptions, keys)) + .isEqualTo(values); + + // NOTE: txn2 updates k1, during txn3 + txn2.put(keys[0], otherValue); + assertThat(txn2.get(readOptions, keys[0])) + .isEqualTo(otherValue); + txn2.commit(); + + try { + txn3.commit(); // should cause an exception! + } catch(final RocksDBException e) { + assertThat(e.getStatus().getCode()).isSameAs(Status.Code.Busy); + return; + } + } + } + + fail("Expected an exception for put after getForUpdate from conflicting" + + "transactions"); + } + } + + @Test + public void multiGetasListForUpdate_conflict() throws RocksDBException { + final byte[][] keys = new byte[][] {"key1".getBytes(UTF_8), "key2".getBytes(UTF_8)}; + final byte[][] values = new byte[][] {"value1".getBytes(UTF_8), "value2".getBytes(UTF_8)}; + final byte[] otherValue = "otherValue".getBytes(UTF_8); + + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions()) { + try (final Transaction txn = dbContainer.beginTransaction()) { + txn.put(keys[0], values[0]); + txn.put(keys[1], values[1]); + assertThat(txn.multiGetAsList(readOptions, Arrays.asList(keys))).containsExactly(values); + txn.commit(); + } + + try (final Transaction txn2 = dbContainer.beginTransaction()) { + try (final Transaction txn3 = dbContainer.beginTransaction()) { + assertThat(txn3.multiGetForUpdateAsList(readOptions, Arrays.asList(keys))) + .containsExactly(values); + + // NOTE: txn2 updates k1, during txn3 + txn2.put(keys[0], otherValue); + assertThat(txn2.get(readOptions, keys[0])).isEqualTo(otherValue); + txn2.commit(); + + try { + txn3.commit(); // should cause an exception! + } catch (final RocksDBException e) { + assertThat(e.getStatus().getCode()).isSameAs(Status.Code.Busy); + return; + } + } + } + + fail("Expected an exception for put after getForUpdate from conflicting" + + "transactions"); + } + } + + @Test + public void undoGetForUpdate_cf_conflict() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + final byte[] v12 = "value12".getBytes(UTF_8); + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + + try(final Transaction txn = dbContainer.beginTransaction()) { + txn.put(testCf, k1, v1); + assertThat(txn.get(testCf, readOptions, k1)).isEqualTo(v1); + txn.commit(); + } + + try(final Transaction txn2 = dbContainer.beginTransaction()) { + try(final Transaction txn3 = dbContainer.beginTransaction()) { + assertThat(txn3.getForUpdate(readOptions, testCf, k1, true)).isEqualTo(v1); + + // undo the getForUpdate + txn3.undoGetForUpdate(testCf, k1); + + // NOTE: txn2 updates k1, during txn3 + txn2.put(testCf, k1, v12); + assertThat(txn2.get(testCf, readOptions, k1)).isEqualTo(v12); + txn2.commit(); + + // should not cause an exception + // because we undid the getForUpdate above! + txn3.commit(); + } + } + } + } + + @Test + public void undoGetForUpdate_conflict() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + final byte[] v12 = "value12".getBytes(UTF_8); + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions()) { + + try(final Transaction txn = dbContainer.beginTransaction()) { + txn.put(k1, v1); + assertThat(txn.get(readOptions, k1)).isEqualTo(v1); + txn.commit(); + } + + try(final Transaction txn2 = dbContainer.beginTransaction()) { + try(final Transaction txn3 = dbContainer.beginTransaction()) { + assertThat(txn3.getForUpdate(readOptions, k1, true)).isEqualTo(v1); + + // undo the getForUpdate + txn3.undoGetForUpdate(k1); + + // NOTE: txn2 updates k1, during txn3 + txn2.put(k1, v12); + assertThat(txn2.get(readOptions, k1)).isEqualTo(v12); + txn2.commit(); + + // should not cause an exception + // because we undid the getForUpdate above! + txn3.commit(); + } + } + } + } + + @Test + public void name() throws RocksDBException { + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.getName()).isEmpty(); + final String name = "my-transaction-" + rand.nextLong(); + + try { + txn.setName(name); + fail("Optimistic transactions cannot be named."); + } catch(final RocksDBException e) { + assertThat(e.getStatus().getCode()).isEqualTo(Status.Code.InvalidArgument); + } + } + } + + @Override + public OptimisticTransactionDBContainer startDb() + throws RocksDBException { + final DBOptions options = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + + final ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions(); + final List columnFamilyDescriptors = + Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor(TXN_TEST_COLUMN_FAMILY, + columnFamilyOptions)); + final List columnFamilyHandles = new ArrayList<>(); + + final OptimisticTransactionDB optimisticTxnDb; + try { + optimisticTxnDb = OptimisticTransactionDB.open( + options, dbFolder.getRoot().getAbsolutePath(), + columnFamilyDescriptors, columnFamilyHandles); + } catch(final RocksDBException e) { + columnFamilyOptions.close(); + options.close(); + throw e; + } + + final WriteOptions writeOptions = new WriteOptions(); + final OptimisticTransactionOptions optimisticTxnOptions = + new OptimisticTransactionOptions(); + + return new OptimisticTransactionDBContainer(optimisticTxnOptions, + writeOptions, columnFamilyHandles, optimisticTxnDb, columnFamilyOptions, + options); + } + + private static class OptimisticTransactionDBContainer + extends DBContainer { + + private final OptimisticTransactionOptions optimisticTxnOptions; + private final OptimisticTransactionDB optimisticTxnDb; + + public OptimisticTransactionDBContainer( + final OptimisticTransactionOptions optimisticTxnOptions, + final WriteOptions writeOptions, + final List columnFamilyHandles, + final OptimisticTransactionDB optimisticTxnDb, + final ColumnFamilyOptions columnFamilyOptions, + final DBOptions options) { + super(writeOptions, columnFamilyHandles, columnFamilyOptions, + options); + this.optimisticTxnOptions = optimisticTxnOptions; + this.optimisticTxnDb = optimisticTxnDb; + } + + @Override + public Transaction beginTransaction() { + return optimisticTxnDb.beginTransaction(writeOptions, + optimisticTxnOptions); + } + + @Override + public Transaction beginTransaction(final WriteOptions writeOptions) { + return optimisticTxnDb.beginTransaction(writeOptions, + optimisticTxnOptions); + } + + @Override + public void close() { + optimisticTxnOptions.close(); + writeOptions.close(); + for(final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandles) { + columnFamilyHandle.close(); + } + optimisticTxnDb.close(); + options.close(); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/OptionsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/OptionsTest.java new file mode 100644 index 0000000..4b59464 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/OptionsTest.java @@ -0,0 +1,1499 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.*; + +import java.nio.file.Paths; +import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.ClassRule; +import org.junit.Test; +import org.rocksdb.test.RemoveEmptyValueCompactionFilterFactory; + +public class OptionsTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + public static final Random rand = PlatformRandomHelper. + getPlatformSpecificRandomFactory(); + + @Test + public void copyConstructor() { + final Options origOpts = new Options(); + origOpts.setNumLevels(rand.nextInt(8)); + origOpts.setTargetFileSizeMultiplier(rand.nextInt(100)); + origOpts.setLevel0StopWritesTrigger(rand.nextInt(50)); + final Options copyOpts = new Options(origOpts); + assertThat(origOpts.numLevels()).isEqualTo(copyOpts.numLevels()); + assertThat(origOpts.targetFileSizeMultiplier()).isEqualTo(copyOpts.targetFileSizeMultiplier()); + assertThat(origOpts.level0StopWritesTrigger()).isEqualTo(copyOpts.level0StopWritesTrigger()); + } + + @Test + public void setIncreaseParallelism() { + try (final Options opt = new Options()) { + final int threads = Runtime.getRuntime().availableProcessors() * 2; + opt.setIncreaseParallelism(threads); + } + } + + @Test + public void writeBufferSize() throws RocksDBException { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setWriteBufferSize(longValue); + assertThat(opt.writeBufferSize()).isEqualTo(longValue); + } + } + + @Test + public void maxWriteBufferNumber() { + try (final Options opt = new Options()) { + final int intValue = rand.nextInt(); + opt.setMaxWriteBufferNumber(intValue); + assertThat(opt.maxWriteBufferNumber()).isEqualTo(intValue); + } + } + + @Test + public void minWriteBufferNumberToMerge() { + try (final Options opt = new Options()) { + final int intValue = rand.nextInt(); + opt.setMinWriteBufferNumberToMerge(intValue); + assertThat(opt.minWriteBufferNumberToMerge()).isEqualTo(intValue); + } + } + + @Test + public void numLevels() { + try (final Options opt = new Options()) { + final int intValue = rand.nextInt(); + opt.setNumLevels(intValue); + assertThat(opt.numLevels()).isEqualTo(intValue); + } + } + + @Test + public void levelZeroFileNumCompactionTrigger() { + try (final Options opt = new Options()) { + final int intValue = rand.nextInt(); + opt.setLevelZeroFileNumCompactionTrigger(intValue); + assertThat(opt.levelZeroFileNumCompactionTrigger()).isEqualTo(intValue); + } + } + + @Test + public void levelZeroSlowdownWritesTrigger() { + try (final Options opt = new Options()) { + final int intValue = rand.nextInt(); + opt.setLevelZeroSlowdownWritesTrigger(intValue); + assertThat(opt.levelZeroSlowdownWritesTrigger()).isEqualTo(intValue); + } + } + + @Test + public void levelZeroStopWritesTrigger() { + try (final Options opt = new Options()) { + final int intValue = rand.nextInt(); + opt.setLevelZeroStopWritesTrigger(intValue); + assertThat(opt.levelZeroStopWritesTrigger()).isEqualTo(intValue); + } + } + + @Test + public void targetFileSizeBase() { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setTargetFileSizeBase(longValue); + assertThat(opt.targetFileSizeBase()).isEqualTo(longValue); + } + } + + @Test + public void targetFileSizeMultiplier() { + try (final Options opt = new Options()) { + final int intValue = rand.nextInt(); + opt.setTargetFileSizeMultiplier(intValue); + assertThat(opt.targetFileSizeMultiplier()).isEqualTo(intValue); + } + } + + @Test + public void maxBytesForLevelBase() { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setMaxBytesForLevelBase(longValue); + assertThat(opt.maxBytesForLevelBase()).isEqualTo(longValue); + } + } + + @Test + public void levelCompactionDynamicLevelBytes() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setLevelCompactionDynamicLevelBytes(boolValue); + assertThat(opt.levelCompactionDynamicLevelBytes()) + .isEqualTo(boolValue); + } + } + + @Test + public void maxBytesForLevelMultiplier() { + try (final Options opt = new Options()) { + final double doubleValue = rand.nextDouble(); + opt.setMaxBytesForLevelMultiplier(doubleValue); + assertThat(opt.maxBytesForLevelMultiplier()).isEqualTo(doubleValue); + } + } + + @Test + public void maxBytesForLevelMultiplierAdditional() { + try (final Options opt = new Options()) { + final int intValue1 = rand.nextInt(); + final int intValue2 = rand.nextInt(); + final int[] ints = new int[]{intValue1, intValue2}; + opt.setMaxBytesForLevelMultiplierAdditional(ints); + assertThat(opt.maxBytesForLevelMultiplierAdditional()).isEqualTo(ints); + } + } + + @Test + public void maxCompactionBytes() { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setMaxCompactionBytes(longValue); + assertThat(opt.maxCompactionBytes()).isEqualTo(longValue); + } + } + + @Test + public void softPendingCompactionBytesLimit() { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setSoftPendingCompactionBytesLimit(longValue); + assertThat(opt.softPendingCompactionBytesLimit()).isEqualTo(longValue); + } + } + + @Test + public void hardPendingCompactionBytesLimit() { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setHardPendingCompactionBytesLimit(longValue); + assertThat(opt.hardPendingCompactionBytesLimit()).isEqualTo(longValue); + } + } + + @Test + public void level0FileNumCompactionTrigger() { + try (final Options opt = new Options()) { + final int intValue = rand.nextInt(); + opt.setLevel0FileNumCompactionTrigger(intValue); + assertThat(opt.level0FileNumCompactionTrigger()).isEqualTo(intValue); + } + } + + @Test + public void level0SlowdownWritesTrigger() { + try (final Options opt = new Options()) { + final int intValue = rand.nextInt(); + opt.setLevel0SlowdownWritesTrigger(intValue); + assertThat(opt.level0SlowdownWritesTrigger()).isEqualTo(intValue); + } + } + + @Test + public void level0StopWritesTrigger() { + try (final Options opt = new Options()) { + final int intValue = rand.nextInt(); + opt.setLevel0StopWritesTrigger(intValue); + assertThat(opt.level0StopWritesTrigger()).isEqualTo(intValue); + } + } + + @Test + public void arenaBlockSize() throws RocksDBException { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setArenaBlockSize(longValue); + assertThat(opt.arenaBlockSize()).isEqualTo(longValue); + } + } + + @Test + public void disableAutoCompactions() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setDisableAutoCompactions(boolValue); + assertThat(opt.disableAutoCompactions()).isEqualTo(boolValue); + } + } + + @Test + public void maxSequentialSkipInIterations() { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setMaxSequentialSkipInIterations(longValue); + assertThat(opt.maxSequentialSkipInIterations()).isEqualTo(longValue); + } + } + + @Test + public void inplaceUpdateSupport() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setInplaceUpdateSupport(boolValue); + assertThat(opt.inplaceUpdateSupport()).isEqualTo(boolValue); + } + } + + @Test + public void inplaceUpdateNumLocks() throws RocksDBException { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setInplaceUpdateNumLocks(longValue); + assertThat(opt.inplaceUpdateNumLocks()).isEqualTo(longValue); + } + } + + @Test + public void memtablePrefixBloomSizeRatio() { + try (final Options opt = new Options()) { + final double doubleValue = rand.nextDouble(); + opt.setMemtablePrefixBloomSizeRatio(doubleValue); + assertThat(opt.memtablePrefixBloomSizeRatio()).isEqualTo(doubleValue); + } + } + + @Test + public void experimentalMempurgeThreshold() { + try (final Options opt = new Options()) { + final double doubleValue = rand.nextDouble(); + opt.setExperimentalMempurgeThreshold(doubleValue); + assertThat(opt.experimentalMempurgeThreshold()).isEqualTo(doubleValue); + } + } + + @Test + public void memtableWholeKeyFiltering() { + try (final Options opt = new Options()) { + final boolean booleanValue = rand.nextBoolean(); + opt.setMemtableWholeKeyFiltering(booleanValue); + assertThat(opt.memtableWholeKeyFiltering()).isEqualTo(booleanValue); + } + } + + @Test + public void memtableHugePageSize() { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setMemtableHugePageSize(longValue); + assertThat(opt.memtableHugePageSize()).isEqualTo(longValue); + } + } + + @Test + public void bloomLocality() { + try (final Options opt = new Options()) { + final int intValue = rand.nextInt(); + opt.setBloomLocality(intValue); + assertThat(opt.bloomLocality()).isEqualTo(intValue); + } + } + + @Test + public void maxSuccessiveMerges() throws RocksDBException { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setMaxSuccessiveMerges(longValue); + assertThat(opt.maxSuccessiveMerges()).isEqualTo(longValue); + } + } + + @Test + public void optimizeFiltersForHits() { + try (final Options opt = new Options()) { + final boolean aBoolean = rand.nextBoolean(); + opt.setOptimizeFiltersForHits(aBoolean); + assertThat(opt.optimizeFiltersForHits()).isEqualTo(aBoolean); + } + } + + @Test + public void createIfMissing() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setCreateIfMissing(boolValue); + assertThat(opt.createIfMissing()). + isEqualTo(boolValue); + } + } + + @Test + public void createMissingColumnFamilies() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setCreateMissingColumnFamilies(boolValue); + assertThat(opt.createMissingColumnFamilies()). + isEqualTo(boolValue); + } + } + + @Test + public void errorIfExists() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setErrorIfExists(boolValue); + assertThat(opt.errorIfExists()).isEqualTo(boolValue); + } + } + + @Test + public void paranoidChecks() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setParanoidChecks(boolValue); + assertThat(opt.paranoidChecks()). + isEqualTo(boolValue); + } + } + + @Test + public void maxTotalWalSize() { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setMaxTotalWalSize(longValue); + assertThat(opt.maxTotalWalSize()). + isEqualTo(longValue); + } + } + + @Test + public void maxOpenFiles() { + try (final Options opt = new Options()) { + final int intValue = rand.nextInt(); + opt.setMaxOpenFiles(intValue); + assertThat(opt.maxOpenFiles()).isEqualTo(intValue); + } + } + + @Test + public void maxFileOpeningThreads() { + try (final Options opt = new Options()) { + final int intValue = rand.nextInt(); + opt.setMaxFileOpeningThreads(intValue); + assertThat(opt.maxFileOpeningThreads()).isEqualTo(intValue); + } + } + + @Test + public void useFsync() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setUseFsync(boolValue); + assertThat(opt.useFsync()).isEqualTo(boolValue); + } + } + + @Test + public void dbPaths() { + final List dbPaths = new ArrayList<>(); + dbPaths.add(new DbPath(Paths.get("/a"), 10)); + dbPaths.add(new DbPath(Paths.get("/b"), 100)); + dbPaths.add(new DbPath(Paths.get("/c"), 1000)); + + try (final Options opt = new Options()) { + assertThat(opt.dbPaths()).isEqualTo(Collections.emptyList()); + + opt.setDbPaths(dbPaths); + + assertThat(opt.dbPaths()).isEqualTo(dbPaths); + } + } + + @Test + public void dbLogDir() { + try (final Options opt = new Options()) { + final String str = "path/to/DbLogDir"; + opt.setDbLogDir(str); + assertThat(opt.dbLogDir()).isEqualTo(str); + } + } + + @Test + public void walDir() { + try (final Options opt = new Options()) { + final String str = "path/to/WalDir"; + opt.setWalDir(str); + assertThat(opt.walDir()).isEqualTo(str); + } + } + + @Test + public void deleteObsoleteFilesPeriodMicros() { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setDeleteObsoleteFilesPeriodMicros(longValue); + assertThat(opt.deleteObsoleteFilesPeriodMicros()). + isEqualTo(longValue); + } + } + + @SuppressWarnings("deprecated") + @Test + public void maxBackgroundCompactions() { + try (final Options opt = new Options()) { + final int intValue = rand.nextInt(); + opt.setMaxBackgroundCompactions(intValue); + assertThat(opt.maxBackgroundCompactions()). + isEqualTo(intValue); + } + } + + @Test + public void maxSubcompactions() { + try (final Options opt = new Options()) { + final int intValue = rand.nextInt(); + opt.setMaxSubcompactions(intValue); + assertThat(opt.maxSubcompactions()). + isEqualTo(intValue); + } + } + + @SuppressWarnings("deprecated") + @Test + public void maxBackgroundFlushes() { + try (final Options opt = new Options()) { + final int intValue = rand.nextInt(); + opt.setMaxBackgroundFlushes(intValue); + assertThat(opt.maxBackgroundFlushes()). + isEqualTo(intValue); + } + } + + @Test + public void maxBackgroundJobs() { + try (final Options opt = new Options()) { + final int intValue = rand.nextInt(); + opt.setMaxBackgroundJobs(intValue); + assertThat(opt.maxBackgroundJobs()).isEqualTo(intValue); + } + } + + @Test + public void maxLogFileSize() throws RocksDBException { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setMaxLogFileSize(longValue); + assertThat(opt.maxLogFileSize()).isEqualTo(longValue); + } + } + + @Test + public void logFileTimeToRoll() throws RocksDBException { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setLogFileTimeToRoll(longValue); + assertThat(opt.logFileTimeToRoll()). + isEqualTo(longValue); + } + } + + @Test + public void keepLogFileNum() throws RocksDBException { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setKeepLogFileNum(longValue); + assertThat(opt.keepLogFileNum()).isEqualTo(longValue); + } + } + + @Test + public void recycleLogFileNum() throws RocksDBException { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setRecycleLogFileNum(longValue); + assertThat(opt.recycleLogFileNum()).isEqualTo(longValue); + } + } + + @Test + public void maxManifestFileSize() { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setMaxManifestFileSize(longValue); + assertThat(opt.maxManifestFileSize()). + isEqualTo(longValue); + } + } + + @Test + public void tableCacheNumshardbits() { + try (final Options opt = new Options()) { + final int intValue = rand.nextInt(); + opt.setTableCacheNumshardbits(intValue); + assertThat(opt.tableCacheNumshardbits()). + isEqualTo(intValue); + } + } + + @Test + public void walSizeLimitMB() { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setWalSizeLimitMB(longValue); + assertThat(opt.walSizeLimitMB()).isEqualTo(longValue); + } + } + + @Test + public void walTtlSeconds() { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setWalTtlSeconds(longValue); + assertThat(opt.walTtlSeconds()).isEqualTo(longValue); + } + } + + @Test + public void manifestPreallocationSize() throws RocksDBException { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setManifestPreallocationSize(longValue); + assertThat(opt.manifestPreallocationSize()). + isEqualTo(longValue); + } + } + + @Test + public void useDirectReads() { + try(final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setUseDirectReads(boolValue); + assertThat(opt.useDirectReads()).isEqualTo(boolValue); + } + } + + @Test + public void useDirectIoForFlushAndCompaction() { + try(final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setUseDirectIoForFlushAndCompaction(boolValue); + assertThat(opt.useDirectIoForFlushAndCompaction()).isEqualTo(boolValue); + } + } + + @Test + public void allowFAllocate() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setAllowFAllocate(boolValue); + assertThat(opt.allowFAllocate()).isEqualTo(boolValue); + } + } + + @Test + public void allowMmapReads() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setAllowMmapReads(boolValue); + assertThat(opt.allowMmapReads()).isEqualTo(boolValue); + } + } + + @Test + public void allowMmapWrites() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setAllowMmapWrites(boolValue); + assertThat(opt.allowMmapWrites()).isEqualTo(boolValue); + } + } + + @Test + public void isFdCloseOnExec() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setIsFdCloseOnExec(boolValue); + assertThat(opt.isFdCloseOnExec()).isEqualTo(boolValue); + } + } + + @Test + public void statsDumpPeriodSec() { + try (final Options opt = new Options()) { + final int intValue = rand.nextInt(); + opt.setStatsDumpPeriodSec(intValue); + assertThat(opt.statsDumpPeriodSec()).isEqualTo(intValue); + } + } + + @Test + public void statsPersistPeriodSec() { + try (final Options opt = new Options()) { + final int intValue = rand.nextInt(); + opt.setStatsPersistPeriodSec(intValue); + assertThat(opt.statsPersistPeriodSec()).isEqualTo(intValue); + } + } + + @Test + public void statsHistoryBufferSize() { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setStatsHistoryBufferSize(longValue); + assertThat(opt.statsHistoryBufferSize()).isEqualTo(longValue); + } + } + + @Test + public void adviseRandomOnOpen() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setAdviseRandomOnOpen(boolValue); + assertThat(opt.adviseRandomOnOpen()).isEqualTo(boolValue); + } + } + + @Test + public void dbWriteBufferSize() { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setDbWriteBufferSize(longValue); + assertThat(opt.dbWriteBufferSize()).isEqualTo(longValue); + } + } + + @Test + public void setWriteBufferManager() throws RocksDBException { + try (final Options opt = new Options(); final Cache cache = new LRUCache(1024 * 1024); + final WriteBufferManager writeBufferManager = new WriteBufferManager(2000L, cache)) { + opt.setWriteBufferManager(writeBufferManager); + assertThat(opt.writeBufferManager()).isEqualTo(writeBufferManager); + } + } + + @Test + public void setWriteBufferManagerWithZeroBufferSize() throws RocksDBException { + try (final Options opt = new Options(); final Cache cache = new LRUCache(1024 * 1024); + final WriteBufferManager writeBufferManager = new WriteBufferManager(0L, cache)) { + opt.setWriteBufferManager(writeBufferManager); + assertThat(opt.writeBufferManager()).isEqualTo(writeBufferManager); + } + } + + @Test + public void setWriteBufferManagerWithAllowStall() throws RocksDBException { + try (final Options opt = new Options(); final Cache cache = new LRUCache(1024 * 1024); + final WriteBufferManager writeBufferManager = new WriteBufferManager(2000L, cache, true)) { + opt.setWriteBufferManager(writeBufferManager); + assertThat(opt.writeBufferManager()).isEqualTo(writeBufferManager); + assertThat(opt.writeBufferManager().allowStall()).isEqualTo(true); + } + } + + @SuppressWarnings("deprecated") + @Test + public void accessHintOnCompactionStart() { + try (final Options opt = new Options()) { + final AccessHint accessHint = AccessHint.SEQUENTIAL; + opt.setAccessHintOnCompactionStart(accessHint); + assertThat(opt.accessHintOnCompactionStart()).isEqualTo(accessHint); + } + } + + @Test + public void compactionReadaheadSize() { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setCompactionReadaheadSize(longValue); + assertThat(opt.compactionReadaheadSize()).isEqualTo(longValue); + } + } + + @Test + public void randomAccessMaxBufferSize() { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setRandomAccessMaxBufferSize(longValue); + assertThat(opt.randomAccessMaxBufferSize()).isEqualTo(longValue); + } + } + + @Test + public void writableFileMaxBufferSize() { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setWritableFileMaxBufferSize(longValue); + assertThat(opt.writableFileMaxBufferSize()).isEqualTo(longValue); + } + } + + @Test + public void useAdaptiveMutex() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setUseAdaptiveMutex(boolValue); + assertThat(opt.useAdaptiveMutex()).isEqualTo(boolValue); + } + } + + @Test + public void bytesPerSync() { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setBytesPerSync(longValue); + assertThat(opt.bytesPerSync()).isEqualTo(longValue); + } + } + + @Test + public void walBytesPerSync() { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setWalBytesPerSync(longValue); + assertThat(opt.walBytesPerSync()).isEqualTo(longValue); + } + } + + @Test + public void strictBytesPerSync() { + try (final Options opt = new Options()) { + assertThat(opt.strictBytesPerSync()).isFalse(); + opt.setStrictBytesPerSync(true); + assertThat(opt.strictBytesPerSync()).isTrue(); + } + } + + @Test + public void enableThreadTracking() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setEnableThreadTracking(boolValue); + assertThat(opt.enableThreadTracking()).isEqualTo(boolValue); + } + } + + @Test + public void delayedWriteRate() { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setDelayedWriteRate(longValue); + assertThat(opt.delayedWriteRate()).isEqualTo(longValue); + } + } + + @Test + public void enablePipelinedWrite() { + try(final Options opt = new Options()) { + assertThat(opt.enablePipelinedWrite()).isFalse(); + opt.setEnablePipelinedWrite(true); + assertThat(opt.enablePipelinedWrite()).isTrue(); + } + } + + @Test + public void unordredWrite() { + try(final Options opt = new Options()) { + assertThat(opt.unorderedWrite()).isFalse(); + opt.setUnorderedWrite(true); + assertThat(opt.unorderedWrite()).isTrue(); + } + } + + @Test + public void allowConcurrentMemtableWrite() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setAllowConcurrentMemtableWrite(boolValue); + assertThat(opt.allowConcurrentMemtableWrite()).isEqualTo(boolValue); + } + } + + @Test + public void enableWriteThreadAdaptiveYield() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setEnableWriteThreadAdaptiveYield(boolValue); + assertThat(opt.enableWriteThreadAdaptiveYield()).isEqualTo(boolValue); + } + } + + @Test + public void writeThreadMaxYieldUsec() { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setWriteThreadMaxYieldUsec(longValue); + assertThat(opt.writeThreadMaxYieldUsec()).isEqualTo(longValue); + } + } + + @Test + public void writeThreadSlowYieldUsec() { + try (final Options opt = new Options()) { + final long longValue = rand.nextLong(); + opt.setWriteThreadSlowYieldUsec(longValue); + assertThat(opt.writeThreadSlowYieldUsec()).isEqualTo(longValue); + } + } + + @Test + public void skipStatsUpdateOnDbOpen() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setSkipStatsUpdateOnDbOpen(boolValue); + assertThat(opt.skipStatsUpdateOnDbOpen()).isEqualTo(boolValue); + } + } + + @Test + public void walRecoveryMode() { + try (final Options opt = new Options()) { + for (final WALRecoveryMode walRecoveryMode : WALRecoveryMode.values()) { + opt.setWalRecoveryMode(walRecoveryMode); + assertThat(opt.walRecoveryMode()).isEqualTo(walRecoveryMode); + } + } + } + + @Test + public void allow2pc() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setAllow2pc(boolValue); + assertThat(opt.allow2pc()).isEqualTo(boolValue); + } + } + + @Test + public void rowCache() { + try (final Options opt = new Options()) { + assertThat(opt.rowCache()).isNull(); + + try(final Cache lruCache = new LRUCache(1000)) { + opt.setRowCache(lruCache); + assertThat(opt.rowCache()).isEqualTo(lruCache); + } + + try(final Cache clockCache = new ClockCache(1000)) { + opt.setRowCache(clockCache); + assertThat(opt.rowCache()).isEqualTo(clockCache); + } + } + } + + @Test + public void walFilter() { + try (final Options opt = new Options()) { + assertThat(opt.walFilter()).isNull(); + + try (final AbstractWalFilter walFilter = new AbstractWalFilter() { + @Override + public void columnFamilyLogNumberMap( + final Map cfLognumber, + final Map cfNameId) { + // no-op + } + + @Override + public LogRecordFoundResult logRecordFound(final long logNumber, + final String logFileName, final WriteBatch batch, + final WriteBatch newBatch) { + return new LogRecordFoundResult( + WalProcessingOption.CONTINUE_PROCESSING, false); + } + + @Override + public String name() { + return "test-wal-filter"; + } + }) { + opt.setWalFilter(walFilter); + assertThat(opt.walFilter()).isEqualTo(walFilter); + } + } + } + + @Test + public void failIfOptionsFileError() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setFailIfOptionsFileError(boolValue); + assertThat(opt.failIfOptionsFileError()).isEqualTo(boolValue); + } + } + + @Test + public void dumpMallocStats() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setDumpMallocStats(boolValue); + assertThat(opt.dumpMallocStats()).isEqualTo(boolValue); + } + } + + @Test + public void avoidFlushDuringRecovery() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setAvoidFlushDuringRecovery(boolValue); + assertThat(opt.avoidFlushDuringRecovery()).isEqualTo(boolValue); + } + } + + @Test + public void avoidFlushDuringShutdown() { + try (final Options opt = new Options()) { + final boolean boolValue = rand.nextBoolean(); + opt.setAvoidFlushDuringShutdown(boolValue); + assertThat(opt.avoidFlushDuringShutdown()).isEqualTo(boolValue); + } + } + + + @Test + public void allowIngestBehind() { + try (final Options opt = new Options()) { + assertThat(opt.allowIngestBehind()).isFalse(); + opt.setAllowIngestBehind(true); + assertThat(opt.allowIngestBehind()).isTrue(); + } + } + + @Test + public void twoWriteQueues() { + try (final Options opt = new Options()) { + assertThat(opt.twoWriteQueues()).isFalse(); + opt.setTwoWriteQueues(true); + assertThat(opt.twoWriteQueues()).isTrue(); + } + } + + @Test + public void manualWalFlush() { + try (final Options opt = new Options()) { + assertThat(opt.manualWalFlush()).isFalse(); + opt.setManualWalFlush(true); + assertThat(opt.manualWalFlush()).isTrue(); + } + } + + @Test + public void atomicFlush() { + try (final Options opt = new Options()) { + assertThat(opt.atomicFlush()).isFalse(); + opt.setAtomicFlush(true); + assertThat(opt.atomicFlush()).isTrue(); + } + } + + @Test + public void env() { + try (final Options options = new Options(); + final Env env = Env.getDefault()) { + options.setEnv(env); + assertThat(options.getEnv()).isSameAs(env); + } + } + + @Test + public void linkageOfPrepMethods() { + try (final Options options = new Options()) { + options.optimizeUniversalStyleCompaction(); + options.optimizeUniversalStyleCompaction(4000); + options.optimizeLevelStyleCompaction(); + options.optimizeLevelStyleCompaction(3000); + options.optimizeForPointLookup(10); + options.optimizeForSmallDb(); + options.prepareForBulkLoad(); + } + } + + @Test + public void compressionTypes() { + try (final Options options = new Options()) { + for (final CompressionType compressionType : + CompressionType.values()) { + options.setCompressionType(compressionType); + assertThat(options.compressionType()). + isEqualTo(compressionType); + assertThat(CompressionType.valueOf("NO_COMPRESSION")). + isEqualTo(CompressionType.NO_COMPRESSION); + } + } + } + + @Test + public void prepopulateBlobCache() { + try (final Options options = new Options()) { + for (final PrepopulateBlobCache prepopulateBlobCache : PrepopulateBlobCache.values()) { + options.setPrepopulateBlobCache(prepopulateBlobCache); + assertThat(options.prepopulateBlobCache()).isEqualTo(prepopulateBlobCache); + assertThat(PrepopulateBlobCache.valueOf("PREPOPULATE_BLOB_DISABLE")) + .isEqualTo(PrepopulateBlobCache.PREPOPULATE_BLOB_DISABLE); + } + } + } + + @Test + public void compressionPerLevel() { + try (final Options options = new Options()) { + assertThat(options.compressionPerLevel()).isEmpty(); + List compressionTypeList = + new ArrayList<>(); + for (int i = 0; i < options.numLevels(); i++) { + compressionTypeList.add(CompressionType.NO_COMPRESSION); + } + options.setCompressionPerLevel(compressionTypeList); + compressionTypeList = options.compressionPerLevel(); + for (final CompressionType compressionType : compressionTypeList) { + assertThat(compressionType).isEqualTo( + CompressionType.NO_COMPRESSION); + } + } + } + + @Test + public void differentCompressionsPerLevel() { + try (final Options options = new Options()) { + options.setNumLevels(3); + + assertThat(options.compressionPerLevel()).isEmpty(); + List compressionTypeList = new ArrayList<>(); + + compressionTypeList.add(CompressionType.BZLIB2_COMPRESSION); + compressionTypeList.add(CompressionType.SNAPPY_COMPRESSION); + compressionTypeList.add(CompressionType.LZ4_COMPRESSION); + + options.setCompressionPerLevel(compressionTypeList); + compressionTypeList = options.compressionPerLevel(); + + assertThat(compressionTypeList.size()).isEqualTo(3); + assertThat(compressionTypeList). + containsExactly( + CompressionType.BZLIB2_COMPRESSION, + CompressionType.SNAPPY_COMPRESSION, + CompressionType.LZ4_COMPRESSION); + + } + } + + @Test + public void bottommostCompressionType() { + try (final Options options = new Options()) { + assertThat(options.bottommostCompressionType()) + .isEqualTo(CompressionType.DISABLE_COMPRESSION_OPTION); + + for (final CompressionType compressionType : CompressionType.values()) { + options.setBottommostCompressionType(compressionType); + assertThat(options.bottommostCompressionType()) + .isEqualTo(compressionType); + } + } + } + + @Test + public void bottommostCompressionOptions() { + try (final Options options = new Options(); + final CompressionOptions bottommostCompressionOptions = new CompressionOptions() + .setMaxDictBytes(123)) { + + options.setBottommostCompressionOptions(bottommostCompressionOptions); + assertThat(options.bottommostCompressionOptions()) + .isEqualTo(bottommostCompressionOptions); + assertThat(options.bottommostCompressionOptions().maxDictBytes()) + .isEqualTo(123); + } + } + + @Test + public void compressionOptions() { + try (final Options options = new Options(); + final CompressionOptions compressionOptions = new CompressionOptions() + .setMaxDictBytes(123)) { + + options.setCompressionOptions(compressionOptions); + assertThat(options.compressionOptions()) + .isEqualTo(compressionOptions); + assertThat(options.compressionOptions().maxDictBytes()) + .isEqualTo(123); + } + } + + @Test + public void compactionStyles() { + try (final Options options = new Options()) { + for (final CompactionStyle compactionStyle : + CompactionStyle.values()) { + options.setCompactionStyle(compactionStyle); + assertThat(options.compactionStyle()). + isEqualTo(compactionStyle); + assertThat(CompactionStyle.valueOf("FIFO")). + isEqualTo(CompactionStyle.FIFO); + } + } + } + + @Test + public void maxTableFilesSizeFIFO() { + try (final Options opt = new Options()) { + long longValue = rand.nextLong(); + // Size has to be positive + longValue = (longValue < 0) ? -longValue : longValue; + longValue = (longValue == 0) ? longValue + 1 : longValue; + opt.setMaxTableFilesSizeFIFO(longValue); + assertThat(opt.maxTableFilesSizeFIFO()). + isEqualTo(longValue); + } + } + + @Test + public void rateLimiter() { + try (final Options options = new Options(); + final Options anotherOptions = new Options(); + final RateLimiter rateLimiter = + new RateLimiter(1000, 100 * 1000, 1)) { + options.setRateLimiter(rateLimiter); + // Test with parameter initialization + anotherOptions.setRateLimiter( + new RateLimiter(1000)); + } + } + + @Test + public void sstFileManager() throws RocksDBException { + try (final Options options = new Options(); + final SstFileManager sstFileManager = + new SstFileManager(Env.getDefault())) { + options.setSstFileManager(sstFileManager); + } + } + + @Test + public void shouldSetTestPrefixExtractor() { + try (final Options options = new Options()) { + options.useFixedLengthPrefixExtractor(100); + options.useFixedLengthPrefixExtractor(10); + } + } + + @Test + public void shouldSetTestCappedPrefixExtractor() { + try (final Options options = new Options()) { + options.useCappedPrefixExtractor(100); + options.useCappedPrefixExtractor(10); + } + } + + @Test + public void shouldTestMemTableFactoryName() + throws RocksDBException { + try (final Options options = new Options()) { + options.setMemTableConfig(new VectorMemTableConfig()); + assertThat(options.memTableFactoryName()). + isEqualTo("VectorRepFactory"); + options.setMemTableConfig( + new HashLinkedListMemTableConfig()); + assertThat(options.memTableFactoryName()). + isEqualTo("HashLinkedListRepFactory"); + } + } + + @Test + public void statistics() { + try(final Options options = new Options()) { + final Statistics statistics = options.statistics(); + assertThat(statistics).isNull(); + } + + try(final Statistics statistics = new Statistics(); + final Options options = new Options().setStatistics(statistics); + final Statistics stats = options.statistics()) { + assertThat(stats).isNotNull(); + } + } + + @Test + public void maxWriteBufferNumberToMaintain() { + try (final Options options = new Options()) { + int intValue = rand.nextInt(); + // Size has to be positive + intValue = (intValue < 0) ? -intValue : intValue; + intValue = (intValue == 0) ? intValue + 1 : intValue; + options.setMaxWriteBufferNumberToMaintain(intValue); + assertThat(options.maxWriteBufferNumberToMaintain()). + isEqualTo(intValue); + } + } + + @Test + public void compactionPriorities() { + try (final Options options = new Options()) { + for (final CompactionPriority compactionPriority : + CompactionPriority.values()) { + options.setCompactionPriority(compactionPriority); + assertThat(options.compactionPriority()). + isEqualTo(compactionPriority); + } + } + } + + @Test + public void reportBgIoStats() { + try (final Options options = new Options()) { + final boolean booleanValue = true; + options.setReportBgIoStats(booleanValue); + assertThat(options.reportBgIoStats()). + isEqualTo(booleanValue); + } + } + + @Test + public void ttl() { + try (final Options options = new Options()) { + options.setTtl(1000 * 60); + assertThat(options.ttl()). + isEqualTo(1000 * 60); + } + } + + @Test + public void periodicCompactionSeconds() { + try (final Options options = new Options()) { + options.setPeriodicCompactionSeconds(1000 * 60); + assertThat(options.periodicCompactionSeconds()).isEqualTo(1000 * 60); + } + } + + @Test + public void compactionOptionsUniversal() { + try (final Options options = new Options(); + final CompactionOptionsUniversal optUni = new CompactionOptionsUniversal() + .setCompressionSizePercent(7)) { + options.setCompactionOptionsUniversal(optUni); + assertThat(options.compactionOptionsUniversal()). + isEqualTo(optUni); + assertThat(options.compactionOptionsUniversal().compressionSizePercent()) + .isEqualTo(7); + } + } + + @Test + public void compactionOptionsFIFO() { + try (final Options options = new Options(); + final CompactionOptionsFIFO optFifo = new CompactionOptionsFIFO() + .setMaxTableFilesSize(2000)) { + options.setCompactionOptionsFIFO(optFifo); + assertThat(options.compactionOptionsFIFO()). + isEqualTo(optFifo); + assertThat(options.compactionOptionsFIFO().maxTableFilesSize()) + .isEqualTo(2000); + } + } + + @Test + public void forceConsistencyChecks() { + try (final Options options = new Options()) { + final boolean booleanValue = true; + options.setForceConsistencyChecks(booleanValue); + assertThat(options.forceConsistencyChecks()). + isEqualTo(booleanValue); + } + } + + @Test + public void compactionFilter() { + try(final Options options = new Options(); + final RemoveEmptyValueCompactionFilter cf = new RemoveEmptyValueCompactionFilter()) { + options.setCompactionFilter(cf); + assertThat(options.compactionFilter()).isEqualTo(cf); + } + } + + @Test + public void compactionFilterFactory() { + try(final Options options = new Options(); + final RemoveEmptyValueCompactionFilterFactory cff = new RemoveEmptyValueCompactionFilterFactory()) { + options.setCompactionFilterFactory(cff); + assertThat(options.compactionFilterFactory()).isEqualTo(cff); + } + } + + @Test + public void compactionThreadLimiter() { + try (final Options options = new Options(); + final ConcurrentTaskLimiter compactionThreadLimiter = + new ConcurrentTaskLimiterImpl("name", 3)) { + options.setCompactionThreadLimiter(compactionThreadLimiter); + assertThat(options.compactionThreadLimiter()).isEqualTo(compactionThreadLimiter); + } + } + + @Test + public void oldDefaults() { + try (final Options options = new Options()) { + options.oldDefaults(4, 6); + assertThat(options.writeBufferSize()).isEqualTo(4 << 20); + assertThat(options.compactionPriority()).isEqualTo(CompactionPriority.ByCompensatedSize); + assertThat(options.targetFileSizeBase()).isEqualTo(2 * 1048576); + assertThat(options.maxBytesForLevelBase()).isEqualTo(10 * 1048576); + assertThat(options.softPendingCompactionBytesLimit()).isEqualTo(0); + assertThat(options.hardPendingCompactionBytesLimit()).isEqualTo(0); + assertThat(options.level0StopWritesTrigger()).isEqualTo(24); + } + } + + @Test + public void optimizeForSmallDbWithCache() { + try (final Options options = new Options(); final Cache cache = new LRUCache(1024)) { + assertThat(options.optimizeForSmallDb(cache)).isEqualTo(options); + } + } + + @Test + public void cfPaths() { + try (final Options options = new Options()) { + final List paths = Arrays.asList( + new DbPath(Paths.get("test1"), 2 << 25), new DbPath(Paths.get("/test2/path"), 2 << 25)); + assertThat(options.cfPaths()).isEqualTo(Collections.emptyList()); + assertThat(options.setCfPaths(paths)).isEqualTo(options); + assertThat(options.cfPaths()).isEqualTo(paths); + } + } + + @Test + public void avoidUnnecessaryBlockingIO() { + try (final Options options = new Options()) { + assertThat(options.avoidUnnecessaryBlockingIO()).isEqualTo(false); + assertThat(options.setAvoidUnnecessaryBlockingIO(true)).isEqualTo(options); + assertThat(options.avoidUnnecessaryBlockingIO()).isEqualTo(true); + } + } + + @Test + public void persistStatsToDisk() { + try (final Options options = new Options()) { + assertThat(options.persistStatsToDisk()).isEqualTo(false); + assertThat(options.setPersistStatsToDisk(true)).isEqualTo(options); + assertThat(options.persistStatsToDisk()).isEqualTo(true); + } + } + + @Test + public void writeDbidToManifest() { + try (final Options options = new Options()) { + assertThat(options.writeDbidToManifest()).isEqualTo(false); + assertThat(options.setWriteDbidToManifest(true)).isEqualTo(options); + assertThat(options.writeDbidToManifest()).isEqualTo(true); + } + } + + @Test + public void logReadaheadSize() { + try (final Options options = new Options()) { + assertThat(options.logReadaheadSize()).isEqualTo(0); + final int size = 1024 * 1024 * 100; + assertThat(options.setLogReadaheadSize(size)).isEqualTo(options); + assertThat(options.logReadaheadSize()).isEqualTo(size); + } + } + + @Test + public void bestEffortsRecovery() { + try (final Options options = new Options()) { + assertThat(options.bestEffortsRecovery()).isEqualTo(false); + assertThat(options.setBestEffortsRecovery(true)).isEqualTo(options); + assertThat(options.bestEffortsRecovery()).isEqualTo(true); + } + } + + @Test + public void maxBgerrorResumeCount() { + try (final Options options = new Options()) { + final int INT_MAX = 2147483647; + assertThat(options.maxBgerrorResumeCount()).isEqualTo(INT_MAX); + assertThat(options.setMaxBgErrorResumeCount(-1)).isEqualTo(options); + assertThat(options.maxBgerrorResumeCount()).isEqualTo(-1); + } + } + + @Test + public void bgerrorResumeRetryInterval() { + try (final Options options = new Options()) { + assertThat(options.bgerrorResumeRetryInterval()).isEqualTo(1000000); + final long newRetryInterval = 24 * 3600 * 1000000L; + assertThat(options.setBgerrorResumeRetryInterval(newRetryInterval)).isEqualTo(options); + assertThat(options.bgerrorResumeRetryInterval()).isEqualTo(newRetryInterval); + } + } + + @Test + public void maxWriteBatchGroupSizeBytes() { + try (final Options options = new Options()) { + assertThat(options.maxWriteBatchGroupSizeBytes()).isEqualTo(1024 * 1024); + final long size = 1024 * 1024 * 1024 * 10L; + assertThat(options.setMaxWriteBatchGroupSizeBytes(size)).isEqualTo(options); + assertThat(options.maxWriteBatchGroupSizeBytes()).isEqualTo(size); + } + } + + @Test + public void skipCheckingSstFileSizesOnDbOpen() { + try (final Options options = new Options()) { + assertThat(options.skipCheckingSstFileSizesOnDbOpen()).isEqualTo(false); + assertThat(options.setSkipCheckingSstFileSizesOnDbOpen(true)).isEqualTo(options); + assertThat(options.skipCheckingSstFileSizesOnDbOpen()).isEqualTo(true); + } + } + + @Test + public void memtableMaxRangeDeletions() { + try (final Options options = new Options()) { + assertThat(options.memtableMaxRangeDeletions()).isEqualTo(0); + final int val = 32; + assertThat(options.setMemtableMaxRangeDeletions(val)).isEqualTo(options); + assertThat(options.memtableMaxRangeDeletions()).isEqualTo(val); + } + } + + @Test + public void eventListeners() { + final AtomicBoolean wasCalled1 = new AtomicBoolean(); + final AtomicBoolean wasCalled2 = new AtomicBoolean(); + try (final Options options = new Options(); + final AbstractEventListener el1 = + new AbstractEventListener() { + @Override + public void onTableFileDeleted(final TableFileDeletionInfo tableFileDeletionInfo) { + wasCalled1.set(true); + } + }; + final AbstractEventListener el2 = + new AbstractEventListener() { + @Override + public void onMemTableSealed(final MemTableInfo memTableInfo) { + wasCalled2.set(true); + } + }) { + assertThat(options.setListeners(Arrays.asList(el1, el2))).isEqualTo(options); + final List listeners = options.listeners(); + assertEquals(el1, listeners.get(0)); + assertEquals(el2, listeners.get(1)); + options.setListeners(Collections.emptyList()); + listeners.get(0).onTableFileDeleted(null); + assertTrue(wasCalled1.get()); + listeners.get(1).onMemTableSealed(null); + assertTrue(wasCalled2.get()); + final List listeners2 = options.listeners(); + assertNotNull(listeners2); + assertEquals(0, listeners2.size()); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/OptionsUtilTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/OptionsUtilTest.java new file mode 100644 index 0000000..c2975ea --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/OptionsUtilTest.java @@ -0,0 +1,129 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.util.*; + +import static org.assertj.core.api.Assertions.assertThat; + +public class OptionsUtilTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = new RocksNativeLibraryResource(); + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + enum TestAPI { LOAD_LATEST_OPTIONS, LOAD_OPTIONS_FROM_FILE } + + @Test + public void loadLatestOptions() throws RocksDBException { + verifyOptions(TestAPI.LOAD_LATEST_OPTIONS); + } + + @Test + public void loadOptionsFromFile() throws RocksDBException { + verifyOptions(TestAPI.LOAD_OPTIONS_FROM_FILE); + } + + @Test + public void getLatestOptionsFileName() throws RocksDBException { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, dbPath)) { + assertThat(db).isNotNull(); + } + + final String fName = OptionsUtil.getLatestOptionsFileName(dbPath, Env.getDefault()); + assertThat(fName).isNotNull(); + assert (fName.startsWith("OPTIONS-")); + // System.out.println("latest options fileName: " + fName); + } + + private void verifyOptions(final TestAPI apiType) throws RocksDBException { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + final Options options = new Options() + .setCreateIfMissing(true) + .setParanoidChecks(false) + .setMaxOpenFiles(478) + .setDelayedWriteRate(1234567L); + final ColumnFamilyOptions baseDefaultCFOpts = new ColumnFamilyOptions(); + final byte[] secondCFName = "new_cf".getBytes(); + final ColumnFamilyOptions baseSecondCFOpts = + new ColumnFamilyOptions() + .setWriteBufferSize(70 * 1024) + .setMaxWriteBufferNumber(7) + .setMaxBytesForLevelBase(53 * 1024 * 1024) + .setLevel0FileNumCompactionTrigger(3) + .setLevel0SlowdownWritesTrigger(51) + .setBottommostCompressionType(CompressionType.ZSTD_COMPRESSION); + + // Create a database with a new column family + try (final RocksDB db = RocksDB.open(options, dbPath)) { + assertThat(db).isNotNull(); + + // create column family + try (final ColumnFamilyHandle columnFamilyHandle = + db.createColumnFamily(new ColumnFamilyDescriptor(secondCFName, baseSecondCFOpts))) { + assert(columnFamilyHandle != null); + } + } + + // Read the options back and verify + final DBOptions dbOptions = new DBOptions(); + final ConfigOptions configOptions = + new ConfigOptions().setIgnoreUnknownOptions(false).setInputStringsEscaped(true).setEnv( + Env.getDefault()); + final List cfDescs = new ArrayList<>(); + String path = dbPath; + if (apiType == TestAPI.LOAD_LATEST_OPTIONS) { + OptionsUtil.loadLatestOptions(configOptions, path, dbOptions, cfDescs); + } else if (apiType == TestAPI.LOAD_OPTIONS_FROM_FILE) { + path = dbPath + "/" + OptionsUtil.getLatestOptionsFileName(dbPath, Env.getDefault()); + OptionsUtil.loadOptionsFromFile(configOptions, path, dbOptions, cfDescs); + } + + assertThat(dbOptions.createIfMissing()).isEqualTo(options.createIfMissing()); + assertThat(dbOptions.paranoidChecks()).isEqualTo(options.paranoidChecks()); + assertThat(dbOptions.maxOpenFiles()).isEqualTo(options.maxOpenFiles()); + assertThat(dbOptions.delayedWriteRate()).isEqualTo(options.delayedWriteRate()); + + assertThat(cfDescs.size()).isEqualTo(2); + assertThat(cfDescs.get(0)).isNotNull(); + assertThat(cfDescs.get(1)).isNotNull(); + assertThat(cfDescs.get(0).getName()).isEqualTo(RocksDB.DEFAULT_COLUMN_FAMILY); + assertThat(cfDescs.get(1).getName()).isEqualTo(secondCFName); + + final ColumnFamilyOptions defaultCFOpts = cfDescs.get(0).getOptions(); + assertThat(defaultCFOpts.writeBufferSize()).isEqualTo(baseDefaultCFOpts.writeBufferSize()); + assertThat(defaultCFOpts.maxWriteBufferNumber()) + .isEqualTo(baseDefaultCFOpts.maxWriteBufferNumber()); + assertThat(defaultCFOpts.maxBytesForLevelBase()) + .isEqualTo(baseDefaultCFOpts.maxBytesForLevelBase()); + assertThat(defaultCFOpts.level0FileNumCompactionTrigger()) + .isEqualTo(baseDefaultCFOpts.level0FileNumCompactionTrigger()); + assertThat(defaultCFOpts.level0SlowdownWritesTrigger()) + .isEqualTo(baseDefaultCFOpts.level0SlowdownWritesTrigger()); + assertThat(defaultCFOpts.bottommostCompressionType()) + .isEqualTo(baseDefaultCFOpts.bottommostCompressionType()); + + final ColumnFamilyOptions secondCFOpts = cfDescs.get(1).getOptions(); + assertThat(secondCFOpts.writeBufferSize()).isEqualTo(baseSecondCFOpts.writeBufferSize()); + assertThat(secondCFOpts.maxWriteBufferNumber()) + .isEqualTo(baseSecondCFOpts.maxWriteBufferNumber()); + assertThat(secondCFOpts.maxBytesForLevelBase()) + .isEqualTo(baseSecondCFOpts.maxBytesForLevelBase()); + assertThat(secondCFOpts.level0FileNumCompactionTrigger()) + .isEqualTo(baseSecondCFOpts.level0FileNumCompactionTrigger()); + assertThat(secondCFOpts.level0SlowdownWritesTrigger()) + .isEqualTo(baseSecondCFOpts.level0SlowdownWritesTrigger()); + assertThat(secondCFOpts.bottommostCompressionType()) + .isEqualTo(baseSecondCFOpts.bottommostCompressionType()); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/PlainTableConfigTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/PlainTableConfigTest.java new file mode 100644 index 0000000..827eb79 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/PlainTableConfigTest.java @@ -0,0 +1,89 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class PlainTableConfigTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Test + public void keySize() { + final PlainTableConfig plainTableConfig = new PlainTableConfig(); + plainTableConfig.setKeySize(5); + assertThat(plainTableConfig.keySize()). + isEqualTo(5); + } + + @Test + public void bloomBitsPerKey() { + final PlainTableConfig plainTableConfig = new PlainTableConfig(); + plainTableConfig.setBloomBitsPerKey(11); + assertThat(plainTableConfig.bloomBitsPerKey()). + isEqualTo(11); + } + + @Test + public void hashTableRatio() { + final PlainTableConfig plainTableConfig = new PlainTableConfig(); + plainTableConfig.setHashTableRatio(0.95); + assertThat(plainTableConfig.hashTableRatio()). + isEqualTo(0.95); + } + + @Test + public void indexSparseness() { + final PlainTableConfig plainTableConfig = new PlainTableConfig(); + plainTableConfig.setIndexSparseness(18); + assertThat(plainTableConfig.indexSparseness()). + isEqualTo(18); + } + + @Test + public void hugePageTlbSize() { + final PlainTableConfig plainTableConfig = new PlainTableConfig(); + plainTableConfig.setHugePageTlbSize(1); + assertThat(plainTableConfig.hugePageTlbSize()). + isEqualTo(1); + } + + @Test + public void encodingType() { + final PlainTableConfig plainTableConfig = new PlainTableConfig(); + plainTableConfig.setEncodingType(EncodingType.kPrefix); + assertThat(plainTableConfig.encodingType()).isEqualTo( + EncodingType.kPrefix); + } + + @Test + public void fullScanMode() { + final PlainTableConfig plainTableConfig = new PlainTableConfig(); + plainTableConfig.setFullScanMode(true); + assertThat(plainTableConfig.fullScanMode()).isTrue(); } + + @Test + public void storeIndexInFile() { + final PlainTableConfig plainTableConfig = new PlainTableConfig(); + plainTableConfig.setStoreIndexInFile(true); + assertThat(plainTableConfig.storeIndexInFile()). + isTrue(); + } + + @Test + public void plainTableConfig() { + try(final Options opt = new Options()) { + final PlainTableConfig plainTableConfig = new PlainTableConfig(); + opt.setTableFormatConfig(plainTableConfig); + assertThat(opt.tableFactoryName()).isEqualTo("PlainTable"); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/PlatformRandomHelper.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/PlatformRandomHelper.java new file mode 100644 index 0000000..80ea4d1 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/PlatformRandomHelper.java @@ -0,0 +1,58 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Random; + +/** + * Helper class to get the appropriate Random class instance dependent + * on the current platform architecture (32bit vs 64bit) + */ +public class PlatformRandomHelper { + /** + * Determine if OS is 32-Bit/64-Bit + * + * @return boolean value indicating if operating system is 64 Bit. + */ + public static boolean isOs64Bit(){ + final boolean is64Bit; + if (System.getProperty("os.name").contains("Windows")) { + is64Bit = (System.getenv("ProgramFiles(x86)") != null); + } else { + is64Bit = (System.getProperty("os.arch").contains("64")); + } + return is64Bit; + } + + /** + * Factory to get a platform specific Random instance + * + * @return {@link java.util.Random} instance. + */ + public static Random getPlatformSpecificRandomFactory(){ + if (isOs64Bit()) { + return new Random(); + } + return new Random32Bit(); + } + + /** + * Random32Bit is a class which overrides {@code nextLong} to + * provide random numbers which fit in size_t. This workaround + * is necessary because there is no unsigned_int < Java 8 + */ + private static class Random32Bit extends Random { + @Override + public long nextLong(){ + return this.nextInt(Integer.MAX_VALUE); + } + } + + /** + * Utility class constructor + */ + private PlatformRandomHelper() { } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/PutMultiplePartsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/PutMultiplePartsTest.java new file mode 100644 index 0000000..7835737 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/PutMultiplePartsTest.java @@ -0,0 +1,164 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class PutMultiplePartsTest { + @Parameterized.Parameters + public static List data() { + return Arrays.asList(2, 3, 250, 20000); + } + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + private final int numParts; + + public PutMultiplePartsTest(final Integer numParts) { + this.numParts = numParts; + } + + @Test + public void putUntracked() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final TransactionDBOptions txnDbOptions = new TransactionDBOptions(); + final TransactionDB txnDB = + TransactionDB.open(options, txnDbOptions, dbFolder.getRoot().getAbsolutePath())) { + try (final Transaction transaction = txnDB.beginTransaction(new WriteOptions())) { + final byte[][] keys = generateItems("key", ":", numParts); + final byte[][] values = generateItems("value", "", numParts); + transaction.putUntracked(keys, values); + transaction.commit(); + } + txnDB.syncWal(); + } + + validateResults(); + } + + @Test + public void put() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final TransactionDBOptions txnDbOptions = new TransactionDBOptions(); + final TransactionDB txnDB = + TransactionDB.open(options, txnDbOptions, dbFolder.getRoot().getAbsolutePath())) { + try (final Transaction transaction = txnDB.beginTransaction(new WriteOptions())) { + final byte[][] keys = generateItems("key", ":", numParts); + final byte[][] values = generateItems("value", "", numParts); + transaction.put(keys, values); + transaction.commit(); + } + txnDB.syncWal(); + } + + validateResults(); + } + + @Test + public void putUntrackedCF() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final TransactionDBOptions txnDbOptions = new TransactionDBOptions(); + final TransactionDB txnDB = + TransactionDB.open(options, txnDbOptions, dbFolder.getRoot().getAbsolutePath()); + final ColumnFamilyHandle columnFamilyHandle = + txnDB.createColumnFamily(new ColumnFamilyDescriptor("cfTest".getBytes()))) { + try (final Transaction transaction = txnDB.beginTransaction(new WriteOptions())) { + final byte[][] keys = generateItems("key", ":", numParts); + final byte[][] values = generateItems("value", "", numParts); + transaction.putUntracked(columnFamilyHandle, keys, values); + transaction.commit(); + } + txnDB.syncWal(); + } + + validateResultsCF(); + } + @Test + public void putCF() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final TransactionDBOptions txnDbOptions = new TransactionDBOptions(); + final TransactionDB txnDB = + TransactionDB.open(options, txnDbOptions, dbFolder.getRoot().getAbsolutePath()); + final ColumnFamilyHandle columnFamilyHandle = + txnDB.createColumnFamily(new ColumnFamilyDescriptor("cfTest".getBytes()))) { + try (final Transaction transaction = txnDB.beginTransaction(new WriteOptions())) { + final byte[][] keys = generateItems("key", ":", numParts); + final byte[][] values = generateItems("value", "", numParts); + transaction.put(columnFamilyHandle, keys, values); + transaction.commit(); + } + txnDB.syncWal(); + } + + validateResultsCF(); + } + + private void validateResults() throws RocksDBException { + try (final RocksDB db = RocksDB.open(new Options(), dbFolder.getRoot().getAbsolutePath())) { + final List keys = generateItemsAsList("key", ":", numParts); + final byte[][] values = generateItems("value", "", numParts); + + final StringBuilder singleKey = new StringBuilder(); + for (int i = 0; i < numParts; i++) { + singleKey.append(new String(keys.get(i), StandardCharsets.UTF_8)); + } + final byte[] result = db.get(singleKey.toString().getBytes()); + final StringBuilder singleValue = new StringBuilder(); + for (int i = 0; i < numParts; i++) { + singleValue.append(new String(values[i], StandardCharsets.UTF_8)); + } + assertThat(result).isEqualTo(singleValue.toString().getBytes()); + } + } + + private void validateResultsCF() throws RocksDBException { + final List columnFamilyDescriptors = new ArrayList<>(); + columnFamilyDescriptors.add(new ColumnFamilyDescriptor("cfTest".getBytes())); + columnFamilyDescriptors.add(new ColumnFamilyDescriptor("default".getBytes())); + final List columnFamilyHandles = new ArrayList<>(); + try (final RocksDB db = RocksDB.open(new DBOptions(), dbFolder.getRoot().getAbsolutePath(), + columnFamilyDescriptors, columnFamilyHandles)) { + final List keys = generateItemsAsList("key", ":", numParts); + final byte[][] values = generateItems("value", "", numParts); + + final StringBuilder singleKey = new StringBuilder(); + for (int i = 0; i < numParts; i++) { + singleKey.append(new String(keys.get(i), StandardCharsets.UTF_8)); + } + final byte[] result = db.get(columnFamilyHandles.get(0), singleKey.toString().getBytes()); + final StringBuilder singleValue = new StringBuilder(); + for (int i = 0; i < numParts; i++) { + singleValue.append(new String(values[i], StandardCharsets.UTF_8)); + } + assertThat(result).isEqualTo(singleValue.toString().getBytes()); + } + } + + private byte[][] generateItems(final String prefix, final String suffix, final int numItems) { + return generateItemsAsList(prefix, suffix, numItems).toArray(new byte[0][0]); + } + + private List generateItemsAsList( + final String prefix, final String suffix, final int numItems) { + final List items = new ArrayList<>(); + for (int i = 0; i < numItems; i++) { + items.add((prefix + i + suffix).getBytes()); + } + return items; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/RateLimiterTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/RateLimiterTest.java new file mode 100644 index 0000000..e7d6e6c --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/RateLimiterTest.java @@ -0,0 +1,65 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.rocksdb.RateLimiter.*; + +public class RateLimiterTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Test + public void bytesPerSecond() { + try(final RateLimiter rateLimiter = + new RateLimiter(1000, DEFAULT_REFILL_PERIOD_MICROS, + DEFAULT_FAIRNESS, DEFAULT_MODE, DEFAULT_AUTOTUNE)) { + assertThat(rateLimiter.getBytesPerSecond()).isGreaterThan(0); + rateLimiter.setBytesPerSecond(2000); + assertThat(rateLimiter.getBytesPerSecond()).isGreaterThan(0); + } + } + + @Test + public void getSingleBurstBytes() { + try(final RateLimiter rateLimiter = + new RateLimiter(1000, DEFAULT_REFILL_PERIOD_MICROS, + DEFAULT_FAIRNESS, DEFAULT_MODE, DEFAULT_AUTOTUNE)) { + assertThat(rateLimiter.getSingleBurstBytes()).isEqualTo(100); + } + } + + @Test + public void getTotalBytesThrough() { + try(final RateLimiter rateLimiter = + new RateLimiter(1000, DEFAULT_REFILL_PERIOD_MICROS, + DEFAULT_FAIRNESS, DEFAULT_MODE, DEFAULT_AUTOTUNE)) { + assertThat(rateLimiter.getTotalBytesThrough()).isEqualTo(0); + } + } + + @Test + public void getTotalRequests() { + try(final RateLimiter rateLimiter = + new RateLimiter(1000, DEFAULT_REFILL_PERIOD_MICROS, + DEFAULT_FAIRNESS, DEFAULT_MODE, DEFAULT_AUTOTUNE)) { + assertThat(rateLimiter.getTotalRequests()).isEqualTo(0); + } + } + + @Test + public void autoTune() { + try(final RateLimiter rateLimiter = + new RateLimiter(1000, DEFAULT_REFILL_PERIOD_MICROS, + DEFAULT_FAIRNESS, DEFAULT_MODE, true)) { + assertThat(rateLimiter.getBytesPerSecond()).isGreaterThan(0); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ReadOnlyTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ReadOnlyTest.java new file mode 100644 index 0000000..99549b6 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ReadOnlyTest.java @@ -0,0 +1,222 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class ReadOnlyTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void readOnlyOpen() throws RocksDBException { + try (final Options options = new Options() + .setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath())) { + db.put("key".getBytes(), "value".getBytes()); + } + try (final RocksDB db = RocksDB.openReadOnly(dbFolder.getRoot().getAbsolutePath())) { + assertThat("value").isEqualTo(new String(db.get("key".getBytes()))); + } + + try (final ColumnFamilyOptions cfOpts = new ColumnFamilyOptions()) { + final List cfDescriptors = new ArrayList<>(); + cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpts)); + final List columnFamilyHandleList = new ArrayList<>(); + try (final RocksDB db = RocksDB.open( + dbFolder.getRoot().getAbsolutePath(), cfDescriptors, columnFamilyHandleList)) { + columnFamilyHandleList.add( + db.createColumnFamily(new ColumnFamilyDescriptor("new_cf".getBytes(), cfOpts))); + columnFamilyHandleList.add( + db.createColumnFamily(new ColumnFamilyDescriptor("new_cf2".getBytes(), cfOpts))); + db.put(columnFamilyHandleList.get(2), "key2".getBytes(), "value2".getBytes()); + } + + columnFamilyHandleList.clear(); + try (final RocksDB db = RocksDB.openReadOnly( + dbFolder.getRoot().getAbsolutePath(), cfDescriptors, columnFamilyHandleList)) { + assertThat(db.get("key2".getBytes())).isNull(); + assertThat(db.get(columnFamilyHandleList.get(0), "key2".getBytes())).isNull(); + } + + cfDescriptors.clear(); + cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpts)); + cfDescriptors.add(new ColumnFamilyDescriptor("new_cf2".getBytes(), cfOpts)); + columnFamilyHandleList.clear(); + try (final RocksDB db = RocksDB.openReadOnly( + dbFolder.getRoot().getAbsolutePath(), cfDescriptors, columnFamilyHandleList)) { + assertThat(new String(db.get(columnFamilyHandleList.get(1), "key2".getBytes()))) + .isEqualTo("value2"); + } + } + } + + @Test(expected = RocksDBException.class) + public void failToWriteInReadOnly() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true)) { + try (final RocksDB ignored = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + // no-op + } + } + + try (final ColumnFamilyOptions cfOpts = new ColumnFamilyOptions()) { + final List cfDescriptors = Collections.singletonList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpts)); + + final List readOnlyColumnFamilyHandleList = new ArrayList<>(); + try (final RocksDB rDb = RocksDB.openReadOnly(dbFolder.getRoot().getAbsolutePath(), + cfDescriptors, readOnlyColumnFamilyHandleList)) { + // test that put fails in readonly mode + rDb.put("key".getBytes(), "value".getBytes()); + } + } + } + + @Test(expected = RocksDBException.class) + public void failToCFWriteInReadOnly() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB ignored = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + //no-op + } + + try (final ColumnFamilyOptions cfOpts = new ColumnFamilyOptions()) { + final List cfDescriptors = Collections.singletonList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpts)); + final List readOnlyColumnFamilyHandleList = + new ArrayList<>(); + try (final RocksDB rDb = RocksDB.openReadOnly( + dbFolder.getRoot().getAbsolutePath(), cfDescriptors, + readOnlyColumnFamilyHandleList)) { + rDb.put(readOnlyColumnFamilyHandleList.get(0), "key".getBytes(), "value".getBytes()); + } + } + } + + @Test(expected = RocksDBException.class) + public void failToRemoveInReadOnly() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB ignored = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + //no-op + } + + try (final ColumnFamilyOptions cfOpts = new ColumnFamilyOptions()) { + final List cfDescriptors = Collections.singletonList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpts)); + + final List readOnlyColumnFamilyHandleList = + new ArrayList<>(); + + try (final RocksDB rDb = RocksDB.openReadOnly( + dbFolder.getRoot().getAbsolutePath(), cfDescriptors, + readOnlyColumnFamilyHandleList)) { + rDb.delete("key".getBytes()); + } + } + } + + @Test(expected = RocksDBException.class) + public void failToCFRemoveInReadOnly() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB ignored = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + //no-op + } + + try (final ColumnFamilyOptions cfOpts = new ColumnFamilyOptions()) { + final List cfDescriptors = Collections.singletonList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpts)); + + final List readOnlyColumnFamilyHandleList = + new ArrayList<>(); + try (final RocksDB rDb = RocksDB.openReadOnly( + dbFolder.getRoot().getAbsolutePath(), cfDescriptors, + readOnlyColumnFamilyHandleList)) { + rDb.delete(readOnlyColumnFamilyHandleList.get(0), "key".getBytes()); + } + } + } + + @Test(expected = RocksDBException.class) + public void failToWriteBatchReadOnly() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB ignored = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + //no-op + } + + try (final ColumnFamilyOptions cfOpts = new ColumnFamilyOptions()) { + final List cfDescriptors = Collections.singletonList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpts)); + + final List readOnlyColumnFamilyHandleList = + new ArrayList<>(); + try (final RocksDB rDb = RocksDB.openReadOnly( + dbFolder.getRoot().getAbsolutePath(), cfDescriptors, + readOnlyColumnFamilyHandleList); + final WriteBatch wb = new WriteBatch(); + final WriteOptions wOpts = new WriteOptions()) { + wb.put("key".getBytes(), "value".getBytes()); + rDb.write(wOpts, wb); + } + } + } + + @Test(expected = RocksDBException.class) + public void failToCFWriteBatchReadOnly() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB ignored = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + //no-op + } + + try (final ColumnFamilyOptions cfOpts = new ColumnFamilyOptions()) { + final List cfDescriptors = Collections.singletonList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpts)); + + final List readOnlyColumnFamilyHandleList = + new ArrayList<>(); + try (final RocksDB rDb = RocksDB.openReadOnly( + dbFolder.getRoot().getAbsolutePath(), cfDescriptors, + readOnlyColumnFamilyHandleList); + final WriteBatch wb = new WriteBatch(); + final WriteOptions wOpts = new WriteOptions()) { + wb.put(readOnlyColumnFamilyHandleList.get(0), "key".getBytes(), "value".getBytes()); + rDb.write(wOpts, wb); + } + } + } + + @Test(expected = RocksDBException.class) + public void errorIfWalFileExists() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB ignored = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + // no-op + } + + try (final ColumnFamilyOptions cfOpts = new ColumnFamilyOptions()) { + final List cfDescriptors = Collections.singletonList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpts)); + + final List readOnlyColumnFamilyHandleList = new ArrayList<>(); + try (final DBOptions options = new DBOptions(); + final RocksDB ignored = + RocksDB.openReadOnly(options, dbFolder.getRoot().getAbsolutePath(), cfDescriptors, + readOnlyColumnFamilyHandleList, true)) { + // no-op... should have raised an error as errorIfWalFileExists=true + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ReadOptionsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ReadOptionsTest.java new file mode 100644 index 0000000..1bc24b9 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/ReadOptionsTest.java @@ -0,0 +1,374 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Arrays; +import java.util.Random; + +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ReadOptionsTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public ExpectedException exception = ExpectedException.none(); + + @Test + public void altConstructor() { + try (final ReadOptions opt = new ReadOptions(true, true)) { + assertThat(opt.verifyChecksums()).isTrue(); + assertThat(opt.fillCache()).isTrue(); + } + } + + @Test + public void copyConstructor() { + try (final ReadOptions opt = new ReadOptions()) { + opt.setVerifyChecksums(false); + opt.setFillCache(false); + opt.setIterateUpperBound(buildRandomSlice()); + opt.setIterateLowerBound(buildRandomSlice()); + opt.setTimestamp(buildRandomSlice()); + opt.setIterStartTs(buildRandomSlice()); + try (final ReadOptions other = new ReadOptions(opt)) { + assertThat(opt.verifyChecksums()).isEqualTo(other.verifyChecksums()); + assertThat(opt.fillCache()).isEqualTo(other.fillCache()); + assertThat(Arrays.equals(opt.iterateUpperBound().data(), other.iterateUpperBound().data())).isTrue(); + assertThat(Arrays.equals(opt.iterateLowerBound().data(), other.iterateLowerBound().data())).isTrue(); + assertThat(Arrays.equals(opt.timestamp().data(), other.timestamp().data())).isTrue(); + assertThat(Arrays.equals(opt.iterStartTs().data(), other.iterStartTs().data())).isTrue(); + } + } + } + + @Test + public void verifyChecksum() { + try (final ReadOptions opt = new ReadOptions()) { + final Random rand = new Random(); + final boolean boolValue = rand.nextBoolean(); + opt.setVerifyChecksums(boolValue); + assertThat(opt.verifyChecksums()).isEqualTo(boolValue); + } + } + + @Test + public void fillCache() { + try (final ReadOptions opt = new ReadOptions()) { + final Random rand = new Random(); + final boolean boolValue = rand.nextBoolean(); + opt.setFillCache(boolValue); + assertThat(opt.fillCache()).isEqualTo(boolValue); + } + } + + @Test + public void tailing() { + try (final ReadOptions opt = new ReadOptions()) { + final Random rand = new Random(); + final boolean boolValue = rand.nextBoolean(); + opt.setTailing(boolValue); + assertThat(opt.tailing()).isEqualTo(boolValue); + } + } + + @Test + public void snapshot() { + try (final ReadOptions opt = new ReadOptions()) { + opt.setSnapshot(null); + assertThat(opt.snapshot()).isNull(); + } + } + + @Test + public void readTier() { + try (final ReadOptions opt = new ReadOptions()) { + opt.setReadTier(ReadTier.BLOCK_CACHE_TIER); + assertThat(opt.readTier()).isEqualTo(ReadTier.BLOCK_CACHE_TIER); + } + } + + @SuppressWarnings("deprecated") + @Test + public void managed() { + try (final ReadOptions opt = new ReadOptions()) { + opt.setManaged(true); + assertThat(opt.managed()).isTrue(); + } + } + + @Test + public void totalOrderSeek() { + try (final ReadOptions opt = new ReadOptions()) { + opt.setTotalOrderSeek(true); + assertThat(opt.totalOrderSeek()).isTrue(); + } + } + + @Test + public void prefixSameAsStart() { + try (final ReadOptions opt = new ReadOptions()) { + opt.setPrefixSameAsStart(true); + assertThat(opt.prefixSameAsStart()).isTrue(); + } + } + + @Test + public void pinData() { + try (final ReadOptions opt = new ReadOptions()) { + opt.setPinData(true); + assertThat(opt.pinData()).isTrue(); + } + } + + @Test + public void backgroundPurgeOnIteratorCleanup() { + try (final ReadOptions opt = new ReadOptions()) { + opt.setBackgroundPurgeOnIteratorCleanup(true); + assertThat(opt.backgroundPurgeOnIteratorCleanup()).isTrue(); + } + } + + @Test + public void readaheadSize() { + try (final ReadOptions opt = new ReadOptions()) { + final Random rand = new Random(); + final int intValue = rand.nextInt(2147483647); + opt.setReadaheadSize(intValue); + assertThat(opt.readaheadSize()).isEqualTo(intValue); + } + } + + @Test + public void ignoreRangeDeletions() { + try (final ReadOptions opt = new ReadOptions()) { + opt.setIgnoreRangeDeletions(true); + assertThat(opt.ignoreRangeDeletions()).isTrue(); + } + } + + @Test + public void iterateUpperBound() { + try (final ReadOptions opt = new ReadOptions()) { + final Slice upperBound = buildRandomSlice(); + opt.setIterateUpperBound(upperBound); + assertThat(Arrays.equals(upperBound.data(), opt.iterateUpperBound().data())).isTrue(); + opt.setIterateUpperBound(null); + assertThat(opt.iterateUpperBound()).isNull(); + } + } + + @Test + public void iterateUpperBoundNull() { + try (final ReadOptions opt = new ReadOptions()) { + assertThat(opt.iterateUpperBound()).isNull(); + } + } + + @Test + public void iterateLowerBound() { + try (final ReadOptions opt = new ReadOptions()) { + final Slice lowerBound = buildRandomSlice(); + opt.setIterateLowerBound(lowerBound); + assertThat(Arrays.equals(lowerBound.data(), opt.iterateLowerBound().data())).isTrue(); + opt.setIterateLowerBound(null); + assertThat(opt.iterateLowerBound()).isNull(); + } + } + + @Test + public void iterateLowerBoundNull() { + try (final ReadOptions opt = new ReadOptions()) { + assertThat(opt.iterateLowerBound()).isNull(); + } + } + + @Test + public void tableFilter() { + try (final ReadOptions opt = new ReadOptions(); + final AbstractTableFilter allTablesFilter = new AllTablesFilter()) { + opt.setTableFilter(allTablesFilter); + } + } + + @Test + public void autoPrefixMode() { + try (final ReadOptions opt = new ReadOptions()) { + opt.setAutoPrefixMode(true); + assertThat(opt.autoPrefixMode()).isTrue(); + } + } + + @Test + public void timestamp() { + try (final ReadOptions opt = new ReadOptions()) { + final Slice timestamp = buildRandomSlice(); + opt.setTimestamp(timestamp); + assertThat(Arrays.equals(timestamp.data(), opt.timestamp().data())).isTrue(); + opt.setTimestamp(null); + assertThat(opt.timestamp()).isNull(); + } + } + + @Test + public void iterStartTs() { + try (final ReadOptions opt = new ReadOptions()) { + final Slice itertStartTsSlice = buildRandomSlice(); + opt.setIterStartTs(itertStartTsSlice); + assertThat(Arrays.equals(itertStartTsSlice.data(), opt.iterStartTs().data())).isTrue(); + opt.setIterStartTs(null); + assertThat(opt.iterStartTs()).isNull(); + } + } + + @Test + public void deadline() { + try (final ReadOptions opt = new ReadOptions()) { + opt.setDeadline(1999L); + assertThat(opt.deadline()).isEqualTo(1999L); + } + } + + @Test + public void ioTimeout() { + try (final ReadOptions opt = new ReadOptions()) { + opt.setIoTimeout(34555L); + assertThat(opt.ioTimeout()).isEqualTo(34555L); + } + } + + @Test + public void valueSizeSoftLimit() { + try (final ReadOptions opt = new ReadOptions()) { + opt.setValueSizeSoftLimit(12134324L); + assertThat(opt.valueSizeSoftLimit()).isEqualTo(12134324L); + } + } + + @Test + public void failSetVerifyChecksumUninitialized() { + try (final ReadOptions readOptions = + setupUninitializedReadOptions(exception)) { + readOptions.setVerifyChecksums(true); + } + } + + @Test + public void failVerifyChecksumUninitialized() { + try (final ReadOptions readOptions = + setupUninitializedReadOptions(exception)) { + readOptions.verifyChecksums(); + } + } + + @Test + public void failSetFillCacheUninitialized() { + try (final ReadOptions readOptions = + setupUninitializedReadOptions(exception)) { + readOptions.setFillCache(true); + } + } + + @Test + public void failFillCacheUninitialized() { + try (final ReadOptions readOptions = + setupUninitializedReadOptions(exception)) { + readOptions.fillCache(); + } + } + + @Test + public void failSetTailingUninitialized() { + try (final ReadOptions readOptions = + setupUninitializedReadOptions(exception)) { + readOptions.setTailing(true); + } + } + + @Test + public void failTailingUninitialized() { + try (final ReadOptions readOptions = + setupUninitializedReadOptions(exception)) { + readOptions.tailing(); + } + } + + @Test + public void failSetSnapshotUninitialized() { + try (final ReadOptions readOptions = + setupUninitializedReadOptions(exception)) { + readOptions.setSnapshot(null); + } + } + + @Test + public void failSnapshotUninitialized() { + try (final ReadOptions readOptions = + setupUninitializedReadOptions(exception)) { + readOptions.snapshot(); + } + } + + @Test + public void failSetIterateUpperBoundUninitialized() { + try (final ReadOptions readOptions = + setupUninitializedReadOptions(exception)) { + readOptions.setIterateUpperBound(null); + } + } + + @Test + public void failIterateUpperBoundUninitialized() { + try (final ReadOptions readOptions = + setupUninitializedReadOptions(exception)) { + readOptions.iterateUpperBound(); + } + } + + @Test + public void failSetIterateLowerBoundUninitialized() { + try (final ReadOptions readOptions = + setupUninitializedReadOptions(exception)) { + readOptions.setIterateLowerBound(null); + } + } + + @Test + public void failIterateLowerBoundUninitialized() { + try (final ReadOptions readOptions = + setupUninitializedReadOptions(exception)) { + readOptions.iterateLowerBound(); + } + } + + private ReadOptions setupUninitializedReadOptions(final ExpectedException exception) { + final ReadOptions readOptions = new ReadOptions(); + readOptions.close(); + exception.expect(AssertionError.class); + return readOptions; + } + + private Slice buildRandomSlice() { + final Random rand = new Random(); + final byte[] sliceBytes = new byte[rand.nextInt(100) + 1]; + rand.nextBytes(sliceBytes); + return new Slice(sliceBytes); + } + + private static class AllTablesFilter extends AbstractTableFilter { + @Override + public boolean filter(final TableProperties tableProperties) { + return true; + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/RocksDBExceptionTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/RocksDBExceptionTest.java new file mode 100644 index 0000000..d3bd4ec --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/RocksDBExceptionTest.java @@ -0,0 +1,115 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.Test; + +import org.rocksdb.Status.Code; +import org.rocksdb.Status.SubCode; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.fail; + +public class RocksDBExceptionTest { + + @Test + public void exception() { + try { + raiseException(); + } catch(final RocksDBException e) { + assertThat(e.getStatus()).isNull(); + assertThat(e.getMessage()).isEqualTo("test message"); + return; + } + fail(); + } + + @Test + public void exceptionWithStatusCode() { + try { + raiseExceptionWithStatusCode(); + } catch(final RocksDBException e) { + assertThat(e.getStatus()).isNotNull(); + assertThat(e.getStatus().getCode()).isEqualTo(Code.NotSupported); + assertThat(e.getStatus().getSubCode()).isEqualTo(SubCode.None); + assertThat(e.getStatus().getState()).isNull(); + assertThat(e.getMessage()).isEqualTo("test message"); + return; + } + fail(); + } + + @Test + public void exceptionNoMsgWithStatusCode() { + try { + raiseExceptionNoMsgWithStatusCode(); + } catch(final RocksDBException e) { + assertThat(e.getStatus()).isNotNull(); + assertThat(e.getStatus().getCode()).isEqualTo(Code.NotSupported); + assertThat(e.getStatus().getSubCode()).isEqualTo(SubCode.None); + assertThat(e.getStatus().getState()).isNull(); + assertThat(e.getMessage()).isEqualTo(Code.NotSupported.name()); + return; + } + fail(); + } + + @Test + public void exceptionWithStatusCodeSubCode() { + try { + raiseExceptionWithStatusCodeSubCode(); + } catch(final RocksDBException e) { + assertThat(e.getStatus()).isNotNull(); + assertThat(e.getStatus().getCode()).isEqualTo(Code.TimedOut); + assertThat(e.getStatus().getSubCode()) + .isEqualTo(Status.SubCode.LockTimeout); + assertThat(e.getStatus().getState()).isNull(); + assertThat(e.getMessage()).isEqualTo("test message"); + return; + } + fail(); + } + + @Test + public void exceptionNoMsgWithStatusCodeSubCode() { + try { + raiseExceptionNoMsgWithStatusCodeSubCode(); + } catch(final RocksDBException e) { + assertThat(e.getStatus()).isNotNull(); + assertThat(e.getStatus().getCode()).isEqualTo(Code.TimedOut); + assertThat(e.getStatus().getSubCode()).isEqualTo(SubCode.LockTimeout); + assertThat(e.getStatus().getState()).isNull(); + assertThat(e.getMessage()).isEqualTo(Code.TimedOut.name() + + "(" + SubCode.LockTimeout.name() + ")"); + return; + } + fail(); + } + + @Test + public void exceptionWithStatusCodeState() { + try { + raiseExceptionWithStatusCodeState(); + } catch(final RocksDBException e) { + assertThat(e.getStatus()).isNotNull(); + assertThat(e.getStatus().getCode()).isEqualTo(Code.NotSupported); + assertThat(e.getStatus().getSubCode()).isEqualTo(SubCode.None); + assertThat(e.getStatus().getState()).isNotNull(); + assertThat(e.getMessage()).isEqualTo("test message"); + return; + } + fail(); + } + + private native void raiseException() throws RocksDBException; + private native void raiseExceptionWithStatusCode() throws RocksDBException; + private native void raiseExceptionNoMsgWithStatusCode() throws RocksDBException; + private native void raiseExceptionWithStatusCodeSubCode() + throws RocksDBException; + private native void raiseExceptionNoMsgWithStatusCodeSubCode() + throws RocksDBException; + private native void raiseExceptionWithStatusCodeState() + throws RocksDBException; +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/RocksDBTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/RocksDBTest.java new file mode 100644 index 0000000..3f6ebc7 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/RocksDBTest.java @@ -0,0 +1,1699 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +import org.junit.*; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; + +import java.nio.ByteBuffer; +import java.util.*; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.fail; + +public class RocksDBTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + public static final Random rand = PlatformRandomHelper. + getPlatformSpecificRandomFactory(); + + @Test + public void open() throws RocksDBException { + try (final RocksDB db = + RocksDB.open(dbFolder.getRoot().getAbsolutePath())) { + assertThat(db).isNotNull(); + } + } + + @Test + public void open_opt() throws RocksDBException { + try (final Options opt = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + assertThat(db).isNotNull(); + } + } + + @Test + public void openWhenOpen() throws RocksDBException { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + + try (final RocksDB ignored = RocksDB.open(dbPath)) { + try (final RocksDB ignored1 = RocksDB.open(dbPath)) { + fail("Should have thrown an exception when opening the same db twice"); + } catch (final RocksDBException e) { + assertThat(e.getStatus().getCode()).isEqualTo(Status.Code.IOError); + assertThat(e.getStatus().getSubCode()).isEqualTo(Status.SubCode.None); + assertThat(e.getStatus().getState()).contains("lock "); + } + } + } + + @Test + public void createColumnFamily() throws RocksDBException { + final byte[] col1Name = "col1".getBytes(UTF_8); + + try (final RocksDB db = RocksDB.open(dbFolder.getRoot().getAbsolutePath()); + final ColumnFamilyOptions cfOpts = new ColumnFamilyOptions() + ) { + try (final ColumnFamilyHandle col1 = + db.createColumnFamily(new ColumnFamilyDescriptor(col1Name, cfOpts))) { + assertThat(col1).isNotNull(); + assertThat(col1.getName()).isEqualTo(col1Name); + } + } + + final List cfHandles = new ArrayList<>(); + try (final RocksDB ignored = RocksDB.open(dbFolder.getRoot().getAbsolutePath(), + Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor(col1Name)), + cfHandles)) { + try { + assertThat(cfHandles.size()).isEqualTo(2); + assertThat(cfHandles.get(1)).isNotNull(); + assertThat(cfHandles.get(1).getName()).isEqualTo(col1Name); + } finally { + for (final ColumnFamilyHandle cfHandle : + cfHandles) { + cfHandle.close(); + } + } + } + } + + + @Test + public void createColumnFamilies() throws RocksDBException { + final byte[] col1Name = "col1".getBytes(UTF_8); + final byte[] col2Name = "col2".getBytes(UTF_8); + + List cfHandles; + try (final RocksDB db = RocksDB.open(dbFolder.getRoot().getAbsolutePath()); + final ColumnFamilyOptions cfOpts = new ColumnFamilyOptions() + ) { + cfHandles = + db.createColumnFamilies(cfOpts, Arrays.asList(col1Name, col2Name)); + try { + assertThat(cfHandles).isNotNull(); + assertThat(cfHandles.size()).isEqualTo(2); + assertThat(cfHandles.get(0).getName()).isEqualTo(col1Name); + assertThat(cfHandles.get(1).getName()).isEqualTo(col2Name); + } finally { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + } + } + + cfHandles = new ArrayList<>(); + try (final RocksDB ignored = RocksDB.open(dbFolder.getRoot().getAbsolutePath(), + Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor(col1Name), new ColumnFamilyDescriptor(col2Name)), + cfHandles)) { + try { + assertThat(cfHandles.size()).isEqualTo(3); + assertThat(cfHandles.get(1)).isNotNull(); + assertThat(cfHandles.get(1).getName()).isEqualTo(col1Name); + assertThat(cfHandles.get(2)).isNotNull(); + assertThat(cfHandles.get(2).getName()).isEqualTo(col2Name); + } finally { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + } + } + } + + @Test + public void createColumnFamiliesfromDescriptors() throws RocksDBException { + final byte[] col1Name = "col1".getBytes(UTF_8); + final byte[] col2Name = "col2".getBytes(UTF_8); + + List cfHandles; + try (final RocksDB db = RocksDB.open(dbFolder.getRoot().getAbsolutePath()); + final ColumnFamilyOptions cfOpts = new ColumnFamilyOptions() + ) { + cfHandles = + db.createColumnFamilies(Arrays.asList( + new ColumnFamilyDescriptor(col1Name, cfOpts), + new ColumnFamilyDescriptor(col2Name, cfOpts))); + try { + assertThat(cfHandles).isNotNull(); + assertThat(cfHandles.size()).isEqualTo(2); + assertThat(cfHandles.get(0).getName()).isEqualTo(col1Name); + assertThat(cfHandles.get(1).getName()).isEqualTo(col2Name); + } finally { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + } + } + + cfHandles = new ArrayList<>(); + try (final RocksDB ignored = RocksDB.open(dbFolder.getRoot().getAbsolutePath(), + Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor(col1Name), new ColumnFamilyDescriptor(col2Name)), + cfHandles)) { + try { + assertThat(cfHandles.size()).isEqualTo(3); + assertThat(cfHandles.get(1)).isNotNull(); + assertThat(cfHandles.get(1).getName()).isEqualTo(col1Name); + assertThat(cfHandles.get(2)).isNotNull(); + assertThat(cfHandles.get(2).getName()).isEqualTo(col2Name); + } finally { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + } + } + } + + @Test + public void put() throws RocksDBException { + try (final RocksDB db = RocksDB.open(dbFolder.getRoot().getAbsolutePath()); + final WriteOptions opt = new WriteOptions(); final ReadOptions optr = new ReadOptions()) { + db.put("key1".getBytes(), "value".getBytes()); + db.put(opt, "key2".getBytes(), "12345678".getBytes()); + assertThat(db.get("key1".getBytes())).isEqualTo( + "value".getBytes()); + assertThat(db.get("key2".getBytes())).isEqualTo( + "12345678".getBytes()); + + final ByteBuffer key = ByteBuffer.allocateDirect(12); + final ByteBuffer value = ByteBuffer.allocateDirect(12); + key.position(4); + key.put("key3".getBytes()); + key.position(4).limit(8); + value.position(4); + value.put("val3".getBytes()); + value.position(4).limit(8); + + db.put(opt, key, value); + + assertThat(key.position()).isEqualTo(8); + assertThat(key.limit()).isEqualTo(8); + + assertThat(value.position()).isEqualTo(8); + assertThat(value.limit()).isEqualTo(8); + + key.position(4); + + final ByteBuffer result = ByteBuffer.allocateDirect(12); + assertThat(db.get(optr, key, result)).isEqualTo(4); + assertThat(result.position()).isEqualTo(0); + assertThat(result.limit()).isEqualTo(4); + assertThat(key.position()).isEqualTo(8); + assertThat(key.limit()).isEqualTo(8); + + final byte[] tmp = new byte[4]; + result.get(tmp); + assertThat(tmp).isEqualTo("val3".getBytes()); + + key.position(4); + + result.clear().position(9); + assertThat(db.get(optr, key, result)).isEqualTo(4); + assertThat(result.position()).isEqualTo(9); + assertThat(result.limit()).isEqualTo(12); + assertThat(key.position()).isEqualTo(8); + assertThat(key.limit()).isEqualTo(8); + final byte[] tmp2 = new byte[3]; + result.get(tmp2); + assertThat(tmp2).isEqualTo("val".getBytes()); + + // put + final Segment key3 = sliceSegment("key3"); + final Segment key4 = sliceSegment("key4"); + final Segment value0 = sliceSegment("value 0"); + final Segment value1 = sliceSegment("value 1"); + db.put(key3.data, key3.offset, key3.len, value0.data, value0.offset, value0.len); + db.put(opt, key4.data, key4.offset, key4.len, value1.data, value1.offset, value1.len); + + // compare + Assert.assertTrue(value0.isSamePayload(db.get(key3.data, key3.offset, key3.len))); + Assert.assertTrue(value1.isSamePayload(db.get(key4.data, key4.offset, key4.len))); + } + } + + private static Segment sliceSegment(final String key) { + final ByteBuffer rawKey = ByteBuffer.allocate(key.length() + 4); + rawKey.put((byte)0); + rawKey.put((byte)0); + rawKey.put(key.getBytes()); + + return new Segment(rawKey.array(), 2, key.length()); + } + + private static class Segment { + final byte[] data; + final int offset; + final int len; + + public boolean isSamePayload(final byte[] value) { + if (value == null) { + return false; + } + if (value.length != len) { + return false; + } + + for (int i = 0; i < value.length; i++) { + if (data[i + offset] != value[i]) { + return false; + } + } + + return true; + } + + public Segment(final byte[] value, final int offset, final int len) { + this.data = value; + this.offset = offset; + this.len = len; + } + } + + @Test + public void write() throws RocksDBException { + try (final StringAppendOperator stringAppendOperator = new StringAppendOperator(); + final Options options = new Options() + .setMergeOperator(stringAppendOperator) + .setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath()); + final WriteOptions opts = new WriteOptions()) { + + try (final WriteBatch wb1 = new WriteBatch()) { + wb1.put("key1".getBytes(), "aa".getBytes()); + wb1.merge("key1".getBytes(), "bb".getBytes()); + + try (final WriteBatch wb2 = new WriteBatch()) { + wb2.put("key2".getBytes(), "xx".getBytes()); + wb2.merge("key2".getBytes(), "yy".getBytes()); + db.write(opts, wb1); + db.write(opts, wb2); + } + } + + assertThat(db.get("key1".getBytes())).isEqualTo( + "aa,bb".getBytes()); + assertThat(db.get("key2".getBytes())).isEqualTo( + "xx,yy".getBytes()); + } + } + + @Test + public void getWithOutValue() throws RocksDBException { + try (final RocksDB db = + RocksDB.open(dbFolder.getRoot().getAbsolutePath())) { + db.put("key1".getBytes(), "value".getBytes()); + db.put("key2".getBytes(), "12345678".getBytes()); + final byte[] outValue = new byte[5]; + // not found value + int getResult = db.get("keyNotFound".getBytes(), outValue); + assertThat(getResult).isEqualTo(RocksDB.NOT_FOUND); + // found value which fits in outValue + getResult = db.get("key1".getBytes(), outValue); + assertThat(getResult).isNotEqualTo(RocksDB.NOT_FOUND); + assertThat(outValue).isEqualTo("value".getBytes()); + // found value which fits partially + getResult = db.get("key2".getBytes(), outValue); + assertThat(getResult).isNotEqualTo(RocksDB.NOT_FOUND); + assertThat(outValue).isEqualTo("12345".getBytes()); + } + } + + @Test + public void getWithOutValueReadOptions() throws RocksDBException { + try (final RocksDB db = RocksDB.open(dbFolder.getRoot().getAbsolutePath()); + final ReadOptions rOpt = new ReadOptions()) { + db.put("key1".getBytes(), "value".getBytes()); + db.put("key2".getBytes(), "12345678".getBytes()); + final byte[] outValue = new byte[5]; + // not found value + int getResult = db.get(rOpt, "keyNotFound".getBytes(), + outValue); + assertThat(getResult).isEqualTo(RocksDB.NOT_FOUND); + // found value which fits in outValue + getResult = db.get(rOpt, "key1".getBytes(), outValue); + assertThat(getResult).isNotEqualTo(RocksDB.NOT_FOUND); + assertThat(outValue).isEqualTo("value".getBytes()); + // found value which fits partially + getResult = db.get(rOpt, "key2".getBytes(), outValue); + assertThat(getResult).isNotEqualTo(RocksDB.NOT_FOUND); + assertThat(outValue).isEqualTo("12345".getBytes()); + } + } + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test + public void getOutOfArrayMaxSizeValue() throws RocksDBException { + final int numberOfValueSplits = 10; + final int splitSize = Integer.MAX_VALUE / numberOfValueSplits; + + final Runtime runtime = Runtime.getRuntime(); + final long neededMemory = ((long) (splitSize)) * (((long) numberOfValueSplits) + 3); + final boolean isEnoughMemory = runtime.maxMemory() - runtime.totalMemory() > neededMemory; + Assume.assumeTrue(isEnoughMemory); + + final byte[] valueSplit = new byte[splitSize]; + final byte[] key = "key".getBytes(); + + thrown.expect(RocksDBException.class); + thrown.expectMessage("Requested array size exceeds VM limit"); + + // merge (numberOfValueSplits + 1) valueSplit's to get value size exceeding Integer.MAX_VALUE + try (final StringAppendOperator stringAppendOperator = new StringAppendOperator(); + final Options opt = new Options() + .setCreateIfMissing(true) + .setMergeOperator(stringAppendOperator); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + db.put(key, valueSplit); + for (int i = 0; i < numberOfValueSplits; i++) { + db.merge(key, valueSplit); + } + db.get(key); + } + } + + @Test + public void multiGetAsList() throws RocksDBException { + try (final RocksDB db = RocksDB.open(dbFolder.getRoot().getAbsolutePath()); + final ReadOptions rOpt = new ReadOptions()) { + db.put("key1".getBytes(), "value".getBytes()); + db.put("key2".getBytes(), "12345678".getBytes()); + final List lookupKeys = new ArrayList<>(); + lookupKeys.add("key1".getBytes()); + lookupKeys.add("key2".getBytes()); + List results = db.multiGetAsList(lookupKeys); + assertThat(results).isNotNull(); + assertThat(results).hasSize(lookupKeys.size()); + assertThat(results). + containsExactly("value".getBytes(), "12345678".getBytes()); + // test same method with ReadOptions + results = db.multiGetAsList(rOpt, lookupKeys); + assertThat(results).isNotNull(); + assertThat(results). + contains("value".getBytes(), "12345678".getBytes()); + + // remove existing key + lookupKeys.remove(1); + // add non existing key + lookupKeys.add("key3".getBytes()); + results = db.multiGetAsList(lookupKeys); + assertThat(results).isNotNull(); + assertThat(results). + containsExactly("value".getBytes(), null); + // test same call with readOptions + results = db.multiGetAsList(rOpt, lookupKeys); + assertThat(results).isNotNull(); + assertThat(results).contains("value".getBytes()); + } + } + + @Test + public void merge() throws RocksDBException { + try (final StringAppendOperator stringAppendOperator = new StringAppendOperator(); + final Options opt = new Options() + .setCreateIfMissing(true) + .setMergeOperator(stringAppendOperator); + final WriteOptions wOpt = new WriteOptions(); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath()) + ) { + db.put("key1".getBytes(), "value".getBytes()); + assertThat(db.get("key1".getBytes())).isEqualTo( + "value".getBytes()); + // merge key1 with another value portion + db.merge("key1".getBytes(), "value2".getBytes()); + assertThat(db.get("key1".getBytes())).isEqualTo( + "value,value2".getBytes()); + // merge key1 with another value portion + db.merge(wOpt, "key1".getBytes(), "value3".getBytes()); + assertThat(db.get("key1".getBytes())).isEqualTo( + "value,value2,value3".getBytes()); + // merge on non existent key shall insert the value + db.merge(wOpt, "key2".getBytes(), "xxxx".getBytes()); + assertThat(db.get("key2".getBytes())).isEqualTo( + "xxxx".getBytes()); + + final Segment key3 = sliceSegment("key3"); + final Segment key4 = sliceSegment("key4"); + final Segment value0 = sliceSegment("value 0"); + final Segment value1 = sliceSegment("value 1"); + + db.merge(key3.data, key3.offset, key3.len, value0.data, value0.offset, value0.len); + db.merge(wOpt, key4.data, key4.offset, key4.len, value1.data, value1.offset, value1.len); + + // compare + Assert.assertTrue(value0.isSamePayload(db.get(key3.data, key3.offset, key3.len))); + Assert.assertTrue(value1.isSamePayload(db.get(key4.data, key4.offset, key4.len))); + } + } + + @Test + public void delete() throws RocksDBException { + try (final RocksDB db = RocksDB.open(dbFolder.getRoot().getAbsolutePath()); + final WriteOptions wOpt = new WriteOptions()) { + db.put("key1".getBytes(), "value".getBytes()); + db.put("key2".getBytes(), "12345678".getBytes()); + db.put("key3".getBytes(), "33".getBytes()); + assertThat(db.get("key1".getBytes())).isEqualTo( + "value".getBytes()); + assertThat(db.get("key2".getBytes())).isEqualTo( + "12345678".getBytes()); + assertThat(db.get("key3".getBytes())).isEqualTo("33".getBytes()); + db.delete("key1".getBytes()); + db.delete(wOpt, "key2".getBytes()); + final ByteBuffer key = ByteBuffer.allocateDirect(16); + key.put("key3".getBytes()).flip(); + db.delete(wOpt, key); + assertThat(key.position()).isEqualTo(4); + assertThat(key.limit()).isEqualTo(4); + + assertThat(db.get("key1".getBytes())).isNull(); + assertThat(db.get("key2".getBytes())).isNull(); + + final Segment key3 = sliceSegment("key3"); + final Segment key4 = sliceSegment("key4"); + db.put("key3".getBytes(), "key3 value".getBytes()); + db.put("key4".getBytes(), "key4 value".getBytes()); + + db.delete(key3.data, key3.offset, key3.len); + db.delete(wOpt, key4.data, key4.offset, key4.len); + + assertThat(db.get("key3".getBytes())).isNull(); + assertThat(db.get("key4".getBytes())).isNull(); + } + } + + @Test + public void singleDelete() throws RocksDBException { + try (final RocksDB db = RocksDB.open(dbFolder.getRoot().getAbsolutePath()); + final WriteOptions wOpt = new WriteOptions()) { + db.put("key1".getBytes(), "value".getBytes()); + db.put("key2".getBytes(), "12345678".getBytes()); + assertThat(db.get("key1".getBytes())).isEqualTo( + "value".getBytes()); + assertThat(db.get("key2".getBytes())).isEqualTo( + "12345678".getBytes()); + db.singleDelete("key1".getBytes()); + db.singleDelete(wOpt, "key2".getBytes()); + assertThat(db.get("key1".getBytes())).isNull(); + assertThat(db.get("key2".getBytes())).isNull(); + } + } + + @Test + public void singleDelete_nonExisting() throws RocksDBException { + try (final RocksDB db = RocksDB.open(dbFolder.getRoot().getAbsolutePath()); + final WriteOptions wOpt = new WriteOptions()) { + db.singleDelete("key1".getBytes()); + db.singleDelete(wOpt, "key2".getBytes()); + assertThat(db.get("key1".getBytes())).isNull(); + assertThat(db.get("key2".getBytes())).isNull(); + } + } + + @Test + public void deleteRange() throws RocksDBException { + try (final RocksDB db = RocksDB.open(dbFolder.getRoot().getAbsolutePath())) { + db.put("key1".getBytes(), "value".getBytes()); + db.put("key2".getBytes(), "12345678".getBytes()); + db.put("key3".getBytes(), "abcdefg".getBytes()); + db.put("key4".getBytes(), "xyz".getBytes()); + assertThat(db.get("key1".getBytes())).isEqualTo("value".getBytes()); + assertThat(db.get("key2".getBytes())).isEqualTo("12345678".getBytes()); + assertThat(db.get("key3".getBytes())).isEqualTo("abcdefg".getBytes()); + assertThat(db.get("key4".getBytes())).isEqualTo("xyz".getBytes()); + db.deleteRange("key2".getBytes(), "key4".getBytes()); + assertThat(db.get("key1".getBytes())).isEqualTo("value".getBytes()); + assertThat(db.get("key2".getBytes())).isNull(); + assertThat(db.get("key3".getBytes())).isNull(); + assertThat(db.get("key4".getBytes())).isEqualTo("xyz".getBytes()); + } + } + + @Test + public void getIntProperty() throws RocksDBException { + try ( + final Options options = new Options() + .setCreateIfMissing(true) + .setMaxWriteBufferNumber(10) + .setMinWriteBufferNumberToMerge(10); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath()); + final WriteOptions wOpt = new WriteOptions().setDisableWAL(true) + ) { + db.put(wOpt, "key1".getBytes(), "value1".getBytes()); + db.put(wOpt, "key2".getBytes(), "value2".getBytes()); + db.put(wOpt, "key3".getBytes(), "value3".getBytes()); + db.put(wOpt, "key4".getBytes(), "value4".getBytes()); + assertThat(db.getLongProperty("rocksdb.num-entries-active-mem-table")) + .isGreaterThan(0); + assertThat(db.getLongProperty("rocksdb.cur-size-active-mem-table")) + .isGreaterThan(0); + } + } + + @Test + public void fullCompactRange() throws RocksDBException { + try (final Options opt = new Options(). + setCreateIfMissing(true). + setDisableAutoCompactions(true). + setCompactionStyle(CompactionStyle.LEVEL). + setNumLevels(4). + setWriteBufferSize(100 << 10). + setLevelZeroFileNumCompactionTrigger(3). + setTargetFileSizeBase(200 << 10). + setTargetFileSizeMultiplier(1). + setMaxBytesForLevelBase(500 << 10). + setMaxBytesForLevelMultiplier(1). + setDisableAutoCompactions(false); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + // fill database with key/value pairs + final byte[] b = new byte[10000]; + for (int i = 0; i < 200; i++) { + rand.nextBytes(b); + db.put((String.valueOf(i)).getBytes(), b); + } + db.compactRange(); + } + } + + @Test + public void fullCompactRangeColumnFamily() + throws RocksDBException { + try ( + final DBOptions opt = new DBOptions(). + setCreateIfMissing(true). + setCreateMissingColumnFamilies(true); + final ColumnFamilyOptions new_cf_opts = new ColumnFamilyOptions(). + setDisableAutoCompactions(true). + setCompactionStyle(CompactionStyle.LEVEL). + setNumLevels(4). + setWriteBufferSize(100 << 10). + setLevelZeroFileNumCompactionTrigger(3). + setTargetFileSizeBase(200 << 10). + setTargetFileSizeMultiplier(1). + setMaxBytesForLevelBase(500 << 10). + setMaxBytesForLevelMultiplier(1). + setDisableAutoCompactions(false) + ) { + final List columnFamilyDescriptors = + Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes(), new_cf_opts)); + + // open database + final List columnFamilyHandles = new ArrayList<>(); + try (final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath(), + columnFamilyDescriptors, + columnFamilyHandles)) { + try { + // fill database with key/value pairs + final byte[] b = new byte[10000]; + for (int i = 0; i < 200; i++) { + rand.nextBytes(b); + db.put(columnFamilyHandles.get(1), + String.valueOf(i).getBytes(), b); + } + db.compactRange(columnFamilyHandles.get(1)); + } finally { + for (final ColumnFamilyHandle handle : columnFamilyHandles) { + handle.close(); + } + } + } + } + } + + @Test + public void compactRangeWithKeys() + throws RocksDBException { + try (final Options opt = new Options(). + setCreateIfMissing(true). + setDisableAutoCompactions(true). + setCompactionStyle(CompactionStyle.LEVEL). + setNumLevels(4). + setWriteBufferSize(100 << 10). + setLevelZeroFileNumCompactionTrigger(3). + setTargetFileSizeBase(200 << 10). + setTargetFileSizeMultiplier(1). + setMaxBytesForLevelBase(500 << 10). + setMaxBytesForLevelMultiplier(1). + setDisableAutoCompactions(false); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + // fill database with key/value pairs + final byte[] b = new byte[10000]; + for (int i = 0; i < 200; i++) { + rand.nextBytes(b); + db.put((String.valueOf(i)).getBytes(), b); + } + db.compactRange("0".getBytes(), "201".getBytes()); + } + } + + @Test + public void compactRangeWithKeysReduce() + throws RocksDBException { + try ( + final Options opt = new Options(). + setCreateIfMissing(true). + setDisableAutoCompactions(true). + setCompactionStyle(CompactionStyle.LEVEL). + setNumLevels(4). + setWriteBufferSize(100 << 10). + setLevelZeroFileNumCompactionTrigger(3). + setTargetFileSizeBase(200 << 10). + setTargetFileSizeMultiplier(1). + setMaxBytesForLevelBase(500 << 10). + setMaxBytesForLevelMultiplier(1). + setDisableAutoCompactions(false); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + // fill database with key/value pairs + final byte[] b = new byte[10000]; + for (int i = 0; i < 200; i++) { + rand.nextBytes(b); + db.put((String.valueOf(i)).getBytes(), b); + } + try (final FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + db.flush(flushOptions); + } + try (final CompactRangeOptions compactRangeOpts = new CompactRangeOptions() + .setChangeLevel(true) + .setTargetLevel(-1) + .setTargetPathId(0)) { + db.compactRange(null, "0".getBytes(), "201".getBytes(), + compactRangeOpts); + } + } + } + + @Test + public void compactRangeWithKeysColumnFamily() + throws RocksDBException { + try (final DBOptions opt = new DBOptions(). + setCreateIfMissing(true). + setCreateMissingColumnFamilies(true); + final ColumnFamilyOptions new_cf_opts = new ColumnFamilyOptions(). + setDisableAutoCompactions(true). + setCompactionStyle(CompactionStyle.LEVEL). + setNumLevels(4). + setWriteBufferSize(100 << 10). + setLevelZeroFileNumCompactionTrigger(3). + setTargetFileSizeBase(200 << 10). + setTargetFileSizeMultiplier(1). + setMaxBytesForLevelBase(500 << 10). + setMaxBytesForLevelMultiplier(1). + setDisableAutoCompactions(false) + ) { + final List columnFamilyDescriptors = + Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes(), new_cf_opts) + ); + + // open database + final List columnFamilyHandles = + new ArrayList<>(); + try (final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath(), + columnFamilyDescriptors, + columnFamilyHandles)) { + try { + // fill database with key/value pairs + final byte[] b = new byte[10000]; + for (int i = 0; i < 200; i++) { + rand.nextBytes(b); + db.put(columnFamilyHandles.get(1), + String.valueOf(i).getBytes(), b); + } + db.compactRange(columnFamilyHandles.get(1), + "0".getBytes(), "201".getBytes()); + } finally { + for (final ColumnFamilyHandle handle : columnFamilyHandles) { + handle.close(); + } + } + } + } + } + + @Test + public void compactRangeWithKeysReduceColumnFamily() + throws RocksDBException { + try (final DBOptions opt = new DBOptions(). + setCreateIfMissing(true). + setCreateMissingColumnFamilies(true); + final ColumnFamilyOptions new_cf_opts = new ColumnFamilyOptions(). + setDisableAutoCompactions(true). + setCompactionStyle(CompactionStyle.LEVEL). + setNumLevels(4). + setWriteBufferSize(100 << 10). + setLevelZeroFileNumCompactionTrigger(3). + setTargetFileSizeBase(200 << 10). + setTargetFileSizeMultiplier(1). + setMaxBytesForLevelBase(500 << 10). + setMaxBytesForLevelMultiplier(1). + setDisableAutoCompactions(false) + ) { + final List columnFamilyDescriptors = + Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes(), new_cf_opts) + ); + + final List columnFamilyHandles = new ArrayList<>(); + // open database + try (final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath(), + columnFamilyDescriptors, + columnFamilyHandles)) { + try (final CompactRangeOptions compactRangeOpts = new CompactRangeOptions() + .setChangeLevel(true) + .setTargetLevel(-1) + .setTargetPathId(0)) { + // fill database with key/value pairs + final byte[] b = new byte[10000]; + for (int i = 0; i < 200; i++) { + rand.nextBytes(b); + db.put(columnFamilyHandles.get(1), + String.valueOf(i).getBytes(), b); + } + db.compactRange(columnFamilyHandles.get(1), "0".getBytes(), + "201".getBytes(), compactRangeOpts); + } finally { + for (final ColumnFamilyHandle handle : columnFamilyHandles) { + handle.close(); + } + } + } + } + } + + @Test + public void compactRangeToLevel() throws RocksDBException { + final int NUM_KEYS_PER_L0_FILE = 100; + final int KEY_SIZE = 20; + final int VALUE_SIZE = 300; + final int L0_FILE_SIZE = + NUM_KEYS_PER_L0_FILE * (KEY_SIZE + VALUE_SIZE); + final int NUM_L0_FILES = 10; + final int TEST_SCALE = 5; + final int KEY_INTERVAL = 100; + try (final Options opt = new Options() + .setCreateIfMissing(true) + .setCompactionStyle(CompactionStyle.LEVEL) + .setLevelCompactionDynamicLevelBytes(false) + .setNumLevels(5) + . + // a slightly bigger write buffer than L0 file + // so that we can ensure manual flush always + // go before background flush happens. + setWriteBufferSize(L0_FILE_SIZE * 2) + . + // Disable auto L0 -> L1 compaction + setLevelZeroFileNumCompactionTrigger(20) + .setTargetFileSizeBase(L0_FILE_SIZE * 100) + .setTargetFileSizeMultiplier(1) + . + // To disable auto compaction + setMaxBytesForLevelBase(NUM_L0_FILES * L0_FILE_SIZE * 100) + .setMaxBytesForLevelMultiplier(2) + .setDisableAutoCompactions(true); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + // fill database with key/value pairs + final byte[] value = new byte[VALUE_SIZE]; + int int_key = 0; + for (int round = 0; round < 5; ++round) { + final int initial_key = int_key; + for (int f = 1; f <= NUM_L0_FILES; ++f) { + for (int i = 0; i < NUM_KEYS_PER_L0_FILE; ++i) { + int_key += KEY_INTERVAL; + rand.nextBytes(value); + + db.put(String.format("%020d", int_key).getBytes(), + value); + } + try (final FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + db.flush(flushOptions); + } + // Make sure we do create one more L0 files. + assertThat( + db.getProperty("rocksdb.num-files-at-level0")). + isEqualTo("" + f); + } + + // Compact all L0 files we just created + db.compactRange( + String.format("%020d", initial_key).getBytes(), + String.format("%020d", int_key - 1).getBytes()); + // Making sure there isn't any L0 files. + assertThat( + db.getProperty("rocksdb.num-files-at-level0")). + isEqualTo("0"); + // Making sure there are some L1 files. + // Here we only use != 0 instead of a specific number + // as we don't want the test make any assumption on + // how compaction works. + assertThat( + db.getProperty("rocksdb.num-files-at-level1")). + isNotEqualTo("0"); + // Because we only compacted those keys we issued + // in this round, there shouldn't be any L1 -> L2 + // compaction. So we expect zero L2 files here. + assertThat( + db.getProperty("rocksdb.num-files-at-level2")). + isEqualTo("0"); + } + } + } + + @Test + public void deleteFilesInRange() throws RocksDBException { + final int KEY_SIZE = 20; + final int VALUE_SIZE = 1000; + final int FILE_SIZE = 64000; + final int NUM_FILES = 10; + + final int KEY_INTERVAL = 10000; + /* + * Intention of these options is to end up reliably with 10 files + * we will be deleting using deleteFilesInRange. + * It is writing roughly number of keys that will fit in 10 files (target size) + * It is writing interleaved so that files from memory on L0 will overlap + * Then compaction cleans everything, and we should end up with 10 files + */ + try (final Options opt = new Options() + .setCreateIfMissing(true) + .setCompressionType(CompressionType.NO_COMPRESSION) + .setTargetFileSizeBase(FILE_SIZE) + .setWriteBufferSize(FILE_SIZE / 2) + .setDisableAutoCompactions(true) + .setLevelCompactionDynamicLevelBytes(false); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + final int records = FILE_SIZE / (KEY_SIZE + VALUE_SIZE); + + // fill database with key/value pairs + final byte[] value = new byte[VALUE_SIZE]; + int key_init = 0; + for (int o = 0; o < NUM_FILES; ++o) { + int int_key = key_init++; + for (int i = 0; i < records; ++i) { + int_key += KEY_INTERVAL; + rand.nextBytes(value); + + db.put(String.format("%020d", int_key).getBytes(), value); + } + } + try (final FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + db.flush(flushOptions); + } + db.compactRange(); + // Make sure we do create one more L0 files. + assertThat(db.getProperty("rocksdb.num-files-at-level0")).isEqualTo("0"); + + // Should be 10, but we are OK with asserting +- 2 + int files = Integer.parseInt(db.getProperty("rocksdb.num-files-at-level1")); + assertThat(files).isBetween(8, 12); + + // Delete lower 60% (roughly). Result should be 5, but we are OK with asserting +- 2 + // Important is that we know something was deleted (JNI call did something) + // Exact assertions are done in C++ unit tests + db.deleteFilesInRanges(null, + Arrays.asList(null, String.format("%020d", records * KEY_INTERVAL * 6 / 10).getBytes()), + false); + files = Integer.parseInt(db.getProperty("rocksdb.num-files-at-level1")); + assertThat(files).isBetween(3, 7); + } + } + + @Test + public void compactRangeToLevelColumnFamily() + throws RocksDBException { + final int NUM_KEYS_PER_L0_FILE = 100; + final int KEY_SIZE = 20; + final int VALUE_SIZE = 300; + final int L0_FILE_SIZE = + NUM_KEYS_PER_L0_FILE * (KEY_SIZE + VALUE_SIZE); + final int NUM_L0_FILES = 10; + final int TEST_SCALE = 5; + final int KEY_INTERVAL = 100; + + try (final DBOptions opt = + new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); + final ColumnFamilyOptions new_cf_opts = + new ColumnFamilyOptions() + .setCompactionStyle(CompactionStyle.LEVEL) + .setLevelCompactionDynamicLevelBytes(false) + .setNumLevels(5) + . + // a slightly bigger write buffer than L0 file + // so that we can ensure manual flush always + // go before background flush happens. + setWriteBufferSize(L0_FILE_SIZE * 2) + . + // Disable auto L0 -> L1 compaction + setLevelZeroFileNumCompactionTrigger(20) + .setTargetFileSizeBase(L0_FILE_SIZE * 100) + .setTargetFileSizeMultiplier(1) + . + // To disable auto compaction + setMaxBytesForLevelBase(NUM_L0_FILES * L0_FILE_SIZE * 100) + .setMaxBytesForLevelMultiplier(2) + .setDisableAutoCompactions(true)) { + final List columnFamilyDescriptors = + Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes(), new_cf_opts) + ); + + final List columnFamilyHandles = new ArrayList<>(); + // open database + try (final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath(), + columnFamilyDescriptors, + columnFamilyHandles)) { + try { + // fill database with key/value pairs + final byte[] value = new byte[VALUE_SIZE]; + int int_key = 0; + for (int round = 0; round < 5; ++round) { + final int initial_key = int_key; + for (int f = 1; f <= NUM_L0_FILES; ++f) { + for (int i = 0; i < NUM_KEYS_PER_L0_FILE; ++i) { + int_key += KEY_INTERVAL; + rand.nextBytes(value); + + db.put(columnFamilyHandles.get(1), + String.format("%020d", int_key).getBytes(), + value); + } + try (final FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + db.flush(flushOptions, columnFamilyHandles.get(1)); + } + // Make sure we do create one more L0 files. + assertThat( + db.getProperty(columnFamilyHandles.get(1), + "rocksdb.num-files-at-level0")). + isEqualTo("" + f); + } + + // Compact all L0 files we just created + db.compactRange( + columnFamilyHandles.get(1), + String.format("%020d", initial_key).getBytes(), + String.format("%020d", int_key - 1).getBytes()); + // Making sure there isn't any L0 files. + assertThat( + db.getProperty(columnFamilyHandles.get(1), + "rocksdb.num-files-at-level0")). + isEqualTo("0"); + // Making sure there are some L1 files. + // Here we only use != 0 instead of a specific number + // as we don't want the test make any assumption on + // how compaction works. + assertThat( + db.getProperty(columnFamilyHandles.get(1), + "rocksdb.num-files-at-level1")). + isNotEqualTo("0"); + // Because we only compacted those keys we issued + // in this round, there shouldn't be any L1 -> L2 + // compaction. So we expect zero L2 files here. + assertThat( + db.getProperty(columnFamilyHandles.get(1), + "rocksdb.num-files-at-level2")). + isEqualTo("0"); + } + } finally { + for (final ColumnFamilyHandle handle : columnFamilyHandles) { + handle.close(); + } + } + } + } + } + + @Test + public void continueBackgroundWorkAfterCancelAllBackgroundWork() throws RocksDBException { + final int KEY_SIZE = 20; + final int VALUE_SIZE = 300; + try (final DBOptions opt = new DBOptions(). + setCreateIfMissing(true). + setCreateMissingColumnFamilies(true); + final ColumnFamilyOptions new_cf_opts = new ColumnFamilyOptions() + ) { + final List columnFamilyDescriptors = + Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes(), new_cf_opts) + ); + + final List columnFamilyHandles = new ArrayList<>(); + // open the database + try (final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath(), + columnFamilyDescriptors, + columnFamilyHandles)) { + try { + db.cancelAllBackgroundWork(true); + try { + db.put(new byte[KEY_SIZE], new byte[VALUE_SIZE]); + try (final FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + db.flush(flushOptions); + } + fail("Expected RocksDBException to be thrown if we attempt to trigger a flush after" + + " all background work is cancelled."); + } catch (final RocksDBException ignored) { + } + } finally { + for (final ColumnFamilyHandle handle : columnFamilyHandles) { + handle.close(); + } + } + } + } + } + + @Test + public void cancelAllBackgroundWorkTwice() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath()) + ) { + // Cancel all background work synchronously + db.cancelAllBackgroundWork(true); + // Cancel all background work asynchronously + db.cancelAllBackgroundWork(false); + } + } + + @Test + public void pauseContinueBackgroundWork() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath()) + ) { + db.pauseBackgroundWork(); + db.continueBackgroundWork(); + db.pauseBackgroundWork(); + db.continueBackgroundWork(); + } + } + + @Test + public void enableDisableFileDeletions() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath()) + ) { + db.disableFileDeletions(); + db.enableFileDeletions(false); + db.disableFileDeletions(); + db.enableFileDeletions(true); + } + } + + @Test + public void setOptions() throws RocksDBException { + try (final DBOptions options = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + final ColumnFamilyOptions new_cf_opts = new ColumnFamilyOptions() + .setWriteBufferSize(4096)) { + + final List columnFamilyDescriptors = + Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes(), new_cf_opts)); + + // open database + final List columnFamilyHandles = new ArrayList<>(); + try (final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath(), columnFamilyDescriptors, columnFamilyHandles)) { + try { + final MutableColumnFamilyOptions mutableOptions = + MutableColumnFamilyOptions.builder() + .setWriteBufferSize(2048) + .build(); + + db.setOptions(columnFamilyHandles.get(1), mutableOptions); + + } finally { + for (final ColumnFamilyHandle handle : columnFamilyHandles) { + handle.close(); + } + } + } + } + } + + @Test + public void destroyDB() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true)) { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath)) { + db.put("key1".getBytes(), "value".getBytes()); + } + assertThat(dbFolder.getRoot().exists() + && Objects.requireNonNull(dbFolder.getRoot().listFiles()).length != 0) + .isTrue(); + RocksDB.destroyDB(dbPath, options); + assertThat(dbFolder.getRoot().exists() + && Objects.requireNonNull(dbFolder.getRoot().listFiles()).length != 0) + .isFalse(); + } + } + + @Test(expected = RocksDBException.class) + public void destroyDBFailIfOpen() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true)) { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB ignored = RocksDB.open(options, dbPath)) { + // Fails as the db is open and locked. + RocksDB.destroyDB(dbPath, options); + } + } + } + + @Test + public void getApproximateSizes() throws RocksDBException { + final byte[] key1 = "key1".getBytes(UTF_8); + final byte[] key2 = "key2".getBytes(UTF_8); + final byte[] key3 = "key3".getBytes(UTF_8); + try (final Options options = new Options().setCreateIfMissing(true)) { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath)) { + db.put(key1, key1); + db.put(key2, key2); + db.put(key3, key3); + + final long[] sizes = db.getApproximateSizes( + Arrays.asList( + new Range(new Slice(key1), new Slice(key1)), + new Range(new Slice(key2), new Slice(key3)) + ), + SizeApproximationFlag.INCLUDE_FILES, + SizeApproximationFlag.INCLUDE_MEMTABLES); + + assertThat(sizes.length).isEqualTo(2); + assertThat(sizes[0]).isEqualTo(0); + assertThat(sizes[1]).isGreaterThanOrEqualTo(1); + } + } + } + + @Test + public void getApproximateMemTableStats() throws RocksDBException { + final byte[] key1 = "key1".getBytes(UTF_8); + final byte[] key2 = "key2".getBytes(UTF_8); + final byte[] key3 = "key3".getBytes(UTF_8); + try (final Options options = new Options().setCreateIfMissing(true)) { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath)) { + db.put(key1, key1); + db.put(key2, key2); + db.put(key3, key3); + + final RocksDB.CountAndSize stats = + db.getApproximateMemTableStats( + new Range(new Slice(key1), new Slice(key3))); + + assertThat(stats).isNotNull(); + assertThat(stats.count).isGreaterThan(1); + assertThat(stats.size).isGreaterThan(1); + } + } + } + + @Test + public void getApproximateMemTableStatsSingleKey() throws RocksDBException { + final byte[] key1 = "key1".getBytes(UTF_8); + final byte[] key3 = "key3".getBytes(UTF_8); + try (final Options options = new Options().setCreateIfMissing(true)) { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath)) { + db.put(key1, key1); + + final RocksDB.CountAndSize stats = + db.getApproximateMemTableStats(new Range(new Slice(key1), new Slice(key3))); + + assertThat(stats).isNotNull(); + assertThat(stats.count).isEqualTo(1); + assertThat(stats.size).isGreaterThan(1); + } + } + } + + @Ignore("TODO(AR) re-enable when ready!") + @Test + public void compactFiles() throws RocksDBException { + final int kTestKeySize = 16; + final int kTestValueSize = 984; + final int kEntrySize = kTestKeySize + kTestValueSize; + final int kEntriesPerBuffer = 100; + final int writeBufferSize = kEntrySize * kEntriesPerBuffer; + final byte[] cfName = "pikachu".getBytes(UTF_8); + + try (final Options options = new Options() + .setCreateIfMissing(true) + .setWriteBufferSize(writeBufferSize) + .setCompactionStyle(CompactionStyle.LEVEL) + .setLevelCompactionDynamicLevelBytes(false) + .setTargetFileSizeBase(writeBufferSize) + .setMaxBytesForLevelBase(writeBufferSize * 2) + .setLevel0StopWritesTrigger(2) + .setMaxBytesForLevelMultiplier(2) + .setCompressionType(CompressionType.NO_COMPRESSION) + .setMaxSubcompactions(4)) { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath); + final ColumnFamilyOptions cfOptions = new ColumnFamilyOptions(options)) { + db.createColumnFamily(new ColumnFamilyDescriptor(cfName, + cfOptions)).close(); + } + + try (final ColumnFamilyOptions cfOptions = new ColumnFamilyOptions(options)) { + final List cfDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions), + new ColumnFamilyDescriptor(cfName, cfOptions) + ); + final List cfHandles = new ArrayList<>(); + try (final DBOptions dbOptions = new DBOptions(options); + final RocksDB db = RocksDB.open(dbOptions, dbPath, cfDescriptors, cfHandles)) { + try (final FlushOptions flushOptions = new FlushOptions() + .setWaitForFlush(true) + .setAllowWriteStall(true); + final CompactionOptions compactionOptions = new CompactionOptions()) { + final Random rnd = new Random(301); + for (int key = 64 * kEntriesPerBuffer; key >= 0; --key) { + final byte[] value = new byte[kTestValueSize]; + rnd.nextBytes(value); + db.put(cfHandles.get(1), Integer.toString(key).getBytes(UTF_8), + value); + } + db.flush(flushOptions, cfHandles); + + final RocksDB.LiveFiles liveFiles = db.getLiveFiles(); + final List compactedFiles = + db.compactFiles(compactionOptions, cfHandles.get(1), + liveFiles.files, 1, -1, null); + assertThat(compactedFiles).isNotEmpty(); + } finally { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + } + } + } + } + } + + @Test + public void enableAutoCompaction() throws RocksDBException { + try (final DBOptions options = new DBOptions() + .setCreateIfMissing(true)) { + final List cfDescs = + Collections.singletonList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY)); + final List cfHandles = new ArrayList<>(); + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath, cfDescs, cfHandles)) { + try { + db.enableAutoCompaction(cfHandles); + } finally { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + } + } + } + } + + @Test + public void numberLevels() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true)) { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath)) { + assertThat(db.numberLevels()).isEqualTo(7); + } + } + } + + @Test + public void maxMemCompactionLevel() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true)) { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath)) { + assertThat(db.maxMemCompactionLevel()).isEqualTo(0); + } + } + } + + @Test + public void level0StopWriteTrigger() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true)) { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath)) { + assertThat(db.level0StopWriteTrigger()).isEqualTo(36); + } + } + } + + @Test + public void getName() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true)) { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath)) { + assertThat(db.getName()).isEqualTo(dbPath); + } + } + } + + @Test + public void getEnv() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true)) { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath)) { + assertThat(db.getEnv()).isEqualTo(Env.getDefault()); + } + } + } + + @Test + public void flush() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true)) { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath); + final FlushOptions flushOptions = new FlushOptions()) { + db.flush(flushOptions); + } + } + } + + @Test + public void flushWal() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true)) { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath)) { + db.flushWal(true); + } + } + } + + @Test + public void syncWal() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true)) { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath)) { + db.syncWal(); + } + } + } + + @Test + public void getLiveFiles() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true)) { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath)) { + final RocksDB.LiveFiles livefiles = db.getLiveFiles(true); + assertThat(livefiles).isNotNull(); + assertThat(livefiles.manifestFileSize).isEqualTo(70); + assertThat(livefiles.files.size()).isEqualTo(3); + assertThat(livefiles.files.get(0)).isEqualTo("/CURRENT"); + assertThat(livefiles.files.get(1)).isEqualTo("/MANIFEST-000005"); + assertThat(livefiles.files.get(2)).isEqualTo("/OPTIONS-000007"); + } + } + } + + @Test + public void getSortedWalFiles() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true)) { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath)) { + db.put("key1".getBytes(UTF_8), "value1".getBytes(UTF_8)); + final List logFiles = db.getSortedWalFiles(); + assertThat(logFiles).isNotNull(); + assertThat(logFiles.size()).isEqualTo(1); + assertThat(logFiles.get(0).type()) + .isEqualTo(WalFileType.kAliveLogFile); + } + } + } + + @Test + public void deleteFile() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true)) { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath)) { + db.deleteFile("unknown"); + } + } + } + + @Test + public void getLiveFilesMetaData() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true)) { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath)) { + db.put("key1".getBytes(UTF_8), "value1".getBytes(UTF_8)); + final List liveFilesMetaData + = db.getLiveFilesMetaData(); + assertThat(liveFilesMetaData).isEmpty(); + } + } + } + + @Test + public void getColumnFamilyMetaData() throws RocksDBException { + try (final DBOptions options = new DBOptions() + .setCreateIfMissing(true)) { + final List cfDescs = + Collections.singletonList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY)); + final List cfHandles = new ArrayList<>(); + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath, cfDescs, cfHandles)) { + db.put(cfHandles.get(0), "key1".getBytes(UTF_8), "value1".getBytes(UTF_8)); + try { + final ColumnFamilyMetaData cfMetadata = + db.getColumnFamilyMetaData(cfHandles.get(0)); + assertThat(cfMetadata).isNotNull(); + assertThat(cfMetadata.name()).isEqualTo(RocksDB.DEFAULT_COLUMN_FAMILY); + assertThat(cfMetadata.levels().size()).isEqualTo(7); + } finally { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + } + } + } + } + + @Test + public void verifyChecksum() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true)) { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath)) { + db.verifyChecksum(); + } + } + } + + @Test + public void getPropertiesOfAllTables() throws RocksDBException { + try (final DBOptions options = new DBOptions() + .setCreateIfMissing(true)) { + final List cfDescs = + Collections.singletonList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY)); + final List cfHandles = new ArrayList<>(); + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath, cfDescs, cfHandles)) { + db.put(cfHandles.get(0), "key1".getBytes(UTF_8), "value1".getBytes(UTF_8)); + try { + final Map properties = + db.getPropertiesOfAllTables(cfHandles.get(0)); + assertThat(properties).isNotNull(); + } finally { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + } + } + } + } + + @Test + public void getPropertiesOfTablesInRange() throws RocksDBException { + try (final DBOptions options = new DBOptions() + .setCreateIfMissing(true)) { + final List cfDescs = + Collections.singletonList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY)); + final List cfHandles = new ArrayList<>(); + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath, cfDescs, cfHandles)) { + db.put(cfHandles.get(0), "key1".getBytes(UTF_8), "value1".getBytes(UTF_8)); + db.put(cfHandles.get(0), "key2".getBytes(UTF_8), "value2".getBytes(UTF_8)); + db.put(cfHandles.get(0), "key3".getBytes(UTF_8), "value3".getBytes(UTF_8)); + try { + final Range range = new Range( + new Slice("key1".getBytes(UTF_8)), + new Slice("key3".getBytes(UTF_8))); + final Map properties = + db.getPropertiesOfTablesInRange(cfHandles.get(0), Collections.singletonList(range)); + assertThat(properties).isNotNull(); + } finally { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + } + } + } + } + + @Test + public void suggestCompactRange() throws RocksDBException { + try (final DBOptions options = new DBOptions() + .setCreateIfMissing(true)) { + final List cfDescs = + Collections.singletonList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY)); + final List cfHandles = new ArrayList<>(); + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath, cfDescs, cfHandles)) { + db.put(cfHandles.get(0), "key1".getBytes(UTF_8), "value1".getBytes(UTF_8)); + db.put(cfHandles.get(0), "key2".getBytes(UTF_8), "value2".getBytes(UTF_8)); + db.put(cfHandles.get(0), "key3".getBytes(UTF_8), "value3".getBytes(UTF_8)); + try { + final Range range = db.suggestCompactRange(cfHandles.get(0)); + assertThat(range).isNotNull(); + } finally { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + } + } + } + } + + @Test + public void promoteL0() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true)) { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath)) { + db.promoteL0(2); + } + } + } + + @Test + public void startTrace() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true)) { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + try (final RocksDB db = RocksDB.open(options, dbPath)) { + final TraceOptions traceOptions = new TraceOptions(); + + try (final InMemoryTraceWriter traceWriter = new InMemoryTraceWriter()) { + db.startTrace(traceOptions, traceWriter); + + db.put("key1".getBytes(UTF_8), "value1".getBytes(UTF_8)); + + db.endTrace(); + + final List writes = traceWriter.getWrites(); + assertThat(writes.size()).isGreaterThan(0); + } + } + } + } + + @Test + public void setDBOptions() throws RocksDBException { + try (final DBOptions options = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + final ColumnFamilyOptions new_cf_opts = new ColumnFamilyOptions() + .setWriteBufferSize(4096)) { + + final List columnFamilyDescriptors = + Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes(), new_cf_opts)); + + // open database + final List columnFamilyHandles = new ArrayList<>(); + try (final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath(), columnFamilyDescriptors, columnFamilyHandles)) { + try { + final MutableDBOptions mutableOptions = + MutableDBOptions.builder() + .setBytesPerSync(1024 * 1027 * 7) + .setAvoidFlushDuringShutdown(false) + .build(); + + db.setDBOptions(mutableOptions); + } finally { + for (final ColumnFamilyHandle handle : columnFamilyHandles) { + handle.close(); + } + } + } + } + } + + @Test + public void rocksdbVersion() { + final RocksDB.Version version = RocksDB.rocksdbVersion(); + assertThat(version).isNotNull(); + assertThat(version.getMajor()).isGreaterThan(1); + } + + private static class InMemoryTraceWriter extends AbstractTraceWriter { + private final List writes = new ArrayList<>(); + private volatile boolean closed = false; + + @Override + public void write(final Slice slice) { + if (closed) { + return; + } + final byte[] data = slice.data(); + final byte[] dataCopy = new byte[data.length]; + System.arraycopy(data, 0, dataCopy, 0, data.length); + writes.add(dataCopy); + } + + @Override + public void closeWriter() { + closed = true; + } + + @Override + public long getFileSize() { + long size = 0; + for (final byte[] write : writes) { + size += write.length; + } + return size; + } + + public List getWrites() { + return writes; + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/RocksIteratorTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/RocksIteratorTest.java new file mode 100644 index 0000000..2a13550 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/RocksIteratorTest.java @@ -0,0 +1,289 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class RocksIteratorTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + private void validateByteBufferResult( + final int fetched, final ByteBuffer byteBuffer, final String expected) { + assertThat(fetched).isEqualTo(expected.length()); + assertThat(byteBuffer.position()).isEqualTo(0); + assertThat(byteBuffer.limit()).isEqualTo(Math.min(byteBuffer.remaining(), expected.length())); + final int bufferSpace = byteBuffer.remaining(); + final byte[] contents = new byte[bufferSpace]; + byteBuffer.get(contents, 0, bufferSpace); + assertThat(contents).isEqualTo( + expected.substring(0, bufferSpace).getBytes(StandardCharsets.UTF_8)); + } + + private void validateKey( + final RocksIterator iterator, final ByteBuffer byteBuffer, final String key) { + validateByteBufferResult(iterator.key(byteBuffer), byteBuffer, key); + } + + private void validateValue( + final RocksIterator iterator, final ByteBuffer byteBuffer, final String value) { + validateByteBufferResult(iterator.value(byteBuffer), byteBuffer, value); + } + + @Test + public void rocksIterator() throws RocksDBException { + try (final Options options = + new Options().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + db.put("key1".getBytes(), "value1".getBytes()); + db.put("key2".getBytes(), "value2".getBytes()); + + try (final RocksIterator iterator = db.newIterator()) { + iterator.seekToFirst(); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key1".getBytes()); + assertThat(iterator.value()).isEqualTo("value1".getBytes()); + + validateKey(iterator, ByteBuffer.allocateDirect(2), "key1"); + validateKey(iterator, ByteBuffer.allocateDirect(2), "key0"); + validateKey(iterator, ByteBuffer.allocateDirect(4), "key1"); + validateKey(iterator, ByteBuffer.allocateDirect(5), "key1"); + validateValue(iterator, ByteBuffer.allocateDirect(2), "value2"); + validateValue(iterator, ByteBuffer.allocateDirect(2), "vasicu"); + validateValue(iterator, ByteBuffer.allocateDirect(8), "value1"); + + validateKey(iterator, ByteBuffer.allocate(2), "key1"); + validateKey(iterator, ByteBuffer.allocate(2), "key0"); + validateKey(iterator, ByteBuffer.allocate(4), "key1"); + validateKey(iterator, ByteBuffer.allocate(5), "key1"); + validateValue(iterator, ByteBuffer.allocate(2), "value1"); + validateValue(iterator, ByteBuffer.allocate(8), "value1"); + + iterator.next(); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key2".getBytes()); + assertThat(iterator.value()).isEqualTo("value2".getBytes()); + iterator.next(); + assertThat(iterator.isValid()).isFalse(); + iterator.seekToLast(); + iterator.prev(); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key1".getBytes()); + assertThat(iterator.value()).isEqualTo("value1".getBytes()); + iterator.seekToFirst(); + iterator.seekToLast(); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key2".getBytes()); + assertThat(iterator.value()).isEqualTo("value2".getBytes()); + iterator.status(); + + { + final ByteBuffer key = ByteBuffer.allocate(12); + key.put("key1".getBytes()).flip(); + iterator.seek(key); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.value()).isEqualTo("value1".getBytes()); + assertThat(key.position()).isEqualTo(4); + assertThat(key.limit()).isEqualTo(4); + + validateValue(iterator, ByteBuffer.allocateDirect(12), "value1"); + validateValue(iterator, ByteBuffer.allocateDirect(4), "valu56"); + } + + { + final ByteBuffer key = ByteBuffer.allocate(12); + key.put("key2".getBytes()).flip(); + iterator.seekForPrev(key); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.value()).isEqualTo("value2".getBytes()); + assertThat(key.position()).isEqualTo(4); + assertThat(key.limit()).isEqualTo(4); + } + + { + final ByteBuffer key = ByteBuffer.allocate(12); + key.put("key1".getBytes()).flip(); + iterator.seek(key); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.value()).isEqualTo("value1".getBytes()); + assertThat(key.position()).isEqualTo(4); + assertThat(key.limit()).isEqualTo(4); + } + + { + // Check offsets of slice byte buffers + final ByteBuffer key0 = ByteBuffer.allocate(24); + key0.put("key2key2".getBytes()); + final ByteBuffer key = key0.slice(); + key.put("key1".getBytes()).flip(); + iterator.seek(key); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.value()).isEqualTo("value1".getBytes()); + assertThat(key.position()).isEqualTo(4); + assertThat(key.limit()).isEqualTo(4); + } + + { + // Check offsets of slice byte buffers + final ByteBuffer key0 = ByteBuffer.allocateDirect(24); + key0.put("key2key2".getBytes()); + final ByteBuffer key = key0.slice(); + key.put("key1".getBytes()).flip(); + iterator.seek(key); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.value()).isEqualTo("value1".getBytes()); + assertThat(key.position()).isEqualTo(4); + assertThat(key.limit()).isEqualTo(4); + } + + { + final ByteBuffer key = ByteBuffer.allocate(12); + key.put("key2".getBytes()).flip(); + iterator.seekForPrev(key); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.value()).isEqualTo("value2".getBytes()); + assertThat(key.position()).isEqualTo(4); + assertThat(key.limit()).isEqualTo(4); + } + } + } + } + + @Test + public void rocksIteratorSeekAndInsert() throws RocksDBException { + try (final Options options = + new Options().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + db.put("key1".getBytes(), "value1".getBytes()); + db.put("key2".getBytes(), "value2".getBytes()); + + try (final RocksIterator iterator = db.newIterator()) { + iterator.seek("key0".getBytes()); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key1".getBytes()); + + iterator.seek("key1".getBytes()); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key1".getBytes()); + + iterator.seek("key1.5".getBytes()); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key2".getBytes()); + + iterator.seek("key2".getBytes()); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key2".getBytes()); + + iterator.seek("key3".getBytes()); + assertThat(iterator.isValid()).isFalse(); + } + + try (final RocksIterator iterator = db.newIterator()) { + iterator.seekForPrev("key0".getBytes()); + assertThat(iterator.isValid()).isFalse(); + + iterator.seekForPrev("key1".getBytes()); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key1".getBytes()); + + iterator.seekForPrev("key1.5".getBytes()); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key1".getBytes()); + + iterator.seekForPrev("key2".getBytes()); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key2".getBytes()); + + iterator.seekForPrev("key3".getBytes()); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key2".getBytes()); + } + + try (final RocksIterator iterator = db.newIterator()) { + iterator.seekToFirst(); + assertThat(iterator.isValid()).isTrue(); + + byte[] lastKey; + do { + lastKey = iterator.key(); + iterator.next(); + } while (iterator.isValid()); + + db.put("key3".getBytes(), "value3".getBytes()); + assertThat(iterator.isValid()).isFalse(); + iterator.refresh(); + iterator.seek(lastKey); + assertThat(iterator.isValid()).isTrue(); + + iterator.next(); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key3".getBytes()); + } + } + } + + @Test + public void rocksIteratorReleaseAfterCfClose() throws RocksDBException { + try (final Options options = new Options() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(options, + this.dbFolder.getRoot().getAbsolutePath())) { + db.put("key".getBytes(), "value".getBytes()); + + // Test case: release iterator after default CF close + try (final RocksIterator iterator = db.newIterator()) { + // In fact, calling close() on default CF has no effect + db.getDefaultColumnFamily().close(); + + iterator.seekToFirst(); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key".getBytes()); + assertThat(iterator.value()).isEqualTo("value".getBytes()); + } + + // Test case: release iterator after custom CF close + final ColumnFamilyDescriptor cfd1 = new ColumnFamilyDescriptor("cf1".getBytes()); + final ColumnFamilyHandle cfHandle1 = db.createColumnFamily(cfd1); + db.put(cfHandle1, "key1".getBytes(), "value1".getBytes()); + + try (final RocksIterator iterator = db.newIterator(cfHandle1)) { + cfHandle1.close(); + + iterator.seekToFirst(); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key1".getBytes()); + assertThat(iterator.value()).isEqualTo("value1".getBytes()); + } + + // Test case: release iterator after custom CF drop & close + final ColumnFamilyDescriptor cfd2 = new ColumnFamilyDescriptor("cf2".getBytes()); + final ColumnFamilyHandle cfHandle2 = db.createColumnFamily(cfd2); + db.put(cfHandle2, "key2".getBytes(), "value2".getBytes()); + + try (final RocksIterator iterator = db.newIterator(cfHandle2)) { + db.dropColumnFamily(cfHandle2); + cfHandle2.close(); + + iterator.seekToFirst(); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key2".getBytes()); + assertThat(iterator.value()).isEqualTo("value2".getBytes()); + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/RocksMemEnvTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/RocksMemEnvTest.java new file mode 100644 index 0000000..40b24ff --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/RocksMemEnvTest.java @@ -0,0 +1,137 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class RocksMemEnvTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Test + public void memEnvFillAndReopen() throws RocksDBException { + + final byte[][] keys = { + "aaa".getBytes(), + "bbb".getBytes(), + "ccc".getBytes() + }; + + final byte[][] values = { + "foo".getBytes(), + "bar".getBytes(), + "baz".getBytes() + }; + + try (final Env env = new RocksMemEnv(Env.getDefault()); + final Options options = new Options().setCreateIfMissing(true).setEnv(env); + final FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + try (final RocksDB db = RocksDB.open(options, "/dir/db")) { + // write key/value pairs using MemEnv + for (int i = 0; i < keys.length; i++) { + db.put(keys[i], values[i]); + } + + // read key/value pairs using MemEnv + for (int i = 0; i < keys.length; i++) { + assertThat(db.get(keys[i])).isEqualTo(values[i]); + } + + // Check iterator access + try (final RocksIterator iterator = db.newIterator()) { + iterator.seekToFirst(); + for (int i = 0; i < keys.length; i++) { + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo(keys[i]); + assertThat(iterator.value()).isEqualTo(values[i]); + iterator.next(); + } + // reached end of database + assertThat(iterator.isValid()).isFalse(); + } + + // flush + db.flush(flushOptions); + + // read key/value pairs after flush using MemEnv + for (int i = 0; i < keys.length; i++) { + assertThat(db.get(keys[i])).isEqualTo(values[i]); + } + } + + options.setCreateIfMissing(false); + + // After reopen the values shall still be in the mem env. + // as long as the env is not freed. + try (final RocksDB db = RocksDB.open(options, "/dir/db")) { + // read key/value pairs using MemEnv + for (int i = 0; i < keys.length; i++) { + assertThat(db.get(keys[i])).isEqualTo(values[i]); + } + } + } + } + + @Test + public void multipleDatabaseInstances() throws RocksDBException { + // db - keys + final byte[][] keys = { + "aaa".getBytes(), + "bbb".getBytes(), + "ccc".getBytes() + }; + // otherDb - keys + final byte[][] otherKeys = { + "111".getBytes(), + "222".getBytes(), + "333".getBytes() + }; + // values + final byte[][] values = { + "foo".getBytes(), + "bar".getBytes(), + "baz".getBytes() + }; + + try (final Env env = new RocksMemEnv(Env.getDefault()); + final Options options = new Options().setCreateIfMissing(true).setEnv(env); + final RocksDB db = RocksDB.open(options, "/dir/db"); + final RocksDB otherDb = RocksDB.open(options, "/dir/otherDb")) { + // write key/value pairs using MemEnv + // to db and to otherDb. + for (int i = 0; i < keys.length; i++) { + db.put(keys[i], values[i]); + otherDb.put(otherKeys[i], values[i]); + } + + // verify key/value pairs after flush using MemEnv + for (int i = 0; i < keys.length; i++) { + // verify db + assertThat(db.get(otherKeys[i])).isNull(); + assertThat(db.get(keys[i])).isEqualTo(values[i]); + + // verify otherDb + assertThat(otherDb.get(keys[i])).isNull(); + assertThat(otherDb.get(otherKeys[i])).isEqualTo(values[i]); + } + } + } + + @Test(expected = RocksDBException.class) + public void createIfMissingFalse() throws RocksDBException { + try (final Env env = new RocksMemEnv(Env.getDefault()); + final Options options = new Options().setCreateIfMissing(false).setEnv(env); + final RocksDB db = RocksDB.open(options, "/db/dir")) { + // shall throw an exception because db dir does not + // exist. + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/RocksNativeLibraryResource.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/RocksNativeLibraryResource.java new file mode 100644 index 0000000..6116f2f --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/RocksNativeLibraryResource.java @@ -0,0 +1,18 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.rules.ExternalResource; + +/** + * Resource to load the RocksDB JNI library. + */ +public class RocksNativeLibraryResource extends ExternalResource { + @Override + protected void before() { + RocksDB.loadLibrary(); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/SecondaryDBTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/SecondaryDBTest.java new file mode 100644 index 0000000..557d4a4 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/SecondaryDBTest.java @@ -0,0 +1,135 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.ArrayList; +import java.util.List; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class SecondaryDBTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Rule public TemporaryFolder secondaryDbFolder = new TemporaryFolder(); + + @Test + public void openAsSecondary() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + db.put("key1".getBytes(), "value1".getBytes()); + db.put("key2".getBytes(), "value2".getBytes()); + db.put("key3".getBytes(), "value3".getBytes()); + + // open secondary + try (final Options secondaryOptions = new Options(); + final RocksDB secondaryDb = + RocksDB.openAsSecondary(secondaryOptions, dbFolder.getRoot().getAbsolutePath(), + secondaryDbFolder.getRoot().getAbsolutePath())) { + assertThat(secondaryDb.get("key1".getBytes())).isEqualTo("value1".getBytes()); + assertThat(secondaryDb.get("key2".getBytes())).isEqualTo("value2".getBytes()); + assertThat(secondaryDb.get("key3".getBytes())).isEqualTo("value3".getBytes()); + + // write to primary + db.put("key4".getBytes(), "value4".getBytes()); + db.put("key5".getBytes(), "value5".getBytes()); + db.put("key6".getBytes(), "value6".getBytes()); + + // tell secondary to catch up + secondaryDb.tryCatchUpWithPrimary(); + + db.put("key7".getBytes(), "value7".getBytes()); + + // check secondary + assertThat(secondaryDb.get("key4".getBytes())).isEqualTo("value4".getBytes()); + assertThat(secondaryDb.get("key5".getBytes())).isEqualTo("value5".getBytes()); + assertThat(secondaryDb.get("key6".getBytes())).isEqualTo("value6".getBytes()); + + assertThat(secondaryDb.get("key7".getBytes())).isNull(); + } + } + } + + @Test + public void openAsSecondaryColumnFamilies() throws RocksDBException { + try (final ColumnFamilyOptions cfOpts = new ColumnFamilyOptions()) { + final List cfDescriptors = new ArrayList<>(); + cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpts)); + cfDescriptors.add(new ColumnFamilyDescriptor("cf1".getBytes(), cfOpts)); + + final List cfHandles = new ArrayList<>(); + + try (final DBOptions options = + new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open( + options, dbFolder.getRoot().getAbsolutePath(), cfDescriptors, cfHandles)) { + try { + final ColumnFamilyHandle cf1 = cfHandles.get(1); + + db.put(cf1, "key1".getBytes(), "value1".getBytes()); + db.put(cf1, "key2".getBytes(), "value2".getBytes()); + db.put(cf1, "key3".getBytes(), "value3".getBytes()); + + final List secondaryCfHandles = new ArrayList<>(); + + // open secondary + try (final DBOptions secondaryOptions = new DBOptions(); + final RocksDB secondaryDb = + RocksDB.openAsSecondary(secondaryOptions, dbFolder.getRoot().getAbsolutePath(), + secondaryDbFolder.getRoot().getAbsolutePath(), cfDescriptors, + secondaryCfHandles)) { + try { + final ColumnFamilyHandle secondaryCf1 = secondaryCfHandles.get(1); + + assertThat(secondaryDb.get(secondaryCf1, "key1".getBytes())) + .isEqualTo("value1".getBytes()); + assertThat(secondaryDb.get(secondaryCf1, "key2".getBytes())) + .isEqualTo("value2".getBytes()); + assertThat(secondaryDb.get(secondaryCf1, "key3".getBytes())) + .isEqualTo("value3".getBytes()); + + // write to primary + db.put(cf1, "key4".getBytes(), "value4".getBytes()); + db.put(cf1, "key5".getBytes(), "value5".getBytes()); + db.put(cf1, "key6".getBytes(), "value6".getBytes()); + + // tell secondary to catch up + secondaryDb.tryCatchUpWithPrimary(); + + db.put(cf1, "key7".getBytes(), "value7".getBytes()); + + // check secondary + assertThat(secondaryDb.get(secondaryCf1, "key4".getBytes())) + .isEqualTo("value4".getBytes()); + assertThat(secondaryDb.get(secondaryCf1, "key5".getBytes())) + .isEqualTo("value5".getBytes()); + assertThat(secondaryDb.get(secondaryCf1, "key6".getBytes())) + .isEqualTo("value6".getBytes()); + + assertThat(secondaryDb.get(secondaryCf1, "key7".getBytes())).isNull(); + + } finally { + for (final ColumnFamilyHandle secondaryCfHandle : secondaryCfHandles) { + secondaryCfHandle.close(); + } + } + } + } finally { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + } + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/SliceTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/SliceTest.java new file mode 100644 index 0000000..c65b019 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/SliceTest.java @@ -0,0 +1,80 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class SliceTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Test + public void slice() { + try (final Slice slice = new Slice("testSlice")) { + assertThat(slice.empty()).isFalse(); + assertThat(slice.size()).isEqualTo(9); + assertThat(slice.data()).isEqualTo("testSlice".getBytes()); + } + + try (final Slice otherSlice = new Slice("otherSlice".getBytes())) { + assertThat(otherSlice.data()).isEqualTo("otherSlice".getBytes()); + } + + try (final Slice thirdSlice = new Slice("otherSlice".getBytes(), 5)) { + assertThat(thirdSlice.data()).isEqualTo("Slice".getBytes()); + } + } + + @Test + public void sliceClear() { + try (final Slice slice = new Slice("abc")) { + assertThat(slice.toString()).isEqualTo("abc"); + slice.clear(); + assertThat(slice.toString()).isEmpty(); + slice.clear(); // make sure we don't double-free + } + } + + @Test + public void sliceRemovePrefix() { + try (final Slice slice = new Slice("abc")) { + assertThat(slice.toString()).isEqualTo("abc"); + slice.removePrefix(1); + assertThat(slice.toString()).isEqualTo("bc"); + } + } + + @Test + public void sliceEquals() { + try (final Slice slice = new Slice("abc"); + final Slice slice2 = new Slice("abc")) { + assertThat(slice.equals(slice2)).isTrue(); + assertThat(slice.hashCode() == slice2.hashCode()).isTrue(); + } + } + + @Test + public void sliceStartWith() { + try (final Slice slice = new Slice("matchpoint"); + final Slice match = new Slice("mat"); + final Slice noMatch = new Slice("nomatch")) { + assertThat(slice.startsWith(match)).isTrue(); + assertThat(slice.startsWith(noMatch)).isFalse(); + } + } + + @Test + public void sliceToString() { + try (final Slice slice = new Slice("stringTest")) { + assertThat(slice.toString()).isEqualTo("stringTest"); + assertThat(slice.toString(true)).isNotEqualTo(""); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/SnapshotTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/SnapshotTest.java new file mode 100644 index 0000000..11f0d56 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/SnapshotTest.java @@ -0,0 +1,169 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import static org.assertj.core.api.Assertions.assertThat; + +public class SnapshotTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void snapshots() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath())) { + db.put("key".getBytes(), "value".getBytes()); + // Get new Snapshot of database + try (final Snapshot snapshot = db.getSnapshot()) { + assertThat(snapshot.getSequenceNumber()).isGreaterThan(0); + assertThat(snapshot.getSequenceNumber()).isEqualTo(1); + try (final ReadOptions readOptions = new ReadOptions()) { + // set snapshot in ReadOptions + readOptions.setSnapshot(snapshot); + + // retrieve key value pair + assertThat(new String(db.get("key".getBytes()))). + isEqualTo("value"); + // retrieve key value pair created before + // the snapshot was made + assertThat(new String(db.get(readOptions, + "key".getBytes()))).isEqualTo("value"); + // add new key/value pair + db.put("newkey".getBytes(), "newvalue".getBytes()); + // using no snapshot the latest db entries + // will be taken into account + assertThat(new String(db.get("newkey".getBytes()))). + isEqualTo("newvalue"); + // snapshopot was created before newkey + assertThat(db.get(readOptions, "newkey".getBytes())). + isNull(); + // Retrieve snapshot from read options + try (final Snapshot sameSnapshot = readOptions.snapshot()) { + readOptions.setSnapshot(sameSnapshot); + // results must be the same with new Snapshot + // instance using the same native pointer + assertThat(new String(db.get(readOptions, + "key".getBytes()))).isEqualTo("value"); + // update key value pair to newvalue + db.put("key".getBytes(), "newvalue".getBytes()); + // read with previously created snapshot will + // read previous version of key value pair + assertThat(new String(db.get(readOptions, + "key".getBytes()))).isEqualTo("value"); + // read for newkey using the snapshot must be + // null + assertThat(db.get(readOptions, "newkey".getBytes())). + isNull(); + // setting null to snapshot in ReadOptions leads + // to no Snapshot being used. + readOptions.setSnapshot(null); + assertThat(new String(db.get(readOptions, + "newkey".getBytes()))).isEqualTo("newvalue"); + // release Snapshot + db.releaseSnapshot(snapshot); + } + } + } + } + } + + @Test + public void iteratorWithSnapshot() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath())) { + db.put("key".getBytes(), "value".getBytes()); + + // Get new Snapshot of database + // set snapshot in ReadOptions + try (final Snapshot snapshot = db.getSnapshot(); + final ReadOptions readOptions = + new ReadOptions().setSnapshot(snapshot)) { + db.put("key2".getBytes(), "value2".getBytes()); + + // iterate over current state of db + try (final RocksIterator iterator = db.newIterator()) { + iterator.seekToFirst(); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key".getBytes()); + iterator.next(); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key2".getBytes()); + iterator.next(); + assertThat(iterator.isValid()).isFalse(); + } + + // iterate using a snapshot + try (final RocksIterator snapshotIterator = + db.newIterator(readOptions)) { + snapshotIterator.seekToFirst(); + assertThat(snapshotIterator.isValid()).isTrue(); + assertThat(snapshotIterator.key()).isEqualTo("key".getBytes()); + snapshotIterator.next(); + assertThat(snapshotIterator.isValid()).isFalse(); + } + + // release Snapshot + db.releaseSnapshot(snapshot); + } + } + } + + @Test + public void iteratorWithSnapshotOnColumnFamily() throws RocksDBException { + try (final Options options = new Options() + .setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath())) { + + db.put("key".getBytes(), "value".getBytes()); + + // Get new Snapshot of database + // set snapshot in ReadOptions + try (final Snapshot snapshot = db.getSnapshot(); + final ReadOptions readOptions = new ReadOptions() + .setSnapshot(snapshot)) { + db.put("key2".getBytes(), "value2".getBytes()); + + // iterate over current state of column family + try (final RocksIterator iterator = db.newIterator( + db.getDefaultColumnFamily())) { + iterator.seekToFirst(); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key".getBytes()); + iterator.next(); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key2".getBytes()); + iterator.next(); + assertThat(iterator.isValid()).isFalse(); + } + + // iterate using a snapshot on default column family + try (final RocksIterator snapshotIterator = db.newIterator( + db.getDefaultColumnFamily(), readOptions)) { + snapshotIterator.seekToFirst(); + assertThat(snapshotIterator.isValid()).isTrue(); + assertThat(snapshotIterator.key()).isEqualTo("key".getBytes()); + snapshotIterator.next(); + assertThat(snapshotIterator.isValid()).isFalse(); + + // release Snapshot + db.releaseSnapshot(snapshot); + } + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/SstFileManagerTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/SstFileManagerTest.java new file mode 100644 index 0000000..2e136e8 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/SstFileManagerTest.java @@ -0,0 +1,66 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.Test; + +import java.util.Collections; + +import static org.assertj.core.api.Assertions.*; + +public class SstFileManagerTest { + + @Test + public void maxAllowedSpaceUsage() throws RocksDBException { + try (final SstFileManager sstFileManager = new SstFileManager(Env.getDefault())) { + sstFileManager.setMaxAllowedSpaceUsage(1024 * 1024 * 64); + assertThat(sstFileManager.isMaxAllowedSpaceReached()).isFalse(); + assertThat(sstFileManager.isMaxAllowedSpaceReachedIncludingCompactions()).isFalse(); + } + } + + @Test + public void compactionBufferSize() throws RocksDBException { + try (final SstFileManager sstFileManager = new SstFileManager(Env.getDefault())) { + sstFileManager.setCompactionBufferSize(1024 * 1024 * 10); + assertThat(sstFileManager.isMaxAllowedSpaceReachedIncludingCompactions()).isFalse(); + } + } + + @Test + public void totalSize() throws RocksDBException { + try (final SstFileManager sstFileManager = new SstFileManager(Env.getDefault())) { + assertThat(sstFileManager.getTotalSize()).isEqualTo(0); + } + } + + @Test + public void trackedFiles() throws RocksDBException { + try (final SstFileManager sstFileManager = new SstFileManager(Env.getDefault())) { + assertThat(sstFileManager.getTrackedFiles()).isEqualTo(Collections.emptyMap()); + } + } + + @Test + public void deleteRateBytesPerSecond() throws RocksDBException { + try (final SstFileManager sstFileManager = new SstFileManager(Env.getDefault())) { + assertThat(sstFileManager.getDeleteRateBytesPerSecond()).isEqualTo(SstFileManager.RATE_BYTES_PER_SEC_DEFAULT); + final long ratePerSecond = 1024 * 1024 * 52; + sstFileManager.setDeleteRateBytesPerSecond(ratePerSecond); + assertThat(sstFileManager.getDeleteRateBytesPerSecond()).isEqualTo(ratePerSecond); + } + } + + @Test + public void maxTrashDBRatio() throws RocksDBException { + try (final SstFileManager sstFileManager = new SstFileManager(Env.getDefault())) { + assertThat(sstFileManager.getMaxTrashDBRatio()).isEqualTo(SstFileManager.MAX_TRASH_DB_RATION_DEFAULT); + final double trashRatio = 0.2; + sstFileManager.setMaxTrashDBRatio(trashRatio); + assertThat(sstFileManager.getMaxTrashDBRatio()).isEqualTo(trashRatio); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/SstFileReaderTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/SstFileReaderTest.java new file mode 100644 index 0000000..ef74b08 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/SstFileReaderTest.java @@ -0,0 +1,222 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.rocksdb.util.ByteBufferAllocator; + +@RunWith(Parameterized.class) +public class SstFileReaderTest { + private static final String SST_FILE_NAME = "test.sst"; + + static class KeyValueWithOp { + KeyValueWithOp(final String key, final String value, final OpType opType) { + this.key = key; + this.value = value; + this.opType = opType; + } + + String getKey() { + return key; + } + + String getValue() { + return value; + } + + OpType getOpType() { + return opType; + } + + private final String key; + private final String value; + private final OpType opType; + } + + @Rule public TemporaryFolder parentFolder = new TemporaryFolder(); + + @Parameterized.Parameters(name = "{0}") + public static Iterable parameters() { + return Arrays.asList(new Object[][] { + {"direct", ByteBufferAllocator.DIRECT}, {"indirect", ByteBufferAllocator.HEAP}}); + } + + @Parameterized.Parameter() public String name; + + @Parameterized.Parameter(1) public ByteBufferAllocator byteBufferAllocator; + + enum OpType { PUT, PUT_BYTES, MERGE, MERGE_BYTES, DELETE, DELETE_BYTES } + + private File newSstFile(final List keyValues) + throws IOException, RocksDBException { + final EnvOptions envOptions = new EnvOptions(); + final StringAppendOperator stringAppendOperator = new StringAppendOperator(); + final Options options = new Options().setMergeOperator(stringAppendOperator); + final SstFileWriter sstFileWriter; + sstFileWriter = new SstFileWriter(envOptions, options); + + final File sstFile = parentFolder.newFile(SST_FILE_NAME); + try { + sstFileWriter.open(sstFile.getAbsolutePath()); + for (final KeyValueWithOp keyValue : keyValues) { + final Slice keySlice = new Slice(keyValue.getKey()); + final Slice valueSlice = new Slice(keyValue.getValue()); + final byte[] keyBytes = keyValue.getKey().getBytes(); + final byte[] valueBytes = keyValue.getValue().getBytes(); + switch (keyValue.getOpType()) { + case PUT: + sstFileWriter.put(keySlice, valueSlice); + break; + case PUT_BYTES: + sstFileWriter.put(keyBytes, valueBytes); + break; + case MERGE: + sstFileWriter.merge(keySlice, valueSlice); + break; + case MERGE_BYTES: + sstFileWriter.merge(keyBytes, valueBytes); + break; + case DELETE: + sstFileWriter.delete(keySlice); + break; + case DELETE_BYTES: + sstFileWriter.delete(keyBytes); + break; + default: + fail("Unsupported op type"); + } + keySlice.close(); + valueSlice.close(); + } + sstFileWriter.finish(); + } finally { + assertThat(sstFileWriter).isNotNull(); + sstFileWriter.close(); + options.close(); + envOptions.close(); + } + return sstFile; + } + + @Test + public void readSstFile() throws RocksDBException, IOException { + final List keyValues = new ArrayList<>(); + keyValues.add(new KeyValueWithOp("key1", "value1", OpType.PUT)); + keyValues.add(new KeyValueWithOp("key2", "value2", OpType.PUT)); + keyValues.add(new KeyValueWithOp("key3", "value3", OpType.PUT)); + + final File sstFile = newSstFile(keyValues); + try (final StringAppendOperator stringAppendOperator = new StringAppendOperator(); + final Options options = + new Options().setCreateIfMissing(true).setMergeOperator(stringAppendOperator); + final SstFileReader reader = new SstFileReader(options)) { + // Open the sst file and iterator + reader.open(sstFile.getAbsolutePath()); + final ReadOptions readOptions = new ReadOptions(); + final SstFileReaderIterator iterator = reader.newIterator(readOptions); + + // Use the iterator to read sst file + iterator.seekToFirst(); + + // Verify Checksum + reader.verifyChecksum(); + + // Verify Table Properties + assertEquals(reader.getTableProperties().getNumEntries(), 3); + + // Check key and value + assertThat(iterator.key()).isEqualTo("key1".getBytes()); + assertThat(iterator.value()).isEqualTo("value1".getBytes()); + + final ByteBuffer byteBuffer = byteBufferAllocator.allocate(128); + byteBuffer.put("key1".getBytes()).flip(); + iterator.seek(byteBuffer); + assertThat(byteBuffer.position()).isEqualTo(4); + assertThat(byteBuffer.limit()).isEqualTo(4); + + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key1".getBytes()); + assertThat(iterator.value()).isEqualTo("value1".getBytes()); + + { + byteBuffer.clear(); + assertThat(iterator.key(byteBuffer)).isEqualTo("key1".getBytes().length); + final byte[] dst = new byte["key1".getBytes().length]; + byteBuffer.get(dst); + assertThat(new String(dst)).isEqualTo("key1"); + } + + { + byteBuffer.clear(); + byteBuffer.put("PREFIX".getBytes()); + final ByteBuffer slice = byteBuffer.slice(); + assertThat(iterator.key(byteBuffer)).isEqualTo("key1".getBytes().length); + final byte[] dst = new byte["key1".getBytes().length]; + slice.get(dst); + assertThat(new String(dst)).isEqualTo("key1"); + } + + { + byteBuffer.clear(); + assertThat(iterator.value(byteBuffer)).isEqualTo("value1".getBytes().length); + final byte[] dst = new byte["value1".getBytes().length]; + byteBuffer.get(dst); + assertThat(new String(dst)).isEqualTo("value1"); + } + + byteBuffer.clear(); + byteBuffer.put("key1point5".getBytes()).flip(); + iterator.seek(byteBuffer); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key2".getBytes()); + assertThat(iterator.value()).isEqualTo("value2".getBytes()); + + byteBuffer.clear(); + byteBuffer.put("key1point5".getBytes()).flip(); + iterator.seekForPrev(byteBuffer); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key1".getBytes()); + assertThat(iterator.value()).isEqualTo("value1".getBytes()); + + byteBuffer.clear(); + byteBuffer.put("key2point5".getBytes()).flip(); + iterator.seek(byteBuffer); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key3".getBytes()); + assertThat(iterator.value()).isEqualTo("value3".getBytes()); + + byteBuffer.clear(); + byteBuffer.put("key2point5".getBytes()).flip(); + iterator.seekForPrev(byteBuffer); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key2".getBytes()); + assertThat(iterator.value()).isEqualTo("value2".getBytes()); + + byteBuffer.clear(); + byteBuffer.put("PREFIX".getBytes()); + final ByteBuffer slice = byteBuffer.slice(); + slice.put("key1point5".getBytes()).flip(); + iterator.seekForPrev(slice); + assertThat(iterator.isValid()).isTrue(); + assertThat(iterator.key()).isEqualTo("key1".getBytes()); + assertThat(iterator.value()).isEqualTo("value1".getBytes()); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/SstFileWriterTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/SstFileWriterTest.java new file mode 100644 index 0000000..c0f4ed9 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/SstFileWriterTest.java @@ -0,0 +1,239 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.fail; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.util.BytewiseComparator; + +public class SstFileWriterTest { + private static final String SST_FILE_NAME = "test.sst"; + private static final String DB_DIRECTORY_NAME = "test_db"; + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE + = new RocksNativeLibraryResource(); + + @Rule public TemporaryFolder parentFolder = new TemporaryFolder(); + + enum OpType { PUT, PUT_BYTES, PUT_DIRECT, MERGE, MERGE_BYTES, DELETE, DELETE_BYTES } + + static class KeyValueWithOp { + KeyValueWithOp(final String key, final String value, final OpType opType) { + this.key = key; + this.value = value; + this.opType = opType; + } + + String getKey() { + return key; + } + + String getValue() { + return value; + } + + OpType getOpType() { + return opType; + } + + private final String key; + private final String value; + private final OpType opType; + } + + private File newSstFile(final List keyValues, + final boolean useJavaBytewiseComparator) throws IOException, RocksDBException { + final EnvOptions envOptions = new EnvOptions(); + final StringAppendOperator stringAppendOperator = new StringAppendOperator(); + final Options options = new Options().setMergeOperator(stringAppendOperator); + final SstFileWriter sstFileWriter; + ComparatorOptions comparatorOptions = null; + BytewiseComparator comparator = null; + if (useJavaBytewiseComparator) { + comparatorOptions = new ComparatorOptions().setUseDirectBuffer(false); + comparator = new BytewiseComparator(comparatorOptions); + options.setComparator(comparator); + sstFileWriter = new SstFileWriter(envOptions, options); + } else { + sstFileWriter = new SstFileWriter(envOptions, options); + } + + final File sstFile = parentFolder.newFile(SST_FILE_NAME); + try { + sstFileWriter.open(sstFile.getAbsolutePath()); + assertThat(sstFileWriter.fileSize()).isEqualTo(0); + for (final KeyValueWithOp keyValue : keyValues) { + final Slice keySlice = new Slice(keyValue.getKey()); + final Slice valueSlice = new Slice(keyValue.getValue()); + final byte[] keyBytes = keyValue.getKey().getBytes(); + final byte[] valueBytes = keyValue.getValue().getBytes(); + final ByteBuffer keyDirect = ByteBuffer.allocateDirect(keyBytes.length); + keyDirect.put(keyBytes); + keyDirect.flip(); + final ByteBuffer valueDirect = ByteBuffer.allocateDirect(valueBytes.length); + valueDirect.put(valueBytes); + valueDirect.flip(); + switch (keyValue.getOpType()) { + case PUT: + sstFileWriter.put(keySlice, valueSlice); + break; + case PUT_BYTES: + sstFileWriter.put(keyBytes, valueBytes); + break; + case PUT_DIRECT: + sstFileWriter.put(keyDirect, valueDirect); + assertThat(keyDirect.position()).isEqualTo(keyBytes.length); + assertThat(keyDirect.limit()).isEqualTo(keyBytes.length); + assertThat(valueDirect.position()).isEqualTo(valueBytes.length); + assertThat(valueDirect.limit()).isEqualTo(valueBytes.length); + break; + case MERGE: + sstFileWriter.merge(keySlice, valueSlice); + break; + case MERGE_BYTES: + sstFileWriter.merge(keyBytes, valueBytes); + break; + case DELETE: + sstFileWriter.delete(keySlice); + break; + case DELETE_BYTES: + sstFileWriter.delete(keyBytes); + break; + default: + fail("Unsupported op type"); + } + keySlice.close(); + valueSlice.close(); + } + sstFileWriter.finish(); + assertThat(sstFileWriter.fileSize()).isGreaterThan(100); + } finally { + assertThat(sstFileWriter).isNotNull(); + sstFileWriter.close(); + options.close(); + envOptions.close(); + if (comparatorOptions != null) { + comparatorOptions.close(); + } + if (comparator != null) { + comparator.close(); + } + } + return sstFile; + } + + @Test + public void generateSstFileWithJavaComparator() + throws RocksDBException, IOException { + final List keyValues = new ArrayList<>(); + keyValues.add(new KeyValueWithOp("key1", "value1", OpType.PUT)); + keyValues.add(new KeyValueWithOp("key2", "value2", OpType.PUT)); + keyValues.add(new KeyValueWithOp("key3", "value3", OpType.MERGE)); + keyValues.add(new KeyValueWithOp("key4", "value4", OpType.MERGE)); + keyValues.add(new KeyValueWithOp("key5", "", OpType.DELETE)); + + newSstFile(keyValues, true); + } + + @Test + public void generateSstFileWithNativeComparator() + throws RocksDBException, IOException { + final List keyValues = new ArrayList<>(); + keyValues.add(new KeyValueWithOp("key1", "value1", OpType.PUT)); + keyValues.add(new KeyValueWithOp("key2", "value2", OpType.PUT)); + keyValues.add(new KeyValueWithOp("key3", "value3", OpType.MERGE)); + keyValues.add(new KeyValueWithOp("key4", "value4", OpType.MERGE)); + keyValues.add(new KeyValueWithOp("key5", "", OpType.DELETE)); + + newSstFile(keyValues, false); + } + + @Test + public void ingestSstFile() throws RocksDBException, IOException { + final List keyValues = new ArrayList<>(); + keyValues.add(new KeyValueWithOp("key1", "value1", OpType.PUT)); + keyValues.add(new KeyValueWithOp("key2", "value2", OpType.PUT_DIRECT)); + keyValues.add(new KeyValueWithOp("key3", "value3", OpType.PUT_BYTES)); + keyValues.add(new KeyValueWithOp("key4", "value4", OpType.MERGE)); + keyValues.add(new KeyValueWithOp("key5", "value5", OpType.MERGE_BYTES)); + keyValues.add(new KeyValueWithOp("key6", "", OpType.DELETE)); + keyValues.add(new KeyValueWithOp("key7", "", OpType.DELETE)); + + + final File sstFile = newSstFile(keyValues, false); + final File dbFolder = parentFolder.newFolder(DB_DIRECTORY_NAME); + try(final StringAppendOperator stringAppendOperator = + new StringAppendOperator(); + final Options options = new Options() + .setCreateIfMissing(true) + .setMergeOperator(stringAppendOperator); + final RocksDB db = RocksDB.open(options, dbFolder.getAbsolutePath()); + final IngestExternalFileOptions ingestExternalFileOptions = + new IngestExternalFileOptions()) { + db.ingestExternalFile( + Collections.singletonList(sstFile.getAbsolutePath()), ingestExternalFileOptions); + + assertThat(db.get("key1".getBytes())).isEqualTo("value1".getBytes()); + assertThat(db.get("key2".getBytes())).isEqualTo("value2".getBytes()); + assertThat(db.get("key3".getBytes())).isEqualTo("value3".getBytes()); + assertThat(db.get("key4".getBytes())).isEqualTo("value4".getBytes()); + assertThat(db.get("key5".getBytes())).isEqualTo("value5".getBytes()); + assertThat(db.get("key6".getBytes())).isEqualTo(null); + assertThat(db.get("key7".getBytes())).isEqualTo(null); + } + } + + @Test + public void ingestSstFile_cf() throws RocksDBException, IOException { + final List keyValues = new ArrayList<>(); + keyValues.add(new KeyValueWithOp("key1", "value1", OpType.PUT)); + keyValues.add(new KeyValueWithOp("key2", "value2", OpType.PUT)); + keyValues.add(new KeyValueWithOp("key3", "value3", OpType.MERGE)); + keyValues.add(new KeyValueWithOp("key4", "", OpType.DELETE)); + + final File sstFile = newSstFile(keyValues, false); + final File dbFolder = parentFolder.newFolder(DB_DIRECTORY_NAME); + try(final StringAppendOperator stringAppendOperator = + new StringAppendOperator(); + final Options options = new Options() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true) + .setMergeOperator(stringAppendOperator); + final RocksDB db = RocksDB.open(options, dbFolder.getAbsolutePath()); + final IngestExternalFileOptions ingestExternalFileOptions = + new IngestExternalFileOptions()) { + + try(final ColumnFamilyOptions cf_opts = new ColumnFamilyOptions() + .setMergeOperator(stringAppendOperator); + final ColumnFamilyHandle cf_handle = db.createColumnFamily( + new ColumnFamilyDescriptor("new_cf".getBytes(), cf_opts))) { + db.ingestExternalFile(cf_handle, Collections.singletonList(sstFile.getAbsolutePath()), + ingestExternalFileOptions); + + assertThat(db.get(cf_handle, + "key1".getBytes())).isEqualTo("value1".getBytes()); + assertThat(db.get(cf_handle, + "key2".getBytes())).isEqualTo("value2".getBytes()); + assertThat(db.get(cf_handle, + "key3".getBytes())).isEqualTo("value3".getBytes()); + assertThat(db.get(cf_handle, + "key4".getBytes())).isEqualTo(null); + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/SstPartitionerTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/SstPartitionerTest.java new file mode 100644 index 0000000..3ee7390 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/SstPartitionerTest.java @@ -0,0 +1,72 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class SstPartitionerTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void sstFixedPrefix() throws RocksDBException { + try (final SstPartitionerFixedPrefixFactory factory = new SstPartitionerFixedPrefixFactory(4); + final Options opt = + new Options().setCreateIfMissing(true).setSstPartitionerFactory(factory); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + // writing (long)100 under key + db.put("aaaa1".getBytes(), "A".getBytes()); + db.put("bbbb1".getBytes(), "B".getBytes()); + db.flush(new FlushOptions()); + + db.put("aaaa0".getBytes(), "A2".getBytes()); + db.put("aaaa2".getBytes(), "A2".getBytes()); + db.flush(new FlushOptions()); + + db.compactRange(); + + final List metadata = db.getLiveFilesMetaData(); + assertThat(metadata.size()).isEqualTo(2); + } + } + + @Test + public void sstFixedPrefixFamily() throws RocksDBException { + final byte[] cfName = "new_cf".getBytes(UTF_8); + final ColumnFamilyDescriptor cfDescriptor = new ColumnFamilyDescriptor(cfName, + new ColumnFamilyOptions().setSstPartitionerFactory( + new SstPartitionerFixedPrefixFactory(4))); + + try (final Options opt = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { + final ColumnFamilyHandle columnFamilyHandle = db.createColumnFamily(cfDescriptor); + + // writing (long)100 under key + db.put(columnFamilyHandle, "aaaa1".getBytes(), "A".getBytes()); + db.put(columnFamilyHandle, "bbbb1".getBytes(), "B".getBytes()); + db.flush(new FlushOptions(), columnFamilyHandle); + + db.put(columnFamilyHandle, "aaaa0".getBytes(), "A2".getBytes()); + db.put(columnFamilyHandle, "aaaa2".getBytes(), "A2".getBytes()); + db.flush(new FlushOptions(), columnFamilyHandle); + + db.compactRange(columnFamilyHandle); + + final List metadata = db.getLiveFilesMetaData(); + assertThat(metadata.size()).isEqualTo(2); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/StatisticsCollectorTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/StatisticsCollectorTest.java new file mode 100644 index 0000000..36721c8 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/StatisticsCollectorTest.java @@ -0,0 +1,55 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Collections; + +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import static org.assertj.core.api.Assertions.assertThat; + +public class StatisticsCollectorTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void statisticsCollector() + throws InterruptedException, RocksDBException { + try (final Statistics statistics = new Statistics(); + final Options opt = new Options() + .setStatistics(statistics) + .setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + + try(final Statistics stats = opt.statistics()) { + + final StatsCallbackMock callback = new StatsCallbackMock(); + final StatsCollectorInput statsInput = + new StatsCollectorInput(stats, callback); + + final StatisticsCollector statsCollector = new StatisticsCollector( + Collections.singletonList(statsInput), 100); + statsCollector.start(); + + Thread.sleep(1000); + + assertThat(callback.tickerCallbackCount).isGreaterThan(0); + assertThat(callback.histCallbackCount).isGreaterThan(0); + + statsCollector.shutDown(1000); + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/StatisticsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/StatisticsTest.java new file mode 100644 index 0000000..de92102 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/StatisticsTest.java @@ -0,0 +1,168 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.nio.charset.StandardCharsets; + +import static org.assertj.core.api.Assertions.assertThat; + +public class StatisticsTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void statsLevel() throws RocksDBException { + final Statistics statistics = new Statistics(); + statistics.setStatsLevel(StatsLevel.ALL); + assertThat(statistics.statsLevel()).isEqualTo(StatsLevel.ALL); + } + + @Test + public void getTickerCount() throws RocksDBException { + try (final Statistics statistics = new Statistics(); + final Options opt = new Options() + .setStatistics(statistics) + .setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + + final byte[] key = "some-key".getBytes(StandardCharsets.UTF_8); + final byte[] value = "some-value".getBytes(StandardCharsets.UTF_8); + + db.put(key, value); + for(int i = 0; i < 10; i++) { + db.get(key); + } + + assertThat(statistics.getTickerCount(TickerType.BYTES_READ)).isGreaterThan(0); + } + } + + @Test + public void getAndResetTickerCount() throws RocksDBException { + try (final Statistics statistics = new Statistics(); + final Options opt = new Options() + .setStatistics(statistics) + .setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + + final byte[] key = "some-key".getBytes(StandardCharsets.UTF_8); + final byte[] value = "some-value".getBytes(StandardCharsets.UTF_8); + + db.put(key, value); + for(int i = 0; i < 10; i++) { + db.get(key); + } + + final long read = statistics.getAndResetTickerCount(TickerType.BYTES_READ); + assertThat(read).isGreaterThan(0); + + final long readAfterReset = statistics.getTickerCount(TickerType.BYTES_READ); + assertThat(readAfterReset).isLessThan(read); + } + } + + @Test + public void getHistogramData() throws RocksDBException { + try (final Statistics statistics = new Statistics(); + final Options opt = new Options() + .setStatistics(statistics) + .setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + + final byte[] key = "some-key".getBytes(StandardCharsets.UTF_8); + final byte[] value = "some-value".getBytes(StandardCharsets.UTF_8); + + db.put(key, value); + for(int i = 0; i < 10; i++) { + db.get(key); + } + + final HistogramData histogramData = statistics.getHistogramData(HistogramType.BYTES_PER_READ); + assertThat(histogramData).isNotNull(); + assertThat(histogramData.getAverage()).isGreaterThan(0); + assertThat(histogramData.getMedian()).isGreaterThan(0); + assertThat(histogramData.getPercentile95()).isGreaterThan(0); + assertThat(histogramData.getPercentile99()).isGreaterThan(0); + assertThat(histogramData.getStandardDeviation()).isEqualTo(0.00); + assertThat(histogramData.getMax()).isGreaterThan(0); + assertThat(histogramData.getCount()).isGreaterThan(0); + assertThat(histogramData.getSum()).isGreaterThan(0); + assertThat(histogramData.getMin()).isGreaterThan(0); + } + } + + @Test + public void getHistogramString() throws RocksDBException { + try (final Statistics statistics = new Statistics(); + final Options opt = new Options() + .setStatistics(statistics) + .setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + + final byte[] key = "some-key".getBytes(StandardCharsets.UTF_8); + final byte[] value = "some-value".getBytes(StandardCharsets.UTF_8); + + for(int i = 0; i < 10; i++) { + db.put(key, value); + } + + assertThat(statistics.getHistogramString(HistogramType.BYTES_PER_WRITE)).isNotNull(); + } + } + + @Test + public void reset() throws RocksDBException { + try (final Statistics statistics = new Statistics(); + final Options opt = new Options() + .setStatistics(statistics) + .setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + + final byte[] key = "some-key".getBytes(StandardCharsets.UTF_8); + final byte[] value = "some-value".getBytes(StandardCharsets.UTF_8); + + db.put(key, value); + for(int i = 0; i < 10; i++) { + db.get(key); + } + + final long read = statistics.getTickerCount(TickerType.BYTES_READ); + assertThat(read).isGreaterThan(0); + + statistics.reset(); + + final long readAfterReset = statistics.getTickerCount(TickerType.BYTES_READ); + assertThat(readAfterReset).isLessThan(read); + } + } + + @Test + public void ToString() throws RocksDBException { + try (final Statistics statistics = new Statistics(); + final Options opt = new Options() + .setStatistics(statistics) + .setCreateIfMissing(true); + final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath())) { + assertThat(statistics.toString()).isNotNull(); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/StatsCallbackMock.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/StatsCallbackMock.java new file mode 100644 index 0000000..c6a7294 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/StatsCallbackMock.java @@ -0,0 +1,19 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +public class StatsCallbackMock implements StatisticsCollectorCallback { + public int tickerCallbackCount = 0; + public int histCallbackCount = 0; + + public void tickerCallback(final TickerType tickerType, final long tickerCount) { + tickerCallbackCount++; + } + + public void histogramCallback(final HistogramType histType, final HistogramData histData) { + histCallbackCount++; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TableFilterTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TableFilterTest.java new file mode 100644 index 0000000..2bd3b17 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TableFilterTest.java @@ -0,0 +1,106 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; + +public class TableFilterTest { + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void readOptions() throws RocksDBException { + try (final DBOptions opt = new DBOptions(). + setCreateIfMissing(true). + setCreateMissingColumnFamilies(true); + final ColumnFamilyOptions new_cf_opts = new ColumnFamilyOptions() + ) { + final List columnFamilyDescriptors = + Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes(), new_cf_opts) + ); + + final List columnFamilyHandles = new ArrayList<>(); + + // open database + try (final RocksDB db = RocksDB.open(opt, + dbFolder.getRoot().getAbsolutePath(), + columnFamilyDescriptors, + columnFamilyHandles)) { + + try (final CfNameCollectionTableFilter cfNameCollectingTableFilter = + new CfNameCollectionTableFilter(); + final FlushOptions flushOptions = + new FlushOptions().setWaitForFlush(true); + final ReadOptions readOptions = + new ReadOptions().setTableFilter(cfNameCollectingTableFilter)) { + + db.put(columnFamilyHandles.get(0), + "key1".getBytes(UTF_8), "value1".getBytes(UTF_8)); + db.put(columnFamilyHandles.get(0), + "key2".getBytes(UTF_8), "value2".getBytes(UTF_8)); + db.put(columnFamilyHandles.get(0), + "key3".getBytes(UTF_8), "value3".getBytes(UTF_8)); + db.put(columnFamilyHandles.get(1), + "key1".getBytes(UTF_8), "value1".getBytes(UTF_8)); + db.put(columnFamilyHandles.get(1), + "key2".getBytes(UTF_8), "value2".getBytes(UTF_8)); + db.put(columnFamilyHandles.get(1), + "key3".getBytes(UTF_8), "value3".getBytes(UTF_8)); + + db.flush(flushOptions, columnFamilyHandles); + + try (final RocksIterator iterator = + db.newIterator(columnFamilyHandles.get(0), readOptions)) { + iterator.seekToFirst(); + while (iterator.isValid()) { + iterator.key(); + iterator.value(); + iterator.next(); + } + } + + try (final RocksIterator iterator = + db.newIterator(columnFamilyHandles.get(1), readOptions)) { + iterator.seekToFirst(); + while (iterator.isValid()) { + iterator.key(); + iterator.value(); + iterator.next(); + } + } + + assertThat(cfNameCollectingTableFilter.cfNames.size()).isEqualTo(2); + assertThat(cfNameCollectingTableFilter.cfNames.get(0)) + .isEqualTo(RocksDB.DEFAULT_COLUMN_FAMILY); + assertThat(cfNameCollectingTableFilter.cfNames.get(1)) + .isEqualTo("new_cf".getBytes(UTF_8)); + } finally { + for (final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandles) { + columnFamilyHandle.close(); + } + } + } + } + } + + private static class CfNameCollectionTableFilter extends AbstractTableFilter { + private final List cfNames = new ArrayList<>(); + + @Override + public boolean filter(final TableProperties tableProperties) { + cfNames.add(tableProperties.getColumnFamilyName()); + return true; + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TimedEnvTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TimedEnvTest.java new file mode 100644 index 0000000..31bad2e --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TimedEnvTest.java @@ -0,0 +1,40 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import static java.nio.charset.StandardCharsets.UTF_8; + +public class TimedEnvTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void construct() throws RocksDBException { + try (final Env env = new TimedEnv(Env.getDefault())) { + // no-op + } + } + + @Test + public void construct_integration() throws RocksDBException { + try (final Env env = new TimedEnv(Env.getDefault()); + final Options options = new Options().setCreateIfMissing(true).setEnv(env)) { + try (final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getPath())) { + db.put("key1".getBytes(UTF_8), "value1".getBytes(UTF_8)); + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TransactionDBOptionsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TransactionDBOptionsTest.java new file mode 100644 index 0000000..7eaa6b1 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TransactionDBOptionsTest.java @@ -0,0 +1,64 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.Test; + +import java.util.Random; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TransactionDBOptionsTest { + + private static final Random rand = PlatformRandomHelper. + getPlatformSpecificRandomFactory(); + + @Test + public void maxNumLocks() { + try (final TransactionDBOptions opt = new TransactionDBOptions()) { + final long longValue = rand.nextLong(); + opt.setMaxNumLocks(longValue); + assertThat(opt.getMaxNumLocks()).isEqualTo(longValue); + } + } + + @Test + public void maxNumStripes() { + try (final TransactionDBOptions opt = new TransactionDBOptions()) { + final long longValue = rand.nextLong(); + opt.setNumStripes(longValue); + assertThat(opt.getNumStripes()).isEqualTo(longValue); + } + } + + @Test + public void transactionLockTimeout() { + try (final TransactionDBOptions opt = new TransactionDBOptions()) { + final long longValue = rand.nextLong(); + opt.setTransactionLockTimeout(longValue); + assertThat(opt.getTransactionLockTimeout()).isEqualTo(longValue); + } + } + + @Test + public void defaultLockTimeout() { + try (final TransactionDBOptions opt = new TransactionDBOptions()) { + final long longValue = rand.nextLong(); + opt.setDefaultLockTimeout(longValue); + assertThat(opt.getDefaultLockTimeout()).isEqualTo(longValue); + } + } + + @Test + public void writePolicy() { + try (final TransactionDBOptions opt = new TransactionDBOptions()) { + final TxnDBWritePolicy writePolicy = TxnDBWritePolicy.WRITE_UNPREPARED; // non-default + opt.setWritePolicy(writePolicy); + assertThat(opt.getWritePolicy()).isEqualTo(writePolicy); + } + } + +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TransactionDBTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TransactionDBTest.java new file mode 100644 index 0000000..56acb21 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TransactionDBTest.java @@ -0,0 +1,177 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.util.*; + +import static org.assertj.core.api.Assertions.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; + +public class TransactionDBTest { + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void open() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final TransactionDBOptions txnDbOptions = new TransactionDBOptions(); + final TransactionDB tdb = TransactionDB.open(options, txnDbOptions, + dbFolder.getRoot().getAbsolutePath())) { + assertThat(tdb).isNotNull(); + } + } + + @Test + public void open_columnFamilies() throws RocksDBException { + try(final DBOptions dbOptions = new DBOptions().setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + final ColumnFamilyOptions myCfOpts = new ColumnFamilyOptions()) { + + final List columnFamilyDescriptors = + Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("myCf".getBytes(), myCfOpts)); + + final List columnFamilyHandles = new ArrayList<>(); + + try (final TransactionDBOptions txnDbOptions = new TransactionDBOptions(); + final TransactionDB tdb = TransactionDB.open(dbOptions, txnDbOptions, + dbFolder.getRoot().getAbsolutePath(), + columnFamilyDescriptors, columnFamilyHandles)) { + try { + assertThat(tdb).isNotNull(); + } finally { + for (final ColumnFamilyHandle handle : columnFamilyHandles) { + handle.close(); + } + } + } + } + } + + @Test + public void beginTransaction() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final TransactionDBOptions txnDbOptions = new TransactionDBOptions(); + final TransactionDB tdb = TransactionDB.open(options, txnDbOptions, + dbFolder.getRoot().getAbsolutePath()); + final WriteOptions writeOptions = new WriteOptions()) { + + try(final Transaction txn = tdb.beginTransaction(writeOptions)) { + assertThat(txn).isNotNull(); + } + } + } + + @Test + public void beginTransaction_transactionOptions() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final TransactionDBOptions txnDbOptions = new TransactionDBOptions(); + final TransactionDB tdb = TransactionDB.open(options, txnDbOptions, + dbFolder.getRoot().getAbsolutePath()); + final WriteOptions writeOptions = new WriteOptions(); + final TransactionOptions txnOptions = new TransactionOptions()) { + + try(final Transaction txn = tdb.beginTransaction(writeOptions, + txnOptions)) { + assertThat(txn).isNotNull(); + } + } + } + + @Test + public void beginTransaction_withOld() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final TransactionDBOptions txnDbOptions = new TransactionDBOptions(); + final TransactionDB tdb = TransactionDB.open(options, txnDbOptions, + dbFolder.getRoot().getAbsolutePath()); + final WriteOptions writeOptions = new WriteOptions()) { + + try(final Transaction txn = tdb.beginTransaction(writeOptions)) { + final Transaction txnReused = tdb.beginTransaction(writeOptions, txn); + assertThat(txnReused).isSameAs(txn); + } + } + } + + @Test + public void beginTransaction_withOld_transactionOptions() + throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final TransactionDBOptions txnDbOptions = new TransactionDBOptions(); + final TransactionDB tdb = TransactionDB.open(options, txnDbOptions, + dbFolder.getRoot().getAbsolutePath()); + final WriteOptions writeOptions = new WriteOptions(); + final TransactionOptions txnOptions = new TransactionOptions()) { + + try(final Transaction txn = tdb.beginTransaction(writeOptions)) { + final Transaction txnReused = tdb.beginTransaction(writeOptions, + txnOptions, txn); + assertThat(txnReused).isSameAs(txn); + } + } + } + + @Test + public void lockStatusData() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final TransactionDBOptions txnDbOptions = new TransactionDBOptions(); + final TransactionDB tdb = TransactionDB.open(options, txnDbOptions, + dbFolder.getRoot().getAbsolutePath()); + final WriteOptions writeOptions = new WriteOptions(); + final ReadOptions readOptions = new ReadOptions()) { + + try (final Transaction txn = tdb.beginTransaction(writeOptions)) { + final byte[] key = "key".getBytes(UTF_8); + final byte[] value = "value".getBytes(UTF_8); + + txn.put(key, value); + assertThat(txn.getForUpdate(readOptions, key, true)).isEqualTo(value); + + final Map lockStatus = + tdb.getLockStatusData(); + + assertThat(lockStatus.size()).isEqualTo(1); + final Set> entrySet = lockStatus.entrySet(); + final Map.Entry entry = entrySet.iterator().next(); + final long columnFamilyId = entry.getKey(); + assertThat(columnFamilyId).isEqualTo(0); + final TransactionDB.KeyLockInfo keyLockInfo = entry.getValue(); + assertThat(keyLockInfo.getKey()).isEqualTo(new String(key, UTF_8)); + assertThat(keyLockInfo.getTransactionIDs().length).isEqualTo(1); + assertThat(keyLockInfo.getTransactionIDs()[0]).isEqualTo(txn.getId()); + assertThat(keyLockInfo.isExclusive()).isTrue(); + } + } + } + + @Test + public void deadlockInfoBuffer() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final TransactionDBOptions txnDbOptions = new TransactionDBOptions(); + final TransactionDB tdb = TransactionDB.open(options, txnDbOptions, + dbFolder.getRoot().getAbsolutePath())) { + + // TODO(AR) can we cause a deadlock so that we can test the output here? + assertThat(tdb.getDeadlockInfoBuffer()).isEmpty(); + } + } + + @Test + public void setDeadlockInfoBufferSize() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final TransactionDBOptions txnDbOptions = new TransactionDBOptions(); + final TransactionDB tdb = TransactionDB.open(options, txnDbOptions, + dbFolder.getRoot().getAbsolutePath())) { + tdb.setDeadlockInfoBufferSize(123); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TransactionLogIteratorTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TransactionLogIteratorTest.java new file mode 100644 index 0000000..3c4dff7 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TransactionLogIteratorTest.java @@ -0,0 +1,139 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TransactionLogIteratorTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void transactionLogIterator() throws RocksDBException { + try (final Options options = new Options() + .setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath()); + final TransactionLogIterator transactionLogIterator = + db.getUpdatesSince(0)) { + //no-op + } + } + + @Test + public void getBatch() throws RocksDBException { + final int numberOfPuts = 5; + try (final Options options = new Options() + .setCreateIfMissing(true) + .setWalTtlSeconds(1000) + .setWalSizeLimitMB(10); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath())) { + + for (int i = 0; i < numberOfPuts; i++) { + db.put(String.valueOf(i).getBytes(), + String.valueOf(i).getBytes()); + } + db.flush(new FlushOptions().setWaitForFlush(true)); + + // the latest sequence number is 5 because 5 puts + // were written beforehand + assertThat(db.getLatestSequenceNumber()). + isEqualTo(numberOfPuts); + + // insert 5 writes into a cf + try (final ColumnFamilyHandle cfHandle = db.createColumnFamily( + new ColumnFamilyDescriptor("new_cf".getBytes()))) { + for (int i = 0; i < numberOfPuts; i++) { + db.put(cfHandle, String.valueOf(i).getBytes(), + String.valueOf(i).getBytes()); + } + // the latest sequence number is 10 because + // (5 + 5) puts were written beforehand + assertThat(db.getLatestSequenceNumber()). + isEqualTo(numberOfPuts + numberOfPuts); + + // Get updates since the beginning + try (final TransactionLogIterator transactionLogIterator = + db.getUpdatesSince(0)) { + assertThat(transactionLogIterator.isValid()).isTrue(); + transactionLogIterator.status(); + + // The first sequence number is 1 + final TransactionLogIterator.BatchResult batchResult = + transactionLogIterator.getBatch(); + assertThat(batchResult.sequenceNumber()).isEqualTo(1); + } + } + } + } + + @Test + public void transactionLogIteratorStallAtLastRecord() + throws RocksDBException { + try (final Options options = new Options() + .setCreateIfMissing(true) + .setWalTtlSeconds(1000) + .setWalSizeLimitMB(10); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath())) { + + db.put("key1".getBytes(), "value1".getBytes()); + // Get updates since the beginning + try (final TransactionLogIterator transactionLogIterator = + db.getUpdatesSince(0)) { + transactionLogIterator.status(); + assertThat(transactionLogIterator.isValid()).isTrue(); + transactionLogIterator.next(); + assertThat(transactionLogIterator.isValid()).isFalse(); + transactionLogIterator.status(); + db.put("key2".getBytes(), "value2".getBytes()); + transactionLogIterator.next(); + transactionLogIterator.status(); + assertThat(transactionLogIterator.isValid()).isTrue(); + } + } + } + + @Test + public void transactionLogIteratorCheckAfterRestart() + throws RocksDBException { + final int numberOfKeys = 2; + try (final Options options = new Options() + .setCreateIfMissing(true) + .setWalTtlSeconds(1000) + .setWalSizeLimitMB(10)) { + + try (final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath())) { + db.put("key1".getBytes(), "value1".getBytes()); + db.put("key2".getBytes(), "value2".getBytes()); + db.flush(new FlushOptions().setWaitForFlush(true)); + + } + + // reopen + try (final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath())) { + assertThat(db.getLatestSequenceNumber()).isEqualTo(numberOfKeys); + + try (final TransactionLogIterator transactionLogIterator = + db.getUpdatesSince(0)) { + for (int i = 0; i < numberOfKeys; i++) { + transactionLogIterator.status(); + assertThat(transactionLogIterator.isValid()).isTrue(); + transactionLogIterator.next(); + } + } + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TransactionOptionsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TransactionOptionsTest.java new file mode 100644 index 0000000..add0439 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TransactionOptionsTest.java @@ -0,0 +1,72 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.Test; + +import java.util.Random; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TransactionOptionsTest { + + private static final Random rand = PlatformRandomHelper. + getPlatformSpecificRandomFactory(); + + @Test + public void snapshot() { + try (final TransactionOptions opt = new TransactionOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setSetSnapshot(boolValue); + assertThat(opt.isSetSnapshot()).isEqualTo(boolValue); + } + } + + @Test + public void deadlockDetect() { + try (final TransactionOptions opt = new TransactionOptions()) { + final boolean boolValue = rand.nextBoolean(); + opt.setDeadlockDetect(boolValue); + assertThat(opt.isDeadlockDetect()).isEqualTo(boolValue); + } + } + + @Test + public void lockTimeout() { + try (final TransactionOptions opt = new TransactionOptions()) { + final long longValue = rand.nextLong(); + opt.setLockTimeout(longValue); + assertThat(opt.getLockTimeout()).isEqualTo(longValue); + } + } + + @Test + public void expiration() { + try (final TransactionOptions opt = new TransactionOptions()) { + final long longValue = rand.nextLong(); + opt.setExpiration(longValue); + assertThat(opt.getExpiration()).isEqualTo(longValue); + } + } + + @Test + public void deadlockDetectDepth() { + try (final TransactionOptions opt = new TransactionOptions()) { + final long longValue = rand.nextLong(); + opt.setDeadlockDetectDepth(longValue); + assertThat(opt.getDeadlockDetectDepth()).isEqualTo(longValue); + } + } + + @Test + public void maxWriteBatchSize() { + try (final TransactionOptions opt = new TransactionOptions()) { + final long longValue = rand.nextLong(); + opt.setMaxWriteBatchSize(longValue); + assertThat(opt.getMaxWriteBatchSize()).isEqualTo(longValue); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TransactionTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TransactionTest.java new file mode 100644 index 0000000..b80445c --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TransactionTest.java @@ -0,0 +1,488 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +public class TransactionTest extends AbstractTransactionTest { + + @Test + public void getForUpdate_cf_conflict() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + final byte[] v12 = "value12".getBytes(UTF_8); + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + + try(final Transaction txn = dbContainer.beginTransaction()) { + txn.put(testCf, k1, v1); + assertThat(txn.getForUpdate(readOptions, testCf, k1, true)).isEqualTo(v1); + txn.commit(); + } + + try(final Transaction txn2 = dbContainer.beginTransaction()) { + try(final Transaction txn3 = dbContainer.beginTransaction()) { + assertThat(txn3.getForUpdate(readOptions, testCf, k1, true)).isEqualTo(v1); + + // NOTE: txn2 updates k1, during txn3 + try { + txn2.put(testCf, k1, v12); // should cause an exception! + } catch(final RocksDBException e) { + assertThat(e.getStatus().getCode()).isSameAs(Status.Code.TimedOut); + return; + } + } + } + + fail("Expected an exception for put after getForUpdate from conflicting" + + "transactions"); + } + } + + @Test + public void prepare_commit() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + final byte[] v12 = "value12".getBytes(UTF_8); + + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions()) { + try (final Transaction txn = dbContainer.beginTransaction()) { + txn.put(k1, v1); + txn.commit(); + } + + try (final Transaction txn = dbContainer.beginTransaction()) { + txn.setName("txnPrepare1"); + txn.put(k1, v12); + txn.prepare(); + txn.commit(); + } + + try (final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.get(readOptions, k1)).isEqualTo(v12); + } + } + } + + @Test + public void prepare_rollback() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + final byte[] v12 = "value12".getBytes(UTF_8); + + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions()) { + try (final Transaction txn = dbContainer.beginTransaction()) { + txn.put(k1, v1); + txn.commit(); + } + + try (final Transaction txn = dbContainer.beginTransaction()) { + txn.setName("txnPrepare1"); + txn.put(k1, v12); + txn.prepare(); + txn.rollback(); + } + + try (final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.get(readOptions, k1)).isEqualTo(v1); + } + } + } + + @Test + public void prepare_read_prepared_commit() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + final byte[] v12 = "value12".getBytes(UTF_8); + + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions()) { + try (final Transaction txn = dbContainer.beginTransaction()) { + txn.put(k1, v1); + txn.commit(); + } + + final Transaction txnPrepare; + txnPrepare = dbContainer.beginTransaction(); + txnPrepare.setName("txnPrepare1"); + txnPrepare.put(k1, v12); + txnPrepare.prepare(); + + try (final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.get(readOptions, k1)).isEqualTo(v1); + } + + txnPrepare.commit(); + + try (final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.get(readOptions, k1)).isEqualTo(v12); + } + } + } + + @Test + public void prepare_read_prepared_rollback() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + final byte[] v12 = "value12".getBytes(UTF_8); + + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions()) { + try (final Transaction txn = dbContainer.beginTransaction()) { + txn.put(k1, v1); + txn.commit(); + } + + final Transaction txnPrepare; + txnPrepare = dbContainer.beginTransaction(); + txnPrepare.setName("txnPrepare1"); + txnPrepare.put(k1, v12); + txnPrepare.prepare(); + + try (final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.get(readOptions, k1)).isEqualTo(v1); + } + + txnPrepare.rollback(); + + try (final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.get(readOptions, k1)).isEqualTo(v1); + } + } + } + + @Test + public void getForUpdate_conflict() throws RocksDBException { + final byte[] k1 = "key1".getBytes(UTF_8); + final byte[] v1 = "value1".getBytes(UTF_8); + final byte[] v12 = "value12".getBytes(UTF_8); + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions()) { + + try(final Transaction txn = dbContainer.beginTransaction()) { + txn.put(k1, v1); + assertThat(txn.getForUpdate(readOptions, k1, true)).isEqualTo(v1); + txn.commit(); + } + + try(final Transaction txn2 = dbContainer.beginTransaction()) { + try(final Transaction txn3 = dbContainer.beginTransaction()) { + assertThat(txn3.getForUpdate(readOptions, k1, true)).isEqualTo(v1); + + // NOTE: txn2 updates k1, during txn3 + try { + txn2.put(k1, v12); // should cause an exception! + } catch(final RocksDBException e) { + assertThat(e.getStatus().getCode()).isSameAs(Status.Code.TimedOut); + return; + } + } + } + + fail("Expected an exception for put after getForUpdate from conflicting" + + "transactions"); + } + } + + @Test + public void multiGetForUpdate_cf_conflict() throws RocksDBException { + final byte[][] keys = new byte[][] {"key1".getBytes(UTF_8), "key2".getBytes(UTF_8)}; + final byte[][] values = new byte[][] {"value1".getBytes(UTF_8), "value2".getBytes(UTF_8)}; + final byte[] otherValue = "otherValue".getBytes(UTF_8); + + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + final List cfList = Arrays.asList(testCf, testCf); + + try(final Transaction txn = dbContainer.beginTransaction()) { + txn.put(testCf, keys[0], values[0]); + txn.put(testCf, keys[1], values[1]); + assertThat(txn.multiGet(readOptions, cfList, keys)).isEqualTo(values); + txn.commit(); + } + + try(final Transaction txn2 = dbContainer.beginTransaction()) { + try(final Transaction txn3 = dbContainer.beginTransaction()) { + assertThat(txn3.multiGetForUpdate(readOptions, cfList, keys)) + .isEqualTo(values); + + // NOTE: txn2 updates k1, during txn3 + try { + txn2.put(testCf, keys[0], otherValue); // should cause an exception! + } catch(final RocksDBException e) { + assertThat(e.getStatus().getCode()).isSameAs(Status.Code.TimedOut); + return; + } + } + } + + fail("Expected an exception for put after getForUpdate from conflicting" + + "transactions"); + } + } + + @Test + public void multiGetAsListForUpdate_cf_conflict() throws RocksDBException { + final byte[][] keys = new byte[][] {"key1".getBytes(UTF_8), "key2".getBytes(UTF_8)}; + final byte[][] values = new byte[][] {"value1".getBytes(UTF_8), "value2".getBytes(UTF_8)}; + final byte[] otherValue = "otherValue".getBytes(UTF_8); + + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions()) { + final ColumnFamilyHandle testCf = dbContainer.getTestColumnFamily(); + final List cfList = Arrays.asList(testCf, testCf); + + try (final Transaction txn = dbContainer.beginTransaction()) { + txn.put(testCf, keys[0], values[0]); + txn.put(testCf, keys[1], values[1]); + assertThat(txn.multiGetAsList(readOptions, cfList, Arrays.asList(keys))) + .containsExactly(values); + txn.commit(); + } + + try (final Transaction txn2 = dbContainer.beginTransaction()) { + try (final Transaction txn3 = dbContainer.beginTransaction()) { + assertThat(txn3.multiGetForUpdateAsList(readOptions, cfList, Arrays.asList(keys))) + .containsExactly(values); + + // NOTE: txn2 updates k1, during txn3 + try { + txn2.put(testCf, keys[0], otherValue); // should cause an exception! + } catch (final RocksDBException e) { + assertThat(e.getStatus().getCode()).isSameAs(Status.Code.TimedOut); + return; + } + } + } + + fail("Expected an exception for put after getForUpdate from conflicting" + + "transactions"); + } + } + + @Test + public void multiGetForUpdate_conflict() throws RocksDBException { + final byte[][] keys = new byte[][] {"key1".getBytes(UTF_8), "key2".getBytes(UTF_8)}; + final byte[][] values = new byte[][] {"value1".getBytes(UTF_8), "value2".getBytes(UTF_8)}; + final byte[] otherValue = "otherValue".getBytes(UTF_8); + + try(final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions()) { + try(final Transaction txn = dbContainer.beginTransaction()) { + txn.put(keys[0], values[0]); + txn.put(keys[1], values[1]); + assertThat(txn.multiGet(readOptions, keys)).isEqualTo(values); + txn.commit(); + } + + try(final Transaction txn2 = dbContainer.beginTransaction()) { + try(final Transaction txn3 = dbContainer.beginTransaction()) { + assertThat(txn3.multiGetForUpdate(readOptions, keys)) + .isEqualTo(values); + + // NOTE: txn2 updates k1, during txn3 + try { + txn2.put(keys[0], otherValue); // should cause an exception! + } catch (final RocksDBException e) { + assertThat(e.getStatus().getCode()).isSameAs(Status.Code.TimedOut); + return; + } + } + } + + fail("Expected an exception for put after getForUpdate from conflicting" + + "transactions"); + } + } + + @Test + public void multiGetAsListForUpdate_conflict() throws RocksDBException { + final byte[][] keys = new byte[][] {"key1".getBytes(UTF_8), "key2".getBytes(UTF_8)}; + final byte[][] values = new byte[][] {"value1".getBytes(UTF_8), "value2".getBytes(UTF_8)}; + final byte[] otherValue = "otherValue".getBytes(UTF_8); + + try (final DBContainer dbContainer = startDb(); + final ReadOptions readOptions = new ReadOptions()) { + try (final Transaction txn = dbContainer.beginTransaction()) { + txn.put(keys[0], values[0]); + txn.put(keys[1], values[1]); + assertThat(txn.multiGetAsList(readOptions, Arrays.asList(keys))).containsExactly(values); + txn.commit(); + } + + try (final Transaction txn2 = dbContainer.beginTransaction()) { + try (final Transaction txn3 = dbContainer.beginTransaction()) { + assertThat(txn3.multiGetForUpdateAsList(readOptions, Arrays.asList(keys))) + .containsExactly(values); + + // NOTE: txn2 updates k1, during txn3 + try { + txn2.put(keys[0], otherValue); // should cause an exception! + } catch(final RocksDBException e) { + assertThat(e.getStatus().getCode()).isSameAs(Status.Code.TimedOut); + return; + } + } + } + + fail("Expected an exception for put after getForUpdate from conflicting" + + "transactions"); + } + } + + @Test + public void name() throws RocksDBException { + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.getName()).isEmpty(); + final String name = "my-transaction-" + rand.nextLong(); + txn.setName(name); + assertThat(txn.getName()).isEqualTo(name); + } + } + + @Test + public void ID() throws RocksDBException { + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.getID()).isGreaterThan(0); + } + } + + @Test + public void deadlockDetect() throws RocksDBException { + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.isDeadlockDetect()).isFalse(); + } + } + + @Test + public void waitingTxns() throws RocksDBException { + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.getWaitingTxns().getTransactionIds().length).isEqualTo(0); + } + } + + @Test + public void state() throws RocksDBException { + try(final DBContainer dbContainer = startDb()) { + + try(final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.getState()) + .isSameAs(Transaction.TransactionState.STARTED); + txn.commit(); + assertThat(txn.getState()) + .isSameAs(Transaction.TransactionState.COMMITTED); + } + + try(final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.getState()) + .isSameAs(Transaction.TransactionState.STARTED); + txn.rollback(); + assertThat(txn.getState()) + .isSameAs(Transaction.TransactionState.STARTED); + } + } + } + + @Test + public void Id() throws RocksDBException { + try(final DBContainer dbContainer = startDb(); + final Transaction txn = dbContainer.beginTransaction()) { + assertThat(txn.getId()).isNotNull(); + } + } + + @Override + public TransactionDBContainer startDb() throws RocksDBException { + final DBOptions options = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true); + final TransactionDBOptions txnDbOptions = new TransactionDBOptions(); + final ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions(); + final List columnFamilyDescriptors = + Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor(TXN_TEST_COLUMN_FAMILY, + columnFamilyOptions)); + final List columnFamilyHandles = new ArrayList<>(); + + final TransactionDB txnDb; + try { + txnDb = TransactionDB.open(options, txnDbOptions, + dbFolder.getRoot().getAbsolutePath(), columnFamilyDescriptors, + columnFamilyHandles); + } catch(final RocksDBException e) { + columnFamilyOptions.close(); + txnDbOptions.close(); + options.close(); + throw e; + } + + final WriteOptions writeOptions = new WriteOptions(); + final TransactionOptions txnOptions = new TransactionOptions(); + + return new TransactionDBContainer(txnOptions, writeOptions, + columnFamilyHandles, txnDb, txnDbOptions, columnFamilyOptions, options); + } + + private static class TransactionDBContainer + extends DBContainer { + private final TransactionOptions txnOptions; + private final TransactionDB txnDb; + private final TransactionDBOptions txnDbOptions; + + public TransactionDBContainer( + final TransactionOptions txnOptions, final WriteOptions writeOptions, + final List columnFamilyHandles, + final TransactionDB txnDb, final TransactionDBOptions txnDbOptions, + final ColumnFamilyOptions columnFamilyOptions, + final DBOptions options) { + super(writeOptions, columnFamilyHandles, columnFamilyOptions, + options); + this.txnOptions = txnOptions; + this.txnDb = txnDb; + this.txnDbOptions = txnDbOptions; + } + + @Override + public Transaction beginTransaction() { + return txnDb.beginTransaction(writeOptions, txnOptions); + } + + @Override + public Transaction beginTransaction(final WriteOptions writeOptions) { + return txnDb.beginTransaction(writeOptions, txnOptions); + } + + @Override + public void close() { + txnOptions.close(); + writeOptions.close(); + for(final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandles) { + columnFamilyHandle.close(); + } + txnDb.close(); + txnDbOptions.close(); + options.close(); + } + } + +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TtlDBTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TtlDBTest.java new file mode 100644 index 0000000..ebf9e9e --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/TtlDBTest.java @@ -0,0 +1,112 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TtlDBTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void ttlDBOpen() throws RocksDBException, InterruptedException { + try (final Options options = new Options().setCreateIfMissing(true).setMaxCompactionBytes(0); + final TtlDB ttlDB = TtlDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + ttlDB.put("key".getBytes(), "value".getBytes()); + assertThat(ttlDB.get("key".getBytes())). + isEqualTo("value".getBytes()); + assertThat(ttlDB.get("key".getBytes())).isNotNull(); + } + } + + @Test + public void ttlDBOpenWithTtl() throws RocksDBException, InterruptedException { + try (final Options options = new Options().setCreateIfMissing(true).setMaxCompactionBytes(0); + final TtlDB ttlDB = TtlDB.open(options, dbFolder.getRoot().getAbsolutePath(), 1, false)) { + ttlDB.put("key".getBytes(), "value".getBytes()); + assertThat(ttlDB.get("key".getBytes())). + isEqualTo("value".getBytes()); + TimeUnit.SECONDS.sleep(2); + ttlDB.compactRange(); + assertThat(ttlDB.get("key".getBytes())).isNull(); + } + } + + @Test + public void ttlDbOpenWithColumnFamilies() throws RocksDBException, + InterruptedException { + final List cfNames = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes()) + ); + final List ttlValues = Arrays.asList(0, 1); + + final List columnFamilyHandleList = new ArrayList<>(); + try (final DBOptions dbOptions = new DBOptions() + .setCreateMissingColumnFamilies(true) + .setCreateIfMissing(true); + final TtlDB ttlDB = TtlDB.open(dbOptions, + dbFolder.getRoot().getAbsolutePath(), cfNames, + columnFamilyHandleList, ttlValues, false)) { + try { + ttlDB.put("key".getBytes(), "value".getBytes()); + assertThat(ttlDB.get("key".getBytes())). + isEqualTo("value".getBytes()); + ttlDB.put(columnFamilyHandleList.get(1), "key".getBytes(), + "value".getBytes()); + assertThat(ttlDB.get(columnFamilyHandleList.get(1), + "key".getBytes())).isEqualTo("value".getBytes()); + TimeUnit.SECONDS.sleep(2); + + ttlDB.compactRange(); + ttlDB.compactRange(columnFamilyHandleList.get(1)); + + assertThat(ttlDB.get("key".getBytes())).isNotNull(); + assertThat(ttlDB.get(columnFamilyHandleList.get(1), + "key".getBytes())).isNull(); + } finally { + for (final ColumnFamilyHandle columnFamilyHandle : + columnFamilyHandleList) { + columnFamilyHandle.close(); + } + } + } + } + + @Test + public void createTtlColumnFamily() throws RocksDBException, + InterruptedException { + try (final Options options = new Options().setCreateIfMissing(true); + final TtlDB ttlDB = TtlDB.open(options, + dbFolder.getRoot().getAbsolutePath()); + final ColumnFamilyHandle columnFamilyHandle = + ttlDB.createColumnFamilyWithTtl( + new ColumnFamilyDescriptor("new_cf".getBytes()), 1)) { + ttlDB.put(columnFamilyHandle, "key".getBytes(), + "value".getBytes()); + assertThat(ttlDB.get(columnFamilyHandle, "key".getBytes())). + isEqualTo("value".getBytes()); + TimeUnit.SECONDS.sleep(2); + ttlDB.compactRange(columnFamilyHandle); + assertThat(ttlDB.get(columnFamilyHandle, "key".getBytes())).isNull(); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/Types.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/Types.java new file mode 100644 index 0000000..a6abdec --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/Types.java @@ -0,0 +1,43 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * Simple type conversion methods + * for use in tests + */ +public class Types { + + /** + * Convert first 4 bytes of a byte array to an int + * + * @param data The byte array + * + * @return An integer + */ + public static int byteToInt(final byte[] data) { + return (data[0] & 0xff) | + ((data[1] & 0xff) << 8) | + ((data[2] & 0xff) << 16) | + ((data[3] & 0xff) << 24); + } + + /** + * Convert an int to 4 bytes + * + * @param v The int + * + * @return A byte array containing 4 bytes + */ + public static byte[] intToByte(final int v) { + return new byte[] { + (byte)((v >>> 0) & 0xff), + (byte)((v >>> 8) & 0xff), + (byte)((v >>> 16) & 0xff), + (byte)((v >>> 24) & 0xff) + }; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/VerifyChecksumsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/VerifyChecksumsTest.java new file mode 100644 index 0000000..ddc2a45 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/VerifyChecksumsTest.java @@ -0,0 +1,213 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class VerifyChecksumsTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + /** + * Class to factor out the specific DB operations within the test + */ + abstract static class Operations { + final int kv_count; + final List elements = new ArrayList<>(); + final List sortedElements = new ArrayList<>(); + + Operations(final int kv_count) { + this.kv_count = kv_count; + for (int i = 0; i < kv_count; i++) elements.add(MessageFormat.format("{0,number,#}", i)); + sortedElements.addAll(elements); + Collections.sort(sortedElements); + } + + void fill(final RocksDB db) throws RocksDBException { + for (int i = 0; i < kv_count; i++) { + final String key = MessageFormat.format("key{0}", elements.get(i)); + final String value = MessageFormat.format("value{0}", elements.get(i)); + // noinspection ObjectAllocationInLoop + db.put(key.getBytes(), value.getBytes()); + } + db.flush(new FlushOptions()); + } + + @SuppressWarnings("ObjectAllocationInLoop") + void get(final RocksDB db, final boolean verifyFlag) throws RocksDBException { + try (final ReadOptions readOptions = new ReadOptions()) { + readOptions.setReadaheadSize(32 * 1024); + readOptions.setFillCache(false); + readOptions.setVerifyChecksums(verifyFlag); + + for (int i = 0; i < kv_count / 10; i++) { + @SuppressWarnings("UnsecureRandomNumberGeneration") + final int index = Double.valueOf(Math.random() * kv_count).intValue(); + final String key = MessageFormat.format("key{0}", sortedElements.get(index)); + final String expectedValue = MessageFormat.format("value{0}", sortedElements.get(index)); + + final byte[] value = db.get(readOptions, key.getBytes()); + assertThat(value).isEqualTo(expectedValue.getBytes()); + } + } + } + + @SuppressWarnings("ObjectAllocationInLoop") + void multiGet(final RocksDB db, final boolean verifyFlag) throws RocksDBException { + try (final ReadOptions readOptions = new ReadOptions()) { + readOptions.setReadaheadSize(32 * 1024); + readOptions.setFillCache(false); + readOptions.setVerifyChecksums(verifyFlag); + + final List keys = new ArrayList<>(); + final List expectedValues = new ArrayList<>(); + + for (int i = 0; i < kv_count / 10; i++) { + @SuppressWarnings("UnsecureRandomNumberGeneration") + final int index = Double.valueOf(Math.random() * kv_count).intValue(); + keys.add(MessageFormat.format("key{0}", sortedElements.get(index)).getBytes()); + + expectedValues.add(MessageFormat.format("value{0}", sortedElements.get(index))); + } + + final List values = db.multiGetAsList(readOptions, keys); + for (int i = 0; i < keys.size(); i++) { + assertThat(values.get(i)).isEqualTo(expectedValues.get(i).getBytes()); + } + } + } + + void iterate(final RocksDB db, final boolean verifyFlag) throws RocksDBException { + final ReadOptions readOptions = new ReadOptions(); + readOptions.setReadaheadSize(32 * 1024); + readOptions.setFillCache(false); + readOptions.setVerifyChecksums(verifyFlag); + int i = 0; + try (final RocksIterator rocksIterator = db.newIterator(readOptions)) { + rocksIterator.seekToFirst(); + rocksIterator.status(); + while (rocksIterator.isValid()) { + final byte[] key = rocksIterator.key(); + final byte[] value = rocksIterator.value(); + // noinspection ObjectAllocationInLoop + assertThat(key).isEqualTo( + (MessageFormat.format("key{0}", sortedElements.get(i))).getBytes()); + // noinspection ObjectAllocationInLoop + assertThat(value).isEqualTo( + (MessageFormat.format("value{0}", sortedElements.get(i))).getBytes()); + rocksIterator.next(); + rocksIterator.status(); + i++; + } + } + assertThat(i).isEqualTo(kv_count); + } + + abstract void performOperations(final RocksDB db, final boolean verifyFlag) + throws RocksDBException; + } + + private static final int KV_COUNT = 10000; + + /** + * Run some operations and count the TickerType.BLOCK_CHECKSUM_COMPUTE_COUNT before and after + * It should GO UP when the read options have checksum verification turned on. + * It shoulld REMAIN UNCHANGED when the read options have checksum verification turned off. + * As the read options refer only to the read operations, there are still a few checksums + * performed outside this (blocks are getting loaded for lots of reasons, not aways directly due + * to reads) but this test provides a good enough proxy for whether the flag is being noticed. + * + * @param operations the DB reading operations to perform which affect the checksum stats + * + * @throws RocksDBException + */ + private void verifyChecksums(final Operations operations) throws RocksDBException { + final String dbPath = dbFolder.getRoot().getAbsolutePath(); + + // noinspection SingleStatementInBlock + try (final Statistics statistics = new Statistics(); + final Options options = new Options().setCreateIfMissing(true).setStatistics(statistics)) { + try (final RocksDB db = RocksDB.open(options, dbPath)) { + // 0 + System.out.println(MessageFormat.format( + "newly open {0}", statistics.getTickerCount(TickerType.BLOCK_CHECKSUM_COMPUTE_COUNT))); + operations.fill(db); + // + System.out.println(MessageFormat.format( + "flushed {0}", statistics.getTickerCount(TickerType.BLOCK_CHECKSUM_COMPUTE_COUNT))); + } + + // 2 + System.out.println(MessageFormat.format("closed-after-write {0}", + statistics.getTickerCount(TickerType.BLOCK_CHECKSUM_COMPUTE_COUNT))); + + for (final boolean verifyFlag : new boolean[] {false, true, false, true}) { + try (final RocksDB db = RocksDB.open(options, dbPath)) { + final long beforeOperationsCount = + statistics.getTickerCount(TickerType.BLOCK_CHECKSUM_COMPUTE_COUNT); + System.out.println(MessageFormat.format("re-opened {0}", beforeOperationsCount)); + operations.performOperations(db, verifyFlag); + final long afterOperationsCount = + statistics.getTickerCount(TickerType.BLOCK_CHECKSUM_COMPUTE_COUNT); + if (verifyFlag) { + // We don't need to be exact - we are checking that the checksums happen + // exactly how many depends on block size etc etc, so may not be entirely stable + System.out.println(MessageFormat.format("verify=true {0}", afterOperationsCount)); + assertThat(afterOperationsCount).isGreaterThan(beforeOperationsCount + 20); + } else { + System.out.println(MessageFormat.format("verify=false {0}", afterOperationsCount)); + assertThat(afterOperationsCount).isEqualTo(beforeOperationsCount); + } + } + } + } + } + + @Test + public void verifyChecksumsInIteration() throws RocksDBException { + // noinspection AnonymousInnerClassMayBeStatic + verifyChecksums(new Operations(KV_COUNT) { + @Override + void performOperations(final RocksDB db, final boolean verifyFlag) throws RocksDBException { + iterate(db, verifyFlag); + } + }); + } + + @Test + public void verifyChecksumsGet() throws RocksDBException { + // noinspection AnonymousInnerClassMayBeStatic + verifyChecksums(new Operations(KV_COUNT) { + @Override + void performOperations(final RocksDB db, final boolean verifyFlag) throws RocksDBException { + get(db, verifyFlag); + } + }); + } + + @Test + public void verifyChecksumsMultiGet() throws RocksDBException { + // noinspection AnonymousInnerClassMayBeStatic + verifyChecksums(new Operations(KV_COUNT) { + @Override + void performOperations(final RocksDB db, final boolean verifyFlag) throws RocksDBException { + multiGet(db, verifyFlag); + } + }); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/WALRecoveryModeTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/WALRecoveryModeTest.java new file mode 100644 index 0000000..2a0133f --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/WALRecoveryModeTest.java @@ -0,0 +1,22 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + + +public class WALRecoveryModeTest { + + @Test + public void getWALRecoveryMode() { + for (final WALRecoveryMode walRecoveryMode : WALRecoveryMode.values()) { + assertThat(WALRecoveryMode.getWALRecoveryMode(walRecoveryMode.getValue())) + .isEqualTo(walRecoveryMode); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/WalFilterTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/WalFilterTest.java new file mode 100644 index 0000000..08bc6ee --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/WalFilterTest.java @@ -0,0 +1,165 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.rocksdb.util.ByteUtil.bytes; +import static org.rocksdb.util.TestUtil.*; + +public class WalFilterTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void walFilter() throws RocksDBException { + // Create 3 batches with two keys each + final byte[][][] batchKeys = { + new byte[][] { + bytes("key1"), + bytes("key2") + }, + new byte[][] { + bytes("key3"), + bytes("key4") + }, + new byte[][] { + bytes("key5"), + bytes("key6") + } + + }; + + final List cfDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor(bytes("pikachu")) + ); + final List cfHandles = new ArrayList<>(); + + // Test with all WAL processing options + for (final WalProcessingOption option : WalProcessingOption.values()) { + try (final Options options = optionsForLogIterTest(); + final DBOptions dbOptions = new DBOptions(options) + .setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open(dbOptions, + dbFolder.getRoot().getAbsolutePath(), + cfDescriptors, cfHandles)) { + try (final WriteOptions writeOptions = new WriteOptions()) { + // Write given keys in given batches + for (final byte[][] batchKey : batchKeys) { + final WriteBatch batch = new WriteBatch(); + for (final byte[] bytes : batchKey) { + batch.put(cfHandles.get(0), bytes, dummyString(1024)); + } + db.write(writeOptions, batch); + } + } finally { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + cfHandles.clear(); + } + } + + // Create a test filter that would apply wal_processing_option at the first + // record + final int applyOptionForRecordIndex = 1; + try (final TestableWalFilter walFilter = + new TestableWalFilter(option, applyOptionForRecordIndex)) { + + try (final Options options = optionsForLogIterTest(); + final DBOptions dbOptions = new DBOptions(options) + .setWalFilter(walFilter)) { + + try (final RocksDB db = RocksDB.open(dbOptions, + dbFolder.getRoot().getAbsolutePath(), + cfDescriptors, cfHandles)) { + + try { + assertThat(walFilter.logNumbers).isNotEmpty(); + assertThat(walFilter.logFileNames).isNotEmpty(); + } finally { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + cfHandles.clear(); + } + } catch (final RocksDBException e) { + if (option != WalProcessingOption.CORRUPTED_RECORD) { + // exception is expected when CORRUPTED_RECORD! + throw e; + } + } + } + } + } + } + + + private static class TestableWalFilter extends AbstractWalFilter { + private final WalProcessingOption walProcessingOption; + private final int applyOptionForRecordIndex; + Map cfLognumber; + Map cfNameId; + final List logNumbers = new ArrayList<>(); + final List logFileNames = new ArrayList<>(); + private int currentRecordIndex = 0; + + public TestableWalFilter(final WalProcessingOption walProcessingOption, + final int applyOptionForRecordIndex) { + super(); + this.walProcessingOption = walProcessingOption; + this.applyOptionForRecordIndex = applyOptionForRecordIndex; + } + + @Override + public void columnFamilyLogNumberMap(final Map cfLognumber, + final Map cfNameId) { + this.cfLognumber = cfLognumber; + this.cfNameId = cfNameId; + } + + @Override + public LogRecordFoundResult logRecordFound( + final long logNumber, final String logFileName, final WriteBatch batch, + final WriteBatch newBatch) { + + logNumbers.add(logNumber); + logFileNames.add(logFileName); + + final WalProcessingOption optionToReturn; + if (currentRecordIndex == applyOptionForRecordIndex) { + optionToReturn = walProcessingOption; + } + else { + optionToReturn = WalProcessingOption.CONTINUE_PROCESSING; + } + + currentRecordIndex++; + + return new LogRecordFoundResult(optionToReturn, false); + } + + @Override + public String name() { + return "testable-wal-filter"; + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/WriteBatchHandlerTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/WriteBatchHandlerTest.java new file mode 100644 index 0000000..2826b12 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/WriteBatchHandlerTest.java @@ -0,0 +1,76 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import java.util.Arrays; +import java.util.List; + +import org.junit.ClassRule; +import org.junit.Test; +import org.rocksdb.util.CapturingWriteBatchHandler; +import org.rocksdb.util.CapturingWriteBatchHandler.Event; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.rocksdb.util.CapturingWriteBatchHandler.Action.*; + + +public class WriteBatchHandlerTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Test + public void writeBatchHandler() throws RocksDBException { + // setup test data + final List testEvents = Arrays.asList( + new Event(DELETE, "k0".getBytes(), null), + new Event(PUT, "k1".getBytes(), "v1".getBytes()), + new Event(PUT, "k2".getBytes(), "v2".getBytes()), + new Event(PUT, "k3".getBytes(), "v3".getBytes()), + new Event(LOG, null, "log1".getBytes()), + new Event(MERGE, "k2".getBytes(), "v22".getBytes()), + new Event(DELETE, "k3".getBytes(), null) + ); + + // load test data to the write batch + try (final WriteBatch batch = new WriteBatch()) { + for (final Event testEvent : testEvents) { + switch (testEvent.action) { + + case PUT: + batch.put(testEvent.key, testEvent.value); + break; + + case MERGE: + batch.merge(testEvent.key, testEvent.value); + break; + + case DELETE: + batch.delete(testEvent.key); + break; + + case LOG: + batch.putLogData(testEvent.value); + break; + } + } + + // attempt to read test data back from the WriteBatch by iterating + // with a handler + try (final CapturingWriteBatchHandler handler = + new CapturingWriteBatchHandler()) { + batch.iterate(handler); + + // compare the results to the test data + final List actualEvents = + handler.getEvents(); + assertThat(testEvents.size()).isSameAs(actualEvents.size()); + + assertThat(testEvents).isEqualTo(actualEvents); + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/WriteBatchTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/WriteBatchTest.java new file mode 100644 index 0000000..cc3ad26 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/WriteBatchTest.java @@ -0,0 +1,528 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +package org.rocksdb; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; +import static org.rocksdb.util.CapturingWriteBatchHandler.Action.DELETE; +import static org.rocksdb.util.CapturingWriteBatchHandler.Action.DELETE_RANGE; +import static org.rocksdb.util.CapturingWriteBatchHandler.Action.LOG; +import static org.rocksdb.util.CapturingWriteBatchHandler.Action.MERGE; +import static org.rocksdb.util.CapturingWriteBatchHandler.Action.PUT; +import static org.rocksdb.util.CapturingWriteBatchHandler.Action.SINGLE_DELETE; + +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.util.CapturingWriteBatchHandler; +import org.rocksdb.util.CapturingWriteBatchHandler.Event; +import org.rocksdb.util.WriteBatchGetter; + +/** + * This class mimics the db/write_batch_test.cc + * in the c++ rocksdb library. + */ +public class WriteBatchTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void emptyWriteBatch() { + try (final WriteBatch batch = new WriteBatch()) { + assertThat(batch.count()).isEqualTo(0); + } + } + + @Test + public void multipleBatchOperations() + throws RocksDBException { + + final byte[] foo = "foo".getBytes(UTF_8); + final byte[] bar = "bar".getBytes(UTF_8); + final byte[] box = "box".getBytes(UTF_8); + final byte[] baz = "baz".getBytes(UTF_8); + final byte[] boo = "boo".getBytes(UTF_8); + final byte[] hoo = "hoo".getBytes(UTF_8); + final byte[] hello = "hello".getBytes(UTF_8); + + try (final WriteBatch batch = new WriteBatch()) { + batch.put(foo, bar); + batch.delete(box); + batch.put(baz, boo); + batch.merge(baz, hoo); + batch.singleDelete(foo); + batch.deleteRange(baz, foo); + batch.putLogData(hello); + + try(final CapturingWriteBatchHandler handler = + new CapturingWriteBatchHandler()) { + batch.iterate(handler); + + assertThat(handler.getEvents().size()).isEqualTo(7); + + assertThat(handler.getEvents().get(0)).isEqualTo(new Event(PUT, foo, bar)); + assertThat(handler.getEvents().get(1)).isEqualTo(new Event(DELETE, box, null)); + assertThat(handler.getEvents().get(2)).isEqualTo(new Event(PUT, baz, boo)); + assertThat(handler.getEvents().get(3)).isEqualTo(new Event(MERGE, baz, hoo)); + assertThat(handler.getEvents().get(4)).isEqualTo(new Event(SINGLE_DELETE, foo, null)); + assertThat(handler.getEvents().get(5)).isEqualTo(new Event(DELETE_RANGE, baz, foo)); + assertThat(handler.getEvents().get(6)).isEqualTo(new Event(LOG, null, hello)); + } + } + } + + @Test + public void multipleBatchOperationsDirect() + throws UnsupportedEncodingException, RocksDBException { + try (WriteBatch batch = new WriteBatch()) { + ByteBuffer key = ByteBuffer.allocateDirect(16); + ByteBuffer value = ByteBuffer.allocateDirect(16); + key.put("foo".getBytes("US-ASCII")).flip(); + value.put("bar".getBytes("US-ASCII")).flip(); + batch.put(key, value); + assertThat(key.position()).isEqualTo(3); + assertThat(key.limit()).isEqualTo(3); + assertThat(value.position()).isEqualTo(3); + assertThat(value.limit()).isEqualTo(3); + + key.clear(); + key.put("box".getBytes("US-ASCII")).flip(); + batch.delete(key); + assertThat(key.position()).isEqualTo(3); + assertThat(key.limit()).isEqualTo(3); + + batch.put("baz".getBytes("US-ASCII"), "boo".getBytes("US-ASCII")); + + WriteBatchTestInternalHelper.setSequence(batch, 100); + assertThat(WriteBatchTestInternalHelper.sequence(batch)).isNotNull().isEqualTo(100); + assertThat(batch.count()).isEqualTo(3); + assertThat(new String(getContents(batch), "US-ASCII")) + .isEqualTo("Put(baz, boo)@102" + + "Delete(box)@101" + + "Put(foo, bar)@100"); + } + } + + @Test + public void testAppendOperation() + throws RocksDBException { + try (final WriteBatch b1 = new WriteBatch(); + final WriteBatch b2 = new WriteBatch()) { + WriteBatchTestInternalHelper.setSequence(b1, 200); + WriteBatchTestInternalHelper.setSequence(b2, 300); + WriteBatchTestInternalHelper.append(b1, b2); + assertThat(getContents(b1).length).isEqualTo(0); + assertThat(b1.count()).isEqualTo(0); + b2.put("a".getBytes(UTF_8), "va".getBytes(UTF_8)); + WriteBatchTestInternalHelper.append(b1, b2); + assertThat("Put(a, va)@200".equals(new String(getContents(b1), + UTF_8))); + assertThat(b1.count()).isEqualTo(1); + b2.clear(); + b2.put("b".getBytes(UTF_8), "vb".getBytes(UTF_8)); + WriteBatchTestInternalHelper.append(b1, b2); + assertThat(("Put(a, va)@200" + + "Put(b, vb)@201") + .equals(new String(getContents(b1), UTF_8))); + assertThat(b1.count()).isEqualTo(2); + b2.delete("foo".getBytes(UTF_8)); + WriteBatchTestInternalHelper.append(b1, b2); + assertThat(("Put(a, va)@200" + + "Put(b, vb)@202" + + "Put(b, vb)@201" + + "Delete(foo)@203") + .equals(new String(getContents(b1), UTF_8))); + assertThat(b1.count()).isEqualTo(4); + } + } + + @Test + public void blobOperation() + throws RocksDBException { + try (final WriteBatch batch = new WriteBatch()) { + batch.put("k1".getBytes(UTF_8), "v1".getBytes(UTF_8)); + batch.put("k2".getBytes(UTF_8), "v2".getBytes(UTF_8)); + batch.put("k3".getBytes(UTF_8), "v3".getBytes(UTF_8)); + batch.putLogData("blob1".getBytes(UTF_8)); + batch.delete("k2".getBytes(UTF_8)); + batch.putLogData("blob2".getBytes(UTF_8)); + batch.merge("foo".getBytes(UTF_8), "bar".getBytes(UTF_8)); + assertThat(batch.count()).isEqualTo(5); + assertThat(("Merge(foo, bar)@4" + + "Put(k1, v1)@0" + + "Delete(k2)@3" + + "Put(k2, v2)@1" + + "Put(k3, v3)@2") + .equals(new String(getContents(batch), UTF_8))); + } + } + + @Test + public void savePoints() + throws RocksDBException { + try (final WriteBatch batch = new WriteBatch()) { + batch.put("k1".getBytes(UTF_8), "v1".getBytes(UTF_8)); + batch.put("k2".getBytes(UTF_8), "v2".getBytes(UTF_8)); + batch.put("k3".getBytes(UTF_8), "v3".getBytes(UTF_8)); + + assertThat(getFromWriteBatch(batch, "k1")).isEqualTo("v1"); + assertThat(getFromWriteBatch(batch, "k2")).isEqualTo("v2"); + assertThat(getFromWriteBatch(batch, "k3")).isEqualTo("v3"); + + batch.setSavePoint(); + + batch.delete("k2".getBytes(UTF_8)); + batch.put("k3".getBytes(UTF_8), "v3-2".getBytes(UTF_8)); + + assertThat(getFromWriteBatch(batch, "k2")).isNull(); + assertThat(getFromWriteBatch(batch, "k3")).isEqualTo("v3-2"); + + + batch.setSavePoint(); + + batch.put("k3".getBytes(UTF_8), "v3-3".getBytes(UTF_8)); + batch.put("k4".getBytes(UTF_8), "v4".getBytes(UTF_8)); + + assertThat(getFromWriteBatch(batch, "k3")).isEqualTo("v3-3"); + assertThat(getFromWriteBatch(batch, "k4")).isEqualTo("v4"); + + + batch.rollbackToSavePoint(); + + assertThat(getFromWriteBatch(batch, "k2")).isNull(); + assertThat(getFromWriteBatch(batch, "k3")).isEqualTo("v3-2"); + assertThat(getFromWriteBatch(batch, "k4")).isNull(); + + + batch.rollbackToSavePoint(); + + assertThat(getFromWriteBatch(batch, "k1")).isEqualTo("v1"); + assertThat(getFromWriteBatch(batch, "k2")).isEqualTo("v2"); + assertThat(getFromWriteBatch(batch, "k3")).isEqualTo("v3"); + assertThat(getFromWriteBatch(batch, "k4")).isNull(); + } + } + + @Test + public void deleteRange() throws RocksDBException { + try (final RocksDB db = RocksDB.open(dbFolder.getRoot().getAbsolutePath()); + final WriteBatch batch = new WriteBatch(); + final WriteOptions wOpt = new WriteOptions()) { + db.put("key1".getBytes(), "value".getBytes()); + db.put("key2".getBytes(), "12345678".getBytes()); + db.put("key3".getBytes(), "abcdefg".getBytes()); + db.put("key4".getBytes(), "xyz".getBytes()); + assertThat(db.get("key1".getBytes())).isEqualTo("value".getBytes()); + assertThat(db.get("key2".getBytes())).isEqualTo("12345678".getBytes()); + assertThat(db.get("key3".getBytes())).isEqualTo("abcdefg".getBytes()); + assertThat(db.get("key4".getBytes())).isEqualTo("xyz".getBytes()); + + batch.deleteRange("key2".getBytes(), "key4".getBytes()); + db.write(wOpt, batch); + + assertThat(db.get("key1".getBytes())).isEqualTo("value".getBytes()); + assertThat(db.get("key2".getBytes())).isNull(); + assertThat(db.get("key3".getBytes())).isNull(); + assertThat(db.get("key4".getBytes())).isEqualTo("xyz".getBytes()); + } + } + + @Test + public void restorePoints() throws RocksDBException { + try (final WriteBatch batch = new WriteBatch()) { + + batch.put("k1".getBytes(), "v1".getBytes()); + batch.put("k2".getBytes(), "v2".getBytes()); + + batch.setSavePoint(); + + batch.put("k1".getBytes(), "123456789".getBytes()); + batch.delete("k2".getBytes()); + + batch.rollbackToSavePoint(); + + try(final CapturingWriteBatchHandler handler = new CapturingWriteBatchHandler()) { + batch.iterate(handler); + + assertThat(handler.getEvents().size()).isEqualTo(2); + assertThat(handler.getEvents().get(0)).isEqualTo(new Event(PUT, "k1".getBytes(), "v1".getBytes())); + assertThat(handler.getEvents().get(1)).isEqualTo(new Event(PUT, "k2".getBytes(), "v2".getBytes())); + } + } + } + + @Test(expected = RocksDBException.class) + public void restorePoints_withoutSavePoints() throws RocksDBException { + try (final WriteBatch batch = new WriteBatch()) { + batch.rollbackToSavePoint(); + } + } + + @Test(expected = RocksDBException.class) + public void restorePoints_withoutSavePoints_nested() throws RocksDBException { + try (final WriteBatch batch = new WriteBatch()) { + + batch.setSavePoint(); + batch.rollbackToSavePoint(); + + // without previous corresponding setSavePoint + batch.rollbackToSavePoint(); + } + } + + @Test + public void popSavePoint() throws RocksDBException { + try (final WriteBatch batch = new WriteBatch()) { + + batch.put("k1".getBytes(), "v1".getBytes()); + batch.put("k2".getBytes(), "v2".getBytes()); + + batch.setSavePoint(); + + batch.put("k1".getBytes(), "123456789".getBytes()); + batch.delete("k2".getBytes()); + + batch.setSavePoint(); + + batch.popSavePoint(); + + batch.rollbackToSavePoint(); + + try(final CapturingWriteBatchHandler handler = new CapturingWriteBatchHandler()) { + batch.iterate(handler); + + assertThat(handler.getEvents().size()).isEqualTo(2); + assertThat(handler.getEvents().get(0)).isEqualTo(new Event(PUT, "k1".getBytes(), "v1".getBytes())); + assertThat(handler.getEvents().get(1)).isEqualTo(new Event(PUT, "k2".getBytes(), "v2".getBytes())); + } + } + } + + @Test(expected = RocksDBException.class) + public void popSavePoint_withoutSavePoints() throws RocksDBException { + try (final WriteBatch batch = new WriteBatch()) { + batch.popSavePoint(); + } + } + + @Test(expected = RocksDBException.class) + public void popSavePoint_withoutSavePoints_nested() throws RocksDBException { + try (final WriteBatch batch = new WriteBatch()) { + + batch.setSavePoint(); + batch.popSavePoint(); + + // without previous corresponding setSavePoint + batch.popSavePoint(); + } + } + + @Test + public void maxBytes() throws RocksDBException { + try (final WriteBatch batch = new WriteBatch()) { + batch.setMaxBytes(19); + + batch.put("k1".getBytes(), "v1".getBytes()); + } + } + + @Test(expected = RocksDBException.class) + public void maxBytes_over() throws RocksDBException { + try (final WriteBatch batch = new WriteBatch()) { + batch.setMaxBytes(1); + + batch.put("k1".getBytes(), "v1".getBytes()); + } + } + + @Test + public void data() throws RocksDBException { + try (final WriteBatch batch1 = new WriteBatch()) { + batch1.delete("k0".getBytes()); + batch1.put("k1".getBytes(), "v1".getBytes()); + batch1.put("k2".getBytes(), "v2".getBytes()); + batch1.put("k3".getBytes(), "v3".getBytes()); + batch1.putLogData("log1".getBytes()); + batch1.merge("k2".getBytes(), "v22".getBytes()); + batch1.delete("k3".getBytes()); + + final byte[] serialized = batch1.data(); + + try(final WriteBatch batch2 = new WriteBatch(serialized)) { + assertThat(batch2.count()).isEqualTo(batch1.count()); + + try(final CapturingWriteBatchHandler handler1 = new CapturingWriteBatchHandler()) { + batch1.iterate(handler1); + + try (final CapturingWriteBatchHandler handler2 = new CapturingWriteBatchHandler()) { + batch2.iterate(handler2); + + assertThat(handler1.getEvents().equals(handler2.getEvents())).isTrue(); + } + } + } + } + } + + @Test + public void dataSize() throws RocksDBException { + try (final WriteBatch batch = new WriteBatch()) { + batch.put("k1".getBytes(), "v1".getBytes()); + + assertThat(batch.getDataSize()).isEqualTo(19); + } + } + + @Test + public void hasPut() throws RocksDBException { + try (final WriteBatch batch = new WriteBatch()) { + assertThat(batch.hasPut()).isFalse(); + + batch.put("k1".getBytes(), "v1".getBytes()); + + assertThat(batch.hasPut()).isTrue(); + } + } + + @Test + public void hasDelete() throws RocksDBException { + try (final WriteBatch batch = new WriteBatch()) { + assertThat(batch.hasDelete()).isFalse(); + + batch.delete("k1".getBytes()); + + assertThat(batch.hasDelete()).isTrue(); + } + } + + @Test + public void hasSingleDelete() throws RocksDBException { + try (final WriteBatch batch = new WriteBatch()) { + assertThat(batch.hasSingleDelete()).isFalse(); + + batch.singleDelete("k1".getBytes()); + + assertThat(batch.hasSingleDelete()).isTrue(); + } + } + + @Test + public void hasDeleteRange() throws RocksDBException { + try (final WriteBatch batch = new WriteBatch()) { + assertThat(batch.hasDeleteRange()).isFalse(); + + batch.deleteRange("k1".getBytes(), "k2".getBytes()); + + assertThat(batch.hasDeleteRange()).isTrue(); + } + } + + @Test + public void hasBeginPrepareRange() throws RocksDBException { + try (final WriteBatch batch = new WriteBatch()) { + assertThat(batch.hasBeginPrepare()).isFalse(); + } + } + + @Test + public void hasEndPrepareRange() throws RocksDBException { + try (final WriteBatch batch = new WriteBatch()) { + assertThat(batch.hasEndPrepare()).isFalse(); + } + } + + @Test + public void hasCommit() throws RocksDBException { + try (final WriteBatch batch = new WriteBatch()) { + assertThat(batch.hasCommit()).isFalse(); + } + } + + @Test + public void hasRollback() throws RocksDBException { + try (final WriteBatch batch = new WriteBatch()) { + assertThat(batch.hasRollback()).isFalse(); + } + } + + @Test + public void walTerminationPoint() throws RocksDBException { + try (final WriteBatch batch = new WriteBatch()) { + WriteBatch.SavePoint walTerminationPoint = batch.getWalTerminationPoint(); + assertThat(walTerminationPoint.isCleared()).isTrue(); + + batch.put("k1".getBytes(UTF_8), "v1".getBytes(UTF_8)); + + batch.markWalTerminationPoint(); + + walTerminationPoint = batch.getWalTerminationPoint(); + assertThat(walTerminationPoint.getSize()).isEqualTo(19); + assertThat(walTerminationPoint.getCount()).isEqualTo(1); + assertThat(walTerminationPoint.getContentFlags()).isEqualTo(2); + } + } + + @Test + public void getWriteBatch() { + try (final WriteBatch batch = new WriteBatch()) { + assertThat(batch.getWriteBatch()).isEqualTo(batch); + } + } + + static byte[] getContents(final WriteBatch wb) { + return getContents(wb.nativeHandle_); + } + + static String getFromWriteBatch(final WriteBatch wb, final String key) + throws RocksDBException { + final WriteBatchGetter getter = + new WriteBatchGetter(key.getBytes(UTF_8)); + wb.iterate(getter); + if(getter.getValue() != null) { + return new String(getter.getValue(), UTF_8); + } else { + return null; + } + } + + private static native byte[] getContents(final long writeBatchHandle); +} + +/** + * Package-private class which provides java api to access + * c++ WriteBatchInternal. + */ +class WriteBatchTestInternalHelper { + static void setSequence(final WriteBatch wb, final long sn) { + setSequence(wb.nativeHandle_, sn); + } + + static long sequence(final WriteBatch wb) { + return sequence(wb.nativeHandle_); + } + + static void append(final WriteBatch wb1, final WriteBatch wb2) { + append(wb1.nativeHandle_, wb2.nativeHandle_); + } + + private static native void setSequence(final long writeBatchHandle, + final long sn); + + private static native long sequence(final long writeBatchHandle); + + private static native void append(final long writeBatchHandle1, + final long writeBatchHandle2); +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/WriteBatchThreadedTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/WriteBatchThreadedTest.java new file mode 100644 index 0000000..0321da3 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/WriteBatchThreadedTest.java @@ -0,0 +1,101 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.*; + +@RunWith(Parameterized.class) +public class WriteBatchThreadedTest { + + @Parameters(name = "WriteBatchThreadedTest(threadCount={0})") + public static Iterable data() { + return Arrays.asList(1, 10, 50, 100); + } + + @Parameter + public int threadCount; + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + RocksDB db; + + @Before + public void setUp() throws Exception { + RocksDB.loadLibrary(); + final Options options = new Options() + .setCreateIfMissing(true) + .setIncreaseParallelism(32); + db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath()); + assert (db != null); + } + + @After + public void tearDown() throws Exception { + if (db != null) { + db.close(); + } + } + + @Test + public void threadedWrites() throws InterruptedException, ExecutionException { + final List> callables = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + final int offset = i * 100; + callables.add(() -> { + try (final WriteBatch wb = new WriteBatch(); + final WriteOptions w_opt = new WriteOptions()) { + for (int i1 = offset; i1 < offset + 100; i1++) { + wb.put(ByteBuffer.allocate(4).putInt(i1).array(), "parallel rocks test".getBytes()); + } + db.write(w_opt, wb); + } + return null; + }); + } + + //submit the callables + final ExecutorService executorService = + Executors.newFixedThreadPool(threadCount); + try { + final ExecutorCompletionService completionService = + new ExecutorCompletionService<>(executorService); + final Set> futures = new HashSet<>(); + for (final Callable callable : callables) { + futures.add(completionService.submit(callable)); + } + + while (futures.size() > 0) { + final Future future = completionService.take(); + futures.remove(future); + + try { + future.get(); + } catch (final ExecutionException e) { + for (final Future f : futures) { + f.cancel(true); + } + + throw e; + } + } + } finally { + executorService.shutdown(); + executorService.awaitTermination(10, TimeUnit.SECONDS); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/WriteBatchWithIndexTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/WriteBatchWithIndexTest.java new file mode 100644 index 0000000..b0a0cdc --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/WriteBatchWithIndexTest.java @@ -0,0 +1,1068 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +package org.rocksdb; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.util.ByteBufferAllocator; + +public class WriteBatchWithIndexTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + @Test + public void readYourOwnWrites() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath())) { + + final byte[] k1 = "key1".getBytes(); + final byte[] v1 = "value1".getBytes(); + final byte[] k2 = "key2".getBytes(); + final byte[] v2 = "value2".getBytes(); + + db.put(k1, v1); + db.put(k2, v2); + + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex(true); + final RocksIterator base = db.newIterator(); + final RocksIterator it = wbwi.newIteratorWithBase(base)) { + it.seek(k1); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(k1); + assertThat(it.value()).isEqualTo(v1); + + it.seek(k2); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(k2); + assertThat(it.value()).isEqualTo(v2); + + //put data to the write batch and make sure we can read it. + final byte[] k3 = "key3".getBytes(); + final byte[] v3 = "value3".getBytes(); + wbwi.put(k3, v3); + it.seek(k3); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(k3); + assertThat(it.value()).isEqualTo(v3); + + //update k2 in the write batch and check the value + final byte[] v2Other = "otherValue2".getBytes(); + wbwi.put(k2, v2Other); + it.seek(k2); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(k2); + assertThat(it.value()).isEqualTo(v2Other); + + //delete k1 and make sure we can read back the write + wbwi.delete(k1); + it.seek(k1); + assertThat(it.key()).isNotEqualTo(k1); + + //reinsert k1 and make sure we see the new value + final byte[] v1Other = "otherValue1".getBytes(); + wbwi.put(k1, v1Other); + it.seek(k1); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(k1); + assertThat(it.value()).isEqualTo(v1Other); + + //single remove k3 and make sure we can read back the write + wbwi.singleDelete(k3); + it.seek(k3); + assertThat(it.isValid()).isEqualTo(false); + + //reinsert k3 and make sure we see the new value + final byte[] v3Other = "otherValue3".getBytes(); + wbwi.put(k3, v3Other); + it.seek(k3); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(k3); + assertThat(it.value()).isEqualTo(v3Other); + } + } + } + + @Test + public void readYourOwnWritesCf() throws RocksDBException { + final List cfNames = + Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes())); + + final List columnFamilyHandleList = new ArrayList<>(); + + // Test open database with column family names + try (final DBOptions options = + new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open( + options, dbFolder.getRoot().getAbsolutePath(), cfNames, columnFamilyHandleList)) { + final ColumnFamilyHandle newCf = columnFamilyHandleList.get(1); + + try { + final byte[] k1 = "key1".getBytes(); + final byte[] v1 = "value1".getBytes(); + final byte[] k2 = "key2".getBytes(); + final byte[] v2 = "value2".getBytes(); + + db.put(newCf, k1, v1); + db.put(newCf, k2, v2); + + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex(true); + final ReadOptions readOptions = new ReadOptions(); + final RocksIterator base = db.newIterator(newCf, readOptions); + final RocksIterator it = wbwi.newIteratorWithBase(newCf, base, readOptions)) { + it.seek(k1); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(k1); + assertThat(it.value()).isEqualTo(v1); + + it.seek(k2); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(k2); + assertThat(it.value()).isEqualTo(v2); + + // put data to the write batch and make sure we can read it. + final byte[] k3 = "key3".getBytes(); + final byte[] v3 = "value3".getBytes(); + wbwi.put(newCf, k3, v3); + it.seek(k3); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(k3); + assertThat(it.value()).isEqualTo(v3); + + // update k2 in the write batch and check the value + final byte[] v2Other = "otherValue2".getBytes(); + wbwi.put(newCf, k2, v2Other); + it.seek(k2); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(k2); + assertThat(it.value()).isEqualTo(v2Other); + + // delete k1 and make sure we can read back the write + wbwi.delete(newCf, k1); + it.seek(k1); + assertThat(it.key()).isNotEqualTo(k1); + + // reinsert k1 and make sure we see the new value + final byte[] v1Other = "otherValue1".getBytes(); + wbwi.put(newCf, k1, v1Other); + it.seek(k1); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(k1); + assertThat(it.value()).isEqualTo(v1Other); + + // single remove k3 and make sure we can read back the write + wbwi.singleDelete(newCf, k3); + it.seek(k3); + assertThat(it.isValid()).isEqualTo(false); + + // reinsert k3 and make sure we see the new value + final byte[] v3Other = "otherValue3".getBytes(); + wbwi.put(newCf, k3, v3Other); + it.seek(k3); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(k3); + assertThat(it.value()).isEqualTo(v3Other); + } + } finally { + for (final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandleList) { + columnFamilyHandle.close(); + } + } + } + } + + @Test + public void readYourOwnWritesCfIterDirectBB() throws RocksDBException { + readYourOwnWritesCfIterDirect(ByteBufferAllocator.DIRECT); + } + + @Test + public void readYourOwnWritesCfIterIndirectBB() throws RocksDBException { + readYourOwnWritesCfIterDirect(ByteBufferAllocator.HEAP); + } + + public void readYourOwnWritesCfIterDirect(final ByteBufferAllocator byteBufferAllocator) + throws RocksDBException { + final List cfNames = + Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes())); + + final List columnFamilyHandleList = new ArrayList<>(); + + // Test open database with column family names + try (final DBOptions options = + new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open( + options, dbFolder.getRoot().getAbsolutePath(), cfNames, columnFamilyHandleList)) { + final ColumnFamilyHandle newCf = columnFamilyHandleList.get(1); + + try { + final byte[] kv1 = "key1".getBytes(); + final byte[] vv1 = "value1".getBytes(); + final ByteBuffer k1 = byteBufferAllocator.allocate(12); + k1.put(kv1); + final byte[] kv2 = "key2".getBytes(); + final byte[] vv2 = "value2".getBytes(); + final ByteBuffer k2 = byteBufferAllocator.allocate(12); + k2.put(kv2); + + db.put(newCf, kv1, vv1); + db.put(newCf, kv2, vv2); + + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex(true); + final ReadOptions readOptions = new ReadOptions(); + final RocksIterator base = db.newIterator(newCf, readOptions); + final RocksIterator it = wbwi.newIteratorWithBase(newCf, base, readOptions)) { + k1.flip(); + it.seek(k1); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(kv1); + assertThat(it.value()).isEqualTo(vv1); + + k2.flip(); + it.seek(k2); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(kv2); + assertThat(it.value()).isEqualTo(vv2); + + final byte[] kv1point5 = "key1point5".getBytes(); + final ByteBuffer k1point5 = byteBufferAllocator.allocate(12); + k1point5.put(kv1point5); + + k1point5.flip(); + it.seek(k1point5); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(kv2); + assertThat(it.value()).isEqualTo(vv2); + + k1point5.flip(); + it.seekForPrev(k1point5); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(kv1); + assertThat(it.value()).isEqualTo(vv1); + + // put data to the write batch and make sure we can read it. + final byte[] kv3 = "key3".getBytes(); + final ByteBuffer k3 = byteBufferAllocator.allocate(12); + k3.put(kv3); + final byte[] vv3 = "value3".getBytes(); + wbwi.put(newCf, kv3, vv3); + k3.flip(); + it.seek(k3); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(kv3); + assertThat(it.value()).isEqualTo(vv3); + + // update k2 in the write batch and check the value + final byte[] v2Other = "otherValue2".getBytes(); + wbwi.put(newCf, kv2, v2Other); + k2.flip(); + it.seek(k2); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(kv2); + assertThat(it.value()).isEqualTo(v2Other); + + // delete k1 and make sure we can read back the write + wbwi.delete(newCf, kv1); + k1.flip(); + it.seek(k1); + assertThat(it.key()).isNotEqualTo(kv1); + + // reinsert k1 and make sure we see the new value + final byte[] v1Other = "otherValue1".getBytes(); + wbwi.put(newCf, kv1, v1Other); + k1.flip(); + it.seek(k1); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(kv1); + assertThat(it.value()).isEqualTo(v1Other); + + // single remove k3 and make sure we can read back the write + wbwi.singleDelete(newCf, kv3); + k3.flip(); + it.seek(k3); + assertThat(it.isValid()).isEqualTo(false); + + // reinsert k3 and make sure we see the new value + final byte[] v3Other = "otherValue3".getBytes(); + wbwi.put(newCf, kv3, v3Other); + k3.flip(); + it.seek(k3); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(kv3); + assertThat(it.value()).isEqualTo(v3Other); + } + } finally { + for (final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandleList) { + columnFamilyHandle.close(); + } + } + } + } + + @Test + public void readYourOwnWritesCfIterIndirect() throws RocksDBException { + final List cfNames = + Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes())); + + final List columnFamilyHandleList = new ArrayList<>(); + + // Test open database with column family names + try (final DBOptions options = + new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open( + options, dbFolder.getRoot().getAbsolutePath(), cfNames, columnFamilyHandleList)) { + final ColumnFamilyHandle newCf = columnFamilyHandleList.get(1); + + try { + final byte[] kv1 = "key1".getBytes(); + final byte[] vv1 = "value1".getBytes(); + final ByteBuffer k1 = ByteBuffer.allocate(12); + k1.put(kv1).flip(); + final byte[] kv2 = "key2".getBytes(); + final byte[] vv2 = "value2".getBytes(); + final ByteBuffer k2 = ByteBuffer.allocate(12); + k2.put(kv2).flip(); + + db.put(newCf, kv1, vv1); + db.put(newCf, kv2, vv2); + + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex(true); + final ReadOptions readOptions = new ReadOptions(); + final RocksIterator base = db.newIterator(newCf, readOptions); + final RocksIterator it = wbwi.newIteratorWithBase(newCf, base, readOptions)) { + it.seek(k1); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(kv1); + assertThat(it.value()).isEqualTo(vv1); + + it.seek(k2); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(kv2); + assertThat(it.value()).isEqualTo(vv2); + + // put data to the write batch and make sure we can read it. + final byte[] kv3 = "key3".getBytes(); + final ByteBuffer k3 = ByteBuffer.allocate(12); + k3.put(kv3); + final byte[] vv3 = "value3".getBytes(); + wbwi.put(newCf, kv3, vv3); + k3.flip(); + it.seek(k3); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(kv3); + assertThat(it.value()).isEqualTo(vv3); + + // update k2 in the write batch and check the value + final byte[] v2Other = "otherValue2".getBytes(); + wbwi.put(newCf, kv2, v2Other); + k2.flip(); + it.seek(k2); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(kv2); + assertThat(it.value()).isEqualTo(v2Other); + + // delete k1 and make sure we can read back the write + wbwi.delete(newCf, kv1); + k1.flip(); + it.seek(k1); + assertThat(it.key()).isNotEqualTo(kv1); + + // reinsert k1 and make sure we see the new value + final byte[] v1Other = "otherValue1".getBytes(); + wbwi.put(newCf, kv1, v1Other); + k1.flip(); + it.seek(k1); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(kv1); + assertThat(it.value()).isEqualTo(v1Other); + + // single remove k3 and make sure we can read back the write + wbwi.singleDelete(newCf, kv3); + k3.flip(); + it.seek(k3); + assertThat(it.isValid()).isEqualTo(false); + + // reinsert k3 and make sure we see the new value + final byte[] v3Other = "otherValue3".getBytes(); + wbwi.put(newCf, kv3, v3Other); + k3.flip(); + it.seek(k3); + assertThat(it.isValid()).isTrue(); + assertThat(it.key()).isEqualTo(kv3); + assertThat(it.value()).isEqualTo(v3Other); + } + } finally { + for (final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandleList) { + columnFamilyHandle.close(); + } + } + } + } + + @Test + public void writeBatchWithIndex() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath())) { + + final byte[] k1 = "key1".getBytes(); + final byte[] v1 = "value1".getBytes(); + final byte[] k2 = "key2".getBytes(); + final byte[] v2 = "value2".getBytes(); + + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex(); + final WriteOptions wOpt = new WriteOptions()) { + wbwi.put(k1, v1); + wbwi.put(k2, v2); + + db.write(wOpt, wbwi); + } + + assertThat(db.get(k1)).isEqualTo(v1); + assertThat(db.get(k2)).isEqualTo(v2); + } + } + + @Test + public void write_writeBatchWithIndexDirect() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + final ByteBuffer k1 = ByteBuffer.allocateDirect(16); + final ByteBuffer v1 = ByteBuffer.allocateDirect(16); + final ByteBuffer k2 = ByteBuffer.allocateDirect(16); + final ByteBuffer v2 = ByteBuffer.allocateDirect(16); + k1.put("key1".getBytes()).flip(); + v1.put("value1".getBytes()).flip(); + k2.put("key2".getBytes()).flip(); + v2.put("value2".getBytes()).flip(); + + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex()) { + wbwi.put(k1, v1); + assertThat(k1.position()).isEqualTo(4); + assertThat(k1.limit()).isEqualTo(4); + assertThat(v1.position()).isEqualTo(6); + assertThat(v1.limit()).isEqualTo(6); + + wbwi.put(k2, v2); + + db.write(new WriteOptions(), wbwi); + } + + assertThat(db.get("key1".getBytes())).isEqualTo("value1".getBytes()); + assertThat(db.get("key2".getBytes())).isEqualTo("value2".getBytes()); + } + } + + @Test + public void iterator() throws RocksDBException { + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex(true)) { + + final String k1 = "key1"; + final String v1 = "value1"; + final String k2 = "key2"; + final String v2 = "value2"; + final String k3 = "key3"; + final String v3 = "value3"; + final String k4 = "key4"; + final String k5 = "key5"; + final String v8 = "value8"; + final byte[] k1b = k1.getBytes(UTF_8); + final byte[] v1b = v1.getBytes(UTF_8); + final byte[] k2b = k2.getBytes(UTF_8); + final byte[] v2b = v2.getBytes(UTF_8); + final byte[] k3b = k3.getBytes(UTF_8); + final byte[] v3b = v3.getBytes(UTF_8); + final byte[] k4b = k4.getBytes(UTF_8); + final byte[] k5b = k5.getBytes(UTF_8); + final byte[] v8b = v8.getBytes(UTF_8); + + final String k1point5 = "key1point5"; + final String k2point5 = "key2point5"; + + // add put records + wbwi.put(k1b, v1b); + wbwi.put(k2b, v2b); + wbwi.put(k3b, v3b); + + // add a deletion record + wbwi.delete(k4b); + + // add a single deletion record + wbwi.singleDelete(k5b); + + // add a log record + wbwi.putLogData(v8b); + + final WBWIRocksIterator.WriteEntry[] expected = { + new WBWIRocksIterator.WriteEntry(WBWIRocksIterator.WriteType.PUT, + new DirectSlice(k1), new DirectSlice(v1)), + new WBWIRocksIterator.WriteEntry(WBWIRocksIterator.WriteType.PUT, + new DirectSlice(k2), new DirectSlice(v2)), + new WBWIRocksIterator.WriteEntry(WBWIRocksIterator.WriteType.PUT, + new DirectSlice(k3), new DirectSlice(v3)), + new WBWIRocksIterator.WriteEntry(WBWIRocksIterator.WriteType.DELETE, + new DirectSlice(k4), DirectSlice.NONE), + new WBWIRocksIterator.WriteEntry(WBWIRocksIterator.WriteType.SINGLE_DELETE, + new DirectSlice(k5), DirectSlice.NONE), + }; + + try (final WBWIRocksIterator it = wbwi.newIterator()) { + //direct access - seek to key offsets + final int[] testOffsets = {2, 0, 3, 4, 1}; + for (final int testOffset : testOffsets) { + final byte[] key = toArray(expected[testOffset].getKey().data()); + + it.seek(key); + assertThat(it.isValid()).isTrue(); + + final WBWIRocksIterator.WriteEntry entry = it.entry(); + assertThat(entry).isEqualTo(expected[testOffset]); + } + + for (final int testOffset : testOffsets) { + final byte[] key = toArray(expected[testOffset].getKey().data()); + + // Direct buffer seek + final ByteBuffer db = expected[testOffset].getKey().data(); + it.seek(db); + assertThat(db.position()).isEqualTo(key.length); + assertThat(it.isValid()).isTrue(); + + final WBWIRocksIterator.WriteEntry entry = it.entry(); + assertThat(entry).isEqualTo(expected[testOffset]); + } + + for (final int testOffset : testOffsets) { + final byte[] key = toArray(expected[testOffset].getKey().data()); + + // Direct buffer seek + final ByteBuffer db = expected[testOffset].getKey().data(); + it.seekForPrev(db); + assertThat(db.position()).isEqualTo(key.length); + assertThat(it.isValid()).isTrue(); + + final WBWIRocksIterator.WriteEntry entry = it.entry(); + assertThat(entry).isEqualTo(expected[testOffset]); + } + + for (final int testOffset : testOffsets) { + final byte[] key = toArray(expected[testOffset].getKey().data()); + + // Indirect buffer seek + final ByteBuffer db = ByteBuffer.allocate(key.length); + System.arraycopy(key, 0, db.array(), 0, key.length); + it.seek(db); + assertThat(db.position()).isEqualTo(key.length); + assertThat(it.isValid()).isTrue(); + + final WBWIRocksIterator.WriteEntry entry = it.entry(); + assertThat(entry).isEqualTo(expected[testOffset]); + } + + for (final int testOffset : testOffsets) { + final byte[] key = toArray(expected[testOffset].getKey().data()); + + // Indirect buffer seek for prev + final ByteBuffer db = ByteBuffer.allocate(key.length); + System.arraycopy(key, 0, db.array(), 0, key.length); + it.seekForPrev(db); + assertThat(db.position()).isEqualTo(key.length); + assertThat(it.isValid()).isTrue(); + + final WBWIRocksIterator.WriteEntry entry = it.entry(); + assertThat(entry).isEqualTo(expected[testOffset]); + } + + { + it.seekForPrev(k2point5.getBytes()); + assertThat(it.isValid()).isTrue(); + final WBWIRocksIterator.WriteEntry entry = it.entry(); + assertThat(entry).isEqualTo(expected[1]); + } + + { + it.seekForPrev(k1point5.getBytes()); + assertThat(it.isValid()).isTrue(); + final WBWIRocksIterator.WriteEntry entry = it.entry(); + assertThat(entry).isEqualTo(expected[0]); + } + + { + final ByteBuffer db = ByteBuffer.allocate(k2point5.length()); + db.put(k2point5.getBytes()); + db.flip(); + it.seekForPrev(db); + assertThat(it.isValid()).isTrue(); + final WBWIRocksIterator.WriteEntry entry = it.entry(); + assertThat(entry).isEqualTo(expected[1]); + } + + { + final ByteBuffer db = ByteBuffer.allocate(k1point5.length()); + db.put(k1point5.getBytes()); + db.flip(); + it.seekForPrev(db); + assertThat(it.isValid()).isTrue(); + final WBWIRocksIterator.WriteEntry entry = it.entry(); + assertThat(entry).isEqualTo(expected[0]); + } + + //forward iterative access + int i = 0; + for (it.seekToFirst(); it.isValid(); it.next()) { + assertThat(it.entry()).isEqualTo(expected[i++]); + } + + //reverse iterative access + i = expected.length - 1; + for (it.seekToLast(); it.isValid(); it.prev()) { + assertThat(it.entry()).isEqualTo(expected[i--]); + } + } + } + } + + @Test + public void zeroByteTests() throws RocksDBException { + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex(true)) { + final byte[] zeroByteValue = new byte[]{0, 0}; + //add zero byte value + wbwi.put(zeroByteValue, zeroByteValue); + + final ByteBuffer buffer = ByteBuffer.allocateDirect(zeroByteValue.length); + buffer.put(zeroByteValue); + + final WBWIRocksIterator.WriteEntry expected = + new WBWIRocksIterator.WriteEntry(WBWIRocksIterator.WriteType.PUT, + new DirectSlice(buffer, zeroByteValue.length), + new DirectSlice(buffer, zeroByteValue.length)); + + try (final WBWIRocksIterator it = wbwi.newIterator()) { + it.seekToFirst(); + final WBWIRocksIterator.WriteEntry actual = it.entry(); + assertThat(actual.equals(expected)).isTrue(); + assertThat(it.entry().hashCode() == expected.hashCode()).isTrue(); + } + } + } + + @Test + public void savePoints() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath())) { + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex(true); + final ReadOptions readOptions = new ReadOptions()) { + wbwi.put("k1".getBytes(), "v1".getBytes()); + wbwi.put("k2".getBytes(), "v2".getBytes()); + wbwi.put("k3".getBytes(), "v3".getBytes()); + + assertThat(getFromWriteBatchWithIndex(db, readOptions, wbwi, "k1")) + .isEqualTo("v1"); + assertThat(getFromWriteBatchWithIndex(db, readOptions, wbwi, "k2")) + .isEqualTo("v2"); + assertThat(getFromWriteBatchWithIndex(db, readOptions, wbwi, "k3")) + .isEqualTo("v3"); + + + wbwi.setSavePoint(); + + wbwi.delete("k2".getBytes()); + wbwi.put("k3".getBytes(), "v3-2".getBytes()); + + assertThat(getFromWriteBatchWithIndex(db, readOptions, wbwi, "k2")) + .isNull(); + assertThat(getFromWriteBatchWithIndex(db, readOptions, wbwi, "k3")) + .isEqualTo("v3-2"); + + + wbwi.setSavePoint(); + + wbwi.put("k3".getBytes(), "v3-3".getBytes()); + wbwi.put("k4".getBytes(), "v4".getBytes()); + + assertThat(getFromWriteBatchWithIndex(db, readOptions, wbwi, "k3")) + .isEqualTo("v3-3"); + assertThat(getFromWriteBatchWithIndex(db, readOptions, wbwi, "k4")) + .isEqualTo("v4"); + + + wbwi.rollbackToSavePoint(); + + assertThat(getFromWriteBatchWithIndex(db, readOptions, wbwi, "k2")) + .isNull(); + assertThat(getFromWriteBatchWithIndex(db, readOptions, wbwi, "k3")) + .isEqualTo("v3-2"); + assertThat(getFromWriteBatchWithIndex(db, readOptions, wbwi, "k4")) + .isNull(); + + + wbwi.rollbackToSavePoint(); + + assertThat(getFromWriteBatchWithIndex(db, readOptions, wbwi, "k1")) + .isEqualTo("v1"); + assertThat(getFromWriteBatchWithIndex(db, readOptions, wbwi, "k2")) + .isEqualTo("v2"); + assertThat(getFromWriteBatchWithIndex(db, readOptions, wbwi, "k3")) + .isEqualTo("v3"); + assertThat(getFromWriteBatchWithIndex(db, readOptions, wbwi, "k4")) + .isNull(); + } + } + } + + @Test + public void restorePoints() throws RocksDBException { + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex()) { + + wbwi.put("k1".getBytes(UTF_8), "v1".getBytes(UTF_8)); + wbwi.put("k2".getBytes(UTF_8), "v2".getBytes(UTF_8)); + + wbwi.setSavePoint(); + + wbwi.put("k1".getBytes(UTF_8), "123456789".getBytes(UTF_8)); + wbwi.delete("k2".getBytes(UTF_8)); + + wbwi.rollbackToSavePoint(); + + try(final DBOptions options = new DBOptions()) { + assertThat(wbwi.getFromBatch(options,"k1".getBytes(UTF_8))).isEqualTo("v1".getBytes()); + assertThat(wbwi.getFromBatch(options,"k2".getBytes(UTF_8))).isEqualTo("v2".getBytes()); + } + } + } + + @Test(expected = RocksDBException.class) + public void restorePoints_withoutSavePoints() throws RocksDBException { + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex()) { + wbwi.rollbackToSavePoint(); + } + } + + @Test(expected = RocksDBException.class) + public void restorePoints_withoutSavePoints_nested() throws RocksDBException { + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex()) { + + wbwi.setSavePoint(); + wbwi.rollbackToSavePoint(); + + // without previous corresponding setSavePoint + wbwi.rollbackToSavePoint(); + } + } + + @Test + public void popSavePoint() throws RocksDBException { + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex()) { + + wbwi.put("k1".getBytes(), "v1".getBytes()); + wbwi.put("k2".getBytes(), "v2".getBytes()); + + wbwi.setSavePoint(); + + wbwi.put("k1".getBytes(), "123456789".getBytes()); + wbwi.delete("k2".getBytes()); + + wbwi.setSavePoint(); + + wbwi.popSavePoint(); + + wbwi.rollbackToSavePoint(); + + try(final DBOptions options = new DBOptions()) { + assertThat(wbwi.getFromBatch(options,"k1".getBytes(UTF_8))).isEqualTo("v1".getBytes()); + assertThat(wbwi.getFromBatch(options,"k2".getBytes(UTF_8))).isEqualTo("v2".getBytes()); + } + } + } + + @Test(expected = RocksDBException.class) + public void popSavePoint_withoutSavePoints() throws RocksDBException { + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex()) { + wbwi.popSavePoint(); + } + } + + @Test(expected = RocksDBException.class) + public void popSavePoint_withoutSavePoints_nested() throws RocksDBException { + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex()) { + + wbwi.setSavePoint(); + wbwi.popSavePoint(); + + // without previous corresponding setSavePoint + wbwi.popSavePoint(); + } + } + + @Test + public void maxBytes() throws RocksDBException { + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex()) { + wbwi.setMaxBytes(19); + + wbwi.put("k1".getBytes(), "v1".getBytes()); + } + } + + @Test(expected = RocksDBException.class) + public void maxBytes_over() throws RocksDBException { + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex()) { + wbwi.setMaxBytes(1); + + wbwi.put("k1".getBytes(), "v1".getBytes()); + } + } + + @Test + public void getWriteBatch() { + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex()) { + + final WriteBatch wb = wbwi.getWriteBatch(); + assertThat(wb).isNotNull(); + assertThat(wb.isOwningHandle()).isFalse(); + } + } + + private static String getFromWriteBatchWithIndex(final RocksDB db, + final ReadOptions readOptions, final WriteBatchWithIndex wbwi, + final String skey) { + final byte[] key = skey.getBytes(); + try (final RocksIterator baseIterator = db.newIterator(readOptions); + final RocksIterator iterator = wbwi.newIteratorWithBase(baseIterator)) { + iterator.seek(key); + + // Arrays.equals(key, iterator.key()) ensures an exact match in Rocks, + // instead of a nearest match + return iterator.isValid() && + Arrays.equals(key, iterator.key()) ? + new String(iterator.value()) : null; + } + } + + @Test + public void getFromBatch() throws RocksDBException { + final byte[] k1 = "k1".getBytes(); + final byte[] k2 = "k2".getBytes(); + final byte[] k3 = "k3".getBytes(); + final byte[] k4 = "k4".getBytes(); + + final byte[] v1 = "v1".getBytes(); + final byte[] v2 = "v2".getBytes(); + final byte[] v3 = "v3".getBytes(); + + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex(true); + final DBOptions dbOptions = new DBOptions()) { + wbwi.put(k1, v1); + wbwi.put(k2, v2); + wbwi.put(k3, v3); + + assertThat(wbwi.getFromBatch(dbOptions, k1)).isEqualTo(v1); + assertThat(wbwi.getFromBatch(dbOptions, k2)).isEqualTo(v2); + assertThat(wbwi.getFromBatch(dbOptions, k3)).isEqualTo(v3); + assertThat(wbwi.getFromBatch(dbOptions, k4)).isNull(); + + wbwi.delete(k2); + + assertThat(wbwi.getFromBatch(dbOptions, k2)).isNull(); + } + } + + @Test + public void getFromBatchAndDB() throws RocksDBException { + final byte[] k1 = "k1".getBytes(); + final byte[] k2 = "k2".getBytes(); + final byte[] k3 = "k3".getBytes(); + final byte[] k4 = "k4".getBytes(); + + final byte[] v1 = "v1".getBytes(); + final byte[] v2 = "v2".getBytes(); + final byte[] v3 = "v3".getBytes(); + final byte[] v4 = "v4".getBytes(); + + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, + dbFolder.getRoot().getAbsolutePath())) { + + db.put(k1, v1); + db.put(k2, v2); + db.put(k4, v4); + + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex(true); + final DBOptions dbOptions = new DBOptions(); + final ReadOptions readOptions = new ReadOptions()) { + + assertThat(wbwi.getFromBatch(dbOptions, k1)).isNull(); + assertThat(wbwi.getFromBatch(dbOptions, k2)).isNull(); + assertThat(wbwi.getFromBatch(dbOptions, k4)).isNull(); + + wbwi.put(k3, v3); + + assertThat(wbwi.getFromBatch(dbOptions, k3)).isEqualTo(v3); + + assertThat(wbwi.getFromBatchAndDB(db, readOptions, k1)).isEqualTo(v1); + assertThat(wbwi.getFromBatchAndDB(db, readOptions, k2)).isEqualTo(v2); + assertThat(wbwi.getFromBatchAndDB(db, readOptions, k3)).isEqualTo(v3); + assertThat(wbwi.getFromBatchAndDB(db, readOptions, k4)).isEqualTo(v4); + + wbwi.delete(k4); + + assertThat(wbwi.getFromBatchAndDB(db, readOptions, k4)).isNull(); + } + } + } + private byte[] toArray(final ByteBuffer buf) { + final byte[] ary = new byte[buf.remaining()]; + buf.get(ary); + return ary; + } + + @Test + public void deleteRange() throws RocksDBException { + try (final RocksDB db = RocksDB.open(dbFolder.getRoot().getAbsolutePath()); + final WriteBatch batch = new WriteBatch(); + final WriteOptions wOpt = new WriteOptions()) { + db.put("key1".getBytes(), "value".getBytes()); + db.put("key2".getBytes(), "12345678".getBytes()); + db.put("key3".getBytes(), "abcdefg".getBytes()); + db.put("key4".getBytes(), "xyz".getBytes()); + assertThat(db.get("key1".getBytes())).isEqualTo("value".getBytes()); + assertThat(db.get("key2".getBytes())).isEqualTo("12345678".getBytes()); + assertThat(db.get("key3".getBytes())).isEqualTo("abcdefg".getBytes()); + assertThat(db.get("key4".getBytes())).isEqualTo("xyz".getBytes()); + + batch.deleteRange("key2".getBytes(), "key4".getBytes()); + db.write(wOpt, batch); + + assertThat(db.get("key1".getBytes())).isEqualTo("value".getBytes()); + assertThat(db.get("key2".getBytes())).isNull(); + assertThat(db.get("key3".getBytes())).isNull(); + assertThat(db.get("key4".getBytes())).isEqualTo("xyz".getBytes()); + } + } + + @Test + public void iteratorWithBaseOverwriteTrue() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex(true); + final RocksIterator baseIter = db.newIterator(); + final RocksIterator wbwiIter = wbwi.newIteratorWithBase(baseIter)) { + assertThat(wbwiIter).isNotNull(); + assertThat(wbwiIter.nativeHandle_).isGreaterThan(0); + wbwiIter.status(); + } + + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex(true); + final RocksIterator baseIter = db.newIterator(); + final ReadOptions readOptions = new ReadOptions(); + final RocksIterator wbwiIter = wbwi.newIteratorWithBase(baseIter, readOptions)) { + assertThat(wbwiIter).isNotNull(); + assertThat(wbwiIter.nativeHandle_).isGreaterThan(0); + wbwiIter.status(); + } + } + + final List cfNames = + Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes())); + final List columnFamilyHandleList = new ArrayList<>(); + try (final DBOptions options = + new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open( + options, dbFolder.getRoot().getAbsolutePath(), cfNames, columnFamilyHandleList)) { + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex(true); + final RocksIterator baseIter = db.newIterator(); + final RocksIterator wbwiIter = + wbwi.newIteratorWithBase(columnFamilyHandleList.get(1), baseIter)) { + assertThat(wbwiIter).isNotNull(); + assertThat(wbwiIter.nativeHandle_).isGreaterThan(0); + wbwiIter.status(); + } + + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex(true); + final RocksIterator baseIter = db.newIterator(); + final ReadOptions readOptions = new ReadOptions(); + final RocksIterator wbwiIter = + wbwi.newIteratorWithBase(columnFamilyHandleList.get(1), baseIter, readOptions)) { + assertThat(wbwiIter).isNotNull(); + assertThat(wbwiIter.nativeHandle_).isGreaterThan(0); + wbwiIter.status(); + } + } + } + + @Test + public void iteratorWithBaseOverwriteFalse() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex(false); + final RocksIterator baseIter = db.newIterator(); + final RocksIterator wbwiIter = wbwi.newIteratorWithBase(baseIter)) { + assertThat(wbwiIter).isNotNull(); + assertThat(wbwiIter.nativeHandle_).isGreaterThan(0); + wbwiIter.status(); + } + + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex(false); + final RocksIterator baseIter = db.newIterator(); + final ReadOptions readOptions = new ReadOptions(); + final RocksIterator wbwiIter = wbwi.newIteratorWithBase(baseIter, readOptions)) { + assertThat(wbwiIter).isNotNull(); + assertThat(wbwiIter.nativeHandle_).isGreaterThan(0); + wbwiIter.status(); + } + } + + final List cfNames = + Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes())); + final List columnFamilyHandleList = new ArrayList<>(); + try (final DBOptions options = + new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); + final RocksDB db = RocksDB.open( + options, dbFolder.getRoot().getAbsolutePath(), cfNames, columnFamilyHandleList)) { + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex(false); + final RocksIterator baseIter = db.newIterator(); + final RocksIterator wbwiIter = + wbwi.newIteratorWithBase(columnFamilyHandleList.get(1), baseIter)) { + assertThat(wbwiIter).isNotNull(); + assertThat(wbwiIter.nativeHandle_).isGreaterThan(0); + wbwiIter.status(); + } + + try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex(false); + final RocksIterator baseIter = db.newIterator(); + final ReadOptions readOptions = new ReadOptions(); + final RocksIterator wbwiIter = + wbwi.newIteratorWithBase(columnFamilyHandleList.get(1), baseIter, readOptions)) { + assertThat(wbwiIter).isNotNull(); + assertThat(wbwiIter.nativeHandle_).isGreaterThan(0); + wbwiIter.status(); + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/WriteOptionsTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/WriteOptionsTest.java new file mode 100644 index 0000000..1e1c93f --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/WriteOptionsTest.java @@ -0,0 +1,75 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Random; +import org.junit.ClassRule; +import org.junit.Test; + +public class WriteOptionsTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + public static final Random rand = PlatformRandomHelper. + getPlatformSpecificRandomFactory(); + + @Test + public void writeOptions() { + try (final WriteOptions writeOptions = new WriteOptions()) { + + writeOptions.setSync(true); + assertThat(writeOptions.sync()).isTrue(); + writeOptions.setSync(false); + assertThat(writeOptions.sync()).isFalse(); + + writeOptions.setDisableWAL(true); + assertThat(writeOptions.disableWAL()).isTrue(); + writeOptions.setDisableWAL(false); + assertThat(writeOptions.disableWAL()).isFalse(); + + + writeOptions.setIgnoreMissingColumnFamilies(true); + assertThat(writeOptions.ignoreMissingColumnFamilies()).isTrue(); + writeOptions.setIgnoreMissingColumnFamilies(false); + assertThat(writeOptions.ignoreMissingColumnFamilies()).isFalse(); + + writeOptions.setNoSlowdown(true); + assertThat(writeOptions.noSlowdown()).isTrue(); + writeOptions.setNoSlowdown(false); + assertThat(writeOptions.noSlowdown()).isFalse(); + + writeOptions.setLowPri(true); + assertThat(writeOptions.lowPri()).isTrue(); + writeOptions.setLowPri(false); + assertThat(writeOptions.lowPri()).isFalse(); + + writeOptions.setMemtableInsertHintPerBatch(true); + assertThat(writeOptions.memtableInsertHintPerBatch()).isTrue(); + writeOptions.setMemtableInsertHintPerBatch(false); + assertThat(writeOptions.memtableInsertHintPerBatch()).isFalse(); + } + } + + @Test + public void copyConstructor() { + final WriteOptions origOpts = new WriteOptions(); + origOpts.setDisableWAL(rand.nextBoolean()); + origOpts.setIgnoreMissingColumnFamilies(rand.nextBoolean()); + origOpts.setSync(rand.nextBoolean()); + origOpts.setMemtableInsertHintPerBatch(true); + final WriteOptions copyOpts = new WriteOptions(origOpts); + assertThat(origOpts.disableWAL()).isEqualTo(copyOpts.disableWAL()); + assertThat(origOpts.ignoreMissingColumnFamilies()).isEqualTo( + copyOpts.ignoreMissingColumnFamilies()); + assertThat(origOpts.sync()).isEqualTo(copyOpts.sync()); + assertThat(origOpts.memtableInsertHintPerBatch()) + .isEqualTo(copyOpts.memtableInsertHintPerBatch()); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/test/RemoveEmptyValueCompactionFilterFactory.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/test/RemoveEmptyValueCompactionFilterFactory.java new file mode 100644 index 0000000..c4e4f25 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/test/RemoveEmptyValueCompactionFilterFactory.java @@ -0,0 +1,21 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb.test; + +import org.rocksdb.AbstractCompactionFilter; +import org.rocksdb.AbstractCompactionFilterFactory; +import org.rocksdb.RemoveEmptyValueCompactionFilter; + +/** + * Simple CompactionFilterFactory class used in tests. Generates RemoveEmptyValueCompactionFilters. + */ +public class RemoveEmptyValueCompactionFilterFactory extends AbstractCompactionFilterFactory { + @Override + public RemoveEmptyValueCompactionFilter createCompactionFilter(final AbstractCompactionFilter.Context context) { + return new RemoveEmptyValueCompactionFilter(); + } + + @Override + public String name() { + return "RemoveEmptyValueCompactionFilterFactory"; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/test/RocksJunitRunner.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/test/RocksJunitRunner.java new file mode 100644 index 0000000..42d3148 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/test/RocksJunitRunner.java @@ -0,0 +1,174 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb.test; + +import org.junit.internal.JUnitSystem; +import org.junit.internal.RealSystem; +import org.junit.internal.TextListener; +import org.junit.runner.Description; +import org.junit.runner.JUnitCore; +import org.junit.runner.Result; +import org.junit.runner.notification.Failure; +import org.rocksdb.RocksDB; + +import java.io.PrintStream; +import java.text.DecimalFormat; +import java.text.NumberFormat; +import java.util.ArrayList; +import java.util.List; + +import static org.rocksdb.test.RocksJunitRunner.RocksJunitListener.Status.*; + +/** + * Custom Junit Runner to print also Test classes + * and executed methods to command prompt. + */ +public class RocksJunitRunner { + + /** + * Listener which overrides default functionality + * to print class and method to system out. + */ + static class RocksJunitListener extends TextListener { + + private final static NumberFormat secsFormat = + new DecimalFormat("###,###.###"); + + private final PrintStream writer; + + private String currentClassName = null; + private String currentMethodName = null; + private Status currentStatus = null; + private long currentTestsStartTime; + private int currentTestsCount = 0; + private int currentTestsIgnoredCount = 0; + private int currentTestsFailureCount = 0; + private int currentTestsErrorCount = 0; + + enum Status { + IGNORED, + FAILURE, + ERROR, + OK + } + + /** + * RocksJunitListener constructor + * + * @param system JUnitSystem + */ + public RocksJunitListener(final JUnitSystem system) { + this(system.out()); + } + + public RocksJunitListener(final PrintStream writer) { + super(writer); + this.writer = writer; + } + + @Override + public void testRunStarted(final Description description) { + writer.format("Starting RocksJava Tests...%n"); + + } + + @Override + public void testStarted(final Description description) { + if(currentClassName == null + || !currentClassName.equals(description.getClassName())) { + if(currentClassName != null) { + printTestsSummary(); + } else { + currentTestsStartTime = System.currentTimeMillis(); + } + writer.format("%nRunning: %s%n", description.getClassName()); + currentClassName = description.getClassName(); + } + currentMethodName = description.getMethodName(); + currentStatus = OK; + currentTestsCount++; + } + + private void printTestsSummary() { + // print summary of last test set + writer.format("Tests run: %d, Failures: %d, Errors: %d, Ignored: %d, Time elapsed: %s sec%n", + currentTestsCount, + currentTestsFailureCount, + currentTestsErrorCount, + currentTestsIgnoredCount, + formatSecs(System.currentTimeMillis() - currentTestsStartTime)); + + // reset counters + currentTestsCount = 0; + currentTestsFailureCount = 0; + currentTestsErrorCount = 0; + currentTestsIgnoredCount = 0; + currentTestsStartTime = System.currentTimeMillis(); + } + + private static String formatSecs(final double milliseconds) { + final double seconds = milliseconds / 1000; + return secsFormat.format(seconds); + } + + @Override + public void testFailure(final Failure failure) { + if (failure.getException() != null + && failure.getException() instanceof AssertionError) { + currentStatus = FAILURE; + currentTestsFailureCount++; + } else { + currentStatus = ERROR; + currentTestsErrorCount++; + } + } + + @Override + public void testIgnored(final Description description) { + currentStatus = IGNORED; + currentTestsIgnoredCount++; + } + + @Override + public void testFinished(final Description description) { + if(currentStatus == OK) { + writer.format("\t%s OK%n",currentMethodName); + } else { + writer.format(" [%s] %s%n", currentStatus.name(), currentMethodName); + } + } + + @Override + public void testRunFinished(final Result result) { + printTestsSummary(); + super.testRunFinished(result); + } + } + + /** + * Main method to execute tests + * + * @param args Test classes as String names + */ + public static void main(final String[] args){ + final JUnitCore runner = new JUnitCore(); + final JUnitSystem system = new RealSystem(); + runner.addListener(new RocksJunitListener(system)); + try { + final List> classes = new ArrayList<>(); + for (final String arg : args) { + classes.add(Class.forName(arg)); + } + final Class[] clazzes = classes.toArray(new Class[classes.size()]); + final Result result = runner.run(clazzes); + if(!result.wasSuccessful()) { + System.exit(-1); + } + } catch (final ClassNotFoundException e) { + e.printStackTrace(); + System.exit(-2); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/test/TestableEventListener.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/test/TestableEventListener.java new file mode 100644 index 0000000..865ad5c --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/test/TestableEventListener.java @@ -0,0 +1,23 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb.test; + +import org.rocksdb.AbstractEventListener; + +public class TestableEventListener extends AbstractEventListener { + public TestableEventListener() { + super(); + } + + public TestableEventListener(final EnabledEventCallback... enabledEventCallbacks) { + super(enabledEventCallbacks); + } + + public void invokeAllCallbacks() { + invokeAllCallbacks(nativeHandle_); + } + + private static native void invokeAllCallbacks(final long handle); +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/ByteBufferAllocator.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/ByteBufferAllocator.java new file mode 100644 index 0000000..8d7956c --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/ByteBufferAllocator.java @@ -0,0 +1,16 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb.util; + +import java.nio.ByteBuffer; + +public interface ByteBufferAllocator { + ByteBuffer allocate(int capacity); + + ByteBufferAllocator DIRECT = new DirectByteBufferAllocator(); + ByteBufferAllocator HEAP = new HeapByteBufferAllocator(); +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/BytewiseComparatorIntTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/BytewiseComparatorIntTest.java new file mode 100644 index 0000000..fb7239c --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/BytewiseComparatorIntTest.java @@ -0,0 +1,267 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb.util; + +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; +import org.rocksdb.*; + +import java.nio.ByteBuffer; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Similar to {@link IntComparatorTest}, but uses {@link BytewiseComparator} + * which ensures the correct ordering of positive integers. + */ +@RunWith(Parameterized.class) +public class BytewiseComparatorIntTest { + + // test with 500 random positive integer keys + private static final int TOTAL_KEYS = 500; + private static final byte[][] keys = new byte[TOTAL_KEYS][4]; + + @BeforeClass + public static void prepareKeys() { + final ByteBuffer buf = ByteBuffer.allocate(4); + final Random random = new Random(); + for (int i = 0; i < TOTAL_KEYS; i++) { + final int ri = random.nextInt() & Integer.MAX_VALUE; // the & ensures positive integer + buf.putInt(ri); + buf.flip(); + final byte[] key = buf.array(); + + // does key already exist (avoid duplicates) + if (keyExists(key, i)) { + i--; // loop round and generate a different key + } else { + System.arraycopy(key, 0, keys[i], 0, 4); + } + } + } + + private static boolean keyExists(final byte[] key, final int limit) { + for (int j = 0; j < limit; j++) { + if (Arrays.equals(key, keys[j])) { + return true; + } + } + return false; + } + + @Parameters(name = "{0}") + public static Iterable parameters() { + return Arrays.asList(new Object[][] { + { "non-direct_reused64_mutex", false, 64, ReusedSynchronisationType.MUTEX }, + { "direct_reused64_mutex", true, 64, ReusedSynchronisationType.MUTEX }, + { "non-direct_reused64_adaptive-mutex", false, 64, ReusedSynchronisationType.ADAPTIVE_MUTEX }, + { "direct_reused64_adaptive-mutex", true, 64, ReusedSynchronisationType.ADAPTIVE_MUTEX }, + { "non-direct_reused64_thread-local", false, 64, ReusedSynchronisationType.THREAD_LOCAL }, + { "direct_reused64_thread-local", true, 64, ReusedSynchronisationType.THREAD_LOCAL }, + { "non-direct_noreuse", false, -1, null }, + { "direct_noreuse", true, -1, null } + }); + } + + @Parameter(0) + public String name; + + @Parameter(1) + public boolean useDirectBuffer; + + @Parameter(2) + public int maxReusedBufferSize; + + @Parameter(3) + public ReusedSynchronisationType reusedSynchronisationType; + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + + @Test + public void javaComparatorDefaultCf() throws RocksDBException { + try (final ComparatorOptions options = new ComparatorOptions() + .setUseDirectBuffer(useDirectBuffer) + .setMaxReusedBufferSize(maxReusedBufferSize) + // if reusedSynchronisationType == null we assume that maxReusedBufferSize <= 0 and so we just set ADAPTIVE_MUTEX, even though it won't be used + .setReusedSynchronisationType(reusedSynchronisationType == null ? ReusedSynchronisationType.ADAPTIVE_MUTEX : reusedSynchronisationType); + final BytewiseComparator comparator = new BytewiseComparator(options)) { + + // test the round-tripability of keys written and read with the Comparator + testRoundtrip(FileSystems.getDefault().getPath( + dbFolder.getRoot().getAbsolutePath()), comparator); + } + } + + @Test + public void javaComparatorNamedCf() throws RocksDBException { + try (final ComparatorOptions options = new ComparatorOptions() + .setUseDirectBuffer(useDirectBuffer) + .setMaxReusedBufferSize(maxReusedBufferSize) + // if reusedSynchronisationType == null we assume that maxReusedBufferSize <= 0 and so we just set ADAPTIVE_MUTEX, even though it won't be used + .setReusedSynchronisationType(reusedSynchronisationType == null ? ReusedSynchronisationType.ADAPTIVE_MUTEX : reusedSynchronisationType); + final BytewiseComparator comparator = new BytewiseComparator(options)) { + + // test the round-tripability of keys written and read with the Comparator + testRoundtripCf(FileSystems.getDefault().getPath( + dbFolder.getRoot().getAbsolutePath()), comparator); + } + } + + /** + * Test which stores random keys into the database + * using an {@link IntComparator} + * it then checks that these keys are read back in + * ascending order + * + * @param db_path A path where we can store database + * files temporarily + * + * @param comparator the comparator + * + * @throws RocksDBException if a database error happens. + */ + private void testRoundtrip(final Path db_path, + final AbstractComparator comparator) throws RocksDBException { + try (final Options opt = new Options() + .setCreateIfMissing(true) + .setComparator(comparator)) { + + // store TOTAL_KEYS into the db + try (final RocksDB db = RocksDB.open(opt, db_path.toString())) { + for (int i = 0; i < TOTAL_KEYS; i++) { + db.put(keys[i], "value".getBytes(UTF_8)); + } + } + + // re-open db and read from start to end + // integer keys should be in ascending + // order as defined by IntComparator + final ByteBuffer key = ByteBuffer.allocate(4); + try (final RocksDB db = RocksDB.open(opt, db_path.toString()); + final RocksIterator it = db.newIterator()) { + it.seekToFirst(); + int lastKey = Integer.MIN_VALUE; + int count = 0; + for (it.seekToFirst(); it.isValid(); it.next()) { + key.put(it.key()); + key.flip(); + final int thisKey = key.getInt(); + key.clear(); + assertThat(thisKey).isGreaterThan(lastKey); + lastKey = thisKey; + count++; + } + assertThat(count).isEqualTo(TOTAL_KEYS); + } + } + } + + /** + * Test which stores random keys into a column family + * in the database + * using an {@link IntComparator} + * it then checks that these keys are read back in + * ascending order + * + * @param db_path A path where we can store database + * files temporarily + * + * @param comparator the comparator + * + * @throws RocksDBException if a database error happens. + */ + private void testRoundtripCf(final Path db_path, + final AbstractComparator comparator) throws RocksDBException { + + final List cfDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes(), + new ColumnFamilyOptions() + .setComparator(comparator)) + ); + + final List cfHandles = new ArrayList<>(); + + try (final DBOptions opt = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true)) { + + try (final RocksDB db = RocksDB.open(opt, db_path.toString(), + cfDescriptors, cfHandles)) { + try { + assertThat(cfDescriptors.size()).isEqualTo(2); + assertThat(cfHandles.size()).isEqualTo(2); + + for (int i = 0; i < TOTAL_KEYS; i++) { + db.put(cfHandles.get(1), keys[i], "value".getBytes(UTF_8)); + } + } finally { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + cfHandles.clear(); + } + } + + // re-open db and read from start to end + // integer keys should be in ascending + // order as defined by SimpleIntComparator + final ByteBuffer key = ByteBuffer.allocate(4); + try (final RocksDB db = RocksDB.open(opt, db_path.toString(), + cfDescriptors, cfHandles); + final RocksIterator it = db.newIterator(cfHandles.get(1))) { + try { + assertThat(cfDescriptors.size()).isEqualTo(2); + assertThat(cfHandles.size()).isEqualTo(2); + + it.seekToFirst(); + int lastKey = Integer.MIN_VALUE; + int count = 0; + for (it.seekToFirst(); it.isValid(); it.next()) { + key.put(it.key()); + key.flip(); + final int thisKey = key.getInt(); + key.clear(); + assertThat(thisKey).isGreaterThan(lastKey); + lastKey = thisKey; + count++; + } + + assertThat(count).isEqualTo(TOTAL_KEYS); + + } finally { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + cfHandles.clear(); + for (final ColumnFamilyDescriptor cfDescriptor : cfDescriptors) { + cfDescriptor.getOptions().close(); + } + } + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/BytewiseComparatorTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/BytewiseComparatorTest.java new file mode 100644 index 0000000..69f2c28 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/BytewiseComparatorTest.java @@ -0,0 +1,531 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb.util; + +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.*; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.*; +import java.util.*; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.*; +import static org.rocksdb.util.ByteUtil.bytes; + +/** + * This is a direct port of various C++ + * tests from db/comparator_db_test.cc + * and some code to adapt it to RocksJava + */ +public class BytewiseComparatorTest { + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + private List source_strings = Arrays.asList("b", "d", "f", "h", "j", "l"); + private List interleaving_strings = Arrays.asList("a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m"); + + /** + * Open the database using the C++ BytewiseComparatorImpl + * and test the results against our Java BytewiseComparator + */ + @Test + public void java_vs_cpp_bytewiseComparator() + throws IOException, RocksDBException { + for(int rand_seed = 301; rand_seed < 306; rand_seed++) { + final Path dbDir = + FileSystems.getDefault().getPath(dbFolder.newFolder().getAbsolutePath()); + try(final RocksDB db = openDatabase(dbDir, + BuiltinComparator.BYTEWISE_COMPARATOR)) { + + final Random rnd = new Random(rand_seed); + try(final ComparatorOptions copt2 = new ComparatorOptions() + .setUseDirectBuffer(false); + final AbstractComparator comparator2 = new BytewiseComparator(copt2)) { + final java.util.Comparator jComparator = toJavaComparator(comparator2); + doRandomIterationTest( + db, + jComparator, + rnd, + 8, 100, 3 + ); + } + } + } + } + + /** + * Open the database using the Java BytewiseComparator + * and test the results against another Java BytewiseComparator + */ + @Test + public void java_vs_java_bytewiseComparator() + throws IOException, RocksDBException { + for(int rand_seed = 301; rand_seed < 306; rand_seed++) { + final Path dbDir = + FileSystems.getDefault().getPath(dbFolder.newFolder().getAbsolutePath()); + try(final ComparatorOptions copt = new ComparatorOptions() + .setUseDirectBuffer(false); + final AbstractComparator comparator = new BytewiseComparator(copt); + final RocksDB db = openDatabase(dbDir, comparator)) { + + final Random rnd = new Random(rand_seed); + try(final ComparatorOptions copt2 = new ComparatorOptions() + .setUseDirectBuffer(false); + final AbstractComparator comparator2 = new BytewiseComparator(copt2)) { + final java.util.Comparator jComparator = toJavaComparator(comparator2); + doRandomIterationTest( + db, + jComparator, + rnd, + 8, 100, 3 + ); + } + } + } + } + + /** + * Open the database using the C++ BytewiseComparatorImpl + * and test the results against our Java DirectBytewiseComparator + */ + @Test + public void java_vs_cpp_directBytewiseComparator() + throws IOException, RocksDBException { + for(int rand_seed = 301; rand_seed < 306; rand_seed++) { + final Path dbDir = + FileSystems.getDefault().getPath(dbFolder.newFolder().getAbsolutePath()); + try(final RocksDB db = openDatabase(dbDir, + BuiltinComparator.BYTEWISE_COMPARATOR)) { + + final Random rnd = new Random(rand_seed); + try(final ComparatorOptions copt2 = new ComparatorOptions() + .setUseDirectBuffer(true); + final AbstractComparator comparator2 = new BytewiseComparator(copt2)) { + final java.util.Comparator jComparator = toJavaComparator(comparator2); + doRandomIterationTest( + db, + jComparator, + rnd, + 8, 100, 3 + ); + } + } + } + } + + /** + * Open the database using the Java DirectBytewiseComparator + * and test the results against another Java DirectBytewiseComparator + */ + @Test + public void java_vs_java_directBytewiseComparator() + throws IOException, RocksDBException { + for(int rand_seed = 301; rand_seed < 306; rand_seed++) { + final Path dbDir = + FileSystems.getDefault().getPath(dbFolder.newFolder().getAbsolutePath()); + try (final ComparatorOptions copt = new ComparatorOptions() + .setUseDirectBuffer(true); + final AbstractComparator comparator = new BytewiseComparator(copt); + final RocksDB db = openDatabase(dbDir, comparator)) { + + final Random rnd = new Random(rand_seed); + try(final ComparatorOptions copt2 = new ComparatorOptions() + .setUseDirectBuffer(true); + final AbstractComparator comparator2 = new BytewiseComparator(copt2)) { + final java.util.Comparator jComparator = toJavaComparator(comparator2); + doRandomIterationTest( + db, + jComparator, + rnd, + 8, 100, 3 + ); + } + } + } + } + + /** + * Open the database using the C++ ReverseBytewiseComparatorImpl + * and test the results against our Java ReverseBytewiseComparator + */ + @Test + public void java_vs_cpp_reverseBytewiseComparator() + throws IOException, RocksDBException { + for(int rand_seed = 301; rand_seed < 306; rand_seed++) { + final Path dbDir = + FileSystems.getDefault().getPath(dbFolder.newFolder().getAbsolutePath()); + try(final RocksDB db = openDatabase(dbDir, + BuiltinComparator.REVERSE_BYTEWISE_COMPARATOR)) { + + final Random rnd = new Random(rand_seed); + try(final ComparatorOptions copt2 = new ComparatorOptions() + .setUseDirectBuffer(false); + final AbstractComparator comparator2 = new ReverseBytewiseComparator(copt2)) { + final java.util.Comparator jComparator = toJavaComparator(comparator2); + doRandomIterationTest( + db, + jComparator, + rnd, + 8, 100, 3 + ); + } + } + } + } + + /** + * Open the database using the Java ReverseBytewiseComparator + * and test the results against another Java ReverseBytewiseComparator + */ + @Test + public void java_vs_java_reverseBytewiseComparator() + throws IOException, RocksDBException { + for(int rand_seed = 301; rand_seed < 306; rand_seed++) { + final Path dbDir = + FileSystems.getDefault().getPath(dbFolder.newFolder().getAbsolutePath()); + try (final ComparatorOptions copt = new ComparatorOptions() + .setUseDirectBuffer(false); + final AbstractComparator comparator = new ReverseBytewiseComparator(copt); + final RocksDB db = openDatabase(dbDir, comparator)) { + + final Random rnd = new Random(rand_seed); + try(final ComparatorOptions copt2 = new ComparatorOptions() + .setUseDirectBuffer(false); + final AbstractComparator comparator2 = new ReverseBytewiseComparator(copt2)) { + final java.util.Comparator jComparator = toJavaComparator(comparator2); + doRandomIterationTest( + db, + jComparator, + rnd, + 8, 100, 3 + ); + } + } + } + } + + private void doRandomIterationTest( + final RocksDB db, final java.util.Comparator javaComparator, + final Random rnd, + final int num_writes, final int num_iter_ops, + final int num_trigger_flush) throws RocksDBException { + + final TreeMap map = new TreeMap<>(javaComparator); + + try (final FlushOptions flushOptions = new FlushOptions(); + final WriteOptions writeOptions = new WriteOptions()) { + for (int i = 0; i < num_writes; i++) { + if (num_trigger_flush > 0 && i != 0 && i % num_trigger_flush == 0) { + db.flush(flushOptions); + } + + final int type = rnd.nextInt(2); + final int index = rnd.nextInt(source_strings.size()); + final String key = source_strings.get(index); + switch (type) { + case 0: + // put + map.put(key, key); + db.put(writeOptions, bytes(key), bytes(key)); + break; + case 1: + // delete + if (map.containsKey(key)) { + map.remove(key); + } + db.delete(writeOptions, bytes(key)); + break; + + default: + fail("Should not be able to generate random outside range 1..2"); + } + } + } + + try (final ReadOptions readOptions = new ReadOptions(); + final RocksIterator iter = db.newIterator(readOptions)) { + final KVIter result_iter = new KVIter<>(map); + + boolean is_valid = false; + for (int i = 0; i < num_iter_ops; i++) { + // Random walk and make sure iter and result_iter returns the + // same key and value + final int type = rnd.nextInt(8); + iter.status(); + switch (type) { + case 0: + // Seek to First + iter.seekToFirst(); + result_iter.seekToFirst(); + break; + case 1: + // Seek to last + iter.seekToLast(); + result_iter.seekToLast(); + break; + case 2: { + // Seek to random (existing or non-existing) key + final int key_idx = rnd.nextInt(interleaving_strings.size()); + final String key = interleaving_strings.get(key_idx); + iter.seek(bytes(key)); + result_iter.seek(bytes(key)); + break; + } + case 3: { + // SeekForPrev to random (existing or non-existing) key + final int key_idx = rnd.nextInt(interleaving_strings.size()); + final String key = interleaving_strings.get(key_idx); + iter.seekForPrev(bytes(key)); + result_iter.seekForPrev(bytes(key)); + break; + } + case 4: + // Next + if (is_valid) { + iter.next(); + result_iter.next(); + } else { + continue; + } + break; + case 5: + // Prev + if (is_valid) { + iter.prev(); + result_iter.prev(); + } else { + continue; + } + break; + case 6: + // Refresh + iter.refresh(); + result_iter.refresh(); + iter.seekToFirst(); + result_iter.seekToFirst(); + break; + default: { + assert (type == 7); + final int key_idx = rnd.nextInt(source_strings.size()); + final String key = source_strings.get(key_idx); + final byte[] result = db.get(readOptions, bytes(key)); + if (!map.containsKey(key)) { + assertNull(result); + } else { + assertArrayEquals(bytes(map.get(key)), result); + } + break; + } + } + + assertEquals(result_iter.isValid(), iter.isValid()); + + is_valid = iter.isValid(); + + if (is_valid) { + assertArrayEquals(bytes(result_iter.key()), iter.key()); + + //note that calling value on a non-valid iterator from the Java API + //results in a SIGSEGV + assertArrayEquals(bytes(result_iter.value()), iter.value()); + } + } + } + } + + /** + * Open the database using a C++ Comparator + */ + private RocksDB openDatabase( + final Path dbDir, final BuiltinComparator cppComparator) + throws IOException, RocksDBException { + final Options options = new Options() + .setCreateIfMissing(true) + .setComparator(cppComparator); + return RocksDB.open(options, dbDir.toAbsolutePath().toString()); + } + + /** + * Open the database using a Java Comparator + */ + private RocksDB openDatabase( + final Path dbDir, + final AbstractComparator javaComparator) + throws IOException, RocksDBException { + final Options options = new Options() + .setCreateIfMissing(true) + .setComparator(javaComparator); + return RocksDB.open(options, dbDir.toAbsolutePath().toString()); + } + + private java.util.Comparator toJavaComparator( + final AbstractComparator rocksComparator) { + return new java.util.Comparator() { + @Override + public int compare(final String s1, final String s2) { + final ByteBuffer bufS1; + final ByteBuffer bufS2; + if (rocksComparator.usingDirectBuffers()) { + bufS1 = ByteBuffer.allocateDirect(s1.length()); + bufS2 = ByteBuffer.allocateDirect(s2.length()); + } else { + bufS1 = ByteBuffer.allocate(s1.length()); + bufS2 = ByteBuffer.allocate(s2.length()); + } + bufS1.put(bytes(s1)); + bufS1.flip(); + bufS2.put(bytes(s2)); + bufS2.flip(); + return rocksComparator.compare(bufS1, bufS2); + } + }; + } + + private static class KVIter implements RocksIteratorInterface { + + private final List> entries; + private final java.util.Comparator comparator; + private int offset = -1; + + private int lastPrefixMatchIdx = -1; + private int lastPrefixMatch = 0; + + public KVIter(final TreeMap map) { + this.entries = new ArrayList<>(); + entries.addAll(map.entrySet()); + this.comparator = map.comparator(); + } + + + @Override + public boolean isValid() { + return offset > -1 && offset < entries.size(); + } + + @Override + public void seekToFirst() { + offset = 0; + } + + @Override + public void seekToLast() { + offset = entries.size() - 1; + } + + @SuppressWarnings("unchecked") + @Override + public void seek(final byte[] target) { + for(offset = 0; offset < entries.size(); offset++) { + if(comparator.compare(entries.get(offset).getKey(), + (K)new String(target, UTF_8)) >= 0) { + return; + } + } + } + + @SuppressWarnings("unchecked") + @Override + public void seekForPrev(final byte[] target) { + for(offset = entries.size()-1; offset >= 0; offset--) { + if(comparator.compare(entries.get(offset).getKey(), + (K)new String(target, UTF_8)) <= 0) { + return; + } + } + } + + /** + * Is `a` a prefix of `b` + * + * @return The length of the matching prefix, or 0 if it is not a prefix + */ + private int isPrefix(final byte[] a, final byte[] b) { + if(b.length >= a.length) { + for(int i = 0; i < a.length; i++) { + if(a[i] != b[i]) { + return i; + } + } + return a.length; + } else { + return 0; + } + } + + @Override + public void next() { + if(offset < entries.size()) { + offset++; + } + } + + @Override + public void prev() { + if(offset >= 0) { + offset--; + } + } + + @Override + public void refresh() throws RocksDBException { + offset = -1; + } + + @Override + public void status() throws RocksDBException { + if(offset < 0 || offset >= entries.size()) { + throw new RocksDBException("Index out of bounds. Size is: " + + entries.size() + ", offset is: " + offset); + } + } + + @SuppressWarnings("unchecked") + public K key() { + if(!isValid()) { + if(entries.isEmpty()) { + return (K)""; + } else if(offset == -1){ + return entries.get(0).getKey(); + } else if(offset == entries.size()) { + return entries.get(offset - 1).getKey(); + } else { + return (K)""; + } + } else { + return entries.get(offset).getKey(); + } + } + + @SuppressWarnings("unchecked") + public V value() { + if(!isValid()) { + return (V)""; + } else { + return entries.get(offset).getValue(); + } + } + + @Override + public void seek(ByteBuffer target) { + throw new IllegalAccessError("Not implemented"); + } + + @Override + public void seekForPrev(ByteBuffer target) { + throw new IllegalAccessError("Not implemented"); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/CapturingWriteBatchHandler.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/CapturingWriteBatchHandler.java new file mode 100644 index 0000000..8ea1043 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/CapturingWriteBatchHandler.java @@ -0,0 +1,190 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb.util; + +import org.rocksdb.RocksDBException; +import org.rocksdb.WriteBatch; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +/** + * A simple WriteBatch Handler which adds a record + * of each event that it receives to a list + */ +public class CapturingWriteBatchHandler extends WriteBatch.Handler { + + private final List events = new ArrayList<>(); + + /** + * Returns a copy of the current events list + * + * @return a list of the events which have happened upto now + */ + public List getEvents() { + return new ArrayList<>(events); + } + + @Override + public void put(final int columnFamilyId, final byte[] key, + final byte[] value) { + events.add(new Event(Action.PUT, columnFamilyId, key, value)); + } + + @Override + public void put(final byte[] key, final byte[] value) { + events.add(new Event(Action.PUT, key, value)); + } + + @Override + public void merge(final int columnFamilyId, final byte[] key, + final byte[] value) { + events.add(new Event(Action.MERGE, columnFamilyId, key, value)); + } + + @Override + public void merge(final byte[] key, final byte[] value) { + events.add(new Event(Action.MERGE, key, value)); + } + + @Override + public void delete(final int columnFamilyId, final byte[] key) { + events.add(new Event(Action.DELETE, columnFamilyId, key, (byte[])null)); + } + + @Override + public void delete(final byte[] key) { + events.add(new Event(Action.DELETE, key, (byte[])null)); + } + + @Override + public void singleDelete(final int columnFamilyId, final byte[] key) { + events.add(new Event(Action.SINGLE_DELETE, + columnFamilyId, key, (byte[])null)); + } + + @Override + public void singleDelete(final byte[] key) { + events.add(new Event(Action.SINGLE_DELETE, key, (byte[])null)); + } + + @Override + public void deleteRange(final int columnFamilyId, final byte[] beginKey, + final byte[] endKey) { + events.add(new Event(Action.DELETE_RANGE, columnFamilyId, beginKey, + endKey)); + } + + @Override + public void deleteRange(final byte[] beginKey, final byte[] endKey) { + events.add(new Event(Action.DELETE_RANGE, beginKey, endKey)); + } + + @Override + public void logData(final byte[] blob) { + events.add(new Event(Action.LOG, (byte[])null, blob)); + } + + @Override + public void putBlobIndex(final int columnFamilyId, final byte[] key, + final byte[] value) { + events.add(new Event(Action.PUT_BLOB_INDEX, key, value)); + } + + @Override + public void markBeginPrepare() throws RocksDBException { + events.add(new Event(Action.MARK_BEGIN_PREPARE, (byte[])null, + (byte[])null)); + } + + @Override + public void markEndPrepare(final byte[] xid) throws RocksDBException { + events.add(new Event(Action.MARK_END_PREPARE, (byte[])null, + (byte[])null)); + } + + @Override + public void markNoop(final boolean emptyBatch) throws RocksDBException { + events.add(new Event(Action.MARK_NOOP, (byte[])null, (byte[])null)); + } + + @Override + public void markRollback(final byte[] xid) throws RocksDBException { + events.add(new Event(Action.MARK_ROLLBACK, (byte[])null, (byte[])null)); + } + + @Override + public void markCommit(final byte[] xid) throws RocksDBException { + events.add(new Event(Action.MARK_COMMIT, (byte[])null, (byte[])null)); + } + + @Override + public void markCommitWithTimestamp(final byte[] xid, final byte[] ts) throws RocksDBException { + events.add(new Event(Action.MARK_COMMIT_WITH_TIMESTAMP, (byte[]) null, (byte[]) null)); + } + + public static class Event { + public final Action action; + public final int columnFamilyId; + public final byte[] key; + public final byte[] value; + + public Event(final Action action, final byte[] key, final byte[] value) { + this(action, 0, key, value); + } + + public Event(final Action action, final int columnFamilyId, final byte[] key, + final byte[] value) { + this.action = action; + this.columnFamilyId = columnFamilyId; + this.key = key; + this.value = value; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final Event event = (Event) o; + return columnFamilyId == event.columnFamilyId && + action == event.action && + ((key == null && event.key == null) + || Arrays.equals(key, event.key)) && + ((value == null && event.value == null) + || Arrays.equals(value, event.value)); + } + + @Override + public int hashCode() { + int result = Objects.hash(action, columnFamilyId); + result = 31 * result + Arrays.hashCode(key); + result = 31 * result + Arrays.hashCode(value); + return result; + } + } + + /** + * Enumeration of Write Batch + * event actions + */ + public enum Action { + PUT, + MERGE, + DELETE, + SINGLE_DELETE, + DELETE_RANGE, + LOG, + PUT_BLOB_INDEX, + MARK_BEGIN_PREPARE, + MARK_END_PREPARE, + MARK_NOOP, + MARK_COMMIT, + MARK_ROLLBACK, + MARK_COMMIT_WITH_TIMESTAMP + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/DirectByteBufferAllocator.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/DirectByteBufferAllocator.java new file mode 100644 index 0000000..d26fb57 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/DirectByteBufferAllocator.java @@ -0,0 +1,18 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb.util; + +import java.nio.ByteBuffer; + +public final class DirectByteBufferAllocator implements ByteBufferAllocator { + DirectByteBufferAllocator(){}; + + @Override + public ByteBuffer allocate(final int capacity) { + return ByteBuffer.allocateDirect(capacity); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/EnvironmentTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/EnvironmentTest.java new file mode 100644 index 0000000..ae340e0 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/EnvironmentTest.java @@ -0,0 +1,304 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb.util; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.hamcrest.Matchers.is; + +import java.lang.reflect.Field; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class EnvironmentTest { + private final static String ARCH_FIELD_NAME = "ARCH"; + private final static String OS_FIELD_NAME = "OS"; + + private final static String MUSL_ENVIRONMENT_FIELD_NAME = "MUSL_ENVIRONMENT"; + private final static String MUSL_LIBC_FIELD_NAME = "MUSL_LIBC"; + + private static String INITIAL_OS; + private static String INITIAL_ARCH; + private static String INITIAL_MUSL_ENVIRONMENT; + private static Boolean INITIAL_MUSL_LIBC; + + @BeforeClass + public static void saveState() { + INITIAL_ARCH = getEnvironmentClassField(ARCH_FIELD_NAME); + INITIAL_OS = getEnvironmentClassField(OS_FIELD_NAME); + INITIAL_MUSL_LIBC = getEnvironmentClassField(MUSL_LIBC_FIELD_NAME); + INITIAL_MUSL_ENVIRONMENT = getEnvironmentClassField(MUSL_ENVIRONMENT_FIELD_NAME); + } + + @Test + public void mac32() { + setEnvironmentClassFields("mac", "32"); + assertThat(Environment.isWindows()).isFalse(); + assertThat(Environment.getJniLibraryExtension()). + isEqualTo(".jnilib"); + assertThat(Environment.getJniLibraryFileName("rocksdb")). + isEqualTo("librocksdbjni-osx.jnilib"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); + assertThat(Environment.getSharedLibraryFileName("rocksdb")). + isEqualTo("librocksdbjni.dylib"); + } + + @Test + public void mac64_x86_64() { + setEnvironmentClassFields("mac", "x86_64"); + assertThat(Environment.isWindows()).isFalse(); + assertThat(Environment.getJniLibraryExtension()). + isEqualTo(".jnilib"); + assertThat(Environment.getJniLibraryFileName("rocksdb")) + .isEqualTo("librocksdbjni-osx-x86_64.jnilib"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")) + .isEqualTo("librocksdbjni-osx.jnilib"); + assertThat(Environment.getSharedLibraryFileName("rocksdb")). + isEqualTo("librocksdbjni.dylib"); + } + + @Test + public void macAarch64() { + setEnvironmentClassFields("mac", "aarch64"); + assertThat(Environment.isWindows()).isFalse(); + assertThat(Environment.getJniLibraryExtension()).isEqualTo(".jnilib"); + assertThat(Environment.getJniLibraryFileName("rocksdb")) + .isEqualTo("librocksdbjni-osx-arm64.jnilib"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")) + .isEqualTo("librocksdbjni-osx.jnilib"); + assertThat(Environment.getSharedLibraryFileName("rocksdb")).isEqualTo("librocksdbjni.dylib"); + } + + @Test + public void nix32() { + // Linux + setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, false); + setEnvironmentClassFields("Linux", "32"); + assertThat(Environment.isWindows()).isFalse(); + assertThat(Environment.getJniLibraryExtension()). + isEqualTo(".so"); + assertThat(Environment.getJniLibraryFileName("rocksdb")). + isEqualTo("librocksdbjni-linux32.so"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); + assertThat(Environment.getSharedLibraryFileName("rocksdb")). + isEqualTo("librocksdbjni.so"); + // Linux musl-libc (Alpine) + setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, true); + assertThat(Environment.isWindows()).isFalse(); + assertThat(Environment.getJniLibraryExtension()). + isEqualTo(".so"); + assertThat(Environment.getJniLibraryFileName("rocksdb")). + isEqualTo("librocksdbjni-linux32-musl.so"); + assertThat(Environment.getSharedLibraryFileName("rocksdb")). + isEqualTo("librocksdbjni.so"); + // UNIX + setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, false); + setEnvironmentClassFields("Unix", "32"); + assertThat(Environment.isWindows()).isFalse(); + assertThat(Environment.getJniLibraryExtension()). + isEqualTo(".so"); + assertThat(Environment.getJniLibraryFileName("rocksdb")). + isEqualTo("librocksdbjni-linux32.so"); + assertThat(Environment.getSharedLibraryFileName("rocksdb")). + isEqualTo("librocksdbjni.so"); + } + + @Test(expected = UnsupportedOperationException.class) + public void aix32() { + // AIX + setEnvironmentClassFields("aix", "32"); + assertThat(Environment.isWindows()).isFalse(); + assertThat(Environment.getJniLibraryExtension()). + isEqualTo(".so"); + assertThat(Environment.getJniLibraryFileName("rocksdb")).isEqualTo("blah"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); + } + + @Test + public void nix64() { + setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, false); + setEnvironmentClassFields("Linux", "x64"); + assertThat(Environment.isWindows()).isFalse(); + assertThat(Environment.getJniLibraryExtension()). + isEqualTo(".so"); + assertThat(Environment.getJniLibraryFileName("rocksdb")). + isEqualTo("librocksdbjni-linux64.so"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); + assertThat(Environment.getSharedLibraryFileName("rocksdb")). + isEqualTo("librocksdbjni.so"); + // Linux musl-libc (Alpine) + setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, true); + assertThat(Environment.isWindows()).isFalse(); + assertThat(Environment.getJniLibraryExtension()). + isEqualTo(".so"); + assertThat(Environment.getJniLibraryFileName("rocksdb")). + isEqualTo("librocksdbjni-linux64-musl.so"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); + assertThat(Environment.getSharedLibraryFileName("rocksdb")). + isEqualTo("librocksdbjni.so"); + // UNIX + setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, false); + setEnvironmentClassFields("Unix", "x64"); + assertThat(Environment.isWindows()).isFalse(); + assertThat(Environment.getJniLibraryExtension()). + isEqualTo(".so"); + assertThat(Environment.getJniLibraryFileName("rocksdb")). + isEqualTo("librocksdbjni-linux64.so"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); + assertThat(Environment.getSharedLibraryFileName("rocksdb")). + isEqualTo("librocksdbjni.so"); + // AIX + setEnvironmentClassFields("aix", "x64"); + assertThat(Environment.isWindows()).isFalse(); + assertThat(Environment.getJniLibraryExtension()). + isEqualTo(".so"); + assertThat(Environment.getJniLibraryFileName("rocksdb")). + isEqualTo("librocksdbjni-aix64.so"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); + assertThat(Environment.getSharedLibraryFileName("rocksdb")). + isEqualTo("librocksdbjni.so"); + } + + @Test + public void detectWindows(){ + setEnvironmentClassFields("win", "x64"); + assertThat(Environment.isWindows()).isTrue(); + } + + @Test + public void win64() { + setEnvironmentClassFields("win", "x64"); + assertThat(Environment.isWindows()).isTrue(); + assertThat(Environment.getJniLibraryExtension()). + isEqualTo(".dll"); + assertThat(Environment.getJniLibraryFileName("rocksdb")). + isEqualTo("librocksdbjni-win64.dll"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); + assertThat(Environment.getSharedLibraryFileName("rocksdb")). + isEqualTo("librocksdbjni.dll"); + } + + @Test + public void ppc64le() { + setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, false); + setEnvironmentClassFields("Linux", "ppc64le"); + assertThat(Environment.isUnix()).isTrue(); + assertThat(Environment.isPowerPC()).isTrue(); + assertThat(Environment.is64Bit()).isTrue(); + assertThat(Environment.getJniLibraryExtension()).isEqualTo(".so"); + assertThat(Environment.getSharedLibraryName("rocksdb")).isEqualTo("rocksdbjni"); + assertThat(Environment.getJniLibraryName("rocksdb")).isEqualTo("rocksdbjni-linux-ppc64le"); + assertThat(Environment.getJniLibraryFileName("rocksdb")) + .isEqualTo("librocksdbjni-linux-ppc64le.so"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); + assertThat(Environment.getSharedLibraryFileName("rocksdb")).isEqualTo("librocksdbjni.so"); + // Linux musl-libc (Alpine) + setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, true); + setEnvironmentClassFields("Linux", "ppc64le"); + assertThat(Environment.isUnix()).isTrue(); + assertThat(Environment.isPowerPC()).isTrue(); + assertThat(Environment.is64Bit()).isTrue(); + assertThat(Environment.getJniLibraryExtension()).isEqualTo(".so"); + assertThat(Environment.getSharedLibraryName("rocksdb")).isEqualTo("rocksdbjni"); + assertThat(Environment.getJniLibraryName("rocksdb")).isEqualTo("rocksdbjni-linux-ppc64le-musl"); + assertThat(Environment.getJniLibraryFileName("rocksdb")) + .isEqualTo("librocksdbjni-linux-ppc64le-musl.so"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); + assertThat(Environment.getSharedLibraryFileName("rocksdb")).isEqualTo("librocksdbjni.so"); + setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, false); + } + + @Test + public void linuxArch64() { + setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, false); + setEnvironmentClassFields("Linux", "aarch64"); + assertThat(Environment.isUnix()).isTrue(); + assertThat(Environment.isAarch64()).isTrue(); + assertThat(Environment.is64Bit()).isTrue(); + assertThat(Environment.getJniLibraryExtension()).isEqualTo(".so"); + assertThat(Environment.getSharedLibraryName("rocksdb")).isEqualTo("rocksdbjni"); + assertThat(Environment.getJniLibraryName("rocksdb")).isEqualTo("rocksdbjni-linux-aarch64"); + assertThat(Environment.getJniLibraryFileName("rocksdb")) + .isEqualTo("librocksdbjni-linux-aarch64.so"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); + assertThat(Environment.getSharedLibraryFileName("rocksdb")).isEqualTo("librocksdbjni.so"); + // Linux musl-libc (Alpine) + setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, true); + setEnvironmentClassFields("Linux", "aarch64"); + assertThat(Environment.isUnix()).isTrue(); + assertThat(Environment.isAarch64()).isTrue(); + assertThat(Environment.is64Bit()).isTrue(); + assertThat(Environment.getJniLibraryExtension()).isEqualTo(".so"); + assertThat(Environment.getSharedLibraryName("rocksdb")).isEqualTo("rocksdbjni"); + assertThat(Environment.getJniLibraryName("rocksdb")).isEqualTo("rocksdbjni-linux-aarch64-musl"); + assertThat(Environment.getJniLibraryFileName("rocksdb")) + .isEqualTo("librocksdbjni-linux-aarch64-musl.so"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); + assertThat(Environment.getSharedLibraryFileName("rocksdb")).isEqualTo("librocksdbjni.so"); + setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, false); + } + + @Test + public void resolveIsMuslLibc() { + setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, null); + setEnvironmentClassFields("win", "anyarch"); + assertThat(Environment.isUnix()).isFalse(); + + // with user input, will resolve to true if set as true. Even on OSs that appear absurd for + // musl. Users choice + assertThat(Environment.initIsMuslLibc()).isFalse(); + setEnvironmentClassField(MUSL_ENVIRONMENT_FIELD_NAME, "true"); + assertThat(Environment.initIsMuslLibc()).isTrue(); + setEnvironmentClassField(MUSL_ENVIRONMENT_FIELD_NAME, "false"); + assertThat(Environment.initIsMuslLibc()).isFalse(); + } + + private void setEnvironmentClassFields(String osName, + String osArch) { + setEnvironmentClassField(OS_FIELD_NAME, osName); + setEnvironmentClassField(ARCH_FIELD_NAME, osArch); + } + + @AfterClass + public static void restoreState() { + setEnvironmentClassField(OS_FIELD_NAME, INITIAL_OS); + setEnvironmentClassField(ARCH_FIELD_NAME, INITIAL_ARCH); + setEnvironmentClassField(MUSL_ENVIRONMENT_FIELD_NAME, INITIAL_MUSL_ENVIRONMENT); + setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, INITIAL_MUSL_LIBC); + } + + @SuppressWarnings("unchecked") + private static T getEnvironmentClassField(String fieldName) { + final Field field; + try { + field = Environment.class.getDeclaredField(fieldName); + field.setAccessible(true); + /* Fails in JDK 13; and not needed unless fields are final + final Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + */ + return (T)field.get(null); + } catch (final NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + private static void setEnvironmentClassField(String fieldName, Object value) { + final Field field; + try { + field = Environment.class.getDeclaredField(fieldName); + field.setAccessible(true); + /* Fails in JDK 13; and not needed unless fields are final + final Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + */ + field.set(null, value); + } catch (final NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/HeapByteBufferAllocator.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/HeapByteBufferAllocator.java new file mode 100644 index 0000000..ad6b8f6 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/HeapByteBufferAllocator.java @@ -0,0 +1,18 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb.util; + +import java.nio.ByteBuffer; + +public final class HeapByteBufferAllocator implements ByteBufferAllocator { + HeapByteBufferAllocator(){}; + + @Override + public ByteBuffer allocate(final int capacity) { + return ByteBuffer.allocate(capacity); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/IntComparatorTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/IntComparatorTest.java new file mode 100644 index 0000000..dd32885 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/IntComparatorTest.java @@ -0,0 +1,266 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb.util; + +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; +import org.rocksdb.*; + +import java.nio.ByteBuffer; +import java.nio.file.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Tests for IntComparator, but more generally + * also for rocksdb::ComparatorJniCallback implementation. + */ +@RunWith(Parameterized.class) +public class IntComparatorTest { + + // test with 500 random integer keys + private static final int TOTAL_KEYS = 500; + private static final byte[][] keys = new byte[TOTAL_KEYS][4]; + + @BeforeClass + public static void prepareKeys() { + final ByteBuffer buf = ByteBuffer.allocate(4); + final Random random = new Random(); + for (int i = 0; i < TOTAL_KEYS; i++) { + final int ri = random.nextInt(); + buf.putInt(ri); + buf.flip(); + final byte[] key = buf.array(); + + // does key already exist (avoid duplicates) + if (keyExists(key, i)) { + i--; // loop round and generate a different key + } else { + System.arraycopy(key, 0, keys[i], 0, 4); + } + } + } + + private static boolean keyExists(final byte[] key, final int limit) { + for (int j = 0; j < limit; j++) { + if (Arrays.equals(key, keys[j])) { + return true; + } + } + return false; + } + + @Parameters(name = "{0}") + public static Iterable parameters() { + return Arrays.asList(new Object[][] { + { "non-direct_reused64_mutex", false, 64, ReusedSynchronisationType.MUTEX }, + { "direct_reused64_mutex", true, 64, ReusedSynchronisationType.MUTEX }, + { "non-direct_reused64_adaptive-mutex", false, 64, ReusedSynchronisationType.ADAPTIVE_MUTEX }, + { "direct_reused64_adaptive-mutex", true, 64, ReusedSynchronisationType.ADAPTIVE_MUTEX }, + { "non-direct_reused64_thread-local", false, 64, ReusedSynchronisationType.THREAD_LOCAL }, + { "direct_reused64_thread-local", true, 64, ReusedSynchronisationType.THREAD_LOCAL }, + { "non-direct_noreuse", false, -1, null }, + { "direct_noreuse", true, -1, null } + }); + } + + @Parameter(0) + public String name; + + @Parameter(1) + public boolean useDirectBuffer; + + @Parameter(2) + public int maxReusedBufferSize; + + @Parameter(3) + public ReusedSynchronisationType reusedSynchronisationType; + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + + @Test + public void javaComparatorDefaultCf() throws RocksDBException { + try (final ComparatorOptions options = new ComparatorOptions() + .setUseDirectBuffer(useDirectBuffer) + .setMaxReusedBufferSize(maxReusedBufferSize) + // if reusedSynchronisationType == null we assume that maxReusedBufferSize <= 0 and so we just set ADAPTIVE_MUTEX, even though it won't be used + .setReusedSynchronisationType(reusedSynchronisationType == null ? ReusedSynchronisationType.ADAPTIVE_MUTEX : reusedSynchronisationType); + final IntComparator comparator = new IntComparator(options)) { + + // test the round-tripability of keys written and read with the Comparator + testRoundtrip(FileSystems.getDefault().getPath( + dbFolder.getRoot().getAbsolutePath()), comparator); + } + } + + @Test + public void javaComparatorNamedCf() throws RocksDBException { + try (final ComparatorOptions options = new ComparatorOptions() + .setUseDirectBuffer(useDirectBuffer) + .setMaxReusedBufferSize(maxReusedBufferSize) + // if reusedSynchronisationType == null we assume that maxReusedBufferSize <= 0 and so we just set ADAPTIVE_MUTEX, even though it won't be used + .setReusedSynchronisationType(reusedSynchronisationType == null ? ReusedSynchronisationType.ADAPTIVE_MUTEX : reusedSynchronisationType); + final IntComparator comparator = new IntComparator(options)) { + + // test the round-tripability of keys written and read with the Comparator + testRoundtripCf(FileSystems.getDefault().getPath( + dbFolder.getRoot().getAbsolutePath()), comparator); + } + } + + /** + * Test which stores random keys into the database + * using an {@link IntComparator} + * it then checks that these keys are read back in + * ascending order + * + * @param db_path A path where we can store database + * files temporarily + * + * @param comparator the comparator + * + * @throws RocksDBException if a database error happens. + */ + private void testRoundtrip(final Path db_path, + final AbstractComparator comparator) throws RocksDBException { + try (final Options opt = new Options() + .setCreateIfMissing(true) + .setComparator(comparator)) { + + // store TOTAL_KEYS into the db + try (final RocksDB db = RocksDB.open(opt, db_path.toString())) { + for (int i = 0; i < TOTAL_KEYS; i++) { + db.put(keys[i], "value".getBytes(UTF_8)); + } + } + + // re-open db and read from start to end + // integer keys should be in ascending + // order as defined by IntComparator + final ByteBuffer key = ByteBuffer.allocate(4); + try (final RocksDB db = RocksDB.open(opt, db_path.toString()); + final RocksIterator it = db.newIterator()) { + it.seekToFirst(); + int lastKey = Integer.MIN_VALUE; + int count = 0; + for (it.seekToFirst(); it.isValid(); it.next()) { + key.put(it.key()); + key.flip(); + final int thisKey = key.getInt(); + key.clear(); + assertThat(thisKey).isGreaterThan(lastKey); + lastKey = thisKey; + count++; + } + assertThat(count).isEqualTo(TOTAL_KEYS); + } + } + } + + /** + * Test which stores random keys into a column family + * in the database + * using an {@link IntComparator} + * it then checks that these keys are read back in + * ascending order + * + * @param db_path A path where we can store database + * files temporarily + * + * @param comparator the comparator + * + * @throws RocksDBException if a database error happens. + */ + private void testRoundtripCf(final Path db_path, + final AbstractComparator comparator) throws RocksDBException { + + final List cfDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes(), + new ColumnFamilyOptions() + .setComparator(comparator)) + ); + + final List cfHandles = new ArrayList<>(); + + try (final DBOptions opt = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true)) { + + try (final RocksDB db = RocksDB.open(opt, db_path.toString(), + cfDescriptors, cfHandles)) { + try { + assertThat(cfDescriptors.size()).isEqualTo(2); + assertThat(cfHandles.size()).isEqualTo(2); + + for (int i = 0; i < TOTAL_KEYS; i++) { + db.put(cfHandles.get(1), keys[i], "value".getBytes(UTF_8)); + } + } finally { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + cfHandles.clear(); + } + } + + // re-open db and read from start to end + // integer keys should be in ascending + // order as defined by SimpleIntComparator + final ByteBuffer key = ByteBuffer.allocate(4); + try (final RocksDB db = RocksDB.open(opt, db_path.toString(), + cfDescriptors, cfHandles); + final RocksIterator it = db.newIterator(cfHandles.get(1))) { + try { + assertThat(cfDescriptors.size()).isEqualTo(2); + assertThat(cfHandles.size()).isEqualTo(2); + + it.seekToFirst(); + int lastKey = Integer.MIN_VALUE; + int count = 0; + for (it.seekToFirst(); it.isValid(); it.next()) { + key.put(it.key()); + key.flip(); + final int thisKey = key.getInt(); + key.clear(); + assertThat(thisKey).isGreaterThan(lastKey); + lastKey = thisKey; + count++; + } + + assertThat(count).isEqualTo(TOTAL_KEYS); + + } finally { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + cfHandles.clear(); + for (final ColumnFamilyDescriptor cfDescriptor : cfDescriptors) { + cfDescriptor.getOptions().close(); + } + } + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/JNIComparatorTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/JNIComparatorTest.java new file mode 100644 index 0000000..a962b8d --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/JNIComparatorTest.java @@ -0,0 +1,180 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb.util; + +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; +import org.rocksdb.*; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.*; +import java.util.Arrays; + +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(Parameterized.class) +public class JNIComparatorTest { + + @Parameters(name = "{0}") + public static Iterable parameters() { + return Arrays.asList(new Object[][] { + { "bytewise_non-direct", BuiltinComparator.BYTEWISE_COMPARATOR, false }, + { "bytewise_direct", BuiltinComparator.BYTEWISE_COMPARATOR, true }, + { "reverse-bytewise_non-direct", BuiltinComparator.REVERSE_BYTEWISE_COMPARATOR, false }, + { "reverse-bytewise_direct", BuiltinComparator.REVERSE_BYTEWISE_COMPARATOR, true }, + }); + } + + @Parameter(0) + public String name; + + @Parameter(1) + public BuiltinComparator builtinComparator; + + @Parameter(2) + public boolean useDirectBuffer; + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + private static final int MIN = Short.MIN_VALUE - 1; + private static final int MAX = Short.MAX_VALUE + 1; + + @Test + public void java_comparator_equals_cpp_comparator() throws RocksDBException, IOException { + final int[] javaKeys; + try (final ComparatorOptions comparatorOptions = new ComparatorOptions(); + final AbstractComparator comparator = builtinComparator == BuiltinComparator.BYTEWISE_COMPARATOR + ? new BytewiseComparator(comparatorOptions) + : new ReverseBytewiseComparator(comparatorOptions)) { + final Path javaDbDir = + FileSystems.getDefault().getPath(dbFolder.newFolder().getAbsolutePath()); + storeWithJavaComparator(javaDbDir, comparator); + javaKeys = readAllWithJavaComparator(javaDbDir, comparator); + } + + final Path cppDbDir = + FileSystems.getDefault().getPath(dbFolder.newFolder().getAbsolutePath()); + storeWithCppComparator(cppDbDir, builtinComparator); + final int[] cppKeys = + readAllWithCppComparator(cppDbDir, builtinComparator); + + assertThat(javaKeys).isEqualTo(cppKeys); + } + + private void storeWithJavaComparator(final Path dir, + final AbstractComparator comparator) throws RocksDBException { + final ByteBuffer buf = ByteBuffer.allocate(4); + try (final Options options = new Options() + .setCreateIfMissing(true) + .setComparator(comparator); + final RocksDB db = + RocksDB.open(options, dir.toAbsolutePath().toString())) { + for (int i = MIN; i < MAX; i++) { + buf.putInt(i); + buf.flip(); + + db.put(buf.array(), buf.array()); + + buf.clear(); + } + } + } + + private void storeWithCppComparator(final Path dir, + final BuiltinComparator builtinComparator) throws RocksDBException { + try (final Options options = new Options() + .setCreateIfMissing(true) + .setComparator(builtinComparator); + final RocksDB db = + RocksDB.open(options, dir.toAbsolutePath().toString())) { + + final ByteBuffer buf = ByteBuffer.allocate(4); + for (int i = MIN; i < MAX; i++) { + buf.putInt(i); + buf.flip(); + + db.put(buf.array(), buf.array()); + + buf.clear(); + } + } + } + + private int[] readAllWithJavaComparator(final Path dir, + final AbstractComparator comparator) throws RocksDBException { + try (final Options options = new Options() + .setCreateIfMissing(true) + .setComparator(comparator); + final RocksDB db = + RocksDB.open(options, dir.toAbsolutePath().toString())) { + + try (final RocksIterator it = db.newIterator()) { + it.seekToFirst(); + + final ByteBuffer buf = ByteBuffer.allocate(4); + final int[] keys = new int[MAX - MIN]; + int idx = 0; + while (it.isValid()) { + buf.put(it.key()); + buf.flip(); + + final int thisKey = buf.getInt(); + keys[idx++] = thisKey; + + buf.clear(); + + it.next(); + } + + return keys; + } + } + } + + private int[] readAllWithCppComparator(final Path dir, + final BuiltinComparator comparator) throws RocksDBException { + try (final Options options = new Options() + .setCreateIfMissing(true) + .setComparator(comparator); + final RocksDB db = + RocksDB.open(options, dir.toAbsolutePath().toString())) { + + try (final RocksIterator it = db.newIterator()) { + it.seekToFirst(); + + final ByteBuffer buf = ByteBuffer.allocate(4); + final int[] keys = new int[MAX - MIN]; + int idx = 0; + while (it.isValid()) { + buf.put(it.key()); + buf.flip(); + + final int thisKey = buf.getInt(); + keys[idx++] = thisKey; + + buf.clear(); + + it.next(); + } + + return keys; + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/ReverseBytewiseComparatorIntTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/ReverseBytewiseComparatorIntTest.java new file mode 100644 index 0000000..ca08d9d --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/ReverseBytewiseComparatorIntTest.java @@ -0,0 +1,270 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb.util; + +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; +import org.rocksdb.*; + +import java.nio.ByteBuffer; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Similar to {@link IntComparatorTest}, but uses + * {@link ReverseBytewiseComparator} which ensures the correct reverse + * ordering of positive integers. + */ +@RunWith(Parameterized.class) +public class ReverseBytewiseComparatorIntTest { + + // test with 500 random positive integer keys + private static final int TOTAL_KEYS = 500; + private static final byte[][] keys = new byte[TOTAL_KEYS][4]; + + @BeforeClass + public static void prepareKeys() { + final ByteBuffer buf = ByteBuffer.allocate(4); + final Random random = new Random(); + for (int i = 0; i < TOTAL_KEYS; i++) { + final int ri = random.nextInt() & Integer.MAX_VALUE; // the & ensures positive integer + buf.putInt(ri); + buf.flip(); + final byte[] key = buf.array(); + + // does key already exist (avoid duplicates) + if (keyExists(key, i)) { + i--; // loop round and generate a different key + } else { + System.arraycopy(key, 0, keys[i], 0, 4); + } + } + } + + private static boolean keyExists(final byte[] key, final int limit) { + for (int j = 0; j < limit; j++) { + if (Arrays.equals(key, keys[j])) { + return true; + } + } + return false; + } + + @Parameters(name = "{0}") + public static Iterable parameters() { + return Arrays.asList(new Object[][] { + { "non-direct_reused64_mutex", false, 64, ReusedSynchronisationType.MUTEX }, + { "direct_reused64_adaptive-mutex", true, 64, ReusedSynchronisationType.MUTEX }, + { "non-direct_reused64_adaptive-mutex", false, 64, ReusedSynchronisationType.ADAPTIVE_MUTEX }, + { "direct_reused64_adaptive-mutex", true, 64, ReusedSynchronisationType.ADAPTIVE_MUTEX }, + { "non-direct_reused64_adaptive-mutex", false, 64, ReusedSynchronisationType.THREAD_LOCAL }, + { "direct_reused64_adaptive-mutex", true, 64, ReusedSynchronisationType.THREAD_LOCAL }, + { "non-direct_noreuse", false, -1, null }, + { "direct_noreuse", true, -1, null } + }); + } + + @Parameter(0) + public String name; + + @Parameter(1) + public boolean useDirectBuffer; + + @Parameter(2) + public int maxReusedBufferSize; + + @Parameter(3) + public ReusedSynchronisationType reusedSynchronisationType; + + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule + public TemporaryFolder dbFolder = new TemporaryFolder(); + + + @Test + public void javaComparatorDefaultCf() throws RocksDBException { + try (final ComparatorOptions options = new ComparatorOptions() + .setUseDirectBuffer(useDirectBuffer) + .setMaxReusedBufferSize(maxReusedBufferSize) + // if reusedSynchronisationType == null we assume that maxReusedBufferSize <= 0 and so we just set ADAPTIVE_MUTEX, even though it won't be used + .setReusedSynchronisationType(reusedSynchronisationType == null ? ReusedSynchronisationType.ADAPTIVE_MUTEX : reusedSynchronisationType); + final ReverseBytewiseComparator comparator = + new ReverseBytewiseComparator(options)) { + + // test the round-tripability of keys written and read with the Comparator + testRoundtrip(FileSystems.getDefault().getPath( + dbFolder.getRoot().getAbsolutePath()), comparator); + } + } + + @Test + public void javaComparatorNamedCf() throws RocksDBException { + try (final ComparatorOptions options = new ComparatorOptions() + .setUseDirectBuffer(useDirectBuffer) + .setMaxReusedBufferSize(maxReusedBufferSize) + // if reusedSynchronisationType == null we assume that maxReusedBufferSize <= 0 and so we just set ADAPTIVE_MUTEX, even though it won't be used + .setReusedSynchronisationType(reusedSynchronisationType == null ? ReusedSynchronisationType.ADAPTIVE_MUTEX : reusedSynchronisationType); + final ReverseBytewiseComparator comparator + = new ReverseBytewiseComparator(options)) { + + // test the round-tripability of keys written and read with the Comparator + testRoundtripCf(FileSystems.getDefault().getPath( + dbFolder.getRoot().getAbsolutePath()), comparator); + } + } + + /** + * Test which stores random keys into the database + * using an {@link IntComparator} + * it then checks that these keys are read back in + * ascending order + * + * @param db_path A path where we can store database + * files temporarily + * + * @param comparator the comparator + * + * @throws RocksDBException if a database error happens. + */ + private void testRoundtrip(final Path db_path, + final AbstractComparator comparator) throws RocksDBException { + try (final Options opt = new Options() + .setCreateIfMissing(true) + .setComparator(comparator)) { + + // store TOTAL_KEYS into the db + try (final RocksDB db = RocksDB.open(opt, db_path.toString())) { + for (int i = 0; i < TOTAL_KEYS; i++) { + db.put(keys[i], "value".getBytes(UTF_8)); + } + } + + // re-open db and read from start to end + // integer keys should be in descending + // order + final ByteBuffer key = ByteBuffer.allocate(4); + try (final RocksDB db = RocksDB.open(opt, db_path.toString()); + final RocksIterator it = db.newIterator()) { + it.seekToFirst(); + int lastKey = Integer.MAX_VALUE; + int count = 0; + for (it.seekToFirst(); it.isValid(); it.next()) { + key.put(it.key()); + key.flip(); + final int thisKey = key.getInt(); + key.clear(); + assertThat(thisKey).isLessThan(lastKey); + lastKey = thisKey; + count++; + } + assertThat(count).isEqualTo(TOTAL_KEYS); + } + } + } + + /** + * Test which stores random keys into a column family + * in the database + * using an {@link IntComparator} + * it then checks that these keys are read back in + * ascending order + * + * @param db_path A path where we can store database + * files temporarily + * + * @param comparator the comparator + * + * @throws RocksDBException if a database error happens. + */ + private void testRoundtripCf(final Path db_path, + final AbstractComparator comparator) throws RocksDBException { + + final List cfDescriptors = Arrays.asList( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), + new ColumnFamilyDescriptor("new_cf".getBytes(), + new ColumnFamilyOptions() + .setComparator(comparator)) + ); + + final List cfHandles = new ArrayList<>(); + + try (final DBOptions opt = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true)) { + + try (final RocksDB db = RocksDB.open(opt, db_path.toString(), + cfDescriptors, cfHandles)) { + try { + assertThat(cfDescriptors.size()).isEqualTo(2); + assertThat(cfHandles.size()).isEqualTo(2); + + for (int i = 0; i < TOTAL_KEYS; i++) { + db.put(cfHandles.get(1), keys[i], "value".getBytes(UTF_8)); + } + } finally { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + cfHandles.clear(); + } + } + + // re-open db and read from start to end + // integer keys should be in descending + // order + final ByteBuffer key = ByteBuffer.allocate(4); + try (final RocksDB db = RocksDB.open(opt, db_path.toString(), + cfDescriptors, cfHandles); + final RocksIterator it = db.newIterator(cfHandles.get(1))) { + try { + assertThat(cfDescriptors.size()).isEqualTo(2); + assertThat(cfHandles.size()).isEqualTo(2); + + it.seekToFirst(); + int lastKey = Integer.MAX_VALUE; + int count = 0; + for (it.seekToFirst(); it.isValid(); it.next()) { + key.put(it.key()); + key.flip(); + final int thisKey = key.getInt(); + key.clear(); + assertThat(thisKey).isLessThan(lastKey); + lastKey = thisKey; + count++; + } + + assertThat(count).isEqualTo(TOTAL_KEYS); + + } finally { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + cfHandles.clear(); + for (final ColumnFamilyDescriptor cfDescriptor : cfDescriptors) { + cfDescriptor.getOptions().close(); + } + } + } + } + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/SizeUnitTest.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/SizeUnitTest.java new file mode 100644 index 0000000..990aa5f --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/SizeUnitTest.java @@ -0,0 +1,27 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb.util; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class SizeUnitTest { + + public static final long COMPUTATION_UNIT = 1024L; + + @Test + public void sizeUnit() { + assertThat(SizeUnit.KB).isEqualTo(COMPUTATION_UNIT); + assertThat(SizeUnit.MB).isEqualTo( + SizeUnit.KB * COMPUTATION_UNIT); + assertThat(SizeUnit.GB).isEqualTo( + SizeUnit.MB * COMPUTATION_UNIT); + assertThat(SizeUnit.TB).isEqualTo( + SizeUnit.GB * COMPUTATION_UNIT); + assertThat(SizeUnit.PB).isEqualTo( + SizeUnit.TB * COMPUTATION_UNIT); + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/TestUtil.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/TestUtil.java new file mode 100644 index 0000000..e4f490c --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/TestUtil.java @@ -0,0 +1,72 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb.util; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.nio.ByteBuffer; +import java.util.Random; +import org.rocksdb.CompactionPriority; +import org.rocksdb.Options; +import org.rocksdb.WALRecoveryMode; + +/** + * General test utilities. + */ +public class TestUtil { + + /** + * Get the options for log iteration tests. + * + * @return the options + */ + public static Options optionsForLogIterTest() { + return defaultOptions() + .setCreateIfMissing(true) + .setWalTtlSeconds(1000); + } + + /** + * Get the default options. + * + * @return the options + */ + public static Options defaultOptions() { + return new Options() + .setWriteBufferSize(4090 * 4096) + .setTargetFileSizeBase(2 * 1024 * 1024) + .setMaxBytesForLevelBase(10 * 1024 * 1024) + .setMaxOpenFiles(5000) + .setWalRecoveryMode(WALRecoveryMode.TolerateCorruptedTailRecords) + .setCompactionPriority(CompactionPriority.ByCompensatedSize); + } + + private static final Random random = new Random(); + + /** + * Generate a random string of bytes. + * + * @param len the length of the string to generate. + * + * @return the random string of bytes + */ + public static byte[] dummyString(final int len) { + final byte[] str = new byte[len]; + random.nextBytes(str); + return str; + } + + /** + * Copy a {@link ByteBuffer} into an array for shorthand ease of test coding + * @param byteBuffer the buffer to copy + * @return a {@link byte[]} containing the same bytes as the input + */ + public static byte[] bufferBytes(final ByteBuffer byteBuffer) { + final byte[] result = new byte[byteBuffer.limit()]; + byteBuffer.get(result); + return result; + } +} diff --git a/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/WriteBatchGetter.java b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/WriteBatchGetter.java new file mode 100644 index 0000000..2efa164 --- /dev/null +++ b/librocksdb-sys/rocksdb/java/src/test/java/org/rocksdb/util/WriteBatchGetter.java @@ -0,0 +1,139 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +package org.rocksdb.util; + +import org.rocksdb.RocksDBException; +import org.rocksdb.WriteBatch; + +import java.util.Arrays; + +public class WriteBatchGetter extends WriteBatch.Handler { + + private int columnFamilyId = -1; + private final byte[] key; + private byte[] value; + + public WriteBatchGetter(final byte[] key) { + this.key = key; + } + + public byte[] getValue() { + return value; + } + + @Override + public void put(final int columnFamilyId, final byte[] key, + final byte[] value) { + if(Arrays.equals(this.key, key)) { + this.columnFamilyId = columnFamilyId; + this.value = value; + } + } + + @Override + public void put(final byte[] key, final byte[] value) { + if(Arrays.equals(this.key, key)) { + this.value = value; + } + } + + @Override + public void merge(final int columnFamilyId, final byte[] key, + final byte[] value) { + if(Arrays.equals(this.key, key)) { + this.columnFamilyId = columnFamilyId; + this.value = value; + } + } + + @Override + public void merge(final byte[] key, final byte[] value) { + if(Arrays.equals(this.key, key)) { + this.value = value; + } + } + + @Override + public void delete(final int columnFamilyId, final byte[] key) { + if(Arrays.equals(this.key, key)) { + this.columnFamilyId = columnFamilyId; + this.value = null; + } + } + + @Override + public void delete(final byte[] key) { + if(Arrays.equals(this.key, key)) { + this.value = null; + } + } + + @Override + public void singleDelete(final int columnFamilyId, final byte[] key) { + if(Arrays.equals(this.key, key)) { + this.columnFamilyId = columnFamilyId; + this.value = null; + } + } + + @Override + public void singleDelete(final byte[] key) { + if(Arrays.equals(this.key, key)) { + this.value = null; + } + } + + @Override + public void deleteRange(final int columnFamilyId, final byte[] beginKey, + final byte[] endKey) { + throw new UnsupportedOperationException(); + } + + @Override + public void deleteRange(final byte[] beginKey, final byte[] endKey) { + throw new UnsupportedOperationException(); + } + + @Override + public void logData(final byte[] blob) { + throw new UnsupportedOperationException(); + } + + @Override + public void putBlobIndex(final int columnFamilyId, final byte[] key, + final byte[] value) { + if(Arrays.equals(this.key, key)) { + this.columnFamilyId = columnFamilyId; + this.value = value; + } + } + + @Override + public void markBeginPrepare() throws RocksDBException { + throw new UnsupportedOperationException(); + } + + @Override + public void markEndPrepare(final byte[] xid) throws RocksDBException { + throw new UnsupportedOperationException(); + } + + @Override + public void markNoop(final boolean emptyBatch) throws RocksDBException { + throw new UnsupportedOperationException(); + } + + @Override + public void markRollback(final byte[] xid) throws RocksDBException { + throw new UnsupportedOperationException(); + } + + @Override + public void markCommit(final byte[] xid) throws RocksDBException { + throw new UnsupportedOperationException(); + } + + @Override + public void markCommitWithTimestamp(final byte[] xid, final byte[] ts) throws RocksDBException { + throw new UnsupportedOperationException(); + } +} diff --git a/librocksdb-sys/rocksdb/java/understanding_options.md b/librocksdb-sys/rocksdb/java/understanding_options.md new file mode 100644 index 0000000..0393aff --- /dev/null +++ b/librocksdb-sys/rocksdb/java/understanding_options.md @@ -0,0 +1,79 @@ +# How RocksDB Options and their Java Wrappers Work + +Options in RocksDB come in many different flavours. This is an attempt at a taxonomy and explanation. + +## RocksDB Options + +Initially, I believe, RocksDB had only database options. I don't know if any of these were mutable. Column families came later. Read on to understand the terminology. + +So to begin, one sets up a collection of options and starts/creates a database with these options. That's a useful way to think about it, because from a Java point-of-view (and I didn't realise this initially and got very confused), despite making native calls to C++, the `API`s are just manipulating a native C++ configuration object. This object is just a record of configuration, and it must later be passed to the database (at create or open time) in order to apply the options. + +### Database versus Column Family + +The concept of the *column family* or `CF` is widespread within RocksDB. I think of it as a data namespace, but conveniently transactions can operate across these namespaces. The concept of a default column family exists, and when operations do not refer to a particular `CF`, it refers to the default. + +We raise this w.r.t. options because many options, perhaps most that users encounter, are *column family options*. That is to say they apply individually to a particular column family, or to the default column family. Crucially also, many/most/all of these same options are exposed as *database options* and then apply as the default for column families which do not have the option set explicitly. Obviously some database options are naturally database-wide; they apply to the operation of the database and don't make any sense applied to a column family. + +### Mutability + +There are 2 kinds of options + +- Mutable options +- Immutable options. We name these in contrast to the mutable ones, but they are usually referred to unqualified. + +Mutable options are those which can be changed on a running `RocksDB` instance. Immutable options can only be configured prior to the start of a database. Of course, we can configure the immutable options at this time too; The entirety of options is a strict superset of the mutable options. + +Mutable options (whether column-family specific or database-wide) are manipulated at runtime with builders, so we have `MutableDBOptions.MutableDBOptionsBuilder` and `MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder` which share tooling classes/hierarchy and maintain and manipulate the relevant options as a `(key,value)` map. + +Mutable options are then passed using `setOptions()` and `setDBOptions()` methods on the live RocksDB, and then take effect immediately (depending on the semantics of the option) on the database. + +### Advanced + +There are 2 classes of options + +- Advanced options +- Non-advanced options + +It's not clear to me what the conceptual distinction is between advanced and not. However, the Java code takes care to reflect it from the underlying C++. + +This leads to 2 separate type hierarchies within column family options, one for each `class` of options. The `kind`s are represented by where the options appear in their hierarchy. + +```java +interface ColumnFamilyOptionsInterface> + extends AdvancedColumnFamilyOptionsInterface +interface MutableColumnFamilyOptionsInterface> + extends AdvancedMutableColumnFamilyOptionsInterface +``` + +And then there is ultimately a single concrete implementation class for CF options: + +```java +class ColumnFamilyOptions extends RocksObject + implements ColumnFamilyOptionsInterface, + MutableColumnFamilyOptionsInterface +``` + +as there is a single concrete implementation class for DB options: + +```java +class DBOptions extends RocksObject + implements DBOptionsInterface, + MutableDBOptionsInterface +``` + +Interestingly `DBOptionsInterface` doesn't extend `MutableDBOptionsInterface`, if only in order to disrupt our belief in consistent basic laws of the Universe. + +## Startup/Creation Options + +```java +class Options extends RocksObject + implements DBOptionsInterface, + MutableDBOptionsInterface, + ColumnFamilyOptionsInterface, + MutableColumnFamilyOptionsInterface +``` + +### Example - Blob Options + +The `enable_blob_files` and `min_blob_size` options are per-column-family, and are mutable. The options also appear in the unqualified database options. So by initial configuration, we can set up a RocksDB database where for every `(key,value)` with a value of size at least `min_blob_size`, the value is written (indirected) to a blob file. Blobs may share a blob file, subject to the configuration values set. Later, using the `MutableColumnFamilyOptionsInterface` of the `ColumnFamilyOptions`, we can choose to turn this off (`enable_blob_files=false`) , or alter the `min_blob_size` for the default column family, or any other column family. It seems to me that we cannot, though, mutate the column family options for all column families using the +`setOptions()` mechanism, either for all existing column families or for all future column families; but maybe we can do the latter on a re-`open()/create()' diff --git a/librocksdb-sys/rocksdb/logging/auto_roll_logger.cc b/librocksdb-sys/rocksdb/logging/auto_roll_logger.cc new file mode 100644 index 0000000..9e9ad45 --- /dev/null +++ b/librocksdb-sys/rocksdb/logging/auto_roll_logger.cc @@ -0,0 +1,368 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#include "logging/auto_roll_logger.h" + +#include + +#include "file/filename.h" +#include "logging/logging.h" +#include "rocksdb/env.h" +#include "rocksdb/file_system.h" +#include "rocksdb/system_clock.h" +#include "util/mutexlock.h" + +namespace ROCKSDB_NAMESPACE { + +// -- AutoRollLogger + +AutoRollLogger::AutoRollLogger(const std::shared_ptr& fs, + const std::shared_ptr& clock, + const std::string& dbname, + const std::string& db_log_dir, + size_t log_max_size, + size_t log_file_time_to_roll, + size_t keep_log_file_num, + const InfoLogLevel log_level) + : Logger(log_level), + dbname_(dbname), + db_log_dir_(db_log_dir), + fs_(fs), + clock_(clock), + status_(Status::OK()), + kMaxLogFileSize(log_max_size), + kLogFileTimeToRoll(log_file_time_to_roll), + kKeepLogFileNum(keep_log_file_num), + cached_now(static_cast(clock_->NowMicros() * 1e-6)), + ctime_(cached_now), + cached_now_access_count(0), + call_NowMicros_every_N_records_(100), + mutex_() { + Status s = fs->GetAbsolutePath(dbname, io_options_, &db_absolute_path_, + &io_context_); + if (s.IsNotSupported()) { + db_absolute_path_ = dbname; + } else { + status_ = s; + } + log_fname_ = InfoLogFileName(dbname_, db_absolute_path_, db_log_dir_); + if (fs_->FileExists(log_fname_, io_options_, &io_context_).ok()) { + RollLogFile(); + } + GetExistingFiles(); + s = ResetLogger(); + if (s.ok() && status_.ok()) { + status_ = TrimOldLogFiles(); + } +} + +Status AutoRollLogger::ResetLogger() { + TEST_SYNC_POINT("AutoRollLogger::ResetLogger:BeforeNewLogger"); + status_ = fs_->NewLogger(log_fname_, io_options_, &logger_, &io_context_); + TEST_SYNC_POINT("AutoRollLogger::ResetLogger:AfterNewLogger"); + + if (!status_.ok()) { + return status_; + } + assert(logger_); + logger_->SetInfoLogLevel(Logger::GetInfoLogLevel()); + + if (logger_->GetLogFileSize() == Logger::kDoNotSupportGetLogFileSize) { + status_ = Status::NotSupported( + "The underlying logger doesn't support GetLogFileSize()"); + } + if (status_.ok()) { + cached_now = static_cast(clock_->NowMicros() * 1e-6); + ctime_ = cached_now; + cached_now_access_count = 0; + } + + return status_; +} + +void AutoRollLogger::RollLogFile() { + // This function is called when log is rotating. Two rotations + // can happen quickly (NowMicro returns same value). To not overwrite + // previous log file we increment by one micro second and try again. + uint64_t now = clock_->NowMicros(); + std::string old_fname; + do { + old_fname = + OldInfoLogFileName(dbname_, now, db_absolute_path_, db_log_dir_); + now++; + } while (fs_->FileExists(old_fname, io_options_, &io_context_).ok()); + // Wait for logger_ reference count to turn to 1 as it might be pinned by + // Flush. Pinned Logger can't be closed till Flush is completed on that + // Logger. + while (logger_.use_count() > 1) { + } + // Close the existing logger first to release the existing handle + // before renaming the file using the file system. If this call + // fails there is nothing much we can do and we will continue with the + // rename and hence ignoring the result status. + if (logger_) { + logger_->Close().PermitUncheckedError(); + } + Status s = fs_->RenameFile(log_fname_, old_fname, io_options_, &io_context_); + if (!s.ok()) { + // What should we do on error? + } + old_log_files_.push(old_fname); +} + +void AutoRollLogger::GetExistingFiles() { + { + // Empty the queue to avoid duplicated entries in the queue. + std::queue empty; + std::swap(old_log_files_, empty); + } + + std::string parent_dir; + std::vector info_log_files; + Status s = + GetInfoLogFiles(fs_, db_log_dir_, dbname_, &parent_dir, &info_log_files); + if (status_.ok()) { + status_ = s; + } + // We need to sort the file before enqueing it so that when we + // delete file from the front, it is the oldest file. + std::sort(info_log_files.begin(), info_log_files.end()); + + for (const std::string& f : info_log_files) { + old_log_files_.push(parent_dir + "/" + f); + } +} + +Status AutoRollLogger::TrimOldLogFiles() { + // Here we directly list info files and delete them through FileSystem. + // The deletion isn't going through DB, so there are shortcomes: + // 1. the deletion is not rate limited by SstFileManager + // 2. there is a chance that an I/O will be issued here + // Since it's going to be complicated to pass DB object down to + // here, we take a simple approach to keep the code easier to + // maintain. + + // old_log_files_.empty() is helpful for the corner case that + // kKeepLogFileNum == 0. We can instead check kKeepLogFileNum != 0 but + // it's essentially the same thing, and checking empty before accessing + // the queue feels safer. + while (!old_log_files_.empty() && old_log_files_.size() >= kKeepLogFileNum) { + Status s = + fs_->DeleteFile(old_log_files_.front(), io_options_, &io_context_); + // Remove the file from the tracking anyway. It's possible that + // DB cleaned up the old log file, or people cleaned it up manually. + old_log_files_.pop(); + // To make the file really go away, we should sync parent directory. + // Since there isn't any consistency issue involved here, skipping + // this part to avoid one I/O here. + if (!s.ok()) { + return s; + } + } + return Status::OK(); +} + +std::string AutoRollLogger::ValistToString(const char* format, + va_list args) const { + // Any log messages longer than 1024 will get truncated. + // The user is responsible for chopping longer messages into multi line log + static const int MAXBUFFERSIZE = 1024; + char buffer[MAXBUFFERSIZE]; + + int count = vsnprintf(buffer, MAXBUFFERSIZE, format, args); + (void)count; + assert(count >= 0); + + return buffer; +} + +void AutoRollLogger::LogInternal(const char* format, ...) { + mutex_.AssertHeld(); + + if (!logger_) { + return; + } + + va_list args; + va_start(args, format); + logger_->Logv(format, args); + va_end(args); +} + +void AutoRollLogger::Logv(const char* format, va_list ap) { + assert(GetStatus().ok()); + if (!logger_) { + return; + } + + std::shared_ptr logger; + { + MutexLock l(&mutex_); + if ((kLogFileTimeToRoll > 0 && LogExpired()) || + (kMaxLogFileSize > 0 && logger_->GetLogFileSize() >= kMaxLogFileSize)) { + RollLogFile(); + Status s = ResetLogger(); + Status s2 = TrimOldLogFiles(); + + if (!s.ok()) { + // can't really log the error if creating a new LOG file failed + return; + } + + WriteHeaderInfo(); + + if (!s2.ok()) { + ROCKS_LOG_WARN(logger.get(), "Fail to trim old info log file: %s", + s2.ToString().c_str()); + } + } + + // pin down the current logger_ instance before releasing the mutex. + logger = logger_; + } + + // Another thread could have put a new Logger instance into logger_ by now. + // However, since logger is still hanging on to the previous instance + // (reference count is not zero), we don't have to worry about it being + // deleted while we are accessing it. + // Note that logv itself is not mutex protected to allow maximum concurrency, + // as thread safety should have been handled by the underlying logger. + logger->Logv(format, ap); +} + +void AutoRollLogger::WriteHeaderInfo() { + mutex_.AssertHeld(); + for (auto& header : headers_) { + LogInternal("%s", header.c_str()); + } +} + +void AutoRollLogger::LogHeader(const char* format, va_list args) { + if (!logger_) { + return; + } + + // header message are to be retained in memory. Since we cannot make any + // assumptions about the data contained in va_list, we will retain them as + // strings + va_list tmp; + va_copy(tmp, args); + std::string data = ValistToString(format, tmp); + va_end(tmp); + + MutexLock l(&mutex_); + headers_.push_back(data); + + // Log the original message to the current log + logger_->Logv(format, args); +} + +bool AutoRollLogger::LogExpired() { + if (cached_now_access_count >= call_NowMicros_every_N_records_) { + cached_now = static_cast(clock_->NowMicros() * 1e-6); + cached_now_access_count = 0; + } + + ++cached_now_access_count; + return cached_now >= ctime_ + kLogFileTimeToRoll; +} + +Status CreateLoggerFromOptions(const std::string& dbname, + const DBOptions& options, + std::shared_ptr* logger) { + if (options.info_log) { + *logger = options.info_log; + return Status::OK(); + } + + Env* env = options.env; + std::string db_absolute_path; + Status s = env->GetAbsolutePath(dbname, &db_absolute_path); + TEST_SYNC_POINT_CALLBACK("rocksdb::CreateLoggerFromOptions:AfterGetPath", &s); + if (!s.ok()) { + return s; + } + std::string fname = + InfoLogFileName(dbname, db_absolute_path, options.db_log_dir); + + const auto& clock = env->GetSystemClock(); + // In case it does not exist. + s = env->CreateDirIfMissing(dbname); + if (!s.ok()) { + if (options.db_log_dir.empty()) { + return s; + } else { + // Ignore the error returned during creation of dbname because dbname and + // db_log_dir can be on different filesystems in which case dbname will + // not exist and error should be ignored. db_log_dir creation will handle + // the error in case there is any error in the creation of dbname on same + // filesystem. + s = Status::OK(); + } + } + assert(s.ok()); + + if (!options.db_log_dir.empty()) { + s = env->CreateDirIfMissing(options.db_log_dir); + if (!s.ok()) { + return s; + } + } + // Currently we only support roll by time-to-roll and log size + if (options.log_file_time_to_roll > 0 || options.max_log_file_size > 0) { + AutoRollLogger* result = new AutoRollLogger( + env->GetFileSystem(), clock, dbname, options.db_log_dir, + options.max_log_file_size, options.log_file_time_to_roll, + options.keep_log_file_num, options.info_log_level); + s = result->GetStatus(); + if (!s.ok()) { + delete result; + } else { + logger->reset(result); + } + return s; + } + // Open a log file in the same directory as the db + s = env->FileExists(fname); + if (s.ok()) { + s = env->RenameFile( + fname, OldInfoLogFileName(dbname, clock->NowMicros(), db_absolute_path, + options.db_log_dir)); + + // The operation sequence of "FileExists -> Rename" is not atomic. It's + // possible that FileExists returns OK but file gets deleted before Rename. + // This can cause Rename to return IOError with subcode PathNotFound. + // Although it may be a rare case and applications should be discouraged + // to not concurrently modifying the contents of the directories accessed + // by the database instance, it is still helpful if we can perform some + // simple handling of this case. Therefore, we do the following: + // 1. if Rename() returns IOError with PathNotFound subcode, then we check + // whether the source file, i.e. LOG, exists. + // 2. if LOG exists, it means Rename() failed due to something else. Then + // we report error. + // 3. if LOG does not exist, it means it may have been removed/renamed by + // someone else. Since it does not exist, we can reset Status to OK so + // that this caller can try creating a new LOG file. If this succeeds, + // we should still allow it. + if (s.IsPathNotFound()) { + s = env->FileExists(fname); + if (s.IsNotFound()) { + s = Status::OK(); + } + } + } else if (s.IsNotFound()) { + // "LOG" is not required to exist since this could be a new DB. + s = Status::OK(); + } + if (s.ok()) { + s = env->NewLogger(fname, logger); + } + if (s.ok() && logger->get() != nullptr) { + (*logger)->SetInfoLogLevel(options.info_log_level); + } + return s; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/logging/auto_roll_logger.h b/librocksdb-sys/rocksdb/logging/auto_roll_logger.h new file mode 100644 index 0000000..dca9996 --- /dev/null +++ b/librocksdb-sys/rocksdb/logging/auto_roll_logger.h @@ -0,0 +1,166 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Logger implementation that can be shared by all environments +// where enough posix functionality is available. + +#pragma once +#include +#include +#include + +#include "file/filename.h" +#include "port/port.h" +#include "port/util_logger.h" +#include "test_util/sync_point.h" +#include "util/mutexlock.h" + +namespace ROCKSDB_NAMESPACE { +class FileSystem; +class SystemClock; + +// Rolls the log file by size and/or time +class AutoRollLogger : public Logger { + public: + AutoRollLogger(const std::shared_ptr& fs, + const std::shared_ptr& clock, + const std::string& dbname, const std::string& db_log_dir, + size_t log_max_size, size_t log_file_time_to_roll, + size_t keep_log_file_num, + const InfoLogLevel log_level = InfoLogLevel::INFO_LEVEL); + + using Logger::Logv; + void Logv(const char* format, va_list ap) override; + + // Write a header entry to the log. All header information will be written + // again every time the log rolls over. + virtual void LogHeader(const char* format, va_list ap) override; + + // check if the logger has encountered any problem. + Status GetStatus() { return status_; } + + size_t GetLogFileSize() const override { + if (!logger_) { + return 0; + } + + std::shared_ptr logger; + { + MutexLock l(&mutex_); + // pin down the current logger_ instance before releasing the mutex. + logger = logger_; + } + return logger->GetLogFileSize(); + } + + void Flush() override { + std::shared_ptr logger; + { + MutexLock l(&mutex_); + // pin down the current logger_ instance before releasing the mutex. + logger = logger_; + } + TEST_SYNC_POINT("AutoRollLogger::Flush:PinnedLogger"); + if (logger) { + logger->Flush(); + } + } + + virtual ~AutoRollLogger() { + if (logger_ && !closed_) { + logger_->Close().PermitUncheckedError(); + } + status_.PermitUncheckedError(); + } + + using Logger::GetInfoLogLevel; + InfoLogLevel GetInfoLogLevel() const override { + MutexLock l(&mutex_); + if (!logger_) { + return Logger::GetInfoLogLevel(); + } + return logger_->GetInfoLogLevel(); + } + + using Logger::SetInfoLogLevel; + void SetInfoLogLevel(const InfoLogLevel log_level) override { + MutexLock lock(&mutex_); + Logger::SetInfoLogLevel(log_level); + if (logger_) { + logger_->SetInfoLogLevel(log_level); + } + } + + void SetCallNowMicrosEveryNRecords(uint64_t call_NowMicros_every_N_records) { + call_NowMicros_every_N_records_ = call_NowMicros_every_N_records; + } + + // Expose the log file path for testing purpose + std::string TEST_log_fname() const { return log_fname_; } + + uint64_t TEST_ctime() const { return ctime_; } + + Logger* TEST_inner_logger() const { return logger_.get(); } + + protected: + // Implementation of Close() + virtual Status CloseImpl() override { + if (logger_) { + return logger_->Close(); + } else { + return Status::OK(); + } + } + + private: + bool LogExpired(); + Status ResetLogger(); + void RollLogFile(); + // Read all names of old log files into old_log_files_ + // If there is any error, put the error code in status_ + void GetExistingFiles(); + // Delete old log files if it excceeds the limit. + Status TrimOldLogFiles(); + // Log message to logger without rolling + void LogInternal(const char* format, ...); + // Serialize the va_list to a string + std::string ValistToString(const char* format, va_list args) const; + // Write the logs marked as headers to the new log file + void WriteHeaderInfo(); + std::string log_fname_; // Current active info log's file name. + std::string dbname_; + std::string db_log_dir_; + std::string db_absolute_path_; + std::shared_ptr fs_; + std::shared_ptr clock_; + std::shared_ptr logger_; + // current status of the logger + Status status_; + const size_t kMaxLogFileSize; + const size_t kLogFileTimeToRoll; + const size_t kKeepLogFileNum; + // header information + std::list headers_; + // List of all existing info log files. Used for enforcing number of + // info log files. + // Full path is stored here. It consumes signifianctly more memory + // than only storing file name. Can optimize if it causes a problem. + std::queue old_log_files_; + // to avoid frequent clock->NowMicros() calls, we cached the current time + uint64_t cached_now; + uint64_t ctime_; + uint64_t cached_now_access_count; + uint64_t call_NowMicros_every_N_records_; + IOOptions io_options_; + IODebugContext io_context_; + mutable port::Mutex mutex_; +}; + +// Facade to craete logger automatically +Status CreateLoggerFromOptions(const std::string& dbname, + const DBOptions& options, + std::shared_ptr* logger); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/logging/auto_roll_logger_test.cc b/librocksdb-sys/rocksdb/logging/auto_roll_logger_test.cc new file mode 100644 index 0000000..3d0ec17 --- /dev/null +++ b/librocksdb-sys/rocksdb/logging/auto_roll_logger_test.cc @@ -0,0 +1,731 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + + +#include "logging/auto_roll_logger.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "db/db_test_util.h" +#include "env/emulated_clock.h" +#include "logging/env_logger.h" +#include "logging/logging.h" +#include "port/port.h" +#include "rocksdb/db.h" +#include "rocksdb/file_system.h" +#include "rocksdb/system_clock.h" +#include "test_util/sync_point.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" + +namespace ROCKSDB_NAMESPACE { + +// In this test we only want to Log some simple log message with +// no format. LogMessage() provides such a simple interface and +// avoids the [format-security] warning which occurs when you +// call ROCKS_LOG_INFO(logger, log_message) directly. +namespace { +void LogMessage(Logger* logger, const char* message) { + ROCKS_LOG_INFO(logger, "%s", message); +} + +void LogMessage(const InfoLogLevel log_level, Logger* logger, + const char* message) { + Log(log_level, logger, "%s", message); +} +} // namespace + +class AutoRollLoggerTest : public testing::Test { + public: + static void InitTestDb() { + // TODO replace the `system` calls with Env/FileSystem APIs. +#ifdef OS_WIN + // Replace all slashes in the path so windows CompSpec does not + // become confused + std::string testDbDir(kTestDbDir); + std::replace_if( + testDbDir.begin(), testDbDir.end(), [](char ch) { return ch == '/'; }, + '\\'); + std::string deleteDbDirCmd = + "if exist " + testDbDir + " rd /s /q " + testDbDir; + ASSERT_TRUE(system(deleteDbDirCmd.c_str()) == 0); + + std::string testDir(kTestDir); + std::replace_if( + testDir.begin(), testDir.end(), [](char ch) { return ch == '/'; }, + '\\'); + std::string deleteCmd = "if exist " + testDir + " rd /s /q " + testDir; +#else + std::string deleteCmd = "rm -rf " + kTestDir + " " + kTestDbDir; +#endif + ASSERT_TRUE(system(deleteCmd.c_str()) == 0); + ASSERT_OK(Env::Default()->CreateDir(kTestDir)); + ASSERT_OK(Env::Default()->CreateDir(kTestDbDir)); + } + + void RollLogFileBySizeTest(AutoRollLogger* logger, size_t log_max_size, + const std::string& log_message); + void RollLogFileByTimeTest(const std::shared_ptr& fs, + const std::shared_ptr& sc, + AutoRollLogger* logger, size_t time, + const std::string& log_message); + // return list of files under kTestDir that contains "LOG" + std::vector GetLogFiles() { + std::vector ret; + std::vector files; + Status s = default_env->GetChildren(kTestDir, &files); + // Should call ASSERT_OK() here but it doesn't compile. It's not + // worth the time figuring out why. + EXPECT_TRUE(s.ok()); + for (const auto& f : files) { + if (f.find("LOG") != std::string::npos) { + ret.push_back(f); + } + } + return ret; + } + + // Delete all log files under kTestDir + void CleanupLogFiles() { + for (const std::string& f : GetLogFiles()) { + ASSERT_OK(default_env->DeleteFile(kTestDir + "/" + f)); + } + } + + void RollNTimesBySize(Logger* auto_roll_logger, size_t file_num, + size_t max_log_file_size) { + // Roll the log 4 times, and it will trim to 3 files. + std::string dummy_large_string; + dummy_large_string.assign(max_log_file_size, '='); + auto_roll_logger->SetInfoLogLevel(InfoLogLevel::INFO_LEVEL); + for (size_t i = 0; i < file_num + 1; i++) { + // Log enough bytes to trigger at least one roll. + LogMessage(auto_roll_logger, dummy_large_string.c_str()); + LogMessage(auto_roll_logger, ""); + } + } + + static const std::string kSampleMessage; + static const std::string kTestDir; + static const std::string kTestDbDir; + static const std::string kLogFile; + static Env* default_env; +}; + +const std::string AutoRollLoggerTest::kSampleMessage( + "this is the message to be written to the log file!!"); +const std::string AutoRollLoggerTest::kTestDir( + test::PerThreadDBPath("db_log_test")); +const std::string AutoRollLoggerTest::kTestDbDir( + test::PerThreadDBPath("db_log_test_db")); +const std::string AutoRollLoggerTest::kLogFile( + test::PerThreadDBPath("db_log_test") + "/LOG"); +Env* AutoRollLoggerTest::default_env = Env::Default(); + +void AutoRollLoggerTest::RollLogFileBySizeTest(AutoRollLogger* logger, + size_t log_max_size, + const std::string& log_message) { + logger->SetInfoLogLevel(InfoLogLevel::INFO_LEVEL); + ASSERT_EQ(InfoLogLevel::INFO_LEVEL, logger->GetInfoLogLevel()); + ASSERT_EQ(InfoLogLevel::INFO_LEVEL, + logger->TEST_inner_logger()->GetInfoLogLevel()); + // measure the size of each message, which is supposed + // to be equal or greater than log_message.size() + LogMessage(logger, log_message.c_str()); + size_t message_size = logger->GetLogFileSize(); + size_t current_log_size = message_size; + + // Test the cases when the log file will not be rolled. + while (current_log_size + message_size < log_max_size) { + LogMessage(logger, log_message.c_str()); + current_log_size += message_size; + ASSERT_EQ(current_log_size, logger->GetLogFileSize()); + } + + // Now the log file will be rolled + LogMessage(logger, log_message.c_str()); + // Since rotation is checked before actual logging, we need to + // trigger the rotation by logging another message. + LogMessage(logger, log_message.c_str()); + + ASSERT_TRUE(message_size == logger->GetLogFileSize()); +} + +void AutoRollLoggerTest::RollLogFileByTimeTest( + const std::shared_ptr& fs, + const std::shared_ptr& sc, AutoRollLogger* logger, size_t time, + const std::string& log_message) { + uint64_t expected_ctime; + uint64_t actual_ctime; + + uint64_t total_log_size; + EXPECT_OK(fs->GetFileSize(kLogFile, IOOptions(), &total_log_size, nullptr)); + expected_ctime = logger->TEST_ctime(); + logger->SetCallNowMicrosEveryNRecords(0); + + // -- Write to the log for several times, which is supposed + // to be finished before time. + for (int i = 0; i < 10; ++i) { + sc->SleepForMicroseconds(50000); + LogMessage(logger, log_message.c_str()); + EXPECT_OK(logger->GetStatus()); + // Make sure we always write to the same log file (by + // checking the create time); + + actual_ctime = logger->TEST_ctime(); + + // Also make sure the log size is increasing. + EXPECT_EQ(expected_ctime, actual_ctime); + EXPECT_GT(logger->GetLogFileSize(), total_log_size); + total_log_size = logger->GetLogFileSize(); + } + + // -- Make the log file expire + sc->SleepForMicroseconds(static_cast(time * 1000000)); + LogMessage(logger, log_message.c_str()); + + // At this time, the new log file should be created. + actual_ctime = logger->TEST_ctime(); + EXPECT_LT(expected_ctime, actual_ctime); + EXPECT_LT(logger->GetLogFileSize(), total_log_size); +} + +TEST_F(AutoRollLoggerTest, RollLogFileBySize) { + InitTestDb(); + size_t log_max_size = 1024 * 5; + size_t keep_log_file_num = 10; + + AutoRollLogger logger(FileSystem::Default(), SystemClock::Default(), kTestDir, + "", log_max_size, 0, keep_log_file_num); + + RollLogFileBySizeTest(&logger, log_max_size, + kSampleMessage + ":RollLogFileBySize"); +} + +TEST_F(AutoRollLoggerTest, RollLogFileByTime) { + auto nsc = + std::make_shared(SystemClock::Default(), true); + + size_t time = 2; + size_t log_size = 1024 * 5; + size_t keep_log_file_num = 10; + + InitTestDb(); + // -- Test the existence of file during the server restart. + ASSERT_EQ(Status::NotFound(), default_env->FileExists(kLogFile)); + AutoRollLogger logger(default_env->GetFileSystem(), nsc, kTestDir, "", + log_size, time, keep_log_file_num); + ASSERT_OK(default_env->FileExists(kLogFile)); + + RollLogFileByTimeTest(default_env->GetFileSystem(), nsc, &logger, time, + kSampleMessage + ":RollLogFileByTime"); +} + +TEST_F(AutoRollLoggerTest, SetInfoLogLevel) { + InitTestDb(); + Options options; + options.info_log_level = InfoLogLevel::FATAL_LEVEL; + options.max_log_file_size = 1024; + std::shared_ptr logger; + ASSERT_OK(CreateLoggerFromOptions(kTestDir, options, &logger)); + auto* auto_roll_logger = dynamic_cast(logger.get()); + ASSERT_NE(nullptr, auto_roll_logger); + ASSERT_EQ(InfoLogLevel::FATAL_LEVEL, auto_roll_logger->GetInfoLogLevel()); + ASSERT_EQ(InfoLogLevel::FATAL_LEVEL, + auto_roll_logger->TEST_inner_logger()->GetInfoLogLevel()); + auto_roll_logger->SetInfoLogLevel(InfoLogLevel::DEBUG_LEVEL); + ASSERT_EQ(InfoLogLevel::DEBUG_LEVEL, auto_roll_logger->GetInfoLogLevel()); + ASSERT_EQ(InfoLogLevel::DEBUG_LEVEL, logger->GetInfoLogLevel()); + ASSERT_EQ(InfoLogLevel::DEBUG_LEVEL, + auto_roll_logger->TEST_inner_logger()->GetInfoLogLevel()); +} + +TEST_F(AutoRollLoggerTest, OpenLogFilesMultipleTimesWithOptionLog_max_size) { + // If only 'log_max_size' options is specified, then every time + // when rocksdb is restarted, a new empty log file will be created. + InitTestDb(); + // WORKAROUND: + // avoid complier's complaint of "comparison between signed + // and unsigned integer expressions" because literal 0 is + // treated as "singed". + size_t kZero = 0; + size_t log_size = 1024; + size_t keep_log_file_num = 10; + + AutoRollLogger* logger = + new AutoRollLogger(FileSystem::Default(), SystemClock::Default(), + kTestDir, "", log_size, 0, keep_log_file_num); + + LogMessage(logger, kSampleMessage.c_str()); + ASSERT_GT(logger->GetLogFileSize(), kZero); + delete logger; + + // reopens the log file and an empty log file will be created. + logger = new AutoRollLogger(FileSystem::Default(), SystemClock::Default(), + kTestDir, "", log_size, 0, 10); + ASSERT_EQ(logger->GetLogFileSize(), kZero); + delete logger; +} + +TEST_F(AutoRollLoggerTest, CompositeRollByTimeAndSizeLogger) { + size_t time = 2, log_max_size = 1024 * 5; + size_t keep_log_file_num = 10; + + InitTestDb(); + + auto nsc = + std::make_shared(SystemClock::Default(), true); + AutoRollLogger logger(FileSystem::Default(), nsc, kTestDir, "", log_max_size, + time, keep_log_file_num); + + // Test the ability to roll by size + RollLogFileBySizeTest(&logger, log_max_size, + kSampleMessage + ":CompositeRollByTimeAndSizeLogger"); + + // Test the ability to roll by Time + RollLogFileByTimeTest(FileSystem::Default(), nsc, &logger, time, + kSampleMessage + ":CompositeRollByTimeAndSizeLogger"); +} + +#ifndef OS_WIN +// TODO: does not build for Windows because of EnvLogger use below. Need to +// port +TEST_F(AutoRollLoggerTest, CreateLoggerFromOptions) { + DBOptions options; + auto nsc = + std::make_shared(SystemClock::Default(), true); + std::unique_ptr nse(new CompositeEnvWrapper(Env::Default(), nsc)); + + std::shared_ptr logger; + + // Normal logger + ASSERT_OK(CreateLoggerFromOptions(kTestDir, options, &logger)); + ASSERT_TRUE(dynamic_cast(logger.get())); + + // Only roll by size + InitTestDb(); + options.max_log_file_size = 1024; + ASSERT_OK(CreateLoggerFromOptions(kTestDir, options, &logger)); + AutoRollLogger* auto_roll_logger = + dynamic_cast(logger.get()); + ASSERT_TRUE(auto_roll_logger); + RollLogFileBySizeTest(auto_roll_logger, options.max_log_file_size, + kSampleMessage + ":CreateLoggerFromOptions - size"); + + // Only roll by Time + options.env = nse.get(); + InitTestDb(); + options.max_log_file_size = 0; + options.log_file_time_to_roll = 2; + ASSERT_OK(CreateLoggerFromOptions(kTestDir, options, &logger)); + auto_roll_logger = dynamic_cast(logger.get()); + RollLogFileByTimeTest(options.env->GetFileSystem(), nsc, auto_roll_logger, + options.log_file_time_to_roll, + kSampleMessage + ":CreateLoggerFromOptions - time"); + + // roll by both Time and size + InitTestDb(); + options.max_log_file_size = 1024 * 5; + options.log_file_time_to_roll = 2; + ASSERT_OK(CreateLoggerFromOptions(kTestDir, options, &logger)); + auto_roll_logger = dynamic_cast(logger.get()); + RollLogFileBySizeTest(auto_roll_logger, options.max_log_file_size, + kSampleMessage + ":CreateLoggerFromOptions - both"); + RollLogFileByTimeTest(options.env->GetFileSystem(), nsc, auto_roll_logger, + options.log_file_time_to_roll, + kSampleMessage + ":CreateLoggerFromOptions - both"); + + // Set keep_log_file_num + { + const size_t kFileNum = 3; + InitTestDb(); + options.max_log_file_size = 512; + options.log_file_time_to_roll = 2; + options.keep_log_file_num = kFileNum; + ASSERT_OK(CreateLoggerFromOptions(kTestDir, options, &logger)); + auto_roll_logger = dynamic_cast(logger.get()); + + // Roll the log 4 times, and it will trim to 3 files. + std::string dummy_large_string; + dummy_large_string.assign(options.max_log_file_size, '='); + auto_roll_logger->SetInfoLogLevel(InfoLogLevel::INFO_LEVEL); + for (size_t i = 0; i < kFileNum + 1; i++) { + // Log enough bytes to trigger at least one roll. + LogMessage(auto_roll_logger, dummy_large_string.c_str()); + LogMessage(auto_roll_logger, ""); + } + + std::vector files = GetLogFiles(); + ASSERT_EQ(kFileNum, files.size()); + + CleanupLogFiles(); + } + + // Set keep_log_file_num and dbname is different from + // db_log_dir. + { + const size_t kFileNum = 3; + InitTestDb(); + options.max_log_file_size = 512; + options.log_file_time_to_roll = 2; + options.keep_log_file_num = kFileNum; + options.db_log_dir = kTestDir; + ASSERT_OK(CreateLoggerFromOptions(kTestDbDir, options, &logger)); + auto_roll_logger = dynamic_cast(logger.get()); + + // Roll the log 4 times, and it will trim to 3 files. + std::string dummy_large_string; + dummy_large_string.assign(options.max_log_file_size, '='); + auto_roll_logger->SetInfoLogLevel(InfoLogLevel::INFO_LEVEL); + for (size_t i = 0; i < kFileNum + 1; i++) { + // Log enough bytes to trigger at least one roll. + LogMessage(auto_roll_logger, dummy_large_string.c_str()); + LogMessage(auto_roll_logger, ""); + } + + std::vector files = GetLogFiles(); + ASSERT_EQ(kFileNum, files.size()); + for (const auto& f : files) { + ASSERT_TRUE(f.find("db_log_test_db") != std::string::npos); + } + + // Cleaning up those files. + CleanupLogFiles(); + } +} + +TEST_F(AutoRollLoggerTest, AutoDeleting) { + for (int attempt = 0; attempt < 2; attempt++) { + // In the first attemp, db_log_dir is not set, while in the + // second it is set. + std::string dbname = (attempt == 0) ? kTestDir : "/test/dummy/dir"; + std::string db_log_dir = (attempt == 0) ? "" : kTestDir; + + InitTestDb(); + const size_t kMaxFileSize = 512; + { + size_t log_num = 8; + AutoRollLogger logger(FileSystem::Default(), SystemClock::Default(), + dbname, db_log_dir, kMaxFileSize, 0, log_num); + RollNTimesBySize(&logger, log_num, kMaxFileSize); + + ASSERT_EQ(log_num, GetLogFiles().size()); + } + // Shrink number of files + { + size_t log_num = 5; + AutoRollLogger logger(FileSystem::Default(), SystemClock::Default(), + dbname, db_log_dir, kMaxFileSize, 0, log_num); + ASSERT_EQ(log_num, GetLogFiles().size()); + + RollNTimesBySize(&logger, 3, kMaxFileSize); + ASSERT_EQ(log_num, GetLogFiles().size()); + } + + // Increase number of files again. + { + size_t log_num = 7; + AutoRollLogger logger(FileSystem::Default(), SystemClock::Default(), + dbname, db_log_dir, kMaxFileSize, 0, log_num); + ASSERT_EQ(6, GetLogFiles().size()); + + RollNTimesBySize(&logger, 3, kMaxFileSize); + ASSERT_EQ(log_num, GetLogFiles().size()); + } + + CleanupLogFiles(); + } +} + +TEST_F(AutoRollLoggerTest, LogFlushWhileRolling) { + DBOptions options; + std::shared_ptr logger; + + InitTestDb(); + options.max_log_file_size = 1024 * 5; + ASSERT_OK(CreateLoggerFromOptions(kTestDir, options, &logger)); + AutoRollLogger* auto_roll_logger = + dynamic_cast(logger.get()); + ASSERT_TRUE(auto_roll_logger); + ROCKSDB_NAMESPACE::port::Thread flush_thread; + + // Notes: + // (1) Need to pin the old logger before beginning the roll, as rolling grabs + // the mutex, which would prevent us from accessing the old logger. This + // also marks flush_thread with AutoRollLogger::Flush:PinnedLogger. + // (2) New logger will be cut in AutoRollLogger::RollLogFile only when flush + // is completed and reference to pinned logger is released. + // (3) EnvLogger::Flush() happens in both threads but its SyncPoints only + // are enabled in flush_thread (the one pinning the old logger). + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependencyAndMarkers( + {{"AutoRollLogger::Flush:PinnedLogger", + "AutoRollLoggerTest::LogFlushWhileRolling:PreRollAndPostThreadInit"}}, + {{"AutoRollLogger::Flush:PinnedLogger", "EnvLogger::Flush:Begin1"}, + {"AutoRollLogger::Flush:PinnedLogger", "EnvLogger::Flush:Begin2"}}); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); + + flush_thread = port::Thread([&]() { auto_roll_logger->Flush(); }); + TEST_SYNC_POINT( + "AutoRollLoggerTest::LogFlushWhileRolling:PreRollAndPostThreadInit"); + RollLogFileBySizeTest(auto_roll_logger, options.max_log_file_size, + kSampleMessage + ":LogFlushWhileRolling"); + flush_thread.join(); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); +} + +#endif // OS_WIN + +TEST_F(AutoRollLoggerTest, InfoLogLevel) { + InitTestDb(); + + size_t log_size = 8192; + size_t log_lines = 0; + // an extra-scope to force the AutoRollLogger to flush the log file when it + // becomes out of scope. + { + AutoRollLogger logger(FileSystem::Default(), SystemClock::Default(), + kTestDir, "", log_size, 0, 10); + for (int log_level = InfoLogLevel::HEADER_LEVEL; + log_level >= InfoLogLevel::DEBUG_LEVEL; log_level--) { + logger.SetInfoLogLevel((InfoLogLevel)log_level); + for (int log_type = InfoLogLevel::DEBUG_LEVEL; + log_type <= InfoLogLevel::HEADER_LEVEL; log_type++) { + // log messages with log level smaller than log_level will not be + // logged. + LogMessage((InfoLogLevel)log_type, &logger, kSampleMessage.c_str()); + } + log_lines += InfoLogLevel::HEADER_LEVEL - log_level + 1; + } + for (int log_level = InfoLogLevel::HEADER_LEVEL; + log_level >= InfoLogLevel::DEBUG_LEVEL; log_level--) { + logger.SetInfoLogLevel((InfoLogLevel)log_level); + + // again, messages with level smaller than log_level will not be logged. + ROCKS_LOG_HEADER(&logger, "%s", kSampleMessage.c_str()); + ROCKS_LOG_DEBUG(&logger, "%s", kSampleMessage.c_str()); + ROCKS_LOG_INFO(&logger, "%s", kSampleMessage.c_str()); + ROCKS_LOG_WARN(&logger, "%s", kSampleMessage.c_str()); + ROCKS_LOG_ERROR(&logger, "%s", kSampleMessage.c_str()); + ROCKS_LOG_FATAL(&logger, "%s", kSampleMessage.c_str()); + log_lines += InfoLogLevel::HEADER_LEVEL - log_level + 1; + } + } + std::ifstream inFile(AutoRollLoggerTest::kLogFile.c_str()); + size_t lines = std::count(std::istreambuf_iterator(inFile), + std::istreambuf_iterator(), '\n'); + ASSERT_EQ(log_lines, lines); + inFile.close(); +} + +TEST_F(AutoRollLoggerTest, Close) { + InitTestDb(); + + size_t log_size = 8192; + size_t log_lines = 0; + AutoRollLogger logger(FileSystem::Default(), SystemClock::Default(), kTestDir, + "", log_size, 0, 10); + for (int log_level = InfoLogLevel::HEADER_LEVEL; + log_level >= InfoLogLevel::DEBUG_LEVEL; log_level--) { + logger.SetInfoLogLevel((InfoLogLevel)log_level); + for (int log_type = InfoLogLevel::DEBUG_LEVEL; + log_type <= InfoLogLevel::HEADER_LEVEL; log_type++) { + // log messages with log level smaller than log_level will not be + // logged. + LogMessage((InfoLogLevel)log_type, &logger, kSampleMessage.c_str()); + } + log_lines += InfoLogLevel::HEADER_LEVEL - log_level + 1; + } + for (int log_level = InfoLogLevel::HEADER_LEVEL; + log_level >= InfoLogLevel::DEBUG_LEVEL; log_level--) { + logger.SetInfoLogLevel((InfoLogLevel)log_level); + + // again, messages with level smaller than log_level will not be logged. + ROCKS_LOG_HEADER(&logger, "%s", kSampleMessage.c_str()); + ROCKS_LOG_DEBUG(&logger, "%s", kSampleMessage.c_str()); + ROCKS_LOG_INFO(&logger, "%s", kSampleMessage.c_str()); + ROCKS_LOG_WARN(&logger, "%s", kSampleMessage.c_str()); + ROCKS_LOG_ERROR(&logger, "%s", kSampleMessage.c_str()); + ROCKS_LOG_FATAL(&logger, "%s", kSampleMessage.c_str()); + log_lines += InfoLogLevel::HEADER_LEVEL - log_level + 1; + } + ASSERT_EQ(logger.Close(), Status::OK()); + + std::ifstream inFile(AutoRollLoggerTest::kLogFile.c_str()); + size_t lines = std::count(std::istreambuf_iterator(inFile), + std::istreambuf_iterator(), '\n'); + ASSERT_EQ(log_lines, lines); + inFile.close(); +} + +// Test the logger Header function for roll over logs +// We expect the new logs creates as roll over to carry the headers specified +static std::vector GetOldFileNames(const std::string& path) { + std::vector ret; + + const std::string dirname = path.substr(/*start=*/0, path.find_last_of("/")); + const std::string fname = path.substr(path.find_last_of("/") + 1); + + std::vector children; + EXPECT_OK(Env::Default()->GetChildren(dirname, &children)); + + // We know that the old log files are named [path] + // Return all entities that match the pattern + for (auto& child : children) { + if (fname != child && child.find(fname) == 0) { + ret.push_back(dirname + "/" + child); + } + } + + return ret; +} + +TEST_F(AutoRollLoggerTest, LogHeaderTest) { + static const size_t MAX_HEADERS = 10; + static const size_t LOG_MAX_SIZE = 1024 * 5; + static const std::string HEADER_STR = "Log header line"; + + // test_num == 0 -> standard call to Header() + // test_num == 1 -> call to Log() with InfoLogLevel::HEADER_LEVEL + for (int test_num = 0; test_num < 2; test_num++) { + InitTestDb(); + + AutoRollLogger logger(FileSystem::Default(), SystemClock::Default(), + kTestDir, /*db_log_dir=*/"", LOG_MAX_SIZE, + /*log_file_time_to_roll=*/0, + /*keep_log_file_num=*/10); + + if (test_num == 0) { + // Log some headers explicitly using Header() + for (size_t i = 0; i < MAX_HEADERS; i++) { + Header(&logger, "%s %" ROCKSDB_PRIszt, HEADER_STR.c_str(), i); + } + } else if (test_num == 1) { + // HEADER_LEVEL should make this behave like calling Header() + for (size_t i = 0; i < MAX_HEADERS; i++) { + ROCKS_LOG_HEADER(&logger, "%s %" ROCKSDB_PRIszt, HEADER_STR.c_str(), i); + } + } + + const std::string newfname = logger.TEST_log_fname(); + + // Log enough data to cause a roll over + int i = 0; + for (size_t iter = 0; iter < 2; iter++) { + while (logger.GetLogFileSize() < LOG_MAX_SIZE) { + Info(&logger, (kSampleMessage + ":LogHeaderTest line %d").c_str(), i); + ++i; + } + + Info(&logger, "Rollover"); + } + + // Flush the log for the latest file + LogFlush(&logger); + + const auto oldfiles = GetOldFileNames(newfname); + + ASSERT_EQ(oldfiles.size(), (size_t)2); + + for (auto& oldfname : oldfiles) { + // verify that the files rolled over + ASSERT_NE(oldfname, newfname); + // verify that the old log contains all the header logs + ASSERT_EQ(test::GetLinesCount(oldfname, HEADER_STR), MAX_HEADERS); + } + } +} + +TEST_F(AutoRollLoggerTest, LogFileExistence) { + ROCKSDB_NAMESPACE::DB* db; + ROCKSDB_NAMESPACE::Options options; +#ifdef OS_WIN + // Replace all slashes in the path so windows CompSpec does not + // become confused + std::string testDir(kTestDir); + std::replace_if( + testDir.begin(), testDir.end(), [](char ch) { return ch == '/'; }, '\\'); + std::string deleteCmd = "if exist " + testDir + " rd /s /q " + testDir; +#else + std::string deleteCmd = "rm -rf " + kTestDir; +#endif + ASSERT_EQ(system(deleteCmd.c_str()), 0); + options.max_log_file_size = 100 * 1024 * 1024; + options.create_if_missing = true; + ASSERT_OK(ROCKSDB_NAMESPACE::DB::Open(options, kTestDir, &db)); + ASSERT_OK(default_env->FileExists(kLogFile)); + delete db; +} + +TEST_F(AutoRollLoggerTest, FileCreateFailure) { + Options options; + options.max_log_file_size = 100 * 1024 * 1024; + options.db_log_dir = "/a/dir/does/not/exist/at/all"; + + std::shared_ptr logger; + ASSERT_NOK(CreateLoggerFromOptions("", options, &logger)); + ASSERT_TRUE(!logger); +} + +TEST_F(AutoRollLoggerTest, RenameOnlyWhenExists) { + InitTestDb(); + SpecialEnv env(Env::Default()); + Options options; + options.env = &env; + + // Originally no LOG exists. Should not see a rename. + { + std::shared_ptr logger; + ASSERT_OK(CreateLoggerFromOptions(kTestDir, options, &logger)); + ASSERT_EQ(0, env.rename_count_); + } + + // Now a LOG exists. Create a new one should see a rename. + { + std::shared_ptr logger; + ASSERT_OK(CreateLoggerFromOptions(kTestDir, options, &logger)); + ASSERT_EQ(1, env.rename_count_); + } +} + +TEST_F(AutoRollLoggerTest, RenameError) { + InitTestDb(); + SpecialEnv env(Env::Default()); + env.rename_error_ = true; + Options options; + options.env = &env; + + // Originally no LOG exists. Should not be impacted by rename error. + { + std::shared_ptr logger; + ASSERT_OK(CreateLoggerFromOptions(kTestDir, options, &logger)); + ASSERT_TRUE(logger != nullptr); + } + + // Now a LOG exists. Rename error should cause failure. + { + std::shared_ptr logger; + ASSERT_NOK(CreateLoggerFromOptions(kTestDir, options, &logger)); + ASSERT_TRUE(logger == nullptr); + } +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/librocksdb-sys/rocksdb/logging/env_logger.h b/librocksdb-sys/rocksdb/logging/env_logger.h new file mode 100644 index 0000000..fc9b245 --- /dev/null +++ b/librocksdb-sys/rocksdb/logging/env_logger.h @@ -0,0 +1,195 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Logger implementation that uses custom Env object for logging. + +#pragma once + +#include + +#include +#include + +#include "file/writable_file_writer.h" +#include "monitoring/iostats_context_imp.h" +#include "port/sys_time.h" +#include "rocksdb/env.h" +#include "rocksdb/file_system.h" +#include "rocksdb/perf_level.h" +#include "rocksdb/slice.h" +#include "test_util/sync_point.h" +#include "util/mutexlock.h" + +namespace ROCKSDB_NAMESPACE { + +class EnvLogger : public Logger { + public: + EnvLogger(std::unique_ptr&& writable_file, + const std::string& fname, const EnvOptions& options, Env* env, + InfoLogLevel log_level = InfoLogLevel::ERROR_LEVEL) + : Logger(log_level), + env_(env), + clock_(env_->GetSystemClock().get()), + file_(std::move(writable_file), fname, options, clock_), + last_flush_micros_(0), + flush_pending_(false) {} + + ~EnvLogger() { + if (!closed_) { + closed_ = true; + CloseHelper().PermitUncheckedError(); + } + } + + private: + // A guard to prepare file operations, such as mutex and skip + // I/O context. + class FileOpGuard { + public: + explicit FileOpGuard(EnvLogger& logger) + : logger_(logger), prev_perf_level_(GetPerfLevel()) { + // Preserve iostats not to pollute writes from user writes. We might + // need a better solution than this. + SetPerfLevel(PerfLevel::kDisable); + IOSTATS_SET_DISABLE(true); + logger.mutex_.Lock(); + } + ~FileOpGuard() { + logger_.mutex_.Unlock(); + IOSTATS_SET_DISABLE(false); + SetPerfLevel(prev_perf_level_); + } + + private: + EnvLogger& logger_; + PerfLevel prev_perf_level_; + }; + + void FlushLocked() { + mutex_.AssertHeld(); + if (flush_pending_) { + flush_pending_ = false; + file_.Flush().PermitUncheckedError(); + file_.reset_seen_error(); + } + last_flush_micros_ = clock_->NowMicros(); + } + + void Flush() override { + TEST_SYNC_POINT("EnvLogger::Flush:Begin1"); + TEST_SYNC_POINT("EnvLogger::Flush:Begin2"); + + FileOpGuard guard(*this); + FlushLocked(); + } + + Status CloseImpl() override { return CloseHelper(); } + + Status CloseHelper() { + FileOpGuard guard(*this); + const auto close_status = file_.Close(); + + if (close_status.ok()) { + return close_status; + } + return Status::IOError("Close of log file failed with error:" + + (close_status.getState() + ? std::string(close_status.getState()) + : std::string())); + } + + using Logger::Logv; + void Logv(const char* format, va_list ap) override { + IOSTATS_TIMER_GUARD(logger_nanos); + + const uint64_t thread_id = env_->GetThreadID(); + + // We try twice: the first time with a fixed-size stack allocated buffer, + // and the second time with a much larger dynamically allocated buffer. + char buffer[500]; + for (int iter = 0; iter < 2; iter++) { + char* base; + int bufsize; + if (iter == 0) { + bufsize = sizeof(buffer); + base = buffer; + } else { + bufsize = 65536; + base = new char[bufsize]; + } + char* p = base; + char* limit = base + bufsize; + + port::TimeVal now_tv; + port::GetTimeOfDay(&now_tv, nullptr); + const time_t seconds = now_tv.tv_sec; + struct tm t; + port::LocalTimeR(&seconds, &t); + p += snprintf(p, limit - p, "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llu ", + t.tm_year + 1900, t.tm_mon + 1, t.tm_mday, t.tm_hour, + t.tm_min, t.tm_sec, static_cast(now_tv.tv_usec), + static_cast(thread_id)); + + // Print the message + if (p < limit) { + va_list backup_ap; + va_copy(backup_ap, ap); + p += vsnprintf(p, limit - p, format, backup_ap); + va_end(backup_ap); + } + + // Truncate to available space if necessary + if (p >= limit) { + if (iter == 0) { + continue; // Try again with larger buffer + } else { + p = limit - 1; + } + } + + // Add newline if necessary + if (p == base || p[-1] != '\n') { + *p++ = '\n'; + } + + assert(p <= limit); + { + FileOpGuard guard(*this); + // We will ignore any error returned by Append(). + file_.Append(Slice(base, p - base)).PermitUncheckedError(); + file_.reset_seen_error(); + flush_pending_ = true; + const uint64_t now_micros = clock_->NowMicros(); + if (now_micros - last_flush_micros_ >= flush_every_seconds_ * 1000000) { + FlushLocked(); + } + } + if (base != buffer) { + delete[] base; + } + break; + } + } + + size_t GetLogFileSize() const override { + MutexLock l(&mutex_); + return file_.GetFileSize(); + } + + private: + Env* env_; + SystemClock* clock_; + WritableFileWriter file_; + mutable port::Mutex mutex_; // Mutex to protect the shared variables below. + const static uint64_t flush_every_seconds_ = 5; + std::atomic_uint_fast64_t last_flush_micros_; + std::atomic flush_pending_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/logging/env_logger_test.cc b/librocksdb-sys/rocksdb/logging/env_logger_test.cc new file mode 100644 index 0000000..467ab06 --- /dev/null +++ b/librocksdb-sys/rocksdb/logging/env_logger_test.cc @@ -0,0 +1,163 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#include "logging/env_logger.h" + +#include "test_util/testharness.h" +#include "test_util/testutil.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { +// In this test we only want to Log some simple log message with +// no format. +void LogMessage(std::shared_ptr logger, const std::string& message) { + Log(logger, "%s", message.c_str()); +} + +// Helper method to write the message num_times in the given logger. +void WriteLogs(std::shared_ptr logger, const std::string& message, + int num_times) { + for (int ii = 0; ii < num_times; ++ii) { + LogMessage(logger, message); + } +} + +} // namespace + +class EnvLoggerTest : public testing::Test { + public: + Env* env_; + + EnvLoggerTest() : env_(Env::Default()) {} + + ~EnvLoggerTest() = default; + + std::shared_ptr CreateLogger() { + std::shared_ptr result; + assert(NewEnvLogger(kLogFile, env_, &result).ok()); + assert(result); + result->SetInfoLogLevel(InfoLogLevel::INFO_LEVEL); + return result; + } + + void DeleteLogFile() { ASSERT_OK(env_->DeleteFile(kLogFile)); } + + static const std::string kSampleMessage; + static const std::string kTestDir; + static const std::string kLogFile; +}; + +const std::string EnvLoggerTest::kSampleMessage = + "this is the message to be written to the log file!!"; +const std::string EnvLoggerTest::kLogFile = test::PerThreadDBPath("log_file"); + +TEST_F(EnvLoggerTest, EmptyLogFile) { + auto logger = CreateLogger(); + ASSERT_EQ(logger->Close(), Status::OK()); + + // Check the size of the log file. + uint64_t file_size; + ASSERT_EQ(env_->GetFileSize(kLogFile, &file_size), Status::OK()); + ASSERT_EQ(file_size, 0); + DeleteLogFile(); +} + +TEST_F(EnvLoggerTest, LogMultipleLines) { + auto logger = CreateLogger(); + + // Write multiple lines. + const int kNumIter = 10; + WriteLogs(logger, kSampleMessage, kNumIter); + + // Flush the logs. + logger->Flush(); + ASSERT_EQ(logger->Close(), Status::OK()); + + // Validate whether the log file has 'kNumIter' number of lines. + ASSERT_EQ(test::GetLinesCount(kLogFile, kSampleMessage), kNumIter); + DeleteLogFile(); +} + +TEST_F(EnvLoggerTest, Overwrite) { + { + auto logger = CreateLogger(); + + // Write multiple lines. + const int kNumIter = 10; + WriteLogs(logger, kSampleMessage, kNumIter); + + ASSERT_EQ(logger->Close(), Status::OK()); + + // Validate whether the log file has 'kNumIter' number of lines. + ASSERT_EQ(test::GetLinesCount(kLogFile, kSampleMessage), kNumIter); + } + + // Now reopen the file again. + { + auto logger = CreateLogger(); + + // File should be empty. + uint64_t file_size; + ASSERT_EQ(env_->GetFileSize(kLogFile, &file_size), Status::OK()); + ASSERT_EQ(file_size, 0); + ASSERT_EQ(logger->GetLogFileSize(), 0); + ASSERT_EQ(logger->Close(), Status::OK()); + } + DeleteLogFile(); +} + +TEST_F(EnvLoggerTest, Close) { + auto logger = CreateLogger(); + + // Write multiple lines. + const int kNumIter = 10; + WriteLogs(logger, kSampleMessage, kNumIter); + + ASSERT_EQ(logger->Close(), Status::OK()); + + // Validate whether the log file has 'kNumIter' number of lines. + ASSERT_EQ(test::GetLinesCount(kLogFile, kSampleMessage), kNumIter); + DeleteLogFile(); +} + +TEST_F(EnvLoggerTest, ConcurrentLogging) { + auto logger = CreateLogger(); + + const int kNumIter = 20; + std::function cb = [&]() { + WriteLogs(logger, kSampleMessage, kNumIter); + logger->Flush(); + }; + + // Write to the logs from multiple threads. + std::vector threads; + const int kNumThreads = 5; + // Create threads. + for (int ii = 0; ii < kNumThreads; ++ii) { + threads.push_back(port::Thread(cb)); + } + + // Wait for them to complete. + for (auto& th : threads) { + th.join(); + } + + ASSERT_EQ(logger->Close(), Status::OK()); + + // Verfiy the log file. + ASSERT_EQ(test::GetLinesCount(kLogFile, kSampleMessage), + kNumIter * kNumThreads); + DeleteLogFile(); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/logging/event_logger.cc b/librocksdb-sys/rocksdb/logging/event_logger.cc new file mode 100644 index 0000000..cb9eca6 --- /dev/null +++ b/librocksdb-sys/rocksdb/logging/event_logger.cc @@ -0,0 +1,68 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "logging/event_logger.h" + +#include +#include +#include +#include + +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +EventLoggerStream::EventLoggerStream(Logger* logger) + : logger_(logger), + log_buffer_(nullptr), + max_log_size_(0), + json_writer_(nullptr) {} + +EventLoggerStream::EventLoggerStream(LogBuffer* log_buffer, + const size_t max_log_size) + : logger_(nullptr), + log_buffer_(log_buffer), + max_log_size_(max_log_size), + json_writer_(nullptr) {} + +EventLoggerStream::~EventLoggerStream() { + if (json_writer_) { + json_writer_->EndObject(); +#ifdef ROCKSDB_PRINT_EVENTS_TO_STDOUT + printf("%s\n", json_writer_->Get().c_str()); +#else + if (logger_) { + EventLogger::Log(logger_, *json_writer_); + } else if (log_buffer_) { + assert(max_log_size_); + EventLogger::LogToBuffer(log_buffer_, *json_writer_, max_log_size_); + } +#endif + delete json_writer_; + } +} + +void EventLogger::Log(const JSONWriter& jwriter) { Log(logger_, jwriter); } + +void EventLogger::Log(Logger* logger, const JSONWriter& jwriter) { +#ifdef ROCKSDB_PRINT_EVENTS_TO_STDOUT + printf("%s\n", jwriter.Get().c_str()); +#else + ROCKSDB_NAMESPACE::Log(logger, "%s %s", Prefix(), jwriter.Get().c_str()); +#endif +} + +void EventLogger::LogToBuffer(LogBuffer* log_buffer, const JSONWriter& jwriter, + const size_t max_log_size) { +#ifdef ROCKSDB_PRINT_EVENTS_TO_STDOUT + printf("%s\n", jwriter.Get().c_str()); +#else + assert(log_buffer); + ROCKSDB_NAMESPACE::LogToBuffer(log_buffer, max_log_size, "%s %s", Prefix(), + jwriter.Get().c_str()); +#endif +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/logging/event_logger.h b/librocksdb-sys/rocksdb/logging/event_logger.h new file mode 100644 index 0000000..9ce982f --- /dev/null +++ b/librocksdb-sys/rocksdb/logging/event_logger.h @@ -0,0 +1,202 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include +#include + +#include "logging/log_buffer.h" +#include "rocksdb/env.h" + +namespace ROCKSDB_NAMESPACE { + +class JSONWriter { + public: + JSONWriter() : state_(kExpectKey), first_element_(true), in_array_(false) { + stream_ << "{"; + } + + void AddKey(const std::string& key) { + assert(state_ == kExpectKey); + if (!first_element_) { + stream_ << ", "; + } + stream_ << "\"" << key << "\": "; + state_ = kExpectValue; + first_element_ = false; + } + + void AddValue(const char* value) { + assert(state_ == kExpectValue || state_ == kInArray); + if (state_ == kInArray && !first_element_) { + stream_ << ", "; + } + stream_ << "\"" << value << "\""; + if (state_ != kInArray) { + state_ = kExpectKey; + } + first_element_ = false; + } + + template + void AddValue(const T& value) { + assert(state_ == kExpectValue || state_ == kInArray); + if (state_ == kInArray && !first_element_) { + stream_ << ", "; + } + stream_ << value; + if (state_ != kInArray) { + state_ = kExpectKey; + } + first_element_ = false; + } + + void StartArray() { + assert(state_ == kExpectValue); + state_ = kInArray; + in_array_ = true; + stream_ << "["; + first_element_ = true; + } + + void EndArray() { + assert(state_ == kInArray); + state_ = kExpectKey; + in_array_ = false; + stream_ << "]"; + first_element_ = false; + } + + void StartObject() { + assert(state_ == kExpectValue); + state_ = kExpectKey; + stream_ << "{"; + first_element_ = true; + } + + void EndObject() { + assert(state_ == kExpectKey); + stream_ << "}"; + first_element_ = false; + } + + void StartArrayedObject() { + assert(state_ == kInArray && in_array_); + state_ = kExpectValue; + if (!first_element_) { + stream_ << ", "; + } + StartObject(); + } + + void EndArrayedObject() { + assert(in_array_); + EndObject(); + state_ = kInArray; + } + + std::string Get() const { return stream_.str(); } + + JSONWriter& operator<<(const char* val) { + if (state_ == kExpectKey) { + AddKey(val); + } else { + AddValue(val); + } + return *this; + } + + JSONWriter& operator<<(const std::string& val) { + return *this << val.c_str(); + } + + template + JSONWriter& operator<<(const T& val) { + assert(state_ != kExpectKey); + AddValue(val); + return *this; + } + + private: + enum JSONWriterState { + kExpectKey, + kExpectValue, + kInArray, + kInArrayedObject, + }; + JSONWriterState state_; + bool first_element_; + bool in_array_; + std::ostringstream stream_; +}; + +class EventLoggerStream { + public: + template + EventLoggerStream& operator<<(const T& val) { + MakeStream(); + *json_writer_ << val; + return *this; + } + + void StartArray() { json_writer_->StartArray(); } + void EndArray() { json_writer_->EndArray(); } + void StartObject() { json_writer_->StartObject(); } + void EndObject() { json_writer_->EndObject(); } + + ~EventLoggerStream(); + + private: + void MakeStream() { + if (!json_writer_) { + json_writer_ = new JSONWriter(); + *this << "time_micros" + << std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + } + } + friend class EventLogger; + explicit EventLoggerStream(Logger* logger); + explicit EventLoggerStream(LogBuffer* log_buffer, const size_t max_log_size); + // exactly one is non-nullptr + Logger* const logger_; + LogBuffer* const log_buffer_; + const size_t max_log_size_; // used only for log_buffer_ + // ownership + JSONWriter* json_writer_; +}; + +// here is an example of the output that will show up in the LOG: +// 2015/01/15-14:13:25.788019 1105ef000 EVENT_LOG_v1 {"time_micros": +// 1421360005788015, "event": "table_file_creation", "file_number": 12, +// "file_size": 1909699} +class EventLogger { + public: + static const char* Prefix() { return "EVENT_LOG_v1"; } + + explicit EventLogger(Logger* logger) : logger_(logger) {} + EventLoggerStream Log() { return EventLoggerStream(logger_); } + EventLoggerStream LogToBuffer(LogBuffer* log_buffer) { + return EventLoggerStream(log_buffer, LogBuffer::kDefaultMaxLogSize); + } + EventLoggerStream LogToBuffer(LogBuffer* log_buffer, + const size_t max_log_size) { + return EventLoggerStream(log_buffer, max_log_size); + } + void Log(const JSONWriter& jwriter); + static void Log(Logger* logger, const JSONWriter& jwriter); + static void LogToBuffer( + LogBuffer* log_buffer, const JSONWriter& jwriter, + const size_t max_log_size = LogBuffer::kDefaultMaxLogSize); + + private: + Logger* logger_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/logging/event_logger_test.cc b/librocksdb-sys/rocksdb/logging/event_logger_test.cc new file mode 100644 index 0000000..582f56c --- /dev/null +++ b/librocksdb-sys/rocksdb/logging/event_logger_test.cc @@ -0,0 +1,45 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "logging/event_logger.h" + +#include + +#include "test_util/testharness.h" + +namespace ROCKSDB_NAMESPACE { + +class EventLoggerTest : public testing::Test {}; + +class StringLogger : public Logger { + public: + using Logger::Logv; + void Logv(const char* format, va_list ap) override { + vsnprintf(buffer_, sizeof(buffer_), format, ap); + } + char* buffer() { return buffer_; } + + private: + char buffer_[1000]; +}; + +TEST_F(EventLoggerTest, SimpleTest) { + StringLogger logger; + EventLogger event_logger(&logger); + event_logger.Log() << "id" << 5 << "event" + << "just_testing"; + std::string output(logger.buffer()); + ASSERT_TRUE(output.find("\"event\": \"just_testing\"") != std::string::npos); + ASSERT_TRUE(output.find("\"id\": 5") != std::string::npos); + ASSERT_TRUE(output.find("\"time_micros\"") != std::string::npos); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/logging/log_buffer.cc b/librocksdb-sys/rocksdb/logging/log_buffer.cc new file mode 100644 index 0000000..2763e61 --- /dev/null +++ b/librocksdb-sys/rocksdb/logging/log_buffer.cc @@ -0,0 +1,91 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "logging/log_buffer.h" + +#include "port/port.h" +#include "port/sys_time.h" + +namespace ROCKSDB_NAMESPACE { + +LogBuffer::LogBuffer(const InfoLogLevel log_level, Logger* info_log) + : log_level_(log_level), info_log_(info_log) {} + +void LogBuffer::AddLogToBuffer(size_t max_log_size, const char* format, + va_list ap) { + if (!info_log_ || log_level_ < info_log_->GetInfoLogLevel()) { + // Skip the level because of its level. + return; + } + + char* alloc_mem = arena_.AllocateAligned(max_log_size); + BufferedLog* buffered_log = new (alloc_mem) BufferedLog(); + char* p = buffered_log->message; + char* limit = alloc_mem + max_log_size - 1; + + // store the time + port::GetTimeOfDay(&(buffered_log->now_tv), nullptr); + + // Print the message + if (p < limit) { + va_list backup_ap; + va_copy(backup_ap, ap); + auto n = vsnprintf(p, limit - p, format, backup_ap); +#ifndef OS_WIN + // MS reports -1 when the buffer is too short + assert(n >= 0); +#endif + if (n > 0) { + p += n; + } else { + p = limit; + } + va_end(backup_ap); + } + + if (p > limit) { + p = limit; + } + + // Add '\0' to the end + *p = '\0'; + + logs_.push_back(buffered_log); +} + +void LogBuffer::FlushBufferToLog() { + for (BufferedLog* log : logs_) { + const time_t seconds = log->now_tv.tv_sec; + struct tm t; + if (port::LocalTimeR(&seconds, &t) != nullptr) { + Log(log_level_, info_log_, + "(Original Log Time %04d/%02d/%02d-%02d:%02d:%02d.%06d) %s", + t.tm_year + 1900, t.tm_mon + 1, t.tm_mday, t.tm_hour, t.tm_min, + t.tm_sec, static_cast(log->now_tv.tv_usec), log->message); + } + } + logs_.clear(); +} + +void LogToBuffer(LogBuffer* log_buffer, size_t max_log_size, const char* format, + ...) { + if (log_buffer != nullptr) { + va_list ap; + va_start(ap, format); + log_buffer->AddLogToBuffer(max_log_size, format, ap); + va_end(ap); + } +} + +void LogToBuffer(LogBuffer* log_buffer, const char* format, ...) { + if (log_buffer != nullptr) { + va_list ap; + va_start(ap, format); + log_buffer->AddLogToBuffer(LogBuffer::kDefaultMaxLogSize, format, ap); + va_end(ap); + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/logging/log_buffer.h b/librocksdb-sys/rocksdb/logging/log_buffer.h new file mode 100644 index 0000000..92d38d1 --- /dev/null +++ b/librocksdb-sys/rocksdb/logging/log_buffer.h @@ -0,0 +1,57 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "memory/arena.h" +#include "port/sys_time.h" +#include "rocksdb/env.h" +#include "util/autovector.h" + +namespace ROCKSDB_NAMESPACE { + +class Logger; + +// A class to buffer info log entries and flush them in the end. +class LogBuffer { + public: + // log_level: the log level for all the logs + // info_log: logger to write the logs to + LogBuffer(const InfoLogLevel log_level, Logger* info_log); + + // Add a log entry to the buffer. Use default max_log_size. + // max_log_size indicates maximize log size, including some metadata. + void AddLogToBuffer(size_t max_log_size, const char* format, va_list ap); + + size_t IsEmpty() const { return logs_.empty(); } + + // Flush all buffered log to the info log. + void FlushBufferToLog(); + static const size_t kDefaultMaxLogSize = 512; + + private: + // One log entry with its timestamp + struct BufferedLog { + port::TimeVal now_tv; // Timestamp of the log + char message[1]; // Beginning of log message + }; + + const InfoLogLevel log_level_; + Logger* info_log_; + Arena arena_; + autovector logs_; +}; + +// Add log to the LogBuffer for a delayed info logging. It can be used when +// we want to add some logs inside a mutex. +// max_log_size indicates maximize log size, including some metadata. +extern void LogToBuffer(LogBuffer* log_buffer, size_t max_log_size, + const char* format, ...); +// Same as previous function, but with default max log size. +extern void LogToBuffer(LogBuffer* log_buffer, const char* format, ...); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/logging/logging.h b/librocksdb-sys/rocksdb/logging/logging.h new file mode 100644 index 0000000..0fa882a --- /dev/null +++ b/librocksdb-sys/rocksdb/logging/logging.h @@ -0,0 +1,62 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Must not be included from any .h files to avoid polluting the namespace +// with macros. + +#pragma once + +// Helper macros that include information about file name and line number +#define ROCKS_LOG_STRINGIFY(x) #x +#define ROCKS_LOG_TOSTRING(x) ROCKS_LOG_STRINGIFY(x) +#define ROCKS_LOG_PREPEND_FILE_LINE(FMT) \ + ("[%s:" ROCKS_LOG_TOSTRING(__LINE__) "] " FMT) + +inline const char* RocksLogShorterFileName(const char* file) { + // 18 is the length of "logging/logging.h". + // If the name of this file changed, please change this number, too. + return file + (sizeof(__FILE__) > 18 ? sizeof(__FILE__) - 18 : 0); +} + +// Don't inclide file/line info in HEADER level +#define ROCKS_LOG_HEADER(LGR, FMT, ...) \ + ROCKSDB_NAMESPACE::Log(InfoLogLevel::HEADER_LEVEL, LGR, FMT, ##__VA_ARGS__) + +#define ROCKS_LOG_AT_LEVEL(LGR, LVL, FMT, ...) \ + ROCKSDB_NAMESPACE::Log((LVL), (LGR), ROCKS_LOG_PREPEND_FILE_LINE(FMT), \ + RocksLogShorterFileName(__FILE__), ##__VA_ARGS__) + +#define ROCKS_LOG_DEBUG(LGR, FMT, ...) \ + ROCKS_LOG_AT_LEVEL((LGR), InfoLogLevel::DEBUG_LEVEL, FMT, ##__VA_ARGS__) + +#define ROCKS_LOG_INFO(LGR, FMT, ...) \ + ROCKS_LOG_AT_LEVEL((LGR), InfoLogLevel::INFO_LEVEL, FMT, ##__VA_ARGS__) + +#define ROCKS_LOG_WARN(LGR, FMT, ...) \ + ROCKS_LOG_AT_LEVEL((LGR), InfoLogLevel::WARN_LEVEL, FMT, ##__VA_ARGS__) + +#define ROCKS_LOG_ERROR(LGR, FMT, ...) \ + ROCKS_LOG_AT_LEVEL((LGR), InfoLogLevel::ERROR_LEVEL, FMT, ##__VA_ARGS__) + +#define ROCKS_LOG_FATAL(LGR, FMT, ...) \ + ROCKS_LOG_AT_LEVEL((LGR), InfoLogLevel::FATAL_LEVEL, FMT, ##__VA_ARGS__) + +#define ROCKS_LOG_BUFFER(LOG_BUF, FMT, ...) \ + ROCKSDB_NAMESPACE::LogToBuffer(LOG_BUF, ROCKS_LOG_PREPEND_FILE_LINE(FMT), \ + RocksLogShorterFileName(__FILE__), \ + ##__VA_ARGS__) + +#define ROCKS_LOG_BUFFER_MAX_SZ(LOG_BUF, MAX_LOG_SIZE, FMT, ...) \ + ROCKSDB_NAMESPACE::LogToBuffer( \ + LOG_BUF, MAX_LOG_SIZE, ROCKS_LOG_PREPEND_FILE_LINE(FMT), \ + RocksLogShorterFileName(__FILE__), ##__VA_ARGS__) + +#define ROCKS_LOG_DETAILS(LGR, FMT, ...) \ + ; // due to overhead by default skip such lines +// ROCKS_LOG_DEBUG(LGR, FMT, ##__VA_ARGS__) diff --git a/librocksdb-sys/rocksdb/memory/allocator.h b/librocksdb-sys/rocksdb/memory/allocator.h new file mode 100644 index 0000000..0d7cd60 --- /dev/null +++ b/librocksdb-sys/rocksdb/memory/allocator.h @@ -0,0 +1,58 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Abstract interface for allocating memory in blocks. This memory is freed +// when the allocator object is destroyed. See the Arena class for more info. + +#pragma once +#include +#include + +#include "rocksdb/write_buffer_manager.h" + +namespace ROCKSDB_NAMESPACE { + +class Logger; + +class Allocator { + public: + virtual ~Allocator() {} + + virtual char* Allocate(size_t bytes) = 0; + virtual char* AllocateAligned(size_t bytes, size_t huge_page_size = 0, + Logger* logger = nullptr) = 0; + + virtual size_t BlockSize() const = 0; +}; + +class AllocTracker { + public: + explicit AllocTracker(WriteBufferManager* write_buffer_manager); + // No copying allowed + AllocTracker(const AllocTracker&) = delete; + void operator=(const AllocTracker&) = delete; + + ~AllocTracker(); + void Allocate(size_t bytes); + // Call when we're finished allocating memory so we can free it from + // the write buffer's limit. + void DoneAllocating(); + + void FreeMem(); + + bool is_freed() const { return write_buffer_manager_ == nullptr || freed_; } + + private: + WriteBufferManager* write_buffer_manager_; + std::atomic bytes_allocated_; + bool done_allocating_; + bool freed_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/memory/arena.cc b/librocksdb-sys/rocksdb/memory/arena.cc new file mode 100644 index 0000000..0a92020 --- /dev/null +++ b/librocksdb-sys/rocksdb/memory/arena.cc @@ -0,0 +1,170 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "memory/arena.h" + +#include + +#include "logging/logging.h" +#include "port/malloc.h" +#include "port/port.h" +#include "rocksdb/env.h" +#include "test_util/sync_point.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +size_t Arena::OptimizeBlockSize(size_t block_size) { + // Make sure block_size is in optimal range + block_size = std::max(Arena::kMinBlockSize, block_size); + block_size = std::min(Arena::kMaxBlockSize, block_size); + + // make sure block_size is the multiple of kAlignUnit + if (block_size % kAlignUnit != 0) { + block_size = (1 + block_size / kAlignUnit) * kAlignUnit; + } + + return block_size; +} + +Arena::Arena(size_t block_size, AllocTracker* tracker, size_t huge_page_size) + : kBlockSize(OptimizeBlockSize(block_size)), tracker_(tracker) { + assert(kBlockSize >= kMinBlockSize && kBlockSize <= kMaxBlockSize && + kBlockSize % kAlignUnit == 0); + TEST_SYNC_POINT_CALLBACK("Arena::Arena:0", const_cast(&kBlockSize)); + alloc_bytes_remaining_ = sizeof(inline_block_); + blocks_memory_ += alloc_bytes_remaining_; + aligned_alloc_ptr_ = inline_block_; + unaligned_alloc_ptr_ = inline_block_ + alloc_bytes_remaining_; + if (MemMapping::kHugePageSupported) { + hugetlb_size_ = huge_page_size; + if (hugetlb_size_ && kBlockSize > hugetlb_size_) { + hugetlb_size_ = ((kBlockSize - 1U) / hugetlb_size_ + 1U) * hugetlb_size_; + } + } + if (tracker_ != nullptr) { + tracker_->Allocate(kInlineSize); + } +} + +Arena::~Arena() { + if (tracker_ != nullptr) { + assert(tracker_->is_freed()); + tracker_->FreeMem(); + } +} + +char* Arena::AllocateFallback(size_t bytes, bool aligned) { + if (bytes > kBlockSize / 4) { + ++irregular_block_num; + // Object is more than a quarter of our block size. Allocate it separately + // to avoid wasting too much space in leftover bytes. + return AllocateNewBlock(bytes); + } + + // We waste the remaining space in the current block. + size_t size = 0; + char* block_head = nullptr; + if (MemMapping::kHugePageSupported && hugetlb_size_ > 0) { + size = hugetlb_size_; + block_head = AllocateFromHugePage(size); + } + if (!block_head) { + size = kBlockSize; + block_head = AllocateNewBlock(size); + } + alloc_bytes_remaining_ = size - bytes; + + if (aligned) { + aligned_alloc_ptr_ = block_head + bytes; + unaligned_alloc_ptr_ = block_head + size; + return block_head; + } else { + aligned_alloc_ptr_ = block_head; + unaligned_alloc_ptr_ = block_head + size - bytes; + return unaligned_alloc_ptr_; + } +} + +char* Arena::AllocateFromHugePage(size_t bytes) { + MemMapping mm = MemMapping::AllocateHuge(bytes); + auto addr = static_cast(mm.Get()); + if (addr) { + huge_blocks_.push_back(std::move(mm)); + blocks_memory_ += bytes; + if (tracker_ != nullptr) { + tracker_->Allocate(bytes); + } + } + return addr; +} + +char* Arena::AllocateAligned(size_t bytes, size_t huge_page_size, + Logger* logger) { + if (MemMapping::kHugePageSupported && hugetlb_size_ > 0 && + huge_page_size > 0 && bytes > 0) { + // Allocate from a huge page TLB table. + size_t reserved_size = + ((bytes - 1U) / huge_page_size + 1U) * huge_page_size; + assert(reserved_size >= bytes); + + char* addr = AllocateFromHugePage(reserved_size); + if (addr == nullptr) { + ROCKS_LOG_WARN(logger, + "AllocateAligned fail to allocate huge TLB pages: %s", + errnoStr(errno).c_str()); + // fail back to malloc + } else { + return addr; + } + } + + size_t current_mod = + reinterpret_cast(aligned_alloc_ptr_) & (kAlignUnit - 1); + size_t slop = (current_mod == 0 ? 0 : kAlignUnit - current_mod); + size_t needed = bytes + slop; + char* result; + if (needed <= alloc_bytes_remaining_) { + result = aligned_alloc_ptr_ + slop; + aligned_alloc_ptr_ += needed; + alloc_bytes_remaining_ -= needed; + } else { + // AllocateFallback always returns aligned memory + result = AllocateFallback(bytes, true /* aligned */); + } + assert((reinterpret_cast(result) & (kAlignUnit - 1)) == 0); + return result; +} + +char* Arena::AllocateNewBlock(size_t block_bytes) { + // NOTE: std::make_unique zero-initializes the block so is not appropriate + // here + char* block = new char[block_bytes]; + blocks_.push_back(std::unique_ptr(block)); + + size_t allocated_size; +#ifdef ROCKSDB_MALLOC_USABLE_SIZE + allocated_size = malloc_usable_size(block); +#ifndef NDEBUG + // It's hard to predict what malloc_usable_size() returns. + // A callback can allow users to change the costed size. + std::pair pair(&allocated_size, &block_bytes); + TEST_SYNC_POINT_CALLBACK("Arena::AllocateNewBlock:0", &pair); +#endif // NDEBUG +#else + allocated_size = block_bytes; +#endif // ROCKSDB_MALLOC_USABLE_SIZE + blocks_memory_ += allocated_size; + if (tracker_ != nullptr) { + tracker_->Allocate(allocated_size); + } + return block; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/memory/arena.h b/librocksdb-sys/rocksdb/memory/arena.h new file mode 100644 index 0000000..39399aa --- /dev/null +++ b/librocksdb-sys/rocksdb/memory/arena.h @@ -0,0 +1,135 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +// Arena is an implementation of Allocator class. For a request of small size, +// it allocates a block with pre-defined block size. For a request of big +// size, it uses malloc to directly get the requested size. + +#pragma once + +#include +#include + +#include "memory/allocator.h" +#include "port/mmap.h" +#include "rocksdb/env.h" + +namespace ROCKSDB_NAMESPACE { + +class Arena : public Allocator { + public: + // No copying allowed + Arena(const Arena&) = delete; + void operator=(const Arena&) = delete; + + static constexpr size_t kInlineSize = 2048; + static constexpr size_t kMinBlockSize = 4096; + static constexpr size_t kMaxBlockSize = 2u << 30; + + static constexpr unsigned kAlignUnit = alignof(std::max_align_t); + static_assert((kAlignUnit & (kAlignUnit - 1)) == 0, + "Pointer size should be power of 2"); + + // huge_page_size: if 0, don't use huge page TLB. If > 0 (should set to the + // supported hugepage size of the system), block allocation will try huge + // page TLB first. If allocation fails, will fall back to normal case. + explicit Arena(size_t block_size = kMinBlockSize, + AllocTracker* tracker = nullptr, size_t huge_page_size = 0); + ~Arena(); + + char* Allocate(size_t bytes) override; + + // huge_page_size: if >0, will try to allocate from huage page TLB. + // The argument will be the size of the page size for huge page TLB. Bytes + // will be rounded up to multiple of the page size to allocate through mmap + // anonymous option with huge page on. The extra space allocated will be + // wasted. If allocation fails, will fall back to normal case. To enable it, + // need to reserve huge pages for it to be allocated, like: + // sysctl -w vm.nr_hugepages=20 + // See linux doc Documentation/vm/hugetlbpage.txt for details. + // huge page allocation can fail. In this case it will fail back to + // normal cases. The messages will be logged to logger. So when calling with + // huge_page_tlb_size > 0, we highly recommend a logger is passed in. + // Otherwise, the error message will be printed out to stderr directly. + char* AllocateAligned(size_t bytes, size_t huge_page_size = 0, + Logger* logger = nullptr) override; + + // Returns an estimate of the total memory usage of data allocated + // by the arena (exclude the space allocated but not yet used for future + // allocations). + size_t ApproximateMemoryUsage() const { + return blocks_memory_ + blocks_.size() * sizeof(char*) - + alloc_bytes_remaining_; + } + + size_t MemoryAllocatedBytes() const { return blocks_memory_; } + + size_t AllocatedAndUnused() const { return alloc_bytes_remaining_; } + + // If an allocation is too big, we'll allocate an irregular block with the + // same size of that allocation. + size_t IrregularBlockNum() const { return irregular_block_num; } + + size_t BlockSize() const override { return kBlockSize; } + + bool IsInInlineBlock() const { + return blocks_.empty() && huge_blocks_.empty(); + } + + // check and adjust the block_size so that the return value is + // 1. in the range of [kMinBlockSize, kMaxBlockSize]. + // 2. the multiple of align unit. + static size_t OptimizeBlockSize(size_t block_size); + + private: + alignas(std::max_align_t) char inline_block_[kInlineSize]; + // Number of bytes allocated in one block + const size_t kBlockSize; + // Allocated memory blocks + std::deque> blocks_; + // Huge page allocations + std::deque huge_blocks_; + size_t irregular_block_num = 0; + + // Stats for current active block. + // For each block, we allocate aligned memory chucks from one end and + // allocate unaligned memory chucks from the other end. Otherwise the + // memory waste for alignment will be higher if we allocate both types of + // memory from one direction. + char* unaligned_alloc_ptr_ = nullptr; + char* aligned_alloc_ptr_ = nullptr; + // How many bytes left in currently active block? + size_t alloc_bytes_remaining_ = 0; + + size_t hugetlb_size_ = 0; + + char* AllocateFromHugePage(size_t bytes); + char* AllocateFallback(size_t bytes, bool aligned); + char* AllocateNewBlock(size_t block_bytes); + + // Bytes of memory in blocks allocated so far + size_t blocks_memory_ = 0; + // Non-owned + AllocTracker* tracker_; +}; + +inline char* Arena::Allocate(size_t bytes) { + // The semantics of what to return are a bit messy if we allow + // 0-byte allocations, so we disallow them here (we don't need + // them for our internal use). + assert(bytes > 0); + if (bytes <= alloc_bytes_remaining_) { + unaligned_alloc_ptr_ -= bytes; + alloc_bytes_remaining_ -= bytes; + return unaligned_alloc_ptr_; + } + return AllocateFallback(bytes, false /* unaligned */); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/memory/arena_test.cc b/librocksdb-sys/rocksdb/memory/arena_test.cc new file mode 100644 index 0000000..21bf7ed --- /dev/null +++ b/librocksdb-sys/rocksdb/memory/arena_test.cc @@ -0,0 +1,295 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "memory/arena.h" + +#ifndef OS_WIN +#include +#endif +#include "port/port.h" +#include "test_util/testharness.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { +const size_t kHugePageSize = 2 * 1024 * 1024; +} // namespace +class ArenaTest : public testing::Test {}; + +TEST_F(ArenaTest, Empty) { Arena arena0; } + +namespace { +bool CheckMemoryAllocated(size_t allocated, size_t expected) { + // The value returned by Arena::MemoryAllocatedBytes() may be greater than + // the requested memory. We choose a somewhat arbitrary upper bound of + // max_expected = expected * 1.1 to detect critical overallocation. + size_t max_expected = expected + expected / 10; + return allocated >= expected && allocated <= max_expected; +} + +void MemoryAllocatedBytesTest(size_t huge_page_size) { + const int N = 17; + size_t req_sz; // requested size + size_t bsz = 32 * 1024; // block size + size_t expected_memory_allocated; + + Arena arena(bsz, nullptr, huge_page_size); + + // requested size > quarter of a block: + // allocate requested size separately + req_sz = 12 * 1024; + for (int i = 0; i < N; i++) { + arena.Allocate(req_sz); + } + expected_memory_allocated = req_sz * N + Arena::kInlineSize; + ASSERT_PRED2(CheckMemoryAllocated, arena.MemoryAllocatedBytes(), + expected_memory_allocated); + + arena.Allocate(Arena::kInlineSize - 1); + + // requested size < quarter of a block: + // allocate a block with the default size, then try to use unused part + // of the block. So one new block will be allocated for the first + // Allocate(99) call. All the remaining calls won't lead to new allocation. + req_sz = 99; + for (int i = 0; i < N; i++) { + arena.Allocate(req_sz); + } + if (huge_page_size) { + ASSERT_TRUE( + CheckMemoryAllocated(arena.MemoryAllocatedBytes(), + expected_memory_allocated + bsz) || + CheckMemoryAllocated(arena.MemoryAllocatedBytes(), + expected_memory_allocated + huge_page_size)); + } else { + expected_memory_allocated += bsz; + ASSERT_PRED2(CheckMemoryAllocated, arena.MemoryAllocatedBytes(), + expected_memory_allocated); + } + + // requested size > size of a block: + // allocate requested size separately + expected_memory_allocated = arena.MemoryAllocatedBytes(); + req_sz = 8 * 1024 * 1024; + for (int i = 0; i < N; i++) { + arena.Allocate(req_sz); + } + expected_memory_allocated += req_sz * N; + ASSERT_PRED2(CheckMemoryAllocated, arena.MemoryAllocatedBytes(), + expected_memory_allocated); +} + +// Make sure we didn't count the allocate but not used memory space in +// Arena::ApproximateMemoryUsage() +static void ApproximateMemoryUsageTest(size_t huge_page_size) { + const size_t kBlockSize = 4096; + const size_t kEntrySize = kBlockSize / 8; + const size_t kZero = 0; + Arena arena(kBlockSize, nullptr, huge_page_size); + ASSERT_EQ(kZero, arena.ApproximateMemoryUsage()); + + // allocate inline bytes + const size_t kAlignUnit = alignof(max_align_t); + EXPECT_TRUE(arena.IsInInlineBlock()); + arena.AllocateAligned(kAlignUnit); + EXPECT_TRUE(arena.IsInInlineBlock()); + arena.AllocateAligned(Arena::kInlineSize / 2 - (2 * kAlignUnit)); + EXPECT_TRUE(arena.IsInInlineBlock()); + arena.AllocateAligned(Arena::kInlineSize / 2); + EXPECT_TRUE(arena.IsInInlineBlock()); + ASSERT_EQ(arena.ApproximateMemoryUsage(), Arena::kInlineSize - kAlignUnit); + ASSERT_PRED2(CheckMemoryAllocated, arena.MemoryAllocatedBytes(), + Arena::kInlineSize); + + auto num_blocks = kBlockSize / kEntrySize; + + // first allocation + arena.AllocateAligned(kEntrySize); + EXPECT_FALSE(arena.IsInInlineBlock()); + auto mem_usage = arena.MemoryAllocatedBytes(); + if (huge_page_size) { + ASSERT_TRUE( + CheckMemoryAllocated(mem_usage, kBlockSize + Arena::kInlineSize) || + CheckMemoryAllocated(mem_usage, huge_page_size + Arena::kInlineSize)); + } else { + ASSERT_PRED2(CheckMemoryAllocated, mem_usage, + kBlockSize + Arena::kInlineSize); + } + auto usage = arena.ApproximateMemoryUsage(); + ASSERT_LT(usage, mem_usage); + for (size_t i = 1; i < num_blocks; ++i) { + arena.AllocateAligned(kEntrySize); + ASSERT_EQ(mem_usage, arena.MemoryAllocatedBytes()); + ASSERT_EQ(arena.ApproximateMemoryUsage(), usage + kEntrySize); + EXPECT_FALSE(arena.IsInInlineBlock()); + usage = arena.ApproximateMemoryUsage(); + } + if (huge_page_size) { + ASSERT_TRUE(usage > mem_usage || + usage + huge_page_size - kBlockSize == mem_usage); + } else { + ASSERT_GT(usage, mem_usage); + } +} + +static void SimpleTest(size_t huge_page_size) { + std::vector> allocated; + Arena arena(Arena::kMinBlockSize, nullptr, huge_page_size); + const int N = 100000; + size_t bytes = 0; + Random rnd(301); + for (int i = 0; i < N; i++) { + size_t s; + if (i % (N / 10) == 0) { + s = i; + } else { + s = rnd.OneIn(4000) + ? rnd.Uniform(6000) + : (rnd.OneIn(10) ? rnd.Uniform(100) : rnd.Uniform(20)); + } + if (s == 0) { + // Our arena disallows size 0 allocations. + s = 1; + } + char* r; + if (rnd.OneIn(10)) { + r = arena.AllocateAligned(s); + } else { + r = arena.Allocate(s); + } + + for (unsigned int b = 0; b < s; b++) { + // Fill the "i"th allocation with a known bit pattern + r[b] = i % 256; + } + bytes += s; + allocated.push_back(std::make_pair(s, r)); + ASSERT_GE(arena.ApproximateMemoryUsage(), bytes); + if (i > N / 10) { + ASSERT_LE(arena.ApproximateMemoryUsage(), bytes * 1.10); + } + } + for (unsigned int i = 0; i < allocated.size(); i++) { + size_t num_bytes = allocated[i].first; + const char* p = allocated[i].second; + for (unsigned int b = 0; b < num_bytes; b++) { + // Check the "i"th allocation for the known bit pattern + ASSERT_EQ(int(p[b]) & 0xff, (int)(i % 256)); + } + } +} +} // namespace + +TEST_F(ArenaTest, MemoryAllocatedBytes) { + MemoryAllocatedBytesTest(0); + MemoryAllocatedBytesTest(kHugePageSize); +} + +TEST_F(ArenaTest, ApproximateMemoryUsage) { + ApproximateMemoryUsageTest(0); + ApproximateMemoryUsageTest(kHugePageSize); +} + +TEST_F(ArenaTest, Simple) { + SimpleTest(0); + SimpleTest(kHugePageSize); +} + +// Number of minor page faults since last call +size_t PopMinorPageFaultCount() { +#ifdef RUSAGE_SELF + static long prev = 0; + struct rusage usage; + EXPECT_EQ(getrusage(RUSAGE_SELF, &usage), 0); + size_t rv = usage.ru_minflt - prev; + prev = usage.ru_minflt; + return rv; +#else + // Conservative + return SIZE_MAX; +#endif // RUSAGE_SELF +} + +TEST(MmapTest, AllocateLazyZeroed) { + // Doesn't have to be page aligned + constexpr size_t len = 1234567; + MemMapping m = MemMapping::AllocateLazyZeroed(len); + auto arr = static_cast(m.Get()); + + // Should generally work + ASSERT_NE(arr, nullptr); + + // Start counting page faults + PopMinorPageFaultCount(); + + // Access half of the allocation + size_t i = 0; + for (; i < len / 2; ++i) { + ASSERT_EQ(arr[i], 0); + arr[i] = static_cast(i & 255); + } + + // Appropriate page faults (maybe more) + size_t faults = PopMinorPageFaultCount(); + ASSERT_GE(faults, len / 2 / port::kPageSize); + + // Access rest of the allocation + for (; i < len; ++i) { + ASSERT_EQ(arr[i], 0); + arr[i] = static_cast(i & 255); + } + + // Appropriate page faults (maybe more) + faults = PopMinorPageFaultCount(); + ASSERT_GE(faults, len / 2 / port::kPageSize); + + // Verify data + for (i = 0; i < len; ++i) { + ASSERT_EQ(arr[i], static_cast(i & 255)); + } +} + +TEST_F(ArenaTest, UnmappedAllocation) { + // Verify that it's possible to get unmapped pages in large allocations, + // for memory efficiency and to ensure we don't accidentally waste time & + // space initializing the memory. + constexpr size_t kBlockSize = 2U << 20; + Arena arena(kBlockSize); + + // The allocator might give us back recycled memory for a while, but + // shouldn't last forever. + for (int i = 0;; ++i) { + char* p = arena.Allocate(kBlockSize); + + // Start counting page faults + PopMinorPageFaultCount(); + + // Overwrite the whole allocation + for (size_t j = 0; j < kBlockSize; ++j) { + p[j] = static_cast(j & 255); + } + + size_t faults = PopMinorPageFaultCount(); + if (faults >= kBlockSize * 3 / 4 / port::kPageSize) { + // Most of the access generated page faults => GOOD + break; + } + // Should have succeeded after enough tries + ASSERT_LT(i, 1000); + } +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/memory/concurrent_arena.cc b/librocksdb-sys/rocksdb/memory/concurrent_arena.cc new file mode 100644 index 0000000..1619bd9 --- /dev/null +++ b/librocksdb-sys/rocksdb/memory/concurrent_arena.cc @@ -0,0 +1,45 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "memory/concurrent_arena.h" + +#include + +#include "port/port.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +thread_local size_t ConcurrentArena::tls_cpuid = 0; + +namespace { +// If the shard block size is too large, in the worst case, every core +// allocates a block without populate it. If the shared block size is +// 1MB, 64 cores will quickly allocate 64MB, and may quickly trigger a +// flush. Cap the size instead. +const size_t kMaxShardBlockSize = size_t{128 * 1024}; +} // namespace + +ConcurrentArena::ConcurrentArena(size_t block_size, AllocTracker* tracker, + size_t huge_page_size) + : shard_block_size_(std::min(kMaxShardBlockSize, block_size / 8)), + shards_(), + arena_(block_size, tracker, huge_page_size) { + Fixup(); +} + +ConcurrentArena::Shard* ConcurrentArena::Repick() { + auto shard_and_index = shards_.AccessElementAndIndex(); + // even if we are cpu 0, use a non-zero tls_cpuid so we can tell we + // have repicked + tls_cpuid = shard_and_index.second | shards_.Size(); + return shard_and_index.first; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/memory/concurrent_arena.h b/librocksdb-sys/rocksdb/memory/concurrent_arena.h new file mode 100644 index 0000000..f14507d --- /dev/null +++ b/librocksdb-sys/rocksdb/memory/concurrent_arena.h @@ -0,0 +1,215 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include +#include +#include + +#include "memory/allocator.h" +#include "memory/arena.h" +#include "port/lang.h" +#include "port/likely.h" +#include "util/core_local.h" +#include "util/mutexlock.h" +#include "util/thread_local.h" + +// Only generate field unused warning for padding array, or build under +// GCC 4.8.1 will fail. +#ifdef __clang__ +#define ROCKSDB_FIELD_UNUSED __attribute__((__unused__)) +#else +#define ROCKSDB_FIELD_UNUSED +#endif // __clang__ + +namespace ROCKSDB_NAMESPACE { + +class Logger; + +// ConcurrentArena wraps an Arena. It makes it thread safe using a fast +// inlined spinlock, and adds small per-core allocation caches to avoid +// contention for small allocations. To avoid any memory waste from the +// per-core shards, they are kept small, they are lazily instantiated +// only if ConcurrentArena actually notices concurrent use, and they +// adjust their size so that there is no fragmentation waste when the +// shard blocks are allocated from the underlying main arena. +class ConcurrentArena : public Allocator { + public: + // block_size and huge_page_size are the same as for Arena (and are + // in fact just passed to the constructor of arena_. The core-local + // shards compute their shard_block_size as a fraction of block_size + // that varies according to the hardware concurrency level. + explicit ConcurrentArena(size_t block_size = Arena::kMinBlockSize, + AllocTracker* tracker = nullptr, + size_t huge_page_size = 0); + + char* Allocate(size_t bytes) override { + return AllocateImpl(bytes, false /*force_arena*/, + [this, bytes]() { return arena_.Allocate(bytes); }); + } + + char* AllocateAligned(size_t bytes, size_t huge_page_size = 0, + Logger* logger = nullptr) override { + size_t rounded_up = ((bytes - 1) | (sizeof(void*) - 1)) + 1; + assert(rounded_up >= bytes && rounded_up < bytes + sizeof(void*) && + (rounded_up % sizeof(void*)) == 0); + + return AllocateImpl(rounded_up, huge_page_size != 0 /*force_arena*/, + [this, rounded_up, huge_page_size, logger]() { + return arena_.AllocateAligned(rounded_up, + huge_page_size, logger); + }); + } + + size_t ApproximateMemoryUsage() const { + std::unique_lock lock(arena_mutex_, std::defer_lock); + lock.lock(); + return arena_.ApproximateMemoryUsage() - ShardAllocatedAndUnused(); + } + + size_t MemoryAllocatedBytes() const { + return memory_allocated_bytes_.load(std::memory_order_relaxed); + } + + size_t AllocatedAndUnused() const { + return arena_allocated_and_unused_.load(std::memory_order_relaxed) + + ShardAllocatedAndUnused(); + } + + size_t IrregularBlockNum() const { + return irregular_block_num_.load(std::memory_order_relaxed); + } + + size_t BlockSize() const override { return arena_.BlockSize(); } + + private: + struct Shard { + char padding[40] ROCKSDB_FIELD_UNUSED; + mutable SpinMutex mutex; + char* free_begin_; + std::atomic allocated_and_unused_; + + Shard() : free_begin_(nullptr), allocated_and_unused_(0) {} + }; + + static thread_local size_t tls_cpuid; + + char padding0[56] ROCKSDB_FIELD_UNUSED; + + size_t shard_block_size_; + + CoreLocalArray shards_; + + Arena arena_; + mutable SpinMutex arena_mutex_; + std::atomic arena_allocated_and_unused_; + std::atomic memory_allocated_bytes_; + std::atomic irregular_block_num_; + + char padding1[56] ROCKSDB_FIELD_UNUSED; + + Shard* Repick(); + + size_t ShardAllocatedAndUnused() const { + size_t total = 0; + for (size_t i = 0; i < shards_.Size(); ++i) { + total += shards_.AccessAtCore(i)->allocated_and_unused_.load( + std::memory_order_relaxed); + } + return total; + } + + template + char* AllocateImpl(size_t bytes, bool force_arena, const Func& func) { + size_t cpu; + + // Go directly to the arena if the allocation is too large, or if + // we've never needed to Repick() and the arena mutex is available + // with no waiting. This keeps the fragmentation penalty of + // concurrency zero unless it might actually confer an advantage. + std::unique_lock arena_lock(arena_mutex_, std::defer_lock); + if (bytes > shard_block_size_ / 4 || force_arena || + ((cpu = tls_cpuid) == 0 && + !shards_.AccessAtCore(0)->allocated_and_unused_.load( + std::memory_order_relaxed) && + arena_lock.try_lock())) { + if (!arena_lock.owns_lock()) { + arena_lock.lock(); + } + auto rv = func(); + Fixup(); + return rv; + } + + // pick a shard from which to allocate + Shard* s = shards_.AccessAtCore(cpu & (shards_.Size() - 1)); + if (!s->mutex.try_lock()) { + s = Repick(); + s->mutex.lock(); + } + std::unique_lock lock(s->mutex, std::adopt_lock); + + size_t avail = s->allocated_and_unused_.load(std::memory_order_relaxed); + if (avail < bytes) { + // reload + std::lock_guard reload_lock(arena_mutex_); + + // If the arena's current block is within a factor of 2 of the right + // size, we adjust our request to avoid arena waste. + auto exact = arena_allocated_and_unused_.load(std::memory_order_relaxed); + assert(exact == arena_.AllocatedAndUnused()); + + if (exact >= bytes && arena_.IsInInlineBlock()) { + // If we haven't exhausted arena's inline block yet, allocate from arena + // directly. This ensures that we'll do the first few small allocations + // without allocating any blocks. + // In particular this prevents empty memtables from using + // disproportionately large amount of memory: a memtable allocates on + // the order of 1 KB of memory when created; we wouldn't want to + // allocate a full arena block (typically a few megabytes) for that, + // especially if there are thousands of empty memtables. + auto rv = func(); + Fixup(); + return rv; + } + + avail = exact >= shard_block_size_ / 2 && exact < shard_block_size_ * 2 + ? exact + : shard_block_size_; + s->free_begin_ = arena_.AllocateAligned(avail); + Fixup(); + } + s->allocated_and_unused_.store(avail - bytes, std::memory_order_relaxed); + + char* rv; + if ((bytes % sizeof(void*)) == 0) { + // aligned allocation from the beginning + rv = s->free_begin_; + s->free_begin_ += bytes; + } else { + // unaligned from the end + rv = s->free_begin_ + avail - bytes; + } + return rv; + } + + void Fixup() { + arena_allocated_and_unused_.store(arena_.AllocatedAndUnused(), + std::memory_order_relaxed); + memory_allocated_bytes_.store(arena_.MemoryAllocatedBytes(), + std::memory_order_relaxed); + irregular_block_num_.store(arena_.IrregularBlockNum(), + std::memory_order_relaxed); + } + + ConcurrentArena(const ConcurrentArena&) = delete; + ConcurrentArena& operator=(const ConcurrentArena&) = delete; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/memory/jemalloc_nodump_allocator.cc b/librocksdb-sys/rocksdb/memory/jemalloc_nodump_allocator.cc new file mode 100644 index 0000000..d052482 --- /dev/null +++ b/librocksdb-sys/rocksdb/memory/jemalloc_nodump_allocator.cc @@ -0,0 +1,303 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "memory/jemalloc_nodump_allocator.h" + +#include +#include + +#include "port/likely.h" +#include "port/port.h" +#include "rocksdb/convenience.h" +#include "rocksdb/utilities/customizable_util.h" +#include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/options_type.h" +#include "util/fastrange.h" +#include "util/random.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +#ifdef ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR +std::atomic JemallocNodumpAllocator::original_alloc_{nullptr}; +#endif // ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR + +static std::unordered_map jemalloc_type_info = { + {"limit_tcache_size", + {offsetof(struct JemallocAllocatorOptions, limit_tcache_size), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"tcache_size_lower_bound", + {offsetof(struct JemallocAllocatorOptions, tcache_size_lower_bound), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"tcache_size_upper_bound", + {offsetof(struct JemallocAllocatorOptions, tcache_size_upper_bound), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"num_arenas", + {offsetof(struct JemallocAllocatorOptions, num_arenas), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, +}; +bool JemallocNodumpAllocator::IsSupported(std::string* why) { +#ifndef ROCKSDB_JEMALLOC + *why = "Not compiled with ROCKSDB_JEMALLOC"; + return false; +#else + static const std::string unsupported = + "JemallocNodumpAllocator only available with jemalloc version >= 5 " + "and MADV_DONTDUMP is available."; + if (!HasJemalloc()) { + *why = unsupported; + return false; + } +#ifndef ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR + *why = unsupported; + return false; +#else + return true; +#endif // ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR +#endif // ROCKSDB_MALLOC +} + +JemallocNodumpAllocator::JemallocNodumpAllocator( + JemallocAllocatorOptions& options) + : options_(options) +#ifdef ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR + , + tcache_(&JemallocNodumpAllocator::DestroyThreadSpecificCache) { +#else // ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR +{ +#endif // ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR + RegisterOptions(&options_, &jemalloc_type_info); +} + +#ifdef ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR +JemallocNodumpAllocator::~JemallocNodumpAllocator() { + // Destroy tcache before destroying arena. + autovector tcache_list; + tcache_.Scrape(&tcache_list, nullptr); + for (void* tcache_index : tcache_list) { + DestroyThreadSpecificCache(tcache_index); + } + for (auto arena_index : arena_indexes_) { + // Destroy arena. Silently ignore error. + Status s = DestroyArena(arena_index); + assert(s.ok()); + s.PermitUncheckedError(); + } +} + +size_t JemallocNodumpAllocator::UsableSize(void* p, + size_t /*allocation_size*/) const { + return malloc_usable_size(static_cast(p)); +} + +void* JemallocNodumpAllocator::Allocate(size_t size) { + int tcache_flag = GetThreadSpecificCache(size); + uint32_t arena_index = GetArenaIndex(); + return mallocx(size, MALLOCX_ARENA(arena_index) | tcache_flag); +} + +void JemallocNodumpAllocator::Deallocate(void* p) { + // Obtain tcache. + size_t size = 0; + if (options_.limit_tcache_size) { + size = malloc_usable_size(p); + } + int tcache_flag = GetThreadSpecificCache(size); + // No need to pass arena index to dallocx(). Jemalloc will find arena index + // from its own metadata. + dallocx(p, tcache_flag); +} + +uint32_t JemallocNodumpAllocator::GetArenaIndex() const { + if (arena_indexes_.size() == 1) { + return arena_indexes_[0]; + } + + static std::atomic next_seed = 0; + // Core-local may work in place of `thread_local` as we should be able to + // tolerate occasional stale reads in thread migration cases. However we need + // to make Random thread-safe and prevent cacheline bouncing. Whether this is + // worthwhile is still an open question. + thread_local Random tl_random(next_seed.fetch_add(1)); + return arena_indexes_[FastRange32(tl_random.Next(), arena_indexes_.size())]; +} + +Status JemallocNodumpAllocator::InitializeArenas() { + assert(!init_); + init_ = true; + + for (size_t i = 0; i < options_.num_arenas; i++) { + // Create arena. + unsigned arena_index; + size_t arena_index_size = sizeof(arena_index); + int ret = + mallctl("arenas.create", &arena_index, &arena_index_size, nullptr, 0); + if (ret != 0) { + return Status::Incomplete( + "Failed to create jemalloc arena, error code: " + + std::to_string(ret)); + } + arena_indexes_.push_back(arena_index); + + // Read existing hooks. + std::string key = + "arena." + std::to_string(arena_indexes_[i]) + ".extent_hooks"; + extent_hooks_t* hooks; + size_t hooks_size = sizeof(hooks); + ret = mallctl(key.c_str(), &hooks, &hooks_size, nullptr, 0); + if (ret != 0) { + return Status::Incomplete("Failed to read existing hooks, error code: " + + std::to_string(ret)); + } + + // Store existing alloc. + extent_alloc_t* original_alloc = hooks->alloc; + extent_alloc_t* expected = nullptr; + bool success = + JemallocNodumpAllocator::original_alloc_.compare_exchange_strong( + expected, original_alloc); + if (!success && original_alloc != expected) { + // This could happen if jemalloc creates new arenas with different initial + // values in their `alloc` function pointers. See `original_alloc_` API + // doc for more details. + return Status::Incomplete("Original alloc conflict."); + } + + // Set the custom hook. + per_arena_hooks_.emplace_back(); + per_arena_hooks_.back().reset(new extent_hooks_t(*hooks)); + per_arena_hooks_.back()->alloc = &JemallocNodumpAllocator::Alloc; + extent_hooks_t* hooks_ptr = per_arena_hooks_.back().get(); + ret = mallctl(key.c_str(), nullptr, nullptr, &hooks_ptr, sizeof(hooks_ptr)); + if (ret != 0) { + return Status::Incomplete("Failed to set custom hook, error code: " + + std::to_string(ret)); + } + } + return Status::OK(); +} + +#endif // ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR + +Status JemallocNodumpAllocator::PrepareOptions( + const ConfigOptions& config_options) { + std::string message; + + if (!IsSupported(&message)) { + return Status::NotSupported(message); + } else if (options_.limit_tcache_size && + options_.tcache_size_lower_bound >= + options_.tcache_size_upper_bound) { + return Status::InvalidArgument( + "tcache_size_lower_bound larger or equal to tcache_size_upper_bound."); + } else if (options_.num_arenas < 1) { + return Status::InvalidArgument("num_arenas must be a positive integer"); + } else if (IsMutable()) { + Status s = MemoryAllocator::PrepareOptions(config_options); +#ifdef ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR + if (s.ok()) { + s = InitializeArenas(); + } +#endif // ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR + return s; + } else { + // Already prepared + return Status::OK(); + } +} + +#ifdef ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR +int JemallocNodumpAllocator::GetThreadSpecificCache(size_t size) { + // We always enable tcache. The only corner case is when there are a ton of + // threads accessing with low frequency, then it could consume a lot of + // memory (may reach # threads * ~1MB) without bringing too much benefit. + if (options_.limit_tcache_size && (size <= options_.tcache_size_lower_bound || + size > options_.tcache_size_upper_bound)) { + return MALLOCX_TCACHE_NONE; + } + unsigned* tcache_index = reinterpret_cast(tcache_.Get()); + if (UNLIKELY(tcache_index == nullptr)) { + // Instantiate tcache. + tcache_index = new unsigned(0); + size_t tcache_index_size = sizeof(unsigned); + int ret = + mallctl("tcache.create", tcache_index, &tcache_index_size, nullptr, 0); + if (ret != 0) { + // No good way to expose the error. Silently disable tcache. + delete tcache_index; + return MALLOCX_TCACHE_NONE; + } + tcache_.Reset(static_cast(tcache_index)); + } + return MALLOCX_TCACHE(*tcache_index); +} +void* JemallocNodumpAllocator::Alloc(extent_hooks_t* extent, void* new_addr, + size_t size, size_t alignment, bool* zero, + bool* commit, unsigned arena_ind) { + extent_alloc_t* original_alloc = + original_alloc_.load(std::memory_order_relaxed); + assert(original_alloc != nullptr); + void* result = original_alloc(extent, new_addr, size, alignment, zero, commit, + arena_ind); + if (result != nullptr) { + int ret = madvise(result, size, MADV_DONTDUMP); + if (ret != 0) { + fprintf( + stderr, + "JemallocNodumpAllocator failed to set MADV_DONTDUMP, error code: %d", + ret); + assert(false); + } + } + return result; +} + +Status JemallocNodumpAllocator::DestroyArena(uint32_t arena_index) { + assert(arena_index != 0); + std::string key = "arena." + std::to_string(arena_index) + ".destroy"; + int ret = mallctl(key.c_str(), nullptr, 0, nullptr, 0); + if (ret != 0) { + return Status::Incomplete("Failed to destroy jemalloc arena, error code: " + + std::to_string(ret)); + } + return Status::OK(); +} + +void JemallocNodumpAllocator::DestroyThreadSpecificCache(void* ptr) { + assert(ptr != nullptr); + unsigned* tcache_index = static_cast(ptr); + size_t tcache_index_size = sizeof(unsigned); + int ret __attribute__((__unused__)) = + mallctl("tcache.destroy", nullptr, 0, tcache_index, tcache_index_size); + // Silently ignore error. + assert(ret == 0); + delete tcache_index; +} + +#endif // ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR + +Status NewJemallocNodumpAllocator( + JemallocAllocatorOptions& options, + std::shared_ptr* memory_allocator) { + if (memory_allocator == nullptr) { + return Status::InvalidArgument("memory_allocator must be non-null."); + } +#ifndef ROCKSDB_JEMALLOC + (void)options; + return Status::NotSupported("Not compiled with JEMALLOC"); +#else + std::unique_ptr allocator( + new JemallocNodumpAllocator(options)); + Status s = allocator->PrepareOptions(ConfigOptions()); + if (s.ok()) { + memory_allocator->reset(allocator.release()); + } + return s; +#endif +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/memory/jemalloc_nodump_allocator.h b/librocksdb-sys/rocksdb/memory/jemalloc_nodump_allocator.h new file mode 100644 index 0000000..2bdbaeb --- /dev/null +++ b/librocksdb-sys/rocksdb/memory/jemalloc_nodump_allocator.h @@ -0,0 +1,99 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include + +#include "port/jemalloc_helper.h" +#include "port/port.h" +#include "rocksdb/memory_allocator.h" +#include "util/thread_local.h" +#include "utilities/memory_allocators.h" + +#if defined(ROCKSDB_JEMALLOC) && defined(ROCKSDB_PLATFORM_POSIX) + +#include + +#if (JEMALLOC_VERSION_MAJOR >= 5) && defined(MADV_DONTDUMP) +#define ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR +#endif // (JEMALLOC_VERSION_MAJOR >= 5) && MADV_DONTDUMP +#endif // ROCKSDB_JEMALLOC && ROCKSDB_PLATFORM_POSIX + +namespace ROCKSDB_NAMESPACE { + +// Allocation requests are randomly sharded across +// `JemallocAllocatorOptions::num_arenas` arenas to reduce contention on per- +// arena mutexes. +class JemallocNodumpAllocator : public BaseMemoryAllocator { + public: + explicit JemallocNodumpAllocator(JemallocAllocatorOptions& options); +#ifdef ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR + ~JemallocNodumpAllocator(); +#endif // ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR + + static const char* kClassName() { return "JemallocNodumpAllocator"; } + const char* Name() const override { return kClassName(); } + static bool IsSupported() { + std::string unused; + return IsSupported(&unused); + } + static bool IsSupported(std::string* why); + bool IsMutable() const { return !init_; } + + Status PrepareOptions(const ConfigOptions& config_options) override; + +#ifdef ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR + void* Allocate(size_t size) override; + void Deallocate(void* p) override; + size_t UsableSize(void* p, size_t allocation_size) const override; +#endif // ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR + + private: +#ifdef ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR + Status InitializeArenas(); + + uint32_t GetArenaIndex() const; + + // Custom alloc hook to replace jemalloc default alloc. + static void* Alloc(extent_hooks_t* extent, void* new_addr, size_t size, + size_t alignment, bool* zero, bool* commit, + unsigned arena_ind); + + // Destroy arena on destruction of the allocator, or on failure. + static Status DestroyArena(uint32_t arena_index); + + // Destroy tcache on destruction of the allocator, or thread exit. + static void DestroyThreadSpecificCache(void* ptr); + + // Get or create tcache. Return flag suitable to use with `mallocx`: + // either MALLOCX_TCACHE_NONE or MALLOCX_TCACHE(tc). + int GetThreadSpecificCache(size_t size); +#endif // ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR + JemallocAllocatorOptions options_; + +#ifdef ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR + // A function pointer to jemalloc default alloc. Use atomic to make sure + // NewJemallocNodumpAllocator is thread-safe. + // + // Hack: original_alloc_ needs to be static for Alloc() to access it. + // alloc needs to be static to pass to jemalloc as function pointer. We can + // use a single process-wide value as long as we assume that any newly created + // arena has the same original value in its `alloc` function pointer. + static std::atomic original_alloc_; + + // Custom hooks has to outlive corresponding arena. + std::vector> per_arena_hooks_; + + // Hold thread-local tcache index. + ThreadLocalPtr tcache_; + + std::vector arena_indexes_; +#endif // ROCKSDB_JEMALLOC_NODUMP_ALLOCATOR + + bool init_ = false; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/memory/memkind_kmem_allocator.cc b/librocksdb-sys/rocksdb/memory/memkind_kmem_allocator.cc new file mode 100644 index 0000000..635c221 --- /dev/null +++ b/librocksdb-sys/rocksdb/memory/memkind_kmem_allocator.cc @@ -0,0 +1,44 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// Copyright (c) 2019 Intel Corporation +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#ifdef MEMKIND +#include +#endif // MEMKIND + +#include "memory/memkind_kmem_allocator.h" + +namespace ROCKSDB_NAMESPACE { +Status MemkindKmemAllocator::PrepareOptions(const ConfigOptions& options) { + std::string message; + if (!IsSupported(&message)) { + return Status::NotSupported(message); + } else { + return MemoryAllocator::PrepareOptions(options); + } +} + +#ifdef MEMKIND +void* MemkindKmemAllocator::Allocate(size_t size) { + void* p = memkind_malloc(MEMKIND_DAX_KMEM, size); + if (p == NULL) { + throw std::bad_alloc(); + } + return p; +} + +void MemkindKmemAllocator::Deallocate(void* p) { + memkind_free(MEMKIND_DAX_KMEM, p); +} + +#ifdef ROCKSDB_MALLOC_USABLE_SIZE +size_t MemkindKmemAllocator::UsableSize(void* p, + size_t /*allocation_size*/) const { + return memkind_malloc_usable_size(MEMKIND_DAX_KMEM, p); +} +#endif // ROCKSDB_MALLOC_USABLE_SIZE +#endif // MEMKIND + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/memory/memkind_kmem_allocator.h b/librocksdb-sys/rocksdb/memory/memkind_kmem_allocator.h new file mode 100644 index 0000000..7176f17 --- /dev/null +++ b/librocksdb-sys/rocksdb/memory/memkind_kmem_allocator.h @@ -0,0 +1,43 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// Copyright (c) 2019 Intel Corporation +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "rocksdb/memory_allocator.h" +#include "utilities/memory_allocators.h" + +namespace ROCKSDB_NAMESPACE { + +class MemkindKmemAllocator : public BaseMemoryAllocator { + public: + static const char* kClassName() { return "MemkindKmemAllocator"; } + const char* Name() const override { return kClassName(); } + static bool IsSupported() { + std::string unused; + return IsSupported(&unused); + } + + static bool IsSupported(std::string* msg) { +#ifdef MEMKIND + (void)msg; + return true; +#else + *msg = "Not compiled with MemKind"; + return false; +#endif + } + Status PrepareOptions(const ConfigOptions& options) override; + +#ifdef MEMKIND + void* Allocate(size_t size) override; + void Deallocate(void* p) override; +#ifdef ROCKSDB_MALLOC_USABLE_SIZE + size_t UsableSize(void* p, size_t /*allocation_size*/) const override; +#endif +#endif // MEMKIND +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/memory/memory_allocator.cc b/librocksdb-sys/rocksdb/memory/memory_allocator.cc new file mode 100644 index 0000000..d0de26b --- /dev/null +++ b/librocksdb-sys/rocksdb/memory/memory_allocator.cc @@ -0,0 +1,80 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "rocksdb/memory_allocator.h" + +#include "memory/jemalloc_nodump_allocator.h" +#include "memory/memkind_kmem_allocator.h" +#include "rocksdb/utilities/customizable_util.h" +#include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/options_type.h" +#include "utilities/memory_allocators.h" + +namespace ROCKSDB_NAMESPACE { +namespace { +static std::unordered_map ma_wrapper_type_info = { + {"target", OptionTypeInfo::AsCustomSharedPtr( + 0, OptionVerificationType::kByName, OptionTypeFlags::kNone)}, +}; + +static int RegisterBuiltinAllocators(ObjectLibrary& library, + const std::string& /*arg*/) { + library.AddFactory( + DefaultMemoryAllocator::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /*errmsg*/) { + guard->reset(new DefaultMemoryAllocator()); + return guard->get(); + }); + library.AddFactory( + CountedMemoryAllocator::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /*errmsg*/) { + guard->reset(new CountedMemoryAllocator( + std::make_shared())); + return guard->get(); + }); + library.AddFactory( + JemallocNodumpAllocator::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* errmsg) { + if (JemallocNodumpAllocator::IsSupported(errmsg)) { + JemallocAllocatorOptions options; + guard->reset(new JemallocNodumpAllocator(options)); + } + return guard->get(); + }); + library.AddFactory( + MemkindKmemAllocator::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* errmsg) { + if (MemkindKmemAllocator::IsSupported(errmsg)) { + guard->reset(new MemkindKmemAllocator()); + } + return guard->get(); + }); + size_t num_types; + return static_cast(library.GetFactoryCount(&num_types)); +} +} // namespace + +MemoryAllocatorWrapper::MemoryAllocatorWrapper( + const std::shared_ptr& t) + : target_(t) { + RegisterOptions("", &target_, &ma_wrapper_type_info); +} + +Status MemoryAllocator::CreateFromString( + const ConfigOptions& options, const std::string& value, + std::shared_ptr* result) { + static std::once_flag once; + std::call_once(once, [&]() { + RegisterBuiltinAllocators(*(ObjectLibrary::Default().get()), ""); + }); + ConfigOptions copy = options; + copy.invoke_prepare_options = true; + return LoadManagedObject(copy, value, result); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/memory/memory_allocator_impl.h b/librocksdb-sys/rocksdb/memory/memory_allocator_impl.h new file mode 100644 index 0000000..68aa35b --- /dev/null +++ b/librocksdb-sys/rocksdb/memory/memory_allocator_impl.h @@ -0,0 +1,47 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#pragma once + +#include + +#include "rocksdb/memory_allocator.h" + +namespace ROCKSDB_NAMESPACE { + +struct CustomDeleter { + CustomDeleter(MemoryAllocator* a = nullptr) : allocator(a) {} + + void operator()(char* ptr) const { + if (allocator) { + allocator->Deallocate(reinterpret_cast(ptr)); + } else { + delete[] ptr; + } + } + + MemoryAllocator* allocator; +}; + +using CacheAllocationPtr = std::unique_ptr; + +inline CacheAllocationPtr AllocateBlock(size_t size, + MemoryAllocator* allocator) { + if (allocator) { + auto block = reinterpret_cast(allocator->Allocate(size)); + return CacheAllocationPtr(block, allocator); + } + return CacheAllocationPtr(new char[size]); +} + +inline CacheAllocationPtr AllocateAndCopyBlock(const Slice& data, + MemoryAllocator* allocator) { + CacheAllocationPtr cap = AllocateBlock(data.size(), allocator); + std::copy_n(data.data(), data.size(), cap.get()); + return cap; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/memory/memory_allocator_test.cc b/librocksdb-sys/rocksdb/memory/memory_allocator_test.cc new file mode 100644 index 0000000..6616e1c --- /dev/null +++ b/librocksdb-sys/rocksdb/memory/memory_allocator_test.cc @@ -0,0 +1,236 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// Copyright (c) 2019 Intel Corporation +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include + +#include "memory/jemalloc_nodump_allocator.h" +#include "memory/memkind_kmem_allocator.h" +#include "rocksdb/cache.h" +#include "rocksdb/convenience.h" +#include "rocksdb/db.h" +#include "rocksdb/options.h" +#include "table/block_based/block_based_table_factory.h" +#include "test_util/testharness.h" +#include "utilities/memory_allocators.h" + +namespace ROCKSDB_NAMESPACE { + +// TODO: the tests do not work in LITE mode due to relying on +// `CreateFromString()` to create non-default memory allocators. + +class MemoryAllocatorTest + : public testing::Test, + public ::testing::WithParamInterface> { + public: + MemoryAllocatorTest() { + std::tie(id_, supported_) = GetParam(); + Status s = + MemoryAllocator::CreateFromString(ConfigOptions(), id_, &allocator_); + EXPECT_EQ(supported_, s.ok()); + } + bool IsSupported() { return supported_; } + + std::shared_ptr allocator_; + std::string id_; + + private: + bool supported_; +}; + +TEST_P(MemoryAllocatorTest, Allocate) { + if (!IsSupported()) { + return; + } + void* p = allocator_->Allocate(1024); + ASSERT_NE(p, nullptr); + size_t size = allocator_->UsableSize(p, 1024); + ASSERT_GE(size, 1024); + allocator_->Deallocate(p); +} + +TEST_P(MemoryAllocatorTest, CreateAllocator) { + ConfigOptions config_options; + config_options.ignore_unknown_options = false; + config_options.ignore_unsupported_options = false; + std::shared_ptr orig, copy; + Status s = MemoryAllocator::CreateFromString(config_options, id_, &orig); + if (!IsSupported()) { + ASSERT_TRUE(s.IsNotSupported()); + } else { + ASSERT_OK(s); + ASSERT_NE(orig, nullptr); + std::string str = orig->ToString(config_options); + ASSERT_OK(MemoryAllocator::CreateFromString(config_options, str, ©)); + ASSERT_EQ(orig, copy); + } +} + +TEST_P(MemoryAllocatorTest, DatabaseBlockCache) { + if (!IsSupported()) { + // Check if a memory node is available for allocation + } + + // Create database with block cache using the MemoryAllocator + Options options; + std::string dbname = test::PerThreadDBPath("allocator_test"); + ASSERT_OK(DestroyDB(dbname, options)); + + options.create_if_missing = true; + BlockBasedTableOptions table_options; + auto cache = NewLRUCache(1024 * 1024, 6, false, 0.0, allocator_); + table_options.block_cache = cache; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DB* db = nullptr; + Status s = DB::Open(options, dbname, &db); + ASSERT_OK(s); + ASSERT_NE(db, nullptr); + ASSERT_LE(cache->GetUsage(), 104); // Cache will contain stats + + // Write 2kB (200 values, each 10 bytes) + int num_keys = 200; + WriteOptions wo; + std::string val = "0123456789"; + for (int i = 0; i < num_keys; i++) { + std::string key = std::to_string(i); + s = db->Put(wo, Slice(key), Slice(val)); + ASSERT_OK(s); + } + ASSERT_OK(db->Flush(FlushOptions())); // Flush all data from memtable so that + // reads are from block cache + + // Read and check block cache usage + ReadOptions ro; + std::string result; + for (int i = 0; i < num_keys; i++) { + std::string key = std::to_string(i); + s = db->Get(ro, key, &result); + ASSERT_OK(s); + ASSERT_EQ(result, val); + } + ASSERT_GT(cache->GetUsage(), 2000); + + // Close database + s = db->Close(); + ASSERT_OK(s); + delete db; + ASSERT_OK(DestroyDB(dbname, options)); +} + +class CreateMemoryAllocatorTest : public testing::Test { + public: + CreateMemoryAllocatorTest() { + config_options_.ignore_unknown_options = false; + config_options_.ignore_unsupported_options = false; + } + ConfigOptions config_options_; +}; + +TEST_F(CreateMemoryAllocatorTest, JemallocOptionsTest) { + std::shared_ptr allocator; + std::string id = std::string("id=") + JemallocNodumpAllocator::kClassName(); + Status s = MemoryAllocator::CreateFromString(config_options_, id, &allocator); + if (!JemallocNodumpAllocator::IsSupported()) { + ASSERT_NOK(s); + ROCKSDB_GTEST_BYPASS("JEMALLOC not supported"); + return; + } + ASSERT_OK(s); + ASSERT_NE(allocator, nullptr); + JemallocAllocatorOptions jopts; + auto opts = allocator->GetOptions(); + ASSERT_NE(opts, nullptr); + ASSERT_EQ(opts->limit_tcache_size, jopts.limit_tcache_size); + ASSERT_EQ(opts->tcache_size_lower_bound, jopts.tcache_size_lower_bound); + ASSERT_EQ(opts->tcache_size_upper_bound, jopts.tcache_size_upper_bound); + + ASSERT_NOK(MemoryAllocator::CreateFromString( + config_options_, + id + "; limit_tcache_size=true; tcache_size_lower_bound=4096; " + "tcache_size_upper_bound=1024", + &allocator)); + ASSERT_OK(MemoryAllocator::CreateFromString( + config_options_, + id + "; limit_tcache_size=false; tcache_size_lower_bound=4096; " + "tcache_size_upper_bound=1024", + &allocator)); + opts = allocator->GetOptions(); + ASSERT_NE(opts, nullptr); + ASSERT_EQ(opts->limit_tcache_size, false); + ASSERT_EQ(opts->tcache_size_lower_bound, 4096U); + ASSERT_EQ(opts->tcache_size_upper_bound, 1024U); + ASSERT_OK(MemoryAllocator::CreateFromString( + config_options_, + id + "; limit_tcache_size=true; tcache_size_upper_bound=4096; " + "tcache_size_lower_bound=1024", + &allocator)); + opts = allocator->GetOptions(); + ASSERT_NE(opts, nullptr); + ASSERT_EQ(opts->limit_tcache_size, true); + ASSERT_EQ(opts->tcache_size_lower_bound, 1024U); + ASSERT_EQ(opts->tcache_size_upper_bound, 4096U); +} + +TEST_F(CreateMemoryAllocatorTest, NewJemallocNodumpAllocator) { + JemallocAllocatorOptions jopts; + std::shared_ptr allocator; + + jopts.limit_tcache_size = true; + jopts.tcache_size_lower_bound = 2 * 1024; + jopts.tcache_size_upper_bound = 1024; + + ASSERT_NOK(NewJemallocNodumpAllocator(jopts, nullptr)); + Status s = NewJemallocNodumpAllocator(jopts, &allocator); + std::string msg; + if (!JemallocNodumpAllocator::IsSupported(&msg)) { + ASSERT_NOK(s); + ROCKSDB_GTEST_BYPASS("JEMALLOC not supported"); + return; + } + ASSERT_NOK(s); // Invalid options + ASSERT_EQ(allocator, nullptr); + + jopts.tcache_size_upper_bound = 4 * 1024; + ASSERT_OK(NewJemallocNodumpAllocator(jopts, &allocator)); + ASSERT_NE(allocator, nullptr); + auto opts = allocator->GetOptions(); + ASSERT_EQ(opts->tcache_size_upper_bound, jopts.tcache_size_upper_bound); + ASSERT_EQ(opts->tcache_size_lower_bound, jopts.tcache_size_lower_bound); + ASSERT_EQ(opts->limit_tcache_size, jopts.limit_tcache_size); + + jopts.limit_tcache_size = false; + ASSERT_OK(NewJemallocNodumpAllocator(jopts, &allocator)); + ASSERT_NE(allocator, nullptr); + opts = allocator->GetOptions(); + ASSERT_EQ(opts->tcache_size_upper_bound, jopts.tcache_size_upper_bound); + ASSERT_EQ(opts->tcache_size_lower_bound, jopts.tcache_size_lower_bound); + ASSERT_EQ(opts->limit_tcache_size, jopts.limit_tcache_size); +} + +INSTANTIATE_TEST_CASE_P(DefaultMemoryAllocator, MemoryAllocatorTest, + ::testing::Values(std::make_tuple( + DefaultMemoryAllocator::kClassName(), true))); +#ifdef MEMKIND +INSTANTIATE_TEST_CASE_P( + MemkindkMemAllocator, MemoryAllocatorTest, + ::testing::Values(std::make_tuple(MemkindKmemAllocator::kClassName(), + MemkindKmemAllocator::IsSupported()))); +#endif // MEMKIND + +#ifdef ROCKSDB_JEMALLOC +INSTANTIATE_TEST_CASE_P( + JemallocNodumpAllocator, MemoryAllocatorTest, + ::testing::Values(std::make_tuple(JemallocNodumpAllocator::kClassName(), + JemallocNodumpAllocator::IsSupported()))); +#endif // ROCKSDB_JEMALLOC + + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/memory/memory_usage.h b/librocksdb-sys/rocksdb/memory/memory_usage.h new file mode 100644 index 0000000..76b9bd1 --- /dev/null +++ b/librocksdb-sys/rocksdb/memory/memory_usage.h @@ -0,0 +1,38 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#ifdef USE_FOLLY +#include +#endif + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +// Helper methods to estimate memroy usage by std containers. + +template +size_t ApproximateMemoryUsage( + const std::unordered_map& umap) { + using Map = std::unordered_map; + return sizeof(umap) + + // Size of all items plus a next pointer for each item. + (sizeof(typename Map::value_type) + sizeof(void*)) * umap.size() + + // Size of hash buckets. + umap.bucket_count() * sizeof(void*); +} + +#ifdef USE_FOLLY +template +size_t ApproximateMemoryUsage(const folly::F14FastMap& umap) { + return sizeof(umap) + umap.getAllocatedMemorySize(); +} +#endif + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/memtable/alloc_tracker.cc b/librocksdb-sys/rocksdb/memtable/alloc_tracker.cc new file mode 100644 index 0000000..4c6d354 --- /dev/null +++ b/librocksdb-sys/rocksdb/memtable/alloc_tracker.cc @@ -0,0 +1,63 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include + +#include "memory/allocator.h" +#include "memory/arena.h" +#include "rocksdb/write_buffer_manager.h" + +namespace ROCKSDB_NAMESPACE { + +AllocTracker::AllocTracker(WriteBufferManager* write_buffer_manager) + : write_buffer_manager_(write_buffer_manager), + bytes_allocated_(0), + done_allocating_(false), + freed_(false) {} + +AllocTracker::~AllocTracker() { FreeMem(); } + +void AllocTracker::Allocate(size_t bytes) { + assert(write_buffer_manager_ != nullptr); + if (write_buffer_manager_->enabled() || + write_buffer_manager_->cost_to_cache()) { + bytes_allocated_.fetch_add(bytes, std::memory_order_relaxed); + write_buffer_manager_->ReserveMem(bytes); + } +} + +void AllocTracker::DoneAllocating() { + if (write_buffer_manager_ != nullptr && !done_allocating_) { + if (write_buffer_manager_->enabled() || + write_buffer_manager_->cost_to_cache()) { + write_buffer_manager_->ScheduleFreeMem( + bytes_allocated_.load(std::memory_order_relaxed)); + } else { + assert(bytes_allocated_.load(std::memory_order_relaxed) == 0); + } + done_allocating_ = true; + } +} + +void AllocTracker::FreeMem() { + if (!done_allocating_) { + DoneAllocating(); + } + if (write_buffer_manager_ != nullptr && !freed_) { + if (write_buffer_manager_->enabled() || + write_buffer_manager_->cost_to_cache()) { + write_buffer_manager_->FreeMem( + bytes_allocated_.load(std::memory_order_relaxed)); + } else { + assert(bytes_allocated_.load(std::memory_order_relaxed) == 0); + } + freed_ = true; + } +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/memtable/hash_linklist_rep.cc b/librocksdb-sys/rocksdb/memtable/hash_linklist_rep.cc new file mode 100644 index 0000000..9e60f9b --- /dev/null +++ b/librocksdb-sys/rocksdb/memtable/hash_linklist_rep.cc @@ -0,0 +1,924 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + + +#include +#include + +#include "db/memtable.h" +#include "memory/arena.h" +#include "memtable/skiplist.h" +#include "monitoring/histogram.h" +#include "port/port.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/slice.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/utilities/options_type.h" +#include "util/hash.h" + +namespace ROCKSDB_NAMESPACE { +namespace { + +using Key = const char*; +using MemtableSkipList = SkipList; +using Pointer = std::atomic; + +// A data structure used as the header of a link list of a hash bucket. +struct BucketHeader { + Pointer next; + std::atomic num_entries; + + explicit BucketHeader(void* n, uint32_t count) + : next(n), num_entries(count) {} + + bool IsSkipListBucket() { + return next.load(std::memory_order_relaxed) == this; + } + + uint32_t GetNumEntries() const { + return num_entries.load(std::memory_order_relaxed); + } + + // REQUIRES: called from single-threaded Insert() + void IncNumEntries() { + // Only one thread can do write at one time. No need to do atomic + // incremental. Update it with relaxed load and store. + num_entries.store(GetNumEntries() + 1, std::memory_order_relaxed); + } +}; + +// A data structure used as the header of a skip list of a hash bucket. +struct SkipListBucketHeader { + BucketHeader Counting_header; + MemtableSkipList skip_list; + + explicit SkipListBucketHeader(const MemTableRep::KeyComparator& cmp, + Allocator* allocator, uint32_t count) + : Counting_header(this, // Pointing to itself to indicate header type. + count), + skip_list(cmp, allocator) {} +}; + +struct Node { + // Accessors/mutators for links. Wrapped in methods so we can + // add the appropriate barriers as necessary. + Node* Next() { + // Use an 'acquire load' so that we observe a fully initialized + // version of the returned Node. + return next_.load(std::memory_order_acquire); + } + void SetNext(Node* x) { + // Use a 'release store' so that anybody who reads through this + // pointer observes a fully initialized version of the inserted node. + next_.store(x, std::memory_order_release); + } + // No-barrier variants that can be safely used in a few locations. + Node* NoBarrier_Next() { return next_.load(std::memory_order_relaxed); } + + void NoBarrier_SetNext(Node* x) { next_.store(x, std::memory_order_relaxed); } + + // Needed for placement new below which is fine + Node() {} + + private: + std::atomic next_; + + // Prohibit copying due to the below + Node(const Node&) = delete; + Node& operator=(const Node&) = delete; + + public: + char key[1]; +}; + +// Memory structure of the mem table: +// It is a hash table, each bucket points to one entry, a linked list or a +// skip list. In order to track total number of records in a bucket to determine +// whether should switch to skip list, a header is added just to indicate +// number of entries in the bucket. +// +// +// +-----> NULL Case 1. Empty bucket +// | +// | +// | +---> +-------+ +// | | | Next +--> NULL +// | | +-------+ +// +-----+ | | | | Case 2. One Entry in bucket. +// | +-+ | | Data | next pointer points to +// +-----+ | | | NULL. All other cases +// | | | | | next pointer is not NULL. +// +-----+ | +-------+ +// | +---+ +// +-----+ +-> +-------+ +> +-------+ +-> +-------+ +// | | | | Next +--+ | Next +--+ | Next +-->NULL +// +-----+ | +-------+ +-------+ +-------+ +// | +-----+ | Count | | | | | +// +-----+ +-------+ | Data | | Data | +// | | | | | | +// +-----+ Case 3. | | | | +// | | A header +-------+ +-------+ +// +-----+ points to +// | | a linked list. Count indicates total number +// +-----+ of rows in this bucket. +// | | +// +-----+ +-> +-------+ <--+ +// | | | | Next +----+ +// +-----+ | +-------+ Case 4. A header points to a skip +// | +----+ | Count | list and next pointer points to +// +-----+ +-------+ itself, to distinguish case 3 or 4. +// | | | | Count still is kept to indicates total +// +-----+ | Skip +--> of entries in the bucket for debugging +// | | | List | Data purpose. +// | | | +--> +// +-----+ | | +// | | +-------+ +// +-----+ +// +// We don't have data race when changing cases because: +// (1) When changing from case 2->3, we create a new bucket header, put the +// single node there first without changing the original node, and do a +// release store when changing the bucket pointer. In that case, a reader +// who sees a stale value of the bucket pointer will read this node, while +// a reader sees the correct value because of the release store. +// (2) When changing case 3->4, a new header is created with skip list points +// to the data, before doing an acquire store to change the bucket pointer. +// The old header and nodes are never changed, so any reader sees any +// of those existing pointers will guarantee to be able to iterate to the +// end of the linked list. +// (3) Header's next pointer in case 3 might change, but they are never equal +// to itself, so no matter a reader sees any stale or newer value, it will +// be able to correctly distinguish case 3 and 4. +// +// The reason that we use case 2 is we want to make the format to be efficient +// when the utilization of buckets is relatively low. If we use case 3 for +// single entry bucket, we will need to waste 12 bytes for every entry, +// which can be significant decrease of memory utilization. +class HashLinkListRep : public MemTableRep { + public: + HashLinkListRep(const MemTableRep::KeyComparator& compare, + Allocator* allocator, const SliceTransform* transform, + size_t bucket_size, uint32_t threshold_use_skiplist, + size_t huge_page_tlb_size, Logger* logger, + int bucket_entries_logging_threshold, + bool if_log_bucket_dist_when_flash); + + KeyHandle Allocate(const size_t len, char** buf) override; + + void Insert(KeyHandle handle) override; + + bool Contains(const char* key) const override; + + size_t ApproximateMemoryUsage() override; + + void Get(const LookupKey& k, void* callback_args, + bool (*callback_func)(void* arg, const char* entry)) override; + + ~HashLinkListRep() override; + + MemTableRep::Iterator* GetIterator(Arena* arena = nullptr) override; + + MemTableRep::Iterator* GetDynamicPrefixIterator( + Arena* arena = nullptr) override; + + private: + friend class DynamicIterator; + + size_t bucket_size_; + + // Maps slices (which are transformed user keys) to buckets of keys sharing + // the same transform. + Pointer* buckets_; + + const uint32_t threshold_use_skiplist_; + + // The user-supplied transform whose domain is the user keys. + const SliceTransform* transform_; + + const MemTableRep::KeyComparator& compare_; + + Logger* logger_; + int bucket_entries_logging_threshold_; + bool if_log_bucket_dist_when_flash_; + + bool LinkListContains(Node* head, const Slice& key) const; + + bool IsEmptyBucket(Pointer& bucket_pointer) const { + return bucket_pointer.load(std::memory_order_acquire) == nullptr; + } + + // Precondition: GetLinkListFirstNode() must have been called first and return + // null so that it must be a skip list bucket + SkipListBucketHeader* GetSkipListBucketHeader(Pointer& bucket_pointer) const; + + // Returning nullptr indicates it is a skip list bucket. + Node* GetLinkListFirstNode(Pointer& bucket_pointer) const; + + Slice GetPrefix(const Slice& internal_key) const { + return transform_->Transform(ExtractUserKey(internal_key)); + } + + size_t GetHash(const Slice& slice) const { + return GetSliceRangedNPHash(slice, bucket_size_); + } + + Pointer& GetBucket(size_t i) const { return buckets_[i]; } + + Pointer& GetBucket(const Slice& slice) const { + return GetBucket(GetHash(slice)); + } + + bool Equal(const Slice& a, const Key& b) const { + return (compare_(b, a) == 0); + } + + bool Equal(const Key& a, const Key& b) const { return (compare_(a, b) == 0); } + + bool KeyIsAfterNode(const Slice& internal_key, const Node* n) const { + // nullptr n is considered infinite + return (n != nullptr) && (compare_(n->key, internal_key) < 0); + } + + bool KeyIsAfterNode(const Key& key, const Node* n) const { + // nullptr n is considered infinite + return (n != nullptr) && (compare_(n->key, key) < 0); + } + + bool KeyIsAfterOrAtNode(const Slice& internal_key, const Node* n) const { + // nullptr n is considered infinite + return (n != nullptr) && (compare_(n->key, internal_key) <= 0); + } + + bool KeyIsAfterOrAtNode(const Key& key, const Node* n) const { + // nullptr n is considered infinite + return (n != nullptr) && (compare_(n->key, key) <= 0); + } + + Node* FindGreaterOrEqualInBucket(Node* head, const Slice& key) const; + Node* FindLessOrEqualInBucket(Node* head, const Slice& key) const; + + class FullListIterator : public MemTableRep::Iterator { + public: + explicit FullListIterator(MemtableSkipList* list, Allocator* allocator) + : iter_(list), full_list_(list), allocator_(allocator) {} + + ~FullListIterator() override {} + + // Returns true iff the iterator is positioned at a valid node. + bool Valid() const override { return iter_.Valid(); } + + // Returns the key at the current position. + // REQUIRES: Valid() + const char* key() const override { + assert(Valid()); + return iter_.key(); + } + + // Advances to the next position. + // REQUIRES: Valid() + void Next() override { + assert(Valid()); + iter_.Next(); + } + + // Advances to the previous position. + // REQUIRES: Valid() + void Prev() override { + assert(Valid()); + iter_.Prev(); + } + + // Advance to the first entry with a key >= target + void Seek(const Slice& internal_key, const char* memtable_key) override { + const char* encoded_key = (memtable_key != nullptr) + ? memtable_key + : EncodeKey(&tmp_, internal_key); + iter_.Seek(encoded_key); + } + + // Retreat to the last entry with a key <= target + void SeekForPrev(const Slice& internal_key, + const char* memtable_key) override { + const char* encoded_key = (memtable_key != nullptr) + ? memtable_key + : EncodeKey(&tmp_, internal_key); + iter_.SeekForPrev(encoded_key); + } + + // Position at the first entry in collection. + // Final state of iterator is Valid() iff collection is not empty. + void SeekToFirst() override { iter_.SeekToFirst(); } + + // Position at the last entry in collection. + // Final state of iterator is Valid() iff collection is not empty. + void SeekToLast() override { iter_.SeekToLast(); } + + private: + MemtableSkipList::Iterator iter_; + // To destruct with the iterator. + std::unique_ptr full_list_; + std::unique_ptr allocator_; + std::string tmp_; // For passing to EncodeKey + }; + + class LinkListIterator : public MemTableRep::Iterator { + public: + explicit LinkListIterator(const HashLinkListRep* const hash_link_list_rep, + Node* head) + : hash_link_list_rep_(hash_link_list_rep), + head_(head), + node_(nullptr) {} + + ~LinkListIterator() override {} + + // Returns true iff the iterator is positioned at a valid node. + bool Valid() const override { return node_ != nullptr; } + + // Returns the key at the current position. + // REQUIRES: Valid() + const char* key() const override { + assert(Valid()); + return node_->key; + } + + // Advances to the next position. + // REQUIRES: Valid() + void Next() override { + assert(Valid()); + node_ = node_->Next(); + } + + // Advances to the previous position. + // REQUIRES: Valid() + void Prev() override { + // Prefix iterator does not support total order. + // We simply set the iterator to invalid state + Reset(nullptr); + } + + // Advance to the first entry with a key >= target + void Seek(const Slice& internal_key, + const char* /*memtable_key*/) override { + node_ = + hash_link_list_rep_->FindGreaterOrEqualInBucket(head_, internal_key); + } + + // Retreat to the last entry with a key <= target + void SeekForPrev(const Slice& /*internal_key*/, + const char* /*memtable_key*/) override { + // Since we do not support Prev() + // We simply do not support SeekForPrev + Reset(nullptr); + } + + // Position at the first entry in collection. + // Final state of iterator is Valid() iff collection is not empty. + void SeekToFirst() override { + // Prefix iterator does not support total order. + // We simply set the iterator to invalid state + Reset(nullptr); + } + + // Position at the last entry in collection. + // Final state of iterator is Valid() iff collection is not empty. + void SeekToLast() override { + // Prefix iterator does not support total order. + // We simply set the iterator to invalid state + Reset(nullptr); + } + + protected: + void Reset(Node* head) { + head_ = head; + node_ = nullptr; + } + + private: + friend class HashLinkListRep; + const HashLinkListRep* const hash_link_list_rep_; + Node* head_; + Node* node_; + + virtual void SeekToHead() { node_ = head_; } + }; + + class DynamicIterator : public HashLinkListRep::LinkListIterator { + public: + explicit DynamicIterator(HashLinkListRep& memtable_rep) + : HashLinkListRep::LinkListIterator(&memtable_rep, nullptr), + memtable_rep_(memtable_rep) {} + + // Advance to the first entry with a key >= target + void Seek(const Slice& k, const char* memtable_key) override { + auto transformed = memtable_rep_.GetPrefix(k); + Pointer& bucket = memtable_rep_.GetBucket(transformed); + + if (memtable_rep_.IsEmptyBucket(bucket)) { + skip_list_iter_.reset(); + Reset(nullptr); + } else { + Node* first_linked_list_node = + memtable_rep_.GetLinkListFirstNode(bucket); + if (first_linked_list_node != nullptr) { + // The bucket is organized as a linked list + skip_list_iter_.reset(); + Reset(first_linked_list_node); + HashLinkListRep::LinkListIterator::Seek(k, memtable_key); + + } else { + SkipListBucketHeader* skip_list_header = + memtable_rep_.GetSkipListBucketHeader(bucket); + assert(skip_list_header != nullptr); + // The bucket is organized as a skip list + if (!skip_list_iter_) { + skip_list_iter_.reset( + new MemtableSkipList::Iterator(&skip_list_header->skip_list)); + } else { + skip_list_iter_->SetList(&skip_list_header->skip_list); + } + if (memtable_key != nullptr) { + skip_list_iter_->Seek(memtable_key); + } else { + IterKey encoded_key; + encoded_key.EncodeLengthPrefixedKey(k); + skip_list_iter_->Seek(encoded_key.GetUserKey().data()); + } + } + } + } + + bool Valid() const override { + if (skip_list_iter_) { + return skip_list_iter_->Valid(); + } + return HashLinkListRep::LinkListIterator::Valid(); + } + + const char* key() const override { + if (skip_list_iter_) { + return skip_list_iter_->key(); + } + return HashLinkListRep::LinkListIterator::key(); + } + + void Next() override { + if (skip_list_iter_) { + skip_list_iter_->Next(); + } else { + HashLinkListRep::LinkListIterator::Next(); + } + } + + private: + // the underlying memtable + const HashLinkListRep& memtable_rep_; + std::unique_ptr skip_list_iter_; + }; + + class EmptyIterator : public MemTableRep::Iterator { + // This is used when there wasn't a bucket. It is cheaper than + // instantiating an empty bucket over which to iterate. + public: + EmptyIterator() {} + bool Valid() const override { return false; } + const char* key() const override { + assert(false); + return nullptr; + } + void Next() override {} + void Prev() override {} + void Seek(const Slice& /*user_key*/, + const char* /*memtable_key*/) override {} + void SeekForPrev(const Slice& /*user_key*/, + const char* /*memtable_key*/) override {} + void SeekToFirst() override {} + void SeekToLast() override {} + + private: + }; +}; + +HashLinkListRep::HashLinkListRep( + const MemTableRep::KeyComparator& compare, Allocator* allocator, + const SliceTransform* transform, size_t bucket_size, + uint32_t threshold_use_skiplist, size_t huge_page_tlb_size, Logger* logger, + int bucket_entries_logging_threshold, bool if_log_bucket_dist_when_flash) + : MemTableRep(allocator), + bucket_size_(bucket_size), + // Threshold to use skip list doesn't make sense if less than 3, so we + // force it to be minimum of 3 to simplify implementation. + threshold_use_skiplist_(std::max(threshold_use_skiplist, 3U)), + transform_(transform), + compare_(compare), + logger_(logger), + bucket_entries_logging_threshold_(bucket_entries_logging_threshold), + if_log_bucket_dist_when_flash_(if_log_bucket_dist_when_flash) { + char* mem = allocator_->AllocateAligned(sizeof(Pointer) * bucket_size, + huge_page_tlb_size, logger); + + buckets_ = new (mem) Pointer[bucket_size]; + + for (size_t i = 0; i < bucket_size_; ++i) { + buckets_[i].store(nullptr, std::memory_order_relaxed); + } +} + +HashLinkListRep::~HashLinkListRep() {} + +KeyHandle HashLinkListRep::Allocate(const size_t len, char** buf) { + char* mem = allocator_->AllocateAligned(sizeof(Node) + len); + Node* x = new (mem) Node(); + *buf = x->key; + return static_cast(x); +} + +SkipListBucketHeader* HashLinkListRep::GetSkipListBucketHeader( + Pointer& bucket_pointer) const { + Pointer* first_next_pointer = + static_cast(bucket_pointer.load(std::memory_order_acquire)); + assert(first_next_pointer != nullptr); + assert(first_next_pointer->load(std::memory_order_relaxed) != nullptr); + + // Counting header + BucketHeader* header = reinterpret_cast(first_next_pointer); + assert(header->IsSkipListBucket()); + assert(header->GetNumEntries() > threshold_use_skiplist_); + auto* skip_list_bucket_header = + reinterpret_cast(header); + assert(skip_list_bucket_header->Counting_header.next.load( + std::memory_order_relaxed) == header); + return skip_list_bucket_header; +} + +Node* HashLinkListRep::GetLinkListFirstNode(Pointer& bucket_pointer) const { + Pointer* first_next_pointer = + static_cast(bucket_pointer.load(std::memory_order_acquire)); + assert(first_next_pointer != nullptr); + if (first_next_pointer->load(std::memory_order_relaxed) == nullptr) { + // Single entry bucket + return reinterpret_cast(first_next_pointer); + } + + // It is possible that after we fetch first_next_pointer it is modified + // and the next is not null anymore. In this case, the bucket should have been + // modified to a counting header, so we should reload the first_next_pointer + // to make sure we see the update. + first_next_pointer = + static_cast(bucket_pointer.load(std::memory_order_acquire)); + // Counting header + BucketHeader* header = reinterpret_cast(first_next_pointer); + if (!header->IsSkipListBucket()) { + assert(header->GetNumEntries() <= threshold_use_skiplist_); + return reinterpret_cast( + header->next.load(std::memory_order_acquire)); + } + assert(header->GetNumEntries() > threshold_use_skiplist_); + return nullptr; +} + +void HashLinkListRep::Insert(KeyHandle handle) { + Node* x = static_cast(handle); + assert(!Contains(x->key)); + Slice internal_key = GetLengthPrefixedSlice(x->key); + auto transformed = GetPrefix(internal_key); + auto& bucket = buckets_[GetHash(transformed)]; + Pointer* first_next_pointer = + static_cast(bucket.load(std::memory_order_relaxed)); + + if (first_next_pointer == nullptr) { + // Case 1. empty bucket + // NoBarrier_SetNext() suffices since we will add a barrier when + // we publish a pointer to "x" in prev[i]. + x->NoBarrier_SetNext(nullptr); + bucket.store(x, std::memory_order_release); + return; + } + + BucketHeader* header = nullptr; + if (first_next_pointer->load(std::memory_order_relaxed) == nullptr) { + // Case 2. only one entry in the bucket + // Need to convert to a Counting bucket and turn to case 4. + Node* first = reinterpret_cast(first_next_pointer); + // Need to add a bucket header. + // We have to first convert it to a bucket with header before inserting + // the new node. Otherwise, we might need to change next pointer of first. + // In that case, a reader might sees the next pointer is NULL and wrongly + // think the node is a bucket header. + auto* mem = allocator_->AllocateAligned(sizeof(BucketHeader)); + header = new (mem) BucketHeader(first, 1); + bucket.store(header, std::memory_order_release); + } else { + header = reinterpret_cast(first_next_pointer); + if (header->IsSkipListBucket()) { + // Case 4. Bucket is already a skip list + assert(header->GetNumEntries() > threshold_use_skiplist_); + auto* skip_list_bucket_header = + reinterpret_cast(header); + // Only one thread can execute Insert() at one time. No need to do atomic + // incremental. + skip_list_bucket_header->Counting_header.IncNumEntries(); + skip_list_bucket_header->skip_list.Insert(x->key); + return; + } + } + + if (bucket_entries_logging_threshold_ > 0 && + header->GetNumEntries() == + static_cast(bucket_entries_logging_threshold_)) { + Info(logger_, + "HashLinkedList bucket %" ROCKSDB_PRIszt + " has more than %d " + "entries. Key to insert: %s", + GetHash(transformed), header->GetNumEntries(), + GetLengthPrefixedSlice(x->key).ToString(true).c_str()); + } + + if (header->GetNumEntries() == threshold_use_skiplist_) { + // Case 3. number of entries reaches the threshold so need to convert to + // skip list. + LinkListIterator bucket_iter( + this, reinterpret_cast( + first_next_pointer->load(std::memory_order_relaxed))); + auto mem = allocator_->AllocateAligned(sizeof(SkipListBucketHeader)); + SkipListBucketHeader* new_skip_list_header = new (mem) + SkipListBucketHeader(compare_, allocator_, header->GetNumEntries() + 1); + auto& skip_list = new_skip_list_header->skip_list; + + // Add all current entries to the skip list + for (bucket_iter.SeekToHead(); bucket_iter.Valid(); bucket_iter.Next()) { + skip_list.Insert(bucket_iter.key()); + } + + // insert the new entry + skip_list.Insert(x->key); + // Set the bucket + bucket.store(new_skip_list_header, std::memory_order_release); + } else { + // Case 5. Need to insert to the sorted linked list without changing the + // header. + Node* first = + reinterpret_cast(header->next.load(std::memory_order_relaxed)); + assert(first != nullptr); + // Advance counter unless the bucket needs to be advanced to skip list. + // In that case, we need to make sure the previous count never exceeds + // threshold_use_skiplist_ to avoid readers to cast to wrong format. + header->IncNumEntries(); + + Node* cur = first; + Node* prev = nullptr; + while (true) { + if (cur == nullptr) { + break; + } + Node* next = cur->Next(); + // Make sure the lists are sorted. + // If x points to head_ or next points nullptr, it is trivially satisfied. + assert((cur == first) || (next == nullptr) || + KeyIsAfterNode(next->key, cur)); + if (KeyIsAfterNode(internal_key, cur)) { + // Keep searching in this list + prev = cur; + cur = next; + } else { + break; + } + } + + // Our data structure does not allow duplicate insertion + assert(cur == nullptr || !Equal(x->key, cur->key)); + + // NoBarrier_SetNext() suffices since we will add a barrier when + // we publish a pointer to "x" in prev[i]. + x->NoBarrier_SetNext(cur); + + if (prev) { + prev->SetNext(x); + } else { + header->next.store(static_cast(x), std::memory_order_release); + } + } +} + +bool HashLinkListRep::Contains(const char* key) const { + Slice internal_key = GetLengthPrefixedSlice(key); + + auto transformed = GetPrefix(internal_key); + Pointer& bucket = GetBucket(transformed); + if (IsEmptyBucket(bucket)) { + return false; + } + + Node* linked_list_node = GetLinkListFirstNode(bucket); + if (linked_list_node != nullptr) { + return LinkListContains(linked_list_node, internal_key); + } + + SkipListBucketHeader* skip_list_header = GetSkipListBucketHeader(bucket); + if (skip_list_header != nullptr) { + return skip_list_header->skip_list.Contains(key); + } + return false; +} + +size_t HashLinkListRep::ApproximateMemoryUsage() { + // Memory is always allocated from the allocator. + return 0; +} + +void HashLinkListRep::Get(const LookupKey& k, void* callback_args, + bool (*callback_func)(void* arg, const char* entry)) { + auto transformed = transform_->Transform(k.user_key()); + Pointer& bucket = GetBucket(transformed); + + if (IsEmptyBucket(bucket)) { + return; + } + + auto* link_list_head = GetLinkListFirstNode(bucket); + if (link_list_head != nullptr) { + LinkListIterator iter(this, link_list_head); + for (iter.Seek(k.internal_key(), nullptr); + iter.Valid() && callback_func(callback_args, iter.key()); + iter.Next()) { + } + } else { + auto* skip_list_header = GetSkipListBucketHeader(bucket); + if (skip_list_header != nullptr) { + // Is a skip list + MemtableSkipList::Iterator iter(&skip_list_header->skip_list); + for (iter.Seek(k.memtable_key().data()); + iter.Valid() && callback_func(callback_args, iter.key()); + iter.Next()) { + } + } + } +} + +MemTableRep::Iterator* HashLinkListRep::GetIterator(Arena* alloc_arena) { + // allocate a new arena of similar size to the one currently in use + Arena* new_arena = new Arena(allocator_->BlockSize()); + auto list = new MemtableSkipList(compare_, new_arena); + HistogramImpl keys_per_bucket_hist; + + for (size_t i = 0; i < bucket_size_; ++i) { + int count = 0; + Pointer& bucket = GetBucket(i); + if (!IsEmptyBucket(bucket)) { + auto* link_list_head = GetLinkListFirstNode(bucket); + if (link_list_head != nullptr) { + LinkListIterator itr(this, link_list_head); + for (itr.SeekToHead(); itr.Valid(); itr.Next()) { + list->Insert(itr.key()); + count++; + } + } else { + auto* skip_list_header = GetSkipListBucketHeader(bucket); + assert(skip_list_header != nullptr); + // Is a skip list + MemtableSkipList::Iterator itr(&skip_list_header->skip_list); + for (itr.SeekToFirst(); itr.Valid(); itr.Next()) { + list->Insert(itr.key()); + count++; + } + } + } + if (if_log_bucket_dist_when_flash_) { + keys_per_bucket_hist.Add(count); + } + } + if (if_log_bucket_dist_when_flash_ && logger_ != nullptr) { + Info(logger_, "hashLinkedList Entry distribution among buckets: %s", + keys_per_bucket_hist.ToString().c_str()); + } + + if (alloc_arena == nullptr) { + return new FullListIterator(list, new_arena); + } else { + auto mem = alloc_arena->AllocateAligned(sizeof(FullListIterator)); + return new (mem) FullListIterator(list, new_arena); + } +} + +MemTableRep::Iterator* HashLinkListRep::GetDynamicPrefixIterator( + Arena* alloc_arena) { + if (alloc_arena == nullptr) { + return new DynamicIterator(*this); + } else { + auto mem = alloc_arena->AllocateAligned(sizeof(DynamicIterator)); + return new (mem) DynamicIterator(*this); + } +} + +bool HashLinkListRep::LinkListContains(Node* head, + const Slice& user_key) const { + Node* x = FindGreaterOrEqualInBucket(head, user_key); + return (x != nullptr && Equal(user_key, x->key)); +} + +Node* HashLinkListRep::FindGreaterOrEqualInBucket(Node* head, + const Slice& key) const { + Node* x = head; + while (true) { + if (x == nullptr) { + return x; + } + Node* next = x->Next(); + // Make sure the lists are sorted. + // If x points to head_ or next points nullptr, it is trivially satisfied. + assert((x == head) || (next == nullptr) || KeyIsAfterNode(next->key, x)); + if (KeyIsAfterNode(key, x)) { + // Keep searching in this list + x = next; + } else { + break; + } + } + return x; +} + +struct HashLinkListRepOptions { + static const char* kName() { return "HashLinkListRepFactoryOptions"; } + size_t bucket_count; + uint32_t threshold_use_skiplist; + size_t huge_page_tlb_size; + int bucket_entries_logging_threshold; + bool if_log_bucket_dist_when_flash; +}; + +static std::unordered_map hash_linklist_info = { + {"bucket_count", + {offsetof(struct HashLinkListRepOptions, bucket_count), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"threshold", + {offsetof(struct HashLinkListRepOptions, threshold_use_skiplist), + OptionType::kUInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"huge_page_size", + {offsetof(struct HashLinkListRepOptions, huge_page_tlb_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"logging_threshold", + {offsetof(struct HashLinkListRepOptions, bucket_entries_logging_threshold), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"log_when_flash", + {offsetof(struct HashLinkListRepOptions, if_log_bucket_dist_when_flash), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, +}; + +class HashLinkListRepFactory : public MemTableRepFactory { + public: + explicit HashLinkListRepFactory(size_t bucket_count, + uint32_t threshold_use_skiplist, + size_t huge_page_tlb_size, + int bucket_entries_logging_threshold, + bool if_log_bucket_dist_when_flash) { + options_.bucket_count = bucket_count; + options_.threshold_use_skiplist = threshold_use_skiplist; + options_.huge_page_tlb_size = huge_page_tlb_size; + options_.bucket_entries_logging_threshold = + bucket_entries_logging_threshold; + options_.if_log_bucket_dist_when_flash = if_log_bucket_dist_when_flash; + RegisterOptions(&options_, &hash_linklist_info); + } + + using MemTableRepFactory::CreateMemTableRep; + virtual MemTableRep* CreateMemTableRep( + const MemTableRep::KeyComparator& compare, Allocator* allocator, + const SliceTransform* transform, Logger* logger) override; + + static const char* kClassName() { return "HashLinkListRepFactory"; } + static const char* kNickName() { return "hash_linkedlist"; } + virtual const char* Name() const override { return kClassName(); } + virtual const char* NickName() const override { return kNickName(); } + + private: + HashLinkListRepOptions options_; +}; + +} // namespace + +MemTableRep* HashLinkListRepFactory::CreateMemTableRep( + const MemTableRep::KeyComparator& compare, Allocator* allocator, + const SliceTransform* transform, Logger* logger) { + return new HashLinkListRep( + compare, allocator, transform, options_.bucket_count, + options_.threshold_use_skiplist, options_.huge_page_tlb_size, logger, + options_.bucket_entries_logging_threshold, + options_.if_log_bucket_dist_when_flash); +} + +MemTableRepFactory* NewHashLinkListRepFactory( + size_t bucket_count, size_t huge_page_tlb_size, + int bucket_entries_logging_threshold, bool if_log_bucket_dist_when_flash, + uint32_t threshold_use_skiplist) { + return new HashLinkListRepFactory( + bucket_count, threshold_use_skiplist, huge_page_tlb_size, + bucket_entries_logging_threshold, if_log_bucket_dist_when_flash); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/memtable/hash_skiplist_rep.cc b/librocksdb-sys/rocksdb/memtable/hash_skiplist_rep.cc new file mode 100644 index 0000000..15ff4f0 --- /dev/null +++ b/librocksdb-sys/rocksdb/memtable/hash_skiplist_rep.cc @@ -0,0 +1,391 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#include + +#include "db/memtable.h" +#include "memory/arena.h" +#include "memtable/skiplist.h" +#include "port/port.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/slice.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/utilities/options_type.h" +#include "util/murmurhash.h" + +namespace ROCKSDB_NAMESPACE { +namespace { + +class HashSkipListRep : public MemTableRep { + public: + HashSkipListRep(const MemTableRep::KeyComparator& compare, + Allocator* allocator, const SliceTransform* transform, + size_t bucket_size, int32_t skiplist_height, + int32_t skiplist_branching_factor); + + void Insert(KeyHandle handle) override; + + bool Contains(const char* key) const override; + + size_t ApproximateMemoryUsage() override; + + void Get(const LookupKey& k, void* callback_args, + bool (*callback_func)(void* arg, const char* entry)) override; + + ~HashSkipListRep() override; + + MemTableRep::Iterator* GetIterator(Arena* arena = nullptr) override; + + MemTableRep::Iterator* GetDynamicPrefixIterator( + Arena* arena = nullptr) override; + + private: + friend class DynamicIterator; + using Bucket = SkipList; + + size_t bucket_size_; + + const int32_t skiplist_height_; + const int32_t skiplist_branching_factor_; + + // Maps slices (which are transformed user keys) to buckets of keys sharing + // the same transform. + std::atomic* buckets_; + + // The user-supplied transform whose domain is the user keys. + const SliceTransform* transform_; + + const MemTableRep::KeyComparator& compare_; + // immutable after construction + Allocator* const allocator_; + + inline size_t GetHash(const Slice& slice) const { + return MurmurHash(slice.data(), static_cast(slice.size()), 0) % + bucket_size_; + } + inline Bucket* GetBucket(size_t i) const { + return buckets_[i].load(std::memory_order_acquire); + } + inline Bucket* GetBucket(const Slice& slice) const { + return GetBucket(GetHash(slice)); + } + // Get a bucket from buckets_. If the bucket hasn't been initialized yet, + // initialize it before returning. + Bucket* GetInitializedBucket(const Slice& transformed); + + class Iterator : public MemTableRep::Iterator { + public: + explicit Iterator(Bucket* list, bool own_list = true, + Arena* arena = nullptr) + : list_(list), iter_(list), own_list_(own_list), arena_(arena) {} + + ~Iterator() override { + // if we own the list, we should also delete it + if (own_list_) { + assert(list_ != nullptr); + delete list_; + } + } + + // Returns true iff the iterator is positioned at a valid node. + bool Valid() const override { return list_ != nullptr && iter_.Valid(); } + + // Returns the key at the current position. + // REQUIRES: Valid() + const char* key() const override { + assert(Valid()); + return iter_.key(); + } + + // Advances to the next position. + // REQUIRES: Valid() + void Next() override { + assert(Valid()); + iter_.Next(); + } + + // Advances to the previous position. + // REQUIRES: Valid() + void Prev() override { + assert(Valid()); + iter_.Prev(); + } + + // Advance to the first entry with a key >= target + void Seek(const Slice& internal_key, const char* memtable_key) override { + if (list_ != nullptr) { + const char* encoded_key = (memtable_key != nullptr) + ? memtable_key + : EncodeKey(&tmp_, internal_key); + iter_.Seek(encoded_key); + } + } + + // Retreat to the last entry with a key <= target + void SeekForPrev(const Slice& /*internal_key*/, + const char* /*memtable_key*/) override { + // not supported + assert(false); + } + + // Position at the first entry in collection. + // Final state of iterator is Valid() iff collection is not empty. + void SeekToFirst() override { + if (list_ != nullptr) { + iter_.SeekToFirst(); + } + } + + // Position at the last entry in collection. + // Final state of iterator is Valid() iff collection is not empty. + void SeekToLast() override { + if (list_ != nullptr) { + iter_.SeekToLast(); + } + } + + protected: + void Reset(Bucket* list) { + if (own_list_) { + assert(list_ != nullptr); + delete list_; + } + list_ = list; + iter_.SetList(list); + own_list_ = false; + } + + private: + // if list_ is nullptr, we should NEVER call any methods on iter_ + // if list_ is nullptr, this Iterator is not Valid() + Bucket* list_; + Bucket::Iterator iter_; + // here we track if we own list_. If we own it, we are also + // responsible for it's cleaning. This is a poor man's std::shared_ptr + bool own_list_; + std::unique_ptr arena_; + std::string tmp_; // For passing to EncodeKey + }; + + class DynamicIterator : public HashSkipListRep::Iterator { + public: + explicit DynamicIterator(const HashSkipListRep& memtable_rep) + : HashSkipListRep::Iterator(nullptr, false), + memtable_rep_(memtable_rep) {} + + // Advance to the first entry with a key >= target + void Seek(const Slice& k, const char* memtable_key) override { + auto transformed = memtable_rep_.transform_->Transform(ExtractUserKey(k)); + Reset(memtable_rep_.GetBucket(transformed)); + HashSkipListRep::Iterator::Seek(k, memtable_key); + } + + // Position at the first entry in collection. + // Final state of iterator is Valid() iff collection is not empty. + void SeekToFirst() override { + // Prefix iterator does not support total order. + // We simply set the iterator to invalid state + Reset(nullptr); + } + + // Position at the last entry in collection. + // Final state of iterator is Valid() iff collection is not empty. + void SeekToLast() override { + // Prefix iterator does not support total order. + // We simply set the iterator to invalid state + Reset(nullptr); + } + + private: + // the underlying memtable + const HashSkipListRep& memtable_rep_; + }; + + class EmptyIterator : public MemTableRep::Iterator { + // This is used when there wasn't a bucket. It is cheaper than + // instantiating an empty bucket over which to iterate. + public: + EmptyIterator() {} + bool Valid() const override { return false; } + const char* key() const override { + assert(false); + return nullptr; + } + void Next() override {} + void Prev() override {} + void Seek(const Slice& /*internal_key*/, + const char* /*memtable_key*/) override {} + void SeekForPrev(const Slice& /*internal_key*/, + const char* /*memtable_key*/) override {} + void SeekToFirst() override {} + void SeekToLast() override {} + + private: + }; +}; + +HashSkipListRep::HashSkipListRep(const MemTableRep::KeyComparator& compare, + Allocator* allocator, + const SliceTransform* transform, + size_t bucket_size, int32_t skiplist_height, + int32_t skiplist_branching_factor) + : MemTableRep(allocator), + bucket_size_(bucket_size), + skiplist_height_(skiplist_height), + skiplist_branching_factor_(skiplist_branching_factor), + transform_(transform), + compare_(compare), + allocator_(allocator) { + auto mem = + allocator->AllocateAligned(sizeof(std::atomic) * bucket_size); + buckets_ = new (mem) std::atomic[bucket_size]; + + for (size_t i = 0; i < bucket_size_; ++i) { + buckets_[i].store(nullptr, std::memory_order_relaxed); + } +} + +HashSkipListRep::~HashSkipListRep() {} + +HashSkipListRep::Bucket* HashSkipListRep::GetInitializedBucket( + const Slice& transformed) { + size_t hash = GetHash(transformed); + auto bucket = GetBucket(hash); + if (bucket == nullptr) { + auto addr = allocator_->AllocateAligned(sizeof(Bucket)); + bucket = new (addr) Bucket(compare_, allocator_, skiplist_height_, + skiplist_branching_factor_); + buckets_[hash].store(bucket, std::memory_order_release); + } + return bucket; +} + +void HashSkipListRep::Insert(KeyHandle handle) { + auto* key = static_cast(handle); + assert(!Contains(key)); + auto transformed = transform_->Transform(UserKey(key)); + auto bucket = GetInitializedBucket(transformed); + bucket->Insert(key); +} + +bool HashSkipListRep::Contains(const char* key) const { + auto transformed = transform_->Transform(UserKey(key)); + auto bucket = GetBucket(transformed); + if (bucket == nullptr) { + return false; + } + return bucket->Contains(key); +} + +size_t HashSkipListRep::ApproximateMemoryUsage() { return 0; } + +void HashSkipListRep::Get(const LookupKey& k, void* callback_args, + bool (*callback_func)(void* arg, const char* entry)) { + auto transformed = transform_->Transform(k.user_key()); + auto bucket = GetBucket(transformed); + if (bucket != nullptr) { + Bucket::Iterator iter(bucket); + for (iter.Seek(k.memtable_key().data()); + iter.Valid() && callback_func(callback_args, iter.key()); + iter.Next()) { + } + } +} + +MemTableRep::Iterator* HashSkipListRep::GetIterator(Arena* arena) { + // allocate a new arena of similar size to the one currently in use + Arena* new_arena = new Arena(allocator_->BlockSize()); + auto list = new Bucket(compare_, new_arena); + for (size_t i = 0; i < bucket_size_; ++i) { + auto bucket = GetBucket(i); + if (bucket != nullptr) { + Bucket::Iterator itr(bucket); + for (itr.SeekToFirst(); itr.Valid(); itr.Next()) { + list->Insert(itr.key()); + } + } + } + if (arena == nullptr) { + return new Iterator(list, true, new_arena); + } else { + auto mem = arena->AllocateAligned(sizeof(Iterator)); + return new (mem) Iterator(list, true, new_arena); + } +} + +MemTableRep::Iterator* HashSkipListRep::GetDynamicPrefixIterator(Arena* arena) { + if (arena == nullptr) { + return new DynamicIterator(*this); + } else { + auto mem = arena->AllocateAligned(sizeof(DynamicIterator)); + return new (mem) DynamicIterator(*this); + } +} + +struct HashSkipListRepOptions { + static const char* kName() { return "HashSkipListRepFactoryOptions"; } + size_t bucket_count; + int32_t skiplist_height; + int32_t skiplist_branching_factor; +}; + +static std::unordered_map hash_skiplist_info = { + {"bucket_count", + {offsetof(struct HashSkipListRepOptions, bucket_count), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"skiplist_height", + {offsetof(struct HashSkipListRepOptions, skiplist_height), + OptionType::kInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"branching_factor", + {offsetof(struct HashSkipListRepOptions, skiplist_branching_factor), + OptionType::kInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, +}; + +class HashSkipListRepFactory : public MemTableRepFactory { + public: + explicit HashSkipListRepFactory(size_t bucket_count, int32_t skiplist_height, + int32_t skiplist_branching_factor) { + options_.bucket_count = bucket_count; + options_.skiplist_height = skiplist_height; + options_.skiplist_branching_factor = skiplist_branching_factor; + RegisterOptions(&options_, &hash_skiplist_info); + } + + using MemTableRepFactory::CreateMemTableRep; + virtual MemTableRep* CreateMemTableRep( + const MemTableRep::KeyComparator& compare, Allocator* allocator, + const SliceTransform* transform, Logger* logger) override; + + static const char* kClassName() { return "HashSkipListRepFactory"; } + static const char* kNickName() { return "prefix_hash"; } + + virtual const char* Name() const override { return kClassName(); } + virtual const char* NickName() const override { return kNickName(); } + + private: + HashSkipListRepOptions options_; +}; + +} // namespace + +MemTableRep* HashSkipListRepFactory::CreateMemTableRep( + const MemTableRep::KeyComparator& compare, Allocator* allocator, + const SliceTransform* transform, Logger* /*logger*/) { + return new HashSkipListRep(compare, allocator, transform, + options_.bucket_count, options_.skiplist_height, + options_.skiplist_branching_factor); +} + +MemTableRepFactory* NewHashSkipListRepFactory( + size_t bucket_count, int32_t skiplist_height, + int32_t skiplist_branching_factor) { + return new HashSkipListRepFactory(bucket_count, skiplist_height, + skiplist_branching_factor); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/memtable/inlineskiplist.h b/librocksdb-sys/rocksdb/memtable/inlineskiplist.h new file mode 100644 index 0000000..abb3c3d --- /dev/null +++ b/librocksdb-sys/rocksdb/memtable/inlineskiplist.h @@ -0,0 +1,1051 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. Use of +// this source code is governed by a BSD-style license that can be found +// in the LICENSE file. See the AUTHORS file for names of contributors. +// +// InlineSkipList is derived from SkipList (skiplist.h), but it optimizes +// the memory layout by requiring that the key storage be allocated through +// the skip list instance. For the common case of SkipList this saves 1 pointer per skip list node and gives better cache +// locality, at the expense of wasted padding from using AllocateAligned +// instead of Allocate for the keys. The unused padding will be from +// 0 to sizeof(void*)-1 bytes, and the space savings are sizeof(void*) +// bytes, so despite the padding the space used is always less than +// SkipList. +// +// Thread safety ------------- +// +// Writes via Insert require external synchronization, most likely a mutex. +// InsertConcurrently can be safely called concurrently with reads and +// with other concurrent inserts. Reads require a guarantee that the +// InlineSkipList will not be destroyed while the read is in progress. +// Apart from that, reads progress without any internal locking or +// synchronization. +// +// Invariants: +// +// (1) Allocated nodes are never deleted until the InlineSkipList is +// destroyed. This is trivially guaranteed by the code since we never +// delete any skip list nodes. +// +// (2) The contents of a Node except for the next/prev pointers are +// immutable after the Node has been linked into the InlineSkipList. +// Only Insert() modifies the list, and it is careful to initialize a +// node and use release-stores to publish the nodes in one or more lists. +// +// ... prev vs. next pointer ordering ... +// + +#pragma once +#include +#include + +#include +#include +#include + +#include "memory/allocator.h" +#include "port/likely.h" +#include "port/port.h" +#include "rocksdb/slice.h" +#include "util/coding.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +template +class InlineSkipList { + private: + struct Node; + struct Splice; + + public: + using DecodedKey = + typename std::remove_reference::type::DecodedType; + + static const uint16_t kMaxPossibleHeight = 32; + + // Create a new InlineSkipList object that will use "cmp" for comparing + // keys, and will allocate memory using "*allocator". Objects allocated + // in the allocator must remain allocated for the lifetime of the + // skiplist object. + explicit InlineSkipList(Comparator cmp, Allocator* allocator, + int32_t max_height = 12, + int32_t branching_factor = 4); + // No copying allowed + InlineSkipList(const InlineSkipList&) = delete; + InlineSkipList& operator=(const InlineSkipList&) = delete; + + // Allocates a key and a skip-list node, returning a pointer to the key + // portion of the node. This method is thread-safe if the allocator + // is thread-safe. + char* AllocateKey(size_t key_size); + + // Allocate a splice using allocator. + Splice* AllocateSplice(); + + // Allocate a splice on heap. + Splice* AllocateSpliceOnHeap(); + + // Inserts a key allocated by AllocateKey, after the actual key value + // has been filled in. + // + // REQUIRES: nothing that compares equal to key is currently in the list. + // REQUIRES: no concurrent calls to any of inserts. + bool Insert(const char* key); + + // Inserts a key allocated by AllocateKey with a hint of last insert + // position in the skip-list. If hint points to nullptr, a new hint will be + // populated, which can be used in subsequent calls. + // + // It can be used to optimize the workload where there are multiple groups + // of keys, and each key is likely to insert to a location close to the last + // inserted key in the same group. One example is sequential inserts. + // + // REQUIRES: nothing that compares equal to key is currently in the list. + // REQUIRES: no concurrent calls to any of inserts. + bool InsertWithHint(const char* key, void** hint); + + // Like InsertConcurrently, but with a hint + // + // REQUIRES: nothing that compares equal to key is currently in the list. + // REQUIRES: no concurrent calls that use same hint + bool InsertWithHintConcurrently(const char* key, void** hint); + + // Like Insert, but external synchronization is not required. + bool InsertConcurrently(const char* key); + + // Inserts a node into the skip list. key must have been allocated by + // AllocateKey and then filled in by the caller. If UseCAS is true, + // then external synchronization is not required, otherwise this method + // may not be called concurrently with any other insertions. + // + // Regardless of whether UseCAS is true, the splice must be owned + // exclusively by the current thread. If allow_partial_splice_fix is + // true, then the cost of insertion is amortized O(log D), where D is + // the distance from the splice to the inserted key (measured as the + // number of intervening nodes). Note that this bound is very good for + // sequential insertions! If allow_partial_splice_fix is false then + // the existing splice will be ignored unless the current key is being + // inserted immediately after the splice. allow_partial_splice_fix == + // false has worse running time for the non-sequential case O(log N), + // but a better constant factor. + template + bool Insert(const char* key, Splice* splice, bool allow_partial_splice_fix); + + // Returns true iff an entry that compares equal to key is in the list. + bool Contains(const char* key) const; + + // Return estimated number of entries smaller than `key`. + uint64_t EstimateCount(const char* key) const; + + // Validate correctness of the skip-list. + void TEST_Validate() const; + + // Iteration over the contents of a skip list + class Iterator { + public: + // Initialize an iterator over the specified list. + // The returned iterator is not valid. + explicit Iterator(const InlineSkipList* list); + + // Change the underlying skiplist used for this iterator + // This enables us not changing the iterator without deallocating + // an old one and then allocating a new one + void SetList(const InlineSkipList* list); + + // Returns true iff the iterator is positioned at a valid node. + bool Valid() const; + + // Returns the key at the current position. + // REQUIRES: Valid() + const char* key() const; + + // Advances to the next position. + // REQUIRES: Valid() + void Next(); + + // Advances to the previous position. + // REQUIRES: Valid() + void Prev(); + + // Advance to the first entry with a key >= target + void Seek(const char* target); + + // Retreat to the last entry with a key <= target + void SeekForPrev(const char* target); + + // Advance to a random entry in the list. + void RandomSeek(); + + // Position at the first entry in list. + // Final state of iterator is Valid() iff list is not empty. + void SeekToFirst(); + + // Position at the last entry in list. + // Final state of iterator is Valid() iff list is not empty. + void SeekToLast(); + + private: + const InlineSkipList* list_; + Node* node_; + // Intentionally copyable + }; + + private: + const uint16_t kMaxHeight_; + const uint16_t kBranching_; + const uint32_t kScaledInverseBranching_; + + Allocator* const allocator_; // Allocator used for allocations of nodes + // Immutable after construction + Comparator const compare_; + Node* const head_; + + // Modified only by Insert(). Read racily by readers, but stale + // values are ok. + std::atomic max_height_; // Height of the entire list + + // seq_splice_ is a Splice used for insertions in the non-concurrent + // case. It caches the prev and next found during the most recent + // non-concurrent insertion. + Splice* seq_splice_; + + inline int GetMaxHeight() const { + return max_height_.load(std::memory_order_relaxed); + } + + int RandomHeight(); + + Node* AllocateNode(size_t key_size, int height); + + bool Equal(const char* a, const char* b) const { + return (compare_(a, b) == 0); + } + + bool LessThan(const char* a, const char* b) const { + return (compare_(a, b) < 0); + } + + // Return true if key is greater than the data stored in "n". Null n + // is considered infinite. n should not be head_. + bool KeyIsAfterNode(const char* key, Node* n) const; + bool KeyIsAfterNode(const DecodedKey& key, Node* n) const; + + // Returns the earliest node with a key >= key. + // Return nullptr if there is no such node. + Node* FindGreaterOrEqual(const char* key) const; + + // Return the latest node with a key < key. + // Return head_ if there is no such node. + // Fills prev[level] with pointer to previous node at "level" for every + // level in [0..max_height_-1], if prev is non-null. + Node* FindLessThan(const char* key, Node** prev = nullptr) const; + + // Return the latest node with a key < key on bottom_level. Start searching + // from root node on the level below top_level. + // Fills prev[level] with pointer to previous node at "level" for every + // level in [bottom_level..top_level-1], if prev is non-null. + Node* FindLessThan(const char* key, Node** prev, Node* root, int top_level, + int bottom_level) const; + + // Return the last node in the list. + // Return head_ if list is empty. + Node* FindLast() const; + + // Returns a random entry. + Node* FindRandomEntry() const; + + // Traverses a single level of the list, setting *out_prev to the last + // node before the key and *out_next to the first node after. Assumes + // that the key is not present in the skip list. On entry, before should + // point to a node that is before the key, and after should point to + // a node that is after the key. after should be nullptr if a good after + // node isn't conveniently available. + template + void FindSpliceForLevel(const DecodedKey& key, Node* before, Node* after, + int level, Node** out_prev, Node** out_next); + + // Recomputes Splice levels from highest_level (inclusive) down to + // lowest_level (inclusive). + void RecomputeSpliceLevels(const DecodedKey& key, Splice* splice, + int recompute_level); +}; + +// Implementation details follow + +template +struct InlineSkipList::Splice { + // The invariant of a Splice is that prev_[i+1].key <= prev_[i].key < + // next_[i].key <= next_[i+1].key for all i. That means that if a + // key is bracketed by prev_[i] and next_[i] then it is bracketed by + // all higher levels. It is _not_ required that prev_[i]->Next(i) == + // next_[i] (it probably did at some point in the past, but intervening + // or concurrent operations might have inserted nodes in between). + int height_ = 0; + Node** prev_; + Node** next_; +}; + +// The Node data type is more of a pointer into custom-managed memory than +// a traditional C++ struct. The key is stored in the bytes immediately +// after the struct, and the next_ pointers for nodes with height > 1 are +// stored immediately _before_ the struct. This avoids the need to include +// any pointer or sizing data, which reduces per-node memory overheads. +template +struct InlineSkipList::Node { + // Stores the height of the node in the memory location normally used for + // next_[0]. This is used for passing data from AllocateKey to Insert. + void StashHeight(const int height) { + assert(sizeof(int) <= sizeof(next_[0])); + memcpy(static_cast(&next_[0]), &height, sizeof(int)); + } + + // Retrieves the value passed to StashHeight. Undefined after a call + // to SetNext or NoBarrier_SetNext. + int UnstashHeight() const { + int rv; + memcpy(&rv, &next_[0], sizeof(int)); + return rv; + } + + const char* Key() const { return reinterpret_cast(&next_[1]); } + + // Accessors/mutators for links. Wrapped in methods so we can add + // the appropriate barriers as necessary, and perform the necessary + // addressing trickery for storing links below the Node in memory. + Node* Next(int n) { + assert(n >= 0); + // Use an 'acquire load' so that we observe a fully initialized + // version of the returned Node. + return ((&next_[0] - n)->load(std::memory_order_acquire)); + } + + void SetNext(int n, Node* x) { + assert(n >= 0); + // Use a 'release store' so that anybody who reads through this + // pointer observes a fully initialized version of the inserted node. + (&next_[0] - n)->store(x, std::memory_order_release); + } + + bool CASNext(int n, Node* expected, Node* x) { + assert(n >= 0); + return (&next_[0] - n)->compare_exchange_strong(expected, x); + } + + // No-barrier variants that can be safely used in a few locations. + Node* NoBarrier_Next(int n) { + assert(n >= 0); + return (&next_[0] - n)->load(std::memory_order_relaxed); + } + + void NoBarrier_SetNext(int n, Node* x) { + assert(n >= 0); + (&next_[0] - n)->store(x, std::memory_order_relaxed); + } + + // Insert node after prev on specific level. + void InsertAfter(Node* prev, int level) { + // NoBarrier_SetNext() suffices since we will add a barrier when + // we publish a pointer to "this" in prev. + NoBarrier_SetNext(level, prev->NoBarrier_Next(level)); + prev->SetNext(level, this); + } + + private: + // next_[0] is the lowest level link (level 0). Higher levels are + // stored _earlier_, so level 1 is at next_[-1]. + std::atomic next_[1]; +}; + +template +inline InlineSkipList::Iterator::Iterator( + const InlineSkipList* list) { + SetList(list); +} + +template +inline void InlineSkipList::Iterator::SetList( + const InlineSkipList* list) { + list_ = list; + node_ = nullptr; +} + +template +inline bool InlineSkipList::Iterator::Valid() const { + return node_ != nullptr; +} + +template +inline const char* InlineSkipList::Iterator::key() const { + assert(Valid()); + return node_->Key(); +} + +template +inline void InlineSkipList::Iterator::Next() { + assert(Valid()); + node_ = node_->Next(0); +} + +template +inline void InlineSkipList::Iterator::Prev() { + // Instead of using explicit "prev" links, we just search for the + // last node that falls before key. + assert(Valid()); + node_ = list_->FindLessThan(node_->Key()); + if (node_ == list_->head_) { + node_ = nullptr; + } +} + +template +inline void InlineSkipList::Iterator::Seek(const char* target) { + node_ = list_->FindGreaterOrEqual(target); +} + +template +inline void InlineSkipList::Iterator::SeekForPrev( + const char* target) { + Seek(target); + if (!Valid()) { + SeekToLast(); + } + while (Valid() && list_->LessThan(target, key())) { + Prev(); + } +} + +template +inline void InlineSkipList::Iterator::RandomSeek() { + node_ = list_->FindRandomEntry(); +} + +template +inline void InlineSkipList::Iterator::SeekToFirst() { + node_ = list_->head_->Next(0); +} + +template +inline void InlineSkipList::Iterator::SeekToLast() { + node_ = list_->FindLast(); + if (node_ == list_->head_) { + node_ = nullptr; + } +} + +template +int InlineSkipList::RandomHeight() { + auto rnd = Random::GetTLSInstance(); + + // Increase height with probability 1 in kBranching + int height = 1; + while (height < kMaxHeight_ && height < kMaxPossibleHeight && + rnd->Next() < kScaledInverseBranching_) { + height++; + } + assert(height > 0); + assert(height <= kMaxHeight_); + assert(height <= kMaxPossibleHeight); + return height; +} + +template +bool InlineSkipList::KeyIsAfterNode(const char* key, + Node* n) const { + // nullptr n is considered infinite + assert(n != head_); + return (n != nullptr) && (compare_(n->Key(), key) < 0); +} + +template +bool InlineSkipList::KeyIsAfterNode(const DecodedKey& key, + Node* n) const { + // nullptr n is considered infinite + assert(n != head_); + return (n != nullptr) && (compare_(n->Key(), key) < 0); +} + +template +typename InlineSkipList::Node* +InlineSkipList::FindGreaterOrEqual(const char* key) const { + // Note: It looks like we could reduce duplication by implementing + // this function as FindLessThan(key)->Next(0), but we wouldn't be able + // to exit early on equality and the result wouldn't even be correct. + // A concurrent insert might occur after FindLessThan(key) but before + // we get a chance to call Next(0). + Node* x = head_; + int level = GetMaxHeight() - 1; + Node* last_bigger = nullptr; + const DecodedKey key_decoded = compare_.decode_key(key); + while (true) { + Node* next = x->Next(level); + if (next != nullptr) { + PREFETCH(next->Next(level), 0, 1); + } + // Make sure the lists are sorted + assert(x == head_ || next == nullptr || KeyIsAfterNode(next->Key(), x)); + // Make sure we haven't overshot during our search + assert(x == head_ || KeyIsAfterNode(key_decoded, x)); + int cmp = (next == nullptr || next == last_bigger) + ? 1 + : compare_(next->Key(), key_decoded); + if (cmp == 0 || (cmp > 0 && level == 0)) { + return next; + } else if (cmp < 0) { + // Keep searching in this list + x = next; + } else { + // Switch to next list, reuse compare_() result + last_bigger = next; + level--; + } + } +} + +template +typename InlineSkipList::Node* +InlineSkipList::FindLessThan(const char* key, Node** prev) const { + return FindLessThan(key, prev, head_, GetMaxHeight(), 0); +} + +template +typename InlineSkipList::Node* +InlineSkipList::FindLessThan(const char* key, Node** prev, + Node* root, int top_level, + int bottom_level) const { + assert(top_level > bottom_level); + int level = top_level - 1; + Node* x = root; + // KeyIsAfter(key, last_not_after) is definitely false + Node* last_not_after = nullptr; + const DecodedKey key_decoded = compare_.decode_key(key); + while (true) { + assert(x != nullptr); + Node* next = x->Next(level); + if (next != nullptr) { + PREFETCH(next->Next(level), 0, 1); + } + assert(x == head_ || next == nullptr || KeyIsAfterNode(next->Key(), x)); + assert(x == head_ || KeyIsAfterNode(key_decoded, x)); + if (next != last_not_after && KeyIsAfterNode(key_decoded, next)) { + // Keep searching in this list + assert(next != nullptr); + x = next; + } else { + if (prev != nullptr) { + prev[level] = x; + } + if (level == bottom_level) { + return x; + } else { + // Switch to next list, reuse KeyIsAfterNode() result + last_not_after = next; + level--; + } + } + } +} + +template +typename InlineSkipList::Node* +InlineSkipList::FindLast() const { + Node* x = head_; + int level = GetMaxHeight() - 1; + while (true) { + Node* next = x->Next(level); + if (next == nullptr) { + if (level == 0) { + return x; + } else { + // Switch to next list + level--; + } + } else { + x = next; + } + } +} + +template +typename InlineSkipList::Node* +InlineSkipList::FindRandomEntry() const { + // TODO(bjlemaire): consider adding PREFETCH calls. + Node *x = head_, *scan_node = nullptr, *limit_node = nullptr; + + // We start at the max level. + // FOr each level, we look at all the nodes at the level, and + // we randomly pick one of them. Then decrement the level + // and reiterate the process. + // eg: assume GetMaxHeight()=5, and there are #100 elements (nodes). + // level 4 nodes: lvl_nodes={#1, #15, #67, #84}. Randomly pick #15. + // We will consider all the nodes between #15 (inclusive) and #67 + // (exclusive). #67 is called 'limit_node' here. + // level 3 nodes: lvl_nodes={#15, #21, #45, #51}. Randomly choose + // #51. #67 remains 'limit_node'. + // [...] + // level 0 nodes: lvl_nodes={#56,#57,#58,#59}. Randomly pick $57. + // Return Node #57. + std::vector lvl_nodes; + Random* rnd = Random::GetTLSInstance(); + int level = GetMaxHeight() - 1; + + while (level >= 0) { + lvl_nodes.clear(); + scan_node = x; + while (scan_node != limit_node) { + lvl_nodes.push_back(scan_node); + scan_node = scan_node->Next(level); + } + uint32_t rnd_idx = rnd->Next() % lvl_nodes.size(); + x = lvl_nodes[rnd_idx]; + if (rnd_idx + 1 < lvl_nodes.size()) { + limit_node = lvl_nodes[rnd_idx + 1]; + } + level--; + } + // There is a special case where x could still be the head_ + // (note that the head_ contains no key). + return x == head_ && head_ != nullptr ? head_->Next(0) : x; +} + +template +uint64_t InlineSkipList::EstimateCount(const char* key) const { + uint64_t count = 0; + + Node* x = head_; + int level = GetMaxHeight() - 1; + const DecodedKey key_decoded = compare_.decode_key(key); + while (true) { + assert(x == head_ || compare_(x->Key(), key_decoded) < 0); + Node* next = x->Next(level); + if (next != nullptr) { + PREFETCH(next->Next(level), 0, 1); + } + if (next == nullptr || compare_(next->Key(), key_decoded) >= 0) { + if (level == 0) { + return count; + } else { + // Switch to next list + count *= kBranching_; + level--; + } + } else { + x = next; + count++; + } + } +} + +template +InlineSkipList::InlineSkipList(const Comparator cmp, + Allocator* allocator, + int32_t max_height, + int32_t branching_factor) + : kMaxHeight_(static_cast(max_height)), + kBranching_(static_cast(branching_factor)), + kScaledInverseBranching_((Random::kMaxNext + 1) / kBranching_), + allocator_(allocator), + compare_(cmp), + head_(AllocateNode(0, max_height)), + max_height_(1), + seq_splice_(AllocateSplice()) { + assert(max_height > 0 && kMaxHeight_ == static_cast(max_height)); + assert(branching_factor > 1 && + kBranching_ == static_cast(branching_factor)); + assert(kScaledInverseBranching_ > 0); + + for (int i = 0; i < kMaxHeight_; ++i) { + head_->SetNext(i, nullptr); + } +} + +template +char* InlineSkipList::AllocateKey(size_t key_size) { + return const_cast(AllocateNode(key_size, RandomHeight())->Key()); +} + +template +typename InlineSkipList::Node* +InlineSkipList::AllocateNode(size_t key_size, int height) { + auto prefix = sizeof(std::atomic) * (height - 1); + + // prefix is space for the height - 1 pointers that we store before + // the Node instance (next_[-(height - 1) .. -1]). Node starts at + // raw + prefix, and holds the bottom-mode (level 0) skip list pointer + // next_[0]. key_size is the bytes for the key, which comes just after + // the Node. + char* raw = allocator_->AllocateAligned(prefix + sizeof(Node) + key_size); + Node* x = reinterpret_cast(raw + prefix); + + // Once we've linked the node into the skip list we don't actually need + // to know its height, because we can implicitly use the fact that we + // traversed into a node at level h to known that h is a valid level + // for that node. We need to convey the height to the Insert step, + // however, so that it can perform the proper links. Since we're not + // using the pointers at the moment, StashHeight temporarily borrow + // storage from next_[0] for that purpose. + x->StashHeight(height); + return x; +} + +template +typename InlineSkipList::Splice* +InlineSkipList::AllocateSplice() { + // size of prev_ and next_ + size_t array_size = sizeof(Node*) * (kMaxHeight_ + 1); + char* raw = allocator_->AllocateAligned(sizeof(Splice) + array_size * 2); + Splice* splice = reinterpret_cast(raw); + splice->height_ = 0; + splice->prev_ = reinterpret_cast(raw + sizeof(Splice)); + splice->next_ = reinterpret_cast(raw + sizeof(Splice) + array_size); + return splice; +} + +template +typename InlineSkipList::Splice* +InlineSkipList::AllocateSpliceOnHeap() { + size_t array_size = sizeof(Node*) * (kMaxHeight_ + 1); + char* raw = new char[sizeof(Splice) + array_size * 2]; + Splice* splice = reinterpret_cast(raw); + splice->height_ = 0; + splice->prev_ = reinterpret_cast(raw + sizeof(Splice)); + splice->next_ = reinterpret_cast(raw + sizeof(Splice) + array_size); + return splice; +} + +template +bool InlineSkipList::Insert(const char* key) { + return Insert(key, seq_splice_, false); +} + +template +bool InlineSkipList::InsertConcurrently(const char* key) { + Node* prev[kMaxPossibleHeight]; + Node* next[kMaxPossibleHeight]; + Splice splice; + splice.prev_ = prev; + splice.next_ = next; + return Insert(key, &splice, false); +} + +template +bool InlineSkipList::InsertWithHint(const char* key, void** hint) { + assert(hint != nullptr); + Splice* splice = reinterpret_cast(*hint); + if (splice == nullptr) { + splice = AllocateSplice(); + *hint = reinterpret_cast(splice); + } + return Insert(key, splice, true); +} + +template +bool InlineSkipList::InsertWithHintConcurrently(const char* key, + void** hint) { + assert(hint != nullptr); + Splice* splice = reinterpret_cast(*hint); + if (splice == nullptr) { + splice = AllocateSpliceOnHeap(); + *hint = reinterpret_cast(splice); + } + return Insert(key, splice, true); +} + +template +template +void InlineSkipList::FindSpliceForLevel(const DecodedKey& key, + Node* before, Node* after, + int level, Node** out_prev, + Node** out_next) { + while (true) { + Node* next = before->Next(level); + if (next != nullptr) { + PREFETCH(next->Next(level), 0, 1); + } + if (prefetch_before == true) { + if (next != nullptr && level > 0) { + PREFETCH(next->Next(level - 1), 0, 1); + } + } + assert(before == head_ || next == nullptr || + KeyIsAfterNode(next->Key(), before)); + assert(before == head_ || KeyIsAfterNode(key, before)); + if (next == after || !KeyIsAfterNode(key, next)) { + // found it + *out_prev = before; + *out_next = next; + return; + } + before = next; + } +} + +template +void InlineSkipList::RecomputeSpliceLevels(const DecodedKey& key, + Splice* splice, + int recompute_level) { + assert(recompute_level > 0); + assert(recompute_level <= splice->height_); + for (int i = recompute_level - 1; i >= 0; --i) { + FindSpliceForLevel(key, splice->prev_[i + 1], splice->next_[i + 1], i, + &splice->prev_[i], &splice->next_[i]); + } +} + +template +template +bool InlineSkipList::Insert(const char* key, Splice* splice, + bool allow_partial_splice_fix) { + Node* x = reinterpret_cast(const_cast(key)) - 1; + const DecodedKey key_decoded = compare_.decode_key(key); + int height = x->UnstashHeight(); + assert(height >= 1 && height <= kMaxHeight_); + + int max_height = max_height_.load(std::memory_order_relaxed); + while (height > max_height) { + if (max_height_.compare_exchange_weak(max_height, height)) { + // successfully updated it + max_height = height; + break; + } + // else retry, possibly exiting the loop because somebody else + // increased it + } + assert(max_height <= kMaxPossibleHeight); + + int recompute_height = 0; + if (splice->height_ < max_height) { + // Either splice has never been used or max_height has grown since + // last use. We could potentially fix it in the latter case, but + // that is tricky. + splice->prev_[max_height] = head_; + splice->next_[max_height] = nullptr; + splice->height_ = max_height; + recompute_height = max_height; + } else { + // Splice is a valid proper-height splice that brackets some + // key, but does it bracket this one? We need to validate it and + // recompute a portion of the splice (levels 0..recompute_height-1) + // that is a superset of all levels that don't bracket the new key. + // Several choices are reasonable, because we have to balance the work + // saved against the extra comparisons required to validate the Splice. + // + // One strategy is just to recompute all of orig_splice_height if the + // bottom level isn't bracketing. This pessimistically assumes that + // we will either get a perfect Splice hit (increasing sequential + // inserts) or have no locality. + // + // Another strategy is to walk up the Splice's levels until we find + // a level that brackets the key. This strategy lets the Splice + // hint help for other cases: it turns insertion from O(log N) into + // O(log D), where D is the number of nodes in between the key that + // produced the Splice and the current insert (insertion is aided + // whether the new key is before or after the splice). If you have + // a way of using a prefix of the key to map directly to the closest + // Splice out of O(sqrt(N)) Splices and we make it so that splices + // can also be used as hints during read, then we end up with Oshman's + // and Shavit's SkipTrie, which has O(log log N) lookup and insertion + // (compare to O(log N) for skip list). + // + // We control the pessimistic strategy with allow_partial_splice_fix. + // A good strategy is probably to be pessimistic for seq_splice_, + // optimistic if the caller actually went to the work of providing + // a Splice. + while (recompute_height < max_height) { + if (splice->prev_[recompute_height]->Next(recompute_height) != + splice->next_[recompute_height]) { + // splice isn't tight at this level, there must have been some inserts + // to this + // location that didn't update the splice. We might only be a little + // stale, but if + // the splice is very stale it would be O(N) to fix it. We haven't used + // up any of + // our budget of comparisons, so always move up even if we are + // pessimistic about + // our chances of success. + ++recompute_height; + } else if (splice->prev_[recompute_height] != head_ && + !KeyIsAfterNode(key_decoded, + splice->prev_[recompute_height])) { + // key is from before splice + if (allow_partial_splice_fix) { + // skip all levels with the same node without more comparisons + Node* bad = splice->prev_[recompute_height]; + while (splice->prev_[recompute_height] == bad) { + ++recompute_height; + } + } else { + // we're pessimistic, recompute everything + recompute_height = max_height; + } + } else if (KeyIsAfterNode(key_decoded, splice->next_[recompute_height])) { + // key is from after splice + if (allow_partial_splice_fix) { + Node* bad = splice->next_[recompute_height]; + while (splice->next_[recompute_height] == bad) { + ++recompute_height; + } + } else { + recompute_height = max_height; + } + } else { + // this level brackets the key, we won! + break; + } + } + } + assert(recompute_height <= max_height); + if (recompute_height > 0) { + RecomputeSpliceLevels(key_decoded, splice, recompute_height); + } + + bool splice_is_valid = true; + if (UseCAS) { + for (int i = 0; i < height; ++i) { + while (true) { + // Checking for duplicate keys on the level 0 is sufficient + if (UNLIKELY(i == 0 && splice->next_[i] != nullptr && + compare_(x->Key(), splice->next_[i]->Key()) >= 0)) { + // duplicate key + return false; + } + if (UNLIKELY(i == 0 && splice->prev_[i] != head_ && + compare_(splice->prev_[i]->Key(), x->Key()) >= 0)) { + // duplicate key + return false; + } + assert(splice->next_[i] == nullptr || + compare_(x->Key(), splice->next_[i]->Key()) < 0); + assert(splice->prev_[i] == head_ || + compare_(splice->prev_[i]->Key(), x->Key()) < 0); + x->NoBarrier_SetNext(i, splice->next_[i]); + if (splice->prev_[i]->CASNext(i, splice->next_[i], x)) { + // success + break; + } + // CAS failed, we need to recompute prev and next. It is unlikely + // to be helpful to try to use a different level as we redo the + // search, because it should be unlikely that lots of nodes have + // been inserted between prev[i] and next[i]. No point in using + // next[i] as the after hint, because we know it is stale. + FindSpliceForLevel(key_decoded, splice->prev_[i], nullptr, i, + &splice->prev_[i], &splice->next_[i]); + + // Since we've narrowed the bracket for level i, we might have + // violated the Splice constraint between i and i-1. Make sure + // we recompute the whole thing next time. + if (i > 0) { + splice_is_valid = false; + } + } + } + } else { + for (int i = 0; i < height; ++i) { + if (i >= recompute_height && + splice->prev_[i]->Next(i) != splice->next_[i]) { + FindSpliceForLevel(key_decoded, splice->prev_[i], nullptr, i, + &splice->prev_[i], &splice->next_[i]); + } + // Checking for duplicate keys on the level 0 is sufficient + if (UNLIKELY(i == 0 && splice->next_[i] != nullptr && + compare_(x->Key(), splice->next_[i]->Key()) >= 0)) { + // duplicate key + return false; + } + if (UNLIKELY(i == 0 && splice->prev_[i] != head_ && + compare_(splice->prev_[i]->Key(), x->Key()) >= 0)) { + // duplicate key + return false; + } + assert(splice->next_[i] == nullptr || + compare_(x->Key(), splice->next_[i]->Key()) < 0); + assert(splice->prev_[i] == head_ || + compare_(splice->prev_[i]->Key(), x->Key()) < 0); + assert(splice->prev_[i]->Next(i) == splice->next_[i]); + x->NoBarrier_SetNext(i, splice->next_[i]); + splice->prev_[i]->SetNext(i, x); + } + } + if (splice_is_valid) { + for (int i = 0; i < height; ++i) { + splice->prev_[i] = x; + } + assert(splice->prev_[splice->height_] == head_); + assert(splice->next_[splice->height_] == nullptr); + for (int i = 0; i < splice->height_; ++i) { + assert(splice->next_[i] == nullptr || + compare_(key, splice->next_[i]->Key()) < 0); + assert(splice->prev_[i] == head_ || + compare_(splice->prev_[i]->Key(), key) <= 0); + assert(splice->prev_[i + 1] == splice->prev_[i] || + splice->prev_[i + 1] == head_ || + compare_(splice->prev_[i + 1]->Key(), splice->prev_[i]->Key()) < + 0); + assert(splice->next_[i + 1] == splice->next_[i] || + splice->next_[i + 1] == nullptr || + compare_(splice->next_[i]->Key(), splice->next_[i + 1]->Key()) < + 0); + } + } else { + splice->height_ = 0; + } + return true; +} + +template +bool InlineSkipList::Contains(const char* key) const { + Node* x = FindGreaterOrEqual(key); + if (x != nullptr && Equal(key, x->Key())) { + return true; + } else { + return false; + } +} + +template +void InlineSkipList::TEST_Validate() const { + // Interate over all levels at the same time, and verify nodes appear in + // the right order, and nodes appear in upper level also appear in lower + // levels. + Node* nodes[kMaxPossibleHeight]; + int max_height = GetMaxHeight(); + assert(max_height > 0); + for (int i = 0; i < max_height; i++) { + nodes[i] = head_; + } + while (nodes[0] != nullptr) { + Node* l0_next = nodes[0]->Next(0); + if (l0_next == nullptr) { + break; + } + assert(nodes[0] == head_ || compare_(nodes[0]->Key(), l0_next->Key()) < 0); + nodes[0] = l0_next; + + int i = 1; + while (i < max_height) { + Node* next = nodes[i]->Next(i); + if (next == nullptr) { + break; + } + auto cmp = compare_(nodes[0]->Key(), next->Key()); + assert(cmp <= 0); + if (cmp == 0) { + assert(next == nodes[0]); + nodes[i] = next; + } else { + break; + } + i++; + } + } + for (int i = 1; i < max_height; i++) { + assert(nodes[i] != nullptr && nodes[i]->Next(i) == nullptr); + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/memtable/inlineskiplist_test.cc b/librocksdb-sys/rocksdb/memtable/inlineskiplist_test.cc new file mode 100644 index 0000000..930574e --- /dev/null +++ b/librocksdb-sys/rocksdb/memtable/inlineskiplist_test.cc @@ -0,0 +1,665 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "memtable/inlineskiplist.h" + +#include +#include + +#include "memory/concurrent_arena.h" +#include "rocksdb/env.h" +#include "test_util/testharness.h" +#include "util/hash.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +// Our test skip list stores 8-byte unsigned integers +using Key = uint64_t; + +static const char* Encode(const uint64_t* key) { + return reinterpret_cast(key); +} + +static Key Decode(const char* key) { + Key rv; + memcpy(&rv, key, sizeof(Key)); + return rv; +} + +struct TestComparator { + using DecodedType = Key; + + static DecodedType decode_key(const char* b) { return Decode(b); } + + int operator()(const char* a, const char* b) const { + if (Decode(a) < Decode(b)) { + return -1; + } else if (Decode(a) > Decode(b)) { + return +1; + } else { + return 0; + } + } + + int operator()(const char* a, const DecodedType b) const { + if (Decode(a) < b) { + return -1; + } else if (Decode(a) > b) { + return +1; + } else { + return 0; + } + } +}; + +using TestInlineSkipList = InlineSkipList; + +class InlineSkipTest : public testing::Test { + public: + void Insert(TestInlineSkipList* list, Key key) { + char* buf = list->AllocateKey(sizeof(Key)); + memcpy(buf, &key, sizeof(Key)); + list->Insert(buf); + keys_.insert(key); + } + + bool InsertWithHint(TestInlineSkipList* list, Key key, void** hint) { + char* buf = list->AllocateKey(sizeof(Key)); + memcpy(buf, &key, sizeof(Key)); + bool res = list->InsertWithHint(buf, hint); + keys_.insert(key); + return res; + } + + void Validate(TestInlineSkipList* list) { + // Check keys exist. + for (Key key : keys_) { + ASSERT_TRUE(list->Contains(Encode(&key))); + } + // Iterate over the list, make sure keys appears in order and no extra + // keys exist. + TestInlineSkipList::Iterator iter(list); + ASSERT_FALSE(iter.Valid()); + Key zero = 0; + iter.Seek(Encode(&zero)); + for (Key key : keys_) { + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(key, Decode(iter.key())); + iter.Next(); + } + ASSERT_FALSE(iter.Valid()); + // Validate the list is well-formed. + list->TEST_Validate(); + } + + private: + std::set keys_; +}; + +TEST_F(InlineSkipTest, Empty) { + Arena arena; + TestComparator cmp; + InlineSkipList list(cmp, &arena); + Key key = 10; + ASSERT_TRUE(!list.Contains(Encode(&key))); + + InlineSkipList::Iterator iter(&list); + ASSERT_TRUE(!iter.Valid()); + iter.SeekToFirst(); + ASSERT_TRUE(!iter.Valid()); + key = 100; + iter.Seek(Encode(&key)); + ASSERT_TRUE(!iter.Valid()); + iter.SeekForPrev(Encode(&key)); + ASSERT_TRUE(!iter.Valid()); + iter.SeekToLast(); + ASSERT_TRUE(!iter.Valid()); +} + +TEST_F(InlineSkipTest, InsertAndLookup) { + const int N = 2000; + const int R = 5000; + Random rnd(1000); + std::set keys; + ConcurrentArena arena; + TestComparator cmp; + InlineSkipList list(cmp, &arena); + for (int i = 0; i < N; i++) { + Key key = rnd.Next() % R; + if (keys.insert(key).second) { + char* buf = list.AllocateKey(sizeof(Key)); + memcpy(buf, &key, sizeof(Key)); + list.Insert(buf); + } + } + + for (Key i = 0; i < R; i++) { + if (list.Contains(Encode(&i))) { + ASSERT_EQ(keys.count(i), 1U); + } else { + ASSERT_EQ(keys.count(i), 0U); + } + } + + // Simple iterator tests + { + InlineSkipList::Iterator iter(&list); + ASSERT_TRUE(!iter.Valid()); + + uint64_t zero = 0; + iter.Seek(Encode(&zero)); + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(*(keys.begin()), Decode(iter.key())); + + uint64_t max_key = R - 1; + iter.SeekForPrev(Encode(&max_key)); + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(*(keys.rbegin()), Decode(iter.key())); + + iter.SeekToFirst(); + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(*(keys.begin()), Decode(iter.key())); + + iter.SeekToLast(); + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(*(keys.rbegin()), Decode(iter.key())); + } + + // Forward iteration test + for (Key i = 0; i < R; i++) { + InlineSkipList::Iterator iter(&list); + iter.Seek(Encode(&i)); + + // Compare against model iterator + std::set::iterator model_iter = keys.lower_bound(i); + for (int j = 0; j < 3; j++) { + if (model_iter == keys.end()) { + ASSERT_TRUE(!iter.Valid()); + break; + } else { + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(*model_iter, Decode(iter.key())); + ++model_iter; + iter.Next(); + } + } + } + + // Backward iteration test + for (Key i = 0; i < R; i++) { + InlineSkipList::Iterator iter(&list); + iter.SeekForPrev(Encode(&i)); + + // Compare against model iterator + std::set::iterator model_iter = keys.upper_bound(i); + for (int j = 0; j < 3; j++) { + if (model_iter == keys.begin()) { + ASSERT_TRUE(!iter.Valid()); + break; + } else { + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(*--model_iter, Decode(iter.key())); + iter.Prev(); + } + } + } +} + +TEST_F(InlineSkipTest, InsertWithHint_Sequential) { + const int N = 100000; + Arena arena; + TestComparator cmp; + TestInlineSkipList list(cmp, &arena); + void* hint = nullptr; + for (int i = 0; i < N; i++) { + Key key = i; + InsertWithHint(&list, key, &hint); + } + Validate(&list); +} + +TEST_F(InlineSkipTest, InsertWithHint_MultipleHints) { + const int N = 100000; + const int S = 100; + Random rnd(534); + Arena arena; + TestComparator cmp; + TestInlineSkipList list(cmp, &arena); + void* hints[S]; + Key last_key[S]; + for (int i = 0; i < S; i++) { + hints[i] = nullptr; + last_key[i] = 0; + } + for (int i = 0; i < N; i++) { + Key s = rnd.Uniform(S); + Key key = (s << 32) + (++last_key[s]); + InsertWithHint(&list, key, &hints[s]); + } + Validate(&list); +} + +TEST_F(InlineSkipTest, InsertWithHint_MultipleHintsRandom) { + const int N = 100000; + const int S = 100; + Random rnd(534); + Arena arena; + TestComparator cmp; + TestInlineSkipList list(cmp, &arena); + void* hints[S]; + for (int i = 0; i < S; i++) { + hints[i] = nullptr; + } + for (int i = 0; i < N; i++) { + Key s = rnd.Uniform(S); + Key key = (s << 32) + rnd.Next(); + InsertWithHint(&list, key, &hints[s]); + } + Validate(&list); +} + +TEST_F(InlineSkipTest, InsertWithHint_CompatibleWithInsertWithoutHint) { + const int N = 100000; + const int S1 = 100; + const int S2 = 100; + Random rnd(534); + Arena arena; + TestComparator cmp; + TestInlineSkipList list(cmp, &arena); + std::unordered_set used; + Key with_hint[S1]; + Key without_hint[S2]; + void* hints[S1]; + for (int i = 0; i < S1; i++) { + hints[i] = nullptr; + while (true) { + Key s = rnd.Next(); + if (used.insert(s).second) { + with_hint[i] = s; + break; + } + } + } + for (int i = 0; i < S2; i++) { + while (true) { + Key s = rnd.Next(); + if (used.insert(s).second) { + without_hint[i] = s; + break; + } + } + } + for (int i = 0; i < N; i++) { + Key s = rnd.Uniform(S1 + S2); + if (s < S1) { + Key key = (with_hint[s] << 32) + rnd.Next(); + InsertWithHint(&list, key, &hints[s]); + } else { + Key key = (without_hint[s - S1] << 32) + rnd.Next(); + Insert(&list, key); + } + } + Validate(&list); +} + +#if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) +// We want to make sure that with a single writer and multiple +// concurrent readers (with no synchronization other than when a +// reader's iterator is created), the reader always observes all the +// data that was present in the skip list when the iterator was +// constructor. Because insertions are happening concurrently, we may +// also observe new values that were inserted since the iterator was +// constructed, but we should never miss any values that were present +// at iterator construction time. +// +// We generate multi-part keys: +// +// where: +// key is in range [0..K-1] +// gen is a generation number for key +// hash is hash(key,gen) +// +// The insertion code picks a random key, sets gen to be 1 + the last +// generation number inserted for that key, and sets hash to Hash(key,gen). +// +// At the beginning of a read, we snapshot the last inserted +// generation number for each key. We then iterate, including random +// calls to Next() and Seek(). For every key we encounter, we +// check that it is either expected given the initial snapshot or has +// been concurrently added since the iterator started. +class ConcurrentTest { + public: + static const uint32_t K = 8; + + private: + static uint64_t key(Key key) { return (key >> 40); } + static uint64_t gen(Key key) { return (key >> 8) & 0xffffffffu; } + static uint64_t hash(Key key) { return key & 0xff; } + + static uint64_t HashNumbers(uint64_t k, uint64_t g) { + uint64_t data[2] = {k, g}; + return Hash(reinterpret_cast(data), sizeof(data), 0); + } + + static Key MakeKey(uint64_t k, uint64_t g) { + assert(sizeof(Key) == sizeof(uint64_t)); + assert(k <= K); // We sometimes pass K to seek to the end of the skiplist + assert(g <= 0xffffffffu); + return ((k << 40) | (g << 8) | (HashNumbers(k, g) & 0xff)); + } + + static bool IsValidKey(Key k) { + return hash(k) == (HashNumbers(key(k), gen(k)) & 0xff); + } + + static Key RandomTarget(Random* rnd) { + switch (rnd->Next() % 10) { + case 0: + // Seek to beginning + return MakeKey(0, 0); + case 1: + // Seek to end + return MakeKey(K, 0); + default: + // Seek to middle + return MakeKey(rnd->Next() % K, 0); + } + } + + // Per-key generation + struct State { + std::atomic generation[K]; + void Set(int k, int v) { + generation[k].store(v, std::memory_order_release); + } + int Get(int k) { return generation[k].load(std::memory_order_acquire); } + + State() { + for (unsigned int k = 0; k < K; k++) { + Set(k, 0); + } + } + }; + + // Current state of the test + State current_; + + ConcurrentArena arena_; + + // InlineSkipList is not protected by mu_. We just use a single writer + // thread to modify it. + InlineSkipList list_; + + public: + ConcurrentTest() : list_(TestComparator(), &arena_) {} + + // REQUIRES: No concurrent calls to WriteStep or ConcurrentWriteStep + void WriteStep(Random* rnd) { + const uint32_t k = rnd->Next() % K; + const int g = current_.Get(k) + 1; + const Key new_key = MakeKey(k, g); + char* buf = list_.AllocateKey(sizeof(Key)); + memcpy(buf, &new_key, sizeof(Key)); + list_.Insert(buf); + current_.Set(k, g); + } + + // REQUIRES: No concurrent calls for the same k + void ConcurrentWriteStep(uint32_t k, bool use_hint = false) { + const int g = current_.Get(k) + 1; + const Key new_key = MakeKey(k, g); + char* buf = list_.AllocateKey(sizeof(Key)); + memcpy(buf, &new_key, sizeof(Key)); + if (use_hint) { + void* hint = nullptr; + list_.InsertWithHintConcurrently(buf, &hint); + delete[] reinterpret_cast(hint); + } else { + list_.InsertConcurrently(buf); + } + ASSERT_EQ(g, current_.Get(k) + 1); + current_.Set(k, g); + } + + void ReadStep(Random* rnd) { + // Remember the initial committed state of the skiplist. + State initial_state; + for (unsigned int k = 0; k < K; k++) { + initial_state.Set(k, current_.Get(k)); + } + + Key pos = RandomTarget(rnd); + InlineSkipList::Iterator iter(&list_); + iter.Seek(Encode(&pos)); + while (true) { + Key current; + if (!iter.Valid()) { + current = MakeKey(K, 0); + } else { + current = Decode(iter.key()); + ASSERT_TRUE(IsValidKey(current)) << current; + } + ASSERT_LE(pos, current) << "should not go backwards"; + + // Verify that everything in [pos,current) was not present in + // initial_state. + while (pos < current) { + ASSERT_LT(key(pos), K) << pos; + + // Note that generation 0 is never inserted, so it is ok if + // <*,0,*> is missing. + ASSERT_TRUE((gen(pos) == 0U) || + (gen(pos) > static_cast(initial_state.Get( + static_cast(key(pos)))))) + << "key: " << key(pos) << "; gen: " << gen(pos) + << "; initgen: " << initial_state.Get(static_cast(key(pos))); + + // Advance to next key in the valid key space + if (key(pos) < key(current)) { + pos = MakeKey(key(pos) + 1, 0); + } else { + pos = MakeKey(key(pos), gen(pos) + 1); + } + } + + if (!iter.Valid()) { + break; + } + + if (rnd->Next() % 2) { + iter.Next(); + pos = MakeKey(key(pos), gen(pos) + 1); + } else { + Key new_target = RandomTarget(rnd); + if (new_target > pos) { + pos = new_target; + iter.Seek(Encode(&new_target)); + } + } + } + } +}; +const uint32_t ConcurrentTest::K; + +// Simple test that does single-threaded testing of the ConcurrentTest +// scaffolding. +TEST_F(InlineSkipTest, ConcurrentReadWithoutThreads) { + ConcurrentTest test; + Random rnd(test::RandomSeed()); + for (int i = 0; i < 10000; i++) { + test.ReadStep(&rnd); + test.WriteStep(&rnd); + } +} + +TEST_F(InlineSkipTest, ConcurrentInsertWithoutThreads) { + ConcurrentTest test; + Random rnd(test::RandomSeed()); + for (int i = 0; i < 10000; i++) { + test.ReadStep(&rnd); + uint32_t base = rnd.Next(); + for (int j = 0; j < 4; ++j) { + test.ConcurrentWriteStep((base + j) % ConcurrentTest::K); + } + } +} + +class TestState { + public: + ConcurrentTest t_; + bool use_hint_; + int seed_; + std::atomic quit_flag_; + std::atomic next_writer_; + + enum ReaderState { STARTING, RUNNING, DONE }; + + explicit TestState(int s) + : seed_(s), + quit_flag_(false), + state_(STARTING), + pending_writers_(0), + state_cv_(&mu_) {} + + void Wait(ReaderState s) { + mu_.Lock(); + while (state_ != s) { + state_cv_.Wait(); + } + mu_.Unlock(); + } + + void Change(ReaderState s) { + mu_.Lock(); + state_ = s; + state_cv_.Signal(); + mu_.Unlock(); + } + + void AdjustPendingWriters(int delta) { + mu_.Lock(); + pending_writers_ += delta; + if (pending_writers_ == 0) { + state_cv_.Signal(); + } + mu_.Unlock(); + } + + void WaitForPendingWriters() { + mu_.Lock(); + while (pending_writers_ != 0) { + state_cv_.Wait(); + } + mu_.Unlock(); + } + + private: + port::Mutex mu_; + ReaderState state_; + int pending_writers_; + port::CondVar state_cv_; +}; + +static void ConcurrentReader(void* arg) { + TestState* state = reinterpret_cast(arg); + Random rnd(state->seed_); + int64_t reads = 0; + state->Change(TestState::RUNNING); + while (!state->quit_flag_.load(std::memory_order_acquire)) { + state->t_.ReadStep(&rnd); + ++reads; + } + (void)reads; + state->Change(TestState::DONE); +} + +static void ConcurrentWriter(void* arg) { + TestState* state = reinterpret_cast(arg); + uint32_t k = state->next_writer_++ % ConcurrentTest::K; + state->t_.ConcurrentWriteStep(k, state->use_hint_); + state->AdjustPendingWriters(-1); +} + +static void RunConcurrentRead(int run) { + const int seed = test::RandomSeed() + (run * 100); + Random rnd(seed); + const int N = 1000; + const int kSize = 1000; + for (int i = 0; i < N; i++) { + if ((i % 100) == 0) { + fprintf(stderr, "Run %d of %d\n", i, N); + } + TestState state(seed + 1); + Env::Default()->SetBackgroundThreads(1); + Env::Default()->Schedule(ConcurrentReader, &state); + state.Wait(TestState::RUNNING); + for (int k = 0; k < kSize; ++k) { + state.t_.WriteStep(&rnd); + } + state.quit_flag_.store(true, std::memory_order_release); + state.Wait(TestState::DONE); + } +} + +static void RunConcurrentInsert(int run, bool use_hint = false, + int write_parallelism = 4) { + Env::Default()->SetBackgroundThreads(1 + write_parallelism, + Env::Priority::LOW); + const int seed = test::RandomSeed() + (run * 100); + Random rnd(seed); + const int N = 1000; + const int kSize = 1000; + for (int i = 0; i < N; i++) { + if ((i % 100) == 0) { + fprintf(stderr, "Run %d of %d\n", i, N); + } + TestState state(seed + 1); + state.use_hint_ = use_hint; + Env::Default()->Schedule(ConcurrentReader, &state); + state.Wait(TestState::RUNNING); + for (int k = 0; k < kSize; k += write_parallelism) { + state.next_writer_ = rnd.Next(); + state.AdjustPendingWriters(write_parallelism); + for (int p = 0; p < write_parallelism; ++p) { + Env::Default()->Schedule(ConcurrentWriter, &state); + } + state.WaitForPendingWriters(); + } + state.quit_flag_.store(true, std::memory_order_release); + state.Wait(TestState::DONE); + } +} + +TEST_F(InlineSkipTest, ConcurrentRead1) { RunConcurrentRead(1); } +TEST_F(InlineSkipTest, ConcurrentRead2) { RunConcurrentRead(2); } +TEST_F(InlineSkipTest, ConcurrentRead3) { RunConcurrentRead(3); } +TEST_F(InlineSkipTest, ConcurrentRead4) { RunConcurrentRead(4); } +TEST_F(InlineSkipTest, ConcurrentRead5) { RunConcurrentRead(5); } +TEST_F(InlineSkipTest, ConcurrentInsert1) { RunConcurrentInsert(1); } +TEST_F(InlineSkipTest, ConcurrentInsert2) { RunConcurrentInsert(2); } +TEST_F(InlineSkipTest, ConcurrentInsert3) { RunConcurrentInsert(3); } +TEST_F(InlineSkipTest, ConcurrentInsertWithHint1) { + RunConcurrentInsert(1, true); +} +TEST_F(InlineSkipTest, ConcurrentInsertWithHint2) { + RunConcurrentInsert(2, true); +} +TEST_F(InlineSkipTest, ConcurrentInsertWithHint3) { + RunConcurrentInsert(3, true); +} + +#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/memtable/memtablerep_bench.cc b/librocksdb-sys/rocksdb/memtable/memtablerep_bench.cc new file mode 100644 index 0000000..83db461 --- /dev/null +++ b/librocksdb-sys/rocksdb/memtable/memtablerep_bench.cc @@ -0,0 +1,687 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef GFLAGS +#include +int main() { + fprintf(stderr, "Please install gflags to run rocksdb tools\n"); + return 1; +} +#else + +#include +#include +#include +#include +#include +#include + +#include "db/dbformat.h" +#include "db/memtable.h" +#include "memory/arena.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/comparator.h" +#include "rocksdb/convenience.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/options.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/system_clock.h" +#include "rocksdb/write_buffer_manager.h" +#include "test_util/testutil.h" +#include "util/gflags_compat.h" +#include "util/mutexlock.h" +#include "util/stop_watch.h" + +using GFLAGS_NAMESPACE::ParseCommandLineFlags; +using GFLAGS_NAMESPACE::RegisterFlagValidator; +using GFLAGS_NAMESPACE::SetUsageMessage; + +DEFINE_string(benchmarks, "fillrandom", + "Comma-separated list of benchmarks to run. Options:\n" + "\tfillrandom -- write N random values\n" + "\tfillseq -- write N values in sequential order\n" + "\treadrandom -- read N values in random order\n" + "\treadseq -- scan the DB\n" + "\treadwrite -- 1 thread writes while N - 1 threads " + "do random\n" + "\t reads\n" + "\tseqreadwrite -- 1 thread writes while N - 1 threads " + "do scans\n"); + +DEFINE_string(memtablerep, "skiplist", + "Which implementation of memtablerep to use. See " + "include/memtablerep.h for\n" + " more details. Options:\n" + "\tskiplist -- backed by a skiplist\n" + "\tvector -- backed by an std::vector\n" + "\thashskiplist -- backed by a hash skip list\n" + "\thashlinklist -- backed by a hash linked list\n" + "\tcuckoo -- backed by a cuckoo hash table"); + +DEFINE_int64(bucket_count, 1000000, + "bucket_count parameter to pass into NewHashSkiplistRepFactory or " + "NewHashLinkListRepFactory"); + +DEFINE_int32( + hashskiplist_height, 4, + "skiplist_height parameter to pass into NewHashSkiplistRepFactory"); + +DEFINE_int32( + hashskiplist_branching_factor, 4, + "branching_factor parameter to pass into NewHashSkiplistRepFactory"); + +DEFINE_int32( + huge_page_tlb_size, 0, + "huge_page_tlb_size parameter to pass into NewHashLinkListRepFactory"); + +DEFINE_int32(bucket_entries_logging_threshold, 4096, + "bucket_entries_logging_threshold parameter to pass into " + "NewHashLinkListRepFactory"); + +DEFINE_bool(if_log_bucket_dist_when_flash, true, + "if_log_bucket_dist_when_flash parameter to pass into " + "NewHashLinkListRepFactory"); + +DEFINE_int32( + threshold_use_skiplist, 256, + "threshold_use_skiplist parameter to pass into NewHashLinkListRepFactory"); + +DEFINE_int64(write_buffer_size, 256, + "write_buffer_size parameter to pass into WriteBufferManager"); + +DEFINE_int32( + num_threads, 1, + "Number of concurrent threads to run. If the benchmark includes writes,\n" + "then at most one thread will be a writer"); + +DEFINE_int32(num_operations, 1000000, + "Number of operations to do for write and random read benchmarks"); + +DEFINE_int32(num_scans, 10, + "Number of times for each thread to scan the memtablerep for " + "sequential read " + "benchmarks"); + +DEFINE_int32(item_size, 100, "Number of bytes each item should be"); + +DEFINE_int32(prefix_length, 8, + "Prefix length to pass into NewFixedPrefixTransform"); + +/* VectorRep settings */ +DEFINE_int64(vectorrep_count, 0, + "Number of entries to reserve on VectorRep initialization"); + +DEFINE_int64(seed, 0, + "Seed base for random number generators. " + "When 0 it is deterministic."); + +namespace ROCKSDB_NAMESPACE { + +namespace { +struct CallbackVerifyArgs { + bool found; + LookupKey* key; + MemTableRep* table; + InternalKeyComparator* comparator; +}; +} // namespace + +// Helper for quickly generating random data. +class RandomGenerator { + private: + std::string data_; + unsigned int pos_; + + public: + RandomGenerator() { + Random rnd(301); + auto size = (unsigned)std::max(1048576, FLAGS_item_size); + data_ = rnd.RandomString(size); + pos_ = 0; + } + + Slice Generate(unsigned int len) { + assert(len <= data_.size()); + if (pos_ + len > data_.size()) { + pos_ = 0; + } + pos_ += len; + return Slice(data_.data() + pos_ - len, len); + } +}; + +enum WriteMode { SEQUENTIAL, RANDOM, UNIQUE_RANDOM }; + +class KeyGenerator { + public: + KeyGenerator(Random64* rand, WriteMode mode, uint64_t num) + : rand_(rand), mode_(mode), num_(num), next_(0) { + if (mode_ == UNIQUE_RANDOM) { + // NOTE: if memory consumption of this approach becomes a concern, + // we can either break it into pieces and only random shuffle a section + // each time. Alternatively, use a bit map implementation + // (https://reviews.facebook.net/differential/diff/54627/) + values_.resize(num_); + for (uint64_t i = 0; i < num_; ++i) { + values_[i] = i; + } + RandomShuffle(values_.begin(), values_.end(), + static_cast(FLAGS_seed)); + } + } + + uint64_t Next() { + switch (mode_) { + case SEQUENTIAL: + return next_++; + case RANDOM: + return rand_->Next() % num_; + case UNIQUE_RANDOM: + return values_[next_++]; + } + assert(false); + return std::numeric_limits::max(); + } + + private: + Random64* rand_; + WriteMode mode_; + const uint64_t num_; + uint64_t next_; + std::vector values_; +}; + +class BenchmarkThread { + public: + explicit BenchmarkThread(MemTableRep* table, KeyGenerator* key_gen, + uint64_t* bytes_written, uint64_t* bytes_read, + uint64_t* sequence, uint64_t num_ops, + uint64_t* read_hits) + : table_(table), + key_gen_(key_gen), + bytes_written_(bytes_written), + bytes_read_(bytes_read), + sequence_(sequence), + num_ops_(num_ops), + read_hits_(read_hits) {} + + virtual void operator()() = 0; + virtual ~BenchmarkThread() {} + + protected: + MemTableRep* table_; + KeyGenerator* key_gen_; + uint64_t* bytes_written_; + uint64_t* bytes_read_; + uint64_t* sequence_; + uint64_t num_ops_; + uint64_t* read_hits_; + RandomGenerator generator_; +}; + +class FillBenchmarkThread : public BenchmarkThread { + public: + FillBenchmarkThread(MemTableRep* table, KeyGenerator* key_gen, + uint64_t* bytes_written, uint64_t* bytes_read, + uint64_t* sequence, uint64_t num_ops, uint64_t* read_hits) + : BenchmarkThread(table, key_gen, bytes_written, bytes_read, sequence, + num_ops, read_hits) {} + + void FillOne() { + char* buf = nullptr; + auto internal_key_size = 16; + auto encoded_len = + FLAGS_item_size + VarintLength(internal_key_size) + internal_key_size; + KeyHandle handle = table_->Allocate(encoded_len, &buf); + assert(buf != nullptr); + char* p = EncodeVarint32(buf, internal_key_size); + auto key = key_gen_->Next(); + EncodeFixed64(p, key); + p += 8; + EncodeFixed64(p, ++(*sequence_)); + p += 8; + Slice bytes = generator_.Generate(FLAGS_item_size); + memcpy(p, bytes.data(), FLAGS_item_size); + p += FLAGS_item_size; + assert(p == buf + encoded_len); + table_->Insert(handle); + *bytes_written_ += encoded_len; + } + + void operator()() override { + for (unsigned int i = 0; i < num_ops_; ++i) { + FillOne(); + } + } +}; + +class ConcurrentFillBenchmarkThread : public FillBenchmarkThread { + public: + ConcurrentFillBenchmarkThread(MemTableRep* table, KeyGenerator* key_gen, + uint64_t* bytes_written, uint64_t* bytes_read, + uint64_t* sequence, uint64_t num_ops, + uint64_t* read_hits, + std::atomic_int* threads_done) + : FillBenchmarkThread(table, key_gen, bytes_written, bytes_read, sequence, + num_ops, read_hits) { + threads_done_ = threads_done; + } + + void operator()() override { + // # of read threads will be total threads - write threads (always 1). Loop + // while all reads complete. + while ((*threads_done_).load() < (FLAGS_num_threads - 1)) { + FillOne(); + } + } + + private: + std::atomic_int* threads_done_; +}; + +class ReadBenchmarkThread : public BenchmarkThread { + public: + ReadBenchmarkThread(MemTableRep* table, KeyGenerator* key_gen, + uint64_t* bytes_written, uint64_t* bytes_read, + uint64_t* sequence, uint64_t num_ops, uint64_t* read_hits) + : BenchmarkThread(table, key_gen, bytes_written, bytes_read, sequence, + num_ops, read_hits) {} + + static bool callback(void* arg, const char* entry) { + CallbackVerifyArgs* callback_args = static_cast(arg); + assert(callback_args != nullptr); + uint32_t key_length; + const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length); + if ((callback_args->comparator) + ->user_comparator() + ->Equal(Slice(key_ptr, key_length - 8), + callback_args->key->user_key())) { + callback_args->found = true; + } + return false; + } + + void ReadOne() { + std::string user_key; + auto key = key_gen_->Next(); + PutFixed64(&user_key, key); + LookupKey lookup_key(user_key, *sequence_); + InternalKeyComparator internal_key_comp(BytewiseComparator()); + CallbackVerifyArgs verify_args; + verify_args.found = false; + verify_args.key = &lookup_key; + verify_args.table = table_; + verify_args.comparator = &internal_key_comp; + table_->Get(lookup_key, &verify_args, callback); + if (verify_args.found) { + *bytes_read_ += VarintLength(16) + 16 + FLAGS_item_size; + ++*read_hits_; + } + } + void operator()() override { + for (unsigned int i = 0; i < num_ops_; ++i) { + ReadOne(); + } + } +}; + +class SeqReadBenchmarkThread : public BenchmarkThread { + public: + SeqReadBenchmarkThread(MemTableRep* table, KeyGenerator* key_gen, + uint64_t* bytes_written, uint64_t* bytes_read, + uint64_t* sequence, uint64_t num_ops, + uint64_t* read_hits) + : BenchmarkThread(table, key_gen, bytes_written, bytes_read, sequence, + num_ops, read_hits) {} + + void ReadOneSeq() { + std::unique_ptr iter(table_->GetIterator()); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + // pretend to read the value + *bytes_read_ += VarintLength(16) + 16 + FLAGS_item_size; + } + ++*read_hits_; + } + + void operator()() override { + for (unsigned int i = 0; i < num_ops_; ++i) { + { ReadOneSeq(); } + } + } +}; + +class ConcurrentReadBenchmarkThread : public ReadBenchmarkThread { + public: + ConcurrentReadBenchmarkThread(MemTableRep* table, KeyGenerator* key_gen, + uint64_t* bytes_written, uint64_t* bytes_read, + uint64_t* sequence, uint64_t num_ops, + uint64_t* read_hits, + std::atomic_int* threads_done) + : ReadBenchmarkThread(table, key_gen, bytes_written, bytes_read, sequence, + num_ops, read_hits) { + threads_done_ = threads_done; + } + + void operator()() override { + for (unsigned int i = 0; i < num_ops_; ++i) { + ReadOne(); + } + ++*threads_done_; + } + + private: + std::atomic_int* threads_done_; +}; + +class SeqConcurrentReadBenchmarkThread : public SeqReadBenchmarkThread { + public: + SeqConcurrentReadBenchmarkThread(MemTableRep* table, KeyGenerator* key_gen, + uint64_t* bytes_written, + uint64_t* bytes_read, uint64_t* sequence, + uint64_t num_ops, uint64_t* read_hits, + std::atomic_int* threads_done) + : SeqReadBenchmarkThread(table, key_gen, bytes_written, bytes_read, + sequence, num_ops, read_hits) { + threads_done_ = threads_done; + } + + void operator()() override { + for (unsigned int i = 0; i < num_ops_; ++i) { + ReadOneSeq(); + } + ++*threads_done_; + } + + private: + std::atomic_int* threads_done_; +}; + +class Benchmark { + public: + explicit Benchmark(MemTableRep* table, KeyGenerator* key_gen, + uint64_t* sequence, uint32_t num_threads) + : table_(table), + key_gen_(key_gen), + sequence_(sequence), + num_threads_(num_threads) {} + + virtual ~Benchmark() {} + virtual void Run() { + std::cout << "Number of threads: " << num_threads_ << std::endl; + std::vector threads; + uint64_t bytes_written = 0; + uint64_t bytes_read = 0; + uint64_t read_hits = 0; + StopWatchNano timer(SystemClock::Default().get(), true); + RunThreads(&threads, &bytes_written, &bytes_read, true, &read_hits); + auto elapsed_time = static_cast(timer.ElapsedNanos() / 1000); + std::cout << "Elapsed time: " << static_cast(elapsed_time) << " us" + << std::endl; + + if (bytes_written > 0) { + auto MiB_written = static_cast(bytes_written) / (1 << 20); + auto write_throughput = MiB_written / (elapsed_time / 1000000); + std::cout << "Total bytes written: " << MiB_written << " MiB" + << std::endl; + std::cout << "Write throughput: " << write_throughput << " MiB/s" + << std::endl; + auto us_per_op = elapsed_time / num_write_ops_per_thread_; + std::cout << "write us/op: " << us_per_op << std::endl; + } + if (bytes_read > 0) { + auto MiB_read = static_cast(bytes_read) / (1 << 20); + auto read_throughput = MiB_read / (elapsed_time / 1000000); + std::cout << "Total bytes read: " << MiB_read << " MiB" << std::endl; + std::cout << "Read throughput: " << read_throughput << " MiB/s" + << std::endl; + auto us_per_op = elapsed_time / num_read_ops_per_thread_; + std::cout << "read us/op: " << us_per_op << std::endl; + } + } + + virtual void RunThreads(std::vector* threads, + uint64_t* bytes_written, uint64_t* bytes_read, + bool write, uint64_t* read_hits) = 0; + + protected: + MemTableRep* table_; + KeyGenerator* key_gen_; + uint64_t* sequence_; + uint64_t num_write_ops_per_thread_ = 0; + uint64_t num_read_ops_per_thread_ = 0; + const uint32_t num_threads_; +}; + +class FillBenchmark : public Benchmark { + public: + explicit FillBenchmark(MemTableRep* table, KeyGenerator* key_gen, + uint64_t* sequence) + : Benchmark(table, key_gen, sequence, 1) { + num_write_ops_per_thread_ = FLAGS_num_operations; + } + + void RunThreads(std::vector* /*threads*/, + uint64_t* bytes_written, uint64_t* bytes_read, bool /*write*/, + uint64_t* read_hits) override { + FillBenchmarkThread(table_, key_gen_, bytes_written, bytes_read, sequence_, + num_write_ops_per_thread_, read_hits)(); + } +}; + +class ReadBenchmark : public Benchmark { + public: + explicit ReadBenchmark(MemTableRep* table, KeyGenerator* key_gen, + uint64_t* sequence) + : Benchmark(table, key_gen, sequence, FLAGS_num_threads) { + num_read_ops_per_thread_ = FLAGS_num_operations / FLAGS_num_threads; + } + + void RunThreads(std::vector* threads, uint64_t* bytes_written, + uint64_t* bytes_read, bool /*write*/, + uint64_t* read_hits) override { + for (int i = 0; i < FLAGS_num_threads; ++i) { + threads->emplace_back( + ReadBenchmarkThread(table_, key_gen_, bytes_written, bytes_read, + sequence_, num_read_ops_per_thread_, read_hits)); + } + for (auto& thread : *threads) { + thread.join(); + } + std::cout << "read hit%: " + << (static_cast(*read_hits) / FLAGS_num_operations) * 100 + << std::endl; + } +}; + +class SeqReadBenchmark : public Benchmark { + public: + explicit SeqReadBenchmark(MemTableRep* table, uint64_t* sequence) + : Benchmark(table, nullptr, sequence, FLAGS_num_threads) { + num_read_ops_per_thread_ = FLAGS_num_scans; + } + + void RunThreads(std::vector* threads, uint64_t* bytes_written, + uint64_t* bytes_read, bool /*write*/, + uint64_t* read_hits) override { + for (int i = 0; i < FLAGS_num_threads; ++i) { + threads->emplace_back(SeqReadBenchmarkThread( + table_, key_gen_, bytes_written, bytes_read, sequence_, + num_read_ops_per_thread_, read_hits)); + } + for (auto& thread : *threads) { + thread.join(); + } + } +}; + +template +class ReadWriteBenchmark : public Benchmark { + public: + explicit ReadWriteBenchmark(MemTableRep* table, KeyGenerator* key_gen, + uint64_t* sequence) + : Benchmark(table, key_gen, sequence, FLAGS_num_threads) { + num_read_ops_per_thread_ = + FLAGS_num_threads <= 1 + ? 0 + : (FLAGS_num_operations / (FLAGS_num_threads - 1)); + num_write_ops_per_thread_ = FLAGS_num_operations; + } + + void RunThreads(std::vector* threads, uint64_t* bytes_written, + uint64_t* bytes_read, bool /*write*/, + uint64_t* read_hits) override { + std::atomic_int threads_done; + threads_done.store(0); + threads->emplace_back(ConcurrentFillBenchmarkThread( + table_, key_gen_, bytes_written, bytes_read, sequence_, + num_write_ops_per_thread_, read_hits, &threads_done)); + for (int i = 1; i < FLAGS_num_threads; ++i) { + threads->emplace_back( + ReadThreadType(table_, key_gen_, bytes_written, bytes_read, sequence_, + num_read_ops_per_thread_, read_hits, &threads_done)); + } + for (auto& thread : *threads) { + thread.join(); + } + } +}; + +} // namespace ROCKSDB_NAMESPACE + +void PrintWarnings() { +#if defined(__GNUC__) && !defined(__OPTIMIZE__) + fprintf(stdout, + "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n"); +#endif +#ifndef NDEBUG + fprintf(stdout, + "WARNING: Assertions are enabled; benchmarks unnecessarily slow\n"); +#endif +} + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) + + " [OPTIONS]..."); + ParseCommandLineFlags(&argc, &argv, true); + + PrintWarnings(); + + ROCKSDB_NAMESPACE::Options options; + + std::unique_ptr factory; + if (FLAGS_memtablerep == "skiplist") { + factory.reset(new ROCKSDB_NAMESPACE::SkipListFactory); + } else if (FLAGS_memtablerep == "vector") { + factory.reset(new ROCKSDB_NAMESPACE::VectorRepFactory); + } else if (FLAGS_memtablerep == "hashskiplist" || + FLAGS_memtablerep == "prefix_hash") { + factory.reset(ROCKSDB_NAMESPACE::NewHashSkipListRepFactory( + FLAGS_bucket_count, FLAGS_hashskiplist_height, + FLAGS_hashskiplist_branching_factor)); + options.prefix_extractor.reset( + ROCKSDB_NAMESPACE::NewFixedPrefixTransform(FLAGS_prefix_length)); + } else if (FLAGS_memtablerep == "hashlinklist" || + FLAGS_memtablerep == "hash_linkedlist") { + factory.reset(ROCKSDB_NAMESPACE::NewHashLinkListRepFactory( + FLAGS_bucket_count, FLAGS_huge_page_tlb_size, + FLAGS_bucket_entries_logging_threshold, + FLAGS_if_log_bucket_dist_when_flash, FLAGS_threshold_use_skiplist)); + options.prefix_extractor.reset( + ROCKSDB_NAMESPACE::NewFixedPrefixTransform(FLAGS_prefix_length)); + } else { + ROCKSDB_NAMESPACE::ConfigOptions config_options; + config_options.ignore_unsupported_options = false; + + ROCKSDB_NAMESPACE::Status s = + ROCKSDB_NAMESPACE::MemTableRepFactory::CreateFromString( + config_options, FLAGS_memtablerep, &factory); + if (!s.ok()) { + fprintf(stdout, "Unknown memtablerep: %s\n", s.ToString().c_str()); + exit(1); + } + } + + ROCKSDB_NAMESPACE::InternalKeyComparator internal_key_comp( + ROCKSDB_NAMESPACE::BytewiseComparator()); + ROCKSDB_NAMESPACE::MemTable::KeyComparator key_comp(internal_key_comp); + ROCKSDB_NAMESPACE::Arena arena; + ROCKSDB_NAMESPACE::WriteBufferManager wb(FLAGS_write_buffer_size); + uint64_t sequence; + auto createMemtableRep = [&] { + sequence = 0; + return factory->CreateMemTableRep(key_comp, &arena, + options.prefix_extractor.get(), + options.info_log.get()); + }; + std::unique_ptr memtablerep; + ROCKSDB_NAMESPACE::Random64 rng(FLAGS_seed); + const char* benchmarks = FLAGS_benchmarks.c_str(); + while (benchmarks != nullptr) { + std::unique_ptr key_gen; + const char* sep = strchr(benchmarks, ','); + ROCKSDB_NAMESPACE::Slice name; + if (sep == nullptr) { + name = benchmarks; + benchmarks = nullptr; + } else { + name = ROCKSDB_NAMESPACE::Slice(benchmarks, sep - benchmarks); + benchmarks = sep + 1; + } + std::unique_ptr benchmark; + if (name == ROCKSDB_NAMESPACE::Slice("fillseq")) { + memtablerep.reset(createMemtableRep()); + key_gen.reset(new ROCKSDB_NAMESPACE::KeyGenerator( + &rng, ROCKSDB_NAMESPACE::SEQUENTIAL, FLAGS_num_operations)); + benchmark.reset(new ROCKSDB_NAMESPACE::FillBenchmark( + memtablerep.get(), key_gen.get(), &sequence)); + } else if (name == ROCKSDB_NAMESPACE::Slice("fillrandom")) { + memtablerep.reset(createMemtableRep()); + key_gen.reset(new ROCKSDB_NAMESPACE::KeyGenerator( + &rng, ROCKSDB_NAMESPACE::UNIQUE_RANDOM, FLAGS_num_operations)); + benchmark.reset(new ROCKSDB_NAMESPACE::FillBenchmark( + memtablerep.get(), key_gen.get(), &sequence)); + } else if (name == ROCKSDB_NAMESPACE::Slice("readrandom")) { + key_gen.reset(new ROCKSDB_NAMESPACE::KeyGenerator( + &rng, ROCKSDB_NAMESPACE::RANDOM, FLAGS_num_operations)); + benchmark.reset(new ROCKSDB_NAMESPACE::ReadBenchmark( + memtablerep.get(), key_gen.get(), &sequence)); + } else if (name == ROCKSDB_NAMESPACE::Slice("readseq")) { + key_gen.reset(new ROCKSDB_NAMESPACE::KeyGenerator( + &rng, ROCKSDB_NAMESPACE::SEQUENTIAL, FLAGS_num_operations)); + benchmark.reset(new ROCKSDB_NAMESPACE::SeqReadBenchmark(memtablerep.get(), + &sequence)); + } else if (name == ROCKSDB_NAMESPACE::Slice("readwrite")) { + memtablerep.reset(createMemtableRep()); + key_gen.reset(new ROCKSDB_NAMESPACE::KeyGenerator( + &rng, ROCKSDB_NAMESPACE::RANDOM, FLAGS_num_operations)); + benchmark.reset(new ROCKSDB_NAMESPACE::ReadWriteBenchmark< + ROCKSDB_NAMESPACE::ConcurrentReadBenchmarkThread>( + memtablerep.get(), key_gen.get(), &sequence)); + } else if (name == ROCKSDB_NAMESPACE::Slice("seqreadwrite")) { + memtablerep.reset(createMemtableRep()); + key_gen.reset(new ROCKSDB_NAMESPACE::KeyGenerator( + &rng, ROCKSDB_NAMESPACE::RANDOM, FLAGS_num_operations)); + benchmark.reset(new ROCKSDB_NAMESPACE::ReadWriteBenchmark< + ROCKSDB_NAMESPACE::SeqConcurrentReadBenchmarkThread>( + memtablerep.get(), key_gen.get(), &sequence)); + } else { + std::cout << "WARNING: skipping unknown benchmark '" << name.ToString() + << std::endl; + continue; + } + std::cout << "Running " << name.ToString() << std::endl; + benchmark->Run(); + } + + return 0; +} + +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/memtable/skiplist.h b/librocksdb-sys/rocksdb/memtable/skiplist.h new file mode 100644 index 0000000..e3cecd3 --- /dev/null +++ b/librocksdb-sys/rocksdb/memtable/skiplist.h @@ -0,0 +1,498 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Thread safety +// ------------- +// +// Writes require external synchronization, most likely a mutex. +// Reads require a guarantee that the SkipList will not be destroyed +// while the read is in progress. Apart from that, reads progress +// without any internal locking or synchronization. +// +// Invariants: +// +// (1) Allocated nodes are never deleted until the SkipList is +// destroyed. This is trivially guaranteed by the code since we +// never delete any skip list nodes. +// +// (2) The contents of a Node except for the next/prev pointers are +// immutable after the Node has been linked into the SkipList. +// Only Insert() modifies the list, and it is careful to initialize +// a node and use release-stores to publish the nodes in one or +// more lists. +// +// ... prev vs. next pointer ordering ... +// + +#pragma once +#include +#include + +#include + +#include "memory/allocator.h" +#include "port/port.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +template +class SkipList { + private: + struct Node; + + public: + // Create a new SkipList object that will use "cmp" for comparing keys, + // and will allocate memory using "*allocator". Objects allocated in the + // allocator must remain allocated for the lifetime of the skiplist object. + explicit SkipList(Comparator cmp, Allocator* allocator, + int32_t max_height = 12, int32_t branching_factor = 4); + // No copying allowed + SkipList(const SkipList&) = delete; + void operator=(const SkipList&) = delete; + + // Insert key into the list. + // REQUIRES: nothing that compares equal to key is currently in the list. + void Insert(const Key& key); + + // Returns true iff an entry that compares equal to key is in the list. + bool Contains(const Key& key) const; + + // Return estimated number of entries smaller than `key`. + uint64_t EstimateCount(const Key& key) const; + + // Iteration over the contents of a skip list + class Iterator { + public: + // Initialize an iterator over the specified list. + // The returned iterator is not valid. + explicit Iterator(const SkipList* list); + + // Change the underlying skiplist used for this iterator + // This enables us not changing the iterator without deallocating + // an old one and then allocating a new one + void SetList(const SkipList* list); + + // Returns true iff the iterator is positioned at a valid node. + bool Valid() const; + + // Returns the key at the current position. + // REQUIRES: Valid() + const Key& key() const; + + // Advances to the next position. + // REQUIRES: Valid() + void Next(); + + // Advances to the previous position. + // REQUIRES: Valid() + void Prev(); + + // Advance to the first entry with a key >= target + void Seek(const Key& target); + + // Retreat to the last entry with a key <= target + void SeekForPrev(const Key& target); + + // Position at the first entry in list. + // Final state of iterator is Valid() iff list is not empty. + void SeekToFirst(); + + // Position at the last entry in list. + // Final state of iterator is Valid() iff list is not empty. + void SeekToLast(); + + private: + const SkipList* list_; + Node* node_; + // Intentionally copyable + }; + + private: + const uint16_t kMaxHeight_; + const uint16_t kBranching_; + const uint32_t kScaledInverseBranching_; + + // Immutable after construction + Comparator const compare_; + Allocator* const allocator_; // Allocator used for allocations of nodes + + Node* const head_; + + // Modified only by Insert(). Read racily by readers, but stale + // values are ok. + std::atomic max_height_; // Height of the entire list + + // Used for optimizing sequential insert patterns. Tricky. prev_[i] for + // i up to max_height_ is the predecessor of prev_[0] and prev_height_ + // is the height of prev_[0]. prev_[0] can only be equal to head before + // insertion, in which case max_height_ and prev_height_ are 1. + Node** prev_; + int32_t prev_height_; + + inline int GetMaxHeight() const { + return max_height_.load(std::memory_order_relaxed); + } + + Node* NewNode(const Key& key, int height); + int RandomHeight(); + bool Equal(const Key& a, const Key& b) const { return (compare_(a, b) == 0); } + bool LessThan(const Key& a, const Key& b) const { + return (compare_(a, b) < 0); + } + + // Return true if key is greater than the data stored in "n" + bool KeyIsAfterNode(const Key& key, Node* n) const; + + // Returns the earliest node with a key >= key. + // Return nullptr if there is no such node. + Node* FindGreaterOrEqual(const Key& key) const; + + // Return the latest node with a key < key. + // Return head_ if there is no such node. + // Fills prev[level] with pointer to previous node at "level" for every + // level in [0..max_height_-1], if prev is non-null. + Node* FindLessThan(const Key& key, Node** prev = nullptr) const; + + // Return the last node in the list. + // Return head_ if list is empty. + Node* FindLast() const; +}; + +// Implementation details follow +template +struct SkipList::Node { + explicit Node(const Key& k) : key(k) {} + + Key const key; + + // Accessors/mutators for links. Wrapped in methods so we can + // add the appropriate barriers as necessary. + Node* Next(int n) { + assert(n >= 0); + // Use an 'acquire load' so that we observe a fully initialized + // version of the returned Node. + return (next_[n].load(std::memory_order_acquire)); + } + void SetNext(int n, Node* x) { + assert(n >= 0); + // Use a 'release store' so that anybody who reads through this + // pointer observes a fully initialized version of the inserted node. + next_[n].store(x, std::memory_order_release); + } + + // No-barrier variants that can be safely used in a few locations. + Node* NoBarrier_Next(int n) { + assert(n >= 0); + return next_[n].load(std::memory_order_relaxed); + } + void NoBarrier_SetNext(int n, Node* x) { + assert(n >= 0); + next_[n].store(x, std::memory_order_relaxed); + } + + private: + // Array of length equal to the node height. next_[0] is lowest level link. + std::atomic next_[1]; +}; + +template +typename SkipList::Node* SkipList::NewNode( + const Key& key, int height) { + char* mem = allocator_->AllocateAligned( + sizeof(Node) + sizeof(std::atomic) * (height - 1)); + return new (mem) Node(key); +} + +template +inline SkipList::Iterator::Iterator(const SkipList* list) { + SetList(list); +} + +template +inline void SkipList::Iterator::SetList(const SkipList* list) { + list_ = list; + node_ = nullptr; +} + +template +inline bool SkipList::Iterator::Valid() const { + return node_ != nullptr; +} + +template +inline const Key& SkipList::Iterator::key() const { + assert(Valid()); + return node_->key; +} + +template +inline void SkipList::Iterator::Next() { + assert(Valid()); + node_ = node_->Next(0); +} + +template +inline void SkipList::Iterator::Prev() { + // Instead of using explicit "prev" links, we just search for the + // last node that falls before key. + assert(Valid()); + node_ = list_->FindLessThan(node_->key); + if (node_ == list_->head_) { + node_ = nullptr; + } +} + +template +inline void SkipList::Iterator::Seek(const Key& target) { + node_ = list_->FindGreaterOrEqual(target); +} + +template +inline void SkipList::Iterator::SeekForPrev( + const Key& target) { + Seek(target); + if (!Valid()) { + SeekToLast(); + } + while (Valid() && list_->LessThan(target, key())) { + Prev(); + } +} + +template +inline void SkipList::Iterator::SeekToFirst() { + node_ = list_->head_->Next(0); +} + +template +inline void SkipList::Iterator::SeekToLast() { + node_ = list_->FindLast(); + if (node_ == list_->head_) { + node_ = nullptr; + } +} + +template +int SkipList::RandomHeight() { + auto rnd = Random::GetTLSInstance(); + + // Increase height with probability 1 in kBranching + int height = 1; + while (height < kMaxHeight_ && rnd->Next() < kScaledInverseBranching_) { + height++; + } + assert(height > 0); + assert(height <= kMaxHeight_); + return height; +} + +template +bool SkipList::KeyIsAfterNode(const Key& key, Node* n) const { + // nullptr n is considered infinite + return (n != nullptr) && (compare_(n->key, key) < 0); +} + +template +typename SkipList::Node* +SkipList::FindGreaterOrEqual(const Key& key) const { + // Note: It looks like we could reduce duplication by implementing + // this function as FindLessThan(key)->Next(0), but we wouldn't be able + // to exit early on equality and the result wouldn't even be correct. + // A concurrent insert might occur after FindLessThan(key) but before + // we get a chance to call Next(0). + Node* x = head_; + int level = GetMaxHeight() - 1; + Node* last_bigger = nullptr; + while (true) { + assert(x != nullptr); + Node* next = x->Next(level); + // Make sure the lists are sorted + assert(x == head_ || next == nullptr || KeyIsAfterNode(next->key, x)); + // Make sure we haven't overshot during our search + assert(x == head_ || KeyIsAfterNode(key, x)); + int cmp = + (next == nullptr || next == last_bigger) ? 1 : compare_(next->key, key); + if (cmp == 0 || (cmp > 0 && level == 0)) { + return next; + } else if (cmp < 0) { + // Keep searching in this list + x = next; + } else { + // Switch to next list, reuse compare_() result + last_bigger = next; + level--; + } + } +} + +template +typename SkipList::Node* +SkipList::FindLessThan(const Key& key, Node** prev) const { + Node* x = head_; + int level = GetMaxHeight() - 1; + // KeyIsAfter(key, last_not_after) is definitely false + Node* last_not_after = nullptr; + while (true) { + assert(x != nullptr); + Node* next = x->Next(level); + assert(x == head_ || next == nullptr || KeyIsAfterNode(next->key, x)); + assert(x == head_ || KeyIsAfterNode(key, x)); + if (next != last_not_after && KeyIsAfterNode(key, next)) { + // Keep searching in this list + x = next; + } else { + if (prev != nullptr) { + prev[level] = x; + } + if (level == 0) { + return x; + } else { + // Switch to next list, reuse KeyIUsAfterNode() result + last_not_after = next; + level--; + } + } + } +} + +template +typename SkipList::Node* SkipList::FindLast() + const { + Node* x = head_; + int level = GetMaxHeight() - 1; + while (true) { + Node* next = x->Next(level); + if (next == nullptr) { + if (level == 0) { + return x; + } else { + // Switch to next list + level--; + } + } else { + x = next; + } + } +} + +template +uint64_t SkipList::EstimateCount(const Key& key) const { + uint64_t count = 0; + + Node* x = head_; + int level = GetMaxHeight() - 1; + while (true) { + assert(x == head_ || compare_(x->key, key) < 0); + Node* next = x->Next(level); + if (next == nullptr || compare_(next->key, key) >= 0) { + if (level == 0) { + return count; + } else { + // Switch to next list + count *= kBranching_; + level--; + } + } else { + x = next; + count++; + } + } +} + +template +SkipList::SkipList(const Comparator cmp, Allocator* allocator, + int32_t max_height, + int32_t branching_factor) + : kMaxHeight_(static_cast(max_height)), + kBranching_(static_cast(branching_factor)), + kScaledInverseBranching_((Random::kMaxNext + 1) / kBranching_), + compare_(cmp), + allocator_(allocator), + head_(NewNode(0 /* any key will do */, max_height)), + max_height_(1), + prev_height_(1) { + assert(max_height > 0 && kMaxHeight_ == static_cast(max_height)); + assert(branching_factor > 0 && + kBranching_ == static_cast(branching_factor)); + assert(kScaledInverseBranching_ > 0); + // Allocate the prev_ Node* array, directly from the passed-in allocator. + // prev_ does not need to be freed, as its life cycle is tied up with + // the allocator as a whole. + prev_ = reinterpret_cast( + allocator_->AllocateAligned(sizeof(Node*) * kMaxHeight_)); + for (int i = 0; i < kMaxHeight_; i++) { + head_->SetNext(i, nullptr); + prev_[i] = head_; + } +} + +template +void SkipList::Insert(const Key& key) { + // fast path for sequential insertion + if (!KeyIsAfterNode(key, prev_[0]->NoBarrier_Next(0)) && + (prev_[0] == head_ || KeyIsAfterNode(key, prev_[0]))) { + assert(prev_[0] != head_ || (prev_height_ == 1 && GetMaxHeight() == 1)); + + // Outside of this method prev_[1..max_height_] is the predecessor + // of prev_[0], and prev_height_ refers to prev_[0]. Inside Insert + // prev_[0..max_height - 1] is the predecessor of key. Switch from + // the external state to the internal + for (int i = 1; i < prev_height_; i++) { + prev_[i] = prev_[0]; + } + } else { + // TODO(opt): we could use a NoBarrier predecessor search as an + // optimization for architectures where memory_order_acquire needs + // a synchronization instruction. Doesn't matter on x86 + FindLessThan(key, prev_); + } + + // Our data structure does not allow duplicate insertion + assert(prev_[0]->Next(0) == nullptr || !Equal(key, prev_[0]->Next(0)->key)); + + int height = RandomHeight(); + if (height > GetMaxHeight()) { + for (int i = GetMaxHeight(); i < height; i++) { + prev_[i] = head_; + } + // fprintf(stderr, "Change height from %d to %d\n", max_height_, height); + + // It is ok to mutate max_height_ without any synchronization + // with concurrent readers. A concurrent reader that observes + // the new value of max_height_ will see either the old value of + // new level pointers from head_ (nullptr), or a new value set in + // the loop below. In the former case the reader will + // immediately drop to the next level since nullptr sorts after all + // keys. In the latter case the reader will use the new node. + max_height_.store(height, std::memory_order_relaxed); + } + + Node* x = NewNode(key, height); + for (int i = 0; i < height; i++) { + // NoBarrier_SetNext() suffices since we will add a barrier when + // we publish a pointer to "x" in prev[i]. + x->NoBarrier_SetNext(i, prev_[i]->NoBarrier_Next(i)); + prev_[i]->SetNext(i, x); + } + prev_[0] = x; + prev_height_ = height; +} + +template +bool SkipList::Contains(const Key& key) const { + Node* x = FindGreaterOrEqual(key); + if (x != nullptr && Equal(key, x->key)) { + return true; + } else { + return false; + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/memtable/skiplist_test.cc b/librocksdb-sys/rocksdb/memtable/skiplist_test.cc new file mode 100644 index 0000000..868c518 --- /dev/null +++ b/librocksdb-sys/rocksdb/memtable/skiplist_test.cc @@ -0,0 +1,388 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "memtable/skiplist.h" + +#include + +#include "memory/arena.h" +#include "rocksdb/env.h" +#include "test_util/testharness.h" +#include "util/hash.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +using Key = uint64_t; + +struct TestComparator { + int operator()(const Key& a, const Key& b) const { + if (a < b) { + return -1; + } else if (a > b) { + return +1; + } else { + return 0; + } + } +}; + +class SkipTest : public testing::Test {}; + +TEST_F(SkipTest, Empty) { + Arena arena; + TestComparator cmp; + SkipList list(cmp, &arena); + ASSERT_TRUE(!list.Contains(10)); + + SkipList::Iterator iter(&list); + ASSERT_TRUE(!iter.Valid()); + iter.SeekToFirst(); + ASSERT_TRUE(!iter.Valid()); + iter.Seek(100); + ASSERT_TRUE(!iter.Valid()); + iter.SeekForPrev(100); + ASSERT_TRUE(!iter.Valid()); + iter.SeekToLast(); + ASSERT_TRUE(!iter.Valid()); +} + +TEST_F(SkipTest, InsertAndLookup) { + const int N = 2000; + const int R = 5000; + Random rnd(1000); + std::set keys; + Arena arena; + TestComparator cmp; + SkipList list(cmp, &arena); + for (int i = 0; i < N; i++) { + Key key = rnd.Next() % R; + if (keys.insert(key).second) { + list.Insert(key); + } + } + + for (int i = 0; i < R; i++) { + if (list.Contains(i)) { + ASSERT_EQ(keys.count(i), 1U); + } else { + ASSERT_EQ(keys.count(i), 0U); + } + } + + // Simple iterator tests + { + SkipList::Iterator iter(&list); + ASSERT_TRUE(!iter.Valid()); + + iter.Seek(0); + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(*(keys.begin()), iter.key()); + + iter.SeekForPrev(R - 1); + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(*(keys.rbegin()), iter.key()); + + iter.SeekToFirst(); + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(*(keys.begin()), iter.key()); + + iter.SeekToLast(); + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(*(keys.rbegin()), iter.key()); + } + + // Forward iteration test + for (int i = 0; i < R; i++) { + SkipList::Iterator iter(&list); + iter.Seek(i); + + // Compare against model iterator + std::set::iterator model_iter = keys.lower_bound(i); + for (int j = 0; j < 3; j++) { + if (model_iter == keys.end()) { + ASSERT_TRUE(!iter.Valid()); + break; + } else { + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(*model_iter, iter.key()); + ++model_iter; + iter.Next(); + } + } + } + + // Backward iteration test + for (int i = 0; i < R; i++) { + SkipList::Iterator iter(&list); + iter.SeekForPrev(i); + + // Compare against model iterator + std::set::iterator model_iter = keys.upper_bound(i); + for (int j = 0; j < 3; j++) { + if (model_iter == keys.begin()) { + ASSERT_TRUE(!iter.Valid()); + break; + } else { + ASSERT_TRUE(iter.Valid()); + ASSERT_EQ(*--model_iter, iter.key()); + iter.Prev(); + } + } + } +} + +// We want to make sure that with a single writer and multiple +// concurrent readers (with no synchronization other than when a +// reader's iterator is created), the reader always observes all the +// data that was present in the skip list when the iterator was +// constructor. Because insertions are happening concurrently, we may +// also observe new values that were inserted since the iterator was +// constructed, but we should never miss any values that were present +// at iterator construction time. +// +// We generate multi-part keys: +// +// where: +// key is in range [0..K-1] +// gen is a generation number for key +// hash is hash(key,gen) +// +// The insertion code picks a random key, sets gen to be 1 + the last +// generation number inserted for that key, and sets hash to Hash(key,gen). +// +// At the beginning of a read, we snapshot the last inserted +// generation number for each key. We then iterate, including random +// calls to Next() and Seek(). For every key we encounter, we +// check that it is either expected given the initial snapshot or has +// been concurrently added since the iterator started. +class ConcurrentTest { + private: + static const uint32_t K = 4; + + static uint64_t key(Key key) { return (key >> 40); } + static uint64_t gen(Key key) { return (key >> 8) & 0xffffffffu; } + static uint64_t hash(Key key) { return key & 0xff; } + + static uint64_t HashNumbers(uint64_t k, uint64_t g) { + uint64_t data[2] = {k, g}; + return Hash(reinterpret_cast(data), sizeof(data), 0); + } + + static Key MakeKey(uint64_t k, uint64_t g) { + assert(sizeof(Key) == sizeof(uint64_t)); + assert(k <= K); // We sometimes pass K to seek to the end of the skiplist + assert(g <= 0xffffffffu); + return ((k << 40) | (g << 8) | (HashNumbers(k, g) & 0xff)); + } + + static bool IsValidKey(Key k) { + return hash(k) == (HashNumbers(key(k), gen(k)) & 0xff); + } + + static Key RandomTarget(Random* rnd) { + switch (rnd->Next() % 10) { + case 0: + // Seek to beginning + return MakeKey(0, 0); + case 1: + // Seek to end + return MakeKey(K, 0); + default: + // Seek to middle + return MakeKey(rnd->Next() % K, 0); + } + } + + // Per-key generation + struct State { + std::atomic generation[K]; + void Set(int k, int v) { + generation[k].store(v, std::memory_order_release); + } + int Get(int k) { return generation[k].load(std::memory_order_acquire); } + + State() { + for (unsigned int k = 0; k < K; k++) { + Set(k, 0); + } + } + }; + + // Current state of the test + State current_; + + Arena arena_; + + // SkipList is not protected by mu_. We just use a single writer + // thread to modify it. + SkipList list_; + + public: + ConcurrentTest() : list_(TestComparator(), &arena_) {} + + // REQUIRES: External synchronization + void WriteStep(Random* rnd) { + const uint32_t k = rnd->Next() % K; + const int g = current_.Get(k) + 1; + const Key new_key = MakeKey(k, g); + list_.Insert(new_key); + current_.Set(k, g); + } + + void ReadStep(Random* rnd) { + // Remember the initial committed state of the skiplist. + State initial_state; + for (unsigned int k = 0; k < K; k++) { + initial_state.Set(k, current_.Get(k)); + } + + Key pos = RandomTarget(rnd); + SkipList::Iterator iter(&list_); + iter.Seek(pos); + while (true) { + Key current; + if (!iter.Valid()) { + current = MakeKey(K, 0); + } else { + current = iter.key(); + ASSERT_TRUE(IsValidKey(current)) << current; + } + ASSERT_LE(pos, current) << "should not go backwards"; + + // Verify that everything in [pos,current) was not present in + // initial_state. + while (pos < current) { + ASSERT_LT(key(pos), K) << pos; + + // Note that generation 0 is never inserted, so it is ok if + // <*,0,*> is missing. + ASSERT_TRUE((gen(pos) == 0U) || + (gen(pos) > static_cast(initial_state.Get( + static_cast(key(pos)))))) + << "key: " << key(pos) << "; gen: " << gen(pos) + << "; initgen: " << initial_state.Get(static_cast(key(pos))); + + // Advance to next key in the valid key space + if (key(pos) < key(current)) { + pos = MakeKey(key(pos) + 1, 0); + } else { + pos = MakeKey(key(pos), gen(pos) + 1); + } + } + + if (!iter.Valid()) { + break; + } + + if (rnd->Next() % 2) { + iter.Next(); + pos = MakeKey(key(pos), gen(pos) + 1); + } else { + Key new_target = RandomTarget(rnd); + if (new_target > pos) { + pos = new_target; + iter.Seek(new_target); + } + } + } + } +}; +const uint32_t ConcurrentTest::K; + +// Simple test that does single-threaded testing of the ConcurrentTest +// scaffolding. +TEST_F(SkipTest, ConcurrentWithoutThreads) { + ConcurrentTest test; + Random rnd(test::RandomSeed()); + for (int i = 0; i < 10000; i++) { + test.ReadStep(&rnd); + test.WriteStep(&rnd); + } +} + +class TestState { + public: + ConcurrentTest t_; + int seed_; + std::atomic quit_flag_; + + enum ReaderState { STARTING, RUNNING, DONE }; + + explicit TestState(int s) + : seed_(s), quit_flag_(false), state_(STARTING), state_cv_(&mu_) {} + + void Wait(ReaderState s) { + mu_.Lock(); + while (state_ != s) { + state_cv_.Wait(); + } + mu_.Unlock(); + } + + void Change(ReaderState s) { + mu_.Lock(); + state_ = s; + state_cv_.Signal(); + mu_.Unlock(); + } + + private: + port::Mutex mu_; + ReaderState state_; + port::CondVar state_cv_; +}; + +static void ConcurrentReader(void* arg) { + TestState* state = reinterpret_cast(arg); + Random rnd(state->seed_); + int64_t reads = 0; + state->Change(TestState::RUNNING); + while (!state->quit_flag_.load(std::memory_order_acquire)) { + state->t_.ReadStep(&rnd); + ++reads; + } + (void)reads; + state->Change(TestState::DONE); +} + +static void RunConcurrent(int run) { + const int seed = test::RandomSeed() + (run * 100); + Random rnd(seed); + const int N = 1000; + const int kSize = 1000; + for (int i = 0; i < N; i++) { + if ((i % 100) == 0) { + fprintf(stderr, "Run %d of %d\n", i, N); + } + TestState state(seed + 1); + Env::Default()->SetBackgroundThreads(1); + Env::Default()->Schedule(ConcurrentReader, &state); + state.Wait(TestState::RUNNING); + for (int k = 0; k < kSize; k++) { + state.t_.WriteStep(&rnd); + } + state.quit_flag_.store(true, std::memory_order_release); + state.Wait(TestState::DONE); + } +} + +TEST_F(SkipTest, Concurrent1) { RunConcurrent(1); } +TEST_F(SkipTest, Concurrent2) { RunConcurrent(2); } +TEST_F(SkipTest, Concurrent3) { RunConcurrent(3); } +TEST_F(SkipTest, Concurrent4) { RunConcurrent(4); } +TEST_F(SkipTest, Concurrent5) { RunConcurrent(5); } + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/memtable/skiplistrep.cc b/librocksdb-sys/rocksdb/memtable/skiplistrep.cc new file mode 100644 index 0000000..c3b4c78 --- /dev/null +++ b/librocksdb-sys/rocksdb/memtable/skiplistrep.cc @@ -0,0 +1,368 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#include + +#include "db/memtable.h" +#include "memory/arena.h" +#include "memtable/inlineskiplist.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/utilities/options_type.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { +namespace { +class SkipListRep : public MemTableRep { + InlineSkipList skip_list_; + const MemTableRep::KeyComparator& cmp_; + const SliceTransform* transform_; + const size_t lookahead_; + + friend class LookaheadIterator; + + public: + explicit SkipListRep(const MemTableRep::KeyComparator& compare, + Allocator* allocator, const SliceTransform* transform, + const size_t lookahead) + : MemTableRep(allocator), + skip_list_(compare, allocator), + cmp_(compare), + transform_(transform), + lookahead_(lookahead) {} + + KeyHandle Allocate(const size_t len, char** buf) override { + *buf = skip_list_.AllocateKey(len); + return static_cast(*buf); + } + + // Insert key into the list. + // REQUIRES: nothing that compares equal to key is currently in the list. + void Insert(KeyHandle handle) override { + skip_list_.Insert(static_cast(handle)); + } + + bool InsertKey(KeyHandle handle) override { + return skip_list_.Insert(static_cast(handle)); + } + + void InsertWithHint(KeyHandle handle, void** hint) override { + skip_list_.InsertWithHint(static_cast(handle), hint); + } + + bool InsertKeyWithHint(KeyHandle handle, void** hint) override { + return skip_list_.InsertWithHint(static_cast(handle), hint); + } + + void InsertWithHintConcurrently(KeyHandle handle, void** hint) override { + skip_list_.InsertWithHintConcurrently(static_cast(handle), hint); + } + + bool InsertKeyWithHintConcurrently(KeyHandle handle, void** hint) override { + return skip_list_.InsertWithHintConcurrently(static_cast(handle), + hint); + } + + void InsertConcurrently(KeyHandle handle) override { + skip_list_.InsertConcurrently(static_cast(handle)); + } + + bool InsertKeyConcurrently(KeyHandle handle) override { + return skip_list_.InsertConcurrently(static_cast(handle)); + } + + // Returns true iff an entry that compares equal to key is in the list. + bool Contains(const char* key) const override { + return skip_list_.Contains(key); + } + + size_t ApproximateMemoryUsage() override { + // All memory is allocated through allocator; nothing to report here + return 0; + } + + void Get(const LookupKey& k, void* callback_args, + bool (*callback_func)(void* arg, const char* entry)) override { + SkipListRep::Iterator iter(&skip_list_); + Slice dummy_slice; + for (iter.Seek(dummy_slice, k.memtable_key().data()); + iter.Valid() && callback_func(callback_args, iter.key()); + iter.Next()) { + } + } + + uint64_t ApproximateNumEntries(const Slice& start_ikey, + const Slice& end_ikey) override { + std::string tmp; + uint64_t start_count = + skip_list_.EstimateCount(EncodeKey(&tmp, start_ikey)); + uint64_t end_count = skip_list_.EstimateCount(EncodeKey(&tmp, end_ikey)); + return (end_count >= start_count) ? (end_count - start_count) : 0; + } + + void UniqueRandomSample(const uint64_t num_entries, + const uint64_t target_sample_size, + std::unordered_set* entries) override { + entries->clear(); + // Avoid divide-by-0. + assert(target_sample_size > 0); + assert(num_entries > 0); + // NOTE: the size of entries is not enforced to be exactly + // target_sample_size at the end of this function, it might be slightly + // greater or smaller. + SkipListRep::Iterator iter(&skip_list_); + // There are two methods to create the subset of samples (size m) + // from the table containing N elements: + // 1-Iterate linearly through the N memtable entries. For each entry i, + // add it to the sample set with a probability + // (target_sample_size - entries.size() ) / (N-i). + // + // 2-Pick m random elements without repetition. + // We pick Option 2 when m sqrt(N). + if (target_sample_size > + static_cast(std::sqrt(1.0 * num_entries))) { + Random* rnd = Random::GetTLSInstance(); + iter.SeekToFirst(); + uint64_t counter = 0, num_samples_left = target_sample_size; + for (; iter.Valid() && (num_samples_left > 0); iter.Next(), counter++) { + // Add entry to sample set with probability + // num_samples_left/(num_entries - counter). + if (rnd->Next() % (num_entries - counter) < num_samples_left) { + entries->insert(iter.key()); + num_samples_left--; + } + } + } else { + // Option 2: pick m random elements with no duplicates. + // If Option 2 is picked, then target_sample_size99.9% for N>4. + // At worst, for the final pick , when m=sqrt(N) there is + // a probability of p= 1/sqrt(N) chances to find a duplicate. + for (uint64_t j = 0; j < 5; j++) { + iter.RandomSeek(); + // unordered_set::insert returns pair. + // The second element is true if an insert successfully happened. + // If element is already in the set, this bool will be false, and + // true otherwise. + if ((entries->insert(iter.key())).second) { + break; + } + } + } + } + } + + ~SkipListRep() override {} + + // Iteration over the contents of a skip list + class Iterator : public MemTableRep::Iterator { + InlineSkipList::Iterator iter_; + + public: + // Initialize an iterator over the specified list. + // The returned iterator is not valid. + explicit Iterator( + const InlineSkipList* list) + : iter_(list) {} + + ~Iterator() override {} + + // Returns true iff the iterator is positioned at a valid node. + bool Valid() const override { return iter_.Valid(); } + + // Returns the key at the current position. + // REQUIRES: Valid() + const char* key() const override { return iter_.key(); } + + // Advances to the next position. + // REQUIRES: Valid() + void Next() override { iter_.Next(); } + + // Advances to the previous position. + // REQUIRES: Valid() + void Prev() override { iter_.Prev(); } + + // Advance to the first entry with a key >= target + void Seek(const Slice& user_key, const char* memtable_key) override { + if (memtable_key != nullptr) { + iter_.Seek(memtable_key); + } else { + iter_.Seek(EncodeKey(&tmp_, user_key)); + } + } + + // Retreat to the last entry with a key <= target + void SeekForPrev(const Slice& user_key, const char* memtable_key) override { + if (memtable_key != nullptr) { + iter_.SeekForPrev(memtable_key); + } else { + iter_.SeekForPrev(EncodeKey(&tmp_, user_key)); + } + } + + void RandomSeek() override { iter_.RandomSeek(); } + + // Position at the first entry in list. + // Final state of iterator is Valid() iff list is not empty. + void SeekToFirst() override { iter_.SeekToFirst(); } + + // Position at the last entry in list. + // Final state of iterator is Valid() iff list is not empty. + void SeekToLast() override { iter_.SeekToLast(); } + + protected: + std::string tmp_; // For passing to EncodeKey + }; + + // Iterator over the contents of a skip list which also keeps track of the + // previously visited node. In Seek(), it examines a few nodes after it + // first, falling back to O(log n) search from the head of the list only if + // the target key hasn't been found. + class LookaheadIterator : public MemTableRep::Iterator { + public: + explicit LookaheadIterator(const SkipListRep& rep) + : rep_(rep), iter_(&rep_.skip_list_), prev_(iter_) {} + + ~LookaheadIterator() override {} + + bool Valid() const override { return iter_.Valid(); } + + const char* key() const override { + assert(Valid()); + return iter_.key(); + } + + void Next() override { + assert(Valid()); + + bool advance_prev = true; + if (prev_.Valid()) { + auto k1 = rep_.UserKey(prev_.key()); + auto k2 = rep_.UserKey(iter_.key()); + + if (k1.compare(k2) == 0) { + // same user key, don't move prev_ + advance_prev = false; + } else if (rep_.transform_) { + // only advance prev_ if it has the same prefix as iter_ + auto t1 = rep_.transform_->Transform(k1); + auto t2 = rep_.transform_->Transform(k2); + advance_prev = t1.compare(t2) == 0; + } + } + + if (advance_prev) { + prev_ = iter_; + } + iter_.Next(); + } + + void Prev() override { + assert(Valid()); + iter_.Prev(); + prev_ = iter_; + } + + void Seek(const Slice& internal_key, const char* memtable_key) override { + const char* encoded_key = (memtable_key != nullptr) + ? memtable_key + : EncodeKey(&tmp_, internal_key); + + if (prev_.Valid() && rep_.cmp_(encoded_key, prev_.key()) >= 0) { + // prev_.key() is smaller or equal to our target key; do a quick + // linear search (at most lookahead_ steps) starting from prev_ + iter_ = prev_; + + size_t cur = 0; + while (cur++ <= rep_.lookahead_ && iter_.Valid()) { + if (rep_.cmp_(encoded_key, iter_.key()) <= 0) { + return; + } + Next(); + } + } + + iter_.Seek(encoded_key); + prev_ = iter_; + } + + void SeekForPrev(const Slice& internal_key, + const char* memtable_key) override { + const char* encoded_key = (memtable_key != nullptr) + ? memtable_key + : EncodeKey(&tmp_, internal_key); + iter_.SeekForPrev(encoded_key); + prev_ = iter_; + } + + void SeekToFirst() override { + iter_.SeekToFirst(); + prev_ = iter_; + } + + void SeekToLast() override { + iter_.SeekToLast(); + prev_ = iter_; + } + + protected: + std::string tmp_; // For passing to EncodeKey + + private: + const SkipListRep& rep_; + InlineSkipList::Iterator iter_; + InlineSkipList::Iterator prev_; + }; + + MemTableRep::Iterator* GetIterator(Arena* arena = nullptr) override { + if (lookahead_ > 0) { + void* mem = + arena ? arena->AllocateAligned(sizeof(SkipListRep::LookaheadIterator)) + : + operator new(sizeof(SkipListRep::LookaheadIterator)); + return new (mem) SkipListRep::LookaheadIterator(*this); + } else { + void* mem = arena ? arena->AllocateAligned(sizeof(SkipListRep::Iterator)) + : + operator new(sizeof(SkipListRep::Iterator)); + return new (mem) SkipListRep::Iterator(&skip_list_); + } + } +}; +} // namespace + +static std::unordered_map skiplist_factory_info = { + {"lookahead", + {0, OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kDontSerialize /*Since it is part of the ID*/}}, +}; + +SkipListFactory::SkipListFactory(size_t lookahead) : lookahead_(lookahead) { + RegisterOptions("SkipListFactoryOptions", &lookahead_, + &skiplist_factory_info); +} + +std::string SkipListFactory::GetId() const { + std::string id = Name(); + if (lookahead_ > 0) { + id.append(":").append(std::to_string(lookahead_)); + } + return id; +} + +MemTableRep* SkipListFactory::CreateMemTableRep( + const MemTableRep::KeyComparator& compare, Allocator* allocator, + const SliceTransform* transform, Logger* /*logger*/) { + return new SkipListRep(compare, allocator, transform, lookahead_); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/memtable/stl_wrappers.h b/librocksdb-sys/rocksdb/memtable/stl_wrappers.h new file mode 100644 index 0000000..783a808 --- /dev/null +++ b/librocksdb-sys/rocksdb/memtable/stl_wrappers.h @@ -0,0 +1,33 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include +#include + +#include "rocksdb/comparator.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/slice.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { +namespace stl_wrappers { + +class Base { + protected: + const MemTableRep::KeyComparator& compare_; + explicit Base(const MemTableRep::KeyComparator& compare) + : compare_(compare) {} +}; + +struct Compare : private Base { + explicit Compare(const MemTableRep::KeyComparator& compare) : Base(compare) {} + inline bool operator()(const char* a, const char* b) const { + return compare_(a, b) < 0; + } +}; + +} // namespace stl_wrappers +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/memtable/vectorrep.cc b/librocksdb-sys/rocksdb/memtable/vectorrep.cc new file mode 100644 index 0000000..e42ae44 --- /dev/null +++ b/librocksdb-sys/rocksdb/memtable/vectorrep.cc @@ -0,0 +1,307 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#include +#include +#include +#include +#include + +#include "db/memtable.h" +#include "memory/arena.h" +#include "memtable/stl_wrappers.h" +#include "port/port.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/utilities/options_type.h" +#include "util/mutexlock.h" + +namespace ROCKSDB_NAMESPACE { +namespace { + +class VectorRep : public MemTableRep { + public: + VectorRep(const KeyComparator& compare, Allocator* allocator, size_t count); + + // Insert key into the collection. (The caller will pack key and value into a + // single buffer and pass that in as the parameter to Insert) + // REQUIRES: nothing that compares equal to key is currently in the + // collection. + void Insert(KeyHandle handle) override; + + // Returns true iff an entry that compares equal to key is in the collection. + bool Contains(const char* key) const override; + + void MarkReadOnly() override; + + size_t ApproximateMemoryUsage() override; + + void Get(const LookupKey& k, void* callback_args, + bool (*callback_func)(void* arg, const char* entry)) override; + + ~VectorRep() override {} + + class Iterator : public MemTableRep::Iterator { + class VectorRep* vrep_; + std::shared_ptr> bucket_; + std::vector::const_iterator mutable cit_; + const KeyComparator& compare_; + std::string tmp_; // For passing to EncodeKey + bool mutable sorted_; + void DoSort() const; + + public: + explicit Iterator(class VectorRep* vrep, + std::shared_ptr> bucket, + const KeyComparator& compare); + + // Initialize an iterator over the specified collection. + // The returned iterator is not valid. + // explicit Iterator(const MemTableRep* collection); + ~Iterator() override{}; + + // Returns true iff the iterator is positioned at a valid node. + bool Valid() const override; + + // Returns the key at the current position. + // REQUIRES: Valid() + const char* key() const override; + + // Advances to the next position. + // REQUIRES: Valid() + void Next() override; + + // Advances to the previous position. + // REQUIRES: Valid() + void Prev() override; + + // Advance to the first entry with a key >= target + void Seek(const Slice& user_key, const char* memtable_key) override; + + // Advance to the first entry with a key <= target + void SeekForPrev(const Slice& user_key, const char* memtable_key) override; + + // Position at the first entry in collection. + // Final state of iterator is Valid() iff collection is not empty. + void SeekToFirst() override; + + // Position at the last entry in collection. + // Final state of iterator is Valid() iff collection is not empty. + void SeekToLast() override; + }; + + // Return an iterator over the keys in this representation. + MemTableRep::Iterator* GetIterator(Arena* arena) override; + + private: + friend class Iterator; + using Bucket = std::vector; + std::shared_ptr bucket_; + mutable port::RWMutex rwlock_; + bool immutable_; + bool sorted_; + const KeyComparator& compare_; +}; + +void VectorRep::Insert(KeyHandle handle) { + auto* key = static_cast(handle); + WriteLock l(&rwlock_); + assert(!immutable_); + bucket_->push_back(key); +} + +// Returns true iff an entry that compares equal to key is in the collection. +bool VectorRep::Contains(const char* key) const { + ReadLock l(&rwlock_); + return std::find(bucket_->begin(), bucket_->end(), key) != bucket_->end(); +} + +void VectorRep::MarkReadOnly() { + WriteLock l(&rwlock_); + immutable_ = true; +} + +size_t VectorRep::ApproximateMemoryUsage() { + return sizeof(bucket_) + sizeof(*bucket_) + + bucket_->size() * + sizeof( + std::remove_reference::type::value_type); +} + +VectorRep::VectorRep(const KeyComparator& compare, Allocator* allocator, + size_t count) + : MemTableRep(allocator), + bucket_(new Bucket()), + immutable_(false), + sorted_(false), + compare_(compare) { + bucket_.get()->reserve(count); +} + +VectorRep::Iterator::Iterator(class VectorRep* vrep, + std::shared_ptr> bucket, + const KeyComparator& compare) + : vrep_(vrep), + bucket_(bucket), + cit_(bucket_->end()), + compare_(compare), + sorted_(false) {} + +void VectorRep::Iterator::DoSort() const { + // vrep is non-null means that we are working on an immutable memtable + if (!sorted_ && vrep_ != nullptr) { + WriteLock l(&vrep_->rwlock_); + if (!vrep_->sorted_) { + std::sort(bucket_->begin(), bucket_->end(), + stl_wrappers::Compare(compare_)); + cit_ = bucket_->begin(); + vrep_->sorted_ = true; + } + sorted_ = true; + } + if (!sorted_) { + std::sort(bucket_->begin(), bucket_->end(), + stl_wrappers::Compare(compare_)); + cit_ = bucket_->begin(); + sorted_ = true; + } + assert(sorted_); + assert(vrep_ == nullptr || vrep_->sorted_); +} + +// Returns true iff the iterator is positioned at a valid node. +bool VectorRep::Iterator::Valid() const { + DoSort(); + return cit_ != bucket_->end(); +} + +// Returns the key at the current position. +// REQUIRES: Valid() +const char* VectorRep::Iterator::key() const { + assert(sorted_); + return *cit_; +} + +// Advances to the next position. +// REQUIRES: Valid() +void VectorRep::Iterator::Next() { + assert(sorted_); + if (cit_ == bucket_->end()) { + return; + } + ++cit_; +} + +// Advances to the previous position. +// REQUIRES: Valid() +void VectorRep::Iterator::Prev() { + assert(sorted_); + if (cit_ == bucket_->begin()) { + // If you try to go back from the first element, the iterator should be + // invalidated. So we set it to past-the-end. This means that you can + // treat the container circularly. + cit_ = bucket_->end(); + } else { + --cit_; + } +} + +// Advance to the first entry with a key >= target +void VectorRep::Iterator::Seek(const Slice& user_key, + const char* memtable_key) { + DoSort(); + // Do binary search to find first value not less than the target + const char* encoded_key = + (memtable_key != nullptr) ? memtable_key : EncodeKey(&tmp_, user_key); + cit_ = std::equal_range(bucket_->begin(), bucket_->end(), encoded_key, + [this](const char* a, const char* b) { + return compare_(a, b) < 0; + }) + .first; +} + +// Advance to the first entry with a key <= target +void VectorRep::Iterator::SeekForPrev(const Slice& /*user_key*/, + const char* /*memtable_key*/) { + assert(false); +} + +// Position at the first entry in collection. +// Final state of iterator is Valid() iff collection is not empty. +void VectorRep::Iterator::SeekToFirst() { + DoSort(); + cit_ = bucket_->begin(); +} + +// Position at the last entry in collection. +// Final state of iterator is Valid() iff collection is not empty. +void VectorRep::Iterator::SeekToLast() { + DoSort(); + cit_ = bucket_->end(); + if (bucket_->size() != 0) { + --cit_; + } +} + +void VectorRep::Get(const LookupKey& k, void* callback_args, + bool (*callback_func)(void* arg, const char* entry)) { + rwlock_.ReadLock(); + VectorRep* vector_rep; + std::shared_ptr bucket; + if (immutable_) { + vector_rep = this; + } else { + vector_rep = nullptr; + bucket.reset(new Bucket(*bucket_)); // make a copy + } + VectorRep::Iterator iter(vector_rep, immutable_ ? bucket_ : bucket, compare_); + rwlock_.ReadUnlock(); + + for (iter.Seek(k.user_key(), k.memtable_key().data()); + iter.Valid() && callback_func(callback_args, iter.key()); iter.Next()) { + } +} + +MemTableRep::Iterator* VectorRep::GetIterator(Arena* arena) { + char* mem = nullptr; + if (arena != nullptr) { + mem = arena->AllocateAligned(sizeof(Iterator)); + } + ReadLock l(&rwlock_); + // Do not sort here. The sorting would be done the first time + // a Seek is performed on the iterator. + if (immutable_) { + if (arena == nullptr) { + return new Iterator(this, bucket_, compare_); + } else { + return new (mem) Iterator(this, bucket_, compare_); + } + } else { + std::shared_ptr tmp; + tmp.reset(new Bucket(*bucket_)); // make a copy + if (arena == nullptr) { + return new Iterator(nullptr, tmp, compare_); + } else { + return new (mem) Iterator(nullptr, tmp, compare_); + } + } +} +} // namespace + +static std::unordered_map vector_rep_table_info = { + {"count", + {0, OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, +}; + +VectorRepFactory::VectorRepFactory(size_t count) : count_(count) { + RegisterOptions("VectorRepFactoryOptions", &count_, &vector_rep_table_info); +} + +MemTableRep* VectorRepFactory::CreateMemTableRep( + const MemTableRep::KeyComparator& compare, Allocator* allocator, + const SliceTransform*, Logger* /*logger*/) { + return new VectorRep(compare, allocator, count_); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/memtable/write_buffer_manager.cc b/librocksdb-sys/rocksdb/memtable/write_buffer_manager.cc new file mode 100644 index 0000000..ce1789c --- /dev/null +++ b/librocksdb-sys/rocksdb/memtable/write_buffer_manager.cc @@ -0,0 +1,185 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "rocksdb/write_buffer_manager.h" + +#include + +#include "cache/cache_entry_roles.h" +#include "cache/cache_reservation_manager.h" +#include "db/db_impl/db_impl.h" +#include "rocksdb/status.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { +WriteBufferManager::WriteBufferManager(size_t _buffer_size, + std::shared_ptr cache, + bool allow_stall) + : buffer_size_(_buffer_size), + mutable_limit_(buffer_size_ * 7 / 8), + memory_used_(0), + memory_active_(0), + cache_res_mgr_(nullptr), + allow_stall_(allow_stall), + stall_active_(false) { + if (cache) { + // Memtable's memory usage tends to fluctuate frequently + // therefore we set delayed_decrease = true to save some dummy entry + // insertion on memory increase right after memory decrease + cache_res_mgr_ = std::make_shared< + CacheReservationManagerImpl>( + cache, true /* delayed_decrease */); + } +} + +WriteBufferManager::~WriteBufferManager() { +#ifndef NDEBUG + std::unique_lock lock(mu_); + assert(queue_.empty()); +#endif +} + +std::size_t WriteBufferManager::dummy_entries_in_cache_usage() const { + if (cache_res_mgr_ != nullptr) { + return cache_res_mgr_->GetTotalReservedCacheSize(); + } else { + return 0; + } +} + +void WriteBufferManager::ReserveMem(size_t mem) { + if (cache_res_mgr_ != nullptr) { + ReserveMemWithCache(mem); + } else if (enabled()) { + memory_used_.fetch_add(mem, std::memory_order_relaxed); + } + if (enabled()) { + memory_active_.fetch_add(mem, std::memory_order_relaxed); + } +} + +// Should only be called from write thread +void WriteBufferManager::ReserveMemWithCache(size_t mem) { + assert(cache_res_mgr_ != nullptr); + // Use a mutex to protect various data structures. Can be optimized to a + // lock-free solution if it ends up with a performance bottleneck. + std::lock_guard lock(cache_res_mgr_mu_); + + size_t new_mem_used = memory_used_.load(std::memory_order_relaxed) + mem; + memory_used_.store(new_mem_used, std::memory_order_relaxed); + Status s = cache_res_mgr_->UpdateCacheReservation(new_mem_used); + + // We absorb the error since WriteBufferManager is not able to handle + // this failure properly. Ideallly we should prevent this allocation + // from happening if this cache charging fails. + // [TODO] We'll need to improve it in the future and figure out what to do on + // error + s.PermitUncheckedError(); +} + +void WriteBufferManager::ScheduleFreeMem(size_t mem) { + if (enabled()) { + memory_active_.fetch_sub(mem, std::memory_order_relaxed); + } +} + +void WriteBufferManager::FreeMem(size_t mem) { + if (cache_res_mgr_ != nullptr) { + FreeMemWithCache(mem); + } else if (enabled()) { + memory_used_.fetch_sub(mem, std::memory_order_relaxed); + } + // Check if stall is active and can be ended. + MaybeEndWriteStall(); +} + +void WriteBufferManager::FreeMemWithCache(size_t mem) { + assert(cache_res_mgr_ != nullptr); + // Use a mutex to protect various data structures. Can be optimized to a + // lock-free solution if it ends up with a performance bottleneck. + std::lock_guard lock(cache_res_mgr_mu_); + size_t new_mem_used = memory_used_.load(std::memory_order_relaxed) - mem; + memory_used_.store(new_mem_used, std::memory_order_relaxed); + Status s = cache_res_mgr_->UpdateCacheReservation(new_mem_used); + + // We absorb the error since WriteBufferManager is not able to handle + // this failure properly. + // [TODO] We'll need to improve it in the future and figure out what to do on + // error + s.PermitUncheckedError(); +} + +void WriteBufferManager::BeginWriteStall(StallInterface* wbm_stall) { + assert(wbm_stall != nullptr); + + // Allocate outside of the lock. + std::list new_node = {wbm_stall}; + + { + std::unique_lock lock(mu_); + // Verify if the stall conditions are stil active. + if (ShouldStall()) { + stall_active_.store(true, std::memory_order_relaxed); + queue_.splice(queue_.end(), std::move(new_node)); + } + } + + // If the node was not consumed, the stall has ended already and we can signal + // the caller. + if (!new_node.empty()) { + new_node.front()->Signal(); + } +} + +// Called when memory is freed in FreeMem or the buffer size has changed. +void WriteBufferManager::MaybeEndWriteStall() { + // Stall conditions have not been resolved. + if (allow_stall_.load(std::memory_order_relaxed) && + IsStallThresholdExceeded()) { + return; + } + + // Perform all deallocations outside of the lock. + std::list cleanup; + + std::unique_lock lock(mu_); + if (!stall_active_.load(std::memory_order_relaxed)) { + return; // Nothing to do. + } + + // Unblock new writers. + stall_active_.store(false, std::memory_order_relaxed); + + // Unblock the writers in the queue. + for (StallInterface* wbm_stall : queue_) { + wbm_stall->Signal(); + } + cleanup = std::move(queue_); +} + +void WriteBufferManager::RemoveDBFromQueue(StallInterface* wbm_stall) { + assert(wbm_stall != nullptr); + + // Deallocate the removed nodes outside of the lock. + std::list cleanup; + + if (enabled() && allow_stall_.load(std::memory_order_relaxed)) { + std::unique_lock lock(mu_); + for (auto it = queue_.begin(); it != queue_.end();) { + auto next = std::next(it); + if (*it == wbm_stall) { + cleanup.splice(cleanup.end(), queue_, std::move(it)); + } + it = next; + } + } + wbm_stall->Signal(); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/memtable/write_buffer_manager_test.cc b/librocksdb-sys/rocksdb/memtable/write_buffer_manager_test.cc new file mode 100644 index 0000000..c992d2e --- /dev/null +++ b/librocksdb-sys/rocksdb/memtable/write_buffer_manager_test.cc @@ -0,0 +1,304 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "rocksdb/write_buffer_manager.h" + +#include "rocksdb/advanced_cache.h" +#include "test_util/testharness.h" + +namespace ROCKSDB_NAMESPACE { +class WriteBufferManagerTest : public testing::Test {}; + +const size_t kSizeDummyEntry = 256 * 1024; + +TEST_F(WriteBufferManagerTest, ShouldFlush) { + // A write buffer manager of size 10MB + std::unique_ptr wbf( + new WriteBufferManager(10 * 1024 * 1024)); + + wbf->ReserveMem(8 * 1024 * 1024); + ASSERT_FALSE(wbf->ShouldFlush()); + // 90% of the hard limit will hit the condition + wbf->ReserveMem(1 * 1024 * 1024); + ASSERT_TRUE(wbf->ShouldFlush()); + // Scheduling for freeing will release the condition + wbf->ScheduleFreeMem(1 * 1024 * 1024); + ASSERT_FALSE(wbf->ShouldFlush()); + + wbf->ReserveMem(2 * 1024 * 1024); + ASSERT_TRUE(wbf->ShouldFlush()); + + wbf->ScheduleFreeMem(4 * 1024 * 1024); + // 11MB total, 6MB mutable. hard limit still hit + ASSERT_TRUE(wbf->ShouldFlush()); + + wbf->ScheduleFreeMem(2 * 1024 * 1024); + // 11MB total, 4MB mutable. hard limit stills but won't flush because more + // than half data is already being flushed. + ASSERT_FALSE(wbf->ShouldFlush()); + + wbf->ReserveMem(4 * 1024 * 1024); + // 15 MB total, 8MB mutable. + ASSERT_TRUE(wbf->ShouldFlush()); + + wbf->FreeMem(7 * 1024 * 1024); + // 8MB total, 8MB mutable. + ASSERT_FALSE(wbf->ShouldFlush()); + + // change size: 8M limit, 7M mutable limit + wbf->SetBufferSize(8 * 1024 * 1024); + // 8MB total, 8MB mutable. + ASSERT_TRUE(wbf->ShouldFlush()); + + wbf->ScheduleFreeMem(2 * 1024 * 1024); + // 8MB total, 6MB mutable. + ASSERT_TRUE(wbf->ShouldFlush()); + + wbf->FreeMem(2 * 1024 * 1024); + // 6MB total, 6MB mutable. + ASSERT_FALSE(wbf->ShouldFlush()); + + wbf->ReserveMem(1 * 1024 * 1024); + // 7MB total, 7MB mutable. + ASSERT_FALSE(wbf->ShouldFlush()); + + wbf->ReserveMem(1 * 1024 * 1024); + // 8MB total, 8MB mutable. + ASSERT_TRUE(wbf->ShouldFlush()); + + wbf->ScheduleFreeMem(1 * 1024 * 1024); + wbf->FreeMem(1 * 1024 * 1024); + // 7MB total, 7MB mutable. + ASSERT_FALSE(wbf->ShouldFlush()); +} + +class ChargeWriteBufferTest : public testing::Test {}; + +TEST_F(ChargeWriteBufferTest, Basic) { + constexpr std::size_t kMetaDataChargeOverhead = 10000; + + LRUCacheOptions co; + // 1GB cache + co.capacity = 1024 * 1024 * 1024; + co.num_shard_bits = 4; + co.metadata_charge_policy = kDontChargeCacheMetadata; + std::shared_ptr cache = NewLRUCache(co); + // A write buffer manager of size 50MB + std::unique_ptr wbf( + new WriteBufferManager(50 * 1024 * 1024, cache)); + + // Allocate 333KB will allocate 512KB, memory_used_ = 333KB + wbf->ReserveMem(333 * 1024); + // 2 dummy entries are added for size 333 KB + ASSERT_EQ(wbf->dummy_entries_in_cache_usage(), 2 * kSizeDummyEntry); + ASSERT_GE(cache->GetPinnedUsage(), 2 * 256 * 1024); + ASSERT_LT(cache->GetPinnedUsage(), 2 * 256 * 1024 + kMetaDataChargeOverhead); + + // Allocate another 512KB, memory_used_ = 845KB + wbf->ReserveMem(512 * 1024); + // 2 more dummy entries are added for size 512 KB + // since ceil((memory_used_ - dummy_entries_in_cache_usage) % kSizeDummyEntry) + // = 2 + ASSERT_EQ(wbf->dummy_entries_in_cache_usage(), 4 * kSizeDummyEntry); + ASSERT_GE(cache->GetPinnedUsage(), 4 * 256 * 1024); + ASSERT_LT(cache->GetPinnedUsage(), 4 * 256 * 1024 + kMetaDataChargeOverhead); + + // Allocate another 10MB, memory_used_ = 11085KB + wbf->ReserveMem(10 * 1024 * 1024); + // 40 more entries are added for size 10 * 1024 * 1024 KB + ASSERT_EQ(wbf->dummy_entries_in_cache_usage(), 44 * kSizeDummyEntry); + ASSERT_GE(cache->GetPinnedUsage(), 44 * 256 * 1024); + ASSERT_LT(cache->GetPinnedUsage(), 44 * 256 * 1024 + kMetaDataChargeOverhead); + + // Free 1MB, memory_used_ = 10061KB + // It will not cause any change in cache cost + // since memory_used_ > dummy_entries_in_cache_usage * (3/4) + wbf->FreeMem(1 * 1024 * 1024); + ASSERT_EQ(wbf->dummy_entries_in_cache_usage(), 44 * kSizeDummyEntry); + ASSERT_GE(cache->GetPinnedUsage(), 44 * 256 * 1024); + ASSERT_LT(cache->GetPinnedUsage(), 44 * 256 * 1024 + kMetaDataChargeOverhead); + ASSERT_FALSE(wbf->ShouldFlush()); + + // Allocate another 41MB, memory_used_ = 52045KB + wbf->ReserveMem(41 * 1024 * 1024); + ASSERT_EQ(wbf->dummy_entries_in_cache_usage(), 204 * kSizeDummyEntry); + ASSERT_GE(cache->GetPinnedUsage(), 204 * 256 * 1024); + ASSERT_LT(cache->GetPinnedUsage(), + 204 * 256 * 1024 + kMetaDataChargeOverhead); + ASSERT_TRUE(wbf->ShouldFlush()); + + ASSERT_TRUE(wbf->ShouldFlush()); + + // Schedule free 20MB, memory_used_ = 52045KB + // It will not cause any change in memory_used and cache cost + wbf->ScheduleFreeMem(20 * 1024 * 1024); + ASSERT_EQ(wbf->dummy_entries_in_cache_usage(), 204 * kSizeDummyEntry); + ASSERT_GE(cache->GetPinnedUsage(), 204 * 256 * 1024); + ASSERT_LT(cache->GetPinnedUsage(), + 204 * 256 * 1024 + kMetaDataChargeOverhead); + // Still need flush as the hard limit hits + ASSERT_TRUE(wbf->ShouldFlush()); + + // Free 20MB, memory_used_ = 31565KB + // It will releae 80 dummy entries from cache since + // since memory_used_ < dummy_entries_in_cache_usage * (3/4) + // and floor((dummy_entries_in_cache_usage - memory_used_) % kSizeDummyEntry) + // = 80 + wbf->FreeMem(20 * 1024 * 1024); + ASSERT_EQ(wbf->dummy_entries_in_cache_usage(), 124 * kSizeDummyEntry); + ASSERT_GE(cache->GetPinnedUsage(), 124 * 256 * 1024); + ASSERT_LT(cache->GetPinnedUsage(), + 124 * 256 * 1024 + kMetaDataChargeOverhead); + + ASSERT_FALSE(wbf->ShouldFlush()); + + // Free 16KB, memory_used_ = 31549KB + // It will not release any dummy entry since memory_used_ >= + // dummy_entries_in_cache_usage * (3/4) + wbf->FreeMem(16 * 1024); + ASSERT_EQ(wbf->dummy_entries_in_cache_usage(), 124 * kSizeDummyEntry); + ASSERT_GE(cache->GetPinnedUsage(), 124 * 256 * 1024); + ASSERT_LT(cache->GetPinnedUsage(), + 124 * 256 * 1024 + kMetaDataChargeOverhead); + + // Free 20MB, memory_used_ = 11069KB + // It will releae 80 dummy entries from cache + // since memory_used_ < dummy_entries_in_cache_usage * (3/4) + // and floor((dummy_entries_in_cache_usage - memory_used_) % kSizeDummyEntry) + // = 80 + wbf->FreeMem(20 * 1024 * 1024); + ASSERT_EQ(wbf->dummy_entries_in_cache_usage(), 44 * kSizeDummyEntry); + ASSERT_GE(cache->GetPinnedUsage(), 44 * 256 * 1024); + ASSERT_LT(cache->GetPinnedUsage(), 44 * 256 * 1024 + kMetaDataChargeOverhead); + + // Free 1MB, memory_used_ = 10045KB + // It will not cause any change in cache cost + // since memory_used_ > dummy_entries_in_cache_usage * (3/4) + wbf->FreeMem(1 * 1024 * 1024); + ASSERT_EQ(wbf->dummy_entries_in_cache_usage(), 44 * kSizeDummyEntry); + ASSERT_GE(cache->GetPinnedUsage(), 44 * 256 * 1024); + ASSERT_LT(cache->GetPinnedUsage(), 44 * 256 * 1024 + kMetaDataChargeOverhead); + + // Reserve 512KB, memory_used_ = 10557KB + // It will not casue any change in cache cost + // since memory_used_ > dummy_entries_in_cache_usage * (3/4) + // which reflects the benefit of saving dummy entry insertion on memory + // reservation after delay decrease + wbf->ReserveMem(512 * 1024); + ASSERT_EQ(wbf->dummy_entries_in_cache_usage(), 44 * kSizeDummyEntry); + ASSERT_GE(cache->GetPinnedUsage(), 44 * 256 * 1024); + ASSERT_LT(cache->GetPinnedUsage(), 44 * 256 * 1024 + kMetaDataChargeOverhead); + + // Destroy write buffer manger should free everything + wbf.reset(); + ASSERT_EQ(cache->GetPinnedUsage(), 0); +} + +TEST_F(ChargeWriteBufferTest, BasicWithNoBufferSizeLimit) { + constexpr std::size_t kMetaDataChargeOverhead = 10000; + // 1GB cache + std::shared_ptr cache = NewLRUCache(1024 * 1024 * 1024, 4); + // A write buffer manager of size 256MB + std::unique_ptr wbf(new WriteBufferManager(0, cache)); + + // Allocate 10MB, memory_used_ = 10240KB + // It will allocate 40 dummy entries + wbf->ReserveMem(10 * 1024 * 1024); + ASSERT_EQ(wbf->dummy_entries_in_cache_usage(), 40 * kSizeDummyEntry); + ASSERT_GE(cache->GetPinnedUsage(), 40 * 256 * 1024); + ASSERT_LT(cache->GetPinnedUsage(), 40 * 256 * 1024 + kMetaDataChargeOverhead); + + ASSERT_FALSE(wbf->ShouldFlush()); + + // Free 9MB, memory_used_ = 1024KB + // It will free 36 dummy entries + wbf->FreeMem(9 * 1024 * 1024); + ASSERT_EQ(wbf->dummy_entries_in_cache_usage(), 4 * kSizeDummyEntry); + ASSERT_GE(cache->GetPinnedUsage(), 4 * 256 * 1024); + ASSERT_LT(cache->GetPinnedUsage(), 4 * 256 * 1024 + kMetaDataChargeOverhead); + + // Free 160KB gradually, memory_used_ = 864KB + // It will not cause any change + // since memory_used_ > dummy_entries_in_cache_usage * 3/4 + for (int i = 0; i < 40; i++) { + wbf->FreeMem(4 * 1024); + } + ASSERT_EQ(wbf->dummy_entries_in_cache_usage(), 4 * kSizeDummyEntry); + ASSERT_GE(cache->GetPinnedUsage(), 4 * 256 * 1024); + ASSERT_LT(cache->GetPinnedUsage(), 4 * 256 * 1024 + kMetaDataChargeOverhead); +} + +TEST_F(ChargeWriteBufferTest, BasicWithCacheFull) { + constexpr std::size_t kMetaDataChargeOverhead = 20000; + + // 12MB cache size with strict capacity + LRUCacheOptions lo; + lo.capacity = 12 * 1024 * 1024; + lo.num_shard_bits = 0; + lo.strict_capacity_limit = true; + std::shared_ptr cache = NewLRUCache(lo); + std::unique_ptr wbf(new WriteBufferManager(0, cache)); + + // Allocate 10MB, memory_used_ = 10240KB + wbf->ReserveMem(10 * 1024 * 1024); + ASSERT_EQ(wbf->dummy_entries_in_cache_usage(), 40 * kSizeDummyEntry); + ASSERT_GE(cache->GetPinnedUsage(), 40 * kSizeDummyEntry); + ASSERT_LT(cache->GetPinnedUsage(), + 40 * kSizeDummyEntry + kMetaDataChargeOverhead); + + // Allocate 10MB, memory_used_ = 20480KB + // Some dummy entry insertion will fail due to full cache + wbf->ReserveMem(10 * 1024 * 1024); + ASSERT_GE(cache->GetPinnedUsage(), 40 * kSizeDummyEntry); + ASSERT_LE(cache->GetPinnedUsage(), 12 * 1024 * 1024); + ASSERT_LT(wbf->dummy_entries_in_cache_usage(), 80 * kSizeDummyEntry); + + // Free 15MB after encoutering cache full, memory_used_ = 5120KB + wbf->FreeMem(15 * 1024 * 1024); + ASSERT_EQ(wbf->dummy_entries_in_cache_usage(), 20 * kSizeDummyEntry); + ASSERT_GE(cache->GetPinnedUsage(), 20 * kSizeDummyEntry); + ASSERT_LT(cache->GetPinnedUsage(), + 20 * kSizeDummyEntry + kMetaDataChargeOverhead); + + // Reserve 15MB, creating cache full again, memory_used_ = 20480KB + wbf->ReserveMem(15 * 1024 * 1024); + ASSERT_LE(cache->GetPinnedUsage(), 12 * 1024 * 1024); + ASSERT_LT(wbf->dummy_entries_in_cache_usage(), 80 * kSizeDummyEntry); + + // Increase capacity so next insert will fully succeed + cache->SetCapacity(40 * 1024 * 1024); + + // Allocate 10MB, memory_used_ = 30720KB + wbf->ReserveMem(10 * 1024 * 1024); + ASSERT_EQ(wbf->dummy_entries_in_cache_usage(), 120 * kSizeDummyEntry); + ASSERT_GE(cache->GetPinnedUsage(), 120 * kSizeDummyEntry); + ASSERT_LT(cache->GetPinnedUsage(), + 120 * kSizeDummyEntry + kMetaDataChargeOverhead); + + // Gradually release 20 MB + // It ended up sequentially releasing 32, 24, 18 dummy entries when + // memory_used_ decreases to 22528KB, 16384KB, 11776KB. + // In total, it releases 74 dummy entries + for (int i = 0; i < 40; i++) { + wbf->FreeMem(512 * 1024); + } + + ASSERT_EQ(wbf->dummy_entries_in_cache_usage(), 46 * kSizeDummyEntry); + ASSERT_GE(cache->GetPinnedUsage(), 46 * kSizeDummyEntry); + ASSERT_LT(cache->GetPinnedUsage(), + 46 * kSizeDummyEntry + kMetaDataChargeOverhead); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/microbench/CMakeLists.txt b/librocksdb-sys/rocksdb/microbench/CMakeLists.txt new file mode 100644 index 0000000..483e979 --- /dev/null +++ b/librocksdb-sys/rocksdb/microbench/CMakeLists.txt @@ -0,0 +1,17 @@ +find_package(benchmark REQUIRED) +find_package(Threads REQUIRED) + +file(GLOB_RECURSE ALL_BENCH_CPP *.cc) +foreach(ONE_BENCH_CPP ${ALL_BENCH_CPP}) + get_filename_component(TARGET_NAME ${ONE_BENCH_CPP} NAME_WE) + add_executable(${TARGET_NAME} ${ONE_BENCH_CPP}) + target_link_libraries(${TARGET_NAME} ${ROCKSDB_LIB} benchmark::benchmark + ${CMAKE_THREAD_LIBS_INIT}) + # run benchmark like a test, if added, the benchmark tests could be run by `ctest -R Bench_` + # add_test(Bench_${TARGET_NAME} ${TARGET_NAME}) + list(APPEND ALL_BENCH_TARGETS ${TARGET_NAME}) +endforeach() +add_custom_target(microbench DEPENDS ${ALL_BENCH_TARGETS}) +add_custom_target(run_microbench + COMMAND for t in ${ALL_BENCH_TARGETS}\; do \.\/$$t \|\| exit 1\; done + DEPENDS ${ALL_BENCH_TARGETS}) diff --git a/librocksdb-sys/rocksdb/microbench/README.md b/librocksdb-sys/rocksdb/microbench/README.md new file mode 100644 index 0000000..290ca58 --- /dev/null +++ b/librocksdb-sys/rocksdb/microbench/README.md @@ -0,0 +1,60 @@ +# RocksDB Micro-Benchmark + +## Overview + +RocksDB micro-benchmark is a set of tests for benchmarking a single component or simple DB operations. The test artificially generates input data and executes the same operation with it to collect and report performance metrics. As it's focusing on testing a single, well-defined operation, the result is more precise and reproducible, which also has its limitation of not representing a real production use case. The test author needs to carefully design the microbench to represent its true purpose. + +The tests are based on [Google Benchmark](https://github.com/google/benchmark) library, which provides a standard framework for writing benchmarks. + +## How to Run +### Prerequisite +Install the [Google Benchmark](https://github.com/google/benchmark) version `1.6.0` or above. + +*Note: Google Benchmark `1.6.x` is incompatible with previous versions like `1.5.x`, please make sure you're using the newer version.* + +### Build and Run +With `Makefile`: +```bash +$ DEBUG_LEVEL=0 make run_microbench +``` +Or with cmake: +```bash +$ mkdir build && cd build && cmake .. -DCMAKE_BUILD_TYPE=Release -DWITH_BENCHMARK +$ make run_microbench +``` + +*Note: Please run the benchmark code in release build.* +### Run Single Test +Example: +```bash +$ make db_basic_bench +$ ./db_basic_bench --benchmark_filter= +``` + +## Best Practices +#### * Use the Same Test Directory Setting as Unittest +Most of the Micro-benchmark tests use the same test directory setup as unittest, so it could be overridden by: +```bash +$ TEST_TMPDIR=/mydata/tmp/ ./db_basic_bench --benchmark_filter= +``` +Please also follow that when designing new tests. + +#### * Avoid Using Debug API +Even though micro-benchmark is a test, avoid using internal Debug API like TEST_WaitForRun() which is designed for unittest. As benchmark tests are designed for release build, don't use any of that. + +#### * Pay Attention to Local Optimization +As a micro-benchmark is focusing on a single component or area, make sure it is a key part for impacting the overall application performance. + +The compiler might be able to optimize the code that not the same way as the whole application, and if the test data input is simple and small, it may be able to all cached in CPU memory, which is leading to a wrong metric. Take these into consideration when designing the tests. + +#### * Names of user-defined counters/metrics has to be `[A-Za-z0-9_]` +It's a restriction of the metrics collecting and reporting system RocksDB is using internally. It will also help integrate with more systems. + +#### * Minimize the Metrics Variation +Try reducing the test result variation, one way to check that is running the test multiple times and check the CV (Coefficient of Variation) reported by gbenchmark. +```bash +$ ./db_basic_bench --benchmark_filter= --benchmark_repetitions=10 +... +_cv 3.2% +``` +RocksDB has background compaction jobs which may cause the test result to vary a lot. If the micro-benchmark is not purposely testing the operation while compaction is in progress, it should wait for the compaction to finish (`db_impl->WaitForCompact()`) or disable auto-compaction. diff --git a/librocksdb-sys/rocksdb/microbench/db_basic_bench.cc b/librocksdb-sys/rocksdb/microbench/db_basic_bench.cc new file mode 100644 index 0000000..20e7182 --- /dev/null +++ b/librocksdb-sys/rocksdb/microbench/db_basic_bench.cc @@ -0,0 +1,1580 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#ifndef OS_WIN +#include +#endif // ! OS_WIN + +#include "benchmark/benchmark.h" +#include "db/db_impl/db_impl.h" +#include "rocksdb/db.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/options.h" +#include "table/block_based/block.h" +#include "table/block_based/block_builder.h" +#include "util/random.h" +#include "utilities/merge_operators.h" + +namespace ROCKSDB_NAMESPACE { + +class KeyGenerator { + public: + // Generate next key + // buff: the caller needs to make sure there's enough space for generated key + // offset: to control the group of the key, 0 means normal key, 1 means + // non-existing key, 2 is reserved prefix_only: only return a prefix + Slice Next(char* buff, int8_t offset = 0, bool prefix_only = false) { + assert(max_key_ < std::numeric_limits::max() / + MULTIPLIER); // TODO: add large key support + + uint32_t k; + if (is_sequential_) { + assert(next_sequential_key_ < max_key_); + k = (next_sequential_key_ % max_key_) * MULTIPLIER + offset; + if (next_sequential_key_ + 1 == max_key_) { + next_sequential_key_ = 0; + } else { + next_sequential_key_++; + } + } else { + k = (rnd_->Next() % max_key_) * MULTIPLIER + offset; + } + // TODO: make sure the buff is large enough + memset(buff, 0, key_size_); + if (prefix_num_ > 0) { + uint32_t prefix = (k % prefix_num_) * MULTIPLIER + offset; + Encode(buff, prefix); + if (prefix_only) { + return {buff, prefix_size_}; + } + } + Encode(buff + prefix_size_, k); + return {buff, key_size_}; + } + + // use internal buffer for generated key, make sure there's only one caller in + // single thread + Slice Next() { return Next(buff_); } + + // user internal buffer for generated prefix + Slice NextPrefix() { + assert(prefix_num_ > 0); + return Next(buff_, 0, true); + } + + // helper function to get non exist key + Slice NextNonExist() { return Next(buff_, 1); } + + Slice MaxKey(char* buff) const { + memset(buff, 0xff, key_size_); + return {buff, key_size_}; + } + + Slice MinKey(char* buff) const { + memset(buff, 0, key_size_); + return {buff, key_size_}; + } + + // max_key: the max key that it could generate + // prefix_num: the max prefix number + // key_size: in bytes + explicit KeyGenerator(Random* rnd, uint64_t max_key = 100 * 1024 * 1024, + size_t prefix_num = 0, size_t key_size = 10) { + prefix_num_ = prefix_num; + key_size_ = key_size; + max_key_ = max_key; + rnd_ = rnd; + if (prefix_num > 0) { + prefix_size_ = 4; // TODO: support different prefix_size + } + } + + // generate sequential keys + explicit KeyGenerator(uint64_t max_key = 100 * 1024 * 1024, + size_t key_size = 10) { + key_size_ = key_size; + max_key_ = max_key; + rnd_ = nullptr; + is_sequential_ = true; + } + + private: + Random* rnd_; + size_t prefix_num_ = 0; + size_t prefix_size_ = 0; + size_t key_size_; + uint64_t max_key_; + bool is_sequential_ = false; + uint32_t next_sequential_key_ = 0; + char buff_[256] = {0}; + const int MULTIPLIER = 3; + + void static Encode(char* buf, uint32_t value) { + if (port::kLittleEndian) { + buf[0] = static_cast((value >> 24) & 0xff); + buf[1] = static_cast((value >> 16) & 0xff); + buf[2] = static_cast((value >> 8) & 0xff); + buf[3] = static_cast(value & 0xff); + } else { + memcpy(buf, &value, sizeof(value)); + } + } +}; + +static void SetupDB(benchmark::State& state, Options& options, + std::unique_ptr* db, + const std::string& test_name = "") { + options.create_if_missing = true; + auto env = Env::Default(); + std::string db_path; + Status s = env->GetTestDirectory(&db_path); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + return; + } + std::string db_name = + db_path + kFilePathSeparator + test_name + std::to_string(getpid()); + DestroyDB(db_name, options); + + DB* db_ptr = nullptr; + s = DB::Open(options, db_name, &db_ptr); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + return; + } + db->reset(db_ptr); +} + +static void TeardownDB(benchmark::State& state, const std::unique_ptr& db, + const Options& options, KeyGenerator& kg) { + char min_buff[256], max_buff[256]; + const Range r(kg.MinKey(min_buff), kg.MaxKey(max_buff)); + uint64_t size; + Status s = db->GetApproximateSizes(&r, 1, &size); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + state.counters["db_size"] = static_cast(size); + + std::string db_name = db->GetName(); + s = db->Close(); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + DestroyDB(db_name, options); +} + +static void DBOpen(benchmark::State& state) { + // create DB + std::unique_ptr db; + Options options; + SetupDB(state, options, &db, "DBOpen"); + + std::string db_name = db->GetName(); + db->Close(); + + options.create_if_missing = false; + + auto rnd = Random(123); + + for (auto _ : state) { + { + DB* db_ptr = nullptr; + Status s = DB::Open(options, db_name, &db_ptr); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + db.reset(db_ptr); + } + state.PauseTiming(); + auto wo = WriteOptions(); + Status s; + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 100; j++) { + s = db->Put(wo, rnd.RandomString(10), rnd.RandomString(100)); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + } + s = db->Flush(FlushOptions()); + } + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + s = db->Close(); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + state.ResumeTiming(); + } + DestroyDB(db_name, options); +} + +BENCHMARK(DBOpen)->Iterations(200); // specify iteration number as the db size + // is impacted by iteration number + +static void DBClose(benchmark::State& state) { + // create DB + std::unique_ptr db; + Options options; + SetupDB(state, options, &db, "DBClose"); + + std::string db_name = db->GetName(); + db->Close(); + + options.create_if_missing = false; + + auto rnd = Random(12345); + + for (auto _ : state) { + state.PauseTiming(); + { + DB* db_ptr = nullptr; + Status s = DB::Open(options, db_name, &db_ptr); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + db.reset(db_ptr); + } + auto wo = WriteOptions(); + Status s; + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 100; j++) { + s = db->Put(wo, rnd.RandomString(10), rnd.RandomString(100)); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + } + s = db->Flush(FlushOptions()); + } + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + state.ResumeTiming(); + s = db->Close(); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + } + DestroyDB(db_name, options); +} + +BENCHMARK(DBClose)->Iterations(200); // specify iteration number as the db size + // is impacted by iteration number + +static void DBPut(benchmark::State& state) { + auto compaction_style = static_cast(state.range(0)); + uint64_t max_data = state.range(1); + uint64_t per_key_size = state.range(2); + bool enable_statistics = state.range(3); + bool enable_wal = state.range(4); + uint64_t key_num = max_data / per_key_size; + + // setup DB + static std::unique_ptr db = nullptr; + Options options; + if (enable_statistics) { + options.statistics = CreateDBStatistics(); + } + options.compaction_style = compaction_style; + + auto rnd = Random(301 + state.thread_index()); + KeyGenerator kg(&rnd, key_num); + + if (state.thread_index() == 0) { + SetupDB(state, options, &db, "DBPut"); + } + + auto wo = WriteOptions(); + wo.disableWAL = !enable_wal; + + for (auto _ : state) { + state.PauseTiming(); + Slice key = kg.Next(); + std::string val = rnd.RandomString(static_cast(per_key_size)); + state.ResumeTiming(); + Status s = db->Put(wo, key, val); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + } + + if (state.thread_index() == 0) { + auto db_full = static_cast_with_check(db.get()); + Status s = db_full->WaitForCompact(WaitForCompactOptions()); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + return; + } + if (enable_statistics) { + HistogramData histogram_data; + options.statistics->histogramData(DB_WRITE, &histogram_data); + state.counters["put_mean"] = histogram_data.average * std::milli::den; + state.counters["put_p95"] = histogram_data.percentile95 * std::milli::den; + state.counters["put_p99"] = histogram_data.percentile99 * std::milli::den; + } + + TeardownDB(state, db, options, kg); + } +} + +static void DBPutArguments(benchmark::internal::Benchmark* b) { + for (int comp_style : {kCompactionStyleLevel, kCompactionStyleUniversal, + kCompactionStyleFIFO}) { + for (int64_t max_data : {100l << 30}) { + for (int64_t per_key_size : {256, 1024}) { + for (bool enable_statistics : {false, true}) { + for (bool wal : {false, true}) { + b->Args( + {comp_style, max_data, per_key_size, enable_statistics, wal}); + } + } + } + } + } + b->ArgNames( + {"comp_style", "max_data", "per_key_size", "enable_statistics", "wal"}); +} + +static const uint64_t DBPutNum = 409600l; +BENCHMARK(DBPut)->Threads(1)->Iterations(DBPutNum)->Apply(DBPutArguments); +BENCHMARK(DBPut)->Threads(8)->Iterations(DBPutNum / 8)->Apply(DBPutArguments); + +static void ManualCompaction(benchmark::State& state) { + auto compaction_style = static_cast(state.range(0)); + uint64_t max_data = state.range(1); + uint64_t per_key_size = state.range(2); + bool enable_statistics = state.range(3); + uint64_t key_num = max_data / per_key_size; + + // setup DB + static std::unique_ptr db; + Options options; + if (enable_statistics) { + options.statistics = CreateDBStatistics(); + } + options.compaction_style = compaction_style; + // No auto compaction + options.disable_auto_compactions = true; + options.level0_file_num_compaction_trigger = (1 << 30); + options.level0_slowdown_writes_trigger = (1 << 30); + options.level0_stop_writes_trigger = (1 << 30); + options.soft_pending_compaction_bytes_limit = 0; + options.hard_pending_compaction_bytes_limit = 0; + + auto rnd = Random(301 + state.thread_index()); + KeyGenerator kg(&rnd, key_num); + + if (state.thread_index() == 0) { + SetupDB(state, options, &db, "ManualCompaction"); + } + + auto wo = WriteOptions(); + wo.disableWAL = true; + uint64_t flush_mod = key_num / 4; // at least generate 4 files for compaction + for (uint64_t i = 0; i < key_num; i++) { + Status s = db->Put(wo, kg.Next(), + rnd.RandomString(static_cast(per_key_size))); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + if (i + 1 % flush_mod == 0) { + s = db->Flush(FlushOptions()); + } + } + FlushOptions fo; + Status s = db->Flush(fo); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + std::vector files_meta; + db->GetLiveFilesMetaData(&files_meta); + std::vector files_before_compact; + files_before_compact.reserve(files_meta.size()); + for (const LiveFileMetaData& file : files_meta) { + files_before_compact.emplace_back(file.name); + } + + SetPerfLevel(kEnableTime); + get_perf_context()->EnablePerLevelPerfContext(); + get_perf_context()->Reset(); + CompactionOptions co; + for (auto _ : state) { + s = db->CompactFiles(co, files_before_compact, 1); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + } + + if (state.thread_index() == 0) { + auto db_full = static_cast_with_check(db.get()); + s = db_full->WaitForCompact(WaitForCompactOptions()); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + return; + } + if (enable_statistics) { + HistogramData histogram_data; + options.statistics->histogramData(COMPACTION_TIME, &histogram_data); + state.counters["comp_time"] = histogram_data.average; + options.statistics->histogramData(COMPACTION_CPU_TIME, &histogram_data); + state.counters["comp_cpu_time"] = histogram_data.average; + options.statistics->histogramData(COMPACTION_OUTFILE_SYNC_MICROS, + &histogram_data); + state.counters["comp_outfile_sync"] = histogram_data.average; + + state.counters["comp_read"] = static_cast( + options.statistics->getTickerCount(COMPACT_READ_BYTES)); + state.counters["comp_write"] = static_cast( + options.statistics->getTickerCount(COMPACT_WRITE_BYTES)); + + state.counters["user_key_comparison_count"] = + static_cast(get_perf_context()->user_key_comparison_count); + state.counters["block_read_count"] = + static_cast(get_perf_context()->block_read_count); + state.counters["block_read_time"] = + static_cast(get_perf_context()->block_read_time); + state.counters["block_read_cpu_time"] = + static_cast(get_perf_context()->block_read_cpu_time); + state.counters["block_checksum_time"] = + static_cast(get_perf_context()->block_checksum_time); + state.counters["new_table_block_iter_nanos"] = + static_cast(get_perf_context()->new_table_block_iter_nanos); + state.counters["new_table_iterator_nanos"] = + static_cast(get_perf_context()->new_table_iterator_nanos); + state.counters["find_table_nanos"] = + static_cast(get_perf_context()->find_table_nanos); + } + + TeardownDB(state, db, options, kg); + } +} + +static void ManualCompactionArguments(benchmark::internal::Benchmark* b) { + for (int comp_style : {kCompactionStyleLevel, kCompactionStyleUniversal}) { + for (int64_t max_data : {32l << 20, 128l << 20}) { + for (int64_t per_key_size : {256, 1024}) { + for (bool enable_statistics : {false, true}) { + b->Args({comp_style, max_data, per_key_size, enable_statistics}); + } + } + } + } + b->ArgNames({"comp_style", "max_data", "per_key_size", "enable_statistics"}); +} + +BENCHMARK(ManualCompaction)->Iterations(1)->Apply(ManualCompactionArguments); + +static void ManualFlush(benchmark::State& state) { + uint64_t key_num = state.range(0); + uint64_t per_key_size = state.range(1); + bool enable_statistics = true; + + // setup DB + static std::unique_ptr db; + Options options; + if (enable_statistics) { + options.statistics = CreateDBStatistics(); + } + options.disable_auto_compactions = true; + options.level0_file_num_compaction_trigger = (1 << 30); + options.level0_slowdown_writes_trigger = (1 << 30); + options.level0_stop_writes_trigger = (1 << 30); + options.soft_pending_compaction_bytes_limit = 0; + options.hard_pending_compaction_bytes_limit = 0; + options.write_buffer_size = 2l << 30; // 2G to avoid auto flush + + auto rnd = Random(301 + state.thread_index()); + KeyGenerator kg(&rnd, key_num); + + if (state.thread_index() == 0) { + SetupDB(state, options, &db, "ManualFlush"); + } + + auto wo = WriteOptions(); + for (auto _ : state) { + state.PauseTiming(); + for (uint64_t i = 0; i < key_num; i++) { + Status s = db->Put(wo, kg.Next(), + rnd.RandomString(static_cast(per_key_size))); + } + FlushOptions fo; + state.ResumeTiming(); + Status s = db->Flush(fo); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + } + + if (state.thread_index() == 0) { + auto db_full = static_cast_with_check(db.get()); + Status s = db_full->WaitForCompact(WaitForCompactOptions()); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + return; + } + if (enable_statistics) { + HistogramData histogram_data; + options.statistics->histogramData(FLUSH_TIME, &histogram_data); + state.counters["flush_time"] = histogram_data.average; + state.counters["flush_write_bytes"] = static_cast( + options.statistics->getTickerCount(FLUSH_WRITE_BYTES)); + } + + TeardownDB(state, db, options, kg); + } +} + +static void ManualFlushArguments(benchmark::internal::Benchmark* b) { + for (int64_t key_num : {1l << 10, 8l << 10, 64l << 10}) { + for (int64_t per_key_size : {256, 1024}) { + b->Args({key_num, per_key_size}); + } + } + b->ArgNames({"key_num", "per_key_size"}); +} + +BENCHMARK(ManualFlush)->Iterations(1)->Apply(ManualFlushArguments); + +static void DBGet(benchmark::State& state) { + auto compaction_style = static_cast(state.range(0)); + uint64_t max_data = state.range(1); + uint64_t per_key_size = state.range(2); + bool enable_statistics = state.range(3); + bool negative_query = state.range(4); + bool enable_filter = state.range(5); + bool mmap = state.range(6); + uint64_t key_num = max_data / per_key_size; + + // setup DB + static std::unique_ptr db; + Options options; + if (enable_statistics) { + options.statistics = CreateDBStatistics(); + } + if (mmap) { + options.allow_mmap_reads = true; + options.compression = kNoCompression; + } + options.compaction_style = compaction_style; + + BlockBasedTableOptions table_options; + if (enable_filter) { + table_options.filter_policy.reset(NewBloomFilterPolicy(10, false)); + } + if (mmap) { + table_options.no_block_cache = true; + table_options.block_restart_interval = 1; + } + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + auto rnd = Random(301 + state.thread_index()); + + if (state.thread_index() == 0) { + KeyGenerator kg_seq(key_num /* max_key */); + SetupDB(state, options, &db, "DBGet"); + + // Load all valid keys into DB. That way, iterations in `!negative_query` + // runs can always find the key even though it is generated from a random + // number. + auto wo = WriteOptions(); + wo.disableWAL = true; + for (uint64_t i = 0; i < key_num; i++) { + Status s = db->Put(wo, kg_seq.Next(), + rnd.RandomString(static_cast(per_key_size))); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + } + + // Compact whole DB into one level, so each iteration will consider the same + // number of files (one). + Status s = db->CompactRange(CompactRangeOptions(), nullptr /* begin */, + nullptr /* end */); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + } + + KeyGenerator kg_rnd(&rnd, key_num /* max_key */); + auto ro = ReadOptions(); + if (mmap) { + ro.verify_checksums = false; + } + size_t not_found = 0; + if (negative_query) { + for (auto _ : state) { + std::string val; + Status s = db->Get(ro, kg_rnd.NextNonExist(), &val); + if (s.IsNotFound()) { + not_found++; + } + } + } else { + for (auto _ : state) { + std::string val; + Status s = db->Get(ro, kg_rnd.Next(), &val); + if (s.IsNotFound()) { + not_found++; + } + } + } + + state.counters["neg_qu_pct"] = benchmark::Counter( + static_cast(not_found * 100), benchmark::Counter::kAvgIterations); + + if (state.thread_index() == 0) { + if (enable_statistics) { + HistogramData histogram_data; + options.statistics->histogramData(DB_GET, &histogram_data); + state.counters["get_mean"] = histogram_data.average * std::milli::den; + state.counters["get_p95"] = histogram_data.percentile95 * std::milli::den; + state.counters["get_p99"] = histogram_data.percentile99 * std::milli::den; + } + + TeardownDB(state, db, options, kg_rnd); + } +} + +static void DBGetArguments(benchmark::internal::Benchmark* b) { + for (int comp_style : {kCompactionStyleLevel, kCompactionStyleUniversal, + kCompactionStyleFIFO}) { + for (int64_t max_data : {128l << 20, 512l << 20}) { + for (int64_t per_key_size : {256, 1024}) { + for (bool enable_statistics : {false, true}) { + for (bool negative_query : {false, true}) { + for (bool enable_filter : {false, true}) { + for (bool mmap : {false, true}) { + b->Args({comp_style, max_data, per_key_size, enable_statistics, + negative_query, enable_filter, mmap}); + } + } + } + } + } + } + } + b->ArgNames({"comp_style", "max_data", "per_key_size", "enable_statistics", + "negative_query", "enable_filter", "mmap"}); +} + +BENCHMARK(DBGet)->Threads(1)->Apply(DBGetArguments); +BENCHMARK(DBGet)->Threads(8)->Apply(DBGetArguments); + +static void SimpleGetWithPerfContext(benchmark::State& state) { + // setup DB + static std::unique_ptr db; + std::string db_name; + Options options; + options.create_if_missing = true; + options.arena_block_size = 8 << 20; + + auto rnd = Random(301 + state.thread_index()); + KeyGenerator kg(&rnd, 1024); + + if (state.thread_index() == 0) { + auto env = Env::Default(); + std::string db_path; + Status s = env->GetTestDirectory(&db_path); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + return; + } + db_name = db_path + "/simple_get_" + std::to_string(getpid()); + DestroyDB(db_name, options); + + { + DB* db_ptr = nullptr; + s = DB::Open(options, db_name, &db_ptr); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + return; + } + db.reset(db_ptr); + } + // load db + auto wo = WriteOptions(); + wo.disableWAL = true; + for (uint64_t i = 0; i < 1024; i++) { + s = db->Put(wo, kg.Next(), rnd.RandomString(1024)); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + } + auto db_full = static_cast_with_check(db.get()); + s = db_full->WaitForCompact(WaitForCompactOptions()); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + return; + } + FlushOptions fo; + s = db->Flush(fo); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + } + + auto ro = ReadOptions(); + size_t not_found = 0; + uint64_t user_key_comparison_count = 0; + uint64_t block_read_time = 0; + uint64_t block_read_cpu_time = 0; + uint64_t block_checksum_time = 0; + uint64_t get_snapshot_time = 0; + uint64_t get_post_process_time = 0; + uint64_t get_from_output_files_time = 0; + uint64_t new_table_block_iter_nanos = 0; + uint64_t block_seek_nanos = 0; + uint64_t get_cpu_nanos = 0; + uint64_t get_from_table_nanos = 0; + SetPerfLevel(kEnableTime); + get_perf_context()->EnablePerLevelPerfContext(); + for (auto _ : state) { + std::string val; + get_perf_context()->Reset(); + Status s = db->Get(ro, kg.NextNonExist(), &val); + if (s.IsNotFound()) { + not_found++; + } + user_key_comparison_count += get_perf_context()->user_key_comparison_count; + block_read_time += get_perf_context()->block_read_time; + block_read_cpu_time += get_perf_context()->block_read_cpu_time; + block_checksum_time += get_perf_context()->block_checksum_time; + get_snapshot_time += get_perf_context()->get_snapshot_time; + get_post_process_time += get_perf_context()->get_post_process_time; + get_from_output_files_time += + get_perf_context()->get_from_output_files_time; + new_table_block_iter_nanos += + get_perf_context()->new_table_block_iter_nanos; + block_seek_nanos += get_perf_context()->block_seek_nanos; + get_cpu_nanos += get_perf_context()->get_cpu_nanos; + get_from_table_nanos += + (*(get_perf_context()->level_to_perf_context))[0].get_from_table_nanos; + } + + state.counters["neg_qu_pct"] = benchmark::Counter( + static_cast(not_found * 100), benchmark::Counter::kAvgIterations); + state.counters["user_key_comparison_count"] = + benchmark::Counter(static_cast(user_key_comparison_count), + benchmark::Counter::kAvgIterations); + state.counters["block_read_time"] = benchmark::Counter( + static_cast(block_read_time), benchmark::Counter::kAvgIterations); + state.counters["block_read_cpu_time"] = + benchmark::Counter(static_cast(block_read_cpu_time), + benchmark::Counter::kAvgIterations); + state.counters["block_checksum_time"] = + benchmark::Counter(static_cast(block_checksum_time), + benchmark::Counter::kAvgIterations); + state.counters["get_snapshot_time"] = + benchmark::Counter(static_cast(get_snapshot_time), + benchmark::Counter::kAvgIterations); + state.counters["get_post_process_time"] = + benchmark::Counter(static_cast(get_post_process_time), + benchmark::Counter::kAvgIterations); + state.counters["get_from_output_files_time"] = + benchmark::Counter(static_cast(get_from_output_files_time), + benchmark::Counter::kAvgIterations); + state.counters["new_table_block_iter_nanos"] = + benchmark::Counter(static_cast(new_table_block_iter_nanos), + benchmark::Counter::kAvgIterations); + state.counters["block_seek_nanos"] = + benchmark::Counter(static_cast(block_seek_nanos), + benchmark::Counter::kAvgIterations); + state.counters["get_cpu_nanos"] = benchmark::Counter( + static_cast(get_cpu_nanos), benchmark::Counter::kAvgIterations); + state.counters["get_from_table_nanos"] = + benchmark::Counter(static_cast(get_from_table_nanos), + benchmark::Counter::kAvgIterations); + + if (state.thread_index() == 0) { + TeardownDB(state, db, options, kg); + } +} + +BENCHMARK(SimpleGetWithPerfContext)->Iterations(1000000); + +static void DBGetMergeOperandsInMemtable(benchmark::State& state) { + const uint64_t kDataLen = 16 << 20; // 16MB + const uint64_t kValueLen = 64; + const uint64_t kNumEntries = kDataLen / kValueLen; + const uint64_t kNumEntriesPerKey = state.range(0); + const uint64_t kNumKeys = kNumEntries / kNumEntriesPerKey; + + // setup DB + static std::unique_ptr db; + + Options options; + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + // Make memtable large enough that automatic flush will not be triggered. + options.write_buffer_size = 2 * kDataLen; + + KeyGenerator sequential_key_gen(kNumKeys); + auto rnd = Random(301 + state.thread_index()); + + if (state.thread_index() == 0) { + SetupDB(state, options, &db, "DBGetMergeOperandsInMemtable"); + + // load db + auto write_opts = WriteOptions(); + write_opts.disableWAL = true; + for (uint64_t i = 0; i < kNumEntries; i++) { + Status s = db->Merge(write_opts, sequential_key_gen.Next(), + rnd.RandomString(static_cast(kValueLen))); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + } + } + + KeyGenerator random_key_gen(kNumKeys); + std::vector value_operands; + value_operands.resize(kNumEntriesPerKey); + GetMergeOperandsOptions get_merge_ops_opts; + get_merge_ops_opts.expected_max_number_of_operands = + static_cast(kNumEntriesPerKey); + for (auto _ : state) { + int num_value_operands = 0; + Status s = db->GetMergeOperands( + ReadOptions(), db->DefaultColumnFamily(), random_key_gen.Next(), + value_operands.data(), &get_merge_ops_opts, &num_value_operands); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + if (num_value_operands != static_cast(kNumEntriesPerKey)) { + state.SkipWithError("Unexpected number of merge operands found for key"); + } + for (auto& value_operand : value_operands) { + value_operand.Reset(); + } + } + + if (state.thread_index() == 0) { + TeardownDB(state, db, options, random_key_gen); + } +} + +static void DBGetMergeOperandsInSstFile(benchmark::State& state) { + const uint64_t kDataLen = 16 << 20; // 16MB + const uint64_t kValueLen = 64; + const uint64_t kNumEntries = kDataLen / kValueLen; + const uint64_t kNumEntriesPerKey = state.range(0); + const uint64_t kNumKeys = kNumEntries / kNumEntriesPerKey; + const bool kMmap = state.range(1); + + // setup DB + static std::unique_ptr db; + + BlockBasedTableOptions table_options; + if (kMmap) { + table_options.no_block_cache = true; + } else { + // Make block cache large enough that eviction will not be triggered. + table_options.block_cache = NewLRUCache(2 * kDataLen); + } + + Options options; + if (kMmap) { + options.allow_mmap_reads = true; + } + options.compression = kNoCompression; + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + // Make memtable large enough that automatic flush will not be triggered. + options.write_buffer_size = 2 * kDataLen; + + KeyGenerator sequential_key_gen(kNumKeys); + auto rnd = Random(301 + state.thread_index()); + + if (state.thread_index() == 0) { + SetupDB(state, options, &db, "DBGetMergeOperandsInBlockCache"); + + // load db + // + // Take a snapshot after each cycle of merges to ensure flush cannot + // merge any entries. + std::vector snapshots; + snapshots.resize(kNumEntriesPerKey); + auto write_opts = WriteOptions(); + write_opts.disableWAL = true; + for (uint64_t i = 0; i < kNumEntriesPerKey; i++) { + for (uint64_t j = 0; j < kNumKeys; j++) { + Status s = db->Merge(write_opts, sequential_key_gen.Next(), + rnd.RandomString(static_cast(kValueLen))); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + } + snapshots[i] = db->GetSnapshot(); + } + + // Flush to an L0 file; read back to prime the cache/mapped memory. + db->Flush(FlushOptions()); + for (uint64_t i = 0; i < kNumKeys; ++i) { + std::string value; + Status s = db->Get(ReadOptions(), sequential_key_gen.Next(), &value); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + } + + if (state.thread_index() == 0) { + for (uint64_t i = 0; i < kNumEntriesPerKey; ++i) { + db->ReleaseSnapshot(snapshots[i]); + } + } + } + + KeyGenerator random_key_gen(kNumKeys); + std::vector value_operands; + value_operands.resize(kNumEntriesPerKey); + GetMergeOperandsOptions get_merge_ops_opts; + get_merge_ops_opts.expected_max_number_of_operands = + static_cast(kNumEntriesPerKey); + for (auto _ : state) { + int num_value_operands = 0; + ReadOptions read_opts; + read_opts.verify_checksums = false; + Status s = db->GetMergeOperands( + read_opts, db->DefaultColumnFamily(), random_key_gen.Next(), + value_operands.data(), &get_merge_ops_opts, &num_value_operands); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + if (num_value_operands != static_cast(kNumEntriesPerKey)) { + state.SkipWithError("Unexpected number of merge operands found for key"); + } + for (auto& value_operand : value_operands) { + value_operand.Reset(); + } + } + + if (state.thread_index() == 0) { + TeardownDB(state, db, options, random_key_gen); + } +} + +static void DBGetMergeOperandsInMemtableArguments( + benchmark::internal::Benchmark* b) { + for (int entries_per_key : {1, 32, 1024}) { + b->Args({entries_per_key}); + } + b->ArgNames({"entries_per_key"}); +} + +static void DBGetMergeOperandsInSstFileArguments( + benchmark::internal::Benchmark* b) { + for (int entries_per_key : {1, 32, 1024}) { + for (bool mmap : {false, true}) { + b->Args({entries_per_key, mmap}); + } + } + b->ArgNames({"entries_per_key", "mmap"}); +} + +BENCHMARK(DBGetMergeOperandsInMemtable) + ->Threads(1) + ->Apply(DBGetMergeOperandsInMemtableArguments); +BENCHMARK(DBGetMergeOperandsInMemtable) + ->Threads(8) + ->Apply(DBGetMergeOperandsInMemtableArguments); +BENCHMARK(DBGetMergeOperandsInSstFile) + ->Threads(1) + ->Apply(DBGetMergeOperandsInSstFileArguments); +BENCHMARK(DBGetMergeOperandsInSstFile) + ->Threads(8) + ->Apply(DBGetMergeOperandsInSstFileArguments); + +std::string GenerateKey(int primary_key, int secondary_key, int padding_size, + Random* rnd) { + char buf[50]; + char* p = &buf[0]; + snprintf(buf, sizeof(buf), "%6d%4d", primary_key, secondary_key); + std::string k(p); + if (padding_size) { + k += rnd->RandomString(padding_size); + } + + return k; +} + +void GenerateRandomKVs(std::vector* keys, + std::vector* values, const int from, + const int len, const int step = 1, + const int padding_size = 0, + const int keys_share_prefix = 1) { + Random rnd(302); + + // generate different prefix + for (int i = from; i < from + len; i += step) { + // generating keys that share the prefix + for (int j = 0; j < keys_share_prefix; ++j) { + keys->emplace_back(GenerateKey(i, j, padding_size, &rnd)); + // 100 bytes values + values->emplace_back(rnd.RandomString(100)); + } + } +} + +// TODO: move it to different files, as it's testing an internal API +static void DataBlockSeek(benchmark::State& state) { + Random rnd(301); + Options options = Options(); + + BlockBuilder builder(16, true, false, + BlockBasedTableOptions::kDataBlockBinarySearch); + + int num_records = 500; + std::vector keys; + std::vector values; + + GenerateRandomKVs(&keys, &values, 0, num_records); + + for (int i = 0; i < num_records; i++) { + std::string ukey(keys[i] + "1"); + InternalKey ikey(ukey, 0, kTypeValue); + builder.Add(ikey.Encode().ToString(), values[i]); + } + + Slice rawblock = builder.Finish(); + + BlockContents contents; + contents.data = rawblock; + Block reader(std::move(contents)); + + SetPerfLevel(kEnableTime); + uint64_t total = 0; + for (auto _ : state) { + DataBlockIter* iter = reader.NewDataIterator(options.comparator, + kDisableGlobalSequenceNumber); + uint32_t index = rnd.Uniform(static_cast(num_records)); + std::string ukey(keys[index] + "1"); + InternalKey ikey(ukey, 0, kTypeValue); + get_perf_context()->Reset(); + bool may_exist = iter->SeekForGet(ikey.Encode().ToString()); + if (!may_exist) { + state.SkipWithError("key not found"); + } + total += get_perf_context()->block_seek_nanos; + delete iter; + } + state.counters["seek_ns"] = benchmark::Counter( + static_cast(total), benchmark::Counter::kAvgIterations); +} + +BENCHMARK(DataBlockSeek)->Iterations(1000000); + +static void IteratorSeek(benchmark::State& state) { + auto compaction_style = static_cast(state.range(0)); + uint64_t max_data = state.range(1); + uint64_t per_key_size = state.range(2); + bool enable_statistics = state.range(3); + bool negative_query = state.range(4); + bool enable_filter = state.range(5); + uint64_t key_num = max_data / per_key_size; + + // setup DB + static std::unique_ptr db; + Options options; + if (enable_statistics) { + options.statistics = CreateDBStatistics(); + } + options.compaction_style = compaction_style; + + if (enable_filter) { + BlockBasedTableOptions table_options; + table_options.filter_policy.reset(NewBloomFilterPolicy(10, false)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + } + + auto rnd = Random(301 + state.thread_index()); + KeyGenerator kg(&rnd, key_num); + + if (state.thread_index() == 0) { + SetupDB(state, options, &db, "IteratorSeek"); + + // load db + auto wo = WriteOptions(); + wo.disableWAL = true; + for (uint64_t i = 0; i < key_num; i++) { + Status s = db->Put(wo, kg.Next(), + rnd.RandomString(static_cast(per_key_size))); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + } + + FlushOptions fo; + Status s = db->Flush(fo); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + + auto db_full = static_cast_with_check(db.get()); + s = db_full->WaitForCompact(WaitForCompactOptions()); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + return; + } + } + + for (auto _ : state) { + std::unique_ptr iter{nullptr}; + state.PauseTiming(); + if (!iter) { + iter.reset(db->NewIterator(ReadOptions())); + } + Slice key = negative_query ? kg.NextNonExist() : kg.Next(); + if (!iter->status().ok()) { + state.SkipWithError(iter->status().ToString().c_str()); + return; + } + state.ResumeTiming(); + iter->Seek(key); + } + + if (state.thread_index() == 0) { + TeardownDB(state, db, options, kg); + } +} + +static void IteratorSeekArguments(benchmark::internal::Benchmark* b) { + for (int comp_style : {kCompactionStyleLevel, kCompactionStyleUniversal, + kCompactionStyleFIFO}) { + for (int64_t max_data : {128l << 20, 512l << 20}) { + for (int64_t per_key_size : {256, 1024}) { + for (bool enable_statistics : {false, true}) { + for (bool negative_query : {false, true}) { + for (bool enable_filter : {false, true}) { + b->Args({comp_style, max_data, per_key_size, enable_statistics, + negative_query, enable_filter}); + } + } + } + } + } + } + b->ArgNames({"comp_style", "max_data", "per_key_size", "enable_statistics", + "negative_query", "enable_filter"}); +} + +static constexpr uint64_t kDBSeekNum = 10l << 10; +BENCHMARK(IteratorSeek) + ->Threads(1) + ->Iterations(kDBSeekNum) + ->Apply(IteratorSeekArguments); +BENCHMARK(IteratorSeek) + ->Threads(8) + ->Iterations(kDBSeekNum / 8) + ->Apply(IteratorSeekArguments); + +static void IteratorNext(benchmark::State& state) { + auto compaction_style = static_cast(state.range(0)); + uint64_t max_data = state.range(1); + uint64_t per_key_size = state.range(2); + uint64_t key_num = max_data / per_key_size; + + // setup DB + static std::unique_ptr db; + Options options; + options.compaction_style = compaction_style; + + auto rnd = Random(301 + state.thread_index()); + KeyGenerator kg(&rnd, key_num); + + if (state.thread_index() == 0) { + SetupDB(state, options, &db, "IteratorNext"); + // load db + auto wo = WriteOptions(); + wo.disableWAL = true; + for (uint64_t i = 0; i < key_num; i++) { + Status s = db->Put(wo, kg.Next(), + rnd.RandomString(static_cast(per_key_size))); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + } + + FlushOptions fo; + Status s = db->Flush(fo); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + + auto db_full = static_cast_with_check(db.get()); + s = db_full->WaitForCompact(WaitForCompactOptions()); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + return; + } + } + + for (auto _ : state) { + std::unique_ptr iter{nullptr}; + state.PauseTiming(); + if (!iter) { + iter.reset(db->NewIterator(ReadOptions())); + } + while (!iter->Valid()) { + iter->Seek(kg.Next()); + if (!iter->status().ok()) { + state.SkipWithError(iter->status().ToString().c_str()); + } + } + state.ResumeTiming(); + iter->Next(); + } + + if (state.thread_index() == 0) { + TeardownDB(state, db, options, kg); + } +} + +static void IteratorNextArguments(benchmark::internal::Benchmark* b) { + for (int comp_style : {kCompactionStyleLevel, kCompactionStyleUniversal, + kCompactionStyleFIFO}) { + for (int64_t max_data : {128l << 20, 512l << 20}) { + for (int64_t per_key_size : {256, 1024}) { + b->Args({comp_style, max_data, per_key_size}); + } + } + } + b->ArgNames({"comp_style", "max_data", "per_key_size"}); +} +static constexpr uint64_t kIteratorNextNum = 10l << 10; +BENCHMARK(IteratorNext) + ->Iterations(kIteratorNextNum) + ->Apply(IteratorNextArguments); + +static void IteratorNextWithPerfContext(benchmark::State& state) { + // setup DB + static std::unique_ptr db; + Options options; + + auto rnd = Random(301 + state.thread_index()); + KeyGenerator kg(&rnd, 1024); + + if (state.thread_index() == 0) { + SetupDB(state, options, &db, "IteratorNextWithPerfContext"); + // load db + auto wo = WriteOptions(); + wo.disableWAL = true; + for (uint64_t i = 0; i < 1024; i++) { + Status s = db->Put(wo, kg.Next(), rnd.RandomString(1024)); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + } + auto db_full = static_cast_with_check(db.get()); + Status s = db_full->WaitForCompact(WaitForCompactOptions()); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + return; + } + FlushOptions fo; + s = db->Flush(fo); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + } + + uint64_t user_key_comparison_count = 0; + uint64_t internal_key_skipped_count = 0; + uint64_t find_next_user_entry_time = 0; + uint64_t iter_next_cpu_nanos = 0; + + SetPerfLevel(kEnableTime); + get_perf_context()->EnablePerLevelPerfContext(); + + for (auto _ : state) { + std::unique_ptr iter{nullptr}; + state.PauseTiming(); + if (!iter) { + iter.reset(db->NewIterator(ReadOptions())); + } + while (!iter->Valid()) { + iter->Seek(kg.Next()); + if (!iter->status().ok()) { + state.SkipWithError(iter->status().ToString().c_str()); + } + } + get_perf_context()->Reset(); + state.ResumeTiming(); + + iter->Next(); + user_key_comparison_count += get_perf_context()->user_key_comparison_count; + internal_key_skipped_count += + get_perf_context()->internal_key_skipped_count; + find_next_user_entry_time += get_perf_context()->find_next_user_entry_time; + iter_next_cpu_nanos += get_perf_context()->iter_next_cpu_nanos; + } + + state.counters["user_key_comparison_count"] = + benchmark::Counter(static_cast(user_key_comparison_count), + benchmark::Counter::kAvgIterations); + state.counters["internal_key_skipped_count"] = + benchmark::Counter(static_cast(internal_key_skipped_count), + benchmark::Counter::kAvgIterations); + state.counters["find_next_user_entry_time"] = + benchmark::Counter(static_cast(find_next_user_entry_time), + benchmark::Counter::kAvgIterations); + state.counters["iter_next_cpu_nanos"] = + benchmark::Counter(static_cast(iter_next_cpu_nanos), + benchmark::Counter::kAvgIterations); + + if (state.thread_index() == 0) { + TeardownDB(state, db, options, kg); + } +} + +BENCHMARK(IteratorNextWithPerfContext)->Iterations(100000); + +static void IteratorPrev(benchmark::State& state) { + auto compaction_style = static_cast(state.range(0)); + uint64_t max_data = state.range(1); + uint64_t per_key_size = state.range(2); + uint64_t key_num = max_data / per_key_size; + + // setup DB + static std::unique_ptr db; + std::string db_name; + Options options; + options.compaction_style = compaction_style; + + auto rnd = Random(301 + state.thread_index()); + KeyGenerator kg(&rnd, key_num); + + if (state.thread_index() == 0) { + SetupDB(state, options, &db, "IteratorPrev"); + // load db + auto wo = WriteOptions(); + wo.disableWAL = true; + for (uint64_t i = 0; i < key_num; i++) { + Status s = db->Put(wo, kg.Next(), + rnd.RandomString(static_cast(per_key_size))); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + } + + FlushOptions fo; + Status s = db->Flush(fo); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + + auto db_full = static_cast_with_check(db.get()); + s = db_full->WaitForCompact(WaitForCompactOptions()); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + return; + } + } + + for (auto _ : state) { + std::unique_ptr iter{nullptr}; + state.PauseTiming(); + if (!iter) { + iter.reset(db->NewIterator(ReadOptions())); + } + while (!iter->Valid()) { + iter->Seek(kg.Next()); + if (!iter->status().ok()) { + state.SkipWithError(iter->status().ToString().c_str()); + } + } + state.ResumeTiming(); + iter->Prev(); + } + + if (state.thread_index() == 0) { + TeardownDB(state, db, options, kg); + } +} + +static void IteratorPrevArguments(benchmark::internal::Benchmark* b) { + for (int comp_style : {kCompactionStyleLevel, kCompactionStyleUniversal, + kCompactionStyleFIFO}) { + for (int64_t max_data : {128l << 20, 512l << 20}) { + for (int64_t per_key_size : {256, 1024}) { + b->Args({comp_style, max_data, per_key_size}); + } + } + } + b->ArgNames({"comp_style", "max_data", "per_key_size"}); +} + +static constexpr uint64_t kIteratorPrevNum = 10l << 10; +BENCHMARK(IteratorPrev) + ->Iterations(kIteratorPrevNum) + ->Apply(IteratorPrevArguments); + +static void PrefixSeek(benchmark::State& state) { + auto compaction_style = static_cast(state.range(0)); + uint64_t max_data = state.range(1); + uint64_t per_key_size = state.range(2); + bool enable_statistics = state.range(3); + bool enable_filter = state.range(4); + uint64_t key_num = max_data / per_key_size; + + // setup DB + static std::unique_ptr db; + Options options; + if (enable_statistics) { + options.statistics = CreateDBStatistics(); + } + options.compaction_style = compaction_style; + options.prefix_extractor.reset(NewFixedPrefixTransform(4)); + + if (enable_filter) { + BlockBasedTableOptions table_options; + table_options.filter_policy.reset(NewBloomFilterPolicy(10, false)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + } + + auto rnd = Random(301 + state.thread_index()); + KeyGenerator kg(&rnd, key_num, key_num / 100); + + if (state.thread_index() == 0) { + SetupDB(state, options, &db, "PrefixSeek"); + + // load db + auto wo = WriteOptions(); + wo.disableWAL = true; + for (uint64_t i = 0; i < key_num; i++) { + Status s = db->Put(wo, kg.Next(), + rnd.RandomString(static_cast(per_key_size))); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + } + + FlushOptions fo; + Status s = db->Flush(fo); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + + auto db_full = static_cast_with_check(db.get()); + s = db_full->WaitForCompact(WaitForCompactOptions()); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + return; + } + } + + for (auto _ : state) { + std::unique_ptr iter{nullptr}; + state.PauseTiming(); + if (!iter) { + iter.reset(db->NewIterator(ReadOptions())); + } + state.ResumeTiming(); + iter->Seek(kg.NextPrefix()); + if (!iter->status().ok()) { + state.SkipWithError(iter->status().ToString().c_str()); + return; + } + } + + if (state.thread_index() == 0) { + TeardownDB(state, db, options, kg); + } +} + +static void PrefixSeekArguments(benchmark::internal::Benchmark* b) { + for (int comp_style : {kCompactionStyleLevel, kCompactionStyleUniversal, + kCompactionStyleFIFO}) { + for (int64_t max_data : {128l << 20, 512l << 20}) { + for (int64_t per_key_size : {256, 1024}) { + for (bool enable_statistics : {false, true}) { + for (bool enable_filter : {false, true}) { + b->Args({comp_style, max_data, per_key_size, enable_statistics, + enable_filter}); + } + } + } + } + } + b->ArgNames({"comp_style", "max_data", "per_key_size", "enable_statistics", + "enable_filter"}); +} + +static constexpr uint64_t kPrefixSeekNum = 10l << 10; +BENCHMARK(PrefixSeek)->Iterations(kPrefixSeekNum)->Apply(PrefixSeekArguments); +BENCHMARK(PrefixSeek) + ->Threads(8) + ->Iterations(kPrefixSeekNum / 8) + ->Apply(PrefixSeekArguments); + +// TODO: move it to different files, as it's testing an internal API +static void RandomAccessFileReaderRead(benchmark::State& state) { + bool enable_statistics = state.range(0); + constexpr int kFileNum = 10; + auto env = Env::Default(); + auto fs = env->GetFileSystem(); + std::string db_path; + Status s = env->GetTestDirectory(&db_path); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + return; + } + + // Setup multiple `RandomAccessFileReader`s with different parameters to be + // used for test + Random rand(301); + std::string fname_base = + db_path + kFilePathSeparator + "random-access-file-reader-read"; + std::vector> readers; + auto statistics_share = CreateDBStatistics(); + Statistics* statistics = enable_statistics ? statistics_share.get() : nullptr; + for (int i = 0; i < kFileNum; i++) { + std::string fname = fname_base + std::to_string(i); + std::string content = rand.RandomString(kDefaultPageSize); + std::unique_ptr tgt_file; + env->NewWritableFile(fname, &tgt_file, EnvOptions()); + tgt_file->Append(content); + tgt_file->Close(); + + std::unique_ptr f; + fs->NewRandomAccessFile(fname, FileOptions(), &f, nullptr); + int rand_num = rand.Next() % 3; + auto temperature = rand_num == 0 ? Temperature::kUnknown + : rand_num == 1 ? Temperature::kWarm + : Temperature::kCold; + readers.emplace_back(new RandomAccessFileReader( + std::move(f), fname, env->GetSystemClock().get(), nullptr, statistics, + Histograms::HISTOGRAM_ENUM_MAX, nullptr, nullptr, {}, temperature, + rand_num == 1)); + } + + IOOptions io_options; + std::unique_ptr scratch(new char[2048]); + Slice result; + uint64_t idx = 0; + for (auto _ : state) { + s = readers[idx++ % kFileNum]->Read(io_options, 0, kDefaultPageSize / 3, + &result, scratch.get(), nullptr, + Env::IO_TOTAL); + if (!s.ok()) { + state.SkipWithError(s.ToString().c_str()); + } + } + + // clean up + for (int i = 0; i < kFileNum; i++) { + std::string fname = fname_base + std::to_string(i); + env->DeleteFile(fname); // ignore return, okay to fail cleanup + } +} + +BENCHMARK(RandomAccessFileReaderRead) + ->Iterations(1000000) + ->Arg(0) + ->Arg(1) + ->ArgName("enable_statistics"); + +} // namespace ROCKSDB_NAMESPACE + +BENCHMARK_MAIN(); diff --git a/librocksdb-sys/rocksdb/microbench/ribbon_bench.cc b/librocksdb-sys/rocksdb/microbench/ribbon_bench.cc new file mode 100644 index 0000000..d0fb2ec --- /dev/null +++ b/librocksdb-sys/rocksdb/microbench/ribbon_bench.cc @@ -0,0 +1,155 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +// this is a simple micro-benchmark for compare ribbon filter vs. other filter +// for more comprehensive, please check the dedicate util/filter_bench. +#include "benchmark/benchmark.h" +#include "table/block_based/filter_policy_internal.h" +#include "table/block_based/mock_block_based_table.h" + +namespace ROCKSDB_NAMESPACE { + +struct KeyMaker { + explicit KeyMaker(size_t avg_size) + : smallest_size_(avg_size), + buf_size_(avg_size + 11), // pad to vary key size and alignment + buf_(new char[buf_size_]) { + memset(buf_.get(), 0, buf_size_); + assert(smallest_size_ > 8); + } + size_t smallest_size_; + size_t buf_size_; + std::unique_ptr buf_; + + // Returns a unique(-ish) key based on the given parameter values. Each + // call returns a Slice from the same buffer so previously returned + // Slices should be considered invalidated. + Slice Get(uint32_t filter_num, uint32_t val_num) const { + size_t start = val_num % 4; + size_t len = smallest_size_; + // To get range [avg_size - 2, avg_size + 2] + // use range [smallest_size, smallest_size + 4] + len += FastRange32((val_num >> 5) * 1234567891, 5); + char *data = buf_.get() + start; + // Populate key data such that all data makes it into a key of at + // least 8 bytes. We also don't want all the within-filter key + // variance confined to a contiguous 32 bits, because then a 32 bit + // hash function can "cheat" the false positive rate by + // approximating a perfect hash. + EncodeFixed32(data, val_num); + EncodeFixed32(data + 4, filter_num + val_num); + // ensure clearing leftovers from different alignment + EncodeFixed32(data + 8, 0); + return {data, len}; + } +}; + +// benchmark arguments: +// 0. filter impl (like filter_bench -impl) +// 1. filter config bits_per_key +// 2. average data key length +// 3. data entry number +static void CustomArguments(benchmark::internal::Benchmark *b) { + const auto kImplCount = + static_cast(BloomLikeFilterPolicy::GetAllFixedImpls().size()); + for (int filter_impl = 0; filter_impl < kImplCount; ++filter_impl) { + for (int bits_per_key : {10, 20}) { + for (int key_len_avg : {10, 100}) { + for (int64_t entry_num : {1 << 10, 1 << 20}) { + b->Args({filter_impl, bits_per_key, key_len_avg, entry_num}); + } + } + } + } + b->ArgNames({"filter_impl", "bits_per_key", "key_len_avg", "entry_num"}); +} + +static void FilterBuild(benchmark::State &state) { + // setup data + auto filter = BloomLikeFilterPolicy::Create( + BloomLikeFilterPolicy::GetAllFixedImpls().at(state.range(0)), + static_cast(state.range(1))); + auto tester = std::make_unique(filter); + KeyMaker km(state.range(2)); + std::unique_ptr owner; + const int64_t kEntryNum = state.range(3); + auto rnd = Random32(12345); + uint32_t filter_num = rnd.Next(); + // run the test + for (auto _ : state) { + std::unique_ptr builder(tester->GetBuilder()); + for (uint32_t i = 0; i < kEntryNum; i++) { + builder->AddKey(km.Get(filter_num, i)); + } + auto ret = builder->Finish(&owner); + state.counters["size"] = static_cast(ret.size()); + } +} +BENCHMARK(FilterBuild)->Apply(CustomArguments); + +static void FilterQueryPositive(benchmark::State &state) { + // setup data + auto filter = BloomLikeFilterPolicy::Create( + BloomLikeFilterPolicy::GetAllFixedImpls().at(state.range(0)), + static_cast(state.range(1))); + auto tester = std::make_unique(filter); + KeyMaker km(state.range(2)); + std::unique_ptr owner; + const int64_t kEntryNum = state.range(3); + auto rnd = Random32(12345); + uint32_t filter_num = rnd.Next(); + std::unique_ptr builder(tester->GetBuilder()); + for (uint32_t i = 0; i < kEntryNum; i++) { + builder->AddKey(km.Get(filter_num, i)); + } + auto data = builder->Finish(&owner); + std::unique_ptr reader{filter->GetFilterBitsReader(data)}; + + // run test + uint32_t i = 0; + for (auto _ : state) { + i++; + i = i % kEntryNum; + reader->MayMatch(km.Get(filter_num, i)); + } +} +BENCHMARK(FilterQueryPositive)->Apply(CustomArguments); + +static void FilterQueryNegative(benchmark::State &state) { + // setup data + auto filter = BloomLikeFilterPolicy::Create( + BloomLikeFilterPolicy::GetAllFixedImpls().at(state.range(0)), + static_cast(state.range(1))); + auto tester = std::make_unique(filter); + KeyMaker km(state.range(2)); + std::unique_ptr owner; + const int64_t kEntryNum = state.range(3); + auto rnd = Random32(12345); + uint32_t filter_num = rnd.Next(); + std::unique_ptr builder(tester->GetBuilder()); + for (uint32_t i = 0; i < kEntryNum; i++) { + builder->AddKey(km.Get(filter_num, i)); + } + auto data = builder->Finish(&owner); + std::unique_ptr reader{filter->GetFilterBitsReader(data)}; + + // run test + uint32_t i = 0; + double fp_cnt = 0; + for (auto _ : state) { + i++; + auto result = reader->MayMatch(km.Get(filter_num + 1, i)); + if (result) { + fp_cnt++; + } + } + state.counters["fp_pct"] = + benchmark::Counter(fp_cnt * 100, benchmark::Counter::kAvgIterations); +} +BENCHMARK(FilterQueryNegative)->Apply(CustomArguments); + +} // namespace ROCKSDB_NAMESPACE + +BENCHMARK_MAIN(); diff --git a/librocksdb-sys/rocksdb/monitoring/file_read_sample.h b/librocksdb-sys/rocksdb/monitoring/file_read_sample.h new file mode 100644 index 0000000..82a933e --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/file_read_sample.h @@ -0,0 +1,23 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#pragma once +#include "db/version_edit.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { +static const uint32_t kFileReadSampleRate = 1024; +extern bool should_sample_file_read(); +extern void sample_file_read_inc(FileMetaData*); + +inline bool should_sample_file_read() { + return (Random::GetTLSInstance()->Next() % kFileReadSampleRate == 307); +} + +inline void sample_file_read_inc(FileMetaData* meta) { + meta->stats.num_reads_sampled.fetch_add(kFileReadSampleRate, + std::memory_order_relaxed); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/histogram.cc b/librocksdb-sys/rocksdb/monitoring/histogram.cc new file mode 100644 index 0000000..61bc6c1 --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/histogram.cc @@ -0,0 +1,270 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "monitoring/histogram.h" + +#include + +#include +#include +#include +#include + +#include "port/port.h" +#include "util/cast_util.h" + +namespace ROCKSDB_NAMESPACE { + +HistogramBucketMapper::HistogramBucketMapper() { + // If you change this, you also need to change + // size of array buckets_ in HistogramImpl + bucketValues_ = {1, 2}; + double bucket_val = static_cast(bucketValues_.back()); + while ((bucket_val = 1.5 * bucket_val) <= + static_cast(std::numeric_limits::max())) { + bucketValues_.push_back(static_cast(bucket_val)); + // Extracts two most significant digits to make histogram buckets more + // human-readable. E.g., 172 becomes 170. + uint64_t pow_of_ten = 1; + while (bucketValues_.back() / 10 > 10) { + bucketValues_.back() /= 10; + pow_of_ten *= 10; + } + bucketValues_.back() *= pow_of_ten; + } + maxBucketValue_ = bucketValues_.back(); + minBucketValue_ = bucketValues_.front(); +} + +size_t HistogramBucketMapper::IndexForValue(const uint64_t value) const { + auto beg = bucketValues_.begin(); + auto end = bucketValues_.end(); + if (value >= maxBucketValue_) + return end - beg - 1; // bucketValues_.size() - 1 + else + return std::lower_bound(beg, end, value) - beg; +} + +namespace { +const HistogramBucketMapper bucketMapper; +} + +HistogramStat::HistogramStat() : num_buckets_(bucketMapper.BucketCount()) { + assert(num_buckets_ == sizeof(buckets_) / sizeof(*buckets_)); + Clear(); +} + +void HistogramStat::Clear() { + min_.store(bucketMapper.LastValue(), std::memory_order_relaxed); + max_.store(0, std::memory_order_relaxed); + num_.store(0, std::memory_order_relaxed); + sum_.store(0, std::memory_order_relaxed); + sum_squares_.store(0, std::memory_order_relaxed); + for (unsigned int b = 0; b < num_buckets_; b++) { + buckets_[b].store(0, std::memory_order_relaxed); + } +}; + +bool HistogramStat::Empty() const { return num() == 0; } + +void HistogramStat::Add(uint64_t value) { + // This function is designed to be lock free, as it's in the critical path + // of any operation. Each individual value is atomic and the order of updates + // by concurrent threads is tolerable. + const size_t index = bucketMapper.IndexForValue(value); + assert(index < num_buckets_); + buckets_[index].store(buckets_[index].load(std::memory_order_relaxed) + 1, + std::memory_order_relaxed); + + uint64_t old_min = min(); + if (value < old_min) { + min_.store(value, std::memory_order_relaxed); + } + + uint64_t old_max = max(); + if (value > old_max) { + max_.store(value, std::memory_order_relaxed); + } + + num_.store(num_.load(std::memory_order_relaxed) + 1, + std::memory_order_relaxed); + sum_.store(sum_.load(std::memory_order_relaxed) + value, + std::memory_order_relaxed); + sum_squares_.store( + sum_squares_.load(std::memory_order_relaxed) + value * value, + std::memory_order_relaxed); +} + +void HistogramStat::Merge(const HistogramStat& other) { + // This function needs to be performned with the outer lock acquired + // However, atomic operation on every member is still need, since Add() + // requires no lock and value update can still happen concurrently + uint64_t old_min = min(); + uint64_t other_min = other.min(); + while (other_min < old_min && + !min_.compare_exchange_weak(old_min, other_min)) { + } + + uint64_t old_max = max(); + uint64_t other_max = other.max(); + while (other_max > old_max && + !max_.compare_exchange_weak(old_max, other_max)) { + } + + num_.fetch_add(other.num(), std::memory_order_relaxed); + sum_.fetch_add(other.sum(), std::memory_order_relaxed); + sum_squares_.fetch_add(other.sum_squares(), std::memory_order_relaxed); + for (unsigned int b = 0; b < num_buckets_; b++) { + buckets_[b].fetch_add(other.bucket_at(b), std::memory_order_relaxed); + } +} + +double HistogramStat::Median() const { return Percentile(50.0); } + +double HistogramStat::Percentile(double p) const { + double threshold = num() * (p / 100.0); + uint64_t cumulative_sum = 0; + for (unsigned int b = 0; b < num_buckets_; b++) { + uint64_t bucket_value = bucket_at(b); + cumulative_sum += bucket_value; + if (cumulative_sum >= threshold) { + // Scale linearly within this bucket + uint64_t left_point = (b == 0) ? 0 : bucketMapper.BucketLimit(b - 1); + uint64_t right_point = bucketMapper.BucketLimit(b); + uint64_t left_sum = cumulative_sum - bucket_value; + uint64_t right_sum = cumulative_sum; + double pos = 0; + uint64_t right_left_diff = right_sum - left_sum; + if (right_left_diff != 0) { + pos = (threshold - left_sum) / right_left_diff; + } + double r = left_point + (right_point - left_point) * pos; + uint64_t cur_min = min(); + uint64_t cur_max = max(); + if (r < cur_min) r = static_cast(cur_min); + if (r > cur_max) r = static_cast(cur_max); + return r; + } + } + return static_cast(max()); +} + +double HistogramStat::Average() const { + uint64_t cur_num = num(); + uint64_t cur_sum = sum(); + if (cur_num == 0) return 0; + return static_cast(cur_sum) / static_cast(cur_num); +} + +double HistogramStat::StandardDeviation() const { + double cur_num = + static_cast(num()); // Use double to avoid integer overflow + double cur_sum = static_cast(sum()); + double cur_sum_squares = static_cast(sum_squares()); + if (cur_num == 0.0) { + return 0.0; + } + double variance = + (cur_sum_squares * cur_num - cur_sum * cur_sum) / (cur_num * cur_num); + return std::sqrt(std::max(variance, 0.0)); +} + +std::string HistogramStat::ToString() const { + uint64_t cur_num = num(); + std::string r; + char buf[1650]; + snprintf(buf, sizeof(buf), "Count: %" PRIu64 " Average: %.4f StdDev: %.2f\n", + cur_num, Average(), StandardDeviation()); + r.append(buf); + snprintf(buf, sizeof(buf), + "Min: %" PRIu64 " Median: %.4f Max: %" PRIu64 "\n", + (cur_num == 0 ? 0 : min()), Median(), (cur_num == 0 ? 0 : max())); + r.append(buf); + snprintf(buf, sizeof(buf), + "Percentiles: " + "P50: %.2f P75: %.2f P99: %.2f P99.9: %.2f P99.99: %.2f\n", + Percentile(50), Percentile(75), Percentile(99), Percentile(99.9), + Percentile(99.99)); + r.append(buf); + r.append("------------------------------------------------------\n"); + if (cur_num == 0) return r; // all buckets are empty + const double mult = 100.0 / cur_num; + uint64_t cumulative_sum = 0; + for (unsigned int b = 0; b < num_buckets_; b++) { + uint64_t bucket_value = bucket_at(b); + if (bucket_value <= 0.0) continue; + cumulative_sum += bucket_value; + snprintf(buf, sizeof(buf), + "%c %7" PRIu64 ", %7" PRIu64 " ] %8" PRIu64 " %7.3f%% %7.3f%% ", + (b == 0) ? '[' : '(', + (b == 0) ? 0 : bucketMapper.BucketLimit(b - 1), // left + bucketMapper.BucketLimit(b), // right + bucket_value, // count + (mult * bucket_value), // percentage + (mult * cumulative_sum)); // cumulative percentage + r.append(buf); + + // Add hash marks based on percentage; 20 marks for 100%. + size_t marks = static_cast(mult * bucket_value / 5 + 0.5); + r.append(marks, '#'); + r.push_back('\n'); + } + return r; +} + +void HistogramStat::Data(HistogramData* const data) const { + assert(data); + data->median = Median(); + data->percentile95 = Percentile(95); + data->percentile99 = Percentile(99); + data->max = static_cast(max()); + data->average = Average(); + data->standard_deviation = StandardDeviation(); + data->count = num(); + data->sum = sum(); + data->min = static_cast(min()); +} + +void HistogramImpl::Clear() { + std::lock_guard lock(mutex_); + stats_.Clear(); +} + +bool HistogramImpl::Empty() const { return stats_.Empty(); } + +void HistogramImpl::Add(uint64_t value) { stats_.Add(value); } + +void HistogramImpl::Merge(const Histogram& other) { + if (strcmp(Name(), other.Name()) == 0) { + Merge(*static_cast_with_check(&other)); + } +} + +void HistogramImpl::Merge(const HistogramImpl& other) { + std::lock_guard lock(mutex_); + stats_.Merge(other.stats_); +} + +double HistogramImpl::Median() const { return stats_.Median(); } + +double HistogramImpl::Percentile(double p) const { + return stats_.Percentile(p); +} + +double HistogramImpl::Average() const { return stats_.Average(); } + +double HistogramImpl::StandardDeviation() const { + return stats_.StandardDeviation(); +} + +std::string HistogramImpl::ToString() const { return stats_.ToString(); } + +void HistogramImpl::Data(HistogramData* const data) const { stats_.Data(data); } + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/histogram.h b/librocksdb-sys/rocksdb/monitoring/histogram.h new file mode 100644 index 0000000..15fee2b --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/histogram.h @@ -0,0 +1,143 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include +#include +#include +#include +#include + +#include "rocksdb/statistics.h" + +namespace ROCKSDB_NAMESPACE { + +class HistogramBucketMapper { + public: + HistogramBucketMapper(); + + // converts a value to the bucket index. + size_t IndexForValue(uint64_t value) const; + // number of buckets required. + + size_t BucketCount() const { return bucketValues_.size(); } + + uint64_t LastValue() const { return maxBucketValue_; } + + uint64_t FirstValue() const { return minBucketValue_; } + + uint64_t BucketLimit(const size_t bucketNumber) const { + assert(bucketNumber < BucketCount()); + return bucketValues_[bucketNumber]; + } + + private: + std::vector bucketValues_; + uint64_t maxBucketValue_; + uint64_t minBucketValue_; +}; + +struct HistogramStat { + HistogramStat(); + ~HistogramStat() {} + + HistogramStat(const HistogramStat&) = delete; + HistogramStat& operator=(const HistogramStat&) = delete; + + void Clear(); + bool Empty() const; + void Add(uint64_t value); + void Merge(const HistogramStat& other); + + inline uint64_t min() const { return min_.load(std::memory_order_relaxed); } + inline uint64_t max() const { return max_.load(std::memory_order_relaxed); } + inline uint64_t num() const { return num_.load(std::memory_order_relaxed); } + inline uint64_t sum() const { return sum_.load(std::memory_order_relaxed); } + inline uint64_t sum_squares() const { + return sum_squares_.load(std::memory_order_relaxed); + } + inline uint64_t bucket_at(size_t b) const { + return buckets_[b].load(std::memory_order_relaxed); + } + + double Median() const; + double Percentile(double p) const; + double Average() const; + double StandardDeviation() const; + void Data(HistogramData* const data) const; + std::string ToString() const; + + // To be able to use HistogramStat as thread local variable, it + // cannot have dynamic allocated member. That's why we're + // using manually values from BucketMapper + std::atomic_uint_fast64_t min_; + std::atomic_uint_fast64_t max_; + std::atomic_uint_fast64_t num_; + std::atomic_uint_fast64_t sum_; + std::atomic_uint_fast64_t sum_squares_; + std::atomic_uint_fast64_t buckets_[109]; // 109==BucketMapper::BucketCount() + const uint64_t num_buckets_; +}; + +class Histogram { + public: + Histogram() {} + virtual ~Histogram(){}; + + virtual void Clear() = 0; + virtual bool Empty() const = 0; + virtual void Add(uint64_t value) = 0; + virtual void Merge(const Histogram&) = 0; + + virtual std::string ToString() const = 0; + virtual const char* Name() const = 0; + virtual uint64_t min() const = 0; + virtual uint64_t max() const = 0; + virtual uint64_t num() const = 0; + virtual double Median() const = 0; + virtual double Percentile(double p) const = 0; + virtual double Average() const = 0; + virtual double StandardDeviation() const = 0; + virtual void Data(HistogramData* const data) const = 0; +}; + +class HistogramImpl : public Histogram { + public: + HistogramImpl() { Clear(); } + + HistogramImpl(const HistogramImpl&) = delete; + HistogramImpl& operator=(const HistogramImpl&) = delete; + + virtual void Clear() override; + virtual bool Empty() const override; + virtual void Add(uint64_t value) override; + virtual void Merge(const Histogram& other) override; + void Merge(const HistogramImpl& other); + + virtual std::string ToString() const override; + virtual const char* Name() const override { return "HistogramImpl"; } + virtual uint64_t min() const override { return stats_.min(); } + virtual uint64_t max() const override { return stats_.max(); } + virtual uint64_t num() const override { return stats_.num(); } + virtual double Median() const override; + virtual double Percentile(double p) const override; + virtual double Average() const override; + virtual double StandardDeviation() const override; + virtual void Data(HistogramData* const data) const override; + + virtual ~HistogramImpl() {} + + inline HistogramStat& TEST_GetStats() { return stats_; } + + private: + HistogramStat stats_; + std::mutex mutex_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/histogram_test.cc b/librocksdb-sys/rocksdb/monitoring/histogram_test.cc new file mode 100644 index 0000000..19e9f15 --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/histogram_test.cc @@ -0,0 +1,254 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#include "monitoring/histogram.h" + +#include + +#include "monitoring/histogram_windowing.h" +#include "rocksdb/system_clock.h" +#include "test_util/mock_time_env.h" +#include "test_util/testharness.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +class HistogramTest : public testing::Test {}; + +namespace { +const double kIota = 0.1; +const HistogramBucketMapper bucketMapper; +std::shared_ptr clock = + std::make_shared(SystemClock::Default()); +} // namespace + +void PopulateHistogram(Histogram& histogram, uint64_t low, uint64_t high, + uint64_t loop = 1) { + Random rnd(test::RandomSeed()); + for (; loop > 0; loop--) { + for (uint64_t i = low; i <= high; i++) { + histogram.Add(i); + // sleep a random microseconds [0-10) + clock->SleepForMicroseconds(rnd.Uniform(10)); + } + } + // make sure each data population at least take some time + clock->SleepForMicroseconds(1); +} + +void BasicOperation(Histogram& histogram) { + PopulateHistogram(histogram, 1, 110, 10); // fill up to bucket [70, 110) + + HistogramData data; + histogram.Data(&data); + + ASSERT_LE(fabs(histogram.Percentile(100.0) - 110.0), kIota); + ASSERT_LE(fabs(data.percentile99 - 108.9), kIota); // 99 * 110 / 100 + ASSERT_LE(fabs(data.percentile95 - 104.5), kIota); // 95 * 110 / 100 + ASSERT_LE(fabs(data.median - 55.0), kIota); // 50 * 110 / 100 + ASSERT_EQ(data.average, 55.5); // (1 + 110) / 2 +} + +void MergeHistogram(Histogram& histogram, Histogram& other) { + PopulateHistogram(histogram, 1, 100); + PopulateHistogram(other, 101, 250); + histogram.Merge(other); + + HistogramData data; + histogram.Data(&data); + + ASSERT_LE(fabs(histogram.Percentile(100.0) - 250.0), kIota); + ASSERT_LE(fabs(data.percentile99 - 247.5), kIota); // 99 * 250 / 100 + ASSERT_LE(fabs(data.percentile95 - 237.5), kIota); // 95 * 250 / 100 + ASSERT_LE(fabs(data.median - 125.0), kIota); // 50 * 250 / 100 + ASSERT_EQ(data.average, 125.5); // (1 + 250) / 2 +} + +void EmptyHistogram(Histogram& histogram) { + ASSERT_EQ(histogram.min(), bucketMapper.LastValue()); + ASSERT_EQ(histogram.max(), 0); + ASSERT_EQ(histogram.num(), 0); + ASSERT_EQ(histogram.Median(), 0.0); + ASSERT_EQ(histogram.Percentile(85.0), 0.0); + ASSERT_EQ(histogram.Average(), 0.0); + ASSERT_EQ(histogram.StandardDeviation(), 0.0); +} + +void ClearHistogram(Histogram& histogram) { + for (uint64_t i = 1; i <= 100; i++) { + histogram.Add(i); + } + histogram.Clear(); + ASSERT_TRUE(histogram.Empty()); + ASSERT_EQ(histogram.Median(), 0); + ASSERT_EQ(histogram.Percentile(85.0), 0); + ASSERT_EQ(histogram.Average(), 0); +} + +TEST_F(HistogramTest, BasicOperation) { + HistogramImpl histogram; + BasicOperation(histogram); + + HistogramWindowingImpl histogramWindowing; + BasicOperation(histogramWindowing); +} + +TEST_F(HistogramTest, BoundaryValue) { + HistogramImpl histogram; + // - both should be in [0, 1] bucket because we place values on bucket + // boundaries in the lower bucket. + // - all points are in [0, 1] bucket, so p50 will be 0.5 + // - the test cannot be written with a single point since histogram won't + // report percentiles lower than the min or greater than the max. + histogram.Add(0); + histogram.Add(1); + + ASSERT_LE(fabs(histogram.Percentile(50.0) - 0.5), kIota); +} + +TEST_F(HistogramTest, MergeHistogram) { + HistogramImpl histogram; + HistogramImpl other; + MergeHistogram(histogram, other); + + HistogramWindowingImpl histogramWindowing; + HistogramWindowingImpl otherWindowing; + MergeHistogram(histogramWindowing, otherWindowing); +} + +TEST_F(HistogramTest, EmptyHistogram) { + HistogramImpl histogram; + EmptyHistogram(histogram); + + HistogramWindowingImpl histogramWindowing; + EmptyHistogram(histogramWindowing); +} + +TEST_F(HistogramTest, ClearHistogram) { + HistogramImpl histogram; + ClearHistogram(histogram); + + HistogramWindowingImpl histogramWindowing; + ClearHistogram(histogramWindowing); +} + +TEST_F(HistogramTest, HistogramWindowingExpire) { + uint64_t num_windows = 3; + int micros_per_window = 1000000; + uint64_t min_num_per_window = 0; + + HistogramWindowingImpl histogramWindowing(num_windows, micros_per_window, + min_num_per_window); + histogramWindowing.TEST_UpdateClock(clock); + PopulateHistogram(histogramWindowing, 1, 1, 100); + clock->SleepForMicroseconds(micros_per_window); + ASSERT_EQ(histogramWindowing.num(), 100); + ASSERT_EQ(histogramWindowing.min(), 1); + ASSERT_EQ(histogramWindowing.max(), 1); + ASSERT_EQ(histogramWindowing.Average(), 1.0); + ASSERT_EQ(histogramWindowing.StandardDeviation(), 0.0); + + PopulateHistogram(histogramWindowing, 2, 2, 100); + clock->SleepForMicroseconds(micros_per_window); + ASSERT_EQ(histogramWindowing.num(), 200); + ASSERT_EQ(histogramWindowing.min(), 1); + ASSERT_EQ(histogramWindowing.max(), 2); + ASSERT_EQ(histogramWindowing.Average(), 1.5); + ASSERT_GT(histogramWindowing.StandardDeviation(), 0.0); + + PopulateHistogram(histogramWindowing, 3, 3, 100); + clock->SleepForMicroseconds(micros_per_window); + ASSERT_EQ(histogramWindowing.num(), 300); + ASSERT_EQ(histogramWindowing.min(), 1); + ASSERT_EQ(histogramWindowing.max(), 3); + ASSERT_EQ(histogramWindowing.Average(), 2.0); + ASSERT_GT(histogramWindowing.StandardDeviation(), 0.0); + + // dropping oldest window with value 1, remaining 2 ~ 4 + PopulateHistogram(histogramWindowing, 4, 4, 100); + clock->SleepForMicroseconds(micros_per_window); + ASSERT_EQ(histogramWindowing.num(), 300); + ASSERT_EQ(histogramWindowing.min(), 2); + ASSERT_EQ(histogramWindowing.max(), 4); + ASSERT_EQ(histogramWindowing.Average(), 3.0); + ASSERT_GT(histogramWindowing.StandardDeviation(), 0.0); + + // dropping oldest window with value 2, remaining 3 ~ 5 + PopulateHistogram(histogramWindowing, 5, 5, 100); + clock->SleepForMicroseconds(micros_per_window); + ASSERT_EQ(histogramWindowing.num(), 300); + ASSERT_EQ(histogramWindowing.min(), 3); + ASSERT_EQ(histogramWindowing.max(), 5); + ASSERT_EQ(histogramWindowing.Average(), 4.0); + ASSERT_GT(histogramWindowing.StandardDeviation(), 0.0); +} + +TEST_F(HistogramTest, HistogramWindowingMerge) { + uint64_t num_windows = 3; + int micros_per_window = 1000000; + uint64_t min_num_per_window = 0; + + HistogramWindowingImpl histogramWindowing(num_windows, micros_per_window, + min_num_per_window); + HistogramWindowingImpl otherWindowing(num_windows, micros_per_window, + min_num_per_window); + histogramWindowing.TEST_UpdateClock(clock); + otherWindowing.TEST_UpdateClock(clock); + + PopulateHistogram(histogramWindowing, 1, 1, 100); + PopulateHistogram(otherWindowing, 1, 1, 100); + clock->SleepForMicroseconds(micros_per_window); + + PopulateHistogram(histogramWindowing, 2, 2, 100); + PopulateHistogram(otherWindowing, 2, 2, 100); + clock->SleepForMicroseconds(micros_per_window); + + PopulateHistogram(histogramWindowing, 3, 3, 100); + PopulateHistogram(otherWindowing, 3, 3, 100); + clock->SleepForMicroseconds(micros_per_window); + + histogramWindowing.Merge(otherWindowing); + ASSERT_EQ(histogramWindowing.num(), 600); + ASSERT_EQ(histogramWindowing.min(), 1); + ASSERT_EQ(histogramWindowing.max(), 3); + ASSERT_EQ(histogramWindowing.Average(), 2.0); + + // dropping oldest window with value 1, remaining 2 ~ 4 + PopulateHistogram(histogramWindowing, 4, 4, 100); + clock->SleepForMicroseconds(micros_per_window); + ASSERT_EQ(histogramWindowing.num(), 500); + ASSERT_EQ(histogramWindowing.min(), 2); + ASSERT_EQ(histogramWindowing.max(), 4); + + // dropping oldest window with value 2, remaining 3 ~ 5 + PopulateHistogram(histogramWindowing, 5, 5, 100); + clock->SleepForMicroseconds(micros_per_window); + ASSERT_EQ(histogramWindowing.num(), 400); + ASSERT_EQ(histogramWindowing.min(), 3); + ASSERT_EQ(histogramWindowing.max(), 5); +} + +TEST_F(HistogramTest, LargeStandardDeviation) { + HistogramImpl histogram; + PopulateHistogram(histogram, 1, 1000000); + ASSERT_LT(fabs(histogram.StandardDeviation() - 288675), 1); +} + +TEST_F(HistogramTest, LostUpdateStandardDeviation) { + HistogramImpl histogram; + PopulateHistogram(histogram, 100, 100, 100); + // Simulate a possible lost update (since they are not atomic) + histogram.TEST_GetStats().sum_squares_ -= 10000; + // Ideally zero, but should never be negative or NaN + ASSERT_GE(histogram.StandardDeviation(), 0.0); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/monitoring/histogram_windowing.cc b/librocksdb-sys/rocksdb/monitoring/histogram_windowing.cc new file mode 100644 index 0000000..c41ae8a --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/histogram_windowing.cc @@ -0,0 +1,194 @@ +// Copyright (c) 2013, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "monitoring/histogram_windowing.h" + +#include + +#include "monitoring/histogram.h" +#include "rocksdb/system_clock.h" +#include "util/cast_util.h" + +namespace ROCKSDB_NAMESPACE { + +HistogramWindowingImpl::HistogramWindowingImpl() { + clock_ = SystemClock::Default(); + window_stats_.reset(new HistogramStat[static_cast(num_windows_)]); + Clear(); +} + +HistogramWindowingImpl::HistogramWindowingImpl(uint64_t num_windows, + uint64_t micros_per_window, + uint64_t min_num_per_window) + : num_windows_(num_windows), + micros_per_window_(micros_per_window), + min_num_per_window_(min_num_per_window) { + clock_ = SystemClock::Default(); + window_stats_.reset(new HistogramStat[static_cast(num_windows_)]); + Clear(); +} + +HistogramWindowingImpl::~HistogramWindowingImpl() {} + +void HistogramWindowingImpl::Clear() { + std::lock_guard lock(mutex_); + + stats_.Clear(); + for (size_t i = 0; i < num_windows_; i++) { + window_stats_[i].Clear(); + } + current_window_.store(0, std::memory_order_relaxed); + last_swap_time_.store(clock_->NowMicros(), std::memory_order_relaxed); +} + +bool HistogramWindowingImpl::Empty() const { return stats_.Empty(); } + +// This function is designed to be lock free, as it's in the critical path +// of any operation. +// Each individual value is atomic, it is just that some samples can go +// in the older bucket which is tolerable. +void HistogramWindowingImpl::Add(uint64_t value) { + TimerTick(); + + // Parent (global) member update + stats_.Add(value); + + // Current window update + window_stats_[static_cast(current_window())].Add(value); +} + +void HistogramWindowingImpl::Merge(const Histogram& other) { + if (strcmp(Name(), other.Name()) == 0) { + Merge(*static_cast_with_check(&other)); + } +} + +void HistogramWindowingImpl::Merge(const HistogramWindowingImpl& other) { + std::lock_guard lock(mutex_); + stats_.Merge(other.stats_); + + if (stats_.num_buckets_ != other.stats_.num_buckets_ || + micros_per_window_ != other.micros_per_window_) { + return; + } + + uint64_t cur_window = current_window(); + uint64_t other_cur_window = other.current_window(); + // going backwards for alignment + for (unsigned int i = 0; i < std::min(num_windows_, other.num_windows_); + i++) { + uint64_t window_index = (cur_window + num_windows_ - i) % num_windows_; + uint64_t other_window_index = + (other_cur_window + other.num_windows_ - i) % other.num_windows_; + size_t windex = static_cast(window_index); + size_t other_windex = static_cast(other_window_index); + + window_stats_[windex].Merge(other.window_stats_[other_windex]); + } +} + +std::string HistogramWindowingImpl::ToString() const { + return stats_.ToString(); +} + +double HistogramWindowingImpl::Median() const { return Percentile(50.0); } + +double HistogramWindowingImpl::Percentile(double p) const { + // Retry 3 times in total + for (int retry = 0; retry < 3; retry++) { + uint64_t start_num = stats_.num(); + double result = stats_.Percentile(p); + // Detect if swap buckets or Clear() was called during calculation + if (stats_.num() >= start_num) { + return result; + } + } + return 0.0; +} + +double HistogramWindowingImpl::Average() const { return stats_.Average(); } + +double HistogramWindowingImpl::StandardDeviation() const { + return stats_.StandardDeviation(); +} + +void HistogramWindowingImpl::Data(HistogramData* const data) const { + stats_.Data(data); +} + +void HistogramWindowingImpl::TimerTick() { + uint64_t curr_time = clock_->NowMicros(); + size_t curr_window_ = static_cast(current_window()); + if (curr_time - last_swap_time() > micros_per_window_ && + window_stats_[curr_window_].num() >= min_num_per_window_) { + SwapHistoryBucket(); + } +} + +void HistogramWindowingImpl::SwapHistoryBucket() { + // Threads executing Add() would be competing for this mutex, the first one + // who got the metex would take care of the bucket swap, other threads + // can skip this. + // If mutex is held by Merge() or Clear(), next Add() will take care of the + // swap, if needed. + if (mutex_.try_lock()) { + last_swap_time_.store(clock_->NowMicros(), std::memory_order_relaxed); + + uint64_t curr_window = current_window(); + uint64_t next_window = + (curr_window == num_windows_ - 1) ? 0 : curr_window + 1; + + // subtract next buckets from totals and swap to next buckets + HistogramStat& stats_to_drop = + window_stats_[static_cast(next_window)]; + + if (!stats_to_drop.Empty()) { + for (size_t b = 0; b < stats_.num_buckets_; b++) { + stats_.buckets_[b].fetch_sub(stats_to_drop.bucket_at(b), + std::memory_order_relaxed); + } + + if (stats_.min() == stats_to_drop.min()) { + uint64_t new_min = std::numeric_limits::max(); + for (unsigned int i = 0; i < num_windows_; i++) { + if (i != next_window) { + uint64_t m = window_stats_[i].min(); + if (m < new_min) new_min = m; + } + } + stats_.min_.store(new_min, std::memory_order_relaxed); + } + + if (stats_.max() == stats_to_drop.max()) { + uint64_t new_max = 0; + for (unsigned int i = 0; i < num_windows_; i++) { + if (i != next_window) { + uint64_t m = window_stats_[i].max(); + if (m > new_max) new_max = m; + } + } + stats_.max_.store(new_max, std::memory_order_relaxed); + } + + stats_.num_.fetch_sub(stats_to_drop.num(), std::memory_order_relaxed); + stats_.sum_.fetch_sub(stats_to_drop.sum(), std::memory_order_relaxed); + stats_.sum_squares_.fetch_sub(stats_to_drop.sum_squares(), + std::memory_order_relaxed); + + stats_to_drop.Clear(); + } + + // advance to next window bucket + current_window_.store(next_window, std::memory_order_relaxed); + + mutex_.unlock(); + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/histogram_windowing.h b/librocksdb-sys/rocksdb/monitoring/histogram_windowing.h new file mode 100644 index 0000000..9a86267 --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/histogram_windowing.h @@ -0,0 +1,84 @@ +// Copyright (c) 2013, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include "monitoring/histogram.h" + +namespace ROCKSDB_NAMESPACE { +class SystemClock; + +class HistogramWindowingImpl : public Histogram { + public: + HistogramWindowingImpl(); + HistogramWindowingImpl(uint64_t num_windows, uint64_t micros_per_window, + uint64_t min_num_per_window); + + HistogramWindowingImpl(const HistogramWindowingImpl&) = delete; + HistogramWindowingImpl& operator=(const HistogramWindowingImpl&) = delete; + + ~HistogramWindowingImpl(); + + virtual void Clear() override; + virtual bool Empty() const override; + virtual void Add(uint64_t value) override; + virtual void Merge(const Histogram& other) override; + void Merge(const HistogramWindowingImpl& other); + + virtual std::string ToString() const override; + virtual const char* Name() const override { return "HistogramWindowingImpl"; } + virtual uint64_t min() const override { return stats_.min(); } + virtual uint64_t max() const override { return stats_.max(); } + virtual uint64_t num() const override { return stats_.num(); } + virtual double Median() const override; + virtual double Percentile(double p) const override; + virtual double Average() const override; + virtual double StandardDeviation() const override; + virtual void Data(HistogramData* const data) const override; + +#ifndef NDEBUG + void TEST_UpdateClock(const std::shared_ptr& clock) { + clock_ = clock; + } +#endif // NDEBUG + + private: + void TimerTick(); + void SwapHistoryBucket(); + inline uint64_t current_window() const { + return current_window_.load(std::memory_order_relaxed); + } + inline uint64_t last_swap_time() const { + return last_swap_time_.load(std::memory_order_relaxed); + } + + std::shared_ptr clock_; + std::mutex mutex_; + + // Aggregated stats over windows_stats_, all the computation is done + // upon aggregated values + HistogramStat stats_; + + // This is a circular array representing the latest N time-windows. + // Each entry stores a time-window of data. Expiration is done + // on window-based. + std::unique_ptr window_stats_; + + std::atomic_uint_fast64_t current_window_; + std::atomic_uint_fast64_t last_swap_time_; + + // Following parameters are configuable + uint64_t num_windows_ = 5; + uint64_t micros_per_window_ = 60000000; + // By default, don't care about the number of values in current window + // when decide whether to swap windows or not. + uint64_t min_num_per_window_ = 0; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/in_memory_stats_history.cc b/librocksdb-sys/rocksdb/monitoring/in_memory_stats_history.cc new file mode 100644 index 0000000..568d8ec --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/in_memory_stats_history.cc @@ -0,0 +1,50 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "monitoring/in_memory_stats_history.h" + +#include "db/db_impl/db_impl.h" + +namespace ROCKSDB_NAMESPACE { + +InMemoryStatsHistoryIterator::~InMemoryStatsHistoryIterator() {} + +bool InMemoryStatsHistoryIterator::Valid() const { return valid_; } + +Status InMemoryStatsHistoryIterator::status() const { return status_; } + +// Because of garbage collection, the next stats snapshot may or may not be +// right after the current one. When reading from DBImpl::stats_history_, this +// call will be protected by DB Mutex so it will not return partial or +// corrupted results. +void InMemoryStatsHistoryIterator::Next() { + // increment start_time by 1 to avoid infinite loop + AdvanceIteratorByTime(GetStatsTime() + 1, end_time_); +} + +uint64_t InMemoryStatsHistoryIterator::GetStatsTime() const { return time_; } + +const std::map& +InMemoryStatsHistoryIterator::GetStatsMap() const { + return stats_map_; +} + +// advance the iterator to the next time between [start_time, end_time) +// if success, update time_ and stats_map_ with new_time and stats_map +void InMemoryStatsHistoryIterator::AdvanceIteratorByTime(uint64_t start_time, + uint64_t end_time) { + // try to find next entry in stats_history_ map + if (db_impl_ != nullptr) { + valid_ = + db_impl_->FindStatsByTime(start_time, end_time, &time_, &stats_map_); + } else { + valid_ = false; + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/in_memory_stats_history.h b/librocksdb-sys/rocksdb/monitoring/in_memory_stats_history.h new file mode 100644 index 0000000..3be864f --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/in_memory_stats_history.h @@ -0,0 +1,74 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include "rocksdb/stats_history.h" + +namespace ROCKSDB_NAMESPACE { + +// InMemoryStatsHistoryIterator can be used to access stats history that was +// stored by an in-memory two level std::map(DBImpl::stats_history_). It keeps +// a copy of the stats snapshot (in stats_map_) that is currently being pointed +// to, which allows the iterator to access the stats snapshot even when +// the background garbage collecting thread purges it from the source of truth +// (`DBImpl::stats_history_`). In that case, the iterator will continue to be +// valid until a call to `Next()` returns no result and invalidates it. In +// some extreme cases, the iterator may also return fragmented segments of +// stats snapshots due to long gaps between `Next()` calls and interleaved +// garbage collection. +class InMemoryStatsHistoryIterator final : public StatsHistoryIterator { + public: + // Setup InMemoryStatsHistoryIterator to return stats snapshots between + // seconds timestamps [start_time, end_time) + InMemoryStatsHistoryIterator(uint64_t start_time, uint64_t end_time, + DBImpl* db_impl) + : start_time_(start_time), + end_time_(end_time), + valid_(true), + db_impl_(db_impl) { + AdvanceIteratorByTime(start_time_, end_time_); + } + // no copying allowed + InMemoryStatsHistoryIterator(const InMemoryStatsHistoryIterator&) = delete; + void operator=(const InMemoryStatsHistoryIterator&) = delete; + InMemoryStatsHistoryIterator(InMemoryStatsHistoryIterator&&) = delete; + InMemoryStatsHistoryIterator& operator=(InMemoryStatsHistoryIterator&&) = + delete; + + ~InMemoryStatsHistoryIterator() override; + bool Valid() const override; + Status status() const override; + + // Move to the next stats snapshot currently available + // This function may invalidate the iterator + // REQUIRES: Valid() + void Next() override; + + // REQUIRES: Valid() + uint64_t GetStatsTime() const override; + + // This function is idempotent + // REQUIRES: Valid() + const std::map& GetStatsMap() const override; + + private: + // advance the iterator to the next stats history record with timestamp + // between [start_time, end_time) + void AdvanceIteratorByTime(uint64_t start_time, uint64_t end_time); + + uint64_t time_; + uint64_t start_time_; + uint64_t end_time_; + std::map stats_map_; + Status status_; + bool valid_; + DBImpl* db_impl_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/instrumented_mutex.cc b/librocksdb-sys/rocksdb/monitoring/instrumented_mutex.cc new file mode 100644 index 0000000..699495a --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/instrumented_mutex.cc @@ -0,0 +1,90 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "monitoring/instrumented_mutex.h" + +#include "monitoring/perf_context_imp.h" +#include "monitoring/thread_status_util.h" +#include "rocksdb/system_clock.h" +#include "test_util/sync_point.h" + +namespace ROCKSDB_NAMESPACE { +namespace { +#ifndef NPERF_CONTEXT +Statistics* stats_for_report(SystemClock* clock, Statistics* stats) { + if (clock != nullptr && stats != nullptr && + stats->get_stats_level() > kExceptTimeForMutex) { + return stats; + } else { + return nullptr; + } +} +#endif // NPERF_CONTEXT +} // namespace + +void InstrumentedMutex::Lock() { + PERF_CONDITIONAL_TIMER_FOR_MUTEX_GUARD( + db_mutex_lock_nanos, stats_code_ == DB_MUTEX_WAIT_MICROS, + stats_for_report(clock_, stats_), stats_code_); + LockInternal(); +} + +void InstrumentedMutex::LockInternal() { +#ifndef NDEBUG + ThreadStatusUtil::TEST_StateDelay(ThreadStatus::STATE_MUTEX_WAIT); +#endif +#ifdef COERCE_CONTEXT_SWITCH + if (stats_code_ == DB_MUTEX_WAIT_MICROS) { + thread_local Random rnd(301); + if (rnd.OneIn(2)) { + if (bg_cv_) { + bg_cv_->SignalAll(); + } + sched_yield(); + } else { + uint32_t sleep_us = rnd.Uniform(11) * 1000; + if (bg_cv_) { + bg_cv_->SignalAll(); + } + SystemClock::Default()->SleepForMicroseconds(sleep_us); + } + } +#endif + mutex_.Lock(); +} + +void InstrumentedCondVar::Wait() { + PERF_CONDITIONAL_TIMER_FOR_MUTEX_GUARD( + db_condition_wait_nanos, stats_code_ == DB_MUTEX_WAIT_MICROS, + stats_for_report(clock_, stats_), stats_code_); + WaitInternal(); +} + +void InstrumentedCondVar::WaitInternal() { +#ifndef NDEBUG + ThreadStatusUtil::TEST_StateDelay(ThreadStatus::STATE_MUTEX_WAIT); +#endif + cond_.Wait(); +} + +bool InstrumentedCondVar::TimedWait(uint64_t abs_time_us) { + PERF_CONDITIONAL_TIMER_FOR_MUTEX_GUARD( + db_condition_wait_nanos, stats_code_ == DB_MUTEX_WAIT_MICROS, + stats_for_report(clock_, stats_), stats_code_); + return TimedWaitInternal(abs_time_us); +} + +bool InstrumentedCondVar::TimedWaitInternal(uint64_t abs_time_us) { +#ifndef NDEBUG + ThreadStatusUtil::TEST_StateDelay(ThreadStatus::STATE_MUTEX_WAIT); +#endif + + TEST_SYNC_POINT_CALLBACK("InstrumentedCondVar::TimedWaitInternal", + &abs_time_us); + + return cond_.TimedWait(abs_time_us); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/instrumented_mutex.h b/librocksdb-sys/rocksdb/monitoring/instrumented_mutex.h new file mode 100644 index 0000000..b97d250 --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/instrumented_mutex.h @@ -0,0 +1,126 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "monitoring/statistics_impl.h" +#include "port/port.h" +#include "rocksdb/statistics.h" +#include "rocksdb/system_clock.h" +#include "rocksdb/thread_status.h" +#include "util/stop_watch.h" + +namespace ROCKSDB_NAMESPACE { +class InstrumentedCondVar; + +// A wrapper class for port::Mutex that provides additional layer +// for collecting stats and instrumentation. +class InstrumentedMutex { + public: + explicit InstrumentedMutex(bool adaptive = false) + : mutex_(adaptive), stats_(nullptr), clock_(nullptr), stats_code_(0) {} + + explicit InstrumentedMutex(SystemClock* clock, bool adaptive = false) + : mutex_(adaptive), stats_(nullptr), clock_(clock), stats_code_(0) {} + + InstrumentedMutex(Statistics* stats, SystemClock* clock, int stats_code, + bool adaptive = false) + : mutex_(adaptive), + stats_(stats), + clock_(clock), + stats_code_(stats_code) {} + +#ifdef COERCE_CONTEXT_SWITCH + InstrumentedMutex(Statistics* stats, SystemClock* clock, int stats_code, + InstrumentedCondVar* bg_cv, bool adaptive = false) + : mutex_(adaptive), + stats_(stats), + clock_(clock), + stats_code_(stats_code), + bg_cv_(bg_cv) {} +#endif + + void Lock(); + + void Unlock() { mutex_.Unlock(); } + + void AssertHeld() { mutex_.AssertHeld(); } + + private: + void LockInternal(); + friend class InstrumentedCondVar; + port::Mutex mutex_; + Statistics* stats_; + SystemClock* clock_; + int stats_code_; +#ifdef COERCE_CONTEXT_SWITCH + InstrumentedCondVar* bg_cv_ = nullptr; +#endif +}; + +class ALIGN_AS(CACHE_LINE_SIZE) CacheAlignedInstrumentedMutex + : public InstrumentedMutex { + using InstrumentedMutex::InstrumentedMutex; +}; +static_assert(alignof(CacheAlignedInstrumentedMutex) != CACHE_LINE_SIZE || + sizeof(CacheAlignedInstrumentedMutex) % CACHE_LINE_SIZE == 0); + +// RAII wrapper for InstrumentedMutex +class InstrumentedMutexLock { + public: + explicit InstrumentedMutexLock(InstrumentedMutex* mutex) : mutex_(mutex) { + mutex_->Lock(); + } + + ~InstrumentedMutexLock() { mutex_->Unlock(); } + + private: + InstrumentedMutex* const mutex_; + InstrumentedMutexLock(const InstrumentedMutexLock&) = delete; + void operator=(const InstrumentedMutexLock&) = delete; +}; + +// RAII wrapper for temporary releasing InstrumentedMutex inside +// InstrumentedMutexLock +class InstrumentedMutexUnlock { + public: + explicit InstrumentedMutexUnlock(InstrumentedMutex* mutex) : mutex_(mutex) { + mutex_->Unlock(); + } + + ~InstrumentedMutexUnlock() { mutex_->Lock(); } + + private: + InstrumentedMutex* const mutex_; + InstrumentedMutexUnlock(const InstrumentedMutexUnlock&) = delete; + void operator=(const InstrumentedMutexUnlock&) = delete; +}; + +class InstrumentedCondVar { + public: + explicit InstrumentedCondVar(InstrumentedMutex* instrumented_mutex) + : cond_(&(instrumented_mutex->mutex_)), + stats_(instrumented_mutex->stats_), + clock_(instrumented_mutex->clock_), + stats_code_(instrumented_mutex->stats_code_) {} + + void Wait(); + + bool TimedWait(uint64_t abs_time_us); + + void Signal() { cond_.Signal(); } + + void SignalAll() { cond_.SignalAll(); } + + private: + void WaitInternal(); + bool TimedWaitInternal(uint64_t abs_time_us); + port::CondVar cond_; + Statistics* stats_; + SystemClock* clock_; + int stats_code_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/iostats_context.cc b/librocksdb-sys/rocksdb/monitoring/iostats_context.cc new file mode 100644 index 0000000..04e9891 --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/iostats_context.cc @@ -0,0 +1,78 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include + +#include "monitoring/iostats_context_imp.h" +#include "rocksdb/env.h" + +namespace ROCKSDB_NAMESPACE { + +#ifdef NIOSTATS_CONTEXT +// Should not be used because the counters are not thread-safe. +// Put here just to make get_iostats_context() simple without ifdef. +static IOStatsContext iostats_context; +#else +thread_local IOStatsContext iostats_context; +#endif + +IOStatsContext* get_iostats_context() { return &iostats_context; } + +void IOStatsContext::Reset() { +#ifndef NIOSTATS_CONTEXT + thread_pool_id = Env::Priority::TOTAL; + bytes_read = 0; + bytes_written = 0; + open_nanos = 0; + allocate_nanos = 0; + write_nanos = 0; + read_nanos = 0; + range_sync_nanos = 0; + prepare_write_nanos = 0; + fsync_nanos = 0; + logger_nanos = 0; + cpu_write_nanos = 0; + cpu_read_nanos = 0; + file_io_stats_by_temperature.Reset(); +#endif //! NIOSTATS_CONTEXT +} + +#define IOSTATS_CONTEXT_OUTPUT(counter) \ + if (!exclude_zero_counters || counter > 0) { \ + ss << #counter << " = " << counter << ", "; \ + } + +std::string IOStatsContext::ToString(bool exclude_zero_counters) const { +#ifdef NIOSTATS_CONTEXT + (void)exclude_zero_counters; + return ""; +#else + std::ostringstream ss; + IOSTATS_CONTEXT_OUTPUT(thread_pool_id); + IOSTATS_CONTEXT_OUTPUT(bytes_read); + IOSTATS_CONTEXT_OUTPUT(bytes_written); + IOSTATS_CONTEXT_OUTPUT(open_nanos); + IOSTATS_CONTEXT_OUTPUT(allocate_nanos); + IOSTATS_CONTEXT_OUTPUT(write_nanos); + IOSTATS_CONTEXT_OUTPUT(read_nanos); + IOSTATS_CONTEXT_OUTPUT(range_sync_nanos); + IOSTATS_CONTEXT_OUTPUT(fsync_nanos); + IOSTATS_CONTEXT_OUTPUT(prepare_write_nanos); + IOSTATS_CONTEXT_OUTPUT(logger_nanos); + IOSTATS_CONTEXT_OUTPUT(cpu_write_nanos); + IOSTATS_CONTEXT_OUTPUT(cpu_read_nanos); + IOSTATS_CONTEXT_OUTPUT(file_io_stats_by_temperature.hot_file_bytes_read); + IOSTATS_CONTEXT_OUTPUT(file_io_stats_by_temperature.warm_file_bytes_read); + IOSTATS_CONTEXT_OUTPUT(file_io_stats_by_temperature.cold_file_bytes_read); + IOSTATS_CONTEXT_OUTPUT(file_io_stats_by_temperature.hot_file_read_count); + IOSTATS_CONTEXT_OUTPUT(file_io_stats_by_temperature.warm_file_read_count); + IOSTATS_CONTEXT_OUTPUT(file_io_stats_by_temperature.cold_file_read_count); + std::string str = ss.str(); + str.erase(str.find_last_not_of(", ") + 1); + return str; +#endif //! NIOSTATS_CONTEXT +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/iostats_context_imp.h b/librocksdb-sys/rocksdb/monitoring/iostats_context_imp.h new file mode 100644 index 0000000..a0b4292 --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/iostats_context_imp.h @@ -0,0 +1,62 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#pragma once +#include "monitoring/perf_step_timer.h" +#include "rocksdb/iostats_context.h" + +#if !defined(NIOSTATS_CONTEXT) +namespace ROCKSDB_NAMESPACE { +extern thread_local IOStatsContext iostats_context; +} // namespace ROCKSDB_NAMESPACE + +// increment a specific counter by the specified value +#define IOSTATS_ADD(metric, value) \ + if (!iostats_context.disable_iostats) { \ + iostats_context.metric += value; \ + } + +// reset a specific counter to zero +#define IOSTATS_RESET(metric) (iostats_context.metric = 0) + +// reset all counters to zero +#define IOSTATS_RESET_ALL() (iostats_context.Reset()) + +#define IOSTATS_SET_THREAD_POOL_ID(value) \ + (iostats_context.thread_pool_id = value) + +#define IOSTATS_THREAD_POOL_ID() (iostats_context.thread_pool_id) + +#define IOSTATS(metric) (iostats_context.metric) + +// Declare and set start time of the timer +#define IOSTATS_TIMER_GUARD(metric) \ + PerfStepTimer iostats_step_timer_##metric(&(iostats_context.metric)); \ + iostats_step_timer_##metric.Start(); + +// Declare and set start time of the timer +#define IOSTATS_CPU_TIMER_GUARD(metric, clock) \ + PerfStepTimer iostats_step_timer_##metric( \ + &(iostats_context.metric), clock, true, \ + PerfLevel::kEnableTimeAndCPUTimeExceptForMutex); \ + iostats_step_timer_##metric.Start(); + +#define IOSTATS_SET_DISABLE(disable) (iostats_context.disable_iostats = disable) + +#else // !NIOSTATS_CONTEXT + +#define IOSTATS_ADD(metric, value) +#define IOSTATS_ADD_IF_POSITIVE(metric, value) +#define IOSTATS_RESET(metric) +#define IOSTATS_RESET_ALL() +#define IOSTATS_SET_THREAD_POOL_ID(value) +#define IOSTATS_THREAD_POOL_ID() +#define IOSTATS(metric) 0 +#define IOSTATS_SET_DISABLE(disable) + +#define IOSTATS_TIMER_GUARD(metric) +#define IOSTATS_CPU_TIMER_GUARD(metric, clock) static_cast(clock) + +#endif // !NIOSTATS_CONTEXT diff --git a/librocksdb-sys/rocksdb/monitoring/iostats_context_test.cc b/librocksdb-sys/rocksdb/monitoring/iostats_context_test.cc new file mode 100644 index 0000000..5fce334 --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/iostats_context_test.cc @@ -0,0 +1,31 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "rocksdb/iostats_context.h" + +#include "test_util/testharness.h" + +namespace ROCKSDB_NAMESPACE { + +TEST(IOStatsContextTest, ToString) { + get_iostats_context()->Reset(); + get_iostats_context()->bytes_read = 12345; + + std::string zero_included = get_iostats_context()->ToString(); + ASSERT_NE(std::string::npos, zero_included.find("= 0")); + ASSERT_NE(std::string::npos, zero_included.find("= 12345")); + + std::string zero_excluded = get_iostats_context()->ToString(true); + ASSERT_EQ(std::string::npos, zero_excluded.find("= 0")); + ASSERT_NE(std::string::npos, zero_excluded.find("= 12345")); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/monitoring/perf_context.cc b/librocksdb-sys/rocksdb/monitoring/perf_context.cc new file mode 100644 index 0000000..6839e90 --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/perf_context.cc @@ -0,0 +1,313 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#include + +#include "monitoring/perf_context_imp.h" + +namespace ROCKSDB_NAMESPACE { + +/* + * Please add new metrics to this macro and appropriate fields will be copied, + * and/or emitted when converted to string. + * When people need to add new metrics please add the metric to the macro below + * and enclose the name of the specific metric within defCmd(). + * The position of the field will be dictated by the + * order in which the macros are enumerated and the offsets of the fields will + * be matched against ''PerfContextByLevelBase'' declared in perf_context.h. + */ +// clang-format off +#define DEF_PERF_CONTEXT_LEVEL_METRICS(defCmd) \ + defCmd(bloom_filter_useful) \ + defCmd(bloom_filter_full_positive) \ + defCmd(bloom_filter_full_true_positive) \ + defCmd(user_key_return_count) \ + defCmd(get_from_table_nanos) \ + defCmd(block_cache_hit_count) \ + defCmd(block_cache_miss_count) +// clang-format on + +// Break down performance counters by level and store per-level perf context in +// PerfContextByLevel +struct PerfContextByLevelInt { +#define EMIT_FIELDS(x) uint64_t x = 0; + DEF_PERF_CONTEXT_LEVEL_METRICS(EMIT_FIELDS) +#undef EMIT_FIELDS +}; + +/* + * Please add new metrics to this macro and appropriate fields will be copied, + * and/or emitted when converted to string. + * When people need to add new metrics please enclose the name of the specific + * metric within defCmd(). The position of the field will be dictated by the + * order in which the macros are enumerated and the offsets of the fields will + * be matched against ''PerfContextBase'' declared in perf_context.h. + */ + +// clang-format off +#define DEF_PERF_CONTEXT_METRICS(defCmd) \ + defCmd(user_key_comparison_count) \ + defCmd(block_cache_hit_count) \ + defCmd(block_read_count) \ + defCmd(block_read_byte) \ + defCmd(block_read_time) \ + defCmd(block_read_cpu_time) \ + defCmd(block_cache_index_hit_count) \ + defCmd(block_cache_standalone_handle_count) \ + defCmd(block_cache_real_handle_count) \ + defCmd(index_block_read_count) \ + defCmd(block_cache_filter_hit_count) \ + defCmd(filter_block_read_count) \ + defCmd(compression_dict_block_read_count) \ + defCmd(secondary_cache_hit_count) \ + defCmd(compressed_sec_cache_insert_real_count) \ + defCmd(compressed_sec_cache_insert_dummy_count) \ + defCmd(compressed_sec_cache_uncompressed_bytes) \ + defCmd(compressed_sec_cache_compressed_bytes) \ + defCmd(block_checksum_time) \ + defCmd(block_decompress_time) \ + defCmd(get_read_bytes) \ + defCmd(multiget_read_bytes) \ + defCmd(iter_read_bytes) \ + defCmd(blob_cache_hit_count) \ + defCmd(blob_read_count) \ + defCmd(blob_read_byte) \ + defCmd(blob_read_time) \ + defCmd(blob_checksum_time) \ + defCmd(blob_decompress_time) \ + defCmd(internal_key_skipped_count) \ + defCmd(internal_delete_skipped_count) \ + defCmd(internal_recent_skipped_count) \ + defCmd(internal_merge_count) \ + defCmd(internal_merge_point_lookup_count) \ + defCmd(internal_range_del_reseek_count) \ + defCmd(get_snapshot_time) \ + defCmd(get_from_memtable_time) \ + defCmd(get_from_memtable_count) \ + defCmd(get_post_process_time) \ + defCmd(get_from_output_files_time) \ + defCmd(seek_on_memtable_time) \ + defCmd(seek_on_memtable_count) \ + defCmd(next_on_memtable_count) \ + defCmd(prev_on_memtable_count) \ + defCmd(seek_child_seek_time) \ + defCmd(seek_child_seek_count) \ + defCmd(seek_min_heap_time) \ + defCmd(seek_max_heap_time) \ + defCmd(seek_internal_seek_time) \ + defCmd(find_next_user_entry_time) \ + defCmd(write_wal_time) \ + defCmd(write_memtable_time) \ + defCmd(write_delay_time) \ + defCmd(write_scheduling_flushes_compactions_time)\ + defCmd(write_pre_and_post_process_time) \ + defCmd(write_thread_wait_nanos) \ + defCmd(db_mutex_lock_nanos) \ + defCmd(db_condition_wait_nanos) \ + defCmd(merge_operator_time_nanos) \ + defCmd(read_index_block_nanos) \ + defCmd(read_filter_block_nanos) \ + defCmd(new_table_block_iter_nanos) \ + defCmd(new_table_iterator_nanos) \ + defCmd(block_seek_nanos) \ + defCmd(find_table_nanos) \ + defCmd(bloom_memtable_hit_count) \ + defCmd(bloom_memtable_miss_count) \ + defCmd(bloom_sst_hit_count) \ + defCmd(bloom_sst_miss_count) \ + defCmd(key_lock_wait_time) \ + defCmd(key_lock_wait_count) \ + defCmd(env_new_sequential_file_nanos) \ + defCmd(env_new_random_access_file_nanos) \ + defCmd(env_new_writable_file_nanos) \ + defCmd(env_reuse_writable_file_nanos) \ + defCmd(env_new_random_rw_file_nanos) \ + defCmd(env_new_directory_nanos) \ + defCmd(env_file_exists_nanos) \ + defCmd(env_get_children_nanos) \ + defCmd(env_get_children_file_attributes_nanos) \ + defCmd(env_delete_file_nanos) \ + defCmd(env_create_dir_nanos) \ + defCmd(env_create_dir_if_missing_nanos) \ + defCmd(env_delete_dir_nanos) \ + defCmd(env_get_file_size_nanos) \ + defCmd(env_get_file_modification_time_nanos) \ + defCmd(env_rename_file_nanos) \ + defCmd(env_link_file_nanos) \ + defCmd(env_lock_file_nanos) \ + defCmd(env_unlock_file_nanos) \ + defCmd(env_new_logger_nanos) \ + defCmd(get_cpu_nanos) \ + defCmd(iter_next_cpu_nanos) \ + defCmd(iter_prev_cpu_nanos) \ + defCmd(iter_seek_cpu_nanos) \ + defCmd(iter_next_count) \ + defCmd(iter_prev_count) \ + defCmd(iter_seek_count) \ + defCmd(encrypt_data_nanos) \ + defCmd(decrypt_data_nanos) \ + defCmd(number_async_seek) +// clang-format on + +struct PerfContextInt { +#define EMIT_FIELDS(x) uint64_t x; + DEF_PERF_CONTEXT_METRICS(EMIT_FIELDS) +#undef EMIT_FIELDS +}; + +#if defined(NPERF_CONTEXT) +// Should not be used because the counters are not thread-safe. +// Put here just to make get_perf_context() simple without ifdef. +PerfContext perf_context; +#else +thread_local PerfContext perf_context; +#endif + +PerfContext* get_perf_context() { + static_assert(sizeof(PerfContextBase) == sizeof(PerfContextInt)); + static_assert(sizeof(PerfContextByLevelBase) == + sizeof(PerfContextByLevelInt)); + /* + * Validate that we have the same fields and offsets between the external user + * facing + * ''PerfContextBase'' and ''PerfContextByLevelBase' structures with the + * internal structures that we generate from the DEF_* macros above. This way + * if people add metrics to the user-facing header file, they will have a + * build failure and need to add similar fields to the macros in this file. + * These are compile-time validations and don't impose any run-time penalties. + */ +#define EMIT_OFFSET_ASSERTION(x) \ + static_assert(offsetof(PerfContextBase, x) == offsetof(PerfContextInt, x)); + DEF_PERF_CONTEXT_METRICS(EMIT_OFFSET_ASSERTION) +#undef EMIT_OFFSET_ASSERTION +#define EMIT_OFFSET_ASSERTION(x) \ + static_assert(offsetof(PerfContextByLevelBase, x) == \ + offsetof(PerfContextByLevelInt, x)); + DEF_PERF_CONTEXT_LEVEL_METRICS(EMIT_OFFSET_ASSERTION) +#undef EMIT_OFFSET_ASSERTION + return &perf_context; +} + +PerfContext::~PerfContext() { +#if !defined(NPERF_CONTEXT) && !defined(OS_SOLARIS) + ClearPerLevelPerfContext(); +#endif +} + +PerfContext::PerfContext(const PerfContext& other) { +#ifdef NPERF_CONTEXT + (void)other; +#else + copyMetrics(&other); +#endif +} + +PerfContext::PerfContext(PerfContext&& other) noexcept { +#ifdef NPERF_CONTEXT + (void)other; +#else + copyMetrics(&other); +#endif +} + +PerfContext& PerfContext::operator=(const PerfContext& other) { +#ifdef NPERF_CONTEXT + (void)other; +#else + copyMetrics(&other); +#endif + return *this; +} + +void PerfContext::copyMetrics(const PerfContext* other) noexcept { +#ifdef NPERF_CONTEXT + (void)other; +#else +#define EMIT_COPY_FIELDS(x) x = other->x; + DEF_PERF_CONTEXT_METRICS(EMIT_COPY_FIELDS) +#undef EMIT_COPY_FIELDS + if (per_level_perf_context_enabled && level_to_perf_context != nullptr) { + ClearPerLevelPerfContext(); + } + if (other->level_to_perf_context != nullptr) { + level_to_perf_context = new std::map(); + *level_to_perf_context = *other->level_to_perf_context; + } + per_level_perf_context_enabled = other->per_level_perf_context_enabled; +#endif +} + +void PerfContext::Reset() { +#ifndef NPERF_CONTEXT +#define EMIT_FIELDS(x) x = 0; + DEF_PERF_CONTEXT_METRICS(EMIT_FIELDS) +#undef EMIT_FIELDS + if (per_level_perf_context_enabled && level_to_perf_context) { + for (auto& kv : *level_to_perf_context) { + kv.second.Reset(); + } + } +#endif +} + +void PerfContextByLevel::Reset() { +#ifndef NPERF_CONTEXT +#define EMIT_FIELDS(x) x = 0; + DEF_PERF_CONTEXT_LEVEL_METRICS(EMIT_FIELDS) +#undef EMIT_FIELDS +#endif +} + +std::string PerfContext::ToString(bool exclude_zero_counters) const { +#ifdef NPERF_CONTEXT + (void)exclude_zero_counters; + return ""; +#else + std::ostringstream ss; +#define PERF_CONTEXT_OUTPUT(counter) \ + if (!exclude_zero_counters || (counter > 0)) { \ + ss << #counter << " = " << counter << ", "; \ + } + DEF_PERF_CONTEXT_METRICS(PERF_CONTEXT_OUTPUT) +#undef PERF_CONTEXT_OUTPUT + if (per_level_perf_context_enabled && level_to_perf_context) { +#define PERF_CONTEXT_BY_LEVEL_OUTPUT_ONE_COUNTER(counter) \ + ss << #counter << " = "; \ + for (auto& kv : *level_to_perf_context) { \ + if (!exclude_zero_counters || (kv.second.counter > 0)) { \ + ss << kv.second.counter << "@level" << kv.first << ", "; \ + } \ + } + DEF_PERF_CONTEXT_LEVEL_METRICS(PERF_CONTEXT_BY_LEVEL_OUTPUT_ONE_COUNTER) +#undef PERF_CONTEXT_BY_LEVEL_OUTPUT_ONE_COUNTER + } + std::string str = ss.str(); + str.erase(str.find_last_not_of(", ") + 1); + return str; +#endif +} + +void PerfContext::EnablePerLevelPerfContext() { + if (level_to_perf_context == nullptr) { + level_to_perf_context = new std::map(); + } + per_level_perf_context_enabled = true; +} + +void PerfContext::DisablePerLevelPerfContext() { + per_level_perf_context_enabled = false; +} + +void PerfContext::ClearPerLevelPerfContext() { + if (level_to_perf_context != nullptr) { + level_to_perf_context->clear(); + delete level_to_perf_context; + level_to_perf_context = nullptr; + } + per_level_perf_context_enabled = false; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/perf_context_imp.h b/librocksdb-sys/rocksdb/monitoring/perf_context_imp.h new file mode 100644 index 0000000..5b66ff2 --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/perf_context_imp.h @@ -0,0 +1,96 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#pragma once +#include "monitoring/perf_step_timer.h" +#include "rocksdb/perf_context.h" +#include "util/stop_watch.h" + +namespace ROCKSDB_NAMESPACE { +#if defined(NPERF_CONTEXT) +extern PerfContext perf_context; +#else +#if defined(OS_SOLARIS) +extern thread_local PerfContext perf_context_; +#define perf_context (*get_perf_context()) +#else +extern thread_local PerfContext perf_context; +#endif +#endif + +#if defined(NPERF_CONTEXT) + +#define PERF_TIMER_STOP(metric) +#define PERF_TIMER_START(metric) +#define PERF_TIMER_GUARD(metric) +#define PERF_TIMER_GUARD_WITH_CLOCK(metric, clock) +#define PERF_CPU_TIMER_GUARD(metric, clock) +#define PERF_CONDITIONAL_TIMER_FOR_MUTEX_GUARD(metric, condition, stats, \ + ticker_type) +#define PERF_TIMER_MEASURE(metric) +#define PERF_COUNTER_ADD(metric, value) +#define PERF_COUNTER_BY_LEVEL_ADD(metric, value, level) + +#else + +// Stop the timer and update the metric +#define PERF_TIMER_STOP(metric) perf_step_timer_##metric.Stop(); + +#define PERF_TIMER_START(metric) perf_step_timer_##metric.Start(); + +// Declare and set start time of the timer +#define PERF_TIMER_GUARD(metric) \ + PerfStepTimer perf_step_timer_##metric(&(perf_context.metric)); \ + perf_step_timer_##metric.Start(); + +// Declare and set start time of the timer +#define PERF_TIMER_GUARD_WITH_CLOCK(metric, clock) \ + PerfStepTimer perf_step_timer_##metric(&(perf_context.metric), clock); \ + perf_step_timer_##metric.Start(); + +// Declare and set start time of the timer +#define PERF_CPU_TIMER_GUARD(metric, clock) \ + PerfStepTimer perf_step_timer_##metric( \ + &(perf_context.metric), clock, true, \ + PerfLevel::kEnableTimeAndCPUTimeExceptForMutex); \ + perf_step_timer_##metric.Start(); + +#define PERF_CONDITIONAL_TIMER_FOR_MUTEX_GUARD(metric, condition, stats, \ + ticker_type) \ + PerfStepTimer perf_step_timer_##metric(&(perf_context.metric), nullptr, \ + false, PerfLevel::kEnableTime, stats, \ + ticker_type); \ + if (condition) { \ + perf_step_timer_##metric.Start(); \ + } + +// Update metric with time elapsed since last START. start time is reset +// to current timestamp. +#define PERF_TIMER_MEASURE(metric) perf_step_timer_##metric.Measure(); + +// Increase metric value +#define PERF_COUNTER_ADD(metric, value) \ + if (perf_level >= PerfLevel::kEnableCount) { \ + perf_context.metric += value; \ + } + +// Increase metric value +#define PERF_COUNTER_BY_LEVEL_ADD(metric, value, level) \ + if (perf_level >= PerfLevel::kEnableCount && \ + perf_context.per_level_perf_context_enabled && \ + perf_context.level_to_perf_context) { \ + if ((*(perf_context.level_to_perf_context)).find(level) != \ + (*(perf_context.level_to_perf_context)).end()) { \ + (*(perf_context.level_to_perf_context))[level].metric += value; \ + } else { \ + PerfContextByLevel empty_context; \ + (*(perf_context.level_to_perf_context))[level] = empty_context; \ + (*(perf_context.level_to_perf_context))[level].metric += value; \ + } \ + } + +#endif + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/perf_level.cc b/librocksdb-sys/rocksdb/monitoring/perf_level.cc new file mode 100644 index 0000000..e350762 --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/perf_level.cc @@ -0,0 +1,23 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#include + +#include "monitoring/perf_level_imp.h" + +namespace ROCKSDB_NAMESPACE { + +thread_local PerfLevel perf_level = kEnableCount; + +void SetPerfLevel(PerfLevel level) { + assert(level > kUninitialized); + assert(level < kOutOfBounds); + perf_level = level; +} + +PerfLevel GetPerfLevel() { return perf_level; } + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/perf_level_imp.h b/librocksdb-sys/rocksdb/monitoring/perf_level_imp.h new file mode 100644 index 0000000..28bd185 --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/perf_level_imp.h @@ -0,0 +1,14 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#pragma once +#include "port/port.h" +#include "rocksdb/perf_level.h" + +namespace ROCKSDB_NAMESPACE { + +extern thread_local PerfLevel perf_level; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/perf_step_timer.h b/librocksdb-sys/rocksdb/monitoring/perf_step_timer.h new file mode 100644 index 0000000..f6c45d7 --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/perf_step_timer.h @@ -0,0 +1,77 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#pragma once +#include "monitoring/perf_level_imp.h" +#include "monitoring/statistics_impl.h" +#include "rocksdb/system_clock.h" + +namespace ROCKSDB_NAMESPACE { + +class PerfStepTimer { + public: + explicit PerfStepTimer( + uint64_t* metric, SystemClock* clock = nullptr, bool use_cpu_time = false, + PerfLevel enable_level = PerfLevel::kEnableTimeExceptForMutex, + Statistics* statistics = nullptr, uint32_t ticker_type = 0) + : perf_counter_enabled_(perf_level >= enable_level), + use_cpu_time_(use_cpu_time), + ticker_type_(ticker_type), + clock_((perf_counter_enabled_ || statistics != nullptr) + ? (clock ? clock : SystemClock::Default().get()) + : nullptr), + start_(0), + metric_(metric), + statistics_(statistics) {} + + ~PerfStepTimer() { Stop(); } + + void Start() { + if (perf_counter_enabled_ || statistics_ != nullptr) { + start_ = time_now(); + } + } + + void Measure() { + if (start_) { + uint64_t now = time_now(); + *metric_ += now - start_; + start_ = now; + } + } + + void Stop() { + if (start_) { + uint64_t duration = time_now() - start_; + if (perf_counter_enabled_) { + *metric_ += duration; + } + + if (statistics_ != nullptr) { + RecordTick(statistics_, ticker_type_, duration); + } + start_ = 0; + } + } + + private: + uint64_t time_now() { + if (!use_cpu_time_) { + return clock_->NowNanos(); + } else { + return clock_->CPUNanos(); + } + } + + const bool perf_counter_enabled_; + const bool use_cpu_time_; + uint32_t ticker_type_; + SystemClock* const clock_; + uint64_t start_; + uint64_t* metric_; + Statistics* statistics_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/persistent_stats_history.cc b/librocksdb-sys/rocksdb/monitoring/persistent_stats_history.cc new file mode 100644 index 0000000..f4c0221 --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/persistent_stats_history.cc @@ -0,0 +1,170 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "monitoring/persistent_stats_history.h" + +#include +#include +#include + +#include "db/db_impl/db_impl.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { +// 10 digit seconds timestamp => [Sep 9, 2001 ~ Nov 20, 2286] +const int kNowSecondsStringLength = 10; +const std::string kFormatVersionKeyString = + "__persistent_stats_format_version__"; +const std::string kCompatibleVersionKeyString = + "__persistent_stats_compatible_version__"; +// Every release maintains two versions numbers for persistents stats: Current +// format version and compatible format version. Current format version +// designates what type of encoding will be used when writing to stats CF; +// compatible format version designates the minimum format version that +// can decode the stats CF encoded using the current format version. +const uint64_t kStatsCFCurrentFormatVersion = 1; +const uint64_t kStatsCFCompatibleFormatVersion = 1; + +Status DecodePersistentStatsVersionNumber(DBImpl* db, StatsVersionKeyType type, + uint64_t* version_number) { + if (type >= StatsVersionKeyType::kKeyTypeMax) { + return Status::InvalidArgument("Invalid stats version key type provided"); + } + std::string key; + if (type == StatsVersionKeyType::kFormatVersion) { + key = kFormatVersionKeyString; + } else if (type == StatsVersionKeyType::kCompatibleVersion) { + key = kCompatibleVersionKeyString; + } + ReadOptions options; + options.verify_checksums = true; + std::string result; + Status s = db->Get(options, db->PersistentStatsColumnFamily(), key, &result); + if (!s.ok() || result.empty()) { + return Status::NotFound("Persistent stats version key " + key + + " not found."); + } + + // read version_number but do nothing in current version + *version_number = ParseUint64(result); + return Status::OK(); +} + +int EncodePersistentStatsKey(uint64_t now_seconds, const std::string& key, + int size, char* buf) { + char timestamp[kNowSecondsStringLength + 1]; + // make time stamp string equal in length to allow sorting by time + snprintf(timestamp, sizeof(timestamp), "%010d", + static_cast(now_seconds)); + timestamp[kNowSecondsStringLength] = '\0'; + return snprintf(buf, size, "%s#%s", timestamp, key.c_str()); +} + +void OptimizeForPersistentStats(ColumnFamilyOptions* cfo) { + cfo->write_buffer_size = 2 << 20; + cfo->target_file_size_base = 2 * 1048576; + cfo->max_bytes_for_level_base = 10 * 1048576; + cfo->soft_pending_compaction_bytes_limit = 256 * 1048576; + cfo->hard_pending_compaction_bytes_limit = 1073741824ul; + cfo->compression = kNoCompression; +} + +PersistentStatsHistoryIterator::~PersistentStatsHistoryIterator() {} + +bool PersistentStatsHistoryIterator::Valid() const { return valid_; } + +Status PersistentStatsHistoryIterator::status() const { return status_; } + +void PersistentStatsHistoryIterator::Next() { + // increment start_time by 1 to avoid infinite loop + AdvanceIteratorByTime(GetStatsTime() + 1, end_time_); +} + +uint64_t PersistentStatsHistoryIterator::GetStatsTime() const { return time_; } + +const std::map& +PersistentStatsHistoryIterator::GetStatsMap() const { + return stats_map_; +} + +std::pair parseKey(const Slice& key, + uint64_t start_time) { + std::pair result; + std::string key_str = key.ToString(); + std::string::size_type pos = key_str.find("#"); + // TODO(Zhongyi): add counters to track parse failures? + if (pos == std::string::npos) { + result.first = std::numeric_limits::max(); + result.second.clear(); + } else { + uint64_t parsed_time = ParseUint64(key_str.substr(0, pos)); + // skip entries with timestamp smaller than start_time + if (parsed_time < start_time) { + result.first = std::numeric_limits::max(); + result.second = ""; + } else { + result.first = parsed_time; + std::string key_resize = key_str.substr(pos + 1); + result.second = key_resize; + } + } + return result; +} + +// advance the iterator to the next time between [start_time, end_time) +// if success, update time_ and stats_map_ with new_time and stats_map +void PersistentStatsHistoryIterator::AdvanceIteratorByTime(uint64_t start_time, + uint64_t end_time) { + // try to find next entry in stats_history_ map + if (db_impl_ != nullptr) { + ReadOptions ro; + Iterator* iter = + db_impl_->NewIterator(ro, db_impl_->PersistentStatsColumnFamily()); + + char timestamp[kNowSecondsStringLength + 1]; + snprintf(timestamp, sizeof(timestamp), "%010d", + static_cast(std::max(time_, start_time))); + timestamp[kNowSecondsStringLength] = '\0'; + + iter->Seek(timestamp); + // no more entries with timestamp >= start_time is found or version key + // is found to be incompatible + if (!iter->Valid()) { + valid_ = false; + delete iter; + return; + } + time_ = parseKey(iter->key(), start_time).first; + valid_ = true; + // check parsed time and invalid if it exceeds end_time + if (time_ > end_time) { + valid_ = false; + delete iter; + return; + } + // find all entries with timestamp equal to time_ + std::map new_stats_map; + std::pair kv; + for (; iter->Valid(); iter->Next()) { + kv = parseKey(iter->key(), start_time); + if (kv.first != time_) { + break; + } + if (kv.second.compare(kFormatVersionKeyString) == 0) { + continue; + } + new_stats_map[kv.second] = ParseUint64(iter->value().ToString()); + } + stats_map_.swap(new_stats_map); + delete iter; + } else { + valid_ = false; + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/persistent_stats_history.h b/librocksdb-sys/rocksdb/monitoring/persistent_stats_history.h new file mode 100644 index 0000000..7c711fe --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/persistent_stats_history.h @@ -0,0 +1,83 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include "db/db_impl/db_impl.h" +#include "rocksdb/stats_history.h" + +namespace ROCKSDB_NAMESPACE { + +extern const std::string kFormatVersionKeyString; +extern const std::string kCompatibleVersionKeyString; +extern const uint64_t kStatsCFCurrentFormatVersion; +extern const uint64_t kStatsCFCompatibleFormatVersion; + +enum StatsVersionKeyType : uint32_t { + kFormatVersion = 1, + kCompatibleVersion = 2, + kKeyTypeMax = 3 +}; + +// Read the version number from persitent stats cf depending on type provided +// stores the version number in `*version_number` +// returns Status::OK() on success, or other status code on failure +Status DecodePersistentStatsVersionNumber(DBImpl* db, StatsVersionKeyType type, + uint64_t* version_number); + +// Encode timestamp and stats key into buf +// Format: timestamp(10 digit) + '#' + key +// Total length of encoded key will be capped at 100 bytes +int EncodePersistentStatsKey(uint64_t timestamp, const std::string& key, + int size, char* buf); + +void OptimizeForPersistentStats(ColumnFamilyOptions* cfo); + +class PersistentStatsHistoryIterator final : public StatsHistoryIterator { + public: + PersistentStatsHistoryIterator(uint64_t start_time, uint64_t end_time, + DBImpl* db_impl) + : time_(0), + start_time_(start_time), + end_time_(end_time), + valid_(true), + db_impl_(db_impl) { + AdvanceIteratorByTime(start_time_, end_time_); + } + ~PersistentStatsHistoryIterator() override; + bool Valid() const override; + Status status() const override; + + void Next() override; + uint64_t GetStatsTime() const override; + + const std::map& GetStatsMap() const override; + + private: + // advance the iterator to the next stats history record with timestamp + // between [start_time, end_time) + void AdvanceIteratorByTime(uint64_t start_time, uint64_t end_time); + + // No copying allowed + PersistentStatsHistoryIterator(const PersistentStatsHistoryIterator&) = + delete; + void operator=(const PersistentStatsHistoryIterator&) = delete; + PersistentStatsHistoryIterator(PersistentStatsHistoryIterator&&) = delete; + PersistentStatsHistoryIterator& operator=(PersistentStatsHistoryIterator&&) = + delete; + + uint64_t time_; + uint64_t start_time_; + uint64_t end_time_; + std::map stats_map_; + Status status_; + bool valid_; + DBImpl* db_impl_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/statistics.cc b/librocksdb-sys/rocksdb/monitoring/statistics.cc new file mode 100644 index 0000000..e4a18f9 --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/statistics.cc @@ -0,0 +1,541 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#include "rocksdb/statistics.h" + +#include +#include +#include + +#include "monitoring/statistics_impl.h" +#include "rocksdb/convenience.h" +#include "rocksdb/utilities/customizable_util.h" +#include "rocksdb/utilities/options_type.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +// The order of items listed in Tickers should be the same as +// the order listed in TickersNameMap +const std::vector> TickersNameMap = { + {BLOCK_CACHE_MISS, "rocksdb.block.cache.miss"}, + {BLOCK_CACHE_HIT, "rocksdb.block.cache.hit"}, + {BLOCK_CACHE_ADD, "rocksdb.block.cache.add"}, + {BLOCK_CACHE_ADD_FAILURES, "rocksdb.block.cache.add.failures"}, + {BLOCK_CACHE_INDEX_MISS, "rocksdb.block.cache.index.miss"}, + {BLOCK_CACHE_INDEX_HIT, "rocksdb.block.cache.index.hit"}, + {BLOCK_CACHE_INDEX_ADD, "rocksdb.block.cache.index.add"}, + {BLOCK_CACHE_INDEX_BYTES_INSERT, "rocksdb.block.cache.index.bytes.insert"}, + {BLOCK_CACHE_FILTER_MISS, "rocksdb.block.cache.filter.miss"}, + {BLOCK_CACHE_FILTER_HIT, "rocksdb.block.cache.filter.hit"}, + {BLOCK_CACHE_FILTER_ADD, "rocksdb.block.cache.filter.add"}, + {BLOCK_CACHE_FILTER_BYTES_INSERT, + "rocksdb.block.cache.filter.bytes.insert"}, + {BLOCK_CACHE_DATA_MISS, "rocksdb.block.cache.data.miss"}, + {BLOCK_CACHE_DATA_HIT, "rocksdb.block.cache.data.hit"}, + {BLOCK_CACHE_DATA_ADD, "rocksdb.block.cache.data.add"}, + {BLOCK_CACHE_DATA_BYTES_INSERT, "rocksdb.block.cache.data.bytes.insert"}, + {BLOCK_CACHE_BYTES_READ, "rocksdb.block.cache.bytes.read"}, + {BLOCK_CACHE_BYTES_WRITE, "rocksdb.block.cache.bytes.write"}, + {BLOOM_FILTER_USEFUL, "rocksdb.bloom.filter.useful"}, + {BLOOM_FILTER_FULL_POSITIVE, "rocksdb.bloom.filter.full.positive"}, + {BLOOM_FILTER_FULL_TRUE_POSITIVE, + "rocksdb.bloom.filter.full.true.positive"}, + {PERSISTENT_CACHE_HIT, "rocksdb.persistent.cache.hit"}, + {PERSISTENT_CACHE_MISS, "rocksdb.persistent.cache.miss"}, + {SIM_BLOCK_CACHE_HIT, "rocksdb.sim.block.cache.hit"}, + {SIM_BLOCK_CACHE_MISS, "rocksdb.sim.block.cache.miss"}, + {MEMTABLE_HIT, "rocksdb.memtable.hit"}, + {MEMTABLE_MISS, "rocksdb.memtable.miss"}, + {GET_HIT_L0, "rocksdb.l0.hit"}, + {GET_HIT_L1, "rocksdb.l1.hit"}, + {GET_HIT_L2_AND_UP, "rocksdb.l2andup.hit"}, + {COMPACTION_KEY_DROP_NEWER_ENTRY, "rocksdb.compaction.key.drop.new"}, + {COMPACTION_KEY_DROP_OBSOLETE, "rocksdb.compaction.key.drop.obsolete"}, + {COMPACTION_KEY_DROP_RANGE_DEL, "rocksdb.compaction.key.drop.range_del"}, + {COMPACTION_KEY_DROP_USER, "rocksdb.compaction.key.drop.user"}, + {COMPACTION_RANGE_DEL_DROP_OBSOLETE, + "rocksdb.compaction.range_del.drop.obsolete"}, + {COMPACTION_OPTIMIZED_DEL_DROP_OBSOLETE, + "rocksdb.compaction.optimized.del.drop.obsolete"}, + {COMPACTION_CANCELLED, "rocksdb.compaction.cancelled"}, + {NUMBER_KEYS_WRITTEN, "rocksdb.number.keys.written"}, + {NUMBER_KEYS_READ, "rocksdb.number.keys.read"}, + {NUMBER_KEYS_UPDATED, "rocksdb.number.keys.updated"}, + {BYTES_WRITTEN, "rocksdb.bytes.written"}, + {BYTES_READ, "rocksdb.bytes.read"}, + {NUMBER_DB_SEEK, "rocksdb.number.db.seek"}, + {NUMBER_DB_NEXT, "rocksdb.number.db.next"}, + {NUMBER_DB_PREV, "rocksdb.number.db.prev"}, + {NUMBER_DB_SEEK_FOUND, "rocksdb.number.db.seek.found"}, + {NUMBER_DB_NEXT_FOUND, "rocksdb.number.db.next.found"}, + {NUMBER_DB_PREV_FOUND, "rocksdb.number.db.prev.found"}, + {ITER_BYTES_READ, "rocksdb.db.iter.bytes.read"}, + {NO_FILE_OPENS, "rocksdb.no.file.opens"}, + {NO_FILE_ERRORS, "rocksdb.no.file.errors"}, + {STALL_MICROS, "rocksdb.stall.micros"}, + {DB_MUTEX_WAIT_MICROS, "rocksdb.db.mutex.wait.micros"}, + {NUMBER_MULTIGET_CALLS, "rocksdb.number.multiget.get"}, + {NUMBER_MULTIGET_KEYS_READ, "rocksdb.number.multiget.keys.read"}, + {NUMBER_MULTIGET_BYTES_READ, "rocksdb.number.multiget.bytes.read"}, + {NUMBER_MERGE_FAILURES, "rocksdb.number.merge.failures"}, + {BLOOM_FILTER_PREFIX_CHECKED, "rocksdb.bloom.filter.prefix.checked"}, + {BLOOM_FILTER_PREFIX_USEFUL, "rocksdb.bloom.filter.prefix.useful"}, + {BLOOM_FILTER_PREFIX_TRUE_POSITIVE, + "rocksdb.bloom.filter.prefix.true.positive"}, + {NUMBER_OF_RESEEKS_IN_ITERATION, "rocksdb.number.reseeks.iteration"}, + {GET_UPDATES_SINCE_CALLS, "rocksdb.getupdatessince.calls"}, + {WAL_FILE_SYNCED, "rocksdb.wal.synced"}, + {WAL_FILE_BYTES, "rocksdb.wal.bytes"}, + {WRITE_DONE_BY_SELF, "rocksdb.write.self"}, + {WRITE_DONE_BY_OTHER, "rocksdb.write.other"}, + {WRITE_WITH_WAL, "rocksdb.write.wal"}, + {COMPACT_READ_BYTES, "rocksdb.compact.read.bytes"}, + {COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes"}, + {FLUSH_WRITE_BYTES, "rocksdb.flush.write.bytes"}, + {COMPACT_READ_BYTES_MARKED, "rocksdb.compact.read.marked.bytes"}, + {COMPACT_READ_BYTES_PERIODIC, "rocksdb.compact.read.periodic.bytes"}, + {COMPACT_READ_BYTES_TTL, "rocksdb.compact.read.ttl.bytes"}, + {COMPACT_WRITE_BYTES_MARKED, "rocksdb.compact.write.marked.bytes"}, + {COMPACT_WRITE_BYTES_PERIODIC, "rocksdb.compact.write.periodic.bytes"}, + {COMPACT_WRITE_BYTES_TTL, "rocksdb.compact.write.ttl.bytes"}, + {NUMBER_DIRECT_LOAD_TABLE_PROPERTIES, + "rocksdb.number.direct.load.table.properties"}, + {NUMBER_SUPERVERSION_ACQUIRES, "rocksdb.number.superversion_acquires"}, + {NUMBER_SUPERVERSION_RELEASES, "rocksdb.number.superversion_releases"}, + {NUMBER_SUPERVERSION_CLEANUPS, "rocksdb.number.superversion_cleanups"}, + {NUMBER_BLOCK_COMPRESSED, "rocksdb.number.block.compressed"}, + {NUMBER_BLOCK_DECOMPRESSED, "rocksdb.number.block.decompressed"}, + {NUMBER_BLOCK_NOT_COMPRESSED, "rocksdb.number.block.not_compressed"}, + {MERGE_OPERATION_TOTAL_TIME, "rocksdb.merge.operation.time.nanos"}, + {FILTER_OPERATION_TOTAL_TIME, "rocksdb.filter.operation.time.nanos"}, + {ROW_CACHE_HIT, "rocksdb.row.cache.hit"}, + {ROW_CACHE_MISS, "rocksdb.row.cache.miss"}, + {READ_AMP_ESTIMATE_USEFUL_BYTES, "rocksdb.read.amp.estimate.useful.bytes"}, + {READ_AMP_TOTAL_READ_BYTES, "rocksdb.read.amp.total.read.bytes"}, + {NUMBER_RATE_LIMITER_DRAINS, "rocksdb.number.rate_limiter.drains"}, + {NUMBER_ITER_SKIP, "rocksdb.number.iter.skip"}, + {BLOB_DB_NUM_PUT, "rocksdb.blobdb.num.put"}, + {BLOB_DB_NUM_WRITE, "rocksdb.blobdb.num.write"}, + {BLOB_DB_NUM_GET, "rocksdb.blobdb.num.get"}, + {BLOB_DB_NUM_MULTIGET, "rocksdb.blobdb.num.multiget"}, + {BLOB_DB_NUM_SEEK, "rocksdb.blobdb.num.seek"}, + {BLOB_DB_NUM_NEXT, "rocksdb.blobdb.num.next"}, + {BLOB_DB_NUM_PREV, "rocksdb.blobdb.num.prev"}, + {BLOB_DB_NUM_KEYS_WRITTEN, "rocksdb.blobdb.num.keys.written"}, + {BLOB_DB_NUM_KEYS_READ, "rocksdb.blobdb.num.keys.read"}, + {BLOB_DB_BYTES_WRITTEN, "rocksdb.blobdb.bytes.written"}, + {BLOB_DB_BYTES_READ, "rocksdb.blobdb.bytes.read"}, + {BLOB_DB_WRITE_INLINED, "rocksdb.blobdb.write.inlined"}, + {BLOB_DB_WRITE_INLINED_TTL, "rocksdb.blobdb.write.inlined.ttl"}, + {BLOB_DB_WRITE_BLOB, "rocksdb.blobdb.write.blob"}, + {BLOB_DB_WRITE_BLOB_TTL, "rocksdb.blobdb.write.blob.ttl"}, + {BLOB_DB_BLOB_FILE_BYTES_WRITTEN, "rocksdb.blobdb.blob.file.bytes.written"}, + {BLOB_DB_BLOB_FILE_BYTES_READ, "rocksdb.blobdb.blob.file.bytes.read"}, + {BLOB_DB_BLOB_FILE_SYNCED, "rocksdb.blobdb.blob.file.synced"}, + {BLOB_DB_BLOB_INDEX_EXPIRED_COUNT, + "rocksdb.blobdb.blob.index.expired.count"}, + {BLOB_DB_BLOB_INDEX_EXPIRED_SIZE, "rocksdb.blobdb.blob.index.expired.size"}, + {BLOB_DB_BLOB_INDEX_EVICTED_COUNT, + "rocksdb.blobdb.blob.index.evicted.count"}, + {BLOB_DB_BLOB_INDEX_EVICTED_SIZE, "rocksdb.blobdb.blob.index.evicted.size"}, + {BLOB_DB_GC_NUM_FILES, "rocksdb.blobdb.gc.num.files"}, + {BLOB_DB_GC_NUM_NEW_FILES, "rocksdb.blobdb.gc.num.new.files"}, + {BLOB_DB_GC_FAILURES, "rocksdb.blobdb.gc.failures"}, + {BLOB_DB_GC_NUM_KEYS_RELOCATED, "rocksdb.blobdb.gc.num.keys.relocated"}, + {BLOB_DB_GC_BYTES_RELOCATED, "rocksdb.blobdb.gc.bytes.relocated"}, + {BLOB_DB_FIFO_NUM_FILES_EVICTED, "rocksdb.blobdb.fifo.num.files.evicted"}, + {BLOB_DB_FIFO_NUM_KEYS_EVICTED, "rocksdb.blobdb.fifo.num.keys.evicted"}, + {BLOB_DB_FIFO_BYTES_EVICTED, "rocksdb.blobdb.fifo.bytes.evicted"}, + {TXN_PREPARE_MUTEX_OVERHEAD, "rocksdb.txn.overhead.mutex.prepare"}, + {TXN_OLD_COMMIT_MAP_MUTEX_OVERHEAD, + "rocksdb.txn.overhead.mutex.old.commit.map"}, + {TXN_DUPLICATE_KEY_OVERHEAD, "rocksdb.txn.overhead.duplicate.key"}, + {TXN_SNAPSHOT_MUTEX_OVERHEAD, "rocksdb.txn.overhead.mutex.snapshot"}, + {TXN_GET_TRY_AGAIN, "rocksdb.txn.get.tryagain"}, + {NUMBER_MULTIGET_KEYS_FOUND, "rocksdb.number.multiget.keys.found"}, + {NO_ITERATOR_CREATED, "rocksdb.num.iterator.created"}, + {NO_ITERATOR_DELETED, "rocksdb.num.iterator.deleted"}, + {BLOCK_CACHE_COMPRESSION_DICT_MISS, + "rocksdb.block.cache.compression.dict.miss"}, + {BLOCK_CACHE_COMPRESSION_DICT_HIT, + "rocksdb.block.cache.compression.dict.hit"}, + {BLOCK_CACHE_COMPRESSION_DICT_ADD, + "rocksdb.block.cache.compression.dict.add"}, + {BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT, + "rocksdb.block.cache.compression.dict.bytes.insert"}, + {BLOCK_CACHE_ADD_REDUNDANT, "rocksdb.block.cache.add.redundant"}, + {BLOCK_CACHE_INDEX_ADD_REDUNDANT, + "rocksdb.block.cache.index.add.redundant"}, + {BLOCK_CACHE_FILTER_ADD_REDUNDANT, + "rocksdb.block.cache.filter.add.redundant"}, + {BLOCK_CACHE_DATA_ADD_REDUNDANT, "rocksdb.block.cache.data.add.redundant"}, + {BLOCK_CACHE_COMPRESSION_DICT_ADD_REDUNDANT, + "rocksdb.block.cache.compression.dict.add.redundant"}, + {FILES_MARKED_TRASH, "rocksdb.files.marked.trash"}, + {FILES_DELETED_FROM_TRASH_QUEUE, "rocksdb.files.marked.trash.deleted"}, + {FILES_DELETED_IMMEDIATELY, "rocksdb.files.deleted.immediately"}, + {ERROR_HANDLER_BG_ERROR_COUNT, "rocksdb.error.handler.bg.error.count"}, + {ERROR_HANDLER_BG_ERROR_COUNT_MISSPELLED, + "rocksdb.error.handler.bg.errro.count"}, + {ERROR_HANDLER_BG_IO_ERROR_COUNT, + "rocksdb.error.handler.bg.io.error.count"}, + {ERROR_HANDLER_BG_IO_ERROR_COUNT_MISSPELLED, + "rocksdb.error.handler.bg.io.errro.count"}, + {ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT, + "rocksdb.error.handler.bg.retryable.io.error.count"}, + {ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT_MISSPELLED, + "rocksdb.error.handler.bg.retryable.io.errro.count"}, + {ERROR_HANDLER_AUTORESUME_COUNT, "rocksdb.error.handler.autoresume.count"}, + {ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT, + "rocksdb.error.handler.autoresume.retry.total.count"}, + {ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT, + "rocksdb.error.handler.autoresume.success.count"}, + {MEMTABLE_PAYLOAD_BYTES_AT_FLUSH, + "rocksdb.memtable.payload.bytes.at.flush"}, + {MEMTABLE_GARBAGE_BYTES_AT_FLUSH, + "rocksdb.memtable.garbage.bytes.at.flush"}, + {SECONDARY_CACHE_HITS, "rocksdb.secondary.cache.hits"}, + {VERIFY_CHECKSUM_READ_BYTES, "rocksdb.verify_checksum.read.bytes"}, + {BACKUP_READ_BYTES, "rocksdb.backup.read.bytes"}, + {BACKUP_WRITE_BYTES, "rocksdb.backup.write.bytes"}, + {REMOTE_COMPACT_READ_BYTES, "rocksdb.remote.compact.read.bytes"}, + {REMOTE_COMPACT_WRITE_BYTES, "rocksdb.remote.compact.write.bytes"}, + {HOT_FILE_READ_BYTES, "rocksdb.hot.file.read.bytes"}, + {WARM_FILE_READ_BYTES, "rocksdb.warm.file.read.bytes"}, + {COLD_FILE_READ_BYTES, "rocksdb.cold.file.read.bytes"}, + {HOT_FILE_READ_COUNT, "rocksdb.hot.file.read.count"}, + {WARM_FILE_READ_COUNT, "rocksdb.warm.file.read.count"}, + {COLD_FILE_READ_COUNT, "rocksdb.cold.file.read.count"}, + {LAST_LEVEL_READ_BYTES, "rocksdb.last.level.read.bytes"}, + {LAST_LEVEL_READ_COUNT, "rocksdb.last.level.read.count"}, + {NON_LAST_LEVEL_READ_BYTES, "rocksdb.non.last.level.read.bytes"}, + {NON_LAST_LEVEL_READ_COUNT, "rocksdb.non.last.level.read.count"}, + {LAST_LEVEL_SEEK_FILTERED, "rocksdb.last.level.seek.filtered"}, + {LAST_LEVEL_SEEK_FILTER_MATCH, "rocksdb.last.level.seek.filter.match"}, + {LAST_LEVEL_SEEK_DATA, "rocksdb.last.level.seek.data"}, + {LAST_LEVEL_SEEK_DATA_USEFUL_NO_FILTER, + "rocksdb.last.level.seek.data.useful.no.filter"}, + {LAST_LEVEL_SEEK_DATA_USEFUL_FILTER_MATCH, + "rocksdb.last.level.seek.data.useful.filter.match"}, + {NON_LAST_LEVEL_SEEK_FILTERED, "rocksdb.non.last.level.seek.filtered"}, + {NON_LAST_LEVEL_SEEK_FILTER_MATCH, + "rocksdb.non.last.level.seek.filter.match"}, + {NON_LAST_LEVEL_SEEK_DATA, "rocksdb.non.last.level.seek.data"}, + {NON_LAST_LEVEL_SEEK_DATA_USEFUL_NO_FILTER, + "rocksdb.non.last.level.seek.data.useful.no.filter"}, + {NON_LAST_LEVEL_SEEK_DATA_USEFUL_FILTER_MATCH, + "rocksdb.non.last.level.seek.data.useful.filter.match"}, + {BLOCK_CHECKSUM_COMPUTE_COUNT, "rocksdb.block.checksum.compute.count"}, + {BLOCK_CHECKSUM_MISMATCH_COUNT, "rocksdb.block.checksum.mismatch.count"}, + {MULTIGET_COROUTINE_COUNT, "rocksdb.multiget.coroutine.count"}, + {BLOB_DB_CACHE_MISS, "rocksdb.blobdb.cache.miss"}, + {BLOB_DB_CACHE_HIT, "rocksdb.blobdb.cache.hit"}, + {BLOB_DB_CACHE_ADD, "rocksdb.blobdb.cache.add"}, + {BLOB_DB_CACHE_ADD_FAILURES, "rocksdb.blobdb.cache.add.failures"}, + {BLOB_DB_CACHE_BYTES_READ, "rocksdb.blobdb.cache.bytes.read"}, + {BLOB_DB_CACHE_BYTES_WRITE, "rocksdb.blobdb.cache.bytes.write"}, + {READ_ASYNC_MICROS, "rocksdb.read.async.micros"}, + {ASYNC_READ_ERROR_COUNT, "rocksdb.async.read.error.count"}, + {SECONDARY_CACHE_FILTER_HITS, "rocksdb.secondary.cache.filter.hits"}, + {SECONDARY_CACHE_INDEX_HITS, "rocksdb.secondary.cache.index.hits"}, + {SECONDARY_CACHE_DATA_HITS, "rocksdb.secondary.cache.data.hits"}, + {TABLE_OPEN_PREFETCH_TAIL_MISS, "rocksdb.table.open.prefetch.tail.miss"}, + {TABLE_OPEN_PREFETCH_TAIL_HIT, "rocksdb.table.open.prefetch.tail.hit"}, + {TIMESTAMP_FILTER_TABLE_CHECKED, "rocksdb.timestamp.filter.table.checked"}, + {TIMESTAMP_FILTER_TABLE_FILTERED, + "rocksdb.timestamp.filter.table.filtered"}, + {BYTES_COMPRESSED_FROM, "rocksdb.bytes.compressed.from"}, + {BYTES_COMPRESSED_TO, "rocksdb.bytes.compressed.to"}, + {BYTES_COMPRESSION_BYPASSED, "rocksdb.bytes.compression_bypassed"}, + {BYTES_COMPRESSION_REJECTED, "rocksdb.bytes.compression.rejected"}, + {NUMBER_BLOCK_COMPRESSION_BYPASSED, + "rocksdb.number.block_compression_bypassed"}, + {NUMBER_BLOCK_COMPRESSION_REJECTED, + "rocksdb.number.block_compression_rejected"}, + {BYTES_DECOMPRESSED_FROM, "rocksdb.bytes.decompressed.from"}, + {BYTES_DECOMPRESSED_TO, "rocksdb.bytes.decompressed.to"}, +}; + +const std::vector> HistogramsNameMap = { + {DB_GET, "rocksdb.db.get.micros"}, + {DB_WRITE, "rocksdb.db.write.micros"}, + {COMPACTION_TIME, "rocksdb.compaction.times.micros"}, + {COMPACTION_CPU_TIME, "rocksdb.compaction.times.cpu_micros"}, + {SUBCOMPACTION_SETUP_TIME, "rocksdb.subcompaction.setup.times.micros"}, + {TABLE_SYNC_MICROS, "rocksdb.table.sync.micros"}, + {COMPACTION_OUTFILE_SYNC_MICROS, "rocksdb.compaction.outfile.sync.micros"}, + {WAL_FILE_SYNC_MICROS, "rocksdb.wal.file.sync.micros"}, + {MANIFEST_FILE_SYNC_MICROS, "rocksdb.manifest.file.sync.micros"}, + {TABLE_OPEN_IO_MICROS, "rocksdb.table.open.io.micros"}, + {DB_MULTIGET, "rocksdb.db.multiget.micros"}, + {READ_BLOCK_COMPACTION_MICROS, "rocksdb.read.block.compaction.micros"}, + {READ_BLOCK_GET_MICROS, "rocksdb.read.block.get.micros"}, + {WRITE_RAW_BLOCK_MICROS, "rocksdb.write.raw.block.micros"}, + {NUM_FILES_IN_SINGLE_COMPACTION, "rocksdb.numfiles.in.singlecompaction"}, + {DB_SEEK, "rocksdb.db.seek.micros"}, + {WRITE_STALL, "rocksdb.db.write.stall"}, + {SST_READ_MICROS, "rocksdb.sst.read.micros"}, + {FILE_READ_FLUSH_MICROS, "rocksdb.file.read.flush.micros"}, + {FILE_READ_COMPACTION_MICROS, "rocksdb.file.read.compaction.micros"}, + {FILE_READ_DB_OPEN_MICROS, "rocksdb.file.read.db.open.micros"}, + {NUM_SUBCOMPACTIONS_SCHEDULED, "rocksdb.num.subcompactions.scheduled"}, + {BYTES_PER_READ, "rocksdb.bytes.per.read"}, + {BYTES_PER_WRITE, "rocksdb.bytes.per.write"}, + {BYTES_PER_MULTIGET, "rocksdb.bytes.per.multiget"}, + {BYTES_COMPRESSED, "rocksdb.bytes.compressed"}, + {BYTES_DECOMPRESSED, "rocksdb.bytes.decompressed"}, + {COMPRESSION_TIMES_NANOS, "rocksdb.compression.times.nanos"}, + {DECOMPRESSION_TIMES_NANOS, "rocksdb.decompression.times.nanos"}, + {READ_NUM_MERGE_OPERANDS, "rocksdb.read.num.merge_operands"}, + {BLOB_DB_KEY_SIZE, "rocksdb.blobdb.key.size"}, + {BLOB_DB_VALUE_SIZE, "rocksdb.blobdb.value.size"}, + {BLOB_DB_WRITE_MICROS, "rocksdb.blobdb.write.micros"}, + {BLOB_DB_GET_MICROS, "rocksdb.blobdb.get.micros"}, + {BLOB_DB_MULTIGET_MICROS, "rocksdb.blobdb.multiget.micros"}, + {BLOB_DB_SEEK_MICROS, "rocksdb.blobdb.seek.micros"}, + {BLOB_DB_NEXT_MICROS, "rocksdb.blobdb.next.micros"}, + {BLOB_DB_PREV_MICROS, "rocksdb.blobdb.prev.micros"}, + {BLOB_DB_BLOB_FILE_WRITE_MICROS, "rocksdb.blobdb.blob.file.write.micros"}, + {BLOB_DB_BLOB_FILE_READ_MICROS, "rocksdb.blobdb.blob.file.read.micros"}, + {BLOB_DB_BLOB_FILE_SYNC_MICROS, "rocksdb.blobdb.blob.file.sync.micros"}, + {BLOB_DB_COMPRESSION_MICROS, "rocksdb.blobdb.compression.micros"}, + {BLOB_DB_DECOMPRESSION_MICROS, "rocksdb.blobdb.decompression.micros"}, + {FLUSH_TIME, "rocksdb.db.flush.micros"}, + {SST_BATCH_SIZE, "rocksdb.sst.batch.size"}, + {NUM_INDEX_AND_FILTER_BLOCKS_READ_PER_LEVEL, + "rocksdb.num.index.and.filter.blocks.read.per.level"}, + {NUM_SST_READ_PER_LEVEL, "rocksdb.num.sst.read.per.level"}, + {ERROR_HANDLER_AUTORESUME_RETRY_COUNT, + "rocksdb.error.handler.autoresume.retry.count"}, + {ASYNC_READ_BYTES, "rocksdb.async.read.bytes"}, + {POLL_WAIT_MICROS, "rocksdb.poll.wait.micros"}, + {PREFETCHED_BYTES_DISCARDED, "rocksdb.prefetched.bytes.discarded"}, + {MULTIGET_IO_BATCH_SIZE, "rocksdb.multiget.io.batch.size"}, + {NUM_LEVEL_READ_PER_MULTIGET, "rocksdb.num.level.read.per.multiget"}, + {ASYNC_PREFETCH_ABORT_MICROS, "rocksdb.async.prefetch.abort.micros"}, + {TABLE_OPEN_PREFETCH_TAIL_READ_BYTES, + "rocksdb.table.open.prefetch.tail.read.bytes"}, +}; + +std::shared_ptr CreateDBStatistics() { + return std::make_shared(nullptr); +} + +static int RegisterBuiltinStatistics(ObjectLibrary& library, + const std::string& /*arg*/) { + library.AddFactory( + StatisticsImpl::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new StatisticsImpl(nullptr)); + return guard->get(); + }); + return 1; +} + +Status Statistics::CreateFromString(const ConfigOptions& config_options, + const std::string& id, + std::shared_ptr* result) { + static std::once_flag once; + std::call_once(once, [&]() { + RegisterBuiltinStatistics(*(ObjectLibrary::Default().get()), ""); + }); + Status s; + if (id == "" || id == StatisticsImpl::kClassName()) { + result->reset(new StatisticsImpl(nullptr)); + } else if (id == kNullptrString) { + result->reset(); + } else { + s = LoadSharedObject(config_options, id, result); + } + return s; +} + +static std::unordered_map stats_type_info = { + {"inner", OptionTypeInfo::AsCustomSharedPtr( + 0, OptionVerificationType::kByNameAllowFromNull, + OptionTypeFlags::kCompareNever)}, +}; + +StatisticsImpl::StatisticsImpl(std::shared_ptr stats) + : stats_(std::move(stats)) { + RegisterOptions("StatisticsOptions", &stats_, &stats_type_info); +} + +StatisticsImpl::~StatisticsImpl() {} + +uint64_t StatisticsImpl::getTickerCount(uint32_t tickerType) const { + MutexLock lock(&aggregate_lock_); + return getTickerCountLocked(tickerType); +} + +uint64_t StatisticsImpl::getTickerCountLocked(uint32_t tickerType) const { + assert(tickerType < TICKER_ENUM_MAX); + uint64_t res = 0; + for (size_t core_idx = 0; core_idx < per_core_stats_.Size(); ++core_idx) { + res += per_core_stats_.AccessAtCore(core_idx)->tickers_[tickerType]; + } + return res; +} + +void StatisticsImpl::histogramData(uint32_t histogramType, + HistogramData* const data) const { + MutexLock lock(&aggregate_lock_); + getHistogramImplLocked(histogramType)->Data(data); +} + +std::unique_ptr StatisticsImpl::getHistogramImplLocked( + uint32_t histogramType) const { + assert(histogramType < HISTOGRAM_ENUM_MAX); + std::unique_ptr res_hist(new HistogramImpl()); + for (size_t core_idx = 0; core_idx < per_core_stats_.Size(); ++core_idx) { + res_hist->Merge( + per_core_stats_.AccessAtCore(core_idx)->histograms_[histogramType]); + } + return res_hist; +} + +std::string StatisticsImpl::getHistogramString(uint32_t histogramType) const { + MutexLock lock(&aggregate_lock_); + return getHistogramImplLocked(histogramType)->ToString(); +} + +void StatisticsImpl::setTickerCount(uint32_t tickerType, uint64_t count) { + { + MutexLock lock(&aggregate_lock_); + setTickerCountLocked(tickerType, count); + } + if (stats_ && tickerType < TICKER_ENUM_MAX) { + stats_->setTickerCount(tickerType, count); + } +} + +void StatisticsImpl::setTickerCountLocked(uint32_t tickerType, uint64_t count) { + assert(tickerType < TICKER_ENUM_MAX); + for (size_t core_idx = 0; core_idx < per_core_stats_.Size(); ++core_idx) { + if (core_idx == 0) { + per_core_stats_.AccessAtCore(core_idx)->tickers_[tickerType] = count; + } else { + per_core_stats_.AccessAtCore(core_idx)->tickers_[tickerType] = 0; + } + } +} + +uint64_t StatisticsImpl::getAndResetTickerCount(uint32_t tickerType) { + uint64_t sum = 0; + { + MutexLock lock(&aggregate_lock_); + assert(tickerType < TICKER_ENUM_MAX); + for (size_t core_idx = 0; core_idx < per_core_stats_.Size(); ++core_idx) { + sum += + per_core_stats_.AccessAtCore(core_idx)->tickers_[tickerType].exchange( + 0, std::memory_order_relaxed); + } + } + if (stats_ && tickerType < TICKER_ENUM_MAX) { + stats_->setTickerCount(tickerType, 0); + } + return sum; +} + +void StatisticsImpl::recordTick(uint32_t tickerType, uint64_t count) { + if (get_stats_level() <= StatsLevel::kExceptTickers) { + return; + } + if (tickerType < TICKER_ENUM_MAX) { + per_core_stats_.Access()->tickers_[tickerType].fetch_add( + count, std::memory_order_relaxed); + if (stats_) { + stats_->recordTick(tickerType, count); + } + } else { + assert(false); + } +} + +void StatisticsImpl::recordInHistogram(uint32_t histogramType, uint64_t value) { + assert(histogramType < HISTOGRAM_ENUM_MAX); + if (get_stats_level() <= StatsLevel::kExceptHistogramOrTimers) { + return; + } + per_core_stats_.Access()->histograms_[histogramType].Add(value); + if (stats_ && histogramType < HISTOGRAM_ENUM_MAX) { + stats_->recordInHistogram(histogramType, value); + } +} + +Status StatisticsImpl::Reset() { + MutexLock lock(&aggregate_lock_); + for (uint32_t i = 0; i < TICKER_ENUM_MAX; ++i) { + setTickerCountLocked(i, 0); + } + for (uint32_t i = 0; i < HISTOGRAM_ENUM_MAX; ++i) { + for (size_t core_idx = 0; core_idx < per_core_stats_.Size(); ++core_idx) { + per_core_stats_.AccessAtCore(core_idx)->histograms_[i].Clear(); + } + } + return Status::OK(); +} + +namespace { + +// a buffer size used for temp string buffers +const int kTmpStrBufferSize = 200; + +} // namespace + +std::string StatisticsImpl::ToString() const { + MutexLock lock(&aggregate_lock_); + std::string res; + res.reserve(20000); + for (const auto& t : TickersNameMap) { + assert(t.first < TICKER_ENUM_MAX); + char buffer[kTmpStrBufferSize]; + snprintf(buffer, kTmpStrBufferSize, "%s COUNT : %" PRIu64 "\n", + t.second.c_str(), getTickerCountLocked(t.first)); + res.append(buffer); + } + for (const auto& h : HistogramsNameMap) { + assert(h.first < HISTOGRAM_ENUM_MAX); + char buffer[kTmpStrBufferSize]; + HistogramData hData; + getHistogramImplLocked(h.first)->Data(&hData); + // don't handle failures - buffer should always be big enough and arguments + // should be provided correctly + int ret = + snprintf(buffer, kTmpStrBufferSize, + "%s P50 : %f P95 : %f P99 : %f P100 : %f COUNT : %" PRIu64 + " SUM : %" PRIu64 "\n", + h.second.c_str(), hData.median, hData.percentile95, + hData.percentile99, hData.max, hData.count, hData.sum); + if (ret < 0 || ret >= kTmpStrBufferSize) { + assert(false); + continue; + } + res.append(buffer); + } + res.shrink_to_fit(); + return res; +} + +bool StatisticsImpl::getTickerMap( + std::map* stats_map) const { + assert(stats_map); + if (!stats_map) return false; + stats_map->clear(); + MutexLock lock(&aggregate_lock_); + for (const auto& t : TickersNameMap) { + assert(t.first < TICKER_ENUM_MAX); + (*stats_map)[t.second.c_str()] = getTickerCountLocked(t.first); + } + return true; +} + +bool StatisticsImpl::HistEnabledForType(uint32_t type) const { + return type < HISTOGRAM_ENUM_MAX; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/statistics_impl.h b/librocksdb-sys/rocksdb/monitoring/statistics_impl.h new file mode 100644 index 0000000..e0dc29d --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/statistics_impl.h @@ -0,0 +1,144 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#pragma once +#include +#include +#include +#include + +#include "monitoring/histogram.h" +#include "port/likely.h" +#include "port/port.h" +#include "rocksdb/statistics.h" +#include "util/core_local.h" +#include "util/mutexlock.h" + +#ifdef __clang__ +#define ROCKSDB_FIELD_UNUSED __attribute__((__unused__)) +#else +#define ROCKSDB_FIELD_UNUSED +#endif // __clang__ + +#ifndef STRINGIFY +#define STRINGIFY(x) #x +#define TOSTRING(x) STRINGIFY(x) +#endif + +namespace ROCKSDB_NAMESPACE { + +enum TickersInternal : uint32_t { + INTERNAL_TICKER_ENUM_START = TICKER_ENUM_MAX, + INTERNAL_TICKER_ENUM_MAX +}; + +enum HistogramsInternal : uint32_t { + INTERNAL_HISTOGRAM_START = HISTOGRAM_ENUM_MAX, + INTERNAL_HISTOGRAM_ENUM_MAX +}; + +class StatisticsImpl : public Statistics { + public: + StatisticsImpl(std::shared_ptr stats); + virtual ~StatisticsImpl(); + const char* Name() const override { return kClassName(); } + static const char* kClassName() { return "BasicStatistics"; } + + virtual uint64_t getTickerCount(uint32_t ticker_type) const override; + virtual void histogramData(uint32_t histogram_type, + HistogramData* const data) const override; + std::string getHistogramString(uint32_t histogram_type) const override; + + virtual void setTickerCount(uint32_t ticker_type, uint64_t count) override; + virtual uint64_t getAndResetTickerCount(uint32_t ticker_type) override; + virtual void recordTick(uint32_t ticker_type, uint64_t count) override; + // The function is implemented for now for backward compatibility reason. + // In case a user explictly calls it, for example, they may have a wrapped + // Statistics object, passing the call to recordTick() into here, nothing + // will break. + void measureTime(uint32_t histogramType, uint64_t time) override { + recordInHistogram(histogramType, time); + } + virtual void recordInHistogram(uint32_t histogram_type, + uint64_t value) override; + + virtual Status Reset() override; + virtual std::string ToString() const override; + virtual bool getTickerMap(std::map*) const override; + virtual bool HistEnabledForType(uint32_t type) const override; + + const Customizable* Inner() const override { return stats_.get(); } + + private: + // If non-nullptr, forwards updates to the object pointed to by `stats_`. + std::shared_ptr stats_; + // Synchronizes anything that operates across other cores' local data, + // such that operations like Reset() can be performed atomically. + mutable port::Mutex aggregate_lock_; + + // The ticker/histogram data are stored in this structure, which we will store + // per-core. It is cache-aligned, so tickers/histograms belonging to different + // cores can never share the same cache line. + // + // Alignment attributes expand to nothing depending on the platform + struct ALIGN_AS(CACHE_LINE_SIZE) StatisticsData { + std::atomic_uint_fast64_t tickers_[INTERNAL_TICKER_ENUM_MAX] = {{0}}; + HistogramImpl histograms_[INTERNAL_HISTOGRAM_ENUM_MAX]; +#ifndef HAVE_ALIGNED_NEW + char + padding[(CACHE_LINE_SIZE - + (INTERNAL_TICKER_ENUM_MAX * sizeof(std::atomic_uint_fast64_t) + + INTERNAL_HISTOGRAM_ENUM_MAX * sizeof(HistogramImpl)) % + CACHE_LINE_SIZE)] ROCKSDB_FIELD_UNUSED; +#endif + void* operator new(size_t s) { return port::cacheline_aligned_alloc(s); } + void* operator new[](size_t s) { return port::cacheline_aligned_alloc(s); } + void operator delete(void* p) { port::cacheline_aligned_free(p); } + void operator delete[](void* p) { port::cacheline_aligned_free(p); } + }; + +#ifndef TEST_CACHE_LINE_SIZE + static_assert(sizeof(StatisticsData) % CACHE_LINE_SIZE == 0, + "Expected " TOSTRING(CACHE_LINE_SIZE) "-byte aligned"); +#endif + + CoreLocalArray per_core_stats_; + + uint64_t getTickerCountLocked(uint32_t ticker_type) const; + std::unique_ptr getHistogramImplLocked( + uint32_t histogram_type) const; + void setTickerCountLocked(uint32_t ticker_type, uint64_t count); +}; + +// Utility functions +inline void RecordInHistogram(Statistics* statistics, uint32_t histogram_type, + uint64_t value) { + if (statistics) { + statistics->recordInHistogram(histogram_type, value); + } +} + +inline void RecordTimeToHistogram(Statistics* statistics, + uint32_t histogram_type, uint64_t value) { + if (statistics) { + statistics->reportTimeToHistogram(histogram_type, value); + } +} + +inline void RecordTick(Statistics* statistics, uint32_t ticker_type, + uint64_t count = 1) { + if (statistics) { + statistics->recordTick(ticker_type, count); + } +} + +inline void SetTickerCount(Statistics* statistics, uint32_t ticker_type, + uint64_t count) { + if (statistics) { + statistics->setTickerCount(ticker_type, count); + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/statistics_test.cc b/librocksdb-sys/rocksdb/monitoring/statistics_test.cc new file mode 100644 index 0000000..98aae0c --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/statistics_test.cc @@ -0,0 +1,88 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#include "rocksdb/statistics.h" + +#include "port/stack_trace.h" +#include "rocksdb/convenience.h" +#include "rocksdb/utilities/options_type.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" + +namespace ROCKSDB_NAMESPACE { + +class StatisticsTest : public testing::Test {}; + +// Sanity check to make sure that contents and order of TickersNameMap +// match Tickers enum +TEST_F(StatisticsTest, SanityTickers) { + EXPECT_EQ(static_cast(Tickers::TICKER_ENUM_MAX), + TickersNameMap.size()); + + for (uint32_t t = 0; t < Tickers::TICKER_ENUM_MAX; t++) { + auto pair = TickersNameMap[static_cast(t)]; + ASSERT_EQ(pair.first, t) << "Miss match at " << pair.second; + } +} + +// Sanity check to make sure that contents and order of HistogramsNameMap +// match Tickers enum +TEST_F(StatisticsTest, SanityHistograms) { + EXPECT_EQ(static_cast(Histograms::HISTOGRAM_ENUM_MAX), + HistogramsNameMap.size()); + + for (uint32_t h = 0; h < Histograms::HISTOGRAM_ENUM_MAX; h++) { + auto pair = HistogramsNameMap[static_cast(h)]; + ASSERT_EQ(pair.first, h) << "Miss match at " << pair.second; + } +} + +TEST_F(StatisticsTest, NoNameStats) { + static std::unordered_map no_name_opt_info = { + {"inner", + OptionTypeInfo::AsCustomSharedPtr( + 0, OptionVerificationType::kByName, + OptionTypeFlags::kAllowNull | OptionTypeFlags::kCompareNever)}, + }; + + class DefaultNameStatistics : public Statistics { + public: + DefaultNameStatistics(const std::shared_ptr& stats = nullptr) + : inner(stats) { + RegisterOptions("", &inner, &no_name_opt_info); + } + + uint64_t getTickerCount(uint32_t /*tickerType*/) const override { + return 0; + } + void histogramData(uint32_t /*type*/, + HistogramData* const /*data*/) const override {} + void recordTick(uint32_t /*tickerType*/, uint64_t /*count*/) override {} + void setTickerCount(uint32_t /*tickerType*/, uint64_t /*count*/) override {} + uint64_t getAndResetTickerCount(uint32_t /*tickerType*/) override { + return 0; + } + std::shared_ptr inner; + }; + ConfigOptions options; + options.ignore_unsupported_options = false; + auto stats = std::make_shared(); + ASSERT_STREQ(stats->Name(), ""); + ASSERT_EQ("", stats->ToString( + options)); // A stats with no name with have no options... + ASSERT_OK(stats->ConfigureFromString(options, "inner=")); + ASSERT_EQ("", stats->ToString( + options)); // A stats with no name with have no options... + ASSERT_NE(stats->inner, nullptr); + ASSERT_NE("", stats->inner->ToString(options)); // ... even if it does... +} +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/monitoring/stats_history_test.cc b/librocksdb-sys/rocksdb/monitoring/stats_history_test.cc new file mode 100644 index 0000000..cfed7ba --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/stats_history_test.cc @@ -0,0 +1,662 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "rocksdb/stats_history.h" + +#include +#include +#include + +#include "db/column_family.h" +#include "db/db_impl/db_impl.h" +#include "db/db_test_util.h" +#include "db/periodic_task_scheduler.h" +#include "monitoring/persistent_stats_history.h" +#include "options/options_helper.h" +#include "port/stack_trace.h" +#include "rocksdb/cache.h" +#include "rocksdb/convenience.h" +#include "rocksdb/rate_limiter.h" +#include "test_util/mock_time_env.h" +#include "test_util/sync_point.h" +#include "test_util/testutil.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +class StatsHistoryTest : public DBTestBase { + public: + StatsHistoryTest() : DBTestBase("stats_history_test", /*env_do_fsync=*/true) { + mock_clock_ = std::make_shared(env_->GetSystemClock()); + mock_env_.reset(new CompositeEnvWrapper(env_, mock_clock_)); + } + + protected: + std::shared_ptr mock_clock_; + std::unique_ptr mock_env_; + + void SetUp() override { + mock_clock_->InstallTimedWaitFixCallback(); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::StartPeriodicTaskScheduler:Init", [&](void* arg) { + auto periodic_task_scheduler_ptr = + reinterpret_cast(arg); + periodic_task_scheduler_ptr->TEST_OverrideTimer(mock_clock_.get()); + }); + } +}; + +TEST_F(StatsHistoryTest, RunStatsDumpPeriodSec) { + constexpr int kPeriodSec = 5; + Options options; + options.create_if_missing = true; + options.stats_dump_period_sec = kPeriodSec; + options.env = mock_env_.get(); + int counter = 0; + SyncPoint::GetInstance()->SetCallBack("DBImpl::DumpStats:1", + [&](void* /*arg*/) { counter++; }); + Reopen(options); + ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_dump_period_sec); + + // Wait for the first stats persist to finish, as the initial delay could be + // different. + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); + + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); + ASSERT_GE(counter, 1); + + // Test cancel job through SetOptions + ASSERT_OK(dbfull()->SetDBOptions({{"stats_dump_period_sec", "0"}})); + int old_val = counter; + for (int i = 1; i < 20; ++i) { + mock_clock_->MockSleepForSeconds(kPeriodSec); + } + ASSERT_EQ(counter, old_val); + Close(); +} + +// Test persistent stats background thread scheduling and cancelling +TEST_F(StatsHistoryTest, StatsPersistScheduling) { + constexpr int kPeriodSec = 5; + Options options; + options.create_if_missing = true; + options.stats_persist_period_sec = kPeriodSec; + options.env = mock_env_.get(); + int counter = 0; + SyncPoint::GetInstance()->SetCallBack("DBImpl::PersistStats:Entry", + [&](void* /*arg*/) { counter++; }); + Reopen(options); + ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_persist_period_sec); + + // Wait for the first stats persist to finish, as the initial delay could be + // different. + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); + + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); + ASSERT_GE(counter, 1); + + // Test cancel job through SetOptions + ASSERT_OK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "0"}})); + int old_val = counter; + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec * 2); }); + ASSERT_EQ(counter, old_val); + + Close(); +} + +// Test enabling persistent stats for the first time +TEST_F(StatsHistoryTest, PersistentStatsFreshInstall) { + constexpr unsigned int kPeriodSec = 5; + Options options; + options.create_if_missing = true; + options.stats_persist_period_sec = 0; + options.env = mock_env_.get(); + int counter = 0; + SyncPoint::GetInstance()->SetCallBack("DBImpl::PersistStats:Entry", + [&](void* /*arg*/) { counter++; }); + Reopen(options); + ASSERT_OK(dbfull()->SetDBOptions( + {{"stats_persist_period_sec", std::to_string(kPeriodSec)}})); + ASSERT_EQ(kPeriodSec, dbfull()->GetDBOptions().stats_persist_period_sec); + + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); + ASSERT_GE(counter, 1); + Close(); +} + +// TODO(Zhongyi): Move persistent stats related tests to a separate file +TEST_F(StatsHistoryTest, GetStatsHistoryInMemory) { + constexpr int kPeriodSec = 5; + Options options; + options.create_if_missing = true; + options.stats_persist_period_sec = kPeriodSec; + options.statistics = CreateDBStatistics(); + options.env = mock_env_.get(); + CreateColumnFamilies({"pikachu"}, options); + ASSERT_OK(Put("foo", "bar")); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + + // make sure the first stats persist to finish + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); + + // Wait for stats persist to finish + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); + + std::unique_ptr stats_iter; + ASSERT_OK( + db_->GetStatsHistory(0, mock_clock_->NowSeconds() + 1, &stats_iter)); + ASSERT_TRUE(stats_iter != nullptr); + // disabled stats snapshots + ASSERT_OK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "0"}})); + size_t stats_count = 0; + for (; stats_iter->Valid(); stats_iter->Next()) { + auto stats_map = stats_iter->GetStatsMap(); + ASSERT_EQ(stats_iter->GetStatsTime(), mock_clock_->NowSeconds()); + stats_count += stats_map.size(); + } + ASSERT_GT(stats_count, 0); + // Wait a bit and verify no more stats are found + for (int i = 0; i < 10; ++i) { + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(1); }); + } + ASSERT_OK(db_->GetStatsHistory(0, mock_clock_->NowSeconds(), &stats_iter)); + ASSERT_TRUE(stats_iter != nullptr); + size_t stats_count_new = 0; + for (; stats_iter->Valid(); stats_iter->Next()) { + stats_count_new += stats_iter->GetStatsMap().size(); + } + ASSERT_EQ(stats_count_new, stats_count); + Close(); +} + +TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) { + constexpr int kPeriodSec = 1; + Options options; + options.create_if_missing = true; + options.statistics = CreateDBStatistics(); + options.stats_persist_period_sec = kPeriodSec; + options.env = mock_env_.get(); + + CreateColumnFamilies({"pikachu"}, options); + ASSERT_OK(Put("foo", "bar")); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + // some random operation to populate statistics + ASSERT_OK(Delete("foo")); + ASSERT_OK(Put("sol", "sol")); + ASSERT_OK(Put("epic", "epic")); + ASSERT_OK(Put("ltd", "ltd")); + ASSERT_EQ("sol", Get("sol")); + ASSERT_EQ("epic", Get("epic")); + ASSERT_EQ("ltd", Get("ltd")); + Iterator* iterator = db_->NewIterator(ReadOptions()); + for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) { + ASSERT_TRUE(iterator->key() == iterator->value()); + } + delete iterator; + ASSERT_OK(Flush()); + ASSERT_OK(Delete("sol")); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + // second round of ops + ASSERT_OK(Put("saigon", "saigon")); + ASSERT_OK(Put("noodle talk", "noodle talk")); + ASSERT_OK(Put("ping bistro", "ping bistro")); + iterator = db_->NewIterator(ReadOptions()); + for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) { + ASSERT_TRUE(iterator->key() == iterator->value()); + } + delete iterator; + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + const int kIterations = 10; + for (int i = 0; i < kIterations; ++i) { + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); + } + + std::unique_ptr stats_iter; + ASSERT_OK( + db_->GetStatsHistory(0, mock_clock_->NowSeconds() + 1, &stats_iter)); + ASSERT_TRUE(stats_iter != nullptr); + size_t stats_count = 0; + int slice_count = 0; + for (; stats_iter->Valid(); stats_iter->Next()) { + slice_count++; + auto stats_map = stats_iter->GetStatsMap(); + stats_count += stats_map.size(); + } + size_t stats_history_size = dbfull()->TEST_EstimateInMemoryStatsHistorySize(); + ASSERT_GE(slice_count, kIterations - 1); + ASSERT_GE(stats_history_size, 15000); + // capping memory cost at 15000 bytes since one slice is around 10000~15000 + ASSERT_OK(dbfull()->SetDBOptions({{"stats_history_buffer_size", "15000"}})); + ASSERT_EQ(15000, dbfull()->GetDBOptions().stats_history_buffer_size); + + // Wait for stats persist to finish + for (int i = 0; i < kIterations; ++i) { + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); + } + + ASSERT_OK( + db_->GetStatsHistory(0, mock_clock_->NowSeconds() + 1, &stats_iter)); + ASSERT_TRUE(stats_iter != nullptr); + size_t stats_count_reopen = 0; + slice_count = 0; + for (; stats_iter->Valid(); stats_iter->Next()) { + slice_count++; + auto stats_map = stats_iter->GetStatsMap(); + stats_count_reopen += stats_map.size(); + } + size_t stats_history_size_reopen = + dbfull()->TEST_EstimateInMemoryStatsHistorySize(); + // only one slice can fit under the new stats_history_buffer_size + ASSERT_LT(slice_count, 2); + ASSERT_TRUE(stats_history_size_reopen < 15000 && + stats_history_size_reopen > 0); + ASSERT_TRUE(stats_count_reopen < stats_count && stats_count_reopen > 0); + Close(); + // TODO: may also want to verify stats timestamp to make sure we are purging + // the correct stats snapshot +} + +int countkeys(Iterator* iter) { + int count = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + count++; + } + return count; +} + +TEST_F(StatsHistoryTest, GetStatsHistoryFromDisk) { + constexpr int kPeriodSec = 5; + Options options; + options.create_if_missing = true; + options.stats_persist_period_sec = kPeriodSec; + options.statistics = CreateDBStatistics(); + options.persist_stats_to_disk = true; + options.env = mock_env_.get(); + CreateColumnFamilies({"pikachu"}, options); + ASSERT_OK(Put("foo", "bar")); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_EQ(Get("foo"), "bar"); + + // Wait for the first stats persist to finish, as the initial delay could be + // different. + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); + + // Wait for stats persist to finish + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); + + auto iter = + db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); + int key_count1 = countkeys(iter); + delete iter; + + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); + iter = + db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); + int key_count2 = countkeys(iter); + delete iter; + + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); + iter = + db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); + int key_count3 = countkeys(iter); + delete iter; + ASSERT_GE(key_count2, key_count1); + ASSERT_GE(key_count3, key_count2); + ASSERT_EQ(key_count3 - key_count2, key_count2 - key_count1); + std::unique_ptr stats_iter; + ASSERT_OK( + db_->GetStatsHistory(0, mock_clock_->NowSeconds() + 1, &stats_iter)); + ASSERT_TRUE(stats_iter != nullptr); + size_t stats_count = 0; + int slice_count = 0; + int non_zero_count = 0; + for (int i = 2; stats_iter->Valid(); stats_iter->Next(), i++) { + slice_count++; + auto stats_map = stats_iter->GetStatsMap(); + ASSERT_EQ(stats_iter->GetStatsTime(), kPeriodSec * i - 1); + for (auto& stat : stats_map) { + if (stat.second != 0) { + non_zero_count++; + } + } + stats_count += stats_map.size(); + } + ASSERT_EQ(slice_count, 3); + // 2 extra keys for format version + ASSERT_EQ(stats_count, key_count3 - 2); + // verify reopen will not cause data loss + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_OK( + db_->GetStatsHistory(0, mock_clock_->NowSeconds() + 1, &stats_iter)); + ASSERT_TRUE(stats_iter != nullptr); + size_t stats_count_reopen = 0; + int slice_count_reopen = 0; + int non_zero_count_recover = 0; + for (; stats_iter->Valid(); stats_iter->Next()) { + slice_count_reopen++; + auto stats_map = stats_iter->GetStatsMap(); + for (auto& stat : stats_map) { + if (stat.second != 0) { + non_zero_count_recover++; + } + } + stats_count_reopen += stats_map.size(); + } + + ASSERT_EQ(non_zero_count, non_zero_count_recover); + ASSERT_EQ(slice_count, slice_count_reopen); + ASSERT_EQ(stats_count, stats_count_reopen); + Close(); +} + +// Test persisted stats matches the value found in options.statistics and +// the stats value retains after DB reopen +TEST_F(StatsHistoryTest, PersitentStatsVerifyValue) { + constexpr int kPeriodSec = 5; + Options options; + options.create_if_missing = true; + options.stats_persist_period_sec = kPeriodSec; + options.statistics = CreateDBStatistics(); + options.persist_stats_to_disk = true; + std::map stats_map_before; + ASSERT_TRUE(options.statistics->getTickerMap(&stats_map_before)); + options.env = mock_env_.get(); + CreateColumnFamilies({"pikachu"}, options); + ASSERT_OK(Put("foo", "bar")); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_EQ(Get("foo"), "bar"); + + // Wait for the first stats persist to finish, as the initial delay could be + // different. + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); + + // Wait for stats persist to finish + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); + auto iter = + db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); + countkeys(iter); + delete iter; + + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); + iter = + db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); + countkeys(iter); + delete iter; + + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); + iter = + db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); + countkeys(iter); + delete iter; + + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); + + std::map stats_map_after; + ASSERT_TRUE(options.statistics->getTickerMap(&stats_map_after)); + std::unique_ptr stats_iter; + ASSERT_OK( + db_->GetStatsHistory(0, mock_clock_->NowSeconds() + 1, &stats_iter)); + ASSERT_TRUE(stats_iter != nullptr); + std::string sample = "rocksdb.num.iterator.deleted"; + uint64_t recovered_value = 0; + for (int i = 2; stats_iter->Valid(); stats_iter->Next(), ++i) { + auto stats_map = stats_iter->GetStatsMap(); + ASSERT_EQ(stats_iter->GetStatsTime(), kPeriodSec * i - 1); + for (const auto& stat : stats_map) { + if (sample.compare(stat.first) == 0) { + recovered_value += stat.second; + } + } + } + ASSERT_EQ(recovered_value, stats_map_after[sample]); + + // test stats value retains after recovery + ReopenWithColumnFamilies({"default", "pikachu"}, options); + ASSERT_OK( + db_->GetStatsHistory(0, mock_clock_->NowSeconds() + 1, &stats_iter)); + ASSERT_TRUE(stats_iter != nullptr); + uint64_t new_recovered_value = 0; + for (int i = 2; stats_iter->Valid(); stats_iter->Next(), i++) { + auto stats_map = stats_iter->GetStatsMap(); + ASSERT_EQ(stats_iter->GetStatsTime(), kPeriodSec * i - 1); + for (const auto& stat : stats_map) { + if (sample.compare(stat.first) == 0) { + new_recovered_value += stat.second; + } + } + } + ASSERT_EQ(recovered_value, new_recovered_value); + + // TODO(Zhongyi): also add test to read raw values from disk and verify + // correctness + Close(); +} + +// TODO(Zhongyi): add test for different format versions + +TEST_F(StatsHistoryTest, PersistentStatsCreateColumnFamilies) { + constexpr int kPeriodSec = 5; + Options options; + options.create_if_missing = true; + options.stats_persist_period_sec = kPeriodSec; + options.statistics = CreateDBStatistics(); + options.persist_stats_to_disk = true; + options.env = mock_env_.get(); + ASSERT_OK(TryReopen(options)); + CreateColumnFamilies({"one", "two", "three"}, options); + ASSERT_OK(Put(1, "foo", "bar")); + ReopenWithColumnFamilies({"default", "one", "two", "three"}, options); + ASSERT_EQ(Get(2, "foo"), "bar"); + CreateColumnFamilies({"four"}, options); + ReopenWithColumnFamilies({"default", "one", "two", "three", "four"}, options); + ASSERT_EQ(Get(2, "foo"), "bar"); + + // make sure the first stats persist to finish + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); + + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); + auto iter = + db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily()); + int key_count = countkeys(iter); + delete iter; + ASSERT_GE(key_count, 0); + uint64_t num_write_wal = 0; + std::string sample = "rocksdb.write.wal"; + std::unique_ptr stats_iter; + ASSERT_OK(db_->GetStatsHistory(0, mock_clock_->NowSeconds(), &stats_iter)); + ASSERT_TRUE(stats_iter != nullptr); + for (; stats_iter->Valid(); stats_iter->Next()) { + auto stats_map = stats_iter->GetStatsMap(); + for (const auto& stat : stats_map) { + if (sample.compare(stat.first) == 0) { + num_write_wal += stat.second; + } + } + } + stats_iter.reset(); + ASSERT_EQ(num_write_wal, 1); + + options.persist_stats_to_disk = false; + ReopenWithColumnFamilies({"default", "one", "two", "three", "four"}, options); + int cf_count = 0; + for (auto cfd : *dbfull()->versions_->GetColumnFamilySet()) { + (void)cfd; + cf_count++; + } + // persistent stats cf will be implicitly opened even if + // persist_stats_to_disk is false + ASSERT_EQ(cf_count, 6); + ASSERT_EQ(Get(2, "foo"), "bar"); + + // attempt to create column family using same name, should fail + ColumnFamilyOptions cf_opts(options); + ColumnFamilyHandle* handle; + ASSERT_NOK(db_->CreateColumnFamily(cf_opts, kPersistentStatsColumnFamilyName, + &handle)); + + options.persist_stats_to_disk = true; + ReopenWithColumnFamilies({"default", "one", "two", "three", "four"}, options); + ASSERT_NOK(db_->CreateColumnFamily(cf_opts, kPersistentStatsColumnFamilyName, + &handle)); + // verify stats is not affected by prior failed CF creation + ASSERT_OK(db_->GetStatsHistory(0, mock_clock_->NowSeconds(), &stats_iter)); + ASSERT_TRUE(stats_iter != nullptr); + num_write_wal = 0; + for (; stats_iter->Valid(); stats_iter->Next()) { + auto stats_map = stats_iter->GetStatsMap(); + for (const auto& stat : stats_map) { + if (sample.compare(stat.first) == 0) { + num_write_wal += stat.second; + } + } + } + ASSERT_EQ(num_write_wal, 1); + + Close(); + Destroy(options); +} + +TEST_F(StatsHistoryTest, PersistentStatsReadOnly) { + ASSERT_OK(Put("bar", "v2")); + Close(); + + auto options = CurrentOptions(); + options.stats_persist_period_sec = 5; + options.persist_stats_to_disk = true; + assert(options.env == env_); + ASSERT_OK(ReadOnlyReopen(options)); + ASSERT_EQ("v2", Get("bar")); + Close(); + + // Reopen and flush memtable. + ASSERT_OK(TryReopen(options)); + ASSERT_OK(Flush()); + Close(); + // Now check keys in read only mode. + ASSERT_OK(ReadOnlyReopen(options)); +} + +TEST_F(StatsHistoryTest, ForceManualFlushStatsCF) { + constexpr int kPeriodSec = 5; + Options options; + options.create_if_missing = true; + options.write_buffer_size = 1024 * 1024 * 10; // 10 Mb + options.stats_persist_period_sec = kPeriodSec; + options.statistics = CreateDBStatistics(); + options.persist_stats_to_disk = true; + options.env = mock_env_.get(); + CreateColumnFamilies({"pikachu"}, options); + ReopenWithColumnFamilies({"default", "pikachu"}, options); + + // Wait for the first stats persist to finish, as the initial delay could be + // different. + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); }); + + ColumnFamilyData* cfd_default = + static_cast(dbfull()->DefaultColumnFamily()) + ->cfd(); + ColumnFamilyData* cfd_stats = static_cast( + dbfull()->PersistentStatsColumnFamily()) + ->cfd(); + ColumnFamilyData* cfd_test = + static_cast(handles_[1])->cfd(); + + ASSERT_OK(Put("foo", "v0")); + ASSERT_OK(Put("bar", "v0")); + ASSERT_EQ("v0", Get("bar")); + ASSERT_EQ("v0", Get("foo")); + ASSERT_OK(Put(1, "Eevee", "v0")); + ASSERT_EQ("v0", Get(1, "Eevee")); + + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); + // writing to all three cf, flush default cf + // LogNumbers: default: 16, stats: 10, pikachu: 5 + // Since in recovery process, cfd_stats column is created after WAL is + // created, synced and MANIFEST is persisted, its log number which depends on + // logfile_number_ will be different. Since "pikachu" is never flushed, thus + // its log_number should be the smallest of the three. + ASSERT_OK(Flush()); + ASSERT_LT(cfd_test->GetLogNumber(), cfd_stats->GetLogNumber()); + ASSERT_LT(cfd_test->GetLogNumber(), cfd_default->GetLogNumber()); + + ASSERT_OK(Put("foo1", "v1")); + ASSERT_OK(Put("bar1", "v1")); + ASSERT_EQ("v1", Get("bar1")); + ASSERT_EQ("v1", Get("foo1")); + ASSERT_OK(Put(1, "Vaporeon", "v1")); + ASSERT_EQ("v1", Get(1, "Vaporeon")); + // writing to default and test cf, flush test cf + // LogNumbers: default: 14, stats: 16, pikachu: 16 + ASSERT_OK(Flush(1)); + ASSERT_EQ(cfd_stats->GetLogNumber(), cfd_test->GetLogNumber()); + ASSERT_GT(cfd_stats->GetLogNumber(), cfd_default->GetLogNumber()); + + ASSERT_OK(Put("foo2", "v2")); + ASSERT_OK(Put("bar2", "v2")); + ASSERT_EQ("v2", Get("bar2")); + ASSERT_EQ("v2", Get("foo2")); + + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); + // writing to default and stats cf, flushing default cf + // LogNumbers: default: 19, stats: 19, pikachu: 19 + ASSERT_OK(Flush()); + ASSERT_EQ(cfd_stats->GetLogNumber(), cfd_test->GetLogNumber()); + ASSERT_EQ(cfd_stats->GetLogNumber(), cfd_default->GetLogNumber()); + + ASSERT_OK(Put("foo3", "v3")); + ASSERT_OK(Put("bar3", "v3")); + ASSERT_EQ("v3", Get("bar3")); + ASSERT_EQ("v3", Get("foo3")); + ASSERT_OK(Put(1, "Jolteon", "v3")); + ASSERT_EQ("v3", Get(1, "Jolteon")); + + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); }); + // writing to all three cf, flushing test cf + // LogNumbers: default: 19, stats: 19, pikachu: 22 + ASSERT_OK(Flush(1)); + ASSERT_LT(cfd_stats->GetLogNumber(), cfd_test->GetLogNumber()); + ASSERT_EQ(cfd_stats->GetLogNumber(), cfd_default->GetLogNumber()); + Close(); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/monitoring/thread_status_impl.cc b/librocksdb-sys/rocksdb/monitoring/thread_status_impl.cc new file mode 100644 index 0000000..9619dfd --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/thread_status_impl.cc @@ -0,0 +1,163 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#include + +#include "rocksdb/env.h" +#include "rocksdb/thread_status.h" +#include "util/string_util.h" +#include "util/thread_operation.h" + +namespace ROCKSDB_NAMESPACE { + +#ifdef ROCKSDB_USING_THREAD_STATUS +std::string ThreadStatus::GetThreadTypeName( + ThreadStatus::ThreadType thread_type) { + switch (thread_type) { + case ThreadStatus::ThreadType::HIGH_PRIORITY: + return "High Pri"; + case ThreadStatus::ThreadType::LOW_PRIORITY: + return "Low Pri"; + case ThreadStatus::ThreadType::USER: + return "User"; + case ThreadStatus::ThreadType::BOTTOM_PRIORITY: + return "Bottom Pri"; + case ThreadStatus::ThreadType::NUM_THREAD_TYPES: + assert(false); + } + return "Unknown"; +} + +const std::string& ThreadStatus::GetOperationName( + ThreadStatus::OperationType op_type) { + if (op_type < 0 || op_type >= NUM_OP_TYPES) { + return global_operation_table[OP_UNKNOWN].name; + } + return global_operation_table[op_type].name; +} + +const std::string& ThreadStatus::GetOperationStageName( + ThreadStatus::OperationStage stage) { + if (stage < 0 || stage >= NUM_OP_STAGES) { + return global_op_stage_table[STAGE_UNKNOWN].name; + } + return global_op_stage_table[stage].name; +} + +const std::string& ThreadStatus::GetStateName( + ThreadStatus::StateType state_type) { + if (state_type < 0 || state_type >= NUM_STATE_TYPES) { + return global_state_table[STATE_UNKNOWN].name; + } + return global_state_table[state_type].name; +} + +const std::string ThreadStatus::MicrosToString(uint64_t micros) { + if (micros == 0) { + return ""; + } + const int kBufferLen = 100; + char buffer[kBufferLen]; + AppendHumanMicros(micros, buffer, kBufferLen, false); + return std::string(buffer); +} + +const std::string& ThreadStatus::GetOperationPropertyName( + ThreadStatus::OperationType op_type, int i) { + static const std::string empty_str = ""; + switch (op_type) { + case ThreadStatus::OP_COMPACTION: + if (i >= NUM_COMPACTION_PROPERTIES) { + return empty_str; + } + return compaction_operation_properties[i].name; + case ThreadStatus::OP_FLUSH: + if (i >= NUM_FLUSH_PROPERTIES) { + return empty_str; + } + return flush_operation_properties[i].name; + default: + return empty_str; + } +} + +std::map ThreadStatus::InterpretOperationProperties( + ThreadStatus::OperationType op_type, const uint64_t* op_properties) { + int num_properties; + switch (op_type) { + case OP_COMPACTION: + num_properties = NUM_COMPACTION_PROPERTIES; + break; + case OP_FLUSH: + num_properties = NUM_FLUSH_PROPERTIES; + break; + default: + num_properties = 0; + } + + std::map property_map; + for (int i = 0; i < num_properties; ++i) { + if (op_type == OP_COMPACTION && i == COMPACTION_INPUT_OUTPUT_LEVEL) { + property_map.insert({"BaseInputLevel", op_properties[i] >> 32}); + property_map.insert( + {"OutputLevel", op_properties[i] % (uint64_t(1) << 32U)}); + } else if (op_type == OP_COMPACTION && i == COMPACTION_PROP_FLAGS) { + property_map.insert({"IsManual", ((op_properties[i] & 2) >> 1)}); + property_map.insert({"IsDeletion", ((op_properties[i] & 4) >> 2)}); + property_map.insert({"IsTrivialMove", ((op_properties[i] & 8) >> 3)}); + } else { + property_map.insert( + {GetOperationPropertyName(op_type, i), op_properties[i]}); + } + } + return property_map; +} + +#else + +std::string ThreadStatus::GetThreadTypeName( + ThreadStatus::ThreadType /*thread_type*/) { + static std::string dummy_str = ""; + return dummy_str; +} + +const std::string& ThreadStatus::GetOperationName( + ThreadStatus::OperationType /*op_type*/) { + static std::string dummy_str = ""; + return dummy_str; +} + +const std::string& ThreadStatus::GetOperationStageName( + ThreadStatus::OperationStage /*stage*/) { + static std::string dummy_str = ""; + return dummy_str; +} + +const std::string& ThreadStatus::GetStateName( + ThreadStatus::StateType /*state_type*/) { + static std::string dummy_str = ""; + return dummy_str; +} + +const std::string ThreadStatus::MicrosToString(uint64_t /*op_elapsed_time*/) { + static std::string dummy_str = ""; + return dummy_str; +} + +const std::string& ThreadStatus::GetOperationPropertyName( + ThreadStatus::OperationType /*op_type*/, int /*i*/) { + static std::string dummy_str = ""; + return dummy_str; +} + +std::map ThreadStatus::InterpretOperationProperties( + ThreadStatus::OperationType /*op_type*/, + const uint64_t* /*op_properties*/) { + return std::map(); +} + +#endif // ROCKSDB_USING_THREAD_STATUS +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/thread_status_updater.cc b/librocksdb-sys/rocksdb/monitoring/thread_status_updater.cc new file mode 100644 index 0000000..37fcef6 --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/thread_status_updater.cc @@ -0,0 +1,328 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "monitoring/thread_status_updater.h" + +#include + +#include "port/likely.h" +#include "rocksdb/env.h" +#include "rocksdb/system_clock.h" +#include "util/mutexlock.h" + +namespace ROCKSDB_NAMESPACE { + +#ifdef ROCKSDB_USING_THREAD_STATUS + +thread_local ThreadStatusData* ThreadStatusUpdater::thread_status_data_ = + nullptr; + +void ThreadStatusUpdater::RegisterThread(ThreadStatus::ThreadType ttype, + uint64_t thread_id) { + if (UNLIKELY(thread_status_data_ == nullptr)) { + thread_status_data_ = new ThreadStatusData(); + thread_status_data_->thread_type = ttype; + thread_status_data_->thread_id = thread_id; + std::lock_guard lck(thread_list_mutex_); + thread_data_set_.insert(thread_status_data_); + } + + ClearThreadOperationProperties(); +} + +void ThreadStatusUpdater::UnregisterThread() { + if (thread_status_data_ != nullptr) { + std::lock_guard lck(thread_list_mutex_); + thread_data_set_.erase(thread_status_data_); + delete thread_status_data_; + thread_status_data_ = nullptr; + } +} + +void ThreadStatusUpdater::ResetThreadStatus() { + ClearThreadState(); + ClearThreadOperation(); + SetColumnFamilyInfoKey(nullptr); +} + +void ThreadStatusUpdater::SetEnableTracking(bool enable_tracking) { + auto* data = Get(); + if (data == nullptr) { + return; + } + data->enable_tracking.store(enable_tracking, std::memory_order_relaxed); +} + +void ThreadStatusUpdater::SetColumnFamilyInfoKey(const void* cf_key) { + auto* data = Get(); + if (data == nullptr) { + return; + } + data->cf_key.store(const_cast(cf_key), std::memory_order_relaxed); +} + +const void* ThreadStatusUpdater::GetColumnFamilyInfoKey() { + auto* data = GetLocalThreadStatus(); + if (data == nullptr) { + return nullptr; + } + return data->cf_key.load(std::memory_order_relaxed); +} + +void ThreadStatusUpdater::SetThreadOperation( + const ThreadStatus::OperationType type) { + auto* data = GetLocalThreadStatus(); + if (data == nullptr) { + return; + } + // NOTE: Our practice here is to set all the thread operation properties + // and stage before we set thread operation, and thread operation + // will be set in std::memory_order_release. This is to ensure + // whenever a thread operation is not OP_UNKNOWN, we will always + // have a consistent information on its properties. + data->operation_type.store(type, std::memory_order_release); + if (type == ThreadStatus::OP_UNKNOWN) { + data->operation_stage.store(ThreadStatus::STAGE_UNKNOWN, + std::memory_order_relaxed); + ClearThreadOperationProperties(); + } +} + +ThreadStatus::OperationType ThreadStatusUpdater::GetThreadOperation() { + ThreadStatusData* data = GetLocalThreadStatus(); + if (data == nullptr) { + return ThreadStatus::OperationType::OP_UNKNOWN; + } + return data->operation_type.load(std::memory_order_relaxed); +} + +void ThreadStatusUpdater::SetThreadOperationProperty(int i, uint64_t value) { + auto* data = GetLocalThreadStatus(); + if (data == nullptr) { + return; + } + data->op_properties[i].store(value, std::memory_order_relaxed); +} + +void ThreadStatusUpdater::IncreaseThreadOperationProperty(int i, + uint64_t delta) { + auto* data = GetLocalThreadStatus(); + if (data == nullptr) { + return; + } + data->op_properties[i].fetch_add(delta, std::memory_order_relaxed); +} + +void ThreadStatusUpdater::SetOperationStartTime(const uint64_t start_time) { + auto* data = GetLocalThreadStatus(); + if (data == nullptr) { + return; + } + data->op_start_time.store(start_time, std::memory_order_relaxed); +} + +void ThreadStatusUpdater::ClearThreadOperation() { + auto* data = GetLocalThreadStatus(); + if (data == nullptr) { + return; + } + data->operation_stage.store(ThreadStatus::STAGE_UNKNOWN, + std::memory_order_relaxed); + data->operation_type.store(ThreadStatus::OP_UNKNOWN, + std::memory_order_relaxed); + ClearThreadOperationProperties(); +} + +void ThreadStatusUpdater::ClearThreadOperationProperties() { + auto* data = GetLocalThreadStatus(); + if (data == nullptr) { + return; + } + for (int i = 0; i < ThreadStatus::kNumOperationProperties; ++i) { + data->op_properties[i].store(0, std::memory_order_relaxed); + } +} + +ThreadStatus::OperationStage ThreadStatusUpdater::SetThreadOperationStage( + ThreadStatus::OperationStage stage) { + auto* data = GetLocalThreadStatus(); + if (data == nullptr) { + return ThreadStatus::STAGE_UNKNOWN; + } + return data->operation_stage.exchange(stage, std::memory_order_relaxed); +} + +void ThreadStatusUpdater::SetThreadState(const ThreadStatus::StateType type) { + auto* data = GetLocalThreadStatus(); + if (data == nullptr) { + return; + } + data->state_type.store(type, std::memory_order_relaxed); +} + +void ThreadStatusUpdater::ClearThreadState() { + auto* data = GetLocalThreadStatus(); + if (data == nullptr) { + return; + } + data->state_type.store(ThreadStatus::STATE_UNKNOWN, + std::memory_order_relaxed); +} + +Status ThreadStatusUpdater::GetThreadList( + std::vector* thread_list) { + thread_list->clear(); + std::vector> valid_list; + uint64_t now_micros = SystemClock::Default()->NowMicros(); + + std::lock_guard lck(thread_list_mutex_); + for (auto* thread_data : thread_data_set_) { + assert(thread_data); + auto thread_id = thread_data->thread_id.load(std::memory_order_relaxed); + auto thread_type = thread_data->thread_type.load(std::memory_order_relaxed); + // Since any change to cf_info_map requires thread_list_mutex, + // which is currently held by GetThreadList(), here we can safely + // use "memory_order_relaxed" to load the cf_key. + auto cf_key = thread_data->cf_key.load(std::memory_order_relaxed); + + ThreadStatus::OperationType op_type = ThreadStatus::OP_UNKNOWN; + ThreadStatus::OperationStage op_stage = ThreadStatus::STAGE_UNKNOWN; + ThreadStatus::StateType state_type = ThreadStatus::STATE_UNKNOWN; + uint64_t op_elapsed_micros = 0; + uint64_t op_props[ThreadStatus::kNumOperationProperties] = {0}; + + auto iter = cf_info_map_.find(cf_key); + if (iter != cf_info_map_.end()) { + op_type = thread_data->operation_type.load(std::memory_order_acquire); + // display lower-level info only when higher-level info is available. + if (op_type != ThreadStatus::OP_UNKNOWN) { + op_elapsed_micros = now_micros - thread_data->op_start_time.load( + std::memory_order_relaxed); + op_stage = thread_data->operation_stage.load(std::memory_order_relaxed); + state_type = thread_data->state_type.load(std::memory_order_relaxed); + for (int i = 0; i < ThreadStatus::kNumOperationProperties; ++i) { + op_props[i] = + thread_data->op_properties[i].load(std::memory_order_relaxed); + } + } + } + + thread_list->emplace_back( + thread_id, thread_type, + iter != cf_info_map_.end() ? iter->second.db_name : "", + iter != cf_info_map_.end() ? iter->second.cf_name : "", op_type, + op_elapsed_micros, op_stage, op_props, state_type); + } + + return Status::OK(); +} + +ThreadStatusData* ThreadStatusUpdater::GetLocalThreadStatus() { + if (thread_status_data_ == nullptr) { + return nullptr; + } + if (!thread_status_data_->enable_tracking.load(std::memory_order_relaxed)) { + return nullptr; + } + return thread_status_data_; +} + +void ThreadStatusUpdater::NewColumnFamilyInfo(const void* db_key, + const std::string& db_name, + const void* cf_key, + const std::string& cf_name) { + // Acquiring same lock as GetThreadList() to guarantee + // a consistent view of global column family table (cf_info_map). + std::lock_guard lck(thread_list_mutex_); + + cf_info_map_.emplace(std::piecewise_construct, std::make_tuple(cf_key), + std::make_tuple(db_key, db_name, cf_name)); + db_key_map_[db_key].insert(cf_key); +} + +void ThreadStatusUpdater::EraseColumnFamilyInfo(const void* cf_key) { + // Acquiring same lock as GetThreadList() to guarantee + // a consistent view of global column family table (cf_info_map). + std::lock_guard lck(thread_list_mutex_); + + auto cf_pair = cf_info_map_.find(cf_key); + if (cf_pair != cf_info_map_.end()) { + // Remove its entry from db_key_map_ by the following steps: + // 1. Obtain the entry in db_key_map_ whose set contains cf_key + // 2. Remove it from the set. + ConstantColumnFamilyInfo& cf_info = cf_pair->second; + auto db_pair = db_key_map_.find(cf_info.db_key); + assert(db_pair != db_key_map_.end()); + size_t result __attribute__((__unused__)); + result = db_pair->second.erase(cf_key); + assert(result); + cf_info_map_.erase(cf_pair); + } +} + +void ThreadStatusUpdater::EraseDatabaseInfo(const void* db_key) { + // Acquiring same lock as GetThreadList() to guarantee + // a consistent view of global column family table (cf_info_map). + std::lock_guard lck(thread_list_mutex_); + auto db_pair = db_key_map_.find(db_key); + if (UNLIKELY(db_pair == db_key_map_.end())) { + // In some occasional cases such as DB::Open fails, we won't + // register ColumnFamilyInfo for a db. + return; + } + + for (auto cf_key : db_pair->second) { + auto cf_pair = cf_info_map_.find(cf_key); + if (cf_pair != cf_info_map_.end()) { + cf_info_map_.erase(cf_pair); + } + } + db_key_map_.erase(db_key); +} + +#else + +void ThreadStatusUpdater::RegisterThread(ThreadStatus::ThreadType /*ttype*/, + uint64_t /*thread_id*/) {} + +void ThreadStatusUpdater::UnregisterThread() {} + +void ThreadStatusUpdater::ResetThreadStatus() {} + +void ThreadStatusUpdater::SetColumnFamilyInfoKey(const void* /*cf_key*/) {} + +void ThreadStatusUpdater::SetThreadOperation( + const ThreadStatus::OperationType /*type*/) {} + +void ThreadStatusUpdater::ClearThreadOperation() {} + +void ThreadStatusUpdater::SetThreadState( + const ThreadStatus::StateType /*type*/) {} + +void ThreadStatusUpdater::ClearThreadState() {} + +Status ThreadStatusUpdater::GetThreadList( + std::vector* /*thread_list*/) { + return Status::NotSupported( + "GetThreadList is not supported in the current running environment."); +} + +void ThreadStatusUpdater::NewColumnFamilyInfo(const void* /*db_key*/, + const std::string& /*db_name*/, + const void* /*cf_key*/, + const std::string& /*cf_name*/) {} + +void ThreadStatusUpdater::EraseColumnFamilyInfo(const void* /*cf_key*/) {} + +void ThreadStatusUpdater::EraseDatabaseInfo(const void* /*db_key*/) {} + +void ThreadStatusUpdater::SetThreadOperationProperty(int /*i*/, + uint64_t /*value*/) {} + +void ThreadStatusUpdater::IncreaseThreadOperationProperty(int /*i*/, + uint64_t /*delta*/) {} + +#endif // ROCKSDB_USING_THREAD_STATUS +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/thread_status_updater.h b/librocksdb-sys/rocksdb/monitoring/thread_status_updater.h new file mode 100644 index 0000000..696063c --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/thread_status_updater.h @@ -0,0 +1,226 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// The implementation of ThreadStatus. +// +// Note that we make get and set access to ThreadStatusData lockless. +// As a result, ThreadStatusData as a whole is not atomic. However, +// we guarantee consistent ThreadStatusData all the time whenever +// user call GetThreadList(). This consistency guarantee is done +// by having the following constraint in the internal implementation +// of set and get order: +// +// 1. When reset any information in ThreadStatusData, always start from +// clearing up the lower-level information first. +// 2. When setting any information in ThreadStatusData, always start from +// setting the higher-level information. +// 3. When returning ThreadStatusData to the user, fields are fetched from +// higher-level to lower-level. In addition, where there's a nullptr +// in one field, then all fields that has lower-level than that field +// should be ignored. +// +// The high to low level information would be: +// thread_id > thread_type > db > cf > operation > state +// +// This means user might not always get full information, but whenever +// returned by the GetThreadList() is guaranteed to be consistent. +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include + +#include "port/port.h" +#include "rocksdb/status.h" +#include "rocksdb/thread_status.h" +#include "util/thread_operation.h" + +namespace ROCKSDB_NAMESPACE { + +class ColumnFamilyHandle; + +// The structure that keeps constant information about a column family. +struct ConstantColumnFamilyInfo { +#ifdef ROCKSDB_USING_THREAD_STATUS + public: + ConstantColumnFamilyInfo(const void* _db_key, const std::string& _db_name, + const std::string& _cf_name) + : db_key(_db_key), db_name(_db_name), cf_name(_cf_name) {} + const void* db_key; + const std::string db_name; + const std::string cf_name; +#endif // ROCKSDB_USING_THREAD_STATUS +}; + +// the internal data-structure that is used to reflect the current +// status of a thread using a set of atomic pointers. +struct ThreadStatusData { +#ifdef ROCKSDB_USING_THREAD_STATUS + explicit ThreadStatusData() { + enable_tracking.store(false); + thread_id.store(0); + thread_type.store(ThreadStatus::USER); + cf_key.store(nullptr); + operation_type.store(ThreadStatus::OP_UNKNOWN); + op_start_time.store(0); + state_type.store(ThreadStatus::STATE_UNKNOWN); + } + + // A flag to indicate whether the thread tracking is enabled + // in the current thread. + // If set to false, then SetThreadOperation and SetThreadState + // will be no-op. + std::atomic enable_tracking; + + std::atomic thread_id; + std::atomic thread_type; + std::atomic cf_key; + std::atomic operation_type; + std::atomic op_start_time; + std::atomic operation_stage; + std::atomic op_properties[ThreadStatus::kNumOperationProperties]; + std::atomic state_type; +#endif // ROCKSDB_USING_THREAD_STATUS +}; + +// The class that stores and updates the status of the current thread +// using a thread-local ThreadStatusData. +// +// In most of the case, you should use ThreadStatusUtil to update +// the status of the current thread instead of using ThreadSatusUpdater +// directly. +// +// @see ThreadStatusUtil +class ThreadStatusUpdater { + public: + ThreadStatusUpdater() {} + + // Releases all ThreadStatusData of all active threads. + virtual ~ThreadStatusUpdater() {} + + // Unregister the current thread. + void UnregisterThread(); + + // Reset the status of the current thread. This includes resetting + // ColumnFamilyInfoKey, ThreadOperation, and ThreadState. + void ResetThreadStatus(); + + // Set the id of the current thread. + void SetThreadID(uint64_t thread_id); + + // Register the current thread for tracking. + void RegisterThread(ThreadStatus::ThreadType ttype, uint64_t thread_id); + + void SetEnableTracking(bool enable_tracking); + + // Update the column-family info of the current thread by setting + // its thread-local pointer of ThreadStatusData to the correct entry. + void SetColumnFamilyInfoKey(const void* cf_key); + + // returns the column family info key. + const void* GetColumnFamilyInfoKey(); + + // Update the thread operation of the current thread. + void SetThreadOperation(const ThreadStatus::OperationType type); + + // Return the thread operation of the current thread. + ThreadStatus::OperationType GetThreadOperation(); + + // The start time of the current thread operation. It is in the format + // of micro-seconds since some fixed point in time. + void SetOperationStartTime(const uint64_t start_time); + + // Set the "i"th property of the current operation. + // + // NOTE: Our practice here is to set all the thread operation properties + // and stage before we set thread operation, and thread operation + // will be set in std::memory_order_release. This is to ensure + // whenever a thread operation is not OP_UNKNOWN, we will always + // have a consistent information on its properties. + void SetThreadOperationProperty(int i, uint64_t value); + + // Increase the "i"th property of the current operation with + // the specified delta. + void IncreaseThreadOperationProperty(int i, uint64_t delta); + + // Update the thread operation stage of the current thread. + ThreadStatus::OperationStage SetThreadOperationStage( + const ThreadStatus::OperationStage stage); + + // Clear thread operation of the current thread. + void ClearThreadOperation(); + + // Reset all thread-operation-properties to 0. + void ClearThreadOperationProperties(); + + // Update the thread state of the current thread. + void SetThreadState(const ThreadStatus::StateType type); + + // Clear the thread state of the current thread. + void ClearThreadState(); + + // Obtain the status of all active registered threads. + Status GetThreadList(std::vector* thread_list); + + // Create an entry in the global ColumnFamilyInfo table for the + // specified column family. This function should be called only + // when the current thread does not hold db_mutex. + void NewColumnFamilyInfo(const void* db_key, const std::string& db_name, + const void* cf_key, const std::string& cf_name); + + // Erase all ConstantColumnFamilyInfo that is associated with the + // specified db instance. This function should be called only when + // the current thread does not hold db_mutex. + void EraseDatabaseInfo(const void* db_key); + + // Erase the ConstantColumnFamilyInfo that is associated with the + // specified ColumnFamilyData. This function should be called only + // when the current thread does not hold db_mutex. + void EraseColumnFamilyInfo(const void* cf_key); + + // Verifies whether the input ColumnFamilyHandles matches + // the information stored in the current cf_info_map. + void TEST_VerifyColumnFamilyInfoMap( + const std::vector& handles, bool check_exist); + + protected: +#ifdef ROCKSDB_USING_THREAD_STATUS + // The thread-local variable for storing thread status. + static thread_local ThreadStatusData* thread_status_data_; + + // Returns the pointer to the thread status data only when the + // thread status data is non-null and has enable_tracking == true. + ThreadStatusData* GetLocalThreadStatus(); + + // Directly returns the pointer to thread_status_data_ without + // checking whether enabling_tracking is true of not. + ThreadStatusData* Get() { return thread_status_data_; } + + // The mutex that protects cf_info_map and db_key_map. + std::mutex thread_list_mutex_; + + // The current status data of all active threads. + std::unordered_set thread_data_set_; + + // A global map that keeps the column family information. It is stored + // globally instead of inside DB is to avoid the situation where DB is + // closing while GetThreadList function already get the pointer to its + // CopnstantColumnFamilyInfo. + std::unordered_map cf_info_map_; + + // A db_key to cf_key map that allows erasing elements in cf_info_map + // associated to the same db_key faster. + std::unordered_map> db_key_map_; + +#else + static ThreadStatusData* thread_status_data_; +#endif // ROCKSDB_USING_THREAD_STATUS +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/thread_status_updater_debug.cc b/librocksdb-sys/rocksdb/monitoring/thread_status_updater_debug.cc new file mode 100644 index 0000000..464c23b --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/thread_status_updater_debug.cc @@ -0,0 +1,43 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include + +#include "db/column_family.h" +#include "monitoring/thread_status_updater.h" +#include "util/cast_util.h" + +namespace ROCKSDB_NAMESPACE { + +#ifndef NDEBUG +#ifdef ROCKSDB_USING_THREAD_STATUS +void ThreadStatusUpdater::TEST_VerifyColumnFamilyInfoMap( + const std::vector& handles, bool check_exist) { + std::unique_lock lock(thread_list_mutex_); + if (check_exist) { + assert(cf_info_map_.size() == handles.size()); + } + for (auto* handle : handles) { + auto* cfd = static_cast_with_check(handle)->cfd(); + auto iter __attribute__((__unused__)) = cf_info_map_.find(cfd); + if (check_exist) { + assert(iter != cf_info_map_.end()); + assert(iter->second.cf_name == cfd->GetName()); + } else { + assert(iter == cf_info_map_.end()); + } + } +} + +#else + +void ThreadStatusUpdater::TEST_VerifyColumnFamilyInfoMap( + const std::vector& /*handles*/, bool /*check_exist*/) { +} + +#endif // ROCKSDB_USING_THREAD_STATUS +#endif // !NDEBUG + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/thread_status_util.cc b/librocksdb-sys/rocksdb/monitoring/thread_status_util.cc new file mode 100644 index 0000000..9b66dc2 --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/thread_status_util.cc @@ -0,0 +1,208 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "monitoring/thread_status_util.h" + +#include "monitoring/thread_status_updater.h" +#include "rocksdb/env.h" +#include "rocksdb/system_clock.h" + +namespace ROCKSDB_NAMESPACE { + +#ifdef ROCKSDB_USING_THREAD_STATUS +thread_local ThreadStatusUpdater* + ThreadStatusUtil::thread_updater_local_cache_ = nullptr; +thread_local bool ThreadStatusUtil::thread_updater_initialized_ = false; + +void ThreadStatusUtil::RegisterThread(const Env* env, + ThreadStatus::ThreadType thread_type) { + if (!MaybeInitThreadLocalUpdater(env)) { + return; + } + assert(thread_updater_local_cache_); + thread_updater_local_cache_->RegisterThread(thread_type, env->GetThreadID()); +} + +void ThreadStatusUtil::UnregisterThread() { + thread_updater_initialized_ = false; + if (thread_updater_local_cache_ != nullptr) { + thread_updater_local_cache_->UnregisterThread(); + thread_updater_local_cache_ = nullptr; + } +} + +void ThreadStatusUtil::SetEnableTracking(bool enable_tracking) { + if (thread_updater_local_cache_ == nullptr) { + return; + } + thread_updater_local_cache_->SetEnableTracking(enable_tracking); +} + +void ThreadStatusUtil::SetColumnFamily(const ColumnFamilyData* cfd) { + if (thread_updater_local_cache_ == nullptr) { + return; + } + assert(cfd); + thread_updater_local_cache_->SetColumnFamilyInfoKey(cfd); +} + +void ThreadStatusUtil::SetThreadOperation(ThreadStatus::OperationType op) { + if (thread_updater_local_cache_ == nullptr) { + return; + } + + if (op != ThreadStatus::OP_UNKNOWN) { + uint64_t current_time = SystemClock::Default()->NowMicros(); + thread_updater_local_cache_->SetOperationStartTime(current_time); + } else { + // TDOO(yhchiang): we could report the time when we set operation to + // OP_UNKNOWN once the whole instrumentation has been done. + thread_updater_local_cache_->SetOperationStartTime(0); + } + thread_updater_local_cache_->SetThreadOperation(op); +} + +ThreadStatus::OperationType ThreadStatusUtil::GetThreadOperation() { + if (thread_updater_local_cache_ == nullptr) { + return ThreadStatus::OperationType::OP_UNKNOWN; + } + return thread_updater_local_cache_->GetThreadOperation(); +} + +ThreadStatus::OperationStage ThreadStatusUtil::SetThreadOperationStage( + ThreadStatus::OperationStage stage) { + if (thread_updater_local_cache_ == nullptr) { + // thread_updater_local_cache_ must be set in SetColumnFamily + // or other ThreadStatusUtil functions. + return ThreadStatus::STAGE_UNKNOWN; + } + + return thread_updater_local_cache_->SetThreadOperationStage(stage); +} + +void ThreadStatusUtil::SetThreadOperationProperty(int code, uint64_t value) { + if (thread_updater_local_cache_ == nullptr) { + // thread_updater_local_cache_ must be set in SetColumnFamily + // or other ThreadStatusUtil functions. + return; + } + + thread_updater_local_cache_->SetThreadOperationProperty(code, value); +} + +void ThreadStatusUtil::IncreaseThreadOperationProperty(int code, + uint64_t delta) { + if (thread_updater_local_cache_ == nullptr) { + // thread_updater_local_cache_ must be set in SetColumnFamily + // or other ThreadStatusUtil functions. + return; + } + + thread_updater_local_cache_->IncreaseThreadOperationProperty(code, delta); +} + +void ThreadStatusUtil::SetThreadState(ThreadStatus::StateType state) { + if (thread_updater_local_cache_ == nullptr) { + // thread_updater_local_cache_ must be set in SetColumnFamily + // or other ThreadStatusUtil functions. + return; + } + + thread_updater_local_cache_->SetThreadState(state); +} + +void ThreadStatusUtil::ResetThreadStatus() { + if (thread_updater_local_cache_ == nullptr) { + return; + } + thread_updater_local_cache_->ResetThreadStatus(); +} + +void ThreadStatusUtil::NewColumnFamilyInfo(const DB* db, + const ColumnFamilyData* cfd, + const std::string& cf_name, + const Env* env) { + if (!MaybeInitThreadLocalUpdater(env)) { + return; + } + assert(thread_updater_local_cache_); + if (thread_updater_local_cache_) { + thread_updater_local_cache_->NewColumnFamilyInfo(db, db->GetName(), cfd, + cf_name); + } +} + +void ThreadStatusUtil::EraseColumnFamilyInfo(const ColumnFamilyData* cfd) { + if (thread_updater_local_cache_ == nullptr) { + return; + } + thread_updater_local_cache_->EraseColumnFamilyInfo(cfd); +} + +void ThreadStatusUtil::EraseDatabaseInfo(const DB* db) { + ThreadStatusUpdater* thread_updater = db->GetEnv()->GetThreadStatusUpdater(); + if (thread_updater == nullptr) { + return; + } + thread_updater->EraseDatabaseInfo(db); +} + +bool ThreadStatusUtil::MaybeInitThreadLocalUpdater(const Env* env) { + if (!thread_updater_initialized_ && env != nullptr) { + thread_updater_initialized_ = true; + thread_updater_local_cache_ = env->GetThreadStatusUpdater(); + } + return (thread_updater_local_cache_ != nullptr); +} + +AutoThreadOperationStageUpdater::AutoThreadOperationStageUpdater( + ThreadStatus::OperationStage stage) { + prev_stage_ = ThreadStatusUtil::SetThreadOperationStage(stage); +} + +AutoThreadOperationStageUpdater::~AutoThreadOperationStageUpdater() { + ThreadStatusUtil::SetThreadOperationStage(prev_stage_); +} + +#else + +ThreadStatusUpdater* ThreadStatusUtil::thread_updater_local_cache_ = nullptr; +bool ThreadStatusUtil::thread_updater_initialized_ = false; + +bool ThreadStatusUtil::MaybeInitThreadLocalUpdater(const Env* /*env*/) { + return false; +} + +void ThreadStatusUtil::SetColumnFamily(const ColumnFamilyData* /*cfd*/) {} + +void ThreadStatusUtil::SetThreadOperation(ThreadStatus::OperationType /*op*/) {} + +void ThreadStatusUtil::SetThreadOperationProperty(int /*code*/, + uint64_t /*value*/) {} + +void ThreadStatusUtil::IncreaseThreadOperationProperty(int /*code*/, + uint64_t /*delta*/) {} + +void ThreadStatusUtil::SetThreadState(ThreadStatus::StateType /*state*/) {} + +void ThreadStatusUtil::NewColumnFamilyInfo(const DB* /*db*/, + const ColumnFamilyData* /*cfd*/, + const std::string& /*cf_name*/, + const Env* env) {} + +void ThreadStatusUtil::EraseColumnFamilyInfo(const ColumnFamilyData* /*cfd*/) {} + +void ThreadStatusUtil::EraseDatabaseInfo(const DB* /*db*/) {} + +void ThreadStatusUtil::ResetThreadStatus() {} + +AutoThreadOperationStageUpdater::AutoThreadOperationStageUpdater( + ThreadStatus::OperationStage /*stage*/) {} + +AutoThreadOperationStageUpdater::~AutoThreadOperationStageUpdater() {} + +#endif // ROCKSDB_USING_THREAD_STATUS + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/thread_status_util.h b/librocksdb-sys/rocksdb/monitoring/thread_status_util.h new file mode 100644 index 0000000..df148a0 --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/thread_status_util.h @@ -0,0 +1,139 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "monitoring/thread_status_updater.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/thread_status.h" + +namespace ROCKSDB_NAMESPACE { + +class ColumnFamilyData; + +// The static utility class for updating thread-local status. +// +// The thread-local status is updated via the thread-local cached +// pointer thread_updater_local_cache_. During each function call, +// when ThreadStatusUtil finds thread_updater_local_cache_ is +// left uninitialized (determined by thread_updater_initialized_), +// it will tries to initialize it using the return value of +// Env::GetThreadStatusUpdater(). When thread_updater_local_cache_ +// is initialized by a non-null pointer, each function call will +// then update the status of the current thread. Otherwise, +// all function calls to ThreadStatusUtil will be no-op. +class ThreadStatusUtil { + public: + // Register the current thread for tracking. + static void RegisterThread(const Env* env, + ThreadStatus::ThreadType thread_type); + + // Unregister the current thread. + static void UnregisterThread(); + + // Create an entry in the global ColumnFamilyInfo table for the + // specified column family. This function should be called only + // when the current thread does not hold db_mutex. + static void NewColumnFamilyInfo(const DB* db, const ColumnFamilyData* cfd, + const std::string& cf_name, const Env* env); + + // Erase the ConstantColumnFamilyInfo that is associated with the + // specified ColumnFamilyData. This function should be called only + // when the current thread does not hold db_mutex. + static void EraseColumnFamilyInfo(const ColumnFamilyData* cfd); + + // Erase all ConstantColumnFamilyInfo that is associated with the + // specified db instance. This function should be called only when + // the current thread does not hold db_mutex. + static void EraseDatabaseInfo(const DB* db); + + static void SetEnableTracking(bool enable_tracking); + + // Update the thread status to indicate the current thread is doing + // something related to the specified column family. + // + // REQUIRES: cfd != nullptr + static void SetColumnFamily(const ColumnFamilyData* cfd); + + static void SetThreadOperation(ThreadStatus::OperationType type); + + static ThreadStatus::OperationType GetThreadOperation(); + + static ThreadStatus::OperationStage SetThreadOperationStage( + ThreadStatus::OperationStage stage); + + static void SetThreadOperationProperty(int code, uint64_t value); + + static void IncreaseThreadOperationProperty(int code, uint64_t delta); + + static void SetThreadState(ThreadStatus::StateType type); + + static void ResetThreadStatus(); + +#ifndef NDEBUG + static void TEST_SetStateDelay(const ThreadStatus::StateType state, + int micro); + static void TEST_StateDelay(const ThreadStatus::StateType state); + + static Env::IOActivity TEST_GetExpectedIOActivity( + ThreadStatus::OperationType thread_op); +#endif + + protected: + // Initialize the thread-local ThreadStatusUpdater when it finds + // the cached value is nullptr. Returns true if it has cached + // a non-null pointer. + static bool MaybeInitThreadLocalUpdater(const Env* env); + +#ifdef ROCKSDB_USING_THREAD_STATUS + // A boolean flag indicating whether thread_updater_local_cache_ + // is initialized. It is set to true when an Env uses any + // ThreadStatusUtil functions using the current thread other + // than UnregisterThread(). It will be set to false when + // UnregisterThread() is called. + // + // When this variable is set to true, thread_updater_local_cache_ + // will not be updated until this variable is again set to false + // in UnregisterThread(). + static thread_local bool thread_updater_initialized_; + + // The thread-local cached ThreadStatusUpdater that caches the + // thread_status_updater_ of the first Env that uses any ThreadStatusUtil + // function other than UnregisterThread(). This variable will + // be cleared when UnregisterThread() is called. + // + // When this variable is set to a non-null pointer, then the status + // of the current thread will be updated when a function of + // ThreadStatusUtil is called. Otherwise, all functions of + // ThreadStatusUtil will be no-op. + // + // When thread_updater_initialized_ is set to true, this variable + // will not be updated until this thread_updater_initialized_ is + // again set to false in UnregisterThread(). + static thread_local ThreadStatusUpdater* thread_updater_local_cache_; +#else + static bool thread_updater_initialized_; + static ThreadStatusUpdater* thread_updater_local_cache_; +#endif +}; + +// A helper class for updating thread state. It will set the +// thread state according to the input parameter in its constructor +// and set the thread state to the previous state in its destructor. +class AutoThreadOperationStageUpdater { + public: + explicit AutoThreadOperationStageUpdater(ThreadStatus::OperationStage stage); + ~AutoThreadOperationStageUpdater(); + +#ifdef ROCKSDB_USING_THREAD_STATUS + private: + ThreadStatus::OperationStage prev_stage_; +#endif +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/monitoring/thread_status_util_debug.cc b/librocksdb-sys/rocksdb/monitoring/thread_status_util_debug.cc new file mode 100644 index 0000000..6e4fe8a --- /dev/null +++ b/librocksdb-sys/rocksdb/monitoring/thread_status_util_debug.cc @@ -0,0 +1,46 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include + +#include "monitoring/thread_status_updater.h" +#include "monitoring/thread_status_util.h" +#include "rocksdb/system_clock.h" + +namespace ROCKSDB_NAMESPACE { + +#ifndef NDEBUG +// the delay for debugging purpose. +static std::atomic states_delay[ThreadStatus::NUM_STATE_TYPES]; + +void ThreadStatusUtil::TEST_SetStateDelay(const ThreadStatus::StateType state, + int micro) { + states_delay[state].store(micro, std::memory_order_relaxed); +} + +void ThreadStatusUtil::TEST_StateDelay(const ThreadStatus::StateType state) { + auto delay = states_delay[state].load(std::memory_order_relaxed); + if (delay > 0) { + SystemClock::Default()->SleepForMicroseconds(delay); + } +} + +Env::IOActivity ThreadStatusUtil::TEST_GetExpectedIOActivity( + ThreadStatus::OperationType thread_op) { + switch (thread_op) { + case ThreadStatus::OperationType::OP_FLUSH: + return Env::IOActivity::kFlush; + case ThreadStatus::OperationType::OP_COMPACTION: + return Env::IOActivity::kCompaction; + case ThreadStatus::OperationType::OP_DBOPEN: + return Env::IOActivity::kDBOpen; + default: + return Env::IOActivity::kUnknown; + } +} + +#endif // !NDEBUG + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/options/cf_options.cc b/librocksdb-sys/rocksdb/options/cf_options.cc new file mode 100644 index 0000000..a425c18 --- /dev/null +++ b/librocksdb-sys/rocksdb/options/cf_options.cc @@ -0,0 +1,1201 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "options/cf_options.h" + +#include +#include +#include +#include + +#include "logging/logging.h" +#include "options/configurable_helper.h" +#include "options/db_options.h" +#include "options/options_helper.h" +#include "options/options_parser.h" +#include "port/port.h" +#include "rocksdb/advanced_cache.h" +#include "rocksdb/compaction_filter.h" +#include "rocksdb/concurrent_task_limiter.h" +#include "rocksdb/configurable.h" +#include "rocksdb/convenience.h" +#include "rocksdb/env.h" +#include "rocksdb/file_system.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/options.h" +#include "rocksdb/table.h" +#include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/options_type.h" +#include "util/cast_util.h" + +// NOTE: in this file, many option flags that were deprecated +// and removed from the rest of the code have to be kept here +// and marked as kDeprecated in order to be able to read old +// OPTIONS files. + +namespace ROCKSDB_NAMESPACE { + +static Status ParseCompressionOptions(const std::string& value, + const std::string& name, + CompressionOptions& compression_opts) { + const char kDelimiter = ':'; + std::istringstream field_stream(value); + std::string field; + + if (!std::getline(field_stream, field, kDelimiter)) { + return Status::InvalidArgument("unable to parse the specified CF option " + + name); + } + compression_opts.window_bits = ParseInt(field); + + if (!std::getline(field_stream, field, kDelimiter)) { + return Status::InvalidArgument("unable to parse the specified CF option " + + name); + } + compression_opts.level = ParseInt(field); + + if (!std::getline(field_stream, field, kDelimiter)) { + return Status::InvalidArgument("unable to parse the specified CF option " + + name); + } + compression_opts.strategy = ParseInt(field); + + // max_dict_bytes is optional for backwards compatibility + if (!field_stream.eof()) { + if (!std::getline(field_stream, field, kDelimiter)) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + compression_opts.max_dict_bytes = ParseInt(field); + } + + // zstd_max_train_bytes is optional for backwards compatibility + if (!field_stream.eof()) { + if (!std::getline(field_stream, field, kDelimiter)) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + compression_opts.zstd_max_train_bytes = ParseInt(field); + } + + // parallel_threads is optional for backwards compatibility + if (!field_stream.eof()) { + if (!std::getline(field_stream, field, kDelimiter)) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + // Since parallel_threads comes before enabled but was added optionally + // later, we need to check if this is the final token (meaning it is the + // enabled bit), or if there are more tokens (meaning this one is + // parallel_threads). + if (!field_stream.eof()) { + compression_opts.parallel_threads = ParseInt(field); + } else { + // parallel_threads is not serialized with this format, but enabled is + compression_opts.enabled = ParseBoolean("", field); + } + } + + // enabled is optional for backwards compatibility + if (!field_stream.eof()) { + if (!std::getline(field_stream, field, kDelimiter)) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + compression_opts.enabled = ParseBoolean("", field); + } + + // max_dict_buffer_bytes is optional for backwards compatibility + if (!field_stream.eof()) { + if (!std::getline(field_stream, field, kDelimiter)) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + compression_opts.max_dict_buffer_bytes = ParseUint64(field); + } + + // use_zstd_dict_trainer is optional for backwards compatibility + if (!field_stream.eof()) { + if (!std::getline(field_stream, field, kDelimiter)) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + compression_opts.use_zstd_dict_trainer = ParseBoolean("", field); + } + + if (!field_stream.eof()) { + return Status::InvalidArgument("unable to parse the specified CF option " + + name); + } + return Status::OK(); +} + +const std::string kOptNameBMCompOpts = "bottommost_compression_opts"; +const std::string kOptNameCompOpts = "compression_opts"; + +// OptionTypeInfo map for CompressionOptions +static std::unordered_map + compression_options_type_info = { + {"window_bits", + {offsetof(struct CompressionOptions, window_bits), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, + {"level", + {offsetof(struct CompressionOptions, level), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, + {"strategy", + {offsetof(struct CompressionOptions, strategy), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, + {"max_compressed_bytes_per_kb", + {offsetof(struct CompressionOptions, max_compressed_bytes_per_kb), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"max_dict_bytes", + {offsetof(struct CompressionOptions, max_dict_bytes), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, + {"zstd_max_train_bytes", + {offsetof(struct CompressionOptions, zstd_max_train_bytes), + OptionType::kUInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"parallel_threads", + {offsetof(struct CompressionOptions, parallel_threads), + OptionType::kUInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"enabled", + {offsetof(struct CompressionOptions, enabled), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, + {"max_dict_buffer_bytes", + {offsetof(struct CompressionOptions, max_dict_buffer_bytes), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"use_zstd_dict_trainer", + {offsetof(struct CompressionOptions, use_zstd_dict_trainer), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, +}; + +static std::unordered_map + file_temperature_age_type_info = { + {"temperature", + {offsetof(struct FileTemperatureAge, temperature), + OptionType::kTemperature, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"age", + {offsetof(struct FileTemperatureAge, age), OptionType::kUInt64T, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, +}; + +static std::unordered_map + fifo_compaction_options_type_info = { + {"max_table_files_size", + {offsetof(struct CompactionOptionsFIFO, max_table_files_size), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"age_for_warm", + {offsetof(struct CompactionOptionsFIFO, age_for_warm), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"ttl", + {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"allow_compaction", + {offsetof(struct CompactionOptionsFIFO, allow_compaction), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"file_temperature_age_thresholds", + OptionTypeInfo::Vector( + offsetof(struct CompactionOptionsFIFO, + file_temperature_age_thresholds), + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + OptionTypeInfo::Struct("file_temperature_age_thresholds", + &file_temperature_age_type_info, 0, + OptionVerificationType::kNormal, + OptionTypeFlags::kMutable))}}; + +static std::unordered_map + universal_compaction_options_type_info = { + {"size_ratio", + {offsetof(class CompactionOptionsUniversal, size_ratio), + OptionType::kUInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"min_merge_width", + {offsetof(class CompactionOptionsUniversal, min_merge_width), + OptionType::kUInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"max_merge_width", + {offsetof(class CompactionOptionsUniversal, max_merge_width), + OptionType::kUInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"max_size_amplification_percent", + {offsetof(class CompactionOptionsUniversal, + max_size_amplification_percent), + OptionType::kUInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"compression_size_percent", + {offsetof(class CompactionOptionsUniversal, compression_size_percent), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"stop_style", + {offsetof(class CompactionOptionsUniversal, stop_style), + OptionType::kCompactionStopStyle, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"incremental", + {offsetof(class CompactionOptionsUniversal, incremental), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"allow_trivial_move", + {offsetof(class CompactionOptionsUniversal, allow_trivial_move), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}}; + +static std::unordered_map + cf_mutable_options_type_info = { + {"report_bg_io_stats", + {offsetof(struct MutableCFOptions, report_bg_io_stats), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"disable_auto_compactions", + {offsetof(struct MutableCFOptions, disable_auto_compactions), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"filter_deletes", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable}}, + {"check_flush_compaction_key_order", + {offsetof(struct MutableCFOptions, check_flush_compaction_key_order), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"paranoid_file_checks", + {offsetof(struct MutableCFOptions, paranoid_file_checks), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"verify_checksums_in_compaction", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable}}, + {"soft_pending_compaction_bytes_limit", + {offsetof(struct MutableCFOptions, + soft_pending_compaction_bytes_limit), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"hard_pending_compaction_bytes_limit", + {offsetof(struct MutableCFOptions, + hard_pending_compaction_bytes_limit), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"hard_rate_limit", + {0, OptionType::kDouble, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable}}, + {"soft_rate_limit", + {0, OptionType::kDouble, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable}}, + {"max_compaction_bytes", + {offsetof(struct MutableCFOptions, max_compaction_bytes), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"ignore_max_compaction_bytes_for_input", + {offsetof(struct MutableCFOptions, + ignore_max_compaction_bytes_for_input), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"expanded_compaction_factor", + {0, OptionType::kInt, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable}}, + {"level0_file_num_compaction_trigger", + {offsetof(struct MutableCFOptions, level0_file_num_compaction_trigger), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"level0_slowdown_writes_trigger", + {offsetof(struct MutableCFOptions, level0_slowdown_writes_trigger), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"level0_stop_writes_trigger", + {offsetof(struct MutableCFOptions, level0_stop_writes_trigger), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"max_grandparent_overlap_factor", + {0, OptionType::kInt, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable}}, + {"max_write_buffer_number", + {offsetof(struct MutableCFOptions, max_write_buffer_number), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"source_compaction_factor", + {0, OptionType::kInt, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable}}, + {"target_file_size_multiplier", + {offsetof(struct MutableCFOptions, target_file_size_multiplier), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"arena_block_size", + {offsetof(struct MutableCFOptions, arena_block_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"inplace_update_num_locks", + {offsetof(struct MutableCFOptions, inplace_update_num_locks), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"max_successive_merges", + {offsetof(struct MutableCFOptions, max_successive_merges), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"memtable_huge_page_size", + {offsetof(struct MutableCFOptions, memtable_huge_page_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"memtable_prefix_bloom_huge_page_tlb_size", + {0, OptionType::kSizeT, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable}}, + {"write_buffer_size", + {offsetof(struct MutableCFOptions, write_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"memtable_prefix_bloom_bits", + {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable}}, + {"memtable_prefix_bloom_size_ratio", + {offsetof(struct MutableCFOptions, memtable_prefix_bloom_size_ratio), + OptionType::kDouble, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"memtable_prefix_bloom_probes", + {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable}}, + {"memtable_whole_key_filtering", + {offsetof(struct MutableCFOptions, memtable_whole_key_filtering), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"min_partial_merge_operands", + {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable}}, + {"max_bytes_for_level_base", + {offsetof(struct MutableCFOptions, max_bytes_for_level_base), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"snap_refresh_nanos", + {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable}}, + {"max_bytes_for_level_multiplier", + {offsetof(struct MutableCFOptions, max_bytes_for_level_multiplier), + OptionType::kDouble, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"max_bytes_for_level_multiplier_additional", + OptionTypeInfo::Vector( + offsetof(struct MutableCFOptions, + max_bytes_for_level_multiplier_additional), + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + {0, OptionType::kInt})}, + {"max_sequential_skip_in_iterations", + {offsetof(struct MutableCFOptions, max_sequential_skip_in_iterations), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"target_file_size_base", + {offsetof(struct MutableCFOptions, target_file_size_base), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"compression", + {offsetof(struct MutableCFOptions, compression), + OptionType::kCompressionType, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"prefix_extractor", + OptionTypeInfo::AsCustomSharedPtr( + offsetof(struct MutableCFOptions, prefix_extractor), + OptionVerificationType::kByNameAllowNull, + (OptionTypeFlags::kMutable | OptionTypeFlags::kAllowNull))}, + {"compaction_options_fifo", + OptionTypeInfo::Struct( + "compaction_options_fifo", &fifo_compaction_options_type_info, + offsetof(struct MutableCFOptions, compaction_options_fifo), + OptionVerificationType::kNormal, OptionTypeFlags::kMutable) + .SetParseFunc([](const ConfigOptions& opts, + const std::string& name, const std::string& value, + void* addr) { + // This is to handle backward compatibility, where + // compaction_options_fifo could be assigned a single scalar + // value, say, like "23", which would be assigned to + // max_table_files_size. + if (name == "compaction_options_fifo" && + value.find("=") == std::string::npos) { + // Old format. Parse just a single uint64_t value. + auto options = static_cast(addr); + options->max_table_files_size = ParseUint64(value); + return Status::OK(); + } else { + return OptionTypeInfo::ParseStruct( + opts, "compaction_options_fifo", + &fifo_compaction_options_type_info, name, value, addr); + } + })}, + {"compaction_options_universal", + OptionTypeInfo::Struct( + "compaction_options_universal", + &universal_compaction_options_type_info, + offsetof(struct MutableCFOptions, compaction_options_universal), + OptionVerificationType::kNormal, OptionTypeFlags::kMutable)}, + {"ttl", + {offsetof(struct MutableCFOptions, ttl), OptionType::kUInt64T, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, + {"periodic_compaction_seconds", + {offsetof(struct MutableCFOptions, periodic_compaction_seconds), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"bottommost_temperature", + {0, OptionType::kTemperature, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable}}, + {"last_level_temperature", + {offsetof(struct MutableCFOptions, last_level_temperature), + OptionType::kTemperature, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"enable_blob_files", + {offsetof(struct MutableCFOptions, enable_blob_files), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"min_blob_size", + {offsetof(struct MutableCFOptions, min_blob_size), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"blob_file_size", + {offsetof(struct MutableCFOptions, blob_file_size), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"blob_compression_type", + {offsetof(struct MutableCFOptions, blob_compression_type), + OptionType::kCompressionType, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"enable_blob_garbage_collection", + {offsetof(struct MutableCFOptions, enable_blob_garbage_collection), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"blob_garbage_collection_age_cutoff", + {offsetof(struct MutableCFOptions, blob_garbage_collection_age_cutoff), + OptionType::kDouble, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"blob_garbage_collection_force_threshold", + {offsetof(struct MutableCFOptions, + blob_garbage_collection_force_threshold), + OptionType::kDouble, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"blob_compaction_readahead_size", + {offsetof(struct MutableCFOptions, blob_compaction_readahead_size), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"blob_file_starting_level", + {offsetof(struct MutableCFOptions, blob_file_starting_level), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"prepopulate_blob_cache", + OptionTypeInfo::Enum( + offsetof(struct MutableCFOptions, prepopulate_blob_cache), + &prepopulate_blob_cache_string_map, OptionTypeFlags::kMutable)}, + {"sample_for_compression", + {offsetof(struct MutableCFOptions, sample_for_compression), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"bottommost_compression", + {offsetof(struct MutableCFOptions, bottommost_compression), + OptionType::kCompressionType, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"compression_per_level", + OptionTypeInfo::Vector( + offsetof(struct MutableCFOptions, compression_per_level), + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + {0, OptionType::kCompressionType})}, + {"experimental_mempurge_threshold", + {offsetof(struct MutableCFOptions, experimental_mempurge_threshold), + OptionType::kDouble, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"memtable_protection_bytes_per_key", + {offsetof(struct MutableCFOptions, memtable_protection_bytes_per_key), + OptionType::kUInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"block_protection_bytes_per_key", + {offsetof(struct MutableCFOptions, block_protection_bytes_per_key), + OptionType::kUInt8T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {kOptNameCompOpts, + OptionTypeInfo::Struct( + kOptNameCompOpts, &compression_options_type_info, + offsetof(struct MutableCFOptions, compression_opts), + OptionVerificationType::kNormal, + (OptionTypeFlags::kMutable | OptionTypeFlags::kCompareNever), + [](const ConfigOptions& opts, const std::string& name, + const std::string& value, void* addr) { + // This is to handle backward compatibility, where + // compression_options was a ":" separated list. + if (name == kOptNameCompOpts && + value.find("=") == std::string::npos) { + auto* compression = static_cast(addr); + return ParseCompressionOptions(value, name, *compression); + } else { + return OptionTypeInfo::ParseStruct( + opts, kOptNameCompOpts, &compression_options_type_info, + name, value, addr); + } + })}, + {kOptNameBMCompOpts, + OptionTypeInfo::Struct( + kOptNameBMCompOpts, &compression_options_type_info, + offsetof(struct MutableCFOptions, bottommost_compression_opts), + OptionVerificationType::kNormal, + (OptionTypeFlags::kMutable | OptionTypeFlags::kCompareNever), + [](const ConfigOptions& opts, const std::string& name, + const std::string& value, void* addr) { + // This is to handle backward compatibility, where + // compression_options was a ":" separated list. + if (name == kOptNameBMCompOpts && + value.find("=") == std::string::npos) { + auto* compression = static_cast(addr); + return ParseCompressionOptions(value, name, *compression); + } else { + return OptionTypeInfo::ParseStruct( + opts, kOptNameBMCompOpts, &compression_options_type_info, + name, value, addr); + } + })}, + // End special case properties + {"memtable_max_range_deletions", + {offsetof(struct MutableCFOptions, memtable_max_range_deletions), + OptionType::kUInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + +}; + +static std::unordered_map + cf_immutable_options_type_info = { + /* not yet supported + CompressionOptions compression_opts; + TablePropertiesCollectorFactories table_properties_collector_factories; + using TablePropertiesCollectorFactories = + std::vector>; + UpdateStatus (*inplace_callback)(char* existing_value, + uint34_t* existing_value_size, + Slice delta_value, + std::string* merged_value); + std::vector cf_paths; + */ + {"compaction_measure_io_stats", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"purge_redundant_kvs_while_flush", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"inplace_update_support", + {offsetof(struct ImmutableCFOptions, inplace_update_support), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"level_compaction_dynamic_level_bytes", + {offsetof(struct ImmutableCFOptions, + level_compaction_dynamic_level_bytes), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"level_compaction_dynamic_file_size", + {offsetof(struct ImmutableCFOptions, + level_compaction_dynamic_file_size), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"optimize_filters_for_hits", + {offsetof(struct ImmutableCFOptions, optimize_filters_for_hits), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"force_consistency_checks", + {offsetof(struct ImmutableCFOptions, force_consistency_checks), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"preclude_last_level_data_seconds", + {offsetof(struct ImmutableCFOptions, preclude_last_level_data_seconds), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"preserve_internal_time_seconds", + {offsetof(struct ImmutableCFOptions, preserve_internal_time_seconds), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + // Need to keep this around to be able to read old OPTIONS files. + {"max_mem_compaction_level", + {0, OptionType::kInt, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"max_write_buffer_number_to_maintain", + {offsetof(struct ImmutableCFOptions, + max_write_buffer_number_to_maintain), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"max_write_buffer_size_to_maintain", + {offsetof(struct ImmutableCFOptions, + max_write_buffer_size_to_maintain), + OptionType::kInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"min_write_buffer_number_to_merge", + {offsetof(struct ImmutableCFOptions, min_write_buffer_number_to_merge), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"num_levels", + {offsetof(struct ImmutableCFOptions, num_levels), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"bloom_locality", + {offsetof(struct ImmutableCFOptions, bloom_locality), + OptionType::kUInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"rate_limit_delay_max_milliseconds", + {0, OptionType::kUInt, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"comparator", + OptionTypeInfo::AsCustomRawPtr( + offsetof(struct ImmutableCFOptions, user_comparator), + OptionVerificationType::kByName, OptionTypeFlags::kCompareLoose) + .SetSerializeFunc( + // Serializes a Comparator + [](const ConfigOptions& opts, const std::string&, + const void* addr, std::string* value) { + // it's a const pointer of const Comparator* + const auto* ptr = + static_cast(addr); + // Since the user-specified comparator will be wrapped by + // InternalKeyComparator, we should persist the + // user-specified one instead of InternalKeyComparator. + if (*ptr == nullptr) { + *value = kNullptrString; + } else if (opts.mutable_options_only) { + *value = ""; + } else { + const Comparator* root_comp = (*ptr)->GetRootComparator(); + if (root_comp == nullptr) { + root_comp = (*ptr); + } + *value = root_comp->ToString(opts); + } + return Status::OK(); + })}, + {"memtable_insert_with_hint_prefix_extractor", + OptionTypeInfo::AsCustomSharedPtr( + offsetof(struct ImmutableCFOptions, + memtable_insert_with_hint_prefix_extractor), + OptionVerificationType::kByNameAllowNull, OptionTypeFlags::kNone)}, + {"memtable_factory", + {offsetof(struct ImmutableCFOptions, memtable_factory), + OptionType::kCustomizable, OptionVerificationType::kByName, + OptionTypeFlags::kShared, + [](const ConfigOptions& opts, const std::string&, + const std::string& value, void* addr) { + std::unique_ptr factory; + auto* shared = + static_cast*>(addr); + Status s = + MemTableRepFactory::CreateFromString(opts, value, shared); + return s; + }}}, + {"memtable", + {offsetof(struct ImmutableCFOptions, memtable_factory), + OptionType::kCustomizable, OptionVerificationType::kAlias, + OptionTypeFlags::kShared, + [](const ConfigOptions& opts, const std::string&, + const std::string& value, void* addr) { + std::unique_ptr factory; + auto* shared = + static_cast*>(addr); + Status s = + MemTableRepFactory::CreateFromString(opts, value, shared); + return s; + }}}, + {"table_factory", + OptionTypeInfo::AsCustomSharedPtr( + offsetof(struct ImmutableCFOptions, table_factory), + OptionVerificationType::kByName, + (OptionTypeFlags::kCompareLoose | + OptionTypeFlags::kStringNameOnly | + OptionTypeFlags::kDontPrepare))}, + {"block_based_table_factory", + {offsetof(struct ImmutableCFOptions, table_factory), + OptionType::kCustomizable, OptionVerificationType::kAlias, + OptionTypeFlags::kShared | OptionTypeFlags::kCompareLoose, + // Parses the input value and creates a BlockBasedTableFactory + [](const ConfigOptions& opts, const std::string& name, + const std::string& value, void* addr) { + BlockBasedTableOptions* old_opts = nullptr; + auto table_factory = + static_cast*>(addr); + if (table_factory->get() != nullptr) { + old_opts = + table_factory->get()->GetOptions(); + } + if (name == "block_based_table_factory") { + std::unique_ptr new_factory; + if (old_opts != nullptr) { + new_factory.reset(NewBlockBasedTableFactory(*old_opts)); + } else { + new_factory.reset(NewBlockBasedTableFactory()); + } + Status s = new_factory->ConfigureFromString(opts, value); + if (s.ok()) { + table_factory->reset(new_factory.release()); + } + return s; + } else if (old_opts != nullptr) { + return table_factory->get()->ConfigureOption(opts, name, value); + } else { + return Status::NotFound("Mismatched table option: ", name); + } + }}}, + {"plain_table_factory", + {offsetof(struct ImmutableCFOptions, table_factory), + OptionType::kCustomizable, OptionVerificationType::kAlias, + OptionTypeFlags::kShared | OptionTypeFlags::kCompareLoose, + // Parses the input value and creates a PlainTableFactory + [](const ConfigOptions& opts, const std::string& name, + const std::string& value, void* addr) { + PlainTableOptions* old_opts = nullptr; + auto table_factory = + static_cast*>(addr); + if (table_factory->get() != nullptr) { + old_opts = table_factory->get()->GetOptions(); + } + if (name == "plain_table_factory") { + std::unique_ptr new_factory; + if (old_opts != nullptr) { + new_factory.reset(NewPlainTableFactory(*old_opts)); + } else { + new_factory.reset(NewPlainTableFactory()); + } + Status s = new_factory->ConfigureFromString(opts, value); + if (s.ok()) { + table_factory->reset(new_factory.release()); + } + return s; + } else if (old_opts != nullptr) { + return table_factory->get()->ConfigureOption(opts, name, value); + } else { + return Status::NotFound("Mismatched table option: ", name); + } + }}}, + {"table_properties_collectors", + OptionTypeInfo::Vector< + std::shared_ptr>( + offsetof(struct ImmutableCFOptions, + table_properties_collector_factories), + OptionVerificationType::kByName, OptionTypeFlags::kNone, + OptionTypeInfo::AsCustomSharedPtr( + 0, OptionVerificationType::kByName, OptionTypeFlags::kNone))}, + {"compaction_filter", + OptionTypeInfo::AsCustomRawPtr( + offsetof(struct ImmutableCFOptions, compaction_filter), + OptionVerificationType::kByName, OptionTypeFlags::kAllowNull)}, + {"compaction_filter_factory", + OptionTypeInfo::AsCustomSharedPtr( + offsetof(struct ImmutableCFOptions, compaction_filter_factory), + OptionVerificationType::kByName, OptionTypeFlags::kAllowNull)}, + {"merge_operator", + OptionTypeInfo::AsCustomSharedPtr( + offsetof(struct ImmutableCFOptions, merge_operator), + OptionVerificationType::kByNameAllowFromNull, + OptionTypeFlags::kCompareLoose | OptionTypeFlags::kAllowNull)}, + {"compaction_style", + {offsetof(struct ImmutableCFOptions, compaction_style), + OptionType::kCompactionStyle, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"compaction_pri", + {offsetof(struct ImmutableCFOptions, compaction_pri), + OptionType::kCompactionPri, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"sst_partitioner_factory", + OptionTypeInfo::AsCustomSharedPtr( + offsetof(struct ImmutableCFOptions, sst_partitioner_factory), + OptionVerificationType::kByName, OptionTypeFlags::kAllowNull)}, + {"blob_cache", + {offsetof(struct ImmutableCFOptions, blob_cache), OptionType::kUnknown, + OptionVerificationType::kNormal, + (OptionTypeFlags::kCompareNever | OptionTypeFlags::kDontSerialize), + // Parses the input value as a Cache + [](const ConfigOptions& opts, const std::string&, + const std::string& value, void* addr) { + auto* cache = static_cast*>(addr); + return Cache::CreateFromString(opts, value, cache); + }}}, + {"persist_user_defined_timestamps", + {offsetof(struct ImmutableCFOptions, persist_user_defined_timestamps), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kCompareLoose}}, +}; + +const std::string OptionsHelper::kCFOptionsName = "ColumnFamilyOptions"; + +class ConfigurableMutableCFOptions : public Configurable { + public: + explicit ConfigurableMutableCFOptions(const MutableCFOptions& mcf) { + mutable_ = mcf; + RegisterOptions(&mutable_, &cf_mutable_options_type_info); + } + + protected: + MutableCFOptions mutable_; +}; + +class ConfigurableCFOptions : public ConfigurableMutableCFOptions { + public: + ConfigurableCFOptions(const ColumnFamilyOptions& opts, + const std::unordered_map* map) + : ConfigurableMutableCFOptions(MutableCFOptions(opts)), + immutable_(opts), + cf_options_(opts), + opt_map_(map) { + RegisterOptions(&immutable_, &cf_immutable_options_type_info); + } + + protected: + Status ConfigureOptions( + const ConfigOptions& config_options, + const std::unordered_map& opts_map, + std::unordered_map* unused) override { + Status s = Configurable::ConfigureOptions(config_options, opts_map, unused); + if (s.ok()) { + UpdateColumnFamilyOptions(mutable_, &cf_options_); + UpdateColumnFamilyOptions(immutable_, &cf_options_); + s = PrepareOptions(config_options); + } + return s; + } + + virtual const void* GetOptionsPtr(const std::string& name) const override { + if (name == OptionsHelper::kCFOptionsName) { + return &cf_options_; + } else { + return ConfigurableMutableCFOptions::GetOptionsPtr(name); + } + } + + bool OptionsAreEqual(const ConfigOptions& config_options, + const OptionTypeInfo& opt_info, + const std::string& opt_name, const void* const this_ptr, + const void* const that_ptr, + std::string* mismatch) const override { + bool equals = opt_info.AreEqual(config_options, opt_name, this_ptr, + that_ptr, mismatch); + if (!equals && opt_info.IsByName()) { + if (opt_map_ == nullptr) { + equals = true; + } else { + const auto& iter = opt_map_->find(opt_name); + if (iter == opt_map_->end()) { + equals = true; + } else { + equals = opt_info.AreEqualByName(config_options, opt_name, this_ptr, + iter->second); + } + } + if (equals) { // False alarm, clear mismatch + *mismatch = ""; + } + } + if (equals && opt_info.IsConfigurable() && opt_map_ != nullptr) { + const auto* this_config = opt_info.AsRawPointer(this_ptr); + if (this_config == nullptr) { + const auto& iter = opt_map_->find(opt_name); + // If the name exists in the map and is not empty/null, + // then the this_config should be set. + if (iter != opt_map_->end() && !iter->second.empty() && + iter->second != kNullptrString) { + *mismatch = opt_name; + equals = false; + } + } + } + return equals; + } + + private: + ImmutableCFOptions immutable_; + ColumnFamilyOptions cf_options_; + const std::unordered_map* opt_map_; +}; + +std::unique_ptr CFOptionsAsConfigurable( + const MutableCFOptions& opts) { + std::unique_ptr ptr(new ConfigurableMutableCFOptions(opts)); + return ptr; +} +std::unique_ptr CFOptionsAsConfigurable( + const ColumnFamilyOptions& opts, + const std::unordered_map* opt_map) { + std::unique_ptr ptr(new ConfigurableCFOptions(opts, opt_map)); + return ptr; +} + +ImmutableCFOptions::ImmutableCFOptions() : ImmutableCFOptions(Options()) {} + +ImmutableCFOptions::ImmutableCFOptions(const ColumnFamilyOptions& cf_options) + : compaction_style(cf_options.compaction_style), + compaction_pri(cf_options.compaction_pri), + user_comparator(cf_options.comparator), + internal_comparator(InternalKeyComparator(cf_options.comparator)), + merge_operator(cf_options.merge_operator), + compaction_filter(cf_options.compaction_filter), + compaction_filter_factory(cf_options.compaction_filter_factory), + min_write_buffer_number_to_merge( + cf_options.min_write_buffer_number_to_merge), + max_write_buffer_number_to_maintain( + cf_options.max_write_buffer_number_to_maintain), + max_write_buffer_size_to_maintain( + cf_options.max_write_buffer_size_to_maintain), + inplace_update_support(cf_options.inplace_update_support), + inplace_callback(cf_options.inplace_callback), + memtable_factory(cf_options.memtable_factory), + table_factory(cf_options.table_factory), + table_properties_collector_factories( + cf_options.table_properties_collector_factories), + bloom_locality(cf_options.bloom_locality), + level_compaction_dynamic_level_bytes( + cf_options.level_compaction_dynamic_level_bytes), + level_compaction_dynamic_file_size( + cf_options.level_compaction_dynamic_file_size), + num_levels(cf_options.num_levels), + optimize_filters_for_hits(cf_options.optimize_filters_for_hits), + force_consistency_checks(cf_options.force_consistency_checks), + preclude_last_level_data_seconds( + cf_options.preclude_last_level_data_seconds), + preserve_internal_time_seconds(cf_options.preserve_internal_time_seconds), + memtable_insert_with_hint_prefix_extractor( + cf_options.memtable_insert_with_hint_prefix_extractor), + cf_paths(cf_options.cf_paths), + compaction_thread_limiter(cf_options.compaction_thread_limiter), + sst_partitioner_factory(cf_options.sst_partitioner_factory), + blob_cache(cf_options.blob_cache), + persist_user_defined_timestamps( + cf_options.persist_user_defined_timestamps) {} + +ImmutableOptions::ImmutableOptions() : ImmutableOptions(Options()) {} + +ImmutableOptions::ImmutableOptions(const Options& options) + : ImmutableOptions(options, options) {} + +ImmutableOptions::ImmutableOptions(const DBOptions& db_options, + const ColumnFamilyOptions& cf_options) + : ImmutableDBOptions(db_options), ImmutableCFOptions(cf_options) {} + +ImmutableOptions::ImmutableOptions(const DBOptions& db_options, + const ImmutableCFOptions& cf_options) + : ImmutableDBOptions(db_options), ImmutableCFOptions(cf_options) {} + +ImmutableOptions::ImmutableOptions(const ImmutableDBOptions& db_options, + const ColumnFamilyOptions& cf_options) + : ImmutableDBOptions(db_options), ImmutableCFOptions(cf_options) {} + +ImmutableOptions::ImmutableOptions(const ImmutableDBOptions& db_options, + const ImmutableCFOptions& cf_options) + : ImmutableDBOptions(db_options), ImmutableCFOptions(cf_options) {} + +// Multiple two operands. If they overflow, return op1. +uint64_t MultiplyCheckOverflow(uint64_t op1, double op2) { + if (op1 == 0 || op2 <= 0) { + return 0; + } + if (std::numeric_limits::max() / op1 < op2) { + return op1; + } + return static_cast(op1 * op2); +} + +// when level_compaction_dynamic_level_bytes is true and leveled compaction +// is used, the base level is not always L1, so precomupted max_file_size can +// no longer be used. Recompute file_size_for_level from base level. +uint64_t MaxFileSizeForLevel(const MutableCFOptions& cf_options, + int level, CompactionStyle compaction_style, int base_level, + bool level_compaction_dynamic_level_bytes) { + if (!level_compaction_dynamic_level_bytes || level < base_level || + compaction_style != kCompactionStyleLevel) { + assert(level >= 0); + assert(level < (int)cf_options.max_file_size.size()); + return cf_options.max_file_size[level]; + } else { + assert(level >= 0 && base_level >= 0); + assert(level - base_level < (int)cf_options.max_file_size.size()); + return cf_options.max_file_size[level - base_level]; + } +} + +size_t MaxFileSizeForL0MetaPin(const MutableCFOptions& cf_options) { + // We do not want to pin meta-blocks that almost certainly came from intra-L0 + // or a former larger `write_buffer_size` value to avoid surprising users with + // pinned memory usage. We use a factor of 1.5 to account for overhead + // introduced during flush in most cases. + if (std::numeric_limits::max() / 3 < + cf_options.write_buffer_size / 2) { + return std::numeric_limits::max(); + } + return cf_options.write_buffer_size / 2 * 3; +} + +void MutableCFOptions::RefreshDerivedOptions(int num_levels, + CompactionStyle compaction_style) { + max_file_size.resize(num_levels); + for (int i = 0; i < num_levels; ++i) { + if (i == 0 && compaction_style == kCompactionStyleUniversal) { + max_file_size[i] = ULLONG_MAX; + } else if (i > 1) { + max_file_size[i] = MultiplyCheckOverflow(max_file_size[i - 1], + target_file_size_multiplier); + } else { + max_file_size[i] = target_file_size_base; + } + } +} + +void MutableCFOptions::Dump(Logger* log) const { + // Memtable related options + ROCKS_LOG_INFO(log, + " write_buffer_size: %" ROCKSDB_PRIszt, + write_buffer_size); + ROCKS_LOG_INFO(log, " max_write_buffer_number: %d", + max_write_buffer_number); + ROCKS_LOG_INFO(log, + " arena_block_size: %" ROCKSDB_PRIszt, + arena_block_size); + ROCKS_LOG_INFO(log, " memtable_prefix_bloom_ratio: %f", + memtable_prefix_bloom_size_ratio); + ROCKS_LOG_INFO(log, " memtable_whole_key_filtering: %d", + memtable_whole_key_filtering); + ROCKS_LOG_INFO(log, + " memtable_huge_page_size: %" ROCKSDB_PRIszt, + memtable_huge_page_size); + ROCKS_LOG_INFO(log, + " max_successive_merges: %" ROCKSDB_PRIszt, + max_successive_merges); + ROCKS_LOG_INFO(log, + " inplace_update_num_locks: %" ROCKSDB_PRIszt, + inplace_update_num_locks); + ROCKS_LOG_INFO(log, " prefix_extractor: %s", + prefix_extractor == nullptr + ? "nullptr" + : prefix_extractor->GetId().c_str()); + ROCKS_LOG_INFO(log, " disable_auto_compactions: %d", + disable_auto_compactions); + ROCKS_LOG_INFO(log, " soft_pending_compaction_bytes_limit: %" PRIu64, + soft_pending_compaction_bytes_limit); + ROCKS_LOG_INFO(log, " hard_pending_compaction_bytes_limit: %" PRIu64, + hard_pending_compaction_bytes_limit); + ROCKS_LOG_INFO(log, " level0_file_num_compaction_trigger: %d", + level0_file_num_compaction_trigger); + ROCKS_LOG_INFO(log, " level0_slowdown_writes_trigger: %d", + level0_slowdown_writes_trigger); + ROCKS_LOG_INFO(log, " level0_stop_writes_trigger: %d", + level0_stop_writes_trigger); + ROCKS_LOG_INFO(log, " max_compaction_bytes: %" PRIu64, + max_compaction_bytes); + ROCKS_LOG_INFO(log, " ignore_max_compaction_bytes_for_input: %s", + ignore_max_compaction_bytes_for_input ? "true" : "false"); + ROCKS_LOG_INFO(log, " target_file_size_base: %" PRIu64, + target_file_size_base); + ROCKS_LOG_INFO(log, " target_file_size_multiplier: %d", + target_file_size_multiplier); + ROCKS_LOG_INFO(log, " max_bytes_for_level_base: %" PRIu64, + max_bytes_for_level_base); + ROCKS_LOG_INFO(log, " max_bytes_for_level_multiplier: %f", + max_bytes_for_level_multiplier); + ROCKS_LOG_INFO(log, " ttl: %" PRIu64, + ttl); + ROCKS_LOG_INFO(log, " periodic_compaction_seconds: %" PRIu64, + periodic_compaction_seconds); + std::string result; + char buf[10]; + for (const auto m : max_bytes_for_level_multiplier_additional) { + snprintf(buf, sizeof(buf), "%d, ", m); + result += buf; + } + if (result.size() >= 2) { + result.resize(result.size() - 2); + } else { + result = ""; + } + + ROCKS_LOG_INFO(log, "max_bytes_for_level_multiplier_additional: %s", + result.c_str()); + ROCKS_LOG_INFO(log, " max_sequential_skip_in_iterations: %" PRIu64, + max_sequential_skip_in_iterations); + ROCKS_LOG_INFO(log, " check_flush_compaction_key_order: %d", + check_flush_compaction_key_order); + ROCKS_LOG_INFO(log, " paranoid_file_checks: %d", + paranoid_file_checks); + ROCKS_LOG_INFO(log, " report_bg_io_stats: %d", + report_bg_io_stats); + ROCKS_LOG_INFO(log, " compression: %d", + static_cast(compression)); + ROCKS_LOG_INFO(log, + " experimental_mempurge_threshold: %f", + experimental_mempurge_threshold); + + // Universal Compaction Options + ROCKS_LOG_INFO(log, "compaction_options_universal.size_ratio : %d", + compaction_options_universal.size_ratio); + ROCKS_LOG_INFO(log, "compaction_options_universal.min_merge_width : %d", + compaction_options_universal.min_merge_width); + ROCKS_LOG_INFO(log, "compaction_options_universal.max_merge_width : %d", + compaction_options_universal.max_merge_width); + ROCKS_LOG_INFO( + log, "compaction_options_universal.max_size_amplification_percent : %d", + compaction_options_universal.max_size_amplification_percent); + ROCKS_LOG_INFO(log, + "compaction_options_universal.compression_size_percent : %d", + compaction_options_universal.compression_size_percent); + ROCKS_LOG_INFO(log, "compaction_options_universal.stop_style : %d", + compaction_options_universal.stop_style); + ROCKS_LOG_INFO( + log, "compaction_options_universal.allow_trivial_move : %d", + static_cast(compaction_options_universal.allow_trivial_move)); + ROCKS_LOG_INFO(log, "compaction_options_universal.incremental : %d", + static_cast(compaction_options_universal.incremental)); + + // FIFO Compaction Options + ROCKS_LOG_INFO(log, "compaction_options_fifo.max_table_files_size : %" PRIu64, + compaction_options_fifo.max_table_files_size); + ROCKS_LOG_INFO(log, "compaction_options_fifo.allow_compaction : %d", + compaction_options_fifo.allow_compaction); + + // Blob file related options + ROCKS_LOG_INFO(log, " enable_blob_files: %s", + enable_blob_files ? "true" : "false"); + ROCKS_LOG_INFO(log, " min_blob_size: %" PRIu64, + min_blob_size); + ROCKS_LOG_INFO(log, " blob_file_size: %" PRIu64, + blob_file_size); + ROCKS_LOG_INFO(log, " blob_compression_type: %s", + CompressionTypeToString(blob_compression_type).c_str()); + ROCKS_LOG_INFO(log, " enable_blob_garbage_collection: %s", + enable_blob_garbage_collection ? "true" : "false"); + ROCKS_LOG_INFO(log, " blob_garbage_collection_age_cutoff: %f", + blob_garbage_collection_age_cutoff); + ROCKS_LOG_INFO(log, " blob_garbage_collection_force_threshold: %f", + blob_garbage_collection_force_threshold); + ROCKS_LOG_INFO(log, " blob_compaction_readahead_size: %" PRIu64, + blob_compaction_readahead_size); + ROCKS_LOG_INFO(log, " blob_file_starting_level: %d", + blob_file_starting_level); + ROCKS_LOG_INFO(log, " prepopulate_blob_cache: %s", + prepopulate_blob_cache == PrepopulateBlobCache::kFlushOnly + ? "flush only" + : "disable"); + ROCKS_LOG_INFO(log, " last_level_temperature: %d", + static_cast(last_level_temperature)); +} + +MutableCFOptions::MutableCFOptions(const Options& options) + : MutableCFOptions(ColumnFamilyOptions(options)) {} + +Status GetMutableOptionsFromStrings( + const MutableCFOptions& base_options, + const std::unordered_map& options_map, + Logger* /*info_log*/, MutableCFOptions* new_options) { + assert(new_options); + *new_options = base_options; + ConfigOptions config_options; + Status s = OptionTypeInfo::ParseType( + config_options, options_map, cf_mutable_options_type_info, new_options); + if (!s.ok()) { + *new_options = base_options; + } + return s; +} + +Status GetStringFromMutableCFOptions(const ConfigOptions& config_options, + const MutableCFOptions& mutable_opts, + std::string* opt_string) { + assert(opt_string); + opt_string->clear(); + return OptionTypeInfo::SerializeType( + config_options, cf_mutable_options_type_info, &mutable_opts, opt_string); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/options/cf_options.h b/librocksdb-sys/rocksdb/options/cf_options.h new file mode 100644 index 0000000..86de78d --- /dev/null +++ b/librocksdb-sys/rocksdb/options/cf_options.h @@ -0,0 +1,350 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include + +#include "db/dbformat.h" +#include "options/db_options.h" +#include "rocksdb/options.h" +#include "util/compression.h" + +namespace ROCKSDB_NAMESPACE { + +// ImmutableCFOptions is a data struct used by RocksDB internal. It contains a +// subset of Options that should not be changed during the entire lifetime +// of DB. Raw pointers defined in this struct do not have ownership to the data +// they point to. Options contains std::shared_ptr to these data. +struct ImmutableCFOptions { + public: + static const char* kName() { return "ImmutableCFOptions"; } + explicit ImmutableCFOptions(); + explicit ImmutableCFOptions(const ColumnFamilyOptions& cf_options); + + CompactionStyle compaction_style; + + CompactionPri compaction_pri; + + const Comparator* user_comparator; + InternalKeyComparator internal_comparator; // Only in Immutable + + std::shared_ptr merge_operator; + + const CompactionFilter* compaction_filter; + + std::shared_ptr compaction_filter_factory; + + int min_write_buffer_number_to_merge; + + int max_write_buffer_number_to_maintain; + + int64_t max_write_buffer_size_to_maintain; + + bool inplace_update_support; + + UpdateStatus (*inplace_callback)(char* existing_value, + uint32_t* existing_value_size, + Slice delta_value, + std::string* merged_value); + + std::shared_ptr memtable_factory; + + std::shared_ptr table_factory; + + Options::TablePropertiesCollectorFactories + table_properties_collector_factories; + + // This options is required by PlainTableReader. May need to move it + // to PlainTableOptions just like bloom_bits_per_key + uint32_t bloom_locality; + + bool level_compaction_dynamic_level_bytes; + + bool level_compaction_dynamic_file_size; + + int num_levels; + + bool optimize_filters_for_hits; + + bool force_consistency_checks; + + uint64_t preclude_last_level_data_seconds; + + uint64_t preserve_internal_time_seconds; + + std::shared_ptr + memtable_insert_with_hint_prefix_extractor; + + std::vector cf_paths; + + std::shared_ptr compaction_thread_limiter; + + std::shared_ptr sst_partitioner_factory; + + std::shared_ptr blob_cache; + + bool persist_user_defined_timestamps; +}; + +struct ImmutableOptions : public ImmutableDBOptions, public ImmutableCFOptions { + explicit ImmutableOptions(); + explicit ImmutableOptions(const Options& options); + + ImmutableOptions(const DBOptions& db_options, + const ColumnFamilyOptions& cf_options); + + ImmutableOptions(const ImmutableDBOptions& db_options, + const ImmutableCFOptions& cf_options); + + ImmutableOptions(const DBOptions& db_options, + const ImmutableCFOptions& cf_options); + + ImmutableOptions(const ImmutableDBOptions& db_options, + const ColumnFamilyOptions& cf_options); +}; + +struct MutableCFOptions { + static const char* kName() { return "MutableCFOptions"; } + explicit MutableCFOptions(const ColumnFamilyOptions& options) + : write_buffer_size(options.write_buffer_size), + max_write_buffer_number(options.max_write_buffer_number), + arena_block_size(options.arena_block_size), + memtable_prefix_bloom_size_ratio( + options.memtable_prefix_bloom_size_ratio), + memtable_whole_key_filtering(options.memtable_whole_key_filtering), + memtable_huge_page_size(options.memtable_huge_page_size), + max_successive_merges(options.max_successive_merges), + inplace_update_num_locks(options.inplace_update_num_locks), + prefix_extractor(options.prefix_extractor), + experimental_mempurge_threshold( + options.experimental_mempurge_threshold), + disable_auto_compactions(options.disable_auto_compactions), + soft_pending_compaction_bytes_limit( + options.soft_pending_compaction_bytes_limit), + hard_pending_compaction_bytes_limit( + options.hard_pending_compaction_bytes_limit), + level0_file_num_compaction_trigger( + options.level0_file_num_compaction_trigger), + level0_slowdown_writes_trigger(options.level0_slowdown_writes_trigger), + level0_stop_writes_trigger(options.level0_stop_writes_trigger), + max_compaction_bytes(options.max_compaction_bytes), + ignore_max_compaction_bytes_for_input( + options.ignore_max_compaction_bytes_for_input), + target_file_size_base(options.target_file_size_base), + target_file_size_multiplier(options.target_file_size_multiplier), + max_bytes_for_level_base(options.max_bytes_for_level_base), + max_bytes_for_level_multiplier(options.max_bytes_for_level_multiplier), + ttl(options.ttl), + periodic_compaction_seconds(options.periodic_compaction_seconds), + max_bytes_for_level_multiplier_additional( + options.max_bytes_for_level_multiplier_additional), + compaction_options_fifo(options.compaction_options_fifo), + compaction_options_universal(options.compaction_options_universal), + enable_blob_files(options.enable_blob_files), + min_blob_size(options.min_blob_size), + blob_file_size(options.blob_file_size), + blob_compression_type(options.blob_compression_type), + enable_blob_garbage_collection(options.enable_blob_garbage_collection), + blob_garbage_collection_age_cutoff( + options.blob_garbage_collection_age_cutoff), + blob_garbage_collection_force_threshold( + options.blob_garbage_collection_force_threshold), + blob_compaction_readahead_size(options.blob_compaction_readahead_size), + blob_file_starting_level(options.blob_file_starting_level), + prepopulate_blob_cache(options.prepopulate_blob_cache), + max_sequential_skip_in_iterations( + options.max_sequential_skip_in_iterations), + check_flush_compaction_key_order( + options.check_flush_compaction_key_order), + paranoid_file_checks(options.paranoid_file_checks), + report_bg_io_stats(options.report_bg_io_stats), + compression(options.compression), + bottommost_compression(options.bottommost_compression), + compression_opts(options.compression_opts), + bottommost_compression_opts(options.bottommost_compression_opts), + last_level_temperature(options.last_level_temperature == + Temperature::kUnknown + ? options.bottommost_temperature + : options.last_level_temperature), + memtable_protection_bytes_per_key( + options.memtable_protection_bytes_per_key), + block_protection_bytes_per_key(options.block_protection_bytes_per_key), + sample_for_compression( + options.sample_for_compression), // TODO: is 0 fine here? + compression_per_level(options.compression_per_level), + memtable_max_range_deletions(options.memtable_max_range_deletions) { + RefreshDerivedOptions(options.num_levels, options.compaction_style); + } + + MutableCFOptions() + : write_buffer_size(0), + max_write_buffer_number(0), + arena_block_size(0), + memtable_prefix_bloom_size_ratio(0), + memtable_whole_key_filtering(false), + memtable_huge_page_size(0), + max_successive_merges(0), + inplace_update_num_locks(0), + prefix_extractor(nullptr), + experimental_mempurge_threshold(0.0), + disable_auto_compactions(false), + soft_pending_compaction_bytes_limit(0), + hard_pending_compaction_bytes_limit(0), + level0_file_num_compaction_trigger(0), + level0_slowdown_writes_trigger(0), + level0_stop_writes_trigger(0), + max_compaction_bytes(0), + ignore_max_compaction_bytes_for_input(true), + target_file_size_base(0), + target_file_size_multiplier(0), + max_bytes_for_level_base(0), + max_bytes_for_level_multiplier(0), + ttl(0), + periodic_compaction_seconds(0), + compaction_options_fifo(), + enable_blob_files(false), + min_blob_size(0), + blob_file_size(0), + blob_compression_type(kNoCompression), + enable_blob_garbage_collection(false), + blob_garbage_collection_age_cutoff(0.0), + blob_garbage_collection_force_threshold(0.0), + blob_compaction_readahead_size(0), + blob_file_starting_level(0), + prepopulate_blob_cache(PrepopulateBlobCache::kDisable), + max_sequential_skip_in_iterations(0), + check_flush_compaction_key_order(true), + paranoid_file_checks(false), + report_bg_io_stats(false), + compression(Snappy_Supported() ? kSnappyCompression : kNoCompression), + bottommost_compression(kDisableCompressionOption), + last_level_temperature(Temperature::kUnknown), + memtable_protection_bytes_per_key(0), + block_protection_bytes_per_key(0), + sample_for_compression(0), + memtable_max_range_deletions(0) {} + + explicit MutableCFOptions(const Options& options); + + // Must be called after any change to MutableCFOptions + void RefreshDerivedOptions(int num_levels, CompactionStyle compaction_style); + + void RefreshDerivedOptions(const ImmutableCFOptions& ioptions) { + RefreshDerivedOptions(ioptions.num_levels, ioptions.compaction_style); + } + + int MaxBytesMultiplerAdditional(int level) const { + if (level >= + static_cast(max_bytes_for_level_multiplier_additional.size())) { + return 1; + } + return max_bytes_for_level_multiplier_additional[level]; + } + + void Dump(Logger* log) const; + + // Memtable related options + size_t write_buffer_size; + int max_write_buffer_number; + size_t arena_block_size; + double memtable_prefix_bloom_size_ratio; + bool memtable_whole_key_filtering; + size_t memtable_huge_page_size; + size_t max_successive_merges; + size_t inplace_update_num_locks; + std::shared_ptr prefix_extractor; + // [experimental] + // Used to activate or deactive the Mempurge feature (memtable garbage + // collection). (deactivated by default). At every flush, the total useful + // payload (total entries minus garbage entries) is estimated as a ratio + // [useful payload bytes]/[size of a memtable (in bytes)]. This ratio is then + // compared to this `threshold` value: + // - if ratio1.0 : aggressive mempurge. + // 0 < threshold < 1.0: mempurge triggered only for very low useful payload + // ratios. + // [experimental] + double experimental_mempurge_threshold; + + // Compaction related options + bool disable_auto_compactions; + uint64_t soft_pending_compaction_bytes_limit; + uint64_t hard_pending_compaction_bytes_limit; + int level0_file_num_compaction_trigger; + int level0_slowdown_writes_trigger; + int level0_stop_writes_trigger; + uint64_t max_compaction_bytes; + bool ignore_max_compaction_bytes_for_input; + uint64_t target_file_size_base; + int target_file_size_multiplier; + uint64_t max_bytes_for_level_base; + double max_bytes_for_level_multiplier; + uint64_t ttl; + uint64_t periodic_compaction_seconds; + std::vector max_bytes_for_level_multiplier_additional; + CompactionOptionsFIFO compaction_options_fifo; + CompactionOptionsUniversal compaction_options_universal; + + // Blob file related options + bool enable_blob_files; + uint64_t min_blob_size; + uint64_t blob_file_size; + CompressionType blob_compression_type; + bool enable_blob_garbage_collection; + double blob_garbage_collection_age_cutoff; + double blob_garbage_collection_force_threshold; + uint64_t blob_compaction_readahead_size; + int blob_file_starting_level; + PrepopulateBlobCache prepopulate_blob_cache; + + // Misc options + uint64_t max_sequential_skip_in_iterations; + bool check_flush_compaction_key_order; + bool paranoid_file_checks; + bool report_bg_io_stats; + CompressionType compression; + CompressionType bottommost_compression; + CompressionOptions compression_opts; + CompressionOptions bottommost_compression_opts; + Temperature last_level_temperature; + uint32_t memtable_protection_bytes_per_key; + uint8_t block_protection_bytes_per_key; + + uint64_t sample_for_compression; + std::vector compression_per_level; + uint32_t memtable_max_range_deletions; + + // Derived options + // Per-level target file size. + std::vector max_file_size; +}; + +uint64_t MultiplyCheckOverflow(uint64_t op1, double op2); + +// Get the max file size in a given level. +uint64_t MaxFileSizeForLevel(const MutableCFOptions& cf_options, + int level, CompactionStyle compaction_style, int base_level = 1, + bool level_compaction_dynamic_level_bytes = false); + +// Get the max size of an L0 file for which we will pin its meta-blocks when +// `pin_l0_filter_and_index_blocks_in_cache` is set. +size_t MaxFileSizeForL0MetaPin(const MutableCFOptions& cf_options); + +Status GetStringFromMutableCFOptions(const ConfigOptions& config_options, + const MutableCFOptions& mutable_opts, + std::string* opt_string); + +Status GetMutableOptionsFromStrings( + const MutableCFOptions& base_options, + const std::unordered_map& options_map, + Logger* info_log, MutableCFOptions* new_options); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/options/configurable.cc b/librocksdb-sys/rocksdb/options/configurable.cc new file mode 100644 index 0000000..5491336 --- /dev/null +++ b/librocksdb-sys/rocksdb/options/configurable.cc @@ -0,0 +1,712 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "rocksdb/configurable.h" + +#include "logging/logging.h" +#include "options/configurable_helper.h" +#include "options/options_helper.h" +#include "rocksdb/customizable.h" +#include "rocksdb/status.h" +#include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/options_type.h" +#include "util/coding.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +void Configurable::RegisterOptions( + const std::string& name, void* opt_ptr, + const std::unordered_map* type_map) { + RegisteredOptions opts; + opts.name = name; + opts.type_map = type_map; + opts.opt_ptr = opt_ptr; + options_.emplace_back(opts); +} + +//************************************************************************* +// +// Methods for Initializing and Validating Configurable Objects +// +//************************************************************************* + +Status Configurable::PrepareOptions(const ConfigOptions& opts) { + // We ignore the invoke_prepare_options here intentionally, + // as if you are here, you must have called PrepareOptions explicitly. + Status status = Status::OK(); + for (auto opt_iter : options_) { + if (opt_iter.type_map != nullptr) { + for (auto map_iter : *(opt_iter.type_map)) { + auto& opt_info = map_iter.second; + if (opt_info.ShouldPrepare()) { + status = opt_info.Prepare(opts, map_iter.first, opt_iter.opt_ptr); + if (!status.ok()) { + return status; + } + } + } + } + } + return status; +} + +Status Configurable::ValidateOptions(const DBOptions& db_opts, + const ColumnFamilyOptions& cf_opts) const { + Status status; + for (auto opt_iter : options_) { + if (opt_iter.type_map != nullptr) { + for (auto map_iter : *(opt_iter.type_map)) { + auto& opt_info = map_iter.second; + if (opt_info.ShouldValidate()) { + status = opt_info.Validate(db_opts, cf_opts, map_iter.first, + opt_iter.opt_ptr); + if (!status.ok()) { + return status; + } + } + } + } + } + return status; +} + +/*********************************************************************************/ +/* */ +/* Methods for Retrieving Options from Configurables */ +/* */ +/*********************************************************************************/ + +const void* Configurable::GetOptionsPtr(const std::string& name) const { + for (auto o : options_) { + if (o.name == name) { + return o.opt_ptr; + } + } + return nullptr; +} + +std::string Configurable::GetOptionName(const std::string& opt_name) const { + return opt_name; +} + +const OptionTypeInfo* ConfigurableHelper::FindOption( + const std::vector& options, + const std::string& short_name, std::string* opt_name, void** opt_ptr) { + for (auto iter : options) { + if (iter.type_map != nullptr) { + const auto opt_info = + OptionTypeInfo::Find(short_name, *(iter.type_map), opt_name); + if (opt_info != nullptr) { + *opt_ptr = iter.opt_ptr; + return opt_info; + } + } + } + return nullptr; +} + +//************************************************************************* +// +// Methods for Configuring Options from Strings/Name-Value Pairs/Maps +// +//************************************************************************* + +Status Configurable::ConfigureFromMap( + const ConfigOptions& config_options, + const std::unordered_map& opts_map) { + Status s = ConfigureFromMap(config_options, opts_map, nullptr); + return s; +} + +Status Configurable::ConfigureFromMap( + const ConfigOptions& config_options, + const std::unordered_map& opts_map, + std::unordered_map* unused) { + return ConfigureOptions(config_options, opts_map, unused); +} + +Status Configurable::ConfigureOptions( + const ConfigOptions& config_options, + const std::unordered_map& opts_map, + std::unordered_map* unused) { + std::string curr_opts; + Status s; + if (!opts_map.empty()) { + // There are options in the map. + // Save the current configuration in curr_opts and then configure the + // options, but do not prepare them now. We will do all the prepare when + // the configuration is complete. + ConfigOptions copy = config_options; + copy.invoke_prepare_options = false; + if (!config_options.ignore_unknown_options) { + // If we are not ignoring unused, get the defaults in case we need to + // reset + copy.depth = ConfigOptions::kDepthDetailed; + copy.delimiter = "; "; + GetOptionString(copy, &curr_opts).PermitUncheckedError(); + } + + s = ConfigurableHelper::ConfigureOptions(copy, *this, opts_map, unused); + } + if (config_options.invoke_prepare_options && s.ok()) { + s = PrepareOptions(config_options); + } + if (!s.ok() && !curr_opts.empty()) { + ConfigOptions reset = config_options; + reset.ignore_unknown_options = true; + reset.invoke_prepare_options = true; + reset.ignore_unsupported_options = true; + // There are some options to reset from this current error + ConfigureFromString(reset, curr_opts).PermitUncheckedError(); + } + return s; +} + +Status Configurable::ParseStringOptions(const ConfigOptions& /*config_options*/, + const std::string& /*opts_str*/) { + return Status::OK(); +} + +Status Configurable::ConfigureFromString(const ConfigOptions& config_options, + const std::string& opts_str) { + Status s; + if (!opts_str.empty()) { + if (opts_str.find(';') != std::string::npos || + opts_str.find('=') != std::string::npos) { + std::unordered_map opt_map; + s = StringToMap(opts_str, &opt_map); + if (s.ok()) { + s = ConfigureFromMap(config_options, opt_map, nullptr); + } + } else { + s = ParseStringOptions(config_options, opts_str); + if (s.ok() && config_options.invoke_prepare_options) { + s = PrepareOptions(config_options); + } + } + } else if (config_options.invoke_prepare_options) { + s = PrepareOptions(config_options); + } else { + s = Status::OK(); + } + return s; +} + +/** + * Sets the value of the named property to the input value, returning OK on + * succcess. + */ +Status Configurable::ConfigureOption(const ConfigOptions& config_options, + const std::string& name, + const std::string& value) { + return ConfigurableHelper::ConfigureSingleOption(config_options, *this, name, + value); +} + +/** + * Looks for the named option amongst the options for this type and sets + * the value for it to be the input value. + * If the name was found, found_option will be set to true and the resulting + * status should be returned. + */ + +Status Configurable::ParseOption(const ConfigOptions& config_options, + const OptionTypeInfo& opt_info, + const std::string& opt_name, + const std::string& opt_value, void* opt_ptr) { + if (opt_info.IsMutable()) { + if (config_options.mutable_options_only) { + // This option is mutable. Treat all of its children as mutable as well + ConfigOptions copy = config_options; + copy.mutable_options_only = false; + return opt_info.Parse(copy, opt_name, opt_value, opt_ptr); + } else { + return opt_info.Parse(config_options, opt_name, opt_value, opt_ptr); + } + } else if (config_options.mutable_options_only) { + return Status::InvalidArgument("Option not changeable: " + opt_name); + } else { + return opt_info.Parse(config_options, opt_name, opt_value, opt_ptr); + } +} + + +Status ConfigurableHelper::ConfigureOptions( + const ConfigOptions& config_options, Configurable& configurable, + const std::unordered_map& opts_map, + std::unordered_map* unused) { + std::unordered_map remaining = opts_map; + Status s = Status::OK(); + if (!opts_map.empty()) { + for (const auto& iter : configurable.options_) { + if (iter.type_map != nullptr) { + s = ConfigureSomeOptions(config_options, configurable, *(iter.type_map), + &remaining, iter.opt_ptr); + if (remaining.empty()) { // Are there more options left? + break; + } else if (!s.ok()) { + break; + } + } + } + } + if (unused != nullptr && !remaining.empty()) { + unused->insert(remaining.begin(), remaining.end()); + } + if (config_options.ignore_unknown_options) { + s = Status::OK(); + } else if (s.ok() && unused == nullptr && !remaining.empty()) { + s = Status::NotFound("Could not find option: ", remaining.begin()->first); + } + return s; +} + +/** + * Updates the object with the named-value property values, returning OK on + * succcess. Any properties that were found are removed from the options list; + * upon return only options that were not found in this opt_map remain. + + * Returns: + * - OK if ignore_unknown_options is set + * - InvalidArgument, if any option was invalid + * - NotSupported, if any option is unsupported and ignore_unsupported_options + is OFF + * - OK, if no option was invalid or not supported (or ignored) + */ +Status ConfigurableHelper::ConfigureSomeOptions( + const ConfigOptions& config_options, Configurable& configurable, + const std::unordered_map& type_map, + std::unordered_map* options, void* opt_ptr) { + Status result = Status::OK(); // The last non-OK result (if any) + Status notsup = Status::OK(); // The last NotSupported result (if any) + std::string elem_name; + int found = 1; + std::unordered_set unsupported; + // While there are unused properties and we processed at least one, + // go through the remaining unused properties and attempt to configure them. + while (found > 0 && !options->empty()) { + found = 0; + notsup = Status::OK(); + for (auto it = options->begin(); it != options->end();) { + const std::string& opt_name = configurable.GetOptionName(it->first); + const std::string& opt_value = it->second; + const auto opt_info = + OptionTypeInfo::Find(opt_name, type_map, &elem_name); + if (opt_info == nullptr) { // Did not find the option. Skip it + ++it; + } else { + Status s = ConfigureOption(config_options, configurable, *opt_info, + opt_name, elem_name, opt_value, opt_ptr); + if (s.IsNotFound()) { + ++it; + } else if (s.IsNotSupported()) { + notsup = s; + unsupported.insert(it->first); + ++it; // Skip it for now + } else { + found++; + it = options->erase(it); + if (!s.ok()) { + result = s; + } + } + } + } // End for all remaining options + } // End while found one or options remain + + // Now that we have been through the list, remove any unsupported + for (auto u : unsupported) { + auto it = options->find(u); + if (it != options->end()) { + options->erase(it); + } + } + if (config_options.ignore_unknown_options) { + if (!result.ok()) result.PermitUncheckedError(); + if (!notsup.ok()) notsup.PermitUncheckedError(); + return Status::OK(); + } else if (!result.ok()) { + if (!notsup.ok()) notsup.PermitUncheckedError(); + return result; + } else if (config_options.ignore_unsupported_options) { + if (!notsup.ok()) notsup.PermitUncheckedError(); + return Status::OK(); + } else { + return notsup; + } +} + +Status ConfigurableHelper::ConfigureSingleOption( + const ConfigOptions& config_options, Configurable& configurable, + const std::string& name, const std::string& value) { + const std::string& opt_name = configurable.GetOptionName(name); + std::string elem_name; + void* opt_ptr = nullptr; + const auto opt_info = + FindOption(configurable.options_, opt_name, &elem_name, &opt_ptr); + if (opt_info == nullptr) { + return Status::NotFound("Could not find option: ", name); + } else { + return ConfigureOption(config_options, configurable, *opt_info, opt_name, + elem_name, value, opt_ptr); + } +} +Status ConfigurableHelper::ConfigureCustomizableOption( + const ConfigOptions& config_options, Configurable& configurable, + const OptionTypeInfo& opt_info, const std::string& opt_name, + const std::string& name, const std::string& value, void* opt_ptr) { + Customizable* custom = opt_info.AsRawPointer(opt_ptr); + ConfigOptions copy = config_options; + if (opt_info.IsMutable()) { + // This option is mutable. Pass that property on to any subsequent calls + copy.mutable_options_only = false; + } + + if (opt_info.IsMutable() || !config_options.mutable_options_only) { + // Either the option is mutable, or we are processing all of the options + if (opt_name == name || name == OptionTypeInfo::kIdPropName() || + EndsWith(opt_name, OptionTypeInfo::kIdPropSuffix())) { + return configurable.ParseOption(copy, opt_info, name, value, opt_ptr); + } else if (value.empty()) { + return Status::OK(); + } else if (custom == nullptr || !StartsWith(name, custom->GetId() + ".")) { + return configurable.ParseOption(copy, opt_info, name, value, opt_ptr); + } else if (value.find("=") != std::string::npos) { + return custom->ConfigureFromString(copy, value); + } else { + return custom->ConfigureOption(copy, name, value); + } + } else { + // We are processing immutable options, which means that we cannot change + // the Customizable object itself, but could change its mutable properties. + // Check to make sure that nothing is trying to change the Customizable + if (custom == nullptr) { + // We do not have a Customizable to configure. This is OK if the + // value is empty (nothing being configured) but an error otherwise + if (value.empty()) { + return Status::OK(); + } else { + return Status::InvalidArgument("Option not changeable: " + opt_name); + } + } else if (EndsWith(opt_name, OptionTypeInfo::kIdPropSuffix()) || + name == OptionTypeInfo::kIdPropName()) { + // We have a property of the form "id=value" or "table.id=value" + // This is OK if we ID/value matches the current customizable object + if (custom->GetId() == value) { + return Status::OK(); + } else { + return Status::InvalidArgument("Option not changeable: " + opt_name); + } + } else if (opt_name == name) { + // The properties are of one of forms: + // name = { id = id; prop1 = value1; ... } + // name = { prop1=value1; prop2=value2; ... } + // name = ID + // Convert the value to a map and extract the ID + // If the ID does not match that of the current customizable, return an + // error. Otherwise, update the current customizable via the properties + // map + std::unordered_map props; + std::string id; + Status s = + Configurable::GetOptionsMap(value, custom->GetId(), &id, &props); + if (!s.ok()) { + return s; + } else if (custom->GetId() != id) { + return Status::InvalidArgument("Option not changeable: " + opt_name); + } else if (props.empty()) { + return Status::OK(); + } else { + return custom->ConfigureFromMap(copy, props); + } + } else { + // Attempting to configure one of the properties of the customizable + // Let it through + return custom->ConfigureOption(copy, name, value); + } + } +} + +Status ConfigurableHelper::ConfigureOption( + const ConfigOptions& config_options, Configurable& configurable, + const OptionTypeInfo& opt_info, const std::string& opt_name, + const std::string& name, const std::string& value, void* opt_ptr) { + if (opt_info.IsCustomizable()) { + return ConfigureCustomizableOption(config_options, configurable, opt_info, + opt_name, name, value, opt_ptr); + } else if (opt_name == name) { + return configurable.ParseOption(config_options, opt_info, opt_name, value, + opt_ptr); + } else if (opt_info.IsStruct() || opt_info.IsConfigurable()) { + return configurable.ParseOption(config_options, opt_info, name, value, + opt_ptr); + } else { + return Status::NotFound("Could not find option: ", name); + } +} + +//******************************************************************************* +// +// Methods for Converting Options into strings +// +//******************************************************************************* + +Status Configurable::GetOptionString(const ConfigOptions& config_options, + std::string* result) const { + assert(result); + result->clear(); + return ConfigurableHelper::SerializeOptions(config_options, *this, "", + result); +} + +std::string Configurable::ToString(const ConfigOptions& config_options, + const std::string& prefix) const { + std::string result = SerializeOptions(config_options, prefix); + if (result.empty() || result.find('=') == std::string::npos) { + return result; + } else { + return "{" + result + "}"; + } +} + +std::string Configurable::SerializeOptions(const ConfigOptions& config_options, + const std::string& header) const { + std::string result; + Status s = ConfigurableHelper::SerializeOptions(config_options, *this, header, + &result); + assert(s.ok()); + return result; +} + +Status Configurable::GetOption(const ConfigOptions& config_options, + const std::string& name, + std::string* value) const { + return ConfigurableHelper::GetOption(config_options, *this, + GetOptionName(name), value); +} + +Status ConfigurableHelper::GetOption(const ConfigOptions& config_options, + const Configurable& configurable, + const std::string& short_name, + std::string* value) { + // Look for option directly + assert(value); + value->clear(); + + std::string opt_name; + void* opt_ptr = nullptr; + const auto opt_info = + FindOption(configurable.options_, short_name, &opt_name, &opt_ptr); + if (opt_info != nullptr) { + ConfigOptions embedded = config_options; + embedded.delimiter = ";"; + if (short_name == opt_name) { + return opt_info->Serialize(embedded, opt_name, opt_ptr, value); + } else if (opt_info->IsStruct()) { + return opt_info->Serialize(embedded, opt_name, opt_ptr, value); + } else if (opt_info->IsConfigurable()) { + auto const* config = opt_info->AsRawPointer(opt_ptr); + if (config != nullptr) { + return config->GetOption(embedded, opt_name, value); + } + } + } + return Status::NotFound("Cannot find option: ", short_name); +} + +Status ConfigurableHelper::SerializeOptions(const ConfigOptions& config_options, + const Configurable& configurable, + const std::string& prefix, + std::string* result) { + assert(result); + for (auto const& opt_iter : configurable.options_) { + if (opt_iter.type_map != nullptr) { + for (const auto& map_iter : *(opt_iter.type_map)) { + const auto& opt_name = map_iter.first; + const auto& opt_info = map_iter.second; + if (opt_info.ShouldSerialize()) { + std::string value; + Status s; + if (!config_options.mutable_options_only) { + s = opt_info.Serialize(config_options, prefix + opt_name, + opt_iter.opt_ptr, &value); + } else if (opt_info.IsMutable()) { + ConfigOptions copy = config_options; + copy.mutable_options_only = false; + s = opt_info.Serialize(copy, prefix + opt_name, opt_iter.opt_ptr, + &value); + } else if (opt_info.IsConfigurable()) { + // If it is a Configurable and we are either printing all of the + // details or not printing only the name, this option should be + // included in the list + if (config_options.IsDetailed() || + !opt_info.IsEnabled(OptionTypeFlags::kStringNameOnly)) { + s = opt_info.Serialize(config_options, prefix + opt_name, + opt_iter.opt_ptr, &value); + } + } + if (!s.ok()) { + return s; + } else if (!value.empty()) { + // = + result->append(prefix + opt_name + "=" + value + + config_options.delimiter); + } + } + } + } + } + return Status::OK(); +} + +//******************************************************************************** +// +// Methods for listing the options from Configurables +// +//******************************************************************************** +Status Configurable::GetOptionNames( + const ConfigOptions& config_options, + std::unordered_set* result) const { + assert(result); + return ConfigurableHelper::ListOptions(config_options, *this, "", result); +} + +Status ConfigurableHelper::ListOptions( + const ConfigOptions& config_options, const Configurable& configurable, + const std::string& prefix, std::unordered_set* result) { + Status status; + for (auto const& opt_iter : configurable.options_) { + if (opt_iter.type_map != nullptr) { + for (const auto& map_iter : *(opt_iter.type_map)) { + const auto& opt_name = map_iter.first; + const auto& opt_info = map_iter.second; + // If the option is no longer used in rocksdb and marked as deprecated, + // we skip it in the serialization. + if (!opt_info.IsDeprecated() && !opt_info.IsAlias()) { + if (!config_options.mutable_options_only) { + result->emplace(prefix + opt_name); + } else if (opt_info.IsMutable()) { + result->emplace(prefix + opt_name); + } + } + } + } + } + return status; +} + +//******************************************************************************* +// +// Methods for Comparing Configurables +// +//******************************************************************************* + +bool Configurable::AreEquivalent(const ConfigOptions& config_options, + const Configurable* other, + std::string* name) const { + assert(name); + name->clear(); + if (this == other || config_options.IsCheckDisabled()) { + return true; + } else if (other != nullptr) { + return ConfigurableHelper::AreEquivalent(config_options, *this, *other, + name); + } else { + return false; + } +} + +bool Configurable::OptionsAreEqual(const ConfigOptions& config_options, + const OptionTypeInfo& opt_info, + const std::string& opt_name, + const void* const this_ptr, + const void* const that_ptr, + std::string* mismatch) const { + if (opt_info.AreEqual(config_options, opt_name, this_ptr, that_ptr, + mismatch)) { + return true; + } else if (opt_info.AreEqualByName(config_options, opt_name, this_ptr, + that_ptr)) { + *mismatch = ""; + return true; + } else { + return false; + } +} + +bool ConfigurableHelper::AreEquivalent(const ConfigOptions& config_options, + const Configurable& this_one, + const Configurable& that_one, + std::string* mismatch) { + assert(mismatch != nullptr); + for (auto const& o : this_one.options_) { + const auto this_offset = this_one.GetOptionsPtr(o.name); + const auto that_offset = that_one.GetOptionsPtr(o.name); + if (this_offset != that_offset) { + if (this_offset == nullptr || that_offset == nullptr) { + return false; + } else if (o.type_map != nullptr) { + for (const auto& map_iter : *(o.type_map)) { + const auto& opt_info = map_iter.second; + if (config_options.IsCheckEnabled(opt_info.GetSanityLevel())) { + if (!config_options.mutable_options_only) { + if (!this_one.OptionsAreEqual(config_options, opt_info, + map_iter.first, this_offset, + that_offset, mismatch)) { + return false; + } + } else if (opt_info.IsMutable()) { + ConfigOptions copy = config_options; + copy.mutable_options_only = false; + if (!this_one.OptionsAreEqual(copy, opt_info, map_iter.first, + this_offset, that_offset, + mismatch)) { + return false; + } + } + } + } + } + } + } + return true; +} + +Status Configurable::GetOptionsMap( + const std::string& value, const std::string& default_id, std::string* id, + std::unordered_map* props) { + assert(id); + assert(props); + Status status; + if (value.empty() || value == kNullptrString) { + *id = default_id; + } else if (value.find('=') == std::string::npos) { + *id = value; + } else { + status = StringToMap(value, props); + if (!status.ok()) { // There was an error creating the map. + *id = value; // Treat the value as id + props->clear(); // Clear the properties + status = Status::OK(); // and ignore the error + } else { + auto iter = props->find(OptionTypeInfo::kIdPropName()); + if (iter != props->end()) { + *id = iter->second; + props->erase(iter); + if (*id == kNullptrString) { + id->clear(); + } + } else if (!default_id.empty()) { + *id = default_id; + } else { // No id property and no default + *id = value; // Treat the value as id + props->clear(); // Clear the properties + } + } + } + return status; +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/options/configurable_helper.h b/librocksdb-sys/rocksdb/options/configurable_helper.h new file mode 100644 index 0000000..5d409f8 --- /dev/null +++ b/librocksdb-sys/rocksdb/options/configurable_helper.h @@ -0,0 +1,185 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include +#include + +#include "rocksdb/configurable.h" +#include "rocksdb/convenience.h" + +namespace ROCKSDB_NAMESPACE { +// Helper class defining static methods for supporting the Configurable +// class. The purpose of this class is to keep the Configurable class +// as tight as possible and provide methods for doing the actual work +// of configuring the objects. +class ConfigurableHelper { + public: + // Configures the input Configurable object based on the parameters. + // On successful completion, the Configurable is updated with the settings + // from the opt_map. + // + // The acceptable values of the name/value pairs are documented with the + // specific class/instance. + // + // @param config_options Controls how the arguments are processed. + // @param opt_map Name/value pairs of the options to update + // @param unused If specified, this value will return the name/value + // pairs from opt_map that were NotFound for this object. + // @return OK If all values in the map were successfully updated + // @return NotFound If any of the names in the opt_map were not valid + // for this object. If unused is specified, it will contain the + // collection of NotFound entries + // @return NotSupported If any of the names are valid but the object does + // not know how to convert the value. This can happen if, for example, + // there is some nested Configurable that cannot be created. + // @return InvalidArgument If any of the values cannot be successfully + // parsed. This can also be returned if PrepareOptions encounters an + // error. + static Status ConfigureOptions( + const ConfigOptions& config_options, Configurable& configurable, + const std::unordered_map& options, + std::unordered_map* unused); + + // Internal method to configure a set of options for this object. + // Classes may override this value to change its behavior. + // @param config_options Controls how the options are being configured + // @param type_name The name that was registered for this set of options + // @param type_map The map of options for this name + // @param opt_ptr Pointer to the object being configured for this option set. + // @param options The option name/values being updated. On return, any + // option that was found is removed from the list. + // @return OK If all of the options were successfully updated. + // @return InvalidArgument If an option was found but the value could not + // be updated. + // @return NotFound If an option name was not found in type_mape + // @return NotSupported If the option was found but no rule for converting + // the value could be found. + static Status ConfigureSomeOptions( + const ConfigOptions& config_options, Configurable& configurable, + const std::unordered_map& type_map, + std::unordered_map* options, void* opt_ptr); + + // Configures a single option in the input Configurable. + // This method will look through the set of option names for this + // Configurable searching for one with the input name. If such an option + // is found, it will be configured via the input value. + // + // @param config_options Controls how the option is being configured + // @param configurable The object to configure + // @param name For options with sub-options (like Structs or + // Configurables), + // this value may be the name of the sub-field of the option being + // updated. For example, if the option is + // "compaction_options_fifo.allow_compaction", then field name would be + // "allow_compaction". For most options, field_name and opt_name will be + // equivalent. + // @param value The new value for this option. + // @param See ConfigureOptions for the possible return values + static Status ConfigureSingleOption(const ConfigOptions& config_options, + Configurable& configurable, + const std::string& name, + const std::string& value); + + // Configures the option referenced by opt_info for this configurable + // This method configures the option based on opt_info for the input + // configurable. + // @param config_options Controls how the option is being configured + // @param configurable The object to configure + // @param opt_name The full option name + // @param name For options with sub-options (like Structs or + // Configurables), + // this value may be the name of the sub-field of the option being + // updated. For example, if the option is + // "compaction_options_fifo.allow_compaction", then field name would be + // "allow_compaction". For most options, field_name and opt_name will be + // equivalent. + // @param value The new value for this option. + // @param See ConfigureOptions for the possible return values + static Status ConfigureOption(const ConfigOptions& config_options, + Configurable& configurable, + const OptionTypeInfo& opt_info, + const std::string& opt_name, + const std::string& name, + const std::string& value, void* opt_ptr); + + // Returns the value of the option associated with the input name + // This method is the functional inverse of ConfigureOption + // @param config_options Controls how the value is returned + // @param configurable The object from which to get the option. + // @param name The name of the option to return a value for. + // @param value The returned value associated with the named option. + // Note that value will be only the serialized version + // of the option and not "name=value" + // @return OK If the named field was successfully updated to value. + // @return NotFound If the name is not valid for this object. + // @param InvalidArgument If the name is valid for this object but + // its value cannot be serialized. + static Status GetOption(const ConfigOptions& config_options, + const Configurable& configurable, + const std::string& name, std::string* value); + + // Serializes the input Configurable into the output result. + // This is the inverse of ConfigureOptions + // @param config_options Controls how serialization happens. + // @param configurable The object to serialize + // @param prefix A prefix to add to the each option as it is serialized. + // @param result The string representation of the configurable. + // @return OK If the options for this object wer successfully serialized. + // @return InvalidArgument If one or more of the options could not be + // serialized. + static Status SerializeOptions(const ConfigOptions& config_options, + const Configurable& configurable, + const std::string& prefix, + std::string* result); + + // Internal method to list the option names for this object. + // Classes may override this value to change its behavior. + // @see ListOptions for more details + static Status ListOptions(const ConfigOptions& config_options, + const Configurable& configurable, + const std::string& prefix, + std::unordered_set* result); + + // Checks to see if the two configurables are equivalent to one other. + // This method assumes that the two objects are of the same class. + // @param config_options Controls how the options are compared. + // @param this_one The object to compare to. + // @param that_one The other object being compared. + // @param mismatch If the objects do not match, this parameter contains + // the name of the option that triggered the match failure. + // @param True if the objects match, false otherwise. + static bool AreEquivalent(const ConfigOptions& config_options, + const Configurable& this_one, + const Configurable& that_one, + std::string* mismatch); + + private: + // Looks for the option specified by name in the RegisteredOptions. + // This method traverses the types in the input options vector. If an entry + // matching name is found, that entry, opt_name, and pointer are returned. + // @param options The vector of options to search through + // @param name The name of the option to search for in the OptionType map + // @param opt_name If the name was found, this value is set to the option name + // associated with the input name/type. + // @param opt_ptr If the name was found, this value is set to the option + // pointer + // in the RegisteredOptions vector associated with this entry + // @return A pointer to the OptionTypeInfo from the options if found, + // nullptr if the name was not found in the input options + static const OptionTypeInfo* FindOption( + const std::vector& options, + const std::string& name, std::string* opt_name, void** opt_ptr); + + static Status ConfigureCustomizableOption( + const ConfigOptions& config_options, Configurable& configurable, + const OptionTypeInfo& opt_info, const std::string& opt_name, + const std::string& name, const std::string& value, void* opt_ptr); +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/options/configurable_test.cc b/librocksdb-sys/rocksdb/options/configurable_test.cc new file mode 100644 index 0000000..a03d8f0 --- /dev/null +++ b/librocksdb-sys/rocksdb/options/configurable_test.cc @@ -0,0 +1,861 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "options/configurable_test.h" + +#include +#include +#include +#include + +#include "options/configurable_helper.h" +#include "options/options_helper.h" +#include "options/options_parser.h" +#include "rocksdb/configurable.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" + +#ifndef GFLAGS +bool FLAGS_enable_print = false; +#else +#include "util/gflags_compat.h" +using GFLAGS_NAMESPACE::ParseCommandLineFlags; +DEFINE_bool(enable_print, false, "Print options generated to console."); +#endif // GFLAGS + +namespace ROCKSDB_NAMESPACE { +namespace test { +class StringLogger : public Logger { + public: + using Logger::Logv; + void Logv(const char* format, va_list ap) override { + char buffer[1000]; + vsnprintf(buffer, sizeof(buffer), format, ap); + string_.append(buffer); + } + const std::string& str() const { return string_; } + void clear() { string_.clear(); } + + private: + std::string string_; +}; +static std::unordered_map struct_option_info = { + {"struct", OptionTypeInfo::Struct("struct", &simple_option_info, 0, + OptionVerificationType::kNormal, + OptionTypeFlags::kMutable)}, +}; + +static std::unordered_map imm_struct_option_info = + { + {"struct", OptionTypeInfo::Struct("struct", &simple_option_info, 0, + OptionVerificationType::kNormal, + OptionTypeFlags::kNone)}, +}; + +class SimpleConfigurable : public TestConfigurable { + public: + static SimpleConfigurable* Create( + const std::string& name = "simple", + int mode = TestConfigMode::kDefaultMode, + const std::unordered_map* map = + &simple_option_info) { + return new SimpleConfigurable(name, mode, map); + } + + SimpleConfigurable(const std::string& name, int mode, + const std::unordered_map* + map = &simple_option_info) + : TestConfigurable(name, mode, map) { + if ((mode & TestConfigMode::kUniqueMode) != 0) { + unique_.reset(SimpleConfigurable::Create("Unique" + name_)); + RegisterOptions(name_ + "Unique", &unique_, &unique_option_info); + } + if ((mode & TestConfigMode::kSharedMode) != 0) { + shared_.reset(SimpleConfigurable::Create("Shared" + name_)); + RegisterOptions(name_ + "Shared", &shared_, &shared_option_info); + } + if ((mode & TestConfigMode::kRawPtrMode) != 0) { + pointer_ = SimpleConfigurable::Create("Pointer" + name_); + RegisterOptions(name_ + "Pointer", &pointer_, &pointer_option_info); + } + } + +}; // End class SimpleConfigurable + +using ConfigTestFactoryFunc = std::function; + +class ConfigurableTest : public testing::Test { + public: + ConfigurableTest() { config_options_.invoke_prepare_options = false; } + + ConfigOptions config_options_; +}; + +TEST_F(ConfigurableTest, GetOptionsPtrTest) { + std::string opt_str; + std::unique_ptr configurable(SimpleConfigurable::Create()); + ASSERT_NE(configurable->GetOptions("simple"), nullptr); + ASSERT_EQ(configurable->GetOptions("bad-opt"), nullptr); +} + +TEST_F(ConfigurableTest, ConfigureFromMapTest) { + std::unique_ptr configurable(SimpleConfigurable::Create()); + auto* opts = configurable->GetOptions("simple"); + ASSERT_OK(configurable->ConfigureFromMap(config_options_, {})); + ASSERT_NE(opts, nullptr); + std::unordered_map options_map = { + {"int", "1"}, {"bool", "true"}, {"string", "string"}}; + ASSERT_OK(configurable->ConfigureFromMap(config_options_, options_map)); + ASSERT_EQ(opts->i, 1); + ASSERT_EQ(opts->b, true); + ASSERT_EQ(opts->s, "string"); +} + +TEST_F(ConfigurableTest, ConfigureFromStringTest) { + std::unique_ptr configurable(SimpleConfigurable::Create()); + auto* opts = configurable->GetOptions("simple"); + ASSERT_OK(configurable->ConfigureFromString(config_options_, "")); + ASSERT_NE(opts, nullptr); + ASSERT_OK(configurable->ConfigureFromString(config_options_, + "int=1;bool=true;string=s")); + ASSERT_EQ(opts->i, 1); + ASSERT_EQ(opts->b, true); + ASSERT_EQ(opts->s, "s"); +} + +TEST_F(ConfigurableTest, ConfigureIgnoreTest) { + std::unique_ptr configurable(SimpleConfigurable::Create()); + std::unordered_map options_map = {{"unused", "u"}}; + ConfigOptions ignore = config_options_; + ignore.ignore_unknown_options = true; + ASSERT_NOK(configurable->ConfigureFromMap(config_options_, options_map)); + ASSERT_OK(configurable->ConfigureFromMap(ignore, options_map)); + ASSERT_NOK(configurable->ConfigureFromString(config_options_, "unused=u")); + ASSERT_OK(configurable->ConfigureFromString(ignore, "unused=u")); +} + +TEST_F(ConfigurableTest, ConfigureNestedOptionsTest) { + std::unique_ptr base, copy; + std::string opt_str; + std::string mismatch; + + base.reset(SimpleConfigurable::Create("simple", TestConfigMode::kAllOptMode)); + copy.reset(SimpleConfigurable::Create("simple", TestConfigMode::kAllOptMode)); + ASSERT_OK(base->ConfigureFromString(config_options_, + "shared={int=10; string=10};" + "unique={int=20; string=20};" + "pointer={int=30; string=30};")); + ASSERT_OK(base->GetOptionString(config_options_, &opt_str)); + ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str)); + ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); +} + +TEST_F(ConfigurableTest, GetOptionsTest) { + std::unique_ptr simple; + + simple.reset( + SimpleConfigurable::Create("simple", TestConfigMode::kAllOptMode)); + int i = 11; + for (auto opt : {"", "shared.", "unique.", "pointer."}) { + std::string value; + std::string expected = std::to_string(i); + std::string opt_name = opt; + ASSERT_OK( + simple->ConfigureOption(config_options_, opt_name + "int", expected)); + ASSERT_OK(simple->GetOption(config_options_, opt_name + "int", &value)); + ASSERT_EQ(expected, value); + ASSERT_OK(simple->ConfigureOption(config_options_, opt_name + "string", + expected)); + ASSERT_OK(simple->GetOption(config_options_, opt_name + "string", &value)); + ASSERT_EQ(expected, value); + + ASSERT_NOK( + simple->ConfigureOption(config_options_, opt_name + "bad", expected)); + ASSERT_NOK(simple->GetOption(config_options_, "bad option", &value)); + ASSERT_TRUE(value.empty()); + i += 11; + } +} + +TEST_F(ConfigurableTest, ConfigureBadOptionsTest) { + std::unique_ptr configurable(SimpleConfigurable::Create()); + auto* opts = configurable->GetOptions("simple"); + ASSERT_NE(opts, nullptr); + ASSERT_OK(configurable->ConfigureOption(config_options_, "int", "42")); + ASSERT_EQ(opts->i, 42); + ASSERT_NOK(configurable->ConfigureOption(config_options_, "int", "fred")); + ASSERT_NOK(configurable->ConfigureOption(config_options_, "bool", "fred")); + ASSERT_NOK( + configurable->ConfigureFromString(config_options_, "int=33;unused=u")); + ASSERT_EQ(opts->i, 42); +} + +TEST_F(ConfigurableTest, InvalidOptionTest) { + std::unique_ptr configurable(SimpleConfigurable::Create()); + std::unordered_map options_map = { + {"bad-option", "bad"}}; + ASSERT_NOK(configurable->ConfigureFromMap(config_options_, options_map)); + ASSERT_NOK( + configurable->ConfigureFromString(config_options_, "bad-option=bad")); + ASSERT_NOK( + configurable->ConfigureOption(config_options_, "bad-option", "bad")); +} + +static std::unordered_map validated_option_info = { + {"validated", + {0, OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, +}; +static std::unordered_map prepared_option_info = { + {"prepared", + {0, OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, +}; +static std::unordered_map + dont_prepare_option_info = { + {"unique", + {0, OptionType::kConfigurable, OptionVerificationType::kNormal, + (OptionTypeFlags::kUnique | OptionTypeFlags::kDontPrepare)}}, + +}; + +class ValidatedConfigurable : public SimpleConfigurable { + public: + ValidatedConfigurable(const std::string& name, unsigned char mode, + bool dont_prepare = false) + : SimpleConfigurable(name, TestConfigMode::kDefaultMode), + validated(false), + prepared(0) { + RegisterOptions("Validated", &validated, &validated_option_info); + RegisterOptions("Prepared", &prepared, &prepared_option_info); + if ((mode & TestConfigMode::kUniqueMode) != 0) { + unique_.reset(new ValidatedConfigurable( + "Unique" + name_, TestConfigMode::kDefaultMode, false)); + if (dont_prepare) { + RegisterOptions(name_ + "Unique", &unique_, &dont_prepare_option_info); + } else { + RegisterOptions(name_ + "Unique", &unique_, &unique_option_info); + } + } + } + + Status PrepareOptions(const ConfigOptions& config_options) override { + if (++prepared <= 0) { + return Status::InvalidArgument("Cannot prepare option"); + } else { + return SimpleConfigurable::PrepareOptions(config_options); + } + } + + Status ValidateOptions(const DBOptions& db_opts, + const ColumnFamilyOptions& cf_opts) const override { + if (!validated) { + return Status::InvalidArgument("Not Validated"); + } else { + return SimpleConfigurable::ValidateOptions(db_opts, cf_opts); + } + } + + private: + bool validated; + int prepared; +}; + +TEST_F(ConfigurableTest, ValidateOptionsTest) { + std::unique_ptr configurable( + new ValidatedConfigurable("validated", TestConfigMode::kDefaultMode)); + ColumnFamilyOptions cf_opts; + DBOptions db_opts; + ASSERT_OK( + configurable->ConfigureOption(config_options_, "validated", "false")); + ASSERT_NOK(configurable->ValidateOptions(db_opts, cf_opts)); + ASSERT_OK( + configurable->ConfigureOption(config_options_, "validated", "true")); + ASSERT_OK(configurable->ValidateOptions(db_opts, cf_opts)); +} + +TEST_F(ConfigurableTest, PrepareOptionsTest) { + std::unique_ptr c( + new ValidatedConfigurable("Simple", TestConfigMode::kUniqueMode, false)); + auto cp = c->GetOptions("Prepared"); + auto u = c->GetOptions>("SimpleUnique"); + auto up = u->get()->GetOptions("Prepared"); + config_options_.invoke_prepare_options = false; + + ASSERT_NE(cp, nullptr); + ASSERT_NE(up, nullptr); + ASSERT_EQ(*cp, 0); + ASSERT_EQ(*up, 0); + ASSERT_OK(c->ConfigureFromMap(config_options_, {})); + ASSERT_EQ(*cp, 0); + ASSERT_EQ(*up, 0); + config_options_.invoke_prepare_options = true; + ASSERT_OK(c->ConfigureFromMap(config_options_, {})); + ASSERT_EQ(*cp, 1); + ASSERT_EQ(*up, 1); + ASSERT_OK(c->ConfigureFromString(config_options_, "prepared=0")); + ASSERT_EQ(*up, 2); + ASSERT_EQ(*cp, 1); + + ASSERT_NOK(c->ConfigureFromString(config_options_, "prepared=-2")); + + c.reset( + new ValidatedConfigurable("Simple", TestConfigMode::kUniqueMode, true)); + cp = c->GetOptions("Prepared"); + u = c->GetOptions>("SimpleUnique"); + up = u->get()->GetOptions("Prepared"); + + ASSERT_OK(c->ConfigureFromString(config_options_, "prepared=0")); + ASSERT_EQ(*cp, 1); + ASSERT_EQ(*up, 0); +} + +TEST_F(ConfigurableTest, CopyObjectTest) { + class CopyConfigurable : public Configurable { + public: + CopyConfigurable() : prepared_(0), validated_(0) {} + Status PrepareOptions(const ConfigOptions& options) override { + prepared_++; + return Configurable::PrepareOptions(options); + } + Status ValidateOptions(const DBOptions& db_opts, + const ColumnFamilyOptions& cf_opts) const override { + validated_++; + return Configurable::ValidateOptions(db_opts, cf_opts); + } + int prepared_; + mutable int validated_; + }; + + CopyConfigurable c1; + ConfigOptions config_options; + Options options; + + ASSERT_OK(c1.PrepareOptions(config_options)); + ASSERT_OK(c1.ValidateOptions(options, options)); + ASSERT_EQ(c1.prepared_, 1); + ASSERT_EQ(c1.validated_, 1); + CopyConfigurable c2 = c1; + ASSERT_OK(c1.PrepareOptions(config_options)); + ASSERT_OK(c1.ValidateOptions(options, options)); + ASSERT_EQ(c2.prepared_, 1); + ASSERT_EQ(c2.validated_, 1); + ASSERT_EQ(c1.prepared_, 2); + ASSERT_EQ(c1.validated_, 2); +} + +TEST_F(ConfigurableTest, MutableOptionsTest) { + static std::unordered_map imm_option_info = { + {"imm", OptionTypeInfo::Struct("imm", &simple_option_info, 0, + OptionVerificationType::kNormal, + OptionTypeFlags::kNone)}, + }; + + class MutableConfigurable : public SimpleConfigurable { + public: + MutableConfigurable() + : SimpleConfigurable("mutable", TestConfigMode::kDefaultMode | + TestConfigMode::kUniqueMode | + TestConfigMode::kSharedMode) { + RegisterOptions("struct", &options_, &struct_option_info); + RegisterOptions("imm", &options_, &imm_option_info); + } + }; + MutableConfigurable mc; + ConfigOptions options = config_options_; + + ASSERT_OK(mc.ConfigureOption(options, "bool", "true")); + ASSERT_OK(mc.ConfigureOption(options, "int", "42")); + auto* opts = mc.GetOptions("mutable"); + ASSERT_NE(opts, nullptr); + ASSERT_EQ(opts->i, 42); + ASSERT_EQ(opts->b, true); + ASSERT_OK(mc.ConfigureOption(options, "struct", "{bool=false;}")); + ASSERT_OK(mc.ConfigureOption(options, "imm", "{int=55;}")); + + options.mutable_options_only = true; + + // Now only mutable options should be settable. + ASSERT_NOK(mc.ConfigureOption(options, "bool", "true")); + ASSERT_OK(mc.ConfigureOption(options, "int", "24")); + ASSERT_EQ(opts->i, 24); + ASSERT_EQ(opts->b, false); + ASSERT_NOK(mc.ConfigureFromString(options, "bool=false;int=33;")); + ASSERT_EQ(opts->i, 24); + ASSERT_EQ(opts->b, false); + + // Setting options through an immutable struct fails + ASSERT_NOK(mc.ConfigureOption(options, "imm", "{int=55;}")); + ASSERT_NOK(mc.ConfigureOption(options, "imm.int", "55")); + ASSERT_EQ(opts->i, 24); + ASSERT_EQ(opts->b, false); + + // Setting options through an mutable struct succeeds + ASSERT_OK(mc.ConfigureOption(options, "struct", "{int=44;}")); + ASSERT_EQ(opts->i, 44); + ASSERT_OK(mc.ConfigureOption(options, "struct.int", "55")); + ASSERT_EQ(opts->i, 55); + + // Setting nested immutable configurable options fail + ASSERT_NOK(mc.ConfigureOption(options, "shared", "{bool=true;}")); + ASSERT_NOK(mc.ConfigureOption(options, "shared.bool", "true")); + + // Setting nested mutable configurable options succeeds + ASSERT_OK(mc.ConfigureOption(options, "unique", "{bool=true}")); + ASSERT_OK(mc.ConfigureOption(options, "unique.bool", "true")); +} + +TEST_F(ConfigurableTest, DeprecatedOptionsTest) { + static std::unordered_map + deprecated_option_info = { + {"deprecated", + {offsetof(struct TestOptions, b), OptionType::kBoolean, + OptionVerificationType::kDeprecated, OptionTypeFlags::kNone}}}; + std::unique_ptr orig; + orig.reset(SimpleConfigurable::Create("simple", TestConfigMode::kDefaultMode, + &deprecated_option_info)); + auto* opts = orig->GetOptions("simple"); + ASSERT_NE(opts, nullptr); + opts->d = true; + ASSERT_OK(orig->ConfigureOption(config_options_, "deprecated", "false")); + ASSERT_TRUE(opts->d); + ASSERT_OK(orig->ConfigureFromString(config_options_, "deprecated=false")); + ASSERT_TRUE(opts->d); +} + +TEST_F(ConfigurableTest, AliasOptionsTest) { + static std::unordered_map alias_option_info = { + {"bool", + {offsetof(struct TestOptions, b), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"alias", + {offsetof(struct TestOptions, b), OptionType::kBoolean, + OptionVerificationType::kAlias, OptionTypeFlags::kNone, 0}}}; + std::unique_ptr orig; + orig.reset(SimpleConfigurable::Create("simple", TestConfigMode::kDefaultMode, + &alias_option_info)); + auto* opts = orig->GetOptions("simple"); + ASSERT_NE(opts, nullptr); + ASSERT_OK(orig->ConfigureOption(config_options_, "bool", "false")); + ASSERT_FALSE(opts->b); + ASSERT_OK(orig->ConfigureOption(config_options_, "alias", "true")); + ASSERT_TRUE(opts->b); + std::string opts_str; + ASSERT_OK(orig->GetOptionString(config_options_, &opts_str)); + ASSERT_EQ(opts_str.find("alias"), std::string::npos); + + ASSERT_OK(orig->ConfigureOption(config_options_, "bool", "false")); + ASSERT_FALSE(opts->b); + ASSERT_OK(orig->GetOption(config_options_, "alias", &opts_str)); + ASSERT_EQ(opts_str, "false"); +} + +TEST_F(ConfigurableTest, NestedUniqueConfigTest) { + std::unique_ptr simple; + simple.reset( + SimpleConfigurable::Create("Outer", TestConfigMode::kAllOptMode)); + const auto outer = simple->GetOptions("Outer"); + const auto unique = + simple->GetOptions>("OuterUnique"); + ASSERT_NE(outer, nullptr); + ASSERT_NE(unique, nullptr); + ASSERT_OK( + simple->ConfigureFromString(config_options_, "int=24;string=outer")); + ASSERT_OK(simple->ConfigureFromString(config_options_, + "unique={int=42;string=nested}")); + const auto inner = unique->get()->GetOptions("UniqueOuter"); + ASSERT_NE(inner, nullptr); + ASSERT_EQ(outer->i, 24); + ASSERT_EQ(outer->s, "outer"); + ASSERT_EQ(inner->i, 42); + ASSERT_EQ(inner->s, "nested"); +} + +TEST_F(ConfigurableTest, NestedSharedConfigTest) { + std::unique_ptr simple; + simple.reset(SimpleConfigurable::Create( + "Outer", TestConfigMode::kDefaultMode | TestConfigMode::kSharedMode)); + ASSERT_OK( + simple->ConfigureFromString(config_options_, "int=24;string=outer")); + ASSERT_OK(simple->ConfigureFromString(config_options_, + "shared={int=42;string=nested}")); + const auto outer = simple->GetOptions("Outer"); + const auto shared = + simple->GetOptions>("OuterShared"); + ASSERT_NE(outer, nullptr); + ASSERT_NE(shared, nullptr); + const auto inner = shared->get()->GetOptions("SharedOuter"); + ASSERT_NE(inner, nullptr); + ASSERT_EQ(outer->i, 24); + ASSERT_EQ(outer->s, "outer"); + ASSERT_EQ(inner->i, 42); + ASSERT_EQ(inner->s, "nested"); +} + +TEST_F(ConfigurableTest, NestedRawConfigTest) { + std::unique_ptr simple; + simple.reset(SimpleConfigurable::Create( + "Outer", TestConfigMode::kDefaultMode | TestConfigMode::kRawPtrMode)); + ASSERT_OK( + simple->ConfigureFromString(config_options_, "int=24;string=outer")); + ASSERT_OK(simple->ConfigureFromString(config_options_, + "pointer={int=42;string=nested}")); + const auto outer = simple->GetOptions("Outer"); + const auto pointer = simple->GetOptions("OuterPointer"); + ASSERT_NE(outer, nullptr); + ASSERT_NE(pointer, nullptr); + const auto inner = (*pointer)->GetOptions("PointerOuter"); + ASSERT_NE(inner, nullptr); + ASSERT_EQ(outer->i, 24); + ASSERT_EQ(outer->s, "outer"); + ASSERT_EQ(inner->i, 42); + ASSERT_EQ(inner->s, "nested"); +} + +TEST_F(ConfigurableTest, MatchesTest) { + std::string mismatch; + std::unique_ptr base, copy; + base.reset(SimpleConfigurable::Create( + "simple", TestConfigMode::kDefaultMode | TestConfigMode::kNestedMode)); + copy.reset(SimpleConfigurable::Create( + "simple", TestConfigMode::kDefaultMode | TestConfigMode::kNestedMode)); + ASSERT_OK(base->ConfigureFromString( + config_options_, + "int=11;string=outer;unique={int=22;string=u};shared={int=33;string=s}")); + ASSERT_OK(copy->ConfigureFromString( + config_options_, + "int=11;string=outer;unique={int=22;string=u};shared={int=33;string=s}")); + ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); + ASSERT_OK(base->ConfigureOption(config_options_, "shared", "int=44")); + ASSERT_FALSE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); + ASSERT_EQ(mismatch, "shared.int"); + std::string c1value, c2value; + ASSERT_OK(base->GetOption(config_options_, mismatch, &c1value)); + ASSERT_OK(copy->GetOption(config_options_, mismatch, &c2value)); + ASSERT_NE(c1value, c2value); +} + +static Configurable* SimpleStructFactory() { + return SimpleConfigurable::Create( + "simple-struct", TestConfigMode::kDefaultMode, &struct_option_info); +} + +TEST_F(ConfigurableTest, ConfigureStructTest) { + std::unique_ptr base(SimpleStructFactory()); + std::unique_ptr copy(SimpleStructFactory()); + std::string opt_str, value; + std::string mismatch; + std::unordered_set names; + + ASSERT_OK( + base->ConfigureFromString(config_options_, "struct={int=10; string=10}")); + ASSERT_OK(base->GetOptionString(config_options_, &opt_str)); + ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str)); + ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); + ASSERT_OK(base->GetOptionNames(config_options_, &names)); + ASSERT_EQ(names.size(), 1); + ASSERT_EQ(*(names.begin()), "struct"); + ASSERT_OK( + base->ConfigureFromString(config_options_, "struct={int=20; string=20}")); + ASSERT_OK(base->GetOption(config_options_, "struct", &value)); + ASSERT_OK(copy->ConfigureOption(config_options_, "struct", value)); + ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); + + ASSERT_NOK(base->ConfigureFromString(config_options_, + "struct={int=10; string=10; bad=11}")); + ASSERT_OK(base->ConfigureOption(config_options_, "struct.int", "42")); + ASSERT_NOK(base->ConfigureOption(config_options_, "struct.bad", "42")); + ASSERT_NOK(base->GetOption(config_options_, "struct.bad", &value)); + ASSERT_OK(base->GetOption(config_options_, "struct.int", &value)); + ASSERT_EQ(value, "42"); +} + +TEST_F(ConfigurableTest, ConfigurableEnumTest) { + std::unique_ptr base, copy; + base.reset(SimpleConfigurable::Create("e", TestConfigMode::kEnumMode)); + copy.reset(SimpleConfigurable::Create("e", TestConfigMode::kEnumMode)); + + std::string opts_str; + std::string mismatch; + + ASSERT_OK(base->ConfigureFromString(config_options_, "enum=B")); + ASSERT_FALSE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); + ASSERT_OK(base->GetOptionString(config_options_, &opts_str)); + ASSERT_OK(copy->ConfigureFromString(config_options_, opts_str)); + ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); + ASSERT_NOK(base->ConfigureOption(config_options_, "enum", "bad")); + ASSERT_NOK(base->ConfigureOption(config_options_, "unknown", "bad")); +} + +static std::unordered_map noserialize_option_info = + { + {"int", + {offsetof(struct TestOptions, i), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kDontSerialize}}, +}; + +TEST_F(ConfigurableTest, TestNoSerialize) { + std::unique_ptr base; + base.reset(SimpleConfigurable::Create("c", TestConfigMode::kDefaultMode, + &noserialize_option_info)); + std::string opts_str, value; + ASSERT_OK(base->ConfigureFromString(config_options_, "int=10")); + ASSERT_OK(base->GetOptionString(config_options_, &opts_str)); + ASSERT_EQ(opts_str, ""); + ASSERT_NOK(base->GetOption(config_options_, "int", &value)); +} + +TEST_F(ConfigurableTest, TestNoCompare) { + std::unordered_map nocomp_option_info = { + {"int", + {offsetof(struct TestOptions, i), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kCompareNever}}, + }; + std::unordered_map normal_option_info = { + {"int", + {offsetof(struct TestOptions, i), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + }; + + std::unique_ptr base, copy; + base.reset(SimpleConfigurable::Create("c", TestConfigMode::kDefaultMode, + &nocomp_option_info)); + copy.reset(SimpleConfigurable::Create("c", TestConfigMode::kDefaultMode, + &normal_option_info)); + ASSERT_OK(base->ConfigureFromString(config_options_, "int=10")); + ASSERT_OK(copy->ConfigureFromString(config_options_, "int=20")); + std::string bvalue, cvalue, mismatch; + ASSERT_OK(base->GetOption(config_options_, "int", &bvalue)); + ASSERT_OK(copy->GetOption(config_options_, "int", &cvalue)); + ASSERT_EQ(bvalue, "10"); + ASSERT_EQ(cvalue, "20"); + ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); + ASSERT_FALSE(copy->AreEquivalent(config_options_, base.get(), &mismatch)); +} + +TEST_F(ConfigurableTest, NullOptionMapTest) { + std::unique_ptr base; + std::unordered_set names; + std::string str; + + base.reset( + SimpleConfigurable::Create("c", TestConfigMode::kDefaultMode, nullptr)); + ASSERT_NOK(base->ConfigureFromString(config_options_, "int=10")); + ASSERT_NOK(base->ConfigureFromString(config_options_, "int=20")); + ASSERT_NOK(base->ConfigureOption(config_options_, "int", "20")); + ASSERT_NOK(base->GetOption(config_options_, "int", &str)); + ASSERT_NE(base->GetOptions("c"), nullptr); + ASSERT_OK(base->GetOptionNames(config_options_, &names)); + ASSERT_EQ(names.size(), 0UL); + ASSERT_OK(base->PrepareOptions(config_options_)); + ASSERT_OK(base->ValidateOptions(DBOptions(), ColumnFamilyOptions())); + std::unique_ptr copy; + copy.reset( + SimpleConfigurable::Create("c", TestConfigMode::kDefaultMode, nullptr)); + ASSERT_OK(base->GetOptionString(config_options_, &str)); + ASSERT_OK(copy->ConfigureFromString(config_options_, str)); + ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &str)); +} + +static std::unordered_map TestFactories = { + {"Simple", []() { return SimpleConfigurable::Create("simple"); }}, + {"Struct", []() { return SimpleStructFactory(); }}, + {"Unique", + []() { + return SimpleConfigurable::Create( + "simple", TestConfigMode::kSimpleMode | TestConfigMode::kUniqueMode); + }}, + {"Shared", + []() { + return SimpleConfigurable::Create( + "simple", TestConfigMode::kSimpleMode | TestConfigMode::kSharedMode); + }}, + {"Nested", + []() { + return SimpleConfigurable::Create( + "simple", TestConfigMode::kSimpleMode | TestConfigMode::kNestedMode); + }}, + {"Mutable", + []() { + return SimpleConfigurable::Create("simple", + TestConfigMode::kMutableMode | + TestConfigMode::kSimpleMode | + TestConfigMode::kNestedMode); + }}, + {"ThreeDeep", + []() { + Configurable* simple = SimpleConfigurable::Create( + "Simple", + TestConfigMode::kUniqueMode | TestConfigMode::kDefaultMode); + auto* unique = + simple->GetOptions>("SimpleUnique"); + unique->reset(SimpleConfigurable::Create( + "Child", + TestConfigMode::kUniqueMode | TestConfigMode::kDefaultMode)); + unique = unique->get()->GetOptions>( + "ChildUnique"); + unique->reset( + SimpleConfigurable::Create("Child", TestConfigMode::kDefaultMode)); + return simple; + }}, + {"DBOptions", + []() { + auto config = DBOptionsAsConfigurable(DBOptions()); + return config.release(); + }}, + {"CFOptions", + []() { + auto config = CFOptionsAsConfigurable(ColumnFamilyOptions()); + return config.release(); + }}, + {"BlockBased", []() { return NewBlockBasedTableFactory(); }}, +}; + +class ConfigurableParamTest : public ConfigurableTest, + virtual public ::testing::WithParamInterface< + std::pair> { + public: + ConfigurableParamTest() { + type_ = GetParam().first; + configuration_ = GetParam().second; + assert(TestFactories.find(type_) != TestFactories.end()); + object_.reset(CreateConfigurable()); + } + + Configurable* CreateConfigurable() { + const auto& iter = TestFactories.find(type_); + return (iter->second)(); + } + + void TestConfigureOptions(const ConfigOptions& opts); + std::string type_; + std::string configuration_; + std::unique_ptr object_; +}; + +void ConfigurableParamTest::TestConfigureOptions( + const ConfigOptions& config_options) { + std::unique_ptr base, copy; + std::unordered_set names; + std::string opt_str, mismatch; + + base.reset(CreateConfigurable()); + copy.reset(CreateConfigurable()); + + ASSERT_OK(base->ConfigureFromString(config_options, configuration_)); + ASSERT_OK(base->GetOptionString(config_options, &opt_str)); + ASSERT_OK(copy->ConfigureFromString(config_options, opt_str)); + ASSERT_OK(copy->GetOptionString(config_options, &opt_str)); + ASSERT_TRUE(base->AreEquivalent(config_options, copy.get(), &mismatch)); + + copy.reset(CreateConfigurable()); + ASSERT_OK(base->GetOptionNames(config_options, &names)); + std::unordered_map unused; + bool found_one = false; + for (auto name : names) { + std::string value; + Status s = base->GetOption(config_options, name, &value); + if (s.ok()) { + s = copy->ConfigureOption(config_options, name, value); + if (s.ok() || s.IsNotSupported()) { + found_one = true; + } else { + unused[name] = value; + } + } else { + ASSERT_TRUE(s.IsNotSupported()); + } + } + ASSERT_TRUE(found_one || names.empty()); + while (found_one && !unused.empty()) { + found_one = false; + for (auto iter = unused.begin(); iter != unused.end();) { + if (copy->ConfigureOption(config_options, iter->first, iter->second) + .ok()) { + found_one = true; + iter = unused.erase(iter); + } else { + ++iter; + } + } + } + ASSERT_EQ(0, unused.size()); + ASSERT_TRUE(base->AreEquivalent(config_options, copy.get(), &mismatch)); +} + +TEST_P(ConfigurableParamTest, GetDefaultOptionsTest) { + TestConfigureOptions(config_options_); +} + +TEST_P(ConfigurableParamTest, ConfigureFromPropsTest) { + std::string opt_str, mismatch; + std::unordered_set names; + std::unique_ptr copy(CreateConfigurable()); + + ASSERT_OK(object_->ConfigureFromString(config_options_, configuration_)); + config_options_.delimiter = "\n"; + ASSERT_OK(object_->GetOptionString(config_options_, &opt_str)); + std::istringstream iss(opt_str); + std::unordered_map copy_map; + std::string line; + for (int line_num = 0; std::getline(iss, line); line_num++) { + std::string name; + std::string value; + ASSERT_OK( + RocksDBOptionsParser::ParseStatement(&name, &value, line, line_num)); + copy_map[name] = value; + } + ASSERT_OK(copy->ConfigureFromMap(config_options_, copy_map)); + ASSERT_TRUE(object_->AreEquivalent(config_options_, copy.get(), &mismatch)); +} + +INSTANTIATE_TEST_CASE_P( + ParamTest, ConfigurableParamTest, + testing::Values( + std::pair("Simple", + "int=42;bool=true;string=s"), + std::pair( + "Mutable", "int=42;unique={int=33;string=unique}"), + std::pair( + "Struct", "struct={int=33;bool=true;string=s;}"), + std::pair("Shared", + "int=33;bool=true;string=outer;" + "shared={int=42;string=shared}"), + std::pair("Unique", + "int=33;bool=true;string=outer;" + "unique={int=42;string=unique}"), + std::pair("Nested", + "int=11;bool=true;string=outer;" + "pointer={int=22;string=pointer};" + "unique={int=33;string=unique};" + "shared={int=44;string=shared}"), + std::pair("ThreeDeep", + "int=11;bool=true;string=outer;" + "unique={int=22;string=inner;" + "unique={int=33;string=unique}};"), + std::pair("DBOptions", + "max_background_jobs=100;" + "max_open_files=200;"), + std::pair("CFOptions", + "table_factory=BlockBasedTable;" + "disable_auto_compactions=true;"), + std::pair("BlockBased", + "block_size=1024;" + "no_block_cache=true;"))); + +} // namespace test +} // namespace ROCKSDB_NAMESPACE +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); +#ifdef GFLAGS + ParseCommandLineFlags(&argc, &argv, true); +#endif // GFLAGS + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/options/configurable_test.h b/librocksdb-sys/rocksdb/options/configurable_test.h new file mode 100644 index 0000000..3d6fe84 --- /dev/null +++ b/librocksdb-sys/rocksdb/options/configurable_test.h @@ -0,0 +1,116 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include +#include +#include + +#include "options/configurable_helper.h" +#include "rocksdb/configurable.h" +#include "rocksdb/utilities/options_type.h" + +namespace ROCKSDB_NAMESPACE { +struct ColumnFamilyOptions; +struct DBOptions; + +namespace test { +enum TestEnum { kTestA, kTestB }; + +static const std::unordered_map test_enum_map = { + {"A", TestEnum::kTestA}, + {"B", TestEnum::kTestB}, +}; + +struct TestOptions { + int i = 0; + bool b = false; + bool d = true; + TestEnum e = TestEnum::kTestA; + std::string s = ""; + std::string u = ""; +}; + +static std::unordered_map simple_option_info = { + {"int", + {offsetof(struct TestOptions, i), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, + {"bool", + {offsetof(struct TestOptions, b), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"string", + {offsetof(struct TestOptions, s), OptionType::kString, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, +}; + +static std::unordered_map enum_option_info = { + {"enum", + OptionTypeInfo::Enum(offsetof(struct TestOptions, e), &test_enum_map)} +}; + +static std::unordered_map unique_option_info = { + {"unique", + {0, OptionType::kConfigurable, OptionVerificationType::kNormal, + (OptionTypeFlags::kUnique | OptionTypeFlags::kMutable)}}, +}; + +static std::unordered_map shared_option_info = { + {"shared", + {0, OptionType::kConfigurable, OptionVerificationType::kNormal, + (OptionTypeFlags::kShared)}}, +}; +static std::unordered_map pointer_option_info = { + {"pointer", + {0, OptionType::kConfigurable, OptionVerificationType::kNormal, + OptionTypeFlags::kRawPointer}}, +}; + +enum TestConfigMode { + kEmptyMode = 0x0, // Don't register anything + kMutableMode = 0x01, // Configuration is mutable + kSimpleMode = 0x02, // Use the simple options + kEnumMode = 0x04, // Use the enum options + kDefaultMode = kSimpleMode, // Use no inner nested configurations + kSharedMode = 0x10, // Use shared configuration + kUniqueMode = 0x20, // Use unique configuration + kRawPtrMode = 0x40, // Use pointer configuration + kNestedMode = (kSharedMode | kUniqueMode | kRawPtrMode), + kAllOptMode = (kNestedMode | kEnumMode | kSimpleMode), +}; + +template +class TestConfigurable : public Configurable { + protected: + std::string name_; + std::string prefix_; + TestOptions options_; + + public: + std::unique_ptr unique_; + std::shared_ptr shared_; + T* pointer_; + + TestConfigurable(const std::string& name, int mode, + const std::unordered_map* map = + &simple_option_info) + : name_(name), pointer_(nullptr) { + prefix_ = "test." + name + "."; + if ((mode & TestConfigMode::kSimpleMode) != 0) { + RegisterOptions(name_, &options_, map); + } + if ((mode & TestConfigMode::kEnumMode) != 0) { + RegisterOptions(name_ + "Enum", &options_, &enum_option_info); + } + } + + ~TestConfigurable() override { delete pointer_; } +}; + +} // namespace test +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/options/customizable.cc b/librocksdb-sys/rocksdb/options/customizable.cc new file mode 100644 index 0000000..2f154d8 --- /dev/null +++ b/librocksdb-sys/rocksdb/options/customizable.cc @@ -0,0 +1,133 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "rocksdb/customizable.h" + +#include + +#include "options/options_helper.h" +#include "port/port.h" +#include "rocksdb/convenience.h" +#include "rocksdb/status.h" +#include "rocksdb/utilities/options_type.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +std::string Customizable::GetOptionName(const std::string& long_name) const { + const std::string& name = Name(); + size_t name_len = name.size(); + if (long_name.size() > name_len + 1 && + long_name.compare(0, name_len, name) == 0 && + long_name.at(name_len) == '.') { + return long_name.substr(name_len + 1); + } else { + return Configurable::GetOptionName(long_name); + } +} + +std::string Customizable::GenerateIndividualId() const { + std::ostringstream ostr; + ostr << Name() << "@" << static_cast(this) << "#" + << port::GetProcessID(); + return ostr.str(); +} + +Status Customizable::GetOption(const ConfigOptions& config_options, + const std::string& opt_name, + std::string* value) const { + if (opt_name == OptionTypeInfo::kIdPropName()) { + *value = GetId(); + return Status::OK(); + } else { + return Configurable::GetOption(config_options, opt_name, value); + } +} + +std::string Customizable::SerializeOptions(const ConfigOptions& config_options, + const std::string& prefix) const { + std::string result; + std::string parent; + std::string id = GetId(); + if (!config_options.IsShallow() && !id.empty()) { + parent = Configurable::SerializeOptions(config_options, ""); + } + if (parent.empty()) { + result = id; + } else { + result.append(prefix); + result.append(OptionTypeInfo::kIdPropName()); + result.append("="); + result.append(id); + result.append(config_options.delimiter); + result.append(parent); + } + return result; +} + + +bool Customizable::AreEquivalent(const ConfigOptions& config_options, + const Configurable* other, + std::string* mismatch) const { + if (config_options.sanity_level > ConfigOptions::kSanityLevelNone && + this != other) { + const Customizable* custom = reinterpret_cast(other); + if (custom == nullptr) { // Cast failed + return false; + } else if (GetId() != custom->GetId()) { + *mismatch = OptionTypeInfo::kIdPropName(); + return false; + } else if (config_options.sanity_level > + ConfigOptions::kSanityLevelLooselyCompatible) { + bool matches = + Configurable::AreEquivalent(config_options, other, mismatch); + return matches; + } + } + return true; +} + +Status Customizable::GetOptionsMap( + const ConfigOptions& config_options, const Customizable* customizable, + const std::string& value, std::string* id, + std::unordered_map* props) { + Status status; + if (value.empty() || value == kNullptrString) { + *id = ""; + props->clear(); + } else if (customizable != nullptr) { + status = + Configurable::GetOptionsMap(value, customizable->GetId(), id, props); + if (status.ok() && customizable->IsInstanceOf(*id)) { + // The new ID and the old ID match, so the objects are the same type. + // Try to get the existing options, ignoring any errors + ConfigOptions embedded = config_options; + embedded.delimiter = ";"; + std::string curr_opts; + if (customizable->GetOptionString(embedded, &curr_opts).ok()) { + std::unordered_map curr_props; + if (StringToMap(curr_opts, &curr_props).ok()) { + props->insert(curr_props.begin(), curr_props.end()); + } + } + } + } else { + status = Configurable::GetOptionsMap(value, "", id, props); + } + return status; +} + +Status Customizable::ConfigureNewObject( + const ConfigOptions& config_options, Customizable* object, + const std::unordered_map& opt_map) { + Status status; + if (object != nullptr) { + status = object->ConfigureFromMap(config_options, opt_map); + } else if (!opt_map.empty()) { + status = Status::InvalidArgument("Cannot configure null object "); + } + return status; +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/options/customizable_test.cc b/librocksdb-sys/rocksdb/options/customizable_test.cc new file mode 100644 index 0000000..d887777 --- /dev/null +++ b/librocksdb-sys/rocksdb/options/customizable_test.cc @@ -0,0 +1,2116 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "rocksdb/customizable.h" + +#include +#include +#include +#include +#include + +#include "db/db_test_util.h" +#include "memory/jemalloc_nodump_allocator.h" +#include "memory/memkind_kmem_allocator.h" +#include "options/options_helper.h" +#include "options/options_parser.h" +#include "port/stack_trace.h" +#include "rocksdb/convenience.h" +#include "rocksdb/env_encryption.h" +#include "rocksdb/file_checksum.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/flush_block_policy.h" +#include "rocksdb/memory_allocator.h" +#include "rocksdb/secondary_cache.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/sst_partitioner.h" +#include "rocksdb/statistics.h" +#include "rocksdb/utilities/customizable_util.h" +#include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/options_type.h" +#include "table/block_based/filter_policy_internal.h" +#include "table/block_based/flush_block_policy_impl.h" +#include "table/mock_table.h" +#include "test_util/mock_time_env.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/file_checksum_helper.h" +#include "util/string_util.h" +#include "utilities/compaction_filters/remove_emptyvalue_compactionfilter.h" +#include "utilities/memory_allocators.h" +#include "utilities/merge_operators/bytesxor.h" +#include "utilities/merge_operators/sortlist.h" +#include "utilities/merge_operators/string_append/stringappend.h" +#include "utilities/merge_operators/string_append/stringappend2.h" + +#ifndef GFLAGS +bool FLAGS_enable_print = false; +#else +#include "util/gflags_compat.h" +using GFLAGS_NAMESPACE::ParseCommandLineFlags; +DEFINE_bool(enable_print, false, "Print options generated to console."); +#endif // GFLAGS + +namespace ROCKSDB_NAMESPACE { +namespace { +class StringLogger : public Logger { + public: + using Logger::Logv; + void Logv(const char* format, va_list ap) override { + char buffer[1000]; + vsnprintf(buffer, sizeof(buffer), format, ap); + string_.append(buffer); + } + const std::string& str() const { return string_; } + void clear() { string_.clear(); } + + private: + std::string string_; +}; + +class TestCustomizable : public Customizable { + public: + TestCustomizable(const std::string& name) : name_(name) {} + // Method to allow CheckedCast to work for this class + static const char* kClassName() { + return "TestCustomizable"; + } + + const char* Name() const override { return name_.c_str(); } + static const char* Type() { return "test.custom"; } + static Status CreateFromString(const ConfigOptions& opts, + const std::string& value, + std::unique_ptr* result); + static Status CreateFromString(const ConfigOptions& opts, + const std::string& value, + std::shared_ptr* result); + static Status CreateFromString(const ConfigOptions& opts, + const std::string& value, + TestCustomizable** result); + bool IsInstanceOf(const std::string& name) const override { + if (name == kClassName()) { + return true; + } else { + return Customizable::IsInstanceOf(name); + } + } + + protected: + const std::string name_; +}; + +struct AOptions { + static const char* kName() { return "A"; } + int i = 0; + bool b = false; +}; + +static std::unordered_map a_option_info = { + {"int", + {offsetof(struct AOptions, i), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, + {"bool", + {offsetof(struct AOptions, b), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, +}; + +class ACustomizable : public TestCustomizable { + public: + explicit ACustomizable(const std::string& id) + : TestCustomizable("A"), id_(id) { + RegisterOptions(&opts_, &a_option_info); + } + std::string GetId() const override { return id_; } + static const char* kClassName() { return "A"; } + + private: + AOptions opts_; + const std::string id_; +}; + +struct BOptions { + std::string s; + bool b = false; +}; + +static std::unordered_map b_option_info = { + {"string", + {offsetof(struct BOptions, s), OptionType::kString, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"bool", + {offsetof(struct BOptions, b), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, +}; + +class BCustomizable : public TestCustomizable { + private: + public: + explicit BCustomizable(const std::string& name) : TestCustomizable(name) { + RegisterOptions(name, &opts_, &b_option_info); + } + static const char* kClassName() { return "B"; } + + private: + BOptions opts_; +}; + +static int A_count = 0; +static int RegisterCustomTestObjects(ObjectLibrary& library, + const std::string& /*arg*/) { + library.AddFactory( + ObjectLibrary::PatternEntry("A", true).AddSeparator("_"), + [](const std::string& name, std::unique_ptr* guard, + std::string* /* msg */) { + guard->reset(new ACustomizable(name)); + A_count++; + return guard->get(); + }); + library.AddFactory( + "B", [](const std::string& name, std::unique_ptr* guard, + std::string* /* msg */) { + guard->reset(new BCustomizable(name)); + return guard->get(); + }); + + library.AddFactory( + "S", [](const std::string& name, + std::unique_ptr* /* guard */, + std::string* /* msg */) { return new BCustomizable(name); }); + size_t num_types; + return static_cast(library.GetFactoryCount(&num_types)); +} + +struct SimpleOptions { + static const char* kName() { return "simple"; } + bool b = true; + std::unique_ptr cu; + std::shared_ptr cs; + TestCustomizable* cp = nullptr; +}; + +static std::unordered_map simple_option_info = { + {"bool", + {offsetof(struct SimpleOptions, b), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"unique", + OptionTypeInfo::AsCustomUniquePtr( + offsetof(struct SimpleOptions, cu), OptionVerificationType::kNormal, + OptionTypeFlags::kAllowNull)}, + {"shared", + OptionTypeInfo::AsCustomSharedPtr( + offsetof(struct SimpleOptions, cs), OptionVerificationType::kNormal, + OptionTypeFlags::kAllowNull)}, + {"pointer", + OptionTypeInfo::AsCustomRawPtr( + offsetof(struct SimpleOptions, cp), OptionVerificationType::kNormal, + OptionTypeFlags::kAllowNull)}, +}; + +class SimpleConfigurable : public Configurable { + private: + SimpleOptions simple_; + + public: + SimpleConfigurable() { RegisterOptions(&simple_, &simple_option_info); } + + explicit SimpleConfigurable( + const std::unordered_map* map) { + RegisterOptions(&simple_, map); + } +}; + +static void GetMapFromProperties( + const std::string& props, + std::unordered_map* map) { + std::istringstream iss(props); + std::unordered_map copy_map; + std::string line; + map->clear(); + for (int line_num = 0; std::getline(iss, line); line_num++) { + std::string name; + std::string value; + ASSERT_OK( + RocksDBOptionsParser::ParseStatement(&name, &value, line, line_num)); + (*map)[name] = value; + } +} +} // namespace + +Status TestCustomizable::CreateFromString( + const ConfigOptions& config_options, const std::string& value, + std::shared_ptr* result) { + return LoadSharedObject(config_options, value, result); +} + +Status TestCustomizable::CreateFromString( + const ConfigOptions& config_options, const std::string& value, + std::unique_ptr* result) { + return LoadUniqueObject(config_options, value, result); +} + +Status TestCustomizable::CreateFromString(const ConfigOptions& config_options, + const std::string& value, + TestCustomizable** result) { + return LoadStaticObject(config_options, value, result); +} + +class CustomizableTest : public testing::Test { + public: + CustomizableTest() { + config_options_.invoke_prepare_options = false; + config_options_.registry->AddLibrary("CustomizableTest", + RegisterCustomTestObjects, ""); + } + + ConfigOptions config_options_; +}; + +// Tests that a Customizable can be created by: +// - a simple name +// - a XXX.id option +// - a property with a name +TEST_F(CustomizableTest, CreateByNameTest) { + ObjectLibrary::Default()->AddFactory( + ObjectLibrary::PatternEntry("TEST", false).AddSeparator("_"), + [](const std::string& name, std::unique_ptr* guard, + std::string* /* msg */) { + guard->reset(new TestCustomizable(name)); + return guard->get(); + }); + std::unique_ptr configurable(new SimpleConfigurable()); + SimpleOptions* simple = configurable->GetOptions(); + ASSERT_NE(simple, nullptr); + ASSERT_OK( + configurable->ConfigureFromString(config_options_, "unique={id=TEST_1}")); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(simple->cu->GetId(), "TEST_1"); + ASSERT_OK( + configurable->ConfigureFromString(config_options_, "unique.id=TEST_2")); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(simple->cu->GetId(), "TEST_2"); + ASSERT_OK( + configurable->ConfigureFromString(config_options_, "unique=TEST_3")); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(simple->cu->GetId(), "TEST_3"); +} + +TEST_F(CustomizableTest, ToStringTest) { + std::unique_ptr custom(new TestCustomizable("test")); + ASSERT_EQ(custom->ToString(config_options_), "test"); +} + +TEST_F(CustomizableTest, SimpleConfigureTest) { + std::unordered_map opt_map = { + {"unique", "id=A;int=1;bool=true"}, + {"shared", "id=B;string=s"}, + }; + std::unique_ptr configurable(new SimpleConfigurable()); + ASSERT_OK(configurable->ConfigureFromMap(config_options_, opt_map)); + SimpleOptions* simple = configurable->GetOptions(); + ASSERT_NE(simple, nullptr); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(simple->cu->GetId(), "A"); + std::string opt_str; + std::string mismatch; + ASSERT_OK(configurable->GetOptionString(config_options_, &opt_str)); + std::unique_ptr copy(new SimpleConfigurable()); + ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str)); + ASSERT_TRUE( + configurable->AreEquivalent(config_options_, copy.get(), &mismatch)); +} + +TEST_F(CustomizableTest, ConfigureFromPropsTest) { + std::unordered_map opt_map = { + {"unique.id", "A"}, {"unique.A.int", "1"}, {"unique.A.bool", "true"}, + {"shared.id", "B"}, {"shared.B.string", "s"}, + }; + std::unique_ptr configurable(new SimpleConfigurable()); + ASSERT_OK(configurable->ConfigureFromMap(config_options_, opt_map)); + SimpleOptions* simple = configurable->GetOptions(); + ASSERT_NE(simple, nullptr); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(simple->cu->GetId(), "A"); + std::string opt_str; + std::string mismatch; + config_options_.delimiter = "\n"; + std::unordered_map props; + ASSERT_OK(configurable->GetOptionString(config_options_, &opt_str)); + GetMapFromProperties(opt_str, &props); + std::unique_ptr copy(new SimpleConfigurable()); + ASSERT_OK(copy->ConfigureFromMap(config_options_, props)); + ASSERT_TRUE( + configurable->AreEquivalent(config_options_, copy.get(), &mismatch)); +} + +TEST_F(CustomizableTest, ConfigureFromShortTest) { + std::unordered_map opt_map = { + {"unique.id", "A"}, {"unique.A.int", "1"}, {"unique.A.bool", "true"}, + {"shared.id", "B"}, {"shared.B.string", "s"}, + }; + std::unique_ptr configurable(new SimpleConfigurable()); + ASSERT_OK(configurable->ConfigureFromMap(config_options_, opt_map)); + SimpleOptions* simple = configurable->GetOptions(); + ASSERT_NE(simple, nullptr); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(simple->cu->GetId(), "A"); +} + +TEST_F(CustomizableTest, AreEquivalentOptionsTest) { + std::unordered_map opt_map = { + {"unique", "id=A;int=1;bool=true"}, + {"shared", "id=A;int=1;bool=true"}, + }; + std::string mismatch; + ConfigOptions config_options = config_options_; + std::unique_ptr c1(new SimpleConfigurable()); + std::unique_ptr c2(new SimpleConfigurable()); + ASSERT_OK(c1->ConfigureFromMap(config_options, opt_map)); + ASSERT_OK(c2->ConfigureFromMap(config_options, opt_map)); + ASSERT_TRUE(c1->AreEquivalent(config_options, c2.get(), &mismatch)); + SimpleOptions* simple = c1->GetOptions(); + ASSERT_TRUE( + simple->cu->AreEquivalent(config_options, simple->cs.get(), &mismatch)); + ASSERT_OK(simple->cu->ConfigureOption(config_options, "int", "2")); + ASSERT_FALSE( + simple->cu->AreEquivalent(config_options, simple->cs.get(), &mismatch)); + ASSERT_FALSE(c1->AreEquivalent(config_options, c2.get(), &mismatch)); + ConfigOptions loosely = config_options; + loosely.sanity_level = ConfigOptions::kSanityLevelLooselyCompatible; + ASSERT_TRUE(c1->AreEquivalent(loosely, c2.get(), &mismatch)); + ASSERT_TRUE(simple->cu->AreEquivalent(loosely, simple->cs.get(), &mismatch)); + + ASSERT_OK(c1->ConfigureOption(config_options, "shared", "id=B;string=3")); + ASSERT_TRUE(c1->AreEquivalent(loosely, c2.get(), &mismatch)); + ASSERT_FALSE(c1->AreEquivalent(config_options, c2.get(), &mismatch)); + ASSERT_FALSE(simple->cs->AreEquivalent(loosely, simple->cu.get(), &mismatch)); + simple->cs.reset(); + ASSERT_TRUE(c1->AreEquivalent(loosely, c2.get(), &mismatch)); + ASSERT_FALSE(c1->AreEquivalent(config_options, c2.get(), &mismatch)); +} + +// Tests that we can initialize a customizable from its options +TEST_F(CustomizableTest, ConfigureStandaloneCustomTest) { + std::unique_ptr base, copy; + const auto& registry = config_options_.registry; + ASSERT_OK(registry->NewUniqueObject("A", &base)); + ASSERT_OK(registry->NewUniqueObject("A", ©)); + ASSERT_OK(base->ConfigureFromString(config_options_, "int=33;bool=true")); + std::string opt_str; + std::string mismatch; + ASSERT_OK(base->GetOptionString(config_options_, &opt_str)); + ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str)); + ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); +} + +// Tests that we fail appropriately if the pattern is not registered +TEST_F(CustomizableTest, BadNameTest) { + config_options_.ignore_unsupported_options = false; + std::unique_ptr c1(new SimpleConfigurable()); + ASSERT_NOK( + c1->ConfigureFromString(config_options_, "unique.shared.id=bad name")); + config_options_.ignore_unsupported_options = true; + ASSERT_OK( + c1->ConfigureFromString(config_options_, "unique.shared.id=bad name")); +} + +// Tests that we fail appropriately if a bad option is passed to the underlying +// configurable +TEST_F(CustomizableTest, BadOptionTest) { + std::unique_ptr c1(new SimpleConfigurable()); + ConfigOptions ignore = config_options_; + ignore.ignore_unknown_options = true; + + ASSERT_NOK(c1->ConfigureFromString(config_options_, "A.int=11")); + ASSERT_NOK(c1->ConfigureFromString(config_options_, "shared={id=B;int=1}")); + ASSERT_OK(c1->ConfigureFromString(ignore, "shared={id=A;string=s}")); + ASSERT_NOK(c1->ConfigureFromString(config_options_, "B.int=11")); + ASSERT_OK(c1->ConfigureFromString(ignore, "B.int=11")); + ASSERT_NOK(c1->ConfigureFromString(config_options_, "A.string=s")); + ASSERT_OK(c1->ConfigureFromString(ignore, "A.string=s")); + // Test as detached + ASSERT_NOK( + c1->ConfigureFromString(config_options_, "shared.id=A;A.string=b}")); + ASSERT_OK(c1->ConfigureFromString(ignore, "shared.id=A;A.string=s}")); +} + +TEST_F(CustomizableTest, FailingFactoryTest) { + std::shared_ptr registry = ObjectRegistry::NewInstance(); + std::unique_ptr c1(new SimpleConfigurable()); + ConfigOptions ignore = config_options_; + + Status s; + ignore.registry->AddLibrary("failing")->AddFactory( + "failing", + [](const std::string& /*uri*/, + std::unique_ptr* /*guard */, std::string* errmsg) { + *errmsg = "Bad Factory"; + return nullptr; + }); + + // If we are ignoring unknown and unsupported options, will see + // different errors for failing versus missing + ignore.ignore_unknown_options = false; + ignore.ignore_unsupported_options = false; + s = c1->ConfigureFromString(ignore, "shared.id=failing"); + ASSERT_TRUE(s.IsInvalidArgument()); + s = c1->ConfigureFromString(ignore, "unique.id=failing"); + ASSERT_TRUE(s.IsInvalidArgument()); + s = c1->ConfigureFromString(ignore, "shared.id=missing"); + ASSERT_TRUE(s.IsNotSupported()); + s = c1->ConfigureFromString(ignore, "unique.id=missing"); + ASSERT_TRUE(s.IsNotSupported()); + + // If we are ignoring unsupported options, will see + // errors for failing but not missing + ignore.ignore_unknown_options = false; + ignore.ignore_unsupported_options = true; + s = c1->ConfigureFromString(ignore, "shared.id=failing"); + ASSERT_TRUE(s.IsInvalidArgument()); + s = c1->ConfigureFromString(ignore, "unique.id=failing"); + ASSERT_TRUE(s.IsInvalidArgument()); + + ASSERT_OK(c1->ConfigureFromString(ignore, "shared.id=missing")); + ASSERT_OK(c1->ConfigureFromString(ignore, "unique.id=missing")); + + // If we are ignoring unknown options, will see no errors + // for failing or missing + ignore.ignore_unknown_options = true; + ignore.ignore_unsupported_options = false; + ASSERT_OK(c1->ConfigureFromString(ignore, "shared.id=failing")); + ASSERT_OK(c1->ConfigureFromString(ignore, "unique.id=failing")); + ASSERT_OK(c1->ConfigureFromString(ignore, "shared.id=missing")); + ASSERT_OK(c1->ConfigureFromString(ignore, "unique.id=missing")); +} + +// Tests that different IDs lead to different objects +TEST_F(CustomizableTest, UniqueIdTest) { + std::unique_ptr base(new SimpleConfigurable()); + ASSERT_OK(base->ConfigureFromString(config_options_, + "unique={id=A_1;int=1;bool=true}")); + SimpleOptions* simple = base->GetOptions(); + ASSERT_NE(simple, nullptr); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(simple->cu->GetId(), std::string("A_1")); + std::string opt_str; + std::string mismatch; + ASSERT_OK(base->GetOptionString(config_options_, &opt_str)); + std::unique_ptr copy(new SimpleConfigurable()); + ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str)); + ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); + ASSERT_OK(base->ConfigureFromString(config_options_, + "unique={id=A_2;int=1;bool=true}")); + ASSERT_FALSE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); + ASSERT_EQ(simple->cu->GetId(), std::string("A_2")); +} + +TEST_F(CustomizableTest, IsInstanceOfTest) { + std::shared_ptr tc = std::make_shared("A_1"); + + ASSERT_EQ(tc->GetId(), std::string("A_1")); + ASSERT_TRUE(tc->IsInstanceOf("A")); + ASSERT_TRUE(tc->IsInstanceOf("TestCustomizable")); + ASSERT_FALSE(tc->IsInstanceOf("B")); + ASSERT_FALSE(tc->IsInstanceOf("A_1")); + ASSERT_EQ(tc->CheckedCast(), tc.get()); + ASSERT_EQ(tc->CheckedCast(), tc.get()); + ASSERT_EQ(tc->CheckedCast(), nullptr); + + tc.reset(new BCustomizable("B")); + ASSERT_TRUE(tc->IsInstanceOf("B")); + ASSERT_TRUE(tc->IsInstanceOf("TestCustomizable")); + ASSERT_FALSE(tc->IsInstanceOf("A")); + ASSERT_EQ(tc->CheckedCast(), tc.get()); + ASSERT_EQ(tc->CheckedCast(), tc.get()); + ASSERT_EQ(tc->CheckedCast(), nullptr); +} + +TEST_F(CustomizableTest, PrepareOptionsTest) { + static std::unordered_map p_option_info = { + {"can_prepare", + {0, OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + }; + + class PrepareCustomizable : public TestCustomizable { + public: + bool can_prepare_ = true; + + PrepareCustomizable() : TestCustomizable("P") { + RegisterOptions("Prepare", &can_prepare_, &p_option_info); + } + + Status PrepareOptions(const ConfigOptions& opts) override { + if (!can_prepare_) { + return Status::InvalidArgument("Cannot Prepare"); + } else { + return TestCustomizable::PrepareOptions(opts); + } + } + }; + + ObjectLibrary::Default()->AddFactory( + "P", + [](const std::string& /*name*/, std::unique_ptr* guard, + std::string* /* msg */) { + guard->reset(new PrepareCustomizable()); + return guard->get(); + }); + + std::unique_ptr base(new SimpleConfigurable()); + ConfigOptions prepared(config_options_); + prepared.invoke_prepare_options = true; + + ASSERT_OK(base->ConfigureFromString( + prepared, "unique=A_1; shared={id=B;string=s}; pointer.id=S")); + SimpleOptions* simple = base->GetOptions(); + ASSERT_NE(simple, nullptr); + ASSERT_NE(simple->cu, nullptr); + ASSERT_NE(simple->cs, nullptr); + ASSERT_NE(simple->cp, nullptr); + delete simple->cp; + base.reset(new SimpleConfigurable()); + ASSERT_OK(base->ConfigureFromString( + config_options_, "unique=A_1; shared={id=B;string=s}; pointer.id=S")); + + simple = base->GetOptions(); + ASSERT_NE(simple, nullptr); + ASSERT_NE(simple->cu, nullptr); + ASSERT_NE(simple->cs, nullptr); + ASSERT_NE(simple->cp, nullptr); + + ASSERT_OK(base->PrepareOptions(config_options_)); + delete simple->cp; + base.reset(new SimpleConfigurable()); + simple = base->GetOptions(); + ASSERT_NE(simple, nullptr); + + ASSERT_NOK( + base->ConfigureFromString(prepared, "unique={id=P; can_prepare=false}")); + ASSERT_EQ(simple->cu, nullptr); + + ASSERT_OK( + base->ConfigureFromString(prepared, "unique={id=P; can_prepare=true}")); + ASSERT_NE(simple->cu, nullptr); + + ASSERT_OK(base->ConfigureFromString(config_options_, + "unique={id=P; can_prepare=true}")); + ASSERT_NE(simple->cu, nullptr); + ASSERT_OK(simple->cu->PrepareOptions(prepared)); + + ASSERT_OK(base->ConfigureFromString(config_options_, + "unique={id=P; can_prepare=false}")); + ASSERT_NE(simple->cu, nullptr); + ASSERT_NOK(simple->cu->PrepareOptions(prepared)); +} + +namespace { +static std::unordered_map inner_option_info = { + {"inner", + OptionTypeInfo::AsCustomSharedPtr( + 0, OptionVerificationType::kNormal, OptionTypeFlags::kStringNameOnly)} +}; + +struct InnerOptions { + static const char* kName() { return "InnerOptions"; } + std::shared_ptr inner; +}; + +class InnerCustomizable : public Customizable { + public: + explicit InnerCustomizable(const std::shared_ptr& w) { + iopts_.inner = w; + RegisterOptions(&iopts_, &inner_option_info); + } + static const char* kClassName() { return "Inner"; } + const char* Name() const override { return kClassName(); } + + bool IsInstanceOf(const std::string& name) const override { + if (name == kClassName()) { + return true; + } else { + return Customizable::IsInstanceOf(name); + } + } + + protected: + const Customizable* Inner() const override { return iopts_.inner.get(); } + + private: + InnerOptions iopts_; +}; + +struct WrappedOptions1 { + static const char* kName() { return "WrappedOptions1"; } + int i = 42; +}; + +class WrappedCustomizable1 : public InnerCustomizable { + public: + explicit WrappedCustomizable1(const std::shared_ptr& w) + : InnerCustomizable(w) { + RegisterOptions(&wopts_, nullptr); + } + const char* Name() const override { return kClassName(); } + static const char* kClassName() { return "Wrapped1"; } + + private: + WrappedOptions1 wopts_; +}; + +struct WrappedOptions2 { + static const char* kName() { return "WrappedOptions2"; } + std::string s = "42"; +}; +class WrappedCustomizable2 : public InnerCustomizable { + public: + explicit WrappedCustomizable2(const std::shared_ptr& w) + : InnerCustomizable(w) {} + const void* GetOptionsPtr(const std::string& name) const override { + if (name == WrappedOptions2::kName()) { + return &wopts_; + } else { + return InnerCustomizable::GetOptionsPtr(name); + } + } + + const char* Name() const override { return kClassName(); } + static const char* kClassName() { return "Wrapped2"; } + + private: + WrappedOptions2 wopts_; +}; +} // namespace + +TEST_F(CustomizableTest, WrappedInnerTest) { + std::shared_ptr ac = + std::make_shared("A"); + + ASSERT_TRUE(ac->IsInstanceOf("A")); + ASSERT_TRUE(ac->IsInstanceOf("TestCustomizable")); + ASSERT_EQ(ac->CheckedCast(), ac.get()); + ASSERT_EQ(ac->CheckedCast(), nullptr); + ASSERT_EQ(ac->CheckedCast(), nullptr); + ASSERT_EQ(ac->CheckedCast(), nullptr); + std::shared_ptr wc1 = + std::make_shared(ac); + + ASSERT_TRUE(wc1->IsInstanceOf(WrappedCustomizable1::kClassName())); + ASSERT_EQ(wc1->CheckedCast(), wc1.get()); + ASSERT_EQ(wc1->CheckedCast(), nullptr); + ASSERT_EQ(wc1->CheckedCast(), wc1.get()); + ASSERT_EQ(wc1->CheckedCast(), ac.get()); + + std::shared_ptr wc2 = + std::make_shared(wc1); + ASSERT_TRUE(wc2->IsInstanceOf(WrappedCustomizable2::kClassName())); + ASSERT_EQ(wc2->CheckedCast(), wc2.get()); + ASSERT_EQ(wc2->CheckedCast(), wc1.get()); + ASSERT_EQ(wc2->CheckedCast(), wc2.get()); + ASSERT_EQ(wc2->CheckedCast(), ac.get()); +} + +TEST_F(CustomizableTest, CustomizableInnerTest) { + std::shared_ptr c = + std::make_shared(std::make_shared("a")); + std::shared_ptr wc1 = std::make_shared(c); + std::shared_ptr wc2 = std::make_shared(c); + auto inner = c->GetOptions(); + ASSERT_NE(inner, nullptr); + + auto aopts = c->GetOptions(); + ASSERT_NE(aopts, nullptr); + ASSERT_EQ(aopts, wc1->GetOptions()); + ASSERT_EQ(aopts, wc2->GetOptions()); + auto w1opts = wc1->GetOptions(); + ASSERT_NE(w1opts, nullptr); + ASSERT_EQ(c->GetOptions(), nullptr); + ASSERT_EQ(wc2->GetOptions(), nullptr); + + auto w2opts = wc2->GetOptions(); + ASSERT_NE(w2opts, nullptr); + ASSERT_EQ(c->GetOptions(), nullptr); + ASSERT_EQ(wc1->GetOptions(), nullptr); +} + +TEST_F(CustomizableTest, CopyObjectTest) { + class CopyCustomizable : public Customizable { + public: + CopyCustomizable() : prepared_(0), validated_(0) {} + const char* Name() const override { return "CopyCustomizable"; } + + Status PrepareOptions(const ConfigOptions& options) override { + prepared_++; + return Customizable::PrepareOptions(options); + } + Status ValidateOptions(const DBOptions& db_opts, + const ColumnFamilyOptions& cf_opts) const override { + validated_++; + return Customizable::ValidateOptions(db_opts, cf_opts); + } + int prepared_; + mutable int validated_; + }; + + CopyCustomizable c1; + ConfigOptions config_options; + Options options; + + ASSERT_OK(c1.PrepareOptions(config_options)); + ASSERT_OK(c1.ValidateOptions(options, options)); + ASSERT_EQ(c1.prepared_, 1); + ASSERT_EQ(c1.validated_, 1); + CopyCustomizable c2 = c1; + ASSERT_OK(c1.PrepareOptions(config_options)); + ASSERT_OK(c1.ValidateOptions(options, options)); + ASSERT_EQ(c2.prepared_, 1); + ASSERT_EQ(c2.validated_, 1); + ASSERT_EQ(c1.prepared_, 2); + ASSERT_EQ(c1.validated_, 2); +} + +TEST_F(CustomizableTest, TestStringDepth) { + ConfigOptions shallow = config_options_; + std::unique_ptr c( + new InnerCustomizable(std::make_shared("a"))); + std::string opt_str; + shallow.depth = ConfigOptions::Depth::kDepthShallow; + ASSERT_OK(c->GetOptionString(shallow, &opt_str)); + ASSERT_EQ(opt_str, "inner=a;"); + shallow.depth = ConfigOptions::Depth::kDepthDetailed; + ASSERT_OK(c->GetOptionString(shallow, &opt_str)); + ASSERT_NE(opt_str, "inner=a;"); +} + +// Tests that we only get a new customizable when it changes +TEST_F(CustomizableTest, NewUniqueCustomizableTest) { + std::unique_ptr base(new SimpleConfigurable()); + A_count = 0; + ASSERT_OK(base->ConfigureFromString(config_options_, + "unique={id=A_1;int=1;bool=true}")); + SimpleOptions* simple = base->GetOptions(); + ASSERT_NE(simple, nullptr); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(A_count, 1); // Created one A + ASSERT_OK(base->ConfigureFromString(config_options_, + "unique={id=A_1;int=1;bool=false}")); + ASSERT_EQ(A_count, 2); // Create another A_1 + ASSERT_OK(base->ConfigureFromString(config_options_, "unique={id=}")); + ASSERT_EQ(simple->cu, nullptr); + ASSERT_EQ(A_count, 2); + ASSERT_OK(base->ConfigureFromString(config_options_, + "unique={id=A_2;int=1;bool=false}")); + ASSERT_EQ(A_count, 3); // Created another A + ASSERT_OK(base->ConfigureFromString(config_options_, "unique.id=")); + ASSERT_EQ(simple->cu, nullptr); + ASSERT_OK(base->ConfigureFromString(config_options_, "unique=nullptr")); + ASSERT_EQ(simple->cu, nullptr); + ASSERT_OK(base->ConfigureFromString(config_options_, "unique.id=nullptr")); + ASSERT_EQ(simple->cu, nullptr); + ASSERT_EQ(A_count, 3); +} + +TEST_F(CustomizableTest, NewEmptyUniqueTest) { + std::unique_ptr base(new SimpleConfigurable()); + SimpleOptions* simple = base->GetOptions(); + ASSERT_EQ(simple->cu, nullptr); + simple->cu.reset(new BCustomizable("B")); + + ASSERT_OK(base->ConfigureFromString(config_options_, "unique={id=}")); + ASSERT_EQ(simple->cu, nullptr); + simple->cu.reset(new BCustomizable("B")); + + ASSERT_OK(base->ConfigureFromString(config_options_, "unique={id=nullptr}")); + ASSERT_EQ(simple->cu, nullptr); + simple->cu.reset(new BCustomizable("B")); + + ASSERT_OK(base->ConfigureFromString(config_options_, "unique.id=")); + ASSERT_EQ(simple->cu, nullptr); + simple->cu.reset(new BCustomizable("B")); + + ASSERT_OK(base->ConfigureFromString(config_options_, "unique=nullptr")); + ASSERT_EQ(simple->cu, nullptr); + simple->cu.reset(new BCustomizable("B")); + + ASSERT_OK(base->ConfigureFromString(config_options_, "unique.id=nullptr")); + ASSERT_EQ(simple->cu, nullptr); +} + +TEST_F(CustomizableTest, NewEmptySharedTest) { + std::unique_ptr base(new SimpleConfigurable()); + + SimpleOptions* simple = base->GetOptions(); + ASSERT_NE(simple, nullptr); + ASSERT_EQ(simple->cs, nullptr); + simple->cs.reset(new BCustomizable("B")); + + ASSERT_OK(base->ConfigureFromString(config_options_, "shared={id=}")); + ASSERT_NE(simple, nullptr); + ASSERT_EQ(simple->cs, nullptr); + simple->cs.reset(new BCustomizable("B")); + + ASSERT_OK(base->ConfigureFromString(config_options_, "shared={id=nullptr}")); + ASSERT_EQ(simple->cs, nullptr); + simple->cs.reset(new BCustomizable("B")); + + ASSERT_OK(base->ConfigureFromString(config_options_, "shared.id=")); + ASSERT_EQ(simple->cs, nullptr); + simple->cs.reset(new BCustomizable("B")); + + ASSERT_OK(base->ConfigureFromString(config_options_, "shared.id=nullptr")); + ASSERT_EQ(simple->cs, nullptr); + simple->cs.reset(new BCustomizable("B")); + + ASSERT_OK(base->ConfigureFromString(config_options_, "shared=nullptr")); + ASSERT_EQ(simple->cs, nullptr); +} + +TEST_F(CustomizableTest, NewEmptyStaticTest) { + std::unique_ptr base(new SimpleConfigurable()); + ASSERT_OK(base->ConfigureFromString(config_options_, "pointer={id=}")); + SimpleOptions* simple = base->GetOptions(); + ASSERT_NE(simple, nullptr); + ASSERT_EQ(simple->cp, nullptr); + ASSERT_OK(base->ConfigureFromString(config_options_, "pointer={id=nullptr}")); + ASSERT_EQ(simple->cp, nullptr); + + ASSERT_OK(base->ConfigureFromString(config_options_, "pointer=")); + ASSERT_EQ(simple->cp, nullptr); + ASSERT_OK(base->ConfigureFromString(config_options_, "pointer=nullptr")); + ASSERT_EQ(simple->cp, nullptr); + + ASSERT_OK(base->ConfigureFromString(config_options_, "pointer.id=")); + ASSERT_EQ(simple->cp, nullptr); + ASSERT_OK(base->ConfigureFromString(config_options_, "pointer.id=nullptr")); + ASSERT_EQ(simple->cp, nullptr); +} + +namespace { +static std::unordered_map vector_option_info = { + {"vector", + OptionTypeInfo::Vector>( + 0, OptionVerificationType::kNormal, + + OptionTypeFlags::kNone, + + OptionTypeInfo::AsCustomSharedPtr( + 0, OptionVerificationType::kNormal, OptionTypeFlags::kNone))}, +}; +class VectorConfigurable : public SimpleConfigurable { + public: + VectorConfigurable() { RegisterOptions("vector", &cv, &vector_option_info); } + std::vector> cv; +}; +} // namespace + +TEST_F(CustomizableTest, VectorConfigTest) { + VectorConfigurable orig, copy; + std::shared_ptr c1, c2; + ASSERT_OK(TestCustomizable::CreateFromString(config_options_, "A", &c1)); + ASSERT_OK(TestCustomizable::CreateFromString(config_options_, "B", &c2)); + orig.cv.push_back(c1); + orig.cv.push_back(c2); + ASSERT_OK(orig.ConfigureFromString(config_options_, "unique=A2")); + std::string opt_str, mismatch; + ASSERT_OK(orig.GetOptionString(config_options_, &opt_str)); + ASSERT_OK(copy.ConfigureFromString(config_options_, opt_str)); + ASSERT_TRUE(orig.AreEquivalent(config_options_, ©, &mismatch)); +} + +TEST_F(CustomizableTest, NoNameTest) { + // If Customizables are created without names, they are not + // part of the serialization (since they cannot be recreated) + VectorConfigurable orig, copy; + auto sopts = orig.GetOptions(); + auto copts = copy.GetOptions(); + sopts->cu.reset(new ACustomizable("")); + orig.cv.push_back(std::make_shared("")); + orig.cv.push_back(std::make_shared("A_1")); + std::string opt_str, mismatch; + ASSERT_OK(orig.GetOptionString(config_options_, &opt_str)); + ASSERT_OK(copy.ConfigureFromString(config_options_, opt_str)); + ASSERT_EQ(copy.cv.size(), 1U); + ASSERT_EQ(copy.cv[0]->GetId(), "A_1"); + ASSERT_EQ(copts->cu, nullptr); +} + + +TEST_F(CustomizableTest, IgnoreUnknownObjects) { + ConfigOptions ignore = config_options_; + std::shared_ptr shared; + std::unique_ptr unique; + TestCustomizable* pointer = nullptr; + ignore.ignore_unsupported_options = false; + ASSERT_NOK(LoadSharedObject(ignore, "Unknown", &shared)); + ASSERT_NOK(LoadUniqueObject(ignore, "Unknown", &unique)); + ASSERT_NOK(LoadStaticObject(ignore, "Unknown", &pointer)); + ASSERT_EQ(shared.get(), nullptr); + ASSERT_EQ(unique.get(), nullptr); + ASSERT_EQ(pointer, nullptr); + ignore.ignore_unsupported_options = true; + ASSERT_OK(LoadSharedObject(ignore, "Unknown", &shared)); + ASSERT_OK(LoadUniqueObject(ignore, "Unknown", &unique)); + ASSERT_OK(LoadStaticObject(ignore, "Unknown", &pointer)); + ASSERT_EQ(shared.get(), nullptr); + ASSERT_EQ(unique.get(), nullptr); + ASSERT_EQ(pointer, nullptr); + ASSERT_OK(LoadSharedObject(ignore, "id=Unknown", &shared)); + ASSERT_OK(LoadUniqueObject(ignore, "id=Unknown", &unique)); + ASSERT_OK(LoadStaticObject(ignore, "id=Unknown", &pointer)); + ASSERT_EQ(shared.get(), nullptr); + ASSERT_EQ(unique.get(), nullptr); + ASSERT_EQ(pointer, nullptr); + ASSERT_OK(LoadSharedObject(ignore, "id=Unknown;option=bad", + &shared)); + ASSERT_OK(LoadUniqueObject(ignore, "id=Unknown;option=bad", + &unique)); + ASSERT_OK(LoadStaticObject(ignore, "id=Unknown;option=bad", + &pointer)); + ASSERT_EQ(shared.get(), nullptr); + ASSERT_EQ(unique.get(), nullptr); + ASSERT_EQ(pointer, nullptr); +} + +TEST_F(CustomizableTest, URLFactoryTest) { + std::unique_ptr unique; + config_options_.registry->AddLibrary("URL")->AddFactory( + ObjectLibrary::PatternEntry("Z", false).AddSeparator(""), + [](const std::string& name, std::unique_ptr* guard, + std::string* /* msg */) { + guard->reset(new TestCustomizable(name)); + return guard->get(); + }); + + ConfigOptions ignore = config_options_; + ignore.ignore_unsupported_options = false; + ignore.ignore_unsupported_options = false; + ASSERT_OK(TestCustomizable::CreateFromString(ignore, "Z=1;x=y", &unique)); + ASSERT_NE(unique, nullptr); + ASSERT_EQ(unique->GetId(), "Z=1;x=y"); + ASSERT_OK(TestCustomizable::CreateFromString(ignore, "Z;x=y", &unique)); + ASSERT_NE(unique, nullptr); + ASSERT_EQ(unique->GetId(), "Z;x=y"); + unique.reset(); + ASSERT_OK(TestCustomizable::CreateFromString(ignore, "Z=1?x=y", &unique)); + ASSERT_NE(unique, nullptr); + ASSERT_EQ(unique->GetId(), "Z=1?x=y"); +} + +TEST_F(CustomizableTest, MutableOptionsTest) { + static std::unordered_map mutable_option_info = { + {"mutable", + OptionTypeInfo::AsCustomSharedPtr( + 0, OptionVerificationType::kNormal, OptionTypeFlags::kMutable)}}; + static std::unordered_map immutable_option_info = + {{"immutable", + OptionTypeInfo::AsCustomSharedPtr( + 0, OptionVerificationType::kNormal, OptionTypeFlags::kAllowNull)}}; + + class MutableCustomizable : public Customizable { + private: + std::shared_ptr mutable_; + std::shared_ptr immutable_; + + public: + MutableCustomizable() { + RegisterOptions("mutable", &mutable_, &mutable_option_info); + RegisterOptions("immutable", &immutable_, &immutable_option_info); + } + const char* Name() const override { return "MutableCustomizable"; } + }; + MutableCustomizable mc, mc2; + std::string mismatch; + std::string opt_str; + + ConfigOptions options = config_options_; + ASSERT_OK(mc.ConfigureOption(options, "mutable", "{id=B;}")); + options.mutable_options_only = true; + ASSERT_OK(mc.GetOptionString(options, &opt_str)); + ASSERT_OK(mc2.ConfigureFromString(options, opt_str)); + ASSERT_TRUE(mc.AreEquivalent(options, &mc2, &mismatch)); + + options.mutable_options_only = false; + ASSERT_OK(mc.ConfigureOption(options, "immutable", "{id=A; int=10}")); + auto* mm = mc.GetOptions>("mutable"); + auto* im = mc.GetOptions>("immutable"); + ASSERT_NE(mm, nullptr); + ASSERT_NE(mm->get(), nullptr); + ASSERT_NE(im, nullptr); + ASSERT_NE(im->get(), nullptr); + + // Now only deal with mutable options + options.mutable_options_only = true; + + // Setting nested immutable customizable options fails + ASSERT_NOK(mc.ConfigureOption(options, "immutable", "{id=B;}")); + ASSERT_NOK(mc.ConfigureOption(options, "immutable.id", "B")); + ASSERT_NOK(mc.ConfigureOption(options, "immutable.bool", "true")); + ASSERT_NOK(mc.ConfigureOption(options, "immutable", "bool=true")); + ASSERT_NOK(mc.ConfigureOption(options, "immutable", "{int=11;bool=true}")); + auto* im_a = im->get()->GetOptions("A"); + ASSERT_NE(im_a, nullptr); + ASSERT_EQ(im_a->i, 10); + ASSERT_EQ(im_a->b, false); + + // Setting nested mutable customizable options succeeds but the object did not + // change + ASSERT_OK(mc.ConfigureOption(options, "immutable.int", "11")); + ASSERT_EQ(im_a->i, 11); + ASSERT_EQ(im_a, im->get()->GetOptions("A")); + + // The mutable configurable itself can be changed + ASSERT_OK(mc.ConfigureOption(options, "mutable.id", "A")); + ASSERT_OK(mc.ConfigureOption(options, "mutable", "A")); + ASSERT_OK(mc.ConfigureOption(options, "mutable", "{id=A}")); + ASSERT_OK(mc.ConfigureOption(options, "mutable", "{bool=true}")); + + // The Nested options in the mutable object can be changed + ASSERT_OK(mc.ConfigureOption(options, "mutable", "{bool=true}")); + auto* mm_a = mm->get()->GetOptions("A"); + ASSERT_EQ(mm_a->b, true); + ASSERT_OK(mc.ConfigureOption(options, "mutable", "{int=22;bool=false}")); + mm_a = mm->get()->GetOptions("A"); + ASSERT_EQ(mm_a->i, 22); + ASSERT_EQ(mm_a->b, false); + + // Only the mutable options should get serialized + options.mutable_options_only = false; + ASSERT_OK(mc.GetOptionString(options, &opt_str)); + ASSERT_OK(mc.ConfigureOption(options, "immutable", "{id=B;}")); + options.mutable_options_only = true; + + ASSERT_OK(mc.GetOptionString(options, &opt_str)); + ASSERT_OK(mc2.ConfigureFromString(options, opt_str)); + ASSERT_TRUE(mc.AreEquivalent(options, &mc2, &mismatch)); + options.mutable_options_only = false; + ASSERT_FALSE(mc.AreEquivalent(options, &mc2, &mismatch)); + ASSERT_EQ(mismatch, "immutable"); +} + +TEST_F(CustomizableTest, CustomManagedObjects) { + std::shared_ptr object1, object2; + ASSERT_OK(LoadManagedObject( + config_options_, "id=A_1;int=1;bool=true", &object1)); + ASSERT_NE(object1, nullptr); + ASSERT_OK( + LoadManagedObject(config_options_, "A_1", &object2)); + ASSERT_EQ(object1, object2); + auto* opts = object2->GetOptions("A"); + ASSERT_NE(opts, nullptr); + ASSERT_EQ(opts->i, 1); + ASSERT_EQ(opts->b, true); + ASSERT_OK( + LoadManagedObject(config_options_, "A_2", &object2)); + ASSERT_NE(object1, object2); + object1.reset(); + ASSERT_OK(LoadManagedObject( + config_options_, "id=A_1;int=2;bool=false", &object1)); + opts = object1->GetOptions("A"); + ASSERT_NE(opts, nullptr); + ASSERT_EQ(opts->i, 2); + ASSERT_EQ(opts->b, false); +} + +TEST_F(CustomizableTest, CreateManagedObjects) { + class ManagedCustomizable : public Customizable { + public: + static const char* Type() { return "ManagedCustomizable"; } + static const char* kClassName() { return "Managed"; } + const char* Name() const override { return kClassName(); } + std::string GetId() const override { return id_; } + ManagedCustomizable() { id_ = GenerateIndividualId(); } + static Status CreateFromString( + const ConfigOptions& opts, const std::string& value, + std::shared_ptr* result) { + return LoadManagedObject(opts, value, result); + } + + private: + std::string id_; + }; + + config_options_.registry->AddLibrary("Managed") + ->AddFactory( + ObjectLibrary::PatternEntry::AsIndividualId( + ManagedCustomizable::kClassName()), + [](const std::string& /*name*/, + std::unique_ptr* guard, + std::string* /* msg */) { + guard->reset(new ManagedCustomizable()); + return guard->get(); + }); + + std::shared_ptr mc1, mc2, mc3, obj; + // Create a "deadbeef" customizable + std::string deadbeef = + std::string(ManagedCustomizable::kClassName()) + "@0xdeadbeef#0001"; + ASSERT_OK( + ManagedCustomizable::CreateFromString(config_options_, deadbeef, &mc1)); + // Create an object with the base/class name + ASSERT_OK(ManagedCustomizable::CreateFromString( + config_options_, ManagedCustomizable::kClassName(), &mc2)); + // Creating another with the base name returns a different object + ASSERT_OK(ManagedCustomizable::CreateFromString( + config_options_, ManagedCustomizable::kClassName(), &mc3)); + // At this point, there should be 4 managed objects (deadbeef, mc1, 2, and 3) + std::vector> objects; + ASSERT_OK(config_options_.registry->ListManagedObjects(&objects)); + ASSERT_EQ(objects.size(), 4U); + objects.clear(); + // Three separate object, none of them equal + ASSERT_NE(mc1, mc2); + ASSERT_NE(mc1, mc3); + ASSERT_NE(mc2, mc3); + + // Creating another object with "deadbeef" object + ASSERT_OK( + ManagedCustomizable::CreateFromString(config_options_, deadbeef, &obj)); + ASSERT_EQ(mc1, obj); + // Create another with the IDs of the instances + ASSERT_OK(ManagedCustomizable::CreateFromString(config_options_, mc1->GetId(), + &obj)); + ASSERT_EQ(mc1, obj); + ASSERT_OK(ManagedCustomizable::CreateFromString(config_options_, mc2->GetId(), + &obj)); + ASSERT_EQ(mc2, obj); + ASSERT_OK(ManagedCustomizable::CreateFromString(config_options_, mc3->GetId(), + &obj)); + ASSERT_EQ(mc3, obj); + + // Now get rid of deadbeef. 2 Objects left (m2+m3) + mc1.reset(); + ASSERT_EQ( + config_options_.registry->GetManagedObject(deadbeef), + nullptr); + ASSERT_OK(config_options_.registry->ListManagedObjects(&objects)); + ASSERT_EQ(objects.size(), 2U); + objects.clear(); + + // Associate deadbeef with #2 + ASSERT_OK(config_options_.registry->SetManagedObject(deadbeef, mc2)); + ASSERT_OK( + ManagedCustomizable::CreateFromString(config_options_, deadbeef, &obj)); + ASSERT_EQ(mc2, obj); + obj.reset(); + + // Get the ID of mc2 and then reset it. 1 Object left + std::string mc2id = mc2->GetId(); + mc2.reset(); + ASSERT_EQ( + config_options_.registry->GetManagedObject(mc2id), + nullptr); + ASSERT_OK(config_options_.registry->ListManagedObjects(&objects)); + ASSERT_EQ(objects.size(), 1U); + objects.clear(); + + // Create another object with the old mc2id. + ASSERT_OK( + ManagedCustomizable::CreateFromString(config_options_, mc2id, &mc2)); + ASSERT_OK( + ManagedCustomizable::CreateFromString(config_options_, mc2id, &obj)); + ASSERT_EQ(mc2, obj); + + // For good measure, create another deadbeef object + ASSERT_OK( + ManagedCustomizable::CreateFromString(config_options_, deadbeef, &mc1)); + ASSERT_OK( + ManagedCustomizable::CreateFromString(config_options_, deadbeef, &obj)); + ASSERT_EQ(mc1, obj); +} + + +namespace { +class TestSecondaryCache : public SecondaryCache { + public: + static const char* kClassName() { return "Test"; } + const char* Name() const override { return kClassName(); } + Status Insert(const Slice& /*key*/, Cache::ObjectPtr /*value*/, + const Cache::CacheItemHelper* /*helper*/) override { + return Status::NotSupported(); + } + std::unique_ptr Lookup( + const Slice& /*key*/, const Cache::CacheItemHelper* /*helper*/, + Cache::CreateContext* /*create_context*/, bool /*wait*/, + bool /*advise_erase*/, bool& kept_in_sec_cache) override { + kept_in_sec_cache = true; + return nullptr; + } + + bool SupportForceErase() const override { return false; } + + void Erase(const Slice& /*key*/) override {} + + // Wait for a collection of handles to become ready + void WaitAll(std::vector /*handles*/) override {} + + std::string GetPrintableOptions() const override { return ""; } +}; + +class TestStatistics : public StatisticsImpl { + public: + TestStatistics() : StatisticsImpl(nullptr) {} + const char* Name() const override { return kClassName(); } + static const char* kClassName() { return "Test"; } +}; + +class TestFlushBlockPolicyFactory : public FlushBlockPolicyFactory { + public: + TestFlushBlockPolicyFactory() {} + + static const char* kClassName() { return "TestFlushBlockPolicyFactory"; } + const char* Name() const override { return kClassName(); } + + FlushBlockPolicy* NewFlushBlockPolicy( + const BlockBasedTableOptions& /*table_options*/, + const BlockBuilder& /*data_block_builder*/) const override { + return nullptr; + } +}; + +class MockSliceTransform : public SliceTransform { + public: + const char* Name() const override { return kClassName(); } + static const char* kClassName() { return "Mock"; } + + Slice Transform(const Slice& /*key*/) const override { return Slice(); } + + bool InDomain(const Slice& /*key*/) const override { return false; } + + bool InRange(const Slice& /*key*/) const override { return false; } +}; + +class MockMemoryAllocator : public BaseMemoryAllocator { + public: + static const char* kClassName() { return "MockMemoryAllocator"; } + const char* Name() const override { return kClassName(); } +}; + +class MockEncryptionProvider : public EncryptionProvider { + public: + explicit MockEncryptionProvider(const std::string& id) : id_(id) {} + static const char* kClassName() { return "Mock"; } + const char* Name() const override { return kClassName(); } + size_t GetPrefixLength() const override { return 0; } + Status CreateNewPrefix(const std::string& /*fname*/, char* /*prefix*/, + size_t /*prefixLength*/) const override { + return Status::NotSupported(); + } + + Status AddCipher(const std::string& /*descriptor*/, const char* /*cipher*/, + size_t /*len*/, bool /*for_write*/) override { + return Status::NotSupported(); + } + + Status CreateCipherStream( + const std::string& /*fname*/, const EnvOptions& /*options*/, + Slice& /*prefix*/, + std::unique_ptr* /*result*/) override { + return Status::NotSupported(); + } + Status ValidateOptions(const DBOptions& db_opts, + const ColumnFamilyOptions& cf_opts) const override { + if (EndsWith(id_, "://test")) { + return EncryptionProvider::ValidateOptions(db_opts, cf_opts); + } else { + return Status::InvalidArgument("MockProvider not initialized"); + } + } + + private: + std::string id_; +}; + +class MockCipher : public BlockCipher { + public: + const char* Name() const override { return "Mock"; } + size_t BlockSize() override { return 0; } + Status Encrypt(char* /*data*/) override { return Status::NotSupported(); } + Status Decrypt(char* data) override { return Encrypt(data); } +}; + +class DummyFileSystem : public FileSystemWrapper { + public: + explicit DummyFileSystem(const std::shared_ptr& t) + : FileSystemWrapper(t) {} + static const char* kClassName() { return "DummyFileSystem"; } + const char* Name() const override { return kClassName(); } +}; + + + +class MockTablePropertiesCollectorFactory + : public TablePropertiesCollectorFactory { + private: + public: + TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context /*context*/) override { + return nullptr; + } + static const char* kClassName() { return "Mock"; } + const char* Name() const override { return kClassName(); } +}; + +class MockSstPartitionerFactory : public SstPartitionerFactory { + public: + static const char* kClassName() { return "Mock"; } + const char* Name() const override { return kClassName(); } + std::unique_ptr CreatePartitioner( + const SstPartitioner::Context& /* context */) const override { + return nullptr; + } +}; + +class MockFileChecksumGenFactory : public FileChecksumGenFactory { + public: + static const char* kClassName() { return "Mock"; } + const char* Name() const override { return kClassName(); } + std::unique_ptr CreateFileChecksumGenerator( + const FileChecksumGenContext& /*context*/) override { + return nullptr; + } +}; + +class MockFilterPolicy : public FilterPolicy { + public: + static const char* kClassName() { return "MockFilterPolicy"; } + const char* Name() const override { return kClassName(); } + const char* CompatibilityName() const override { return Name(); } + FilterBitsBuilder* GetBuilderWithContext( + const FilterBuildingContext&) const override { + return nullptr; + } + FilterBitsReader* GetFilterBitsReader( + const Slice& /*contents*/) const override { + return nullptr; + } +}; + +static int RegisterLocalObjects(ObjectLibrary& library, + const std::string& /*arg*/) { + size_t num_types; + library.AddFactory( + mock::MockTableFactory::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new mock::MockTableFactory()); + return guard->get(); + }); + library.AddFactory( + OnFileDeletionListener::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new OnFileDeletionListener()); + return guard->get(); + }); + library.AddFactory( + FlushCounterListener::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new FlushCounterListener()); + return guard->get(); + }); + // Load any locally defined objects here + library.AddFactory( + MockSliceTransform::kClassName(), + [](const std::string& /*uri*/, + std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new MockSliceTransform()); + return guard->get(); + }); + library.AddFactory( + TestStatistics::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new TestStatistics()); + return guard->get(); + }); + + library.AddFactory( + ObjectLibrary::PatternEntry(MockEncryptionProvider::kClassName(), true) + .AddSuffix("://test"), + [](const std::string& uri, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new MockEncryptionProvider(uri)); + return guard->get(); + }); + library.AddFactory( + "Mock", + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new MockCipher()); + return guard->get(); + }); + library.AddFactory( + MockMemoryAllocator::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new MockMemoryAllocator()); + return guard->get(); + }); + library.AddFactory( + TestFlushBlockPolicyFactory::kClassName(), + [](const std::string& /*uri*/, + std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new TestFlushBlockPolicyFactory()); + return guard->get(); + }); + + library.AddFactory( + TestSecondaryCache::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new TestSecondaryCache()); + return guard->get(); + }); + + library.AddFactory( + DummyFileSystem::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new DummyFileSystem(nullptr)); + return guard->get(); + }); + + library.AddFactory( + MockSstPartitionerFactory::kClassName(), + [](const std::string& /*uri*/, + std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new MockSstPartitionerFactory()); + return guard->get(); + }); + + library.AddFactory( + MockFileChecksumGenFactory::kClassName(), + [](const std::string& /*uri*/, + std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new MockFileChecksumGenFactory()); + return guard->get(); + }); + + library.AddFactory( + MockTablePropertiesCollectorFactory::kClassName(), + [](const std::string& /*uri*/, + std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new MockTablePropertiesCollectorFactory()); + return guard->get(); + }); + + library.AddFactory( + MockFilterPolicy::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new MockFilterPolicy()); + return guard->get(); + }); + + return static_cast(library.GetFactoryCount(&num_types)); +} +} // namespace + +class LoadCustomizableTest : public testing::Test { + public: + LoadCustomizableTest() { + config_options_.ignore_unsupported_options = false; + config_options_.invoke_prepare_options = false; + } + bool RegisterTests(const std::string& arg) { + config_options_.registry->AddLibrary("custom-tests", + test::RegisterTestObjects, arg); + config_options_.registry->AddLibrary("local-tests", RegisterLocalObjects, + arg); + return true; + } + + template + Status TestCreateStatic(const std::string& name, U** result, + bool delete_result = false) { + Status s = T::CreateFromString(config_options_, name, result); + if (s.ok()) { + EXPECT_NE(*result, nullptr); + EXPECT_TRUE(*result != nullptr && (*result)->IsInstanceOf(name)); + } + if (delete_result) { + delete *result; + *result = nullptr; + } + return s; + } + + template + std::shared_ptr ExpectCreateShared(const std::string& name, + std::shared_ptr* object) { + EXPECT_OK(T::CreateFromString(config_options_, name, object)); + EXPECT_NE(object->get(), nullptr); + EXPECT_TRUE(object->get()->IsInstanceOf(name)); + return *object; + } + + template + std::shared_ptr ExpectCreateShared(const std::string& name) { + std::shared_ptr result; + return ExpectCreateShared(name, &result); + } + + template + Status TestExpectedBuiltins( + const std::string& mock, const std::unordered_set& expected, + std::shared_ptr* object, std::vector* failed, + const std::function(const std::string&)>& alt = + nullptr) { + std::unordered_set factories = expected; + Status s = T::CreateFromString(config_options_, mock, object); + EXPECT_NOK(s); + std::vector builtins; + ObjectLibrary::Default()->GetFactoryNames(T::Type(), &builtins); + factories.insert(builtins.begin(), builtins.end()); + Status result; + int created = 0; + for (const auto& name : factories) { + created++; + s = T::CreateFromString(config_options_, name, object); + if (!s.ok() && alt != nullptr) { + for (const auto& alt_name : alt(name)) { + s = T::CreateFromString(config_options_, alt_name, object); + if (s.ok()) { + break; + } + } + } + if (!s.ok()) { + result = s; + failed->push_back(name); + } else { + EXPECT_NE(object->get(), nullptr); + EXPECT_TRUE(object->get()->IsInstanceOf(name)); + } + } + std::vector plugins; + ObjectRegistry::Default()->GetFactoryNames(T::Type(), &plugins); + if (plugins.size() > builtins.size()) { + for (const auto& name : plugins) { + if (factories.find(name) == factories.end()) { + created++; + s = T::CreateFromString(config_options_, name, object); + if (!s.ok() && alt != nullptr) { + for (const auto& alt_name : alt(name)) { + s = T::CreateFromString(config_options_, alt_name, object); + if (s.ok()) { + break; + } + } + } + if (!s.ok()) { + failed->push_back(name); + if (result.ok()) { + result = s; + } + printf("%s: Failed creating plugin[%s]: %s\n", T::Type(), + name.c_str(), s.ToString().c_str()); + } else if (object->get() == nullptr || + !object->get()->IsInstanceOf(name)) { + failed->push_back(name); + printf("%s: Invalid plugin[%s]\n", T::Type(), name.c_str()); + } + } + } + } + printf("%s: Created %d (expected+builtins+plugins %d+%d+%d) %d Failed\n", + T::Type(), created, (int)expected.size(), + (int)(factories.size() - expected.size()), + (int)(plugins.size() - builtins.size()), (int)failed->size()); + return result; + } + + template + Status TestSharedBuiltins(const std::string& mock, + const std::string& expected, + std::vector* failed = nullptr) { + std::unordered_set values; + if (!expected.empty()) { + values.insert(expected); + } + std::shared_ptr object; + if (failed != nullptr) { + return TestExpectedBuiltins(mock, values, &object, failed); + } else { + std::vector failures; + Status s = TestExpectedBuiltins(mock, values, &object, &failures); + EXPECT_EQ(0U, failures.size()); + return s; + } + } + + template + Status TestStaticBuiltins(const std::string& mock, U** object, + const std::unordered_set& expected, + std::vector* failed, + bool delete_objects = false) { + std::unordered_set factories = expected; + Status s = TestCreateStatic(mock, object, delete_objects); + EXPECT_NOK(s); + std::vector builtins; + ObjectLibrary::Default()->GetFactoryNames(T::Type(), &builtins); + factories.insert(builtins.begin(), builtins.end()); + int created = 0; + Status result; + for (const auto& name : factories) { + created++; + s = TestCreateStatic(name, object, delete_objects); + if (!s.ok()) { + result = s; + failed->push_back(name); + } + } + std::vector plugins; + ObjectRegistry::Default()->GetFactoryNames(T::Type(), &plugins); + if (plugins.size() > builtins.size()) { + for (const auto& name : plugins) { + if (factories.find(name) == factories.end()) { + created++; + s = T::CreateFromString(config_options_, name, object); + if (!s.ok() || *object == nullptr || + !((*object)->IsInstanceOf(name))) { + failed->push_back(name); + if (result.ok() && !s.ok()) { + result = s; + } + printf("%s: Failed creating plugin[%s]: %s\n", T::Type(), + name.c_str(), s.ToString().c_str()); + } + if (delete_objects) { + delete *object; + *object = nullptr; + } + } + } + } + printf("%s: Created %d (expected+builtins+plugins %d+%d+%d) %d Failed\n", + T::Type(), created, (int)expected.size(), + (int)(factories.size() - expected.size()), + (int)(plugins.size() - builtins.size()), (int)failed->size()); + return result; + } + + protected: + DBOptions db_opts_; + ColumnFamilyOptions cf_opts_; + ConfigOptions config_options_; +}; + +TEST_F(LoadCustomizableTest, LoadTableFactoryTest) { + ASSERT_OK( + TestSharedBuiltins(mock::MockTableFactory::kClassName(), + TableFactory::kBlockBasedTableName())); + std::string opts_str = "table_factory="; + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options_, cf_opts_, + opts_str + TableFactory::kBlockBasedTableName(), &cf_opts_)); + ASSERT_NE(cf_opts_.table_factory.get(), nullptr); + ASSERT_STREQ(cf_opts_.table_factory->Name(), + TableFactory::kBlockBasedTableName()); + if (RegisterTests("Test")) { + ExpectCreateShared(mock::MockTableFactory::kClassName()); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options_, cf_opts_, + opts_str + mock::MockTableFactory::kClassName(), &cf_opts_)); + ASSERT_NE(cf_opts_.table_factory.get(), nullptr); + ASSERT_STREQ(cf_opts_.table_factory->Name(), + mock::MockTableFactory::kClassName()); + } +} + +TEST_F(LoadCustomizableTest, LoadFileSystemTest) { + ASSERT_OK(TestSharedBuiltins(DummyFileSystem::kClassName(), + FileSystem::kDefaultName())); + if (RegisterTests("Test")) { + auto fs = ExpectCreateShared(DummyFileSystem::kClassName()); + ASSERT_FALSE(fs->IsInstanceOf(FileSystem::kDefaultName())); + } +} + +TEST_F(LoadCustomizableTest, LoadSecondaryCacheTest) { + ASSERT_OK( + TestSharedBuiltins(TestSecondaryCache::kClassName(), "")); + if (RegisterTests("Test")) { + ExpectCreateShared(TestSecondaryCache::kClassName()); + } +} + +TEST_F(LoadCustomizableTest, LoadSstPartitionerFactoryTest) { + ASSERT_OK(TestSharedBuiltins( + "Mock", SstPartitionerFixedPrefixFactory::kClassName())); + if (RegisterTests("Test")) { + ExpectCreateShared("Mock"); + } +} + +TEST_F(LoadCustomizableTest, LoadChecksumGenFactoryTest) { + ASSERT_OK(TestSharedBuiltins("Mock", "")); + if (RegisterTests("Test")) { + ExpectCreateShared("Mock"); + } +} + +TEST_F(LoadCustomizableTest, LoadTablePropertiesCollectorFactoryTest) { + ASSERT_OK(TestSharedBuiltins( + MockTablePropertiesCollectorFactory::kClassName(), "")); + if (RegisterTests("Test")) { + ExpectCreateShared( + MockTablePropertiesCollectorFactory::kClassName()); + } +} + +TEST_F(LoadCustomizableTest, LoadComparatorTest) { + const Comparator* bytewise = BytewiseComparator(); + const Comparator* reverse = ReverseBytewiseComparator(); + const Comparator* result = nullptr; + std::unordered_set expected = {bytewise->Name(), + reverse->Name()}; + std::vector failures; + ASSERT_OK(TestStaticBuiltins( + test::SimpleSuffixReverseComparator::kClassName(), &result, expected, + &failures)); + if (RegisterTests("Test")) { + ASSERT_OK(TestCreateStatic( + test::SimpleSuffixReverseComparator::kClassName(), &result)); + } +} + +TEST_F(LoadCustomizableTest, LoadSliceTransformFactoryTest) { + std::shared_ptr result; + std::vector failures; + std::unordered_set expected = {"rocksdb.Noop", "fixed", + "rocksdb.FixedPrefix", "capped", + "rocksdb.CappedPrefix"}; + ASSERT_OK(TestExpectedBuiltins( + "Mock", expected, &result, &failures, [](const std::string& name) { + std::vector names = {name + ":22", name + ".22"}; + return names; + })); + ASSERT_OK(SliceTransform::CreateFromString( + config_options_, "rocksdb.FixedPrefix.22", &result)); + ASSERT_NE(result.get(), nullptr); + ASSERT_TRUE(result->IsInstanceOf("fixed")); + ASSERT_OK(SliceTransform::CreateFromString( + config_options_, "rocksdb.CappedPrefix.22", &result)); + ASSERT_NE(result.get(), nullptr); + ASSERT_TRUE(result->IsInstanceOf("capped")); + if (RegisterTests("Test")) { + ExpectCreateShared("Mock", &result); + } +} + +TEST_F(LoadCustomizableTest, LoadStatisticsTest) { + ASSERT_OK(TestSharedBuiltins(TestStatistics::kClassName(), + "BasicStatistics")); + // Empty will create a default BasicStatistics + ASSERT_OK( + Statistics::CreateFromString(config_options_, "", &db_opts_.statistics)); + ASSERT_NE(db_opts_.statistics, nullptr); + ASSERT_STREQ(db_opts_.statistics->Name(), "BasicStatistics"); + + ASSERT_NOK(GetDBOptionsFromString(config_options_, db_opts_, + "statistics=Test", &db_opts_)); + ASSERT_OK(GetDBOptionsFromString(config_options_, db_opts_, + "statistics=BasicStatistics", &db_opts_)); + ASSERT_NE(db_opts_.statistics, nullptr); + ASSERT_STREQ(db_opts_.statistics->Name(), "BasicStatistics"); + + if (RegisterTests("test")) { + auto stats = ExpectCreateShared(TestStatistics::kClassName()); + + ASSERT_OK(GetDBOptionsFromString(config_options_, db_opts_, + "statistics=Test", &db_opts_)); + ASSERT_NE(db_opts_.statistics, nullptr); + ASSERT_STREQ(db_opts_.statistics->Name(), TestStatistics::kClassName()); + + ASSERT_OK(GetDBOptionsFromString( + config_options_, db_opts_, "statistics={id=Test;inner=BasicStatistics}", + &db_opts_)); + ASSERT_NE(db_opts_.statistics, nullptr); + ASSERT_STREQ(db_opts_.statistics->Name(), TestStatistics::kClassName()); + auto* inner = db_opts_.statistics->GetOptions>( + "StatisticsOptions"); + ASSERT_NE(inner, nullptr); + ASSERT_NE(inner->get(), nullptr); + ASSERT_STREQ(inner->get()->Name(), "BasicStatistics"); + + ASSERT_OK(Statistics::CreateFromString( + config_options_, "id=BasicStatistics;inner=Test", &stats)); + ASSERT_NE(stats, nullptr); + ASSERT_STREQ(stats->Name(), "BasicStatistics"); + inner = stats->GetOptions>("StatisticsOptions"); + ASSERT_NE(inner, nullptr); + ASSERT_NE(inner->get(), nullptr); + ASSERT_STREQ(inner->get()->Name(), TestStatistics::kClassName()); + } +} + +TEST_F(LoadCustomizableTest, LoadMemTableRepFactoryTest) { + std::unordered_set expected = { + SkipListFactory::kClassName(), + SkipListFactory::kNickName(), + }; + + std::vector failures; + std::shared_ptr factory; + Status s = TestExpectedBuiltins( + "SpecialSkipListFactory", expected, &factory, &failures); + // There is a "cuckoo" factory registered that we expect to fail. Ignore the + // error if this is the one + if (s.ok() || failures.size() > 1 || failures[0] != "cuckoo") { + ASSERT_OK(s); + } + if (RegisterTests("Test")) { + ExpectCreateShared("SpecialSkipListFactory"); + } +} + +TEST_F(LoadCustomizableTest, LoadMergeOperatorTest) { + std::shared_ptr result; + std::vector failed; + std::unordered_set expected = { + "put", "put_v1", "PutOperator", "uint64add", "UInt64AddOperator", + "max", "MaxOperator", + }; + expected.insert({ + StringAppendOperator::kClassName(), + StringAppendOperator::kNickName(), + StringAppendTESTOperator::kClassName(), + StringAppendTESTOperator::kNickName(), + SortList::kClassName(), + SortList::kNickName(), + BytesXOROperator::kClassName(), + BytesXOROperator::kNickName(), + }); + + ASSERT_OK(TestExpectedBuiltins("Changling", expected, &result, + &failed)); + if (RegisterTests("Test")) { + ExpectCreateShared("Changling"); + } +} + +TEST_F(LoadCustomizableTest, LoadCompactionFilterFactoryTest) { + ASSERT_OK(TestSharedBuiltins("Changling", "")); + if (RegisterTests("Test")) { + ExpectCreateShared("Changling"); + } +} + +TEST_F(LoadCustomizableTest, LoadCompactionFilterTest) { + const CompactionFilter* result = nullptr; + std::vector failures; + ASSERT_OK(TestStaticBuiltins("Changling", &result, {}, + &failures, true)); + if (RegisterTests("Test")) { + ASSERT_OK(TestCreateStatic("Changling", &result, true)); + } +} + +TEST_F(LoadCustomizableTest, LoadEventListenerTest) { + ASSERT_OK(TestSharedBuiltins( + OnFileDeletionListener::kClassName(), "")); + if (RegisterTests("Test")) { + ExpectCreateShared(OnFileDeletionListener::kClassName()); + ExpectCreateShared(FlushCounterListener::kClassName()); + } +} + +TEST_F(LoadCustomizableTest, LoadEncryptionProviderTest) { + std::vector failures; + std::shared_ptr result; + ASSERT_OK( + TestExpectedBuiltins("Mock", {}, &result, &failures)); + if (!failures.empty()) { + ASSERT_EQ(failures[0], "1://test"); + ASSERT_EQ(failures.size(), 1U); + } + + result = ExpectCreateShared("CTR"); + ASSERT_NOK(result->ValidateOptions(db_opts_, cf_opts_)); + ASSERT_OK(EncryptionProvider::CreateFromString(config_options_, "CTR://test", + &result)); + ASSERT_NE(result, nullptr); + ASSERT_STREQ(result->Name(), "CTR"); + ASSERT_OK(result->ValidateOptions(db_opts_, cf_opts_)); + + if (RegisterTests("Test")) { + ExpectCreateShared("Mock"); + ASSERT_OK(EncryptionProvider::CreateFromString(config_options_, + "Mock://test", &result)); + ASSERT_NE(result, nullptr); + ASSERT_STREQ(result->Name(), "Mock"); + ASSERT_OK(result->ValidateOptions(db_opts_, cf_opts_)); + } +} + +TEST_F(LoadCustomizableTest, LoadEncryptionCipherTest) { + ASSERT_OK(TestSharedBuiltins("Mock", "ROT13")); + if (RegisterTests("Test")) { + ExpectCreateShared("Mock"); + } +} + +TEST_F(LoadCustomizableTest, LoadSystemClockTest) { + ASSERT_OK(TestSharedBuiltins(MockSystemClock::kClassName(), + SystemClock::kDefaultName())); + if (RegisterTests("Test")) { + auto result = + ExpectCreateShared(MockSystemClock::kClassName()); + ASSERT_FALSE(result->IsInstanceOf(SystemClock::kDefaultName())); + } +} + +TEST_F(LoadCustomizableTest, LoadMemoryAllocatorTest) { + std::vector failures; + Status s = TestSharedBuiltins( + MockMemoryAllocator::kClassName(), DefaultMemoryAllocator::kClassName(), + &failures); + if (failures.empty()) { + ASSERT_OK(s); + } else { + ASSERT_NOK(s); + for (const auto& failure : failures) { + if (failure == JemallocNodumpAllocator::kClassName()) { + ASSERT_FALSE(JemallocNodumpAllocator::IsSupported()); + } else if (failure == MemkindKmemAllocator::kClassName()) { + ASSERT_FALSE(MemkindKmemAllocator::IsSupported()); + } else { + printf("BYPASSED: %s -- %s\n", failure.c_str(), s.ToString().c_str()); + } + } + } + if (RegisterTests("Test")) { + ExpectCreateShared(MockMemoryAllocator::kClassName()); + } +} + +TEST_F(LoadCustomizableTest, LoadFilterPolicyTest) { + const std::string kAutoBloom = BloomFilterPolicy::kClassName(); + const std::string kAutoRibbon = RibbonFilterPolicy::kClassName(); + + std::shared_ptr result; + std::vector failures; + std::unordered_set expected = { + ReadOnlyBuiltinFilterPolicy::kClassName(), + }; + + expected.insert({ + kAutoBloom, + BloomFilterPolicy::kNickName(), + kAutoRibbon, + RibbonFilterPolicy::kNickName(), + }); + ASSERT_OK(TestExpectedBuiltins( + "Mock", expected, &result, &failures, [](const std::string& name) { + std::vector names = {name + ":1.234"}; + return names; + })); + ASSERT_OK(FilterPolicy::CreateFromString( + config_options_, kAutoBloom + ":1.234:false", &result)); + ASSERT_NE(result.get(), nullptr); + ASSERT_TRUE(result->IsInstanceOf(kAutoBloom)); + ASSERT_OK(FilterPolicy::CreateFromString( + config_options_, kAutoBloom + ":1.234:false", &result)); + ASSERT_NE(result.get(), nullptr); + ASSERT_TRUE(result->IsInstanceOf(kAutoBloom)); + ASSERT_OK(FilterPolicy::CreateFromString(config_options_, + kAutoRibbon + ":1.234:-1", &result)); + ASSERT_NE(result.get(), nullptr); + ASSERT_TRUE(result->IsInstanceOf(kAutoRibbon)); + ASSERT_OK(FilterPolicy::CreateFromString(config_options_, + kAutoRibbon + ":1.234:56", &result)); + ASSERT_NE(result.get(), nullptr); + ASSERT_TRUE(result->IsInstanceOf(kAutoRibbon)); + + if (RegisterTests("Test")) { + ExpectCreateShared(MockFilterPolicy::kClassName(), &result); + } + + std::shared_ptr table; + + std::string table_opts = "id=BlockBasedTable; filter_policy="; + ASSERT_OK(TableFactory::CreateFromString(config_options_, + table_opts + "nullptr", &table)); + ASSERT_NE(table.get(), nullptr); + auto bbto = table->GetOptions(); + ASSERT_NE(bbto, nullptr); + ASSERT_EQ(bbto->filter_policy.get(), nullptr); + ASSERT_OK(TableFactory::CreateFromString( + config_options_, table_opts + ReadOnlyBuiltinFilterPolicy::kClassName(), + &table)); + bbto = table->GetOptions(); + ASSERT_NE(bbto, nullptr); + ASSERT_NE(bbto->filter_policy.get(), nullptr); + ASSERT_STREQ(bbto->filter_policy->Name(), + ReadOnlyBuiltinFilterPolicy::kClassName()); + ASSERT_OK(TableFactory::CreateFromString( + config_options_, table_opts + MockFilterPolicy::kClassName(), &table)); + bbto = table->GetOptions(); + ASSERT_NE(bbto, nullptr); + ASSERT_NE(bbto->filter_policy.get(), nullptr); + ASSERT_TRUE( + bbto->filter_policy->IsInstanceOf(MockFilterPolicy::kClassName())); +} + +TEST_F(LoadCustomizableTest, LoadFlushBlockPolicyFactoryTest) { + std::shared_ptr result; + std::shared_ptr table; + std::vector failed; + std::unordered_set expected = { + FlushBlockBySizePolicyFactory::kClassName(), + FlushBlockEveryKeyPolicyFactory::kClassName(), + }; + + ASSERT_OK(TestExpectedBuiltins( + TestFlushBlockPolicyFactory::kClassName(), expected, &result, &failed)); + + // An empty policy name creates a BySize policy + ASSERT_OK( + FlushBlockPolicyFactory::CreateFromString(config_options_, "", &result)); + ASSERT_NE(result, nullptr); + ASSERT_STREQ(result->Name(), FlushBlockBySizePolicyFactory::kClassName()); + + std::string table_opts = "id=BlockBasedTable; flush_block_policy_factory="; + ASSERT_OK(TableFactory::CreateFromString( + config_options_, + table_opts + FlushBlockEveryKeyPolicyFactory::kClassName(), &table)); + auto bbto = table->GetOptions(); + ASSERT_NE(bbto, nullptr); + ASSERT_NE(bbto->flush_block_policy_factory.get(), nullptr); + ASSERT_STREQ(bbto->flush_block_policy_factory->Name(), + FlushBlockEveryKeyPolicyFactory::kClassName()); + if (RegisterTests("Test")) { + ExpectCreateShared( + TestFlushBlockPolicyFactory::kClassName()); + ASSERT_OK(TableFactory::CreateFromString( + config_options_, table_opts + TestFlushBlockPolicyFactory::kClassName(), + &table)); + bbto = table->GetOptions(); + ASSERT_NE(bbto, nullptr); + ASSERT_NE(bbto->flush_block_policy_factory.get(), nullptr); + ASSERT_STREQ(bbto->flush_block_policy_factory->Name(), + TestFlushBlockPolicyFactory::kClassName()); + } +} + +} // namespace ROCKSDB_NAMESPACE +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); +#ifdef GFLAGS + ParseCommandLineFlags(&argc, &argv, true); +#endif // GFLAGS + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/options/db_options.cc b/librocksdb-sys/rocksdb/options/db_options.cc new file mode 100644 index 0000000..f009c1a --- /dev/null +++ b/librocksdb-sys/rocksdb/options/db_options.cc @@ -0,0 +1,1086 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "options/db_options.h" + +#include + +#include "logging/logging.h" +#include "options/configurable_helper.h" +#include "options/options_helper.h" +#include "options/options_parser.h" +#include "port/port.h" +#include "rocksdb/advanced_cache.h" +#include "rocksdb/configurable.h" +#include "rocksdb/env.h" +#include "rocksdb/file_system.h" +#include "rocksdb/listener.h" +#include "rocksdb/rate_limiter.h" +#include "rocksdb/sst_file_manager.h" +#include "rocksdb/statistics.h" +#include "rocksdb/system_clock.h" +#include "rocksdb/utilities/options_type.h" +#include "rocksdb/wal_filter.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { +static std::unordered_map + wal_recovery_mode_string_map = { + {"kTolerateCorruptedTailRecords", + WALRecoveryMode::kTolerateCorruptedTailRecords}, + {"kAbsoluteConsistency", WALRecoveryMode::kAbsoluteConsistency}, + {"kPointInTimeRecovery", WALRecoveryMode::kPointInTimeRecovery}, + {"kSkipAnyCorruptedRecords", + WALRecoveryMode::kSkipAnyCorruptedRecords}}; + +static std::unordered_map + access_hint_string_map = {{"NONE", DBOptions::AccessHint::NONE}, + {"NORMAL", DBOptions::AccessHint::NORMAL}, + {"SEQUENTIAL", DBOptions::AccessHint::SEQUENTIAL}, + {"WILLNEED", DBOptions::AccessHint::WILLNEED}}; + +static std::unordered_map cache_tier_string_map = { + {"kVolatileTier", CacheTier::kVolatileTier}, + {"kNonVolatileBlockTier", CacheTier::kNonVolatileBlockTier}}; + +static std::unordered_map info_log_level_string_map = + {{"DEBUG_LEVEL", InfoLogLevel::DEBUG_LEVEL}, + {"INFO_LEVEL", InfoLogLevel::INFO_LEVEL}, + {"WARN_LEVEL", InfoLogLevel::WARN_LEVEL}, + {"ERROR_LEVEL", InfoLogLevel::ERROR_LEVEL}, + {"FATAL_LEVEL", InfoLogLevel::FATAL_LEVEL}, + {"HEADER_LEVEL", InfoLogLevel::HEADER_LEVEL}}; + +static std::unordered_map + db_mutable_options_type_info = { + {"allow_os_buffer", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable}}, + {"base_background_compactions", + {0, OptionType::kInt, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable}}, + {"max_background_jobs", + {offsetof(struct MutableDBOptions, max_background_jobs), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"max_background_compactions", + {offsetof(struct MutableDBOptions, max_background_compactions), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"max_subcompactions", + {offsetof(struct MutableDBOptions, max_subcompactions), + OptionType::kUInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"avoid_flush_during_shutdown", + {offsetof(struct MutableDBOptions, avoid_flush_during_shutdown), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"writable_file_max_buffer_size", + {offsetof(struct MutableDBOptions, writable_file_max_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"delayed_write_rate", + {offsetof(struct MutableDBOptions, delayed_write_rate), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"max_total_wal_size", + {offsetof(struct MutableDBOptions, max_total_wal_size), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"delete_obsolete_files_period_micros", + {offsetof(struct MutableDBOptions, + delete_obsolete_files_period_micros), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"stats_dump_period_sec", + {offsetof(struct MutableDBOptions, stats_dump_period_sec), + OptionType::kUInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"stats_persist_period_sec", + {offsetof(struct MutableDBOptions, stats_persist_period_sec), + OptionType::kUInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"stats_history_buffer_size", + {offsetof(struct MutableDBOptions, stats_history_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"max_open_files", + {offsetof(struct MutableDBOptions, max_open_files), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, + {"bytes_per_sync", + {offsetof(struct MutableDBOptions, bytes_per_sync), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"wal_bytes_per_sync", + {offsetof(struct MutableDBOptions, wal_bytes_per_sync), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"strict_bytes_per_sync", + {offsetof(struct MutableDBOptions, strict_bytes_per_sync), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"compaction_readahead_size", + {offsetof(struct MutableDBOptions, compaction_readahead_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"max_background_flushes", + {offsetof(struct MutableDBOptions, max_background_flushes), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, +}; + +static std::unordered_map + db_immutable_options_type_info = { + /* + // not yet supported + std::shared_ptr row_cache; + std::shared_ptr delete_scheduler; + std::shared_ptr info_log; + std::shared_ptr rate_limiter; + std::shared_ptr statistics; + std::vector db_paths; + FileTypeSet checksum_handoff_file_types; + */ + {"advise_random_on_open", + {offsetof(struct ImmutableDBOptions, advise_random_on_open), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"allow_mmap_reads", + {offsetof(struct ImmutableDBOptions, allow_mmap_reads), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"allow_fallocate", + {offsetof(struct ImmutableDBOptions, allow_fallocate), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"allow_mmap_writes", + {offsetof(struct ImmutableDBOptions, allow_mmap_writes), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"use_direct_reads", + {offsetof(struct ImmutableDBOptions, use_direct_reads), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"use_direct_writes", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"use_direct_io_for_flush_and_compaction", + {offsetof(struct ImmutableDBOptions, + use_direct_io_for_flush_and_compaction), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"allow_2pc", + {offsetof(struct ImmutableDBOptions, allow_2pc), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"wal_filter", + OptionTypeInfo::AsCustomRawPtr( + offsetof(struct ImmutableDBOptions, wal_filter), + OptionVerificationType::kByName, + (OptionTypeFlags::kAllowNull | OptionTypeFlags::kCompareNever))}, + {"create_if_missing", + {offsetof(struct ImmutableDBOptions, create_if_missing), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"create_missing_column_families", + {offsetof(struct ImmutableDBOptions, create_missing_column_families), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"disableDataSync", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"disable_data_sync", // for compatibility + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"enable_thread_tracking", + {offsetof(struct ImmutableDBOptions, enable_thread_tracking), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"error_if_exists", + {offsetof(struct ImmutableDBOptions, error_if_exists), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"experimental_allow_mempurge", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"experimental_mempurge_policy", + {0, OptionType::kString, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"experimental_mempurge_threshold", + {0, OptionType::kDouble, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"is_fd_close_on_exec", + {offsetof(struct ImmutableDBOptions, is_fd_close_on_exec), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"paranoid_checks", + {offsetof(struct ImmutableDBOptions, paranoid_checks), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"flush_verify_memtable_count", + {offsetof(struct ImmutableDBOptions, flush_verify_memtable_count), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"compaction_verify_record_count", + {offsetof(struct ImmutableDBOptions, compaction_verify_record_count), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"track_and_verify_wals_in_manifest", + {offsetof(struct ImmutableDBOptions, + track_and_verify_wals_in_manifest), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"verify_sst_unique_id_in_manifest", + {offsetof(struct ImmutableDBOptions, verify_sst_unique_id_in_manifest), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"skip_log_error_on_recovery", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"skip_stats_update_on_db_open", + {offsetof(struct ImmutableDBOptions, skip_stats_update_on_db_open), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"skip_checking_sst_file_sizes_on_db_open", + {offsetof(struct ImmutableDBOptions, + skip_checking_sst_file_sizes_on_db_open), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"new_table_reader_for_compaction_inputs", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"random_access_max_buffer_size", + {offsetof(struct ImmutableDBOptions, random_access_max_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"use_adaptive_mutex", + {offsetof(struct ImmutableDBOptions, use_adaptive_mutex), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"use_fsync", + {offsetof(struct ImmutableDBOptions, use_fsync), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"max_file_opening_threads", + {offsetof(struct ImmutableDBOptions, max_file_opening_threads), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"table_cache_numshardbits", + {offsetof(struct ImmutableDBOptions, table_cache_numshardbits), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"db_write_buffer_size", + {offsetof(struct ImmutableDBOptions, db_write_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"keep_log_file_num", + {offsetof(struct ImmutableDBOptions, keep_log_file_num), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"recycle_log_file_num", + {offsetof(struct ImmutableDBOptions, recycle_log_file_num), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"log_file_time_to_roll", + {offsetof(struct ImmutableDBOptions, log_file_time_to_roll), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"manifest_preallocation_size", + {offsetof(struct ImmutableDBOptions, manifest_preallocation_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"max_log_file_size", + {offsetof(struct ImmutableDBOptions, max_log_file_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"db_log_dir", + {offsetof(struct ImmutableDBOptions, db_log_dir), OptionType::kString, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"wal_dir", + {offsetof(struct ImmutableDBOptions, wal_dir), OptionType::kString, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"WAL_size_limit_MB", + {offsetof(struct ImmutableDBOptions, WAL_size_limit_MB), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"WAL_ttl_seconds", + {offsetof(struct ImmutableDBOptions, WAL_ttl_seconds), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"max_manifest_file_size", + {offsetof(struct ImmutableDBOptions, max_manifest_file_size), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"persist_stats_to_disk", + {offsetof(struct ImmutableDBOptions, persist_stats_to_disk), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"fail_if_options_file_error", + {offsetof(struct ImmutableDBOptions, fail_if_options_file_error), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"enable_pipelined_write", + {offsetof(struct ImmutableDBOptions, enable_pipelined_write), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"unordered_write", + {offsetof(struct ImmutableDBOptions, unordered_write), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"allow_concurrent_memtable_write", + {offsetof(struct ImmutableDBOptions, allow_concurrent_memtable_write), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"wal_recovery_mode", + OptionTypeInfo::Enum( + offsetof(struct ImmutableDBOptions, wal_recovery_mode), + &wal_recovery_mode_string_map)}, + {"enable_write_thread_adaptive_yield", + {offsetof(struct ImmutableDBOptions, + enable_write_thread_adaptive_yield), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"write_thread_slow_yield_usec", + {offsetof(struct ImmutableDBOptions, write_thread_slow_yield_usec), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"max_write_batch_group_size_bytes", + {offsetof(struct ImmutableDBOptions, max_write_batch_group_size_bytes), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"write_thread_max_yield_usec", + {offsetof(struct ImmutableDBOptions, write_thread_max_yield_usec), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"access_hint_on_compaction_start", + OptionTypeInfo::Enum( + offsetof(struct ImmutableDBOptions, + access_hint_on_compaction_start), + &access_hint_string_map)}, + {"info_log_level", + OptionTypeInfo::Enum( + offsetof(struct ImmutableDBOptions, info_log_level), + &info_log_level_string_map)}, + {"dump_malloc_stats", + {offsetof(struct ImmutableDBOptions, dump_malloc_stats), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"avoid_flush_during_recovery", + {offsetof(struct ImmutableDBOptions, avoid_flush_during_recovery), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"allow_ingest_behind", + {offsetof(struct ImmutableDBOptions, allow_ingest_behind), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"preserve_deletes", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"concurrent_prepare", // Deprecated by two_write_queues + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"two_write_queues", + {offsetof(struct ImmutableDBOptions, two_write_queues), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"manual_wal_flush", + {offsetof(struct ImmutableDBOptions, manual_wal_flush), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"wal_compression", + {offsetof(struct ImmutableDBOptions, wal_compression), + OptionType::kCompressionType, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"seq_per_batch", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"atomic_flush", + {offsetof(struct ImmutableDBOptions, atomic_flush), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"avoid_unnecessary_blocking_io", + {offsetof(struct ImmutableDBOptions, avoid_unnecessary_blocking_io), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"write_dbid_to_manifest", + {offsetof(struct ImmutableDBOptions, write_dbid_to_manifest), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"log_readahead_size", + {offsetof(struct ImmutableDBOptions, log_readahead_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"best_efforts_recovery", + {offsetof(struct ImmutableDBOptions, best_efforts_recovery), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"max_bgerror_resume_count", + {offsetof(struct ImmutableDBOptions, max_bgerror_resume_count), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"bgerror_resume_retry_interval", + {offsetof(struct ImmutableDBOptions, bgerror_resume_retry_interval), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"db_host_id", + {offsetof(struct ImmutableDBOptions, db_host_id), OptionType::kString, + OptionVerificationType::kNormal, OptionTypeFlags::kCompareNever}}, + // Temporarily deprecated due to race conditions (examples in PR 10375). + {"rate_limiter", + {offsetof(struct ImmutableDBOptions, rate_limiter), + OptionType::kUnknown, OptionVerificationType::kDeprecated, + OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever}}, + // The following properties were handled as special cases in ParseOption + // This means that the properties could be read from the options file + // but never written to the file or compared to each other. + {"rate_limiter_bytes_per_sec", + {offsetof(struct ImmutableDBOptions, rate_limiter), + OptionType::kUnknown, OptionVerificationType::kNormal, + (OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever), + // Parse the input value as a RateLimiter + [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + const std::string& value, void* addr) { + auto limiter = static_cast*>(addr); + limiter->reset(NewGenericRateLimiter( + static_cast(ParseUint64(value)))); + return Status::OK(); + }}}, + {"env", //**TODO: Should this be kCustomizable? + OptionTypeInfo( + offsetof(struct ImmutableDBOptions, env), OptionType::kUnknown, + OptionVerificationType::kNormal, + (OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever)) + .SetParseFunc([](const ConfigOptions& opts, + const std::string& /*name*/, + const std::string& value, void* addr) { + // Parse the input value as an Env + auto old_env = static_cast(addr); // Get the old value + Env* new_env = *old_env; // Set new to old + Status s = Env::CreateFromString(opts, value, + &new_env); // Update new value + if (s.ok()) { // It worked + *old_env = new_env; // Update the old one + } + return s; + }) + .SetPrepareFunc([](const ConfigOptions& opts, + const std::string& /*name*/, void* addr) { + auto env = static_cast(addr); + return (*env)->PrepareOptions(opts); + }) + .SetValidateFunc([](const DBOptions& db_opts, + const ColumnFamilyOptions& cf_opts, + const std::string& /*name*/, + const void* addr) { + const auto env = static_cast(addr); + return (*env)->ValidateOptions(db_opts, cf_opts); + })}, + {"allow_data_in_errors", + {offsetof(struct ImmutableDBOptions, allow_data_in_errors), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"file_checksum_gen_factory", + OptionTypeInfo::AsCustomSharedPtr( + offsetof(struct ImmutableDBOptions, file_checksum_gen_factory), + OptionVerificationType::kByNameAllowFromNull, + OptionTypeFlags::kAllowNull)}, + {"statistics", + OptionTypeInfo::AsCustomSharedPtr( + // Statistics should not be compared and can be null + // Statistics are maked "don't serialize" until they can be shared + // between DBs + offsetof(struct ImmutableDBOptions, statistics), + OptionVerificationType::kNormal, + OptionTypeFlags::kCompareNever | OptionTypeFlags::kDontSerialize | + OptionTypeFlags::kAllowNull)}, + // Allow EventListeners that have a non-empty Name() to be read/written + // as options Each listener will either be + // - A simple name (e.g. "MyEventListener") + // - A name with properties (e.g. "{id=MyListener1; timeout=60}" + // Multiple listeners will be separated by a ":": + // - "MyListener0;{id=MyListener1; timeout=60} + {"listeners", + {offsetof(struct ImmutableDBOptions, listeners), OptionType::kVector, + OptionVerificationType::kByNameAllowNull, + OptionTypeFlags::kCompareNever, + [](const ConfigOptions& opts, const std::string& /*name*/, + const std::string& value, void* addr) { + ConfigOptions embedded = opts; + embedded.ignore_unsupported_options = true; + std::vector> listeners; + Status s; + for (size_t start = 0, end = 0; + s.ok() && start < value.size() && end != std::string::npos; + start = end + 1) { + std::string token; + s = OptionTypeInfo::NextToken(value, ':', start, &end, &token); + if (s.ok() && !token.empty()) { + std::shared_ptr listener; + s = EventListener::CreateFromString(embedded, token, &listener); + if (s.ok() && listener != nullptr) { + listeners.push_back(listener); + } + } + } + if (s.ok()) { // It worked + *(static_cast>*>( + addr)) = listeners; + } + return s; + }, + [](const ConfigOptions& opts, const std::string& /*name*/, + const void* addr, std::string* value) { + const auto listeners = + static_cast>*>( + addr); + ConfigOptions embedded = opts; + embedded.delimiter = ";"; + int printed = 0; + for (const auto& listener : *listeners) { + auto id = listener->GetId(); + if (!id.empty()) { + std::string elem_str = listener->ToString(embedded, ""); + if (printed++ == 0) { + value->append("{"); + } else { + value->append(":"); + } + value->append(elem_str); + } + } + if (printed > 0) { + value->append("}"); + } + return Status::OK(); + }, + nullptr}}, + {"lowest_used_cache_tier", + OptionTypeInfo::Enum( + offsetof(struct ImmutableDBOptions, lowest_used_cache_tier), + &cache_tier_string_map, OptionTypeFlags::kNone)}, + {"enforce_single_del_contracts", + {offsetof(struct ImmutableDBOptions, enforce_single_del_contracts), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, +}; + +const std::string OptionsHelper::kDBOptionsName = "DBOptions"; + +class MutableDBConfigurable : public Configurable { + public: + explicit MutableDBConfigurable( + const MutableDBOptions& mdb, + const std::unordered_map* map = nullptr) + : mutable_(mdb), opt_map_(map) { + RegisterOptions(&mutable_, &db_mutable_options_type_info); + } + + bool OptionsAreEqual(const ConfigOptions& config_options, + const OptionTypeInfo& opt_info, + const std::string& opt_name, const void* const this_ptr, + const void* const that_ptr, + std::string* mismatch) const override { + bool equals = opt_info.AreEqual(config_options, opt_name, this_ptr, + that_ptr, mismatch); + if (!equals && opt_info.IsByName()) { + if (opt_map_ == nullptr) { + equals = true; + } else { + const auto& iter = opt_map_->find(opt_name); + if (iter == opt_map_->end()) { + equals = true; + } else { + equals = opt_info.AreEqualByName(config_options, opt_name, this_ptr, + iter->second); + } + } + if (equals) { // False alarm, clear mismatch + *mismatch = ""; + } + } + if (equals && opt_info.IsConfigurable() && opt_map_ != nullptr) { + const auto* this_config = opt_info.AsRawPointer(this_ptr); + if (this_config == nullptr) { + const auto& iter = opt_map_->find(opt_name); + // If the name exists in the map and is not empty/null, + // then the this_config should be set. + if (iter != opt_map_->end() && !iter->second.empty() && + iter->second != kNullptrString) { + *mismatch = opt_name; + equals = false; + } + } + } + return equals; + } + + protected: + MutableDBOptions mutable_; + const std::unordered_map* opt_map_; +}; + +class DBOptionsConfigurable : public MutableDBConfigurable { + public: + explicit DBOptionsConfigurable( + const DBOptions& opts, + const std::unordered_map* map = nullptr) + : MutableDBConfigurable(MutableDBOptions(opts), map), db_options_(opts) { + // The ImmutableDBOptions currently requires the env to be non-null. Make + // sure it is + if (opts.env != nullptr) { + immutable_ = ImmutableDBOptions(opts); + } else { + DBOptions copy = opts; + copy.env = Env::Default(); + immutable_ = ImmutableDBOptions(copy); + } + RegisterOptions(&immutable_, &db_immutable_options_type_info); + } + + protected: + Status ConfigureOptions( + const ConfigOptions& config_options, + const std::unordered_map& opts_map, + std::unordered_map* unused) override { + Status s = Configurable::ConfigureOptions(config_options, opts_map, unused); + if (s.ok()) { + db_options_ = BuildDBOptions(immutable_, mutable_); + s = PrepareOptions(config_options); + } + return s; + } + + const void* GetOptionsPtr(const std::string& name) const override { + if (name == OptionsHelper::kDBOptionsName) { + return &db_options_; + } else { + return MutableDBConfigurable::GetOptionsPtr(name); + } + } + + private: + ImmutableDBOptions immutable_; + DBOptions db_options_; +}; + +std::unique_ptr DBOptionsAsConfigurable( + const MutableDBOptions& opts) { + std::unique_ptr ptr(new MutableDBConfigurable(opts)); + return ptr; +} +std::unique_ptr DBOptionsAsConfigurable( + const DBOptions& opts, + const std::unordered_map* opt_map) { + std::unique_ptr ptr(new DBOptionsConfigurable(opts, opt_map)); + return ptr; +} + +ImmutableDBOptions::ImmutableDBOptions() : ImmutableDBOptions(Options()) {} + +ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options) + : create_if_missing(options.create_if_missing), + create_missing_column_families(options.create_missing_column_families), + error_if_exists(options.error_if_exists), + paranoid_checks(options.paranoid_checks), + flush_verify_memtable_count(options.flush_verify_memtable_count), + compaction_verify_record_count(options.compaction_verify_record_count), + track_and_verify_wals_in_manifest( + options.track_and_verify_wals_in_manifest), + verify_sst_unique_id_in_manifest( + options.verify_sst_unique_id_in_manifest), + env(options.env), + rate_limiter(options.rate_limiter), + sst_file_manager(options.sst_file_manager), + info_log(options.info_log), + info_log_level(options.info_log_level), + max_file_opening_threads(options.max_file_opening_threads), + statistics(options.statistics), + use_fsync(options.use_fsync), + db_paths(options.db_paths), + db_log_dir(options.db_log_dir), + wal_dir(options.wal_dir), + max_log_file_size(options.max_log_file_size), + log_file_time_to_roll(options.log_file_time_to_roll), + keep_log_file_num(options.keep_log_file_num), + recycle_log_file_num(options.recycle_log_file_num), + max_manifest_file_size(options.max_manifest_file_size), + table_cache_numshardbits(options.table_cache_numshardbits), + WAL_ttl_seconds(options.WAL_ttl_seconds), + WAL_size_limit_MB(options.WAL_size_limit_MB), + max_write_batch_group_size_bytes( + options.max_write_batch_group_size_bytes), + manifest_preallocation_size(options.manifest_preallocation_size), + allow_mmap_reads(options.allow_mmap_reads), + allow_mmap_writes(options.allow_mmap_writes), + use_direct_reads(options.use_direct_reads), + use_direct_io_for_flush_and_compaction( + options.use_direct_io_for_flush_and_compaction), + allow_fallocate(options.allow_fallocate), + is_fd_close_on_exec(options.is_fd_close_on_exec), + advise_random_on_open(options.advise_random_on_open), + db_write_buffer_size(options.db_write_buffer_size), + write_buffer_manager(options.write_buffer_manager), + access_hint_on_compaction_start(options.access_hint_on_compaction_start), + random_access_max_buffer_size(options.random_access_max_buffer_size), + use_adaptive_mutex(options.use_adaptive_mutex), + listeners(options.listeners), + enable_thread_tracking(options.enable_thread_tracking), + enable_pipelined_write(options.enable_pipelined_write), + unordered_write(options.unordered_write), + allow_concurrent_memtable_write(options.allow_concurrent_memtable_write), + enable_write_thread_adaptive_yield( + options.enable_write_thread_adaptive_yield), + write_thread_max_yield_usec(options.write_thread_max_yield_usec), + write_thread_slow_yield_usec(options.write_thread_slow_yield_usec), + skip_stats_update_on_db_open(options.skip_stats_update_on_db_open), + skip_checking_sst_file_sizes_on_db_open( + options.skip_checking_sst_file_sizes_on_db_open), + wal_recovery_mode(options.wal_recovery_mode), + allow_2pc(options.allow_2pc), + row_cache(options.row_cache), + wal_filter(options.wal_filter), + fail_if_options_file_error(options.fail_if_options_file_error), + dump_malloc_stats(options.dump_malloc_stats), + avoid_flush_during_recovery(options.avoid_flush_during_recovery), + allow_ingest_behind(options.allow_ingest_behind), + two_write_queues(options.two_write_queues), + manual_wal_flush(options.manual_wal_flush), + wal_compression(options.wal_compression), + atomic_flush(options.atomic_flush), + avoid_unnecessary_blocking_io(options.avoid_unnecessary_blocking_io), + persist_stats_to_disk(options.persist_stats_to_disk), + write_dbid_to_manifest(options.write_dbid_to_manifest), + log_readahead_size(options.log_readahead_size), + file_checksum_gen_factory(options.file_checksum_gen_factory), + best_efforts_recovery(options.best_efforts_recovery), + max_bgerror_resume_count(options.max_bgerror_resume_count), + bgerror_resume_retry_interval(options.bgerror_resume_retry_interval), + allow_data_in_errors(options.allow_data_in_errors), + db_host_id(options.db_host_id), + checksum_handoff_file_types(options.checksum_handoff_file_types), + lowest_used_cache_tier(options.lowest_used_cache_tier), + compaction_service(options.compaction_service), + enforce_single_del_contracts(options.enforce_single_del_contracts) { + fs = env->GetFileSystem(); + clock = env->GetSystemClock().get(); + logger = info_log.get(); + stats = statistics.get(); +} + +void ImmutableDBOptions::Dump(Logger* log) const { + ROCKS_LOG_HEADER(log, " Options.error_if_exists: %d", + error_if_exists); + ROCKS_LOG_HEADER(log, " Options.create_if_missing: %d", + create_if_missing); + ROCKS_LOG_HEADER(log, " Options.paranoid_checks: %d", + paranoid_checks); + ROCKS_LOG_HEADER(log, " Options.flush_verify_memtable_count: %d", + flush_verify_memtable_count); + ROCKS_LOG_HEADER(log, " Options.compaction_verify_record_count: %d", + compaction_verify_record_count); + ROCKS_LOG_HEADER(log, + " " + "Options.track_and_verify_wals_in_manifest: %d", + track_and_verify_wals_in_manifest); + ROCKS_LOG_HEADER(log, " Options.verify_sst_unique_id_in_manifest: %d", + verify_sst_unique_id_in_manifest); + ROCKS_LOG_HEADER(log, " Options.env: %p", + env); + ROCKS_LOG_HEADER(log, " Options.fs: %s", + fs->Name()); + ROCKS_LOG_HEADER(log, " Options.info_log: %p", + info_log.get()); + ROCKS_LOG_HEADER(log, " Options.max_file_opening_threads: %d", + max_file_opening_threads); + ROCKS_LOG_HEADER(log, " Options.statistics: %p", + stats); + ROCKS_LOG_HEADER(log, " Options.use_fsync: %d", + use_fsync); + ROCKS_LOG_HEADER( + log, " Options.max_log_file_size: %" ROCKSDB_PRIszt, + max_log_file_size); + ROCKS_LOG_HEADER(log, + " Options.max_manifest_file_size: %" PRIu64, + max_manifest_file_size); + ROCKS_LOG_HEADER( + log, " Options.log_file_time_to_roll: %" ROCKSDB_PRIszt, + log_file_time_to_roll); + ROCKS_LOG_HEADER( + log, " Options.keep_log_file_num: %" ROCKSDB_PRIszt, + keep_log_file_num); + ROCKS_LOG_HEADER( + log, " Options.recycle_log_file_num: %" ROCKSDB_PRIszt, + recycle_log_file_num); + ROCKS_LOG_HEADER(log, " Options.allow_fallocate: %d", + allow_fallocate); + ROCKS_LOG_HEADER(log, " Options.allow_mmap_reads: %d", + allow_mmap_reads); + ROCKS_LOG_HEADER(log, " Options.allow_mmap_writes: %d", + allow_mmap_writes); + ROCKS_LOG_HEADER(log, " Options.use_direct_reads: %d", + use_direct_reads); + ROCKS_LOG_HEADER(log, + " " + "Options.use_direct_io_for_flush_and_compaction: %d", + use_direct_io_for_flush_and_compaction); + ROCKS_LOG_HEADER(log, " Options.create_missing_column_families: %d", + create_missing_column_families); + ROCKS_LOG_HEADER(log, " Options.db_log_dir: %s", + db_log_dir.c_str()); + ROCKS_LOG_HEADER(log, " Options.wal_dir: %s", + wal_dir.c_str()); + ROCKS_LOG_HEADER(log, " Options.table_cache_numshardbits: %d", + table_cache_numshardbits); + ROCKS_LOG_HEADER(log, + " Options.WAL_ttl_seconds: %" PRIu64, + WAL_ttl_seconds); + ROCKS_LOG_HEADER(log, + " Options.WAL_size_limit_MB: %" PRIu64, + WAL_size_limit_MB); + ROCKS_LOG_HEADER(log, + " " + "Options.max_write_batch_group_size_bytes: %" PRIu64, + max_write_batch_group_size_bytes); + ROCKS_LOG_HEADER( + log, " Options.manifest_preallocation_size: %" ROCKSDB_PRIszt, + manifest_preallocation_size); + ROCKS_LOG_HEADER(log, " Options.is_fd_close_on_exec: %d", + is_fd_close_on_exec); + ROCKS_LOG_HEADER(log, " Options.advise_random_on_open: %d", + advise_random_on_open); + ROCKS_LOG_HEADER( + log, " Options.db_write_buffer_size: %" ROCKSDB_PRIszt, + db_write_buffer_size); + ROCKS_LOG_HEADER(log, " Options.write_buffer_manager: %p", + write_buffer_manager.get()); + ROCKS_LOG_HEADER(log, " Options.access_hint_on_compaction_start: %d", + static_cast(access_hint_on_compaction_start)); + ROCKS_LOG_HEADER( + log, " Options.random_access_max_buffer_size: %" ROCKSDB_PRIszt, + random_access_max_buffer_size); + ROCKS_LOG_HEADER(log, " Options.use_adaptive_mutex: %d", + use_adaptive_mutex); + ROCKS_LOG_HEADER(log, " Options.rate_limiter: %p", + rate_limiter.get()); + Header( + log, " Options.sst_file_manager.rate_bytes_per_sec: %" PRIi64, + sst_file_manager ? sst_file_manager->GetDeleteRateBytesPerSecond() : 0); + ROCKS_LOG_HEADER(log, " Options.wal_recovery_mode: %d", + static_cast(wal_recovery_mode)); + ROCKS_LOG_HEADER(log, " Options.enable_thread_tracking: %d", + enable_thread_tracking); + ROCKS_LOG_HEADER(log, " Options.enable_pipelined_write: %d", + enable_pipelined_write); + ROCKS_LOG_HEADER(log, " Options.unordered_write: %d", + unordered_write); + ROCKS_LOG_HEADER(log, " Options.allow_concurrent_memtable_write: %d", + allow_concurrent_memtable_write); + ROCKS_LOG_HEADER(log, " Options.enable_write_thread_adaptive_yield: %d", + enable_write_thread_adaptive_yield); + ROCKS_LOG_HEADER(log, + " Options.write_thread_max_yield_usec: %" PRIu64, + write_thread_max_yield_usec); + ROCKS_LOG_HEADER(log, + " Options.write_thread_slow_yield_usec: %" PRIu64, + write_thread_slow_yield_usec); + if (row_cache) { + ROCKS_LOG_HEADER( + log, + " Options.row_cache: %" ROCKSDB_PRIszt, + row_cache->GetCapacity()); + } else { + ROCKS_LOG_HEADER(log, + " Options.row_cache: None"); + } + ROCKS_LOG_HEADER(log, " Options.wal_filter: %s", + wal_filter ? wal_filter->Name() : "None"); + + ROCKS_LOG_HEADER(log, " Options.avoid_flush_during_recovery: %d", + avoid_flush_during_recovery); + ROCKS_LOG_HEADER(log, " Options.allow_ingest_behind: %d", + allow_ingest_behind); + ROCKS_LOG_HEADER(log, " Options.two_write_queues: %d", + two_write_queues); + ROCKS_LOG_HEADER(log, " Options.manual_wal_flush: %d", + manual_wal_flush); + ROCKS_LOG_HEADER(log, " Options.wal_compression: %d", + wal_compression); + ROCKS_LOG_HEADER(log, " Options.atomic_flush: %d", atomic_flush); + ROCKS_LOG_HEADER(log, + " Options.avoid_unnecessary_blocking_io: %d", + avoid_unnecessary_blocking_io); + ROCKS_LOG_HEADER(log, " Options.persist_stats_to_disk: %u", + persist_stats_to_disk); + ROCKS_LOG_HEADER(log, " Options.write_dbid_to_manifest: %d", + write_dbid_to_manifest); + ROCKS_LOG_HEADER( + log, " Options.log_readahead_size: %" ROCKSDB_PRIszt, + log_readahead_size); + ROCKS_LOG_HEADER(log, " Options.file_checksum_gen_factory: %s", + file_checksum_gen_factory ? file_checksum_gen_factory->Name() + : kUnknownFileChecksumFuncName); + ROCKS_LOG_HEADER(log, " Options.best_efforts_recovery: %d", + static_cast(best_efforts_recovery)); + ROCKS_LOG_HEADER(log, " Options.max_bgerror_resume_count: %d", + max_bgerror_resume_count); + ROCKS_LOG_HEADER(log, + " Options.bgerror_resume_retry_interval: %" PRIu64, + bgerror_resume_retry_interval); + ROCKS_LOG_HEADER(log, " Options.allow_data_in_errors: %d", + allow_data_in_errors); + ROCKS_LOG_HEADER(log, " Options.db_host_id: %s", + db_host_id.c_str()); + ROCKS_LOG_HEADER(log, " Options.enforce_single_del_contracts: %s", + enforce_single_del_contracts ? "true" : "false"); +} + +bool ImmutableDBOptions::IsWalDirSameAsDBPath() const { + assert(!db_paths.empty()); + return IsWalDirSameAsDBPath(db_paths[0].path); +} + +bool ImmutableDBOptions::IsWalDirSameAsDBPath( + const std::string& db_path) const { + bool same = wal_dir.empty(); + if (!same) { + Status s = env->AreFilesSame(wal_dir, db_path, &same); + if (s.IsNotSupported()) { + same = wal_dir == db_path; + } + } + return same; +} + +const std::string& ImmutableDBOptions::GetWalDir() const { + if (wal_dir.empty()) { + assert(!db_paths.empty()); + return db_paths[0].path; + } else { + return wal_dir; + } +} + +const std::string& ImmutableDBOptions::GetWalDir( + const std::string& path) const { + if (wal_dir.empty()) { + return path; + } else { + return wal_dir; + } +} + +MutableDBOptions::MutableDBOptions() + : max_background_jobs(2), + max_background_compactions(-1), + max_subcompactions(0), + avoid_flush_during_shutdown(false), + writable_file_max_buffer_size(1024 * 1024), + delayed_write_rate(2 * 1024U * 1024U), + max_total_wal_size(0), + delete_obsolete_files_period_micros(6ULL * 60 * 60 * 1000000), + stats_dump_period_sec(600), + stats_persist_period_sec(600), + stats_history_buffer_size(1024 * 1024), + max_open_files(-1), + bytes_per_sync(0), + wal_bytes_per_sync(0), + strict_bytes_per_sync(false), + compaction_readahead_size(0), + max_background_flushes(-1) {} + +MutableDBOptions::MutableDBOptions(const DBOptions& options) + : max_background_jobs(options.max_background_jobs), + max_background_compactions(options.max_background_compactions), + max_subcompactions(options.max_subcompactions), + avoid_flush_during_shutdown(options.avoid_flush_during_shutdown), + writable_file_max_buffer_size(options.writable_file_max_buffer_size), + delayed_write_rate(options.delayed_write_rate), + max_total_wal_size(options.max_total_wal_size), + delete_obsolete_files_period_micros( + options.delete_obsolete_files_period_micros), + stats_dump_period_sec(options.stats_dump_period_sec), + stats_persist_period_sec(options.stats_persist_period_sec), + stats_history_buffer_size(options.stats_history_buffer_size), + max_open_files(options.max_open_files), + bytes_per_sync(options.bytes_per_sync), + wal_bytes_per_sync(options.wal_bytes_per_sync), + strict_bytes_per_sync(options.strict_bytes_per_sync), + compaction_readahead_size(options.compaction_readahead_size), + max_background_flushes(options.max_background_flushes) {} + +void MutableDBOptions::Dump(Logger* log) const { + ROCKS_LOG_HEADER(log, " Options.max_background_jobs: %d", + max_background_jobs); + ROCKS_LOG_HEADER(log, " Options.max_background_compactions: %d", + max_background_compactions); + ROCKS_LOG_HEADER(log, " Options.max_subcompactions: %" PRIu32, + max_subcompactions); + ROCKS_LOG_HEADER(log, " Options.avoid_flush_during_shutdown: %d", + avoid_flush_during_shutdown); + ROCKS_LOG_HEADER( + log, " Options.writable_file_max_buffer_size: %" ROCKSDB_PRIszt, + writable_file_max_buffer_size); + ROCKS_LOG_HEADER(log, " Options.delayed_write_rate : %" PRIu64, + delayed_write_rate); + ROCKS_LOG_HEADER(log, " Options.max_total_wal_size: %" PRIu64, + max_total_wal_size); + ROCKS_LOG_HEADER( + log, " Options.delete_obsolete_files_period_micros: %" PRIu64, + delete_obsolete_files_period_micros); + ROCKS_LOG_HEADER(log, " Options.stats_dump_period_sec: %u", + stats_dump_period_sec); + ROCKS_LOG_HEADER(log, " Options.stats_persist_period_sec: %d", + stats_persist_period_sec); + ROCKS_LOG_HEADER( + log, + " Options.stats_history_buffer_size: %" ROCKSDB_PRIszt, + stats_history_buffer_size); + ROCKS_LOG_HEADER(log, " Options.max_open_files: %d", + max_open_files); + ROCKS_LOG_HEADER(log, + " Options.bytes_per_sync: %" PRIu64, + bytes_per_sync); + ROCKS_LOG_HEADER(log, + " Options.wal_bytes_per_sync: %" PRIu64, + wal_bytes_per_sync); + ROCKS_LOG_HEADER(log, + " Options.strict_bytes_per_sync: %d", + strict_bytes_per_sync); + ROCKS_LOG_HEADER(log, + " Options.compaction_readahead_size: %" ROCKSDB_PRIszt, + compaction_readahead_size); + ROCKS_LOG_HEADER(log, " Options.max_background_flushes: %d", + max_background_flushes); +} + +Status GetMutableDBOptionsFromStrings( + const MutableDBOptions& base_options, + const std::unordered_map& options_map, + MutableDBOptions* new_options) { + assert(new_options); + *new_options = base_options; + ConfigOptions config_options; + Status s = OptionTypeInfo::ParseType( + config_options, options_map, db_mutable_options_type_info, new_options); + if (!s.ok()) { + *new_options = base_options; + } + return s; +} + +bool MutableDBOptionsAreEqual(const MutableDBOptions& this_options, + const MutableDBOptions& that_options) { + ConfigOptions config_options; + std::string mismatch; + return OptionTypeInfo::StructsAreEqual( + config_options, "MutableDBOptions", &db_mutable_options_type_info, + "MutableDBOptions", &this_options, &that_options, &mismatch); +} + +Status GetStringFromMutableDBOptions(const ConfigOptions& config_options, + const MutableDBOptions& mutable_opts, + std::string* opt_string) { + return OptionTypeInfo::SerializeType( + config_options, db_mutable_options_type_info, &mutable_opts, opt_string); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/options/db_options.h b/librocksdb-sys/rocksdb/options/db_options.h new file mode 100644 index 0000000..d00a067 --- /dev/null +++ b/librocksdb-sys/rocksdb/options/db_options.h @@ -0,0 +1,153 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include + +#include "rocksdb/options.h" + +namespace ROCKSDB_NAMESPACE { +class SystemClock; + +struct ImmutableDBOptions { + static const char* kName() { return "ImmutableDBOptions"; } + ImmutableDBOptions(); + explicit ImmutableDBOptions(const DBOptions& options); + + void Dump(Logger* log) const; + + bool create_if_missing; + bool create_missing_column_families; + bool error_if_exists; + bool paranoid_checks; + bool flush_verify_memtable_count; + bool compaction_verify_record_count; + bool track_and_verify_wals_in_manifest; + bool verify_sst_unique_id_in_manifest; + Env* env; + std::shared_ptr rate_limiter; + std::shared_ptr sst_file_manager; + std::shared_ptr info_log; + InfoLogLevel info_log_level; + int max_file_opening_threads; + std::shared_ptr statistics; + bool use_fsync; + std::vector db_paths; + std::string db_log_dir; + // The wal_dir option from the file. To determine the + // directory in use, the GetWalDir or IsWalDirSameAsDBPath + // methods should be used instead of accessing this variable directly. + std::string wal_dir; + size_t max_log_file_size; + size_t log_file_time_to_roll; + size_t keep_log_file_num; + size_t recycle_log_file_num; + uint64_t max_manifest_file_size; + int table_cache_numshardbits; + uint64_t WAL_ttl_seconds; + uint64_t WAL_size_limit_MB; + uint64_t max_write_batch_group_size_bytes; + size_t manifest_preallocation_size; + bool allow_mmap_reads; + bool allow_mmap_writes; + bool use_direct_reads; + bool use_direct_io_for_flush_and_compaction; + bool allow_fallocate; + bool is_fd_close_on_exec; + bool advise_random_on_open; + size_t db_write_buffer_size; + std::shared_ptr write_buffer_manager; + DBOptions::AccessHint access_hint_on_compaction_start; + size_t random_access_max_buffer_size; + bool use_adaptive_mutex; + std::vector> listeners; + bool enable_thread_tracking; + bool enable_pipelined_write; + bool unordered_write; + bool allow_concurrent_memtable_write; + bool enable_write_thread_adaptive_yield; + uint64_t write_thread_max_yield_usec; + uint64_t write_thread_slow_yield_usec; + bool skip_stats_update_on_db_open; + bool skip_checking_sst_file_sizes_on_db_open; + WALRecoveryMode wal_recovery_mode; + bool allow_2pc; + std::shared_ptr row_cache; + WalFilter* wal_filter; + bool fail_if_options_file_error; + bool dump_malloc_stats; + bool avoid_flush_during_recovery; + bool allow_ingest_behind; + bool two_write_queues; + bool manual_wal_flush; + CompressionType wal_compression; + bool atomic_flush; + bool avoid_unnecessary_blocking_io; + bool persist_stats_to_disk; + bool write_dbid_to_manifest; + size_t log_readahead_size; + std::shared_ptr file_checksum_gen_factory; + bool best_efforts_recovery; + int max_bgerror_resume_count; + uint64_t bgerror_resume_retry_interval; + bool allow_data_in_errors; + std::string db_host_id; + FileTypeSet checksum_handoff_file_types; + CacheTier lowest_used_cache_tier; + // Convenience/Helper objects that are not part of the base DBOptions + std::shared_ptr fs; + SystemClock* clock; + Statistics* stats; + Logger* logger; + std::shared_ptr compaction_service; + bool enforce_single_del_contracts; + + bool IsWalDirSameAsDBPath() const; + bool IsWalDirSameAsDBPath(const std::string& path) const; + const std::string& GetWalDir() const; + const std::string& GetWalDir(const std::string& path) const; +}; + +struct MutableDBOptions { + static const char* kName() { return "MutableDBOptions"; } + MutableDBOptions(); + explicit MutableDBOptions(const DBOptions& options); + + void Dump(Logger* log) const; + + int max_background_jobs; + int max_background_compactions; + uint32_t max_subcompactions; + bool avoid_flush_during_shutdown; + size_t writable_file_max_buffer_size; + uint64_t delayed_write_rate; + uint64_t max_total_wal_size; + uint64_t delete_obsolete_files_period_micros; + unsigned int stats_dump_period_sec; + unsigned int stats_persist_period_sec; + size_t stats_history_buffer_size; + int max_open_files; + uint64_t bytes_per_sync; + uint64_t wal_bytes_per_sync; + bool strict_bytes_per_sync; + size_t compaction_readahead_size; + int max_background_flushes; +}; + +Status GetStringFromMutableDBOptions(const ConfigOptions& config_options, + const MutableDBOptions& mutable_opts, + std::string* opt_string); + +Status GetMutableDBOptionsFromStrings( + const MutableDBOptions& base_options, + const std::unordered_map& options_map, + MutableDBOptions* new_options); + +bool MutableDBOptionsAreEqual(const MutableDBOptions& this_options, + const MutableDBOptions& that_options); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/options/options.cc b/librocksdb-sys/rocksdb/options/options.cc new file mode 100644 index 0000000..4e3ac41 --- /dev/null +++ b/librocksdb-sys/rocksdb/options/options.cc @@ -0,0 +1,694 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "rocksdb/options.h" + +#include +#include + +#include "logging/logging.h" +#include "monitoring/statistics_impl.h" +#include "options/db_options.h" +#include "options/options_helper.h" +#include "rocksdb/cache.h" +#include "rocksdb/compaction_filter.h" +#include "rocksdb/comparator.h" +#include "rocksdb/env.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/slice.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/sst_file_manager.h" +#include "rocksdb/sst_partitioner.h" +#include "rocksdb/table.h" +#include "rocksdb/table_properties.h" +#include "rocksdb/wal_filter.h" +#include "table/block_based/block_based_table_factory.h" +#include "util/compression.h" + +namespace ROCKSDB_NAMESPACE { + +AdvancedColumnFamilyOptions::AdvancedColumnFamilyOptions() { + assert(memtable_factory.get() != nullptr); +} + +AdvancedColumnFamilyOptions::AdvancedColumnFamilyOptions(const Options& options) + : max_write_buffer_number(options.max_write_buffer_number), + min_write_buffer_number_to_merge( + options.min_write_buffer_number_to_merge), + max_write_buffer_number_to_maintain( + options.max_write_buffer_number_to_maintain), + max_write_buffer_size_to_maintain( + options.max_write_buffer_size_to_maintain), + inplace_update_support(options.inplace_update_support), + inplace_update_num_locks(options.inplace_update_num_locks), + experimental_mempurge_threshold(options.experimental_mempurge_threshold), + inplace_callback(options.inplace_callback), + memtable_prefix_bloom_size_ratio( + options.memtable_prefix_bloom_size_ratio), + memtable_whole_key_filtering(options.memtable_whole_key_filtering), + memtable_huge_page_size(options.memtable_huge_page_size), + memtable_insert_with_hint_prefix_extractor( + options.memtable_insert_with_hint_prefix_extractor), + bloom_locality(options.bloom_locality), + arena_block_size(options.arena_block_size), + compression_per_level(options.compression_per_level), + num_levels(options.num_levels), + level0_slowdown_writes_trigger(options.level0_slowdown_writes_trigger), + level0_stop_writes_trigger(options.level0_stop_writes_trigger), + target_file_size_base(options.target_file_size_base), + target_file_size_multiplier(options.target_file_size_multiplier), + level_compaction_dynamic_level_bytes( + options.level_compaction_dynamic_level_bytes), + max_bytes_for_level_multiplier(options.max_bytes_for_level_multiplier), + max_bytes_for_level_multiplier_additional( + options.max_bytes_for_level_multiplier_additional), + max_compaction_bytes(options.max_compaction_bytes), + ignore_max_compaction_bytes_for_input( + options.ignore_max_compaction_bytes_for_input), + soft_pending_compaction_bytes_limit( + options.soft_pending_compaction_bytes_limit), + hard_pending_compaction_bytes_limit( + options.hard_pending_compaction_bytes_limit), + compaction_style(options.compaction_style), + compaction_pri(options.compaction_pri), + compaction_options_universal(options.compaction_options_universal), + compaction_options_fifo(options.compaction_options_fifo), + max_sequential_skip_in_iterations( + options.max_sequential_skip_in_iterations), + memtable_factory(options.memtable_factory), + table_properties_collector_factories( + options.table_properties_collector_factories), + max_successive_merges(options.max_successive_merges), + optimize_filters_for_hits(options.optimize_filters_for_hits), + paranoid_file_checks(options.paranoid_file_checks), + force_consistency_checks(options.force_consistency_checks), + report_bg_io_stats(options.report_bg_io_stats), + ttl(options.ttl), + periodic_compaction_seconds(options.periodic_compaction_seconds), + sample_for_compression(options.sample_for_compression), + preclude_last_level_data_seconds( + options.preclude_last_level_data_seconds), + preserve_internal_time_seconds(options.preserve_internal_time_seconds), + enable_blob_files(options.enable_blob_files), + min_blob_size(options.min_blob_size), + blob_file_size(options.blob_file_size), + blob_compression_type(options.blob_compression_type), + enable_blob_garbage_collection(options.enable_blob_garbage_collection), + blob_garbage_collection_age_cutoff( + options.blob_garbage_collection_age_cutoff), + blob_garbage_collection_force_threshold( + options.blob_garbage_collection_force_threshold), + blob_compaction_readahead_size(options.blob_compaction_readahead_size), + blob_file_starting_level(options.blob_file_starting_level), + blob_cache(options.blob_cache), + prepopulate_blob_cache(options.prepopulate_blob_cache), + persist_user_defined_timestamps(options.persist_user_defined_timestamps) { + assert(memtable_factory.get() != nullptr); + if (max_bytes_for_level_multiplier_additional.size() < + static_cast(num_levels)) { + max_bytes_for_level_multiplier_additional.resize(num_levels, 1); + } +} + +ColumnFamilyOptions::ColumnFamilyOptions() + : compression(Snappy_Supported() ? kSnappyCompression : kNoCompression), + table_factory( + std::shared_ptr(new BlockBasedTableFactory())) {} + +ColumnFamilyOptions::ColumnFamilyOptions(const Options& options) + : ColumnFamilyOptions(*static_cast(&options)) {} + +DBOptions::DBOptions() {} +DBOptions::DBOptions(const Options& options) + : DBOptions(*static_cast(&options)) {} + +void DBOptions::Dump(Logger* log) const { + ImmutableDBOptions(*this).Dump(log); + MutableDBOptions(*this).Dump(log); +} // DBOptions::Dump + +void ColumnFamilyOptions::Dump(Logger* log) const { + ROCKS_LOG_HEADER(log, " Options.comparator: %s", + comparator->Name()); + if (comparator->timestamp_size() > 0) { + ROCKS_LOG_HEADER( + log, " Options.persist_user_defined_timestamps: %s", + persist_user_defined_timestamps ? "true" : "false"); + } + ROCKS_LOG_HEADER(log, " Options.merge_operator: %s", + merge_operator ? merge_operator->Name() : "None"); + ROCKS_LOG_HEADER(log, " Options.compaction_filter: %s", + compaction_filter ? compaction_filter->Name() : "None"); + ROCKS_LOG_HEADER( + log, " Options.compaction_filter_factory: %s", + compaction_filter_factory ? compaction_filter_factory->Name() : "None"); + ROCKS_LOG_HEADER( + log, " Options.sst_partitioner_factory: %s", + sst_partitioner_factory ? sst_partitioner_factory->Name() : "None"); + ROCKS_LOG_HEADER(log, " Options.memtable_factory: %s", + memtable_factory->Name()); + ROCKS_LOG_HEADER(log, " Options.table_factory: %s", + table_factory->Name()); + ROCKS_LOG_HEADER(log, " table_factory options: %s", + table_factory->GetPrintableOptions().c_str()); + ROCKS_LOG_HEADER(log, " Options.write_buffer_size: %" ROCKSDB_PRIszt, + write_buffer_size); + ROCKS_LOG_HEADER(log, " Options.max_write_buffer_number: %d", + max_write_buffer_number); + if (!compression_per_level.empty()) { + for (unsigned int i = 0; i < compression_per_level.size(); i++) { + ROCKS_LOG_HEADER( + log, " Options.compression[%d]: %s", i, + CompressionTypeToString(compression_per_level[i]).c_str()); + } + } else { + ROCKS_LOG_HEADER(log, " Options.compression: %s", + CompressionTypeToString(compression).c_str()); + } + ROCKS_LOG_HEADER( + log, " Options.bottommost_compression: %s", + bottommost_compression == kDisableCompressionOption + ? "Disabled" + : CompressionTypeToString(bottommost_compression).c_str()); + ROCKS_LOG_HEADER( + log, " Options.prefix_extractor: %s", + prefix_extractor == nullptr ? "nullptr" : prefix_extractor->Name()); + ROCKS_LOG_HEADER(log, + " Options.memtable_insert_with_hint_prefix_extractor: %s", + memtable_insert_with_hint_prefix_extractor == nullptr + ? "nullptr" + : memtable_insert_with_hint_prefix_extractor->Name()); + ROCKS_LOG_HEADER(log, " Options.num_levels: %d", num_levels); + ROCKS_LOG_HEADER(log, " Options.min_write_buffer_number_to_merge: %d", + min_write_buffer_number_to_merge); + ROCKS_LOG_HEADER(log, " Options.max_write_buffer_number_to_maintain: %d", + max_write_buffer_number_to_maintain); + ROCKS_LOG_HEADER(log, + " Options.max_write_buffer_size_to_maintain: %" PRIu64, + max_write_buffer_size_to_maintain); + ROCKS_LOG_HEADER( + log, " Options.bottommost_compression_opts.window_bits: %d", + bottommost_compression_opts.window_bits); + ROCKS_LOG_HEADER( + log, " Options.bottommost_compression_opts.level: %d", + bottommost_compression_opts.level); + ROCKS_LOG_HEADER( + log, " Options.bottommost_compression_opts.strategy: %d", + bottommost_compression_opts.strategy); + ROCKS_LOG_HEADER( + log, + " Options.bottommost_compression_opts.max_dict_bytes: " + "%" PRIu32, + bottommost_compression_opts.max_dict_bytes); + ROCKS_LOG_HEADER( + log, + " Options.bottommost_compression_opts.zstd_max_train_bytes: " + "%" PRIu32, + bottommost_compression_opts.zstd_max_train_bytes); + ROCKS_LOG_HEADER( + log, + " Options.bottommost_compression_opts.parallel_threads: " + "%" PRIu32, + bottommost_compression_opts.parallel_threads); + ROCKS_LOG_HEADER( + log, " Options.bottommost_compression_opts.enabled: %s", + bottommost_compression_opts.enabled ? "true" : "false"); + ROCKS_LOG_HEADER( + log, + " Options.bottommost_compression_opts.max_dict_buffer_bytes: " + "%" PRIu64, + bottommost_compression_opts.max_dict_buffer_bytes); + ROCKS_LOG_HEADER( + log, + " Options.bottommost_compression_opts.use_zstd_dict_trainer: %s", + bottommost_compression_opts.use_zstd_dict_trainer ? "true" : "false"); + ROCKS_LOG_HEADER(log, " Options.compression_opts.window_bits: %d", + compression_opts.window_bits); + ROCKS_LOG_HEADER(log, " Options.compression_opts.level: %d", + compression_opts.level); + ROCKS_LOG_HEADER(log, " Options.compression_opts.strategy: %d", + compression_opts.strategy); + ROCKS_LOG_HEADER( + log, + " Options.compression_opts.max_dict_bytes: %" PRIu32, + compression_opts.max_dict_bytes); + ROCKS_LOG_HEADER(log, + " Options.compression_opts.zstd_max_train_bytes: " + "%" PRIu32, + compression_opts.zstd_max_train_bytes); + ROCKS_LOG_HEADER( + log, " Options.compression_opts.use_zstd_dict_trainer: %s", + compression_opts.use_zstd_dict_trainer ? "true" : "false"); + ROCKS_LOG_HEADER(log, + " Options.compression_opts.parallel_threads: " + "%" PRIu32, + compression_opts.parallel_threads); + ROCKS_LOG_HEADER(log, + " Options.compression_opts.enabled: %s", + compression_opts.enabled ? "true" : "false"); + ROCKS_LOG_HEADER(log, + " Options.compression_opts.max_dict_buffer_bytes: " + "%" PRIu64, + compression_opts.max_dict_buffer_bytes); + ROCKS_LOG_HEADER(log, " Options.level0_file_num_compaction_trigger: %d", + level0_file_num_compaction_trigger); + ROCKS_LOG_HEADER(log, " Options.level0_slowdown_writes_trigger: %d", + level0_slowdown_writes_trigger); + ROCKS_LOG_HEADER(log, " Options.level0_stop_writes_trigger: %d", + level0_stop_writes_trigger); + ROCKS_LOG_HEADER( + log, " Options.target_file_size_base: %" PRIu64, + target_file_size_base); + ROCKS_LOG_HEADER(log, " Options.target_file_size_multiplier: %d", + target_file_size_multiplier); + ROCKS_LOG_HEADER( + log, " Options.max_bytes_for_level_base: %" PRIu64, + max_bytes_for_level_base); + ROCKS_LOG_HEADER(log, "Options.level_compaction_dynamic_level_bytes: %d", + level_compaction_dynamic_level_bytes); + ROCKS_LOG_HEADER(log, " Options.max_bytes_for_level_multiplier: %f", + max_bytes_for_level_multiplier); + for (size_t i = 0; i < max_bytes_for_level_multiplier_additional.size(); + i++) { + ROCKS_LOG_HEADER( + log, "Options.max_bytes_for_level_multiplier_addtl[%" ROCKSDB_PRIszt + "]: %d", + i, max_bytes_for_level_multiplier_additional[i]); + } + ROCKS_LOG_HEADER( + log, " Options.max_sequential_skip_in_iterations: %" PRIu64, + max_sequential_skip_in_iterations); + ROCKS_LOG_HEADER( + log, " Options.max_compaction_bytes: %" PRIu64, + max_compaction_bytes); + ROCKS_LOG_HEADER(log, " Options.ignore_max_compaction_bytes_for_input: %s", + ignore_max_compaction_bytes_for_input ? "true" : "false"); + ROCKS_LOG_HEADER( + log, + " Options.arena_block_size: %" ROCKSDB_PRIszt, + arena_block_size); + ROCKS_LOG_HEADER(log, + " Options.soft_pending_compaction_bytes_limit: %" PRIu64, + soft_pending_compaction_bytes_limit); + ROCKS_LOG_HEADER(log, + " Options.hard_pending_compaction_bytes_limit: %" PRIu64, + hard_pending_compaction_bytes_limit); + ROCKS_LOG_HEADER(log, " Options.disable_auto_compactions: %d", + disable_auto_compactions); + + const auto& it_compaction_style = + compaction_style_to_string.find(compaction_style); + std::string str_compaction_style; + if (it_compaction_style == compaction_style_to_string.end()) { + assert(false); + str_compaction_style = "unknown_" + std::to_string(compaction_style); + } else { + str_compaction_style = it_compaction_style->second; + } + ROCKS_LOG_HEADER(log, + " Options.compaction_style: %s", + str_compaction_style.c_str()); + + const auto& it_compaction_pri = + compaction_pri_to_string.find(compaction_pri); + std::string str_compaction_pri; + if (it_compaction_pri == compaction_pri_to_string.end()) { + assert(false); + str_compaction_pri = "unknown_" + std::to_string(compaction_pri); + } else { + str_compaction_pri = it_compaction_pri->second; + } + ROCKS_LOG_HEADER(log, + " Options.compaction_pri: %s", + str_compaction_pri.c_str()); + ROCKS_LOG_HEADER(log, + "Options.compaction_options_universal.size_ratio: %u", + compaction_options_universal.size_ratio); + ROCKS_LOG_HEADER(log, + "Options.compaction_options_universal.min_merge_width: %u", + compaction_options_universal.min_merge_width); + ROCKS_LOG_HEADER(log, + "Options.compaction_options_universal.max_merge_width: %u", + compaction_options_universal.max_merge_width); + ROCKS_LOG_HEADER( + log, + "Options.compaction_options_universal." + "max_size_amplification_percent: %u", + compaction_options_universal.max_size_amplification_percent); + ROCKS_LOG_HEADER( + log, + "Options.compaction_options_universal.compression_size_percent: %d", + compaction_options_universal.compression_size_percent); + const auto& it_compaction_stop_style = compaction_stop_style_to_string.find( + compaction_options_universal.stop_style); + std::string str_compaction_stop_style; + if (it_compaction_stop_style == compaction_stop_style_to_string.end()) { + assert(false); + str_compaction_stop_style = + "unknown_" + std::to_string(compaction_options_universal.stop_style); + } else { + str_compaction_stop_style = it_compaction_stop_style->second; + } + ROCKS_LOG_HEADER(log, + "Options.compaction_options_universal.stop_style: %s", + str_compaction_stop_style.c_str()); + ROCKS_LOG_HEADER( + log, "Options.compaction_options_fifo.max_table_files_size: %" PRIu64, + compaction_options_fifo.max_table_files_size); + ROCKS_LOG_HEADER(log, + "Options.compaction_options_fifo.allow_compaction: %d", + compaction_options_fifo.allow_compaction); + std::ostringstream collector_info; + for (const auto& collector_factory : table_properties_collector_factories) { + collector_info << collector_factory->ToString() << ';'; + } + ROCKS_LOG_HEADER( + log, " Options.table_properties_collectors: %s", + collector_info.str().c_str()); + ROCKS_LOG_HEADER(log, + " Options.inplace_update_support: %d", + inplace_update_support); + ROCKS_LOG_HEADER( + log, + " Options.inplace_update_num_locks: %" ROCKSDB_PRIszt, + inplace_update_num_locks); + // TODO: easier config for bloom (maybe based on avg key/value size) + ROCKS_LOG_HEADER( + log, " Options.memtable_prefix_bloom_size_ratio: %f", + memtable_prefix_bloom_size_ratio); + ROCKS_LOG_HEADER(log, + " Options.memtable_whole_key_filtering: %d", + memtable_whole_key_filtering); + + ROCKS_LOG_HEADER(log, " Options.memtable_huge_page_size: %" ROCKSDB_PRIszt, + memtable_huge_page_size); + ROCKS_LOG_HEADER(log, + " Options.bloom_locality: %d", + bloom_locality); + + ROCKS_LOG_HEADER( + log, + " Options.max_successive_merges: %" ROCKSDB_PRIszt, + max_successive_merges); + ROCKS_LOG_HEADER(log, + " Options.optimize_filters_for_hits: %d", + optimize_filters_for_hits); + ROCKS_LOG_HEADER(log, " Options.paranoid_file_checks: %d", + paranoid_file_checks); + ROCKS_LOG_HEADER(log, " Options.force_consistency_checks: %d", + force_consistency_checks); + ROCKS_LOG_HEADER(log, " Options.report_bg_io_stats: %d", + report_bg_io_stats); + ROCKS_LOG_HEADER(log, " Options.ttl: %" PRIu64, + ttl); + ROCKS_LOG_HEADER(log, + " Options.periodic_compaction_seconds: %" PRIu64, + periodic_compaction_seconds); + ROCKS_LOG_HEADER(log, " Options.preclude_last_level_data_seconds: %" PRIu64, + preclude_last_level_data_seconds); + ROCKS_LOG_HEADER(log, " Options.preserve_internal_time_seconds: %" PRIu64, + preserve_internal_time_seconds); + ROCKS_LOG_HEADER(log, " Options.enable_blob_files: %s", + enable_blob_files ? "true" : "false"); + ROCKS_LOG_HEADER( + log, " Options.min_blob_size: %" PRIu64, + min_blob_size); + ROCKS_LOG_HEADER( + log, " Options.blob_file_size: %" PRIu64, + blob_file_size); + ROCKS_LOG_HEADER(log, " Options.blob_compression_type: %s", + CompressionTypeToString(blob_compression_type).c_str()); + ROCKS_LOG_HEADER(log, " Options.enable_blob_garbage_collection: %s", + enable_blob_garbage_collection ? "true" : "false"); + ROCKS_LOG_HEADER(log, " Options.blob_garbage_collection_age_cutoff: %f", + blob_garbage_collection_age_cutoff); + ROCKS_LOG_HEADER(log, "Options.blob_garbage_collection_force_threshold: %f", + blob_garbage_collection_force_threshold); + ROCKS_LOG_HEADER( + log, " Options.blob_compaction_readahead_size: %" PRIu64, + blob_compaction_readahead_size); + ROCKS_LOG_HEADER(log, " Options.blob_file_starting_level: %d", + blob_file_starting_level); + if (blob_cache) { + ROCKS_LOG_HEADER(log, " Options.blob_cache: %s", + blob_cache->Name()); + ROCKS_LOG_HEADER(log, " blob_cache options: %s", + blob_cache->GetPrintableOptions().c_str()); + ROCKS_LOG_HEADER( + log, " blob_cache prepopulated: %s", + prepopulate_blob_cache == PrepopulateBlobCache::kFlushOnly + ? "flush only" + : "disabled"); + } + ROCKS_LOG_HEADER(log, " Options.experimental_mempurge_threshold: %f", + experimental_mempurge_threshold); + ROCKS_LOG_HEADER(log, " Options.memtable_max_range_deletions: %d", + memtable_max_range_deletions); +} // ColumnFamilyOptions::Dump + +void Options::Dump(Logger* log) const { + DBOptions::Dump(log); + ColumnFamilyOptions::Dump(log); +} // Options::Dump + +void Options::DumpCFOptions(Logger* log) const { + ColumnFamilyOptions::Dump(log); +} // Options::DumpCFOptions + +// +// The goal of this method is to create a configuration that +// allows an application to write all files into L0 and +// then do a single compaction to output all files into L1. +Options* +Options::PrepareForBulkLoad() +{ + // never slowdown ingest. + level0_file_num_compaction_trigger = (1<<30); + level0_slowdown_writes_trigger = (1<<30); + level0_stop_writes_trigger = (1<<30); + soft_pending_compaction_bytes_limit = 0; + hard_pending_compaction_bytes_limit = 0; + + // no auto compactions please. The application should issue a + // manual compaction after all data is loaded into L0. + disable_auto_compactions = true; + // A manual compaction run should pick all files in L0 in + // a single compaction run. + max_compaction_bytes = (static_cast(1) << 60); + + // It is better to have only 2 levels, otherwise a manual + // compaction would compact at every possible level, thereby + // increasing the total time needed for compactions. + num_levels = 2; + + // Need to allow more write buffers to allow more parallism + // of flushes. + max_write_buffer_number = 6; + min_write_buffer_number_to_merge = 1; + + // When compaction is disabled, more parallel flush threads can + // help with write throughput. + max_background_flushes = 4; + + // Prevent a memtable flush to automatically promote files + // to L1. This is helpful so that all files that are + // input to the manual compaction are all at L0. + max_background_compactions = 2; + + // The compaction would create large files in L1. + target_file_size_base = 256 * 1024 * 1024; + return this; +} + +Options* Options::OptimizeForSmallDb() { + // 16MB block cache + std::shared_ptr cache = NewLRUCache(16 << 20); + + ColumnFamilyOptions::OptimizeForSmallDb(&cache); + DBOptions::OptimizeForSmallDb(&cache); + return this; +} + +Options* Options::DisableExtraChecks() { + // See https://github.com/facebook/rocksdb/issues/9354 + force_consistency_checks = false; + // Considered but no clear performance impact seen: + // * check_flush_compaction_key_order + // * paranoid_checks + // * flush_verify_memtable_count + // By current API contract, not including + // * verify_checksums + // because checking storage data integrity is a more standard practice. + return this; +} + +Options* Options::OldDefaults(int rocksdb_major_version, + int rocksdb_minor_version) { + ColumnFamilyOptions::OldDefaults(rocksdb_major_version, + rocksdb_minor_version); + DBOptions::OldDefaults(rocksdb_major_version, rocksdb_minor_version); + return this; +} + +DBOptions* DBOptions::OldDefaults(int rocksdb_major_version, + int rocksdb_minor_version) { + if (rocksdb_major_version < 4 || + (rocksdb_major_version == 4 && rocksdb_minor_version < 7)) { + max_file_opening_threads = 1; + table_cache_numshardbits = 4; + } + if (rocksdb_major_version < 5 || + (rocksdb_major_version == 5 && rocksdb_minor_version < 2)) { + delayed_write_rate = 2 * 1024U * 1024U; + } else if (rocksdb_major_version < 5 || + (rocksdb_major_version == 5 && rocksdb_minor_version < 6)) { + delayed_write_rate = 16 * 1024U * 1024U; + } + max_open_files = 5000; + wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords; + return this; +} + +ColumnFamilyOptions* ColumnFamilyOptions::OldDefaults( + int rocksdb_major_version, int rocksdb_minor_version) { + if (rocksdb_major_version < 5 || + (rocksdb_major_version == 5 && rocksdb_minor_version <= 18)) { + compaction_pri = CompactionPri::kByCompensatedSize; + } + if (rocksdb_major_version < 4 || + (rocksdb_major_version == 4 && rocksdb_minor_version < 7)) { + write_buffer_size = 4 << 20; + target_file_size_base = 2 * 1048576; + max_bytes_for_level_base = 10 * 1048576; + soft_pending_compaction_bytes_limit = 0; + hard_pending_compaction_bytes_limit = 0; + } + if (rocksdb_major_version < 5) { + level0_stop_writes_trigger = 24; + } else if (rocksdb_major_version == 5 && rocksdb_minor_version < 2) { + level0_stop_writes_trigger = 30; + } + + return this; +} + +// Optimization functions +DBOptions* DBOptions::OptimizeForSmallDb(std::shared_ptr* cache) { + max_file_opening_threads = 1; + max_open_files = 5000; + + // Cost memtable to block cache too. + std::shared_ptr wbm = + std::make_shared( + 0, (cache != nullptr) ? *cache : std::shared_ptr()); + write_buffer_manager = wbm; + + return this; +} + +ColumnFamilyOptions* ColumnFamilyOptions::OptimizeForSmallDb( + std::shared_ptr* cache) { + write_buffer_size = 2 << 20; + target_file_size_base = 2 * 1048576; + max_bytes_for_level_base = 10 * 1048576; + soft_pending_compaction_bytes_limit = 256 * 1048576; + hard_pending_compaction_bytes_limit = 1073741824ul; + + BlockBasedTableOptions table_options; + table_options.block_cache = + (cache != nullptr) ? *cache : std::shared_ptr(); + table_options.cache_index_and_filter_blocks = true; + // Two level iterator to avoid LRU cache imbalance + table_options.index_type = + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + table_factory.reset(new BlockBasedTableFactory(table_options)); + + return this; +} + +ColumnFamilyOptions* ColumnFamilyOptions::OptimizeForPointLookup( + uint64_t block_cache_size_mb) { + BlockBasedTableOptions block_based_options; + block_based_options.data_block_index_type = + BlockBasedTableOptions::kDataBlockBinaryAndHash; + block_based_options.data_block_hash_table_util_ratio = 0.75; + block_based_options.filter_policy.reset(NewBloomFilterPolicy(10)); + block_based_options.block_cache = + NewLRUCache(static_cast(block_cache_size_mb * 1024 * 1024)); + table_factory.reset(new BlockBasedTableFactory(block_based_options)); + memtable_prefix_bloom_size_ratio = 0.02; + memtable_whole_key_filtering = true; + return this; +} + +ColumnFamilyOptions* ColumnFamilyOptions::OptimizeLevelStyleCompaction( + uint64_t memtable_memory_budget) { + write_buffer_size = static_cast(memtable_memory_budget / 4); + // merge two memtables when flushing to L0 + min_write_buffer_number_to_merge = 2; + // this means we'll use 50% extra memory in the worst case, but will reduce + // write stalls. + max_write_buffer_number = 6; + // start flushing L0->L1 as soon as possible. each file on level0 is + // (memtable_memory_budget / 2). This will flush level 0 when it's bigger than + // memtable_memory_budget. + level0_file_num_compaction_trigger = 2; + // doesn't really matter much, but we don't want to create too many files + target_file_size_base = memtable_memory_budget / 8; + // make Level1 size equal to Level0 size, so that L0->L1 compactions are fast + max_bytes_for_level_base = memtable_memory_budget; + + // level style compaction + compaction_style = kCompactionStyleLevel; + + // only compress levels >= 2 + compression_per_level.resize(num_levels); + for (int i = 0; i < num_levels; ++i) { + if (i < 2) { + compression_per_level[i] = kNoCompression; + } else { + compression_per_level[i] = + LZ4_Supported() + ? kLZ4Compression + : (Snappy_Supported() ? kSnappyCompression : kNoCompression); + } + } + return this; +} + +ColumnFamilyOptions* ColumnFamilyOptions::OptimizeUniversalStyleCompaction( + uint64_t memtable_memory_budget) { + write_buffer_size = static_cast(memtable_memory_budget / 4); + // merge two memtables when flushing to L0 + min_write_buffer_number_to_merge = 2; + // this means we'll use 50% extra memory in the worst case, but will reduce + // write stalls. + max_write_buffer_number = 6; + // universal style compaction + compaction_style = kCompactionStyleUniversal; + compaction_options_universal.compression_size_percent = 80; + return this; +} + +DBOptions* DBOptions::IncreaseParallelism(int total_threads) { + max_background_jobs = total_threads; + env->SetBackgroundThreads(total_threads, Env::LOW); + env->SetBackgroundThreads(1, Env::HIGH); + return this; +} + +ReadOptions::ReadOptions(bool _verify_checksums, bool _fill_cache) + : verify_checksums(_verify_checksums), fill_cache(_fill_cache) {} + +ReadOptions::ReadOptions(Env::IOActivity _io_activity) + : io_activity(_io_activity) {} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/options/options_helper.cc b/librocksdb-sys/rocksdb/options/options_helper.cc new file mode 100644 index 0000000..83b4e97 --- /dev/null +++ b/librocksdb-sys/rocksdb/options/options_helper.cc @@ -0,0 +1,1427 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#include "options/options_helper.h" + +#include +#include +#include +#include +#include +#include + +#include "options/cf_options.h" +#include "options/db_options.h" +#include "rocksdb/cache.h" +#include "rocksdb/compaction_filter.h" +#include "rocksdb/convenience.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/flush_block_policy.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/options.h" +#include "rocksdb/rate_limiter.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/table.h" +#include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/options_type.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { +ConfigOptions::ConfigOptions() + : registry(ObjectRegistry::NewInstance()) +{ + env = Env::Default(); +} + +ConfigOptions::ConfigOptions(const DBOptions& db_opts) : env(db_opts.env) { + registry = ObjectRegistry::NewInstance(); +} + +Status ValidateOptions(const DBOptions& db_opts, + const ColumnFamilyOptions& cf_opts) { + Status s; + auto db_cfg = DBOptionsAsConfigurable(db_opts); + auto cf_cfg = CFOptionsAsConfigurable(cf_opts); + s = db_cfg->ValidateOptions(db_opts, cf_opts); + if (s.ok()) s = cf_cfg->ValidateOptions(db_opts, cf_opts); + return s; +} + +DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options, + const MutableDBOptions& mutable_db_options) { + DBOptions options; + + options.create_if_missing = immutable_db_options.create_if_missing; + options.create_missing_column_families = + immutable_db_options.create_missing_column_families; + options.error_if_exists = immutable_db_options.error_if_exists; + options.paranoid_checks = immutable_db_options.paranoid_checks; + options.flush_verify_memtable_count = + immutable_db_options.flush_verify_memtable_count; + options.compaction_verify_record_count = + immutable_db_options.compaction_verify_record_count; + options.track_and_verify_wals_in_manifest = + immutable_db_options.track_and_verify_wals_in_manifest; + options.verify_sst_unique_id_in_manifest = + immutable_db_options.verify_sst_unique_id_in_manifest; + options.env = immutable_db_options.env; + options.rate_limiter = immutable_db_options.rate_limiter; + options.sst_file_manager = immutable_db_options.sst_file_manager; + options.info_log = immutable_db_options.info_log; + options.info_log_level = immutable_db_options.info_log_level; + options.max_open_files = mutable_db_options.max_open_files; + options.max_file_opening_threads = + immutable_db_options.max_file_opening_threads; + options.max_total_wal_size = mutable_db_options.max_total_wal_size; + options.statistics = immutable_db_options.statistics; + options.use_fsync = immutable_db_options.use_fsync; + options.db_paths = immutable_db_options.db_paths; + options.db_log_dir = immutable_db_options.db_log_dir; + options.wal_dir = immutable_db_options.wal_dir; + options.delete_obsolete_files_period_micros = + mutable_db_options.delete_obsolete_files_period_micros; + options.max_background_jobs = mutable_db_options.max_background_jobs; + options.max_background_compactions = + mutable_db_options.max_background_compactions; + options.bytes_per_sync = mutable_db_options.bytes_per_sync; + options.wal_bytes_per_sync = mutable_db_options.wal_bytes_per_sync; + options.strict_bytes_per_sync = mutable_db_options.strict_bytes_per_sync; + options.max_subcompactions = mutable_db_options.max_subcompactions; + options.max_background_flushes = mutable_db_options.max_background_flushes; + options.max_log_file_size = immutable_db_options.max_log_file_size; + options.log_file_time_to_roll = immutable_db_options.log_file_time_to_roll; + options.keep_log_file_num = immutable_db_options.keep_log_file_num; + options.recycle_log_file_num = immutable_db_options.recycle_log_file_num; + options.max_manifest_file_size = immutable_db_options.max_manifest_file_size; + options.table_cache_numshardbits = + immutable_db_options.table_cache_numshardbits; + options.WAL_ttl_seconds = immutable_db_options.WAL_ttl_seconds; + options.WAL_size_limit_MB = immutable_db_options.WAL_size_limit_MB; + options.manifest_preallocation_size = + immutable_db_options.manifest_preallocation_size; + options.allow_mmap_reads = immutable_db_options.allow_mmap_reads; + options.allow_mmap_writes = immutable_db_options.allow_mmap_writes; + options.use_direct_reads = immutable_db_options.use_direct_reads; + options.use_direct_io_for_flush_and_compaction = + immutable_db_options.use_direct_io_for_flush_and_compaction; + options.allow_fallocate = immutable_db_options.allow_fallocate; + options.is_fd_close_on_exec = immutable_db_options.is_fd_close_on_exec; + options.stats_dump_period_sec = mutable_db_options.stats_dump_period_sec; + options.stats_persist_period_sec = + mutable_db_options.stats_persist_period_sec; + options.persist_stats_to_disk = immutable_db_options.persist_stats_to_disk; + options.stats_history_buffer_size = + mutable_db_options.stats_history_buffer_size; + options.advise_random_on_open = immutable_db_options.advise_random_on_open; + options.db_write_buffer_size = immutable_db_options.db_write_buffer_size; + options.write_buffer_manager = immutable_db_options.write_buffer_manager; + options.access_hint_on_compaction_start = + immutable_db_options.access_hint_on_compaction_start; + options.compaction_readahead_size = + mutable_db_options.compaction_readahead_size; + options.random_access_max_buffer_size = + immutable_db_options.random_access_max_buffer_size; + options.writable_file_max_buffer_size = + mutable_db_options.writable_file_max_buffer_size; + options.use_adaptive_mutex = immutable_db_options.use_adaptive_mutex; + options.listeners = immutable_db_options.listeners; + options.enable_thread_tracking = immutable_db_options.enable_thread_tracking; + options.delayed_write_rate = mutable_db_options.delayed_write_rate; + options.enable_pipelined_write = immutable_db_options.enable_pipelined_write; + options.unordered_write = immutable_db_options.unordered_write; + options.allow_concurrent_memtable_write = + immutable_db_options.allow_concurrent_memtable_write; + options.enable_write_thread_adaptive_yield = + immutable_db_options.enable_write_thread_adaptive_yield; + options.max_write_batch_group_size_bytes = + immutable_db_options.max_write_batch_group_size_bytes; + options.write_thread_max_yield_usec = + immutable_db_options.write_thread_max_yield_usec; + options.write_thread_slow_yield_usec = + immutable_db_options.write_thread_slow_yield_usec; + options.skip_stats_update_on_db_open = + immutable_db_options.skip_stats_update_on_db_open; + options.skip_checking_sst_file_sizes_on_db_open = + immutable_db_options.skip_checking_sst_file_sizes_on_db_open; + options.wal_recovery_mode = immutable_db_options.wal_recovery_mode; + options.allow_2pc = immutable_db_options.allow_2pc; + options.row_cache = immutable_db_options.row_cache; + options.wal_filter = immutable_db_options.wal_filter; + options.fail_if_options_file_error = + immutable_db_options.fail_if_options_file_error; + options.dump_malloc_stats = immutable_db_options.dump_malloc_stats; + options.avoid_flush_during_recovery = + immutable_db_options.avoid_flush_during_recovery; + options.avoid_flush_during_shutdown = + mutable_db_options.avoid_flush_during_shutdown; + options.allow_ingest_behind = immutable_db_options.allow_ingest_behind; + options.two_write_queues = immutable_db_options.two_write_queues; + options.manual_wal_flush = immutable_db_options.manual_wal_flush; + options.wal_compression = immutable_db_options.wal_compression; + options.atomic_flush = immutable_db_options.atomic_flush; + options.avoid_unnecessary_blocking_io = + immutable_db_options.avoid_unnecessary_blocking_io; + options.log_readahead_size = immutable_db_options.log_readahead_size; + options.file_checksum_gen_factory = + immutable_db_options.file_checksum_gen_factory; + options.best_efforts_recovery = immutable_db_options.best_efforts_recovery; + options.max_bgerror_resume_count = + immutable_db_options.max_bgerror_resume_count; + options.bgerror_resume_retry_interval = + immutable_db_options.bgerror_resume_retry_interval; + options.db_host_id = immutable_db_options.db_host_id; + options.allow_data_in_errors = immutable_db_options.allow_data_in_errors; + options.checksum_handoff_file_types = + immutable_db_options.checksum_handoff_file_types; + options.lowest_used_cache_tier = immutable_db_options.lowest_used_cache_tier; + options.enforce_single_del_contracts = + immutable_db_options.enforce_single_del_contracts; + return options; +} + +ColumnFamilyOptions BuildColumnFamilyOptions( + const ColumnFamilyOptions& options, + const MutableCFOptions& mutable_cf_options) { + ColumnFamilyOptions cf_opts(options); + UpdateColumnFamilyOptions(mutable_cf_options, &cf_opts); + // TODO(yhchiang): find some way to handle the following derived options + // * max_file_size + return cf_opts; +} + +void UpdateColumnFamilyOptions(const MutableCFOptions& moptions, + ColumnFamilyOptions* cf_opts) { + // Memtable related options + cf_opts->write_buffer_size = moptions.write_buffer_size; + cf_opts->max_write_buffer_number = moptions.max_write_buffer_number; + cf_opts->arena_block_size = moptions.arena_block_size; + cf_opts->memtable_prefix_bloom_size_ratio = + moptions.memtable_prefix_bloom_size_ratio; + cf_opts->memtable_whole_key_filtering = moptions.memtable_whole_key_filtering; + cf_opts->memtable_huge_page_size = moptions.memtable_huge_page_size; + cf_opts->max_successive_merges = moptions.max_successive_merges; + cf_opts->inplace_update_num_locks = moptions.inplace_update_num_locks; + cf_opts->prefix_extractor = moptions.prefix_extractor; + cf_opts->experimental_mempurge_threshold = + moptions.experimental_mempurge_threshold; + cf_opts->memtable_protection_bytes_per_key = + moptions.memtable_protection_bytes_per_key; + cf_opts->block_protection_bytes_per_key = + moptions.block_protection_bytes_per_key; + + // Compaction related options + cf_opts->disable_auto_compactions = moptions.disable_auto_compactions; + cf_opts->soft_pending_compaction_bytes_limit = + moptions.soft_pending_compaction_bytes_limit; + cf_opts->hard_pending_compaction_bytes_limit = + moptions.hard_pending_compaction_bytes_limit; + cf_opts->level0_file_num_compaction_trigger = + moptions.level0_file_num_compaction_trigger; + cf_opts->level0_slowdown_writes_trigger = + moptions.level0_slowdown_writes_trigger; + cf_opts->level0_stop_writes_trigger = moptions.level0_stop_writes_trigger; + cf_opts->max_compaction_bytes = moptions.max_compaction_bytes; + cf_opts->ignore_max_compaction_bytes_for_input = + moptions.ignore_max_compaction_bytes_for_input; + cf_opts->target_file_size_base = moptions.target_file_size_base; + cf_opts->target_file_size_multiplier = moptions.target_file_size_multiplier; + cf_opts->max_bytes_for_level_base = moptions.max_bytes_for_level_base; + cf_opts->max_bytes_for_level_multiplier = + moptions.max_bytes_for_level_multiplier; + cf_opts->ttl = moptions.ttl; + cf_opts->periodic_compaction_seconds = moptions.periodic_compaction_seconds; + + cf_opts->max_bytes_for_level_multiplier_additional.clear(); + for (auto value : moptions.max_bytes_for_level_multiplier_additional) { + cf_opts->max_bytes_for_level_multiplier_additional.emplace_back(value); + } + + cf_opts->compaction_options_fifo = moptions.compaction_options_fifo; + cf_opts->compaction_options_universal = moptions.compaction_options_universal; + + // Blob file related options + cf_opts->enable_blob_files = moptions.enable_blob_files; + cf_opts->min_blob_size = moptions.min_blob_size; + cf_opts->blob_file_size = moptions.blob_file_size; + cf_opts->blob_compression_type = moptions.blob_compression_type; + cf_opts->enable_blob_garbage_collection = + moptions.enable_blob_garbage_collection; + cf_opts->blob_garbage_collection_age_cutoff = + moptions.blob_garbage_collection_age_cutoff; + cf_opts->blob_garbage_collection_force_threshold = + moptions.blob_garbage_collection_force_threshold; + cf_opts->blob_compaction_readahead_size = + moptions.blob_compaction_readahead_size; + cf_opts->blob_file_starting_level = moptions.blob_file_starting_level; + cf_opts->prepopulate_blob_cache = moptions.prepopulate_blob_cache; + + // Misc options + cf_opts->max_sequential_skip_in_iterations = + moptions.max_sequential_skip_in_iterations; + cf_opts->check_flush_compaction_key_order = + moptions.check_flush_compaction_key_order; + cf_opts->paranoid_file_checks = moptions.paranoid_file_checks; + cf_opts->report_bg_io_stats = moptions.report_bg_io_stats; + cf_opts->compression = moptions.compression; + cf_opts->compression_opts = moptions.compression_opts; + cf_opts->bottommost_compression = moptions.bottommost_compression; + cf_opts->bottommost_compression_opts = moptions.bottommost_compression_opts; + cf_opts->sample_for_compression = moptions.sample_for_compression; + cf_opts->compression_per_level = moptions.compression_per_level; + cf_opts->last_level_temperature = moptions.last_level_temperature; + cf_opts->bottommost_temperature = moptions.last_level_temperature; + cf_opts->memtable_max_range_deletions = moptions.memtable_max_range_deletions; +} + +void UpdateColumnFamilyOptions(const ImmutableCFOptions& ioptions, + ColumnFamilyOptions* cf_opts) { + cf_opts->compaction_style = ioptions.compaction_style; + cf_opts->compaction_pri = ioptions.compaction_pri; + cf_opts->comparator = ioptions.user_comparator; + cf_opts->merge_operator = ioptions.merge_operator; + cf_opts->compaction_filter = ioptions.compaction_filter; + cf_opts->compaction_filter_factory = ioptions.compaction_filter_factory; + cf_opts->min_write_buffer_number_to_merge = + ioptions.min_write_buffer_number_to_merge; + cf_opts->max_write_buffer_number_to_maintain = + ioptions.max_write_buffer_number_to_maintain; + cf_opts->max_write_buffer_size_to_maintain = + ioptions.max_write_buffer_size_to_maintain; + cf_opts->inplace_update_support = ioptions.inplace_update_support; + cf_opts->inplace_callback = ioptions.inplace_callback; + cf_opts->memtable_factory = ioptions.memtable_factory; + cf_opts->table_factory = ioptions.table_factory; + cf_opts->table_properties_collector_factories = + ioptions.table_properties_collector_factories; + cf_opts->bloom_locality = ioptions.bloom_locality; + cf_opts->level_compaction_dynamic_level_bytes = + ioptions.level_compaction_dynamic_level_bytes; + cf_opts->level_compaction_dynamic_file_size = + ioptions.level_compaction_dynamic_file_size; + cf_opts->num_levels = ioptions.num_levels; + cf_opts->optimize_filters_for_hits = ioptions.optimize_filters_for_hits; + cf_opts->force_consistency_checks = ioptions.force_consistency_checks; + cf_opts->memtable_insert_with_hint_prefix_extractor = + ioptions.memtable_insert_with_hint_prefix_extractor; + cf_opts->cf_paths = ioptions.cf_paths; + cf_opts->compaction_thread_limiter = ioptions.compaction_thread_limiter; + cf_opts->sst_partitioner_factory = ioptions.sst_partitioner_factory; + cf_opts->blob_cache = ioptions.blob_cache; + cf_opts->preclude_last_level_data_seconds = + ioptions.preclude_last_level_data_seconds; + cf_opts->preserve_internal_time_seconds = + ioptions.preserve_internal_time_seconds; + cf_opts->persist_user_defined_timestamps = + ioptions.persist_user_defined_timestamps; + + // TODO(yhchiang): find some way to handle the following derived options + // * max_file_size +} + +std::map + OptionsHelper::compaction_style_to_string = { + {kCompactionStyleLevel, "kCompactionStyleLevel"}, + {kCompactionStyleUniversal, "kCompactionStyleUniversal"}, + {kCompactionStyleFIFO, "kCompactionStyleFIFO"}, + {kCompactionStyleNone, "kCompactionStyleNone"}}; + +std::map OptionsHelper::compaction_pri_to_string = { + {kByCompensatedSize, "kByCompensatedSize"}, + {kOldestLargestSeqFirst, "kOldestLargestSeqFirst"}, + {kOldestSmallestSeqFirst, "kOldestSmallestSeqFirst"}, + {kMinOverlappingRatio, "kMinOverlappingRatio"}, + {kRoundRobin, "kRoundRobin"}}; + +std::map + OptionsHelper::compaction_stop_style_to_string = { + {kCompactionStopStyleSimilarSize, "kCompactionStopStyleSimilarSize"}, + {kCompactionStopStyleTotalSize, "kCompactionStopStyleTotalSize"}}; + +std::map OptionsHelper::temperature_to_string = { + {Temperature::kUnknown, "kUnknown"}, + {Temperature::kHot, "kHot"}, + {Temperature::kWarm, "kWarm"}, + {Temperature::kCold, "kCold"}}; + +std::unordered_map + OptionsHelper::checksum_type_string_map = {{"kNoChecksum", kNoChecksum}, + {"kCRC32c", kCRC32c}, + {"kxxHash", kxxHash}, + {"kxxHash64", kxxHash64}, + {"kXXH3", kXXH3}}; + +std::unordered_map + OptionsHelper::compression_type_string_map = { + {"kNoCompression", kNoCompression}, + {"kSnappyCompression", kSnappyCompression}, + {"kZlibCompression", kZlibCompression}, + {"kBZip2Compression", kBZip2Compression}, + {"kLZ4Compression", kLZ4Compression}, + {"kLZ4HCCompression", kLZ4HCCompression}, + {"kXpressCompression", kXpressCompression}, + {"kZSTD", kZSTD}, + {"kZSTDNotFinalCompression", kZSTDNotFinalCompression}, + {"kDisableCompressionOption", kDisableCompressionOption}}; + +std::vector GetSupportedCompressions() { + // std::set internally to deduplicate potential name aliases + std::set supported_compressions; + for (const auto& comp_to_name : OptionsHelper::compression_type_string_map) { + CompressionType t = comp_to_name.second; + if (t != kDisableCompressionOption && CompressionTypeSupported(t)) { + supported_compressions.insert(t); + } + } + return std::vector(supported_compressions.begin(), + supported_compressions.end()); +} + +std::vector GetSupportedDictCompressions() { + std::set dict_compression_types; + for (const auto& comp_to_name : OptionsHelper::compression_type_string_map) { + CompressionType t = comp_to_name.second; + if (t != kDisableCompressionOption && DictCompressionTypeSupported(t)) { + dict_compression_types.insert(t); + } + } + return std::vector(dict_compression_types.begin(), + dict_compression_types.end()); +} + +std::vector GetSupportedChecksums() { + std::set checksum_types; + for (const auto& e : OptionsHelper::checksum_type_string_map) { + checksum_types.insert(e.second); + } + return std::vector(checksum_types.begin(), + checksum_types.end()); +} + +static bool ParseOptionHelper(void* opt_address, const OptionType& opt_type, + const std::string& value) { + switch (opt_type) { + case OptionType::kBoolean: + *static_cast(opt_address) = ParseBoolean("", value); + break; + case OptionType::kInt: + *static_cast(opt_address) = ParseInt(value); + break; + case OptionType::kInt32T: + *static_cast(opt_address) = ParseInt32(value); + break; + case OptionType::kInt64T: + PutUnaligned(static_cast(opt_address), ParseInt64(value)); + break; + case OptionType::kUInt: + *static_cast(opt_address) = ParseUint32(value); + break; + case OptionType::kUInt8T: + *static_cast(opt_address) = ParseUint8(value); + break; + case OptionType::kUInt32T: + *static_cast(opt_address) = ParseUint32(value); + break; + case OptionType::kUInt64T: + PutUnaligned(static_cast(opt_address), ParseUint64(value)); + break; + case OptionType::kSizeT: + PutUnaligned(static_cast(opt_address), ParseSizeT(value)); + break; + case OptionType::kString: + *static_cast(opt_address) = value; + break; + case OptionType::kDouble: + *static_cast(opt_address) = ParseDouble(value); + break; + case OptionType::kCompactionStyle: + return ParseEnum( + compaction_style_string_map, value, + static_cast(opt_address)); + case OptionType::kCompactionPri: + return ParseEnum(compaction_pri_string_map, value, + static_cast(opt_address)); + case OptionType::kCompressionType: + return ParseEnum( + compression_type_string_map, value, + static_cast(opt_address)); + case OptionType::kChecksumType: + return ParseEnum(checksum_type_string_map, value, + static_cast(opt_address)); + case OptionType::kEncodingType: + return ParseEnum(encoding_type_string_map, value, + static_cast(opt_address)); + case OptionType::kCompactionStopStyle: + return ParseEnum( + compaction_stop_style_string_map, value, + static_cast(opt_address)); + case OptionType::kEncodedString: { + std::string* output_addr = static_cast(opt_address); + (Slice(value)).DecodeHex(output_addr); + break; + } + case OptionType::kTemperature: { + return ParseEnum(temperature_string_map, value, + static_cast(opt_address)); + } + default: + return false; + } + return true; +} + +bool SerializeSingleOptionHelper(const void* opt_address, + const OptionType opt_type, + std::string* value) { + assert(value); + switch (opt_type) { + case OptionType::kBoolean: + *value = *(static_cast(opt_address)) ? "true" : "false"; + break; + case OptionType::kInt: + *value = std::to_string(*(static_cast(opt_address))); + break; + case OptionType::kInt32T: + *value = std::to_string(*(static_cast(opt_address))); + break; + case OptionType::kInt64T: + { + int64_t v; + GetUnaligned(static_cast(opt_address), &v); + *value = std::to_string(v); + } + break; + case OptionType::kUInt: + *value = std::to_string(*(static_cast(opt_address))); + break; + case OptionType::kUInt8T: + *value = std::to_string(*(static_cast(opt_address))); + break; + case OptionType::kUInt32T: + *value = std::to_string(*(static_cast(opt_address))); + break; + case OptionType::kUInt64T: + { + uint64_t v; + GetUnaligned(static_cast(opt_address), &v); + *value = std::to_string(v); + } + break; + case OptionType::kSizeT: + { + size_t v; + GetUnaligned(static_cast(opt_address), &v); + *value = std::to_string(v); + } + break; + case OptionType::kDouble: + *value = std::to_string(*(static_cast(opt_address))); + break; + case OptionType::kString: + *value = + EscapeOptionString(*(static_cast(opt_address))); + break; + case OptionType::kCompactionStyle: + return SerializeEnum( + compaction_style_string_map, + *(static_cast(opt_address)), value); + case OptionType::kCompactionPri: + return SerializeEnum( + compaction_pri_string_map, + *(static_cast(opt_address)), value); + case OptionType::kCompressionType: + return SerializeEnum( + compression_type_string_map, + *(static_cast(opt_address)), value); + break; + case OptionType::kChecksumType: + return SerializeEnum( + checksum_type_string_map, + *static_cast(opt_address), value); + case OptionType::kEncodingType: + return SerializeEnum( + encoding_type_string_map, + *static_cast(opt_address), value); + case OptionType::kCompactionStopStyle: + return SerializeEnum( + compaction_stop_style_string_map, + *static_cast(opt_address), value); + case OptionType::kEncodedString: { + const auto* ptr = static_cast(opt_address); + *value = (Slice(*ptr)).ToString(true); + break; + } + case OptionType::kTemperature: { + return SerializeEnum( + temperature_string_map, *static_cast(opt_address), + value); + } + default: + return false; + } + return true; +} + +template +Status ConfigureFromMap( + const ConfigOptions& config_options, + const std::unordered_map& opt_map, + const std::string& option_name, Configurable* config, T* new_opts) { + Status s = config->ConfigureFromMap(config_options, opt_map); + if (s.ok()) { + *new_opts = *(config->GetOptions(option_name)); + } + return s; +} + + +Status StringToMap(const std::string& opts_str, + std::unordered_map* opts_map) { + assert(opts_map); + // Example: + // opts_str = "write_buffer_size=1024;max_write_buffer_number=2;" + // "nested_opt={opt1=1;opt2=2};max_bytes_for_level_base=100" + size_t pos = 0; + std::string opts = trim(opts_str); + // If the input string starts and ends with "{...}", strip off the brackets + while (opts.size() > 2 && opts[0] == '{' && opts[opts.size() - 1] == '}') { + opts = trim(opts.substr(1, opts.size() - 2)); + } + + while (pos < opts.size()) { + size_t eq_pos = opts.find_first_of("={};", pos); + if (eq_pos == std::string::npos) { + return Status::InvalidArgument("Mismatched key value pair, '=' expected"); + } else if (opts[eq_pos] != '=') { + return Status::InvalidArgument("Unexpected char in key"); + } + + std::string key = trim(opts.substr(pos, eq_pos - pos)); + if (key.empty()) { + return Status::InvalidArgument("Empty key found"); + } + + std::string value; + Status s = OptionTypeInfo::NextToken(opts, ';', eq_pos + 1, &pos, &value); + if (!s.ok()) { + return s; + } else { + (*opts_map)[key] = value; + if (pos == std::string::npos) { + break; + } else { + pos++; + } + } + } + + return Status::OK(); +} + + +Status GetStringFromDBOptions(std::string* opt_string, + const DBOptions& db_options, + const std::string& delimiter) { + ConfigOptions config_options(db_options); + config_options.delimiter = delimiter; + return GetStringFromDBOptions(config_options, db_options, opt_string); +} + +Status GetStringFromDBOptions(const ConfigOptions& config_options, + const DBOptions& db_options, + std::string* opt_string) { + assert(opt_string); + opt_string->clear(); + auto config = DBOptionsAsConfigurable(db_options); + return config->GetOptionString(config_options, opt_string); +} + + +Status GetStringFromColumnFamilyOptions(std::string* opt_string, + const ColumnFamilyOptions& cf_options, + const std::string& delimiter) { + ConfigOptions config_options; + config_options.delimiter = delimiter; + return GetStringFromColumnFamilyOptions(config_options, cf_options, + opt_string); +} + +Status GetStringFromColumnFamilyOptions(const ConfigOptions& config_options, + const ColumnFamilyOptions& cf_options, + std::string* opt_string) { + const auto config = CFOptionsAsConfigurable(cf_options); + return config->GetOptionString(config_options, opt_string); +} + +Status GetStringFromCompressionType(std::string* compression_str, + CompressionType compression_type) { + bool ok = SerializeEnum(compression_type_string_map, + compression_type, compression_str); + if (ok) { + return Status::OK(); + } else { + return Status::InvalidArgument("Invalid compression types"); + } +} + +Status GetColumnFamilyOptionsFromMap( + const ConfigOptions& config_options, + const ColumnFamilyOptions& base_options, + const std::unordered_map& opts_map, + ColumnFamilyOptions* new_options) { + assert(new_options); + + *new_options = base_options; + + const auto config = CFOptionsAsConfigurable(base_options); + Status s = ConfigureFromMap( + config_options, opts_map, OptionsHelper::kCFOptionsName, config.get(), + new_options); + // Translate any errors (NotFound, NotSupported, to InvalidArgument + if (s.ok() || s.IsInvalidArgument()) { + return s; + } else { + return Status::InvalidArgument(s.getState()); + } +} + +Status GetColumnFamilyOptionsFromString(const ConfigOptions& config_options, + const ColumnFamilyOptions& base_options, + const std::string& opts_str, + ColumnFamilyOptions* new_options) { + std::unordered_map opts_map; + Status s = StringToMap(opts_str, &opts_map); + if (!s.ok()) { + *new_options = base_options; + return s; + } + return GetColumnFamilyOptionsFromMap(config_options, base_options, opts_map, + new_options); +} + +Status GetDBOptionsFromMap( + const ConfigOptions& config_options, const DBOptions& base_options, + const std::unordered_map& opts_map, + DBOptions* new_options) { + assert(new_options); + *new_options = base_options; + auto config = DBOptionsAsConfigurable(base_options); + Status s = ConfigureFromMap(config_options, opts_map, + OptionsHelper::kDBOptionsName, + config.get(), new_options); + // Translate any errors (NotFound, NotSupported, to InvalidArgument + if (s.ok() || s.IsInvalidArgument()) { + return s; + } else { + return Status::InvalidArgument(s.getState()); + } +} + +Status GetDBOptionsFromString(const ConfigOptions& config_options, + const DBOptions& base_options, + const std::string& opts_str, + DBOptions* new_options) { + std::unordered_map opts_map; + Status s = StringToMap(opts_str, &opts_map); + if (!s.ok()) { + *new_options = base_options; + return s; + } + return GetDBOptionsFromMap(config_options, base_options, opts_map, + new_options); +} + +Status GetOptionsFromString(const Options& base_options, + const std::string& opts_str, Options* new_options) { + ConfigOptions config_options(base_options); + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + + return GetOptionsFromString(config_options, base_options, opts_str, + new_options); +} + +Status GetOptionsFromString(const ConfigOptions& config_options, + const Options& base_options, + const std::string& opts_str, Options* new_options) { + ColumnFamilyOptions new_cf_options; + std::unordered_map unused_opts; + std::unordered_map opts_map; + + assert(new_options); + *new_options = base_options; + Status s = StringToMap(opts_str, &opts_map); + if (!s.ok()) { + return s; + } + auto config = DBOptionsAsConfigurable(base_options); + s = config->ConfigureFromMap(config_options, opts_map, &unused_opts); + + if (s.ok()) { + DBOptions* new_db_options = + config->GetOptions(OptionsHelper::kDBOptionsName); + if (!unused_opts.empty()) { + s = GetColumnFamilyOptionsFromMap(config_options, base_options, + unused_opts, &new_cf_options); + if (s.ok()) { + *new_options = Options(*new_db_options, new_cf_options); + } + } else { + *new_options = Options(*new_db_options, base_options); + } + } + // Translate any errors (NotFound, NotSupported, to InvalidArgument + if (s.ok() || s.IsInvalidArgument()) { + return s; + } else { + return Status::InvalidArgument(s.getState()); + } +} + +std::unordered_map + OptionsHelper::encoding_type_string_map = {{"kPlain", kPlain}, + {"kPrefix", kPrefix}}; + +std::unordered_map + OptionsHelper::compaction_style_string_map = { + {"kCompactionStyleLevel", kCompactionStyleLevel}, + {"kCompactionStyleUniversal", kCompactionStyleUniversal}, + {"kCompactionStyleFIFO", kCompactionStyleFIFO}, + {"kCompactionStyleNone", kCompactionStyleNone}}; + +std::unordered_map + OptionsHelper::compaction_pri_string_map = { + {"kByCompensatedSize", kByCompensatedSize}, + {"kOldestLargestSeqFirst", kOldestLargestSeqFirst}, + {"kOldestSmallestSeqFirst", kOldestSmallestSeqFirst}, + {"kMinOverlappingRatio", kMinOverlappingRatio}, + {"kRoundRobin", kRoundRobin}}; + +std::unordered_map + OptionsHelper::compaction_stop_style_string_map = { + {"kCompactionStopStyleSimilarSize", kCompactionStopStyleSimilarSize}, + {"kCompactionStopStyleTotalSize", kCompactionStopStyleTotalSize}}; + +std::unordered_map + OptionsHelper::temperature_string_map = { + {"kUnknown", Temperature::kUnknown}, + {"kHot", Temperature::kHot}, + {"kWarm", Temperature::kWarm}, + {"kCold", Temperature::kCold}}; + +std::unordered_map + OptionsHelper::prepopulate_blob_cache_string_map = { + {"kDisable", PrepopulateBlobCache::kDisable}, + {"kFlushOnly", PrepopulateBlobCache::kFlushOnly}}; + +Status OptionTypeInfo::NextToken(const std::string& opts, char delimiter, + size_t pos, size_t* end, std::string* token) { + while (pos < opts.size() && isspace(opts[pos])) { + ++pos; + } + // Empty value at the end + if (pos >= opts.size()) { + *token = ""; + *end = std::string::npos; + return Status::OK(); + } else if (opts[pos] == '{') { + int count = 1; + size_t brace_pos = pos + 1; + while (brace_pos < opts.size()) { + if (opts[brace_pos] == '{') { + ++count; + } else if (opts[brace_pos] == '}') { + --count; + if (count == 0) { + break; + } + } + ++brace_pos; + } + // found the matching closing brace + if (count == 0) { + *token = trim(opts.substr(pos + 1, brace_pos - pos - 1)); + // skip all whitespace and move to the next delimiter + // brace_pos points to the next position after the matching '}' + pos = brace_pos + 1; + while (pos < opts.size() && isspace(opts[pos])) { + ++pos; + } + if (pos < opts.size() && opts[pos] != delimiter) { + return Status::InvalidArgument("Unexpected chars after nested options"); + } + *end = pos; + } else { + return Status::InvalidArgument( + "Mismatched curly braces for nested options"); + } + } else { + *end = opts.find(delimiter, pos); + if (*end == std::string::npos) { + // It either ends with a trailing semi-colon or the last key-value pair + *token = trim(opts.substr(pos)); + } else { + *token = trim(opts.substr(pos, *end - pos)); + } + } + return Status::OK(); +} + +Status OptionTypeInfo::Parse(const ConfigOptions& config_options, + const std::string& opt_name, + const std::string& value, void* opt_ptr) const { + if (IsDeprecated()) { + return Status::OK(); + } + try { + const std::string& opt_value = config_options.input_strings_escaped + ? UnescapeOptionString(value) + : value; + + if (opt_ptr == nullptr) { + return Status::NotFound("Could not find option", opt_name); + } else if (parse_func_ != nullptr) { + ConfigOptions copy = config_options; + copy.invoke_prepare_options = false; + void* opt_addr = GetOffset(opt_ptr); + return parse_func_(copy, opt_name, opt_value, opt_addr); + } else if (ParseOptionHelper(GetOffset(opt_ptr), type_, opt_value)) { + return Status::OK(); + } else if (IsConfigurable()) { + // The option is . + Configurable* config = AsRawPointer(opt_ptr); + if (opt_value.empty()) { + return Status::OK(); + } else if (config == nullptr) { + return Status::NotFound("Could not find configurable: ", opt_name); + } else { + ConfigOptions copy = config_options; + copy.ignore_unknown_options = false; + copy.invoke_prepare_options = false; + if (opt_value.find("=") != std::string::npos) { + return config->ConfigureFromString(copy, opt_value); + } else { + return config->ConfigureOption(copy, opt_name, opt_value); + } + } + } else if (IsByName()) { + return Status::NotSupported("Deserializing the option " + opt_name + + " is not supported"); + } else { + return Status::InvalidArgument("Error parsing:", opt_name); + } + } catch (std::exception& e) { + return Status::InvalidArgument("Error parsing " + opt_name + ":" + + std::string(e.what())); + } +} + +Status OptionTypeInfo::ParseType( + const ConfigOptions& config_options, const std::string& opts_str, + const std::unordered_map& type_map, + void* opt_addr, std::unordered_map* unused) { + std::unordered_map opts_map; + Status status = StringToMap(opts_str, &opts_map); + if (!status.ok()) { + return status; + } else { + return ParseType(config_options, opts_map, type_map, opt_addr, unused); + } +} + +Status OptionTypeInfo::ParseType( + const ConfigOptions& config_options, + const std::unordered_map& opts_map, + const std::unordered_map& type_map, + void* opt_addr, std::unordered_map* unused) { + for (const auto& opts_iter : opts_map) { + std::string opt_name; + const auto* opt_info = Find(opts_iter.first, type_map, &opt_name); + if (opt_info != nullptr) { + Status status = + opt_info->Parse(config_options, opt_name, opts_iter.second, opt_addr); + if (!status.ok()) { + return status; + } + } else if (unused != nullptr) { + (*unused)[opts_iter.first] = opts_iter.second; + } else if (!config_options.ignore_unknown_options) { + return Status::NotFound("Unrecognized option", opts_iter.first); + } + } + return Status::OK(); +} + +Status OptionTypeInfo::ParseStruct( + const ConfigOptions& config_options, const std::string& struct_name, + const std::unordered_map* struct_map, + const std::string& opt_name, const std::string& opt_value, void* opt_addr) { + assert(struct_map); + Status status; + if (opt_name == struct_name || EndsWith(opt_name, "." + struct_name)) { + // This option represents the entire struct + std::unordered_map unused; + status = + ParseType(config_options, opt_value, *struct_map, opt_addr, &unused); + if (status.ok() && !unused.empty()) { + status = Status::InvalidArgument( + "Unrecognized option", struct_name + "." + unused.begin()->first); + } + } else if (StartsWith(opt_name, struct_name + ".")) { + // This option represents a nested field in the struct (e.g, struct.field) + std::string elem_name; + const auto opt_info = + Find(opt_name.substr(struct_name.size() + 1), *struct_map, &elem_name); + if (opt_info != nullptr) { + status = opt_info->Parse(config_options, elem_name, opt_value, opt_addr); + } else { + status = Status::InvalidArgument("Unrecognized option", opt_name); + } + } else { + // This option represents a field in the struct (e.g. field) + std::string elem_name; + const auto opt_info = Find(opt_name, *struct_map, &elem_name); + if (opt_info != nullptr) { + status = opt_info->Parse(config_options, elem_name, opt_value, opt_addr); + } else { + status = Status::InvalidArgument("Unrecognized option", + struct_name + "." + opt_name); + } + } + return status; +} + +Status OptionTypeInfo::Serialize(const ConfigOptions& config_options, + const std::string& opt_name, + const void* const opt_ptr, + std::string* opt_value) const { + // If the option is no longer used in rocksdb and marked as deprecated, + // we skip it in the serialization. + if (opt_ptr == nullptr || IsDeprecated()) { + return Status::OK(); + } else if (IsEnabled(OptionTypeFlags::kDontSerialize)) { + return Status::NotSupported("Cannot serialize option: ", opt_name); + } else if (serialize_func_ != nullptr) { + const void* opt_addr = GetOffset(opt_ptr); + return serialize_func_(config_options, opt_name, opt_addr, opt_value); + } else if (IsCustomizable()) { + const Customizable* custom = AsRawPointer(opt_ptr); + opt_value->clear(); + if (custom == nullptr) { + // We do not have a custom object to serialize. + // If the option is not mutable and we are doing only mutable options, + // we return an empty string (which will cause the option not to be + // printed). Otherwise, we return the "nullptr" string, which will result + // in "option=nullptr" being printed. + if (IsMutable() || !config_options.mutable_options_only) { + *opt_value = kNullptrString; + } else { + *opt_value = ""; + } + } else if (IsEnabled(OptionTypeFlags::kStringNameOnly) && + !config_options.IsDetailed()) { + if (!config_options.mutable_options_only || IsMutable()) { + *opt_value = custom->GetId(); + } + } else { + ConfigOptions embedded = config_options; + embedded.delimiter = ";"; + // If this option is mutable, everything inside it should be considered + // mutable + if (IsMutable()) { + embedded.mutable_options_only = false; + } + std::string value = custom->ToString(embedded); + if (!embedded.mutable_options_only || + value.find("=") != std::string::npos) { + *opt_value = value; + } else { + *opt_value = ""; + } + } + return Status::OK(); + } else if (IsConfigurable()) { + const Configurable* config = AsRawPointer(opt_ptr); + if (config != nullptr) { + ConfigOptions embedded = config_options; + embedded.delimiter = ";"; + *opt_value = config->ToString(embedded); + } + return Status::OK(); + } else if (config_options.mutable_options_only && !IsMutable()) { + return Status::OK(); + } else if (SerializeSingleOptionHelper(GetOffset(opt_ptr), type_, + opt_value)) { + return Status::OK(); + } else { + return Status::InvalidArgument("Cannot serialize option: ", opt_name); + } +} + +Status OptionTypeInfo::SerializeType( + const ConfigOptions& config_options, + const std::unordered_map& type_map, + const void* opt_addr, std::string* result) { + Status status; + for (const auto& iter : type_map) { + std::string single; + const auto& opt_info = iter.second; + if (opt_info.ShouldSerialize()) { + status = + opt_info.Serialize(config_options, iter.first, opt_addr, &single); + if (!status.ok()) { + return status; + } else { + result->append(iter.first + "=" + single + config_options.delimiter); + } + } + } + return status; +} + +Status OptionTypeInfo::SerializeStruct( + const ConfigOptions& config_options, const std::string& struct_name, + const std::unordered_map* struct_map, + const std::string& opt_name, const void* opt_addr, std::string* value) { + assert(struct_map); + Status status; + if (EndsWith(opt_name, struct_name)) { + // We are going to write the struct as "{ prop1=value1; prop2=value2;}. + // Set the delimiter to ";" so that the everything will be on one line. + ConfigOptions embedded = config_options; + embedded.delimiter = ";"; + + // This option represents the entire struct + std::string result; + status = SerializeType(embedded, *struct_map, opt_addr, &result); + if (!status.ok()) { + return status; + } else { + *value = "{" + result + "}"; + } + } else if (StartsWith(opt_name, struct_name + ".")) { + // This option represents a nested field in the struct (e.g, struct.field) + std::string elem_name; + const auto opt_info = + Find(opt_name.substr(struct_name.size() + 1), *struct_map, &elem_name); + if (opt_info != nullptr) { + status = opt_info->Serialize(config_options, elem_name, opt_addr, value); + } else { + status = Status::InvalidArgument("Unrecognized option", opt_name); + } + } else { + // This option represents a field in the struct (e.g. field) + std::string elem_name; + const auto opt_info = Find(opt_name, *struct_map, &elem_name); + if (opt_info == nullptr) { + status = Status::InvalidArgument("Unrecognized option", opt_name); + } else if (opt_info->ShouldSerialize()) { + status = opt_info->Serialize(config_options, opt_name + "." + elem_name, + opt_addr, value); + } + } + return status; +} + +template +bool IsOptionEqual(const void* offset1, const void* offset2) { + return (*static_cast(offset1) == *static_cast(offset2)); +} + +static bool AreEqualDoubles(const double a, const double b) { + return (fabs(a - b) < 0.00001); +} + +static bool AreOptionsEqual(OptionType type, const void* this_offset, + const void* that_offset) { + switch (type) { + case OptionType::kBoolean: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kInt: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kUInt: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kInt32T: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kInt64T: { + int64_t v1, v2; + GetUnaligned(static_cast(this_offset), &v1); + GetUnaligned(static_cast(that_offset), &v2); + return (v1 == v2); + } + case OptionType::kUInt8T: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kUInt32T: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kUInt64T: { + uint64_t v1, v2; + GetUnaligned(static_cast(this_offset), &v1); + GetUnaligned(static_cast(that_offset), &v2); + return (v1 == v2); + } + case OptionType::kSizeT: { + size_t v1, v2; + GetUnaligned(static_cast(this_offset), &v1); + GetUnaligned(static_cast(that_offset), &v2); + return (v1 == v2); + } + case OptionType::kString: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kDouble: + return AreEqualDoubles(*static_cast(this_offset), + *static_cast(that_offset)); + case OptionType::kCompactionStyle: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kCompactionStopStyle: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kCompactionPri: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kCompressionType: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kChecksumType: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kEncodingType: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kEncodedString: + return IsOptionEqual(this_offset, that_offset); + case OptionType::kTemperature: + return IsOptionEqual(this_offset, that_offset); + default: + return false; + } // End switch +} + +bool OptionTypeInfo::AreEqual(const ConfigOptions& config_options, + const std::string& opt_name, + const void* const this_ptr, + const void* const that_ptr, + std::string* mismatch) const { + auto level = GetSanityLevel(); + if (!config_options.IsCheckEnabled(level)) { + return true; // If the sanity level is not being checked, skip it + } + if (this_ptr == nullptr || that_ptr == nullptr) { + if (this_ptr == that_ptr) { + return true; + } + } else if (equals_func_ != nullptr) { + const void* this_addr = GetOffset(this_ptr); + const void* that_addr = GetOffset(that_ptr); + if (equals_func_(config_options, opt_name, this_addr, that_addr, + mismatch)) { + return true; + } + } else { + const void* this_addr = GetOffset(this_ptr); + const void* that_addr = GetOffset(that_ptr); + if (AreOptionsEqual(type_, this_addr, that_addr)) { + return true; + } else if (IsConfigurable()) { + const auto* this_config = AsRawPointer(this_ptr); + const auto* that_config = AsRawPointer(that_ptr); + if (this_config == that_config) { + return true; + } else if (this_config != nullptr && that_config != nullptr) { + std::string bad_name; + bool matches; + if (level < config_options.sanity_level) { + ConfigOptions copy = config_options; + copy.sanity_level = level; + matches = this_config->AreEquivalent(copy, that_config, &bad_name); + } else { + matches = this_config->AreEquivalent(config_options, that_config, + &bad_name); + } + if (!matches) { + *mismatch = opt_name + "." + bad_name; + } + return matches; + } + } + } + if (mismatch->empty()) { + *mismatch = opt_name; + } + return false; +} + +bool OptionTypeInfo::TypesAreEqual( + const ConfigOptions& config_options, + const std::unordered_map& type_map, + const void* this_addr, const void* that_addr, std::string* mismatch) { + for (const auto& iter : type_map) { + const auto& opt_info = iter.second; + if (!opt_info.AreEqual(config_options, iter.first, this_addr, that_addr, + mismatch)) { + return false; + } + } + return true; +} + +bool OptionTypeInfo::StructsAreEqual( + const ConfigOptions& config_options, const std::string& struct_name, + const std::unordered_map* struct_map, + const std::string& opt_name, const void* this_addr, const void* that_addr, + std::string* mismatch) { + assert(struct_map); + bool matches = true; + std::string result; + if (EndsWith(opt_name, struct_name)) { + // This option represents the entire struct + matches = TypesAreEqual(config_options, *struct_map, this_addr, that_addr, + &result); + if (!matches) { + *mismatch = struct_name + "." + result; + return false; + } + } else if (StartsWith(opt_name, struct_name + ".")) { + // This option represents a nested field in the struct (e.g, struct.field) + std::string elem_name; + const auto opt_info = + Find(opt_name.substr(struct_name.size() + 1), *struct_map, &elem_name); + assert(opt_info); + if (opt_info == nullptr) { + *mismatch = opt_name; + matches = false; + } else if (!opt_info->AreEqual(config_options, elem_name, this_addr, + that_addr, &result)) { + matches = false; + *mismatch = struct_name + "." + result; + } + } else { + // This option represents a field in the struct (e.g. field) + std::string elem_name; + const auto opt_info = Find(opt_name, *struct_map, &elem_name); + assert(opt_info); + if (opt_info == nullptr) { + *mismatch = struct_name + "." + opt_name; + matches = false; + } else if (!opt_info->AreEqual(config_options, elem_name, this_addr, + that_addr, &result)) { + matches = false; + *mismatch = struct_name + "." + result; + } + } + return matches; +} + +bool MatchesOptionsTypeFromMap( + const ConfigOptions& config_options, + const std::unordered_map& type_map, + const void* const this_ptr, const void* const that_ptr, + std::string* mismatch) { + for (auto& pair : type_map) { + // We skip checking deprecated variables as they might + // contain random values since they might not be initialized + if (config_options.IsCheckEnabled(pair.second.GetSanityLevel())) { + if (!pair.second.AreEqual(config_options, pair.first, this_ptr, that_ptr, + mismatch) && + !pair.second.AreEqualByName(config_options, pair.first, this_ptr, + that_ptr)) { + return false; + } + } + } + return true; +} + +bool OptionTypeInfo::AreEqualByName(const ConfigOptions& config_options, + const std::string& opt_name, + const void* const this_ptr, + const void* const that_ptr) const { + if (IsByName()) { + std::string that_value; + if (Serialize(config_options, opt_name, that_ptr, &that_value).ok()) { + return AreEqualByName(config_options, opt_name, this_ptr, that_value); + } + } + return false; +} + +bool OptionTypeInfo::AreEqualByName(const ConfigOptions& config_options, + const std::string& opt_name, + const void* const opt_ptr, + const std::string& that_value) const { + std::string this_value; + if (!IsByName()) { + return false; + } else if (!Serialize(config_options, opt_name, opt_ptr, &this_value).ok()) { + return false; + } else if (IsEnabled(OptionVerificationType::kByNameAllowFromNull)) { + if (that_value == kNullptrString) { + return true; + } + } else if (IsEnabled(OptionVerificationType::kByNameAllowNull)) { + if (that_value == kNullptrString) { + return true; + } + } + return (this_value == that_value); +} + +Status OptionTypeInfo::Prepare(const ConfigOptions& config_options, + const std::string& name, void* opt_ptr) const { + if (ShouldPrepare()) { + if (prepare_func_ != nullptr) { + void* opt_addr = GetOffset(opt_ptr); + return prepare_func_(config_options, name, opt_addr); + } else if (IsConfigurable()) { + Configurable* config = AsRawPointer(opt_ptr); + if (config != nullptr) { + return config->PrepareOptions(config_options); + } else if (!CanBeNull()) { + return Status::NotFound("Missing configurable object", name); + } + } + } + return Status::OK(); +} + +Status OptionTypeInfo::Validate(const DBOptions& db_opts, + const ColumnFamilyOptions& cf_opts, + const std::string& name, + const void* opt_ptr) const { + if (ShouldValidate()) { + if (validate_func_ != nullptr) { + const void* opt_addr = GetOffset(opt_ptr); + return validate_func_(db_opts, cf_opts, name, opt_addr); + } else if (IsConfigurable()) { + const Configurable* config = AsRawPointer(opt_ptr); + if (config != nullptr) { + return config->ValidateOptions(db_opts, cf_opts); + } else if (!CanBeNull()) { + return Status::NotFound("Missing configurable object", name); + } + } + } + return Status::OK(); +} + +const OptionTypeInfo* OptionTypeInfo::Find( + const std::string& opt_name, + const std::unordered_map& opt_map, + std::string* elem_name) { + const auto iter = opt_map.find(opt_name); // Look up the value in the map + if (iter != opt_map.end()) { // Found the option in the map + *elem_name = opt_name; // Return the name + return &(iter->second); // Return the contents of the iterator + } else { + auto idx = opt_name.find("."); // Look for a separator + if (idx > 0 && idx != std::string::npos) { // We found a separator + auto siter = + opt_map.find(opt_name.substr(0, idx)); // Look for the short name + if (siter != opt_map.end()) { // We found the short name + if (siter->second.IsStruct() || // If the object is a struct + siter->second.IsConfigurable()) { // or a Configurable + *elem_name = opt_name.substr(idx + 1); // Return the rest + return &(siter->second); // Return the contents of the iterator + } + } + } + } + return nullptr; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/options/options_helper.h b/librocksdb-sys/rocksdb/options/options_helper.h new file mode 100644 index 0000000..76e312a --- /dev/null +++ b/librocksdb-sys/rocksdb/options/options_helper.h @@ -0,0 +1,116 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include +#include + +#include "rocksdb/advanced_options.h" +#include "rocksdb/options.h" +#include "rocksdb/status.h" +#include "rocksdb/table.h" + +namespace ROCKSDB_NAMESPACE { +struct ColumnFamilyOptions; +struct ConfigOptions; +struct DBOptions; +struct ImmutableCFOptions; +struct ImmutableDBOptions; +struct MutableDBOptions; +struct MutableCFOptions; +struct Options; + +std::vector GetSupportedCompressions(); + +std::vector GetSupportedDictCompressions(); + +std::vector GetSupportedChecksums(); + +inline bool IsSupportedChecksumType(ChecksumType type) { + // Avoid annoying compiler warning-as-error (-Werror=type-limits) + auto min = kNoChecksum; + auto max = kXXH3; + return type >= min && type <= max; +} + +// Checks that the combination of DBOptions and ColumnFamilyOptions are valid +Status ValidateOptions(const DBOptions& db_opts, + const ColumnFamilyOptions& cf_opts); + +DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options, + const MutableDBOptions& mutable_db_options); + +ColumnFamilyOptions BuildColumnFamilyOptions( + const ColumnFamilyOptions& ioptions, + const MutableCFOptions& mutable_cf_options); + +void UpdateColumnFamilyOptions(const ImmutableCFOptions& ioptions, + ColumnFamilyOptions* cf_opts); +void UpdateColumnFamilyOptions(const MutableCFOptions& moptions, + ColumnFamilyOptions* cf_opts); + +std::unique_ptr DBOptionsAsConfigurable( + const MutableDBOptions& opts); +std::unique_ptr DBOptionsAsConfigurable( + const DBOptions& opts, + const std::unordered_map* opt_map = nullptr); +std::unique_ptr CFOptionsAsConfigurable( + const MutableCFOptions& opts); +std::unique_ptr CFOptionsAsConfigurable( + const ColumnFamilyOptions& opts, + const std::unordered_map* opt_map = nullptr); + +extern Status StringToMap( + const std::string& opts_str, + std::unordered_map* opts_map); + +struct OptionsHelper { + static const std::string kCFOptionsName /*= "ColumnFamilyOptions"*/; + static const std::string kDBOptionsName /*= "DBOptions" */; + static std::map compaction_style_to_string; + static std::map compaction_pri_to_string; + static std::map + compaction_stop_style_to_string; + static std::map temperature_to_string; + static std::unordered_map checksum_type_string_map; + static std::unordered_map + compression_type_string_map; + static std::unordered_map + prepopulate_blob_cache_string_map; + static std::unordered_map + compaction_stop_style_string_map; + static std::unordered_map encoding_type_string_map; + static std::unordered_map + compaction_style_string_map; + static std::unordered_map + compaction_pri_string_map; + static std::unordered_map temperature_string_map; +}; + +// Some aliasing +static auto& compaction_style_to_string = + OptionsHelper::compaction_style_to_string; +static auto& compaction_pri_to_string = OptionsHelper::compaction_pri_to_string; +static auto& compaction_stop_style_to_string = + OptionsHelper::compaction_stop_style_to_string; +static auto& temperature_to_string = OptionsHelper::temperature_to_string; +static auto& checksum_type_string_map = OptionsHelper::checksum_type_string_map; +static auto& compaction_stop_style_string_map = + OptionsHelper::compaction_stop_style_string_map; +static auto& compression_type_string_map = + OptionsHelper::compression_type_string_map; +static auto& encoding_type_string_map = OptionsHelper::encoding_type_string_map; +static auto& compaction_style_string_map = + OptionsHelper::compaction_style_string_map; +static auto& compaction_pri_string_map = + OptionsHelper::compaction_pri_string_map; +static auto& temperature_string_map = OptionsHelper::temperature_string_map; +static auto& prepopulate_blob_cache_string_map = + OptionsHelper::prepopulate_blob_cache_string_map; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/options/options_parser.cc b/librocksdb-sys/rocksdb/options/options_parser.cc new file mode 100644 index 0000000..a8c855d --- /dev/null +++ b/librocksdb-sys/rocksdb/options/options_parser.cc @@ -0,0 +1,736 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + + +#include "options/options_parser.h" + +#include +#include +#include +#include +#include + +#include "file/line_file_reader.h" +#include "file/writable_file_writer.h" +#include "options/cf_options.h" +#include "options/db_options.h" +#include "options/options_helper.h" +#include "port/port.h" +#include "rocksdb/convenience.h" +#include "rocksdb/db.h" +#include "rocksdb/utilities/options_type.h" +#include "test_util/sync_point.h" +#include "util/cast_util.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +static const std::string option_file_header = + "# This is a RocksDB option file.\n" + "#\n" + "# For detailed file format spec, please refer to the example file\n" + "# in examples/rocksdb_option_file_example.ini\n" + "#\n" + "\n"; + +Status PersistRocksDBOptions(const DBOptions& db_opt, + const std::vector& cf_names, + const std::vector& cf_opts, + const std::string& file_name, FileSystem* fs) { + ConfigOptions + config_options; // Use default for escaped(true) and check (exact) + config_options.delimiter = "\n "; + // Do not invoke PrepareOptions when we are doing validation. + config_options.invoke_prepare_options = false; + // If a readahead size was set in the input options, use it + if (db_opt.log_readahead_size > 0) { + config_options.file_readahead_size = db_opt.log_readahead_size; + } + return PersistRocksDBOptions(config_options, db_opt, cf_names, cf_opts, + file_name, fs); +} + +Status PersistRocksDBOptions(const ConfigOptions& config_options_in, + const DBOptions& db_opt, + const std::vector& cf_names, + const std::vector& cf_opts, + const std::string& file_name, FileSystem* fs) { + ConfigOptions config_options = config_options_in; + config_options.delimiter = "\n "; // Override the default to nl + + TEST_SYNC_POINT("PersistRocksDBOptions:start"); + if (cf_names.size() != cf_opts.size()) { + return Status::InvalidArgument( + "cf_names.size() and cf_opts.size() must be the same"); + } + std::unique_ptr wf; + + Status s = + fs->NewWritableFile(file_name, FileOptions(), &wf, nullptr); + if (!s.ok()) { + return s; + } + std::unique_ptr writable; + writable.reset(new WritableFileWriter(std::move(wf), file_name, EnvOptions(), + nullptr /* statistics */)); + TEST_SYNC_POINT("PersistRocksDBOptions:create"); + + std::string options_file_content; + + s = writable->Append( + option_file_header + "[" + opt_section_titles[kOptionSectionVersion] + + "]\n" + " rocksdb_version=" + + std::to_string(ROCKSDB_MAJOR) + "." + std::to_string(ROCKSDB_MINOR) + + "." + std::to_string(ROCKSDB_PATCH) + "\n"); + if (s.ok()) { + s = writable->Append( + " options_file_version=" + std::to_string(ROCKSDB_OPTION_FILE_MAJOR) + + "." + std::to_string(ROCKSDB_OPTION_FILE_MINOR) + "\n"); + } + if (s.ok()) { + s = writable->Append("\n[" + opt_section_titles[kOptionSectionDBOptions] + + "]\n "); + } + + if (s.ok()) { + s = GetStringFromDBOptions(config_options, db_opt, &options_file_content); + } + if (s.ok()) { + s = writable->Append(options_file_content + "\n"); + } + + for (size_t i = 0; s.ok() && i < cf_opts.size(); ++i) { + // CFOptions section + s = writable->Append("\n[" + opt_section_titles[kOptionSectionCFOptions] + + " \"" + EscapeOptionString(cf_names[i]) + "\"]\n "); + if (s.ok()) { + s = GetStringFromColumnFamilyOptions(config_options, cf_opts[i], + &options_file_content); + } + if (s.ok()) { + s = writable->Append(options_file_content + "\n"); + } + // TableOptions section + auto* tf = cf_opts[i].table_factory.get(); + if (tf != nullptr) { + if (s.ok()) { + s = writable->Append( + "[" + opt_section_titles[kOptionSectionTableOptions] + tf->Name() + + " \"" + EscapeOptionString(cf_names[i]) + "\"]\n "); + } + if (s.ok()) { + options_file_content.clear(); + s = tf->GetOptionString(config_options, &options_file_content); + } + if (s.ok()) { + s = writable->Append(options_file_content + "\n"); + } + } + } + if (s.ok()) { + s = writable->Sync(true /* use_fsync */); + } + if (s.ok()) { + s = writable->Close(); + } + TEST_SYNC_POINT("PersistRocksDBOptions:written"); + if (s.ok()) { + return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( + config_options, db_opt, cf_names, cf_opts, file_name, fs); + } + return s; +} + +RocksDBOptionsParser::RocksDBOptionsParser() { Reset(); } + +void RocksDBOptionsParser::Reset() { + db_opt_ = DBOptions(); + db_opt_map_.clear(); + cf_names_.clear(); + cf_opts_.clear(); + cf_opt_maps_.clear(); + has_version_section_ = false; + has_db_options_ = false; + has_default_cf_options_ = false; + for (int i = 0; i < 3; ++i) { + db_version[i] = 0; + opt_file_version[i] = 0; + } +} + +bool RocksDBOptionsParser::IsSection(const std::string& line) { + if (line.size() < 2) { + return false; + } + if (line[0] != '[' || line[line.size() - 1] != ']') { + return false; + } + return true; +} + +Status RocksDBOptionsParser::ParseSection(OptionSection* section, + std::string* title, + std::string* argument, + const std::string& line, + const int line_num) { + *section = kOptionSectionUnknown; + // A section is of the form [ ""], where + // "" is optional. + size_t arg_start_pos = line.find("\""); + size_t arg_end_pos = line.rfind("\""); + // The following if-then check tries to identify whether the input + // section has the optional section argument. + if (arg_start_pos != std::string::npos && arg_start_pos != arg_end_pos) { + *title = TrimAndRemoveComment(line.substr(1, arg_start_pos - 1), true); + *argument = UnescapeOptionString( + line.substr(arg_start_pos + 1, arg_end_pos - arg_start_pos - 1)); + } else { + *title = TrimAndRemoveComment(line.substr(1, line.size() - 2), true); + *argument = ""; + } + for (int i = 0; i < kOptionSectionUnknown; ++i) { + if (title->find(opt_section_titles[i]) == 0) { + if (i == kOptionSectionVersion || i == kOptionSectionDBOptions || + i == kOptionSectionCFOptions) { + if (title->size() == opt_section_titles[i].size()) { + // if true, then it indicats equal + *section = static_cast(i); + return CheckSection(*section, *argument, line_num); + } + } else if (i == kOptionSectionTableOptions) { + // This type of sections has a sufffix at the end of the + // section title + if (title->size() > opt_section_titles[i].size()) { + *section = static_cast(i); + return CheckSection(*section, *argument, line_num); + } + } + } + } + return Status::InvalidArgument(std::string("Unknown section ") + line); +} + +Status RocksDBOptionsParser::InvalidArgument(const int line_num, + const std::string& message) { + return Status::InvalidArgument( + "[RocksDBOptionsParser Error] ", + message + " (at line " + std::to_string(line_num) + ")"); +} + +Status RocksDBOptionsParser::ParseStatement(std::string* name, + std::string* value, + const std::string& line, + const int line_num) { + size_t eq_pos = line.find("="); + if (eq_pos == std::string::npos) { + return InvalidArgument(line_num, "A valid statement must have a '='."); + } + + *name = TrimAndRemoveComment(line.substr(0, eq_pos), true); + *value = + TrimAndRemoveComment(line.substr(eq_pos + 1, line.size() - eq_pos - 1)); + if (name->empty()) { + return InvalidArgument(line_num, + "A valid statement must have a variable name."); + } + return Status::OK(); +} + +Status RocksDBOptionsParser::Parse(const std::string& file_name, FileSystem* fs, + bool ignore_unknown_options, + size_t file_readahead_size) { + ConfigOptions + config_options; // Use default for escaped(true) and check (exact) + config_options.ignore_unknown_options = ignore_unknown_options; + if (file_readahead_size > 0) { + config_options.file_readahead_size = file_readahead_size; + } + return Parse(config_options, file_name, fs); +} + +Status RocksDBOptionsParser::Parse(const ConfigOptions& config_options_in, + const std::string& file_name, + FileSystem* fs) { + Reset(); + ConfigOptions config_options = config_options_in; + + std::unique_ptr seq_file; + Status s = fs->NewSequentialFile(file_name, FileOptions(), &seq_file, + nullptr); + if (!s.ok()) { + return s; + } + LineFileReader lf_reader(std::move(seq_file), file_name, + config_options.file_readahead_size); + + OptionSection section = kOptionSectionUnknown; + std::string title; + std::string argument; + std::unordered_map opt_map; + std::string line; + // we only support single-lined statement. + while (lf_reader.ReadLine(&line, Env::IO_TOTAL /* rate_limiter_priority */)) { + int line_num = static_cast(lf_reader.GetLineNumber()); + line = TrimAndRemoveComment(line); + if (line.empty()) { + continue; + } + if (IsSection(line)) { + s = EndSection(config_options, section, title, argument, opt_map); + opt_map.clear(); + if (!s.ok()) { + return s; + } + + // If the option file is not generated by a higher minor version, + // there shouldn't be any unknown option. + if (config_options.ignore_unknown_options && + section == kOptionSectionVersion) { + if (db_version[0] < ROCKSDB_MAJOR || (db_version[0] == ROCKSDB_MAJOR && + db_version[1] <= ROCKSDB_MINOR)) { + config_options.ignore_unknown_options = false; + } + } + + s = ParseSection(§ion, &title, &argument, line, line_num); + if (!s.ok()) { + return s; + } + } else { + std::string name; + std::string value; + s = ParseStatement(&name, &value, line, line_num); + if (!s.ok()) { + return s; + } + opt_map.insert({name, value}); + } + } + s = lf_reader.GetStatus(); + if (!s.ok()) { + return s; + } + + s = EndSection(config_options, section, title, argument, opt_map); + opt_map.clear(); + if (!s.ok()) { + return s; + } + return ValidityCheck(); +} + +Status RocksDBOptionsParser::CheckSection(const OptionSection section, + const std::string& section_arg, + const int line_num) { + if (section == kOptionSectionDBOptions) { + if (has_db_options_) { + return InvalidArgument( + line_num, + "More than one DBOption section found in the option config file"); + } + has_db_options_ = true; + } else if (section == kOptionSectionCFOptions) { + bool is_default_cf = (section_arg == kDefaultColumnFamilyName); + if (cf_opts_.size() == 0 && !is_default_cf) { + return InvalidArgument( + line_num, + "Default column family must be the first CFOptions section " + "in the option config file"); + } else if (cf_opts_.size() != 0 && is_default_cf) { + return InvalidArgument( + line_num, + "Default column family must be the first CFOptions section " + "in the optio/n config file"); + } else if (GetCFOptions(section_arg) != nullptr) { + return InvalidArgument( + line_num, + "Two identical column families found in option config file"); + } + has_default_cf_options_ |= is_default_cf; + } else if (section == kOptionSectionTableOptions) { + if (GetCFOptions(section_arg) == nullptr) { + return InvalidArgument( + line_num, std::string( + "Does not find a matched column family name in " + "TableOptions section. Column Family Name:") + + section_arg); + } + } else if (section == kOptionSectionVersion) { + if (has_version_section_) { + return InvalidArgument( + line_num, + "More than one Version section found in the option config file."); + } + has_version_section_ = true; + } + return Status::OK(); +} + +Status RocksDBOptionsParser::ParseVersionNumber(const std::string& ver_name, + const std::string& ver_string, + const int max_count, + int* version) { + int version_index = 0; + int current_number = 0; + int current_digit_count = 0; + bool has_dot = false; + for (int i = 0; i < max_count; ++i) { + version[i] = 0; + } + constexpr int kBufferSize = 200; + char buffer[kBufferSize]; + for (size_t i = 0; i < ver_string.size(); ++i) { + if (ver_string[i] == '.') { + if (version_index >= max_count - 1) { + snprintf(buffer, sizeof(buffer) - 1, + "A valid %s can only contains at most %d dots.", + ver_name.c_str(), max_count - 1); + return Status::InvalidArgument(buffer); + } + if (current_digit_count == 0) { + snprintf(buffer, sizeof(buffer) - 1, + "A valid %s must have at least one digit before each dot.", + ver_name.c_str()); + return Status::InvalidArgument(buffer); + } + version[version_index++] = current_number; + current_number = 0; + current_digit_count = 0; + has_dot = true; + } else if (isdigit(ver_string[i])) { + current_number = current_number * 10 + (ver_string[i] - '0'); + current_digit_count++; + } else { + snprintf(buffer, sizeof(buffer) - 1, + "A valid %s can only contains dots and numbers.", + ver_name.c_str()); + return Status::InvalidArgument(buffer); + } + } + version[version_index] = current_number; + if (has_dot && current_digit_count == 0) { + snprintf(buffer, sizeof(buffer) - 1, + "A valid %s must have at least one digit after each dot.", + ver_name.c_str()); + return Status::InvalidArgument(buffer); + } + return Status::OK(); +} + +Status RocksDBOptionsParser::EndSection( + const ConfigOptions& config_options, const OptionSection section, + const std::string& section_title, const std::string& section_arg, + const std::unordered_map& opt_map) { + Status s; + if (section == kOptionSectionDBOptions) { + s = GetDBOptionsFromMap(config_options, DBOptions(), opt_map, &db_opt_); + if (!s.ok()) { + return s; + } + db_opt_map_ = opt_map; + } else if (section == kOptionSectionCFOptions) { + // This condition should be ensured earlier in ParseSection + // so we make an assertion here. + assert(GetCFOptions(section_arg) == nullptr); + cf_names_.emplace_back(section_arg); + cf_opts_.emplace_back(); + s = GetColumnFamilyOptionsFromMap(config_options, ColumnFamilyOptions(), + opt_map, &cf_opts_.back()); + if (!s.ok()) { + return s; + } + // keep the parsed string. + cf_opt_maps_.emplace_back(opt_map); + } else if (section == kOptionSectionTableOptions) { + assert(GetCFOptions(section_arg) != nullptr); + auto* cf_opt = GetCFOptionsImpl(section_arg); + if (cf_opt == nullptr) { + return Status::InvalidArgument( + "The specified column family must be defined before the " + "TableOptions section:", + section_arg); + } + // Ignore error as table factory deserialization is optional + cf_opt->table_factory.reset(); + s = TableFactory::CreateFromString( + config_options, + section_title.substr( + opt_section_titles[kOptionSectionTableOptions].size()), + &(cf_opt->table_factory)); + if (s.ok() && cf_opt->table_factory != nullptr) { + s = cf_opt->table_factory->ConfigureFromMap(config_options, opt_map); + // Translate any errors (NotFound, NotSupported, to InvalidArgument + if (s.ok() || s.IsInvalidArgument()) { + return s; + } else { + return Status::InvalidArgument(s.getState()); + } + } else { + // Return OK for not supported table factories as TableFactory + // Deserialization is optional. + cf_opt->table_factory.reset(); + return Status::OK(); + } + } else if (section == kOptionSectionVersion) { + for (const auto& pair : opt_map) { + if (pair.first == "rocksdb_version") { + s = ParseVersionNumber(pair.first, pair.second, 3, db_version); + if (!s.ok()) { + return s; + } + } else if (pair.first == "options_file_version") { + s = ParseVersionNumber(pair.first, pair.second, 2, opt_file_version); + if (!s.ok()) { + return s; + } + if (opt_file_version[0] < 1) { + return Status::InvalidArgument( + "A valid options_file_version must be at least 1."); + } + } + } + } + return s; +} + +Status RocksDBOptionsParser::ValidityCheck() { + if (!has_db_options_) { + return Status::Corruption( + "A RocksDB Option file must have a single DBOptions section"); + } + if (!has_default_cf_options_) { + return Status::Corruption( + "A RocksDB Option file must have a single CFOptions:default section"); + } + + return Status::OK(); +} + +std::string RocksDBOptionsParser::TrimAndRemoveComment(const std::string& line, + bool trim_only) { + size_t start = 0; + size_t end = line.size(); + + // we only support "#" style comment + if (!trim_only) { + size_t search_pos = 0; + while (search_pos < line.size()) { + size_t comment_pos = line.find('#', search_pos); + if (comment_pos == std::string::npos) { + break; + } + if (comment_pos == 0 || line[comment_pos - 1] != '\\') { + end = comment_pos; + break; + } + search_pos = comment_pos + 1; + } + } + + while (start < end && isspace(line[start]) != 0) { + ++start; + } + + // start < end implies end > 0. + while (start < end && isspace(line[end - 1]) != 0) { + --end; + } + + if (start < end) { + return line.substr(start, end - start); + } + + return ""; +} + +Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( + const ConfigOptions& config_options_in, const DBOptions& db_opt, + const std::vector& cf_names, + const std::vector& cf_opts, + const std::string& file_name, FileSystem* fs) { + RocksDBOptionsParser parser; + ConfigOptions config_options = config_options_in; + config_options.invoke_prepare_options = + false; // No need to do a prepare for verify + if (config_options.sanity_level < ConfigOptions::kSanityLevelExactMatch) { + // If we are not doing an exact comparison, we should ignore + // unsupported options, as they may cause the Parse to fail + // (if the ObjectRegistry is not initialized) + config_options.ignore_unsupported_options = true; + } + Status s = parser.Parse(config_options, file_name, fs); + if (!s.ok()) { + return s; + } + + // Verify DBOptions + s = VerifyDBOptions(config_options, db_opt, *parser.db_opt(), + parser.db_opt_map()); + if (!s.ok()) { + return s; + } + + // Verify ColumnFamily Name + if (cf_names.size() != parser.cf_names()->size()) { + if (config_options.sanity_level >= + ConfigOptions::kSanityLevelLooselyCompatible) { + return Status::InvalidArgument( + "[RocksDBOptionParser Error] The persisted options does not have " + "the same number of column family names as the db instance."); + } else if (cf_opts.size() > parser.cf_opts()->size()) { + return Status::InvalidArgument( + "[RocksDBOptionsParser Error]", + "The persisted options file has less number of column family " + "names than that of the specified one."); + } + } + for (size_t i = 0; i < cf_names.size(); ++i) { + if (cf_names[i] != parser.cf_names()->at(i)) { + return Status::InvalidArgument( + "[RocksDBOptionParser Error] The persisted options and the db" + "instance does not have the same name for column family ", + std::to_string(i)); + } + } + + // Verify Column Family Options + if (cf_opts.size() != parser.cf_opts()->size()) { + if (config_options.sanity_level >= + ConfigOptions::kSanityLevelLooselyCompatible) { + return Status::InvalidArgument( + "[RocksDBOptionsParser Error]", + "The persisted options does not have the same number of " + "column families as the db instance."); + } else if (cf_opts.size() > parser.cf_opts()->size()) { + return Status::InvalidArgument( + "[RocksDBOptionsParser Error]", + "The persisted options file has less number of column families " + "than that of the specified number."); + } + } + for (size_t i = 0; i < cf_opts.size(); ++i) { + s = VerifyCFOptions(config_options, cf_opts[i], parser.cf_opts()->at(i), + &(parser.cf_opt_maps()->at(i))); + if (!s.ok()) { + return s; + } + s = VerifyTableFactory(config_options, cf_opts[i].table_factory.get(), + parser.cf_opts()->at(i).table_factory.get()); + if (!s.ok()) { + return s; + } + } + + return Status::OK(); +} + +Status RocksDBOptionsParser::VerifyDBOptions( + const ConfigOptions& config_options, const DBOptions& base_opt, + const DBOptions& file_opt, + const std::unordered_map* opt_map) { + auto base_config = DBOptionsAsConfigurable(base_opt, opt_map); + auto file_config = DBOptionsAsConfigurable(file_opt, opt_map); + std::string mismatch; + if (!base_config->AreEquivalent(config_options, file_config.get(), + &mismatch)) { + const size_t kBufferSize = 2048; + char buffer[kBufferSize]; + std::string base_value; + std::string file_value; + int offset = snprintf(buffer, sizeof(buffer), + "[RocksDBOptionsParser]: " + "failed the verification on DBOptions::%s -- ", + mismatch.c_str()); + Status s = base_config->GetOption(config_options, mismatch, &base_value); + if (s.ok()) { + s = file_config->GetOption(config_options, mismatch, &file_value); + } + assert(offset >= 0); + assert(static_cast(offset) < sizeof(buffer)); + if (s.ok()) { + snprintf(buffer + offset, sizeof(buffer) - static_cast(offset), + "-- The specified one is %s while the persisted one is %s.\n", + base_value.c_str(), file_value.c_str()); + } else { + snprintf(buffer + offset, sizeof(buffer) - static_cast(offset), + "-- Unable to re-serialize an option: %s.\n", + s.ToString().c_str()); + } + return Status::InvalidArgument(Slice(buffer, strlen(buffer))); + } + return Status::OK(); +} + +Status RocksDBOptionsParser::VerifyCFOptions( + const ConfigOptions& config_options, const ColumnFamilyOptions& base_opt, + const ColumnFamilyOptions& file_opt, + const std::unordered_map* opt_map) { + auto base_config = CFOptionsAsConfigurable(base_opt, opt_map); + auto file_config = CFOptionsAsConfigurable(file_opt, opt_map); + std::string mismatch; + if (!base_config->AreEquivalent(config_options, file_config.get(), + &mismatch)) { + std::string base_value; + std::string file_value; + // The options do not match + const size_t kBufferSize = 2048; + char buffer[kBufferSize]; + Status s = base_config->GetOption(config_options, mismatch, &base_value); + if (s.ok()) { + s = file_config->GetOption(config_options, mismatch, &file_value); + // In file_opt, certain options like MergeOperator may be nullptr due to + // factor methods not available. So we use opt_map to get + // option value to use in the error message below. + if (s.ok() && file_value == kNullptrString && opt_map) { + auto const& opt_val_str = (opt_map->find(mismatch)); + if (opt_val_str != opt_map->end()) { + file_value = opt_val_str->second; + } + } + } + int offset = snprintf(buffer, sizeof(buffer), + "[RocksDBOptionsParser]: " + "failed the verification on ColumnFamilyOptions::%s", + mismatch.c_str()); + assert(offset >= 0); + assert(static_cast(offset) < sizeof(buffer)); + if (s.ok()) { + snprintf(buffer + offset, sizeof(buffer) - static_cast(offset), + "--- The specified one is %s while the persisted one is %s.\n", + base_value.c_str(), file_value.c_str()); + } else { + snprintf(buffer + offset, sizeof(buffer) - static_cast(offset), + "--- Unable to re-serialize an option: %s.\n", + s.ToString().c_str()); + } + return Status::InvalidArgument(Slice(buffer, sizeof(buffer))); + } // For each option + return Status::OK(); +} + +Status RocksDBOptionsParser::VerifyTableFactory( + const ConfigOptions& config_options, const TableFactory* base_tf, + const TableFactory* file_tf) { + std::string mismatch; + if (base_tf && file_tf) { + if (config_options.sanity_level > ConfigOptions::kSanityLevelNone && + std::string(base_tf->Name()) != std::string(file_tf->Name())) { + return Status::Corruption( + "[RocksDBOptionsParser]: " + "failed the verification on TableFactory->Name()"); + } else if (!base_tf->AreEquivalent(config_options, file_tf, &mismatch)) { + return Status::Corruption(std::string("[RocksDBOptionsParser]:" + "failed the verification on ") + + base_tf->Name() + "::", + mismatch); + } + } else { + // TODO(yhchiang): further support sanity check here + } + return Status::OK(); +} +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/options/options_parser.h b/librocksdb-sys/rocksdb/options/options_parser.h new file mode 100644 index 0000000..4268051 --- /dev/null +++ b/librocksdb-sys/rocksdb/options/options_parser.h @@ -0,0 +1,149 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include + +#include "rocksdb/env.h" +#include "rocksdb/options.h" + +namespace ROCKSDB_NAMESPACE { + +struct ConfigOptions; +class OptionTypeInfo; +class TableFactory; + +#define ROCKSDB_OPTION_FILE_MAJOR 1 +#define ROCKSDB_OPTION_FILE_MINOR 1 + +enum OptionSection : char { + kOptionSectionVersion = 0, + kOptionSectionDBOptions, + kOptionSectionCFOptions, + kOptionSectionTableOptions, + kOptionSectionUnknown +}; + +static const std::string opt_section_titles[] = { + "Version", "DBOptions", "CFOptions", "TableOptions/", "Unknown"}; + +Status PersistRocksDBOptions(const DBOptions& db_opt, + const std::vector& cf_names, + const std::vector& cf_opts, + const std::string& file_name, FileSystem* fs); +Status PersistRocksDBOptions(const ConfigOptions& config_options, + const DBOptions& db_opt, + const std::vector& cf_names, + const std::vector& cf_opts, + const std::string& file_name, FileSystem* fs); + +class RocksDBOptionsParser { + public: + explicit RocksDBOptionsParser(); + ~RocksDBOptionsParser() {} + void Reset(); + + // `file_readahead_size` is used for readahead for the option file. + // If 0 is given, a default value will be used. + Status Parse(const std::string& file_name, FileSystem* fs, + bool ignore_unknown_options, size_t file_readahead_size); + + Status Parse(const ConfigOptions& config_options, + const std::string& file_name, FileSystem* fs); + + static std::string TrimAndRemoveComment(const std::string& line, + const bool trim_only = false); + + const DBOptions* db_opt() const { return &db_opt_; } + const std::unordered_map* db_opt_map() const { + return &db_opt_map_; + } + const std::vector* cf_opts() const { return &cf_opts_; } + const std::vector* cf_names() const { return &cf_names_; } + const std::vector>* cf_opt_maps() + const { + return &cf_opt_maps_; + } + + const ColumnFamilyOptions* GetCFOptions(const std::string& name) { + return GetCFOptionsImpl(name); + } + size_t NumColumnFamilies() { return cf_opts_.size(); } + static Status VerifyRocksDBOptionsFromFile( + const ConfigOptions& config_options, const DBOptions& db_opt, + const std::vector& cf_names, + const std::vector& cf_opts, + const std::string& file_name, FileSystem* fs); + static Status VerifyDBOptions( + const ConfigOptions& config_options, const DBOptions& base_opt, + const DBOptions& new_opt, + const std::unordered_map* new_opt_map = + nullptr); + + static Status VerifyCFOptions( + const ConfigOptions& config_options, const ColumnFamilyOptions& base_opt, + const ColumnFamilyOptions& new_opt, + const std::unordered_map* new_opt_map = + nullptr); + + static Status VerifyTableFactory(const ConfigOptions& config_options, + const TableFactory* base_tf, + const TableFactory* file_tf); + + static Status ExtraParserCheck(const RocksDBOptionsParser& input_parser); + + static Status ParseStatement(std::string* name, std::string* value, + const std::string& line, const int line_num); + + protected: + bool IsSection(const std::string& line); + Status ParseSection(OptionSection* section, std::string* title, + std::string* argument, const std::string& line, + const int line_num); + + Status CheckSection(const OptionSection section, + const std::string& section_arg, const int line_num); + + Status EndSection( + const ConfigOptions& config_options, const OptionSection section, + const std::string& title, const std::string& section_arg, + const std::unordered_map& opt_map); + + Status ValidityCheck(); + + static Status InvalidArgument(const int line_num, const std::string& message); + + Status ParseVersionNumber(const std::string& ver_name, + const std::string& ver_string, const int max_count, + int* version); + + ColumnFamilyOptions* GetCFOptionsImpl(const std::string& name) { + assert(cf_names_.size() == cf_opts_.size()); + for (size_t i = 0; i < cf_names_.size(); ++i) { + if (cf_names_[i] == name) { + return &cf_opts_[i]; + } + } + return nullptr; + } + + private: + DBOptions db_opt_; + std::unordered_map db_opt_map_; + std::vector cf_names_; + std::vector cf_opts_; + std::vector> cf_opt_maps_; + bool has_version_section_; + bool has_db_options_; + bool has_default_cf_options_; + int db_version[3]; + int opt_file_version[3]; +}; + + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/options/options_settable_test.cc b/librocksdb-sys/rocksdb/options/options_settable_test.cc new file mode 100644 index 0000000..c2bf864 --- /dev/null +++ b/librocksdb-sys/rocksdb/options/options_settable_test.cc @@ -0,0 +1,657 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include + +#include "options/cf_options.h" +#include "options/db_options.h" +#include "options/options_helper.h" +#include "rocksdb/convenience.h" +#include "test_util/testharness.h" + +#ifndef GFLAGS +bool FLAGS_enable_print = false; +#else +#include "util/gflags_compat.h" +using GFLAGS_NAMESPACE::ParseCommandLineFlags; +DEFINE_bool(enable_print, false, "Print options generated to console."); +#endif // GFLAGS + +namespace ROCKSDB_NAMESPACE { + +// Verify options are settable from options strings. +// We take the approach that depends on compiler behavior that copy constructor +// won't touch implicit padding bytes, so that the test is fragile. +// As a result, we only run the tests to verify new fields in options are +// settable through string on limited platforms as it depends on behavior of +// compilers. +#if defined OS_LINUX || defined OS_WIN +#ifndef __clang__ +#ifndef ROCKSDB_UBSAN_RUN + +class OptionsSettableTest : public testing::Test { + public: + OptionsSettableTest() {} +}; + +const char kSpecialChar = 'z'; +using OffsetGap = std::vector>; + +void FillWithSpecialChar(char* start_ptr, size_t total_size, + const OffsetGap& excluded, + char special_char = kSpecialChar) { + size_t offset = 0; + // The excluded vector contains pairs of bytes, (first, second). + // The first bytes are all set to the special char (represented as 'c' below). + // The second bytes are simply skipped (padding bytes). + // ccccc[skipped]cccccccc[skiped]cccccccc[skipped] + for (auto& pair : excluded) { + std::memset(start_ptr + offset, special_char, pair.first - offset); + offset = pair.first + pair.second; + } + // The rest of the structure is filled with the special characters. + // ccccc[skipped]cccccccc[skiped]cccccccc[skipped]cccccccccccccccc + std::memset(start_ptr + offset, special_char, total_size - offset); +} + +int NumUnsetBytes(char* start_ptr, size_t total_size, + const OffsetGap& excluded) { + int total_unset_bytes_base = 0; + size_t offset = 0; + for (auto& pair : excluded) { + // The first part of the structure contains memory spaces that can be + // set (pair.first), and memory spaces that cannot be set (pair.second). + // Therefore total_unset_bytes_base only agregates bytes set to kSpecialChar + // in the pair.first bytes, but skips the pair.second bytes (padding bytes). + for (char* ptr = start_ptr + offset; ptr < start_ptr + pair.first; ptr++) { + if (*ptr == kSpecialChar) { + total_unset_bytes_base++; + } + } + offset = pair.first + pair.second; + } + // Then total_unset_bytes_base aggregates the bytes + // set to kSpecialChar in the rest of the structure + for (char* ptr = start_ptr + offset; ptr < start_ptr + total_size; ptr++) { + if (*ptr == kSpecialChar) { + total_unset_bytes_base++; + } + } + return total_unset_bytes_base; +} + +// Return true iff two structs are the same except excluded fields. +bool CompareBytes(char* start_ptr1, char* start_ptr2, size_t total_size, + const OffsetGap& excluded) { + size_t offset = 0; + for (auto& pair : excluded) { + for (; offset < pair.first; offset++) { + if (*(start_ptr1 + offset) != *(start_ptr2 + offset)) { + return false; + } + } + offset = pair.first + pair.second; + } + for (; offset < total_size; offset++) { + if (*(start_ptr1 + offset) != *(start_ptr2 + offset)) { + return false; + } + } + return true; +} + +// If the test fails, likely a new option is added to BlockBasedTableOptions +// but it cannot be set through GetBlockBasedTableOptionsFromString(), or the +// test is not updated accordingly. +// After adding an option, we need to make sure it is settable by +// GetBlockBasedTableOptionsFromString() and add the option to the input string +// passed to the GetBlockBasedTableOptionsFromString() in this test. +// If it is a complicated type, you also need to add the field to +// kBbtoExcluded, and maybe add customized verification for it. +TEST_F(OptionsSettableTest, BlockBasedTableOptionsAllFieldsSettable) { + // Items in the form of . Need to be in ascending order + // and not overlapping. Need to update if new option to be excluded is added + // (e.g, pointer-type) + const OffsetGap kBbtoExcluded = { + {offsetof(struct BlockBasedTableOptions, flush_block_policy_factory), + sizeof(std::shared_ptr)}, + {offsetof(struct BlockBasedTableOptions, block_cache), + sizeof(std::shared_ptr)}, + {offsetof(struct BlockBasedTableOptions, persistent_cache), + sizeof(std::shared_ptr)}, + {offsetof(struct BlockBasedTableOptions, cache_usage_options), + sizeof(CacheUsageOptions)}, + {offsetof(struct BlockBasedTableOptions, filter_policy), + sizeof(std::shared_ptr)}, + }; + + // In this test, we catch a new option of BlockBasedTableOptions that is not + // settable through GetBlockBasedTableOptionsFromString(). + // We count padding bytes of the option struct, and assert it to be the same + // as unset bytes of an option struct initialized by + // GetBlockBasedTableOptionsFromString(). + + char* bbto_ptr = new char[sizeof(BlockBasedTableOptions)]; + + // Count padding bytes by setting all bytes in the memory to a special char, + // copy a well constructed struct to this memory and see how many special + // bytes left. + BlockBasedTableOptions* bbto = new (bbto_ptr) BlockBasedTableOptions(); + FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoExcluded); + // It based on the behavior of compiler that padding bytes are not changed + // when copying the struct. It's prone to failure when compiler behavior + // changes. We verify there is unset bytes to detect the case. + *bbto = BlockBasedTableOptions(); + int unset_bytes_base = + NumUnsetBytes(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoExcluded); + ASSERT_GT(unset_bytes_base, 0); + bbto->~BlockBasedTableOptions(); + + // Construct the base option passed into + // GetBlockBasedTableOptionsFromString(). + bbto = new (bbto_ptr) BlockBasedTableOptions(); + FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoExcluded); + // This option is not setable: + bbto->use_delta_encoding = true; + + char* new_bbto_ptr = new char[sizeof(BlockBasedTableOptions)]; + BlockBasedTableOptions* new_bbto = + new (new_bbto_ptr) BlockBasedTableOptions(); + FillWithSpecialChar(new_bbto_ptr, sizeof(BlockBasedTableOptions), + kBbtoExcluded); + + // Need to update the option string if a new option is added. + ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + config_options.invoke_prepare_options = false; + config_options.ignore_unsupported_options = false; + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, *bbto, + "cache_index_and_filter_blocks=1;" + "cache_index_and_filter_blocks_with_high_priority=true;" + "metadata_cache_options={top_level_index_pinning=kFallback;" + "partition_pinning=kAll;" + "unpartitioned_pinning=kFlushedAndSimilar;};" + "pin_l0_filter_and_index_blocks_in_cache=1;" + "pin_top_level_index_and_filter=1;" + "index_type=kHashSearch;" + "data_block_index_type=kDataBlockBinaryAndHash;" + "index_shortening=kNoShortening;" + "data_block_hash_table_util_ratio=0.75;" + "checksum=kxxHash;no_block_cache=1;" + "block_cache=1M;block_cache_compressed=1k;block_size=1024;" + "block_size_deviation=8;block_restart_interval=4; " + "metadata_block_size=1024;" + "partition_filters=false;" + "optimize_filters_for_memory=true;" + "index_block_restart_interval=4;" + "filter_policy=bloomfilter:4:true;whole_key_filtering=1;detect_filter_" + "construct_corruption=false;" + "format_version=1;" + "verify_compression=true;read_amp_bytes_per_bit=0;" + "enable_index_compression=false;" + "block_align=true;" + "max_auto_readahead_size=0;" + "prepopulate_block_cache=kDisable;" + "initial_auto_readahead_size=0;" + "num_file_reads_for_auto_readahead=0", + new_bbto)); + + ASSERT_EQ(unset_bytes_base, + NumUnsetBytes(new_bbto_ptr, sizeof(BlockBasedTableOptions), + kBbtoExcluded)); + + ASSERT_TRUE(new_bbto->block_cache.get() != nullptr); + ASSERT_TRUE(new_bbto->filter_policy.get() != nullptr); + + bbto->~BlockBasedTableOptions(); + new_bbto->~BlockBasedTableOptions(); + + delete[] bbto_ptr; + delete[] new_bbto_ptr; +} + +// If the test fails, likely a new option is added to DBOptions +// but it cannot be set through GetDBOptionsFromString(), or the test is not +// updated accordingly. +// After adding an option, we need to make sure it is settable by +// GetDBOptionsFromString() and add the option to the input string passed to +// DBOptionsFromString()in this test. +// If it is a complicated type, you also need to add the field to +// kDBOptionsExcluded, and maybe add customized verification for it. +TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) { + const OffsetGap kDBOptionsExcluded = { + {offsetof(struct DBOptions, env), sizeof(Env*)}, + {offsetof(struct DBOptions, rate_limiter), + sizeof(std::shared_ptr)}, + {offsetof(struct DBOptions, sst_file_manager), + sizeof(std::shared_ptr)}, + {offsetof(struct DBOptions, info_log), sizeof(std::shared_ptr)}, + {offsetof(struct DBOptions, statistics), + sizeof(std::shared_ptr)}, + {offsetof(struct DBOptions, db_paths), sizeof(std::vector)}, + {offsetof(struct DBOptions, db_log_dir), sizeof(std::string)}, + {offsetof(struct DBOptions, wal_dir), sizeof(std::string)}, + {offsetof(struct DBOptions, write_buffer_manager), + sizeof(std::shared_ptr)}, + {offsetof(struct DBOptions, listeners), + sizeof(std::vector>)}, + {offsetof(struct DBOptions, row_cache), sizeof(std::shared_ptr)}, + {offsetof(struct DBOptions, wal_filter), sizeof(const WalFilter*)}, + {offsetof(struct DBOptions, file_checksum_gen_factory), + sizeof(std::shared_ptr)}, + {offsetof(struct DBOptions, db_host_id), sizeof(std::string)}, + {offsetof(struct DBOptions, checksum_handoff_file_types), + sizeof(FileTypeSet)}, + {offsetof(struct DBOptions, compaction_service), + sizeof(std::shared_ptr)}, + }; + + char* options_ptr = new char[sizeof(DBOptions)]; + + // Count padding bytes by setting all bytes in the memory to a special char, + // copy a well constructed struct to this memory and see how many special + // bytes left. + DBOptions* options = new (options_ptr) DBOptions(); + FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsExcluded); + // It based on the behavior of compiler that padding bytes are not changed + // when copying the struct. It's prone to failure when compiler behavior + // changes. We verify there is unset bytes to detect the case. + *options = DBOptions(); + int unset_bytes_base = + NumUnsetBytes(options_ptr, sizeof(DBOptions), kDBOptionsExcluded); + ASSERT_GT(unset_bytes_base, 0); + options->~DBOptions(); + + options = new (options_ptr) DBOptions(); + FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsExcluded); + + char* new_options_ptr = new char[sizeof(DBOptions)]; + DBOptions* new_options = new (new_options_ptr) DBOptions(); + FillWithSpecialChar(new_options_ptr, sizeof(DBOptions), kDBOptionsExcluded); + + // Need to update the option string if a new option is added. + ConfigOptions config_options(*options); + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + ASSERT_OK( + GetDBOptionsFromString(config_options, *options, + "wal_bytes_per_sync=4295048118;" + "delete_obsolete_files_period_micros=4294967758;" + "WAL_ttl_seconds=4295008036;" + "WAL_size_limit_MB=4295036161;" + "max_write_batch_group_size_bytes=1048576;" + "wal_dir=path/to/wal_dir;" + "db_write_buffer_size=2587;" + "max_subcompactions=64330;" + "table_cache_numshardbits=28;" + "max_open_files=72;" + "max_file_opening_threads=35;" + "max_background_jobs=8;" + "max_background_compactions=33;" + "use_fsync=true;" + "use_adaptive_mutex=false;" + "max_total_wal_size=4295005604;" + "compaction_readahead_size=0;" + "keep_log_file_num=4890;" + "skip_stats_update_on_db_open=false;" + "skip_checking_sst_file_sizes_on_db_open=false;" + "max_manifest_file_size=4295009941;" + "db_log_dir=path/to/db_log_dir;" + "writable_file_max_buffer_size=1048576;" + "paranoid_checks=true;" + "flush_verify_memtable_count=true;" + "compaction_verify_record_count=true;" + "track_and_verify_wals_in_manifest=true;" + "verify_sst_unique_id_in_manifest=true;" + "is_fd_close_on_exec=false;" + "bytes_per_sync=4295013613;" + "strict_bytes_per_sync=true;" + "enable_thread_tracking=false;" + "recycle_log_file_num=0;" + "create_missing_column_families=true;" + "log_file_time_to_roll=3097;" + "max_background_flushes=35;" + "create_if_missing=false;" + "error_if_exists=true;" + "delayed_write_rate=4294976214;" + "manifest_preallocation_size=1222;" + "allow_mmap_writes=false;" + "stats_dump_period_sec=70127;" + "stats_persist_period_sec=54321;" + "persist_stats_to_disk=true;" + "stats_history_buffer_size=14159;" + "allow_fallocate=true;" + "allow_mmap_reads=false;" + "use_direct_reads=false;" + "use_direct_io_for_flush_and_compaction=false;" + "max_log_file_size=4607;" + "random_access_max_buffer_size=1048576;" + "advise_random_on_open=true;" + "fail_if_options_file_error=false;" + "enable_pipelined_write=false;" + "unordered_write=false;" + "allow_concurrent_memtable_write=true;" + "wal_recovery_mode=kPointInTimeRecovery;" + "enable_write_thread_adaptive_yield=true;" + "write_thread_slow_yield_usec=5;" + "write_thread_max_yield_usec=1000;" + "access_hint_on_compaction_start=NONE;" + "info_log_level=DEBUG_LEVEL;" + "dump_malloc_stats=false;" + "allow_2pc=false;" + "avoid_flush_during_recovery=false;" + "avoid_flush_during_shutdown=false;" + "allow_ingest_behind=false;" + "concurrent_prepare=false;" + "two_write_queues=false;" + "manual_wal_flush=false;" + "wal_compression=kZSTD;" + "seq_per_batch=false;" + "atomic_flush=false;" + "avoid_unnecessary_blocking_io=false;" + "log_readahead_size=0;" + "write_dbid_to_manifest=false;" + "best_efforts_recovery=false;" + "max_bgerror_resume_count=2;" + "bgerror_resume_retry_interval=1000000;" + "db_host_id=hostname;" + "lowest_used_cache_tier=kNonVolatileBlockTier;" + "allow_data_in_errors=false;" + "enforce_single_del_contracts=false;", + new_options)); + + ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_options_ptr, sizeof(DBOptions), + kDBOptionsExcluded)); + + options->~DBOptions(); + new_options->~DBOptions(); + + delete[] options_ptr; + delete[] new_options_ptr; +} + +// If the test fails, likely a new option is added to ColumnFamilyOptions +// but it cannot be set through GetColumnFamilyOptionsFromString(), or the +// test is not updated accordingly. +// After adding an option, we need to make sure it is settable by +// GetColumnFamilyOptionsFromString() and add the option to the input +// string passed to GetColumnFamilyOptionsFromString() in this test. +// If it is a complicated type, you also need to add the field to +// kColumnFamilyOptionsExcluded, and maybe add customized verification +// for it. +TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) { + // options in the excluded set need to appear in the same order as in + // ColumnFamilyOptions. + const OffsetGap kColumnFamilyOptionsExcluded = { + {offsetof(struct ColumnFamilyOptions, inplace_callback), + sizeof(UpdateStatus(*)(char*, uint32_t*, Slice, std::string*))}, + {offsetof(struct ColumnFamilyOptions, + memtable_insert_with_hint_prefix_extractor), + sizeof(std::shared_ptr)}, + {offsetof(struct ColumnFamilyOptions, compression_per_level), + sizeof(std::vector)}, + {offsetof(struct ColumnFamilyOptions, + max_bytes_for_level_multiplier_additional), + sizeof(std::vector)}, + {offsetof(struct ColumnFamilyOptions, compaction_options_fifo), + sizeof(struct CompactionOptionsFIFO)}, + {offsetof(struct ColumnFamilyOptions, memtable_factory), + sizeof(std::shared_ptr)}, + {offsetof(struct ColumnFamilyOptions, + table_properties_collector_factories), + sizeof(ColumnFamilyOptions::TablePropertiesCollectorFactories)}, + {offsetof(struct ColumnFamilyOptions, preclude_last_level_data_seconds), + sizeof(uint64_t)}, + {offsetof(struct ColumnFamilyOptions, preserve_internal_time_seconds), + sizeof(uint64_t)}, + {offsetof(struct ColumnFamilyOptions, blob_cache), + sizeof(std::shared_ptr)}, + {offsetof(struct ColumnFamilyOptions, comparator), sizeof(Comparator*)}, + {offsetof(struct ColumnFamilyOptions, merge_operator), + sizeof(std::shared_ptr)}, + {offsetof(struct ColumnFamilyOptions, compaction_filter), + sizeof(const CompactionFilter*)}, + {offsetof(struct ColumnFamilyOptions, compaction_filter_factory), + sizeof(std::shared_ptr)}, + {offsetof(struct ColumnFamilyOptions, prefix_extractor), + sizeof(std::shared_ptr)}, + {offsetof(struct ColumnFamilyOptions, snap_refresh_nanos), + sizeof(uint64_t)}, + {offsetof(struct ColumnFamilyOptions, table_factory), + sizeof(std::shared_ptr)}, + {offsetof(struct ColumnFamilyOptions, cf_paths), + sizeof(std::vector)}, + {offsetof(struct ColumnFamilyOptions, compaction_thread_limiter), + sizeof(std::shared_ptr)}, + {offsetof(struct ColumnFamilyOptions, sst_partitioner_factory), + sizeof(std::shared_ptr)}, + }; + + char* options_ptr = new char[sizeof(ColumnFamilyOptions)]; + + // Count padding bytes by setting all bytes in the memory to a special char, + // copy a well constructed struct to this memory and see how many special + // bytes left. + FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions), + kColumnFamilyOptionsExcluded); + + // Invoke a user-defined constructor in the hope that it does not overwrite + // padding bytes. Note that previously we relied on the implicitly-defined + // copy-assignment operator (i.e., `*options = ColumnFamilyOptions();`) here, + // which did in fact modify padding bytes. + ColumnFamilyOptions* options = new (options_ptr) ColumnFamilyOptions(); + + int unset_bytes_base = NumUnsetBytes(options_ptr, sizeof(ColumnFamilyOptions), + kColumnFamilyOptionsExcluded); + ASSERT_GT(unset_bytes_base, 0); + options->~ColumnFamilyOptions(); + + options = new (options_ptr) ColumnFamilyOptions(); + FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions), + kColumnFamilyOptionsExcluded); + + // Following options are not settable through + // GetColumnFamilyOptionsFromString(): + options->compaction_options_universal = CompactionOptionsUniversal(); + options->num_levels = 42; // Initialize options for MutableCF + options->compaction_filter = nullptr; + options->sst_partitioner_factory = nullptr; + + char* new_options_ptr = new char[sizeof(ColumnFamilyOptions)]; + ColumnFamilyOptions* new_options = + new (new_options_ptr) ColumnFamilyOptions(); + FillWithSpecialChar(new_options_ptr, sizeof(ColumnFamilyOptions), + kColumnFamilyOptionsExcluded); + + // Need to update the option string if a new option is added. + ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, *options, + "compaction_filter_factory=mpudlojcujCompactionFilterFactory;" + "table_factory=PlainTable;" + "prefix_extractor=rocksdb.CappedPrefix.13;" + "comparator=leveldb.BytewiseComparator;" + "compression_per_level=kBZip2Compression:kBZip2Compression:" + "kBZip2Compression:kNoCompression:kZlibCompression:kBZip2Compression:" + "kSnappyCompression;" + "max_bytes_for_level_base=986;" + "bloom_locality=8016;" + "target_file_size_base=4294976376;" + "memtable_huge_page_size=2557;" + "max_successive_merges=5497;" + "max_sequential_skip_in_iterations=4294971408;" + "arena_block_size=1893;" + "target_file_size_multiplier=35;" + "min_write_buffer_number_to_merge=9;" + "max_write_buffer_number=84;" + "write_buffer_size=1653;" + "max_compaction_bytes=64;" + "ignore_max_compaction_bytes_for_input=true;" + "max_bytes_for_level_multiplier=60;" + "memtable_factory=SkipListFactory;" + "compression=kNoCompression;" + "compression_opts={max_dict_buffer_bytes=5;use_zstd_dict_trainer=true;" + "enabled=false;parallel_threads=6;zstd_max_train_bytes=7;strategy=8;max_" + "dict_bytes=9;level=10;window_bits=11;max_compressed_bytes_per_kb=987;};" + "bottommost_compression_opts={max_dict_buffer_bytes=4;use_zstd_dict_" + "trainer=true;enabled=true;parallel_threads=5;zstd_max_train_bytes=6;" + "strategy=7;max_dict_bytes=8;level=9;window_bits=10;max_compressed_bytes_" + "per_kb=876;};" + "bottommost_compression=kDisableCompressionOption;" + "level0_stop_writes_trigger=33;" + "num_levels=99;" + "level0_slowdown_writes_trigger=22;" + "level0_file_num_compaction_trigger=14;" + "compaction_filter=urxcqstuwnCompactionFilter;" + "soft_pending_compaction_bytes_limit=0;" + "max_write_buffer_number_to_maintain=84;" + "max_write_buffer_size_to_maintain=2147483648;" + "merge_operator=aabcxehazrMergeOperator;" + "memtable_prefix_bloom_size_ratio=0.4642;" + "memtable_whole_key_filtering=true;" + "memtable_insert_with_hint_prefix_extractor=rocksdb.CappedPrefix.13;" + "check_flush_compaction_key_order=false;" + "paranoid_file_checks=true;" + "force_consistency_checks=true;" + "inplace_update_num_locks=7429;" + "experimental_mempurge_threshold=0.0001;" + "optimize_filters_for_hits=false;" + "level_compaction_dynamic_level_bytes=false;" + "level_compaction_dynamic_file_size=true;" + "inplace_update_support=false;" + "compaction_style=kCompactionStyleFIFO;" + "compaction_pri=kMinOverlappingRatio;" + "hard_pending_compaction_bytes_limit=0;" + "disable_auto_compactions=false;" + "report_bg_io_stats=true;" + "ttl=60;" + "periodic_compaction_seconds=3600;" + "sample_for_compression=0;" + "enable_blob_files=true;" + "min_blob_size=256;" + "blob_file_size=1000000;" + "blob_compression_type=kBZip2Compression;" + "enable_blob_garbage_collection=true;" + "blob_garbage_collection_age_cutoff=0.5;" + "blob_garbage_collection_force_threshold=0.75;" + "blob_compaction_readahead_size=262144;" + "blob_file_starting_level=1;" + "prepopulate_blob_cache=kDisable;" + "bottommost_temperature=kWarm;" + "last_level_temperature=kWarm;" + "preclude_last_level_data_seconds=86400;" + "preserve_internal_time_seconds=86400;" + "compaction_options_fifo={max_table_files_size=3;allow_" + "compaction=true;age_for_warm=0;file_temperature_age_thresholds={{" + "temperature=kCold;age=12345}};};" + "blob_cache=1M;" + "memtable_protection_bytes_per_key=2;" + "persist_user_defined_timestamps=true;" + "block_protection_bytes_per_key=1;" + "memtable_max_range_deletions=999999;", + new_options)); + + ASSERT_NE(new_options->blob_cache.get(), nullptr); + + ASSERT_EQ(unset_bytes_base, + NumUnsetBytes(new_options_ptr, sizeof(ColumnFamilyOptions), + kColumnFamilyOptionsExcluded)); + + // Custom verification since compaction_options_fifo was in + // kColumnFamilyOptionsExcluded + ASSERT_EQ(new_options->compaction_options_fifo.max_table_files_size, 3); + ASSERT_EQ(new_options->compaction_options_fifo.allow_compaction, true); + ASSERT_EQ(new_options->compaction_options_fifo.file_temperature_age_thresholds + .size(), + 1); + ASSERT_EQ( + new_options->compaction_options_fifo.file_temperature_age_thresholds[0] + .temperature, + Temperature::kCold); + ASSERT_EQ( + new_options->compaction_options_fifo.file_temperature_age_thresholds[0] + .age, + 12345); + + ColumnFamilyOptions rnd_filled_options = *new_options; + + options->~ColumnFamilyOptions(); + new_options->~ColumnFamilyOptions(); + + delete[] options_ptr; + delete[] new_options_ptr; + + // Test copying to mutabable and immutable options and copy back the mutable + // part. + const OffsetGap kMutableCFOptionsExcluded = { + {offsetof(struct MutableCFOptions, prefix_extractor), + sizeof(std::shared_ptr)}, + {offsetof(struct MutableCFOptions, + max_bytes_for_level_multiplier_additional), + sizeof(std::vector)}, + {offsetof(struct MutableCFOptions, compaction_options_fifo), + sizeof(struct CompactionOptionsFIFO)}, + {offsetof(struct MutableCFOptions, compression_per_level), + sizeof(std::vector)}, + {offsetof(struct MutableCFOptions, max_file_size), + sizeof(std::vector)}, + }; + + // For all memory used for options, pre-fill every char. Otherwise, the + // padding bytes might be different so that byte-wise comparison doesn't + // general equal results even if objects are equal. + const char kMySpecialChar = 'x'; + char* mcfo1_ptr = new char[sizeof(MutableCFOptions)]; + FillWithSpecialChar(mcfo1_ptr, sizeof(MutableCFOptions), + kMutableCFOptionsExcluded, kMySpecialChar); + char* mcfo2_ptr = new char[sizeof(MutableCFOptions)]; + FillWithSpecialChar(mcfo2_ptr, sizeof(MutableCFOptions), + kMutableCFOptionsExcluded, kMySpecialChar); + + // A clean column family options is constructed after filling the same special + // char as the initial one. So that the padding bytes are the same. + char* cfo_clean_ptr = new char[sizeof(ColumnFamilyOptions)]; + FillWithSpecialChar(cfo_clean_ptr, sizeof(ColumnFamilyOptions), + kColumnFamilyOptionsExcluded); + rnd_filled_options.num_levels = 66; + ColumnFamilyOptions* cfo_clean = new (cfo_clean_ptr) ColumnFamilyOptions(); + + MutableCFOptions* mcfo1 = + new (mcfo1_ptr) MutableCFOptions(rnd_filled_options); + ColumnFamilyOptions cfo_back = BuildColumnFamilyOptions(*cfo_clean, *mcfo1); + MutableCFOptions* mcfo2 = new (mcfo2_ptr) MutableCFOptions(cfo_back); + + ASSERT_TRUE(CompareBytes(mcfo1_ptr, mcfo2_ptr, sizeof(MutableCFOptions), + kMutableCFOptionsExcluded)); + + cfo_clean->~ColumnFamilyOptions(); + mcfo1->~MutableCFOptions(); + mcfo2->~MutableCFOptions(); + delete[] mcfo1_ptr; + delete[] mcfo2_ptr; + delete[] cfo_clean_ptr; +} +#endif // !ROCKSDB_UBSAN_RUN +#endif // !__clang__ +#endif // OS_LINUX || OS_WIN + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); +#ifdef GFLAGS + ParseCommandLineFlags(&argc, &argv, true); +#endif // GFLAGS + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/options/options_test.cc b/librocksdb-sys/rocksdb/options/options_test.cc new file mode 100644 index 0000000..067b00b --- /dev/null +++ b/librocksdb-sys/rocksdb/options/options_test.cc @@ -0,0 +1,5064 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include +#include +#include + +#include "cache/lru_cache.h" +#include "cache/sharded_cache.h" +#include "options/options_helper.h" +#include "options/options_parser.h" +#include "port/port.h" +#include "rocksdb/cache.h" +#include "rocksdb/convenience.h" +#include "rocksdb/file_checksum.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/utilities/leveldb_options.h" +#include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/options_type.h" +#include "table/block_based/filter_policy_internal.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/random.h" +#include "util/stderr_logger.h" +#include "util/string_util.h" +#include "utilities/merge_operators/bytesxor.h" +#include "utilities/merge_operators/sortlist.h" +#include "utilities/merge_operators/string_append/stringappend.h" +#include "utilities/merge_operators/string_append/stringappend2.h" + +#ifndef GFLAGS +bool FLAGS_enable_print = false; +#else +#include "util/gflags_compat.h" +using GFLAGS_NAMESPACE::ParseCommandLineFlags; +DEFINE_bool(enable_print, false, "Print options generated to console."); +#endif // GFLAGS + +namespace ROCKSDB_NAMESPACE { + +class OptionsTest : public testing::Test {}; + +class UnregisteredTableFactory : public TableFactory { + public: + UnregisteredTableFactory() {} + const char* Name() const override { return "Unregistered"; } + using TableFactory::NewTableReader; + Status NewTableReader(const ReadOptions&, const TableReaderOptions&, + std::unique_ptr&&, uint64_t, + std::unique_ptr*, bool) const override { + return Status::NotSupported(); + } + TableBuilder* NewTableBuilder(const TableBuilderOptions&, + WritableFileWriter*) const override { + return nullptr; + } +}; + +TEST_F(OptionsTest, GetOptionsFromMapTest) { + std::unordered_map cf_options_map = { + {"write_buffer_size", "1"}, + {"max_write_buffer_number", "2"}, + {"min_write_buffer_number_to_merge", "3"}, + {"max_write_buffer_number_to_maintain", "99"}, + {"max_write_buffer_size_to_maintain", "-99999"}, + {"compression", "kSnappyCompression"}, + {"compression_per_level", + "kNoCompression:" + "kSnappyCompression:" + "kZlibCompression:" + "kBZip2Compression:" + "kLZ4Compression:" + "kLZ4HCCompression:" + "kXpressCompression:" + "kZSTD:" + "kZSTDNotFinalCompression"}, + {"bottommost_compression", "kLZ4Compression"}, + {"bottommost_compression_opts", "5:6:7:8:10:true"}, + {"compression_opts", "4:5:6:7:8:2:true:100:false"}, + {"num_levels", "8"}, + {"level0_file_num_compaction_trigger", "8"}, + {"level0_slowdown_writes_trigger", "9"}, + {"level0_stop_writes_trigger", "10"}, + {"target_file_size_base", "12"}, + {"target_file_size_multiplier", "13"}, + {"max_bytes_for_level_base", "14"}, + {"level_compaction_dynamic_level_bytes", "true"}, + {"max_bytes_for_level_multiplier", "15.0"}, + {"max_bytes_for_level_multiplier_additional", "16:17:18"}, + {"max_compaction_bytes", "21"}, + {"hard_pending_compaction_bytes_limit", "211"}, + {"arena_block_size", "22"}, + {"disable_auto_compactions", "true"}, + {"compaction_style", "kCompactionStyleLevel"}, + {"compaction_pri", "kOldestSmallestSeqFirst"}, + {"verify_checksums_in_compaction", "false"}, + {"compaction_options_fifo", + "{allow_compaction=true;max_table_files_size=11002244;" + "file_temperature_age_thresholds={{temperature=kCold;age=12345}}}"}, + {"max_sequential_skip_in_iterations", "24"}, + {"inplace_update_support", "true"}, + {"report_bg_io_stats", "true"}, + {"compaction_measure_io_stats", "false"}, + {"purge_redundant_kvs_while_flush", "false"}, + {"inplace_update_num_locks", "25"}, + {"memtable_prefix_bloom_size_ratio", "0.26"}, + {"memtable_whole_key_filtering", "true"}, + {"memtable_huge_page_size", "28"}, + {"bloom_locality", "29"}, + {"max_successive_merges", "30"}, + {"min_partial_merge_operands", "31"}, + {"prefix_extractor", "fixed:31"}, + {"experimental_mempurge_threshold", "0.003"}, + {"optimize_filters_for_hits", "true"}, + {"enable_blob_files", "true"}, + {"min_blob_size", "1K"}, + {"blob_file_size", "1G"}, + {"blob_compression_type", "kZSTD"}, + {"enable_blob_garbage_collection", "true"}, + {"blob_garbage_collection_age_cutoff", "0.5"}, + {"blob_garbage_collection_force_threshold", "0.75"}, + {"blob_compaction_readahead_size", "256K"}, + {"blob_file_starting_level", "1"}, + {"prepopulate_blob_cache", "kDisable"}, + {"last_level_temperature", "kWarm"}, + {"persist_user_defined_timestamps", "true"}, + {"memtable_max_range_deletions", "0"}, + }; + + std::unordered_map db_options_map = { + {"create_if_missing", "false"}, + {"create_missing_column_families", "true"}, + {"error_if_exists", "false"}, + {"paranoid_checks", "true"}, + {"track_and_verify_wals_in_manifest", "true"}, + {"verify_sst_unique_id_in_manifest", "true"}, + {"max_open_files", "32"}, + {"max_total_wal_size", "33"}, + {"use_fsync", "true"}, + {"db_log_dir", "/db_log_dir"}, + {"wal_dir", "/wal_dir"}, + {"delete_obsolete_files_period_micros", "34"}, + {"max_background_compactions", "35"}, + {"max_background_flushes", "36"}, + {"max_log_file_size", "37"}, + {"log_file_time_to_roll", "38"}, + {"keep_log_file_num", "39"}, + {"recycle_log_file_num", "5"}, + {"max_manifest_file_size", "40"}, + {"table_cache_numshardbits", "41"}, + {"WAL_ttl_seconds", "43"}, + {"WAL_size_limit_MB", "44"}, + {"manifest_preallocation_size", "45"}, + {"allow_mmap_reads", "true"}, + {"allow_mmap_writes", "false"}, + {"use_direct_reads", "false"}, + {"use_direct_io_for_flush_and_compaction", "false"}, + {"is_fd_close_on_exec", "true"}, + {"skip_log_error_on_recovery", "false"}, + {"stats_dump_period_sec", "46"}, + {"stats_persist_period_sec", "57"}, + {"persist_stats_to_disk", "false"}, + {"stats_history_buffer_size", "69"}, + {"advise_random_on_open", "true"}, + {"use_adaptive_mutex", "false"}, + {"compaction_readahead_size", "100"}, + {"random_access_max_buffer_size", "3145728"}, + {"writable_file_max_buffer_size", "314159"}, + {"bytes_per_sync", "47"}, + {"wal_bytes_per_sync", "48"}, + {"strict_bytes_per_sync", "true"}, + {"preserve_deletes", "false"}, + }; + + ColumnFamilyOptions base_cf_opt; + ColumnFamilyOptions new_cf_opt; + ConfigOptions exact, loose; + exact.input_strings_escaped = false; + exact.ignore_unknown_options = false; + exact.sanity_level = ConfigOptions::kSanityLevelExactMatch; + loose.sanity_level = ConfigOptions::kSanityLevelLooselyCompatible; + + loose.input_strings_escaped = false; + loose.ignore_unknown_options = true; + ASSERT_OK(GetColumnFamilyOptionsFromMap(exact, base_cf_opt, cf_options_map, + &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 1U); + ASSERT_EQ(new_cf_opt.max_write_buffer_number, 2); + ASSERT_EQ(new_cf_opt.min_write_buffer_number_to_merge, 3); + ASSERT_EQ(new_cf_opt.max_write_buffer_number_to_maintain, 99); + ASSERT_EQ(new_cf_opt.max_write_buffer_size_to_maintain, -99999); + ASSERT_EQ(new_cf_opt.compression, kSnappyCompression); + ASSERT_EQ(new_cf_opt.compression_per_level.size(), 9U); + ASSERT_EQ(new_cf_opt.compression_per_level[0], kNoCompression); + ASSERT_EQ(new_cf_opt.compression_per_level[1], kSnappyCompression); + ASSERT_EQ(new_cf_opt.compression_per_level[2], kZlibCompression); + ASSERT_EQ(new_cf_opt.compression_per_level[3], kBZip2Compression); + ASSERT_EQ(new_cf_opt.compression_per_level[4], kLZ4Compression); + ASSERT_EQ(new_cf_opt.compression_per_level[5], kLZ4HCCompression); + ASSERT_EQ(new_cf_opt.compression_per_level[6], kXpressCompression); + ASSERT_EQ(new_cf_opt.compression_per_level[7], kZSTD); + ASSERT_EQ(new_cf_opt.compression_per_level[8], kZSTDNotFinalCompression); + ASSERT_EQ(new_cf_opt.compression_opts.window_bits, 4); + ASSERT_EQ(new_cf_opt.compression_opts.level, 5); + ASSERT_EQ(new_cf_opt.compression_opts.strategy, 6); + ASSERT_EQ(new_cf_opt.compression_opts.max_dict_bytes, 7u); + ASSERT_EQ(new_cf_opt.compression_opts.zstd_max_train_bytes, 8u); + ASSERT_EQ(new_cf_opt.compression_opts.parallel_threads, 2u); + ASSERT_EQ(new_cf_opt.compression_opts.enabled, true); + ASSERT_EQ(new_cf_opt.compression_opts.max_dict_buffer_bytes, 100u); + ASSERT_EQ(new_cf_opt.compression_opts.use_zstd_dict_trainer, false); + ASSERT_EQ(new_cf_opt.bottommost_compression, kLZ4Compression); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.window_bits, 5); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.level, 6); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.strategy, 7); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.max_dict_bytes, 8u); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.zstd_max_train_bytes, 10u); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.parallel_threads, + CompressionOptions().parallel_threads); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.enabled, true); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.use_zstd_dict_trainer, + CompressionOptions().use_zstd_dict_trainer); + ASSERT_EQ(new_cf_opt.num_levels, 8); + ASSERT_EQ(new_cf_opt.level0_file_num_compaction_trigger, 8); + ASSERT_EQ(new_cf_opt.level0_slowdown_writes_trigger, 9); + ASSERT_EQ(new_cf_opt.level0_stop_writes_trigger, 10); + ASSERT_EQ(new_cf_opt.target_file_size_base, static_cast(12)); + ASSERT_EQ(new_cf_opt.target_file_size_multiplier, 13); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_base, 14U); + ASSERT_EQ(new_cf_opt.level_compaction_dynamic_level_bytes, true); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier, 15.0); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional.size(), 3U); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[0], 16); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[1], 17); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[2], 18); + ASSERT_EQ(new_cf_opt.max_compaction_bytes, 21); + ASSERT_EQ(new_cf_opt.hard_pending_compaction_bytes_limit, 211); + ASSERT_EQ(new_cf_opt.arena_block_size, 22U); + ASSERT_EQ(new_cf_opt.disable_auto_compactions, true); + ASSERT_EQ(new_cf_opt.compaction_style, kCompactionStyleLevel); + ASSERT_EQ(new_cf_opt.compaction_pri, kOldestSmallestSeqFirst); + ASSERT_EQ(new_cf_opt.compaction_options_fifo.max_table_files_size, + static_cast(11002244)); + ASSERT_EQ(new_cf_opt.compaction_options_fifo.allow_compaction, true); + ASSERT_EQ( + new_cf_opt.compaction_options_fifo.file_temperature_age_thresholds.size(), + 1); + ASSERT_EQ( + new_cf_opt.compaction_options_fifo.file_temperature_age_thresholds[0] + .temperature, + Temperature::kCold); + ASSERT_EQ( + new_cf_opt.compaction_options_fifo.file_temperature_age_thresholds[0].age, + 12345); + ASSERT_EQ(new_cf_opt.max_sequential_skip_in_iterations, + static_cast(24)); + ASSERT_EQ(new_cf_opt.inplace_update_support, true); + ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 25U); + ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_size_ratio, 0.26); + ASSERT_EQ(new_cf_opt.memtable_whole_key_filtering, true); + ASSERT_EQ(new_cf_opt.memtable_huge_page_size, 28U); + ASSERT_EQ(new_cf_opt.bloom_locality, 29U); + ASSERT_EQ(new_cf_opt.max_successive_merges, 30U); + ASSERT_TRUE(new_cf_opt.prefix_extractor != nullptr); + ASSERT_EQ(new_cf_opt.optimize_filters_for_hits, true); + ASSERT_EQ(new_cf_opt.prefix_extractor->AsString(), "rocksdb.FixedPrefix.31"); + ASSERT_EQ(new_cf_opt.experimental_mempurge_threshold, 0.003); + ASSERT_EQ(new_cf_opt.enable_blob_files, true); + ASSERT_EQ(new_cf_opt.min_blob_size, 1ULL << 10); + ASSERT_EQ(new_cf_opt.blob_file_size, 1ULL << 30); + ASSERT_EQ(new_cf_opt.blob_compression_type, kZSTD); + ASSERT_EQ(new_cf_opt.enable_blob_garbage_collection, true); + ASSERT_EQ(new_cf_opt.blob_garbage_collection_age_cutoff, 0.5); + ASSERT_EQ(new_cf_opt.blob_garbage_collection_force_threshold, 0.75); + ASSERT_EQ(new_cf_opt.blob_compaction_readahead_size, 262144); + ASSERT_EQ(new_cf_opt.blob_file_starting_level, 1); + ASSERT_EQ(new_cf_opt.prepopulate_blob_cache, PrepopulateBlobCache::kDisable); + ASSERT_EQ(new_cf_opt.last_level_temperature, Temperature::kWarm); + ASSERT_EQ(new_cf_opt.bottommost_temperature, Temperature::kWarm); + ASSERT_EQ(new_cf_opt.persist_user_defined_timestamps, true); + ASSERT_EQ(new_cf_opt.memtable_max_range_deletions, 0); + + cf_options_map["write_buffer_size"] = "hello"; + ASSERT_NOK(GetColumnFamilyOptionsFromMap(exact, base_cf_opt, cf_options_map, + &new_cf_opt)); + ASSERT_OK( + RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); + + cf_options_map["write_buffer_size"] = "1"; + ASSERT_OK(GetColumnFamilyOptionsFromMap(exact, base_cf_opt, cf_options_map, + &new_cf_opt)); + + cf_options_map["unknown_option"] = "1"; + ASSERT_NOK(GetColumnFamilyOptionsFromMap(exact, base_cf_opt, cf_options_map, + &new_cf_opt)); + ASSERT_OK( + RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); + + // ignore_unknown_options=true;input_strings_escaped=false + ASSERT_OK(GetColumnFamilyOptionsFromMap(loose, base_cf_opt, cf_options_map, + &new_cf_opt)); + ASSERT_OK( + RocksDBOptionsParser::VerifyCFOptions(loose, base_cf_opt, new_cf_opt)); + ASSERT_NOK( + RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); + + DBOptions base_db_opt; + DBOptions new_db_opt; + ASSERT_OK( + GetDBOptionsFromMap(exact, base_db_opt, db_options_map, &new_db_opt)); + ASSERT_EQ(new_db_opt.create_if_missing, false); + ASSERT_EQ(new_db_opt.create_missing_column_families, true); + ASSERT_EQ(new_db_opt.error_if_exists, false); + ASSERT_EQ(new_db_opt.paranoid_checks, true); + ASSERT_EQ(new_db_opt.track_and_verify_wals_in_manifest, true); + ASSERT_EQ(new_db_opt.verify_sst_unique_id_in_manifest, true); + ASSERT_EQ(new_db_opt.max_open_files, 32); + ASSERT_EQ(new_db_opt.max_total_wal_size, static_cast(33)); + ASSERT_EQ(new_db_opt.use_fsync, true); + ASSERT_EQ(new_db_opt.db_log_dir, "/db_log_dir"); + ASSERT_EQ(new_db_opt.wal_dir, "/wal_dir"); + ASSERT_EQ(new_db_opt.delete_obsolete_files_period_micros, + static_cast(34)); + ASSERT_EQ(new_db_opt.max_background_compactions, 35); + ASSERT_EQ(new_db_opt.max_background_flushes, 36); + ASSERT_EQ(new_db_opt.max_log_file_size, 37U); + ASSERT_EQ(new_db_opt.log_file_time_to_roll, 38U); + ASSERT_EQ(new_db_opt.keep_log_file_num, 39U); + ASSERT_EQ(new_db_opt.recycle_log_file_num, 5U); + ASSERT_EQ(new_db_opt.max_manifest_file_size, static_cast(40)); + ASSERT_EQ(new_db_opt.table_cache_numshardbits, 41); + ASSERT_EQ(new_db_opt.WAL_ttl_seconds, static_cast(43)); + ASSERT_EQ(new_db_opt.WAL_size_limit_MB, static_cast(44)); + ASSERT_EQ(new_db_opt.manifest_preallocation_size, 45U); + ASSERT_EQ(new_db_opt.allow_mmap_reads, true); + ASSERT_EQ(new_db_opt.allow_mmap_writes, false); + ASSERT_EQ(new_db_opt.use_direct_reads, false); + ASSERT_EQ(new_db_opt.use_direct_io_for_flush_and_compaction, false); + ASSERT_EQ(new_db_opt.is_fd_close_on_exec, true); + ASSERT_EQ(new_db_opt.stats_dump_period_sec, 46U); + ASSERT_EQ(new_db_opt.stats_persist_period_sec, 57U); + ASSERT_EQ(new_db_opt.persist_stats_to_disk, false); + ASSERT_EQ(new_db_opt.stats_history_buffer_size, 69U); + ASSERT_EQ(new_db_opt.advise_random_on_open, true); + ASSERT_EQ(new_db_opt.use_adaptive_mutex, false); + ASSERT_EQ(new_db_opt.compaction_readahead_size, 100); + ASSERT_EQ(new_db_opt.random_access_max_buffer_size, 3145728); + ASSERT_EQ(new_db_opt.writable_file_max_buffer_size, 314159); + ASSERT_EQ(new_db_opt.bytes_per_sync, static_cast(47)); + ASSERT_EQ(new_db_opt.wal_bytes_per_sync, static_cast(48)); + ASSERT_EQ(new_db_opt.strict_bytes_per_sync, true); + + db_options_map["max_open_files"] = "hello"; + Status s = + GetDBOptionsFromMap(exact, base_db_opt, db_options_map, &new_db_opt); + ASSERT_NOK(s); + ASSERT_TRUE(s.IsInvalidArgument()); + + ASSERT_OK( + RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt)); + ASSERT_OK( + RocksDBOptionsParser::VerifyDBOptions(loose, base_db_opt, new_db_opt)); + + // unknow options should fail parsing without ignore_unknown_options = true + db_options_map["unknown_db_option"] = "1"; + s = GetDBOptionsFromMap(exact, base_db_opt, db_options_map, &new_db_opt); + ASSERT_NOK(s); + ASSERT_TRUE(s.IsInvalidArgument()); + ASSERT_OK( + RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt)); + + ASSERT_OK( + GetDBOptionsFromMap(loose, base_db_opt, db_options_map, &new_db_opt)); + ASSERT_OK( + RocksDBOptionsParser::VerifyDBOptions(loose, base_db_opt, new_db_opt)); + ASSERT_NOK( + RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt)); +} + +TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) { + ColumnFamilyOptions base_cf_opt; + ColumnFamilyOptions new_cf_opt; + ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + + base_cf_opt.table_factory.reset(); + ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt, "", + &new_cf_opt)); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, "write_buffer_size=5", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 5U); + ASSERT_TRUE(new_cf_opt.table_factory == nullptr); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, "write_buffer_size=6;", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 6U); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, " write_buffer_size = 7 ", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 7U); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, " write_buffer_size = 8 ; ", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 8U); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=9;max_write_buffer_number=10", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 9U); + ASSERT_EQ(new_cf_opt.max_write_buffer_number, 10); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=11; max_write_buffer_number = 12 ;", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 11U); + ASSERT_EQ(new_cf_opt.max_write_buffer_number, 12); + // Wrong name "max_write_buffer_number_" + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=13;max_write_buffer_number_=14;", &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt, + new_cf_opt)); + + // Comparator from object registry + std::string kCompName = "reverse_comp"; + ObjectLibrary::Default()->AddFactory( + kCompName, + [](const std::string& /*name*/, + std::unique_ptr* /*guard*/, + std::string* /* errmsg */) { return ReverseBytewiseComparator(); }); + + ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt, + "comparator=" + kCompName + ";", + &new_cf_opt)); + ASSERT_EQ(new_cf_opt.comparator, ReverseBytewiseComparator()); + + // MergeOperator from object registry + std::unique_ptr bxo(new BytesXOROperator()); + std::string kMoName = bxo->Name(); + + ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt, + "merge_operator=" + kMoName + ";", + &new_cf_opt)); + ASSERT_EQ(kMoName, std::string(new_cf_opt.merge_operator->Name())); + + // Wrong key/value pair + Status s = GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=13;max_write_buffer_number;", &new_cf_opt); + ASSERT_NOK(s); + ASSERT_TRUE(s.IsInvalidArgument()); + + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt, + new_cf_opt)); + + // Error Parsing value + s = GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=13;max_write_buffer_number=;", &new_cf_opt); + ASSERT_NOK(s); + ASSERT_TRUE(s.IsInvalidArgument()); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt, + new_cf_opt)); + + // Missing option name + s = GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, "write_buffer_size=13; =100;", &new_cf_opt); + ASSERT_NOK(s); + ASSERT_TRUE(s.IsInvalidArgument()); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt, + new_cf_opt)); + + const uint64_t kilo = 1024UL; + const uint64_t mega = 1024 * kilo; + const uint64_t giga = 1024 * mega; + const uint64_t tera = 1024 * giga; + + // Units (k) + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, "max_write_buffer_number=15K", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.max_write_buffer_number, 15 * kilo); + // Units (m) + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "max_write_buffer_number=16m;inplace_update_num_locks=17M", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.max_write_buffer_number, 16 * mega); + ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 17u * mega); + // Units (g) + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=18g;prefix_extractor=capped:8;" + "arena_block_size=19G", + &new_cf_opt)); + + ASSERT_EQ(new_cf_opt.write_buffer_size, 18 * giga); + ASSERT_EQ(new_cf_opt.arena_block_size, 19 * giga); + ASSERT_TRUE(new_cf_opt.prefix_extractor.get() != nullptr); + ASSERT_EQ(new_cf_opt.prefix_extractor->AsString(), "rocksdb.CappedPrefix.8"); + + // Units (t) + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, "write_buffer_size=20t;arena_block_size=21T", + &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 20 * tera); + ASSERT_EQ(new_cf_opt.arena_block_size, 21 * tera); + + // Nested block based table options + // Empty + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={};arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.table_factory != nullptr); + // Non-empty + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_cache=1M;block_size=4;};" + "arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.table_factory != nullptr); + // Last one + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_cache=1M;block_size=4;}", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.table_factory != nullptr); + // Mismatch curly braces + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={{{block_size=4;};" + "arena_block_size=1024", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt, + new_cf_opt)); + + // Unexpected chars after closing curly brace + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_size=4;}};" + "arena_block_size=1024", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt, + new_cf_opt)); + + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_size=4;}xdfa;" + "arena_block_size=1024", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt, + new_cf_opt)); + + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_size=4;}xdfa", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt, + new_cf_opt)); + + // Invalid block based table option + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={xx_block_size=4;}", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt, + new_cf_opt)); + + ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt, + "optimize_filters_for_hits=true", + &new_cf_opt)); + ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt, + "optimize_filters_for_hits=false", + &new_cf_opt)); + + ASSERT_NOK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt, + "optimize_filters_for_hits=junk", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt, + new_cf_opt)); + + // Nested plain table options + // Empty + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "plain_table_factory={};arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.table_factory != nullptr); + ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable"); + // Non-empty + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "plain_table_factory={user_key_len=66;bloom_bits_per_key=20;};" + "arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.table_factory != nullptr); + ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable"); + + // memtable factory + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "memtable=skip_list:10;arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.memtable_factory != nullptr); + ASSERT_EQ(std::string(new_cf_opt.memtable_factory->Name()), "SkipListFactory"); + ASSERT_TRUE(new_cf_opt.memtable_factory->IsInstanceOf("SkipListFactory")); + + // blob cache + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "blob_cache={capacity=1M;num_shard_bits=4;" + "strict_capacity_limit=true;high_pri_pool_ratio=0.5;};", + &new_cf_opt)); + ASSERT_NE(new_cf_opt.blob_cache, nullptr); + ASSERT_EQ(new_cf_opt.blob_cache->GetCapacity(), 1024UL * 1024UL); + ASSERT_EQ(static_cast(new_cf_opt.blob_cache.get()) + ->GetNumShardBits(), + 4); + ASSERT_EQ(new_cf_opt.blob_cache->HasStrictCapacityLimit(), true); + ASSERT_EQ(static_cast(new_cf_opt.blob_cache.get()) + ->GetHighPriPoolRatio(), + 0.5); +} + +TEST_F(OptionsTest, CompressionOptionsFromString) { + ColumnFamilyOptions base_cf_opt; + ColumnFamilyOptions new_cf_opt; + ConfigOptions config_options; + std::string opts_str; + config_options.ignore_unknown_options = false; + CompressionOptions dflt; + // Test with some optional values removed.... + ASSERT_OK( + GetColumnFamilyOptionsFromString(config_options, ColumnFamilyOptions(), + "compression_opts=3:4:5; " + "bottommost_compression_opts=4:5:6:7", + &base_cf_opt)); + ASSERT_EQ(base_cf_opt.compression_opts.window_bits, 3); + ASSERT_EQ(base_cf_opt.compression_opts.level, 4); + ASSERT_EQ(base_cf_opt.compression_opts.strategy, 5); + ASSERT_EQ(base_cf_opt.compression_opts.max_dict_bytes, dflt.max_dict_bytes); + ASSERT_EQ(base_cf_opt.compression_opts.zstd_max_train_bytes, + dflt.zstd_max_train_bytes); + ASSERT_EQ(base_cf_opt.compression_opts.parallel_threads, + dflt.parallel_threads); + ASSERT_EQ(base_cf_opt.compression_opts.enabled, dflt.enabled); + ASSERT_EQ(base_cf_opt.compression_opts.use_zstd_dict_trainer, + dflt.use_zstd_dict_trainer); + ASSERT_EQ(base_cf_opt.bottommost_compression_opts.window_bits, 4); + ASSERT_EQ(base_cf_opt.bottommost_compression_opts.level, 5); + ASSERT_EQ(base_cf_opt.bottommost_compression_opts.strategy, 6); + ASSERT_EQ(base_cf_opt.bottommost_compression_opts.max_dict_bytes, 7u); + ASSERT_EQ(base_cf_opt.bottommost_compression_opts.zstd_max_train_bytes, + dflt.zstd_max_train_bytes); + ASSERT_EQ(base_cf_opt.bottommost_compression_opts.parallel_threads, + dflt.parallel_threads); + ASSERT_EQ(base_cf_opt.bottommost_compression_opts.enabled, dflt.enabled); + ASSERT_EQ(base_cf_opt.bottommost_compression_opts.use_zstd_dict_trainer, + dflt.use_zstd_dict_trainer); + + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, ColumnFamilyOptions(), + "compression_opts=4:5:6:7:8:9:true:10:false; " + "bottommost_compression_opts=5:6:7:8:9:false", + &base_cf_opt)); + ASSERT_EQ(base_cf_opt.compression_opts.window_bits, 4); + ASSERT_EQ(base_cf_opt.compression_opts.level, 5); + ASSERT_EQ(base_cf_opt.compression_opts.strategy, 6); + ASSERT_EQ(base_cf_opt.compression_opts.max_dict_bytes, 7u); + ASSERT_EQ(base_cf_opt.compression_opts.zstd_max_train_bytes, 8u); + ASSERT_EQ(base_cf_opt.compression_opts.parallel_threads, 9u); + ASSERT_EQ(base_cf_opt.compression_opts.enabled, true); + ASSERT_EQ(base_cf_opt.compression_opts.max_dict_buffer_bytes, 10u); + ASSERT_EQ(base_cf_opt.compression_opts.use_zstd_dict_trainer, false); + ASSERT_EQ(base_cf_opt.bottommost_compression_opts.window_bits, 5); + ASSERT_EQ(base_cf_opt.bottommost_compression_opts.level, 6); + ASSERT_EQ(base_cf_opt.bottommost_compression_opts.strategy, 7); + ASSERT_EQ(base_cf_opt.bottommost_compression_opts.max_dict_bytes, 8u); + ASSERT_EQ(base_cf_opt.bottommost_compression_opts.zstd_max_train_bytes, 9u); + ASSERT_EQ(base_cf_opt.bottommost_compression_opts.parallel_threads, + dflt.parallel_threads); + ASSERT_EQ(base_cf_opt.bottommost_compression_opts.enabled, false); + ASSERT_EQ(base_cf_opt.bottommost_compression_opts.use_zstd_dict_trainer, + dflt.use_zstd_dict_trainer); + + ASSERT_OK( + GetStringFromColumnFamilyOptions(config_options, base_cf_opt, &opts_str)); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, ColumnFamilyOptions(), opts_str, &new_cf_opt)); + ASSERT_EQ(new_cf_opt.compression_opts.window_bits, 4); + ASSERT_EQ(new_cf_opt.compression_opts.level, 5); + ASSERT_EQ(new_cf_opt.compression_opts.strategy, 6); + ASSERT_EQ(new_cf_opt.compression_opts.max_dict_bytes, 7u); + ASSERT_EQ(new_cf_opt.compression_opts.zstd_max_train_bytes, 8u); + ASSERT_EQ(new_cf_opt.compression_opts.parallel_threads, 9u); + ASSERT_EQ(new_cf_opt.compression_opts.enabled, true); + ASSERT_EQ(base_cf_opt.compression_opts.max_dict_buffer_bytes, 10u); + ASSERT_EQ(base_cf_opt.compression_opts.use_zstd_dict_trainer, false); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.window_bits, 5); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.level, 6); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.strategy, 7); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.max_dict_bytes, 8u); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.zstd_max_train_bytes, 9u); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.parallel_threads, + dflt.parallel_threads); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.enabled, false); + ASSERT_EQ(base_cf_opt.bottommost_compression_opts.use_zstd_dict_trainer, + dflt.use_zstd_dict_trainer); + + // Test as struct values + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, ColumnFamilyOptions(), + "compression_opts={window_bits=5; level=6; strategy=7; max_dict_bytes=8;" + "zstd_max_train_bytes=9;parallel_threads=10;enabled=true;use_zstd_dict_" + "trainer=false}; " + "bottommost_compression_opts={window_bits=4; level=5; strategy=6;" + " max_dict_bytes=7;zstd_max_train_bytes=8;parallel_threads=9;" + "enabled=false;use_zstd_dict_trainer=true}; ", + &new_cf_opt)); + ASSERT_EQ(new_cf_opt.compression_opts.window_bits, 5); + ASSERT_EQ(new_cf_opt.compression_opts.level, 6); + ASSERT_EQ(new_cf_opt.compression_opts.strategy, 7); + ASSERT_EQ(new_cf_opt.compression_opts.max_dict_bytes, 8u); + ASSERT_EQ(new_cf_opt.compression_opts.zstd_max_train_bytes, 9u); + ASSERT_EQ(new_cf_opt.compression_opts.parallel_threads, 10u); + ASSERT_EQ(new_cf_opt.compression_opts.enabled, true); + ASSERT_EQ(new_cf_opt.compression_opts.use_zstd_dict_trainer, false); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.window_bits, 4); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.level, 5); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.strategy, 6); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.max_dict_bytes, 7u); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.zstd_max_train_bytes, 8u); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.parallel_threads, 9u); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.enabled, false); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.use_zstd_dict_trainer, true); + + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "compression_opts={window_bits=4; strategy=5;};" + "bottommost_compression_opts={level=6; strategy=7;}", + &new_cf_opt)); + ASSERT_EQ(new_cf_opt.compression_opts.window_bits, 4); + ASSERT_EQ(new_cf_opt.compression_opts.strategy, 5); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.level, 6); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.strategy, 7); + + ASSERT_EQ(new_cf_opt.compression_opts.level, + base_cf_opt.compression_opts.level); + ASSERT_EQ(new_cf_opt.compression_opts.max_dict_bytes, + base_cf_opt.compression_opts.max_dict_bytes); + ASSERT_EQ(new_cf_opt.compression_opts.zstd_max_train_bytes, + base_cf_opt.compression_opts.zstd_max_train_bytes); + ASSERT_EQ(new_cf_opt.compression_opts.parallel_threads, + base_cf_opt.compression_opts.parallel_threads); + ASSERT_EQ(new_cf_opt.compression_opts.enabled, + base_cf_opt.compression_opts.enabled); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.window_bits, + base_cf_opt.bottommost_compression_opts.window_bits); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.max_dict_bytes, + base_cf_opt.bottommost_compression_opts.max_dict_bytes); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.zstd_max_train_bytes, + base_cf_opt.bottommost_compression_opts.zstd_max_train_bytes); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.parallel_threads, + base_cf_opt.bottommost_compression_opts.parallel_threads); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.enabled, + base_cf_opt.bottommost_compression_opts.enabled); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.use_zstd_dict_trainer, + base_cf_opt.bottommost_compression_opts.use_zstd_dict_trainer); + + // Test a few individual struct values + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "compression_opts.enabled=false; " + "bottommost_compression_opts.enabled=true; ", + &new_cf_opt)); + ASSERT_EQ(new_cf_opt.compression_opts.enabled, false); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.enabled, true); + + // Now test some illegal values + ConfigOptions ignore; + ignore.ignore_unknown_options = true; + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, ColumnFamilyOptions(), + "compression_opts=5:6:7:8:9:x:false", &base_cf_opt)); + ASSERT_OK(GetColumnFamilyOptionsFromString( + ignore, ColumnFamilyOptions(), "compression_opts=5:6:7:8:9:x:false", + &base_cf_opt)); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, ColumnFamilyOptions(), + "compression_opts=1:2:3:4:5:6:true:8", &base_cf_opt)); + ASSERT_OK(GetColumnFamilyOptionsFromString( + ignore, ColumnFamilyOptions(), "compression_opts=1:2:3:4:5:6:true:8", + &base_cf_opt)); + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, ColumnFamilyOptions(), + "compression_opts=1:2:3:4:5:6:true:8:9", &base_cf_opt)); + ASSERT_OK(GetColumnFamilyOptionsFromString( + ignore, ColumnFamilyOptions(), "compression_opts=1:2:3:4:5:6:true:8:9", + &base_cf_opt)); + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, ColumnFamilyOptions(), "compression_opts={unknown=bad;}", + &base_cf_opt)); + ASSERT_OK(GetColumnFamilyOptionsFromString(ignore, ColumnFamilyOptions(), + "compression_opts={unknown=bad;}", + &base_cf_opt)); + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, ColumnFamilyOptions(), "compression_opts.unknown=bad", + &base_cf_opt)); + ASSERT_OK(GetColumnFamilyOptionsFromString(ignore, ColumnFamilyOptions(), + "compression_opts.unknown=bad", + &base_cf_opt)); +} + +TEST_F(OptionsTest, OldInterfaceTest) { + ColumnFamilyOptions base_cf_opt; + ColumnFamilyOptions new_cf_opt; + ConfigOptions exact; + ConfigOptions cf_config_options; + cf_config_options.input_strings_escaped = false; + cf_config_options.ignore_unknown_options = false; + ASSERT_OK(GetColumnFamilyOptionsFromString( + cf_config_options, base_cf_opt, + "write_buffer_size=18;prefix_extractor=capped:8;" + "arena_block_size=19", + &new_cf_opt)); + + ASSERT_EQ(new_cf_opt.write_buffer_size, 18); + ASSERT_EQ(new_cf_opt.arena_block_size, 19); + ASSERT_TRUE(new_cf_opt.prefix_extractor.get() != nullptr); + + // And with a bad option + ASSERT_NOK(GetColumnFamilyOptionsFromString( + cf_config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={xx_block_size=4;}", + &new_cf_opt)); + ASSERT_OK( + RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); + + std::unordered_map cf_options_map = { + {"write_buffer_size", "1"}, + {"max_write_buffer_number", "2"}, + {"min_write_buffer_number_to_merge", "3"}, + }; + ASSERT_OK(GetColumnFamilyOptionsFromMap(cf_config_options, base_cf_opt, + cf_options_map, &new_cf_opt)); + cf_options_map["unknown_option"] = "1"; + ASSERT_NOK(GetColumnFamilyOptionsFromMap(cf_config_options, base_cf_opt, + cf_options_map, &new_cf_opt)); + ASSERT_OK( + RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); + cf_config_options.input_strings_escaped = true; + cf_config_options.ignore_unknown_options = true; + ASSERT_OK(GetColumnFamilyOptionsFromMap(cf_config_options, base_cf_opt, + cf_options_map, &new_cf_opt)); + + DBOptions base_db_opt; + DBOptions new_db_opt; + std::unordered_map db_options_map = { + {"create_if_missing", "false"}, + {"create_missing_column_families", "true"}, + {"error_if_exists", "false"}, + {"paranoid_checks", "true"}, + {"track_and_verify_wals_in_manifest", "true"}, + {"verify_sst_unique_id_in_manifest", "true"}, + {"max_open_files", "32"}, + }; + + ConfigOptions db_config_options(base_db_opt); + db_config_options.input_strings_escaped = false; + db_config_options.ignore_unknown_options = false; + ASSERT_OK(GetDBOptionsFromMap(db_config_options, base_db_opt, db_options_map, + &new_db_opt)); + ASSERT_EQ(new_db_opt.create_if_missing, false); + ASSERT_EQ(new_db_opt.create_missing_column_families, true); + ASSERT_EQ(new_db_opt.error_if_exists, false); + ASSERT_EQ(new_db_opt.paranoid_checks, true); + ASSERT_EQ(new_db_opt.track_and_verify_wals_in_manifest, true); + ASSERT_EQ(new_db_opt.verify_sst_unique_id_in_manifest, true); + ASSERT_EQ(new_db_opt.max_open_files, 32); + db_options_map["unknown_option"] = "1"; + Status s = GetDBOptionsFromMap(db_config_options, base_db_opt, db_options_map, + &new_db_opt); + ASSERT_NOK(s); + ASSERT_TRUE(s.IsInvalidArgument()); + + ASSERT_OK( + RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt)); + db_config_options.input_strings_escaped = true; + db_config_options.ignore_unknown_options = true; + ASSERT_OK(GetDBOptionsFromMap(db_config_options, base_db_opt, db_options_map, + &new_db_opt)); + db_config_options.input_strings_escaped = false; + db_config_options.ignore_unknown_options = false; + ASSERT_OK(GetDBOptionsFromString( + db_config_options, base_db_opt, + "create_if_missing=false;error_if_exists=false;max_open_files=42;", + &new_db_opt)); + ASSERT_EQ(new_db_opt.create_if_missing, false); + ASSERT_EQ(new_db_opt.error_if_exists, false); + ASSERT_EQ(new_db_opt.max_open_files, 42); + s = GetDBOptionsFromString( + db_config_options, base_db_opt, + "create_if_missing=false;error_if_exists=false;max_open_files=42;" + "unknown_option=1;", + &new_db_opt); + ASSERT_NOK(s); + ASSERT_TRUE(s.IsInvalidArgument()); + ASSERT_OK( + RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt)); +} + + +TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) { + BlockBasedTableOptions table_opt; + BlockBasedTableOptions new_opt; + ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + config_options.ignore_unsupported_options = false; + + // make sure default values are overwritten by something else + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "cache_index_and_filter_blocks=1;index_type=kHashSearch;" + "checksum=kxxHash;" + "block_cache=1M;block_cache_compressed=1k;block_size=1024;" + "block_size_deviation=8;block_restart_interval=4;" + "format_version=5;whole_key_filtering=1;" + "filter_policy=bloomfilter:4.567:false;detect_filter_construct_" + "corruption=true;" + // A bug caused read_amp_bytes_per_bit to be a large integer in OPTIONS + // file generated by 6.10 to 6.14. Though bug is fixed in these releases, + // we need to handle the case of loading OPTIONS file generated before the + // fix. + "read_amp_bytes_per_bit=17179869185;", + &new_opt)); + ASSERT_TRUE(new_opt.cache_index_and_filter_blocks); + ASSERT_EQ(new_opt.index_type, BlockBasedTableOptions::kHashSearch); + ASSERT_EQ(new_opt.checksum, ChecksumType::kxxHash); + ASSERT_TRUE(new_opt.block_cache != nullptr); + ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL); + ASSERT_EQ(new_opt.block_size, 1024UL); + ASSERT_EQ(new_opt.block_size_deviation, 8); + ASSERT_EQ(new_opt.block_restart_interval, 4); + ASSERT_EQ(new_opt.format_version, 5U); + ASSERT_EQ(new_opt.whole_key_filtering, true); + ASSERT_EQ(new_opt.detect_filter_construct_corruption, true); + ASSERT_TRUE(new_opt.filter_policy != nullptr); + auto bfp = new_opt.filter_policy->CheckedCast(); + ASSERT_NE(bfp, nullptr); + EXPECT_EQ(bfp->GetMillibitsPerKey(), 4567); + EXPECT_EQ(bfp->GetWholeBitsPerKey(), 5); + // Verify that only the lower 32bits are stored in + // new_opt.read_amp_bytes_per_bit. + EXPECT_EQ(1U, new_opt.read_amp_bytes_per_bit); + + // unknown option + Status s = GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "cache_index_and_filter_blocks=1;index_type=kBinarySearch;" + "bad_option=1", + &new_opt); + ASSERT_NOK(s); + ASSERT_TRUE(s.IsInvalidArgument()); + ASSERT_EQ(static_cast(table_opt.cache_index_and_filter_blocks), + new_opt.cache_index_and_filter_blocks); + ASSERT_EQ(table_opt.index_type, new_opt.index_type); + + // unrecognized index type + s = GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "cache_index_and_filter_blocks=1;index_type=kBinarySearchXX", &new_opt); + ASSERT_NOK(s); + ASSERT_TRUE(s.IsInvalidArgument()); + ASSERT_EQ(table_opt.cache_index_and_filter_blocks, + new_opt.cache_index_and_filter_blocks); + ASSERT_EQ(table_opt.index_type, new_opt.index_type); + + // unrecognized checksum type + ASSERT_NOK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "cache_index_and_filter_blocks=1;checksum=kxxHashXX", &new_opt)); + ASSERT_EQ(table_opt.cache_index_and_filter_blocks, + new_opt.cache_index_and_filter_blocks); + ASSERT_EQ(table_opt.index_type, new_opt.index_type); + + // unrecognized filter policy name + s = GetBlockBasedTableOptionsFromString(config_options, table_opt, + "filter_policy=bloomfilterxx:4:true", + &new_opt); + ASSERT_NOK(s); + ASSERT_TRUE(s.IsInvalidArgument()); + + // missing bits per key + s = GetBlockBasedTableOptionsFromString( + config_options, table_opt, "filter_policy=bloomfilter", &new_opt); + ASSERT_NOK(s); + ASSERT_TRUE(s.IsInvalidArgument()); + + // Used to be rejected, now accepted + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, "filter_policy=bloomfilter:4", &new_opt)); + bfp = dynamic_cast(new_opt.filter_policy.get()); + EXPECT_EQ(bfp->GetMillibitsPerKey(), 4000); + EXPECT_EQ(bfp->GetWholeBitsPerKey(), 4); + + // use_block_based_builder=true now ignored in public API (same as false) + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, "filter_policy=bloomfilter:4:true", &new_opt)); + bfp = dynamic_cast(new_opt.filter_policy.get()); + EXPECT_EQ(bfp->GetMillibitsPerKey(), 4000); + EXPECT_EQ(bfp->GetWholeBitsPerKey(), 4); + + // Test configuring using other internal names + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "filter_policy=rocksdb.internal.LegacyBloomFilter:3", &new_opt)); + auto builtin = + dynamic_cast(new_opt.filter_policy.get()); + EXPECT_EQ(builtin->GetId(), "rocksdb.internal.LegacyBloomFilter:3"); + + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "filter_policy=rocksdb.internal.FastLocalBloomFilter:1.234", &new_opt)); + builtin = + dynamic_cast(new_opt.filter_policy.get()); + EXPECT_EQ(builtin->GetId(), "rocksdb.internal.FastLocalBloomFilter:1.234"); + + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "filter_policy=rocksdb.internal.Standard128RibbonFilter:1.234", + &new_opt)); + builtin = + dynamic_cast(new_opt.filter_policy.get()); + EXPECT_EQ(builtin->GetId(), "rocksdb.internal.Standard128RibbonFilter:1.234"); + + // Ribbon filter policy (no Bloom hybrid) + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, "filter_policy=ribbonfilter:5.678:-1;", + &new_opt)); + ASSERT_TRUE(new_opt.filter_policy != nullptr); + auto rfp = + dynamic_cast(new_opt.filter_policy.get()); + EXPECT_EQ(rfp->GetMillibitsPerKey(), 5678); + EXPECT_EQ(rfp->GetBloomBeforeLevel(), -1); + + // Ribbon filter policy (default Bloom hybrid) + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, "filter_policy=ribbonfilter:6.789;", + &new_opt)); + ASSERT_TRUE(new_opt.filter_policy != nullptr); + rfp = dynamic_cast(new_opt.filter_policy.get()); + EXPECT_EQ(rfp->GetMillibitsPerKey(), 6789); + EXPECT_EQ(rfp->GetBloomBeforeLevel(), 0); + + // Ribbon filter policy (custom Bloom hybrid) + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, "filter_policy=ribbonfilter:6.789:5;", + &new_opt)); + ASSERT_TRUE(new_opt.filter_policy != nullptr); + rfp = dynamic_cast(new_opt.filter_policy.get()); + EXPECT_EQ(rfp->GetMillibitsPerKey(), 6789); + EXPECT_EQ(rfp->GetBloomBeforeLevel(), 5); + + // Check block cache options are overwritten when specified + // in new format as a struct. + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "block_cache={capacity=1M;num_shard_bits=4;" + "strict_capacity_limit=true;high_pri_pool_ratio=0.5;};" + "block_cache_compressed={capacity=1M;num_shard_bits=4;" + "strict_capacity_limit=true;high_pri_pool_ratio=0.5;}", + &new_opt)); + ASSERT_TRUE(new_opt.block_cache != nullptr); + ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL); + ASSERT_EQ(std::dynamic_pointer_cast(new_opt.block_cache) + ->GetNumShardBits(), + 4); + ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true); + ASSERT_EQ(std::dynamic_pointer_cast( + new_opt.block_cache)->GetHighPriPoolRatio(), 0.5); + + // Set only block cache capacity. Check other values are + // reset to default values. + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "block_cache={capacity=2M};" + "block_cache_compressed={capacity=2M}", + &new_opt)); + ASSERT_TRUE(new_opt.block_cache != nullptr); + ASSERT_EQ(new_opt.block_cache->GetCapacity(), 2*1024UL*1024UL); + // Default values + ASSERT_EQ(std::dynamic_pointer_cast(new_opt.block_cache) + ->GetNumShardBits(), + GetDefaultCacheShardBits(new_opt.block_cache->GetCapacity())); + ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false); + ASSERT_EQ(std::dynamic_pointer_cast(new_opt.block_cache) + ->GetHighPriPoolRatio(), + 0.5); + + // Set couple of block cache options. + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "block_cache={num_shard_bits=5;high_pri_pool_ratio=0.5;};" + "block_cache_compressed={num_shard_bits=5;" + "high_pri_pool_ratio=0.0;}", + &new_opt)); + ASSERT_EQ(new_opt.block_cache->GetCapacity(), 0); + ASSERT_EQ(std::dynamic_pointer_cast(new_opt.block_cache) + ->GetNumShardBits(), + 5); + ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false); + ASSERT_EQ(std::dynamic_pointer_cast( + new_opt.block_cache)->GetHighPriPoolRatio(), 0.5); + + // Set couple of block cache options. + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "block_cache={capacity=1M;num_shard_bits=4;" + "strict_capacity_limit=true;};" + "block_cache_compressed={capacity=1M;num_shard_bits=4;" + "strict_capacity_limit=true;}", + &new_opt)); + ASSERT_TRUE(new_opt.block_cache != nullptr); + ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL); + ASSERT_EQ(std::dynamic_pointer_cast(new_opt.block_cache) + ->GetNumShardBits(), + 4); + ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true); + ASSERT_EQ(std::dynamic_pointer_cast(new_opt.block_cache) + ->GetHighPriPoolRatio(), + 0.5); + + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, "filter_policy=rocksdb.BloomFilter:1.234", + &new_opt)); + ASSERT_TRUE(new_opt.filter_policy != nullptr); + ASSERT_TRUE( + new_opt.filter_policy->IsInstanceOf(BloomFilterPolicy::kClassName())); + ASSERT_TRUE( + new_opt.filter_policy->IsInstanceOf(BloomFilterPolicy::kNickName())); + + // Ribbon filter policy alternative name + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, "filter_policy=rocksdb.RibbonFilter:6.789:5;", + &new_opt)); + ASSERT_TRUE(new_opt.filter_policy != nullptr); + ASSERT_TRUE( + new_opt.filter_policy->IsInstanceOf(RibbonFilterPolicy::kClassName())); + ASSERT_TRUE( + new_opt.filter_policy->IsInstanceOf(RibbonFilterPolicy::kNickName())); +} + + +TEST_F(OptionsTest, GetPlainTableOptionsFromString) { + PlainTableOptions table_opt; + PlainTableOptions new_opt; + ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + // make sure default values are overwritten by something else + ASSERT_OK(GetPlainTableOptionsFromString( + config_options, table_opt, + "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" + "index_sparseness=8;huge_page_tlb_size=4;encoding_type=kPrefix;" + "full_scan_mode=true;store_index_in_file=true", + &new_opt)); + ASSERT_EQ(new_opt.user_key_len, 66u); + ASSERT_EQ(new_opt.bloom_bits_per_key, 20); + ASSERT_EQ(new_opt.hash_table_ratio, 0.5); + ASSERT_EQ(new_opt.index_sparseness, 8); + ASSERT_EQ(new_opt.huge_page_tlb_size, 4); + ASSERT_EQ(new_opt.encoding_type, EncodingType::kPrefix); + ASSERT_TRUE(new_opt.full_scan_mode); + ASSERT_TRUE(new_opt.store_index_in_file); + + // unknown option + Status s = GetPlainTableOptionsFromString( + config_options, table_opt, + "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" + "bad_option=1", + &new_opt); + ASSERT_NOK(s); + ASSERT_TRUE(s.IsInvalidArgument()); + + // unrecognized EncodingType + s = GetPlainTableOptionsFromString( + config_options, table_opt, + "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" + "encoding_type=kPrefixXX", + &new_opt); + ASSERT_NOK(s); + ASSERT_TRUE(s.IsInvalidArgument()); +} + +TEST_F(OptionsTest, GetMemTableRepFactoryFromString) { + std::unique_ptr new_mem_factory = nullptr; + + ASSERT_OK(GetMemTableRepFactoryFromString("skip_list", &new_mem_factory)); + ASSERT_OK(GetMemTableRepFactoryFromString("skip_list:16", &new_mem_factory)); + ASSERT_STREQ(new_mem_factory->Name(), "SkipListFactory"); + ASSERT_NOK(GetMemTableRepFactoryFromString("skip_list:16:invalid_opt", + &new_mem_factory)); + + ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash", &new_mem_factory)); + ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash:1000", + &new_mem_factory)); + ASSERT_STREQ(new_mem_factory->Name(), "HashSkipListRepFactory"); + ASSERT_NOK(GetMemTableRepFactoryFromString("prefix_hash:1000:invalid_opt", + &new_mem_factory)); + + ASSERT_OK(GetMemTableRepFactoryFromString("hash_linkedlist", + &new_mem_factory)); + ASSERT_OK(GetMemTableRepFactoryFromString("hash_linkedlist:1000", + &new_mem_factory)); + ASSERT_EQ(std::string(new_mem_factory->Name()), "HashLinkListRepFactory"); + ASSERT_NOK(GetMemTableRepFactoryFromString("hash_linkedlist:1000:invalid_opt", + &new_mem_factory)); + + ASSERT_OK(GetMemTableRepFactoryFromString("vector", &new_mem_factory)); + ASSERT_OK(GetMemTableRepFactoryFromString("vector:1024", &new_mem_factory)); + ASSERT_EQ(std::string(new_mem_factory->Name()), "VectorRepFactory"); + ASSERT_NOK(GetMemTableRepFactoryFromString("vector:1024:invalid_opt", + &new_mem_factory)); + + ASSERT_NOK(GetMemTableRepFactoryFromString("cuckoo", &new_mem_factory)); + // CuckooHash memtable is already removed. + ASSERT_NOK(GetMemTableRepFactoryFromString("cuckoo:1024", &new_mem_factory)); + + ASSERT_NOK(GetMemTableRepFactoryFromString("bad_factory", &new_mem_factory)); +} + +TEST_F(OptionsTest, MemTableRepFactoryCreateFromString) { + std::unique_ptr new_mem_factory = nullptr; + ConfigOptions config_options; + config_options.ignore_unsupported_options = false; + config_options.ignore_unknown_options = false; + + ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "skip_list", + &new_mem_factory)); + ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "skip_list:16", + &new_mem_factory)); + ASSERT_STREQ(new_mem_factory->Name(), "SkipListFactory"); + ASSERT_TRUE(new_mem_factory->IsInstanceOf("skip_list")); + ASSERT_TRUE(new_mem_factory->IsInstanceOf("SkipListFactory")); + ASSERT_NOK(MemTableRepFactory::CreateFromString( + config_options, "skip_list:16:invalid_opt", &new_mem_factory)); + + ASSERT_NOK(MemTableRepFactory::CreateFromString( + config_options, "invalid_opt=10", &new_mem_factory)); + + // Test a reset + ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "", + &new_mem_factory)); + ASSERT_EQ(new_mem_factory, nullptr); + ASSERT_NOK(MemTableRepFactory::CreateFromString( + config_options, "invalid_opt=10", &new_mem_factory)); + + ASSERT_OK(MemTableRepFactory::CreateFromString( + config_options, "id=skip_list; lookahead=32", &new_mem_factory)); + ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "prefix_hash", + &new_mem_factory)); + ASSERT_OK(MemTableRepFactory::CreateFromString( + config_options, "prefix_hash:1000", &new_mem_factory)); + ASSERT_STREQ(new_mem_factory->Name(), "HashSkipListRepFactory"); + ASSERT_TRUE(new_mem_factory->IsInstanceOf("prefix_hash")); + ASSERT_TRUE(new_mem_factory->IsInstanceOf("HashSkipListRepFactory")); + ASSERT_NOK(MemTableRepFactory::CreateFromString( + config_options, "prefix_hash:1000:invalid_opt", &new_mem_factory)); + ASSERT_OK(MemTableRepFactory::CreateFromString( + config_options, + "id=prefix_hash; bucket_count=32; skiplist_height=64; " + "branching_factor=16", + &new_mem_factory)); + ASSERT_NOK(MemTableRepFactory::CreateFromString( + config_options, + "id=prefix_hash; bucket_count=32; skiplist_height=64; " + "branching_factor=16; invalid=unknown", + &new_mem_factory)); + + ASSERT_OK(MemTableRepFactory::CreateFromString( + config_options, "hash_linkedlist", &new_mem_factory)); + ASSERT_OK(MemTableRepFactory::CreateFromString( + config_options, "hash_linkedlist:1000", &new_mem_factory)); + ASSERT_STREQ(new_mem_factory->Name(), "HashLinkListRepFactory"); + ASSERT_TRUE(new_mem_factory->IsInstanceOf("hash_linkedlist")); + ASSERT_TRUE(new_mem_factory->IsInstanceOf("HashLinkListRepFactory")); + ASSERT_NOK(MemTableRepFactory::CreateFromString( + config_options, "hash_linkedlist:1000:invalid_opt", &new_mem_factory)); + ASSERT_OK(MemTableRepFactory::CreateFromString( + config_options, + "id=hash_linkedlist; bucket_count=32; threshold=64; huge_page_size=16; " + "logging_threshold=12; log_when_flash=true", + &new_mem_factory)); + ASSERT_NOK(MemTableRepFactory::CreateFromString( + config_options, + "id=hash_linkedlist; bucket_count=32; threshold=64; huge_page_size=16; " + "logging_threshold=12; log_when_flash=true; invalid=unknown", + &new_mem_factory)); + + ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "vector", + &new_mem_factory)); + ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "vector:1024", + &new_mem_factory)); + ASSERT_STREQ(new_mem_factory->Name(), "VectorRepFactory"); + ASSERT_TRUE(new_mem_factory->IsInstanceOf("vector")); + ASSERT_TRUE(new_mem_factory->IsInstanceOf("VectorRepFactory")); + ASSERT_NOK(MemTableRepFactory::CreateFromString( + config_options, "vector:1024:invalid_opt", &new_mem_factory)); + ASSERT_OK(MemTableRepFactory::CreateFromString( + config_options, "id=vector; count=42", &new_mem_factory)); + ASSERT_NOK(MemTableRepFactory::CreateFromString( + config_options, "id=vector; invalid=unknown", &new_mem_factory)); + ASSERT_NOK(MemTableRepFactory::CreateFromString(config_options, "cuckoo", + &new_mem_factory)); + // CuckooHash memtable is already removed. + ASSERT_NOK(MemTableRepFactory::CreateFromString(config_options, "cuckoo:1024", + &new_mem_factory)); + + ASSERT_NOK(MemTableRepFactory::CreateFromString(config_options, "bad_factory", + &new_mem_factory)); +} + +class CustomEnv : public EnvWrapper { + public: + explicit CustomEnv(Env* _target) : EnvWrapper(_target) {} + static const char* kClassName() { return "CustomEnv"; } + const char* Name() const override { return kClassName(); } +}; + +TEST_F(OptionsTest, GetOptionsFromStringTest) { + Options base_options, new_options; + ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + + base_options.write_buffer_size = 20; + base_options.min_write_buffer_number_to_merge = 15; + BlockBasedTableOptions block_based_table_options; + block_based_table_options.cache_index_and_filter_blocks = true; + base_options.table_factory.reset( + NewBlockBasedTableFactory(block_based_table_options)); + + // Register an Env with object registry. + ObjectLibrary::Default()->AddFactory( + CustomEnv::kClassName(), + [](const std::string& /*name*/, std::unique_ptr* /*env_guard*/, + std::string* /* errmsg */) { + static CustomEnv env(Env::Default()); + return &env; + }); + + ASSERT_OK(GetOptionsFromString( + config_options, base_options, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_cache=1M;block_size=4;};" + "compression_opts=4:5:6;create_if_missing=true;max_open_files=1;" + "bottommost_compression_opts=5:6:7;create_if_missing=true;max_open_files=" + "1;" + "rate_limiter_bytes_per_sec=1024;env=CustomEnv", + &new_options)); + + ASSERT_EQ(new_options.compression_opts.window_bits, 4); + ASSERT_EQ(new_options.compression_opts.level, 5); + ASSERT_EQ(new_options.compression_opts.strategy, 6); + ASSERT_EQ(new_options.compression_opts.max_dict_bytes, 0u); + ASSERT_EQ(new_options.compression_opts.zstd_max_train_bytes, 0u); + ASSERT_EQ(new_options.compression_opts.parallel_threads, 1u); + ASSERT_EQ(new_options.compression_opts.enabled, false); + ASSERT_EQ(new_options.compression_opts.use_zstd_dict_trainer, true); + ASSERT_EQ(new_options.bottommost_compression, kDisableCompressionOption); + ASSERT_EQ(new_options.bottommost_compression_opts.window_bits, 5); + ASSERT_EQ(new_options.bottommost_compression_opts.level, 6); + ASSERT_EQ(new_options.bottommost_compression_opts.strategy, 7); + ASSERT_EQ(new_options.bottommost_compression_opts.max_dict_bytes, 0u); + ASSERT_EQ(new_options.bottommost_compression_opts.zstd_max_train_bytes, 0u); + ASSERT_EQ(new_options.bottommost_compression_opts.parallel_threads, 1u); + ASSERT_EQ(new_options.bottommost_compression_opts.enabled, false); + ASSERT_EQ(new_options.bottommost_compression_opts.use_zstd_dict_trainer, + true); + ASSERT_EQ(new_options.write_buffer_size, 10U); + ASSERT_EQ(new_options.max_write_buffer_number, 16); + const auto new_bbto = + new_options.table_factory->GetOptions(); + ASSERT_NE(new_bbto, nullptr); + ASSERT_EQ(new_bbto->block_cache->GetCapacity(), 1U << 20); + ASSERT_EQ(new_bbto->block_size, 4U); + // don't overwrite block based table options + ASSERT_TRUE(new_bbto->cache_index_and_filter_blocks); + + ASSERT_EQ(new_options.create_if_missing, true); + ASSERT_EQ(new_options.max_open_files, 1); + ASSERT_TRUE(new_options.rate_limiter.get() != nullptr); + Env* newEnv = new_options.env; + ASSERT_OK(Env::CreateFromString({}, CustomEnv::kClassName(), &newEnv)); + ASSERT_EQ(newEnv, new_options.env); + + config_options.ignore_unknown_options = false; + // Test a bad value for a DBOption returns a failure + base_options.dump_malloc_stats = false; + base_options.write_buffer_size = 1024; + Options bad_options = new_options; + Status s = GetOptionsFromString(config_options, base_options, + "create_if_missing=XX;dump_malloc_stats=true", + &bad_options); + ASSERT_NOK(s); + ASSERT_TRUE(s.IsInvalidArgument()); + ASSERT_EQ(bad_options.dump_malloc_stats, false); + + bad_options = new_options; + s = GetOptionsFromString(config_options, base_options, + "write_buffer_size=XX;dump_malloc_stats=true", + &bad_options); + ASSERT_NOK(s); + ASSERT_TRUE(s.IsInvalidArgument()); + + ASSERT_EQ(bad_options.dump_malloc_stats, false); + + // Test a bad value for a TableFactory Option returns a failure + bad_options = new_options; + s = GetOptionsFromString(config_options, base_options, + "write_buffer_size=16;dump_malloc_stats=true" + "block_based_table_factory={block_size=XX;};", + &bad_options); + ASSERT_TRUE(s.IsInvalidArgument()); + ASSERT_EQ(bad_options.dump_malloc_stats, false); + ASSERT_EQ(bad_options.write_buffer_size, 1024); + + config_options.ignore_unknown_options = true; + ASSERT_OK(GetOptionsFromString(config_options, base_options, + "create_if_missing=XX;dump_malloc_stats=true;" + "write_buffer_size=XX;" + "block_based_table_factory={block_size=XX;};", + &bad_options)); + ASSERT_EQ(bad_options.create_if_missing, base_options.create_if_missing); + ASSERT_EQ(bad_options.dump_malloc_stats, true); + ASSERT_EQ(bad_options.write_buffer_size, base_options.write_buffer_size); + + // Test the old interface + ASSERT_OK(GetOptionsFromString( + base_options, + "write_buffer_size=22;max_write_buffer_number=33;max_open_files=44;", + &new_options)); + ASSERT_EQ(new_options.write_buffer_size, 22U); + ASSERT_EQ(new_options.max_write_buffer_number, 33); + ASSERT_EQ(new_options.max_open_files, 44); +} + +TEST_F(OptionsTest, DBOptionsSerialization) { + Options base_options, new_options; + Random rnd(301); + ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + + // Phase 1: Make big change in base_options + test::RandomInitDBOptions(&base_options, &rnd); + + // Phase 2: obtain a string from base_option + std::string base_options_file_content; + ASSERT_OK(GetStringFromDBOptions(config_options, base_options, + &base_options_file_content)); + + // Phase 3: Set new_options from the derived string and expect + // new_options == base_options + ASSERT_OK(GetDBOptionsFromString(config_options, DBOptions(), + base_options_file_content, &new_options)); + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(config_options, base_options, + new_options)); +} + +TEST_F(OptionsTest, OptionsComposeDecompose) { + // build an Options from DBOptions + CFOptions, then decompose it to verify + // we get same constituent options. + DBOptions base_db_opts; + ColumnFamilyOptions base_cf_opts; + ConfigOptions + config_options; // Use default for ignore(false) and check (exact) + config_options.input_strings_escaped = false; + + Random rnd(301); + test::RandomInitDBOptions(&base_db_opts, &rnd); + test::RandomInitCFOptions(&base_cf_opts, base_db_opts, &rnd); + + Options base_opts(base_db_opts, base_cf_opts); + DBOptions new_db_opts(base_opts); + ColumnFamilyOptions new_cf_opts(base_opts); + + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(config_options, base_db_opts, + new_db_opts)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opts, + new_cf_opts)); + delete new_cf_opts.compaction_filter; +} + +TEST_F(OptionsTest, DBOptionsComposeImmutable) { + // Build a DBOptions from an Immutable/Mutable one and verify that + // we get same constituent options. + ConfigOptions config_options; + Random rnd(301); + DBOptions base_opts, new_opts; + test::RandomInitDBOptions(&base_opts, &rnd); + MutableDBOptions m_opts(base_opts); + ImmutableDBOptions i_opts(base_opts); + new_opts = BuildDBOptions(i_opts, m_opts); + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(config_options, base_opts, + new_opts)); +} + +TEST_F(OptionsTest, GetMutableDBOptions) { + Random rnd(228); + DBOptions base_opts; + std::string opts_str; + std::unordered_map opts_map; + ConfigOptions config_options; + + test::RandomInitDBOptions(&base_opts, &rnd); + ImmutableDBOptions i_opts(base_opts); + MutableDBOptions m_opts(base_opts); + MutableDBOptions new_opts; + ASSERT_OK(GetStringFromMutableDBOptions(config_options, m_opts, &opts_str)); + ASSERT_OK(StringToMap(opts_str, &opts_map)); + ASSERT_OK(GetMutableDBOptionsFromStrings(m_opts, opts_map, &new_opts)); + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions( + config_options, base_opts, BuildDBOptions(i_opts, new_opts))); +} + +TEST_F(OptionsTest, CFOptionsComposeImmutable) { + // Build a DBOptions from an Immutable/Mutable one and verify that + // we get same constituent options. + ConfigOptions config_options; + Random rnd(301); + ColumnFamilyOptions base_opts, new_opts; + DBOptions dummy; // Needed to create ImmutableCFOptions + test::RandomInitCFOptions(&base_opts, dummy, &rnd); + MutableCFOptions m_opts(base_opts); + ImmutableCFOptions i_opts(base_opts); + UpdateColumnFamilyOptions(i_opts, &new_opts); + UpdateColumnFamilyOptions(m_opts, &new_opts); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_opts, + new_opts)); + delete new_opts.compaction_filter; +} + +TEST_F(OptionsTest, GetMutableCFOptions) { + Random rnd(228); + ColumnFamilyOptions base, copy; + std::string opts_str; + std::unordered_map opts_map; + ConfigOptions config_options; + DBOptions dummy; // Needed to create ImmutableCFOptions + + test::RandomInitCFOptions(&base, dummy, &rnd); + ColumnFamilyOptions result; + MutableCFOptions m_opts(base), new_opts; + + ASSERT_OK(GetStringFromMutableCFOptions(config_options, m_opts, &opts_str)); + ASSERT_OK(StringToMap(opts_str, &opts_map)); + ASSERT_OK(GetMutableOptionsFromStrings(m_opts, opts_map, nullptr, &new_opts)); + UpdateColumnFamilyOptions(ImmutableCFOptions(base), ©); + UpdateColumnFamilyOptions(new_opts, ©); + + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base, copy)); + delete copy.compaction_filter; +} + +TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) { + Options options; + ColumnFamilyOptions base_opt, new_opt; + Random rnd(302); + ConfigOptions config_options; + config_options.input_strings_escaped = false; + + // Phase 1: randomly assign base_opt + // custom type options + test::RandomInitCFOptions(&base_opt, options, &rnd); + + // Phase 2: obtain a string from base_opt + std::string base_options_file_content; + ASSERT_OK(GetStringFromColumnFamilyOptions(config_options, base_opt, + &base_options_file_content)); + + // Phase 3: Set new_opt from the derived string and expect + // new_opt == base_opt + ASSERT_OK( + GetColumnFamilyOptionsFromString(config_options, ColumnFamilyOptions(), + base_options_file_content, &new_opt)); + ASSERT_OK( + RocksDBOptionsParser::VerifyCFOptions(config_options, base_opt, new_opt)); + if (base_opt.compaction_filter) { + delete base_opt.compaction_filter; + } +} + +TEST_F(OptionsTest, CheckBlockBasedTableOptions) { + ColumnFamilyOptions cf_opts; + DBOptions db_opts; + ConfigOptions config_opts; + + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_opts, cf_opts, "prefix_extractor=capped:8", &cf_opts)); + ASSERT_OK(TableFactory::CreateFromString(config_opts, "BlockBasedTable", + &cf_opts.table_factory)); + ASSERT_NE(cf_opts.table_factory.get(), nullptr); + ASSERT_TRUE(cf_opts.table_factory->IsInstanceOf( + TableFactory::kBlockBasedTableName())); + auto bbto = cf_opts.table_factory->GetOptions(); + ASSERT_OK(cf_opts.table_factory->ConfigureFromString( + config_opts, + "block_cache={capacity=1M;num_shard_bits=4;};" + "block_size_deviation=101;" + "block_restart_interval=0;" + "index_block_restart_interval=5;" + "partition_filters=true;" + "index_type=kHashSearch;" + "no_block_cache=1;")); + ASSERT_NE(bbto, nullptr); + ASSERT_EQ(bbto->block_cache.get(), nullptr); + ASSERT_EQ(bbto->block_size_deviation, 0); + ASSERT_EQ(bbto->block_restart_interval, 1); + ASSERT_EQ(bbto->index_block_restart_interval, 1); + ASSERT_FALSE(bbto->partition_filters); + ASSERT_OK(TableFactory::CreateFromString(config_opts, "BlockBasedTable", + &cf_opts.table_factory)); + bbto = cf_opts.table_factory->GetOptions(); + + ASSERT_OK(cf_opts.table_factory->ConfigureFromString(config_opts, + "no_block_cache=0;")); + ASSERT_NE(bbto->block_cache.get(), nullptr); + ASSERT_OK(cf_opts.table_factory->ValidateOptions(db_opts, cf_opts)); +} + +TEST_F(OptionsTest, MutableTableOptions) { + ConfigOptions config_options; + std::shared_ptr bbtf; + bbtf.reset(NewBlockBasedTableFactory()); + auto bbto = bbtf->GetOptions(); + ASSERT_NE(bbto, nullptr); + ASSERT_OK(bbtf->ConfigureOption(config_options, "block_align", "true")); + ASSERT_OK(bbtf->ConfigureOption(config_options, "block_size", "1024")); + ASSERT_EQ(bbto->block_align, true); + ASSERT_EQ(bbto->block_size, 1024); + ASSERT_OK(bbtf->PrepareOptions(config_options)); + config_options.mutable_options_only = true; + ASSERT_OK(bbtf->ConfigureOption(config_options, "block_size", "1024")); + ASSERT_EQ(bbto->block_align, true); + ASSERT_NOK(bbtf->ConfigureOption(config_options, "block_align", "false")); + ASSERT_OK(bbtf->ConfigureOption(config_options, "block_size", "2048")); + ASSERT_EQ(bbto->block_align, true); + ASSERT_EQ(bbto->block_size, 2048); + + ColumnFamilyOptions cf_opts; + cf_opts.table_factory = bbtf; + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, cf_opts, "block_based_table_factory.block_align=false", + &cf_opts)); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, cf_opts, "block_based_table_factory.block_size=8192", + &cf_opts)); + ASSERT_EQ(bbto->block_align, true); + ASSERT_EQ(bbto->block_size, 8192); +} + +TEST_F(OptionsTest, MutableCFOptions) { + ConfigOptions config_options; + ColumnFamilyOptions cf_opts; + + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, cf_opts, + "paranoid_file_checks=true; block_based_table_factory.block_align=false; " + "block_based_table_factory.block_size=8192;", + &cf_opts)); + ASSERT_TRUE(cf_opts.paranoid_file_checks); + ASSERT_NE(cf_opts.table_factory.get(), nullptr); + const auto bbto = cf_opts.table_factory->GetOptions(); + ASSERT_NE(bbto, nullptr); + ASSERT_EQ(bbto->block_size, 8192); + ASSERT_EQ(bbto->block_align, false); + std::unordered_map unused_opts; + ASSERT_OK(GetColumnFamilyOptionsFromMap( + config_options, cf_opts, {{"paranoid_file_checks", "false"}}, &cf_opts)); + ASSERT_EQ(cf_opts.paranoid_file_checks, false); + + ASSERT_OK(GetColumnFamilyOptionsFromMap( + config_options, cf_opts, + {{"block_based_table_factory.block_size", "16384"}}, &cf_opts)); + ASSERT_EQ(bbto, cf_opts.table_factory->GetOptions()); + ASSERT_EQ(bbto->block_size, 16384); + + config_options.mutable_options_only = true; + // Force consistency checks is not mutable + ASSERT_NOK(GetColumnFamilyOptionsFromMap( + config_options, cf_opts, {{"force_consistency_checks", "true"}}, + &cf_opts)); + + // Attempt to change the table. It is not mutable, so this should fail and + // leave the original intact + ASSERT_NOK(GetColumnFamilyOptionsFromMap( + config_options, cf_opts, {{"table_factory", "PlainTable"}}, &cf_opts)); + ASSERT_NOK(GetColumnFamilyOptionsFromMap( + config_options, cf_opts, {{"table_factory.id", "PlainTable"}}, &cf_opts)); + ASSERT_NE(cf_opts.table_factory.get(), nullptr); + ASSERT_EQ(bbto, cf_opts.table_factory->GetOptions()); + + // Change the block size. Should update the value in the current table + ASSERT_OK(GetColumnFamilyOptionsFromMap( + config_options, cf_opts, + {{"block_based_table_factory.block_size", "8192"}}, &cf_opts)); + ASSERT_EQ(bbto, cf_opts.table_factory->GetOptions()); + ASSERT_EQ(bbto->block_size, 8192); + + // Attempt to turn off block cache fails, as this option is not mutable + ASSERT_NOK(GetColumnFamilyOptionsFromMap( + config_options, cf_opts, + {{"block_based_table_factory.no_block_cache", "true"}}, &cf_opts)); + ASSERT_EQ(bbto, cf_opts.table_factory->GetOptions()); + + // Attempt to change the block size via a config string/map. Should update + // the current value + ASSERT_OK(GetColumnFamilyOptionsFromMap( + config_options, cf_opts, + {{"block_based_table_factory", "{block_size=32768}"}}, &cf_opts)); + ASSERT_EQ(bbto, cf_opts.table_factory->GetOptions()); + ASSERT_EQ(bbto->block_size, 32768); + + // Attempt to change the block size and no cache through the map. Should + // fail, leaving the old values intact + ASSERT_NOK(GetColumnFamilyOptionsFromMap( + config_options, cf_opts, + {{"block_based_table_factory", + "{block_size=16384; no_block_cache=true}"}}, + &cf_opts)); + ASSERT_EQ(bbto, cf_opts.table_factory->GetOptions()); + ASSERT_EQ(bbto->block_size, 32768); +} + + +Status StringToMap( + const std::string& opts_str, + std::unordered_map* opts_map); + +TEST_F(OptionsTest, StringToMapTest) { + std::unordered_map opts_map; + // Regular options + ASSERT_OK(StringToMap("k1=v1;k2=v2;k3=v3", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], "v2"); + ASSERT_EQ(opts_map["k3"], "v3"); + // Value with '=' + opts_map.clear(); + ASSERT_OK(StringToMap("k1==v1;k2=v2=;", &opts_map)); + ASSERT_EQ(opts_map["k1"], "=v1"); + ASSERT_EQ(opts_map["k2"], "v2="); + // Overwrriten option + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k1=v2;k3=v3", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v2"); + ASSERT_EQ(opts_map["k3"], "v3"); + // Empty value + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2=;k3=v3;k4=", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_TRUE(opts_map.find("k2") != opts_map.end()); + ASSERT_EQ(opts_map["k2"], ""); + ASSERT_EQ(opts_map["k3"], "v3"); + ASSERT_TRUE(opts_map.find("k4") != opts_map.end()); + ASSERT_EQ(opts_map["k4"], ""); + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2=;k3=v3;k4= ", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_TRUE(opts_map.find("k2") != opts_map.end()); + ASSERT_EQ(opts_map["k2"], ""); + ASSERT_EQ(opts_map["k3"], "v3"); + ASSERT_TRUE(opts_map.find("k4") != opts_map.end()); + ASSERT_EQ(opts_map["k4"], ""); + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2=;k3=", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_TRUE(opts_map.find("k2") != opts_map.end()); + ASSERT_EQ(opts_map["k2"], ""); + ASSERT_TRUE(opts_map.find("k3") != opts_map.end()); + ASSERT_EQ(opts_map["k3"], ""); + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2=;k3=;", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_TRUE(opts_map.find("k2") != opts_map.end()); + ASSERT_EQ(opts_map["k2"], ""); + ASSERT_TRUE(opts_map.find("k3") != opts_map.end()); + ASSERT_EQ(opts_map["k3"], ""); + // Regular nested options + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2={nk1=nv1;nk2=nv2};k3=v3", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], "nk1=nv1;nk2=nv2"); + ASSERT_EQ(opts_map["k3"], "v3"); + // Multi-level nested options + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2={nk1=nv1;nk2={nnk1=nnk2}};" + "k3={nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}};k4=v4", + &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], "nk1=nv1;nk2={nnk1=nnk2}"); + ASSERT_EQ(opts_map["k3"], "nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}"); + ASSERT_EQ(opts_map["k4"], "v4"); + // Garbage inside curly braces + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2={dfad=};k3={=};k4=v4", + &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], "dfad="); + ASSERT_EQ(opts_map["k3"], "="); + ASSERT_EQ(opts_map["k4"], "v4"); + // Empty nested options + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2={};", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], ""); + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2={{{{}}}{}{}};", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], "{{{}}}{}{}"); + // With random spaces + opts_map.clear(); + ASSERT_OK(StringToMap(" k1 = v1 ; k2= {nk1=nv1; nk2={nnk1=nnk2}} ; " + "k3={ { } }; k4= v4 ", + &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], "nk1=nv1; nk2={nnk1=nnk2}"); + ASSERT_EQ(opts_map["k3"], "{ }"); + ASSERT_EQ(opts_map["k4"], "v4"); + + // Empty key + ASSERT_NOK(StringToMap("k1=v1;k2=v2;=", &opts_map)); + ASSERT_NOK(StringToMap("=v1;k2=v2", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2v2;", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2=v2;fadfa", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2=v2;;", &opts_map)); + // Mismatch curly braces + ASSERT_NOK(StringToMap("k1=v1;k2={;k3=v3", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={{};k3=v3", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={}};k3=v3", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={{}{}}};k3=v3", &opts_map)); + // However this is valid! + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2=};k3=v3", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], "}"); + ASSERT_EQ(opts_map["k3"], "v3"); + + // Invalid chars after closing curly brace + ASSERT_NOK(StringToMap("k1=v1;k2={{}}{};k3=v3", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={{}}cfda;k3=v3", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={{}} cfda;k3=v3", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={{}} cfda", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={{}}{}", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={{dfdl}adfa}{}", &opts_map)); +} + +TEST_F(OptionsTest, StringToMapRandomTest) { + std::unordered_map opts_map; + // Make sure segfault is not hit by semi-random strings + + std::vector bases = { + "a={aa={};tt={xxx={}}};c=defff", + "a={aa={};tt={xxx={}}};c=defff;d={{}yxx{}3{xx}}", + "abc={{}{}{}{{{}}}{{}{}{}{}{}{}{}"}; + + for (std::string base : bases) { + for (int rand_seed = 301; rand_seed < 401; rand_seed++) { + Random rnd(rand_seed); + for (int attempt = 0; attempt < 10; attempt++) { + std::string str = base; + // Replace random position to space + size_t pos = static_cast( + rnd.Uniform(static_cast(base.size()))); + str[pos] = ' '; + Status s = StringToMap(str, &opts_map); + ASSERT_TRUE(s.ok() || s.IsInvalidArgument()); + opts_map.clear(); + } + } + } + + // Random Construct a string + std::vector chars = {'{', '}', ' ', '=', ';', 'c'}; + for (int rand_seed = 301; rand_seed < 1301; rand_seed++) { + Random rnd(rand_seed); + int len = rnd.Uniform(30); + std::string str = ""; + for (int attempt = 0; attempt < len; attempt++) { + // Add a random character + size_t pos = static_cast( + rnd.Uniform(static_cast(chars.size()))); + str.append(1, chars[pos]); + } + Status s = StringToMap(str, &opts_map); + ASSERT_TRUE(s.ok() || s.IsInvalidArgument()); + s = StringToMap("name=" + str, &opts_map); + ASSERT_TRUE(s.ok() || s.IsInvalidArgument()); + opts_map.clear(); + } +} + +TEST_F(OptionsTest, GetStringFromCompressionType) { + std::string res; + + ASSERT_OK(GetStringFromCompressionType(&res, kNoCompression)); + ASSERT_EQ(res, "kNoCompression"); + + ASSERT_OK(GetStringFromCompressionType(&res, kSnappyCompression)); + ASSERT_EQ(res, "kSnappyCompression"); + + ASSERT_OK(GetStringFromCompressionType(&res, kDisableCompressionOption)); + ASSERT_EQ(res, "kDisableCompressionOption"); + + ASSERT_OK(GetStringFromCompressionType(&res, kLZ4Compression)); + ASSERT_EQ(res, "kLZ4Compression"); + + ASSERT_OK(GetStringFromCompressionType(&res, kZlibCompression)); + ASSERT_EQ(res, "kZlibCompression"); + + ASSERT_NOK( + GetStringFromCompressionType(&res, static_cast(-10))); +} + +TEST_F(OptionsTest, OnlyMutableDBOptions) { + std::string opt_str; + Random rnd(302); + ConfigOptions cfg_opts; + DBOptions db_opts; + DBOptions mdb_opts; + std::unordered_set m_names; + std::unordered_set a_names; + + test::RandomInitDBOptions(&db_opts, &rnd); + auto db_config = DBOptionsAsConfigurable(db_opts); + + // Get all of the DB Option names (mutable or not) + ASSERT_OK(db_config->GetOptionNames(cfg_opts, &a_names)); + + // Get only the mutable options from db_opts and set those in mdb_opts + cfg_opts.mutable_options_only = true; + + // Get only the Mutable DB Option names + ASSERT_OK(db_config->GetOptionNames(cfg_opts, &m_names)); + ASSERT_OK(GetStringFromDBOptions(cfg_opts, db_opts, &opt_str)); + ASSERT_OK(GetDBOptionsFromString(cfg_opts, mdb_opts, opt_str, &mdb_opts)); + std::string mismatch; + // Comparing only the mutable options, the two are equivalent + auto mdb_config = DBOptionsAsConfigurable(mdb_opts); + ASSERT_TRUE(mdb_config->AreEquivalent(cfg_opts, db_config.get(), &mismatch)); + ASSERT_TRUE(db_config->AreEquivalent(cfg_opts, mdb_config.get(), &mismatch)); + + ASSERT_GT(a_names.size(), m_names.size()); + for (const auto& n : m_names) { + std::string m, d; + ASSERT_OK(mdb_config->GetOption(cfg_opts, n, &m)); + ASSERT_OK(db_config->GetOption(cfg_opts, n, &d)); + ASSERT_EQ(m, d); + } + + cfg_opts.mutable_options_only = false; + // Comparing all of the options, the two are not equivalent + ASSERT_FALSE(mdb_config->AreEquivalent(cfg_opts, db_config.get(), &mismatch)); + ASSERT_FALSE(db_config->AreEquivalent(cfg_opts, mdb_config.get(), &mismatch)); + + // Make sure there are only mutable options being configured + ASSERT_OK(GetDBOptionsFromString(cfg_opts, DBOptions(), opt_str, &db_opts)); +} + +TEST_F(OptionsTest, OnlyMutableCFOptions) { + std::string opt_str; + Random rnd(302); + ConfigOptions cfg_opts; + DBOptions db_opts; + ColumnFamilyOptions mcf_opts; + ColumnFamilyOptions cf_opts; + std::unordered_set m_names; + std::unordered_set a_names; + + test::RandomInitCFOptions(&cf_opts, db_opts, &rnd); + cf_opts.comparator = ReverseBytewiseComparator(); + auto cf_config = CFOptionsAsConfigurable(cf_opts); + + // Get all of the CF Option names (mutable or not) + ASSERT_OK(cf_config->GetOptionNames(cfg_opts, &a_names)); + + // Get only the mutable options from cf_opts and set those in mcf_opts + cfg_opts.mutable_options_only = true; + // Get only the Mutable CF Option names + ASSERT_OK(cf_config->GetOptionNames(cfg_opts, &m_names)); + ASSERT_OK(GetStringFromColumnFamilyOptions(cfg_opts, cf_opts, &opt_str)); + ASSERT_OK( + GetColumnFamilyOptionsFromString(cfg_opts, mcf_opts, opt_str, &mcf_opts)); + std::string mismatch; + + auto mcf_config = CFOptionsAsConfigurable(mcf_opts); + // Comparing only the mutable options, the two are equivalent + ASSERT_TRUE(mcf_config->AreEquivalent(cfg_opts, cf_config.get(), &mismatch)); + ASSERT_TRUE(cf_config->AreEquivalent(cfg_opts, mcf_config.get(), &mismatch)); + + ASSERT_GT(a_names.size(), m_names.size()); + for (const auto& n : m_names) { + std::string m, d; + ASSERT_OK(mcf_config->GetOption(cfg_opts, n, &m)); + ASSERT_OK(cf_config->GetOption(cfg_opts, n, &d)); + ASSERT_EQ(m, d); + } + + cfg_opts.mutable_options_only = false; + // Comparing all of the options, the two are not equivalent + ASSERT_FALSE(mcf_config->AreEquivalent(cfg_opts, cf_config.get(), &mismatch)); + ASSERT_FALSE(cf_config->AreEquivalent(cfg_opts, mcf_config.get(), &mismatch)); + delete cf_opts.compaction_filter; + + // Make sure the options string contains only mutable options + ASSERT_OK(GetColumnFamilyOptionsFromString(cfg_opts, ColumnFamilyOptions(), + opt_str, &cf_opts)); + delete cf_opts.compaction_filter; +} + +TEST_F(OptionsTest, SstPartitionerTest) { + ConfigOptions cfg_opts; + ColumnFamilyOptions cf_opts, new_opt; + std::string opts_str, mismatch; + + ASSERT_OK(SstPartitionerFactory::CreateFromString( + cfg_opts, SstPartitionerFixedPrefixFactory::kClassName(), + &cf_opts.sst_partitioner_factory)); + ASSERT_NE(cf_opts.sst_partitioner_factory, nullptr); + ASSERT_STREQ(cf_opts.sst_partitioner_factory->Name(), + SstPartitionerFixedPrefixFactory::kClassName()); + ASSERT_NOK(GetColumnFamilyOptionsFromString( + cfg_opts, ColumnFamilyOptions(), + std::string("sst_partitioner_factory={id=") + + SstPartitionerFixedPrefixFactory::kClassName() + "; unknown=10;}", + &cf_opts)); + ASSERT_OK(GetColumnFamilyOptionsFromString( + cfg_opts, ColumnFamilyOptions(), + std::string("sst_partitioner_factory={id=") + + SstPartitionerFixedPrefixFactory::kClassName() + "; length=10;}", + &cf_opts)); + ASSERT_NE(cf_opts.sst_partitioner_factory, nullptr); + ASSERT_STREQ(cf_opts.sst_partitioner_factory->Name(), + SstPartitionerFixedPrefixFactory::kClassName()); + ASSERT_OK(GetStringFromColumnFamilyOptions(cfg_opts, cf_opts, &opts_str)); + ASSERT_OK( + GetColumnFamilyOptionsFromString(cfg_opts, cf_opts, opts_str, &new_opt)); + ASSERT_NE(new_opt.sst_partitioner_factory, nullptr); + ASSERT_STREQ(new_opt.sst_partitioner_factory->Name(), + SstPartitionerFixedPrefixFactory::kClassName()); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(cfg_opts, cf_opts, new_opt)); + ASSERT_TRUE(cf_opts.sst_partitioner_factory->AreEquivalent( + cfg_opts, new_opt.sst_partitioner_factory.get(), &mismatch)); +} + +TEST_F(OptionsTest, FileChecksumGenFactoryTest) { + ConfigOptions cfg_opts; + DBOptions db_opts, new_opt; + std::string opts_str, mismatch; + auto factory = GetFileChecksumGenCrc32cFactory(); + + cfg_opts.ignore_unsupported_options = false; + + ASSERT_OK(GetStringFromDBOptions(cfg_opts, db_opts, &opts_str)); + ASSERT_OK(GetDBOptionsFromString(cfg_opts, db_opts, opts_str, &new_opt)); + + ASSERT_NE(factory, nullptr); + ASSERT_OK(FileChecksumGenFactory::CreateFromString( + cfg_opts, factory->Name(), &db_opts.file_checksum_gen_factory)); + ASSERT_NE(db_opts.file_checksum_gen_factory, nullptr); + ASSERT_STREQ(db_opts.file_checksum_gen_factory->Name(), factory->Name()); + ASSERT_NOK(GetDBOptionsFromString( + cfg_opts, DBOptions(), "file_checksum_gen_factory=unknown", &db_opts)); + ASSERT_OK(GetDBOptionsFromString( + cfg_opts, DBOptions(), + std::string("file_checksum_gen_factory=") + factory->Name(), &db_opts)); + ASSERT_NE(db_opts.file_checksum_gen_factory, nullptr); + ASSERT_STREQ(db_opts.file_checksum_gen_factory->Name(), factory->Name()); + + ASSERT_OK(GetStringFromDBOptions(cfg_opts, db_opts, &opts_str)); + ASSERT_OK(GetDBOptionsFromString(cfg_opts, db_opts, opts_str, &new_opt)); + ASSERT_NE(new_opt.file_checksum_gen_factory, nullptr); + ASSERT_STREQ(new_opt.file_checksum_gen_factory->Name(), factory->Name()); + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(cfg_opts, db_opts, new_opt)); + ASSERT_TRUE(factory->AreEquivalent( + cfg_opts, new_opt.file_checksum_gen_factory.get(), &mismatch)); + ASSERT_TRUE(db_opts.file_checksum_gen_factory->AreEquivalent( + cfg_opts, new_opt.file_checksum_gen_factory.get(), &mismatch)); +} + +class TestTablePropertiesCollectorFactory + : public TablePropertiesCollectorFactory { + private: + std::string id_; + + public: + explicit TestTablePropertiesCollectorFactory(const std::string& id) + : id_(id) {} + TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context /*context*/) override { + return nullptr; + } + static const char* kClassName() { return "TestCollector"; } + const char* Name() const override { return kClassName(); } + std::string GetId() const override { + return std::string(kClassName()) + ":" + id_; + } +}; + +TEST_F(OptionsTest, OptionTablePropertiesTest) { + ConfigOptions cfg_opts; + ColumnFamilyOptions orig, copy; + orig.table_properties_collector_factories.push_back( + std::make_shared("1")); + orig.table_properties_collector_factories.push_back( + std::make_shared("2")); + + // Push two TablePropertiesCollectorFactories then create a new + // ColumnFamilyOptions based on those settings. The copy should + // have no properties but still match the original + std::string opts_str; + ASSERT_OK(GetStringFromColumnFamilyOptions(cfg_opts, orig, &opts_str)); + ASSERT_OK(GetColumnFamilyOptionsFromString(cfg_opts, orig, opts_str, ©)); + ASSERT_EQ(copy.table_properties_collector_factories.size(), 0); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(cfg_opts, orig, copy)); + + // Now register a TablePropertiesCollectorFactory + // Repeat the experiment. The copy should have the same + // properties as the original + cfg_opts.registry->AddLibrary("collector") + ->AddFactory( + ObjectLibrary::PatternEntry( + TestTablePropertiesCollectorFactory::kClassName(), false) + .AddSeparator(":"), + [](const std::string& name, + std::unique_ptr* guard, + std::string* /* errmsg */) { + std::string id = name.substr( + strlen(TestTablePropertiesCollectorFactory::kClassName()) + 1); + guard->reset(new TestTablePropertiesCollectorFactory(id)); + return guard->get(); + }); + + ASSERT_OK(GetColumnFamilyOptionsFromString(cfg_opts, orig, opts_str, ©)); + ASSERT_EQ(copy.table_properties_collector_factories.size(), 2); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(cfg_opts, orig, copy)); +} + +TEST_F(OptionsTest, ConvertOptionsTest) { + LevelDBOptions leveldb_opt; + Options converted_opt = ConvertOptions(leveldb_opt); + + ASSERT_EQ(converted_opt.create_if_missing, leveldb_opt.create_if_missing); + ASSERT_EQ(converted_opt.error_if_exists, leveldb_opt.error_if_exists); + ASSERT_EQ(converted_opt.paranoid_checks, leveldb_opt.paranoid_checks); + ASSERT_EQ(converted_opt.env, leveldb_opt.env); + ASSERT_EQ(converted_opt.info_log.get(), leveldb_opt.info_log); + ASSERT_EQ(converted_opt.write_buffer_size, leveldb_opt.write_buffer_size); + ASSERT_EQ(converted_opt.max_open_files, leveldb_opt.max_open_files); + ASSERT_EQ(converted_opt.compression, leveldb_opt.compression); + + std::shared_ptr table_factory = converted_opt.table_factory; + const auto table_opt = table_factory->GetOptions(); + ASSERT_NE(table_opt, nullptr); + + ASSERT_EQ(table_opt->block_cache->GetCapacity(), 32UL << 20); + ASSERT_EQ(table_opt->block_size, leveldb_opt.block_size); + ASSERT_EQ(table_opt->block_restart_interval, + leveldb_opt.block_restart_interval); + ASSERT_EQ(table_opt->filter_policy.get(), leveldb_opt.filter_policy); +} +class TestEventListener : public EventListener { + private: + std::string id_; + + public: + explicit TestEventListener(const std::string& id) : id_("Test" + id) {} + const char* Name() const override { return id_.c_str(); } +}; + +static std::unordered_map + test_listener_option_info = { + {"s", + {0, OptionType::kString, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + +}; + +class TestConfigEventListener : public TestEventListener { + private: + std::string s_; + + public: + explicit TestConfigEventListener(const std::string& id) + : TestEventListener("Config" + id) { + s_ = id; + RegisterOptions("Test", &s_, &test_listener_option_info); + } +}; + +static int RegisterTestEventListener(ObjectLibrary& library, + const std::string& arg) { + library.AddFactory( + "Test" + arg, + [](const std::string& name, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new TestEventListener(name.substr(4))); + return guard->get(); + }); + library.AddFactory( + "TestConfig" + arg, + [](const std::string& name, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new TestConfigEventListener(name.substr(10))); + return guard->get(); + }); + return 1; +} +TEST_F(OptionsTest, OptionsListenerTest) { + DBOptions orig, copy; + orig.listeners.push_back(std::make_shared("1")); + orig.listeners.push_back(std::make_shared("2")); + orig.listeners.push_back(std::make_shared("")); + orig.listeners.push_back(std::make_shared("1")); + orig.listeners.push_back(std::make_shared("2")); + orig.listeners.push_back(std::make_shared("")); + ConfigOptions config_opts(orig); + config_opts.registry->AddLibrary("listener", RegisterTestEventListener, "1"); + std::string opts_str; + ASSERT_OK(GetStringFromDBOptions(config_opts, orig, &opts_str)); + ASSERT_OK(GetDBOptionsFromString(config_opts, orig, opts_str, ©)); + ASSERT_OK(GetStringFromDBOptions(config_opts, copy, &opts_str)); + ASSERT_EQ( + copy.listeners.size(), + 2); // The Test{Config}1 Listeners could be loaded but not the others + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(config_opts, orig, copy)); +} + +const static std::string kCustomEnvName = "Custom"; +const static std::string kCustomEnvProp = "env=" + kCustomEnvName; + +static int RegisterCustomEnv(ObjectLibrary& library, const std::string& arg) { + library.AddFactory( + arg, [](const std::string& /*name*/, std::unique_ptr* /*env_guard*/, + std::string* /* errmsg */) { + static CustomEnv env(Env::Default()); + return &env; + }); + return 1; +} + +// This test suite tests the old APIs into the Configure options methods. +// Once those APIs are officially deprecated, this test suite can be deleted. +class OptionsOldApiTest : public testing::Test {}; + +TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) { + std::unordered_map cf_options_map = { + {"write_buffer_size", "1"}, + {"max_write_buffer_number", "2"}, + {"min_write_buffer_number_to_merge", "3"}, + {"max_write_buffer_number_to_maintain", "99"}, + {"max_write_buffer_size_to_maintain", "-99999"}, + {"compression", "kSnappyCompression"}, + {"compression_per_level", + "kNoCompression:" + "kSnappyCompression:" + "kZlibCompression:" + "kBZip2Compression:" + "kLZ4Compression:" + "kLZ4HCCompression:" + "kXpressCompression:" + "kZSTD:" + "kZSTDNotFinalCompression"}, + {"bottommost_compression", "kLZ4Compression"}, + {"bottommost_compression_opts", "5:6:7:8:9:true"}, + {"compression_opts", "4:5:6:7:8:9:true:10:false"}, + {"num_levels", "8"}, + {"level0_file_num_compaction_trigger", "8"}, + {"level0_slowdown_writes_trigger", "9"}, + {"level0_stop_writes_trigger", "10"}, + {"target_file_size_base", "12"}, + {"target_file_size_multiplier", "13"}, + {"max_bytes_for_level_base", "14"}, + {"level_compaction_dynamic_level_bytes", "true"}, + {"level_compaction_dynamic_file_size", "true"}, + {"max_bytes_for_level_multiplier", "15.0"}, + {"max_bytes_for_level_multiplier_additional", "16:17:18"}, + {"max_compaction_bytes", "21"}, + {"soft_rate_limit", "1.1"}, + {"hard_rate_limit", "2.1"}, + {"rate_limit_delay_max_milliseconds", "100"}, + {"hard_pending_compaction_bytes_limit", "211"}, + {"arena_block_size", "22"}, + {"disable_auto_compactions", "true"}, + {"compaction_style", "kCompactionStyleLevel"}, + {"compaction_pri", "kOldestSmallestSeqFirst"}, + {"verify_checksums_in_compaction", "false"}, + {"compaction_options_fifo", + "{allow_compaction=true;max_table_files_size=11002244;" + "file_temperature_age_thresholds={{temperature=kCold;age=12345}}}"}, + {"max_sequential_skip_in_iterations", "24"}, + {"inplace_update_support", "true"}, + {"report_bg_io_stats", "true"}, + {"compaction_measure_io_stats", "false"}, + {"purge_redundant_kvs_while_flush", "false"}, + {"inplace_update_num_locks", "25"}, + {"memtable_prefix_bloom_size_ratio", "0.26"}, + {"memtable_whole_key_filtering", "true"}, + {"memtable_huge_page_size", "28"}, + {"bloom_locality", "29"}, + {"max_successive_merges", "30"}, + {"min_partial_merge_operands", "31"}, + {"prefix_extractor", "fixed:31"}, + {"experimental_mempurge_threshold", "0.003"}, + {"optimize_filters_for_hits", "true"}, + {"enable_blob_files", "true"}, + {"min_blob_size", "1K"}, + {"blob_file_size", "1G"}, + {"blob_compression_type", "kZSTD"}, + {"enable_blob_garbage_collection", "true"}, + {"blob_garbage_collection_age_cutoff", "0.5"}, + {"blob_garbage_collection_force_threshold", "0.75"}, + {"blob_compaction_readahead_size", "256K"}, + {"blob_file_starting_level", "1"}, + {"prepopulate_blob_cache", "kDisable"}, + {"last_level_temperature", "kWarm"}, + {"persist_user_defined_timestamps", "true"}, + {"memtable_max_range_deletions", "0"}, + }; + + std::unordered_map db_options_map = { + {"create_if_missing", "false"}, + {"create_missing_column_families", "true"}, + {"error_if_exists", "false"}, + {"paranoid_checks", "true"}, + {"track_and_verify_wals_in_manifest", "true"}, + {"verify_sst_unique_id_in_manifest", "true"}, + {"max_open_files", "32"}, + {"max_total_wal_size", "33"}, + {"use_fsync", "true"}, + {"db_log_dir", "/db_log_dir"}, + {"wal_dir", "/wal_dir"}, + {"delete_obsolete_files_period_micros", "34"}, + {"max_background_compactions", "35"}, + {"max_background_flushes", "36"}, + {"max_log_file_size", "37"}, + {"log_file_time_to_roll", "38"}, + {"keep_log_file_num", "39"}, + {"recycle_log_file_num", "5"}, + {"max_manifest_file_size", "40"}, + {"table_cache_numshardbits", "41"}, + {"WAL_ttl_seconds", "43"}, + {"WAL_size_limit_MB", "44"}, + {"manifest_preallocation_size", "45"}, + {"allow_mmap_reads", "true"}, + {"allow_mmap_writes", "false"}, + {"use_direct_reads", "false"}, + {"use_direct_io_for_flush_and_compaction", "false"}, + {"is_fd_close_on_exec", "true"}, + {"skip_log_error_on_recovery", "false"}, + {"stats_dump_period_sec", "46"}, + {"stats_persist_period_sec", "57"}, + {"persist_stats_to_disk", "false"}, + {"stats_history_buffer_size", "69"}, + {"advise_random_on_open", "true"}, + {"use_adaptive_mutex", "false"}, + {"compaction_readahead_size", "100"}, + {"random_access_max_buffer_size", "3145728"}, + {"writable_file_max_buffer_size", "314159"}, + {"bytes_per_sync", "47"}, + {"wal_bytes_per_sync", "48"}, + {"strict_bytes_per_sync", "true"}, + {"preserve_deletes", "false"}, + }; + + ColumnFamilyOptions base_cf_opt; + ColumnFamilyOptions new_cf_opt; + ConfigOptions cf_config_options; + cf_config_options.ignore_unknown_options = false; + cf_config_options.input_strings_escaped = false; + ASSERT_OK(GetColumnFamilyOptionsFromMap(cf_config_options, base_cf_opt, + cf_options_map, &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 1U); + ASSERT_EQ(new_cf_opt.max_write_buffer_number, 2); + ASSERT_EQ(new_cf_opt.min_write_buffer_number_to_merge, 3); + ASSERT_EQ(new_cf_opt.max_write_buffer_number_to_maintain, 99); + ASSERT_EQ(new_cf_opt.max_write_buffer_size_to_maintain, -99999); + ASSERT_EQ(new_cf_opt.compression, kSnappyCompression); + ASSERT_EQ(new_cf_opt.compression_per_level.size(), 9U); + ASSERT_EQ(new_cf_opt.compression_per_level[0], kNoCompression); + ASSERT_EQ(new_cf_opt.compression_per_level[1], kSnappyCompression); + ASSERT_EQ(new_cf_opt.compression_per_level[2], kZlibCompression); + ASSERT_EQ(new_cf_opt.compression_per_level[3], kBZip2Compression); + ASSERT_EQ(new_cf_opt.compression_per_level[4], kLZ4Compression); + ASSERT_EQ(new_cf_opt.compression_per_level[5], kLZ4HCCompression); + ASSERT_EQ(new_cf_opt.compression_per_level[6], kXpressCompression); + ASSERT_EQ(new_cf_opt.compression_per_level[7], kZSTD); + ASSERT_EQ(new_cf_opt.compression_per_level[8], kZSTDNotFinalCompression); + ASSERT_EQ(new_cf_opt.compression_opts.window_bits, 4); + ASSERT_EQ(new_cf_opt.compression_opts.level, 5); + ASSERT_EQ(new_cf_opt.compression_opts.strategy, 6); + ASSERT_EQ(new_cf_opt.compression_opts.max_dict_bytes, 7u); + ASSERT_EQ(new_cf_opt.compression_opts.zstd_max_train_bytes, 8u); + ASSERT_EQ(new_cf_opt.compression_opts.parallel_threads, 9u); + ASSERT_EQ(new_cf_opt.compression_opts.enabled, true); + ASSERT_EQ(new_cf_opt.compression_opts.max_dict_buffer_bytes, 10u); + ASSERT_EQ(new_cf_opt.compression_opts.use_zstd_dict_trainer, false); + ASSERT_EQ(new_cf_opt.bottommost_compression, kLZ4Compression); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.window_bits, 5); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.level, 6); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.strategy, 7); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.max_dict_bytes, 8u); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.zstd_max_train_bytes, 9u); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.parallel_threads, + CompressionOptions().parallel_threads); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.enabled, true); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.max_dict_buffer_bytes, + CompressionOptions().max_dict_buffer_bytes); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.use_zstd_dict_trainer, + CompressionOptions().use_zstd_dict_trainer); + ASSERT_EQ(new_cf_opt.num_levels, 8); + ASSERT_EQ(new_cf_opt.level0_file_num_compaction_trigger, 8); + ASSERT_EQ(new_cf_opt.level0_slowdown_writes_trigger, 9); + ASSERT_EQ(new_cf_opt.level0_stop_writes_trigger, 10); + ASSERT_EQ(new_cf_opt.target_file_size_base, static_cast(12)); + ASSERT_EQ(new_cf_opt.target_file_size_multiplier, 13); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_base, 14U); + ASSERT_EQ(new_cf_opt.level_compaction_dynamic_level_bytes, true); + ASSERT_EQ(new_cf_opt.level_compaction_dynamic_file_size, true); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier, 15.0); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional.size(), 3U); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[0], 16); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[1], 17); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[2], 18); + ASSERT_EQ(new_cf_opt.max_compaction_bytes, 21); + ASSERT_EQ(new_cf_opt.hard_pending_compaction_bytes_limit, 211); + ASSERT_EQ(new_cf_opt.arena_block_size, 22U); + ASSERT_EQ(new_cf_opt.disable_auto_compactions, true); + ASSERT_EQ(new_cf_opt.compaction_style, kCompactionStyleLevel); + ASSERT_EQ(new_cf_opt.compaction_pri, kOldestSmallestSeqFirst); + ASSERT_EQ(new_cf_opt.compaction_options_fifo.max_table_files_size, + static_cast(11002244)); + ASSERT_EQ(new_cf_opt.compaction_options_fifo.allow_compaction, true); + ASSERT_EQ( + new_cf_opt.compaction_options_fifo.file_temperature_age_thresholds.size(), + 1); + ASSERT_EQ( + new_cf_opt.compaction_options_fifo.file_temperature_age_thresholds[0] + .temperature, + Temperature::kCold); + ASSERT_EQ( + new_cf_opt.compaction_options_fifo.file_temperature_age_thresholds[0].age, + 12345); + ASSERT_EQ(new_cf_opt.max_sequential_skip_in_iterations, + static_cast(24)); + ASSERT_EQ(new_cf_opt.inplace_update_support, true); + ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 25U); + ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_size_ratio, 0.26); + ASSERT_EQ(new_cf_opt.memtable_whole_key_filtering, true); + ASSERT_EQ(new_cf_opt.memtable_huge_page_size, 28U); + ASSERT_EQ(new_cf_opt.bloom_locality, 29U); + ASSERT_EQ(new_cf_opt.max_successive_merges, 30U); + ASSERT_TRUE(new_cf_opt.prefix_extractor != nullptr); + ASSERT_EQ(new_cf_opt.optimize_filters_for_hits, true); + ASSERT_EQ(new_cf_opt.prefix_extractor->AsString(), "rocksdb.FixedPrefix.31"); + ASSERT_EQ(new_cf_opt.experimental_mempurge_threshold, 0.003); + ASSERT_EQ(new_cf_opt.enable_blob_files, true); + ASSERT_EQ(new_cf_opt.min_blob_size, 1ULL << 10); + ASSERT_EQ(new_cf_opt.blob_file_size, 1ULL << 30); + ASSERT_EQ(new_cf_opt.blob_compression_type, kZSTD); + ASSERT_EQ(new_cf_opt.enable_blob_garbage_collection, true); + ASSERT_EQ(new_cf_opt.blob_garbage_collection_age_cutoff, 0.5); + ASSERT_EQ(new_cf_opt.blob_garbage_collection_force_threshold, 0.75); + ASSERT_EQ(new_cf_opt.blob_compaction_readahead_size, 262144); + ASSERT_EQ(new_cf_opt.blob_file_starting_level, 1); + ASSERT_EQ(new_cf_opt.prepopulate_blob_cache, PrepopulateBlobCache::kDisable); + ASSERT_EQ(new_cf_opt.last_level_temperature, Temperature::kWarm); + ASSERT_EQ(new_cf_opt.bottommost_temperature, Temperature::kWarm); + ASSERT_EQ(new_cf_opt.persist_user_defined_timestamps, true); + ASSERT_EQ(new_cf_opt.memtable_max_range_deletions, 0); + + cf_options_map["write_buffer_size"] = "hello"; + ASSERT_NOK(GetColumnFamilyOptionsFromMap(cf_config_options, base_cf_opt, + cf_options_map, &new_cf_opt)); + ConfigOptions exact, loose; + exact.sanity_level = ConfigOptions::kSanityLevelExactMatch; + loose.sanity_level = ConfigOptions::kSanityLevelLooselyCompatible; + + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); + + cf_options_map["write_buffer_size"] = "1"; + ASSERT_OK(GetColumnFamilyOptionsFromMap(cf_config_options, base_cf_opt, + cf_options_map, &new_cf_opt)); + + cf_options_map["unknown_option"] = "1"; + ASSERT_NOK(GetColumnFamilyOptionsFromMap(cf_config_options, base_cf_opt, + cf_options_map, &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); + + cf_config_options.input_strings_escaped = false; + cf_config_options.ignore_unknown_options = true; + ASSERT_OK(GetColumnFamilyOptionsFromMap(cf_config_options, base_cf_opt, + cf_options_map, &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( + loose, base_cf_opt, new_cf_opt, nullptr /* new_opt_map */)); + ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( + exact /* default for VerifyCFOptions */, base_cf_opt, new_cf_opt, nullptr)); + + DBOptions base_db_opt; + DBOptions new_db_opt; + ConfigOptions db_config_options(base_db_opt); + db_config_options.input_strings_escaped = false; + db_config_options.ignore_unknown_options = false; + ASSERT_OK(GetDBOptionsFromMap(db_config_options, base_db_opt, db_options_map, + &new_db_opt)); + ASSERT_EQ(new_db_opt.create_if_missing, false); + ASSERT_EQ(new_db_opt.create_missing_column_families, true); + ASSERT_EQ(new_db_opt.error_if_exists, false); + ASSERT_EQ(new_db_opt.paranoid_checks, true); + ASSERT_EQ(new_db_opt.track_and_verify_wals_in_manifest, true); + ASSERT_EQ(new_db_opt.max_open_files, 32); + ASSERT_EQ(new_db_opt.max_total_wal_size, static_cast(33)); + ASSERT_EQ(new_db_opt.use_fsync, true); + ASSERT_EQ(new_db_opt.db_log_dir, "/db_log_dir"); + ASSERT_EQ(new_db_opt.wal_dir, "/wal_dir"); + ASSERT_EQ(new_db_opt.delete_obsolete_files_period_micros, + static_cast(34)); + ASSERT_EQ(new_db_opt.max_background_compactions, 35); + ASSERT_EQ(new_db_opt.max_background_flushes, 36); + ASSERT_EQ(new_db_opt.max_log_file_size, 37U); + ASSERT_EQ(new_db_opt.log_file_time_to_roll, 38U); + ASSERT_EQ(new_db_opt.keep_log_file_num, 39U); + ASSERT_EQ(new_db_opt.recycle_log_file_num, 5U); + ASSERT_EQ(new_db_opt.max_manifest_file_size, static_cast(40)); + ASSERT_EQ(new_db_opt.table_cache_numshardbits, 41); + ASSERT_EQ(new_db_opt.WAL_ttl_seconds, static_cast(43)); + ASSERT_EQ(new_db_opt.WAL_size_limit_MB, static_cast(44)); + ASSERT_EQ(new_db_opt.manifest_preallocation_size, 45U); + ASSERT_EQ(new_db_opt.allow_mmap_reads, true); + ASSERT_EQ(new_db_opt.allow_mmap_writes, false); + ASSERT_EQ(new_db_opt.use_direct_reads, false); + ASSERT_EQ(new_db_opt.use_direct_io_for_flush_and_compaction, false); + ASSERT_EQ(new_db_opt.is_fd_close_on_exec, true); + ASSERT_EQ(new_db_opt.stats_dump_period_sec, 46U); + ASSERT_EQ(new_db_opt.stats_persist_period_sec, 57U); + ASSERT_EQ(new_db_opt.persist_stats_to_disk, false); + ASSERT_EQ(new_db_opt.stats_history_buffer_size, 69U); + ASSERT_EQ(new_db_opt.advise_random_on_open, true); + ASSERT_EQ(new_db_opt.use_adaptive_mutex, false); + ASSERT_EQ(new_db_opt.compaction_readahead_size, 100); + ASSERT_EQ(new_db_opt.random_access_max_buffer_size, 3145728); + ASSERT_EQ(new_db_opt.writable_file_max_buffer_size, 314159); + ASSERT_EQ(new_db_opt.bytes_per_sync, static_cast(47)); + ASSERT_EQ(new_db_opt.wal_bytes_per_sync, static_cast(48)); + ASSERT_EQ(new_db_opt.strict_bytes_per_sync, true); + + db_options_map["max_open_files"] = "hello"; + ASSERT_NOK(GetDBOptionsFromMap(db_config_options, base_db_opt, db_options_map, + &new_db_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(loose, base_db_opt, new_db_opt)); + + // unknow options should fail parsing without ignore_unknown_options = true + db_options_map["unknown_db_option"] = "1"; + ASSERT_NOK(GetDBOptionsFromMap(db_config_options, base_db_opt, db_options_map, + &new_db_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt)); + + db_config_options.input_strings_escaped = false; + db_config_options.ignore_unknown_options = true; + ASSERT_OK(GetDBOptionsFromMap(db_config_options, base_db_opt, db_options_map, + &new_db_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(loose, base_db_opt, new_db_opt)); + ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt)); +} + +TEST_F(OptionsOldApiTest, GetColumnFamilyOptionsFromStringTest) { + ColumnFamilyOptions base_cf_opt; + ColumnFamilyOptions new_cf_opt; + base_cf_opt.table_factory.reset(); + ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt, "", + &new_cf_opt)); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, "write_buffer_size=5", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 5U); + ASSERT_TRUE(new_cf_opt.table_factory == nullptr); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, "write_buffer_size=6;", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 6U); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, " write_buffer_size = 7 ", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 7U); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, " write_buffer_size = 8 ; ", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 8U); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=9;max_write_buffer_number=10", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 9U); + ASSERT_EQ(new_cf_opt.max_write_buffer_number, 10); + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=11; max_write_buffer_number = 12 ;", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 11U); + ASSERT_EQ(new_cf_opt.max_write_buffer_number, 12); + // Wrong name "max_write_buffer_number_" + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=13;max_write_buffer_number_=14;", &new_cf_opt)); + ConfigOptions exact; + exact.sanity_level = ConfigOptions::kSanityLevelExactMatch; + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); + + // Comparator from object registry + std::string kCompName = "reverse_comp"; + ObjectLibrary::Default()->AddFactory( + kCompName, + [](const std::string& /*name*/, + std::unique_ptr* /*guard*/, + std::string* /* errmsg */) { return ReverseBytewiseComparator(); }); + + ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt, + "comparator=" + kCompName + ";", + &new_cf_opt)); + ASSERT_EQ(new_cf_opt.comparator, ReverseBytewiseComparator()); + + // MergeOperator from object registry + std::unique_ptr bxo(new BytesXOROperator()); + std::string kMoName = bxo->Name(); + ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt, + "merge_operator=" + kMoName + ";", + &new_cf_opt)); + ASSERT_EQ(kMoName, std::string(new_cf_opt.merge_operator->Name())); + + // Wrong key/value pair + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=13;max_write_buffer_number;", &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); + + // Error Paring value + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=13;max_write_buffer_number=;", &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); + + // Missing option name + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, "write_buffer_size=13; =100;", &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); + + const uint64_t kilo = 1024UL; + const uint64_t mega = 1024 * kilo; + const uint64_t giga = 1024 * mega; + const uint64_t tera = 1024 * giga; + + // Units (k) + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, "max_write_buffer_number=15K", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.max_write_buffer_number, 15 * kilo); + // Units (m) + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "max_write_buffer_number=16m;inplace_update_num_locks=17M", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.max_write_buffer_number, 16 * mega); + ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 17u * mega); + // Units (g) + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=18g;prefix_extractor=capped:8;" + "arena_block_size=19G", + &new_cf_opt)); + + ASSERT_EQ(new_cf_opt.write_buffer_size, 18 * giga); + ASSERT_EQ(new_cf_opt.arena_block_size, 19 * giga); + ASSERT_TRUE(new_cf_opt.prefix_extractor.get() != nullptr); + ASSERT_EQ(new_cf_opt.prefix_extractor->AsString(), "rocksdb.CappedPrefix.8"); + + // Units (t) + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, "write_buffer_size=20t;arena_block_size=21T", + &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 20 * tera); + ASSERT_EQ(new_cf_opt.arena_block_size, 21 * tera); + + // Nested block based table options + // Empty + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={};arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.table_factory != nullptr); + // Non-empty + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_cache=1M;block_size=4;};" + "arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.table_factory != nullptr); + // Last one + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_cache=1M;block_size=4;}", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.table_factory != nullptr); + // Mismatch curly braces + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={{{block_size=4;};" + "arena_block_size=1024", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); + + // Unexpected chars after closing curly brace + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_size=4;}};" + "arena_block_size=1024", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); + + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_size=4;}xdfa;" + "arena_block_size=1024", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); + + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_size=4;}xdfa", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); + + // Invalid block based table option + ASSERT_NOK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={xx_block_size=4;}", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); + + ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt, + "optimize_filters_for_hits=true", + &new_cf_opt)); + ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt, + "optimize_filters_for_hits=false", + &new_cf_opt)); + + ASSERT_NOK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt, + "optimize_filters_for_hits=junk", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt)); + + // Nested plain table options + // Empty + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "plain_table_factory={};arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.table_factory != nullptr); + ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable"); + // Non-empty + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "plain_table_factory={user_key_len=66;bloom_bits_per_key=20;};" + "arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.table_factory != nullptr); + ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable"); + + // memtable factory + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "memtable=skip_list:10;arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.memtable_factory != nullptr); + ASSERT_TRUE(new_cf_opt.memtable_factory->IsInstanceOf("SkipListFactory")); + + // blob cache + ASSERT_OK(GetColumnFamilyOptionsFromString( + config_options, base_cf_opt, + "blob_cache={capacity=1M;num_shard_bits=4;" + "strict_capacity_limit=true;high_pri_pool_ratio=0.5;};", + &new_cf_opt)); + ASSERT_NE(new_cf_opt.blob_cache, nullptr); + ASSERT_EQ(new_cf_opt.blob_cache->GetCapacity(), 1024UL * 1024UL); + ASSERT_EQ(static_cast(new_cf_opt.blob_cache.get()) + ->GetNumShardBits(), + 4); + ASSERT_EQ(new_cf_opt.blob_cache->HasStrictCapacityLimit(), true); + ASSERT_EQ(static_cast(new_cf_opt.blob_cache.get()) + ->GetHighPriPoolRatio(), + 0.5); +} + +TEST_F(OptionsTest, SliceTransformCreateFromString) { + std::shared_ptr transform = nullptr; + ConfigOptions config_options; + config_options.ignore_unsupported_options = false; + config_options.ignore_unknown_options = false; + + ASSERT_OK( + SliceTransform::CreateFromString(config_options, "fixed:31", &transform)); + ASSERT_NE(transform, nullptr); + ASSERT_FALSE(transform->IsInstanceOf("capped")); + ASSERT_TRUE(transform->IsInstanceOf("fixed")); + ASSERT_TRUE(transform->IsInstanceOf("rocksdb.FixedPrefix")); + ASSERT_EQ(transform->GetId(), "rocksdb.FixedPrefix.31"); + ASSERT_OK(SliceTransform::CreateFromString( + config_options, "rocksdb.FixedPrefix.42", &transform)); + ASSERT_NE(transform, nullptr); + ASSERT_EQ(transform->GetId(), "rocksdb.FixedPrefix.42"); + + ASSERT_OK(SliceTransform::CreateFromString(config_options, "capped:16", + &transform)); + ASSERT_NE(transform, nullptr); + ASSERT_FALSE(transform->IsInstanceOf("fixed")); + ASSERT_TRUE(transform->IsInstanceOf("capped")); + ASSERT_TRUE(transform->IsInstanceOf("rocksdb.CappedPrefix")); + ASSERT_EQ(transform->GetId(), "rocksdb.CappedPrefix.16"); + ASSERT_OK(SliceTransform::CreateFromString( + config_options, "rocksdb.CappedPrefix.42", &transform)); + ASSERT_NE(transform, nullptr); + ASSERT_EQ(transform->GetId(), "rocksdb.CappedPrefix.42"); + + ASSERT_OK(SliceTransform::CreateFromString(config_options, "rocksdb.Noop", + &transform)); + ASSERT_NE(transform, nullptr); + + ASSERT_NOK(SliceTransform::CreateFromString(config_options, + "fixed:21:invalid", &transform)); + ASSERT_NOK(SliceTransform::CreateFromString(config_options, + "capped:21:invalid", &transform)); + ASSERT_NOK( + SliceTransform::CreateFromString(config_options, "fixed", &transform)); + ASSERT_NOK( + SliceTransform::CreateFromString(config_options, "capped", &transform)); + ASSERT_NOK( + SliceTransform::CreateFromString(config_options, "fixed:", &transform)); + ASSERT_NOK( + SliceTransform::CreateFromString(config_options, "capped:", &transform)); + ASSERT_NOK(SliceTransform::CreateFromString( + config_options, "rocksdb.FixedPrefix:42", &transform)); + ASSERT_NOK(SliceTransform::CreateFromString( + config_options, "rocksdb.CappedPrefix:42", &transform)); + ASSERT_NOK(SliceTransform::CreateFromString( + config_options, "rocksdb.FixedPrefix", &transform)); + ASSERT_NOK(SliceTransform::CreateFromString( + config_options, "rocksdb.CappedPrefix", &transform)); + ASSERT_NOK(SliceTransform::CreateFromString( + config_options, "rocksdb.FixedPrefix.", &transform)); + ASSERT_NOK(SliceTransform::CreateFromString( + config_options, "rocksdb.CappedPrefix.", &transform)); + ASSERT_NOK( + SliceTransform::CreateFromString(config_options, "invalid", &transform)); + + ASSERT_OK(SliceTransform::CreateFromString( + config_options, "rocksdb.CappedPrefix.11", &transform)); + ASSERT_NE(transform, nullptr); + ASSERT_EQ(transform->GetId(), "rocksdb.CappedPrefix.11"); + ASSERT_TRUE(transform->IsInstanceOf("capped")); + ASSERT_TRUE(transform->IsInstanceOf("capped:11")); + ASSERT_TRUE(transform->IsInstanceOf("rocksdb.CappedPrefix")); + ASSERT_TRUE(transform->IsInstanceOf("rocksdb.CappedPrefix.11")); + ASSERT_FALSE(transform->IsInstanceOf("fixed")); + ASSERT_FALSE(transform->IsInstanceOf("fixed:11")); + ASSERT_FALSE(transform->IsInstanceOf("rocksdb.FixedPrefix")); + ASSERT_FALSE(transform->IsInstanceOf("rocksdb.FixedPrefix.11")); + + ASSERT_OK(SliceTransform::CreateFromString( + config_options, "rocksdb.FixedPrefix.11", &transform)); + ASSERT_TRUE(transform->IsInstanceOf("fixed")); + ASSERT_TRUE(transform->IsInstanceOf("fixed:11")); + ASSERT_TRUE(transform->IsInstanceOf("rocksdb.FixedPrefix")); + ASSERT_TRUE(transform->IsInstanceOf("rocksdb.FixedPrefix.11")); + ASSERT_FALSE(transform->IsInstanceOf("capped")); + ASSERT_FALSE(transform->IsInstanceOf("capped:11")); + ASSERT_FALSE(transform->IsInstanceOf("rocksdb.CappedPrefix")); + ASSERT_FALSE(transform->IsInstanceOf("rocksdb.CappedPrefix.11")); +} + +TEST_F(OptionsOldApiTest, GetBlockBasedTableOptionsFromString) { + BlockBasedTableOptions table_opt; + BlockBasedTableOptions new_opt; + ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + config_options.invoke_prepare_options = false; + config_options.ignore_unsupported_options = false; + + // make sure default values are overwritten by something else + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "cache_index_and_filter_blocks=1;index_type=kHashSearch;" + "checksum=kxxHash;no_block_cache=1;" + "block_cache=1M;block_cache_compressed=1k;block_size=1024;" + "block_size_deviation=8;block_restart_interval=4;" + "format_version=5;whole_key_filtering=1;" + "filter_policy=bloomfilter:4.567:false;", + &new_opt)); + ASSERT_TRUE(new_opt.cache_index_and_filter_blocks); + ASSERT_EQ(new_opt.index_type, BlockBasedTableOptions::kHashSearch); + ASSERT_EQ(new_opt.checksum, ChecksumType::kxxHash); + ASSERT_TRUE(new_opt.no_block_cache); + ASSERT_TRUE(new_opt.block_cache != nullptr); + ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL); + ASSERT_EQ(new_opt.block_size, 1024UL); + ASSERT_EQ(new_opt.block_size_deviation, 8); + ASSERT_EQ(new_opt.block_restart_interval, 4); + ASSERT_EQ(new_opt.format_version, 5U); + ASSERT_EQ(new_opt.whole_key_filtering, true); + ASSERT_TRUE(new_opt.filter_policy != nullptr); + const BloomFilterPolicy* bfp = + dynamic_cast(new_opt.filter_policy.get()); + EXPECT_EQ(bfp->GetMillibitsPerKey(), 4567); + EXPECT_EQ(bfp->GetWholeBitsPerKey(), 5); + + // unknown option + ASSERT_NOK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "cache_index_and_filter_blocks=1;index_type=kBinarySearch;" + "bad_option=1", + &new_opt)); + ASSERT_EQ(static_cast(table_opt.cache_index_and_filter_blocks), + new_opt.cache_index_and_filter_blocks); + ASSERT_EQ(table_opt.index_type, new_opt.index_type); + + // unrecognized index type + ASSERT_NOK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "cache_index_and_filter_blocks=1;index_type=kBinarySearchXX", &new_opt)); + ASSERT_EQ(table_opt.cache_index_and_filter_blocks, + new_opt.cache_index_and_filter_blocks); + ASSERT_EQ(table_opt.index_type, new_opt.index_type); + + // unrecognized checksum type + ASSERT_NOK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "cache_index_and_filter_blocks=1;checksum=kxxHashXX", &new_opt)); + ASSERT_EQ(table_opt.cache_index_and_filter_blocks, + new_opt.cache_index_and_filter_blocks); + ASSERT_EQ(table_opt.index_type, new_opt.index_type); + + // unrecognized filter policy name + ASSERT_NOK( + GetBlockBasedTableOptionsFromString(config_options, table_opt, + "cache_index_and_filter_blocks=1;" + "filter_policy=bloomfilterxx:4:true", + &new_opt)); + ASSERT_EQ(table_opt.cache_index_and_filter_blocks, + new_opt.cache_index_and_filter_blocks); + ASSERT_EQ(table_opt.filter_policy, new_opt.filter_policy); + + // Used to be rejected, now accepted + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, "filter_policy=bloomfilter:4", &new_opt)); + bfp = dynamic_cast(new_opt.filter_policy.get()); + EXPECT_EQ(bfp->GetMillibitsPerKey(), 4000); + EXPECT_EQ(bfp->GetWholeBitsPerKey(), 4); + + // Check block cache options are overwritten when specified + // in new format as a struct. + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "block_cache={capacity=1M;num_shard_bits=4;" + "strict_capacity_limit=true;high_pri_pool_ratio=0.5;};" + "block_cache_compressed={capacity=1M;num_shard_bits=4;" + "strict_capacity_limit=true;high_pri_pool_ratio=0.5;}", + &new_opt)); + ASSERT_TRUE(new_opt.block_cache != nullptr); + ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL); + ASSERT_EQ(std::dynamic_pointer_cast(new_opt.block_cache) + ->GetNumShardBits(), + 4); + ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true); + ASSERT_EQ(std::dynamic_pointer_cast( + new_opt.block_cache)->GetHighPriPoolRatio(), 0.5); + + // Set only block cache capacity. Check other values are + // reset to default values. + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "block_cache={capacity=2M};" + "block_cache_compressed={capacity=2M}", + &new_opt)); + ASSERT_TRUE(new_opt.block_cache != nullptr); + ASSERT_EQ(new_opt.block_cache->GetCapacity(), 2*1024UL*1024UL); + // Default values + ASSERT_EQ(std::dynamic_pointer_cast(new_opt.block_cache) + ->GetNumShardBits(), + GetDefaultCacheShardBits(new_opt.block_cache->GetCapacity())); + ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false); + ASSERT_EQ(std::dynamic_pointer_cast(new_opt.block_cache) + ->GetHighPriPoolRatio(), + 0.5); + + // Set couple of block cache options. + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "block_cache={num_shard_bits=5;high_pri_pool_ratio=0.5;};" + "block_cache_compressed={num_shard_bits=5;" + "high_pri_pool_ratio=0.0;}", + &new_opt)); + ASSERT_EQ(new_opt.block_cache->GetCapacity(), 0); + ASSERT_EQ(std::dynamic_pointer_cast(new_opt.block_cache) + ->GetNumShardBits(), + 5); + ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false); + ASSERT_EQ(std::dynamic_pointer_cast( + new_opt.block_cache)->GetHighPriPoolRatio(), 0.5); + + // Set couple of block cache options. + ASSERT_OK(GetBlockBasedTableOptionsFromString( + config_options, table_opt, + "block_cache={capacity=1M;num_shard_bits=4;" + "strict_capacity_limit=true;};" + "block_cache_compressed={capacity=1M;num_shard_bits=4;" + "strict_capacity_limit=true;}", + &new_opt)); + ASSERT_TRUE(new_opt.block_cache != nullptr); + ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL); + ASSERT_EQ(std::dynamic_pointer_cast(new_opt.block_cache) + ->GetNumShardBits(), + 4); + ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true); + ASSERT_EQ(std::dynamic_pointer_cast(new_opt.block_cache) + ->GetHighPriPoolRatio(), + 0.5); +} + +TEST_F(OptionsOldApiTest, GetPlainTableOptionsFromString) { + PlainTableOptions table_opt; + PlainTableOptions new_opt; + // make sure default values are overwritten by something else + ConfigOptions config_options_from_string; + config_options_from_string.input_strings_escaped = false; + config_options_from_string.ignore_unknown_options = false; + config_options_from_string.invoke_prepare_options = false; + ASSERT_OK(GetPlainTableOptionsFromString( + config_options_from_string, table_opt, + "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" + "index_sparseness=8;huge_page_tlb_size=4;encoding_type=kPrefix;" + "full_scan_mode=true;store_index_in_file=true", + &new_opt)); + ASSERT_EQ(new_opt.user_key_len, 66u); + ASSERT_EQ(new_opt.bloom_bits_per_key, 20); + ASSERT_EQ(new_opt.hash_table_ratio, 0.5); + ASSERT_EQ(new_opt.index_sparseness, 8); + ASSERT_EQ(new_opt.huge_page_tlb_size, 4); + ASSERT_EQ(new_opt.encoding_type, EncodingType::kPrefix); + ASSERT_TRUE(new_opt.full_scan_mode); + ASSERT_TRUE(new_opt.store_index_in_file); + + std::unordered_map opt_map; + ASSERT_OK(StringToMap( + "user_key_len=55;bloom_bits_per_key=10;huge_page_tlb_size=8;", &opt_map)); + ConfigOptions config_options_from_map; + config_options_from_map.input_strings_escaped = false; + config_options_from_map.ignore_unknown_options = false; + ASSERT_OK(GetPlainTableOptionsFromMap(config_options_from_map, table_opt, + opt_map, &new_opt)); + ASSERT_EQ(new_opt.user_key_len, 55u); + ASSERT_EQ(new_opt.bloom_bits_per_key, 10); + ASSERT_EQ(new_opt.huge_page_tlb_size, 8); + + // unknown option + ASSERT_NOK(GetPlainTableOptionsFromString( + config_options_from_string, table_opt, + "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" + "bad_option=1", + &new_opt)); + + // unrecognized EncodingType + ASSERT_NOK(GetPlainTableOptionsFromString( + config_options_from_string, table_opt, + "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" + "encoding_type=kPrefixXX", + &new_opt)); +} + +TEST_F(OptionsOldApiTest, GetOptionsFromStringTest) { + Options base_options, new_options; + base_options.write_buffer_size = 20; + base_options.min_write_buffer_number_to_merge = 15; + BlockBasedTableOptions block_based_table_options; + block_based_table_options.cache_index_and_filter_blocks = true; + base_options.table_factory.reset( + NewBlockBasedTableFactory(block_based_table_options)); + + // Register an Env with object registry. + ObjectLibrary::Default()->AddFactory( + "CustomEnvDefault", + [](const std::string& /*name*/, std::unique_ptr* /*env_guard*/, + std::string* /* errmsg */) { + static CustomEnv env(Env::Default()); + return &env; + }); + + ASSERT_OK(GetOptionsFromString( + base_options, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_cache=1M;block_size=4;};" + "compression_opts=4:5:6;create_if_missing=true;max_open_files=1;" + "bottommost_compression_opts=5:6:7;create_if_missing=true;max_open_files=" + "1;" + "rate_limiter_bytes_per_sec=1024;env=CustomEnvDefault", + &new_options)); + + ASSERT_EQ(new_options.compression_opts.window_bits, 4); + ASSERT_EQ(new_options.compression_opts.level, 5); + ASSERT_EQ(new_options.compression_opts.strategy, 6); + ASSERT_EQ(new_options.compression_opts.max_dict_bytes, 0u); + ASSERT_EQ(new_options.compression_opts.zstd_max_train_bytes, 0u); + ASSERT_EQ(new_options.compression_opts.parallel_threads, 1u); + ASSERT_EQ(new_options.compression_opts.enabled, false); + ASSERT_EQ(new_options.compression_opts.use_zstd_dict_trainer, true); + ASSERT_EQ(new_options.bottommost_compression, kDisableCompressionOption); + ASSERT_EQ(new_options.bottommost_compression_opts.window_bits, 5); + ASSERT_EQ(new_options.bottommost_compression_opts.level, 6); + ASSERT_EQ(new_options.bottommost_compression_opts.strategy, 7); + ASSERT_EQ(new_options.bottommost_compression_opts.max_dict_bytes, 0u); + ASSERT_EQ(new_options.bottommost_compression_opts.zstd_max_train_bytes, 0u); + ASSERT_EQ(new_options.bottommost_compression_opts.parallel_threads, 1u); + ASSERT_EQ(new_options.bottommost_compression_opts.enabled, false); + ASSERT_EQ(new_options.bottommost_compression_opts.use_zstd_dict_trainer, + true); + ASSERT_EQ(new_options.write_buffer_size, 10U); + ASSERT_EQ(new_options.max_write_buffer_number, 16); + + auto new_block_based_table_options = + new_options.table_factory->GetOptions(); + ASSERT_NE(new_block_based_table_options, nullptr); + ASSERT_EQ(new_block_based_table_options->block_cache->GetCapacity(), + 1U << 20); + ASSERT_EQ(new_block_based_table_options->block_size, 4U); + // don't overwrite block based table options + ASSERT_TRUE(new_block_based_table_options->cache_index_and_filter_blocks); + + ASSERT_EQ(new_options.create_if_missing, true); + ASSERT_EQ(new_options.max_open_files, 1); + ASSERT_TRUE(new_options.rate_limiter.get() != nullptr); + Env* newEnv = new_options.env; + ASSERT_OK(Env::CreateFromString({}, "CustomEnvDefault", &newEnv)); + ASSERT_EQ(newEnv, new_options.env); +} + +TEST_F(OptionsOldApiTest, DBOptionsSerialization) { + Options base_options, new_options; + Random rnd(301); + + // Phase 1: Make big change in base_options + test::RandomInitDBOptions(&base_options, &rnd); + + // Phase 2: obtain a string from base_option + std::string base_options_file_content; + ASSERT_OK(GetStringFromDBOptions(&base_options_file_content, base_options)); + + // Phase 3: Set new_options from the derived string and expect + // new_options == base_options + const DBOptions base_db_options; + ConfigOptions db_config_options(base_db_options); + db_config_options.input_strings_escaped = false; + db_config_options.ignore_unknown_options = false; + ASSERT_OK(GetDBOptionsFromString(db_config_options, base_db_options, + base_options_file_content, &new_options)); + ConfigOptions verify_db_config_options; + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(verify_db_config_options, + base_options, new_options)); +} + +TEST_F(OptionsOldApiTest, ColumnFamilyOptionsSerialization) { + Options options; + ColumnFamilyOptions base_opt, new_opt; + Random rnd(302); + // Phase 1: randomly assign base_opt + // custom type options + test::RandomInitCFOptions(&base_opt, options, &rnd); + + // Phase 2: obtain a string from base_opt + std::string base_options_file_content; + ASSERT_OK( + GetStringFromColumnFamilyOptions(&base_options_file_content, base_opt)); + + // Phase 3: Set new_opt from the derived string and expect + // new_opt == base_opt + ConfigOptions cf_config_options; + cf_config_options.input_strings_escaped = false; + cf_config_options.ignore_unknown_options = false; + ASSERT_OK( + GetColumnFamilyOptionsFromString(cf_config_options, ColumnFamilyOptions(), + base_options_file_content, &new_opt)); + ConfigOptions verify_cf_config_options; + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(verify_cf_config_options, + base_opt, new_opt)); + if (base_opt.compaction_filter) { + delete base_opt.compaction_filter; + } +} + +class OptionsParserTest : public testing::Test { + public: + OptionsParserTest() { fs_.reset(new test::StringFS(FileSystem::Default())); } + + protected: + std::shared_ptr fs_; +}; + +TEST_F(OptionsParserTest, Comment) { + DBOptions db_opt; + db_opt.max_open_files = 12345; + db_opt.max_background_flushes = 301; + db_opt.max_total_wal_size = 1024; + ColumnFamilyOptions cf_opt; + + std::string options_file_content = + "# This is a testing option string.\n" + "# Currently we only support \"#\" styled comment.\n" + "\n" + "[Version]\n" + " rocksdb_version=3.14.0\n" + " options_file_version=1\n" + "[ DBOptions ]\n" + " # note that we don't support space around \"=\"\n" + " max_open_files=12345;\n" + " max_background_flushes=301 # comment after a statement is fine\n" + " # max_background_flushes=1000 # this line would be ignored\n" + " # max_background_compactions=2000 # so does this one\n" + " max_total_wal_size=1024 # keep_log_file_num=1000\n" + "[CFOptions \"default\"] # column family must be specified\n" + " # in the correct order\n" + " # if a section is blank, we will use the default\n"; + + const std::string kTestFileName = "test-rocksdb-options.ini"; + ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content)); + RocksDBOptionsParser parser; + ASSERT_OK( + parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */)); + + ConfigOptions exact; + exact.input_strings_escaped = false; + exact.sanity_level = ConfigOptions::kSanityLevelExactMatch; + ASSERT_OK( + RocksDBOptionsParser::VerifyDBOptions(exact, *parser.db_opt(), db_opt)); + ASSERT_EQ(parser.NumColumnFamilies(), 1U); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( + exact, *parser.GetCFOptions("default"), cf_opt)); +} + +TEST_F(OptionsParserTest, ExtraSpace) { + std::string options_file_content = + "# This is a testing option string.\n" + "# Currently we only support \"#\" styled comment.\n" + "\n" + "[ Version ]\n" + " rocksdb_version = 3.14.0 \n" + " options_file_version=1 # some comment\n" + "[DBOptions ] # some comment\n" + "max_open_files=12345 \n" + " max_background_flushes = 301 \n" + " max_total_wal_size = 1024 # keep_log_file_num=1000\n" + " [CFOptions \"default\" ]\n" + " # if a section is blank, we will use the default\n"; + + const std::string kTestFileName = "test-rocksdb-options.ini"; + ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content)); + RocksDBOptionsParser parser; + ASSERT_OK( + parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */)); +} + +TEST_F(OptionsParserTest, MissingDBOptions) { + std::string options_file_content = + "# This is a testing option string.\n" + "# Currently we only support \"#\" styled comment.\n" + "\n" + "[Version]\n" + " rocksdb_version=3.14.0\n" + " options_file_version=1\n" + "[CFOptions \"default\"]\n" + " # if a section is blank, we will use the default\n"; + + const std::string kTestFileName = "test-rocksdb-options.ini"; + ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content)); + RocksDBOptionsParser parser; + ASSERT_NOK( + parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */)); + ; +} + +TEST_F(OptionsParserTest, DoubleDBOptions) { + DBOptions db_opt; + db_opt.max_open_files = 12345; + db_opt.max_background_flushes = 301; + db_opt.max_total_wal_size = 1024; + ColumnFamilyOptions cf_opt; + + std::string options_file_content = + "# This is a testing option string.\n" + "# Currently we only support \"#\" styled comment.\n" + "\n" + "[Version]\n" + " rocksdb_version=3.14.0\n" + " options_file_version=1\n" + "[DBOptions]\n" + " max_open_files=12345\n" + " max_background_flushes=301\n" + " max_total_wal_size=1024 # keep_log_file_num=1000\n" + "[DBOptions]\n" + "[CFOptions \"default\"]\n" + " # if a section is blank, we will use the default\n"; + + const std::string kTestFileName = "test-rocksdb-options.ini"; + ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content)); + RocksDBOptionsParser parser; + ASSERT_NOK( + parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */)); +} + +TEST_F(OptionsParserTest, NoDefaultCFOptions) { + DBOptions db_opt; + db_opt.max_open_files = 12345; + db_opt.max_background_flushes = 301; + db_opt.max_total_wal_size = 1024; + ColumnFamilyOptions cf_opt; + + std::string options_file_content = + "# This is a testing option string.\n" + "# Currently we only support \"#\" styled comment.\n" + "\n" + "[Version]\n" + " rocksdb_version=3.14.0\n" + " options_file_version=1\n" + "[DBOptions]\n" + " max_open_files=12345\n" + " max_background_flushes=301\n" + " max_total_wal_size=1024 # keep_log_file_num=1000\n" + "[CFOptions \"something_else\"]\n" + " # if a section is blank, we will use the default\n"; + + const std::string kTestFileName = "test-rocksdb-options.ini"; + ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content)); + RocksDBOptionsParser parser; + ASSERT_NOK( + parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */)); +} + +TEST_F(OptionsParserTest, DefaultCFOptionsMustBeTheFirst) { + DBOptions db_opt; + db_opt.max_open_files = 12345; + db_opt.max_background_flushes = 301; + db_opt.max_total_wal_size = 1024; + ColumnFamilyOptions cf_opt; + + std::string options_file_content = + "# This is a testing option string.\n" + "# Currently we only support \"#\" styled comment.\n" + "\n" + "[Version]\n" + " rocksdb_version=3.14.0\n" + " options_file_version=1\n" + "[DBOptions]\n" + " max_open_files=12345\n" + " max_background_flushes=301\n" + " max_total_wal_size=1024 # keep_log_file_num=1000\n" + "[CFOptions \"something_else\"]\n" + " # if a section is blank, we will use the default\n" + "[CFOptions \"default\"]\n" + " # if a section is blank, we will use the default\n"; + + const std::string kTestFileName = "test-rocksdb-options.ini"; + ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content)); + RocksDBOptionsParser parser; + ASSERT_NOK( + parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */)); +} + +TEST_F(OptionsParserTest, DuplicateCFOptions) { + DBOptions db_opt; + db_opt.max_open_files = 12345; + db_opt.max_background_flushes = 301; + db_opt.max_total_wal_size = 1024; + ColumnFamilyOptions cf_opt; + + std::string options_file_content = + "# This is a testing option string.\n" + "# Currently we only support \"#\" styled comment.\n" + "\n" + "[Version]\n" + " rocksdb_version=3.14.0\n" + " options_file_version=1\n" + "[DBOptions]\n" + " max_open_files=12345\n" + " max_background_flushes=301\n" + " max_total_wal_size=1024 # keep_log_file_num=1000\n" + "[CFOptions \"default\"]\n" + "[CFOptions \"something_else\"]\n" + "[CFOptions \"something_else\"]\n"; + + const std::string kTestFileName = "test-rocksdb-options.ini"; + ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content)); + RocksDBOptionsParser parser; + ASSERT_NOK( + parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */)); +} + +TEST_F(OptionsParserTest, IgnoreUnknownOptions) { + for (int case_id = 0; case_id < 5; case_id++) { + DBOptions db_opt; + db_opt.max_open_files = 12345; + db_opt.max_background_flushes = 301; + db_opt.max_total_wal_size = 1024; + ColumnFamilyOptions cf_opt; + + std::string version_string; + bool should_ignore = true; + if (case_id == 0) { + // same version + should_ignore = false; + version_string = std::to_string(ROCKSDB_MAJOR) + "." + + std::to_string(ROCKSDB_MINOR) + ".0"; + } else if (case_id == 1) { + // higher minor version + should_ignore = true; + version_string = std::to_string(ROCKSDB_MAJOR) + "." + + std::to_string(ROCKSDB_MINOR + 1) + ".0"; + } else if (case_id == 2) { + // higher major version. + should_ignore = true; + version_string = std::to_string(ROCKSDB_MAJOR + 1) + ".0.0"; + } else if (case_id == 3) { + // lower minor version +#if ROCKSDB_MINOR == 0 + continue; +#else + version_string = std::to_string(ROCKSDB_MAJOR) + "." + + std::to_string(ROCKSDB_MINOR - 1) + ".0"; + should_ignore = false; +#endif + } else { + // lower major version + should_ignore = false; + version_string = std::to_string(ROCKSDB_MAJOR - 1) + "." + + std::to_string(ROCKSDB_MINOR) + ".0"; + } + + std::string options_file_content = + "# This is a testing option string.\n" + "# Currently we only support \"#\" styled comment.\n" + "\n" + "[Version]\n" + " rocksdb_version=" + + version_string + + "\n" + " options_file_version=1\n" + "[DBOptions]\n" + " max_open_files=12345\n" + " max_background_flushes=301\n" + " max_total_wal_size=1024 # keep_log_file_num=1000\n" + " unknown_db_option1=321\n" + " unknown_db_option2=false\n" + "[CFOptions \"default\"]\n" + " unknown_cf_option1=hello\n" + "[CFOptions \"something_else\"]\n" + " unknown_cf_option2=world\n" + " # if a section is blank, we will use the default\n"; + + const std::string kTestFileName = "test-rocksdb-options.ini"; + auto s = fs_->FileExists(kTestFileName, IOOptions(), nullptr); + ASSERT_TRUE(s.ok() || s.IsNotFound()); + if (s.ok()) { + ASSERT_OK(fs_->DeleteFile(kTestFileName, IOOptions(), nullptr)); + } + ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content)); + RocksDBOptionsParser parser; + ASSERT_NOK(parser.Parse(kTestFileName, fs_.get(), false, + 4096 /* readahead_size */)); + if (should_ignore) { + ASSERT_OK(parser.Parse(kTestFileName, fs_.get(), + true /* ignore_unknown_options */, + 4096 /* readahead_size */)); + } else { + ASSERT_NOK(parser.Parse(kTestFileName, fs_.get(), + true /* ignore_unknown_options */, + 4096 /* readahead_size */)); + } + } +} + +TEST_F(OptionsParserTest, ParseVersion) { + DBOptions db_opt; + db_opt.max_open_files = 12345; + db_opt.max_background_flushes = 301; + db_opt.max_total_wal_size = 1024; + ColumnFamilyOptions cf_opt; + + std::string file_template = + "# This is a testing option string.\n" + "# Currently we only support \"#\" styled comment.\n" + "\n" + "[Version]\n" + " rocksdb_version=3.13.1\n" + " options_file_version=%s\n" + "[DBOptions]\n" + "[CFOptions \"default\"]\n"; + const int kLength = 1000; + char buffer[kLength]; + RocksDBOptionsParser parser; + + const std::vector invalid_versions = { + "a.b.c", "3.2.2b", "3.-12", "3. 1", // only digits and dots are allowed + "1.2.3.4", + "1.2.3" // can only contains at most one dot. + "0", // options_file_version must be at least one + "3..2", + ".", ".1.2", // must have at least one digit before each dot + "1.2.", "1.", "2.34."}; // must have at least one digit after each dot + for (auto iv : invalid_versions) { + snprintf(buffer, kLength - 1, file_template.c_str(), iv.c_str()); + + parser.Reset(); + ASSERT_OK(fs_->WriteToNewFile(iv, buffer)); + ASSERT_NOK(parser.Parse(iv, fs_.get(), false, 0 /* readahead_size */)); + } + + const std::vector valid_versions = { + "1.232", "100", "3.12", "1", "12.3 ", " 1.25 "}; + for (auto vv : valid_versions) { + snprintf(buffer, kLength - 1, file_template.c_str(), vv.c_str()); + parser.Reset(); + ASSERT_OK(fs_->WriteToNewFile(vv, buffer)); + ASSERT_OK(parser.Parse(vv, fs_.get(), false, 0 /* readahead_size */)); + } +} + +void VerifyCFPointerTypedOptions( + ColumnFamilyOptions* base_cf_opt, const ColumnFamilyOptions* new_cf_opt, + const std::unordered_map* new_cf_opt_map) { + std::string name_buffer; + ConfigOptions config_options; + config_options.input_strings_escaped = false; + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, *base_cf_opt, + *new_cf_opt, new_cf_opt_map)); + + // change the name of merge operator back-and-forth + { + auto* merge_operator = base_cf_opt->merge_operator + ->CheckedCast(); + if (merge_operator != nullptr) { + name_buffer = merge_operator->Name(); + // change the name and expect non-ok status + merge_operator->SetName("some-other-name"); + ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( + config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map)); + // change the name back and expect ok status + merge_operator->SetName(name_buffer); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( + config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map)); + } + } + + // change the name of the compaction filter factory back-and-forth + { + auto* compaction_filter_factory = + base_cf_opt->compaction_filter_factory + ->CheckedCast(); + if (compaction_filter_factory != nullptr) { + name_buffer = compaction_filter_factory->Name(); + // change the name and expect non-ok status + compaction_filter_factory->SetName("some-other-name"); + ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( + config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map)); + // change the name back and expect ok status + compaction_filter_factory->SetName(name_buffer); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( + config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map)); + } + } + + // test by setting compaction_filter to nullptr + { + auto* tmp_compaction_filter = base_cf_opt->compaction_filter; + if (tmp_compaction_filter != nullptr) { + base_cf_opt->compaction_filter = nullptr; + // set compaction_filter to nullptr and expect non-ok status + ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( + config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map)); + // set the value back and expect ok status + base_cf_opt->compaction_filter = tmp_compaction_filter; + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( + config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map)); + } + } + + // test by setting table_factory to nullptr + { + auto tmp_table_factory = base_cf_opt->table_factory; + if (tmp_table_factory != nullptr) { + base_cf_opt->table_factory.reset(); + // set table_factory to nullptr and expect non-ok status + ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( + config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map)); + // set the value back and expect ok status + base_cf_opt->table_factory = tmp_table_factory; + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( + config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map)); + } + } + + // test by setting memtable_factory to nullptr + { + auto tmp_memtable_factory = base_cf_opt->memtable_factory; + if (tmp_memtable_factory != nullptr) { + base_cf_opt->memtable_factory.reset(); + // set memtable_factory to nullptr and expect non-ok status + ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( + config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map)); + // set the value back and expect ok status + base_cf_opt->memtable_factory = tmp_memtable_factory; + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( + config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map)); + } + } +} + +TEST_F(OptionsParserTest, Readahead) { + DBOptions base_db_opt; + std::vector base_cf_opts; + base_cf_opts.emplace_back(); + base_cf_opts.emplace_back(); + + std::string one_mb_string = std::string(1024 * 1024, 'x'); + std::vector cf_names = {"default", one_mb_string}; + const std::string kOptionsFileName = "test-persisted-options.ini"; + + ASSERT_OK(PersistRocksDBOptions(base_db_opt, cf_names, base_cf_opts, + kOptionsFileName, fs_.get())); + + uint64_t file_size = 0; + ASSERT_OK( + fs_->GetFileSize(kOptionsFileName, IOOptions(), &file_size, nullptr)); + assert(file_size > 0); + + RocksDBOptionsParser parser; + + fs_->num_seq_file_read_ = 0; + size_t readahead_size = 128 * 1024; + + ASSERT_OK(parser.Parse(kOptionsFileName, fs_.get(), false, readahead_size)); + ASSERT_EQ(fs_->num_seq_file_read_.load(), + (file_size - 1) / readahead_size + 1); + + fs_->num_seq_file_read_.store(0); + readahead_size = 1024 * 1024; + ASSERT_OK(parser.Parse(kOptionsFileName, fs_.get(), false, readahead_size)); + ASSERT_EQ(fs_->num_seq_file_read_.load(), + (file_size - 1) / readahead_size + 1); + + // Tiny readahead. 8 KB is read each time. + fs_->num_seq_file_read_.store(0); + ASSERT_OK( + parser.Parse(kOptionsFileName, fs_.get(), false, 1 /* readahead_size */)); + ASSERT_GE(fs_->num_seq_file_read_.load(), file_size / (8 * 1024)); + ASSERT_LT(fs_->num_seq_file_read_.load(), file_size / (8 * 1024) * 2); + + // Disable readahead means 512KB readahead. + fs_->num_seq_file_read_.store(0); + ASSERT_OK( + parser.Parse(kOptionsFileName, fs_.get(), false, 0 /* readahead_size */)); + ASSERT_GE(fs_->num_seq_file_read_.load(), (file_size - 1) / (512 * 1024) + 1); +} + +TEST_F(OptionsParserTest, DumpAndParse) { + DBOptions base_db_opt; + std::vector base_cf_opts; + std::vector cf_names = {"default", "cf1", "cf2", "cf3", + "c:f:4:4:4" + "p\\i\\k\\a\\chu\\\\\\", + "###rocksdb#1-testcf#2###"}; + const int num_cf = static_cast(cf_names.size()); + Random rnd(302); + test::RandomInitDBOptions(&base_db_opt, &rnd); + base_db_opt.db_log_dir += "/#odd #but #could #happen #path #/\\\\#OMG"; + + BlockBasedTableOptions special_bbto; + special_bbto.cache_index_and_filter_blocks = true; + special_bbto.block_size = 999999; + + for (int c = 0; c < num_cf; ++c) { + ColumnFamilyOptions cf_opt; + Random cf_rnd(0xFB + c); + test::RandomInitCFOptions(&cf_opt, base_db_opt, &cf_rnd); + if (c < 4) { + cf_opt.prefix_extractor.reset(test::RandomSliceTransform(&rnd, c)); + } + if (c < 3) { + cf_opt.table_factory.reset(test::RandomTableFactory(&rnd, c)); + } else if (c == 4) { + cf_opt.table_factory.reset(NewBlockBasedTableFactory(special_bbto)); + } else if (c == 5) { + // A table factory that doesn't support deserialization should be + // supported. + cf_opt.table_factory.reset(new UnregisteredTableFactory()); + } + base_cf_opts.emplace_back(cf_opt); + } + + const std::string kOptionsFileName = "test-persisted-options.ini"; + // Use default for escaped(true), unknown(false) and check (exact) + ConfigOptions config_options; + ASSERT_OK(PersistRocksDBOptions(base_db_opt, cf_names, base_cf_opts, + kOptionsFileName, fs_.get())); + + RocksDBOptionsParser parser; + ASSERT_OK(parser.Parse(config_options, kOptionsFileName, fs_.get())); + + // Make sure block-based table factory options was deserialized correctly + std::shared_ptr ttf = (*parser.cf_opts())[4].table_factory; + ASSERT_EQ(TableFactory::kBlockBasedTableName(), std::string(ttf->Name())); + const auto parsed_bbto = ttf->GetOptions(); + ASSERT_NE(parsed_bbto, nullptr); + ASSERT_EQ(special_bbto.block_size, parsed_bbto->block_size); + ASSERT_EQ(special_bbto.cache_index_and_filter_blocks, + parsed_bbto->cache_index_and_filter_blocks); + + ASSERT_OK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( + config_options, base_db_opt, cf_names, base_cf_opts, kOptionsFileName, + fs_.get())); + + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions( + config_options, *parser.db_opt(), base_db_opt)); + for (int c = 0; c < num_cf; ++c) { + const auto* cf_opt = parser.GetCFOptions(cf_names[c]); + ASSERT_NE(cf_opt, nullptr); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( + config_options, base_cf_opts[c], *cf_opt, + &(parser.cf_opt_maps()->at(c)))); + } + + // Further verify pointer-typed options + for (int c = 0; c < num_cf; ++c) { + const auto* cf_opt = parser.GetCFOptions(cf_names[c]); + ASSERT_NE(cf_opt, nullptr); + VerifyCFPointerTypedOptions(&base_cf_opts[c], cf_opt, + &(parser.cf_opt_maps()->at(c))); + } + + ASSERT_EQ(parser.GetCFOptions("does not exist"), nullptr); + + base_db_opt.max_open_files++; + ASSERT_NOK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( + config_options, base_db_opt, cf_names, base_cf_opts, kOptionsFileName, + fs_.get())); + + for (int c = 0; c < num_cf; ++c) { + if (base_cf_opts[c].compaction_filter) { + delete base_cf_opts[c].compaction_filter; + } + } +} + +TEST_F(OptionsParserTest, DifferentDefault) { + const std::string kOptionsFileName = "test-persisted-options.ini"; + + ColumnFamilyOptions cf_level_opts; + ASSERT_EQ(CompactionPri::kMinOverlappingRatio, cf_level_opts.compaction_pri); + cf_level_opts.OptimizeLevelStyleCompaction(); + + ColumnFamilyOptions cf_univ_opts; + cf_univ_opts.OptimizeUniversalStyleCompaction(); + + ASSERT_OK(PersistRocksDBOptions(DBOptions(), {"default", "universal"}, + {cf_level_opts, cf_univ_opts}, + kOptionsFileName, fs_.get())); + + RocksDBOptionsParser parser; + ASSERT_OK(parser.Parse(kOptionsFileName, fs_.get(), false, + 4096 /* readahead_size */)); + + { + Options old_default_opts; + old_default_opts.OldDefaults(); + ASSERT_EQ(10 * 1048576, old_default_opts.max_bytes_for_level_base); + ASSERT_EQ(5000, old_default_opts.max_open_files); + ASSERT_EQ(2 * 1024U * 1024U, old_default_opts.delayed_write_rate); + ASSERT_EQ(WALRecoveryMode::kTolerateCorruptedTailRecords, + old_default_opts.wal_recovery_mode); + } + { + Options old_default_opts; + old_default_opts.OldDefaults(4, 6); + ASSERT_EQ(10 * 1048576, old_default_opts.max_bytes_for_level_base); + ASSERT_EQ(5000, old_default_opts.max_open_files); + } + { + Options old_default_opts; + old_default_opts.OldDefaults(4, 7); + ASSERT_NE(10 * 1048576, old_default_opts.max_bytes_for_level_base); + ASSERT_NE(4, old_default_opts.table_cache_numshardbits); + ASSERT_EQ(5000, old_default_opts.max_open_files); + ASSERT_EQ(2 * 1024U * 1024U, old_default_opts.delayed_write_rate); + } + { + ColumnFamilyOptions old_default_cf_opts; + old_default_cf_opts.OldDefaults(); + ASSERT_EQ(2 * 1048576, old_default_cf_opts.target_file_size_base); + ASSERT_EQ(4 << 20, old_default_cf_opts.write_buffer_size); + ASSERT_EQ(2 * 1048576, old_default_cf_opts.target_file_size_base); + ASSERT_EQ(0, old_default_cf_opts.soft_pending_compaction_bytes_limit); + ASSERT_EQ(0, old_default_cf_opts.hard_pending_compaction_bytes_limit); + ASSERT_EQ(CompactionPri::kByCompensatedSize, + old_default_cf_opts.compaction_pri); + } + { + ColumnFamilyOptions old_default_cf_opts; + old_default_cf_opts.OldDefaults(4, 6); + ASSERT_EQ(2 * 1048576, old_default_cf_opts.target_file_size_base); + ASSERT_EQ(CompactionPri::kByCompensatedSize, + old_default_cf_opts.compaction_pri); + } + { + ColumnFamilyOptions old_default_cf_opts; + old_default_cf_opts.OldDefaults(4, 7); + ASSERT_NE(2 * 1048576, old_default_cf_opts.target_file_size_base); + ASSERT_EQ(CompactionPri::kByCompensatedSize, + old_default_cf_opts.compaction_pri); + } + { + Options old_default_opts; + old_default_opts.OldDefaults(5, 1); + ASSERT_EQ(2 * 1024U * 1024U, old_default_opts.delayed_write_rate); + } + { + Options old_default_opts; + old_default_opts.OldDefaults(5, 2); + ASSERT_EQ(16 * 1024U * 1024U, old_default_opts.delayed_write_rate); + ASSERT_TRUE(old_default_opts.compaction_pri == + CompactionPri::kByCompensatedSize); + } + { + Options old_default_opts; + old_default_opts.OldDefaults(5, 18); + ASSERT_TRUE(old_default_opts.compaction_pri == + CompactionPri::kByCompensatedSize); + } + + Options small_opts; + small_opts.OptimizeForSmallDb(); + ASSERT_EQ(2 << 20, small_opts.write_buffer_size); + ASSERT_EQ(5000, small_opts.max_open_files); +} + +class OptionsSanityCheckTest : public OptionsParserTest, + public ::testing::WithParamInterface { + protected: + ConfigOptions config_options_; + + public: + OptionsSanityCheckTest() { + config_options_.ignore_unknown_options = false; + config_options_.ignore_unsupported_options = GetParam(); + config_options_.input_strings_escaped = true; + } + + protected: + Status SanityCheckOptions(const DBOptions& db_opts, + const ColumnFamilyOptions& cf_opts, + ConfigOptions::SanityLevel level) { + config_options_.sanity_level = level; + return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( + config_options_, db_opts, {"default"}, {cf_opts}, kOptionsFileName, + fs_.get()); + } + + Status SanityCheckCFOptions(const ColumnFamilyOptions& cf_opts, + ConfigOptions::SanityLevel level) { + return SanityCheckOptions(DBOptions(), cf_opts, level); + } + + void SanityCheckCFOptions(const ColumnFamilyOptions& opts, bool exact) { + ASSERT_OK(SanityCheckCFOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone)); + if (exact) { + ASSERT_OK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); + } else { + ASSERT_NOK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); + } + } + + Status SanityCheckDBOptions(const DBOptions& db_opts, + ConfigOptions::SanityLevel level) { + return SanityCheckOptions(db_opts, ColumnFamilyOptions(), level); + } + + void SanityCheckDBOptions(const DBOptions& opts, bool exact) { + ASSERT_OK(SanityCheckDBOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckDBOptions(opts, ConfigOptions::kSanityLevelNone)); + if (exact) { + ASSERT_OK( + SanityCheckDBOptions(opts, ConfigOptions::kSanityLevelExactMatch)); + } else { + ASSERT_NOK( + SanityCheckDBOptions(opts, ConfigOptions::kSanityLevelExactMatch)); + } + } + + Status PersistOptions(const DBOptions& db_opts, + const ColumnFamilyOptions& cf_opts) { + Status s = fs_->DeleteFile(kOptionsFileName, IOOptions(), nullptr); + if (!s.ok()) { + return s; + } + return PersistRocksDBOptions(db_opts, {"default"}, {cf_opts}, + kOptionsFileName, fs_.get()); + } + + Status PersistCFOptions(const ColumnFamilyOptions& cf_opts) { + return PersistOptions(DBOptions(), cf_opts); + } + + Status PersistDBOptions(const DBOptions& db_opts) { + return PersistOptions(db_opts, ColumnFamilyOptions()); + } + + const std::string kOptionsFileName = "OPTIONS"; +}; + +TEST_P(OptionsSanityCheckTest, MergeOperatorErrorMessage) { + ColumnFamilyOptions opts; + Random rnd(301); + opts.merge_operator.reset(test::RandomMergeOperator(&rnd)); + std::string merge_op_name = opts.merge_operator->Name(); + ASSERT_OK(PersistCFOptions(opts)); + + // Test when going from merge operator -> nullptr + opts.merge_operator = nullptr; + Status s = + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelLooselyCompatible); + ASSERT_TRUE(s.IsInvalidArgument()); + std::string err_msg = s.ToString(); + std::string specified = "The specified one is " + kNullptrString; + std::string persisted = "the persisted one is " + merge_op_name; + ASSERT_TRUE(err_msg.find(specified) != std::string::npos); + ASSERT_TRUE(err_msg.find(persisted) != std::string::npos); + + // Test when using a different merge operator + opts.merge_operator.reset(test::RandomMergeOperator(&rnd)); + s = SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelLooselyCompatible); + ASSERT_TRUE(s.IsInvalidArgument()); + err_msg = s.ToString(); + specified = + "The specified one is " + std::string(opts.merge_operator->Name()); + persisted = "the persisted one is " + merge_op_name; + ASSERT_TRUE(err_msg.find(specified) != std::string::npos); + ASSERT_TRUE(err_msg.find(persisted) != std::string::npos); +} + +TEST_P(OptionsSanityCheckTest, CFOptionsSanityCheck) { + ColumnFamilyOptions opts; + Random rnd(301); + + // default ColumnFamilyOptions + { + ASSERT_OK(PersistCFOptions(opts)); + ASSERT_OK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); + } + + // prefix_extractor + { + // Okay to change prefix_extractor form nullptr to non-nullptr + ASSERT_EQ(opts.prefix_extractor.get(), nullptr); + opts.prefix_extractor.reset(NewCappedPrefixTransform(10)); + ASSERT_OK(SanityCheckCFOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone)); + + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + ASSERT_OK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); + + // use same prefix extractor but with different parameter + opts.prefix_extractor.reset(NewCappedPrefixTransform(15)); + // expect pass only in + // ConfigOptions::kSanityLevelLooselyCompatible + ASSERT_NOK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); + ASSERT_OK(SanityCheckCFOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone)); + + // repeat the test with FixedPrefixTransform + opts.prefix_extractor.reset(NewFixedPrefixTransform(10)); + ASSERT_NOK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); + ASSERT_OK(SanityCheckCFOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone)); + + // persist the change of prefix_extractor + ASSERT_OK(PersistCFOptions(opts)); + ASSERT_OK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); + + // use same prefix extractor but with different parameter + opts.prefix_extractor.reset(NewFixedPrefixTransform(15)); + // expect pass only in + // ConfigOptions::kSanityLevelLooselyCompatible + SanityCheckCFOptions(opts, false); + + // Change prefix extractor from non-nullptr to nullptr + opts.prefix_extractor.reset(); + // expect pass as it's safe to change prefix_extractor + // from non-null to null + ASSERT_OK(SanityCheckCFOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone)); + } + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); + + // table_factory + { + for (int tb = 0; tb <= 2; ++tb) { + // change the table factory + opts.table_factory.reset(test::RandomTableFactory(&rnd, tb)); + ASSERT_NOK(SanityCheckCFOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone)); + + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + ASSERT_OK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); + } + } + + // merge_operator + { + // Test when going from nullptr -> merge operator + opts.merge_operator.reset(test::RandomMergeOperator(&rnd)); + ASSERT_OK(SanityCheckCFOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone)); + + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + SanityCheckCFOptions(opts, config_options_.ignore_unsupported_options); + + for (int test = 0; test < 5; ++test) { + // change the merge operator + opts.merge_operator.reset(test::RandomMergeOperator(&rnd)); + ASSERT_NOK(SanityCheckCFOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone)); + + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + SanityCheckCFOptions(opts, config_options_.ignore_unsupported_options); + } + + // Test when going from merge operator -> nullptr + opts.merge_operator = nullptr; + ASSERT_NOK(SanityCheckCFOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone)); + + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + SanityCheckCFOptions(opts, true); + } + + // compaction_filter + { + for (int test = 0; test < 5; ++test) { + // change the compaction filter + opts.compaction_filter = test::RandomCompactionFilter(&rnd); + SanityCheckCFOptions(opts, false); + + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + SanityCheckCFOptions(opts, config_options_.ignore_unsupported_options); + delete opts.compaction_filter; + opts.compaction_filter = nullptr; + } + } + + // compaction_filter_factory + { + for (int test = 0; test < 5; ++test) { + // change the compaction filter factory + opts.compaction_filter_factory.reset( + test::RandomCompactionFilterFactory(&rnd)); + SanityCheckCFOptions(opts, false); + + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + SanityCheckCFOptions(opts, config_options_.ignore_unsupported_options); + } + } + + // persist_user_defined_timestamps + { + // Test change from true to false not allowed in loose and exact mode. + opts.persist_user_defined_timestamps = false; + ASSERT_NOK(SanityCheckCFOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); + ASSERT_NOK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); + + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + SanityCheckCFOptions(opts, config_options_.ignore_unsupported_options); + + // Test change from false to true not allowed in loose and exact mode. + opts.persist_user_defined_timestamps = true; + ASSERT_NOK(SanityCheckCFOptions( + opts, ConfigOptions::kSanityLevelLooselyCompatible)); + ASSERT_NOK( + SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch)); + + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + } +} + +TEST_P(OptionsSanityCheckTest, DBOptionsSanityCheck) { + DBOptions opts; + Random rnd(301); + + // default DBOptions + { + ASSERT_OK(PersistDBOptions(opts)); + ASSERT_OK( + SanityCheckDBOptions(opts, ConfigOptions::kSanityLevelExactMatch)); + } + + // File checksum generator + { + class MockFileChecksumGenFactory : public FileChecksumGenFactory { + public: + static const char* kClassName() { return "Mock"; } + const char* Name() const override { return kClassName(); } + std::unique_ptr CreateFileChecksumGenerator( + const FileChecksumGenContext& /*context*/) override { + return nullptr; + } + }; + + // Okay to change file_checksum_gen_factory form nullptr to non-nullptr + ASSERT_EQ(opts.file_checksum_gen_factory.get(), nullptr); + opts.file_checksum_gen_factory.reset(new MockFileChecksumGenFactory()); + + // persist the change + ASSERT_OK(PersistDBOptions(opts)); + SanityCheckDBOptions(opts, config_options_.ignore_unsupported_options); + + // Change file_checksum_gen_factory from non-nullptr to nullptr + opts.file_checksum_gen_factory.reset(); + // expect pass as it's safe to change file_checksum_gen_factory + // from non-null to null + SanityCheckDBOptions(opts, false); + } + // persist the change + ASSERT_OK(PersistDBOptions(opts)); + ASSERT_OK(SanityCheckDBOptions(opts, ConfigOptions::kSanityLevelExactMatch)); +} + +namespace { +bool IsEscapedString(const std::string& str) { + for (size_t i = 0; i < str.size(); ++i) { + if (str[i] == '\\') { + // since we already handle those two consecutive '\'s in + // the next if-then branch, any '\' appear at the end + // of an escaped string in such case is not valid. + if (i == str.size() - 1) { + return false; + } + if (str[i + 1] == '\\') { + // if there're two consecutive '\'s, skip the second one. + i++; + continue; + } + switch (str[i + 1]) { + case ':': + case '\\': + case '#': + continue; + default: + // if true, '\' together with str[i + 1] is not a valid escape. + if (UnescapeChar(str[i + 1]) == str[i + 1]) { + return false; + } + } + } else if (isSpecialChar(str[i]) && (i == 0 || str[i - 1] != '\\')) { + return false; + } + } + return true; +} +} // namespace + +TEST_F(OptionsParserTest, IntegerParsing) { + ASSERT_EQ(ParseUint64("18446744073709551615"), 18446744073709551615U); + ASSERT_EQ(ParseUint32("4294967295"), 4294967295U); + ASSERT_EQ(ParseSizeT("18446744073709551615"), 18446744073709551615U); + ASSERT_EQ(ParseInt64("9223372036854775807"), 9223372036854775807); + ASSERT_EQ(ParseInt64("-9223372036854775808"), + std::numeric_limits::min()); + ASSERT_EQ(ParseInt32("2147483647"), 2147483647); + ASSERT_EQ(ParseInt32("-2147483648"), std::numeric_limits::min()); + ASSERT_EQ(ParseInt("-32767"), -32767); + ASSERT_EQ(ParseDouble("-1.234567"), -1.234567); +} + +TEST_F(OptionsParserTest, EscapeOptionString) { + ASSERT_EQ(UnescapeOptionString( + "This is a test string with \\# \\: and \\\\ escape chars."), + "This is a test string with # : and \\ escape chars."); + + ASSERT_EQ( + EscapeOptionString("This is a test string with # : and \\ escape chars."), + "This is a test string with \\# \\: and \\\\ escape chars."); + + std::string readible_chars = + "A String like this \"1234567890-=_)(*&^%$#@!ertyuiop[]{POIU" + "YTREWQasdfghjkl;':LKJHGFDSAzxcvbnm,.?>" + " +void TestOptInfo(const ConfigOptions& config_options, OptionType opt_type, + T* base, T* comp) { + std::string result; + OptionTypeInfo opt_info(0, opt_type); + ASSERT_FALSE(opt_info.AreEqual(config_options, "base", base, comp, &result)); + ASSERT_EQ(result, "base"); + ASSERT_NE(*base, *comp); + TestAndCompareOption(config_options, opt_info, "base", base, comp); + ASSERT_EQ(*base, *comp); +} + +class OptionTypeInfoTest : public testing::Test {}; + +TEST_F(OptionTypeInfoTest, BasicTypes) { + ConfigOptions config_options; + { + bool a = true, b = false; + TestOptInfo(config_options, OptionType::kBoolean, &a, &b); + } + { + int a = 100, b = 200; + TestOptInfo(config_options, OptionType::kInt, &a, &b); + } + { + int32_t a = 100, b = 200; + TestOptInfo(config_options, OptionType::kInt32T, &a, &b); + } + { + int64_t a = 100, b = 200; + TestOptInfo(config_options, OptionType::kInt64T, &a, &b); + } + { + unsigned int a = 100, b = 200; + TestOptInfo(config_options, OptionType::kUInt, &a, &b); + } + { + uint32_t a = 100, b = 200; + TestOptInfo(config_options, OptionType::kUInt32T, &a, &b); + } + { + uint64_t a = 100, b = 200; + TestOptInfo(config_options, OptionType::kUInt64T, &a, &b); + } + { + size_t a = 100, b = 200; + TestOptInfo(config_options, OptionType::kSizeT, &a, &b); + } + { + std::string a = "100", b = "200"; + TestOptInfo(config_options, OptionType::kString, &a, &b); + } + { + double a = 1.0, b = 2.0; + TestOptInfo(config_options, OptionType::kDouble, &a, &b); + } +} + +TEST_F(OptionTypeInfoTest, TestInvalidArgs) { + ConfigOptions config_options; + bool b; + int i; + int32_t i32; + int64_t i64; + unsigned int u; + int32_t u32; + int64_t u64; + size_t sz; + double d; + + ASSERT_NOK(OptionTypeInfo(0, OptionType::kBoolean) + .Parse(config_options, "b", "x", &b)); + ASSERT_NOK( + OptionTypeInfo(0, OptionType::kInt).Parse(config_options, "b", "x", &i)); + ASSERT_NOK(OptionTypeInfo(0, OptionType::kInt32T) + .Parse(config_options, "b", "x", &i32)); + ASSERT_NOK(OptionTypeInfo(0, OptionType::kInt64T) + .Parse(config_options, "b", "x", &i64)); + ASSERT_NOK( + OptionTypeInfo(0, OptionType::kUInt).Parse(config_options, "b", "x", &u)); + ASSERT_NOK(OptionTypeInfo(0, OptionType::kUInt32T) + .Parse(config_options, "b", "x", &u32)); + ASSERT_NOK(OptionTypeInfo(0, OptionType::kUInt64T) + .Parse(config_options, "b", "x", &u64)); + ASSERT_NOK(OptionTypeInfo(0, OptionType::kSizeT) + .Parse(config_options, "b", "x", &sz)); + ASSERT_NOK(OptionTypeInfo(0, OptionType::kDouble) + .Parse(config_options, "b", "x", &d)); + + // Don't know how to convert Unknowns to anything else + ASSERT_NOK(OptionTypeInfo(0, OptionType::kUnknown) + .Parse(config_options, "b", "x", &d)); + + // Verify that if the parse function throws an exception, it is also trapped + OptionTypeInfo func_info(0, OptionType::kUnknown, + OptionVerificationType::kNormal, + OptionTypeFlags::kNone, + [](const ConfigOptions&, const std::string&, + const std::string& value, void* addr) { + auto ptr = static_cast(addr); + *ptr = ParseInt(value); + return Status::OK(); + }); + ASSERT_OK(func_info.Parse(config_options, "b", "1", &i)); + ASSERT_NOK(func_info.Parse(config_options, "b", "x", &i)); +} + +TEST_F(OptionTypeInfoTest, TestParseFunc) { + OptionTypeInfo opt_info(0, OptionType::kUnknown, + OptionVerificationType::kNormal, + OptionTypeFlags::kNone); + opt_info.SetParseFunc([](const ConfigOptions& /*opts*/, + const std::string& name, const std::string& value, + void* addr) { + auto ptr = static_cast(addr); + if (name == "Oops") { + return Status::InvalidArgument(value); + } else { + *ptr = value + " " + name; + return Status::OK(); + } + }); + ConfigOptions config_options; + std::string base; + ASSERT_OK(opt_info.Parse(config_options, "World", "Hello", &base)); + ASSERT_EQ(base, "Hello World"); + ASSERT_NOK(opt_info.Parse(config_options, "Oops", "Hello", &base)); +} + +TEST_F(OptionTypeInfoTest, TestSerializeFunc) { + OptionTypeInfo opt_info(0, OptionType::kString, + OptionVerificationType::kNormal, + OptionTypeFlags::kNone); + opt_info.SetSerializeFunc([](const ConfigOptions& /*opts*/, + const std::string& name, const void* /*addr*/, + std::string* value) { + if (name == "Oops") { + return Status::InvalidArgument(name); + } else { + *value = name; + return Status::OK(); + } + }); + ConfigOptions config_options; + std::string base; + std::string value; + ASSERT_OK(opt_info.Serialize(config_options, "Hello", &base, &value)); + ASSERT_EQ(value, "Hello"); + ASSERT_NOK(opt_info.Serialize(config_options, "Oops", &base, &value)); +} + +TEST_F(OptionTypeInfoTest, TestEqualsFunc) { + OptionTypeInfo opt_info(0, OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone); + opt_info.SetEqualsFunc([](const ConfigOptions& /*opts*/, + const std::string& name, const void* addr1, + const void* addr2, std::string* mismatch) { + auto i1 = *(static_cast(addr1)); + auto i2 = *(static_cast(addr2)); + if (name == "LT") { + return i1 < i2; + } else if (name == "GT") { + return i1 > i2; + } else if (name == "EQ") { + return i1 == i2; + } else { + *mismatch = name + "???"; + return false; + } + }); + + ConfigOptions config_options; + int int1 = 100; + int int2 = 200; + std::string mismatch; + ASSERT_TRUE(opt_info.AreEqual(config_options, "LT", &int1, &int2, &mismatch)); + ASSERT_EQ(mismatch, ""); + ASSERT_FALSE( + opt_info.AreEqual(config_options, "GT", &int1, &int2, &mismatch)); + ASSERT_EQ(mismatch, "GT"); + ASSERT_FALSE( + opt_info.AreEqual(config_options, "NO", &int1, &int2, &mismatch)); + ASSERT_EQ(mismatch, "NO???"); +} + +TEST_F(OptionTypeInfoTest, TestPrepareFunc) { + OptionTypeInfo opt_info(0, OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone); + opt_info.SetPrepareFunc( + [](const ConfigOptions& /*opts*/, const std::string& name, void* addr) { + auto i1 = static_cast(addr); + if (name == "x2") { + *i1 *= 2; + } else if (name == "/2") { + *i1 /= 2; + } else { + return Status::InvalidArgument("Bad Argument", name); + } + return Status::OK(); + }); + ConfigOptions config_options; + int int1 = 100; + ASSERT_OK(opt_info.Prepare(config_options, "x2", &int1)); + ASSERT_EQ(int1, 200); + ASSERT_OK(opt_info.Prepare(config_options, "/2", &int1)); + ASSERT_EQ(int1, 100); + ASSERT_NOK(opt_info.Prepare(config_options, "??", &int1)); + ASSERT_EQ(int1, 100); +} +TEST_F(OptionTypeInfoTest, TestValidateFunc) { + OptionTypeInfo opt_info(0, OptionType::kSizeT, + OptionVerificationType::kNormal, + OptionTypeFlags::kNone); + opt_info.SetValidateFunc([](const DBOptions& db_opts, + const ColumnFamilyOptions& cf_opts, + const std::string& name, const void* addr) { + const auto sz = static_cast(addr); + bool is_valid = false; + if (name == "keep_log_file_num") { + is_valid = (*sz == db_opts.keep_log_file_num); + } else if (name == "write_buffer_size") { + is_valid = (*sz == cf_opts.write_buffer_size); + } + if (is_valid) { + return Status::OK(); + } else { + return Status::InvalidArgument("Mismatched value", name); + } + }); + ConfigOptions config_options; + DBOptions db_options; + ColumnFamilyOptions cf_options; + + ASSERT_OK(opt_info.Validate(db_options, cf_options, "keep_log_file_num", + &db_options.keep_log_file_num)); + ASSERT_OK(opt_info.Validate(db_options, cf_options, "write_buffer_size", + &cf_options.write_buffer_size)); + ASSERT_NOK(opt_info.Validate(db_options, cf_options, "keep_log_file_num", + &cf_options.write_buffer_size)); + ASSERT_NOK(opt_info.Validate(db_options, cf_options, "write_buffer_size", + &db_options.keep_log_file_num)); +} + +TEST_F(OptionTypeInfoTest, TestOptionFlags) { + OptionTypeInfo opt_none(0, OptionType::kString, + OptionVerificationType::kNormal, + OptionTypeFlags::kDontSerialize); + OptionTypeInfo opt_never(0, OptionType::kString, + OptionVerificationType::kNormal, + OptionTypeFlags::kCompareNever); + OptionTypeInfo opt_alias(0, OptionType::kString, + OptionVerificationType::kAlias, + OptionTypeFlags::kNone); + OptionTypeInfo opt_deprecated(0, OptionType::kString, + OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone); + ConfigOptions config_options; + std::string opts_str; + std::string base = "base"; + std::string comp = "comp"; + + // If marked string none, the serialization returns not supported + ASSERT_NOK(opt_none.Serialize(config_options, "None", &base, &opts_str)); + // If marked never compare, they match even when they do not + ASSERT_TRUE(opt_never.AreEqual(config_options, "Never", &base, &comp, &base)); + ASSERT_FALSE(opt_none.AreEqual(config_options, "Never", &base, &comp, &base)); + + // An alias can change the value via parse, but does nothing on serialize on + // match + std::string result; + ASSERT_OK(opt_alias.Parse(config_options, "Alias", "Alias", &base)); + ASSERT_OK(opt_alias.Serialize(config_options, "Alias", &base, &result)); + ASSERT_TRUE( + opt_alias.AreEqual(config_options, "Alias", &base, &comp, &result)); + ASSERT_EQ(base, "Alias"); + ASSERT_NE(base, comp); + + // Deprecated options do nothing on any of the commands + ASSERT_OK(opt_deprecated.Parse(config_options, "Alias", "Deprecated", &base)); + ASSERT_OK(opt_deprecated.Serialize(config_options, "Alias", &base, &result)); + ASSERT_TRUE( + opt_deprecated.AreEqual(config_options, "Alias", &base, &comp, &result)); + ASSERT_EQ(base, "Alias"); + ASSERT_NE(base, comp); +} + +TEST_F(OptionTypeInfoTest, TestCustomEnum) { + enum TestEnum { kA, kB, kC }; + std::unordered_map enum_map = { + {"A", TestEnum::kA}, + {"B", TestEnum::kB}, + {"C", TestEnum::kC}, + }; + OptionTypeInfo opt_info = OptionTypeInfo::Enum(0, &enum_map); + TestEnum e1, e2; + ConfigOptions config_options; + std::string result, mismatch; + + e2 = TestEnum::kA; + + ASSERT_OK(opt_info.Parse(config_options, "", "B", &e1)); + ASSERT_OK(opt_info.Serialize(config_options, "", &e1, &result)); + ASSERT_EQ(e1, TestEnum::kB); + ASSERT_EQ(result, "B"); + + ASSERT_FALSE(opt_info.AreEqual(config_options, "Enum", &e1, &e2, &mismatch)); + ASSERT_EQ(mismatch, "Enum"); + + TestParseAndCompareOption(config_options, opt_info, "", "C", &e1, &e2); + ASSERT_EQ(e2, TestEnum::kC); + + ASSERT_NOK(opt_info.Parse(config_options, "", "D", &e1)); + ASSERT_EQ(e1, TestEnum::kC); +} + +TEST_F(OptionTypeInfoTest, TestBuiltinEnum) { + ConfigOptions config_options; + for (auto iter : OptionsHelper::compaction_style_string_map) { + CompactionStyle e1, e2; + TestParseAndCompareOption(config_options, + OptionTypeInfo(0, OptionType::kCompactionStyle), + "CompactionStyle", iter.first, &e1, &e2); + ASSERT_EQ(e1, iter.second); + } + for (auto iter : OptionsHelper::compaction_pri_string_map) { + CompactionPri e1, e2; + TestParseAndCompareOption(config_options, + OptionTypeInfo(0, OptionType::kCompactionPri), + "CompactionPri", iter.first, &e1, &e2); + ASSERT_EQ(e1, iter.second); + } + for (auto iter : OptionsHelper::compression_type_string_map) { + CompressionType e1, e2; + TestParseAndCompareOption(config_options, + OptionTypeInfo(0, OptionType::kCompressionType), + "CompressionType", iter.first, &e1, &e2); + ASSERT_EQ(e1, iter.second); + } + for (auto iter : OptionsHelper::compaction_stop_style_string_map) { + CompactionStopStyle e1, e2; + TestParseAndCompareOption( + config_options, OptionTypeInfo(0, OptionType::kCompactionStopStyle), + "CompactionStopStyle", iter.first, &e1, &e2); + ASSERT_EQ(e1, iter.second); + } + for (auto iter : OptionsHelper::checksum_type_string_map) { + ChecksumType e1, e2; + TestParseAndCompareOption(config_options, + OptionTypeInfo(0, OptionType::kChecksumType), + "CheckSumType", iter.first, &e1, &e2); + ASSERT_EQ(e1, iter.second); + } + for (auto iter : OptionsHelper::encoding_type_string_map) { + EncodingType e1, e2; + TestParseAndCompareOption(config_options, + OptionTypeInfo(0, OptionType::kEncodingType), + "EncodingType", iter.first, &e1, &e2); + ASSERT_EQ(e1, iter.second); + } +} + +TEST_F(OptionTypeInfoTest, TestStruct) { + struct Basic { + int i = 42; + std::string s = "Hello"; + }; + + struct Extended { + int j = 11; + Basic b; + }; + + std::unordered_map basic_type_map = { + {"i", {offsetof(struct Basic, i), OptionType::kInt}}, + {"s", {offsetof(struct Basic, s), OptionType::kString}}, + }; + OptionTypeInfo basic_info = OptionTypeInfo::Struct( + "b", &basic_type_map, 0, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable); + + std::unordered_map extended_type_map = { + {"j", {offsetof(struct Extended, j), OptionType::kInt}}, + {"b", OptionTypeInfo::Struct( + "b", &basic_type_map, offsetof(struct Extended, b), + OptionVerificationType::kNormal, OptionTypeFlags::kNone)}, + {"m", OptionTypeInfo::Struct( + "m", &basic_type_map, offsetof(struct Extended, b), + OptionVerificationType::kNormal, OptionTypeFlags::kMutable)}, + }; + OptionTypeInfo extended_info = OptionTypeInfo::Struct( + "e", &extended_type_map, 0, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable); + Extended e1, e2; + ConfigOptions config_options; + std::string mismatch; + TestParseAndCompareOption(config_options, basic_info, "b", "{i=33;s=33}", + &e1.b, &e2.b); + ASSERT_EQ(e1.b.i, 33); + ASSERT_EQ(e1.b.s, "33"); + + TestParseAndCompareOption(config_options, basic_info, "b.i", "44", &e1.b, + &e2.b); + ASSERT_EQ(e1.b.i, 44); + + TestParseAndCompareOption(config_options, basic_info, "i", "55", &e1.b, + &e2.b); + ASSERT_EQ(e1.b.i, 55); + + e1.b.i = 0; + + ASSERT_FALSE( + basic_info.AreEqual(config_options, "b", &e1.b, &e2.b, &mismatch)); + ASSERT_EQ(mismatch, "b.i"); + mismatch.clear(); + ASSERT_FALSE( + basic_info.AreEqual(config_options, "b.i", &e1.b, &e2.b, &mismatch)); + ASSERT_EQ(mismatch, "b.i"); + mismatch.clear(); + ASSERT_FALSE( + basic_info.AreEqual(config_options, "i", &e1.b, &e2.b, &mismatch)); + ASSERT_EQ(mismatch, "b.i"); + mismatch.clear(); + + e1 = e2; + ASSERT_NOK(basic_info.Parse(config_options, "b", "{i=33;s=33;j=44}", &e1.b)); + ASSERT_NOK(basic_info.Parse(config_options, "b.j", "44", &e1.b)); + ASSERT_NOK(basic_info.Parse(config_options, "j", "44", &e1.b)); + + TestParseAndCompareOption(config_options, extended_info, "e", + "b={i=55;s=55}; j=22;", &e1, &e2); + ASSERT_EQ(e1.b.i, 55); + ASSERT_EQ(e1.j, 22); + ASSERT_EQ(e1.b.s, "55"); + TestParseAndCompareOption(config_options, extended_info, "e.b", + "{i=66;s=66;}", &e1, &e2); + ASSERT_EQ(e1.b.i, 66); + ASSERT_EQ(e1.j, 22); + ASSERT_EQ(e1.b.s, "66"); + TestParseAndCompareOption(config_options, extended_info, "e.b.i", "77", &e1, + &e2); + ASSERT_EQ(e1.b.i, 77); + ASSERT_EQ(e1.j, 22); + ASSERT_EQ(e1.b.s, "66"); +} + +TEST_F(OptionTypeInfoTest, TestArrayType) { + OptionTypeInfo array_info = OptionTypeInfo::Array( + 0, OptionVerificationType::kNormal, OptionTypeFlags::kNone, + {0, OptionType::kString}); + std::array array1, array2; + std::string mismatch; + + ConfigOptions config_options; + TestParseAndCompareOption(config_options, array_info, "v", "a:b:c:d", &array1, + &array2); + + ASSERT_EQ(array1.size(), 4); + ASSERT_EQ(array1[0], "a"); + ASSERT_EQ(array1[1], "b"); + ASSERT_EQ(array1[2], "c"); + ASSERT_EQ(array1[3], "d"); + array1[3] = "e"; + ASSERT_FALSE( + array_info.AreEqual(config_options, "v", &array1, &array2, &mismatch)); + ASSERT_EQ(mismatch, "v"); + + // Test vectors with inner brackets + TestParseAndCompareOption(config_options, array_info, "v", "a:{b}:c:d", + &array1, &array2); + ASSERT_EQ(array1.size(), 4); + ASSERT_EQ(array1[0], "a"); + ASSERT_EQ(array1[1], "b"); + ASSERT_EQ(array1[2], "c"); + ASSERT_EQ(array1[3], "d"); + + std::array array3, array4; + OptionTypeInfo bar_info = OptionTypeInfo::Array( + 0, OptionVerificationType::kNormal, OptionTypeFlags::kNone, + {0, OptionType::kString}, '|'); + TestParseAndCompareOption(config_options, bar_info, "v", "x|y|z", &array3, + &array4); + + // Test arrays with inner array + TestParseAndCompareOption(config_options, bar_info, "v", + "a|{b1|b2}|{c1|c2|{d1|d2}}", &array3, &array4, + false); + ASSERT_EQ(array3.size(), 3); + ASSERT_EQ(array3[0], "a"); + ASSERT_EQ(array3[1], "b1|b2"); + ASSERT_EQ(array3[2], "c1|c2|{d1|d2}"); + + TestParseAndCompareOption(config_options, bar_info, "v", + "{a1|a2}|{b1|{c1|c2}}|d1", &array3, &array4, true); + ASSERT_EQ(array3.size(), 3); + ASSERT_EQ(array3[0], "a1|a2"); + ASSERT_EQ(array3[1], "b1|{c1|c2}"); + ASSERT_EQ(array3[2], "d1"); + + // Test invalid input: less element than requested + auto s = bar_info.Parse(config_options, "opt_name1", "a1|a2", &array3); + ASSERT_TRUE(s.IsInvalidArgument()); + + // Test invalid input: more element than requested + s = bar_info.Parse(config_options, "opt_name2", "a1|b|c1|d3", &array3); + ASSERT_TRUE(s.IsInvalidArgument()); +} + +TEST_F(OptionTypeInfoTest, TestVectorType) { + OptionTypeInfo vec_info = OptionTypeInfo::Vector( + 0, OptionVerificationType::kNormal, OptionTypeFlags::kNone, + {0, OptionType::kString}); + std::vector vec1, vec2; + std::string mismatch; + + ConfigOptions config_options; + TestParseAndCompareOption(config_options, vec_info, "v", "a:b:c:d", &vec1, + &vec2); + ASSERT_EQ(vec1.size(), 4); + ASSERT_EQ(vec1[0], "a"); + ASSERT_EQ(vec1[1], "b"); + ASSERT_EQ(vec1[2], "c"); + ASSERT_EQ(vec1[3], "d"); + vec1[3] = "e"; + ASSERT_FALSE(vec_info.AreEqual(config_options, "v", &vec1, &vec2, &mismatch)); + ASSERT_EQ(mismatch, "v"); + + // Test vectors with inner brackets + TestParseAndCompareOption(config_options, vec_info, "v", "a:{b}:c:d", &vec1, + &vec2); + ASSERT_EQ(vec1.size(), 4); + ASSERT_EQ(vec1[0], "a"); + ASSERT_EQ(vec1[1], "b"); + ASSERT_EQ(vec1[2], "c"); + ASSERT_EQ(vec1[3], "d"); + + OptionTypeInfo bar_info = OptionTypeInfo::Vector( + 0, OptionVerificationType::kNormal, OptionTypeFlags::kNone, + {0, OptionType::kString}, '|'); + TestParseAndCompareOption(config_options, vec_info, "v", "x|y|z", &vec1, + &vec2); + // Test vectors with inner vector + TestParseAndCompareOption(config_options, bar_info, "v", + "a|{b1|b2}|{c1|c2|{d1|d2}}", &vec1, &vec2, false); + ASSERT_EQ(vec1.size(), 3); + ASSERT_EQ(vec1[0], "a"); + ASSERT_EQ(vec1[1], "b1|b2"); + ASSERT_EQ(vec1[2], "c1|c2|{d1|d2}"); + + TestParseAndCompareOption(config_options, bar_info, "v", + "{a1|a2}|{b1|{c1|c2}}|d1", &vec1, &vec2, true); + ASSERT_EQ(vec1.size(), 3); + ASSERT_EQ(vec1[0], "a1|a2"); + ASSERT_EQ(vec1[1], "b1|{c1|c2}"); + ASSERT_EQ(vec1[2], "d1"); + + TestParseAndCompareOption(config_options, bar_info, "v", "{a1}", &vec1, &vec2, + false); + ASSERT_EQ(vec1.size(), 1); + ASSERT_EQ(vec1[0], "a1"); + + TestParseAndCompareOption(config_options, bar_info, "v", "{a1|a2}|{b1|b2}", + &vec1, &vec2, true); + ASSERT_EQ(vec1.size(), 2); + ASSERT_EQ(vec1[0], "a1|a2"); + ASSERT_EQ(vec1[1], "b1|b2"); +} + +TEST_F(OptionTypeInfoTest, TestStaticType) { + struct SimpleOptions { + size_t size = 0; + bool verify = true; + }; + + static std::unordered_map type_map = { + {"size", {offsetof(struct SimpleOptions, size), OptionType::kSizeT}}, + {"verify", + {offsetof(struct SimpleOptions, verify), OptionType::kBoolean}}, + }; + + ConfigOptions config_options; + SimpleOptions opts, copy; + opts.size = 12345; + opts.verify = false; + std::string str, mismatch; + + ASSERT_OK( + OptionTypeInfo::SerializeType(config_options, type_map, &opts, &str)); + ASSERT_FALSE(OptionTypeInfo::TypesAreEqual(config_options, type_map, &opts, + ©, &mismatch)); + ASSERT_OK(OptionTypeInfo::ParseType(config_options, str, type_map, ©)); + ASSERT_TRUE(OptionTypeInfo::TypesAreEqual(config_options, type_map, &opts, + ©, &mismatch)); +} + +class ConfigOptionsTest : public testing::Test {}; + +TEST_F(ConfigOptionsTest, EnvFromConfigOptions) { + ConfigOptions config_options; + DBOptions db_opts; + Options opts; + Env* mem_env = NewMemEnv(Env::Default()); + config_options.registry->AddLibrary("custom-env", RegisterCustomEnv, + kCustomEnvName); + + config_options.env = mem_env; + // First test that we can get the env as expected + ASSERT_OK(GetDBOptionsFromString(config_options, DBOptions(), kCustomEnvProp, + &db_opts)); + ASSERT_OK( + GetOptionsFromString(config_options, Options(), kCustomEnvProp, &opts)); + ASSERT_NE(config_options.env, db_opts.env); + ASSERT_EQ(opts.env, db_opts.env); + Env* custom_env = db_opts.env; + + // Now try a "bad" env" and check that nothing changed + config_options.ignore_unsupported_options = true; + ASSERT_OK( + GetDBOptionsFromString(config_options, db_opts, "env=unknown", &db_opts)); + ASSERT_OK(GetOptionsFromString(config_options, opts, "env=unknown", &opts)); + ASSERT_EQ(config_options.env, mem_env); + ASSERT_EQ(db_opts.env, custom_env); + ASSERT_EQ(opts.env, db_opts.env); + + // Now try a "bad" env" ignoring unknown objects + config_options.ignore_unsupported_options = false; + ASSERT_NOK( + GetDBOptionsFromString(config_options, db_opts, "env=unknown", &db_opts)); + ASSERT_EQ(config_options.env, mem_env); + ASSERT_EQ(db_opts.env, custom_env); + ASSERT_EQ(opts.env, db_opts.env); + + delete mem_env; +} +TEST_F(ConfigOptionsTest, MergeOperatorFromString) { + ConfigOptions config_options; + std::shared_ptr merge_op; + + ASSERT_OK(MergeOperator::CreateFromString(config_options, "put", &merge_op)); + ASSERT_NE(merge_op, nullptr); + ASSERT_TRUE(merge_op->IsInstanceOf("put")); + ASSERT_STREQ(merge_op->Name(), "PutOperator"); + + ASSERT_OK( + MergeOperator::CreateFromString(config_options, "put_v1", &merge_op)); + ASSERT_NE(merge_op, nullptr); + ASSERT_TRUE(merge_op->IsInstanceOf("PutOperator")); + + ASSERT_OK( + MergeOperator::CreateFromString(config_options, "uint64add", &merge_op)); + ASSERT_NE(merge_op, nullptr); + ASSERT_TRUE(merge_op->IsInstanceOf("uint64add")); + ASSERT_STREQ(merge_op->Name(), "UInt64AddOperator"); + + ASSERT_OK(MergeOperator::CreateFromString(config_options, "max", &merge_op)); + ASSERT_NE(merge_op, nullptr); + ASSERT_TRUE(merge_op->IsInstanceOf("max")); + ASSERT_STREQ(merge_op->Name(), "MaxOperator"); + + ASSERT_OK( + MergeOperator::CreateFromString(config_options, "bytesxor", &merge_op)); + ASSERT_NE(merge_op, nullptr); + ASSERT_TRUE(merge_op->IsInstanceOf("bytesxor")); + ASSERT_STREQ(merge_op->Name(), BytesXOROperator::kClassName()); + + ASSERT_OK( + MergeOperator::CreateFromString(config_options, "sortlist", &merge_op)); + ASSERT_NE(merge_op, nullptr); + ASSERT_TRUE(merge_op->IsInstanceOf("sortlist")); + ASSERT_STREQ(merge_op->Name(), SortList::kClassName()); + + ASSERT_OK(MergeOperator::CreateFromString(config_options, "stringappend", + &merge_op)); + ASSERT_NE(merge_op, nullptr); + ASSERT_TRUE(merge_op->IsInstanceOf("stringappend")); + ASSERT_STREQ(merge_op->Name(), StringAppendOperator::kClassName()); + auto delimiter = merge_op->GetOptions("Delimiter"); + ASSERT_NE(delimiter, nullptr); + ASSERT_EQ(*delimiter, ","); + + ASSERT_OK(MergeOperator::CreateFromString(config_options, "stringappendtest", + &merge_op)); + ASSERT_NE(merge_op, nullptr); + ASSERT_TRUE(merge_op->IsInstanceOf("stringappendtest")); + ASSERT_STREQ(merge_op->Name(), StringAppendTESTOperator::kClassName()); + delimiter = merge_op->GetOptions("Delimiter"); + ASSERT_NE(delimiter, nullptr); + ASSERT_EQ(*delimiter, ","); + + ASSERT_OK(MergeOperator::CreateFromString( + config_options, "id=stringappend; delimiter=||", &merge_op)); + ASSERT_NE(merge_op, nullptr); + ASSERT_TRUE(merge_op->IsInstanceOf("stringappend")); + ASSERT_STREQ(merge_op->Name(), StringAppendOperator::kClassName()); + delimiter = merge_op->GetOptions("Delimiter"); + ASSERT_NE(delimiter, nullptr); + ASSERT_EQ(*delimiter, "||"); + + ASSERT_OK(MergeOperator::CreateFromString( + config_options, "id=stringappendtest; delimiter=&&", &merge_op)); + ASSERT_NE(merge_op, nullptr); + ASSERT_TRUE(merge_op->IsInstanceOf("stringappendtest")); + ASSERT_STREQ(merge_op->Name(), StringAppendTESTOperator::kClassName()); + delimiter = merge_op->GetOptions("Delimiter"); + ASSERT_NE(delimiter, nullptr); + ASSERT_EQ(*delimiter, "&&"); + + std::shared_ptr copy; + std::string mismatch; + std::string opts_str = merge_op->ToString(config_options); + + ASSERT_OK(MergeOperator::CreateFromString(config_options, opts_str, ©)); + ASSERT_TRUE(merge_op->AreEquivalent(config_options, copy.get(), &mismatch)); + ASSERT_NE(copy, nullptr); + delimiter = copy->GetOptions("Delimiter"); + ASSERT_NE(delimiter, nullptr); + ASSERT_EQ(*delimiter, "&&"); +} + +TEST_F(ConfigOptionsTest, ConfiguringOptionsDoesNotRevertRateLimiterBandwidth) { + // Regression test for bug where rate limiter's dynamically set bandwidth + // could be silently reverted when configuring an options structure with an + // existing `rate_limiter`. + Options base_options; + base_options.rate_limiter.reset( + NewGenericRateLimiter(1 << 20 /* rate_bytes_per_sec */)); + Options copy_options(base_options); + + base_options.rate_limiter->SetBytesPerSecond(2 << 20); + ASSERT_EQ(2 << 20, base_options.rate_limiter->GetBytesPerSecond()); + + ASSERT_OK(GetOptionsFromString(base_options, "", ©_options)); + ASSERT_EQ(2 << 20, base_options.rate_limiter->GetBytesPerSecond()); +} + +INSTANTIATE_TEST_CASE_P(OptionsSanityCheckTest, OptionsSanityCheckTest, + ::testing::Bool()); + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); +#ifdef GFLAGS + ParseCommandLineFlags(&argc, &argv, true); +#endif // GFLAGS + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/plugin/README.md b/librocksdb-sys/rocksdb/plugin/README.md new file mode 100644 index 0000000..5cd8994 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/README.md @@ -0,0 +1,43 @@ +## Building external plugins together with RocksDB + +RocksDB offers several plugin interfaces for developers to customize its behavior. One difficulty developers face is how to make their plugin available to end users. The approach discussed here involves building the external code together with the RocksDB code into a single binary. Note another approach we plan to support involves loading plugins dynamically from shared libraries. + +### Discovery + +We hope developers will mention their work in "PLUGINS.md" so users can easily discover and reuse solutions for customizing RocksDB. + +### Directory organization + +External plugins will be linked according to their name into a subdirectory of "plugin/". For example, a plugin called "dedupfs" would be linked into "plugin/dedupfs/". + +### Build standard + +Currently the only supported build system are make and cmake. + +For make, files in the plugin directory ending in the .mk extension can define the following variables. + +* `$(PLUGIN_NAME)_SOURCES`: these files will be compiled and linked with RocksDB. They can access RocksDB public header files. +* `$(PLUGIN_NAME)_HEADERS`: these files will be installed in the RocksDB header directory. Their paths will be prefixed by "rocksdb/plugin/$(PLUGIN_NAME)/". +* `$(PLUGIN_NAME)_LDFLAGS`: these flags will be passed to the final link step. For example, library dependencies can be propagated here, or symbols can be forcibly included, e.g., for static registration. +* `$(PLUGIN_NAME)_CXXFLAGS`: these flags will be passed to the compiler. For example, they can specify locations of header files in non-standard locations. + +Users will run the usual make commands from the RocksDB directory, specifying the plugins to include in a space-separated list in the variable `ROCKSDB_PLUGINS`. + +For CMake, the CMakeLists.txt file in the plugin directory can define the following variables. + +* `${PLUGIN_NAME}_SOURCES`: these files will be compiled and linked with RocksDB. They can access RocksDB public header files. +* `${PLUGIN_NAME}_COMPILE_FLAGS`: these flags will be passed to the compiler. For example, they can specify locations of header files in non-standard locations. +* `${PLUGIN_NAME}_INCLUDE_PATHS`: paths to directories to search for plugin-specific header files during compilation. +* `${PLUGIN_NAME}_LIBS`: list of library names required to build the plugin, e.g. `dl`, `java`, `jvm`, `rados`, etc. CMake will generate proper flags for linking. +* `${PLUGIN_NAME}_LINK_PATHS`: list of paths for the linker to search for required libraries in additional to standard locations. +* `${PLUGIN_NAME}_CMAKE_SHARED_LINKER_FLAGS` additional linker flags used to generate shared libraries. For example, symbols can be forcibly included, e.g., for static registration. +* `${PLUGIN_NAME}_CMAKE_EXE_LINKER_FLAGS`: additional linker flags used to generate executables. For example, symbols can be forcibly included, e.g., for static registration. + +Users will run the usual cmake commands, specifying the plugins to include in a space-separated list in the command line variable `ROCKSDB_PLUGINS` when invoking cmake. +``` +cmake .. -DROCKSDB_PLUGINS="dedupfs hdfs rados" +``` + +### Example + +For a working example, see [Dedupfs](https://github.com/ajkr/dedupfs). diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/CODE_OF_CONDUCT.md b/librocksdb-sys/rocksdb/plugin/ippcp/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..58dba18 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/CODE_OF_CONDUCT.md @@ -0,0 +1,131 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, + without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +CommunityCodeOfConduct AT intel DOT com. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/CONTRIBUTING.md b/librocksdb-sys/rocksdb/plugin/ippcp/CONTRIBUTING.md new file mode 100644 index 0000000..4fbf8d1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/CONTRIBUTING.md @@ -0,0 +1,57 @@ +# Contributing + +### License + +Intel® Integrated Performance Primitives Cryptography Plugin for RocksDB* Storage Engine is licensed under the terms in [Apache License](LICENSE). By contributing to the project, you agree to the license and copyright terms therein and release your contribution under these terms. + +### Sign your work + +Please use the sign-off line at the end of the patch. Your signature certifies that you wrote the patch or otherwise have the right to pass it on as an open-source patch. The rules are pretty simple: if you can certify +the below (from [developercertificate.org](http://developercertificate.org/)): + +``` +Developer Certificate of Origin +Version 1.1 + +Copyright (C) 2004, 2006 The Linux Foundation and its contributors. +660 York Street, Suite 102, +San Francisco, CA 94110 USA + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + +Developer's Certificate of Origin 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + +(b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. +``` + +Then you just add a line to every git commit message: + + Signed-off-by: Joe Smith + +Use your real name (sorry, no pseudonyms or anonymous contributions.) + +If you set your `user.name` and `user.email` git configs, you can sign your +commit automatically with `git commit -s`. diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/LICENSE b/librocksdb-sys/rocksdb/plugin/ippcp/LICENSE new file mode 100644 index 0000000..9c8f3ea --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/README.md b/librocksdb-sys/rocksdb/plugin/ippcp/README.md new file mode 100644 index 0000000..6c6a35d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/README.md @@ -0,0 +1,62 @@ +# Intel® Integrated Performance Primitives Cryptography Plugin for RocksDB* Storage Engine + +`ippcp` is an encryption provider for RocksDB that is based on Intel's Integrated Performance Primitives for Cryptography (IPPCP). IPPCP is a lightweight cryptography library that is highly optimized for various Intel CPUs. It's used here to provide AES-128/192/256 encryption, with a CTR mode of operation, for RocksDB. + +## Prerequisite + +There is a dependency on ipp cryptograhy library (ippcp) which needs to be installed. Please refer below link for installtion. +https://www.intel.com/content/www/us/en/develop/documentation/get-started-with-ipp-crypto-for-oneapi-linux/top.html + +Once Installed source /opt/intel/oneapi/ippcp/latest/env/var.sh + + +## Build + +The code first needs to be linked under RocksDB's "plugin/" directory. In your RocksDB directory, run: + +``` +$ pushd ./plugin/ +$ git clone https://github.com/intel/ippcp-plugin-rocksdb.git ippcp +``` + +Next, we can build and install RocksDB with this plugin as follows: + +``` +$ popd +$ make clean && ROCKSDB_PLUGINS=ippcp make -j48 release +``` + +## Testing + +* Install ipp cryptograhy library (ippcp) as described in the previous section. +* Install https://github.com/google/googletest +* Build RocksDB as a shared library + +``` +LIB_MODE=shared make -j release + +``` + +* Go to the tests directory of ippcp plugin and build as mentioned below: + +``` +cd plugin/ippcp/tests/ +mkdir build +cd build +cmake -DROCKSDB_PATH= -DIPPCRYPTOROOT= .. +make run + +``` +## Tool usage + +For RocksDB binaries (such as the `db_bench` we built above), the plugin can be enabled through configuration. `db_bench` in particular takes a `--fs_uri` where we can specify "dedupfs" , which is the name registered by this plugin. Example usage: + +``` +$ ./db_bench --benchmarks=fillseq --env_uri=ippcp_db_bench_env --compression_type=none +``` + +## Application usage + +The plugin's interface is also exposed to applications, which can enable it either through configuration or through code. Example available under the "examples/" directory. + +``` \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/examples/Makefile b/librocksdb-sys/rocksdb/plugin/ippcp/examples/Makefile new file mode 100644 index 0000000..81226b5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/examples/Makefile @@ -0,0 +1,12 @@ +include ../../../make_config.mk + +PLATFORM_LDFLAGS += -lrocksdb -lippcp -L../../.. -L../library/lib +.PHONY: clean + +all: ippcp_example + +ippcp_example: ippcp_example.cc + $(CXX) $(CXXFLAGS) $@.cc -o$@ -I../../../include -O2 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) -I../library/include + +clean: + rm -rf ./ippcp_example diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/examples/ippcp_example b/librocksdb-sys/rocksdb/plugin/ippcp/examples/ippcp_example new file mode 100755 index 0000000..d830b46 Binary files /dev/null and b/librocksdb-sys/rocksdb/plugin/ippcp/examples/ippcp_example differ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/examples/ippcp_example.cc b/librocksdb-sys/rocksdb/plugin/ippcp/examples/ippcp_example.cc new file mode 100644 index 0000000..f7b66d9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/examples/ippcp_example.cc @@ -0,0 +1,68 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// Copyright (c) 2020 Intel Corporation +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include +#include +#include +#include +#include +#include + +#include + +#include "../ippcp_provider.h" + +using namespace ROCKSDB_NAMESPACE; + +std::string kDBPath = "/tmp/ipp_aes_example"; + +int main() { + DB* db; + Options options; + options.create_if_missing = true; + + std::shared_ptr provider; + Status status = EncryptionProvider::CreateFromString( + ConfigOptions(), IppcpProvider::kName(), &provider); + assert(status.ok()); + + status = + provider->AddCipher("", "a6d2ae2816157e2b3c4fcf098815f7xb", 32, false); + assert(status.ok()); + + options.env = NewEncryptedEnv(Env::Default(), provider); + + status = DB::Open(options, kDBPath, &db); + assert(status.ok()); + + setbuf(stdout, NULL); + printf("writing 1M records..."); + WriteOptions w_opts; + for (int i = 0; i < 1000000; ++i) { + status = db->Put(w_opts, std::to_string(i), std::to_string(i * i)); + assert(status.ok()); + } + db->Flush(FlushOptions()); + printf("done.\n"); + + printf("reading 1M records..."); + std::string value; + ReadOptions r_opts; + for (int i = 0; i < 1000000; ++i) { + status = db->Get(r_opts, std::to_string(i), &value); + assert(status.ok()); + assert(value == std::to_string(i * i)); + } + printf("done.\n"); + + // Close database + status = db->Close(); + assert(status.ok()); + //status = DestroyDB(kDBPath, options); + //assert(status.ok()); + + return 0; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/examples2/Makefile b/librocksdb-sys/rocksdb/plugin/ippcp/examples2/Makefile new file mode 100644 index 0000000..81226b5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/examples2/Makefile @@ -0,0 +1,12 @@ +include ../../../make_config.mk + +PLATFORM_LDFLAGS += -lrocksdb -lippcp -L../../.. -L../library/lib +.PHONY: clean + +all: ippcp_example + +ippcp_example: ippcp_example.cc + $(CXX) $(CXXFLAGS) $@.cc -o$@ -I../../../include -O2 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) -I../library/include + +clean: + rm -rf ./ippcp_example diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/examples2/ippcp_example b/librocksdb-sys/rocksdb/plugin/ippcp/examples2/ippcp_example new file mode 100755 index 0000000..3ed6d67 Binary files /dev/null and b/librocksdb-sys/rocksdb/plugin/ippcp/examples2/ippcp_example differ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/examples2/ippcp_example.cc b/librocksdb-sys/rocksdb/plugin/ippcp/examples2/ippcp_example.cc new file mode 100644 index 0000000..75b2e10 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/examples2/ippcp_example.cc @@ -0,0 +1,66 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// Copyright (c) 2020 Intel Corporation +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include +#include +#include +#include +#include +#include + +#include + +#include "../ippcp_provider.h" + +using namespace ROCKSDB_NAMESPACE; + +std::string kDBPath = "/tmp/ipp_aes_example2"; + +int main() { + DB* db; + Options options; + options.create_if_missing = true; + + // std::shared_ptr provider; + // Status status = EncryptionProvider::CreateFromString( + // ConfigOptions(), IppcpProvider::kName(), &provider); + // assert(status.ok()); + + // status = + // provider->AddCipher("", "a6d2ae2816157e2b3c4fcf098815f7xb", 32, false); + // assert(status.ok()); + + Status status = DB::Open(options, kDBPath, &db); + assert(status.ok()); + + setbuf(stdout, NULL); + printf("writing 1M records..."); + WriteOptions w_opts; + for (int i = 0; i < 1000000; ++i) { + status = db->Put(w_opts, std::to_string(i), std::to_string(i * i)); + assert(status.ok()); + } + db->Flush(FlushOptions()); + printf("done.\n"); + + printf("reading 1M records..."); + std::string value; + ReadOptions r_opts; + for (int i = 0; i < 1000000; ++i) { + status = db->Get(r_opts, std::to_string(i), &value); + assert(status.ok()); + assert(value == std::to_string(i * i)); + } + printf("done.\n"); + + // Close database + status = db->Close(); + assert(status.ok()); + //status = DestroyDB(kDBPath, options); + //assert(status.ok()); + + return 0; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/ippcp.mk b/librocksdb-sys/rocksdb/plugin/ippcp/ippcp.mk new file mode 100644 index 0000000..c30049b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/ippcp.mk @@ -0,0 +1,4 @@ +ippcp_SOURCES = ippcp_provider.cc +ippcp_HEADERS = ippcp_provider.h +ippcp_LDFLAGS = -lippcp +ippcp_CXXFLAGS = -Iplugin/ippcp/library/include \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/ippcp_db_bench_env.cc b/librocksdb-sys/rocksdb/plugin/ippcp/ippcp_db_bench_env.cc new file mode 100644 index 0000000..5a7e5af --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/ippcp_db_bench_env.cc @@ -0,0 +1,32 @@ +// Copyright (c) 2021-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include + +#include "ippcp_provider.h" + +namespace ROCKSDB_NAMESPACE { + +#ifndef ROCKSDB_LITE + +extern "C" FactoryFunc ippcp_db_bench_env; + +// Registers a sample ippcp encrypted environment that can be used in db_bench +// by passing --env_uri=ippcp_db_bench_env parameter. + +FactoryFunc ippcp_db_bench_env = ObjectLibrary::Default()->AddFactory( + "ippcp_db_bench_env", + [](const std::string& /* uri */, std::unique_ptr* f, + std::string* /* errmsg */) { + auto provider = + std::shared_ptr(IppcpProvider::CreateProvider()); + provider->AddCipher("", "a6d2ae2816157e2b3c4fcf098815f7xb", 32, false); + *f = std::unique_ptr(NewEncryptedEnv(Env::Default(), provider)); + return f->get(); + }); + +#endif // ROCKSDB_LITE + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/ippcp_provider.cc b/librocksdb-sys/rocksdb/plugin/ippcp/ippcp_provider.cc new file mode 100644 index 0000000..e819a91 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/ippcp_provider.cc @@ -0,0 +1,250 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// Copyright (c) 2020 Intel Corporation +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#ifndef ROCKSDB_LITE + +#include "ippcp_provider.h" + +#include +#include +#include +#include "rocksdb/utilities/customizable_util.h" +#include + +#endif + +namespace ROCKSDB_NAMESPACE { + +#ifndef ROCKSDB_LITE + + +static void RegisterEncryptionAES() { + static std::once_flag once; + std::call_once(once, [&]() { + + ObjectLibrary::Default()->AddFactory( + IppcpProvider::kName(), + [](const std::string& /* uri */, std::unique_ptr* f, + std::string* /* errmsg */) { + *f = IppcpProvider::CreateProvider(); + return f->get(); + }); + + }); +} + +Status EncryptionProvider::CreateFromString( + const ConfigOptions& config_options, const std::string& value, + std::shared_ptr* result) { + RegisterEncryptionAES(); + return LoadSharedObject(config_options, value, result); +} + +// extern "C" FactoryFunc ippcp_reg; + +// FactoryFunc ippcp_reg = +// ObjectLibrary::Default()->AddFactory( +// IppcpProvider::kName(), +// [](const std::string& /* uri */, std::unique_ptr* f, +// std::string* /* errmsg */) { +// *f = IppcpProvider::CreateProvider(); +// return f->get(); +// }); + +// IppcpCipherStream implements BlockAccessCipherStream using AES block +// cipher and a CTR mode of operation. +// +// Since ipp-crypto can handle block sizes larger than kBlockSize (16 bytes for +// AES) by chopping them internally into KBlockSize bytes, there is no need to +// support the EncryptBlock and DecryptBlock member functions (and they will +// never be called). +// +// See https://github.com/intel/ipp-crypto#documentation +class IppcpCipherStream : public BlockAccessCipherStream { + public: + static constexpr size_t kBlockSize = 16; // in bytes + static constexpr size_t kCounterLen = 64; // in bits + + IppcpCipherStream(IppsAESSpec* aes_ctx, const char* init_vector); + + virtual Status Encrypt(uint64_t fileOffset, char* data, + size_t dataSize) override; + virtual Status Decrypt(uint64_t fileOffset, char* data, + size_t dataSize) override; + virtual size_t BlockSize() override { return kBlockSize; } + + protected: + // These functions are not needed and will never be called! + virtual void AllocateScratch(std::string&) override {} + virtual Status EncryptBlock(uint64_t, char*, char*) override { + return Status::NotSupported("Operation not supported."); + } + virtual Status DecryptBlock(uint64_t, char*, char*) override { + return Status::NotSupported("Operation not supported."); + } + + private: + IppsAESSpec* aes_ctx_; + __m128i init_vector_; +}; + +IppcpCipherStream::IppcpCipherStream(IppsAESSpec* aes_ctx, + const char* init_vector) + : aes_ctx_(aes_ctx) { + init_vector_ = _mm_loadu_si128((__m128i*)init_vector); +} + +Status IppcpCipherStream::Encrypt(uint64_t fileOffset, char* data, + size_t dataSize) { + if (dataSize == 0) return Status::OK(); + + size_t index = fileOffset / kBlockSize; + size_t offset = fileOffset % kBlockSize; + + Ipp8u ctr_block[kBlockSize]; + + // evaluate the counter block from the block index + __m128i counter = _mm_add_epi64(init_vector_, _mm_cvtsi64_si128(index)); + Ipp8u* ptr_counter = (Ipp8u*)&counter; + for (size_t i = 0; i < kBlockSize; ++i) + ctr_block[i] = ptr_counter[kBlockSize - 1 - i]; + + IppStatus ipp_status = ippStsNoErr; + + //- If offset is != 0, that means we would have first encrypt a partial block at the + //beginning of the offset. That requires us to take the block index at that position and + //manually do the xor operation – first we encrypt a block (called zero_block), and then + //xor it starting at the offset. + //Once that block is encrypted, we may exit (if the dataSize is less than kBlockSize) or + //let ippcrypto start encrypting beginning a kBlockSize aligned offset + //kCounterLen is 64 bits same as index size so that 64 bits are incremented + // and counter stream generated by ipp and above match + // https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38a.pdf + + if (offset == 0) { + ipp_status = ippsAESEncryptCTR((Ipp8u*)(data), (Ipp8u*)data, static_cast(dataSize), + aes_ctx_, ctr_block, kCounterLen); + } else { + Ipp8u zero_block[kBlockSize]{0}; + ipp_status = ippsAESEncryptCTR(zero_block, zero_block, kBlockSize, aes_ctx_, + ctr_block, kCounterLen); + if (ipp_status != ippStsNoErr) + return Status::Aborted(ippcpGetStatusString(ipp_status)); + + size_t n = std::min(kBlockSize - offset, dataSize); + for (size_t i = 0; i < n; ++i) data[i] ^= zero_block[offset + i]; + memset(zero_block, 0, kBlockSize); + + n = kBlockSize - offset; + if (dataSize > n) { + Ipp8u* ptr = (Ipp8u*)(data + n); + ipp_status = ippsAESEncryptCTR(ptr, ptr, static_cast(dataSize - n), aes_ctx_, + ctr_block, kCounterLen); + } + } + if (ipp_status == ippStsNoErr) return Status::OK(); + + return Status::Aborted(ippcpGetStatusString(ipp_status)); +} + +Status IppcpCipherStream::Decrypt(uint64_t fileOffset, char* data, + size_t dataSize) { + // Decryption is implemented as encryption in CTR mode of operation + return Encrypt(fileOffset, data, dataSize); +} + +std::unique_ptr IppcpProvider::CreateProvider() { + return std::unique_ptr(new IppcpProvider); +} + +Status IppcpProvider::AddCipher(const std::string& /*descriptor*/, + const char* cipher, size_t len, + bool /*for_write*/) { + // We currently don't support more than one encryption key + if (aes_ctx_ != nullptr) { + return Status::InvalidArgument("Multiple encryption keys not supported."); + } + + // AES supports key sizes of only 16, 24, or 32 bytes + if (len != 16 && len != 24 && len != 32) { + return Status::InvalidArgument("Invalid key size in provider."); + } + + // len is in bytes + switch (len) { + case 16: + key_size_ = KeySize::AES_128; + break; + case 24: + key_size_ = KeySize::AES_192; + break; + case 32: + key_size_ = KeySize::AES_256; + break; + } + + // get size for context + IppStatus ipp_status = ippsAESGetSize(&ctx_size_); + if (ipp_status != ippStsNoErr) { + return Status::Aborted("Failed to create provider."); + } + + // allocate memory for context + aes_ctx_ = (IppsAESSpec*)(new Ipp8u[ctx_size_]); + assert(aes_ctx_ != nullptr); + + // initialize context + const Ipp8u* key = (const Ipp8u*)(cipher); + ipp_status = + ippsAESInit(key, static_cast(key_size_), aes_ctx_, ctx_size_); + + if (ipp_status != ippStsNoErr) { + // clean up context and abort! + ippsAESInit(0, static_cast(key_size_), aes_ctx_, ctx_size_); + delete[](Ipp8u*) aes_ctx_; + return Status::Aborted("Failed to create provider."); + } + return Status::OK(); +} + +Status IppcpProvider::CreateNewPrefix(const std::string& /*fname*/, + char* prefix, size_t prefixLength) const { + IppStatus ipp_status; + Ipp32u rnd; + const size_t rnd_size = sizeof(Ipp32u); + assert(prefixLength % rnd_size == 0); + for (size_t i = 0; i < prefixLength; i += rnd_size) { + // generate a cryptographically secured random number + ipp_status = ippsPRNGenRDRAND(&rnd, rnd_size << 3, nullptr); + if (ipp_status != ippStsNoErr) + return Status::Aborted(ippcpGetStatusString(ipp_status)); + memcpy(prefix + i, &rnd, rnd_size); + } + IppcpCipherStream cs(aes_ctx_, prefix); + return cs.Encrypt(0, prefix + IppcpCipherStream::kBlockSize, + prefixLength - IppcpCipherStream::kBlockSize); +} + +Status IppcpProvider::CreateCipherStream( + const std::string& /*fname*/, const EnvOptions& /*options*/, Slice& prefix, + std::unique_ptr* result) { + assert(result != nullptr); + assert(prefix.size() >= IppcpCipherStream::kBlockSize); + result->reset(new IppcpCipherStream(aes_ctx_, prefix.data())); + Status ipp_status = (*result)->Decrypt( + 0, (char*)prefix.data() + IppcpCipherStream::kBlockSize, + prefix.size() - IppcpCipherStream::kBlockSize); + return ipp_status; +} + +IppcpProvider::~IppcpProvider() { + ippsAESInit(0, static_cast(key_size_), aes_ctx_, ctx_size_); + delete[](Ipp8u*) aes_ctx_; +} + +#endif // ROCKSDB_LITE + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/ippcp_provider.h b/librocksdb-sys/rocksdb/plugin/ippcp/ippcp_provider.h new file mode 100644 index 0000000..939a4cd --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/ippcp_provider.h @@ -0,0 +1,73 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// Copyright (c) 2020 Intel Corporation +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#if !defined(ROCKSDB_LITE) + +// Includes Intel's Integrated Performance Primitives for Cryptography (IPPCP). +// IPPCP is lightweight cryptography library that is highly-optimized for +// various Intel CPUs. +// +// We use it here to provide an AES-128/192/256 encryption with a CTR mode of +// operation. +// +// Download URL: https://github.com/intel/ipp-crypto. +// + +#include +#include + +#include + +namespace ROCKSDB_NAMESPACE { + +// AES-128, AES-192, and AES-256 encryptions are all supported. +enum struct KeySize { AES_128 = 16, AES_192 = 24, AES_256 = 32 }; + +// This encryption provider uses AES block cipher and a CTR mode of operation +// with a cryptographically secure IV that is randomly generated. +// +// Note: a prefix size of 4096 (4K) is chosen for optimal performance. +// +class IppcpProvider : public EncryptionProvider { + public: + static constexpr size_t kPrefixSize = 4096; + + static std::unique_ptr CreateProvider(); + + static const char* kName() { return "ippcp"; } + + virtual const char* Name() const override { return kName(); } + + virtual size_t GetPrefixLength() const override { return kPrefixSize; } + + virtual Status AddCipher(const std::string& /*descriptor*/, + const char* /*cipher*/, size_t /*len*/, + bool /*for_write*/) override; + + virtual Status CreateNewPrefix(const std::string& fname, char* prefix, + size_t prefixLength) const override; + + virtual Status CreateCipherStream( + const std::string& fname, const EnvOptions& options, Slice& prefix, + std::unique_ptr* result) override; + + virtual ~IppcpProvider(); + + private: + int ctx_size_; + KeySize key_size_; + IppsAESSpec* aes_ctx_; + IppcpProvider() + : ctx_size_(0), key_size_(KeySize::AES_256), aes_ctx_(nullptr) {} + IppcpProvider(const IppcpProvider&) = delete; + IppcpProvider& operator=(const IppcpProvider&) = delete; +}; + +} // namespace ROCKSDB_NAMESPACE + +#endif // !defined(ROCKSDB_LITE) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/cpu_features.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/cpu_features.h new file mode 100644 index 0000000..577f81d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/cpu_features.h @@ -0,0 +1,113 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef CPU_FEATURES_H +#define CPU_FEATURES_H + +#include + +/* definition of cpu features */ +#define mbcpCPUID_MMX 0x00000001LL /* Intel® architecture with MMX(TM) technology supported */ +#define mbcpCPUID_SSE 0x00000002LL /* Intel® Streaming SIMD Extensions (Intel® SSE) instruction set */ +#define mbcpCPUID_SSE2 0x00000004LL /* Intel® Streaming SIMD Extensions 2 (Intel® SSE2) instruction set */ +#define mbcpCPUID_SSE3 0x00000008LL /* Intel® Streaming SIMD Extensions 3 (Intel® SSE3) instruction set */ +#define mbcpCPUID_SSSE3 0x00000010LL /* Supplemental Streaming SIMD Extensions 3 (SSSE3) instruction set */ +#define mbcpCPUID_MOVBE 0x00000020LL /* Intel® instruction MOVBE */ +#define mbcpCPUID_SSE41 0x00000040LL /* Intel® Streaming SIMD Extensions 4.1 (Intel® SSE4.1) instruction set */ +#define mbcpCPUID_SSE42 0x00000080LL /* Intel® Streaming SIMD Extensions 4.2 (Intel® SSE4.2) instruction set */ +#define mbcpCPUID_AVX 0x00000100LL /* Intel® Advanced Vector Extensions (Intel® AVX) instruction set */ +#define mbcpAVX_ENABLEDBYOS 0x00000200LL /* Intel® Advanced Vector Extensions (Intel® AVX) instruction set is supported by OS */ +#define mbcpCPUID_AES 0x00000400LL /* AES */ +#define mbcpCPUID_CLMUL 0x00000800LL /* Intel® instruction PCLMULQDQ */ +#define mbcpCPUID_ABR 0x00001000LL /* Reserved */ +#define mbcpCPUID_RDRAND 0x00002000LL /* Intel® instruction RDRAND */ +#define mbcpCPUID_F16C 0x00004000LL /* Intel® instruction F16C */ +#define mbcpCPUID_AVX2 0x00008000LL /* Intel® Advanced Vector Extensions 2 (Intel® AVX2) */ +#define mbcpCPUID_ADX 0x00010000LL /* Intel® instructions ADOX/ADCX */ +#define mbcpCPUID_RDSEED 0x00020000LL /* Intel® instruction RDSEED */ +#define mbcpCPUID_PREFETCHW 0x00040000LL /* Intel® instruction PREFETCHW */ +#define mbcpCPUID_SHA 0x00080000LL /* Intel® Secure Hash Algorithm Extensions (Intel® SHA Extensions) */ +#define mbcpCPUID_AVX512F 0x00100000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) Foundation instruction set */ +#define mbcpCPUID_AVX512CD 0x00200000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) CD instruction set */ +#define mbcpCPUID_AVX512ER 0x00400000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) ER instruction set */ +#define mbcpCPUID_AVX512PF 0x00800000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) PF instruction set */ +#define mbcpCPUID_AVX512BW 0x01000000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) BW instruction set */ +#define mbcpCPUID_AVX512DQ 0x02000000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) DQ instruction set */ +#define mbcpCPUID_AVX512VL 0x04000000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) VL instruction set */ +#define mbcpCPUID_AVX512VBMI 0x08000000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) Bit Manipulation instructions */ +#define mbcpCPUID_MPX 0x10000000LL /* Intel® Memory Protection Extensions (Intel® MPX) */ +#define mbcpCPUID_AVX512_4FMADDPS 0x20000000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) DL floating-point single precision */ +#define mbcpCPUID_AVX512_4VNNIW 0x40000000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) DL enhanced word variable precision */ +#define mbcpCPUID_KNC 0x80000000LL /* Intel® Xeon® Phi(TM) Coprocessor */ +#define mbcpCPUID_AVX512IFMA 0x100000000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) IFMA (PMADD52) instruction set */ +#define mbcpAVX512_ENABLEDBYOS 0x200000000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) is supported by OS */ +#define mbcpCPUID_AVX512GFNI 0x400000000LL /* GFNI */ +#define mbcpCPUID_AVX512VAES 0x800000000LL /* VAES */ +#define mbcpCPUID_AVX512VCLMUL 0x1000000000LL /* VCLMUL */ +#define mbcpCPUID_AVX512VBMI2 0x2000000000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) Bit Manipulation instructions 2 */ +#define mbcpCPUID_BMI1 0x4000000000LL /* BMI1 */ +#define mbcpCPUID_BMI2 0x8000000000LL /* BMI2 */ + +/* map cpu features */ +EXTERN_C int64u mbx_get_cpu_features(void); + +/* check if crypto_mb is applicable */ +EXTERN_C int mbx_is_crypto_mb_applicable(int64u cpu_features); + +/* supported algorithm */ +enum MBX_ALGO { + MBX_ALGO_RSA_1K, + MBX_ALGO_RSA_2K, + MBX_ALGO_RSA_3K, + MBX_ALGO_RSA_4K, + MBX_ALGO_X25519, + MBX_ALGO_EC_NIST_P256, + MBX_ALGO_ECDHE_NIST_P256 = MBX_ALGO_EC_NIST_P256, + MBX_ALGO_ECDSA_NIST_P256 = MBX_ALGO_EC_NIST_P256, + MBX_ALGO_EC_NIST_P384, + MBX_ALGO_ECDHE_NIST_P384 = MBX_ALGO_EC_NIST_P384, + MBX_ALGO_ECDSA_NIST_P384 = MBX_ALGO_EC_NIST_P384, + MBX_ALGO_EC_NIST_P521, + MBX_ALGO_ECDHE_NIST_P521 = MBX_ALGO_EC_NIST_P521, + MBX_ALGO_ECDSA_NIST_P521 = MBX_ALGO_EC_NIST_P521, + MBX_ALGO_EC_SM2, + MBX_ALGO_ECDHE_SM2 = MBX_ALGO_EC_SM2, + MBX_ALGO_ECDSA_SM2 = MBX_ALGO_EC_SM2, + MBX_ALGO_SM3, + MBX_ALGO_SM4, + MBX_ALGO_ECB_SM4 = MBX_ALGO_SM4, + MBX_ALGO_CBC_SM4 = MBX_ALGO_SM4, + MBX_ALGO_CTR_SM4 = MBX_ALGO_SM4, + MBX_ALGO_OFB_SM4 = MBX_ALGO_SM4, + MBX_ALGO_OFB128_SM4 = MBX_ALGO_SM4, +}; + +/* multi-buffer width implemented by library */ +enum MBX_WIDTH { + MBX_WIDTH_MB8 = 8, + MBX_WIDTH_MB16 = 16, + MBX_WIDTH_ANY = (1 << 16) - 1 +}; + +typedef int64u MBX_ALGO_INFO; + +/* check if algorithm is supported on current platform + * returns: multi-buffer width mask or 0 if algorithm not supported +*/ +EXTERN_C MBX_ALGO_INFO mbx_get_algo_info(enum MBX_ALGO algo); + +#endif /* CPU_FEATURES_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/defs.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/defs.h new file mode 100644 index 0000000..aa5a234 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/defs.h @@ -0,0 +1,68 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef DEFS_H +#define DEFS_H + +/* data types */ +typedef unsigned char int8u; +typedef unsigned short int16u; +typedef unsigned int int32u; +typedef unsigned long long int64u; + +#ifndef NULL + #define NULL ((void *)0) +#endif + +/* alignment & inline */ +#if defined(__GNUC__) + #if !defined(__ALIGN64) + #define __ALIGN64 __attribute__((aligned(64))) + #endif + + #if !defined(__INLINE) + #define __INLINE static __inline__ + #endif + + #if !defined(__NOINLINE) + #define __NOINLINE __attribute__((noinline)) + #endif +#else + #if !defined(__ALIGN64) + #define __ALIGN64 __declspec(align(64)) + #endif + + #if !defined(__INLINE) + #define __INLINE static __forceinline + #endif + + #if !defined(__NOINLINE) + #define __NOINLINE __declspec(noinline) + #endif +#endif + + +/* externals */ +#undef EXTERN_C + +#ifdef __cplusplus + #define EXTERN_C extern "C" +#else + #define EXTERN_C +#endif + +#endif /* DEFS_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/ec_nistp256.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/ec_nistp256.h new file mode 100644 index 0000000..2ffa7c5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/ec_nistp256.h @@ -0,0 +1,187 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef EC_NISTP256_H +#define EC_NISTP256_H + +#include +#include + +#ifndef BN_OPENSSL_DISABLE + #include + #include +#endif // BN_OPENSSL_DISABLE + + +/* +// ECDHE +*/ + +/* +// Computes public key +// pa_pubx[] array of pointers to the public keys X-coordinates +// pa_puby[] array of pointers to the public keys Y-coordinates +// pa_pubz[] array of pointers to the public keys Z-coordinates +// pa_skey[] array of pointers to the private keys +// pBuffer pointer to the scratch buffer +// +// Note: +// output public key is represented by (X:Y:Z) projective Jacobian coordinates +*/ +#ifndef BN_OPENSSL_DISABLE +EXTERN_C mbx_status mbx_nistp256_ecpublic_key_ssl_mb8(BIGNUM* pa_pubx[8], + BIGNUM* pa_puby[8], + BIGNUM* pa_pubz[8], + const BIGNUM* const pa_skey[8], + int8u* pBuffer); +#endif // BN_OPENSSL_DISABLE + + +EXTERN_C mbx_status mbx_nistp256_ecpublic_key_mb8(int64u* pa_pubx[8], + int64u* pa_puby[8], + int64u* pa_pubz[8], + const int64u* const pa_skey[8], + int8u* pBuffer); +/* +// Computes shared key +// pa_shared_key[] array of pointers to the shared keys +// pa_skey[] array of pointers to the own (ephemeral) private keys +// pa_pubx[] array of pointers to the party's public keys X-coordinates +// pa_puby[] array of pointers to the party's public keys Y-coordinates +// pa_pubz[] array of pointers to the party's public keys Z-coordinates +// pBuffer pointer to the scratch buffer +// +// Note: +// input party's public key is represented by (X:Y:Z) projective Jacobian coordinates +*/ +#ifndef BN_OPENSSL_DISABLE +EXTERN_C mbx_status mbx_nistp256_ecdh_ssl_mb8(int8u* pa_shared_key[8], + const BIGNUM* const pa_skey[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer); +#endif // BN_OPENSSL_DISABLE + + +EXTERN_C mbx_status mbx_nistp256_ecdh_mb8(int8u* pa_shared_key[8], + const int64u* const pa_skey[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer); + + +/* +// ECDSA signature generation +*/ + +/* +// Pre-computes ECDSA signature +// pa_inv_eph_skey[] array of pointers to the inversion of signer's ephemeral private keys +// pa_sign_rp[] array of pointers to the pre-computed r-components of the signatures +// pa_eph_skey[] array of pointers to the signer's ephemeral private keys +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp256_ecdsa_sign_setup_mb8(int64u* pa_inv_eph_skey[8], + int64u* pa_sign_rp[8], + const int64u* const pa_eph_skey[8], + int8u* pBuffer); +/* +// computes ECDSA signature +// +// pa_sign_pr[] array of pointers to the r-components of the signatures +// pa_sign_ps[] array of pointers to the s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// pa_sign_rp[] array of pointers to the pre-computed r-components of the signatures +// pa_inv_eph_skey[] array of pointers to the inversion of signer's ephemeral private keys +// pa_reg_skey[] array of pointers to the regular signer's ephemeral private keys +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp256_ecdsa_sign_complete_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_sgn_rp[8], + const int64u* const pa_inv_eph_skey[8], + const int64u* const pa_reg_skey[8], + int8u* pBuffer); +/* +// Computes ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// pa_eph_skey[] array of pointers to the signer's ephemeral private keys +// pa_reg_skey[] array of pointers to the signer's regular private keys +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp256_ecdsa_sign_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_eph_skey[8], + const int64u* const pa_reg_skey[8], + int8u* pBuffer); + +/* +// Verifies ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_msg[] array of pointers to the messages that have been signed +// pa_pubx[] array of pointers to the signer's public keys X-coordinates +// pa_puby[] array of pointers to the signer's public keys Y-coordinates +// pa_pubz[] array of pointers to the signer's public keys Z-coordinates (or NULL, if affine coordinate requested) +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp256_ecdsa_verify_mb8(const int8u* const pa_sign_r[8], + const int8u* const pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer); +/* +// OpenSSL's specific similar APIs +*/ +#ifndef BN_OPENSSL_DISABLE +EXTERN_C mbx_status mbx_nistp256_ecdsa_sign_setup_ssl_mb8(BIGNUM* pa_inv_eph_skey[8], + BIGNUM* pa_sign_pr[8], + const BIGNUM* const pa_eph_skey[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_nistp256_ecdsa_sign_complete_ssl_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const BIGNUM* const pa_sgn_rp[8], + const BIGNUM* const pa_inv_eph_skey[8], + const BIGNUM* const pa_reg_skey[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_nistp256_ecdsa_sign_ssl_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const BIGNUM* const pa_eph_skey[8], + const BIGNUM* const pa_reg_skey[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_nistp256_ecdsa_verify_ssl_mb8(const ECDSA_SIG* const pa_sig[8], + const int8u* const pa_msg[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer); + +#endif // BN_OPENSSL_DISABLE +#endif /* EC_NISTP256_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/ec_nistp384.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/ec_nistp384.h new file mode 100644 index 0000000..4c3fa98 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/ec_nistp384.h @@ -0,0 +1,186 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef EC_NISTP384_H +#define EC_NISTP384_H + +#include +#include + +#ifndef BN_OPENSSL_DISABLE + #include + #include +#endif // BN_OPENSSL_DISABLE + + +/* +// ECDHE +*/ + +/* +// Computes public key +// pa_pubx[] array of pointers to the public keys X-coordinates +// pa_puby[] array of pointers to the public keys Y-coordinates +// pa_pubz[] array of pointers to the public keys Z-coordinates +// pa_skey[] array of pointers to the private keys +// pBuffer pointer to the scratch buffer +// +// Note: +// output public key is represented by (X:Y:Z) projective Jacobian coordinates +*/ +#ifndef BN_OPENSSL_DISABLE +EXTERN_C mbx_status mbx_nistp384_ecpublic_key_ssl_mb8(BIGNUM* pa_pubx[8], + BIGNUM* pa_puby[8], + BIGNUM* pa_pubz[8], + const BIGNUM* const pa_skey[8], + int8u* pBuffer); +#endif // BN_OPENSSL_DISABLE + + +EXTERN_C mbx_status mbx_nistp384_ecpublic_key_mb8(int64u* pa_pubx[8], + int64u* pa_puby[8], + int64u* pa_pubz[8], + const int64u* const pa_skey[8], + int8u* pBuffer); +/* +// Computes shared key +// pa_shared_key[] array of pointers to the shared keys +// pa_skey[] array of pointers to the own (ephemeral) private keys +// pa_pubx[] array of pointers to the party's public keys X-coordinates +// pa_puby[] array of pointers to the party's public keys Y-coordinates +// pa_pubz[] array of pointers to the party's public keys Z-coordinates +// pBuffer pointer to the scratch buffer +// +// Note: +// input party's public key is represented by (X:Y:Z) projective Jacobian coordinates +*/ +#ifndef BN_OPENSSL_DISABLE +EXTERN_C mbx_status mbx_nistp384_ecdh_ssl_mb8(int8u* pa_shared_key[8], + const BIGNUM* const pa_skey[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer); +#endif // BN_OPENSSL_DISABLE + + +EXTERN_C mbx_status mbx_nistp384_ecdh_mb8(int8u* pa_shared_key[8], + const int64u* const pa_skey[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer); + + +/* +// ECDSA signature generation +*/ + +/* +// Pre-computes ECDSA signature +// pa_inv_eph_skey[] array of pointers to the inversion of signer's ephemeral private keys +// pa_sign_rp[] array of pointers to the pre-computed r-components of the signatures +// pa_eph_skey[] array of pointers to the signer's ephemeral private keys +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp384_ecdsa_sign_setup_mb8(int64u* pa_inv_eph_skey[8], + int64u* pa_sign_rp[8], + const int64u* const pa_eph_skey[8], + int8u* pBuffer); +/* +// computes ECDSA signature +// +// pa_sign_pr[] array of pointers to the r-components of the signatures +// pa_sign_ps[] array of pointers to the s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// pa_sign_rp[] array of pointers to the pre-computed r-components of the signatures +// pa_inv_eph_skey[] array of pointers to the inversion of signer's ephemeral private keys +// pa_reg_skey[] array of pointers to the regular signer's ephemeral private keys +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp384_ecdsa_sign_complete_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_sgn_rp[8], + const int64u* const pa_inv_eph_skey[8], + const int64u* const pa_reg_skey[8], + int8u* pBuffer); +/* +// Computes ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// pa_eph_skey[] array of pointers to the signer's ephemeral private keys +// pa_reg_skey[] array of pointers to the signer's regular private keys +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp384_ecdsa_sign_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_eph_skey[8], + const int64u* const pa_reg_skey[8], + int8u* pBuffer); + +/* +// Verifies ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_msg[] array of pointers to the messages that have been signed +// pa_pubx[] array of pointers to the signer's public keys X-coordinates +// pa_puby[] array of pointers to the signer's public keys Y-coordinates +// pa_pubz[] array of pointers to the signer's public keys Z-coordinates (or NULL, if affine coordinate requested) +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp384_ecdsa_verify_mb8(const int8u* const pa_sign_r[8], + const int8u* const pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer); +/* +// OpenSSL's specific similar APIs +*/ +#ifndef BN_OPENSSL_DISABLE +EXTERN_C mbx_status mbx_nistp384_ecdsa_sign_setup_ssl_mb8(BIGNUM* pa_inv_eph_skey[8], + BIGNUM* pa_sign_pr[8], + const BIGNUM* const pa_eph_skey[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_nistp384_ecdsa_sign_complete_ssl_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const BIGNUM* const pa_sgn_rp[8], + const BIGNUM* const pa_inv_eph_skey[8], + const BIGNUM* const pa_reg_skey[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_nistp384_ecdsa_sign_ssl_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const BIGNUM* const pa_eph_skey[8], + const BIGNUM* const pa_reg_skey[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_nistp384_ecdsa_verify_ssl_mb8(const ECDSA_SIG* const pa_sign[8], + const int8u* const pa_msg[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer); +#endif // BN_OPENSSL_DISABLE +#endif /* EC_NISTP384_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/ec_nistp521.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/ec_nistp521.h new file mode 100644 index 0000000..1e938c3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/ec_nistp521.h @@ -0,0 +1,187 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef EC_NISTP521_H +#define EC_NISTP521_H + +#include +#include + +#ifndef BN_OPENSSL_DISABLE + #include + #include +#endif // BN_OPENSSL_DISABLE + + +/* +// ECDHE +*/ + +/* +// Computes public key +// pa_pubx[] array of pointers to the public keys X-coordinates +// pa_puby[] array of pointers to the public keys Y-coordinates +// pa_pubz[] array of pointers to the public keys Z-coordinates +// pa_skey[] array of pointers to the private keys +// pBuffer pointer to the scratch buffer +// +// Note: +// output public key is represented by (X:Y:Z) projective Jacobian coordinates +*/ +#ifndef BN_OPENSSL_DISABLE +EXTERN_C mbx_status mbx_nistp521_ecpublic_key_ssl_mb8(BIGNUM* pa_pubx[8], + BIGNUM* pa_puby[8], + BIGNUM* pa_pubz[8], + const BIGNUM* const pa_skey[8], + int8u* pBuffer); +#endif // BN_OPENSSL_DISABLE + + +EXTERN_C mbx_status mbx_nistp521_ecpublic_key_mb8(int64u* pa_pubx[8], + int64u* pa_puby[8], + int64u* pa_pubz[8], + const int64u* const pa_skey[8], + int8u* pBuffer); +/* +// Computes shared key +// pa_shared_key[] array of pointers to the shared keys +// pa_skey[] array of pointers to the own (ephemeral) private keys +// pa_pubx[] array of pointers to the party's public keys X-coordinates +// pa_puby[] array of pointers to the party's public keys Y-coordinates +// pa_pubz[] array of pointers to the party's public keys Z-coordinates +// pBuffer pointer to the scratch buffer +// +// Note: +// input party's public key is represented by (X:Y:Z) projective Jacobian coordinates +*/ +#ifndef BN_OPENSSL_DISABLE +EXTERN_C mbx_status mbx_nistp521_ecdh_ssl_mb8(int8u* pa_shared_key[8], + const BIGNUM* const pa_skey[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer); +#endif // BN_OPENSSL_DISABLE + + +EXTERN_C mbx_status mbx_nistp521_ecdh_mb8(int8u* pa_shared_key[8], + const int64u* const pa_skey[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer); + + +/* +// ECDSA signature generation +*/ + +/* +// Pre-computes ECDSA signature +// pa_inv_eph_skey[] array of pointers to the inversion of signer's ephemeral private keys +// pa_sign_rp[] array of pointers to the pre-computed r-components of the signatures +// pa_eph_skey[] array of pointers to the signer's ephemeral private keys +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp521_ecdsa_sign_setup_mb8(int64u* pa_inv_eph_skey[8], + int64u* pa_sign_rp[8], + const int64u* const pa_eph_skey[8], + int8u* pBuffer); +/* +// computes ECDSA signature +// +// pa_sign_pr[] array of pointers to the r-components of the signatures +// pa_sign_ps[] array of pointers to the s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// pa_sign_rp[] array of pointers to the pre-computed r-components of the signatures +// pa_inv_eph_skey[] array of pointers to the inversion of signer's ephemeral private keys +// pa_reg_skey[] array of pointers to the regular signer's ephemeral private keys +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp521_ecdsa_sign_complete_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_sgn_rp[8], + const int64u* const pa_inv_eph_skey[8], + const int64u* const pa_reg_skey[8], + int8u* pBuffer); +/* +// Computes ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// pa_eph_skey[] array of pointers to the signer's ephemeral private keys +// pa_reg_skey[] array of pointers to the signer's regular private keys +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp521_ecdsa_sign_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_eph_skey[8], + const int64u* const pa_reg_skey[8], + int8u* pBuffer); + +/* +// Verifies ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// pa_pubx[] array of pointers to the public keys X-coordinates +// pa_puby[] array of pointers to the public keys Y-coordinates +// pa_pubz[] array of pointers to the public keys Z-coordinates +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp521_ecdsa_verify_mb8(const int8u* const pa_sign_r[8], + const int8u* const pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer); +/* +// OpenSSL's specific similar APIs +*/ +#ifndef BN_OPENSSL_DISABLE +EXTERN_C mbx_status mbx_nistp521_ecdsa_sign_setup_ssl_mb8(BIGNUM* pa_inv_eph_skey[8], + BIGNUM* pa_sign_pr[8], + const BIGNUM* const pa_eph_skey[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_nistp521_ecdsa_sign_complete_ssl_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const BIGNUM* const pa_sgn_rp[8], + const BIGNUM* const pa_inv_eph_skey[8], + const BIGNUM* const pa_reg_skey[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_nistp521_ecdsa_sign_ssl_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const BIGNUM* const pa_eph_skey[8], + const BIGNUM* const pa_reg_skey[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_nistp521_ecdsa_verify_ssl_mb8(const ECDSA_SIG* const pa_sig[8], + const int8u* const pa_msg[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer); + +#endif // BN_OPENSSL_DISABLE +#endif /* EC_NISTP521_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/ec_sm2.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/ec_sm2.h new file mode 100644 index 0000000..43d6f97 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/ec_sm2.h @@ -0,0 +1,154 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef EC_SM2_H +#define EC_SM2_H + +#include +#include + +#ifndef BN_OPENSSL_DISABLE + #include + #include +#endif // BN_OPENSSL_DISABLE + +/* +// Computes public key +// pa_pubx[] array of pointers to the public keys X-coordinates +// pa_puby[] array of pointers to the public keys Y-coordinates +// pa_pubz[] array of pointers to the public keys Z-coordinates (or NULL, if affine coordinate requested) +// pa_skey[] array of pointers to the private keys +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_sm2_ecpublic_key_mb8(int64u* pa_pubx[8], + int64u* pa_puby[8], + int64u* pa_pubz[8], + const int64u* const pa_skey[8], + int8u* pBuffer); + +/* +// Computes shared key +// pa_shared_key[] array of pointers to the shared keys +// pa_skey[] array of pointers to the own (ephemeral) private keys +// pa_pubx[] array of pointers to the party's public keys X-coordinates +// pa_puby[] array of pointers to the party's public keys Y-coordinates +// pa_pubz[] array of pointers to the party's public keys Z-coordinates (or NULL, if affine coordinate requested) +// pBuffer pointer to the scratch buffer +// +// Note: +// This function implements ECDHE over SM2 curve according to IEEE 1363-2000 standard. +*/ +EXTERN_C mbx_status mbx_sm2_ecdh_mb8(int8u* pa_shared_key[8], + const int64u* const pa_skey[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer); + +/* +// Computes SM2 ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_user_id[] array of pointers to the users ID +// user_id_len[] array of users ID length +// pa_msg[] array of pointers to the messages are being signed +// msg_len[] array of messages length +// pa_eph_skey[] array of pointers to the signer's ephemeral private keys +// pa_reg_skey[] array of pointers to the signer's regular private keys +// pa_pubx[] array of pointers to the party's public keys X-coordinates +// pa_puby[] array of pointers to the party's public keys Y-coordinates +// pa_pubz[] array of pointers to the party's public keys Z-coordinates (or NULL, if affine coordinate requested) +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_sm2_ecdsa_sign_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_user_id[8], + const int user_id_len[8], + const int8u* const pa_msg[8], + const int msg_len[8], + const int64u* const pa_eph_skey[8], + const int64u* const pa_reg_skey[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer); + +/* +// Verifies SM2 ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_user_id[] array of pointers to the users ID +// user_id_len[] array of users ID length +// pa_msg[] array of pointers to the messages are being signed +// msg_len[] array of messages length +// pa_pubx[] array of pointers to the signer's public keys X-coordinates +// pa_puby[] array of pointers to the signer's public keys Y-coordinates +// pa_pubz[] array of pointers to the signer's public keys Z-coordinates (or NULL, if affine coordinate requested) +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_sm2_ecdsa_verify_mb8(const int8u* const pa_sign_r[8], + const int8u* const pa_sign_s[8], + const int8u* const pa_user_id[8], + const int user_id_len[8], + const int8u* const pa_msg[8], + const int msg_len[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer); + +/* +// OpenSSL's specific similar APIs +*/ +#ifndef BN_OPENSSL_DISABLE +EXTERN_C mbx_status mbx_sm2_ecpublic_key_ssl_mb8(BIGNUM* pa_pubx[8], + BIGNUM* pa_puby[8], + BIGNUM* pa_pubz[8], + const BIGNUM* const pa_skey[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_sm2_ecdh_ssl_mb8(int8u* pa_shared_key[8], + const BIGNUM* const pa_skey[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_sm2_ecdsa_sign_ssl_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_user_id[8], + const int user_id_len[8], + const int8u* const pa_msg[8], + const int msg_len[8], + const BIGNUM* const pa_eph_skey[8], + const BIGNUM* const pa_reg_skey[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_sm2_ecdsa_verify_ssl_mb8(const ECDSA_SIG* const pa_sig[8], + const int8u* const pa_user_id[8], + const int user_id_len[8], + const int8u* const pa_msg[8], + const int msg_len[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer); +#endif // BN_OPENSSL_DISABLE +#endif /* EC_SM2_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/ed25519.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/ed25519.h new file mode 100644 index 0000000..8cc737d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/ed25519.h @@ -0,0 +1,66 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef ED25519_H +#define ED25519_H + +#include +#include + +typedef int8u ed25519_sign_component[32]; +typedef ed25519_sign_component ed25519_sign[2]; + +typedef int8u ed25519_public_key[32]; +typedef int8u ed25519_private_key[32]; + +/* +// Computes ed25519 public key +// pa_public_key[] array of pointers to the public keys +// pa_private_key[] array of pointers to the public keys Y-coordinates +*/ +EXTERN_C mbx_status mbx_ed25519_public_key_mb8(ed25519_public_key* pa_public_key[8], + const ed25519_private_key* const pa_private_key[8]); + +/* +// Computes ed25519 signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// msgLen[] lengths of the messages are being signed +// pa_private_key[] array of pointers to the signer's private keys +// pa_public_key[] array of pointers to the signer's public keys +*/ +EXTERN_C mbx_status mbx_ed25519_sign_mb8(ed25519_sign_component* pa_sign_r[8], + ed25519_sign_component* pa_sign_s[8], + const int8u* const pa_msg[8], const int32u msgLen[8], + const ed25519_private_key* const pa_private_key[8], + const ed25519_public_key* const pa_public_key[8]); + +/* +// Verifies ed25519 signature +// pa_sign_r[] array of pointers to the r-components of the verified signatures +// pa_sign_s[] array of pointers to the s-components of the verified signatures +// pa_msg[] array of pointers to the signed messages +// msgLen[] array of signed messages lengths +// pa_public_key[] array of pointers to the signer's public keys +*/ +EXTERN_C mbx_status mbx_ed25519_verify_mb8(const ed25519_sign_component* const pa_sign_r[8], + const ed25519_sign_component* const pa_sign_s[8], + const int8u* const pa_msg[8], const int32u msgLen[8], + const ed25519_public_key* const pa_public_key[8]); + +#endif /* ED25519_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/exp.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/exp.h new file mode 100644 index 0000000..b5a76b8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/exp.h @@ -0,0 +1,59 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef EXP_H +#define EXP_H + +#include +#include + + +/* size of scratch buffer */ +EXTERN_C int mbx_exp_BufferSize(int modulusBits); + +/* exp operation */ +EXTERN_C mbx_status mbx_exp1024_mb8(int64u* const out_pa[8], + const int64u* const base_pa[8], + const int64u* const exp_pa[8], int exp_bits, + const int64u* const mod_pa[8], int mod_bits, + int8u* pBuffer, int bufferLen); + +EXTERN_C mbx_status mbx_exp2048_mb8(int64u* const out_pa[8], + const int64u* const base_pa[8], + const int64u* const exp_pa[8], int exp_bits, + const int64u* const mod_pa[8], int mod_bits, + int8u* pBuffer, int bufferLen); + +EXTERN_C mbx_status mbx_exp3072_mb8(int64u* const out_pa[8], + const int64u* const base_pa[8], + const int64u* const exp_pa[8], int exp_bits, + const int64u* const mod_pa[8], int mod_bits, + int8u* pBuffer, int bufferLen); + +EXTERN_C mbx_status mbx_exp4096_mb8(int64u* const out_pa[8], + const int64u* const base_pa[8], + const int64u* const exp_pa[8], int exp_bits, + const int64u* const mod_pa[8], int mod_bits, + int8u* pBuffer, int bufferLen); + +EXTERN_C mbx_status mbx_exp_mb8(int64u* const out_pa[8], + const int64u* const base_pa[8], + const int64u* const exp_pa[8], int exp_bits, + const int64u* const mod_pa[8], int mod_bits, + int8u* pBuffer, int bufferLen); + +#endif /* EXP_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/rsa.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/rsa.h new file mode 100644 index 0000000..f71f2d5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/rsa.h @@ -0,0 +1,104 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef RSA_H +#define RSA_H + +#include +#include + +#ifndef BN_OPENSSL_DISABLE +#include + +EXTERN_C mbx_status mbx_rsa_public_ssl_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const e_pa[8], + const BIGNUM* const n_pa[8], + int expected_rsa_bitsize); + +EXTERN_C mbx_status mbx_rsa_private_ssl_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const d_pa[8], + const BIGNUM* const n_pa[8], + int expected_rsa_bitsize); + +EXTERN_C mbx_status mbx_rsa_private_crt_ssl_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const p_pa[8], + const BIGNUM* const q_pa[8], + const BIGNUM* const dp_pa[8], + const BIGNUM* const dq_pa[8], + const BIGNUM* const iq_pa[8], + int expected_rsa_bitsize); +#endif /* BN_OPENSSL_DISABLE */ + + +/* +// rsa cp methods +*/ +typedef struct _ifma_rsa_method mbx_RSA_Method; + +/* rsa public key opertaion */ +EXTERN_C const mbx_RSA_Method* mbx_RSA1K_pub65537_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA2K_pub65537_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA3K_pub65537_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA4K_pub65537_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA_pub65537_Method(int rsaBitsize); + +/* rsa private key opertaion */ +EXTERN_C const mbx_RSA_Method* mbx_RSA1K_private_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA2K_private_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA3K_private_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA4K_private_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA_private_Method(int rsaBitsize); + +/* rsa private key opertaion (ctr) */ +EXTERN_C const mbx_RSA_Method* mbx_RSA1K_private_crt_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA2K_private_crt_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA3K_private_crt_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA4K_private_crt_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA_private_crt_Method(int rsaBitsize); + +EXTERN_C int mbx_RSA_Method_BufSize(const mbx_RSA_Method* m); + + +EXTERN_C mbx_status mbx_rsa_public_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const int64u* const n_pa[8], + int rsaBitlen, + const mbx_RSA_Method* m, + int8u* pBuffer); + +EXTERN_C mbx_status mbx_rsa_private_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const int64u* const d_pa[8], + const int64u* const n_pa[8], + int rsaBitlen, + const mbx_RSA_Method* m, + int8u* pBuffer); + +EXTERN_C mbx_status mbx_rsa_private_crt_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const int64u* const p_pa[8], + const int64u* const q_pa[8], + const int64u* const dp_pa[8], + const int64u* const dq_pa[8], + const int64u* const iq_pa[8], + int rsaBitlen, + const mbx_RSA_Method* m, + int8u* pBuffer); +#endif /* RSA_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/sm3.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/sm3.h new file mode 100644 index 0000000..8f7f7d8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/sm3.h @@ -0,0 +1,60 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + + +#ifndef SM3_H +#define SM3_H + +#include +#include + +#define SM3_SIZE_IN_BITS (256) /* sm3 size in bits */ +#define SM3_SIZE_IN_WORDS (SM3_SIZE_IN_BITS/(sizeof(int32u)*8)) /* sm3 hash size in words */ +#define SM3_MSG_BLOCK_SIZE (64) /* messge block size */ + +#define SM3_NUM_BUFFERS (16) /* max number of buffers in sm3 multi-buffer */ + +/* +// sm3 context for mb16 +*/ + +typedef int32u sm3_hash_mb[SM3_SIZE_IN_WORDS][SM3_NUM_BUFFERS]; /* sm3 hash value in multi-buffer format */ + +struct _sm3_context_mb16 { + int msg_buff_idx[SM3_NUM_BUFFERS]; /* buffer entry */ + int64u msg_len[SM3_NUM_BUFFERS]; /* message length */ + int8u msg_buffer[SM3_NUM_BUFFERS][SM3_MSG_BLOCK_SIZE]; /* buffer */ + __ALIGN64 + sm3_hash_mb msg_hash; /* intermediate hash */ +}; + +typedef struct _sm3_context_mb16 SM3_CTX_mb16; + +EXTERN_C mbx_status16 mbx_sm3_init_mb16(SM3_CTX_mb16* p_state); + +EXTERN_C mbx_status16 mbx_sm3_update_mb16(const int8u* const msg_pa[16], + int len[16], + SM3_CTX_mb16* p_state); + +EXTERN_C mbx_status16 mbx_sm3_final_mb16(int8u* hash_pa[16], + SM3_CTX_mb16* p_state); + +EXTERN_C mbx_status16 mbx_sm3_msg_digest_mb16(const int8u* const msg_pa[16], + int len[16], + int8u* hash_pa[16]); + +#endif /* SM3_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/sm4.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/sm4.h new file mode 100644 index 0000000..144fb6f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/sm4.h @@ -0,0 +1,59 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + + +#ifndef SM4_H +#define SM4_H + +#include +#include + +#define SM4_LINES (16) /* Max number of buffers */ +#define SM4_BLOCK_SIZE (16) /* SM4 data block size (bytes) */ +#define SM4_KEY_SIZE (16) /* SM4 key size (bytes) */ +#define SM4_ROUNDS (32) /* SM4 number of rounds */ +#define SM4_XTS_MAX_SIZE ((1 << 20) * SM4_BLOCK_SIZE) /* SM4 max buffer size (bytes) */ + +typedef int8u sm4_key[SM4_KEY_SIZE]; +typedef int8u sm4_xts_key[SM4_KEY_SIZE*2]; +typedef int32u mbx_sm4_key_schedule[SM4_ROUNDS][SM4_LINES]; + +EXTERN_C mbx_status16 mbx_sm4_set_key_mb16(mbx_sm4_key_schedule* key_sched, const sm4_key* pa_key[SM4_LINES]); +EXTERN_C mbx_status16 mbx_sm4_xts_set_keys_mb16(mbx_sm4_key_schedule* key_sched1, mbx_sm4_key_schedule* key_sched2, const sm4_xts_key* pa_key[SM4_LINES]); + +EXTERN_C mbx_status16 mbx_sm4_encrypt_ecb_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched); +EXTERN_C mbx_status16 mbx_sm4_decrypt_ecb_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched); + +EXTERN_C mbx_status16 mbx_sm4_encrypt_cbc_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, const int8u* pa_iv[SM4_LINES]); +EXTERN_C mbx_status16 mbx_sm4_decrypt_cbc_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, const int8u* pa_iv[SM4_LINES]); + +EXTERN_C mbx_status16 mbx_sm4_encrypt_ctr128_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, int8u* pa_ctr[SM4_LINES]); +EXTERN_C mbx_status16 mbx_sm4_decrypt_ctr128_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, int8u* pa_ctr[SM4_LINES]); + +EXTERN_C mbx_status16 mbx_sm4_encrypt_ofb_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, int8u* pa_iv[SM4_LINES]); +EXTERN_C mbx_status16 mbx_sm4_decrypt_ofb_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, int8u* pa_iv[SM4_LINES]); + +EXTERN_C mbx_status16 mbx_sm4_encrypt_cfb128_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, const int8u* pa_iv[SM4_LINES]); +EXTERN_C mbx_status16 mbx_sm4_decrypt_cfb128_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, const int8u* pa_iv[SM4_LINES]); + +EXTERN_C mbx_status16 mbx_sm4_xts_encrypt_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], + const mbx_sm4_key_schedule* key_sched1, const mbx_sm4_key_schedule* key_sched2, + const int8u* pa_tweak[SM4_LINES]); +EXTERN_C mbx_status16 mbx_sm4_xts_decrypt_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], + const mbx_sm4_key_schedule* key_sched1, const mbx_sm4_key_schedule* key_sched2, + const int8u* pa_tweak[SM4_LINES]); +#endif /* SM4_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/sm4_ccm.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/sm4_ccm.h new file mode 100644 index 0000000..45128b7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/sm4_ccm.h @@ -0,0 +1,143 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#ifndef SM4_CCM_H +#define SM4_CCM_H + +#include + +#include + +#define MIN_CCM_IV_LENGTH 7 +#define MAX_CCM_IV_LENGTH 13 +#define MIN_CCM_TAG_LENGTH 4 +#define MAX_CCM_TAG_LENGTH 16 +#define MAX_CCM_AAD_LENGTH 65280 /* 2^16 - 2^8 */ + +#define SM4_CCM_CONTEXT_BUFFER_SLOT_TYPE int64u + +#define SM4_CCM_CONTEXT_BUFFER_SLOT_SIZE_BYTES (sizeof(SM4_CCM_CONTEXT_BUFFER_SLOT_TYPE)) +#define SM4_CCM_CONTEXT_BUFFER_SIZE_BYTES ((SM4_LINES * SM4_BLOCK_SIZE) / SM4_CCM_CONTEXT_BUFFER_SLOT_SIZE_BYTES) + +/* +// Enum to control call sequence +// +// Valid call sequence: +// +// 1) mbx_sm4_ccm_init_mb16 +// 2) mbx_sm4_ccm_update_aad_mb16 – optional +// 3) mbx_sm4_ccm_encrypt_mb16/mbx_sm4_ccm_decrypt_mb16 – optional, can be called as many times as necessary +// 4) mbx_sm4_ccm_get_tag_mb16 +// +// Call sequence restrictions: +// +// * mbx_sm4_ccm_get_tag_mb16 can be called after mbx_sm4_ccm_init_mb16 has been called. +// * functions at steps 2-3 can be called as many times as needed to process payload while this functions processes buffers +// with full blocks (Blocks of 16 bytes size) or empty buffers and length of processed payload is not overflowed. +// * if functions at steps 2-3 called to process a partial block, it can’t be called again. +// * if mbx_sm4_ccm_encrypt_mb16 or mbx_sm4_ccm_decrypt_mb16 was called, mbx_sm4_ccm_update_aad_mb16 can’t be called. +// * if mbx_sm4_ccm_encrypt_mb16 was called, mbx_sm4_ccm_decrypt_mb16 can’t be called. +// * if mbx_sm4_ccm_decrypt_mb16 was called, mbx_sm4_ccm_encrypt_mb16 can’t be called. +*/ +typedef enum { sm4_ccm_update_aad = 0xF0A1, sm4_ccm_start_encdec, sm4_ccm_enc, sm4_ccm_dec, sm4_ccm_get_tag } sm4_ccm_state; + +struct _sm4_ccm_context_mb16 { + int64u msg_len[SM4_LINES]; /* Message length (in bytes) of all lines */ + int64u total_processed_len[SM4_LINES]; /* Total processed plaintext/ciphertext length (in bytes) of all lines */ + int tag_len[SM4_LINES]; /* Tag length (in bytes) of all lines */ + int iv_len[SM4_LINES]; /* Total IV length (in bytes) of all lines */ + __m128i ctr0[SM4_LINES]; /* CTR0 content */ + __m128i ctr[SM4_LINES]; /* CTR content */ + __m128i hash[SM4_LINES]; /* hash value accumulator for AAD and TXT processing */ + + mbx_sm4_key_schedule key_sched; /* SM4 key schedule */ + sm4_ccm_state state; /* call sequence state */ +}; + +typedef struct _sm4_ccm_context_mb16 SM4_CCM_CTX_mb16; + +/* + * Initializes SM4-CCM context. + * + * @param[in] pa_key Array of key pointers + * @param[in] pa_iv Array of IV pointers + * @param[in] iv_len Array of IV lengths + * @param[in] tag_len Array of authentication tag lengths + * @param[in] msg_len Array of total message lengths + * @param[in/out] p_context SM4-CCM context + * + * @return Bitmask of operation status + */ +EXTERN_C mbx_status16 mbx_sm4_ccm_init_mb16(const sm4_key *const pa_key[SM4_LINES], + const int8u *const pa_iv[SM4_LINES], + const int iv_len[SM4_LINES], + const int tag_len[SM4_LINES], + const int64u msg_len[SM4_LINES], + SM4_CCM_CTX_mb16 *p_context); +/* + * Digests additional authenticated data (AAD) for 16 buffers + * + * @param[in] pa_aad Array of AAD pointers + * @param[in] aad_len Array of AAD lengths + * @param[in/out] p_context SM4-CCM context + * + * @return Bitmask of operation status + */ +EXTERN_C mbx_status16 mbx_sm4_ccm_update_aad_mb16(const int8u *const pa_aad[SM4_LINES], + const int aad_len[SM4_LINES], + SM4_CCM_CTX_mb16 *p_context); +/* + * Retrieves authentication tag for 16 buffers + * + * @param[out] pa_tag Array of authentication tag pointers + * @param[in] tag_len Array of tag lengths + * @param[in/out] p_context SM4-CCM context + * + * @return Bitmask of operation status + */ +EXTERN_C mbx_status16 mbx_sm4_ccm_get_tag_mb16(int8u *pa_tag[SM4_LINES], + const int tag_len[SM4_LINES], + SM4_CCM_CTX_mb16 *p_context); +/* + * Encrypts 16 buffers with SM4-CCM. + * + * @param[out] pa_out Array of ciphertext pointers + * @param[in] pa_in Array of plaintext pointers + * @param[in] in_len Array of plaintext lengths + * @param[in/out] p_context SM4-CCM context + * + * @return Bitmask of operation status + */ +EXTERN_C mbx_status16 mbx_sm4_ccm_encrypt_mb16(int8u *pa_out[SM4_LINES], + const int8u *const pa_in[SM4_LINES], + const int in_len[SM4_LINES], + SM4_CCM_CTX_mb16 *p_context); +/* + * Decrypts 16 buffers with SM4-CCM. + * + * @param[out] pa_out Array of plaintext pointers + * @param[in] pa_in Array of ciphertext pointers + * @param[in] in_len Array of ciphertext lengths + * @param[in/out] p_context SM4-CCM context + * + * @return Bitmask of operation status + */ +EXTERN_C mbx_status16 mbx_sm4_ccm_decrypt_mb16(int8u *pa_out[SM4_LINES], + const int8u *const pa_in[SM4_LINES], + const int in_len[SM4_LINES], + SM4_CCM_CTX_mb16 *p_context); +#endif /* SM4_CCM_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/sm4_gcm.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/sm4_gcm.h new file mode 100644 index 0000000..9dede98 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/sm4_gcm.h @@ -0,0 +1,113 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#ifndef SM4_GCM_H +#define SM4_GCM_H + +#include + +#include + +#define SM4_GCM_CONTEXT_BUFFER_SLOT_TYPE int64u + +#define SM4_GCM_CONTEXT_BUFFER_SLOT_SIZE_BYTES (sizeof(SM4_GCM_CONTEXT_BUFFER_SLOT_TYPE)) +#define SM4_GCM_CONTEXT_BUFFER_SIZE_BYTES ((SM4_LINES * SM4_BLOCK_SIZE) / SM4_GCM_CONTEXT_BUFFER_SLOT_SIZE_BYTES) + +#define SM4_GCM_HASHKEY_PWR_NUM 8 + +/* +// Enum to control call sequence +// +// Valid call sequence: +// +// 1) mbx_sm4_gcm_init_mb16 +// 2) mbx_sm4_gcm_update_iv_mb16 – optional, can be called as many times as necessary +// 3) mbx_sm4_gcm_update_aad_mb16 – optional, can be called as many times as necessary +// 4) mbx_sm4_gcm_encrypt_mb16/mbx_sm4_gcm_decrypt_mb16 – optional, can be called as many times as necessary +// 5) mbx_sm4_gcm_get_tag_mb16 +// +// Call sequence restrictions: +// +// * mbx_sm4_gcm_get_tag_mb16 can be called after IV is fully processed. +// IV is fully processed if buffer with partial block (Block of less than 16 bytes size) was processed or if mbx_sm4_gcm_update_aad_mb16 was called +// * functions at steps 2-4 can be called as many times as needed to process payload while this functions processes buffers +// with full blocks (Blocks of 16 bytes size) or empty buffers and length of processed payload is not overflowed. +// * if functions at steps 2-4 called to process a partial block, it can’t be called again. +// * if mbx_sm4_gcm_update_aad_mb16 was called, mbx_sm4_gcm_update_iv_mb16 can’t be called. +// * if mbx_sm4_gcm_encrypt_mb16 or mbx_sm4_gcm_decrypt_mb16 was called, mbx_sm4_gcm_update_aad_mb16 and mbx_sm4_gcm_update_iv_mb16 can’t be called. +// * if mbx_sm4_gcm_encrypt_mb16 was called, mbx_sm4_gcm_decrypt_mb16 can’t be called. +// * if mbx_sm4_gcm_decrypt_mb16 was called, mbx_sm4_gcm_encrypt_mb16 can’t be called. +*/ +typedef enum { sm4_gcm_update_iv = 0xF0A1, sm4_gcm_update_aad, sm4_gcm_start_encdec, sm4_gcm_enc, sm4_gcm_dec, sm4_gcm_get_tag } sm4_gcm_state; + +struct _sm4_gcm_context_mb16 { + __m128i hashkey[SM4_GCM_HASHKEY_PWR_NUM][SM4_LINES]; /* Set of hashkeys for ghash computation */ + __m128i j0[SM4_LINES]; /* J0 value accumulator for IV processing */ + __m128i ghash[SM4_LINES]; /* ghash value accumulator for AAD and TXT processing */ + __m128i ctr[SM4_LINES]; /* counter for gctr encryption */ + + /* + // buffer to store IV, AAD and TXT length in bytes + // + // this buffer is used to store IV length to compute J0 block + // and reused to store AAD and TXT length to compute ghash + // + // length is stored as follow: + // + // J0 computation: + // [64 bits with IV len (buffer 0)] + // [64 bits with IV len (buffer 1)] + // .. + // [64 bits with IV len (buffer SM4_LINES-1)] + // + // Only half of buffer is used for J0 computation + // + // ghash computation: + // [64 bits with AAD len (buffer 0)][64 bits with TXT len (buffer 0)] + // [64 bits with AAD len (buffer 1)][64 bits with TXT len (buffer 1)] + // .. + // [64 bits with AAD len (buffer SM4_LINES-1)][64 bits with TXT len (buffer SM4_LINES-1)] + // + */ + int64u len[SM4_LINES * 2]; + + mbx_sm4_key_schedule key_sched; /* SM4 key schedule */ + sm4_gcm_state state; /* call sequence state */ +}; + +typedef struct _sm4_gcm_context_mb16 SM4_GCM_CTX_mb16; + +EXTERN_C mbx_status16 mbx_sm4_gcm_init_mb16(const sm4_key *const pa_key[SM4_LINES], + const int8u *const pa_iv[SM4_LINES], + const int iv_len[SM4_LINES], + SM4_GCM_CTX_mb16 *p_context); + +EXTERN_C mbx_status16 mbx_sm4_gcm_update_iv_mb16(const int8u *const pa_iv[SM4_LINES], const int iv_len[SM4_LINES], SM4_GCM_CTX_mb16 *p_state); +EXTERN_C mbx_status16 mbx_sm4_gcm_update_aad_mb16(const int8u *const pa_aad[SM4_LINES], const int aad_len[SM4_LINES], SM4_GCM_CTX_mb16 *p_state); + +EXTERN_C mbx_status16 mbx_sm4_gcm_encrypt_mb16(int8u *pa_out[SM4_LINES], + const int8u *const pa_in[SM4_LINES], + const int in_len[SM4_LINES], + SM4_GCM_CTX_mb16 *p_context); +EXTERN_C mbx_status16 mbx_sm4_gcm_decrypt_mb16(int8u *pa_out[SM4_LINES], + const int8u *const pa_in[SM4_LINES], + const int in_len[SM4_LINES], + SM4_GCM_CTX_mb16 *p_context); + +EXTERN_C mbx_status16 mbx_sm4_gcm_get_tag_mb16(int8u *pa_tag[SM4_LINES], const int tag_len[SM4_LINES], SM4_GCM_CTX_mb16 *p_context); + +#endif /* SM4_GCM_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/status.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/status.h new file mode 100644 index 0000000..add3b5c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/status.h @@ -0,0 +1,117 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef STATUS_H +#define STATUS_H + +#include + +typedef int32u mbx_status; +typedef int64u mbx_status16; + +// error statuses and manipulators +#define MBX_STATUS_OK (0) +#define MBX_STATUS_MISMATCH_PARAM_ERR (1) +#define MBX_STATUS_NULL_PARAM_ERR (2) +#define MBX_STATUS_LOW_ORDER_ERR (4) +#define MBX_STATUS_SIGNATURE_ERR (8) + +__INLINE mbx_status MBX_SET_STS(mbx_status status, int numb, mbx_status sttVal) +{ + numb &= 7; /* 0 <= numb < 8 */ + status &= (mbx_status)(~(0xF << (numb*4))); + return status |= (sttVal & 0xF) << (numb*4); +} + +__INLINE mbx_status MBX_GET_STS(mbx_status status, int numb) +{ + return (status >>(numb*4)) & 0xF; +} +__INLINE mbx_status MBX_SET_STS_ALL(mbx_status stsVal) +{ + return (stsVal<<4*7) | (stsVal<<4*6) | (stsVal<<4*5) | (stsVal<<4*4) | (stsVal<<4*3) | (stsVal<<4*2) | (stsVal<<4*1) | stsVal; +} + +__INLINE mbx_status MBX_SET_STS_BY_MASK(mbx_status status, int8u mask, mbx_status sttVal) +{ + int numb; + + for(numb=0; numb<8; numb++) { + mbx_status buf_stt = (0 - ((mask>>numb) &1)) & sttVal; + status = MBX_SET_STS(status, numb, buf_stt); + } + return status; +} + +__INLINE int MBX_IS_ANY_OK_STS(mbx_status status) +{ + int ret = MBX_STATUS_OK==MBX_GET_STS(status, 0) + || MBX_STATUS_OK==MBX_GET_STS(status, 1) + || MBX_STATUS_OK==MBX_GET_STS(status, 2) + || MBX_STATUS_OK==MBX_GET_STS(status, 3) + || MBX_STATUS_OK==MBX_GET_STS(status, 4) + || MBX_STATUS_OK==MBX_GET_STS(status, 5) + || MBX_STATUS_OK==MBX_GET_STS(status, 6) + || MBX_STATUS_OK==MBX_GET_STS(status, 7); + return ret; +} + +/* +// Helpers for 64-bit status mbx_status16 +*/ + +/* Accessors for the low and high part of 64-bit status */ +__INLINE mbx_status MBX_GET_HIGH_PART_STS16(mbx_status16 status16) +{ + return ((mbx_status)(((mbx_status16)(status16) >> 32) & 0xFFFFFFFF)); +} + +__INLINE mbx_status MBX_GET_LOW_PART_STS16(mbx_status16 status16) +{ + return ((mbx_status)(status16)); +} + +__INLINE mbx_status16 MBX_SET_STS16_ALL(mbx_status16 stsVal) +{ + return (stsVal<<4*15) | (stsVal<<4*14) | (stsVal<<4*13) | (stsVal<<4*12) | (stsVal<<4*11) | (stsVal<<4*10) | (stsVal<<4*9) | (stsVal<<4*8) | \ + (stsVal<<4*7) | (stsVal<<4*6) | (stsVal<<4*5) | (stsVal<<4*4) | (stsVal<<4*3) | (stsVal<<4*2) | (stsVal<<4*1) | stsVal; +} + +__INLINE mbx_status16 MBX_SET_STS16(mbx_status16 status, int numb, mbx_status16 sttVal) +{ + numb &= 15; /* 0 <= numb < 16 */ + status &= (mbx_status16)(~((int64u)0xF << (numb*4))); + return status |= (sttVal & 0xF) << (numb*4); +} + +__INLINE mbx_status16 MBX_SET_STS16_BY_MASK(mbx_status16 status, int16u mask, mbx_status16 sttVal) +{ + int numb; + for (numb = 0; numb < 16; numb++) { + mbx_status16 buf_stt = (0 - ((mask >> numb) & 1)) & sttVal; + status = MBX_SET_STS16(status, numb, buf_stt); + } + return status; +} + +__INLINE int MBX_IS_ANY_OK_STS16(mbx_status16 status) +{ + return MBX_IS_ANY_OK_STS(MBX_GET_HIGH_PART_STS16(status)) || \ + MBX_IS_ANY_OK_STS(MBX_GET_LOW_PART_STS16(status)); +} + +#endif /* STATUS_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/version.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/version.h new file mode 100644 index 0000000..8e6c863 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/version.h @@ -0,0 +1,45 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef VERSION_H +#define VERSION_H + +#include + +/* crypto_mb name & version */ +#define MBX_LIB_NAME() "crypto_mb" +#define MBX_VER_MAJOR 1 +#define MBX_VER_MINOR 0 +#define MBX_VER_REV 9 + +/* major interface version */ +#define MBX_INTERFACE_VERSION_MAJOR 11 +/* minor interface version */ +#define MBX_INTERFACE_VERSION_MINOR 9 + +typedef struct { + int major; /* e.g. 1 */ + int minor; /* e.g. 2 */ + int revision; /* e.g. 3 */ + const char* name; /* e,g. "crypto_mb" */ + const char* buildDate; /* e.g. "Oct 28 2019" */ + const char* strVersion;/* e.g. "crypto_mb (ver 1.2.3 Oct 28 2019)" */ +} mbxVersion; + +EXTERN_C const mbxVersion* mbx_getversion(void); + +#endif /* VERSION_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/x25519.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/x25519.h new file mode 100644 index 0000000..ab0231f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/crypto_mb/x25519.h @@ -0,0 +1,31 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef X25519_H +#define X25519_H + +#include +#include + +EXTERN_C mbx_status mbx_x25519_public_key_mb8(int8u* const pa_public_key[8], + const int8u* const pa_private_key[8]); + +EXTERN_C mbx_status mbx_x25519_mb8(int8u* const pa_shared_key[8], + const int8u* const pa_private_key[8], + const int8u* const pa_public_key[8]); + +#endif /* X25519_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/ippcp.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/ippcp.h new file mode 100644 index 0000000..9d7e467 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/ippcp.h @@ -0,0 +1,1541 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) +// +*/ + +#if !defined( IPPCP_H__ ) || defined( _OWN_BLDPCS ) +#define IPPCP_H__ + + +#ifndef IPPCPDEFS_H__ + #include "ippcpdefs.h" +#endif + + +#ifdef __cplusplus +extern "C" { +#endif + +#if !defined( IPP_NO_DEFAULT_LIB ) + #if defined( _IPP_SEQUENTIAL_DYNAMIC ) + #pragma comment( lib, __FILE__ "/../../lib/" INTEL_PLATFORM "ippcp" ) + #elif defined( _IPP_SEQUENTIAL_STATIC ) + #pragma comment( lib, __FILE__ "/../../lib/" INTEL_PLATFORM "ippcpmt" ) + #endif +#endif + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && !defined(__INTEL_LLVM_COMPILER) +#pragma warning(push) +#pragma warning(disable : 4100) // for MSVC, unreferenced parameter +#endif + +/* ///////////////////////////////////////////////////////////////////////////// +// Name: ippcpGetLibVersion +// Purpose: getting of the library version +// Returns: the structure of information about version of ippCP library +// Parameters: +// +// Notes: not necessary to release the returned structure +*/ +IPPAPI( const IppLibraryVersion*, ippcpGetLibVersion, (void) ) + + +/* +// ========================================================= +// Symmetric Ciphers +// ========================================================= +*/ + +/* TDES */ + +#define TDES_DEPRECATED "This algorithm is considered weak due to known attacks on it. \ +The functionality remains in the library, but the implementation will no be longer \ +optimized and no security patches will be applied. A more secure alternative is available: AES" + +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsDESGetSize,(int *size)) +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsDESInit,(const Ipp8u* pKey, IppsDESSpec* pCtx)) + +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsDESPack,(const IppsDESSpec* pCtx, Ipp8u* pBuffer)) +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsDESUnpack,(const Ipp8u* pBuffer, IppsDESSpec* pCtx)) + +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsTDESEncryptECB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsDESSpec* pCtx1, const IppsDESSpec* pCtx2, const IppsDESSpec* pCtx3, + IppsCPPadding padding)) +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsTDESDecryptECB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsDESSpec* pCtx1, const IppsDESSpec* pCtx2, const IppsDESSpec* pCtx3, + IppsCPPadding padding)) + +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsTDESEncryptCBC,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsDESSpec* pCtx1, const IppsDESSpec* pCtx2, const IppsDESSpec* pCtx3, + const Ipp8u* pIV, + IppsCPPadding padding)) +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsTDESDecryptCBC,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsDESSpec* pCtx1, const IppsDESSpec* pCtx2, const IppsDESSpec* pCtx3, + const Ipp8u* pIV, + IppsCPPadding padding)) + +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsTDESEncryptCFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int cfbBlkSize, + const IppsDESSpec* pCtx1, const IppsDESSpec* pCtx2, const IppsDESSpec* pCtx3, + const Ipp8u* pIV, + IppsCPPadding padding)) +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsTDESDecryptCFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int cfbBlkSize, + const IppsDESSpec* pCtx1, const IppsDESSpec* pCtx2, const IppsDESSpec* pCtx3, + const Ipp8u* pIV, + IppsCPPadding padding)) + +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsTDESEncryptOFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int ofbBlkSize, + const IppsDESSpec* pCtx1, + const IppsDESSpec* pCtx2, + const IppsDESSpec* pCtx3, + Ipp8u* pIV)) +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsTDESDecryptOFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int ofbBlkSize, + const IppsDESSpec* pCtx1, + const IppsDESSpec* pCtx2, + const IppsDESSpec* pCtx3, + Ipp8u* pIV)) + +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsTDESEncryptCTR,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsDESSpec* pCtx1, + const IppsDESSpec* pCtx2, + const IppsDESSpec* pCtx3, + Ipp8u* pCtrValue, int ctrNumBitSize)) +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsTDESDecryptCTR,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsDESSpec* pCtx1, + const IppsDESSpec* pCtx2, + const IppsDESSpec* pCtx3, + Ipp8u* pCtrValue, int ctrNumBitSize)) + +/* AES */ +IPPAPI(IppStatus, ippsAESGetSize,(int *pSize)) +IPPAPI(IppStatus, ippsAESInit,(const Ipp8u* pKey, int keyLen, IppsAESSpec* pCtx, int ctxSize)) +IPPAPI(IppStatus, ippsAESSetKey,(const Ipp8u* pKey, int keyLen, IppsAESSpec* pCtx)) + +IPPAPI(IppStatus, ippsAESPack,(const IppsAESSpec* pCtx, Ipp8u* pBuffer, int bufSize)) +IPPAPI(IppStatus, ippsAESUnpack,(const Ipp8u* pBuffer, IppsAESSpec* pCtx, int ctxSize)) + +#define ECB_DEPRECATED "ECB functionality remains in the library, but it is not safe when used as is. \ +It is recommended to use any other mode, for example CBC." + +IPP_DEPRECATED(ECB_DEPRECATED) \ +IPPAPI(IppStatus, ippsAESEncryptECB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx)) +IPP_DEPRECATED(ECB_DEPRECATED) \ +IPPAPI(IppStatus, ippsAESDecryptECB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx)) + +IPPAPI(IppStatus, ippsAESEncryptCBC,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsAESEncryptCBC_CS1,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsAESEncryptCBC_CS2,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsAESEncryptCBC_CS3,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsAESDecryptCBC,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsAESDecryptCBC_CS1,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsAESDecryptCBC_CS2,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsAESDecryptCBC_CS3,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + const Ipp8u* pIV)) + +IPPAPI(IppStatus, ippsAESEncryptCFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int cfbBlkSize, + const IppsAESSpec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsAESDecryptCFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int cfbBlkSize, + const IppsAESSpec* pCtx, + const Ipp8u* pIV)) + +IPPAPI(IppStatus, ippsAESEncryptOFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int ofbBlkSize, + const IppsAESSpec* pCtx, + Ipp8u* pIV)) +IPPAPI(IppStatus, ippsAESDecryptOFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int ofbBlkSize, + const IppsAESSpec* pCtx, + Ipp8u* pIV)) + +IPPAPI(IppStatus, ippsAESEncryptCTR,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + Ipp8u* pCtrValue, int ctrNumBitSize)) +IPPAPI(IppStatus, ippsAESDecryptCTR,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + Ipp8u* pCtrValue, int ctrNumBitSize)) + +IPPAPI(IppStatus, ippsAESEncryptXTS_Direct,(const Ipp8u* pSrc, Ipp8u* pDst, int encBitsize, int aesBlkNo, + const Ipp8u* pTweakPT, + const Ipp8u* pKey, int keyBitsize, + int dataUnitBitsize)) +IPPAPI(IppStatus, ippsAESDecryptXTS_Direct,(const Ipp8u* pSrc, Ipp8u* pDst, int encBitsize, int aesBlkNo, + const Ipp8u* pTweakPT, + const Ipp8u* pKey, int keyBitsize, + int dataUnitBitsize)) + +IPPAPI(IppStatus, ippsAESSetupNoise,(Ipp32u noiseLevel, IppsAESSpec* pCtx)) +IPPAPI(IppStatus, ippsAES_GCMSetupNoise,(Ipp32u noiseLevel, IppsAES_GCMState* pState)) +IPPAPI(IppStatus, ippsAES_CMACSetupNoise,(Ipp32u noiseLevel, IppsAES_CMACState* pState)) + +/* AES multi-buffer functions */ +IPPAPI(IppStatus, ippsAES_EncryptCFB16_MB, (const Ipp8u* pSrc[], Ipp8u* pDst[], int len[], + const IppsAESSpec* pCtx[], + const Ipp8u* pIV[], + IppStatus status[], + int numBuffers)) + +/* SMS4 */ +IPPAPI(IppStatus, ippsSMS4GetSize,(int *pSize)) +IPPAPI(IppStatus, ippsSMS4Init,(const Ipp8u* pKey, int keyLen, IppsSMS4Spec* pCtx, int ctxSize)) +IPPAPI(IppStatus, ippsSMS4SetKey,(const Ipp8u* pKey, int keyLen, IppsSMS4Spec* pCtx)) + +IPP_DEPRECATED(ECB_DEPRECATED) \ +IPPAPI(IppStatus, ippsSMS4EncryptECB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx)) +IPP_DEPRECATED(ECB_DEPRECATED) \ +IPPAPI(IppStatus, ippsSMS4DecryptECB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx)) + +IPPAPI(IppStatus, ippsSMS4EncryptCBC,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsSMS4EncryptCBC_CS1,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsSMS4EncryptCBC_CS2,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsSMS4EncryptCBC_CS3,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsSMS4DecryptCBC,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsSMS4DecryptCBC_CS1,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsSMS4DecryptCBC_CS2,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsSMS4DecryptCBC_CS3,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx, + const Ipp8u* pIV)) + +IPPAPI(IppStatus, ippsSMS4EncryptCFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int cfbBlkSize, + const IppsSMS4Spec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsSMS4DecryptCFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int cfbBlkSize, + const IppsSMS4Spec* pCtx, + const Ipp8u* pIV)) + +IPPAPI(IppStatus, ippsSMS4EncryptOFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int ofbBlkSize, + const IppsSMS4Spec* pCtx, + Ipp8u* pIV)) +IPPAPI(IppStatus, ippsSMS4DecryptOFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int ofbBlkSize, + const IppsSMS4Spec* pCtx, + Ipp8u* pIV)) + +IPPAPI(IppStatus, ippsSMS4EncryptCTR,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx, + Ipp8u* pCtrValue, int ctrNumBitSize)) +IPPAPI(IppStatus, ippsSMS4DecryptCTR,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx, + Ipp8u* pCtrValue, int ctrNumBitSize)) + +/* SMS4-CCM */ +IPPAPI(IppStatus, ippsSMS4_CCMGetSize,(int* pSize)) +IPPAPI(IppStatus, ippsSMS4_CCMInit,(const Ipp8u* pKey, int keyLen, IppsSMS4_CCMState* pCtx, int ctxSize)) + +IPPAPI(IppStatus, ippsSMS4_CCMMessageLen,(Ipp64u msgLen, IppsSMS4_CCMState* pCtx)) +IPPAPI(IppStatus, ippsSMS4_CCMTagLen,(int tagLen, IppsSMS4_CCMState* pCtx)) + +IPPAPI(IppStatus, ippsSMS4_CCMStart,(const Ipp8u* pIV, int ivLen, const Ipp8u* pAD, int adLen, IppsSMS4_CCMState* pCtx)) +IPPAPI(IppStatus, ippsSMS4_CCMEncrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int len, IppsSMS4_CCMState* pCtx)) +IPPAPI(IppStatus, ippsSMS4_CCMDecrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int len, IppsSMS4_CCMState* pCtx)) +IPPAPI(IppStatus, ippsSMS4_CCMGetTag,(Ipp8u* pTag, int tagLen, const IppsSMS4_CCMState* pCtx)) + +/* +// ========================================================= +// AES based authentication & confidence Primitives +// ========================================================= +*/ + +/* AES-CCM */ +IPPAPI(IppStatus, ippsAES_CCMGetSize,(int* pSize)) +IPPAPI(IppStatus, ippsAES_CCMInit,(const Ipp8u* pKey, int keyLen, IppsAES_CCMState* pState, int ctxSize)) + +IPPAPI(IppStatus, ippsAES_CCMMessageLen,(Ipp64u msgLen, IppsAES_CCMState* pState)) +IPPAPI(IppStatus, ippsAES_CCMTagLen,(int tagLen, IppsAES_CCMState* pState)) + +IPPAPI(IppStatus, ippsAES_CCMStart,(const Ipp8u* pIV, int ivLen, const Ipp8u* pAD, int adLen, IppsAES_CCMState* pState)) +IPPAPI(IppStatus, ippsAES_CCMEncrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int len, IppsAES_CCMState* pState)) +IPPAPI(IppStatus, ippsAES_CCMDecrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int len, IppsAES_CCMState* pState)) +IPPAPI(IppStatus, ippsAES_CCMGetTag,(Ipp8u* pTag, int tagLen, const IppsAES_CCMState* pState)) + +/* AES-GCM */ +IPPAPI(IppStatus, ippsAES_GCMGetSize,(int * pSize)) +IPPAPI(IppStatus, ippsAES_GCMInit,(const Ipp8u* pKey, int keyLen, IppsAES_GCMState* pState, int ctxSize)) +IPPAPI(IppStatus, ippsAES_GCMReinit,(IppsAES_GCMState* pState)) + +IPPAPI(IppStatus, ippsAES_GCMReset,(IppsAES_GCMState* pState)) +IPPAPI(IppStatus, ippsAES_GCMProcessIV,(const Ipp8u* pIV, int ivLen, + IppsAES_GCMState* pState)) +IPPAPI(IppStatus, ippsAES_GCMProcessAAD,(const Ipp8u* pAAD, int aadLen, + IppsAES_GCMState* pState)) +IPPAPI(IppStatus, ippsAES_GCMStart,(const Ipp8u* pIV, int ivLen, + const Ipp8u* pAAD, int aadLen, + IppsAES_GCMState* pState)) +IPPAPI(IppStatus, ippsAES_GCMEncrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int len, IppsAES_GCMState* pState)) +IPPAPI(IppStatus, ippsAES_GCMDecrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int len, IppsAES_GCMState* pState)) +IPPAPI(IppStatus, ippsAES_GCMGetTag,(Ipp8u* pDstTag, int tagLen, const IppsAES_GCMState* pState)) + +/* AES-XTS */ +IPPAPI(IppStatus, ippsAES_XTSGetSize,(int * pSize)) +IPPAPI(IppStatus, ippsAES_XTSInit,(const Ipp8u* pKey, int keyLen, + int duBitsize, + IppsAES_XTSSpec* pCtx,int ctxSize)) +IPPAPI(IppStatus, ippsAES_XTSEncrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int bitSizeLen, + const IppsAES_XTSSpec* pCtx, + const Ipp8u* pTweak, + int startCipherBlkNo)) +IPPAPI(IppStatus, ippsAES_XTSDecrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int bitSizeLen, + const IppsAES_XTSSpec* pCtx, + const Ipp8u* pTweak, + int startCipherBlkNo)) + +/* AES-SIV (RFC 5297) */ +IPPAPI(IppStatus, ippsAES_S2V_CMAC,(const Ipp8u* pKey, int keyLen, + const Ipp8u* pAD[], const int pADlen[], int numAD, + Ipp8u* pV)) +IPPAPI(IppStatus, ippsAES_SIVEncrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + Ipp8u* pSIV, + const Ipp8u* pAuthKey, const Ipp8u* pConfKey, int keyLen, + const Ipp8u* pAD[], const int pADlen[], int numAD)) +IPPAPI(IppStatus, ippsAES_SIVDecrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + int* pAuthPassed, + const Ipp8u* pAuthKey, const Ipp8u* pConfKey, int keyLen, + const Ipp8u* pAD[], const int pADlen[], int numAD, + const Ipp8u* pSIV)) + +/* AES-CMAC */ +IPPAPI(IppStatus, ippsAES_CMACGetSize,(int* pSize)) +IPPAPI(IppStatus, ippsAES_CMACInit,(const Ipp8u* pKey, int keyLen, IppsAES_CMACState* pState, int ctxSize)) + +IPPAPI(IppStatus, ippsAES_CMACUpdate,(const Ipp8u* pSrc, int len, IppsAES_CMACState* pState)) +IPPAPI(IppStatus, ippsAES_CMACFinal,(Ipp8u* pMD, int mdLen, IppsAES_CMACState* pState)) +IPPAPI(IppStatus, ippsAES_CMACGetTag,(Ipp8u* pMD, int mdLen, const IppsAES_CMACState* pState)) + + +/* +// ========================================================= +// RC4 Stream Ciphers +// ========================================================= +*/ + +#define RC4_DEPRECATED "is deprecated. This algorithm is considered weak due to known attacks on it. \ +It is obsolete and will be removed in one of the future Intel IPP Cryptography releases." + +IPP_DEPRECATED(RC4_DEPRECATED) \ +IPPAPI(IppStatus, ippsARCFourCheckKey, (const Ipp8u *pKey, int keyLen, IppBool* pIsWeak)) + +IPP_DEPRECATED(RC4_DEPRECATED) \ +IPPAPI(IppStatus, ippsARCFourGetSize, (int* pSize)) +IPP_DEPRECATED(RC4_DEPRECATED) \ +IPPAPI(IppStatus, ippsARCFourInit, (const Ipp8u *pKey, int keyLen, IppsARCFourState *pCtx)) +IPP_DEPRECATED(RC4_DEPRECATED) \ +IPPAPI(IppStatus, ippsARCFourReset, (IppsARCFourState* pCtx)) + +IPP_DEPRECATED(RC4_DEPRECATED) \ +IPPAPI(IppStatus, ippsARCFourPack,(const IppsARCFourState* pCtx, Ipp8u* pBuffer)) +IPP_DEPRECATED(RC4_DEPRECATED) \ +IPPAPI(IppStatus, ippsARCFourUnpack,(const Ipp8u* pBuffer, IppsARCFourState* pCtx)) + +IPP_DEPRECATED(RC4_DEPRECATED) \ +IPPAPI(IppStatus, ippsARCFourEncrypt, (const Ipp8u *pSrc, Ipp8u *pDst, int length, IppsARCFourState *pCtx)) +IPP_DEPRECATED(RC4_DEPRECATED) \ +IPPAPI(IppStatus, ippsARCFourDecrypt, (const Ipp8u *pSrc, Ipp8u *pDst, int length, IppsARCFourState *pCtx)) + + +/* +// ========================================================= +// One-Way Hash Functions +// ========================================================= +*/ + +#define OBSOLETE_API "is deprecated. This API is considered obsolete and will be removed in one of future Intel IPP Cryptography releases. \ +Use the following link for opening a ticket and providing feedback: https://supporttickets.intel.com/ if you have concerns." + +#define SHA1_DEPRECATED "This algorithm is considered weak due to known attacks on it. \ +The functionality remains in the library, but the implementation will no be longer \ +optimized and no security patches will be applied. A more secure alternative is available: SHA-2" + +/* SHA1 Hash Primitives */ +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI(IppStatus, ippsSHA1GetSize,(int* pSize)) +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI(IppStatus, ippsSHA1Init,(IppsSHA1State* pState)) +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI(IppStatus, ippsSHA1Duplicate,(const IppsSHA1State* pSrcState, IppsSHA1State* pDstState)) + +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI(IppStatus, ippsSHA1Pack,(const IppsSHA1State* pState, Ipp8u* pBuffer)) +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI(IppStatus, ippsSHA1Unpack,(const Ipp8u* pBuffer, IppsSHA1State* pState)) + +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI(IppStatus, ippsSHA1Update,(const Ipp8u* pSrc, int len, IppsSHA1State* pState)) +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI(IppStatus, ippsSHA1GetTag,(Ipp8u* pTag, Ipp32u tagLen, const IppsSHA1State* pState)) +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI(IppStatus, ippsSHA1Final,(Ipp8u* pMD, IppsSHA1State* pState)) +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI(IppStatus, ippsSHA1MessageDigest,(const Ipp8u* pMsg, int len, Ipp8u* pMD)) + +/* SHA224 Hash Primitives */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA224GetSize,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA224Init,(IppsSHA224State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA224Duplicate,(const IppsSHA224State* pSrcState, IppsSHA224State* pDstState)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA224Pack,(const IppsSHA224State* pState, Ipp8u* pBuffer)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA224Unpack,(const Ipp8u* pBuffer, IppsSHA224State* pState)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA224Update,(const Ipp8u* pSrc, int len, IppsSHA224State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA224GetTag,(Ipp8u* pTag, Ipp32u tagLen, const IppsSHA224State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA224Final,(Ipp8u* pMD, IppsSHA224State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA224MessageDigest,(const Ipp8u* pMsg, int len, Ipp8u* pMD)) + +/* SHA256 Hash Primitives */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA256GetSize,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA256Init,(IppsSHA256State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA256Duplicate,(const IppsSHA256State* pSrcState, IppsSHA256State* pDstState)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA256Pack,(const IppsSHA256State* pState, Ipp8u* pBuffer)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA256Unpack,(const Ipp8u* pBuffer, IppsSHA256State* pState)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA256Update,(const Ipp8u* pSrc, int len, IppsSHA256State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA256GetTag,(Ipp8u* pTag, Ipp32u tagLen, const IppsSHA256State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA256Final,(Ipp8u* pMD, IppsSHA256State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA256MessageDigest,(const Ipp8u* pMsg, int len, Ipp8u* pMD)) + +/* SHA384 Hash Primitives */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA384GetSize,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA384Init,(IppsSHA384State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA384Duplicate,(const IppsSHA384State* pSrcState, IppsSHA384State* pDstState)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA384Pack,(const IppsSHA384State* pState, Ipp8u* pBuffer)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA384Unpack,(const Ipp8u* pBuffer, IppsSHA384State* pState)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA384Update,(const Ipp8u* pSrc, int len, IppsSHA384State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA384GetTag,(Ipp8u* pTag, Ipp32u tagLen, const IppsSHA384State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA384Final,(Ipp8u* pMD, IppsSHA384State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA384MessageDigest,(const Ipp8u* pMsg, int len, Ipp8u* pMD)) + +/* SHA512 Hash Primitives */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA512GetSize,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA512Init,(IppsSHA512State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA512Duplicate,(const IppsSHA512State* pSrcState, IppsSHA512State* pDstState)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA512Pack,(const IppsSHA512State* pState, Ipp8u* pBuffer)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA512Unpack,(const Ipp8u* pBuffer, IppsSHA512State* pState)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA512Update,(const Ipp8u* pSrc, int len, IppsSHA512State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA512GetTag,(Ipp8u* pTag, Ipp32u tagLen, const IppsSHA512State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA512Final,(Ipp8u* pMD, IppsSHA512State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA512MessageDigest,(const Ipp8u* pMsg, int len, Ipp8u* pMD)) + +/* MD5 Hash Primitives */ + +#define MD5_DEPRECATED "This algorithm is considered weak due to known attacks on it. \ +The functionality remains in the library, but the implementation will no be longer \ +optimized and no security patches will be applied. A more secure alternative is available: SHA-2" + +IPP_DEPRECATED(MD5_DEPRECATED) \ +IPPAPI(IppStatus, ippsMD5GetSize,(int* pSize)) +IPP_DEPRECATED(MD5_DEPRECATED) \ +IPPAPI(IppStatus, ippsMD5Init,(IppsMD5State* pState)) +IPP_DEPRECATED(MD5_DEPRECATED) \ +IPPAPI(IppStatus, ippsMD5Duplicate,(const IppsMD5State* pSrcState, IppsMD5State* pDstState)) + +IPP_DEPRECATED(MD5_DEPRECATED) \ +IPPAPI(IppStatus, ippsMD5Pack,(const IppsMD5State* pState, Ipp8u* pBuffer)) +IPP_DEPRECATED(MD5_DEPRECATED) \ +IPPAPI(IppStatus, ippsMD5Unpack,(const Ipp8u* pBuffer, IppsMD5State* pState)) + +IPP_DEPRECATED(MD5_DEPRECATED) \ +IPPAPI(IppStatus, ippsMD5Update,(const Ipp8u* pSrc, int len, IppsMD5State* pState)) +IPP_DEPRECATED(MD5_DEPRECATED) \ +IPPAPI(IppStatus, ippsMD5GetTag,(Ipp8u* pTag, Ipp32u tagLen, const IppsMD5State* pState)) +IPP_DEPRECATED(MD5_DEPRECATED) \ +IPPAPI(IppStatus, ippsMD5Final,(Ipp8u* pMD, IppsMD5State* pState)) +IPP_DEPRECATED(MD5_DEPRECATED) \ +IPPAPI(IppStatus, ippsMD5MessageDigest,(const Ipp8u* pMsg, int len, Ipp8u* pMD)) + +/* SM3 Hash Primitives */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSM3GetSize,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSM3Init,(IppsSM3State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSM3Duplicate,(const IppsSM3State* pSrcState, IppsSM3State* pDstState)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSM3Pack,(const IppsSM3State* pState, Ipp8u* pBuffer)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSM3Unpack,(const Ipp8u* pBuffer, IppsSM3State* pState)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSM3Update,(const Ipp8u* pSrc, int len, IppsSM3State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSM3GetTag,(Ipp8u* pTag, Ipp32u tagLen, const IppsSM3State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSM3Final,(Ipp8u* pMD, IppsSM3State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSM3MessageDigest,(const Ipp8u* pMsg, int len, Ipp8u* pMD)) + +/* generalized Hash Primitives */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHashGetSize,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHashInit,(IppsHashState* pState, IppHashAlgId hashAlg)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHashPack,(const IppsHashState* pState, Ipp8u* pBuffer, int bufSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHashUnpack,(const Ipp8u* pBuffer, IppsHashState* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHashDuplicate,(const IppsHashState* pSrcState, IppsHashState* pDstState)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHashUpdate,(const Ipp8u* pSrc, int len, IppsHashState* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHashGetTag,(Ipp8u* pTag, int tagLen, const IppsHashState* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHashFinal,(Ipp8u* pMD, IppsHashState* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHashMessage,(const Ipp8u* pMsg, int len, Ipp8u* pMD, IppHashAlgId hashAlg)) + +/* method based generalized (reduced memory footprint) Hash Primitives */ +IPP_DEPRECATED(MD5_DEPRECATED) \ +IPPAPI( const IppsHashMethod*, ippsHashMethod_MD5, (void) ) +IPPAPI( const IppsHashMethod*, ippsHashMethod_SM3, (void) ) + +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA1, (void) ) +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA1_NI, (void) ) +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA1_TT, (void) ) + +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA256, (void) ) +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA256_NI, (void) ) +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA256_TT, (void) ) + +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA224, (void) ) +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA224_NI, (void) ) +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA224_TT, (void) ) + +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA512, (void) ) +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA384, (void) ) +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA512_256, (void) ) +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA512_224, (void) ) + +IPPAPI( IppStatus, ippsHashMethodGetSize, (int* pSize) ) +IPP_DEPRECATED(MD5_DEPRECATED) \ +IPPAPI( IppStatus, ippsHashMethodSet_MD5, (IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashMethodSet_SM3, (IppsHashMethod* pMethod) ) + +IPPAPI( IppStatus, ippsHashStateMethodSet_SM3, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) + +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI( IppStatus, ippsHashMethodSet_SHA1, (IppsHashMethod* pMethod) ) +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI( IppStatus, ippsHashMethodSet_SHA1_NI, (IppsHashMethod* pMethod) ) +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI( IppStatus, ippsHashMethodSet_SHA1_TT, (IppsHashMethod* pMethod) ) + +IPPAPI( IppStatus, ippsHashMethodSet_SHA256, (IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashMethodSet_SHA256_NI, (IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashMethodSet_SHA256_TT, (IppsHashMethod* pMethod) ) + +IPPAPI( IppStatus, ippsHashStateMethodSet_SHA256, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashStateMethodSet_SHA256_NI, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashStateMethodSet_SHA256_TT, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) + +IPPAPI( IppStatus, ippsHashMethodSet_SHA224, (IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashMethodSet_SHA224_NI, (IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashMethodSet_SHA224_TT, (IppsHashMethod* pMethod) ) + +IPPAPI( IppStatus, ippsHashStateMethodSet_SHA224, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashStateMethodSet_SHA224_NI, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashStateMethodSet_SHA224_TT, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) + +IPPAPI( IppStatus, ippsHashMethodSet_SHA512, (IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashMethodSet_SHA384, (IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashMethodSet_SHA512_256, (IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashMethodSet_SHA512_224, (IppsHashMethod* pMethod) ) + +IPPAPI( IppStatus, ippsHashStateMethodSet_SHA512, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashStateMethodSet_SHA384, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashStateMethodSet_SHA512_256, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashStateMethodSet_SHA512_224, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) + +IPPAPI(IppStatus, ippsHashGetSize_rmf,(int* pSize)) +IPPAPI(IppStatus, ippsHashInit_rmf,(IppsHashState_rmf* pState, const IppsHashMethod* pMethod)) + +IPPAPI(IppStatus, ippsHashPack_rmf,(const IppsHashState_rmf* pState, Ipp8u* pBuffer, int bufSize)) +IPPAPI(IppStatus, ippsHashUnpack_rmf,(const Ipp8u* pBuffer, IppsHashState_rmf* pState)) +IPPAPI(IppStatus, ippsHashDuplicate_rmf,(const IppsHashState_rmf* pSrcState, IppsHashState_rmf* pDstState)) + +IPPAPI(IppStatus, ippsHashUpdate_rmf,(const Ipp8u* pSrc, int len, IppsHashState_rmf* pState)) +IPPAPI(IppStatus, ippsHashGetTag_rmf,(Ipp8u* pMD, int tagLen, const IppsHashState_rmf* pState)) +IPPAPI(IppStatus, ippsHashFinal_rmf,(Ipp8u* pMD, IppsHashState_rmf* pState)) +IPPAPI(IppStatus, ippsHashMessage_rmf,(const Ipp8u* pMsg, int len, Ipp8u* pMD, const IppsHashMethod* pMethod)) + +IPPAPI(IppStatus, ippsHashMethodGetInfo,(IppsHashInfo* pInfo, const IppsHashMethod* pMethod)) +IPPAPI(IppStatus, ippsHashGetInfo_rmf,(IppsHashInfo* pInfo, const IppsHashState_rmf* pState)) + +/* general MGF Primitives*/ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsMGF,(const Ipp8u* pSeed, int seedLen, Ipp8u* pMask, int maskLen, IppHashAlgId hashAlg)) +IPPAPI(IppStatus, ippsMGF1_rmf,(const Ipp8u* pSeed, int seedLen, Ipp8u* pMask, int maskLen, const IppsHashMethod* pMethod)) +IPPAPI(IppStatus, ippsMGF2_rmf,(const Ipp8u* pSeed, int seedLen, Ipp8u* pMask, int maskLen, const IppsHashMethod* pMethod)) + + +/* +// ========================================================= +// Keyed-Hash Message Authentication Codes +// ========================================================= +*/ + +/* generalized Keyed HMAC primitives */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHMAC_GetSize,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHMAC_Init,(const Ipp8u* pKey, int keyLen, IppsHMACState* pCtx, IppHashAlgId hashAlg)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHMAC_Pack,(const IppsHMACState* pCtx, Ipp8u* pBuffer, int bufSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHMAC_Unpack,(const Ipp8u* pBuffer, IppsHMACState* pCtx)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHMAC_Duplicate,(const IppsHMACState* pSrcCtx, IppsHMACState* pDstCtx)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHMAC_Update,(const Ipp8u* pSrc, int len, IppsHMACState* pCtx)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHMAC_Final,(Ipp8u* pMD, int mdLen, IppsHMACState* pCtx)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHMAC_GetTag,(Ipp8u* pMD, int mdLen, const IppsHMACState* pCtx)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHMAC_Message,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pKey, int keyLen, + Ipp8u* pMD, int mdLen, + IppHashAlgId hashAlg)) + +/* method based generalized (reduced memory footprint) Keyed HMAC primitives */ +IPPAPI(IppStatus, ippsHMACGetSize_rmf,(int* pSize)) +IPPAPI(IppStatus, ippsHMACInit_rmf,(const Ipp8u* pKey, int keyLen, + IppsHMACState_rmf* pCtx, + const IppsHashMethod* pMethod)) + +IPPAPI(IppStatus, ippsHMACPack_rmf,(const IppsHMACState_rmf* pCtx, Ipp8u* pBuffer, int bufSize)) +IPPAPI(IppStatus, ippsHMACUnpack_rmf,(const Ipp8u* pBuffer, IppsHMACState_rmf* pCtx)) +IPPAPI(IppStatus, ippsHMACDuplicate_rmf,(const IppsHMACState_rmf* pSrcCtx, IppsHMACState_rmf* pDstCtx)) + +IPPAPI(IppStatus, ippsHMACUpdate_rmf,(const Ipp8u* pSrc, int len, IppsHMACState_rmf* pCtx)) +IPPAPI(IppStatus, ippsHMACFinal_rmf,(Ipp8u* pMD, int mdLen, IppsHMACState_rmf* pCtx)) +IPPAPI(IppStatus, ippsHMACGetTag_rmf,(Ipp8u* pMD, int mdLen, const IppsHMACState_rmf* pCtx)) +IPPAPI(IppStatus, ippsHMACMessage_rmf,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pKey, int keyLen, + Ipp8u* pMD, int mdLen, + const IppsHashMethod* pMethod)) + + +/* +// ========================================================= +// Big Number Integer Arithmetic +// ========================================================= +*/ + +/* Signed BigNum Operations */ +IPPAPI(IppStatus, ippsBigNumGetSize,(int length, int* pSize)) +IPPAPI(IppStatus, ippsBigNumInit,(int length, IppsBigNumState* pBN)) + +IPPAPI(IppStatus, ippsCmpZero_BN,(const IppsBigNumState* pBN, Ipp32u* pResult)) +IPPAPI(IppStatus, ippsCmp_BN,(const IppsBigNumState* pA, const IppsBigNumState* pB, Ipp32u* pResult)) + +IPPAPI(IppStatus, ippsGetSize_BN,(const IppsBigNumState* pBN, int* pSize)) +IPPAPI(IppStatus, ippsSet_BN,(IppsBigNumSGN sgn, + int length, const Ipp32u* pData, + IppsBigNumState* pBN)) +IPPAPI(IppStatus, ippsGet_BN,(IppsBigNumSGN* pSgn, + int* pLength, Ipp32u* pData, + const IppsBigNumState* pBN)) +IPPAPI(IppStatus, ippsRef_BN,(IppsBigNumSGN* pSgn, int* bitSize, Ipp32u** const ppData, + const IppsBigNumState* pBN)) +IPPAPI(IppStatus, ippsExtGet_BN,(IppsBigNumSGN* pSgn, + int* pBitSize, Ipp32u* pData, + const IppsBigNumState* pBN)) + +IPPAPI(IppStatus, ippsAdd_BN, (IppsBigNumState* pA, IppsBigNumState* pB, IppsBigNumState* pR)) +IPPAPI(IppStatus, ippsSub_BN, (IppsBigNumState* pA, IppsBigNumState* pB, IppsBigNumState* pR)) +IPPAPI(IppStatus, ippsMul_BN, (IppsBigNumState* pA, IppsBigNumState* pB, IppsBigNumState* pR)) +IPPAPI(IppStatus, ippsMAC_BN_I, (IppsBigNumState* pA, IppsBigNumState* pB, IppsBigNumState* pR)) +IPPAPI(IppStatus, ippsDiv_BN, (IppsBigNumState* pA, IppsBigNumState* pB, IppsBigNumState* pQ, IppsBigNumState* pR)) +IPPAPI(IppStatus, ippsMod_BN, (IppsBigNumState* pA, IppsBigNumState* pM, IppsBigNumState* pR)) +IPPAPI(IppStatus, ippsGcd_BN, (IppsBigNumState* pA, IppsBigNumState* pB, IppsBigNumState* pGCD)) +IPPAPI(IppStatus, ippsModInv_BN,(IppsBigNumState* pA, IppsBigNumState* pM, IppsBigNumState* pInv)) + +IPPAPI(IppStatus, ippsSetOctString_BN,(const Ipp8u* pStr, int strLen, IppsBigNumState* pBN)) +IPPAPI(IppStatus, ippsGetOctString_BN,(Ipp8u* pStr, int strLen, const IppsBigNumState* pBN)) + +/* Montgomery Operations */ +IPPAPI(IppStatus, ippsMontGetSize,(IppsExpMethod method, int length, int* pSize)) +IPPAPI(IppStatus, ippsMontInit,(IppsExpMethod method, int length, IppsMontState* pCtx)) + +IPPAPI(IppStatus, ippsMontSet,(const Ipp32u* pModulo, int size, IppsMontState* pCtx)) +IPPAPI(IppStatus, ippsMontGet,(Ipp32u* pModulo, int* pSize, const IppsMontState* pCtx)) + +IPPAPI(IppStatus, ippsMontForm,(const IppsBigNumState* pA, IppsMontState* pCtx, IppsBigNumState* pR)) +IPPAPI(IppStatus, ippsMontMul, (const IppsBigNumState* pA, const IppsBigNumState* pB, IppsMontState* m, IppsBigNumState* pR)) +IPPAPI(IppStatus, ippsMontExp, (const IppsBigNumState* pA, const IppsBigNumState* pE, IppsMontState* m, IppsBigNumState* pR)) + +/* Pseudo-Random Number Generation */ +IPPAPI(IppStatus, ippsPRNGGetSize,(int* pSize)) +IPPAPI(IppStatus, ippsPRNGInit, (int seedBits, IppsPRNGState* pCtx)) + +IPPAPI(IppStatus, ippsPRNGSetModulus,(const IppsBigNumState* pMod, IppsPRNGState* pCtx)) +IPPAPI(IppStatus, ippsPRNGSetH0, (const IppsBigNumState* pH0, IppsPRNGState* pCtx)) +IPPAPI(IppStatus, ippsPRNGSetAugment,(const IppsBigNumState* pAug, IppsPRNGState* pCtx)) +IPPAPI(IppStatus, ippsPRNGSetSeed, (const IppsBigNumState* pSeed,IppsPRNGState* pCtx)) +IPPAPI(IppStatus, ippsPRNGGetSeed, (const IppsPRNGState* pCtx,IppsBigNumState* pSeed)) + +IPPAPI(IppStatus, ippsPRNGen, (Ipp32u* pRand, int nBits, void* pCtx)) +IPPAPI(IppStatus, ippsPRNGen_BN, (IppsBigNumState* pRand, int nBits, void* pCtx)) +IPPAPI(IppStatus, ippsPRNGenRDRAND, (Ipp32u* pRand, int nBits, void* pCtx)) +IPPAPI(IppStatus, ippsPRNGenRDRAND_BN,(IppsBigNumState* pRand, int nBits, void* pCtx)) +IPPAPI(IppStatus, ippsTRNGenRDSEED, (Ipp32u* pRand, int nBits, void* pCtx)) +IPPAPI(IppStatus, ippsTRNGenRDSEED_BN,(IppsBigNumState* pRand, int nBits, void* pCtx)) + +/* Probable Prime Number Generation */ +IPPAPI(IppStatus, ippsPrimeGetSize,(int nMaxBits, int* pSize)) +IPPAPI(IppStatus, ippsPrimeInit, (int nMaxBits, IppsPrimeState* pCtx)) + +IPPAPI(IppStatus, ippsPrimeGen, (int nBits, int nTrials, IppsPrimeState* pCtx, + IppBitSupplier rndFunc, void* pRndParam)) +IPPAPI(IppStatus, ippsPrimeTest,(int nTrials, Ipp32u* pResult, IppsPrimeState* pCtx, + IppBitSupplier rndFunc, void* pRndParam)) +IPPAPI(IppStatus, ippsPrimeGen_BN,(IppsBigNumState* pPrime, int nBits, int nTrials, IppsPrimeState* pCtx, + IppBitSupplier rndFunc, void* pRndParam)) +IPPAPI(IppStatus, ippsPrimeTest_BN,(const IppsBigNumState* pPrime, int nTrials, Ipp32u* pResult, IppsPrimeState* pCtx, + IppBitSupplier rndFunc, void* pRndParam)) + +IPPAPI(IppStatus, ippsPrimeGet, (Ipp32u* pPrime, int* pLen, const IppsPrimeState* pCtx)) +IPPAPI(IppStatus, ippsPrimeGet_BN,(IppsBigNumState* pPrime, const IppsPrimeState* pCtx)) + +IPPAPI(IppStatus, ippsPrimeSet, (const Ipp32u* pPrime, int nBits, IppsPrimeState* pCtx)) +IPPAPI(IppStatus, ippsPrimeSet_BN,(const IppsBigNumState* pPrime, IppsPrimeState* pCtx)) + + +/* +// ========================================================= +// RSA Cryptography +// ========================================================= +*/ +IPPAPI(IppStatus, ippsRSA_GetSizePublicKey,(int rsaModulusBitSize, int pubicExpBitSize, int* pKeySize)) +IPPAPI(IppStatus, ippsRSA_InitPublicKey,(int rsaModulusBitSize, int publicExpBitSize, + IppsRSAPublicKeyState* pKey, int keyCtxSize)) +IPPAPI(IppStatus, ippsRSA_SetPublicKey,(const IppsBigNumState* pModulus, + const IppsBigNumState* pPublicExp, + IppsRSAPublicKeyState* pKey)) +IPPAPI(IppStatus, ippsRSA_GetPublicKey,(IppsBigNumState* pModulus, + IppsBigNumState* pPublicExp, + const IppsRSAPublicKeyState* pKey)) + +IPPAPI(IppStatus, ippsRSA_GetSizePrivateKeyType1,(int rsaModulusBitSize, int privateExpBitSize, int* pKeySize)) +IPPAPI(IppStatus, ippsRSA_InitPrivateKeyType1,(int rsaModulusBitSize, int privateExpBitSize, + IppsRSAPrivateKeyState* pKey, int keyCtxSize)) +IPPAPI(IppStatus, ippsRSA_SetPrivateKeyType1,(const IppsBigNumState* pModulus, + const IppsBigNumState* pPrivateExp, + IppsRSAPrivateKeyState* pKey)) +IPPAPI(IppStatus, ippsRSA_GetPrivateKeyType1,(IppsBigNumState* pModulus, + IppsBigNumState* pPrivateExp, + const IppsRSAPrivateKeyState* pKey)) + +IPPAPI(IppStatus, ippsRSA_GetSizePrivateKeyType2,(int factorPbitSize, int factorQbitSize, int* pKeySize)) +IPPAPI(IppStatus, ippsRSA_InitPrivateKeyType2,(int factorPbitSize, int factorQbitSize, + IppsRSAPrivateKeyState* pKey, int keyCtxSize)) +IPPAPI(IppStatus, ippsRSA_SetPrivateKeyType2,(const IppsBigNumState* pFactorP, + const IppsBigNumState* pFactorQ, + const IppsBigNumState* pCrtExpP, + const IppsBigNumState* pCrtExpQ, + const IppsBigNumState* pInverseQ, + IppsRSAPrivateKeyState* pKey)) +IPPAPI(IppStatus, ippsRSA_GetPrivateKeyType2,(IppsBigNumState* pFactorP, + IppsBigNumState* pFactorQ, + IppsBigNumState* pCrtExpP, + IppsBigNumState* pCrtExpQ, + IppsBigNumState* pInverseQ, + const IppsRSAPrivateKeyState* pKey)) + +IPPAPI(IppStatus, ippsRSA_GetBufferSizePublicKey,(int* pBufferSize, const IppsRSAPublicKeyState* pKey)) +IPPAPI(IppStatus, ippsRSA_GetBufferSizePrivateKey,(int* pBufferSize, const IppsRSAPrivateKeyState* pKey)) + +IPPAPI(IppStatus, ippsRSA_Encrypt,(const IppsBigNumState* pPtxt, + IppsBigNumState* pCtxt, + const IppsRSAPublicKeyState* pKey, + Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsRSA_Decrypt,(const IppsBigNumState* pCtxt, + IppsBigNumState* pPtxt, + const IppsRSAPrivateKeyState* pKey, + Ipp8u* pScratchBuffer)) + +IPPAPI(IppStatus, ippsRSA_GenerateKeys,(const IppsBigNumState* pSrcPublicExp, + IppsBigNumState* pModulus, + IppsBigNumState* pPublicExp, + IppsBigNumState* pPrivateExp, + IppsRSAPrivateKeyState* pPrivateKeyType2, + Ipp8u* pScratchBuffer, + int nTrials, + IppsPrimeState* pPrimeGen, + IppBitSupplier rndFunc, void* pRndParam)) + +IPPAPI(IppStatus, ippsRSA_ValidateKeys,(int* pResult, + const IppsRSAPublicKeyState* pPublicKey, + const IppsRSAPrivateKeyState* pPrivateKeyType2, + const IppsRSAPrivateKeyState* pPrivateKeyType1, + Ipp8u* pScratchBuffer, + int nTrials, + IppsPrimeState* pPrimeGen, + IppBitSupplier rndFunc, void* pRndParam)) + +/* encryption scheme: RSAES-OAEP */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsRSAEncrypt_OAEP,(const Ipp8u* pSrc, int srcLen, + const Ipp8u* pLabel, int labLen, + const Ipp8u* pSeed, + Ipp8u* pDst, + const IppsRSAPublicKeyState* pKey, + IppHashAlgId hashAlg, + Ipp8u* pBuffer)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsRSADecrypt_OAEP,(const Ipp8u* pSrc, + const Ipp8u* pLab, int labLen, + Ipp8u* pDst, int* pDstLen, + const IppsRSAPrivateKeyState* pKey, + IppHashAlgId hashAlg, + Ipp8u* pBuffer)) + +IPPAPI(IppStatus, ippsRSAEncrypt_OAEP_rmf,(const Ipp8u* pSrc, int srcLen, + const Ipp8u* pLabel, int labLen, + const Ipp8u* pSeed, + Ipp8u* pDst, + const IppsRSAPublicKeyState* pKey, + const IppsHashMethod* pMethod, + Ipp8u* pBuffer)) + +IPPAPI(IppStatus, ippsRSADecrypt_OAEP_rmf,(const Ipp8u* pSrc, + const Ipp8u* pLab, int labLen, + Ipp8u* pDst, int* pDstLen, + const IppsRSAPrivateKeyState* pKey, + const IppsHashMethod* pMethod, + Ipp8u* pBuffer)) + +/* encryption scheme: RSAES-PKCS_v1_5 */ + +#define PKCS_DEPRECATED "This algorithm is considered weak due to known attacks on it. \ +The functionality remains in the library, but the implementation will no be longer \ +optimized and no security patches will be applied. A more secure alternative is available: RSA-OAEP" + +IPP_DEPRECATED(PKCS_DEPRECATED) \ +IPPAPI(IppStatus, ippsRSAEncrypt_PKCSv15,(const Ipp8u* pSrc, int srcLen, + const Ipp8u* pRndPS, + Ipp8u* pDst, + const IppsRSAPublicKeyState* pKey, + Ipp8u* pBuffer)) + +IPP_DEPRECATED(PKCS_DEPRECATED) \ +IPPAPI(IppStatus, ippsRSADecrypt_PKCSv15,(const Ipp8u* pSrc, + Ipp8u* pDst, int* pDstLen, + const IppsRSAPrivateKeyState* pKey, + Ipp8u* pBuffer)) + +/* signature scheme : RSA-SSA-PSS */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsRSASign_PSS,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pSalt, int saltLen, + Ipp8u* pSign, + const IppsRSAPrivateKeyState* pPrvKey, + const IppsRSAPublicKeyState* pPubKey, + IppHashAlgId hashAlg, + Ipp8u* pBuffer)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsRSAVerify_PSS,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pSign, + int* pIsValid, + const IppsRSAPublicKeyState* pKey, + IppHashAlgId hashAlg, + Ipp8u* pBuffer)) + +IPPAPI(IppStatus, ippsRSASign_PSS_rmf,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pSalt, int saltLen, + Ipp8u* pSign, + const IppsRSAPrivateKeyState* pPrvKey, + const IppsRSAPublicKeyState* pPubKey, + const IppsHashMethod* pMethod, + Ipp8u* pBuffer)) + +IPPAPI(IppStatus, ippsRSAVerify_PSS_rmf,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pSign, + int* pIsValid, + const IppsRSAPublicKeyState* pKey, + const IppsHashMethod* pMethod, + Ipp8u* pBuffer)) + +/* signature scheme : RSA-SSA-PKCS1-v1_5 */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsRSASign_PKCS1v15,(const Ipp8u* pMsg, int msgLen, + Ipp8u* pSign, + const IppsRSAPrivateKeyState* pPrvKey, + const IppsRSAPublicKeyState* pPubKey, + IppHashAlgId hashAlg, + Ipp8u* pBuffer)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsRSAVerify_PKCS1v15,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pSign, int* pIsValid, + const IppsRSAPublicKeyState* pKey, + IppHashAlgId hashAlg, + Ipp8u* pBuffer)) + +IPPAPI(IppStatus, ippsRSASign_PKCS1v15_rmf,(const Ipp8u* pMsg, int msgLen, + Ipp8u* pSign, + const IppsRSAPrivateKeyState* pPrvKey, + const IppsRSAPublicKeyState* pPubKey, + const IppsHashMethod* pMethod, + Ipp8u* pBuffer)) + +IPPAPI(IppStatus, ippsRSAVerify_PKCS1v15_rmf,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pSign, int* pIsValid, + const IppsRSAPublicKeyState* pKey, + const IppsHashMethod* pMethod, + Ipp8u* pBuffer)) + +/* +// ========================================================= +// DL Cryptography +// ========================================================= +*/ +IPPAPI( const char*, ippsDLGetResultString, (IppDLResult code)) + +/* Initialization */ +IPPAPI(IppStatus, ippsDLPGetSize,(int bitSizeP, int bitSizeR, int* pSize)) +IPPAPI(IppStatus, ippsDLPInit, (int bitSizeP, int bitSizeR, IppsDLPState* pCtx)) + +IPPAPI(IppStatus, ippsDLPPack,(const IppsDLPState* pCtx, Ipp8u* pBuffer)) +IPPAPI(IppStatus, ippsDLPUnpack,(const Ipp8u* pBuffer, IppsDLPState* pCtx)) + +/* Set Up and Retrieve Domain Parameters */ +IPPAPI(IppStatus, ippsDLPSet,(const IppsBigNumState* pP, + const IppsBigNumState* pR, + const IppsBigNumState* pG, + IppsDLPState* pCtx)) +IPPAPI(IppStatus, ippsDLPGet,(IppsBigNumState* pP, + IppsBigNumState* pR, + IppsBigNumState* pG, + IppsDLPState* pCtx)) +IPPAPI(IppStatus, ippsDLPSetDP,(const IppsBigNumState* pDP, IppDLPKeyTag tag, IppsDLPState* pCtx)) +IPPAPI(IppStatus, ippsDLPGetDP,(IppsBigNumState* pDP, IppDLPKeyTag tag, const IppsDLPState* pCtx)) + +/* Key Generation, Validation and Set Up */ +IPPAPI(IppStatus, ippsDLPGenKeyPair,(IppsBigNumState* pPrvKey, IppsBigNumState* pPubKey, + IppsDLPState* pCtx, + IppBitSupplier rndFunc, void* pRndParam)) +IPPAPI(IppStatus, ippsDLPPublicKey, (const IppsBigNumState* pPrvKey, + IppsBigNumState* pPubKey, + IppsDLPState* pCtx)) +IPPAPI(IppStatus, ippsDLPValidateKeyPair,(const IppsBigNumState* pPrvKey, + const IppsBigNumState* pPubKey, + IppDLResult* pResult, + IppsDLPState* pCtx)) + +IPPAPI(IppStatus, ippsDLPSetKeyPair,(const IppsBigNumState* pPrvKey, + const IppsBigNumState* pPubKey, + IppsDLPState* pCtx)) + +/* Signing/Verifying (DSA version) */ +IPPAPI(IppStatus, ippsDLPSignDSA, (const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pPrvKey, + IppsBigNumState* pSignR, IppsBigNumState* pSignS, + IppsDLPState* pCtx)) +IPPAPI(IppStatus, ippsDLPVerifyDSA,(const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pSignR, const IppsBigNumState* pSignS, + IppDLResult* pResult, + IppsDLPState* pCtx)) + +/* Shared Secret Element (DH version) */ +IPPAPI(IppStatus, ippsDLPSharedSecretDH,(const IppsBigNumState* pPrvKeyA, + const IppsBigNumState* pPubKeyB, + IppsBigNumState* pShare, + IppsDLPState* pCtx)) + +/* DSA's parameter Generation and Validation */ +IPPAPI(IppStatus, ippsDLPGenerateDSA,(const IppsBigNumState* pSeedIn, + int nTrials, IppsDLPState* pCtx, + IppsBigNumState* pSeedOut, int* pCounter, + IppBitSupplier rndFunc, void* pRndParam)) +IPPAPI(IppStatus, ippsDLPValidateDSA,(int nTrials, IppDLResult* pResult, IppsDLPState* pCtx, + IppBitSupplier rndFunc, void* pRndParam)) + +/* DH parameter's Generation and Validation */ +IPPAPI(IppStatus, ippsDLPGenerateDH,(const IppsBigNumState* pSeedIn, + int nTrials, IppsDLPState* pCtx, + IppsBigNumState* pSeedOut, int* pCounter, + IppBitSupplier rndFunc, void* pRndParam)) +IPPAPI(IppStatus, ippsDLPValidateDH,(int nTrials, IppDLResult* pResult, IppsDLPState* pCtx, + IppBitSupplier rndFunc, void* pRndParam)) + + +/* +// ========================================================= +// EC Cryptography +// ========================================================= +*/ +IPPAPI( const char*, ippsECCGetResultString, (IppECResult code)) + +/* +// EC over Prime Fields +*/ +/* general EC initialization */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGetSize,(int feBitSize, int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGetSizeStd128r1,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGetSizeStd128r2,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGetSizeStd192r1,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGetSizeStd224r1,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGetSizeStd256r1,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGetSizeStd384r1,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGetSizeStd521r1,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGetSizeStdSM2, (int* pSize)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPInit,(int feBitSize, IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPInitStd128r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPInitStd128r2,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPInitStd192r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPInitStd224r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPInitStd256r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPInitStd384r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPInitStd521r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPInitStdSM2, (IppsECCPState* pEC)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSet,(const IppsBigNumState* pPrime, + const IppsBigNumState* pA, const IppsBigNumState* pB, + const IppsBigNumState* pGX,const IppsBigNumState* pGY,const IppsBigNumState* pOrder, + int cofactor, + IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetStd,(IppECCType flag, IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetStd128r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetStd128r2,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetStd192r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetStd224r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetStd256r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetStd384r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetStd521r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetStdSM2, (IppsECCPState* pEC)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPBindGxyTblStd192r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPBindGxyTblStd224r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPBindGxyTblStd256r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPBindGxyTblStd384r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPBindGxyTblStd521r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPBindGxyTblStdSM2, (IppsECCPState* pEC)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGet,(IppsBigNumState* pPrime, + IppsBigNumState* pA, IppsBigNumState* pB, + IppsBigNumState* pGX,IppsBigNumState* pGY,IppsBigNumState* pOrder, + int* cofactor, + IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGetOrderBitSize,(int* pBitSize, IppsECCPState* pEC)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPValidate,(int nTrials, IppECResult* pResult, IppsECCPState* pEC, + IppBitSupplier rndFunc, void* pRndParam)) + +/* EC Point */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPPointGetSize,(int feBitSize, int* pSize)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPPointInit,(int feBitSize, IppsECCPPointState* pPoint)) + +/* Setup/retrieve point's coordinates */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetPoint,(const IppsBigNumState* pX, const IppsBigNumState* pY, + IppsECCPPointState* pPoint, IppsECCPState* pEC)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetPointAtInfinity,(IppsECCPPointState* pPoint, IppsECCPState* pEC)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGetPoint,(IppsBigNumState* pX, IppsBigNumState* pY, + const IppsECCPPointState* pPoint, IppsECCPState* pEC)) + +/* EC Point Operations */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPCheckPoint,(const IppsECCPPointState* pP, + IppECResult* pResult, IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPComparePoint,(const IppsECCPPointState* pP, const IppsECCPPointState* pQ, + IppECResult* pResult, IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPNegativePoint,(const IppsECCPPointState* pP, + IppsECCPPointState* pR, IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPAddPoint,(const IppsECCPPointState* pP, const IppsECCPPointState* pQ, + IppsECCPPointState* pR, IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPMulPointScalar,(const IppsECCPPointState* pP, const IppsBigNumState* pK, + IppsECCPPointState* pR, IppsECCPState* pEC)) + +/* Key Generation, Setup and Validation */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGenKeyPair,(IppsBigNumState* pPrivate, IppsECCPPointState* pPublic, + IppsECCPState* pEC, + IppBitSupplier rndFunc, void* pRndParam)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPPublicKey,(const IppsBigNumState* pPrivate, + IppsECCPPointState* pPublic, + IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPValidateKeyPair,(const IppsBigNumState* pPrivate, const IppsECCPPointState* pPublic, + IppECResult* pResult, + IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetKeyPair,(const IppsBigNumState* pPrivate, const IppsECCPPointState* pPublic, + IppBool regular, + IppsECCPState* pEC)) + +/* Shared Secret (DH scheme ) */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSharedSecretDH,(const IppsBigNumState* pPrivateA, + const IppsECCPPointState* pPublicB, + IppsBigNumState* pShare, + IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSharedSecretDHC,(const IppsBigNumState* pPrivateA, + const IppsECCPPointState* pPublicB, + IppsBigNumState* pShare, + IppsECCPState* pEC)) + +/* Sign/Verify */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSignDSA,(const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pPrivate, + IppsBigNumState* pSignX, IppsBigNumState* pSignY, + IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPVerifyDSA,(const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pSignX, const IppsBigNumState* pSignY, + IppECResult* pResult, + IppsECCPState* pEC)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSignNR,(const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pPrivate, + IppsBigNumState* pSignX, IppsBigNumState* pSignY, + IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPVerifyNR,(const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pSignX, const IppsBigNumState* pSignY, + IppECResult* pResult, + IppsECCPState* pEC)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSignSM2,(const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pRegPrivate, + IppsBigNumState* pEphPrivate, + IppsBigNumState* pSignR, IppsBigNumState* pSignS, + IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPVerifySM2,(const IppsBigNumState* pMsgDigest, + const IppsECCPPointState* pRegPublic, + const IppsBigNumState* pSignR, const IppsBigNumState* pSignS, + IppECResult* pResult, + IppsECCPState* pEC)) + +/* +// GF over prime and its extension +*/ +IPPAPI(IppStatus, ippsGFpGetSize, (int feBitSize, int* pSize)) +IPPAPI(IppStatus, ippsGFpInitArbitrary,(const IppsBigNumState* pPrime, int primeBitSize, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpInitFixed,(int primeBitSize, const IppsGFpMethod* pGFpMethod, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpInit, (const IppsBigNumState* pPrime, int primeBitSize, const IppsGFpMethod* pGFpMethod, IppsGFpState* pGFp)) +IPPAPI( const IppsGFpMethod*, ippsGFpMethod_p192r1, (void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpMethod_p224r1, (void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpMethod_p256r1, (void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpMethod_p384r1, (void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpMethod_p521r1, (void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpMethod_p256sm2,(void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpMethod_p256bn, (void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpMethod_p256, (void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpMethod_pArb, (void) ) + +IPPAPI(IppStatus, ippsGFpxGetSize,(const IppsGFpState* pGroundGF, int degree, int* pSize)) +IPPAPI(IppStatus, ippsGFpxInit, (const IppsGFpState* pGroundGF, int extDeg, const IppsGFpElement* const ppGroundElm[], int nElm, const IppsGFpMethod* pGFpMethod, IppsGFpState* pGFpx)) +IPPAPI(IppStatus, ippsGFpxInitBinomial,(const IppsGFpState* pGroundGF, int extDeg, const IppsGFpElement* pGroundElm, const IppsGFpMethod* pGFpMethod, IppsGFpState* pGFpx)) +IPPAPI( const IppsGFpMethod*, ippsGFpxMethod_binom2_epid2,(void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpxMethod_binom3_epid2,(void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpxMethod_binom2, (void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpxMethod_binom3, (void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpxMethod_binom, (void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpxMethod_com, (void) ) + +IPPAPI(IppStatus, ippsGFpScratchBufferSize,(int nExponents, int ExpBitSize, const IppsGFpState* pGFp, int* pBufferSize)) + +IPPAPI(IppStatus, ippsGFpElementGetSize,(const IppsGFpState* pGFp, int* pElementSize)) +IPPAPI(IppStatus, ippsGFpElementInit, (const Ipp32u* pA, int lenA, IppsGFpElement* pR, IppsGFpState* pGFp)) + +IPPAPI(IppStatus, ippsGFpSetElement, (const Ipp32u* pA, int lenA, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpSetElementRegular,(const IppsBigNumState* pBN, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpSetElementOctString,(const Ipp8u* pStr, int strSize, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpSetElementRandom,(IppsGFpElement* pR, IppsGFpState* pGFp, IppBitSupplier rndFunc, void* pRndParam)) +IPPAPI(IppStatus, ippsGFpSetElementHash,(const Ipp8u* pMsg, int msgLen, IppsGFpElement* pElm, IppsGFpState* pGFp, IppHashAlgId hashID)) +IPPAPI(IppStatus, ippsGFpSetElementHash_rmf,(const Ipp8u* pMsg, int msgLen, IppsGFpElement* pElm, IppsGFpState* pGFp, const IppsHashMethod* pMethod)) +IPPAPI(IppStatus, ippsGFpCpyElement,(const IppsGFpElement* pA, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpGetElement,(const IppsGFpElement* pA, Ipp32u* pDataA, int lenA, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpGetElementOctString,(const IppsGFpElement* pA, Ipp8u* pStr, int strSize, IppsGFpState* pGFp)) + +IPPAPI(IppStatus, ippsGFpCmpElement,(const IppsGFpElement* pA, const IppsGFpElement* pB, int* pResult, const IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpIsZeroElement,(const IppsGFpElement* pA, int* pResult, const IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpIsUnityElement,(const IppsGFpElement* pA, int* pResult, const IppsGFpState* pGFp)) + +IPPAPI(IppStatus, ippsGFpConj,(const IppsGFpElement* pA, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpNeg, (const IppsGFpElement* pA, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpInv, (const IppsGFpElement* pA, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpSqrt,(const IppsGFpElement* pA, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpSqr, (const IppsGFpElement* pA, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpAdd, (const IppsGFpElement* pA, const IppsGFpElement* pB, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpSub, (const IppsGFpElement* pA, const IppsGFpElement* pB, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpMul, (const IppsGFpElement* pA, const IppsGFpElement* pB, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpExp, (const IppsGFpElement* pA, const IppsBigNumState* pE, IppsGFpElement* pR, IppsGFpState* pGFp, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpMultiExp,(const IppsGFpElement* const ppElmA[], const IppsBigNumState* const ppE[], int nItems, IppsGFpElement* pR, IppsGFpState* pGFp, Ipp8u* pScratchBuffer)) + +IPPAPI(IppStatus, ippsGFpAdd_PE,(const IppsGFpElement* pA, const IppsGFpElement* pParentB, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpSub_PE,(const IppsGFpElement* pA, const IppsGFpElement* pParentB, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpMul_PE,(const IppsGFpElement* pA, const IppsGFpElement* pParentB, IppsGFpElement* pR, IppsGFpState* pGFp)) + +IPPAPI(IppStatus, ippsGFpGetInfo, (IppsGFpInfo* pInfo, const IppsGFpState* pGFp)) + +/* ================== */ +IPPAPI(IppStatus, ippsGFpECGetSize,(const IppsGFpState* pGFp, int* pSize)) +IPPAPI(IppStatus, ippsGFpECInit, (const IppsGFpState* pGFp, + const IppsGFpElement* pA, const IppsGFpElement* pB, + IppsGFpECState* pEC)) + +IPPAPI(IppStatus, ippsGFpECSet,(const IppsGFpElement* pA, const IppsGFpElement* pB, + IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECSetSubgroup,(const IppsGFpElement* pX, const IppsGFpElement* pY, + const IppsBigNumState* pOrder, + const IppsBigNumState* pCofactor, + IppsGFpECState* pEC)) + +IPPAPI(IppStatus, ippsGFpECInitStd128r1,(const IppsGFpState* pGFp, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECInitStd128r2,(const IppsGFpState* pGFp, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECInitStd192r1,(const IppsGFpState* pGFp, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECInitStd224r1,(const IppsGFpState* pGFp, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECInitStd256r1,(const IppsGFpState* pGFp, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECInitStd384r1,(const IppsGFpState* pGFp, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECInitStd521r1,(const IppsGFpState* pGFp, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECInitStdSM2, (const IppsGFpState* pGFp, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECInitStdBN256,(const IppsGFpState* pGFp, IppsGFpECState* pEC)) + +IPPAPI(IppStatus, ippsGFpECBindGxyTblStd192r1,(IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECBindGxyTblStd224r1,(IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECBindGxyTblStd256r1,(IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECBindGxyTblStd384r1,(IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECBindGxyTblStd521r1,(IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECBindGxyTblStdSM2, (IppsGFpECState* pEC)) + +IPPAPI(IppStatus, ippsGFpECGet,(IppsGFpState** const ppGFp, + IppsGFpElement* pA, IppsGFpElement* pB, + const IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECGetSubgroup,(IppsGFpState** const ppGFp, + IppsGFpElement* pX, IppsGFpElement* pY, + IppsBigNumState* pOrder,IppsBigNumState* pCofactor, + const IppsGFpECState* pEC)) + +IPPAPI(IppStatus, ippsGFpECScratchBufferSize,(int nScalars, const IppsGFpECState* pEC, int* pBufferSize)) + +IPPAPI(IppStatus, ippsGFpECVerify,(IppECResult* pResult, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) + +IPPAPI(IppStatus, ippsGFpECPointGetSize,(const IppsGFpECState* pEC, int* pSize)) +IPPAPI(IppStatus, ippsGFpECPointInit, (const IppsGFpElement* pX, const IppsGFpElement* pY, IppsGFpECPoint* pPoint, IppsGFpECState* pEC)) + +IPPAPI(IppStatus, ippsGFpECSetPointAtInfinity,(IppsGFpECPoint* pPoint, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECSetPoint,(const IppsGFpElement* pX, const IppsGFpElement* pY, IppsGFpECPoint* pPoint, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECSetPointRegular,(const IppsBigNumState* pX, const IppsBigNumState* pY, IppsGFpECPoint* pPoint, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECSetPointRandom,(IppsGFpECPoint* pPoint, IppsGFpECState* pEC, IppBitSupplier rndFunc, void* pRndParam, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECMakePoint,(const IppsGFpElement* pX, IppsGFpECPoint* pPoint, IppsGFpECState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsGFpECSetPointHash,(Ipp32u hdr, const Ipp8u* pMsg, int msgLen, IppsGFpECPoint* pPoint, IppsGFpECState* pEC, IppHashAlgId hashID, Ipp8u* pScratchBuffer)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsGFpECSetPointHashBackCompatible,(Ipp32u hdr, const Ipp8u* pMsg, int msgLen, IppsGFpECPoint* pPoint, IppsGFpECState* pEC, IppHashAlgId hashID, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECSetPointHash_rmf,(Ipp32u hdr, const Ipp8u* pMsg, int msgLen, IppsGFpECPoint* pPoint, IppsGFpECState* pEC, const IppsHashMethod* pMethod, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECSetPointHashBackCompatible_rmf,(Ipp32u hdr, const Ipp8u* pMsg, int msgLen, IppsGFpECPoint* pPoint, IppsGFpECState* pEC, const IppsHashMethod* pMethod, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECGetPoint,(const IppsGFpECPoint* pPoint, IppsGFpElement* pX, IppsGFpElement* pY, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECGetPointRegular,(const IppsGFpECPoint* pPoint, IppsBigNumState* pX, IppsBigNumState* pY, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECSetPointOctString,(const Ipp8u* pStr, int strLen, IppsGFpECPoint* pPoint, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECGetPointOctString,(const IppsGFpECPoint* pPoint, Ipp8u* pStr, int strLen, IppsGFpECState* pEC)) + +IPPAPI(IppStatus, ippsGFpECTstPoint,(const IppsGFpECPoint* pP, IppECResult* pResult, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECTstPointInSubgroup,(const IppsGFpECPoint* pP, IppECResult* pResult, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECCpyPoint,(const IppsGFpECPoint* pA, IppsGFpECPoint* pR, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECCmpPoint,(const IppsGFpECPoint* pP, const IppsGFpECPoint* pQ, IppECResult* pResult, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECNegPoint,(const IppsGFpECPoint* pP, IppsGFpECPoint* pR, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECAddPoint,(const IppsGFpECPoint* pP, const IppsGFpECPoint* pQ, IppsGFpECPoint* pR, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECMulPoint,(const IppsGFpECPoint* pP, const IppsBigNumState* pN, IppsGFpECPoint* pR, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) + +/* keys */ +IPPAPI(IppStatus, ippsGFpECPrivateKey,(IppsBigNumState* pPrivate, IppsGFpECState* pEC, + IppBitSupplier rndFunc, void* pRndParam)) +IPPAPI(IppStatus, ippsGFpECPublicKey,(const IppsBigNumState* pPrivate, IppsGFpECPoint* pPublic, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECTstKeyPair,(const IppsBigNumState* pPrivate, const IppsGFpECPoint* pPublic, IppECResult* pResult, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) + +/* DH shared secret */ +IPPAPI(IppStatus, ippsGFpECSharedSecretDH,(const IppsBigNumState* pPrivateA, const IppsGFpECPoint* pPublicB, + IppsBigNumState* pShare, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECSharedSecretDHC,(const IppsBigNumState* pPrivateA, + const IppsGFpECPoint* pPublicB, + IppsBigNumState* pShare, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) + +/* sign generation/verification of DSA, NR, SM2 */ +IPPAPI(IppStatus, ippsGFpECMessageRepresentationSM2, (IppsBigNumState* pMsgDigest, + const Ipp8u* pMsg, int msgLen, + const Ipp8u* pUserID, int userIDLen, + const IppsGFpECPoint* pRegPublic, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECSignDSA, (const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pRegPrivate, + IppsBigNumState* pEphPrivate, + IppsBigNumState* pSignR, IppsBigNumState* pSignS, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECVerifyDSA, (const IppsBigNumState* pMsgDigest, + const IppsGFpECPoint* pRegPublic, + const IppsBigNumState* pSignR, const IppsBigNumState* pSignS, + IppECResult* pResult, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) + +IPPAPI(IppStatus, ippsGFpECSignNR, (const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pRegPrivate, + IppsBigNumState* pEphPrivate, + IppsBigNumState* pSignR, IppsBigNumState* pSignS, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECVerifyNR, (const IppsBigNumState* pMsgDigest, + const IppsGFpECPoint* pRegPublic, + const IppsBigNumState* pSignR, const IppsBigNumState* pSignS, + IppECResult* pResult, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) + +IPPAPI(IppStatus, ippsGFpECSignSM2, (const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pRegPrivate, + IppsBigNumState* pEphPrivate, + IppsBigNumState* pSignR, IppsBigNumState* pSignS, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECVerifySM2, (const IppsBigNumState* pMsgDigest, + const IppsGFpECPoint* pRegPublic, + const IppsBigNumState* pSignR, const IppsBigNumState* pSignS, + IppECResult* pResult, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) + +/* SM2 UserIDHash */ +IPPAPI(IppStatus, ippsGFpECUserIDHashSM2, (Ipp8u* pZaDigest, + const Ipp8u* pUserID, int userIDLen, + const IppsGFpECPoint* pPublicKey, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) + + +/* SM2 Key Exchange */ +IPPAPI(IppStatus, ippsGFpECKeyExchangeSM2_GetSize, (const IppsGFpECState* pEC, int* pSize)) +IPPAPI(IppStatus, ippsGFpECKeyExchangeSM2_Init, (IppsGFpECKeyExchangeSM2State* pKE, IppsKeyExchangeRoleSM2 role, IppsGFpECState* pEC)) + +IPPAPI(IppStatus, ippsGFpECKeyExchangeSM2_Setup, (const Ipp8u pZSelf[IPP_SM3_DIGEST_BYTESIZE], + const Ipp8u pZPeer[IPP_SM3_DIGEST_BYTESIZE], + const IppsGFpECPoint *pPublicKeySelf, + const IppsGFpECPoint *pPublicKeyPeer, + const IppsGFpECPoint *pEphPublicKeySelf, + const IppsGFpECPoint *pEphPublicKeyPeer, + IppsGFpECKeyExchangeSM2State *pKE)) + +IPPAPI(IppStatus, ippsGFpECKeyExchangeSM2_SharedKey, (Ipp8u* pSharedKey, int sharedKeySize, + Ipp8u* pSSelf, + const IppsBigNumState* pPrvKey, + IppsBigNumState* pEphPrvKey, + IppsGFpECKeyExchangeSM2State *pKE, Ipp8u* pScratchBuffer)) + +IPPAPI(IppStatus, ippsGFpECKeyExchangeSM2_Confirm, (const Ipp8u pSPeer[IPP_SM3_DIGEST_BYTESIZE], + int* pStatus, + IppsGFpECKeyExchangeSM2State* pKE)) + +/* SM2 Encryption/Decryption */ +IPPAPI(IppStatus, ippsGFpECGetInfo_GF,(IppsGFpInfo* pInfo, const IppsGFpECState* pEC)) + +IPPAPI(IppStatus, ippsGFpECESGetSize_SM2, (const IppsGFpECState* pEC, int* pSize)) +IPPAPI(IppStatus, ippsGFpECESInit_SM2, (IppsGFpECState* pEC, + IppsECESState_SM2* pState, int avaliableCtxSize)) +IPPAPI(IppStatus, ippsGFpECESSetKey_SM2, (const IppsBigNumState* pPrivate, + const IppsGFpECPoint* pPublic, + IppsECESState_SM2* pState, + IppsGFpECState* pEC, + Ipp8u* pEcScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECESStart_SM2, (IppsECESState_SM2* pState)) +IPPAPI(IppStatus, ippsGFpECESEncrypt_SM2, (const Ipp8u* pInput, Ipp8u* pOutput, + int dataLen, IppsECESState_SM2* pState)) +IPPAPI(IppStatus, ippsGFpECESDecrypt_SM2, (const Ipp8u* pInput, Ipp8u* pOutput, + int dataLen, IppsECESState_SM2* pState)) +IPPAPI(IppStatus, ippsGFpECESFinal_SM2, (Ipp8u* pTag, int tagLen, IppsECESState_SM2* pState)) +IPPAPI(IppStatus, ippsGFpECESGetBuffersSize_SM2, (int* pPublicKeySize, + int* pMaximumTagSize, const IppsECESState_SM2* pState)) + +/* SM2 Encryption/Decryption Extended API */ +/* Encryption */ +IPPAPI(IppStatus, ippsGFpECEncryptSM2_Ext_EncMsgSize, (const IppsGFpECState *pEC, int ptMsgSize, int *pSize)) + +IPPAPI(IppStatus, ippsGFpECEncryptSM2_Ext, (Ipp8u *pOut, int maxOutLen, + int *pOutSize, + const Ipp8u *pInp, int inpLen, + const IppsGFpECPoint *pPublicKey, + IppsGFpECPoint *pEhpPublicKey, IppsBigNumState *pEphPrvKey, + IppsGFpECState *pEC, Ipp8u *pScratchBuffer)) + +/* Decryption */ +IPPAPI(IppStatus, ippsGFpECDecryptSM2_Ext_DecMsgSize, (const IppsGFpECState *pEC, int ctMsgSize, int *pSize)) + +IPPAPI(IppStatus, ippsGFpECDecryptSM2_Ext, (Ipp8u *pOut, int maxOutLen, + int *pOutSize, + const Ipp8u *pInp, int inpLen, + const IppsBigNumState *pPrvKey, + IppsGFpECState *pEC, Ipp8u *pScratchBuffer)) + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && !defined(__INTEL_LLVM_COMPILER) +#pragma warning(pop) +#endif +#ifdef __cplusplus +} +#endif + + +#endif /* IPPCP_H__ */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/ippcpdefs.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/ippcpdefs.h new file mode 100644 index 0000000..50417c6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/ippcpdefs.h @@ -0,0 +1,860 @@ +/******************************************************************************* +* Copyright (C) 2012 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) +// +// Purpose: Basic Types and Macro Definitions +// +*/ + + +#ifndef IPPBASE_H__ +#define IPPBASE_H__ + +#ifdef __cplusplus +extern "C" { +#endif +#if defined (_WIN64) +#define INTEL_PLATFORM "intel64/" +#elif defined (_WIN32) +#define INTEL_PLATFORM "ia32/" +#endif + +#if !defined( IPPAPI ) + + /* Specify explicit calling convention for public functions */ + #if defined( IPP_W32DLL ) && (defined( _WIN32 ) || defined( _WIN64 )) + #if defined( _MSC_VER ) || defined( __ICL ) + #define IPPAPI( type,name,arg ) \ + __declspec(dllimport) type IPP_CALL name arg; + #else + #define IPPAPI( type,name,arg ) type IPP_CALL name arg; + #endif + #else + #define IPPAPI( type,name,arg ) type IPP_CALL name arg; + #endif + +#endif + +/* icc 2021 supports short float data type, icx supports _Float16 data type */ +#define _FLOAT_16 2 +#define _SHORT_FLOAT 1 +#define _NO_FLOAT_16 0 +#if defined(__INTEL_LLVM_COMPILER) && defined(__AVX512FP16__) +# define COMPILER_SUPPORT_SHORT_FLOAT _FLOAT_16 +#else +# if defined(__INTEL_COMPILER) +# if(__INTEL_COMPILER >= 2021) +# define COMPILER_SUPPORT_SHORT_FLOAT _SHORT_FLOAT +# endif +# endif +#endif +#if !(defined(COMPILER_SUPPORT_SHORT_FLOAT)) +# define COMPILER_SUPPORT_SHORT_FLOAT _NO_FLOAT_16 +#endif + +#if !defined(_NO_IPP_DEPRECATED) + #if (defined( __ICL ) || defined( __ECL ) || defined(_MSC_VER)) && !defined( _PCS ) && !defined( _PCS_GENSTUBS ) + #if( __INTEL_COMPILER >= 1100 ) /* icl 11.0 supports additional comment */ + #if( _MSC_VER >= 1400 ) + #define IPP_DEPRECATED( comment ) __declspec( deprecated ( comment )) + #else + #pragma message ("your icl version supports additional comment for deprecated functions but it can't be displayed") + #pragma message ("because internal _MSC_VER macro variable setting requires compatibility with MSVC7.1") + #pragma message ("use -Qvc8 switch for icl command line to see these additional comments") + #define IPP_DEPRECATED( comment ) __declspec( deprecated ) + #endif + #elif( _MSC_FULL_VER >= 140050727 )&&( !defined( __INTEL_COMPILER ))&&( !defined(__INTEL_LLVM_COMPILER)) /* VS2005 supports additional comment */ + #define IPP_DEPRECATED( comment ) __declspec( deprecated ( comment )) + #elif( _MSC_VER <= 1200 )&&( !defined( __INTEL_COMPILER ))&&( !defined(__INTEL_LLVM_COMPILER)) /* VS 6 doesn't support deprecation */ + #define IPP_DEPRECATED( comment ) + #else + #define IPP_DEPRECATED( comment ) __declspec( deprecated ) + #endif + #elif (defined(__ICC) || defined(__ECC) || defined( __GNUC__ )) && !defined( _PCS ) && !defined( _PCS_GENSTUBS ) + #if defined( __GNUC__ ) + #if (__GNUC__ * 100 + __GNUC_MINOR__) >= 405 + #define IPP_DEPRECATED( message ) __attribute__(( deprecated( message ))) + #else + #define IPP_DEPRECATED( message ) __attribute__(( deprecated )) + #endif + #else + #define IPP_DEPRECATED( comment ) __attribute__(( deprecated )) + #endif + #else + #define IPP_DEPRECATED( comment ) + #endif +#else + #define IPP_DEPRECATED( comment ) +#endif + +#if (defined( __ICL ) || defined( __ECL ) || defined(_MSC_VER)) + #if !defined( IPP_NO_DEFAULT_LIB ) + #if ((defined( _IPP_SEQUENTIAL_DYNAMIC ) && !defined( _IPP_SEQUENTIAL_STATIC )) || \ + (!defined( _IPP_SEQUENTIAL_DYNAMIC ) && defined( _IPP_SEQUENTIAL_STATIC ))) + #elif (!defined( _IPP_SEQUENTIAL_DYNAMIC ) && !defined( _IPP_SEQUENTIAL_STATIC )) + #define IPP_NO_DEFAULT_LIB + #else + #error Illegal combination of _IPP_SEQUENTIAL_DYNAMIC/_IPP_SEQUENTIAL_STATIC, only one definition can be defined + #endif + #endif +#else + #define IPP_NO_DEFAULT_LIB + #if (defined(_IPP_SEQUENTIAL_DYNAMIC) || defined(_IPP_SEQUENTIAL_STATIC)) + #pragma message ("defines _IPP_SEQUENTIAL_DYNAMIC/_IPP_SEQUENTIAL_STATIC do not have any effect in current configuration") + #endif +#endif + + +#if defined (_MSC_VER) + #define IPP_CDECL __cdecl +#elif (defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || defined (__GNUC__ ) || defined (__clang__)) && defined (_ARCH_IA32) + #define IPP_CDECL __attribute((cdecl)) +#else + #define IPP_CDECL +#endif + +#if defined( _WIN32 ) || defined( _WIN64 ) + #define IPP_STDCALL __stdcall + #define IPP_CALL IPP_STDCALL + #define IPP_INT64 __int64 + #define IPP_UINT64 unsigned __int64 +#else + #define IPP_STDCALL + #define IPP_CALL IPP_CDECL + #define IPP_INT64 long long + #define IPP_UINT64 unsigned long long +#endif + +#define IPP_COUNT_OF( obj ) (sizeof(obj)/sizeof(obj[0])) + +#define IPP_PI ( 3.14159265358979323846 ) /* ANSI C does not support M_PI */ +#define IPP_2PI ( 6.28318530717958647692 ) /* 2*pi */ +#define IPP_PI2 ( 1.57079632679489661923 ) /* pi/2 */ +#define IPP_PI4 ( 0.78539816339744830961 ) /* pi/4 */ +#define IPP_PI180 ( 0.01745329251994329577 ) /* pi/180 */ +#define IPP_RPI ( 0.31830988618379067154 ) /* 1/pi */ +#define IPP_SQRT2 ( 1.41421356237309504880 ) /* sqrt(2) */ +#define IPP_SQRT3 ( 1.73205080756887729353 ) /* sqrt(3) */ +#define IPP_LN2 ( 0.69314718055994530942 ) /* ln(2) */ +#define IPP_LN3 ( 1.09861228866810969139 ) /* ln(3) */ +#define IPP_E ( 2.71828182845904523536 ) /* e */ +#define IPP_RE ( 0.36787944117144232159 ) /* 1/e */ +#define IPP_EPS23 ( 1.19209289e-07f ) +#define IPP_EPS52 ( 2.2204460492503131e-016 ) + +#define IPP_MAX_8U ( 0xFF ) +#define IPP_MAX_16U ( 0xFFFF ) +#define IPP_MAX_32U ( 0xFFFFFFFF ) +#define IPP_MIN_8U ( 0 ) +#define IPP_MIN_16U ( 0 ) +#define IPP_MIN_32U ( 0 ) +#define IPP_MIN_8S (-128 ) +#define IPP_MAX_8S ( 127 ) +#define IPP_MIN_16S (-32768 ) +#define IPP_MAX_16S ( 32767 ) +#define IPP_MIN_32S (-2147483647 - 1 ) +#define IPP_MAX_32S ( 2147483647 ) +#define IPP_MIN_64U ( 0 ) + +#if defined( _WIN32 ) || defined ( _WIN64 ) + #define IPP_MAX_64S ( 9223372036854775807i64 ) + #define IPP_MIN_64S (-9223372036854775807i64 - 1 ) + #define IPP_MAX_64U ( 0xffffffffffffffffL ) /* 18446744073709551615 */ +#else + #define IPP_MAX_64S ( 9223372036854775807LL ) + #define IPP_MIN_64S (-9223372036854775807LL - 1 ) + #define IPP_MAX_64U ( 0xffffffffffffffffLL ) /* 18446744073709551615 */ +#endif + +#define IPP_MINABS_32F ( 1.175494351e-38f ) +#define IPP_MAXABS_32F ( 3.402823466e+38f ) +#define IPP_EPS_32F ( 1.192092890e-07f ) +#define IPP_MINABS_64F ( 2.2250738585072014e-308 ) +#define IPP_MAXABS_64F ( 1.7976931348623158e+308 ) +#define IPP_EPS_64F ( 2.2204460492503131e-016 ) + +#define IPP_MAX( a, b ) ( ((a) > (b)) ? (a) : (b) ) +#define IPP_MIN( a, b ) ( ((a) < (b)) ? (a) : (b) ) + +#define IPP_ABS( a ) ( ((a) < 0) ? (-(a)) : (a) ) + +typedef struct { + int major; /* e.g. 1 */ + int minor; /* e.g. 2 */ + int majorBuild; /* e.g. 3 */ + unsigned int revision; /* e.g. 0xf6f5e5bc */ + char targetCpu[4]; /* corresponding to Intel® processor */ + const char* Name; /* e.g. "ippsw7" */ + const char* Version; /* e.g. "v1.2 Beta" */ + const char* BuildDate; /* e.g. "Jul 20 99" */ +} IppLibraryVersion; + +typedef unsigned char Ipp8u; +typedef unsigned short Ipp16u; +typedef unsigned int Ipp32u; +typedef signed char Ipp8s; +typedef signed short Ipp16s; +typedef signed int Ipp32s; +typedef float Ipp32f; +typedef IPP_INT64 Ipp64s; +typedef IPP_UINT64 Ipp64u; +typedef double Ipp64f; + +#if (COMPILER_SUPPORT_SHORT_FLOAT == _FLOAT_16) + typedef _Float16 Ipp16f; +#endif +#if (COMPILER_SUPPORT_SHORT_FLOAT == _SHORT_FLOAT) + typedef short float Ipp16f; +#endif +#if (COMPILER_SUPPORT_SHORT_FLOAT == _NO_FLOAT_16) + typedef Ipp16s Ipp16f; +#endif + +typedef struct { + Ipp8s re; + Ipp8s im; +} Ipp8sc; + +typedef struct { + Ipp16s re; + Ipp16s im; +} Ipp16sc; + +typedef struct { + Ipp16u re; + Ipp16u im; +} Ipp16uc; + +typedef struct { + Ipp32s re; + Ipp32s im; +} Ipp32sc; + +typedef struct { + Ipp32f re; + Ipp32f im; +} Ipp32fc; + +typedef struct { + Ipp64s re; + Ipp64s im; +} Ipp64sc; + +typedef struct { + Ipp64f re; + Ipp64f im; +} Ipp64fc; + +typedef struct { + Ipp16f re; + Ipp16f im; +} Ipp16fc; + +typedef enum { + ippUndef = -1, + ipp1u = 0, + ipp8u = 1, + ipp8uc = 2, + ipp8s = 3, + ipp8sc = 4, + ipp16u = 5, + ipp16uc = 6, + ipp16s = 7, + ipp16sc = 8, + ipp32u = 9, + ipp32uc = 10, + ipp32s = 11, + ipp32sc = 12, + ipp32f = 13, + ipp32fc = 14, + ipp64u = 15, + ipp64uc = 16, + ipp64s = 17, + ipp64sc = 18, + ipp64f = 19, + ipp64fc = 20, + ipp16fc = 21 /* This is necessary for TS */ +} IppDataType; + +typedef enum { + ippFalse = 0, + ippTrue = 1 +} IppBool; + +#ifdef __cplusplus +} +#endif + +#endif /* IPPBASE_H__ */ + +#ifndef IPP_CPU_FEATURES__ +#define IPP_CPU_FEATURES__ + +#define ippCPUID_MMX 0x00000001 /* Intel® architecture with MMX(TM) technology supported */ +#define ippCPUID_SSE 0x00000002 /* Intel® Streaming SIMD Extensions (Intel® SSE) instruction set */ +#define ippCPUID_SSE2 0x00000004 /* Intel® Streaming SIMD Extensions 2 (Intel® SSE2) instruction set */ +#define ippCPUID_SSE3 0x00000008 /* Intel® Streaming SIMD Extensions 3 (Intel® SSE3) instruction set */ +#define ippCPUID_SSSE3 0x00000010 /* Supplemental Streaming SIMD Extensions 3 (SSSE3) instruction set */ +#define ippCPUID_MOVBE 0x00000020 /* Intel® instruction MOVBE */ +#define ippCPUID_SSE41 0x00000040 /* Intel® Streaming SIMD Extensions 4.1 (Intel® SSE4.1) instruction set */ +#define ippCPUID_SSE42 0x00000080 /* Intel® Streaming SIMD Extensions 4.2 (Intel® SSE4.2) instruction set */ +#define ippCPUID_AVX 0x00000100 /* Intel® Advanced Vector Extensions instruction set */ +#define ippAVX_ENABLEDBYOS 0x00000200 /* Intel® Advanced Vector Extensions instruction set is supported by OS */ +#define ippCPUID_AES 0x00000400 /* */ +#define ippCPUID_CLMUL 0x00000800 /* Intel® instruction PCLMULQDQ */ +#define ippCPUID_ABR 0x00001000 /* Reserved */ +#define ippCPUID_RDRAND 0x00002000 /* Intel® instruction RDRAND */ +#define ippCPUID_F16C 0x00004000 /* Intel® instruction F16C */ +#define ippCPUID_AVX2 0x00008000 /* Intel® Advanced Vector Extensions 2 */ +#define ippCPUID_ADCOX 0x00010000 /* Intel® instructions ADOX/ADCX */ +#define ippCPUID_RDSEED 0x00020000 /* Intel® instruction RDSEED */ +#define ippCPUID_PREFETCHW 0x00040000 /* Intel® instruction PREFETCHW */ +#define ippCPUID_SHA 0x00080000 /* Intel® Secure Hash Algorithm Extensions */ +#define ippCPUID_AVX512F 0x00100000 /* Intel® Advanced Vector Extensions 512 Foundation instruction set */ +#define ippCPUID_AVX512CD 0x00200000 /* Intel® Advanced Vector Extensions 512 CD instruction set */ +#define ippCPUID_AVX512ER 0x00400000 /* Intel® Advanced Vector Extensions 512 ER instruction set */ +#define ippCPUID_AVX512PF 0x00800000 /* Intel® Advanced Vector Extensions 512 PF instruction set */ +#define ippCPUID_AVX512BW 0x01000000 /* Intel® Advanced Vector Extensions 512 BW instruction set */ +#define ippCPUID_AVX512DQ 0x02000000 /* Intel® Advanced Vector Extensions 512 DQ instruction set */ +#define ippCPUID_AVX512VL 0x04000000 /* Intel® Advanced Vector Extensions 512 VL instruction set */ +#define ippCPUID_AVX512VBMI 0x08000000 /* Intel® Advanced Vector Extensions 512 Bit Manipulation instructions */ +#define ippCPUID_MPX 0x10000000 /* Intel® Memory Protection Extensions */ +#define ippCPUID_AVX512_4FMADDPS 0x20000000 /* Intel® Advanced Vector Extensions 512 DL floating-point single precision */ +#define ippCPUID_AVX512_4VNNIW 0x40000000 /* Intel® Advanced Vector Extensions 512 DL enhanced word variable precision */ +#define ippCPUID_KNC 0x80000000 /* Intel® Xeon® Phi(TM) Coprocessor */ +#if defined( _WIN32 ) || defined ( _WIN64 ) + #define INT64_SUFFIX(name) name##L +#else + #define INT64_SUFFIX(name) name##LL +#endif + #define ippCPUID_AVX512IFMA INT64_SUFFIX(0x100000000) /* Intel® Advanced Vector Extensions 512 IFMA (PMADD52) instruction set */ + #define ippCPUID_NOCHECK INT64_SUFFIX(0x8000000000000000) /* Force ippSetCpuFeatures to set CPU features without check */ + #define ippCPUID_GETINFO_A INT64_SUFFIX(0x616f666e69746567) /* Force ippGetCpuFeatures to work as cpuid instruction */ + #define ippAVX512_ENABLEDBYOS INT64_SUFFIX(0x200000000) /* Intel® Advanced Vector Extensions 512 is supported by OS */ + + #define ippCPUID_AVX512GFNI INT64_SUFFIX(0x400000000) /* */ + #define ippCPUID_AVX512VAES INT64_SUFFIX(0x800000000) /* */ + #define ippCPUID_AVX512VCLMUL INT64_SUFFIX(0x1000000000) /* */ + #define ippCPUID_AVX512VBMI2 INT64_SUFFIX(0x2000000000) /* Intel® Advanced Vector Extensions 512 Bit Manipulation instructions 2 */ + #define ippCPUID_AVX512_FP16 INT64_SUFFIX(0x1000000000) /* Intel(R) Advanced Vector Extensions 512 16-bit floating point (FP16) instruction set */ + + #define ippCPUID_AVX2VAES INT64_SUFFIX(0x4000000000) /* Intel® Advanced Vector Extensions 256 Bit Vector AES instructions */ + #define ippCPUID_AVX2VCLMUL INT64_SUFFIX(0x8000000000) /* Intel® instruction VPCLMULQDQ */ + +#endif /* IPP_CPU_FEATURES__ */ + +/* Macros are necessary to build custom Intel® IPP Cryptography static 1cpu library (enable specific features at compile-time) */ +#if (!defined(_MERGED_BLD) && defined(IPPCP_CUSTOM_BUILD)) + +#ifndef IPP_CUSTOM_CPU_FEATURES__ +#define IPP_CUSTOM_CPU_FEATURES__ + +#ifndef IPPCP_AES_ON +#define IPPCP_AES_ON (0) +#endif +#ifndef IPPCP_CLMUL_ON +#define IPPCP_CLMUL_ON (0) +#endif +#ifndef IPPCP_VAES_ON +#define IPPCP_VAES_ON (0) +#endif +#ifndef IPPCP_VCLMUL_ON +#define IPPCP_VCLMUL_ON (0) +#endif +#define IPP_CUSTOM_ENABLED_FEATURES (ippCPUID_AES*IPPCP_AES_ON | ippCPUID_CLMUL*IPPCP_CLMUL_ON | ippCPUID_AVX512VAES*IPPCP_VAES_ON | ippCPUID_AVX512VCLMUL*IPPCP_VCLMUL_ON) + +#endif /* IPP_CUSTOM_CPU_FEATURES__ */ +#endif /* !defined(_MERGED_BLD) && defined(IPPCP_CUSTOM_BUILD) */ + +#ifndef IPPSTATUS_H__ +#define IPPSTATUS_H__ + +#ifdef __cplusplus +extern "C" { +#endif +typedef signed int IppStatus; + + /* start of common with ippCrypto part - any changes MUST be done in both repositories - IPP & ippCrypto */ +#define ippStsCpuNotSupportedErr -9999 /* The target CPU is not supported. */ +#define ippStsUnknownStatusCodeErr -216 /* Unknown status code. */ +#define ippStsLoadDynErr -221 /* Error when loading the dynamic library. */ +#define ippStsLengthErr -15 /* Incorrect value for string length. */ +#define ippStsNotSupportedModeErr -14 /* The requested mode is currently not supported. */ +#define ippStsContextMatchErr -13 /* Context parameter does not match the operation. */ +#define ippStsScaleRangeErr -12 /* Scale bounds are out of range. */ +#define ippStsOutOfRangeErr -11 /* Argument is out of range, or point is outside the image. */ +#define ippStsDivByZeroErr -10 /* An attempt to divide by zero. */ +#define ippStsMemAllocErr -9 /* Memory allocated for the operation is not enough. */ +#define ippStsNullPtrErr -8 /* Null pointer error. */ +#define ippStsRangeErr -7 /* Incorrect values for bounds: the lower bound is greater than the upper bound. */ +#define ippStsSizeErr -6 /* Incorrect value for data size. */ +#define ippStsBadArgErr -5 /* Incorrect arg/param of the function. */ +#define ippStsNoMemErr -4 /* Not enough memory for the operation. */ +#define ippStsErr -2 /* Unknown/unspecified error */ + /* no errors */ +#define ippStsNoErr 0 /* No errors. */ + /* warnings */ +#define ippStsNoOperation 1 /* No operation has been executed. */ +#define ippStsDivByZero 2 /* Zero value(s) for the divisor in the Div function. */ +#define ippStsWaterfall 43 /* Cannot load required library, waterfall is used. */ +#define ippStsFeaturesCombination 51 /* Wrong combination of features. */ + /* end of common with ippCrypto part */ + +#ifdef __cplusplus +} +#endif + +#endif /* IPPSTATUS_H__ */ + + /* ippCrypto specific statuses - any changes MUST be done in both repositories - IPP & ippCrypto */ +#define ippStsInvalidPoint -1017 /* ippStsInvalidPoint ECC: Invalid point (out of EC).*/ +#define ippStsQuadraticNonResidueErr -1016 /* SQRT operation on quadratic non-residue value. */ +#define ippStsPointAtInfinity -1015 /* Point at infinity is detected. */ +#define ippStsOFBSizeErr -1014 /* Incorrect value for crypto OFB block size. */ +#define ippStsIncompleteContextErr -1013 /* Crypto: set up of context is not complete. */ +#define ippStsCTRSizeErr -1012 /* Incorrect value for crypto CTR block size. */ +#define ippStsEphemeralKeyErr -1011 /* ECC: Invalid ephemeral key. */ +#define ippStsMessageErr -1010 /* ECC: Invalid message digest. */ +#define ippStsShareKeyErr -1009 /* ECC: Invalid share key. */ +#define ippStsInvalidPrivateKey -1008 /* ECC: Invalid private key. */ +#define ippStsOutOfECErr -1007 /* ECC: Point out of EC. */ +#define ippStsECCInvalidFlagErr -1006 /* ECC: Invalid Flag. */ +#define ippStsUnderRunErr -1005 /* Error in data under run. */ +#define ippStsPaddingErr -1004 /* Detected padding error indicates the possible data corruption. */ +#define ippStsCFBSizeErr -1003 /* Incorrect value for crypto CFB block size. */ +#define ippStsPaddingSchemeErr -1002 /* Invalid padding scheme. */ +#define ippStsBadModulusErr -1001 /* Bad modulus caused a failure in module inversion. */ +#define ippStsInsufficientEntropy 25 /* Generation of the prime/key failed due to insufficient entropy in the random seed and stimulus bit string. */ +#define ippStsNotSupportedCpu 36 /* The CPU is not supported. */ +#define ippStsMbWarning 53 /* Error(s) in statuses array. */ + /* end of ippCrypto specific statuses - any changes MUST be done in both repositories - IPP & ippCrypto */ + +#if (!defined IPPCPDEFS_H__) || defined( _OWN_BLDPCS ) +#define IPPCPDEFS_H__ + +#ifdef __cplusplus +extern "C" { +#endif + + + +#if !defined( _OWN_BLDPCS ) + +typedef Ipp32u IppAlgId; + +/* +// ========================================================= +// Symmetric Ciphers +// ========================================================= +*/ +typedef enum { + ippPaddingNONE = 0, /*NONE = 0,*/ IppsCPPaddingNONE = 0, + ippPaddingPKCS7 = 1, /*PKCS7 = 1,*/ IppsCPPaddingPKCS7 = 1, + ippPaddingZEROS = 2, /*ZEROS = 2,*/ IppsCPPaddingZEROS = 2 +} IppsPadding, IppsCPPadding; + +typedef struct _cpDES IppsDESSpec; +typedef struct _cpRijndael128 IppsAESSpec; +typedef struct _cpRijndael128 IppsRijndael128Spec; +typedef struct _cpSMS4 IppsSMS4Spec; + +/* TDES */ +#define DES_BLOCKSIZE (64) /* cipher blocksize (bits) */ +#define TDES_BLOCKSIZE DES_BLOCKSIZE + +#define DES_KEYSIZE (64) /* cipher keysize (bits) */ +#define TDES_KEYSIZE DES_KEYSIZE + +/* AES */ +#define IPP_AES_BLOCK_BITSIZE (128) /* cipher blocksizes (bits) */ + +/* Rijndael */ +typedef enum { + ippRijndaelKey128 = 128, IppsRijndaelKey128 = 128, /* 128-bit key */ + ippRijndaelKey192 = 192, IppsRijndaelKey192 = 192, /* 192-bit key */ + ippRijndaelKey256 = 256, IppsRijndaelKey256 = 256 /* 256-bit key */ +} IppsRijndaelKeyLength; + +/* AES-CCM (authentication & confidence) */ +typedef struct _cpAES_CCM IppsAES_CCMState; +/* AES-GCM (authentication & confidence) */ +typedef struct _cpAES_GCM IppsAES_GCMState; +/* AES-XTS (confidence) */ +typedef struct _cpAES_XTS IppsAES_XTSSpec; + +/* SMS4-CCM (authentication & confidence) */ +typedef struct _cpSMS4_CCM IppsSMS4_CCMState; + +/* +// ========================================================= +// ARCFOUR Stream Cipher +// ========================================================= +*/ +typedef struct _cpARCfour IppsARCFourState; + +#define IPP_ARCFOUR_KEYMAX_SIZE (256) /* max key length (bytes) */ +#define MAX_ARCFOUR_KEY_LEN IPP_ARCFOUR_KEYMAX_SIZE /* obsolete */ + +/* +// ========================================================= +// One-Way Hash Functions +// ========================================================= +*/ +typedef enum { + ippHashAlg_Unknown, + ippHashAlg_SHA1, + ippHashAlg_SHA256, + ippHashAlg_SHA224, + ippHashAlg_SHA512, + ippHashAlg_SHA384, + ippHashAlg_MD5, + ippHashAlg_SM3, + ippHashAlg_SHA512_224, + ippHashAlg_SHA512_256, + ippHashAlg_MaxNo +} IppHashAlgId; + +#define IPP_ALG_HASH_UNKNOWN (ippHashAlg_Unknown) /* unknown */ +#define IPP_ALG_HASH_SHA1 (ippHashAlg_SHA1) /* SHA1 */ +#define IPP_ALG_HASH_SHA256 (ippHashAlg_SHA256) /* SHA256 */ +#define IPP_ALG_HASH_SHA224 (ippHashAlg_SHA224) /* SHA224 or SHA256/224 */ +#define IPP_ALG_HASH_SHA512 (ippHashAlg_SHA512) /* SHA512 */ +#define IPP_ALG_HASH_SHA384 (ippHashAlg_SHA384) /* SHA384 or SHA512/384 */ +#define IPP_ALG_HASH_MD5 (ippHashAlg_MD5) /* MD5 */ +#define IPP_ALG_HASH_SM3 (ippHashAlg_SM3) /* SM3 */ +#define IPP_ALG_HASH_SHA512_224 (ippHashAlg_SHA512_224) /* SHA512/224 */ +#define IPP_ALG_HASH_SHA512_256 (ippHashAlg_SHA512_256) /* SHA512/256 */ +#define IPP_ALG_HASH_LIMIT (ippHashAlg_MaxNo) /* hash alg limiter*/ + +typedef struct _cpSHA1 IppsSHA1State; +typedef struct _cpSHA256 IppsSHA256State; +typedef struct _cpSHA256 IppsSHA224State; +typedef struct _cpSHA512 IppsSHA512State; +typedef struct _cpSHA512 IppsSHA384State; +typedef struct _cpMD5 IppsMD5State; +typedef struct _cpSM3 IppsSM3State; +typedef struct _cpHashCtx IppsHashState; + +typedef struct _cpHashMethod_rmf IppsHashMethod; +typedef struct _cpHashCtx_rmf IppsHashState_rmf; + +#define IPP_SHA1_DIGEST_BITSIZE 160 /* digest size (bits) */ +#define IPP_SHA256_DIGEST_BITSIZE 256 +#define IPP_SHA224_DIGEST_BITSIZE 224 +#define IPP_SHA384_DIGEST_BITSIZE 384 +#define IPP_SHA512_DIGEST_BITSIZE 512 +#define IPP_MD5_DIGEST_BITSIZE 128 +#define IPP_SM3_DIGEST_BITSIZE 256 +#define IPP_SHA512_224_DIGEST_BITSIZE 224 +#define IPP_SHA512_256_DIGEST_BITSIZE 256 + +/* +// ========================================================= +// Keyed-Hash Message Authentication Codes +// ========================================================= +*/ +typedef struct _cpHMAC IppsHMACState; +typedef struct _cpHMAC IppsHMACSHA1State; +typedef struct _cpHMAC IppsHMACSHA256State; +typedef struct _cpHMAC IppsHMACSHA224State; +typedef struct _cpHMAC IppsHMACSHA384State; +typedef struct _cpHMAC IppsHMACSHA512State; +typedef struct _cpHMAC IppsHMACMD5State; +typedef struct _cpHMAC_rmf IppsHMACState_rmf; + +/* +// ========================================================= +// Data Authentication Codes +// ========================================================= +*/ +typedef struct _cpAES_CMAC IppsAES_CMACState; + +/* +// ========================================================= +// Big Number Integer Arithmetic +// ========================================================= +*/ +#define BN_MAXBITSIZE (16*1024) /* bn max size (bits) */ + + +typedef enum { + ippBigNumNEG = 0, IppsBigNumNEG = 0, + ippBigNumPOS = 1, IppsBigNumPOS = 1 +} IppsBigNumSGN; + +typedef enum { + ippBinaryMethod = 0, IppsBinaryMethod = 0, + ippSlidingWindows = 1, IppsSlidingWindows = 1 +} IppsExpMethod; + +typedef struct _cpBigNum IppsBigNumState; +typedef struct _cpMontgomery IppsMontState; +typedef struct _cpPRNG IppsPRNGState; +typedef struct _cpPrime IppsPrimeState; + +/* External Bit Supplier */ +typedef IppStatus (IPP_CALL *IppBitSupplier)(Ipp32u* pRand, int nBits, void* pEbsParams); + +#define IPP_IS_EQ (0) +#define IPP_IS_GT (1) +#define IPP_IS_LT (2) +#define IPP_IS_NE (3) +#define IPP_IS_NA (4) + +#define IPP_IS_PRIME (5) +#define IPP_IS_COMPOSITE (6) + +#define IPP_IS_VALID (7) +#define IPP_IS_INVALID (8) +#define IPP_IS_INCOMPLETE (9) +#define IPP_IS_ATINFINITY (10) + +#define IS_ZERO IPP_IS_EQ +#define GREATER_THAN_ZERO IPP_IS_GT +#define LESS_THAN_ZERO IPP_IS_LT +#define IS_PRIME IPP_IS_PRIME +#define IS_COMPOSITE IPP_IS_COMPOSITE +#define IS_VALID_KEY IPP_IS_VALID +#define IS_INVALID_KEY IPP_IS_INVALID +#define IS_INCOMPLETED_KEY IPP_IS_INCOMPLETE + +/* +// ========================================================= +// RSA Cryptography +// ========================================================= +*/ +#define MIN_RSA_SIZE (8) +#define MAX_RSA_SIZE (16*1024) + +typedef struct _cpRSA IppsRSAState; + +/* key types */ +typedef enum { + ippRSApublic = 0x20000000, IppRSApublic = 0x20000000, + ippRSAprivate = 0x40000000, IppRSAprivate = 0x40000000 +} IppRSAKeyType; + +/* key component's tag */ +typedef enum { + ippRSAkeyN = 0x01, IppRSAkeyN = 0x01, + ippRSAkeyE = 0x02, IppRSAkeyE = 0x02, + ippRSAkeyD = 0x04, IppRSAkeyD = 0x04, + ippRSAkeyP = 0x08, IppRSAkeyP = 0x08, + ippRSAkeyQ = 0x10, IppRSAkeyQ = 0x10, + ippRSAkeyDp = 0x20, IppRSAkeyDp = 0x20, + ippRSAkeyDq = 0x40, IppRSAkeyDq = 0x40, + ippRSAkeyQinv = 0x80, IppRSAkeyQinv = 0x80 +} IppRSAKeyTag; + +typedef struct _cpRSA_public_key IppsRSAPublicKeyState; +typedef struct _cpRSA_private_key IppsRSAPrivateKeyState; + + +/* +// ========================================================= +// DL Cryptography +// ========================================================= +*/ +#define MIN_DLP_BITSIZE (512) +#define MIN_DLP_BITSIZER (160) + +#define MIN_DLPDH_BITSIZE (512) +#define MIN_DLPDH_BITSIZER (160) +#define DEF_DLPDH_BITSIZER (160) + +#define MIN_DLPDSA_BITSIZE (512) +#define MAX_DLPDSA_BITSIZE (1024) +#define MIN_DLPDSA_BITSIZER (160) +#define DEF_DLPDSA_BITSIZER (160) +#define MAX_DLPDSA_BITSIZER (160) +#define MIN_DLPDSA_SEEDSIZE (160) + +typedef struct _cpDLP IppsDLPState; + +/* domain parameter tags */ +typedef enum { + ippDLPkeyP = 0x01, IppDLPkeyP = 0x01, + ippDLPkeyR = 0x02, IppDLPkeyR = 0x02, + ippDLPkeyG = 0x04, IppDLPkeyG = 0x04 +} IppDLPKeyTag; + +typedef enum { + ippDLValid, /* validation pass successfully */ + + ippDLBaseIsEven, /* !(P is odd) */ + ippDLOrderIsEven, /* !(R is odd) */ + ippDLInvalidBaseRange, /* !(2^(L-1) < P < 2^L) */ + ippDLInvalidOrderRange, /* !(2^(M-1) < R < 2^M) */ + ippDLCompositeBase, + ippDLCompositeOrder, + ippDLInvalidCofactor, /* !( R|(P-1) ) */ + ippDLInvalidGenerator, /* !( G^R == 1 (mod P) ) */ + /* !(1 < G < (P-1)) */ + ippDLInvalidPrivateKey, /* !(1 < private < (R-1)) */ + ippDLInvalidPublicKey, /* !(1 < public <=(P-1)) */ + ippDLInvalidKeyPair, /* !(G^private == public */ + + ippDLInvalidSignature /* invalid signature */ +} IppDLResult; + +/* +// ========================================================= +// EC Cryptography +// ========================================================= +*/ +#define EC_GFP_MAXBITSIZE (1024) + +/* operation result */ +typedef enum { + ippECValid, /* validation pass successfully */ + + ippECCompositeBase, /* field based on composite */ + ippECComplicatedBase, /* number of non-zero terms in the polynomial (> PRIME_ARR_MAX) */ + ippECIsZeroDiscriminant,/* zero discriminant */ + ippECCompositeOrder, /* composite order of base point */ + ippECInvalidOrder, /* invalid base point order */ + ippECIsWeakMOV, /* weak Meneze-Okamoto-Vanstone reduction attack */ + ippECIsWeakSSSA, /* weak Semaev-Smart,Satoh-Araki reduction attack */ + ippECIsSupersingular, /* supersingular curve */ + + ippECInvalidPrivateKey, /* !(0 < Private < order) */ + ippECInvalidPublicKey, /* (order*PublicKey != Infinity) */ + ippECInvalidKeyPair, /* (Private*BasePoint != PublicKey) */ + + ippECPointOutOfGroup, /* out of group (order*P != Infinity) */ + ippECPointIsAtInfinite, /* point (P=(Px,Py)) at Infinity */ + ippECPointIsNotValid, /* point (P=(Px,Py)) out-of EC */ + + ippECPointIsEqual, /* compared points are equal */ + ippECPointIsNotEqual, /* compared points are different */ + + ippECInvalidSignature /* invalid signature */ +} IppECResult; + +/* domain parameter set/get flags */ +typedef enum { + ippECarbitrary =0x00000, IppECCArbitrary = 0x00000, /* arbitrary ECC */ + + ippECPstd = 0x10000, IppECCPStd = 0x10000, /* random (recommended) EC over FG(p): */ + ippECPstd112r1 = ippECPstd, IppECCPStd112r1 = IppECCPStd, /* secp112r1 curve */ + ippECPstd112r2 = ippECPstd+1, IppECCPStd112r2 = IppECCPStd+1, /* secp112r2 curve */ + ippECPstd128r1 = ippECPstd+2, IppECCPStd128r1 = IppECCPStd+2, /* secp128r1 curve */ + ippECPstd128r2 = ippECPstd+3, IppECCPStd128r2 = IppECCPStd+3, /* secp128r2 curve */ + ippECPstd160r1 = ippECPstd+4, IppECCPStd160r1 = IppECCPStd+4, /* secp160r1 curve */ + ippECPstd160r2 = ippECPstd+5, IppECCPStd160r2 = IppECCPStd+5, /* secp160r2 curve */ + ippECPstd192r1 = ippECPstd+6, IppECCPStd192r1 = IppECCPStd+6, /* secp192r1 curve */ + ippECPstd224r1 = ippECPstd+7, IppECCPStd224r1 = IppECCPStd+7, /* secp224r1 curve */ + ippECPstd256r1 = ippECPstd+8, IppECCPStd256r1 = IppECCPStd+8, /* secp256r1 curve */ + ippECPstd384r1 = ippECPstd+9, IppECCPStd384r1 = IppECCPStd+9, /* secp384r1 curve */ + ippECPstd521r1 = ippECPstd+10, IppECCPStd521r1 = IppECCPStd+10, /* secp521r1 curve */ + ippECPstdSM2 = ippECPstd+11, IppECCPStdSM2 = IppECCPStd+11, /* TMP SM2 curve */ + ippEC_TPM_SM2_P256= ippECPstd+11, + ippEC_TPM_BN_P256 = ippECPstd+12, /* TPM BN_P256 curve */ + + /* curves over binary finit fields are not supported in Intel® IPP 9.0 */ + IppECCBStd = 0x20000, /* random (recommended) EC over FG(2^m): */ + IppECCBStd113r1 = IppECCBStd, /* sect113r1 curve */ + IppECCBStd113r2 = IppECCBStd+1, /* sect113r2 curve */ + IppECCBStd131r1 = IppECCBStd+2, /* sect131r1 curve */ + IppECCBStd131r2 = IppECCBStd+3, /* sect131r2 curve */ + IppECCBStd163r1 = IppECCBStd+4, /* sect163r1 curve */ + IppECCBStd163r2 = IppECCBStd+5, /* sect163r2 curve */ + IppECCBStd193r1 = IppECCBStd+6, /* sect193r1 curve */ + IppECCBStd193r2 = IppECCBStd+7, /* sect193r2 curve */ + IppECCBStd233r1 = IppECCBStd+8, /* sect233r1 curve */ + IppECCBStd283r1 = IppECCBStd+9, /* sect283r1 curve */ + IppECCBStd409r1 = IppECCBStd+10, /* sect409r1 curve */ + IppECCBStd571r1 = IppECCBStd+11, /* sect571r1 curve */ + + IppECCKStd = 0x40000, /* Koblitz (recommended) EC over FG(2^m): */ + IppECCBStd163k1 = IppECCKStd, /* Koblitz 163 curve */ + IppECCBStd233k1 = IppECCKStd+1, /* Koblitz 233 curve */ + IppECCBStd239k1 = IppECCKStd+2, /* Koblitz 239 curve */ + IppECCBStd283k1 = IppECCKStd+3, /* Koblitz 283 curve */ + IppECCBStd409k1 = IppECCKStd+4, /* Koblitz 409 curve */ + IppECCBStd571k1 = IppECCKStd+5 /* Koblitz 571 curve */ +} IppsECType, IppECCType; + +/* +// GF over prime and its extension +*/ +#define IPP_MIN_GF_CHAR (3) /* min characteristic of GF */ + +#define IPP_MIN_GF_BITSIZE (2) /* min bitsize of element over prime GF */ +#define IPP_MAX_GF_BITSIZE (1024) /* max bitsize of element over prime GF */ + +#define IPP_MIN_GF_EXTDEG (2) /* min GF extension degree */ +#define IPP_MAX_GF_EXTDEG (8) /* max GF extension degree */ + +#define IPP_MAX_EXPONENT_NUM (6) /* max number of exponents, equals to LOG_CACHE_LINE_SIZE */ + +typedef struct _cpGFpMethod IppsGFpMethod; + +typedef struct _cpGFp IppsGFpState; +typedef struct _cpGFpElement IppsGFpElement; + +typedef struct _cpGFpEC IppsGFpECState; +typedef struct _cpGFpECPoint IppsGFpECPoint; + +typedef struct _cpGFpEC IppsECCPState; +typedef struct _cpGFpECPoint IppsECCPPointState; + +typedef struct { + int hashSize; + int msgBlockSize; +} IppsHashInfo; + +typedef struct { + //const IppsGFpState* pBasicGF; + //const IppsGFpState* pGroundGF; + int parentGFdegree; + int basicGFdegree; + int basicElmBitSize; +} IppsGFpInfo; + +/* SM3 Digest Bytes Size */ +#define IPP_SM3_DIGEST_BYTESIZE ((IPP_SM3_DIGEST_BITSIZE + 7) / 8) + +typedef struct _cpStateECES_SM2 IppsECESState_SM2; + +typedef enum { + ippKESM2Requester = 0xF, /* corresponds to A user/participant */ + ippKESM2Responder /* corresponds to B user/participant */ +} IppsKeyExchangeRoleSM2; + +typedef struct _GFpECKeyExchangeSM2 IppsGFpECKeyExchangeSM2State; + +#endif /* !defined( _OWN_BLDPCS ) */ + +IPPAPI( IppStatus, ippcpGetCpuFeatures, ( Ipp64u* pFeaturesMask )) +IPPAPI( IppStatus, ippcpSetCpuFeatures, ( Ipp64u features )) +IPPAPI( Ipp64u, ippcpGetEnabledCpuFeatures, ( void ) ) +IPPAPI( IppStatus, ippcpSetNumThreads, ( int numThr )) +IPPAPI( IppStatus, ippcpInit,( void )) +IPPAPI( IppStatus, ippcpGetNumThreads, (int* pNumThr) ) +IPPAPI( const char*, ippcpGetStatusString, ( IppStatus StsCode )) +IPPAPI( int, ippcpGetEnabledNumThreads, ( void ) ) +IPPAPI( Ipp64u, ippcpGetCpuClocks, (void) ) + +#ifdef __cplusplus +} +#endif + +#endif /* !defined IPPCPDEFS_H__ || defined( _OWN_BLDPCS ) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/ippversion.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/ippversion.h new file mode 100644 index 0000000..7137614 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/ippversion.h @@ -0,0 +1,41 @@ +/******************************************************************************* +* Copyright (C) 2001 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) +// +// Purpose: Describes the Intel IPP Cryptography version +// +*/ + + +#if !defined( IPPVERSION_H__ ) +#define IPPVERSION_H__ + +#define IPP_VERSION_MAJOR 2021 +#define IPP_VERSION_MINOR 9 +#define IPP_VERSION_UPDATE 0 + +// Major interface version +#define IPP_INTERFACE_VERSION_MAJOR 11 +// Minor interface version +#define IPP_INTERFACE_VERSION_MINOR 9 + +#define IPP_VERSION_STR STR(IPP_VERSION_MAJOR) "." STR(IPP_VERSION_MINOR) "." STR(IPP_VERSION_UPDATE) " (" STR(IPP_INTERFACE_VERSION_MAJOR) "." STR(IPP_INTERFACE_VERSION_MINOR) " )" + +#endif /* IPPVERSION_H__ */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_e9.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_e9.h new file mode 100644 index 0000000..80c52e2 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_e9.h @@ -0,0 +1,557 @@ +/******************************************************************************* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + + #define ippcpGetLibVersion e9_ippcpGetLibVersion +#define ippsDESGetSize e9_ippsDESGetSize +#define ippsDESInit e9_ippsDESInit +#define ippsDESPack e9_ippsDESPack +#define ippsDESUnpack e9_ippsDESUnpack +#define ippsTDESEncryptECB e9_ippsTDESEncryptECB +#define ippsTDESDecryptECB e9_ippsTDESDecryptECB +#define ippsTDESEncryptCBC e9_ippsTDESEncryptCBC +#define ippsTDESDecryptCBC e9_ippsTDESDecryptCBC +#define ippsTDESEncryptCFB e9_ippsTDESEncryptCFB +#define ippsTDESDecryptCFB e9_ippsTDESDecryptCFB +#define ippsTDESEncryptOFB e9_ippsTDESEncryptOFB +#define ippsTDESDecryptOFB e9_ippsTDESDecryptOFB +#define ippsTDESEncryptCTR e9_ippsTDESEncryptCTR +#define ippsTDESDecryptCTR e9_ippsTDESDecryptCTR +#define ippsAESGetSize e9_ippsAESGetSize +#define ippsAESInit e9_ippsAESInit +#define ippsAESSetKey e9_ippsAESSetKey +#define ippsAESPack e9_ippsAESPack +#define ippsAESUnpack e9_ippsAESUnpack +#define ippsAESEncryptECB e9_ippsAESEncryptECB +#define ippsAESDecryptECB e9_ippsAESDecryptECB +#define ippsAESEncryptCBC e9_ippsAESEncryptCBC +#define ippsAESEncryptCBC_CS1 e9_ippsAESEncryptCBC_CS1 +#define ippsAESEncryptCBC_CS2 e9_ippsAESEncryptCBC_CS2 +#define ippsAESEncryptCBC_CS3 e9_ippsAESEncryptCBC_CS3 +#define ippsAESDecryptCBC e9_ippsAESDecryptCBC +#define ippsAESDecryptCBC_CS1 e9_ippsAESDecryptCBC_CS1 +#define ippsAESDecryptCBC_CS2 e9_ippsAESDecryptCBC_CS2 +#define ippsAESDecryptCBC_CS3 e9_ippsAESDecryptCBC_CS3 +#define ippsAESEncryptCFB e9_ippsAESEncryptCFB +#define ippsAESDecryptCFB e9_ippsAESDecryptCFB +#define ippsAESEncryptOFB e9_ippsAESEncryptOFB +#define ippsAESDecryptOFB e9_ippsAESDecryptOFB +#define ippsAESEncryptCTR e9_ippsAESEncryptCTR +#define ippsAESDecryptCTR e9_ippsAESDecryptCTR +#define ippsAESEncryptXTS_Direct e9_ippsAESEncryptXTS_Direct +#define ippsAESDecryptXTS_Direct e9_ippsAESDecryptXTS_Direct +#define ippsAESSetupNoise e9_ippsAESSetupNoise +#define ippsAES_GCMSetupNoise e9_ippsAES_GCMSetupNoise +#define ippsAES_CMACSetupNoise e9_ippsAES_CMACSetupNoise +#define ippsAES_EncryptCFB16_MB e9_ippsAES_EncryptCFB16_MB +#define ippsSMS4GetSize e9_ippsSMS4GetSize +#define ippsSMS4Init e9_ippsSMS4Init +#define ippsSMS4SetKey e9_ippsSMS4SetKey +#define ippsSMS4EncryptECB e9_ippsSMS4EncryptECB +#define ippsSMS4DecryptECB e9_ippsSMS4DecryptECB +#define ippsSMS4EncryptCBC e9_ippsSMS4EncryptCBC +#define ippsSMS4EncryptCBC_CS1 e9_ippsSMS4EncryptCBC_CS1 +#define ippsSMS4EncryptCBC_CS2 e9_ippsSMS4EncryptCBC_CS2 +#define ippsSMS4EncryptCBC_CS3 e9_ippsSMS4EncryptCBC_CS3 +#define ippsSMS4DecryptCBC e9_ippsSMS4DecryptCBC +#define ippsSMS4DecryptCBC_CS1 e9_ippsSMS4DecryptCBC_CS1 +#define ippsSMS4DecryptCBC_CS2 e9_ippsSMS4DecryptCBC_CS2 +#define ippsSMS4DecryptCBC_CS3 e9_ippsSMS4DecryptCBC_CS3 +#define ippsSMS4EncryptCFB e9_ippsSMS4EncryptCFB +#define ippsSMS4DecryptCFB e9_ippsSMS4DecryptCFB +#define ippsSMS4EncryptOFB e9_ippsSMS4EncryptOFB +#define ippsSMS4DecryptOFB e9_ippsSMS4DecryptOFB +#define ippsSMS4EncryptCTR e9_ippsSMS4EncryptCTR +#define ippsSMS4DecryptCTR e9_ippsSMS4DecryptCTR +#define ippsSMS4_CCMGetSize e9_ippsSMS4_CCMGetSize +#define ippsSMS4_CCMInit e9_ippsSMS4_CCMInit +#define ippsSMS4_CCMMessageLen e9_ippsSMS4_CCMMessageLen +#define ippsSMS4_CCMTagLen e9_ippsSMS4_CCMTagLen +#define ippsSMS4_CCMStart e9_ippsSMS4_CCMStart +#define ippsSMS4_CCMEncrypt e9_ippsSMS4_CCMEncrypt +#define ippsSMS4_CCMDecrypt e9_ippsSMS4_CCMDecrypt +#define ippsSMS4_CCMGetTag e9_ippsSMS4_CCMGetTag +#define ippsAES_CCMGetSize e9_ippsAES_CCMGetSize +#define ippsAES_CCMInit e9_ippsAES_CCMInit +#define ippsAES_CCMMessageLen e9_ippsAES_CCMMessageLen +#define ippsAES_CCMTagLen e9_ippsAES_CCMTagLen +#define ippsAES_CCMStart e9_ippsAES_CCMStart +#define ippsAES_CCMEncrypt e9_ippsAES_CCMEncrypt +#define ippsAES_CCMDecrypt e9_ippsAES_CCMDecrypt +#define ippsAES_CCMGetTag e9_ippsAES_CCMGetTag +#define ippsAES_GCMGetSize e9_ippsAES_GCMGetSize +#define ippsAES_GCMInit e9_ippsAES_GCMInit +#define ippsAES_GCMReinit e9_ippsAES_GCMReinit +#define ippsAES_GCMReset e9_ippsAES_GCMReset +#define ippsAES_GCMProcessIV e9_ippsAES_GCMProcessIV +#define ippsAES_GCMProcessAAD e9_ippsAES_GCMProcessAAD +#define ippsAES_GCMStart e9_ippsAES_GCMStart +#define ippsAES_GCMEncrypt e9_ippsAES_GCMEncrypt +#define ippsAES_GCMDecrypt e9_ippsAES_GCMDecrypt +#define ippsAES_GCMGetTag e9_ippsAES_GCMGetTag +#define ippsAES_XTSGetSize e9_ippsAES_XTSGetSize +#define ippsAES_XTSInit e9_ippsAES_XTSInit +#define ippsAES_XTSEncrypt e9_ippsAES_XTSEncrypt +#define ippsAES_XTSDecrypt e9_ippsAES_XTSDecrypt +#define ippsAES_S2V_CMAC e9_ippsAES_S2V_CMAC +#define ippsAES_SIVEncrypt e9_ippsAES_SIVEncrypt +#define ippsAES_SIVDecrypt e9_ippsAES_SIVDecrypt +#define ippsAES_CMACGetSize e9_ippsAES_CMACGetSize +#define ippsAES_CMACInit e9_ippsAES_CMACInit +#define ippsAES_CMACUpdate e9_ippsAES_CMACUpdate +#define ippsAES_CMACFinal e9_ippsAES_CMACFinal +#define ippsAES_CMACGetTag e9_ippsAES_CMACGetTag +#define ippsARCFourCheckKey e9_ippsARCFourCheckKey +#define ippsARCFourGetSize e9_ippsARCFourGetSize +#define ippsARCFourInit e9_ippsARCFourInit +#define ippsARCFourReset e9_ippsARCFourReset +#define ippsARCFourPack e9_ippsARCFourPack +#define ippsARCFourUnpack e9_ippsARCFourUnpack +#define ippsARCFourEncrypt e9_ippsARCFourEncrypt +#define ippsARCFourDecrypt e9_ippsARCFourDecrypt +#define ippsSHA1GetSize e9_ippsSHA1GetSize +#define ippsSHA1Init e9_ippsSHA1Init +#define ippsSHA1Duplicate e9_ippsSHA1Duplicate +#define ippsSHA1Pack e9_ippsSHA1Pack +#define ippsSHA1Unpack e9_ippsSHA1Unpack +#define ippsSHA1Update e9_ippsSHA1Update +#define ippsSHA1GetTag e9_ippsSHA1GetTag +#define ippsSHA1Final e9_ippsSHA1Final +#define ippsSHA1MessageDigest e9_ippsSHA1MessageDigest +#define ippsSHA224GetSize e9_ippsSHA224GetSize +#define ippsSHA224Init e9_ippsSHA224Init +#define ippsSHA224Duplicate e9_ippsSHA224Duplicate +#define ippsSHA224Pack e9_ippsSHA224Pack +#define ippsSHA224Unpack e9_ippsSHA224Unpack +#define ippsSHA224Update e9_ippsSHA224Update +#define ippsSHA224GetTag e9_ippsSHA224GetTag +#define ippsSHA224Final e9_ippsSHA224Final +#define ippsSHA224MessageDigest e9_ippsSHA224MessageDigest +#define ippsSHA256GetSize e9_ippsSHA256GetSize +#define ippsSHA256Init e9_ippsSHA256Init +#define ippsSHA256Duplicate e9_ippsSHA256Duplicate +#define ippsSHA256Pack e9_ippsSHA256Pack +#define ippsSHA256Unpack e9_ippsSHA256Unpack +#define ippsSHA256Update e9_ippsSHA256Update +#define ippsSHA256GetTag e9_ippsSHA256GetTag +#define ippsSHA256Final e9_ippsSHA256Final +#define ippsSHA256MessageDigest e9_ippsSHA256MessageDigest +#define ippsSHA384GetSize e9_ippsSHA384GetSize +#define ippsSHA384Init e9_ippsSHA384Init +#define ippsSHA384Duplicate e9_ippsSHA384Duplicate +#define ippsSHA384Pack e9_ippsSHA384Pack +#define ippsSHA384Unpack e9_ippsSHA384Unpack +#define ippsSHA384Update e9_ippsSHA384Update +#define ippsSHA384GetTag e9_ippsSHA384GetTag +#define ippsSHA384Final e9_ippsSHA384Final +#define ippsSHA384MessageDigest e9_ippsSHA384MessageDigest +#define ippsSHA512GetSize e9_ippsSHA512GetSize +#define ippsSHA512Init e9_ippsSHA512Init +#define ippsSHA512Duplicate e9_ippsSHA512Duplicate +#define ippsSHA512Pack e9_ippsSHA512Pack +#define ippsSHA512Unpack e9_ippsSHA512Unpack +#define ippsSHA512Update e9_ippsSHA512Update +#define ippsSHA512GetTag e9_ippsSHA512GetTag +#define ippsSHA512Final e9_ippsSHA512Final +#define ippsSHA512MessageDigest e9_ippsSHA512MessageDigest +#define ippsMD5GetSize e9_ippsMD5GetSize +#define ippsMD5Init e9_ippsMD5Init +#define ippsMD5Duplicate e9_ippsMD5Duplicate +#define ippsMD5Pack e9_ippsMD5Pack +#define ippsMD5Unpack e9_ippsMD5Unpack +#define ippsMD5Update e9_ippsMD5Update +#define ippsMD5GetTag e9_ippsMD5GetTag +#define ippsMD5Final e9_ippsMD5Final +#define ippsMD5MessageDigest e9_ippsMD5MessageDigest +#define ippsSM3GetSize e9_ippsSM3GetSize +#define ippsSM3Init e9_ippsSM3Init +#define ippsSM3Duplicate e9_ippsSM3Duplicate +#define ippsSM3Pack e9_ippsSM3Pack +#define ippsSM3Unpack e9_ippsSM3Unpack +#define ippsSM3Update e9_ippsSM3Update +#define ippsSM3GetTag e9_ippsSM3GetTag +#define ippsSM3Final e9_ippsSM3Final +#define ippsSM3MessageDigest e9_ippsSM3MessageDigest +#define ippsHashGetSize e9_ippsHashGetSize +#define ippsHashInit e9_ippsHashInit +#define ippsHashPack e9_ippsHashPack +#define ippsHashUnpack e9_ippsHashUnpack +#define ippsHashDuplicate e9_ippsHashDuplicate +#define ippsHashUpdate e9_ippsHashUpdate +#define ippsHashGetTag e9_ippsHashGetTag +#define ippsHashFinal e9_ippsHashFinal +#define ippsHashMessage e9_ippsHashMessage +#define ippsHashMethod_MD5 e9_ippsHashMethod_MD5 +#define ippsHashMethod_SM3 e9_ippsHashMethod_SM3 +#define ippsHashMethod_SHA1 e9_ippsHashMethod_SHA1 +#define ippsHashMethod_SHA1_NI e9_ippsHashMethod_SHA1_NI +#define ippsHashMethod_SHA1_TT e9_ippsHashMethod_SHA1_TT +#define ippsHashMethod_SHA256 e9_ippsHashMethod_SHA256 +#define ippsHashMethod_SHA256_NI e9_ippsHashMethod_SHA256_NI +#define ippsHashMethod_SHA256_TT e9_ippsHashMethod_SHA256_TT +#define ippsHashMethod_SHA224 e9_ippsHashMethod_SHA224 +#define ippsHashMethod_SHA224_NI e9_ippsHashMethod_SHA224_NI +#define ippsHashMethod_SHA224_TT e9_ippsHashMethod_SHA224_TT +#define ippsHashMethod_SHA512 e9_ippsHashMethod_SHA512 +#define ippsHashMethod_SHA384 e9_ippsHashMethod_SHA384 +#define ippsHashMethod_SHA512_256 e9_ippsHashMethod_SHA512_256 +#define ippsHashMethod_SHA512_224 e9_ippsHashMethod_SHA512_224 +#define ippsHashMethodGetSize e9_ippsHashMethodGetSize +#define ippsHashMethodSet_MD5 e9_ippsHashMethodSet_MD5 +#define ippsHashMethodSet_SM3 e9_ippsHashMethodSet_SM3 +#define ippsHashStateMethodSet_SM3 e9_ippsHashStateMethodSet_SM3 +#define ippsHashMethodSet_SHA1 e9_ippsHashMethodSet_SHA1 +#define ippsHashMethodSet_SHA1_NI e9_ippsHashMethodSet_SHA1_NI +#define ippsHashMethodSet_SHA1_TT e9_ippsHashMethodSet_SHA1_TT +#define ippsHashMethodSet_SHA256 e9_ippsHashMethodSet_SHA256 +#define ippsHashMethodSet_SHA256_NI e9_ippsHashMethodSet_SHA256_NI +#define ippsHashMethodSet_SHA256_TT e9_ippsHashMethodSet_SHA256_TT +#define ippsHashStateMethodSet_SHA256 e9_ippsHashStateMethodSet_SHA256 +#define ippsHashStateMethodSet_SHA256_NI e9_ippsHashStateMethodSet_SHA256_NI +#define ippsHashStateMethodSet_SHA256_TT e9_ippsHashStateMethodSet_SHA256_TT +#define ippsHashMethodSet_SHA224 e9_ippsHashMethodSet_SHA224 +#define ippsHashMethodSet_SHA224_NI e9_ippsHashMethodSet_SHA224_NI +#define ippsHashMethodSet_SHA224_TT e9_ippsHashMethodSet_SHA224_TT +#define ippsHashStateMethodSet_SHA224 e9_ippsHashStateMethodSet_SHA224 +#define ippsHashStateMethodSet_SHA224_NI e9_ippsHashStateMethodSet_SHA224_NI +#define ippsHashStateMethodSet_SHA224_TT e9_ippsHashStateMethodSet_SHA224_TT +#define ippsHashMethodSet_SHA512 e9_ippsHashMethodSet_SHA512 +#define ippsHashMethodSet_SHA384 e9_ippsHashMethodSet_SHA384 +#define ippsHashMethodSet_SHA512_256 e9_ippsHashMethodSet_SHA512_256 +#define ippsHashMethodSet_SHA512_224 e9_ippsHashMethodSet_SHA512_224 +#define ippsHashStateMethodSet_SHA512 e9_ippsHashStateMethodSet_SHA512 +#define ippsHashStateMethodSet_SHA384 e9_ippsHashStateMethodSet_SHA384 +#define ippsHashStateMethodSet_SHA512_256 e9_ippsHashStateMethodSet_SHA512_256 +#define ippsHashStateMethodSet_SHA512_224 e9_ippsHashStateMethodSet_SHA512_224 +#define ippsHashGetSize_rmf e9_ippsHashGetSize_rmf +#define ippsHashInit_rmf e9_ippsHashInit_rmf +#define ippsHashPack_rmf e9_ippsHashPack_rmf +#define ippsHashUnpack_rmf e9_ippsHashUnpack_rmf +#define ippsHashDuplicate_rmf e9_ippsHashDuplicate_rmf +#define ippsHashUpdate_rmf e9_ippsHashUpdate_rmf +#define ippsHashGetTag_rmf e9_ippsHashGetTag_rmf +#define ippsHashFinal_rmf e9_ippsHashFinal_rmf +#define ippsHashMessage_rmf e9_ippsHashMessage_rmf +#define ippsHashMethodGetInfo e9_ippsHashMethodGetInfo +#define ippsHashGetInfo_rmf e9_ippsHashGetInfo_rmf +#define ippsMGF e9_ippsMGF +#define ippsMGF1_rmf e9_ippsMGF1_rmf +#define ippsMGF2_rmf e9_ippsMGF2_rmf +#define ippsHMAC_GetSize e9_ippsHMAC_GetSize +#define ippsHMAC_Init e9_ippsHMAC_Init +#define ippsHMAC_Pack e9_ippsHMAC_Pack +#define ippsHMAC_Unpack e9_ippsHMAC_Unpack +#define ippsHMAC_Duplicate e9_ippsHMAC_Duplicate +#define ippsHMAC_Update e9_ippsHMAC_Update +#define ippsHMAC_Final e9_ippsHMAC_Final +#define ippsHMAC_GetTag e9_ippsHMAC_GetTag +#define ippsHMAC_Message e9_ippsHMAC_Message +#define ippsHMACGetSize_rmf e9_ippsHMACGetSize_rmf +#define ippsHMACInit_rmf e9_ippsHMACInit_rmf +#define ippsHMACPack_rmf e9_ippsHMACPack_rmf +#define ippsHMACUnpack_rmf e9_ippsHMACUnpack_rmf +#define ippsHMACDuplicate_rmf e9_ippsHMACDuplicate_rmf +#define ippsHMACUpdate_rmf e9_ippsHMACUpdate_rmf +#define ippsHMACFinal_rmf e9_ippsHMACFinal_rmf +#define ippsHMACGetTag_rmf e9_ippsHMACGetTag_rmf +#define ippsHMACMessage_rmf e9_ippsHMACMessage_rmf +#define ippsBigNumGetSize e9_ippsBigNumGetSize +#define ippsBigNumInit e9_ippsBigNumInit +#define ippsCmpZero_BN e9_ippsCmpZero_BN +#define ippsCmp_BN e9_ippsCmp_BN +#define ippsGetSize_BN e9_ippsGetSize_BN +#define ippsSet_BN e9_ippsSet_BN +#define ippsGet_BN e9_ippsGet_BN +#define ippsRef_BN e9_ippsRef_BN +#define ippsExtGet_BN e9_ippsExtGet_BN +#define ippsAdd_BN e9_ippsAdd_BN +#define ippsSub_BN e9_ippsSub_BN +#define ippsMul_BN e9_ippsMul_BN +#define ippsMAC_BN_I e9_ippsMAC_BN_I +#define ippsDiv_BN e9_ippsDiv_BN +#define ippsMod_BN e9_ippsMod_BN +#define ippsGcd_BN e9_ippsGcd_BN +#define ippsModInv_BN e9_ippsModInv_BN +#define ippsSetOctString_BN e9_ippsSetOctString_BN +#define ippsGetOctString_BN e9_ippsGetOctString_BN +#define ippsMontGetSize e9_ippsMontGetSize +#define ippsMontInit e9_ippsMontInit +#define ippsMontSet e9_ippsMontSet +#define ippsMontGet e9_ippsMontGet +#define ippsMontForm e9_ippsMontForm +#define ippsMontMul e9_ippsMontMul +#define ippsMontExp e9_ippsMontExp +#define ippsPRNGGetSize e9_ippsPRNGGetSize +#define ippsPRNGInit e9_ippsPRNGInit +#define ippsPRNGSetModulus e9_ippsPRNGSetModulus +#define ippsPRNGSetH0 e9_ippsPRNGSetH0 +#define ippsPRNGSetAugment e9_ippsPRNGSetAugment +#define ippsPRNGSetSeed e9_ippsPRNGSetSeed +#define ippsPRNGGetSeed e9_ippsPRNGGetSeed +#define ippsPRNGen e9_ippsPRNGen +#define ippsPRNGen_BN e9_ippsPRNGen_BN +#define ippsPRNGenRDRAND e9_ippsPRNGenRDRAND +#define ippsPRNGenRDRAND_BN e9_ippsPRNGenRDRAND_BN +#define ippsTRNGenRDSEED e9_ippsTRNGenRDSEED +#define ippsTRNGenRDSEED_BN e9_ippsTRNGenRDSEED_BN +#define ippsPrimeGetSize e9_ippsPrimeGetSize +#define ippsPrimeInit e9_ippsPrimeInit +#define ippsPrimeGen e9_ippsPrimeGen +#define ippsPrimeTest e9_ippsPrimeTest +#define ippsPrimeGen_BN e9_ippsPrimeGen_BN +#define ippsPrimeTest_BN e9_ippsPrimeTest_BN +#define ippsPrimeGet e9_ippsPrimeGet +#define ippsPrimeGet_BN e9_ippsPrimeGet_BN +#define ippsPrimeSet e9_ippsPrimeSet +#define ippsPrimeSet_BN e9_ippsPrimeSet_BN +#define ippsRSA_GetSizePublicKey e9_ippsRSA_GetSizePublicKey +#define ippsRSA_InitPublicKey e9_ippsRSA_InitPublicKey +#define ippsRSA_SetPublicKey e9_ippsRSA_SetPublicKey +#define ippsRSA_GetPublicKey e9_ippsRSA_GetPublicKey +#define ippsRSA_GetSizePrivateKeyType1 e9_ippsRSA_GetSizePrivateKeyType1 +#define ippsRSA_InitPrivateKeyType1 e9_ippsRSA_InitPrivateKeyType1 +#define ippsRSA_SetPrivateKeyType1 e9_ippsRSA_SetPrivateKeyType1 +#define ippsRSA_GetPrivateKeyType1 e9_ippsRSA_GetPrivateKeyType1 +#define ippsRSA_GetSizePrivateKeyType2 e9_ippsRSA_GetSizePrivateKeyType2 +#define ippsRSA_InitPrivateKeyType2 e9_ippsRSA_InitPrivateKeyType2 +#define ippsRSA_SetPrivateKeyType2 e9_ippsRSA_SetPrivateKeyType2 +#define ippsRSA_GetPrivateKeyType2 e9_ippsRSA_GetPrivateKeyType2 +#define ippsRSA_GetBufferSizePublicKey e9_ippsRSA_GetBufferSizePublicKey +#define ippsRSA_GetBufferSizePrivateKey e9_ippsRSA_GetBufferSizePrivateKey +#define ippsRSA_Encrypt e9_ippsRSA_Encrypt +#define ippsRSA_Decrypt e9_ippsRSA_Decrypt +#define ippsRSA_GenerateKeys e9_ippsRSA_GenerateKeys +#define ippsRSA_ValidateKeys e9_ippsRSA_ValidateKeys +#define ippsRSAEncrypt_OAEP e9_ippsRSAEncrypt_OAEP +#define ippsRSADecrypt_OAEP e9_ippsRSADecrypt_OAEP +#define ippsRSAEncrypt_OAEP_rmf e9_ippsRSAEncrypt_OAEP_rmf +#define ippsRSADecrypt_OAEP_rmf e9_ippsRSADecrypt_OAEP_rmf +#define ippsRSAEncrypt_PKCSv15 e9_ippsRSAEncrypt_PKCSv15 +#define ippsRSADecrypt_PKCSv15 e9_ippsRSADecrypt_PKCSv15 +#define ippsRSASign_PSS e9_ippsRSASign_PSS +#define ippsRSAVerify_PSS e9_ippsRSAVerify_PSS +#define ippsRSASign_PSS_rmf e9_ippsRSASign_PSS_rmf +#define ippsRSAVerify_PSS_rmf e9_ippsRSAVerify_PSS_rmf +#define ippsRSASign_PKCS1v15 e9_ippsRSASign_PKCS1v15 +#define ippsRSAVerify_PKCS1v15 e9_ippsRSAVerify_PKCS1v15 +#define ippsRSASign_PKCS1v15_rmf e9_ippsRSASign_PKCS1v15_rmf +#define ippsRSAVerify_PKCS1v15_rmf e9_ippsRSAVerify_PKCS1v15_rmf +#define ippsDLGetResultString e9_ippsDLGetResultString +#define ippsDLPGetSize e9_ippsDLPGetSize +#define ippsDLPInit e9_ippsDLPInit +#define ippsDLPPack e9_ippsDLPPack +#define ippsDLPUnpack e9_ippsDLPUnpack +#define ippsDLPSet e9_ippsDLPSet +#define ippsDLPGet e9_ippsDLPGet +#define ippsDLPSetDP e9_ippsDLPSetDP +#define ippsDLPGetDP e9_ippsDLPGetDP +#define ippsDLPGenKeyPair e9_ippsDLPGenKeyPair +#define ippsDLPPublicKey e9_ippsDLPPublicKey +#define ippsDLPValidateKeyPair e9_ippsDLPValidateKeyPair +#define ippsDLPSetKeyPair e9_ippsDLPSetKeyPair +#define ippsDLPSignDSA e9_ippsDLPSignDSA +#define ippsDLPVerifyDSA e9_ippsDLPVerifyDSA +#define ippsDLPSharedSecretDH e9_ippsDLPSharedSecretDH +#define ippsDLPGenerateDSA e9_ippsDLPGenerateDSA +#define ippsDLPValidateDSA e9_ippsDLPValidateDSA +#define ippsDLPGenerateDH e9_ippsDLPGenerateDH +#define ippsDLPValidateDH e9_ippsDLPValidateDH +#define ippsECCGetResultString e9_ippsECCGetResultString +#define ippsECCPGetSize e9_ippsECCPGetSize +#define ippsECCPGetSizeStd128r1 e9_ippsECCPGetSizeStd128r1 +#define ippsECCPGetSizeStd128r2 e9_ippsECCPGetSizeStd128r2 +#define ippsECCPGetSizeStd192r1 e9_ippsECCPGetSizeStd192r1 +#define ippsECCPGetSizeStd224r1 e9_ippsECCPGetSizeStd224r1 +#define ippsECCPGetSizeStd256r1 e9_ippsECCPGetSizeStd256r1 +#define ippsECCPGetSizeStd384r1 e9_ippsECCPGetSizeStd384r1 +#define ippsECCPGetSizeStd521r1 e9_ippsECCPGetSizeStd521r1 +#define ippsECCPGetSizeStdSM2 e9_ippsECCPGetSizeStdSM2 +#define ippsECCPInit e9_ippsECCPInit +#define ippsECCPInitStd128r1 e9_ippsECCPInitStd128r1 +#define ippsECCPInitStd128r2 e9_ippsECCPInitStd128r2 +#define ippsECCPInitStd192r1 e9_ippsECCPInitStd192r1 +#define ippsECCPInitStd224r1 e9_ippsECCPInitStd224r1 +#define ippsECCPInitStd256r1 e9_ippsECCPInitStd256r1 +#define ippsECCPInitStd384r1 e9_ippsECCPInitStd384r1 +#define ippsECCPInitStd521r1 e9_ippsECCPInitStd521r1 +#define ippsECCPInitStdSM2 e9_ippsECCPInitStdSM2 +#define ippsECCPSet e9_ippsECCPSet +#define ippsECCPSetStd e9_ippsECCPSetStd +#define ippsECCPSetStd128r1 e9_ippsECCPSetStd128r1 +#define ippsECCPSetStd128r2 e9_ippsECCPSetStd128r2 +#define ippsECCPSetStd192r1 e9_ippsECCPSetStd192r1 +#define ippsECCPSetStd224r1 e9_ippsECCPSetStd224r1 +#define ippsECCPSetStd256r1 e9_ippsECCPSetStd256r1 +#define ippsECCPSetStd384r1 e9_ippsECCPSetStd384r1 +#define ippsECCPSetStd521r1 e9_ippsECCPSetStd521r1 +#define ippsECCPSetStdSM2 e9_ippsECCPSetStdSM2 +#define ippsECCPBindGxyTblStd192r1 e9_ippsECCPBindGxyTblStd192r1 +#define ippsECCPBindGxyTblStd224r1 e9_ippsECCPBindGxyTblStd224r1 +#define ippsECCPBindGxyTblStd256r1 e9_ippsECCPBindGxyTblStd256r1 +#define ippsECCPBindGxyTblStd384r1 e9_ippsECCPBindGxyTblStd384r1 +#define ippsECCPBindGxyTblStd521r1 e9_ippsECCPBindGxyTblStd521r1 +#define ippsECCPBindGxyTblStdSM2 e9_ippsECCPBindGxyTblStdSM2 +#define ippsECCPGet e9_ippsECCPGet +#define ippsECCPGetOrderBitSize e9_ippsECCPGetOrderBitSize +#define ippsECCPValidate e9_ippsECCPValidate +#define ippsECCPPointGetSize e9_ippsECCPPointGetSize +#define ippsECCPPointInit e9_ippsECCPPointInit +#define ippsECCPSetPoint e9_ippsECCPSetPoint +#define ippsECCPSetPointAtInfinity e9_ippsECCPSetPointAtInfinity +#define ippsECCPGetPoint e9_ippsECCPGetPoint +#define ippsECCPCheckPoint e9_ippsECCPCheckPoint +#define ippsECCPComparePoint e9_ippsECCPComparePoint +#define ippsECCPNegativePoint e9_ippsECCPNegativePoint +#define ippsECCPAddPoint e9_ippsECCPAddPoint +#define ippsECCPMulPointScalar e9_ippsECCPMulPointScalar +#define ippsECCPGenKeyPair e9_ippsECCPGenKeyPair +#define ippsECCPPublicKey e9_ippsECCPPublicKey +#define ippsECCPValidateKeyPair e9_ippsECCPValidateKeyPair +#define ippsECCPSetKeyPair e9_ippsECCPSetKeyPair +#define ippsECCPSharedSecretDH e9_ippsECCPSharedSecretDH +#define ippsECCPSharedSecretDHC e9_ippsECCPSharedSecretDHC +#define ippsECCPSignDSA e9_ippsECCPSignDSA +#define ippsECCPVerifyDSA e9_ippsECCPVerifyDSA +#define ippsECCPSignNR e9_ippsECCPSignNR +#define ippsECCPVerifyNR e9_ippsECCPVerifyNR +#define ippsECCPSignSM2 e9_ippsECCPSignSM2 +#define ippsECCPVerifySM2 e9_ippsECCPVerifySM2 +#define ippsGFpGetSize e9_ippsGFpGetSize +#define ippsGFpInitArbitrary e9_ippsGFpInitArbitrary +#define ippsGFpInitFixed e9_ippsGFpInitFixed +#define ippsGFpInit e9_ippsGFpInit +#define ippsGFpMethod_p192r1 e9_ippsGFpMethod_p192r1 +#define ippsGFpMethod_p224r1 e9_ippsGFpMethod_p224r1 +#define ippsGFpMethod_p256r1 e9_ippsGFpMethod_p256r1 +#define ippsGFpMethod_p384r1 e9_ippsGFpMethod_p384r1 +#define ippsGFpMethod_p521r1 e9_ippsGFpMethod_p521r1 +#define ippsGFpMethod_p256sm2 e9_ippsGFpMethod_p256sm2 +#define ippsGFpMethod_p256bn e9_ippsGFpMethod_p256bn +#define ippsGFpMethod_p256 e9_ippsGFpMethod_p256 +#define ippsGFpMethod_pArb e9_ippsGFpMethod_pArb +#define ippsGFpxGetSize e9_ippsGFpxGetSize +#define ippsGFpxInit e9_ippsGFpxInit +#define ippsGFpxInitBinomial e9_ippsGFpxInitBinomial +#define ippsGFpxMethod_binom2_epid2 e9_ippsGFpxMethod_binom2_epid2 +#define ippsGFpxMethod_binom3_epid2 e9_ippsGFpxMethod_binom3_epid2 +#define ippsGFpxMethod_binom2 e9_ippsGFpxMethod_binom2 +#define ippsGFpxMethod_binom3 e9_ippsGFpxMethod_binom3 +#define ippsGFpxMethod_binom e9_ippsGFpxMethod_binom +#define ippsGFpxMethod_com e9_ippsGFpxMethod_com +#define ippsGFpScratchBufferSize e9_ippsGFpScratchBufferSize +#define ippsGFpElementGetSize e9_ippsGFpElementGetSize +#define ippsGFpElementInit e9_ippsGFpElementInit +#define ippsGFpSetElement e9_ippsGFpSetElement +#define ippsGFpSetElementRegular e9_ippsGFpSetElementRegular +#define ippsGFpSetElementOctString e9_ippsGFpSetElementOctString +#define ippsGFpSetElementRandom e9_ippsGFpSetElementRandom +#define ippsGFpSetElementHash e9_ippsGFpSetElementHash +#define ippsGFpSetElementHash_rmf e9_ippsGFpSetElementHash_rmf +#define ippsGFpCpyElement e9_ippsGFpCpyElement +#define ippsGFpGetElement e9_ippsGFpGetElement +#define ippsGFpGetElementOctString e9_ippsGFpGetElementOctString +#define ippsGFpCmpElement e9_ippsGFpCmpElement +#define ippsGFpIsZeroElement e9_ippsGFpIsZeroElement +#define ippsGFpIsUnityElement e9_ippsGFpIsUnityElement +#define ippsGFpConj e9_ippsGFpConj +#define ippsGFpNeg e9_ippsGFpNeg +#define ippsGFpInv e9_ippsGFpInv +#define ippsGFpSqrt e9_ippsGFpSqrt +#define ippsGFpSqr e9_ippsGFpSqr +#define ippsGFpAdd e9_ippsGFpAdd +#define ippsGFpSub e9_ippsGFpSub +#define ippsGFpMul e9_ippsGFpMul +#define ippsGFpExp e9_ippsGFpExp +#define ippsGFpMultiExp e9_ippsGFpMultiExp +#define ippsGFpAdd_PE e9_ippsGFpAdd_PE +#define ippsGFpSub_PE e9_ippsGFpSub_PE +#define ippsGFpMul_PE e9_ippsGFpMul_PE +#define ippsGFpGetInfo e9_ippsGFpGetInfo +#define ippsGFpECGetSize e9_ippsGFpECGetSize +#define ippsGFpECInit e9_ippsGFpECInit +#define ippsGFpECSet e9_ippsGFpECSet +#define ippsGFpECSetSubgroup e9_ippsGFpECSetSubgroup +#define ippsGFpECInitStd128r1 e9_ippsGFpECInitStd128r1 +#define ippsGFpECInitStd128r2 e9_ippsGFpECInitStd128r2 +#define ippsGFpECInitStd192r1 e9_ippsGFpECInitStd192r1 +#define ippsGFpECInitStd224r1 e9_ippsGFpECInitStd224r1 +#define ippsGFpECInitStd256r1 e9_ippsGFpECInitStd256r1 +#define ippsGFpECInitStd384r1 e9_ippsGFpECInitStd384r1 +#define ippsGFpECInitStd521r1 e9_ippsGFpECInitStd521r1 +#define ippsGFpECInitStdSM2 e9_ippsGFpECInitStdSM2 +#define ippsGFpECInitStdBN256 e9_ippsGFpECInitStdBN256 +#define ippsGFpECBindGxyTblStd192r1 e9_ippsGFpECBindGxyTblStd192r1 +#define ippsGFpECBindGxyTblStd224r1 e9_ippsGFpECBindGxyTblStd224r1 +#define ippsGFpECBindGxyTblStd256r1 e9_ippsGFpECBindGxyTblStd256r1 +#define ippsGFpECBindGxyTblStd384r1 e9_ippsGFpECBindGxyTblStd384r1 +#define ippsGFpECBindGxyTblStd521r1 e9_ippsGFpECBindGxyTblStd521r1 +#define ippsGFpECBindGxyTblStdSM2 e9_ippsGFpECBindGxyTblStdSM2 +#define ippsGFpECGet e9_ippsGFpECGet +#define ippsGFpECGetSubgroup e9_ippsGFpECGetSubgroup +#define ippsGFpECScratchBufferSize e9_ippsGFpECScratchBufferSize +#define ippsGFpECVerify e9_ippsGFpECVerify +#define ippsGFpECPointGetSize e9_ippsGFpECPointGetSize +#define ippsGFpECPointInit e9_ippsGFpECPointInit +#define ippsGFpECSetPointAtInfinity e9_ippsGFpECSetPointAtInfinity +#define ippsGFpECSetPoint e9_ippsGFpECSetPoint +#define ippsGFpECSetPointRegular e9_ippsGFpECSetPointRegular +#define ippsGFpECSetPointRandom e9_ippsGFpECSetPointRandom +#define ippsGFpECMakePoint e9_ippsGFpECMakePoint +#define ippsGFpECSetPointHash e9_ippsGFpECSetPointHash +#define ippsGFpECSetPointHashBackCompatible e9_ippsGFpECSetPointHashBackCompatible +#define ippsGFpECSetPointHash_rmf e9_ippsGFpECSetPointHash_rmf +#define ippsGFpECSetPointHashBackCompatible_rmf e9_ippsGFpECSetPointHashBackCompatible_rmf +#define ippsGFpECGetPoint e9_ippsGFpECGetPoint +#define ippsGFpECGetPointRegular e9_ippsGFpECGetPointRegular +#define ippsGFpECSetPointOctString e9_ippsGFpECSetPointOctString +#define ippsGFpECGetPointOctString e9_ippsGFpECGetPointOctString +#define ippsGFpECTstPoint e9_ippsGFpECTstPoint +#define ippsGFpECTstPointInSubgroup e9_ippsGFpECTstPointInSubgroup +#define ippsGFpECCpyPoint e9_ippsGFpECCpyPoint +#define ippsGFpECCmpPoint e9_ippsGFpECCmpPoint +#define ippsGFpECNegPoint e9_ippsGFpECNegPoint +#define ippsGFpECAddPoint e9_ippsGFpECAddPoint +#define ippsGFpECMulPoint e9_ippsGFpECMulPoint +#define ippsGFpECPrivateKey e9_ippsGFpECPrivateKey +#define ippsGFpECPublicKey e9_ippsGFpECPublicKey +#define ippsGFpECTstKeyPair e9_ippsGFpECTstKeyPair +#define ippsGFpECSharedSecretDH e9_ippsGFpECSharedSecretDH +#define ippsGFpECSharedSecretDHC e9_ippsGFpECSharedSecretDHC +#define ippsGFpECMessageRepresentationSM2 e9_ippsGFpECMessageRepresentationSM2 +#define ippsGFpECSignDSA e9_ippsGFpECSignDSA +#define ippsGFpECVerifyDSA e9_ippsGFpECVerifyDSA +#define ippsGFpECSignNR e9_ippsGFpECSignNR +#define ippsGFpECVerifyNR e9_ippsGFpECVerifyNR +#define ippsGFpECSignSM2 e9_ippsGFpECSignSM2 +#define ippsGFpECVerifySM2 e9_ippsGFpECVerifySM2 +#define ippsGFpECUserIDHashSM2 e9_ippsGFpECUserIDHashSM2 +#define ippsGFpECKeyExchangeSM2_GetSize e9_ippsGFpECKeyExchangeSM2_GetSize +#define ippsGFpECKeyExchangeSM2_Init e9_ippsGFpECKeyExchangeSM2_Init +#define ippsGFpECKeyExchangeSM2_Setup e9_ippsGFpECKeyExchangeSM2_Setup +#define ippsGFpECKeyExchangeSM2_SharedKey e9_ippsGFpECKeyExchangeSM2_SharedKey +#define ippsGFpECKeyExchangeSM2_Confirm e9_ippsGFpECKeyExchangeSM2_Confirm +#define ippsGFpECGetInfo_GF e9_ippsGFpECGetInfo_GF +#define ippsGFpECESGetSize_SM2 e9_ippsGFpECESGetSize_SM2 +#define ippsGFpECESInit_SM2 e9_ippsGFpECESInit_SM2 +#define ippsGFpECESSetKey_SM2 e9_ippsGFpECESSetKey_SM2 +#define ippsGFpECESStart_SM2 e9_ippsGFpECESStart_SM2 +#define ippsGFpECESEncrypt_SM2 e9_ippsGFpECESEncrypt_SM2 +#define ippsGFpECESDecrypt_SM2 e9_ippsGFpECESDecrypt_SM2 +#define ippsGFpECESFinal_SM2 e9_ippsGFpECESFinal_SM2 +#define ippsGFpECESGetBuffersSize_SM2 e9_ippsGFpECESGetBuffersSize_SM2 +#define ippsGFpECEncryptSM2_Ext_EncMsgSize e9_ippsGFpECEncryptSM2_Ext_EncMsgSize +#define ippsGFpECEncryptSM2_Ext e9_ippsGFpECEncryptSM2_Ext +#define ippsGFpECDecryptSM2_Ext_DecMsgSize e9_ippsGFpECDecryptSM2_Ext_DecMsgSize +#define ippsGFpECDecryptSM2_Ext e9_ippsGFpECDecryptSM2_Ext diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_g9.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_g9.h new file mode 100644 index 0000000..de27b94 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_g9.h @@ -0,0 +1,557 @@ +/******************************************************************************* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + + #define ippcpGetLibVersion g9_ippcpGetLibVersion +#define ippsDESGetSize g9_ippsDESGetSize +#define ippsDESInit g9_ippsDESInit +#define ippsDESPack g9_ippsDESPack +#define ippsDESUnpack g9_ippsDESUnpack +#define ippsTDESEncryptECB g9_ippsTDESEncryptECB +#define ippsTDESDecryptECB g9_ippsTDESDecryptECB +#define ippsTDESEncryptCBC g9_ippsTDESEncryptCBC +#define ippsTDESDecryptCBC g9_ippsTDESDecryptCBC +#define ippsTDESEncryptCFB g9_ippsTDESEncryptCFB +#define ippsTDESDecryptCFB g9_ippsTDESDecryptCFB +#define ippsTDESEncryptOFB g9_ippsTDESEncryptOFB +#define ippsTDESDecryptOFB g9_ippsTDESDecryptOFB +#define ippsTDESEncryptCTR g9_ippsTDESEncryptCTR +#define ippsTDESDecryptCTR g9_ippsTDESDecryptCTR +#define ippsAESGetSize g9_ippsAESGetSize +#define ippsAESInit g9_ippsAESInit +#define ippsAESSetKey g9_ippsAESSetKey +#define ippsAESPack g9_ippsAESPack +#define ippsAESUnpack g9_ippsAESUnpack +#define ippsAESEncryptECB g9_ippsAESEncryptECB +#define ippsAESDecryptECB g9_ippsAESDecryptECB +#define ippsAESEncryptCBC g9_ippsAESEncryptCBC +#define ippsAESEncryptCBC_CS1 g9_ippsAESEncryptCBC_CS1 +#define ippsAESEncryptCBC_CS2 g9_ippsAESEncryptCBC_CS2 +#define ippsAESEncryptCBC_CS3 g9_ippsAESEncryptCBC_CS3 +#define ippsAESDecryptCBC g9_ippsAESDecryptCBC +#define ippsAESDecryptCBC_CS1 g9_ippsAESDecryptCBC_CS1 +#define ippsAESDecryptCBC_CS2 g9_ippsAESDecryptCBC_CS2 +#define ippsAESDecryptCBC_CS3 g9_ippsAESDecryptCBC_CS3 +#define ippsAESEncryptCFB g9_ippsAESEncryptCFB +#define ippsAESDecryptCFB g9_ippsAESDecryptCFB +#define ippsAESEncryptOFB g9_ippsAESEncryptOFB +#define ippsAESDecryptOFB g9_ippsAESDecryptOFB +#define ippsAESEncryptCTR g9_ippsAESEncryptCTR +#define ippsAESDecryptCTR g9_ippsAESDecryptCTR +#define ippsAESEncryptXTS_Direct g9_ippsAESEncryptXTS_Direct +#define ippsAESDecryptXTS_Direct g9_ippsAESDecryptXTS_Direct +#define ippsAESSetupNoise g9_ippsAESSetupNoise +#define ippsAES_GCMSetupNoise g9_ippsAES_GCMSetupNoise +#define ippsAES_CMACSetupNoise g9_ippsAES_CMACSetupNoise +#define ippsAES_EncryptCFB16_MB g9_ippsAES_EncryptCFB16_MB +#define ippsSMS4GetSize g9_ippsSMS4GetSize +#define ippsSMS4Init g9_ippsSMS4Init +#define ippsSMS4SetKey g9_ippsSMS4SetKey +#define ippsSMS4EncryptECB g9_ippsSMS4EncryptECB +#define ippsSMS4DecryptECB g9_ippsSMS4DecryptECB +#define ippsSMS4EncryptCBC g9_ippsSMS4EncryptCBC +#define ippsSMS4EncryptCBC_CS1 g9_ippsSMS4EncryptCBC_CS1 +#define ippsSMS4EncryptCBC_CS2 g9_ippsSMS4EncryptCBC_CS2 +#define ippsSMS4EncryptCBC_CS3 g9_ippsSMS4EncryptCBC_CS3 +#define ippsSMS4DecryptCBC g9_ippsSMS4DecryptCBC +#define ippsSMS4DecryptCBC_CS1 g9_ippsSMS4DecryptCBC_CS1 +#define ippsSMS4DecryptCBC_CS2 g9_ippsSMS4DecryptCBC_CS2 +#define ippsSMS4DecryptCBC_CS3 g9_ippsSMS4DecryptCBC_CS3 +#define ippsSMS4EncryptCFB g9_ippsSMS4EncryptCFB +#define ippsSMS4DecryptCFB g9_ippsSMS4DecryptCFB +#define ippsSMS4EncryptOFB g9_ippsSMS4EncryptOFB +#define ippsSMS4DecryptOFB g9_ippsSMS4DecryptOFB +#define ippsSMS4EncryptCTR g9_ippsSMS4EncryptCTR +#define ippsSMS4DecryptCTR g9_ippsSMS4DecryptCTR +#define ippsSMS4_CCMGetSize g9_ippsSMS4_CCMGetSize +#define ippsSMS4_CCMInit g9_ippsSMS4_CCMInit +#define ippsSMS4_CCMMessageLen g9_ippsSMS4_CCMMessageLen +#define ippsSMS4_CCMTagLen g9_ippsSMS4_CCMTagLen +#define ippsSMS4_CCMStart g9_ippsSMS4_CCMStart +#define ippsSMS4_CCMEncrypt g9_ippsSMS4_CCMEncrypt +#define ippsSMS4_CCMDecrypt g9_ippsSMS4_CCMDecrypt +#define ippsSMS4_CCMGetTag g9_ippsSMS4_CCMGetTag +#define ippsAES_CCMGetSize g9_ippsAES_CCMGetSize +#define ippsAES_CCMInit g9_ippsAES_CCMInit +#define ippsAES_CCMMessageLen g9_ippsAES_CCMMessageLen +#define ippsAES_CCMTagLen g9_ippsAES_CCMTagLen +#define ippsAES_CCMStart g9_ippsAES_CCMStart +#define ippsAES_CCMEncrypt g9_ippsAES_CCMEncrypt +#define ippsAES_CCMDecrypt g9_ippsAES_CCMDecrypt +#define ippsAES_CCMGetTag g9_ippsAES_CCMGetTag +#define ippsAES_GCMGetSize g9_ippsAES_GCMGetSize +#define ippsAES_GCMInit g9_ippsAES_GCMInit +#define ippsAES_GCMReinit g9_ippsAES_GCMReinit +#define ippsAES_GCMReset g9_ippsAES_GCMReset +#define ippsAES_GCMProcessIV g9_ippsAES_GCMProcessIV +#define ippsAES_GCMProcessAAD g9_ippsAES_GCMProcessAAD +#define ippsAES_GCMStart g9_ippsAES_GCMStart +#define ippsAES_GCMEncrypt g9_ippsAES_GCMEncrypt +#define ippsAES_GCMDecrypt g9_ippsAES_GCMDecrypt +#define ippsAES_GCMGetTag g9_ippsAES_GCMGetTag +#define ippsAES_XTSGetSize g9_ippsAES_XTSGetSize +#define ippsAES_XTSInit g9_ippsAES_XTSInit +#define ippsAES_XTSEncrypt g9_ippsAES_XTSEncrypt +#define ippsAES_XTSDecrypt g9_ippsAES_XTSDecrypt +#define ippsAES_S2V_CMAC g9_ippsAES_S2V_CMAC +#define ippsAES_SIVEncrypt g9_ippsAES_SIVEncrypt +#define ippsAES_SIVDecrypt g9_ippsAES_SIVDecrypt +#define ippsAES_CMACGetSize g9_ippsAES_CMACGetSize +#define ippsAES_CMACInit g9_ippsAES_CMACInit +#define ippsAES_CMACUpdate g9_ippsAES_CMACUpdate +#define ippsAES_CMACFinal g9_ippsAES_CMACFinal +#define ippsAES_CMACGetTag g9_ippsAES_CMACGetTag +#define ippsARCFourCheckKey g9_ippsARCFourCheckKey +#define ippsARCFourGetSize g9_ippsARCFourGetSize +#define ippsARCFourInit g9_ippsARCFourInit +#define ippsARCFourReset g9_ippsARCFourReset +#define ippsARCFourPack g9_ippsARCFourPack +#define ippsARCFourUnpack g9_ippsARCFourUnpack +#define ippsARCFourEncrypt g9_ippsARCFourEncrypt +#define ippsARCFourDecrypt g9_ippsARCFourDecrypt +#define ippsSHA1GetSize g9_ippsSHA1GetSize +#define ippsSHA1Init g9_ippsSHA1Init +#define ippsSHA1Duplicate g9_ippsSHA1Duplicate +#define ippsSHA1Pack g9_ippsSHA1Pack +#define ippsSHA1Unpack g9_ippsSHA1Unpack +#define ippsSHA1Update g9_ippsSHA1Update +#define ippsSHA1GetTag g9_ippsSHA1GetTag +#define ippsSHA1Final g9_ippsSHA1Final +#define ippsSHA1MessageDigest g9_ippsSHA1MessageDigest +#define ippsSHA224GetSize g9_ippsSHA224GetSize +#define ippsSHA224Init g9_ippsSHA224Init +#define ippsSHA224Duplicate g9_ippsSHA224Duplicate +#define ippsSHA224Pack g9_ippsSHA224Pack +#define ippsSHA224Unpack g9_ippsSHA224Unpack +#define ippsSHA224Update g9_ippsSHA224Update +#define ippsSHA224GetTag g9_ippsSHA224GetTag +#define ippsSHA224Final g9_ippsSHA224Final +#define ippsSHA224MessageDigest g9_ippsSHA224MessageDigest +#define ippsSHA256GetSize g9_ippsSHA256GetSize +#define ippsSHA256Init g9_ippsSHA256Init +#define ippsSHA256Duplicate g9_ippsSHA256Duplicate +#define ippsSHA256Pack g9_ippsSHA256Pack +#define ippsSHA256Unpack g9_ippsSHA256Unpack +#define ippsSHA256Update g9_ippsSHA256Update +#define ippsSHA256GetTag g9_ippsSHA256GetTag +#define ippsSHA256Final g9_ippsSHA256Final +#define ippsSHA256MessageDigest g9_ippsSHA256MessageDigest +#define ippsSHA384GetSize g9_ippsSHA384GetSize +#define ippsSHA384Init g9_ippsSHA384Init +#define ippsSHA384Duplicate g9_ippsSHA384Duplicate +#define ippsSHA384Pack g9_ippsSHA384Pack +#define ippsSHA384Unpack g9_ippsSHA384Unpack +#define ippsSHA384Update g9_ippsSHA384Update +#define ippsSHA384GetTag g9_ippsSHA384GetTag +#define ippsSHA384Final g9_ippsSHA384Final +#define ippsSHA384MessageDigest g9_ippsSHA384MessageDigest +#define ippsSHA512GetSize g9_ippsSHA512GetSize +#define ippsSHA512Init g9_ippsSHA512Init +#define ippsSHA512Duplicate g9_ippsSHA512Duplicate +#define ippsSHA512Pack g9_ippsSHA512Pack +#define ippsSHA512Unpack g9_ippsSHA512Unpack +#define ippsSHA512Update g9_ippsSHA512Update +#define ippsSHA512GetTag g9_ippsSHA512GetTag +#define ippsSHA512Final g9_ippsSHA512Final +#define ippsSHA512MessageDigest g9_ippsSHA512MessageDigest +#define ippsMD5GetSize g9_ippsMD5GetSize +#define ippsMD5Init g9_ippsMD5Init +#define ippsMD5Duplicate g9_ippsMD5Duplicate +#define ippsMD5Pack g9_ippsMD5Pack +#define ippsMD5Unpack g9_ippsMD5Unpack +#define ippsMD5Update g9_ippsMD5Update +#define ippsMD5GetTag g9_ippsMD5GetTag +#define ippsMD5Final g9_ippsMD5Final +#define ippsMD5MessageDigest g9_ippsMD5MessageDigest +#define ippsSM3GetSize g9_ippsSM3GetSize +#define ippsSM3Init g9_ippsSM3Init +#define ippsSM3Duplicate g9_ippsSM3Duplicate +#define ippsSM3Pack g9_ippsSM3Pack +#define ippsSM3Unpack g9_ippsSM3Unpack +#define ippsSM3Update g9_ippsSM3Update +#define ippsSM3GetTag g9_ippsSM3GetTag +#define ippsSM3Final g9_ippsSM3Final +#define ippsSM3MessageDigest g9_ippsSM3MessageDigest +#define ippsHashGetSize g9_ippsHashGetSize +#define ippsHashInit g9_ippsHashInit +#define ippsHashPack g9_ippsHashPack +#define ippsHashUnpack g9_ippsHashUnpack +#define ippsHashDuplicate g9_ippsHashDuplicate +#define ippsHashUpdate g9_ippsHashUpdate +#define ippsHashGetTag g9_ippsHashGetTag +#define ippsHashFinal g9_ippsHashFinal +#define ippsHashMessage g9_ippsHashMessage +#define ippsHashMethod_MD5 g9_ippsHashMethod_MD5 +#define ippsHashMethod_SM3 g9_ippsHashMethod_SM3 +#define ippsHashMethod_SHA1 g9_ippsHashMethod_SHA1 +#define ippsHashMethod_SHA1_NI g9_ippsHashMethod_SHA1_NI +#define ippsHashMethod_SHA1_TT g9_ippsHashMethod_SHA1_TT +#define ippsHashMethod_SHA256 g9_ippsHashMethod_SHA256 +#define ippsHashMethod_SHA256_NI g9_ippsHashMethod_SHA256_NI +#define ippsHashMethod_SHA256_TT g9_ippsHashMethod_SHA256_TT +#define ippsHashMethod_SHA224 g9_ippsHashMethod_SHA224 +#define ippsHashMethod_SHA224_NI g9_ippsHashMethod_SHA224_NI +#define ippsHashMethod_SHA224_TT g9_ippsHashMethod_SHA224_TT +#define ippsHashMethod_SHA512 g9_ippsHashMethod_SHA512 +#define ippsHashMethod_SHA384 g9_ippsHashMethod_SHA384 +#define ippsHashMethod_SHA512_256 g9_ippsHashMethod_SHA512_256 +#define ippsHashMethod_SHA512_224 g9_ippsHashMethod_SHA512_224 +#define ippsHashMethodGetSize g9_ippsHashMethodGetSize +#define ippsHashMethodSet_MD5 g9_ippsHashMethodSet_MD5 +#define ippsHashMethodSet_SM3 g9_ippsHashMethodSet_SM3 +#define ippsHashStateMethodSet_SM3 g9_ippsHashStateMethodSet_SM3 +#define ippsHashMethodSet_SHA1 g9_ippsHashMethodSet_SHA1 +#define ippsHashMethodSet_SHA1_NI g9_ippsHashMethodSet_SHA1_NI +#define ippsHashMethodSet_SHA1_TT g9_ippsHashMethodSet_SHA1_TT +#define ippsHashMethodSet_SHA256 g9_ippsHashMethodSet_SHA256 +#define ippsHashMethodSet_SHA256_NI g9_ippsHashMethodSet_SHA256_NI +#define ippsHashMethodSet_SHA256_TT g9_ippsHashMethodSet_SHA256_TT +#define ippsHashStateMethodSet_SHA256 g9_ippsHashStateMethodSet_SHA256 +#define ippsHashStateMethodSet_SHA256_NI g9_ippsHashStateMethodSet_SHA256_NI +#define ippsHashStateMethodSet_SHA256_TT g9_ippsHashStateMethodSet_SHA256_TT +#define ippsHashMethodSet_SHA224 g9_ippsHashMethodSet_SHA224 +#define ippsHashMethodSet_SHA224_NI g9_ippsHashMethodSet_SHA224_NI +#define ippsHashMethodSet_SHA224_TT g9_ippsHashMethodSet_SHA224_TT +#define ippsHashStateMethodSet_SHA224 g9_ippsHashStateMethodSet_SHA224 +#define ippsHashStateMethodSet_SHA224_NI g9_ippsHashStateMethodSet_SHA224_NI +#define ippsHashStateMethodSet_SHA224_TT g9_ippsHashStateMethodSet_SHA224_TT +#define ippsHashMethodSet_SHA512 g9_ippsHashMethodSet_SHA512 +#define ippsHashMethodSet_SHA384 g9_ippsHashMethodSet_SHA384 +#define ippsHashMethodSet_SHA512_256 g9_ippsHashMethodSet_SHA512_256 +#define ippsHashMethodSet_SHA512_224 g9_ippsHashMethodSet_SHA512_224 +#define ippsHashStateMethodSet_SHA512 g9_ippsHashStateMethodSet_SHA512 +#define ippsHashStateMethodSet_SHA384 g9_ippsHashStateMethodSet_SHA384 +#define ippsHashStateMethodSet_SHA512_256 g9_ippsHashStateMethodSet_SHA512_256 +#define ippsHashStateMethodSet_SHA512_224 g9_ippsHashStateMethodSet_SHA512_224 +#define ippsHashGetSize_rmf g9_ippsHashGetSize_rmf +#define ippsHashInit_rmf g9_ippsHashInit_rmf +#define ippsHashPack_rmf g9_ippsHashPack_rmf +#define ippsHashUnpack_rmf g9_ippsHashUnpack_rmf +#define ippsHashDuplicate_rmf g9_ippsHashDuplicate_rmf +#define ippsHashUpdate_rmf g9_ippsHashUpdate_rmf +#define ippsHashGetTag_rmf g9_ippsHashGetTag_rmf +#define ippsHashFinal_rmf g9_ippsHashFinal_rmf +#define ippsHashMessage_rmf g9_ippsHashMessage_rmf +#define ippsHashMethodGetInfo g9_ippsHashMethodGetInfo +#define ippsHashGetInfo_rmf g9_ippsHashGetInfo_rmf +#define ippsMGF g9_ippsMGF +#define ippsMGF1_rmf g9_ippsMGF1_rmf +#define ippsMGF2_rmf g9_ippsMGF2_rmf +#define ippsHMAC_GetSize g9_ippsHMAC_GetSize +#define ippsHMAC_Init g9_ippsHMAC_Init +#define ippsHMAC_Pack g9_ippsHMAC_Pack +#define ippsHMAC_Unpack g9_ippsHMAC_Unpack +#define ippsHMAC_Duplicate g9_ippsHMAC_Duplicate +#define ippsHMAC_Update g9_ippsHMAC_Update +#define ippsHMAC_Final g9_ippsHMAC_Final +#define ippsHMAC_GetTag g9_ippsHMAC_GetTag +#define ippsHMAC_Message g9_ippsHMAC_Message +#define ippsHMACGetSize_rmf g9_ippsHMACGetSize_rmf +#define ippsHMACInit_rmf g9_ippsHMACInit_rmf +#define ippsHMACPack_rmf g9_ippsHMACPack_rmf +#define ippsHMACUnpack_rmf g9_ippsHMACUnpack_rmf +#define ippsHMACDuplicate_rmf g9_ippsHMACDuplicate_rmf +#define ippsHMACUpdate_rmf g9_ippsHMACUpdate_rmf +#define ippsHMACFinal_rmf g9_ippsHMACFinal_rmf +#define ippsHMACGetTag_rmf g9_ippsHMACGetTag_rmf +#define ippsHMACMessage_rmf g9_ippsHMACMessage_rmf +#define ippsBigNumGetSize g9_ippsBigNumGetSize +#define ippsBigNumInit g9_ippsBigNumInit +#define ippsCmpZero_BN g9_ippsCmpZero_BN +#define ippsCmp_BN g9_ippsCmp_BN +#define ippsGetSize_BN g9_ippsGetSize_BN +#define ippsSet_BN g9_ippsSet_BN +#define ippsGet_BN g9_ippsGet_BN +#define ippsRef_BN g9_ippsRef_BN +#define ippsExtGet_BN g9_ippsExtGet_BN +#define ippsAdd_BN g9_ippsAdd_BN +#define ippsSub_BN g9_ippsSub_BN +#define ippsMul_BN g9_ippsMul_BN +#define ippsMAC_BN_I g9_ippsMAC_BN_I +#define ippsDiv_BN g9_ippsDiv_BN +#define ippsMod_BN g9_ippsMod_BN +#define ippsGcd_BN g9_ippsGcd_BN +#define ippsModInv_BN g9_ippsModInv_BN +#define ippsSetOctString_BN g9_ippsSetOctString_BN +#define ippsGetOctString_BN g9_ippsGetOctString_BN +#define ippsMontGetSize g9_ippsMontGetSize +#define ippsMontInit g9_ippsMontInit +#define ippsMontSet g9_ippsMontSet +#define ippsMontGet g9_ippsMontGet +#define ippsMontForm g9_ippsMontForm +#define ippsMontMul g9_ippsMontMul +#define ippsMontExp g9_ippsMontExp +#define ippsPRNGGetSize g9_ippsPRNGGetSize +#define ippsPRNGInit g9_ippsPRNGInit +#define ippsPRNGSetModulus g9_ippsPRNGSetModulus +#define ippsPRNGSetH0 g9_ippsPRNGSetH0 +#define ippsPRNGSetAugment g9_ippsPRNGSetAugment +#define ippsPRNGSetSeed g9_ippsPRNGSetSeed +#define ippsPRNGGetSeed g9_ippsPRNGGetSeed +#define ippsPRNGen g9_ippsPRNGen +#define ippsPRNGen_BN g9_ippsPRNGen_BN +#define ippsPRNGenRDRAND g9_ippsPRNGenRDRAND +#define ippsPRNGenRDRAND_BN g9_ippsPRNGenRDRAND_BN +#define ippsTRNGenRDSEED g9_ippsTRNGenRDSEED +#define ippsTRNGenRDSEED_BN g9_ippsTRNGenRDSEED_BN +#define ippsPrimeGetSize g9_ippsPrimeGetSize +#define ippsPrimeInit g9_ippsPrimeInit +#define ippsPrimeGen g9_ippsPrimeGen +#define ippsPrimeTest g9_ippsPrimeTest +#define ippsPrimeGen_BN g9_ippsPrimeGen_BN +#define ippsPrimeTest_BN g9_ippsPrimeTest_BN +#define ippsPrimeGet g9_ippsPrimeGet +#define ippsPrimeGet_BN g9_ippsPrimeGet_BN +#define ippsPrimeSet g9_ippsPrimeSet +#define ippsPrimeSet_BN g9_ippsPrimeSet_BN +#define ippsRSA_GetSizePublicKey g9_ippsRSA_GetSizePublicKey +#define ippsRSA_InitPublicKey g9_ippsRSA_InitPublicKey +#define ippsRSA_SetPublicKey g9_ippsRSA_SetPublicKey +#define ippsRSA_GetPublicKey g9_ippsRSA_GetPublicKey +#define ippsRSA_GetSizePrivateKeyType1 g9_ippsRSA_GetSizePrivateKeyType1 +#define ippsRSA_InitPrivateKeyType1 g9_ippsRSA_InitPrivateKeyType1 +#define ippsRSA_SetPrivateKeyType1 g9_ippsRSA_SetPrivateKeyType1 +#define ippsRSA_GetPrivateKeyType1 g9_ippsRSA_GetPrivateKeyType1 +#define ippsRSA_GetSizePrivateKeyType2 g9_ippsRSA_GetSizePrivateKeyType2 +#define ippsRSA_InitPrivateKeyType2 g9_ippsRSA_InitPrivateKeyType2 +#define ippsRSA_SetPrivateKeyType2 g9_ippsRSA_SetPrivateKeyType2 +#define ippsRSA_GetPrivateKeyType2 g9_ippsRSA_GetPrivateKeyType2 +#define ippsRSA_GetBufferSizePublicKey g9_ippsRSA_GetBufferSizePublicKey +#define ippsRSA_GetBufferSizePrivateKey g9_ippsRSA_GetBufferSizePrivateKey +#define ippsRSA_Encrypt g9_ippsRSA_Encrypt +#define ippsRSA_Decrypt g9_ippsRSA_Decrypt +#define ippsRSA_GenerateKeys g9_ippsRSA_GenerateKeys +#define ippsRSA_ValidateKeys g9_ippsRSA_ValidateKeys +#define ippsRSAEncrypt_OAEP g9_ippsRSAEncrypt_OAEP +#define ippsRSADecrypt_OAEP g9_ippsRSADecrypt_OAEP +#define ippsRSAEncrypt_OAEP_rmf g9_ippsRSAEncrypt_OAEP_rmf +#define ippsRSADecrypt_OAEP_rmf g9_ippsRSADecrypt_OAEP_rmf +#define ippsRSAEncrypt_PKCSv15 g9_ippsRSAEncrypt_PKCSv15 +#define ippsRSADecrypt_PKCSv15 g9_ippsRSADecrypt_PKCSv15 +#define ippsRSASign_PSS g9_ippsRSASign_PSS +#define ippsRSAVerify_PSS g9_ippsRSAVerify_PSS +#define ippsRSASign_PSS_rmf g9_ippsRSASign_PSS_rmf +#define ippsRSAVerify_PSS_rmf g9_ippsRSAVerify_PSS_rmf +#define ippsRSASign_PKCS1v15 g9_ippsRSASign_PKCS1v15 +#define ippsRSAVerify_PKCS1v15 g9_ippsRSAVerify_PKCS1v15 +#define ippsRSASign_PKCS1v15_rmf g9_ippsRSASign_PKCS1v15_rmf +#define ippsRSAVerify_PKCS1v15_rmf g9_ippsRSAVerify_PKCS1v15_rmf +#define ippsDLGetResultString g9_ippsDLGetResultString +#define ippsDLPGetSize g9_ippsDLPGetSize +#define ippsDLPInit g9_ippsDLPInit +#define ippsDLPPack g9_ippsDLPPack +#define ippsDLPUnpack g9_ippsDLPUnpack +#define ippsDLPSet g9_ippsDLPSet +#define ippsDLPGet g9_ippsDLPGet +#define ippsDLPSetDP g9_ippsDLPSetDP +#define ippsDLPGetDP g9_ippsDLPGetDP +#define ippsDLPGenKeyPair g9_ippsDLPGenKeyPair +#define ippsDLPPublicKey g9_ippsDLPPublicKey +#define ippsDLPValidateKeyPair g9_ippsDLPValidateKeyPair +#define ippsDLPSetKeyPair g9_ippsDLPSetKeyPair +#define ippsDLPSignDSA g9_ippsDLPSignDSA +#define ippsDLPVerifyDSA g9_ippsDLPVerifyDSA +#define ippsDLPSharedSecretDH g9_ippsDLPSharedSecretDH +#define ippsDLPGenerateDSA g9_ippsDLPGenerateDSA +#define ippsDLPValidateDSA g9_ippsDLPValidateDSA +#define ippsDLPGenerateDH g9_ippsDLPGenerateDH +#define ippsDLPValidateDH g9_ippsDLPValidateDH +#define ippsECCGetResultString g9_ippsECCGetResultString +#define ippsECCPGetSize g9_ippsECCPGetSize +#define ippsECCPGetSizeStd128r1 g9_ippsECCPGetSizeStd128r1 +#define ippsECCPGetSizeStd128r2 g9_ippsECCPGetSizeStd128r2 +#define ippsECCPGetSizeStd192r1 g9_ippsECCPGetSizeStd192r1 +#define ippsECCPGetSizeStd224r1 g9_ippsECCPGetSizeStd224r1 +#define ippsECCPGetSizeStd256r1 g9_ippsECCPGetSizeStd256r1 +#define ippsECCPGetSizeStd384r1 g9_ippsECCPGetSizeStd384r1 +#define ippsECCPGetSizeStd521r1 g9_ippsECCPGetSizeStd521r1 +#define ippsECCPGetSizeStdSM2 g9_ippsECCPGetSizeStdSM2 +#define ippsECCPInit g9_ippsECCPInit +#define ippsECCPInitStd128r1 g9_ippsECCPInitStd128r1 +#define ippsECCPInitStd128r2 g9_ippsECCPInitStd128r2 +#define ippsECCPInitStd192r1 g9_ippsECCPInitStd192r1 +#define ippsECCPInitStd224r1 g9_ippsECCPInitStd224r1 +#define ippsECCPInitStd256r1 g9_ippsECCPInitStd256r1 +#define ippsECCPInitStd384r1 g9_ippsECCPInitStd384r1 +#define ippsECCPInitStd521r1 g9_ippsECCPInitStd521r1 +#define ippsECCPInitStdSM2 g9_ippsECCPInitStdSM2 +#define ippsECCPSet g9_ippsECCPSet +#define ippsECCPSetStd g9_ippsECCPSetStd +#define ippsECCPSetStd128r1 g9_ippsECCPSetStd128r1 +#define ippsECCPSetStd128r2 g9_ippsECCPSetStd128r2 +#define ippsECCPSetStd192r1 g9_ippsECCPSetStd192r1 +#define ippsECCPSetStd224r1 g9_ippsECCPSetStd224r1 +#define ippsECCPSetStd256r1 g9_ippsECCPSetStd256r1 +#define ippsECCPSetStd384r1 g9_ippsECCPSetStd384r1 +#define ippsECCPSetStd521r1 g9_ippsECCPSetStd521r1 +#define ippsECCPSetStdSM2 g9_ippsECCPSetStdSM2 +#define ippsECCPBindGxyTblStd192r1 g9_ippsECCPBindGxyTblStd192r1 +#define ippsECCPBindGxyTblStd224r1 g9_ippsECCPBindGxyTblStd224r1 +#define ippsECCPBindGxyTblStd256r1 g9_ippsECCPBindGxyTblStd256r1 +#define ippsECCPBindGxyTblStd384r1 g9_ippsECCPBindGxyTblStd384r1 +#define ippsECCPBindGxyTblStd521r1 g9_ippsECCPBindGxyTblStd521r1 +#define ippsECCPBindGxyTblStdSM2 g9_ippsECCPBindGxyTblStdSM2 +#define ippsECCPGet g9_ippsECCPGet +#define ippsECCPGetOrderBitSize g9_ippsECCPGetOrderBitSize +#define ippsECCPValidate g9_ippsECCPValidate +#define ippsECCPPointGetSize g9_ippsECCPPointGetSize +#define ippsECCPPointInit g9_ippsECCPPointInit +#define ippsECCPSetPoint g9_ippsECCPSetPoint +#define ippsECCPSetPointAtInfinity g9_ippsECCPSetPointAtInfinity +#define ippsECCPGetPoint g9_ippsECCPGetPoint +#define ippsECCPCheckPoint g9_ippsECCPCheckPoint +#define ippsECCPComparePoint g9_ippsECCPComparePoint +#define ippsECCPNegativePoint g9_ippsECCPNegativePoint +#define ippsECCPAddPoint g9_ippsECCPAddPoint +#define ippsECCPMulPointScalar g9_ippsECCPMulPointScalar +#define ippsECCPGenKeyPair g9_ippsECCPGenKeyPair +#define ippsECCPPublicKey g9_ippsECCPPublicKey +#define ippsECCPValidateKeyPair g9_ippsECCPValidateKeyPair +#define ippsECCPSetKeyPair g9_ippsECCPSetKeyPair +#define ippsECCPSharedSecretDH g9_ippsECCPSharedSecretDH +#define ippsECCPSharedSecretDHC g9_ippsECCPSharedSecretDHC +#define ippsECCPSignDSA g9_ippsECCPSignDSA +#define ippsECCPVerifyDSA g9_ippsECCPVerifyDSA +#define ippsECCPSignNR g9_ippsECCPSignNR +#define ippsECCPVerifyNR g9_ippsECCPVerifyNR +#define ippsECCPSignSM2 g9_ippsECCPSignSM2 +#define ippsECCPVerifySM2 g9_ippsECCPVerifySM2 +#define ippsGFpGetSize g9_ippsGFpGetSize +#define ippsGFpInitArbitrary g9_ippsGFpInitArbitrary +#define ippsGFpInitFixed g9_ippsGFpInitFixed +#define ippsGFpInit g9_ippsGFpInit +#define ippsGFpMethod_p192r1 g9_ippsGFpMethod_p192r1 +#define ippsGFpMethod_p224r1 g9_ippsGFpMethod_p224r1 +#define ippsGFpMethod_p256r1 g9_ippsGFpMethod_p256r1 +#define ippsGFpMethod_p384r1 g9_ippsGFpMethod_p384r1 +#define ippsGFpMethod_p521r1 g9_ippsGFpMethod_p521r1 +#define ippsGFpMethod_p256sm2 g9_ippsGFpMethod_p256sm2 +#define ippsGFpMethod_p256bn g9_ippsGFpMethod_p256bn +#define ippsGFpMethod_p256 g9_ippsGFpMethod_p256 +#define ippsGFpMethod_pArb g9_ippsGFpMethod_pArb +#define ippsGFpxGetSize g9_ippsGFpxGetSize +#define ippsGFpxInit g9_ippsGFpxInit +#define ippsGFpxInitBinomial g9_ippsGFpxInitBinomial +#define ippsGFpxMethod_binom2_epid2 g9_ippsGFpxMethod_binom2_epid2 +#define ippsGFpxMethod_binom3_epid2 g9_ippsGFpxMethod_binom3_epid2 +#define ippsGFpxMethod_binom2 g9_ippsGFpxMethod_binom2 +#define ippsGFpxMethod_binom3 g9_ippsGFpxMethod_binom3 +#define ippsGFpxMethod_binom g9_ippsGFpxMethod_binom +#define ippsGFpxMethod_com g9_ippsGFpxMethod_com +#define ippsGFpScratchBufferSize g9_ippsGFpScratchBufferSize +#define ippsGFpElementGetSize g9_ippsGFpElementGetSize +#define ippsGFpElementInit g9_ippsGFpElementInit +#define ippsGFpSetElement g9_ippsGFpSetElement +#define ippsGFpSetElementRegular g9_ippsGFpSetElementRegular +#define ippsGFpSetElementOctString g9_ippsGFpSetElementOctString +#define ippsGFpSetElementRandom g9_ippsGFpSetElementRandom +#define ippsGFpSetElementHash g9_ippsGFpSetElementHash +#define ippsGFpSetElementHash_rmf g9_ippsGFpSetElementHash_rmf +#define ippsGFpCpyElement g9_ippsGFpCpyElement +#define ippsGFpGetElement g9_ippsGFpGetElement +#define ippsGFpGetElementOctString g9_ippsGFpGetElementOctString +#define ippsGFpCmpElement g9_ippsGFpCmpElement +#define ippsGFpIsZeroElement g9_ippsGFpIsZeroElement +#define ippsGFpIsUnityElement g9_ippsGFpIsUnityElement +#define ippsGFpConj g9_ippsGFpConj +#define ippsGFpNeg g9_ippsGFpNeg +#define ippsGFpInv g9_ippsGFpInv +#define ippsGFpSqrt g9_ippsGFpSqrt +#define ippsGFpSqr g9_ippsGFpSqr +#define ippsGFpAdd g9_ippsGFpAdd +#define ippsGFpSub g9_ippsGFpSub +#define ippsGFpMul g9_ippsGFpMul +#define ippsGFpExp g9_ippsGFpExp +#define ippsGFpMultiExp g9_ippsGFpMultiExp +#define ippsGFpAdd_PE g9_ippsGFpAdd_PE +#define ippsGFpSub_PE g9_ippsGFpSub_PE +#define ippsGFpMul_PE g9_ippsGFpMul_PE +#define ippsGFpGetInfo g9_ippsGFpGetInfo +#define ippsGFpECGetSize g9_ippsGFpECGetSize +#define ippsGFpECInit g9_ippsGFpECInit +#define ippsGFpECSet g9_ippsGFpECSet +#define ippsGFpECSetSubgroup g9_ippsGFpECSetSubgroup +#define ippsGFpECInitStd128r1 g9_ippsGFpECInitStd128r1 +#define ippsGFpECInitStd128r2 g9_ippsGFpECInitStd128r2 +#define ippsGFpECInitStd192r1 g9_ippsGFpECInitStd192r1 +#define ippsGFpECInitStd224r1 g9_ippsGFpECInitStd224r1 +#define ippsGFpECInitStd256r1 g9_ippsGFpECInitStd256r1 +#define ippsGFpECInitStd384r1 g9_ippsGFpECInitStd384r1 +#define ippsGFpECInitStd521r1 g9_ippsGFpECInitStd521r1 +#define ippsGFpECInitStdSM2 g9_ippsGFpECInitStdSM2 +#define ippsGFpECInitStdBN256 g9_ippsGFpECInitStdBN256 +#define ippsGFpECBindGxyTblStd192r1 g9_ippsGFpECBindGxyTblStd192r1 +#define ippsGFpECBindGxyTblStd224r1 g9_ippsGFpECBindGxyTblStd224r1 +#define ippsGFpECBindGxyTblStd256r1 g9_ippsGFpECBindGxyTblStd256r1 +#define ippsGFpECBindGxyTblStd384r1 g9_ippsGFpECBindGxyTblStd384r1 +#define ippsGFpECBindGxyTblStd521r1 g9_ippsGFpECBindGxyTblStd521r1 +#define ippsGFpECBindGxyTblStdSM2 g9_ippsGFpECBindGxyTblStdSM2 +#define ippsGFpECGet g9_ippsGFpECGet +#define ippsGFpECGetSubgroup g9_ippsGFpECGetSubgroup +#define ippsGFpECScratchBufferSize g9_ippsGFpECScratchBufferSize +#define ippsGFpECVerify g9_ippsGFpECVerify +#define ippsGFpECPointGetSize g9_ippsGFpECPointGetSize +#define ippsGFpECPointInit g9_ippsGFpECPointInit +#define ippsGFpECSetPointAtInfinity g9_ippsGFpECSetPointAtInfinity +#define ippsGFpECSetPoint g9_ippsGFpECSetPoint +#define ippsGFpECSetPointRegular g9_ippsGFpECSetPointRegular +#define ippsGFpECSetPointRandom g9_ippsGFpECSetPointRandom +#define ippsGFpECMakePoint g9_ippsGFpECMakePoint +#define ippsGFpECSetPointHash g9_ippsGFpECSetPointHash +#define ippsGFpECSetPointHashBackCompatible g9_ippsGFpECSetPointHashBackCompatible +#define ippsGFpECSetPointHash_rmf g9_ippsGFpECSetPointHash_rmf +#define ippsGFpECSetPointHashBackCompatible_rmf g9_ippsGFpECSetPointHashBackCompatible_rmf +#define ippsGFpECGetPoint g9_ippsGFpECGetPoint +#define ippsGFpECGetPointRegular g9_ippsGFpECGetPointRegular +#define ippsGFpECSetPointOctString g9_ippsGFpECSetPointOctString +#define ippsGFpECGetPointOctString g9_ippsGFpECGetPointOctString +#define ippsGFpECTstPoint g9_ippsGFpECTstPoint +#define ippsGFpECTstPointInSubgroup g9_ippsGFpECTstPointInSubgroup +#define ippsGFpECCpyPoint g9_ippsGFpECCpyPoint +#define ippsGFpECCmpPoint g9_ippsGFpECCmpPoint +#define ippsGFpECNegPoint g9_ippsGFpECNegPoint +#define ippsGFpECAddPoint g9_ippsGFpECAddPoint +#define ippsGFpECMulPoint g9_ippsGFpECMulPoint +#define ippsGFpECPrivateKey g9_ippsGFpECPrivateKey +#define ippsGFpECPublicKey g9_ippsGFpECPublicKey +#define ippsGFpECTstKeyPair g9_ippsGFpECTstKeyPair +#define ippsGFpECSharedSecretDH g9_ippsGFpECSharedSecretDH +#define ippsGFpECSharedSecretDHC g9_ippsGFpECSharedSecretDHC +#define ippsGFpECMessageRepresentationSM2 g9_ippsGFpECMessageRepresentationSM2 +#define ippsGFpECSignDSA g9_ippsGFpECSignDSA +#define ippsGFpECVerifyDSA g9_ippsGFpECVerifyDSA +#define ippsGFpECSignNR g9_ippsGFpECSignNR +#define ippsGFpECVerifyNR g9_ippsGFpECVerifyNR +#define ippsGFpECSignSM2 g9_ippsGFpECSignSM2 +#define ippsGFpECVerifySM2 g9_ippsGFpECVerifySM2 +#define ippsGFpECUserIDHashSM2 g9_ippsGFpECUserIDHashSM2 +#define ippsGFpECKeyExchangeSM2_GetSize g9_ippsGFpECKeyExchangeSM2_GetSize +#define ippsGFpECKeyExchangeSM2_Init g9_ippsGFpECKeyExchangeSM2_Init +#define ippsGFpECKeyExchangeSM2_Setup g9_ippsGFpECKeyExchangeSM2_Setup +#define ippsGFpECKeyExchangeSM2_SharedKey g9_ippsGFpECKeyExchangeSM2_SharedKey +#define ippsGFpECKeyExchangeSM2_Confirm g9_ippsGFpECKeyExchangeSM2_Confirm +#define ippsGFpECGetInfo_GF g9_ippsGFpECGetInfo_GF +#define ippsGFpECESGetSize_SM2 g9_ippsGFpECESGetSize_SM2 +#define ippsGFpECESInit_SM2 g9_ippsGFpECESInit_SM2 +#define ippsGFpECESSetKey_SM2 g9_ippsGFpECESSetKey_SM2 +#define ippsGFpECESStart_SM2 g9_ippsGFpECESStart_SM2 +#define ippsGFpECESEncrypt_SM2 g9_ippsGFpECESEncrypt_SM2 +#define ippsGFpECESDecrypt_SM2 g9_ippsGFpECESDecrypt_SM2 +#define ippsGFpECESFinal_SM2 g9_ippsGFpECESFinal_SM2 +#define ippsGFpECESGetBuffersSize_SM2 g9_ippsGFpECESGetBuffersSize_SM2 +#define ippsGFpECEncryptSM2_Ext_EncMsgSize g9_ippsGFpECEncryptSM2_Ext_EncMsgSize +#define ippsGFpECEncryptSM2_Ext g9_ippsGFpECEncryptSM2_Ext +#define ippsGFpECDecryptSM2_Ext_DecMsgSize g9_ippsGFpECDecryptSM2_Ext_DecMsgSize +#define ippsGFpECDecryptSM2_Ext g9_ippsGFpECDecryptSM2_Ext diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_h9.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_h9.h new file mode 100644 index 0000000..94eae24 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_h9.h @@ -0,0 +1,557 @@ +/******************************************************************************* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + + #define ippcpGetLibVersion h9_ippcpGetLibVersion +#define ippsDESGetSize h9_ippsDESGetSize +#define ippsDESInit h9_ippsDESInit +#define ippsDESPack h9_ippsDESPack +#define ippsDESUnpack h9_ippsDESUnpack +#define ippsTDESEncryptECB h9_ippsTDESEncryptECB +#define ippsTDESDecryptECB h9_ippsTDESDecryptECB +#define ippsTDESEncryptCBC h9_ippsTDESEncryptCBC +#define ippsTDESDecryptCBC h9_ippsTDESDecryptCBC +#define ippsTDESEncryptCFB h9_ippsTDESEncryptCFB +#define ippsTDESDecryptCFB h9_ippsTDESDecryptCFB +#define ippsTDESEncryptOFB h9_ippsTDESEncryptOFB +#define ippsTDESDecryptOFB h9_ippsTDESDecryptOFB +#define ippsTDESEncryptCTR h9_ippsTDESEncryptCTR +#define ippsTDESDecryptCTR h9_ippsTDESDecryptCTR +#define ippsAESGetSize h9_ippsAESGetSize +#define ippsAESInit h9_ippsAESInit +#define ippsAESSetKey h9_ippsAESSetKey +#define ippsAESPack h9_ippsAESPack +#define ippsAESUnpack h9_ippsAESUnpack +#define ippsAESEncryptECB h9_ippsAESEncryptECB +#define ippsAESDecryptECB h9_ippsAESDecryptECB +#define ippsAESEncryptCBC h9_ippsAESEncryptCBC +#define ippsAESEncryptCBC_CS1 h9_ippsAESEncryptCBC_CS1 +#define ippsAESEncryptCBC_CS2 h9_ippsAESEncryptCBC_CS2 +#define ippsAESEncryptCBC_CS3 h9_ippsAESEncryptCBC_CS3 +#define ippsAESDecryptCBC h9_ippsAESDecryptCBC +#define ippsAESDecryptCBC_CS1 h9_ippsAESDecryptCBC_CS1 +#define ippsAESDecryptCBC_CS2 h9_ippsAESDecryptCBC_CS2 +#define ippsAESDecryptCBC_CS3 h9_ippsAESDecryptCBC_CS3 +#define ippsAESEncryptCFB h9_ippsAESEncryptCFB +#define ippsAESDecryptCFB h9_ippsAESDecryptCFB +#define ippsAESEncryptOFB h9_ippsAESEncryptOFB +#define ippsAESDecryptOFB h9_ippsAESDecryptOFB +#define ippsAESEncryptCTR h9_ippsAESEncryptCTR +#define ippsAESDecryptCTR h9_ippsAESDecryptCTR +#define ippsAESEncryptXTS_Direct h9_ippsAESEncryptXTS_Direct +#define ippsAESDecryptXTS_Direct h9_ippsAESDecryptXTS_Direct +#define ippsAESSetupNoise h9_ippsAESSetupNoise +#define ippsAES_GCMSetupNoise h9_ippsAES_GCMSetupNoise +#define ippsAES_CMACSetupNoise h9_ippsAES_CMACSetupNoise +#define ippsAES_EncryptCFB16_MB h9_ippsAES_EncryptCFB16_MB +#define ippsSMS4GetSize h9_ippsSMS4GetSize +#define ippsSMS4Init h9_ippsSMS4Init +#define ippsSMS4SetKey h9_ippsSMS4SetKey +#define ippsSMS4EncryptECB h9_ippsSMS4EncryptECB +#define ippsSMS4DecryptECB h9_ippsSMS4DecryptECB +#define ippsSMS4EncryptCBC h9_ippsSMS4EncryptCBC +#define ippsSMS4EncryptCBC_CS1 h9_ippsSMS4EncryptCBC_CS1 +#define ippsSMS4EncryptCBC_CS2 h9_ippsSMS4EncryptCBC_CS2 +#define ippsSMS4EncryptCBC_CS3 h9_ippsSMS4EncryptCBC_CS3 +#define ippsSMS4DecryptCBC h9_ippsSMS4DecryptCBC +#define ippsSMS4DecryptCBC_CS1 h9_ippsSMS4DecryptCBC_CS1 +#define ippsSMS4DecryptCBC_CS2 h9_ippsSMS4DecryptCBC_CS2 +#define ippsSMS4DecryptCBC_CS3 h9_ippsSMS4DecryptCBC_CS3 +#define ippsSMS4EncryptCFB h9_ippsSMS4EncryptCFB +#define ippsSMS4DecryptCFB h9_ippsSMS4DecryptCFB +#define ippsSMS4EncryptOFB h9_ippsSMS4EncryptOFB +#define ippsSMS4DecryptOFB h9_ippsSMS4DecryptOFB +#define ippsSMS4EncryptCTR h9_ippsSMS4EncryptCTR +#define ippsSMS4DecryptCTR h9_ippsSMS4DecryptCTR +#define ippsSMS4_CCMGetSize h9_ippsSMS4_CCMGetSize +#define ippsSMS4_CCMInit h9_ippsSMS4_CCMInit +#define ippsSMS4_CCMMessageLen h9_ippsSMS4_CCMMessageLen +#define ippsSMS4_CCMTagLen h9_ippsSMS4_CCMTagLen +#define ippsSMS4_CCMStart h9_ippsSMS4_CCMStart +#define ippsSMS4_CCMEncrypt h9_ippsSMS4_CCMEncrypt +#define ippsSMS4_CCMDecrypt h9_ippsSMS4_CCMDecrypt +#define ippsSMS4_CCMGetTag h9_ippsSMS4_CCMGetTag +#define ippsAES_CCMGetSize h9_ippsAES_CCMGetSize +#define ippsAES_CCMInit h9_ippsAES_CCMInit +#define ippsAES_CCMMessageLen h9_ippsAES_CCMMessageLen +#define ippsAES_CCMTagLen h9_ippsAES_CCMTagLen +#define ippsAES_CCMStart h9_ippsAES_CCMStart +#define ippsAES_CCMEncrypt h9_ippsAES_CCMEncrypt +#define ippsAES_CCMDecrypt h9_ippsAES_CCMDecrypt +#define ippsAES_CCMGetTag h9_ippsAES_CCMGetTag +#define ippsAES_GCMGetSize h9_ippsAES_GCMGetSize +#define ippsAES_GCMInit h9_ippsAES_GCMInit +#define ippsAES_GCMReinit h9_ippsAES_GCMReinit +#define ippsAES_GCMReset h9_ippsAES_GCMReset +#define ippsAES_GCMProcessIV h9_ippsAES_GCMProcessIV +#define ippsAES_GCMProcessAAD h9_ippsAES_GCMProcessAAD +#define ippsAES_GCMStart h9_ippsAES_GCMStart +#define ippsAES_GCMEncrypt h9_ippsAES_GCMEncrypt +#define ippsAES_GCMDecrypt h9_ippsAES_GCMDecrypt +#define ippsAES_GCMGetTag h9_ippsAES_GCMGetTag +#define ippsAES_XTSGetSize h9_ippsAES_XTSGetSize +#define ippsAES_XTSInit h9_ippsAES_XTSInit +#define ippsAES_XTSEncrypt h9_ippsAES_XTSEncrypt +#define ippsAES_XTSDecrypt h9_ippsAES_XTSDecrypt +#define ippsAES_S2V_CMAC h9_ippsAES_S2V_CMAC +#define ippsAES_SIVEncrypt h9_ippsAES_SIVEncrypt +#define ippsAES_SIVDecrypt h9_ippsAES_SIVDecrypt +#define ippsAES_CMACGetSize h9_ippsAES_CMACGetSize +#define ippsAES_CMACInit h9_ippsAES_CMACInit +#define ippsAES_CMACUpdate h9_ippsAES_CMACUpdate +#define ippsAES_CMACFinal h9_ippsAES_CMACFinal +#define ippsAES_CMACGetTag h9_ippsAES_CMACGetTag +#define ippsARCFourCheckKey h9_ippsARCFourCheckKey +#define ippsARCFourGetSize h9_ippsARCFourGetSize +#define ippsARCFourInit h9_ippsARCFourInit +#define ippsARCFourReset h9_ippsARCFourReset +#define ippsARCFourPack h9_ippsARCFourPack +#define ippsARCFourUnpack h9_ippsARCFourUnpack +#define ippsARCFourEncrypt h9_ippsARCFourEncrypt +#define ippsARCFourDecrypt h9_ippsARCFourDecrypt +#define ippsSHA1GetSize h9_ippsSHA1GetSize +#define ippsSHA1Init h9_ippsSHA1Init +#define ippsSHA1Duplicate h9_ippsSHA1Duplicate +#define ippsSHA1Pack h9_ippsSHA1Pack +#define ippsSHA1Unpack h9_ippsSHA1Unpack +#define ippsSHA1Update h9_ippsSHA1Update +#define ippsSHA1GetTag h9_ippsSHA1GetTag +#define ippsSHA1Final h9_ippsSHA1Final +#define ippsSHA1MessageDigest h9_ippsSHA1MessageDigest +#define ippsSHA224GetSize h9_ippsSHA224GetSize +#define ippsSHA224Init h9_ippsSHA224Init +#define ippsSHA224Duplicate h9_ippsSHA224Duplicate +#define ippsSHA224Pack h9_ippsSHA224Pack +#define ippsSHA224Unpack h9_ippsSHA224Unpack +#define ippsSHA224Update h9_ippsSHA224Update +#define ippsSHA224GetTag h9_ippsSHA224GetTag +#define ippsSHA224Final h9_ippsSHA224Final +#define ippsSHA224MessageDigest h9_ippsSHA224MessageDigest +#define ippsSHA256GetSize h9_ippsSHA256GetSize +#define ippsSHA256Init h9_ippsSHA256Init +#define ippsSHA256Duplicate h9_ippsSHA256Duplicate +#define ippsSHA256Pack h9_ippsSHA256Pack +#define ippsSHA256Unpack h9_ippsSHA256Unpack +#define ippsSHA256Update h9_ippsSHA256Update +#define ippsSHA256GetTag h9_ippsSHA256GetTag +#define ippsSHA256Final h9_ippsSHA256Final +#define ippsSHA256MessageDigest h9_ippsSHA256MessageDigest +#define ippsSHA384GetSize h9_ippsSHA384GetSize +#define ippsSHA384Init h9_ippsSHA384Init +#define ippsSHA384Duplicate h9_ippsSHA384Duplicate +#define ippsSHA384Pack h9_ippsSHA384Pack +#define ippsSHA384Unpack h9_ippsSHA384Unpack +#define ippsSHA384Update h9_ippsSHA384Update +#define ippsSHA384GetTag h9_ippsSHA384GetTag +#define ippsSHA384Final h9_ippsSHA384Final +#define ippsSHA384MessageDigest h9_ippsSHA384MessageDigest +#define ippsSHA512GetSize h9_ippsSHA512GetSize +#define ippsSHA512Init h9_ippsSHA512Init +#define ippsSHA512Duplicate h9_ippsSHA512Duplicate +#define ippsSHA512Pack h9_ippsSHA512Pack +#define ippsSHA512Unpack h9_ippsSHA512Unpack +#define ippsSHA512Update h9_ippsSHA512Update +#define ippsSHA512GetTag h9_ippsSHA512GetTag +#define ippsSHA512Final h9_ippsSHA512Final +#define ippsSHA512MessageDigest h9_ippsSHA512MessageDigest +#define ippsMD5GetSize h9_ippsMD5GetSize +#define ippsMD5Init h9_ippsMD5Init +#define ippsMD5Duplicate h9_ippsMD5Duplicate +#define ippsMD5Pack h9_ippsMD5Pack +#define ippsMD5Unpack h9_ippsMD5Unpack +#define ippsMD5Update h9_ippsMD5Update +#define ippsMD5GetTag h9_ippsMD5GetTag +#define ippsMD5Final h9_ippsMD5Final +#define ippsMD5MessageDigest h9_ippsMD5MessageDigest +#define ippsSM3GetSize h9_ippsSM3GetSize +#define ippsSM3Init h9_ippsSM3Init +#define ippsSM3Duplicate h9_ippsSM3Duplicate +#define ippsSM3Pack h9_ippsSM3Pack +#define ippsSM3Unpack h9_ippsSM3Unpack +#define ippsSM3Update h9_ippsSM3Update +#define ippsSM3GetTag h9_ippsSM3GetTag +#define ippsSM3Final h9_ippsSM3Final +#define ippsSM3MessageDigest h9_ippsSM3MessageDigest +#define ippsHashGetSize h9_ippsHashGetSize +#define ippsHashInit h9_ippsHashInit +#define ippsHashPack h9_ippsHashPack +#define ippsHashUnpack h9_ippsHashUnpack +#define ippsHashDuplicate h9_ippsHashDuplicate +#define ippsHashUpdate h9_ippsHashUpdate +#define ippsHashGetTag h9_ippsHashGetTag +#define ippsHashFinal h9_ippsHashFinal +#define ippsHashMessage h9_ippsHashMessage +#define ippsHashMethod_MD5 h9_ippsHashMethod_MD5 +#define ippsHashMethod_SM3 h9_ippsHashMethod_SM3 +#define ippsHashMethod_SHA1 h9_ippsHashMethod_SHA1 +#define ippsHashMethod_SHA1_NI h9_ippsHashMethod_SHA1_NI +#define ippsHashMethod_SHA1_TT h9_ippsHashMethod_SHA1_TT +#define ippsHashMethod_SHA256 h9_ippsHashMethod_SHA256 +#define ippsHashMethod_SHA256_NI h9_ippsHashMethod_SHA256_NI +#define ippsHashMethod_SHA256_TT h9_ippsHashMethod_SHA256_TT +#define ippsHashMethod_SHA224 h9_ippsHashMethod_SHA224 +#define ippsHashMethod_SHA224_NI h9_ippsHashMethod_SHA224_NI +#define ippsHashMethod_SHA224_TT h9_ippsHashMethod_SHA224_TT +#define ippsHashMethod_SHA512 h9_ippsHashMethod_SHA512 +#define ippsHashMethod_SHA384 h9_ippsHashMethod_SHA384 +#define ippsHashMethod_SHA512_256 h9_ippsHashMethod_SHA512_256 +#define ippsHashMethod_SHA512_224 h9_ippsHashMethod_SHA512_224 +#define ippsHashMethodGetSize h9_ippsHashMethodGetSize +#define ippsHashMethodSet_MD5 h9_ippsHashMethodSet_MD5 +#define ippsHashMethodSet_SM3 h9_ippsHashMethodSet_SM3 +#define ippsHashStateMethodSet_SM3 h9_ippsHashStateMethodSet_SM3 +#define ippsHashMethodSet_SHA1 h9_ippsHashMethodSet_SHA1 +#define ippsHashMethodSet_SHA1_NI h9_ippsHashMethodSet_SHA1_NI +#define ippsHashMethodSet_SHA1_TT h9_ippsHashMethodSet_SHA1_TT +#define ippsHashMethodSet_SHA256 h9_ippsHashMethodSet_SHA256 +#define ippsHashMethodSet_SHA256_NI h9_ippsHashMethodSet_SHA256_NI +#define ippsHashMethodSet_SHA256_TT h9_ippsHashMethodSet_SHA256_TT +#define ippsHashStateMethodSet_SHA256 h9_ippsHashStateMethodSet_SHA256 +#define ippsHashStateMethodSet_SHA256_NI h9_ippsHashStateMethodSet_SHA256_NI +#define ippsHashStateMethodSet_SHA256_TT h9_ippsHashStateMethodSet_SHA256_TT +#define ippsHashMethodSet_SHA224 h9_ippsHashMethodSet_SHA224 +#define ippsHashMethodSet_SHA224_NI h9_ippsHashMethodSet_SHA224_NI +#define ippsHashMethodSet_SHA224_TT h9_ippsHashMethodSet_SHA224_TT +#define ippsHashStateMethodSet_SHA224 h9_ippsHashStateMethodSet_SHA224 +#define ippsHashStateMethodSet_SHA224_NI h9_ippsHashStateMethodSet_SHA224_NI +#define ippsHashStateMethodSet_SHA224_TT h9_ippsHashStateMethodSet_SHA224_TT +#define ippsHashMethodSet_SHA512 h9_ippsHashMethodSet_SHA512 +#define ippsHashMethodSet_SHA384 h9_ippsHashMethodSet_SHA384 +#define ippsHashMethodSet_SHA512_256 h9_ippsHashMethodSet_SHA512_256 +#define ippsHashMethodSet_SHA512_224 h9_ippsHashMethodSet_SHA512_224 +#define ippsHashStateMethodSet_SHA512 h9_ippsHashStateMethodSet_SHA512 +#define ippsHashStateMethodSet_SHA384 h9_ippsHashStateMethodSet_SHA384 +#define ippsHashStateMethodSet_SHA512_256 h9_ippsHashStateMethodSet_SHA512_256 +#define ippsHashStateMethodSet_SHA512_224 h9_ippsHashStateMethodSet_SHA512_224 +#define ippsHashGetSize_rmf h9_ippsHashGetSize_rmf +#define ippsHashInit_rmf h9_ippsHashInit_rmf +#define ippsHashPack_rmf h9_ippsHashPack_rmf +#define ippsHashUnpack_rmf h9_ippsHashUnpack_rmf +#define ippsHashDuplicate_rmf h9_ippsHashDuplicate_rmf +#define ippsHashUpdate_rmf h9_ippsHashUpdate_rmf +#define ippsHashGetTag_rmf h9_ippsHashGetTag_rmf +#define ippsHashFinal_rmf h9_ippsHashFinal_rmf +#define ippsHashMessage_rmf h9_ippsHashMessage_rmf +#define ippsHashMethodGetInfo h9_ippsHashMethodGetInfo +#define ippsHashGetInfo_rmf h9_ippsHashGetInfo_rmf +#define ippsMGF h9_ippsMGF +#define ippsMGF1_rmf h9_ippsMGF1_rmf +#define ippsMGF2_rmf h9_ippsMGF2_rmf +#define ippsHMAC_GetSize h9_ippsHMAC_GetSize +#define ippsHMAC_Init h9_ippsHMAC_Init +#define ippsHMAC_Pack h9_ippsHMAC_Pack +#define ippsHMAC_Unpack h9_ippsHMAC_Unpack +#define ippsHMAC_Duplicate h9_ippsHMAC_Duplicate +#define ippsHMAC_Update h9_ippsHMAC_Update +#define ippsHMAC_Final h9_ippsHMAC_Final +#define ippsHMAC_GetTag h9_ippsHMAC_GetTag +#define ippsHMAC_Message h9_ippsHMAC_Message +#define ippsHMACGetSize_rmf h9_ippsHMACGetSize_rmf +#define ippsHMACInit_rmf h9_ippsHMACInit_rmf +#define ippsHMACPack_rmf h9_ippsHMACPack_rmf +#define ippsHMACUnpack_rmf h9_ippsHMACUnpack_rmf +#define ippsHMACDuplicate_rmf h9_ippsHMACDuplicate_rmf +#define ippsHMACUpdate_rmf h9_ippsHMACUpdate_rmf +#define ippsHMACFinal_rmf h9_ippsHMACFinal_rmf +#define ippsHMACGetTag_rmf h9_ippsHMACGetTag_rmf +#define ippsHMACMessage_rmf h9_ippsHMACMessage_rmf +#define ippsBigNumGetSize h9_ippsBigNumGetSize +#define ippsBigNumInit h9_ippsBigNumInit +#define ippsCmpZero_BN h9_ippsCmpZero_BN +#define ippsCmp_BN h9_ippsCmp_BN +#define ippsGetSize_BN h9_ippsGetSize_BN +#define ippsSet_BN h9_ippsSet_BN +#define ippsGet_BN h9_ippsGet_BN +#define ippsRef_BN h9_ippsRef_BN +#define ippsExtGet_BN h9_ippsExtGet_BN +#define ippsAdd_BN h9_ippsAdd_BN +#define ippsSub_BN h9_ippsSub_BN +#define ippsMul_BN h9_ippsMul_BN +#define ippsMAC_BN_I h9_ippsMAC_BN_I +#define ippsDiv_BN h9_ippsDiv_BN +#define ippsMod_BN h9_ippsMod_BN +#define ippsGcd_BN h9_ippsGcd_BN +#define ippsModInv_BN h9_ippsModInv_BN +#define ippsSetOctString_BN h9_ippsSetOctString_BN +#define ippsGetOctString_BN h9_ippsGetOctString_BN +#define ippsMontGetSize h9_ippsMontGetSize +#define ippsMontInit h9_ippsMontInit +#define ippsMontSet h9_ippsMontSet +#define ippsMontGet h9_ippsMontGet +#define ippsMontForm h9_ippsMontForm +#define ippsMontMul h9_ippsMontMul +#define ippsMontExp h9_ippsMontExp +#define ippsPRNGGetSize h9_ippsPRNGGetSize +#define ippsPRNGInit h9_ippsPRNGInit +#define ippsPRNGSetModulus h9_ippsPRNGSetModulus +#define ippsPRNGSetH0 h9_ippsPRNGSetH0 +#define ippsPRNGSetAugment h9_ippsPRNGSetAugment +#define ippsPRNGSetSeed h9_ippsPRNGSetSeed +#define ippsPRNGGetSeed h9_ippsPRNGGetSeed +#define ippsPRNGen h9_ippsPRNGen +#define ippsPRNGen_BN h9_ippsPRNGen_BN +#define ippsPRNGenRDRAND h9_ippsPRNGenRDRAND +#define ippsPRNGenRDRAND_BN h9_ippsPRNGenRDRAND_BN +#define ippsTRNGenRDSEED h9_ippsTRNGenRDSEED +#define ippsTRNGenRDSEED_BN h9_ippsTRNGenRDSEED_BN +#define ippsPrimeGetSize h9_ippsPrimeGetSize +#define ippsPrimeInit h9_ippsPrimeInit +#define ippsPrimeGen h9_ippsPrimeGen +#define ippsPrimeTest h9_ippsPrimeTest +#define ippsPrimeGen_BN h9_ippsPrimeGen_BN +#define ippsPrimeTest_BN h9_ippsPrimeTest_BN +#define ippsPrimeGet h9_ippsPrimeGet +#define ippsPrimeGet_BN h9_ippsPrimeGet_BN +#define ippsPrimeSet h9_ippsPrimeSet +#define ippsPrimeSet_BN h9_ippsPrimeSet_BN +#define ippsRSA_GetSizePublicKey h9_ippsRSA_GetSizePublicKey +#define ippsRSA_InitPublicKey h9_ippsRSA_InitPublicKey +#define ippsRSA_SetPublicKey h9_ippsRSA_SetPublicKey +#define ippsRSA_GetPublicKey h9_ippsRSA_GetPublicKey +#define ippsRSA_GetSizePrivateKeyType1 h9_ippsRSA_GetSizePrivateKeyType1 +#define ippsRSA_InitPrivateKeyType1 h9_ippsRSA_InitPrivateKeyType1 +#define ippsRSA_SetPrivateKeyType1 h9_ippsRSA_SetPrivateKeyType1 +#define ippsRSA_GetPrivateKeyType1 h9_ippsRSA_GetPrivateKeyType1 +#define ippsRSA_GetSizePrivateKeyType2 h9_ippsRSA_GetSizePrivateKeyType2 +#define ippsRSA_InitPrivateKeyType2 h9_ippsRSA_InitPrivateKeyType2 +#define ippsRSA_SetPrivateKeyType2 h9_ippsRSA_SetPrivateKeyType2 +#define ippsRSA_GetPrivateKeyType2 h9_ippsRSA_GetPrivateKeyType2 +#define ippsRSA_GetBufferSizePublicKey h9_ippsRSA_GetBufferSizePublicKey +#define ippsRSA_GetBufferSizePrivateKey h9_ippsRSA_GetBufferSizePrivateKey +#define ippsRSA_Encrypt h9_ippsRSA_Encrypt +#define ippsRSA_Decrypt h9_ippsRSA_Decrypt +#define ippsRSA_GenerateKeys h9_ippsRSA_GenerateKeys +#define ippsRSA_ValidateKeys h9_ippsRSA_ValidateKeys +#define ippsRSAEncrypt_OAEP h9_ippsRSAEncrypt_OAEP +#define ippsRSADecrypt_OAEP h9_ippsRSADecrypt_OAEP +#define ippsRSAEncrypt_OAEP_rmf h9_ippsRSAEncrypt_OAEP_rmf +#define ippsRSADecrypt_OAEP_rmf h9_ippsRSADecrypt_OAEP_rmf +#define ippsRSAEncrypt_PKCSv15 h9_ippsRSAEncrypt_PKCSv15 +#define ippsRSADecrypt_PKCSv15 h9_ippsRSADecrypt_PKCSv15 +#define ippsRSASign_PSS h9_ippsRSASign_PSS +#define ippsRSAVerify_PSS h9_ippsRSAVerify_PSS +#define ippsRSASign_PSS_rmf h9_ippsRSASign_PSS_rmf +#define ippsRSAVerify_PSS_rmf h9_ippsRSAVerify_PSS_rmf +#define ippsRSASign_PKCS1v15 h9_ippsRSASign_PKCS1v15 +#define ippsRSAVerify_PKCS1v15 h9_ippsRSAVerify_PKCS1v15 +#define ippsRSASign_PKCS1v15_rmf h9_ippsRSASign_PKCS1v15_rmf +#define ippsRSAVerify_PKCS1v15_rmf h9_ippsRSAVerify_PKCS1v15_rmf +#define ippsDLGetResultString h9_ippsDLGetResultString +#define ippsDLPGetSize h9_ippsDLPGetSize +#define ippsDLPInit h9_ippsDLPInit +#define ippsDLPPack h9_ippsDLPPack +#define ippsDLPUnpack h9_ippsDLPUnpack +#define ippsDLPSet h9_ippsDLPSet +#define ippsDLPGet h9_ippsDLPGet +#define ippsDLPSetDP h9_ippsDLPSetDP +#define ippsDLPGetDP h9_ippsDLPGetDP +#define ippsDLPGenKeyPair h9_ippsDLPGenKeyPair +#define ippsDLPPublicKey h9_ippsDLPPublicKey +#define ippsDLPValidateKeyPair h9_ippsDLPValidateKeyPair +#define ippsDLPSetKeyPair h9_ippsDLPSetKeyPair +#define ippsDLPSignDSA h9_ippsDLPSignDSA +#define ippsDLPVerifyDSA h9_ippsDLPVerifyDSA +#define ippsDLPSharedSecretDH h9_ippsDLPSharedSecretDH +#define ippsDLPGenerateDSA h9_ippsDLPGenerateDSA +#define ippsDLPValidateDSA h9_ippsDLPValidateDSA +#define ippsDLPGenerateDH h9_ippsDLPGenerateDH +#define ippsDLPValidateDH h9_ippsDLPValidateDH +#define ippsECCGetResultString h9_ippsECCGetResultString +#define ippsECCPGetSize h9_ippsECCPGetSize +#define ippsECCPGetSizeStd128r1 h9_ippsECCPGetSizeStd128r1 +#define ippsECCPGetSizeStd128r2 h9_ippsECCPGetSizeStd128r2 +#define ippsECCPGetSizeStd192r1 h9_ippsECCPGetSizeStd192r1 +#define ippsECCPGetSizeStd224r1 h9_ippsECCPGetSizeStd224r1 +#define ippsECCPGetSizeStd256r1 h9_ippsECCPGetSizeStd256r1 +#define ippsECCPGetSizeStd384r1 h9_ippsECCPGetSizeStd384r1 +#define ippsECCPGetSizeStd521r1 h9_ippsECCPGetSizeStd521r1 +#define ippsECCPGetSizeStdSM2 h9_ippsECCPGetSizeStdSM2 +#define ippsECCPInit h9_ippsECCPInit +#define ippsECCPInitStd128r1 h9_ippsECCPInitStd128r1 +#define ippsECCPInitStd128r2 h9_ippsECCPInitStd128r2 +#define ippsECCPInitStd192r1 h9_ippsECCPInitStd192r1 +#define ippsECCPInitStd224r1 h9_ippsECCPInitStd224r1 +#define ippsECCPInitStd256r1 h9_ippsECCPInitStd256r1 +#define ippsECCPInitStd384r1 h9_ippsECCPInitStd384r1 +#define ippsECCPInitStd521r1 h9_ippsECCPInitStd521r1 +#define ippsECCPInitStdSM2 h9_ippsECCPInitStdSM2 +#define ippsECCPSet h9_ippsECCPSet +#define ippsECCPSetStd h9_ippsECCPSetStd +#define ippsECCPSetStd128r1 h9_ippsECCPSetStd128r1 +#define ippsECCPSetStd128r2 h9_ippsECCPSetStd128r2 +#define ippsECCPSetStd192r1 h9_ippsECCPSetStd192r1 +#define ippsECCPSetStd224r1 h9_ippsECCPSetStd224r1 +#define ippsECCPSetStd256r1 h9_ippsECCPSetStd256r1 +#define ippsECCPSetStd384r1 h9_ippsECCPSetStd384r1 +#define ippsECCPSetStd521r1 h9_ippsECCPSetStd521r1 +#define ippsECCPSetStdSM2 h9_ippsECCPSetStdSM2 +#define ippsECCPBindGxyTblStd192r1 h9_ippsECCPBindGxyTblStd192r1 +#define ippsECCPBindGxyTblStd224r1 h9_ippsECCPBindGxyTblStd224r1 +#define ippsECCPBindGxyTblStd256r1 h9_ippsECCPBindGxyTblStd256r1 +#define ippsECCPBindGxyTblStd384r1 h9_ippsECCPBindGxyTblStd384r1 +#define ippsECCPBindGxyTblStd521r1 h9_ippsECCPBindGxyTblStd521r1 +#define ippsECCPBindGxyTblStdSM2 h9_ippsECCPBindGxyTblStdSM2 +#define ippsECCPGet h9_ippsECCPGet +#define ippsECCPGetOrderBitSize h9_ippsECCPGetOrderBitSize +#define ippsECCPValidate h9_ippsECCPValidate +#define ippsECCPPointGetSize h9_ippsECCPPointGetSize +#define ippsECCPPointInit h9_ippsECCPPointInit +#define ippsECCPSetPoint h9_ippsECCPSetPoint +#define ippsECCPSetPointAtInfinity h9_ippsECCPSetPointAtInfinity +#define ippsECCPGetPoint h9_ippsECCPGetPoint +#define ippsECCPCheckPoint h9_ippsECCPCheckPoint +#define ippsECCPComparePoint h9_ippsECCPComparePoint +#define ippsECCPNegativePoint h9_ippsECCPNegativePoint +#define ippsECCPAddPoint h9_ippsECCPAddPoint +#define ippsECCPMulPointScalar h9_ippsECCPMulPointScalar +#define ippsECCPGenKeyPair h9_ippsECCPGenKeyPair +#define ippsECCPPublicKey h9_ippsECCPPublicKey +#define ippsECCPValidateKeyPair h9_ippsECCPValidateKeyPair +#define ippsECCPSetKeyPair h9_ippsECCPSetKeyPair +#define ippsECCPSharedSecretDH h9_ippsECCPSharedSecretDH +#define ippsECCPSharedSecretDHC h9_ippsECCPSharedSecretDHC +#define ippsECCPSignDSA h9_ippsECCPSignDSA +#define ippsECCPVerifyDSA h9_ippsECCPVerifyDSA +#define ippsECCPSignNR h9_ippsECCPSignNR +#define ippsECCPVerifyNR h9_ippsECCPVerifyNR +#define ippsECCPSignSM2 h9_ippsECCPSignSM2 +#define ippsECCPVerifySM2 h9_ippsECCPVerifySM2 +#define ippsGFpGetSize h9_ippsGFpGetSize +#define ippsGFpInitArbitrary h9_ippsGFpInitArbitrary +#define ippsGFpInitFixed h9_ippsGFpInitFixed +#define ippsGFpInit h9_ippsGFpInit +#define ippsGFpMethod_p192r1 h9_ippsGFpMethod_p192r1 +#define ippsGFpMethod_p224r1 h9_ippsGFpMethod_p224r1 +#define ippsGFpMethod_p256r1 h9_ippsGFpMethod_p256r1 +#define ippsGFpMethod_p384r1 h9_ippsGFpMethod_p384r1 +#define ippsGFpMethod_p521r1 h9_ippsGFpMethod_p521r1 +#define ippsGFpMethod_p256sm2 h9_ippsGFpMethod_p256sm2 +#define ippsGFpMethod_p256bn h9_ippsGFpMethod_p256bn +#define ippsGFpMethod_p256 h9_ippsGFpMethod_p256 +#define ippsGFpMethod_pArb h9_ippsGFpMethod_pArb +#define ippsGFpxGetSize h9_ippsGFpxGetSize +#define ippsGFpxInit h9_ippsGFpxInit +#define ippsGFpxInitBinomial h9_ippsGFpxInitBinomial +#define ippsGFpxMethod_binom2_epid2 h9_ippsGFpxMethod_binom2_epid2 +#define ippsGFpxMethod_binom3_epid2 h9_ippsGFpxMethod_binom3_epid2 +#define ippsGFpxMethod_binom2 h9_ippsGFpxMethod_binom2 +#define ippsGFpxMethod_binom3 h9_ippsGFpxMethod_binom3 +#define ippsGFpxMethod_binom h9_ippsGFpxMethod_binom +#define ippsGFpxMethod_com h9_ippsGFpxMethod_com +#define ippsGFpScratchBufferSize h9_ippsGFpScratchBufferSize +#define ippsGFpElementGetSize h9_ippsGFpElementGetSize +#define ippsGFpElementInit h9_ippsGFpElementInit +#define ippsGFpSetElement h9_ippsGFpSetElement +#define ippsGFpSetElementRegular h9_ippsGFpSetElementRegular +#define ippsGFpSetElementOctString h9_ippsGFpSetElementOctString +#define ippsGFpSetElementRandom h9_ippsGFpSetElementRandom +#define ippsGFpSetElementHash h9_ippsGFpSetElementHash +#define ippsGFpSetElementHash_rmf h9_ippsGFpSetElementHash_rmf +#define ippsGFpCpyElement h9_ippsGFpCpyElement +#define ippsGFpGetElement h9_ippsGFpGetElement +#define ippsGFpGetElementOctString h9_ippsGFpGetElementOctString +#define ippsGFpCmpElement h9_ippsGFpCmpElement +#define ippsGFpIsZeroElement h9_ippsGFpIsZeroElement +#define ippsGFpIsUnityElement h9_ippsGFpIsUnityElement +#define ippsGFpConj h9_ippsGFpConj +#define ippsGFpNeg h9_ippsGFpNeg +#define ippsGFpInv h9_ippsGFpInv +#define ippsGFpSqrt h9_ippsGFpSqrt +#define ippsGFpSqr h9_ippsGFpSqr +#define ippsGFpAdd h9_ippsGFpAdd +#define ippsGFpSub h9_ippsGFpSub +#define ippsGFpMul h9_ippsGFpMul +#define ippsGFpExp h9_ippsGFpExp +#define ippsGFpMultiExp h9_ippsGFpMultiExp +#define ippsGFpAdd_PE h9_ippsGFpAdd_PE +#define ippsGFpSub_PE h9_ippsGFpSub_PE +#define ippsGFpMul_PE h9_ippsGFpMul_PE +#define ippsGFpGetInfo h9_ippsGFpGetInfo +#define ippsGFpECGetSize h9_ippsGFpECGetSize +#define ippsGFpECInit h9_ippsGFpECInit +#define ippsGFpECSet h9_ippsGFpECSet +#define ippsGFpECSetSubgroup h9_ippsGFpECSetSubgroup +#define ippsGFpECInitStd128r1 h9_ippsGFpECInitStd128r1 +#define ippsGFpECInitStd128r2 h9_ippsGFpECInitStd128r2 +#define ippsGFpECInitStd192r1 h9_ippsGFpECInitStd192r1 +#define ippsGFpECInitStd224r1 h9_ippsGFpECInitStd224r1 +#define ippsGFpECInitStd256r1 h9_ippsGFpECInitStd256r1 +#define ippsGFpECInitStd384r1 h9_ippsGFpECInitStd384r1 +#define ippsGFpECInitStd521r1 h9_ippsGFpECInitStd521r1 +#define ippsGFpECInitStdSM2 h9_ippsGFpECInitStdSM2 +#define ippsGFpECInitStdBN256 h9_ippsGFpECInitStdBN256 +#define ippsGFpECBindGxyTblStd192r1 h9_ippsGFpECBindGxyTblStd192r1 +#define ippsGFpECBindGxyTblStd224r1 h9_ippsGFpECBindGxyTblStd224r1 +#define ippsGFpECBindGxyTblStd256r1 h9_ippsGFpECBindGxyTblStd256r1 +#define ippsGFpECBindGxyTblStd384r1 h9_ippsGFpECBindGxyTblStd384r1 +#define ippsGFpECBindGxyTblStd521r1 h9_ippsGFpECBindGxyTblStd521r1 +#define ippsGFpECBindGxyTblStdSM2 h9_ippsGFpECBindGxyTblStdSM2 +#define ippsGFpECGet h9_ippsGFpECGet +#define ippsGFpECGetSubgroup h9_ippsGFpECGetSubgroup +#define ippsGFpECScratchBufferSize h9_ippsGFpECScratchBufferSize +#define ippsGFpECVerify h9_ippsGFpECVerify +#define ippsGFpECPointGetSize h9_ippsGFpECPointGetSize +#define ippsGFpECPointInit h9_ippsGFpECPointInit +#define ippsGFpECSetPointAtInfinity h9_ippsGFpECSetPointAtInfinity +#define ippsGFpECSetPoint h9_ippsGFpECSetPoint +#define ippsGFpECSetPointRegular h9_ippsGFpECSetPointRegular +#define ippsGFpECSetPointRandom h9_ippsGFpECSetPointRandom +#define ippsGFpECMakePoint h9_ippsGFpECMakePoint +#define ippsGFpECSetPointHash h9_ippsGFpECSetPointHash +#define ippsGFpECSetPointHashBackCompatible h9_ippsGFpECSetPointHashBackCompatible +#define ippsGFpECSetPointHash_rmf h9_ippsGFpECSetPointHash_rmf +#define ippsGFpECSetPointHashBackCompatible_rmf h9_ippsGFpECSetPointHashBackCompatible_rmf +#define ippsGFpECGetPoint h9_ippsGFpECGetPoint +#define ippsGFpECGetPointRegular h9_ippsGFpECGetPointRegular +#define ippsGFpECSetPointOctString h9_ippsGFpECSetPointOctString +#define ippsGFpECGetPointOctString h9_ippsGFpECGetPointOctString +#define ippsGFpECTstPoint h9_ippsGFpECTstPoint +#define ippsGFpECTstPointInSubgroup h9_ippsGFpECTstPointInSubgroup +#define ippsGFpECCpyPoint h9_ippsGFpECCpyPoint +#define ippsGFpECCmpPoint h9_ippsGFpECCmpPoint +#define ippsGFpECNegPoint h9_ippsGFpECNegPoint +#define ippsGFpECAddPoint h9_ippsGFpECAddPoint +#define ippsGFpECMulPoint h9_ippsGFpECMulPoint +#define ippsGFpECPrivateKey h9_ippsGFpECPrivateKey +#define ippsGFpECPublicKey h9_ippsGFpECPublicKey +#define ippsGFpECTstKeyPair h9_ippsGFpECTstKeyPair +#define ippsGFpECSharedSecretDH h9_ippsGFpECSharedSecretDH +#define ippsGFpECSharedSecretDHC h9_ippsGFpECSharedSecretDHC +#define ippsGFpECMessageRepresentationSM2 h9_ippsGFpECMessageRepresentationSM2 +#define ippsGFpECSignDSA h9_ippsGFpECSignDSA +#define ippsGFpECVerifyDSA h9_ippsGFpECVerifyDSA +#define ippsGFpECSignNR h9_ippsGFpECSignNR +#define ippsGFpECVerifyNR h9_ippsGFpECVerifyNR +#define ippsGFpECSignSM2 h9_ippsGFpECSignSM2 +#define ippsGFpECVerifySM2 h9_ippsGFpECVerifySM2 +#define ippsGFpECUserIDHashSM2 h9_ippsGFpECUserIDHashSM2 +#define ippsGFpECKeyExchangeSM2_GetSize h9_ippsGFpECKeyExchangeSM2_GetSize +#define ippsGFpECKeyExchangeSM2_Init h9_ippsGFpECKeyExchangeSM2_Init +#define ippsGFpECKeyExchangeSM2_Setup h9_ippsGFpECKeyExchangeSM2_Setup +#define ippsGFpECKeyExchangeSM2_SharedKey h9_ippsGFpECKeyExchangeSM2_SharedKey +#define ippsGFpECKeyExchangeSM2_Confirm h9_ippsGFpECKeyExchangeSM2_Confirm +#define ippsGFpECGetInfo_GF h9_ippsGFpECGetInfo_GF +#define ippsGFpECESGetSize_SM2 h9_ippsGFpECESGetSize_SM2 +#define ippsGFpECESInit_SM2 h9_ippsGFpECESInit_SM2 +#define ippsGFpECESSetKey_SM2 h9_ippsGFpECESSetKey_SM2 +#define ippsGFpECESStart_SM2 h9_ippsGFpECESStart_SM2 +#define ippsGFpECESEncrypt_SM2 h9_ippsGFpECESEncrypt_SM2 +#define ippsGFpECESDecrypt_SM2 h9_ippsGFpECESDecrypt_SM2 +#define ippsGFpECESFinal_SM2 h9_ippsGFpECESFinal_SM2 +#define ippsGFpECESGetBuffersSize_SM2 h9_ippsGFpECESGetBuffersSize_SM2 +#define ippsGFpECEncryptSM2_Ext_EncMsgSize h9_ippsGFpECEncryptSM2_Ext_EncMsgSize +#define ippsGFpECEncryptSM2_Ext h9_ippsGFpECEncryptSM2_Ext +#define ippsGFpECDecryptSM2_Ext_DecMsgSize h9_ippsGFpECDecryptSM2_Ext_DecMsgSize +#define ippsGFpECDecryptSM2_Ext h9_ippsGFpECDecryptSM2_Ext diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_k0.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_k0.h new file mode 100644 index 0000000..8e1a740 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_k0.h @@ -0,0 +1,557 @@ +/******************************************************************************* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + + #define ippcpGetLibVersion k0_ippcpGetLibVersion +#define ippsDESGetSize k0_ippsDESGetSize +#define ippsDESInit k0_ippsDESInit +#define ippsDESPack k0_ippsDESPack +#define ippsDESUnpack k0_ippsDESUnpack +#define ippsTDESEncryptECB k0_ippsTDESEncryptECB +#define ippsTDESDecryptECB k0_ippsTDESDecryptECB +#define ippsTDESEncryptCBC k0_ippsTDESEncryptCBC +#define ippsTDESDecryptCBC k0_ippsTDESDecryptCBC +#define ippsTDESEncryptCFB k0_ippsTDESEncryptCFB +#define ippsTDESDecryptCFB k0_ippsTDESDecryptCFB +#define ippsTDESEncryptOFB k0_ippsTDESEncryptOFB +#define ippsTDESDecryptOFB k0_ippsTDESDecryptOFB +#define ippsTDESEncryptCTR k0_ippsTDESEncryptCTR +#define ippsTDESDecryptCTR k0_ippsTDESDecryptCTR +#define ippsAESGetSize k0_ippsAESGetSize +#define ippsAESInit k0_ippsAESInit +#define ippsAESSetKey k0_ippsAESSetKey +#define ippsAESPack k0_ippsAESPack +#define ippsAESUnpack k0_ippsAESUnpack +#define ippsAESEncryptECB k0_ippsAESEncryptECB +#define ippsAESDecryptECB k0_ippsAESDecryptECB +#define ippsAESEncryptCBC k0_ippsAESEncryptCBC +#define ippsAESEncryptCBC_CS1 k0_ippsAESEncryptCBC_CS1 +#define ippsAESEncryptCBC_CS2 k0_ippsAESEncryptCBC_CS2 +#define ippsAESEncryptCBC_CS3 k0_ippsAESEncryptCBC_CS3 +#define ippsAESDecryptCBC k0_ippsAESDecryptCBC +#define ippsAESDecryptCBC_CS1 k0_ippsAESDecryptCBC_CS1 +#define ippsAESDecryptCBC_CS2 k0_ippsAESDecryptCBC_CS2 +#define ippsAESDecryptCBC_CS3 k0_ippsAESDecryptCBC_CS3 +#define ippsAESEncryptCFB k0_ippsAESEncryptCFB +#define ippsAESDecryptCFB k0_ippsAESDecryptCFB +#define ippsAESEncryptOFB k0_ippsAESEncryptOFB +#define ippsAESDecryptOFB k0_ippsAESDecryptOFB +#define ippsAESEncryptCTR k0_ippsAESEncryptCTR +#define ippsAESDecryptCTR k0_ippsAESDecryptCTR +#define ippsAESEncryptXTS_Direct k0_ippsAESEncryptXTS_Direct +#define ippsAESDecryptXTS_Direct k0_ippsAESDecryptXTS_Direct +#define ippsAESSetupNoise k0_ippsAESSetupNoise +#define ippsAES_GCMSetupNoise k0_ippsAES_GCMSetupNoise +#define ippsAES_CMACSetupNoise k0_ippsAES_CMACSetupNoise +#define ippsAES_EncryptCFB16_MB k0_ippsAES_EncryptCFB16_MB +#define ippsSMS4GetSize k0_ippsSMS4GetSize +#define ippsSMS4Init k0_ippsSMS4Init +#define ippsSMS4SetKey k0_ippsSMS4SetKey +#define ippsSMS4EncryptECB k0_ippsSMS4EncryptECB +#define ippsSMS4DecryptECB k0_ippsSMS4DecryptECB +#define ippsSMS4EncryptCBC k0_ippsSMS4EncryptCBC +#define ippsSMS4EncryptCBC_CS1 k0_ippsSMS4EncryptCBC_CS1 +#define ippsSMS4EncryptCBC_CS2 k0_ippsSMS4EncryptCBC_CS2 +#define ippsSMS4EncryptCBC_CS3 k0_ippsSMS4EncryptCBC_CS3 +#define ippsSMS4DecryptCBC k0_ippsSMS4DecryptCBC +#define ippsSMS4DecryptCBC_CS1 k0_ippsSMS4DecryptCBC_CS1 +#define ippsSMS4DecryptCBC_CS2 k0_ippsSMS4DecryptCBC_CS2 +#define ippsSMS4DecryptCBC_CS3 k0_ippsSMS4DecryptCBC_CS3 +#define ippsSMS4EncryptCFB k0_ippsSMS4EncryptCFB +#define ippsSMS4DecryptCFB k0_ippsSMS4DecryptCFB +#define ippsSMS4EncryptOFB k0_ippsSMS4EncryptOFB +#define ippsSMS4DecryptOFB k0_ippsSMS4DecryptOFB +#define ippsSMS4EncryptCTR k0_ippsSMS4EncryptCTR +#define ippsSMS4DecryptCTR k0_ippsSMS4DecryptCTR +#define ippsSMS4_CCMGetSize k0_ippsSMS4_CCMGetSize +#define ippsSMS4_CCMInit k0_ippsSMS4_CCMInit +#define ippsSMS4_CCMMessageLen k0_ippsSMS4_CCMMessageLen +#define ippsSMS4_CCMTagLen k0_ippsSMS4_CCMTagLen +#define ippsSMS4_CCMStart k0_ippsSMS4_CCMStart +#define ippsSMS4_CCMEncrypt k0_ippsSMS4_CCMEncrypt +#define ippsSMS4_CCMDecrypt k0_ippsSMS4_CCMDecrypt +#define ippsSMS4_CCMGetTag k0_ippsSMS4_CCMGetTag +#define ippsAES_CCMGetSize k0_ippsAES_CCMGetSize +#define ippsAES_CCMInit k0_ippsAES_CCMInit +#define ippsAES_CCMMessageLen k0_ippsAES_CCMMessageLen +#define ippsAES_CCMTagLen k0_ippsAES_CCMTagLen +#define ippsAES_CCMStart k0_ippsAES_CCMStart +#define ippsAES_CCMEncrypt k0_ippsAES_CCMEncrypt +#define ippsAES_CCMDecrypt k0_ippsAES_CCMDecrypt +#define ippsAES_CCMGetTag k0_ippsAES_CCMGetTag +#define ippsAES_GCMGetSize k0_ippsAES_GCMGetSize +#define ippsAES_GCMInit k0_ippsAES_GCMInit +#define ippsAES_GCMReinit k0_ippsAES_GCMReinit +#define ippsAES_GCMReset k0_ippsAES_GCMReset +#define ippsAES_GCMProcessIV k0_ippsAES_GCMProcessIV +#define ippsAES_GCMProcessAAD k0_ippsAES_GCMProcessAAD +#define ippsAES_GCMStart k0_ippsAES_GCMStart +#define ippsAES_GCMEncrypt k0_ippsAES_GCMEncrypt +#define ippsAES_GCMDecrypt k0_ippsAES_GCMDecrypt +#define ippsAES_GCMGetTag k0_ippsAES_GCMGetTag +#define ippsAES_XTSGetSize k0_ippsAES_XTSGetSize +#define ippsAES_XTSInit k0_ippsAES_XTSInit +#define ippsAES_XTSEncrypt k0_ippsAES_XTSEncrypt +#define ippsAES_XTSDecrypt k0_ippsAES_XTSDecrypt +#define ippsAES_S2V_CMAC k0_ippsAES_S2V_CMAC +#define ippsAES_SIVEncrypt k0_ippsAES_SIVEncrypt +#define ippsAES_SIVDecrypt k0_ippsAES_SIVDecrypt +#define ippsAES_CMACGetSize k0_ippsAES_CMACGetSize +#define ippsAES_CMACInit k0_ippsAES_CMACInit +#define ippsAES_CMACUpdate k0_ippsAES_CMACUpdate +#define ippsAES_CMACFinal k0_ippsAES_CMACFinal +#define ippsAES_CMACGetTag k0_ippsAES_CMACGetTag +#define ippsARCFourCheckKey k0_ippsARCFourCheckKey +#define ippsARCFourGetSize k0_ippsARCFourGetSize +#define ippsARCFourInit k0_ippsARCFourInit +#define ippsARCFourReset k0_ippsARCFourReset +#define ippsARCFourPack k0_ippsARCFourPack +#define ippsARCFourUnpack k0_ippsARCFourUnpack +#define ippsARCFourEncrypt k0_ippsARCFourEncrypt +#define ippsARCFourDecrypt k0_ippsARCFourDecrypt +#define ippsSHA1GetSize k0_ippsSHA1GetSize +#define ippsSHA1Init k0_ippsSHA1Init +#define ippsSHA1Duplicate k0_ippsSHA1Duplicate +#define ippsSHA1Pack k0_ippsSHA1Pack +#define ippsSHA1Unpack k0_ippsSHA1Unpack +#define ippsSHA1Update k0_ippsSHA1Update +#define ippsSHA1GetTag k0_ippsSHA1GetTag +#define ippsSHA1Final k0_ippsSHA1Final +#define ippsSHA1MessageDigest k0_ippsSHA1MessageDigest +#define ippsSHA224GetSize k0_ippsSHA224GetSize +#define ippsSHA224Init k0_ippsSHA224Init +#define ippsSHA224Duplicate k0_ippsSHA224Duplicate +#define ippsSHA224Pack k0_ippsSHA224Pack +#define ippsSHA224Unpack k0_ippsSHA224Unpack +#define ippsSHA224Update k0_ippsSHA224Update +#define ippsSHA224GetTag k0_ippsSHA224GetTag +#define ippsSHA224Final k0_ippsSHA224Final +#define ippsSHA224MessageDigest k0_ippsSHA224MessageDigest +#define ippsSHA256GetSize k0_ippsSHA256GetSize +#define ippsSHA256Init k0_ippsSHA256Init +#define ippsSHA256Duplicate k0_ippsSHA256Duplicate +#define ippsSHA256Pack k0_ippsSHA256Pack +#define ippsSHA256Unpack k0_ippsSHA256Unpack +#define ippsSHA256Update k0_ippsSHA256Update +#define ippsSHA256GetTag k0_ippsSHA256GetTag +#define ippsSHA256Final k0_ippsSHA256Final +#define ippsSHA256MessageDigest k0_ippsSHA256MessageDigest +#define ippsSHA384GetSize k0_ippsSHA384GetSize +#define ippsSHA384Init k0_ippsSHA384Init +#define ippsSHA384Duplicate k0_ippsSHA384Duplicate +#define ippsSHA384Pack k0_ippsSHA384Pack +#define ippsSHA384Unpack k0_ippsSHA384Unpack +#define ippsSHA384Update k0_ippsSHA384Update +#define ippsSHA384GetTag k0_ippsSHA384GetTag +#define ippsSHA384Final k0_ippsSHA384Final +#define ippsSHA384MessageDigest k0_ippsSHA384MessageDigest +#define ippsSHA512GetSize k0_ippsSHA512GetSize +#define ippsSHA512Init k0_ippsSHA512Init +#define ippsSHA512Duplicate k0_ippsSHA512Duplicate +#define ippsSHA512Pack k0_ippsSHA512Pack +#define ippsSHA512Unpack k0_ippsSHA512Unpack +#define ippsSHA512Update k0_ippsSHA512Update +#define ippsSHA512GetTag k0_ippsSHA512GetTag +#define ippsSHA512Final k0_ippsSHA512Final +#define ippsSHA512MessageDigest k0_ippsSHA512MessageDigest +#define ippsMD5GetSize k0_ippsMD5GetSize +#define ippsMD5Init k0_ippsMD5Init +#define ippsMD5Duplicate k0_ippsMD5Duplicate +#define ippsMD5Pack k0_ippsMD5Pack +#define ippsMD5Unpack k0_ippsMD5Unpack +#define ippsMD5Update k0_ippsMD5Update +#define ippsMD5GetTag k0_ippsMD5GetTag +#define ippsMD5Final k0_ippsMD5Final +#define ippsMD5MessageDigest k0_ippsMD5MessageDigest +#define ippsSM3GetSize k0_ippsSM3GetSize +#define ippsSM3Init k0_ippsSM3Init +#define ippsSM3Duplicate k0_ippsSM3Duplicate +#define ippsSM3Pack k0_ippsSM3Pack +#define ippsSM3Unpack k0_ippsSM3Unpack +#define ippsSM3Update k0_ippsSM3Update +#define ippsSM3GetTag k0_ippsSM3GetTag +#define ippsSM3Final k0_ippsSM3Final +#define ippsSM3MessageDigest k0_ippsSM3MessageDigest +#define ippsHashGetSize k0_ippsHashGetSize +#define ippsHashInit k0_ippsHashInit +#define ippsHashPack k0_ippsHashPack +#define ippsHashUnpack k0_ippsHashUnpack +#define ippsHashDuplicate k0_ippsHashDuplicate +#define ippsHashUpdate k0_ippsHashUpdate +#define ippsHashGetTag k0_ippsHashGetTag +#define ippsHashFinal k0_ippsHashFinal +#define ippsHashMessage k0_ippsHashMessage +#define ippsHashMethod_MD5 k0_ippsHashMethod_MD5 +#define ippsHashMethod_SM3 k0_ippsHashMethod_SM3 +#define ippsHashMethod_SHA1 k0_ippsHashMethod_SHA1 +#define ippsHashMethod_SHA1_NI k0_ippsHashMethod_SHA1_NI +#define ippsHashMethod_SHA1_TT k0_ippsHashMethod_SHA1_TT +#define ippsHashMethod_SHA256 k0_ippsHashMethod_SHA256 +#define ippsHashMethod_SHA256_NI k0_ippsHashMethod_SHA256_NI +#define ippsHashMethod_SHA256_TT k0_ippsHashMethod_SHA256_TT +#define ippsHashMethod_SHA224 k0_ippsHashMethod_SHA224 +#define ippsHashMethod_SHA224_NI k0_ippsHashMethod_SHA224_NI +#define ippsHashMethod_SHA224_TT k0_ippsHashMethod_SHA224_TT +#define ippsHashMethod_SHA512 k0_ippsHashMethod_SHA512 +#define ippsHashMethod_SHA384 k0_ippsHashMethod_SHA384 +#define ippsHashMethod_SHA512_256 k0_ippsHashMethod_SHA512_256 +#define ippsHashMethod_SHA512_224 k0_ippsHashMethod_SHA512_224 +#define ippsHashMethodGetSize k0_ippsHashMethodGetSize +#define ippsHashMethodSet_MD5 k0_ippsHashMethodSet_MD5 +#define ippsHashMethodSet_SM3 k0_ippsHashMethodSet_SM3 +#define ippsHashStateMethodSet_SM3 k0_ippsHashStateMethodSet_SM3 +#define ippsHashMethodSet_SHA1 k0_ippsHashMethodSet_SHA1 +#define ippsHashMethodSet_SHA1_NI k0_ippsHashMethodSet_SHA1_NI +#define ippsHashMethodSet_SHA1_TT k0_ippsHashMethodSet_SHA1_TT +#define ippsHashMethodSet_SHA256 k0_ippsHashMethodSet_SHA256 +#define ippsHashMethodSet_SHA256_NI k0_ippsHashMethodSet_SHA256_NI +#define ippsHashMethodSet_SHA256_TT k0_ippsHashMethodSet_SHA256_TT +#define ippsHashStateMethodSet_SHA256 k0_ippsHashStateMethodSet_SHA256 +#define ippsHashStateMethodSet_SHA256_NI k0_ippsHashStateMethodSet_SHA256_NI +#define ippsHashStateMethodSet_SHA256_TT k0_ippsHashStateMethodSet_SHA256_TT +#define ippsHashMethodSet_SHA224 k0_ippsHashMethodSet_SHA224 +#define ippsHashMethodSet_SHA224_NI k0_ippsHashMethodSet_SHA224_NI +#define ippsHashMethodSet_SHA224_TT k0_ippsHashMethodSet_SHA224_TT +#define ippsHashStateMethodSet_SHA224 k0_ippsHashStateMethodSet_SHA224 +#define ippsHashStateMethodSet_SHA224_NI k0_ippsHashStateMethodSet_SHA224_NI +#define ippsHashStateMethodSet_SHA224_TT k0_ippsHashStateMethodSet_SHA224_TT +#define ippsHashMethodSet_SHA512 k0_ippsHashMethodSet_SHA512 +#define ippsHashMethodSet_SHA384 k0_ippsHashMethodSet_SHA384 +#define ippsHashMethodSet_SHA512_256 k0_ippsHashMethodSet_SHA512_256 +#define ippsHashMethodSet_SHA512_224 k0_ippsHashMethodSet_SHA512_224 +#define ippsHashStateMethodSet_SHA512 k0_ippsHashStateMethodSet_SHA512 +#define ippsHashStateMethodSet_SHA384 k0_ippsHashStateMethodSet_SHA384 +#define ippsHashStateMethodSet_SHA512_256 k0_ippsHashStateMethodSet_SHA512_256 +#define ippsHashStateMethodSet_SHA512_224 k0_ippsHashStateMethodSet_SHA512_224 +#define ippsHashGetSize_rmf k0_ippsHashGetSize_rmf +#define ippsHashInit_rmf k0_ippsHashInit_rmf +#define ippsHashPack_rmf k0_ippsHashPack_rmf +#define ippsHashUnpack_rmf k0_ippsHashUnpack_rmf +#define ippsHashDuplicate_rmf k0_ippsHashDuplicate_rmf +#define ippsHashUpdate_rmf k0_ippsHashUpdate_rmf +#define ippsHashGetTag_rmf k0_ippsHashGetTag_rmf +#define ippsHashFinal_rmf k0_ippsHashFinal_rmf +#define ippsHashMessage_rmf k0_ippsHashMessage_rmf +#define ippsHashMethodGetInfo k0_ippsHashMethodGetInfo +#define ippsHashGetInfo_rmf k0_ippsHashGetInfo_rmf +#define ippsMGF k0_ippsMGF +#define ippsMGF1_rmf k0_ippsMGF1_rmf +#define ippsMGF2_rmf k0_ippsMGF2_rmf +#define ippsHMAC_GetSize k0_ippsHMAC_GetSize +#define ippsHMAC_Init k0_ippsHMAC_Init +#define ippsHMAC_Pack k0_ippsHMAC_Pack +#define ippsHMAC_Unpack k0_ippsHMAC_Unpack +#define ippsHMAC_Duplicate k0_ippsHMAC_Duplicate +#define ippsHMAC_Update k0_ippsHMAC_Update +#define ippsHMAC_Final k0_ippsHMAC_Final +#define ippsHMAC_GetTag k0_ippsHMAC_GetTag +#define ippsHMAC_Message k0_ippsHMAC_Message +#define ippsHMACGetSize_rmf k0_ippsHMACGetSize_rmf +#define ippsHMACInit_rmf k0_ippsHMACInit_rmf +#define ippsHMACPack_rmf k0_ippsHMACPack_rmf +#define ippsHMACUnpack_rmf k0_ippsHMACUnpack_rmf +#define ippsHMACDuplicate_rmf k0_ippsHMACDuplicate_rmf +#define ippsHMACUpdate_rmf k0_ippsHMACUpdate_rmf +#define ippsHMACFinal_rmf k0_ippsHMACFinal_rmf +#define ippsHMACGetTag_rmf k0_ippsHMACGetTag_rmf +#define ippsHMACMessage_rmf k0_ippsHMACMessage_rmf +#define ippsBigNumGetSize k0_ippsBigNumGetSize +#define ippsBigNumInit k0_ippsBigNumInit +#define ippsCmpZero_BN k0_ippsCmpZero_BN +#define ippsCmp_BN k0_ippsCmp_BN +#define ippsGetSize_BN k0_ippsGetSize_BN +#define ippsSet_BN k0_ippsSet_BN +#define ippsGet_BN k0_ippsGet_BN +#define ippsRef_BN k0_ippsRef_BN +#define ippsExtGet_BN k0_ippsExtGet_BN +#define ippsAdd_BN k0_ippsAdd_BN +#define ippsSub_BN k0_ippsSub_BN +#define ippsMul_BN k0_ippsMul_BN +#define ippsMAC_BN_I k0_ippsMAC_BN_I +#define ippsDiv_BN k0_ippsDiv_BN +#define ippsMod_BN k0_ippsMod_BN +#define ippsGcd_BN k0_ippsGcd_BN +#define ippsModInv_BN k0_ippsModInv_BN +#define ippsSetOctString_BN k0_ippsSetOctString_BN +#define ippsGetOctString_BN k0_ippsGetOctString_BN +#define ippsMontGetSize k0_ippsMontGetSize +#define ippsMontInit k0_ippsMontInit +#define ippsMontSet k0_ippsMontSet +#define ippsMontGet k0_ippsMontGet +#define ippsMontForm k0_ippsMontForm +#define ippsMontMul k0_ippsMontMul +#define ippsMontExp k0_ippsMontExp +#define ippsPRNGGetSize k0_ippsPRNGGetSize +#define ippsPRNGInit k0_ippsPRNGInit +#define ippsPRNGSetModulus k0_ippsPRNGSetModulus +#define ippsPRNGSetH0 k0_ippsPRNGSetH0 +#define ippsPRNGSetAugment k0_ippsPRNGSetAugment +#define ippsPRNGSetSeed k0_ippsPRNGSetSeed +#define ippsPRNGGetSeed k0_ippsPRNGGetSeed +#define ippsPRNGen k0_ippsPRNGen +#define ippsPRNGen_BN k0_ippsPRNGen_BN +#define ippsPRNGenRDRAND k0_ippsPRNGenRDRAND +#define ippsPRNGenRDRAND_BN k0_ippsPRNGenRDRAND_BN +#define ippsTRNGenRDSEED k0_ippsTRNGenRDSEED +#define ippsTRNGenRDSEED_BN k0_ippsTRNGenRDSEED_BN +#define ippsPrimeGetSize k0_ippsPrimeGetSize +#define ippsPrimeInit k0_ippsPrimeInit +#define ippsPrimeGen k0_ippsPrimeGen +#define ippsPrimeTest k0_ippsPrimeTest +#define ippsPrimeGen_BN k0_ippsPrimeGen_BN +#define ippsPrimeTest_BN k0_ippsPrimeTest_BN +#define ippsPrimeGet k0_ippsPrimeGet +#define ippsPrimeGet_BN k0_ippsPrimeGet_BN +#define ippsPrimeSet k0_ippsPrimeSet +#define ippsPrimeSet_BN k0_ippsPrimeSet_BN +#define ippsRSA_GetSizePublicKey k0_ippsRSA_GetSizePublicKey +#define ippsRSA_InitPublicKey k0_ippsRSA_InitPublicKey +#define ippsRSA_SetPublicKey k0_ippsRSA_SetPublicKey +#define ippsRSA_GetPublicKey k0_ippsRSA_GetPublicKey +#define ippsRSA_GetSizePrivateKeyType1 k0_ippsRSA_GetSizePrivateKeyType1 +#define ippsRSA_InitPrivateKeyType1 k0_ippsRSA_InitPrivateKeyType1 +#define ippsRSA_SetPrivateKeyType1 k0_ippsRSA_SetPrivateKeyType1 +#define ippsRSA_GetPrivateKeyType1 k0_ippsRSA_GetPrivateKeyType1 +#define ippsRSA_GetSizePrivateKeyType2 k0_ippsRSA_GetSizePrivateKeyType2 +#define ippsRSA_InitPrivateKeyType2 k0_ippsRSA_InitPrivateKeyType2 +#define ippsRSA_SetPrivateKeyType2 k0_ippsRSA_SetPrivateKeyType2 +#define ippsRSA_GetPrivateKeyType2 k0_ippsRSA_GetPrivateKeyType2 +#define ippsRSA_GetBufferSizePublicKey k0_ippsRSA_GetBufferSizePublicKey +#define ippsRSA_GetBufferSizePrivateKey k0_ippsRSA_GetBufferSizePrivateKey +#define ippsRSA_Encrypt k0_ippsRSA_Encrypt +#define ippsRSA_Decrypt k0_ippsRSA_Decrypt +#define ippsRSA_GenerateKeys k0_ippsRSA_GenerateKeys +#define ippsRSA_ValidateKeys k0_ippsRSA_ValidateKeys +#define ippsRSAEncrypt_OAEP k0_ippsRSAEncrypt_OAEP +#define ippsRSADecrypt_OAEP k0_ippsRSADecrypt_OAEP +#define ippsRSAEncrypt_OAEP_rmf k0_ippsRSAEncrypt_OAEP_rmf +#define ippsRSADecrypt_OAEP_rmf k0_ippsRSADecrypt_OAEP_rmf +#define ippsRSAEncrypt_PKCSv15 k0_ippsRSAEncrypt_PKCSv15 +#define ippsRSADecrypt_PKCSv15 k0_ippsRSADecrypt_PKCSv15 +#define ippsRSASign_PSS k0_ippsRSASign_PSS +#define ippsRSAVerify_PSS k0_ippsRSAVerify_PSS +#define ippsRSASign_PSS_rmf k0_ippsRSASign_PSS_rmf +#define ippsRSAVerify_PSS_rmf k0_ippsRSAVerify_PSS_rmf +#define ippsRSASign_PKCS1v15 k0_ippsRSASign_PKCS1v15 +#define ippsRSAVerify_PKCS1v15 k0_ippsRSAVerify_PKCS1v15 +#define ippsRSASign_PKCS1v15_rmf k0_ippsRSASign_PKCS1v15_rmf +#define ippsRSAVerify_PKCS1v15_rmf k0_ippsRSAVerify_PKCS1v15_rmf +#define ippsDLGetResultString k0_ippsDLGetResultString +#define ippsDLPGetSize k0_ippsDLPGetSize +#define ippsDLPInit k0_ippsDLPInit +#define ippsDLPPack k0_ippsDLPPack +#define ippsDLPUnpack k0_ippsDLPUnpack +#define ippsDLPSet k0_ippsDLPSet +#define ippsDLPGet k0_ippsDLPGet +#define ippsDLPSetDP k0_ippsDLPSetDP +#define ippsDLPGetDP k0_ippsDLPGetDP +#define ippsDLPGenKeyPair k0_ippsDLPGenKeyPair +#define ippsDLPPublicKey k0_ippsDLPPublicKey +#define ippsDLPValidateKeyPair k0_ippsDLPValidateKeyPair +#define ippsDLPSetKeyPair k0_ippsDLPSetKeyPair +#define ippsDLPSignDSA k0_ippsDLPSignDSA +#define ippsDLPVerifyDSA k0_ippsDLPVerifyDSA +#define ippsDLPSharedSecretDH k0_ippsDLPSharedSecretDH +#define ippsDLPGenerateDSA k0_ippsDLPGenerateDSA +#define ippsDLPValidateDSA k0_ippsDLPValidateDSA +#define ippsDLPGenerateDH k0_ippsDLPGenerateDH +#define ippsDLPValidateDH k0_ippsDLPValidateDH +#define ippsECCGetResultString k0_ippsECCGetResultString +#define ippsECCPGetSize k0_ippsECCPGetSize +#define ippsECCPGetSizeStd128r1 k0_ippsECCPGetSizeStd128r1 +#define ippsECCPGetSizeStd128r2 k0_ippsECCPGetSizeStd128r2 +#define ippsECCPGetSizeStd192r1 k0_ippsECCPGetSizeStd192r1 +#define ippsECCPGetSizeStd224r1 k0_ippsECCPGetSizeStd224r1 +#define ippsECCPGetSizeStd256r1 k0_ippsECCPGetSizeStd256r1 +#define ippsECCPGetSizeStd384r1 k0_ippsECCPGetSizeStd384r1 +#define ippsECCPGetSizeStd521r1 k0_ippsECCPGetSizeStd521r1 +#define ippsECCPGetSizeStdSM2 k0_ippsECCPGetSizeStdSM2 +#define ippsECCPInit k0_ippsECCPInit +#define ippsECCPInitStd128r1 k0_ippsECCPInitStd128r1 +#define ippsECCPInitStd128r2 k0_ippsECCPInitStd128r2 +#define ippsECCPInitStd192r1 k0_ippsECCPInitStd192r1 +#define ippsECCPInitStd224r1 k0_ippsECCPInitStd224r1 +#define ippsECCPInitStd256r1 k0_ippsECCPInitStd256r1 +#define ippsECCPInitStd384r1 k0_ippsECCPInitStd384r1 +#define ippsECCPInitStd521r1 k0_ippsECCPInitStd521r1 +#define ippsECCPInitStdSM2 k0_ippsECCPInitStdSM2 +#define ippsECCPSet k0_ippsECCPSet +#define ippsECCPSetStd k0_ippsECCPSetStd +#define ippsECCPSetStd128r1 k0_ippsECCPSetStd128r1 +#define ippsECCPSetStd128r2 k0_ippsECCPSetStd128r2 +#define ippsECCPSetStd192r1 k0_ippsECCPSetStd192r1 +#define ippsECCPSetStd224r1 k0_ippsECCPSetStd224r1 +#define ippsECCPSetStd256r1 k0_ippsECCPSetStd256r1 +#define ippsECCPSetStd384r1 k0_ippsECCPSetStd384r1 +#define ippsECCPSetStd521r1 k0_ippsECCPSetStd521r1 +#define ippsECCPSetStdSM2 k0_ippsECCPSetStdSM2 +#define ippsECCPBindGxyTblStd192r1 k0_ippsECCPBindGxyTblStd192r1 +#define ippsECCPBindGxyTblStd224r1 k0_ippsECCPBindGxyTblStd224r1 +#define ippsECCPBindGxyTblStd256r1 k0_ippsECCPBindGxyTblStd256r1 +#define ippsECCPBindGxyTblStd384r1 k0_ippsECCPBindGxyTblStd384r1 +#define ippsECCPBindGxyTblStd521r1 k0_ippsECCPBindGxyTblStd521r1 +#define ippsECCPBindGxyTblStdSM2 k0_ippsECCPBindGxyTblStdSM2 +#define ippsECCPGet k0_ippsECCPGet +#define ippsECCPGetOrderBitSize k0_ippsECCPGetOrderBitSize +#define ippsECCPValidate k0_ippsECCPValidate +#define ippsECCPPointGetSize k0_ippsECCPPointGetSize +#define ippsECCPPointInit k0_ippsECCPPointInit +#define ippsECCPSetPoint k0_ippsECCPSetPoint +#define ippsECCPSetPointAtInfinity k0_ippsECCPSetPointAtInfinity +#define ippsECCPGetPoint k0_ippsECCPGetPoint +#define ippsECCPCheckPoint k0_ippsECCPCheckPoint +#define ippsECCPComparePoint k0_ippsECCPComparePoint +#define ippsECCPNegativePoint k0_ippsECCPNegativePoint +#define ippsECCPAddPoint k0_ippsECCPAddPoint +#define ippsECCPMulPointScalar k0_ippsECCPMulPointScalar +#define ippsECCPGenKeyPair k0_ippsECCPGenKeyPair +#define ippsECCPPublicKey k0_ippsECCPPublicKey +#define ippsECCPValidateKeyPair k0_ippsECCPValidateKeyPair +#define ippsECCPSetKeyPair k0_ippsECCPSetKeyPair +#define ippsECCPSharedSecretDH k0_ippsECCPSharedSecretDH +#define ippsECCPSharedSecretDHC k0_ippsECCPSharedSecretDHC +#define ippsECCPSignDSA k0_ippsECCPSignDSA +#define ippsECCPVerifyDSA k0_ippsECCPVerifyDSA +#define ippsECCPSignNR k0_ippsECCPSignNR +#define ippsECCPVerifyNR k0_ippsECCPVerifyNR +#define ippsECCPSignSM2 k0_ippsECCPSignSM2 +#define ippsECCPVerifySM2 k0_ippsECCPVerifySM2 +#define ippsGFpGetSize k0_ippsGFpGetSize +#define ippsGFpInitArbitrary k0_ippsGFpInitArbitrary +#define ippsGFpInitFixed k0_ippsGFpInitFixed +#define ippsGFpInit k0_ippsGFpInit +#define ippsGFpMethod_p192r1 k0_ippsGFpMethod_p192r1 +#define ippsGFpMethod_p224r1 k0_ippsGFpMethod_p224r1 +#define ippsGFpMethod_p256r1 k0_ippsGFpMethod_p256r1 +#define ippsGFpMethod_p384r1 k0_ippsGFpMethod_p384r1 +#define ippsGFpMethod_p521r1 k0_ippsGFpMethod_p521r1 +#define ippsGFpMethod_p256sm2 k0_ippsGFpMethod_p256sm2 +#define ippsGFpMethod_p256bn k0_ippsGFpMethod_p256bn +#define ippsGFpMethod_p256 k0_ippsGFpMethod_p256 +#define ippsGFpMethod_pArb k0_ippsGFpMethod_pArb +#define ippsGFpxGetSize k0_ippsGFpxGetSize +#define ippsGFpxInit k0_ippsGFpxInit +#define ippsGFpxInitBinomial k0_ippsGFpxInitBinomial +#define ippsGFpxMethod_binom2_epid2 k0_ippsGFpxMethod_binom2_epid2 +#define ippsGFpxMethod_binom3_epid2 k0_ippsGFpxMethod_binom3_epid2 +#define ippsGFpxMethod_binom2 k0_ippsGFpxMethod_binom2 +#define ippsGFpxMethod_binom3 k0_ippsGFpxMethod_binom3 +#define ippsGFpxMethod_binom k0_ippsGFpxMethod_binom +#define ippsGFpxMethod_com k0_ippsGFpxMethod_com +#define ippsGFpScratchBufferSize k0_ippsGFpScratchBufferSize +#define ippsGFpElementGetSize k0_ippsGFpElementGetSize +#define ippsGFpElementInit k0_ippsGFpElementInit +#define ippsGFpSetElement k0_ippsGFpSetElement +#define ippsGFpSetElementRegular k0_ippsGFpSetElementRegular +#define ippsGFpSetElementOctString k0_ippsGFpSetElementOctString +#define ippsGFpSetElementRandom k0_ippsGFpSetElementRandom +#define ippsGFpSetElementHash k0_ippsGFpSetElementHash +#define ippsGFpSetElementHash_rmf k0_ippsGFpSetElementHash_rmf +#define ippsGFpCpyElement k0_ippsGFpCpyElement +#define ippsGFpGetElement k0_ippsGFpGetElement +#define ippsGFpGetElementOctString k0_ippsGFpGetElementOctString +#define ippsGFpCmpElement k0_ippsGFpCmpElement +#define ippsGFpIsZeroElement k0_ippsGFpIsZeroElement +#define ippsGFpIsUnityElement k0_ippsGFpIsUnityElement +#define ippsGFpConj k0_ippsGFpConj +#define ippsGFpNeg k0_ippsGFpNeg +#define ippsGFpInv k0_ippsGFpInv +#define ippsGFpSqrt k0_ippsGFpSqrt +#define ippsGFpSqr k0_ippsGFpSqr +#define ippsGFpAdd k0_ippsGFpAdd +#define ippsGFpSub k0_ippsGFpSub +#define ippsGFpMul k0_ippsGFpMul +#define ippsGFpExp k0_ippsGFpExp +#define ippsGFpMultiExp k0_ippsGFpMultiExp +#define ippsGFpAdd_PE k0_ippsGFpAdd_PE +#define ippsGFpSub_PE k0_ippsGFpSub_PE +#define ippsGFpMul_PE k0_ippsGFpMul_PE +#define ippsGFpGetInfo k0_ippsGFpGetInfo +#define ippsGFpECGetSize k0_ippsGFpECGetSize +#define ippsGFpECInit k0_ippsGFpECInit +#define ippsGFpECSet k0_ippsGFpECSet +#define ippsGFpECSetSubgroup k0_ippsGFpECSetSubgroup +#define ippsGFpECInitStd128r1 k0_ippsGFpECInitStd128r1 +#define ippsGFpECInitStd128r2 k0_ippsGFpECInitStd128r2 +#define ippsGFpECInitStd192r1 k0_ippsGFpECInitStd192r1 +#define ippsGFpECInitStd224r1 k0_ippsGFpECInitStd224r1 +#define ippsGFpECInitStd256r1 k0_ippsGFpECInitStd256r1 +#define ippsGFpECInitStd384r1 k0_ippsGFpECInitStd384r1 +#define ippsGFpECInitStd521r1 k0_ippsGFpECInitStd521r1 +#define ippsGFpECInitStdSM2 k0_ippsGFpECInitStdSM2 +#define ippsGFpECInitStdBN256 k0_ippsGFpECInitStdBN256 +#define ippsGFpECBindGxyTblStd192r1 k0_ippsGFpECBindGxyTblStd192r1 +#define ippsGFpECBindGxyTblStd224r1 k0_ippsGFpECBindGxyTblStd224r1 +#define ippsGFpECBindGxyTblStd256r1 k0_ippsGFpECBindGxyTblStd256r1 +#define ippsGFpECBindGxyTblStd384r1 k0_ippsGFpECBindGxyTblStd384r1 +#define ippsGFpECBindGxyTblStd521r1 k0_ippsGFpECBindGxyTblStd521r1 +#define ippsGFpECBindGxyTblStdSM2 k0_ippsGFpECBindGxyTblStdSM2 +#define ippsGFpECGet k0_ippsGFpECGet +#define ippsGFpECGetSubgroup k0_ippsGFpECGetSubgroup +#define ippsGFpECScratchBufferSize k0_ippsGFpECScratchBufferSize +#define ippsGFpECVerify k0_ippsGFpECVerify +#define ippsGFpECPointGetSize k0_ippsGFpECPointGetSize +#define ippsGFpECPointInit k0_ippsGFpECPointInit +#define ippsGFpECSetPointAtInfinity k0_ippsGFpECSetPointAtInfinity +#define ippsGFpECSetPoint k0_ippsGFpECSetPoint +#define ippsGFpECSetPointRegular k0_ippsGFpECSetPointRegular +#define ippsGFpECSetPointRandom k0_ippsGFpECSetPointRandom +#define ippsGFpECMakePoint k0_ippsGFpECMakePoint +#define ippsGFpECSetPointHash k0_ippsGFpECSetPointHash +#define ippsGFpECSetPointHashBackCompatible k0_ippsGFpECSetPointHashBackCompatible +#define ippsGFpECSetPointHash_rmf k0_ippsGFpECSetPointHash_rmf +#define ippsGFpECSetPointHashBackCompatible_rmf k0_ippsGFpECSetPointHashBackCompatible_rmf +#define ippsGFpECGetPoint k0_ippsGFpECGetPoint +#define ippsGFpECGetPointRegular k0_ippsGFpECGetPointRegular +#define ippsGFpECSetPointOctString k0_ippsGFpECSetPointOctString +#define ippsGFpECGetPointOctString k0_ippsGFpECGetPointOctString +#define ippsGFpECTstPoint k0_ippsGFpECTstPoint +#define ippsGFpECTstPointInSubgroup k0_ippsGFpECTstPointInSubgroup +#define ippsGFpECCpyPoint k0_ippsGFpECCpyPoint +#define ippsGFpECCmpPoint k0_ippsGFpECCmpPoint +#define ippsGFpECNegPoint k0_ippsGFpECNegPoint +#define ippsGFpECAddPoint k0_ippsGFpECAddPoint +#define ippsGFpECMulPoint k0_ippsGFpECMulPoint +#define ippsGFpECPrivateKey k0_ippsGFpECPrivateKey +#define ippsGFpECPublicKey k0_ippsGFpECPublicKey +#define ippsGFpECTstKeyPair k0_ippsGFpECTstKeyPair +#define ippsGFpECSharedSecretDH k0_ippsGFpECSharedSecretDH +#define ippsGFpECSharedSecretDHC k0_ippsGFpECSharedSecretDHC +#define ippsGFpECMessageRepresentationSM2 k0_ippsGFpECMessageRepresentationSM2 +#define ippsGFpECSignDSA k0_ippsGFpECSignDSA +#define ippsGFpECVerifyDSA k0_ippsGFpECVerifyDSA +#define ippsGFpECSignNR k0_ippsGFpECSignNR +#define ippsGFpECVerifyNR k0_ippsGFpECVerifyNR +#define ippsGFpECSignSM2 k0_ippsGFpECSignSM2 +#define ippsGFpECVerifySM2 k0_ippsGFpECVerifySM2 +#define ippsGFpECUserIDHashSM2 k0_ippsGFpECUserIDHashSM2 +#define ippsGFpECKeyExchangeSM2_GetSize k0_ippsGFpECKeyExchangeSM2_GetSize +#define ippsGFpECKeyExchangeSM2_Init k0_ippsGFpECKeyExchangeSM2_Init +#define ippsGFpECKeyExchangeSM2_Setup k0_ippsGFpECKeyExchangeSM2_Setup +#define ippsGFpECKeyExchangeSM2_SharedKey k0_ippsGFpECKeyExchangeSM2_SharedKey +#define ippsGFpECKeyExchangeSM2_Confirm k0_ippsGFpECKeyExchangeSM2_Confirm +#define ippsGFpECGetInfo_GF k0_ippsGFpECGetInfo_GF +#define ippsGFpECESGetSize_SM2 k0_ippsGFpECESGetSize_SM2 +#define ippsGFpECESInit_SM2 k0_ippsGFpECESInit_SM2 +#define ippsGFpECESSetKey_SM2 k0_ippsGFpECESSetKey_SM2 +#define ippsGFpECESStart_SM2 k0_ippsGFpECESStart_SM2 +#define ippsGFpECESEncrypt_SM2 k0_ippsGFpECESEncrypt_SM2 +#define ippsGFpECESDecrypt_SM2 k0_ippsGFpECESDecrypt_SM2 +#define ippsGFpECESFinal_SM2 k0_ippsGFpECESFinal_SM2 +#define ippsGFpECESGetBuffersSize_SM2 k0_ippsGFpECESGetBuffersSize_SM2 +#define ippsGFpECEncryptSM2_Ext_EncMsgSize k0_ippsGFpECEncryptSM2_Ext_EncMsgSize +#define ippsGFpECEncryptSM2_Ext k0_ippsGFpECEncryptSM2_Ext +#define ippsGFpECDecryptSM2_Ext_DecMsgSize k0_ippsGFpECDecryptSM2_Ext_DecMsgSize +#define ippsGFpECDecryptSM2_Ext k0_ippsGFpECDecryptSM2_Ext diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_k1.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_k1.h new file mode 100644 index 0000000..8f5aad3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_k1.h @@ -0,0 +1,557 @@ +/******************************************************************************* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + + #define ippcpGetLibVersion k1_ippcpGetLibVersion +#define ippsDESGetSize k1_ippsDESGetSize +#define ippsDESInit k1_ippsDESInit +#define ippsDESPack k1_ippsDESPack +#define ippsDESUnpack k1_ippsDESUnpack +#define ippsTDESEncryptECB k1_ippsTDESEncryptECB +#define ippsTDESDecryptECB k1_ippsTDESDecryptECB +#define ippsTDESEncryptCBC k1_ippsTDESEncryptCBC +#define ippsTDESDecryptCBC k1_ippsTDESDecryptCBC +#define ippsTDESEncryptCFB k1_ippsTDESEncryptCFB +#define ippsTDESDecryptCFB k1_ippsTDESDecryptCFB +#define ippsTDESEncryptOFB k1_ippsTDESEncryptOFB +#define ippsTDESDecryptOFB k1_ippsTDESDecryptOFB +#define ippsTDESEncryptCTR k1_ippsTDESEncryptCTR +#define ippsTDESDecryptCTR k1_ippsTDESDecryptCTR +#define ippsAESGetSize k1_ippsAESGetSize +#define ippsAESInit k1_ippsAESInit +#define ippsAESSetKey k1_ippsAESSetKey +#define ippsAESPack k1_ippsAESPack +#define ippsAESUnpack k1_ippsAESUnpack +#define ippsAESEncryptECB k1_ippsAESEncryptECB +#define ippsAESDecryptECB k1_ippsAESDecryptECB +#define ippsAESEncryptCBC k1_ippsAESEncryptCBC +#define ippsAESEncryptCBC_CS1 k1_ippsAESEncryptCBC_CS1 +#define ippsAESEncryptCBC_CS2 k1_ippsAESEncryptCBC_CS2 +#define ippsAESEncryptCBC_CS3 k1_ippsAESEncryptCBC_CS3 +#define ippsAESDecryptCBC k1_ippsAESDecryptCBC +#define ippsAESDecryptCBC_CS1 k1_ippsAESDecryptCBC_CS1 +#define ippsAESDecryptCBC_CS2 k1_ippsAESDecryptCBC_CS2 +#define ippsAESDecryptCBC_CS3 k1_ippsAESDecryptCBC_CS3 +#define ippsAESEncryptCFB k1_ippsAESEncryptCFB +#define ippsAESDecryptCFB k1_ippsAESDecryptCFB +#define ippsAESEncryptOFB k1_ippsAESEncryptOFB +#define ippsAESDecryptOFB k1_ippsAESDecryptOFB +#define ippsAESEncryptCTR k1_ippsAESEncryptCTR +#define ippsAESDecryptCTR k1_ippsAESDecryptCTR +#define ippsAESEncryptXTS_Direct k1_ippsAESEncryptXTS_Direct +#define ippsAESDecryptXTS_Direct k1_ippsAESDecryptXTS_Direct +#define ippsAESSetupNoise k1_ippsAESSetupNoise +#define ippsAES_GCMSetupNoise k1_ippsAES_GCMSetupNoise +#define ippsAES_CMACSetupNoise k1_ippsAES_CMACSetupNoise +#define ippsAES_EncryptCFB16_MB k1_ippsAES_EncryptCFB16_MB +#define ippsSMS4GetSize k1_ippsSMS4GetSize +#define ippsSMS4Init k1_ippsSMS4Init +#define ippsSMS4SetKey k1_ippsSMS4SetKey +#define ippsSMS4EncryptECB k1_ippsSMS4EncryptECB +#define ippsSMS4DecryptECB k1_ippsSMS4DecryptECB +#define ippsSMS4EncryptCBC k1_ippsSMS4EncryptCBC +#define ippsSMS4EncryptCBC_CS1 k1_ippsSMS4EncryptCBC_CS1 +#define ippsSMS4EncryptCBC_CS2 k1_ippsSMS4EncryptCBC_CS2 +#define ippsSMS4EncryptCBC_CS3 k1_ippsSMS4EncryptCBC_CS3 +#define ippsSMS4DecryptCBC k1_ippsSMS4DecryptCBC +#define ippsSMS4DecryptCBC_CS1 k1_ippsSMS4DecryptCBC_CS1 +#define ippsSMS4DecryptCBC_CS2 k1_ippsSMS4DecryptCBC_CS2 +#define ippsSMS4DecryptCBC_CS3 k1_ippsSMS4DecryptCBC_CS3 +#define ippsSMS4EncryptCFB k1_ippsSMS4EncryptCFB +#define ippsSMS4DecryptCFB k1_ippsSMS4DecryptCFB +#define ippsSMS4EncryptOFB k1_ippsSMS4EncryptOFB +#define ippsSMS4DecryptOFB k1_ippsSMS4DecryptOFB +#define ippsSMS4EncryptCTR k1_ippsSMS4EncryptCTR +#define ippsSMS4DecryptCTR k1_ippsSMS4DecryptCTR +#define ippsSMS4_CCMGetSize k1_ippsSMS4_CCMGetSize +#define ippsSMS4_CCMInit k1_ippsSMS4_CCMInit +#define ippsSMS4_CCMMessageLen k1_ippsSMS4_CCMMessageLen +#define ippsSMS4_CCMTagLen k1_ippsSMS4_CCMTagLen +#define ippsSMS4_CCMStart k1_ippsSMS4_CCMStart +#define ippsSMS4_CCMEncrypt k1_ippsSMS4_CCMEncrypt +#define ippsSMS4_CCMDecrypt k1_ippsSMS4_CCMDecrypt +#define ippsSMS4_CCMGetTag k1_ippsSMS4_CCMGetTag +#define ippsAES_CCMGetSize k1_ippsAES_CCMGetSize +#define ippsAES_CCMInit k1_ippsAES_CCMInit +#define ippsAES_CCMMessageLen k1_ippsAES_CCMMessageLen +#define ippsAES_CCMTagLen k1_ippsAES_CCMTagLen +#define ippsAES_CCMStart k1_ippsAES_CCMStart +#define ippsAES_CCMEncrypt k1_ippsAES_CCMEncrypt +#define ippsAES_CCMDecrypt k1_ippsAES_CCMDecrypt +#define ippsAES_CCMGetTag k1_ippsAES_CCMGetTag +#define ippsAES_GCMGetSize k1_ippsAES_GCMGetSize +#define ippsAES_GCMInit k1_ippsAES_GCMInit +#define ippsAES_GCMReinit k1_ippsAES_GCMReinit +#define ippsAES_GCMReset k1_ippsAES_GCMReset +#define ippsAES_GCMProcessIV k1_ippsAES_GCMProcessIV +#define ippsAES_GCMProcessAAD k1_ippsAES_GCMProcessAAD +#define ippsAES_GCMStart k1_ippsAES_GCMStart +#define ippsAES_GCMEncrypt k1_ippsAES_GCMEncrypt +#define ippsAES_GCMDecrypt k1_ippsAES_GCMDecrypt +#define ippsAES_GCMGetTag k1_ippsAES_GCMGetTag +#define ippsAES_XTSGetSize k1_ippsAES_XTSGetSize +#define ippsAES_XTSInit k1_ippsAES_XTSInit +#define ippsAES_XTSEncrypt k1_ippsAES_XTSEncrypt +#define ippsAES_XTSDecrypt k1_ippsAES_XTSDecrypt +#define ippsAES_S2V_CMAC k1_ippsAES_S2V_CMAC +#define ippsAES_SIVEncrypt k1_ippsAES_SIVEncrypt +#define ippsAES_SIVDecrypt k1_ippsAES_SIVDecrypt +#define ippsAES_CMACGetSize k1_ippsAES_CMACGetSize +#define ippsAES_CMACInit k1_ippsAES_CMACInit +#define ippsAES_CMACUpdate k1_ippsAES_CMACUpdate +#define ippsAES_CMACFinal k1_ippsAES_CMACFinal +#define ippsAES_CMACGetTag k1_ippsAES_CMACGetTag +#define ippsARCFourCheckKey k1_ippsARCFourCheckKey +#define ippsARCFourGetSize k1_ippsARCFourGetSize +#define ippsARCFourInit k1_ippsARCFourInit +#define ippsARCFourReset k1_ippsARCFourReset +#define ippsARCFourPack k1_ippsARCFourPack +#define ippsARCFourUnpack k1_ippsARCFourUnpack +#define ippsARCFourEncrypt k1_ippsARCFourEncrypt +#define ippsARCFourDecrypt k1_ippsARCFourDecrypt +#define ippsSHA1GetSize k1_ippsSHA1GetSize +#define ippsSHA1Init k1_ippsSHA1Init +#define ippsSHA1Duplicate k1_ippsSHA1Duplicate +#define ippsSHA1Pack k1_ippsSHA1Pack +#define ippsSHA1Unpack k1_ippsSHA1Unpack +#define ippsSHA1Update k1_ippsSHA1Update +#define ippsSHA1GetTag k1_ippsSHA1GetTag +#define ippsSHA1Final k1_ippsSHA1Final +#define ippsSHA1MessageDigest k1_ippsSHA1MessageDigest +#define ippsSHA224GetSize k1_ippsSHA224GetSize +#define ippsSHA224Init k1_ippsSHA224Init +#define ippsSHA224Duplicate k1_ippsSHA224Duplicate +#define ippsSHA224Pack k1_ippsSHA224Pack +#define ippsSHA224Unpack k1_ippsSHA224Unpack +#define ippsSHA224Update k1_ippsSHA224Update +#define ippsSHA224GetTag k1_ippsSHA224GetTag +#define ippsSHA224Final k1_ippsSHA224Final +#define ippsSHA224MessageDigest k1_ippsSHA224MessageDigest +#define ippsSHA256GetSize k1_ippsSHA256GetSize +#define ippsSHA256Init k1_ippsSHA256Init +#define ippsSHA256Duplicate k1_ippsSHA256Duplicate +#define ippsSHA256Pack k1_ippsSHA256Pack +#define ippsSHA256Unpack k1_ippsSHA256Unpack +#define ippsSHA256Update k1_ippsSHA256Update +#define ippsSHA256GetTag k1_ippsSHA256GetTag +#define ippsSHA256Final k1_ippsSHA256Final +#define ippsSHA256MessageDigest k1_ippsSHA256MessageDigest +#define ippsSHA384GetSize k1_ippsSHA384GetSize +#define ippsSHA384Init k1_ippsSHA384Init +#define ippsSHA384Duplicate k1_ippsSHA384Duplicate +#define ippsSHA384Pack k1_ippsSHA384Pack +#define ippsSHA384Unpack k1_ippsSHA384Unpack +#define ippsSHA384Update k1_ippsSHA384Update +#define ippsSHA384GetTag k1_ippsSHA384GetTag +#define ippsSHA384Final k1_ippsSHA384Final +#define ippsSHA384MessageDigest k1_ippsSHA384MessageDigest +#define ippsSHA512GetSize k1_ippsSHA512GetSize +#define ippsSHA512Init k1_ippsSHA512Init +#define ippsSHA512Duplicate k1_ippsSHA512Duplicate +#define ippsSHA512Pack k1_ippsSHA512Pack +#define ippsSHA512Unpack k1_ippsSHA512Unpack +#define ippsSHA512Update k1_ippsSHA512Update +#define ippsSHA512GetTag k1_ippsSHA512GetTag +#define ippsSHA512Final k1_ippsSHA512Final +#define ippsSHA512MessageDigest k1_ippsSHA512MessageDigest +#define ippsMD5GetSize k1_ippsMD5GetSize +#define ippsMD5Init k1_ippsMD5Init +#define ippsMD5Duplicate k1_ippsMD5Duplicate +#define ippsMD5Pack k1_ippsMD5Pack +#define ippsMD5Unpack k1_ippsMD5Unpack +#define ippsMD5Update k1_ippsMD5Update +#define ippsMD5GetTag k1_ippsMD5GetTag +#define ippsMD5Final k1_ippsMD5Final +#define ippsMD5MessageDigest k1_ippsMD5MessageDigest +#define ippsSM3GetSize k1_ippsSM3GetSize +#define ippsSM3Init k1_ippsSM3Init +#define ippsSM3Duplicate k1_ippsSM3Duplicate +#define ippsSM3Pack k1_ippsSM3Pack +#define ippsSM3Unpack k1_ippsSM3Unpack +#define ippsSM3Update k1_ippsSM3Update +#define ippsSM3GetTag k1_ippsSM3GetTag +#define ippsSM3Final k1_ippsSM3Final +#define ippsSM3MessageDigest k1_ippsSM3MessageDigest +#define ippsHashGetSize k1_ippsHashGetSize +#define ippsHashInit k1_ippsHashInit +#define ippsHashPack k1_ippsHashPack +#define ippsHashUnpack k1_ippsHashUnpack +#define ippsHashDuplicate k1_ippsHashDuplicate +#define ippsHashUpdate k1_ippsHashUpdate +#define ippsHashGetTag k1_ippsHashGetTag +#define ippsHashFinal k1_ippsHashFinal +#define ippsHashMessage k1_ippsHashMessage +#define ippsHashMethod_MD5 k1_ippsHashMethod_MD5 +#define ippsHashMethod_SM3 k1_ippsHashMethod_SM3 +#define ippsHashMethod_SHA1 k1_ippsHashMethod_SHA1 +#define ippsHashMethod_SHA1_NI k1_ippsHashMethod_SHA1_NI +#define ippsHashMethod_SHA1_TT k1_ippsHashMethod_SHA1_TT +#define ippsHashMethod_SHA256 k1_ippsHashMethod_SHA256 +#define ippsHashMethod_SHA256_NI k1_ippsHashMethod_SHA256_NI +#define ippsHashMethod_SHA256_TT k1_ippsHashMethod_SHA256_TT +#define ippsHashMethod_SHA224 k1_ippsHashMethod_SHA224 +#define ippsHashMethod_SHA224_NI k1_ippsHashMethod_SHA224_NI +#define ippsHashMethod_SHA224_TT k1_ippsHashMethod_SHA224_TT +#define ippsHashMethod_SHA512 k1_ippsHashMethod_SHA512 +#define ippsHashMethod_SHA384 k1_ippsHashMethod_SHA384 +#define ippsHashMethod_SHA512_256 k1_ippsHashMethod_SHA512_256 +#define ippsHashMethod_SHA512_224 k1_ippsHashMethod_SHA512_224 +#define ippsHashMethodGetSize k1_ippsHashMethodGetSize +#define ippsHashMethodSet_MD5 k1_ippsHashMethodSet_MD5 +#define ippsHashMethodSet_SM3 k1_ippsHashMethodSet_SM3 +#define ippsHashStateMethodSet_SM3 k1_ippsHashStateMethodSet_SM3 +#define ippsHashMethodSet_SHA1 k1_ippsHashMethodSet_SHA1 +#define ippsHashMethodSet_SHA1_NI k1_ippsHashMethodSet_SHA1_NI +#define ippsHashMethodSet_SHA1_TT k1_ippsHashMethodSet_SHA1_TT +#define ippsHashMethodSet_SHA256 k1_ippsHashMethodSet_SHA256 +#define ippsHashMethodSet_SHA256_NI k1_ippsHashMethodSet_SHA256_NI +#define ippsHashMethodSet_SHA256_TT k1_ippsHashMethodSet_SHA256_TT +#define ippsHashStateMethodSet_SHA256 k1_ippsHashStateMethodSet_SHA256 +#define ippsHashStateMethodSet_SHA256_NI k1_ippsHashStateMethodSet_SHA256_NI +#define ippsHashStateMethodSet_SHA256_TT k1_ippsHashStateMethodSet_SHA256_TT +#define ippsHashMethodSet_SHA224 k1_ippsHashMethodSet_SHA224 +#define ippsHashMethodSet_SHA224_NI k1_ippsHashMethodSet_SHA224_NI +#define ippsHashMethodSet_SHA224_TT k1_ippsHashMethodSet_SHA224_TT +#define ippsHashStateMethodSet_SHA224 k1_ippsHashStateMethodSet_SHA224 +#define ippsHashStateMethodSet_SHA224_NI k1_ippsHashStateMethodSet_SHA224_NI +#define ippsHashStateMethodSet_SHA224_TT k1_ippsHashStateMethodSet_SHA224_TT +#define ippsHashMethodSet_SHA512 k1_ippsHashMethodSet_SHA512 +#define ippsHashMethodSet_SHA384 k1_ippsHashMethodSet_SHA384 +#define ippsHashMethodSet_SHA512_256 k1_ippsHashMethodSet_SHA512_256 +#define ippsHashMethodSet_SHA512_224 k1_ippsHashMethodSet_SHA512_224 +#define ippsHashStateMethodSet_SHA512 k1_ippsHashStateMethodSet_SHA512 +#define ippsHashStateMethodSet_SHA384 k1_ippsHashStateMethodSet_SHA384 +#define ippsHashStateMethodSet_SHA512_256 k1_ippsHashStateMethodSet_SHA512_256 +#define ippsHashStateMethodSet_SHA512_224 k1_ippsHashStateMethodSet_SHA512_224 +#define ippsHashGetSize_rmf k1_ippsHashGetSize_rmf +#define ippsHashInit_rmf k1_ippsHashInit_rmf +#define ippsHashPack_rmf k1_ippsHashPack_rmf +#define ippsHashUnpack_rmf k1_ippsHashUnpack_rmf +#define ippsHashDuplicate_rmf k1_ippsHashDuplicate_rmf +#define ippsHashUpdate_rmf k1_ippsHashUpdate_rmf +#define ippsHashGetTag_rmf k1_ippsHashGetTag_rmf +#define ippsHashFinal_rmf k1_ippsHashFinal_rmf +#define ippsHashMessage_rmf k1_ippsHashMessage_rmf +#define ippsHashMethodGetInfo k1_ippsHashMethodGetInfo +#define ippsHashGetInfo_rmf k1_ippsHashGetInfo_rmf +#define ippsMGF k1_ippsMGF +#define ippsMGF1_rmf k1_ippsMGF1_rmf +#define ippsMGF2_rmf k1_ippsMGF2_rmf +#define ippsHMAC_GetSize k1_ippsHMAC_GetSize +#define ippsHMAC_Init k1_ippsHMAC_Init +#define ippsHMAC_Pack k1_ippsHMAC_Pack +#define ippsHMAC_Unpack k1_ippsHMAC_Unpack +#define ippsHMAC_Duplicate k1_ippsHMAC_Duplicate +#define ippsHMAC_Update k1_ippsHMAC_Update +#define ippsHMAC_Final k1_ippsHMAC_Final +#define ippsHMAC_GetTag k1_ippsHMAC_GetTag +#define ippsHMAC_Message k1_ippsHMAC_Message +#define ippsHMACGetSize_rmf k1_ippsHMACGetSize_rmf +#define ippsHMACInit_rmf k1_ippsHMACInit_rmf +#define ippsHMACPack_rmf k1_ippsHMACPack_rmf +#define ippsHMACUnpack_rmf k1_ippsHMACUnpack_rmf +#define ippsHMACDuplicate_rmf k1_ippsHMACDuplicate_rmf +#define ippsHMACUpdate_rmf k1_ippsHMACUpdate_rmf +#define ippsHMACFinal_rmf k1_ippsHMACFinal_rmf +#define ippsHMACGetTag_rmf k1_ippsHMACGetTag_rmf +#define ippsHMACMessage_rmf k1_ippsHMACMessage_rmf +#define ippsBigNumGetSize k1_ippsBigNumGetSize +#define ippsBigNumInit k1_ippsBigNumInit +#define ippsCmpZero_BN k1_ippsCmpZero_BN +#define ippsCmp_BN k1_ippsCmp_BN +#define ippsGetSize_BN k1_ippsGetSize_BN +#define ippsSet_BN k1_ippsSet_BN +#define ippsGet_BN k1_ippsGet_BN +#define ippsRef_BN k1_ippsRef_BN +#define ippsExtGet_BN k1_ippsExtGet_BN +#define ippsAdd_BN k1_ippsAdd_BN +#define ippsSub_BN k1_ippsSub_BN +#define ippsMul_BN k1_ippsMul_BN +#define ippsMAC_BN_I k1_ippsMAC_BN_I +#define ippsDiv_BN k1_ippsDiv_BN +#define ippsMod_BN k1_ippsMod_BN +#define ippsGcd_BN k1_ippsGcd_BN +#define ippsModInv_BN k1_ippsModInv_BN +#define ippsSetOctString_BN k1_ippsSetOctString_BN +#define ippsGetOctString_BN k1_ippsGetOctString_BN +#define ippsMontGetSize k1_ippsMontGetSize +#define ippsMontInit k1_ippsMontInit +#define ippsMontSet k1_ippsMontSet +#define ippsMontGet k1_ippsMontGet +#define ippsMontForm k1_ippsMontForm +#define ippsMontMul k1_ippsMontMul +#define ippsMontExp k1_ippsMontExp +#define ippsPRNGGetSize k1_ippsPRNGGetSize +#define ippsPRNGInit k1_ippsPRNGInit +#define ippsPRNGSetModulus k1_ippsPRNGSetModulus +#define ippsPRNGSetH0 k1_ippsPRNGSetH0 +#define ippsPRNGSetAugment k1_ippsPRNGSetAugment +#define ippsPRNGSetSeed k1_ippsPRNGSetSeed +#define ippsPRNGGetSeed k1_ippsPRNGGetSeed +#define ippsPRNGen k1_ippsPRNGen +#define ippsPRNGen_BN k1_ippsPRNGen_BN +#define ippsPRNGenRDRAND k1_ippsPRNGenRDRAND +#define ippsPRNGenRDRAND_BN k1_ippsPRNGenRDRAND_BN +#define ippsTRNGenRDSEED k1_ippsTRNGenRDSEED +#define ippsTRNGenRDSEED_BN k1_ippsTRNGenRDSEED_BN +#define ippsPrimeGetSize k1_ippsPrimeGetSize +#define ippsPrimeInit k1_ippsPrimeInit +#define ippsPrimeGen k1_ippsPrimeGen +#define ippsPrimeTest k1_ippsPrimeTest +#define ippsPrimeGen_BN k1_ippsPrimeGen_BN +#define ippsPrimeTest_BN k1_ippsPrimeTest_BN +#define ippsPrimeGet k1_ippsPrimeGet +#define ippsPrimeGet_BN k1_ippsPrimeGet_BN +#define ippsPrimeSet k1_ippsPrimeSet +#define ippsPrimeSet_BN k1_ippsPrimeSet_BN +#define ippsRSA_GetSizePublicKey k1_ippsRSA_GetSizePublicKey +#define ippsRSA_InitPublicKey k1_ippsRSA_InitPublicKey +#define ippsRSA_SetPublicKey k1_ippsRSA_SetPublicKey +#define ippsRSA_GetPublicKey k1_ippsRSA_GetPublicKey +#define ippsRSA_GetSizePrivateKeyType1 k1_ippsRSA_GetSizePrivateKeyType1 +#define ippsRSA_InitPrivateKeyType1 k1_ippsRSA_InitPrivateKeyType1 +#define ippsRSA_SetPrivateKeyType1 k1_ippsRSA_SetPrivateKeyType1 +#define ippsRSA_GetPrivateKeyType1 k1_ippsRSA_GetPrivateKeyType1 +#define ippsRSA_GetSizePrivateKeyType2 k1_ippsRSA_GetSizePrivateKeyType2 +#define ippsRSA_InitPrivateKeyType2 k1_ippsRSA_InitPrivateKeyType2 +#define ippsRSA_SetPrivateKeyType2 k1_ippsRSA_SetPrivateKeyType2 +#define ippsRSA_GetPrivateKeyType2 k1_ippsRSA_GetPrivateKeyType2 +#define ippsRSA_GetBufferSizePublicKey k1_ippsRSA_GetBufferSizePublicKey +#define ippsRSA_GetBufferSizePrivateKey k1_ippsRSA_GetBufferSizePrivateKey +#define ippsRSA_Encrypt k1_ippsRSA_Encrypt +#define ippsRSA_Decrypt k1_ippsRSA_Decrypt +#define ippsRSA_GenerateKeys k1_ippsRSA_GenerateKeys +#define ippsRSA_ValidateKeys k1_ippsRSA_ValidateKeys +#define ippsRSAEncrypt_OAEP k1_ippsRSAEncrypt_OAEP +#define ippsRSADecrypt_OAEP k1_ippsRSADecrypt_OAEP +#define ippsRSAEncrypt_OAEP_rmf k1_ippsRSAEncrypt_OAEP_rmf +#define ippsRSADecrypt_OAEP_rmf k1_ippsRSADecrypt_OAEP_rmf +#define ippsRSAEncrypt_PKCSv15 k1_ippsRSAEncrypt_PKCSv15 +#define ippsRSADecrypt_PKCSv15 k1_ippsRSADecrypt_PKCSv15 +#define ippsRSASign_PSS k1_ippsRSASign_PSS +#define ippsRSAVerify_PSS k1_ippsRSAVerify_PSS +#define ippsRSASign_PSS_rmf k1_ippsRSASign_PSS_rmf +#define ippsRSAVerify_PSS_rmf k1_ippsRSAVerify_PSS_rmf +#define ippsRSASign_PKCS1v15 k1_ippsRSASign_PKCS1v15 +#define ippsRSAVerify_PKCS1v15 k1_ippsRSAVerify_PKCS1v15 +#define ippsRSASign_PKCS1v15_rmf k1_ippsRSASign_PKCS1v15_rmf +#define ippsRSAVerify_PKCS1v15_rmf k1_ippsRSAVerify_PKCS1v15_rmf +#define ippsDLGetResultString k1_ippsDLGetResultString +#define ippsDLPGetSize k1_ippsDLPGetSize +#define ippsDLPInit k1_ippsDLPInit +#define ippsDLPPack k1_ippsDLPPack +#define ippsDLPUnpack k1_ippsDLPUnpack +#define ippsDLPSet k1_ippsDLPSet +#define ippsDLPGet k1_ippsDLPGet +#define ippsDLPSetDP k1_ippsDLPSetDP +#define ippsDLPGetDP k1_ippsDLPGetDP +#define ippsDLPGenKeyPair k1_ippsDLPGenKeyPair +#define ippsDLPPublicKey k1_ippsDLPPublicKey +#define ippsDLPValidateKeyPair k1_ippsDLPValidateKeyPair +#define ippsDLPSetKeyPair k1_ippsDLPSetKeyPair +#define ippsDLPSignDSA k1_ippsDLPSignDSA +#define ippsDLPVerifyDSA k1_ippsDLPVerifyDSA +#define ippsDLPSharedSecretDH k1_ippsDLPSharedSecretDH +#define ippsDLPGenerateDSA k1_ippsDLPGenerateDSA +#define ippsDLPValidateDSA k1_ippsDLPValidateDSA +#define ippsDLPGenerateDH k1_ippsDLPGenerateDH +#define ippsDLPValidateDH k1_ippsDLPValidateDH +#define ippsECCGetResultString k1_ippsECCGetResultString +#define ippsECCPGetSize k1_ippsECCPGetSize +#define ippsECCPGetSizeStd128r1 k1_ippsECCPGetSizeStd128r1 +#define ippsECCPGetSizeStd128r2 k1_ippsECCPGetSizeStd128r2 +#define ippsECCPGetSizeStd192r1 k1_ippsECCPGetSizeStd192r1 +#define ippsECCPGetSizeStd224r1 k1_ippsECCPGetSizeStd224r1 +#define ippsECCPGetSizeStd256r1 k1_ippsECCPGetSizeStd256r1 +#define ippsECCPGetSizeStd384r1 k1_ippsECCPGetSizeStd384r1 +#define ippsECCPGetSizeStd521r1 k1_ippsECCPGetSizeStd521r1 +#define ippsECCPGetSizeStdSM2 k1_ippsECCPGetSizeStdSM2 +#define ippsECCPInit k1_ippsECCPInit +#define ippsECCPInitStd128r1 k1_ippsECCPInitStd128r1 +#define ippsECCPInitStd128r2 k1_ippsECCPInitStd128r2 +#define ippsECCPInitStd192r1 k1_ippsECCPInitStd192r1 +#define ippsECCPInitStd224r1 k1_ippsECCPInitStd224r1 +#define ippsECCPInitStd256r1 k1_ippsECCPInitStd256r1 +#define ippsECCPInitStd384r1 k1_ippsECCPInitStd384r1 +#define ippsECCPInitStd521r1 k1_ippsECCPInitStd521r1 +#define ippsECCPInitStdSM2 k1_ippsECCPInitStdSM2 +#define ippsECCPSet k1_ippsECCPSet +#define ippsECCPSetStd k1_ippsECCPSetStd +#define ippsECCPSetStd128r1 k1_ippsECCPSetStd128r1 +#define ippsECCPSetStd128r2 k1_ippsECCPSetStd128r2 +#define ippsECCPSetStd192r1 k1_ippsECCPSetStd192r1 +#define ippsECCPSetStd224r1 k1_ippsECCPSetStd224r1 +#define ippsECCPSetStd256r1 k1_ippsECCPSetStd256r1 +#define ippsECCPSetStd384r1 k1_ippsECCPSetStd384r1 +#define ippsECCPSetStd521r1 k1_ippsECCPSetStd521r1 +#define ippsECCPSetStdSM2 k1_ippsECCPSetStdSM2 +#define ippsECCPBindGxyTblStd192r1 k1_ippsECCPBindGxyTblStd192r1 +#define ippsECCPBindGxyTblStd224r1 k1_ippsECCPBindGxyTblStd224r1 +#define ippsECCPBindGxyTblStd256r1 k1_ippsECCPBindGxyTblStd256r1 +#define ippsECCPBindGxyTblStd384r1 k1_ippsECCPBindGxyTblStd384r1 +#define ippsECCPBindGxyTblStd521r1 k1_ippsECCPBindGxyTblStd521r1 +#define ippsECCPBindGxyTblStdSM2 k1_ippsECCPBindGxyTblStdSM2 +#define ippsECCPGet k1_ippsECCPGet +#define ippsECCPGetOrderBitSize k1_ippsECCPGetOrderBitSize +#define ippsECCPValidate k1_ippsECCPValidate +#define ippsECCPPointGetSize k1_ippsECCPPointGetSize +#define ippsECCPPointInit k1_ippsECCPPointInit +#define ippsECCPSetPoint k1_ippsECCPSetPoint +#define ippsECCPSetPointAtInfinity k1_ippsECCPSetPointAtInfinity +#define ippsECCPGetPoint k1_ippsECCPGetPoint +#define ippsECCPCheckPoint k1_ippsECCPCheckPoint +#define ippsECCPComparePoint k1_ippsECCPComparePoint +#define ippsECCPNegativePoint k1_ippsECCPNegativePoint +#define ippsECCPAddPoint k1_ippsECCPAddPoint +#define ippsECCPMulPointScalar k1_ippsECCPMulPointScalar +#define ippsECCPGenKeyPair k1_ippsECCPGenKeyPair +#define ippsECCPPublicKey k1_ippsECCPPublicKey +#define ippsECCPValidateKeyPair k1_ippsECCPValidateKeyPair +#define ippsECCPSetKeyPair k1_ippsECCPSetKeyPair +#define ippsECCPSharedSecretDH k1_ippsECCPSharedSecretDH +#define ippsECCPSharedSecretDHC k1_ippsECCPSharedSecretDHC +#define ippsECCPSignDSA k1_ippsECCPSignDSA +#define ippsECCPVerifyDSA k1_ippsECCPVerifyDSA +#define ippsECCPSignNR k1_ippsECCPSignNR +#define ippsECCPVerifyNR k1_ippsECCPVerifyNR +#define ippsECCPSignSM2 k1_ippsECCPSignSM2 +#define ippsECCPVerifySM2 k1_ippsECCPVerifySM2 +#define ippsGFpGetSize k1_ippsGFpGetSize +#define ippsGFpInitArbitrary k1_ippsGFpInitArbitrary +#define ippsGFpInitFixed k1_ippsGFpInitFixed +#define ippsGFpInit k1_ippsGFpInit +#define ippsGFpMethod_p192r1 k1_ippsGFpMethod_p192r1 +#define ippsGFpMethod_p224r1 k1_ippsGFpMethod_p224r1 +#define ippsGFpMethod_p256r1 k1_ippsGFpMethod_p256r1 +#define ippsGFpMethod_p384r1 k1_ippsGFpMethod_p384r1 +#define ippsGFpMethod_p521r1 k1_ippsGFpMethod_p521r1 +#define ippsGFpMethod_p256sm2 k1_ippsGFpMethod_p256sm2 +#define ippsGFpMethod_p256bn k1_ippsGFpMethod_p256bn +#define ippsGFpMethod_p256 k1_ippsGFpMethod_p256 +#define ippsGFpMethod_pArb k1_ippsGFpMethod_pArb +#define ippsGFpxGetSize k1_ippsGFpxGetSize +#define ippsGFpxInit k1_ippsGFpxInit +#define ippsGFpxInitBinomial k1_ippsGFpxInitBinomial +#define ippsGFpxMethod_binom2_epid2 k1_ippsGFpxMethod_binom2_epid2 +#define ippsGFpxMethod_binom3_epid2 k1_ippsGFpxMethod_binom3_epid2 +#define ippsGFpxMethod_binom2 k1_ippsGFpxMethod_binom2 +#define ippsGFpxMethod_binom3 k1_ippsGFpxMethod_binom3 +#define ippsGFpxMethod_binom k1_ippsGFpxMethod_binom +#define ippsGFpxMethod_com k1_ippsGFpxMethod_com +#define ippsGFpScratchBufferSize k1_ippsGFpScratchBufferSize +#define ippsGFpElementGetSize k1_ippsGFpElementGetSize +#define ippsGFpElementInit k1_ippsGFpElementInit +#define ippsGFpSetElement k1_ippsGFpSetElement +#define ippsGFpSetElementRegular k1_ippsGFpSetElementRegular +#define ippsGFpSetElementOctString k1_ippsGFpSetElementOctString +#define ippsGFpSetElementRandom k1_ippsGFpSetElementRandom +#define ippsGFpSetElementHash k1_ippsGFpSetElementHash +#define ippsGFpSetElementHash_rmf k1_ippsGFpSetElementHash_rmf +#define ippsGFpCpyElement k1_ippsGFpCpyElement +#define ippsGFpGetElement k1_ippsGFpGetElement +#define ippsGFpGetElementOctString k1_ippsGFpGetElementOctString +#define ippsGFpCmpElement k1_ippsGFpCmpElement +#define ippsGFpIsZeroElement k1_ippsGFpIsZeroElement +#define ippsGFpIsUnityElement k1_ippsGFpIsUnityElement +#define ippsGFpConj k1_ippsGFpConj +#define ippsGFpNeg k1_ippsGFpNeg +#define ippsGFpInv k1_ippsGFpInv +#define ippsGFpSqrt k1_ippsGFpSqrt +#define ippsGFpSqr k1_ippsGFpSqr +#define ippsGFpAdd k1_ippsGFpAdd +#define ippsGFpSub k1_ippsGFpSub +#define ippsGFpMul k1_ippsGFpMul +#define ippsGFpExp k1_ippsGFpExp +#define ippsGFpMultiExp k1_ippsGFpMultiExp +#define ippsGFpAdd_PE k1_ippsGFpAdd_PE +#define ippsGFpSub_PE k1_ippsGFpSub_PE +#define ippsGFpMul_PE k1_ippsGFpMul_PE +#define ippsGFpGetInfo k1_ippsGFpGetInfo +#define ippsGFpECGetSize k1_ippsGFpECGetSize +#define ippsGFpECInit k1_ippsGFpECInit +#define ippsGFpECSet k1_ippsGFpECSet +#define ippsGFpECSetSubgroup k1_ippsGFpECSetSubgroup +#define ippsGFpECInitStd128r1 k1_ippsGFpECInitStd128r1 +#define ippsGFpECInitStd128r2 k1_ippsGFpECInitStd128r2 +#define ippsGFpECInitStd192r1 k1_ippsGFpECInitStd192r1 +#define ippsGFpECInitStd224r1 k1_ippsGFpECInitStd224r1 +#define ippsGFpECInitStd256r1 k1_ippsGFpECInitStd256r1 +#define ippsGFpECInitStd384r1 k1_ippsGFpECInitStd384r1 +#define ippsGFpECInitStd521r1 k1_ippsGFpECInitStd521r1 +#define ippsGFpECInitStdSM2 k1_ippsGFpECInitStdSM2 +#define ippsGFpECInitStdBN256 k1_ippsGFpECInitStdBN256 +#define ippsGFpECBindGxyTblStd192r1 k1_ippsGFpECBindGxyTblStd192r1 +#define ippsGFpECBindGxyTblStd224r1 k1_ippsGFpECBindGxyTblStd224r1 +#define ippsGFpECBindGxyTblStd256r1 k1_ippsGFpECBindGxyTblStd256r1 +#define ippsGFpECBindGxyTblStd384r1 k1_ippsGFpECBindGxyTblStd384r1 +#define ippsGFpECBindGxyTblStd521r1 k1_ippsGFpECBindGxyTblStd521r1 +#define ippsGFpECBindGxyTblStdSM2 k1_ippsGFpECBindGxyTblStdSM2 +#define ippsGFpECGet k1_ippsGFpECGet +#define ippsGFpECGetSubgroup k1_ippsGFpECGetSubgroup +#define ippsGFpECScratchBufferSize k1_ippsGFpECScratchBufferSize +#define ippsGFpECVerify k1_ippsGFpECVerify +#define ippsGFpECPointGetSize k1_ippsGFpECPointGetSize +#define ippsGFpECPointInit k1_ippsGFpECPointInit +#define ippsGFpECSetPointAtInfinity k1_ippsGFpECSetPointAtInfinity +#define ippsGFpECSetPoint k1_ippsGFpECSetPoint +#define ippsGFpECSetPointRegular k1_ippsGFpECSetPointRegular +#define ippsGFpECSetPointRandom k1_ippsGFpECSetPointRandom +#define ippsGFpECMakePoint k1_ippsGFpECMakePoint +#define ippsGFpECSetPointHash k1_ippsGFpECSetPointHash +#define ippsGFpECSetPointHashBackCompatible k1_ippsGFpECSetPointHashBackCompatible +#define ippsGFpECSetPointHash_rmf k1_ippsGFpECSetPointHash_rmf +#define ippsGFpECSetPointHashBackCompatible_rmf k1_ippsGFpECSetPointHashBackCompatible_rmf +#define ippsGFpECGetPoint k1_ippsGFpECGetPoint +#define ippsGFpECGetPointRegular k1_ippsGFpECGetPointRegular +#define ippsGFpECSetPointOctString k1_ippsGFpECSetPointOctString +#define ippsGFpECGetPointOctString k1_ippsGFpECGetPointOctString +#define ippsGFpECTstPoint k1_ippsGFpECTstPoint +#define ippsGFpECTstPointInSubgroup k1_ippsGFpECTstPointInSubgroup +#define ippsGFpECCpyPoint k1_ippsGFpECCpyPoint +#define ippsGFpECCmpPoint k1_ippsGFpECCmpPoint +#define ippsGFpECNegPoint k1_ippsGFpECNegPoint +#define ippsGFpECAddPoint k1_ippsGFpECAddPoint +#define ippsGFpECMulPoint k1_ippsGFpECMulPoint +#define ippsGFpECPrivateKey k1_ippsGFpECPrivateKey +#define ippsGFpECPublicKey k1_ippsGFpECPublicKey +#define ippsGFpECTstKeyPair k1_ippsGFpECTstKeyPair +#define ippsGFpECSharedSecretDH k1_ippsGFpECSharedSecretDH +#define ippsGFpECSharedSecretDHC k1_ippsGFpECSharedSecretDHC +#define ippsGFpECMessageRepresentationSM2 k1_ippsGFpECMessageRepresentationSM2 +#define ippsGFpECSignDSA k1_ippsGFpECSignDSA +#define ippsGFpECVerifyDSA k1_ippsGFpECVerifyDSA +#define ippsGFpECSignNR k1_ippsGFpECSignNR +#define ippsGFpECVerifyNR k1_ippsGFpECVerifyNR +#define ippsGFpECSignSM2 k1_ippsGFpECSignSM2 +#define ippsGFpECVerifySM2 k1_ippsGFpECVerifySM2 +#define ippsGFpECUserIDHashSM2 k1_ippsGFpECUserIDHashSM2 +#define ippsGFpECKeyExchangeSM2_GetSize k1_ippsGFpECKeyExchangeSM2_GetSize +#define ippsGFpECKeyExchangeSM2_Init k1_ippsGFpECKeyExchangeSM2_Init +#define ippsGFpECKeyExchangeSM2_Setup k1_ippsGFpECKeyExchangeSM2_Setup +#define ippsGFpECKeyExchangeSM2_SharedKey k1_ippsGFpECKeyExchangeSM2_SharedKey +#define ippsGFpECKeyExchangeSM2_Confirm k1_ippsGFpECKeyExchangeSM2_Confirm +#define ippsGFpECGetInfo_GF k1_ippsGFpECGetInfo_GF +#define ippsGFpECESGetSize_SM2 k1_ippsGFpECESGetSize_SM2 +#define ippsGFpECESInit_SM2 k1_ippsGFpECESInit_SM2 +#define ippsGFpECESSetKey_SM2 k1_ippsGFpECESSetKey_SM2 +#define ippsGFpECESStart_SM2 k1_ippsGFpECESStart_SM2 +#define ippsGFpECESEncrypt_SM2 k1_ippsGFpECESEncrypt_SM2 +#define ippsGFpECESDecrypt_SM2 k1_ippsGFpECESDecrypt_SM2 +#define ippsGFpECESFinal_SM2 k1_ippsGFpECESFinal_SM2 +#define ippsGFpECESGetBuffersSize_SM2 k1_ippsGFpECESGetBuffersSize_SM2 +#define ippsGFpECEncryptSM2_Ext_EncMsgSize k1_ippsGFpECEncryptSM2_Ext_EncMsgSize +#define ippsGFpECEncryptSM2_Ext k1_ippsGFpECEncryptSM2_Ext +#define ippsGFpECDecryptSM2_Ext_DecMsgSize k1_ippsGFpECDecryptSM2_Ext_DecMsgSize +#define ippsGFpECDecryptSM2_Ext k1_ippsGFpECDecryptSM2_Ext diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_l9.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_l9.h new file mode 100644 index 0000000..76c5d16 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_l9.h @@ -0,0 +1,557 @@ +/******************************************************************************* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + + #define ippcpGetLibVersion l9_ippcpGetLibVersion +#define ippsDESGetSize l9_ippsDESGetSize +#define ippsDESInit l9_ippsDESInit +#define ippsDESPack l9_ippsDESPack +#define ippsDESUnpack l9_ippsDESUnpack +#define ippsTDESEncryptECB l9_ippsTDESEncryptECB +#define ippsTDESDecryptECB l9_ippsTDESDecryptECB +#define ippsTDESEncryptCBC l9_ippsTDESEncryptCBC +#define ippsTDESDecryptCBC l9_ippsTDESDecryptCBC +#define ippsTDESEncryptCFB l9_ippsTDESEncryptCFB +#define ippsTDESDecryptCFB l9_ippsTDESDecryptCFB +#define ippsTDESEncryptOFB l9_ippsTDESEncryptOFB +#define ippsTDESDecryptOFB l9_ippsTDESDecryptOFB +#define ippsTDESEncryptCTR l9_ippsTDESEncryptCTR +#define ippsTDESDecryptCTR l9_ippsTDESDecryptCTR +#define ippsAESGetSize l9_ippsAESGetSize +#define ippsAESInit l9_ippsAESInit +#define ippsAESSetKey l9_ippsAESSetKey +#define ippsAESPack l9_ippsAESPack +#define ippsAESUnpack l9_ippsAESUnpack +#define ippsAESEncryptECB l9_ippsAESEncryptECB +#define ippsAESDecryptECB l9_ippsAESDecryptECB +#define ippsAESEncryptCBC l9_ippsAESEncryptCBC +#define ippsAESEncryptCBC_CS1 l9_ippsAESEncryptCBC_CS1 +#define ippsAESEncryptCBC_CS2 l9_ippsAESEncryptCBC_CS2 +#define ippsAESEncryptCBC_CS3 l9_ippsAESEncryptCBC_CS3 +#define ippsAESDecryptCBC l9_ippsAESDecryptCBC +#define ippsAESDecryptCBC_CS1 l9_ippsAESDecryptCBC_CS1 +#define ippsAESDecryptCBC_CS2 l9_ippsAESDecryptCBC_CS2 +#define ippsAESDecryptCBC_CS3 l9_ippsAESDecryptCBC_CS3 +#define ippsAESEncryptCFB l9_ippsAESEncryptCFB +#define ippsAESDecryptCFB l9_ippsAESDecryptCFB +#define ippsAESEncryptOFB l9_ippsAESEncryptOFB +#define ippsAESDecryptOFB l9_ippsAESDecryptOFB +#define ippsAESEncryptCTR l9_ippsAESEncryptCTR +#define ippsAESDecryptCTR l9_ippsAESDecryptCTR +#define ippsAESEncryptXTS_Direct l9_ippsAESEncryptXTS_Direct +#define ippsAESDecryptXTS_Direct l9_ippsAESDecryptXTS_Direct +#define ippsAESSetupNoise l9_ippsAESSetupNoise +#define ippsAES_GCMSetupNoise l9_ippsAES_GCMSetupNoise +#define ippsAES_CMACSetupNoise l9_ippsAES_CMACSetupNoise +#define ippsAES_EncryptCFB16_MB l9_ippsAES_EncryptCFB16_MB +#define ippsSMS4GetSize l9_ippsSMS4GetSize +#define ippsSMS4Init l9_ippsSMS4Init +#define ippsSMS4SetKey l9_ippsSMS4SetKey +#define ippsSMS4EncryptECB l9_ippsSMS4EncryptECB +#define ippsSMS4DecryptECB l9_ippsSMS4DecryptECB +#define ippsSMS4EncryptCBC l9_ippsSMS4EncryptCBC +#define ippsSMS4EncryptCBC_CS1 l9_ippsSMS4EncryptCBC_CS1 +#define ippsSMS4EncryptCBC_CS2 l9_ippsSMS4EncryptCBC_CS2 +#define ippsSMS4EncryptCBC_CS3 l9_ippsSMS4EncryptCBC_CS3 +#define ippsSMS4DecryptCBC l9_ippsSMS4DecryptCBC +#define ippsSMS4DecryptCBC_CS1 l9_ippsSMS4DecryptCBC_CS1 +#define ippsSMS4DecryptCBC_CS2 l9_ippsSMS4DecryptCBC_CS2 +#define ippsSMS4DecryptCBC_CS3 l9_ippsSMS4DecryptCBC_CS3 +#define ippsSMS4EncryptCFB l9_ippsSMS4EncryptCFB +#define ippsSMS4DecryptCFB l9_ippsSMS4DecryptCFB +#define ippsSMS4EncryptOFB l9_ippsSMS4EncryptOFB +#define ippsSMS4DecryptOFB l9_ippsSMS4DecryptOFB +#define ippsSMS4EncryptCTR l9_ippsSMS4EncryptCTR +#define ippsSMS4DecryptCTR l9_ippsSMS4DecryptCTR +#define ippsSMS4_CCMGetSize l9_ippsSMS4_CCMGetSize +#define ippsSMS4_CCMInit l9_ippsSMS4_CCMInit +#define ippsSMS4_CCMMessageLen l9_ippsSMS4_CCMMessageLen +#define ippsSMS4_CCMTagLen l9_ippsSMS4_CCMTagLen +#define ippsSMS4_CCMStart l9_ippsSMS4_CCMStart +#define ippsSMS4_CCMEncrypt l9_ippsSMS4_CCMEncrypt +#define ippsSMS4_CCMDecrypt l9_ippsSMS4_CCMDecrypt +#define ippsSMS4_CCMGetTag l9_ippsSMS4_CCMGetTag +#define ippsAES_CCMGetSize l9_ippsAES_CCMGetSize +#define ippsAES_CCMInit l9_ippsAES_CCMInit +#define ippsAES_CCMMessageLen l9_ippsAES_CCMMessageLen +#define ippsAES_CCMTagLen l9_ippsAES_CCMTagLen +#define ippsAES_CCMStart l9_ippsAES_CCMStart +#define ippsAES_CCMEncrypt l9_ippsAES_CCMEncrypt +#define ippsAES_CCMDecrypt l9_ippsAES_CCMDecrypt +#define ippsAES_CCMGetTag l9_ippsAES_CCMGetTag +#define ippsAES_GCMGetSize l9_ippsAES_GCMGetSize +#define ippsAES_GCMInit l9_ippsAES_GCMInit +#define ippsAES_GCMReinit l9_ippsAES_GCMReinit +#define ippsAES_GCMReset l9_ippsAES_GCMReset +#define ippsAES_GCMProcessIV l9_ippsAES_GCMProcessIV +#define ippsAES_GCMProcessAAD l9_ippsAES_GCMProcessAAD +#define ippsAES_GCMStart l9_ippsAES_GCMStart +#define ippsAES_GCMEncrypt l9_ippsAES_GCMEncrypt +#define ippsAES_GCMDecrypt l9_ippsAES_GCMDecrypt +#define ippsAES_GCMGetTag l9_ippsAES_GCMGetTag +#define ippsAES_XTSGetSize l9_ippsAES_XTSGetSize +#define ippsAES_XTSInit l9_ippsAES_XTSInit +#define ippsAES_XTSEncrypt l9_ippsAES_XTSEncrypt +#define ippsAES_XTSDecrypt l9_ippsAES_XTSDecrypt +#define ippsAES_S2V_CMAC l9_ippsAES_S2V_CMAC +#define ippsAES_SIVEncrypt l9_ippsAES_SIVEncrypt +#define ippsAES_SIVDecrypt l9_ippsAES_SIVDecrypt +#define ippsAES_CMACGetSize l9_ippsAES_CMACGetSize +#define ippsAES_CMACInit l9_ippsAES_CMACInit +#define ippsAES_CMACUpdate l9_ippsAES_CMACUpdate +#define ippsAES_CMACFinal l9_ippsAES_CMACFinal +#define ippsAES_CMACGetTag l9_ippsAES_CMACGetTag +#define ippsARCFourCheckKey l9_ippsARCFourCheckKey +#define ippsARCFourGetSize l9_ippsARCFourGetSize +#define ippsARCFourInit l9_ippsARCFourInit +#define ippsARCFourReset l9_ippsARCFourReset +#define ippsARCFourPack l9_ippsARCFourPack +#define ippsARCFourUnpack l9_ippsARCFourUnpack +#define ippsARCFourEncrypt l9_ippsARCFourEncrypt +#define ippsARCFourDecrypt l9_ippsARCFourDecrypt +#define ippsSHA1GetSize l9_ippsSHA1GetSize +#define ippsSHA1Init l9_ippsSHA1Init +#define ippsSHA1Duplicate l9_ippsSHA1Duplicate +#define ippsSHA1Pack l9_ippsSHA1Pack +#define ippsSHA1Unpack l9_ippsSHA1Unpack +#define ippsSHA1Update l9_ippsSHA1Update +#define ippsSHA1GetTag l9_ippsSHA1GetTag +#define ippsSHA1Final l9_ippsSHA1Final +#define ippsSHA1MessageDigest l9_ippsSHA1MessageDigest +#define ippsSHA224GetSize l9_ippsSHA224GetSize +#define ippsSHA224Init l9_ippsSHA224Init +#define ippsSHA224Duplicate l9_ippsSHA224Duplicate +#define ippsSHA224Pack l9_ippsSHA224Pack +#define ippsSHA224Unpack l9_ippsSHA224Unpack +#define ippsSHA224Update l9_ippsSHA224Update +#define ippsSHA224GetTag l9_ippsSHA224GetTag +#define ippsSHA224Final l9_ippsSHA224Final +#define ippsSHA224MessageDigest l9_ippsSHA224MessageDigest +#define ippsSHA256GetSize l9_ippsSHA256GetSize +#define ippsSHA256Init l9_ippsSHA256Init +#define ippsSHA256Duplicate l9_ippsSHA256Duplicate +#define ippsSHA256Pack l9_ippsSHA256Pack +#define ippsSHA256Unpack l9_ippsSHA256Unpack +#define ippsSHA256Update l9_ippsSHA256Update +#define ippsSHA256GetTag l9_ippsSHA256GetTag +#define ippsSHA256Final l9_ippsSHA256Final +#define ippsSHA256MessageDigest l9_ippsSHA256MessageDigest +#define ippsSHA384GetSize l9_ippsSHA384GetSize +#define ippsSHA384Init l9_ippsSHA384Init +#define ippsSHA384Duplicate l9_ippsSHA384Duplicate +#define ippsSHA384Pack l9_ippsSHA384Pack +#define ippsSHA384Unpack l9_ippsSHA384Unpack +#define ippsSHA384Update l9_ippsSHA384Update +#define ippsSHA384GetTag l9_ippsSHA384GetTag +#define ippsSHA384Final l9_ippsSHA384Final +#define ippsSHA384MessageDigest l9_ippsSHA384MessageDigest +#define ippsSHA512GetSize l9_ippsSHA512GetSize +#define ippsSHA512Init l9_ippsSHA512Init +#define ippsSHA512Duplicate l9_ippsSHA512Duplicate +#define ippsSHA512Pack l9_ippsSHA512Pack +#define ippsSHA512Unpack l9_ippsSHA512Unpack +#define ippsSHA512Update l9_ippsSHA512Update +#define ippsSHA512GetTag l9_ippsSHA512GetTag +#define ippsSHA512Final l9_ippsSHA512Final +#define ippsSHA512MessageDigest l9_ippsSHA512MessageDigest +#define ippsMD5GetSize l9_ippsMD5GetSize +#define ippsMD5Init l9_ippsMD5Init +#define ippsMD5Duplicate l9_ippsMD5Duplicate +#define ippsMD5Pack l9_ippsMD5Pack +#define ippsMD5Unpack l9_ippsMD5Unpack +#define ippsMD5Update l9_ippsMD5Update +#define ippsMD5GetTag l9_ippsMD5GetTag +#define ippsMD5Final l9_ippsMD5Final +#define ippsMD5MessageDigest l9_ippsMD5MessageDigest +#define ippsSM3GetSize l9_ippsSM3GetSize +#define ippsSM3Init l9_ippsSM3Init +#define ippsSM3Duplicate l9_ippsSM3Duplicate +#define ippsSM3Pack l9_ippsSM3Pack +#define ippsSM3Unpack l9_ippsSM3Unpack +#define ippsSM3Update l9_ippsSM3Update +#define ippsSM3GetTag l9_ippsSM3GetTag +#define ippsSM3Final l9_ippsSM3Final +#define ippsSM3MessageDigest l9_ippsSM3MessageDigest +#define ippsHashGetSize l9_ippsHashGetSize +#define ippsHashInit l9_ippsHashInit +#define ippsHashPack l9_ippsHashPack +#define ippsHashUnpack l9_ippsHashUnpack +#define ippsHashDuplicate l9_ippsHashDuplicate +#define ippsHashUpdate l9_ippsHashUpdate +#define ippsHashGetTag l9_ippsHashGetTag +#define ippsHashFinal l9_ippsHashFinal +#define ippsHashMessage l9_ippsHashMessage +#define ippsHashMethod_MD5 l9_ippsHashMethod_MD5 +#define ippsHashMethod_SM3 l9_ippsHashMethod_SM3 +#define ippsHashMethod_SHA1 l9_ippsHashMethod_SHA1 +#define ippsHashMethod_SHA1_NI l9_ippsHashMethod_SHA1_NI +#define ippsHashMethod_SHA1_TT l9_ippsHashMethod_SHA1_TT +#define ippsHashMethod_SHA256 l9_ippsHashMethod_SHA256 +#define ippsHashMethod_SHA256_NI l9_ippsHashMethod_SHA256_NI +#define ippsHashMethod_SHA256_TT l9_ippsHashMethod_SHA256_TT +#define ippsHashMethod_SHA224 l9_ippsHashMethod_SHA224 +#define ippsHashMethod_SHA224_NI l9_ippsHashMethod_SHA224_NI +#define ippsHashMethod_SHA224_TT l9_ippsHashMethod_SHA224_TT +#define ippsHashMethod_SHA512 l9_ippsHashMethod_SHA512 +#define ippsHashMethod_SHA384 l9_ippsHashMethod_SHA384 +#define ippsHashMethod_SHA512_256 l9_ippsHashMethod_SHA512_256 +#define ippsHashMethod_SHA512_224 l9_ippsHashMethod_SHA512_224 +#define ippsHashMethodGetSize l9_ippsHashMethodGetSize +#define ippsHashMethodSet_MD5 l9_ippsHashMethodSet_MD5 +#define ippsHashMethodSet_SM3 l9_ippsHashMethodSet_SM3 +#define ippsHashStateMethodSet_SM3 l9_ippsHashStateMethodSet_SM3 +#define ippsHashMethodSet_SHA1 l9_ippsHashMethodSet_SHA1 +#define ippsHashMethodSet_SHA1_NI l9_ippsHashMethodSet_SHA1_NI +#define ippsHashMethodSet_SHA1_TT l9_ippsHashMethodSet_SHA1_TT +#define ippsHashMethodSet_SHA256 l9_ippsHashMethodSet_SHA256 +#define ippsHashMethodSet_SHA256_NI l9_ippsHashMethodSet_SHA256_NI +#define ippsHashMethodSet_SHA256_TT l9_ippsHashMethodSet_SHA256_TT +#define ippsHashStateMethodSet_SHA256 l9_ippsHashStateMethodSet_SHA256 +#define ippsHashStateMethodSet_SHA256_NI l9_ippsHashStateMethodSet_SHA256_NI +#define ippsHashStateMethodSet_SHA256_TT l9_ippsHashStateMethodSet_SHA256_TT +#define ippsHashMethodSet_SHA224 l9_ippsHashMethodSet_SHA224 +#define ippsHashMethodSet_SHA224_NI l9_ippsHashMethodSet_SHA224_NI +#define ippsHashMethodSet_SHA224_TT l9_ippsHashMethodSet_SHA224_TT +#define ippsHashStateMethodSet_SHA224 l9_ippsHashStateMethodSet_SHA224 +#define ippsHashStateMethodSet_SHA224_NI l9_ippsHashStateMethodSet_SHA224_NI +#define ippsHashStateMethodSet_SHA224_TT l9_ippsHashStateMethodSet_SHA224_TT +#define ippsHashMethodSet_SHA512 l9_ippsHashMethodSet_SHA512 +#define ippsHashMethodSet_SHA384 l9_ippsHashMethodSet_SHA384 +#define ippsHashMethodSet_SHA512_256 l9_ippsHashMethodSet_SHA512_256 +#define ippsHashMethodSet_SHA512_224 l9_ippsHashMethodSet_SHA512_224 +#define ippsHashStateMethodSet_SHA512 l9_ippsHashStateMethodSet_SHA512 +#define ippsHashStateMethodSet_SHA384 l9_ippsHashStateMethodSet_SHA384 +#define ippsHashStateMethodSet_SHA512_256 l9_ippsHashStateMethodSet_SHA512_256 +#define ippsHashStateMethodSet_SHA512_224 l9_ippsHashStateMethodSet_SHA512_224 +#define ippsHashGetSize_rmf l9_ippsHashGetSize_rmf +#define ippsHashInit_rmf l9_ippsHashInit_rmf +#define ippsHashPack_rmf l9_ippsHashPack_rmf +#define ippsHashUnpack_rmf l9_ippsHashUnpack_rmf +#define ippsHashDuplicate_rmf l9_ippsHashDuplicate_rmf +#define ippsHashUpdate_rmf l9_ippsHashUpdate_rmf +#define ippsHashGetTag_rmf l9_ippsHashGetTag_rmf +#define ippsHashFinal_rmf l9_ippsHashFinal_rmf +#define ippsHashMessage_rmf l9_ippsHashMessage_rmf +#define ippsHashMethodGetInfo l9_ippsHashMethodGetInfo +#define ippsHashGetInfo_rmf l9_ippsHashGetInfo_rmf +#define ippsMGF l9_ippsMGF +#define ippsMGF1_rmf l9_ippsMGF1_rmf +#define ippsMGF2_rmf l9_ippsMGF2_rmf +#define ippsHMAC_GetSize l9_ippsHMAC_GetSize +#define ippsHMAC_Init l9_ippsHMAC_Init +#define ippsHMAC_Pack l9_ippsHMAC_Pack +#define ippsHMAC_Unpack l9_ippsHMAC_Unpack +#define ippsHMAC_Duplicate l9_ippsHMAC_Duplicate +#define ippsHMAC_Update l9_ippsHMAC_Update +#define ippsHMAC_Final l9_ippsHMAC_Final +#define ippsHMAC_GetTag l9_ippsHMAC_GetTag +#define ippsHMAC_Message l9_ippsHMAC_Message +#define ippsHMACGetSize_rmf l9_ippsHMACGetSize_rmf +#define ippsHMACInit_rmf l9_ippsHMACInit_rmf +#define ippsHMACPack_rmf l9_ippsHMACPack_rmf +#define ippsHMACUnpack_rmf l9_ippsHMACUnpack_rmf +#define ippsHMACDuplicate_rmf l9_ippsHMACDuplicate_rmf +#define ippsHMACUpdate_rmf l9_ippsHMACUpdate_rmf +#define ippsHMACFinal_rmf l9_ippsHMACFinal_rmf +#define ippsHMACGetTag_rmf l9_ippsHMACGetTag_rmf +#define ippsHMACMessage_rmf l9_ippsHMACMessage_rmf +#define ippsBigNumGetSize l9_ippsBigNumGetSize +#define ippsBigNumInit l9_ippsBigNumInit +#define ippsCmpZero_BN l9_ippsCmpZero_BN +#define ippsCmp_BN l9_ippsCmp_BN +#define ippsGetSize_BN l9_ippsGetSize_BN +#define ippsSet_BN l9_ippsSet_BN +#define ippsGet_BN l9_ippsGet_BN +#define ippsRef_BN l9_ippsRef_BN +#define ippsExtGet_BN l9_ippsExtGet_BN +#define ippsAdd_BN l9_ippsAdd_BN +#define ippsSub_BN l9_ippsSub_BN +#define ippsMul_BN l9_ippsMul_BN +#define ippsMAC_BN_I l9_ippsMAC_BN_I +#define ippsDiv_BN l9_ippsDiv_BN +#define ippsMod_BN l9_ippsMod_BN +#define ippsGcd_BN l9_ippsGcd_BN +#define ippsModInv_BN l9_ippsModInv_BN +#define ippsSetOctString_BN l9_ippsSetOctString_BN +#define ippsGetOctString_BN l9_ippsGetOctString_BN +#define ippsMontGetSize l9_ippsMontGetSize +#define ippsMontInit l9_ippsMontInit +#define ippsMontSet l9_ippsMontSet +#define ippsMontGet l9_ippsMontGet +#define ippsMontForm l9_ippsMontForm +#define ippsMontMul l9_ippsMontMul +#define ippsMontExp l9_ippsMontExp +#define ippsPRNGGetSize l9_ippsPRNGGetSize +#define ippsPRNGInit l9_ippsPRNGInit +#define ippsPRNGSetModulus l9_ippsPRNGSetModulus +#define ippsPRNGSetH0 l9_ippsPRNGSetH0 +#define ippsPRNGSetAugment l9_ippsPRNGSetAugment +#define ippsPRNGSetSeed l9_ippsPRNGSetSeed +#define ippsPRNGGetSeed l9_ippsPRNGGetSeed +#define ippsPRNGen l9_ippsPRNGen +#define ippsPRNGen_BN l9_ippsPRNGen_BN +#define ippsPRNGenRDRAND l9_ippsPRNGenRDRAND +#define ippsPRNGenRDRAND_BN l9_ippsPRNGenRDRAND_BN +#define ippsTRNGenRDSEED l9_ippsTRNGenRDSEED +#define ippsTRNGenRDSEED_BN l9_ippsTRNGenRDSEED_BN +#define ippsPrimeGetSize l9_ippsPrimeGetSize +#define ippsPrimeInit l9_ippsPrimeInit +#define ippsPrimeGen l9_ippsPrimeGen +#define ippsPrimeTest l9_ippsPrimeTest +#define ippsPrimeGen_BN l9_ippsPrimeGen_BN +#define ippsPrimeTest_BN l9_ippsPrimeTest_BN +#define ippsPrimeGet l9_ippsPrimeGet +#define ippsPrimeGet_BN l9_ippsPrimeGet_BN +#define ippsPrimeSet l9_ippsPrimeSet +#define ippsPrimeSet_BN l9_ippsPrimeSet_BN +#define ippsRSA_GetSizePublicKey l9_ippsRSA_GetSizePublicKey +#define ippsRSA_InitPublicKey l9_ippsRSA_InitPublicKey +#define ippsRSA_SetPublicKey l9_ippsRSA_SetPublicKey +#define ippsRSA_GetPublicKey l9_ippsRSA_GetPublicKey +#define ippsRSA_GetSizePrivateKeyType1 l9_ippsRSA_GetSizePrivateKeyType1 +#define ippsRSA_InitPrivateKeyType1 l9_ippsRSA_InitPrivateKeyType1 +#define ippsRSA_SetPrivateKeyType1 l9_ippsRSA_SetPrivateKeyType1 +#define ippsRSA_GetPrivateKeyType1 l9_ippsRSA_GetPrivateKeyType1 +#define ippsRSA_GetSizePrivateKeyType2 l9_ippsRSA_GetSizePrivateKeyType2 +#define ippsRSA_InitPrivateKeyType2 l9_ippsRSA_InitPrivateKeyType2 +#define ippsRSA_SetPrivateKeyType2 l9_ippsRSA_SetPrivateKeyType2 +#define ippsRSA_GetPrivateKeyType2 l9_ippsRSA_GetPrivateKeyType2 +#define ippsRSA_GetBufferSizePublicKey l9_ippsRSA_GetBufferSizePublicKey +#define ippsRSA_GetBufferSizePrivateKey l9_ippsRSA_GetBufferSizePrivateKey +#define ippsRSA_Encrypt l9_ippsRSA_Encrypt +#define ippsRSA_Decrypt l9_ippsRSA_Decrypt +#define ippsRSA_GenerateKeys l9_ippsRSA_GenerateKeys +#define ippsRSA_ValidateKeys l9_ippsRSA_ValidateKeys +#define ippsRSAEncrypt_OAEP l9_ippsRSAEncrypt_OAEP +#define ippsRSADecrypt_OAEP l9_ippsRSADecrypt_OAEP +#define ippsRSAEncrypt_OAEP_rmf l9_ippsRSAEncrypt_OAEP_rmf +#define ippsRSADecrypt_OAEP_rmf l9_ippsRSADecrypt_OAEP_rmf +#define ippsRSAEncrypt_PKCSv15 l9_ippsRSAEncrypt_PKCSv15 +#define ippsRSADecrypt_PKCSv15 l9_ippsRSADecrypt_PKCSv15 +#define ippsRSASign_PSS l9_ippsRSASign_PSS +#define ippsRSAVerify_PSS l9_ippsRSAVerify_PSS +#define ippsRSASign_PSS_rmf l9_ippsRSASign_PSS_rmf +#define ippsRSAVerify_PSS_rmf l9_ippsRSAVerify_PSS_rmf +#define ippsRSASign_PKCS1v15 l9_ippsRSASign_PKCS1v15 +#define ippsRSAVerify_PKCS1v15 l9_ippsRSAVerify_PKCS1v15 +#define ippsRSASign_PKCS1v15_rmf l9_ippsRSASign_PKCS1v15_rmf +#define ippsRSAVerify_PKCS1v15_rmf l9_ippsRSAVerify_PKCS1v15_rmf +#define ippsDLGetResultString l9_ippsDLGetResultString +#define ippsDLPGetSize l9_ippsDLPGetSize +#define ippsDLPInit l9_ippsDLPInit +#define ippsDLPPack l9_ippsDLPPack +#define ippsDLPUnpack l9_ippsDLPUnpack +#define ippsDLPSet l9_ippsDLPSet +#define ippsDLPGet l9_ippsDLPGet +#define ippsDLPSetDP l9_ippsDLPSetDP +#define ippsDLPGetDP l9_ippsDLPGetDP +#define ippsDLPGenKeyPair l9_ippsDLPGenKeyPair +#define ippsDLPPublicKey l9_ippsDLPPublicKey +#define ippsDLPValidateKeyPair l9_ippsDLPValidateKeyPair +#define ippsDLPSetKeyPair l9_ippsDLPSetKeyPair +#define ippsDLPSignDSA l9_ippsDLPSignDSA +#define ippsDLPVerifyDSA l9_ippsDLPVerifyDSA +#define ippsDLPSharedSecretDH l9_ippsDLPSharedSecretDH +#define ippsDLPGenerateDSA l9_ippsDLPGenerateDSA +#define ippsDLPValidateDSA l9_ippsDLPValidateDSA +#define ippsDLPGenerateDH l9_ippsDLPGenerateDH +#define ippsDLPValidateDH l9_ippsDLPValidateDH +#define ippsECCGetResultString l9_ippsECCGetResultString +#define ippsECCPGetSize l9_ippsECCPGetSize +#define ippsECCPGetSizeStd128r1 l9_ippsECCPGetSizeStd128r1 +#define ippsECCPGetSizeStd128r2 l9_ippsECCPGetSizeStd128r2 +#define ippsECCPGetSizeStd192r1 l9_ippsECCPGetSizeStd192r1 +#define ippsECCPGetSizeStd224r1 l9_ippsECCPGetSizeStd224r1 +#define ippsECCPGetSizeStd256r1 l9_ippsECCPGetSizeStd256r1 +#define ippsECCPGetSizeStd384r1 l9_ippsECCPGetSizeStd384r1 +#define ippsECCPGetSizeStd521r1 l9_ippsECCPGetSizeStd521r1 +#define ippsECCPGetSizeStdSM2 l9_ippsECCPGetSizeStdSM2 +#define ippsECCPInit l9_ippsECCPInit +#define ippsECCPInitStd128r1 l9_ippsECCPInitStd128r1 +#define ippsECCPInitStd128r2 l9_ippsECCPInitStd128r2 +#define ippsECCPInitStd192r1 l9_ippsECCPInitStd192r1 +#define ippsECCPInitStd224r1 l9_ippsECCPInitStd224r1 +#define ippsECCPInitStd256r1 l9_ippsECCPInitStd256r1 +#define ippsECCPInitStd384r1 l9_ippsECCPInitStd384r1 +#define ippsECCPInitStd521r1 l9_ippsECCPInitStd521r1 +#define ippsECCPInitStdSM2 l9_ippsECCPInitStdSM2 +#define ippsECCPSet l9_ippsECCPSet +#define ippsECCPSetStd l9_ippsECCPSetStd +#define ippsECCPSetStd128r1 l9_ippsECCPSetStd128r1 +#define ippsECCPSetStd128r2 l9_ippsECCPSetStd128r2 +#define ippsECCPSetStd192r1 l9_ippsECCPSetStd192r1 +#define ippsECCPSetStd224r1 l9_ippsECCPSetStd224r1 +#define ippsECCPSetStd256r1 l9_ippsECCPSetStd256r1 +#define ippsECCPSetStd384r1 l9_ippsECCPSetStd384r1 +#define ippsECCPSetStd521r1 l9_ippsECCPSetStd521r1 +#define ippsECCPSetStdSM2 l9_ippsECCPSetStdSM2 +#define ippsECCPBindGxyTblStd192r1 l9_ippsECCPBindGxyTblStd192r1 +#define ippsECCPBindGxyTblStd224r1 l9_ippsECCPBindGxyTblStd224r1 +#define ippsECCPBindGxyTblStd256r1 l9_ippsECCPBindGxyTblStd256r1 +#define ippsECCPBindGxyTblStd384r1 l9_ippsECCPBindGxyTblStd384r1 +#define ippsECCPBindGxyTblStd521r1 l9_ippsECCPBindGxyTblStd521r1 +#define ippsECCPBindGxyTblStdSM2 l9_ippsECCPBindGxyTblStdSM2 +#define ippsECCPGet l9_ippsECCPGet +#define ippsECCPGetOrderBitSize l9_ippsECCPGetOrderBitSize +#define ippsECCPValidate l9_ippsECCPValidate +#define ippsECCPPointGetSize l9_ippsECCPPointGetSize +#define ippsECCPPointInit l9_ippsECCPPointInit +#define ippsECCPSetPoint l9_ippsECCPSetPoint +#define ippsECCPSetPointAtInfinity l9_ippsECCPSetPointAtInfinity +#define ippsECCPGetPoint l9_ippsECCPGetPoint +#define ippsECCPCheckPoint l9_ippsECCPCheckPoint +#define ippsECCPComparePoint l9_ippsECCPComparePoint +#define ippsECCPNegativePoint l9_ippsECCPNegativePoint +#define ippsECCPAddPoint l9_ippsECCPAddPoint +#define ippsECCPMulPointScalar l9_ippsECCPMulPointScalar +#define ippsECCPGenKeyPair l9_ippsECCPGenKeyPair +#define ippsECCPPublicKey l9_ippsECCPPublicKey +#define ippsECCPValidateKeyPair l9_ippsECCPValidateKeyPair +#define ippsECCPSetKeyPair l9_ippsECCPSetKeyPair +#define ippsECCPSharedSecretDH l9_ippsECCPSharedSecretDH +#define ippsECCPSharedSecretDHC l9_ippsECCPSharedSecretDHC +#define ippsECCPSignDSA l9_ippsECCPSignDSA +#define ippsECCPVerifyDSA l9_ippsECCPVerifyDSA +#define ippsECCPSignNR l9_ippsECCPSignNR +#define ippsECCPVerifyNR l9_ippsECCPVerifyNR +#define ippsECCPSignSM2 l9_ippsECCPSignSM2 +#define ippsECCPVerifySM2 l9_ippsECCPVerifySM2 +#define ippsGFpGetSize l9_ippsGFpGetSize +#define ippsGFpInitArbitrary l9_ippsGFpInitArbitrary +#define ippsGFpInitFixed l9_ippsGFpInitFixed +#define ippsGFpInit l9_ippsGFpInit +#define ippsGFpMethod_p192r1 l9_ippsGFpMethod_p192r1 +#define ippsGFpMethod_p224r1 l9_ippsGFpMethod_p224r1 +#define ippsGFpMethod_p256r1 l9_ippsGFpMethod_p256r1 +#define ippsGFpMethod_p384r1 l9_ippsGFpMethod_p384r1 +#define ippsGFpMethod_p521r1 l9_ippsGFpMethod_p521r1 +#define ippsGFpMethod_p256sm2 l9_ippsGFpMethod_p256sm2 +#define ippsGFpMethod_p256bn l9_ippsGFpMethod_p256bn +#define ippsGFpMethod_p256 l9_ippsGFpMethod_p256 +#define ippsGFpMethod_pArb l9_ippsGFpMethod_pArb +#define ippsGFpxGetSize l9_ippsGFpxGetSize +#define ippsGFpxInit l9_ippsGFpxInit +#define ippsGFpxInitBinomial l9_ippsGFpxInitBinomial +#define ippsGFpxMethod_binom2_epid2 l9_ippsGFpxMethod_binom2_epid2 +#define ippsGFpxMethod_binom3_epid2 l9_ippsGFpxMethod_binom3_epid2 +#define ippsGFpxMethod_binom2 l9_ippsGFpxMethod_binom2 +#define ippsGFpxMethod_binom3 l9_ippsGFpxMethod_binom3 +#define ippsGFpxMethod_binom l9_ippsGFpxMethod_binom +#define ippsGFpxMethod_com l9_ippsGFpxMethod_com +#define ippsGFpScratchBufferSize l9_ippsGFpScratchBufferSize +#define ippsGFpElementGetSize l9_ippsGFpElementGetSize +#define ippsGFpElementInit l9_ippsGFpElementInit +#define ippsGFpSetElement l9_ippsGFpSetElement +#define ippsGFpSetElementRegular l9_ippsGFpSetElementRegular +#define ippsGFpSetElementOctString l9_ippsGFpSetElementOctString +#define ippsGFpSetElementRandom l9_ippsGFpSetElementRandom +#define ippsGFpSetElementHash l9_ippsGFpSetElementHash +#define ippsGFpSetElementHash_rmf l9_ippsGFpSetElementHash_rmf +#define ippsGFpCpyElement l9_ippsGFpCpyElement +#define ippsGFpGetElement l9_ippsGFpGetElement +#define ippsGFpGetElementOctString l9_ippsGFpGetElementOctString +#define ippsGFpCmpElement l9_ippsGFpCmpElement +#define ippsGFpIsZeroElement l9_ippsGFpIsZeroElement +#define ippsGFpIsUnityElement l9_ippsGFpIsUnityElement +#define ippsGFpConj l9_ippsGFpConj +#define ippsGFpNeg l9_ippsGFpNeg +#define ippsGFpInv l9_ippsGFpInv +#define ippsGFpSqrt l9_ippsGFpSqrt +#define ippsGFpSqr l9_ippsGFpSqr +#define ippsGFpAdd l9_ippsGFpAdd +#define ippsGFpSub l9_ippsGFpSub +#define ippsGFpMul l9_ippsGFpMul +#define ippsGFpExp l9_ippsGFpExp +#define ippsGFpMultiExp l9_ippsGFpMultiExp +#define ippsGFpAdd_PE l9_ippsGFpAdd_PE +#define ippsGFpSub_PE l9_ippsGFpSub_PE +#define ippsGFpMul_PE l9_ippsGFpMul_PE +#define ippsGFpGetInfo l9_ippsGFpGetInfo +#define ippsGFpECGetSize l9_ippsGFpECGetSize +#define ippsGFpECInit l9_ippsGFpECInit +#define ippsGFpECSet l9_ippsGFpECSet +#define ippsGFpECSetSubgroup l9_ippsGFpECSetSubgroup +#define ippsGFpECInitStd128r1 l9_ippsGFpECInitStd128r1 +#define ippsGFpECInitStd128r2 l9_ippsGFpECInitStd128r2 +#define ippsGFpECInitStd192r1 l9_ippsGFpECInitStd192r1 +#define ippsGFpECInitStd224r1 l9_ippsGFpECInitStd224r1 +#define ippsGFpECInitStd256r1 l9_ippsGFpECInitStd256r1 +#define ippsGFpECInitStd384r1 l9_ippsGFpECInitStd384r1 +#define ippsGFpECInitStd521r1 l9_ippsGFpECInitStd521r1 +#define ippsGFpECInitStdSM2 l9_ippsGFpECInitStdSM2 +#define ippsGFpECInitStdBN256 l9_ippsGFpECInitStdBN256 +#define ippsGFpECBindGxyTblStd192r1 l9_ippsGFpECBindGxyTblStd192r1 +#define ippsGFpECBindGxyTblStd224r1 l9_ippsGFpECBindGxyTblStd224r1 +#define ippsGFpECBindGxyTblStd256r1 l9_ippsGFpECBindGxyTblStd256r1 +#define ippsGFpECBindGxyTblStd384r1 l9_ippsGFpECBindGxyTblStd384r1 +#define ippsGFpECBindGxyTblStd521r1 l9_ippsGFpECBindGxyTblStd521r1 +#define ippsGFpECBindGxyTblStdSM2 l9_ippsGFpECBindGxyTblStdSM2 +#define ippsGFpECGet l9_ippsGFpECGet +#define ippsGFpECGetSubgroup l9_ippsGFpECGetSubgroup +#define ippsGFpECScratchBufferSize l9_ippsGFpECScratchBufferSize +#define ippsGFpECVerify l9_ippsGFpECVerify +#define ippsGFpECPointGetSize l9_ippsGFpECPointGetSize +#define ippsGFpECPointInit l9_ippsGFpECPointInit +#define ippsGFpECSetPointAtInfinity l9_ippsGFpECSetPointAtInfinity +#define ippsGFpECSetPoint l9_ippsGFpECSetPoint +#define ippsGFpECSetPointRegular l9_ippsGFpECSetPointRegular +#define ippsGFpECSetPointRandom l9_ippsGFpECSetPointRandom +#define ippsGFpECMakePoint l9_ippsGFpECMakePoint +#define ippsGFpECSetPointHash l9_ippsGFpECSetPointHash +#define ippsGFpECSetPointHashBackCompatible l9_ippsGFpECSetPointHashBackCompatible +#define ippsGFpECSetPointHash_rmf l9_ippsGFpECSetPointHash_rmf +#define ippsGFpECSetPointHashBackCompatible_rmf l9_ippsGFpECSetPointHashBackCompatible_rmf +#define ippsGFpECGetPoint l9_ippsGFpECGetPoint +#define ippsGFpECGetPointRegular l9_ippsGFpECGetPointRegular +#define ippsGFpECSetPointOctString l9_ippsGFpECSetPointOctString +#define ippsGFpECGetPointOctString l9_ippsGFpECGetPointOctString +#define ippsGFpECTstPoint l9_ippsGFpECTstPoint +#define ippsGFpECTstPointInSubgroup l9_ippsGFpECTstPointInSubgroup +#define ippsGFpECCpyPoint l9_ippsGFpECCpyPoint +#define ippsGFpECCmpPoint l9_ippsGFpECCmpPoint +#define ippsGFpECNegPoint l9_ippsGFpECNegPoint +#define ippsGFpECAddPoint l9_ippsGFpECAddPoint +#define ippsGFpECMulPoint l9_ippsGFpECMulPoint +#define ippsGFpECPrivateKey l9_ippsGFpECPrivateKey +#define ippsGFpECPublicKey l9_ippsGFpECPublicKey +#define ippsGFpECTstKeyPair l9_ippsGFpECTstKeyPair +#define ippsGFpECSharedSecretDH l9_ippsGFpECSharedSecretDH +#define ippsGFpECSharedSecretDHC l9_ippsGFpECSharedSecretDHC +#define ippsGFpECMessageRepresentationSM2 l9_ippsGFpECMessageRepresentationSM2 +#define ippsGFpECSignDSA l9_ippsGFpECSignDSA +#define ippsGFpECVerifyDSA l9_ippsGFpECVerifyDSA +#define ippsGFpECSignNR l9_ippsGFpECSignNR +#define ippsGFpECVerifyNR l9_ippsGFpECVerifyNR +#define ippsGFpECSignSM2 l9_ippsGFpECSignSM2 +#define ippsGFpECVerifySM2 l9_ippsGFpECVerifySM2 +#define ippsGFpECUserIDHashSM2 l9_ippsGFpECUserIDHashSM2 +#define ippsGFpECKeyExchangeSM2_GetSize l9_ippsGFpECKeyExchangeSM2_GetSize +#define ippsGFpECKeyExchangeSM2_Init l9_ippsGFpECKeyExchangeSM2_Init +#define ippsGFpECKeyExchangeSM2_Setup l9_ippsGFpECKeyExchangeSM2_Setup +#define ippsGFpECKeyExchangeSM2_SharedKey l9_ippsGFpECKeyExchangeSM2_SharedKey +#define ippsGFpECKeyExchangeSM2_Confirm l9_ippsGFpECKeyExchangeSM2_Confirm +#define ippsGFpECGetInfo_GF l9_ippsGFpECGetInfo_GF +#define ippsGFpECESGetSize_SM2 l9_ippsGFpECESGetSize_SM2 +#define ippsGFpECESInit_SM2 l9_ippsGFpECESInit_SM2 +#define ippsGFpECESSetKey_SM2 l9_ippsGFpECESSetKey_SM2 +#define ippsGFpECESStart_SM2 l9_ippsGFpECESStart_SM2 +#define ippsGFpECESEncrypt_SM2 l9_ippsGFpECESEncrypt_SM2 +#define ippsGFpECESDecrypt_SM2 l9_ippsGFpECESDecrypt_SM2 +#define ippsGFpECESFinal_SM2 l9_ippsGFpECESFinal_SM2 +#define ippsGFpECESGetBuffersSize_SM2 l9_ippsGFpECESGetBuffersSize_SM2 +#define ippsGFpECEncryptSM2_Ext_EncMsgSize l9_ippsGFpECEncryptSM2_Ext_EncMsgSize +#define ippsGFpECEncryptSM2_Ext l9_ippsGFpECEncryptSM2_Ext +#define ippsGFpECDecryptSM2_Ext_DecMsgSize l9_ippsGFpECDecryptSM2_Ext_DecMsgSize +#define ippsGFpECDecryptSM2_Ext l9_ippsGFpECDecryptSM2_Ext diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_m7.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_m7.h new file mode 100644 index 0000000..f9365f8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_m7.h @@ -0,0 +1,557 @@ +/******************************************************************************* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + + #define ippcpGetLibVersion m7_ippcpGetLibVersion +#define ippsDESGetSize m7_ippsDESGetSize +#define ippsDESInit m7_ippsDESInit +#define ippsDESPack m7_ippsDESPack +#define ippsDESUnpack m7_ippsDESUnpack +#define ippsTDESEncryptECB m7_ippsTDESEncryptECB +#define ippsTDESDecryptECB m7_ippsTDESDecryptECB +#define ippsTDESEncryptCBC m7_ippsTDESEncryptCBC +#define ippsTDESDecryptCBC m7_ippsTDESDecryptCBC +#define ippsTDESEncryptCFB m7_ippsTDESEncryptCFB +#define ippsTDESDecryptCFB m7_ippsTDESDecryptCFB +#define ippsTDESEncryptOFB m7_ippsTDESEncryptOFB +#define ippsTDESDecryptOFB m7_ippsTDESDecryptOFB +#define ippsTDESEncryptCTR m7_ippsTDESEncryptCTR +#define ippsTDESDecryptCTR m7_ippsTDESDecryptCTR +#define ippsAESGetSize m7_ippsAESGetSize +#define ippsAESInit m7_ippsAESInit +#define ippsAESSetKey m7_ippsAESSetKey +#define ippsAESPack m7_ippsAESPack +#define ippsAESUnpack m7_ippsAESUnpack +#define ippsAESEncryptECB m7_ippsAESEncryptECB +#define ippsAESDecryptECB m7_ippsAESDecryptECB +#define ippsAESEncryptCBC m7_ippsAESEncryptCBC +#define ippsAESEncryptCBC_CS1 m7_ippsAESEncryptCBC_CS1 +#define ippsAESEncryptCBC_CS2 m7_ippsAESEncryptCBC_CS2 +#define ippsAESEncryptCBC_CS3 m7_ippsAESEncryptCBC_CS3 +#define ippsAESDecryptCBC m7_ippsAESDecryptCBC +#define ippsAESDecryptCBC_CS1 m7_ippsAESDecryptCBC_CS1 +#define ippsAESDecryptCBC_CS2 m7_ippsAESDecryptCBC_CS2 +#define ippsAESDecryptCBC_CS3 m7_ippsAESDecryptCBC_CS3 +#define ippsAESEncryptCFB m7_ippsAESEncryptCFB +#define ippsAESDecryptCFB m7_ippsAESDecryptCFB +#define ippsAESEncryptOFB m7_ippsAESEncryptOFB +#define ippsAESDecryptOFB m7_ippsAESDecryptOFB +#define ippsAESEncryptCTR m7_ippsAESEncryptCTR +#define ippsAESDecryptCTR m7_ippsAESDecryptCTR +#define ippsAESEncryptXTS_Direct m7_ippsAESEncryptXTS_Direct +#define ippsAESDecryptXTS_Direct m7_ippsAESDecryptXTS_Direct +#define ippsAESSetupNoise m7_ippsAESSetupNoise +#define ippsAES_GCMSetupNoise m7_ippsAES_GCMSetupNoise +#define ippsAES_CMACSetupNoise m7_ippsAES_CMACSetupNoise +#define ippsAES_EncryptCFB16_MB m7_ippsAES_EncryptCFB16_MB +#define ippsSMS4GetSize m7_ippsSMS4GetSize +#define ippsSMS4Init m7_ippsSMS4Init +#define ippsSMS4SetKey m7_ippsSMS4SetKey +#define ippsSMS4EncryptECB m7_ippsSMS4EncryptECB +#define ippsSMS4DecryptECB m7_ippsSMS4DecryptECB +#define ippsSMS4EncryptCBC m7_ippsSMS4EncryptCBC +#define ippsSMS4EncryptCBC_CS1 m7_ippsSMS4EncryptCBC_CS1 +#define ippsSMS4EncryptCBC_CS2 m7_ippsSMS4EncryptCBC_CS2 +#define ippsSMS4EncryptCBC_CS3 m7_ippsSMS4EncryptCBC_CS3 +#define ippsSMS4DecryptCBC m7_ippsSMS4DecryptCBC +#define ippsSMS4DecryptCBC_CS1 m7_ippsSMS4DecryptCBC_CS1 +#define ippsSMS4DecryptCBC_CS2 m7_ippsSMS4DecryptCBC_CS2 +#define ippsSMS4DecryptCBC_CS3 m7_ippsSMS4DecryptCBC_CS3 +#define ippsSMS4EncryptCFB m7_ippsSMS4EncryptCFB +#define ippsSMS4DecryptCFB m7_ippsSMS4DecryptCFB +#define ippsSMS4EncryptOFB m7_ippsSMS4EncryptOFB +#define ippsSMS4DecryptOFB m7_ippsSMS4DecryptOFB +#define ippsSMS4EncryptCTR m7_ippsSMS4EncryptCTR +#define ippsSMS4DecryptCTR m7_ippsSMS4DecryptCTR +#define ippsSMS4_CCMGetSize m7_ippsSMS4_CCMGetSize +#define ippsSMS4_CCMInit m7_ippsSMS4_CCMInit +#define ippsSMS4_CCMMessageLen m7_ippsSMS4_CCMMessageLen +#define ippsSMS4_CCMTagLen m7_ippsSMS4_CCMTagLen +#define ippsSMS4_CCMStart m7_ippsSMS4_CCMStart +#define ippsSMS4_CCMEncrypt m7_ippsSMS4_CCMEncrypt +#define ippsSMS4_CCMDecrypt m7_ippsSMS4_CCMDecrypt +#define ippsSMS4_CCMGetTag m7_ippsSMS4_CCMGetTag +#define ippsAES_CCMGetSize m7_ippsAES_CCMGetSize +#define ippsAES_CCMInit m7_ippsAES_CCMInit +#define ippsAES_CCMMessageLen m7_ippsAES_CCMMessageLen +#define ippsAES_CCMTagLen m7_ippsAES_CCMTagLen +#define ippsAES_CCMStart m7_ippsAES_CCMStart +#define ippsAES_CCMEncrypt m7_ippsAES_CCMEncrypt +#define ippsAES_CCMDecrypt m7_ippsAES_CCMDecrypt +#define ippsAES_CCMGetTag m7_ippsAES_CCMGetTag +#define ippsAES_GCMGetSize m7_ippsAES_GCMGetSize +#define ippsAES_GCMInit m7_ippsAES_GCMInit +#define ippsAES_GCMReinit m7_ippsAES_GCMReinit +#define ippsAES_GCMReset m7_ippsAES_GCMReset +#define ippsAES_GCMProcessIV m7_ippsAES_GCMProcessIV +#define ippsAES_GCMProcessAAD m7_ippsAES_GCMProcessAAD +#define ippsAES_GCMStart m7_ippsAES_GCMStart +#define ippsAES_GCMEncrypt m7_ippsAES_GCMEncrypt +#define ippsAES_GCMDecrypt m7_ippsAES_GCMDecrypt +#define ippsAES_GCMGetTag m7_ippsAES_GCMGetTag +#define ippsAES_XTSGetSize m7_ippsAES_XTSGetSize +#define ippsAES_XTSInit m7_ippsAES_XTSInit +#define ippsAES_XTSEncrypt m7_ippsAES_XTSEncrypt +#define ippsAES_XTSDecrypt m7_ippsAES_XTSDecrypt +#define ippsAES_S2V_CMAC m7_ippsAES_S2V_CMAC +#define ippsAES_SIVEncrypt m7_ippsAES_SIVEncrypt +#define ippsAES_SIVDecrypt m7_ippsAES_SIVDecrypt +#define ippsAES_CMACGetSize m7_ippsAES_CMACGetSize +#define ippsAES_CMACInit m7_ippsAES_CMACInit +#define ippsAES_CMACUpdate m7_ippsAES_CMACUpdate +#define ippsAES_CMACFinal m7_ippsAES_CMACFinal +#define ippsAES_CMACGetTag m7_ippsAES_CMACGetTag +#define ippsARCFourCheckKey m7_ippsARCFourCheckKey +#define ippsARCFourGetSize m7_ippsARCFourGetSize +#define ippsARCFourInit m7_ippsARCFourInit +#define ippsARCFourReset m7_ippsARCFourReset +#define ippsARCFourPack m7_ippsARCFourPack +#define ippsARCFourUnpack m7_ippsARCFourUnpack +#define ippsARCFourEncrypt m7_ippsARCFourEncrypt +#define ippsARCFourDecrypt m7_ippsARCFourDecrypt +#define ippsSHA1GetSize m7_ippsSHA1GetSize +#define ippsSHA1Init m7_ippsSHA1Init +#define ippsSHA1Duplicate m7_ippsSHA1Duplicate +#define ippsSHA1Pack m7_ippsSHA1Pack +#define ippsSHA1Unpack m7_ippsSHA1Unpack +#define ippsSHA1Update m7_ippsSHA1Update +#define ippsSHA1GetTag m7_ippsSHA1GetTag +#define ippsSHA1Final m7_ippsSHA1Final +#define ippsSHA1MessageDigest m7_ippsSHA1MessageDigest +#define ippsSHA224GetSize m7_ippsSHA224GetSize +#define ippsSHA224Init m7_ippsSHA224Init +#define ippsSHA224Duplicate m7_ippsSHA224Duplicate +#define ippsSHA224Pack m7_ippsSHA224Pack +#define ippsSHA224Unpack m7_ippsSHA224Unpack +#define ippsSHA224Update m7_ippsSHA224Update +#define ippsSHA224GetTag m7_ippsSHA224GetTag +#define ippsSHA224Final m7_ippsSHA224Final +#define ippsSHA224MessageDigest m7_ippsSHA224MessageDigest +#define ippsSHA256GetSize m7_ippsSHA256GetSize +#define ippsSHA256Init m7_ippsSHA256Init +#define ippsSHA256Duplicate m7_ippsSHA256Duplicate +#define ippsSHA256Pack m7_ippsSHA256Pack +#define ippsSHA256Unpack m7_ippsSHA256Unpack +#define ippsSHA256Update m7_ippsSHA256Update +#define ippsSHA256GetTag m7_ippsSHA256GetTag +#define ippsSHA256Final m7_ippsSHA256Final +#define ippsSHA256MessageDigest m7_ippsSHA256MessageDigest +#define ippsSHA384GetSize m7_ippsSHA384GetSize +#define ippsSHA384Init m7_ippsSHA384Init +#define ippsSHA384Duplicate m7_ippsSHA384Duplicate +#define ippsSHA384Pack m7_ippsSHA384Pack +#define ippsSHA384Unpack m7_ippsSHA384Unpack +#define ippsSHA384Update m7_ippsSHA384Update +#define ippsSHA384GetTag m7_ippsSHA384GetTag +#define ippsSHA384Final m7_ippsSHA384Final +#define ippsSHA384MessageDigest m7_ippsSHA384MessageDigest +#define ippsSHA512GetSize m7_ippsSHA512GetSize +#define ippsSHA512Init m7_ippsSHA512Init +#define ippsSHA512Duplicate m7_ippsSHA512Duplicate +#define ippsSHA512Pack m7_ippsSHA512Pack +#define ippsSHA512Unpack m7_ippsSHA512Unpack +#define ippsSHA512Update m7_ippsSHA512Update +#define ippsSHA512GetTag m7_ippsSHA512GetTag +#define ippsSHA512Final m7_ippsSHA512Final +#define ippsSHA512MessageDigest m7_ippsSHA512MessageDigest +#define ippsMD5GetSize m7_ippsMD5GetSize +#define ippsMD5Init m7_ippsMD5Init +#define ippsMD5Duplicate m7_ippsMD5Duplicate +#define ippsMD5Pack m7_ippsMD5Pack +#define ippsMD5Unpack m7_ippsMD5Unpack +#define ippsMD5Update m7_ippsMD5Update +#define ippsMD5GetTag m7_ippsMD5GetTag +#define ippsMD5Final m7_ippsMD5Final +#define ippsMD5MessageDigest m7_ippsMD5MessageDigest +#define ippsSM3GetSize m7_ippsSM3GetSize +#define ippsSM3Init m7_ippsSM3Init +#define ippsSM3Duplicate m7_ippsSM3Duplicate +#define ippsSM3Pack m7_ippsSM3Pack +#define ippsSM3Unpack m7_ippsSM3Unpack +#define ippsSM3Update m7_ippsSM3Update +#define ippsSM3GetTag m7_ippsSM3GetTag +#define ippsSM3Final m7_ippsSM3Final +#define ippsSM3MessageDigest m7_ippsSM3MessageDigest +#define ippsHashGetSize m7_ippsHashGetSize +#define ippsHashInit m7_ippsHashInit +#define ippsHashPack m7_ippsHashPack +#define ippsHashUnpack m7_ippsHashUnpack +#define ippsHashDuplicate m7_ippsHashDuplicate +#define ippsHashUpdate m7_ippsHashUpdate +#define ippsHashGetTag m7_ippsHashGetTag +#define ippsHashFinal m7_ippsHashFinal +#define ippsHashMessage m7_ippsHashMessage +#define ippsHashMethod_MD5 m7_ippsHashMethod_MD5 +#define ippsHashMethod_SM3 m7_ippsHashMethod_SM3 +#define ippsHashMethod_SHA1 m7_ippsHashMethod_SHA1 +#define ippsHashMethod_SHA1_NI m7_ippsHashMethod_SHA1_NI +#define ippsHashMethod_SHA1_TT m7_ippsHashMethod_SHA1_TT +#define ippsHashMethod_SHA256 m7_ippsHashMethod_SHA256 +#define ippsHashMethod_SHA256_NI m7_ippsHashMethod_SHA256_NI +#define ippsHashMethod_SHA256_TT m7_ippsHashMethod_SHA256_TT +#define ippsHashMethod_SHA224 m7_ippsHashMethod_SHA224 +#define ippsHashMethod_SHA224_NI m7_ippsHashMethod_SHA224_NI +#define ippsHashMethod_SHA224_TT m7_ippsHashMethod_SHA224_TT +#define ippsHashMethod_SHA512 m7_ippsHashMethod_SHA512 +#define ippsHashMethod_SHA384 m7_ippsHashMethod_SHA384 +#define ippsHashMethod_SHA512_256 m7_ippsHashMethod_SHA512_256 +#define ippsHashMethod_SHA512_224 m7_ippsHashMethod_SHA512_224 +#define ippsHashMethodGetSize m7_ippsHashMethodGetSize +#define ippsHashMethodSet_MD5 m7_ippsHashMethodSet_MD5 +#define ippsHashMethodSet_SM3 m7_ippsHashMethodSet_SM3 +#define ippsHashStateMethodSet_SM3 m7_ippsHashStateMethodSet_SM3 +#define ippsHashMethodSet_SHA1 m7_ippsHashMethodSet_SHA1 +#define ippsHashMethodSet_SHA1_NI m7_ippsHashMethodSet_SHA1_NI +#define ippsHashMethodSet_SHA1_TT m7_ippsHashMethodSet_SHA1_TT +#define ippsHashMethodSet_SHA256 m7_ippsHashMethodSet_SHA256 +#define ippsHashMethodSet_SHA256_NI m7_ippsHashMethodSet_SHA256_NI +#define ippsHashMethodSet_SHA256_TT m7_ippsHashMethodSet_SHA256_TT +#define ippsHashStateMethodSet_SHA256 m7_ippsHashStateMethodSet_SHA256 +#define ippsHashStateMethodSet_SHA256_NI m7_ippsHashStateMethodSet_SHA256_NI +#define ippsHashStateMethodSet_SHA256_TT m7_ippsHashStateMethodSet_SHA256_TT +#define ippsHashMethodSet_SHA224 m7_ippsHashMethodSet_SHA224 +#define ippsHashMethodSet_SHA224_NI m7_ippsHashMethodSet_SHA224_NI +#define ippsHashMethodSet_SHA224_TT m7_ippsHashMethodSet_SHA224_TT +#define ippsHashStateMethodSet_SHA224 m7_ippsHashStateMethodSet_SHA224 +#define ippsHashStateMethodSet_SHA224_NI m7_ippsHashStateMethodSet_SHA224_NI +#define ippsHashStateMethodSet_SHA224_TT m7_ippsHashStateMethodSet_SHA224_TT +#define ippsHashMethodSet_SHA512 m7_ippsHashMethodSet_SHA512 +#define ippsHashMethodSet_SHA384 m7_ippsHashMethodSet_SHA384 +#define ippsHashMethodSet_SHA512_256 m7_ippsHashMethodSet_SHA512_256 +#define ippsHashMethodSet_SHA512_224 m7_ippsHashMethodSet_SHA512_224 +#define ippsHashStateMethodSet_SHA512 m7_ippsHashStateMethodSet_SHA512 +#define ippsHashStateMethodSet_SHA384 m7_ippsHashStateMethodSet_SHA384 +#define ippsHashStateMethodSet_SHA512_256 m7_ippsHashStateMethodSet_SHA512_256 +#define ippsHashStateMethodSet_SHA512_224 m7_ippsHashStateMethodSet_SHA512_224 +#define ippsHashGetSize_rmf m7_ippsHashGetSize_rmf +#define ippsHashInit_rmf m7_ippsHashInit_rmf +#define ippsHashPack_rmf m7_ippsHashPack_rmf +#define ippsHashUnpack_rmf m7_ippsHashUnpack_rmf +#define ippsHashDuplicate_rmf m7_ippsHashDuplicate_rmf +#define ippsHashUpdate_rmf m7_ippsHashUpdate_rmf +#define ippsHashGetTag_rmf m7_ippsHashGetTag_rmf +#define ippsHashFinal_rmf m7_ippsHashFinal_rmf +#define ippsHashMessage_rmf m7_ippsHashMessage_rmf +#define ippsHashMethodGetInfo m7_ippsHashMethodGetInfo +#define ippsHashGetInfo_rmf m7_ippsHashGetInfo_rmf +#define ippsMGF m7_ippsMGF +#define ippsMGF1_rmf m7_ippsMGF1_rmf +#define ippsMGF2_rmf m7_ippsMGF2_rmf +#define ippsHMAC_GetSize m7_ippsHMAC_GetSize +#define ippsHMAC_Init m7_ippsHMAC_Init +#define ippsHMAC_Pack m7_ippsHMAC_Pack +#define ippsHMAC_Unpack m7_ippsHMAC_Unpack +#define ippsHMAC_Duplicate m7_ippsHMAC_Duplicate +#define ippsHMAC_Update m7_ippsHMAC_Update +#define ippsHMAC_Final m7_ippsHMAC_Final +#define ippsHMAC_GetTag m7_ippsHMAC_GetTag +#define ippsHMAC_Message m7_ippsHMAC_Message +#define ippsHMACGetSize_rmf m7_ippsHMACGetSize_rmf +#define ippsHMACInit_rmf m7_ippsHMACInit_rmf +#define ippsHMACPack_rmf m7_ippsHMACPack_rmf +#define ippsHMACUnpack_rmf m7_ippsHMACUnpack_rmf +#define ippsHMACDuplicate_rmf m7_ippsHMACDuplicate_rmf +#define ippsHMACUpdate_rmf m7_ippsHMACUpdate_rmf +#define ippsHMACFinal_rmf m7_ippsHMACFinal_rmf +#define ippsHMACGetTag_rmf m7_ippsHMACGetTag_rmf +#define ippsHMACMessage_rmf m7_ippsHMACMessage_rmf +#define ippsBigNumGetSize m7_ippsBigNumGetSize +#define ippsBigNumInit m7_ippsBigNumInit +#define ippsCmpZero_BN m7_ippsCmpZero_BN +#define ippsCmp_BN m7_ippsCmp_BN +#define ippsGetSize_BN m7_ippsGetSize_BN +#define ippsSet_BN m7_ippsSet_BN +#define ippsGet_BN m7_ippsGet_BN +#define ippsRef_BN m7_ippsRef_BN +#define ippsExtGet_BN m7_ippsExtGet_BN +#define ippsAdd_BN m7_ippsAdd_BN +#define ippsSub_BN m7_ippsSub_BN +#define ippsMul_BN m7_ippsMul_BN +#define ippsMAC_BN_I m7_ippsMAC_BN_I +#define ippsDiv_BN m7_ippsDiv_BN +#define ippsMod_BN m7_ippsMod_BN +#define ippsGcd_BN m7_ippsGcd_BN +#define ippsModInv_BN m7_ippsModInv_BN +#define ippsSetOctString_BN m7_ippsSetOctString_BN +#define ippsGetOctString_BN m7_ippsGetOctString_BN +#define ippsMontGetSize m7_ippsMontGetSize +#define ippsMontInit m7_ippsMontInit +#define ippsMontSet m7_ippsMontSet +#define ippsMontGet m7_ippsMontGet +#define ippsMontForm m7_ippsMontForm +#define ippsMontMul m7_ippsMontMul +#define ippsMontExp m7_ippsMontExp +#define ippsPRNGGetSize m7_ippsPRNGGetSize +#define ippsPRNGInit m7_ippsPRNGInit +#define ippsPRNGSetModulus m7_ippsPRNGSetModulus +#define ippsPRNGSetH0 m7_ippsPRNGSetH0 +#define ippsPRNGSetAugment m7_ippsPRNGSetAugment +#define ippsPRNGSetSeed m7_ippsPRNGSetSeed +#define ippsPRNGGetSeed m7_ippsPRNGGetSeed +#define ippsPRNGen m7_ippsPRNGen +#define ippsPRNGen_BN m7_ippsPRNGen_BN +#define ippsPRNGenRDRAND m7_ippsPRNGenRDRAND +#define ippsPRNGenRDRAND_BN m7_ippsPRNGenRDRAND_BN +#define ippsTRNGenRDSEED m7_ippsTRNGenRDSEED +#define ippsTRNGenRDSEED_BN m7_ippsTRNGenRDSEED_BN +#define ippsPrimeGetSize m7_ippsPrimeGetSize +#define ippsPrimeInit m7_ippsPrimeInit +#define ippsPrimeGen m7_ippsPrimeGen +#define ippsPrimeTest m7_ippsPrimeTest +#define ippsPrimeGen_BN m7_ippsPrimeGen_BN +#define ippsPrimeTest_BN m7_ippsPrimeTest_BN +#define ippsPrimeGet m7_ippsPrimeGet +#define ippsPrimeGet_BN m7_ippsPrimeGet_BN +#define ippsPrimeSet m7_ippsPrimeSet +#define ippsPrimeSet_BN m7_ippsPrimeSet_BN +#define ippsRSA_GetSizePublicKey m7_ippsRSA_GetSizePublicKey +#define ippsRSA_InitPublicKey m7_ippsRSA_InitPublicKey +#define ippsRSA_SetPublicKey m7_ippsRSA_SetPublicKey +#define ippsRSA_GetPublicKey m7_ippsRSA_GetPublicKey +#define ippsRSA_GetSizePrivateKeyType1 m7_ippsRSA_GetSizePrivateKeyType1 +#define ippsRSA_InitPrivateKeyType1 m7_ippsRSA_InitPrivateKeyType1 +#define ippsRSA_SetPrivateKeyType1 m7_ippsRSA_SetPrivateKeyType1 +#define ippsRSA_GetPrivateKeyType1 m7_ippsRSA_GetPrivateKeyType1 +#define ippsRSA_GetSizePrivateKeyType2 m7_ippsRSA_GetSizePrivateKeyType2 +#define ippsRSA_InitPrivateKeyType2 m7_ippsRSA_InitPrivateKeyType2 +#define ippsRSA_SetPrivateKeyType2 m7_ippsRSA_SetPrivateKeyType2 +#define ippsRSA_GetPrivateKeyType2 m7_ippsRSA_GetPrivateKeyType2 +#define ippsRSA_GetBufferSizePublicKey m7_ippsRSA_GetBufferSizePublicKey +#define ippsRSA_GetBufferSizePrivateKey m7_ippsRSA_GetBufferSizePrivateKey +#define ippsRSA_Encrypt m7_ippsRSA_Encrypt +#define ippsRSA_Decrypt m7_ippsRSA_Decrypt +#define ippsRSA_GenerateKeys m7_ippsRSA_GenerateKeys +#define ippsRSA_ValidateKeys m7_ippsRSA_ValidateKeys +#define ippsRSAEncrypt_OAEP m7_ippsRSAEncrypt_OAEP +#define ippsRSADecrypt_OAEP m7_ippsRSADecrypt_OAEP +#define ippsRSAEncrypt_OAEP_rmf m7_ippsRSAEncrypt_OAEP_rmf +#define ippsRSADecrypt_OAEP_rmf m7_ippsRSADecrypt_OAEP_rmf +#define ippsRSAEncrypt_PKCSv15 m7_ippsRSAEncrypt_PKCSv15 +#define ippsRSADecrypt_PKCSv15 m7_ippsRSADecrypt_PKCSv15 +#define ippsRSASign_PSS m7_ippsRSASign_PSS +#define ippsRSAVerify_PSS m7_ippsRSAVerify_PSS +#define ippsRSASign_PSS_rmf m7_ippsRSASign_PSS_rmf +#define ippsRSAVerify_PSS_rmf m7_ippsRSAVerify_PSS_rmf +#define ippsRSASign_PKCS1v15 m7_ippsRSASign_PKCS1v15 +#define ippsRSAVerify_PKCS1v15 m7_ippsRSAVerify_PKCS1v15 +#define ippsRSASign_PKCS1v15_rmf m7_ippsRSASign_PKCS1v15_rmf +#define ippsRSAVerify_PKCS1v15_rmf m7_ippsRSAVerify_PKCS1v15_rmf +#define ippsDLGetResultString m7_ippsDLGetResultString +#define ippsDLPGetSize m7_ippsDLPGetSize +#define ippsDLPInit m7_ippsDLPInit +#define ippsDLPPack m7_ippsDLPPack +#define ippsDLPUnpack m7_ippsDLPUnpack +#define ippsDLPSet m7_ippsDLPSet +#define ippsDLPGet m7_ippsDLPGet +#define ippsDLPSetDP m7_ippsDLPSetDP +#define ippsDLPGetDP m7_ippsDLPGetDP +#define ippsDLPGenKeyPair m7_ippsDLPGenKeyPair +#define ippsDLPPublicKey m7_ippsDLPPublicKey +#define ippsDLPValidateKeyPair m7_ippsDLPValidateKeyPair +#define ippsDLPSetKeyPair m7_ippsDLPSetKeyPair +#define ippsDLPSignDSA m7_ippsDLPSignDSA +#define ippsDLPVerifyDSA m7_ippsDLPVerifyDSA +#define ippsDLPSharedSecretDH m7_ippsDLPSharedSecretDH +#define ippsDLPGenerateDSA m7_ippsDLPGenerateDSA +#define ippsDLPValidateDSA m7_ippsDLPValidateDSA +#define ippsDLPGenerateDH m7_ippsDLPGenerateDH +#define ippsDLPValidateDH m7_ippsDLPValidateDH +#define ippsECCGetResultString m7_ippsECCGetResultString +#define ippsECCPGetSize m7_ippsECCPGetSize +#define ippsECCPGetSizeStd128r1 m7_ippsECCPGetSizeStd128r1 +#define ippsECCPGetSizeStd128r2 m7_ippsECCPGetSizeStd128r2 +#define ippsECCPGetSizeStd192r1 m7_ippsECCPGetSizeStd192r1 +#define ippsECCPGetSizeStd224r1 m7_ippsECCPGetSizeStd224r1 +#define ippsECCPGetSizeStd256r1 m7_ippsECCPGetSizeStd256r1 +#define ippsECCPGetSizeStd384r1 m7_ippsECCPGetSizeStd384r1 +#define ippsECCPGetSizeStd521r1 m7_ippsECCPGetSizeStd521r1 +#define ippsECCPGetSizeStdSM2 m7_ippsECCPGetSizeStdSM2 +#define ippsECCPInit m7_ippsECCPInit +#define ippsECCPInitStd128r1 m7_ippsECCPInitStd128r1 +#define ippsECCPInitStd128r2 m7_ippsECCPInitStd128r2 +#define ippsECCPInitStd192r1 m7_ippsECCPInitStd192r1 +#define ippsECCPInitStd224r1 m7_ippsECCPInitStd224r1 +#define ippsECCPInitStd256r1 m7_ippsECCPInitStd256r1 +#define ippsECCPInitStd384r1 m7_ippsECCPInitStd384r1 +#define ippsECCPInitStd521r1 m7_ippsECCPInitStd521r1 +#define ippsECCPInitStdSM2 m7_ippsECCPInitStdSM2 +#define ippsECCPSet m7_ippsECCPSet +#define ippsECCPSetStd m7_ippsECCPSetStd +#define ippsECCPSetStd128r1 m7_ippsECCPSetStd128r1 +#define ippsECCPSetStd128r2 m7_ippsECCPSetStd128r2 +#define ippsECCPSetStd192r1 m7_ippsECCPSetStd192r1 +#define ippsECCPSetStd224r1 m7_ippsECCPSetStd224r1 +#define ippsECCPSetStd256r1 m7_ippsECCPSetStd256r1 +#define ippsECCPSetStd384r1 m7_ippsECCPSetStd384r1 +#define ippsECCPSetStd521r1 m7_ippsECCPSetStd521r1 +#define ippsECCPSetStdSM2 m7_ippsECCPSetStdSM2 +#define ippsECCPBindGxyTblStd192r1 m7_ippsECCPBindGxyTblStd192r1 +#define ippsECCPBindGxyTblStd224r1 m7_ippsECCPBindGxyTblStd224r1 +#define ippsECCPBindGxyTblStd256r1 m7_ippsECCPBindGxyTblStd256r1 +#define ippsECCPBindGxyTblStd384r1 m7_ippsECCPBindGxyTblStd384r1 +#define ippsECCPBindGxyTblStd521r1 m7_ippsECCPBindGxyTblStd521r1 +#define ippsECCPBindGxyTblStdSM2 m7_ippsECCPBindGxyTblStdSM2 +#define ippsECCPGet m7_ippsECCPGet +#define ippsECCPGetOrderBitSize m7_ippsECCPGetOrderBitSize +#define ippsECCPValidate m7_ippsECCPValidate +#define ippsECCPPointGetSize m7_ippsECCPPointGetSize +#define ippsECCPPointInit m7_ippsECCPPointInit +#define ippsECCPSetPoint m7_ippsECCPSetPoint +#define ippsECCPSetPointAtInfinity m7_ippsECCPSetPointAtInfinity +#define ippsECCPGetPoint m7_ippsECCPGetPoint +#define ippsECCPCheckPoint m7_ippsECCPCheckPoint +#define ippsECCPComparePoint m7_ippsECCPComparePoint +#define ippsECCPNegativePoint m7_ippsECCPNegativePoint +#define ippsECCPAddPoint m7_ippsECCPAddPoint +#define ippsECCPMulPointScalar m7_ippsECCPMulPointScalar +#define ippsECCPGenKeyPair m7_ippsECCPGenKeyPair +#define ippsECCPPublicKey m7_ippsECCPPublicKey +#define ippsECCPValidateKeyPair m7_ippsECCPValidateKeyPair +#define ippsECCPSetKeyPair m7_ippsECCPSetKeyPair +#define ippsECCPSharedSecretDH m7_ippsECCPSharedSecretDH +#define ippsECCPSharedSecretDHC m7_ippsECCPSharedSecretDHC +#define ippsECCPSignDSA m7_ippsECCPSignDSA +#define ippsECCPVerifyDSA m7_ippsECCPVerifyDSA +#define ippsECCPSignNR m7_ippsECCPSignNR +#define ippsECCPVerifyNR m7_ippsECCPVerifyNR +#define ippsECCPSignSM2 m7_ippsECCPSignSM2 +#define ippsECCPVerifySM2 m7_ippsECCPVerifySM2 +#define ippsGFpGetSize m7_ippsGFpGetSize +#define ippsGFpInitArbitrary m7_ippsGFpInitArbitrary +#define ippsGFpInitFixed m7_ippsGFpInitFixed +#define ippsGFpInit m7_ippsGFpInit +#define ippsGFpMethod_p192r1 m7_ippsGFpMethod_p192r1 +#define ippsGFpMethod_p224r1 m7_ippsGFpMethod_p224r1 +#define ippsGFpMethod_p256r1 m7_ippsGFpMethod_p256r1 +#define ippsGFpMethod_p384r1 m7_ippsGFpMethod_p384r1 +#define ippsGFpMethod_p521r1 m7_ippsGFpMethod_p521r1 +#define ippsGFpMethod_p256sm2 m7_ippsGFpMethod_p256sm2 +#define ippsGFpMethod_p256bn m7_ippsGFpMethod_p256bn +#define ippsGFpMethod_p256 m7_ippsGFpMethod_p256 +#define ippsGFpMethod_pArb m7_ippsGFpMethod_pArb +#define ippsGFpxGetSize m7_ippsGFpxGetSize +#define ippsGFpxInit m7_ippsGFpxInit +#define ippsGFpxInitBinomial m7_ippsGFpxInitBinomial +#define ippsGFpxMethod_binom2_epid2 m7_ippsGFpxMethod_binom2_epid2 +#define ippsGFpxMethod_binom3_epid2 m7_ippsGFpxMethod_binom3_epid2 +#define ippsGFpxMethod_binom2 m7_ippsGFpxMethod_binom2 +#define ippsGFpxMethod_binom3 m7_ippsGFpxMethod_binom3 +#define ippsGFpxMethod_binom m7_ippsGFpxMethod_binom +#define ippsGFpxMethod_com m7_ippsGFpxMethod_com +#define ippsGFpScratchBufferSize m7_ippsGFpScratchBufferSize +#define ippsGFpElementGetSize m7_ippsGFpElementGetSize +#define ippsGFpElementInit m7_ippsGFpElementInit +#define ippsGFpSetElement m7_ippsGFpSetElement +#define ippsGFpSetElementRegular m7_ippsGFpSetElementRegular +#define ippsGFpSetElementOctString m7_ippsGFpSetElementOctString +#define ippsGFpSetElementRandom m7_ippsGFpSetElementRandom +#define ippsGFpSetElementHash m7_ippsGFpSetElementHash +#define ippsGFpSetElementHash_rmf m7_ippsGFpSetElementHash_rmf +#define ippsGFpCpyElement m7_ippsGFpCpyElement +#define ippsGFpGetElement m7_ippsGFpGetElement +#define ippsGFpGetElementOctString m7_ippsGFpGetElementOctString +#define ippsGFpCmpElement m7_ippsGFpCmpElement +#define ippsGFpIsZeroElement m7_ippsGFpIsZeroElement +#define ippsGFpIsUnityElement m7_ippsGFpIsUnityElement +#define ippsGFpConj m7_ippsGFpConj +#define ippsGFpNeg m7_ippsGFpNeg +#define ippsGFpInv m7_ippsGFpInv +#define ippsGFpSqrt m7_ippsGFpSqrt +#define ippsGFpSqr m7_ippsGFpSqr +#define ippsGFpAdd m7_ippsGFpAdd +#define ippsGFpSub m7_ippsGFpSub +#define ippsGFpMul m7_ippsGFpMul +#define ippsGFpExp m7_ippsGFpExp +#define ippsGFpMultiExp m7_ippsGFpMultiExp +#define ippsGFpAdd_PE m7_ippsGFpAdd_PE +#define ippsGFpSub_PE m7_ippsGFpSub_PE +#define ippsGFpMul_PE m7_ippsGFpMul_PE +#define ippsGFpGetInfo m7_ippsGFpGetInfo +#define ippsGFpECGetSize m7_ippsGFpECGetSize +#define ippsGFpECInit m7_ippsGFpECInit +#define ippsGFpECSet m7_ippsGFpECSet +#define ippsGFpECSetSubgroup m7_ippsGFpECSetSubgroup +#define ippsGFpECInitStd128r1 m7_ippsGFpECInitStd128r1 +#define ippsGFpECInitStd128r2 m7_ippsGFpECInitStd128r2 +#define ippsGFpECInitStd192r1 m7_ippsGFpECInitStd192r1 +#define ippsGFpECInitStd224r1 m7_ippsGFpECInitStd224r1 +#define ippsGFpECInitStd256r1 m7_ippsGFpECInitStd256r1 +#define ippsGFpECInitStd384r1 m7_ippsGFpECInitStd384r1 +#define ippsGFpECInitStd521r1 m7_ippsGFpECInitStd521r1 +#define ippsGFpECInitStdSM2 m7_ippsGFpECInitStdSM2 +#define ippsGFpECInitStdBN256 m7_ippsGFpECInitStdBN256 +#define ippsGFpECBindGxyTblStd192r1 m7_ippsGFpECBindGxyTblStd192r1 +#define ippsGFpECBindGxyTblStd224r1 m7_ippsGFpECBindGxyTblStd224r1 +#define ippsGFpECBindGxyTblStd256r1 m7_ippsGFpECBindGxyTblStd256r1 +#define ippsGFpECBindGxyTblStd384r1 m7_ippsGFpECBindGxyTblStd384r1 +#define ippsGFpECBindGxyTblStd521r1 m7_ippsGFpECBindGxyTblStd521r1 +#define ippsGFpECBindGxyTblStdSM2 m7_ippsGFpECBindGxyTblStdSM2 +#define ippsGFpECGet m7_ippsGFpECGet +#define ippsGFpECGetSubgroup m7_ippsGFpECGetSubgroup +#define ippsGFpECScratchBufferSize m7_ippsGFpECScratchBufferSize +#define ippsGFpECVerify m7_ippsGFpECVerify +#define ippsGFpECPointGetSize m7_ippsGFpECPointGetSize +#define ippsGFpECPointInit m7_ippsGFpECPointInit +#define ippsGFpECSetPointAtInfinity m7_ippsGFpECSetPointAtInfinity +#define ippsGFpECSetPoint m7_ippsGFpECSetPoint +#define ippsGFpECSetPointRegular m7_ippsGFpECSetPointRegular +#define ippsGFpECSetPointRandom m7_ippsGFpECSetPointRandom +#define ippsGFpECMakePoint m7_ippsGFpECMakePoint +#define ippsGFpECSetPointHash m7_ippsGFpECSetPointHash +#define ippsGFpECSetPointHashBackCompatible m7_ippsGFpECSetPointHashBackCompatible +#define ippsGFpECSetPointHash_rmf m7_ippsGFpECSetPointHash_rmf +#define ippsGFpECSetPointHashBackCompatible_rmf m7_ippsGFpECSetPointHashBackCompatible_rmf +#define ippsGFpECGetPoint m7_ippsGFpECGetPoint +#define ippsGFpECGetPointRegular m7_ippsGFpECGetPointRegular +#define ippsGFpECSetPointOctString m7_ippsGFpECSetPointOctString +#define ippsGFpECGetPointOctString m7_ippsGFpECGetPointOctString +#define ippsGFpECTstPoint m7_ippsGFpECTstPoint +#define ippsGFpECTstPointInSubgroup m7_ippsGFpECTstPointInSubgroup +#define ippsGFpECCpyPoint m7_ippsGFpECCpyPoint +#define ippsGFpECCmpPoint m7_ippsGFpECCmpPoint +#define ippsGFpECNegPoint m7_ippsGFpECNegPoint +#define ippsGFpECAddPoint m7_ippsGFpECAddPoint +#define ippsGFpECMulPoint m7_ippsGFpECMulPoint +#define ippsGFpECPrivateKey m7_ippsGFpECPrivateKey +#define ippsGFpECPublicKey m7_ippsGFpECPublicKey +#define ippsGFpECTstKeyPair m7_ippsGFpECTstKeyPair +#define ippsGFpECSharedSecretDH m7_ippsGFpECSharedSecretDH +#define ippsGFpECSharedSecretDHC m7_ippsGFpECSharedSecretDHC +#define ippsGFpECMessageRepresentationSM2 m7_ippsGFpECMessageRepresentationSM2 +#define ippsGFpECSignDSA m7_ippsGFpECSignDSA +#define ippsGFpECVerifyDSA m7_ippsGFpECVerifyDSA +#define ippsGFpECSignNR m7_ippsGFpECSignNR +#define ippsGFpECVerifyNR m7_ippsGFpECVerifyNR +#define ippsGFpECSignSM2 m7_ippsGFpECSignSM2 +#define ippsGFpECVerifySM2 m7_ippsGFpECVerifySM2 +#define ippsGFpECUserIDHashSM2 m7_ippsGFpECUserIDHashSM2 +#define ippsGFpECKeyExchangeSM2_GetSize m7_ippsGFpECKeyExchangeSM2_GetSize +#define ippsGFpECKeyExchangeSM2_Init m7_ippsGFpECKeyExchangeSM2_Init +#define ippsGFpECKeyExchangeSM2_Setup m7_ippsGFpECKeyExchangeSM2_Setup +#define ippsGFpECKeyExchangeSM2_SharedKey m7_ippsGFpECKeyExchangeSM2_SharedKey +#define ippsGFpECKeyExchangeSM2_Confirm m7_ippsGFpECKeyExchangeSM2_Confirm +#define ippsGFpECGetInfo_GF m7_ippsGFpECGetInfo_GF +#define ippsGFpECESGetSize_SM2 m7_ippsGFpECESGetSize_SM2 +#define ippsGFpECESInit_SM2 m7_ippsGFpECESInit_SM2 +#define ippsGFpECESSetKey_SM2 m7_ippsGFpECESSetKey_SM2 +#define ippsGFpECESStart_SM2 m7_ippsGFpECESStart_SM2 +#define ippsGFpECESEncrypt_SM2 m7_ippsGFpECESEncrypt_SM2 +#define ippsGFpECESDecrypt_SM2 m7_ippsGFpECESDecrypt_SM2 +#define ippsGFpECESFinal_SM2 m7_ippsGFpECESFinal_SM2 +#define ippsGFpECESGetBuffersSize_SM2 m7_ippsGFpECESGetBuffersSize_SM2 +#define ippsGFpECEncryptSM2_Ext_EncMsgSize m7_ippsGFpECEncryptSM2_Ext_EncMsgSize +#define ippsGFpECEncryptSM2_Ext m7_ippsGFpECEncryptSM2_Ext +#define ippsGFpECDecryptSM2_Ext_DecMsgSize m7_ippsGFpECDecryptSM2_Ext_DecMsgSize +#define ippsGFpECDecryptSM2_Ext m7_ippsGFpECDecryptSM2_Ext diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_n0.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_n0.h new file mode 100644 index 0000000..a3d367b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_n0.h @@ -0,0 +1,557 @@ +/******************************************************************************* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + + #define ippcpGetLibVersion n0_ippcpGetLibVersion +#define ippsDESGetSize n0_ippsDESGetSize +#define ippsDESInit n0_ippsDESInit +#define ippsDESPack n0_ippsDESPack +#define ippsDESUnpack n0_ippsDESUnpack +#define ippsTDESEncryptECB n0_ippsTDESEncryptECB +#define ippsTDESDecryptECB n0_ippsTDESDecryptECB +#define ippsTDESEncryptCBC n0_ippsTDESEncryptCBC +#define ippsTDESDecryptCBC n0_ippsTDESDecryptCBC +#define ippsTDESEncryptCFB n0_ippsTDESEncryptCFB +#define ippsTDESDecryptCFB n0_ippsTDESDecryptCFB +#define ippsTDESEncryptOFB n0_ippsTDESEncryptOFB +#define ippsTDESDecryptOFB n0_ippsTDESDecryptOFB +#define ippsTDESEncryptCTR n0_ippsTDESEncryptCTR +#define ippsTDESDecryptCTR n0_ippsTDESDecryptCTR +#define ippsAESGetSize n0_ippsAESGetSize +#define ippsAESInit n0_ippsAESInit +#define ippsAESSetKey n0_ippsAESSetKey +#define ippsAESPack n0_ippsAESPack +#define ippsAESUnpack n0_ippsAESUnpack +#define ippsAESEncryptECB n0_ippsAESEncryptECB +#define ippsAESDecryptECB n0_ippsAESDecryptECB +#define ippsAESEncryptCBC n0_ippsAESEncryptCBC +#define ippsAESEncryptCBC_CS1 n0_ippsAESEncryptCBC_CS1 +#define ippsAESEncryptCBC_CS2 n0_ippsAESEncryptCBC_CS2 +#define ippsAESEncryptCBC_CS3 n0_ippsAESEncryptCBC_CS3 +#define ippsAESDecryptCBC n0_ippsAESDecryptCBC +#define ippsAESDecryptCBC_CS1 n0_ippsAESDecryptCBC_CS1 +#define ippsAESDecryptCBC_CS2 n0_ippsAESDecryptCBC_CS2 +#define ippsAESDecryptCBC_CS3 n0_ippsAESDecryptCBC_CS3 +#define ippsAESEncryptCFB n0_ippsAESEncryptCFB +#define ippsAESDecryptCFB n0_ippsAESDecryptCFB +#define ippsAESEncryptOFB n0_ippsAESEncryptOFB +#define ippsAESDecryptOFB n0_ippsAESDecryptOFB +#define ippsAESEncryptCTR n0_ippsAESEncryptCTR +#define ippsAESDecryptCTR n0_ippsAESDecryptCTR +#define ippsAESEncryptXTS_Direct n0_ippsAESEncryptXTS_Direct +#define ippsAESDecryptXTS_Direct n0_ippsAESDecryptXTS_Direct +#define ippsAESSetupNoise n0_ippsAESSetupNoise +#define ippsAES_GCMSetupNoise n0_ippsAES_GCMSetupNoise +#define ippsAES_CMACSetupNoise n0_ippsAES_CMACSetupNoise +#define ippsAES_EncryptCFB16_MB n0_ippsAES_EncryptCFB16_MB +#define ippsSMS4GetSize n0_ippsSMS4GetSize +#define ippsSMS4Init n0_ippsSMS4Init +#define ippsSMS4SetKey n0_ippsSMS4SetKey +#define ippsSMS4EncryptECB n0_ippsSMS4EncryptECB +#define ippsSMS4DecryptECB n0_ippsSMS4DecryptECB +#define ippsSMS4EncryptCBC n0_ippsSMS4EncryptCBC +#define ippsSMS4EncryptCBC_CS1 n0_ippsSMS4EncryptCBC_CS1 +#define ippsSMS4EncryptCBC_CS2 n0_ippsSMS4EncryptCBC_CS2 +#define ippsSMS4EncryptCBC_CS3 n0_ippsSMS4EncryptCBC_CS3 +#define ippsSMS4DecryptCBC n0_ippsSMS4DecryptCBC +#define ippsSMS4DecryptCBC_CS1 n0_ippsSMS4DecryptCBC_CS1 +#define ippsSMS4DecryptCBC_CS2 n0_ippsSMS4DecryptCBC_CS2 +#define ippsSMS4DecryptCBC_CS3 n0_ippsSMS4DecryptCBC_CS3 +#define ippsSMS4EncryptCFB n0_ippsSMS4EncryptCFB +#define ippsSMS4DecryptCFB n0_ippsSMS4DecryptCFB +#define ippsSMS4EncryptOFB n0_ippsSMS4EncryptOFB +#define ippsSMS4DecryptOFB n0_ippsSMS4DecryptOFB +#define ippsSMS4EncryptCTR n0_ippsSMS4EncryptCTR +#define ippsSMS4DecryptCTR n0_ippsSMS4DecryptCTR +#define ippsSMS4_CCMGetSize n0_ippsSMS4_CCMGetSize +#define ippsSMS4_CCMInit n0_ippsSMS4_CCMInit +#define ippsSMS4_CCMMessageLen n0_ippsSMS4_CCMMessageLen +#define ippsSMS4_CCMTagLen n0_ippsSMS4_CCMTagLen +#define ippsSMS4_CCMStart n0_ippsSMS4_CCMStart +#define ippsSMS4_CCMEncrypt n0_ippsSMS4_CCMEncrypt +#define ippsSMS4_CCMDecrypt n0_ippsSMS4_CCMDecrypt +#define ippsSMS4_CCMGetTag n0_ippsSMS4_CCMGetTag +#define ippsAES_CCMGetSize n0_ippsAES_CCMGetSize +#define ippsAES_CCMInit n0_ippsAES_CCMInit +#define ippsAES_CCMMessageLen n0_ippsAES_CCMMessageLen +#define ippsAES_CCMTagLen n0_ippsAES_CCMTagLen +#define ippsAES_CCMStart n0_ippsAES_CCMStart +#define ippsAES_CCMEncrypt n0_ippsAES_CCMEncrypt +#define ippsAES_CCMDecrypt n0_ippsAES_CCMDecrypt +#define ippsAES_CCMGetTag n0_ippsAES_CCMGetTag +#define ippsAES_GCMGetSize n0_ippsAES_GCMGetSize +#define ippsAES_GCMInit n0_ippsAES_GCMInit +#define ippsAES_GCMReinit n0_ippsAES_GCMReinit +#define ippsAES_GCMReset n0_ippsAES_GCMReset +#define ippsAES_GCMProcessIV n0_ippsAES_GCMProcessIV +#define ippsAES_GCMProcessAAD n0_ippsAES_GCMProcessAAD +#define ippsAES_GCMStart n0_ippsAES_GCMStart +#define ippsAES_GCMEncrypt n0_ippsAES_GCMEncrypt +#define ippsAES_GCMDecrypt n0_ippsAES_GCMDecrypt +#define ippsAES_GCMGetTag n0_ippsAES_GCMGetTag +#define ippsAES_XTSGetSize n0_ippsAES_XTSGetSize +#define ippsAES_XTSInit n0_ippsAES_XTSInit +#define ippsAES_XTSEncrypt n0_ippsAES_XTSEncrypt +#define ippsAES_XTSDecrypt n0_ippsAES_XTSDecrypt +#define ippsAES_S2V_CMAC n0_ippsAES_S2V_CMAC +#define ippsAES_SIVEncrypt n0_ippsAES_SIVEncrypt +#define ippsAES_SIVDecrypt n0_ippsAES_SIVDecrypt +#define ippsAES_CMACGetSize n0_ippsAES_CMACGetSize +#define ippsAES_CMACInit n0_ippsAES_CMACInit +#define ippsAES_CMACUpdate n0_ippsAES_CMACUpdate +#define ippsAES_CMACFinal n0_ippsAES_CMACFinal +#define ippsAES_CMACGetTag n0_ippsAES_CMACGetTag +#define ippsARCFourCheckKey n0_ippsARCFourCheckKey +#define ippsARCFourGetSize n0_ippsARCFourGetSize +#define ippsARCFourInit n0_ippsARCFourInit +#define ippsARCFourReset n0_ippsARCFourReset +#define ippsARCFourPack n0_ippsARCFourPack +#define ippsARCFourUnpack n0_ippsARCFourUnpack +#define ippsARCFourEncrypt n0_ippsARCFourEncrypt +#define ippsARCFourDecrypt n0_ippsARCFourDecrypt +#define ippsSHA1GetSize n0_ippsSHA1GetSize +#define ippsSHA1Init n0_ippsSHA1Init +#define ippsSHA1Duplicate n0_ippsSHA1Duplicate +#define ippsSHA1Pack n0_ippsSHA1Pack +#define ippsSHA1Unpack n0_ippsSHA1Unpack +#define ippsSHA1Update n0_ippsSHA1Update +#define ippsSHA1GetTag n0_ippsSHA1GetTag +#define ippsSHA1Final n0_ippsSHA1Final +#define ippsSHA1MessageDigest n0_ippsSHA1MessageDigest +#define ippsSHA224GetSize n0_ippsSHA224GetSize +#define ippsSHA224Init n0_ippsSHA224Init +#define ippsSHA224Duplicate n0_ippsSHA224Duplicate +#define ippsSHA224Pack n0_ippsSHA224Pack +#define ippsSHA224Unpack n0_ippsSHA224Unpack +#define ippsSHA224Update n0_ippsSHA224Update +#define ippsSHA224GetTag n0_ippsSHA224GetTag +#define ippsSHA224Final n0_ippsSHA224Final +#define ippsSHA224MessageDigest n0_ippsSHA224MessageDigest +#define ippsSHA256GetSize n0_ippsSHA256GetSize +#define ippsSHA256Init n0_ippsSHA256Init +#define ippsSHA256Duplicate n0_ippsSHA256Duplicate +#define ippsSHA256Pack n0_ippsSHA256Pack +#define ippsSHA256Unpack n0_ippsSHA256Unpack +#define ippsSHA256Update n0_ippsSHA256Update +#define ippsSHA256GetTag n0_ippsSHA256GetTag +#define ippsSHA256Final n0_ippsSHA256Final +#define ippsSHA256MessageDigest n0_ippsSHA256MessageDigest +#define ippsSHA384GetSize n0_ippsSHA384GetSize +#define ippsSHA384Init n0_ippsSHA384Init +#define ippsSHA384Duplicate n0_ippsSHA384Duplicate +#define ippsSHA384Pack n0_ippsSHA384Pack +#define ippsSHA384Unpack n0_ippsSHA384Unpack +#define ippsSHA384Update n0_ippsSHA384Update +#define ippsSHA384GetTag n0_ippsSHA384GetTag +#define ippsSHA384Final n0_ippsSHA384Final +#define ippsSHA384MessageDigest n0_ippsSHA384MessageDigest +#define ippsSHA512GetSize n0_ippsSHA512GetSize +#define ippsSHA512Init n0_ippsSHA512Init +#define ippsSHA512Duplicate n0_ippsSHA512Duplicate +#define ippsSHA512Pack n0_ippsSHA512Pack +#define ippsSHA512Unpack n0_ippsSHA512Unpack +#define ippsSHA512Update n0_ippsSHA512Update +#define ippsSHA512GetTag n0_ippsSHA512GetTag +#define ippsSHA512Final n0_ippsSHA512Final +#define ippsSHA512MessageDigest n0_ippsSHA512MessageDigest +#define ippsMD5GetSize n0_ippsMD5GetSize +#define ippsMD5Init n0_ippsMD5Init +#define ippsMD5Duplicate n0_ippsMD5Duplicate +#define ippsMD5Pack n0_ippsMD5Pack +#define ippsMD5Unpack n0_ippsMD5Unpack +#define ippsMD5Update n0_ippsMD5Update +#define ippsMD5GetTag n0_ippsMD5GetTag +#define ippsMD5Final n0_ippsMD5Final +#define ippsMD5MessageDigest n0_ippsMD5MessageDigest +#define ippsSM3GetSize n0_ippsSM3GetSize +#define ippsSM3Init n0_ippsSM3Init +#define ippsSM3Duplicate n0_ippsSM3Duplicate +#define ippsSM3Pack n0_ippsSM3Pack +#define ippsSM3Unpack n0_ippsSM3Unpack +#define ippsSM3Update n0_ippsSM3Update +#define ippsSM3GetTag n0_ippsSM3GetTag +#define ippsSM3Final n0_ippsSM3Final +#define ippsSM3MessageDigest n0_ippsSM3MessageDigest +#define ippsHashGetSize n0_ippsHashGetSize +#define ippsHashInit n0_ippsHashInit +#define ippsHashPack n0_ippsHashPack +#define ippsHashUnpack n0_ippsHashUnpack +#define ippsHashDuplicate n0_ippsHashDuplicate +#define ippsHashUpdate n0_ippsHashUpdate +#define ippsHashGetTag n0_ippsHashGetTag +#define ippsHashFinal n0_ippsHashFinal +#define ippsHashMessage n0_ippsHashMessage +#define ippsHashMethod_MD5 n0_ippsHashMethod_MD5 +#define ippsHashMethod_SM3 n0_ippsHashMethod_SM3 +#define ippsHashMethod_SHA1 n0_ippsHashMethod_SHA1 +#define ippsHashMethod_SHA1_NI n0_ippsHashMethod_SHA1_NI +#define ippsHashMethod_SHA1_TT n0_ippsHashMethod_SHA1_TT +#define ippsHashMethod_SHA256 n0_ippsHashMethod_SHA256 +#define ippsHashMethod_SHA256_NI n0_ippsHashMethod_SHA256_NI +#define ippsHashMethod_SHA256_TT n0_ippsHashMethod_SHA256_TT +#define ippsHashMethod_SHA224 n0_ippsHashMethod_SHA224 +#define ippsHashMethod_SHA224_NI n0_ippsHashMethod_SHA224_NI +#define ippsHashMethod_SHA224_TT n0_ippsHashMethod_SHA224_TT +#define ippsHashMethod_SHA512 n0_ippsHashMethod_SHA512 +#define ippsHashMethod_SHA384 n0_ippsHashMethod_SHA384 +#define ippsHashMethod_SHA512_256 n0_ippsHashMethod_SHA512_256 +#define ippsHashMethod_SHA512_224 n0_ippsHashMethod_SHA512_224 +#define ippsHashMethodGetSize n0_ippsHashMethodGetSize +#define ippsHashMethodSet_MD5 n0_ippsHashMethodSet_MD5 +#define ippsHashMethodSet_SM3 n0_ippsHashMethodSet_SM3 +#define ippsHashStateMethodSet_SM3 n0_ippsHashStateMethodSet_SM3 +#define ippsHashMethodSet_SHA1 n0_ippsHashMethodSet_SHA1 +#define ippsHashMethodSet_SHA1_NI n0_ippsHashMethodSet_SHA1_NI +#define ippsHashMethodSet_SHA1_TT n0_ippsHashMethodSet_SHA1_TT +#define ippsHashMethodSet_SHA256 n0_ippsHashMethodSet_SHA256 +#define ippsHashMethodSet_SHA256_NI n0_ippsHashMethodSet_SHA256_NI +#define ippsHashMethodSet_SHA256_TT n0_ippsHashMethodSet_SHA256_TT +#define ippsHashStateMethodSet_SHA256 n0_ippsHashStateMethodSet_SHA256 +#define ippsHashStateMethodSet_SHA256_NI n0_ippsHashStateMethodSet_SHA256_NI +#define ippsHashStateMethodSet_SHA256_TT n0_ippsHashStateMethodSet_SHA256_TT +#define ippsHashMethodSet_SHA224 n0_ippsHashMethodSet_SHA224 +#define ippsHashMethodSet_SHA224_NI n0_ippsHashMethodSet_SHA224_NI +#define ippsHashMethodSet_SHA224_TT n0_ippsHashMethodSet_SHA224_TT +#define ippsHashStateMethodSet_SHA224 n0_ippsHashStateMethodSet_SHA224 +#define ippsHashStateMethodSet_SHA224_NI n0_ippsHashStateMethodSet_SHA224_NI +#define ippsHashStateMethodSet_SHA224_TT n0_ippsHashStateMethodSet_SHA224_TT +#define ippsHashMethodSet_SHA512 n0_ippsHashMethodSet_SHA512 +#define ippsHashMethodSet_SHA384 n0_ippsHashMethodSet_SHA384 +#define ippsHashMethodSet_SHA512_256 n0_ippsHashMethodSet_SHA512_256 +#define ippsHashMethodSet_SHA512_224 n0_ippsHashMethodSet_SHA512_224 +#define ippsHashStateMethodSet_SHA512 n0_ippsHashStateMethodSet_SHA512 +#define ippsHashStateMethodSet_SHA384 n0_ippsHashStateMethodSet_SHA384 +#define ippsHashStateMethodSet_SHA512_256 n0_ippsHashStateMethodSet_SHA512_256 +#define ippsHashStateMethodSet_SHA512_224 n0_ippsHashStateMethodSet_SHA512_224 +#define ippsHashGetSize_rmf n0_ippsHashGetSize_rmf +#define ippsHashInit_rmf n0_ippsHashInit_rmf +#define ippsHashPack_rmf n0_ippsHashPack_rmf +#define ippsHashUnpack_rmf n0_ippsHashUnpack_rmf +#define ippsHashDuplicate_rmf n0_ippsHashDuplicate_rmf +#define ippsHashUpdate_rmf n0_ippsHashUpdate_rmf +#define ippsHashGetTag_rmf n0_ippsHashGetTag_rmf +#define ippsHashFinal_rmf n0_ippsHashFinal_rmf +#define ippsHashMessage_rmf n0_ippsHashMessage_rmf +#define ippsHashMethodGetInfo n0_ippsHashMethodGetInfo +#define ippsHashGetInfo_rmf n0_ippsHashGetInfo_rmf +#define ippsMGF n0_ippsMGF +#define ippsMGF1_rmf n0_ippsMGF1_rmf +#define ippsMGF2_rmf n0_ippsMGF2_rmf +#define ippsHMAC_GetSize n0_ippsHMAC_GetSize +#define ippsHMAC_Init n0_ippsHMAC_Init +#define ippsHMAC_Pack n0_ippsHMAC_Pack +#define ippsHMAC_Unpack n0_ippsHMAC_Unpack +#define ippsHMAC_Duplicate n0_ippsHMAC_Duplicate +#define ippsHMAC_Update n0_ippsHMAC_Update +#define ippsHMAC_Final n0_ippsHMAC_Final +#define ippsHMAC_GetTag n0_ippsHMAC_GetTag +#define ippsHMAC_Message n0_ippsHMAC_Message +#define ippsHMACGetSize_rmf n0_ippsHMACGetSize_rmf +#define ippsHMACInit_rmf n0_ippsHMACInit_rmf +#define ippsHMACPack_rmf n0_ippsHMACPack_rmf +#define ippsHMACUnpack_rmf n0_ippsHMACUnpack_rmf +#define ippsHMACDuplicate_rmf n0_ippsHMACDuplicate_rmf +#define ippsHMACUpdate_rmf n0_ippsHMACUpdate_rmf +#define ippsHMACFinal_rmf n0_ippsHMACFinal_rmf +#define ippsHMACGetTag_rmf n0_ippsHMACGetTag_rmf +#define ippsHMACMessage_rmf n0_ippsHMACMessage_rmf +#define ippsBigNumGetSize n0_ippsBigNumGetSize +#define ippsBigNumInit n0_ippsBigNumInit +#define ippsCmpZero_BN n0_ippsCmpZero_BN +#define ippsCmp_BN n0_ippsCmp_BN +#define ippsGetSize_BN n0_ippsGetSize_BN +#define ippsSet_BN n0_ippsSet_BN +#define ippsGet_BN n0_ippsGet_BN +#define ippsRef_BN n0_ippsRef_BN +#define ippsExtGet_BN n0_ippsExtGet_BN +#define ippsAdd_BN n0_ippsAdd_BN +#define ippsSub_BN n0_ippsSub_BN +#define ippsMul_BN n0_ippsMul_BN +#define ippsMAC_BN_I n0_ippsMAC_BN_I +#define ippsDiv_BN n0_ippsDiv_BN +#define ippsMod_BN n0_ippsMod_BN +#define ippsGcd_BN n0_ippsGcd_BN +#define ippsModInv_BN n0_ippsModInv_BN +#define ippsSetOctString_BN n0_ippsSetOctString_BN +#define ippsGetOctString_BN n0_ippsGetOctString_BN +#define ippsMontGetSize n0_ippsMontGetSize +#define ippsMontInit n0_ippsMontInit +#define ippsMontSet n0_ippsMontSet +#define ippsMontGet n0_ippsMontGet +#define ippsMontForm n0_ippsMontForm +#define ippsMontMul n0_ippsMontMul +#define ippsMontExp n0_ippsMontExp +#define ippsPRNGGetSize n0_ippsPRNGGetSize +#define ippsPRNGInit n0_ippsPRNGInit +#define ippsPRNGSetModulus n0_ippsPRNGSetModulus +#define ippsPRNGSetH0 n0_ippsPRNGSetH0 +#define ippsPRNGSetAugment n0_ippsPRNGSetAugment +#define ippsPRNGSetSeed n0_ippsPRNGSetSeed +#define ippsPRNGGetSeed n0_ippsPRNGGetSeed +#define ippsPRNGen n0_ippsPRNGen +#define ippsPRNGen_BN n0_ippsPRNGen_BN +#define ippsPRNGenRDRAND n0_ippsPRNGenRDRAND +#define ippsPRNGenRDRAND_BN n0_ippsPRNGenRDRAND_BN +#define ippsTRNGenRDSEED n0_ippsTRNGenRDSEED +#define ippsTRNGenRDSEED_BN n0_ippsTRNGenRDSEED_BN +#define ippsPrimeGetSize n0_ippsPrimeGetSize +#define ippsPrimeInit n0_ippsPrimeInit +#define ippsPrimeGen n0_ippsPrimeGen +#define ippsPrimeTest n0_ippsPrimeTest +#define ippsPrimeGen_BN n0_ippsPrimeGen_BN +#define ippsPrimeTest_BN n0_ippsPrimeTest_BN +#define ippsPrimeGet n0_ippsPrimeGet +#define ippsPrimeGet_BN n0_ippsPrimeGet_BN +#define ippsPrimeSet n0_ippsPrimeSet +#define ippsPrimeSet_BN n0_ippsPrimeSet_BN +#define ippsRSA_GetSizePublicKey n0_ippsRSA_GetSizePublicKey +#define ippsRSA_InitPublicKey n0_ippsRSA_InitPublicKey +#define ippsRSA_SetPublicKey n0_ippsRSA_SetPublicKey +#define ippsRSA_GetPublicKey n0_ippsRSA_GetPublicKey +#define ippsRSA_GetSizePrivateKeyType1 n0_ippsRSA_GetSizePrivateKeyType1 +#define ippsRSA_InitPrivateKeyType1 n0_ippsRSA_InitPrivateKeyType1 +#define ippsRSA_SetPrivateKeyType1 n0_ippsRSA_SetPrivateKeyType1 +#define ippsRSA_GetPrivateKeyType1 n0_ippsRSA_GetPrivateKeyType1 +#define ippsRSA_GetSizePrivateKeyType2 n0_ippsRSA_GetSizePrivateKeyType2 +#define ippsRSA_InitPrivateKeyType2 n0_ippsRSA_InitPrivateKeyType2 +#define ippsRSA_SetPrivateKeyType2 n0_ippsRSA_SetPrivateKeyType2 +#define ippsRSA_GetPrivateKeyType2 n0_ippsRSA_GetPrivateKeyType2 +#define ippsRSA_GetBufferSizePublicKey n0_ippsRSA_GetBufferSizePublicKey +#define ippsRSA_GetBufferSizePrivateKey n0_ippsRSA_GetBufferSizePrivateKey +#define ippsRSA_Encrypt n0_ippsRSA_Encrypt +#define ippsRSA_Decrypt n0_ippsRSA_Decrypt +#define ippsRSA_GenerateKeys n0_ippsRSA_GenerateKeys +#define ippsRSA_ValidateKeys n0_ippsRSA_ValidateKeys +#define ippsRSAEncrypt_OAEP n0_ippsRSAEncrypt_OAEP +#define ippsRSADecrypt_OAEP n0_ippsRSADecrypt_OAEP +#define ippsRSAEncrypt_OAEP_rmf n0_ippsRSAEncrypt_OAEP_rmf +#define ippsRSADecrypt_OAEP_rmf n0_ippsRSADecrypt_OAEP_rmf +#define ippsRSAEncrypt_PKCSv15 n0_ippsRSAEncrypt_PKCSv15 +#define ippsRSADecrypt_PKCSv15 n0_ippsRSADecrypt_PKCSv15 +#define ippsRSASign_PSS n0_ippsRSASign_PSS +#define ippsRSAVerify_PSS n0_ippsRSAVerify_PSS +#define ippsRSASign_PSS_rmf n0_ippsRSASign_PSS_rmf +#define ippsRSAVerify_PSS_rmf n0_ippsRSAVerify_PSS_rmf +#define ippsRSASign_PKCS1v15 n0_ippsRSASign_PKCS1v15 +#define ippsRSAVerify_PKCS1v15 n0_ippsRSAVerify_PKCS1v15 +#define ippsRSASign_PKCS1v15_rmf n0_ippsRSASign_PKCS1v15_rmf +#define ippsRSAVerify_PKCS1v15_rmf n0_ippsRSAVerify_PKCS1v15_rmf +#define ippsDLGetResultString n0_ippsDLGetResultString +#define ippsDLPGetSize n0_ippsDLPGetSize +#define ippsDLPInit n0_ippsDLPInit +#define ippsDLPPack n0_ippsDLPPack +#define ippsDLPUnpack n0_ippsDLPUnpack +#define ippsDLPSet n0_ippsDLPSet +#define ippsDLPGet n0_ippsDLPGet +#define ippsDLPSetDP n0_ippsDLPSetDP +#define ippsDLPGetDP n0_ippsDLPGetDP +#define ippsDLPGenKeyPair n0_ippsDLPGenKeyPair +#define ippsDLPPublicKey n0_ippsDLPPublicKey +#define ippsDLPValidateKeyPair n0_ippsDLPValidateKeyPair +#define ippsDLPSetKeyPair n0_ippsDLPSetKeyPair +#define ippsDLPSignDSA n0_ippsDLPSignDSA +#define ippsDLPVerifyDSA n0_ippsDLPVerifyDSA +#define ippsDLPSharedSecretDH n0_ippsDLPSharedSecretDH +#define ippsDLPGenerateDSA n0_ippsDLPGenerateDSA +#define ippsDLPValidateDSA n0_ippsDLPValidateDSA +#define ippsDLPGenerateDH n0_ippsDLPGenerateDH +#define ippsDLPValidateDH n0_ippsDLPValidateDH +#define ippsECCGetResultString n0_ippsECCGetResultString +#define ippsECCPGetSize n0_ippsECCPGetSize +#define ippsECCPGetSizeStd128r1 n0_ippsECCPGetSizeStd128r1 +#define ippsECCPGetSizeStd128r2 n0_ippsECCPGetSizeStd128r2 +#define ippsECCPGetSizeStd192r1 n0_ippsECCPGetSizeStd192r1 +#define ippsECCPGetSizeStd224r1 n0_ippsECCPGetSizeStd224r1 +#define ippsECCPGetSizeStd256r1 n0_ippsECCPGetSizeStd256r1 +#define ippsECCPGetSizeStd384r1 n0_ippsECCPGetSizeStd384r1 +#define ippsECCPGetSizeStd521r1 n0_ippsECCPGetSizeStd521r1 +#define ippsECCPGetSizeStdSM2 n0_ippsECCPGetSizeStdSM2 +#define ippsECCPInit n0_ippsECCPInit +#define ippsECCPInitStd128r1 n0_ippsECCPInitStd128r1 +#define ippsECCPInitStd128r2 n0_ippsECCPInitStd128r2 +#define ippsECCPInitStd192r1 n0_ippsECCPInitStd192r1 +#define ippsECCPInitStd224r1 n0_ippsECCPInitStd224r1 +#define ippsECCPInitStd256r1 n0_ippsECCPInitStd256r1 +#define ippsECCPInitStd384r1 n0_ippsECCPInitStd384r1 +#define ippsECCPInitStd521r1 n0_ippsECCPInitStd521r1 +#define ippsECCPInitStdSM2 n0_ippsECCPInitStdSM2 +#define ippsECCPSet n0_ippsECCPSet +#define ippsECCPSetStd n0_ippsECCPSetStd +#define ippsECCPSetStd128r1 n0_ippsECCPSetStd128r1 +#define ippsECCPSetStd128r2 n0_ippsECCPSetStd128r2 +#define ippsECCPSetStd192r1 n0_ippsECCPSetStd192r1 +#define ippsECCPSetStd224r1 n0_ippsECCPSetStd224r1 +#define ippsECCPSetStd256r1 n0_ippsECCPSetStd256r1 +#define ippsECCPSetStd384r1 n0_ippsECCPSetStd384r1 +#define ippsECCPSetStd521r1 n0_ippsECCPSetStd521r1 +#define ippsECCPSetStdSM2 n0_ippsECCPSetStdSM2 +#define ippsECCPBindGxyTblStd192r1 n0_ippsECCPBindGxyTblStd192r1 +#define ippsECCPBindGxyTblStd224r1 n0_ippsECCPBindGxyTblStd224r1 +#define ippsECCPBindGxyTblStd256r1 n0_ippsECCPBindGxyTblStd256r1 +#define ippsECCPBindGxyTblStd384r1 n0_ippsECCPBindGxyTblStd384r1 +#define ippsECCPBindGxyTblStd521r1 n0_ippsECCPBindGxyTblStd521r1 +#define ippsECCPBindGxyTblStdSM2 n0_ippsECCPBindGxyTblStdSM2 +#define ippsECCPGet n0_ippsECCPGet +#define ippsECCPGetOrderBitSize n0_ippsECCPGetOrderBitSize +#define ippsECCPValidate n0_ippsECCPValidate +#define ippsECCPPointGetSize n0_ippsECCPPointGetSize +#define ippsECCPPointInit n0_ippsECCPPointInit +#define ippsECCPSetPoint n0_ippsECCPSetPoint +#define ippsECCPSetPointAtInfinity n0_ippsECCPSetPointAtInfinity +#define ippsECCPGetPoint n0_ippsECCPGetPoint +#define ippsECCPCheckPoint n0_ippsECCPCheckPoint +#define ippsECCPComparePoint n0_ippsECCPComparePoint +#define ippsECCPNegativePoint n0_ippsECCPNegativePoint +#define ippsECCPAddPoint n0_ippsECCPAddPoint +#define ippsECCPMulPointScalar n0_ippsECCPMulPointScalar +#define ippsECCPGenKeyPair n0_ippsECCPGenKeyPair +#define ippsECCPPublicKey n0_ippsECCPPublicKey +#define ippsECCPValidateKeyPair n0_ippsECCPValidateKeyPair +#define ippsECCPSetKeyPair n0_ippsECCPSetKeyPair +#define ippsECCPSharedSecretDH n0_ippsECCPSharedSecretDH +#define ippsECCPSharedSecretDHC n0_ippsECCPSharedSecretDHC +#define ippsECCPSignDSA n0_ippsECCPSignDSA +#define ippsECCPVerifyDSA n0_ippsECCPVerifyDSA +#define ippsECCPSignNR n0_ippsECCPSignNR +#define ippsECCPVerifyNR n0_ippsECCPVerifyNR +#define ippsECCPSignSM2 n0_ippsECCPSignSM2 +#define ippsECCPVerifySM2 n0_ippsECCPVerifySM2 +#define ippsGFpGetSize n0_ippsGFpGetSize +#define ippsGFpInitArbitrary n0_ippsGFpInitArbitrary +#define ippsGFpInitFixed n0_ippsGFpInitFixed +#define ippsGFpInit n0_ippsGFpInit +#define ippsGFpMethod_p192r1 n0_ippsGFpMethod_p192r1 +#define ippsGFpMethod_p224r1 n0_ippsGFpMethod_p224r1 +#define ippsGFpMethod_p256r1 n0_ippsGFpMethod_p256r1 +#define ippsGFpMethod_p384r1 n0_ippsGFpMethod_p384r1 +#define ippsGFpMethod_p521r1 n0_ippsGFpMethod_p521r1 +#define ippsGFpMethod_p256sm2 n0_ippsGFpMethod_p256sm2 +#define ippsGFpMethod_p256bn n0_ippsGFpMethod_p256bn +#define ippsGFpMethod_p256 n0_ippsGFpMethod_p256 +#define ippsGFpMethod_pArb n0_ippsGFpMethod_pArb +#define ippsGFpxGetSize n0_ippsGFpxGetSize +#define ippsGFpxInit n0_ippsGFpxInit +#define ippsGFpxInitBinomial n0_ippsGFpxInitBinomial +#define ippsGFpxMethod_binom2_epid2 n0_ippsGFpxMethod_binom2_epid2 +#define ippsGFpxMethod_binom3_epid2 n0_ippsGFpxMethod_binom3_epid2 +#define ippsGFpxMethod_binom2 n0_ippsGFpxMethod_binom2 +#define ippsGFpxMethod_binom3 n0_ippsGFpxMethod_binom3 +#define ippsGFpxMethod_binom n0_ippsGFpxMethod_binom +#define ippsGFpxMethod_com n0_ippsGFpxMethod_com +#define ippsGFpScratchBufferSize n0_ippsGFpScratchBufferSize +#define ippsGFpElementGetSize n0_ippsGFpElementGetSize +#define ippsGFpElementInit n0_ippsGFpElementInit +#define ippsGFpSetElement n0_ippsGFpSetElement +#define ippsGFpSetElementRegular n0_ippsGFpSetElementRegular +#define ippsGFpSetElementOctString n0_ippsGFpSetElementOctString +#define ippsGFpSetElementRandom n0_ippsGFpSetElementRandom +#define ippsGFpSetElementHash n0_ippsGFpSetElementHash +#define ippsGFpSetElementHash_rmf n0_ippsGFpSetElementHash_rmf +#define ippsGFpCpyElement n0_ippsGFpCpyElement +#define ippsGFpGetElement n0_ippsGFpGetElement +#define ippsGFpGetElementOctString n0_ippsGFpGetElementOctString +#define ippsGFpCmpElement n0_ippsGFpCmpElement +#define ippsGFpIsZeroElement n0_ippsGFpIsZeroElement +#define ippsGFpIsUnityElement n0_ippsGFpIsUnityElement +#define ippsGFpConj n0_ippsGFpConj +#define ippsGFpNeg n0_ippsGFpNeg +#define ippsGFpInv n0_ippsGFpInv +#define ippsGFpSqrt n0_ippsGFpSqrt +#define ippsGFpSqr n0_ippsGFpSqr +#define ippsGFpAdd n0_ippsGFpAdd +#define ippsGFpSub n0_ippsGFpSub +#define ippsGFpMul n0_ippsGFpMul +#define ippsGFpExp n0_ippsGFpExp +#define ippsGFpMultiExp n0_ippsGFpMultiExp +#define ippsGFpAdd_PE n0_ippsGFpAdd_PE +#define ippsGFpSub_PE n0_ippsGFpSub_PE +#define ippsGFpMul_PE n0_ippsGFpMul_PE +#define ippsGFpGetInfo n0_ippsGFpGetInfo +#define ippsGFpECGetSize n0_ippsGFpECGetSize +#define ippsGFpECInit n0_ippsGFpECInit +#define ippsGFpECSet n0_ippsGFpECSet +#define ippsGFpECSetSubgroup n0_ippsGFpECSetSubgroup +#define ippsGFpECInitStd128r1 n0_ippsGFpECInitStd128r1 +#define ippsGFpECInitStd128r2 n0_ippsGFpECInitStd128r2 +#define ippsGFpECInitStd192r1 n0_ippsGFpECInitStd192r1 +#define ippsGFpECInitStd224r1 n0_ippsGFpECInitStd224r1 +#define ippsGFpECInitStd256r1 n0_ippsGFpECInitStd256r1 +#define ippsGFpECInitStd384r1 n0_ippsGFpECInitStd384r1 +#define ippsGFpECInitStd521r1 n0_ippsGFpECInitStd521r1 +#define ippsGFpECInitStdSM2 n0_ippsGFpECInitStdSM2 +#define ippsGFpECInitStdBN256 n0_ippsGFpECInitStdBN256 +#define ippsGFpECBindGxyTblStd192r1 n0_ippsGFpECBindGxyTblStd192r1 +#define ippsGFpECBindGxyTblStd224r1 n0_ippsGFpECBindGxyTblStd224r1 +#define ippsGFpECBindGxyTblStd256r1 n0_ippsGFpECBindGxyTblStd256r1 +#define ippsGFpECBindGxyTblStd384r1 n0_ippsGFpECBindGxyTblStd384r1 +#define ippsGFpECBindGxyTblStd521r1 n0_ippsGFpECBindGxyTblStd521r1 +#define ippsGFpECBindGxyTblStdSM2 n0_ippsGFpECBindGxyTblStdSM2 +#define ippsGFpECGet n0_ippsGFpECGet +#define ippsGFpECGetSubgroup n0_ippsGFpECGetSubgroup +#define ippsGFpECScratchBufferSize n0_ippsGFpECScratchBufferSize +#define ippsGFpECVerify n0_ippsGFpECVerify +#define ippsGFpECPointGetSize n0_ippsGFpECPointGetSize +#define ippsGFpECPointInit n0_ippsGFpECPointInit +#define ippsGFpECSetPointAtInfinity n0_ippsGFpECSetPointAtInfinity +#define ippsGFpECSetPoint n0_ippsGFpECSetPoint +#define ippsGFpECSetPointRegular n0_ippsGFpECSetPointRegular +#define ippsGFpECSetPointRandom n0_ippsGFpECSetPointRandom +#define ippsGFpECMakePoint n0_ippsGFpECMakePoint +#define ippsGFpECSetPointHash n0_ippsGFpECSetPointHash +#define ippsGFpECSetPointHashBackCompatible n0_ippsGFpECSetPointHashBackCompatible +#define ippsGFpECSetPointHash_rmf n0_ippsGFpECSetPointHash_rmf +#define ippsGFpECSetPointHashBackCompatible_rmf n0_ippsGFpECSetPointHashBackCompatible_rmf +#define ippsGFpECGetPoint n0_ippsGFpECGetPoint +#define ippsGFpECGetPointRegular n0_ippsGFpECGetPointRegular +#define ippsGFpECSetPointOctString n0_ippsGFpECSetPointOctString +#define ippsGFpECGetPointOctString n0_ippsGFpECGetPointOctString +#define ippsGFpECTstPoint n0_ippsGFpECTstPoint +#define ippsGFpECTstPointInSubgroup n0_ippsGFpECTstPointInSubgroup +#define ippsGFpECCpyPoint n0_ippsGFpECCpyPoint +#define ippsGFpECCmpPoint n0_ippsGFpECCmpPoint +#define ippsGFpECNegPoint n0_ippsGFpECNegPoint +#define ippsGFpECAddPoint n0_ippsGFpECAddPoint +#define ippsGFpECMulPoint n0_ippsGFpECMulPoint +#define ippsGFpECPrivateKey n0_ippsGFpECPrivateKey +#define ippsGFpECPublicKey n0_ippsGFpECPublicKey +#define ippsGFpECTstKeyPair n0_ippsGFpECTstKeyPair +#define ippsGFpECSharedSecretDH n0_ippsGFpECSharedSecretDH +#define ippsGFpECSharedSecretDHC n0_ippsGFpECSharedSecretDHC +#define ippsGFpECMessageRepresentationSM2 n0_ippsGFpECMessageRepresentationSM2 +#define ippsGFpECSignDSA n0_ippsGFpECSignDSA +#define ippsGFpECVerifyDSA n0_ippsGFpECVerifyDSA +#define ippsGFpECSignNR n0_ippsGFpECSignNR +#define ippsGFpECVerifyNR n0_ippsGFpECVerifyNR +#define ippsGFpECSignSM2 n0_ippsGFpECSignSM2 +#define ippsGFpECVerifySM2 n0_ippsGFpECVerifySM2 +#define ippsGFpECUserIDHashSM2 n0_ippsGFpECUserIDHashSM2 +#define ippsGFpECKeyExchangeSM2_GetSize n0_ippsGFpECKeyExchangeSM2_GetSize +#define ippsGFpECKeyExchangeSM2_Init n0_ippsGFpECKeyExchangeSM2_Init +#define ippsGFpECKeyExchangeSM2_Setup n0_ippsGFpECKeyExchangeSM2_Setup +#define ippsGFpECKeyExchangeSM2_SharedKey n0_ippsGFpECKeyExchangeSM2_SharedKey +#define ippsGFpECKeyExchangeSM2_Confirm n0_ippsGFpECKeyExchangeSM2_Confirm +#define ippsGFpECGetInfo_GF n0_ippsGFpECGetInfo_GF +#define ippsGFpECESGetSize_SM2 n0_ippsGFpECESGetSize_SM2 +#define ippsGFpECESInit_SM2 n0_ippsGFpECESInit_SM2 +#define ippsGFpECESSetKey_SM2 n0_ippsGFpECESSetKey_SM2 +#define ippsGFpECESStart_SM2 n0_ippsGFpECESStart_SM2 +#define ippsGFpECESEncrypt_SM2 n0_ippsGFpECESEncrypt_SM2 +#define ippsGFpECESDecrypt_SM2 n0_ippsGFpECESDecrypt_SM2 +#define ippsGFpECESFinal_SM2 n0_ippsGFpECESFinal_SM2 +#define ippsGFpECESGetBuffersSize_SM2 n0_ippsGFpECESGetBuffersSize_SM2 +#define ippsGFpECEncryptSM2_Ext_EncMsgSize n0_ippsGFpECEncryptSM2_Ext_EncMsgSize +#define ippsGFpECEncryptSM2_Ext n0_ippsGFpECEncryptSM2_Ext +#define ippsGFpECDecryptSM2_Ext_DecMsgSize n0_ippsGFpECDecryptSM2_Ext_DecMsgSize +#define ippsGFpECDecryptSM2_Ext n0_ippsGFpECDecryptSM2_Ext diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_n8.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_n8.h new file mode 100644 index 0000000..b036560 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_n8.h @@ -0,0 +1,557 @@ +/******************************************************************************* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + + #define ippcpGetLibVersion n8_ippcpGetLibVersion +#define ippsDESGetSize n8_ippsDESGetSize +#define ippsDESInit n8_ippsDESInit +#define ippsDESPack n8_ippsDESPack +#define ippsDESUnpack n8_ippsDESUnpack +#define ippsTDESEncryptECB n8_ippsTDESEncryptECB +#define ippsTDESDecryptECB n8_ippsTDESDecryptECB +#define ippsTDESEncryptCBC n8_ippsTDESEncryptCBC +#define ippsTDESDecryptCBC n8_ippsTDESDecryptCBC +#define ippsTDESEncryptCFB n8_ippsTDESEncryptCFB +#define ippsTDESDecryptCFB n8_ippsTDESDecryptCFB +#define ippsTDESEncryptOFB n8_ippsTDESEncryptOFB +#define ippsTDESDecryptOFB n8_ippsTDESDecryptOFB +#define ippsTDESEncryptCTR n8_ippsTDESEncryptCTR +#define ippsTDESDecryptCTR n8_ippsTDESDecryptCTR +#define ippsAESGetSize n8_ippsAESGetSize +#define ippsAESInit n8_ippsAESInit +#define ippsAESSetKey n8_ippsAESSetKey +#define ippsAESPack n8_ippsAESPack +#define ippsAESUnpack n8_ippsAESUnpack +#define ippsAESEncryptECB n8_ippsAESEncryptECB +#define ippsAESDecryptECB n8_ippsAESDecryptECB +#define ippsAESEncryptCBC n8_ippsAESEncryptCBC +#define ippsAESEncryptCBC_CS1 n8_ippsAESEncryptCBC_CS1 +#define ippsAESEncryptCBC_CS2 n8_ippsAESEncryptCBC_CS2 +#define ippsAESEncryptCBC_CS3 n8_ippsAESEncryptCBC_CS3 +#define ippsAESDecryptCBC n8_ippsAESDecryptCBC +#define ippsAESDecryptCBC_CS1 n8_ippsAESDecryptCBC_CS1 +#define ippsAESDecryptCBC_CS2 n8_ippsAESDecryptCBC_CS2 +#define ippsAESDecryptCBC_CS3 n8_ippsAESDecryptCBC_CS3 +#define ippsAESEncryptCFB n8_ippsAESEncryptCFB +#define ippsAESDecryptCFB n8_ippsAESDecryptCFB +#define ippsAESEncryptOFB n8_ippsAESEncryptOFB +#define ippsAESDecryptOFB n8_ippsAESDecryptOFB +#define ippsAESEncryptCTR n8_ippsAESEncryptCTR +#define ippsAESDecryptCTR n8_ippsAESDecryptCTR +#define ippsAESEncryptXTS_Direct n8_ippsAESEncryptXTS_Direct +#define ippsAESDecryptXTS_Direct n8_ippsAESDecryptXTS_Direct +#define ippsAESSetupNoise n8_ippsAESSetupNoise +#define ippsAES_GCMSetupNoise n8_ippsAES_GCMSetupNoise +#define ippsAES_CMACSetupNoise n8_ippsAES_CMACSetupNoise +#define ippsAES_EncryptCFB16_MB n8_ippsAES_EncryptCFB16_MB +#define ippsSMS4GetSize n8_ippsSMS4GetSize +#define ippsSMS4Init n8_ippsSMS4Init +#define ippsSMS4SetKey n8_ippsSMS4SetKey +#define ippsSMS4EncryptECB n8_ippsSMS4EncryptECB +#define ippsSMS4DecryptECB n8_ippsSMS4DecryptECB +#define ippsSMS4EncryptCBC n8_ippsSMS4EncryptCBC +#define ippsSMS4EncryptCBC_CS1 n8_ippsSMS4EncryptCBC_CS1 +#define ippsSMS4EncryptCBC_CS2 n8_ippsSMS4EncryptCBC_CS2 +#define ippsSMS4EncryptCBC_CS3 n8_ippsSMS4EncryptCBC_CS3 +#define ippsSMS4DecryptCBC n8_ippsSMS4DecryptCBC +#define ippsSMS4DecryptCBC_CS1 n8_ippsSMS4DecryptCBC_CS1 +#define ippsSMS4DecryptCBC_CS2 n8_ippsSMS4DecryptCBC_CS2 +#define ippsSMS4DecryptCBC_CS3 n8_ippsSMS4DecryptCBC_CS3 +#define ippsSMS4EncryptCFB n8_ippsSMS4EncryptCFB +#define ippsSMS4DecryptCFB n8_ippsSMS4DecryptCFB +#define ippsSMS4EncryptOFB n8_ippsSMS4EncryptOFB +#define ippsSMS4DecryptOFB n8_ippsSMS4DecryptOFB +#define ippsSMS4EncryptCTR n8_ippsSMS4EncryptCTR +#define ippsSMS4DecryptCTR n8_ippsSMS4DecryptCTR +#define ippsSMS4_CCMGetSize n8_ippsSMS4_CCMGetSize +#define ippsSMS4_CCMInit n8_ippsSMS4_CCMInit +#define ippsSMS4_CCMMessageLen n8_ippsSMS4_CCMMessageLen +#define ippsSMS4_CCMTagLen n8_ippsSMS4_CCMTagLen +#define ippsSMS4_CCMStart n8_ippsSMS4_CCMStart +#define ippsSMS4_CCMEncrypt n8_ippsSMS4_CCMEncrypt +#define ippsSMS4_CCMDecrypt n8_ippsSMS4_CCMDecrypt +#define ippsSMS4_CCMGetTag n8_ippsSMS4_CCMGetTag +#define ippsAES_CCMGetSize n8_ippsAES_CCMGetSize +#define ippsAES_CCMInit n8_ippsAES_CCMInit +#define ippsAES_CCMMessageLen n8_ippsAES_CCMMessageLen +#define ippsAES_CCMTagLen n8_ippsAES_CCMTagLen +#define ippsAES_CCMStart n8_ippsAES_CCMStart +#define ippsAES_CCMEncrypt n8_ippsAES_CCMEncrypt +#define ippsAES_CCMDecrypt n8_ippsAES_CCMDecrypt +#define ippsAES_CCMGetTag n8_ippsAES_CCMGetTag +#define ippsAES_GCMGetSize n8_ippsAES_GCMGetSize +#define ippsAES_GCMInit n8_ippsAES_GCMInit +#define ippsAES_GCMReinit n8_ippsAES_GCMReinit +#define ippsAES_GCMReset n8_ippsAES_GCMReset +#define ippsAES_GCMProcessIV n8_ippsAES_GCMProcessIV +#define ippsAES_GCMProcessAAD n8_ippsAES_GCMProcessAAD +#define ippsAES_GCMStart n8_ippsAES_GCMStart +#define ippsAES_GCMEncrypt n8_ippsAES_GCMEncrypt +#define ippsAES_GCMDecrypt n8_ippsAES_GCMDecrypt +#define ippsAES_GCMGetTag n8_ippsAES_GCMGetTag +#define ippsAES_XTSGetSize n8_ippsAES_XTSGetSize +#define ippsAES_XTSInit n8_ippsAES_XTSInit +#define ippsAES_XTSEncrypt n8_ippsAES_XTSEncrypt +#define ippsAES_XTSDecrypt n8_ippsAES_XTSDecrypt +#define ippsAES_S2V_CMAC n8_ippsAES_S2V_CMAC +#define ippsAES_SIVEncrypt n8_ippsAES_SIVEncrypt +#define ippsAES_SIVDecrypt n8_ippsAES_SIVDecrypt +#define ippsAES_CMACGetSize n8_ippsAES_CMACGetSize +#define ippsAES_CMACInit n8_ippsAES_CMACInit +#define ippsAES_CMACUpdate n8_ippsAES_CMACUpdate +#define ippsAES_CMACFinal n8_ippsAES_CMACFinal +#define ippsAES_CMACGetTag n8_ippsAES_CMACGetTag +#define ippsARCFourCheckKey n8_ippsARCFourCheckKey +#define ippsARCFourGetSize n8_ippsARCFourGetSize +#define ippsARCFourInit n8_ippsARCFourInit +#define ippsARCFourReset n8_ippsARCFourReset +#define ippsARCFourPack n8_ippsARCFourPack +#define ippsARCFourUnpack n8_ippsARCFourUnpack +#define ippsARCFourEncrypt n8_ippsARCFourEncrypt +#define ippsARCFourDecrypt n8_ippsARCFourDecrypt +#define ippsSHA1GetSize n8_ippsSHA1GetSize +#define ippsSHA1Init n8_ippsSHA1Init +#define ippsSHA1Duplicate n8_ippsSHA1Duplicate +#define ippsSHA1Pack n8_ippsSHA1Pack +#define ippsSHA1Unpack n8_ippsSHA1Unpack +#define ippsSHA1Update n8_ippsSHA1Update +#define ippsSHA1GetTag n8_ippsSHA1GetTag +#define ippsSHA1Final n8_ippsSHA1Final +#define ippsSHA1MessageDigest n8_ippsSHA1MessageDigest +#define ippsSHA224GetSize n8_ippsSHA224GetSize +#define ippsSHA224Init n8_ippsSHA224Init +#define ippsSHA224Duplicate n8_ippsSHA224Duplicate +#define ippsSHA224Pack n8_ippsSHA224Pack +#define ippsSHA224Unpack n8_ippsSHA224Unpack +#define ippsSHA224Update n8_ippsSHA224Update +#define ippsSHA224GetTag n8_ippsSHA224GetTag +#define ippsSHA224Final n8_ippsSHA224Final +#define ippsSHA224MessageDigest n8_ippsSHA224MessageDigest +#define ippsSHA256GetSize n8_ippsSHA256GetSize +#define ippsSHA256Init n8_ippsSHA256Init +#define ippsSHA256Duplicate n8_ippsSHA256Duplicate +#define ippsSHA256Pack n8_ippsSHA256Pack +#define ippsSHA256Unpack n8_ippsSHA256Unpack +#define ippsSHA256Update n8_ippsSHA256Update +#define ippsSHA256GetTag n8_ippsSHA256GetTag +#define ippsSHA256Final n8_ippsSHA256Final +#define ippsSHA256MessageDigest n8_ippsSHA256MessageDigest +#define ippsSHA384GetSize n8_ippsSHA384GetSize +#define ippsSHA384Init n8_ippsSHA384Init +#define ippsSHA384Duplicate n8_ippsSHA384Duplicate +#define ippsSHA384Pack n8_ippsSHA384Pack +#define ippsSHA384Unpack n8_ippsSHA384Unpack +#define ippsSHA384Update n8_ippsSHA384Update +#define ippsSHA384GetTag n8_ippsSHA384GetTag +#define ippsSHA384Final n8_ippsSHA384Final +#define ippsSHA384MessageDigest n8_ippsSHA384MessageDigest +#define ippsSHA512GetSize n8_ippsSHA512GetSize +#define ippsSHA512Init n8_ippsSHA512Init +#define ippsSHA512Duplicate n8_ippsSHA512Duplicate +#define ippsSHA512Pack n8_ippsSHA512Pack +#define ippsSHA512Unpack n8_ippsSHA512Unpack +#define ippsSHA512Update n8_ippsSHA512Update +#define ippsSHA512GetTag n8_ippsSHA512GetTag +#define ippsSHA512Final n8_ippsSHA512Final +#define ippsSHA512MessageDigest n8_ippsSHA512MessageDigest +#define ippsMD5GetSize n8_ippsMD5GetSize +#define ippsMD5Init n8_ippsMD5Init +#define ippsMD5Duplicate n8_ippsMD5Duplicate +#define ippsMD5Pack n8_ippsMD5Pack +#define ippsMD5Unpack n8_ippsMD5Unpack +#define ippsMD5Update n8_ippsMD5Update +#define ippsMD5GetTag n8_ippsMD5GetTag +#define ippsMD5Final n8_ippsMD5Final +#define ippsMD5MessageDigest n8_ippsMD5MessageDigest +#define ippsSM3GetSize n8_ippsSM3GetSize +#define ippsSM3Init n8_ippsSM3Init +#define ippsSM3Duplicate n8_ippsSM3Duplicate +#define ippsSM3Pack n8_ippsSM3Pack +#define ippsSM3Unpack n8_ippsSM3Unpack +#define ippsSM3Update n8_ippsSM3Update +#define ippsSM3GetTag n8_ippsSM3GetTag +#define ippsSM3Final n8_ippsSM3Final +#define ippsSM3MessageDigest n8_ippsSM3MessageDigest +#define ippsHashGetSize n8_ippsHashGetSize +#define ippsHashInit n8_ippsHashInit +#define ippsHashPack n8_ippsHashPack +#define ippsHashUnpack n8_ippsHashUnpack +#define ippsHashDuplicate n8_ippsHashDuplicate +#define ippsHashUpdate n8_ippsHashUpdate +#define ippsHashGetTag n8_ippsHashGetTag +#define ippsHashFinal n8_ippsHashFinal +#define ippsHashMessage n8_ippsHashMessage +#define ippsHashMethod_MD5 n8_ippsHashMethod_MD5 +#define ippsHashMethod_SM3 n8_ippsHashMethod_SM3 +#define ippsHashMethod_SHA1 n8_ippsHashMethod_SHA1 +#define ippsHashMethod_SHA1_NI n8_ippsHashMethod_SHA1_NI +#define ippsHashMethod_SHA1_TT n8_ippsHashMethod_SHA1_TT +#define ippsHashMethod_SHA256 n8_ippsHashMethod_SHA256 +#define ippsHashMethod_SHA256_NI n8_ippsHashMethod_SHA256_NI +#define ippsHashMethod_SHA256_TT n8_ippsHashMethod_SHA256_TT +#define ippsHashMethod_SHA224 n8_ippsHashMethod_SHA224 +#define ippsHashMethod_SHA224_NI n8_ippsHashMethod_SHA224_NI +#define ippsHashMethod_SHA224_TT n8_ippsHashMethod_SHA224_TT +#define ippsHashMethod_SHA512 n8_ippsHashMethod_SHA512 +#define ippsHashMethod_SHA384 n8_ippsHashMethod_SHA384 +#define ippsHashMethod_SHA512_256 n8_ippsHashMethod_SHA512_256 +#define ippsHashMethod_SHA512_224 n8_ippsHashMethod_SHA512_224 +#define ippsHashMethodGetSize n8_ippsHashMethodGetSize +#define ippsHashMethodSet_MD5 n8_ippsHashMethodSet_MD5 +#define ippsHashMethodSet_SM3 n8_ippsHashMethodSet_SM3 +#define ippsHashStateMethodSet_SM3 n8_ippsHashStateMethodSet_SM3 +#define ippsHashMethodSet_SHA1 n8_ippsHashMethodSet_SHA1 +#define ippsHashMethodSet_SHA1_NI n8_ippsHashMethodSet_SHA1_NI +#define ippsHashMethodSet_SHA1_TT n8_ippsHashMethodSet_SHA1_TT +#define ippsHashMethodSet_SHA256 n8_ippsHashMethodSet_SHA256 +#define ippsHashMethodSet_SHA256_NI n8_ippsHashMethodSet_SHA256_NI +#define ippsHashMethodSet_SHA256_TT n8_ippsHashMethodSet_SHA256_TT +#define ippsHashStateMethodSet_SHA256 n8_ippsHashStateMethodSet_SHA256 +#define ippsHashStateMethodSet_SHA256_NI n8_ippsHashStateMethodSet_SHA256_NI +#define ippsHashStateMethodSet_SHA256_TT n8_ippsHashStateMethodSet_SHA256_TT +#define ippsHashMethodSet_SHA224 n8_ippsHashMethodSet_SHA224 +#define ippsHashMethodSet_SHA224_NI n8_ippsHashMethodSet_SHA224_NI +#define ippsHashMethodSet_SHA224_TT n8_ippsHashMethodSet_SHA224_TT +#define ippsHashStateMethodSet_SHA224 n8_ippsHashStateMethodSet_SHA224 +#define ippsHashStateMethodSet_SHA224_NI n8_ippsHashStateMethodSet_SHA224_NI +#define ippsHashStateMethodSet_SHA224_TT n8_ippsHashStateMethodSet_SHA224_TT +#define ippsHashMethodSet_SHA512 n8_ippsHashMethodSet_SHA512 +#define ippsHashMethodSet_SHA384 n8_ippsHashMethodSet_SHA384 +#define ippsHashMethodSet_SHA512_256 n8_ippsHashMethodSet_SHA512_256 +#define ippsHashMethodSet_SHA512_224 n8_ippsHashMethodSet_SHA512_224 +#define ippsHashStateMethodSet_SHA512 n8_ippsHashStateMethodSet_SHA512 +#define ippsHashStateMethodSet_SHA384 n8_ippsHashStateMethodSet_SHA384 +#define ippsHashStateMethodSet_SHA512_256 n8_ippsHashStateMethodSet_SHA512_256 +#define ippsHashStateMethodSet_SHA512_224 n8_ippsHashStateMethodSet_SHA512_224 +#define ippsHashGetSize_rmf n8_ippsHashGetSize_rmf +#define ippsHashInit_rmf n8_ippsHashInit_rmf +#define ippsHashPack_rmf n8_ippsHashPack_rmf +#define ippsHashUnpack_rmf n8_ippsHashUnpack_rmf +#define ippsHashDuplicate_rmf n8_ippsHashDuplicate_rmf +#define ippsHashUpdate_rmf n8_ippsHashUpdate_rmf +#define ippsHashGetTag_rmf n8_ippsHashGetTag_rmf +#define ippsHashFinal_rmf n8_ippsHashFinal_rmf +#define ippsHashMessage_rmf n8_ippsHashMessage_rmf +#define ippsHashMethodGetInfo n8_ippsHashMethodGetInfo +#define ippsHashGetInfo_rmf n8_ippsHashGetInfo_rmf +#define ippsMGF n8_ippsMGF +#define ippsMGF1_rmf n8_ippsMGF1_rmf +#define ippsMGF2_rmf n8_ippsMGF2_rmf +#define ippsHMAC_GetSize n8_ippsHMAC_GetSize +#define ippsHMAC_Init n8_ippsHMAC_Init +#define ippsHMAC_Pack n8_ippsHMAC_Pack +#define ippsHMAC_Unpack n8_ippsHMAC_Unpack +#define ippsHMAC_Duplicate n8_ippsHMAC_Duplicate +#define ippsHMAC_Update n8_ippsHMAC_Update +#define ippsHMAC_Final n8_ippsHMAC_Final +#define ippsHMAC_GetTag n8_ippsHMAC_GetTag +#define ippsHMAC_Message n8_ippsHMAC_Message +#define ippsHMACGetSize_rmf n8_ippsHMACGetSize_rmf +#define ippsHMACInit_rmf n8_ippsHMACInit_rmf +#define ippsHMACPack_rmf n8_ippsHMACPack_rmf +#define ippsHMACUnpack_rmf n8_ippsHMACUnpack_rmf +#define ippsHMACDuplicate_rmf n8_ippsHMACDuplicate_rmf +#define ippsHMACUpdate_rmf n8_ippsHMACUpdate_rmf +#define ippsHMACFinal_rmf n8_ippsHMACFinal_rmf +#define ippsHMACGetTag_rmf n8_ippsHMACGetTag_rmf +#define ippsHMACMessage_rmf n8_ippsHMACMessage_rmf +#define ippsBigNumGetSize n8_ippsBigNumGetSize +#define ippsBigNumInit n8_ippsBigNumInit +#define ippsCmpZero_BN n8_ippsCmpZero_BN +#define ippsCmp_BN n8_ippsCmp_BN +#define ippsGetSize_BN n8_ippsGetSize_BN +#define ippsSet_BN n8_ippsSet_BN +#define ippsGet_BN n8_ippsGet_BN +#define ippsRef_BN n8_ippsRef_BN +#define ippsExtGet_BN n8_ippsExtGet_BN +#define ippsAdd_BN n8_ippsAdd_BN +#define ippsSub_BN n8_ippsSub_BN +#define ippsMul_BN n8_ippsMul_BN +#define ippsMAC_BN_I n8_ippsMAC_BN_I +#define ippsDiv_BN n8_ippsDiv_BN +#define ippsMod_BN n8_ippsMod_BN +#define ippsGcd_BN n8_ippsGcd_BN +#define ippsModInv_BN n8_ippsModInv_BN +#define ippsSetOctString_BN n8_ippsSetOctString_BN +#define ippsGetOctString_BN n8_ippsGetOctString_BN +#define ippsMontGetSize n8_ippsMontGetSize +#define ippsMontInit n8_ippsMontInit +#define ippsMontSet n8_ippsMontSet +#define ippsMontGet n8_ippsMontGet +#define ippsMontForm n8_ippsMontForm +#define ippsMontMul n8_ippsMontMul +#define ippsMontExp n8_ippsMontExp +#define ippsPRNGGetSize n8_ippsPRNGGetSize +#define ippsPRNGInit n8_ippsPRNGInit +#define ippsPRNGSetModulus n8_ippsPRNGSetModulus +#define ippsPRNGSetH0 n8_ippsPRNGSetH0 +#define ippsPRNGSetAugment n8_ippsPRNGSetAugment +#define ippsPRNGSetSeed n8_ippsPRNGSetSeed +#define ippsPRNGGetSeed n8_ippsPRNGGetSeed +#define ippsPRNGen n8_ippsPRNGen +#define ippsPRNGen_BN n8_ippsPRNGen_BN +#define ippsPRNGenRDRAND n8_ippsPRNGenRDRAND +#define ippsPRNGenRDRAND_BN n8_ippsPRNGenRDRAND_BN +#define ippsTRNGenRDSEED n8_ippsTRNGenRDSEED +#define ippsTRNGenRDSEED_BN n8_ippsTRNGenRDSEED_BN +#define ippsPrimeGetSize n8_ippsPrimeGetSize +#define ippsPrimeInit n8_ippsPrimeInit +#define ippsPrimeGen n8_ippsPrimeGen +#define ippsPrimeTest n8_ippsPrimeTest +#define ippsPrimeGen_BN n8_ippsPrimeGen_BN +#define ippsPrimeTest_BN n8_ippsPrimeTest_BN +#define ippsPrimeGet n8_ippsPrimeGet +#define ippsPrimeGet_BN n8_ippsPrimeGet_BN +#define ippsPrimeSet n8_ippsPrimeSet +#define ippsPrimeSet_BN n8_ippsPrimeSet_BN +#define ippsRSA_GetSizePublicKey n8_ippsRSA_GetSizePublicKey +#define ippsRSA_InitPublicKey n8_ippsRSA_InitPublicKey +#define ippsRSA_SetPublicKey n8_ippsRSA_SetPublicKey +#define ippsRSA_GetPublicKey n8_ippsRSA_GetPublicKey +#define ippsRSA_GetSizePrivateKeyType1 n8_ippsRSA_GetSizePrivateKeyType1 +#define ippsRSA_InitPrivateKeyType1 n8_ippsRSA_InitPrivateKeyType1 +#define ippsRSA_SetPrivateKeyType1 n8_ippsRSA_SetPrivateKeyType1 +#define ippsRSA_GetPrivateKeyType1 n8_ippsRSA_GetPrivateKeyType1 +#define ippsRSA_GetSizePrivateKeyType2 n8_ippsRSA_GetSizePrivateKeyType2 +#define ippsRSA_InitPrivateKeyType2 n8_ippsRSA_InitPrivateKeyType2 +#define ippsRSA_SetPrivateKeyType2 n8_ippsRSA_SetPrivateKeyType2 +#define ippsRSA_GetPrivateKeyType2 n8_ippsRSA_GetPrivateKeyType2 +#define ippsRSA_GetBufferSizePublicKey n8_ippsRSA_GetBufferSizePublicKey +#define ippsRSA_GetBufferSizePrivateKey n8_ippsRSA_GetBufferSizePrivateKey +#define ippsRSA_Encrypt n8_ippsRSA_Encrypt +#define ippsRSA_Decrypt n8_ippsRSA_Decrypt +#define ippsRSA_GenerateKeys n8_ippsRSA_GenerateKeys +#define ippsRSA_ValidateKeys n8_ippsRSA_ValidateKeys +#define ippsRSAEncrypt_OAEP n8_ippsRSAEncrypt_OAEP +#define ippsRSADecrypt_OAEP n8_ippsRSADecrypt_OAEP +#define ippsRSAEncrypt_OAEP_rmf n8_ippsRSAEncrypt_OAEP_rmf +#define ippsRSADecrypt_OAEP_rmf n8_ippsRSADecrypt_OAEP_rmf +#define ippsRSAEncrypt_PKCSv15 n8_ippsRSAEncrypt_PKCSv15 +#define ippsRSADecrypt_PKCSv15 n8_ippsRSADecrypt_PKCSv15 +#define ippsRSASign_PSS n8_ippsRSASign_PSS +#define ippsRSAVerify_PSS n8_ippsRSAVerify_PSS +#define ippsRSASign_PSS_rmf n8_ippsRSASign_PSS_rmf +#define ippsRSAVerify_PSS_rmf n8_ippsRSAVerify_PSS_rmf +#define ippsRSASign_PKCS1v15 n8_ippsRSASign_PKCS1v15 +#define ippsRSAVerify_PKCS1v15 n8_ippsRSAVerify_PKCS1v15 +#define ippsRSASign_PKCS1v15_rmf n8_ippsRSASign_PKCS1v15_rmf +#define ippsRSAVerify_PKCS1v15_rmf n8_ippsRSAVerify_PKCS1v15_rmf +#define ippsDLGetResultString n8_ippsDLGetResultString +#define ippsDLPGetSize n8_ippsDLPGetSize +#define ippsDLPInit n8_ippsDLPInit +#define ippsDLPPack n8_ippsDLPPack +#define ippsDLPUnpack n8_ippsDLPUnpack +#define ippsDLPSet n8_ippsDLPSet +#define ippsDLPGet n8_ippsDLPGet +#define ippsDLPSetDP n8_ippsDLPSetDP +#define ippsDLPGetDP n8_ippsDLPGetDP +#define ippsDLPGenKeyPair n8_ippsDLPGenKeyPair +#define ippsDLPPublicKey n8_ippsDLPPublicKey +#define ippsDLPValidateKeyPair n8_ippsDLPValidateKeyPair +#define ippsDLPSetKeyPair n8_ippsDLPSetKeyPair +#define ippsDLPSignDSA n8_ippsDLPSignDSA +#define ippsDLPVerifyDSA n8_ippsDLPVerifyDSA +#define ippsDLPSharedSecretDH n8_ippsDLPSharedSecretDH +#define ippsDLPGenerateDSA n8_ippsDLPGenerateDSA +#define ippsDLPValidateDSA n8_ippsDLPValidateDSA +#define ippsDLPGenerateDH n8_ippsDLPGenerateDH +#define ippsDLPValidateDH n8_ippsDLPValidateDH +#define ippsECCGetResultString n8_ippsECCGetResultString +#define ippsECCPGetSize n8_ippsECCPGetSize +#define ippsECCPGetSizeStd128r1 n8_ippsECCPGetSizeStd128r1 +#define ippsECCPGetSizeStd128r2 n8_ippsECCPGetSizeStd128r2 +#define ippsECCPGetSizeStd192r1 n8_ippsECCPGetSizeStd192r1 +#define ippsECCPGetSizeStd224r1 n8_ippsECCPGetSizeStd224r1 +#define ippsECCPGetSizeStd256r1 n8_ippsECCPGetSizeStd256r1 +#define ippsECCPGetSizeStd384r1 n8_ippsECCPGetSizeStd384r1 +#define ippsECCPGetSizeStd521r1 n8_ippsECCPGetSizeStd521r1 +#define ippsECCPGetSizeStdSM2 n8_ippsECCPGetSizeStdSM2 +#define ippsECCPInit n8_ippsECCPInit +#define ippsECCPInitStd128r1 n8_ippsECCPInitStd128r1 +#define ippsECCPInitStd128r2 n8_ippsECCPInitStd128r2 +#define ippsECCPInitStd192r1 n8_ippsECCPInitStd192r1 +#define ippsECCPInitStd224r1 n8_ippsECCPInitStd224r1 +#define ippsECCPInitStd256r1 n8_ippsECCPInitStd256r1 +#define ippsECCPInitStd384r1 n8_ippsECCPInitStd384r1 +#define ippsECCPInitStd521r1 n8_ippsECCPInitStd521r1 +#define ippsECCPInitStdSM2 n8_ippsECCPInitStdSM2 +#define ippsECCPSet n8_ippsECCPSet +#define ippsECCPSetStd n8_ippsECCPSetStd +#define ippsECCPSetStd128r1 n8_ippsECCPSetStd128r1 +#define ippsECCPSetStd128r2 n8_ippsECCPSetStd128r2 +#define ippsECCPSetStd192r1 n8_ippsECCPSetStd192r1 +#define ippsECCPSetStd224r1 n8_ippsECCPSetStd224r1 +#define ippsECCPSetStd256r1 n8_ippsECCPSetStd256r1 +#define ippsECCPSetStd384r1 n8_ippsECCPSetStd384r1 +#define ippsECCPSetStd521r1 n8_ippsECCPSetStd521r1 +#define ippsECCPSetStdSM2 n8_ippsECCPSetStdSM2 +#define ippsECCPBindGxyTblStd192r1 n8_ippsECCPBindGxyTblStd192r1 +#define ippsECCPBindGxyTblStd224r1 n8_ippsECCPBindGxyTblStd224r1 +#define ippsECCPBindGxyTblStd256r1 n8_ippsECCPBindGxyTblStd256r1 +#define ippsECCPBindGxyTblStd384r1 n8_ippsECCPBindGxyTblStd384r1 +#define ippsECCPBindGxyTblStd521r1 n8_ippsECCPBindGxyTblStd521r1 +#define ippsECCPBindGxyTblStdSM2 n8_ippsECCPBindGxyTblStdSM2 +#define ippsECCPGet n8_ippsECCPGet +#define ippsECCPGetOrderBitSize n8_ippsECCPGetOrderBitSize +#define ippsECCPValidate n8_ippsECCPValidate +#define ippsECCPPointGetSize n8_ippsECCPPointGetSize +#define ippsECCPPointInit n8_ippsECCPPointInit +#define ippsECCPSetPoint n8_ippsECCPSetPoint +#define ippsECCPSetPointAtInfinity n8_ippsECCPSetPointAtInfinity +#define ippsECCPGetPoint n8_ippsECCPGetPoint +#define ippsECCPCheckPoint n8_ippsECCPCheckPoint +#define ippsECCPComparePoint n8_ippsECCPComparePoint +#define ippsECCPNegativePoint n8_ippsECCPNegativePoint +#define ippsECCPAddPoint n8_ippsECCPAddPoint +#define ippsECCPMulPointScalar n8_ippsECCPMulPointScalar +#define ippsECCPGenKeyPair n8_ippsECCPGenKeyPair +#define ippsECCPPublicKey n8_ippsECCPPublicKey +#define ippsECCPValidateKeyPair n8_ippsECCPValidateKeyPair +#define ippsECCPSetKeyPair n8_ippsECCPSetKeyPair +#define ippsECCPSharedSecretDH n8_ippsECCPSharedSecretDH +#define ippsECCPSharedSecretDHC n8_ippsECCPSharedSecretDHC +#define ippsECCPSignDSA n8_ippsECCPSignDSA +#define ippsECCPVerifyDSA n8_ippsECCPVerifyDSA +#define ippsECCPSignNR n8_ippsECCPSignNR +#define ippsECCPVerifyNR n8_ippsECCPVerifyNR +#define ippsECCPSignSM2 n8_ippsECCPSignSM2 +#define ippsECCPVerifySM2 n8_ippsECCPVerifySM2 +#define ippsGFpGetSize n8_ippsGFpGetSize +#define ippsGFpInitArbitrary n8_ippsGFpInitArbitrary +#define ippsGFpInitFixed n8_ippsGFpInitFixed +#define ippsGFpInit n8_ippsGFpInit +#define ippsGFpMethod_p192r1 n8_ippsGFpMethod_p192r1 +#define ippsGFpMethod_p224r1 n8_ippsGFpMethod_p224r1 +#define ippsGFpMethod_p256r1 n8_ippsGFpMethod_p256r1 +#define ippsGFpMethod_p384r1 n8_ippsGFpMethod_p384r1 +#define ippsGFpMethod_p521r1 n8_ippsGFpMethod_p521r1 +#define ippsGFpMethod_p256sm2 n8_ippsGFpMethod_p256sm2 +#define ippsGFpMethod_p256bn n8_ippsGFpMethod_p256bn +#define ippsGFpMethod_p256 n8_ippsGFpMethod_p256 +#define ippsGFpMethod_pArb n8_ippsGFpMethod_pArb +#define ippsGFpxGetSize n8_ippsGFpxGetSize +#define ippsGFpxInit n8_ippsGFpxInit +#define ippsGFpxInitBinomial n8_ippsGFpxInitBinomial +#define ippsGFpxMethod_binom2_epid2 n8_ippsGFpxMethod_binom2_epid2 +#define ippsGFpxMethod_binom3_epid2 n8_ippsGFpxMethod_binom3_epid2 +#define ippsGFpxMethod_binom2 n8_ippsGFpxMethod_binom2 +#define ippsGFpxMethod_binom3 n8_ippsGFpxMethod_binom3 +#define ippsGFpxMethod_binom n8_ippsGFpxMethod_binom +#define ippsGFpxMethod_com n8_ippsGFpxMethod_com +#define ippsGFpScratchBufferSize n8_ippsGFpScratchBufferSize +#define ippsGFpElementGetSize n8_ippsGFpElementGetSize +#define ippsGFpElementInit n8_ippsGFpElementInit +#define ippsGFpSetElement n8_ippsGFpSetElement +#define ippsGFpSetElementRegular n8_ippsGFpSetElementRegular +#define ippsGFpSetElementOctString n8_ippsGFpSetElementOctString +#define ippsGFpSetElementRandom n8_ippsGFpSetElementRandom +#define ippsGFpSetElementHash n8_ippsGFpSetElementHash +#define ippsGFpSetElementHash_rmf n8_ippsGFpSetElementHash_rmf +#define ippsGFpCpyElement n8_ippsGFpCpyElement +#define ippsGFpGetElement n8_ippsGFpGetElement +#define ippsGFpGetElementOctString n8_ippsGFpGetElementOctString +#define ippsGFpCmpElement n8_ippsGFpCmpElement +#define ippsGFpIsZeroElement n8_ippsGFpIsZeroElement +#define ippsGFpIsUnityElement n8_ippsGFpIsUnityElement +#define ippsGFpConj n8_ippsGFpConj +#define ippsGFpNeg n8_ippsGFpNeg +#define ippsGFpInv n8_ippsGFpInv +#define ippsGFpSqrt n8_ippsGFpSqrt +#define ippsGFpSqr n8_ippsGFpSqr +#define ippsGFpAdd n8_ippsGFpAdd +#define ippsGFpSub n8_ippsGFpSub +#define ippsGFpMul n8_ippsGFpMul +#define ippsGFpExp n8_ippsGFpExp +#define ippsGFpMultiExp n8_ippsGFpMultiExp +#define ippsGFpAdd_PE n8_ippsGFpAdd_PE +#define ippsGFpSub_PE n8_ippsGFpSub_PE +#define ippsGFpMul_PE n8_ippsGFpMul_PE +#define ippsGFpGetInfo n8_ippsGFpGetInfo +#define ippsGFpECGetSize n8_ippsGFpECGetSize +#define ippsGFpECInit n8_ippsGFpECInit +#define ippsGFpECSet n8_ippsGFpECSet +#define ippsGFpECSetSubgroup n8_ippsGFpECSetSubgroup +#define ippsGFpECInitStd128r1 n8_ippsGFpECInitStd128r1 +#define ippsGFpECInitStd128r2 n8_ippsGFpECInitStd128r2 +#define ippsGFpECInitStd192r1 n8_ippsGFpECInitStd192r1 +#define ippsGFpECInitStd224r1 n8_ippsGFpECInitStd224r1 +#define ippsGFpECInitStd256r1 n8_ippsGFpECInitStd256r1 +#define ippsGFpECInitStd384r1 n8_ippsGFpECInitStd384r1 +#define ippsGFpECInitStd521r1 n8_ippsGFpECInitStd521r1 +#define ippsGFpECInitStdSM2 n8_ippsGFpECInitStdSM2 +#define ippsGFpECInitStdBN256 n8_ippsGFpECInitStdBN256 +#define ippsGFpECBindGxyTblStd192r1 n8_ippsGFpECBindGxyTblStd192r1 +#define ippsGFpECBindGxyTblStd224r1 n8_ippsGFpECBindGxyTblStd224r1 +#define ippsGFpECBindGxyTblStd256r1 n8_ippsGFpECBindGxyTblStd256r1 +#define ippsGFpECBindGxyTblStd384r1 n8_ippsGFpECBindGxyTblStd384r1 +#define ippsGFpECBindGxyTblStd521r1 n8_ippsGFpECBindGxyTblStd521r1 +#define ippsGFpECBindGxyTblStdSM2 n8_ippsGFpECBindGxyTblStdSM2 +#define ippsGFpECGet n8_ippsGFpECGet +#define ippsGFpECGetSubgroup n8_ippsGFpECGetSubgroup +#define ippsGFpECScratchBufferSize n8_ippsGFpECScratchBufferSize +#define ippsGFpECVerify n8_ippsGFpECVerify +#define ippsGFpECPointGetSize n8_ippsGFpECPointGetSize +#define ippsGFpECPointInit n8_ippsGFpECPointInit +#define ippsGFpECSetPointAtInfinity n8_ippsGFpECSetPointAtInfinity +#define ippsGFpECSetPoint n8_ippsGFpECSetPoint +#define ippsGFpECSetPointRegular n8_ippsGFpECSetPointRegular +#define ippsGFpECSetPointRandom n8_ippsGFpECSetPointRandom +#define ippsGFpECMakePoint n8_ippsGFpECMakePoint +#define ippsGFpECSetPointHash n8_ippsGFpECSetPointHash +#define ippsGFpECSetPointHashBackCompatible n8_ippsGFpECSetPointHashBackCompatible +#define ippsGFpECSetPointHash_rmf n8_ippsGFpECSetPointHash_rmf +#define ippsGFpECSetPointHashBackCompatible_rmf n8_ippsGFpECSetPointHashBackCompatible_rmf +#define ippsGFpECGetPoint n8_ippsGFpECGetPoint +#define ippsGFpECGetPointRegular n8_ippsGFpECGetPointRegular +#define ippsGFpECSetPointOctString n8_ippsGFpECSetPointOctString +#define ippsGFpECGetPointOctString n8_ippsGFpECGetPointOctString +#define ippsGFpECTstPoint n8_ippsGFpECTstPoint +#define ippsGFpECTstPointInSubgroup n8_ippsGFpECTstPointInSubgroup +#define ippsGFpECCpyPoint n8_ippsGFpECCpyPoint +#define ippsGFpECCmpPoint n8_ippsGFpECCmpPoint +#define ippsGFpECNegPoint n8_ippsGFpECNegPoint +#define ippsGFpECAddPoint n8_ippsGFpECAddPoint +#define ippsGFpECMulPoint n8_ippsGFpECMulPoint +#define ippsGFpECPrivateKey n8_ippsGFpECPrivateKey +#define ippsGFpECPublicKey n8_ippsGFpECPublicKey +#define ippsGFpECTstKeyPair n8_ippsGFpECTstKeyPair +#define ippsGFpECSharedSecretDH n8_ippsGFpECSharedSecretDH +#define ippsGFpECSharedSecretDHC n8_ippsGFpECSharedSecretDHC +#define ippsGFpECMessageRepresentationSM2 n8_ippsGFpECMessageRepresentationSM2 +#define ippsGFpECSignDSA n8_ippsGFpECSignDSA +#define ippsGFpECVerifyDSA n8_ippsGFpECVerifyDSA +#define ippsGFpECSignNR n8_ippsGFpECSignNR +#define ippsGFpECVerifyNR n8_ippsGFpECVerifyNR +#define ippsGFpECSignSM2 n8_ippsGFpECSignSM2 +#define ippsGFpECVerifySM2 n8_ippsGFpECVerifySM2 +#define ippsGFpECUserIDHashSM2 n8_ippsGFpECUserIDHashSM2 +#define ippsGFpECKeyExchangeSM2_GetSize n8_ippsGFpECKeyExchangeSM2_GetSize +#define ippsGFpECKeyExchangeSM2_Init n8_ippsGFpECKeyExchangeSM2_Init +#define ippsGFpECKeyExchangeSM2_Setup n8_ippsGFpECKeyExchangeSM2_Setup +#define ippsGFpECKeyExchangeSM2_SharedKey n8_ippsGFpECKeyExchangeSM2_SharedKey +#define ippsGFpECKeyExchangeSM2_Confirm n8_ippsGFpECKeyExchangeSM2_Confirm +#define ippsGFpECGetInfo_GF n8_ippsGFpECGetInfo_GF +#define ippsGFpECESGetSize_SM2 n8_ippsGFpECESGetSize_SM2 +#define ippsGFpECESInit_SM2 n8_ippsGFpECESInit_SM2 +#define ippsGFpECESSetKey_SM2 n8_ippsGFpECESSetKey_SM2 +#define ippsGFpECESStart_SM2 n8_ippsGFpECESStart_SM2 +#define ippsGFpECESEncrypt_SM2 n8_ippsGFpECESEncrypt_SM2 +#define ippsGFpECESDecrypt_SM2 n8_ippsGFpECESDecrypt_SM2 +#define ippsGFpECESFinal_SM2 n8_ippsGFpECESFinal_SM2 +#define ippsGFpECESGetBuffersSize_SM2 n8_ippsGFpECESGetBuffersSize_SM2 +#define ippsGFpECEncryptSM2_Ext_EncMsgSize n8_ippsGFpECEncryptSM2_Ext_EncMsgSize +#define ippsGFpECEncryptSM2_Ext n8_ippsGFpECEncryptSM2_Ext +#define ippsGFpECDecryptSM2_Ext_DecMsgSize n8_ippsGFpECDecryptSM2_Ext_DecMsgSize +#define ippsGFpECDecryptSM2_Ext n8_ippsGFpECDecryptSM2_Ext diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_p8.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_p8.h new file mode 100644 index 0000000..0fac050 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_p8.h @@ -0,0 +1,557 @@ +/******************************************************************************* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + + #define ippcpGetLibVersion p8_ippcpGetLibVersion +#define ippsDESGetSize p8_ippsDESGetSize +#define ippsDESInit p8_ippsDESInit +#define ippsDESPack p8_ippsDESPack +#define ippsDESUnpack p8_ippsDESUnpack +#define ippsTDESEncryptECB p8_ippsTDESEncryptECB +#define ippsTDESDecryptECB p8_ippsTDESDecryptECB +#define ippsTDESEncryptCBC p8_ippsTDESEncryptCBC +#define ippsTDESDecryptCBC p8_ippsTDESDecryptCBC +#define ippsTDESEncryptCFB p8_ippsTDESEncryptCFB +#define ippsTDESDecryptCFB p8_ippsTDESDecryptCFB +#define ippsTDESEncryptOFB p8_ippsTDESEncryptOFB +#define ippsTDESDecryptOFB p8_ippsTDESDecryptOFB +#define ippsTDESEncryptCTR p8_ippsTDESEncryptCTR +#define ippsTDESDecryptCTR p8_ippsTDESDecryptCTR +#define ippsAESGetSize p8_ippsAESGetSize +#define ippsAESInit p8_ippsAESInit +#define ippsAESSetKey p8_ippsAESSetKey +#define ippsAESPack p8_ippsAESPack +#define ippsAESUnpack p8_ippsAESUnpack +#define ippsAESEncryptECB p8_ippsAESEncryptECB +#define ippsAESDecryptECB p8_ippsAESDecryptECB +#define ippsAESEncryptCBC p8_ippsAESEncryptCBC +#define ippsAESEncryptCBC_CS1 p8_ippsAESEncryptCBC_CS1 +#define ippsAESEncryptCBC_CS2 p8_ippsAESEncryptCBC_CS2 +#define ippsAESEncryptCBC_CS3 p8_ippsAESEncryptCBC_CS3 +#define ippsAESDecryptCBC p8_ippsAESDecryptCBC +#define ippsAESDecryptCBC_CS1 p8_ippsAESDecryptCBC_CS1 +#define ippsAESDecryptCBC_CS2 p8_ippsAESDecryptCBC_CS2 +#define ippsAESDecryptCBC_CS3 p8_ippsAESDecryptCBC_CS3 +#define ippsAESEncryptCFB p8_ippsAESEncryptCFB +#define ippsAESDecryptCFB p8_ippsAESDecryptCFB +#define ippsAESEncryptOFB p8_ippsAESEncryptOFB +#define ippsAESDecryptOFB p8_ippsAESDecryptOFB +#define ippsAESEncryptCTR p8_ippsAESEncryptCTR +#define ippsAESDecryptCTR p8_ippsAESDecryptCTR +#define ippsAESEncryptXTS_Direct p8_ippsAESEncryptXTS_Direct +#define ippsAESDecryptXTS_Direct p8_ippsAESDecryptXTS_Direct +#define ippsAESSetupNoise p8_ippsAESSetupNoise +#define ippsAES_GCMSetupNoise p8_ippsAES_GCMSetupNoise +#define ippsAES_CMACSetupNoise p8_ippsAES_CMACSetupNoise +#define ippsAES_EncryptCFB16_MB p8_ippsAES_EncryptCFB16_MB +#define ippsSMS4GetSize p8_ippsSMS4GetSize +#define ippsSMS4Init p8_ippsSMS4Init +#define ippsSMS4SetKey p8_ippsSMS4SetKey +#define ippsSMS4EncryptECB p8_ippsSMS4EncryptECB +#define ippsSMS4DecryptECB p8_ippsSMS4DecryptECB +#define ippsSMS4EncryptCBC p8_ippsSMS4EncryptCBC +#define ippsSMS4EncryptCBC_CS1 p8_ippsSMS4EncryptCBC_CS1 +#define ippsSMS4EncryptCBC_CS2 p8_ippsSMS4EncryptCBC_CS2 +#define ippsSMS4EncryptCBC_CS3 p8_ippsSMS4EncryptCBC_CS3 +#define ippsSMS4DecryptCBC p8_ippsSMS4DecryptCBC +#define ippsSMS4DecryptCBC_CS1 p8_ippsSMS4DecryptCBC_CS1 +#define ippsSMS4DecryptCBC_CS2 p8_ippsSMS4DecryptCBC_CS2 +#define ippsSMS4DecryptCBC_CS3 p8_ippsSMS4DecryptCBC_CS3 +#define ippsSMS4EncryptCFB p8_ippsSMS4EncryptCFB +#define ippsSMS4DecryptCFB p8_ippsSMS4DecryptCFB +#define ippsSMS4EncryptOFB p8_ippsSMS4EncryptOFB +#define ippsSMS4DecryptOFB p8_ippsSMS4DecryptOFB +#define ippsSMS4EncryptCTR p8_ippsSMS4EncryptCTR +#define ippsSMS4DecryptCTR p8_ippsSMS4DecryptCTR +#define ippsSMS4_CCMGetSize p8_ippsSMS4_CCMGetSize +#define ippsSMS4_CCMInit p8_ippsSMS4_CCMInit +#define ippsSMS4_CCMMessageLen p8_ippsSMS4_CCMMessageLen +#define ippsSMS4_CCMTagLen p8_ippsSMS4_CCMTagLen +#define ippsSMS4_CCMStart p8_ippsSMS4_CCMStart +#define ippsSMS4_CCMEncrypt p8_ippsSMS4_CCMEncrypt +#define ippsSMS4_CCMDecrypt p8_ippsSMS4_CCMDecrypt +#define ippsSMS4_CCMGetTag p8_ippsSMS4_CCMGetTag +#define ippsAES_CCMGetSize p8_ippsAES_CCMGetSize +#define ippsAES_CCMInit p8_ippsAES_CCMInit +#define ippsAES_CCMMessageLen p8_ippsAES_CCMMessageLen +#define ippsAES_CCMTagLen p8_ippsAES_CCMTagLen +#define ippsAES_CCMStart p8_ippsAES_CCMStart +#define ippsAES_CCMEncrypt p8_ippsAES_CCMEncrypt +#define ippsAES_CCMDecrypt p8_ippsAES_CCMDecrypt +#define ippsAES_CCMGetTag p8_ippsAES_CCMGetTag +#define ippsAES_GCMGetSize p8_ippsAES_GCMGetSize +#define ippsAES_GCMInit p8_ippsAES_GCMInit +#define ippsAES_GCMReinit p8_ippsAES_GCMReinit +#define ippsAES_GCMReset p8_ippsAES_GCMReset +#define ippsAES_GCMProcessIV p8_ippsAES_GCMProcessIV +#define ippsAES_GCMProcessAAD p8_ippsAES_GCMProcessAAD +#define ippsAES_GCMStart p8_ippsAES_GCMStart +#define ippsAES_GCMEncrypt p8_ippsAES_GCMEncrypt +#define ippsAES_GCMDecrypt p8_ippsAES_GCMDecrypt +#define ippsAES_GCMGetTag p8_ippsAES_GCMGetTag +#define ippsAES_XTSGetSize p8_ippsAES_XTSGetSize +#define ippsAES_XTSInit p8_ippsAES_XTSInit +#define ippsAES_XTSEncrypt p8_ippsAES_XTSEncrypt +#define ippsAES_XTSDecrypt p8_ippsAES_XTSDecrypt +#define ippsAES_S2V_CMAC p8_ippsAES_S2V_CMAC +#define ippsAES_SIVEncrypt p8_ippsAES_SIVEncrypt +#define ippsAES_SIVDecrypt p8_ippsAES_SIVDecrypt +#define ippsAES_CMACGetSize p8_ippsAES_CMACGetSize +#define ippsAES_CMACInit p8_ippsAES_CMACInit +#define ippsAES_CMACUpdate p8_ippsAES_CMACUpdate +#define ippsAES_CMACFinal p8_ippsAES_CMACFinal +#define ippsAES_CMACGetTag p8_ippsAES_CMACGetTag +#define ippsARCFourCheckKey p8_ippsARCFourCheckKey +#define ippsARCFourGetSize p8_ippsARCFourGetSize +#define ippsARCFourInit p8_ippsARCFourInit +#define ippsARCFourReset p8_ippsARCFourReset +#define ippsARCFourPack p8_ippsARCFourPack +#define ippsARCFourUnpack p8_ippsARCFourUnpack +#define ippsARCFourEncrypt p8_ippsARCFourEncrypt +#define ippsARCFourDecrypt p8_ippsARCFourDecrypt +#define ippsSHA1GetSize p8_ippsSHA1GetSize +#define ippsSHA1Init p8_ippsSHA1Init +#define ippsSHA1Duplicate p8_ippsSHA1Duplicate +#define ippsSHA1Pack p8_ippsSHA1Pack +#define ippsSHA1Unpack p8_ippsSHA1Unpack +#define ippsSHA1Update p8_ippsSHA1Update +#define ippsSHA1GetTag p8_ippsSHA1GetTag +#define ippsSHA1Final p8_ippsSHA1Final +#define ippsSHA1MessageDigest p8_ippsSHA1MessageDigest +#define ippsSHA224GetSize p8_ippsSHA224GetSize +#define ippsSHA224Init p8_ippsSHA224Init +#define ippsSHA224Duplicate p8_ippsSHA224Duplicate +#define ippsSHA224Pack p8_ippsSHA224Pack +#define ippsSHA224Unpack p8_ippsSHA224Unpack +#define ippsSHA224Update p8_ippsSHA224Update +#define ippsSHA224GetTag p8_ippsSHA224GetTag +#define ippsSHA224Final p8_ippsSHA224Final +#define ippsSHA224MessageDigest p8_ippsSHA224MessageDigest +#define ippsSHA256GetSize p8_ippsSHA256GetSize +#define ippsSHA256Init p8_ippsSHA256Init +#define ippsSHA256Duplicate p8_ippsSHA256Duplicate +#define ippsSHA256Pack p8_ippsSHA256Pack +#define ippsSHA256Unpack p8_ippsSHA256Unpack +#define ippsSHA256Update p8_ippsSHA256Update +#define ippsSHA256GetTag p8_ippsSHA256GetTag +#define ippsSHA256Final p8_ippsSHA256Final +#define ippsSHA256MessageDigest p8_ippsSHA256MessageDigest +#define ippsSHA384GetSize p8_ippsSHA384GetSize +#define ippsSHA384Init p8_ippsSHA384Init +#define ippsSHA384Duplicate p8_ippsSHA384Duplicate +#define ippsSHA384Pack p8_ippsSHA384Pack +#define ippsSHA384Unpack p8_ippsSHA384Unpack +#define ippsSHA384Update p8_ippsSHA384Update +#define ippsSHA384GetTag p8_ippsSHA384GetTag +#define ippsSHA384Final p8_ippsSHA384Final +#define ippsSHA384MessageDigest p8_ippsSHA384MessageDigest +#define ippsSHA512GetSize p8_ippsSHA512GetSize +#define ippsSHA512Init p8_ippsSHA512Init +#define ippsSHA512Duplicate p8_ippsSHA512Duplicate +#define ippsSHA512Pack p8_ippsSHA512Pack +#define ippsSHA512Unpack p8_ippsSHA512Unpack +#define ippsSHA512Update p8_ippsSHA512Update +#define ippsSHA512GetTag p8_ippsSHA512GetTag +#define ippsSHA512Final p8_ippsSHA512Final +#define ippsSHA512MessageDigest p8_ippsSHA512MessageDigest +#define ippsMD5GetSize p8_ippsMD5GetSize +#define ippsMD5Init p8_ippsMD5Init +#define ippsMD5Duplicate p8_ippsMD5Duplicate +#define ippsMD5Pack p8_ippsMD5Pack +#define ippsMD5Unpack p8_ippsMD5Unpack +#define ippsMD5Update p8_ippsMD5Update +#define ippsMD5GetTag p8_ippsMD5GetTag +#define ippsMD5Final p8_ippsMD5Final +#define ippsMD5MessageDigest p8_ippsMD5MessageDigest +#define ippsSM3GetSize p8_ippsSM3GetSize +#define ippsSM3Init p8_ippsSM3Init +#define ippsSM3Duplicate p8_ippsSM3Duplicate +#define ippsSM3Pack p8_ippsSM3Pack +#define ippsSM3Unpack p8_ippsSM3Unpack +#define ippsSM3Update p8_ippsSM3Update +#define ippsSM3GetTag p8_ippsSM3GetTag +#define ippsSM3Final p8_ippsSM3Final +#define ippsSM3MessageDigest p8_ippsSM3MessageDigest +#define ippsHashGetSize p8_ippsHashGetSize +#define ippsHashInit p8_ippsHashInit +#define ippsHashPack p8_ippsHashPack +#define ippsHashUnpack p8_ippsHashUnpack +#define ippsHashDuplicate p8_ippsHashDuplicate +#define ippsHashUpdate p8_ippsHashUpdate +#define ippsHashGetTag p8_ippsHashGetTag +#define ippsHashFinal p8_ippsHashFinal +#define ippsHashMessage p8_ippsHashMessage +#define ippsHashMethod_MD5 p8_ippsHashMethod_MD5 +#define ippsHashMethod_SM3 p8_ippsHashMethod_SM3 +#define ippsHashMethod_SHA1 p8_ippsHashMethod_SHA1 +#define ippsHashMethod_SHA1_NI p8_ippsHashMethod_SHA1_NI +#define ippsHashMethod_SHA1_TT p8_ippsHashMethod_SHA1_TT +#define ippsHashMethod_SHA256 p8_ippsHashMethod_SHA256 +#define ippsHashMethod_SHA256_NI p8_ippsHashMethod_SHA256_NI +#define ippsHashMethod_SHA256_TT p8_ippsHashMethod_SHA256_TT +#define ippsHashMethod_SHA224 p8_ippsHashMethod_SHA224 +#define ippsHashMethod_SHA224_NI p8_ippsHashMethod_SHA224_NI +#define ippsHashMethod_SHA224_TT p8_ippsHashMethod_SHA224_TT +#define ippsHashMethod_SHA512 p8_ippsHashMethod_SHA512 +#define ippsHashMethod_SHA384 p8_ippsHashMethod_SHA384 +#define ippsHashMethod_SHA512_256 p8_ippsHashMethod_SHA512_256 +#define ippsHashMethod_SHA512_224 p8_ippsHashMethod_SHA512_224 +#define ippsHashMethodGetSize p8_ippsHashMethodGetSize +#define ippsHashMethodSet_MD5 p8_ippsHashMethodSet_MD5 +#define ippsHashMethodSet_SM3 p8_ippsHashMethodSet_SM3 +#define ippsHashStateMethodSet_SM3 p8_ippsHashStateMethodSet_SM3 +#define ippsHashMethodSet_SHA1 p8_ippsHashMethodSet_SHA1 +#define ippsHashMethodSet_SHA1_NI p8_ippsHashMethodSet_SHA1_NI +#define ippsHashMethodSet_SHA1_TT p8_ippsHashMethodSet_SHA1_TT +#define ippsHashMethodSet_SHA256 p8_ippsHashMethodSet_SHA256 +#define ippsHashMethodSet_SHA256_NI p8_ippsHashMethodSet_SHA256_NI +#define ippsHashMethodSet_SHA256_TT p8_ippsHashMethodSet_SHA256_TT +#define ippsHashStateMethodSet_SHA256 p8_ippsHashStateMethodSet_SHA256 +#define ippsHashStateMethodSet_SHA256_NI p8_ippsHashStateMethodSet_SHA256_NI +#define ippsHashStateMethodSet_SHA256_TT p8_ippsHashStateMethodSet_SHA256_TT +#define ippsHashMethodSet_SHA224 p8_ippsHashMethodSet_SHA224 +#define ippsHashMethodSet_SHA224_NI p8_ippsHashMethodSet_SHA224_NI +#define ippsHashMethodSet_SHA224_TT p8_ippsHashMethodSet_SHA224_TT +#define ippsHashStateMethodSet_SHA224 p8_ippsHashStateMethodSet_SHA224 +#define ippsHashStateMethodSet_SHA224_NI p8_ippsHashStateMethodSet_SHA224_NI +#define ippsHashStateMethodSet_SHA224_TT p8_ippsHashStateMethodSet_SHA224_TT +#define ippsHashMethodSet_SHA512 p8_ippsHashMethodSet_SHA512 +#define ippsHashMethodSet_SHA384 p8_ippsHashMethodSet_SHA384 +#define ippsHashMethodSet_SHA512_256 p8_ippsHashMethodSet_SHA512_256 +#define ippsHashMethodSet_SHA512_224 p8_ippsHashMethodSet_SHA512_224 +#define ippsHashStateMethodSet_SHA512 p8_ippsHashStateMethodSet_SHA512 +#define ippsHashStateMethodSet_SHA384 p8_ippsHashStateMethodSet_SHA384 +#define ippsHashStateMethodSet_SHA512_256 p8_ippsHashStateMethodSet_SHA512_256 +#define ippsHashStateMethodSet_SHA512_224 p8_ippsHashStateMethodSet_SHA512_224 +#define ippsHashGetSize_rmf p8_ippsHashGetSize_rmf +#define ippsHashInit_rmf p8_ippsHashInit_rmf +#define ippsHashPack_rmf p8_ippsHashPack_rmf +#define ippsHashUnpack_rmf p8_ippsHashUnpack_rmf +#define ippsHashDuplicate_rmf p8_ippsHashDuplicate_rmf +#define ippsHashUpdate_rmf p8_ippsHashUpdate_rmf +#define ippsHashGetTag_rmf p8_ippsHashGetTag_rmf +#define ippsHashFinal_rmf p8_ippsHashFinal_rmf +#define ippsHashMessage_rmf p8_ippsHashMessage_rmf +#define ippsHashMethodGetInfo p8_ippsHashMethodGetInfo +#define ippsHashGetInfo_rmf p8_ippsHashGetInfo_rmf +#define ippsMGF p8_ippsMGF +#define ippsMGF1_rmf p8_ippsMGF1_rmf +#define ippsMGF2_rmf p8_ippsMGF2_rmf +#define ippsHMAC_GetSize p8_ippsHMAC_GetSize +#define ippsHMAC_Init p8_ippsHMAC_Init +#define ippsHMAC_Pack p8_ippsHMAC_Pack +#define ippsHMAC_Unpack p8_ippsHMAC_Unpack +#define ippsHMAC_Duplicate p8_ippsHMAC_Duplicate +#define ippsHMAC_Update p8_ippsHMAC_Update +#define ippsHMAC_Final p8_ippsHMAC_Final +#define ippsHMAC_GetTag p8_ippsHMAC_GetTag +#define ippsHMAC_Message p8_ippsHMAC_Message +#define ippsHMACGetSize_rmf p8_ippsHMACGetSize_rmf +#define ippsHMACInit_rmf p8_ippsHMACInit_rmf +#define ippsHMACPack_rmf p8_ippsHMACPack_rmf +#define ippsHMACUnpack_rmf p8_ippsHMACUnpack_rmf +#define ippsHMACDuplicate_rmf p8_ippsHMACDuplicate_rmf +#define ippsHMACUpdate_rmf p8_ippsHMACUpdate_rmf +#define ippsHMACFinal_rmf p8_ippsHMACFinal_rmf +#define ippsHMACGetTag_rmf p8_ippsHMACGetTag_rmf +#define ippsHMACMessage_rmf p8_ippsHMACMessage_rmf +#define ippsBigNumGetSize p8_ippsBigNumGetSize +#define ippsBigNumInit p8_ippsBigNumInit +#define ippsCmpZero_BN p8_ippsCmpZero_BN +#define ippsCmp_BN p8_ippsCmp_BN +#define ippsGetSize_BN p8_ippsGetSize_BN +#define ippsSet_BN p8_ippsSet_BN +#define ippsGet_BN p8_ippsGet_BN +#define ippsRef_BN p8_ippsRef_BN +#define ippsExtGet_BN p8_ippsExtGet_BN +#define ippsAdd_BN p8_ippsAdd_BN +#define ippsSub_BN p8_ippsSub_BN +#define ippsMul_BN p8_ippsMul_BN +#define ippsMAC_BN_I p8_ippsMAC_BN_I +#define ippsDiv_BN p8_ippsDiv_BN +#define ippsMod_BN p8_ippsMod_BN +#define ippsGcd_BN p8_ippsGcd_BN +#define ippsModInv_BN p8_ippsModInv_BN +#define ippsSetOctString_BN p8_ippsSetOctString_BN +#define ippsGetOctString_BN p8_ippsGetOctString_BN +#define ippsMontGetSize p8_ippsMontGetSize +#define ippsMontInit p8_ippsMontInit +#define ippsMontSet p8_ippsMontSet +#define ippsMontGet p8_ippsMontGet +#define ippsMontForm p8_ippsMontForm +#define ippsMontMul p8_ippsMontMul +#define ippsMontExp p8_ippsMontExp +#define ippsPRNGGetSize p8_ippsPRNGGetSize +#define ippsPRNGInit p8_ippsPRNGInit +#define ippsPRNGSetModulus p8_ippsPRNGSetModulus +#define ippsPRNGSetH0 p8_ippsPRNGSetH0 +#define ippsPRNGSetAugment p8_ippsPRNGSetAugment +#define ippsPRNGSetSeed p8_ippsPRNGSetSeed +#define ippsPRNGGetSeed p8_ippsPRNGGetSeed +#define ippsPRNGen p8_ippsPRNGen +#define ippsPRNGen_BN p8_ippsPRNGen_BN +#define ippsPRNGenRDRAND p8_ippsPRNGenRDRAND +#define ippsPRNGenRDRAND_BN p8_ippsPRNGenRDRAND_BN +#define ippsTRNGenRDSEED p8_ippsTRNGenRDSEED +#define ippsTRNGenRDSEED_BN p8_ippsTRNGenRDSEED_BN +#define ippsPrimeGetSize p8_ippsPrimeGetSize +#define ippsPrimeInit p8_ippsPrimeInit +#define ippsPrimeGen p8_ippsPrimeGen +#define ippsPrimeTest p8_ippsPrimeTest +#define ippsPrimeGen_BN p8_ippsPrimeGen_BN +#define ippsPrimeTest_BN p8_ippsPrimeTest_BN +#define ippsPrimeGet p8_ippsPrimeGet +#define ippsPrimeGet_BN p8_ippsPrimeGet_BN +#define ippsPrimeSet p8_ippsPrimeSet +#define ippsPrimeSet_BN p8_ippsPrimeSet_BN +#define ippsRSA_GetSizePublicKey p8_ippsRSA_GetSizePublicKey +#define ippsRSA_InitPublicKey p8_ippsRSA_InitPublicKey +#define ippsRSA_SetPublicKey p8_ippsRSA_SetPublicKey +#define ippsRSA_GetPublicKey p8_ippsRSA_GetPublicKey +#define ippsRSA_GetSizePrivateKeyType1 p8_ippsRSA_GetSizePrivateKeyType1 +#define ippsRSA_InitPrivateKeyType1 p8_ippsRSA_InitPrivateKeyType1 +#define ippsRSA_SetPrivateKeyType1 p8_ippsRSA_SetPrivateKeyType1 +#define ippsRSA_GetPrivateKeyType1 p8_ippsRSA_GetPrivateKeyType1 +#define ippsRSA_GetSizePrivateKeyType2 p8_ippsRSA_GetSizePrivateKeyType2 +#define ippsRSA_InitPrivateKeyType2 p8_ippsRSA_InitPrivateKeyType2 +#define ippsRSA_SetPrivateKeyType2 p8_ippsRSA_SetPrivateKeyType2 +#define ippsRSA_GetPrivateKeyType2 p8_ippsRSA_GetPrivateKeyType2 +#define ippsRSA_GetBufferSizePublicKey p8_ippsRSA_GetBufferSizePublicKey +#define ippsRSA_GetBufferSizePrivateKey p8_ippsRSA_GetBufferSizePrivateKey +#define ippsRSA_Encrypt p8_ippsRSA_Encrypt +#define ippsRSA_Decrypt p8_ippsRSA_Decrypt +#define ippsRSA_GenerateKeys p8_ippsRSA_GenerateKeys +#define ippsRSA_ValidateKeys p8_ippsRSA_ValidateKeys +#define ippsRSAEncrypt_OAEP p8_ippsRSAEncrypt_OAEP +#define ippsRSADecrypt_OAEP p8_ippsRSADecrypt_OAEP +#define ippsRSAEncrypt_OAEP_rmf p8_ippsRSAEncrypt_OAEP_rmf +#define ippsRSADecrypt_OAEP_rmf p8_ippsRSADecrypt_OAEP_rmf +#define ippsRSAEncrypt_PKCSv15 p8_ippsRSAEncrypt_PKCSv15 +#define ippsRSADecrypt_PKCSv15 p8_ippsRSADecrypt_PKCSv15 +#define ippsRSASign_PSS p8_ippsRSASign_PSS +#define ippsRSAVerify_PSS p8_ippsRSAVerify_PSS +#define ippsRSASign_PSS_rmf p8_ippsRSASign_PSS_rmf +#define ippsRSAVerify_PSS_rmf p8_ippsRSAVerify_PSS_rmf +#define ippsRSASign_PKCS1v15 p8_ippsRSASign_PKCS1v15 +#define ippsRSAVerify_PKCS1v15 p8_ippsRSAVerify_PKCS1v15 +#define ippsRSASign_PKCS1v15_rmf p8_ippsRSASign_PKCS1v15_rmf +#define ippsRSAVerify_PKCS1v15_rmf p8_ippsRSAVerify_PKCS1v15_rmf +#define ippsDLGetResultString p8_ippsDLGetResultString +#define ippsDLPGetSize p8_ippsDLPGetSize +#define ippsDLPInit p8_ippsDLPInit +#define ippsDLPPack p8_ippsDLPPack +#define ippsDLPUnpack p8_ippsDLPUnpack +#define ippsDLPSet p8_ippsDLPSet +#define ippsDLPGet p8_ippsDLPGet +#define ippsDLPSetDP p8_ippsDLPSetDP +#define ippsDLPGetDP p8_ippsDLPGetDP +#define ippsDLPGenKeyPair p8_ippsDLPGenKeyPair +#define ippsDLPPublicKey p8_ippsDLPPublicKey +#define ippsDLPValidateKeyPair p8_ippsDLPValidateKeyPair +#define ippsDLPSetKeyPair p8_ippsDLPSetKeyPair +#define ippsDLPSignDSA p8_ippsDLPSignDSA +#define ippsDLPVerifyDSA p8_ippsDLPVerifyDSA +#define ippsDLPSharedSecretDH p8_ippsDLPSharedSecretDH +#define ippsDLPGenerateDSA p8_ippsDLPGenerateDSA +#define ippsDLPValidateDSA p8_ippsDLPValidateDSA +#define ippsDLPGenerateDH p8_ippsDLPGenerateDH +#define ippsDLPValidateDH p8_ippsDLPValidateDH +#define ippsECCGetResultString p8_ippsECCGetResultString +#define ippsECCPGetSize p8_ippsECCPGetSize +#define ippsECCPGetSizeStd128r1 p8_ippsECCPGetSizeStd128r1 +#define ippsECCPGetSizeStd128r2 p8_ippsECCPGetSizeStd128r2 +#define ippsECCPGetSizeStd192r1 p8_ippsECCPGetSizeStd192r1 +#define ippsECCPGetSizeStd224r1 p8_ippsECCPGetSizeStd224r1 +#define ippsECCPGetSizeStd256r1 p8_ippsECCPGetSizeStd256r1 +#define ippsECCPGetSizeStd384r1 p8_ippsECCPGetSizeStd384r1 +#define ippsECCPGetSizeStd521r1 p8_ippsECCPGetSizeStd521r1 +#define ippsECCPGetSizeStdSM2 p8_ippsECCPGetSizeStdSM2 +#define ippsECCPInit p8_ippsECCPInit +#define ippsECCPInitStd128r1 p8_ippsECCPInitStd128r1 +#define ippsECCPInitStd128r2 p8_ippsECCPInitStd128r2 +#define ippsECCPInitStd192r1 p8_ippsECCPInitStd192r1 +#define ippsECCPInitStd224r1 p8_ippsECCPInitStd224r1 +#define ippsECCPInitStd256r1 p8_ippsECCPInitStd256r1 +#define ippsECCPInitStd384r1 p8_ippsECCPInitStd384r1 +#define ippsECCPInitStd521r1 p8_ippsECCPInitStd521r1 +#define ippsECCPInitStdSM2 p8_ippsECCPInitStdSM2 +#define ippsECCPSet p8_ippsECCPSet +#define ippsECCPSetStd p8_ippsECCPSetStd +#define ippsECCPSetStd128r1 p8_ippsECCPSetStd128r1 +#define ippsECCPSetStd128r2 p8_ippsECCPSetStd128r2 +#define ippsECCPSetStd192r1 p8_ippsECCPSetStd192r1 +#define ippsECCPSetStd224r1 p8_ippsECCPSetStd224r1 +#define ippsECCPSetStd256r1 p8_ippsECCPSetStd256r1 +#define ippsECCPSetStd384r1 p8_ippsECCPSetStd384r1 +#define ippsECCPSetStd521r1 p8_ippsECCPSetStd521r1 +#define ippsECCPSetStdSM2 p8_ippsECCPSetStdSM2 +#define ippsECCPBindGxyTblStd192r1 p8_ippsECCPBindGxyTblStd192r1 +#define ippsECCPBindGxyTblStd224r1 p8_ippsECCPBindGxyTblStd224r1 +#define ippsECCPBindGxyTblStd256r1 p8_ippsECCPBindGxyTblStd256r1 +#define ippsECCPBindGxyTblStd384r1 p8_ippsECCPBindGxyTblStd384r1 +#define ippsECCPBindGxyTblStd521r1 p8_ippsECCPBindGxyTblStd521r1 +#define ippsECCPBindGxyTblStdSM2 p8_ippsECCPBindGxyTblStdSM2 +#define ippsECCPGet p8_ippsECCPGet +#define ippsECCPGetOrderBitSize p8_ippsECCPGetOrderBitSize +#define ippsECCPValidate p8_ippsECCPValidate +#define ippsECCPPointGetSize p8_ippsECCPPointGetSize +#define ippsECCPPointInit p8_ippsECCPPointInit +#define ippsECCPSetPoint p8_ippsECCPSetPoint +#define ippsECCPSetPointAtInfinity p8_ippsECCPSetPointAtInfinity +#define ippsECCPGetPoint p8_ippsECCPGetPoint +#define ippsECCPCheckPoint p8_ippsECCPCheckPoint +#define ippsECCPComparePoint p8_ippsECCPComparePoint +#define ippsECCPNegativePoint p8_ippsECCPNegativePoint +#define ippsECCPAddPoint p8_ippsECCPAddPoint +#define ippsECCPMulPointScalar p8_ippsECCPMulPointScalar +#define ippsECCPGenKeyPair p8_ippsECCPGenKeyPair +#define ippsECCPPublicKey p8_ippsECCPPublicKey +#define ippsECCPValidateKeyPair p8_ippsECCPValidateKeyPair +#define ippsECCPSetKeyPair p8_ippsECCPSetKeyPair +#define ippsECCPSharedSecretDH p8_ippsECCPSharedSecretDH +#define ippsECCPSharedSecretDHC p8_ippsECCPSharedSecretDHC +#define ippsECCPSignDSA p8_ippsECCPSignDSA +#define ippsECCPVerifyDSA p8_ippsECCPVerifyDSA +#define ippsECCPSignNR p8_ippsECCPSignNR +#define ippsECCPVerifyNR p8_ippsECCPVerifyNR +#define ippsECCPSignSM2 p8_ippsECCPSignSM2 +#define ippsECCPVerifySM2 p8_ippsECCPVerifySM2 +#define ippsGFpGetSize p8_ippsGFpGetSize +#define ippsGFpInitArbitrary p8_ippsGFpInitArbitrary +#define ippsGFpInitFixed p8_ippsGFpInitFixed +#define ippsGFpInit p8_ippsGFpInit +#define ippsGFpMethod_p192r1 p8_ippsGFpMethod_p192r1 +#define ippsGFpMethod_p224r1 p8_ippsGFpMethod_p224r1 +#define ippsGFpMethod_p256r1 p8_ippsGFpMethod_p256r1 +#define ippsGFpMethod_p384r1 p8_ippsGFpMethod_p384r1 +#define ippsGFpMethod_p521r1 p8_ippsGFpMethod_p521r1 +#define ippsGFpMethod_p256sm2 p8_ippsGFpMethod_p256sm2 +#define ippsGFpMethod_p256bn p8_ippsGFpMethod_p256bn +#define ippsGFpMethod_p256 p8_ippsGFpMethod_p256 +#define ippsGFpMethod_pArb p8_ippsGFpMethod_pArb +#define ippsGFpxGetSize p8_ippsGFpxGetSize +#define ippsGFpxInit p8_ippsGFpxInit +#define ippsGFpxInitBinomial p8_ippsGFpxInitBinomial +#define ippsGFpxMethod_binom2_epid2 p8_ippsGFpxMethod_binom2_epid2 +#define ippsGFpxMethod_binom3_epid2 p8_ippsGFpxMethod_binom3_epid2 +#define ippsGFpxMethod_binom2 p8_ippsGFpxMethod_binom2 +#define ippsGFpxMethod_binom3 p8_ippsGFpxMethod_binom3 +#define ippsGFpxMethod_binom p8_ippsGFpxMethod_binom +#define ippsGFpxMethod_com p8_ippsGFpxMethod_com +#define ippsGFpScratchBufferSize p8_ippsGFpScratchBufferSize +#define ippsGFpElementGetSize p8_ippsGFpElementGetSize +#define ippsGFpElementInit p8_ippsGFpElementInit +#define ippsGFpSetElement p8_ippsGFpSetElement +#define ippsGFpSetElementRegular p8_ippsGFpSetElementRegular +#define ippsGFpSetElementOctString p8_ippsGFpSetElementOctString +#define ippsGFpSetElementRandom p8_ippsGFpSetElementRandom +#define ippsGFpSetElementHash p8_ippsGFpSetElementHash +#define ippsGFpSetElementHash_rmf p8_ippsGFpSetElementHash_rmf +#define ippsGFpCpyElement p8_ippsGFpCpyElement +#define ippsGFpGetElement p8_ippsGFpGetElement +#define ippsGFpGetElementOctString p8_ippsGFpGetElementOctString +#define ippsGFpCmpElement p8_ippsGFpCmpElement +#define ippsGFpIsZeroElement p8_ippsGFpIsZeroElement +#define ippsGFpIsUnityElement p8_ippsGFpIsUnityElement +#define ippsGFpConj p8_ippsGFpConj +#define ippsGFpNeg p8_ippsGFpNeg +#define ippsGFpInv p8_ippsGFpInv +#define ippsGFpSqrt p8_ippsGFpSqrt +#define ippsGFpSqr p8_ippsGFpSqr +#define ippsGFpAdd p8_ippsGFpAdd +#define ippsGFpSub p8_ippsGFpSub +#define ippsGFpMul p8_ippsGFpMul +#define ippsGFpExp p8_ippsGFpExp +#define ippsGFpMultiExp p8_ippsGFpMultiExp +#define ippsGFpAdd_PE p8_ippsGFpAdd_PE +#define ippsGFpSub_PE p8_ippsGFpSub_PE +#define ippsGFpMul_PE p8_ippsGFpMul_PE +#define ippsGFpGetInfo p8_ippsGFpGetInfo +#define ippsGFpECGetSize p8_ippsGFpECGetSize +#define ippsGFpECInit p8_ippsGFpECInit +#define ippsGFpECSet p8_ippsGFpECSet +#define ippsGFpECSetSubgroup p8_ippsGFpECSetSubgroup +#define ippsGFpECInitStd128r1 p8_ippsGFpECInitStd128r1 +#define ippsGFpECInitStd128r2 p8_ippsGFpECInitStd128r2 +#define ippsGFpECInitStd192r1 p8_ippsGFpECInitStd192r1 +#define ippsGFpECInitStd224r1 p8_ippsGFpECInitStd224r1 +#define ippsGFpECInitStd256r1 p8_ippsGFpECInitStd256r1 +#define ippsGFpECInitStd384r1 p8_ippsGFpECInitStd384r1 +#define ippsGFpECInitStd521r1 p8_ippsGFpECInitStd521r1 +#define ippsGFpECInitStdSM2 p8_ippsGFpECInitStdSM2 +#define ippsGFpECInitStdBN256 p8_ippsGFpECInitStdBN256 +#define ippsGFpECBindGxyTblStd192r1 p8_ippsGFpECBindGxyTblStd192r1 +#define ippsGFpECBindGxyTblStd224r1 p8_ippsGFpECBindGxyTblStd224r1 +#define ippsGFpECBindGxyTblStd256r1 p8_ippsGFpECBindGxyTblStd256r1 +#define ippsGFpECBindGxyTblStd384r1 p8_ippsGFpECBindGxyTblStd384r1 +#define ippsGFpECBindGxyTblStd521r1 p8_ippsGFpECBindGxyTblStd521r1 +#define ippsGFpECBindGxyTblStdSM2 p8_ippsGFpECBindGxyTblStdSM2 +#define ippsGFpECGet p8_ippsGFpECGet +#define ippsGFpECGetSubgroup p8_ippsGFpECGetSubgroup +#define ippsGFpECScratchBufferSize p8_ippsGFpECScratchBufferSize +#define ippsGFpECVerify p8_ippsGFpECVerify +#define ippsGFpECPointGetSize p8_ippsGFpECPointGetSize +#define ippsGFpECPointInit p8_ippsGFpECPointInit +#define ippsGFpECSetPointAtInfinity p8_ippsGFpECSetPointAtInfinity +#define ippsGFpECSetPoint p8_ippsGFpECSetPoint +#define ippsGFpECSetPointRegular p8_ippsGFpECSetPointRegular +#define ippsGFpECSetPointRandom p8_ippsGFpECSetPointRandom +#define ippsGFpECMakePoint p8_ippsGFpECMakePoint +#define ippsGFpECSetPointHash p8_ippsGFpECSetPointHash +#define ippsGFpECSetPointHashBackCompatible p8_ippsGFpECSetPointHashBackCompatible +#define ippsGFpECSetPointHash_rmf p8_ippsGFpECSetPointHash_rmf +#define ippsGFpECSetPointHashBackCompatible_rmf p8_ippsGFpECSetPointHashBackCompatible_rmf +#define ippsGFpECGetPoint p8_ippsGFpECGetPoint +#define ippsGFpECGetPointRegular p8_ippsGFpECGetPointRegular +#define ippsGFpECSetPointOctString p8_ippsGFpECSetPointOctString +#define ippsGFpECGetPointOctString p8_ippsGFpECGetPointOctString +#define ippsGFpECTstPoint p8_ippsGFpECTstPoint +#define ippsGFpECTstPointInSubgroup p8_ippsGFpECTstPointInSubgroup +#define ippsGFpECCpyPoint p8_ippsGFpECCpyPoint +#define ippsGFpECCmpPoint p8_ippsGFpECCmpPoint +#define ippsGFpECNegPoint p8_ippsGFpECNegPoint +#define ippsGFpECAddPoint p8_ippsGFpECAddPoint +#define ippsGFpECMulPoint p8_ippsGFpECMulPoint +#define ippsGFpECPrivateKey p8_ippsGFpECPrivateKey +#define ippsGFpECPublicKey p8_ippsGFpECPublicKey +#define ippsGFpECTstKeyPair p8_ippsGFpECTstKeyPair +#define ippsGFpECSharedSecretDH p8_ippsGFpECSharedSecretDH +#define ippsGFpECSharedSecretDHC p8_ippsGFpECSharedSecretDHC +#define ippsGFpECMessageRepresentationSM2 p8_ippsGFpECMessageRepresentationSM2 +#define ippsGFpECSignDSA p8_ippsGFpECSignDSA +#define ippsGFpECVerifyDSA p8_ippsGFpECVerifyDSA +#define ippsGFpECSignNR p8_ippsGFpECSignNR +#define ippsGFpECVerifyNR p8_ippsGFpECVerifyNR +#define ippsGFpECSignSM2 p8_ippsGFpECSignSM2 +#define ippsGFpECVerifySM2 p8_ippsGFpECVerifySM2 +#define ippsGFpECUserIDHashSM2 p8_ippsGFpECUserIDHashSM2 +#define ippsGFpECKeyExchangeSM2_GetSize p8_ippsGFpECKeyExchangeSM2_GetSize +#define ippsGFpECKeyExchangeSM2_Init p8_ippsGFpECKeyExchangeSM2_Init +#define ippsGFpECKeyExchangeSM2_Setup p8_ippsGFpECKeyExchangeSM2_Setup +#define ippsGFpECKeyExchangeSM2_SharedKey p8_ippsGFpECKeyExchangeSM2_SharedKey +#define ippsGFpECKeyExchangeSM2_Confirm p8_ippsGFpECKeyExchangeSM2_Confirm +#define ippsGFpECGetInfo_GF p8_ippsGFpECGetInfo_GF +#define ippsGFpECESGetSize_SM2 p8_ippsGFpECESGetSize_SM2 +#define ippsGFpECESInit_SM2 p8_ippsGFpECESInit_SM2 +#define ippsGFpECESSetKey_SM2 p8_ippsGFpECESSetKey_SM2 +#define ippsGFpECESStart_SM2 p8_ippsGFpECESStart_SM2 +#define ippsGFpECESEncrypt_SM2 p8_ippsGFpECESEncrypt_SM2 +#define ippsGFpECESDecrypt_SM2 p8_ippsGFpECESDecrypt_SM2 +#define ippsGFpECESFinal_SM2 p8_ippsGFpECESFinal_SM2 +#define ippsGFpECESGetBuffersSize_SM2 p8_ippsGFpECESGetBuffersSize_SM2 +#define ippsGFpECEncryptSM2_Ext_EncMsgSize p8_ippsGFpECEncryptSM2_Ext_EncMsgSize +#define ippsGFpECEncryptSM2_Ext p8_ippsGFpECEncryptSM2_Ext +#define ippsGFpECDecryptSM2_Ext_DecMsgSize p8_ippsGFpECDecryptSM2_Ext_DecMsgSize +#define ippsGFpECDecryptSM2_Ext p8_ippsGFpECDecryptSM2_Ext diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_s8.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_s8.h new file mode 100644 index 0000000..ec0f35a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_s8.h @@ -0,0 +1,557 @@ +/******************************************************************************* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + + #define ippcpGetLibVersion s8_ippcpGetLibVersion +#define ippsDESGetSize s8_ippsDESGetSize +#define ippsDESInit s8_ippsDESInit +#define ippsDESPack s8_ippsDESPack +#define ippsDESUnpack s8_ippsDESUnpack +#define ippsTDESEncryptECB s8_ippsTDESEncryptECB +#define ippsTDESDecryptECB s8_ippsTDESDecryptECB +#define ippsTDESEncryptCBC s8_ippsTDESEncryptCBC +#define ippsTDESDecryptCBC s8_ippsTDESDecryptCBC +#define ippsTDESEncryptCFB s8_ippsTDESEncryptCFB +#define ippsTDESDecryptCFB s8_ippsTDESDecryptCFB +#define ippsTDESEncryptOFB s8_ippsTDESEncryptOFB +#define ippsTDESDecryptOFB s8_ippsTDESDecryptOFB +#define ippsTDESEncryptCTR s8_ippsTDESEncryptCTR +#define ippsTDESDecryptCTR s8_ippsTDESDecryptCTR +#define ippsAESGetSize s8_ippsAESGetSize +#define ippsAESInit s8_ippsAESInit +#define ippsAESSetKey s8_ippsAESSetKey +#define ippsAESPack s8_ippsAESPack +#define ippsAESUnpack s8_ippsAESUnpack +#define ippsAESEncryptECB s8_ippsAESEncryptECB +#define ippsAESDecryptECB s8_ippsAESDecryptECB +#define ippsAESEncryptCBC s8_ippsAESEncryptCBC +#define ippsAESEncryptCBC_CS1 s8_ippsAESEncryptCBC_CS1 +#define ippsAESEncryptCBC_CS2 s8_ippsAESEncryptCBC_CS2 +#define ippsAESEncryptCBC_CS3 s8_ippsAESEncryptCBC_CS3 +#define ippsAESDecryptCBC s8_ippsAESDecryptCBC +#define ippsAESDecryptCBC_CS1 s8_ippsAESDecryptCBC_CS1 +#define ippsAESDecryptCBC_CS2 s8_ippsAESDecryptCBC_CS2 +#define ippsAESDecryptCBC_CS3 s8_ippsAESDecryptCBC_CS3 +#define ippsAESEncryptCFB s8_ippsAESEncryptCFB +#define ippsAESDecryptCFB s8_ippsAESDecryptCFB +#define ippsAESEncryptOFB s8_ippsAESEncryptOFB +#define ippsAESDecryptOFB s8_ippsAESDecryptOFB +#define ippsAESEncryptCTR s8_ippsAESEncryptCTR +#define ippsAESDecryptCTR s8_ippsAESDecryptCTR +#define ippsAESEncryptXTS_Direct s8_ippsAESEncryptXTS_Direct +#define ippsAESDecryptXTS_Direct s8_ippsAESDecryptXTS_Direct +#define ippsAESSetupNoise s8_ippsAESSetupNoise +#define ippsAES_GCMSetupNoise s8_ippsAES_GCMSetupNoise +#define ippsAES_CMACSetupNoise s8_ippsAES_CMACSetupNoise +#define ippsAES_EncryptCFB16_MB s8_ippsAES_EncryptCFB16_MB +#define ippsSMS4GetSize s8_ippsSMS4GetSize +#define ippsSMS4Init s8_ippsSMS4Init +#define ippsSMS4SetKey s8_ippsSMS4SetKey +#define ippsSMS4EncryptECB s8_ippsSMS4EncryptECB +#define ippsSMS4DecryptECB s8_ippsSMS4DecryptECB +#define ippsSMS4EncryptCBC s8_ippsSMS4EncryptCBC +#define ippsSMS4EncryptCBC_CS1 s8_ippsSMS4EncryptCBC_CS1 +#define ippsSMS4EncryptCBC_CS2 s8_ippsSMS4EncryptCBC_CS2 +#define ippsSMS4EncryptCBC_CS3 s8_ippsSMS4EncryptCBC_CS3 +#define ippsSMS4DecryptCBC s8_ippsSMS4DecryptCBC +#define ippsSMS4DecryptCBC_CS1 s8_ippsSMS4DecryptCBC_CS1 +#define ippsSMS4DecryptCBC_CS2 s8_ippsSMS4DecryptCBC_CS2 +#define ippsSMS4DecryptCBC_CS3 s8_ippsSMS4DecryptCBC_CS3 +#define ippsSMS4EncryptCFB s8_ippsSMS4EncryptCFB +#define ippsSMS4DecryptCFB s8_ippsSMS4DecryptCFB +#define ippsSMS4EncryptOFB s8_ippsSMS4EncryptOFB +#define ippsSMS4DecryptOFB s8_ippsSMS4DecryptOFB +#define ippsSMS4EncryptCTR s8_ippsSMS4EncryptCTR +#define ippsSMS4DecryptCTR s8_ippsSMS4DecryptCTR +#define ippsSMS4_CCMGetSize s8_ippsSMS4_CCMGetSize +#define ippsSMS4_CCMInit s8_ippsSMS4_CCMInit +#define ippsSMS4_CCMMessageLen s8_ippsSMS4_CCMMessageLen +#define ippsSMS4_CCMTagLen s8_ippsSMS4_CCMTagLen +#define ippsSMS4_CCMStart s8_ippsSMS4_CCMStart +#define ippsSMS4_CCMEncrypt s8_ippsSMS4_CCMEncrypt +#define ippsSMS4_CCMDecrypt s8_ippsSMS4_CCMDecrypt +#define ippsSMS4_CCMGetTag s8_ippsSMS4_CCMGetTag +#define ippsAES_CCMGetSize s8_ippsAES_CCMGetSize +#define ippsAES_CCMInit s8_ippsAES_CCMInit +#define ippsAES_CCMMessageLen s8_ippsAES_CCMMessageLen +#define ippsAES_CCMTagLen s8_ippsAES_CCMTagLen +#define ippsAES_CCMStart s8_ippsAES_CCMStart +#define ippsAES_CCMEncrypt s8_ippsAES_CCMEncrypt +#define ippsAES_CCMDecrypt s8_ippsAES_CCMDecrypt +#define ippsAES_CCMGetTag s8_ippsAES_CCMGetTag +#define ippsAES_GCMGetSize s8_ippsAES_GCMGetSize +#define ippsAES_GCMInit s8_ippsAES_GCMInit +#define ippsAES_GCMReinit s8_ippsAES_GCMReinit +#define ippsAES_GCMReset s8_ippsAES_GCMReset +#define ippsAES_GCMProcessIV s8_ippsAES_GCMProcessIV +#define ippsAES_GCMProcessAAD s8_ippsAES_GCMProcessAAD +#define ippsAES_GCMStart s8_ippsAES_GCMStart +#define ippsAES_GCMEncrypt s8_ippsAES_GCMEncrypt +#define ippsAES_GCMDecrypt s8_ippsAES_GCMDecrypt +#define ippsAES_GCMGetTag s8_ippsAES_GCMGetTag +#define ippsAES_XTSGetSize s8_ippsAES_XTSGetSize +#define ippsAES_XTSInit s8_ippsAES_XTSInit +#define ippsAES_XTSEncrypt s8_ippsAES_XTSEncrypt +#define ippsAES_XTSDecrypt s8_ippsAES_XTSDecrypt +#define ippsAES_S2V_CMAC s8_ippsAES_S2V_CMAC +#define ippsAES_SIVEncrypt s8_ippsAES_SIVEncrypt +#define ippsAES_SIVDecrypt s8_ippsAES_SIVDecrypt +#define ippsAES_CMACGetSize s8_ippsAES_CMACGetSize +#define ippsAES_CMACInit s8_ippsAES_CMACInit +#define ippsAES_CMACUpdate s8_ippsAES_CMACUpdate +#define ippsAES_CMACFinal s8_ippsAES_CMACFinal +#define ippsAES_CMACGetTag s8_ippsAES_CMACGetTag +#define ippsARCFourCheckKey s8_ippsARCFourCheckKey +#define ippsARCFourGetSize s8_ippsARCFourGetSize +#define ippsARCFourInit s8_ippsARCFourInit +#define ippsARCFourReset s8_ippsARCFourReset +#define ippsARCFourPack s8_ippsARCFourPack +#define ippsARCFourUnpack s8_ippsARCFourUnpack +#define ippsARCFourEncrypt s8_ippsARCFourEncrypt +#define ippsARCFourDecrypt s8_ippsARCFourDecrypt +#define ippsSHA1GetSize s8_ippsSHA1GetSize +#define ippsSHA1Init s8_ippsSHA1Init +#define ippsSHA1Duplicate s8_ippsSHA1Duplicate +#define ippsSHA1Pack s8_ippsSHA1Pack +#define ippsSHA1Unpack s8_ippsSHA1Unpack +#define ippsSHA1Update s8_ippsSHA1Update +#define ippsSHA1GetTag s8_ippsSHA1GetTag +#define ippsSHA1Final s8_ippsSHA1Final +#define ippsSHA1MessageDigest s8_ippsSHA1MessageDigest +#define ippsSHA224GetSize s8_ippsSHA224GetSize +#define ippsSHA224Init s8_ippsSHA224Init +#define ippsSHA224Duplicate s8_ippsSHA224Duplicate +#define ippsSHA224Pack s8_ippsSHA224Pack +#define ippsSHA224Unpack s8_ippsSHA224Unpack +#define ippsSHA224Update s8_ippsSHA224Update +#define ippsSHA224GetTag s8_ippsSHA224GetTag +#define ippsSHA224Final s8_ippsSHA224Final +#define ippsSHA224MessageDigest s8_ippsSHA224MessageDigest +#define ippsSHA256GetSize s8_ippsSHA256GetSize +#define ippsSHA256Init s8_ippsSHA256Init +#define ippsSHA256Duplicate s8_ippsSHA256Duplicate +#define ippsSHA256Pack s8_ippsSHA256Pack +#define ippsSHA256Unpack s8_ippsSHA256Unpack +#define ippsSHA256Update s8_ippsSHA256Update +#define ippsSHA256GetTag s8_ippsSHA256GetTag +#define ippsSHA256Final s8_ippsSHA256Final +#define ippsSHA256MessageDigest s8_ippsSHA256MessageDigest +#define ippsSHA384GetSize s8_ippsSHA384GetSize +#define ippsSHA384Init s8_ippsSHA384Init +#define ippsSHA384Duplicate s8_ippsSHA384Duplicate +#define ippsSHA384Pack s8_ippsSHA384Pack +#define ippsSHA384Unpack s8_ippsSHA384Unpack +#define ippsSHA384Update s8_ippsSHA384Update +#define ippsSHA384GetTag s8_ippsSHA384GetTag +#define ippsSHA384Final s8_ippsSHA384Final +#define ippsSHA384MessageDigest s8_ippsSHA384MessageDigest +#define ippsSHA512GetSize s8_ippsSHA512GetSize +#define ippsSHA512Init s8_ippsSHA512Init +#define ippsSHA512Duplicate s8_ippsSHA512Duplicate +#define ippsSHA512Pack s8_ippsSHA512Pack +#define ippsSHA512Unpack s8_ippsSHA512Unpack +#define ippsSHA512Update s8_ippsSHA512Update +#define ippsSHA512GetTag s8_ippsSHA512GetTag +#define ippsSHA512Final s8_ippsSHA512Final +#define ippsSHA512MessageDigest s8_ippsSHA512MessageDigest +#define ippsMD5GetSize s8_ippsMD5GetSize +#define ippsMD5Init s8_ippsMD5Init +#define ippsMD5Duplicate s8_ippsMD5Duplicate +#define ippsMD5Pack s8_ippsMD5Pack +#define ippsMD5Unpack s8_ippsMD5Unpack +#define ippsMD5Update s8_ippsMD5Update +#define ippsMD5GetTag s8_ippsMD5GetTag +#define ippsMD5Final s8_ippsMD5Final +#define ippsMD5MessageDigest s8_ippsMD5MessageDigest +#define ippsSM3GetSize s8_ippsSM3GetSize +#define ippsSM3Init s8_ippsSM3Init +#define ippsSM3Duplicate s8_ippsSM3Duplicate +#define ippsSM3Pack s8_ippsSM3Pack +#define ippsSM3Unpack s8_ippsSM3Unpack +#define ippsSM3Update s8_ippsSM3Update +#define ippsSM3GetTag s8_ippsSM3GetTag +#define ippsSM3Final s8_ippsSM3Final +#define ippsSM3MessageDigest s8_ippsSM3MessageDigest +#define ippsHashGetSize s8_ippsHashGetSize +#define ippsHashInit s8_ippsHashInit +#define ippsHashPack s8_ippsHashPack +#define ippsHashUnpack s8_ippsHashUnpack +#define ippsHashDuplicate s8_ippsHashDuplicate +#define ippsHashUpdate s8_ippsHashUpdate +#define ippsHashGetTag s8_ippsHashGetTag +#define ippsHashFinal s8_ippsHashFinal +#define ippsHashMessage s8_ippsHashMessage +#define ippsHashMethod_MD5 s8_ippsHashMethod_MD5 +#define ippsHashMethod_SM3 s8_ippsHashMethod_SM3 +#define ippsHashMethod_SHA1 s8_ippsHashMethod_SHA1 +#define ippsHashMethod_SHA1_NI s8_ippsHashMethod_SHA1_NI +#define ippsHashMethod_SHA1_TT s8_ippsHashMethod_SHA1_TT +#define ippsHashMethod_SHA256 s8_ippsHashMethod_SHA256 +#define ippsHashMethod_SHA256_NI s8_ippsHashMethod_SHA256_NI +#define ippsHashMethod_SHA256_TT s8_ippsHashMethod_SHA256_TT +#define ippsHashMethod_SHA224 s8_ippsHashMethod_SHA224 +#define ippsHashMethod_SHA224_NI s8_ippsHashMethod_SHA224_NI +#define ippsHashMethod_SHA224_TT s8_ippsHashMethod_SHA224_TT +#define ippsHashMethod_SHA512 s8_ippsHashMethod_SHA512 +#define ippsHashMethod_SHA384 s8_ippsHashMethod_SHA384 +#define ippsHashMethod_SHA512_256 s8_ippsHashMethod_SHA512_256 +#define ippsHashMethod_SHA512_224 s8_ippsHashMethod_SHA512_224 +#define ippsHashMethodGetSize s8_ippsHashMethodGetSize +#define ippsHashMethodSet_MD5 s8_ippsHashMethodSet_MD5 +#define ippsHashMethodSet_SM3 s8_ippsHashMethodSet_SM3 +#define ippsHashStateMethodSet_SM3 s8_ippsHashStateMethodSet_SM3 +#define ippsHashMethodSet_SHA1 s8_ippsHashMethodSet_SHA1 +#define ippsHashMethodSet_SHA1_NI s8_ippsHashMethodSet_SHA1_NI +#define ippsHashMethodSet_SHA1_TT s8_ippsHashMethodSet_SHA1_TT +#define ippsHashMethodSet_SHA256 s8_ippsHashMethodSet_SHA256 +#define ippsHashMethodSet_SHA256_NI s8_ippsHashMethodSet_SHA256_NI +#define ippsHashMethodSet_SHA256_TT s8_ippsHashMethodSet_SHA256_TT +#define ippsHashStateMethodSet_SHA256 s8_ippsHashStateMethodSet_SHA256 +#define ippsHashStateMethodSet_SHA256_NI s8_ippsHashStateMethodSet_SHA256_NI +#define ippsHashStateMethodSet_SHA256_TT s8_ippsHashStateMethodSet_SHA256_TT +#define ippsHashMethodSet_SHA224 s8_ippsHashMethodSet_SHA224 +#define ippsHashMethodSet_SHA224_NI s8_ippsHashMethodSet_SHA224_NI +#define ippsHashMethodSet_SHA224_TT s8_ippsHashMethodSet_SHA224_TT +#define ippsHashStateMethodSet_SHA224 s8_ippsHashStateMethodSet_SHA224 +#define ippsHashStateMethodSet_SHA224_NI s8_ippsHashStateMethodSet_SHA224_NI +#define ippsHashStateMethodSet_SHA224_TT s8_ippsHashStateMethodSet_SHA224_TT +#define ippsHashMethodSet_SHA512 s8_ippsHashMethodSet_SHA512 +#define ippsHashMethodSet_SHA384 s8_ippsHashMethodSet_SHA384 +#define ippsHashMethodSet_SHA512_256 s8_ippsHashMethodSet_SHA512_256 +#define ippsHashMethodSet_SHA512_224 s8_ippsHashMethodSet_SHA512_224 +#define ippsHashStateMethodSet_SHA512 s8_ippsHashStateMethodSet_SHA512 +#define ippsHashStateMethodSet_SHA384 s8_ippsHashStateMethodSet_SHA384 +#define ippsHashStateMethodSet_SHA512_256 s8_ippsHashStateMethodSet_SHA512_256 +#define ippsHashStateMethodSet_SHA512_224 s8_ippsHashStateMethodSet_SHA512_224 +#define ippsHashGetSize_rmf s8_ippsHashGetSize_rmf +#define ippsHashInit_rmf s8_ippsHashInit_rmf +#define ippsHashPack_rmf s8_ippsHashPack_rmf +#define ippsHashUnpack_rmf s8_ippsHashUnpack_rmf +#define ippsHashDuplicate_rmf s8_ippsHashDuplicate_rmf +#define ippsHashUpdate_rmf s8_ippsHashUpdate_rmf +#define ippsHashGetTag_rmf s8_ippsHashGetTag_rmf +#define ippsHashFinal_rmf s8_ippsHashFinal_rmf +#define ippsHashMessage_rmf s8_ippsHashMessage_rmf +#define ippsHashMethodGetInfo s8_ippsHashMethodGetInfo +#define ippsHashGetInfo_rmf s8_ippsHashGetInfo_rmf +#define ippsMGF s8_ippsMGF +#define ippsMGF1_rmf s8_ippsMGF1_rmf +#define ippsMGF2_rmf s8_ippsMGF2_rmf +#define ippsHMAC_GetSize s8_ippsHMAC_GetSize +#define ippsHMAC_Init s8_ippsHMAC_Init +#define ippsHMAC_Pack s8_ippsHMAC_Pack +#define ippsHMAC_Unpack s8_ippsHMAC_Unpack +#define ippsHMAC_Duplicate s8_ippsHMAC_Duplicate +#define ippsHMAC_Update s8_ippsHMAC_Update +#define ippsHMAC_Final s8_ippsHMAC_Final +#define ippsHMAC_GetTag s8_ippsHMAC_GetTag +#define ippsHMAC_Message s8_ippsHMAC_Message +#define ippsHMACGetSize_rmf s8_ippsHMACGetSize_rmf +#define ippsHMACInit_rmf s8_ippsHMACInit_rmf +#define ippsHMACPack_rmf s8_ippsHMACPack_rmf +#define ippsHMACUnpack_rmf s8_ippsHMACUnpack_rmf +#define ippsHMACDuplicate_rmf s8_ippsHMACDuplicate_rmf +#define ippsHMACUpdate_rmf s8_ippsHMACUpdate_rmf +#define ippsHMACFinal_rmf s8_ippsHMACFinal_rmf +#define ippsHMACGetTag_rmf s8_ippsHMACGetTag_rmf +#define ippsHMACMessage_rmf s8_ippsHMACMessage_rmf +#define ippsBigNumGetSize s8_ippsBigNumGetSize +#define ippsBigNumInit s8_ippsBigNumInit +#define ippsCmpZero_BN s8_ippsCmpZero_BN +#define ippsCmp_BN s8_ippsCmp_BN +#define ippsGetSize_BN s8_ippsGetSize_BN +#define ippsSet_BN s8_ippsSet_BN +#define ippsGet_BN s8_ippsGet_BN +#define ippsRef_BN s8_ippsRef_BN +#define ippsExtGet_BN s8_ippsExtGet_BN +#define ippsAdd_BN s8_ippsAdd_BN +#define ippsSub_BN s8_ippsSub_BN +#define ippsMul_BN s8_ippsMul_BN +#define ippsMAC_BN_I s8_ippsMAC_BN_I +#define ippsDiv_BN s8_ippsDiv_BN +#define ippsMod_BN s8_ippsMod_BN +#define ippsGcd_BN s8_ippsGcd_BN +#define ippsModInv_BN s8_ippsModInv_BN +#define ippsSetOctString_BN s8_ippsSetOctString_BN +#define ippsGetOctString_BN s8_ippsGetOctString_BN +#define ippsMontGetSize s8_ippsMontGetSize +#define ippsMontInit s8_ippsMontInit +#define ippsMontSet s8_ippsMontSet +#define ippsMontGet s8_ippsMontGet +#define ippsMontForm s8_ippsMontForm +#define ippsMontMul s8_ippsMontMul +#define ippsMontExp s8_ippsMontExp +#define ippsPRNGGetSize s8_ippsPRNGGetSize +#define ippsPRNGInit s8_ippsPRNGInit +#define ippsPRNGSetModulus s8_ippsPRNGSetModulus +#define ippsPRNGSetH0 s8_ippsPRNGSetH0 +#define ippsPRNGSetAugment s8_ippsPRNGSetAugment +#define ippsPRNGSetSeed s8_ippsPRNGSetSeed +#define ippsPRNGGetSeed s8_ippsPRNGGetSeed +#define ippsPRNGen s8_ippsPRNGen +#define ippsPRNGen_BN s8_ippsPRNGen_BN +#define ippsPRNGenRDRAND s8_ippsPRNGenRDRAND +#define ippsPRNGenRDRAND_BN s8_ippsPRNGenRDRAND_BN +#define ippsTRNGenRDSEED s8_ippsTRNGenRDSEED +#define ippsTRNGenRDSEED_BN s8_ippsTRNGenRDSEED_BN +#define ippsPrimeGetSize s8_ippsPrimeGetSize +#define ippsPrimeInit s8_ippsPrimeInit +#define ippsPrimeGen s8_ippsPrimeGen +#define ippsPrimeTest s8_ippsPrimeTest +#define ippsPrimeGen_BN s8_ippsPrimeGen_BN +#define ippsPrimeTest_BN s8_ippsPrimeTest_BN +#define ippsPrimeGet s8_ippsPrimeGet +#define ippsPrimeGet_BN s8_ippsPrimeGet_BN +#define ippsPrimeSet s8_ippsPrimeSet +#define ippsPrimeSet_BN s8_ippsPrimeSet_BN +#define ippsRSA_GetSizePublicKey s8_ippsRSA_GetSizePublicKey +#define ippsRSA_InitPublicKey s8_ippsRSA_InitPublicKey +#define ippsRSA_SetPublicKey s8_ippsRSA_SetPublicKey +#define ippsRSA_GetPublicKey s8_ippsRSA_GetPublicKey +#define ippsRSA_GetSizePrivateKeyType1 s8_ippsRSA_GetSizePrivateKeyType1 +#define ippsRSA_InitPrivateKeyType1 s8_ippsRSA_InitPrivateKeyType1 +#define ippsRSA_SetPrivateKeyType1 s8_ippsRSA_SetPrivateKeyType1 +#define ippsRSA_GetPrivateKeyType1 s8_ippsRSA_GetPrivateKeyType1 +#define ippsRSA_GetSizePrivateKeyType2 s8_ippsRSA_GetSizePrivateKeyType2 +#define ippsRSA_InitPrivateKeyType2 s8_ippsRSA_InitPrivateKeyType2 +#define ippsRSA_SetPrivateKeyType2 s8_ippsRSA_SetPrivateKeyType2 +#define ippsRSA_GetPrivateKeyType2 s8_ippsRSA_GetPrivateKeyType2 +#define ippsRSA_GetBufferSizePublicKey s8_ippsRSA_GetBufferSizePublicKey +#define ippsRSA_GetBufferSizePrivateKey s8_ippsRSA_GetBufferSizePrivateKey +#define ippsRSA_Encrypt s8_ippsRSA_Encrypt +#define ippsRSA_Decrypt s8_ippsRSA_Decrypt +#define ippsRSA_GenerateKeys s8_ippsRSA_GenerateKeys +#define ippsRSA_ValidateKeys s8_ippsRSA_ValidateKeys +#define ippsRSAEncrypt_OAEP s8_ippsRSAEncrypt_OAEP +#define ippsRSADecrypt_OAEP s8_ippsRSADecrypt_OAEP +#define ippsRSAEncrypt_OAEP_rmf s8_ippsRSAEncrypt_OAEP_rmf +#define ippsRSADecrypt_OAEP_rmf s8_ippsRSADecrypt_OAEP_rmf +#define ippsRSAEncrypt_PKCSv15 s8_ippsRSAEncrypt_PKCSv15 +#define ippsRSADecrypt_PKCSv15 s8_ippsRSADecrypt_PKCSv15 +#define ippsRSASign_PSS s8_ippsRSASign_PSS +#define ippsRSAVerify_PSS s8_ippsRSAVerify_PSS +#define ippsRSASign_PSS_rmf s8_ippsRSASign_PSS_rmf +#define ippsRSAVerify_PSS_rmf s8_ippsRSAVerify_PSS_rmf +#define ippsRSASign_PKCS1v15 s8_ippsRSASign_PKCS1v15 +#define ippsRSAVerify_PKCS1v15 s8_ippsRSAVerify_PKCS1v15 +#define ippsRSASign_PKCS1v15_rmf s8_ippsRSASign_PKCS1v15_rmf +#define ippsRSAVerify_PKCS1v15_rmf s8_ippsRSAVerify_PKCS1v15_rmf +#define ippsDLGetResultString s8_ippsDLGetResultString +#define ippsDLPGetSize s8_ippsDLPGetSize +#define ippsDLPInit s8_ippsDLPInit +#define ippsDLPPack s8_ippsDLPPack +#define ippsDLPUnpack s8_ippsDLPUnpack +#define ippsDLPSet s8_ippsDLPSet +#define ippsDLPGet s8_ippsDLPGet +#define ippsDLPSetDP s8_ippsDLPSetDP +#define ippsDLPGetDP s8_ippsDLPGetDP +#define ippsDLPGenKeyPair s8_ippsDLPGenKeyPair +#define ippsDLPPublicKey s8_ippsDLPPublicKey +#define ippsDLPValidateKeyPair s8_ippsDLPValidateKeyPair +#define ippsDLPSetKeyPair s8_ippsDLPSetKeyPair +#define ippsDLPSignDSA s8_ippsDLPSignDSA +#define ippsDLPVerifyDSA s8_ippsDLPVerifyDSA +#define ippsDLPSharedSecretDH s8_ippsDLPSharedSecretDH +#define ippsDLPGenerateDSA s8_ippsDLPGenerateDSA +#define ippsDLPValidateDSA s8_ippsDLPValidateDSA +#define ippsDLPGenerateDH s8_ippsDLPGenerateDH +#define ippsDLPValidateDH s8_ippsDLPValidateDH +#define ippsECCGetResultString s8_ippsECCGetResultString +#define ippsECCPGetSize s8_ippsECCPGetSize +#define ippsECCPGetSizeStd128r1 s8_ippsECCPGetSizeStd128r1 +#define ippsECCPGetSizeStd128r2 s8_ippsECCPGetSizeStd128r2 +#define ippsECCPGetSizeStd192r1 s8_ippsECCPGetSizeStd192r1 +#define ippsECCPGetSizeStd224r1 s8_ippsECCPGetSizeStd224r1 +#define ippsECCPGetSizeStd256r1 s8_ippsECCPGetSizeStd256r1 +#define ippsECCPGetSizeStd384r1 s8_ippsECCPGetSizeStd384r1 +#define ippsECCPGetSizeStd521r1 s8_ippsECCPGetSizeStd521r1 +#define ippsECCPGetSizeStdSM2 s8_ippsECCPGetSizeStdSM2 +#define ippsECCPInit s8_ippsECCPInit +#define ippsECCPInitStd128r1 s8_ippsECCPInitStd128r1 +#define ippsECCPInitStd128r2 s8_ippsECCPInitStd128r2 +#define ippsECCPInitStd192r1 s8_ippsECCPInitStd192r1 +#define ippsECCPInitStd224r1 s8_ippsECCPInitStd224r1 +#define ippsECCPInitStd256r1 s8_ippsECCPInitStd256r1 +#define ippsECCPInitStd384r1 s8_ippsECCPInitStd384r1 +#define ippsECCPInitStd521r1 s8_ippsECCPInitStd521r1 +#define ippsECCPInitStdSM2 s8_ippsECCPInitStdSM2 +#define ippsECCPSet s8_ippsECCPSet +#define ippsECCPSetStd s8_ippsECCPSetStd +#define ippsECCPSetStd128r1 s8_ippsECCPSetStd128r1 +#define ippsECCPSetStd128r2 s8_ippsECCPSetStd128r2 +#define ippsECCPSetStd192r1 s8_ippsECCPSetStd192r1 +#define ippsECCPSetStd224r1 s8_ippsECCPSetStd224r1 +#define ippsECCPSetStd256r1 s8_ippsECCPSetStd256r1 +#define ippsECCPSetStd384r1 s8_ippsECCPSetStd384r1 +#define ippsECCPSetStd521r1 s8_ippsECCPSetStd521r1 +#define ippsECCPSetStdSM2 s8_ippsECCPSetStdSM2 +#define ippsECCPBindGxyTblStd192r1 s8_ippsECCPBindGxyTblStd192r1 +#define ippsECCPBindGxyTblStd224r1 s8_ippsECCPBindGxyTblStd224r1 +#define ippsECCPBindGxyTblStd256r1 s8_ippsECCPBindGxyTblStd256r1 +#define ippsECCPBindGxyTblStd384r1 s8_ippsECCPBindGxyTblStd384r1 +#define ippsECCPBindGxyTblStd521r1 s8_ippsECCPBindGxyTblStd521r1 +#define ippsECCPBindGxyTblStdSM2 s8_ippsECCPBindGxyTblStdSM2 +#define ippsECCPGet s8_ippsECCPGet +#define ippsECCPGetOrderBitSize s8_ippsECCPGetOrderBitSize +#define ippsECCPValidate s8_ippsECCPValidate +#define ippsECCPPointGetSize s8_ippsECCPPointGetSize +#define ippsECCPPointInit s8_ippsECCPPointInit +#define ippsECCPSetPoint s8_ippsECCPSetPoint +#define ippsECCPSetPointAtInfinity s8_ippsECCPSetPointAtInfinity +#define ippsECCPGetPoint s8_ippsECCPGetPoint +#define ippsECCPCheckPoint s8_ippsECCPCheckPoint +#define ippsECCPComparePoint s8_ippsECCPComparePoint +#define ippsECCPNegativePoint s8_ippsECCPNegativePoint +#define ippsECCPAddPoint s8_ippsECCPAddPoint +#define ippsECCPMulPointScalar s8_ippsECCPMulPointScalar +#define ippsECCPGenKeyPair s8_ippsECCPGenKeyPair +#define ippsECCPPublicKey s8_ippsECCPPublicKey +#define ippsECCPValidateKeyPair s8_ippsECCPValidateKeyPair +#define ippsECCPSetKeyPair s8_ippsECCPSetKeyPair +#define ippsECCPSharedSecretDH s8_ippsECCPSharedSecretDH +#define ippsECCPSharedSecretDHC s8_ippsECCPSharedSecretDHC +#define ippsECCPSignDSA s8_ippsECCPSignDSA +#define ippsECCPVerifyDSA s8_ippsECCPVerifyDSA +#define ippsECCPSignNR s8_ippsECCPSignNR +#define ippsECCPVerifyNR s8_ippsECCPVerifyNR +#define ippsECCPSignSM2 s8_ippsECCPSignSM2 +#define ippsECCPVerifySM2 s8_ippsECCPVerifySM2 +#define ippsGFpGetSize s8_ippsGFpGetSize +#define ippsGFpInitArbitrary s8_ippsGFpInitArbitrary +#define ippsGFpInitFixed s8_ippsGFpInitFixed +#define ippsGFpInit s8_ippsGFpInit +#define ippsGFpMethod_p192r1 s8_ippsGFpMethod_p192r1 +#define ippsGFpMethod_p224r1 s8_ippsGFpMethod_p224r1 +#define ippsGFpMethod_p256r1 s8_ippsGFpMethod_p256r1 +#define ippsGFpMethod_p384r1 s8_ippsGFpMethod_p384r1 +#define ippsGFpMethod_p521r1 s8_ippsGFpMethod_p521r1 +#define ippsGFpMethod_p256sm2 s8_ippsGFpMethod_p256sm2 +#define ippsGFpMethod_p256bn s8_ippsGFpMethod_p256bn +#define ippsGFpMethod_p256 s8_ippsGFpMethod_p256 +#define ippsGFpMethod_pArb s8_ippsGFpMethod_pArb +#define ippsGFpxGetSize s8_ippsGFpxGetSize +#define ippsGFpxInit s8_ippsGFpxInit +#define ippsGFpxInitBinomial s8_ippsGFpxInitBinomial +#define ippsGFpxMethod_binom2_epid2 s8_ippsGFpxMethod_binom2_epid2 +#define ippsGFpxMethod_binom3_epid2 s8_ippsGFpxMethod_binom3_epid2 +#define ippsGFpxMethod_binom2 s8_ippsGFpxMethod_binom2 +#define ippsGFpxMethod_binom3 s8_ippsGFpxMethod_binom3 +#define ippsGFpxMethod_binom s8_ippsGFpxMethod_binom +#define ippsGFpxMethod_com s8_ippsGFpxMethod_com +#define ippsGFpScratchBufferSize s8_ippsGFpScratchBufferSize +#define ippsGFpElementGetSize s8_ippsGFpElementGetSize +#define ippsGFpElementInit s8_ippsGFpElementInit +#define ippsGFpSetElement s8_ippsGFpSetElement +#define ippsGFpSetElementRegular s8_ippsGFpSetElementRegular +#define ippsGFpSetElementOctString s8_ippsGFpSetElementOctString +#define ippsGFpSetElementRandom s8_ippsGFpSetElementRandom +#define ippsGFpSetElementHash s8_ippsGFpSetElementHash +#define ippsGFpSetElementHash_rmf s8_ippsGFpSetElementHash_rmf +#define ippsGFpCpyElement s8_ippsGFpCpyElement +#define ippsGFpGetElement s8_ippsGFpGetElement +#define ippsGFpGetElementOctString s8_ippsGFpGetElementOctString +#define ippsGFpCmpElement s8_ippsGFpCmpElement +#define ippsGFpIsZeroElement s8_ippsGFpIsZeroElement +#define ippsGFpIsUnityElement s8_ippsGFpIsUnityElement +#define ippsGFpConj s8_ippsGFpConj +#define ippsGFpNeg s8_ippsGFpNeg +#define ippsGFpInv s8_ippsGFpInv +#define ippsGFpSqrt s8_ippsGFpSqrt +#define ippsGFpSqr s8_ippsGFpSqr +#define ippsGFpAdd s8_ippsGFpAdd +#define ippsGFpSub s8_ippsGFpSub +#define ippsGFpMul s8_ippsGFpMul +#define ippsGFpExp s8_ippsGFpExp +#define ippsGFpMultiExp s8_ippsGFpMultiExp +#define ippsGFpAdd_PE s8_ippsGFpAdd_PE +#define ippsGFpSub_PE s8_ippsGFpSub_PE +#define ippsGFpMul_PE s8_ippsGFpMul_PE +#define ippsGFpGetInfo s8_ippsGFpGetInfo +#define ippsGFpECGetSize s8_ippsGFpECGetSize +#define ippsGFpECInit s8_ippsGFpECInit +#define ippsGFpECSet s8_ippsGFpECSet +#define ippsGFpECSetSubgroup s8_ippsGFpECSetSubgroup +#define ippsGFpECInitStd128r1 s8_ippsGFpECInitStd128r1 +#define ippsGFpECInitStd128r2 s8_ippsGFpECInitStd128r2 +#define ippsGFpECInitStd192r1 s8_ippsGFpECInitStd192r1 +#define ippsGFpECInitStd224r1 s8_ippsGFpECInitStd224r1 +#define ippsGFpECInitStd256r1 s8_ippsGFpECInitStd256r1 +#define ippsGFpECInitStd384r1 s8_ippsGFpECInitStd384r1 +#define ippsGFpECInitStd521r1 s8_ippsGFpECInitStd521r1 +#define ippsGFpECInitStdSM2 s8_ippsGFpECInitStdSM2 +#define ippsGFpECInitStdBN256 s8_ippsGFpECInitStdBN256 +#define ippsGFpECBindGxyTblStd192r1 s8_ippsGFpECBindGxyTblStd192r1 +#define ippsGFpECBindGxyTblStd224r1 s8_ippsGFpECBindGxyTblStd224r1 +#define ippsGFpECBindGxyTblStd256r1 s8_ippsGFpECBindGxyTblStd256r1 +#define ippsGFpECBindGxyTblStd384r1 s8_ippsGFpECBindGxyTblStd384r1 +#define ippsGFpECBindGxyTblStd521r1 s8_ippsGFpECBindGxyTblStd521r1 +#define ippsGFpECBindGxyTblStdSM2 s8_ippsGFpECBindGxyTblStdSM2 +#define ippsGFpECGet s8_ippsGFpECGet +#define ippsGFpECGetSubgroup s8_ippsGFpECGetSubgroup +#define ippsGFpECScratchBufferSize s8_ippsGFpECScratchBufferSize +#define ippsGFpECVerify s8_ippsGFpECVerify +#define ippsGFpECPointGetSize s8_ippsGFpECPointGetSize +#define ippsGFpECPointInit s8_ippsGFpECPointInit +#define ippsGFpECSetPointAtInfinity s8_ippsGFpECSetPointAtInfinity +#define ippsGFpECSetPoint s8_ippsGFpECSetPoint +#define ippsGFpECSetPointRegular s8_ippsGFpECSetPointRegular +#define ippsGFpECSetPointRandom s8_ippsGFpECSetPointRandom +#define ippsGFpECMakePoint s8_ippsGFpECMakePoint +#define ippsGFpECSetPointHash s8_ippsGFpECSetPointHash +#define ippsGFpECSetPointHashBackCompatible s8_ippsGFpECSetPointHashBackCompatible +#define ippsGFpECSetPointHash_rmf s8_ippsGFpECSetPointHash_rmf +#define ippsGFpECSetPointHashBackCompatible_rmf s8_ippsGFpECSetPointHashBackCompatible_rmf +#define ippsGFpECGetPoint s8_ippsGFpECGetPoint +#define ippsGFpECGetPointRegular s8_ippsGFpECGetPointRegular +#define ippsGFpECSetPointOctString s8_ippsGFpECSetPointOctString +#define ippsGFpECGetPointOctString s8_ippsGFpECGetPointOctString +#define ippsGFpECTstPoint s8_ippsGFpECTstPoint +#define ippsGFpECTstPointInSubgroup s8_ippsGFpECTstPointInSubgroup +#define ippsGFpECCpyPoint s8_ippsGFpECCpyPoint +#define ippsGFpECCmpPoint s8_ippsGFpECCmpPoint +#define ippsGFpECNegPoint s8_ippsGFpECNegPoint +#define ippsGFpECAddPoint s8_ippsGFpECAddPoint +#define ippsGFpECMulPoint s8_ippsGFpECMulPoint +#define ippsGFpECPrivateKey s8_ippsGFpECPrivateKey +#define ippsGFpECPublicKey s8_ippsGFpECPublicKey +#define ippsGFpECTstKeyPair s8_ippsGFpECTstKeyPair +#define ippsGFpECSharedSecretDH s8_ippsGFpECSharedSecretDH +#define ippsGFpECSharedSecretDHC s8_ippsGFpECSharedSecretDHC +#define ippsGFpECMessageRepresentationSM2 s8_ippsGFpECMessageRepresentationSM2 +#define ippsGFpECSignDSA s8_ippsGFpECSignDSA +#define ippsGFpECVerifyDSA s8_ippsGFpECVerifyDSA +#define ippsGFpECSignNR s8_ippsGFpECSignNR +#define ippsGFpECVerifyNR s8_ippsGFpECVerifyNR +#define ippsGFpECSignSM2 s8_ippsGFpECSignSM2 +#define ippsGFpECVerifySM2 s8_ippsGFpECVerifySM2 +#define ippsGFpECUserIDHashSM2 s8_ippsGFpECUserIDHashSM2 +#define ippsGFpECKeyExchangeSM2_GetSize s8_ippsGFpECKeyExchangeSM2_GetSize +#define ippsGFpECKeyExchangeSM2_Init s8_ippsGFpECKeyExchangeSM2_Init +#define ippsGFpECKeyExchangeSM2_Setup s8_ippsGFpECKeyExchangeSM2_Setup +#define ippsGFpECKeyExchangeSM2_SharedKey s8_ippsGFpECKeyExchangeSM2_SharedKey +#define ippsGFpECKeyExchangeSM2_Confirm s8_ippsGFpECKeyExchangeSM2_Confirm +#define ippsGFpECGetInfo_GF s8_ippsGFpECGetInfo_GF +#define ippsGFpECESGetSize_SM2 s8_ippsGFpECESGetSize_SM2 +#define ippsGFpECESInit_SM2 s8_ippsGFpECESInit_SM2 +#define ippsGFpECESSetKey_SM2 s8_ippsGFpECESSetKey_SM2 +#define ippsGFpECESStart_SM2 s8_ippsGFpECESStart_SM2 +#define ippsGFpECESEncrypt_SM2 s8_ippsGFpECESEncrypt_SM2 +#define ippsGFpECESDecrypt_SM2 s8_ippsGFpECESDecrypt_SM2 +#define ippsGFpECESFinal_SM2 s8_ippsGFpECESFinal_SM2 +#define ippsGFpECESGetBuffersSize_SM2 s8_ippsGFpECESGetBuffersSize_SM2 +#define ippsGFpECEncryptSM2_Ext_EncMsgSize s8_ippsGFpECEncryptSM2_Ext_EncMsgSize +#define ippsGFpECEncryptSM2_Ext s8_ippsGFpECEncryptSM2_Ext +#define ippsGFpECDecryptSM2_Ext_DecMsgSize s8_ippsGFpECDecryptSM2_Ext_DecMsgSize +#define ippsGFpECDecryptSM2_Ext s8_ippsGFpECDecryptSM2_Ext diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_w7.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_w7.h new file mode 100644 index 0000000..5ddea08 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_w7.h @@ -0,0 +1,557 @@ +/******************************************************************************* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + + #define ippcpGetLibVersion w7_ippcpGetLibVersion +#define ippsDESGetSize w7_ippsDESGetSize +#define ippsDESInit w7_ippsDESInit +#define ippsDESPack w7_ippsDESPack +#define ippsDESUnpack w7_ippsDESUnpack +#define ippsTDESEncryptECB w7_ippsTDESEncryptECB +#define ippsTDESDecryptECB w7_ippsTDESDecryptECB +#define ippsTDESEncryptCBC w7_ippsTDESEncryptCBC +#define ippsTDESDecryptCBC w7_ippsTDESDecryptCBC +#define ippsTDESEncryptCFB w7_ippsTDESEncryptCFB +#define ippsTDESDecryptCFB w7_ippsTDESDecryptCFB +#define ippsTDESEncryptOFB w7_ippsTDESEncryptOFB +#define ippsTDESDecryptOFB w7_ippsTDESDecryptOFB +#define ippsTDESEncryptCTR w7_ippsTDESEncryptCTR +#define ippsTDESDecryptCTR w7_ippsTDESDecryptCTR +#define ippsAESGetSize w7_ippsAESGetSize +#define ippsAESInit w7_ippsAESInit +#define ippsAESSetKey w7_ippsAESSetKey +#define ippsAESPack w7_ippsAESPack +#define ippsAESUnpack w7_ippsAESUnpack +#define ippsAESEncryptECB w7_ippsAESEncryptECB +#define ippsAESDecryptECB w7_ippsAESDecryptECB +#define ippsAESEncryptCBC w7_ippsAESEncryptCBC +#define ippsAESEncryptCBC_CS1 w7_ippsAESEncryptCBC_CS1 +#define ippsAESEncryptCBC_CS2 w7_ippsAESEncryptCBC_CS2 +#define ippsAESEncryptCBC_CS3 w7_ippsAESEncryptCBC_CS3 +#define ippsAESDecryptCBC w7_ippsAESDecryptCBC +#define ippsAESDecryptCBC_CS1 w7_ippsAESDecryptCBC_CS1 +#define ippsAESDecryptCBC_CS2 w7_ippsAESDecryptCBC_CS2 +#define ippsAESDecryptCBC_CS3 w7_ippsAESDecryptCBC_CS3 +#define ippsAESEncryptCFB w7_ippsAESEncryptCFB +#define ippsAESDecryptCFB w7_ippsAESDecryptCFB +#define ippsAESEncryptOFB w7_ippsAESEncryptOFB +#define ippsAESDecryptOFB w7_ippsAESDecryptOFB +#define ippsAESEncryptCTR w7_ippsAESEncryptCTR +#define ippsAESDecryptCTR w7_ippsAESDecryptCTR +#define ippsAESEncryptXTS_Direct w7_ippsAESEncryptXTS_Direct +#define ippsAESDecryptXTS_Direct w7_ippsAESDecryptXTS_Direct +#define ippsAESSetupNoise w7_ippsAESSetupNoise +#define ippsAES_GCMSetupNoise w7_ippsAES_GCMSetupNoise +#define ippsAES_CMACSetupNoise w7_ippsAES_CMACSetupNoise +#define ippsAES_EncryptCFB16_MB w7_ippsAES_EncryptCFB16_MB +#define ippsSMS4GetSize w7_ippsSMS4GetSize +#define ippsSMS4Init w7_ippsSMS4Init +#define ippsSMS4SetKey w7_ippsSMS4SetKey +#define ippsSMS4EncryptECB w7_ippsSMS4EncryptECB +#define ippsSMS4DecryptECB w7_ippsSMS4DecryptECB +#define ippsSMS4EncryptCBC w7_ippsSMS4EncryptCBC +#define ippsSMS4EncryptCBC_CS1 w7_ippsSMS4EncryptCBC_CS1 +#define ippsSMS4EncryptCBC_CS2 w7_ippsSMS4EncryptCBC_CS2 +#define ippsSMS4EncryptCBC_CS3 w7_ippsSMS4EncryptCBC_CS3 +#define ippsSMS4DecryptCBC w7_ippsSMS4DecryptCBC +#define ippsSMS4DecryptCBC_CS1 w7_ippsSMS4DecryptCBC_CS1 +#define ippsSMS4DecryptCBC_CS2 w7_ippsSMS4DecryptCBC_CS2 +#define ippsSMS4DecryptCBC_CS3 w7_ippsSMS4DecryptCBC_CS3 +#define ippsSMS4EncryptCFB w7_ippsSMS4EncryptCFB +#define ippsSMS4DecryptCFB w7_ippsSMS4DecryptCFB +#define ippsSMS4EncryptOFB w7_ippsSMS4EncryptOFB +#define ippsSMS4DecryptOFB w7_ippsSMS4DecryptOFB +#define ippsSMS4EncryptCTR w7_ippsSMS4EncryptCTR +#define ippsSMS4DecryptCTR w7_ippsSMS4DecryptCTR +#define ippsSMS4_CCMGetSize w7_ippsSMS4_CCMGetSize +#define ippsSMS4_CCMInit w7_ippsSMS4_CCMInit +#define ippsSMS4_CCMMessageLen w7_ippsSMS4_CCMMessageLen +#define ippsSMS4_CCMTagLen w7_ippsSMS4_CCMTagLen +#define ippsSMS4_CCMStart w7_ippsSMS4_CCMStart +#define ippsSMS4_CCMEncrypt w7_ippsSMS4_CCMEncrypt +#define ippsSMS4_CCMDecrypt w7_ippsSMS4_CCMDecrypt +#define ippsSMS4_CCMGetTag w7_ippsSMS4_CCMGetTag +#define ippsAES_CCMGetSize w7_ippsAES_CCMGetSize +#define ippsAES_CCMInit w7_ippsAES_CCMInit +#define ippsAES_CCMMessageLen w7_ippsAES_CCMMessageLen +#define ippsAES_CCMTagLen w7_ippsAES_CCMTagLen +#define ippsAES_CCMStart w7_ippsAES_CCMStart +#define ippsAES_CCMEncrypt w7_ippsAES_CCMEncrypt +#define ippsAES_CCMDecrypt w7_ippsAES_CCMDecrypt +#define ippsAES_CCMGetTag w7_ippsAES_CCMGetTag +#define ippsAES_GCMGetSize w7_ippsAES_GCMGetSize +#define ippsAES_GCMInit w7_ippsAES_GCMInit +#define ippsAES_GCMReinit w7_ippsAES_GCMReinit +#define ippsAES_GCMReset w7_ippsAES_GCMReset +#define ippsAES_GCMProcessIV w7_ippsAES_GCMProcessIV +#define ippsAES_GCMProcessAAD w7_ippsAES_GCMProcessAAD +#define ippsAES_GCMStart w7_ippsAES_GCMStart +#define ippsAES_GCMEncrypt w7_ippsAES_GCMEncrypt +#define ippsAES_GCMDecrypt w7_ippsAES_GCMDecrypt +#define ippsAES_GCMGetTag w7_ippsAES_GCMGetTag +#define ippsAES_XTSGetSize w7_ippsAES_XTSGetSize +#define ippsAES_XTSInit w7_ippsAES_XTSInit +#define ippsAES_XTSEncrypt w7_ippsAES_XTSEncrypt +#define ippsAES_XTSDecrypt w7_ippsAES_XTSDecrypt +#define ippsAES_S2V_CMAC w7_ippsAES_S2V_CMAC +#define ippsAES_SIVEncrypt w7_ippsAES_SIVEncrypt +#define ippsAES_SIVDecrypt w7_ippsAES_SIVDecrypt +#define ippsAES_CMACGetSize w7_ippsAES_CMACGetSize +#define ippsAES_CMACInit w7_ippsAES_CMACInit +#define ippsAES_CMACUpdate w7_ippsAES_CMACUpdate +#define ippsAES_CMACFinal w7_ippsAES_CMACFinal +#define ippsAES_CMACGetTag w7_ippsAES_CMACGetTag +#define ippsARCFourCheckKey w7_ippsARCFourCheckKey +#define ippsARCFourGetSize w7_ippsARCFourGetSize +#define ippsARCFourInit w7_ippsARCFourInit +#define ippsARCFourReset w7_ippsARCFourReset +#define ippsARCFourPack w7_ippsARCFourPack +#define ippsARCFourUnpack w7_ippsARCFourUnpack +#define ippsARCFourEncrypt w7_ippsARCFourEncrypt +#define ippsARCFourDecrypt w7_ippsARCFourDecrypt +#define ippsSHA1GetSize w7_ippsSHA1GetSize +#define ippsSHA1Init w7_ippsSHA1Init +#define ippsSHA1Duplicate w7_ippsSHA1Duplicate +#define ippsSHA1Pack w7_ippsSHA1Pack +#define ippsSHA1Unpack w7_ippsSHA1Unpack +#define ippsSHA1Update w7_ippsSHA1Update +#define ippsSHA1GetTag w7_ippsSHA1GetTag +#define ippsSHA1Final w7_ippsSHA1Final +#define ippsSHA1MessageDigest w7_ippsSHA1MessageDigest +#define ippsSHA224GetSize w7_ippsSHA224GetSize +#define ippsSHA224Init w7_ippsSHA224Init +#define ippsSHA224Duplicate w7_ippsSHA224Duplicate +#define ippsSHA224Pack w7_ippsSHA224Pack +#define ippsSHA224Unpack w7_ippsSHA224Unpack +#define ippsSHA224Update w7_ippsSHA224Update +#define ippsSHA224GetTag w7_ippsSHA224GetTag +#define ippsSHA224Final w7_ippsSHA224Final +#define ippsSHA224MessageDigest w7_ippsSHA224MessageDigest +#define ippsSHA256GetSize w7_ippsSHA256GetSize +#define ippsSHA256Init w7_ippsSHA256Init +#define ippsSHA256Duplicate w7_ippsSHA256Duplicate +#define ippsSHA256Pack w7_ippsSHA256Pack +#define ippsSHA256Unpack w7_ippsSHA256Unpack +#define ippsSHA256Update w7_ippsSHA256Update +#define ippsSHA256GetTag w7_ippsSHA256GetTag +#define ippsSHA256Final w7_ippsSHA256Final +#define ippsSHA256MessageDigest w7_ippsSHA256MessageDigest +#define ippsSHA384GetSize w7_ippsSHA384GetSize +#define ippsSHA384Init w7_ippsSHA384Init +#define ippsSHA384Duplicate w7_ippsSHA384Duplicate +#define ippsSHA384Pack w7_ippsSHA384Pack +#define ippsSHA384Unpack w7_ippsSHA384Unpack +#define ippsSHA384Update w7_ippsSHA384Update +#define ippsSHA384GetTag w7_ippsSHA384GetTag +#define ippsSHA384Final w7_ippsSHA384Final +#define ippsSHA384MessageDigest w7_ippsSHA384MessageDigest +#define ippsSHA512GetSize w7_ippsSHA512GetSize +#define ippsSHA512Init w7_ippsSHA512Init +#define ippsSHA512Duplicate w7_ippsSHA512Duplicate +#define ippsSHA512Pack w7_ippsSHA512Pack +#define ippsSHA512Unpack w7_ippsSHA512Unpack +#define ippsSHA512Update w7_ippsSHA512Update +#define ippsSHA512GetTag w7_ippsSHA512GetTag +#define ippsSHA512Final w7_ippsSHA512Final +#define ippsSHA512MessageDigest w7_ippsSHA512MessageDigest +#define ippsMD5GetSize w7_ippsMD5GetSize +#define ippsMD5Init w7_ippsMD5Init +#define ippsMD5Duplicate w7_ippsMD5Duplicate +#define ippsMD5Pack w7_ippsMD5Pack +#define ippsMD5Unpack w7_ippsMD5Unpack +#define ippsMD5Update w7_ippsMD5Update +#define ippsMD5GetTag w7_ippsMD5GetTag +#define ippsMD5Final w7_ippsMD5Final +#define ippsMD5MessageDigest w7_ippsMD5MessageDigest +#define ippsSM3GetSize w7_ippsSM3GetSize +#define ippsSM3Init w7_ippsSM3Init +#define ippsSM3Duplicate w7_ippsSM3Duplicate +#define ippsSM3Pack w7_ippsSM3Pack +#define ippsSM3Unpack w7_ippsSM3Unpack +#define ippsSM3Update w7_ippsSM3Update +#define ippsSM3GetTag w7_ippsSM3GetTag +#define ippsSM3Final w7_ippsSM3Final +#define ippsSM3MessageDigest w7_ippsSM3MessageDigest +#define ippsHashGetSize w7_ippsHashGetSize +#define ippsHashInit w7_ippsHashInit +#define ippsHashPack w7_ippsHashPack +#define ippsHashUnpack w7_ippsHashUnpack +#define ippsHashDuplicate w7_ippsHashDuplicate +#define ippsHashUpdate w7_ippsHashUpdate +#define ippsHashGetTag w7_ippsHashGetTag +#define ippsHashFinal w7_ippsHashFinal +#define ippsHashMessage w7_ippsHashMessage +#define ippsHashMethod_MD5 w7_ippsHashMethod_MD5 +#define ippsHashMethod_SM3 w7_ippsHashMethod_SM3 +#define ippsHashMethod_SHA1 w7_ippsHashMethod_SHA1 +#define ippsHashMethod_SHA1_NI w7_ippsHashMethod_SHA1_NI +#define ippsHashMethod_SHA1_TT w7_ippsHashMethod_SHA1_TT +#define ippsHashMethod_SHA256 w7_ippsHashMethod_SHA256 +#define ippsHashMethod_SHA256_NI w7_ippsHashMethod_SHA256_NI +#define ippsHashMethod_SHA256_TT w7_ippsHashMethod_SHA256_TT +#define ippsHashMethod_SHA224 w7_ippsHashMethod_SHA224 +#define ippsHashMethod_SHA224_NI w7_ippsHashMethod_SHA224_NI +#define ippsHashMethod_SHA224_TT w7_ippsHashMethod_SHA224_TT +#define ippsHashMethod_SHA512 w7_ippsHashMethod_SHA512 +#define ippsHashMethod_SHA384 w7_ippsHashMethod_SHA384 +#define ippsHashMethod_SHA512_256 w7_ippsHashMethod_SHA512_256 +#define ippsHashMethod_SHA512_224 w7_ippsHashMethod_SHA512_224 +#define ippsHashMethodGetSize w7_ippsHashMethodGetSize +#define ippsHashMethodSet_MD5 w7_ippsHashMethodSet_MD5 +#define ippsHashMethodSet_SM3 w7_ippsHashMethodSet_SM3 +#define ippsHashStateMethodSet_SM3 w7_ippsHashStateMethodSet_SM3 +#define ippsHashMethodSet_SHA1 w7_ippsHashMethodSet_SHA1 +#define ippsHashMethodSet_SHA1_NI w7_ippsHashMethodSet_SHA1_NI +#define ippsHashMethodSet_SHA1_TT w7_ippsHashMethodSet_SHA1_TT +#define ippsHashMethodSet_SHA256 w7_ippsHashMethodSet_SHA256 +#define ippsHashMethodSet_SHA256_NI w7_ippsHashMethodSet_SHA256_NI +#define ippsHashMethodSet_SHA256_TT w7_ippsHashMethodSet_SHA256_TT +#define ippsHashStateMethodSet_SHA256 w7_ippsHashStateMethodSet_SHA256 +#define ippsHashStateMethodSet_SHA256_NI w7_ippsHashStateMethodSet_SHA256_NI +#define ippsHashStateMethodSet_SHA256_TT w7_ippsHashStateMethodSet_SHA256_TT +#define ippsHashMethodSet_SHA224 w7_ippsHashMethodSet_SHA224 +#define ippsHashMethodSet_SHA224_NI w7_ippsHashMethodSet_SHA224_NI +#define ippsHashMethodSet_SHA224_TT w7_ippsHashMethodSet_SHA224_TT +#define ippsHashStateMethodSet_SHA224 w7_ippsHashStateMethodSet_SHA224 +#define ippsHashStateMethodSet_SHA224_NI w7_ippsHashStateMethodSet_SHA224_NI +#define ippsHashStateMethodSet_SHA224_TT w7_ippsHashStateMethodSet_SHA224_TT +#define ippsHashMethodSet_SHA512 w7_ippsHashMethodSet_SHA512 +#define ippsHashMethodSet_SHA384 w7_ippsHashMethodSet_SHA384 +#define ippsHashMethodSet_SHA512_256 w7_ippsHashMethodSet_SHA512_256 +#define ippsHashMethodSet_SHA512_224 w7_ippsHashMethodSet_SHA512_224 +#define ippsHashStateMethodSet_SHA512 w7_ippsHashStateMethodSet_SHA512 +#define ippsHashStateMethodSet_SHA384 w7_ippsHashStateMethodSet_SHA384 +#define ippsHashStateMethodSet_SHA512_256 w7_ippsHashStateMethodSet_SHA512_256 +#define ippsHashStateMethodSet_SHA512_224 w7_ippsHashStateMethodSet_SHA512_224 +#define ippsHashGetSize_rmf w7_ippsHashGetSize_rmf +#define ippsHashInit_rmf w7_ippsHashInit_rmf +#define ippsHashPack_rmf w7_ippsHashPack_rmf +#define ippsHashUnpack_rmf w7_ippsHashUnpack_rmf +#define ippsHashDuplicate_rmf w7_ippsHashDuplicate_rmf +#define ippsHashUpdate_rmf w7_ippsHashUpdate_rmf +#define ippsHashGetTag_rmf w7_ippsHashGetTag_rmf +#define ippsHashFinal_rmf w7_ippsHashFinal_rmf +#define ippsHashMessage_rmf w7_ippsHashMessage_rmf +#define ippsHashMethodGetInfo w7_ippsHashMethodGetInfo +#define ippsHashGetInfo_rmf w7_ippsHashGetInfo_rmf +#define ippsMGF w7_ippsMGF +#define ippsMGF1_rmf w7_ippsMGF1_rmf +#define ippsMGF2_rmf w7_ippsMGF2_rmf +#define ippsHMAC_GetSize w7_ippsHMAC_GetSize +#define ippsHMAC_Init w7_ippsHMAC_Init +#define ippsHMAC_Pack w7_ippsHMAC_Pack +#define ippsHMAC_Unpack w7_ippsHMAC_Unpack +#define ippsHMAC_Duplicate w7_ippsHMAC_Duplicate +#define ippsHMAC_Update w7_ippsHMAC_Update +#define ippsHMAC_Final w7_ippsHMAC_Final +#define ippsHMAC_GetTag w7_ippsHMAC_GetTag +#define ippsHMAC_Message w7_ippsHMAC_Message +#define ippsHMACGetSize_rmf w7_ippsHMACGetSize_rmf +#define ippsHMACInit_rmf w7_ippsHMACInit_rmf +#define ippsHMACPack_rmf w7_ippsHMACPack_rmf +#define ippsHMACUnpack_rmf w7_ippsHMACUnpack_rmf +#define ippsHMACDuplicate_rmf w7_ippsHMACDuplicate_rmf +#define ippsHMACUpdate_rmf w7_ippsHMACUpdate_rmf +#define ippsHMACFinal_rmf w7_ippsHMACFinal_rmf +#define ippsHMACGetTag_rmf w7_ippsHMACGetTag_rmf +#define ippsHMACMessage_rmf w7_ippsHMACMessage_rmf +#define ippsBigNumGetSize w7_ippsBigNumGetSize +#define ippsBigNumInit w7_ippsBigNumInit +#define ippsCmpZero_BN w7_ippsCmpZero_BN +#define ippsCmp_BN w7_ippsCmp_BN +#define ippsGetSize_BN w7_ippsGetSize_BN +#define ippsSet_BN w7_ippsSet_BN +#define ippsGet_BN w7_ippsGet_BN +#define ippsRef_BN w7_ippsRef_BN +#define ippsExtGet_BN w7_ippsExtGet_BN +#define ippsAdd_BN w7_ippsAdd_BN +#define ippsSub_BN w7_ippsSub_BN +#define ippsMul_BN w7_ippsMul_BN +#define ippsMAC_BN_I w7_ippsMAC_BN_I +#define ippsDiv_BN w7_ippsDiv_BN +#define ippsMod_BN w7_ippsMod_BN +#define ippsGcd_BN w7_ippsGcd_BN +#define ippsModInv_BN w7_ippsModInv_BN +#define ippsSetOctString_BN w7_ippsSetOctString_BN +#define ippsGetOctString_BN w7_ippsGetOctString_BN +#define ippsMontGetSize w7_ippsMontGetSize +#define ippsMontInit w7_ippsMontInit +#define ippsMontSet w7_ippsMontSet +#define ippsMontGet w7_ippsMontGet +#define ippsMontForm w7_ippsMontForm +#define ippsMontMul w7_ippsMontMul +#define ippsMontExp w7_ippsMontExp +#define ippsPRNGGetSize w7_ippsPRNGGetSize +#define ippsPRNGInit w7_ippsPRNGInit +#define ippsPRNGSetModulus w7_ippsPRNGSetModulus +#define ippsPRNGSetH0 w7_ippsPRNGSetH0 +#define ippsPRNGSetAugment w7_ippsPRNGSetAugment +#define ippsPRNGSetSeed w7_ippsPRNGSetSeed +#define ippsPRNGGetSeed w7_ippsPRNGGetSeed +#define ippsPRNGen w7_ippsPRNGen +#define ippsPRNGen_BN w7_ippsPRNGen_BN +#define ippsPRNGenRDRAND w7_ippsPRNGenRDRAND +#define ippsPRNGenRDRAND_BN w7_ippsPRNGenRDRAND_BN +#define ippsTRNGenRDSEED w7_ippsTRNGenRDSEED +#define ippsTRNGenRDSEED_BN w7_ippsTRNGenRDSEED_BN +#define ippsPrimeGetSize w7_ippsPrimeGetSize +#define ippsPrimeInit w7_ippsPrimeInit +#define ippsPrimeGen w7_ippsPrimeGen +#define ippsPrimeTest w7_ippsPrimeTest +#define ippsPrimeGen_BN w7_ippsPrimeGen_BN +#define ippsPrimeTest_BN w7_ippsPrimeTest_BN +#define ippsPrimeGet w7_ippsPrimeGet +#define ippsPrimeGet_BN w7_ippsPrimeGet_BN +#define ippsPrimeSet w7_ippsPrimeSet +#define ippsPrimeSet_BN w7_ippsPrimeSet_BN +#define ippsRSA_GetSizePublicKey w7_ippsRSA_GetSizePublicKey +#define ippsRSA_InitPublicKey w7_ippsRSA_InitPublicKey +#define ippsRSA_SetPublicKey w7_ippsRSA_SetPublicKey +#define ippsRSA_GetPublicKey w7_ippsRSA_GetPublicKey +#define ippsRSA_GetSizePrivateKeyType1 w7_ippsRSA_GetSizePrivateKeyType1 +#define ippsRSA_InitPrivateKeyType1 w7_ippsRSA_InitPrivateKeyType1 +#define ippsRSA_SetPrivateKeyType1 w7_ippsRSA_SetPrivateKeyType1 +#define ippsRSA_GetPrivateKeyType1 w7_ippsRSA_GetPrivateKeyType1 +#define ippsRSA_GetSizePrivateKeyType2 w7_ippsRSA_GetSizePrivateKeyType2 +#define ippsRSA_InitPrivateKeyType2 w7_ippsRSA_InitPrivateKeyType2 +#define ippsRSA_SetPrivateKeyType2 w7_ippsRSA_SetPrivateKeyType2 +#define ippsRSA_GetPrivateKeyType2 w7_ippsRSA_GetPrivateKeyType2 +#define ippsRSA_GetBufferSizePublicKey w7_ippsRSA_GetBufferSizePublicKey +#define ippsRSA_GetBufferSizePrivateKey w7_ippsRSA_GetBufferSizePrivateKey +#define ippsRSA_Encrypt w7_ippsRSA_Encrypt +#define ippsRSA_Decrypt w7_ippsRSA_Decrypt +#define ippsRSA_GenerateKeys w7_ippsRSA_GenerateKeys +#define ippsRSA_ValidateKeys w7_ippsRSA_ValidateKeys +#define ippsRSAEncrypt_OAEP w7_ippsRSAEncrypt_OAEP +#define ippsRSADecrypt_OAEP w7_ippsRSADecrypt_OAEP +#define ippsRSAEncrypt_OAEP_rmf w7_ippsRSAEncrypt_OAEP_rmf +#define ippsRSADecrypt_OAEP_rmf w7_ippsRSADecrypt_OAEP_rmf +#define ippsRSAEncrypt_PKCSv15 w7_ippsRSAEncrypt_PKCSv15 +#define ippsRSADecrypt_PKCSv15 w7_ippsRSADecrypt_PKCSv15 +#define ippsRSASign_PSS w7_ippsRSASign_PSS +#define ippsRSAVerify_PSS w7_ippsRSAVerify_PSS +#define ippsRSASign_PSS_rmf w7_ippsRSASign_PSS_rmf +#define ippsRSAVerify_PSS_rmf w7_ippsRSAVerify_PSS_rmf +#define ippsRSASign_PKCS1v15 w7_ippsRSASign_PKCS1v15 +#define ippsRSAVerify_PKCS1v15 w7_ippsRSAVerify_PKCS1v15 +#define ippsRSASign_PKCS1v15_rmf w7_ippsRSASign_PKCS1v15_rmf +#define ippsRSAVerify_PKCS1v15_rmf w7_ippsRSAVerify_PKCS1v15_rmf +#define ippsDLGetResultString w7_ippsDLGetResultString +#define ippsDLPGetSize w7_ippsDLPGetSize +#define ippsDLPInit w7_ippsDLPInit +#define ippsDLPPack w7_ippsDLPPack +#define ippsDLPUnpack w7_ippsDLPUnpack +#define ippsDLPSet w7_ippsDLPSet +#define ippsDLPGet w7_ippsDLPGet +#define ippsDLPSetDP w7_ippsDLPSetDP +#define ippsDLPGetDP w7_ippsDLPGetDP +#define ippsDLPGenKeyPair w7_ippsDLPGenKeyPair +#define ippsDLPPublicKey w7_ippsDLPPublicKey +#define ippsDLPValidateKeyPair w7_ippsDLPValidateKeyPair +#define ippsDLPSetKeyPair w7_ippsDLPSetKeyPair +#define ippsDLPSignDSA w7_ippsDLPSignDSA +#define ippsDLPVerifyDSA w7_ippsDLPVerifyDSA +#define ippsDLPSharedSecretDH w7_ippsDLPSharedSecretDH +#define ippsDLPGenerateDSA w7_ippsDLPGenerateDSA +#define ippsDLPValidateDSA w7_ippsDLPValidateDSA +#define ippsDLPGenerateDH w7_ippsDLPGenerateDH +#define ippsDLPValidateDH w7_ippsDLPValidateDH +#define ippsECCGetResultString w7_ippsECCGetResultString +#define ippsECCPGetSize w7_ippsECCPGetSize +#define ippsECCPGetSizeStd128r1 w7_ippsECCPGetSizeStd128r1 +#define ippsECCPGetSizeStd128r2 w7_ippsECCPGetSizeStd128r2 +#define ippsECCPGetSizeStd192r1 w7_ippsECCPGetSizeStd192r1 +#define ippsECCPGetSizeStd224r1 w7_ippsECCPGetSizeStd224r1 +#define ippsECCPGetSizeStd256r1 w7_ippsECCPGetSizeStd256r1 +#define ippsECCPGetSizeStd384r1 w7_ippsECCPGetSizeStd384r1 +#define ippsECCPGetSizeStd521r1 w7_ippsECCPGetSizeStd521r1 +#define ippsECCPGetSizeStdSM2 w7_ippsECCPGetSizeStdSM2 +#define ippsECCPInit w7_ippsECCPInit +#define ippsECCPInitStd128r1 w7_ippsECCPInitStd128r1 +#define ippsECCPInitStd128r2 w7_ippsECCPInitStd128r2 +#define ippsECCPInitStd192r1 w7_ippsECCPInitStd192r1 +#define ippsECCPInitStd224r1 w7_ippsECCPInitStd224r1 +#define ippsECCPInitStd256r1 w7_ippsECCPInitStd256r1 +#define ippsECCPInitStd384r1 w7_ippsECCPInitStd384r1 +#define ippsECCPInitStd521r1 w7_ippsECCPInitStd521r1 +#define ippsECCPInitStdSM2 w7_ippsECCPInitStdSM2 +#define ippsECCPSet w7_ippsECCPSet +#define ippsECCPSetStd w7_ippsECCPSetStd +#define ippsECCPSetStd128r1 w7_ippsECCPSetStd128r1 +#define ippsECCPSetStd128r2 w7_ippsECCPSetStd128r2 +#define ippsECCPSetStd192r1 w7_ippsECCPSetStd192r1 +#define ippsECCPSetStd224r1 w7_ippsECCPSetStd224r1 +#define ippsECCPSetStd256r1 w7_ippsECCPSetStd256r1 +#define ippsECCPSetStd384r1 w7_ippsECCPSetStd384r1 +#define ippsECCPSetStd521r1 w7_ippsECCPSetStd521r1 +#define ippsECCPSetStdSM2 w7_ippsECCPSetStdSM2 +#define ippsECCPBindGxyTblStd192r1 w7_ippsECCPBindGxyTblStd192r1 +#define ippsECCPBindGxyTblStd224r1 w7_ippsECCPBindGxyTblStd224r1 +#define ippsECCPBindGxyTblStd256r1 w7_ippsECCPBindGxyTblStd256r1 +#define ippsECCPBindGxyTblStd384r1 w7_ippsECCPBindGxyTblStd384r1 +#define ippsECCPBindGxyTblStd521r1 w7_ippsECCPBindGxyTblStd521r1 +#define ippsECCPBindGxyTblStdSM2 w7_ippsECCPBindGxyTblStdSM2 +#define ippsECCPGet w7_ippsECCPGet +#define ippsECCPGetOrderBitSize w7_ippsECCPGetOrderBitSize +#define ippsECCPValidate w7_ippsECCPValidate +#define ippsECCPPointGetSize w7_ippsECCPPointGetSize +#define ippsECCPPointInit w7_ippsECCPPointInit +#define ippsECCPSetPoint w7_ippsECCPSetPoint +#define ippsECCPSetPointAtInfinity w7_ippsECCPSetPointAtInfinity +#define ippsECCPGetPoint w7_ippsECCPGetPoint +#define ippsECCPCheckPoint w7_ippsECCPCheckPoint +#define ippsECCPComparePoint w7_ippsECCPComparePoint +#define ippsECCPNegativePoint w7_ippsECCPNegativePoint +#define ippsECCPAddPoint w7_ippsECCPAddPoint +#define ippsECCPMulPointScalar w7_ippsECCPMulPointScalar +#define ippsECCPGenKeyPair w7_ippsECCPGenKeyPair +#define ippsECCPPublicKey w7_ippsECCPPublicKey +#define ippsECCPValidateKeyPair w7_ippsECCPValidateKeyPair +#define ippsECCPSetKeyPair w7_ippsECCPSetKeyPair +#define ippsECCPSharedSecretDH w7_ippsECCPSharedSecretDH +#define ippsECCPSharedSecretDHC w7_ippsECCPSharedSecretDHC +#define ippsECCPSignDSA w7_ippsECCPSignDSA +#define ippsECCPVerifyDSA w7_ippsECCPVerifyDSA +#define ippsECCPSignNR w7_ippsECCPSignNR +#define ippsECCPVerifyNR w7_ippsECCPVerifyNR +#define ippsECCPSignSM2 w7_ippsECCPSignSM2 +#define ippsECCPVerifySM2 w7_ippsECCPVerifySM2 +#define ippsGFpGetSize w7_ippsGFpGetSize +#define ippsGFpInitArbitrary w7_ippsGFpInitArbitrary +#define ippsGFpInitFixed w7_ippsGFpInitFixed +#define ippsGFpInit w7_ippsGFpInit +#define ippsGFpMethod_p192r1 w7_ippsGFpMethod_p192r1 +#define ippsGFpMethod_p224r1 w7_ippsGFpMethod_p224r1 +#define ippsGFpMethod_p256r1 w7_ippsGFpMethod_p256r1 +#define ippsGFpMethod_p384r1 w7_ippsGFpMethod_p384r1 +#define ippsGFpMethod_p521r1 w7_ippsGFpMethod_p521r1 +#define ippsGFpMethod_p256sm2 w7_ippsGFpMethod_p256sm2 +#define ippsGFpMethod_p256bn w7_ippsGFpMethod_p256bn +#define ippsGFpMethod_p256 w7_ippsGFpMethod_p256 +#define ippsGFpMethod_pArb w7_ippsGFpMethod_pArb +#define ippsGFpxGetSize w7_ippsGFpxGetSize +#define ippsGFpxInit w7_ippsGFpxInit +#define ippsGFpxInitBinomial w7_ippsGFpxInitBinomial +#define ippsGFpxMethod_binom2_epid2 w7_ippsGFpxMethod_binom2_epid2 +#define ippsGFpxMethod_binom3_epid2 w7_ippsGFpxMethod_binom3_epid2 +#define ippsGFpxMethod_binom2 w7_ippsGFpxMethod_binom2 +#define ippsGFpxMethod_binom3 w7_ippsGFpxMethod_binom3 +#define ippsGFpxMethod_binom w7_ippsGFpxMethod_binom +#define ippsGFpxMethod_com w7_ippsGFpxMethod_com +#define ippsGFpScratchBufferSize w7_ippsGFpScratchBufferSize +#define ippsGFpElementGetSize w7_ippsGFpElementGetSize +#define ippsGFpElementInit w7_ippsGFpElementInit +#define ippsGFpSetElement w7_ippsGFpSetElement +#define ippsGFpSetElementRegular w7_ippsGFpSetElementRegular +#define ippsGFpSetElementOctString w7_ippsGFpSetElementOctString +#define ippsGFpSetElementRandom w7_ippsGFpSetElementRandom +#define ippsGFpSetElementHash w7_ippsGFpSetElementHash +#define ippsGFpSetElementHash_rmf w7_ippsGFpSetElementHash_rmf +#define ippsGFpCpyElement w7_ippsGFpCpyElement +#define ippsGFpGetElement w7_ippsGFpGetElement +#define ippsGFpGetElementOctString w7_ippsGFpGetElementOctString +#define ippsGFpCmpElement w7_ippsGFpCmpElement +#define ippsGFpIsZeroElement w7_ippsGFpIsZeroElement +#define ippsGFpIsUnityElement w7_ippsGFpIsUnityElement +#define ippsGFpConj w7_ippsGFpConj +#define ippsGFpNeg w7_ippsGFpNeg +#define ippsGFpInv w7_ippsGFpInv +#define ippsGFpSqrt w7_ippsGFpSqrt +#define ippsGFpSqr w7_ippsGFpSqr +#define ippsGFpAdd w7_ippsGFpAdd +#define ippsGFpSub w7_ippsGFpSub +#define ippsGFpMul w7_ippsGFpMul +#define ippsGFpExp w7_ippsGFpExp +#define ippsGFpMultiExp w7_ippsGFpMultiExp +#define ippsGFpAdd_PE w7_ippsGFpAdd_PE +#define ippsGFpSub_PE w7_ippsGFpSub_PE +#define ippsGFpMul_PE w7_ippsGFpMul_PE +#define ippsGFpGetInfo w7_ippsGFpGetInfo +#define ippsGFpECGetSize w7_ippsGFpECGetSize +#define ippsGFpECInit w7_ippsGFpECInit +#define ippsGFpECSet w7_ippsGFpECSet +#define ippsGFpECSetSubgroup w7_ippsGFpECSetSubgroup +#define ippsGFpECInitStd128r1 w7_ippsGFpECInitStd128r1 +#define ippsGFpECInitStd128r2 w7_ippsGFpECInitStd128r2 +#define ippsGFpECInitStd192r1 w7_ippsGFpECInitStd192r1 +#define ippsGFpECInitStd224r1 w7_ippsGFpECInitStd224r1 +#define ippsGFpECInitStd256r1 w7_ippsGFpECInitStd256r1 +#define ippsGFpECInitStd384r1 w7_ippsGFpECInitStd384r1 +#define ippsGFpECInitStd521r1 w7_ippsGFpECInitStd521r1 +#define ippsGFpECInitStdSM2 w7_ippsGFpECInitStdSM2 +#define ippsGFpECInitStdBN256 w7_ippsGFpECInitStdBN256 +#define ippsGFpECBindGxyTblStd192r1 w7_ippsGFpECBindGxyTblStd192r1 +#define ippsGFpECBindGxyTblStd224r1 w7_ippsGFpECBindGxyTblStd224r1 +#define ippsGFpECBindGxyTblStd256r1 w7_ippsGFpECBindGxyTblStd256r1 +#define ippsGFpECBindGxyTblStd384r1 w7_ippsGFpECBindGxyTblStd384r1 +#define ippsGFpECBindGxyTblStd521r1 w7_ippsGFpECBindGxyTblStd521r1 +#define ippsGFpECBindGxyTblStdSM2 w7_ippsGFpECBindGxyTblStdSM2 +#define ippsGFpECGet w7_ippsGFpECGet +#define ippsGFpECGetSubgroup w7_ippsGFpECGetSubgroup +#define ippsGFpECScratchBufferSize w7_ippsGFpECScratchBufferSize +#define ippsGFpECVerify w7_ippsGFpECVerify +#define ippsGFpECPointGetSize w7_ippsGFpECPointGetSize +#define ippsGFpECPointInit w7_ippsGFpECPointInit +#define ippsGFpECSetPointAtInfinity w7_ippsGFpECSetPointAtInfinity +#define ippsGFpECSetPoint w7_ippsGFpECSetPoint +#define ippsGFpECSetPointRegular w7_ippsGFpECSetPointRegular +#define ippsGFpECSetPointRandom w7_ippsGFpECSetPointRandom +#define ippsGFpECMakePoint w7_ippsGFpECMakePoint +#define ippsGFpECSetPointHash w7_ippsGFpECSetPointHash +#define ippsGFpECSetPointHashBackCompatible w7_ippsGFpECSetPointHashBackCompatible +#define ippsGFpECSetPointHash_rmf w7_ippsGFpECSetPointHash_rmf +#define ippsGFpECSetPointHashBackCompatible_rmf w7_ippsGFpECSetPointHashBackCompatible_rmf +#define ippsGFpECGetPoint w7_ippsGFpECGetPoint +#define ippsGFpECGetPointRegular w7_ippsGFpECGetPointRegular +#define ippsGFpECSetPointOctString w7_ippsGFpECSetPointOctString +#define ippsGFpECGetPointOctString w7_ippsGFpECGetPointOctString +#define ippsGFpECTstPoint w7_ippsGFpECTstPoint +#define ippsGFpECTstPointInSubgroup w7_ippsGFpECTstPointInSubgroup +#define ippsGFpECCpyPoint w7_ippsGFpECCpyPoint +#define ippsGFpECCmpPoint w7_ippsGFpECCmpPoint +#define ippsGFpECNegPoint w7_ippsGFpECNegPoint +#define ippsGFpECAddPoint w7_ippsGFpECAddPoint +#define ippsGFpECMulPoint w7_ippsGFpECMulPoint +#define ippsGFpECPrivateKey w7_ippsGFpECPrivateKey +#define ippsGFpECPublicKey w7_ippsGFpECPublicKey +#define ippsGFpECTstKeyPair w7_ippsGFpECTstKeyPair +#define ippsGFpECSharedSecretDH w7_ippsGFpECSharedSecretDH +#define ippsGFpECSharedSecretDHC w7_ippsGFpECSharedSecretDHC +#define ippsGFpECMessageRepresentationSM2 w7_ippsGFpECMessageRepresentationSM2 +#define ippsGFpECSignDSA w7_ippsGFpECSignDSA +#define ippsGFpECVerifyDSA w7_ippsGFpECVerifyDSA +#define ippsGFpECSignNR w7_ippsGFpECSignNR +#define ippsGFpECVerifyNR w7_ippsGFpECVerifyNR +#define ippsGFpECSignSM2 w7_ippsGFpECSignSM2 +#define ippsGFpECVerifySM2 w7_ippsGFpECVerifySM2 +#define ippsGFpECUserIDHashSM2 w7_ippsGFpECUserIDHashSM2 +#define ippsGFpECKeyExchangeSM2_GetSize w7_ippsGFpECKeyExchangeSM2_GetSize +#define ippsGFpECKeyExchangeSM2_Init w7_ippsGFpECKeyExchangeSM2_Init +#define ippsGFpECKeyExchangeSM2_Setup w7_ippsGFpECKeyExchangeSM2_Setup +#define ippsGFpECKeyExchangeSM2_SharedKey w7_ippsGFpECKeyExchangeSM2_SharedKey +#define ippsGFpECKeyExchangeSM2_Confirm w7_ippsGFpECKeyExchangeSM2_Confirm +#define ippsGFpECGetInfo_GF w7_ippsGFpECGetInfo_GF +#define ippsGFpECESGetSize_SM2 w7_ippsGFpECESGetSize_SM2 +#define ippsGFpECESInit_SM2 w7_ippsGFpECESInit_SM2 +#define ippsGFpECESSetKey_SM2 w7_ippsGFpECESSetKey_SM2 +#define ippsGFpECESStart_SM2 w7_ippsGFpECESStart_SM2 +#define ippsGFpECESEncrypt_SM2 w7_ippsGFpECESEncrypt_SM2 +#define ippsGFpECESDecrypt_SM2 w7_ippsGFpECESDecrypt_SM2 +#define ippsGFpECESFinal_SM2 w7_ippsGFpECESFinal_SM2 +#define ippsGFpECESGetBuffersSize_SM2 w7_ippsGFpECESGetBuffersSize_SM2 +#define ippsGFpECEncryptSM2_Ext_EncMsgSize w7_ippsGFpECEncryptSM2_Ext_EncMsgSize +#define ippsGFpECEncryptSM2_Ext w7_ippsGFpECEncryptSM2_Ext +#define ippsGFpECDecryptSM2_Ext_DecMsgSize w7_ippsGFpECDecryptSM2_Ext_DecMsgSize +#define ippsGFpECDecryptSM2_Ext w7_ippsGFpECDecryptSM2_Ext diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_y8.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_y8.h new file mode 100644 index 0000000..12c727d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/include/single_cpu/ippcp_y8.h @@ -0,0 +1,557 @@ +/******************************************************************************* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + + #define ippcpGetLibVersion y8_ippcpGetLibVersion +#define ippsDESGetSize y8_ippsDESGetSize +#define ippsDESInit y8_ippsDESInit +#define ippsDESPack y8_ippsDESPack +#define ippsDESUnpack y8_ippsDESUnpack +#define ippsTDESEncryptECB y8_ippsTDESEncryptECB +#define ippsTDESDecryptECB y8_ippsTDESDecryptECB +#define ippsTDESEncryptCBC y8_ippsTDESEncryptCBC +#define ippsTDESDecryptCBC y8_ippsTDESDecryptCBC +#define ippsTDESEncryptCFB y8_ippsTDESEncryptCFB +#define ippsTDESDecryptCFB y8_ippsTDESDecryptCFB +#define ippsTDESEncryptOFB y8_ippsTDESEncryptOFB +#define ippsTDESDecryptOFB y8_ippsTDESDecryptOFB +#define ippsTDESEncryptCTR y8_ippsTDESEncryptCTR +#define ippsTDESDecryptCTR y8_ippsTDESDecryptCTR +#define ippsAESGetSize y8_ippsAESGetSize +#define ippsAESInit y8_ippsAESInit +#define ippsAESSetKey y8_ippsAESSetKey +#define ippsAESPack y8_ippsAESPack +#define ippsAESUnpack y8_ippsAESUnpack +#define ippsAESEncryptECB y8_ippsAESEncryptECB +#define ippsAESDecryptECB y8_ippsAESDecryptECB +#define ippsAESEncryptCBC y8_ippsAESEncryptCBC +#define ippsAESEncryptCBC_CS1 y8_ippsAESEncryptCBC_CS1 +#define ippsAESEncryptCBC_CS2 y8_ippsAESEncryptCBC_CS2 +#define ippsAESEncryptCBC_CS3 y8_ippsAESEncryptCBC_CS3 +#define ippsAESDecryptCBC y8_ippsAESDecryptCBC +#define ippsAESDecryptCBC_CS1 y8_ippsAESDecryptCBC_CS1 +#define ippsAESDecryptCBC_CS2 y8_ippsAESDecryptCBC_CS2 +#define ippsAESDecryptCBC_CS3 y8_ippsAESDecryptCBC_CS3 +#define ippsAESEncryptCFB y8_ippsAESEncryptCFB +#define ippsAESDecryptCFB y8_ippsAESDecryptCFB +#define ippsAESEncryptOFB y8_ippsAESEncryptOFB +#define ippsAESDecryptOFB y8_ippsAESDecryptOFB +#define ippsAESEncryptCTR y8_ippsAESEncryptCTR +#define ippsAESDecryptCTR y8_ippsAESDecryptCTR +#define ippsAESEncryptXTS_Direct y8_ippsAESEncryptXTS_Direct +#define ippsAESDecryptXTS_Direct y8_ippsAESDecryptXTS_Direct +#define ippsAESSetupNoise y8_ippsAESSetupNoise +#define ippsAES_GCMSetupNoise y8_ippsAES_GCMSetupNoise +#define ippsAES_CMACSetupNoise y8_ippsAES_CMACSetupNoise +#define ippsAES_EncryptCFB16_MB y8_ippsAES_EncryptCFB16_MB +#define ippsSMS4GetSize y8_ippsSMS4GetSize +#define ippsSMS4Init y8_ippsSMS4Init +#define ippsSMS4SetKey y8_ippsSMS4SetKey +#define ippsSMS4EncryptECB y8_ippsSMS4EncryptECB +#define ippsSMS4DecryptECB y8_ippsSMS4DecryptECB +#define ippsSMS4EncryptCBC y8_ippsSMS4EncryptCBC +#define ippsSMS4EncryptCBC_CS1 y8_ippsSMS4EncryptCBC_CS1 +#define ippsSMS4EncryptCBC_CS2 y8_ippsSMS4EncryptCBC_CS2 +#define ippsSMS4EncryptCBC_CS3 y8_ippsSMS4EncryptCBC_CS3 +#define ippsSMS4DecryptCBC y8_ippsSMS4DecryptCBC +#define ippsSMS4DecryptCBC_CS1 y8_ippsSMS4DecryptCBC_CS1 +#define ippsSMS4DecryptCBC_CS2 y8_ippsSMS4DecryptCBC_CS2 +#define ippsSMS4DecryptCBC_CS3 y8_ippsSMS4DecryptCBC_CS3 +#define ippsSMS4EncryptCFB y8_ippsSMS4EncryptCFB +#define ippsSMS4DecryptCFB y8_ippsSMS4DecryptCFB +#define ippsSMS4EncryptOFB y8_ippsSMS4EncryptOFB +#define ippsSMS4DecryptOFB y8_ippsSMS4DecryptOFB +#define ippsSMS4EncryptCTR y8_ippsSMS4EncryptCTR +#define ippsSMS4DecryptCTR y8_ippsSMS4DecryptCTR +#define ippsSMS4_CCMGetSize y8_ippsSMS4_CCMGetSize +#define ippsSMS4_CCMInit y8_ippsSMS4_CCMInit +#define ippsSMS4_CCMMessageLen y8_ippsSMS4_CCMMessageLen +#define ippsSMS4_CCMTagLen y8_ippsSMS4_CCMTagLen +#define ippsSMS4_CCMStart y8_ippsSMS4_CCMStart +#define ippsSMS4_CCMEncrypt y8_ippsSMS4_CCMEncrypt +#define ippsSMS4_CCMDecrypt y8_ippsSMS4_CCMDecrypt +#define ippsSMS4_CCMGetTag y8_ippsSMS4_CCMGetTag +#define ippsAES_CCMGetSize y8_ippsAES_CCMGetSize +#define ippsAES_CCMInit y8_ippsAES_CCMInit +#define ippsAES_CCMMessageLen y8_ippsAES_CCMMessageLen +#define ippsAES_CCMTagLen y8_ippsAES_CCMTagLen +#define ippsAES_CCMStart y8_ippsAES_CCMStart +#define ippsAES_CCMEncrypt y8_ippsAES_CCMEncrypt +#define ippsAES_CCMDecrypt y8_ippsAES_CCMDecrypt +#define ippsAES_CCMGetTag y8_ippsAES_CCMGetTag +#define ippsAES_GCMGetSize y8_ippsAES_GCMGetSize +#define ippsAES_GCMInit y8_ippsAES_GCMInit +#define ippsAES_GCMReinit y8_ippsAES_GCMReinit +#define ippsAES_GCMReset y8_ippsAES_GCMReset +#define ippsAES_GCMProcessIV y8_ippsAES_GCMProcessIV +#define ippsAES_GCMProcessAAD y8_ippsAES_GCMProcessAAD +#define ippsAES_GCMStart y8_ippsAES_GCMStart +#define ippsAES_GCMEncrypt y8_ippsAES_GCMEncrypt +#define ippsAES_GCMDecrypt y8_ippsAES_GCMDecrypt +#define ippsAES_GCMGetTag y8_ippsAES_GCMGetTag +#define ippsAES_XTSGetSize y8_ippsAES_XTSGetSize +#define ippsAES_XTSInit y8_ippsAES_XTSInit +#define ippsAES_XTSEncrypt y8_ippsAES_XTSEncrypt +#define ippsAES_XTSDecrypt y8_ippsAES_XTSDecrypt +#define ippsAES_S2V_CMAC y8_ippsAES_S2V_CMAC +#define ippsAES_SIVEncrypt y8_ippsAES_SIVEncrypt +#define ippsAES_SIVDecrypt y8_ippsAES_SIVDecrypt +#define ippsAES_CMACGetSize y8_ippsAES_CMACGetSize +#define ippsAES_CMACInit y8_ippsAES_CMACInit +#define ippsAES_CMACUpdate y8_ippsAES_CMACUpdate +#define ippsAES_CMACFinal y8_ippsAES_CMACFinal +#define ippsAES_CMACGetTag y8_ippsAES_CMACGetTag +#define ippsARCFourCheckKey y8_ippsARCFourCheckKey +#define ippsARCFourGetSize y8_ippsARCFourGetSize +#define ippsARCFourInit y8_ippsARCFourInit +#define ippsARCFourReset y8_ippsARCFourReset +#define ippsARCFourPack y8_ippsARCFourPack +#define ippsARCFourUnpack y8_ippsARCFourUnpack +#define ippsARCFourEncrypt y8_ippsARCFourEncrypt +#define ippsARCFourDecrypt y8_ippsARCFourDecrypt +#define ippsSHA1GetSize y8_ippsSHA1GetSize +#define ippsSHA1Init y8_ippsSHA1Init +#define ippsSHA1Duplicate y8_ippsSHA1Duplicate +#define ippsSHA1Pack y8_ippsSHA1Pack +#define ippsSHA1Unpack y8_ippsSHA1Unpack +#define ippsSHA1Update y8_ippsSHA1Update +#define ippsSHA1GetTag y8_ippsSHA1GetTag +#define ippsSHA1Final y8_ippsSHA1Final +#define ippsSHA1MessageDigest y8_ippsSHA1MessageDigest +#define ippsSHA224GetSize y8_ippsSHA224GetSize +#define ippsSHA224Init y8_ippsSHA224Init +#define ippsSHA224Duplicate y8_ippsSHA224Duplicate +#define ippsSHA224Pack y8_ippsSHA224Pack +#define ippsSHA224Unpack y8_ippsSHA224Unpack +#define ippsSHA224Update y8_ippsSHA224Update +#define ippsSHA224GetTag y8_ippsSHA224GetTag +#define ippsSHA224Final y8_ippsSHA224Final +#define ippsSHA224MessageDigest y8_ippsSHA224MessageDigest +#define ippsSHA256GetSize y8_ippsSHA256GetSize +#define ippsSHA256Init y8_ippsSHA256Init +#define ippsSHA256Duplicate y8_ippsSHA256Duplicate +#define ippsSHA256Pack y8_ippsSHA256Pack +#define ippsSHA256Unpack y8_ippsSHA256Unpack +#define ippsSHA256Update y8_ippsSHA256Update +#define ippsSHA256GetTag y8_ippsSHA256GetTag +#define ippsSHA256Final y8_ippsSHA256Final +#define ippsSHA256MessageDigest y8_ippsSHA256MessageDigest +#define ippsSHA384GetSize y8_ippsSHA384GetSize +#define ippsSHA384Init y8_ippsSHA384Init +#define ippsSHA384Duplicate y8_ippsSHA384Duplicate +#define ippsSHA384Pack y8_ippsSHA384Pack +#define ippsSHA384Unpack y8_ippsSHA384Unpack +#define ippsSHA384Update y8_ippsSHA384Update +#define ippsSHA384GetTag y8_ippsSHA384GetTag +#define ippsSHA384Final y8_ippsSHA384Final +#define ippsSHA384MessageDigest y8_ippsSHA384MessageDigest +#define ippsSHA512GetSize y8_ippsSHA512GetSize +#define ippsSHA512Init y8_ippsSHA512Init +#define ippsSHA512Duplicate y8_ippsSHA512Duplicate +#define ippsSHA512Pack y8_ippsSHA512Pack +#define ippsSHA512Unpack y8_ippsSHA512Unpack +#define ippsSHA512Update y8_ippsSHA512Update +#define ippsSHA512GetTag y8_ippsSHA512GetTag +#define ippsSHA512Final y8_ippsSHA512Final +#define ippsSHA512MessageDigest y8_ippsSHA512MessageDigest +#define ippsMD5GetSize y8_ippsMD5GetSize +#define ippsMD5Init y8_ippsMD5Init +#define ippsMD5Duplicate y8_ippsMD5Duplicate +#define ippsMD5Pack y8_ippsMD5Pack +#define ippsMD5Unpack y8_ippsMD5Unpack +#define ippsMD5Update y8_ippsMD5Update +#define ippsMD5GetTag y8_ippsMD5GetTag +#define ippsMD5Final y8_ippsMD5Final +#define ippsMD5MessageDigest y8_ippsMD5MessageDigest +#define ippsSM3GetSize y8_ippsSM3GetSize +#define ippsSM3Init y8_ippsSM3Init +#define ippsSM3Duplicate y8_ippsSM3Duplicate +#define ippsSM3Pack y8_ippsSM3Pack +#define ippsSM3Unpack y8_ippsSM3Unpack +#define ippsSM3Update y8_ippsSM3Update +#define ippsSM3GetTag y8_ippsSM3GetTag +#define ippsSM3Final y8_ippsSM3Final +#define ippsSM3MessageDigest y8_ippsSM3MessageDigest +#define ippsHashGetSize y8_ippsHashGetSize +#define ippsHashInit y8_ippsHashInit +#define ippsHashPack y8_ippsHashPack +#define ippsHashUnpack y8_ippsHashUnpack +#define ippsHashDuplicate y8_ippsHashDuplicate +#define ippsHashUpdate y8_ippsHashUpdate +#define ippsHashGetTag y8_ippsHashGetTag +#define ippsHashFinal y8_ippsHashFinal +#define ippsHashMessage y8_ippsHashMessage +#define ippsHashMethod_MD5 y8_ippsHashMethod_MD5 +#define ippsHashMethod_SM3 y8_ippsHashMethod_SM3 +#define ippsHashMethod_SHA1 y8_ippsHashMethod_SHA1 +#define ippsHashMethod_SHA1_NI y8_ippsHashMethod_SHA1_NI +#define ippsHashMethod_SHA1_TT y8_ippsHashMethod_SHA1_TT +#define ippsHashMethod_SHA256 y8_ippsHashMethod_SHA256 +#define ippsHashMethod_SHA256_NI y8_ippsHashMethod_SHA256_NI +#define ippsHashMethod_SHA256_TT y8_ippsHashMethod_SHA256_TT +#define ippsHashMethod_SHA224 y8_ippsHashMethod_SHA224 +#define ippsHashMethod_SHA224_NI y8_ippsHashMethod_SHA224_NI +#define ippsHashMethod_SHA224_TT y8_ippsHashMethod_SHA224_TT +#define ippsHashMethod_SHA512 y8_ippsHashMethod_SHA512 +#define ippsHashMethod_SHA384 y8_ippsHashMethod_SHA384 +#define ippsHashMethod_SHA512_256 y8_ippsHashMethod_SHA512_256 +#define ippsHashMethod_SHA512_224 y8_ippsHashMethod_SHA512_224 +#define ippsHashMethodGetSize y8_ippsHashMethodGetSize +#define ippsHashMethodSet_MD5 y8_ippsHashMethodSet_MD5 +#define ippsHashMethodSet_SM3 y8_ippsHashMethodSet_SM3 +#define ippsHashStateMethodSet_SM3 y8_ippsHashStateMethodSet_SM3 +#define ippsHashMethodSet_SHA1 y8_ippsHashMethodSet_SHA1 +#define ippsHashMethodSet_SHA1_NI y8_ippsHashMethodSet_SHA1_NI +#define ippsHashMethodSet_SHA1_TT y8_ippsHashMethodSet_SHA1_TT +#define ippsHashMethodSet_SHA256 y8_ippsHashMethodSet_SHA256 +#define ippsHashMethodSet_SHA256_NI y8_ippsHashMethodSet_SHA256_NI +#define ippsHashMethodSet_SHA256_TT y8_ippsHashMethodSet_SHA256_TT +#define ippsHashStateMethodSet_SHA256 y8_ippsHashStateMethodSet_SHA256 +#define ippsHashStateMethodSet_SHA256_NI y8_ippsHashStateMethodSet_SHA256_NI +#define ippsHashStateMethodSet_SHA256_TT y8_ippsHashStateMethodSet_SHA256_TT +#define ippsHashMethodSet_SHA224 y8_ippsHashMethodSet_SHA224 +#define ippsHashMethodSet_SHA224_NI y8_ippsHashMethodSet_SHA224_NI +#define ippsHashMethodSet_SHA224_TT y8_ippsHashMethodSet_SHA224_TT +#define ippsHashStateMethodSet_SHA224 y8_ippsHashStateMethodSet_SHA224 +#define ippsHashStateMethodSet_SHA224_NI y8_ippsHashStateMethodSet_SHA224_NI +#define ippsHashStateMethodSet_SHA224_TT y8_ippsHashStateMethodSet_SHA224_TT +#define ippsHashMethodSet_SHA512 y8_ippsHashMethodSet_SHA512 +#define ippsHashMethodSet_SHA384 y8_ippsHashMethodSet_SHA384 +#define ippsHashMethodSet_SHA512_256 y8_ippsHashMethodSet_SHA512_256 +#define ippsHashMethodSet_SHA512_224 y8_ippsHashMethodSet_SHA512_224 +#define ippsHashStateMethodSet_SHA512 y8_ippsHashStateMethodSet_SHA512 +#define ippsHashStateMethodSet_SHA384 y8_ippsHashStateMethodSet_SHA384 +#define ippsHashStateMethodSet_SHA512_256 y8_ippsHashStateMethodSet_SHA512_256 +#define ippsHashStateMethodSet_SHA512_224 y8_ippsHashStateMethodSet_SHA512_224 +#define ippsHashGetSize_rmf y8_ippsHashGetSize_rmf +#define ippsHashInit_rmf y8_ippsHashInit_rmf +#define ippsHashPack_rmf y8_ippsHashPack_rmf +#define ippsHashUnpack_rmf y8_ippsHashUnpack_rmf +#define ippsHashDuplicate_rmf y8_ippsHashDuplicate_rmf +#define ippsHashUpdate_rmf y8_ippsHashUpdate_rmf +#define ippsHashGetTag_rmf y8_ippsHashGetTag_rmf +#define ippsHashFinal_rmf y8_ippsHashFinal_rmf +#define ippsHashMessage_rmf y8_ippsHashMessage_rmf +#define ippsHashMethodGetInfo y8_ippsHashMethodGetInfo +#define ippsHashGetInfo_rmf y8_ippsHashGetInfo_rmf +#define ippsMGF y8_ippsMGF +#define ippsMGF1_rmf y8_ippsMGF1_rmf +#define ippsMGF2_rmf y8_ippsMGF2_rmf +#define ippsHMAC_GetSize y8_ippsHMAC_GetSize +#define ippsHMAC_Init y8_ippsHMAC_Init +#define ippsHMAC_Pack y8_ippsHMAC_Pack +#define ippsHMAC_Unpack y8_ippsHMAC_Unpack +#define ippsHMAC_Duplicate y8_ippsHMAC_Duplicate +#define ippsHMAC_Update y8_ippsHMAC_Update +#define ippsHMAC_Final y8_ippsHMAC_Final +#define ippsHMAC_GetTag y8_ippsHMAC_GetTag +#define ippsHMAC_Message y8_ippsHMAC_Message +#define ippsHMACGetSize_rmf y8_ippsHMACGetSize_rmf +#define ippsHMACInit_rmf y8_ippsHMACInit_rmf +#define ippsHMACPack_rmf y8_ippsHMACPack_rmf +#define ippsHMACUnpack_rmf y8_ippsHMACUnpack_rmf +#define ippsHMACDuplicate_rmf y8_ippsHMACDuplicate_rmf +#define ippsHMACUpdate_rmf y8_ippsHMACUpdate_rmf +#define ippsHMACFinal_rmf y8_ippsHMACFinal_rmf +#define ippsHMACGetTag_rmf y8_ippsHMACGetTag_rmf +#define ippsHMACMessage_rmf y8_ippsHMACMessage_rmf +#define ippsBigNumGetSize y8_ippsBigNumGetSize +#define ippsBigNumInit y8_ippsBigNumInit +#define ippsCmpZero_BN y8_ippsCmpZero_BN +#define ippsCmp_BN y8_ippsCmp_BN +#define ippsGetSize_BN y8_ippsGetSize_BN +#define ippsSet_BN y8_ippsSet_BN +#define ippsGet_BN y8_ippsGet_BN +#define ippsRef_BN y8_ippsRef_BN +#define ippsExtGet_BN y8_ippsExtGet_BN +#define ippsAdd_BN y8_ippsAdd_BN +#define ippsSub_BN y8_ippsSub_BN +#define ippsMul_BN y8_ippsMul_BN +#define ippsMAC_BN_I y8_ippsMAC_BN_I +#define ippsDiv_BN y8_ippsDiv_BN +#define ippsMod_BN y8_ippsMod_BN +#define ippsGcd_BN y8_ippsGcd_BN +#define ippsModInv_BN y8_ippsModInv_BN +#define ippsSetOctString_BN y8_ippsSetOctString_BN +#define ippsGetOctString_BN y8_ippsGetOctString_BN +#define ippsMontGetSize y8_ippsMontGetSize +#define ippsMontInit y8_ippsMontInit +#define ippsMontSet y8_ippsMontSet +#define ippsMontGet y8_ippsMontGet +#define ippsMontForm y8_ippsMontForm +#define ippsMontMul y8_ippsMontMul +#define ippsMontExp y8_ippsMontExp +#define ippsPRNGGetSize y8_ippsPRNGGetSize +#define ippsPRNGInit y8_ippsPRNGInit +#define ippsPRNGSetModulus y8_ippsPRNGSetModulus +#define ippsPRNGSetH0 y8_ippsPRNGSetH0 +#define ippsPRNGSetAugment y8_ippsPRNGSetAugment +#define ippsPRNGSetSeed y8_ippsPRNGSetSeed +#define ippsPRNGGetSeed y8_ippsPRNGGetSeed +#define ippsPRNGen y8_ippsPRNGen +#define ippsPRNGen_BN y8_ippsPRNGen_BN +#define ippsPRNGenRDRAND y8_ippsPRNGenRDRAND +#define ippsPRNGenRDRAND_BN y8_ippsPRNGenRDRAND_BN +#define ippsTRNGenRDSEED y8_ippsTRNGenRDSEED +#define ippsTRNGenRDSEED_BN y8_ippsTRNGenRDSEED_BN +#define ippsPrimeGetSize y8_ippsPrimeGetSize +#define ippsPrimeInit y8_ippsPrimeInit +#define ippsPrimeGen y8_ippsPrimeGen +#define ippsPrimeTest y8_ippsPrimeTest +#define ippsPrimeGen_BN y8_ippsPrimeGen_BN +#define ippsPrimeTest_BN y8_ippsPrimeTest_BN +#define ippsPrimeGet y8_ippsPrimeGet +#define ippsPrimeGet_BN y8_ippsPrimeGet_BN +#define ippsPrimeSet y8_ippsPrimeSet +#define ippsPrimeSet_BN y8_ippsPrimeSet_BN +#define ippsRSA_GetSizePublicKey y8_ippsRSA_GetSizePublicKey +#define ippsRSA_InitPublicKey y8_ippsRSA_InitPublicKey +#define ippsRSA_SetPublicKey y8_ippsRSA_SetPublicKey +#define ippsRSA_GetPublicKey y8_ippsRSA_GetPublicKey +#define ippsRSA_GetSizePrivateKeyType1 y8_ippsRSA_GetSizePrivateKeyType1 +#define ippsRSA_InitPrivateKeyType1 y8_ippsRSA_InitPrivateKeyType1 +#define ippsRSA_SetPrivateKeyType1 y8_ippsRSA_SetPrivateKeyType1 +#define ippsRSA_GetPrivateKeyType1 y8_ippsRSA_GetPrivateKeyType1 +#define ippsRSA_GetSizePrivateKeyType2 y8_ippsRSA_GetSizePrivateKeyType2 +#define ippsRSA_InitPrivateKeyType2 y8_ippsRSA_InitPrivateKeyType2 +#define ippsRSA_SetPrivateKeyType2 y8_ippsRSA_SetPrivateKeyType2 +#define ippsRSA_GetPrivateKeyType2 y8_ippsRSA_GetPrivateKeyType2 +#define ippsRSA_GetBufferSizePublicKey y8_ippsRSA_GetBufferSizePublicKey +#define ippsRSA_GetBufferSizePrivateKey y8_ippsRSA_GetBufferSizePrivateKey +#define ippsRSA_Encrypt y8_ippsRSA_Encrypt +#define ippsRSA_Decrypt y8_ippsRSA_Decrypt +#define ippsRSA_GenerateKeys y8_ippsRSA_GenerateKeys +#define ippsRSA_ValidateKeys y8_ippsRSA_ValidateKeys +#define ippsRSAEncrypt_OAEP y8_ippsRSAEncrypt_OAEP +#define ippsRSADecrypt_OAEP y8_ippsRSADecrypt_OAEP +#define ippsRSAEncrypt_OAEP_rmf y8_ippsRSAEncrypt_OAEP_rmf +#define ippsRSADecrypt_OAEP_rmf y8_ippsRSADecrypt_OAEP_rmf +#define ippsRSAEncrypt_PKCSv15 y8_ippsRSAEncrypt_PKCSv15 +#define ippsRSADecrypt_PKCSv15 y8_ippsRSADecrypt_PKCSv15 +#define ippsRSASign_PSS y8_ippsRSASign_PSS +#define ippsRSAVerify_PSS y8_ippsRSAVerify_PSS +#define ippsRSASign_PSS_rmf y8_ippsRSASign_PSS_rmf +#define ippsRSAVerify_PSS_rmf y8_ippsRSAVerify_PSS_rmf +#define ippsRSASign_PKCS1v15 y8_ippsRSASign_PKCS1v15 +#define ippsRSAVerify_PKCS1v15 y8_ippsRSAVerify_PKCS1v15 +#define ippsRSASign_PKCS1v15_rmf y8_ippsRSASign_PKCS1v15_rmf +#define ippsRSAVerify_PKCS1v15_rmf y8_ippsRSAVerify_PKCS1v15_rmf +#define ippsDLGetResultString y8_ippsDLGetResultString +#define ippsDLPGetSize y8_ippsDLPGetSize +#define ippsDLPInit y8_ippsDLPInit +#define ippsDLPPack y8_ippsDLPPack +#define ippsDLPUnpack y8_ippsDLPUnpack +#define ippsDLPSet y8_ippsDLPSet +#define ippsDLPGet y8_ippsDLPGet +#define ippsDLPSetDP y8_ippsDLPSetDP +#define ippsDLPGetDP y8_ippsDLPGetDP +#define ippsDLPGenKeyPair y8_ippsDLPGenKeyPair +#define ippsDLPPublicKey y8_ippsDLPPublicKey +#define ippsDLPValidateKeyPair y8_ippsDLPValidateKeyPair +#define ippsDLPSetKeyPair y8_ippsDLPSetKeyPair +#define ippsDLPSignDSA y8_ippsDLPSignDSA +#define ippsDLPVerifyDSA y8_ippsDLPVerifyDSA +#define ippsDLPSharedSecretDH y8_ippsDLPSharedSecretDH +#define ippsDLPGenerateDSA y8_ippsDLPGenerateDSA +#define ippsDLPValidateDSA y8_ippsDLPValidateDSA +#define ippsDLPGenerateDH y8_ippsDLPGenerateDH +#define ippsDLPValidateDH y8_ippsDLPValidateDH +#define ippsECCGetResultString y8_ippsECCGetResultString +#define ippsECCPGetSize y8_ippsECCPGetSize +#define ippsECCPGetSizeStd128r1 y8_ippsECCPGetSizeStd128r1 +#define ippsECCPGetSizeStd128r2 y8_ippsECCPGetSizeStd128r2 +#define ippsECCPGetSizeStd192r1 y8_ippsECCPGetSizeStd192r1 +#define ippsECCPGetSizeStd224r1 y8_ippsECCPGetSizeStd224r1 +#define ippsECCPGetSizeStd256r1 y8_ippsECCPGetSizeStd256r1 +#define ippsECCPGetSizeStd384r1 y8_ippsECCPGetSizeStd384r1 +#define ippsECCPGetSizeStd521r1 y8_ippsECCPGetSizeStd521r1 +#define ippsECCPGetSizeStdSM2 y8_ippsECCPGetSizeStdSM2 +#define ippsECCPInit y8_ippsECCPInit +#define ippsECCPInitStd128r1 y8_ippsECCPInitStd128r1 +#define ippsECCPInitStd128r2 y8_ippsECCPInitStd128r2 +#define ippsECCPInitStd192r1 y8_ippsECCPInitStd192r1 +#define ippsECCPInitStd224r1 y8_ippsECCPInitStd224r1 +#define ippsECCPInitStd256r1 y8_ippsECCPInitStd256r1 +#define ippsECCPInitStd384r1 y8_ippsECCPInitStd384r1 +#define ippsECCPInitStd521r1 y8_ippsECCPInitStd521r1 +#define ippsECCPInitStdSM2 y8_ippsECCPInitStdSM2 +#define ippsECCPSet y8_ippsECCPSet +#define ippsECCPSetStd y8_ippsECCPSetStd +#define ippsECCPSetStd128r1 y8_ippsECCPSetStd128r1 +#define ippsECCPSetStd128r2 y8_ippsECCPSetStd128r2 +#define ippsECCPSetStd192r1 y8_ippsECCPSetStd192r1 +#define ippsECCPSetStd224r1 y8_ippsECCPSetStd224r1 +#define ippsECCPSetStd256r1 y8_ippsECCPSetStd256r1 +#define ippsECCPSetStd384r1 y8_ippsECCPSetStd384r1 +#define ippsECCPSetStd521r1 y8_ippsECCPSetStd521r1 +#define ippsECCPSetStdSM2 y8_ippsECCPSetStdSM2 +#define ippsECCPBindGxyTblStd192r1 y8_ippsECCPBindGxyTblStd192r1 +#define ippsECCPBindGxyTblStd224r1 y8_ippsECCPBindGxyTblStd224r1 +#define ippsECCPBindGxyTblStd256r1 y8_ippsECCPBindGxyTblStd256r1 +#define ippsECCPBindGxyTblStd384r1 y8_ippsECCPBindGxyTblStd384r1 +#define ippsECCPBindGxyTblStd521r1 y8_ippsECCPBindGxyTblStd521r1 +#define ippsECCPBindGxyTblStdSM2 y8_ippsECCPBindGxyTblStdSM2 +#define ippsECCPGet y8_ippsECCPGet +#define ippsECCPGetOrderBitSize y8_ippsECCPGetOrderBitSize +#define ippsECCPValidate y8_ippsECCPValidate +#define ippsECCPPointGetSize y8_ippsECCPPointGetSize +#define ippsECCPPointInit y8_ippsECCPPointInit +#define ippsECCPSetPoint y8_ippsECCPSetPoint +#define ippsECCPSetPointAtInfinity y8_ippsECCPSetPointAtInfinity +#define ippsECCPGetPoint y8_ippsECCPGetPoint +#define ippsECCPCheckPoint y8_ippsECCPCheckPoint +#define ippsECCPComparePoint y8_ippsECCPComparePoint +#define ippsECCPNegativePoint y8_ippsECCPNegativePoint +#define ippsECCPAddPoint y8_ippsECCPAddPoint +#define ippsECCPMulPointScalar y8_ippsECCPMulPointScalar +#define ippsECCPGenKeyPair y8_ippsECCPGenKeyPair +#define ippsECCPPublicKey y8_ippsECCPPublicKey +#define ippsECCPValidateKeyPair y8_ippsECCPValidateKeyPair +#define ippsECCPSetKeyPair y8_ippsECCPSetKeyPair +#define ippsECCPSharedSecretDH y8_ippsECCPSharedSecretDH +#define ippsECCPSharedSecretDHC y8_ippsECCPSharedSecretDHC +#define ippsECCPSignDSA y8_ippsECCPSignDSA +#define ippsECCPVerifyDSA y8_ippsECCPVerifyDSA +#define ippsECCPSignNR y8_ippsECCPSignNR +#define ippsECCPVerifyNR y8_ippsECCPVerifyNR +#define ippsECCPSignSM2 y8_ippsECCPSignSM2 +#define ippsECCPVerifySM2 y8_ippsECCPVerifySM2 +#define ippsGFpGetSize y8_ippsGFpGetSize +#define ippsGFpInitArbitrary y8_ippsGFpInitArbitrary +#define ippsGFpInitFixed y8_ippsGFpInitFixed +#define ippsGFpInit y8_ippsGFpInit +#define ippsGFpMethod_p192r1 y8_ippsGFpMethod_p192r1 +#define ippsGFpMethod_p224r1 y8_ippsGFpMethod_p224r1 +#define ippsGFpMethod_p256r1 y8_ippsGFpMethod_p256r1 +#define ippsGFpMethod_p384r1 y8_ippsGFpMethod_p384r1 +#define ippsGFpMethod_p521r1 y8_ippsGFpMethod_p521r1 +#define ippsGFpMethod_p256sm2 y8_ippsGFpMethod_p256sm2 +#define ippsGFpMethod_p256bn y8_ippsGFpMethod_p256bn +#define ippsGFpMethod_p256 y8_ippsGFpMethod_p256 +#define ippsGFpMethod_pArb y8_ippsGFpMethod_pArb +#define ippsGFpxGetSize y8_ippsGFpxGetSize +#define ippsGFpxInit y8_ippsGFpxInit +#define ippsGFpxInitBinomial y8_ippsGFpxInitBinomial +#define ippsGFpxMethod_binom2_epid2 y8_ippsGFpxMethod_binom2_epid2 +#define ippsGFpxMethod_binom3_epid2 y8_ippsGFpxMethod_binom3_epid2 +#define ippsGFpxMethod_binom2 y8_ippsGFpxMethod_binom2 +#define ippsGFpxMethod_binom3 y8_ippsGFpxMethod_binom3 +#define ippsGFpxMethod_binom y8_ippsGFpxMethod_binom +#define ippsGFpxMethod_com y8_ippsGFpxMethod_com +#define ippsGFpScratchBufferSize y8_ippsGFpScratchBufferSize +#define ippsGFpElementGetSize y8_ippsGFpElementGetSize +#define ippsGFpElementInit y8_ippsGFpElementInit +#define ippsGFpSetElement y8_ippsGFpSetElement +#define ippsGFpSetElementRegular y8_ippsGFpSetElementRegular +#define ippsGFpSetElementOctString y8_ippsGFpSetElementOctString +#define ippsGFpSetElementRandom y8_ippsGFpSetElementRandom +#define ippsGFpSetElementHash y8_ippsGFpSetElementHash +#define ippsGFpSetElementHash_rmf y8_ippsGFpSetElementHash_rmf +#define ippsGFpCpyElement y8_ippsGFpCpyElement +#define ippsGFpGetElement y8_ippsGFpGetElement +#define ippsGFpGetElementOctString y8_ippsGFpGetElementOctString +#define ippsGFpCmpElement y8_ippsGFpCmpElement +#define ippsGFpIsZeroElement y8_ippsGFpIsZeroElement +#define ippsGFpIsUnityElement y8_ippsGFpIsUnityElement +#define ippsGFpConj y8_ippsGFpConj +#define ippsGFpNeg y8_ippsGFpNeg +#define ippsGFpInv y8_ippsGFpInv +#define ippsGFpSqrt y8_ippsGFpSqrt +#define ippsGFpSqr y8_ippsGFpSqr +#define ippsGFpAdd y8_ippsGFpAdd +#define ippsGFpSub y8_ippsGFpSub +#define ippsGFpMul y8_ippsGFpMul +#define ippsGFpExp y8_ippsGFpExp +#define ippsGFpMultiExp y8_ippsGFpMultiExp +#define ippsGFpAdd_PE y8_ippsGFpAdd_PE +#define ippsGFpSub_PE y8_ippsGFpSub_PE +#define ippsGFpMul_PE y8_ippsGFpMul_PE +#define ippsGFpGetInfo y8_ippsGFpGetInfo +#define ippsGFpECGetSize y8_ippsGFpECGetSize +#define ippsGFpECInit y8_ippsGFpECInit +#define ippsGFpECSet y8_ippsGFpECSet +#define ippsGFpECSetSubgroup y8_ippsGFpECSetSubgroup +#define ippsGFpECInitStd128r1 y8_ippsGFpECInitStd128r1 +#define ippsGFpECInitStd128r2 y8_ippsGFpECInitStd128r2 +#define ippsGFpECInitStd192r1 y8_ippsGFpECInitStd192r1 +#define ippsGFpECInitStd224r1 y8_ippsGFpECInitStd224r1 +#define ippsGFpECInitStd256r1 y8_ippsGFpECInitStd256r1 +#define ippsGFpECInitStd384r1 y8_ippsGFpECInitStd384r1 +#define ippsGFpECInitStd521r1 y8_ippsGFpECInitStd521r1 +#define ippsGFpECInitStdSM2 y8_ippsGFpECInitStdSM2 +#define ippsGFpECInitStdBN256 y8_ippsGFpECInitStdBN256 +#define ippsGFpECBindGxyTblStd192r1 y8_ippsGFpECBindGxyTblStd192r1 +#define ippsGFpECBindGxyTblStd224r1 y8_ippsGFpECBindGxyTblStd224r1 +#define ippsGFpECBindGxyTblStd256r1 y8_ippsGFpECBindGxyTblStd256r1 +#define ippsGFpECBindGxyTblStd384r1 y8_ippsGFpECBindGxyTblStd384r1 +#define ippsGFpECBindGxyTblStd521r1 y8_ippsGFpECBindGxyTblStd521r1 +#define ippsGFpECBindGxyTblStdSM2 y8_ippsGFpECBindGxyTblStdSM2 +#define ippsGFpECGet y8_ippsGFpECGet +#define ippsGFpECGetSubgroup y8_ippsGFpECGetSubgroup +#define ippsGFpECScratchBufferSize y8_ippsGFpECScratchBufferSize +#define ippsGFpECVerify y8_ippsGFpECVerify +#define ippsGFpECPointGetSize y8_ippsGFpECPointGetSize +#define ippsGFpECPointInit y8_ippsGFpECPointInit +#define ippsGFpECSetPointAtInfinity y8_ippsGFpECSetPointAtInfinity +#define ippsGFpECSetPoint y8_ippsGFpECSetPoint +#define ippsGFpECSetPointRegular y8_ippsGFpECSetPointRegular +#define ippsGFpECSetPointRandom y8_ippsGFpECSetPointRandom +#define ippsGFpECMakePoint y8_ippsGFpECMakePoint +#define ippsGFpECSetPointHash y8_ippsGFpECSetPointHash +#define ippsGFpECSetPointHashBackCompatible y8_ippsGFpECSetPointHashBackCompatible +#define ippsGFpECSetPointHash_rmf y8_ippsGFpECSetPointHash_rmf +#define ippsGFpECSetPointHashBackCompatible_rmf y8_ippsGFpECSetPointHashBackCompatible_rmf +#define ippsGFpECGetPoint y8_ippsGFpECGetPoint +#define ippsGFpECGetPointRegular y8_ippsGFpECGetPointRegular +#define ippsGFpECSetPointOctString y8_ippsGFpECSetPointOctString +#define ippsGFpECGetPointOctString y8_ippsGFpECGetPointOctString +#define ippsGFpECTstPoint y8_ippsGFpECTstPoint +#define ippsGFpECTstPointInSubgroup y8_ippsGFpECTstPointInSubgroup +#define ippsGFpECCpyPoint y8_ippsGFpECCpyPoint +#define ippsGFpECCmpPoint y8_ippsGFpECCmpPoint +#define ippsGFpECNegPoint y8_ippsGFpECNegPoint +#define ippsGFpECAddPoint y8_ippsGFpECAddPoint +#define ippsGFpECMulPoint y8_ippsGFpECMulPoint +#define ippsGFpECPrivateKey y8_ippsGFpECPrivateKey +#define ippsGFpECPublicKey y8_ippsGFpECPublicKey +#define ippsGFpECTstKeyPair y8_ippsGFpECTstKeyPair +#define ippsGFpECSharedSecretDH y8_ippsGFpECSharedSecretDH +#define ippsGFpECSharedSecretDHC y8_ippsGFpECSharedSecretDHC +#define ippsGFpECMessageRepresentationSM2 y8_ippsGFpECMessageRepresentationSM2 +#define ippsGFpECSignDSA y8_ippsGFpECSignDSA +#define ippsGFpECVerifyDSA y8_ippsGFpECVerifyDSA +#define ippsGFpECSignNR y8_ippsGFpECSignNR +#define ippsGFpECVerifyNR y8_ippsGFpECVerifyNR +#define ippsGFpECSignSM2 y8_ippsGFpECSignSM2 +#define ippsGFpECVerifySM2 y8_ippsGFpECVerifySM2 +#define ippsGFpECUserIDHashSM2 y8_ippsGFpECUserIDHashSM2 +#define ippsGFpECKeyExchangeSM2_GetSize y8_ippsGFpECKeyExchangeSM2_GetSize +#define ippsGFpECKeyExchangeSM2_Init y8_ippsGFpECKeyExchangeSM2_Init +#define ippsGFpECKeyExchangeSM2_Setup y8_ippsGFpECKeyExchangeSM2_Setup +#define ippsGFpECKeyExchangeSM2_SharedKey y8_ippsGFpECKeyExchangeSM2_SharedKey +#define ippsGFpECKeyExchangeSM2_Confirm y8_ippsGFpECKeyExchangeSM2_Confirm +#define ippsGFpECGetInfo_GF y8_ippsGFpECGetInfo_GF +#define ippsGFpECESGetSize_SM2 y8_ippsGFpECESGetSize_SM2 +#define ippsGFpECESInit_SM2 y8_ippsGFpECESInit_SM2 +#define ippsGFpECESSetKey_SM2 y8_ippsGFpECESSetKey_SM2 +#define ippsGFpECESStart_SM2 y8_ippsGFpECESStart_SM2 +#define ippsGFpECESEncrypt_SM2 y8_ippsGFpECESEncrypt_SM2 +#define ippsGFpECESDecrypt_SM2 y8_ippsGFpECESDecrypt_SM2 +#define ippsGFpECESFinal_SM2 y8_ippsGFpECESFinal_SM2 +#define ippsGFpECESGetBuffersSize_SM2 y8_ippsGFpECESGetBuffersSize_SM2 +#define ippsGFpECEncryptSM2_Ext_EncMsgSize y8_ippsGFpECEncryptSM2_Ext_EncMsgSize +#define ippsGFpECEncryptSM2_Ext y8_ippsGFpECEncryptSM2_Ext +#define ippsGFpECDecryptSM2_Ext_DecMsgSize y8_ippsGFpECDecryptSM2_Ext_DecMsgSize +#define ippsGFpECDecryptSM2_Ext y8_ippsGFpECDecryptSM2_Ext diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/lib/libippcp.a b/librocksdb-sys/rocksdb/plugin/ippcp/library/lib/libippcp.a new file mode 100644 index 0000000..a5c22c1 Binary files /dev/null and b/librocksdb-sys/rocksdb/plugin/ippcp/library/lib/libippcp.a differ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/.clang-format b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/.clang-format new file mode 100644 index 0000000..ce65d7e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/.clang-format @@ -0,0 +1,35 @@ + +#=============================================================================== +# Copyright (C) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. +# +#=============================================================================== + +BasedOnStyle: WebKit + +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: true +AlignEscapedNewlines: Left +AlignOperands: true +AlignTrailingComments: true +AllowAllParametersOfDeclarationOnNextLine: false +BinPackArguments: false +BinPackParameters: false +BreakBeforeBinaryOperators: None +ColumnLimit: 150 +ContinuationIndentWidth: 3 +IndentWidth: 3 +MaxEmptyLinesToKeep: 3 +PointerAlignment: Right +SortIncludes: false diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/.gitattributes b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/.gitattributes new file mode 100644 index 0000000..30e4773 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/.gitattributes @@ -0,0 +1,33 @@ +# Set the default behavior, in case people don't have core.autocrlf set. +* text=auto + +# Explicitly declare text files you want to always be normalized and converted +# to native line endings on checkout. +*.c text +*.h text +*.cpp text +*.def text +*.rc text +*.i text +*.sh text +*.csh text +*.mk text +*.java text +*.csv text +*.lst text + +# Declare files that will always have CRLF line endings on checkout. +*.sln text eol=crlf +*.bat text eol=crlf + +# Declare files that will always have LF line endings on checkout. +*.py text eol=lf + +# Denote all files that are truly binary and should not be modified. +*.png binary +*.jpg binary +*.lib binary +*.dll binary +*.dylib binary +*.so binary +*.a binary diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/BUILD.md b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/BUILD.md new file mode 100644 index 0000000..c23d5bd --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/BUILD.md @@ -0,0 +1,304 @@ +# How to Build Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) + +- [Software Requirements](#software-requirements) + - [Common tools](#common-tools) + - [Linux* OS](#linux-os) + - [Windows* OS](#windows-os) + - [macOS*](#macos) +- [Building Intel IPP Cryptography on Linux\* OS](#building-intel-ipp-cryptography-on-linux-os) +- [Building Intel IPP Cryptography on Windows\* OS](#building-intel-ipp-cryptography-on-windows-os) +- [Building Intel IPP Cryptography on macOS\*](#building-intel-ipp-cryptography-on-macos) +- [CMake Build Options](#cmake-build-options) + - [Common for all operating systems](#common-for-all-operating-systems) + - [Windows\* OS](#windows-os) + - [Linux\* OS](#linux-os) +- [CMake Commands FAQ](#cmake-commands-faq) + - [How to build a 32-bit library?](#how-to-build-a-32-bit-library) + - [How to build a 64-bit generic library without any CPU-specific optimizations?](#how-to-build-a-64-bit-generic-library-without-any-cpu-specific-optimizations) + - [How to build two libraries with optimizations for Intel® Advanced Vector Extensions 2 and Intel® Advanced Vector Extensions 512 instruction sets?](#how-to-build-two-libraries-with-optimizations-for-intel-advanced-vector-extensions-2-and-intel-advanced-vector-extensions-512-instruction-sets) + - [How to build a library to work in a kernel space?](#how-to-build-a-library-to-work-in-a-kernel-space) +- [Incorporating Intel® IPP Cryptography sources into custom build system](#incorporating-intel-ipp-cryptography-sources-into-custom-build-system) + + +## Software Requirements +### Common tools +- [CMake\*](https://cmake.org/download) 3.15 or higher +- Python 3.8.1 +- The Netwide Assembler (NASM) 2.15 +- OpenSSL\* 3.0.8 or higher + +### Linux* OS +- [Common tools](#common-tools) +- Intel® C++ Compiler Classic 2021.9 for Linux\* OS +- GCC 8.3 +- GCC 9.1 +- GCC 10.1 +- GCC 11.1 +- Clang 9.0 +- Clang 12.0 +- GNU binutils 2.32 +### Windows* OS +- [Common tools](#common-tools) +- Intel® C++ Compiler Classic 2021.9 for Windows\* OS +- Microsoft Visual C++ Compiler\* version 19.16 provided by Microsoft Visual Studio\* 2017 version 15.9 +> **NOTE:** Support for this compiler version will be removed from Intel IPP Cryptography starting 2021.4 release. If you use it for building Intel IPP Cryptography library, please plan on migrating to a newer supported version of Microsoft Visual C++ Compiler\*. +- Microsoft Visual C++ Compiler\* version 19.24 provided by Microsoft Visual Studio\* 2019 version 16.4 +- Microsoft Visual C++ Compiler\* version 19.30 provided by Microsoft Visual Studio\* 2022 version 17.0 +> **NOTE:** [CMake\*](https://cmake.org/download) 3.21 or higher is required to build using Microsoft Visual Studio\* 2022. +### macOS* +- [Common tools](#common-tools) +- Intel® C++ Compiler Classic 2021.9 for macOS\* +## Building Intel IPP Cryptography on Linux\* OS + +The software was validated on: + +- Red Hat\* Enterprise Linux\* 8 + +To build the Intel IPP Cryptography library on Linux\* OS, complete the following steps: +1. Clone the source code from GitHub\* as follows: + + ``` bash + git clone --recursive https://github.com/intel/ipp-crypto + ``` + +2. Set the environment for one of the supported C/C++ compilers. + + *Example for Intel® Compiler:* + + ```bash + source /opt/intel/bin/compilervars.sh intel64 + ``` + + For details, refer to the [Intel® C++ Compiler Developer Guide and Reference](https://www.intel.com/content/www/us/en/docs/cpp-compiler/developer-guide-reference/current/specifying-the-location-of-compiler-components.html). + +3. Run CMake\* in the command line. + + *Examples*: + + For Intel® C++ Compiler: + + ``` bash + CC=icc CXX=icpc cmake CMakeLists.txt -B_build -DARCH=intel64 + ``` + + For GCC: + + ``` bash + CC=gcc CXX=g++ cmake CMakeLists.txt -B_build -DARCH=intel64 + ``` + + For the complete list of supported CMake options, refer to the [CMake Build Options](#cmake-build-options) section. + +4. Navigate to the build folder specified in the CMake command line and start the build: + + ```bash + cd _build + make all + ``` + + You can find the built libraries in the `/.build//lib` directory. + +## Building Intel IPP Cryptography on Windows\* OS + +The software was validated on: + +- Windows Server\* 2019 + +To build the Intel IPP Cryptography library on Windows* OS, complete the following steps: + +1. Clone the source code from GitHub\* as follows: + + ``` bash + git clone --recursive https://github.com/intel/ipp-crypto + ``` + +2. Set the environment variables for one of the supported C/C++ compilers. + For Intel® Compiler instructions, refer to the [Intel® C++ Compiler Developer Guide and Reference](https://www.intel.com/content/www/us/en/docs/cpp-compiler/developer-guide-reference/current/overview.html). + For MSVC* Compiler, refer to [Use the MSVC toolset from the command line](https://docs.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=vs-2017). + +3. Run CMake\* in the command line. + + *Examples*: + + For Intel® C++ Compiler and Visual Studio\* 2019: + + ``` bash + cmake CMakeLists.txt -B_build -G"Visual Studio 16 2019" -T"Intel C++ Compiler 19.2" -Ax64 + ``` + + For MSVC\* Compiler and Visual Studio\* 2019: + + ``` bash + cmake CMakeLists.txt -B_build -G"Visual Studio 16 2019" -Ax64 + ``` + + For the complete list of supported CMake options, please refer to the [CMake Build Options](#cmake-build-options) section. + +4. Navigate to the build folder, specified in the CMake command line and start build either from Visual Studio\* or in the command line. + + *Build from command line:* + + ```bash + cmake --build . --parallel 4 --target ALL_BUILD --config Release + ``` + + *Build from Visual Studio\*:* + Open the Microsoft Visual Studio\* solution `Intel(R) IPP Crypto.sln`, choose project (build target) from the Solution Explorer and run the build. + +## Building Intel IPP Cryptography on macOS\* + +The software was validated on: + +- macOS\* 12.0 + +To build the Intel IPP Cryptography library on macOS*, complete the following steps: + +1. Clone the source code from GitHub\* as follows: + + ``` bash + git clone --recursive https://github.com/intel/ipp-crypto + ``` + +2. Set the environment variables for one of the supported C/C++ compilers. + + *Example for Intel® Compiler:* + + ```bash + source /opt/intel/bin/compilervars.sh intel64 + ``` + + For details, refer to the [Intel® C++ Compiler Developer Guide and Reference](https://www.intel.com/content/www/us/en/docs/cpp-compiler/developer-guide-reference/current/specifying-the-location-of-compiler-components.html) + +3. Run CMake\* in the command line. + + *Examples*: + + For Intel® C++ Compiler: + + ``` bash + CC=icc CXX=icpc cmake CMakeLists.txt -B_build -DARCH=intel64 + ``` + + For the complete list of supported CMake options, refer to the [CMake Build Options](#cmake-build-options) section. + +4. Navigate to the build folder specified in the CMake command line and start the build: + + ```bash + cd _build + make all + ``` + + You can find the built libraries in the `/.build//lib` directory. + +## CMake Build Options + +### Common for all operating systems + +- `-B` - defines the build directory. This is the directory where CMake puts the generated Microsoft Visual Studio\* solution or makefiles. + +- `-DARCH=` - on Linux* OS and macOS*, defines the target architecture for the build of the Intel IPP Cryptography library. + > **NOTE:** On Windows* OS, use `-G`/`-A` instead. See the description of these options [below](#windows-os-1). + +- `-DMERGED_BLD:BOOL=` - optional. Defines the configuration of the Intel IPP Cryptography library to build: + + - `-DMERGED_BLD:BOOL=on`: default configuration. It includes the following steps: + - Build of a dispatched static library with all available optimizations + - Build of a dispatched dynamic library with all available optimizations + - Generation of the single-CPU headers (for details, refer to [this](./OVERVIEW.md) section) + + - `-DMERGED_BLD:BOOL=off`: build of one static library per optimization; build of one dynamic library per optimization. + +- `-DPLATFORM_LIST=""` - optional, works only if `-DMERGED_BLD:BOOL=off` is set. Sets target platforms for the code to be compiled. See the supported platforms list [here](./OVERVIEW.md). + + - Example for Linux\* OS and the IA-32 architecture: + `-DPLATFORM_LIST="m7;s8;p8;g9;h9"` + + - Example for Linux\* OS and the Intel® 64 architecture: + `-DPLATFORM_LIST="w7;n8;y8;e9;l9;k0"` +- `-DIPPCP_CUSTOM_BUILD=""` - optional, works only if `-DMERGED_BLD:BOOL=off` is set, i.e. only for 1CPU libraries. Enables the CPU feature dispatching mask at compile-time based on the provided list. + + - Currently supported by the library custom features dispatching: + 1. Intel® Advanced Encryption Standard New Instructions (Intel® AES-NI) code-path enabling: `IPPCP_AES_ON;IPPCP_CLMUL_ON` + 2. Intel® Advanced Vector Extensions 512 (Intel(R) AVX-512) and vector extensions of Intel(R) AES New Instructions (Intel(R) AES-NI) code-path enabling: `IPPCP_VAES_ON;IPPCP_VCLMUL_ON` + - Example: + `-DPLATFORM_LIST="IPPCP_AES_ON;IPPCP_CLMUL_ON"` - this combination enables Intel® AES-NI in all 1CPU libraries, which contains this code path. + - Example of using a combination of CPU features: + `-DPLATFORM_LIST="IPPCP_AES_ON;IPPCP_CLMUL_ON;IPPCP_VAES_ON;IPPCP_VCLMUL_ON"` - in this combination the highest available feature in each 1CPU library will be enabled (e.g. for `"y8"` it’s Intel® AES-NI and for `"k1"` - Intel AVX-512 VAES) + +### Windows\* OS + +- `-G""` - defines the native build system CMake will generate from the input files. + Refer to CMake [documentation](https://cmake.org/cmake/help/latest/manual/cmake-generators.7.html#visual-studio-generators) for the Visual Studio\* generators options. + +- `-A` - for Visual Studio\* 2019+, defines the target architecture for the build of the Intel IPP Cryptography library. + +- `-T` - defines the compiler for building. + For example, to use Intel® Compiler, specify `-T"Intel C++ Compiler 19.1"`. + +> **NOTE:** Refer to CMake [documentation](https://cmake.org/cmake/help/latest/manual/ccmake.1.html) for more information on these options. + +### Linux\* OS + +- `-DNONPIC_LIB:BOOL=` - optional. Defines whether the built library is position-dependent or not. + + - `-DNONPIC_LIB:BOOL=off:` default. Position-independent code. + + - `-DNONPIC_LIB:BOOL=on:` position-dependent code. + +## CMake Commands FAQ + +### How to build a 32-bit library? + +`cmake CMakeLists.txt -B_build -DARCH=ia32` + +### How to build a 64-bit generic library without any CPU-specific optimizations? + +`cmake CMakeLists.txt -B_build -DARCH=intel64 -DMERGED_BLD:BOOL=off -DPLATFORM_LIST=mx` + +### How to build two libraries with optimizations for Intel® Advanced Vector Extensions 2 and Intel® Advanced Vector Extensions 512 instruction sets? + +`cmake CMakeLists.txt -B_build -DARCH=intel64 -DMERGED_BLD:BOOL=off -DPLATFORM_LIST="l9;k0"` + +### How to build a library to work in a kernel space? + +`cmake CMakeLists.txt -B_build -DARCH=intel64 -DNONPIC_LIB:BOOL=on` + +### How to specify path to OpenSSL\* +`cmake CMakeLists.txt -B_build -DARCH=intel64 -DOPENSSL_INCLUDE_DIR=/path/to/openssl/include -DOPENSSL_LIBRARIES=/path/to/openssl/lib -DOPENSSL_ROOT_DIR=/path/to/openssl` + +## Incorporating Intel® IPP Cryptography sources into custom build system + +You can include Intel IPP Cryptography sources into some arbitrary project's CMake build system and build them with it. + +Here is the minimal working example: + +``` bash +cmake_minimum_required(VERSION 3.14) + +project("test_proj") + +# `crypto` is the repository root folder of Intel IPP Cryptography +add_subdirectory(crypto) +include_directories(crypto/include) + +# 'main.cpp' is some arbitrary project's source file +add_executable("test_proj" main.cpp) +# `ippcp_s` is the target name of static library in the Intel IPP Cryptography build system. +# This static library will be built automatically, when you build your project. +target_link_libraries("test_proj" "ippcp_s") +``` + +Also you can use CMake module to find the Intel IPP Cryptography library installed on the system. The module location is `examples/FindIPPCrypto.cmake` and here is the example of its usage: + +``` bash +find_package(IPPCrypto REQUIRED MODULE) + +if (NOT IPPCRYPTO_FOUND) + message(FATAL_ERROR "No Intel IPP Cryptography library found on the system.") +endif() + +# If Intel IPP Cryptography is found, the following variables will be defined: +# `IPPCRYPTO_LIBRARIES` - static library name +# `IPPCRYPTO_INCLUDE_DIRS` - path to Intel IPP Cryptography headers +# `IPPCRYPTO_ROOT_DIR` - library root dir (a folder with 'include' and 'lib' directories) +``` diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/CHANGELOG.md b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/CHANGELOG.md new file mode 100644 index 0000000..9c38d14 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/CHANGELOG.md @@ -0,0 +1,128 @@ +# Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) + +This is a list of notable changes to Intel(R) IPP Cryptography, in reverse chronological order. + +## Intel(R) IPP Cryptography 2021.9 +- Added optimized RSA-2048 code for multi-buffer (8 buffers) Intel® AVX-512 implementation. +- Added Intel® Advanced Vector Extensions 2 (Intel® AVX2) vector extensions of Intel® AES New Instructions (Intel® AES-NI) optimization for AES-GCM algorithm. + +## Intel(R) IPP Cryptography 2021.8 +- Crypto Multi-buffer library was extended with XTS mode of SM4 algorithm. + +## Intel(R) IPP Cryptography 2021.7.1 +- Added re-initialization API for AES-GCM context - ippsAES_GCMReinit. The use-case of this function is very specific, please, refer to the documentation for more details. + +## Intel(R) IPP Cryptography 2021.7 +- Mitigation for Frequency Throttling Side-Channel Attack ([Frequency Throttling Side Channel Software Guidance for Cryptography Implementations](https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/technical-documentation/frequency-throttling-side-channel-guidance.html)) was added for ECB, CMAC and GCM modes of AES (for more details refer to ippsAESSetupNoise, ippsAES_GCMSetupNoise and ippsAES_CMACSetupNoise). +- Crypto Multi-buffer library was extended with CCM and GCM modes of SM4 algorithm. +- In Crypto Multi-buffer library in-place mode of executing (when pa_out == pa_inp) was fixed for SM4 CBC and CFB modes. +- New API that updates pointer to IppsHashMethod context inside the IppsHashState_rmf state was added (please, refer to ippsHashStateMethodSet_ API). +- ippsHMAC_Pack(_rmf), ippsHMAC_Unpack(_rmf) APIs were fixed - context id is set up properly now during the packing-unpacking process. + +## Intel(R) IPP Cryptography 2021.6 +- EC cryptography functionality for the NIST curves p256r1, p384r1, p521r1 and SM2 curve was enabled with Intel® Advanced Vector Extensions 512 (Intel® AVX-512) IFMA instructions. +- EC cryptography functionality input parameters checking was improved by adding keys boundaries check. +- AES-GCM input parameters checking was improved by adding input text boundary check. +- An issue in AES-CTR Intel® Advanced Vector Extensions 512 (Intel® AVX-512) code path related to handling of last partial block was fixed. +- The ability to build Intel® IPP Cryptography library and Crypto Multi buffer library with Microsoft Visual C++ Compiler\* version 19.30 provided by Microsoft Visual Studio\* 2022 version 17.0 was added. + +## Intel(R) IPP Cryptography 2021.5 +- AES-GCM small packets processing was optimized for Intel® Advanced Vector Extensions 512 (Intel® AVX-512) code path. +- The ability to build Intel® IPP Cryptography library and Crypto Multi buffer library with GCC\* 10, GCC\* 11, Clang\* 9, Clang\* 12 and Intel® C++ Compiler Classic 2021.3 was added. +- The ability to build Crypto Multi buffer library with OpenSSL\* 3.0 was added. + +## Intel(R) IPP Cryptography 2021.4 +- Crypto Multi buffer library was extended with ECB, CBC, CTR, OFB and CFB modes of SM4 algorithm. +- Crypto Multi buffer library was extended with EC SM2 public key generation, ECDHE and ECDSA (Sign and Verify) algorithms. +- Crypto Multi buffer library extended with ECDSA Ed25519 verify algorithm. +- Crypto Multi buffer library extended with modular exponent algorithm. + +## Intel(R) IPP Cryptography 2021.3 +- Enabled ECDSA Ed25519sign crypto multi buffer API within Intel® IPP Cryptography for 3rd Generation Intel® Xeon® Processor Scalable and 10th Gen Intel® Core™ Processors. +- Enabled RSA single buffer 3k, 4k within Intel® IPP Cryptography for 3rd Generation Intel® Xeon® Processor Scalable and 10th Gen Intel® Core™ Processors. +- Fixed Intel® IPP Cryptography AES-GCM decryption incorrect tag issue while dispatching on processors supported with Intel® Advanced Vector Extensions 512 (Intel® AVX-512). + +## Intel(R) IPP Cryptography 2021.2 +- Crypto Multi buffer library was extended with ECDSA (Sign) and ECDHE for the NIST curves p521r1. +- Added ECDSA verify with new Instruction Set Architecture(ISA) for the NIST curve p384r1, p256r1 and p521r1. +- Added SM3 multi buffer with new Instruction Set Architecture(ISA). +- Added new Intel® IPP Cryptography pre-defined hash algorithm APIs ippsHashMethodGetSize and ippsHashMethodInit. +- RSA-2048 decryption (CRT) was enabled for Intel(R) Microarchitecture Code Named Ice Lake. +- Crypto Multi buffer library was extended with ECDSA (Sign) and ECDHE for the NIST curves p256r1 and p384r1. +- Fixed a build issue that affect build of the Intel(R) IPP Cryptography library with MSVC\* compiler on Windows\* OS. +- Duplicated APIs of HASH, HMAC, MGF, RSA, ECCP functionality were marked as deprecated. For more information see [Deprecation notes](./DEPRECATION_NOTES.md) +- Added examples demonstrating usage of SMS4-CBC encryption and decryption. + +## Intel(R) IPP Cryptography 2020 Update 3 +- Refactoring of Crypto Multi buffer library, added build and installation of crypto_mb dynamic library and CPU features detection. +- Added multi buffer implementation of AES-CFB optimized with Intel(R) AES New Instructions (Intel(R) AES-NI) and vector extensions of Intel(R) AES New Instructions (Intel(R) AES-NI) instruction sets. +- Fixed compatibility issue with x64 ABI (restored non-volatile registers after function call in Intel® Advanced Vector Extensions (Intel® AVX)/Intel® Advanced Vector Extensions 2 (Intel® AVX2) assembly code). +- Updated Intel IPP Custom Library Tool. + +## Intel(R) IPP Cryptography 2020 Update 2 +- AES-GCM algorithm was optimized for Intel(R) Microarchitecture Code Named Cascade Lake with Intel(R) AES New Instructions (Intel(R) AES-NI). +- Crypto Multi buffer library installation instructions update. The Readme file of Crypto Multi buffer Library was updated by information about possible installation fails in specific environment. +- Position Independent Execution (PIE) option was added to Crypto Multi buffer Library build scripts. +- AES-XTS optimization for Intel(R) Microarchitecture Code Named Ice Lake with vector extensions of Intel(R) AES New Instructions (Intel(R) AES-NI) was improved. +- SM4-ECB, SM4-CBC and SM4-CTR were enabled for Intel(R) Microarchitecture Code Named Ice Lake with Intel(R) Advanced Vector Extensions 512 (Intel(R) AVX-512) GFNI instructions. +- Added support of Clang 9.0 for Linux and Clang 11.0 for MacOS compilers. +- Added example of RSA Multi Buffer Encryption/Decryption usage. +- The library was enabled with Intel® Control-Flow Enforcement Technology (Intel® CET) on Linux and Windows. +- Changed API of ippsGFpECSignDSA, ippsGFpECSignNR and ippsGFpECSignSM2 functions: const-ness requirement of private ephemeral keys is removed and now the ephemeral keys are cleaned up after signing. + +## Intel(R) IPP Cryptography 2020 Update 1 +- Added RSA IFMA Muti-buffer Library. +- Added RSA PSS multi buffer signature generation and verification. +- Added RSA PKCS#1 v1.5 multi buffer signature generation and verification. +- Removed Android support. Use Linux libraries instead. +- Fixed all build warnings for supported GCC\* and MSVC\* compilers. +- Assembler sources were migrated to NASM\* assembler. + +## Intel(R) IPP Cryptography 2020 +- Added RSA multi buffer encryption and decryption. +- Added Intel(R) IPP Cryptography library examples: AES-CTR, RSA-OAEP, RSA-PSS. +- Fixed code generation for kernel code model in Linux* Intel(R) 64 non-PIC builds. +- Fixes in Intel IPP Custom Library Tool. +- Added Microsoft\* Visual Studio\* 2019 build support. +- A dynamic dispatcher library and a set of CPU-optimized dynamic libraries were replaced by a single merged dynamic library with an internal dispatcher. +- Removed deprecated multi-threaded version of the library. +- Removed Supplemental Streaming SIMD Extensions 3 (SSSE3) support on macOS. + +## Intel(R) IPP Cryptography 2019 Update 5 +- 1024, 2048, 3072 and 4096 bit RSA were enabled with AVX512 IFMA instructions. +- AES-GCM was enabled with vector extensions of Intel(R) AES New Instructions (Intel(R) AES-NI). +- AES-XTS was enabled with vector extensions of Intel(R) AES New Instructions (Intel(R) AES-NI). +- Fixed GCC\* and MSVC\* builds of IA32 generic CPU code (pure C) with -DMERGED_BLD:BOOL=off option. +- Added single-CPU headers generation. +- Aligned structure of the build output directory across all supported operation systems. +- AES-CFB was enabled with vector extensions of Intel(R) AES New Instructions (Intel(R) AES-NI). +- ippsGFpECGetPointOctString and ippsGFpECSetPointOctString now support elliptic curves over both prime and extended finite fields. +- Added the ability to build the Intel(R) IPP Cryptography library with GCC\* 8.2. + +## Intel(R) IPP Cryptography 2019 Update 4 +- AES-ECB, AES-CBC and AES-CTR were enabled with vector extensions of Intel(R) AES New Instructions (Intel(R) AES-NI). +- Improved optimization of Intel(R) AES-NI based CMAC. +- Added the ippsGFpGetInfo function, which returns information about a finite field. +- Added the ippsHashGetInfo_rmf function, which returns information about a hash algorithm. +- Fixed selection of CPU-specific code in dynamic/shared libraries. + +## Intel(R) IPP Cryptography 2019 Update 3 +- Added the ability to build the Intel(R) IPP Cryptography library with the Intel(R) C++ Compiler 19. +- Added Intel IPP Custom Library Tool based on Python. + +## Intel(R) IPP Cryptography 2019 Update 1 +- Added the ability to build the Intel(R) IPP Cryptography library with the Microsoft\* Visual C++ Compiler 2017. +- Added the new SM2 encryption scheme. +- Changed the range of the message being signed or verified by EC and DLP. +- Deprecated the ARCFour functionality. +- Fixed a potential security problem in the signing functions over elliptic curves. +- Fixed a potential security problem in the key expansion function for AES Encryption. +- Fixed a potential security problem in the AES-CTR cipher functions. +- Fixed a potential security problem in the AES-GCM cipher functions. +- Fixed a potential security problem in the DLP signing and key generation functions. +- Fixed minor issues with DLP functions. +- Fixed some of the compilation warnings observed when building the static dispatcher on Windows\* OS. + +------------------------------------------------------------------------ +Intel is a trademark of Intel Corporation or its subsidiaries in the U.S. and/or other countries. +\* Other names and brands may be claimed as the property of others. diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/CMakeLists.txt b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/CMakeLists.txt new file mode 100644 index 0000000..552a812 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/CMakeLists.txt @@ -0,0 +1,238 @@ +#=============================================================================== +# Copyright (C) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. +# +#=============================================================================== + +# +# Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) +# + +cmake_minimum_required(VERSION 3.12) + +include("${CMAKE_CURRENT_SOURCE_DIR}/sources/cmake/ippcp-utils.cmake") +ippcp_getlibversion("${CMAKE_CURRENT_SOURCE_DIR}/include/ippversion.h") +if ((NOT DEFINED IPPCP_VERSION_MAJOR) OR + (NOT DEFINED IPPCP_VERSION_MINOR) OR + (NOT DEFINED IPPCP_VERSION_UPDATE) OR + (NOT DEFINED IPPCP_INTERFACE_VERSION_MAJOR) OR + (NOT DEFINED IPPCP_INTERFACE_VERSION_MINOR)) + message(WARNING "Cannot parse version from ippversion.h file. The project might be corrupted.") +endif() + +set(PROJECT_FULL_NAME "Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography)") +set(PROJECT_NAME "Intel(R) IPP Crypto") +set(PROJECT_VERSION ${IPPCP_VERSION}) + +set(LIB_NAME ippcp) + +set(CMAKE_CONFIGURATION_TYPES "Debug;Release" CACHE STRING "" FORCE) + +cmake_policy(SET CMP0042 NEW) +cmake_policy(SET CMP0054 NEW) +cmake_policy(SET CMP0068 NEW) + +if("${CMAKE_GENERATOR}" STREQUAL "NMake Makefiles") + if(NOT(C_COMPILER STREQUAL "")) + set(CMAKE_C_COMPILER ${C_COMPILER}) + endif() + if(NOT(CXX_COMPILER STREQUAL "")) + set(CMAKE_CXX_COMPILER ${CXX_COMPILER}) + endif() +endif() + +project(${PROJECT_NAME} + VERSION ${PROJECT_VERSION} + LANGUAGES C CXX) + +if("${CMAKE_BUILD_TYPE}" STREQUAL "") + message(STATUS "CMAKE_BUILD_TYPE is unset, defaulting to Release") + set(CMAKE_BUILD_TYPE "Release") +endif() + +find_package(Python REQUIRED) + +if(WIN32 AND (${CMAKE_GENERATOR} MATCHES "Visual Studio")) + if(CMAKE_GENERATOR_PLATFORM) # VS 2019+ -A param + if(${CMAKE_GENERATOR_PLATFORM} MATCHES "x64") + set(ARCH intel64) + else() + set(ARCH ia32) + endif() + else() + if(${CMAKE_GENERATOR} MATCHES "Win64") # old way of platform setting for VS + set(ARCH intel64) + else() + set(ARCH ia32) + endif() + endif() +else() + if (NOT "${ARCH}" STREQUAL "intel64" AND NOT "${ARCH}" STREQUAL "ia32") + message(FATAL_ERROR "Please, set the ARCH parameter to ia32 or intel64") + endif() +endif(WIN32 AND (${CMAKE_GENERATOR} MATCHES "Visual Studio")) + +if ((NOT NONPIC_LIB) AND (NOT CODE_COVERAGE)) + set(DYNAMIC_LIB ON) +else() + set(DYNAMIC_LIB OFF) +endif() + +if("${MERGED_BLD}" STREQUAL "") + set(MERGED_BLD ON) +endif() + +# Set default installation directories +if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) + if (UNIX) + set(CMAKE_INSTALL_PREFIX "$ENV{HOME}/intel/ippcp_${PROJECT_VERSION}" CACHE PATH "..." FORCE) + else() + set(CMAKE_INSTALL_PREFIX "c:/Program Files (x86)/IntelSWTools/ippcp_${PROJECT_VERSION}" CACHE PATH "..." FORCE) + endif() +endif() + +set(IPP_CRYPTO_DIR "${CMAKE_CURRENT_SOURCE_DIR}") +set(IPP_CRYPTO_INCLUDE_DIR "${IPP_CRYPTO_DIR}/include") +set(IPP_CRYPTO_SOURCES_INCLUDE_DIR "${IPP_CRYPTO_DIR}/sources/include") +set(IPP_CRYPTO_SOURCES_DIR "${IPP_CRYPTO_DIR}/sources/ippcp") +set(IPP_CRYPTO_DISPATCHER_DIR "${IPP_CRYPTO_DIR}/sources/dispatcher") +set(TOOLS_DIR "${IPP_CRYPTO_DIR}/tools") + +if(NOT CMAKE_OUTPUT_DIR) + set(CMAKE_OUTPUT_DIR "${CMAKE_BINARY_DIR}/.build") +endif() + +message (STATUS "CMAKE_VERSION ......................... " ${CMAKE_VERSION}) + +if(UNIX AND NOT APPLE) + if(NONPIC_LIB) + message (STATUS "NONPIC_LIB ............................ on") + else() + message (STATUS "NONPIC_LIB ............................ off") + endif() +else() + set(${NONPIC_LIB} false) +endif(UNIX AND NOT APPLE) + +set(NONPIC_SUBDIRECTORY "") +if(NONPIC_LIB) + set(NONPIC_SUBDIRECTORY "/nonpic") +endif() + +foreach( OUTPUTCONFIG ${CMAKE_CONFIGURATION_TYPES} ) + string( TOUPPER ${OUTPUTCONFIG} OUTPUTCONFIG ) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_${OUTPUTCONFIG} "${CMAKE_OUTPUT_DIR}/${OUTPUTCONFIG}/lib${NONPIC_SUBDIRECTORY}") + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${OUTPUTCONFIG} "${CMAKE_OUTPUT_DIR}/${OUTPUTCONFIG}/lib${NONPIC_SUBDIRECTORY}") + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_${OUTPUTCONFIG} "${CMAKE_OUTPUT_DIR}/${OUTPUTCONFIG}/lib${NONPIC_SUBDIRECTORY}") +endforeach( OUTPUTCONFIG CMAKE_CONFIGURATION_TYPES ) + +if(${CMAKE_BUILD_TYPE} STREQUAL "Release") + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE}") + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_ARCHIVE_OUTPUT_DIRECTORY_RELEASE}") + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE}") +endif() + +if(${CMAKE_BUILD_TYPE} STREQUAL "Debug") + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG}") + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_ARCHIVE_OUTPUT_DIRECTORY_DEBUG}") + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG}") +endif() + +message (STATUS "PROJECT ............................... " ${PROJECT_FULL_NAME}) +message (STATUS "CMAKE_BINARY_DIR ...................... " ${CMAKE_BINARY_DIR}) +message (STATUS "CMAKE_OUTPUT_DIR ...................... " ${CMAKE_OUTPUT_DIR}) +message (STATUS "CMAKE_SOURCE_DIR ...................... " ${CMAKE_SOURCE_DIR}) +message (STATUS "IPP_CRYPTO_DIR ........................ " ${IPP_CRYPTO_DIR}) +message (STATUS "CMAKE_GENERATOR ....................... " ${CMAKE_GENERATOR}) +message (STATUS "CMAKE_C_COMPILER_ID ................... " ${CMAKE_C_COMPILER_ID}) +message (STATUS "CMAKE_CXX_COMPILER_ID ................. " ${CMAKE_CXX_COMPILER_ID}) +message (STATUS "IPP_CRYPTO_INCLUDE_DIR ................ " ${IPP_CRYPTO_INCLUDE_DIR}) +message (STATUS "IPP_CRYPTO_SOURCES_INCLUDE_DIR ........ " ${IPP_CRYPTO_SOURCES_INCLUDE_DIR}) +message (STATUS "IPP_CRYPTO_SOURCES_DIR ................ " ${IPP_CRYPTO_SOURCES_DIR}) +message (STATUS "ARCH .................................. " ${ARCH}) +message (STATUS "DYNAMIC_LIB ........................... " ${DYNAMIC_LIB}) +message (STATUS "CMAKE_INSTALL_PREFIX .................. " ${CMAKE_INSTALL_PREFIX}) + +if(Python_Interpreter_FOUND) + message (STATUS "PYTHON_VERSION_STRING ................. " ${Python_VERSION}) +else() + message (STATUS "PYTHON_VERSION_STRING ................. Python not found" ) +endif() + +if(MERGED_BLD) + message (STATUS "MERGED_BLD ............................ on") +else() + message (STATUS "MERGED_BLD ............................ off") +endif() + +option(BUILD_EXAMPLES "Build examples" OFF) +if(BUILD_EXAMPLES) + message (STATUS "BUILD_EXAMPLES ........................ on") +else() + message (STATUS "BUILD_EXAMPLES ........................ off") +endif() + +# Build with sanitizers +set(SANITIZERS OFF) +if(BUILD_WITH_SANITIZERS AND UNIX AND ${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang" AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER_EQUAL "12.0.0") + set(ASAN OFF) + set(UBSAN OFF) + set(MSAN OFF) + if("${BUILD_WITH_SANITIZERS}" MATCHES "address") + set(ASAN ON) + list(APPEND PRINT_TYPES_OF_SANITIZERS_LIST "address") + endif() + if("${BUILD_WITH_SANITIZERS}" MATCHES "undefined") + set(UBSAN ON) + list(APPEND PRINT_TYPES_OF_SANITIZERS_LIST "undefined") + endif() + if("${BUILD_WITH_SANITIZERS}" MATCHES "memory") + set(MSAN ON) + list(APPEND PRINT_TYPES_OF_SANITIZERS_LIST "memory") + endif() + if((ASAN OR UBSAN) AND MSAN) + message (FATAL_ERROR "Can not use address | undefined sanitizers with memory sanitizer") + endif() + if(ASAN OR UBSAN OR MSAN) + set(SANITIZERS ON) + endif() +endif() +if(SANITIZERS) + list(JOIN PRINT_TYPES_OF_SANITIZERS_LIST "," PRINT_TYPES_OF_SANITIZERS_STRING) + message (STATUS "BUILD_WITH_SANITIZERS ................. on (${PRINT_TYPES_OF_SANITIZERS_STRING})") +else() + message (STATUS "BUILD_WITH_SANITIZERS ................. off - use -DBUILD_WITH_SANITIZERS=[memory,address,undefined] with CLANG compiler to enable this option") +endif(SANITIZERS) + +if((UNIX) AND (NOT APPLE)) + set(LINUX ON) +else() + set(LINUX OFF) +endif() + +add_subdirectory(sources/ippcp) +if(EXISTS "${IPP_CRYPTO_DIR}/tests/") + add_subdirectory(tests) +endif() +if(EXISTS "${IPP_CRYPTO_DIR}/perf_tests/") + add_subdirectory(perf_tests) +endif() + +if(BUILD_EXAMPLES) + # This helps to organize examples projects structure in IDE by folders + set_property(GLOBAL PROPERTY USE_FOLDERS ON) + set_property(GLOBAL PROPERTY PREDEFINED_TARGETS_FOLDER "CMakeTargets") + + add_subdirectory(examples) +endif() diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/CONST_TIME_EXECUTION_TESTING.md b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/CONST_TIME_EXECUTION_TESTING.md new file mode 100644 index 0000000..6186836 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/CONST_TIME_EXECUTION_TESTING.md @@ -0,0 +1,162 @@ +# Scope of the Constant-time execution testing of Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) + +- [General information](#general) +- [Scope for ippcp library](#ippcp) +- [Scope for crypto_mb library](#cryptomb) + +## General information

    +- Testing is conducted under Linux for 64-bit Intel® IPP Cryptography built with the following compilers: + - Intel® C++ Compiler 19.1 + - Intel® C++ Compiler Classic 2021.9 + - GCC 8.3 + - GCC 9.1 + - GCC 10.1 + - GCC 11.1 + - Clang 9.0 + - Clang 12.0 +- Tested platforms: w7, n8, y8, e9, l9, k0 (see the supported platforms list [here](./OVERVIEW.md#target-optimization-codes-in-function-names)). +- Testing scope described below is guaranteed to pass for **`release`** branches. This is not guaranteed for the **`develop`** branch ([branches description](./OVERVIEW.md#branches-description)) +- Information about Pin-Based Constant Execution Checker can be found [here](https://github.com/intel/pin_based_cec) + +## ippcp library
    +| Tested Function | Parameters | +| :----------------------: | :----------------------------------------------------------------------------------------: | +| ippsAESEncryptCBC | Different key length:
    128, 192, 256 bits | +| ippsAESDecryptCBC | Different key length:
    128, 192, 256 bits | +| ippsAESEncryptCBC_CS1 | Different key length:
    128, 192, 256 bits | +| ippsAESDecryptCBC_CS1 | Different key length:
    128, 192, 256 bits | +| ippsAESEncryptCBC_CS2 | Different key length:
    128, 192, 256 bits | +| ippsAESDecryptCBC_CS2 | Different key length:
    128, 192, 256 bits | +| ippsAESEncryptCBC_CS3 | Different key length:
    128, 192, 256 bits | +| ippsAESDecryptCBC_CS3 | Different key length:
    128, 192, 256 bits | +| ippsAES_CCMEncrypt | Different key length:
    128, 192, 256 bits | +| ippsAES_CCMDecrypt | Different key length:
    128, 192, 256 bits | +| ippsAESEncryptCFB | Different key length:
    128, 192, 256 bits | +| ippsAESDecryptCFB | Different key length:
    128, 192, 256 bits | +| ippsAES_CMACUpdate | Different key length:
    128, 192, 256 bits | +| ippsAES_CMACFinal | Different key length:
    128, 192, 256 bits | +| ippsAESEncryptCTR | Different key length:
    128, 192, 256 bits | +| ippsAESDecryptCTR | Different key length:
    128, 192, 256 bits | +| ippsAESEncryptECB | Different key length:
    128, 192, 256 bits | +| ippsAESDecryptECB | Different key length:
    128, 192, 256 bits | +| ippsAES_GCMEncrypt | Different key length:
    128, 192, 256 bits | +| ippsAES_GCMDecrypt | Different key length:
    128, 192, 256 bits | +| ippsAES_GCMStart | Different key length:
    128, 192, 256 bits | +| ippsAESEncryptOFB | Different key length:
    128, 192, 256 bits | +| ippsAESDecryptOFB | Different key length:
    128, 192, 256 bits | +| ippsAES_S2V_CMAC | Different key length:
    128, 192, 256 bits | +| ippsAESSetKey | Different key length:
    128, 192, 256 bits | +| ippsAES_SIVEncrypt | Different key length:
    128, 192, 256 bits | +| ippsAES_SIVDecrypt | Different key length:
    128, 192, 256 bits | +| ippsAES_XTSEncrypt | Different key length:
    256, 512 bits | +| ippsAES_XTSDecrypt | Different key length:
    256, 512 bits | +| ippsAESEncryptXTS_Direct | Different key length:
    256, 512 bits | +| ippsAESDecryptXTS_Direct | Different key length:
    256, 512 bits | +| ippsCmp_BN | - | +| ippsDLPPublicKey | - | +| ippsDLPSharedSecretDH | - | +| ippsGFpAdd | - | +| ippsGFpAdd_PE | - | +| ippsGFpConj | - | +| ippsGFpECAddPoint | Different curves:
    p256r1, p384r1, p521r1 | +| ippsGFpECMulPoint | Different curves:
    p224r1, p256r1, p384r1 | +| ippsGFpECNegPoint | Different curves:
    p256r1, p384r1, p521r1 | +| ippsGFpECPublicKey | Different curves:
    p256r1, p384r1, p521r1 | +| ippsGFpECSharedSecretDH | Different curves:
    p256r1, p384r1, p521r1 | +| ippsGFpECSharedSecretDHC | Different curves:
    p256r1, p384r1, p521r1 | +| ippsGFpECSignDSA | Different curves:
    p256r1, p384r1 | +| ippsGFpECSignNR | Different curves:
    p256r1, p384r1 | +| ippsGFpECSignSM2 | - | +| ippsGFpECESStart_SM2 | - | +| ippsGFpECESEncrypt_SM2 | - | +| ippsGFpECESDecrypt_SM2 | - | +| ippsGFpECESFinal_SM2 | - | +| ippsGFpExp | - | +| ippsGFpInv | - | +| ippsGFpMul | - | +| ippsGFpMul_PE | - | +| ippsGFpMultiExp | - | +| ippsGFpNeg | - | +| ippsGFpSub | - | +| ippsGFpSub_PE | - | +| ippsHMACInit_rmf | Different hashes:
    sha1, sha256, sha224, sha384, sha512,
    sha512-256, sha512-224, sm3 | +| ippsRSA_Decrypt | Different key types and key length:
    key type 1, 512 bits
    key type 2, 512 bits | +| ippsRSADecrypt_OAEP | Different key types and key length:
    key type 1, 512 bits
    key type 2, 512 bits | +| ippsRSADecrypt_OAEP_rmf | Different key types and key length:
    key type 1, 512 bits
    key type 2, 512 bits | +| ippsRSASign_PKCS1v15 | Different key types and key length:
    key type 1, 512 bits
    key type 2, 512 bits | +| ippsRSASign_PKCS1v15_rmf | Different key types and key length:
    key type 1, 512 bits
    key type 2, 512 bits | +| ippsRSASign_PSS | Different key types and key length:
    key type 1, 512 bits
    key type 2, 512 bits | +| ippsRSASign_PSS_rmf | Different key types and key length:
    key type 1, 512 bits
    key type 2, 512 bits | +| ippsSMS4EncryptCBC | - | +| ippsSMS4DecryptCBC | - | +| ippsSMS4EncryptCBC_CS1 | - | +| ippsSMS4DecryptCBC_CS1 | - | +| ippsSMS4EncryptCBC_CS2 | - | +| ippsSMS4DecryptCBC_CS2 | - | +| ippsSMS4EncryptCBC_CS3 | - | +| ippsSMS4DecryptCBC_CS3 | - | +| ippsSMS4_CCMEncrypt | - | +| ippsSMS4_CCMDecrypt | - | +| ippsSMS4EncryptCFB | - | +| ippsSMS4DecryptCFB | - | +| ippsSMS4EncryptCTR | - | +| ippsSMS4DecryptCTR | - | +| ippsSMS4EncryptECB | - | +| ippsSMS4DecryptECB | - | +| ippsSMS4EncryptOFB | - | +| ippsSMS4DecryptOFB | - | +| ippsSMS4SetKey | - | + +## crypto_mb library
    +| Function | Parameters | +| :--------------------------------------: | :---------------------------------------------------: | +| mbx_nistp256_ecpublic_key_mb8 | projective coordinates
    affine coordinates | +| mbx_nistp384_ecpublic_key_mb8 | projective coordinates
    affine coordinates | +| mbx_nistp521_ecpublic_key_mb8 | projective coordinates
    affine coordinates | +| mbx_sm2_ecpublic_key_mb8 | projective coordinates
    affine coordinates | +| mbx_nistp256_ecdh_mb8 | projective coordinates
    affine coordinates | +| mbx_nistp384_ecdh_mb8 | projective coordinates
    affine coordinates | +| mbx_nistp521_ecdh_mb8 | projective coordinates
    affine coordinates | +| mbx_nistp256_ecdsa_sign_mb8 | - | +| mbx_nistp384_ecdsa_sign_mb8 | - | +| mbx_nistp521_ecdsa_sign_mb8 | - | +| mbx_nistp256_ecdsa_sign_setup_mb8 | - | +| mbx_nistp384_ecdsa_sign_setup_mb8 | - | +| mbx_nistp521_ecdsa_sign_setup_mb8 | - | +| mbx_nistp256_ecdsa_sign_complete_mb8 | - | +| mbx_nistp384_ecdsa_sign_complete_mb8 | - | +| mbx_nistp521_ecdsa_sign_complete_mb8 | - | +| mbx_nistp256_ecpublic_key_ssl_mb8 | projective coordinates
    affine coordinates | +| mbx_nistp384_ecpublic_key_ssl_mb8 | projective coordinates
    affine coordinates | +| mbx_nistp521_ecpublic_key_ssl_mb8 | projective coordinates
    affine coordinates | +| mbx_sm2_ecpublic_key_ssl_mb8 | projective coordinates
    affine coordinates | +| mbx_nistp256_ecdh_ssl_mb8 | projective coordinates
    affine coordinates | +| mbx_nistp384_ecdh_ssl_mb8 | projective coordinates
    affine coordinates | +| mbx_nistp521_ecdh_ssl_mb8 | projective coordinates
    affine coordinates | +| mbx_nistp256_ecdsa_sign_ssl_mb8 | - | +| mbx_nistp384_ecdsa_sign_ssl_mb8 | - | +| mbx_nistp521_ecdsa_sign_ssl_mb8 | - | +| mbx_nistp256_ecdsa_sign_setup_ssl_mb8 | - | +| mbx_nistp384_ecdsa_sign_setup_ssl_mb8 | - | +| mbx_nistp521_ecdsa_sign_setup_ssl_mb8 | - | +| mbx_nistp256_ecdsa_sign_complete_ssl_mb8 | - | +| mbx_nistp384_ecdsa_sign_complete_ssl_mb8 | - | +| mbx_nistp521_ecdsa_sign_complete_ssl_mb8 | - | +| mbx_ed25519_public_key_mb8 | - | +| mbx_ed25519_sign_mb8 | - | +| mbx_rsa_private_mb8 | Different key length:
    1024, 2048, 3072, 4096 bits | +| mbx_rsa_private_crt_mb8 | Different key length:
    1024, 2048, 3072, 4096 bits | +| mbx_rsa_private_ssl_mb8 | Different key length:
    1024, 2048, 3072, 4096 bits | +| mbx_rsa_private_crt_ssl_mb8 | Different key length:
    1024, 2048, 3072, 4096 bits | +| mbx_sm2_ecdsa_sign_mb8 | - | +| mbx_sm2_ecdsa_sign_ssl_mb8 | - | +| mbx_sm4_encrypt_ecb_mb16 | - | +| mbx_sm4_decrypt_ecb_mb16 | - | +| mbx_sm4_encrypt_cbc_mb16 | - | +| mbx_sm4_decrypt_cbc_mb16 | - | +| mbx_sm4_encrypt_ctr128_mb16 | - | +| mbx_sm4_encrypt_ofb_mb16 | - | +| mbx_sm4_encrypt_cfb128_mb16 | - | +| mbx_sm4_decrypt_cfb128_mb16 | - | +| mbx_x25519_public_key_mb8 | - | +| mbx_x25519_mb8 | - | diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/DEPRECATION_NOTES.md b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/DEPRECATION_NOTES.md new file mode 100644 index 0000000..3ef6475 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/DEPRECATION_NOTES.md @@ -0,0 +1,86 @@ +# Deprecated API in Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) + +This document describes deprecated API in different Intel IPP Cryptography versions and recommendations for transition. + +The deprecated API means it is obsolete and will be removed in one of future Intel IPP Cryptography releases. If you have any concerns, please use the following link for opening a ticket and providing feedback: + +## 2020 Update1 (branch [ipp-crypto_2020_update1](https://github.com/intel/ipp-crypto/tree/ipp-crypto_2020_update1)) + +### Hash Functionality + +| Deprecated | Recommended replacement | +| :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :-------------------------------------------------------------: | +| ippsSHA1GetSize
    ippsSHA224GetSize
    ippsSHA256GetSize
    ippsSHA384GetSize
    ippsSHA512GetSize
    ippsSM3GetSize
    ippsMD5GetSize | ippsHashGetSize_rmf | +| ippsSHA1Init
    ippsSHA224Init
    ippsSHA256Init
    ippsSHA384Init
    ippsSHA512Init
    ippsSM3Init
    ippsMD5Init | ippsHashInit_rmf \* | +| ippsSHA1Duplicate
    ippsSHA224Duplicate
    ippsSHA256Duplicate
    ippsSHA384Duplicate
    ippsSHA512Duplicate
    ippsSM3Duplicate
    ippsMD5Duplicate | ippsHashDuplicate_rmf | +| ippsSHA1Pack, ippsSHA1Unpack
    ippsSHA224Pack, ippsSHA224Unpack
    ippsSHA256Pack, ippsSHA256Unpack
    ippsSHA384Pack, ippsSHA384Unpack
    ippsSHA512Pack, ippsSHA512Unpack
    ippsSM3Pack, ippsSM3Unpack
    ippsMD5Pack, ippsMD5Unpack | ippsHashPack_rmf,
    ippsHashUnpack_rmf | +| ippsSHA1Update, ippsSHA1GetTag, ippsSHA1Final
    ippsSHA224Update, ippsSHA224GetTag, ippsSHA224Final
    ippsSHA256Update, ippsSHA256GetTag, ippsSHA256Final
    ippsSHA384Update, ippsSHA384GetTag, ippsSHA384Final
    ippsSHA512Update, ippsSHA512GetTag, ippsSHA512Final
    ippsSM3Update, ippsSM3GetTag, ippsSM3Final
    ippsMD5Update, ippsMD5GetTag, ippsMD5Final | ippsHashUpdate_rmf,
    ippsHashGetTag_rmf,
    ippsHashFinal_rmf | +| ippsSHA1MessageDigest
    ippsSHA224MessageDigest
    ippsSHA256MessageDigest
    ippsSHA384MessageDigest
    ippsSHA512MessageDigest
    ippsSM3MessageDigest
    ippsMD5MessageDigest | ippsHashMessage_rmf \* | +| ippsHashGetSize | ippsHashGetSize_rmf | +| ippsHashInit \*\* | ippsHashInit_rmf \* | +| ippsHashDuplicate | ippsHashDuplicate_rmf | +| ippsHashPack, ippsHashUnpack | ippsHashPack_rmf, ippsHashUnpack_rmf | +| ippsHashUpdate, ippsHashGetTag,ippsHashFinal | ippsHashUpdate_rmf,ippsHashGetTag_rmf,ippsHashFinal_rmf | +| ippsHashMessage \*\* | ippsHashMessage_rmf \* | + +>\* To choose hash algorithm, specify [IppsHashMethod parameter](#ippshashalgid-to-ippshashmethod-parameter-map) +>\*\* IppsHashAlgId parameter used in ippsHMAC_Init and in ippsHMAC_Message for choosing hash algorithm is deprecated (see Recommended replacement column for alternative in [IppsHashAlgId to IppsHashMethod Parameter Map](#ippshashalgid-to-ippshashmethod-parameter-map) + +### Keyed HMAC Functionality + +| Deprecated | Recommended replacement | +| :------------------------------------------------ | :------------------------------------------------------------: | +| ippsHMAC_GetSize | ippsHMAC_GetSize_rmf | +| ippsHMAC_Init \*\* | ippsHMAC_Init_rmf \* | +| ippsHMAC_Pack, ippsHMAC_Unack, ippsHMAC_Duplicate | ippsHMAC_Pack_rmf, ippsHMAC_Unpack_rmf, ippsHMAC_Duplicate_rmf | +| ippsHMAC_Update, ippsHMAC_Final, ippsHMAC_GetTag | ippsHMAC_Update_rmf, ippsHMAC_Final_rmf, ippsHMAC_GetTag_rmf | +| ippsHMAC_Message \*\* | ippsHMAC_Message_rmf \* | + +>\* To choose hash algorithm, specify [IppsHashMethod parameter](#ippshashalgid-to-ippshashmethod-parameter-map) +>\*\* IppsHashAlgId parameter used in 'ippsHMAC_Init' and in ippsHMAC_Message for choosing hash algorithm is deprecated (see Recommended replacement column for alternative in [IppsHashAlgId to IppsHashMethod Parameter Map](#ippshashalgid-to-ippshashmethod-parameter-map) + + +### MGF Functionality + +| Deprecated | Recommended replacement | +| :--------------- | :---------------------: | +| ippsHMAC_GetSize | ippsHMAC_GetSize_rmf | + +### RSA Encryption and Signature Schemes + +| Deprecated | Recommended replacement | +| :------------------------------------------- | :--------------------------------------------------: | +| ippsRSAEncrypt_OAEP, ippsRSADecrypt_OAEP | ippsRSAEncrypt_OAEP_rmf, ippsRSADecrypt_OAEP_rmf | +| ippsRSASign_PSS, ippsRSAVerify_PSS | ippsRSASign_PSS_rmf, ippsRSAVerify_PSS_rmf | +| ippsRSASign_PKCS1v15, ippsRSAVerify_PKCS1v15 | ippsRSASign_PKCS1v15_rmf, ippsRSAVerify_PKCS1v15_rmf | + + + +### Elliptic Curve Cryptography (ECC) + +| Deprecated | Recommended replacement | +| :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | +| ippsECCPGetSize
    ippsECCPGetSizeStd128r1
    ippsECCPGetSizeStd128r2
    ippsECCPGetSizeStd192r1
    ippsECCPGetSizeStd224r1
    ippsECCPGetSizeStd256r1
    ippsECCPGetSizeStd384r1
    ippsECCPGetSizeStd521r1
    ippsECCPGetSizeStdSM2 | ippsGFpECGetSize | +| ippsECCPInit
    ippsECCPInitStd128r1
    ippsECCPInitStd128r2
    ippsECCPInitStd192r1
    ippsECCPInitStd224r1
    ippsECCPInitStd256r1
    ippsECCPInitStd384r1
    ippsECCPInitStd521r1
    ippsECCPInitStdSM2 | ippsGFpECInitStd \*
    * ippsGFpECInitStd functions provides both initialization
    and set up standard EC set of parameters | +| ippsECCPSet | ippsGFpECSet | +| ippsECCPSetStd | ippsGFpECInitStd \*
    * ippsGFpECInitStd functions provides both initialization
    and set up standard EC set of parameters | +| ippsECCPSetStd128r1
    ippsECCPSetStd128r2
    ippsECCPSetStd192r1
    ippsECCPSetStd224r1
    ippsECCPSetStd256r1
    ippsECCPSetStd384r1
    ippsECCPSetStd521r1
    ippsECCPSetStdSM2 | ippsGFpECInitStd128r1
    ippsGFpECInitStd128r2
    ippsGFpECInitStd192r1
    ippsGFpECInitStd224r1
    ippsGFpECInitStd256r1
    ippsGFpECInitStd384r1
    ippsGFpECInitStd521r1
    ippsGFpECInitStdSM2 | +| ippsECCPBindGxyTblStd192r1
    ippsECCPBindGxyTblStd224r1
    ippsECCPBindGxyTblStd256r1
    ippsECCPBindGxyTblStd384r1
    ippsECCPBindGxyTblStd521r1
    ippsECCPBindGxyTblStdSM2 | ippsGFpECBindGxyTblStd192r1
    ippsGFpECBindGxyTblStd224r1
    ippsGFpECBindGxyTblStd256r1
    ippsGFpECBindGxyTblStd384r1
    ippsGFpECBindGxyTblStd521r1
    ippsGFpECBindGxyTblStdSM2 | +| ippsECCPGet
    ippsECCPGetOrderBitSize
    ippsECCPValidate
    ippsECCPPointGetSize, ippsECCPPointInit
    ippsECCPSetPointAtInfinity
    ippsECCPSetPoint,ippsECCPGetPoint
    ippsECCPCheckPoint
    ippsECCPComparePoint
    ippsECCPNegativePoint
    ippsECCPAddPoint
    ippsECCPMulPointScalar | ippsGFpECGet
    ippsGFpECGetSubgroup
    ippsGFpECVerify
    ippsGFpECPointGetSize, ippsGFpECPointInit
    ippsGFpECSetPointAtInfinity
    ippsGFpECSetPointRegular,ippsGFpECGetPointRegular
    ippsGFpECTstPoint
    ippsGFpECCmpPoint
    ippsGFpECNegPoint
    ippsGFpECAddPoint
    ippsGFpECMulPoint | +| ippsECCPGenKeyPair
    ippsECCPPublicKey
    ippsECCPValidateKeyPair
    ippsECCPSetKeyPair | ippsGFpECPrivateKey
    ippsGFpECPublicKey
    ippsGFpECTstKeyPair
    n/a | +| ippsECCPSharedSecretDH
    ippsECCPSharedSecretDHC | ippsGFpECSharedSecretDH
    ippsGFpECSharedSecretDHC | +| ippsECCPSignDSA
    ippsECCPVerifyDSA
    ippsECCPSignNR
    ippsECCPVerifyNR
    ippsECCPSignSM2
    ippsECCPVerifySM2 | ippsGFpECSignDSA
    ippsGFpECVerifyDSA
    ippsGFpECSignNR
    ippsGFpECVerifyNR
    ippsGFpECSignSM2
    ippsGFpECVerifySM2 | + +### IppsHashAlgId to IppsHashMethod Parameter Map + +| Algorithm | IppsHashAlgId (deprecated) | IppsHashMethod (recommended) | Notes | +| :--------: | :------------------------: | :---------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------: | +| SHA1 | ippsHashAlg_SHA1 | ippsHashMethod_SHA1
    ippsHashMethod_SHA1_NI
    ippsHashMethod_SHA1_TT | Intel® Secure Hash Algorithm - New Instructions (Intel® SHA-NI) not supported
    Intel SHA-NI only supported
    Automatic switch on Intel SHA-NI, if possible (tick-tock) | +| SHA224 | ippsHashAlg_SHA224 | ippsHashMethod_SHA224
    ippsHashMethod_SHA224_NI
    ippsHashMethod_SHA224_TT | Intel SHA-NI not supported
    Intel SHA-NI only supported
    Automatic switch on Intel SHA-NI, if possible supported | +| SHA256 | ippsHashAlg_SHA256 | ippsHashMethod_SHA256
    ippsHashMethod_SHA256_NI
    ippsHashMethod_SHA256_TT | Intel SHA-NI not supported
    Intel SHA-NI only supported
    Automatic switch on Intel SHA-NI, if possible supported | +| SHA384 | ippsHashAlg_SHA384 | ippsHashMethod_SHA384 | - | +| SHA512 | ippsHashAgl_SHA512 | ippsHashMethod_SHA512 | - | +| SM3 | ippsHashAlg_SM3 | ippsHashMethod_SM3 | - | +| MD5 | ippsHashAlg_MD5 | ippsHashMethod_MD5 | - | +| SHA512-224 | ippsHashAlg_SHA512_224 | ippsHashMethod_SHA512_224 | - | +| SHA512-256 | ippsHashAlg_SHA512_256 | ippsHashMethod_SHA512_256 | - | diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/LICENSE b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/LICENSE new file mode 100644 index 0000000..c7047e1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/LICENSE @@ -0,0 +1,234 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + CMake + ------------------------------ + CMake - Cross Platform Makefile Generator + Copyright 2000-2021 Kitware, Inc. and Contributors + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of Kitware, Inc. nor the names of Contributors + may be used to endorse or promote products derived from this + software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/MAC.md b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/MAC.md new file mode 100644 index 0000000..1f2d1a1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/MAC.md @@ -0,0 +1,23 @@ +## How to build on macOS + +- download Intel CPP Classic compiler from [here](https://www.intel.com/content/www/us/en/developer/articles/tool/oneapi-standalone-components.html#dpcpp-cpp) +- choose offline installer m_cpp-compiler-classic_p_2023.2.0.48999_offline.dmg +- `wget https://www.nasm.us/pub/nasm/releasebuilds/2.16.01/macosx/nasm-2.16.01-macosx.zip` +- extract zip file +- `wget https://www.openssl.org/source/openssl-3.1.2.tar.gz` +- in `openssl-3.1.2` extracted folder : + +``` +./Configure +make +``` + +- in this `src` folder: (or `git clone https://github.com/intel/ipp-crypto` for newer) + +``` +source /opt/intel/oneapi/compiler/2023.2.0/env/vars.sh intel64 +export ASM_NASM=/[your_path_here]/nasm-2.16.01/nasm +CC=icc CXX=icpc cmake CMakeLists.txt -B_build -DARCH=intel64 -DOPENSSL_INCLUDE_DIR=/[your_path_here]/openssl-3.1.2/include -DOPENSSL_LIBRARIES=/[your_path_here]/openssl-3.1.2 -DOPENSSL_ROOT_DIR=/[your_path_here]/openssl-3.1.2 +cd _build +make ippcp_s +``` diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/OVERVIEW.md b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/OVERVIEW.md new file mode 100644 index 0000000..63b7059 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/OVERVIEW.md @@ -0,0 +1,208 @@ +# Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) Library Overview + +- [Repository Structure](#repository-structure) + - [Branches Description](#branches-description) +- [Dispatcher](#dispatcher) + - [CPU Dispatching](#cpu-dispatching) + - [Target Optimization Codes in Function Names](#target-optimization-codes-in-function-names) + - [CPU Feature Dispatching](#cpu-feature-dispatching) + - [How to Avoid Dispatcher in All CPUs Static Library](#how-to-avoid-dispatcher-in-all-cpus-static-library) +- [Library Configurations](#library-configurations) + - [Linkage Mode](#linkage-mode) + - [Code Position](#code-position) + - [Target Instruction Set Architecture (ISA)](#target-instruction-set-architecture-isa) + - [All CPUs Library](#all-cpus-library) + - [Specific ISA Library](#specific-isa-library) + - [Choosing specific ISA from the All CPUs Static Library](#choosing-specific-isa-from-the-all-cpus-static-library) + - [Functionality](#functionality) + - [Static Library with Custom functionality](#static-library-with-custom-functionality) + - [Dynamic Library with Custom functionality](#dynamic-library-with-custom-functionality) + + +[sha256-dispatching]: ./data/images/README-pictures-0-dispatcher.png "Intel IPP Crypto function dispatching scheme" +[library configurations]: ./data/images/README-pictures-1-library-configurations.png "Library configurations picture" +[build targets]: ./data/images/README-pictures-1a-build-targets.png "Build targets picture" +[merged library]: ./data/images/README-pictures-2-merged-library.png "Merged library scheme" +[CPU-specific libraries]: ./data/images/README-pictures-3-cpu-specific-libraries.png "CPU-specific libraries scheme" +[link-with-merged-library]: ./data/images/README-pictures-4a-1CPU.png "Link with Merged Library picture" +[1cpu-link-with-merged-library]: ./data/images/README-pictures-4b-1CPU.png "1CPU link with Merged Library picture" + + +## Repository Structure + +``` bash +├── CHANGELOG.md +├── CMakeLists.txt < Main CMake file +├── examples < Examples of the library usage +├── include < Public headers +├── LICENSE +├── README.md +├── sources +│   ├── cmake < OS-specific CMake files +│   │   ├── linux +│   │   ├── macosx +│   │   └── windows +│   ├── dispatcher < CPU dispatcher generator +│   ├── gen_cpu_spc_header < Single CPU headers generator +│   ├── include < Internal headers +│   └── ippcp < C-sources +│   ├── asm_ia32 < IA-32 Assembler sources +│   ├── asm_intel64 < Intel® 64 Assembler sources +│   └── ifma_rsa_mb < Sources of RSA IFMA Multi-buffer library +└── tools + └── ipp_custom_library_tool_python < Custom Library Tool +``` + +### Branches Description + +- `develop` - snapshots of the library under active development. +Contains code that may not be fully functional and that Intel may substantially modify in development of a production version. +- `ipp_crypto_` - source code of the official production release ``. + +## Dispatcher + +### CPU Dispatching + +For the best performance, Intel IPP Cryptography uses multiple implementations of each function, optimized for various CPUs, and the [library version targeted for any CPU](#all-cpus-library) contains all of these implementations. + +With the dispatcher, the library detects an available CPU in a runtime and chooses the best for the current hardware version of a function. The process of dispatching is transparent and you can always call a generic function as illustrated at the picture below. + +![Dispatcher picture][sha256-dispatching] + +The prefix before the function name ("m7_", "n8_", etc) is a naming convention for the function implementations that are included in the [library with dispatcher](#all-cpus-library). It refers to the CPU instruction set for which the function is optimized (for all available prefixes see the [table](#target-optimization-codes-in-function-names) below). + +The dispatcher is designed to add no performance overhead when the library is initialized, in other words, when CPU features are detected. You can initialize the library either explicitly in advance by calling the dedicated function [ippcpInit()](https://www.intel.com/content/www/us/en/docs/ipp-crypto/developer-reference/current/init.html) or it will be done implicitly during the first call of any function of the library. + +By default, the dispatcher chooses the most appropriate optimization for the current hardware, but it is possible to apply the user-chosen one using the [ippcpSetCpuFeatures()](https://www.intel.com/content/www/us/en/docs/ipp-crypto/developer-guide-oneapi/current/support-functions.html) function. + +#### Target Optimization Codes in Function Names + +| IA-32 Intel® architecture | Intel® 64 architecture | Meaning | +| ------------------------- | ---------------------- | ------------------------------------------------------------------------------------------------------------------ | +| px | mx | Generic code without hardware specific optimizations suitable for any CPU | +| w7 | - | Optimized for processors with Intel® Streaming SIMD Extensions 2 (Intel® SSE2) | +| - | m7 | Optimized for processors with Intel® SSE3 | +| s8 | n8 | Optimized for processors with Supplemental Streaming SIMD Extensions 3 (SSSE3) | +| p8 | y8 | Optimized for processors with Intel® SSE4.2 | +| g9 | e9 | Optimized for processors with Intel® Advanced Vector Extensions (Intel® AVX) | +| h9 | l9 | Optimized for processors with Intel® Advanced Vector Extensions 2 (Intel® AVX2) | +| - | k0 | Optimized for processors with Intel® Advanced Vector Extensions 512 (Intel® AVX-512) (formerly codenamed SkyLake) | +| - | k1 | Optimized for processors with Intel® Advanced Vector Extensions 512 (Intel® AVX-512) (formerly codenamed IceLake) | + +### CPU Feature Dispatching + +Besides CPU dispatching that lets the library choose the general instruction set targeted implementation (for example, Intel SSE4.2, Intel AVX-512, and others), there is more granular dispatching that allows configuring usage of particular CPU features within a single instruction set. For example, Intel AVX-512 instruction set contains a VAES (AES Vector Extensions) feature subset, but not all CPUs that have Intel AVX-512 on board support VAES, so the library can automatically detect it in a runtime and enable corresponding optimizations if the feature subset is available. + +List of CPU feature subsets that the library has special optimizations for: + +- Intel ADX (ADCX, ADOX) +- Intel® Advanced Encryption Standard New Instructions (Intel® AES-NI) +- Intel® Secure Hash Algorithm - New Instructions (Intel® SHA-NI) +- RDRAND +- RDSEED +- CLMUL +- Intel AVX-512 VAES +- Intel AVX-512 IFMA +- Intel AVX-512 GFNI + + > **NOTE:** For some features there is also an opportunity to force their dispatching inside the 1CPU libraries manually during the compile time. For more information please, refer to [common for all operating systems CMake build options](./BUILD.md/#common-for-all-operating-systems). + +### How to Avoid Dispatcher in All CPUs Static Library + +To leave only specific ISA when linking with an [All CPUs Static Library](#all-cpus-library) and drop dispatcher, please refer to [this section](#choosing-specific-isa-from-the-all-cpus-static-library). + +## Library Configurations + +The Intel IPP Cryptography library supports configurations by: + +1) [*Linkage Mode*](#linkage-mode): to produce a static or dynamic library + +2) [*Code Position*](#code-position): to make position independent code (PIC) or non-PIC library + +3) *Target Instruction Set Architecture (ISA)*: + - [All CPUs Library](#all-cpus-library) + - [Specific Instruction Set (ISA) Targeted Library](#specific-isa-library) + - [Choosing specific ISA from the All CPUs Static Library](#choosing-specific-isa-from-the-all-cpus-static-library) + +4) *Functionality*: + - [Library with All functionality](#functionality) + - [Static Library with Custom functionality](#static-library-with-custom-functionality) + - [Dynamic Library with Custom functionality](#dynamic-library-with-custom-functionality) + +All possible configuration combinations are shown in the picture below. + +![Library configurations picture][library configurations] + +### Linkage Mode + +The build system is designed to create a dynamic library from the static one, so both build targets for dynamic and static libraries are always generated during CMake phase. + +The corresponding build target names for the libraries are shown at the picture below (same target names can be used on Linux* OS in the `make` command as well). + +![Build targets picture][build targets] + +### Code Position + +Be default, the Intel IPP Cryptography library is built with the [Position Independent Code (PIC)](https://en.wikipedia.org/wiki/Position-independent_code) option. + +But on Linux* OS, when the library is supposed to work in kernel space, it is possible to compile the static library in a non-PIC mode. For more information about build options, refer to the [Linux* OS build options](./BUILD.md). + +### Target Instruction Set Architecture (ISA) + +#### All CPUs Library + +Each function of the library is built in several instances with optimizations for each supported instruction set (see example for the `func2` function in green). Those functions instances are all included into a single library along with the [dispatcher](#dispatcher) that lets the library choose right function instance depending on current CPU. Such library build is called a merged library build. + +![Merged library picture][merged library] + +The advantage of this configuration is that the library works on any CPU. + +CMake build option: `-DMERGED_BLD:BOOL=on` + +#### Specific ISA Library + +The build system produces several separate libraries each optimized for its own instruction set. In this case, there is no need in [CPU dispatcher](#cpu-dispatching), so the dispatcher is not included, although [features dispatching](#cpu-feature-dispatching) within a single instruction set is in place. + +To specify for what instruction set targeted libraries must be produced, use the `PLATFORM_LIST` CMake variable. It contains semicolon-delimited list of CPU codes (for the complete list of codes see the table in [this](#target-optimization-codes-in-function-names) section). + +For example, to create two libraries - one with SSE4.2 optimizations and another with Intel AVX-512 optimizations, specify `-DPLATFORM_LIST="y8;k0"`. + +![CPU specific libraries picture][CPU-specific libraries] + +The advantage of this configuration is that libraries that contain function versions optimized for only one instruction set have much smaller footprint size than a big merged library. But the price of this advantage is that such libraries only work on a CPU that supports a corresponding instruction set. + +CMake build options: `-DMERGED_BLD:BOOL=off -DPLATFORM_LIST=""` + +#### Choosing specific ISA from the All CPUs Static Library + +When application is being statically linked with All CPUs Static Library, it receives functions implementations for all instruction sets with corresponding [dispatcher](#dispatcher) routines. This works well when CPU, where an application is going to work, is unknown. + +![Link with Merged Library picture][link-with-merged-library] + +But when target CPU is defined, it is possible to take from the static library only required instruction set implementations and avoid [dispatcher](#dispatcher) inclusion. + +![1CPU link with Merged Library picture][1cpu-link-with-merged-library] + +For this purpose, there are several CPU-specific headers (each targets a specific CPU optimization) generated during the merged library build. They are located in the `/.build//include/autogen` directory. + +To enable linking of CPU-specific versions of the library functions, include the appropriate header from the directory above before the primary library header `ippcp.h`. + +It is important to ensure that both processor and operating system supports full capabilities of the target processor. + +### Functionality + +By default, Intel IPP Cryptography libraries (both static and dynamic) contain all functionality that exists in the product. But when footprint size matters, the library can contain only required functionality and have no unused code. + +#### Static Library with Custom functionality + +With the static linking having only required functionality in the library is not so actual as leaving only those parts of a library that are used by application, is automatically managed by linker. + +Considering Intel IPP Cryptography design that implies minimal internal dependencies, the application linked with the Intel IPP Cryptography static library contains only relevant library functionality, and has minimal footprint size. + +#### Dynamic Library with Custom functionality + +To build your own dynamic library containing only the functionality that is necessary for your application, you can use the Intel® IPP Custom Library Tool - a Python tool that consumes pre-built merged (all CPUs) static library to produce a tiny dynamic library. + +The tool is located in the `tools/ipp_custom_library_tool_python` directory. + +Please refer to the [tool documentation](https://www.intel.com/content/www/us/en/docs/ipp/developer-guide-oneapi/current/ipp-custom-library-tool.html) for more information. diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/README.md b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/README.md new file mode 100644 index 0000000..5e2e6d5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/README.md @@ -0,0 +1,72 @@ +# Intel® Integrated Performance Primitives Cryptography + +[Build Instructions](./BUILD.md) | [Contributing Guide](#how-to-contribute) | [Documentation](#documentation) | [Get Help](#get-help) | [Intel IPP Product Page](https://www.intel.com/content/www/us/en/developer/tools/oneapi/ipp.html) + +Intel® Integrated Performance Primitives (Intel® IPP) Cryptography is a secure, fast and lightweight library of building blocks for cryptography, highly-optimized for various Intel® CPUs. + +## Key Features +The library provides a comprehensive set of routines commonly used for cryptographic operations, including: + - Symmetric Cryptography Primitive Functions: + - AES (ECB, CBC, CTR, OFB, CFB, XTS, GCM, CCM, SIV) + - SM4 (ECB, CBC, CTR, OFB, CFB, CCM) + - TDES (ECB, CBC, CTR, OFB, CFB) + - RC4 +- One-Way Hash Primitives: + - SHA-1, SHA-224, SHA-256, SHA-384, SHA-512 + - MD5 + - SM3 +- Data Authentication Primitive Functions: + - HMAC + - AES-CMAC +- Public Key Cryptography Functions: + - RSA, RSA-OAEP, RSA-PKCS_v15, RSA-PSS + - DLP, DLP-DSA, DLP-DH + - ECC (NIST curves), ECDSA, ECDH, EC-SM2 +- Multi-buffer RSA, ECDSA, SM3, x25519 +- Finite Field Arithmetic Functions +- Big Number Integer Arithmetic Functions +- PRNG/TRNG and Prime Numbers Generation + +## Reasons to Use Intel IPP Cryptography +- Security (constant-time execution for secret processing functions) +- Designed for the small footprint size +- Optimized for different Intel CPUs and instruction set architectures (including hardware cryptography instructions support): + - Intel® Streaming SIMD Extensions 2 (Intel® SSE2) + - Intel® SSE3 + - Intel® SSE4.2 + - Intel® Advanced Vector Extensions (Intel® AVX) + - Intel® Advanced Vector Extensions 2 (Intel® AVX2) + - Intel® Advanced Vector Extensions 512 (Intel® AVX-512) +- Configurable CPU dispatching for the best performance +- Kernel mode compatibility +- Thread-safe design + +## Installation + +[How to Get and Build the Intel IPP Cryptography Library](./BUILD.md) + +## Documentation + +- [Introduction to Intel IPP Cryptography Library](./OVERVIEW.md) +- [Introduction to Crypto Multi-buffer Library](./sources/ippcp/crypto_mb/Readme.md) +- [Intel IPP Cryptography Build Instructions](./BUILD.md) +- [Intel IPP Release Notes](https://www.intel.com/content/www/us/en/developer/articles/release-notes/release-notes-for-oneapi-integrated-performance-primitives.html) +- [Intel IPP Cryptography Developer Reference](https://www.intel.com/content/www/us/en/docs/ipp-crypto/developer-reference/current/overview.html) +- [Intel IPP Documentation](https://www.intel.com/content/www/us/en/developer/tools/oneapi/ipp-documentation.html) + +## Branches Description + +- `develop` - snapshots of the library under active development. +Contains code that may not be fully functional and that Intel may substantially modify in development of a production version. +- `ipp_crypto_` - source code of the official production release ``. + +## How to Contribute + +We welcome community contributions to Intel IPP Cryptography. If you have an idea how to improve the product, let us know about your proposal via the Intel IPP Forum or GitHub* Issues. + +### License +Intel IPP Cryptography is licensed under Apache License, Version 2.0. By contributing to the project, you agree to the license and copyright terms therein and release your contribution under these terms. + +## Certification + +Intel IPP Cryptography library is not certified for FIPS-140-2 (Security Requirements for Cryptographic Modules) and CMVP (Cryptographic Module Validation Program). diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/SECURITY.md b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/SECURITY.md new file mode 100644 index 0000000..ca4ec45 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/SECURITY.md @@ -0,0 +1,15 @@ +# Security Policy + +## Reporting a Security Vulnerability + +Visit [https://intel.com/security](https://intel.com/security) for information on how to report security vulnerabilities. + +**Do not report any security vulnerability as a regular issue in this repository.** + +Include the following information in your report: + +- Description of the security problem +- Steps to reproduce +- Expected behavior +- Environment where the problem is reproduced: OS, compiler (including version), versions of ippcp / crypto_mb libraries +- Any additional information that may be relevant diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/THIRD-PARTY-PROGRAMS.txt b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/THIRD-PARTY-PROGRAMS.txt new file mode 100644 index 0000000..ba4c3b5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/THIRD-PARTY-PROGRAMS.txt @@ -0,0 +1,37 @@ +Intel(R) Integrated Performance Primitives Cryptography Third Party Programs File + +This file contains the list of third party software (“third party programs”) contained in the Intel software and their required notices and/or license terms. +This third party software, even if included with the distribution of the Intel software, may be governed by separate license terms, including without limitation, third party license terms, other Intel software license terms, and open source software license terms. +These separate license terms govern your use of the third party programs as set forth in the “third-party-programs.txt” or other similarly-named text file. + +Third party programs and their corresponding required notices and/or license terms are listed +below. + +------------------------------------------------------------- + +1. Intel(R) Multi-Buffer Crypto for IPSec Library + +Copyright (c) 2012-2021, Intel Corporation + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/data/images/README-pictures-0-dispatcher.png b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/data/images/README-pictures-0-dispatcher.png new file mode 100644 index 0000000..2113b31 Binary files /dev/null and b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/data/images/README-pictures-0-dispatcher.png differ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/data/images/README-pictures-1-library-configurations.png b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/data/images/README-pictures-1-library-configurations.png new file mode 100644 index 0000000..cabfca5 Binary files /dev/null and b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/data/images/README-pictures-1-library-configurations.png differ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/data/images/README-pictures-1a-build-targets.png b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/data/images/README-pictures-1a-build-targets.png new file mode 100644 index 0000000..02df8b1 Binary files /dev/null and b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/data/images/README-pictures-1a-build-targets.png differ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/data/images/README-pictures-2-merged-library.png b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/data/images/README-pictures-2-merged-library.png new file mode 100644 index 0000000..d844f4f Binary files /dev/null and b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/data/images/README-pictures-2-merged-library.png differ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/data/images/README-pictures-3-cpu-specific-libraries.png b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/data/images/README-pictures-3-cpu-specific-libraries.png new file mode 100644 index 0000000..623c2c2 Binary files /dev/null and b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/data/images/README-pictures-3-cpu-specific-libraries.png differ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/data/images/README-pictures-4a-1CPU.png b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/data/images/README-pictures-4a-1CPU.png new file mode 100644 index 0000000..ffde859 Binary files /dev/null and b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/data/images/README-pictures-4a-1CPU.png differ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/data/images/README-pictures-4b-1CPU.png b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/data/images/README-pictures-4b-1CPU.png new file mode 100644 index 0000000..3dd807b Binary files /dev/null and b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/data/images/README-pictures-4b-1CPU.png differ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/CMakeLists.txt b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/CMakeLists.txt new file mode 100644 index 0000000..cfb184d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/CMakeLists.txt @@ -0,0 +1,123 @@ +#=============================================================================== +# Copyright (C) 2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. +# +#=============================================================================== + +# +# Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) examples +# + +# List of examples for targets generation +set(IPPCP_EXAMPLES + # AES examples + aes/aes-256-ctr-encryption.cpp + aes/aes-256-ctr-decryption.cpp + # DSA + dsa/dsa-dlp-sha-1-verification.cpp + dsa/dsa-dlp-sha-256-verification.cpp + # RSA SSA-PSS examples + rsa/rsa-3k-pss-sha384-type1-signature.cpp + rsa/rsa-1k-pss-sha1-verification.cpp + # RSA OAEP examples + rsa/rsa-1k-oaep-sha1-encryption.cpp + rsa/rsa-1k-oaep-sha1-type2-decryption.cpp + # SMS4 examples + sms4/sms4-128-cbc-encryption.cpp + sms4/sms4-128-cbc-decryption.cpp + ) + +cmake_policy(SET CMP0003 NEW) + +# Custom target to build ALL examples at once +add_custom_target(ippcp_examples_all) +set_target_properties(ippcp_examples_all PROPERTIES FOLDER "examples") + +function(ippcp_define_example out_target source_file category) + # Extract file name without directory or longest extension + get_filename_component(name "${source_file}" NAME_WE) + # Add suffix for nonpic build + if (NONPIC_LIB) + set(suffix "-nonpic") + endif() + set(local_target "example_${name}${suffix}") + # link additional sources if defined in categoryOptions.cmake + set(additional_sources "${category}_CATEGORY_COMMON_SOURCES") + add_executable(${local_target} "${source_file}" + $<$:${${additional_sources}}>) + # Static linking with merged lib is only supported + set(LIBRARY_NAMES_LIST ${IPPCP_LIB_MERGED}) + ippcp_example_set_build_options(${local_target} "${LIBRARY_NAMES_LIST}") + set_target_properties(${local_target} PROPERTIES + PROJECT_LABEL "(example) ${name}" # Set name of the target in IDE + FOLDER "examples/${category}" # Group projects in solution folder + RUNTIME_OUTPUT_DIRECTORY_RELEASE "${CMAKE_OUTPUT_DIR}/$>/examples" # Set output directory for examples in the build folder + RUNTIME_OUTPUT_DIRECTORY_DEBUG "${CMAKE_OUTPUT_DIR}/$>/examples" + COMPILE_DEFINITIONS _NO_IPP_DEPRECATED) # ignore deprecation warnings + + # Add a single target to build all examples of the same category (e.g. 'make ippcp_examples_aes') + set(parent_target ippcp_examples_${category}) + if(NOT TARGET ${parent_target}) + add_custom_target(${parent_target}) + set_target_properties(${parent_target} PROPERTIES FOLDER "examples") + if(TARGET ippcp_examples_all) + add_dependencies(ippcp_examples_all ${parent_target}) + endif() + endif() + add_dependencies(${parent_target} ${local_target}) + set(${out_target} ${local_target} PARENT_SCOPE) +endfunction() + +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_LIST_DIR) +# Build with standalone library + cmake_minimum_required(VERSION 3.1) + + project("Intel IPP Cryptography Examples" CXX) + set(CMAKE_CONFIGURATION_TYPES "Debug;Release" CACHE STRING "" FORCE) + + option(BUILD_EXAMPLES "Build examples" ON) + + list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}) + find_package(IPPCrypto REQUIRED MODULE) + + if (NOT IPPCRYPTO_FOUND) + message(FATAL_ERROR "No Intel IPP Cryptography library found on the system. To build examples with pre-built library, please specify -DIPPCRYPTO_ROOT_DIR= option, where is the path to directory that contains include/ and lib/ folders of Intel IPP Cryptography product.") + endif() + + # Define library to link + list(GET IPPCRYPTO_LIBRARIES 0 IPPCP_LIB_MERGED) + # Define include folder + set(IPP_CRYPTO_INCLUDE_DIR ${IPPCRYPTO_INCLUDE_DIRS}) + # Define output directory + if(NOT CMAKE_OUTPUT_DIR) + set(CMAKE_OUTPUT_DIR "${CMAKE_BINARY_DIR}/.build") + endif() +else() +# Build with library sources + if(NOT BUILD_EXAMPLES OR NOT MERGED_BLD) + message(FATAL_ERROR "Only merged library build is currently supported for Intel IPP Cryptography examples. Use -DMERGED_BLD:BOOL=on options.") + endif() +endif() + +include(examplesBuildOptions.cmake) + +foreach(example_filename ${IPPCP_EXAMPLES}) + # Extract example category from its subdirectory + get_filename_component(category "${example_filename}" DIRECTORY) + + # Source additional options that may exist for category + include(${category}/categoryOptions.cmake OPTIONAL) + + ippcp_define_example(example ${example_filename} ${category}) +endforeach() diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/README.md b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/README.md new file mode 100644 index 0000000..a262415 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/README.md @@ -0,0 +1,95 @@ +# Building usage examples of Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) library + +## System requirements + +- CMake 3.15 + +## Build with library sources + +Only merged library (cmake option `-DMERGED_BLD:BOOL=ON`) builds are supported for the examples. + +1. Navigate to the library sources root and run cmake to generate a library build procedure with the `-DBUILD_EXAMPLES:BOOL=ON` + option. + + On Linux\*/macOS\*: + + `cmake CMakeLists.txt -B_build -DARCH=intel64 -DMERGED_BLD:BOOL=ON -DBUILD_EXAMPLES:BOOL=ON` + + On Windows\*: + + `cmake CMakeLists.txt -B_build -G [-T"Intel® C++ Compiler "] -DBUILD_EXAMPLES:BOOL=ON` + + For the Visual Studio\* generators options, please refer to the CMake help. + The toolchain switch is optional, specify it if you want to build the library and examples using Intel® C++ Compiler. + + For the list of supported compiler versions or other cmake build options, please refer to the library root README.md file. + +2. On Linux\*/macOS\*, build with `make -j8 `. You can use the following targets: + + - To build an invididual example, use targets started with the *example_* string (like *example_aes-256-ctr-encryption*). + + - To build all examples of a single specific category, use target *ippcp_examples_\* (like *ippcp_examples_aes*). + + - To build all examples, use target *ippcp_examples_all*. + +3. On Windows\* OS open generated Visual Studio\* solution in the IDE, select the appropriate project (individual example, + all examples by category or the whole set of examples) in the *examples* folder of project structure in IDE and run **Build**. + + To run the build from the command line, open "Developer Command Prompt for VS \" console and run the cmake command: + + `cmake --build _build --target --config Release`, + + where '_build' is the path to CMake build directory. + +## Build with pre-built library + +1. Navigate to the *examples* folder and run the cmake command below. + + On Linux\*/macOS\*: + + `cmake CMakeLists.txt -B_build` + + On Windows\* OS it is required to specify a generator (`-G` option) and optionally a toolchain (`-T` option) + to build with Intel® C++ Compiler. Example: + + `cmake CMakeLists.txt -B_build -G [-T"Intel C++ Compiler "]` + + For the Visual Studio\* generators options, please refer to the CMake help. + +2. The build system will scan the system for the Intel IPP Cryptography library. + If it is found, you’ll see the following message: + + ``` + -- Found Intel IPP Cryptography at: /home/user/intel/ippcp + -- Configuring done + ``` + + If the library is not found automatically, you can specify the path to the library root folder + (where the include/ and lib/ directories are located) using the `-DIPPCRYPTO_ROOT_DIR=` option. + +3. Run the build process as described in the [Build with library sources](#build-with-library-sources). + + +# How to add a new example into Intel IPP Cryptography library: + +1. Choose a category (a folder), where to put the example, and a filename. Use + existing folders where applicable. + The file name should be as follows: "\-\-\-\.cpp" + E.g.: "rsa-1k-oaep-sha1-type2-decryption.cpp" for the example of RSA category. + +2. Write an example keeping its source code formatting consistent with other + examples as much as possible. The "aes/aes-256-ctr-encryption.cpp" can be used + as a reference. + +3. Use Doxygen annotations for the file description, global variables and + macros. The *main()* function shall not use doxygen annotations inside + (otherwise they disappear in the source code section of an example page in + the generated documentation). + +4. Add the example to the build: open *examples/CMakeLists.txt* file and add the + new file to the *IPPCP_EXAMPLES* list. + +5. Make sure it can be built using Intel IPP Cryptography examples build procedure, and it + works correctly. + +You are ready to submit a pull request! diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/aes/aes-256-ctr-decryption.cpp b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/aes/aes-256-ctr-decryption.cpp new file mode 100644 index 0000000..3945c4a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/aes/aes-256-ctr-decryption.cpp @@ -0,0 +1,137 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/*! + * + * \file + * + * \brief AES Counter mode of operation (CTR) example + * + * This example demonstrates usage of AES block cipher with 256-bit key + * run with CTR mode of operation. Decryption scheme. + * + * The CTR mode of operation is implemented according to the + * "NIST Special Publication 800-38A: Recommendation for Block Cipher Modes of + * Operation" document: + * + * https://csrc.nist.gov/publications/detail/sp/800-38a/final + * + */ + +#include + +#include "ippcp.h" +#include "examples_common.h" + +/*! AES block size in bytes */ +static const int AES_BLOCK_SIZE = 16; + +/*! Key size in bytes */ +static const int KEY_SIZE = 32; + +/*! Message size in bytes */ +static const int SRC_LEN = 16; + +/*! Plain text */ +static Ipp8u plainText[SRC_LEN] = { + 0x6b,0xc1,0xbe,0xe2,0x2e,0x40,0x9f,0x96, + 0xe9,0x3d,0x7e,0x11,0x73,0x93,0x17,0x2a +}; + +/*! Cipher text */ +static Ipp8u cipherText[SRC_LEN] = { + 0x60,0x1e,0xc3,0x13,0x77,0x57,0x89,0xa5, + 0xb7,0xa7,0xf5,0x04,0xbb,0xf3,0xd2,0x28 +}; + +/*! 256-bit secret key */ +static Ipp8u key256[KEY_SIZE] = { + 0x60,0x3d,0xeb,0x10,0x15,0xca,0x71,0xbe, + 0x2b,0x73,0xae,0xf0,0x85,0x7d,0x77,0x81, + 0x1f,0x35,0x2c,0x07,0x3b,0x61,0x08,0xd7, + 0x2d,0x98,0x10,0xa3,0x09,0x14,0xdf,0xf4 +}; + +/*! Initial counter for CTR mode. + * Size of counter for AES-CTR shall be equal to the size of AES block (16 bytes). + */ +static Ipp8u initialCounter[AES_BLOCK_SIZE] = { + 0xf0,0xf1,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7, + 0xf8,0xf9,0xfa,0xfb,0xfc,0xfd,0xfe,0xff +}; + +/*! Main function */ +int main(void) +{ + /* Length of changeable bits in a counter (can be value starting from 1 till block size 128) */ + const Ipp32u counterLen = 128; + + /* Size of AES context structure. It will be set up in ippsAESGetSize(). */ + int ctxSize = 0; + + Ipp8u pOut[SRC_LEN] = {}; + Ipp8u pCounter[AES_BLOCK_SIZE] = {}; + + /* Internal function status */ + IppStatus status = ippStsNoErr; + + /* Pointer to AES context structure */ + IppsAESSpec* pAES = 0; + + do { + /* 1. Get size needed for AES context structure */ + status = ippsAESGetSize(&ctxSize); + if (!checkStatus("ippsAESGetSize", ippStsNoErr, status)) + return status; + + /* 2. Allocate memory for AES context structure */ + pAES = (IppsAESSpec*)(new Ipp8u[ctxSize]); + if (NULL == pAES) { + printf("ERROR: Cannot allocate memory (%d bytes) for AES context\n", ctxSize); + return -1; + } + + /* 3. Initialize AES context */ + status = ippsAESInit(key256, sizeof(key256), pAES, ctxSize); + if (!checkStatus("ippsAESInit", ippStsNoErr, status)) + break; + + /* Initialize counter before decryption. + * An updated counter value will be stored here after ippsAESDecryptCTR finishes. + */ + memcpy(pCounter, initialCounter, sizeof(initialCounter)); + + /* 4. Decryption */ + status = ippsAESDecryptCTR(cipherText, pOut, sizeof(cipherText), pAES, pCounter, counterLen); + if (!checkStatus("ippsAESDecryptCTR", ippStsNoErr, status)) + break; + + /* Compare decrypted message and original text */ + if (0 != memcmp(pOut, plainText, sizeof(plainText))) { + printf("ERROR: Decrypted and plain text messages do not match\n"); + break; + } + } while (0); + + /* 5. Remove secret and release resources */ + ippsAESInit(0, KEY_SIZE, pAES, ctxSize); + if (pAES) delete [] (Ipp8u*)pAES; + + PRINT_EXAMPLE_STATUS("ippsAESDecryptCTR", "AES-CTR 256 Decryption", !status); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/aes/aes-256-ctr-encryption.cpp b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/aes/aes-256-ctr-encryption.cpp new file mode 100644 index 0000000..a384a16 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/aes/aes-256-ctr-encryption.cpp @@ -0,0 +1,137 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/*! + * + * \file + * + * \brief AES Counter mode of operation (CTR) example + * + * This example demonstrates usage of AES block cipher with 256-bit key + * run with CTR mode of operation. Encryption scheme. + * + * The CTR mode of operation is implemented according to the + * "NIST Special Publication 800-38A: Recommendation for Block Cipher Modes of + * Operation" document: + * + * https://csrc.nist.gov/publications/detail/sp/800-38a/final + * + */ + +#include + +#include "ippcp.h" +#include "examples_common.h" + +/*! AES block size in bytes */ +static const int AES_BLOCK_SIZE = 16; + +/*! Key size in bytes */ +static const int KEY_SIZE = 32; + +/*! Message size in bytes */ +static const int SRC_LEN = 16; + +/*! Plain text */ +static Ipp8u plainText[SRC_LEN] = { + 0x6b,0xc1,0xbe,0xe2,0x2e,0x40,0x9f,0x96, + 0xe9,0x3d,0x7e,0x11,0x73,0x93,0x17,0x2a +}; + +/*! Cipher text */ +static Ipp8u cipherText[SRC_LEN] = { + 0x60,0x1e,0xc3,0x13,0x77,0x57,0x89,0xa5, + 0xb7,0xa7,0xf5,0x04,0xbb,0xf3,0xd2,0x28 +}; + +/*! 256-bit secret key */ +static Ipp8u key256[KEY_SIZE] = { + 0x60,0x3d,0xeb,0x10,0x15,0xca,0x71,0xbe, + 0x2b,0x73,0xae,0xf0,0x85,0x7d,0x77,0x81, + 0x1f,0x35,0x2c,0x07,0x3b,0x61,0x08,0xd7, + 0x2d,0x98,0x10,0xa3,0x09,0x14,0xdf,0xf4 +}; + +/*! Initial counter for CTR mode. + * Size of counter for AES-CTR shall be equal to the size of AES block (16 bytes). + */ +static Ipp8u initialCounter[AES_BLOCK_SIZE] = { + 0xf0,0xf1,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7, + 0xf8,0xf9,0xfa,0xfb,0xfc,0xfd,0xfe,0xff +}; + +/*! Main function */ +int main(void) +{ + /* Length of changeable bits in a counter (can be value starting from 1 till block size 128) */ + const Ipp32u counterLen = 128; + + /* Size of AES context structure. It will be set up in ippsAESGetSize(). */ + int ctxSize = 0; + + Ipp8u pOut[SRC_LEN] = {}; + Ipp8u pCounter[AES_BLOCK_SIZE] = {}; + + /* Internal function status */ + IppStatus status = ippStsNoErr; + + /* Pointer to AES context structure */ + IppsAESSpec* pAES = 0; + + do { + /* 1. Get size needed for AES context structure */ + status = ippsAESGetSize(&ctxSize); + if (!checkStatus("ippsAESGetSize", ippStsNoErr, status)) + return status; + + /* 2. Allocate memory for AES context structure */ + pAES = (IppsAESSpec*)(new Ipp8u[ctxSize]); + if (NULL == pAES) { + printf("ERROR: Cannot allocate memory (%d bytes) for AES context\n", ctxSize); + return -1; + } + + /* 3. Initialize AES context */ + status = ippsAESInit(key256, sizeof(key256), pAES, ctxSize); + if (!checkStatus("ippsAESInit", ippStsNoErr, status)) + break; + + /* Initialize counter before encryption. + * An updated counter value will be stored here after ippsAESEncryptCTR finishes. + */ + memcpy(pCounter, initialCounter, sizeof(initialCounter)); + + /* 4. Encryption */ + status = ippsAESEncryptCTR(plainText, pOut, sizeof(plainText), pAES, pCounter, counterLen); + if (!checkStatus("ippsAESEncryptCTR", ippStsNoErr, status)) + break; + + /* Compare encrypted message and reference text */ + if (0 != memcmp(pOut, cipherText, sizeof(cipherText))) { + printf("ERROR: Encrypted and reference messages do not match\n"); + break; + } + } while (0); + + /* 5. Remove secret and release resources */ + ippsAESInit(0, KEY_SIZE, pAES, ctxSize); + if (pAES) delete [] (Ipp8u*)pAES; + + PRINT_EXAMPLE_STATUS("ippsAESEncryptCTR", "AES-CTR 256 Encryption", !status) + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/dsa/dsa-dlp-sha-1-verification.cpp b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/dsa/dsa-dlp-sha-1-verification.cpp new file mode 100644 index 0000000..ce6f1fa --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/dsa/dsa-dlp-sha-1-verification.cpp @@ -0,0 +1,183 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/*! + * + * \file + * + * \brief DSA-DLP verification scheme example + * + * This example demonstrates message verification according to + * DSA-DLP scheme with (L = 1024, N = 160) DSA parameters and SHA-1 hash function. + * + * The DSA-DLP scheme is implemented according : Digital Signature Standard (DSS) (FIPS PUB 186-4) (July 2013) + * + * available at: + * + * http://dx.doi.org/10.6028/NIST.FIPS.186-4. + * + */ + +#include + +#include "bignum.h" +#include "examples_common.h" +#include "ippcp.h" + +/*! Parameters DSA-DLP scheme */ +static const int L_BIT = 1024; +static const int N_BIT = 160; + +/*! Message size in bytes */ +static const int MSG_LEN_BYTE = 6; + +/*! Message text */ +static Ipp8u MSG[MSG_LEN_BYTE] = {0x31, 0x32, 0x33, + 0x34, 0x30, 0x30}; + +/*! The generator of the multiplicative subgroup */ +static const BigNumber G( + "0x" + "0835AA8C358BBF01A1846D1206323FABE408B0E98789FCC6239DA14D4B3F86C2" + "76A8F48AA85A59507E620AD1BC745F0F1CBF63EC98C229C2610D77C634D1642E" + "404354771655B2D5662F7A45227178CE3430AF0F6B3BB94B52F7F51E97BAD659" + "B1BA0684E208BE624C28D82FB1162F18DD9DCE45216461654CF3374624D15A8D"); + +/*! The modulus p */ +static const BigNumber P( + "0x" + "B34CE9C1E78294D3258473842005D2A48C8C566CFCA8F84C0606F2529B59A6D3" + "8AAE071B53BB2167EAA4FC3B01FE176E787E481B6037AAC62CBC3D089799536A" + "869FA8CDFEA1E8B1FD2D1CD3A30350859A2CD6B3EC2F9BFBB68BB11B4BBE2ADA" + "A18D64A93639543AE5E16293E311C0CF8C8D6E180DF05D08C2FD2D93D570751F"); + +/*! The order of the generator g */ +static const BigNumber Q("0xB90B38BA0A50A43EC6898D3F9B68049777F489B1"); + +/*! The public key value */ +static const BigNumber Y( + "0x" + "173931DDA31EFF32F24B383091BF77EACDC6EFD557624911D8E9B9DEBF0F256D" + "0CFFAC5567B33F6EAAE9D3275BBED7EF9F5F94C4003C959E49A1ED3F58C31B21" + "BACCC0ED8840B46145F121B8906D072129BAE01F071947997E8EF760D2D9EA21" + "D08A5EB7E89390B21A85664713C549E25FEDA6E9E6C31970866BDFBC8FA981F6"); + +/*! R digital signature component */ +static const BigNumber sigR("0xAA6A258FBF7D90E15614676D377DF8B10E38DB4A"); + +/*! S digital signature component */ +static const BigNumber sigS("0x496D5220B5F67D3532D1F991203BC3523B964C3B"); + +int main(void) { + /*! Internal function status */ + IppStatus status = ippStsNoErr; + + /* Pointer to DSA DLP context structure */ + IppsDLPState* pDL = NULL; + + /* Result verification DSA DLP */ + IppDLResult result = ippDLValid; + + /* Size of DSA-DLP context structure. It will be set up in ippsDLPGetSize(). */ + int DLSize = 0; + + /* Digest size */ + const int digestSizeBit = IPP_SHA1_DIGEST_BITSIZE; + const int digestSizeByte = digestSizeBit / 8; + /* Pointer to the SHA-1 hash method */ + const IppsHashMethod* hashMethod = ippsHashMethod_SHA1(); + + /*! Algorithm */ + do { + /* 1. Create a digest by message */ + /*! Buffer create digest */ + Ipp8u md[digestSizeByte] = {}; + + /*! Create digest by message */ + status = ippsHashMessage_rmf(MSG, + MSG_LEN_BYTE, + md, + hashMethod); + /*! Check status create digest */ + if (ippStsNoErr != status) + break; + + /*! + * (!) Allocate BigNumber container for the shrank message digest. + * Note, the DSA algorithm uses only leftmost |minSizeDigestBit| bits of the original message digest + */ + const int minSizeDigestBit = IPP_MIN(N_BIT, digestSizeBit); + BigNumber digest(NULL, bitSizeInWords(minSizeDigestBit)); + + /*! Set digest to BigNumber */ + status = ippsSetOctString_BN(md, + bitSizeInBytes(minSizeDigestBit), + digest); + if (ippStsNoErr != status) + break; + + /* 2. Get size needed for DSA DLP context structure */ + status = ippsDLPGetSize(L_BIT, N_BIT, + &DLSize); + if (ippStsNoErr != status) + break; + + /* 3. Allocate memory for DSA DLP context structure */ + pDL = (IppsDLPState*)(new Ipp8u[DLSize]); + if (NULL == pDL) { + printf("ERROR: Cannot allocate memory (%d bytes) for DSA DLP context\n", DLSize); + return -1; + } + + /* 4. Initialize DSA DLP context */ + status = ippsDLPInit(L_BIT, N_BIT, + pDL); + if (ippStsNoErr != status) + break; + + /* 5. Set DL Domain Parameters */ + status = ippsDLPSet(P, Q, G, pDL); + if (ippStsNoErr != status) + break; + + /* 6. Set up Key Pair into the DL context */ + status = ippsDLPSetKeyPair(NULL, /* optional Private Key Set */ + Y, + pDL); + if (ippStsNoErr != status) + break; + + /* 7. Verify Signature DSA DLP */ + status = ippsDLPVerifyDSA(digest, + sigR, sigS, + &result, + pDL); + if (ippStsNoErr != status) + break; + if (ippDLValid != result) + status = ippStsErr; + + } while (0); /* end Algorithm */ + + /* 8. Remove secret and release resources */ + if (NULL != pDL) + delete[](Ipp8u*) pDL; + + PRINT_EXAMPLE_STATUS("ippsDLPVerifyDSA", "DSA-DLP Verification Hash Method Message SHA-1", ippStsNoErr == status); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/dsa/dsa-dlp-sha-256-verification.cpp b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/dsa/dsa-dlp-sha-256-verification.cpp new file mode 100644 index 0000000..98aa71f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/dsa/dsa-dlp-sha-256-verification.cpp @@ -0,0 +1,207 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/*! + * + * \file + * + * \brief DSA-DLP verification scheme example + * + * This example demonstrates message verification according to + * DSA-DLP scheme with (L = 3072, N = 256) DSA parameters and SHA-256 hash function. + * + * The DSA-DLP scheme is implemented according : Digital Signature Standard (DSS) (FIPS PUB 186-4) (July 2013) + * + * available at: + * + * http://dx.doi.org/10.6028/NIST.FIPS.186-4. + * + */ + +#include + +#include "bignum.h" +#include "examples_common.h" +#include "ippcp.h" + +/*! Parameters DSA-DLP scheme */ +static const int L_BIT = 3072; +static const int N_BIT = 256; + +/*! Message size in bytes */ +static const int MSG_LEN_BYTE = 6; + +/*! Message text */ +static Ipp8u MSG[MSG_LEN_BYTE] = {0x31, 0x32, 0x33, + 0x34, 0x30, 0x30}; + +/*! The generator of the multiplicative subgroup */ +static const BigNumber G( + "0x" + "6DEA4B8C3FE3AB91E3229FB14C1CFA822915769AF161405F48B7FADFE1EC5D9F" + "EC4EF0CFBB2233FFDDFA5A554CFC68C6BC6A0BA30CEF6F51309294E622B58D4F" + "ACE00AE9669D9172B15696839ED332AFD906E3F427D85A9AF73562B845BE53A3" + "713C0219402A4C208E9B6A6873235E0BC20442E70AB69EDD46E8F3F7D58CB35E" + "A3690C673F54CD37377725739F00EBE2B3B53BDAF89DDAC74012F8486BD3F521" + "7579B4A303F61BCCC98931FABA969C8C2A27ACB04BC21201EDF9A7F6B42E10F7" + "5DD23C3AB073D7290D173EBE6CB1919607BFE2BF0D829A609D8D3CDA7044FF8D" + "FBBD463E68C9403A45834EC547A7D4FD5ABC68C5997CDC397120698F879356E0" + "E74B62FE1A2938A5D1B486B53A5E0CB875E23A2E834EA563A4A9D4BE44045877" + "DF020C30E22E55603F63D74ED2CAFDE18180EC294A7CE263D56EB280562687F6" + "1F898F3C7D2B37D7F00250A43CA989DE16FA1AAB7D83E0DBF6AA66EDC36AD79E" + "ECFE2F91CFAB6285BA10AE713126F69326540C461E44E45BDF076E4ED8D3E924"); + +/*! The modulus p */ +static const BigNumber P( + "0x" + "DCD2F71FBA7AEB46AEEA858AB76F2102FA97A953ABCE9D791AA269F0161733AC" + "3DF25F5C9DB3448F82846E355E23089614046D42B030298D94F5365D942CBB54" + "90E40A1D5E6E577CC646A807F049A1FB42B97A9E64EFA1AA9EF93BB3C7120DDB" + "F9C403E580431F1789127F0A64EA7B036EF12D07F02103655D63DDDA3C44AD32" + "8F727C1D060FC92E3616976CF11BF1FEEFFF033490D98929252B585CD92C081A" + "FCC71DAE6341AE8DD05E62AE297AD2B00560EC94F1F64482816E3AF052FC1DAF" + "F0A9BF52034012594D4246036D040FA5E741E693E36B064BEDB224EA1F7C6C86" + "171CA8FCFAC98C5DB6E34DAD307C5BFDECE4E578F0E18FCAAEE9D5B330ED69A7" + "2D8FDDF878A58A57914247825AE6ED1CB8A6B241EA694B77F843EEE40F1BE90F" + "26B26154813647D1E1AF01254CF21CFDD2E9EBA7E431BD8DB6164D05A3D3AE93" + "71AF5D0D39A3A9B9F07BA61233C77A6BFC273515FB844DB8FAFD69B559CE844C" + "7A3D686EA4991D9FE74CAD560489F3C1DBB4FD171EA8AE7874E302207C02A7B1"); + +/*! The order of the generator g */ +static const BigNumber Q("0xF870D35CA9F84E6ACAD808D6AC35B13EE4073F26EA84FEFE08C4D9A565754037"); + +/*! The public key value */ +static const BigNumber Y( + "0x" + "7E0062E6F32ECAB1EFFA38391B71A523221D6E97A61F55238F0A623CC42980B9" + "87FB22EC6138E8D1C0D36C05D059CA0CA3F1A526A5F67B216341ABCD04105BB8" + "EFC9E479C2532F9DEA6CDDFB4F57DE5B9D6964E3D314EB89693A3D57F82B9FF9" + "3E0E0D11D72FA4F1BD82BB2B20F1B59547AFF7711DB319D7D06E6964BEB294E4" + "4D34C2A21C7AC7CDAC5E91F2F6D183042AFC3644B09837FA2225A074CEB65D49" + "9F73CEE04C705C82BB912F97D765D5F9C8CB442019E7DAC1E1CCCCEE990335EA" + "3B8C837583595CD4F83169D4787FE4675386D604E8E205B977C7AB2369504282" + "54E3B836BD00296257238D22BDA16A722E405DF82029E3384931FB0E4903C3F8" + "771FB15708D4CB3238E7B2A68131BE518A08D6EFD483A01537A432046DCBD1FF" + "A5FF831E0257B292012D5E1A44C6E32019A6B3AE176A67EDAF12EB27E68FA60A" + "05AF4E5448D606C392B4A672B44298B1775A16B9440B131EB0D91CA3FDE1A1E5" + "28B5FFFC31FFDF1449169C2F4ABD96809A75FB6C85AE845940C45D5AF8334057"); + +/*! R digital signature component */ +static const BigNumber sigR("0x8C7F9B7F5D53A4D3877185D2EF69B2B3AB16321D996940C2EFCBEF01C0B598B4"); + +/*! S digital signature component */ +static const BigNumber sigS("0x33708B0FD4109873550D02340918EC93E5383D471C9D0A322AA76C2872783CBA"); + +int main(void) { + /*! Internal function status */ + IppStatus status = ippStsNoErr; + + /* Pointer to DSA DLP context structure */ + IppsDLPState* pDL = NULL; + + /* Result verification DSA DLP */ + IppDLResult result = ippDLValid; + + /* Size of DSA-DLP context structure. It will be set up in ippsDLPGetSize(). */ + int DLSize = 0; + + /* Digest size */ + const int digestSizeBit = IPP_SHA256_DIGEST_BITSIZE; + const int digestSizeByte = digestSizeBit / 8; + /* Pointer to the SHA-256 hash method */ + const IppsHashMethod* hashMethod = ippsHashMethod_SHA256(); + + /*! Algorithm */ + do { + /* 1. Create a digest by message */ + /*! Buffer create digest */ + Ipp8u md[digestSizeByte] = {}; + + /*! Create digest by message */ + status = ippsHashMessage_rmf(MSG, + MSG_LEN_BYTE, + md, + hashMethod); + /*! Check status create digest */ + if (ippStsNoErr != status) + break; + + /*! + * (!) Allocate BigNumber container for the shrank message digest. + * Note, the DSA algorithm uses only leftmost |minSizeDigestBit| bits of the original message digest + */ + const int minSizeDigestBit = IPP_MIN(N_BIT, digestSizeBit); + BigNumber digest(NULL, bitSizeInWords(minSizeDigestBit)); + + /*! Set digest to BigNumber */ + status = ippsSetOctString_BN(md, + bitSizeInBytes(minSizeDigestBit), + digest); + if (ippStsNoErr != status) + break; + + /* 2. Get size needed for DSA DLP context structure */ + status = ippsDLPGetSize(L_BIT, N_BIT, + &DLSize); + if (ippStsNoErr != status) + break; + + /* 3. Allocate memory for DSA DLP context structure */ + pDL = (IppsDLPState*)(new Ipp8u[DLSize]); + if (NULL == pDL) { + printf("ERROR: Cannot allocate memory (%d bytes) for DSA DLP context\n", DLSize); + return -1; + } + + /* 4. Initialize DSA DLP context */ + status = ippsDLPInit(L_BIT, N_BIT, + pDL); + if (ippStsNoErr != status) + break; + + /* 5. Set DL Domain Parameters */ + status = ippsDLPSet(P, Q, G, pDL); + if (ippStsNoErr != status) + break; + + /* 6. Set up Key Pair into the DL context */ + status = ippsDLPSetKeyPair(NULL, /* optional Private Key Set */ + Y, + pDL); + if (ippStsNoErr != status) + break; + + /* 7. Verify Signature DSA DLP */ + status = ippsDLPVerifyDSA(digest, + sigR, sigS, + &result, + pDL); + if (ippStsNoErr != status) + break; + if (ippDLValid != result) + status = ippStsErr; + + } while (0); /* end Algorithm */ + + /* 8. Remove secret and release resources */ + if (NULL != pDL) + delete[](Ipp8u*) pDL; + + PRINT_EXAMPLE_STATUS("ippsDLPVerifyDSA", "DSA-DLP Verification Hash Method Message SHA-256", ippStsNoErr == status); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/rsa/rsa-1k-oaep-sha1-encryption.cpp b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/rsa/rsa-1k-oaep-sha1-encryption.cpp new file mode 100644 index 0000000..3550d1c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/rsa/rsa-1k-oaep-sha1-encryption.cpp @@ -0,0 +1,131 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/*! + * + * \file + * + * \brief RSA-OAEP encryption scheme usage example. + * + * This example demonstrates message encryption according to + * RSA-OAEP scheme with 1024-bit RSA modulus and SHA-1 hash function. + * It uses Reduced Memory Footprint (_rmf) version of the function. + * + * The RSASSA-OAEP scheme is implemented according to the PKCS#1 v2.1: RSA Cryptography Standard (June 2002), + * available at: + * + * https://tools.ietf.org/html/rfc3447. + * + */ + +#include + +#include "ippcp.h" +#include "examples_common.h" +#include "bignum.h" + +/*! 1024-bit RSA Modulus N = P*Q */ +static BigNumber N("0xBBF82F090682CE9C2338AC2B9DA871F7368D07EED41043A440D6B6F07454F51FB8DFBAAF035C02AB61EA48CEEB6FCD4876ED520D60E1E" + "C4619719D8A5B8B807FAFB8E0A3DFC737723EE6B4B7D93A2584EE6A649D060953748834B2454598394EE0AAB12D7B61A51F527A9A41F6C1" + "687FE2537298CA2A8F5946F8E5FD091DBDCB"); + +/*! Public exponent */ +static BigNumber E("0x11"); + +/*! Plain text source message */ +static Ipp8u sourceMessage[] = + "\xd4\x36\xe9\x95\x69\xfd\x32\xa7" + "\xc8\xa0\x5b\xbc\x90\xd3\x2c\x49"; + +/*! Seed string of hash size */ +static Ipp8u seed[] = "\xaa\xfd\x12\xf6\x59\xca\xe6\x34\x89\xb4" + "\x79\xe5\x07\x6d\xde\xc2\xf0\x6c\xb5\x8f"; + +/*! Reference cipher text. Length is equal to RSA modulus size */ +static Ipp8u cipherTextRef[] = + "\x12\x53\xE0\x4D\xC0\xA5\x39\x7B\xB4\x4A\x7A\xB8\x7E\x9B\xF2\xA0" + "\x39\xA3\x3D\x1E\x99\x6F\xC8\x2A\x94\xCC\xD3\x00\x74\xC9\x5D\xF7" + "\x63\x72\x20\x17\x06\x9E\x52\x68\xDA\x5D\x1C\x0B\x4F\x87\x2C\xF6" + "\x53\xC1\x1D\xF8\x23\x14\xA6\x79\x68\xDF\xEA\xE2\x8D\xEF\x04\xBB" + "\x6D\x84\xB1\xC3\x1D\x65\x4A\x19\x70\xE5\x78\x3B\xD6\xEB\x96\xA0" + "\x24\xC2\xCA\x2F\x4A\x90\xFE\x9F\x2E\xF5\xC9\xC1\x40\xE5\xBB\x48" + "\xDA\x95\x36\xAD\x87\x00\xC8\x4F\xC9\x13\x0A\xDE\xA7\x4E\x55\x8D" + "\x51\xA7\x4D\xDF\x85\xD8\xB5\x0D\xE9\x68\x38\xD6\x06\x3E\x09\x55"; + +/*! Main function */ +int main(void) +{ + /* Internal function status */ + IppStatus status = ippStsNoErr; + + /* Size in bits of RSA modulus */ + const int bitSizeN = N.BitSize(); + /* Size in bits of RSA public exponent */ + const int bitSizeE = E.BitSize(); + + /* Allocate memory for public key. */ + int keySize = 0; + ippsRSA_GetSizePublicKey(bitSizeN, bitSizeE, &keySize); + IppsRSAPublicKeyState* pPubKey = (IppsRSAPublicKeyState*)(new Ipp8u[keySize]); + ippsRSA_InitPublicKey(bitSizeN, bitSizeE, pPubKey, keySize); + + /* Allocate memory for cipher text, not less than RSA modulus size. */ + int cipherTextLen = bitSizeInBytes(bitSizeN); + Ipp8u* pCipherText = new Ipp8u[cipherTextLen]; + + do { + /* Set public key */ + status = ippsRSA_SetPublicKey(N, E, pPubKey); + if (!checkStatus("ippsRSA_SetPublicKey", ippStsNoErr, status)) + break; + + /* Calculate temporary buffer size */ + int pubBufSize = 0; + status = ippsRSA_GetBufferSizePublicKey(&pubBufSize, pPubKey); + if (!checkStatus("ippsRSA_GetBufferSizePublicKey", ippStsNoErr, status)) + break; + + /* Allocate memory for temporary buffer */ + Ipp8u* pScratchBuffer = new Ipp8u[pubBufSize]; + + /* Encrypt message */ + status = ippsRSAEncrypt_OAEP_rmf(sourceMessage, sizeof(sourceMessage)-1, + 0 /* optional label assotiated with the sourceMessage */, + 0, /* label length */ + seed, pCipherText, pPubKey, + ippsHashMethod_SHA1(), + pScratchBuffer); + + if (pScratchBuffer) delete [] pScratchBuffer; + + if (!checkStatus("ippsRSAEncrypt_OAEP_rmf", ippStsNoErr, status)) + break; + + if (0 != memcmp(cipherTextRef, pCipherText, sizeof(cipherTextRef)-1)) { + printf("ERROR: Encrypted and reference messages do not match\n"); + status = ippStsErr; + } + } while (0); + + PRINT_EXAMPLE_STATUS("ippsRSAEncrypt_OAEP_rmf", "RSA-OAEP 1024 (SHA1) Encryption", ippStsNoErr == status); + + if (pCipherText) delete [] pCipherText; + if (pPubKey) delete [] (Ipp8u*)pPubKey; + + return status; +} + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/rsa/rsa-1k-oaep-sha1-type2-decryption.cpp b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/rsa/rsa-1k-oaep-sha1-type2-decryption.cpp new file mode 100644 index 0000000..3f43d9e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/rsa/rsa-1k-oaep-sha1-type2-decryption.cpp @@ -0,0 +1,146 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/*! + * + * \file + * + * \brief RSA-OAEP decryption scheme usage example. + * + * This example demonstrates message decryption according to + * RSA-OAEP scheme with 1024-bit RSA modulus and SHA-1 hash function. + * The private key of Type2 (to be able to apply Chinese Reminder Theorem) is used in this example. + * It uses Reduced Memory Footprint (_rmf) version of the function. + * + * The RSASSA-OAEP scheme is implemented according to the PKCS#1 v2.1: RSA Cryptography Standard (June 2002), + * available at: + * + * https://tools.ietf.org/html/rfc3447. + * + */ + +#include + +#include "ippcp.h" +#include "examples_common.h" +#include "bignum.h" + +/*! Prime P factor */ +static BigNumber P("0xEECFAE81B1B9B3C908810B10A1B5600199EB9F44AEF4FDA493B81A9E3D84F632" + "124EF0236E5D1E3B7E28FAE7AA040A2D5B252176459D1F397541BA2A58FB6599"); + +/*! Prime Q factor */ +static BigNumber Q("0xC97FB1F027F453F6341233EAAAD1D9353F6C42D08866B1D05A0F2035028B9D86" + "9840B41666B42E92EA0DA3B43204B5CFCE3352524D0416A5A441E700AF461503"); + +/*! D mod (p-1) factor */ +static BigNumber DP("0x54494CA63EBA0337E4E24023FCD69A5AEB07DDDC0183A4D0AC9B54B051F2B13E" + "D9490975EAB77414FF59C1F7692E9A2E202B38FC910A474174ADC93C1F67C981"); + +/*! D mod (q-1) factor */ +static BigNumber DQ("0x471E0290FF0AF0750351B7F878864CA961ADBD3A8A7E991C5C0556A94C3146A7" + "F9803F8F6F8AE342E931FD8AE47A220D1B99A495849807FE39F9245A9836DA3D"); + +/*! Q^-1 mod p factor */ +static BigNumber InvQ("0xB06C4FDABB6301198D265BDBAE9423B380F271F73453885093077FCD39E2119F" + "C98632154F5883B167A967BF402B4E9E2E0F9656E698EA3666EDFB25798039F7"); + +/*! Plain text */ +static Ipp8u sourceMessageRef[] = + "\xd4\x36\xe9\x95\x69\xfd\x32\xa7" + "\xc8\xa0\x5b\xbc\x90\xd3\x2c\x49"; + +/*! Cipher text to decrypt. */ +static Ipp8u cipherText[] = + "\x12\x53\xE0\x4D\xC0\xA5\x39\x7B\xB4\x4A\x7A\xB8\x7E\x9B\xF2\xA0" + "\x39\xA3\x3D\x1E\x99\x6F\xC8\x2A\x94\xCC\xD3\x00\x74\xC9\x5D\xF7" + "\x63\x72\x20\x17\x06\x9E\x52\x68\xDA\x5D\x1C\x0B\x4F\x87\x2C\xF6" + "\x53\xC1\x1D\xF8\x23\x14\xA6\x79\x68\xDF\xEA\xE2\x8D\xEF\x04\xBB" + "\x6D\x84\xB1\xC3\x1D\x65\x4A\x19\x70\xE5\x78\x3B\xD6\xEB\x96\xA0" + "\x24\xC2\xCA\x2F\x4A\x90\xFE\x9F\x2E\xF5\xC9\xC1\x40\xE5\xBB\x48" + "\xDA\x95\x36\xAD\x87\x00\xC8\x4F\xC9\x13\x0A\xDE\xA7\x4E\x55\x8D" + "\x51\xA7\x4D\xDF\x85\xD8\xB5\x0D\xE9\x68\x38\xD6\x06\x3E\x09\x55"; + +/*! Main function */ +int main(void) +{ + /* Internal function status */ + IppStatus status = ippStsNoErr; + + /* Size in bits of P factor */ + const int bitSizeP = P.BitSize(); + /* Size in bits of Q factor */ + const int bitSizeQ = Q.BitSize(); + + /* Allocate memory for private key. + * There are two types of private keys that are supported: Type1 and Type2. + * You can choose any of them, depending on your private key representation. + * This example uses Type2 key. + * For more information, see https://www.intel.com/content/www/us/en/docs/ipp-crypto/developer-reference/current/crypto-ref-getsize-public-private1-private2.html + */ + int keySize = 0; + ippsRSA_GetSizePrivateKeyType2(bitSizeP, bitSizeQ, &keySize); + IppsRSAPrivateKeyState* pPrvKeyType2 = (IppsRSAPrivateKeyState*)(new Ipp8u [keySize]); + ippsRSA_InitPrivateKeyType2(bitSizeP, bitSizeQ, pPrvKeyType2, keySize); + + /* Allocate memory for decrypted plain text, not less than RSA modulus size. */ + int plainTextLen = bitSizeInBytes(bitSizeP + bitSizeQ); + Ipp8u* pPlainText = new Ipp8u[plainTextLen]; + + do { + /* Set private key */ + status = ippsRSA_SetPrivateKeyType2(P, Q, DP, DQ, InvQ, pPrvKeyType2); + if (!checkStatus("ippsRSA_SetPrivateKeyType2", ippStsNoErr, status)) + break; + + /* Calculate temporary buffer size */ + int bufSize = 0; + status = ippsRSA_GetBufferSizePrivateKey(&bufSize, pPrvKeyType2); + if (!checkStatus("ippsRSA_GetBufferSizePrivateKey", ippStsNoErr, status)) + break; + + /* Allocate memory for temporary buffer */ + Ipp8u* pScratchBuffer = new Ipp8u[bufSize]; + + /* Decrypt message */ + status = ippsRSADecrypt_OAEP_rmf(cipherText, + 0 /* optional label to be assotiated with the message */, + 0, /* label length */ + pPlainText, &plainTextLen, + pPrvKeyType2, + ippsHashMethod_SHA1(), + pScratchBuffer); + + if (pScratchBuffer) delete [] pScratchBuffer; + + if (!checkStatus("ippsRSADecrypt_OAEP_rmf", ippStsNoErr, status)) + break; + + if (0 != memcmp(sourceMessageRef, pPlainText, sizeof(sourceMessageRef)-1)) { + printf("ERROR: Decrypted and plain text messages do not match\n"); + status = ippStsErr; + } + } while (0); + + PRINT_EXAMPLE_STATUS("ippsRSADecrypt_OAEP_rmf", "RSA-OAEP 1024 (SHA1) Type2 decryption", ippStsNoErr == status) + + if (pPlainText) delete [] pPlainText; + if (pPrvKeyType2) delete [] (Ipp8u*)pPrvKeyType2; + + return status; +} + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/rsa/rsa-1k-pss-sha1-verification.cpp b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/rsa/rsa-1k-pss-sha1-verification.cpp new file mode 100644 index 0000000..7800c95 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/rsa/rsa-1k-pss-sha1-verification.cpp @@ -0,0 +1,124 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/*! + * + * \file + * + * \brief RSASSA-PSS verification scheme usage example. + * + * This example demonstrates message verification according to + * RSASSA-PSS scheme with 1024-bit RSA modulus and SHA-1 hash function. + * It uses Reduced Memory Footprint (_rmf) version of the function. + * + * The RSASSA-PSS scheme is implemented according to the PKCS#1 v2.1: RSA Cryptography Standard (June 2002), + * available at: + * + * https://tools.ietf.org/html/rfc3447. + * + */ +#include "ippcp.h" +#include "examples_common.h" +#include "bignum.h" + +/*! 1024-bit RSA Modulus N = P*Q */ +static BigNumber N("0xA2BA40EE07E3B2BD2F02CE227F36A195024486E49C19CB41BBBDFBBA98B22B0E577C2EEAFFA20D883A76E65E394C69D4" + "B3C05A1E8FADDA27EDB2A42BC000FE888B9B32C22D15ADD0CD76B3E7936E19955B220DD17D4EA904B1EC102B2E4DE775" + "1222AA99151024C7CB41CC5EA21D00EEB41F7C800834D2C6E06BCE3BCE7EA9A5"); + +/*! Public exponent */ +static BigNumber E("0x10001"); + +/*! Message to be signed */ +static +Ipp8u sourceMessage[] = "\x85\x9e\xef\x2f\xd7\x8a\xca\x00\x30\x8b\xdc\x47\x11\x93\xbf\x55" + "\xbf\x9d\x78\xdb\x8f\x8a\x67\x2b\x48\x46\x34\xf3\xc9\xc2\x6e\x64" + "\x78\xae\x10\x26\x0f\xe0\xdd\x8c\x08\x2e\x53\xa5\x29\x3a\xf2\x17" + "\x3c\xd5\x0c\x6d\x5d\x35\x4f\xeb\xf7\x8b\x26\x02\x1c\x25\xc0\x27" + "\x12\xe7\x8c\xd4\x69\x4c\x9f\x46\x97\x77\xe4\x51\xe7\xf8\xe9\xe0" + "\x4c\xd3\x73\x9c\x6b\xbf\xed\xae\x48\x7f\xb5\x56\x44\xe9\xca\x74" + "\xff\x77\xa5\x3c\xb7\x29\x80\x2f\x6e\xd4\xa5\xff\xa8\xba\x15\x98" + "\x90\xfc"; + +/*! Signature to verify */ +static +Ipp8u signatureRef[] = "\x8d\xaa\x62\x7d\x3d\xe7\x59\x5d\x63\x05\x6c\x7e\xc6\x59\xe5\x44" + "\x06\xf1\x06\x10\x12\x8b\xaa\xe8\x21\xc8\xb2\xa0\xf3\x93\x6d\x54" + "\xdc\x3b\xdc\xe4\x66\x89\xf6\xb7\x95\x1b\xb1\x8e\x84\x05\x42\x76" + "\x97\x18\xd5\x71\x5d\x21\x0d\x85\xef\xbb\x59\x61\x92\x03\x2c\x42" + "\xbe\x4c\x29\x97\x2c\x85\x62\x75\xeb\x6d\x5a\x45\xf0\x5f\x51\x87" + "\x6f\xc6\x74\x3d\xed\xdd\x28\xca\xec\x9b\xb3\x0e\xa9\x9e\x02\xc3" + "\x48\x82\x69\x60\x4f\xe4\x97\xf7\x4c\xcd\x7c\x7f\xca\x16\x71\x89" + "\x71\x23\xcb\xd3\x0d\xef\x5d\x54\xa2\xb5\x53\x6a\xd9\x0a\x74\x7e"; + +/*! Main function */ +int main(void) +{ + /* Internal function status */ + IppStatus status = ippStsNoErr; + + /* Size in bits of RSA modulus */ + const int bitSizeN = N.BitSize(); + /* Size in bits of RSA public exponent */ + const int bitSizeE = E.BitSize(); + + /* Allocate memory for public key. */ + int keySize = 0; + ippsRSA_GetSizePublicKey(bitSizeN, bitSizeE, &keySize); + IppsRSAPublicKeyState* pPubKey = (IppsRSAPublicKeyState*)(new Ipp8u[keySize]); + ippsRSA_InitPublicKey(bitSizeN, bitSizeE, pPubKey, keySize); + + if (pPubKey) { + do { + /* Set public key */ + status = ippsRSA_SetPublicKey(N, E, pPubKey); + if (!checkStatus("ippsRSA_SetPublicKey", ippStsNoErr, status)) + break; + + /* Calculate temporary buffer size */ + int pubBufSize = 0; + status = ippsRSA_GetBufferSizePublicKey(&pubBufSize, pPubKey); + if (!checkStatus("ippsRSA_GetBufferSizePublicKey", ippStsNoErr, status)) + break; + + Ipp8u* pScratchBuffer = new Ipp8u[pubBufSize]; + + /* Verify message with use of SHA-1 hash function. The verification result will be placed + * into isValid variable. */ + int isValid = 0; + status = ippsRSAVerify_PSS_rmf(sourceMessage, sizeof(sourceMessage)-1, + signatureRef, &isValid, + pPubKey, + ippsHashMethod_SHA1(), + pScratchBuffer); + + if (pScratchBuffer) delete [] pScratchBuffer; + + if (!checkStatus("ippsRSAVerify_PSS_rmf", ippStsNoErr, status)) + break; + + /* If isValid is zero, then verification fails */ + status = isValid ? ippStsNoErr : ippStsErr; + } while (0); + } + + if (pPubKey) delete [] (Ipp8u*)pPubKey; + + PRINT_EXAMPLE_STATUS("ippsRSAVerify_PSS_rmf", "RSA-PSS 1024 (SHA1) Verification", ippStsNoErr == status); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/rsa/rsa-3k-pss-sha384-type1-signature.cpp b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/rsa/rsa-3k-pss-sha384-type1-signature.cpp new file mode 100644 index 0000000..120f1fb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/rsa/rsa-3k-pss-sha384-type1-signature.cpp @@ -0,0 +1,166 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/*! + * + * \file + * + * \brief RSASSA-PSS signature generation scheme usage example. + * + * This example demonstrates message signature generation according to + * RSASSA-PSS scheme with 3072-bit RSA modulus and SHA-384 hash function. + * It uses Reduced Memory Footprint (_rmf) version of the function. + * + * The RSASSA-PSS scheme is implemented according to the PKCS#1 v2.1: RSA Cryptography Standard (June 2002), + * available at: + * + * https://tools.ietf.org/html/rfc3447. + * + */ + +#include + +#include "ippcp.h" +#include "examples_common.h" +#include "bignum.h" + +/*! 3072-bit RSA Modulus N = P*Q */ +BigNumber N("0xA7A1882A7FB896786034D07FB1B9F6327C27BDD7CE6FE39C285AE3B6C34259ADC0DC4F7B9C7DEC3CA4A20D3407339EEDD\ +7A12A421DA18F5954673CAC2FF059156ECC73C6861EC761E6A0F2A5A033A6768C6A42D8B459E1B4932349E84EFD92DF59B45935F3D0E3081\ +7C66201AA99D07AE36C5D74F408D69CC08F044151FF4960E531360CB19077833ADF7BCE77ECFAA133C0CCC63C93B856814569E0B9884EE55\ +4061B9A20AB46C38263C094DAE791AA61A17F8D16F0E85B7E5CE3B067ECE89E20BC4E8F1AE814B276D234E04F4E766F501DA74EA7E3817C2\ +4EA35D016676CECE652B823B051625573CA92757FC720D254ECF1DCBBFD21D98307561ECAAB545480C7C52AD7E9FA6B597F5FE550559C2FE\ +923205AC1761A99737CA02D7B19822E008A8969349C87FB874C81620E38F613C8521F0381FE5BA55B74827DAD3E1CF2AA29C6933629F2B28\ +6AD11BE88FA6436E7E3F64A75E3595290DC0D1CD5EEE7AAAC54959CC53BD5A934A365E72DD81A2BD4FB9A67821BFFEDF2EF2BD94913DE8B"); + +/*! Public exponent */ +BigNumber E("0x1415a7"); + +/*! Private exponent */ +BigNumber D("0x073A5FC4CD642F6113DFFC4F84035CEE3A2B8ACC549703751A1D6A5EAA13487229A58EF7D7A522BB9F4F25510F1AA0F74\ +C6A8FC8A5C5BE8B91A674EDE50E92F7E34A90A3C9DA999FFFB1D695E4588F451256C163484C151350CB9C7825A7D910845EE5CF826FECF9A\ +7C0FBBBBA22BB4A531C131D2E7761BA898F002EBEF8AB87218511F81D3266E1EC07A7CA8622514C6DFDC86C67679A2C8F5F031DE9A0C22B5\ +A88060B46EE0C64D3B9AF3C0A379BCD9C6A1B51CF6480456D3FD6DEF94CD2A6C171DD3F010E3C9D662BC857208248C94EBCB9FD997B9FF4A\ +7E5FD95558569906525E741D78344F6F6CFDBD59D4FAA52EE3FA964FB7CCCB2D6BE1935D211FE1498217716273939A946081FD8509913FD4\ +7747C5C2F03EFD4D6FC9C6FCFD8402E9F40A0A5B3DE3CA2B3C0FAC9456938FAA6CF2C20E3912E5981C9876D8CA1FF29B87A15EEAE0CCCE3F\ +8A8F1E405091C083B98BCC5FE0D0DEAAE33C67C0394437F0ECCB385B7EFB17AEEBBA8AFAECCA30A2F63EAC8F0AC8F1EACAD85BBCAF3960B"); + +/*! Reference value of signature */ +static const Ipp8u signatureRef[] = +"\x96\x87\x11\x5b\xe4\x78\xe4\xb6\x42\xcd\x36\x93\x92\xb9\xdd\x0f\x35\x76\xe7\x04\xaf\x72\x18\xb1\xf9\x4d\x7f\x8f\ +\xe7\xf0\x70\x73\xe3\xe8\xe1\x18\x6f\xa7\x68\x97\x7d\x6b\x51\x4e\x51\x34\x59\xf2\x37\x3d\xf6\xec\x52\xe3\xde\x9b\ +\xd8\x3f\xcc\x5c\xc3\xe6\xb9\x7f\x8b\x3f\xb5\x34\x16\x3c\x64\xf5\x26\x76\x20\x70\x0e\x9d\x8c\x52\xb3\xdf\x61\xa7\ +\xc3\x74\x8e\xf1\x59\xd6\xb3\x90\x89\x5a\xfa\x3a\xf5\x91\x09\xa5\x47\x8d\x01\x6d\x96\xc4\x9f\x68\xdf\xc7\x35\xba\ +\x2a\xaf\xd5\x01\x2c\x13\x51\x5e\xd6\x64\x4f\x0d\x41\x09\xc4\x55\x56\xe1\x4a\x38\x21\xe1\xaa\x24\xbe\xb8\xa8\x1a\ +\x48\xda\x27\xf1\x31\xde\x84\xf7\xba\x51\x58\x1d\x81\xb8\xff\x31\xba\x92\xb8\xa1\xfd\xe8\x67\xf0\x7e\x32\xe6\xc2\ +\x70\x92\x53\x44\x81\x74\xdd\x31\x32\x4d\xbc\x32\xb0\x5f\x07\x58\x7f\x76\xa9\x99\x7d\xec\xb8\x0f\x38\xd8\xc1\x3d\ +\x0f\x6e\xb3\xc1\x0e\x3d\x96\xa2\x29\x3f\x74\x64\xf1\xe0\x46\x02\xef\x6e\x84\xc2\xd0\x24\x5d\x7d\xb2\x56\xa6\x7d\ +\x13\x2a\x47\xca\xe9\xab\xe0\x6b\x61\xa8\x96\x8f\x50\xa1\x74\x99\x95\xdc\x15\xef\x0d\xcb\x1d\x5f\x59\x59\xe4\xd4\ +\x54\xc8\x54\x7b\xbb\x4d\x19\x56\x98\xf4\x84\x61\x7b\xfd\x12\x2a\xca\xae\x2d\x0e\x8c\x76\xd2\x8b\x24\x00\x5a\xb0\ +\x3c\xaa\x78\x1e\xa9\x7b\x1c\x4d\x93\x96\xa1\x6f\x79\x98\xee\xe7\xdd\xd9\xde\x4c\xab\xe5\x70\x32\xd9\x43\x8a\x5d\ +\x99\xc6\xb3\x4a\x95\x61\x22\x35\x02\x63\xc7\xe9\x98\xbc\x61\xde\xc9\x13\x81\x01\x2e\x68\x6d\x07\x9e\x39\xe9\x6b\ +\x1e\xa4\xbf\xdb\x7c\xdf\x63\x0d\xdb\x42\x2c\x6b\x58\x0e\x55\x06\xc9\xcc\x3d\x6c\x10\x0f\x20\x41\xd1\x7c\xea\xaa\ +\xa5\x45\x89\x24\x9f\x04\xa1\x37\x0f\xfa\x3b\xf3\xff\x1a\xde\xb8\x90\x68\x86\x98"; + +/*! Message to be signed */ +static const Ipp8u sourceMessage[] = +"\x92\x21\xf0\xfe\x91\x15\x84\x35\x54\xd5\x68\x5d\x9f\xe6\x9d\xc4\x9e\x95\xce\xb5\x79\x39\x86\xe4\x28\xb8\xa1\x0b\ +\x89\x4c\x01\xd6\xaf\x87\x82\xfd\x7d\x95\x2f\xaf\x74\xc2\xb6\x37\xca\x3b\x19\xda\xbc\x19\xa7\xfe\x25\x9b\x2b\x92\ +\x4e\xb3\x63\xa9\x08\xc5\xb3\x68\xf8\xab\x1b\x23\x33\xfc\x67\xc3\x0b\x8e\xa5\x6b\x28\x39\xdc\x5b\xda\xde\xfb\x14\ +\xad\xa8\x10\xbc\x3e\x92\xba\xc5\x4e\x2a\xe1\xca\x15\x94\xa4\xb9\xd8\xd1\x93\x37\xbe\x42\x1f\x40\xe0\x67\x4e\x0e\ +\x9f\xed\xb4\x3d\x3a\xe8\x9e\x2c\xa0\x5d\x90\xa6\x82\x03\xf2\xc2"; + +/*! Salt */ +static const Ipp8u salt[] = +"\x61\xa7\x62\xf8\x96\x8d\x5f\x36\x7e\x2d\xbc\xac\xb4\x02\x16\x53\xdc\x75\x43\x7d\x90\x00\xe3\x16\x9d\x94\x37\x29\ +\x70\x38\x37\xa5\xcb\xf4\xde\x62\xbd\xed\xc9\x5f\xd0\xd1\x00\x4e\x84\x75\x14\x52"; + + +/*! Main function */ +int main(void) +{ + /* RSA Modulus N = P*Q in bits */ + const int RSA_MODULUS = 3072; + + /* Internal function status */ + IppStatus status = ippStsNoErr; + + /* Size in bits of RSA modulus */ + const int bitSizeN = N.BitSize(); + /* Size in bits of RSA private exponent */ + const int bitSizeD = D.BitSize(); + + /* Allocate memory for signature. + * Size shall be equal to the RSA modulus size. + */ + const int signatureLen = bitSizeInBytes(RSA_MODULUS); + Ipp8u* pSignature = new Ipp8u[signatureLen]; + + /* Allocate memory private key. + * There are two types of private keys that are supported: Type1 and Type2. + * You can choose any of them, depending on your private key representation. + * This example uses Type1 key. + * For more information, see https://www.intel.com/content/www/us/en/docs/ipp-crypto/developer-reference/current/crypto-ref-getsize-public-private1-private2.html + */ + int keySize = 0; + ippsRSA_GetSizePrivateKeyType1(bitSizeN, bitSizeD, &keySize); + IppsRSAPrivateKeyState* pPrvKeyType1 = (IppsRSAPrivateKeyState*)(new Ipp8u[keySize]); + ippsRSA_InitPrivateKeyType1(bitSizeN, bitSizeD, pPrvKeyType1, keySize); + + if (pPrvKeyType1) { + do { + /* Set private key */ + status = ippsRSA_SetPrivateKeyType1(N, D, pPrvKeyType1); + if (!checkStatus("ippsRSA_SetPrivateKeyType1", ippStsNoErr, status)) + break; + + /* Calculate temporary buffer size */ + int bufSize = 0; + status = ippsRSA_GetBufferSizePrivateKey(&bufSize, pPrvKeyType1); + if (!checkStatus("ippsRSA_GetBufferSizePrivateKey", ippStsNoErr, status)) + break; + + Ipp8u* pScratchBuffer = new Ipp8u[bufSize]; + + /* Sign message with use of SHA384 hash function */ + status = ippsRSASign_PSS_rmf(sourceMessage, sizeof(sourceMessage)-1, + salt, sizeof(salt)-1, + pSignature, + pPrvKeyType1, NULL /* public key */, + ippsHashMethod_SHA384(), + pScratchBuffer); + + if (pScratchBuffer) delete [] pScratchBuffer; + + if (!checkStatus("ippsRSASign_PSS_rmf", ippStsNoErr, status)) + break; + + /* Compare signature with expected value */ + if (0 != memcmp(signatureRef, pSignature, signatureLen)) { + printf("ERROR: Signature and reference value do not match\n"); + status = ippStsErr; + } + } while (0); + } + + if (pPrvKeyType1) delete [] (Ipp8u*)pPrvKeyType1; + if (pSignature) delete [] pSignature; + + PRINT_EXAMPLE_STATUS("ippsRSASign_PSS_rmf", "RSA-PSS 3072 (SHA-384) Type1 Signature", ippStsNoErr == status); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/sms4/sms4-128-cbc-decryption.cpp b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/sms4/sms4-128-cbc-decryption.cpp new file mode 100644 index 0000000..c72e617 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/sms4/sms4-128-cbc-decryption.cpp @@ -0,0 +1,126 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/*! + * + * \file + * + * \brief SMS4 Cipher Block Chaining mode of operation (CBC) example + * + * This example demonstrates usage of SMS4 block cipher + * run with CBC mode of operation. Decryption scheme. + * + * The CBC mode of operation is implemented according to the + * "NIST Special Publication 800-38A: Recommendation for Block Cipher Modes of + * Operation" document: + * + * https://csrc.nist.gov/publications/detail/sp/800-38a/final + * + */ + +#include + +#include "ippcp.h" +#include "examples_common.h" + +/*! SMS4 block size in bytes */ +static const int SMS4_BLOCK_SIZE = 16; + +/*! Key size in bytes */ +static const int KEY_SIZE = 16; + +/*! Message size in bytes */ +static const int SRC_LEN = 16; + +/*! Plain text */ +static Ipp8u plainText[SRC_LEN] = { + 0xAA,0xAA,0xAA,0xAA,0xBB,0xBB,0xBB,0xBB, + 0xCC,0xCC,0xCC,0xCC,0xDD,0xDD,0xDD,0xDD +}; + +/*! Cipher text */ +static Ipp8u cipherText[SRC_LEN] = { + 0x78,0xEB,0xB1,0x1C,0xC4,0x0B,0x0A,0x48, + 0x31,0x2A,0xAE,0xB2,0x04,0x02,0x44,0xCB +}; + +/*! 128-bit secret key */ +static Ipp8u key[KEY_SIZE] = { + 0x01,0x23,0x45,0x67,0x89,0xAB,0xCD,0xEF, + 0xFE,0xDC,0xBA,0x98,0x76,0x54,0x32,0x10 +}; + +/*! Initialization vector for CBC mode. + * Size of initialization vector for SMS4-CBC shall be equal to the size of SMS4 block (16 bytes). + */ +static Ipp8u iv[SMS4_BLOCK_SIZE] = { + 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07, + 0x08,0x09,0x0A,0x0B,0x0C,0x0D,0x0E,0x0F +}; + +/*! Main function */ +int main(void) +{ + /* Size of SMS4 context structure. It will be set up in ippsSMS4GetSize(). */ + int ctxSize = 0; + + Ipp8u pOut[SRC_LEN] = {}; + + /* Internal function status */ + IppStatus status = ippStsNoErr; + + /* Pointer to SMS4 context structure */ + IppsSMS4Spec* pSMS4 = 0; + + do { + /* 1. Get size needed for SMS4 context structure */ + status = ippsSMS4GetSize(&ctxSize); + if (!checkStatus("ippsSMS4GetSize", ippStsNoErr, status)) + return status; + + /* 2. Allocate memory for SMS4 context structure */ + pSMS4 = (IppsSMS4Spec*)(new Ipp8u[ctxSize]); + if (NULL == pSMS4) { + printf("ERROR: Cannot allocate memory (%d bytes) for SMS4 context\n", ctxSize); + return -1; + } + + /* 3. Initialize SMS4 context */ + status = ippsSMS4Init(key, sizeof(key), pSMS4, ctxSize); + if (!checkStatus("ippsSMS4Init", ippStsNoErr, status)) + break; + + /* 4. Decryption */ + status = ippsSMS4DecryptCBC(cipherText, pOut, sizeof(cipherText), pSMS4, iv); + if (!checkStatus("ippsSMS4DecryptCBC", ippStsNoErr, status)) + break; + + /* Compare decrypted message and reference text */ + if (0 != memcmp(pOut, plainText, sizeof(plainText))) { + printf("ERROR: Decrypted and reference messages do not match\n"); + break; + } + } while (0); + + /* 5. Remove secret and release resources */ + ippsSMS4Init(0, KEY_SIZE, pSMS4, ctxSize); + if (pSMS4) delete [] (Ipp8u*)pSMS4; + + PRINT_EXAMPLE_STATUS("ippsSMS4DecryptCBC", "SMS4-CBC Decryption", !status) + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/sms4/sms4-128-cbc-encryption.cpp b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/sms4/sms4-128-cbc-encryption.cpp new file mode 100644 index 0000000..aa6c431 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/sms4/sms4-128-cbc-encryption.cpp @@ -0,0 +1,126 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/*! + * + * \file + * + * \brief SMS4 Cipher Block Chaining mode of operation (CBC) example + * + * This example demonstrates usage of SMS4 block cipher + * run with CBC mode of operation. Encryption scheme. + * + * The CBC mode of operation is implemented according to the + * "NIST Special Publication 800-38A: Recommendation for Block Cipher Modes of + * Operation" document: + * + * https://csrc.nist.gov/publications/detail/sp/800-38a/final + * + */ + +#include + +#include "ippcp.h" +#include "examples_common.h" + +/*! SMS4 block size in bytes */ +static const int SMS4_BLOCK_SIZE = 16; + +/*! Key size in bytes */ +static const int KEY_SIZE = 16; + +/*! Message size in bytes */ +static const int SRC_LEN = 16; + +/*! Plain text */ +static Ipp8u plainText[SRC_LEN] = { + 0xAA,0xAA,0xAA,0xAA,0xBB,0xBB,0xBB,0xBB, + 0xCC,0xCC,0xCC,0xCC,0xDD,0xDD,0xDD,0xDD +}; + +/*! Cipher text */ +static Ipp8u cipherText[SRC_LEN] = { + 0x78,0xEB,0xB1,0x1C,0xC4,0x0B,0x0A,0x48, + 0x31,0x2A,0xAE,0xB2,0x04,0x02,0x44,0xCB +}; + +/*! 128-bit secret key */ +static Ipp8u key[KEY_SIZE] = { + 0x01,0x23,0x45,0x67,0x89,0xAB,0xCD,0xEF, + 0xFE,0xDC,0xBA,0x98,0x76,0x54,0x32,0x10 +}; + +/*! Initialization vector for CBC mode. + * Size of initialization vector for SMS4-CBC shall be equal to the size of SMS4 block (16 bytes). + */ +static Ipp8u iv[SMS4_BLOCK_SIZE] = { + 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07, + 0x08,0x09,0x0A,0x0B,0x0C,0x0D,0x0E,0x0F +}; + +/*! Main function */ +int main(void) +{ + /* Size of SMS4 context structure. It will be set up in ippsSMS4GetSize(). */ + int ctxSize = 0; + + Ipp8u pOut[SRC_LEN] = {}; + + /* Internal function status */ + IppStatus status = ippStsNoErr; + + /* Pointer to SMS4 context structure */ + IppsSMS4Spec* pSMS4 = 0; + + do { + /* 1. Get size needed for SMS4 context structure */ + status = ippsSMS4GetSize(&ctxSize); + if (!checkStatus("ippsSMS4GetSize", ippStsNoErr, status)) + return status; + + /* 2. Allocate memory for SMS4 context structure */ + pSMS4 = (IppsSMS4Spec*)(new Ipp8u[ctxSize]); + if (NULL == pSMS4) { + printf("ERROR: Cannot allocate memory (%d bytes) for SMS4 context\n", ctxSize); + return -1; + } + + /* 3. Initialize SMS4 context */ + status = ippsSMS4Init(key, sizeof(key), pSMS4, ctxSize); + if (!checkStatus("ippsSMS4Init", ippStsNoErr, status)) + break; + + /* 4. Encryption */ + status = ippsSMS4EncryptCBC(plainText, pOut, sizeof(plainText), pSMS4, iv); + if (!checkStatus("ippsSMS4EncryptCBC", ippStsNoErr, status)) + break; + + /* Compare encrypted message and reference text */ + if (0 != memcmp(pOut, cipherText, sizeof(cipherText))) { + printf("ERROR: Encrypted and reference messages do not match\n"); + break; + } + } while (0); + + /* 5. Remove secret and release resources */ + ippsSMS4Init(0, KEY_SIZE, pSMS4, ctxSize); + if (pSMS4) delete [] (Ipp8u*)pSMS4; + + PRINT_EXAMPLE_STATUS("ippsSMS4EncryptCBC", "SMS4-CBC Encryption", !status) + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/utils/bignum.cpp b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/utils/bignum.cpp new file mode 100644 index 0000000..f9bac9b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/utils/bignum.cpp @@ -0,0 +1,438 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "bignum.h" +#include +#include +#include "utils.h" + +////////////////////////////////////////////////////////////////////// +// +// BigNumber +// +////////////////////////////////////////////////////////////////////// +BigNumber::~BigNumber() +{ + delete [] (Ipp8u*)m_pBN; +} + +bool BigNumber::create(const Ipp32u* pData, int length, IppsBigNumSGN sgn) +{ + int size; + ippsBigNumGetSize(length, &size); + m_pBN = (IppsBigNumState*)( new Ipp8u[size] ); + if(!m_pBN) + return false; + ippsBigNumInit(length, m_pBN); + if (pData) + ippsSet_BN(sgn, length, pData, m_pBN); + return true; +} + +// +// constructors +// +BigNumber::BigNumber(Ipp32u value) +{ + create(&value, 1, IppsBigNumPOS); +} + +BigNumber::BigNumber(Ipp32s value) +{ + Ipp32s avalue = abs(value); + create((Ipp32u*)&avalue, 1, (value<0)? IppsBigNumNEG : IppsBigNumPOS); +} + +BigNumber::BigNumber(const IppsBigNumState* pBN) +{ + IppsBigNumSGN bnSgn; + int bnBitLen; + Ipp32u* bnData; + ippsRef_BN(&bnSgn, &bnBitLen, &bnData, pBN); + + create(bnData, BITSIZE_WORD(bnBitLen), bnSgn); +} + +BigNumber::BigNumber(const Ipp32u* pData, int length, IppsBigNumSGN sgn) +{ + create(pData, length, sgn); +} + +static char HexDigitList[] = "0123456789ABCDEF"; + +BigNumber::BigNumber(const char* s) +{ + bool neg = '-' == s[0]; + if(neg) s++; + bool hex = ('0'==s[0]) && (('x'==s[1]) || ('X'==s[1])); + + int dataLen; + Ipp32u base; + if(hex) { + s += 2; + base = 0x10; + dataLen = (int)(strlen_safe(s) + 7)/8; + } + else { + base = 10; + dataLen = (int)(strlen_safe(s) + 9)/10; + } + + create(0, dataLen); + *(this) = Zero(); + while(*s) { + char tmp[2] = {s[0],0}; + Ipp32u digit = (Ipp32u)strcspn(HexDigitList, tmp); + *this = (*this) * base + BigNumber( digit ); + s++; + } + + if(neg) + (*this) = Zero()- (*this); +} + +BigNumber::BigNumber(const BigNumber& bn) +{ + IppsBigNumSGN bnSgn; + int bnBitLen; + Ipp32u* bnData; + ippsRef_BN(&bnSgn, &bnBitLen, &bnData, bn); + + create(bnData, BITSIZE_WORD(bnBitLen), bnSgn); +} + +// +// set value +// +void BigNumber::Set(const Ipp32u* pData, int length, IppsBigNumSGN sgn) +{ + ippsSet_BN(sgn, length, pData, BN(*this)); +} + +// +// constants +// +const BigNumber& BigNumber::Zero() +{ + static const BigNumber zero(0); + return zero; +} + +const BigNumber& BigNumber::One() +{ + static const BigNumber one(1); + return one; +} + +const BigNumber& BigNumber::Two() +{ + static const BigNumber two(2); + return two; +} + +// +// arithmetic operators +// +BigNumber& BigNumber::operator =(const BigNumber& bn) +{ + if(this != &bn) { // prevent self copy + IppsBigNumSGN bnSgn; + int bnBitLen; + Ipp32u* bnData; + ippsRef_BN(&bnSgn, &bnBitLen, &bnData, bn); + + delete [] (Ipp8u*)m_pBN; + create(bnData, BITSIZE_WORD(bnBitLen), bnSgn); + } + return *this; +} + +BigNumber& BigNumber::operator += (const BigNumber& bn) +{ + int aBitLen; + ippsRef_BN(NULL, &aBitLen, NULL, *this); + int bBitLen; + ippsRef_BN(NULL, &bBitLen, NULL, bn); + int rBitLen = IPP_MAX(aBitLen, bBitLen) + 1; + + BigNumber result(0, BITSIZE_WORD(rBitLen)); + ippsAdd_BN(*this, bn, result); + *this = result; + return *this; +} + +BigNumber& BigNumber::operator -= (const BigNumber& bn) +{ + int aBitLen; + ippsRef_BN(NULL, &aBitLen, NULL, *this); + int bBitLen; + ippsRef_BN(NULL, &bBitLen, NULL, bn); + int rBitLen = IPP_MAX(aBitLen, bBitLen); + + BigNumber result(0, BITSIZE_WORD(rBitLen)); + ippsSub_BN(*this, bn, result); + *this = result; + return *this; +} + +BigNumber& BigNumber::operator *= (const BigNumber& bn) +{ + int aBitLen; + ippsRef_BN(NULL, &aBitLen, NULL, *this); + int bBitLen; + ippsRef_BN(NULL, &bBitLen, NULL, bn); + int rBitLen = aBitLen + bBitLen; + + BigNumber result(0, BITSIZE_WORD(rBitLen)); + ippsMul_BN(*this, bn, result); + *this = result; + return *this; +} + +BigNumber& BigNumber::operator *= (Ipp32u n) +{ + int aBitLen; + ippsRef_BN(NULL, &aBitLen, NULL, *this); + + BigNumber result(0, BITSIZE_WORD(aBitLen+32)); + BigNumber bn(n); + ippsMul_BN(*this, bn, result); + *this = result; + return *this; +} + +BigNumber& BigNumber::operator %= (const BigNumber& bn) +{ + BigNumber remainder(bn); + ippsMod_BN(BN(*this), BN(bn), BN(remainder)); + *this = remainder; + return *this; +} + +BigNumber& BigNumber::operator /= (const BigNumber& bn) +{ + BigNumber quotient(*this); + BigNumber remainder(bn); + ippsDiv_BN(BN(*this), BN(bn), BN(quotient), BN(remainder)); + *this = quotient; + return *this; +} + +BigNumber operator + (const BigNumber& a, const BigNumber& b ) +{ + BigNumber r(a); + return r += b; +} + +BigNumber operator - (const BigNumber& a, const BigNumber& b ) +{ + BigNumber r(a); + return r -= b; +} + +BigNumber operator * (const BigNumber& a, const BigNumber& b ) +{ + BigNumber r(a); + return r *= b; +} + +BigNumber operator * (const BigNumber& a, Ipp32u n) +{ + BigNumber r(a); + return r *= n; +} + +BigNumber operator / (const BigNumber& a, const BigNumber& b ) +{ + BigNumber q(a); + return q /= b; +} + +BigNumber operator % (const BigNumber& a, const BigNumber& b ) +{ + BigNumber r(b); + ippsMod_BN(BN(a), BN(b), BN(r)); + return r; +} + +// +// modulo arithmetic +// +BigNumber BigNumber::Modulo(const BigNumber& a) const +{ + return a % *this; +} + +BigNumber BigNumber::InverseAdd(const BigNumber& a) const +{ + BigNumber t = Modulo(a); + if(t==BigNumber::Zero()) + return t; + else + return *this - t; +} + +BigNumber BigNumber::InverseMul(const BigNumber& a) const +{ + BigNumber r(*this); + ippsModInv_BN(BN(a), BN(*this), BN(r)); + return r; +} + +BigNumber BigNumber::ModAdd(const BigNumber& a, const BigNumber& b) const +{ + BigNumber r = this->Modulo(a+b); + return r; +} + +BigNumber BigNumber::ModSub(const BigNumber& a, const BigNumber& b) const +{ + BigNumber r = this->Modulo(a + this->InverseAdd(b)); + return r; +} + +BigNumber BigNumber::ModMul(const BigNumber& a, const BigNumber& b) const +{ + BigNumber r = this->Modulo(a*b); + return r; +} + +// +// comparison +// +int BigNumber::compare(const BigNumber &bn) const +{ + Ipp32u result; + BigNumber tmp = *this - bn; + ippsCmpZero_BN(BN(tmp), &result); + return (result==IS_ZERO)? 0 : (result==GREATER_THAN_ZERO)? 1 : -1; +} + +bool operator < (const BigNumber &a, const BigNumber &b) { return a.compare(b) < 0; } +bool operator > (const BigNumber &a, const BigNumber &b) { return a.compare(b) > 0; } +bool operator == (const BigNumber &a, const BigNumber &b) { return 0 == a.compare(b);} +bool operator != (const BigNumber &a, const BigNumber &b) { return 0 != a.compare(b);} + +// easy tests +// +bool BigNumber::IsOdd() const +{ + Ipp32u* bnData; + ippsRef_BN(NULL, NULL, &bnData, *this); + return bnData[0]&1; +} + +// +// size of BigNumber +// +int BigNumber::LSB() const +{ + if( *this == BigNumber::Zero() ) + return 0; + + vector v; + num2vec(v); + + int lsb = 0; + vector::iterator i; + for(i=v.begin(); i!=v.end(); i++) { + Ipp32u x = *i; + if(0==x) + lsb += 32; + else { + while(0==(x&1)) { + lsb++; + x >>= 1; + } + break; + } + } + return lsb; +} + +int BigNumber::MSB() const +{ + if( *this == BigNumber::Zero() ) + return 0; + + vector v; + num2vec(v); + + int msb = (int)v.size()*32 -1; + vector::reverse_iterator i; + for(i=v.rbegin(); i!=v.rend(); i++) { + Ipp32u x = *i; + if(0==x) + msb -=32; + else { + while(!(x&0x80000000)) { + msb--; + x <<= 1; + } + break; + } + } + return msb; +} + +int Bit(const vector& v, int n) +{ + return 0 != ( v[n>>5] & (1<<(n&0x1F)) ); +} + +// +// conversions and output +// +void BigNumber::num2vec( vector& v ) const +{ + int bnBitLen; + Ipp32u* bnData; + ippsRef_BN(NULL, &bnBitLen, &bnData, *this); + + int len = BITSIZE_WORD(bnBitLen);; + for(int n=0; n0; n--) { + Ipp32u x = bnData[n-1]; + for(int nd=8; nd>0; nd--) { + char c = HexDigitList[(x>>(nd-1)*4)&0xF]; + s.append(1, c); + } + } +} + +ostream& operator << ( ostream &os, const BigNumber& a) +{ + string s; + a.num2hex(s); + os << s.c_str(); + return os; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/utils/bignum.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/utils/bignum.h new file mode 100644 index 0000000..8af9952 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/utils/bignum.h @@ -0,0 +1,107 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#if !defined _BIGNUMBER_H_ +#define _BIGNUMBER_H_ + +#include "ippcp.h" + +#include +#include +#include + +using namespace std; + +class BigNumber +{ +public: + BigNumber(Ipp32u value=0); + BigNumber(Ipp32s value); + BigNumber(const IppsBigNumState* pBN); + BigNumber(const Ipp32u* pData, int length=1, IppsBigNumSGN sgn=IppsBigNumPOS); + BigNumber(const BigNumber& bn); + BigNumber(const char *s); + virtual ~BigNumber(); + + // set value + void Set(const Ipp32u* pData, int length=1, IppsBigNumSGN sgn=IppsBigNumPOS); + // conversion to IppsBigNumState + friend IppsBigNumState* BN(const BigNumber& bn) {return bn.m_pBN;} + operator IppsBigNumState* () const { return m_pBN; } + + // some useful constatns + static const BigNumber& Zero(); + static const BigNumber& One(); + static const BigNumber& Two(); + + // arithmetic operators probably need + BigNumber& operator = (const BigNumber& bn); + BigNumber& operator += (const BigNumber& bn); + BigNumber& operator -= (const BigNumber& bn); + BigNumber& operator *= (Ipp32u n); + BigNumber& operator *= (const BigNumber& bn); + BigNumber& operator /= (const BigNumber& bn); + BigNumber& operator %= (const BigNumber& bn); + friend BigNumber operator + (const BigNumber& a, const BigNumber& b); + friend BigNumber operator - (const BigNumber& a, const BigNumber& b); + friend BigNumber operator * (const BigNumber& a, const BigNumber& b); + friend BigNumber operator * (const BigNumber& a, Ipp32u); + friend BigNumber operator % (const BigNumber& a, const BigNumber& b); + friend BigNumber operator / (const BigNumber& a, const BigNumber& b); + + // modulo arithmetic + BigNumber Modulo(const BigNumber& a) const; + BigNumber ModAdd(const BigNumber& a, const BigNumber& b) const; + BigNumber ModSub(const BigNumber& a, const BigNumber& b) const; + BigNumber ModMul(const BigNumber& a, const BigNumber& b) const; + BigNumber InverseAdd(const BigNumber& a) const; + BigNumber InverseMul(const BigNumber& a) const; + + // comparisons + friend bool operator < (const BigNumber& a, const BigNumber& b); + friend bool operator > (const BigNumber& a, const BigNumber& b); + friend bool operator == (const BigNumber& a, const BigNumber& b); + friend bool operator != (const BigNumber& a, const BigNumber& b); + friend bool operator <= (const BigNumber& a, const BigNumber& b) {return !(a>b);} + friend bool operator >= (const BigNumber& a, const BigNumber& b) {return !(a>5;} + friend int Bit(const vector& v, int n); + + // conversion and output + void num2hex( string& s ) const; // convert to hex string + void num2vec( vector& v ) const; // convert to 32-bit word vector + friend ostream& operator << (ostream& os, const BigNumber& a); + +protected: + bool create(const Ipp32u* pData, int length, IppsBigNumSGN sgn=IppsBigNumPOS); + int compare(const BigNumber& ) const; + IppsBigNumState* m_pBN; +}; + +// convert bit size into 32-bit words +#define BITSIZE_WORD(n) ((((n)+31)>>5)) + +#endif // _BIGNUMBER_H_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/utils/examples_common.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/utils/examples_common.h new file mode 100644 index 0000000..c9a17fa --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/utils/examples_common.h @@ -0,0 +1,90 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) */ + +/*! + * + * \file + * \brief Common header for Intel IPP Cryptography examples + * + */ + +#ifndef EXAMPLES_COMMON_H_ +#define EXAMPLES_COMMON_H_ + +#include + +/*! Macro that prints status message depending on condition */ +#define PRINT_EXAMPLE_STATUS(function_name, description, success_condition) \ + printf("+--------------------------------------------------------------|\n"); \ + printf(" Function: %s\n", function_name); \ + printf(" Description: %s\n", description); \ + if (success_condition) { \ + printf(" Status: PASSED!\n"); \ + } else { \ + printf(" Status: FAILED!\n"); \ + } \ + printf("+--------------------------------------------------------------|\n"); + +/*! + * Helper function to compare expected and actual function return statuses and display + * an error mesage if those are different. + * + * \param[in] Function name to display + * \param[in] Expected status + * \param[in] Actual status + * + * \return zero if statuses are not equal, otherwise - non-zero value + */ +inline int checkStatus(const char* funcName, IppStatus expectedStatus, IppStatus status) +{ + if (expectedStatus != status) { + printf("%s: unexpected return status\n", funcName); + printf("Expected: %s\n", ippcpGetStatusString(expectedStatus)); + printf("Received: %s\n", ippcpGetStatusString(status)); + return 0; + } + return 1; +} + +/*! + * Helper function to convert bit size into byte size. + * + * \param[in] Size in bits + * + * \return size in bytes + */ +inline int bitSizeInBytes(int nBits) +{ + return (nBits + 7) >> 3; +} + +/*! + * Helper function to convert bit size into word size. + * + * \param[in] Size in bits + * + * \return size in words + */ + +inline int bitSizeInWords(int nBits) +{ + return (nBits + 31) >> 5; +} + +#endif /* #ifndef EXAMPLES_COMMON_H_ */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/utils/requests.cpp b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/utils/requests.cpp new file mode 100644 index 0000000..3bb2414 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/utils/requests.cpp @@ -0,0 +1,39 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/*! + * + * \file + * \brief The source file contains the implementation of Request class which simulates a queue in the example of using the multi buffer RSA. + * + */ + +#include "bignum.h" +#include "requests.h" +#include "examples_common.h" + +/* Allocated memory for cipher and decipher texts is not less than RSA modulus size and source plain text */ +Request::Request(const BigNumber& pPlainText, const BigNumber& N, const BigNumber& E, const BigNumber& D) + : m_plainText(pPlainText) + , m_N(N) + , m_E(E) + , m_D(D) + , m_cipherText(N) + , m_decipherText(pPlainText) + , m_isCompatible(true) {} + +Request::~Request(){} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/utils/requests.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/utils/requests.h new file mode 100644 index 0000000..4d7f9b5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/utils/requests.h @@ -0,0 +1,98 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/*! + * + * \file + * \brief The header contains the declaration of Request class which simulates a queue in the example of using the multi buffer RSA. + * + */ + +#if !defined _REQUESTS_H_ +#define _REQUESTS_H_ + +#include "bignum.h" + +class Request { + BigNumber m_plainText; + BigNumber m_cipherText; + BigNumber m_decipherText; + BigNumber m_N; + BigNumber m_E; + BigNumber m_D; + bool m_isCompatible; + + public: + Request(const BigNumber& pPlainText, const BigNumber& N, const BigNumber& E, const BigNumber& D); + ~Request(); + + IppsBigNumState* GetPlainText() const + { + return m_plainText; + } + + IppsBigNumState* GetCipherText() const + { + return m_cipherText; + } + + IppsBigNumState* GetDecipherText() const + { + return m_decipherText; + } + + int GetBitSizeN() const + { + return m_N.BitSize(); + } + + const BigNumber& GetValueN() const + { + return m_N; + } + + int GetBitSizeE() const + { + return m_E.BitSize(); + } + + const BigNumber& GetValueE() const + { + return m_E; + } + + int GetBitSizeD() const + { + return m_D.BitSize(); + } + + const BigNumber& GetValueD() const + { + return m_D; + } + void SetCompatibilityStatus(bool status) + { + m_isCompatible = status; + } + + bool IsCompatible() const + { + return m_isCompatible; + } +}; + +#endif /* #ifndef _REQUESTS_H_ */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/utils/utils.cpp b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/utils/utils.cpp new file mode 100644 index 0000000..c3c721c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/utils/utils.cpp @@ -0,0 +1,47 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + + +#include "utils.h" + +size_t strlen_safe(const char* dest, size_t dmax) { + size_t count; + + /* check null pointer */ + if (NULL == dest) { + return 0UL; + } + + /* check max equal zero */ + if (0UL == dmax) { + return 0UL; + } + + /* check dmax > 4Kb */ + if (dmax > RSIZE_MAX_STR) { + return 0UL; + } + + count = 0UL; + while (*dest && dmax) { + ++count; + --dmax; + ++dest; + } + + return count; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/utils/utils.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/utils/utils.h new file mode 100644 index 0000000..5eebbc9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/examples/utils/utils.h @@ -0,0 +1,35 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ +#ifndef _UTILS_H_ +#define _UTILS_H_ + +#include + +#define RSIZE_MAX_STR (4UL << 10) /* 4Kb */ + +/** + * \brief + * The strnlen_s function computes the length of the string pointed to by dest. + * \param[in] dest pointer to string + * \param[in] dmax restricted maximum length. (default 4Kb) + * \return size_t + * The function returns the string length, excluding the terminating + * null character. If dest is NULL, then strnlen_s returns 0. + */ +size_t strlen_safe(const char* dest, size_t dmax = RSIZE_MAX_STR); + +#endif // _UTILS_H_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/include/ippcp.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/include/ippcp.h new file mode 100644 index 0000000..9d7e467 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/include/ippcp.h @@ -0,0 +1,1541 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) +// +*/ + +#if !defined( IPPCP_H__ ) || defined( _OWN_BLDPCS ) +#define IPPCP_H__ + + +#ifndef IPPCPDEFS_H__ + #include "ippcpdefs.h" +#endif + + +#ifdef __cplusplus +extern "C" { +#endif + +#if !defined( IPP_NO_DEFAULT_LIB ) + #if defined( _IPP_SEQUENTIAL_DYNAMIC ) + #pragma comment( lib, __FILE__ "/../../lib/" INTEL_PLATFORM "ippcp" ) + #elif defined( _IPP_SEQUENTIAL_STATIC ) + #pragma comment( lib, __FILE__ "/../../lib/" INTEL_PLATFORM "ippcpmt" ) + #endif +#endif + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && !defined(__INTEL_LLVM_COMPILER) +#pragma warning(push) +#pragma warning(disable : 4100) // for MSVC, unreferenced parameter +#endif + +/* ///////////////////////////////////////////////////////////////////////////// +// Name: ippcpGetLibVersion +// Purpose: getting of the library version +// Returns: the structure of information about version of ippCP library +// Parameters: +// +// Notes: not necessary to release the returned structure +*/ +IPPAPI( const IppLibraryVersion*, ippcpGetLibVersion, (void) ) + + +/* +// ========================================================= +// Symmetric Ciphers +// ========================================================= +*/ + +/* TDES */ + +#define TDES_DEPRECATED "This algorithm is considered weak due to known attacks on it. \ +The functionality remains in the library, but the implementation will no be longer \ +optimized and no security patches will be applied. A more secure alternative is available: AES" + +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsDESGetSize,(int *size)) +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsDESInit,(const Ipp8u* pKey, IppsDESSpec* pCtx)) + +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsDESPack,(const IppsDESSpec* pCtx, Ipp8u* pBuffer)) +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsDESUnpack,(const Ipp8u* pBuffer, IppsDESSpec* pCtx)) + +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsTDESEncryptECB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsDESSpec* pCtx1, const IppsDESSpec* pCtx2, const IppsDESSpec* pCtx3, + IppsCPPadding padding)) +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsTDESDecryptECB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsDESSpec* pCtx1, const IppsDESSpec* pCtx2, const IppsDESSpec* pCtx3, + IppsCPPadding padding)) + +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsTDESEncryptCBC,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsDESSpec* pCtx1, const IppsDESSpec* pCtx2, const IppsDESSpec* pCtx3, + const Ipp8u* pIV, + IppsCPPadding padding)) +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsTDESDecryptCBC,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsDESSpec* pCtx1, const IppsDESSpec* pCtx2, const IppsDESSpec* pCtx3, + const Ipp8u* pIV, + IppsCPPadding padding)) + +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsTDESEncryptCFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int cfbBlkSize, + const IppsDESSpec* pCtx1, const IppsDESSpec* pCtx2, const IppsDESSpec* pCtx3, + const Ipp8u* pIV, + IppsCPPadding padding)) +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsTDESDecryptCFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int cfbBlkSize, + const IppsDESSpec* pCtx1, const IppsDESSpec* pCtx2, const IppsDESSpec* pCtx3, + const Ipp8u* pIV, + IppsCPPadding padding)) + +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsTDESEncryptOFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int ofbBlkSize, + const IppsDESSpec* pCtx1, + const IppsDESSpec* pCtx2, + const IppsDESSpec* pCtx3, + Ipp8u* pIV)) +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsTDESDecryptOFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int ofbBlkSize, + const IppsDESSpec* pCtx1, + const IppsDESSpec* pCtx2, + const IppsDESSpec* pCtx3, + Ipp8u* pIV)) + +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsTDESEncryptCTR,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsDESSpec* pCtx1, + const IppsDESSpec* pCtx2, + const IppsDESSpec* pCtx3, + Ipp8u* pCtrValue, int ctrNumBitSize)) +IPP_DEPRECATED(TDES_DEPRECATED) \ +IPPAPI(IppStatus, ippsTDESDecryptCTR,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsDESSpec* pCtx1, + const IppsDESSpec* pCtx2, + const IppsDESSpec* pCtx3, + Ipp8u* pCtrValue, int ctrNumBitSize)) + +/* AES */ +IPPAPI(IppStatus, ippsAESGetSize,(int *pSize)) +IPPAPI(IppStatus, ippsAESInit,(const Ipp8u* pKey, int keyLen, IppsAESSpec* pCtx, int ctxSize)) +IPPAPI(IppStatus, ippsAESSetKey,(const Ipp8u* pKey, int keyLen, IppsAESSpec* pCtx)) + +IPPAPI(IppStatus, ippsAESPack,(const IppsAESSpec* pCtx, Ipp8u* pBuffer, int bufSize)) +IPPAPI(IppStatus, ippsAESUnpack,(const Ipp8u* pBuffer, IppsAESSpec* pCtx, int ctxSize)) + +#define ECB_DEPRECATED "ECB functionality remains in the library, but it is not safe when used as is. \ +It is recommended to use any other mode, for example CBC." + +IPP_DEPRECATED(ECB_DEPRECATED) \ +IPPAPI(IppStatus, ippsAESEncryptECB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx)) +IPP_DEPRECATED(ECB_DEPRECATED) \ +IPPAPI(IppStatus, ippsAESDecryptECB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx)) + +IPPAPI(IppStatus, ippsAESEncryptCBC,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsAESEncryptCBC_CS1,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsAESEncryptCBC_CS2,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsAESEncryptCBC_CS3,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsAESDecryptCBC,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsAESDecryptCBC_CS1,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsAESDecryptCBC_CS2,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsAESDecryptCBC_CS3,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + const Ipp8u* pIV)) + +IPPAPI(IppStatus, ippsAESEncryptCFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int cfbBlkSize, + const IppsAESSpec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsAESDecryptCFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int cfbBlkSize, + const IppsAESSpec* pCtx, + const Ipp8u* pIV)) + +IPPAPI(IppStatus, ippsAESEncryptOFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int ofbBlkSize, + const IppsAESSpec* pCtx, + Ipp8u* pIV)) +IPPAPI(IppStatus, ippsAESDecryptOFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int ofbBlkSize, + const IppsAESSpec* pCtx, + Ipp8u* pIV)) + +IPPAPI(IppStatus, ippsAESEncryptCTR,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + Ipp8u* pCtrValue, int ctrNumBitSize)) +IPPAPI(IppStatus, ippsAESDecryptCTR,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + Ipp8u* pCtrValue, int ctrNumBitSize)) + +IPPAPI(IppStatus, ippsAESEncryptXTS_Direct,(const Ipp8u* pSrc, Ipp8u* pDst, int encBitsize, int aesBlkNo, + const Ipp8u* pTweakPT, + const Ipp8u* pKey, int keyBitsize, + int dataUnitBitsize)) +IPPAPI(IppStatus, ippsAESDecryptXTS_Direct,(const Ipp8u* pSrc, Ipp8u* pDst, int encBitsize, int aesBlkNo, + const Ipp8u* pTweakPT, + const Ipp8u* pKey, int keyBitsize, + int dataUnitBitsize)) + +IPPAPI(IppStatus, ippsAESSetupNoise,(Ipp32u noiseLevel, IppsAESSpec* pCtx)) +IPPAPI(IppStatus, ippsAES_GCMSetupNoise,(Ipp32u noiseLevel, IppsAES_GCMState* pState)) +IPPAPI(IppStatus, ippsAES_CMACSetupNoise,(Ipp32u noiseLevel, IppsAES_CMACState* pState)) + +/* AES multi-buffer functions */ +IPPAPI(IppStatus, ippsAES_EncryptCFB16_MB, (const Ipp8u* pSrc[], Ipp8u* pDst[], int len[], + const IppsAESSpec* pCtx[], + const Ipp8u* pIV[], + IppStatus status[], + int numBuffers)) + +/* SMS4 */ +IPPAPI(IppStatus, ippsSMS4GetSize,(int *pSize)) +IPPAPI(IppStatus, ippsSMS4Init,(const Ipp8u* pKey, int keyLen, IppsSMS4Spec* pCtx, int ctxSize)) +IPPAPI(IppStatus, ippsSMS4SetKey,(const Ipp8u* pKey, int keyLen, IppsSMS4Spec* pCtx)) + +IPP_DEPRECATED(ECB_DEPRECATED) \ +IPPAPI(IppStatus, ippsSMS4EncryptECB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx)) +IPP_DEPRECATED(ECB_DEPRECATED) \ +IPPAPI(IppStatus, ippsSMS4DecryptECB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx)) + +IPPAPI(IppStatus, ippsSMS4EncryptCBC,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsSMS4EncryptCBC_CS1,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsSMS4EncryptCBC_CS2,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsSMS4EncryptCBC_CS3,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsSMS4DecryptCBC,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsSMS4DecryptCBC_CS1,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsSMS4DecryptCBC_CS2,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsSMS4DecryptCBC_CS3,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx, + const Ipp8u* pIV)) + +IPPAPI(IppStatus, ippsSMS4EncryptCFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int cfbBlkSize, + const IppsSMS4Spec* pCtx, + const Ipp8u* pIV)) +IPPAPI(IppStatus, ippsSMS4DecryptCFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int cfbBlkSize, + const IppsSMS4Spec* pCtx, + const Ipp8u* pIV)) + +IPPAPI(IppStatus, ippsSMS4EncryptOFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int ofbBlkSize, + const IppsSMS4Spec* pCtx, + Ipp8u* pIV)) +IPPAPI(IppStatus, ippsSMS4DecryptOFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int ofbBlkSize, + const IppsSMS4Spec* pCtx, + Ipp8u* pIV)) + +IPPAPI(IppStatus, ippsSMS4EncryptCTR,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx, + Ipp8u* pCtrValue, int ctrNumBitSize)) +IPPAPI(IppStatus, ippsSMS4DecryptCTR,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx, + Ipp8u* pCtrValue, int ctrNumBitSize)) + +/* SMS4-CCM */ +IPPAPI(IppStatus, ippsSMS4_CCMGetSize,(int* pSize)) +IPPAPI(IppStatus, ippsSMS4_CCMInit,(const Ipp8u* pKey, int keyLen, IppsSMS4_CCMState* pCtx, int ctxSize)) + +IPPAPI(IppStatus, ippsSMS4_CCMMessageLen,(Ipp64u msgLen, IppsSMS4_CCMState* pCtx)) +IPPAPI(IppStatus, ippsSMS4_CCMTagLen,(int tagLen, IppsSMS4_CCMState* pCtx)) + +IPPAPI(IppStatus, ippsSMS4_CCMStart,(const Ipp8u* pIV, int ivLen, const Ipp8u* pAD, int adLen, IppsSMS4_CCMState* pCtx)) +IPPAPI(IppStatus, ippsSMS4_CCMEncrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int len, IppsSMS4_CCMState* pCtx)) +IPPAPI(IppStatus, ippsSMS4_CCMDecrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int len, IppsSMS4_CCMState* pCtx)) +IPPAPI(IppStatus, ippsSMS4_CCMGetTag,(Ipp8u* pTag, int tagLen, const IppsSMS4_CCMState* pCtx)) + +/* +// ========================================================= +// AES based authentication & confidence Primitives +// ========================================================= +*/ + +/* AES-CCM */ +IPPAPI(IppStatus, ippsAES_CCMGetSize,(int* pSize)) +IPPAPI(IppStatus, ippsAES_CCMInit,(const Ipp8u* pKey, int keyLen, IppsAES_CCMState* pState, int ctxSize)) + +IPPAPI(IppStatus, ippsAES_CCMMessageLen,(Ipp64u msgLen, IppsAES_CCMState* pState)) +IPPAPI(IppStatus, ippsAES_CCMTagLen,(int tagLen, IppsAES_CCMState* pState)) + +IPPAPI(IppStatus, ippsAES_CCMStart,(const Ipp8u* pIV, int ivLen, const Ipp8u* pAD, int adLen, IppsAES_CCMState* pState)) +IPPAPI(IppStatus, ippsAES_CCMEncrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int len, IppsAES_CCMState* pState)) +IPPAPI(IppStatus, ippsAES_CCMDecrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int len, IppsAES_CCMState* pState)) +IPPAPI(IppStatus, ippsAES_CCMGetTag,(Ipp8u* pTag, int tagLen, const IppsAES_CCMState* pState)) + +/* AES-GCM */ +IPPAPI(IppStatus, ippsAES_GCMGetSize,(int * pSize)) +IPPAPI(IppStatus, ippsAES_GCMInit,(const Ipp8u* pKey, int keyLen, IppsAES_GCMState* pState, int ctxSize)) +IPPAPI(IppStatus, ippsAES_GCMReinit,(IppsAES_GCMState* pState)) + +IPPAPI(IppStatus, ippsAES_GCMReset,(IppsAES_GCMState* pState)) +IPPAPI(IppStatus, ippsAES_GCMProcessIV,(const Ipp8u* pIV, int ivLen, + IppsAES_GCMState* pState)) +IPPAPI(IppStatus, ippsAES_GCMProcessAAD,(const Ipp8u* pAAD, int aadLen, + IppsAES_GCMState* pState)) +IPPAPI(IppStatus, ippsAES_GCMStart,(const Ipp8u* pIV, int ivLen, + const Ipp8u* pAAD, int aadLen, + IppsAES_GCMState* pState)) +IPPAPI(IppStatus, ippsAES_GCMEncrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int len, IppsAES_GCMState* pState)) +IPPAPI(IppStatus, ippsAES_GCMDecrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int len, IppsAES_GCMState* pState)) +IPPAPI(IppStatus, ippsAES_GCMGetTag,(Ipp8u* pDstTag, int tagLen, const IppsAES_GCMState* pState)) + +/* AES-XTS */ +IPPAPI(IppStatus, ippsAES_XTSGetSize,(int * pSize)) +IPPAPI(IppStatus, ippsAES_XTSInit,(const Ipp8u* pKey, int keyLen, + int duBitsize, + IppsAES_XTSSpec* pCtx,int ctxSize)) +IPPAPI(IppStatus, ippsAES_XTSEncrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int bitSizeLen, + const IppsAES_XTSSpec* pCtx, + const Ipp8u* pTweak, + int startCipherBlkNo)) +IPPAPI(IppStatus, ippsAES_XTSDecrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int bitSizeLen, + const IppsAES_XTSSpec* pCtx, + const Ipp8u* pTweak, + int startCipherBlkNo)) + +/* AES-SIV (RFC 5297) */ +IPPAPI(IppStatus, ippsAES_S2V_CMAC,(const Ipp8u* pKey, int keyLen, + const Ipp8u* pAD[], const int pADlen[], int numAD, + Ipp8u* pV)) +IPPAPI(IppStatus, ippsAES_SIVEncrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + Ipp8u* pSIV, + const Ipp8u* pAuthKey, const Ipp8u* pConfKey, int keyLen, + const Ipp8u* pAD[], const int pADlen[], int numAD)) +IPPAPI(IppStatus, ippsAES_SIVDecrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + int* pAuthPassed, + const Ipp8u* pAuthKey, const Ipp8u* pConfKey, int keyLen, + const Ipp8u* pAD[], const int pADlen[], int numAD, + const Ipp8u* pSIV)) + +/* AES-CMAC */ +IPPAPI(IppStatus, ippsAES_CMACGetSize,(int* pSize)) +IPPAPI(IppStatus, ippsAES_CMACInit,(const Ipp8u* pKey, int keyLen, IppsAES_CMACState* pState, int ctxSize)) + +IPPAPI(IppStatus, ippsAES_CMACUpdate,(const Ipp8u* pSrc, int len, IppsAES_CMACState* pState)) +IPPAPI(IppStatus, ippsAES_CMACFinal,(Ipp8u* pMD, int mdLen, IppsAES_CMACState* pState)) +IPPAPI(IppStatus, ippsAES_CMACGetTag,(Ipp8u* pMD, int mdLen, const IppsAES_CMACState* pState)) + + +/* +// ========================================================= +// RC4 Stream Ciphers +// ========================================================= +*/ + +#define RC4_DEPRECATED "is deprecated. This algorithm is considered weak due to known attacks on it. \ +It is obsolete and will be removed in one of the future Intel IPP Cryptography releases." + +IPP_DEPRECATED(RC4_DEPRECATED) \ +IPPAPI(IppStatus, ippsARCFourCheckKey, (const Ipp8u *pKey, int keyLen, IppBool* pIsWeak)) + +IPP_DEPRECATED(RC4_DEPRECATED) \ +IPPAPI(IppStatus, ippsARCFourGetSize, (int* pSize)) +IPP_DEPRECATED(RC4_DEPRECATED) \ +IPPAPI(IppStatus, ippsARCFourInit, (const Ipp8u *pKey, int keyLen, IppsARCFourState *pCtx)) +IPP_DEPRECATED(RC4_DEPRECATED) \ +IPPAPI(IppStatus, ippsARCFourReset, (IppsARCFourState* pCtx)) + +IPP_DEPRECATED(RC4_DEPRECATED) \ +IPPAPI(IppStatus, ippsARCFourPack,(const IppsARCFourState* pCtx, Ipp8u* pBuffer)) +IPP_DEPRECATED(RC4_DEPRECATED) \ +IPPAPI(IppStatus, ippsARCFourUnpack,(const Ipp8u* pBuffer, IppsARCFourState* pCtx)) + +IPP_DEPRECATED(RC4_DEPRECATED) \ +IPPAPI(IppStatus, ippsARCFourEncrypt, (const Ipp8u *pSrc, Ipp8u *pDst, int length, IppsARCFourState *pCtx)) +IPP_DEPRECATED(RC4_DEPRECATED) \ +IPPAPI(IppStatus, ippsARCFourDecrypt, (const Ipp8u *pSrc, Ipp8u *pDst, int length, IppsARCFourState *pCtx)) + + +/* +// ========================================================= +// One-Way Hash Functions +// ========================================================= +*/ + +#define OBSOLETE_API "is deprecated. This API is considered obsolete and will be removed in one of future Intel IPP Cryptography releases. \ +Use the following link for opening a ticket and providing feedback: https://supporttickets.intel.com/ if you have concerns." + +#define SHA1_DEPRECATED "This algorithm is considered weak due to known attacks on it. \ +The functionality remains in the library, but the implementation will no be longer \ +optimized and no security patches will be applied. A more secure alternative is available: SHA-2" + +/* SHA1 Hash Primitives */ +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI(IppStatus, ippsSHA1GetSize,(int* pSize)) +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI(IppStatus, ippsSHA1Init,(IppsSHA1State* pState)) +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI(IppStatus, ippsSHA1Duplicate,(const IppsSHA1State* pSrcState, IppsSHA1State* pDstState)) + +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI(IppStatus, ippsSHA1Pack,(const IppsSHA1State* pState, Ipp8u* pBuffer)) +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI(IppStatus, ippsSHA1Unpack,(const Ipp8u* pBuffer, IppsSHA1State* pState)) + +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI(IppStatus, ippsSHA1Update,(const Ipp8u* pSrc, int len, IppsSHA1State* pState)) +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI(IppStatus, ippsSHA1GetTag,(Ipp8u* pTag, Ipp32u tagLen, const IppsSHA1State* pState)) +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI(IppStatus, ippsSHA1Final,(Ipp8u* pMD, IppsSHA1State* pState)) +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI(IppStatus, ippsSHA1MessageDigest,(const Ipp8u* pMsg, int len, Ipp8u* pMD)) + +/* SHA224 Hash Primitives */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA224GetSize,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA224Init,(IppsSHA224State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA224Duplicate,(const IppsSHA224State* pSrcState, IppsSHA224State* pDstState)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA224Pack,(const IppsSHA224State* pState, Ipp8u* pBuffer)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA224Unpack,(const Ipp8u* pBuffer, IppsSHA224State* pState)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA224Update,(const Ipp8u* pSrc, int len, IppsSHA224State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA224GetTag,(Ipp8u* pTag, Ipp32u tagLen, const IppsSHA224State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA224Final,(Ipp8u* pMD, IppsSHA224State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA224MessageDigest,(const Ipp8u* pMsg, int len, Ipp8u* pMD)) + +/* SHA256 Hash Primitives */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA256GetSize,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA256Init,(IppsSHA256State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA256Duplicate,(const IppsSHA256State* pSrcState, IppsSHA256State* pDstState)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA256Pack,(const IppsSHA256State* pState, Ipp8u* pBuffer)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA256Unpack,(const Ipp8u* pBuffer, IppsSHA256State* pState)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA256Update,(const Ipp8u* pSrc, int len, IppsSHA256State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA256GetTag,(Ipp8u* pTag, Ipp32u tagLen, const IppsSHA256State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA256Final,(Ipp8u* pMD, IppsSHA256State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA256MessageDigest,(const Ipp8u* pMsg, int len, Ipp8u* pMD)) + +/* SHA384 Hash Primitives */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA384GetSize,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA384Init,(IppsSHA384State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA384Duplicate,(const IppsSHA384State* pSrcState, IppsSHA384State* pDstState)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA384Pack,(const IppsSHA384State* pState, Ipp8u* pBuffer)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA384Unpack,(const Ipp8u* pBuffer, IppsSHA384State* pState)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA384Update,(const Ipp8u* pSrc, int len, IppsSHA384State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA384GetTag,(Ipp8u* pTag, Ipp32u tagLen, const IppsSHA384State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA384Final,(Ipp8u* pMD, IppsSHA384State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA384MessageDigest,(const Ipp8u* pMsg, int len, Ipp8u* pMD)) + +/* SHA512 Hash Primitives */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA512GetSize,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA512Init,(IppsSHA512State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA512Duplicate,(const IppsSHA512State* pSrcState, IppsSHA512State* pDstState)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA512Pack,(const IppsSHA512State* pState, Ipp8u* pBuffer)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA512Unpack,(const Ipp8u* pBuffer, IppsSHA512State* pState)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA512Update,(const Ipp8u* pSrc, int len, IppsSHA512State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA512GetTag,(Ipp8u* pTag, Ipp32u tagLen, const IppsSHA512State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA512Final,(Ipp8u* pMD, IppsSHA512State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSHA512MessageDigest,(const Ipp8u* pMsg, int len, Ipp8u* pMD)) + +/* MD5 Hash Primitives */ + +#define MD5_DEPRECATED "This algorithm is considered weak due to known attacks on it. \ +The functionality remains in the library, but the implementation will no be longer \ +optimized and no security patches will be applied. A more secure alternative is available: SHA-2" + +IPP_DEPRECATED(MD5_DEPRECATED) \ +IPPAPI(IppStatus, ippsMD5GetSize,(int* pSize)) +IPP_DEPRECATED(MD5_DEPRECATED) \ +IPPAPI(IppStatus, ippsMD5Init,(IppsMD5State* pState)) +IPP_DEPRECATED(MD5_DEPRECATED) \ +IPPAPI(IppStatus, ippsMD5Duplicate,(const IppsMD5State* pSrcState, IppsMD5State* pDstState)) + +IPP_DEPRECATED(MD5_DEPRECATED) \ +IPPAPI(IppStatus, ippsMD5Pack,(const IppsMD5State* pState, Ipp8u* pBuffer)) +IPP_DEPRECATED(MD5_DEPRECATED) \ +IPPAPI(IppStatus, ippsMD5Unpack,(const Ipp8u* pBuffer, IppsMD5State* pState)) + +IPP_DEPRECATED(MD5_DEPRECATED) \ +IPPAPI(IppStatus, ippsMD5Update,(const Ipp8u* pSrc, int len, IppsMD5State* pState)) +IPP_DEPRECATED(MD5_DEPRECATED) \ +IPPAPI(IppStatus, ippsMD5GetTag,(Ipp8u* pTag, Ipp32u tagLen, const IppsMD5State* pState)) +IPP_DEPRECATED(MD5_DEPRECATED) \ +IPPAPI(IppStatus, ippsMD5Final,(Ipp8u* pMD, IppsMD5State* pState)) +IPP_DEPRECATED(MD5_DEPRECATED) \ +IPPAPI(IppStatus, ippsMD5MessageDigest,(const Ipp8u* pMsg, int len, Ipp8u* pMD)) + +/* SM3 Hash Primitives */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSM3GetSize,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSM3Init,(IppsSM3State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSM3Duplicate,(const IppsSM3State* pSrcState, IppsSM3State* pDstState)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSM3Pack,(const IppsSM3State* pState, Ipp8u* pBuffer)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSM3Unpack,(const Ipp8u* pBuffer, IppsSM3State* pState)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSM3Update,(const Ipp8u* pSrc, int len, IppsSM3State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSM3GetTag,(Ipp8u* pTag, Ipp32u tagLen, const IppsSM3State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSM3Final,(Ipp8u* pMD, IppsSM3State* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsSM3MessageDigest,(const Ipp8u* pMsg, int len, Ipp8u* pMD)) + +/* generalized Hash Primitives */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHashGetSize,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHashInit,(IppsHashState* pState, IppHashAlgId hashAlg)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHashPack,(const IppsHashState* pState, Ipp8u* pBuffer, int bufSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHashUnpack,(const Ipp8u* pBuffer, IppsHashState* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHashDuplicate,(const IppsHashState* pSrcState, IppsHashState* pDstState)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHashUpdate,(const Ipp8u* pSrc, int len, IppsHashState* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHashGetTag,(Ipp8u* pTag, int tagLen, const IppsHashState* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHashFinal,(Ipp8u* pMD, IppsHashState* pState)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHashMessage,(const Ipp8u* pMsg, int len, Ipp8u* pMD, IppHashAlgId hashAlg)) + +/* method based generalized (reduced memory footprint) Hash Primitives */ +IPP_DEPRECATED(MD5_DEPRECATED) \ +IPPAPI( const IppsHashMethod*, ippsHashMethod_MD5, (void) ) +IPPAPI( const IppsHashMethod*, ippsHashMethod_SM3, (void) ) + +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA1, (void) ) +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA1_NI, (void) ) +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA1_TT, (void) ) + +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA256, (void) ) +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA256_NI, (void) ) +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA256_TT, (void) ) + +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA224, (void) ) +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA224_NI, (void) ) +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA224_TT, (void) ) + +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA512, (void) ) +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA384, (void) ) +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA512_256, (void) ) +IPPAPI( const IppsHashMethod*, ippsHashMethod_SHA512_224, (void) ) + +IPPAPI( IppStatus, ippsHashMethodGetSize, (int* pSize) ) +IPP_DEPRECATED(MD5_DEPRECATED) \ +IPPAPI( IppStatus, ippsHashMethodSet_MD5, (IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashMethodSet_SM3, (IppsHashMethod* pMethod) ) + +IPPAPI( IppStatus, ippsHashStateMethodSet_SM3, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) + +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI( IppStatus, ippsHashMethodSet_SHA1, (IppsHashMethod* pMethod) ) +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI( IppStatus, ippsHashMethodSet_SHA1_NI, (IppsHashMethod* pMethod) ) +IPP_DEPRECATED(SHA1_DEPRECATED) \ +IPPAPI( IppStatus, ippsHashMethodSet_SHA1_TT, (IppsHashMethod* pMethod) ) + +IPPAPI( IppStatus, ippsHashMethodSet_SHA256, (IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashMethodSet_SHA256_NI, (IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashMethodSet_SHA256_TT, (IppsHashMethod* pMethod) ) + +IPPAPI( IppStatus, ippsHashStateMethodSet_SHA256, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashStateMethodSet_SHA256_NI, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashStateMethodSet_SHA256_TT, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) + +IPPAPI( IppStatus, ippsHashMethodSet_SHA224, (IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashMethodSet_SHA224_NI, (IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashMethodSet_SHA224_TT, (IppsHashMethod* pMethod) ) + +IPPAPI( IppStatus, ippsHashStateMethodSet_SHA224, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashStateMethodSet_SHA224_NI, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashStateMethodSet_SHA224_TT, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) + +IPPAPI( IppStatus, ippsHashMethodSet_SHA512, (IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashMethodSet_SHA384, (IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashMethodSet_SHA512_256, (IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashMethodSet_SHA512_224, (IppsHashMethod* pMethod) ) + +IPPAPI( IppStatus, ippsHashStateMethodSet_SHA512, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashStateMethodSet_SHA384, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashStateMethodSet_SHA512_256, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +IPPAPI( IppStatus, ippsHashStateMethodSet_SHA512_224, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) + +IPPAPI(IppStatus, ippsHashGetSize_rmf,(int* pSize)) +IPPAPI(IppStatus, ippsHashInit_rmf,(IppsHashState_rmf* pState, const IppsHashMethod* pMethod)) + +IPPAPI(IppStatus, ippsHashPack_rmf,(const IppsHashState_rmf* pState, Ipp8u* pBuffer, int bufSize)) +IPPAPI(IppStatus, ippsHashUnpack_rmf,(const Ipp8u* pBuffer, IppsHashState_rmf* pState)) +IPPAPI(IppStatus, ippsHashDuplicate_rmf,(const IppsHashState_rmf* pSrcState, IppsHashState_rmf* pDstState)) + +IPPAPI(IppStatus, ippsHashUpdate_rmf,(const Ipp8u* pSrc, int len, IppsHashState_rmf* pState)) +IPPAPI(IppStatus, ippsHashGetTag_rmf,(Ipp8u* pMD, int tagLen, const IppsHashState_rmf* pState)) +IPPAPI(IppStatus, ippsHashFinal_rmf,(Ipp8u* pMD, IppsHashState_rmf* pState)) +IPPAPI(IppStatus, ippsHashMessage_rmf,(const Ipp8u* pMsg, int len, Ipp8u* pMD, const IppsHashMethod* pMethod)) + +IPPAPI(IppStatus, ippsHashMethodGetInfo,(IppsHashInfo* pInfo, const IppsHashMethod* pMethod)) +IPPAPI(IppStatus, ippsHashGetInfo_rmf,(IppsHashInfo* pInfo, const IppsHashState_rmf* pState)) + +/* general MGF Primitives*/ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsMGF,(const Ipp8u* pSeed, int seedLen, Ipp8u* pMask, int maskLen, IppHashAlgId hashAlg)) +IPPAPI(IppStatus, ippsMGF1_rmf,(const Ipp8u* pSeed, int seedLen, Ipp8u* pMask, int maskLen, const IppsHashMethod* pMethod)) +IPPAPI(IppStatus, ippsMGF2_rmf,(const Ipp8u* pSeed, int seedLen, Ipp8u* pMask, int maskLen, const IppsHashMethod* pMethod)) + + +/* +// ========================================================= +// Keyed-Hash Message Authentication Codes +// ========================================================= +*/ + +/* generalized Keyed HMAC primitives */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHMAC_GetSize,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHMAC_Init,(const Ipp8u* pKey, int keyLen, IppsHMACState* pCtx, IppHashAlgId hashAlg)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHMAC_Pack,(const IppsHMACState* pCtx, Ipp8u* pBuffer, int bufSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHMAC_Unpack,(const Ipp8u* pBuffer, IppsHMACState* pCtx)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHMAC_Duplicate,(const IppsHMACState* pSrcCtx, IppsHMACState* pDstCtx)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHMAC_Update,(const Ipp8u* pSrc, int len, IppsHMACState* pCtx)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHMAC_Final,(Ipp8u* pMD, int mdLen, IppsHMACState* pCtx)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHMAC_GetTag,(Ipp8u* pMD, int mdLen, const IppsHMACState* pCtx)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsHMAC_Message,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pKey, int keyLen, + Ipp8u* pMD, int mdLen, + IppHashAlgId hashAlg)) + +/* method based generalized (reduced memory footprint) Keyed HMAC primitives */ +IPPAPI(IppStatus, ippsHMACGetSize_rmf,(int* pSize)) +IPPAPI(IppStatus, ippsHMACInit_rmf,(const Ipp8u* pKey, int keyLen, + IppsHMACState_rmf* pCtx, + const IppsHashMethod* pMethod)) + +IPPAPI(IppStatus, ippsHMACPack_rmf,(const IppsHMACState_rmf* pCtx, Ipp8u* pBuffer, int bufSize)) +IPPAPI(IppStatus, ippsHMACUnpack_rmf,(const Ipp8u* pBuffer, IppsHMACState_rmf* pCtx)) +IPPAPI(IppStatus, ippsHMACDuplicate_rmf,(const IppsHMACState_rmf* pSrcCtx, IppsHMACState_rmf* pDstCtx)) + +IPPAPI(IppStatus, ippsHMACUpdate_rmf,(const Ipp8u* pSrc, int len, IppsHMACState_rmf* pCtx)) +IPPAPI(IppStatus, ippsHMACFinal_rmf,(Ipp8u* pMD, int mdLen, IppsHMACState_rmf* pCtx)) +IPPAPI(IppStatus, ippsHMACGetTag_rmf,(Ipp8u* pMD, int mdLen, const IppsHMACState_rmf* pCtx)) +IPPAPI(IppStatus, ippsHMACMessage_rmf,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pKey, int keyLen, + Ipp8u* pMD, int mdLen, + const IppsHashMethod* pMethod)) + + +/* +// ========================================================= +// Big Number Integer Arithmetic +// ========================================================= +*/ + +/* Signed BigNum Operations */ +IPPAPI(IppStatus, ippsBigNumGetSize,(int length, int* pSize)) +IPPAPI(IppStatus, ippsBigNumInit,(int length, IppsBigNumState* pBN)) + +IPPAPI(IppStatus, ippsCmpZero_BN,(const IppsBigNumState* pBN, Ipp32u* pResult)) +IPPAPI(IppStatus, ippsCmp_BN,(const IppsBigNumState* pA, const IppsBigNumState* pB, Ipp32u* pResult)) + +IPPAPI(IppStatus, ippsGetSize_BN,(const IppsBigNumState* pBN, int* pSize)) +IPPAPI(IppStatus, ippsSet_BN,(IppsBigNumSGN sgn, + int length, const Ipp32u* pData, + IppsBigNumState* pBN)) +IPPAPI(IppStatus, ippsGet_BN,(IppsBigNumSGN* pSgn, + int* pLength, Ipp32u* pData, + const IppsBigNumState* pBN)) +IPPAPI(IppStatus, ippsRef_BN,(IppsBigNumSGN* pSgn, int* bitSize, Ipp32u** const ppData, + const IppsBigNumState* pBN)) +IPPAPI(IppStatus, ippsExtGet_BN,(IppsBigNumSGN* pSgn, + int* pBitSize, Ipp32u* pData, + const IppsBigNumState* pBN)) + +IPPAPI(IppStatus, ippsAdd_BN, (IppsBigNumState* pA, IppsBigNumState* pB, IppsBigNumState* pR)) +IPPAPI(IppStatus, ippsSub_BN, (IppsBigNumState* pA, IppsBigNumState* pB, IppsBigNumState* pR)) +IPPAPI(IppStatus, ippsMul_BN, (IppsBigNumState* pA, IppsBigNumState* pB, IppsBigNumState* pR)) +IPPAPI(IppStatus, ippsMAC_BN_I, (IppsBigNumState* pA, IppsBigNumState* pB, IppsBigNumState* pR)) +IPPAPI(IppStatus, ippsDiv_BN, (IppsBigNumState* pA, IppsBigNumState* pB, IppsBigNumState* pQ, IppsBigNumState* pR)) +IPPAPI(IppStatus, ippsMod_BN, (IppsBigNumState* pA, IppsBigNumState* pM, IppsBigNumState* pR)) +IPPAPI(IppStatus, ippsGcd_BN, (IppsBigNumState* pA, IppsBigNumState* pB, IppsBigNumState* pGCD)) +IPPAPI(IppStatus, ippsModInv_BN,(IppsBigNumState* pA, IppsBigNumState* pM, IppsBigNumState* pInv)) + +IPPAPI(IppStatus, ippsSetOctString_BN,(const Ipp8u* pStr, int strLen, IppsBigNumState* pBN)) +IPPAPI(IppStatus, ippsGetOctString_BN,(Ipp8u* pStr, int strLen, const IppsBigNumState* pBN)) + +/* Montgomery Operations */ +IPPAPI(IppStatus, ippsMontGetSize,(IppsExpMethod method, int length, int* pSize)) +IPPAPI(IppStatus, ippsMontInit,(IppsExpMethod method, int length, IppsMontState* pCtx)) + +IPPAPI(IppStatus, ippsMontSet,(const Ipp32u* pModulo, int size, IppsMontState* pCtx)) +IPPAPI(IppStatus, ippsMontGet,(Ipp32u* pModulo, int* pSize, const IppsMontState* pCtx)) + +IPPAPI(IppStatus, ippsMontForm,(const IppsBigNumState* pA, IppsMontState* pCtx, IppsBigNumState* pR)) +IPPAPI(IppStatus, ippsMontMul, (const IppsBigNumState* pA, const IppsBigNumState* pB, IppsMontState* m, IppsBigNumState* pR)) +IPPAPI(IppStatus, ippsMontExp, (const IppsBigNumState* pA, const IppsBigNumState* pE, IppsMontState* m, IppsBigNumState* pR)) + +/* Pseudo-Random Number Generation */ +IPPAPI(IppStatus, ippsPRNGGetSize,(int* pSize)) +IPPAPI(IppStatus, ippsPRNGInit, (int seedBits, IppsPRNGState* pCtx)) + +IPPAPI(IppStatus, ippsPRNGSetModulus,(const IppsBigNumState* pMod, IppsPRNGState* pCtx)) +IPPAPI(IppStatus, ippsPRNGSetH0, (const IppsBigNumState* pH0, IppsPRNGState* pCtx)) +IPPAPI(IppStatus, ippsPRNGSetAugment,(const IppsBigNumState* pAug, IppsPRNGState* pCtx)) +IPPAPI(IppStatus, ippsPRNGSetSeed, (const IppsBigNumState* pSeed,IppsPRNGState* pCtx)) +IPPAPI(IppStatus, ippsPRNGGetSeed, (const IppsPRNGState* pCtx,IppsBigNumState* pSeed)) + +IPPAPI(IppStatus, ippsPRNGen, (Ipp32u* pRand, int nBits, void* pCtx)) +IPPAPI(IppStatus, ippsPRNGen_BN, (IppsBigNumState* pRand, int nBits, void* pCtx)) +IPPAPI(IppStatus, ippsPRNGenRDRAND, (Ipp32u* pRand, int nBits, void* pCtx)) +IPPAPI(IppStatus, ippsPRNGenRDRAND_BN,(IppsBigNumState* pRand, int nBits, void* pCtx)) +IPPAPI(IppStatus, ippsTRNGenRDSEED, (Ipp32u* pRand, int nBits, void* pCtx)) +IPPAPI(IppStatus, ippsTRNGenRDSEED_BN,(IppsBigNumState* pRand, int nBits, void* pCtx)) + +/* Probable Prime Number Generation */ +IPPAPI(IppStatus, ippsPrimeGetSize,(int nMaxBits, int* pSize)) +IPPAPI(IppStatus, ippsPrimeInit, (int nMaxBits, IppsPrimeState* pCtx)) + +IPPAPI(IppStatus, ippsPrimeGen, (int nBits, int nTrials, IppsPrimeState* pCtx, + IppBitSupplier rndFunc, void* pRndParam)) +IPPAPI(IppStatus, ippsPrimeTest,(int nTrials, Ipp32u* pResult, IppsPrimeState* pCtx, + IppBitSupplier rndFunc, void* pRndParam)) +IPPAPI(IppStatus, ippsPrimeGen_BN,(IppsBigNumState* pPrime, int nBits, int nTrials, IppsPrimeState* pCtx, + IppBitSupplier rndFunc, void* pRndParam)) +IPPAPI(IppStatus, ippsPrimeTest_BN,(const IppsBigNumState* pPrime, int nTrials, Ipp32u* pResult, IppsPrimeState* pCtx, + IppBitSupplier rndFunc, void* pRndParam)) + +IPPAPI(IppStatus, ippsPrimeGet, (Ipp32u* pPrime, int* pLen, const IppsPrimeState* pCtx)) +IPPAPI(IppStatus, ippsPrimeGet_BN,(IppsBigNumState* pPrime, const IppsPrimeState* pCtx)) + +IPPAPI(IppStatus, ippsPrimeSet, (const Ipp32u* pPrime, int nBits, IppsPrimeState* pCtx)) +IPPAPI(IppStatus, ippsPrimeSet_BN,(const IppsBigNumState* pPrime, IppsPrimeState* pCtx)) + + +/* +// ========================================================= +// RSA Cryptography +// ========================================================= +*/ +IPPAPI(IppStatus, ippsRSA_GetSizePublicKey,(int rsaModulusBitSize, int pubicExpBitSize, int* pKeySize)) +IPPAPI(IppStatus, ippsRSA_InitPublicKey,(int rsaModulusBitSize, int publicExpBitSize, + IppsRSAPublicKeyState* pKey, int keyCtxSize)) +IPPAPI(IppStatus, ippsRSA_SetPublicKey,(const IppsBigNumState* pModulus, + const IppsBigNumState* pPublicExp, + IppsRSAPublicKeyState* pKey)) +IPPAPI(IppStatus, ippsRSA_GetPublicKey,(IppsBigNumState* pModulus, + IppsBigNumState* pPublicExp, + const IppsRSAPublicKeyState* pKey)) + +IPPAPI(IppStatus, ippsRSA_GetSizePrivateKeyType1,(int rsaModulusBitSize, int privateExpBitSize, int* pKeySize)) +IPPAPI(IppStatus, ippsRSA_InitPrivateKeyType1,(int rsaModulusBitSize, int privateExpBitSize, + IppsRSAPrivateKeyState* pKey, int keyCtxSize)) +IPPAPI(IppStatus, ippsRSA_SetPrivateKeyType1,(const IppsBigNumState* pModulus, + const IppsBigNumState* pPrivateExp, + IppsRSAPrivateKeyState* pKey)) +IPPAPI(IppStatus, ippsRSA_GetPrivateKeyType1,(IppsBigNumState* pModulus, + IppsBigNumState* pPrivateExp, + const IppsRSAPrivateKeyState* pKey)) + +IPPAPI(IppStatus, ippsRSA_GetSizePrivateKeyType2,(int factorPbitSize, int factorQbitSize, int* pKeySize)) +IPPAPI(IppStatus, ippsRSA_InitPrivateKeyType2,(int factorPbitSize, int factorQbitSize, + IppsRSAPrivateKeyState* pKey, int keyCtxSize)) +IPPAPI(IppStatus, ippsRSA_SetPrivateKeyType2,(const IppsBigNumState* pFactorP, + const IppsBigNumState* pFactorQ, + const IppsBigNumState* pCrtExpP, + const IppsBigNumState* pCrtExpQ, + const IppsBigNumState* pInverseQ, + IppsRSAPrivateKeyState* pKey)) +IPPAPI(IppStatus, ippsRSA_GetPrivateKeyType2,(IppsBigNumState* pFactorP, + IppsBigNumState* pFactorQ, + IppsBigNumState* pCrtExpP, + IppsBigNumState* pCrtExpQ, + IppsBigNumState* pInverseQ, + const IppsRSAPrivateKeyState* pKey)) + +IPPAPI(IppStatus, ippsRSA_GetBufferSizePublicKey,(int* pBufferSize, const IppsRSAPublicKeyState* pKey)) +IPPAPI(IppStatus, ippsRSA_GetBufferSizePrivateKey,(int* pBufferSize, const IppsRSAPrivateKeyState* pKey)) + +IPPAPI(IppStatus, ippsRSA_Encrypt,(const IppsBigNumState* pPtxt, + IppsBigNumState* pCtxt, + const IppsRSAPublicKeyState* pKey, + Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsRSA_Decrypt,(const IppsBigNumState* pCtxt, + IppsBigNumState* pPtxt, + const IppsRSAPrivateKeyState* pKey, + Ipp8u* pScratchBuffer)) + +IPPAPI(IppStatus, ippsRSA_GenerateKeys,(const IppsBigNumState* pSrcPublicExp, + IppsBigNumState* pModulus, + IppsBigNumState* pPublicExp, + IppsBigNumState* pPrivateExp, + IppsRSAPrivateKeyState* pPrivateKeyType2, + Ipp8u* pScratchBuffer, + int nTrials, + IppsPrimeState* pPrimeGen, + IppBitSupplier rndFunc, void* pRndParam)) + +IPPAPI(IppStatus, ippsRSA_ValidateKeys,(int* pResult, + const IppsRSAPublicKeyState* pPublicKey, + const IppsRSAPrivateKeyState* pPrivateKeyType2, + const IppsRSAPrivateKeyState* pPrivateKeyType1, + Ipp8u* pScratchBuffer, + int nTrials, + IppsPrimeState* pPrimeGen, + IppBitSupplier rndFunc, void* pRndParam)) + +/* encryption scheme: RSAES-OAEP */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsRSAEncrypt_OAEP,(const Ipp8u* pSrc, int srcLen, + const Ipp8u* pLabel, int labLen, + const Ipp8u* pSeed, + Ipp8u* pDst, + const IppsRSAPublicKeyState* pKey, + IppHashAlgId hashAlg, + Ipp8u* pBuffer)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsRSADecrypt_OAEP,(const Ipp8u* pSrc, + const Ipp8u* pLab, int labLen, + Ipp8u* pDst, int* pDstLen, + const IppsRSAPrivateKeyState* pKey, + IppHashAlgId hashAlg, + Ipp8u* pBuffer)) + +IPPAPI(IppStatus, ippsRSAEncrypt_OAEP_rmf,(const Ipp8u* pSrc, int srcLen, + const Ipp8u* pLabel, int labLen, + const Ipp8u* pSeed, + Ipp8u* pDst, + const IppsRSAPublicKeyState* pKey, + const IppsHashMethod* pMethod, + Ipp8u* pBuffer)) + +IPPAPI(IppStatus, ippsRSADecrypt_OAEP_rmf,(const Ipp8u* pSrc, + const Ipp8u* pLab, int labLen, + Ipp8u* pDst, int* pDstLen, + const IppsRSAPrivateKeyState* pKey, + const IppsHashMethod* pMethod, + Ipp8u* pBuffer)) + +/* encryption scheme: RSAES-PKCS_v1_5 */ + +#define PKCS_DEPRECATED "This algorithm is considered weak due to known attacks on it. \ +The functionality remains in the library, but the implementation will no be longer \ +optimized and no security patches will be applied. A more secure alternative is available: RSA-OAEP" + +IPP_DEPRECATED(PKCS_DEPRECATED) \ +IPPAPI(IppStatus, ippsRSAEncrypt_PKCSv15,(const Ipp8u* pSrc, int srcLen, + const Ipp8u* pRndPS, + Ipp8u* pDst, + const IppsRSAPublicKeyState* pKey, + Ipp8u* pBuffer)) + +IPP_DEPRECATED(PKCS_DEPRECATED) \ +IPPAPI(IppStatus, ippsRSADecrypt_PKCSv15,(const Ipp8u* pSrc, + Ipp8u* pDst, int* pDstLen, + const IppsRSAPrivateKeyState* pKey, + Ipp8u* pBuffer)) + +/* signature scheme : RSA-SSA-PSS */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsRSASign_PSS,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pSalt, int saltLen, + Ipp8u* pSign, + const IppsRSAPrivateKeyState* pPrvKey, + const IppsRSAPublicKeyState* pPubKey, + IppHashAlgId hashAlg, + Ipp8u* pBuffer)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsRSAVerify_PSS,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pSign, + int* pIsValid, + const IppsRSAPublicKeyState* pKey, + IppHashAlgId hashAlg, + Ipp8u* pBuffer)) + +IPPAPI(IppStatus, ippsRSASign_PSS_rmf,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pSalt, int saltLen, + Ipp8u* pSign, + const IppsRSAPrivateKeyState* pPrvKey, + const IppsRSAPublicKeyState* pPubKey, + const IppsHashMethod* pMethod, + Ipp8u* pBuffer)) + +IPPAPI(IppStatus, ippsRSAVerify_PSS_rmf,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pSign, + int* pIsValid, + const IppsRSAPublicKeyState* pKey, + const IppsHashMethod* pMethod, + Ipp8u* pBuffer)) + +/* signature scheme : RSA-SSA-PKCS1-v1_5 */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsRSASign_PKCS1v15,(const Ipp8u* pMsg, int msgLen, + Ipp8u* pSign, + const IppsRSAPrivateKeyState* pPrvKey, + const IppsRSAPublicKeyState* pPubKey, + IppHashAlgId hashAlg, + Ipp8u* pBuffer)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsRSAVerify_PKCS1v15,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pSign, int* pIsValid, + const IppsRSAPublicKeyState* pKey, + IppHashAlgId hashAlg, + Ipp8u* pBuffer)) + +IPPAPI(IppStatus, ippsRSASign_PKCS1v15_rmf,(const Ipp8u* pMsg, int msgLen, + Ipp8u* pSign, + const IppsRSAPrivateKeyState* pPrvKey, + const IppsRSAPublicKeyState* pPubKey, + const IppsHashMethod* pMethod, + Ipp8u* pBuffer)) + +IPPAPI(IppStatus, ippsRSAVerify_PKCS1v15_rmf,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pSign, int* pIsValid, + const IppsRSAPublicKeyState* pKey, + const IppsHashMethod* pMethod, + Ipp8u* pBuffer)) + +/* +// ========================================================= +// DL Cryptography +// ========================================================= +*/ +IPPAPI( const char*, ippsDLGetResultString, (IppDLResult code)) + +/* Initialization */ +IPPAPI(IppStatus, ippsDLPGetSize,(int bitSizeP, int bitSizeR, int* pSize)) +IPPAPI(IppStatus, ippsDLPInit, (int bitSizeP, int bitSizeR, IppsDLPState* pCtx)) + +IPPAPI(IppStatus, ippsDLPPack,(const IppsDLPState* pCtx, Ipp8u* pBuffer)) +IPPAPI(IppStatus, ippsDLPUnpack,(const Ipp8u* pBuffer, IppsDLPState* pCtx)) + +/* Set Up and Retrieve Domain Parameters */ +IPPAPI(IppStatus, ippsDLPSet,(const IppsBigNumState* pP, + const IppsBigNumState* pR, + const IppsBigNumState* pG, + IppsDLPState* pCtx)) +IPPAPI(IppStatus, ippsDLPGet,(IppsBigNumState* pP, + IppsBigNumState* pR, + IppsBigNumState* pG, + IppsDLPState* pCtx)) +IPPAPI(IppStatus, ippsDLPSetDP,(const IppsBigNumState* pDP, IppDLPKeyTag tag, IppsDLPState* pCtx)) +IPPAPI(IppStatus, ippsDLPGetDP,(IppsBigNumState* pDP, IppDLPKeyTag tag, const IppsDLPState* pCtx)) + +/* Key Generation, Validation and Set Up */ +IPPAPI(IppStatus, ippsDLPGenKeyPair,(IppsBigNumState* pPrvKey, IppsBigNumState* pPubKey, + IppsDLPState* pCtx, + IppBitSupplier rndFunc, void* pRndParam)) +IPPAPI(IppStatus, ippsDLPPublicKey, (const IppsBigNumState* pPrvKey, + IppsBigNumState* pPubKey, + IppsDLPState* pCtx)) +IPPAPI(IppStatus, ippsDLPValidateKeyPair,(const IppsBigNumState* pPrvKey, + const IppsBigNumState* pPubKey, + IppDLResult* pResult, + IppsDLPState* pCtx)) + +IPPAPI(IppStatus, ippsDLPSetKeyPair,(const IppsBigNumState* pPrvKey, + const IppsBigNumState* pPubKey, + IppsDLPState* pCtx)) + +/* Signing/Verifying (DSA version) */ +IPPAPI(IppStatus, ippsDLPSignDSA, (const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pPrvKey, + IppsBigNumState* pSignR, IppsBigNumState* pSignS, + IppsDLPState* pCtx)) +IPPAPI(IppStatus, ippsDLPVerifyDSA,(const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pSignR, const IppsBigNumState* pSignS, + IppDLResult* pResult, + IppsDLPState* pCtx)) + +/* Shared Secret Element (DH version) */ +IPPAPI(IppStatus, ippsDLPSharedSecretDH,(const IppsBigNumState* pPrvKeyA, + const IppsBigNumState* pPubKeyB, + IppsBigNumState* pShare, + IppsDLPState* pCtx)) + +/* DSA's parameter Generation and Validation */ +IPPAPI(IppStatus, ippsDLPGenerateDSA,(const IppsBigNumState* pSeedIn, + int nTrials, IppsDLPState* pCtx, + IppsBigNumState* pSeedOut, int* pCounter, + IppBitSupplier rndFunc, void* pRndParam)) +IPPAPI(IppStatus, ippsDLPValidateDSA,(int nTrials, IppDLResult* pResult, IppsDLPState* pCtx, + IppBitSupplier rndFunc, void* pRndParam)) + +/* DH parameter's Generation and Validation */ +IPPAPI(IppStatus, ippsDLPGenerateDH,(const IppsBigNumState* pSeedIn, + int nTrials, IppsDLPState* pCtx, + IppsBigNumState* pSeedOut, int* pCounter, + IppBitSupplier rndFunc, void* pRndParam)) +IPPAPI(IppStatus, ippsDLPValidateDH,(int nTrials, IppDLResult* pResult, IppsDLPState* pCtx, + IppBitSupplier rndFunc, void* pRndParam)) + + +/* +// ========================================================= +// EC Cryptography +// ========================================================= +*/ +IPPAPI( const char*, ippsECCGetResultString, (IppECResult code)) + +/* +// EC over Prime Fields +*/ +/* general EC initialization */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGetSize,(int feBitSize, int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGetSizeStd128r1,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGetSizeStd128r2,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGetSizeStd192r1,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGetSizeStd224r1,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGetSizeStd256r1,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGetSizeStd384r1,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGetSizeStd521r1,(int* pSize)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGetSizeStdSM2, (int* pSize)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPInit,(int feBitSize, IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPInitStd128r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPInitStd128r2,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPInitStd192r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPInitStd224r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPInitStd256r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPInitStd384r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPInitStd521r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPInitStdSM2, (IppsECCPState* pEC)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSet,(const IppsBigNumState* pPrime, + const IppsBigNumState* pA, const IppsBigNumState* pB, + const IppsBigNumState* pGX,const IppsBigNumState* pGY,const IppsBigNumState* pOrder, + int cofactor, + IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetStd,(IppECCType flag, IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetStd128r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetStd128r2,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetStd192r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetStd224r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetStd256r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetStd384r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetStd521r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetStdSM2, (IppsECCPState* pEC)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPBindGxyTblStd192r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPBindGxyTblStd224r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPBindGxyTblStd256r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPBindGxyTblStd384r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPBindGxyTblStd521r1,(IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPBindGxyTblStdSM2, (IppsECCPState* pEC)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGet,(IppsBigNumState* pPrime, + IppsBigNumState* pA, IppsBigNumState* pB, + IppsBigNumState* pGX,IppsBigNumState* pGY,IppsBigNumState* pOrder, + int* cofactor, + IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGetOrderBitSize,(int* pBitSize, IppsECCPState* pEC)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPValidate,(int nTrials, IppECResult* pResult, IppsECCPState* pEC, + IppBitSupplier rndFunc, void* pRndParam)) + +/* EC Point */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPPointGetSize,(int feBitSize, int* pSize)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPPointInit,(int feBitSize, IppsECCPPointState* pPoint)) + +/* Setup/retrieve point's coordinates */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetPoint,(const IppsBigNumState* pX, const IppsBigNumState* pY, + IppsECCPPointState* pPoint, IppsECCPState* pEC)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetPointAtInfinity,(IppsECCPPointState* pPoint, IppsECCPState* pEC)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGetPoint,(IppsBigNumState* pX, IppsBigNumState* pY, + const IppsECCPPointState* pPoint, IppsECCPState* pEC)) + +/* EC Point Operations */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPCheckPoint,(const IppsECCPPointState* pP, + IppECResult* pResult, IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPComparePoint,(const IppsECCPPointState* pP, const IppsECCPPointState* pQ, + IppECResult* pResult, IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPNegativePoint,(const IppsECCPPointState* pP, + IppsECCPPointState* pR, IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPAddPoint,(const IppsECCPPointState* pP, const IppsECCPPointState* pQ, + IppsECCPPointState* pR, IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPMulPointScalar,(const IppsECCPPointState* pP, const IppsBigNumState* pK, + IppsECCPPointState* pR, IppsECCPState* pEC)) + +/* Key Generation, Setup and Validation */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPGenKeyPair,(IppsBigNumState* pPrivate, IppsECCPPointState* pPublic, + IppsECCPState* pEC, + IppBitSupplier rndFunc, void* pRndParam)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPPublicKey,(const IppsBigNumState* pPrivate, + IppsECCPPointState* pPublic, + IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPValidateKeyPair,(const IppsBigNumState* pPrivate, const IppsECCPPointState* pPublic, + IppECResult* pResult, + IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSetKeyPair,(const IppsBigNumState* pPrivate, const IppsECCPPointState* pPublic, + IppBool regular, + IppsECCPState* pEC)) + +/* Shared Secret (DH scheme ) */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSharedSecretDH,(const IppsBigNumState* pPrivateA, + const IppsECCPPointState* pPublicB, + IppsBigNumState* pShare, + IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSharedSecretDHC,(const IppsBigNumState* pPrivateA, + const IppsECCPPointState* pPublicB, + IppsBigNumState* pShare, + IppsECCPState* pEC)) + +/* Sign/Verify */ +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSignDSA,(const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pPrivate, + IppsBigNumState* pSignX, IppsBigNumState* pSignY, + IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPVerifyDSA,(const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pSignX, const IppsBigNumState* pSignY, + IppECResult* pResult, + IppsECCPState* pEC)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSignNR,(const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pPrivate, + IppsBigNumState* pSignX, IppsBigNumState* pSignY, + IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPVerifyNR,(const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pSignX, const IppsBigNumState* pSignY, + IppECResult* pResult, + IppsECCPState* pEC)) + +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPSignSM2,(const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pRegPrivate, + IppsBigNumState* pEphPrivate, + IppsBigNumState* pSignR, IppsBigNumState* pSignS, + IppsECCPState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsECCPVerifySM2,(const IppsBigNumState* pMsgDigest, + const IppsECCPPointState* pRegPublic, + const IppsBigNumState* pSignR, const IppsBigNumState* pSignS, + IppECResult* pResult, + IppsECCPState* pEC)) + +/* +// GF over prime and its extension +*/ +IPPAPI(IppStatus, ippsGFpGetSize, (int feBitSize, int* pSize)) +IPPAPI(IppStatus, ippsGFpInitArbitrary,(const IppsBigNumState* pPrime, int primeBitSize, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpInitFixed,(int primeBitSize, const IppsGFpMethod* pGFpMethod, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpInit, (const IppsBigNumState* pPrime, int primeBitSize, const IppsGFpMethod* pGFpMethod, IppsGFpState* pGFp)) +IPPAPI( const IppsGFpMethod*, ippsGFpMethod_p192r1, (void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpMethod_p224r1, (void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpMethod_p256r1, (void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpMethod_p384r1, (void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpMethod_p521r1, (void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpMethod_p256sm2,(void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpMethod_p256bn, (void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpMethod_p256, (void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpMethod_pArb, (void) ) + +IPPAPI(IppStatus, ippsGFpxGetSize,(const IppsGFpState* pGroundGF, int degree, int* pSize)) +IPPAPI(IppStatus, ippsGFpxInit, (const IppsGFpState* pGroundGF, int extDeg, const IppsGFpElement* const ppGroundElm[], int nElm, const IppsGFpMethod* pGFpMethod, IppsGFpState* pGFpx)) +IPPAPI(IppStatus, ippsGFpxInitBinomial,(const IppsGFpState* pGroundGF, int extDeg, const IppsGFpElement* pGroundElm, const IppsGFpMethod* pGFpMethod, IppsGFpState* pGFpx)) +IPPAPI( const IppsGFpMethod*, ippsGFpxMethod_binom2_epid2,(void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpxMethod_binom3_epid2,(void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpxMethod_binom2, (void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpxMethod_binom3, (void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpxMethod_binom, (void) ) +IPPAPI( const IppsGFpMethod*, ippsGFpxMethod_com, (void) ) + +IPPAPI(IppStatus, ippsGFpScratchBufferSize,(int nExponents, int ExpBitSize, const IppsGFpState* pGFp, int* pBufferSize)) + +IPPAPI(IppStatus, ippsGFpElementGetSize,(const IppsGFpState* pGFp, int* pElementSize)) +IPPAPI(IppStatus, ippsGFpElementInit, (const Ipp32u* pA, int lenA, IppsGFpElement* pR, IppsGFpState* pGFp)) + +IPPAPI(IppStatus, ippsGFpSetElement, (const Ipp32u* pA, int lenA, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpSetElementRegular,(const IppsBigNumState* pBN, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpSetElementOctString,(const Ipp8u* pStr, int strSize, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpSetElementRandom,(IppsGFpElement* pR, IppsGFpState* pGFp, IppBitSupplier rndFunc, void* pRndParam)) +IPPAPI(IppStatus, ippsGFpSetElementHash,(const Ipp8u* pMsg, int msgLen, IppsGFpElement* pElm, IppsGFpState* pGFp, IppHashAlgId hashID)) +IPPAPI(IppStatus, ippsGFpSetElementHash_rmf,(const Ipp8u* pMsg, int msgLen, IppsGFpElement* pElm, IppsGFpState* pGFp, const IppsHashMethod* pMethod)) +IPPAPI(IppStatus, ippsGFpCpyElement,(const IppsGFpElement* pA, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpGetElement,(const IppsGFpElement* pA, Ipp32u* pDataA, int lenA, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpGetElementOctString,(const IppsGFpElement* pA, Ipp8u* pStr, int strSize, IppsGFpState* pGFp)) + +IPPAPI(IppStatus, ippsGFpCmpElement,(const IppsGFpElement* pA, const IppsGFpElement* pB, int* pResult, const IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpIsZeroElement,(const IppsGFpElement* pA, int* pResult, const IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpIsUnityElement,(const IppsGFpElement* pA, int* pResult, const IppsGFpState* pGFp)) + +IPPAPI(IppStatus, ippsGFpConj,(const IppsGFpElement* pA, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpNeg, (const IppsGFpElement* pA, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpInv, (const IppsGFpElement* pA, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpSqrt,(const IppsGFpElement* pA, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpSqr, (const IppsGFpElement* pA, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpAdd, (const IppsGFpElement* pA, const IppsGFpElement* pB, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpSub, (const IppsGFpElement* pA, const IppsGFpElement* pB, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpMul, (const IppsGFpElement* pA, const IppsGFpElement* pB, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpExp, (const IppsGFpElement* pA, const IppsBigNumState* pE, IppsGFpElement* pR, IppsGFpState* pGFp, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpMultiExp,(const IppsGFpElement* const ppElmA[], const IppsBigNumState* const ppE[], int nItems, IppsGFpElement* pR, IppsGFpState* pGFp, Ipp8u* pScratchBuffer)) + +IPPAPI(IppStatus, ippsGFpAdd_PE,(const IppsGFpElement* pA, const IppsGFpElement* pParentB, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpSub_PE,(const IppsGFpElement* pA, const IppsGFpElement* pParentB, IppsGFpElement* pR, IppsGFpState* pGFp)) +IPPAPI(IppStatus, ippsGFpMul_PE,(const IppsGFpElement* pA, const IppsGFpElement* pParentB, IppsGFpElement* pR, IppsGFpState* pGFp)) + +IPPAPI(IppStatus, ippsGFpGetInfo, (IppsGFpInfo* pInfo, const IppsGFpState* pGFp)) + +/* ================== */ +IPPAPI(IppStatus, ippsGFpECGetSize,(const IppsGFpState* pGFp, int* pSize)) +IPPAPI(IppStatus, ippsGFpECInit, (const IppsGFpState* pGFp, + const IppsGFpElement* pA, const IppsGFpElement* pB, + IppsGFpECState* pEC)) + +IPPAPI(IppStatus, ippsGFpECSet,(const IppsGFpElement* pA, const IppsGFpElement* pB, + IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECSetSubgroup,(const IppsGFpElement* pX, const IppsGFpElement* pY, + const IppsBigNumState* pOrder, + const IppsBigNumState* pCofactor, + IppsGFpECState* pEC)) + +IPPAPI(IppStatus, ippsGFpECInitStd128r1,(const IppsGFpState* pGFp, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECInitStd128r2,(const IppsGFpState* pGFp, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECInitStd192r1,(const IppsGFpState* pGFp, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECInitStd224r1,(const IppsGFpState* pGFp, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECInitStd256r1,(const IppsGFpState* pGFp, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECInitStd384r1,(const IppsGFpState* pGFp, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECInitStd521r1,(const IppsGFpState* pGFp, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECInitStdSM2, (const IppsGFpState* pGFp, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECInitStdBN256,(const IppsGFpState* pGFp, IppsGFpECState* pEC)) + +IPPAPI(IppStatus, ippsGFpECBindGxyTblStd192r1,(IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECBindGxyTblStd224r1,(IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECBindGxyTblStd256r1,(IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECBindGxyTblStd384r1,(IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECBindGxyTblStd521r1,(IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECBindGxyTblStdSM2, (IppsGFpECState* pEC)) + +IPPAPI(IppStatus, ippsGFpECGet,(IppsGFpState** const ppGFp, + IppsGFpElement* pA, IppsGFpElement* pB, + const IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECGetSubgroup,(IppsGFpState** const ppGFp, + IppsGFpElement* pX, IppsGFpElement* pY, + IppsBigNumState* pOrder,IppsBigNumState* pCofactor, + const IppsGFpECState* pEC)) + +IPPAPI(IppStatus, ippsGFpECScratchBufferSize,(int nScalars, const IppsGFpECState* pEC, int* pBufferSize)) + +IPPAPI(IppStatus, ippsGFpECVerify,(IppECResult* pResult, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) + +IPPAPI(IppStatus, ippsGFpECPointGetSize,(const IppsGFpECState* pEC, int* pSize)) +IPPAPI(IppStatus, ippsGFpECPointInit, (const IppsGFpElement* pX, const IppsGFpElement* pY, IppsGFpECPoint* pPoint, IppsGFpECState* pEC)) + +IPPAPI(IppStatus, ippsGFpECSetPointAtInfinity,(IppsGFpECPoint* pPoint, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECSetPoint,(const IppsGFpElement* pX, const IppsGFpElement* pY, IppsGFpECPoint* pPoint, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECSetPointRegular,(const IppsBigNumState* pX, const IppsBigNumState* pY, IppsGFpECPoint* pPoint, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECSetPointRandom,(IppsGFpECPoint* pPoint, IppsGFpECState* pEC, IppBitSupplier rndFunc, void* pRndParam, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECMakePoint,(const IppsGFpElement* pX, IppsGFpECPoint* pPoint, IppsGFpECState* pEC)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsGFpECSetPointHash,(Ipp32u hdr, const Ipp8u* pMsg, int msgLen, IppsGFpECPoint* pPoint, IppsGFpECState* pEC, IppHashAlgId hashID, Ipp8u* pScratchBuffer)) +IPP_DEPRECATED(OBSOLETE_API) \ +IPPAPI(IppStatus, ippsGFpECSetPointHashBackCompatible,(Ipp32u hdr, const Ipp8u* pMsg, int msgLen, IppsGFpECPoint* pPoint, IppsGFpECState* pEC, IppHashAlgId hashID, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECSetPointHash_rmf,(Ipp32u hdr, const Ipp8u* pMsg, int msgLen, IppsGFpECPoint* pPoint, IppsGFpECState* pEC, const IppsHashMethod* pMethod, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECSetPointHashBackCompatible_rmf,(Ipp32u hdr, const Ipp8u* pMsg, int msgLen, IppsGFpECPoint* pPoint, IppsGFpECState* pEC, const IppsHashMethod* pMethod, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECGetPoint,(const IppsGFpECPoint* pPoint, IppsGFpElement* pX, IppsGFpElement* pY, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECGetPointRegular,(const IppsGFpECPoint* pPoint, IppsBigNumState* pX, IppsBigNumState* pY, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECSetPointOctString,(const Ipp8u* pStr, int strLen, IppsGFpECPoint* pPoint, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECGetPointOctString,(const IppsGFpECPoint* pPoint, Ipp8u* pStr, int strLen, IppsGFpECState* pEC)) + +IPPAPI(IppStatus, ippsGFpECTstPoint,(const IppsGFpECPoint* pP, IppECResult* pResult, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECTstPointInSubgroup,(const IppsGFpECPoint* pP, IppECResult* pResult, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECCpyPoint,(const IppsGFpECPoint* pA, IppsGFpECPoint* pR, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECCmpPoint,(const IppsGFpECPoint* pP, const IppsGFpECPoint* pQ, IppECResult* pResult, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECNegPoint,(const IppsGFpECPoint* pP, IppsGFpECPoint* pR, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECAddPoint,(const IppsGFpECPoint* pP, const IppsGFpECPoint* pQ, IppsGFpECPoint* pR, IppsGFpECState* pEC)) +IPPAPI(IppStatus, ippsGFpECMulPoint,(const IppsGFpECPoint* pP, const IppsBigNumState* pN, IppsGFpECPoint* pR, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) + +/* keys */ +IPPAPI(IppStatus, ippsGFpECPrivateKey,(IppsBigNumState* pPrivate, IppsGFpECState* pEC, + IppBitSupplier rndFunc, void* pRndParam)) +IPPAPI(IppStatus, ippsGFpECPublicKey,(const IppsBigNumState* pPrivate, IppsGFpECPoint* pPublic, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECTstKeyPair,(const IppsBigNumState* pPrivate, const IppsGFpECPoint* pPublic, IppECResult* pResult, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) + +/* DH shared secret */ +IPPAPI(IppStatus, ippsGFpECSharedSecretDH,(const IppsBigNumState* pPrivateA, const IppsGFpECPoint* pPublicB, + IppsBigNumState* pShare, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECSharedSecretDHC,(const IppsBigNumState* pPrivateA, + const IppsGFpECPoint* pPublicB, + IppsBigNumState* pShare, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) + +/* sign generation/verification of DSA, NR, SM2 */ +IPPAPI(IppStatus, ippsGFpECMessageRepresentationSM2, (IppsBigNumState* pMsgDigest, + const Ipp8u* pMsg, int msgLen, + const Ipp8u* pUserID, int userIDLen, + const IppsGFpECPoint* pRegPublic, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECSignDSA, (const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pRegPrivate, + IppsBigNumState* pEphPrivate, + IppsBigNumState* pSignR, IppsBigNumState* pSignS, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECVerifyDSA, (const IppsBigNumState* pMsgDigest, + const IppsGFpECPoint* pRegPublic, + const IppsBigNumState* pSignR, const IppsBigNumState* pSignS, + IppECResult* pResult, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) + +IPPAPI(IppStatus, ippsGFpECSignNR, (const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pRegPrivate, + IppsBigNumState* pEphPrivate, + IppsBigNumState* pSignR, IppsBigNumState* pSignS, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECVerifyNR, (const IppsBigNumState* pMsgDigest, + const IppsGFpECPoint* pRegPublic, + const IppsBigNumState* pSignR, const IppsBigNumState* pSignS, + IppECResult* pResult, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) + +IPPAPI(IppStatus, ippsGFpECSignSM2, (const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pRegPrivate, + IppsBigNumState* pEphPrivate, + IppsBigNumState* pSignR, IppsBigNumState* pSignS, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECVerifySM2, (const IppsBigNumState* pMsgDigest, + const IppsGFpECPoint* pRegPublic, + const IppsBigNumState* pSignR, const IppsBigNumState* pSignS, + IppECResult* pResult, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) + +/* SM2 UserIDHash */ +IPPAPI(IppStatus, ippsGFpECUserIDHashSM2, (Ipp8u* pZaDigest, + const Ipp8u* pUserID, int userIDLen, + const IppsGFpECPoint* pPublicKey, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) + + +/* SM2 Key Exchange */ +IPPAPI(IppStatus, ippsGFpECKeyExchangeSM2_GetSize, (const IppsGFpECState* pEC, int* pSize)) +IPPAPI(IppStatus, ippsGFpECKeyExchangeSM2_Init, (IppsGFpECKeyExchangeSM2State* pKE, IppsKeyExchangeRoleSM2 role, IppsGFpECState* pEC)) + +IPPAPI(IppStatus, ippsGFpECKeyExchangeSM2_Setup, (const Ipp8u pZSelf[IPP_SM3_DIGEST_BYTESIZE], + const Ipp8u pZPeer[IPP_SM3_DIGEST_BYTESIZE], + const IppsGFpECPoint *pPublicKeySelf, + const IppsGFpECPoint *pPublicKeyPeer, + const IppsGFpECPoint *pEphPublicKeySelf, + const IppsGFpECPoint *pEphPublicKeyPeer, + IppsGFpECKeyExchangeSM2State *pKE)) + +IPPAPI(IppStatus, ippsGFpECKeyExchangeSM2_SharedKey, (Ipp8u* pSharedKey, int sharedKeySize, + Ipp8u* pSSelf, + const IppsBigNumState* pPrvKey, + IppsBigNumState* pEphPrvKey, + IppsGFpECKeyExchangeSM2State *pKE, Ipp8u* pScratchBuffer)) + +IPPAPI(IppStatus, ippsGFpECKeyExchangeSM2_Confirm, (const Ipp8u pSPeer[IPP_SM3_DIGEST_BYTESIZE], + int* pStatus, + IppsGFpECKeyExchangeSM2State* pKE)) + +/* SM2 Encryption/Decryption */ +IPPAPI(IppStatus, ippsGFpECGetInfo_GF,(IppsGFpInfo* pInfo, const IppsGFpECState* pEC)) + +IPPAPI(IppStatus, ippsGFpECESGetSize_SM2, (const IppsGFpECState* pEC, int* pSize)) +IPPAPI(IppStatus, ippsGFpECESInit_SM2, (IppsGFpECState* pEC, + IppsECESState_SM2* pState, int avaliableCtxSize)) +IPPAPI(IppStatus, ippsGFpECESSetKey_SM2, (const IppsBigNumState* pPrivate, + const IppsGFpECPoint* pPublic, + IppsECESState_SM2* pState, + IppsGFpECState* pEC, + Ipp8u* pEcScratchBuffer)) +IPPAPI(IppStatus, ippsGFpECESStart_SM2, (IppsECESState_SM2* pState)) +IPPAPI(IppStatus, ippsGFpECESEncrypt_SM2, (const Ipp8u* pInput, Ipp8u* pOutput, + int dataLen, IppsECESState_SM2* pState)) +IPPAPI(IppStatus, ippsGFpECESDecrypt_SM2, (const Ipp8u* pInput, Ipp8u* pOutput, + int dataLen, IppsECESState_SM2* pState)) +IPPAPI(IppStatus, ippsGFpECESFinal_SM2, (Ipp8u* pTag, int tagLen, IppsECESState_SM2* pState)) +IPPAPI(IppStatus, ippsGFpECESGetBuffersSize_SM2, (int* pPublicKeySize, + int* pMaximumTagSize, const IppsECESState_SM2* pState)) + +/* SM2 Encryption/Decryption Extended API */ +/* Encryption */ +IPPAPI(IppStatus, ippsGFpECEncryptSM2_Ext_EncMsgSize, (const IppsGFpECState *pEC, int ptMsgSize, int *pSize)) + +IPPAPI(IppStatus, ippsGFpECEncryptSM2_Ext, (Ipp8u *pOut, int maxOutLen, + int *pOutSize, + const Ipp8u *pInp, int inpLen, + const IppsGFpECPoint *pPublicKey, + IppsGFpECPoint *pEhpPublicKey, IppsBigNumState *pEphPrvKey, + IppsGFpECState *pEC, Ipp8u *pScratchBuffer)) + +/* Decryption */ +IPPAPI(IppStatus, ippsGFpECDecryptSM2_Ext_DecMsgSize, (const IppsGFpECState *pEC, int ctMsgSize, int *pSize)) + +IPPAPI(IppStatus, ippsGFpECDecryptSM2_Ext, (Ipp8u *pOut, int maxOutLen, + int *pOutSize, + const Ipp8u *pInp, int inpLen, + const IppsBigNumState *pPrvKey, + IppsGFpECState *pEC, Ipp8u *pScratchBuffer)) + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && !defined(__INTEL_LLVM_COMPILER) +#pragma warning(pop) +#endif +#ifdef __cplusplus +} +#endif + + +#endif /* IPPCP_H__ */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/include/ippcpdefs.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/include/ippcpdefs.h new file mode 100644 index 0000000..50417c6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/include/ippcpdefs.h @@ -0,0 +1,860 @@ +/******************************************************************************* +* Copyright (C) 2012 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) +// +// Purpose: Basic Types and Macro Definitions +// +*/ + + +#ifndef IPPBASE_H__ +#define IPPBASE_H__ + +#ifdef __cplusplus +extern "C" { +#endif +#if defined (_WIN64) +#define INTEL_PLATFORM "intel64/" +#elif defined (_WIN32) +#define INTEL_PLATFORM "ia32/" +#endif + +#if !defined( IPPAPI ) + + /* Specify explicit calling convention for public functions */ + #if defined( IPP_W32DLL ) && (defined( _WIN32 ) || defined( _WIN64 )) + #if defined( _MSC_VER ) || defined( __ICL ) + #define IPPAPI( type,name,arg ) \ + __declspec(dllimport) type IPP_CALL name arg; + #else + #define IPPAPI( type,name,arg ) type IPP_CALL name arg; + #endif + #else + #define IPPAPI( type,name,arg ) type IPP_CALL name arg; + #endif + +#endif + +/* icc 2021 supports short float data type, icx supports _Float16 data type */ +#define _FLOAT_16 2 +#define _SHORT_FLOAT 1 +#define _NO_FLOAT_16 0 +#if defined(__INTEL_LLVM_COMPILER) && defined(__AVX512FP16__) +# define COMPILER_SUPPORT_SHORT_FLOAT _FLOAT_16 +#else +# if defined(__INTEL_COMPILER) +# if(__INTEL_COMPILER >= 2021) +# define COMPILER_SUPPORT_SHORT_FLOAT _SHORT_FLOAT +# endif +# endif +#endif +#if !(defined(COMPILER_SUPPORT_SHORT_FLOAT)) +# define COMPILER_SUPPORT_SHORT_FLOAT _NO_FLOAT_16 +#endif + +#if !defined(_NO_IPP_DEPRECATED) + #if (defined( __ICL ) || defined( __ECL ) || defined(_MSC_VER)) && !defined( _PCS ) && !defined( _PCS_GENSTUBS ) + #if( __INTEL_COMPILER >= 1100 ) /* icl 11.0 supports additional comment */ + #if( _MSC_VER >= 1400 ) + #define IPP_DEPRECATED( comment ) __declspec( deprecated ( comment )) + #else + #pragma message ("your icl version supports additional comment for deprecated functions but it can't be displayed") + #pragma message ("because internal _MSC_VER macro variable setting requires compatibility with MSVC7.1") + #pragma message ("use -Qvc8 switch for icl command line to see these additional comments") + #define IPP_DEPRECATED( comment ) __declspec( deprecated ) + #endif + #elif( _MSC_FULL_VER >= 140050727 )&&( !defined( __INTEL_COMPILER ))&&( !defined(__INTEL_LLVM_COMPILER)) /* VS2005 supports additional comment */ + #define IPP_DEPRECATED( comment ) __declspec( deprecated ( comment )) + #elif( _MSC_VER <= 1200 )&&( !defined( __INTEL_COMPILER ))&&( !defined(__INTEL_LLVM_COMPILER)) /* VS 6 doesn't support deprecation */ + #define IPP_DEPRECATED( comment ) + #else + #define IPP_DEPRECATED( comment ) __declspec( deprecated ) + #endif + #elif (defined(__ICC) || defined(__ECC) || defined( __GNUC__ )) && !defined( _PCS ) && !defined( _PCS_GENSTUBS ) + #if defined( __GNUC__ ) + #if (__GNUC__ * 100 + __GNUC_MINOR__) >= 405 + #define IPP_DEPRECATED( message ) __attribute__(( deprecated( message ))) + #else + #define IPP_DEPRECATED( message ) __attribute__(( deprecated )) + #endif + #else + #define IPP_DEPRECATED( comment ) __attribute__(( deprecated )) + #endif + #else + #define IPP_DEPRECATED( comment ) + #endif +#else + #define IPP_DEPRECATED( comment ) +#endif + +#if (defined( __ICL ) || defined( __ECL ) || defined(_MSC_VER)) + #if !defined( IPP_NO_DEFAULT_LIB ) + #if ((defined( _IPP_SEQUENTIAL_DYNAMIC ) && !defined( _IPP_SEQUENTIAL_STATIC )) || \ + (!defined( _IPP_SEQUENTIAL_DYNAMIC ) && defined( _IPP_SEQUENTIAL_STATIC ))) + #elif (!defined( _IPP_SEQUENTIAL_DYNAMIC ) && !defined( _IPP_SEQUENTIAL_STATIC )) + #define IPP_NO_DEFAULT_LIB + #else + #error Illegal combination of _IPP_SEQUENTIAL_DYNAMIC/_IPP_SEQUENTIAL_STATIC, only one definition can be defined + #endif + #endif +#else + #define IPP_NO_DEFAULT_LIB + #if (defined(_IPP_SEQUENTIAL_DYNAMIC) || defined(_IPP_SEQUENTIAL_STATIC)) + #pragma message ("defines _IPP_SEQUENTIAL_DYNAMIC/_IPP_SEQUENTIAL_STATIC do not have any effect in current configuration") + #endif +#endif + + +#if defined (_MSC_VER) + #define IPP_CDECL __cdecl +#elif (defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || defined (__GNUC__ ) || defined (__clang__)) && defined (_ARCH_IA32) + #define IPP_CDECL __attribute((cdecl)) +#else + #define IPP_CDECL +#endif + +#if defined( _WIN32 ) || defined( _WIN64 ) + #define IPP_STDCALL __stdcall + #define IPP_CALL IPP_STDCALL + #define IPP_INT64 __int64 + #define IPP_UINT64 unsigned __int64 +#else + #define IPP_STDCALL + #define IPP_CALL IPP_CDECL + #define IPP_INT64 long long + #define IPP_UINT64 unsigned long long +#endif + +#define IPP_COUNT_OF( obj ) (sizeof(obj)/sizeof(obj[0])) + +#define IPP_PI ( 3.14159265358979323846 ) /* ANSI C does not support M_PI */ +#define IPP_2PI ( 6.28318530717958647692 ) /* 2*pi */ +#define IPP_PI2 ( 1.57079632679489661923 ) /* pi/2 */ +#define IPP_PI4 ( 0.78539816339744830961 ) /* pi/4 */ +#define IPP_PI180 ( 0.01745329251994329577 ) /* pi/180 */ +#define IPP_RPI ( 0.31830988618379067154 ) /* 1/pi */ +#define IPP_SQRT2 ( 1.41421356237309504880 ) /* sqrt(2) */ +#define IPP_SQRT3 ( 1.73205080756887729353 ) /* sqrt(3) */ +#define IPP_LN2 ( 0.69314718055994530942 ) /* ln(2) */ +#define IPP_LN3 ( 1.09861228866810969139 ) /* ln(3) */ +#define IPP_E ( 2.71828182845904523536 ) /* e */ +#define IPP_RE ( 0.36787944117144232159 ) /* 1/e */ +#define IPP_EPS23 ( 1.19209289e-07f ) +#define IPP_EPS52 ( 2.2204460492503131e-016 ) + +#define IPP_MAX_8U ( 0xFF ) +#define IPP_MAX_16U ( 0xFFFF ) +#define IPP_MAX_32U ( 0xFFFFFFFF ) +#define IPP_MIN_8U ( 0 ) +#define IPP_MIN_16U ( 0 ) +#define IPP_MIN_32U ( 0 ) +#define IPP_MIN_8S (-128 ) +#define IPP_MAX_8S ( 127 ) +#define IPP_MIN_16S (-32768 ) +#define IPP_MAX_16S ( 32767 ) +#define IPP_MIN_32S (-2147483647 - 1 ) +#define IPP_MAX_32S ( 2147483647 ) +#define IPP_MIN_64U ( 0 ) + +#if defined( _WIN32 ) || defined ( _WIN64 ) + #define IPP_MAX_64S ( 9223372036854775807i64 ) + #define IPP_MIN_64S (-9223372036854775807i64 - 1 ) + #define IPP_MAX_64U ( 0xffffffffffffffffL ) /* 18446744073709551615 */ +#else + #define IPP_MAX_64S ( 9223372036854775807LL ) + #define IPP_MIN_64S (-9223372036854775807LL - 1 ) + #define IPP_MAX_64U ( 0xffffffffffffffffLL ) /* 18446744073709551615 */ +#endif + +#define IPP_MINABS_32F ( 1.175494351e-38f ) +#define IPP_MAXABS_32F ( 3.402823466e+38f ) +#define IPP_EPS_32F ( 1.192092890e-07f ) +#define IPP_MINABS_64F ( 2.2250738585072014e-308 ) +#define IPP_MAXABS_64F ( 1.7976931348623158e+308 ) +#define IPP_EPS_64F ( 2.2204460492503131e-016 ) + +#define IPP_MAX( a, b ) ( ((a) > (b)) ? (a) : (b) ) +#define IPP_MIN( a, b ) ( ((a) < (b)) ? (a) : (b) ) + +#define IPP_ABS( a ) ( ((a) < 0) ? (-(a)) : (a) ) + +typedef struct { + int major; /* e.g. 1 */ + int minor; /* e.g. 2 */ + int majorBuild; /* e.g. 3 */ + unsigned int revision; /* e.g. 0xf6f5e5bc */ + char targetCpu[4]; /* corresponding to Intel® processor */ + const char* Name; /* e.g. "ippsw7" */ + const char* Version; /* e.g. "v1.2 Beta" */ + const char* BuildDate; /* e.g. "Jul 20 99" */ +} IppLibraryVersion; + +typedef unsigned char Ipp8u; +typedef unsigned short Ipp16u; +typedef unsigned int Ipp32u; +typedef signed char Ipp8s; +typedef signed short Ipp16s; +typedef signed int Ipp32s; +typedef float Ipp32f; +typedef IPP_INT64 Ipp64s; +typedef IPP_UINT64 Ipp64u; +typedef double Ipp64f; + +#if (COMPILER_SUPPORT_SHORT_FLOAT == _FLOAT_16) + typedef _Float16 Ipp16f; +#endif +#if (COMPILER_SUPPORT_SHORT_FLOAT == _SHORT_FLOAT) + typedef short float Ipp16f; +#endif +#if (COMPILER_SUPPORT_SHORT_FLOAT == _NO_FLOAT_16) + typedef Ipp16s Ipp16f; +#endif + +typedef struct { + Ipp8s re; + Ipp8s im; +} Ipp8sc; + +typedef struct { + Ipp16s re; + Ipp16s im; +} Ipp16sc; + +typedef struct { + Ipp16u re; + Ipp16u im; +} Ipp16uc; + +typedef struct { + Ipp32s re; + Ipp32s im; +} Ipp32sc; + +typedef struct { + Ipp32f re; + Ipp32f im; +} Ipp32fc; + +typedef struct { + Ipp64s re; + Ipp64s im; +} Ipp64sc; + +typedef struct { + Ipp64f re; + Ipp64f im; +} Ipp64fc; + +typedef struct { + Ipp16f re; + Ipp16f im; +} Ipp16fc; + +typedef enum { + ippUndef = -1, + ipp1u = 0, + ipp8u = 1, + ipp8uc = 2, + ipp8s = 3, + ipp8sc = 4, + ipp16u = 5, + ipp16uc = 6, + ipp16s = 7, + ipp16sc = 8, + ipp32u = 9, + ipp32uc = 10, + ipp32s = 11, + ipp32sc = 12, + ipp32f = 13, + ipp32fc = 14, + ipp64u = 15, + ipp64uc = 16, + ipp64s = 17, + ipp64sc = 18, + ipp64f = 19, + ipp64fc = 20, + ipp16fc = 21 /* This is necessary for TS */ +} IppDataType; + +typedef enum { + ippFalse = 0, + ippTrue = 1 +} IppBool; + +#ifdef __cplusplus +} +#endif + +#endif /* IPPBASE_H__ */ + +#ifndef IPP_CPU_FEATURES__ +#define IPP_CPU_FEATURES__ + +#define ippCPUID_MMX 0x00000001 /* Intel® architecture with MMX(TM) technology supported */ +#define ippCPUID_SSE 0x00000002 /* Intel® Streaming SIMD Extensions (Intel® SSE) instruction set */ +#define ippCPUID_SSE2 0x00000004 /* Intel® Streaming SIMD Extensions 2 (Intel® SSE2) instruction set */ +#define ippCPUID_SSE3 0x00000008 /* Intel® Streaming SIMD Extensions 3 (Intel® SSE3) instruction set */ +#define ippCPUID_SSSE3 0x00000010 /* Supplemental Streaming SIMD Extensions 3 (SSSE3) instruction set */ +#define ippCPUID_MOVBE 0x00000020 /* Intel® instruction MOVBE */ +#define ippCPUID_SSE41 0x00000040 /* Intel® Streaming SIMD Extensions 4.1 (Intel® SSE4.1) instruction set */ +#define ippCPUID_SSE42 0x00000080 /* Intel® Streaming SIMD Extensions 4.2 (Intel® SSE4.2) instruction set */ +#define ippCPUID_AVX 0x00000100 /* Intel® Advanced Vector Extensions instruction set */ +#define ippAVX_ENABLEDBYOS 0x00000200 /* Intel® Advanced Vector Extensions instruction set is supported by OS */ +#define ippCPUID_AES 0x00000400 /* */ +#define ippCPUID_CLMUL 0x00000800 /* Intel® instruction PCLMULQDQ */ +#define ippCPUID_ABR 0x00001000 /* Reserved */ +#define ippCPUID_RDRAND 0x00002000 /* Intel® instruction RDRAND */ +#define ippCPUID_F16C 0x00004000 /* Intel® instruction F16C */ +#define ippCPUID_AVX2 0x00008000 /* Intel® Advanced Vector Extensions 2 */ +#define ippCPUID_ADCOX 0x00010000 /* Intel® instructions ADOX/ADCX */ +#define ippCPUID_RDSEED 0x00020000 /* Intel® instruction RDSEED */ +#define ippCPUID_PREFETCHW 0x00040000 /* Intel® instruction PREFETCHW */ +#define ippCPUID_SHA 0x00080000 /* Intel® Secure Hash Algorithm Extensions */ +#define ippCPUID_AVX512F 0x00100000 /* Intel® Advanced Vector Extensions 512 Foundation instruction set */ +#define ippCPUID_AVX512CD 0x00200000 /* Intel® Advanced Vector Extensions 512 CD instruction set */ +#define ippCPUID_AVX512ER 0x00400000 /* Intel® Advanced Vector Extensions 512 ER instruction set */ +#define ippCPUID_AVX512PF 0x00800000 /* Intel® Advanced Vector Extensions 512 PF instruction set */ +#define ippCPUID_AVX512BW 0x01000000 /* Intel® Advanced Vector Extensions 512 BW instruction set */ +#define ippCPUID_AVX512DQ 0x02000000 /* Intel® Advanced Vector Extensions 512 DQ instruction set */ +#define ippCPUID_AVX512VL 0x04000000 /* Intel® Advanced Vector Extensions 512 VL instruction set */ +#define ippCPUID_AVX512VBMI 0x08000000 /* Intel® Advanced Vector Extensions 512 Bit Manipulation instructions */ +#define ippCPUID_MPX 0x10000000 /* Intel® Memory Protection Extensions */ +#define ippCPUID_AVX512_4FMADDPS 0x20000000 /* Intel® Advanced Vector Extensions 512 DL floating-point single precision */ +#define ippCPUID_AVX512_4VNNIW 0x40000000 /* Intel® Advanced Vector Extensions 512 DL enhanced word variable precision */ +#define ippCPUID_KNC 0x80000000 /* Intel® Xeon® Phi(TM) Coprocessor */ +#if defined( _WIN32 ) || defined ( _WIN64 ) + #define INT64_SUFFIX(name) name##L +#else + #define INT64_SUFFIX(name) name##LL +#endif + #define ippCPUID_AVX512IFMA INT64_SUFFIX(0x100000000) /* Intel® Advanced Vector Extensions 512 IFMA (PMADD52) instruction set */ + #define ippCPUID_NOCHECK INT64_SUFFIX(0x8000000000000000) /* Force ippSetCpuFeatures to set CPU features without check */ + #define ippCPUID_GETINFO_A INT64_SUFFIX(0x616f666e69746567) /* Force ippGetCpuFeatures to work as cpuid instruction */ + #define ippAVX512_ENABLEDBYOS INT64_SUFFIX(0x200000000) /* Intel® Advanced Vector Extensions 512 is supported by OS */ + + #define ippCPUID_AVX512GFNI INT64_SUFFIX(0x400000000) /* */ + #define ippCPUID_AVX512VAES INT64_SUFFIX(0x800000000) /* */ + #define ippCPUID_AVX512VCLMUL INT64_SUFFIX(0x1000000000) /* */ + #define ippCPUID_AVX512VBMI2 INT64_SUFFIX(0x2000000000) /* Intel® Advanced Vector Extensions 512 Bit Manipulation instructions 2 */ + #define ippCPUID_AVX512_FP16 INT64_SUFFIX(0x1000000000) /* Intel(R) Advanced Vector Extensions 512 16-bit floating point (FP16) instruction set */ + + #define ippCPUID_AVX2VAES INT64_SUFFIX(0x4000000000) /* Intel® Advanced Vector Extensions 256 Bit Vector AES instructions */ + #define ippCPUID_AVX2VCLMUL INT64_SUFFIX(0x8000000000) /* Intel® instruction VPCLMULQDQ */ + +#endif /* IPP_CPU_FEATURES__ */ + +/* Macros are necessary to build custom Intel® IPP Cryptography static 1cpu library (enable specific features at compile-time) */ +#if (!defined(_MERGED_BLD) && defined(IPPCP_CUSTOM_BUILD)) + +#ifndef IPP_CUSTOM_CPU_FEATURES__ +#define IPP_CUSTOM_CPU_FEATURES__ + +#ifndef IPPCP_AES_ON +#define IPPCP_AES_ON (0) +#endif +#ifndef IPPCP_CLMUL_ON +#define IPPCP_CLMUL_ON (0) +#endif +#ifndef IPPCP_VAES_ON +#define IPPCP_VAES_ON (0) +#endif +#ifndef IPPCP_VCLMUL_ON +#define IPPCP_VCLMUL_ON (0) +#endif +#define IPP_CUSTOM_ENABLED_FEATURES (ippCPUID_AES*IPPCP_AES_ON | ippCPUID_CLMUL*IPPCP_CLMUL_ON | ippCPUID_AVX512VAES*IPPCP_VAES_ON | ippCPUID_AVX512VCLMUL*IPPCP_VCLMUL_ON) + +#endif /* IPP_CUSTOM_CPU_FEATURES__ */ +#endif /* !defined(_MERGED_BLD) && defined(IPPCP_CUSTOM_BUILD) */ + +#ifndef IPPSTATUS_H__ +#define IPPSTATUS_H__ + +#ifdef __cplusplus +extern "C" { +#endif +typedef signed int IppStatus; + + /* start of common with ippCrypto part - any changes MUST be done in both repositories - IPP & ippCrypto */ +#define ippStsCpuNotSupportedErr -9999 /* The target CPU is not supported. */ +#define ippStsUnknownStatusCodeErr -216 /* Unknown status code. */ +#define ippStsLoadDynErr -221 /* Error when loading the dynamic library. */ +#define ippStsLengthErr -15 /* Incorrect value for string length. */ +#define ippStsNotSupportedModeErr -14 /* The requested mode is currently not supported. */ +#define ippStsContextMatchErr -13 /* Context parameter does not match the operation. */ +#define ippStsScaleRangeErr -12 /* Scale bounds are out of range. */ +#define ippStsOutOfRangeErr -11 /* Argument is out of range, or point is outside the image. */ +#define ippStsDivByZeroErr -10 /* An attempt to divide by zero. */ +#define ippStsMemAllocErr -9 /* Memory allocated for the operation is not enough. */ +#define ippStsNullPtrErr -8 /* Null pointer error. */ +#define ippStsRangeErr -7 /* Incorrect values for bounds: the lower bound is greater than the upper bound. */ +#define ippStsSizeErr -6 /* Incorrect value for data size. */ +#define ippStsBadArgErr -5 /* Incorrect arg/param of the function. */ +#define ippStsNoMemErr -4 /* Not enough memory for the operation. */ +#define ippStsErr -2 /* Unknown/unspecified error */ + /* no errors */ +#define ippStsNoErr 0 /* No errors. */ + /* warnings */ +#define ippStsNoOperation 1 /* No operation has been executed. */ +#define ippStsDivByZero 2 /* Zero value(s) for the divisor in the Div function. */ +#define ippStsWaterfall 43 /* Cannot load required library, waterfall is used. */ +#define ippStsFeaturesCombination 51 /* Wrong combination of features. */ + /* end of common with ippCrypto part */ + +#ifdef __cplusplus +} +#endif + +#endif /* IPPSTATUS_H__ */ + + /* ippCrypto specific statuses - any changes MUST be done in both repositories - IPP & ippCrypto */ +#define ippStsInvalidPoint -1017 /* ippStsInvalidPoint ECC: Invalid point (out of EC).*/ +#define ippStsQuadraticNonResidueErr -1016 /* SQRT operation on quadratic non-residue value. */ +#define ippStsPointAtInfinity -1015 /* Point at infinity is detected. */ +#define ippStsOFBSizeErr -1014 /* Incorrect value for crypto OFB block size. */ +#define ippStsIncompleteContextErr -1013 /* Crypto: set up of context is not complete. */ +#define ippStsCTRSizeErr -1012 /* Incorrect value for crypto CTR block size. */ +#define ippStsEphemeralKeyErr -1011 /* ECC: Invalid ephemeral key. */ +#define ippStsMessageErr -1010 /* ECC: Invalid message digest. */ +#define ippStsShareKeyErr -1009 /* ECC: Invalid share key. */ +#define ippStsInvalidPrivateKey -1008 /* ECC: Invalid private key. */ +#define ippStsOutOfECErr -1007 /* ECC: Point out of EC. */ +#define ippStsECCInvalidFlagErr -1006 /* ECC: Invalid Flag. */ +#define ippStsUnderRunErr -1005 /* Error in data under run. */ +#define ippStsPaddingErr -1004 /* Detected padding error indicates the possible data corruption. */ +#define ippStsCFBSizeErr -1003 /* Incorrect value for crypto CFB block size. */ +#define ippStsPaddingSchemeErr -1002 /* Invalid padding scheme. */ +#define ippStsBadModulusErr -1001 /* Bad modulus caused a failure in module inversion. */ +#define ippStsInsufficientEntropy 25 /* Generation of the prime/key failed due to insufficient entropy in the random seed and stimulus bit string. */ +#define ippStsNotSupportedCpu 36 /* The CPU is not supported. */ +#define ippStsMbWarning 53 /* Error(s) in statuses array. */ + /* end of ippCrypto specific statuses - any changes MUST be done in both repositories - IPP & ippCrypto */ + +#if (!defined IPPCPDEFS_H__) || defined( _OWN_BLDPCS ) +#define IPPCPDEFS_H__ + +#ifdef __cplusplus +extern "C" { +#endif + + + +#if !defined( _OWN_BLDPCS ) + +typedef Ipp32u IppAlgId; + +/* +// ========================================================= +// Symmetric Ciphers +// ========================================================= +*/ +typedef enum { + ippPaddingNONE = 0, /*NONE = 0,*/ IppsCPPaddingNONE = 0, + ippPaddingPKCS7 = 1, /*PKCS7 = 1,*/ IppsCPPaddingPKCS7 = 1, + ippPaddingZEROS = 2, /*ZEROS = 2,*/ IppsCPPaddingZEROS = 2 +} IppsPadding, IppsCPPadding; + +typedef struct _cpDES IppsDESSpec; +typedef struct _cpRijndael128 IppsAESSpec; +typedef struct _cpRijndael128 IppsRijndael128Spec; +typedef struct _cpSMS4 IppsSMS4Spec; + +/* TDES */ +#define DES_BLOCKSIZE (64) /* cipher blocksize (bits) */ +#define TDES_BLOCKSIZE DES_BLOCKSIZE + +#define DES_KEYSIZE (64) /* cipher keysize (bits) */ +#define TDES_KEYSIZE DES_KEYSIZE + +/* AES */ +#define IPP_AES_BLOCK_BITSIZE (128) /* cipher blocksizes (bits) */ + +/* Rijndael */ +typedef enum { + ippRijndaelKey128 = 128, IppsRijndaelKey128 = 128, /* 128-bit key */ + ippRijndaelKey192 = 192, IppsRijndaelKey192 = 192, /* 192-bit key */ + ippRijndaelKey256 = 256, IppsRijndaelKey256 = 256 /* 256-bit key */ +} IppsRijndaelKeyLength; + +/* AES-CCM (authentication & confidence) */ +typedef struct _cpAES_CCM IppsAES_CCMState; +/* AES-GCM (authentication & confidence) */ +typedef struct _cpAES_GCM IppsAES_GCMState; +/* AES-XTS (confidence) */ +typedef struct _cpAES_XTS IppsAES_XTSSpec; + +/* SMS4-CCM (authentication & confidence) */ +typedef struct _cpSMS4_CCM IppsSMS4_CCMState; + +/* +// ========================================================= +// ARCFOUR Stream Cipher +// ========================================================= +*/ +typedef struct _cpARCfour IppsARCFourState; + +#define IPP_ARCFOUR_KEYMAX_SIZE (256) /* max key length (bytes) */ +#define MAX_ARCFOUR_KEY_LEN IPP_ARCFOUR_KEYMAX_SIZE /* obsolete */ + +/* +// ========================================================= +// One-Way Hash Functions +// ========================================================= +*/ +typedef enum { + ippHashAlg_Unknown, + ippHashAlg_SHA1, + ippHashAlg_SHA256, + ippHashAlg_SHA224, + ippHashAlg_SHA512, + ippHashAlg_SHA384, + ippHashAlg_MD5, + ippHashAlg_SM3, + ippHashAlg_SHA512_224, + ippHashAlg_SHA512_256, + ippHashAlg_MaxNo +} IppHashAlgId; + +#define IPP_ALG_HASH_UNKNOWN (ippHashAlg_Unknown) /* unknown */ +#define IPP_ALG_HASH_SHA1 (ippHashAlg_SHA1) /* SHA1 */ +#define IPP_ALG_HASH_SHA256 (ippHashAlg_SHA256) /* SHA256 */ +#define IPP_ALG_HASH_SHA224 (ippHashAlg_SHA224) /* SHA224 or SHA256/224 */ +#define IPP_ALG_HASH_SHA512 (ippHashAlg_SHA512) /* SHA512 */ +#define IPP_ALG_HASH_SHA384 (ippHashAlg_SHA384) /* SHA384 or SHA512/384 */ +#define IPP_ALG_HASH_MD5 (ippHashAlg_MD5) /* MD5 */ +#define IPP_ALG_HASH_SM3 (ippHashAlg_SM3) /* SM3 */ +#define IPP_ALG_HASH_SHA512_224 (ippHashAlg_SHA512_224) /* SHA512/224 */ +#define IPP_ALG_HASH_SHA512_256 (ippHashAlg_SHA512_256) /* SHA512/256 */ +#define IPP_ALG_HASH_LIMIT (ippHashAlg_MaxNo) /* hash alg limiter*/ + +typedef struct _cpSHA1 IppsSHA1State; +typedef struct _cpSHA256 IppsSHA256State; +typedef struct _cpSHA256 IppsSHA224State; +typedef struct _cpSHA512 IppsSHA512State; +typedef struct _cpSHA512 IppsSHA384State; +typedef struct _cpMD5 IppsMD5State; +typedef struct _cpSM3 IppsSM3State; +typedef struct _cpHashCtx IppsHashState; + +typedef struct _cpHashMethod_rmf IppsHashMethod; +typedef struct _cpHashCtx_rmf IppsHashState_rmf; + +#define IPP_SHA1_DIGEST_BITSIZE 160 /* digest size (bits) */ +#define IPP_SHA256_DIGEST_BITSIZE 256 +#define IPP_SHA224_DIGEST_BITSIZE 224 +#define IPP_SHA384_DIGEST_BITSIZE 384 +#define IPP_SHA512_DIGEST_BITSIZE 512 +#define IPP_MD5_DIGEST_BITSIZE 128 +#define IPP_SM3_DIGEST_BITSIZE 256 +#define IPP_SHA512_224_DIGEST_BITSIZE 224 +#define IPP_SHA512_256_DIGEST_BITSIZE 256 + +/* +// ========================================================= +// Keyed-Hash Message Authentication Codes +// ========================================================= +*/ +typedef struct _cpHMAC IppsHMACState; +typedef struct _cpHMAC IppsHMACSHA1State; +typedef struct _cpHMAC IppsHMACSHA256State; +typedef struct _cpHMAC IppsHMACSHA224State; +typedef struct _cpHMAC IppsHMACSHA384State; +typedef struct _cpHMAC IppsHMACSHA512State; +typedef struct _cpHMAC IppsHMACMD5State; +typedef struct _cpHMAC_rmf IppsHMACState_rmf; + +/* +// ========================================================= +// Data Authentication Codes +// ========================================================= +*/ +typedef struct _cpAES_CMAC IppsAES_CMACState; + +/* +// ========================================================= +// Big Number Integer Arithmetic +// ========================================================= +*/ +#define BN_MAXBITSIZE (16*1024) /* bn max size (bits) */ + + +typedef enum { + ippBigNumNEG = 0, IppsBigNumNEG = 0, + ippBigNumPOS = 1, IppsBigNumPOS = 1 +} IppsBigNumSGN; + +typedef enum { + ippBinaryMethod = 0, IppsBinaryMethod = 0, + ippSlidingWindows = 1, IppsSlidingWindows = 1 +} IppsExpMethod; + +typedef struct _cpBigNum IppsBigNumState; +typedef struct _cpMontgomery IppsMontState; +typedef struct _cpPRNG IppsPRNGState; +typedef struct _cpPrime IppsPrimeState; + +/* External Bit Supplier */ +typedef IppStatus (IPP_CALL *IppBitSupplier)(Ipp32u* pRand, int nBits, void* pEbsParams); + +#define IPP_IS_EQ (0) +#define IPP_IS_GT (1) +#define IPP_IS_LT (2) +#define IPP_IS_NE (3) +#define IPP_IS_NA (4) + +#define IPP_IS_PRIME (5) +#define IPP_IS_COMPOSITE (6) + +#define IPP_IS_VALID (7) +#define IPP_IS_INVALID (8) +#define IPP_IS_INCOMPLETE (9) +#define IPP_IS_ATINFINITY (10) + +#define IS_ZERO IPP_IS_EQ +#define GREATER_THAN_ZERO IPP_IS_GT +#define LESS_THAN_ZERO IPP_IS_LT +#define IS_PRIME IPP_IS_PRIME +#define IS_COMPOSITE IPP_IS_COMPOSITE +#define IS_VALID_KEY IPP_IS_VALID +#define IS_INVALID_KEY IPP_IS_INVALID +#define IS_INCOMPLETED_KEY IPP_IS_INCOMPLETE + +/* +// ========================================================= +// RSA Cryptography +// ========================================================= +*/ +#define MIN_RSA_SIZE (8) +#define MAX_RSA_SIZE (16*1024) + +typedef struct _cpRSA IppsRSAState; + +/* key types */ +typedef enum { + ippRSApublic = 0x20000000, IppRSApublic = 0x20000000, + ippRSAprivate = 0x40000000, IppRSAprivate = 0x40000000 +} IppRSAKeyType; + +/* key component's tag */ +typedef enum { + ippRSAkeyN = 0x01, IppRSAkeyN = 0x01, + ippRSAkeyE = 0x02, IppRSAkeyE = 0x02, + ippRSAkeyD = 0x04, IppRSAkeyD = 0x04, + ippRSAkeyP = 0x08, IppRSAkeyP = 0x08, + ippRSAkeyQ = 0x10, IppRSAkeyQ = 0x10, + ippRSAkeyDp = 0x20, IppRSAkeyDp = 0x20, + ippRSAkeyDq = 0x40, IppRSAkeyDq = 0x40, + ippRSAkeyQinv = 0x80, IppRSAkeyQinv = 0x80 +} IppRSAKeyTag; + +typedef struct _cpRSA_public_key IppsRSAPublicKeyState; +typedef struct _cpRSA_private_key IppsRSAPrivateKeyState; + + +/* +// ========================================================= +// DL Cryptography +// ========================================================= +*/ +#define MIN_DLP_BITSIZE (512) +#define MIN_DLP_BITSIZER (160) + +#define MIN_DLPDH_BITSIZE (512) +#define MIN_DLPDH_BITSIZER (160) +#define DEF_DLPDH_BITSIZER (160) + +#define MIN_DLPDSA_BITSIZE (512) +#define MAX_DLPDSA_BITSIZE (1024) +#define MIN_DLPDSA_BITSIZER (160) +#define DEF_DLPDSA_BITSIZER (160) +#define MAX_DLPDSA_BITSIZER (160) +#define MIN_DLPDSA_SEEDSIZE (160) + +typedef struct _cpDLP IppsDLPState; + +/* domain parameter tags */ +typedef enum { + ippDLPkeyP = 0x01, IppDLPkeyP = 0x01, + ippDLPkeyR = 0x02, IppDLPkeyR = 0x02, + ippDLPkeyG = 0x04, IppDLPkeyG = 0x04 +} IppDLPKeyTag; + +typedef enum { + ippDLValid, /* validation pass successfully */ + + ippDLBaseIsEven, /* !(P is odd) */ + ippDLOrderIsEven, /* !(R is odd) */ + ippDLInvalidBaseRange, /* !(2^(L-1) < P < 2^L) */ + ippDLInvalidOrderRange, /* !(2^(M-1) < R < 2^M) */ + ippDLCompositeBase, + ippDLCompositeOrder, + ippDLInvalidCofactor, /* !( R|(P-1) ) */ + ippDLInvalidGenerator, /* !( G^R == 1 (mod P) ) */ + /* !(1 < G < (P-1)) */ + ippDLInvalidPrivateKey, /* !(1 < private < (R-1)) */ + ippDLInvalidPublicKey, /* !(1 < public <=(P-1)) */ + ippDLInvalidKeyPair, /* !(G^private == public */ + + ippDLInvalidSignature /* invalid signature */ +} IppDLResult; + +/* +// ========================================================= +// EC Cryptography +// ========================================================= +*/ +#define EC_GFP_MAXBITSIZE (1024) + +/* operation result */ +typedef enum { + ippECValid, /* validation pass successfully */ + + ippECCompositeBase, /* field based on composite */ + ippECComplicatedBase, /* number of non-zero terms in the polynomial (> PRIME_ARR_MAX) */ + ippECIsZeroDiscriminant,/* zero discriminant */ + ippECCompositeOrder, /* composite order of base point */ + ippECInvalidOrder, /* invalid base point order */ + ippECIsWeakMOV, /* weak Meneze-Okamoto-Vanstone reduction attack */ + ippECIsWeakSSSA, /* weak Semaev-Smart,Satoh-Araki reduction attack */ + ippECIsSupersingular, /* supersingular curve */ + + ippECInvalidPrivateKey, /* !(0 < Private < order) */ + ippECInvalidPublicKey, /* (order*PublicKey != Infinity) */ + ippECInvalidKeyPair, /* (Private*BasePoint != PublicKey) */ + + ippECPointOutOfGroup, /* out of group (order*P != Infinity) */ + ippECPointIsAtInfinite, /* point (P=(Px,Py)) at Infinity */ + ippECPointIsNotValid, /* point (P=(Px,Py)) out-of EC */ + + ippECPointIsEqual, /* compared points are equal */ + ippECPointIsNotEqual, /* compared points are different */ + + ippECInvalidSignature /* invalid signature */ +} IppECResult; + +/* domain parameter set/get flags */ +typedef enum { + ippECarbitrary =0x00000, IppECCArbitrary = 0x00000, /* arbitrary ECC */ + + ippECPstd = 0x10000, IppECCPStd = 0x10000, /* random (recommended) EC over FG(p): */ + ippECPstd112r1 = ippECPstd, IppECCPStd112r1 = IppECCPStd, /* secp112r1 curve */ + ippECPstd112r2 = ippECPstd+1, IppECCPStd112r2 = IppECCPStd+1, /* secp112r2 curve */ + ippECPstd128r1 = ippECPstd+2, IppECCPStd128r1 = IppECCPStd+2, /* secp128r1 curve */ + ippECPstd128r2 = ippECPstd+3, IppECCPStd128r2 = IppECCPStd+3, /* secp128r2 curve */ + ippECPstd160r1 = ippECPstd+4, IppECCPStd160r1 = IppECCPStd+4, /* secp160r1 curve */ + ippECPstd160r2 = ippECPstd+5, IppECCPStd160r2 = IppECCPStd+5, /* secp160r2 curve */ + ippECPstd192r1 = ippECPstd+6, IppECCPStd192r1 = IppECCPStd+6, /* secp192r1 curve */ + ippECPstd224r1 = ippECPstd+7, IppECCPStd224r1 = IppECCPStd+7, /* secp224r1 curve */ + ippECPstd256r1 = ippECPstd+8, IppECCPStd256r1 = IppECCPStd+8, /* secp256r1 curve */ + ippECPstd384r1 = ippECPstd+9, IppECCPStd384r1 = IppECCPStd+9, /* secp384r1 curve */ + ippECPstd521r1 = ippECPstd+10, IppECCPStd521r1 = IppECCPStd+10, /* secp521r1 curve */ + ippECPstdSM2 = ippECPstd+11, IppECCPStdSM2 = IppECCPStd+11, /* TMP SM2 curve */ + ippEC_TPM_SM2_P256= ippECPstd+11, + ippEC_TPM_BN_P256 = ippECPstd+12, /* TPM BN_P256 curve */ + + /* curves over binary finit fields are not supported in Intel® IPP 9.0 */ + IppECCBStd = 0x20000, /* random (recommended) EC over FG(2^m): */ + IppECCBStd113r1 = IppECCBStd, /* sect113r1 curve */ + IppECCBStd113r2 = IppECCBStd+1, /* sect113r2 curve */ + IppECCBStd131r1 = IppECCBStd+2, /* sect131r1 curve */ + IppECCBStd131r2 = IppECCBStd+3, /* sect131r2 curve */ + IppECCBStd163r1 = IppECCBStd+4, /* sect163r1 curve */ + IppECCBStd163r2 = IppECCBStd+5, /* sect163r2 curve */ + IppECCBStd193r1 = IppECCBStd+6, /* sect193r1 curve */ + IppECCBStd193r2 = IppECCBStd+7, /* sect193r2 curve */ + IppECCBStd233r1 = IppECCBStd+8, /* sect233r1 curve */ + IppECCBStd283r1 = IppECCBStd+9, /* sect283r1 curve */ + IppECCBStd409r1 = IppECCBStd+10, /* sect409r1 curve */ + IppECCBStd571r1 = IppECCBStd+11, /* sect571r1 curve */ + + IppECCKStd = 0x40000, /* Koblitz (recommended) EC over FG(2^m): */ + IppECCBStd163k1 = IppECCKStd, /* Koblitz 163 curve */ + IppECCBStd233k1 = IppECCKStd+1, /* Koblitz 233 curve */ + IppECCBStd239k1 = IppECCKStd+2, /* Koblitz 239 curve */ + IppECCBStd283k1 = IppECCKStd+3, /* Koblitz 283 curve */ + IppECCBStd409k1 = IppECCKStd+4, /* Koblitz 409 curve */ + IppECCBStd571k1 = IppECCKStd+5 /* Koblitz 571 curve */ +} IppsECType, IppECCType; + +/* +// GF over prime and its extension +*/ +#define IPP_MIN_GF_CHAR (3) /* min characteristic of GF */ + +#define IPP_MIN_GF_BITSIZE (2) /* min bitsize of element over prime GF */ +#define IPP_MAX_GF_BITSIZE (1024) /* max bitsize of element over prime GF */ + +#define IPP_MIN_GF_EXTDEG (2) /* min GF extension degree */ +#define IPP_MAX_GF_EXTDEG (8) /* max GF extension degree */ + +#define IPP_MAX_EXPONENT_NUM (6) /* max number of exponents, equals to LOG_CACHE_LINE_SIZE */ + +typedef struct _cpGFpMethod IppsGFpMethod; + +typedef struct _cpGFp IppsGFpState; +typedef struct _cpGFpElement IppsGFpElement; + +typedef struct _cpGFpEC IppsGFpECState; +typedef struct _cpGFpECPoint IppsGFpECPoint; + +typedef struct _cpGFpEC IppsECCPState; +typedef struct _cpGFpECPoint IppsECCPPointState; + +typedef struct { + int hashSize; + int msgBlockSize; +} IppsHashInfo; + +typedef struct { + //const IppsGFpState* pBasicGF; + //const IppsGFpState* pGroundGF; + int parentGFdegree; + int basicGFdegree; + int basicElmBitSize; +} IppsGFpInfo; + +/* SM3 Digest Bytes Size */ +#define IPP_SM3_DIGEST_BYTESIZE ((IPP_SM3_DIGEST_BITSIZE + 7) / 8) + +typedef struct _cpStateECES_SM2 IppsECESState_SM2; + +typedef enum { + ippKESM2Requester = 0xF, /* corresponds to A user/participant */ + ippKESM2Responder /* corresponds to B user/participant */ +} IppsKeyExchangeRoleSM2; + +typedef struct _GFpECKeyExchangeSM2 IppsGFpECKeyExchangeSM2State; + +#endif /* !defined( _OWN_BLDPCS ) */ + +IPPAPI( IppStatus, ippcpGetCpuFeatures, ( Ipp64u* pFeaturesMask )) +IPPAPI( IppStatus, ippcpSetCpuFeatures, ( Ipp64u features )) +IPPAPI( Ipp64u, ippcpGetEnabledCpuFeatures, ( void ) ) +IPPAPI( IppStatus, ippcpSetNumThreads, ( int numThr )) +IPPAPI( IppStatus, ippcpInit,( void )) +IPPAPI( IppStatus, ippcpGetNumThreads, (int* pNumThr) ) +IPPAPI( const char*, ippcpGetStatusString, ( IppStatus StsCode )) +IPPAPI( int, ippcpGetEnabledNumThreads, ( void ) ) +IPPAPI( Ipp64u, ippcpGetCpuClocks, (void) ) + +#ifdef __cplusplus +} +#endif + +#endif /* !defined IPPCPDEFS_H__ || defined( _OWN_BLDPCS ) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/include/ippversion.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/include/ippversion.h new file mode 100644 index 0000000..7137614 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/include/ippversion.h @@ -0,0 +1,41 @@ +/******************************************************************************* +* Copyright (C) 2001 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) +// +// Purpose: Describes the Intel IPP Cryptography version +// +*/ + + +#if !defined( IPPVERSION_H__ ) +#define IPPVERSION_H__ + +#define IPP_VERSION_MAJOR 2021 +#define IPP_VERSION_MINOR 9 +#define IPP_VERSION_UPDATE 0 + +// Major interface version +#define IPP_INTERFACE_VERSION_MAJOR 11 +// Minor interface version +#define IPP_INTERFACE_VERSION_MINOR 9 + +#define IPP_VERSION_STR STR(IPP_VERSION_MAJOR) "." STR(IPP_VERSION_MINOR) "." STR(IPP_VERSION_UPDATE) " (" STR(IPP_INTERFACE_VERSION_MAJOR) "." STR(IPP_INTERFACE_VERSION_MINOR) " )" + +#endif /* IPPVERSION_H__ */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/CMakeASM_NASMOptions.txt b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/CMakeASM_NASMOptions.txt new file mode 100644 index 0000000..8204215 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/CMakeASM_NASMOptions.txt @@ -0,0 +1,43 @@ +#=============================================================================== +# Copyright (C) 2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. +# +#=============================================================================== + +# +# Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) +# + +if (UNIX) + set(CMAKE_ASM_NASM_DEBUG_OPTIONS -g) + if (APPLE) + set(CMAKE_ASM_NASM_OBJECT_FORMAT macho64) + else() + if (${ARCH} MATCHES "ia32") + set(CMAKE_ASM_NASM_OBJECT_FORMAT elf32) + else() + set(CMAKE_ASM_NASM_OBJECT_FORMAT elf64) + endif() + endif() +else() # Windows + if (${ARCH} MATCHES "ia32") + set(CMAKE_ASM_NASM_OBJECT_FORMAT win32) # MS extended COFF for Win32 + else() + set(CMAKE_ASM_NASM_OBJECT_FORMAT win64) + endif() +endif() + + +set(CMAKE_ASM_NASM_COMPILE_OBJECT " -f ${CMAKE_ASM_NASM_OBJECT_FORMAT} -o ") +set(CMAKE_ASM_NASM_FLAGS_INIT "${LIBRARY_DEFINES}") diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/ippcp-config-version.cmake.in b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/ippcp-config-version.cmake.in new file mode 100644 index 0000000..87bf1e3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/ippcp-config-version.cmake.in @@ -0,0 +1,36 @@ +#=============================================================================== +# Copyright (C) 2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. +# +#=============================================================================== + +# +# Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) +# library detection routine (version compatibility rules). +# + +set(PACKAGE_VERSION @IPPCP_INTERFACE_VERSION@) + +set(PACKAGE_VERSION_EXACT False) +set(PACKAGE_VERSION_COMPATIBLE False) + +if(PACKAGE_FIND_VERSION VERSION_EQUAL PACKAGE_VERSION) + set(PACKAGE_VERSION_EXACT True) + set(PACKAGE_VERSION_COMPATIBLE True) +endif() + +if(PACKAGE_FIND_VERSION_MAJOR EQUAL @IPPCP_INTERFACE_VERSION_MAJOR@ + AND PACKAGE_FIND_VERSION VERSION_LESS PACKAGE_VERSION) + set(PACKAGE_VERSION_COMPATIBLE True) +endif() diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/ippcp-config.cmake.in b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/ippcp-config.cmake.in new file mode 100644 index 0000000..a1e94fb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/ippcp-config.cmake.in @@ -0,0 +1,149 @@ +#=============================================================================== +# Copyright (C) 2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. +# +#=============================================================================== + +# +# Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) +# library detection routine. +# +# To use it, add the lines below to your CMakeLists.txt: +# find_package(IPPCP REQUIRED) +# target_link_libraries(mytarget ${IPPCP_LIBRARIES}) +# +# List of the variables defined in this file: +# IPPCP_FOUND +# IPPCP_LIBRARIES - list of all imported targets +# +# Configuration variables available: +# IPPCP_SHARED - set this to True before find_package() to search for shared library. +# IPPCP_ARCH - set this to 'ia32' or 'intel64' before find_package() to use library of particular arch +# (this variable can be auto-defined) +# + +# Initialize to default values +if (NOT IPPCP_LIBRARIES) + set(IPPCP_LIBRARIES "") +endif() + +# Determine ARCH if not defined outside +if (NOT DEFINED IPPCP_ARCH) + set(IPPCP_ARCH "ia32") + if(CMAKE_CXX_SIZEOF_DATA_PTR EQUAL 8) + set(IPPCP_ARCH "intel64") + endif() + if(CMAKE_SIZEOF_VOID_P) + set(IPPCP_ARCH "intel64") + endif() +endif() + +if (NOT IPPCP_FIND_COMPONENTS) + set(IPPCP_FIND_COMPONENTS "ippcp") + + # crypto_mb library is only for intel64 + if(${IPPCP_ARCH} MATCHES "intel64") + set(IPPCP_BIN_REL_PATH @IPPCP_BIN64_REL_PATH@) + set(IPPCP_LIB_REL_PATH @IPPCP_LIB64_REL_PATH@) + + set(IPPCP_FIND_COMPONENTS "${IPPCP_FIND_COMPONENTS}" "crypto_mb") + else() + set(IPPCP_BIN_REL_PATH @IPPCP_BIN32_REL_PATH@) + set(IPPCP_LIB_REL_PATH @IPPCP_LIB32_REL_PATH@) + endif() + + foreach (_component ${IPPCP_FIND_COMPONENTS}) + set(IPPCP_FIND_REQUIRED_${_component} 1) + endforeach() +endif() + +if (WIN32) + set(_ippcp_library_prefix "") + set(_ippcp_static_library_suffix "mt.lib") + set(_ippcp_shared_library_suffix ".dll") + set(_ippcp_import_library_suffix ".lib") +else() + set(_ippcp_library_prefix "lib") + set(_ippcp_static_library_suffix ".a") + if (APPLE) + set(_ippcp_shared_library_suffix ".dylib") + else() + set(_ippcp_shared_library_suffix ".so") + endif() + set(_ippcp_import_library_suffix "") +endif() + +get_filename_component(_ippcrypto_root "${CMAKE_CURRENT_LIST_DIR}" REALPATH) +set(_ippcrypto_root "${_ippcrypto_root}/../../..") + +macro(add_imported_library_target PATH_TO_LIBRARY PATH_TO_IMPORT_LIB LINKAGE_TYPE) + if (EXISTS "${PATH_TO_LIBRARY}") + if (NOT TARGET IPPCP::${_component}) + add_library(IPPCP::${_component} ${LINKAGE_TYPE} IMPORTED) + get_filename_component(_include_dir "${_ippcrypto_root}/@IPPCP_INC_REL_PATH@" REALPATH) + if (EXISTS "${_include_dir}") + set_target_properties(IPPCP::${_component} PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${_include_dir}" + IMPORTED_LOCATION "${PATH_TO_LIBRARY}") + if (WIN32) + set_target_properties(IPPCP::${_component} PROPERTIES IMPORTED_IMPLIB "${PATH_TO_IMPORT_LIB}") + endif() + else() + message(WARNING "IPPCP: Include directory does not exist: '${_include_dir}'. Intel IPP Cryptography installation might be broken.") + endif() + unset(_include_dir) + endif() + list(APPEND IPPCP_LIBRARIES IPPCP::${_component}) + set(IPPCP_${_component}_FOUND 1) + elseif (IPPCP_FIND_REQUIRED AND IPPCP_FIND_REQUIRED_${_component}) + message(STATUS "Missed required Intel IPP Cryptography component: ${_component}") + message(STATUS " library not found:\n ${PATH_TO_LIBRARY}") + if (${LINKAGE_TYPE} MATCHES "SHARED") + message(STATUS "You may try to search for static library by unsetting IPPCP_SHARED variable.") + endif() + set(IPPCP_FOUND FALSE) + endif() +endmacro(add_imported_library_target) + +foreach (_component ${IPPCP_FIND_COMPONENTS}) + set(IPPCP_${_component}_FOUND 0) + + if (IPPCP_SHARED) + set(_ippcp_library_suffix "${_ippcp_shared_library_suffix}") + set(_linkage_type "SHARED") + else() + set(_ippcp_library_suffix "${_ippcp_static_library_suffix}") + set(_linkage_type "STATIC") + endif() + + if (WIN32 AND ${_linkage_type} MATCHES "SHARED") + get_filename_component(_lib "${_ippcrypto_root}/${IPPCP_BIN_REL_PATH}/${_ippcp_library_prefix}${_component}${_ippcp_library_suffix}" REALPATH) + get_filename_component(_imp_lib "${_ippcrypto_root}/${PPCP_LIB_REL_PATH}/${_ippcp_library_prefix}${_component}${_ippcp_import_library_suffix}" REALPATH) + else() + get_filename_component(_lib "${_ippcrypto_root}/${IPPCP_LIB_REL_PATH}/${_ippcp_library_prefix}${_component}${_ippcp_library_suffix}" REALPATH) + set(_imp_lib "") + endif() + + add_imported_library_target("${_lib}" "${_imp_lib}" "${_linkage_type}") +endforeach() + +list(REMOVE_DUPLICATES IPPCP_LIBRARIES) +unset(_ippcp_library_prefix) +unset(_ippcp_static_library_suffix) +unset(_ippcp_shared_library_suffix) +unset(_ippcp_import_library_suffix) +unset(_ippcp_library_suffix) +unset(_linkage_type) +unset(_lib) +unset(_imp_lib) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/linux/sanitizers_ignorelist.txt b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/linux/sanitizers_ignorelist.txt new file mode 100644 index 0000000..7dd77ca --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/linux/sanitizers_ignorelist.txt @@ -0,0 +1,22 @@ +#=============================================================================== +# Copyright 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#=============================================================================== + +[memory] +fun:*cpGetFeatures* +fun:*IntelCpu* +fun:*ppSetIppEnvironment* +src:tests/testlib/* +src:perf_tests/perflib/* \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/pkg-config/crypto_mb-dynamic.pc.in b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/pkg-config/crypto_mb-dynamic.pc.in new file mode 100644 index 0000000..98ae0f9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/pkg-config/crypto_mb-dynamic.pc.in @@ -0,0 +1,27 @@ +#=============================================================================== +# Copyright (C) 2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. +# +#=============================================================================== + +prefix=@PREFIX_FOR_PC_FILE@ +libdir=@LIBDIR_FOR_PC_FILE@ +includedir=@INCDIR_FOR_PC_FILE@ + +Name: Intel® Integrated Performance Primitives (Intel® IPP) Cryptography Crypto Multi-Buffer Library +Description: This library consists of highly-optimized kernels taking advantage of Intel’s multi-buffer processing and Intel® AVX-512 instruction set. +URL: https://github.com/intel/ipp-crypto +Version: @IPPCP_VERSION@ +Libs: -L${libdir} -l@CRYPTO_MB_PC_LIB_NAME@ +Cflags: -I${includedir} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/pkg-config/crypto_mb-static.pc.in b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/pkg-config/crypto_mb-static.pc.in new file mode 100644 index 0000000..4a1df44 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/pkg-config/crypto_mb-static.pc.in @@ -0,0 +1,27 @@ +#=============================================================================== +# Copyright (C) 2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. +# +#=============================================================================== + +prefix=@PREFIX_FOR_PC_FILE@ +libdir=@LIBDIR_FOR_PC_FILE@ +includedir=@INCDIR_FOR_PC_FILE@ + +Name: Intel® Integrated Performance Primitives (Intel® IPP) Cryptography Crypto Multi-Buffer Library +Description: This library consists of highly-optimized kernels taking advantage of Intel’s multi-buffer processing and Intel® AVX-512 instruction set. +URL: https://github.com/intel/ipp-crypto +Version: @IPPCP_VERSION@ +Libs: ${libdir}/@STATIC_LIBRARY_PREFIX@@CRYPTO_MB_PC_LIB_NAME@@STATIC_LIBRARY_SUFFIX@ +Cflags: -I${includedir} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/pkg-config/ippcp-dynamic.pc.in b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/pkg-config/ippcp-dynamic.pc.in new file mode 100644 index 0000000..94d2773 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/pkg-config/ippcp-dynamic.pc.in @@ -0,0 +1,27 @@ +#=============================================================================== +# Copyright (C) 2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. +# +#=============================================================================== + +prefix=@PREFIX_FOR_PC_FILE@ +libdir=@LIBDIR_FOR_PC_FILE@ +includedir=@INCDIR_FOR_PC_FILE@ + +Name: Intel® Integrated Performance Primitives (Intel® IPP) Cryptography Library +Description: Secure, fast and lightweight library of building blocks for cryptography, highly-optimized for various Intel® CPUs. +URL: https://github.com/intel/ipp-crypto +Version: @IPPCP_VERSION@ +Libs: -L${libdir} -l@IPPCP_PC_LIB_NAME@ +Cflags: -I${includedir} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/pkg-config/ippcp-static.pc.in b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/pkg-config/ippcp-static.pc.in new file mode 100644 index 0000000..7732e58 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/cmake/pkg-config/ippcp-static.pc.in @@ -0,0 +1,27 @@ +#=============================================================================== +# Copyright (C) 2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. +# +#=============================================================================== + +prefix=@PREFIX_FOR_PC_FILE@ +libdir=@LIBDIR_FOR_PC_FILE@ +includedir=@INCDIR_FOR_PC_FILE@ + +Name: Intel® Integrated Performance Primitives (Intel® IPP) Cryptography Library +Description: Secure, fast and lightweight library of building blocks for cryptography, highly-optimized for various Intel® CPUs. +URL: https://github.com/intel/ipp-crypto +Version: @IPPCP_VERSION@ +Libs: ${libdir}/@STATIC_LIBRARY_PREFIX@@IPPCP_PC_LIB_NAME@@STATIC_LIBRARY_SUFFIX@ +Cflags: -I${includedir} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/__pycache__/gen_disp_common.cpython-39.pyc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/__pycache__/gen_disp_common.cpython-39.pyc new file mode 100644 index 0000000..9b53757 Binary files /dev/null and b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/__pycache__/gen_disp_common.cpython-39.pyc differ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_common.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_common.py new file mode 100644 index 0000000..fce76b7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_common.py @@ -0,0 +1,67 @@ +#=============================================================================== +# Copyright (C) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. +# +#=============================================================================== + +# +# Intel(R) Integrated Performance Primitives Cryptography (Intel(R) IPP Cryptography) +# + +import re +import sys +import os +import hashlib + +def readNextFunction(header, curLine, headerID): ## read next function with arguments + ## find header ID macros + FunName = '' + FunArg = '' + FunType = '' + success = False + while (curLine < len(header) and success == False): + if not headerID and re.match( '\s*#\s*if\s*!\s*defined\s*\(\s*__IPP', header[curLine]): + headerID= re.sub( '.*__IPP', '__IPP', header[curLine] ) + headerID= re.sub( "\)", '', headerID) + headerID= re.sub( '[\n\s]', '', headerID ) + + if re.match( '^\s*IPPAPI\s*\(.*', header[curLine] ) : + FunStr= header[curLine]; + FunStr= re.sub('\n','',FunStr) ## remove EOL symbols + + while not re.match('.*\)\s*\)\s*$', FunStr): ## concatinate strinng if string is not completed + curLine= curLine+1 + FunStr= FunStr+header[curLine] + FunStr= re.sub('\n','',FunStr) ## remove EOL symbols + + FunStr= re.sub('\s+', ' ', FunStr) + + s= FunStr.split(',') + + ## Extract funtion name + FunName= s[1] + FunName= re.sub('\s', '', FunName) + + ## Extract function type + FunType= re.sub( '.*\(', '', s[0] ) + #FunType= re.sub(' ', '', FunType ) + + ## Extract function arguments + FunArg= re.sub('.*\(.*,.+,\s*\(', '(', FunStr) + FunArg= re.sub('\)\s*\)', ')', FunArg) + success = True + + curLine = curLine + 1 + + return {'curLine':curLine, 'FunType':FunType, 'FunName':FunName, 'FunArg':FunArg, 'success':success } \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_lin32.nonpic.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_lin32.nonpic.py new file mode 100644 index 0000000..b2155c4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_lin32.nonpic.py @@ -0,0 +1,175 @@ +#=============================================================================== +# Copyright (C) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. +# +#=============================================================================== + +# +# Intel(R) Integrated Performance Primitives Cryptography (Intel(R) IPP Cryptography) +# + +import re +import sys +import os +import hashlib +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument('-i', '--header', action='store', required=True, help='Intel IPP Cryptography dispatcher will be generated for fucntions in Header') +parser.add_argument('-o', '--out-directory', action='store', required=True, help='Output folder for generated files') +parser.add_argument('-l', '--cpu-list', action='store', required=True, help='Actual CPU list: semicolon separated string') +parser.add_argument('-c', '--compiler', action='store', required=True, help='Compiler') +args = parser.parse_args() +Header = args.header +OutDir = args.out_directory +cpulist = args.cpu_list.split(';') +compiler = args.compiler + +headerID= False ## Header ID define to avoid multiple include like: #if !defined( __IPPCP_H__ ) + +from gen_disp_common import readNextFunction + +HDR= open( Header, 'r' ) +h= HDR.readlines() +HDR.close() + + +## keep filename only +(incdir, Header)= os.path.split(Header) + +## original header name to declare external functions as internal for dispatcher +OrgH= Header + +isFunctionFound = True +curLine = 0 +FunName = "" +FunArg = "" + +if(compiler == "GNU" or compiler == "Clang" or compiler == "IntelLLVM"): + while (isFunctionFound == True): + + result = readNextFunction(h, curLine, headerID) + + curLine = result['curLine'] + FunName = result['FunName'] + FunArg = result['FunArg'] + isFunctionFound = result['success'] + + if (isFunctionFound == True): + + ################################################## + ## create dispatcher files ASM + ################################################## + ASMDISP= open( os.sep.join([OutDir, "jmp_" + FunName+"_" + hashlib.sha512(FunName.encode('utf-8')).hexdigest()[:8] +".asm"]), 'w' ) + + # Symbol type setting for extern functions initially appeared in version 2.15 + ASMDISP.write("%if ((__NASM_MAJOR__ > 2) || ((__NASM_MAJOR__ == 2) && (__NASM_MINOR__ > 14)))\n"); + ASMDISP.write(" %xdefine elf_symbol_type :function\n"); + ASMDISP.write("%else\n"); + ASMDISP.write(" %xdefine elf_symbol_type\n"); + ASMDISP.write("%endif\n"); + for cpu in cpulist: + ASMDISP.write("extern "+cpu+"_"+FunName+"%+elf_symbol_type\n") + + ASMDISP.write("extern ippcpJumpIndexForMergedLibs\n") + ASMDISP.write("extern ippcpInit%+elf_symbol_type\n\n") + ASMDISP.write(""" +segment .data +align 4 + +dd in_{FunName} +arraddr_{FunName}: + +""".format(FunName=FunName)) + size = 4 + for cpu in cpulist: + size = size + 4 + ASMDISP.write(" dd "+cpu+"_"+FunName+"\n") + + ASMDISP.write(""" + +segment .text +global {FunName}:function ({FunName}.LEnd{FunName} - {FunName}) +in_{FunName}: + {endbr32} + call ippcpInit + align 16 +{FunName}: + {endbr32} + mov eax, dword [ippcpJumpIndexForMergedLibs] + jmp dword [rel arraddr_{FunName} + eax*4] +.LEnd{FunName}: +""".format(FunName=FunName, size=size, endbr32='db 0xf3, 0x0f, 0x1e, 0xfb')) + ASMDISP.close() +else: + while (isFunctionFound == True): + + result = readNextFunction(h, curLine, headerID) + + curLine = result['curLine'] + FunName = result['FunName'] + FunArg = result['FunArg'] + isFunctionFound = result['success'] + + if (isFunctionFound == True): + + ################################################## + ## create dispatcher files: C file with inline asm + ################################################## + DISP= open( os.sep.join([OutDir, "jmp_"+FunName+"_" + hashlib.sha512(FunName.encode('utf-8')).hexdigest()[:8] + ".c"]), 'w' ) + + DISP.write("""#include "ippcpdefs.h"\n\n""") + + DISP.write("typedef void (*IPP_PROC)(void);\n\n") + DISP.write("extern int ippcpJumpIndexForMergedLibs;\n") + DISP.write("extern IPP_CALL ippcpInit();\n\n") + + DISP.write("extern IppStatus in_"+FunName+FunArg+";\n") + + for cpu in cpulist: + DISP.write("extern IppStatus "+cpu+"_"+FunName+FunArg+";\n") + + DISP.write(""" +__asm( " .data"); +__asm( " .align 4"); +__asm( "arraddr:"); +__asm( " .long in_{FunName}");""".format(FunName=FunName)) + size = 4 + for cpu in cpulist: + size = size + 4 + DISP.write("""\n__asm( " .long {cpu}_{FunName}");""".format(FunName=FunName, cpu=cpu)) + + DISP.write(""" +__asm( " .type arraddr,@object"); +__asm( " .size arraddr,{size}"); +__asm( " .data");\n""".format(size=size)) + + DISP.write(""" +#undef IPPAPI +#define IPPAPI(type,name,arg) __declspec(naked) void name arg +IPPAPI(IppStatus, {FunName},{FunArg}) +{{ + __asm( ".L0: mov ippcpJumpIndexForMergedLibs, %eax"); + __asm( "mov (arraddr+4)(,%eax,4), %eax" ); + __asm( "jmp *%eax" ); + __asm( ".global in_{FunName}" ); + __asm( "in_{FunName}:" ); + {endbr32} + __asm( "call ippcpInit" ); + __asm( "jmp .L0" ); +}}; +""".format(FunName=FunName, FunArg=FunArg, endbr32='__asm( ".byte 0xf3, 0x0f, 0x1e, 0xfb" );')) + + DISP.close() + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_lin32.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_lin32.py new file mode 100644 index 0000000..7dda598 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_lin32.py @@ -0,0 +1,191 @@ +#=============================================================================== +# Copyright (C) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. +# +#=============================================================================== + +# +# Intel(R) Integrated Performance Primitives Cryptography (Intel(R) IPP Cryptography) +# + +import re +import sys +import os +import hashlib +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument('-i', '--header', action='store', required=True, help='Intel IPP Cryptography dispatcher will be generated for fucntions in Header') +parser.add_argument('-o', '--out-directory', action='store', required=True, help='Output folder for generated files') +parser.add_argument('-l', '--cpu-list', action='store', required=True, help='Actual CPU list: semicolon separated string') +parser.add_argument('-c', '--compiler', action='store', required=True, help='Compiler') +args = parser.parse_args() +Header = args.header +OutDir = args.out_directory +cpulist = args.cpu_list.split(';') +compiler = args.compiler + +headerID= False ## Header ID define to avoid multiple include like: #if !defined( __IPPCP_H__ ) + +from gen_disp_common import readNextFunction + +HDR= open( Header, 'r' ) +h= HDR.readlines() +HDR.close() + + +## keep filename only +(incdir, Header)= os.path.split(Header) + +## original header name to declare external functions as internal for dispatcher +OrgH= Header + +isFunctionFound = True +curLine = 0 +FunName = "" +FunArg = "" + +if(compiler == "GNU" or compiler == "Clang" or compiler == "IntelLLVM"): + while (isFunctionFound == True): + + result = readNextFunction(h, curLine, headerID) + + curLine = result['curLine'] + FunName = result['FunName'] + FunArg = result['FunArg'] + isFunctionFound = result['success'] + + if (isFunctionFound == True): + + ################################################## + ## create dispatcher files ASM + ################################################## + ASMDISP= open( os.sep.join([OutDir, "jmp_" + FunName+"_" + hashlib.sha512(FunName.encode('utf-8')).hexdigest()[:8] +".asm"]), 'w' ) + + # Symbol type setting for extern functions initially appeared in version 2.15 + ASMDISP.write("%if ((__NASM_MAJOR__ > 2) || ((__NASM_MAJOR__ == 2) && (__NASM_MINOR__ > 14)))\n"); + ASMDISP.write(" %xdefine elf_symbol_type :function\n"); + ASMDISP.write("%else\n"); + ASMDISP.write(" %xdefine elf_symbol_type\n"); + ASMDISP.write("%endif\n"); + for cpu in cpulist: + ASMDISP.write("extern "+cpu+"_"+FunName+"%+elf_symbol_type\n") + + ASMDISP.write("extern ippcpJumpIndexForMergedLibs\n") + ASMDISP.write("extern ippcpInit%+elf_symbol_type\n\n") + + ASMDISP.write(""" +segment .data +align 4 +dd .Lin_{FunName} +.Larraddr_{FunName}: +""".format(FunName=FunName)) + + for cpu in cpulist: + ASMDISP.write(" dd "+cpu+"_"+FunName+"\n") + + ASMDISP.write(""" + +segment .text +extern _GLOBAL_OFFSET_TABLE_ +global {FunName}:function ({FunName}.LEnd{FunName} - {FunName}) +.Lin_{FunName}: + {endbr32} + push ebx + mov ebx, eax + call ippcpInit wrt ..plt + pop ebx + align 16 +{FunName}: + {endbr32} + call .L1 +.L1: + pop eax + add eax, _GLOBAL_OFFSET_TABLE_ + $$ - .L1 wrt ..gotpc + mov edx, [eax + ippcpJumpIndexForMergedLibs wrt ..got] + mov edx, [edx] + jmp dword [eax + edx*4 + .Larraddr_{FunName} wrt ..gotoff] +.LEnd{FunName}: +""".format(FunName=FunName, endbr32='db 0xf3, 0x0f, 0x1e, 0xfb')) + ASMDISP.close() +else: + + while (isFunctionFound == True): + + result = readNextFunction(h, curLine, headerID) + + curLine = result['curLine'] + FunName = result['FunName'] + FunArg = result['FunArg'] + isFunctionFound = result['success'] + + if (isFunctionFound == True): + + ################################################## + ## create dispatcher files: C file with inline asm + ################################################## + DISP= open( os.sep.join([OutDir, "jmp_"+FunName+"_" + hashlib.sha512(FunName.encode('utf-8')).hexdigest()[:8] + ".c"]), 'w' ) + + DISP.write("""#include "ippcpdefs.h"\n\n""") + + DISP.write("typedef void (*IPP_PROC)(void);\n\n") + DISP.write("extern int ippcpJumpIndexForMergedLibs;\n") + DISP.write("extern IPP_CALL ippcpInit();\n\n") + + DISP.write("extern IppStatus IPP_CALL in_"+FunName+FunArg+";\n") + + for cpu in cpulist: + DISP.write("extern IppStatus IPP_CALL "+cpu+"_"+FunName+FunArg+";\n") + + DISP.write(""" +__asm( " .data"); +__asm( " .align 4"); +__asm( "arraddr:"); +__asm( " .long in_{FunName}");""".format(FunName=FunName)) + size = 4 + for cpu in cpulist: + size = size + 4 + DISP.write("""\n__asm( " .long {cpu}_{FunName}");""".format(FunName=FunName, cpu=cpu)) + + DISP.write(""" +__asm( " .type arraddr,@object"); +__asm( " .size arraddr,{size}"); +__asm( " .data");\n""".format(size=size)) + + DISP.write(""" +#undef IPPAPI +#define IPPAPI(type,name,arg) __declspec(naked) void IPP_CALL name arg +__declspec(naked) IPP_PROC {FunName}{FunArg} +{{ + __asm( ".L0: call .L1"); + __asm( ".L1: pop %eax"); + __asm( "add $_GLOBAL_OFFSET_TABLE_ - .L1, %eax" ); + __asm( "movd ippcpJumpIndexForMergedLibs@GOT(%eax), %xmm0" ); + __asm( "movd %xmm0, %edx" ); + __asm( "mov (%edx), %edx" ); + __asm( "mov (arraddr@GOTOFF+4)(%eax,%edx,4), %edx" ); + __asm( "jmp *%edx" ); + __asm( ".global in_{FunName}" ); + __asm( "in_{FunName}:" ); + {endbr32} + __asm( "push %ebx" ); + __asm( "mov %eax, %ebx" ); + __asm( "call ippcpInit@PLT" ); + __asm( "pop %ebx" ); + __asm( "jmp .L0" ); +}}; +""".format(FunName=FunName, FunArg=FunArg, endbr32='__asm( ".byte 0xf3, 0x0f, 0x1e, 0xfb" );')) + + DISP.close() + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_lin64.nonpic.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_lin64.nonpic.py new file mode 100644 index 0000000..8439ef1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_lin64.nonpic.py @@ -0,0 +1,175 @@ +#=============================================================================== +# Copyright (C) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. +# +#=============================================================================== + +# +# Intel(R) Integrated Performance Primitives Cryptography (Intel(R) IPP Cryptography) +# + +import re +import sys +import os +import hashlib +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument('-i', '--header', action='store', required=True, help='Intel IPP Cryptography dispatcher will be generated for fucntions in Header') +parser.add_argument('-o', '--out-directory', action='store', required=True, help='Output folder for generated files') +parser.add_argument('-l', '--cpu-list', action='store', required=True, help='Actual CPU list: semicolon separated string') +parser.add_argument('-c', '--compiler', action='store', required=True, help='Compiler') +args = parser.parse_args() +Header = args.header +OutDir = args.out_directory +cpulist = args.cpu_list.split(';') +compiler = args.compiler + +headerID= False ## Header ID define to avoid multiple include like: #if !defined( __IPPCP_H__ ) + +from gen_disp_common import readNextFunction + +HDR= open( Header, 'r' ) +h= HDR.readlines() +HDR.close() + + +## keep filename only +(incdir, Header)= os.path.split(Header) + +## original header name to declare external functions as internal for dispatcher +OrgH= Header + +isFunctionFound = True +curLine = 0 +FunType = "" +FunName = "" +FunArg = "" + +if(compiler == "GNU" or compiler == "Clang" or compiler == "IntelLLVM"): + while (isFunctionFound == True): + + result = readNextFunction(h, curLine, headerID) + + curLine = result['curLine'] + FunType = result['FunType'] + FunName = result['FunName'] + FunArg = result['FunArg'] + isFunctionFound = result['success'] + + if (isFunctionFound == True): + + ################################################## + ## create dispatcher files ASM + ################################################## + ASMDISP= open( os.sep.join([OutDir, "jmp_" + FunName+"_" + hashlib.sha512(FunName.encode('utf-8')).hexdigest()[:8] +".asm"]), 'w' ) + + # Symbol type setting for extern functions initially appeared in version 2.15 + ASMDISP.write("%if ((__NASM_MAJOR__ > 2) || ((__NASM_MAJOR__ == 2) && (__NASM_MINOR__ > 14)))\n"); + ASMDISP.write(" %xdefine elf_symbol_type :function\n"); + ASMDISP.write("%else\n"); + ASMDISP.write(" %xdefine elf_symbol_type\n"); + ASMDISP.write("%endif\n"); + for cpu in cpulist: + ASMDISP.write("extern "+cpu+"_"+FunName+"%+elf_symbol_type\n") + + ASMDISP.write("extern ippcpJumpIndexForMergedLibs\n") + ASMDISP.write("extern ippcpSafeInit%+elf_symbol_type\n\n") + ASMDISP.write(""" +segment .data +align 8 +dq .Lin_{FunName} +.Larraddr_{FunName}: +""".format(FunName=FunName)) + + for cpu in cpulist: + ASMDISP.write(" dq "+cpu+"_"+FunName+"\n") + + ASMDISP.write(""" +segment .text +global {FunName}:function ({FunName}.LEnd{FunName} - {FunName}) +.Lin_{FunName}: + {endbr64} + call ippcpSafeInit + align 16 + +{FunName}: + {endbr64} + movsxd rax, dword [ippcpJumpIndexForMergedLibs] + lea r11, [rel .Larraddr_{FunName}] + mov r11, qword [r11 + rax*8] + jmp r11 +.LEnd{FunName}: +""".format(FunName=FunName, endbr64='db 0xf3, 0x0f, 0x1e, 0xfa')) + ASMDISP.close() +else: + while (isFunctionFound == True): + + result = readNextFunction(h, curLine, headerID) + + curLine = result['curLine'] + FunType = result['FunType'] + FunName = result['FunName'] + FunArg = result['FunArg'] + isFunctionFound = result['success'] + + if (isFunctionFound == True): + + ################################################## + ## create dispatcher files: C file with inline asm + ################################################## + DISP= open( os.sep.join([OutDir, "jmp_"+FunName+"_" + hashlib.sha512(FunName.encode('utf-8')).hexdigest()[:8] + ".c"]), 'w' ) + + DISP.write("""#include "ippcp.h"\n\n#pragma warning(disable : 1478 1786) // deprecated\n\n""") + + DISP.write("typedef "+FunType+" (*IPP_PROC)"+FunArg+";\n\n") + DISP.write("extern int ippcpJumpIndexForMergedLibs;\n") + DISP.write("extern IppStatus ippcpSafeInit( void );\n\n") + + DISP.write("extern "+FunType+" in_"+FunName+FunArg+";\n") + + for cpu in cpulist: + DISP.write("extern "+FunType+" "+cpu+"_"+FunName+FunArg+";\n") + + DISP.write("static IPP_PROC arraddr[] =\n{{\n (IPP_PROC)in_{}".format(FunName)) + + for cpu in cpulist: + DISP.write(",\n (IPP_PROC)"+cpu+"_"+FunName+"") + + DISP.write("\n};") + + DISP.write(""" +#undef IPPAPI +#define IPPAPI(type,name,arg) __declspec(naked) type name arg + +IPPAPI({FunType}, {FunName},{FunArg}) +{{ + __asm{{ + movsxd rax, dword ptr ippcpJumpIndexForMergedLibs + mov rax, qword ptr [8*rax+8+arraddr] + jmp rax + }} +}} + +IPPAPI({FunType}, in_{FunName},{FunArg}) +{{ + __asm{{ + call ippcpSafeInit + lea rax, {FunName} + jmp rax + }} +}}; +""".format(FunType=FunType, FunName=FunName, FunArg=FunArg)) + + DISP.close() diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_lin64.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_lin64.py new file mode 100644 index 0000000..d5645d5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_lin64.py @@ -0,0 +1,174 @@ +#=============================================================================== +# Copyright (C) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. +# +#=============================================================================== + +# +# Intel(R) Integrated Performance Primitives Cryptography (Intel(R) IPP Cryptography) +# + +import re +import sys +import os +import hashlib +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument('-i', '--header', action='store', required=True, help='Intel IPP Cryptography dispatcher will be generated for fucntions in Header') +parser.add_argument('-o', '--out-directory', action='store', required=True, help='Output folder for generated files') +parser.add_argument('-l', '--cpu-list', action='store', required=True, help='Actual CPU list: semicolon separated string') +parser.add_argument('-c', '--compiler', action='store', required=True, help='Compiler') +args = parser.parse_args() +Header = args.header +OutDir = args.out_directory +cpulist = args.cpu_list.split(';') +compiler = args.compiler + +headerID= False ## Header ID define to avoid multiple include like: #if !defined( __IPPCP_H__ ) + +from gen_disp_common import readNextFunction + +HDR= open( Header, 'r' ) +h= HDR.readlines() +HDR.close() + + +## keep filename only +(incdir, Header)= os.path.split(Header) + +## original header name to declare external functions as internal for dispatcher +OrgH= Header + +isFunctionFound = True +curLine = 0 +FunType = "" +FunName = "" +FunArg = "" + +if(compiler == "GNU" or compiler == "Clang" or compiler == "IntelLLVM"): + + while (isFunctionFound == True): + + result = readNextFunction(h, curLine, headerID) + + curLine = result['curLine'] + FunType = result['FunType'] + FunName = result['FunName'] + FunArg = result['FunArg'] + isFunctionFound = result['success'] + + if (isFunctionFound == True): + + ################################################## + ## create dispatcher files ASM + ################################################## + ASMDISP= open( os.sep.join([OutDir, "jmp_" + FunName+"_" + hashlib.sha512(FunName.encode('utf-8')).hexdigest()[:8] +".asm"]), 'w' ) + + # Symbol type setting for extern functions initially appeared in version 2.15 + ASMDISP.write("%if ((__NASM_MAJOR__ > 2) || ((__NASM_MAJOR__ == 2) && (__NASM_MINOR__ > 14)))\n"); + ASMDISP.write(" %xdefine elf_symbol_type :function\n"); + ASMDISP.write("%else\n"); + ASMDISP.write(" %xdefine elf_symbol_type\n"); + ASMDISP.write("%endif\n"); + for cpu in cpulist: + ASMDISP.write("extern "+cpu+"_"+FunName+"%+elf_symbol_type\n") + + ASMDISP.write("extern ippcpJumpIndexForMergedLibs\n") + ASMDISP.write("extern ippcpSafeInit%+elf_symbol_type\n\n") + ASMDISP.write(""" +segment .data +align 8 +dq .Lin_{FunName} +.Larraddr_{FunName}: +""".format(FunName=FunName)) + + for cpu in cpulist: + ASMDISP.write(" dq "+cpu+"_"+FunName+"\n") + + ASMDISP.write(""" +segment .text +global {FunName}:function ({FunName}.LEnd{FunName} - {FunName}) +.Lin_{FunName}: + {endbr64} + call ippcpSafeInit wrt ..plt + align 16 + +{FunName}: + {endbr64} + mov rax, qword [rel ippcpJumpIndexForMergedLibs wrt ..gotpc] + movsxd rax, dword [rax] + lea r11, [rel .Larraddr_{FunName}] + mov r11, qword [r11+rax*8] + jmp r11 +.LEnd{FunName}: +""".format(FunName=FunName, endbr64='db 0xf3, 0x0f, 0x1e, 0xfa')) + ASMDISP.close() + +else: + while (isFunctionFound == True): + + result = readNextFunction(h, curLine, headerID) + + curLine = result['curLine'] + FunType = result['FunType'] + FunName = result['FunName'] + FunArg = result['FunArg'] + isFunctionFound = result['success'] + + if (isFunctionFound == True): + + ################################################## + ## create dispatcher files: C file with inline asm + ################################################## + DISP= open( os.sep.join([OutDir, "jmp_"+FunName+"_" + hashlib.sha512(FunName.encode('utf-8')).hexdigest()[:8] + ".c"]), 'w' ) + + DISP.write("""#include "ippcp.h"\n\n#pragma warning(disable : 1478 1786) // deprecated\n\n""") + + DISP.write("typedef "+FunType+" (*IPP_PROC)"+FunArg+";\n\n") + DISP.write("extern int ippcpJumpIndexForMergedLibs;\n") + DISP.write("extern IppStatus ippcpSafeInit( void );\n\n") + + DISP.write("extern "+FunType+" in_"+FunName+FunArg+";\n") + + for cpu in cpulist: + DISP.write("extern "+FunType+" "+cpu+"_"+FunName+FunArg+";\n") + + DISP.write("static IPP_PROC arraddr[] =\n{{\n (IPP_PROC)in_{}".format(FunName)) + + for cpu in cpulist: + DISP.write(",\n (IPP_PROC)"+cpu+"_"+FunName+"") + + DISP.write("\n};") + + DISP.write(""" +#undef IPPAPI +#define IPPAPI(type,name,arg) __declspec(naked) type name arg +IPPAPI({FunType}, {FunName},{FunArg}) +{{ + register unsigned long long i __asm__("rax"); + i = (unsigned long long)arraddr[ippcpJumpIndexForMergedLibs+1]; + _asm{{ jmp rax }} +}}; +IPPAPI({FunType}, in_{FunName},{FunArg}) +{{ + __asm{{ + call ippcpSafeInit + mov rax, qword ptr [{FunName}] + jmp rax + }} +}}; +""".format(FunType=FunType, FunName=FunName, FunArg=FunArg)) + + DISP.close() diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_mac64.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_mac64.py new file mode 100644 index 0000000..badc127 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_mac64.py @@ -0,0 +1,168 @@ +#=============================================================================== +# Copyright (C) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. +# +#=============================================================================== + +# +# Intel(R) Integrated Performance Primitives Cryptography (Intel(R) IPP Cryptography) +# + +import re +import sys +import os +import hashlib +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument('-i', '--header', action='store', required=True, help='Intel IPP Cryptography dispatcher will be generated for fucntions in Header') +parser.add_argument('-o', '--out-directory', action='store', required=True, help='Output folder for generated files') +parser.add_argument('-l', '--cpu-list', action='store', required=True, help='Actual CPU list: semicolon separated string') +parser.add_argument('-c', '--compiler', action='store', help='Compiler') # is not used +args = parser.parse_args() +Header = args.header +OutDir = args.out_directory +cpulist = args.cpu_list.split(';') +compiler = args.compiler + +headerID= False ## Header ID define to avoid multiple include like: #if !defined( __IPPCP_H__ ) + +from gen_disp_common import readNextFunction + +HDR= open( Header, 'r' ) +h= HDR.readlines() +HDR.close() + + +## keep filename only +(incdir, Header)= os.path.split(Header) + +## original header name to declare external functions as internal for dispatcher +OrgH= Header + +isFunctionFound = True +curLine = 0 +FunType = "" +FunName = "" +FunArg = "" + +if(compiler == "AppleClang"): + while (isFunctionFound == True): + + result = readNextFunction(h, curLine, headerID) + + curLine = result['curLine'] + FunType = result['FunType'] + FunName = result['FunName'] + FunArg = result['FunArg'] + isFunctionFound = result['success'] + + if (isFunctionFound == True): + + ################################################## + ## create dispatcher files ASM + ################################################## + ASMDISP= open( os.sep.join([OutDir, "jmp_" + FunName+"_" + hashlib.sha512(FunName.encode('utf-8')).hexdigest()[:8] +".asm"]), 'w' ) + + for cpu in cpulist: + ASMDISP.write("extern "+"_"+cpu+"_"+FunName+"\n") + + ASMDISP.write("extern _ippcpJumpIndexForMergedLibs\n") + ASMDISP.write("extern _ippcpSafeInit\n\n") + ASMDISP.write(""" + segment .data + align 8 + dq .Lin_{FunName} + .Larraddr_{FunName}: + """.format(FunName=FunName)) + + for cpu in cpulist: + ASMDISP.write(" dq "+"_"+cpu+"_"+FunName+"\n") + + ASMDISP.write(""" + segment .text + global _{FunName} + .Lin_{FunName}: + {endbr64} + call _ippcpSafeInit + align 16 + + _{FunName}: + {endbr64} + mov rax, qword [rel _ippcpJumpIndexForMergedLibs wrt ..gotpcrel] + movsxd rax, dword [rax] + lea r11, [rel .Larraddr_{FunName}] + mov r11, qword [r11+rax*8] + jmp r11 + .LEnd{FunName}: + """.format(FunName=FunName, endbr64='db 0xf3, 0x0f, 0x1e, 0xfa')) + ASMDISP.close() +else: + while (isFunctionFound == True): + + result = readNextFunction(h, curLine, headerID) + + curLine = result['curLine'] + FunType = result['FunType'] + FunName = result['FunName'] + FunArg = result['FunArg'] + isFunctionFound = result['success'] + + if (isFunctionFound == True): + + ################################################## + ## create dispatcher files: C file with inline asm + ################################################## + DISP= open( os.sep.join([OutDir, "jmp_"+FunName+"_" + hashlib.sha512(FunName.encode('utf-8')).hexdigest()[:8] + ".c"]), 'w' ) + + DISP.write("""#include "ippcp.h"\n\n#pragma warning(disable : 1478 1786) // deprecated\n\n""") + + DISP.write("typedef "+FunType+" (*IPP_PROC)"+FunArg+";\n\n") + DISP.write("extern int ippcpJumpIndexForMergedLibs;\n") + DISP.write("extern IppStatus ippcpSafeInit( void );\n\n") + + DISP.write("extern "+FunType+" in_"+FunName+FunArg+";\n") + + for cpu in cpulist: + DISP.write("extern "+FunType+" "+cpu+"_"+FunName+FunArg+";\n") + + DISP.write("static IPP_PROC arraddr[] =\n{{\n (IPP_PROC)in_{}".format(FunName)) + + for cpu in cpulist: + DISP.write(",\n (IPP_PROC)"+cpu+"_"+FunName+"") + + DISP.write("\n};") + + DISP.write(""" + #undef IPPAPI + #define IPPAPI(type,name,arg) __declspec(naked) type name arg + + IPPAPI({FunType}, {FunName},{FunArg}) + {{ + register unsigned long long i __asm__("rax"); + i = (unsigned long long)arraddr[ippcpJumpIndexForMergedLibs+1]; + __asm{{ jmp rax }} + }}; + + IPPAPI({FunType}, in_{FunName},{FunArg}) + {{ + __asm{{ + call ippcpSafeInit + mov rax, qword ptr [{FunName}] + jmp rax + }} + }}; + """.format(FunType=FunType, FunName=FunName, FunArg=FunArg)) + + DISP.close() diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_win32.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_win32.py new file mode 100644 index 0000000..4534b70 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_win32.py @@ -0,0 +1,117 @@ +#=============================================================================== +# Copyright (C) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. +# +#=============================================================================== + +# +# Intel(R) Integrated Performance Primitives Cryptography (Intel(R) IPP Cryptography) +# + +import re +import sys +import os +import hashlib +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument('-i', '--header', action='store', required=True, help='Intel IPP Cryptography dispatcher will be generated for fucntions in Header') +parser.add_argument('-o', '--out-directory', action='store', required=True, help='Output folder for generated files') +parser.add_argument('-l', '--cpu-list', action='store', required=True, help='Actual CPU list: semicolon separated string') +parser.add_argument('-c', '--compiler', action='store', help='Compiler') # is not used +args = parser.parse_args() +Header = args.header +OutDir = args.out_directory +cpulist = args.cpu_list.split(';') + +headerID= False ## Header ID define to avoid multiple include like: #if !defined( __IPPCP_H__ ) + +from gen_disp_common import readNextFunction + +HDR= open( Header, 'r' ) +h= HDR.readlines() +HDR.close() + + +## keep filename only +(incdir, Header)= os.path.split(Header) + +## original header name to declare external functions as internal for dispatcher +OrgH= Header + +isFunctionFound = True +curLine = 0 +FunName = "" +FunType = "" +FunArg = "" + +while (isFunctionFound == True): + + result = readNextFunction(h, curLine, headerID) + + curLine = result['curLine'] + FunType = result['FunType'] + FunName = result['FunName'] + FunArg = result['FunArg'] + isFunctionFound = result['success'] + + if (isFunctionFound == True): + + ################################################## + ## create dispatcher files: C file with inline asm + ################################################## + DISP= open( os.sep.join([OutDir, "jmp_"+FunName+"_" + hashlib.sha512(FunName.encode('utf-8')).hexdigest()[:8] + ".c"]), 'w' ) + + DISP.write("""#include "ippcpdefs.h" """) + + DISP.write("\n\ntypedef " + FunType + "(*IPP_PROC)(void);\n\n") + DISP.write("extern int ippcpJumpIndexForMergedLibs;\n") + DISP.write("extern IppStatus IPP_CALL ippcpInit();\n\n") + + DISP.write("extern " + FunType + " IPP_CALL in_"+FunName+FunArg+";\n") + + for cpu in cpulist: + DISP.write("extern " + FunType + " IPP_CALL "+cpu+"_"+FunName+FunArg+";\n") + + DISP.write("static IPP_PROC arraddr[] =\n{{\n (IPP_PROC)in_{}".format(FunName)) + + for cpu in cpulist: + DISP.write(",\n (IPP_PROC)"+cpu+"_"+FunName+"") + + DISP.write("\n};") + + DISP.write(""" +#ifdef _MSC_VER +#pragma warning(disable: 4100) // for MSVC, unreferenced param +#endif +#undef IPPAPI +#define IPPAPI(type,name,arg) __declspec(naked) type __stdcall name arg +IPPAPI({FunType}, {FunName},{FunArg}) +{{ + __asm {{mov eax, dword ptr ippcpJumpIndexForMergedLibs}} + __asm {{mov eax, arraddr[eax*4+4]}} + __asm {{jmp eax}} +}}; + +IPPAPI({FunType}, in_{FunName},{FunArg}) +{{ + __asm {{call ippcpInit}} + __asm {{mov eax, dword ptr ippcpJumpIndexForMergedLibs }} + __asm {{mov eax, arraddr[eax*4+4]}} + __asm {{jmp eax}} +}}; +""".format(FunType=FunType, FunName=FunName, FunArg=FunArg)) + + DISP.close() + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_win64.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_win64.py new file mode 100644 index 0000000..9c52b29 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/dispatcher/gen_disp_win64.py @@ -0,0 +1,107 @@ +#=============================================================================== +# Copyright (C) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. +# +#=============================================================================== + +# +# Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) +# + +import re +import sys +import os +import hashlib +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument('-i', '--header', action='store', required=True, help='Intel IPP Cryptography dispatcher will be generated for fucntions in Header') +parser.add_argument('-o', '--out-directory', action='store', required=True, help='Output folder for generated files') +parser.add_argument('-l', '--cpu-list', action='store', required=True, help='Actual CPU list: semicolon separated string') +parser.add_argument('-c', '--compiler', action='store', help='Compiler') # is not used +args = parser.parse_args() +Header = args.header +OutDir = args.out_directory +cpulist = args.cpu_list.split(';') + +headerID= False ## Header ID define to avoid multiple include like: #if !defined( __IPPCP_H__ ) + +from gen_disp_common import readNextFunction + +HDR= open( Header, 'r' ) +h= HDR.readlines() +HDR.close() + +## keep filename only +(incdir, Header)= os.path.split(Header) + +## original header name to declare external functions as internal for dispatcher +OrgH= Header + +isFunctionFound = True +curLine = 0 +FunName = "" +FunArg = "" + +while (isFunctionFound == True): + + result = readNextFunction(h, curLine, headerID) + + curLine = result['curLine'] + FunName = result['FunName'] + FunArg = result['FunArg'] + isFunctionFound = result['success'] + + if (isFunctionFound == True): + ################################################## + ## create dispatcher files: C file with inline asm + ################################################## + filename = "jmp_{}_{}".format(FunName, hashlib.sha512(FunName.encode('utf-8')).hexdigest()[:8]) + + DISP= open( os.sep.join([OutDir, filename + ".asm"]), 'w' ) + + for cpu in cpulist: + DISP.write("extern "+cpu+"_"+FunName+"\n") + + DISP.write("extern ippcpJumpIndexForMergedLibs\n") + DISP.write("extern ippcpSafeInit\n\n") + + DISP.write("segment data\n\n") + + DISP.write(" DQ in_"+FunName+"\n") + DISP.write(FunName+"_arraddr:\n") + + for cpu in cpulist: + DISP.write(" DQ "+cpu+"_"+FunName+"\n") + + DISP.write(""" + +segment text + +global {FunName} + +in_{FunName}: + {endbr64} + call ippcpSafeInit + align 16 +{FunName}: + {endbr64} + movsxd rax, dword [rel ippcpJumpIndexForMergedLibs] + lea r10, [rel {FunName}_arraddr] + mov r10, qword [r10+rax*8] + jmp r10 + +""".format(FunName=FunName, endbr64='db 0xf3, 0x0f, 0x1e, 0xfa')) + + DISP.close() diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/gen_cpu_spc_header/gen_cpu_spc_1cpu_header.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/gen_cpu_spc_header/gen_cpu_spc_1cpu_header.py new file mode 100644 index 0000000..c044020 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/gen_cpu_spc_header/gen_cpu_spc_1cpu_header.py @@ -0,0 +1,86 @@ +#=============================================================================== +# Copyright (C) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. +# +#=============================================================================== + +# +# Intel(R) Integrated Performance Primitives Cryptography (Intel(R) IPP Cryptography) +# + +import sys +import os +import datetime + +sys.path.append(os.path.join(sys.path[0], '../dispatcher')) + +from gen_disp_common import readNextFunction + +Header = sys.argv[1] +OutDir = sys.argv[2] + +Header = os.path.abspath(Header) +Filename = "" + +HDR= open(Header, 'r') +h= HDR.readlines() +HDR.close() + +headerID= False +FunName = "" + + +if not os.path.exists(OutDir): + os.makedirs(OutDir) + + +Filename="ippcp" +Filenames=["h9", "p8", "s8", "w7", "e9", "k0", "k1", "l9", "m7", "n0", "n8", "y8", "g9"] + +for name in Filenames: + OutFile = os.sep.join([OutDir, Filename + "_"+ name + ".h"]) + + OUT= open( OutFile, 'w' ) + OUT.write("""/******************************************************************************* + * Copyright {year} Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + + """.format(year=datetime.datetime.today().year)) + + curLine = 0 + isFunctionFound = True + + while (isFunctionFound): + + result = readNextFunction(h, curLine, headerID) + + curLine = result['curLine'] + FunName = result['FunName'] + isFunctionFound = result['success'] + + if (isFunctionFound): + OUT.write("#define " + FunName + " " + name +"_" + FunName + "\n") + OUT.close() diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/gen_cpu_spc_header/gen_cpu_spc_header.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/gen_cpu_spc_header/gen_cpu_spc_header.py new file mode 100644 index 0000000..0bf8f26 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/gen_cpu_spc_header/gen_cpu_spc_header.py @@ -0,0 +1,81 @@ +#=============================================================================== +# Copyright (C) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. +# +#=============================================================================== + +# +# Intel(R) Integrated Performance Primitives Cryptography (Intel(R) IPP Cryptography) +# + +import sys +import os +import datetime + +sys.path.append(os.path.join(sys.path[0], '../dispatcher')) + +from gen_disp_common import readNextFunction + +Header = sys.argv[1] +OutDir = sys.argv[2] + +Header = os.path.abspath(Header) +Filename = "" + +HDR= open(Header, 'r') +h= HDR.readlines() +HDR.close() + +headerID= False +FunName = "" + +if not os.path.exists(OutDir): + os.makedirs(OutDir) + +Filename = "ippcp_cpuspc" +OutFile = os.sep.join([OutDir, Filename + ".h"]) + +OUT= open( OutFile, 'w' ) +OUT.write("""/******************************************************************************* +* Copyright {year} Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +""".format(year=datetime.datetime.today().year)) + +curLine = 0 +isFunctionFound = True + +while (isFunctionFound): + + result = readNextFunction(h, curLine, headerID) + + curLine = result['curLine'] + FunName = result['FunName'] + isFunctionFound = result['success'] + + if (isFunctionFound): + OUT.write("#define " + FunName + " " + "OWNAPI(" + FunName + ")\n") +OUT.close() diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/asmdefs.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/asmdefs.inc new file mode 100644 index 0000000..a7a3daa --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/asmdefs.inc @@ -0,0 +1,175 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +%ifndef __ASMDEFS_INC__ +%define __ASMDEFS_INC__ 1 + +%assign _IPP_PX 0 ; pure C-code ia32 +%assign _IPP_M5 1 ; Intel(R) Quark(TM) processor - ia32 +%assign _IPP_W7 8 ; Intel(R) Streaming SIMD Extensions 2 - ia32 +%assign _IPP_T7 16 ; Intel(R) Streaming SIMD Extensions 3 - ia32 +%assign _IPP_V8 32 ; Supplemental Streaming SIMD Extensions 3 (SSSE3) +%assign _IPP_S8 33 ; SSSE3 + MOVBE instruction - ia32 +%assign _IPP_P8 64 ; Intel(R) Streaming SIMD Extensions 4.2 - ia32 +%assign _IPP_G9 128 ; Intel(R) Advanced Vector Extensions - ia32 +%assign _IPP_H9 256 ; Intel(R) Advanced Vector Extensions 2 - ia32 +%assign _IPP_I0 512 ; Intel(R) Advanced Vector Extensions 512 - Intel(R) Xeon Phi(TM) processor (formerly Knight Landing) - ia32 +%assign _IPP_S0 1024 ; Intel(R) Advanced Vector Extensions 512 - Intel(R) Xeon(R) processor (formerly Skylake) - ia32 + +%assign _IPP32E_PX _IPP_PX ; pure C-code x64 +%assign _IPP32E_M7 32 ; Intel(R) Streaming SIMD Extensions 3 - intel64 +%assign _IPP32E_U8 64 ; Supplemental Streaming SIMD Extensions 3 (SSSE3) - intel64 +%assign _IPP32E_N8 65 ; SSSE3 + MOVBE instruction - intel64 +%assign _IPP32E_Y8 128 ; Intel(R) Streaming SIMD Extensions 4.2 - intel64 +%assign _IPP32E_E9 256 ; Intel(R) Advanced Vector Extensions - intel64 +%assign _IPP32E_L9 512 ; Intel(R) Advanced Vector Extensions 2 - intel64 +%assign _IPP32E_N0 1024 ; Intel(R) Advanced Vector Extensions 512 - Intel(R) Xeon Phi(TM) processor (formerly Knight Landing) - intel64 +%assign _IPP32E_K0 2048 ; Intel(R) Advanced Vector Extensions 512 - Intel(R) Xeon(R) processor (formerly Skylake) - intel64 +%assign _IPP32E_K1 4096 ; Intel(R) Advanced Vector Extensions 512 - Intel(R) Xeon(R) processor (formerly Icelake) - intel64 + +%assign _IPP _IPP_PX +%assign _IPP32E _IPP32E_PX + +%ifdef _M5 ; Intel(R) Quark(TM) processor - ia32 + %assign _IPP _IPP_M5 +%elifdef _W7 ; Intel(R) Streaming SIMD Extensions 2 - ia32 + %assign _IPP _IPP_W7 +%elifdef _T7 ; Intel(R) Streaming SIMD Extensions 3 - ia32 + %assign _IPP _IPP_T7 +%elifdef _V8 ; Supplemental Streaming SIMD Extensions 3 (SSSE3) + %assign _IPP _IPP_V8 +%elifdef _S8 ; SSSE3 + MOVBE instruction - ia32 + %assign _IPP _IPP_S8 +%elifdef _P8 ; Intel(R) Streaming SIMD Extensions 4.2 - ia32 + %assign _IPP _IPP_P8 +%elifdef _G9 ; Intel(R) Advanced Vector Extensions - ia32 + %assign IPP_ALIGN_FACTOR 32 + %assign _IPP _IPP_G9 +%elifdef _H9 ; Intel(R) Advanced Vector Extensions 2 - ia32 + %assign IPP_ALIGN_FACTOR 32 + %assign _IPP _IPP_H9 +%elifdef _S0 ; Intel(R) Advanced Vector Extensions 512 - Intel(R) Xeon(R) processor (formerly Skylake) - ia32 + %assign IPP_ALIGN_FACTOR 64 + %assign _IPP _IPP_S0 +%elifdef _M7 ; Intel(R) Streaming SIMD Extensions 3 - intel64 + %assign _IPP _IPP_PX + %assign _IPP32E _IPP32E_M7 +%elifdef _U8 ; Supplemental Streaming SIMD Extensions 3 (SSSE3) - intel64 + %assign _IPP _IPP_PX + %assign _IPP32E _IPP32E_U8 +%elifdef _N8 ; SSSE3 + MOVBE instruction - intel64 + %assign _IPP _IPP_PX + %assign _IPP32E _IPP32E_N8 +%elifdef _Y8 ; Intel(R) Streaming SIMD Extensions 4.2 - intel64 + %assign _IPP _IPP_PX + %assign _IPP32E _IPP32E_Y8 +%elifdef _E9 ; Intel(R) Advanced Vector Extensions - intel64 + %assign IPP_ALIGN_FACTOR 32 + %assign _IPP _IPP_PX + %assign _IPP32E _IPP32E_E9 +%elifdef _L9 ; Intel(R) Advanced Vector Extensions 2 - intel64 + %assign IPP_ALIGN_FACTOR 32 + %assign _IPP _IPP_PX + %assign _IPP32E _IPP32E_L9 +%elifdef _N0 ; Intel(R) Advanced Vector Extensions 512 (formerly Knights Landing) - intel64 + %assign IPP_ALIGN_FACTOR 64 + %assign _IPP _IPP_PX + %assign _IPP32E _IPP32E_N0 +%elifdef _K0 ; Intel(R) Advanced Vector Extensions 512 - Intel(R) Xeon(R) processor (formerly Skylake) - intel64 + %assign IPP_ALIGN_FACTOR 64 + %assign _IPP _IPP_PX + %assign _IPP32E _IPP32E_K0 +%elifdef _K1 ; Intel(R) Advanced Vector Extensions 512 - Intel(R) Xeon(R) processor (formerly Icelake) - intel64 + %assign IPP_ALIGN_FACTOR 64 + %assign _IPP _IPP_PX + %assign _IPP32E _IPP32E_K1 +%else + %assign _IPP _IPP_PX ; pure C-code +%endif + +%if (_IPP > _IPP_H9) || (_IPP32E > _IPP32E_L9) + %assign IPP_ALIGN_FACTOR 64 +%elif (_IPP > _IPP_P8) || (_IPP32E > _IPP32E_Y8) + %assign IPP_ALIGN_FACTOR 32 +%else + %assign IPP_ALIGN_FACTOR 16 +%endif + +; noexec stack +%ifdef LINUX32 + %ifndef OSX32 +section .note.GNU-stack noalloc noexec nowrite progbits + %endif +%endif + +; noexec stack +%ifdef LINUX32E + %ifndef OSXEM64T + %ifndef _ARCH_KNC +section .note.GNU-stack noalloc noexec nowrite progbits + %endif + %endif +%endif + + +%ifidn __OUTPUT_FORMAT__, elf32 + %assign IPP_BINARY_FORMAT 0 +%elifidn __OUTPUT_FORMAT__, elf64 + %assign IPP_BINARY_FORMAT 1 +%elifidn __OUTPUT_FORMAT__, macho64 + %assign IPP_BINARY_FORMAT 2 +%elifidn __OUTPUT_FORMAT__, win32 + %assign IPP_BINARY_FORMAT 3 +%elifidn __OUTPUT_FORMAT__, win64 + %assign IPP_BINARY_FORMAT 4 +%else + %fatal Unsupported output format: __OUTPUT_FORMAT__. Shall be: elf32, elf64, win32, win64, macho64 +%endif + +%ifdef _MERGED_BLD + %assign _OWN_MERGED_BLD 1 +%endif ; _MERGED_BLD + +; data compilation definitions: merged builds shall compile data only as +; part of one single object build to avoid multiple definition warnings at link time +%ifndef _MERGED_BLD + %assign _IPP_DATA 1 +%else + %if (_IPP == _IPP_G9) || (_IPP32E == _IPP32E_E9) + %assign _IPP_DATA 1 + %endif +%endif ; _MERGED_BLD + +; Definitions of sizeof(type) +%iassign ZWORD_size 64 ; zmm-word +%iassign YWORD_size 32 ; ymm-word +%iassign OWORD_size 16 ; octo-word +%iassign TWORD_size 10 ; ten-bytes word +%iassign QWORD_size 8 ; quad-word +%iassign DWORD_size 4 ; double-word +%iassign WORD_size 2 +%iassign BYTE_size 1 + +%idefine YMMWORD YWORD +%idefine XMMWORD OWORD +%iassign YMMWORD_size YWORD_size +%iassign XMMWORD_size OWORD_size + +%idefine sizeof(_x_) _x_%+_size + +%endif +;;;;;;;;;;;;;;;;;;;;;;;;;; End of file "asmdefs.inc" ;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/dispatcher.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/dispatcher.h new file mode 100644 index 0000000..8290df6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/dispatcher.h @@ -0,0 +1,293 @@ +/******************************************************************************* +* Copyright (C) 2009 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +// +// Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) +// + +#ifndef __DISPATCHER_H__ +#define __DISPATCHER_H__ + +#if defined( __cplusplus ) +extern "C" { +#endif + +/* + Intel IPP Cryptography libraries and CPU features mask fitness. Implemented only for IA32 and Intel64 (emt) +*/ + +#if defined( _ARCH_IA32 ) +#define M5_FM ( 0 ) +#ifdef _IPP_QUARK +#define PX_FM ( 0 ) +#else +#define PX_FM ( ippCPUID_MMX | ippCPUID_SSE ) +#endif +#define W7_FM ( PX_FM | ippCPUID_SSE2 ) +#define V8_FM ( W7_FM | ippCPUID_SSE3 | ippCPUID_SSSE3 ) +#define S8_FM ( V8_FM | ippCPUID_MOVBE ) +#define P8_FM ( V8_FM | ippCPUID_SSE41 | ippCPUID_SSE42 | ippCPUID_AES | ippCPUID_CLMUL | ippCPUID_SHA ) +#define G9_FM ( P8_FM | ippCPUID_AVX | ippAVX_ENABLEDBYOS | ippCPUID_RDRAND | ippCPUID_F16C ) +#define H9_FM ( G9_FM | ippCPUID_MOVBE | ippCPUID_AVX2 | ippCPUID_ADCOX | ippCPUID_RDSEED | ippCPUID_PREFETCHW ) +#define I0_FM ( H9_FM | ippCPUID_AVX512F | ippCPUID_AVX512CD | ippCPUID_AVX512PF | ippCPUID_AVX512ER | ippAVX512_ENABLEDBYOS ) +#define S0_FM ( H9_FM | ippCPUID_AVX512F | ippCPUID_AVX512CD | ippCPUID_AVX512VL | ippCPUID_AVX512BW | ippCPUID_AVX512DQ | ippAVX512_ENABLEDBYOS ) + +#elif defined (_ARCH_EM64T) + +#define PX_FM ( ippCPUID_MMX | ippCPUID_SSE | ippCPUID_SSE2 ) +#define M7_FM ( PX_FM | ippCPUID_SSE3 ) +#define U8_FM ( M7_FM | ippCPUID_SSSE3 ) +#define N8_FM ( U8_FM | ippCPUID_MOVBE ) +#define Y8_FM ( U8_FM | ippCPUID_SSE41 | ippCPUID_SSE42 | ippCPUID_AES | ippCPUID_CLMUL | ippCPUID_SHA ) +#define E9_FM ( Y8_FM | ippCPUID_AVX | ippAVX_ENABLEDBYOS | ippCPUID_RDRAND | ippCPUID_F16C ) +#define L9_FM ( E9_FM | ippCPUID_MOVBE | ippCPUID_AVX2 | ippCPUID_ADCOX | ippCPUID_RDSEED | ippCPUID_PREFETCHW ) +#define N0_FM ( L9_FM | ippCPUID_AVX512F | ippCPUID_AVX512CD | ippCPUID_AVX512PF | ippCPUID_AVX512ER | ippAVX512_ENABLEDBYOS ) +#define K0_FM ( L9_FM | ippCPUID_AVX512F | ippCPUID_AVX512CD | ippCPUID_AVX512VL | ippCPUID_AVX512BW | ippCPUID_AVX512DQ | ippAVX512_ENABLEDBYOS ) + +#elif defined (_ARCH_LRB2) + #define PX_FM ( ippCPUID_KNC ) +#else + #error undefined architecture +#endif + +#define PX_MSK ( 0 ) +#define MMX_MSK ( ippCPUID_MMX ) +#define SSE_MSK ( MMX_MSK | ippCPUID_SSE ) +#define SSE2_MSK ( SSE_MSK | ippCPUID_SSE2 ) +#define SSE3_MSK ( SSE2_MSK | ippCPUID_SSE3 ) +#define SSSE3_MSK ( SSE3_MSK | ippCPUID_SSSE3 ) +#define ATOM_MSK ( SSE3_MSK | ippCPUID_SSSE3 | ippCPUID_MOVBE ) +#define SSE41_MSK ( SSSE3_MSK | ippCPUID_SSE41 ) +#define SSE42_MSK ( SSE41_MSK | ippCPUID_SSE42 ) +#define AVX_MSK ( SSE42_MSK | ippCPUID_AVX ) +#define AVX2_MSK ( AVX_MSK | ippCPUID_AVX2 ) +#define AVX3X_MSK ( AVX2_MSK | ippCPUID_AVX512F | ippCPUID_AVX512CD | ippCPUID_AVX512VL | ippCPUID_AVX512BW | ippCPUID_AVX512DQ ) +#define AVX3M_MSK ( AVX2_MSK | ippCPUID_AVX512F | ippCPUID_AVX512CD | ippCPUID_AVX512PF | ippCPUID_AVX512ER ) +#define AVX3I_MSK ( AVX3X_MSK | ippCPUID_SHA | ippCPUID_AVX512VBMI | ippCPUID_AVX512VBMI2 | ippCPUID_AVX512IFMA | ippCPUID_AVX512GFNI | ippCPUID_AVX512VAES | ippCPUID_AVX512VCLMUL ) + +#if defined( _ARCH_IA32 ) && !defined( OSX32 ) + enum lib_enum { + LIB_W7=0, LIB_S8=1, LIB_P8=2, LIB_G9=3, LIB_H9=4, LIB_NOMORE + }; + #define LIB_PX LIB_W7 +#elif defined( OSX32 ) + enum lib_enum { + LIB_S8=0, LIB_P8=1, LIB_G9=2, LIB_H9=3, LIB_NOMORE + }; + #define LIB_PX LIB_S8 + #define LIB_W7 LIB_S8 +#elif defined( _ARCH_EM64T ) && !defined( OSXEM64T ) && !defined( WIN32E ) /* Linux* OS Intel64 supports N0 */ + enum lib_enum { + LIB_M7=0, LIB_N8=1, LIB_Y8=2, LIB_E9=3, LIB_L9=4, LIB_N0=5, LIB_K0=6, LIB_K1=7,LIB_NOMORE + }; + #define LIB_PX LIB_M7 +#elif defined( _ARCH_EM64T ) && !defined( OSXEM64T ) /* Windows* OS Intel64 doesn't support N0 */ + enum lib_enum { + LIB_M7=0, LIB_N8=1, LIB_Y8=2, LIB_E9=3, LIB_L9=4, LIB_K0=5, LIB_K1=6, LIB_NOMORE + }; + #define LIB_PX LIB_M7 + #define LIB_N0 LIB_L9 +#elif defined( OSXEM64T ) + enum lib_enum { + LIB_Y8=0, LIB_E9=1, LIB_L9=2, LIB_K0=3, LIB_K1=4, LIB_NOMORE + }; + #define LIB_PX LIB_Y8 + #define LIB_M7 LIB_Y8 + #define LIB_N8 LIB_Y8 + #define LIB_N0 LIB_L9 +#elif defined( _ARCH_LRB2 ) + enum lib_enum { + LIB_PX=0, LIB_B2=1, LIB_NOMORE + }; + #define LIB_MIC LIB_B2 +#else + #error "lib_enum isn't defined!" +#endif + +#if defined( _ARCH_IA32 ) + #if defined( OSX32 ) /* OSX supports starting with Intel® architecture formerly codenamed Penryn only */ + #define LIB_MMX LIB_S8 + #define LIB_SSE LIB_S8 + #define LIB_SSE2 LIB_S8 + #define LIB_SSE3 LIB_S8 + #define LIB_ATOM LIB_S8 + #define LIB_SSSE3 LIB_S8 + #define LIB_SSE41 LIB_S8 + #define LIB_SSE42 LIB_P8 + #define LIB_AVX LIB_G9 + #define LIB_AVX2 LIB_H9 + #define LIB_AVX3M LIB_H9 /* no ia32 library for Intel® Xeon® Phi(TM) processor (formerly Knight Landing) */ + #define LIB_AVX3X LIB_H9 /* no ia32 library for Intel® Xeon® processor (formerly Skylake) */ + #define LIB_AVX3I LIB_H9 /* no ia32 library for Intel® Xeon® processor (formerly Icelake) */ +#else + #define LIB_MMX LIB_W7 + #define LIB_SSE LIB_W7 + #define LIB_SSE2 LIB_W7 + #define LIB_SSE3 LIB_W7 + #define LIB_ATOM LIB_S8 + #define LIB_SSSE3 LIB_S8 + #define LIB_SSE41 LIB_S8 /* P8 is oriented for new Intel Atom® processor (formerly Silvermont) */ + #define LIB_SSE42 LIB_P8 + #define LIB_AVX LIB_G9 + #define LIB_AVX2 LIB_H9 + #define LIB_AVX3M LIB_H9 /* no ia32 library for Intel® Xeon® Phi(TM) processor (formerly Knight Landing) */ + #define LIB_AVX3X LIB_H9 /* no ia32 library for Intel® Xeon® processor (formerly Skylake) */ + #define LIB_AVX3I LIB_H9 /* no ia32 library for Intel® Xeon® processor (formerly Icelake) */ +#endif +#elif defined (_ARCH_EM64T) + #if defined( OSXEM64T ) /* OSX supports starting PNR only */ + #define LIB_MMX LIB_N8 + #define LIB_SSE LIB_N8 + #define LIB_SSE2 LIB_N8 + #define LIB_SSE3 LIB_N8 + #define LIB_ATOM LIB_N8 + #define LIB_SSSE3 LIB_N8 + #define LIB_SSE41 LIB_N8 + #define LIB_SSE42 LIB_Y8 + #define LIB_AVX LIB_E9 + #define LIB_AVX2 LIB_L9 + #define LIB_AVX3M LIB_L9 + #define LIB_AVX3X LIB_K0 + #define LIB_AVX3I LIB_K1 +#else + #define LIB_MMX LIB_M7 + #define LIB_SSE LIB_M7 + #define LIB_SSE2 LIB_M7 + #define LIB_SSE3 LIB_M7 + #define LIB_ATOM LIB_N8 + #define LIB_SSSE3 LIB_N8 + #define LIB_SSE41 LIB_N8 /* Y8 is oriented for new Intel Atom® processor (formerly Silvermont) */ + #define LIB_SSE42 LIB_Y8 + #define LIB_AVX LIB_E9 + #define LIB_AVX2 LIB_L9 + #define LIB_AVX3M LIB_N0 + #define LIB_AVX3X LIB_K0 + #define LIB_AVX3I LIB_K1 +#endif +#elif defined (_ARCH_LRB2) + #define LIB_MIC LIB_B2 +#endif + +//gres: #if defined( _IPP_DYNAMIC ) +#if defined( _PCS ) +#if defined( _ARCH_IA32 ) + +/* Describe Intel CPUs and libraries */ +typedef enum{CPU_W7=0, CPU_S8, CPU_P8, CPU_G9, CPU_H9, CPU_NOMORE} cpu_enum; +typedef enum{DLL_W7=0, DLL_S8, DLL_P8, DLL_G9, DLL_H9, DLL_NOMORE} dll_enum; + +/* New cpu can use some libraries for old cpu */ +static const dll_enum dllUsage[][DLL_NOMORE+1] = { + /* DLL_H9, DLL_G9, DLL_P8, DLL_S8, DLL_W7, DLL_NOMORE */ +/*CPU_W7*/ { DLL_W7, DLL_NOMORE }, +/*CPU_S8*/ { DLL_S8, DLL_W7, DLL_NOMORE }, +/*CPU_P8*/ { DLL_P8, DLL_S8, DLL_W7, DLL_NOMORE }, +/*CPU_G9*/ { DLL_G9, DLL_P8, DLL_S8, DLL_W7, DLL_NOMORE }, +/*CPU_H9*/ { DLL_H9, DLL_G9, DLL_P8, DLL_S8, DLL_W7, DLL_NOMORE } +}; + +#elif defined (_ARCH_EM64T) +/* Describe Intel CPUs and libraries */ +typedef enum{CPU_M7=0, CPU_N8, CPU_Y8, CPU_E9, CPU_L9, CPU_N0, CPU_K0, CPU_K1, CPU_NOMORE} cpu_enum; +typedef enum{DLL_M7=0, DLL_N8, DLL_Y8, DLL_E9, DLL_L9, DLL_N0, DLL_K0, DLL_K1, DLL_NOMORE} dll_enum; + +/* New cpu can use some libraries for old cpu */ +static const dll_enum dllUsage[][DLL_NOMORE+1] = { + /* DLL_K1, DLL_K0, DLL_N0, DLL_L9, DLL_E9, DLL_Y8, DLL_N8, DLL_M7, DLL_NOMORE */ +/*CPU_M7*/ { DLL_M7, DLL_NOMORE }, +/*CPU_N8*/ { DLL_N8, DLL_M7, DLL_NOMORE }, +/*CPU_Y8*/ { DLL_Y8, DLL_N8, DLL_M7, DLL_NOMORE }, +/*CPU_E9*/ { DLL_E9, DLL_Y8, DLL_N8, DLL_M7, DLL_NOMORE }, +/*CPU_L9*/ { DLL_L9, DLL_E9, DLL_Y8, DLL_N8, DLL_M7, DLL_NOMORE }, +/*CPU_N0*/ { DLL_N0, DLL_L9, DLL_E9, DLL_Y8, DLL_N8, DLL_M7, DLL_NOMORE }, +/*CPU_K0*/ { DLL_K0, DLL_N0, DLL_L9, DLL_E9, DLL_Y8, DLL_N8, DLL_M7, DLL_NOMORE } +/*CPU_K1*/ { DLL_K1, DLL_K0, DLL_N0, DLL_L9, DLL_E9, DLL_Y8, DLL_N8, DLL_M7, DLL_NOMORE } +}; + +#endif + +#if defined( _PCS ) + +/* Names of the Intel libraries which can be loaded */ +#if defined ( WIN32 ) +static const _TCHAR* dllNames[DLL_NOMORE] = { + _T(IPP_LIB_PREFIX()) _T("w7") _T(".dll"), + _T(IPP_LIB_PREFIX()) _T("s8") _T(".dll"), + _T(IPP_LIB_PREFIX()) _T("p8") _T(".dll"), + _T(IPP_LIB_PREFIX()) _T("g9") _T(".dll"), + _T(IPP_LIB_PREFIX()) _T("h9") _T(".dll") +}; +#elif defined(LINUX32) +static const _TCHAR* dllNames[DLL_NOMORE] = { + _T("lib") _T(IPP_LIB_PREFIX()) _T("w7.so"), + _T("lib") _T(IPP_LIB_PREFIX()) _T("s8.so"), + _T("lib") _T(IPP_LIB_PREFIX()) _T("p8.so"), + _T("lib") _T(IPP_LIB_PREFIX()) _T("g9.so"), + _T("lib") _T(IPP_LIB_PREFIX()) _T("h9.so") +}; +#elif defined( OSX32 ) +static const _TCHAR* dllNames[DLL_NOMORE] = { + _T("lib") _T(IPP_LIB_PREFIX()) _T("s8") _T(".dylib"), + _T("lib") _T(IPP_LIB_PREFIX()) _T("p8") _T(".dylib"), + _T("lib") _T(IPP_LIB_PREFIX()) _T("g9") _T(".dylib"), + _T("lib") _T(IPP_LIB_PREFIX()) _T("h9") _T(".dylib") +}; +#elif defined( WIN32E ) +static const _TCHAR* dllNames[DLL_NOMORE] = { + _T(IPP_LIB_PREFIX()) _T("m7") _T(".dll"), + _T(IPP_LIB_PREFIX()) _T("n8") _T(".dll"), + _T(IPP_LIB_PREFIX()) _T("y8") _T(".dll"), + _T(IPP_LIB_PREFIX()) _T("e9") _T(".dll"), + _T(IPP_LIB_PREFIX()) _T("l9") _T(".dll"), /* no support for N0 on win */ + _T(IPP_LIB_PREFIX()) _T("k0") _T(".dll"), + _T(IPP_LIB_PREFIX()) _T("k1") _T(".dll") +}; +#elif defined( OSXEM64T ) +static const _TCHAR* dllNames[DLL_NOMORE] = { + _T("lib") _T(IPP_LIB_PREFIX()) _T("n8") _T(".dylib"), + _T("lib") _T(IPP_LIB_PREFIX()) _T("y8") _T(".dylib"), + _T("lib") _T(IPP_LIB_PREFIX()) _T("e9") _T(".dylib"), + _T("lib") _T(IPP_LIB_PREFIX()) _T("l9") _T(".dylib"), + _T("lib") _T(IPP_LIB_PREFIX()) _T("k0") _T(".dylib"), + _T("lib") _T(IPP_LIB_PREFIX()) _T("k1") _T(".dylib") +}; +#elif defined( LINUX32E ) +static const _TCHAR* dllNames[DLL_NOMORE] = { + _T("lib") _T(IPP_LIB_PREFIX()) _T("m7.so"), + _T("lib") _T(IPP_LIB_PREFIX()) _T("n8.so"), + _T("lib") _T(IPP_LIB_PREFIX()) _T("y8.so"), + _T("lib") _T(IPP_LIB_PREFIX()) _T("e9.so"), + _T("lib") _T(IPP_LIB_PREFIX()) _T("l9.so"), + _T("lib") _T(IPP_LIB_PREFIX()) _T("n0.so"), + _T("lib") _T(IPP_LIB_PREFIX()) _T("k0.so"), + _T("lib") _T(IPP_LIB_PREFIX()) _T("k1.so") +}; +#endif + +#endif /* _PCS */ + +#else /*_IPP_DYNAMIC */ + + +#endif + + +#if defined( __cplusplus ) +} +#endif + +#endif /* __DISPATCHER_H__ */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/ia_32e.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/ia_32e.inc new file mode 100644 index 0000000..fecfe44 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/ia_32e.inc @@ -0,0 +1,532 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + + +%include "asmdefs.inc" +%include "ia_common.inc" +%include "utils.inc" + +%ifndef __IA_32E_INC__ +%define __IA_32E_INC__ 1 + +%ifndef LINUX32E + %ifndef WIN32E + %fatal \ + LINUX32E or WIN32E - Linux ABI (parameter passing in rdi, rsi, rdx, rcx, r8, r9...) + %endif +%endif + +; Force RIP-relative addressing +; default rel + +%ifdef LINUX32E + %ifdef STACK_ABI + %assign IPP_ABI 2 + %else + %assign IPP_ABI 3 + %endif +%endif + +%ifdef WIN32E + %ifdef STACK_ABI + %assign IPP_ABI 1 + %else + %assign IPP_ABI 0 + %endif +%endif + +; Decorates function name with appropriate CPU prefix (for the merged library). +; The macro is context-dependent and returns decorated name in the %$decorated_func_name +; context variable along with the decoration length in the %$decoration_length context variable. +%macro CPU_PREFIX_DECORATE 1.nolist + %ifnctx _CPU_PREFIX_DECORATE_CTX_ + %fatal "Not in the context: _CPU_PREFIX_DECORATE_CTX_" + %endif + + ; Add CPU-specific suffix for the dispatched library + %ifdef _OWN_MERGED_BLD + %if (_IPP32E == _IPP32E_PX) + %xdefine %%func_name mx_%1 + %assign %%decoration_length 3 + %endif + %if (_IPP32E == _IPP32E_M7) + %xdefine %%func_name m7_%1 + %assign %%decoration_length 3 + %endif + %if (_IPP32E == _IPP32E_U8) + %xdefine %%func_name u8_%1 + %assign %%decoration_length 3 + %endif + %if (_IPP32E == _IPP32E_N8) + %xdefine %%func_name n8_%1 + %assign %%decoration_length 3 + %endif + %if (_IPP32E == _IPP32E_Y8) + %xdefine %%func_name y8_%1 + %assign %%decoration_length 3 + %endif + %if (_IPP32E == _IPP32E_E9) + %xdefine %%func_name e9_%1 + %assign %%decoration_length 3 + %endif + %if (_IPP32E == _IPP32E_L9) + %xdefine %%func_name l9_%1 + %assign %%decoration_length 3 + %endif + %if (_IPP32E == _IPP32E_N0) + %xdefine %%func_name n0_%1 + %assign %%decoration_length 3 + %endif + %if (_IPP32E == _IPP32E_K0) + %xdefine %%func_name k0_%1 + %assign %%decoration_length 3 + %endif + %if (_IPP32E == _IPP32E_K1) + %xdefine %%func_name k1_%1 + %assign %%decoration_length 3 + %endif + %else + %xdefine %%func_name %1 + %assign %%decoration_length 0 + %endif + + %ifndef %%func_name + %fatal "CPU_PREFIX_DECORATE: unknown decoration for : _IPP32E = " _IPP32E + %endif + %xdefine %$decorated_func_name %[%%func_name] + %assign %$decoration_length %%decoration_length +%endmacro + +; Lists of non-volatile registers that needs to be preserved in a function call +%define NONVOLATILE_REGS_LIN64_GPR rbx,rbp,r12,r13,r14,r15 +%define NONVOLATILE_REGS_WIN64_GPR rbx,rbp,r12,r13,r14,r15,rsi,rdi +%define NONVOLATILE_REGS_WIN64_XMM xmm6,xmm7,xmm8,xmm9,xmm10,xmm11,xmm12,xmm13,xmm14,xmm15 +%define NONVOLATILE_REGS_WIN64_YMM ymm6,ymm7,ymm8,ymm9,ymm10,ymm11,ymm12,ymm13,ymm14,ymm15 + +; Saves non-volatile GPR registers on stack. +; Input - list of used registers. +%macro USES_GPR 1+.nolist + %assign GPR_FRAME 0 + %define GPR_CUR + + %if (IPP_ABI < 2) ; Win64 + %define %%nonvolatile_regs_list %[NONVOLATILE_REGS_WIN64_GPR] + %else ; Lin64 + %define %%nonvolatile_regs_list %[NONVOLATILE_REGS_LIN64_GPR] + %endif + BEGIN_INTERSECT + INTERSECT {%1},{%%nonvolatile_regs_list} + ; List of non-volatile GPR registers in the order they will be pushed on stack + %xdefine GPR_CUR %[%$intersection] + %assign GPR_FRAME %$cardinality * 8 + END_INTERSECT + + ; Push non-volatile GPRs on stack + FOREACH GPR_CUR,{push} +%endmacro + +; Restore preliminary saved by USES_GPR non-volatile GPR registers from the stack. +; The macro shall be called after function processing. +%macro REST_GPR 0.nolist + %ifndef GPR_CUR + %fatal "REST_GPR: no GPR_CUR defined" + %endif + + ; Pop saved registers from the stack + RFOREACH GPR_CUR,{pop} +%endmacro + +; Saves XMM register on stack (SSE version). +; An offset from RSP, where the register will be saved, shall be provided in the calling context. +%macro PUSH_XMM_REG 1.nolist + movdqa [rsp + %$rsp_offset], %1 + %assign %$rsp_offset %$rsp_offset + 16 +%endmacro + +; Saves non-volatile XMM registers on stack and allocates stack size for +; local variables if needed. +; Input - list of used registers (can be empty). +%macro USES_XMM 0-*.nolist + ; LOCAL_FRAME - stack size required for all local variables of the procedure. Shall be defined before USES_XMM macro call if + ; local variables are used in the procedure. + %ifndef LOCAL_FRAME + %assign LOCAL_FRAME 0 + %endif + ; to align size for local variables size on 16-bytes + %assign LOCAL_FRAME (LOCAL_FRAME + 15) & (-16) + %assign S_FRAME 0 + %define XMM_CUR + + ; Convert parameters to the list variable if there are arguments + %if (%0 > 0) + %xdefine %%param_list %1 + %rotate 1 + %rep %0-1 + %xdefine %%param_list %[%%param_list],%1 + %rotate 1 + %endrep + %endif + + %if (IPP_ABI < 2) ; Win64 + %assign %%T_FRAME 0 + BEGIN_INTERSECT + INTERSECT {%%param_list},{%[NONVOLATILE_REGS_WIN64_XMM]} + %define XMM_CUR %[%$intersection] + %assign %%T_FRAME %$cardinality * 16 + END_INTERSECT + + ; Adjust offset depending on function frame + %if ((%%T_FRAME > 0) || (LOCAL_FRAME > 0)) + %assign S_FRAME %%T_FRAME + LOCAL_FRAME + %if (((S_FRAME + GPR_FRAME ) & 8) == 0) + %assign S_FRAME S_FRAME + 8 + %endif + %endif + + ; Allocate space on stack and push XMM registers + %if (S_FRAME > 0) + sub rsp, S_FRAME + %push %?? + %assign %$rsp_offset LOCAL_FRAME + FOREACH %[XMM_CUR],{PUSH_XMM_REG} + %pop %?? + %endif + %else + ; Linux x86_64 ABI does not count MM registers as non-volatile, so they do not need to be + ; preserved, so just allocate stack space for local variables and duplicated register parameters if needed. + %if (IPP_ABI == 2) ; LINUX32S + %assign S_FRAME LOCAL_FRAME + 48 ; 48 = 6 * 8 - stack frame for 6 register inputs + %if (((S_FRAME + GPR_FRAME) & 8 ) == 0) + %assign S_FRAME S_FRAME + 8 + %endif + %assign INP_FRAME S_FRAME - 48 ; for Linux32s-key stack-frame for 6 registers inputs + %else ; LINUX32E + %if (LOCAL_FRAME > 0) + %assign S_FRAME LOCAL_FRAME + %if (((S_FRAME + GPR_FRAME) & 8 ) == 0) + %assign S_FRAME S_FRAME + 8 + %endif + %endif + %endif + %if (S_FRAME > 0) + sub rsp, S_FRAME + %endif + %endif +%endmacro + +; Pop input list of XMM registers from the stack. +; The offset from RSP, where the registers will be taken from, shall be provided in the calling context. +%macro POP_XMM_REG 1.nolist + movdqa %1, [rsp + %$rsp_offset] + %assign %$rsp_offset %$rsp_offset + 16 +%endmacro + +; Restore preliminary saved by USES_XMM non-volatile XMM registers from the stack. +; The macro shall be called after function processing. +%macro REST_XMM 0.nolist + %if (IPP_ABI < 2) + %if (S_FRAME > 0) + %push %?? + %assign %$rsp_offset LOCAL_FRAME + FOREACH %[XMM_CUR],{POP_XMM_REG} + %pop %?? + %endif + %endif + %if (S_FRAME > 0) + add rsp, S_FRAME + %endif + %if (_IPP32E >= _IPP32E_E9) + %if (_IPP32E != _IPP32E_N0) + vzeroupper + %endif + %endif +%endmacro + +; Saves XMM or YMM register on stack (AVX version). +; An offset from RSP, where the registers will be saved, shall be provided in the calling context. +%macro PUSH_XMM_AVX_REG 1.nolist + ; Process registers depending on type + %defstr %%reg_str %1 ; convert register token to the string + %substr %%reg_type %%reg_str 1, 3 ; take first 3 elems of the string + %ifidni %%reg_type, 'xmm' + vmovdqa oword [rsp + %$rsp_offset], %1 + %assign %$rsp_offset %$rsp_offset + 16 + %elifidni %%reg_type, 'ymm' + vmovdqu ymmword [rsp + %$rsp_offset], %1 + %assign %$rsp_offset %$rsp_offset + 32 + %else + %fatal PUSH_XMM_AVX_REG: inconsistent usage - only XMM/YMM registers supported, found: %%reg_type + %endif +%endmacro + +; Saves non-volatile XMM/YMM registers on stack and allocates stack size for +; local variables if needed (AVX version). +; Input - list of used registers (can be empty). +%macro USES_XMM_AVX 0-*.nolist + ; LOCAL_FRAME - stack size required for all local variables of the procedure. Shall be defined before USES_XMM macro call if + ; local variables are used in the procedure. + %ifndef LOCAL_FRAME + %assign LOCAL_FRAME 0 + %endif + ; to align size for local variables size on 16-bytes + %assign LOCAL_FRAME (LOCAL_FRAME + 15) & (-16) + %assign S_FRAME 0 + %define XMM_CUR + %define YMM_CUR + + ; Convert parameters to the list variable if there are arguments + %if (%0 > 0) + %xdefine %%param_list %1 + %rotate 1 + %rep %0-1 + %xdefine %%param_list %[%%param_list],%1 + %rotate 1 + %endrep + %endif + + %if (IPP_ABI < 2) ; Win64 + %assign %%T_FRAME 0 + ; Process XMM registers + BEGIN_INTERSECT + INTERSECT {%%param_list},{%[NONVOLATILE_REGS_WIN64_XMM]} + %define XMM_CUR %[%$intersection] + %assign %%T_FRAME %$cardinality * 16 + END_INTERSECT + + ; Process YMM registers + BEGIN_INTERSECT + INTERSECT {%%param_list},{%[NONVOLATILE_REGS_WIN64_YMM]} + %define YMM_CUR %[%$intersection] + %assign %%T_FRAME %%T_FRAME + %$cardinality * 32 + END_INTERSECT + + ; Adjust offset depending on function frame + %if ((%%T_FRAME > 0) || (LOCAL_FRAME > 0)) + %assign S_FRAME %%T_FRAME + LOCAL_FRAME + %if (((S_FRAME + GPR_FRAME ) & 8) == 0) + %assign S_FRAME S_FRAME + 8 + %endif + %endif + + ; Allocate space on stack and push registers + %if (S_FRAME > 0) + sub rsp, S_FRAME + CONCATENATE {%[XMM_CUR]},{%[YMM_CUR]},%%XMM_YMM_CUR + %push %?? + %assign %$rsp_offset LOCAL_FRAME + FOREACH %[%%XMM_YMM_CUR],{PUSH_XMM_AVX_REG} + %pop %?? + %endif + %else + ; Linux x86_64 ABI does not count MM registers as non-volatile, so they do not need to be + ; preserved, so just allocate stack space for local variables and duplicated register parameters if needed. + %if (IPP_ABI == 2) ; LINUX32S + %assign S_FRAME LOCAL_FRAME + 48 ; 48 = 6 * 8 - stack frame for 6 register inputs + %if (((S_FRAME + GPR_FRAME) & 8 ) == 0) + %assign S_FRAME S_FRAME + 8 + %endif + %assign INP_FRAME S_FRAME - 48 ; for Linux32s-key stack-frame for 6 registers inputs + %else ; LINUX32E + %if (LOCAL_FRAME > 0) + %assign S_FRAME LOCAL_FRAME + %if (((S_FRAME + GPR_FRAME) & 8 ) == 0) + %assign S_FRAME S_FRAME + 8 + %endif + %endif + %endif + %if (S_FRAME > 0) + sub rsp, S_FRAME + %endif + %endif +%endmacro + +; Pop XMM or YMM register from the stack. +; The offset from RSP, where the registers will be taken from, shall be provided in the calling context. +%macro POP_XMM_AVX_REG 1.nolist + ; Process registers depending on type + %defstr %%reg_str %1 ; convert register token to the string + %substr %%reg_type %%reg_str 1, 3 ; take first 3 elems of the string + %ifidni %%reg_type, 'xmm' + vmovdqa %1, oword [rsp + %$rsp_offset] + %assign %$rsp_offset %$rsp_offset + 16 + %elifidni %%reg_type, 'ymm' + vmovdqu %1, ymmword [rsp + %$rsp_offset] + %assign %$rsp_offset %$rsp_offset + 32 + %else + %fatal POP_XMM_AVX_REG: inconsistent usage - only XMM/YMM registers supported, found: %%reg_type + %endif +%endmacro + +; Restore preliminary saved by USES_XMM_AVX non-volatile XMM/YMM registers from the stack. +; The macro shall be called after function processing. +%macro REST_XMM_AVX 0.nolist + %if (IPP_ABI < 2) + %if (S_FRAME > 0) + CONCATENATE {%[XMM_CUR]},{%[YMM_CUR]},%%XMM_YMM_CUR + %push %?? + %assign %$rsp_offset LOCAL_FRAME + FOREACH %[%%XMM_YMM_CUR],{POP_XMM_AVX_REG} + %pop %?? + %endif + %endif + %if (S_FRAME > 0) + add rsp, S_FRAME + %endif + %if (_IPP32E != _IPP32E_N0) + vzeroupper + %endif +%endmacro + +; Helper macro to align different ABIs parameters for uniform usage +%macro COMP_ABI 1.nolist + %if (IPP_ABI == 0) ;; if defined WIN32E + %if (%1 > 0) + mov rdi, rcx ;; ARG_1 + %endif + %if (%1 > 1) + mov rsi, rdx ;; ARG_2 + %endif + %if (%1 > 2) + mov rdx, r8 ;; ARG_3 + %endif + %if (%1 > 3) + mov rcx, r9 ;; ARG_4 + %endif + %if (%1 > 4) + mov r8, [rsp + S_FRAME + GPR_FRAME + 40] ;; ARG_5 + %endif + %if (%1 > 5) + mov r9, [rsp + S_FRAME + GPR_FRAME + 48] ;; ARG_6 + %endif + %if (%1 > 6) + %assign FIRST_P S_FRAME + GPR_FRAME + 56 ;; ARG_7 + %assign ARG_7 S_FRAME + GPR_FRAME + 56 + %endif + %endif + %if (IPP_ABI == 1) ;; if defined WIN32S + %assign FIRST_P S_FRAME + GPR_FRAME + 8 + %if (%1 > 0) + mov [rsp + FIRST_P],rcx + %assign ARG_1 FIRST_P + %endif + %if (%1 > 1) + mov [rsp + FIRST_P + 8],rdx + %assign ARG_2 ARG_1 + 8 + %endif + %if (%1 > 2) + mov [rsp + FIRST_P + 16],r8 + %assign ARG_3 ARG_2 + 8 + %endif + %if (%1 > 3) + mov [rsp + FIRST_P + 24],r9 + %assign ARG_4 ARG_3 + 8 + %endif + %if (%1 > 4) + %assign ARG_5 ARG_4 + 8 + %endif + %if (%1 > 5) + %assign ARG_6 ARG_5 + 8 + %endif + %if (%1 > 6) + %assign ARG_7 ARG_6 + 8 ;; ARG_7 + %endif + %endif + %if (IPP_ABI == 2) ;; if defined LINUX32S + %assign FIRST_P INP_FRAME + %if (%1 > 0) + mov [rsp + FIRST_P],rdi + %assign ARG_1 FIRST_P + %endif + %if (%1 > 1) + mov [rsp + FIRST_P + 8],rsi + %assign ARG_2 ARG_1 + 8 + %endif + %if (%1 > 2) + mov [rsp + FIRST_P + 16],rdx + %assign ARG_3 ARG_2 + 8 + %endif + %if (%1 > 3) + mov [rsp + FIRST_P + 24],rcx + %assign ARG_4 ARG_3 + 8 + %endif + %if (%1 > 4) + mov [rsp + FIRST_P + 32],r8 + %assign ARG_5 ARG_4 + 8 + %endif + %if (%1 > 5) + mov [rsp + FIRST_P + 40],r9 + %assign ARG_6 ARG_5 + 8 + %endif + %if (%1 > 6) + %assign ARG_7 S_FRAME+GPR_FRAME+8 + %endif + %endif + %if (IPP_ABI == 3) + %if (%1 > 6) ;; ARG_1 = rdi ARG_2 = rsi ARG_3 = rdx ARG_4 = rcx ARG_5 = r8 ARG_6 = r9 + %assign FIRST_P S_FRAME + GPR_FRAME + 8 ;; ARG_7 + %assign ARG_7 S_FRAME + GPR_FRAME + 8 + %endif + %endif + %if (%1 > 7) + %assign ARG_8 ARG_7 + 8 ;; ARG_8 + %endif + %if (%1 > 8) + %assign ARG_9 ARG_8 + 8 ;; ARG_9 + %endif + %if (%1 > 9) + %assign ARG_10 ARG_9 + 8 ;; ARG_10 + %endif + %if (%1 > 10) + %assign ARG_11 ARG_10 + 8 ;; ARG_11 + %endif + %if (%1 > 11) + %assign ARG_12 ARG_11 + 8 ;; ARG_12 + %endif + %if (%1 > 12) + %assign ARG_13 ARG_12 + 8 ;; ARG_13 + %endif + %if (%1 > 13) + %assign ARG_14 ARG_13 + 8 ;; ARG_14 + %endif + %if (%1 > 14) + %assign ARG_15 ARG_14 + 8 ;; ARG_15 + %endif + %if (%1 > 15) + %assign ARG_16 ARG_15 + 8 ;; ARG_16 + %endif + %if (%1 > 16) + %assign ARG_17 ARG_16 + 8 ;; ARG_17 + %endif + %if (%1 > 17) + %assign ARG_18 ARG_17 + 8 ;; ARG_18 + %endif +%endmacro + +%macro LD_ADDR 2.nolist + %xdefine %%reg %1 + %xdefine %%addr %2 + +%ifdef xxIPP_PIC + call %%LABEL +%%LABEL: pop %%reg + sub %%reg, %%LABEL-%%addr +%else + lea %%reg, [%%addr] +%endif +%endmacro + +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/ia_common.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/ia_common.inc new file mode 100644 index 0000000..0598056 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/ia_common.inc @@ -0,0 +1,186 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +%ifndef __IA_COMMON_INC__ +%define __IA_COMMON_INC__ 1 + +; use multi-byte nop's sequences to align loops and jmp's when threshold is reached +%use smartalign +ALIGNMODE p6,16 + +; Declares function, sets visibility and binding and adds __cdecl decoration when needed. +%macro DECLARE_FUNC 2-3.nolist + %xdefine %%func_name %1 + %xdefine %%visibility %2 + %xdefine %%binding %3 + + %ifctx _DECLARE_FUNC_CTX_ + %fatal "DECLARE_FUNC: already in the context, need to call ENDFUNC" + %endif + + ; Accepted visibility values are PUBLIC and PRIVATE + %ifnidni %%visibility, PUBLIC + %ifnidni %%visibility, PRIVATE + %fatal Function %%func_name visibility is not properly defined. Shall be: PRIVATE or PUBLIC. + %endif + %endif + + ; Accepted binding values are WEAK or STRONG (default) + %ifnempty %%binding + %ifnidni %%binding, WEAK + %ifnidni %%binding, STRONG + %%fatal Function %%func_name binding is not properly defined. Shall be: WEAK or STRONG. + %endif + %endif + %endif + + ; Function decoration length + %assign %%decoration_length 0 + + ; The __cdecl calling convention name decoration (to have interoperability with C). + ; Only public functions are decorated + %ifidni %%visibility, PUBLIC + %if ((IPP_BINARY_FORMAT == 2) || (IPP_BINARY_FORMAT == 3)) ; WIN32 or OSXEM64T + %xdefine %%func_name _%[%%func_name] + %assign %%decoration_length %%decoration_length+1 + %endif + %endif + + ; If current macro is called from IPPASM macro, then function might be decorated by CPU-prefix + %ifctx _IPPASM_CTX_ + %assign %%decoration_length %%decoration_length + %$decoration_length ; %$decoration_length belongs to _IPPASM_CTX_ + %endif + + %push _DECLARE_FUNC_CTX_ + ; setup context variables to use in ENDFUNC + %xdefine %$func_name_ctx %%func_name + %assign %$decoration_length %%decoration_length ; %$decoration_length belongs to _DECLARE_FUNC_CTX_ + + %ifidn %%visibility, PUBLIC + %if (IPP_BINARY_FORMAT < 2) ; LINUX32 or LINUX32E + %ifnempty %%binding + global %%func_name:function %%binding (%%func_name%+.LEnd_%+%%func_name - %%func_name) + %else + global %%func_name:function (%%func_name%+.LEnd_%+%%func_name - %%func_name) + %endif + %else + global %%func_name + %endif + %endif + %%func_name: + + ; CET enabling (macOS not supported) + %if ((IPP_BINARY_FORMAT == 0) || (IPP_BINARY_FORMAT == 3)) ; elf32/win32 + db 0F3h, 00Fh, 01Eh, 0FBh ; endbr32 + %elif ((IPP_BINARY_FORMAT == 1) || (IPP_BINARY_FORMAT == 4)) ; elf64/win64 + db 0F3h, 00Fh, 01Eh, 0FAh ; endbr64 + %endif +%endmacro + +; Calls assembler function declared by DECLARE_FUNC +; Default visibility is PRIVATE (affects decoration) +%macro CALL_FUNC 1-2.nolist PRIVATE + %xdefine %%func_name %1 + %xdefine %%visibility %2 + + ; Accepted visibility values are PUBLIC and PRIVATE + %ifnidni %%visibility, PUBLIC + %ifnidni %%visibility, PRIVATE + %fatal Function %%func_name visibility is not properly defined. Shall be: PRIVATE or PUBLIC. + %endif + %endif + + ; __cdecl on WIN32/OSXEM64T obligates to have undersore prefix decoration. + ; Only PUBLIC functions are decorated. + %ifidni %%visibility, PUBLIC + %if ((IPP_BINARY_FORMAT == 2) || (IPP_BINARY_FORMAT == 3)) ; WIN32 or OSXEM64T + %xdefine %%func_name _%1 + %endif + %endif + + call %%func_name +%endmacro + +; Declares function decorated by appropriate CPU prefix (for the merged library) +; Default visibility (if not defined) is PUBLIC. +%macro IPPASM 1-2.nolist PUBLIC + %xdefine %%func_name %1 + %xdefine %%visibility %2 + + %ifctx _IPPASM_CTX_ + %fatal "IPPASM: already in the context, need to call ENDFUNC" + %endif + %push _IPPASM_CTX_ + + %push _CPU_PREFIX_DECORATE_CTX_ + CPU_PREFIX_DECORATE %%func_name + %xdefine %%func_name %$decorated_func_name + %assign %$$decoration_length %$decoration_length + %pop _CPU_PREFIX_DECORATE_CTX_ + + DECLARE_FUNC %%func_name, %%visibility +%endmacro + +; Calls assembler function declared by IPPASM +; Default visibility is PRIVATE (affects decoration) +%macro CALL_IPPASM 1-2.nolist PRIVATE + %xdefine %%func_name %1 + %xdefine %%visibility %2 + + ; Accepted visibility values are PUBLIC and PRIVATE + %ifnidni %%visibility, PUBLIC + %ifnidni %%visibility, PRIVATE + %fatal Function %%func_name visibility is not properly defined. Shall be: PRIVATE or PUBLIC. + %endif + %endif + + %push _CPU_PREFIX_DECORATE_CTX_ + CPU_PREFIX_DECORATE %%func_name + %xdefine %%func_name %$decorated_func_name + %pop _CPU_PREFIX_DECORATE_CTX_ + + CALL_FUNC %%func_name,%%visibility +%endmacro + +; End function macro - required to be called after IPPASM or DECLARE_FUNC macro invokation. +%macro ENDFUNC 1.nolist + %xdefine %%func_name %1 + %ifnctx _DECLARE_FUNC_CTX_ + %fatal "Not in the context: _DECLARE_FUNC_CTX_" + %endif + + ; Cross-check of context variable with macro parameter + %defstr %%func_name_str %%func_name + %defstr %%func_name_ctx_str %$func_name_ctx + %substr %%func_name_ctx_str_not_decorated %%func_name_ctx_str %[%$decoration_length+1],-1 ; remove decoration (first X symbols) + %ifnidn %%func_name_str,%%func_name_ctx_str + %ifnidn %%func_name_str,%%func_name_ctx_str_not_decorated + %fatal ENDFUNC: function name [%%func_name] does match context: [%$func_name_ctx] + %endif + %endif + + ; Add local label to be able calculate function size + ; Take function name from the context (real declaration name) +.LEnd_%+%$func_name_ctx: + %pop _DECLARE_FUNC_CTX_ + + %ifctx _IPPASM_CTX_ + %pop _IPPASM_CTX_ + %endif +%endmacro + +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/ia_emm.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/ia_emm.inc new file mode 100644 index 0000000..1136c92 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/ia_emm.inc @@ -0,0 +1,116 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +%include "asmdefs.inc" +%include "ia_common.inc" +%include "utils.inc" + +; Decorates function name with appropriate CPU prefix (for the merged library). +; The macro is context-dependent and returns decorated name in the %$decorated_func_name +; context variable. +%macro CPU_PREFIX_DECORATE 1.nolist + %ifnctx _CPU_PREFIX_DECORATE_CTX_ + %fatal "Not in the context: _CPU_PREFIX_DECORATE_CTX_" + %endif + + ; Add CPU-specific suffix for the dispatched library + %ifdef _OWN_MERGED_BLD + %if (_IPP == _IPP_PX) + %xdefine %%func_name px_%1 + %assign %%decoration_length 3 + %endif + %if (_IPP == _IPP_W7) + %xdefine %%func_name w7_%1 + %assign %%decoration_length 3 + %endif + %if (_IPP == _IPP_V8) + %xdefine %%func_name v8_%1 + %assign %%decoration_length 3 + %endif + %if (_IPP == _IPP_S8) + %xdefine %%func_name s8_%1 + %assign %%decoration_length 3 + %endif + %if (_IPP == _IPP_P8) + %xdefine %%func_name p8_%1 + %assign %%decoration_length 3 + %endif + %if (_IPP == _IPP_G9) + %xdefine %%func_name g9_%1 + %assign %%decoration_length 3 + %endif + %if (_IPP == _IPP_H9) + %xdefine %%func_name h9_%1 + %assign %%decoration_length 3 + %endif + %else + %xdefine %%func_name %1 + %assign %%decoration_length 0 + %endif + + %ifndef %%func_name + %fatal "CPU_PREFIX_DECORATE: unknown decoration for: _IPP = " _IPP + %endif + %xdefine %$decorated_func_name %[%%func_name] + %assign %$decoration_length %%decoration_length +%endmacro + +%define NONVOLATILE_REGS_32_GPR ebp,ebx,esi,edi + +; Saves non-volatile GPR registers on stack. +; Input - list of used registers. +%macro USES_GPR 1+.nolist + %assign LOCAL_FRAME 0 + %assign GPR_FRAME 0 + %define GPR_CUR + + BEGIN_INTERSECT + INTERSECT {%1},{%[NONVOLATILE_REGS_32_GPR]} + ; List of non-volatile GPR registers in the order they will be pushed on stack + %xdefine GPR_CUR %$intersection + %assign GPR_FRAME %$cardinality * 4 + END_INTERSECT + + ; Push non-volatile GPRs on stack + FOREACH GPR_CUR,{push} + + ; Set up offset of arguments from ESP + %assign ARG_1 %[GPR_FRAME + 4] +%endmacro + +; Restore preliminary saved by USES_GPR non-volatile GPR registers from the stack. +; The macro shall be called after function processing. +%macro REST_GPR 0.nolist + %ifndef GPR_CUR + %fatal "REST_GPR: no GPR_CUR defined" + %endif + ; Pop saved GPRs from the stack + RFOREACH GPR_CUR,{pop} +%endmacro + +%macro LD_ADDR 2.nolist + %xdefine %%reg %1 + %xdefine %%addr %2 + +%ifdef IPP_PIC + call %%LABEL +%%LABEL: pop %%reg + sub %%reg, %%LABEL-%%addr +%else + lea %%reg, [%%addr] +%endif +%endmacro diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/ippres.gen b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/ippres.gen new file mode 100644 index 0000000..4e827e5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/ippres.gen @@ -0,0 +1,159 @@ +/******************************************************************************* +* Copyright (C) 1999 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + + +#include "winres.h" + +#define STR2(x) #x +#define STR(x) STR2(x) + +#ifndef _IPP_VERSION +#define _IPP_VERSION "" +#endif + +VS_VERSION_INFO VERSIONINFO + FILEVERSION VERSION() + PRODUCTVERSION VERSION() + FILEFLAGSMASK 0x3fL +#ifdef _DEBUG + FILEFLAGS 0x1L +#else + FILEFLAGS 0x0L +#endif + FILEOS VOS_NT_WINDOWS32 + FILETYPE VFT_DLL + FILESUBTYPE 0x0L +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904b0" + BEGIN + VALUE "CompanyName", "Intel Corporation.\0" + VALUE "FileVersion", STR( VERSION() ) "\0" + VALUE "ProductName", IPP_LIB_SHORTNAME() ". Intel(R) Integrated Performance Primitives. " IPP_LIB_LONGNAME() ".\0" + VALUE "ProductVersion", STR_VERSION() "\0" + VALUE "LegalCopyright", "Copyright (C) 1999-2021, Intel Corporation. All rights reserved.\0" + + #if defined(_MERGED_BLD) + #if defined (WIN32) && !defined (_WIN64) && !defined (WIN32E) + VALUE "Comments", "Intel(R) Integrated Performance Primitives. " IPP_LIB_LONGNAME() ".\0" + VALUE "FileDescription", IPP_LIB_PREFIX() _IPP_VERSION ".dll is the ia32 " IPP_LIB_SHORTNAME() " dynamic library\0" + VALUE "InternalName", IPP_LIB_PREFIX() _IPP_VERSION ".dll\0" + VALUE "OriginalFilename", IPP_LIB_PREFIX() _IPP_VERSION ".dll\0" + #endif + + #if defined(WIN32E) + VALUE "Comments", "Intel(R) Integrated Performance Primitives. " IPP_LIB_LONGNAME() ".\0" + VALUE "FileDescription", IPP_LIB_PREFIX() _IPP_VERSION ".dll is the intel64 " IPP_LIB_SHORTNAME() " dynamic library\0" + VALUE "InternalName", IPP_LIB_PREFIX() _IPP_VERSION ".dll\0" + VALUE "OriginalFilename", IPP_LIB_PREFIX() _IPP_VERSION ".dll\0" + #endif + + #else + #if defined (_PX) && defined (WIN32) && !defined (_WIN64) && !defined (WIN32E) + VALUE "Comments", "Intel(R) Integrated Performance Primitives. " IPP_LIB_LONGNAME() ". For Intel(R) Pentium(R) processors\0" + VALUE "FileDescription", IPP_LIB_PREFIX() "px" _IPP_VERSION ".dll is an " IPP_LIB_SHORTNAME() " dynamic library\0" + VALUE "InternalName", IPP_LIB_PREFIX() "px" _IPP_VERSION ".dll\0" + VALUE "OriginalFilename", IPP_LIB_PREFIX() "px" _IPP_VERSION ".dll\0" + #endif + #if defined(_PX) && defined(WIN32E) + VALUE "Comments", "Intel(R) Integrated Performance Primitives. " IPP_LIB_LONGNAME() ". For Intel(R) 64 Instruction Set Architecture (ISA) processors\0" + VALUE "FileDescription", IPP_LIB_PREFIX() "mx" _IPP_VERSION ".dll is an " IPP_LIB_SHORTNAME() " dynamic library\0" + VALUE "InternalName", IPP_LIB_PREFIX() "mx" _IPP_VERSION ".dll\0" + VALUE "OriginalFilename", IPP_LIB_PREFIX() "mx" _IPP_VERSION ".dll\0" + #endif + #if defined(_W7) + VALUE "Comments", "Intel(R) Integrated Performance Primitives. " IPP_LIB_LONGNAME() ". Optimized for processors with Streaming SIMD Extensions 2 (SSE2)\0" + VALUE "FileDescription", IPP_LIB_PREFIX() "w7" _IPP_VERSION ".dll is an " IPP_LIB_SHORTNAME() " dynamic library\0" + VALUE "InternalName", IPP_LIB_PREFIX() "w7" _IPP_VERSION ".dll\0" + VALUE "OriginalFilename", IPP_LIB_PREFIX() "w7" _IPP_VERSION ".dll\0" + #endif + #if defined(_P8) + VALUE "Comments", "Intel(R) Integrated Performance Primitives. " IPP_LIB_LONGNAME() ". Optimized for processors with Streaming SIMD Extensions 4.2 (SSE4.2)\0" + VALUE "FileDescription", IPP_LIB_PREFIX() "p8" _IPP_VERSION ".dll is an " IPP_LIB_SHORTNAME() " dynamic library\0" + VALUE "InternalName", IPP_LIB_PREFIX() "p8" _IPP_VERSION ".dll\0" + VALUE "OriginalFilename", IPP_LIB_PREFIX() "p8" _IPP_VERSION ".dll\0" + #endif + #if defined(_G9) + VALUE "Comments", "Intel(R) Integrated Performance Primitives. " IPP_LIB_LONGNAME() ". Optimized for processors with Advanced Vector Extensions (AVX)\0" + VALUE "FileDescription", IPP_LIB_PREFIX() "g9" _IPP_VERSION ".dll is an " IPP_LIB_SHORTNAME() " dynamic library\0" + VALUE "InternalName", IPP_LIB_PREFIX() "g9" _IPP_VERSION ".dll\0" + VALUE "OriginalFilename", IPP_LIB_PREFIX() "g9" _IPP_VERSION ".dll\0" + #endif + #if defined(_H9) + VALUE "Comments", "Intel(R) Integrated Performance Primitives. " IPP_LIB_LONGNAME() ". Optimized for processors with Advanced Vector Extensions 2 (AVX2)\0" + VALUE "FileDescription", IPP_LIB_PREFIX() "h9" _IPP_VERSION ".dll is an " IPP_LIB_SHORTNAME() " dynamic library\0" + VALUE "InternalName", IPP_LIB_PREFIX() "h9" _IPP_VERSION ".dll\0" + VALUE "OriginalFilename", IPP_LIB_PREFIX() "h9" _IPP_VERSION ".dll\0" + #endif + #if defined(_M7) + VALUE "Comments", "Intel(R) Integrated Performance Primitives. " IPP_LIB_LONGNAME() ". Optimized for Intel(R) 64 Instruction Set Architecture (ISA)\0" + VALUE "FileDescription", IPP_LIB_PREFIX() "m7" _IPP_VERSION ".dll is an " IPP_LIB_SHORTNAME() " dynamic library\0" + VALUE "InternalName", IPP_LIB_PREFIX() "m7" _IPP_VERSION ".dll\0" + VALUE "OriginalFilename", IPP_LIB_PREFIX() "m7" _IPP_VERSION ".dll\0" + #endif + #if defined(_Y8) + VALUE "Comments", "Intel(R) Integrated Performance Primitives. " IPP_LIB_LONGNAME() ". Optimized for processors with Streaming SIMD Extensions 4.2 (SSE4.2)\0" + VALUE "FileDescription", IPP_LIB_PREFIX() "y8" _IPP_VERSION ".dll is an " IPP_LIB_SHORTNAME() " dynamic library\0" + VALUE "InternalName", IPP_LIB_PREFIX() "y8" _IPP_VERSION ".dll\0" + VALUE "OriginalFilename", IPP_LIB_PREFIX() "y8" _IPP_VERSION ".dll\0" + #endif + #if defined(_E9) + VALUE "Comments", "Intel(R) Integrated Performance Primitives. " IPP_LIB_LONGNAME() ". Optimized for processors with Advanced Vector Extensions (AVX)\0" + VALUE "FileDescription", IPP_LIB_PREFIX() "e9" _IPP_VERSION ".dll is an " IPP_LIB_SHORTNAME() " dynamic library\0" + VALUE "InternalName", IPP_LIB_PREFIX() "e9" _IPP_VERSION ".dll\0" + VALUE "OriginalFilename", IPP_LIB_PREFIX() "e9" _IPP_VERSION ".dll\0" + #endif + #if defined(_L9) + VALUE "Comments", "Intel(R) Integrated Performance Primitives. " IPP_LIB_LONGNAME() ". Optimized for processors with Advanced Vector Extensions 2 (AVX2)\0" + VALUE "FileDescription", IPP_LIB_PREFIX() "l9" _IPP_VERSION ".dll is an " IPP_LIB_SHORTNAME() " dynamic library\0" + VALUE "InternalName", IPP_LIB_PREFIX() "l9" _IPP_VERSION ".dll\0" + VALUE "OriginalFilename", IPP_LIB_PREFIX() "l9" _IPP_VERSION ".dll\0" + #endif + #if defined(_K0) + VALUE "Comments", "Intel(R) Integrated Performance Primitives. " IPP_LIB_LONGNAME() ". Optimized for processors with AVX512F/CD/BW/DQ/VL support\0" + VALUE "FileDescription", IPP_LIB_PREFIX() "k0" _IPP_VERSION ".dll is an " IPP_LIB_SHORTNAME() " dynamic library\0" + VALUE "InternalName", IPP_LIB_PREFIX() "k0" _IPP_VERSION ".dll\0" + VALUE "OriginalFilename", IPP_LIB_PREFIX() "k0" _IPP_VERSION ".dll\0" + #endif + #if defined(_N0) + VALUE "Comments", "Intel(R) Integrated Performance Primitives. " IPP_LIB_LONGNAME() ". Optimized for processors with AVX512F/CD/ER/PF support\0" + VALUE "FileDescription", IPP_LIB_PREFIX() "n0" _IPP_VERSION ".dll is an " IPP_LIB_SHORTNAME() " dynamic library\0" + VALUE "InternalName", IPP_LIB_PREFIX() "n0" _IPP_VERSION ".dll\0" + VALUE "OriginalFilename", IPP_LIB_PREFIX() "n0" _IPP_VERSION ".dll\0" + #endif + #if defined(_S8) + VALUE "Comments", "Intel(R) Integrated Performance Primitives. " IPP_LIB_LONGNAME() ". Optimized for Intel(R) Atom(TM) processor\0" + VALUE "FileDescription", IPP_LIB_PREFIX() "s8" _IPP_VERSION ".dll is an " IPP_LIB_SHORTNAME() " dynamic library\0" + VALUE "InternalName", IPP_LIB_PREFIX() "s8" _IPP_VERSION ".dll\0" + VALUE "OriginalFilename", IPP_LIB_PREFIX() "s8" _IPP_VERSION ".dll\0" + #endif + #if defined(_N8) + VALUE "Comments", "Intel(R) Integrated Performance Primitives. " IPP_LIB_LONGNAME() ". Optimized for Intel(R) Atom(TM) processor\0" + VALUE "FileDescription", IPP_LIB_PREFIX() "n8" _IPP_VERSION ".dll is an " IPP_LIB_SHORTNAME() " dynamic library\0" + VALUE "InternalName", IPP_LIB_PREFIX() "n8" _IPP_VERSION ".dll\0" + VALUE "OriginalFilename", IPP_LIB_PREFIX() "n8" _IPP_VERSION ".dll\0" + #endif + #endif + + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1200 + END +END diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/ippver.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/ippver.h new file mode 100644 index 0000000..fe9ec9b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/ippver.h @@ -0,0 +1,43 @@ +/******************************************************************************* +* Copyright (C) 2001 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) +// +// Purpose: Describes the base Intel IPP Cryptography version +// +*/ + + +#include "ippversion.h" +#ifndef BASE_VERSION +#define BASE_VERSION() IPP_VERSION_MAJOR,IPP_VERSION_MINOR,IPP_VERSION_UPDATE +#endif + +#define STR2(x) #x +#define STR(x) STR2(x) + +#ifndef STR_VERSION + #ifdef IPP_REVISION + #define STR_VERSION() IPP_VERSION_STR " (r" STR( IPP_REVISION ) ")" + #else + #define STR_VERSION() IPP_VERSION_STR " (-)" + #endif +#endif + + +/* ////////////////////////////// End of file /////////////////////////////// */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/owndefs.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/owndefs.h new file mode 100644 index 0000000..dcc1ede --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/owndefs.h @@ -0,0 +1,658 @@ +/******************************************************************************* +* Copyright (C) 1999 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +// +// Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) +// +// Purpose: +// Internal definitions +// +// + +#ifndef __OWNDEFS_H__ +#define __OWNDEFS_H__ + +#if defined( _VXWORKS ) + #include + #undef NONE +#endif + +#ifndef IPPCPDEFS_H__ + #include "ippcpdefs.h" +#endif + +#if !defined(__INLINE) +#if defined(__INTEL_COMPILER) || defined(__INTEL_LLVM_COMPILER) || defined(_MSC_VER) + #define __INLINE static __inline +#elif defined( __GNUC__ ) + #define __INLINE static __inline__ +#else + #define __INLINE static +#endif +#endif /*__INLINE*/ + +/* TODO: to check ICX compiler */ +#if !defined(__NOINLINE) +#if defined(__INTEL_COMPILER) || defined(_MSC_VER) + #define __NOINLINE __declspec(noinline) +#elif defined( __GNUC__ ) + #define __NOINLINE __attribute__((noinline)) +#else + #define __NOINLINE +#endif +#endif /*__NOINLINE*/ + +#if !defined(__FORCEINLINE) +#if defined(_MSC_VER) + #define __FORCEINLINE __forceinline +#elif defined(__INTEL_COMPILER) || defined(__INTEL_LLVM_COMPILER) || defined( __GNUC__ ) + #define __FORCEINLINE __INLINE __attribute__((always_inline)) +#else + #define __FORCEINLINE +#endif +#endif /*__FORCEINLINE*/ + +#if defined(__INTEL_COMPILER) || defined(__INTEL_LLVM_COMPILER) + #define __RESTRICT restrict +#elif !defined( __RESTRICT ) + #define __RESTRICT +#endif + +#if defined( IPP_W32DLL ) + #if defined( _MSC_VER ) || defined( __INTEL_COMPILER ) || defined(__INTEL_LLVM_COMPILER) + #define IPPDEF(type) __declspec(dllexport) type + #else + #define IPPDEF(type) type + #endif +#else + #define IPPDEF(type) type +#endif + +/* ia32 */ +#define _IPP_PX 0 /* pure C-code */ +#define _IPP_M5 1 /* Intel® Quark(TM) processor */ +#define _IPP_W7 8 /* Intel® Streaming SIMD Extensions 2 (Intel® SSE2) */ +#define _IPP_T7 16 /* Intel® Streaming SIMD Extensions 3 (Intel® SSE3) */ +#define _IPP_V8 32 /* Supplemental Streaming SIMD Extensions 3 (SSSE3) */ +#define _IPP_S8 33 /* Supplemental Streaming SIMD Extensions 3 (SSSE3) + MOVBE instruction */ +#define _IPP_P8 64 /* Intel® Streaming SIMD Extensions 4.2 (Intel® SSE4.2) */ +#define _IPP_G9 128 /* Intel® Advanced Vector Extensions (Intel® AVX) */ +#define _IPP_H9 256 /* Intel® Advanced Vector Extensions 2 (Intel® AVX2) */ +#define _IPP_I0 512 /* Intel® Advanced Vector Extensions 512 (Intel® AVX512) - Intel® Xeon® Phi(TM) Processor (formerly Knights Landing) */ +#define _IPP_S0 1024 /* Intel® Advanced Vector Extensions 512 (Intel® AVX512) - Intel® Xeon® Processor (formerly codenamed Skylake) */ + +/* intel64 */ +#define _IPP32E_PX _IPP_PX /* pure C-code */ +#define _IPP32E_M7 32 /* Intel® Streaming SIMD Extensions 3 (Intel® SSE3) */ +#define _IPP32E_U8 64 /* Supplemental Streaming SIMD Extensions 3 (SSSE3) */ +#define _IPP32E_N8 65 /* Supplemental Streaming SIMD Extensions 3 (SSSE3) + MOVBE instruction */ +#define _IPP32E_Y8 128 /* Intel® Streaming SIMD Extensions 4.2 (Intel® SSE4.2) */ +#define _IPP32E_E9 256 /* Intel® Advanced Vector Extensions (Intel® AVX) */ +#define _IPP32E_L9 512 /* Intel® Advanced Vector Extensions 2 (Intel® AVX2) */ +#define _IPP32E_N0 1024 /* Intel® Advanced Vector Extensions 512 (Intel® AVX512) - Intel® Xeon® Phi(TM) Processor (formerly Knights Landing) */ +#define _IPP32E_K0 2048 /* Intel® Advanced Vector Extensions 512 (Intel® AVX512) - Intel® Xeon® Processor (formerly codenamed Skylake) */ +#define _IPP32E_K1 4096 /* Intel® Advanced Vector Extensions 512 (Intel® AVX512) - Intel® Xeon® Processor (formerly codenamed Icelake) */ + + +#if defined(__INTEL_COMPILER) || defined(__INTEL_LLVM_COMPILER) || (_MSC_VER >= 1300) + #define __ALIGN8 __declspec (align(8)) + #define __ALIGN16 __declspec (align(16)) + #define __ALIGN32 __declspec (align(32)) + #if !defined(__ALIGN64) + #define __ALIGN64 __declspec (align(64)) + #endif/*__ALIGN64*/ +#elif defined(__GNUC__) || defined(__CLANG__) + #define __ALIGN8 __attribute__((aligned(8))) + #define __ALIGN16 __attribute__((aligned(16))) + #define __ALIGN32 __attribute__((aligned(32))) + #if !defined(__ALIGN64) + #define __ALIGN64 __attribute__((aligned(64))) + #endif/*__ALIGN64*/ +#else + #error Intel, MS or GNU C compiler required +#endif + +/* ia32 */ +#if defined ( _M5 ) /* Intel® Quark(TM) processor */ + #define _IPP _IPP_M5 + #define _IPP32E _IPP32E_PX + #define OWNAPI(name) m5_##name + +#elif defined( _W7 ) /* Intel® SSE2 */ + #define _IPP _IPP_W7 + #define _IPP32E _IPP32E_PX + #define OWNAPI(name) w7_##name + +#elif defined( _T7 ) /* Intel® SSE3 */ + #define _IPP _IPP_T7 + #define _IPP32E _IPP32E_PX + #define OWNAPI(name) t7_##name + +#elif defined( _V8 ) /* SSSE3 */ + #define _IPP _IPP_V8 + #define _IPP32E _IPP32E_PX + #define OWNAPI(name) v8_##name + +#elif defined( _S8 ) /* SSSE3 + MOVBE instruction */ + #define _IPP _IPP_S8 + #define _IPP32E _IPP32E_PX + #define OWNAPI(name) s8_##name + +#elif defined( _P8 ) /* Intel® SSE4.2 */ + #define _IPP _IPP_P8 + #define _IPP32E _IPP32E_PX + #define OWNAPI(name) p8_##name + +#elif defined( _G9 ) /* Intel® AVX */ + #define _IPP _IPP_G9 + #define _IPP32E _IPP32E_PX + #define OWNAPI(name) g9_##name + +#elif defined( _H9 ) /* Intel® AVX2 */ + #define _IPP _IPP_H9 + #define _IPP32E _IPP32E_PX + #define OWNAPI(name) h9_##name + +/* intel64 */ +#elif defined( _M7 ) /* Intel® SSE3 */ + #define _IPP _IPP_PX + #define _IPP32E _IPP32E_M7 + #define OWNAPI(name) m7_##name + +#elif defined( _U8 ) /* SSSE3 */ + #define _IPP _IPP_PX + #define _IPP32E _IPP32E_U8 + #define OWNAPI(name) u8_##name + +#elif defined( _N8 ) /* SSSE3 + MOVBE instruction */ + #define _IPP _IPP_PX + #define _IPP32E _IPP32E_N8 + #define OWNAPI(name) n8_##name + +#elif defined( _Y8 ) /* Intel® SSE4.2 */ + #define _IPP _IPP_PX + #define _IPP32E _IPP32E_Y8 + #define OWNAPI(name) y8_##name + +#elif defined( _E9 ) /* Intel® AVX */ + #define _IPP _IPP_PX + #define _IPP32E _IPP32E_E9 + #define OWNAPI(name) e9_##name + +#elif defined( _L9 ) /* Intel® AVX2 */ + #define _IPP _IPP_PX + #define _IPP32E _IPP32E_L9 + #define OWNAPI(name) l9_##name + +#elif defined( _N0 ) /* Intel® AVX512 - formerly Knights Landing */ + #define _IPP _IPP_PX + #define _IPP32E _IPP32E_N0 + #define OWNAPI(name) n0_##name + +#elif defined( _K0 ) /* Intel® AVX512 - formerly codenamed Skylake */ + #define _IPP _IPP_PX + #define _IPP32E _IPP32E_K0 + #define OWNAPI(name) k0_##name + +#elif defined( _K1 ) /* Intel® AVX512 - formerly codenamed Icelake */ +#define _IPP _IPP_PX +#define _IPP32E _IPP32E_K1 +#define OWNAPI(name) k1_##name + +#else + #define _IPP _IPP_PX + #define _IPP32E _IPP32E_PX + #if defined (_M_AMD64) || defined (__x86_64__) || defined ( _ARCH_EM64T ) + #define OWNAPI(name) mx_##name + #else + #define OWNAPI(name) px_##name + #endif +#endif + +#if defined(_MERGED_BLD) +#define _OWN_MERGED_BLD +#endif + +#ifndef _OWN_MERGED_BLD + #undef OWNAPI + #define OWNAPI(name) name +#endif + +/* Force __cdecl calling convention for internal functions declarations */ +#define IPP_OWN_DECL(type,name,param) type IPP_CDECL name param ; +#define IPP_OWN_DEFN(type,name,param) type IPP_CDECL name param +#define IPP_OWN_FUNPTR(type,name,param) typedef type (IPP_CDECL *name) param ; + +#if defined( IPP_W32DLL ) + #if defined( _MSC_VER ) || defined( __INTEL_COMPILER ) || defined(__INTEL_LLVM_COMPILER) + #define IPPFUN(type,name,arg) __declspec(dllexport) type IPP_CALL name arg + #else + #define IPPFUN(type,name,arg) extern type IPP_CALL name arg + #endif +#else + #if defined(LINUX32E) && !defined(IPP_PIC) + #define IPPFUN(type,name,arg) __attribute__((force_align_arg_pointer)) extern type IPP_CALL name arg + #else + #define IPPFUN(type,name,arg) extern type IPP_CALL name arg + #endif +#endif + +#define _IPP_ARCH_IA32 1 +#define _IPP_ARCH_EM64T 4 +#define _IPP_ARCH_LRB 16 +#define _IPP_ARCH_LRB2 128 +#define _IPP_ARCH_LP64 0 + +#if defined ( _ARCH_IA32 ) + #define _IPP_ARCH _IPP_ARCH_IA32 + +#elif defined( _ARCH_EM64T ) + #define _IPP_ARCH _IPP_ARCH_EM64T + +#else + #if defined(_M_AMD64) || defined(__x86_64) || defined(__x86_64__) + #define _IPP_ARCH _IPP_ARCH_EM64T + + #else + #define _IPP_ARCH _IPP_ARCH_IA32 + + #endif +#endif + +#if ((_IPP_ARCH == _IPP_ARCH_IA32)) +__INLINE Ipp32s IPP_INT_PTR ( const void* ptr ) +{ + union { + void* Ptr; + Ipp32s Int; + } dd; + dd.Ptr = (void*)ptr; + return dd.Int; +} + +__INLINE Ipp32u IPP_UINT_PTR( const void* ptr ) +{ + union { + void* Ptr; + Ipp32u Int; + } dd; + dd.Ptr = (void*)ptr; + return dd.Int; +} +#elif ((_IPP_ARCH == _IPP_ARCH_EM64T) || (_IPP_ARCH == _IPP_ARCH_LRB2)) +__INLINE Ipp64s IPP_INT_PTR( const void* ptr ) +{ + union { + void* Ptr; + Ipp64s Int; + } dd; + dd.Ptr = (void*)ptr; + return dd.Int; +} + +__INLINE Ipp64u IPP_UINT_PTR( const void* ptr ) +{ + union { + void* Ptr; + Ipp64u Int; + } dd; + dd.Ptr = (void*)ptr; + return dd.Int; +} +#else + #define IPP_INT_PTR( ptr ) ( (long)(ptr) ) + #define IPP_UINT_PTR( ptr ) ( (unsigned long)(ptr) ) +#endif + +#define IPP_ALIGN_TYPE(type, align) ((align)/sizeof(type)-1) +#define IPP_BYTES_TO_ALIGN(ptr, align) ((~(IPP_INT_PTR(ptr)&((align)-1))+1)&((align)-1)) +#define IPP_ALIGNED_PTR(ptr, align) (void*)( (unsigned char*)(ptr) + (IPP_BYTES_TO_ALIGN( ptr, align )) ) + +#define IPP_ALIGNED_SIZE(size, align) (((size)+(align)-1)&~((align)-1)) + +#define IPP_MALLOC_ALIGNED_BYTES 64 +#define IPP_MALLOC_ALIGNED_8BYTES 8 +#define IPP_MALLOC_ALIGNED_16BYTES 16 +#define IPP_MALLOC_ALIGNED_32BYTES 32 + +#define IPP_ALIGNED_ARRAY(align,arrtype,arrname,arrlength)\ + char arrname##AlignedArrBuff[sizeof(arrtype)*(arrlength)+IPP_ALIGN_TYPE(char, align)];\ + arrtype *arrname = (arrtype*)IPP_ALIGNED_PTR(arrname##AlignedArrBuff,align) + +#if defined( __cplusplus ) +extern "C" { +#endif + +/* ///////////////////////////////////////////////////////////////////////////// + + Intel IPP Cryptography Context Identification + + /////////////////////////////////////////////////////////////////////////// */ + +#define IPP_CONTEXT( a, b, c, d) \ + (int)(((unsigned)(a) << 24) | ((unsigned)(b) << 16) | \ + ((unsigned)(c) << 8) | (unsigned)(d)) + +typedef enum { + idCtxUnknown = 0, + idCtxDES = IPP_CONTEXT( ' ', 'D', 'E', 'S'), + idCtxRijndael = IPP_CONTEXT( ' ', 'R', 'I', 'J'), + idCtxSMS4 = IPP_CONTEXT( 'S', 'M', 'S', '4'), + idCtxARCFOUR = IPP_CONTEXT( ' ', 'R', 'C', '4'), + idCtxSHA1 = IPP_CONTEXT( 'S', 'H', 'S', '1'), + idCtxSHA224 = IPP_CONTEXT( 'S', 'H', 'S', '3'), + idCtxSHA256 = IPP_CONTEXT( 'S', 'H', 'S', '2'), + idCtxSHA384 = IPP_CONTEXT( 'S', 'H', 'S', '4'), + idCtxSHA512 = IPP_CONTEXT( 'S', 'H', 'S', '5'), + idCtxMD5 = IPP_CONTEXT( ' ', 'M', 'D', '5'), + idCtxHMAC = IPP_CONTEXT( 'H', 'M', 'A', 'C'), + idCtxBigNum = IPP_CONTEXT( 'B', 'I', 'G', 'N'), + idCtxMontgomery = IPP_CONTEXT( 'M', 'O', 'N', 'T'), + idCtxPrimeNumber = IPP_CONTEXT( 'P', 'R', 'I', 'M'), + idCtxPRNG = IPP_CONTEXT( 'P', 'R', 'N', 'G'), + idCtxRSA = IPP_CONTEXT( ' ', 'R', 'S', 'A'), + idCtxRSA_PubKey = IPP_CONTEXT( 'R', 'S', 'A', '0'), + idCtxRSA_PrvKey1 = IPP_CONTEXT( 'R', 'S', 'A', '1'), + idCtxRSA_PrvKey2 = IPP_CONTEXT( 'R', 'S', 'A', '2'), + idCtxDSA = IPP_CONTEXT( ' ', 'D', 'S', 'A'), + idCtxECCP = IPP_CONTEXT( ' ', 'E', 'C', 'P'), + idCtxECCB = IPP_CONTEXT( ' ', 'E', 'C', 'B'), + idCtxECCPPoint = IPP_CONTEXT( 'P', 'E', 'C', 'P'), + idCtxECCBPoint = IPP_CONTEXT( 'P', 'E', 'C', 'B'), + idCtxDH = IPP_CONTEXT( ' ', ' ', 'D', 'H'), + idCtxDLP = IPP_CONTEXT( ' ', 'D', 'L', 'P'), + idCtxCMAC = IPP_CONTEXT( 'C', 'M', 'A', 'C'), + idCtxAESXCBC, + idCtxAESCCM, + idCtxAESGCM, + idCtxGFP, + idCtxGFPE, + idCtxGFPX, + idCtxGFPXE, + idCtxGFPXQX, + idCtxGFPXQXE, + idCtxGFPEC, + idCtxGFPPoint, + idCtxGFPXEC, + idCtxGFPXECPoint, + idCtxHash, + idCtxSM3, + idCtxAESXTS, + idxCtxECES_SM2, + idCtxGFPECKE +} IppCtxId; + + + + +/* ///////////////////////////////////////////////////////////////////////////// + Helpers + /////////////////////////////////////////////////////////////////////////// */ + +#define IPP_NOERROR_RET() return ippStsNoErr +#define IPP_ERROR_RET( ErrCode ) return (ErrCode) + +#define IPP_BADARG_RET( expr, ErrCode )\ + {if (expr) { IPP_ERROR_RET( ErrCode ); }} + +#define IPP_BAD_SIZE_RET( n )\ + IPP_BADARG_RET( (n)<=0, ippStsSizeErr ) + +#define IPP_BAD_STEP_RET( n )\ + IPP_BADARG_RET( (n)<=0, ippStsStepErr ) + +#define IPP_BAD_PTR1_RET( ptr )\ + IPP_BADARG_RET( NULL==(ptr), ippStsNullPtrErr ) + +#define IPP_BAD_PTR2_RET( ptr1, ptr2 )\ + {IPP_BAD_PTR1_RET( ptr1 ); IPP_BAD_PTR1_RET( ptr2 )} + +#define IPP_BAD_PTR3_RET( ptr1, ptr2, ptr3 )\ + {IPP_BAD_PTR2_RET( ptr1, ptr2 ); IPP_BAD_PTR1_RET( ptr3 )} + +#define IPP_BAD_PTR4_RET( ptr1, ptr2, ptr3, ptr4 )\ + {IPP_BAD_PTR2_RET( ptr1, ptr2 ); IPP_BAD_PTR2_RET( ptr3, ptr4 )} + +#define IPP_BAD_ISIZE_RET(roi) \ + IPP_BADARG_RET( ((roi).width<=0 || (roi).height<=0), ippStsSizeErr) + +/* ////////////////////////////////////////////////////////////////////////// */ +/* internal messages */ + +#define MSG_LOAD_DLL_ERR (-9700) /* Error at loading of %s library */ +#define MSG_NO_DLL (-9701) /* No DLLs were found in the Waterfall procedure */ +#define MSG_NO_SHARED (-9702) /* No shared libraries were found in the Waterfall procedure */ + +/* ////////////////////////////////////////////////////////////////////////// */ + + +typedef union { /* double precision */ + Ipp64s hex; + Ipp64f fp; +} IppFP_64f; + +typedef union { /* single precision */ + Ipp32s hex; + Ipp32f fp; +} IppFP_32f; + + +/* ////////////////////////////////////////////////////////////////////////// */ + +/* Define NULL pointer value */ +#ifndef NULL +#ifdef __cplusplus +#define NULL 0 +#else +#define NULL ((void *)0) +#endif +#endif + +#define IPP_UNREFERENCED_PARAMETER(p) (void)(p) + +#if defined( _IPP_MARK_LIBRARY ) +static char G[] = {73, 80, 80, 71, 101, 110, 117, 105, 110, 101, 243, 193, 210, 207, 215}; +#endif + + +/* +// endian definition +*/ +#define IPP_LITTLE_ENDIAN (0) +#define IPP_BIG_ENDIAN (1) + +#if defined( _IPP_LE ) + #define IPP_ENDIAN IPP_LITTLE_ENDIAN + +#elif defined( _IPP_BE ) + #define IPP_ENDIAN IPP_BIG_ENDIAN + +#else + #if defined( __ARMEB__ ) + #define IPP_ENDIAN IPP_BIG_ENDIAN + + #else + #define IPP_ENDIAN IPP_LITTLE_ENDIAN + + #endif +#endif + + +/* ////////////////////////////////////////////////////////////////////////// */ + +/* intrinsics */ +#if (_IPP >= _IPP_W7) || (_IPP32E >= _IPP32E_M7) + #if defined(__INTEL_COMPILER) || (_MSC_VER >= 1300) + #if (_IPP == _IPP_W7) + #if defined(__INTEL_COMPILER) + #include "emmintrin.h" + #else + #undef _W7 + #include "emmintrin.h" + #define _W7 + #endif + #define _mm_loadu _mm_loadu_si128 + #elif (_IPP32E == _IPP32E_M7) + #if defined(__INTEL_COMPILER) + #include "pmmintrin.h" + #define _mm_loadu _mm_lddqu_si128 + #elif (_MSC_FULL_VER >= 140050110) + #include "intrin.h" + #define _mm_loadu _mm_lddqu_si128 + #elif (_MSC_FULL_VER < 140050110) + #include "emmintrin.h" + #define _mm_loadu _mm_loadu_si128 + #endif + #elif ((_IPP == _IPP_V8) || (_IPP32E == _IPP32E_U8) || (_IPP == _IPP_S8) || (_IPP32E == _IPP32E_N8)) + #if defined(__INTEL_COMPILER) + #include "tmmintrin.h" + #define _mm_loadu _mm_lddqu_si128 + #elif (_MSC_FULL_VER >= 140050110) + #include "intrin.h" + #define _mm_loadu _mm_lddqu_si128 + #elif (_MSC_FULL_VER < 140050110) + #include "emmintrin.h" + #define _mm_loadu _mm_loadu_si128 + #endif + #elif (_IPP == _IPP_P8) || (_IPP32E == _IPP32E_Y8) + #if defined(__INTEL_COMPILER) + #include "smmintrin.h" + #define _mm_loadu _mm_lddqu_si128 + #elif (_MSC_FULL_VER >= 140050110) + #include "intrin.h" + #define _mm_loadu _mm_lddqu_si128 + #elif (_MSC_FULL_VER < 140050110) + #include "emmintrin.h" + #define _mm_loadu _mm_loadu_si128 + #endif + #elif (_IPP >= _IPP_G9) || (_IPP32E >= _IPP32E_E9) + #if defined(__INTEL_COMPILER) + #include "immintrin.h" + #define _mm_loadu _mm_lddqu_si128 + #elif (_MSC_FULL_VER >= 160021003) + #include "immintrin.h" + #define _mm_loadu _mm_lddqu_si128 + #endif + #endif + #endif + #if defined(__GNUC__) && !defined(__INTEL_COMPILER) + #include "x86intrin.h" + #endif +#endif + +// **** intrinsics for bit casting **** +#if defined(__INTEL_COMPILER) +extern unsigned int __intel_castf32_u32(float val); +extern float __intel_castu32_f32(unsigned int val); +extern unsigned __int64 __intel_castf64_u64(double val); +extern double __intel_castu64_f64(unsigned __int64 val); + #define __CAST_32f32u(val) __intel_castf32_u32((Ipp32f)val) + #define __CAST_32u32f(val) __intel_castu32_f32((Ipp32u)val) + #define __CAST_64f64u(val) __intel_castf64_u64((Ipp64f)val) + #define __CAST_64u64f(val) __intel_castu64_f64((Ipp64u)val) +#else + #define __CAST_32f32u(val) ( *((Ipp32u*)&val) ) + #define __CAST_32u32f(val) ( *((Ipp32f*)&val) ) + #define __CAST_64f64u(val) ( *((Ipp64u*)&val) ) + #define __CAST_64u64f(val) ( *((Ipp64f*)&val) ) +#endif + + +// short names for vector registers casting +#define _pd2ps _mm_castpd_ps +#define _ps2pd _mm_castps_pd +#define _pd2pi _mm_castpd_si128 +#define _pi2pd _mm_castsi128_pd +#define _ps2pi _mm_castps_si128 +#define _pi2ps _mm_castsi128_ps + +#define _ypd2ypi _mm256_castpd_si256 +#define _ypi2ypd _mm256_castsi256_pd +#define _yps2ypi _mm256_castps_si256 +#define _ypi2yps _mm256_castsi256_ps +#define _ypd2yps _mm256_castpd_ps +#define _yps2ypd _mm256_castps_pd + +#define _yps2ps _mm256_castps256_ps128 +#define _ypi2pi _mm256_castsi256_si128 +#define _ypd2pd _mm256_castpd256_pd128 +#define _ps2yps _mm256_castps128_ps256 +#define _pi2ypi _mm256_castsi128_si256 +#define _pd2ypd _mm256_castpd128_pd256 + +#define _zpd2zpi _mm512_castpd_si512 +#define _zpi2zpd _mm512_castsi512_pd +#define _zps2zpi _mm512_castps_si512 +#define _zpi2zps _mm512_castsi512_ps +#define _zpd2zps _mm512_castpd_ps +#define _zps2zpd _mm512_castps_pd + +#define _zps2ps _mm512_castps512_ps128 +#define _zpi2pi _mm512_castsi512_si128 +#define _zpd2pd _mm512_castpd512_pd128 +#define _ps2zps _mm512_castps128_ps512 +#define _pi2zpi _mm512_castsi128_si512 +#define _pd2zpd _mm512_castpd128_pd512 + +#define _zps2yps _mm512_castps512_ps256 +#define _zpi2ypi _mm512_castsi512_si256 +#define _zpd2ypd _mm512_castpd512_pd256 +#define _yps2zps _mm512_castps256_ps512 +#define _ypi2zpi _mm512_castsi256_si512 +#define _ypd2zpd _mm512_castpd256_pd512 + + +#if defined(__INTEL_COMPILER) +#define __IVDEP ivdep +#else +#define __IVDEP message("message :: 'ivdep' is not defined") +#endif + +#if defined( _MERGED_BLD ) + #if !defined( _IPP_DYNAMIC ) + /* WIN-32, WIN-64 */ + #if defined(WIN32) || defined(WIN32E) + #if ( defined(_W7) || defined(_M7) ) + #define _IPP_DATA 1 + #endif + + #elif defined(linux) + /* LIN-32, LIN-64 */ + #if ( defined(_W7) || defined(_M7) ) + #define _IPP_DATA 1 + #endif + + + /* OSX-32, OSX-64 */ + #elif defined(OSX32) || defined(OSXEM64T) + #if ( defined(_Y8) ) + #define _IPP_DATA 1 + #endif + #endif + #endif +#else + /* compile data unconditionally */ + #define _IPP_DATA 1 +#endif + + +#if defined( __cplusplus ) +} +#endif + +#endif /* __OWNDEFS_H__ */ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/utils.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/utils.inc new file mode 100644 index 0000000..d4abb92 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/include/utils.inc @@ -0,0 +1,132 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +%ifndef __UTILS_INC__ +%define __UTILS_INC__ 1 + +; Apply a functor provided as a last parameter to each element of the list, provided as sequence of first parameters. +; A list is processed in direct order. Note: an input list can be empty. +%macro FOREACH 2-*.nolist + %rotate -1 + %xdefine %%functor %1 + %rep %0-1 + %rotate 1 + %ifnempty %1 + %%functor %1 + %endif + %endrep +%endmacro + +; Apply a functor provided as a last parameter to each element of the list, provided as sequence of first parameters. +; A list is processed in reverse order. Note: an input list can be empty. +%macro RFOREACH 2-*.nolist + %rotate -1 + %xdefine %%functor %1 + %rep %0-1 + %rotate -1 + %ifnempty %1 + %%functor %1 + %endif + %endrep +%endmacro + +; Shall be called before INTERSECT macro to open corresponding context. +%macro BEGIN_INTERSECT 0.nolist + %push _INTERSECT_CTX_ + %xdefine %$intersection + %assign %$cardinality 0 +%endmacro + +; Shall be called after INTERSECT macro to close corresponding context. +%macro END_INTERSECT 0.nolist + %pop _INTERSECT_CTX_ +%endmacro + +; The macro searches intersection between two lists. +; Input: two comma-separated lists, enclosed in curly braces. +; Output: +; - Intersection will be located in the %$instersection context macro (can be empty). +; - Count of intersection elements list will be stored in the %$cardinality context variable. +%macro INTERSECT 2.nolist + %ifnctx _INTERSECT_CTX_ + %fatal "Not in the context: _INTERSECT_CTX_" + %endif + + %xdefine %%list1 %1 + %xdefine %%list2 %2 + + FOREACH %%list1,{?INTERSECT_BODY {%%list2},} +%endmacro + +; Helper macro to concatenate two lists. +; The result will be stored in the 3rd parameter that must be a macro identifier. +%macro CONCATENATE 3.nolist + %ifnid %3 + %fatal "CONCATENATE: 3rd parameter must be a macro identifier." + %endif + %define %3 %[%1] + %ifnempty %3 + %ifnempty %2 + %define %3 %[%3],%[%2] + %endif + %else + %define %3 %[%2] + %endif +%endmacro + +; Helper macro that searches the specified element in the input list. +; Input: +; - Last parameter - target element +; - First parameters refer to the list where the search is processed. +; Output: +; - The macro is context dependent and upon the element is found, the context macro %$elem_exists will be defined. +%macro ?FIND 2-*.nolist + %ifnctx _FIND_CTX_ + %fatal "Not in the context: _FIND_CTX_" + %endif + %rotate -1 + %xdefine %%elem_to_check %1 + %undef %$elem_exists + + %rep %0-1 + %rotate -1 + %ifidni %%elem_to_check, %1 + %define %$elem_exists %1 + %exitrep + %endif + %endrep +%endmacro + +; Macro that finds and collects intersection elements. To be used as INTERSECT macro functor. +%macro ?INTERSECT_BODY 2.nolist + %xdefine %%list %1 + %xdefine %%elem %2 + + %push _FIND_CTX_ + ?FIND %%list,%%elem + %ifdef %$elem_exists + %ifempty %$$intersection + %define %$$intersection %2 + %else + %define %$$intersection %[%$$intersection],%%elem + %endif + %assign %$$cardinality %$$cardinality + 1 + %endif + %pop _FIND_CTX_ +%endmacro + +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/CMakeLists.txt b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/CMakeLists.txt new file mode 100644 index 0000000..a7f38aa --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/CMakeLists.txt @@ -0,0 +1,595 @@ +#=============================================================================== +# Copyright (C) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. +# +#=============================================================================== + +# +# Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) +# + +include (GenerateExportHeader) + +set(TARGET_NAME ${LIB_NAME}) + +#Set platform list +if (NOT MERGED_BLD) + if(${ARCH} MATCHES "ia32") + set(BASE_PLATFORM_LIST px) + else() + set(BASE_PLATFORM_LIST mx) + endif() +endif() +if(WIN32) + if(${ARCH} MATCHES "ia32") + set(BASE_PLATFORM_LIST ${BASE_PLATFORM_LIST} w7 s8 p8 g9 h9) + else() + set(BASE_PLATFORM_LIST ${BASE_PLATFORM_LIST} m7 n8 y8 e9 l9 k0 k1) + endif() +endif(WIN32) +if(UNIX) + if(APPLE) + set(BASE_PLATFORM_LIST ${BASE_PLATFORM_LIST} y8 e9 l9 k0 k1) + else() + if (${ARCH} MATCHES "ia32") + set(BASE_PLATFORM_LIST ${BASE_PLATFORM_LIST} w7 s8 p8 g9 h9) + else() + set(BASE_PLATFORM_LIST ${BASE_PLATFORM_LIST} m7 n8 y8 e9 l9 n0 k0 k1) + endif(${ARCH} MATCHES "ia32") + endif(APPLE) +endif(UNIX) + +if(PLATFORM_LIST) + if (NOT MERGED_BLD) + foreach(opt ${PLATFORM_LIST}) + set(FOUND_PLATFORM false) + foreach(base_opt ${BASE_PLATFORM_LIST}) + string(STRIP "${opt}" opt_strip) + if(opt_strip STREQUAL base_opt) + set(FOUND_PLATFORM true) + endif() + endforeach(base_opt ${BASE_PLATFORM_LIST}) + if(NOT FOUND_PLATFORM) + message (FATAL_ERROR "Incorrect platform: " ${opt}) + endif() + endforeach(opt ${PLATFORM_LIST}) + if (PLATFORM_LIST STREQUAL "") + message (FATAL_ERROR "PLATFORM_LIST cannot be empty") + endif(PLATFORM_LIST STREQUAL "") + message (STATUS "Platform list ......................... " ${PLATFORM_LIST}) + else() + message (FATAL_ERROR "PLATFORM_LIST cannot be set, when MERGED_BLD is on") + endif(NOT MERGED_BLD) +endif() + +# Platform-specific definitions +set(px_def "_PX") +set(w7_def "_W7") +set(s8_def "_S8") +set(p8_def "_P8") +set(g9_def "_G9") +set(h9_def "_H9") +set(mx_def "_PX") +set(m7_def "_M7") +set(n8_def "_N8") +set(y8_def "_Y8") +set(e9_def "_E9") +set(l9_def "_L9") +set(n0_def "_N0") +set(k0_def "_K0") +set(k1_def "_K1") + +if(NOT PLATFORM_LIST) + set(PLATFORM_LIST ${BASE_PLATFORM_LIST}) +endif() + +if (NOT MERGED_BLD) + set(PLATFORM_LIST ${PLATFORM_LIST} PARENT_SCOPE) +endif(NOT MERGED_BLD) + +# define defaults for every supported compiler +set(DEFAULT_Intel_COMPILER_VER 19.0.0) +set(DEFAULT_MSVC_COMPILER_VER 19.14) +set(DEFAULT_GNU_COMPILER_VER 8.2.0) +set(DEFAULT_APPLE_CLANG_COMPILER_VER 12.0.0) +set(DEFAULT_Clang_COMPILER_VER 9.0.0) + +set(DEFAULT_Intel18_COMPILER_VER 18.0.0) +set(DEFAULT_Intel19_COMPILER_VER 19.0.0) +set(DEFAULT_MSVC19_COMPILER_VER 19.14) +set(DEFAULT_IntelLLVM2023_COMPILER_VER 2023.1.0) + +string(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+).*$" "\\1.\\2.\\3" CMAKE_C_COMPILER_VERSION_SHORT ${CMAKE_C_COMPILER_VERSION}) +string(REGEX REPLACE "^([0-9]+)\\..*$" "\\1" CMAKE_C_COMPILER_VERSION_MAJOR ${CMAKE_C_COMPILER_VERSION}) + +if(UNIX) + if(APPLE) + set(OS_STRING "macosx") + else() + set(OS_STRING "linux") + endif() +else() + set(OS_STRING "windows") +endif() + +# common build options and ${OS_DEFAULT_COMPILER} +include("${IPP_CRYPTO_DIR}/sources/cmake/${OS_STRING}/common.cmake") +include("${IPP_CRYPTO_DIR}/sources/cmake/os_common.cmake") + +if(("${CMAKE_C_COMPILER_ID}" STREQUAL "MSVC") AND (CMAKE_C_COMPILER_VERSION VERSION_LESS DEFAULT_MSVC_COMPILER_VER)) + message(FATAL_ERROR "Microsoft Visual C++ Compiler version must be 19.14 or higher (MSVC 19.14 or higher)") +endif() + +if(("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") AND (CMAKE_C_COMPILER_VERSION VERSION_LESS DEFAULT_GNU_COMPILER_VER)) + message(FATAL_ERROR "GNU C Compiler version must be 8.2 or higher") +endif() + +if(("${CMAKE_C_COMPILER_ID}" STREQUAL "AppleClang") AND (CMAKE_C_COMPILER_VERSION VERSION_LESS DEFAULT_APPLE_CLANG_COMPILER_VER)) + message(FATAL_ERROR "Apple Clang C Compiler version must be 12.0 or higher") +endif() + +if(("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang") AND (CMAKE_C_COMPILER_VERSION VERSION_LESS DEFAULT_CLANG_COMPILER_VER)) + message(FATAL_ERROR "Clang C Compiler version must be 9.0 or higher") +endif() + +if(("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang") OR ("${CMAKE_C_COMPILER_ID}" STREQUAL "AppleClang")) + set(CLANG_COMPILER ON) + set(CLANG_COMPILER ${CLANG_COMPILER} PARENT_SCOPE) +endif() + +# Compiler options file +set(COMPILER_OPTIONS_FILE "${IPP_CRYPTO_DIR}/sources/cmake/${OS_STRING}") +if (EXISTS "${COMPILER_OPTIONS_FILE}/${CMAKE_C_COMPILER_ID}${CMAKE_C_COMPILER_VERSION_SHORT}.cmake") + set(COMPILER_OPTIONS_FILE "${COMPILER_OPTIONS_FILE}/${CMAKE_C_COMPILER_ID}${CMAKE_C_COMPILER_VERSION_SHORT}.cmake") +else() + if ("${DEFAULT_${CMAKE_C_COMPILER_ID}${CMAKE_C_COMPILER_VERSION_MAJOR}_COMPILER_VER}" STREQUAL "") + if ("${DEFAULT_${CMAKE_C_COMPILER_ID}_COMPILER_VER}" STREQUAL "") + set(COMPILER_OPTIONS_FILE "${COMPILER_OPTIONS_FILE}/${OS_DEFAULT_COMPILER}.cmake") + message(WARNING "Unknown compiler, using options from the OS default one: ${OS_DEFAULT_COMPILER}") + else() + set(COMPILER_OPTIONS_FILE "${COMPILER_OPTIONS_FILE}/${CMAKE_C_COMPILER_ID}${DEFAULT_${CMAKE_C_COMPILER_ID}_COMPILER_VER}.cmake") + if(CMAKE_C_COMPILER_VERSION VERSION_LESS ${DEFAULT_${CMAKE_C_COMPILER_ID}_COMPILER_VER}) + message(WARNING "Version of the compiler is lower than default, using options from: ${DEFAULT_${CMAKE_C_COMPILER_ID}_COMPILER_VER}") + endif() + endif() + else() + set(COMPILER_OPTIONS_FILE "${COMPILER_OPTIONS_FILE}/${CMAKE_C_COMPILER_ID}${DEFAULT_${CMAKE_C_COMPILER_ID}${CMAKE_C_COMPILER_VERSION_MAJOR}_COMPILER_VER}.cmake") + endif() +endif() +message (STATUS "Using compiler options from ........... " ${COMPILER_OPTIONS_FILE}) + +# Assembler options +# Note: do not move this initialization after LIBRARY_DEFINES extension +set(CMAKE_USER_MAKE_RULES_OVERRIDE_ASM "${IPP_CRYPTO_DIR}/sources/cmake/CMakeASM_NASMOptions.txt") +enable_language(ASM_NASM) +if(NOT CMAKE_ASM_NASM_COMPILER_LOADED) + message(FATAL_ERROR "Can't find assembler") +endif() +message(STATUS "ASM compiler version .................. " ${CMAKE_ASM_NASM_COMPILER}) +message(STATUS "ASM object format ..................... " ${CMAKE_ASM_NASM_OBJECT_FORMAT}) + +# set Intel IPP Cryptography revision, if required +if(IPP_REVISION) + # IPP_REVISION - release revision id, added into LibraryVersion + set(LIBRARY_DEFINES "${LIBRARY_DEFINES} -DIPP_REVISION=${IPP_REVISION}") +endif() + +set(LIBRARY_DEFINES "${LIBRARY_DEFINES} -D_NO_IPP_DEPRECATED") # do not warn about ippcp deprecated functions +# set BN_OPENSSL_DISABLE for Intel IPP Cryptography +# set(LIBRARY_DEFINES "${LIBRARY_DEFINES} -DBN_OPENSSL_DISABLE") + +include(${COMPILER_OPTIONS_FILE}) # uses ${CMAKE_C_FLAGS} ${LIBRARY_DEFINES} ${opt} from the scope + +if(UNIX) + set(CMAKE_C_CREATE_SHARED_LIBRARY " ${LIB_EXPORT_FILE} -o ") + set(CMAKE_CXX_CREATE_SHARED_LIBRARY " ${LIB_EXPORT_FILE} -o ") + if("${CMAKE_C_COMPILER_ID}" STREQUAL "AppleClang") + string(REPLACE "" " " CMAKE_C_CREATE_SHARED_LIBRARY ${CMAKE_C_CREATE_SHARED_LIBRARY}) + string(REPLACE "" " " CMAKE_CXX_CREATE_SHARED_LIBRARY ${CMAKE_CXX_CREATE_SHARED_LIBRARY}) + endif() +endif(UNIX) + +if(NOT MERGED_BLD) + set(IPPCP_LIB_STATIC "") + set(IPPCP_LIB_DYNAMIC "") +endif(NOT MERGED_BLD) + +file(GLOB IPPCP_PUBLIC_HEADERS + ${IPP_CRYPTO_INCLUDE_DIR}/*.h) + +file(GLOB LIBRARY_HEADERS + ${IPP_CRYPTO_SOURCES_DIR}/*.h + ${IPP_CRYPTO_SOURCES_DIR}/ecnist/*.h + ${IPP_CRYPTO_SOURCES_DIR}/sm2/*.h + ${IPP_CRYPTO_SOURCES_INCLUDE_DIR}/*.h + ${IPP_CRYPTO_INCLUDE_DIR}/ippcp*.h + # RSA_SB (ifma) uses crypto_mb headers + ${IPP_CRYPTO_SOURCES_DIR}/crypto_mb/include/crypto_mb/*.h + ${IPP_CRYPTO_SOURCES_DIR}/crypto_mb/include/internal/*.h + ) + +file(GLOB LIBRARY_C_SOURCES_ORIGINAL + ${IPP_CRYPTO_SOURCES_DIR}/*.c + ${IPP_CRYPTO_SOURCES_DIR}/ecnist/*.c + ${IPP_CRYPTO_SOURCES_DIR}/sm2/*.c + ) + +file(GLOB LIBRARY_ASM_SOURCES_ORIGINAL + ${IPP_CRYPTO_SOURCES_DIR}/asm_${ARCH}/*.asm + ) + +set(INTERNAL_INCLUDE_DIR ${CMAKE_BINARY_DIR}/include/autogen) + +set (C_INCLUDE_DIRECTORIES + ${IPP_CRYPTO_SOURCES_DIR} + ${IPP_CRYPTO_SOURCES_DIR}/ecnist + ${IPP_CRYPTO_SOURCES_DIR}/sm2 + ${IPP_CRYPTO_SOURCES_INCLUDE_DIR} + ${IPP_CRYPTO_INCLUDE_DIR} + ${INTERNAL_INCLUDE_DIR} + # RSA_SB (ifma) uses crypto_mb headers + ${IPP_CRYPTO_SOURCES_DIR}/crypto_mb/include + $<$:$ENV{ROOT}/compiler/include $ENV{ROOT}/compiler/include/icc> + $<$>:${CMAKE_SYSTEM_INCLUDE_PATH}> + $<$,$>:$ENV{INCLUDE}> + ) + +set (ASM_INCLUDE_DIRECTORIES + ${IPP_CRYPTO_SOURCES_INCLUDE_DIR} + ${IPP_CRYPTO_SOURCES_DIR}/asm_${ARCH} + ) + +# Because of CMake bug (https://gitlab.kitware.com/cmake/cmake/issues/19178) it is impossible to add +# target specific NASM compiler options/definitions when using VS generator, but it is possible to specify +# them based on source files. So, here is the workaround - we spawn same assembler sources across different +# platform-specific directories and assign different definitions for each directory using set_source_files_properties(). +# +# The same trick is used for C sources, but it aims a different purpose: to rename object files in the library archive, +# so that they do not intersect in a 'merged' library. +foreach(opt ${PLATFORM_LIST}) + # Popullate ASM source files in to corresponding folders per library 'letter' + set(asm_cache_dir "${CMAKE_BINARY_DIR}/asm_sources/${opt}/asm_${ARCH}") + file(MAKE_DIRECTORY ${asm_cache_dir}) + # Prefer configure_file() over simple file(COPY) as it creates dependencies, so that cmake project + # will be regenerated each time when any original file is changed. This is needed to keep all copied files up-to-date. + foreach (file ${LIBRARY_ASM_SOURCES_ORIGINAL}) + get_filename_component(basename ${file} NAME) + configure_file(${file} ${asm_cache_dir}/${opt}_${basename} COPYONLY) + endforeach() + file (GLOB LIBRARY_ASM_SOURCES_${opt} + ${asm_cache_dir}/*.asm + ) + set_source_files_properties(${LIBRARY_ASM_SOURCES_${opt}} PROPERTIES COMPILE_DEFINITIONS "$<$:_MERGED_BLD>;${${opt}_def}" + COMPILE_OPTIONS "$<$:${CMAKE_ASM_NASM_DEBUG_OPTIONS}>" + INCLUDE_DIRECTORIES "${ASM_INCLUDE_DIRECTORIES}") + + ######################################################################################### + # Popullate C source files in to corresponding folders per library 'letter' + if (NOT CODE_COVERAGE) + set(c_cache_dir "${CMAKE_BINARY_DIR}/c_sources/${opt}/c_${ARCH}") + file(MAKE_DIRECTORY ${c_cache_dir}) + # Add a prefix to the source files, so that corresponding object files in the merged library are unique named + foreach (file ${LIBRARY_C_SOURCES_ORIGINAL}) + get_filename_component(basename ${file} NAME) + configure_file(${file} ${c_cache_dir}/${opt}_${basename} COPYONLY) + endforeach() + file (GLOB LIBRARY_C_SOURCES_${opt} + ${c_cache_dir}/*.c + ) + else() + # Code coverage data gathering is a bit easier, when sources for ISA-specific builds point to the same location on a filesystem + set(LIBRARY_C_SOURCES_${opt} "${LIBRARY_C_SOURCES_ORIGINAL}") + endif() + set_source_files_properties(${LIBRARY_C_SOURCES_${opt}} pcpver.rc PROPERTIES INCLUDE_DIRECTORIES "${C_INCLUDE_DIRECTORIES}") + # Set optimization flag for cpinit.c to -O1 + # Fix for problem related with MSVC compiler -O2 optimization + if((${ARCH} STREQUAL "intel64") AND ("${CMAKE_C_COMPILER_ID}" STREQUAL "MSVC")) + set_source_files_properties("${c_cache_dir}/${opt}_cpinit.c" PROPERTIES COMPILE_FLAGS " /O1") + endif() +endforeach() + +# Generate single-CPU headers +if(MERGED_BLD) + set(ONE_CPU_FOLDER ${INTERNAL_INCLUDE_DIR}/single_cpu) + set(ONE_CPU_GENERATOR ${IPP_CRYPTO_DIR}/sources/gen_cpu_spc_header/gen_cpu_spc_1cpu_header.py) + execute_process(COMMAND ${Python_EXECUTABLE} ${ONE_CPU_GENERATOR} ${IPP_CRYPTO_INCLUDE_DIR}/ippcp.h ${ONE_CPU_FOLDER}) + file(GLOB ONE_CPU_HEADERS "${ONE_CPU_FOLDER}/*.h") + + set(INTERNAL_GENERATOR ${IPP_CRYPTO_DIR}/sources/gen_cpu_spc_header/gen_cpu_spc_header.py) + execute_process(COMMAND ${Python_EXECUTABLE} ${INTERNAL_GENERATOR} ${IPP_CRYPTO_INCLUDE_DIR}/ippcp.h ${INTERNAL_INCLUDE_DIR}) + set(HEADERS ${LIBRARY_HEADERS} ${INTERNAL_INCLUDE_DIR}/ippcp_cpuspc.h) +endif(MERGED_BLD) + +# Copy headers to the output directory +foreach( OUTPUTCONFIG ${CMAKE_CONFIGURATION_TYPES}) + string( TOUPPER ${OUTPUTCONFIG} OUTPUTCONFIG ) + file(COPY ${IPP_CRYPTO_INCLUDE_DIR}/ippcp.h + ${IPP_CRYPTO_INCLUDE_DIR}/ippcpdefs.h + ${IPP_CRYPTO_INCLUDE_DIR}/ippversion.h + ${ONE_CPU_FOLDER} + DESTINATION "${CMAKE_OUTPUT_DIR}/${OUTPUTCONFIG}/include") +endforeach( OUTPUTCONFIG CMAKE_CONFIGURATION_TYPES ) + +foreach(opt ${PLATFORM_LIST}) + set(OPT_FLAGS_${opt} ${${opt}_opt}) + + if(DYNAMIC_LIB AND NOT MERGED_BLD) + set(IPPCP_DYN_ITER ${TARGET_NAME}_dyn_${opt}) + set(IPPCP_DYN_ITER_ASMOBJS ${TARGET_NAME}_dyn_${opt}-asmobjs) + if(WIN32) + add_library(${IPPCP_DYN_ITER_ASMOBJS} OBJECT ${LIBRARY_ASM_SOURCES_${opt}}) + add_library(${IPPCP_DYN_ITER} SHARED ippcp.def + pcpver.rc + ${LIBRARY_HEADERS} + ${LIBRARY_C_SOURCES_${opt}} + $) + + set_target_properties(${IPPCP_DYN_ITER} PROPERTIES LINK_FLAGS ${LINK_FLAG_DYNAMIC_WINDOWS}) + foreach(link ${LINK_LIB_STATIC_DEBUG}) + target_link_libraries(${IPPCP_DYN_ITER} debug ${link}) + endforeach() + foreach(link ${LINK_LIB_STATIC_RELEASE}) + target_link_libraries(${IPPCP_DYN_ITER} optimized ${link}) + endforeach() + else() + add_library(${IPPCP_DYN_ITER_ASMOBJS} OBJECT ${LIBRARY_ASM_SOURCES_${opt}}) + add_library(${IPPCP_DYN_ITER} SHARED ${LIBRARY_HEADERS} + ${LIBRARY_C_SOURCES_${opt}} + $) + if(APPLE) + ## gres: copy LINK_FLAG_DYNAMIC_LINUX + set(LINK_FLAGS ${LINK_FLAG_DYNAMIC_MACOSX}) + ## add export file + set(LINK_FLAGS "${LINK_FLAGS} -exported_symbols_list ${IPP_CRYPTO_SOURCES_DIR}/exports.macosx.lib-export") + set_target_properties(${IPPCP_DYN_ITER} PROPERTIES LINK_FLAGS "${LINK_FLAGS}") + add_custom_command(TARGET ${IPPCP_DYN_ITER} POST_BUILD COMMAND + ${CMAKE_INSTALL_NAME_TOOL} -id @rpath/lib${TARGET_NAME}${opt}.dylib $) + else() + ## gres: copy LINK_FLAG_DYNAMIC_LINUX + set(LINK_FLAGS ${LINK_FLAG_DYNAMIC_LINUX}) + ## add export file + set(LINK_FLAGS "${LINK_FLAGS} ${IPP_CRYPTO_SOURCES_DIR}/exports.linux.lib-export") + set_target_properties(${IPPCP_DYN_ITER} PROPERTIES LINK_FLAGS ${LINK_FLAGS}) + endif() + if(NOT APPLE) + if("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") + target_link_libraries(${IPPCP_DYN_ITER} gcc) # gcc is because of -nostdlib + target_link_libraries(${IPPCP_DYN_ITER} c) # for printf - debug purpose only, not used + endif() + endif() + endif() + set_target_properties(${IPPCP_DYN_ITER} PROPERTIES OUTPUT_NAME "${TARGET_NAME}${opt}" + COMPILE_FLAGS "${OPT_FLAGS_${opt}}" + COMPILE_DEFINITIONS "_IPP_DYNAMIC;${${opt}_def}" + PUBLIC_HEADER "${IPPCP_PUBLIC_HEADERS}" + ) + if(UNIX) + set_target_properties(${IPPCP_DYN_ITER} PROPERTIES VERSION ${IPPCP_INTERFACE_VERSION} + SOVERSION ${IPPCP_INTERFACE_VERSION_MAJOR}) + endif() + + install(TARGETS ${IPPCP_DYN_ITER} + LIBRARY DESTINATION "lib/${ARCH}/$<$:nonpic>" + RUNTIME DESTINATION "lib/${ARCH}/$<$:nonpic>" + PUBLIC_HEADER DESTINATION "include") + list(APPEND IPPCP_LIB_DYNAMIC ${IPPCP_DYN_ITER}) + endif(DYNAMIC_LIB AND NOT MERGED_BLD) + + set(IPPCP_ST_ITER ${TARGET_NAME}_s_${opt}) + set(IPPCP_ST_ITER_ASMOBJS ${TARGET_NAME}_s_${opt}-asmobjs) + if(MERGED_BLD) + add_library(${IPPCP_ST_ITER} OBJECT ${LIBRARY_HEADERS} ${LIBRARY_C_SOURCES_${opt}}) + add_library(${IPPCP_ST_ITER_ASMOBJS} OBJECT ${LIBRARY_ASM_SOURCES_${opt}}) + set(merged_dependency ${merged_dependency} $ + $) + else() + add_library(${IPPCP_ST_ITER_ASMOBJS} OBJECT ${LIBRARY_ASM_SOURCES_${opt}}) + add_library(${IPPCP_ST_ITER} STATIC ${LIBRARY_HEADERS} + ${LIBRARY_C_SOURCES_${opt}} + $) + endif() + + if(WIN32) + set_target_properties(${IPPCP_ST_ITER} ${IPPCP_ST_ITER_ASMOBJS} PROPERTIES STATIC_LIBRARY_FLAGS ${LINK_FLAG_STATIC_WINDOWS}) + endif() + + set_target_properties(${IPPCP_ST_ITER} PROPERTIES COMPILE_FLAGS "${OPT_FLAGS_${opt}}" + COMPILE_DEFINITIONS "$<$:_MERGED_BLD>;${${opt}_def}") + # Merged build install is handled in another target + if (NOT MERGED_BLD) + set_target_properties(${IPPCP_ST_ITER} PROPERTIES PUBLIC_HEADER "${IPPCP_PUBLIC_HEADERS}") + install(TARGETS ${IPPCP_ST_ITER} + ARCHIVE DESTINATION "lib/${ARCH}/$<$:nonpic>" + PUBLIC_HEADER DESTINATION "include") + endif() + + list(APPEND IPPCP_LIB_STATIC ${IPPCP_ST_ITER}) +endforeach() + +if(NOT MERGED_BLD) + set(IPPCP_LIB_STATIC ${IPPCP_LIB_STATIC} PARENT_SCOPE) + set(IPPCP_LIB_DYNAMIC ${IPPCP_LIB_DYNAMIC} PARENT_SCOPE) +endif() + +if(MERGED_BLD) + set(DISPATCHER ${CMAKE_BINARY_DIR}/dispatcher) + set(IPPCP_API ${IPP_CRYPTO_INCLUDE_DIR}/ippcp.h) + file(MAKE_DIRECTORY ${DISPATCHER}) + + if(WIN32) + if(${ARCH} MATCHES "ia32") + set(DISPATCHER_GENERATOR ${IPP_CRYPTO_DISPATCHER_DIR}/gen_disp_win32.py) + else() + set(DISPATCHER_GENERATOR ${IPP_CRYPTO_DISPATCHER_DIR}/gen_disp_win64.py) + endif() + elseif(UNIX) + if(APPLE) + set(DISPATCHER_GENERATOR ${IPP_CRYPTO_DISPATCHER_DIR}/gen_disp_mac64.py) + else() + if(NOT NONPIC_LIB) + if(${ARCH} MATCHES "ia32") + set(DISPATCHER_GENERATOR ${IPP_CRYPTO_DISPATCHER_DIR}/gen_disp_lin32.py) + else() + set(DISPATCHER_GENERATOR ${IPP_CRYPTO_DISPATCHER_DIR}/gen_disp_lin64.py) + endif() + else() + if(${ARCH} MATCHES "ia32") + set(DISPATCHER_GENERATOR ${IPP_CRYPTO_DISPATCHER_DIR}/gen_disp_lin32.nonpic.py) + else() + set(DISPATCHER_GENERATOR ${IPP_CRYPTO_DISPATCHER_DIR}/gen_disp_lin64.nonpic.py) + endif() + endif() + endif() + endif(WIN32) + + execute_process(COMMAND ${Python_EXECUTABLE} ${DISPATCHER_GENERATOR} -i ${IPPCP_API} -o ${DISPATCHER} -l "${PLATFORM_LIST}" -c ${CMAKE_C_COMPILER_ID} + RESULT_VARIABLE result + ) + + file(GLOB DISPATCHER_HEADERS + ${CMAKE_BINARY_DIR}/dispatcher/*.h + ) + + file(GLOB DISPATCHER_C_SOURCES + ${CMAKE_BINARY_DIR}/dispatcher/*.c + ) + + file(GLOB DISPATCHER_ASM_SOURCES + ${CMAKE_BINARY_DIR}/dispatcher/*.asm + ) + + set(IPPCP_LIB_MERGED ${TARGET_NAME}_s) + add_library(${IPPCP_LIB_MERGED} STATIC ${DISPATCHER_HEADERS} ${DISPATCHER_C_SOURCES} ${DISPATCHER_ASM_SOURCES} ${merged_dependency}) + + set(IPPCP_LIB_MERGED ${IPPCP_LIB_MERGED} PARENT_SCOPE) + + if(WIN32) + set_target_properties(${IPPCP_LIB_MERGED} PROPERTIES OUTPUT_NAME "${TARGET_NAME}mt") + set_target_properties(${IPPCP_LIB_MERGED} PROPERTIES STATIC_LIBRARY_FLAGS ${LINK_FLAG_STATIC_WINDOWS}) + else() + set_target_properties(${IPPCP_LIB_MERGED} PROPERTIES OUTPUT_NAME "${TARGET_NAME}") + endif(WIN32) + + set_target_properties(${IPPCP_LIB_MERGED} PROPERTIES PUBLIC_HEADER "${IPPCP_PUBLIC_HEADERS}" + PRIVATE_HEADER "${ONE_CPU_HEADERS}") + + install(TARGETS ${IPPCP_LIB_MERGED} + ARCHIVE DESTINATION "lib/${ARCH}/$<$:nonpic>" + PUBLIC_HEADER DESTINATION "include" + PRIVATE_HEADER DESTINATION "tools/${ARCH}/staticlib") + + set_source_files_properties(${DISPATCHER_C_SOURCES} pcpver.rc PROPERTIES INCLUDE_DIRECTORIES "${C_INCLUDE_DIRECTORIES}") + # protection (_FORTIFY_SOURCE) and optimization flags for dispatcher + if(UNIX) + if(${ARCH} MATCHES "ia32") + set_source_files_properties(${DISPATCHER_C_SOURCES} PROPERTIES COMPILE_FLAGS "${CC_FLAGS_INLINE_ASM_UNIX_IA32} -D_GNU_SOURCE -D_FORTIFY_SOURCE=2 -O2") + else() + set_source_files_properties(${DISPATCHER_C_SOURCES} PROPERTIES COMPILE_FLAGS "${CC_FLAGS_INLINE_ASM_UNIX_INTEL64} -D_GNU_SOURCE -D_FORTIFY_SOURCE=2 -O2") + endif() + endif() + + # Single merged dynamic lib + if(DYNAMIC_LIB) + set(IPPCP_LIB_PCS ${TARGET_NAME}_dyn) + set(IPPCP_LIB_PCS ${IPPCP_LIB_PCS} PARENT_SCOPE) + + if(WIN32) + add_library(${IPPCP_LIB_PCS} SHARED ippcp.def emptyfile.c pcpver.rc) # emptyfile.c - Visual Studio does not produce a .dll without source files + else() + add_library(${IPPCP_LIB_PCS} SHARED emptyfile.c) # emptyfile.c to suppress the cmake warning + endif() + + set_target_properties(${IPPCP_LIB_PCS} PROPERTIES LINKER_LANGUAGE C + COMPILE_DEFINITIONS "_MERGED_BLD" + PUBLIC_HEADER "${IPPCP_PUBLIC_HEADERS}" + PRIVATE_HEADER "${ONE_CPU_HEADERS}") + if(UNIX) + set_target_properties(${IPPCP_LIB_PCS} PROPERTIES VERSION ${IPPCP_INTERFACE_VERSION} + SOVERSION ${IPPCP_INTERFACE_VERSION_MAJOR}) + endif() + + target_link_libraries(${IPPCP_LIB_PCS} ${IPPCP_LIB_MERGED}) # link to the static merged + set_target_properties(${IPPCP_LIB_PCS} PROPERTIES OUTPUT_NAME "${TARGET_NAME}") + + install(TARGETS ${IPPCP_LIB_PCS} + LIBRARY DESTINATION "lib/${ARCH}/$<$:nonpic>" + RUNTIME DESTINATION "lib/${ARCH}/$<$:nonpic>" + PUBLIC_HEADER DESTINATION "include" + PRIVATE_HEADER DESTINATION "tools/${ARCH}/staticlib") + + if(WIN32) + set_target_properties(${IPPCP_LIB_PCS} PROPERTIES LINK_FLAGS ${LINK_FLAG_DYNAMIC_WINDOWS}) + foreach(link ${LINK_LIB_STATIC_DEBUG}) + target_link_libraries(${IPPCP_LIB_PCS} debug ${link}) + endforeach() + foreach(link ${LINK_LIB_STATIC_RELEASE}) + target_link_libraries(${IPPCP_LIB_PCS} optimized ${link}) + endforeach() + else() + if(APPLE) + set(LINK_FLAGS ${LINK_FLAG_PCS_MACOSX}) + ## add export file + set(LINK_FLAGS "${LINK_FLAGS} -exported_symbols_list ${IPP_CRYPTO_SOURCES_DIR}/exports.macosx.lib-export") + set_target_properties(${IPPCP_LIB_PCS} PROPERTIES LINK_FLAGS "${LINK_FLAGS}") + add_custom_command(TARGET ${IPPCP_LIB_PCS} POST_BUILD COMMAND + ${CMAKE_INSTALL_NAME_TOOL} -id @rpath/lib${TARGET_NAME}.dylib $) + else() + set(LINK_FLAGS ${LINK_FLAG_PCS_LINUX}) + ## add export file + set(LINK_FLAGS "${LINK_FLAGS} ${IPP_CRYPTO_SOURCES_DIR}/exports.linux.lib-export") + set_target_properties(${IPPCP_LIB_PCS} PROPERTIES LINK_FLAGS ${LINK_FLAGS}) + if("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") + target_link_libraries(${IPPCP_LIB_PCS} gcc) # gcc is because of -nostdlib + target_link_libraries(${IPPCP_LIB_PCS} c) # for stack check on gcc + endif() + endif() + endif(WIN32) + + endif(DYNAMIC_LIB) +endif() + +# Generate CMake configs to let external projects find ippcp and crypto_mb libraries easily +include("${IPP_CRYPTO_DIR}/sources/cmake/ippcp-gen-config.cmake") + +# Install Custom Library tool +install(DIRECTORY "${TOOLS_DIR}/ipp_custom_library_tool_python/" + DESTINATION "tools/custom_library_tool_python") + +# Crypto multi-buffer library +if ((NOT NO_CRYPTO_MB) AND ("${ARCH}" STREQUAL "intel64")) + # MB_STANDALONE adjusts some build settings in crypto_mb + # (e.g. output directories, make some crypto_mb build variables visible in parent scope, etc) + set(MB_STANDALONE false) + add_subdirectory(crypto_mb) + + # Throw these variables further up, so that tests can grab them and link to crypto_mb libraries + set(MB_DYN_LIB_TARGET "${MB_DYN_LIB_TARGET}" PARENT_SCOPE) + set(MB_STATIC_LIB_TARGET "${MB_STATIC_LIB_TARGET}" PARENT_SCOPE) + + # Copy crypto_mb public headers to the output directory + foreach( OUTPUTCONFIG ${CMAKE_CONFIGURATION_TYPES}) + string( TOUPPER ${OUTPUTCONFIG} OUTPUTCONFIG ) + foreach(HEADER ${MB_PUBLIC_HEADERS}) + file(COPY "${HEADER}" DESTINATION "${CMAKE_OUTPUT_DIR}/${OUTPUTCONFIG}/include/crypto_mb") + endforeach(HEADER ${MB_PUBLIC_HEADERS}) + endforeach( OUTPUTCONFIG CMAKE_CONFIGURATION_TYPES ) + + # Interface versions (a.k.a. shared library versions) of ippcp and crypto_mb shall be equal, + # as they share single cmake-config routine with same "package name" IPPCP (in terms of find_package()). + # The intention behind this is ippcp and crypto_mb are part of the same product, and should + # share single find_package() call that can accept just one version common for all package components. + if(NOT "${IPPCP_INTERFACE_VERSION}" MATCHES "${MBX_INTERFACE_VERSION}") + message(SEND_ERROR "IPPCP_INTERFACE_VERSION and MBX_INTERFACE_VERSION don't match") + endif() +endif() diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_cfb_aesni_mb.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_cfb_aesni_mb.h new file mode 100644 index 0000000..991f96a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_cfb_aesni_mb.h @@ -0,0 +1,31 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#if !defined(_AES_CFB_AESNI_MB) +#define _AES_CFB_AESNI_MB + +#include "owndefs.h" +#include "owncp.h" + +#if (_IPP32E>=_IPP32E_Y8) + +#define aes_cfb16_enc_aesni_mb4 OWNAPI(aes_cfb16_enc_aesni_mb4) + IPP_OWN_DECL (void, aes_cfb16_enc_aesni_mb4, (const Ipp8u* const source_pa[4], Ipp8u* const dst_pa[4], const int len[4], const int num_of_rounds, const Ipp32u* enc_keys[4], const Ipp8u* pIV[4])) + +#endif + +#endif /* _AES_CFB_AESNI_MB */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_cfb_aesni_mb4.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_cfb_aesni_mb4.c new file mode 100644 index 0000000..be702b1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_cfb_aesni_mb4.c @@ -0,0 +1,102 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" +#include "aes_cfb_aesni_mb.h" + +#if (_IPP32E>=_IPP32E_Y8) + +static inline void aes_encrypt4_aesni_mb4(__m128i blocks[4], __m128i enc_keys[4][15], int cipherRounds) +{ + + blocks[0] = _mm_xor_si128(blocks[0], enc_keys[0][0]); + blocks[1] = _mm_xor_si128(blocks[1], enc_keys[1][0]); + blocks[2] = _mm_xor_si128(blocks[2], enc_keys[2][0]); + blocks[3] = _mm_xor_si128(blocks[3], enc_keys[3][0]); + + int nr; + + for (nr = 1; nr < cipherRounds; nr += 1) { + + blocks[0] = _mm_aesenc_si128(blocks[0], enc_keys[0][nr]); + blocks[1] = _mm_aesenc_si128(blocks[1], enc_keys[1][nr]); + blocks[2] = _mm_aesenc_si128(blocks[2], enc_keys[2][nr]); + blocks[3] = _mm_aesenc_si128(blocks[3], enc_keys[3][nr]); + + } + + blocks[0] = _mm_aesenclast_si128(blocks[0], enc_keys[0][nr]); + blocks[1] = _mm_aesenclast_si128(blocks[1], enc_keys[1][nr]); + blocks[2] = _mm_aesenclast_si128(blocks[2], enc_keys[2][nr]); + blocks[3] = _mm_aesenclast_si128(blocks[3], enc_keys[3][nr]); +} + + +IPP_OWN_DEFN (void, aes_cfb16_enc_aesni_mb4, (const Ipp8u* const source_pa[4], Ipp8u* const dst_pa[4], const int len[4], const int cipherRounds, const Ipp32u* enc_keys[4], const Ipp8u* pIV[4])) +{ + __m128i* pSrc[4]; + __m128i* pDst[4]; + + __m128i blocks[4]; + __m128i plainBlocks[4]; + + int nBlocks[4]; + + int maxBlocks = 0; + + __m128i keySchedule[4][15]; + + for (int i = 0; i < 4; i++) { + pSrc[i] = (__m128i*)source_pa[i]; + pDst[i] = (__m128i*)dst_pa[i]; + + nBlocks[i] = len[i] / CFB16_BLOCK_SIZE; + + if(nBlocks[i] > 0) { + blocks[i] = _mm_loadu_si128((__m128i const*)(pIV[i])); + + for (int j = 0; j <= cipherRounds; j++) { + keySchedule[i][j] = _mm_loadu_si128((__m128i const*)enc_keys[i] + j); + } + } + + if (nBlocks[i] > maxBlocks) { + maxBlocks = nBlocks[i]; + } + } + + for (int block = 0; block < maxBlocks; block++) { + aes_encrypt4_aesni_mb4(blocks, keySchedule, cipherRounds); + + for (int i = 0; i < 4; i++) { + if (nBlocks[i] > 0) { + plainBlocks[i] = _mm_loadu_si128(pSrc[i]); + blocks[i] = _mm_xor_si128(blocks[i], plainBlocks[i]); + _mm_storeu_si128(pDst[i], blocks[i]); + + pSrc[i]+= 1; + pDst[i]+= 1; + nBlocks[i] -= 1; + } + } + } +} + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_cfb_vaes_mb.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_cfb_vaes_mb.h new file mode 100644 index 0000000..7894c56 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_cfb_vaes_mb.h @@ -0,0 +1,54 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#if !defined(_AES_CFB_VAES_MB) +#define _AES_CFB_VAES_MB + +#include "owndefs.h" +#include "owncp.h" + +#if (_IPP32E>=_IPP32E_K1) + +#define TRANSPOSE_4x4_I128(X0_, X1_ ,X2_ ,X3_) {\ + __m512i T0_ = _mm512_maskz_shuffle_i64x2((__mmask8)0xFF, X0_, X1_, 0b01000100); \ + __m512i T1_ = _mm512_maskz_shuffle_i64x2((__mmask8)0xFF, X2_, X3_, 0b01000100); \ + __m512i T2_ = _mm512_maskz_shuffle_i64x2((__mmask8)0xFF, X0_, X1_, 0b11101110); \ + __m512i T3_ = _mm512_maskz_shuffle_i64x2((__mmask8)0xFF, X2_, X3_, 0b11101110); \ + \ + X0_ = _mm512_maskz_shuffle_i64x2((__mmask8)0xFF, T0_, T1_, 0b10001000); \ + X1_ = _mm512_maskz_shuffle_i64x2((__mmask8)0xFF, T0_, T1_, 0b11011101); \ + X2_ = _mm512_maskz_shuffle_i64x2((__mmask8)0xFF, T2_, T3_, 0b10001000); \ + X3_ = _mm512_maskz_shuffle_i64x2((__mmask8)0xFF, T2_, T3_, 0b11011101); \ +} + +#define UPDATE_MASK(len64, mask) {\ + if (len64 < 2 * 4 && len64 >= 0) \ + mask = (__mmask8)(((1 << len64) - 1) & 0xFF); \ + else if (len64 < 0) \ + mask = 0; \ +} + +#define aes_cfb16_enc_vaes_mb4 OWNAPI(aes_cfb16_enc_vaes_mb4) + IPP_OWN_DECL (void, aes_cfb16_enc_vaes_mb4, (const Ipp8u* const source_pa[4], Ipp8u* const dst_pa[4], const int len[4], const int num_of_rounds, const Ipp32u* enc_keys[4], const Ipp8u* pIV[4])) +#define aes_cfb16_enc_vaes_mb8 OWNAPI(aes_cfb16_enc_vaes_mb8) + IPP_OWN_DECL (void, aes_cfb16_enc_vaes_mb8, (const Ipp8u* const source_pa[8], Ipp8u* const dst_pa[8], const int len[8], const int num_of_rounds, const Ipp32u* enc_keys[8], const Ipp8u* pIV[8])) +#define aes_cfb16_enc_vaes_mb16 OWNAPI(aes_cfb16_enc_vaes_mb16) + IPP_OWN_DECL (void, aes_cfb16_enc_vaes_mb16, (const Ipp8u* const source_pa[16], Ipp8u* const dst_pa[16], const int len[16], const int num_of_rounds, const Ipp32u* enc_keys[16], const Ipp8u* pIV[16])) + +#endif + +#endif /* _AES_CFB_VAES_MB */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_cfb_vaes_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_cfb_vaes_mb16.c new file mode 100644 index 0000000..889f132 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_cfb_vaes_mb16.c @@ -0,0 +1,257 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" +#include "aes_cfb_vaes_mb.h" + +#if(_IPP32E>=_IPP32E_K1) + +#define AES_ENCRYPT_VAES_MB16(b0, b1, b2, b3, pRkey, num_rounds) { \ + __m512i (*tkeys)[4] = &pRkey[num_rounds-9]; \ + b0 = _mm512_xor_si512(b0, pRkey[0][0]); \ + b1 = _mm512_xor_si512(b1, pRkey[0][1]); \ + b2 = _mm512_xor_si512(b2, pRkey[0][2]); \ + b3 = _mm512_xor_si512(b3, pRkey[0][3]); \ + switch(num_rounds) { \ + case 14: \ + b0 = _mm512_aesenc_epi128(b0, tkeys[-4][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[-4][1]); \ + b2 = _mm512_aesenc_epi128(b2, tkeys[-4][2]); \ + b3 = _mm512_aesenc_epi128(b3, tkeys[-4][3]); \ + \ + b0 = _mm512_aesenc_epi128(b0, tkeys[-3][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[-3][1]); \ + b2 = _mm512_aesenc_epi128(b2, tkeys[-3][2]); \ + b3 = _mm512_aesenc_epi128(b3, tkeys[-3][3]); \ + case 12: \ + b0 = _mm512_aesenc_epi128(b0, tkeys[-2][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[-2][1]); \ + b2 = _mm512_aesenc_epi128(b2, tkeys[-2][2]); \ + b3 = _mm512_aesenc_epi128(b3, tkeys[-2][3]); \ + \ + b0 = _mm512_aesenc_epi128(b0, tkeys[-1][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[-1][1]); \ + b2 = _mm512_aesenc_epi128(b2, tkeys[-1][2]); \ + b3 = _mm512_aesenc_epi128(b3, tkeys[-1][3]); \ + default: \ + b0 = _mm512_aesenc_epi128(b0, tkeys[0][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[0][1]); \ + b2 = _mm512_aesenc_epi128(b2, tkeys[0][2]); \ + b3 = _mm512_aesenc_epi128(b3, tkeys[0][3]); \ + \ + b0 = _mm512_aesenc_epi128(b0, tkeys[1][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[1][1]); \ + b2 = _mm512_aesenc_epi128(b2, tkeys[1][2]); \ + b3 = _mm512_aesenc_epi128(b3, tkeys[1][3]); \ + \ + b0 = _mm512_aesenc_epi128(b0, tkeys[2][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[2][1]); \ + b2 = _mm512_aesenc_epi128(b2, tkeys[2][2]); \ + b3 = _mm512_aesenc_epi128(b3, tkeys[2][3]); \ + \ + b0 = _mm512_aesenc_epi128(b0, tkeys[3][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[3][1]); \ + b2 = _mm512_aesenc_epi128(b2, tkeys[3][2]); \ + b3 = _mm512_aesenc_epi128(b3, tkeys[3][3]); \ + \ + b0 = _mm512_aesenc_epi128(b0, tkeys[4][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[4][1]); \ + b2 = _mm512_aesenc_epi128(b2, tkeys[4][2]); \ + b3 = _mm512_aesenc_epi128(b3, tkeys[4][3]); \ + \ + b0 = _mm512_aesenc_epi128(b0, tkeys[5][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[5][1]); \ + b2 = _mm512_aesenc_epi128(b2, tkeys[5][2]); \ + b3 = _mm512_aesenc_epi128(b3, tkeys[5][3]); \ + \ + b0 = _mm512_aesenc_epi128(b0, tkeys[6][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[6][1]); \ + b2 = _mm512_aesenc_epi128(b2, tkeys[6][2]); \ + b3 = _mm512_aesenc_epi128(b3, tkeys[6][3]); \ + \ + b0 = _mm512_aesenc_epi128(b0, tkeys[7][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[7][1]); \ + b2 = _mm512_aesenc_epi128(b2, tkeys[7][2]); \ + b3 = _mm512_aesenc_epi128(b3, tkeys[7][3]); \ + \ + b0 = _mm512_aesenc_epi128(b0, tkeys[8][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[8][1]); \ + b2 = _mm512_aesenc_epi128(b2, tkeys[8][2]); \ + b3 = _mm512_aesenc_epi128(b3, tkeys[8][3]); \ + \ + b0 = _mm512_aesenclast_epi128(b0, tkeys[9][0]); \ + b1 = _mm512_aesenclast_epi128(b1, tkeys[9][1]); \ + b2 = _mm512_aesenclast_epi128(b2, tkeys[9][2]); \ + b3 = _mm512_aesenclast_epi128(b3, tkeys[9][3]); \ + } \ +} + +// Disable optimization for MSVC +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + #pragma optimize( "", off ) +#endif + +IPP_OWN_DEFN(void, aes_cfb16_enc_vaes_mb16, (const Ipp8u* const source_pa[16], Ipp8u* const dst_pa[16], const int arr_len[16], const int num_rounds, const Ipp32u* enc_keys[16], const Ipp8u* iv_pa[16])) +{ + int i, j, k; + int maxLen = 0; + int loc_len64[16]; + Ipp8u* loc_src[16]; + Ipp8u* loc_dst[16]; + __m512i iv512[4]; + + __mmask8 mbMask128[16] = { 0x03, 0x0C, 0x30, 0xC0, 0x03, 0x0C, 0x30, 0xC0, 0x03, 0x0C, 0x30, 0xC0, 0x03, 0x0C, 0x30, 0xC0 }; + __mmask8 mbMask[16] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; + + // - Local copy of length, source and target pointers, maxLen calculation + for (i = 0; i < 16; i++) { + // The case of the empty input buffer + if (arr_len[i] == 0) + { + mbMask128[i] = 0; + mbMask[i] = 0; + loc_len64[i] = 0; + continue; + } + + loc_src[i] = (Ipp8u*)source_pa[i]; + loc_dst[i] = (Ipp8u*)dst_pa[i]; + int len64 = arr_len[i] / (Ipp32s)sizeof(Ipp64u); // length in 64-bit chunks + loc_len64[i] = len64; + + if (len64 < 8) + mbMask[i] = (__mmask8)(((1 << len64) - 1) & 0xFF); + if (len64 > maxLen) + maxLen = len64; + } + + // Load the necessary number of 128-bit IV + j = 0; + for (i = 0; i < 16; i += 4) { + iv512[j] = _mm512_setzero_si512(); + + iv512[j] = _mm512_mask_expandloadu_epi64(iv512[j], mbMask128[i], iv_pa[i]); + iv512[j] = _mm512_mask_expandloadu_epi64(iv512[j], mbMask128[i + 1], iv_pa[i + 1]); + iv512[j] = _mm512_mask_expandloadu_epi64(iv512[j], mbMask128[i + 2], iv_pa[i + 2]); + iv512[j] = _mm512_mask_expandloadu_epi64(iv512[j], mbMask128[i + 3], iv_pa[i + 3]); + j += 1; + } + + // Temporary block to left IV unchanged to use it on the next round + __m512i chip0 = iv512[0]; + __m512i chip1 = iv512[1]; + __m512i chip2 = iv512[2]; + __m512i chip3 = iv512[3]; + + // Prepare array with key schedule + __m512i keySchedule[15][4]; + __m512i tmpKeyMb = _mm512_setzero_si512(); + for (i = 0; i <= num_rounds; i++) + { + k = 0; + for (j = 0; j < 16; j += 4) { + tmpKeyMb = _mm512_setzero_si512(); + tmpKeyMb = _mm512_mask_expandloadu_epi64(tmpKeyMb, mbMask128[j], (const void *)(enc_keys[j] + (Ipp32u)i * sizeof(Ipp32u))); + tmpKeyMb = _mm512_mask_expandloadu_epi64(tmpKeyMb, mbMask128[j + 1], (const void *)(enc_keys[j + 1] + (Ipp32u)i * sizeof(Ipp32u))); + tmpKeyMb = _mm512_mask_expandloadu_epi64(tmpKeyMb, mbMask128[j + 2], (const void *)(enc_keys[j + 2] + (Ipp32u)i * sizeof(Ipp32u))); + tmpKeyMb = _mm512_mask_expandloadu_epi64(tmpKeyMb, mbMask128[j + 3], (const void *)(enc_keys[j + 3] + (Ipp32u)i * sizeof(Ipp32u))); + + keySchedule[i][k] = _mm512_loadu_si512(&tmpKeyMb); + k += 1; + } + } + + for (; maxLen >= 0; maxLen -= 8) + { + __m512i b0 = _mm512_maskz_loadu_epi64(mbMask[0], loc_src[0]); loc_src[0] += CFB16_BLOCK_SIZE * 4; + __m512i b1 = _mm512_maskz_loadu_epi64(mbMask[1], loc_src[1]); loc_src[1] += CFB16_BLOCK_SIZE * 4; + __m512i b2 = _mm512_maskz_loadu_epi64(mbMask[2], loc_src[2]); loc_src[2] += CFB16_BLOCK_SIZE * 4; + __m512i b3 = _mm512_maskz_loadu_epi64(mbMask[3], loc_src[3]); loc_src[3] += CFB16_BLOCK_SIZE * 4; + __m512i b4 = _mm512_maskz_loadu_epi64(mbMask[4], loc_src[4]); loc_src[4] += CFB16_BLOCK_SIZE * 4; + __m512i b5 = _mm512_maskz_loadu_epi64(mbMask[5], loc_src[5]); loc_src[5] += CFB16_BLOCK_SIZE * 4; + __m512i b6 = _mm512_maskz_loadu_epi64(mbMask[6], loc_src[6]); loc_src[6] += CFB16_BLOCK_SIZE * 4; + __m512i b7 = _mm512_maskz_loadu_epi64(mbMask[7], loc_src[7]); loc_src[7] += CFB16_BLOCK_SIZE * 4; + __m512i b8 = _mm512_maskz_loadu_epi64(mbMask[8], loc_src[8]); loc_src[8] += CFB16_BLOCK_SIZE * 4; + __m512i b9 = _mm512_maskz_loadu_epi64(mbMask[9], loc_src[9]); loc_src[9] += CFB16_BLOCK_SIZE * 4; + __m512i b10 = _mm512_maskz_loadu_epi64(mbMask[10], loc_src[10]); loc_src[10] += CFB16_BLOCK_SIZE * 4; + __m512i b11 = _mm512_maskz_loadu_epi64(mbMask[11], loc_src[11]); loc_src[11] += CFB16_BLOCK_SIZE * 4; + __m512i b12 = _mm512_maskz_loadu_epi64(mbMask[12], loc_src[12]); loc_src[12] += CFB16_BLOCK_SIZE * 4; + __m512i b13 = _mm512_maskz_loadu_epi64(mbMask[13], loc_src[13]); loc_src[13] += CFB16_BLOCK_SIZE * 4; + __m512i b14 = _mm512_maskz_loadu_epi64(mbMask[14], loc_src[14]); loc_src[14] += CFB16_BLOCK_SIZE * 4; + __m512i b15 = _mm512_maskz_loadu_epi64(mbMask[15], loc_src[15]); loc_src[15] += CFB16_BLOCK_SIZE * 4; + + TRANSPOSE_4x4_I128(b0, b1, b2, b3); // {0,0,0,0}, {1,1,1,1}, {2,2,2,2}, {3,3,3,3} + TRANSPOSE_4x4_I128(b4, b5, b6, b7); // {0,0,0,0}, {1,1,1,1}, {2,2,2,2}, {3,3,3,3} + TRANSPOSE_4x4_I128(b8, b9, b10, b11); // {0,0,0,0}, {1,1,1,1}, {2,2,2,2}, {3,3,3,3} + TRANSPOSE_4x4_I128(b12, b13, b14, b15); // {0,0,0,0}, {1,1,1,1}, {2,2,2,2}, {3,3,3,3} + + AES_ENCRYPT_VAES_MB16(chip0, chip1, chip2, chip3, keySchedule, num_rounds); + chip0 = b0 = _mm512_xor_si512(b0, chip0); + chip1 = b4 = _mm512_xor_si512(b4, chip1); + chip2 = b8 = _mm512_xor_si512(b8, chip2); + chip3 = b12 = _mm512_xor_si512(b12, chip3); + + AES_ENCRYPT_VAES_MB16(chip0, chip1, chip2, chip3, keySchedule, num_rounds); + chip0 = b1 = _mm512_xor_si512(b1, chip0); + chip1 = b5 = _mm512_xor_si512(b5, chip1); + chip2 = b9 = _mm512_xor_si512(b9, chip2); + chip3 = b13 = _mm512_xor_si512(b13, chip3); + + AES_ENCRYPT_VAES_MB16(chip0, chip1, chip2, chip3, keySchedule, num_rounds); + chip0 = b2 = _mm512_xor_si512(b2, chip0); + chip1 = b6 = _mm512_xor_si512(b6, chip1); + chip2 = b10 = _mm512_xor_si512(b10, chip2); + chip3 = b14 = _mm512_xor_si512(b14, chip3); + + AES_ENCRYPT_VAES_MB16(chip0, chip1, chip2, chip3, keySchedule, num_rounds); + chip0 = b3 = _mm512_xor_si512(b3, chip0); + chip1 = b7 = _mm512_xor_si512(b7, chip1); + chip2 = b11 = _mm512_xor_si512(b11, chip2); + chip3 = b15 = _mm512_xor_si512(b15, chip3); + + TRANSPOSE_4x4_I128(b0, b1, b2, b3); + TRANSPOSE_4x4_I128(b4, b5, b6, b7); + TRANSPOSE_4x4_I128(b8, b9, b10, b11); + TRANSPOSE_4x4_I128(b12, b13, b14, b15); + + _mm512_mask_storeu_epi64(loc_dst[0], mbMask[0], b0); loc_dst[0] += CFB16_BLOCK_SIZE * 4; loc_len64[0] -= 2 * 4; UPDATE_MASK(loc_len64[0], mbMask[0]); + _mm512_mask_storeu_epi64(loc_dst[1], mbMask[1], b1); loc_dst[1] += CFB16_BLOCK_SIZE * 4; loc_len64[1] -= 2 * 4; UPDATE_MASK(loc_len64[1], mbMask[1]); + _mm512_mask_storeu_epi64(loc_dst[2], mbMask[2], b2); loc_dst[2] += CFB16_BLOCK_SIZE * 4; loc_len64[2] -= 2 * 4; UPDATE_MASK(loc_len64[2], mbMask[2]); + _mm512_mask_storeu_epi64(loc_dst[3], mbMask[3], b3); loc_dst[3] += CFB16_BLOCK_SIZE * 4; loc_len64[3] -= 2 * 4; UPDATE_MASK(loc_len64[3], mbMask[3]); + _mm512_mask_storeu_epi64(loc_dst[4], mbMask[4], b4); loc_dst[4] += CFB16_BLOCK_SIZE * 4; loc_len64[4] -= 2 * 4; UPDATE_MASK(loc_len64[4], mbMask[4]); + _mm512_mask_storeu_epi64(loc_dst[5], mbMask[5], b5); loc_dst[5] += CFB16_BLOCK_SIZE * 4; loc_len64[5] -= 2 * 4; UPDATE_MASK(loc_len64[5], mbMask[5]); + _mm512_mask_storeu_epi64(loc_dst[6], mbMask[6], b6); loc_dst[6] += CFB16_BLOCK_SIZE * 4; loc_len64[6] -= 2 * 4; UPDATE_MASK(loc_len64[6], mbMask[6]); + _mm512_mask_storeu_epi64(loc_dst[7], mbMask[7], b7); loc_dst[7] += CFB16_BLOCK_SIZE * 4; loc_len64[7] -= 2 * 4; UPDATE_MASK(loc_len64[7], mbMask[7]); + _mm512_mask_storeu_epi64(loc_dst[8], mbMask[8], b8); loc_dst[8] += CFB16_BLOCK_SIZE * 4; loc_len64[8] -= 2 * 4; UPDATE_MASK(loc_len64[8], mbMask[8]); + _mm512_mask_storeu_epi64(loc_dst[9], mbMask[9], b9); loc_dst[9] += CFB16_BLOCK_SIZE * 4; loc_len64[9] -= 2 * 4; UPDATE_MASK(loc_len64[9], mbMask[9]); + _mm512_mask_storeu_epi64(loc_dst[10], mbMask[10], b10); loc_dst[10] += CFB16_BLOCK_SIZE * 4; loc_len64[10] -= 2 * 4; UPDATE_MASK(loc_len64[10], mbMask[10]); + _mm512_mask_storeu_epi64(loc_dst[11], mbMask[11], b11); loc_dst[11] += CFB16_BLOCK_SIZE * 4; loc_len64[11] -= 2 * 4; UPDATE_MASK(loc_len64[11], mbMask[11]); + _mm512_mask_storeu_epi64(loc_dst[12], mbMask[12], b12); loc_dst[12] += CFB16_BLOCK_SIZE * 4; loc_len64[12] -= 2 * 4; UPDATE_MASK(loc_len64[12], mbMask[12]); + _mm512_mask_storeu_epi64(loc_dst[13], mbMask[13], b13); loc_dst[13] += CFB16_BLOCK_SIZE * 4; loc_len64[13] -= 2 * 4; UPDATE_MASK(loc_len64[13], mbMask[13]); + _mm512_mask_storeu_epi64(loc_dst[14], mbMask[14], b14); loc_dst[14] += CFB16_BLOCK_SIZE * 4; loc_len64[14] -= 2 * 4; UPDATE_MASK(loc_len64[14], mbMask[14]); + _mm512_mask_storeu_epi64(loc_dst[15], mbMask[15], b15); loc_dst[15] += CFB16_BLOCK_SIZE * 4; loc_len64[15] -= 2 * 4; UPDATE_MASK(loc_len64[15], mbMask[15]); + } +} + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + #pragma optimize( "", on ) +#endif + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_cfb_vaes_mb4.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_cfb_vaes_mb4.c new file mode 100644 index 0000000..5bfb651 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_cfb_vaes_mb4.c @@ -0,0 +1,146 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" +#include "aes_cfb_vaes_mb.h" + +#if(_IPP32E>=_IPP32E_K1) + +#define AES_ENCRYPT_VAES_MB16(b0, pRkey, num_rounds) { \ + __m512i (*tkeys) = &pRkey[num_rounds-9]; \ + b0 = _mm512_xor_si512(b0, pRkey[0]); \ + switch(num_rounds) { \ + case 14: \ + b0 = _mm512_aesenc_epi128(b0, tkeys[-4]); \ + b0 = _mm512_aesenc_epi128(b0, tkeys[-3]); \ + case 12: \ + b0 = _mm512_aesenc_epi128(b0, tkeys[-2]); \ + b0 = _mm512_aesenc_epi128(b0, tkeys[-1]); \ + default: \ + b0 = _mm512_aesenc_epi128(b0, tkeys[0]); \ + b0 = _mm512_aesenc_epi128(b0, tkeys[1]); \ + b0 = _mm512_aesenc_epi128(b0, tkeys[2]); \ + b0 = _mm512_aesenc_epi128(b0, tkeys[3]); \ + b0 = _mm512_aesenc_epi128(b0, tkeys[4]); \ + b0 = _mm512_aesenc_epi128(b0, tkeys[5]); \ + b0 = _mm512_aesenc_epi128(b0, tkeys[6]); \ + b0 = _mm512_aesenc_epi128(b0, tkeys[7]); \ + b0 = _mm512_aesenc_epi128(b0, tkeys[8]); \ + b0 = _mm512_aesenclast_epi128(b0, tkeys[9]); \ + } \ +} + +// Disable optimization for MSVC +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + #pragma optimize( "", off ) +#endif + +IPP_OWN_DEFN (void, aes_cfb16_enc_vaes_mb4, (const Ipp8u* const source_pa[4], Ipp8u* const dst_pa[4], const int arr_len[4], const int num_rounds, const Ipp32u* enc_keys[4], const Ipp8u* iv_pa[4])) +{ + int i; + int maxLen = 0; + int loc_len64[4]; + Ipp8u* loc_src[4]; + Ipp8u* loc_dst[4]; + + __mmask8 mbMask128[4] = { 0x03, 0x0C, 0x30, 0xC0 }; + __mmask8 mbMask[4] = { 0xFF, 0xFF, 0xFF, 0xFF }; + + for (i = 0; i < 4; i++) { + // The case of the empty input buffer + if (arr_len[i] == 0) + { + mbMask128[i] = 0; + mbMask[i] = 0; + loc_len64[i] = 0; + loc_src[i] = NULL; + loc_dst[i] = NULL; + continue; + } + loc_src[i] = (Ipp8u*)source_pa[i]; + loc_dst[i] = (Ipp8u*)dst_pa[i]; + int len64 = arr_len[i] / (Ipp32s)sizeof(Ipp64u); // length in 64-bit chunks + loc_len64[i] = len64; + + if (len64 < 8) + mbMask[i] = (__mmask8)(((1 << len64) - 1) & 0xFF); + if (len64 > maxLen) + maxLen = len64; + } + + // Load the necessary number of 128-bit IV + __m512i iv512 = _mm512_setzero_si512(); + iv512 = _mm512_mask_expandloadu_epi64(iv512, mbMask128[0], iv_pa[0]); // 0 0 0 IV1 + iv512 = _mm512_mask_expandloadu_epi64(iv512, mbMask128[1], iv_pa[1]); // 0 0 IV2 0 + iv512 = _mm512_mask_expandloadu_epi64(iv512, mbMask128[2], iv_pa[2]); // 0 IV3 0 0 + iv512 = _mm512_mask_expandloadu_epi64(iv512, mbMask128[3], iv_pa[3]); // IV4 0 0 0 + + // Temporary block to left IV unchanged to use it on the next round + __m512i chip0 = iv512; + + // Prepare array with key schedule + __m512i keySchedule[15]; + __m512i tmpKeyMb = _mm512_setzero_si512(); + for (i = 0; i <= num_rounds; i++) + { + tmpKeyMb = _mm512_mask_expandloadu_epi64(tmpKeyMb, mbMask128[0], (const void *)(enc_keys[0] + (Ipp32u)i * sizeof(Ipp32u))); + tmpKeyMb = _mm512_mask_expandloadu_epi64(tmpKeyMb, mbMask128[1], (const void *)(enc_keys[1] + (Ipp32u)i * sizeof(Ipp32u))); + tmpKeyMb = _mm512_mask_expandloadu_epi64(tmpKeyMb, mbMask128[2], (const void *)(enc_keys[2] + (Ipp32u)i * sizeof(Ipp32u))); + tmpKeyMb = _mm512_mask_expandloadu_epi64(tmpKeyMb, mbMask128[3], (const void *)(enc_keys[3] + (Ipp32u)i * sizeof(Ipp32u))); + + keySchedule[i] = _mm512_loadu_si512(&tmpKeyMb); + } + + for (; maxLen >= 0; maxLen -= 8) + { + // Load plain text from the different buffers + __m512i b0 = _mm512_maskz_loadu_epi64(mbMask[0], loc_src[0]); loc_src[0] += CFB16_BLOCK_SIZE * 4; + __m512i b1 = _mm512_maskz_loadu_epi64(mbMask[1], loc_src[1]); loc_src[1] += CFB16_BLOCK_SIZE * 4; + __m512i b2 = _mm512_maskz_loadu_epi64(mbMask[2], loc_src[2]); loc_src[2] += CFB16_BLOCK_SIZE * 4; + __m512i b3 = _mm512_maskz_loadu_epi64(mbMask[3], loc_src[3]); loc_src[3] += CFB16_BLOCK_SIZE * 4; + + TRANSPOSE_4x4_I128(b0, b1, b2, b3); // {0,0,0,0}, {1,1,1,1}, {2,2,2,2}, {3,3,3,3} + + AES_ENCRYPT_VAES_MB16(chip0, keySchedule, num_rounds); + chip0 = b0 = _mm512_xor_si512(b0, chip0); + + AES_ENCRYPT_VAES_MB16(chip0, keySchedule, num_rounds); + chip0 = b1 = _mm512_xor_si512(b1, chip0); + + AES_ENCRYPT_VAES_MB16(chip0, keySchedule, num_rounds); + chip0 = b2 = _mm512_xor_si512(b2, chip0); + + AES_ENCRYPT_VAES_MB16(chip0, keySchedule, num_rounds); + chip0 = b3 = _mm512_xor_si512(b3, chip0); + + TRANSPOSE_4x4_I128(b0, b1, b2, b3); + + _mm512_mask_storeu_epi64(loc_dst[0], mbMask[0], b0); loc_dst[0] += CFB16_BLOCK_SIZE * 4; loc_len64[0] -= 2 * 4; UPDATE_MASK(loc_len64[0], mbMask[0]); + _mm512_mask_storeu_epi64(loc_dst[1], mbMask[1], b1); loc_dst[1] += CFB16_BLOCK_SIZE * 4; loc_len64[1] -= 2 * 4; UPDATE_MASK(loc_len64[1], mbMask[1]); + _mm512_mask_storeu_epi64(loc_dst[2], mbMask[2], b2); loc_dst[2] += CFB16_BLOCK_SIZE * 4; loc_len64[2] -= 2 * 4; UPDATE_MASK(loc_len64[2], mbMask[2]); + _mm512_mask_storeu_epi64(loc_dst[3], mbMask[3], b3); loc_dst[3] += CFB16_BLOCK_SIZE * 4; loc_len64[3] -= 2 * 4; UPDATE_MASK(loc_len64[3], mbMask[3]); + } +} + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + #pragma optimize( "", on ) +#endif + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_cfb_vaes_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_cfb_vaes_mb8.c new file mode 100644 index 0000000..68957e6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_cfb_vaes_mb8.c @@ -0,0 +1,204 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" +#include "aes_cfb_vaes_mb.h" + +#if(_IPP32E>=_IPP32E_K1) + +#define AES_ENCRYPT_VAES_MB16(b0, b1, pRkey, num_rounds) { \ + __m512i (*tkeys)[2] = &pRkey[num_rounds-9]; \ + b0 = _mm512_xor_si512(b0, pRkey[0][0]); \ + b1 = _mm512_xor_si512(b1, pRkey[0][1]); \ + switch(num_rounds) { \ + case 14: \ + b0 = _mm512_aesenc_epi128(b0, tkeys[-4][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[-4][1]); \ + \ + b0 = _mm512_aesenc_epi128(b0, tkeys[-3][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[-3][1]); \ + case 12: \ + b0 = _mm512_aesenc_epi128(b0, tkeys[-2][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[-2][1]); \ + \ + b0 = _mm512_aesenc_epi128(b0, tkeys[-1][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[-1][1]); \ + default: \ + b0 = _mm512_aesenc_epi128(b0, tkeys[0][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[0][1]); \ + \ + b0 = _mm512_aesenc_epi128(b0, tkeys[1][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[1][1]); \ + \ + b0 = _mm512_aesenc_epi128(b0, tkeys[2][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[2][1]); \ + \ + b0 = _mm512_aesenc_epi128(b0, tkeys[3][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[3][1]); \ + \ + b0 = _mm512_aesenc_epi128(b0, tkeys[4][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[4][1]); \ + \ + b0 = _mm512_aesenc_epi128(b0, tkeys[5][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[5][1]); \ + \ + b0 = _mm512_aesenc_epi128(b0, tkeys[6][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[6][1]); \ + \ + b0 = _mm512_aesenc_epi128(b0, tkeys[7][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[7][1]); \ + \ + b0 = _mm512_aesenc_epi128(b0, tkeys[8][0]); \ + b1 = _mm512_aesenc_epi128(b1, tkeys[8][1]); \ + \ + b0 = _mm512_aesenclast_epi128(b0, tkeys[9][0]); \ + b1 = _mm512_aesenclast_epi128(b1, tkeys[9][1]); \ + } \ +} + +// Disable optimization for MSVC +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + #pragma optimize( "", off ) +#endif + +IPP_OWN_DEFN (void, aes_cfb16_enc_vaes_mb8, (const Ipp8u* const source_pa[8], Ipp8u* const dst_pa[8], const int arr_len[8], const int num_rounds, const Ipp32u* enc_keys[8], const Ipp8u* iv_pa[8])) +{ + int i; + int maxLen = 0; + int loc_len64[8]; + Ipp8u* loc_src[8]; + Ipp8u* loc_dst[8]; + __m512i iv512[2]; + + __mmask8 mbMask128[8] = { 0x03, 0x0C, 0x30, 0xC0, 0x03, 0x0C, 0x30, 0xC0 }; + __mmask8 mbMask[8] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; + + for (i = 0; i < 8; i++) { + // The case of the empty input buffer + if (arr_len[i] == 0) + { + mbMask128[i] = 0; + mbMask[i] = 0; + loc_len64[i] = 0; + loc_src[i] = NULL; + loc_dst[i] = NULL; + continue; + } + + loc_src[i] = (Ipp8u*)source_pa[i]; + loc_dst[i] = (Ipp8u*)dst_pa[i]; + int len64 = arr_len[i] / (Ipp32s)sizeof(Ipp64u); // length in 64-bit chunks + loc_len64[i] = len64; + + if (len64 < 8) + mbMask[i] = (__mmask8)(((1 << len64) - 1) & 0xFF); + if (len64 > maxLen) + maxLen = len64; + } + + // Load the necessary number of 128-bit IV + iv512[0] = _mm512_setzero_si512(); + iv512[0] = _mm512_mask_expandloadu_epi64(iv512[0], mbMask128[0], iv_pa[0]); + iv512[0] = _mm512_mask_expandloadu_epi64(iv512[0], mbMask128[1], iv_pa[1]); + iv512[0] = _mm512_mask_expandloadu_epi64(iv512[0], mbMask128[2], iv_pa[2]); + iv512[0] = _mm512_mask_expandloadu_epi64(iv512[0], mbMask128[3], iv_pa[3]); + + iv512[1] = _mm512_setzero_si512(); + iv512[1] = _mm512_mask_expandloadu_epi64(iv512[1], mbMask128[4], iv_pa[4]); + iv512[1] = _mm512_mask_expandloadu_epi64(iv512[1], mbMask128[5], iv_pa[5]); + iv512[1] = _mm512_mask_expandloadu_epi64(iv512[1], mbMask128[6], iv_pa[6]); + iv512[1] = _mm512_mask_expandloadu_epi64(iv512[1], mbMask128[7], iv_pa[7]); + + + // Temporary block to left IV unchanged to use it on the next round + __m512i chip0 = iv512[0]; + __m512i chip1 = iv512[1]; + + // Prepare array with key schedule + __m512i keySchedule[15][2]; + __m512i tmpKeyMb = _mm512_setzero_si512(); + + for (i = 0; i <= num_rounds; i++) + { + tmpKeyMb = _mm512_mask_expandloadu_epi64(tmpKeyMb, mbMask128[0], (const void *)(enc_keys[0] + (Ipp32u)i * sizeof(Ipp32u))); + tmpKeyMb = _mm512_mask_expandloadu_epi64(tmpKeyMb, mbMask128[1], (const void *)(enc_keys[1] + (Ipp32u)i * sizeof(Ipp32u))); + tmpKeyMb = _mm512_mask_expandloadu_epi64(tmpKeyMb, mbMask128[2], (const void *)(enc_keys[2] + (Ipp32u)i * sizeof(Ipp32u))); + tmpKeyMb = _mm512_mask_expandloadu_epi64(tmpKeyMb, mbMask128[3], (const void *)(enc_keys[3] + (Ipp32u)i * sizeof(Ipp32u))); + + keySchedule[i][0] = _mm512_loadu_si512(&tmpKeyMb); + + tmpKeyMb = _mm512_mask_expandloadu_epi64(tmpKeyMb, mbMask128[4], (const void *)(enc_keys[4] + (Ipp32u)i * sizeof(Ipp32u))); + tmpKeyMb = _mm512_mask_expandloadu_epi64(tmpKeyMb, mbMask128[5], (const void *)(enc_keys[5] + (Ipp32u)i * sizeof(Ipp32u))); + tmpKeyMb = _mm512_mask_expandloadu_epi64(tmpKeyMb, mbMask128[6], (const void *)(enc_keys[6] + (Ipp32u)i * sizeof(Ipp32u))); + tmpKeyMb = _mm512_mask_expandloadu_epi64(tmpKeyMb, mbMask128[7], (const void *)(enc_keys[7] + (Ipp32u)i * sizeof(Ipp32u))); + + keySchedule[i][1] = _mm512_loadu_si512(&tmpKeyMb); + } + + for (; maxLen >= 0; maxLen -= 8) + { + // Load plain text from the different buffers + __m512i b0 = _mm512_maskz_loadu_epi64(mbMask[0], loc_src[0]); loc_src[0] += CFB16_BLOCK_SIZE * 4; + __m512i b1 = _mm512_maskz_loadu_epi64(mbMask[1], loc_src[1]); loc_src[1] += CFB16_BLOCK_SIZE * 4; + __m512i b2 = _mm512_maskz_loadu_epi64(mbMask[2], loc_src[2]); loc_src[2] += CFB16_BLOCK_SIZE * 4; + __m512i b3 = _mm512_maskz_loadu_epi64(mbMask[3], loc_src[3]); loc_src[3] += CFB16_BLOCK_SIZE * 4; + __m512i b4 = _mm512_maskz_loadu_epi64(mbMask[4], loc_src[4]); loc_src[4] += CFB16_BLOCK_SIZE * 4; + __m512i b5 = _mm512_maskz_loadu_epi64(mbMask[5], loc_src[5]); loc_src[5] += CFB16_BLOCK_SIZE * 4; + __m512i b6 = _mm512_maskz_loadu_epi64(mbMask[6], loc_src[6]); loc_src[6] += CFB16_BLOCK_SIZE * 4; + __m512i b7 = _mm512_maskz_loadu_epi64(mbMask[7], loc_src[7]); loc_src[7] += CFB16_BLOCK_SIZE * 4; + + TRANSPOSE_4x4_I128(b0, b1, b2, b3); // {0,0,0,0}, {1,1,1,1}, {2,2,2,2}, {3,3,3,3} + TRANSPOSE_4x4_I128(b4, b5, b6, b7); // {0,0,0,0}, {1,1,1,1}, {2,2,2,2}, {3,3,3,3} + + AES_ENCRYPT_VAES_MB16(chip0, chip1, keySchedule, num_rounds); + chip0 = b0 = _mm512_xor_si512(b0, chip0); + chip1 = b4 = _mm512_xor_si512(b4, chip1); + + AES_ENCRYPT_VAES_MB16(chip0, chip1, keySchedule, num_rounds); + chip0 = b1 = _mm512_xor_si512(b1, chip0); + chip1 = b5 = _mm512_xor_si512(b5, chip1); + + AES_ENCRYPT_VAES_MB16(chip0, chip1, keySchedule, num_rounds); + chip0 = b2 = _mm512_xor_si512(b2, chip0); + chip1 = b6 = _mm512_xor_si512(b6, chip1); + + AES_ENCRYPT_VAES_MB16(chip0, chip1, keySchedule, num_rounds); + chip0 = b3 = _mm512_xor_si512(b3, chip0); + chip1 = b7 = _mm512_xor_si512(b7, chip1); + + TRANSPOSE_4x4_I128(b0, b1, b2, b3); + TRANSPOSE_4x4_I128(b4, b5, b6, b7); + + _mm512_mask_storeu_epi64(loc_dst[0], mbMask[0], b0); loc_dst[0] += CFB16_BLOCK_SIZE * 4; loc_len64[0] -= 2 * 4; UPDATE_MASK(loc_len64[0], mbMask[0]); + _mm512_mask_storeu_epi64(loc_dst[1], mbMask[1], b1); loc_dst[1] += CFB16_BLOCK_SIZE * 4; loc_len64[1] -= 2 * 4; UPDATE_MASK(loc_len64[1], mbMask[1]); + _mm512_mask_storeu_epi64(loc_dst[2], mbMask[2], b2); loc_dst[2] += CFB16_BLOCK_SIZE * 4; loc_len64[2] -= 2 * 4; UPDATE_MASK(loc_len64[2], mbMask[2]); + _mm512_mask_storeu_epi64(loc_dst[3], mbMask[3], b3); loc_dst[3] += CFB16_BLOCK_SIZE * 4; loc_len64[3] -= 2 * 4; UPDATE_MASK(loc_len64[3], mbMask[3]); + _mm512_mask_storeu_epi64(loc_dst[4], mbMask[4], b4); loc_dst[4] += CFB16_BLOCK_SIZE * 4; loc_len64[4] -= 2 * 4; UPDATE_MASK(loc_len64[4], mbMask[4]); + _mm512_mask_storeu_epi64(loc_dst[5], mbMask[5], b5); loc_dst[5] += CFB16_BLOCK_SIZE * 4; loc_len64[5] -= 2 * 4; UPDATE_MASK(loc_len64[5], mbMask[5]); + _mm512_mask_storeu_epi64(loc_dst[6], mbMask[6], b6); loc_dst[6] += CFB16_BLOCK_SIZE * 4; loc_len64[6] -= 2 * 4; UPDATE_MASK(loc_len64[6], mbMask[6]); + _mm512_mask_storeu_epi64(loc_dst[7], mbMask[7], b7); loc_dst[7] += CFB16_BLOCK_SIZE * 4; loc_len64[7] -= 2 * 4; UPDATE_MASK(loc_len64[7], mbMask[7]); + } +} + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + #pragma optimize( "", on ) +#endif + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_gcm_avx512.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_gcm_avx512.h new file mode 100644 index 0000000..8c89bc4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_gcm_avx512.h @@ -0,0 +1,78 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES GCM AVX512 +// Internal Functions Prototypes +// +*/ + +#ifndef __AES_GCM_AVX512_H_ +#define __AES_GCM_AVX512_H_ + +#include "owndefs.h" +#include "owncp.h" + +#include "aes_gcm_avx512_structures.h" + +#if(_IPP32E>=_IPP32E_K0) + +#define aes_gcm_enc_128_update_avx512 OWNAPI(aes_gcm_enc_128_update_avx512) + IPP_OWN_DECL (void, aes_gcm_enc_128_update_avx512, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, Ipp8u *out, const Ipp8u *in, Ipp64u len)) +#define aes_gcm_enc_192_update_avx512 OWNAPI(aes_gcm_enc_192_update_avx512) + IPP_OWN_DECL (void, aes_gcm_enc_192_update_avx512, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, Ipp8u *out, const Ipp8u *in, Ipp64u len)) +#define aes_gcm_enc_256_update_avx512 OWNAPI(aes_gcm_enc_256_update_avx512) + IPP_OWN_DECL (void, aes_gcm_enc_256_update_avx512, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, Ipp8u *out, const Ipp8u *in, Ipp64u len)) + +#define aes_gcm_dec_128_update_avx512 OWNAPI(aes_gcm_dec_128_update_avx512) + IPP_OWN_DECL (void, aes_gcm_dec_128_update_avx512, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, Ipp8u *out, const Ipp8u *in, Ipp64u len)) +#define aes_gcm_dec_192_update_avx512 OWNAPI(aes_gcm_dec_192_update_avx512) + IPP_OWN_DECL (void, aes_gcm_dec_192_update_avx512, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, Ipp8u *out, const Ipp8u *in, Ipp64u len)) +#define aes_gcm_dec_256_update_avx512 OWNAPI(aes_gcm_dec_256_update_avx512) + IPP_OWN_DECL (void, aes_gcm_dec_256_update_avx512, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, Ipp8u *out, const Ipp8u *in, Ipp64u len)) + +#define aes_gcm_gettag_128_avx512 OWNAPI(aes_gcm_gettag_128_avx512) + IPP_OWN_DECL (void, aes_gcm_gettag_128_avx512, (const struct gcm_key_data *key_data, const struct gcm_context_data *context_data, Ipp8u *auth_tag, Ipp64u auth_tag_len)) +#define aes_gcm_gettag_192_avx512 OWNAPI(aes_gcm_gettag_192_avx512) + IPP_OWN_DECL (void, aes_gcm_gettag_192_avx512, (const struct gcm_key_data *key_data, const struct gcm_context_data *context_data, Ipp8u *auth_tag, Ipp64u auth_tag_len)) +#define aes_gcm_gettag_256_avx512 OWNAPI(aes_gcm_gettag_256_avx512) + IPP_OWN_DECL (void, aes_gcm_gettag_256_avx512, (const struct gcm_key_data *key_data, const struct gcm_context_data *context_data, Ipp8u *auth_tag, Ipp64u auth_tag_len)) + +#define aes_gcm_precomp_128_avx512 OWNAPI(aes_gcm_precomp_128_avx512) + IPP_OWN_DECL (void, aes_gcm_precomp_128_avx512, (struct gcm_key_data *key_data)) +#define aes_gcm_precomp_192_avx512 OWNAPI(aes_gcm_precomp_192_avx512) + IPP_OWN_DECL (void, aes_gcm_precomp_192_avx512, (struct gcm_key_data *key_data)) +#define aes_gcm_precomp_256_avx512 OWNAPI(aes_gcm_precomp_256_avx512) + IPP_OWN_DECL (void, aes_gcm_precomp_256_avx512, (struct gcm_key_data *key_data)) + +#define aes_gcm_aad_hash_update_avx512 OWNAPI(aes_gcm_aad_hash_update_avx512) + IPP_OWN_DECL (void, aes_gcm_aad_hash_update_avx512, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const Ipp8u *aad, const Ipp64u aad_len)) + +#define aes_gcm_iv_hash_update_avx512 OWNAPI(aes_gcm_iv_hash_update_avx512) + IPP_OWN_DECL (void, aes_gcm_iv_hash_update_avx512, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const Ipp8u *iv, const Ipp64u iv_len)) +#define aes_gcm_iv_hash_finalize_avx512 OWNAPI(aes_gcm_iv_hash_finalize_avx512) + IPP_OWN_DECL (void, aes_gcm_iv_hash_finalize_avx512, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const Ipp8u *iv, const Ipp64u iv_len, const Ipp64u iv_general_len)) + +#define aes_gcm_gmult_avx512 OWNAPI(aes_gcm_gmult_avx512) + IPP_OWN_DECL (void, aes_gcm_gmult_avx512, (const struct gcm_key_data *key_data, Ipp8u *ghash)) + +#endif /* #if(_IPP32E>=_IPP32E_K0) */ + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_gcm_avx512_structures.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_gcm_avx512_structures.h new file mode 100644 index 0000000..b604c5a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_gcm_avx512_structures.h @@ -0,0 +1,76 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES GCM otimized for AVX512 and AVX512-VAES features +// Internal Definitions +// +// +*/ + +#ifndef __AES_GCM_AVX512_STRUCTURES_H_ +#define __AES_GCM_AVX512_STRUCTURES_H_ + +#include "owndefs.h" +#include "owncp.h" + +// declaration of internal structures used for AVX512 AES GCM optimization + +/* GCM data structures */ +#define GCM_BLOCK_LEN 16 + +/** + * \brief holds GCM operation context + */ +struct gcm_context_data { + /* init, update and finalize context data */ + Ipp8u aad_hash[GCM_BLOCK_LEN]; + Ipp64u aad_length; + Ipp64u in_length; + Ipp8u partial_block_enc_key[GCM_BLOCK_LEN]; + Ipp8u orig_IV[GCM_BLOCK_LEN]; + Ipp8u current_counter[GCM_BLOCK_LEN]; + Ipp64u partial_block_length; +}; + +/* #define GCM_BLOCK_LEN 16 */ +#define GCM_ENC_KEY_LEN 16 +#define GCM_KEY_SETS (15) /*exp key + 14 exp round keys*/ + +/** + * \brief holds intermediate key data needed to improve performance + * + * gcm_key_data hold internal key information used by gcm128, gcm192 and gcm256. + */ +struct gcm_key_data { + Ipp8u expanded_keys[GCM_ENC_KEY_LEN * GCM_KEY_SETS]; + /* + * (HashKey<<1 mod poly), (HashKey^2<<1 mod poly), ..., + * (Hashkey^48<<1 mod poly) + */ + Ipp8u shifted_hkey[GCM_ENC_KEY_LEN * 48]; +} +#ifdef LINUX +__attribute__((aligned(64))); +#else +; +#endif + +#endif // __AES_GCM_AVX512_STRUCTURES_H_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_gcm_vaes.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_gcm_vaes.h new file mode 100644 index 0000000..fa706d7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_gcm_vaes.h @@ -0,0 +1,78 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES GCM AVX512-VAES +// Internal Functions Prototypes +// +*/ + +#ifndef __AES_GCM_VAES_H_ +#define __AES_GCM_VAES_H_ + +#include "owndefs.h" +#include "owncp.h" + +#include "aes_gcm_avx512_structures.h" + +#if(_IPP32E>=_IPP32E_K0) + +#define aes_gcm_enc_128_update_vaes_avx512 OWNAPI(aes_gcm_enc_128_update_vaes_avx512) + IPP_OWN_DECL (void, aes_gcm_enc_128_update_vaes_avx512, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, Ipp8u *out, const Ipp8u *in, Ipp64u len)) +#define aes_gcm_enc_192_update_vaes_avx512 OWNAPI(aes_gcm_enc_192_update_vaes_avx512) + IPP_OWN_DECL (void, aes_gcm_enc_192_update_vaes_avx512, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, Ipp8u *out, const Ipp8u *in, Ipp64u len)) +#define aes_gcm_enc_256_update_vaes_avx512 OWNAPI(aes_gcm_enc_256_update_vaes_avx512) + IPP_OWN_DECL (void, aes_gcm_enc_256_update_vaes_avx512, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, Ipp8u *out, const Ipp8u *in, Ipp64u len)) + +#define aes_gcm_dec_128_update_vaes_avx512 OWNAPI(aes_gcm_dec_128_update_vaes_avx512) + IPP_OWN_DECL (void, aes_gcm_dec_128_update_vaes_avx512, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, Ipp8u *out, const Ipp8u *in, Ipp64u len)) +#define aes_gcm_dec_192_update_vaes_avx512 OWNAPI(aes_gcm_dec_192_update_vaes_avx512) + IPP_OWN_DECL (void, aes_gcm_dec_192_update_vaes_avx512, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, Ipp8u *out, const Ipp8u *in, Ipp64u len)) +#define aes_gcm_dec_256_update_vaes_avx512 OWNAPI(aes_gcm_dec_256_update_vaes_avx512) + IPP_OWN_DECL (void, aes_gcm_dec_256_update_vaes_avx512, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, Ipp8u *out, const Ipp8u *in, Ipp64u len)) + +#define aes_gcm_gettag_128_vaes_avx512 OWNAPI(aes_gcm_gettag_128_vaes_avx512) + IPP_OWN_DECL (void, aes_gcm_gettag_128_vaes_avx512, (const struct gcm_key_data *key_data, const struct gcm_context_data *context_data, Ipp8u *auth_tag, Ipp64u auth_tag_len)) +#define aes_gcm_gettag_192_vaes_avx512 OWNAPI(aes_gcm_gettag_192_vaes_avx512) + IPP_OWN_DECL (void, aes_gcm_gettag_192_vaes_avx512, (const struct gcm_key_data *key_data, const struct gcm_context_data *context_data, Ipp8u *auth_tag, Ipp64u auth_tag_len)) +#define aes_gcm_gettag_256_vaes_avx512 OWNAPI(aes_gcm_gettag_256_vaes_avx512) + IPP_OWN_DECL (void, aes_gcm_gettag_256_vaes_avx512, (const struct gcm_key_data *key_data, const struct gcm_context_data *context_data, Ipp8u *auth_tag, Ipp64u auth_tag_len)) + +#define aes_gcm_precomp_128_vaes_avx512 OWNAPI(aes_gcm_precomp_128_vaes_avx512) + IPP_OWN_DECL (void, aes_gcm_precomp_128_vaes_avx512, (struct gcm_key_data *key_data)) +#define aes_gcm_precomp_192_vaes_avx512 OWNAPI(aes_gcm_precomp_192_vaes_avx512) + IPP_OWN_DECL (void, aes_gcm_precomp_192_vaes_avx512, (struct gcm_key_data *key_data)) +#define aes_gcm_precomp_256_vaes_avx512 OWNAPI(aes_gcm_precomp_256_vaes_avx512) + IPP_OWN_DECL (void, aes_gcm_precomp_256_vaes_avx512, (struct gcm_key_data *key_data)) + +#define aes_gcm_aad_hash_update_vaes512 OWNAPI(aes_gcm_aad_hash_update_vaes512) + IPP_OWN_DECL (void, aes_gcm_aad_hash_update_vaes512, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const Ipp8u *aad, const Ipp64u aad_len)) + +#define aes_gcm_iv_hash_update_vaes512 OWNAPI(aes_gcm_iv_hash_update_vaes512) + IPP_OWN_DECL (void, aes_gcm_iv_hash_update_vaes512, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const Ipp8u *iv, const Ipp64u iv_len)) +#define aes_gcm_iv_hash_finalize_vaes512 OWNAPI(aes_gcm_iv_hash_finalize_vaes512) + IPP_OWN_DECL (void, aes_gcm_iv_hash_finalize_vaes512, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const Ipp8u *iv, const Ipp64u iv_len, const Ipp64u iv_general_len)) + +#define aes_gcm_gmult_vaes512 OWNAPI(aes_gcm_gmult_vaes512) + IPP_OWN_DECL (void, aes_gcm_gmult_vaes512, (const struct gcm_key_data *key_data, Ipp8u *ghash)) + +#endif /* #if(_IPP32E>=_IPP32E_K0) */ + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_keyexp.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_keyexp.h new file mode 100644 index 0000000..81e5d3f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/aes_keyexp.h @@ -0,0 +1,48 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES Key Expansion +// Internal Definitions +// +*/ + +#ifndef __AES_KEYEXP_H_ +#define __AES_KEYEXP_H_ + +#include "owndefs.h" +#include "owncp.h" + +#include "pcpaesauthgcm_avx512.h" + +// These functions for key expansion are used only with AVX512 and AVX512-VAES optimizations for AES GCM +// TODO: replase AVX2 keyexp with AVX512 keyexp +#if(_IPP32E>=_IPP32E_K0) + +#define aes_keyexp_128_enc OWNAPI(aes_keyexp_128_enc) + IPP_OWN_DECL (void, aes_keyexp_128_enc, (const Ipp8u* key, struct gcm_key_data *key_data)) +#define aes_keyexp_192_enc OWNAPI(aes_keyexp_192_enc) + IPP_OWN_DECL (void, aes_keyexp_192_enc, (const Ipp8u* key, struct gcm_key_data *key_data)) +#define aes_keyexp_256_enc OWNAPI(aes_keyexp_256_enc) + IPP_OWN_DECL (void, aes_keyexp_256_enc, (const Ipp8u* key, struct gcm_key_data *key_data)) + +#endif // (_IPP32E>=_IPP32E_K0) + +#endif // __AES_KEYEXP_H_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/cpinitas.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/cpinitas.asm new file mode 100644 index 0000000..7209f0c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/cpinitas.asm @@ -0,0 +1,733 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%assign LOCAL_ALIGN_FACTOR 32 + +segment .text align=LOCAL_ALIGN_FACTOR + +%ifdef _IPP_DATA + +;#################################################################### +;# void ownGetReg( int* buf, int valueEAX, int valueECX ); # +;#################################################################### + +%define buf [esp+12] +%define valueEAX [esp+16] +%define valueECX [esp+20] + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC cpGetReg,PUBLIC + + push ebx + push esi + mov eax, valueEAX + mov ecx, valueECX + xor ebx, ebx + xor edx, edx + mov esi, buf + cpuid + mov [esi], eax + mov [esi + 4], ebx + mov [esi + 8], ecx + mov [esi + 12], edx + pop esi + pop ebx + ret +ENDFUNC cpGetReg + +;################################################### + +; Feature information after XGETBV(ECX=0), EAX, bits 2,1 ( XMM state and YMM state are enabled by OS ) +%assign XGETBV_MASK 06h +; OSXSAVE support, feature information after cpuid(1), ECX, bit 27 ( XGETBV is enabled by OS ) +%assign XSAVEXGETBV_FLAG 8000000h +%assign XGETBV_AVX512_MASK 0E0h + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC cp_is_avx_extension,PUBLIC + USES_GPR ebx,edx,ecx + mov eax, 1 + cpuid + xor eax, eax + and ecx, 018000000h + cmp ecx, 018000000h + jne .not_avx + xor ecx, ecx + db 00fh,001h,0d0h ; xgetbv + mov ecx, eax + xor eax, eax + and ecx, XGETBV_MASK + cmp ecx, XGETBV_MASK + jne .not_avx + mov eax, 1 +.not_avx: + REST_GPR + ret +ENDFUNC cp_is_avx_extension + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC cp_is_avx512_extension,PUBLIC + USES_GPR ebx,edx,ecx + mov eax, 1 + cpuid + xor eax, eax + and ecx, XSAVEXGETBV_FLAG + cmp ecx, XSAVEXGETBV_FLAG + jne .not_avx512 + xor ecx, ecx + db 00fh,001h,0d0h ; xgetbv + mov ecx, eax + xor eax, eax + and ecx, XGETBV_AVX512_MASK + cmp ecx, XGETBV_AVX512_MASK + jne .not_avx512 + mov eax, 1 +.not_avx512: + REST_GPR + ret +ENDFUNC cp_is_avx512_extension + +%ifdef LINUX32 + %ifndef OSX32 + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC __ashldi3,PUBLIC,WEAK + mov eax, [esp+4] + mov edx, [esp+8] + mov ecx, [esp+12] + test cl, 20H + je .less + mov edx, eax + xor eax, eax + shl edx, cl + ret +.less: + shld edx, eax, cl + shl eax, cl + ret +ENDFUNC __ashldi3 + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC __ashrdi3,PUBLIC,WEAK + mov eax, [esp+4] + mov edx, [esp+8] + mov ecx, [esp+12] + test cl, 20H + je .less + mov eax, edx + sar edx, 1FH + sar eax, cl + ret +.less: + shrd eax, edx, cl + sar edx, cl + ret +ENDFUNC __ashrdi3 + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC __divdi3,PUBLIC,WEAK + xor ecx, ecx + mov eax, dword [8+esp] + or eax, eax + jge .Apositive + mov ecx, 1 + mov edx, dword [4+esp] + neg eax + neg edx + sbb eax, 0 + mov dword [4+esp], edx + mov dword [8+esp], eax +.Apositive: + mov eax, dword [16+esp] + or eax,eax + jge .ABpositive + sub ecx, 1 + mov edx, dword [12+esp] + neg eax + neg edx + sbb eax, 0 + mov dword [12+esp], edx + mov dword [16+esp], eax +.ABpositive: + mov eax, dword [16+esp] + xor edx, edx + push ecx + test eax, eax + jne .non_zero_hi + mov eax, dword [12+esp] + div dword [16+esp] + mov ecx, eax + mov eax, dword [8+esp] + div dword [16+esp] + mov edx, ecx + jmp .return +.non_zero_hi: + mov ecx, dword [12+esp] + cmp eax, ecx + jb .divisor_greater + jne .return_zero + mov ecx, dword [16+esp] + mov eax, dword [8+esp] + cmp ecx, eax + ja .return_zero +.return_one: + mov eax, 1 + jmp .return +.return_zero: + add esp, 4 + xor eax, eax + ret +.divisor_greater: + test eax, 80000000h + jne .return_one +.find_hi_bit: + bsr ecx, eax + add ecx, 1 +.hi_bit_found: + mov edx, dword [16+esp] + push ebx + shrd edx, eax, cl + mov ebx, edx + mov eax, dword [12+esp] + mov edx, dword [16+esp] + shrd eax, edx, cl + shr edx, cl +.make_div: + div ebx + mov ebx, eax + mul dword [24+esp] + mov ecx, eax + mov eax, dword [20+esp] + mul ebx + add edx, ecx + jb .need_dec + cmp dword [16+esp], edx + jb .need_dec + ja .after_dec + cmp dword [12+esp], eax + jae .after_dec +.need_dec: + sub ebx, 1 +.after_dec: + xor edx, edx + mov eax, ebx + pop ebx +.return: + pop ecx + test ecx, ecx + jne .ch_sign + ret +.ch_sign: + neg edx + neg eax + sbb edx, 0 + ret +ENDFUNC __divdi3 + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC __udivdi3,PUBLIC,WEAK + xor ecx,ecx +.ABpositive: + mov eax, dword [16+esp] + xor edx, edx + push ecx + test eax, eax + jne .non_zero_hi + mov eax, dword [12+esp] + div dword [16+esp] + mov ecx, eax + mov eax, dword [8+esp] + div dword [16+esp] + mov edx, ecx + jmp .return +.non_zero_hi: + mov ecx, dword [12+esp] + cmp eax, ecx + jb .divisor_greater + jne .return_zero + mov ecx, dword [16+esp] + mov eax, dword [8+esp] + cmp ecx, eax + ja .return_zero +.return_one: + mov eax, 1 + jmp .return +.return_zero: + add esp, 4 + xor eax, eax + ret +.divisor_greater: + test eax, 80000000h + jne .return_one +.find_hi_bit: + bsr ecx, eax + add ecx, 1 +.hi_bit_found: + mov edx, dword [16+esp] + push ebx + shrd edx, eax, cl + mov ebx, edx + mov eax, dword [12+esp] + mov edx, dword [16+esp] + shrd eax, edx, cl + shr edx, cl +.make_div: + div ebx + mov ebx, eax + mul dword [24+esp] + mov ecx, eax + mov eax, dword [20+esp] + mul ebx + add edx, ecx + jb .need_dec + cmp dword [16+esp], edx + jb .need_dec + ja .after_dec + cmp dword [12+esp], eax + jae .after_dec +.need_dec: + sub ebx, 1 +.after_dec: + xor edx, edx + mov eax, ebx + pop ebx +.return: + pop ecx + test ecx, ecx + jne .ch_sign + ret +.ch_sign: + neg edx + neg eax + sbb edx, 0 + ret +ENDFUNC __udivdi3 + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC __moddi3,PUBLIC,WEAK + sub esp, 8 + mov dword [esp], 0 + mov eax, dword [esp+16] + or eax, eax + jge .Apositive + inc dword [esp] + mov edx, dword [esp+12] + neg eax + neg edx + sbb eax, 0 + mov dword [esp+16], eax + mov dword [esp+12], edx + +.Apositive: + mov eax, dword [esp+24] + or eax, eax + jge .ABpositive + mov edx, dword [esp+20] + neg eax + neg edx + sbb eax, 0 + mov dword [esp+24], eax + mov dword [esp+20], edx + jmp .ABpositive + lea esi, [esi] + lea edi, [edi] + + sub esp, 8 + mov dword [esp], 0 + +.ABpositive: + mov eax, dword [esp+24] + test eax, eax + jne .non_zero_hi + mov eax, dword [esp+16] + mov edx, 0 + div dword [esp+20] + mov ecx, eax + mov eax, dword [12+esp] + div dword [20+esp] + mov eax, edx + xor edx, edx + jmp .return + +.non_zero_hi: + mov ecx, dword [16+esp] + cmp eax, ecx + jb .divisor_greater + jne .return_devisor + mov ecx, dword [20+esp] + cmp ecx, dword [12+esp] + ja .return_devisor + +.return_dif: + mov eax, dword [12+esp] + mov edx, dword [16+esp] + sub eax, dword [20+esp] + sbb edx, dword [24+esp] + jmp .return + +.return_devisor: + mov eax, dword [12+esp] + mov edx, dword [16+esp] + jmp .return + +.divisor_greater: + test eax, 80000000h + jne .return_dif + +.find_hi_bit: + bsr ecx, eax + add ecx, 1 + +.hi_bit_found: + push ebx + mov edx, dword [24+esp] + shrd edx, eax, cl + mov ebx, edx + mov eax, dword [16+esp] + mov edx, dword [20+esp] + shrd eax, edx, cl + shr edx, cl + div ebx + mov ebx, eax + +.multiple: + mul dword [28+esp] + mov ecx, eax + mov eax, dword [24+esp] + mul ebx + add edx, ecx + jb .need_dec + cmp dword [20+esp], edx + jb .need_dec + ja .after_dec + cmp dword [16+esp], eax + jb .need_dec + +.after_dec: + mov ebx, eax + mov eax, dword [16+esp] + sub eax, ebx + mov ebx, edx + mov edx, dword [20+esp] + sbb edx, ebx + pop ebx + +.return: + mov dword [4+esp], eax + mov eax, dword [esp] + test eax, eax + jne .ch_sign + mov eax, dword [4+esp] + add esp, 8 + ret + +.ch_sign: + mov eax, dword [4+esp] + neg edx + neg eax + sbb edx, 0 + add esp, 8 + ret + +.need_dec: + dec ebx + mov eax, ebx + jmp .multiple +ENDFUNC __moddi3 + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC __umoddi3,PUBLIC,WEAK + sub esp, 8 + mov dword [esp], 0 + +.ABpositive: + mov eax, dword [esp+24] + test eax, eax + jne .non_zero_hi + mov eax, dword [esp+16] + mov edx, 0 + div dword [esp+20] + mov ecx, eax + mov eax, dword [12+esp] + div dword [20+esp] + mov eax, edx + xor edx, edx + jmp .return + +.non_zero_hi: + mov ecx, dword [16+esp] + cmp eax, ecx + jb .divisor_greater + jne .return_devisor + mov ecx, dword [20+esp] + cmp ecx, dword [12+esp] + ja .return_devisor + +.return_dif: + mov eax, dword [12+esp] + mov edx, dword [16+esp] + sub eax, dword [20+esp] + sbb edx, dword [24+esp] + jmp .return + +.return_devisor: + mov eax, dword [12+esp] + mov edx, dword [16+esp] + jmp .return + +.divisor_greater: + test eax, 80000000h + jne .return_dif + +.find_hi_bit: + bsr ecx, eax + add ecx, 1 + +.hi_bit_found: + push ebx + mov edx, dword [24+esp] + shrd edx, eax, cl + mov ebx, edx + mov eax, dword [16+esp] + mov edx, dword [20+esp] + shrd eax, edx, cl + shr edx, cl + div ebx + mov ebx, eax + +.multiple: + mul dword [28+esp] + mov ecx, eax + mov eax, dword [24+esp] + mul ebx + add edx, ecx + jb .need_dec + cmp dword [20+esp], edx + jb .need_dec + ja .after_dec + cmp dword [16+esp], eax + jb .need_dec + +.after_dec: + mov ebx, eax + mov eax, dword [16+esp] + sub eax, ebx + mov ebx, edx + mov edx, dword [20+esp] + sbb edx, ebx + pop ebx + +.return: + mov dword [4+esp], eax + mov eax, dword [esp] + test eax, eax + jne .ch_sign + mov eax, dword [4+esp] + add esp, 8 + ret + +.ch_sign: + mov eax, dword [4+esp] + neg edx + neg eax + sbb edx, 0 + add esp, 8 + ret + +.need_dec: + dec ebx + mov eax, ebx + jmp .multiple +ENDFUNC __umoddi3 + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC __muldi3,PUBLIC,WEAK + mov eax, dword [esp+8] + mul dword [esp+12] + mov ecx, eax + mov eax, dword [esp+4] + mul dword [esp+16] + add ecx, eax + mov eax, dword [esp+4] + mul dword [esp+12] + add edx, ecx + ret +ENDFUNC __muldi3 + + %endif; IFNDEF OSX32 +%endif; IFDEF LINUX32 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC cp_get_pentium_counter,PUBLIC + rdtsc + ret +ENDFUNC cp_get_pentium_counter + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC cpStartTscp,PUBLIC + push ebx + xor eax, eax + cpuid + pop ebx + rdtscp + ret +ENDFUNC cpStartTscp + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC cpStopTscp,PUBLIC + rdtscp + push eax + push edx + push ebx + xor eax, eax + cpuid + pop ebx + pop edx + pop eax + ret +ENDFUNC cpStopTscp + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC cpStartTsc,PUBLIC + push ebx + xor eax, eax + cpuid + pop ebx + rdtsc + ret +ENDFUNC cpStartTsc + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC cpStopTsc,PUBLIC + rdtsc + push eax + push edx + push ebx + xor eax, eax + cpuid + pop ebx + pop edx + pop eax + ret +ENDFUNC cpStopTsc + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;***************************************** +; int cpGetCacheSize( int* tableCache ); +align LOCAL_ALIGN_FACTOR +%define table [36+esp] +DECLARE_FUNC cpGetCacheSize,PUBLIC + push edi + push esi + push ebx + push ebp + sub esp, 16 + + mov edi, table + mov ebp, esp + xor esi, esi + + mov eax, 2 + cpuid + + cmp al, 1 + jne .GetCacheSize_11 + + test eax, 080000000h + jz .GetCacheSize_00 + xor eax, eax +.GetCacheSize_00: + test ebx, 080000000h + jz .GetCacheSize_01 + xor ebx, ebx +.GetCacheSize_01: + test ecx, 080000000h + jz .GetCacheSize_02 + xor ecx, ecx +.GetCacheSize_02: + test edx, 080000000h + jz .GetCacheSize_03 + xor edx, edx + +.GetCacheSize_03: + test eax, eax + jz .GetCacheSize_04 + mov [ebp], eax + add ebp, 4 + add esi, 3 +.GetCacheSize_04: + test ebx, ebx + jz .GetCacheSize_05 + mov [ebp], ebx + add ebp, 4 + add esi, 4 +.GetCacheSize_05: + test ecx, ecx + jz .GetCacheSize_06 + mov [ebp], ecx + add ebp, 4 + add esi, 4 +.GetCacheSize_06: + test edx, edx + jz .GetCacheSize_07 + mov [ebp], edx + add esi, 4 + +.GetCacheSize_07: + test esi, esi + jz .GetCacheSize_11 + mov eax, -1 +.GetCacheSize_08: + xor edx, edx + add edx, [edi] + jz .ExitGetCacheSize00 + add edi, 8 + mov ecx, esi +.GetCacheSize_09: + cmp dl, BYTE [esp + ecx] + je .GetCacheSize_10 + dec ecx + jnz .GetCacheSize_09 + jmp .GetCacheSize_08 + +.GetCacheSize_10: + mov eax, [edi - 4] + +.ExitGetCacheSize00: + add esp, 16 + pop ebp + pop ebx + pop esi + pop edi + ret + +.GetCacheSize_11: + mov eax, -1 + jmp .ExitGetCacheSize00 +ENDFUNC cpGetCacheSize + +; ***************************************** + +%endif ; IPP_DATA diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/emulator.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/emulator.inc new file mode 100644 index 0000000..667b3f9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/emulator.inc @@ -0,0 +1,441 @@ +;=============================================================================== +; Copyright (C) 2009 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: x86 Cryptography Primitive. +; +; +; + +%macro my_pclmulqdq 3.nolist + %define %%xxDst %1 + %define %%xxSrc %2 + %define %%xxOp %3 + +%if (my_emulator == 0) + pclmulqdq %%xxDst, %%xxSrc, %%xxOp +%else +;;;; esp => &p1 +;; +4 => &p2 +;; +8 => p3 +;; +12 => p2 (16 bytes) +;; +28 => p1 (16 bytes) +;; +44 => eax + + %assign %%stackSize (4+4+4)+(16+16)+(4+4+4+4+4+4) + sub esp,%%stackSize + + mov dword [esp+44], eax ;; save eax + mov dword [esp+48], ebx ;; save ebx + mov dword [esp+52], ecx ;; save ecx + mov dword [esp+56], edx ;; save edx + mov dword [esp+60], esi ;; save esi + mov dword [esp+64], edi ;; save edi + + movdqu oword [esp+28], %%xxDst ;; store mmDst + movdqu oword [esp+12], %%xxSrc ;; store mmSrc + mov dword [esp+8], %%xxOp ;; put p3 (mmOp) into the stack + lea eax, [esp+28] ;; &p1 (&mmSrc) + mov dword [esp], eax ;; put &p1 into the stack + lea eax, [esp+12] ;; &p2 + mov dword [esp+4], eax ;; put &p2 into the stack + + call emu_pclmulqdq + movdqu %%xxDst, oword [esp+28] ;; load result + + mov eax, dword [esp+44] ;; restore eax + mov ebx, dword [esp+48] ;; restore ebx + mov ecx, dword [esp+52] ;; restore ecx + mov edx, dword [esp+56] ;; restore edx + mov esi, dword [esp+60] ;; restore esi + mov edi, dword [esp+64] ;; restore edi + add esp, %%stackSize +%endif +%endmacro + +%macro my_palignr 3.nolist + %define %%xxDst %1 + %define %%xxSrc %2 + %define %%xxOp %3 + +%if (my_emulator == 0) + palignr %%xxDst, %%xxSrc, %%xxOp +%else +;; +;; esp => &p1 +;; +4 => &p2 +;; +8 => p3 +;; +12 => p2 (16 bytes) +;; +28 => p1 (16 bytes) +;; +44 => eax + + %assign %%stackSize (4+4+4)+(16+16)+(4+4+4+4+4+4) + sub esp,%%stackSize + + mov dword [esp+44], eax ;; save eax + mov dword [esp+48], ebx ;; save ebx + mov dword [esp+52], ecx ;; save ecx + mov dword [esp+56], edx ;; save edx + mov dword [esp+60], esi ;; save esi + mov dword [esp+64], edi ;; save edi + + movdqu oword [esp+28], %%xxDst ;; store mmDst + movdqu oword [esp+12], %%xxSrc ;; store mmSrc + mov dword [esp+8], %%xxOp ;; put p3 (mmOp) into the stack + lea eax, [esp+28] ;; &p1 (&mmSrc) + mov dword [esp], eax ;; put &p1 into the stack + lea eax, [esp+12] ;; &p2 + mov dword [esp+4], eax ;; put &p2 into the stack + + call emu_palignr + movdqu %%xxDst, oword [esp+28] ;; load result + + mov eax, dword [esp+44] ;; restore eax + mov ebx, dword [esp+48] ;; restore ebx + mov ecx, dword [esp+52] ;; restore ecx + mov edx, dword [esp+56] ;; restore edx + mov esi, dword [esp+60] ;; restore esi + mov edi, dword [esp+64] ;; restore edi + add esp, %%stackSize +%endif +%endmacro + +%macro my_pshufd 3.nolist + %define %%xxDst %1 + %define %%xxSrc %2 + %define %%xxOp %3 + +%if (my_emulator == 0) + pshufd %%xxDst, %%xxSrc, %%xxOp +%else +;; +;; esp => &p1 +;; +4 => &p2 +;; +8 => p3 +;; +12 => p2 (16 bytes) +;; +28 => p1 (16 bytes) +;; +44 => eax + + %assign %%stackSize (4+4+4)+(16+16)+(4+4+4+4+4+4) + sub esp,%%stackSize + + mov dword [esp+44], eax ;; save eax + mov dword [esp+48], ebx ;; save ebx + mov dword [esp+52], ecx ;; save ecx + mov dword [esp+56], edx ;; save edx + mov dword [esp+60], esi ;; save esi + mov dword [esp+64], edi ;; save edi + + movdqu oword [esp+28], %%xxDst ;; store mmDst + movdqu oword [esp+12], %%xxSrc ;; store mmSrc + mov dword [esp+8], %%xxOp ;; put p3 (mmOp) into the stack + lea eax, [esp+28] ;; &p1 (&mmSrc) + mov dword [esp], eax ;; put &p1 into the stack + lea eax, [esp+12] ;; &p2 + mov dword [esp+4], eax ;; put &p2 into the stack + + call emu_pshufd + movdqu %%xxDst, oword [esp+28] ;; load result + + mov eax, dword [esp+44] ;; restore eax + mov ebx, dword [esp+48] ;; restore ebx + mov ecx, dword [esp+52] ;; restore ecx + mov edx, dword [esp+56] ;; restore edx + mov esi, dword [esp+60] ;; restore esi + mov edi, dword [esp+64] ;; restore edi + add esp, %%stackSize +%endif +%endmacro + + +%macro my_pshufb 2.nolist + %define %%xxDst %1 + %define %%xxSrc %2 + +%if (my_emulator == 0) + pshufb %%xxDst, %%xxSrc +%else +;; +;; esp => &p1 +;; +4 => &p2 +;; +8 => p2 (16 bytes) +;; +24 => p1 (16 bytes) +;; +40 => eax + + %assign %%stackSize (4+4)+(16+16)+(4+4+4+4+4+4) + sub esp,%%stackSize + + mov dword [esp+40], eax ;; save eax + mov dword [esp+44], ebx ;; save ebx + mov dword [esp+48], ecx ;; save ecx + mov dword [esp+52], edx ;; save edx + mov dword [esp+56], esi ;; save esi + mov dword [esp+60], edi ;; save edi + + movdqu oword [esp+24], %%xxDst ;; store mmDst + movdqu oword [esp+8], %%xxSrc ;; store mmSrc + lea eax, [esp+24] ;; &p1 (&mmSrc) + mov dword [esp], eax ;; put &p1 into the stack + lea eax, [esp+8] ;; &p2 + mov dword [esp+4], eax ;; put &p2 into the stack + + call emu_pshufb + movdqu %%xxDst, oword [esp+24] ;; load result + + mov eax, dword [esp+40] ;; restore eax + mov ebx, dword [esp+44] ;; restore ebx + mov ecx, dword [esp+48] ;; restore ecx + mov edx, dword [esp+52] ;; restore edx + mov esi, dword [esp+56] ;; restore esi + mov edi, dword [esp+60] ;; restore edi + add esp, %%stackSize +%endif +%endmacro + +%macro my_pshufbM 2.nolist + %define %%xxDst %1 + %define %%xxSrc %2 + +%if (my_emulator == 0) + pshufb %%xxDst, %%xxSrc +%else +;; +;; esp => &p1 +;; +4 => &p2 +;; +8 => p2 (16 bytes) +;; +24 => p1 (16 bytes) +;; +40 => eax + + %assign %%stackSize (4+4)+(16+16)+(4+4+4+4+4+4) + sub esp,%%stackSize + + mov dword [esp+40], eax ;; save eax + mov dword [esp+44], ebx ;; save ebx + mov dword [esp+48], ecx ;; save ecx + mov dword [esp+52], edx ;; save edx + mov dword [esp+56], esi ;; save esi + mov dword [esp+60], edi ;; save edi + + movdqu oword [esp+24], %%xxDst ;; store mmDst + movdqu %%xxDst, %%xxSrc + movdqu oword [esp+8], %%xxDst ;; store mmSrc + lea eax, [esp+24] ;; &p1 (&mmSrc) + mov dword [esp], eax ;; put &p1 into the stack + lea eax, [esp+8] ;; &p2 + mov dword [esp+4], eax ;; put &p2 into the stack + + call emu_pshufb + movdqu %%xxDst, oword [esp+24] ;; load result + + mov eax, dword [esp+40] ;; restore eax + mov ebx, dword [esp+44] ;; restore ebx + mov ecx, dword [esp+48] ;; restore ecx + mov edx, dword [esp+52] ;; restore edx + mov esi, dword [esp+56] ;; restore esi + mov edi, dword [esp+60] ;; restore edi + add esp, %%stackSize +%endif +%endmacro + +%macro my_aesenc 2.nolist + %define %%xxDst %1 + %define %%xxSrc %2 + +%if (my_emulator == 0) + aesenc %%xxDst, %%xxSrc +%else +;; +;; esp => &p1 +;; +4 => &p2 +;; +8 => p1 (16 bytes) +;; +24 => p2 (16 bytes) +;; +40 => eax + + %assign %%stackSize (4+4)+(16+16)+(4+4+4+4+4+4) + sub esp,%%stackSize + + mov dword [esp+40], eax ;; save eax + mov dword [esp+44], ebx ;; save ebx + mov dword [esp+48], ecx ;; save ecx + mov dword [esp+52], edx ;; save edx + mov dword [esp+56], esi ;; save esi + mov dword [esp+60], edi ;; save edi + + movdqu oword [esp+8], %%xxDst ;; store mmDst + movdqu oword [esp+24], %%xxSrc ;; store mmSrc + lea eax, [esp+8] ;; &p1 (&mmSrc) + mov dword [esp], eax ;; put &p1 into the stack + lea eax, [esp+24] ;; &p2 + mov dword [esp+4], eax ;; put &p2 into the stack + + call emu_aesenc + movdqu %%xxDst, oword [esp+8] ;; load result + + mov eax, dword [esp+40] ;; restore eax + mov ebx, dword [esp+44] ;; restore ebx + mov ecx, dword [esp+48] ;; restore ecx + mov edx, dword [esp+52] ;; restore edx + mov esi, dword [esp+56] ;; restore esi + mov edi, dword [esp+60] ;; restore edi + add esp, %%stackSize +%endif +%endmacro + +%macro my_aesenclast 2.nolist + %define %%xxDst %1 + %define %%xxSrc %2 + +%if (my_emulator == 0) + aesenclast %%xxDst, %%xxSrc +%else +;; +;; esp => &p1 +;; +4 => &p2 +;; +8 => p1 (16 bytes) +;; +24 => p2 (16 bytes) +;; +40 => eax + + %assign %%stackSize (4+4)+(16+16)+(4+4+4+4+4+4) + sub esp,%%stackSize + + mov dword [esp+40], eax ;; save eax + mov dword [esp+44], ebx ;; save ebx + mov dword [esp+48], ecx ;; save ecx + mov dword [esp+52], edx ;; save edx + mov dword [esp+56], esi ;; save esi + mov dword [esp+60], edi ;; save edi + + movdqu oword [esp+8], %%xxDst ;; store mmDst + movdqu oword [esp+24], %%xxSrc ;; store mmSrc + lea eax, [esp+8] ;; &p1 (&mmSrc) + mov dword [esp], eax ;; put &p1 into the stack + lea eax, [esp+24] ;; &p2 + mov dword [esp+4], eax ;; put &p2 into the stack + + call emu_aesenclast + movdqu %%xxDst, oword [esp+8] ;; load result + + mov eax, dword [esp+40] ;; restore eax + mov ebx, dword [esp+44] ;; restore ebx + mov ecx, dword [esp+48] ;; restore ecx + mov edx, dword [esp+52] ;; restore edx + mov esi, dword [esp+56] ;; restore esi + mov edi, dword [esp+60] ;; restore edi + add esp, %%stackSize +%endif +%endmacro + + +%macro my_aesdec 2.nolist + %define %%xxDst %1 + %define %%xxSrc %2 + +%if (my_emulator == 0) + aesdec %%xxDst, %%xxSrc +%else +;; +;; esp => &p1 +;; +4 => &p2 +;; +8 => p1 (16 bytes) +;; +24 => p2 (16 bytes) +;; +40 => eax + + %assign %%stackSize (4+4)+(16+16)+(4+4+4+4+4+4) + sub esp,%%stackSize + + mov dword [esp+40], eax ;; save eax + mov dword [esp+44], ebx ;; save ebx + mov dword [esp+48], ecx ;; save ecx + mov dword [esp+52], edx ;; save edx + mov dword [esp+56], esi ;; save esi + mov dword [esp+60], edi ;; save edi + + movdqu oword [esp+8], %%xxDst ;; store mmDst + movdqu oword [esp+24], %%xxSrc ;; store mmSrc + lea eax, [esp+8] ;; &p1 (&mmSrc) + mov dword [esp], eax ;; put &p1 into the stack + lea eax, [esp+24] ;; &p2 + mov dword [esp+4], eax ;; put &p2 into the stack + + call emu_aesdec + movdqu %%xxDst, oword [esp+8] ;; load result + + mov eax, dword [esp+40] ;; restore eax + mov ebx, dword [esp+44] ;; restore ebx + mov ecx, dword [esp+48] ;; restore ecx + mov edx, dword [esp+52] ;; restore edx + mov esi, dword [esp+56] ;; restore esi + mov edi, dword [esp+60] ;; restore edi + add esp, %%stackSize +%endif +%endmacro + +%macro my_aesdeclast 2.nolist + %define %%xxDst %1 + %define %%xxSrc %2 + +%if (my_emulator == 0) + aesdeclast %%xxDst, %%xxSrc +%else +;; +;; esp => &p1 +;; +4 => &p2 +;; +8 => p1 (16 bytes) +;; +24 => p2 (16 bytes) +;; +40 => eax + + %assign %%stackSize (4+4)+(16+16)+(4+4+4+4+4+4) + sub esp,%%stackSize + + mov dword [esp+40], eax ;; save eax + mov dword [esp+44], ebx ;; save ebx + mov dword [esp+48], ecx ;; save ecx + mov dword [esp+52], edx ;; save edx + mov dword [esp+56], esi ;; save esi + mov dword [esp+60], edi ;; save edi + + movdqu oword [esp+8], %%xxDst ;; store mmDst + movdqu oword [esp+24], %%xxSrc ;; store mmSrc + lea eax, [esp+8] ;; &p1 (&mmSrc) + mov dword [esp], eax ;; put &p1 into the stack + lea eax, [esp+24] ;; &p2 + mov dword [esp+4], eax ;; put &p2 into the stack + + call emu_aesdeclast + movdqu %%xxDst, oword [esp+8] ;; load result + + mov eax, dword [esp+40] ;; restore eax + mov ebx, dword [esp+44] ;; restore ebx + mov ecx, dword [esp+48] ;; restore ecx + mov edx, dword [esp+52] ;; restore edx + mov esi, dword [esp+56] ;; restore esi + mov edi, dword [esp+60] ;; restore edi + add esp, %%stackSize +%endif +%endmacro + +%if (my_emulator != 0) + extern emu_pclmulqdq + extern emu_palignr + extern emu_pshufd + extern emu_pshufb + extern emu_aesenc + extern emu_aesenclast + extern emu_aesdec + extern emu_aesdeclast +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpaesgcmg9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpaesgcmg9as.asm new file mode 100644 index 0000000..6f19e77 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpaesgcmg9as.asm @@ -0,0 +1,875 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; AES-GCM function +; +; Content: +; AesGcmPrecompute_avx() +; AesGcmMulGcm_avx() +; AesGcmAuth_avx() +; AesGcmEnc_avx() +; AesGcmDec_avx() +; +; +; + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%assign my_emulator 0; set 1 for emulation +%include "emulator.inc" + +;; +;; a = a*b mod g(x), g(x) = x^128 + x^7 + x^2 +x +1 +;; +%macro sse_clmul_gcm 5.nolist + %xdefine %%GH %1 + %xdefine %%HK %2 + %xdefine %%tmpX0 %3 + %xdefine %%tmpX1 %4 + %xdefine %%tmpX2 %5 + + ;; GH, HK hold the values for the two operands which are carry-less multiplied + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; Karatsuba Method + ;; + ;; GH = [GH1:GH0] + ;; HK = [HK1:HK0] + ;; + pshufd %%tmpX2, %%GH, 01001110b ;; xmm2 = {GH0:GH1} + pshufd %%tmpX0, %%HK, 01001110b ;; xmm0 = {HK0:HK1} + pxor %%tmpX2, %%GH ;; xmm2 = {GH0+GH1:GH1+GH0} + pxor %%tmpX0, %%HK ;; xmm0 = {HK0+HK1:HK1+HK0} + +my_pclmulqdq %%tmpX2, %%tmpX0,00h ;; tmpX2 = (a1+a0)*(b1+b0) xmm2 = (GH1+GH0)*(HK1+HK0) + movdqa %%tmpX1, %%GH +my_pclmulqdq %%GH, %%HK, 00h ;; GH = a0*b0 GH = GH0*HK0 + pxor %%tmpX0, %%tmpX0 +my_pclmulqdq %%tmpX1, %%HK, 11h ;; tmpX1 = a1*b1 xmm1 = GH1*HK1 + pxor %%tmpX2, %%GH ;; xmm2 = (GH1+GH0)*(HK1+HK0) + GH0*HK0 + pxor %%tmpX2, %%tmpX1 ;; tmpX2 = a0*b1+a1*b0 xmm2 = (GH1+GH0)*(HK1+HK0) + GH0*HK0 + GH1*HK1 = GH0*HK1+GH1*HK0 + + palignr %%tmpX0, %%tmpX2, 8 ;; tmpX0 = {Zeros : HI(a0*b1+a1*b0)} + pslldq %%tmpX2, 8 ;; tmpX2 = {LO(HI(a0*b1+a1*b0)) : Zeros} + pxor %%tmpX1, %%tmpX0 ;; holds the result of the carry-less multiplication of GH by HK + pxor %%GH, %%tmpX2 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;first phase of the reduction: + ; Most( (product_H * g1), 128)) product_H = GH + ; g1 = 2^256/g = g = 1+x+x^2+x^7+x^128 + ; + movdqa %%tmpX0, %%GH + psllq %%tmpX0, 1 ; GH<<1 + pxor %%tmpX0, %%GH + psllq %%tmpX0, 5 ; ((GH<<1) ^ GH)<<5 + pxor %%tmpX0, %%GH + psllq %%tmpX0, 57 ; (((GH<<1) ^ GH)<<5) ^ GH)<<57 <==> GH<<63 ^ GH<<62 ^ GH<<57 + + movdqa %%tmpX2, %%tmpX0 + pslldq %%tmpX2, 8 ; shift-L tmpX2 2 DWs + psrldq %%tmpX0, 8 ; shift-R xmm2 2 DWs + + pxor %%GH, %%tmpX2 ; first phase of the reduction complete + pxor %%tmpX1, %%tmpX0 ; save the lost MS 1-2-7 bits from first phase + + ;second phase of the reduction + movdqa %%tmpX2, %%GH ; move GH into xmm15 + psrlq %%tmpX2, 5 ; packed right shifting >> 5 + pxor %%tmpX2, %%GH ; xor shifted versions + psrlq %%tmpX2, 1 ; packed right shifting >> 1 + pxor %%tmpX2, %%GH ; xor shifted versions + psrlq %%tmpX2, 1 ; packed right shifting >> 1 + + pxor %%GH, %%tmpX2 ; second phase of the reduction complete + pxor %%GH, %%tmpX1 ; the result is in GH +%endmacro + +%if (_IPP >= _IPP_P8) + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR +CONST_TABLE: +_poly DQ 00000000000000001h,0C200000000000000h ;; 0xC2000000000000000000000000000001 +_twoone DQ 00000000000000001h,00000000100000000h ;; 0x00000001000000000000000000000001 +_u128_str DB 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 +_mask1 DQ 0ffffffffffffffffh,00000000000000000h ;; 0x0000000000000000ffffffffffffffff +_mask2 DQ 00000000000000000h,0ffffffffffffffffh ;; 0xffffffffffffffff0000000000000000 +_inc1 DQ 1,0 + +%xdefine POLY [esi+(_poly - CONST_TABLE)] +%xdefine TWOONE [esi+(_twoone - CONST_TABLE)] +%xdefine u128_str [esi+(_u128_str - CONST_TABLE)] +%xdefine MASK1 [esi+(_mask1 - CONST_TABLE)] +%xdefine MASK2 [esi+(_mask2 - CONST_TABLE)] +%xdefine inc1 [esi+(_inc1 - CONST_TABLE)] + +%assign sizeof_oword_ (16) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; +;;; void GCMpipePrecomute(const Ipp8u* pRefHkey, Ipp8u* pMultipliers); +;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM AesGcmPrecompute_avx,PUBLIC + USES_GPR esi + +%xdefine pHkey [esp + ARG_1 + 0*sizeof(dword)] ; pointer to the reflected hkey +%xdefine pMultipliers [esp + ARG_1 + 1*sizeof(dword)] ; output to the precomputed multipliers + + LD_ADDR esi, CONST_TABLE + + mov eax, pHkey + movdqu xmm0, oword [eax] ; xmm0 holds HashKey + pshufb xmm0, u128_str + + ; precompute HashKey<<1 mod poly from the HashKey + movdqa xmm4, xmm0 + psllq xmm0, 1 + psrlq xmm4, 63 + movdqa xmm3, xmm4 + pslldq xmm4, 8 + psrldq xmm3, 8 + por xmm0, xmm4 + ;reduction + pshufd xmm4, xmm3, 00100100b + pcmpeqd xmm4, oword TWOONE ; TWOONE = 0x00000001000000000000000000000001 + pand xmm4, oword POLY + pxor xmm0, xmm4 ; xmm0 holds the HashKey<<1 mod poly + + movdqa xmm1, xmm0 + sse_clmul_gcm xmm1, xmm0, xmm3, xmm4, xmm5 ; xmm1 holds (HashKey^2)<<1 mod poly + + movdqa xmm2, xmm1 + sse_clmul_gcm xmm2, xmm1, xmm3, xmm4, xmm5 ; xmm2 holds (HashKey^4)<<1 mod poly + + mov eax, pMultipliers + movdqu oword [eax+sizeof_oword_*0], xmm0 + movdqu oword [eax+sizeof_oword_*1], xmm1 + movdqu oword [eax+sizeof_oword_*2], xmm2 + + REST_GPR + ret +ENDFUNC AesGcmPrecompute_avx + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; +;;; void AesGcmMulGcm_avx(Ipp8u* pHash, const Ipp8u* pHKey) +;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM AesGcmMulGcm_avx,PUBLIC + USES_GPR esi,edi + +%xdefine pHash [esp + ARG_1 + 0*sizeof(dword)] +%xdefine pHKey [esp + ARG_1 + 1*sizeof(dword)] + + LD_ADDR esi, CONST_TABLE + + mov edi, pHash ; (edi) pointer to the Hash value + mov eax, pHKey ; (eax) pointer to the (hkey<<1) value + + movdqa xmm0, oword [edi] + pshufb xmm0, u128_str + movdqa xmm1, oword [eax] + + sse_clmul_gcm xmm0, xmm1, xmm2, xmm3, xmm4 ; xmm0 holds Hash*HKey mod poly + + pshufb xmm0, u128_str + movdqa oword [edi], xmm0 + + REST_GPR + ret +ENDFUNC AesGcmMulGcm_avx + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; +;;; void AesGcmAuth_avx(Ipp8u* pHash, const Ipp8u* pSrc, int len, const Ipp8u* pHKey +;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM AesGcmAuth_avx,PUBLIC + USES_GPR esi,edi + +%xdefine pHash [esp + ARG_1 + 0*sizeof(dword)] +%xdefine pSrc [esp + ARG_1 + 1*sizeof(dword)] +%xdefine len [esp + ARG_1 + 2*sizeof(dword)] +%xdefine pHKey [esp + ARG_1 + 3*sizeof(dword)] + +%assign BYTES_PER_BLK (16) + + LD_ADDR esi, CONST_TABLE + + mov edi, pHash + movdqa xmm0, oword [edi] + pshufb xmm0, u128_str + mov eax, pHKey + movdqa xmm1, oword [eax] + + mov ecx, pSrc + mov edx, len + +align IPP_ALIGN_FACTOR +.auth_loop: + movdqu xmm2, oword [ecx] ; src[] + pshufb xmm2, u128_str + add ecx, BYTES_PER_BLK + pxor xmm0, xmm2 ; hash ^= src[] + + sse_clmul_gcm xmm0, xmm1, xmm2, xmm3, xmm4 ; xmm0 holds Hash*HKey mod poly + + sub edx, BYTES_PER_BLK + jnz .auth_loop + + pshufb xmm0, u128_str + movdqa oword [edi], xmm0 + + REST_GPR + ret +ENDFUNC AesGcmAuth_avx + + +;*************************************************************** +;* Purpose: pipelined AES-GCM encryption +;* +;* void AesGcmEnc_avx(Ipp8u* pDst, +;* const Ipp8u* pSrc, +;* int length, +;* RijnCipher cipher, +;* int nr, +;* const Ipp8u* pRKey, +;* Ipp8u* pGhash, +;* Ipp8u* pCtrValue, +;* Ipp8u* pEncCtrValue, +;* const Ipp8u* pPrecomData) +;*************************************************************** + +;; +;; Lib = P8, G9 +;; +;; Caller = ippsRijndael128GCMEncrypt +;; +align IPP_ALIGN_FACTOR +IPPASM AesGcmEnc_avx,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pDst [ebp + ARG_1 + 0*sizeof(dword)] ; output block address +%xdefine pSrc [ebp + ARG_1 + 1*sizeof(dword)] ; input block address +%xdefine len [ebp + ARG_1 + 2*sizeof(dword)] ; length(byte) +%xdefine cipher [ebp + ARG_1 + 3*sizeof(dword)] +%xdefine nr [ebp + ARG_1 + 4*sizeof(dword)] ; number of rounds +%xdefine pKey [ebp + ARG_1 + 5*sizeof(dword)] ; key material address +%xdefine pGhash [ebp + ARG_1 + 6*sizeof(dword)] ; hash +%xdefine pCounter [ebp + ARG_1 + 7*sizeof(dword)] ; counter +%xdefine pEcounter [ebp + ARG_1 + 8*sizeof(dword)] ; enc. counter +%xdefine pPrecomData [ebp + ARG_1 + 9*sizeof(dword)] ; const multipliers + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) +%assign BYTES_PER_BLK (16) +%assign BYTES_PER_LOOP (BYTES_PER_BLK*BLKS_PER_LOOP) + +;; +;; stack structure: +%assign CNT (0) +%assign ECNT (CNT+sizeof_oword_) +%assign GHASH (ECNT+sizeof_oword_) + +%assign GHASH0 (GHASH) +%assign GHASH1 (GHASH0+sizeof_oword_) +%assign GHASH2 (GHASH1+sizeof_oword_) +%assign GHASH3 (GHASH2+sizeof_oword_) + +%assign SHUF_CONST (GHASH3+sizeof_oword_) +%assign INC_1 (SHUF_CONST+sizeof_oword_) + +%assign BLKS4 (INC_1+sizeof_oword_) +%assign BLKS (BLKS4+sizeof(dword)) +%assign STACK_SIZE (BLKS+sizeof(dword)+sizeof_oword_) + + sub esp, STACK_SIZE ; alocate stack + lea ebx, [esp+sizeof_oword_] ; align stack + and ebx, -sizeof_oword_ + mov eax, cipher ; due to bug in ml12 - dummy instruction + LD_ADDR esi, CONST_TABLE + movdqa xmm4, oword u128_str + movdqa xmm5, oword inc1 + + mov eax, pCounter ; address of the counter + mov ecx, pEcounter ; address of the encrypted counter + mov edx, pGhash ; address of hash value + + movdqu xmm0, oword [eax] ; counter value + movdqu xmm1, oword [ecx] ; encrypted counter value + movdqu xmm2, oword [edx] ; hash value + +my_pshufb xmm0, xmm4 ; convert counter and + movdqa oword [ebx+CNT], xmm0 ; and store into the stack + movdqa oword [ebx+ECNT], xmm1 ; store encrypted counter into the stack + +my_pshufb xmm2, xmm4 ; convert hash value + pxor xmm1, xmm1 + movdqa oword [ebx+GHASH0], xmm2 ; store hash into the stack + movdqa oword [ebx+GHASH1], xmm1 ; + movdqa oword [ebx+GHASH2], xmm1 ; + movdqa oword [ebx+GHASH3], xmm1 ; + + movdqa oword [ebx+SHUF_CONST], xmm4 ; store constants into the stack + movdqa oword [ebx+INC_1], xmm5 + + mov ecx, pKey ; key marerial + mov esi, pSrc ; src/dst pointers + mov edi, pDst + + mov eax, len + mov edx, BYTES_PER_LOOP-1 + and edx, eax + and eax,-BYTES_PER_LOOP + mov dword [ebx+BLKS4], eax ; 4-blks counter + mov dword [ebx+BLKS], edx ; rest counter + jz .single_block_proc + +;; +;; pipelined processing +;; +align IPP_ALIGN_FACTOR +.blks4_loop: + ;; + ;; ctr encryption + ;; + movdqa xmm5, oword [ebx+INC_1] + + movdqa xmm1, xmm0 ; counter+1 + paddd xmm1, xmm5 + movdqa xmm2, xmm1 ; counter+2 + paddd xmm2, xmm5 + movdqa xmm3, xmm2 ; counter+3 + paddd xmm3, xmm5 + movdqa xmm4, xmm3 ; counter+4 + paddd xmm4, xmm5 + movdqa oword [ebx+CNT], xmm4 + + movdqa xmm5,oword [ebx+SHUF_CONST] + + movdqa xmm0, oword [ecx] ; pre-load whitening keys + lea eax, [ecx+16] ; pointer to the round's key material + +my_pshufb xmm1, xmm5 ; counter, counter+1, counter+2, counter+3 +my_pshufb xmm2, xmm5 ; ready to be encrypted +my_pshufb xmm3, xmm5 +my_pshufb xmm4, xmm5 + + pxor xmm1, xmm0 ; whitening + pxor xmm2, xmm0 + pxor xmm3, xmm0 + pxor xmm4, xmm0 + + movdqa xmm0, oword [eax] ; pre load round keys + add eax, 16 + + mov edx, nr ; counter depending on key length + sub edx, 1 + +align IPP_ALIGN_FACTOR +.cipher4_loop: +my_aesenc xmm1, xmm0 ; regular round +my_aesenc xmm2, xmm0 +my_aesenc xmm3, xmm0 +my_aesenc xmm4, xmm0 + movdqa xmm0, oword [eax] + add eax, 16 + dec edx + jnz .cipher4_loop +my_aesenclast xmm1, xmm0 +my_aesenclast xmm2, xmm0 +my_aesenclast xmm3, xmm0 +my_aesenclast xmm4, xmm0 + + movdqa xmm0, oword [ebx+ECNT] ; load pre-calculated encrypted counter + movdqa oword [ebx+ECNT], xmm4 ; save encrypted counter+4 + + movdqu xmm4, oword [esi+0*BYTES_PER_BLK] ; ctr encryption of 4 input blocks + movdqu xmm5, oword [esi+1*BYTES_PER_BLK] + movdqu xmm6, oword [esi+2*BYTES_PER_BLK] + movdqu xmm7, oword [esi+3*BYTES_PER_BLK] + add esi, BYTES_PER_LOOP + + pxor xmm0, xmm4 ; ctr encryption + movdqu oword [edi+0*BYTES_PER_BLK], xmm0 ; store result +my_pshufbM xmm0, [ebx+SHUF_CONST] ; convert for multiplication and + pxor xmm0, oword [ebx+GHASH0] + + pxor xmm1, xmm5 + movdqu oword [edi+1*BYTES_PER_BLK], xmm1 +my_pshufbM xmm1, [ebx+SHUF_CONST] + pxor xmm1, oword [ebx+GHASH1] + + pxor xmm2, xmm6 + movdqu oword [edi+2*BYTES_PER_BLK], xmm2 +my_pshufbM xmm2, [ebx+SHUF_CONST] + pxor xmm2, oword [ebx+GHASH2] + + pxor xmm3, xmm7 + movdqu oword [edi+3*BYTES_PER_BLK], xmm3 +my_pshufbM xmm3, [ebx+SHUF_CONST] + pxor xmm3, oword [ebx+GHASH3] + + add edi, BYTES_PER_LOOP + + mov eax, pPrecomData ; pointer to the {hk<<1,hk^2<<1,kh^4<<1} multipliers + movdqa xmm7,oword [eax+sizeof_oword_*2] + + cmp dword [ebx+BLKS4], BYTES_PER_LOOP + je .combine_hash + + ;; + ;; update hash value + ;; + sse_clmul_gcm xmm0, xmm7, xmm4, xmm5, xmm6 ; gHash0 = gHash0 * (HashKey^4)<<1 mod poly + sse_clmul_gcm xmm1, xmm7, xmm4, xmm5, xmm6 ; gHash1 = gHash0 * (HashKey^4)<<1 mod poly + sse_clmul_gcm xmm2, xmm7, xmm4, xmm5, xmm6 ; gHash2 = gHash0 * (HashKey^4)<<1 mod poly + sse_clmul_gcm xmm3, xmm7, xmm4, xmm5, xmm6 ; gHash3 = gHash0 * (HashKey^4)<<1 mod poly + + movdqa oword [ebx+GHASH0], xmm0 + movdqa oword [ebx+GHASH1], xmm1 + movdqa oword [ebx+GHASH2], xmm2 + movdqa oword [ebx+GHASH3], xmm3 + + movdqa xmm0, oword [ebx+CNT] ; next counter value + sub dword [ebx+BLKS4], BYTES_PER_LOOP + jge .blks4_loop + +.combine_hash: + sse_clmul_gcm xmm0, xmm7, xmm4, xmm5, xmm6 ; gHash0 = gHash0 * (HashKey^4)<<1 mod poly + movdqa xmm7,oword [eax+sizeof_oword_*1] + sse_clmul_gcm xmm1, xmm7, xmm4, xmm5, xmm6 ; gHash1 = gHash1 * (HashKey^2)<<1 mod poly + movdqa xmm7,oword [eax+sizeof_oword_*0] + sse_clmul_gcm xmm2, xmm7, xmm4, xmm5, xmm6 ; gHash2 = gHash2 * (HashKey^1)<<1 mod poly + + pxor xmm3, xmm1 + pxor xmm3, xmm2 + sse_clmul_gcm xmm3, xmm7, xmm4, xmm5, xmm6 ; gHash3 = gHash3 * (HashKey)<<1 mod poly + + pxor xmm3, xmm0 + movdqa oword [ebx+GHASH0], xmm3 ; store ghash + +;; +;; rest of input processing (1-3 blocks) +;; +.single_block_proc: + cmp dword [ebx+BLKS],0 + jz .quit + +align IPP_ALIGN_FACTOR +.blk_loop: + movdqa xmm0, oword [ebx+CNT] ; advance counter value + movdqa xmm1, xmm0 + paddd xmm1, oword [ebx+INC_1] + movdqa oword [ebx+CNT], xmm1 + + movdqa xmm0, oword [ecx] ; pre-load whitening keys + lea eax, [ecx+16] + +my_pshufb xmm1, [ebx+SHUF_CONST] ; counter is ready to be encrypted + + pxor xmm1, xmm0 ; whitening + + movdqa xmm0, oword [eax] + add eax, 16 + + mov edx, nr ; counter depending on key length + sub edx, 1 + +align IPP_ALIGN_FACTOR +.cipher_loop: +my_aesenc xmm1, xmm0 ; regular round + movdqa xmm0, oword [eax] + add eax, 16 + dec edx + jnz .cipher_loop +my_aesenclast xmm1, xmm0 + + movdqa xmm0, oword [ebx+ECNT] ; load pre-calculated encrypted counter + movdqa oword [ebx+ECNT], xmm1 ; save encrypted counter + + movdqu xmm1, oword [esi] ; input block + add esi, BYTES_PER_BLK + pxor xmm0, xmm1 ; ctr encryption + movdqu oword [edi], xmm0 + add edi, BYTES_PER_BLK + + mov eax, pPrecomData + + pshufb xmm0, [ebx+SHUF_CONST] + pxor xmm0, oword [ebx+GHASH0] + movdqa xmm1, oword [eax] + sse_clmul_gcm xmm0, xmm1, xmm2, xmm3, xmm4 ; update hash value + movdqa oword [ebx+GHASH0], xmm0 + + sub dword [ebx+BLKS], BYTES_PER_BLK + jg .blk_loop + +;; +;; exit +;; +.quit: + movdqa xmm4, oword [ebx+SHUF_CONST] + + movdqa xmm0, oword [ebx+CNT] ; counter + movdqa xmm1, oword [ebx+ECNT] ; encrypted counter + movdqa xmm2, oword [ebx+GHASH0] ; hash + + mov eax, pCounter ; address of the counter + mov ecx, pEcounter ; address of the encrypted counter + mov edx, pGhash ; address of hash value + +my_pshufb xmm0, xmm4 ; convert counter back and + movdqu oword [eax], xmm0 ; and store + + movdqu oword [ecx], xmm1 ; store encrypted counter into the context + +my_pshufb xmm2, xmm4 ; convert hash value back + movdqu oword [edx], xmm2 ; store hash into the context + + add esp, STACK_SIZE ; free stack + REST_GPR + ret +ENDFUNC AesGcmEnc_avx + + +;*************************************************************** +;* Purpose: pipelined AES-GCM decryption +;* +;* void AesGcmEnc_avx(Ipp8u* pDst, +;* const Ipp8u* pSrc, +;* int length, +;* RijnCipher cipher, +;* int nr, +;* const Ipp8u* pRKey, +;* Ipp8u* pGhash, +;* Ipp8u* pCtrValue, +;* Ipp8u* pEncCtrValue, +;* const Ipp8u* pPrecomData) +;*************************************************************** + +;; +;; Lib = P8, G9 +;; +;; Caller = ippsRijndael128GCMDecrypt +;; +align IPP_ALIGN_FACTOR +IPPASM AesGcmDec_avx,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pDst [ebp + ARG_1 + 0*sizeof(dword)] ; output block address +%xdefine pSrc [ebp + ARG_1 + 1*sizeof(dword)] ; input block address +%xdefine len [ebp + ARG_1 + 2*sizeof(dword)] ; length(byte) +%xdefine cipher [ebp + ARG_1 + 3*sizeof(dword)] +%xdefine nr [ebp + ARG_1 + 4*sizeof(dword)] ; number of rounds +%xdefine pKey [ebp + ARG_1 + 5*sizeof(dword)] ; key material address +%xdefine pGhash [ebp + ARG_1 + 6*sizeof(dword)] ; hash +%xdefine pCounter [ebp + ARG_1 + 7*sizeof(dword)] ; counter +%xdefine pEcounter [ebp + ARG_1 + 8*sizeof(dword)] ; enc. counter +%xdefine pPrecomData [ebp + ARG_1 + 9*sizeof(dword)] ; const multipliers + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) +%assign BYTES_PER_BLK (16) +%assign BYTES_PER_LOOP (BYTES_PER_BLK*BLKS_PER_LOOP) + +;; +;; stack structure: +%assign CNT (0) +%assign ECNT (CNT+sizeof_oword_) +%assign GHASH (ECNT+sizeof_oword_) + +%assign GHASH0 (GHASH) +%assign GHASH1 (GHASH0+sizeof_oword_) +%assign GHASH2 (GHASH1+sizeof_oword_) +%assign GHASH3 (GHASH2+sizeof_oword_) + +%assign SHUF_CONST (GHASH3+sizeof_oword_) +%assign INC_1 (SHUF_CONST+sizeof_oword_) + +%assign BLKS4 (INC_1+sizeof_oword_) +%assign BLKS (BLKS4+sizeof(dword)) +%assign STACK_SIZE (BLKS+sizeof(dword)+sizeof_oword_) + + sub esp, STACK_SIZE ; alocate stack + lea ebx, [esp+sizeof_oword_] ; align stack + and ebx, -sizeof_oword_ + mov eax, cipher ; due to bug in ml12 - dummy instruction + + LD_ADDR esi, CONST_TABLE + movdqa xmm4, oword u128_str + movdqa xmm5, oword inc1 + + mov eax, pCounter ; address of the counter + mov ecx, pEcounter ; address of the encrypted counter + mov edx, pGhash ; address of hash value + + movdqu xmm0, oword [eax] ; counter value + movdqu xmm1, oword [ecx] ; encrypted counter value + movdqu xmm2, oword [edx] ; hash value + +my_pshufb xmm0, xmm4 ; convert counter and + movdqa oword [ebx+CNT], xmm0 ; and store into the stack + movdqa oword [ebx+ECNT], xmm1 ; store encrypted counter into the stack + +my_pshufb xmm2, xmm4 ; convert hash value + pxor xmm1, xmm1 + movdqa oword [ebx+GHASH0], xmm2 ; store hash into the stack + movdqa oword [ebx+GHASH1], xmm1 ; + movdqa oword [ebx+GHASH2], xmm1 ; + movdqa oword [ebx+GHASH3], xmm1 ; + + movdqa oword [ebx+SHUF_CONST], xmm4 ; store constants into the stack + movdqa oword [ebx+INC_1], xmm5 + + mov ecx, pKey ; key marerial + mov esi, pSrc ; src/dst pointers + mov edi, pDst + + mov eax, len + mov edx, BYTES_PER_LOOP-1 + and edx, eax + and eax,-BYTES_PER_LOOP + mov dword [ebx+BLKS4], eax ; 4-blks counter + mov dword [ebx+BLKS], edx ; rest counter + jz .single_block_proc + +;; +;; pipelined processing +;; +align IPP_ALIGN_FACTOR +.blks4_loop: + ;; + ;; ctr encryption + ;; + movdqa xmm5, oword [ebx+INC_1] + + movdqa xmm1, xmm0 ; counter+1 + paddd xmm1, xmm5 + movdqa xmm2, xmm1 ; counter+2 + paddd xmm2, xmm5 + movdqa xmm3, xmm2 ; counter+3 + paddd xmm3, xmm5 + movdqa xmm4, xmm3 ; counter+4 + paddd xmm4, xmm5 + movdqa oword [ebx+CNT], xmm4 + + movdqa xmm5,oword [ebx+SHUF_CONST] + + movdqa xmm0, oword [ecx] ; pre-load whitening keys + lea eax, [ecx+16] ; pointer to the round's key material + +my_pshufb xmm1, xmm5 ; counter, counter+1, counter+2, counter+3 +my_pshufb xmm2, xmm5 ; ready to be encrypted +my_pshufb xmm3, xmm5 +my_pshufb xmm4, xmm5 + + pxor xmm1, xmm0 ; whitening + pxor xmm2, xmm0 + pxor xmm3, xmm0 + pxor xmm4, xmm0 + + movdqa xmm0, oword [eax] ; pre load round keys + add eax, 16 + + mov edx, nr ; counter depending on key length + sub edx, 1 + +align IPP_ALIGN_FACTOR +.cipher4_loop: +my_aesenc xmm1, xmm0 ; regular round +my_aesenc xmm2, xmm0 +my_aesenc xmm3, xmm0 +my_aesenc xmm4, xmm0 + movdqa xmm0, oword [eax] + add eax, 16 + dec edx + jnz .cipher4_loop +my_aesenclast xmm1, xmm0 +my_aesenclast xmm2, xmm0 +my_aesenclast xmm3, xmm0 +my_aesenclast xmm4, xmm0 + + movdqa xmm0, oword [ebx+ECNT] ; load pre-calculated encrypted counter + movdqa oword [ebx+ECNT], xmm4 ; save encrypted counter+4 + + movdqu xmm4, oword [esi+0*BYTES_PER_BLK] ; ctr encryption of 4 input blocks + movdqu xmm5, oword [esi+1*BYTES_PER_BLK] + movdqu xmm6, oword [esi+2*BYTES_PER_BLK] + movdqu xmm7, oword [esi+3*BYTES_PER_BLK] + add esi, BYTES_PER_LOOP + + pxor xmm0, xmm4 ; ctr encryption + movdqu oword [edi+0*BYTES_PER_BLK], xmm0 ; store result +my_pshufbM xmm4, [ebx+SHUF_CONST] ; convert for multiplication and + pxor xmm4, oword [ebx+GHASH0] + + pxor xmm1, xmm5 + movdqu oword [edi+1*BYTES_PER_BLK], xmm1 +my_pshufbM xmm5, [ebx+SHUF_CONST] + pxor xmm5, oword [ebx+GHASH1] + + pxor xmm2, xmm6 + movdqu oword [edi+2*BYTES_PER_BLK], xmm2 + ;pshufb xmm6, [ebx+SHUF_CONST] +my_pshufbM xmm6, [ebx+SHUF_CONST] + pxor xmm6, oword [ebx+GHASH2] + + pxor xmm3, xmm7 + movdqu oword [edi+3*BYTES_PER_BLK], xmm3 +my_pshufbM xmm7, [ebx+SHUF_CONST] + pxor xmm7, oword [ebx+GHASH3] + + add edi, BYTES_PER_LOOP + + mov eax, pPrecomData ; pointer to the const multipliers (c^1, c^2, c^4) + movdqa xmm0,oword [eax+sizeof_oword_*2] + + cmp dword [ebx+BLKS4], BYTES_PER_LOOP + je .combine_hash + + ;; + ;; update hash value + ;; + sse_clmul_gcm xmm4, xmm0, xmm1, xmm2, xmm3 ; gHash0 = gHash0 * (HashKey^4)<<1 mod poly + sse_clmul_gcm xmm5, xmm0, xmm1, xmm2, xmm3 ; gHash1 = gHash0 * (HashKey^4)<<1 mod poly + sse_clmul_gcm xmm6, xmm0, xmm1, xmm2, xmm3 ; gHash2 = gHash0 * (HashKey^4)<<1 mod poly + sse_clmul_gcm xmm7, xmm0, xmm1, xmm2, xmm3 ; gHash3 = gHash0 * (HashKey^4)<<1 mod poly + + movdqa oword [ebx+GHASH0], xmm4 + movdqa oword [ebx+GHASH1], xmm5 + movdqa oword [ebx+GHASH2], xmm6 + movdqa oword [ebx+GHASH3], xmm7 + + movdqa xmm0, oword [ebx+CNT] ; next counter value + sub dword [ebx+BLKS4], BYTES_PER_LOOP + jge .blks4_loop + +.combine_hash: + sse_clmul_gcm xmm4, xmm0, xmm1, xmm2, xmm3 ; gHash0 = gHash0 * (HashKey^4)<<1 mod poly + movdqa xmm0, oword [eax+sizeof_oword_*1] + sse_clmul_gcm xmm5, xmm0, xmm1, xmm2, xmm3 ; gHash1 = gHash1 * (HashKey^2)<<1 mod poly + movdqa xmm0, oword [eax+sizeof_oword_*0] + sse_clmul_gcm xmm6, xmm0, xmm1, xmm2, xmm3 ; gHash2 = gHash2 * (HashKey^1)<<1 mod poly + + pxor xmm7, xmm5 + pxor xmm7, xmm6 + sse_clmul_gcm xmm7, xmm0, xmm1, xmm2, xmm3 ; gHash3 = gHash3 * (HashKey)<<1 mod poly + + pxor xmm7, xmm4 + movdqa oword [ebx+GHASH0], xmm7 ; store ghash + +;; +;; rest of input processing (1-3 blocks) +;; +.single_block_proc: + cmp dword [ebx+BLKS],0 + jz .quit + +align IPP_ALIGN_FACTOR +.blk_loop: + movdqa xmm0, oword [ebx+CNT] ; advance counter value + movdqa xmm1, xmm0 + paddd xmm1, oword [ebx+INC_1] + movdqa oword [ebx+CNT], xmm1 + + movdqa xmm0, oword [ecx] ; pre-load whitening keys + lea eax, [ecx+16] + +my_pshufb xmm1, [ebx+SHUF_CONST] ; counter is ready to be encrypted + + pxor xmm1, xmm0 ; whitening + + movdqa xmm0, oword [eax] + add eax, 16 + + mov edx, nr ; counter depending on key length + sub edx, 1 + +align IPP_ALIGN_FACTOR +.cipher_loop: +my_aesenc xmm1, xmm0 ; regular round + movdqa xmm0, oword [eax] + add eax, 16 + dec edx + jnz .cipher_loop +my_aesenclast xmm1, xmm0 + + movdqa xmm0, oword [ebx+ECNT] ; load pre-calculated encrypted counter + movdqa oword [ebx+ECNT], xmm1 ; save encrypted counter + + movdqu xmm1, oword [esi] ; input block + add esi, BYTES_PER_BLK + pxor xmm0, xmm1 ; ctr encryption + movdqu oword [edi], xmm0 + add edi, BYTES_PER_BLK + + mov eax, pPrecomData + + pshufb xmm1, [ebx+SHUF_CONST] + pxor xmm1, oword [ebx+GHASH0] + movdqa xmm0, oword [eax] + sse_clmul_gcm xmm1, xmm0, xmm2, xmm3, xmm4 ; update hash value + movdqa oword [ebx+GHASH0], xmm1 + + sub dword [ebx+BLKS], BYTES_PER_BLK + jg .blk_loop + +;; +;; exit +;; +.quit: + movdqa xmm4, oword [ebx+SHUF_CONST] + + movdqa xmm0, oword [ebx+CNT] ; counter + movdqa xmm1, oword [ebx+ECNT] ; encrypted counter + movdqa xmm2, oword [ebx+GHASH0] ; hash + + mov eax, pCounter ; address of the counter + mov ecx, pEcounter ; address of the encrypted counter + mov edx, pGhash ; address of hash value + +my_pshufb xmm0, xmm4 ; convert counter back and + movdqu oword [eax], xmm0 ; and store + + movdqu oword [ecx], xmm1 ; store encrypted counter into the context + +my_pshufb xmm2, xmm4 ; convert hash value back + movdqu oword [edx], xmm2 ; store hash into the context + + add esp, STACK_SIZE ; free stack + REST_GPR + ret +ENDFUNC AesGcmDec_avx + +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpaesgcmtable2kv8as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpaesgcmtable2kv8as.asm new file mode 100644 index 0000000..9deb3ec --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpaesgcmtable2kv8as.asm @@ -0,0 +1,449 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Encrypt/Decrypt byte data stream according to Rijndael128 (GCM mode) +; +; Content: +; AesGcmMulGcm_table2K() +; AesGcmAuth_table2K() +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%if (_IPP >= _IPP_V8) + +segment .text align=IPP_ALIGN_FACTOR + +; +; getAesGcmConst_table_ct provides c-e-t access to pre-computed Ipp16u AesGcmConst_table[256] +; +; input: +; edx: address of the AesGcmConst_table +; ecx: index in the table +; +; output: +; eax +; +; register ecx destoyed +; registers mmx2, mmx3, mmx6, and mmx7 destoyed +; +align IPP_ALIGN_FACTOR +_CONST_DATA: +_INIT_IDX DW 000h,001h,002h,003h,004h,005h,006h,007h ;; initial search inx = {0:1:2:3:4:5:6:7} +_INCR_IDX DW 008h,008h,008h,008h,008h,008h,008h,008h ;; index increment = {8:8:8:8:8:8:8:8} + +%xdefine INIT_IDX [ebx+(_INIT_IDX - _CONST_DATA)] +%xdefine INCR_IDX [ebx+(_INCR_IDX - _CONST_DATA)] + +align IPP_ALIGN_FACTOR +IPPASM getAesGcmConst_table_ct,PRIVATE + push ebx + LD_ADDR ebx, _CONST_DATA + + pxor xmm2, xmm2 ;; accumulator xmm2 = 0 + + mov eax, ecx ;; broadcast inx into dword + shl ecx, 16 + or ecx, eax + movd xmm3, ecx + pshufd xmm3, xmm3, 00b ;; search index xmm3 = broadcast(idx) + + movdqa xmm6, xmmword INIT_IDX ;; current indexes + + xor eax, eax +align IPP_ALIGN_FACTOR +.search_loop: + movdqa xmm7, xmm6 ;; copy current indexes + paddw xmm6, xmmword INCR_IDX ;; advance current indexes + + pcmpeqw xmm7, xmm3 ;; selection mask + pand xmm7, xmmword [edx+eax*sizeof(word)];; mask data + + add eax, 8 + cmp eax, 256 + + por xmm2, xmm7 ;; and accumulate + jl .search_loop + + movdqa xmm3, xmm2 ;; pack result in qword + psrldq xmm2, sizeof(xmmword)/2 + por xmm2, xmm3 + movdqa xmm3, xmm2 ;; pack result in dword + psrldq xmm2, sizeof(xmmword)/4 + por xmm2, xmm3 + movd eax, xmm2 + + pop ebx + + and ecx, 3 ;; select tbl[idx] value + shl ecx, 4 ;; rcx *=16 = sizeof(word)*8 + shr eax, cl + ret +ENDFUNC getAesGcmConst_table_ct + + +; +; void AesGcmMulGcm_table2K(Ipp8u* pHash, const Ipp8u* pPrecomputedData, , const void* pParam)) +; +align IPP_ALIGN_FACTOR +IPPASM AesGcmMulGcm_table2K,PUBLIC + USES_GPR esi,edi,ebx + +%xdefine pHash [esp + ARG_1 + 0*sizeof(dword)] +%xdefine pMulTbl [esp + ARG_1 + 1*sizeof(dword)] +%xdefine pParam [esp + ARG_1 + 2*sizeof(dword)] + + mov edi, pHash + movdqu xmm0, [edi] ; hash value + + mov esi, pMulTbl + mov edx, pParam ; pointer to the fixed table + + movd ebx, xmm0 ; ebx = hash.0 + mov eax, 0f0f0f0f0h + and eax, ebx ; eax = 4 x 4_bits + shl ebx, 4 + and ebx, 0f0f0f0f0h ; ebx = 4 x 4_bits (another) + movzx ecx, ah + movdqa xmm5, oword [esi+1024+ecx] + movzx ecx, al + movdqa xmm4, oword [esi+1024+ecx] + shr eax, 16 + movzx ecx, ah + movdqa xmm3, oword [esi+1024+ecx] + movzx ecx, al + movdqa xmm2, oword [esi+1024+ecx] + + psrldq xmm0, 4 ; shift xmm0 + movd eax, xmm0 ; eax = hash[1] + and eax, 0f0f0f0f0h ; eax = 4 x 4_bits + + movzx ecx, bh + pxor xmm5, oword [esi+ 0*256 + ecx] + movzx ecx, bl + pxor xmm4, oword [esi+ 0*256 + ecx] + shr ebx, 16 + movzx ecx, bh + pxor xmm3, oword [esi+ 0*256 + ecx] + movzx ecx, bl + pxor xmm2, oword [esi+ 0*256 + ecx] + + movd ebx, xmm0 ; ebx = hash[1] + shl ebx, 4 ; another 4 x 4_bits + and ebx, 0f0f0f0f0h + + movzx ecx, ah + pxor xmm5, oword [esi+1024+ 1*256 + ecx] + movzx ecx, al + pxor xmm4, oword [esi+1024+ 1*256 + ecx] + shr eax, 16 + movzx ecx, ah + pxor xmm3, oword [esi+1024+ 1*256 + ecx] + movzx ecx, al + pxor xmm2, oword [esi+1024+ 1*256 + ecx] + psrldq xmm0, 4 + + movd eax, xmm0 ; eax = hash.2 + and eax, 0f0f0f0f0h + + movzx ecx, bh + pxor xmm5, oword [esi+ 1*256 + ecx] + movzx ecx, bl + pxor xmm4, oword [esi+ 1*256 + ecx] + shr ebx, 16 + movzx ecx, bh + pxor xmm3, oword [esi+ 1*256 + ecx] + movzx ecx, bl + pxor xmm2, oword [esi+ 1*256 + ecx] + + movd ebx, xmm0 + shl ebx, 4 + and ebx, 0f0f0f0f0h + + movzx ecx, ah + pxor xmm5, oword [esi+1024+ 2*256 + ecx] + movzx ecx, al + pxor xmm4, oword [esi+1024+ 2*256 + ecx] + shr eax, 16 + movzx ecx, ah + pxor xmm3, oword [esi+1024+ 2*256 + ecx] + movzx ecx, al + pxor xmm2, oword [esi+1024+ 2*256 + ecx] + + psrldq xmm0, 4 + movd eax, xmm0 ; eax = hash.3 + and eax, 0f0f0f0f0h + + movzx ecx, bh + pxor xmm5, oword [esi+ 2*256 + ecx] + movzx ecx, bl + pxor xmm4, oword [esi+ 2*256 + ecx] + shr ebx, 16 + movzx ecx, bh + pxor xmm3, oword [esi+ 2*256 + ecx] + movzx ecx, bl + pxor xmm2, oword [esi+ 2*256 + ecx] + + movd ebx, xmm0 + shl ebx, 4 + and ebx, 0f0f0f0f0h + + movzx ecx, ah + pxor xmm5, oword [esi+1024+ 3*256 + ecx] + movzx ecx, al + pxor xmm4, oword [esi+1024+ 3*256 + ecx] + shr eax, 16 + movzx ecx, ah + pxor xmm3, oword [esi+1024+ 3*256 + ecx] + movzx ecx, al + pxor xmm2, oword [esi+1024+ 3*256 + ecx] + + movzx ecx, bh + pxor xmm5, oword [esi+ 3*256 + ecx] + movzx ecx, bl + pxor xmm4, oword [esi+ 3*256 + ecx] + shr ebx, 16 + movzx ecx, bh + pxor xmm3, oword [esi+ 3*256 + ecx] + movzx ecx, bl + pxor xmm2, oword [esi+ 3*256 + ecx] + + movdqa xmm0, xmm3 + pslldq xmm3, 1 + pxor xmm2, xmm3 + movdqa xmm1, xmm2 + pslldq xmm2, 1 + pxor xmm5, xmm2 + + psrldq xmm0, 15 + movd ecx, xmm0 + CALL_IPPASM getAesGcmConst_table_ct ;;movzx eax, word [edx + ecx*sizeof(word)] + shl eax, 8 + + movdqa xmm0, xmm5 + pslldq xmm5, 1 + pxor xmm4, xmm5 + + psrldq xmm1, 15 + movd ecx, xmm1 + mov ebx, eax ;;xor ax, word [edx + ecx*sizeof(word)] + CALL_IPPASM getAesGcmConst_table_ct ;; + xor eax, ebx ;; + shl eax, 8 + + psrldq xmm0, 15 + movd ecx, xmm0 + mov ebx, eax ;;xor ax, word [edx + ecx*sizeof(word)] + CALL_IPPASM getAesGcmConst_table_ct ;; + xor eax, ebx ;; + + movd xmm0, eax + pxor xmm0, xmm4 + + movdqu oword [edi], xmm0 ; store hash value + + REST_GPR + ret +ENDFUNC AesGcmMulGcm_table2K + +; +; void AesGcmAuth_table2K(Ipp8u* pHash, const Ipp8u* pSrc, int len, const Ipp8u* pPrecomputedData, const void* pParam) +; +align IPP_ALIGN_FACTOR +IPPASM AesGcmAuth_table2K,PUBLIC + USES_GPR esi,edi,ebx + +%xdefine pHash [esp + ARG_1 + 0*sizeof(dword)] +%xdefine pSrc [esp + ARG_1 + 1*sizeof(dword)] +%xdefine len [esp + ARG_1 + 2*sizeof(dword)] +%xdefine pMulTbl [esp + ARG_1 + 3*sizeof(dword)] +%xdefine pParam [esp + ARG_1 + 4*sizeof(dword)] + + mov edi, pHash + movdqu xmm0, [edi] ; hash value + + mov esi, pMulTbl + mov edi, pSrc + + mov edx, pParam ; pointer to the fixed table + +align IPP_ALIGN_FACTOR +.auth_loop: + movdqu xmm4, [edi] ; get src[] + pxor xmm0, xmm4 ; hash ^= src[] + + movd ebx, xmm0 ; ebx = hash.0 + mov eax, 0f0f0f0f0h + and eax, ebx ; eax = 4 x 4_bits + shl ebx, 4 + and ebx, 0f0f0f0f0h ; ebx = 4 x 4_bits (another) + movzx ecx, ah + movdqa xmm5, oword [esi+1024+ecx] + movzx ecx, al + movdqa xmm4, oword [esi+1024+ecx] + shr eax, 16 + movzx ecx, ah + movdqa xmm3, oword [esi+1024+ecx] + movzx ecx, al + movdqa xmm2, oword [esi+1024+ecx] + + psrldq xmm0, 4 ; shift xmm0 + movd eax, xmm0 ; eax = hash[1] + and eax, 0f0f0f0f0h ; eax = 4 x 4_bits + + movzx ecx, bh + pxor xmm5, oword [esi+ 0*256 + ecx] + movzx ecx, bl + pxor xmm4, oword [esi+ 0*256 + ecx] + shr ebx, 16 + movzx ecx, bh + pxor xmm3, oword [esi+ 0*256 + ecx] + movzx ecx, bl + pxor xmm2, oword [esi+ 0*256 + ecx] + + movd ebx, xmm0 ; ebx = hash[1] + shl ebx, 4 ; another 4 x 4_bits + and ebx, 0f0f0f0f0h + + movzx ecx, ah + pxor xmm5, oword [esi+1024+ 1*256 + ecx] + movzx ecx, al + pxor xmm4, oword [esi+1024+ 1*256 + ecx] + shr eax, 16 + movzx ecx, ah + pxor xmm3, oword [esi+1024+ 1*256 + ecx] + movzx ecx, al + pxor xmm2, oword [esi+1024+ 1*256 + ecx] + + psrldq xmm0, 4 + movd eax, xmm0 ; eax = hash[2] + and eax, 0f0f0f0f0h + + movzx ecx, bh + pxor xmm5, oword [esi+ 1*256 + ecx] + movzx ecx, bl + pxor xmm4, oword [esi+ 1*256 + ecx] + shr ebx, 16 + movzx ecx, bh + pxor xmm3, oword [esi+ 1*256 + ecx] + movzx ecx, bl + pxor xmm2, oword [esi+ 1*256 + ecx] + + movd ebx, xmm0 + shl ebx, 4 + and ebx, 0f0f0f0f0h + + movzx ecx, ah + pxor xmm5, oword [esi+1024+ 2*256 + ecx] + movzx ecx, al + pxor xmm4, oword [esi+1024+ 2*256 + ecx] + shr eax, 16 + movzx ecx, ah + pxor xmm3, oword [esi+1024+ 2*256 + ecx] + movzx ecx, al + pxor xmm2, oword [esi+1024+ 2*256 + ecx] + + psrldq xmm0, 4 + movd eax, xmm0 ; eax = hash[3] + and eax, 0f0f0f0f0h + + movzx ecx, bh + pxor xmm5, oword [esi+ 2*256 + ecx] + movzx ecx, bl + pxor xmm4, oword [esi+ 2*256 + ecx] + shr ebx, 16 + movzx ecx, bh + pxor xmm3, oword [esi+ 2*256 + ecx] + movzx ecx, bl + pxor xmm2, oword [esi+ 2*256 + ecx] + + movd ebx, xmm0 + shl ebx, 4 + and ebx, 0f0f0f0f0h + + movzx ecx, ah + pxor xmm5, oword [esi+1024+ 3*256 + ecx] + movzx ecx, al + pxor xmm4, oword [esi+1024+ 3*256 + ecx] + shr eax, 16 + movzx ecx, ah + pxor xmm3, oword [esi+1024+ 3*256 + ecx] + movzx ecx, al + pxor xmm2, oword [esi+1024+ 3*256 + ecx] + + movzx ecx, bh + pxor xmm5, oword [esi+ 3*256 + ecx] + movzx ecx, bl + pxor xmm4, oword [esi+ 3*256 + ecx] + shr ebx, 16 + movzx ecx, bh + pxor xmm3, oword [esi+ 3*256 + ecx] + movzx ecx, bl + pxor xmm2, oword [esi+ 3*256 + ecx] + + movdqa xmm0, xmm3 + pslldq xmm3, 1 + pxor xmm2, xmm3 + movdqa xmm1, xmm2 + pslldq xmm2, 1 + pxor xmm5, xmm2 + psrldq xmm0, 15 + + movd ecx, xmm0 + CALL_IPPASM getAesGcmConst_table_ct ;;movzx eax, word [edx + ecx*sizeof(word)] + shl eax, 8 + + movdqa xmm0, xmm5 + pslldq xmm5, 1 + pxor xmm4, xmm5 + + psrldq xmm1, 15 + movd ecx, xmm1 + mov ebx, eax ;;xor ax, word [edx + ecx*sizeof(word)] + CALL_IPPASM getAesGcmConst_table_ct ;; + xor eax, ebx ;; + shl eax, 8 + + psrldq xmm0, 15 + movd ecx, xmm0 + mov ebx, eax ;;xor ax, word [edx + ecx*sizeof(word)] + CALL_IPPASM getAesGcmConst_table_ct ;; + xor eax, ebx ;; + + movd xmm0, eax + pxor xmm0, xmm4 + + add edi, sizeof(oword) ; advance src address + sub dword len, sizeof(oword) ; decrease counter + jnz .auth_loop ; process next block + + mov edi, pHash + movdqu oword [edi], xmm0 ; store hash value + + REST_GPR + ret +ENDFUNC AesGcmAuth_table2K + +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpbnu.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpbnu.inc new file mode 100644 index 0000000..87f72ad --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpbnu.inc @@ -0,0 +1,377 @@ +;=============================================================================== +; Copyright (C) 2006 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Big Number macros +; +; Content: +; COPY_BNU +; CMP_BNU +; FIX_BNU +; +; SUB_FIX_BNU +; ADD_FIX_BNU +; + +;; +;; CMP_BNU comare arbitrary BNUs +;; +;; input +;; rSrc1 points BNU1 +;; rSrc2 points BNU2 +;; rLen size of BNUs +;; +;; output +;; flags +;; +%macro CMP_BNU 4.nolist + %xdefine %%rSrc1 %1 + %xdefine %%rSrc2 %2 + %xdefine %%rLen %3 + %xdefine %%tmp %4 + +%%cmp_loop: + mov %%tmp,[%%rSrc1+%%rLen*4-4] + cmp %%tmp,[%%rSrc2+%%rLen*4-4] + jne %%cmp_quit + sub %%rLen,1 + jg %%cmp_loop +%%cmp_quit: +%endmacro + + +;; +;; SUB_FIX_BNU subtract fixed size BNUs +;; +;; input +;; rVal points src/dst BNU +;; rSrc points source BNU +;; immLen BNU size +;; +%macro SUB_FIX_BNU 3.nolist + %xdefine %%rVal %1 + %xdefine %%rSrc %2 + %xdefine %%immLen %3 + + pxor mm0,mm0 + %assign %%i 0 + %rep %%immLen + movd mm1,DWORD [%%rVal+%%i*4] + movd mm2,DWORD [%%rSrc+%%i*4] + %if %%i != 0 + paddq mm1,mm0 + %endif + psubq mm1,mm2 + movd DWORD [%%rVal+%%i*4],mm1 + %if %%i < (%%immLen-1) + pshufw mm0,mm1,11111110b + %endif + %assign %%i %%i+1 + %endrep +%endmacro + + +;; +;; ADD_FIX_BNU add fixed size BNUs +;; +;; input +;; rVal points src/dst BNU +;; rSrc points source BNU +;; immLen BNU size +;; +%macro ADD_FIX_BNU 3.nolist + %xdefine %%rVal %1 + %xdefine %%rSrc %2 + %xdefine %%immLen %3 + + pxor mm0,mm0 + %assign %%i 0 + %rep %%immLen + movd mm1,DWORD [%%rVal+%%i*4] + movd mm2,DWORD [%%rSrc+%%i*4] + %if %%i != 0 + paddq mm1,mm0 + %endif + paddq mm1,mm2 + movd DWORD [%%rVal+%%i*4],mm1 + %if %%i < (%%immLen-1) + pshufw mm0,mm1,11111110b + %endif + %assign %%i %%i+1 + %endrep +%endmacro + + +;; +;; COPY_BNU copy arbitrary BNU +;; +%macro COPY_BNU 5.nolist + %xdefine %%rSrc %1 + %xdefine %%rDst %2 + %xdefine %%rLen %3 + %xdefine %%rIdx %4 + %xdefine %%rTmp %5 + + xor %%rIdx,%%rIdx +%%copy_bnu: + mov %%rTmp,[%%rSrc+%%rIdx*4] + mov [%%rDst+%%rIdx*4],%%rTmp + add %%rIdx,1 + cmp %%rIdx,%%rLen + jl %%copy_bnu +%endmacro + + +;; +;; FIX_BNU returns actual length of BNU +;; +;; input +;; rSrc points BNU +;; rLen initial BNU size +;; +;; output +;; rLen actual BNU size +;; +%macro FIX_BNU 3.nolist + %xdefine %%rSrc %1 + %xdefine %%rLen %2 + %xdefine %%tmp %3 + +%%fix_bnu_loop: + mov %%tmp,[%%rSrc+%%rLen*4-4] ;; value + test %%tmp,%%tmp ;; test BNU component + jnz %%fix_bnu_quit + sub %%rLen,1 + jg %%fix_bnu_loop + add %%rLen,1 +%%fix_bnu_quit: +%endmacro + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro MULADD_START 2.nolist + %xdefine %%i %1 + %xdefine %%j %2 + + movd mm1,DWORD [eax + 4*%%j] + movd mm3,DWORD [eax + 4*%%j] + pmuludq mm1,mm0 + paddq mm7,mm1 + movd DWORD [edx + 4*(%%i+%%j)],mm7 + pand mm3,mm6 + psrlq mm7,32 + paddq mm7,mm3 +%endmacro + + +%macro MULADD 2.nolist + %xdefine %%i %1 + %xdefine %%j %2 + + movd mm1,DWORD [eax + 4*%%j] + movd mm3,DWORD [eax + 4*%%j] + movd mm2,DWORD [edx + 4*(%%i+%%j)] + pmuludq mm1,mm0 + pand mm3,mm6 + paddq mm1,mm2 + paddq mm7,mm1 + movd DWORD [edx + 4*(%%i+%%j)],mm7 + psrlq mm7,32 + paddq mm7,mm3 +%endmacro + + +%macro SQR_DECOMPOSE 1.nolist + %xdefine %%i %1 + + movd mm7,DWORD [eax + 4*%%i] + movd mm0,DWORD [eax + 4*%%i] + movd mm6,DWORD [eax + 4*%%i] + %if %%i != 0 + movd mm1,DWORD [edx + 4*(2*%%i)] + %endif + pslld mm0,1 + pmuludq mm7,mm7 + psrad mm6,32 + %if %%i != 0 + paddq mm7,mm1 + %endif + movd DWORD [edx + 4*(2*%%i)],mm7 + psrlq mm7,32 +%endmacro + + +%macro STORE_CARRY 2.nolist + %xdefine %%i %1 + %xdefine %%s %2 + + movq DWORD [edx + 4*(%%i + %%s)],mm7 +%endmacro + + +%macro STORE_CARRY_NEXT 2.nolist + %xdefine %%i %1 + %xdefine %%s %2 + + movd mm4,DWORD [edx + 4*(%%i + %%s)] + paddq mm4,mm7 + movd DWORD [edx + 4*(%%i + %%s)],mm4 + psrlq mm7,32 + movd DWORD [edx + 4*(%%i + %%s + 1)],mm7 +%endmacro + + +%macro LAST_STEP 1.nolist + %xdefine %%s %1 + + movd mm7,DWORD [eax + 4*(%%s - 1)] + movd mm2,DWORD [edx + 4*(2*%%s - 2)] + pmuludq mm7,mm7 + paddq mm7,mm2 + movd mm4,DWORD [edx + 4*(2*%%s - 1)] + movd DWORD [edx + 4*(2*%%s - 2)],mm7 + psrlq mm7,32 + paddq mm4,mm7 + movd DWORD [edx + 4*(2*%%s - 1)],mm4 +%endmacro + + +%macro INNER_LOOP 2.nolist + %xdefine %%i %1 + %xdefine %%nsize %2 + + %assign %%j %%i + 1 + %assign %%s %%nsize - %%i - 1 + + SQR_DECOMPOSE %%i + + %rep %%s + %if %%i == 0 + MULADD_START %%i,%%j + el%%se + MULADD %%i,%%j + %endif + %assign %%j %%j + 1 + %endrep + + %if %%i == 0 + STORE_CARRY %%i,%%nsize + el%%se + STORE_CARRY_NEXT %%i,%%nsize + %endif +%endmacro + + +%macro OUTER_LOOP 1.nolist + %xdefine %%nsize %1 + + %assign %%i 0 + %rep ns%%ize - 1 + INNER_LOOP %%i,%%nsize + %assign %%i %%i + 1 + %endrep + + LAST_STEP %%nsize +%endmacro + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro MULADD_START_wt_carry 1.nolist + %xdefine %%i %1 + + movd mm7,DWORD [eax] + movd mm2,DWORD [edx + 4*(%%i)] + pmuludq mm7,mm0 + paddq mm7,mm2 + movd DWORD [edx + 4*(%%i)],mm7 + psrlq mm7,32 +%endmacro + + +%macro MULADD_START1 2.nolist + %xdefine %%i %1 + %xdefine %%j %2 + + movd mm1,DWORD [eax + 4*%%j] + pmuludq mm1,mm0 + paddq mm7,mm1 + movd DWORD [edx + 4*(%%i+%%j)],mm7 + psrlq mm7,32 +%endmacro + + +%macro MULADD_START_wt_carry1 0.nolist + movd mm7,DWORD [eax] + pmuludq mm7,mm0 + movd DWORD [edx],mm7 + psrlq mm7,32 +%endmacro + + +%macro MULADD1 2.nolist + %xdefine %%i %1 + %xdefine %%j %2 + + movd mm1,DWORD [eax + 4*%%j] + movd mm2,DWORD [edx + 4*(%%i+%%j)] + pmuludq mm1,mm0 + paddq mm1,mm2 + paddq mm7,mm1 + movd DWORD [edx + 4*(%%i+%%j)],mm7 + psrlq mm7,32 +%endmacro + + +%macro INNER_LOOP1 2.nolist + %xdefine %%i %1 + %xdefine %%nsize %2 + + %assign %%j 0 + movd mm0,DWORD [ebx + 4*%%i] + + %rep %%nsize +%if %%i == 0 + %if %%j == 0 + MULADD_START_wt_carry1 + %else + MULADD_START1 %%i,%%j + %endif +%else + %if %%j == 0 + MULADD_START_wt_carry %%i + %else + MULADD1 %%i,%%j + %endif +%endif + %assign %%j %%j + 1 + %endrep + movd DWORD [edx + 4*(%%i + %%nsize)],mm7 +%endmacro + + +%macro OUTER_LOOP1 1.nolist + %xdefine %%nsize %1 + + %assign %%i 0 + %rep %%nsize + INNER_LOOP1 %%i,%%nsize + %assign %%i %%i + 1 + %endrep +%endmacro + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpbnuaddw7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpbnuaddw7as.asm new file mode 100644 index 0000000..51542ca --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpbnuaddw7as.asm @@ -0,0 +1,80 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; Purpose: Cryptography Primitive. +; Big Number Arithmetic +; +; Content: +; cpAddMul_BNU() +; + + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%if (_IPP >= _IPP_W7) +%include "pcpvariant.inc" + +segment .text align=IPP_ALIGN_FACTOR + +%if (_USE_C_cpAdd_BNU_ == 0) +align IPP_ALIGN_FACTOR + +IPPASM cpAdd_BNU,PUBLIC + USES_GPR esi,edi,ebx + +%xdefine pDst [esp + ARG_1 + 0*sizeof(dword)] ; target address +%xdefine pSrc1 [esp + ARG_1 + 1*sizeof(dword)] ; source address +%xdefine pSrc2 [esp + ARG_1 + 2*sizeof(dword)] ; source address +%xdefine len [esp + ARG_1 + 3*sizeof(dword)] ; length of BNU + + mov eax,pSrc1 ; src1 + mov ebx,pSrc2 ; src2 + mov edx,pDst ; dst + mov edi,len ; length + shl edi,2 + + xor ecx,ecx + pandn mm0,mm0 + +align IPP_ALIGN_FACTOR +.main_loop: + movd mm1,DWORD [eax + ecx] + movd mm2,DWORD [ebx + ecx] + + paddq mm0,mm1 + paddq mm0,mm2 + movd DWORD [edx + ecx],mm0 + pshufw mm0,mm0,11111110b + + add ecx,4 + cmp ecx,edi + jl .main_loop + + movd eax,mm0 + emms + REST_GPR + ret +ENDFUNC cpAdd_BNU +%endif + +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpbnuincw7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpbnuincw7as.asm new file mode 100644 index 0000000..4603e00 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpbnuincw7as.asm @@ -0,0 +1,77 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; Purpose: Cryptography Primitive. +; +; Content: +; cpInc_BNU() +; + + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%if (_IPP >= _IPP_W7) +%include "pcpvariant.inc" + +%if (_USE_C_cpInc_BNU_ == 0) + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR +IPPASM cpInc_BNU,PUBLIC + USES_GPR esi,edi,ebx + +%xdefine pDst [esp + ARG_1 + 0*sizeof(dword)] ; target address +%xdefine pSrc [esp + ARG_1 + 1*sizeof(dword)] ; source address +%xdefine len [esp + ARG_1 + 2*sizeof(dword)] ; length of BNU +%xdefine value [esp + ARG_1 + 3*sizeof(dword)] ; increment val + + mov edi, pDst ; dst + mov esi, pSrc ; src + mov eax, value ; value + movd mm0, value + mov edx,len ; length + shl edx,2 + + xor ecx,ecx + +align IPP_ALIGN_FACTOR +.main_loop: + movd mm1,DWORD [esi + ecx] + paddq mm1,mm0 + movd DWORD [edi + ecx],mm1 + pshufw mm0,mm1,11111110b + movd eax, mm0 + + add ecx,4 + cmp ecx,edx + jl .main_loop + +.exit_loop: + emms + REST_GPR + ret +ENDFUNC cpInc_BNU +%endif + +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpbnumuldgtw7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpbnumuldgtw7as.asm new file mode 100644 index 0000000..d340abf --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpbnumuldgtw7as.asm @@ -0,0 +1,82 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; Purpose: Cryptography Primitive. +; Big Number Arithmetic +; +; Content: +; cpMulDgt_BNU() +; cpAddMulDgt_BNU() +; cpSubMulDgtBNU() +; + + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%if (_IPP >= _IPP_W7) +%include "pcpvariant.inc" + +segment .text align=IPP_ALIGN_FACTOR + +%if (_USE_C_cpAddMulDgt_BNU_ == 0) +;; +;; Ipp32u cpAddMulDgt_BNU(Ipp32u* pDst, const Ipp32u* pSrc, cpSize len, Ipp32u dgt) +;; +IPPASM cpAddMulDgt_BNU,PUBLIC + USES_GPR edi + +%xdefine pDst [esp + ARG_1 + 0*sizeof(dword)] ; target address +%xdefine pSrc [esp + ARG_1 + 1*sizeof(dword)] ; source address +%xdefine len [esp + ARG_1 + 2*sizeof(dword)] ; BNU length +%xdefine dgt [esp + ARG_1 + 3*sizeof(dword)] ; 32-bit multiplier + + mov eax,pSrc ; src + mov edx,pDst ; dst + mov edi,len ; length + + xor ecx,ecx + shl edi,2 + movd mm0,dgt + pandn mm7,mm7 ;c + +.main_loop: + movd mm1,DWORD [eax + ecx] + movd mm2,DWORD [edx + ecx] + pmuludq mm1,mm0 + paddq mm7,mm1 + paddq mm7,mm2 + movd DWORD [edx + ecx],mm7 + psrlq mm7,32 + add ecx,4 + cmp ecx,edi + jl .main_loop + + movd eax,mm7 + + emms + REST_GPR + ret +ENDFUNC cpAddMulDgt_BNU +%endif + +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpbnumulschoolv8as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpbnumulschoolv8as.asm new file mode 100644 index 0000000..29e6797 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpbnumulschoolv8as.asm @@ -0,0 +1,1217 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Big Number Arithmetic +; +; Content: +; cpMulAdc_BNU_school() +; +; +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%if (_IPP >= _IPP_V8) +%include "pcpvariant.inc" + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; +;; +;; Short-Length Squaring +;; +%macro INIT 3.nolist + %xdefine %%r %1 + %xdefine %%a %2 + %xdefine %%n %3 + + movd xmm0,DWORD [%%a] ;; a[0] + pxor xmm7,xmm7 ;; clear carry + movd DWORD [%%r],xmm7 ;; r[0] = 0 + + %assign %%i 1 + movd xmm1,DWORD [%%a+%%i*4] ;; a[i] + pmuludq xmm1,xmm0 ;; t0 = a[i]*a[0] + movd xmm2,DWORD [%%a+%%i*4+4] ;; a[i+1] + pmuludq xmm2,xmm0 ;; t1 = a[i+1]*a[0] + + %rep (%%n-1)/2 + paddq xmm7,xmm1 ;; t0 += carry + %if %%n-(%%i+2) > 0 + movd xmm1,DWORD [%%a+(%%i+2)*4] ;; read in advance a[i+2] + pmuludq xmm1,xmm0 ;; multiply in advance a[i+2]*a[0] + %endif + movd DWORD [%%r+%%i*4],xmm7 ;; r[i] = LO(t0) + psrlq xmm7,32 ;; carry = HI(t0) + + paddq xmm7,xmm2 ;; t1 += carry + %if %%n-(%%i+3) > 0 + movd xmm2,DWORD [%%a+(%%i+2)*4+4] ;; read in advance a[i+3] + pmuludq xmm2,xmm0 ;; multiply in advance a[i+3]*a[0] + %endif + movd DWORD [%%r+%%i*4+4],xmm7 ;; r[i+1] = LO(t1) + psrlq xmm7,32 ;; carry = HI(t1) + + %assign %%i %%i+2 + %endrep + + %if (((%%n-1) & 1) != 0) + paddq xmm7,xmm1 ;; t0 += carry + movd DWORD [%%r+%%i*4],xmm7 ;; r[i] = LO(t0) + psrlq xmm7,32 ;; carry = HI(t0) + %endif + + movd DWORD [%%r+%%n*4],xmm7 ;; r[n] = extension +%endmacro + + +%macro UPDATE_MUL 3.nolist + %xdefine %%r %1 + %xdefine %%a %2 + %xdefine %%n %3 + + %assign %%i 1 + %rep (%%n-2) + movd xmm0,DWORD [%%a+%%i*4] ;; a[i] + pxor xmm7,xmm7 ;; clear carry + + %assign %%j %%i+1 + movd xmm1,DWORD [%%a+%%j*4] ;; a[j] + pmuludq xmm1,xmm0 ;; t0 = a[j]*a[i] + movd xmm3,DWORD [%%r+(%%i+%%j)*4] ;; r[i+j] + %if (%%j+1) < %%n + movd xmm2,DWORD [%%a+%%j*4+4] ;; a[j+1] + pmuludq xmm2,xmm0 ;; t1 = a[j+1]*a[i] + movd xmm4,DWORD [%%r+(%%i+%%j)*4+4];; r[i+j+1] + %endif + + %assign %%tn %%n-%%j + %rep %%tn/2 + paddq xmm7,xmm1 ;; t0 += carry+r[i+j] + %if %%n-(%%j+2) > 0 + movd xmm1,DWORD [%%a+(%%j+2)*4] ;; read in advance a[j+2] + pmuludq xmm1,xmm0 ;; multiply in advance a[j+2]*a[i] + %endif + paddq xmm7,xmm3 + %if %%n-(%%j+2) > 0 + movd xmm3,DWORD [%%r+(%%i+2+%%j)*4];; read in advance r[i+j+2] + %endif + movd DWORD [%%r+(%%i+%%j)*4],xmm7 ;; r[i+j] = LO(t0) + psrlq xmm7,32 ;; carry = HI(t0) + + paddq xmm7,xmm2 ;; t1 += carry+r[i+j] + %if %%n-(%%j+3) > 0 + movd xmm2,DWORD [%%a+(%%j+2)*4+4];; read in advance a[j+3] + pmuludq xmm2,xmm0 ;; multiply in advance a[j+3]*a[i] + %endif + paddq xmm7,xmm4 + %if %%n-(%%j+3) > 0 + movd xmm4,DWORD [%%r+(%%i+2+%%j)*4+4];; read in advance r[i+j+3] + %endif + movd DWORD [%%r+(%%i+%%j)*4+4],xmm7 ;; r[i+j+1] = LO(t1) + psrlq xmm7,32 ;; carry = HI(t1) + + %assign %%j %%j+2 + %endrep + + %if ((%%tn & 1) != 0) + paddq xmm7,xmm1 + paddq xmm7,xmm3 + movd DWORD [%%r+(%%i+%%j)*4],xmm7 + psrlq xmm7,32 ;; carry = HI(t1) + %endif + + movd DWORD [%%r+(%%n+%%i)*4],xmm7 ;; r[i+j] = extension + + %assign %%i %%i+1 + %endrep + + pandn xmm7,xmm7 ;; clear carry + movd DWORD [%%r+(2*%%n-1)*4],xmm7 ;; r[2*n-1] = zero extension +%endmacro + + + +%macro FINALIZE 3.nolist + %xdefine %%r %1 + %xdefine %%a %2 + %xdefine %%n %3 + + %assign %%i 0 + movd xmm0,DWORD [%%a+%%i*4] ;; a[i] + pmuludq xmm0,xmm0 ;; p = a[i]*a[i] + + movd xmm2,DWORD [%%r+(2*%%i)*4] ;; r[2*i] + paddq xmm2,xmm2 + movd xmm3,DWORD [%%r+(2*%%i+1)*4] ;; r[2*i+1] + paddq xmm3,xmm3 + + pcmpeqd xmm6,xmm6 ;; mm6 = low 32 bit mask + psrlq xmm6,32 + + pandn xmm7,xmm7 ;; clear carry + + %rep %%n + movq xmm1,xmm0 + pand xmm0,xmm6 ;; mm0 = LO(p) + psrlq xmm1,32 ;; mm1 = HI(p) + + paddq xmm7,xmm0 ;; q = carry + 2*r[2*i] + LO(p) + %if %%n-(%%i+1) > 0 + movd xmm0,DWORD [%%a+(%%i+1)*4] ;; a[i] + pmuludq xmm0,xmm0 ;; p = a[i]*a[i] + %endif + paddq xmm7,xmm2 + %if %%n-(%%i+1) > 0 + movd xmm2,DWORD [%%r+(2*%%i+2)*4] ;; r[2*(i+1)] + paddq xmm2,xmm2 + %endif + movd DWORD [%%r+(2*%%i)*4],xmm7 ;; r[2*i] = LO(q) + psrlq xmm7,32 ;; carry = HI(q) + + paddq xmm7,xmm1 ;; q = carry + 2*r[2*i+1] + HI(p) + paddq xmm7,xmm3 + %if %%n-(%%i+1) > 0 + movd xmm3,DWORD [%%r+(2*%%i+2)*4+4];; r[2*(i+1)+1] + paddq xmm3,xmm3 + %endif + movd DWORD [%%r+(2*%%i+1)*4],xmm7 ;; r[2*i+1] = LO(q) + psrlq xmm7,32 ;; carry = HI(q) + + %assign %%i %%i+1 + %endrep +%endmacro + + +%macro SQR_SHORT 3 + %xdefine %%r %1 + %xdefine %%a %2 + %xdefine %%n %3 + + INIT %%r,%%a,%%n + UPDATE_MUL %%r,%%a,%%n + FINALIZE %%r,%%a,%%n +%endmacro + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; +;; Middle-Length Squaring +;; +%macro SQR_DECOMPOSE 1.nolist + %xdefine %%i %1 + + movd xmm7,DWORD [eax + 4*%%i] + movd xmm0,DWORD [eax + 4*%%i] + movd xmm6,DWORD [eax + 4*%%i] + + %if %%i != 0 + movd xmm1,DWORD [edx + 4*(2*%%i)] + %endif + + psllq xmm0,1 + pmuludq xmm7,xmm7 + psrad xmm6,32 + + %if %%i != 0 + paddq xmm7,xmm1 + %endif + + movd DWORD [edx + 4*(2*%%i)],xmm7 + psrlq xmm7,32 +%endmacro + + +%macro MULADD_START 2.nolist + %xdefine %%i %1 + %xdefine %%j %2 + + movd xmm1,DWORD [eax + 4*%%j] + movd xmm3,DWORD [eax + 4*%%j] + pmuludq xmm1,xmm0 + paddq xmm7,xmm1 + movd DWORD [edx + 4*(%%i+%%j)],xmm7 + pand xmm3,xmm6 + psrlq xmm7,32 + paddq xmm7,xmm3 +%endmacro + + +%macro MULADD 2.nolist + %xdefine %%i %1 + %xdefine %%j %2 + + movd xmm1,DWORD [eax + 4*%%j] + movd xmm3,DWORD [eax + 4*%%j] + movd xmm2,DWORD [edx + 4*(%%i+%%j)] + pmuludq xmm1,xmm0 + pand xmm3,xmm6 + paddq xmm1,xmm2 + paddq xmm7,xmm1 + movd DWORD [edx + 4*(%%i+%%j)],xmm7 + psrlq xmm7,32 + paddq xmm7,xmm3 +%endmacro + + +%macro STORE_CARRY 2.nolist + %xdefine %%i %1 + %xdefine %%nsize %2 + + movq QWORD [edx + 4*(%%i + %%nsize)],xmm7 +%endmacro + + +%macro STORE_CARRY_NEXT 2.nolist + %xdefine %%i %1 + %xdefine %%nsize %2 + + movd xmm4,DWORD [edx + 4*(%%i + %%nsize)] + paddq xmm4,xmm7 + movd DWORD [edx + 4*(%%i + %%nsize)],xmm4 + psrlq xmm7,32 + movd DWORD [edx + 4*(%%i + %%nsize + 1)],xmm7 +%endmacro + + +%macro INNER_LOOP 2.nolist + %xdefine %%i %1 + %xdefine %%nsize %2 + + %assign %%j %%i + 1 + %assign %%s %%nsize - %%i - 1 + + SQR_DECOMPOSE %%i + + %rep %%s + %if %%i == 0 + MULADD_START %%i,%%j + el%%se + MULADD %%i,%%j + %endif + + %assign %%j %%j + 1 + %endrep + + %if %%i == 0 + STORE_CARRY %%i,%%nsize + el%%se + STORE_CARRY_NEXT %%i,%%nsize + %endif +%endmacro + + +%macro LAST_STEP 1.nolist + %xdefine %%nsize %1 + + movd xmm7,DWORD [eax + 4*(%%nsize - 1)] + pmuludq xmm7,xmm7 + movd xmm2,DWORD [edx + 4*(2*%%nsize - 2)] + paddq xmm7,xmm2 + movd xmm4,DWORD [edx + 4*(2*%%nsize - 1)] + movd DWORD [edx + 4*(2*%%nsize - 2)],xmm7 + psrlq xmm7,32 + paddq xmm7,xmm4 + movd DWORD [edx + 4*(2*%%nsize - 1)],xmm7 +%endmacro + + +%macro SQR_MIDDL 1.nolist + %xdefine %%nsize %1 + + %assign %%i 0 + %rep %%nsize - 1 + INNER_LOOP %%i,%%nsize + %assign %%i %%i + 1 + %endrep + LAST_STEP %%nsize +%endmacro + + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%macro UNROLL16_MULADD 0.nolist + movd xmm1,DWORD [eax + ecx] + movd xmm2,DWORD [edx + ecx] + movd xmm3,DWORD [eax + ecx + 4] + movd xmm4,DWORD [edx + ecx + 4] + movd xmm5,DWORD [eax + ecx + 8] + movd xmm6,DWORD [edx + ecx + 8] + + pmuludq xmm1,xmm0 + paddq xmm1,xmm2 + pmuludq xmm3,xmm0 + paddq xmm3,xmm4 + pmuludq xmm5,xmm0 + paddq xmm5,xmm6 + + paddq xmm7,xmm1 + movd xmm1,DWORD [eax + ecx + 12] + movd xmm2,DWORD [edx + ecx + 12] + movd DWORD [edx + ecx],xmm7 + psrlq xmm7,32 + + pmuludq xmm1,xmm0 + paddq xmm1,xmm2 + + paddq xmm7,xmm3 + movd xmm3,DWORD [eax + ecx + 16] + movd xmm4,DWORD [edx + ecx + 16] + movd DWORD [edx + ecx + 4],xmm7 + psrlq xmm7,32 + + pmuludq xmm3,xmm0 + paddq xmm3,xmm4 + + paddq xmm7,xmm5 + movd xmm5,DWORD [eax + ecx + 20] + movd xmm6,DWORD [edx + ecx + 20] + movd DWORD [edx + ecx + 8],xmm7 + psrlq xmm7,32 + + pmuludq xmm5,xmm0 + paddq xmm5,xmm6 + + paddq xmm7,xmm1 + movd xmm1,DWORD [eax + ecx + 24] + movd xmm2,DWORD [edx + ecx + 24] + movd DWORD [edx + ecx + 12],xmm7 + psrlq xmm7,32 + + pmuludq xmm1,xmm0 + paddq xmm1,xmm2 + + paddq xmm7,xmm3 + movd xmm3,DWORD [eax + ecx + 28] + movd xmm4,DWORD [edx + ecx + 28] + movd DWORD [edx + ecx + 16],xmm7 + psrlq xmm7,32 + + pmuludq xmm3,xmm0 + paddq xmm3,xmm4 + + paddq xmm7,xmm5 + movd xmm5,DWORD [eax + ecx + 32] + movd xmm6,DWORD [edx + ecx + 32] + movd DWORD [edx + ecx + 20],xmm7 + psrlq xmm7,32 + + pmuludq xmm5,xmm0 + paddq xmm5,xmm6 + + paddq xmm7,xmm1 + movd xmm1,DWORD [eax + ecx + 36] + movd xmm2,DWORD [edx + ecx + 36] + movd DWORD [edx + ecx + 24],xmm7 + psrlq xmm7,32 + + pmuludq xmm1,xmm0 + paddq xmm1,xmm2 + + paddq xmm7,xmm3 + movd xmm3,DWORD [eax + ecx + 40] + movd xmm4,DWORD [edx + ecx + 40] + movd DWORD [edx + ecx + 28],xmm7 + psrlq xmm7,32 + + pmuludq xmm3,xmm0 + paddq xmm3,xmm4 + + paddq xmm7,xmm5 + movd xmm5,DWORD [eax + ecx + 44] + movd xmm6,DWORD [edx + ecx + 44] + movd DWORD [edx + ecx + 32],xmm7 + psrlq xmm7,32 + + pmuludq xmm5,xmm0 + paddq xmm5,xmm6 + + paddq xmm7,xmm1 + movd xmm1,DWORD [eax + ecx + 48] + movd xmm2,DWORD [edx + ecx + 48] + movd DWORD [edx + ecx + 36],xmm7 + psrlq xmm7,32 + + pmuludq xmm1,xmm0 + paddq xmm1,xmm2 + + paddq xmm7,xmm3 + movd xmm3,DWORD [eax + ecx + 52] + movd xmm4,DWORD [edx + ecx + 52] + movd DWORD [edx + ecx + 40],xmm7 + psrlq xmm7,32 + + pmuludq xmm3,xmm0 + paddq xmm3,xmm4 + + paddq xmm7,xmm5 + movd xmm5,DWORD [eax + ecx + 56] + movd xmm6,DWORD [edx + ecx + 56] + movd DWORD [edx + ecx + 44],xmm7 + psrlq xmm7,32 + + pmuludq xmm5,xmm0 + paddq xmm5,xmm6 + + paddq xmm7,xmm1 + movd xmm1,DWORD [eax + ecx + 60] + movd xmm2,DWORD [edx + ecx + 60] + movd DWORD [edx + ecx + 48],xmm7 + psrlq xmm7,32 + + pmuludq xmm1,xmm0 + paddq xmm1,xmm2 + + paddq xmm7,xmm3 + movd DWORD [edx + ecx + 52],xmm7 + psrlq xmm7,32 + + paddq xmm7,xmm5 + movd DWORD [edx + ecx + 56],xmm7 + psrlq xmm7,32 + + paddq xmm7,xmm1 + movd DWORD [edx + ecx + 60],xmm7 + psrlq xmm7,32 +%endmacro + + +%macro UNROLL8_MULADD 0.nolist + movd xmm1,DWORD [eax + ecx] + movd xmm2,DWORD [edx + ecx] + movd xmm3,DWORD [eax + ecx + 4] + movd xmm4,DWORD [edx + ecx + 4] + movd xmm5,DWORD [eax + ecx + 8] + movd xmm6,DWORD [edx + ecx + 8] + + pmuludq xmm1,xmm0 + paddq xmm1,xmm2 + pmuludq xmm3,xmm0 + paddq xmm3,xmm4 + pmuludq xmm5,xmm0 + paddq xmm5,xmm6 + + paddq xmm7,xmm1 + movd xmm1,DWORD [eax + ecx + 12] + movd xmm2,DWORD [edx + ecx + 12] + movd DWORD [edx + ecx],xmm7 + psrlq xmm7,32 + + pmuludq xmm1,xmm0 + paddq xmm1,xmm2 + + paddq xmm7,xmm3 + movd xmm3,DWORD [eax + ecx + 16] + movd xmm4,DWORD [edx + ecx + 16] + movd DWORD [edx + ecx + 4],xmm7 + psrlq xmm7,32 + + pmuludq xmm3,xmm0 + paddq xmm3,xmm4 + + paddq xmm7,xmm5 + movd xmm5,DWORD [eax + ecx + 20] + movd xmm6,DWORD [edx + ecx + 20] + movd DWORD [edx + ecx + 8],xmm7 + psrlq xmm7,32 + + pmuludq xmm5,xmm0 + paddq xmm5,xmm6 + + paddq xmm7,xmm1 + movd xmm1,DWORD [eax + ecx + 24] + movd xmm2,DWORD [edx + ecx + 24] + movd DWORD [edx + ecx + 12],xmm7 + psrlq xmm7,32 + + pmuludq xmm1,xmm0 + paddq xmm1,xmm2 + + paddq xmm7,xmm3 + movd xmm3,DWORD [eax + ecx + 28] + movd xmm4,DWORD [edx + ecx + 28] + movd DWORD [edx + ecx + 16],xmm7 + psrlq xmm7,32 + + pmuludq xmm3,xmm0 + paddq xmm3,xmm4 + + paddq xmm7,xmm5 + movd DWORD [edx + ecx + 20],xmm7 + psrlq xmm7,32 + + paddq xmm7,xmm1 + movd DWORD [edx + ecx + 24],xmm7 + psrlq xmm7,32 + + paddq xmm7,xmm3 + movd DWORD [edx + ecx + 28],xmm7 + psrlq xmm7,32 +%endmacro + + + +%macro MUL_MAC0_START 0.nolist + movd xmm7,DWORD [eax] + pmuludq xmm7,xmm0 + movd DWORD [edx],xmm7 + psrlq xmm7,32 +%endmacro + + +%macro MUL_MAC0_START2 0.nolist + movd xmm1,DWORD [eax] + pmuludq xmm1,xmm0 + movd DWORD [edx],xmm1 + psrlq xmm1,32 + + movd xmm7,DWORD [eax+4] + pmuludq xmm7,xmm0 + paddq xmm7,xmm1 + movd DWORD [edx+4],xmm7 + psrlq xmm7,32 +%endmacro + + +%macro MUL_MAC0 1.nolist + %xdefine %%j %1 + + movd xmm1,DWORD [eax + 4*%%j] + pmuludq xmm1,xmm0 + paddq xmm1,xmm7 + movd DWORD [edx + 4*%%j],xmm1 + psrlq xmm1,32 + + movd xmm7,DWORD [eax + 4*(%%j+1)] + pmuludq xmm7,xmm0 + paddq xmm7,xmm1 + movd DWORD [edx + 4*(%%j+1)],xmm7 + psrlq xmm7,32 +%endmacro + + +%macro MUL_MAC_START 1.nolist + %xdefine %%i %1 + + movd xmm7,DWORD [eax] + pmuludq xmm7,xmm0 + movd xmm2,DWORD [edx + 4*(%%i)] + paddq xmm7,xmm2 + movd DWORD [edx + 4*(%%i)],xmm7 + psrlq xmm7,32 +%endmacro + + +%macro MUL_MAC_START2 1.nolist + %xdefine %%i %1 + + movd xmm1,DWORD [eax] + pmuludq xmm1,xmm0 + movd xmm2,DWORD [edx + 4*%%i] + paddq xmm1,xmm2 + movd DWORD [edx + 4*%%i],xmm1 + psrlq xmm1,32 + + movd xmm7,DWORD [eax+4] + pmuludq xmm7,xmm0 + movd xmm2,DWORD [edx + 4*(%%i+1)] + paddq xmm7,xmm2 + paddq xmm7,xmm1 + movd DWORD [edx + 4*(%%i+1)],xmm7 + psrlq xmm7,32 +%endmacro + + +%macro MUL_MAC 2.nolist + %xdefine %%i %1 + %xdefine %%j %2 + + movd xmm1,DWORD [eax + 4*%%j] + pmuludq xmm1,xmm0 + movd xmm2,DWORD [edx + 4*(%%i+%%j)] + paddq xmm1,xmm2 + paddq xmm1,xmm7 + movd DWORD [edx + 4*(%%i+%%j)],xmm1 + psrlq xmm1,32 + + movd xmm7,DWORD [eax + 4*(%%j+1)] + pmuludq xmm7,xmm0 + movd xmm2,DWORD [edx + 4*(%%i+%%j+1)] + paddq xmm7,xmm2 + paddq xmm7,xmm1 + movd DWORD [edx + 4*(%%i+%%j+1)],xmm7 + psrlq xmm7,32 +%endmacro + + +%macro INNER_LOOP1 2.nolist + %xdefine %%i %1 + %xdefine %%nsize %2 + + %assign %%j 0 + movd xmm0,DWORD [ebx + 4*%%i] + + %if %%i == 0 + %if %%nsize & 1 + MUL_MAC0_START + %assign %%j %%j + 1 + %else + MUL_MAC0_START2 + %assign %%j %%j + 2 + %endif + %else + %if %%nsize & 1 + MUL_MAC_START %%i + %assign %%j %%j + 1 + %else + MUL_MAC_START2 %%i + %assign %%j %%j + 2 + %endif + %endif + + %rep ((%%nsize-%%j)/2) + %if %%i == 0 + MUL_MAC0 %%j + %else + MUL_MAC %%i,%%j + %endif + %assign %%j %%j + 2 + %endrep + movd DWORD [edx + 4*(%%i + %%nsize)],xmm7 +%endmacro + + +%macro OUTER_LOOP1 1.nolist + %xdefine %%nsize %1 + + %assign %%i 0 + %rep %%nsize + INNER_LOOP1 %%i,%%nsize + %assign %%i %%i+1 + %endrep +%endmacro + + + +segment .text align=IPP_ALIGN_FACTOR + +%if (_USE_C_cpMulAdc_BNU_school_ == 0) +;************************************************************* +;* +;* void cpMulAdc_BNU_school(Ipp32u* pR, +;* const Ipp32u* pA, int aSize, +;* const Ipp32u* pB, int bSize) +;* +;************************************************************* + +;; +;; Lib = V8 +;; +;; Caller = ippsMontMul +;; Caller = ippsMontExp +;; Caller = ippsModInv_BN +;; +;; Caller = ippsECCPGetPoint +;; Caller = ippsECCPCheckPoint +;; Caller = ippsECCPAddPoint +;; Caller = ippsECCPMulPointScalar +;; Caller = ippsECCPGenKeyPair +;; Caller = ippsECCPPublicKey +;; Caller = ippsECCPValidateKey +;; Caller = ippsECCPShareSecretKeyDH +;; Caller = ippsECCPShareSecretKeyDHC +;; Caller = ippsECCPSignDSA +;; Caller = ippsECCPSignNR +;; Caller = ippsECCPVerifyDSA +;; Caller = ippsECCPVerifyNR +;; +IPPASM cpMulAdc_BNU_school,PUBLIC + USES_GPR esi,edi,ebx + +%xdefine pR [esp + ARG_1 + 0*sizeof(dword)] ; target address +%xdefine pA [esp + ARG_1 + 1*sizeof(dword)] ; source address +%xdefine aSize [esp + ARG_1 + 2*sizeof(dword)] ; BNU length +%xdefine pB [esp + ARG_1 + 3*sizeof(dword)] ; source address +%xdefine bSize [esp + ARG_1 + 4*sizeof(dword)] ; BNU length + + mov eax,pA ; pA + mov edi,aSize ; aSzie_len + mov ebx,pB ; pB + mov esi,bSize ; bSize + mov edx,pR ; pR + + cmp eax,ebx + jne .multiplication + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; squaring +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + cmp edi,1 + jg .sqr2 +.sqr1: + movd xmm0,DWORD [eax] + pmuludq xmm0,xmm0 + movq QWORD [edx],xmm0 + jmp .finish + +.sqr2: + cmp edi,2 + jg .sqr3 + movd xmm0,DWORD [eax] + movd xmm1,DWORD [eax+4] + + movq xmm2,xmm0 + pmuludq xmm0,xmm0 ; a[0]*a[0] + pmuludq xmm2,xmm1 ; a[0]*a[1] + pmuludq xmm1,xmm1 ; a[1]*a[1] + + pcmpeqd xmm7,xmm7 ; mm7 = low 32 bit mask + psrlq xmm7,32 + + pand xmm7,xmm2 ; mm7 = LO(a[0]*a[1]) + paddq xmm7,xmm7 + psrlq xmm2,32 ; mm2 = HI(a[0]*a[1]) + paddq xmm2,xmm2 + + movd DWORD [edx],xmm0 + psrlq xmm0,32 + + paddq xmm0,xmm7 + movd DWORD [edx+4],xmm0 + psrlq xmm0,32 + + paddq xmm2,xmm1 + paddq xmm2,xmm0 + movq QWORD [edx+8],xmm2 + jmp .finish + +.sqr3: + cmp edi,3 + jg .sqr4 + SQR_SHORT edx,eax,3 + jmp .finish + +.sqr4: + cmp edi,4 + jg .sqr5 + SQR_SHORT edx,eax,4 + jmp .finish + +.sqr5: + cmp edi,5 + jg .sqr6 + SQR_SHORT edx,eax,5 + jmp .finish + +.sqr6: + cmp edi,6 + jg .sqr7 + SQR_SHORT edx,eax,6 + jmp .finish + +.sqr7: + cmp edi,7 + jg .sqr8 + SQR_SHORT edx,eax,7 + jmp .finish + +.sqr8: + cmp edi,8 + jg .sqr9 + ;SQR_MIDDL 8 + SQR_SHORT edx,eax,8 + jmp .finish + +.sqr9: + cmp edi,9 + jg .sqr10 + ;SQR_MIDDL 9 + SQR_SHORT edx,eax,9 + jmp .finish + +.sqr10: + cmp edi,10 + jg .sqr11 + ;SQR_MIDDL 10 + SQR_SHORT edx,eax,10 + jmp .finish + +.sqr11: + cmp edi,11 + jg .sqr12 + ;SQR_MIDDL 11 + SQR_SHORT edx,eax,11 + jmp .finish + +.sqr12: + cmp edi,12 + jg .sqr13 + ;SQR_MIDDL 12 + SQR_SHORT edx,eax,12 + jmp .finish + +.sqr13: + cmp edi,13 + jg .sqr14 + ;SQR_MIDDL 13 + SQR_SHORT edx,eax,13 + jmp .finish + +.sqr14: + cmp edi,14 + jg .sqr15 + ;SQR_MIDDL 14 + SQR_SHORT edx,eax,14 + jmp .finish + +.sqr15: + cmp edi,15 + jg .sqr16 + ;SQR_MIDDL 15 + SQR_SHORT edx,eax,15 + jmp .finish + +.sqr16: + cmp edi,16 + jg .sqr17 + ;SQR_MIDDL 16 + SQR_SHORT edx,eax,16 + jmp .finish + +.sqr17: + cmp edi,17 + jg .common_case_sqr + ;SQR_MIDDL 17 + SQR_SHORT edx,eax,17 + jmp .finish + +;; +;; General case Squaring (aSize > 17) +;; +.common_case_sqr: + mov ebx,edx ; copy pR + mov ecx,edi ; copy aSize +;; +;; init result: +;; r = (a[1],a[2],..a[N-1]) * a[0] +;; + mov edx,1 ; i=1 + movd xmm0,DWORD [eax] ; a[0] + movd xmm1,DWORD [eax+edx*4] ; a[i] + pmuludq xmm1,xmm0 ; p = a[i]*a[0] + pandn xmm7,xmm7 ; clear carry + movd DWORD [ebx],xmm7 ; r[0] = 0 +.sqr_init_loop: + movd xmm2,DWORD [eax+edx*4+4] ; read in advance a[i+1] + pmuludq xmm2,xmm0 ; q = a[i+1]*a[0] + + paddq xmm7,xmm1 ; p+= carry + movd DWORD [ebx+edx*4],xmm7 ; r[i] = LO(p) + psrlq xmm7,32 ; carry = HI(p) + + add edx,2 + cmp edx,ecx + jg .sqr_break_init_loop + + movd xmm1,DWORD [eax+edx*4] ; next a[i] + pmuludq xmm1,xmm0 ; p = a[i]*a[0] + + paddq xmm7,xmm2 ; q += carry + movd DWORD [ebx+edx*4-4],xmm7 ; r[i+1] = LO(q) + psrlq xmm7,32 ; carry = HI(q) + + jl .sqr_init_loop + +.sqr_break_init_loop: + movd DWORD [ebx+ecx*4],xmm7 ; r[aSize] = carry + +;; +;; add other a[i]*a[j], i=1,..,N-1, j=i+1,..,N-1 +;; + mov edx,1 ; i=1 + +.sqr_update_mul_loop: + mov esi,edx ; j=i+1 + add esi,1 + mov edi,edx ; i+j + add edi,esi + + movd xmm0,DWORD [eax+edx*4] ; a[i] + movd xmm1,DWORD [eax+esi*4] ; a[j] + pmuludq xmm1,xmm0 ; p = a[j]*a[i] + movd xmm3,DWORD [ebx+edi*4] ; r[i+j] + pandn xmm7,xmm7 ; clear carry + +.sqr_update_mul_inner_loop: + paddq xmm7,xmm1 ; p += carry+r[i+j] + paddq xmm7,xmm3 + movd DWORD [ebx+edi*4],xmm7 ; r[i+j] = LO(p) + psrlq xmm7,32 ; carry = HI(p) + + movd xmm1,DWORD [eax+esi*4+4] ; read in advance a[i+1] + pmuludq xmm1,xmm0 ; p = a[j+1]*a[i] + movd xmm3,DWORD [ebx+edi*4+4] ; read in advance r[i+j+1] + + add edi,1 + add esi,1 + cmp esi,ecx + jl .sqr_update_mul_inner_loop + + movd DWORD [ebx+edi*4],xmm7 ; r[i+j] = carry + add edx,1 + sub esi,1 + cmp edx,esi + jl .sqr_update_mul_loop + pandn xmm7,xmm7 ; clear carry + movd DWORD [ebx+edi*4+4],xmm7 ; r[i+j+1] = 0 + +;; +;; double a[i]*a[j] and add a[i]^2 +;; + pcmpeqd xmm6,xmm6 ; xmm6 = low 32 bit mask + psrlq xmm6,32 + + movd xmm0,DWORD [eax] ; a[i] + pmuludq xmm0,xmm0 ; p = a[i]*a[i] + mov edx,0 ; i=0 + movd xmm2,DWORD [ebx] ; r[2*i] + movd xmm3,DWORD [ebx+4] ; r[2*i+1] + pandn xmm7,xmm7 ; clear carry +.sqr_loop: + paddq xmm2,xmm2 ; 2*r[2*i] + paddq xmm3,xmm3 ; 2*r[2*i+1] + + movq xmm1,xmm0 + pand xmm0,xmm6 ; xmm0 = LO(p) + psrlq xmm1,32 ; xmm1 = HI(p) + + paddq xmm7,xmm2 ; q = carry + 2*r[2*i] + LO(p) + paddq xmm7,xmm0 + movd DWORD [ebx+edx*8],xmm7 ; r[2*i] = LO(q) + psrlq xmm7,32 ; carry = HI(q) + + movd xmm0,DWORD [eax+edx*4+4] ; read in advance a[i+1] + pmuludq xmm0,xmm0 ; p = a[i+1]*a[i+1] + + paddq xmm7,xmm3 ; q = carry + 2*r[2*i+1] + HI(p) + paddq xmm7,xmm1 + movd DWORD [ebx+edx*8+4],xmm7 ; r[2*i+1] = LO(q) + psrlq xmm7,32 ; carry = HI(q) + + add edx,1 + movd xmm2,DWORD [ebx+edx*8] ; r[2*(i+1)] + movd xmm3,DWORD [ebx+edx*8+4] ; r[2*(i+1)+1] + cmp edx,ecx + jl .sqr_loop + + jmp .finish + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; multiplication +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +.multiplication: + cmp edi,esi + jne .common_case_mul + +.mul4: + cmp edi,4 + jne .mul5 + OUTER_LOOP1 4 + jmp .finish + +.mul5: + cmp edi,5 + jne .mul6 + OUTER_LOOP1 5 + jmp .finish + +.mul6: + cmp edi,6 + jne .mul7 + OUTER_LOOP1 6 + jmp .finish + +.mul7: + cmp edi,7 + jne .mul8 + OUTER_LOOP1 7 + jmp .finish + +.mul8: + cmp edi,8 + jne .mul9 + OUTER_LOOP1 8 + jmp .finish + +.mul9: + cmp edi,9 + jne .mul10 + OUTER_LOOP1 9 + jmp .finish + +.mul10: + cmp edi,10 + jne .mul11 + OUTER_LOOP1 10 + jmp .finish + +.mul11: + cmp edi,11 + jne .mul12 + OUTER_LOOP1 11 + jmp .finish + +.mul12: + cmp edi,12 + jne .mul13 + OUTER_LOOP1 12 + jmp .finish + +.mul13: + cmp edi,13 + jne .mul14 + OUTER_LOOP1 13 + jmp .finish + +.mul14: + cmp edi,14 + jne .mul15 + OUTER_LOOP1 14 + jmp .finish + +.mul15: + cmp edi,15 + jne .mul16 + OUTER_LOOP1 15 + jmp .finish + +.mul16: + cmp edi,16 + jne .mul17 + OUTER_LOOP1 16 + jmp .finish + +.mul17: + cmp edi,17 + jne .common_case_mul + OUTER_LOOP1 17 + jmp .finish + +;; +;; General case Multiplication +;; +.common_case_mul: + cmp esi,edi ; bSize ~ aSize + jae .ini_result + + xor edi,esi ; swap(aSize,bSize) + xor esi,edi + xor edi,esi + mov edi,aSize + mov esi,bSize + + xor eax,ebx ; swap(pA,pB) + xor ebx,eax + xor eax,ebx + mov eax,pA + mov ebx,pB + +.ini_result: + add esi,edi + xor ecx,ecx +.mul_init_loop: + mov [edx],ecx + add edx,4 + sub esi,1 + jne .mul_init_loop + + mov esi,edi ; restore aSize + mov edx,pR ; restore pR + + shl edi,2 ; aSize*4 +; +; multiplication loop +; +.macLoop: + movd xmm0,DWORD [ebx] ; pB[] + pxor xmm7,xmm7 + mov esi,edi ; number of loops (==aSize*4) + xor ecx,ecx ; init loop counter + + and esi,7*4 + jz .testSize_8 +.tiny_loop: + movd xmm1,DWORD [eax+ecx] + movd xmm2,DWORD [edx+ecx] + + pmuludq xmm1,xmm0 + paddq xmm1,xmm2 + paddq xmm7,xmm1 + movd DWORD [edx+ecx],xmm7 + psrlq xmm7,32 + + add ecx,4 + cmp ecx,esi + jl .tiny_loop + +.testSize_8: + mov esi,edi ; restore aSize*4 + and esi,8*4 + jz .testSize_16 + + UNROLL8_MULADD + add ecx,8*4 + +.testSize_16: + mov esi,edi ; restore aSize*4 + and esi,0FFFFFFC0h + jz .next_term + +.huge_loop: + UNROLL16_MULADD + add ecx,16*4 + cmp ecx,esi + jl .huge_loop + +.next_term: + movd DWORD [edx + ecx],xmm7 + + add ebx,4 ; move to the next B[] + add edx,4 ; move pR + sub dword bSize,1 ; decrease bSize + jne .macLoop + +.finish: + REST_GPR + ret +ENDFUNC cpMulAdc_BNU_school +%endif + +%endif ;; _IPP_V8 diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpbnusqrw7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpbnusqrw7as.asm new file mode 100644 index 0000000..4f77f91 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpbnusqrw7as.asm @@ -0,0 +1,706 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; Purpose: Cryptography Primitive. +; Big Number Arithmetic +; +; Content: +; cpSqrAdc_BNU_school() +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%if (_IPP >= _IPP_W7) +%include "pcpvariant.inc" + +;; +;; Short-Length Squaring +;; +%macro INIT 3.nolist + %xdefine %%r %1 + %xdefine %%a %2 + %xdefine %%n %3 + + movd mm0,DWORD [%%a] ;; a[0] + pandn mm7,mm7 ;; clear carry + movd DWORD [%%r],mm7 ;; r[0] = 0 + + %assign %%i 1 + movd mm1,DWORD [%%a+%%i*4] ;; a[i] + pmuludq mm1,mm0 ;; t0 = a[i]*a[0] + movd mm2,DWORD [%%a+%%i*4+4] ;; a[i+1] + pmuludq mm2,mm0 ;; t1 = a[i+1]*a[0] + + %rep (%%n-1)/2 + paddq mm7,mm1 ;; t0 += carry + %if %%n-(%%i+2) > 0 + movd mm1,DWORD [%%a+(%%i+2)*4] ;; read in advance a[i+2] + pmuludq mm1,mm0 ;; multiply in advance a[i+2]*a[0] + %endif + movd DWORD [%%r+%%i*4],mm7 ;; r[i] = LO(t0) + psrlq mm7,32 ;; carry = HI(t0) + + paddq mm7,mm2 ;; t1 += carry + %if %%n-(%%i+3) > 0 + movd mm2,DWORD [%%a+(%%i+2)*4+4] ;; read in advance a[i+3] + pmuludq mm2,mm0 ;; multiply in advance a[i+3]*a[0] + %endif + movd DWORD [%%r+%%i*4+4],mm7 ;; r[i+1] = LO(t1) + psrlq mm7,32 ;; carry = HI(t1) + + %assign %%i %%i+2 + %endrep + + %if (((%%n-1) & 1) != 0) + paddq mm7,mm1 ;; t0 += carry + movd DWORD [%%r+%%i*4],mm7 ;; r[i] = LO(t0) + psrlq mm7,32 ;; carry = HI(t0) + %endif + + movd DWORD [%%r+%%n*4],mm7 ;; r[n] = extension +%endmacro + + +%macro UPDATE_MUL 3.nolist + %xdefine %%r %1 + %xdefine %%a %2 + %xdefine %%n %3 + + %assign %%i 1 + %rep (%%n-2) + movd mm0,DWORD [%%a+%%i*4] ;; a[i] + pandn mm7,mm7 ;; clear carry + + %assign %%j %%i+1 + movd mm1,DWORD [%%a+%%j*4] ;; a[j] + pmuludq mm1,mm0 ;; t0 = a[j]*a[i] + movd mm3,DWORD [%%r+(%%i+%%j)*4] ;; r[i+j] + %if (%%j+1) < %%n + movd mm2,DWORD [%%a+%%j*4+4] ;; a[j+1] + pmuludq mm2,mm0 ;; t1 = a[j+1]*a[i] + movd mm4,DWORD [%%r+(%%i+%%j)*4+4] ;; r[i+j+1] + %endif + + %assign %%tn %%n-%%j + %rep %%tn/2 + paddq mm7,mm1 ;; t0 += carry+r[i+j] + %if %%n-(%%j+2) > 0 + movd mm1,DWORD [%%a+(%%j+2)*4] ;; read in advance a[j+2] + pmuludq mm1,mm0 ;; multiply in advance a[j+2]*a[i] + %endif + paddq mm7,mm3 + %if %%n-(%%j+2) > 0 + movd mm3,DWORD [%%r+(%%i+2+%%j)*4] ;; read in advance r[i+j+2] + %endif + movd DWORD [%%r+(%%i+%%j)*4],mm7 ;; r[i+j] = LO(t0) + psrlq mm7,32 ;; carry = HI(t0) + + paddq mm7,mm2 ;; t1 += carry+r[i+j] + %if %%n-(%%j+3) > 0 + movd mm2,DWORD [%%a+(%%j+2)*4+4] ;; read in advance a[j+3] + pmuludq mm2,mm0 ;; multiply in advance a[j+3]*a[i] + %endif + paddq mm7,mm4 + %if %%n-(%%j+3) > 0 + movd mm4,DWORD [%%r+(%%i+2+%%j)*4+4];; read in advance r[i+j+3] + %endif + movd DWORD [%%r+(%%i+%%j)*4+4],mm7 ;; r[i+j+1] = LO(t1) + psrlq mm7,32 ;; carry = HI(t1) + + %assign %%j %%j+2 + %endrep + + %if ((%%tn & 1) != 0) + paddq mm7,mm1 + paddq mm7,mm3 + movd DWORD [%%r+(%%i+%%j)*4],mm7 + psrlq mm7,32 ;; carry = HI(t1) + %endif + + movd DWORD [%%r+(%%n+%%i)*4],mm7 ;; r[i+j] = extension + + %assign %%i %%i+1 + %endrep + + pandn mm7,mm7 ;; clear carry + movd DWORD [%%r+(2*%%n-1)*4],mm7 ;; r[2*n-1] = zero extension +%endmacro + + + +%macro FINALIZE 3.nolist + %xdefine %%r %1 + %xdefine %%a %2 + %xdefine %%n %3 + + %assign %%i 0 + movd mm0,DWORD [%%a+%%i*4] ;; a[i] + pmuludq mm0,mm0 ;; p = a[i]*a[i] + + movd mm2,DWORD [%%r+(2*%%i)*4] ;; r[2*i] + paddq mm2,mm2 + movd mm3,DWORD [%%r+(2*%%i+1)*4];; r[2*i+1] + paddq mm3,mm3 + + pcmpeqd mm6,mm6 ;; mm6 = low 32 bit mask + psrlq mm6,32 + + pandn mm7,mm7 ;; clear carry + + %rep %%n + movq mm1,mm0 + pand mm0,mm6 ;; mm0 = LO(p) + psrlq mm1,32 ;; mm1 = HI(p) + + paddq mm7,mm0 ;; q = carry + 2*r[2*i] + LO(p) + %if %%n-(%%i+1) > 0 + movd mm0,DWORD [%%a+(%%i+1)*4] ;; a[i] + pmuludq mm0,mm0 ;; p = a[i]*a[i] + %endif + paddq mm7,mm2 + %if %%n-(%%i+1) > 0 + movd mm2,DWORD [%%r+(2*%%i+2)*4] ;; r[2*(i+1)] + paddq mm2,mm2 + %endif + movd DWORD [%%r+(2*%%i)*4],mm7 ;; r[2*i] = LO(q) + psrlq mm7,32 ;; carry = HI(q) + + paddq mm7,mm1 ;; q = carry + 2*r[2*i+1] + HI(p) + paddq mm7,mm3 + %if %%n-(%%i+1) > 0 + movd mm3,DWORD [%%r+(2*%%i+2)*4+4] ;; r[2*(i+1)+1] + paddq mm3,mm3 + %endif + movd DWORD [%%r+(2*%%i+1)*4],mm7 ;; r[2*i+1] = LO(q) + psrlq mm7,32 ;; carry = HI(q) + + %assign %%i %%i+1 + %endrep +%endmacro + + +%macro INIT1 3.nolist + %xdefine %%r %1 + %xdefine %%a %2 + %xdefine %%n %3 + + movd mm0,DWORD [%%a] ;; a[0] + pandn mm7,mm7 ;; clear carry + movd DWORD [%%r],mm7 ;; r[0] = 0 + + %assign %%i 1 + %rep (%%n-1) + movd mm1,DWORD [%%a+%%i*4] ;; a[i] + pmuludq mm1,mm0 ;; t0 = a[i]*a[0] + paddq mm7,mm1 ;; t0 += carry + movd DWORD [%%r+%%i*4],mm7 ;; r[i] = LO(t0) + psrlq mm7,32 ;; carry = HI(t0) + %assign %%i %%i+1 + %endrep + movd DWORD [%%r+%%n*4],mm7 ;; r[n] = extension +%endmacro + + +%macro UPDATE_MUL1 3.nolist + %xdefine %%r %1 + %xdefine %%a %2 + %xdefine %%n %3 + + %assign %%i 1 + %rep (%%n-2) + movd mm0,DWORD [%%a+%%i*4] ;; a[i] + pandn mm7,mm7 ;; clear carry + + %assign %%j %%i+1 + %rep (%%n-%%j) + movd mm1,DWORD [%%a+%%j*4] ;; a[j]) + pmuludq mm1,mm0 ;; t0 = a[j]*a[i] + movd mm3,DWORD [%%r+(%%i+%%j)*4] ;; r[i+j] + paddq mm7,mm1 ;; t0 += carry+r[i+j] + paddq mm7,mm3 + movd DWORD [%%r+(%%i+%%j)*4],mm7 ;; r[i+j] = LO(t0) + psrlq mm7,32 ;; carry = HI(t0) + %assign %%j %%j+1 + %endrep + + movd DWORD [%%r+(%%n+%%i)*4],mm7 ;; r[i+j] = extension + %assign %%i %%i+1 + %endrep + + pandn mm7,mm7 ;; clear carry + movd DWORD [%%r+(2*%%n-1)*4],mm7 ;; r[2*n-1] = zero extension +%endmacro + + +%macro SQUARE_SHORT 3.nolist + %xdefine %%r %1 + %xdefine %%a %2 + %xdefine %%n %3 + + INIT %%r,%%a,%%n + UPDATE_MUL %%r,%%a,%%n + FINALIZE %%r,%%a,%%n + emms +%endmacro + + + +;; +;; Middle-Length Squaring +;; +%macro SQR_DECOMPOSE 1.nolist + %xdefine %%i %1 + + movd mm7,DWORD [eax + 4*%%i] + movd mm0,DWORD [eax + 4*%%i] + movd mm6,DWORD [eax + 4*%%i] + %if %%i != 0 + movd mm1,DWORD [ebx + 4*(2*%%i)] + %endif + pslld mm0,1 + pmuludq mm7,mm7 + psrad mm6,32 + %if %%i != 0 + paddq mm7,mm1 + %endif + movd DWORD [ebx + 4*(2*%%i)],mm7 + psrlq mm7,32 +%endmacro + + + +%macro MULADD_START 2.nolist + %xdefine %%i %1 + %xdefine %%j %2 + + movd mm1,DWORD [eax + 4*%%j] + movd mm3,DWORD [eax + 4*%%j] + pmuludq mm1,mm0 + paddq mm7,mm1 + movd DWORD [ebx + 4*(%%i+%%j)],mm7 + pand mm3,mm6 + psrlq mm7,32 + paddq mm7,mm3 +%endmacro + + +%macro MULADD 2.nolist + %xdefine %%i %1 + %xdefine %%j %2 + + movd mm1,DWORD [eax + 4*%%j] + movd mm3,DWORD [eax + 4*%%j] + movd mm2,DWORD [ebx + 4*(%%i+%%j)] + pmuludq mm1,mm0 + pand mm3,mm6 + paddq mm1,mm2 + paddq mm7,mm1 + movd DWORD [ebx + 4*(%%i+%%j)],mm7 + psrlq mm7,32 + paddq mm7,mm3 +%endmacro + + +%macro STORE_CARRY 2.nolist + %xdefine %%i %1 + %xdefine %%s %2 + + movq [ebx + 4*(%%i + %%s)],mm7 +%endmacro + + +%macro STORE_CARRY_NEXT 2.nolist + %xdefine %%i %1 + %xdefine %%s %2 + + movd mm4,DWORD [ebx + 4*(%%i + %%s)] + paddq mm4,mm7 + movd DWORD [ebx + 4*(%%i + %%s)],mm4 + psrlq mm7,32 + movd DWORD [ebx + 4*(%%i + %%s + 1)],mm7 +%endmacro + + +%macro INNER_LOOP 2.nolist + %xdefine %%i %1 + %xdefine %%nsize %2 + + %assign %%j %%i + 1 + %assign %%s %%nsize - %%i - 1 + + SQR_DECOMPOSE %%i + + %rep %%s + %if %%i == 0 + MULADD_START %%i,%%j + el%%se + MULADD %%i,%%j + %endif + %assign %%j %%j + 1 + %endrep + + %if %%i == 0 + STORE_CARRY %%i,%%nsize + %else + STORE_CARRY_NEXT %%i,%%nsize + %endif +%endmacro + + +%macro LAST_STEP 1.nolist + %xdefine %%s %1 + + movd mm7,DWORD [eax + 4*(%%s - 1)] + movd mm2,DWORD [ebx + 4*(2*%%s - 2)] + pmuludq mm7,mm7 + paddq mm7,mm2 + movd mm4,DWORD [ebx + 4*(2*%%s - 1)] + movd DWORD [ebx + 4*(2*%%s - 2)],mm7 + psrlq mm7,32 + paddq mm4,mm7 + movd DWORD [ebx + 4*(2*%%s - 1)],mm4 +%endmacro + + +%macro SQUARE_MIDL 1.nolist + %xdefine %%nsize %1 + + %assign %%i 0 + %rep %%nsize - 1 + INNER_LOOP %%i,%%nsize + %assign %%i %%i + 1 + %endrep + LAST_STEP %%nsize + emms +%endmacro + + +%if (_USE_C_cpSqrAdc_BNU_school_ == 0) + +segment .text align=IPP_ALIGN_FACTOR + +;************************************************************* +;* void cpSqrAdc_BNU_school(Ipp32u* pR, const Ipp32u* pA, int aSize) +;* +;************************************************************* + +;; +;; Lib = W7 +;; +;; Caller = ippsMontMul +;; Caller = ippsMontExp +;; +;; Caller = ippsECCPGetPoint +;; Caller = ippsECCPCheckPoint +;; Caller = ippsECCPAddPoint +;; Caller = ippsECCPMulPointScalar +;; Caller = ippsECCPGenKeyPair +;; Caller = ippsECCPPublicKey +;; Caller = ippsECCPValidateKey +;; Caller = ippsECCPShareSecretKeyDH +;; Caller = ippsECCPShareSecretKeyDHC +;; Caller = ippsECCPSignDSA +;; Caller = ippsECCPSignNR +;; Caller = ippsECCPVerifyDSA +;; Caller = ippsECCPVerifyNR +;; +IPPASM cpSqrAdc_BNU_school,PUBLIC + USES_GPR esi,edi,ebx + +%xdefine pR [esp + ARG_1 + 0*sizeof(dword)] ; target address +%xdefine pA [esp + ARG_1 + 1*sizeof(dword)] ; source address +%xdefine aSize [esp + ARG_1 + 2*sizeof(dword)] ; BNU length + + mov eax,pA ; src + mov ebx,pR ; dst + mov ecx,aSize ; length + +;; +;; switch +;; +.len1: + cmp ecx,1 + jg .len2 + + movd mm0,DWORD [eax] + pmuludq mm0,mm0 + movq QWORD [ebx],mm0 + jmp .finish + +.len2: + cmp ecx,2 + jg .len3 + + movd mm0,DWORD [eax] + movd mm1,DWORD [eax+4] + + movq mm2,mm0 + pmuludq mm0,mm0 ; a[0]*a[0] + pmuludq mm2,mm1 ; a[0]*a[1] + pmuludq mm1,mm1 ; a[1]*a[1] + + pcmpeqd mm7,mm7 ; mm7 = low 32 bit mask + psrlq mm7,32 + + pand mm7,mm2 ; mm7 = LO(a[0]*a[1]) + paddq mm7,mm7 + psrlq mm2,32 ; mm2 = HI(a[0]*a[1]) + paddq mm2,mm2 + + movd DWORD [ebx],mm0 + psrlq mm0,32 + + paddq mm0,mm7 + movd DWORD [ebx+4],mm0 + psrlq mm0,32 + + paddq mm2,mm1 + paddq mm2,mm0 + movq QWORD [ebx+8],mm2 + + jmp .finish + +.len3: + cmp ecx,3 + jg .len4 + SQUARE_SHORT ebx,eax,3 + jmp .finish + +.len4: + cmp ecx,4 + jg .len5 + SQUARE_SHORT ebx,eax,4 + jmp .finish + +.len5: + cmp ecx,5 + jg .len6 + SQUARE_SHORT ebx,eax,5 + jmp .finish + +.len6: + cmp ecx,6 + jg .len7 + SQUARE_SHORT ebx,eax,6 + jmp .finish + +.len7: + cmp ecx,7 + jg .len8 + SQUARE_SHORT ebx,eax,7 + jmp .finish + +.len8: + cmp ecx,8 + jg .len9 + ;SQUARE_MIDL 8 + SQUARE_SHORT ebx,eax,8 + jmp .finish + +.len9: + cmp ecx,9 + jg .len10 + ;SQUARE_MIDL 9 + SQUARE_SHORT ebx,eax,9 + jmp .finish + +.len10: + cmp ecx,10 + jg .len11 + ;SQUARE_MIDL 10 + SQUARE_SHORT ebx,eax,10 + jmp .finish + +.len11: + cmp ecx,11 + jg .len12 + ;SQUARE_MIDL 11 + SQUARE_SHORT ebx,eax,11 + jmp .finish + +.len12: + cmp ecx,12 + jg .len13 + ;SQUARE_MIDL 12 + SQUARE_SHORT ebx,eax,12 + jmp .finish + +.len13: + cmp ecx,13 + jg .len14 + ;SQUARE_MIDL 13 + SQUARE_SHORT ebx,eax,13 + jmp .finish + +.len14: + cmp ecx,14 + jg .len15 + ;SQUARE_MIDL 14 + SQUARE_SHORT ebx,eax,14 + jmp .finish + +.len15: + cmp ecx,15 + jg .len16 + ;SQUARE_MIDL 15 + SQUARE_SHORT ebx,eax,15 + jmp .finish + +.len16: + cmp ecx,16 + jg .len17 + ;SQUARE_MIDL 16 + SQUARE_SHORT ebx,eax,16 + jmp .finish + +.len17: + cmp ecx,17 + jg .common_case + ;SQUARE_MIDL 17 + SQUARE_SHORT ebx,eax,17 + jmp .finish + +;; +;; Ceneral case (aSize > 17) +;; +.common_case: +;; +;; init result: +;; r = (a[1],a[2],..a[N-1]) * a[0] +;; + mov edx,1 ; i=1 + movd mm0,DWORD [eax] ; a[0] + movd mm1,DWORD [eax+edx*4] ; a[i] + pmuludq mm1,mm0 ; p = a[i]*a[0] + pandn mm7,mm7 ; clear carry + movd DWORD [ebx],mm7 ; r[0] = 0 +.init_loop: + movd mm2,DWORD [eax+edx*4+4] ; read in advance a[i+1] + pmuludq mm2,mm0 ; q = a[i+1]*a[0] + + paddq mm7,mm1 ; p+= carry + movd DWORD [ebx+edx*4],mm7 ; r[i] = LO(p) + psrlq mm7,32 ; carry = HI(p) + + add edx,2 + cmp edx,ecx + jg .break_init_loop + + movd mm1,DWORD [eax+edx*4] ; next a[i] + pmuludq mm1,mm0 ; p = a[i]*a[0] + + paddq mm7,mm2 ; q += carry + movd DWORD [ebx+edx*4-4],mm7 ; r[i+1] = LO(q) + psrlq mm7,32 ; carry = HI(q) + + jl .init_loop + +.break_init_loop: + movd DWORD [ebx+ecx*4],mm7 ; r[aSize] = carry + +;; +;; add other a[i]*a[j], i=1,..,N-1, j=i+1,..,N-1 +;; + mov edx,1 ; i=1 + +.update_mul_loop: + mov esi,edx ; j=i+1 + add esi,1 + mov edi,edx ; i+j + add edi,esi + + movd mm0,DWORD [eax+edx*4] ; a[i] + movd mm1,DWORD [eax+esi*4] ; a[j] + pmuludq mm1,mm0 ; p = a[j]*a[i] + movd mm3,DWORD [ebx+edi*4] ; r[i+j] + pandn mm7,mm7 ; clear carry + +.update_mul_inner_loop: + paddq mm7,mm1 ; p += carry+r[i+j] + paddq mm7,mm3 + movd DWORD [ebx+edi*4],mm7 ; r[i+j] = LO(p) + psrlq mm7,32 ; carry = HI(p) + + movd mm1,DWORD [eax+esi*4+4] ; read in advance a[i+1] + pmuludq mm1,mm0 ; p = a[j+1]*a[i] + movd mm3,DWORD [ebx+edi*4+4] ; read in advance r[i+j+1] + + add edi,1 + add esi,1 + cmp esi,ecx + jl .update_mul_inner_loop + + movd DWORD [ebx+edi*4],mm7 ; r[i+j] = carry + add edx,1 + sub esi,1 + cmp edx,esi + jl .update_mul_loop + pandn mm7,mm7 ; clear carry + movd DWORD [ebx+edi*4+4],mm7 ; r[i+j+1] = 0 + +;; +;; double a[i]*a[j] and add a[i]^2 +;; + pcmpeqd mm6,mm6 ; mm6 = low 32 bit mask + psrlq mm6,32 + + movd mm0,DWORD [eax] ; a[i] + pmuludq mm0,mm0 ; p = a[i]*a[i] + mov edx,0 ; i=0 + movd mm2,DWORD [ebx] ; r[2*i] + movd mm3,DWORD [ebx+4] ; r[2*i+1] + pandn mm7,mm7 ; clear carry +.sqr_loop: + paddq mm2,mm2 ; 2*r[2*i] + paddq mm3,mm3 ; 2*r[2*i+1] + + movq mm1,mm0 + pand mm0,mm6 ; mm0 = LO(p) + psrlq mm1,32 ; mm1 = HI(p) + + paddq mm7,mm2 ; q = carry + 2*r[2*i] + LO(p) + paddq mm7,mm0 + movd DWORD [ebx+edx*8],mm7 ; r[2*i] = LO(q) + psrlq mm7,32 ; carry = HI(q) + + movd mm0,DWORD [eax+edx*4+4] ; read in advance a[i+1] + pmuludq mm0,mm0 ; p = a[i+1]*a[i+1] + + paddq mm7,mm3 ; q = carry + 2*r[2*i+1] + HI(p) + paddq mm7,mm1 + movd DWORD [ebx+edx*8+4],mm7 ; r[2*i+1] = LO(q) + psrlq mm7,32 ; carry = HI(q) + + add edx,1 + movd mm2,DWORD [ebx+edx*8] ; r[2*(i+1)] + movd mm3,DWORD [ebx+edx*8+4] ; r[2*(i+1)+1] + cmp edx,ecx + jl .sqr_loop + +.finish: + emms + REST_GPR + ret +ENDFUNC cpSqrAdc_BNU_school + +%endif +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpbnusubw7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpbnusubw7as.asm new file mode 100644 index 0000000..85b59ee --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpbnusubw7as.asm @@ -0,0 +1,82 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; Purpose: Cryptography Primitive. +; Big Number Arithmetic +; +; Content: +; cpSubMul_BNU() +; + + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%if (_IPP >= _IPP_W7) +%include "pcpvariant.inc" + +segment .text align=IPP_ALIGN_FACTOR + +%if (_USE_C_cpSub_BNU_ == 0) + +align IPP_ALIGN_FACTOR +IPPASM cpSub_BNU,PUBLIC + USES_GPR esi,edi,ebx + +%xdefine pDst [esp + ARG_1 + 0*sizeof(dword)] ; target address +%xdefine pSrc1 [esp + ARG_1 + 1*sizeof(dword)] ; source address +%xdefine pSrc2 [esp + ARG_1 + 2*sizeof(dword)] ; source address +%xdefine len [esp + ARG_1 + 3*sizeof(dword)] ; length of BNU + + mov eax,pSrc1 ; src1 + mov ebx,pSrc2 ; src2 + mov edx,pDst ; dst + mov edi,len ; length + shl edi,2 + + xor ecx,ecx + pandn mm0,mm0 + +align IPP_ALIGN_FACTOR +.main_loop: + movd mm1,DWORD [eax + ecx] + movd mm2,DWORD [ebx + ecx] + + paddq mm0,mm1 + psubq mm0,mm2 + movd DWORD [edx + ecx],mm0 + pshufw mm0,mm0,11111110b + + add ecx,4 + cmp ecx,edi + jl .main_loop + + movd eax,mm0 + neg eax + + emms + REST_GPR + ret +ENDFUNC cpSub_BNU +%endif + +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpdelay.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpdelay.asm new file mode 100644 index 0000000..e7a1879 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpdelay.asm @@ -0,0 +1,51 @@ +;=============================================================================== +; Copyright 2022 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the "License"); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, software +; distributed under the License is distributed on an "AS IS" BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions and +; limitations under the License. +;=============================================================================== + +; +; +; Purpose: Delay function, ia32 version +; +; Content: +; _ippcpDelay() +; + +%include "asmdefs.inc" +%include "ia_emm.inc" + +segment .text align=IPP_ALIGN_FACTOR + +;*************************************************************** +;* Purpose: delay +;* +;* void _ippcpDelay(Ipp32u value) +;* +;* Caller = cpAESRandomNoise +;*************************************************************** +align IPP_ALIGN_FACTOR +IPPASM _ippcpDelay,PUBLIC + +%xdefine delayVal [esp + 4] ; indent through the return address + + mov ecx,delayVal + +.Loop: + mov eax,ecx + dec ecx + test eax,eax + jnz .Loop + + ret +ENDFUNC _ippcpDelay diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpmd5w7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpmd5w7as.asm new file mode 100644 index 0000000..bad43e7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpmd5w7as.asm @@ -0,0 +1,315 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to MD5 +; (derived from the RSA Data Security, Inc. MD5 Message-Digest Algorithm) +; +; Content: +; UpdateMD5 +; +; + + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_MD5_) +%if (_IPP >= _IPP_M5) + +;; +;; Magic functions defined in RFC 1321 +;; +%macro MAGIC_F 4.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%Y %3 + %xdefine %%Z %4 + + mov %%F,%%Z + xor %%F,%%Y + and %%F,%%X + xor %%F,%%Z +%endmacro + + +%macro MAGIC_G 4.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%Y %3 + %xdefine %%Z %4 + + MAGIC_F %%F,%%Z,%%X,%%Y +%endmacro + + +%macro MAGIC_H 4.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%Y %3 + %xdefine %%Z %4 + + mov %%F,%%Z + xor %%F,%%Y + xor %%F,%%X +%endmacro + + +%macro MAGIC_I 4.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%Y %3 + %xdefine %%Z %4 + + mov %%F,%%Z + not %%F + or %%F,%%X + xor %%F,%%Y +%endmacro + + + +;; +;; single MD5 step +;; +;; A = B +ROL32((A +MAGIC(B,C,D) +data +const), nrot) +;; +%macro MD5_STEP 9.nolist + %xdefine %%MAGIC_FUN %1 + %xdefine %%A %2 + %xdefine %%B %3 + %xdefine %%C %4 + %xdefine %%D %5 + %xdefine %%FUN %6 + %xdefine %%data %7 + %xdefine %%MD5const %8 + %xdefine %%nrot %9 + + add %%A,%%MD5const + add %%A,[%%data] + MAGIC_FUN %%FUN, %%B,%%C,%%D + add %%A,%%FUN + rol %%A,%%nrot + add %%A,%%B +%endmacro + + +%macro MD5_RND 9.nolist + %xdefine %%MAGIC_FUN %1 + %xdefine %%A %2 + %xdefine %%B %3 + %xdefine %%C %4 + %xdefine %%D %5 + %xdefine %%FUN %6 + %xdefine %%MD5const %7 + %xdefine %%nrot %8 + %xdefine %%nextdata %9 + + %%MAGIC_FUN %%FUN, %%B,%%C,%%D + lea %%A,[%%A+ebp+%%MD5const] + %ifnempty %%nextdata + mov ebp,[%%nextdata] + %endif +; MAGIC_FUN FUN, B,C,D + add %%A,%%FUN + rol %%A,%%nrot + add %%A,%%B +%endmacro + + + +segment .text align=IPP_ALIGN_FACTOR + + +;***************************************************************************************** +;* Purpose: Update internal digest according to message block +;* +;* void UpdateMD5(DigestMD5digest, const Ipp32u* mblk, int mlen, const void* pParam) +;* +;***************************************************************************************** + +;; +;; MD5 left rotations (number of bits) +;; +%assign rot11 7 +%assign rot12 12 +%assign rot13 17 +%assign rot14 22 +%assign rot21 5 +%assign rot22 9 +%assign rot23 14 +%assign rot24 20 +%assign rot31 4 +%assign rot32 11 +%assign rot33 16 +%assign rot34 23 +%assign rot41 6 +%assign rot42 10 +%assign rot43 15 +%assign rot44 21 + +;; +;; Lib = W7 +;; +;; Caller = ippsSHA1Update +;; Caller = ippsSHA1Final +;; Caller = ippsSHA1MessageDigest +;; +;; Caller = ippsHMACSHA1Update +;; Caller = ippsHMACSHA1Final +;; Caller = ippsHMACSHA1MessageDigest +;; + +align IPP_ALIGN_FACTOR +IPPASM UpdateMD5,PUBLIC + USES_GPR esi,edi,ebx,ebp + +%xdefine digest [esp + ARG_1 + 0*sizeof(dword)] ; digest address +%xdefine mblk [esp + ARG_1 + 1*sizeof(dword)] ; buffer address +%xdefine mlen [esp + ARG_1 + 2*sizeof(dword)] ; buffer length +%xdefine pParam [esp + ARG_1 + 3*sizeof(dword)] ; dummy parameter + +%xdefine MBS_MD5 (64) + mov eax, pParam ; due to bug in ml12 - dummy instruction + + mov edi,digest ; digest address + mov esi,mblk ; source data address + mov eax,mlen ; data length + + sub esp,2*sizeof(dword) + mov [esp+0*sizeof(dword)],edi ; save digest address + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +.md5_block_loop: + mov [esp+1*sizeof(dword)],eax ; save data length + + mov ebp,[esi+ 0*4] ; preload data + +;; +;; init A, B, C, D by the internal digest +;; + mov eax,[edi+0*4] ; eax = digest[0] (A) + mov ebx,[edi+1*4] ; ebx = digest[1] (B) + mov ecx,[edi+2*4] ; ecx = digest[2] (C) + mov edx,[edi+3*4] ; edx = digest[3] (D) + +;; +;; perform 0-63 steps +;; +;; MAGIC A, B, C, D, FUN, cnt, nrot, pNextData (ebp) +;; ------------------------------------------------------------ + MD5_RND MAGIC_F, eax,ebx,ecx,edx, edi, 0d76aa478h, rot11, {esi+ 1*4} + MD5_RND MAGIC_F, edx,eax,ebx,ecx, edi, 0e8c7b756h, rot12, {esi+ 2*4} + MD5_RND MAGIC_F, ecx,edx,eax,ebx, edi, 0242070dbh, rot13, {esi+ 3*4} + MD5_RND MAGIC_F, ebx,ecx,edx,eax, edi, 0c1bdceeeh, rot14, {esi+ 4*4} + MD5_RND MAGIC_F, eax,ebx,ecx,edx, edi, 0f57c0fafh, rot11, {esi+ 5*4} + MD5_RND MAGIC_F, edx,eax,ebx,ecx, edi, 04787c62ah, rot12, {esi+ 6*4} + MD5_RND MAGIC_F, ecx,edx,eax,ebx, edi, 0a8304613h, rot13, {esi+ 7*4} + MD5_RND MAGIC_F, ebx,ecx,edx,eax, edi, 0fd469501h, rot14, {esi+ 8*4} + MD5_RND MAGIC_F, eax,ebx,ecx,edx, edi, 0698098d8h, rot11, {esi+ 9*4} + MD5_RND MAGIC_F, edx,eax,ebx,ecx, edi, 08b44f7afh, rot12, {esi+10*4} + MD5_RND MAGIC_F, ecx,edx,eax,ebx, edi, 0ffff5bb1h, rot13, {esi+11*4} + MD5_RND MAGIC_F, ebx,ecx,edx,eax, edi, 0895cd7beh, rot14, {esi+12*4} + MD5_RND MAGIC_F, eax,ebx,ecx,edx, edi, 06b901122h, rot11, {esi+13*4} + MD5_RND MAGIC_F, edx,eax,ebx,ecx, edi, 0fd987193h, rot12, {esi+14*4} + MD5_RND MAGIC_F, ecx,edx,eax,ebx, edi, 0a679438eh, rot13, {esi+15*4} + MD5_RND MAGIC_F, ebx,ecx,edx,eax, edi, 049b40821h, rot14, {esi+ 1*4} + + MD5_RND MAGIC_G, eax,ebx,ecx,edx, edi, 0f61e2562h, rot21, {esi+ 6*4} + MD5_RND MAGIC_G, edx,eax,ebx,ecx, edi, 0c040b340h, rot22, {esi+11*4} + MD5_RND MAGIC_G, ecx,edx,eax,ebx, edi, 0265e5a51h, rot23, {esi+ 0*4} + MD5_RND MAGIC_G, ebx,ecx,edx,eax, edi, 0e9b6c7aah, rot24, {esi+ 5*4} + MD5_RND MAGIC_G, eax,ebx,ecx,edx, edi, 0d62f105dh, rot21, {esi+10*4} + MD5_RND MAGIC_G, edx,eax,ebx,ecx, edi, 002441453h, rot22, {esi+15*4} + MD5_RND MAGIC_G, ecx,edx,eax,ebx, edi, 0d8a1e681h, rot23, {esi+ 4*4} + MD5_RND MAGIC_G, ebx,ecx,edx,eax, edi, 0e7d3fbc8h, rot24, {esi+ 9*4} + MD5_RND MAGIC_G, eax,ebx,ecx,edx, edi, 021e1cde6h, rot21, {esi+14*4} + MD5_RND MAGIC_G, edx,eax,ebx,ecx, edi, 0c33707d6h, rot22, {esi+ 3*4} + MD5_RND MAGIC_G, ecx,edx,eax,ebx, edi, 0f4d50d87h, rot23, {esi+ 8*4} + MD5_RND MAGIC_G, ebx,ecx,edx,eax, edi, 0455a14edh, rot24, {esi+13*4} + MD5_RND MAGIC_G, eax,ebx,ecx,edx, edi, 0a9e3e905h, rot21, {esi+ 2*4} + MD5_RND MAGIC_G, edx,eax,ebx,ecx, edi, 0fcefa3f8h, rot22, {esi+ 7*4} + MD5_RND MAGIC_G, ecx,edx,eax,ebx, edi, 0676f02d9h, rot23, {esi+12*4} + MD5_RND MAGIC_G, ebx,ecx,edx,eax, edi, 08d2a4c8ah, rot24, {esi+ 5*4} + + MD5_RND MAGIC_H, eax,ebx,ecx,edx, edi, 0fffa3942h, rot31, {esi+ 8*4} + MD5_RND MAGIC_H, edx,eax,ebx,ecx, edi, 08771f681h, rot32, {esi+11*4} + MD5_RND MAGIC_H, ecx,edx,eax,ebx, edi, 06d9d6122h, rot33, {esi+14*4} + MD5_RND MAGIC_H, ebx,ecx,edx,eax, edi, 0fde5380ch, rot34, {esi+ 1*4} + MD5_RND MAGIC_H, eax,ebx,ecx,edx, edi, 0a4beea44h, rot31, {esi+ 4*4} + MD5_RND MAGIC_H, edx,eax,ebx,ecx, edi, 04bdecfa9h, rot32, {esi+ 7*4} + MD5_RND MAGIC_H, ecx,edx,eax,ebx, edi, 0f6bb4b60h, rot33, {esi+10*4} + MD5_RND MAGIC_H, ebx,ecx,edx,eax, edi, 0bebfbc70h, rot34, {esi+13*4} + MD5_RND MAGIC_H, eax,ebx,ecx,edx, edi, 0289b7ec6h, rot31, {esi+ 0*4} + MD5_RND MAGIC_H, edx,eax,ebx,ecx, edi, 0eaa127fah, rot32, {esi+ 3*4} + MD5_RND MAGIC_H, ecx,edx,eax,ebx, edi, 0d4ef3085h, rot33, {esi+ 6*4} + MD5_RND MAGIC_H, ebx,ecx,edx,eax, edi, 004881d05h, rot34, {esi+ 9*4} + MD5_RND MAGIC_H, eax,ebx,ecx,edx, edi, 0d9d4d039h, rot31, {esi+12*4} + MD5_RND MAGIC_H, edx,eax,ebx,ecx, edi, 0e6db99e5h, rot32, {esi+15*4} + MD5_RND MAGIC_H, ecx,edx,eax,ebx, edi, 01fa27cf8h, rot33, {esi+ 2*4} + MD5_RND MAGIC_H, ebx,ecx,edx,eax, edi, 0c4ac5665h, rot34, {esi+ 0*4} + + MD5_RND MAGIC_I, eax,ebx,ecx,edx, edi, 0f4292244h, rot41, {esi+ 7*4} + MD5_RND MAGIC_I, edx,eax,ebx,ecx, edi, 0432aff97h, rot42, {esi+14*4} + MD5_RND MAGIC_I, ecx,edx,eax,ebx, edi, 0ab9423a7h, rot43, {esi+ 5*4} + MD5_RND MAGIC_I, ebx,ecx,edx,eax, edi, 0fc93a039h, rot44, {esi+12*4} + MD5_RND MAGIC_I, eax,ebx,ecx,edx, edi, 0655b59c3h, rot41, {esi+ 3*4} + MD5_RND MAGIC_I, edx,eax,ebx,ecx, edi, 08f0ccc92h, rot42, {esi+10*4} + MD5_RND MAGIC_I, ecx,edx,eax,ebx, edi, 0ffeff47dh, rot43, {esi+ 1*4} + MD5_RND MAGIC_I, ebx,ecx,edx,eax, edi, 085845dd1h, rot44, {esi+ 8*4} + MD5_RND MAGIC_I, eax,ebx,ecx,edx, edi, 06fa87e4fh, rot41, {esi+15*4} + MD5_RND MAGIC_I, edx,eax,ebx,ecx, edi, 0fe2ce6e0h, rot42, {esi+ 6*4} + MD5_RND MAGIC_I, ecx,edx,eax,ebx, edi, 0a3014314h, rot43, {esi+13*4} + MD5_RND MAGIC_I, ebx,ecx,edx,eax, edi, 04e0811a1h, rot44, {esi+ 4*4} + MD5_RND MAGIC_I, eax,ebx,ecx,edx, edi, 0f7537e82h, rot41, {esi+11*4} + MD5_RND MAGIC_I, edx,eax,ebx,ecx, edi, 0bd3af235h, rot42, {esi+ 2*4} + MD5_RND MAGIC_I, ecx,edx,eax,ebx, edi, 02ad7d2bbh, rot43, {esi+ 9*4} + MD5_RND MAGIC_I, ebx,ecx,edx,eax, edi, 0eb86d391h, rot44, {esp} + +;; +;; update digest +;; + add [ebp+0*4],eax ; advance digest + mov eax, dword [esp+1*sizeof(dword)] + add [ebp+1*4],ebx + add [ebp+2*4],ecx + add [ebp+3*4],edx + + mov edi, ebp ; restore hash address + add esi, MBS_MD5 + sub eax, MBS_MD5 + jg .md5_block_loop + + add esp,2*sizeof(dword) + REST_GPR + ret +ENDFUNC UpdateMD5 + +%endif +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpmontreductionv8as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpmontreductionv8as.asm new file mode 100644 index 0000000..f13135d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpmontreductionv8as.asm @@ -0,0 +1,603 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; Purpose: Cryptography Primitive. +; Big Number Arithmetic (Montgomery Reduction) +; +; Content: +; cpMontRedAdc_BNU() +; +; History: +; This implementation used instead of previous one +; (see pcpmontredictw7as.asm) +; +; Extra reduction (R=A-M) has been added to perform MontReduction safe +; + + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%if (_IPP >= _IPP_V8) +%include "pcpvariant.inc" +%include "pcpbnu.inc" + +%if _USE_NN_MONTMUL_ == _NUSE + + +%macro MULADD_wt_carry 1.nolist + %xdefine %%i %1 + + movd xmm7,DWORD [eax] ;; pM[0] + pmuludq xmm7,xmm0 ;; t = pM[0] * u + movd xmm2,DWORD [edx+4*(%%i)] ;; pBuffer[i] + paddq xmm7,xmm2 ;; t += pBuffer[i] + movd DWORD [edx+4*(%%i)],xmm7 ;; pBuffer[i] = LO(t) + psrlq xmm7,32 ;; carryLcl = HI(t) +%endmacro + + +%macro MULADD1 3.nolist + %xdefine %%i %1 + %xdefine %%j %2 + %xdefine %%nsize %3 + + movd xmm1,DWORD [eax+4*%%j] ;; pM[j] + pmuludq xmm1,xmm0 ;; t = pM[j] * u + movd xmm2,DWORD [edx+4*(%%i+%%j)] ;; pBuffer[i+j] + paddq xmm1,xmm2 ;; t +=pBuffer[i+j] + paddq xmm7,xmm1 ;; +carryLcl + movd DWORD [edx+4*(%%i+%%j)],xmm7 ;; pBuffer[i+j] = LO(t) + psrlq xmm7,32 ;; carryLcl = HI(t) +%endmacro + + +; INNER_LOOP1 is defined in pcpbnu.inc as well +%ifmacro INNER_LOOP1 2 +%unmacro INNER_LOOP1 2 +%endif +%macro INNER_LOOP1 2.nolist + %xdefine %%i %1 + %xdefine %%size %2 + + %assign %%j 0 + movd xmm0,DWORD [edx+4*%%i] ;; pBuffer[i] + + pmuludq xmm0,xmm5 ;; u = (Ipp32u)( m0*pBuffer[i] ) + movd xmm4,DWORD [edx+4*(%%i+%%size)] ;; w = pBuffer[i+mSize] + paddq xmm4,xmm6 ;; +carryGbl + + %rep %%size + %if %%j == 0 + MULADD_wt_carry %%i + %else + MULADD1 %%i,%%j,%%size + %endif + %assign %%j %%j + 1 + %endrep + + paddq xmm7,xmm4 ;; w+= carryLcl + + movd DWORD [edx+4*(%%i+%%size)],xmm7 ;; pBuffer[i+mSize] = LO(w) +; pshufw xmm6,xmm7,11111110b ;; carryGbl = HI(w) + pshuflw xmm6,xmm7,11111110b ;; carryGbl = HI(w) +%endmacro + + +; OUTER_LOOP1 is defined in pcpbnu.inc as well +%ifmacro OUTER_LOOP1 1 +%unmacro OUTER_LOOP1 1 +%endif +%macro OUTER_LOOP1 1.nolist + %xdefine %%nsize %1 + + movd xmm5,DWORD m0 ; m0 + pandn xmm6,xmm6 ; init carryGbl = 0 + + %assign %%i 0 + %rep %%nsize + INNER_LOOP1 %%i,%%nsize + %assign %%i %%i + 1 + %endrep + + psrlq xmm7,32 +%endmacro + + + +%macro UNROLL8 0.nolist + movd xmm1,DWORD [eax+ecx] + movd xmm2,DWORD [edx+ecx] + movd xmm3,DWORD [eax+ecx+4] + movd xmm4,DWORD [edx+ecx+4] + movd xmm5,DWORD [eax+ecx+8] + movd xmm6,DWORD [edx+ecx+8] + + pmuludq xmm1,xmm0 + paddq xmm1,xmm2 + pmuludq xmm3,xmm0 + paddq xmm3,xmm4 + pmuludq xmm5,xmm0 + paddq xmm5,xmm6 + + paddq xmm7,xmm1 + movd xmm1,DWORD [eax+ecx+12] + movd xmm2,DWORD [edx+ecx+12] + movd DWORD [edx+ecx],xmm7 + psrlq xmm7,32 + + pmuludq xmm1,xmm0 + paddq xmm1,xmm2 + + paddq xmm7,xmm3 + movd xmm3,DWORD [eax+ecx+16] + movd xmm4,DWORD [edx+ecx+16] + movd DWORD [edx+ecx+4],xmm7 + psrlq xmm7,32 + + pmuludq xmm3,xmm0 + paddq xmm3,xmm4 + + paddq xmm7,xmm5 + movd xmm5,DWORD [eax+ecx+20] + movd xmm6,DWORD [edx+ecx+20] + movd DWORD [edx+ecx+8],xmm7 + psrlq xmm7,32 + + pmuludq xmm5,xmm0 + paddq xmm5,xmm6 + + paddq xmm7,xmm1 + movd xmm1,DWORD [eax+ecx+24] + movd xmm2,DWORD [edx+ecx+24] + movd DWORD [edx+ecx+12],xmm7 + psrlq xmm7,32 + + pmuludq xmm1,xmm0 + paddq xmm1,xmm2 + + paddq xmm7,xmm3 + movd xmm3,DWORD [eax+ecx+28] + movd xmm4,DWORD [edx+ecx+28] + movd DWORD [edx+ecx+16],xmm7 + psrlq xmm7,32 + + pmuludq xmm3,xmm0 + paddq xmm3,xmm4 + + paddq xmm7,xmm5 + movd DWORD [edx+ecx+20],xmm7 + psrlq xmm7,32 + + paddq xmm7,xmm1 + movd DWORD [edx+ecx+24],xmm7 + psrlq xmm7,32 + + paddq xmm7,xmm3 + movd DWORD [edx+ecx+28],xmm7 + psrlq xmm7,32 +%endmacro + + +%macro UNROLL16 0.nolist + movd xmm1,DWORD [eax+ecx] + movd xmm2,DWORD [edx+ecx] + movd xmm3,DWORD [eax+ecx+4] + movd xmm4,DWORD [edx+ecx+4] + movd xmm5,DWORD [eax+ecx+8] + movd xmm6,DWORD [edx+ecx+8] + + pmuludq xmm1,xmm0 + paddq xmm1,xmm2 + pmuludq xmm3,xmm0 + paddq xmm3,xmm4 + pmuludq xmm5,xmm0 + paddq xmm5,xmm6 + + paddq xmm7,xmm1 + movd xmm1,DWORD [eax+ecx+12] + movd xmm2,DWORD [edx+ecx+12] + movd DWORD [edx+ecx],xmm7 + psrlq xmm7,32 + + pmuludq xmm1,xmm0 + paddq xmm1,xmm2 + + paddq xmm7,xmm3 + movd xmm3,DWORD [eax+ecx+16] + movd xmm4,DWORD [edx+ecx+16] + movd DWORD [edx+ecx+4],xmm7 + psrlq xmm7,32 + + pmuludq xmm3,xmm0 + paddq xmm3,xmm4 + + paddq xmm7,xmm5 + movd xmm5,DWORD [eax+ecx+20] + movd xmm6,DWORD [edx+ecx+20] + movd DWORD [edx+ecx+8],xmm7 + psrlq xmm7,32 + + pmuludq xmm5,xmm0 + paddq xmm5,xmm6 + + paddq xmm7,xmm1 + movd xmm1,DWORD [eax+ecx+24] + movd xmm2,DWORD [edx+ecx+24] + movd DWORD [edx+ecx+12],xmm7 + psrlq xmm7,32 + + pmuludq xmm1,xmm0 + paddq xmm1,xmm2 + + paddq xmm7,xmm3 + movd xmm3,DWORD [eax+ecx+28] + movd xmm4,DWORD [edx+ecx+28] + movd DWORD [edx+ecx+16],xmm7 + psrlq xmm7,32 + + pmuludq xmm3,xmm0 + paddq xmm3,xmm4 + + paddq xmm7,xmm5 + movd xmm5,DWORD [eax+ecx+32] + movd xmm6,DWORD [edx+ecx+32] + movd DWORD [edx+ecx+20],xmm7 + psrlq xmm7,32 + + pmuludq xmm5,xmm0 + paddq xmm5,xmm6 + + paddq xmm7,xmm1 + movd xmm1,DWORD [eax+ecx+36] + movd xmm2,DWORD [edx+ecx+36] + movd DWORD [edx+ecx+24],xmm7 + psrlq xmm7,32 + + pmuludq xmm1,xmm0 + paddq xmm1,xmm2 + + paddq xmm7,xmm3 + movd xmm3,DWORD [eax+ecx+40] + movd xmm4,DWORD [edx+ecx+40] + movd DWORD [edx+ecx+28],xmm7 + psrlq xmm7,32 + + pmuludq xmm3,xmm0 + paddq xmm3,xmm4 + + paddq xmm7,xmm5 + movd xmm5,DWORD [eax+ecx+44] + movd xmm6,DWORD [edx+ecx+44] + movd DWORD [edx+ecx+32],xmm7 + psrlq xmm7,32 + + pmuludq xmm5,xmm0 + paddq xmm5,xmm6 + + paddq xmm7,xmm1 + movd xmm1,DWORD [eax+ecx+48] + movd xmm2,DWORD [edx+ecx+48] + movd DWORD [edx+ecx+36],xmm7 + psrlq xmm7,32 + + pmuludq xmm1,xmm0 + paddq xmm1,xmm2 + + paddq xmm7,xmm3 + movd xmm3,DWORD [eax+ecx+52] + movd xmm4,DWORD [edx+ecx+52] + movd DWORD [edx+ecx+40],xmm7 + psrlq xmm7,32 + + pmuludq xmm3,xmm0 + paddq xmm3,xmm4 + + paddq xmm7,xmm5 + movd xmm5,DWORD [eax+ecx+56] + movd xmm6,DWORD [edx+ecx+56] + movd DWORD [edx+ecx+44],xmm7 + psrlq xmm7,32 + + pmuludq xmm5,xmm0 + paddq xmm5,xmm6 + + paddq xmm7,xmm1 + movd xmm1,DWORD [eax+ecx+60] + movd xmm2,DWORD [edx+ecx+60] + movd DWORD [edx+ecx+48],xmm7 + psrlq xmm7,32 + + pmuludq xmm1,xmm0 + paddq xmm1,xmm2 + + paddq xmm7,xmm3 + movd DWORD [edx+ecx+52],xmm7 + psrlq xmm7,32 + + paddq xmm7,xmm5 + movd DWORD [edx+ecx+56],xmm7 + psrlq xmm7,32 + + paddq xmm7,xmm1 + movd DWORD [edx+ecx+60],xmm7 + psrlq xmm7,32 +%endmacro + + + +segment .text align=IPP_ALIGN_FACTOR + +%if (_USE_C_cpMontRedAdc_BNU_ == 0) +;************************************************************* +;* void cpMontRedAdc_BNU(Ipp32u* pR, +;* Ipp32u* pBuffer, +;* const Ipp32u* pModulo, int mSize, Ipp32u m0) +;* +;************************************************************* + +;; +;; Lib = V8 +;; +;; Caller = ippsMontMul +;; +IPPASM cpMontRedAdc_BNU,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; pointer to the reduction +%xdefine pBuffer [ebp + ARG_1 + 1*sizeof(dword)] ; pointer to the product +%xdefine pModulo [ebp + ARG_1 + 2*sizeof(dword)] ; pointer to the modulo +%xdefine mSize [ebp + ARG_1 + 3*sizeof(dword)] ; modulo size +%xdefine m0 [ebp + ARG_1 + 4*sizeof(dword)] ; helper + + mov eax,pModulo ; modulo address + mov edi,mSize ; modulo size + mov edx,pBuffer ; temp product address + +; +; spesial cases +; +.tst_reduct4: + cmp edi,4 ; special case + jne .tst_reduct5 + OUTER_LOOP1 4 + add edx,4*4 + jmp .finish + +.tst_reduct5: + cmp edi,5 ; special case + jne .tst_reduct6 + OUTER_LOOP1 5 + add edx,4*5 + jmp .finish + +.tst_reduct6: + cmp edi,6 ; special case + jne .tst_reduct7 + OUTER_LOOP1 6 + add edx,4*6 + jmp .finish + +.tst_reduct7: + cmp edi,7 ; special case + jne .tst_reduct8 + OUTER_LOOP1 7 + add edx,4*7 + jmp .finish + +.tst_reduct8: + cmp edi,8 ; special case + jne .tst_reduct9 + OUTER_LOOP1 8 + add edx,4*8 + jmp .finish + +.tst_reduct9: + cmp edi,9 ; special case + jne .tst_reduct10 + OUTER_LOOP1 9 + add edx,4*9 + jmp .finish + +.tst_reduct10: + cmp edi,10 ; special case + jne .tst_reduct11 + OUTER_LOOP1 10 + add edx,4*10 + jmp .finish + +.tst_reduct11: + cmp edi,11 ; special case + jne .tst_reduct12 + OUTER_LOOP1 11 + add edx,4*11 + jmp .finish + +.tst_reduct12: + cmp edi,12 ; special case + jne .tst_reduct13 + OUTER_LOOP1 12 + add edx,4*12 + jmp .finish + +.tst_reduct13: + cmp edi,13 ; special case + jne .tst_reduct14 + OUTER_LOOP1 13 + add edx,4*13 + jmp .finish + +.tst_reduct14: + cmp edi,14 ; special case + jne .tst_reduct15 + OUTER_LOOP1 14 + add edx,4*14 + jmp .finish + +.tst_reduct15: + cmp edi,15 ; special case + jne .tst_reduct16 + OUTER_LOOP1 15 + add edx,4*15 + jmp .finish + +.tst_reduct16: + cmp edi,16 ; special case + jne .reduct_general + OUTER_LOOP1 16 + add edx,4*16 + jmp .finish + +; +; general case +; +.reduct_general: + sub esp,4 ; allocate slot for carryGbl + + pandn xmm6,xmm6 ; init carryGbl = 0 + + mov ebx,edi + shl ebx,2 ; modulo size in bytes (outer counter) + shl edi,2 + +.mainLoop: + movd xmm0,DWORD m0 ; m0 helper + movd xmm1,DWORD [edx] ; pBuffer[i] + movd DWORD [esp],xmm6 ; save carryGbl + + pmuludq xmm0,xmm1 ; u = (Ipp32u)( m0*pBuffer[i] ) + + xor ecx,ecx ; inner index + pandn xmm7,xmm7 ; int carryLcl = 0 + +; +; case: 0 != mSize%8 +; + mov esi,edi ; copy modulo size copy + and esi,7*4 ; and test (inner counter) + jz .testSize_8 + +.small_loop: + movd xmm1,DWORD [eax+ecx] ; pModulo[] + pmuludq xmm1,xmm0 ; t = pModulo[]*u + movd xmm2,DWORD [edx+ecx] ; pBuffer[] + paddq xmm1,xmm2 ; t +=pBuffer[] + paddq xmm7,xmm1 ; +carryLcl + movd DWORD [edx+ecx],xmm7 ; pBuffer[] = LO(t) + psrlq xmm7,32 ; carryLcl = HI(t) + + add ecx,4 + cmp ecx,esi + jl .small_loop + +; +; case: mSize==8 +; +.testSize_8: + mov esi,edi ; copy modulo size copy + and esi,8*4 ; and test + jz .testSize_16 + + UNROLL8 + add ecx,8*4 + +; +; case: mSize==16*n +; +.testSize_16: + mov esi,edi ; copy modulo size copy + and esi,0FFFFFFC0h ; and test + jz .next_term + +.unroll16_loop: + UNROLL16 + add ecx,16*4 + cmp ecx,esi + jl .unroll16_loop + +.next_term: + movd xmm1,DWORD [edx+ecx] ; pBuffer[] + paddq xmm7,xmm1 ; t = pBuffer[]+carryLcl + movd xmm6,DWORD [esp] ; carryGbl + paddq xmm6,xmm7 ; t +=carryGbl + movd DWORD [edx+ecx],xmm6 ; pBuffer[] = LO(t) + psrlq xmm6,32 ; carryGbl = HI(t) + + add edx,4 ; advance pBuffer + sub ebx,4 ; decrease outer counter + jg .mainLoop + + add esp,4 ; release slot for carryGbl + shr edi,2 ; restore mSize + + +;; +;; finish +;; +.finish: + pxor xmm7,xmm7 ; converr carryGbl into the mask + psubd xmm7,xmm6 + + mov esi,pR ; pointer to the result + pandn xmm0,xmm0 ; borrow=0 + xor ecx,ecx ; index =0 + ; perform pR[] = pBuffer[] - pModulus[] +.subtract_loop: + movd xmm1,DWORD [edx+ecx*4]; pBuffer[] + paddq xmm0,xmm1 + movd xmm2,DWORD [eax+ecx*4]; pModulus[] + psubq xmm0,xmm2 + movd DWORD [esi+ecx*4],xmm0 + pshuflw xmm0,xmm0,11111110b + + add ecx,1 + cmp ecx,edi + jl .subtract_loop + + pcmpeqd xmm6,xmm6 ; convert borrow into the mask + pxor xmm0,xmm6 + por xmm0,xmm7 ; common (carryGbl and borrow) mask + + pcmpeqd xmm7,xmm7 ; mask and + pxor xmm7,xmm0 ; ~mask + + xor ecx,ecx ; index =0 + ; masked copy: pR[] = (mask & pR[]) | (~mask & pBuffer[]) +.masked_copy_loop: + movd xmm1,DWORD [esi+ecx*4]; pR[] + pand xmm1,xmm0 + movd xmm2,DWORD [edx+ecx*4]; pBuffer[] + pand xmm2,xmm7 + por xmm1,xmm2 + movd DWORD [esi+ecx*4],xmm1; pR[] + + add ecx,1 + cmp ecx,edi + jl .masked_copy_loop + REST_GPR + ret +ENDFUNC cpMontRedAdc_BNU +%endif + +%endif ;; _IPP_V8 +%endif ;; _USE_NN_MONTMUL_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpmontreductionw7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpmontreductionw7as.asm new file mode 100644 index 0000000..e94fce9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpmontreductionw7as.asm @@ -0,0 +1,618 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Big Number Arithmetic (Montgomery Reduction) +; +; Content: +; cpMontRedAdc_BNU() +; +; History: +; This implementation used instead of previous one +; (see pcpmontredictw7as.asm) +; +; Extra reduction (R=A-M) has been added to perform MontReduction safe +; +; + + + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%if (_IPP == _IPP_W7) || (_IPP == _IPP_T7) +%include "pcpvariant.inc" +%include "pcpbnu.inc" + +%if _USE_NN_MONTMUL_ == _NUSE + + +%macro MULADD_wt_carry 1.nolist + %xdefine %%i %1 + + movd mm7,DWORD [eax] ;; pM[0] + movd mm2,DWORD [edx+4*(%%i)] ;; pBuffer[i] + pmuludq mm7,mm0 ;; t = pM[0] * u + paddq mm7,mm2 ;; +pBuffer[i] + movd DWORD [edx+4*(%%i)],mm7 ;; pBuffer[i] = LO(t) + psrlq mm7,32 ;; carryLcl = HI(t) +%endmacro + + +%macro MULADD1 3.nolist + %xdefine %%i %1 + %xdefine %%j %2 + %xdefine %%nsize %3 + + movd mm1,DWORD [eax+4*%%j] ;; pM[j] + movd mm2,DWORD [edx+4*(%%i+%%j)] ;; pBuffer[i+j] + pmuludq mm1,mm0 ;; t = pM[j] * u + paddq mm1,mm2 ;; +pBuffer[i+j] + paddq mm7,mm1 ;; +carryLcl + movd DWORD [edx+4*(%%i+%%j)],mm7 ;; pBuffer[i+j] = LO(t) + psrlq mm7,32 ;; carryLcl = HI(t) +%endmacro + +; INNER_LOOP1 is defined in pcpbnu.inc as well +%ifmacro INNER_LOOP1 2 +%unmacro INNER_LOOP1 2 +%endif +%macro INNER_LOOP1 2.nolist + %xdefine %%i %1 + %xdefine %%size %2 + + %assign %%j 0 + movd mm0,DWORD [edx+4*%%i] ;; pBuffer[i] + + pmuludq mm0,mm5 ;; u = (Ipp32u)( m0*pBuffer[i] ) + movd mm4,DWORD [edx+4*(%%i+%%size)] ;; w = pBuffer[i+mSize] + paddq mm4,mm6 ;; +carryGbl + + %rep %%size + %if %%j == 0 + MULADD_wt_carry %%i + %else + MULADD1 %%i,%%j,%%size + %endif + %assign %%j %%j + 1 + %endrep + + paddq mm7,mm4 ;; w+= carryLcl + + movd DWORD [edx+4*(%%i+%%size)],mm7 ;; pBuffer[i+mSize] = LO(w) + pshufw mm6,mm7,11111110b ;; carryGbl = HI(w) +%endmacro + + +; OUTER_LOOP1 is defined in pcpbnu.inc as well +%ifmacro OUTER_LOOP1 1 +%unmacro OUTER_LOOP1 1 +%endif +%macro OUTER_LOOP1 1.nolist + %xdefine %%nsize %1 + + movd mm5,DWORD m0 ; m0 + pandn mm6,mm6 ; init carryGbl = 0 + + %assign %%i 0 + %rep %%nsize + INNER_LOOP1 %%i,%%nsize + %assign %%i %%i + 1 + %endrep + + psrlq mm7,32 +%endmacro + + + +%macro UNROLL8 0.nolist + movd mm1,DWORD [eax+ecx] + movd mm2,DWORD [edx+ecx] + movd mm3,DWORD [eax+ecx+4] + movd mm4,DWORD [edx+ecx+4] + movd mm5,DWORD [eax+ecx+8] + movd mm6,DWORD [edx+ecx+8] + + pmuludq mm1,mm0 + paddq mm1,mm2 + pmuludq mm3,mm0 + paddq mm3,mm4 + pmuludq mm5,mm0 + paddq mm5,mm6 + + movd mm2,DWORD [edx+ecx+12] + paddq mm7,mm1 + movd mm1,DWORD [eax+ecx+12] + + pmuludq mm1,mm0 + paddq mm1,mm2 + movd mm2,DWORD [edx+ecx+24] + movd DWORD [edx+ecx],mm7 + + psrlq mm7,32 + paddq mm7,mm3 + + movd mm3,DWORD [eax+ecx+16] + movd mm4,DWORD [edx+ecx+16] + + pmuludq mm3,mm0 + + movd DWORD [edx+ecx+4],mm7 + + psrlq mm7,32 + paddq mm3,mm4 + movd mm4,DWORD [edx+ecx+28] + paddq mm7,mm5 + movd mm5,DWORD [eax+ecx+20] + movd mm6,DWORD [edx+ecx+20] + + pmuludq mm5,mm0 + movd DWORD [edx+ecx+8],mm7 + psrlq mm7,32 + + paddq mm5,mm6 + paddq mm7,mm1 + + movd mm1,DWORD [eax+ecx+24] + + pmuludq mm1,mm0 + + movd DWORD [edx+ecx+12],mm7 + psrlq mm7,32 + + paddq mm1,mm2 + paddq mm7,mm3 + + movd mm3,DWORD [eax+ecx+28] + pmuludq mm3,mm0 + + movd DWORD [edx+ecx+16],mm7 + psrlq mm7,32 + paddq mm3,mm4 + paddq mm7,mm5 + + movd DWORD [edx+ecx+20],mm7 + psrlq mm7,32 + + paddq mm7,mm1 + movd DWORD [edx+ecx+24],mm7 + psrlq mm7,32 + + paddq mm7,mm3 + + movd DWORD [edx+ecx+28],mm7 + psrlq mm7,32 +%endmacro + + +%macro UNROLL16 0.nolist + movd mm1,DWORD [eax+ecx] + movd mm2,DWORD [edx+ecx] + movd mm3,DWORD [eax+ecx+4] + movd mm4,DWORD [edx+ecx+4] + movd mm5,DWORD [eax+ecx+8] + movd mm6,DWORD [edx+ecx+8] + + pmuludq mm1,mm0 + paddq mm1,mm2 + pmuludq mm3,mm0 + paddq mm3,mm4 + pmuludq mm5,mm0 + paddq mm5,mm6 + + movd mm2,DWORD [edx+ecx+12] + paddq mm7,mm1 + movd mm1,DWORD [eax+ecx+12] + + pmuludq mm1,mm0 + paddq mm1,mm2 + movd mm2,DWORD [edx+ecx+24] + movd DWORD [edx+ecx],mm7 + + psrlq mm7,32 + paddq mm7,mm3 + + movd mm3,DWORD [eax+ecx+16] + movd mm4,DWORD [edx+ecx+16] + + pmuludq mm3,mm0 + + movd DWORD [edx+ecx+4],mm7 + + psrlq mm7,32 + paddq mm3,mm4 + movd mm4,DWORD [edx+ecx+28] + paddq mm7,mm5 + movd mm5,DWORD [eax+ecx+20] + movd mm6,DWORD [edx+ecx+20] + + pmuludq mm5,mm0 + movd DWORD [edx+ecx+8],mm7 + psrlq mm7,32 + + paddq mm5,mm6 + movd mm6,DWORD [edx+ecx+32] + paddq mm7,mm1 + + movd mm1,DWORD [eax+ecx+24] + + pmuludq mm1,mm0 + + movd DWORD [edx+ecx+12],mm7 + psrlq mm7,32 + + paddq mm1,mm2 + movd mm2,DWORD [edx+ecx+36] + paddq mm7,mm3 + + movd mm3,DWORD [eax+ecx+28] + pmuludq mm3,mm0 + + movd DWORD [edx+ecx+16],mm7 + psrlq mm7,32 + paddq mm3,mm4 + movd mm4,DWORD [edx+ecx+40] + paddq mm7,mm5 + + movd mm5,DWORD [eax+ecx+32] + pmuludq mm5,mm0 + + movd DWORD [edx+ecx+20],mm7 + psrlq mm7,32 + + paddq mm5,mm6 + movd mm6,DWORD [edx+ecx+44] + paddq mm7,mm1 + movd mm1,DWORD [eax+ecx+36] + pmuludq mm1,mm0 + movd DWORD [edx+ecx+24],mm7 + psrlq mm7,32 + + paddq mm1,mm2 + movd mm2,DWORD [edx+ecx+48] + paddq mm7,mm3 + movd mm3,DWORD [eax+ecx+40] + pmuludq mm3,mm0 + + movd DWORD [edx+ecx+28],mm7 + psrlq mm7,32 + paddq mm3,mm4 + movd mm4,DWORD [edx+ecx+52] + paddq mm7,mm5 + + movd mm5,DWORD [eax+ecx+44] + pmuludq mm5,mm0 + + movd DWORD [edx+ecx+32],mm7 + psrlq mm7,32 + paddq mm5,mm6 + movd mm6,DWORD [edx+ecx+56] + paddq mm7,mm1 + + movd mm1,DWORD [eax+ecx+48] + pmuludq mm1,mm0 + + movd DWORD [edx+ecx+36],mm7 + psrlq mm7,32 + + paddq mm1,mm2 + movd mm2,DWORD [edx+ecx+60] + paddq mm7,mm3 + + movd mm3,DWORD [eax+ecx+52] + pmuludq mm3,mm0 + + movd DWORD [edx+ecx+40],mm7 + psrlq mm7,32 + paddq mm3,mm4 + paddq mm7,mm5 + + movd mm5,DWORD [eax+ecx+56] + + pmuludq mm5,mm0 + movd DWORD [edx+ecx+44],mm7 + psrlq mm7,32 + paddq mm5,mm6 + paddq mm7,mm1 + + movd mm1,DWORD [eax+ecx+60] + + pmuludq mm1,mm0 + movd DWORD [edx+ecx+48],mm7 + psrlq mm7,32 + + paddq mm7,mm3 + movd DWORD [edx+ecx+52],mm7 + psrlq mm7,32 + paddq mm1,mm2 + paddq mm7,mm5 + movd DWORD [edx+ecx+56],mm7 + psrlq mm7,32 + + paddq mm7,mm1 + movd DWORD [edx+ecx+60],mm7 + psrlq mm7,32 +%endmacro + + + +segment .text align=IPP_ALIGN_FACTOR + + +;************************************************************* +;* void cpMontRedAdc_BNU(Ipp32u* pR, +;* Ipp32u* pBuffer, +;* const Ipp32u* pModulo, int mSize, Ipp32u m0) +;* +;************************************************************* + +;; +;; Lib = W7 +;; +;; Caller = ippsMontMul +;; +IPPASM cpMontRedAdc_BNU,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; pointer to the reduction +%xdefine pBuffer [ebp + ARG_1 + 1*sizeof(dword)] ; pointer to the product +%xdefine pModulo [ebp + ARG_1 + 2*sizeof(dword)] ; pointer to the modulo +%xdefine mSize [ebp + ARG_1 + 3*sizeof(dword)] ; modulo size +%xdefine m0 [ebp + ARG_1 + 4*sizeof(dword)] ; helper + + mov eax,pModulo ; modulo address + mov edi,mSize ; modulo size + mov edx,pBuffer ; temp product address + +; +; spesial cases +; +.tst_reduct4: + cmp edi,4 ; special case + jne .tst_reduct5 + OUTER_LOOP1 4 + add edx,4*4 + jmp .finish + +.tst_reduct5: + cmp edi,5 ; special case + jne .tst_reduct6 + OUTER_LOOP1 5 + add edx,4*5 + jmp .finish + +.tst_reduct6: + cmp edi,6 ; special case + jne .tst_reduct7 + OUTER_LOOP1 6 + add edx,4*6 + jmp .finish + +.tst_reduct7: + cmp edi,7 ; special case + jne .tst_reduct8 + OUTER_LOOP1 7 + add edx,4*7 + jmp .finish + +.tst_reduct8: + cmp edi,8 ; special case + jne .tst_reduct9 + OUTER_LOOP1 8 + add edx,4*8 + jmp .finish + +.tst_reduct9: + cmp edi,9 ; special case + jne .tst_reduct10 + OUTER_LOOP1 9 + add edx,4*9 + jmp .finish + +.tst_reduct10: + cmp edi,10 ; special case + jne .tst_reduct11 + OUTER_LOOP1 10 + add edx,4*10 + jmp .finish + +.tst_reduct11: + cmp edi,11 ; special case + jne .tst_reduct12 + OUTER_LOOP1 11 + add edx,4*11 + jmp .finish + +.tst_reduct12: + cmp edi,12 ; special case + jne .tst_reduct13 + OUTER_LOOP1 12 + add edx,4*12 + jmp .finish + +.tst_reduct13: + cmp edi,13 ; special case + jne .tst_reduct14 + OUTER_LOOP1 13 + add edx,4*13 + jmp .finish + +.tst_reduct14: + cmp edi,14 ; special case + jne .tst_reduct15 + OUTER_LOOP1 14 + add edx,4*14 + jmp .finish + +.tst_reduct15: + cmp edi,15 ; special case + jne .tst_reduct16 + OUTER_LOOP1 15 + add edx,4*15 + jmp .finish + +.tst_reduct16: + cmp edi,16 ; special case + jne .reduct_general + OUTER_LOOP1 16 + add edx,4*16 + jmp .finish + +; +; general case +; +.reduct_general: + sub esp,4 ; allocate slot for carryGbl + + pandn mm6,mm6 ; init carryGbl = 0 + + mov ebx,edi + shl ebx,2 ; modulo size in bytes (outer counter) + shl edi,2 + +.mainLoop: + movd mm0,DWORD m0 ; m0 helper + movd mm1,DWORD [edx] ; pBuffer[i] + movd DWORD [esp],mm6 ; save carryGbl + + pmuludq mm0,mm1 ; u = (Ipp32u)( m0*pBuffer[i] ) + + xor ecx,ecx ; inner index + pandn mm7,mm7 ; int carryLcl = 0 + +; +; case: 0 != mSize%8 +; + mov esi,edi ; copy modulo size copy + and esi,7*4 ; and test (inner counter) + jz .testSize_8 + +.small_loop: + movd mm1,DWORD [eax+ecx] ; pModulo[] + movd mm2,DWORD [edx+ecx] ; pBuffer[] + + pmuludq mm1,mm0 ; t = pModulo[]*u + paddq mm2,mm1 ; +pBuffer[] + paddq mm7,mm2 ; +carryLcl + movd DWORD [edx+ecx],mm7 ; pBuffer[] = LO(t) + psrlq mm7,32 ; carryLcl = HI(t) + + add ecx,4 + cmp ecx,esi + jl .small_loop + +; +; case: mSize==8 +; +.testSize_8: + mov esi,edi ; copy modulo size copy + and esi,8*4 ; and test + jz .testSize_16 + + UNROLL8 + add ecx,8*4 + +; +; case: mSize==16*n +; +.testSize_16: + mov esi,edi ; copy modulo size copy + and esi,0FFFFFFC0h ; and test + jz .next_term + +.unroll16_loop: + UNROLL16 + add ecx,16*4 + cmp ecx,esi + jl .unroll16_loop + +.next_term: + movd mm1,DWORD [edx+ecx] ; pBuffer[] + movd mm6,DWORD [esp] ; carryGbl + paddq mm7,mm1 ; t = pBuffer[]+carryLcl + paddq mm6,mm7 ; +carryGbl + movd DWORD [edx+ecx],mm6 ; pBuffer[] = LO(t) + psrlq mm6,32 ; carryGbl = HI(t) + + add edx,4 ; advance pBuffer + sub ebx,4 ; decrease outer counter + jg .mainLoop + + add esp,4 ; release slot for carryGbl + shr edi,2 ; restore mSize + + +;; +;; finish +;; +.finish: + pxor mm7,mm7 ; converr carryGbl into the mask + psubd mm7,mm6 + + mov esi,pR ; pointer to the result + mov ebx,edi ; restore mSize + pandn mm0,mm0 ; borrow=0 + xor ecx,ecx ; index =0 + ; perform pR[] = pBuffer[] - pModulus[] +.subtract_loop: + movd mm1,DWORD [edx+ecx*4]; pBuffer[] + movd mm2,DWORD [eax+ecx*4]; pModulus[] + + psubq mm1,mm2 + paddq mm1,mm0 + movd DWORD [esi+ecx*4],mm1 + pshufw mm0,mm1,11111110b + + add ecx,1 + cmp ecx,edi + jl .subtract_loop + + pcmpeqd mm6,mm6 ; convert borrow into the mask + pxor mm0,mm6 + por mm0,mm7 ; common (carryGbl and borrow) mask + + pcmpeqd mm7,mm7 ; mask and + pxor mm7,mm0 ; ~mask + + xor ecx,ecx ; index =0 + ; masked copy: pR[] = (mask & pR[]) | (~mask & pBuffer[]) +.masked_copy_loop: + movd mm1,DWORD [esi+ecx*4]; pR[] + pand mm1,mm0 + movd mm2,DWORD [edx+ecx*4]; pBuffer[] + pand mm2,mm7 + por mm1,mm2 + movd DWORD [esi+ecx*4],mm1; pR[] + + add ecx,1 + cmp ecx,edi + jl .masked_copy_loop + + emms + REST_GPR + ret +ENDFUNC cpMontRedAdc_BNU + +%endif ;; _IPP_W7 || _IPP_T7 +%endif ;; _USE_NN_MONTMUL_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpp192r1arith_mont_slm.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpp192r1arith_mont_slm.asm new file mode 100644 index 0000000..31a43e6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpp192r1arith_mont_slm.asm @@ -0,0 +1,810 @@ +;=============================================================================== +; Copyright (C) 2016 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; P192r1 basic arithmetic function +; +; Content: +; p192r1_add +; p192r1_sub +; p192r1_neg +; p192r1_div_by_2 +; p192r1_mul_mont_slm +; p192r1_sqr_mont_slm +; p192r1_mred +; p192r1_select_ap_w7 +; + + + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" +%include "pcpvariant.inc" + +%if (_IPP >= _IPP_P8) + +segment .text align=IPP_ALIGN_FACTOR + +;; +;; some p384r1 constants +;; +p192r1_data: +_prime192r1 DD 0FFFFFFFFh,0FFFFFFFFh,0FFFFFFFEh,0FFFFFFFFh,0FFFFFFFFh,0FFFFFFFFh + +%assign LEN192 (192/32) ; dword's length of operands + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; Ipp32u add_192(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +;; input: edi = r +;; esi = a +;; ebx = b +;; +;; output: eax = carry = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM add_192,PRIVATE + ; r = a+b + mov eax, dword [esi] + add eax, dword [ebx] + mov dword [edi], eax + + mov eax, dword [esi+sizeof(dword)] + adc eax, dword [ebx+sizeof(dword)] + mov dword [edi+sizeof(dword)], eax + + mov eax, dword [esi+sizeof(dword)*2] + adc eax, dword [ebx+sizeof(dword)*2] + mov dword [edi+sizeof(dword)*2], eax + + mov eax, dword [esi+sizeof(dword)*3] + adc eax, dword [ebx+sizeof(dword)*3] + mov dword [edi+sizeof(dword)*3], eax + + mov eax, dword [esi+sizeof(dword)*4] + adc eax, dword [ebx+sizeof(dword)*4] + mov dword [edi+sizeof(dword)*4], eax + + mov eax, dword [esi+sizeof(dword)*5] + adc eax, dword [ebx+sizeof(dword)*5] + mov dword [edi+sizeof(dword)*5], eax + + mov eax, 0 + adc eax, 0 + ret +ENDFUNC add_192 + +;; +;; Ipp32u sub_192(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +;; input: edi = r +;; esi = a +;; ebx = b +;; +;; output: eax = borrow = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM sub_192,PRIVATE + ; r = a-b + mov eax, dword [esi] + sub eax, dword [ebx] + mov dword [edi], eax + + mov eax, dword [esi+sizeof(dword)] + sbb eax, dword [ebx+sizeof(dword)] + mov dword [edi+sizeof(dword)], eax + + mov eax, dword [esi+sizeof(dword)*2] + sbb eax, dword [ebx+sizeof(dword)*2] + mov dword [edi+sizeof(dword)*2], eax + + mov eax, dword [esi+sizeof(dword)*3] + sbb eax, dword [ebx+sizeof(dword)*3] + mov dword [edi+sizeof(dword)*3], eax + + mov eax, dword [esi+sizeof(dword)*4] + sbb eax, dword [ebx+sizeof(dword)*4] + mov dword [edi+sizeof(dword)*4], eax + + mov eax, dword [esi+sizeof(dword)*5] + sbb eax, dword [ebx+sizeof(dword)*5] + mov dword [edi+sizeof(dword)*5], eax + + mov eax, 0 + adc eax, 0 + ret +ENDFUNC sub_192 + +;; +;; Ipp32u shl_192(Ipp32u* r, const Ipp32u* a) +;; +;; input: edi = r +;; esi = a +;; +;; output: eax = extension = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM shl_192,PRIVATE + mov eax, dword [esi+(LEN192-1)*sizeof(dword)] + ; r = a<<1 + movq xmm2, qword [esi+sizeof(oword)] + movdqu xmm1, oword [esi] + + movdqa xmm3, xmm2 + palignr xmm3, xmm1, sizeof(qword) + psllq xmm2, 1 + psrlq xmm3, 63 + por xmm2, xmm3 + movq qword [edi+sizeof(oword)], xmm2 + + movdqa xmm3, xmm1 + pslldq xmm3, sizeof(qword) + psllq xmm1, 1 + psrlq xmm3, 63 + por xmm1, xmm3 + movdqu oword [edi], xmm1 + + shr eax, 31 + ret +ENDFUNC shl_192 + +;; +;; void shr_192(Ipp32u* r, const Ipp32u* a) +;; +;; input: edi = r +;; esi = a +;; eax = ext +;; output: eax = extension = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM shr_192,PRIVATE + ; r = a>>1 + movdqu xmm2, oword [esi] + movq xmm1, qword [esi+sizeof(oword)] + + movdqa xmm3, xmm1 + palignr xmm3, xmm2, sizeof(qword) + psrlq xmm2, 1 + psllq xmm3, 63 + por xmm2, xmm3 + movdqu oword [edi], xmm2 + + movdqa xmm3, xmm0 + psrlq xmm1, 1 + psllq xmm3, 63 + por xmm1, xmm3 + movq qword [edi+sizeof(oword)], xmm1 + + ret +ENDFUNC shr_192 + +;; +;; void cpy_192(Ipp32u* r, const Ipp32u* a) +;; +%macro cpy_192 2.nolist + %xdefine %%pdst %1 + %xdefine %%psrc %2 + + movdqu xmm0, oword [%%psrc] + movq xmm1, qword [%%psrc+sizeof(oword)] + movdqu oword [%%pdst], xmm0 + movq qword [%%pdst+sizeof(oword)], xmm1 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p192r1_add(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +align IPP_ALIGN_FACTOR +IPPASM p192r1_add,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +%xdefine pB [ebp + ARG_1 + 2*sizeof(dword)] ; source B address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN192] +%assign _sp_ _buf_+(LEN192)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_+sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + mov edi, pR ; pR + mov esi, pA ; pA + mov ebx, pB ; pB + CALL_IPPASM add_192 ; R = A+B + mov edx, eax + + lea edi, [esp+_buf_] ; T + mov esi, pR ; R + LD_ADDR ebx, p192r1_data ; modulus + lea ebx, [ebx+(_prime192r1-p192r1_data)] + CALL_IPPASM sub_192 ; T = R-modulus + + lea esi,[esp+_buf_] + mov edi, pR + sub edx, eax ; R = T<0? R : T + cmovnz esi, edi + cpy_192 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p192r1_add + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p192r1_sub(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +align IPP_ALIGN_FACTOR +IPPASM p192r1_sub,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +%xdefine pB [ebp + ARG_1 + 2*sizeof(dword)] ; source B address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN192] +%assign _sp_ _buf_+(LEN192)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_+sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + mov edi, pR ; pR + mov esi, pA ; pA + mov ebx, pB ; pB + CALL_IPPASM sub_192 ; R = A-B + mov edx, eax + + lea edi, [esp+_buf_] ; T + mov esi, pR ; R + LD_ADDR ebx, p192r1_data ; modulus + lea ebx, [ebx+(_prime192r1-p192r1_data)] + CALL_IPPASM add_192 ; T = R+modulus + + lea esi,[esp+_buf_] + mov edi, pR + test edx, edx ; R = T<0? R : T + cmovz esi, edi + cpy_192 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p192r1_sub + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p192r1_neg(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p192r1_neg,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN192] +%assign _sp_ _buf_+(LEN192)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + mov edi, pR ; outpur pR + mov esi, pA ; input pA + + ; r = 0-a + mov eax, 0 + sub eax, dword [esi] + mov dword [edi], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)] + mov dword [edi+sizeof(dword)], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*2] + mov dword [edi+sizeof(dword)*2], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*3] + mov dword [edi+sizeof(dword)*3], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*4] + mov dword [edi+sizeof(dword)*4], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*5] + mov dword [edi+sizeof(dword)*5], eax + sbb edx,edx + + lea edi, [esp+_buf_] ; T + mov esi, pR ; R + LD_ADDR ebx, p192r1_data ; modulus + lea ebx, [ebx+(_prime192r1-p192r1_data)] + CALL_IPPASM add_192 ; T = R+modulus + + lea esi,[esp+_buf_] + mov edi, pR + test edx, edx ; R = T<0? R : T + cmovz esi, edi + cpy_192 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p192r1_neg + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p192r1_mul_by_2(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p192r1_mul_by_2,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN192] +%assign _sp_ _buf_+(LEN192)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_+sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + lea edi, [esp+_buf_] ; T + mov esi, pA ; pA + CALL_IPPASM shl_192 ; T = A<<1 + mov edx, eax + + mov esi, edi ; T + mov edi, pR ; R + LD_ADDR ebx, p192r1_data ; modulus + lea ebx, [ebx+(_prime192r1-p192r1_data)] + CALL_IPPASM sub_192 ; R = T-modulus + + sub edx, eax ; R = R<0? T : R + cmovz esi, edi + cpy_192 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p192r1_mul_by_2 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p192r1_mul_by_3(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p192r1_mul_by_3,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _bufT_ 0 ; T buffer[LEN192] +%assign _bufU_ _bufT_+(LEN192)*sizeof(dword) ; U buffer[LEN192] +%assign _mod_ _bufU_+(LEN192)*sizeof(dword) ; modulus address [1] +%assign _sp_ _mod_+sizeof(dword) ; esp [1] +%assign _frame_ _sp_+sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + LD_ADDR eax, p192r1_data ; srore modulus address + lea eax, [eax+(_prime192r1-p192r1_data)] + mov dword [esp+_mod_], eax + + lea edi, [esp+_bufT_] ; T + mov esi, pA ; A + CALL_IPPASM shl_192 ; T = A<<1 + mov edx, eax + + mov esi, edi ; T + lea edi, [esp+_bufU_] ; U + mov ebx, [esp+_mod_] ; modulus + CALL_IPPASM sub_192 ; U = T-modulus + + sub edx, eax ; T = U<0? T : U + cmovz esi, edi + cpy_192 edi, esi + + mov esi, edi + mov ebx, pA + CALL_IPPASM add_192 ; T +=A + mov edx, eax + + mov edi, pR ; R + mov ebx, [esp+_mod_] ; modulus + CALL_IPPASM sub_192 ; R = T-modulus + + sub edx, eax ; R = T<0? R : T + cmovz esi, edi + cpy_192 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p192r1_mul_by_3 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p192r1_div_by_2(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p192r1_div_by_2,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN192] +%assign _sp_ _buf_+(LEN192)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_+sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + lea edi, [esp+_buf_] ; T + mov esi, pA ; A + LD_ADDR ebx, p192r1_data ; modulus + lea ebx, [ebx+(_prime192r1-p192r1_data)] + CALL_IPPASM add_192 ; R = A+modulus + mov edx, 0 + + mov ecx, dword [esi] ; shifted_data = (a[0]&1)? T : A + and ecx, 1 + cmovnz esi, edi + cmovz eax, edx + movd xmm0, eax + mov edi, pR + CALL_IPPASM shr_192 + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p192r1_div_by_2 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p192r1_mul_mont_slm(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +align IPP_ALIGN_FACTOR +IPPASM p192r1_mul_mont_slm,PUBLIC + USES_GPR ebp,ebx,esi,edi + +%xdefine pR [eax + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [eax + ARG_1 + 1*sizeof(dword)] ; source A address +%xdefine pB [eax + ARG_1 + 2*sizeof(dword)] ; source B address +; +; stack layout: +; +%assign _buf_ 0 +%assign _rp_ _buf_+(LEN192+1)*sizeof(dword) ; pR +%assign _ap_ _rp_+sizeof(dword) ; pA +%assign _bp_ _ap_+sizeof(dword) ; pB +%assign _sp_ _bp_+sizeof(dword) ; esp storage +%assign _ssize_ _sp_+sizeof(dword) ; size allocated stack + + mov eax, esp ; save esp + sub esp, _ssize_ ; allocate stack + and esp, -16 ; provide 16-byte stack alignment + mov dword [esp+_sp_], eax ; store original esp + + ; clear buffer + pxor mm0, mm0 + movq [esp+_buf_], mm0 + movq [esp+_buf_+sizeof(qword)], mm0 + movq [esp+_buf_+sizeof(qword)*2], mm0 + movq [esp+_buf_+sizeof(qword)*3], mm0 + + ; store parameters into the stack + ; note: eax here stores an original esp, so it can be used to reach function parameters + mov edi, pR + mov esi, pA + mov ebp, pB + mov dword [esp+_rp_], edi + mov dword [esp+_ap_], esi + mov dword [esp+_bp_], ebp + + mov edi, LEN192 + + movd mm1, dword [esi+sizeof(dword)] ; pre load a[1], a[2] + movd mm2, dword [esi+sizeof(dword)*2] + + +align IPP_ALIGN_FACTOR +.mmul_loop: +; +; i-st pass +; modulus = 2^192 -2^64 -1 +; [6] [2] [0] +; m0 = 1 +; + movd mm7, edi ; save pass counter + + mov edx, dword [ebp] ; b = b[i] + mov eax, dword [esi] ; a[0] + movd mm0, edx + add ebp, sizeof(dword) + mov dword [esp+_bp_], ebp + + pmuludq mm1, mm0 ; a[1]*b[i] + + mul edx ; (E:u) = (edx:eax) = a[0]*b[i]+buf[0] + add eax, dword [esp+_buf_] + adc edx, 0 + + pmuludq mm2, mm0 ; a[2]*b[i] + +; multiplication round 1 - round 2 + movd ecx, mm1 ; p = a[1]*b[i] + E + psrlq mm1, 32 + add ecx, edx + movd edx, mm1 + adc edx, 0 + add ecx, dword [esp+_buf_+sizeof(dword)*1] + movd mm1, dword [esi+sizeof(dword)*3] + adc edx, 0 + + movd ebx, mm2 ; p = a[2]*b[i] + E + psrlq mm2, 32 + add ebx, edx + movd edx, mm2 + adc edx, 0 + add ebx, dword [esp+_buf_+sizeof(dword)*2] + movd mm2, dword [esi+sizeof(dword)*4] + movd mm3, dword [esi+sizeof(dword)*5] + adc edx, 0 + + pmuludq mm1, mm0 ; a[3]*b[i] + pmuludq mm2, mm0 ; a[4]*b[i] + pmuludq mm3, mm0 ; a[5]*b[i] + +;;; and reduction ;;; + mov dword [esp+_buf_+sizeof(dword)*0], ecx ; +0 + sub ebx, eax ; -u0 + mov edi, 0 + mov dword [esp+_buf_+sizeof(dword)*1], ebx + adc edi, 0 ; ssave bf + +; multiplication round 3 - round 5 + movd ecx, mm1 ; p = a[3]*b[i] + E + psrlq mm1, 32 + add ecx, edx + movd edx, mm1 + adc edx, 0 + add ecx, dword [esp+_buf_+sizeof(dword)*3] + adc edx, 0 + + movd ebx, mm2 ; p = a[4]*b[i] + E + psrlq mm2, 32 + add ebx, edx + movd edx, mm2 + adc edx, 0 + add ebx, dword [esp+_buf_+sizeof(dword)*4] + adc edx, 0 + + movd ebp, mm3 ; p = a[5]*b[i] + E + psrlq mm3, 32 + add ebp, edx + movd edx, mm3 + adc edx, 0 + add ebp, dword [esp+_buf_+sizeof(dword)*5] + adc edx, 0 + +;;; and reduction ;;; + sub ecx, edi ; -cb + mov dword [esp+_buf_+sizeof(dword)*2], ecx + sbb ebx, 0 ; -bf + mov dword [esp+_buf_+sizeof(dword)*3], ebx + sbb ebp, 0 ; -bf + mov dword [esp+_buf_+sizeof(dword)*4], ebp + +;;; last multiplication round 6 + movd edi, mm7 ; restore pass counter + + sbb eax, 0 ; u0 -bf + mov ebx, 0 + add edx, dword [esp+_buf_+sizeof(dword)*6] + adc ebx, 0 + add edx, eax + adc ebx, 0 + mov dword [esp+_buf_+sizeof(dword)*5], edx + mov dword [esp+_buf_+sizeof(dword)*6], ebx + + sub edi, 1 + movd mm1, dword [esi+sizeof(dword)] ; speculative load a[1], a[2], a[3], a[4] + movd mm2, dword [esi+sizeof(dword)*2] + jz .exit_mmul_loop + + mov ebp, dword [esp+_bp_] ; restore pB + jmp .mmul_loop + +.exit_mmul_loop: + emms + +; final reduction + mov edi, [esp+_rp_] ; result + lea esi, [esp+_buf_] ; buffer + LD_ADDR ebx, p192r1_data ; modulus + lea ebx, [ebx+(_prime192r1-p192r1_data)] + CALL_IPPASM sub_192 + mov edx, dword [esp+_buf_+sizeof(dword)*LEN192] + sub edx, eax + +; copy + cmovz esi, edi + cpy_192 edi, esi + + mov esp, [esp+_sp_] ; release stack + REST_GPR + ret +ENDFUNC p192r1_mul_mont_slm + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p192r1_sqr_mont_slm(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p192r1_sqr_mont_slm,PUBLIC + USES_GPR esi,edi + +%xdefine pR [esp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [esp + ARG_1 + 1*sizeof(dword)] ; source A address + + ;; use p192r1_mul_mont_slm to compute sqr + mov esi, pA + mov edi, pR + push esi + push esi + push edi + CALL_IPPASM p192r1_mul_mont_slm,PUBLIC + add esp, sizeof(dword)*3 + REST_GPR + ret +ENDFUNC p192r1_sqr_mont_slm + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p192r1_mred(Ipp32u* r, Ipp32u* prod) +;; +; modulus = 2^192 -2^64 -1 +; [6] [2] [0] +; m0 = 1 +; +align IPP_ALIGN_FACTOR +IPPASM p192r1_mred,PUBLIC + USES_GPR ebx,esi,edi + +%xdefine pR [esp + ARG_1 + 0*sizeof(dword)] ; reduction address +%xdefine pA [esp + ARG_1 + 1*sizeof(dword)] ; source product address + + ; get parameters: + mov esi, pA + + mov ecx, LEN192 + xor edx, edx +align IPP_ALIGN_FACTOR +.mred_loop: + mov eax, dword [esi] + + mov ebx, 0 + mov dword [esi], ebx + + mov ebx, dword [esi+sizeof(dword)] + mov dword [esi+sizeof(dword)], ebx + + mov ebx, dword [esi+sizeof(dword)*2] + sub ebx, eax + mov dword [esi+sizeof(dword)*2], ebx + + mov ebx, dword [esi+sizeof(dword)*3] + sbb ebx, 0 + mov dword [esi+sizeof(dword)*3], ebx + + mov ebx, dword [esi+sizeof(dword)*4] + sbb ebx, 0 + mov dword [esi+sizeof(dword)*4], ebx + + mov ebx, dword [esi+sizeof(dword)*5] + sbb ebx, 0 + mov dword [esi+sizeof(dword)*5], ebx + + mov ebx, dword [esi+sizeof(dword)*6] + sbb eax, 0 + add eax, edx + mov edx, 0 + adc edx, 0 + add ebx, eax + mov dword [esi+sizeof(dword)*6], ebx + adc edx, 0 + + lea esi, [esi+sizeof(dword)] + sub ecx, 1 + jnz .mred_loop + + ; final reduction + mov edi, pR ; result + LD_ADDR ebx, p192r1_data ; addres of the modulus + lea ebx, [ebx+(_prime192r1-p192r1_data)] + CALL_IPPASM sub_192 + + sub edx, eax + cmovz esi, edi + cpy_192 edi, esi + + REST_GPR + ret +ENDFUNC p192r1_mred + +%endif ;; _IPP >= _IPP_P8 + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpp224r1arith_mont_slm.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpp224r1arith_mont_slm.asm new file mode 100644 index 0000000..24879e6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpp224r1arith_mont_slm.asm @@ -0,0 +1,867 @@ +;=============================================================================== +; Copyright (C) 2016 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; P224r1 basic arithmetic function +; +; Content: +; p224r1_add +; p224r1_sub +; p224r1_neg +; p224r1_div_by_2 +; p224r1_mul_mont_slm +; p224r1_sqr_mont_slm +; p224r1_mred +; p224r1_mont_back +; p224r1_select_ap_w7 +; + + + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%if (_IPP >= _IPP_P8) + +segment .text align=IPP_ALIGN_FACTOR + +;; +;; some p224r1 constants +;; +p224r1_data: +_prime224r1 DD 000000001h,000000000h,000000000h,0FFFFFFFFh,0FFFFFFFFh,0FFFFFFFFh,0FFFFFFFFh + +%assign LEN224 (224/32) ; dword's length of operands + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; Ipp32u add_224(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +;; input: edi = r +;; esi = a +;; ebx = b +;; +;; output: eax = carry = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM add_224,PRIVATE + ; r = a+b + mov eax, dword [esi] + add eax, dword [ebx] + mov dword [edi], eax + + mov eax, dword [esi+sizeof(dword)] + adc eax, dword [ebx+sizeof(dword)] + mov dword [edi+sizeof(dword)], eax + + mov eax, dword [esi+sizeof(dword)*2] + adc eax, dword [ebx+sizeof(dword)*2] + mov dword [edi+sizeof(dword)*2], eax + + mov eax, dword [esi+sizeof(dword)*3] + adc eax, dword [ebx+sizeof(dword)*3] + mov dword [edi+sizeof(dword)*3], eax + + mov eax, dword [esi+sizeof(dword)*4] + adc eax, dword [ebx+sizeof(dword)*4] + mov dword [edi+sizeof(dword)*4], eax + + mov eax, dword [esi+sizeof(dword)*5] + adc eax, dword [ebx+sizeof(dword)*5] + mov dword [edi+sizeof(dword)*5], eax + + mov eax, dword [esi+sizeof(dword)*6] + adc eax, dword [ebx+sizeof(dword)*6] + mov dword [edi+sizeof(dword)*6], eax + + mov eax, 0 + adc eax, 0 + ret +ENDFUNC add_224 + +;; +;; Ipp32u sub_224(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +;; input: edi = r +;; esi = a +;; ebx = b +;; +;; output: eax = borrow = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM sub_224,PRIVATE + ; r = a-b + mov eax, dword [esi] + sub eax, dword [ebx] + mov dword [edi], eax + + mov eax, dword [esi+sizeof(dword)] + sbb eax, dword [ebx+sizeof(dword)] + mov dword [edi+sizeof(dword)], eax + + mov eax, dword [esi+sizeof(dword)*2] + sbb eax, dword [ebx+sizeof(dword)*2] + mov dword [edi+sizeof(dword)*2], eax + + mov eax, dword [esi+sizeof(dword)*3] + sbb eax, dword [ebx+sizeof(dword)*3] + mov dword [edi+sizeof(dword)*3], eax + + mov eax, dword [esi+sizeof(dword)*4] + sbb eax, dword [ebx+sizeof(dword)*4] + mov dword [edi+sizeof(dword)*4], eax + + mov eax, dword [esi+sizeof(dword)*5] + sbb eax, dword [ebx+sizeof(dword)*5] + mov dword [edi+sizeof(dword)*5], eax + + mov eax, dword [esi+sizeof(dword)*6] + sbb eax, dword [ebx+sizeof(dword)*6] + mov dword [edi+sizeof(dword)*6], eax + + mov eax, 0 + adc eax, 0 + ret +ENDFUNC sub_224 + +;; +;; Ipp32u shl_224(Ipp32u* r, const Ipp32u* a) +;; +;; input: edi = r +;; esi = a +;; +;; output: eax = extension = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM shl_224,PRIVATE + ; r = a<<1 + movdqu xmm0, oword [esi] + movdqu xmm1, oword [esi+LEN224*sizeof(dword)-sizeof(oword)] + mov eax, dword [esi+(LEN224-1)*sizeof(dword)] + psrldq xmm1, sizeof(dword) + + movdqa xmm2, xmm0 + psllq xmm0, 1 + psrlq xmm2, 63 + movdqa xmm3, xmm1 + psllq xmm1, 1 + psrlq xmm3, 63 + + palignr xmm3, xmm2, sizeof(qword) + pslldq xmm2, sizeof(qword) + + por xmm1, xmm3 + por xmm0, xmm2 + movdqu oword [edi], xmm0 + movq qword [edi+sizeof(oword)], xmm1 + psrldq xmm1, sizeof(qword) + movd dword [edi+sizeof(oword)+sizeof(qword)], xmm1 + + shr eax, 31 + ret +ENDFUNC shl_224 + +;; +;; void shr_224(Ipp32u* r, const Ipp32u* a) +;; +;; input: edi = r +;; esi = a +;; eax = ext +;; output: eax = extension = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM shr_224,PRIVATE + ; r = a>>1 + movdqu xmm0, oword [esi] + movdqu xmm2, oword [esi+LEN224*sizeof(dword)-sizeof(oword)] + movd xmm1, eax + palignr xmm1, xmm2, sizeof(dword) + + movdqa xmm2, xmm0 + psrlq xmm0, 1 + psllq xmm2, 63 + movdqa xmm3, xmm1 + psrlq xmm1, 1 + psllq xmm3, 63 + + movdqa xmm4, xmm3 + palignr xmm3, xmm2, sizeof(qword) + psrldq xmm4, sizeof(qword) + + por xmm0, xmm3 + por xmm1, xmm4 + movdqu oword [edi], xmm0 + movq qword [edi+sizeof(oword)], xmm1 + psrldq xmm1, sizeof(qword) + movd dword [edi+sizeof(oword)+sizeof(qword)], xmm1 + + ret +ENDFUNC shr_224 + +;; +;; void cpy_224(Ipp32u* r, const Ipp32u* a) +;; +%macro cpy_224 2.nolist + %xdefine %%pdst %1 + %xdefine %%psrc %2 + + movdqu xmm0, oword [%%psrc] + movq xmm1, qword [%%psrc+sizeof(oword)] + movd xmm2, dword [%%psrc+sizeof(oword)+sizeof(qword)] + movdqu oword [%%pdst], xmm0 + movq qword [%%pdst+sizeof(oword)], xmm1 + movd dword [%%pdst+sizeof(oword)+sizeof(qword)], xmm2 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p224r1_add(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +align IPP_ALIGN_FACTOR +IPPASM p224r1_add,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +%xdefine pB [ebp + ARG_1 + 2*sizeof(dword)] ; source B address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN224] +%assign _sp_ _buf_+(LEN224)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + mov edi, pR ; pR + mov esi, pA ; pA + mov ebx, pB ; pB + CALL_IPPASM add_224 ; R = A+B + mov edx, eax + + lea edi, [esp+_buf_] ; T + mov esi, pR ; R + LD_ADDR ebx, p224r1_data ; modulus + lea ebx, [ebx+(_prime224r1-p224r1_data)] + CALL_IPPASM sub_224 ; T = R-modulus + + lea esi,[esp+_buf_] + mov edi, pR + sub edx, eax ; R = T<0? R : T + cmovnz esi, edi + cpy_224 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p224r1_add + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p224r1_sub(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +align IPP_ALIGN_FACTOR +IPPASM p224r1_sub,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +%xdefine pB [ebp + ARG_1 + 2*sizeof(dword)] ; source B address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN224] +%assign _sp_ _buf_+(LEN224)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + mov edi, pR ; pR + mov esi, pA ; pA + mov ebx, pB ; pB + CALL_IPPASM sub_224 ; R = A-B + mov edx, eax + + lea edi, [esp+_buf_] ; T + mov esi, pR ; R + LD_ADDR ebx, p224r1_data ; modulus + lea ebx, [ebx+(_prime224r1-p224r1_data)] + CALL_IPPASM add_224 ; T = R+modulus + + lea esi,[esp+_buf_] + mov edi, pR + test edx, edx ; R = T<0? R : T + cmovz esi, edi + cpy_224 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p224r1_sub + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p224r1_neg(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p224r1_neg,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN224] +%assign _sp_ _buf_+(LEN224)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + mov edi, pR ; outpur pR + mov esi, pA ; input pA + + ; r = 0-a + mov eax, 0 + sub eax, dword [esi] + mov dword [edi], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)] + mov dword [edi+sizeof(dword)], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*2] + mov dword [edi+sizeof(dword)*2], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*3] + mov dword [edi+sizeof(dword)*3], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*4] + mov dword [edi+sizeof(dword)*4], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*5] + mov dword [edi+sizeof(dword)*5], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*6] + mov dword [edi+sizeof(dword)*6], eax + sbb edx,edx + + lea edi, [esp+_buf_] ; T + mov esi, pR ; R + LD_ADDR ebx, p224r1_data ; modulus + lea ebx, [ebx+(_prime224r1-p224r1_data)] + CALL_IPPASM add_224 ; T = R+modulus + + lea esi,[esp+_buf_] + mov edi, pR + test edx, edx ; R = T<0? R : T + cmovz esi, edi + cpy_224 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p224r1_neg + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p224r1_mul_by_2(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p224r1_mul_by_2,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN224] +%assign _sp_ _buf_+(LEN224)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + lea edi, [esp+_buf_] ; T + mov esi, pA ; pA + CALL_IPPASM shl_224 ; T = A<<1 + mov edx, eax + + mov esi, edi ; T + mov edi, pR ; R + LD_ADDR ebx, p224r1_data ; modulus + lea ebx, [ebx+(_prime224r1-p224r1_data)] + CALL_IPPASM sub_224 ; R = T-modulus + + sub edx, eax ; R = R<0? T : R + cmovz esi, edi + cpy_224 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p224r1_mul_by_2 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p224r1_mul_by_3(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p224r1_mul_by_3,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _bufT_ 0 ; T buffer[LEN224] +%assign _bufU_ _bufT_+(LEN224)*sizeof(dword) ; U buffer[LEN224] +%assign _mod_ _bufU_+(LEN224)*sizeof(dword) ; modulus address [1] +%assign _sp_ _mod_+sizeof(dword) ; esp [1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + LD_ADDR eax, p224r1_data ; srore modulus address + lea eax, [eax+(_prime224r1-p224r1_data)] + mov dword [esp+_mod_], eax + + lea edi, [esp+_bufT_] ; T + mov esi, pA ; A + CALL_IPPASM shl_224 ; T = A<<1 + mov edx, eax + + mov esi, edi ; T + lea edi, [esp+_bufU_] ; U + mov ebx, [esp+_mod_] ; modulus + CALL_IPPASM sub_224 ; U = T-modulus + + sub edx, eax ; T = U<0? T : U + cmovz esi, edi + cpy_224 edi, esi + + mov esi, edi + mov ebx, pA + CALL_IPPASM add_224 ; T +=A + mov edx, eax + + mov edi, pR ; R + mov ebx, [esp+_mod_] ; modulus + CALL_IPPASM sub_224 ; R = T-modulus + + sub edx, eax ; R = T<0? R : T + cmovz esi, edi + cpy_224 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p224r1_mul_by_3 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p224r1_div_by_2(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p224r1_div_by_2,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN224] +%assign _sp_ _buf_+(LEN224)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + lea edi, [esp+_buf_] ; T + mov esi, pA ; A + LD_ADDR ebx, p224r1_data ; modulus + lea ebx, [ebx+(_prime224r1-p224r1_data)] + CALL_IPPASM add_224 ; R = A+modulus + mov edx, 0 + + mov ecx, dword [esi] ; shifted_data = (a[0]&1)? T : A + and ecx, 1 + cmovnz esi, edi + cmovz eax, edx + mov edi, pR + CALL_IPPASM shr_224 + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p224r1_div_by_2 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p224r1_mul_mont_slm(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +align IPP_ALIGN_FACTOR +IPPASM p224r1_mul_mont_slm,PUBLIC + USES_GPR ebp,ebx,esi,edi + +%xdefine pR [eax + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [eax + ARG_1 + 1*sizeof(dword)] ; source A address +%xdefine pB [eax + ARG_1 + 2*sizeof(dword)] ; source B address +; +; stack layout: +; +%assign _buf_ 0 +%assign _rp_ _buf_+(LEN224+1)*sizeof(dword) ; pR +%assign _ap_ _rp_ +sizeof(dword) ; pA +%assign _bp_ _ap_+sizeof(dword) ; pB +%assign _sp_ _bp_+sizeof(dword) ; esp storage +%assign _ssize_ _sp_+sizeof(dword) ; size allocated stack + + mov eax, esp ; save esp + sub esp, _ssize_ ; allocate stack + and esp, -16 ; provide 16-byte stack alignment + mov dword [esp+_sp_], eax ; store original esp + + ; clear buffer + pxor mm0, mm0 + movq [esp+_buf_], mm0 + movq [esp+_buf_+sizeof(qword)], mm0 + movq [esp+_buf_+sizeof(qword)*2], mm0 + movq [esp+_buf_+sizeof(qword)*3], mm0 + + ; store parameters into the stack + ; note: eax here stores an original esp, so it can be used to reach function parameters + mov edi, pR + mov esi, pA + mov ebp, pB + mov dword [esp+_rp_], edi + mov dword [esp+_ap_], esi + mov dword [esp+_bp_], ebp + + mov edi, LEN224 + + movd mm1, dword [esi+sizeof(dword)] ; pre load a[1], a[2], a[3], a[4] + movd mm2, dword [esi+sizeof(dword)*2] + movd mm3, dword [esi+sizeof(dword)*3] + movd mm4, dword [esi+sizeof(dword)*4] + +align IPP_ALIGN_FACTOR +.mmul_loop: +; +; i-st pass +; modulus = 2^224 -2^96 +1 +; [7] [3] [0] +; m0 = 1 +; + movd mm7, edi ; save pass counter + + mov edx, dword [ebp] ; b = b[i] + mov eax, dword [esi] ; a[0] + movd mm0, edx + add ebp, sizeof(dword) + mov dword [esp+_bp_], ebp + + pmuludq mm1, mm0 ; a[1]*b[i] + pmuludq mm2, mm0 ; a[2]*b[i] + + mul edx ; (E:u) = (edx:eax) = a[0]*b[i]+buf[0] + add eax, dword [esp+_buf_] + adc edx, 0 + + pmuludq mm3, mm0 ; a[3]*b[i] + pmuludq mm4, mm0 ; a[4]*b[i] + +; multiplication round 1 - round 4 + movd ecx, mm1 ; p = a[1]*b[i] + E + psrlq mm1, 32 + add ecx, edx + movd edx, mm1 + adc edx, 0 + add ecx, dword [esp+_buf_+sizeof(dword)*1] + movd mm1, dword [esi+sizeof(dword)*5] + adc edx, 0 + + movd ebx, mm2 ; p = a[2]*b[i] + E + psrlq mm2, 32 + add ebx, edx + movd edx, mm2 + adc edx, 0 + add ebx, dword [esp+_buf_+sizeof(dword)*2] + movd mm2, dword [esi+sizeof(dword)*6] + adc edx, 0 + + pmuludq mm1, mm0 ; a[5]*b[i] + pmuludq mm2, mm0 ; a[6]*b[i] + + movd ebp, mm3 ; p = a[3]*b[i] + E + psrlq mm3, 32 + add ebp, edx + movd edx, mm3 + adc edx, 0 + add ebp, dword [esp+_buf_+sizeof(dword)*3] + adc edx, 0 + + movd edi, mm4 ; p = a[4]*b[i] + E + psrlq mm4, 32 + add edi, edx + movd edx, mm4 + adc edx, 0 + add edi, dword [esp+_buf_+sizeof(dword)*4] + adc edx, 0 + +;;; and reduction ;;; + neg eax ; u0 + + adc ecx, 0 ; +cf + mov dword [esp+_buf_+sizeof(dword)*0], ecx + adc ebx, 0 ; +cf + mov dword [esp+_buf_+sizeof(dword)*1], ebx + + mov ecx, eax ; save u0 + + sbb eax, 0 ; u0-cf + sub ebp, eax ; -u0+cf + mov dword [esp+_buf_+sizeof(dword)*2], ebp + + mov eax, ecx ; restore u0 + mov ebp, 0 + + sbb edi, 0 ; -bf + mov dword [esp+_buf_+sizeof(dword)*3], edi + adc ebp, 0 ; save bf + +; multiplication round 5 - round 6 + movd ecx, mm1 ; p = a[5]*b[i] + E + psrlq mm1, 32 + add ecx, edx + movd edx, mm1 + adc edx, 0 + add ecx, dword [esp+_buf_+sizeof(dword)*5] + adc edx, 0 + + movd ebx, mm2 ; p = a[6]*b[i] + E + psrlq mm2, 32 + add ebx, edx + movd edx, mm2 + adc edx, 0 + add ebx, dword [esp+_buf_+sizeof(dword)*6] + adc edx, 0 + +;;; and reduction ;;; + sub ecx, ebp ; -bf + mov dword [esp+_buf_+sizeof(dword)*4], ecx + sbb ebx, 0 ; -bf + mov dword [esp+_buf_+sizeof(dword)*5], ebx + +; last multiplication round 7 + movd edi, mm7 ; restore pass counter + + sbb eax, 0 ; u-bf + mov ebx, 0 + + add edx, dword [esp+_buf_+sizeof(dword)*7] + adc ebx, 0 + add edx, eax + mov dword [esp+_buf_+sizeof(dword)*6], edx + adc ebx, 0 + mov dword [esp+_buf_+sizeof(dword)*7], ebx + + sub edi, 1 + movd mm1, dword [esi+sizeof(dword)] ; speculative load a[1], a[2], a[3], a[4] + movd mm2, dword [esi+sizeof(dword)*2] + movd mm3, dword [esi+sizeof(dword)*3] + movd mm4, dword [esi+sizeof(dword)*4] + jz .exit_mmul_loop + + mov ebp, dword [esp+_bp_] ; restore pB + jmp .mmul_loop + +.exit_mmul_loop: + emms + +; final reduction + mov edi, [esp+_rp_] ; result + lea esi, [esp+_buf_] ; buffer + LD_ADDR ebx, p224r1_data ; modulus + lea ebx, [ebx+(_prime224r1-p224r1_data)] + CALL_IPPASM sub_224 + mov edx, dword [esp+_buf_+LEN224*sizeof(dword)] + sub edx, eax + +; copy + cmovz esi, edi + cpy_224 edi, esi + + mov esp, [esp+_sp_] ; release stack + REST_GPR + ret +ENDFUNC p224r1_mul_mont_slm + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p224r1_sqr_mont_slm(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p224r1_sqr_mont_slm,PUBLIC + USES_GPR esi,edi + +%xdefine pR [esp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [esp + ARG_1 + 1*sizeof(dword)] ; source A address + + ;; use p224r1_mul_mont_slm to compute sqr + mov esi, pA + mov edi, pR + push esi + push esi + push edi + CALL_IPPASM p224r1_mul_mont_slm,PUBLIC + add esp, sizeof(dword)*3 + REST_GPR + ret +ENDFUNC p224r1_sqr_mont_slm + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p224r1_mred(Ipp32u* r, Ipp32u* prod) +;; +; modulus = 2^224 -2^96 +1 +; [7] [3] [0] +; m0 = -1 +; +align IPP_ALIGN_FACTOR +IPPASM p224r1_mred,PUBLIC + USES_GPR ebx,esi,edi + +%xdefine pR [esp + ARG_1 + 0*sizeof(dword)] ; reduction address +%xdefine pA [esp + ARG_1 + 1*sizeof(dword)] ; source product address + + ; get parameters: + mov esi, pA + + mov ecx, LEN224 + xor edx, edx +align IPP_ALIGN_FACTOR +.mred_loop: + mov eax, dword [esi] + neg eax + + mov ebx, 0 + mov dword [esi], ebx + + mov ebx, dword [esi+sizeof(dword)] + adc ebx, 0 + mov dword [esi+sizeof(dword)], ebx + + mov ebx, dword [esi+sizeof(dword)*2] + adc ebx, 0 + mov dword [esi+sizeof(dword)*2], ebx + + push eax + mov ebx, dword [esi+sizeof(dword)*3] + sbb eax, 0 + sub ebx, eax + mov dword [esi+sizeof(dword)*3], ebx + pop eax + + mov ebx, dword [esi+sizeof(dword)*4] + sbb ebx, 0 + mov dword [esi+sizeof(dword)*4], ebx + + mov ebx, dword [esi+sizeof(dword)*5] + sbb ebx, 0 + mov dword [esi+sizeof(dword)*5], ebx + + mov ebx, dword [esi+sizeof(dword)*6] + sbb ebx, 0 + mov dword [esi+sizeof(dword)*6], ebx + + mov ebx, dword [esi+sizeof(dword)*7] + sbb eax, 0 + add eax, edx + mov edx, 0 + adc edx, 0 + add ebx, eax + mov dword [esi+sizeof(dword)*7], ebx + adc edx, 0 + + lea esi, [esi+sizeof(dword)] + sub ecx, 1 + jnz .mred_loop + + ; final reduction + mov edi, pR ; result + LD_ADDR ebx, p224r1_data ; addres of the modulus + lea ebx, [ebx+(_prime224r1-p224r1_data)] + CALL_IPPASM sub_224 + + sub edx, eax + cmovz esi, edi + cpy_224 edi, esi + + REST_GPR + ret +ENDFUNC p224r1_mred + +%endif ;; _IPP >= _IPP_P8 + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpp256r1arith_mont_slm.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpp256r1arith_mont_slm.asm new file mode 100644 index 0000000..b21452b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpp256r1arith_mont_slm.asm @@ -0,0 +1,874 @@ +;=============================================================================== +; Copyright (C) 2016 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; P256r1 basic arithmetic function +; +; Content: +; p256r1_add +; p256r1_sub +; p256r1_neg +; p256r1_div_by_2 +; p256r1_mul_mont_slm +; p256r1_sqr_mont_slm +; p256r1_mred +; p256r1_mont_back +; p256r1_select_ap_w7 +; + + + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%if (_IPP >= _IPP_P8) + +segment .text align=IPP_ALIGN_FACTOR + +;; +;; some p256r1 constants +;; +p256r1_data: +_prime256r1 DD 0FFFFFFFFh,0FFFFFFFFh,0FFFFFFFFh,000000000h,000000000h,000000000h,000000001h, 0FFFFFFFFh + +%assign LEN256 (256/32) ; dword's length of operands + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; Ipp32u add_256(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +;; input: edi = r +;; esi = a +;; ebx = b +;; +;; output: eax = carry = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM add_256,PRIVATE + ; r = a+b + mov eax, dword [esi] + add eax, dword [ebx] + mov dword [edi], eax + + mov eax, dword [esi+sizeof(dword)] + adc eax, dword [ebx+sizeof(dword)] + mov dword [edi+sizeof(dword)], eax + + mov eax, dword [esi+sizeof(dword)*2] + adc eax, dword [ebx+sizeof(dword)*2] + mov dword [edi+sizeof(dword)*2], eax + + mov eax, dword [esi+sizeof(dword)*3] + adc eax, dword [ebx+sizeof(dword)*3] + mov dword [edi+sizeof(dword)*3], eax + + mov eax, dword [esi+sizeof(dword)*4] + adc eax, dword [ebx+sizeof(dword)*4] + mov dword [edi+sizeof(dword)*4], eax + + mov eax, dword [esi+sizeof(dword)*5] + adc eax, dword [ebx+sizeof(dword)*5] + mov dword [edi+sizeof(dword)*5], eax + + mov eax, dword [esi+sizeof(dword)*6] + adc eax, dword [ebx+sizeof(dword)*6] + mov dword [edi+sizeof(dword)*6], eax + + mov eax, dword [esi+sizeof(dword)*7] + adc eax, dword [ebx+sizeof(dword)*7] + mov dword [edi+sizeof(dword)*7], eax + + mov eax, 0 + adc eax, 0 + ret +ENDFUNC add_256 + +;; +;; Ipp32u sub_256(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +;; input: edi = r +;; esi = a +;; ebx = b +;; +;; output: eax = borrow = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM sub_256,PRIVATE + ; r = a-b + mov eax, dword [esi] + sub eax, dword [ebx] + mov dword [edi], eax + + mov eax, dword [esi+sizeof(dword)] + sbb eax, dword [ebx+sizeof(dword)] + mov dword [edi+sizeof(dword)], eax + + mov eax, dword [esi+sizeof(dword)*2] + sbb eax, dword [ebx+sizeof(dword)*2] + mov dword [edi+sizeof(dword)*2], eax + + mov eax, dword [esi+sizeof(dword)*3] + sbb eax, dword [ebx+sizeof(dword)*3] + mov dword [edi+sizeof(dword)*3], eax + + mov eax, dword [esi+sizeof(dword)*4] + sbb eax, dword [ebx+sizeof(dword)*4] + mov dword [edi+sizeof(dword)*4], eax + + mov eax, dword [esi+sizeof(dword)*5] + sbb eax, dword [ebx+sizeof(dword)*5] + mov dword [edi+sizeof(dword)*5], eax + + mov eax, dword [esi+sizeof(dword)*6] + sbb eax, dword [ebx+sizeof(dword)*6] + mov dword [edi+sizeof(dword)*6], eax + + mov eax, dword [esi+sizeof(dword)*7] + sbb eax, dword [ebx+sizeof(dword)*7] + mov dword [edi+sizeof(dword)*7], eax + + mov eax, 0 + adc eax, 0 + ret +ENDFUNC sub_256 + +;; +;; Ipp32u shl_256(Ipp32u* r, const Ipp32u* a) +;; +;; input: edi = r +;; esi = a +;; +;; output: eax = extension = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM shl_256,PRIVATE + ; r = a<<1 + movdqu xmm0, oword [esi] + movdqu xmm1, oword [esi+sizeof(oword)] + mov eax, dword [esi+(LEN256-1)*sizeof(dword)] + + movdqa xmm2, xmm0 + psllq xmm0, 1 + psrlq xmm2, 63 + movdqa xmm3, xmm1 + psllq xmm1, 1 + psrlq xmm3, 63 + + palignr xmm3, xmm2, sizeof(qword) + pslldq xmm2, sizeof(qword) + + por xmm1, xmm3 + por xmm0, xmm2 + movdqu oword [edi], xmm0 + movdqu oword [edi+sizeof(oword)], xmm1 + + shr eax, 31 + ret +ENDFUNC shl_256 + +;; +;; void shr_256(Ipp32u* r, const Ipp32u* a) +;; +;; input: edi = r +;; esi = a +;; eax = ext +;; output: eax = extension = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM shr_256,PRIVATE + ; r = a>>1 + movd xmm4, eax + movdqu xmm0, oword [esi] + movdqu xmm1, oword [esi+sizeof(oword)] + + psllq xmm4, 63 + movdqa xmm2, xmm0 + psrlq xmm0, 1 + psllq xmm2, 63 + movdqa xmm3, xmm1 + psrlq xmm1, 1 + psllq xmm3, 63 + + palignr xmm4, xmm3, sizeof(qword) + palignr xmm3, xmm2, sizeof(qword) + + por xmm1, xmm4 + por xmm0, xmm3 + movdqu oword [edi], xmm0 + movdqu oword [edi+sizeof(oword)], xmm1 + + ret +ENDFUNC shr_256 + +;; +;; void cpy_256(Ipp32u* r, const Ipp32u* a) +;; +%macro cpy_256 2.nolist + %xdefine %%pdst %1 + %xdefine %%psrc %2 + + movdqu xmm0, oword [%%psrc] + movdqu xmm1, oword [%%psrc+sizeof(oword)] + movdqu oword [%%pdst], xmm0 + movdqu oword [%%pdst+sizeof(oword)], xmm1 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p256r1_add(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +align IPP_ALIGN_FACTOR +IPPASM p256r1_add,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +%xdefine pB [ebp + ARG_1 + 2*sizeof(dword)] ; source B address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN256] +%assign _sp_ _buf_+(LEN256)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + mov edi, pR ; pR + mov esi, pA ; pA + mov ebx, pB ; pB + CALL_IPPASM add_256 ; R = A+B + mov edx, eax + + lea edi, [esp+_buf_] ; T + mov esi, pR ; R + LD_ADDR ebx, p256r1_data ; modulus + lea ebx, [ebx+(_prime256r1-p256r1_data)] + CALL_IPPASM sub_256 ; T = R-modulus + + lea esi,[esp+_buf_] + mov edi, pR + sub edx, eax ; R = T<0? R : T + cmovnz esi, edi + cpy_256 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p256r1_add + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p256r1_sub(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +align IPP_ALIGN_FACTOR +IPPASM p256r1_sub,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +%xdefine pB [ebp + ARG_1 + 2*sizeof(dword)] ; source B address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN256] +%assign _sp_ _buf_+(LEN256)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + mov edi, pR ; pR + mov esi, pA ; pA + mov ebx, pB ; pB + CALL_IPPASM sub_256 ; R = A-B + mov edx, eax + + lea edi, [esp+_buf_] ; T + mov esi, pR ; R + LD_ADDR ebx, p256r1_data ; modulus + lea ebx, [ebx+(_prime256r1-p256r1_data)] + CALL_IPPASM add_256 ; T = R+modulus + + lea esi,[esp+_buf_] + mov edi, pR + test edx, edx ; R = T<0? R : T + cmovz esi, edi + cpy_256 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p256r1_sub + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p256r1_neg(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p256r1_neg,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN256] +%assign _sp_ _buf_+(LEN256)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + mov edi, pR ; outpur pR + mov esi, pA ; input pA + + ; r = 0-a + mov eax, 0 + sub eax, dword [esi] + mov dword [edi], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)] + mov dword [edi+sizeof(dword)], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*2] + mov dword [edi+sizeof(dword)*2], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*3] + mov dword [edi+sizeof(dword)*3], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*4] + mov dword [edi+sizeof(dword)*4], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*5] + mov dword [edi+sizeof(dword)*5], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*6] + mov dword [edi+sizeof(dword)*6], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*7] + mov dword [edi+sizeof(dword)*7], eax + sbb edx,edx + + lea edi, [esp+_buf_] ; T + mov esi, pR ; R + LD_ADDR ebx, p256r1_data ; modulus + lea ebx, [ebx+(_prime256r1-p256r1_data)] + CALL_IPPASM add_256 ; T = R+modulus + + lea esi,[esp+_buf_] + mov edi, pR + test edx, edx ; R = T<0? R : T + cmovz esi, edi + cpy_256 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p256r1_neg + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p256r1_mul_by_2(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p256r1_mul_by_2,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN256] +%assign _sp_ _buf_+(LEN256)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + lea edi, [esp+_buf_] ; T + mov esi, pA ; pA + CALL_IPPASM shl_256 ; T = A<<1 + mov edx, eax + + mov esi, edi ; T + mov edi, pR ; R + LD_ADDR ebx, p256r1_data ; modulus + lea ebx, [ebx+(_prime256r1-p256r1_data)] + CALL_IPPASM sub_256 ; R = T-modulus + + sub edx, eax ; R = R<0? T : R + cmovz esi, edi + cpy_256 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p256r1_mul_by_2 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p256r1_mul_by_3(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p256r1_mul_by_3,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _bufT_ 0 ; T buffer[LEN256] +%assign _bufU_ _bufT_+(LEN256)*sizeof(dword) ; U buffer[LEN256] +%assign _mod_ _bufU_+(LEN256)*sizeof(dword) ; modulus address [1] +%assign _sp_ _mod_+sizeof(dword) ; esp [1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + LD_ADDR eax, p256r1_data ; srore modulus address + lea eax, [eax+(_prime256r1-p256r1_data)] + mov dword [esp+_mod_], eax + + lea edi, [esp+_bufT_] ; T + mov esi, pA ; A + CALL_IPPASM shl_256 ; T = A<<1 + mov edx, eax + + mov esi, edi ; T + lea edi, [esp+_bufU_] ; U + mov ebx, [esp+_mod_] ; modulus + CALL_IPPASM sub_256 ; U = T-modulus + + sub edx, eax ; T = U<0? T : U + cmovz esi, edi + cpy_256 edi, esi + + mov esi, edi + mov ebx, pA + CALL_IPPASM add_256 ; T +=A + mov edx, eax + + mov edi, pR ; R + mov ebx, [esp+_mod_] ; modulus + CALL_IPPASM sub_256 ; R = T-modulus + + sub edx, eax ; R = T<0? R : T + cmovz esi, edi + cpy_256 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p256r1_mul_by_3 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p256r1_div_by_2(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p256r1_div_by_2,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN256] +%assign _sp_ _buf_+(LEN256)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + lea edi, [esp+_buf_] ; T + mov esi, pA ; A + LD_ADDR ebx, p256r1_data ; modulus + lea ebx, [ebx+(_prime256r1-p256r1_data)] + CALL_IPPASM add_256 ; R = A+modulus + mov edx, 0 + + mov ecx, dword [esi] ; shifted_data = (a[0]&1)? T : A + and ecx, 1 + cmovnz esi, edi + cmovz eax, edx + mov edi, pR + CALL_IPPASM shr_256 + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p256r1_div_by_2 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p256r1_mul_mont_slm(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +align IPP_ALIGN_FACTOR +IPPASM p256r1_mul_mont_slm,PUBLIC + USES_GPR ebp,ebx,esi,edi + +%xdefine pR [eax + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [eax + ARG_1 + 1*sizeof(dword)] ; source A address +%xdefine pB [eax + ARG_1 + 2*sizeof(dword)] ; source B address +; +; stack layout: +; +%assign _buf_ 0 +%assign _rp_ _buf_+(LEN256+1)*sizeof(dword) ; pR +%assign _ap_ _rp_ +sizeof(dword) ; pA +%assign _bp_ _ap_+sizeof(dword) ; pB +%assign _sp_ _bp_+sizeof(dword) ; esp storage +%assign _ssize_ _sp_+sizeof(dword) ; size allocated stack + + mov eax, esp ; save esp + sub esp, _ssize_ ; allocate stack + and esp, -16 ; provide 16-byte stack alignment + mov dword [esp+_sp_], eax ; store original esp + + ; clear buffer + pxor mm0, mm0 + movq qword [esp+_buf_], mm0 + movq qword [esp+_buf_+sizeof(qword)], mm0 + movq qword [esp+_buf_+sizeof(qword)*2], mm0 + movq qword [esp+_buf_+sizeof(qword)*3], mm0 + movd dword [esp+_buf_+sizeof(qword)*4], mm0 + + ; store parameters into the stack + ; note: eax here stores an original esp, so it can be used to reach function parameters + mov edi, pR + mov esi, pA + mov ebp, pB + mov dword [esp+_rp_], edi + mov dword [esp+_ap_], esi + mov dword [esp+_bp_], ebp + + mov edi, LEN256 + + movd mm1, dword [esi+sizeof(dword)] ; pre load a[1], a[2], a[3], a[4] + movd mm2, dword [esi+sizeof(dword)*2] + movd mm3, dword [esi+sizeof(dword)*3] + movd mm4, dword [esi+sizeof(dword)*4] + +align IPP_ALIGN_FACTOR +.mmul_loop: +; +; i-st pass +; modulus = 2^256 -2^224 +2^192 +2^96 -1 +; [8] [7] [6] [3] [0] +; m0 = 1 +; + movd mm7, edi ; save pass counter + + mov edx, dword [ebp] ; b = b[i] + mov eax, dword [esi] ; a[0] + movd mm0, edx + add ebp, sizeof(dword) + mov dword [esp+_bp_], ebp + + pmuludq mm1, mm0 ; a[1]*b[i] + pmuludq mm2, mm0 ; a[2]*b[i] + + mul edx ; (E:u) = (edx:eax) = a[0]*b[i]+buf[0] + add eax, dword [esp+_buf_] + adc edx, 0 + + pmuludq mm3, mm0 ; a[3]*b[i] + pmuludq mm4, mm0 ; a[4]*b[i] + +; multiplication round 1 - round 4 + movd ecx, mm1 ; p = a[1]*b[i] + E + psrlq mm1, 32 + add ecx, edx + movd edx, mm1 + adc edx, 0 + add ecx, dword [esp+_buf_+sizeof(dword)*1] + movd mm1, dword [esi+sizeof(dword)*5] + adc edx, 0 + + movd ebx, mm2 ; p = a[2]*b[i] + E + psrlq mm2, 32 + add ebx, edx + movd edx, mm2 + adc edx, 0 + add ebx, dword [esp+_buf_+sizeof(dword)*2] + movd mm2, dword [esi+sizeof(dword)*6] + adc edx, 0 + + pmuludq mm1, mm0 ; a[5]*b[i] + pmuludq mm2, mm0 ; a[6]*b[i] + + movd ebp, mm3 ; p = a[3]*b[i] + E + psrlq mm3, 32 + add ebp, edx + movd edx, mm3 + adc edx, 0 + add ebp, dword [esp+_buf_+sizeof(dword)*3] + movd mm3, dword [esi+sizeof(dword)*7] + adc edx, 0 + + movd edi, mm4 ; p = a[4]*b[i] + E + psrlq mm4, 32 + add edi, edx + movd edx, mm4 + adc edx, 0 + add edi, dword [esp+_buf_+sizeof(dword)*4] + adc edx, 0 + + pmuludq mm3, mm0 ; a[7]*b[i] + +;;; and reduction ;;; + ; eax =u0 + + mov dword [esp+_buf_+sizeof(dword)*0], ecx ; copy + mov dword [esp+_buf_+sizeof(dword)*1], ebx ; copy + + add ebp, eax ; +u0 + mov dword [esp+_buf_+sizeof(dword)*2], ebp + + adc edi, 0 ; +cf + mov dword [esp+_buf_+sizeof(dword)*3], edi + mov edi, 0 ; save edi = cf + adc edi, 0 + +; multiplication round 5 - round 7 + movd ecx, mm1 ; p = a[5]*b[i] + E + psrlq mm1, 32 + add ecx, edx + movd edx, mm1 + adc edx, 0 + add ecx, dword [esp+_buf_+sizeof(dword)*5] + adc edx, 0 + + movd ebx, mm2 ; p = a[6]*b[i] + E + psrlq mm2, 32 + add ebx, edx + movd edx, mm2 + adc edx, 0 + add ebx, dword [esp+_buf_+sizeof(dword)*6] + adc edx, 0 + + movd ebp, mm3 ; p = a[7]*b[i] + E + psrlq mm3, 32 + add ebp, edx + movd edx, mm3 + adc edx, 0 + add ebp, dword [esp+_buf_+sizeof(dword)*7] + adc edx, 0 + +;;; and reduction ;;; + add ecx, edi ; +cf + mov dword [esp+_buf_+sizeof(dword)*4], ecx + adc ebx, eax ; +u0+cf + mov dword [esp+_buf_+sizeof(dword)*5], ebx + mov ecx, eax + sbb eax, 0 + sub ebp, eax ; -u0+cf + mov dword [esp+_buf_+sizeof(dword)*6], ebp + +; last multiplication round 8 + movd edi, mm7 ; restore pass counter + + sbb ecx, 0 ; u-bf + mov ebx, 0 + + add edx, dword [esp+_buf_+sizeof(dword)*8] + adc ebx, 0 + add edx, ecx + mov dword [esp+_buf_+sizeof(dword)*7], edx + adc ebx, 0 + mov dword [esp+_buf_+sizeof(dword)*8], ebx + + sub edi, 1 + movd mm1, dword [esi+sizeof(dword)] ; speculative load a[1], a[2], a[3], a[4] + movd mm2, dword [esi+sizeof(dword)*2] + movd mm3, dword [esi+sizeof(dword)*3] + movd mm4, dword [esi+sizeof(dword)*4] + jz .exit_mmul_loop + + mov ebp, dword [esp+_bp_] ; restore pB + jmp .mmul_loop + +.exit_mmul_loop: + emms + +; final reduction + mov edi, [esp+_rp_] ; result + lea esi, [esp+_buf_] ; buffer + LD_ADDR ebx, p256r1_data ; modulus + lea ebx, [ebx+(_prime256r1-p256r1_data)] + CALL_IPPASM sub_256 + mov edx, dword [esp+_buf_+LEN256*sizeof(dword)] + sub edx, eax + +; copy + cmovz esi, edi + cpy_256 edi, esi + + mov esp, [esp+_sp_] ; release stack + REST_GPR + ret +ENDFUNC p256r1_mul_mont_slm + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p256r1_sqr_mont_slm(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p256r1_sqr_mont_slm,PUBLIC + USES_GPR esi,edi + +%xdefine pR [esp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [esp + ARG_1 + 1*sizeof(dword)] ; source A address + + ;; use p256r1_mul_mont_slm to compute sqr + mov esi, pA + mov edi, pR + push esi + push esi + push edi + CALL_IPPASM p256r1_mul_mont_slm,PUBLIC + add esp, sizeof(dword)*3 + REST_GPR + ret +ENDFUNC p256r1_sqr_mont_slm + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p256r1_mred(Ipp32u* r, Ipp32u* prod) +;; +; modulus = 2^256 -2^224 +2^192 +2^96 -1 +; [8] [7] [6] [3] [0] +; m0 = 1 +; +align IPP_ALIGN_FACTOR +IPPASM p256r1_mred,PUBLIC + USES_GPR ebx,esi,edi + +%xdefine pR [esp + ARG_1 + 0*sizeof(dword)] ; reduction address +%xdefine pA [esp + ARG_1 + 1*sizeof(dword)] ; source product address + + ; get parameters: + mov esi, pA + + mov ecx, LEN256 + xor edx, edx +align IPP_ALIGN_FACTOR +.mred_loop: + mov eax, dword [esi] + + mov ebx, 0 + mov dword [esi], ebx + + mov ebx, dword [esi+sizeof(dword)*3] + add ebx, eax + mov dword [esi+sizeof(dword)*3], ebx + + mov ebx, dword [esi+sizeof(dword)*4] + adc ebx, 0 + mov dword [esi+sizeof(dword)*4], ebx + + mov ebx, dword [esi+sizeof(dword)*5] + adc ebx, 0 + mov dword [esi+sizeof(dword)*5], ebx + + mov ebx, dword [esi+sizeof(dword)*6] + adc ebx, eax + mov dword [esi+sizeof(dword)*6], ebx + + mov ebx, dword [esi+sizeof(dword)*7] + push eax + sbb eax, 0 + sub ebx, eax + mov dword [esi+sizeof(dword)*7], ebx + pop eax + + mov ebx, dword [esi+sizeof(dword)*8] + sbb eax, 0 + add eax, edx + mov edx, 0 + adc edx, 0 + add ebx, eax + mov dword [esi+sizeof(dword)*8], ebx + adc edx, 0 + + lea esi, [esi+sizeof(dword)] + sub ecx, 1 + jnz .mred_loop + + ; final reduction + mov edi, pR ; result + LD_ADDR ebx, p256r1_data ; addres of the modulus + lea ebx, [ebx+(_prime256r1-p256r1_data)] + CALL_IPPASM sub_256 + + sub edx, eax + cmovz esi, edi + cpy_256 edi, esi + + REST_GPR + ret +ENDFUNC p256r1_mred + +%endif ;; _IPP >= _IPP_P8 + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpp384r1arith_mont_slm.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpp384r1arith_mont_slm.asm new file mode 100644 index 0000000..fe2228d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpp384r1arith_mont_slm.asm @@ -0,0 +1,1030 @@ +;=============================================================================== +; Copyright (C) 2016 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; P384r1 basic arithmetic function +; +; Content: +; p384r1_add +; p384r1_sub +; p384r1_neg +; p384r1_div_by_2 +; p384r1_mul_mont_slm +; p384r1_sqr_mont_slm +; p384r1_mred +; p384r1_select_ap_w5 +; + + + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" +%include "pcpvariant.inc" + +%if (_IPP >= _IPP_P8) + +segment .text align=IPP_ALIGN_FACTOR + +;; +;; some p384r1 constants +;; +p384r1_data: +_prime384r1 DD 0FFFFFFFFh,000000000h,000000000h,0FFFFFFFFh,0FFFFFFFEh,0FFFFFFFFh + DD 0FFFFFFFFh,0FFFFFFFFh,0FFFFFFFFh,0FFFFFFFFh,0FFFFFFFFh,0FFFFFFFFh + +%assign LEN384 (384/32) ; dword's length of operands + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; Ipp32u add_384(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +;; input: edi = r +;; esi = a +;; ebx = b +;; +;; output: eax = carry = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM add_384,PRIVATE + ; r = a+b + mov eax, dword [esi] + add eax, dword [ebx] + mov dword [edi], eax + + mov eax, dword [esi+sizeof(dword)] + adc eax, dword [ebx+sizeof(dword)] + mov dword [edi+sizeof(dword)], eax + + mov eax, dword [esi+sizeof(dword)*2] + adc eax, dword [ebx+sizeof(dword)*2] + mov dword [edi+sizeof(dword)*2], eax + + mov eax, dword [esi+sizeof(dword)*3] + adc eax, dword [ebx+sizeof(dword)*3] + mov dword [edi+sizeof(dword)*3], eax + + mov eax, dword [esi+sizeof(dword)*4] + adc eax, dword [ebx+sizeof(dword)*4] + mov dword [edi+sizeof(dword)*4], eax + + mov eax, dword [esi+sizeof(dword)*5] + adc eax, dword [ebx+sizeof(dword)*5] + mov dword [edi+sizeof(dword)*5], eax + + mov eax, dword [esi+sizeof(dword)*6] + adc eax, dword [ebx+sizeof(dword)*6] + mov dword [edi+sizeof(dword)*6], eax + + mov eax, dword [esi+sizeof(dword)*7] + adc eax, dword [ebx+sizeof(dword)*7] + mov dword [edi+sizeof(dword)*7], eax + + mov eax, dword [esi+sizeof(dword)*8] + adc eax, dword [ebx+sizeof(dword)*8] + mov dword [edi+sizeof(dword)*8], eax + + mov eax, dword [esi+sizeof(dword)*9] + adc eax, dword [ebx+sizeof(dword)*9] + mov dword [edi+sizeof(dword)*9], eax + + mov eax, dword [esi+sizeof(dword)*10] + adc eax, dword [ebx+sizeof(dword)*10] + mov dword [edi+sizeof(dword)*10], eax + + mov eax, dword [esi+sizeof(dword)*11] + adc eax, dword [ebx+sizeof(dword)*11] + mov dword [edi+sizeof(dword)*11], eax + mov eax, 0 + adc eax, 0 + ret +ENDFUNC add_384 + +;; +;; Ipp32u sub_384(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +;; input: edi = r +;; esi = a +;; ebx = b +;; +;; output: eax = borrow = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM sub_384,PRIVATE + ; r = a-b + mov eax, dword [esi] + sub eax, dword [ebx] + mov dword [edi], eax + + mov eax, dword [esi+sizeof(dword)] + sbb eax, dword [ebx+sizeof(dword)] + mov dword [edi+sizeof(dword)], eax + + mov eax, dword [esi+sizeof(dword)*2] + sbb eax, dword [ebx+sizeof(dword)*2] + mov dword [edi+sizeof(dword)*2], eax + + mov eax, dword [esi+sizeof(dword)*3] + sbb eax, dword [ebx+sizeof(dword)*3] + mov dword [edi+sizeof(dword)*3], eax + + mov eax, dword [esi+sizeof(dword)*4] + sbb eax, dword [ebx+sizeof(dword)*4] + mov dword [edi+sizeof(dword)*4], eax + + mov eax, dword [esi+sizeof(dword)*5] + sbb eax, dword [ebx+sizeof(dword)*5] + mov dword [edi+sizeof(dword)*5], eax + + mov eax, dword [esi+sizeof(dword)*6] + sbb eax, dword [ebx+sizeof(dword)*6] + mov dword [edi+sizeof(dword)*6], eax + + mov eax, dword [esi+sizeof(dword)*7] + sbb eax, dword [ebx+sizeof(dword)*7] + mov dword [edi+sizeof(dword)*7], eax + + mov eax, dword [esi+sizeof(dword)*8] + sbb eax, dword [ebx+sizeof(dword)*8] + mov dword [edi+sizeof(dword)*8], eax + + mov eax, dword [esi+sizeof(dword)*9] + sbb eax, dword [ebx+sizeof(dword)*9] + mov dword [edi+sizeof(dword)*9], eax + + mov eax, dword [esi+sizeof(dword)*10] + sbb eax, dword [ebx+sizeof(dword)*10] + mov dword [edi+sizeof(dword)*10], eax + + mov eax, dword [esi+sizeof(dword)*11] + sbb eax, dword [ebx+sizeof(dword)*11] + mov dword [edi+sizeof(dword)*11], eax + mov eax, 0 + adc eax, 0 + ret +ENDFUNC sub_384 + +;; +;; Ipp32u shl_384(Ipp32u* r, const Ipp32u* a) +;; +;; input: edi = r +;; esi = a +;; +;; output: eax = extension = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM shl_384,PRIVATE + mov eax, dword [esi+(LEN384-1)*sizeof(dword)] + ; r = a<<1 + movdqu xmm3, oword [esi+sizeof(oword)*2] + movdqu xmm2, oword [esi+sizeof(oword)] + movdqu xmm1, oword [esi] + + movdqa xmm4, xmm3 + palignr xmm4, xmm2, sizeof(qword) + psllq xmm3, 1 + psrlq xmm4, 63 + por xmm3, xmm4 + movdqu oword [edi+sizeof(oword)*2], xmm3 + + movdqa xmm4, xmm2 + palignr xmm4, xmm1, sizeof(qword) + psllq xmm2, 1 + psrlq xmm4, 63 + por xmm2, xmm4 + movdqu oword [edi+sizeof(oword)], xmm2 + + movdqa xmm4, xmm1 + pslldq xmm4, sizeof(qword) + psllq xmm1, 1 + psrlq xmm4, 63 + por xmm1, xmm4 + movdqu oword [edi], xmm1 + + shr eax, 31 + ret +ENDFUNC shl_384 + +;; +;; void shr_384(Ipp32u* r, const Ipp32u* a) +;; +;; input: edi = r +;; esi = a +;; eax = ext +;; output: eax = extension = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM shr_384,PRIVATE + ; r = a>>1 + movdqu xmm3, oword [esi] + movdqu xmm2, oword [esi+sizeof(oword)] + movdqu xmm1, oword [esi+sizeof(oword)*2] + + movdqa xmm4, xmm2 + palignr xmm4, xmm3, sizeof(qword) + psrlq xmm3, 1 + psllq xmm4, 63 + por xmm3, xmm4 + movdqu oword [edi], xmm3 + + movdqa xmm4, xmm1 + palignr xmm4, xmm2, sizeof(qword) + psrlq xmm2, 1 + psllq xmm4, 63 + por xmm2, xmm4 + movdqu oword [edi+sizeof(oword)], xmm2 + + movdqa xmm4, xmm0 + palignr xmm4, xmm1, sizeof(qword) + psrlq xmm1, 1 + psllq xmm4, 63 + por xmm1, xmm4 + movdqu oword [edi+sizeof(oword)*2], xmm1 + + ret +ENDFUNC shr_384 + +;; +;; void cpy_384(Ipp32u* r, const Ipp32u* a) +;; +%macro cpy_384 2.nolist + %xdefine %%pdst %1 + %xdefine %%psrc %2 + + movdqu xmm0, oword [%%psrc] + movdqu xmm1, oword [%%psrc+sizeof(oword)] + movdqu xmm2, oword [%%psrc+sizeof(oword)*2] + movdqu oword [%%pdst], xmm0 + movdqu oword [%%pdst+sizeof(oword)], xmm1 + movdqu oword [%%pdst+sizeof(oword)*2], xmm2 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p384r1_add(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +align IPP_ALIGN_FACTOR +IPPASM p384r1_add,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +%xdefine pB [ebp + ARG_1 + 2*sizeof(dword)] ; source B address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN384] +%assign _sp_ _buf_+(LEN384)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + mov edi, pR ; pR + mov esi, pA ; pA + mov ebx, pB ; pB + CALL_IPPASM add_384 ; R = A+B + mov edx, eax + + lea edi, [esp+_buf_] ; T + mov esi, pR ; R + LD_ADDR ebx, p384r1_data ; modulus + lea ebx, [ebx+(_prime384r1-p384r1_data)] + CALL_IPPASM sub_384 ; T = R-modulus + + lea esi,[esp+_buf_] + mov edi, pR + sub edx, eax ; R = T<0? R : T + cmovnz esi, edi + cpy_384 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p384r1_add + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p384r1_sub(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +align IPP_ALIGN_FACTOR +IPPASM p384r1_sub,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +%xdefine pB [ebp + ARG_1 + 2*sizeof(dword)] ; source B address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN384] +%assign _sp_ _buf_+(LEN384)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + mov edi, pR ; pR + mov esi, pA ; pA + mov ebx, pB ; pB + CALL_IPPASM sub_384 ; R = A-B + mov edx, eax + + lea edi, [esp+_buf_] ; T + mov esi, pR ; R + LD_ADDR ebx, p384r1_data ; modulus + lea ebx, [ebx+(_prime384r1-p384r1_data)] + CALL_IPPASM add_384 ; T = R+modulus + + lea esi,[esp+_buf_] + mov edi, pR + test edx, edx ; R = T<0? R : T + cmovz esi, edi + cpy_384 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p384r1_sub + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p384r1_neg(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p384r1_neg,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN384] +%assign _sp_ _buf_+(LEN384)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + mov edi, pR ; outpur pR + mov esi, pA ; input pA + + ; r = 0-a + mov eax, 0 + sub eax, dword [esi] + mov dword [edi], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)] + mov dword [edi+sizeof(dword)], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*2] + mov dword [edi+sizeof(dword)*2], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*3] + mov dword [edi+sizeof(dword)*3], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*4] + mov dword [edi+sizeof(dword)*4], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*5] + mov dword [edi+sizeof(dword)*5], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*6] + mov dword [edi+sizeof(dword)*6], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*7] + mov dword [edi+sizeof(dword)*7], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*8] + mov dword [edi+sizeof(dword)*8], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*9] + mov dword [edi+sizeof(dword)*9], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*10] + mov dword [edi+sizeof(dword)*10], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*11] + mov dword [edi+sizeof(dword)*11], eax + sbb edx,edx + + lea edi, [esp+_buf_] ; T + mov esi, pR ; R + LD_ADDR ebx, p384r1_data ; modulus + lea ebx, [ebx+(_prime384r1-p384r1_data)] + CALL_IPPASM add_384 ; T = R+modulus + + lea esi,[esp+_buf_] + mov edi, pR + test edx, edx ; R = T<0? R : T + cmovz esi, edi + cpy_384 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p384r1_neg + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p384r1_mul_by_2(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p384r1_mul_by_2,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN384] +%assign _sp_ _buf_+(LEN384)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + lea edi, [esp+_buf_] ; T + mov esi, pA ; pA + CALL_IPPASM shl_384 ; T = A<<1 + mov edx, eax + + mov esi, edi ; T + mov edi, pR ; R + LD_ADDR ebx, p384r1_data ; modulus + lea ebx, [ebx+(_prime384r1-p384r1_data)] + CALL_IPPASM sub_384 ; R = T-modulus + + sub edx, eax ; R = R<0? T : R + cmovz esi, edi + cpy_384 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p384r1_mul_by_2 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p384r1_mul_by_3(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p384r1_mul_by_3,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _bufT_ 0 ; T buffer[LEN384] +%assign _bufU_ _bufT_+(LEN384)*sizeof(dword) ; U buffer[LEN384] +%assign _mod_ _bufU_+(LEN384)*sizeof(dword) ; modulus address [1] +%assign _sp_ _mod_+sizeof(dword) ; esp [1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + LD_ADDR eax, p384r1_data ; srore modulus address + lea eax, [eax+(_prime384r1-p384r1_data)] + mov dword [esp+_mod_], eax + + lea edi, [esp+_bufT_] ; T + mov esi, pA ; A + CALL_IPPASM shl_384 ; T = A<<1 + mov edx, eax + + mov esi, edi ; T + lea edi, [esp+_bufU_] ; U + mov ebx, [esp+_mod_] ; modulus + CALL_IPPASM sub_384 ; U = T-modulus + + sub edx, eax ; T = U<0? T : U + cmovz esi, edi + cpy_384 edi, esi + + mov esi, edi + mov ebx, pA + CALL_IPPASM add_384 ; T +=A + mov edx, eax + + mov edi, pR ; R + mov ebx, [esp+_mod_] ; modulus + CALL_IPPASM sub_384 ; R = T-modulus + + sub edx, eax ; R = T<0? R : T + cmovz esi, edi + cpy_384 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p384r1_mul_by_3 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p384r1_div_by_2(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p384r1_div_by_2,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN384] +%assign _sp_ _buf_+(LEN384)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + lea edi, [esp+_buf_] ; T + mov esi, pA ; A + LD_ADDR ebx, p384r1_data ; modulus + lea ebx, [ebx+(_prime384r1-p384r1_data)] + CALL_IPPASM add_384 ; R = A+modulus + mov edx, 0 + + mov ecx, dword [esi] ; shifted_data = (a[0]&1)? T : A + and ecx, 1 + cmovnz esi, edi + cmovz eax, edx + movd xmm0, eax + mov edi, pR + CALL_IPPASM shr_384 + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p384r1_div_by_2 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p384r1_mul_mont_slm(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +align IPP_ALIGN_FACTOR +IPPASM p384r1_mul_mont_slm,PUBLIC + USES_GPR ebp,ebx,esi,edi + +%xdefine pR [eax + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [eax + ARG_1 + 1*sizeof(dword)] ; source A address +%xdefine pB [eax + ARG_1 + 2*sizeof(dword)] ; source B address +; +; stack layout: +; +%assign _buf_ 0 +%assign _rp_ _buf_+(LEN384+1)*sizeof(dword) ; pR +%assign _ap_ _rp_ +sizeof(dword) ; pA +%assign _bp_ _ap_+sizeof(dword) ; pB +%assign _sp_ _bp_+sizeof(dword) ; esp storage +%assign _ssize_ _sp_+sizeof(dword) ; size allocated stack + + mov eax, esp ; save esp + sub esp, _ssize_ ; allocate stack + and esp, -16 ; provide 16-byte stack alignment + mov dword [esp+_sp_], eax ; store original esp + + ; clear buffer + pxor mm0, mm0 + movq [esp+_buf_], mm0 + movq [esp+_buf_+sizeof(qword)], mm0 + movq [esp+_buf_+sizeof(qword)*2], mm0 + movq [esp+_buf_+sizeof(qword)*3], mm0 + movq [esp+_buf_+sizeof(qword)*4], mm0 + movq [esp+_buf_+sizeof(qword)*5], mm0 + movq [esp+_buf_+sizeof(qword)*6], mm0 + + ; store parameters into the stack + ; note: eax here stores an original esp, so it can be used to reach function parameters + mov edi, pR + mov esi, pA + mov ebp, pB + mov dword [esp+_rp_], edi + mov dword [esp+_ap_], esi + mov dword [esp+_bp_], ebp + + mov eax, LEN384 + + movd mm1, dword [esi+sizeof(dword)] ; pre load a[1], a[2], a[3], a[4] + movd mm2, dword [esi+sizeof(dword)*2] + movd mm3, dword [esi+sizeof(dword)*3] + movd mm4, dword [esi+sizeof(dword)*4] + + +align IPP_ALIGN_FACTOR +.mmul_loop: +; +; i-st pass +; modulus = 2^384 -2^128 -2^96 +2^32 -1 +; [12] [4] [3] [1] [0] +; m0 = 1 +; +; T0 = a[ 0]*b[i] + p[0] +; e0 = HI(T0), u = LO(T0) +; +; T1 = a[ 1]*b[i] +e0 +p[ 1], (cf=0), e1 = HI( T1), p1 =LO( T1), (cf1 ,p1 ) = p1 +u, p[ 0] = p1, cf1 +; T2 = a[ 2]*b[i] +e1 +p[ 2], (cf=0), e2 = HI( T2), p2 =LO( T2), (cf2 ,p2 ) = p2 +cf1, p[ 1] = p2, cf2 +; T3 = a[ 3]*b[i] +e2 +p[ 3], (cf=0), e3 = HI( T3), p3 =LO( T3), (bf3 ,p3 ) = p3 +cf2 -u, p[ 2] = p3, bf3 +; T4 = a[ 4]*b[i] +e3 +p[ 4], (cf=0), e4 = HI( T4), p4 =LO( T4), (bf4 ,p4 ) = p4 -bf3 -u, p[ 3] = p4, bf4 +; T5 = a[ 5]*b[i] +e4 +p[ 5], (cf=0), e5 = HI( T5), p5 =LO( T5), (bf5 ,p5 ) = p5 -bf4, p[ 4] = p5, bf5 +; T6 = a[ 6]*b[i] +e5 +p[ 6], (cf=0), e6 = HI( T6), p6 =LO( T6), (bf6 ,p6 ) = p6 -bf5, p[ 5] = p6, bf6 +; T7 = a[ 7]*b[i] +e6 +p[ 7], (cf=0), e7 = HI( T7), p7 =LO( T7), (bf7 ,p7 ) = p7 -bf6, p[ 6] = p7, bf7 +; T8 = a[ 8]*b[i] +e7 +p[ 8], (cf=0), e8 = HI( T8), p8 =LO( T8), (bf8 ,p8 ) = p8 -bf7, p[ 7] = p8, bf8 +; T9 = a[ 9]*b[i] +e8 +p[ 9], (cf=0), e9 = HI( T9), p9 =LO( T9), (bf9 ,p9 ) = p9 -bf8, p[ 8] = p9, bf9 +; T10 = a[10]*b[i] +e9 +p[10], (cf=0), e10= HI(T10), p10=LO(T10), (bf10,p10) = p10-bf9, p[ 9] = p10, bf9 +; T11 = a[11]*b[i] +e10 +p[11], (cf=0), e11= HI(T11), p11=LO(T11), (bf11,p11) = p11-bf10, p[10] = p11, bf10 +; (cf12,p12) = e11-bf11+u p[11] = p12, cf12 +; p[13] = cf12 + movd mm7, eax ; save pass counter + + mov edx, dword [ebp] ; b = b[i] + mov eax, dword [esi] ; a[0] + movd mm0, edx + add ebp, sizeof(dword) + mov dword [esp+_bp_], ebp + + pmuludq mm1, mm0 ; a[1]*b[i] + pmuludq mm2, mm0 ; a[2]*b[i] + + mul edx ; (E:u) = (edx:eax) = a[0]*b[i]+buf[0] + add eax, dword [esp+_buf_] + adc edx, 0 + + pmuludq mm3, mm0 ; a[3]*b[i] + pmuludq mm4, mm0 ; a[4]*b[i] + +; multiplication round 1 - round 4 + movd ecx, mm1 ; p = a[1]*b[i] + E + psrlq mm1, 32 + add ecx, edx + movd edx, mm1 + adc edx, 0 + add ecx, dword [esp+_buf_+sizeof(dword)*1] + movd mm1, dword [esi+sizeof(dword)*5] + adc edx, 0 + ;pmuludq mm1, mm0 + + movd ebx, mm2 ; p = a[2]*b[i] + E + psrlq mm2, 32 + add ebx, edx + movd edx, mm2 + adc edx, 0 + add ebx, dword [esp+_buf_+sizeof(dword)*2] + movd mm2, dword [esi+sizeof(dword)*6] + adc edx, 0 + pmuludq mm1, mm0 + pmuludq mm2, mm0 + + movd ebp, mm3 ; p = a[3]*b[i] + E + psrlq mm3, 32 + add ebp, edx + movd edx, mm3 + adc edx, 0 + add ebp, dword [esp+_buf_+sizeof(dword)*3] + movd mm3, dword [esi+sizeof(dword)*7] + adc edx, 0 + ;pmuludq mm3, mm0 + + movd edi, mm4 ; p = a[4]*b[i] + E + psrlq mm4, 32 + add edi, edx + movd edx, mm4 + adc edx, 0 + add edi, dword [esp+_buf_+sizeof(dword)*4] + movd mm4, dword [esi+sizeof(dword)*8] + adc edx, 0 + pmuludq mm3, mm0 + pmuludq mm4, mm0 + +;;; and reduction ;;; + add ecx, eax ; +u0 + mov dword [esp+_buf_+sizeof(dword)*0], ecx + mov ecx, eax ; save u0 + adc ebx, 0 ; +cf + mov dword [esp+_buf_+sizeof(dword)*1], ebx + sbb eax, 0 ; eax = u0-cf + sub ebp, eax ; ebp-eax = ebp+cf-u0 + mov dword [esp+_buf_+sizeof(dword)*2], ebp + sbb edi, ecx ; edi-u0-bf + mov eax, 0 + mov dword [esp+_buf_+sizeof(dword)*3], edi + movd mm6, ecx + ;sbb eax, eax ; save bf signu: eax = bf? 0xffffffff: 0x00000000 + adc eax, 0 + +; multiplication round 5 - round 8 + movd ecx, mm1 ; p = a[5]*b[i] + E + psrlq mm1, 32 + add ecx, edx + movd edx, mm1 + adc edx, 0 + add ecx, dword [esp+_buf_+sizeof(dword)*5] + movd mm1, dword [esi+sizeof(dword)*9] + adc edx, 0 + ;pmuludq mm1, mm0 + + movd ebx, mm2 ; p = a[6]*b[i] + E + psrlq mm2, 32 + add ebx, edx + movd edx, mm2 + adc edx, 0 + add ebx, dword [esp+_buf_+sizeof(dword)*6] + movd mm2, dword [esi+sizeof(dword)*10] + adc edx, 0 + pmuludq mm1, mm0 + pmuludq mm2, mm0 + + movd ebp, mm3 ; p = a[7]*b[i] + E + psrlq mm3, 32 + add ebp, edx + movd edx, mm3 + adc edx, 0 + add ebp, dword [esp+_buf_+sizeof(dword)*7] + movd mm3, dword [esi+sizeof(dword)*11] + adc edx, 0 + pmuludq mm3, mm0 + + movd edi, mm4 ; p = a[8]*b[i] + E + psrlq mm4, 32 + add edi, edx + movd edx, mm4 + adc edx, 0 + add edi, dword [esp+_buf_+sizeof(dword)*8] + adc edx, 0 + +;;; and reduction ;;; + ;add eax, eax ; restore bf + ;sbb ecx, 0 ; -bf + sub ecx, eax + movd eax, mm6 + mov dword [esp+_buf_+sizeof(dword)*4], ecx + sbb ebx, 0 ; -bf + mov dword [esp+_buf_+sizeof(dword)*5], ebx + sbb ebp, 0 ; -bf + mov dword [esp+_buf_+sizeof(dword)*6], ebp + sbb edi, 0 ; -bf + mov eax, 0 + mov dword [esp+_buf_+sizeof(dword)*7], edi + ;sbb eax, eax + adc eax, 0 + +; multiplication round 9 - round 11 + movd ecx, mm1 ; p = a[9]*b[i] + E + psrlq mm1, 32 + add ecx, edx + movd edx, mm1 + adc edx, 0 + add ecx, dword [esp+_buf_+sizeof(dword)*9] + adc edx, 0 + + movd ebx, mm2 ; p = a[10]*b[i] + E + psrlq mm2, 32 + add ebx, edx + movd edx, mm2 + adc edx, 0 + add ebx, dword [esp+_buf_+sizeof(dword)*10] + adc edx, 0 + + movd ebp, mm3 ; p = a[11]*b[i] + E + psrlq mm3, 32 + add ebp, edx + movd edx, mm3 + adc edx, 0 + add ebp, dword [esp+_buf_+sizeof(dword)*11] + adc edx, 0 + +;;; and reduction ;;; + ;add eax, eax ; restore bf + ;sbb ecx, 0 ; u0-bf + sub ecx, eax + mov dword [esp+_buf_+sizeof(dword)*8], ecx + movd ecx, mm6 + sbb ebx, 0 ; -bf + mov dword [esp+_buf_+sizeof(dword)*9], ebx + sbb ebp, 0 ; -bf + mov dword [esp+_buf_+sizeof(dword)*10], ebp + sbb ecx, 0 ; u0-bf + mov ebx, 0 + +; last multiplication round 12 + movd eax, mm7 ; restore pass counter + + add edx, dword [esp+_buf_+sizeof(dword)*12] + adc ebx, 0 + add edx, ecx + adc ebx, 0 + mov dword [esp+_buf_+sizeof(dword)*11], edx + mov dword [esp+_buf_+sizeof(dword)*12], ebx + + sub eax, 1 + movd mm1, dword [esi+sizeof(dword)] ; speculative load a[1], a[2], a[3], a[4] + movd mm2, dword [esi+sizeof(dword)*2] + movd mm3, dword [esi+sizeof(dword)*3] + movd mm4, dword [esi+sizeof(dword)*4] + jz .exit_mmul_loop + + mov ebp, dword [esp+_bp_] ; restore pB + jmp .mmul_loop + +.exit_mmul_loop: + emms + +; final reduction + mov edi, [esp+_rp_] ; result + lea esi, [esp+_buf_] ; buffer + LD_ADDR ebx, p384r1_data ; modulus + lea ebx, [ebx+(_prime384r1-p384r1_data)] + CALL_IPPASM sub_384 + mov edx, dword [esp+_buf_+sizeof(dword)*12] + sub edx, eax + +; copy + cmovz esi, edi + cpy_384 edi, esi + + mov esp, [esp+_sp_] ; release stack + REST_GPR + ret +ENDFUNC p384r1_mul_mont_slm + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p384r1_sqr_mont_slm(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p384r1_sqr_mont_slm,PUBLIC + USES_GPR esi,edi + +%xdefine pR [esp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [esp + ARG_1 + 1*sizeof(dword)] ; source A address + + ;; use p384r1_mul_mont_slm to compute sqr + mov esi, pA + mov edi, pR + push esi + push esi + push edi + CALL_IPPASM p384r1_mul_mont_slm,PUBLIC + add esp, sizeof(dword)*3 + REST_GPR + ret +ENDFUNC p384r1_sqr_mont_slm + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p384r1_mred(Ipp32u* r, Ipp32u* prod) +;; +; modulus = 2^384 -2^128 -2^96 +2^32 -1 +; [12] [4] [3] [1] [0] +; m0 = 1 +; +align IPP_ALIGN_FACTOR +IPPASM p384r1_mred,PUBLIC + USES_GPR ebx,esi,edi + +%xdefine pR [esp + ARG_1 + 0*sizeof(dword)] ; reduction address +%xdefine pA [esp + ARG_1 + 1*sizeof(dword)] ; source product address + + ; get parameters: + mov esi, pA + + mov ecx, LEN384 + xor edx, edx +align IPP_ALIGN_FACTOR +.mred_loop: + mov eax, dword [esi] + + mov ebx, dword [esi+sizeof(dword)] + add ebx, eax + mov dword [esi+sizeof(dword)], ebx + + mov ebx, dword [esi+sizeof(dword)*2] + adc ebx, 0 + mov dword [esi+sizeof(dword)*2], ebx + + mov ebx, dword [esi+sizeof(dword)*3] + sbb eax, 0 + sub ebx, eax + mov eax, dword [esi] + mov dword [esi+sizeof(dword)*3], ebx + + mov ebx, dword [esi+sizeof(dword)*4] + sbb ebx, eax + mov dword [esi+sizeof(dword)*4], ebx + + mov ebx, dword [esi+sizeof(dword)*5] + sbb ebx, 0 + mov dword [esi+sizeof(dword)*5], ebx + + mov ebx, dword [esi+sizeof(dword)*6] + sbb ebx, 0 + mov dword [esi+sizeof(dword)*6], ebx + + mov ebx, dword [esi+sizeof(dword)*7] + sbb ebx, 0 + mov dword [esi+sizeof(dword)*7], ebx + + mov ebx, dword [esi+sizeof(dword)*8] + sbb ebx, 0 + mov dword [esi+sizeof(dword)*8], ebx + + mov ebx, dword [esi+sizeof(dword)*9] + sbb ebx, 0 + mov dword [esi+sizeof(dword)*9], ebx + + mov ebx, dword [esi+sizeof(dword)*10] + sbb ebx, 0 + mov dword [esi+sizeof(dword)*10], ebx + + mov ebx, dword [esi+sizeof(dword)*11] + sbb ebx, 0 + mov dword [esi+sizeof(dword)*11], ebx + + mov ebx, dword [esi+sizeof(dword)*12] + sbb eax, 0 + add eax, edx + mov edx, 0 + adc edx, 0 + add ebx, eax + mov dword [esi+sizeof(dword)*12], ebx + adc edx, 0 + + lea esi, [esi+sizeof(dword)] + sub ecx, 1 + jnz .mred_loop + + ; final reduction + mov edi, pR ; result + LD_ADDR ebx, p384r1_data ; addres of the modulus + lea ebx, [ebx+(_prime384r1-p384r1_data)] + CALL_IPPASM sub_384 + + sub edx, eax + cmovz esi, edi + cpy_384 edi, esi + + REST_GPR + ret +ENDFUNC p384r1_mred + +%endif ;; _IPP >= _IPP_P8 + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpp521r1arith_mont_slm.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpp521r1arith_mont_slm.asm new file mode 100644 index 0000000..cbafa02 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpp521r1arith_mont_slm.asm @@ -0,0 +1,1069 @@ +;=============================================================================== +; Copyright (C) 2016 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; P521r1 basic arithmetic function +; +; Content: +; p521r1_add +; p521r1_sub +; p521r1_neg +; p521r1_div_by_2 +; p521r1_mul_mont_slm +; p521r1_sqr_mont_slm +; p521r1_mred +; p521r1_mont_back +; p521r1_select_ap_w7 +; + + + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%if (_IPP >= _IPP_P8) + +segment .text align=IPP_ALIGN_FACTOR + +;; +;; some p256r1 constants +;; +p521r1_data: +_prime521r1 DD 0FFFFFFFFh,0FFFFFFFFh,0FFFFFFFFh,0FFFFFFFFh,0FFFFFFFFh,0FFFFFFFFh,0FFFFFFFFh, 0FFFFFFFFh + DD 0FFFFFFFFh,0FFFFFFFFh,0FFFFFFFFh,0FFFFFFFFh,0FFFFFFFFh,0FFFFFFFFh,0FFFFFFFFh, 0FFFFFFFFh + DD 0000001FFh + +%assign LEN521 ((521+32-1)/32) ; dword's length of operands + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; Ipp32u add_521(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +;; input: edi = r +;; esi = a +;; ebx = b +;; +;; output: eax = carry = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM add_521,PRIVATE + ; r = a+b + mov eax, dword [esi] + add eax, dword [ebx] + mov dword [edi], eax + + mov eax, dword [esi+sizeof(dword)] + adc eax, dword [ebx+sizeof(dword)] + mov dword [edi+sizeof(dword)], eax + + mov eax, dword [esi+sizeof(dword)*2] + adc eax, dword [ebx+sizeof(dword)*2] + mov dword [edi+sizeof(dword)*2], eax + + mov eax, dword [esi+sizeof(dword)*3] + adc eax, dword [ebx+sizeof(dword)*3] + mov dword [edi+sizeof(dword)*3], eax + + mov eax, dword [esi+sizeof(dword)*4] + adc eax, dword [ebx+sizeof(dword)*4] + mov dword [edi+sizeof(dword)*4], eax + + mov eax, dword [esi+sizeof(dword)*5] + adc eax, dword [ebx+sizeof(dword)*5] + mov dword [edi+sizeof(dword)*5], eax + + mov eax, dword [esi+sizeof(dword)*6] + adc eax, dword [ebx+sizeof(dword)*6] + mov dword [edi+sizeof(dword)*6], eax + + mov eax, dword [esi+sizeof(dword)*7] + adc eax, dword [ebx+sizeof(dword)*7] + mov dword [edi+sizeof(dword)*7], eax + + mov eax, dword [esi+sizeof(dword)*8] + adc eax, dword [ebx+sizeof(dword)*8] + mov dword [edi+sizeof(dword)*8], eax + + mov eax, dword [esi+sizeof(dword)*9] + adc eax, dword [ebx+sizeof(dword)*9] + mov dword [edi+sizeof(dword)*9], eax + + mov eax, dword [esi+sizeof(dword)*10] + adc eax, dword [ebx+sizeof(dword)*10] + mov dword [edi+sizeof(dword)*10], eax + + mov eax, dword [esi+sizeof(dword)*11] + adc eax, dword [ebx+sizeof(dword)*11] + mov dword [edi+sizeof(dword)*11], eax + + mov eax, dword [esi+sizeof(dword)*12] + adc eax, dword [ebx+sizeof(dword)*12] + mov dword [edi+sizeof(dword)*12], eax + + mov eax, dword [esi+sizeof(dword)*13] + adc eax, dword [ebx+sizeof(dword)*13] + mov dword [edi+sizeof(dword)*13], eax + + mov eax, dword [esi+sizeof(dword)*14] + adc eax, dword [ebx+sizeof(dword)*14] + mov dword [edi+sizeof(dword)*14], eax + + mov eax, dword [esi+sizeof(dword)*15] + adc eax, dword [ebx+sizeof(dword)*15] + mov dword [edi+sizeof(dword)*15], eax + + mov eax, dword [esi+sizeof(dword)*16] + adc eax, dword [ebx+sizeof(dword)*16] + mov dword [edi+sizeof(dword)*16], eax + + mov eax, 0 + adc eax, 0 + ret +ENDFUNC add_521 + +;; +;; Ipp32u sub_521(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +;; input: edi = r +;; esi = a +;; ebx = b +;; +;; output: eax = borrow = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM sub_521,PRIVATE + ; r = a-b + mov eax, dword [esi] + sub eax, dword [ebx] + mov dword [edi], eax + + mov eax, dword [esi+sizeof(dword)] + sbb eax, dword [ebx+sizeof(dword)] + mov dword [edi+sizeof(dword)], eax + + mov eax, dword [esi+sizeof(dword)*2] + sbb eax, dword [ebx+sizeof(dword)*2] + mov dword [edi+sizeof(dword)*2], eax + + mov eax, dword [esi+sizeof(dword)*3] + sbb eax, dword [ebx+sizeof(dword)*3] + mov dword [edi+sizeof(dword)*3], eax + + mov eax, dword [esi+sizeof(dword)*4] + sbb eax, dword [ebx+sizeof(dword)*4] + mov dword [edi+sizeof(dword)*4], eax + + mov eax, dword [esi+sizeof(dword)*5] + sbb eax, dword [ebx+sizeof(dword)*5] + mov dword [edi+sizeof(dword)*5], eax + + mov eax, dword [esi+sizeof(dword)*6] + sbb eax, dword [ebx+sizeof(dword)*6] + mov dword [edi+sizeof(dword)*6], eax + + mov eax, dword [esi+sizeof(dword)*7] + sbb eax, dword [ebx+sizeof(dword)*7] + mov dword [edi+sizeof(dword)*7], eax + + mov eax, dword [esi+sizeof(dword)*8] + sbb eax, dword [ebx+sizeof(dword)*8] + mov dword [edi+sizeof(dword)*8], eax + + mov eax, dword [esi+sizeof(dword)*9] + sbb eax, dword [ebx+sizeof(dword)*9] + mov dword [edi+sizeof(dword)*9], eax + + mov eax, dword [esi+sizeof(dword)*10] + sbb eax, dword [ebx+sizeof(dword)*10] + mov dword [edi+sizeof(dword)*10], eax + + mov eax, dword [esi+sizeof(dword)*11] + sbb eax, dword [ebx+sizeof(dword)*11] + mov dword [edi+sizeof(dword)*11], eax + + mov eax, dword [esi+sizeof(dword)*12] + sbb eax, dword [ebx+sizeof(dword)*12] + mov dword [edi+sizeof(dword)*12], eax + + mov eax, dword [esi+sizeof(dword)*13] + sbb eax, dword [ebx+sizeof(dword)*13] + mov dword [edi+sizeof(dword)*13], eax + + mov eax, dword [esi+sizeof(dword)*14] + sbb eax, dword [ebx+sizeof(dword)*14] + mov dword [edi+sizeof(dword)*14], eax + + mov eax, dword [esi+sizeof(dword)*15] + sbb eax, dword [ebx+sizeof(dword)*15] + mov dword [edi+sizeof(dword)*15], eax + + mov eax, dword [esi+sizeof(dword)*16] + sbb eax, dword [ebx+sizeof(dword)*16] + mov dword [edi+sizeof(dword)*16], eax + + mov eax, 0 + adc eax, 0 + ret +ENDFUNC sub_521 + +;; +;; Ipp32u shl_521(Ipp32u* r, const Ipp32u* a) +;; +;; input: edi = r +;; esi = a +;; +;; output: eax = extension = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM shl_521,PRIVATE + ; r = a<<1 + push ecx + mov ecx, LEN521-4 + + pxor xmm1, xmm1 +.shl_loop: + movdqu xmm0, oword [esi] + movdqa xmm2, xmm0 + add esi, sizeof(oword) + palignr xmm2, xmm1, sizeof(qword) + movdqa xmm1, xmm0 + + psllq xmm0, 1 + psrlq xmm2, 63 + por xmm0, xmm2 + movdqu oword [edi], xmm0 + add edi, sizeof(oword) + sub ecx, 4 + jg .shl_loop + + movd xmm0, dword [esi] + palignr xmm0, xmm1, sizeof(dword)*3 + psllq xmm0, 1 + psrldq xmm0, sizeof(dword) + movd dword [edi], xmm0 + + sub esi, sizeof(oword)*4 + sub edi, sizeof(oword)*4 + pop ecx + + xor eax, eax + ret +ENDFUNC shl_521 + +;; +;; void shr_521(Ipp32u* r, const Ipp32u* a) +;; +;; input: edi = r +;; esi = a +;; eax = ext +;; output: eax = extension = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM shr_521,PRIVATE + ; r = a>>1 + mov eax, dword [esi+(LEN521-1)*sizeof(dword)] + + push ecx + mov ecx, LEN521-(4*2) +.shr_loop: + movdqu xmm0, oword [esi] + movdqu xmm1, oword [esi+sizeof(oword)] + palignr xmm1, xmm0, sizeof(qword) + add esi, sizeof(oword) + + psrlq xmm0, 1 + psllq xmm1, 63 + por xmm0, xmm1 + movdqu oword [edi], xmm0 + add edi, sizeof(oword) + + sub ecx, 4 + jg .shr_loop + pop ecx + + movdqu xmm0, oword [esi] + movd xmm1, eax + palignr xmm1, xmm0, sizeof(qword) + + psrlq xmm0, 1 + psllq xmm1, 63 + por xmm0, xmm1 + movdqu oword [edi], xmm0 + shr eax, 1 + mov dword [edi+sizeof(oword)], eax + + sub esi, sizeof(qword)*3 + sub esi, sizeof(qword)*3 + ret +ENDFUNC shr_521 +;; +;; void cpy_521(Ipp32u* r, const Ipp32u* a) +;; +%macro cpy_521 2.nolist + %xdefine %%pdst %1 + %xdefine %%psrc %2 + + movdqu xmm0, oword [%%psrc] + movdqu xmm1, oword [%%psrc+sizeof(oword)] + movdqu xmm2, oword [%%psrc+sizeof(oword)*2] + movdqu xmm3, oword [%%psrc+sizeof(oword)*3] + mov eax, dword [%%psrc+sizeof(oword)*4] + movdqu oword [%%pdst], xmm0 + movdqu oword [%%pdst+sizeof(oword)], xmm1 + movdqu oword [%%pdst+sizeof(oword)*2], xmm2 + movdqu oword [%%pdst+sizeof(oword)*3], xmm3 + mov dword [%%pdst+sizeof(oword)*4], eax +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p521r1_add(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +align IPP_ALIGN_FACTOR +IPPASM p521r1_add,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +%xdefine pB [ebp + ARG_1 + 2*sizeof(dword)] ; source B address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN521] +%assign _sp_ _buf_+(LEN521)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + mov edi, pR ; pR + mov esi, pA ; pA + mov ebx, pB ; pB + CALL_IPPASM add_521 ; R = A+B + mov edx, eax + + lea edi, [esp+_buf_] ; T + mov esi, pR ; R + LD_ADDR ebx, p521r1_data ; modulus + lea ebx, [ebx+(_prime521r1-p521r1_data)] + CALL_IPPASM sub_521 ; T = R-modulus + + lea esi,[esp+_buf_] + mov edi, pR + sub edx, eax ; R = T<0? R : T + cmovnz esi, edi + cpy_521 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p521r1_add + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p521r1_sub(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +align IPP_ALIGN_FACTOR +IPPASM p521r1_sub,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +%xdefine pB [ebp + ARG_1 + 2*sizeof(dword)] ; source B address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN521] +%assign _sp_ _buf_+(LEN521)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + mov edi, pR ; pR + mov esi, pA ; pA + mov ebx, pB ; pB + CALL_IPPASM sub_521 ; R = A-B + mov edx, eax + + lea edi, [esp+_buf_] ; T + mov esi, pR ; R + LD_ADDR ebx, p521r1_data ; modulus + lea ebx, [ebx+(_prime521r1-p521r1_data)] + CALL_IPPASM add_521 ; T = R+modulus + + lea esi,[esp+_buf_] + mov edi, pR + test edx, edx ; R = T<0? R : T + cmovz esi, edi + cpy_521 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p521r1_sub + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p521r1_neg(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p521r1_neg,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN521] +%assign _sp_ _buf_+(LEN521)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + mov edi, pR ; outpur pR + mov esi, pA ; input pA + + ; r = 0-a + mov eax, 0 + sub eax, dword [esi] + mov dword [edi], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)] + mov dword [edi+sizeof(dword)], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*2] + mov dword [edi+sizeof(dword)*2], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*3] + mov dword [edi+sizeof(dword)*3], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*4] + mov dword [edi+sizeof(dword)*4], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*5] + mov dword [edi+sizeof(dword)*5], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*6] + mov dword [edi+sizeof(dword)*6], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*7] + mov dword [edi+sizeof(dword)*7], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*8] + mov dword [edi+sizeof(dword)*8], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*9] + mov dword [edi+sizeof(dword)*9], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*10] + mov dword [edi+sizeof(dword)*10], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*11] + mov dword [edi+sizeof(dword)*11], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*12] + mov dword [edi+sizeof(dword)*12], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*13] + mov dword [edi+sizeof(dword)*13], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*14] + mov dword [edi+sizeof(dword)*14], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*15] + mov dword [edi+sizeof(dword)*15], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*16] + mov dword [edi+sizeof(dword)*16], eax + + sbb edx,edx + + lea edi, [esp+_buf_] ; T + mov esi, pR ; R + LD_ADDR ebx, p521r1_data ; modulus + lea ebx, [ebx+(_prime521r1-p521r1_data)] + CALL_IPPASM add_521 ; T = R+modulus + + lea esi,[esp+_buf_] + mov edi, pR + test edx, edx ; R = T<0? R : T + cmovz esi, edi + cpy_521 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p521r1_neg + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p521r1_mul_by_2(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p521r1_mul_by_2,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN521] +%assign _sp_ _buf_+(LEN521)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + lea edi, [esp+_buf_] ; T + mov esi, pA ; pA + CALL_IPPASM shl_521 ; T = A<<1 + mov edx, eax + + mov esi, edi ; T + mov edi, pR ; R + LD_ADDR ebx, p521r1_data ; modulus + lea ebx, [ebx+(_prime521r1-p521r1_data)] + CALL_IPPASM sub_521 ; R = T-modulus + + sub edx, eax ; R = R<0? T : R + cmovz esi, edi + cpy_521 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p521r1_mul_by_2 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p521r1_mul_by_3(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p521r1_mul_by_3,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _bufT_ 0 ; T buffer[LEN521] +%assign _bufU_ _bufT_+(LEN521)*sizeof(dword) ; U buffer[LEN521] +%assign _mod_ _bufU_+(LEN521)*sizeof(dword) ; modulus address [1] +%assign _sp_ _mod_+sizeof(dword) ; esp [1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + LD_ADDR eax, p521r1_data ; srore modulus address + lea eax, [eax+(_prime521r1-p521r1_data)] + mov dword [esp+_mod_], eax + + lea edi, [esp+_bufT_] ; T + mov esi, pA ; A + CALL_IPPASM shl_521 ; T = A<<1 + mov edx, eax + + mov esi, edi ; T + lea edi, [esp+_bufU_] ; U + mov ebx, [esp+_mod_] ; modulus + CALL_IPPASM sub_521 ; U = T-modulus + + sub edx, eax ; T = U<0? T : U + cmovz esi, edi + cpy_521 edi, esi + + mov esi, edi + mov ebx, pA + CALL_IPPASM add_521 ; T +=A + mov edx, eax + + mov edi, pR ; R + mov ebx, [esp+_mod_] ; modulus + CALL_IPPASM sub_521 ; R = T-modulus + + sub edx, eax ; R = T<0? R : T + cmovz esi, edi + cpy_521 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p521r1_mul_by_3 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p521r1_div_by_2(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p521r1_div_by_2,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LEN521] +%assign _sp_ _buf_+(LEN521)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_ +sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + lea edi, [esp+_buf_] ; T + mov esi, pA ; A + LD_ADDR ebx, p521r1_data ; modulus + lea ebx, [ebx+(_prime521r1-p521r1_data)] + CALL_IPPASM add_521 ; R = A+modulus + mov edx, 0 + + mov ecx, dword [esi] ; shifted_data = (a[0]&1)? T : A + and ecx, 1 + cmovnz esi, edi + cmovz eax, edx + mov edi, pR + CALL_IPPASM shr_521 + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC p521r1_div_by_2 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p521r1_mul_mont_slm(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +align IPP_ALIGN_FACTOR +IPPASM p521r1_mul_mont_slm,PUBLIC + USES_GPR ebp,ebx,esi,edi + +%xdefine pR [eax + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [eax + ARG_1 + 1*sizeof(dword)] ; source A address +%xdefine pB [eax + ARG_1 + 2*sizeof(dword)] ; source B address +; +; stack layout: +; +%assign _buf_ 0 +%assign _rp_ _buf_+(LEN521+1)*sizeof(dword) ; pR +%assign _ap_ _rp_ +sizeof(dword) ; pA +%assign _bp_ _ap_ +sizeof(dword) ; pB +%assign _sp_ _bp_ +sizeof(dword) ; esp storage +%assign _ssize_ _sp_ +sizeof(dword) ; size allocated stack + + mov eax, esp ; save esp + sub esp, _ssize_ ; allocate stack + and esp, -16 ; provide 16-byte stack alignment + mov dword [esp+_sp_], eax ; store original esp + + ; clear buffer + pxor xmm0, xmm0 + movdqa oword [esp+_buf_], xmm0 + movdqa oword [esp+_buf_+sizeof(oword)], xmm0 + movdqa oword [esp+_buf_+sizeof(oword)*2], xmm0 + movdqa oword [esp+_buf_+sizeof(oword)*3], xmm0 + movq qword [esp+_buf_+sizeof(oword)*4], xmm0 + + ; store parameters into the stack + ; note: eax here stores an original esp, so it can be used to reach function parameters + mov edi, pR + mov esi, pA + mov ebp, pB + mov dword [esp+_rp_], edi + mov dword [esp+_ap_], esi + mov dword [esp+_bp_], ebp + + mov ebx, LEN521 + + movd mm1, dword [esi+sizeof(dword)] ; pre load a[1], a[2], a[3], a[4] + movd mm2, dword [esi+sizeof(dword)*2] + movd mm3, dword [esi+sizeof(dword)*3] + movd mm4, dword [esi+sizeof(dword)*4] + +align IPP_ALIGN_FACTOR +.mmul_loop: +; +; i-st pass +; modulus = 2^521 -1 +; [17] [0] +; m0 = 1 +; + movd mm7, ebx ; save pass counter + + mov edx, dword [ebp] ; b = b[i] + mov eax, dword [esi] ; a[0] + movd mm0, edx + add ebp, sizeof(dword) + mov dword [esp+_bp_], ebp + + pmuludq mm1, mm0 ; a[1]*b[i] + pmuludq mm2, mm0 ; a[2]*b[i] + + mul edx ; (E:u) = (edx:eax) = a[0]*b[i]+buf[0] + add eax, dword [esp+_buf_] + adc edx, 0 + + pmuludq mm3, mm0 ; a[3]*b[i] + pmuludq mm4, mm0 ; a[4]*b[i] + +; multiplication round 1 - round 4 + movd ecx, mm1 ; p = a[1]*b[i] + E + psrlq mm1, 32 + add ecx, edx + movd edx, mm1 + adc edx, 0 + add ecx, dword [esp+_buf_+sizeof(dword)*1] + movd mm1, dword [esi+sizeof(dword)*5] + mov dword [esp+_buf_+sizeof(dword)*0], ecx + adc edx, 0 + + movd ebx, mm2 ; p = a[2]*b[i] + E + psrlq mm2, 32 + add ebx, edx + movd edx, mm2 + adc edx, 0 + add ebx, dword [esp+_buf_+sizeof(dword)*2] + movd mm2, dword [esi+sizeof(dword)*6] + mov dword [esp+_buf_+sizeof(dword)*1], ebx + adc edx, 0 + + pmuludq mm1, mm0 ; a[5]*b[i] + pmuludq mm2, mm0 ; a[6]*b[i] + + movd ebp, mm3 ; p = a[3]*b[i] + E + psrlq mm3, 32 + add ebp, edx + movd edx, mm3 + adc edx, 0 + add ebp, dword [esp+_buf_+sizeof(dword)*3] + movd mm3, dword [esi+sizeof(dword)*7] + mov dword [esp+_buf_+sizeof(dword)*2], ebp + adc edx, 0 + + movd edi, mm4 ; p = a[4]*b[i] + E + psrlq mm4, 32 + add edi, edx + movd edx, mm4 + adc edx, 0 + add edi, dword [esp+_buf_+sizeof(dword)*4] + movd mm4, dword [esi+sizeof(dword)*8] + mov dword [esp+_buf_+sizeof(dword)*3], edi + adc edx, 0 + + pmuludq mm3, mm0 ; a[7]*b[i] + pmuludq mm4, mm0 ; a[8]*b[i] + +; multiplication round 5 - round 8 + movd ecx, mm1 ; p = a[5]*b[i] + E + psrlq mm1, 32 + add ecx, edx + movd edx, mm1 + adc edx, 0 + add ecx, dword [esp+_buf_+sizeof(dword)*5] + movd mm1, dword [esi+sizeof(dword)*9] + mov dword [esp+_buf_+sizeof(dword)*4], ecx + adc edx, 0 + + movd ebx, mm2 ; p = a[6]*b[i] + E + psrlq mm2, 32 + add ebx, edx + movd edx, mm2 + adc edx, 0 + add ebx, dword [esp+_buf_+sizeof(dword)*6] + movd mm2, dword [esi+sizeof(dword)*10] + mov dword [esp+_buf_+sizeof(dword)*5], ebx + adc edx, 0 + + pmuludq mm1, mm0 ; a[9]*b[i] + pmuludq mm2, mm0 ; a[10]*b[i] + + movd ebp, mm3 ; p = a[7]*b[i] + E + psrlq mm3, 32 + add ebp, edx + movd edx, mm3 + adc edx, 0 + add ebp, dword [esp+_buf_+sizeof(dword)*7] + movd mm3, dword [esi+sizeof(dword)*11] + mov dword [esp+_buf_+sizeof(dword)*6], ebp + adc edx, 0 + + movd edi, mm4 ; p = a[8]*b[i] + E + psrlq mm4, 32 + add edi, edx + movd edx, mm4 + adc edx, 0 + add edi, dword [esp+_buf_+sizeof(dword)*8] + movd mm4, dword [esi+sizeof(dword)*12] + mov dword [esp+_buf_+sizeof(dword)*7], edi + adc edx, 0 + + pmuludq mm3, mm0 ; a[11]*b[i] + pmuludq mm4, mm0 ; a[12]*b[i] + +; multiplication round 9 - round 12 + movd ecx, mm1 ; p = a[9]*b[i] + E + psrlq mm1, 32 + add ecx, edx + movd edx, mm1 + adc edx, 0 + add ecx, dword [esp+_buf_+sizeof(dword)*9] + movd mm1, dword [esi+sizeof(dword)*13] + mov dword [esp+_buf_+sizeof(dword)*8], ecx + adc edx, 0 + + movd ebx, mm2 ; p = a[10]*b[i] + E + psrlq mm2, 32 + add ebx, edx + movd edx, mm2 + adc edx, 0 + add ebx, dword [esp+_buf_+sizeof(dword)*10] + movd mm2, dword [esi+sizeof(dword)*14] + mov dword [esp+_buf_+sizeof(dword)*9], ebx + adc edx, 0 + + pmuludq mm1, mm0 ; a[13]*b[i] + pmuludq mm2, mm0 ; a[14]*b[i] + + movd ebp, mm3 ; p = a[11]*b[i] + E + psrlq mm3, 32 + add ebp, edx + movd edx, mm3 + adc edx, 0 + add ebp, dword [esp+_buf_+sizeof(dword)*11] + movd mm3, dword [esi+sizeof(dword)*15] + mov dword [esp+_buf_+sizeof(dword)*10], ebp + adc edx, 0 + + movd edi, mm4 ; p = a[12]*b[i] + E + psrlq mm4, 32 + add edi, edx + movd edx, mm4 + adc edx, 0 + add edi, dword [esp+_buf_+sizeof(dword)*12] + movd mm4, dword [esi+sizeof(dword)*16] + mov dword [esp+_buf_+sizeof(dword)*11], edi + adc edx, 0 + + pmuludq mm3, mm0 ; a[15]*b[i] + pmuludq mm4, mm0 ; a[16]*b[i] + +; multiplication round 13 - round 16 + movd ecx, mm1 ; p = a[13]*b[i] + E + psrlq mm1, 32 + add ecx, edx + movd edx, mm1 + adc edx, 0 + add ecx, dword [esp+_buf_+sizeof(dword)*13] + mov dword [esp+_buf_+sizeof(dword)*12], ecx + adc edx, 0 + + movd ebx, mm2 ; p = a[14]*b[i] + E + psrlq mm2, 32 + add ebx, edx + movd edx, mm2 + adc edx, 0 + add ebx, dword [esp+_buf_+sizeof(dword)*14] + mov dword [esp+_buf_+sizeof(dword)*13], ebx + adc edx, 0 + + movd ebp, mm3 ; p = a[15]*b[i] + E + psrlq mm3, 32 + add ebp, edx + movd edx, mm3 + adc edx, 0 + add ebp, dword [esp+_buf_+sizeof(dword)*15] + mov dword [esp+_buf_+sizeof(dword)*14], ebp + adc edx, 0 + + movd edi, mm4 ; p = a[16]*b[i] + E + psrlq mm4, 32 + add edi, edx + movd edx, mm4 + adc edx, 0 + add edi, dword [esp+_buf_+sizeof(dword)*16] + adc edx, 0 + +;;; and reduction and last multiplication round 17 + movd ebx, mm7 ; restore pass counter + + mov ecx, eax ; u0 <<= 9 + shl eax, (521-512) + shr ecx, (32-(521-512)) + + add edi, eax + mov dword [esp+_buf_+sizeof(dword)*15], edi + adc edx, ecx + mov dword [esp+_buf_+sizeof(dword)*16], edx + + sub ebx, 1 + movd mm1, dword [esi+sizeof(dword)] ; speculative load a[1], a[2], a[3], a[4] + movd mm2, dword [esi+sizeof(dword)*2] + movd mm3, dword [esi+sizeof(dword)*3] + movd mm4, dword [esi+sizeof(dword)*4] + jz .exit_mmul_loop + + mov ebp, dword [esp+_bp_] ; restore pB + jmp .mmul_loop + +.exit_mmul_loop: + emms + +; final reduction + mov edi, [esp+_rp_] ; result + lea esi, [esp+_buf_] ; buffer + LD_ADDR ebx, p521r1_data ; modulus + lea ebx, [ebx+(_prime521r1-p521r1_data)] + CALL_IPPASM sub_521 + mov edx, dword [esp+_buf_+LEN521*sizeof(dword)] + +; copy + cmovz esi, edi + cpy_521 edi, esi + + mov esp, [esp+_sp_] ; release stack + REST_GPR + ret +ENDFUNC p521r1_mul_mont_slm + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p521r1_sqr_mont_slm(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM p521r1_sqr_mont_slm,PUBLIC + USES_GPR esi,edi + +%xdefine pR [esp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [esp + ARG_1 + 1*sizeof(dword)] ; source A address + + ;; use p521r1_mul_mont_slm to compute sqr + mov esi, pA + mov edi, pR + push esi + push esi + push edi + CALL_IPPASM p521r1_mul_mont_slm,PUBLIC + add esp, sizeof(dword)*3 + REST_GPR + ret +ENDFUNC p521r1_sqr_mont_slm + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void p521r1_mred(Ipp32u* r, Ipp32u* prod) +;; +; modulus = 2^521 -1 +; [17] [0] +; m0 = 1 +; +align IPP_ALIGN_FACTOR +IPPASM p521r1_mred,PUBLIC + USES_GPR ebx,esi,edi + +%xdefine pR [esp + ARG_1 + 0*sizeof(dword)] ; reduction address +%xdefine pA [esp + ARG_1 + 1*sizeof(dword)] ; source product address + + ; get parameters: + mov esi, pA + + mov ecx, LEN521 + xor edx, edx +align IPP_ALIGN_FACTOR +.mred_loop: + mov ebx, dword [esi] ; [ebx:eax] = u0<<9 + mov eax, dword [esi] + shr ebx,(32-(521-512)) + shl eax,(521-512) + add ebx, edx + + ;; [esi+i*sizeof(dword)] = [esi+i*sizeof(dword)], i=1,..,16 + + add eax, dword [esi+sizeof(dword)*16] + mov edx, dword [esi+sizeof(dword)*17] + adc ebx, edx + mov edx, 0 + + mov dword [esi+sizeof(dword)*16], eax + mov dword [esi+sizeof(dword)*17], ebx + adc edx, 0 + + lea esi, [esi+sizeof(dword)] + sub ecx, 1 + jnz .mred_loop + + ; final reduction + mov edi, pR ; result + LD_ADDR ebx, p521r1_data ; addres of the modulus + lea ebx, [ebx+(_prime521r1-p521r1_data)] + CALL_IPPASM sub_521 + + sub edx, eax + cmovz esi, edi + cpy_521 edi, esi + + REST_GPR + ret +ENDFUNC p521r1_mred + +%endif ;; _IPP >= _IPP_P8 + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcppurgeblkw7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcppurgeblkw7as.asm new file mode 100644 index 0000000..be27371 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcppurgeblkw7as.asm @@ -0,0 +1,80 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Purge block +; +; +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%if (_IPP >= _IPP_W7) + +segment .text align=IPP_ALIGN_FACTOR + + +;*************************************************************** +;* Purpose: Clear memory block +;* +;* void PurgeBlock(Ipp8u *pDst, int len) +;* +;*************************************************************** + +;; +;; Lib = W7 +;; +align IPP_ALIGN_FACTOR +IPPASM PurgeBlock,PUBLIC + USES_GPR edi + +%xdefine pDst [esp + ARG_1 + 0*sizeof(dword)] ; target address +%xdefine len [esp + ARG_1 + 1*sizeof(dword)] ; length + + mov edi, pDst ; mem address + mov ecx, len ; length + xor eax, eax + sub ecx, sizeof(dword) + jl .test_purge +.purge4: + mov dword [edi], eax ; clear + add edi, sizeof(dword) + sub ecx, sizeof(dword) + jge .purge4 + +.test_purge: + add ecx, sizeof(dword) + jz .quit +.purge1: + mov byte [edi], al + add edi, sizeof(byte) + sub ecx, sizeof(byte) + jg .purge1 + +.quit: + REST_GPR + ret +ENDFUNC PurgeBlock + +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprc4v8as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprc4v8as.asm new file mode 100644 index 0000000..666b08b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprc4v8as.asm @@ -0,0 +1,123 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; ARCFour +; +; Content: +; ARCFourKernel() +; +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%if (_IPP >= _IPP_V8) + +segment .text align=IPP_ALIGN_FACTOR + + +;*************************************************************** +;* Purpose: RC4 kernel +;* +;* void ARCFourProcessData(const Ipp8u *pSrc, Ipp8u *pDst, int len, +;* IppsARCFourState* pCtx) +;* +;*************************************************************** + +;; +;; Lib = V8 +;; +;; Caller = ippsARCFourEncrypt +;; Caller = ippsARCFourDecrypt +;; +align IPP_ALIGN_FACTOR +IPPASM ARCFourProcessData,PUBLIC + USES_GPR esi,edi,ebx,ebp + +%xdefine pSrc [esp + ARG_1 + 0*sizeof(dword)] +%xdefine pDst [esp + ARG_1 + 1*sizeof(dword)] +%xdefine len [esp + ARG_1 + 2*sizeof(dword)] +%xdefine pCtx [esp + ARG_1 + 3*sizeof(dword)] + + mov edx, len ; data length + mov esi, pSrc ; source data + mov edi, pDst ; target data + mov ebp, pCtx ; context + + test edx, edx ; test length + jz .quit + + mov eax, dword [ebp+4] ; extract x + mov ebx, dword [ebp+8] ; extract y + + lea ebp, [ebp+12] ; sbox + + add eax,1 ; x = (x+1)&0xFF + and eax, 0FFh + mov ecx, dword [ebp+eax*4] ; tx = S[x] + + lea edx, [esi+edx] ; store stop data address + push edx + +;; +;; main code +;; +align IPP_ALIGN_FACTOR +.main_loop: + add ebx, ecx ; y = (x+tx)&0xFF + movzx ebx, bl + mov edx, dword [ebp+ebx*4] ; ty = S[y] + + mov dword [ebp+ebx*4],ecx ; S[y] = tx + add ecx, edx ; tmp_idx = (tx+ty)&0xFF + movzx ecx, cl + mov dword [ebp+eax*4],edx ; S[x] = ty + + add eax, 1 ; next x = (x+1)&0xFF + mov dl, byte [ebp+ecx*4] ; byte of gamma + + movzx eax, al + + xor dl,byte [esi] ; gamma ^= src + add esi, 1 + mov ecx, dword [ebp+eax*4] ; next tx = S[x] + mov byte [edi],dl ; store result + add edi, 1 + cmp esi, dword [esp] + jb .main_loop + + lea ebp, [ebp-12] ; pointer to context + pop edx ; remove local variable + + dec eax ; actual new x counter + and eax, 0FFh + mov dword [ebp+4], eax ; update x conter + mov dword [ebp+8], ebx ; updtae y counter + +.quit: + REST_GPR + ret +ENDFUNC ARCFourProcessData + +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprc4w7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprc4w7as.asm new file mode 100644 index 0000000..71ac114 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprc4w7as.asm @@ -0,0 +1,118 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; ARCFour +; +; Content: +; ARCFourKernel() +; +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%if (_IPP >= _IPP_M5) && (_IPP < _IPP_V8) + +segment .text align=IPP_ALIGN_FACTOR + + +;*************************************************************** +;* Purpose: RC4 kernel +;* +;* void ARCFourProcessData(const Ipp8u *pSrc, Ipp8u *pDst, int len, +;* IppsARCFourState* pCtx) +;* +;*************************************************************** + +;; +;; Lib = W7 +;; +;; Caller = ippsARCFourEncrypt +;; Caller = ippsARCFourDecrypt +;; +IPPASM ARCFourProcessData,PUBLIC + USES_GPR esi,edi,ebx,ebp + +%xdefine pSrc [esp + ARG_1 + 0*sizeof(dword)] +%xdefine pDst [esp + ARG_1 + 1*sizeof(dword)] +%xdefine len [esp + ARG_1 + 2*sizeof(dword)] +%xdefine pCtx [esp + ARG_1 + 3*sizeof(dword)] + + mov edx, len ; data length + mov esi, pSrc ; source data + mov edi, pDst ; target data + mov ebp, pCtx ; context + + test edx, edx ; test length + jz .quit + + xor eax, eax + movzx eax, byte [ebp+4] ; extract x + xor ebx, ebx + movzx ebx, byte [ebp+8] ; extract y + + lea ebp, [ebp+12] ; sbox + + inc al ; x = (x+1)&0xFF + movzx ecx, byte [ebp+eax] ; tx = S[x] + + lea edx, [esi+edx] ; store stop data address + push edx + +;; +;; main code +;; +align IPP_ALIGN_FACTOR +.main_loop: + add bl,cl ; y = (x+tx)&0xFF + movzx edx,byte [ebx+ebp] ; ty = S[y] + + mov byte [ebx+ebp],cl ; S[y] = tx + mov byte [eax+ebp],dl ; S[x] = ty + add dl, cl ; tmp_idx = (tx+ty)&0xFF + + movzx edx,byte [edx+ebp] ; byte of gamma + + add al,1 ; next x = (x+1)&0xFF + xor dl,byte [esi] ; gamma ^= src + lea esi,[esi+1] + movzx ecx,byte [eax+ebp] ; next tx = S[x] + cmp esi, dword [esp] + mov byte [edi],dl ; store result + lea edi,[edi+1] + jb .main_loop + + lea ebp, [ebp-12] ; pointer to context + pop edx ; remove local variable + + dec eax ; actual new x counter + mov byte [ebp+4], al ; update x conter + mov byte [ebp+8], bl ; updtae y counter + +.quit: + REST_GPR + ret +ENDFUNC ARCFourProcessData + +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128ccmg9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128ccmg9as.asm new file mode 100644 index 0000000..c788dab --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128ccmg9as.asm @@ -0,0 +1,298 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Inverse Cipher function +; +; Content: +; AuthEncrypt_RIJ128_AES_NI() +; DecryptAuth_RIJ128_AES_NI() +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + + +%if (_IPP >= _IPP_P8) + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR +ENCODE_DATA: +u128_str DB 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 +increment DQ 1,0 + +;*************************************************************** +;* Purpose: Authenticate and Encrypt +;* +;* void AuthEncrypt_RIJ128_AES_NI(Ipp8u* outBlk, +;* const Ipp8u* inpBlk, +;* int nr, +;* const Ipp8u* pRKey, +;* Ipp32u len, +;* Ipp8u* pLocalState) +;* inp localCtx: +;* MAC +;* CTRi +;* CTRi mask +;* +;* out localCtx: +;* new MAC +;* S = enc(CTRi) +;*************************************************************** + +;; +;; Lib = P8 +;; +;; Caller = ippsRijndael128CCMEncrypt +;; Caller = ippsRijndael128CCMEncryptMessage +;; +align IPP_ALIGN_FACTOR +IPPASM AuthEncrypt_RIJ128_AES_NI,PUBLIC + USES_GPR esi,edi,ebx + +%xdefine pInpBlk [esp + ARG_1 + 0*sizeof(dword)] ; input blocks address +%xdefine pOutBlk [esp + ARG_1 + 1*sizeof(dword)] ; output blocks address +%xdefine nr [esp + ARG_1 + 2*sizeof(dword)] ; number of rounds +%xdefine pKey [esp + ARG_1 + 3*sizeof(dword)] ; key material address +%xdefine len [esp + ARG_1 + 4*sizeof(dword)] ; length (bytes) +%xdefine pLocCtx [esp + ARG_1 + 5*sizeof(dword)] ; pointer to the localState + +%assign BYTES_PER_BLK (16) + + mov eax, pLocCtx + movdqa xmm0, oword [eax] ; MAC + movdqa xmm2, oword [eax+sizeof(oword)] ; CTRi block + movdqa xmm1, oword [eax+sizeof(oword)*2] ; CTR mask + + LD_ADDR eax, ENCODE_DATA + + movdqa xmm7, oword [eax+(u128_str-ENCODE_DATA)] + + pshufb xmm2, xmm7 ; CTRi block (LE) + pshufb xmm1, xmm7 ; CTR mask + + movdqa xmm3, xmm1 + pandn xmm3, xmm2 ; CTR block template + pand xmm2, xmm1 ; CTR value + + mov edx, nr ; number of rounds + mov ecx, pKey ; and keys + + lea edx, [edx*4] ; nrCounter = -nr*16 + lea edx, [edx*4] ; pKey += nr*16 + lea ecx, [ecx+edx] + neg edx + mov ebx, edx + + mov esi, pInpBlk + mov edi, pOutBlk + +align IPP_ALIGN_FACTOR +;; +;; block-by-block processing +;; +.blk_loop: + movdqu xmm4, oword [esi] ; input block src[i] + pxor xmm0, xmm4 ; MAC ^= src[i] + + movdqa xmm5, xmm3 + paddq xmm2, oword [eax+(increment-ENCODE_DATA)] ; advance counter bits + pand xmm2, xmm1 ; and mask them + por xmm5, xmm2 + pshufb xmm5, xmm7 ; CTRi (BE) + + movdqa xmm6, oword [ecx+edx] ; keys for whitening + add edx, 16 + + pxor xmm5, xmm6 ; whitening (CTRi) + pxor xmm0, xmm6 ; whitening (MAC) + + movdqa xmm6, oword [ecx+edx] ; pre load operation's keys + +align IPP_ALIGN_FACTOR +.cipher_loop: + aesenc xmm5, xmm6 ; regular round (CTRi) + aesenc xmm0, xmm6 ; regular round (MAC) + movdqa xmm6, oword [ecx+edx+16] + add edx, 16 + jnz .cipher_loop + aesenclast xmm5, xmm6 ; irregular round (CTRi) + aesenclast xmm0, xmm6 ; irregular round (MAC) + + pxor xmm4, xmm5 ; dst[i] = src[i] ^ ENC(CTRi) + movdqu oword [edi], xmm4 + + mov edx, ebx + add esi, BYTES_PER_BLK + add edi, BYTES_PER_BLK + sub dword len, BYTES_PER_BLK + jnz .blk_loop + + mov eax, pLocCtx + movdqu oword [eax], xmm0 ; update MAC value + movdqu oword [eax+sizeof(oword)], xmm5 ; update ENC(Ctri) + + REST_GPR + ret +ENDFUNC AuthEncrypt_RIJ128_AES_NI + + +;*************************************************************** +;* Purpose: Decrypt and Authenticate +;* +;* void DecryptAuth_RIJ128_AES_NI(Ipp8u* outBlk, +;* const Ipp8u* inpBlk, +;* int nr, +;* const Ipp8u* pRKey, +;* Ipp32u len, +;* Ipp8u* pLocalState) +;* inp localCtx: +;* MAC +;* CTRi +;* CTRi mask +;* +;* out localCtx: +;* new MAC +;* S = enc(CTRi) +;*************************************************************** + +;; +;; Lib = P8 +;; +;; Caller = ippsRijndael128CCMDecrypt +;; Caller = ippsRijndael128CCMDecryptMessage +;; +align IPP_ALIGN_FACTOR +IPPASM DecryptAuth_RIJ128_AES_NI,PUBLIC + USES_GPR esi,edi,ebx + +%xdefine pInpBlk [esp + ARG_1 + 0*sizeof(dword)] ; input blocks address +%xdefine pOutBlk [esp + ARG_1 + 1*sizeof(dword)] ; output blocks address +%xdefine nr [esp + ARG_1 + 2*sizeof(dword)] ; number of rounds +%xdefine pKey [esp + ARG_1 + 3*sizeof(dword)] ; key material address +%xdefine len [esp + ARG_1 + 4*sizeof(dword)] ; length (bytes) +%xdefine pLocCtx [esp + ARG_1 + 5*sizeof(dword)] ; pointer to the localState + +%assign BYTES_PER_BLK (16) + + mov eax, pLocCtx + movdqa xmm0, oword [eax] ; MAC + movdqa xmm2, oword [eax+sizeof(oword)] ; CTRi block + movdqa xmm1, oword [eax+sizeof(oword)*2] ; CTR mask + + LD_ADDR eax, ENCODE_DATA + + movdqa xmm7, oword [eax+(u128_str-ENCODE_DATA)] + + pshufb xmm2, xmm7 ; CTRi block (LE) + pshufb xmm1, xmm7 ; CTR mask + + movdqa xmm3, xmm1 + pandn xmm3, xmm2 ; CTR block template + pand xmm2, xmm1 ; CTR value + + mov edx, nr ; number of rounds + mov ecx, pKey ; and keys + + lea edx, [edx*4] ; nrCounter = -nr*16 + lea edx, [edx*4] ; pKey += nr*16 + lea ecx, [ecx+edx] + neg edx + mov ebx, edx + + mov esi, pInpBlk + mov edi, pOutBlk + +align IPP_ALIGN_FACTOR +;; +;; block-by-block processing +;; +.blk_loop: + + ;;;;;;;;;;;;;;;;; + ;; decryption + ;;;;;;;;;;;;;;;;; + movdqu xmm4, oword [esi] ; input block src[i] + + movdqa xmm5, xmm3 + paddq xmm2, oword [eax+(increment-ENCODE_DATA)] ; advance counter bits + pand xmm2, xmm1 ; and mask them + por xmm5, xmm2 + pshufb xmm5, xmm7 ; CTRi (BE) + + movdqa xmm6, oword [ecx+edx] ; keys for whitening + add edx, 16 + + pxor xmm5, xmm6 ; whitening (CTRi) + movdqa xmm6, oword [ecx+edx] ; pre load operation's keys + +align IPP_ALIGN_FACTOR +.cipher_loop: + aesenc xmm5, xmm6 ; regular round (CTRi) + movdqa xmm6, oword [ecx+edx+16] + add edx, 16 + jnz .cipher_loop + aesenclast xmm5, xmm6 ; irregular round (CTRi) + + pxor xmm4, xmm5 ; dst[i] = src[i] ^ ENC(CTRi) + movdqu oword [edi], xmm4 + + ;;;;;;;;;;;;;;;;; + ;; update MAC + ;;;;;;;;;;;;;;;;; + mov edx, ebx + + movdqa xmm6, oword [ecx+edx] ; keys for whitening + add edx, 16 + + pxor xmm0, xmm4 ; MAC ^= dst[i] + pxor xmm0, xmm6 ; whitening (MAC) + + movdqa xmm6, oword [ecx+edx] ; pre load operation's keys + +align IPP_ALIGN_FACTOR +.auth_loop: + aesenc xmm0, xmm6 ; regular round (MAC) + movdqa xmm6, oword [ecx+edx+16] + add edx, 16 + jnz .auth_loop + aesenclast xmm0, xmm6 ; irregular round (MAC) + + + mov edx, ebx + add esi, BYTES_PER_BLK + add edi, BYTES_PER_BLK + sub dword len, BYTES_PER_BLK + jnz .blk_loop + + mov eax, pLocCtx + movdqu oword [eax], xmm0 ; update MAC value + movdqu oword [eax+sizeof(oword)], xmm6 ; update ENC(Ctri) + + REST_GPR + ret +ENDFUNC DecryptAuth_RIJ128_AES_NI + +%endif + + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128cmacg9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128cmacg9as.asm new file mode 100644 index 0000000..f9cdfc8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128cmacg9as.asm @@ -0,0 +1,117 @@ +;=============================================================================== +; Copyright (C) 2018 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Cipher function +; +; Content: +; cpAESCMAC_Update_AES_NI() +; +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + + +;*************************************************************** +;* Purpose: AES-CMAC update +;* +;* void cpAESCMAC_Update_AES_NI(Ipp8u* digest, +;* const Ipp8u* input, +;* int inpLen, +;* int nr, +;* const Ipp32u* pRKey) +;*************************************************************** + +;%if (_IPP >= _IPP_P8) && (_IPP < _IPP_G9) +%if (_IPP >= _IPP_P8) +;; +;; Lib = P8 +;; +;; Caller = ippsAES_CMACUpdate +;; + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR +IPPASM cpAESCMAC_Update_AES_NI,PUBLIC + USES_GPR esi,edi + +%xdefine pDigest [esp + ARG_1 + 0*sizeof(dword)] ; input/output digest +%xdefine pInpBlk [esp + ARG_1 + 1*sizeof(dword)] ; input blocks +%xdefine len [esp + ARG_1 + 2*sizeof(dword)] ; length (bytes) +%xdefine nr [esp + ARG_1 + 3*sizeof(dword)] ; number of rounds +%xdefine pKey [esp + ARG_1 + 4*sizeof(dword)] ; key material address + +%xdefine SC (4) +%assign BYTES_PER_BLK (16) + + mov edi, pDigest ; pointer to digest + mov esi,pInpBlk ; input data address + mov ecx,pKey ; key material address + mov eax,nr ; number of rounds + + movdqu xmm0, oword [edi] ; digest + + mov edx, len ; length of stream + +align IPP_ALIGN_FACTOR +;; +;; block-by-block processing +;; +.blks_loop: + movdqu xmm1, oword [esi] ; input block + + movdqa xmm4, oword [ecx] ; preload key material + + pxor xmm0, xmm1 ; digest ^ src[] + + pxor xmm0, xmm4 ; whitening + + movdqa xmm4, oword [ecx+16] ; preload key material + add ecx, 16 + + sub eax, 1 ; counter depending on key length +align IPP_ALIGN_FACTOR +.cipher_loop: + aesenc xmm0, xmm4 ; regular round + movdqa xmm4, oword [ecx+16] + add ecx, 16 + sub eax, 1 + jnz .cipher_loop + aesenclast xmm0, xmm4 ; irregular round + + mov ecx, pKey ; restore key pointer + mov eax, nr ; resrore number of rounds + + add esi, BYTES_PER_BLK ; advance pointers + sub edx, BYTES_PER_BLK ; decrease counter + jnz .blks_loop + + pxor xmm4, xmm4 + movdqu oword [edi], xmm0 ; store output block + + REST_GPR + ret +ENDFUNC cpAESCMAC_Update_AES_NI +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128decryptcbcpipeg9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128decryptcbcpipeg9as.asm new file mode 100644 index 0000000..cbf5b80 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128decryptcbcpipeg9as.asm @@ -0,0 +1,208 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Inverse Cipher function +; +; Content: +; DecryptCBC_RIJ128pipe_AES_NI() +; +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + + +;*************************************************************** +;* Purpose: pipelined RIJ128 CBC decryption +;* +;* void DecryptCBC_RIJ128pipe_AES_NI(const Ipp32u* inpBlk, +;* Ipp32u* outBlk, +;* int nr, +;* const Ipp32u* pRKey, +;* int len, +;* const Ipp8u* pIV) +;*************************************************************** + +;%if (_IPP >= _IPP_P8) && (_IPP < _IPP_G9) +%if (_IPP >= _IPP_P8) +;; +;; Lib = G9 +;; +;; Caller = ippsRijndael128DecryptCBC +;; + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR +IPPASM DecryptCBC_RIJ128pipe_AES_NI,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pInpBlk [ebp + ARG_1 + 0*sizeof(dword)] ; input block address +%xdefine pOutBlk [ebp + ARG_1 + 1*sizeof(dword)] ; output block address +%xdefine nr [ebp + ARG_1 + 2*sizeof(dword)] ; number of rounds +%xdefine pKey [ebp + ARG_1 + 3*sizeof(dword)] ; key material address +%xdefine len [ebp + ARG_1 + 4*sizeof(dword)] ; length(byte) +%xdefine pIV [ebp + ARG_1 + 5*sizeof(dword)] ; IV + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) +%assign BYTES_PER_BLK (16) +%assign BYTES_PER_LOOP (BYTES_PER_BLK*BLKS_PER_LOOP) + + mov esi,pInpBlk ; input data address + mov edi,pOutBlk ; output data address + mov ecx,pKey ; key material address + mov eax,nr ; number of rounds + + sub esp, 16*(1+4+1) ; allocate stack + lea edx, [esp+16] + and edx, -16 + + mov ebx, pIV + movdqu xmm4, oword [ebx] ; save IV + movdqa oword [edx+0*16], xmm4 ; into the stack + + sub dword len, BYTES_PER_LOOP + jl .short_input + + lea ebx,[eax*SC] ; keys offset + lea ecx,[ecx+ebx*4] +;; +;; pipelined processing +;; +.blks_loop: + movdqa xmm4, oword [ecx] ; keys for whitening + lea ebx, [ecx-16] ; set pointer to the round's key material + + movdqu xmm0, oword [esi+0*BYTES_PER_BLK] ; get input blocks + movdqu xmm1, oword [esi+1*BYTES_PER_BLK] + movdqu xmm2, oword [esi+2*BYTES_PER_BLK] + movdqu xmm3, oword [esi+3*BYTES_PER_BLK] + movdqa oword [edx+1*16], xmm0 ; and save as IVx + movdqa oword [edx+2*16], xmm1 ; for next operations + movdqa oword [edx+3*16], xmm2 ; into the stack + movdqa oword [edx+4*16], xmm3 + + pxor xmm0, xmm4 ;whitening + pxor xmm1, xmm4 + pxor xmm2, xmm4 + pxor xmm3, xmm4 + + movdqa xmm4, oword [ebx] ; pre load operation's keys + sub ebx, 16 + + mov eax, nr ; counter depending on key length + sub eax, 1 +.cipher_loop: + aesdec xmm0, xmm4 ; regular round + aesdec xmm1, xmm4 + aesdec xmm2, xmm4 + aesdec xmm3, xmm4 + movdqa xmm4, oword [ebx] ; pre load operation's keys + sub ebx, 16 + dec eax + jnz .cipher_loop + + aesdeclast xmm0, xmm4 ; irregular round and IV + pxor xmm0, oword [edx+0*16] ; xor with IV + movdqu oword [edi+0*16], xmm0 ; and store output blocls + + aesdeclast xmm1, xmm4 + pxor xmm1, oword [edx+1*16] + movdqu oword [edi+1*16], xmm1 + + aesdeclast xmm2, xmm4 + pxor xmm2, oword [edx+2*16] + movdqu oword [edi+2*16], xmm2 + + aesdeclast xmm3, xmm4 + pxor xmm3, oword [edx+3*16] + movdqu oword [edi+3*16], xmm3 + + movdqa xmm4, oword [edx+4*16] ; update IV + movdqa oword [edx+0*16], xmm4 + + add esi, BYTES_PER_LOOP + add edi, BYTES_PER_LOOP + sub dword len, BYTES_PER_LOOP + jge .blks_loop + +;; +;; block-by-block processing +;; +.short_input: + add dword len, BYTES_PER_LOOP + jz .quit + + mov eax, nr + mov ecx, pKey + lea ebx,[eax*SC] ; set pointer to the key material + lea ebx,[ecx+ebx*4] + +.single_blk_loop: + movdqu xmm0, oword [esi] ; get input block + movdqa oword [edx+16], xmm0 ; and save as IV for future + pxor xmm0, oword [ebx] ; whitening + + cmp eax,12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + +.key_256_s: + aesdec xmm0, oword [ecx+9*SC*4+4*SC*4] + aesdec xmm0, oword [ecx+9*SC*4+3*SC*4] +.key_192_s: + aesdec xmm0, oword [ecx+9*SC*4+2*SC*4] + aesdec xmm0, oword [ecx+9*SC*4+1*SC*4] +.key_128_s: + aesdec xmm0, oword [ecx+9*SC*4-0*SC*4] + aesdec xmm0, oword [ecx+9*SC*4-1*SC*4] + aesdec xmm0, oword [ecx+9*SC*4-2*SC*4] + aesdec xmm0, oword [ecx+9*SC*4-3*SC*4] + aesdec xmm0, oword [ecx+9*SC*4-4*SC*4] + aesdec xmm0, oword [ecx+9*SC*4-5*SC*4] + aesdec xmm0, oword [ecx+9*SC*4-6*SC*4] + aesdec xmm0, oword [ecx+9*SC*4-7*SC*4] + aesdec xmm0, oword [ecx+9*SC*4-8*SC*4] + aesdeclast xmm0, oword [ecx+9*SC*4-9*SC*4] + + pxor xmm0, oword [edx+0*16] ; add IV + movdqu oword [edi], xmm0 ; and save output blocl + + movdqa xmm4, oword [edx+1*16] ; update IV + movdqa oword [edx+0*16], xmm4 + + add esi, BYTES_PER_BLK + add edi, BYTES_PER_BLK + sub dword len, BYTES_PER_BLK + jnz .single_blk_loop + +.quit: + add esp, 16*(1+4+1) ; free stack + REST_GPR + ret +ENDFUNC DecryptCBC_RIJ128pipe_AES_NI +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128decryptcfbpipeg9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128decryptcfbpipeg9as.asm new file mode 100644 index 0000000..f7f1110 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128decryptcfbpipeg9as.asm @@ -0,0 +1,581 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Inverse Cipher function +; +; Content: +; Decrypt_RIJ128_AES_NI() +; +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + + +%macro COPY_8U 4.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + %xdefine %%limit %3 + %xdefine %%tmp %4 + + xor ecx, ecx +%%next_byte: + mov %%tmp, byte [%%src+ecx] + mov byte [%%dst+ecx], %%tmp + add ecx, 1 + cmp ecx, %%limit + jl %%next_byte +%endmacro + + +%macro COPY_32U 4.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + %xdefine %%limit %3 + %xdefine %%tmp %4 + + xor ecx, ecx +%%next_dword: + mov %%tmp, dword [%%src+ecx] + mov dword [%%dst+ecx], %%tmp + add ecx, 4 + cmp ecx, %%limit + jl %%next_dword +%endmacro + + +%macro COPY_128U 4.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + %xdefine %%limit %3 + %xdefine %%tmp %4 + + xor ecx, ecx +%%next_oword: + movdqu %%tmp, oword [%%src+ecx] + movdqu oword [%%dst+ecx], %%tmp + add ecx, 16 + cmp ecx, %%limit + jl %%next_oword +%endmacro + + +;*************************************************************** +;* Purpose: pipelined RIJ128 CFB decryption +;* +;* void DecryptCFB_RIJ128pipe_AES_NI(const Ipp32u* inpBlk, +;* Ipp32u* outBlk, +;* int nr, +;* const Ipp32u* pRKey, +;* int cfbBlks, +;* int cfbSize, +;* const Ipp8u* pIV) +;*************************************************************** + +;%if (_IPP >= _IPP_P8) && (_IPP < _IPP_G9) +%if (_IPP >= _IPP_P8) +;; +;; Lib = P8 +;; +;; Caller = ippsRijndael128DecryptCFB +;; + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR +IPPASM DecryptCFB_RIJ128pipe_AES_NI,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pInpBlk [ebp + ARG_1 + 0*sizeof(dword)] ; input block address +%xdefine pOutBlk [ebp + ARG_1 + 1*sizeof(dword)] ; output block address +%xdefine nr [ebp + ARG_1 + 2*sizeof(dword)] ; number of rounds +%xdefine pKey [ebp + ARG_1 + 3*sizeof(dword)] ; key material address +%xdefine cfbBlks [ebp + ARG_1 + 4*sizeof(dword)] ; length of stream in cfbSize +%xdefine cfbSize [ebp + ARG_1 + 5*sizeof(dword)] ; cfb blk size +%xdefine pIV [ebp + ARG_1 + 6*sizeof(dword)] ; IV + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) + + sub esp,16*(1+4+4) ; allocate stask: + ; +0*16 IV + ; +1*16 inp0, inp1, inp2, inp3 + ; +5*16 out0, out1, out2, out3 + + mov eax, pIV ; IV address + movdqu xmm4, oword [eax] ; get IV + movdqu oword [esp+0*16], xmm4 ; into the stack + + mov esi,pInpBlk ; input data address + mov edi,pOutBlk ; output data address + mov edx,cfbSize ; size of block + + sub dword cfbBlks, BLKS_PER_LOOP + jl .short_input + +;; +;; pipelined processing +;; +.blks_loop: + lea eax, [edx*BLKS_PER_LOOP] + COPY_32U {esp+16}, esi, eax, ebx ; move 4 input blocks to stack + + mov ecx, pKey + movdqa xmm4, oword [ecx] ; keys for whitening + + lea ebx, [edx+edx*2] + movdqu xmm0, oword [esp] ; get encoded blocks + movdqu xmm1, oword [esp+edx] + movdqu xmm2, oword [esp+edx*2] + movdqu xmm3, oword [esp+ebx] + + lea ebx, [ecx+16] ; pointer to the round's key material + + pxor xmm0, xmm4 ; whitening + pxor xmm1, xmm4 + pxor xmm2, xmm4 + pxor xmm3, xmm4 + + movdqa xmm4, oword [ebx] ; pre load operation's keys + add ebx, 16 + + mov eax, nr ; counter depending on key length + sub eax, 1 +.cipher_loop: + aesenc xmm0, xmm4 ; regular round + aesenc xmm1, xmm4 + aesenc xmm2, xmm4 + aesenc xmm3, xmm4 + movdqa xmm4, oword [ebx] ; pre load operation's keys + add ebx, 16 + dec eax + jnz .cipher_loop + + aesenclast xmm0, xmm4 ; irregular round and IV + aesenclast xmm1, xmm4 + aesenclast xmm2, xmm4 + aesenclast xmm3, xmm4 + + lea ebx, [edx+edx*2] ; get src blocks from the stack + movdqu xmm4, oword [esp+16] + movdqu xmm5, oword [esp+16+edx] + movdqu xmm6, oword [esp+16+edx*2] + movdqu xmm7, oword [esp+16+ebx] + + pxor xmm0, xmm4 ; xor src + movdqu oword [esp+5*16],xmm0;and store into the stack + pxor xmm1, xmm5 + movdqu oword [esp+5*16+edx], xmm1 + pxor xmm2, xmm6 + movdqu oword [esp+5*16+edx*2], xmm2 + pxor xmm3, xmm7 + movdqu oword [esp+5*16+ebx], xmm3 + + lea eax, [edx*BLKS_PER_LOOP] + COPY_32U edi, {esp+5*16}, eax, ebx ; move 4 blocks to output + + movdqu xmm0, oword [esp+eax] ; update IV + movdqu oword [esp], xmm0 + + + add esi, eax + add edi, eax + sub dword cfbBlks, BLKS_PER_LOOP + jge .blks_loop + +;; +;; block-by-block processing +;; +.short_input: + add dword cfbBlks, BLKS_PER_LOOP + jz .quit + + lea ebx, [edx*2] + lea ecx, [edx+edx*2] + cmp dword cfbBlks, 2 + cmovl ebx, edx + cmovg ebx, ecx + COPY_8U {esp+16}, esi, ebx, al ; move recent input blocks to stack + + ; get actual address of key material: pRKeys += (nr-9) * SC + mov ecx, pKey + mov eax, nr + lea esi,[eax*4] + lea esi, [ecx+esi*4-9*(SC)*4] ; AES-128 round keys + + xor eax, eax ; index +.single_blk_loop: + movdqu xmm0, oword [esp+eax] ; get encoded block + + pxor xmm0, oword [ecx] ; whitening + + cmp dword nr, 12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + +.key_256_s: + aesenc xmm0, oword [esi-4*4*SC] + aesenc xmm0, oword [esi-3*4*SC] +.key_192_s: + aesenc xmm0, oword [esi-2*4*SC] + aesenc xmm0, oword [esi-1*4*SC] +.key_128_s: + aesenc xmm0, oword [esi+0*4*SC] + aesenc xmm0, oword [esi+1*4*SC] + aesenc xmm0, oword [esi+2*4*SC] + aesenc xmm0, oword [esi+3*4*SC] + aesenc xmm0, oword [esi+4*4*SC] + aesenc xmm0, oword [esi+5*4*SC] + aesenc xmm0, oword [esi+6*4*SC] + aesenc xmm0, oword [esi+7*4*SC] + aesenc xmm0, oword [esi+8*4*SC] + aesenclast xmm0, oword [esi+9*4*SC] + + movdqu xmm1, oword [esp+eax+16] ; get input block from the stack + pxor xmm0, xmm1 ; xor src + movdqu oword [esp+5*16+eax], xmm0 ; and save output + + add eax, edx + dec dword cfbBlks + jnz .single_blk_loop + + COPY_8U edi, {esp+5*16}, ebx, al ; copy rest output from the stack + +.quit: + add esp, 16*(1+4+4) ; free stack + REST_GPR + ret +ENDFUNC DecryptCFB_RIJ128pipe_AES_NI + +align IPP_ALIGN_FACTOR +IPPASM DecryptCFB32_RIJ128pipe_AES_NI,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pInpBlk [ebp + ARG_1 + 0*sizeof(dword)] ; input block address +%xdefine pOutBlk [ebp + ARG_1 + 1*sizeof(dword)] ; output block address +%xdefine nr [ebp + ARG_1 + 2*sizeof(dword)] ; number of rounds +%xdefine pKey [ebp + ARG_1 + 3*sizeof(dword)] ; key material address +%xdefine cfbBlks [ebp + ARG_1 + 4*sizeof(dword)] ; length of stream in cfbSize +%xdefine cfbSize [ebp + ARG_1 + 5*sizeof(dword)] ; cfb blk size (4 bytes multible) +%xdefine pIV [ebp + ARG_1 + 6*sizeof(dword)] ; IV + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) + + sub esp,16*(1+4+4) ; allocate stask: + ; +0*16 IV + ; +1*16 inp0, inp1, inp2, inp3 + ; +5*16 out0, out1, out2, out3 + + mov eax, pIV ; IV address + movdqu xmm4, oword [eax] ; get IV + movdqu oword [esp+0*16], xmm4 ; into the stack + + mov esi,pInpBlk ; input data address + mov edi,pOutBlk ; output data address + mov edx,cfbSize ; size of block + + sub dword cfbBlks, BLKS_PER_LOOP + jl .short_input + +;; +;; pipelined processing +;; +.blks_loop: + lea eax, [edx*BLKS_PER_LOOP] + COPY_128U {esp+16}, esi, eax, xmm0 ; move 4 input blocks to stack + + mov ecx, pKey + movdqa xmm4, oword [ecx] ; keys for whitening + + lea ebx, [edx+edx*2] + movdqu xmm0, oword [esp] ; get encoded blocks + movdqu xmm1, oword [esp+edx] + movdqu xmm2, oword [esp+edx*2] + movdqu xmm3, oword [esp+ebx] + + lea ebx, [ecx+16] ; pointer to the round's key material + + pxor xmm0, xmm4 ; whitening + pxor xmm1, xmm4 + pxor xmm2, xmm4 + pxor xmm3, xmm4 + + movdqa xmm4, oword [ebx] ; pre load operation's keys + add ebx, 16 + + mov eax, nr ; counter depending on key length + sub eax, 1 +.cipher_loop: + aesenc xmm0, xmm4 ; regular round + aesenc xmm1, xmm4 + aesenc xmm2, xmm4 + aesenc xmm3, xmm4 + movdqa xmm4, oword [ebx] ; pre load operation's keys + add ebx, 16 + dec eax + jnz .cipher_loop + + aesenclast xmm0, xmm4 ; irregular round and IV + aesenclast xmm1, xmm4 + aesenclast xmm2, xmm4 + aesenclast xmm3, xmm4 + + lea ebx, [edx+edx*2] ; get src blocks from the stack + movdqu xmm4, oword [esp+16] + movdqu xmm5, oword [esp+16+edx] + movdqu xmm6, oword [esp+16+edx*2] + movdqu xmm7, oword [esp+16+ebx] + + pxor xmm0, xmm4 ; xor src + movdqu oword [esp+5*16],xmm0;and store into the stack + pxor xmm1, xmm5 + movdqu oword [esp+5*16+edx], xmm1 + pxor xmm2, xmm6 + movdqu oword [esp+5*16+edx*2], xmm2 + pxor xmm3, xmm7 + movdqu oword [esp+5*16+ebx], xmm3 + + lea eax, [edx*BLKS_PER_LOOP] + COPY_128U edi, {esp+5*16}, eax, xmm0 ; move 4 blocks to output + + movdqu xmm0, oword [esp+eax] ; update IV + movdqu oword [esp], xmm0 + + + add esi, eax + add edi, eax + sub dword cfbBlks, BLKS_PER_LOOP + jge .blks_loop + +;; +;; block-by-block processing +;; +.short_input: + add dword cfbBlks, BLKS_PER_LOOP + jz .quit + + lea ebx, [edx*2] + lea ecx, [edx+edx*2] + cmp dword cfbBlks, 2 + cmovl ebx, edx + cmovg ebx, ecx + COPY_32U {esp+16}, esi, ebx, eax ; move recent input blocks to stack + + ; get actual address of key material: pRKeys += (nr-9) * SC + mov ecx, pKey + mov eax, nr + lea esi,[eax*4] + lea esi, [ecx+esi*4-9*(SC)*4] ; AES-128 round keys + + xor eax, eax ; index +.single_blk_loop: + movdqu xmm0, oword [esp+eax] ; get encoded block + + pxor xmm0, oword [ecx] ; whitening + + cmp dword nr, 12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + +.key_256_s: + aesenc xmm0, oword [esi-4*4*SC] + aesenc xmm0, oword [esi-3*4*SC] +.key_192_s: + aesenc xmm0, oword [esi-2*4*SC] + aesenc xmm0, oword [esi-1*4*SC] +.key_128_s: + aesenc xmm0, oword [esi+0*4*SC] + aesenc xmm0, oword [esi+1*4*SC] + aesenc xmm0, oword [esi+2*4*SC] + aesenc xmm0, oword [esi+3*4*SC] + aesenc xmm0, oword [esi+4*4*SC] + aesenc xmm0, oword [esi+5*4*SC] + aesenc xmm0, oword [esi+6*4*SC] + aesenc xmm0, oword [esi+7*4*SC] + aesenc xmm0, oword [esi+8*4*SC] + aesenclast xmm0, oword [esi+9*4*SC] + + movdqu xmm1, oword [esp+eax+16] ; get input block from the stack + pxor xmm0, xmm1 ; xor src + movdqu oword [esp+5*16+eax], xmm0 ; and save output + + add eax, edx + dec dword cfbBlks + jnz .single_blk_loop + + COPY_32U edi, {esp+5*16}, ebx, eax ; copy rest output from the stack + +.quit: + add esp, 16*(1+4+4) ; free stack + REST_GPR + ret +ENDFUNC DecryptCFB32_RIJ128pipe_AES_NI + +;; +;; Lib = G9 +;; +;; Caller = ippsRijndael128DecryptCFB +;; +align IPP_ALIGN_FACTOR +IPPASM DecryptCFB128_RIJ128pipe_AES_NI,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pInpBlk [ebp + ARG_1 + 0*sizeof(dword)] ; input block address +%xdefine pOutBlk [ebp + ARG_1 + 1*sizeof(dword)] ; output block address +%xdefine nr [ebp + ARG_1 + 2*sizeof(dword)] ; number of rounds +%xdefine pKey [ebp + ARG_1 + 3*sizeof(dword)] ; key material address +%xdefine len [ebp + ARG_1 + 4*sizeof(dword)] ; length of stream in bytes +%xdefine pIV [ebp + ARG_1 + 5*sizeof(dword)] ; IV + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) +%assign BYTES_PER_BLK (16) +%assign BYTES_PER_LOOP (BYTES_PER_BLK*BLKS_PER_LOOP) + + mov esi,pInpBlk ; input data address + mov edi,pOutBlk ; output data address + mov ecx,pKey ; keys + mov edx, len + + mov eax, pIV + movdqu xmm0, oword [eax] ; get IV + + sub edx, BYTES_PER_LOOP ; test length of the stream + jl .short_input + +;; +;; pipelined processing +;; +.blks_loop: + movdqa xmm7, oword [ecx] ; keys for whitening + lea ebx, [ecx+16] ; pointer to the round's key material + + movdqu xmm1, oword [esi+0*BYTES_PER_BLK] ; get another encoded cblocks + movdqu xmm2, oword [esi+1*BYTES_PER_BLK] + movdqu xmm3, oword [esi+2*BYTES_PER_BLK] + + pxor xmm0, xmm7 ; whitening + pxor xmm1, xmm7 + pxor xmm2, xmm7 + pxor xmm3, xmm7 + + movdqa xmm7, oword [ebx] ; pre load operation's keys + add ebx, 16 + + mov eax, nr ; counter depending on key length + sub eax, 1 +.cipher_loop: + aesenc xmm0, xmm7 ; regular round + aesenc xmm1, xmm7 + aesenc xmm2, xmm7 + aesenc xmm3, xmm7 + movdqa xmm7, oword [ebx] ; pre load operation's keys + add ebx, 16 + dec eax + jnz .cipher_loop + + aesenclast xmm0, xmm7 ; irregular round and IV for 4 input blocks + movdqu xmm4, oword [esi+0*BYTES_PER_BLK] + aesenclast xmm1, xmm7 + movdqu xmm5, oword [esi+1*BYTES_PER_BLK] + aesenclast xmm2, xmm7 + movdqu xmm6, oword [esi+2*BYTES_PER_BLK] + aesenclast xmm3, xmm7 + movdqu xmm7, oword [esi+3*BYTES_PER_BLK] + add esi, BYTES_PER_LOOP + + pxor xmm0, xmm4 ; 4 output blocks + movdqu oword [edi+0*BYTES_PER_BLK], xmm0 + pxor xmm1, xmm5 + movdqu oword [edi+1*BYTES_PER_BLK], xmm1 + pxor xmm2, xmm6 + movdqu oword [edi+2*BYTES_PER_BLK], xmm2 + pxor xmm3, xmm7 + movdqu oword [edi+3*BYTES_PER_BLK], xmm3 + add edi, BYTES_PER_LOOP + + movdqa xmm0, xmm7 ; update IV + sub edx, BYTES_PER_LOOP + jge .blks_loop + +;; +;; block-by-block processing +;; +.short_input: + add edx, BYTES_PER_LOOP + jz .quit + + ; get actual address of key material: pRKeys += (nr-9) * SC + mov eax, nr + lea ebx,[eax*4] + lea ebx, [ecx+ebx*4-9*(SC)*4] ; AES-128 round keys + +.single_blk_loop: + pxor xmm0, oword [ecx] ; whitening + + cmp eax,12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + +.key_256_s: + aesenc xmm0, oword [ebx-4*4*SC] + aesenc xmm0, oword [ebx-3*4*SC] +.key_192_s: + aesenc xmm0, oword [ebx-2*4*SC] + aesenc xmm0, oword [ebx-1*4*SC] +.key_128_s: + aesenc xmm0, oword [ebx+0*4*SC] + aesenc xmm0, oword [ebx+1*4*SC] + aesenc xmm0, oword [ebx+2*4*SC] + aesenc xmm0, oword [ebx+3*4*SC] + aesenc xmm0, oword [ebx+4*4*SC] + aesenc xmm0, oword [ebx+5*4*SC] + aesenc xmm0, oword [ebx+6*4*SC] + aesenc xmm0, oword [ebx+7*4*SC] + aesenc xmm0, oword [ebx+8*4*SC] + aesenclast xmm0, oword [ebx+9*4*SC] + + movdqu xmm1, oword [esi] ; input block from the stream + add esi, BYTES_PER_BLK + + pxor xmm0, xmm1 ; xor src + movdqu oword [edi], xmm0 ; and save output + add edi, BYTES_PER_BLK + + movdqa xmm0, xmm1 ; update IV + sub edx, BYTES_PER_BLK + jnz .single_blk_loop + +.quit: + REST_GPR + ret +ENDFUNC DecryptCFB128_RIJ128pipe_AES_NI +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128decryptecbpipeg9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128decryptecbpipeg9as.asm new file mode 100644 index 0000000..431a844 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128decryptecbpipeg9as.asm @@ -0,0 +1,176 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Inverse Cipher function +; +; Content: +; DecryptECB_RIJ128pipe_AES_NI() +; +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +;*************************************************************** +;* Purpose: pipelined RIJ128 ECB decryption +;* +;* void DecryptECB_RIJ128pipe_AES_NI(const Ipp32u* inpBlk, +;* Ipp32u* outBlk, +;* int nr, +;* const Ipp32u* pRKey, +;* int len) +;*************************************************************** + +;%if (_IPP >= _IPP_P8) && (_IPP < _IPP_G9) +%if (_IPP >= _IPP_P8) +;; +;; Lib = P8 +;; +;; Caller = ippsRijndael128DecryptECB +;; + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR +IPPASM DecryptECB_RIJ128pipe_AES_NI,PUBLIC + USES_GPR esi,edi,ebx + +%xdefine pInpBlk [esp + ARG_1 + 0*sizeof(dword)] ; input block address +%xdefine pOutBlk [esp + ARG_1 + 1*sizeof(dword)] ; output block address +%xdefine nr [esp + ARG_1 + 2*sizeof(dword)] ; number of rounds +%xdefine pKey [esp + ARG_1 + 3*sizeof(dword)] ; key material address +%xdefine len [esp + ARG_1 + 4*sizeof(dword)] ; length(byte) + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) +%assign BYTES_PER_BLK (16) +%assign BYTES_PER_LOOP (BYTES_PER_BLK*BLKS_PER_LOOP) + + mov esi,pInpBlk ; input data address + mov edi,pOutBlk ; output data address + mov ecx,pKey ; key material address + mov edx,len ; length + mov eax,nr ; number of rounds + + sub edx, BYTES_PER_LOOP + jl .short_input + + lea ebx,[eax*SC] ; keys offset + lea ecx,[ecx+ebx*4] +;; +;; pipelined processing +;; +.blks_loop: + movdqa xmm4, oword [ecx] ; keys for whitening + lea ebx, [ecx-16] ; set pointer to the round's key material + + movdqu xmm0, oword [esi+0*BYTES_PER_BLK] ; get input blocks + movdqu xmm1, oword [esi+1*BYTES_PER_BLK] + movdqu xmm2, oword [esi+2*BYTES_PER_BLK] + movdqu xmm3, oword [esi+3*BYTES_PER_BLK] + add esi, BYTES_PER_LOOP + + pxor xmm0, xmm4 ; whitening + pxor xmm1, xmm4 + pxor xmm2, xmm4 + pxor xmm3, xmm4 + + movdqa xmm4, oword [ebx] ; pre load round keys + sub ebx, 16 + + mov eax, nr ; counter depending on key length + sub eax, 1 +.cipher_loop: + aesdec xmm0, xmm4 ; regular round + aesdec xmm1, xmm4 + aesdec xmm2, xmm4 + aesdec xmm3, xmm4 + movdqa xmm4, oword [ebx] ; pre load operation's keys + sub ebx, 16 + dec eax + jnz .cipher_loop + + aesdeclast xmm0, xmm4 ; irregular round + movdqu oword [edi+0*BYTES_PER_BLK], xmm0 ; store output blocks + aesdeclast xmm1, xmm4 + movdqu oword [edi+1*BYTES_PER_BLK], xmm1 + aesdeclast xmm2, xmm4 + movdqu oword [edi+2*BYTES_PER_BLK], xmm2 + aesdeclast xmm3, xmm4 + movdqu oword [edi+3*BYTES_PER_BLK], xmm3 + + add edi, BYTES_PER_LOOP + sub edx, BYTES_PER_LOOP + jge .blks_loop + +;; +;; block-by-block processing +;; +.short_input: + add edx, BYTES_PER_LOOP + jz .quit + + mov eax, nr + mov ecx, pKey + lea ebx,[eax*SC] ; set pointer to the key material + lea ebx,[ecx+ebx*4] + +.single_blk_loop: + movdqu xmm0, oword [esi] ; get input block + add esi, BYTES_PER_BLK + pxor xmm0, oword [ebx] ; whitening + + cmp eax,12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + +.key_256_s: + aesdec xmm0, oword [ecx+9*SC*4+4*SC*4] + aesdec xmm0, oword [ecx+9*SC*4+3*SC*4] +.key_192_s: + aesdec xmm0, oword [ecx+9*SC*4+2*SC*4] + aesdec xmm0, oword [ecx+9*SC*4+1*SC*4] +.key_128_s: + aesdec xmm0, oword [ecx+9*SC*4-0*SC*4] + aesdec xmm0, oword [ecx+9*SC*4-1*SC*4] + aesdec xmm0, oword [ecx+9*SC*4-2*SC*4] + aesdec xmm0, oword [ecx+9*SC*4-3*SC*4] + aesdec xmm0, oword [ecx+9*SC*4-4*SC*4] + aesdec xmm0, oword [ecx+9*SC*4-5*SC*4] + aesdec xmm0, oword [ecx+9*SC*4-6*SC*4] + aesdec xmm0, oword [ecx+9*SC*4-7*SC*4] + aesdec xmm0, oword [ecx+9*SC*4-8*SC*4] + aesdeclast xmm0, oword [ecx+9*SC*4-9*SC*4] + + movdqu oword [edi], xmm0 ; save output block + add edi, BYTES_PER_BLK + + sub edx, BYTES_PER_BLK + jnz .single_blk_loop + +.quit: + REST_GPR + ret +ENDFUNC DecryptECB_RIJ128pipe_AES_NI +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128decryptg9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128decryptg9as.asm new file mode 100644 index 0000000..29512c9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128decryptg9as.asm @@ -0,0 +1,114 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Inverse Cipher function +; +; Content: +; Decrypt_RIJ128_AES_NI() +; +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + + +;*************************************************************** +;* Purpose: single block RIJ128 Inverse Cipher +;* +;* void Encrypt_RIJ128(const Ipp32u* inpBlk, +;* Ipp32u* outBlk, +;* int nr, +;* const Ipp32u* pRKey, +;* const Ipp32u Tables[][256]) +;* +;*************************************************************** + +;%if (_IPP >= _IPP_P8) && (_IPP < _IPP_G9) +%if (_IPP >= _IPP_P8) +;; +;; Lib = P8 +;; +;; Caller = ippsRijndael128DecryptECB +;; Caller = ippsRijndael128DecryptCBC +;; + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR +IPPASM Decrypt_RIJ128_AES_NI,PUBLIC + USES_GPR esi,edi + +%xdefine pInpBlk [esp + ARG_1 + 0*sizeof(dword)] ; input block address +%xdefine pOutBlk [esp + ARG_1 + 1*sizeof(dword)] ; output block address +%xdefine nr [esp + ARG_1 + 2*sizeof(dword)] ; number of rounds +%xdefine pKey [esp + ARG_1 + 3*sizeof(dword)] ; key material address + +%xdefine SC (4) + + + mov esi,pInpBlk ; input data address + mov ecx,pKey ; key material address + mov eax,nr ; number of rounds + mov edi,pOutBlk ; output data address + + lea edx,[eax*4] + + movdqu xmm0, oword [esi] ; input block + + ;;whitening + pxor xmm0, oword [ecx+edx*4] ; whitening + + cmp eax,12 ; switch according to number of rounds + jl .key_128 + jz .key_192 + + ;; + ;; regular rounds + ;; +.key_256: + aesdec xmm0,oword [ecx+9*SC*4+4*4*SC] + aesdec xmm0,oword [ecx+9*SC*4+3*4*SC] +.key_192: + aesdec xmm0,oword [ecx+9*SC*4+2*4*SC] + aesdec xmm0,oword [ecx+9*SC*4+1*4*SC] +.key_128: + aesdec xmm0,oword [ecx+9*SC*4-0*4*SC] + aesdec xmm0,oword [ecx+9*SC*4-1*4*SC] + aesdec xmm0,oword [ecx+9*SC*4-2*4*SC] + aesdec xmm0,oword [ecx+9*SC*4-3*4*SC] + aesdec xmm0,oword [ecx+9*SC*4-4*4*SC] + aesdec xmm0,oword [ecx+9*SC*4-5*4*SC] + aesdec xmm0,oword [ecx+9*SC*4-6*4*SC] + aesdec xmm0,oword [ecx+9*SC*4-7*4*SC] + aesdec xmm0,oword [ecx+9*SC*4-8*4*SC] + ;; + ;; last rounds + ;; + aesdeclast xmm0,oword [ecx+9*SC*4-9*4*SC] + + movdqu oword [edi], xmm0 ; output block + REST_GPR + ret +ENDFUNC Decrypt_RIJ128_AES_NI +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128decryptxtsg9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128decryptxtsg9as.asm new file mode 100644 index 0000000..2d2294d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128decryptxtsg9as.asm @@ -0,0 +1,265 @@ +;=============================================================================== +; Copyright (C) 2016 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; AES function +; +; Content: +; cpAESDecryptXTS_AES_NI() +; +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%macro MUL_X 4.nolist + %xdefine %%y %1 + %xdefine %%x %2 + %xdefine %%t %3 + %xdefine %%cnt %4 + + pxor %%t, %%t + movdqa %%y, %%x + pcmpgtq %%t, %%x + paddq %%y, %%y + palignr %%t, %%t, sizeof(qword) + pand %%t, %%cnt + pxor %%y, %%t +%endmacro + + +;*************************************************************** +;* Purpose: AES-XTS decryption +;* +;* void cpAESDecryptXTS_AES_NI(Ipp8u* outBlk, +;* const Ipp8u* inpBlk, +;* int length, +;* const Ipp8u* pRKey, +;* int nr, +;* const Ipp8u* pTweak) +;*************************************************************** + +%if (_IPP >= _IPP_P8) +;; +;; Lib = P8 +;; +;; Caller = ippsAESDecryptXTS +;; + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR +IPPASM cpAESDecryptXTS_AES_NI,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pOutBlk [ebp + ARG_1 + 0*sizeof(dword)] ; output block address +%xdefine pInpBlk [ebp + ARG_1 + 1*sizeof(dword)] ; input block address +%xdefine blks [ebp + ARG_1 + 2*sizeof(dword)] ; length(blocks) +%xdefine pKey [ebp + ARG_1 + 3*sizeof(dword)] ; key material address +%xdefine nr [ebp + ARG_1 + 4*sizeof(dword)] ; number of rounds +%xdefine pTweak [ebp + ARG_1 + 5*sizeof(dword)] ; tweaks + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) +%assign BYTES_PER_BLK (16) +%assign BYTES_PER_LOOP (BYTES_PER_BLK*BLKS_PER_LOOP) + + mov esi,pInpBlk ; input data address + mov edi,pOutBlk ; output data address + +; +; stack layout: +; +%assign _tweakCnt 0 ; constant +%assign _tweak0 _tweakCnt+sizeof(oword) ; * alpha +%assign _tweak1 _tweak0+sizeof(oword) ; * alpha^2 +%assign _tweak2 _tweak1+sizeof(oword) ; * alpha^3 +%assign _tweak3 _tweak2+sizeof(oword) ; * alpha^4 +%assign _tkey _tweak3+sizeof(oword) ; decryption keys (top) +%assign _bkey _tkey+sizeof(dword) ; decryption keys (bottom) +%assign _nr _bkey+sizeof(dword) ; # rounds +%assign _sp _nr+sizeof(dword) ; esp +%assign stackSize _sp+sizeof(dword) ; stack size + + mov eax, esp + sub esp, stackSize ; aligned stack + and esp, (-16) + mov dword [esp+_sp], eax ; store esp + + mov eax, pTweak + movdqu xmm4, oword [eax] ; initial tweak + movdqa xmm7, xmm4 + + mov eax, 087h ; create tweakCnt = {0x01:0x87} + mov ecx, 1 + movd xmm0, eax + movd xmm1, ecx + punpcklqdq xmm0, xmm1 + + mov ecx,pKey ; key material address + mov eax,nr ; num of rounds + mov edx,blks ; num of blocks + + ; move parameters to local stack frame + movdqa oword [esp+_tweakCnt], xmm0 + mov dword [esp+_tkey], ecx + lea ebx, [eax*SC] + lea ecx, [ecx+ebx*sizeof(dword)] + mov dword [esp+_bkey], ecx + mov dword [esp+_nr], eax + + sub edx, BLKS_PER_LOOP + jl .short_input + jmp .blks_loop_ep + +;; +;; pipelined processing +;; +.blks_loop: + MUL_X xmm4, xmm7, xmm1, xmm0 +.blks_loop_ep: + movdqa oword [esp+_tweak0], xmm4 + MUL_X xmm5, xmm4, xmm1, xmm0 + movdqa oword [esp+_tweak1], xmm5 + MUL_X xmm6, xmm5, xmm1, xmm0 + movdqa oword [esp+_tweak2], xmm6 + MUL_X xmm7, xmm6, xmm1, xmm0 + movdqa oword [esp+_tweak3], xmm7 + + movdqu xmm0, oword [esi+0*BYTES_PER_BLK] ; get input blocks + movdqu xmm1, oword [esi+1*BYTES_PER_BLK] + movdqu xmm2, oword [esi+2*BYTES_PER_BLK] + movdqu xmm3, oword [esi+3*BYTES_PER_BLK] + add esi, BYTES_PER_BLK*BLKS_PER_LOOP + + pxor xmm0, xmm4 ; tweak pre-whitening + pxor xmm1, xmm5 + pxor xmm2, xmm6 + pxor xmm3, xmm7 + + movdqa xmm4, oword [ecx] ; keys for whitening + lea ebx, [ecx-BYTES_PER_BLK]; pointer to the round's key material + + pxor xmm0, xmm4 ; whitening + pxor xmm1, xmm4 + pxor xmm2, xmm4 + pxor xmm3, xmm4 + + movdqa xmm4, oword [ebx] ; pre load round keys + sub ebx, BYTES_PER_BLK + sub eax, 1 +.cipher_loop: + aesdec xmm0, xmm4 ; regular round + aesdec xmm1, xmm4 + aesdec xmm2, xmm4 + aesdec xmm3, xmm4 + movdqa xmm4, oword [ebx] + sub ebx, BYTES_PER_BLK + dec eax + jnz .cipher_loop + + aesdeclast xmm0, xmm4 ; irregular round + aesdeclast xmm1, xmm4 + aesdeclast xmm2, xmm4 + aesdeclast xmm3, xmm4 + + movdqa xmm4, oword [esp+_tweak0] ; tweak post-whitening + movdqa xmm5, oword [esp+_tweak1] + movdqa xmm6, oword [esp+_tweak2] + movdqa xmm7, oword [esp+_tweak3] + pxor xmm0, xmm4 + pxor xmm1, xmm5 + pxor xmm2, xmm6 + pxor xmm3, xmm7 + + movdqu oword [edi+0*BYTES_PER_BLK], xmm0 ; store output blocks + movdqu oword [edi+1*BYTES_PER_BLK], xmm1 + movdqu oword [edi+2*BYTES_PER_BLK], xmm2 + movdqu oword [edi+3*BYTES_PER_BLK], xmm3 + add edi, BYTES_PER_BLK*BLKS_PER_LOOP + + movdqa xmm0, oword [esp+_tweakCnt] ; restore tweak const + mov eax, dword [esp+_nr] ; number of rounds + sub edx, BLKS_PER_LOOP + jge .blks_loop + +;; +;; block-by-block processing +;; +.short_input: + add edx, BLKS_PER_LOOP + jz .quit + + mov ecx, dword [esp+_tkey] + mov ebx, dword [esp+_bkey] + mov eax, dword [esp+_nr] + jmp .single_blk_loop_ep + +.single_blk_loop: + MUL_X xmm7, xmm7, xmm1, xmm0 + +.single_blk_loop_ep: + movdqu xmm1, oword [esi] ; input block + add esi, BYTES_PER_BLK + pxor xmm1, xmm7 ; tweak pre-whitening + pxor xmm1, oword [ebx] ; AES whitening + + cmp eax,12 ; switch according to number of rounds + jl .key_128_s + +.key_256_s: + aesdec xmm1,oword [ecx+9*SC*4+4*SC*4] + aesdec xmm1,oword [ecx+9*SC*4+3*SC*4] + aesdec xmm1,oword [ecx+9*SC*4+2*SC*4] + aesdec xmm1,oword [ecx+9*SC*4+1*SC*4] +.key_128_s: + aesdec xmm1,oword [ecx+9*SC*4-0*SC*4] + aesdec xmm1,oword [ecx+9*SC*4-1*SC*4] + aesdec xmm1,oword [ecx+9*SC*4-2*SC*4] + aesdec xmm1,oword [ecx+9*SC*4-3*SC*4] + aesdec xmm1,oword [ecx+9*SC*4-4*SC*4] + aesdec xmm1,oword [ecx+9*SC*4-5*SC*4] + aesdec xmm1,oword [ecx+9*SC*4-6*SC*4] + aesdec xmm1,oword [ecx+9*SC*4-7*SC*4] + aesdec xmm1,oword [ecx+9*SC*4-8*SC*4] + aesdeclast xmm1,oword [ecx+9*SC*4-9*SC*4] + + pxor xmm1, xmm7 ; tweak post-whitening + movdqu oword [edi], xmm1 ; output block + add edi, BYTES_PER_BLK + + sub edx, 1 + jnz .single_blk_loop + +.quit: + mov eax, pTweak ; save tweak + movdqu oword [eax], xmm7 + + mov esp, [esp+_sp] + REST_GPR + ret +ENDFUNC cpAESDecryptXTS_AES_NI +%endif + + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128encryptcbcg9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128encryptcbcg9as.asm new file mode 100644 index 0000000..4728b85 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128encryptcbcg9as.asm @@ -0,0 +1,119 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Cipher function +; +; Content: +; EncryptCBC_RIJ128pipe_AES_NI() +; +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +;*************************************************************** +;* Purpose: RIJ128 CBC encryption +;* +;* void EncryptCBC_RIJ128_AES_NI(const Ipp32u* inpBlk, +;* Ipp32u* outBlk, +;* int nr, +;* const Ipp32u* pRKey, +;* int len, +;* const Ipp8u* pIV) +;*************************************************************** + +;%if (_IPP >= _IPP_P8) && (_IPP < _IPP_G9) +%if (_IPP >= _IPP_P8) +;; +;; Lib = P8 +;; +;; Caller = ippsRijndael128EncryptCBC +;; + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR +IPPASM EncryptCBC_RIJ128_AES_NI,PUBLIC + USES_GPR esi,edi + +%xdefine pInpBlk [esp + ARG_1 + 0*sizeof(dword)] ; input block address +%xdefine pOutBlk [esp + ARG_1 + 1*sizeof(dword)] ; output block address +%xdefine nr [esp + ARG_1 + 2*sizeof(dword)] ; number of rounds +%xdefine pKey [esp + ARG_1 + 3*sizeof(dword)] ; key material address +%xdefine len [esp + ARG_1 + 4*sizeof(dword)] ; length (bytes) +%xdefine pIV [esp + ARG_1 + 5*sizeof(dword)] ; IV + +%xdefine SC (4) +%assign BYTES_PER_BLK (16) + + mov edx, pIV ; IV address + mov esi,pInpBlk ; input data address + mov ecx,pKey ; key material address + mov eax,nr ; number of rounds + mov edi,pOutBlk ; output data address + + movdqu xmm0, oword [edx] ; IV + + mov edx, len ; length of stream + +align IPP_ALIGN_FACTOR +;; +;; block-by-block processing +;; +.blks_loop: + movdqu xmm1, oword [esi] ; input block + + movdqa xmm4, oword [ecx] ; preload key material + + pxor xmm0, xmm1 ; src[] ^ iv + + pxor xmm0, xmm4 ; whitening + + movdqa xmm4, oword [ecx+16] ; preload key material + add ecx, 16 + + sub eax, 1 ; counter depending on key length +align IPP_ALIGN_FACTOR +.cipher_loop: + aesenc xmm0, xmm4 ; regular round + movdqa xmm4, oword [ecx+16] + add ecx, 16 + sub eax, 1 + jnz .cipher_loop + aesenclast xmm0, xmm4 ; irregular round + + movdqu oword [edi], xmm0 ; store output block + + mov ecx, pKey ; restore key pointer + mov eax, nr ; resrore number of rounds + + add esi, BYTES_PER_BLK ; advance pointers + add edi, BYTES_PER_BLK + sub edx, BYTES_PER_BLK ; decrease counter + jnz .blks_loop + + REST_GPR + ret +ENDFUNC EncryptCBC_RIJ128_AES_NI +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128encryptcfbg9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128encryptcfbg9as.asm new file mode 100644 index 0000000..0ef62a1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128encryptcfbg9as.asm @@ -0,0 +1,376 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Inverse Cipher function +; +; Content: +; Encrypt_RIJ128_AES_NI() +; +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + + +%macro COPY_8U 4.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + %xdefine %%limit %3 + %xdefine %%tmp %4 + + xor ecx, ecx +%%next_byte: + mov %%tmp, byte [%%src+ecx] + mov byte [%%dst+ecx], %%tmp + add ecx, 1 + cmp ecx, %%limit + jl %%next_byte +%endmacro + + +%macro COPY_32U 4.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + %xdefine %%limit %3 + %xdefine %%tmp %4 + + xor ecx, ecx +%%next_dword: + mov %%tmp, dword [%%src+ecx] + mov dword [%%dst+ecx], %%tmp + add ecx, 4 + cmp ecx, %%limit + jl %%next_dword +%endmacro + + +%macro COPY_128U 4.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + %xdefine %%limit %3 + %xdefine %%tmp %4 + + xor ecx, ecx +%%next_oword: + movdqu %%tmp, oword [%%src+ecx] + movdqu oword [%%dst+ecx], %%tmp + add ecx, 16 + cmp ecx, %%limit + jl %%next_oword +%endmacro + + +;*************************************************************** +;* Purpose: RIJ128 CFB encryption +;* +;* void EncryptCFB_RIJ128_AES_NI(const Ipp32u* inpBlk, +;* Ipp32u* outBlk, +;* int nr, +;* const Ipp32u* pRKey, +;* int len, +;* int cfbSize, +;* const Ipp8u* pIV) +;*************************************************************** + +%if (_IPP >= _IPP_P8) +;; +;; Lib = P8 +;; +;; Caller = ippsRijndael128EncryptCFB +;; + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR +IPPASM EncryptCFB_RIJ128_AES_NI,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pInpBlk [ebp + ARG_1 + 0*sizeof(dword)] ; input blocks address +%xdefine pOutBlk [ebp + ARG_1 + 1*sizeof(dword)] ; output blocks address +%xdefine nr [ebp + ARG_1 + 2*sizeof(dword)] ; number of rounds +%xdefine pKey [ebp + ARG_1 + 3*sizeof(dword)] ; key material address +%xdefine len [ebp + ARG_1 + 4*sizeof(dword)] ; length of stream in bytes +%xdefine cfbSize [ebp + ARG_1 + 5*sizeof(dword)] ; cfb blk size +%xdefine pIV [ebp + ARG_1 + 6*sizeof(dword)] ; pointer to the IV + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) + + sub esp,16*(1+4+4) ; allocate stask: + ; +0*16 IV + ; +1*16 inp0, inp1, inp2, inp3 + ; +5*16 out0, out1, out2, out3 + + mov eax, pIV ; IV address + movdqu xmm4, oword [eax] ; get IV + movdqu oword [esp+0*16], xmm4 ; into the stack + +;; +;; processing +;; +.blks_loop: + mov esi,pInpBlk ; input data address + + mov edx,cfbSize ; size of block + lea ebx, [edx*BLKS_PER_LOOP] ; 4 cfb block + mov edx, len + cmp edx, ebx + cmovl ebx, edx + COPY_8U {esp+5*16}, esi, ebx, dl ; move 1-4 input blocks to stack + + ; get actual address of key material: pRKeys += (nr-9) * SC + mov ecx, pKey + mov edx, nr + lea eax,[edx*4] + lea eax, [ecx+eax*4-9*(SC)*4] ; AES-128 round keys + + xor esi, esi ; index + mov edi, ebx +.single_blk: + movdqu xmm0, oword [esp+esi] ; get processing blocks + + pxor xmm0, oword [ecx] ; whitening + + cmp edx,12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + ; do encryption +.key_256_s: + aesenc xmm0, oword [eax-4*4*SC] + aesenc xmm0, oword [eax-3*4*SC] +.key_192_s: + aesenc xmm0, oword [eax-2*4*SC] + aesenc xmm0, oword [eax-1*4*SC] +.key_128_s: + aesenc xmm0, oword [eax+0*4*SC] + aesenc xmm0, oword [eax+1*4*SC] + aesenc xmm0, oword [eax+2*4*SC] + aesenc xmm0, oword [eax+3*4*SC] + aesenc xmm0, oword [eax+4*4*SC] + aesenc xmm0, oword [eax+5*4*SC] + aesenc xmm0, oword [eax+6*4*SC] + aesenc xmm0, oword [eax+7*4*SC] + aesenc xmm0, oword [eax+8*4*SC] + aesenclast xmm0, oword [eax+9*4*SC] + + movdqu xmm1, oword [esp+5*16+esi] ; get src blocks from the stack + pxor xmm0, xmm1 ; xor src + movdqu oword [esp+1*16+esi],xmm0 ;and store into the stack + + add esi, cfbSize ; advance index + sub edi, cfbSize ; decrease length + jg .single_blk + + mov edi,pOutBlk ; output data address + COPY_8U edi, {esp+1*16}, ebx, dl ; move 1-4 blocks to output + + movdqu xmm0, oword [esp+ebx]; update IV + movdqu oword [esp], xmm0 + + add pInpBlk, ebx + add pOutBlk, ebx + sub len, ebx + jg .blks_loop + + add esp, 16*(1+4+4) + REST_GPR + ret +ENDFUNC EncryptCFB_RIJ128_AES_NI + +align IPP_ALIGN_FACTOR +IPPASM EncryptCFB32_RIJ128_AES_NI,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pInpBlk [ebp + ARG_1 + 0*sizeof(dword)] ; input blocks address +%xdefine pOutBlk [ebp + ARG_1 + 1*sizeof(dword)] ; output blocks address +%xdefine nr [ebp + ARG_1 + 2*sizeof(dword)] ; number of rounds +%xdefine pKey [ebp + ARG_1 + 3*sizeof(dword)] ; key material address +%xdefine len [ebp + ARG_1 + 4*sizeof(dword)] ; length of stream in bytes +%xdefine cfbSize [ebp + ARG_1 + 5*sizeof(dword)] ; cfb blk size +%xdefine pIV [ebp + ARG_1 + 6*sizeof(dword)] ; pointer to the IV + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) + + sub esp,16*(1+4+4) ; allocate stask: + ; +0*16 IV + ; +1*16 inp0, inp1, inp2, inp3 + ; +5*16 out0, out1, out2, out3 + + mov eax, pIV ; IV address + movdqu xmm4, oword [eax] ; get IV + movdqu oword [esp+0*16], xmm4 ; into the stack + +;; +;; processing +;; +.blks_loop: + mov esi,pInpBlk ; input data address + + mov edx,cfbSize ; size of block + lea ebx, [edx*BLKS_PER_LOOP] ; 4 cfb block + mov edx, len + cmp edx, ebx + cmovl ebx, edx + COPY_32U {esp+5*16}, esi, ebx, edx ; move 1-4 input blocks to stack + + ; get actual address of key material: pRKeys += (nr-9) * SC + mov ecx, pKey + mov edx, nr + lea eax,[edx*4] + lea eax, [ecx+eax*4-9*(SC)*4] ; AES-128 round keys + + xor esi, esi ; index + mov edi, ebx +.single_blk: + movdqu xmm0, oword [esp+esi] ; get processing blocks + + pxor xmm0, oword [ecx] ; whitening + + cmp edx,12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + ; do encryption +.key_256_s: + aesenc xmm0, oword [eax-4*4*SC] + aesenc xmm0, oword [eax-3*4*SC] +.key_192_s: + aesenc xmm0, oword [eax-2*4*SC] + aesenc xmm0, oword [eax-1*4*SC] +.key_128_s: + aesenc xmm0, oword [eax+0*4*SC] + aesenc xmm0, oword [eax+1*4*SC] + aesenc xmm0, oword [eax+2*4*SC] + aesenc xmm0, oword [eax+3*4*SC] + aesenc xmm0, oword [eax+4*4*SC] + aesenc xmm0, oword [eax+5*4*SC] + aesenc xmm0, oword [eax+6*4*SC] + aesenc xmm0, oword [eax+7*4*SC] + aesenc xmm0, oword [eax+8*4*SC] + aesenclast xmm0, oword [eax+9*4*SC] + + movdqu xmm1, oword [esp+5*16+esi] ; get src blocks from the stack + pxor xmm0, xmm1 ; xor src + movdqu oword [esp+1*16+esi],xmm0 ;and store into the stack + + add esi, cfbSize ; advance index + sub edi, cfbSize ; decrease length + jg .single_blk + + mov edi,pOutBlk ; output data address + COPY_32U edi, {esp+1*16}, ebx, edx ; move 1-4 blocks to output + + movdqu xmm0, oword [esp+ebx] ; update IV + movdqu oword [esp], xmm0 + + add pInpBlk, ebx + add pOutBlk, ebx + sub len, ebx + jg .blks_loop + + add esp, 16*(1+4+4) + REST_GPR + ret +ENDFUNC EncryptCFB32_RIJ128_AES_NI + + +align IPP_ALIGN_FACTOR +IPPASM EncryptCFB128_RIJ128_AES_NI,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pInpBlk [ebp + ARG_1 + 0*sizeof(dword)] ; input blocks address +%xdefine pOutBlk [ebp + ARG_1 + 1*sizeof(dword)] ; output blocks address +%xdefine nr [ebp + ARG_1 + 2*sizeof(dword)] ; number of rounds +%xdefine pKey [ebp + ARG_1 + 3*sizeof(dword)] ; key material address +%xdefine len [ebp + ARG_1 + 4*sizeof(dword)] ; length of stream in bytes +%xdefine pIV [ebp + ARG_1 + 5*sizeof(dword)] ; pointer to the IV + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) + + mov eax, pIV ; IV address + movdqu xmm0, oword [eax] ; get IV + + mov esi,pInpBlk ; input data address + mov edi,pOutBlk ; output data address + mov ebx, len + + ; get actual address of key material: pRKeys += (nr-9) * SC + mov ecx, pKey + mov edx, nr + lea eax,[edx*4] + lea eax, [ecx+eax*4-9*(SC)*4] ; AES-128 round keys + + +;; +;; processing +;; +.blks_loop: + pxor xmm0, oword [ecx] ; whitening + + movdqu xmm1, oword [esi] ; input blocks + + + cmp edx,12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + ; do encryption +.key_256_s: + aesenc xmm0, oword [eax-4*4*SC] + aesenc xmm0, oword [eax-3*4*SC] +.key_192_s: + aesenc xmm0, oword [eax-2*4*SC] + aesenc xmm0, oword [eax-1*4*SC] +.key_128_s: + aesenc xmm0, oword [eax+0*4*SC] + aesenc xmm0, oword [eax+1*4*SC] + aesenc xmm0, oword [eax+2*4*SC] + aesenc xmm0, oword [eax+3*4*SC] + aesenc xmm0, oword [eax+4*4*SC] + aesenc xmm0, oword [eax+5*4*SC] + aesenc xmm0, oword [eax+6*4*SC] + aesenc xmm0, oword [eax+7*4*SC] + aesenc xmm0, oword [eax+8*4*SC] + aesenclast xmm0, oword [eax+9*4*SC] + + pxor xmm0, xmm1 ; xor src + movdqu oword [edi],xmm0 ;and store into the dst + + add esi, 16 + add edi, 16 + sub ebx, 16 + jg .blks_loop + + REST_GPR + ret +ENDFUNC EncryptCFB128_RIJ128_AES_NI + +%endif + + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128encryptctrpipeg9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128encryptctrpipeg9as.asm new file mode 100644 index 0000000..49e12bc --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128encryptctrpipeg9as.asm @@ -0,0 +1,318 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Cipher function +; +; Content: +; EncryptCTR_RIJ128pipe_AES_NI() +; +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + + + + +;*************************************************************** +;* Purpose: pipelined RIJ128 CTR encryption/decryption +;* +;* void EncryptCTR_RIJ128pipe_AES_NI(const Ipp32u* inpBlk, +;* Ipp32u* outBlk, +;* int nr, +;* const Ipp32u* pRKey, +;* int length, +;* Ipp8u* pCtrValue, +;* Ipp8u* pCtrBitMask) +;*************************************************************** + +;%if (_IPP >= _IPP_P8) && (_IPP < _IPP_G9) +%if (_IPP >= _IPP_P8) +;; +;; Lib = P8 +;; +;; Caller = ippsRijndael128EncryptCTR +;; + +segment .text align=IPP_ALIGN_FACTOR +align IPP_ALIGN_FACTOR +CONST_TABLE: +u128_str DB 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 + +align IPP_ALIGN_FACTOR +IPPASM EncryptCTR_RIJ128pipe_AES_NI,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pInpBlk [ebp + ARG_1 + 0*sizeof(dword)] ; input block address +%xdefine pOutBlk [ebp + ARG_1 + 1*sizeof(dword)] ; output block address +%xdefine nr [ebp + ARG_1 + 2*sizeof(dword)] ; number of rounds +%xdefine pKey [ebp + ARG_1 + 3*sizeof(dword)] ; key material address +%xdefine len [ebp + ARG_1 + 4*sizeof(dword)] ; number of blocks being processed +%xdefine pCtrValue [ebp + ARG_1 + 5*sizeof(dword)] ; pointer to the Counter +%xdefine pCtrBitMask [ebp + ARG_1 + 6*sizeof(dword)] ; pointer to the Counter Bit Mask + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) +%assign BYTES_PER_BLK (16) +%assign BYTES_PER_LOOP (BYTES_PER_BLK*BLKS_PER_LOOP) + + mov esi, pCtrBitMask + mov edi, pCtrValue + movdqu xmm6, oword [esi] ; counter bit mask + movdqu xmm1, oword [edi] ; initial counter + movdqu xmm5, xmm6 ; counter bit mask + pandn xmm6, xmm1 ; counter template + + sub esp, (4*4) ; allocate stack + + LD_ADDR eax, CONST_TABLE ; load bswap conversion tbl + movdqa xmm4, oword [eax+(u128_str - CONST_TABLE)] + + ;; + ;; init counter + ;; + mov edx, dword [edi] + mov ecx, dword [edi+4] + mov ebx, dword [edi+8] + mov eax, dword [edi+12] + bswap edx + bswap ecx + bswap ebx + bswap eax + + mov dword [esp], eax ; store counter + mov dword [esp+4], ebx + mov dword [esp+8], ecx + mov dword [esp+12], edx + + mov esi,pInpBlk ; input data address + mov edi,pOutBlk ; output data address + + sub dword len, BYTES_PER_LOOP + jl .short_input + +;; +;; pipelined processing +;; +.blks_loop: + mov eax, dword [esp] ; get coutnter value + mov ebx, dword [esp+4] + mov ecx, dword [esp+8] + mov edx, dword [esp+12] + + pinsrd xmm0, eax, 0 ; set counter value + pinsrd xmm0, ebx, 1 + pinsrd xmm0, ecx, 2 + pinsrd xmm0, edx, 3 + pshufb xmm0, xmm4 ; convert int the octet string + pand xmm0, xmm5 ; select counter bits + por xmm0, xmm6 ; add unchanged bits + + add eax, 1 ; increment counter + adc ebx, 0 + adc ecx, 0 + adc edx, 0 + pinsrd xmm1, eax, 0 ; set counter value + pinsrd xmm1, ebx, 1 + pinsrd xmm1, ecx, 2 + pinsrd xmm1, edx, 3 + pshufb xmm1, xmm4 ; convert int the octet string + pand xmm1, xmm5 ; select counter bits + por xmm1, xmm6 ; add unchanged bits + + add eax, 1 ; increment counter + adc ebx, 0 + adc ecx, 0 + adc edx, 0 + pinsrd xmm2, eax, 0 ; set counter value + pinsrd xmm2, ebx, 1 + pinsrd xmm2, ecx, 2 + pinsrd xmm2, edx, 3 + pshufb xmm2, xmm4 ; convert int the octet string + pand xmm2, xmm5 ; select counter bits + por xmm2, xmm6 ; add unchanged bits + + add eax, 1 ; increment counter + adc ebx, 0 + adc ecx, 0 + adc edx, 0 + pinsrd xmm3, eax, 0 ; set counter value + pinsrd xmm3, ebx, 1 + pinsrd xmm3, ecx, 2 + pinsrd xmm3, edx, 3 + pshufb xmm3, xmm4 ; convert int the octet string + pand xmm3, xmm5 ; select counter bits + por xmm3, xmm6 ; add unchanged bits + + add eax, 1 ; increment counter + adc ebx, 0 + adc ecx, 0 + adc edx, 0 + mov dword [esp], eax ; and store for next itteration + mov dword [esp+4], ebx + mov dword [esp+8], ecx + mov dword [esp+12], edx + + mov ecx, pKey ; get key material address + movdqa xmm7, oword [ecx] + lea ebx, [ecx+16] ; pointer to the round's key material + + pxor xmm0, xmm7 ; whitening + pxor xmm1, xmm7 + pxor xmm2, xmm7 + pxor xmm3, xmm7 + + movdqa xmm7, oword [ebx] ; pre load round keys + add ebx, 16 + + mov eax, nr ; counter depending on key length + sub eax, 1 +.cipher_loop: + aesenc xmm0, xmm7 ; regular round + aesenc xmm1, xmm7 + aesenc xmm2, xmm7 + aesenc xmm3, xmm7 + movdqa xmm7, oword [ebx] + add ebx, 16 + dec eax + jnz .cipher_loop + + aesenclast xmm0, xmm7 ; irregular round + aesenclast xmm1, xmm7 + aesenclast xmm2, xmm7 + aesenclast xmm3, xmm7 + + movdqu xmm7, oword [esi+0*BYTES_PER_BLK] ; xor input blocks + pxor xmm0, xmm7 + movdqu oword [edi+0*BYTES_PER_BLK], xmm0 + + movdqu xmm7, oword [esi+1*BYTES_PER_BLK] + pxor xmm1, xmm7 + movdqu oword [edi+1*BYTES_PER_BLK], xmm1 + + movdqu xmm7, oword [esi+2*BYTES_PER_BLK] + pxor xmm2, xmm7 + movdqu oword [edi+2*BYTES_PER_BLK], xmm2 + + movdqu xmm7, oword [esi+3*BYTES_PER_BLK] + pxor xmm3, xmm7 + movdqu oword [edi+3*BYTES_PER_BLK], xmm3 + + add esi, BYTES_PER_LOOP + add edi, BYTES_PER_LOOP + + sub dword len, BYTES_PER_LOOP + jge .blks_loop + +;; +;; block-by-block processing +;; +.short_input: + add dword len, BYTES_PER_LOOP + jz .quit + + mov ecx,pKey ; key material address + + ; get actual address of key material: pRKeys += (nr-9) * SC + mov eax, nr + lea ebx,[eax*4] + lea ebx,[ecx+ebx*4-9*(SC)*4] ; AES-128 round keys + +.single_blk_loop: + movdqu xmm0, oword [esp] ; get counter value + pshufb xmm0, xmm4 ; convert int the octet string + pand xmm0, xmm5 ; select counter bits + por xmm0, xmm6 ; add unchanged bits + + pxor xmm0, oword [ecx] ; whitening + + cmp eax,12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + +.key_256_s: + aesenc xmm0,oword [ebx-4*4*SC] + aesenc xmm0,oword [ebx-3*4*SC] +.key_192_s: + aesenc xmm0,oword [ebx-2*4*SC] + aesenc xmm0,oword [ebx-1*4*SC] +.key_128_s: + aesenc xmm0,oword [ebx+0*4*SC] + aesenc xmm0,oword [ebx+1*4*SC] + aesenc xmm0,oword [ebx+2*4*SC] + aesenc xmm0,oword [ebx+3*4*SC] + aesenc xmm0,oword [ebx+4*4*SC] + aesenc xmm0,oword [ebx+5*4*SC] + aesenc xmm0,oword [ebx+6*4*SC] + aesenc xmm0,oword [ebx+7*4*SC] + aesenc xmm0,oword [ebx+8*4*SC] + aesenclast xmm0,oword [ebx+9*4*SC] + + add dword [esp], 1 ; advance counter value + adc dword [esp+4], 0 + adc dword [esp+8], 0 + adc dword [esp+12], 0 + + sub dword len, BYTES_PER_BLK + jl .partial_block + + movdqu xmm1, oword [esi] ; input block + add esi, BYTES_PER_BLK + pxor xmm0, xmm1 ; output block + movdqu oword [edi], xmm0 ; save output block + add edi, BYTES_PER_BLK + + cmp dword len, 0 + jz .quit + jmp .single_blk_loop + +.partial_block: + add dword len, BYTES_PER_BLK + +.partial_block_loop: + pextrb eax, xmm0, 0 + psrldq xmm0, 1 + movzx ebx, byte [esi] + xor eax, ebx + mov byte [edi], al + inc esi + inc edi + dec dword len + jnz .partial_block_loop + +.quit: + mov eax, pCtrValue + movdqu xmm0, oword [esp] ; get counter value + pshufb xmm0, xmm4 ; convert int the octet string + pand xmm0, xmm5 ; select counter bits + por xmm0, xmm6 ; add unchanged bits + movdqu oword [eax], xmm0 ; return updated counter + + add esp, (4*4) ; free stack + REST_GPR + ret +ENDFUNC EncryptCTR_RIJ128pipe_AES_NI +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128encryptecbpipeg9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128encryptecbpipeg9as.asm new file mode 100644 index 0000000..2d99391 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128encryptecbpipeg9as.asm @@ -0,0 +1,173 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Cipher function +; +; Content: +; EncryptECB_RIJ128pipe_AES_NI() +; +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +;*************************************************************** +;* Purpose: pipelined RIJ128 ECB encryption +;* +;* void EncryptECB_RIJ128pipe_AES_NI(const Ipp32u* inpBlk, +;* Ipp32u* outBlk, +;* int nr, +;* const Ipp32u* pRKey, +;* int len) +;*************************************************************** + +;%if (_IPP >= _IPP_P8) && (_IPP < _IPP_G9) +%if (_IPP >= _IPP_P8) +;; +;; Lib = P8 +;; +;; Caller = ippsRijndael128EncryptECB +;; + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR +IPPASM EncryptECB_RIJ128pipe_AES_NI,PUBLIC + USES_GPR esi,edi,ebx + +%xdefine pInpBlk [esp + ARG_1 + 0*sizeof(dword)] ; input block address +%xdefine pOutBlk [esp + ARG_1 + 1*sizeof(dword)] ; output block address +%xdefine nr [esp + ARG_1 + 2*sizeof(dword)] ; number of rounds +%xdefine pKey [esp + ARG_1 + 3*sizeof(dword)] ; key material address +%xdefine len [esp + ARG_1 + 4*sizeof(dword)] ; length(byte) + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) +%assign BYTES_PER_BLK (16) +%assign BYTES_PER_LOOP (BYTES_PER_BLK*BLKS_PER_LOOP) + + mov esi,pInpBlk ; input data address + mov edi,pOutBlk ; output data address + mov ecx,pKey ; key material address + mov edx,len ; length + + sub edx, BYTES_PER_LOOP + jl .short_input + +;; +;; pipelined processing +;; +.blks_loop: + movdqa xmm4, oword [ecx] ; keys for whitening + lea ebx, [ecx+16] ; pointer to the round's key material + + movdqu xmm0, oword [esi+0*BYTES_PER_BLK] ; get input blocks + movdqu xmm1, oword [esi+1*BYTES_PER_BLK] + movdqu xmm2, oword [esi+2*BYTES_PER_BLK] + movdqu xmm3, oword [esi+3*BYTES_PER_BLK] + add esi, BYTES_PER_LOOP + + pxor xmm0, xmm4 ; whitening + pxor xmm1, xmm4 + pxor xmm2, xmm4 + pxor xmm3, xmm4 + + movdqa xmm4, oword [ebx] ; pre load round keys + add ebx, 16 + + mov eax,nr ; number of rounds depending on key length + sub eax, 1 +.cipher_loop: + aesenc xmm0, xmm4 ; regular round + aesenc xmm1, xmm4 + aesenc xmm2, xmm4 + aesenc xmm3, xmm4 + movdqa xmm4, oword [ebx] + add ebx, 16 + dec eax + jnz .cipher_loop + + aesenclast xmm0, xmm4 ; irregular round + movdqu oword [edi+0*BYTES_PER_BLK], xmm0 ; store output blocks + aesenclast xmm1, xmm4 + movdqu oword [edi+1*BYTES_PER_BLK], xmm1 + aesenclast xmm2, xmm4 + movdqu oword [edi+2*BYTES_PER_BLK], xmm2 + aesenclast xmm3, xmm4 + movdqu oword [edi+3*BYTES_PER_BLK], xmm3 + + add edi, BYTES_PER_LOOP + sub edx, BYTES_PER_LOOP + jge .blks_loop + +;; +;; block-by-block processing +;; +.short_input: + add edx, BYTES_PER_LOOP + jz .quit + + ; get actual address of key material: pRKeys += (nr-9) * SC + mov eax, nr + lea ebx,[eax*4] + lea ebx,[ecx+ebx*4-9*(SC)*4] ; AES-128 round keys + +.single_blk_loop: + movdqu xmm0, oword [esi] ; get input block + add esi, BYTES_PER_BLK + pxor xmm0, oword [ecx] ; whitening + + cmp eax,12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + +.key_256_s: + aesenc xmm0,oword [ebx-4*4*SC] + aesenc xmm0,oword [ebx-3*4*SC] +.key_192_s: + aesenc xmm0,oword [ebx-2*4*SC] + aesenc xmm0,oword [ebx-1*4*SC] +.key_128_s: + aesenc xmm0,oword [ebx+0*4*SC] + aesenc xmm0,oword [ebx+1*4*SC] + aesenc xmm0,oword [ebx+2*4*SC] + aesenc xmm0,oword [ebx+3*4*SC] + aesenc xmm0,oword [ebx+4*4*SC] + aesenc xmm0,oword [ebx+5*4*SC] + aesenc xmm0,oword [ebx+6*4*SC] + aesenc xmm0,oword [ebx+7*4*SC] + aesenc xmm0,oword [ebx+8*4*SC] + aesenclast xmm0,oword [ebx+9*4*SC] + + movdqu oword [edi], xmm0 ; save output block + add edi, BYTES_PER_BLK + + sub edx, BYTES_PER_BLK + jnz .single_blk_loop + +.quit: + REST_GPR + ret +ENDFUNC EncryptECB_RIJ128pipe_AES_NI +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128encryptg9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128encryptg9as.asm new file mode 100644 index 0000000..5ec0205 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128encryptg9as.asm @@ -0,0 +1,119 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Cipher function +; +; Content: +; Encrypt_RIJ128_AES_NI() +; +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +;*************************************************************** +;* Purpose: single block RIJ128 Cipher +;* +;* void Encrypt_RIJ128_AES_NI(const Ipp32u* inpBlk, +;* Ipp32u* outBlk, +;* int nr, +;* const Ipp32u* pRKey, +;* const Ipp32u Tables[][256]) +;* +;*************************************************************** + +;%if (_IPP >= _IPP_P8) && (_IPP < _IPP_G9) +%if (_IPP >= _IPP_P8) +;; +;; Lib = P8 +;; +;; Caller = ippsRijndael128EncryptECB +;; Caller = ippsRijndael128EncryptCBC +;; Caller = ippsRijndael128EncryptCFB +;; Caller = ippsRijndael128DecryptCFB +;; +;; Caller = ippsDAARijndael128Update +;; Caller = ippsDAARijndael128Final +;; Caller = ippsDAARijndael128MessageDigest +;; + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR +IPPASM Encrypt_RIJ128_AES_NI,PUBLIC + USES_GPR esi,edi + +%xdefine pInpBlk [esp + ARG_1 + 0*sizeof(dword)] ; input block address +%xdefine pOutBlk [esp + ARG_1 + 1*sizeof(dword)] ; output block address +%xdefine nr [esp + ARG_1 + 2*sizeof(dword)] ; number of rounds +%xdefine pKey [esp + ARG_1 + 3*sizeof(dword)] ; key material address + +%xdefine SC (4) + + mov esi,pInpBlk ; input data address + mov ecx,pKey ; key material address + mov eax,nr ; number of rounds + mov edi,pOutBlk ; output data address + + movdqu xmm0, oword [esi] ; input block + pxor xmm0, oword [ecx] ; whitening + + ; get actual address of key material: pRKeys += (nr-9) * SC + lea edx,[eax*4] + lea ecx,[ecx+edx*4-9*(SC)*4]; AES-128-keys + + cmp eax,12 ; switch according to number of rounds + jl .key_128 + jz .key_192 + + ;; + ;; regular rounds + ;; +.key_256: + aesenc xmm0,oword [ecx-4*4*SC] + aesenc xmm0,oword [ecx-3*4*SC] +.key_192: + aesenc xmm0,oword [ecx-2*4*SC] + aesenc xmm0,oword [ecx-1*4*SC] +.key_128: + aesenc xmm0,oword [ecx+0*4*SC] + aesenc xmm0,oword [ecx+1*4*SC] + aesenc xmm0,oword [ecx+2*4*SC] + aesenc xmm0,oword [ecx+3*4*SC] + aesenc xmm0,oword [ecx+4*4*SC] + aesenc xmm0,oword [ecx+5*4*SC] + aesenc xmm0,oword [ecx+6*4*SC] + aesenc xmm0,oword [ecx+7*4*SC] + aesenc xmm0,oword [ecx+8*4*SC] + ;; + ;; last rounds + ;; + aesenclast xmm0,oword [ecx+9*4*SC] + + movdqu oword [edi], xmm0 ; output block + + REST_GPR + ret +ENDFUNC Encrypt_RIJ128_AES_NI +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128encryptofbg9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128encryptofbg9as.asm new file mode 100644 index 0000000..46af14f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128encryptofbg9as.asm @@ -0,0 +1,251 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Inverse Cipher function +; +; Content: +; Encrypt_RIJ128_AES_NI() +; +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + + +%macro COPY_8U 4.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + %xdefine %%limit %3 + %xdefine %%tmp %4 + + xor ecx, ecx +%%next_byte: + mov %%tmp, byte [%%src+ecx] + mov byte [%%dst+ecx], %%tmp + add ecx, 1 + cmp ecx, %%limit + jl %%next_byte +%endmacro + + +;*************************************************************** +;* Purpose: RIJ128 OFB encryption +;* +;* void EncryptOFB_RIJ128_AES_NI(const Ipp32u* inpBlk, +;* Ipp32u* outBlk, +;* int nr, +;* const Ipp32u* pRKey, +;* int length, +;* int ofbBlks, +;* const Ipp8u* pIV) +;*************************************************************** + +%if (_IPP >= _IPP_P8) +;; +;; Lib = P8 +;; +;; Caller = ippsRijndael128DecryptOFB +;; + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR +IPPASM EncryptOFB_RIJ128_AES_NI,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pInpBlk [ebp + ARG_1 + 0*sizeof(dword)] ; input block address +%xdefine pOutBlk [ebp + ARG_1 + 1*sizeof(dword)] ; output block address +%xdefine nr [ebp + ARG_1 + 2*sizeof(dword)] ; number of rounds +%xdefine pKey [ebp + ARG_1 + 3*sizeof(dword)] ; key material address +%xdefine len [ebp + ARG_1 + 4*sizeof(dword)] ; length of stream in bytes +%xdefine ofbSize [ebp + ARG_1 + 5*sizeof(dword)] ; ofb blk size +%xdefine pIV [ebp + ARG_1 + 6*sizeof(dword)] ; IV + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) + +%xdefine tmpInp esp +%xdefine tmpOut tmpInp+sizeof(oword) +%xdefine locDst tmpOut+sizeof(oword) +%xdefine locSrc locDst+sizeof(oword)*4 +%xdefine locLen locSrc+sizeof(oword)*4 +%xdefine stackLen sizeof(oword)+sizeof(oword)+sizeof(oword)*4+sizeof(oword)*4+sizeof(dword) + + sub esp,stackLen ; allocate stack + + mov eax, pIV ; get IV + movdqu xmm0, oword [eax] ; + movdqu oword [tmpInp], xmm0 ; and save into the stack + + mov eax, nr ; number of rounds + mov ecx, pKey ; key material address + lea eax, [eax*4] ; nr*16 offset (bytes) to end of key material + lea eax, [eax*4] + lea ecx, [ecx+eax] + neg eax ; save -nr*16 + mov nr, eax + mov pKey, ecx ; save key material address + + mov esi, pInpBlk ; input stream + mov edi, pOutBlk ; output stream + mov ebx, ofbSize ; cfb blk size + +align IPP_ALIGN_FACTOR +;; +;; processing +;; +.blks_loop: + lea ebx, [ebx*BLKS_PER_LOOP] ; 4 cfb block + + cmp len, ebx + cmovl ebx, len + COPY_8U {locSrc}, esi, ebx, dl ; move 1-4 input blocks to stack + + mov ecx, pKey + mov eax, nr + + mov dword [locLen], ebx + xor edx, edx ; index + +align IPP_ALIGN_FACTOR +.single_blk: + movdqa xmm3, oword [ecx+eax] ; preload key material + add eax, 16 + + movdqa xmm4, oword [ecx+eax] ; preload next key material + pxor xmm0, xmm3 ; whitening + +align IPP_ALIGN_FACTOR +.cipher_loop: + add eax, 16 + aesenc xmm0, xmm4 ; regular round + movdqa xmm4, oword [ecx+eax] + jnz .cipher_loop + aesenclast xmm0, xmm4 ; irregular round + movdqu oword [tmpOut], xmm0 ; save chipher output + + mov eax, ofbSize ; cfb blk size + + movdqu xmm1, oword [locSrc+edx] ; get src blocks from the stack + pxor xmm1, xmm0 ; xor src + movdqu oword [locDst+edx],xmm1 ;and store into the stack + + movdqu xmm0, oword [tmpInp+eax] ; update chiper input (IV) + movdqu oword [tmpInp], xmm0 + + add edx, eax ; advance index + mov eax, nr + cmp edx, ebx + jl .single_blk + + COPY_8U edi, {locDst}, edx, bl ; move 1-4 blocks to output + + mov ebx, ofbSize ; restore cfb blk size + add esi, edx ; advance pointers + add edi, edx + sub len, edx ; decrease stream counter + jg .blks_loop + + mov eax, pIV ; IV address + movdqu xmm0, oword [tmpInp] ; update IV before return + movdqu oword [eax], xmm0 + + add esp,stackLen ; remove local variables + REST_GPR + ret +ENDFUNC EncryptOFB_RIJ128_AES_NI + + +align IPP_ALIGN_FACTOR +IPPASM EncryptOFB128_RIJ128_AES_NI,PUBLIC + USES_GPR esi,edi + +%xdefine pInpBlk [esp + ARG_1 + 0*sizeof(dword)] ; input block address +%xdefine pOutBlk [esp + ARG_1 + 1*sizeof(dword)] ; output block address +%xdefine nr [esp + ARG_1 + 2*sizeof(dword)] ; number of rounds +%xdefine pKey [esp + ARG_1 + 3*sizeof(dword)] ; key material address +%xdefine len [esp + ARG_1 + 4*sizeof(dword)] ; length of stream in bytes +%xdefine pIV [esp + ARG_1 + 5*sizeof(dword)] ; IV + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) + + mov eax, pIV ; get IV + movdqu xmm0, oword [eax] ; + + mov eax, nr ; number of rounds + mov ecx, pKey ; key material address + lea eax, [eax*4] ; nr*16 offset (bytes) to end of key material + lea eax, [eax*4] + lea ecx, [ecx+eax] + neg eax ; save -nr*16 + mov nr, eax + + mov esi, pInpBlk ; input stream + mov edi, pOutBlk ; output stream + mov edx, len ; length of stream + +align IPP_ALIGN_FACTOR +;; +;; processing +;; +.blks_loop: + movdqa xmm3, oword [ecx+eax] ; preload key material + add eax, 16 + +align IPP_ALIGN_FACTOR +.single_blk: + movdqa xmm4, oword [ecx+eax] ; preload next key material + pxor xmm0, xmm3 ; whitening + + movdqu xmm1, oword [esi] ; input block + +align IPP_ALIGN_FACTOR +.cipher_loop: + add eax, 16 + aesenc xmm0, xmm4 ; regular round + movdqa xmm4, oword [ecx+eax] + jnz .cipher_loop + aesenclast xmm0, xmm4 ; irregular round + + pxor xmm1, xmm0 ; xor src + movdqu oword [edi],xmm1 ; and store into the dst + + mov eax, nr ; restore key material counter + add esi, 16 ; advance pointers + add edi, 16 + sub edx, 16 ; decrease stream counter + jg .blks_loop + + mov eax, pIV ; get IV address + movdqu oword [eax], xmm0 ; update IV before return + + REST_GPR + ret +ENDFUNC EncryptOFB128_RIJ128_AES_NI + +%endif + + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128encryptxtsg9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128encryptxtsg9as.asm new file mode 100644 index 0000000..0b14658 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128encryptxtsg9as.asm @@ -0,0 +1,259 @@ +;=============================================================================== +; Copyright (C) 2016 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; AES function +; +; Content: +; cpAESEncryptXTS_AES_NI() +; +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%macro MUL_X 4.nolist + %xdefine %%y %1 + %xdefine %%x %2 + %xdefine %%t %3 + %xdefine %%cnt %4 + + pxor %%t, %%t + movdqa %%y, %%x + pcmpgtq %%t, %%x + paddq %%y, %%y + palignr %%t, %%t, sizeof(qword) + pand %%t, %%cnt + pxor %%y, %%t +%endmacro + + +;*************************************************************** +;* Purpose: AES-XTS encryption +;* +;* void cpAESEncryptXTS_AES_NI(Ipp8u* outBlk, +;* const Ipp8u* inpBlk, +;* int length, +;* const Ipp8u* pRKey, +;* int nr, +;* const Ipp8u* pTweak) +;*************************************************************** + +%if (_IPP >= _IPP_P8) +;; +;; Lib = P8 +;; +;; Caller = ippsAESEncryptXTS +;; + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR +IPPASM cpAESEncryptXTS_AES_NI,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pOutBlk [ebp + ARG_1 + 0*sizeof(dword)] ; output block address +%xdefine pInpBlk [ebp + ARG_1 + 1*sizeof(dword)] ; input block address +%xdefine blks [ebp + ARG_1 + 2*sizeof(dword)] ; length(blocks) +%xdefine pKey [ebp + ARG_1 + 3*sizeof(dword)] ; key material address +%xdefine nr [ebp + ARG_1 + 4*sizeof(dword)] ; number of rounds +%xdefine pTweak [ebp + ARG_1 + 5*sizeof(dword)] ; tweaks + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) +%assign BYTES_PER_BLK (16) +%assign BYTES_PER_LOOP (BYTES_PER_BLK*BLKS_PER_LOOP) + + mov esi,pInpBlk ; input data address + mov edi,pOutBlk ; output data address + +; +; stack layout: +; +%assign _tweakCnt 0 ; constant +%assign _tweak0 _tweakCnt+sizeof(oword) ; * alpha +%assign _tweak1 _tweak0+sizeof(oword) ; * alpha^2 +%assign _tweak2 _tweak1+sizeof(oword) ; * alpha^3 +%assign _tweak3 _tweak2+sizeof(oword) ; * alpha^4 +%assign _nr _tweak3+sizeof(oword) ; # rounds +%assign _sp _nr+sizeof(dword) ; esp +%assign stackSize _sp+sizeof(dword) ; stack size + + mov eax, esp + sub esp, stackSize ; aligned stack + and esp, (-16) + mov dword [esp+_sp], eax ; store esp + + mov eax, pTweak + movdqu xmm4, oword [eax] ; initial tweak + movdqa xmm7, xmm4 + + mov eax, 087h ; create tweakCnt = {0x01:0x87} + mov ecx, 1 + movd xmm0, eax + movd xmm1, ecx + punpcklqdq xmm0, xmm1 + + mov ecx,pKey ; key material address + mov edx,blks ; num of blocks + + ; move parameters to local stack frame + mov eax,nr ; num of rounds + movdqa oword [esp+_tweakCnt], xmm0 + mov dword [esp+_nr], eax + + sub edx, BLKS_PER_LOOP + jl .short_input + jmp .blks_loop_ep + +;; +;; pipelined processing +;; +.blks_loop: + MUL_X xmm4, xmm7, xmm1, xmm0 +.blks_loop_ep: + movdqa oword [esp+_tweak0], xmm4 + MUL_X xmm5, xmm4, xmm1, xmm0 + movdqa oword [esp+_tweak1], xmm5 + MUL_X xmm6, xmm5, xmm1, xmm0 + movdqa oword [esp+_tweak2], xmm6 + MUL_X xmm7, xmm6, xmm1, xmm0 + movdqa oword [esp+_tweak3], xmm7 + + movdqu xmm0, oword [esi+0*BYTES_PER_BLK] ; get input blocks + movdqu xmm1, oword [esi+1*BYTES_PER_BLK] + movdqu xmm2, oword [esi+2*BYTES_PER_BLK] + movdqu xmm3, oword [esi+3*BYTES_PER_BLK] + add esi, BYTES_PER_BLK*BLKS_PER_LOOP + + pxor xmm0, xmm4 ; tweak pre-whitening + pxor xmm1, xmm5 + pxor xmm2, xmm6 + pxor xmm3, xmm7 + + movdqa xmm4, oword [ecx] ; keys for whitening + lea ebx, [ecx+BYTES_PER_BLK]; pointer to the round's key material + + pxor xmm0, xmm4 ; whitening + pxor xmm1, xmm4 + pxor xmm2, xmm4 + pxor xmm3, xmm4 + + movdqa xmm4, oword [ebx] ; pre load round keys + add ebx, BYTES_PER_BLK + sub eax, 1 +.cipher_loop: + aesenc xmm0, xmm4 ; regular round + aesenc xmm1, xmm4 + aesenc xmm2, xmm4 + aesenc xmm3, xmm4 + movdqa xmm4, oword [ebx] + add ebx, BYTES_PER_BLK + dec eax + jnz .cipher_loop + + aesenclast xmm0, xmm4 ; irregular round + aesenclast xmm1, xmm4 + aesenclast xmm2, xmm4 + aesenclast xmm3, xmm4 + + movdqa xmm4, oword [esp+_tweak0] ; tweak post-whitening + movdqa xmm5, oword [esp+_tweak1] + movdqa xmm6, oword [esp+_tweak2] + movdqa xmm7, oword [esp+_tweak3] + pxor xmm0, xmm4 + pxor xmm1, xmm5 + pxor xmm2, xmm6 + pxor xmm3, xmm7 + + movdqu oword [edi+0*BYTES_PER_BLK], xmm0 ; store output blocks + movdqu oword [edi+1*BYTES_PER_BLK], xmm1 + movdqu oword [edi+2*BYTES_PER_BLK], xmm2 + movdqu oword [edi+3*BYTES_PER_BLK], xmm3 + add edi, BYTES_PER_BLK*BLKS_PER_LOOP + + movdqa xmm0, oword [esp+_tweakCnt] ; restore tweak const + mov eax, dword [esp+_nr] ; number of rounds + sub edx, BLKS_PER_LOOP + jge .blks_loop + +;; +;; block-by-block processing +;; +.short_input: + add edx, BLKS_PER_LOOP + jz .quit + + mov eax, dword [esp+_nr] ; bottom of the keys + lea ebx,[eax*sizeof(dword)] + lea ebx,[ecx+ebx*4-9*(SC)*4] ; AES-128 keys + jmp .single_blk_loop_ep + +.single_blk_loop: + MUL_X xmm7, xmm7, xmm1, xmm0 + +.single_blk_loop_ep: + movdqu xmm1, oword [esi] ; input block + add esi, BYTES_PER_BLK + pxor xmm1, xmm7 ; tweak pre-whitening + pxor xmm1, oword [ecx] ; AES whitening + + cmp eax,12 ; switch according to number of rounds + jl .key_128_s + +.key_256_s: + aesenc xmm1,oword [ebx-4*4*SC] + aesenc xmm1,oword [ebx-3*4*SC] + aesenc xmm1,oword [ebx-2*4*SC] + aesenc xmm1,oword [ebx-1*4*SC] +.key_128_s: + aesenc xmm1,oword [ebx+0*4*SC] + aesenc xmm1,oword [ebx+1*4*SC] + aesenc xmm1,oword [ebx+2*4*SC] + aesenc xmm1,oword [ebx+3*4*SC] + aesenc xmm1,oword [ebx+4*4*SC] + aesenc xmm1,oword [ebx+5*4*SC] + aesenc xmm1,oword [ebx+6*4*SC] + aesenc xmm1,oword [ebx+7*4*SC] + aesenc xmm1,oword [ebx+8*4*SC] + aesenclast xmm1,oword [ebx+9*4*SC] + + pxor xmm1, xmm7 ; tweak post-whitening + movdqu oword [edi], xmm1 ; output block + add edi, BYTES_PER_BLK + + sub edx, 1 + jnz .single_blk_loop + +.quit: + mov eax, pTweak ; save tweak + movdqu oword [eax], xmm7 + + mov esp, [esp+_sp] + REST_GPR + ret +ENDFUNC cpAESEncryptXTS_AES_NI +%endif + + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128safedecm5as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128safedecm5as.asm new file mode 100644 index 0000000..c0c7d98 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128safedecm5as.asm @@ -0,0 +1,448 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael-128 (AES) inverse cipher functions. +; (It's the special free from Sbox/tables implementation) +; +; Content: +; SafeDecrypt_RIJ128() +; +; History: +; +; Notes. +; The implementation is based on compact S-box usage. +; +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%if (_IPP == _IPP_M5) + +;; +;; transpose 16x16 martix to [dst] +;; eax,ebx,ecx,edx constins matrix rows +;; +%macro TRANSPOSE 1.nolist + %xdefine %%dst %1 + + mov byte [%%dst+0 ], al + shr eax, 8 + mov byte [%%dst+1 ], bl + shr ebx, 8 + mov byte [%%dst+2 ], cl + shr ecx, 8 + mov byte [%%dst+3 ], dl + shr edx, 8 + + mov byte [%%dst+4 ], al + shr eax, 8 + mov byte [%%dst+5 ], bl + shr ebx, 8 + mov byte [%%dst+6 ], cl + shr ecx, 8 + mov byte [%%dst+7 ], dl + shr edx, 8 + + mov byte [%%dst+8 ], al + shr eax, 8 + mov byte [%%dst+9 ], bl + shr ebx, 8 + mov byte [%%dst+10], cl + shr ecx, 8 + mov byte [%%dst+11], dl + shr edx, 8 + + mov byte [%%dst+12], al + shr eax, 8 + mov byte [%%dst+13], bl + shr ebx, 8 + mov byte [%%dst+14], cl + shr ecx, 8 + mov byte [%%dst+15], dl + shr edx, 8 +%endmacro + + +;; +;; SubBute +;; +%macro SBOX_ZERO 1.nolist + %xdefine %%inpb %1 + + mov al, byte [esi+%%inpb] +%endmacro + + +%macro SBOX_QUAD 1.nolist + %xdefine %%inpb %1 + + mov eax, 03Fh + and eax, %%inpb ; x%64 + shr %%inpb, 6 ; x/64 + add esi, eax ; &Sbox[x%64] + + mov al, byte [esi+64*0] + mov bl, byte [esi+64*1] + mov cl, byte [esi+64*2] + mov dl, byte [esi+64*3] + mov byte [esp+oBuffer+1*0], al + mov byte [esp+oBuffer+1*1], bl + mov byte [esp+oBuffer+1*2], cl + mov byte [esp+oBuffer+1*3], dl + + mov al, byte [esp+oBuffer+%%inpb] +%endmacro + + +%macro SBOX_FULL 1.nolist + %xdefine %%inpb %1 + + mov eax, 0Fh + and eax, %%inpb ; x%16 + shr %%inpb, 4 ; x/16 + add esi, eax ; &Sbox[x%16] + + movzx eax, byte [esi+16*15] + movzx ebx, byte [esi+16*14] + movzx ecx, byte [esi+16*13] + movzx edx, byte [esi+16*12] + push eax + push ebx + push ecx + push edx + + movzx eax, byte [esi+16*11] + movzx ebx, byte [esi+16*10] + movzx ecx, byte [esi+16*9] + movzx edx, byte [esi+16*8] + push eax + push ebx + push ecx + push edx + + movzx eax, byte [esi+16*7] + movzx ebx, byte [esi+16*6] + movzx ecx, byte [esi+16*5] + movzx edx, byte [esi+16*4] + push eax + push ebx + push ecx + push edx + + movzx eax, byte [esi+16*3] + movzx ebx, byte [esi+16*2] + movzx ecx, byte [esi+16*1] + movzx edx, byte [esi+16*0] + push eax + push ebx + push ecx + push edx + + mov eax, dword [esp+%%inpb*sizeof(dword)] + add esp, 16*sizeof(dword) +%endmacro + + +;; +;; AddRoundKey +;; +%macro ADD_ROUND_KEY 5.nolist + %xdefine %%x0 %1 + %xdefine %%x1 %2 + %xdefine %%x2 %3 + %xdefine %%x3 %4 + %xdefine %%key %5 + + xor %%x0, dword [%%key+sizeof(dword)*0] + xor %%x1, dword [%%key+sizeof(dword)*1] + xor %%x2, dword [%%key+sizeof(dword)*2] + xor %%x3, dword [%%key+sizeof(dword)*3] +%endmacro + + +;; +;; GFMULx x, t0,t1 +;; +;; mask = x & 0x80808080 +;; mask = (mask<<1) - (mask>>7) +;; +;; x = (x<<=1) & 0xFEFEFEFE +;; x ^= msk & 0x1B1B1B1B +;; +%macro GFMULx 3.nolist + %xdefine %%x %1 + %xdefine %%msk %2 + %xdefine %%t %3 + + mov %%t, %%x + add %%x, %%x ;; mul: x = (x<<=1) & 0xFEFEFEFE + and %%t, 080808080h ;; mask: t = x & 0x80808080 + and %%x, 0FEFEFEFEh ;; + lea %%msk, [%%t+%%t] ;; mask: msk = (t<<1) - (t>>7) + shr %%t, 7 ;; mask: + sub %%msk, %%t ;; mask: + and %%msk, 01B1B1B1Bh ;; mul: x ^= msk & 0x1B1B1B1B + xor %%x, %%msk +%endmacro + + +;; +;; MixColumn +;; +%macro MIX_COLUMNS 6.nolist + %xdefine %%x0 %1 + %xdefine %%x1 %2 + %xdefine %%x2 %3 + %xdefine %%x3 %4 + %xdefine %%t0 %5 + %xdefine %%t1 %6 + + ;; Ipp32u y0 = state[1] ^ state[2] ^ state[3]; + ;; Ipp32u y1 = state[0] ^ state[2] ^ state[3]; + ;; Ipp32u y2 = state[0] ^ state[1] ^ state[3]; + ;; Ipp32u y3 = state[0] ^ state[1] ^ state[2]; + mov %%t0, %%x1 + xor %%t0, %%x2 + xor %%t0, %%x3 + mov %%t1, %%x0 + xor %%t1, %%x2 + xor %%t1, %%x3 + mov dword [esp+oBuffer+sizeof(dword)*0], %%t0 + mov dword [esp+oBuffer+sizeof(dword)*1], %%t1 + mov %%t0, %%x0 + xor %%t0, %%x1 + xor %%t0, %%x3 + mov %%t1, %%x0 + xor %%t1, %%x1 + xor %%t1, %%x2 + mov dword [esp+oBuffer+sizeof(dword)*2], %%t0 + mov dword [esp+oBuffer+sizeof(dword)*3], %%t1 + + ;; state[0] = xtime4(state[0]); + ;; state[1] = xtime4(state[1]); + ;; state[2] = xtime4(state[2]); + ;; state[3] = xtime4(state[3]); + GFMULx %%x0, %%t0,%%t1 + GFMULx %%x1, %%t0,%%t1 + GFMULx %%x2, %%t0,%%t1 + GFMULx %%x3, %%t0,%%t1 + + ;; y0 ^= state[0] ^ state[1]; + ;; y1 ^= state[1] ^ state[2]; + ;; y2 ^= state[2] ^ state[3]; + ;; y3 ^= state[3] ^ state[0]; + mov %%t0, dword [esp+oBuffer+sizeof(dword)*0] + mov %%t1, dword [esp+oBuffer+sizeof(dword)*1] + xor %%t0, %%x0 + xor %%t1, %%x1 + xor %%t0, %%x1 + xor %%t1, %%x2 + mov dword [esp+oBuffer+sizeof(dword)*0], %%t0 + mov dword [esp+oBuffer+sizeof(dword)*1], %%t1 + + mov %%t0, dword [esp+oBuffer+sizeof(dword)*2] + mov %%t1, dword [esp+oBuffer+sizeof(dword)*3] + xor %%t0, %%x2 + xor %%t1, %%x3 + xor %%t0, %%x3 + xor %%t1, %%x0 + mov dword [esp+oBuffer+sizeof(dword)*2], %%t0 + mov dword [esp+oBuffer+sizeof(dword)*3], %%t1 + + ;; t02 = state[0] ^ state[2]; + ;; t13 = state[1] ^ state[3]; + ;; t02 = xtime4(t02); + ;; t13 = xtime4(t13); + mov %%t0, %%x0 + mov %%t1, %%x1 + xor %%t0, %%x2 + xor %%t1, %%x3 + GFMULx %%t0, %%x0,%%x1 ;; t02 + GFMULx %%t1, %%x0,%%x1 ;; t13 + + ;; t0123 = t02^t13; + ;; t0123 = xtime4(t0123); + mov %%x0, %%t0 + xor %%x0, %%t1 + GFMULx %%x0, %%x1,%%x2 + + ;; state[0] = y0 ^t02 ^t0123; + ;; state[1] = y1 ^t13 ^t0123; + ;; state[2] = y2 ^t02 ^t0123; + ;; state[3] = y3 ^t13 ^t0123; + xor %%t0, %%x0 ;; t02^t0123 + xor %%t1, %%x0 ;; t13^t0123 + mov %%x0, dword [esp+oBuffer+sizeof(dword)*0] + mov %%x1, dword [esp+oBuffer+sizeof(dword)*1] + mov %%x2, dword [esp+oBuffer+sizeof(dword)*2] + mov %%x3, dword [esp+oBuffer+sizeof(dword)*3] + xor %%x0, %%t0 + xor %%x1, %%t1 + xor %%x2, %%t0 + xor %%x3, %%t1 +%endmacro + + +segment .text align=IPP_ALIGN_FACTOR + +IPPASM Safe2Decrypt_RIJ128,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pInp [ebp + ARG_1 + 0*sizeof(dword)] ; input buffer +%xdefine pOut [ebp + ARG_1 + 1*sizeof(dword)] ; outpu buffer +%xdefine nrounds [ebp + ARG_1 + 2*sizeof(dword)] ; number of rounds +%xdefine pRK [ebp + ARG_1 + 3*sizeof(dword)] ; round keys +%xdefine pSbox [ebp + ARG_1 + 4*sizeof(dword)] ; S-box + +;; stack +%assign oState 0 ; 4*dword state +%assign oBuffer oState+sizeof(dword)*4 ; 4*dword buffer +%assign oSbox oBuffer+sizeof(dword)*4 ; S-box address +%assign oSaveEBP oSbox+sizeof(dword) ; save EBP slot + +%assign stackSize oSaveEBP+sizeof(dword) ; stack size + + sub esp, stackSize ; state on the stack + + mov edi, nrounds + + ; read input block and transpose one + mov esi, pInp + mov eax, dword [esi+sizeof(dword)*0] + mov ebx, dword [esi+sizeof(dword)*1] + mov ecx, dword [esi+sizeof(dword)*2] + mov edx, dword [esi+sizeof(dword)*3] + TRANSPOSE {esp+oState} + + shl edi, 4 ; nrounds*16 + mov esi, pRK + add esi, edi + + ; read input block + mov eax, dword [esp+oState+sizeof(dword)*0] + mov ebx, dword [esp+oState+sizeof(dword)*1] + mov ecx, dword [esp+oState+sizeof(dword)*2] + mov edx, dword [esp+oState+sizeof(dword)*3] + + ; add round key + ADD_ROUND_KEY eax,ebx,ecx,edx, esi ; add round key + sub esi, sizeof(byte)*16 + mov pRK, esi + mov dword [esp+oState+sizeof(dword)*0], eax + mov dword [esp+oState+sizeof(dword)*1], ebx + mov dword [esp+oState+sizeof(dword)*2], ecx + mov dword [esp+oState+sizeof(dword)*3], edx + + dec nrounds + mov esi, pSbox + mov dword [esp+oSbox], esi + mov dword [esp+oSaveEBP], ebp + + ;; regular rounds +.next_aes_round: + + ;; shift rows + ror ebx, 24 + ror ecx, 16 + ror edx, 8 + mov dword [esp+oState+sizeof(dword)*0], eax + mov dword [esp+oState+sizeof(dword)*1], ebx + mov dword [esp+oState+sizeof(dword)*2], ecx + mov dword [esp+oState+sizeof(dword)*3], edx + + ;; sub bytes + xor ebp, ebp +.sub_byte_loop: + mov esi, dword [esp+oSbox] ; Sbox address + movzx edi, byte [esp+oState+ebp] ; x input byte + add ebp, 1 + SBOX_FULL edi + ;;SBOX_QUAD edi + ;;SBOX_ZERO edi + mov byte [esp+oState+ebp-1], al + cmp ebp, 16 + jl .sub_byte_loop + + ;; add round key + mov ebp, dword [esp+oSaveEBP] + mov esi, pRK + + mov eax, dword [esp+oState+sizeof(dword)*0] + mov ebx, dword [esp+oState+sizeof(dword)*1] + mov ecx, dword [esp+oState+sizeof(dword)*2] + mov edx, dword [esp+oState+sizeof(dword)*3] + ADD_ROUND_KEY eax,ebx,ecx,edx, esi ; add round key + sub esi, sizeof(byte)*16 + mov pRK, esi + + ;; mix columns + MIX_COLUMNS eax,ebx,ecx,edx, esi, edi ; mix columns + + sub nrounds, 1 + jne .next_aes_round + + ;; irregular round: shift rows + ror ebx, 24 + ror ecx, 16 + ror edx, 8 + mov dword [esp+oState+sizeof(dword)*0], eax + mov dword [esp+oState+sizeof(dword)*1], ebx + mov dword [esp+oState+sizeof(dword)*2], ecx + mov dword [esp+oState+sizeof(dword)*3], edx + + ;; irregular round: sub bytes + xor ebp, ebp +.sub_byte_irr_loop: + mov esi, dword [esp+oSbox] + movzx edi, byte [esp+oState+ebp] ; x input byte + add ebp, 1 + SBOX_FULL edi + ;;SBOX_QUAD edi + ;;SBOX_ZERO edi + mov byte [esp+oState+ebp-1], al + cmp ebp, 16 + jl .sub_byte_irr_loop + + ;; irregular round: add round key + mov ebp, dword [esp+oSaveEBP] + mov esi, pRK + + mov eax, dword [esp+oState+sizeof(dword)*0] + mov ebx, dword [esp+oState+sizeof(dword)*1] + mov ecx, dword [esp+oState+sizeof(dword)*2] + mov edx, dword [esp+oState+sizeof(dword)*3] + ADD_ROUND_KEY eax,ebx,ecx,edx, esi ; add round key + + mov edi, pOut + TRANSPOSE edi + + add esp, stackSize ; remove state + REST_GPR + ret +ENDFUNC Safe2Decrypt_RIJ128 + + +%endif ; _IPP == _IPP_M5 diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128safedecv8as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128safedecv8as.asm new file mode 100644 index 0000000..fdbfe89 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128safedecv8as.asm @@ -0,0 +1,442 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael-128 (AES) cipher functions. +; (It's the special free from Sbox/tables implementation) +; +; Content: +; SafeDecrypt_RIJ128() +; +; History: +; +; Notes. +; The implementation is based on +; isomorphism between native GF(2^8) and composite GF((2^4)^2). +; +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%if (_IPP >= _IPP_V8) + +%macro PTRANSFORM 6.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + %xdefine %%memTransLO %3 + %xdefine %%memTransHI %4 + %xdefine %%tmp %5 + %xdefine %%srcLO %6 + + movdqa %%dst, oword [%%memTransLO] ;; LO transformation + movdqa %%tmp, oword [%%memTransHI] ;; HI transformation + + movdqa %%srcLO, %%src ;; split src: + psrlw %%src, 4 ;; + pand %%srcLO, xmm7 ;; low 4 bits -> srcLO + pand %%src, xmm7 ;; upper 4 bits -> src + + pshufb %%dst, %%srcLO ;; transformation + pshufb %%tmp, %%src + pxor %%dst, %%tmp +%endmacro + + +%macro PLOOKUP_MEM 3.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + %xdefine %%Table %3 + + movdqa %%dst, OWORD %%Table + pshufb %%dst,%%src +%endmacro + + +%macro PREDUCE_MOD15 2.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + + movdqa %%dst, %%src + pcmpgtb %%src, xmm7 + psubb %%dst, %%src +%endmacro + + +%macro PINVERSE_GF16_INV 6.nolist + %xdefine %%xmmB %1 + %xdefine %%xmmC %2 + %xdefine %%xmmP %3 + %xdefine %%xmmQ %4 + %xdefine %%xmmD %5 + %xdefine %%xmmT %6 + + PLOOKUP_MEM %%xmmT, %%xmmC, [eax+(GF16_logTbl-DECODE_DATA)] ;; xmmT = index_of(c) + pxor %%xmmC, %%xmmB + PLOOKUP_MEM %%xmmQ, %%xmmC, [eax+(GF16_logTbl-DECODE_DATA)] ;; xmmQ = index_of(b xor c) + + PLOOKUP_MEM %%xmmD, %%xmmB, [eax+(GF16_sqr1-DECODE_DATA)] ;; xmmD = sqr(b)*beta^14 + PLOOKUP_MEM %%xmmP, %%xmmB, [eax+(GF16_logTbl-DECODE_DATA)] ;; xmmP = index_of(b) + + paddb %%xmmT, %%xmmQ ;; xmmT = index_of(c) + index_of(b xor c) + PREDUCE_MOD15 %%xmmC, %%xmmT ;; + PLOOKUP_MEM %%xmmT, %%xmmC, [eax+(GF16_expTbl-DECODE_DATA)] ;; c*(b xor c) + + pxor %%xmmD, %%xmmT ;; xmmD = delta = (c*(b xor c)) xor (sqr(b)*beta^14) + PLOOKUP_MEM %%xmmT, %%xmmD, [eax+(GF16_invLog-DECODE_DATA)] ;; xmmT = index_of( inv(delta) ) + + paddb %%xmmQ, %%xmmT ;; xmmQ = index_of((b xor c) * inv(delta)) + paddb %%xmmP, %%xmmT ;; xmmP = index_of(b * inv(delta)) + PREDUCE_MOD15 %%xmmT, %%xmmQ + PLOOKUP_MEM %%xmmC, %%xmmT, [eax+(GF16_expTbl-DECODE_DATA)] + PREDUCE_MOD15 %%xmmT, %%xmmP + PLOOKUP_MEM %%xmmB, %%xmmT, [eax+(GF16_expTbl-DECODE_DATA)] +%endmacro + + + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR + +DECODE_DATA: + +;; (forward) native GF(2^8) to composite GF((2^4)^2) transformation : {0x01,0x2E,0x49,0x43,0x35,0xD0,0x3D,0xE9} +TransFwdLO \ + DB 000h ;; 000h ;; 0 + DB 001h ;; 001h ;; 1 + DB 02Eh ;; 02Eh ;; 2 + DB 02Fh ;; 02Eh XOR 001h ;; 3 + DB 049h ;; 049h ;; 4 + DB 048h ;; 049h XOR 001h ;; 5 + DB 067h ;; 049h XOR 02Eh ;; 6 + DB 066h ;; 049h XOR 02Eh XOR 001h ;; 7 + DB 043h ;; 043h ;; 8 + DB 042h ;; 043h XOR 001h ;; 9 + DB 06Dh ;; 043h XOR 02Eh ;; a + DB 06Ch ;; 043h XOR 02Eh XOR 001h ;; b + DB 00Ah ;; 043h XOR 049h ;; c + DB 00Bh ;; 043h XOR 049h XOR 001h ;; d + DB 024h ;; 043h XOR 049h XOR 02Eh ;; e + DB 025h ;; 043h XOR 049h XOR 02Eh XOR 001h ;; f +TransFwdHI \ + DB 000h ;; 000h ;; 0 + DB 035h ;; 035h ;; 1 + DB 0D0h ;; 0D0h ;; 2 + DB 0E5h ;; 0D0h XOR 035h ;; 3 + DB 03Dh ;; 03Dh ;; 4 + DB 008h ;; 03Dh XOR 035h ;; 5 + DB 0EDh ;; 03Dh XOR 0D0h ;; 6 + DB 0D8h ;; 03Dh XOR 0D0h XOR 035h ;; 7 + DB 0E9h ;; 0E9h ;; 8 + DB 0DCh ;; 0E9h XOR 035h ;; 9 + DB 039h ;; 0E9h XOR 0D0h ;; a + DB 00Ch ;; 0E9h XOR 0D0h XOR 035h ;; b + DB 0D4h ;; 0E9h XOR 03Dh ;; c + DB 0E1h ;; 0E9h XOR 03Dh XOR 035h ;; d + DB 004h ;; 0E9h XOR 03Dh XOR 0D0h ;; e + DB 031h ;; 0E9h XOR 03Dh XOR 0D0h XOR 035h ;; f + +;; (inverse) composite GF((2^4)^2) to native GF(2^8) transformation : {0x01,0x5C,0xE0,0x50,0x1F,0xEE,0x55,0x6A} +TransInvLO \ + DB 000h ;; 000h ;; 0 + DB 001h ;; 001h ;; 1 + DB 05Ch ;; 05Ch ;; 2 + DB 05Dh ;; 05Ch XOR 001h ;; 3 + DB 0E0h ;; 0E0h ;; 4 + DB 0E1h ;; 0E0h XOR 001h ;; 5 + DB 0BCh ;; 0E0h XOR 05Ch ;; 6 + DB 0BDh ;; 0E0h XOR 05Ch XOR 001h ;; 7 + DB 050h ;; 050h ;; 8 + DB 051h ;; 050h XOR 001h ;; 9 + DB 00Ch ;; 050h XOR 05Ch ;; a + DB 00Dh ;; 050h XOR 05Ch XOR 001h ;; b + DB 0B0h ;; 050h XOR 0E0h ;; c + DB 0B1h ;; 050h XOR 0E0h XOR 001h ;; d + DB 0ECh ;; 050h XOR 0E0h XOR 05Ch ;; e + DB 0EDh ;; 050h XOR 0E0h XOR 05Ch XOR 001h ;; f +TransInvHI \ + DB 000h ;; 000h ;; 0 + DB 01Fh ;; 01Fh ;; 1 + DB 0EEh ;; 0EEh ;; 2 + DB 0F1h ;; 0EEh XOR 01Fh ;; 3 + DB 055h ;; 055h ;; 4 + DB 04Ah ;; 055h XOR 01Fh ;; 5 + DB 0BBh ;; 055h XOR 0EEh ;; 6 + DB 0A4h ;; 055h XOR 0EEh XOR 01Fh ;; 7 + DB 06Ah ;; 06Ah ;; 8 + DB 075h ;; 06Ah XOR 01Fh ;; 9 + DB 084h ;; 06Ah XOR 0EEh ;; a + DB 09Bh ;; 06Ah XOR 0EEh XOR 01Fh ;; b + DB 03Fh ;; 06Ah XOR 055h ;; c + DB 020h ;; 06Ah XOR 055h XOR 01Fh ;; d + DB 0D1h ;; 06Ah XOR 055h XOR 0EEh ;; e + DB 0CEh ;; 06Ah XOR 055h XOR 0EEh XOR 01Fh ;; f + + +GF16_csize DB 00Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh + +;; GF16 elements: +;; 0 1 2 3 4 5 6 7 8 9 A B C D E F +GF16_logTbl \ + DB 0C0h,00h,01h,04h,02h,08h,05h,0Ah,03h,0Eh,09h,07h,06h,0Dh,0Bh,0Ch +GF16_expTbl \ + DB 001h,02h,04h,08h,03h,06h,0Ch,0Bh,05h,0Ah,07h,0Eh,0Fh,0Dh,09h,01h +GF16_sqr1 \ + DB 000h,09h,02h,0Bh,08h,01h,0Ah,03h,06h,0Fh,04h,0Dh,0Eh,07h,0Ch,05h ;; sqr(GF16_element) * beta^14 +GF16_invLog \ + DB 0C0h,00h,0Eh,0Bh,0Dh,07h,0Ah,05h,0Ch,01h,06h,08h,09h,02h,04h,03h + +;; affine transformation matrix (inverse cipher) : {0x50,0x36,0x15,0x82,0x01,0x34,0x40,0x3E} +InvAffineLO \ + DB 000h ;; 000h ;; 0 + DB 050h ;; 050h ;; 1 + DB 036h ;; 036h ;; 2 + DB 066h ;; 036h XOR 050h ;; 3 + DB 015h ;; 015h ;; 4 + DB 045h ;; 015h XOR 050h ;; 5 + DB 023h ;; 015h XOR 036h ;; 6 + DB 073h ;; 015h XOR 036h XOR 050h ;; 7 + DB 082h ;; 082h ;; 8 + DB 0D2h ;; 082h XOR 050h ;; 9 + DB 0B4h ;; 082h XOR 036h ;; a + DB 0E4h ;; 082h XOR 036h XOR 050h ;; b + DB 097h ;; 082h XOR 015h ;; c + DB 0C7h ;; 082h XOR 015h XOR 050h ;; d + DB 0A1h ;; 082h XOR 015h XOR 036h ;; e + DB 0F1h ;; 082h XOR 015h XOR 036h XOR 050h ;; f +InvAffineHI \ + DB 000h ;; 000h ;; 0 + DB 001h ;; 001h ;; 1 + DB 034h ;; 034h ;; 2 + DB 035h ;; 034h XOR 001h ;; 3 + DB 040h ;; 040h ;; 4 + DB 041h ;; 040h XOR 001h ;; 5 + DB 074h ;; 040h XOR 034h ;; 6 + DB 075h ;; 040h XOR 034h XOR 001h ;; 7 + DB 03Eh ;; 03Eh ;; 8 + DB 03Fh ;; 03Eh XOR 001h ;; 9 + DB 00Ah ;; 03Eh XOR 034h ;; a + DB 00Bh ;; 03Eh XOR 034h XOR 001h ;; b + DB 07Eh ;; 03Eh XOR 040h ;; c + DB 07Fh ;; 03Eh XOR 040h XOR 001h ;; d + DB 04Ah ;; 03Eh XOR 040h XOR 034h ;; e + DB 04Bh ;; 03Eh XOR 040h XOR 034h XOR 001h ;; f + +;; affine transformation constant (inverse cipher) +InvAffineCnt \ + DQ 04848484848484848h,04848484848484848h + +;; shift rows transformation (inverse cipher) +InvShiftRows \ + DB 0,13,10,7,4,1,14,11,8,5,2,15,12,9,6,3 + +;; mix columns transformation (inverse cipher) +GF16mul_4_2x \ + DB 000h,024h,048h,06Ch,083h,0A7h,0CBh,0EFh,036h,012h,07Eh,05Ah,0B5h,091h,0FDh,0D9h ;; *(4+2x) +GF16mul_1_6x \ + DB 000h,061h,0C2h,0A3h,0B4h,0D5h,076h,017h,058h,039h,09Ah,0FBh,0ECh,08Dh,02Eh,04Fh ;; *(1+6x) + +GF16mul_C_6x \ + DB 000h,06Ch,0CBh,0A7h,0B5h,0D9h,07Eh,012h,05Ah,036h,091h,0FDh,0EFh,083h,024h,048h ;; *(C+6x) +GF16mul_3_Ax \ + DB 000h,0A3h,076h,0D5h,0ECh,04Fh,09Ah,039h,0FBh,058h,08Dh,02Eh,017h,0B4h,061h,0C2h ;; *(3+Ax) + +GF16mul_B_0x \ + DB 000h,00Bh,005h,00Eh,00Ah,001h,00Fh,004h,007h,00Ch,002h,009h,00Dh,006h,008h,003h ;; *(B+0x) +GF16mul_0_Bx \ + DB 000h,0B0h,050h,0E0h,0A0h,010h,0F0h,040h,070h,0C0h,020h,090h,0D0h,060h,080h,030h ;; *(0+Bx) + +GF16mul_2_4x \ + DB 000h,042h,084h,0C6h,038h,07Ah,0BCh,0FEh,063h,021h,0E7h,0A5h,05Bh,019h,0DFh,09Dh ;; *(2+4x) +GF16mul_2_6x \ + DB 000h,062h,0C4h,0A6h,0B8h,0DAh,07Ch,01Eh,053h,031h,097h,0F5h,0EBh,089h,02Fh,04Dh ;; *(2+6x) + +ColumnROR \ + DB 1,2,3,0,5,6,7,4,9,10,11,8,13,14,15,12 + + +;************************************************************* +; convert GF(2^128) -> GF((2^4)^2) +;************************************************************* +align IPP_ALIGN_FACTOR +IPPASM TransformNative2Composite,PUBLIC + USES_GPR esi,edi + +%xdefine pOutBlk [esp + ARG_1 + 0*sizeof(dword)] ; output block address +%xdefine pInpBlk [esp + ARG_1 + 1*sizeof(dword)] ; input block address + + LD_ADDR eax, DECODE_DATA + mov edi,pOutBlk ; output data address + mov esi,pInpBlk ; input data address + movdqa xmm7, oword [eax+(GF16_csize-DECODE_DATA)] + + ;; convert input into the composite GF((2^4)^2) + movdqu xmm0, oword [esi] ; input block + PTRANSFORM xmm1, xmm0, {eax+(TransFwdLO-DECODE_DATA)},{eax+(TransFwdHI-DECODE_DATA)}, xmm2, xmm3 + + movdqu oword [edi], xmm1 ; output block + REST_GPR + ret +ENDFUNC TransformNative2Composite + +align IPP_ALIGN_FACTOR +;************************************************************* +;* void SafeDecrypt_RIJ128( +;* const Ipp32u* pInpBlk, +;* Ipp32u* pOutBlk, +;* int nr, +;* const Ipp32u* pKeys, +;* const void* Tables) +;************************************************************* + +;; +;; Lib = V8 +;; +IPPASM SafeDecrypt_RIJ128,PUBLIC + USES_GPR esi,edi + +%xdefine pInpBlk [esp + ARG_1 + 0*sizeof(dword)] ; input block address +%xdefine pOutBlk [esp + ARG_1 + 1*sizeof(dword)] ; output block address +%xdefine nr [esp + ARG_1 + 2*sizeof(dword)] ; number of rounds +%xdefine pKey [esp + ARG_1 + 3*sizeof(dword)] ; key material address + +%assign RSIZE sizeof(dword) ; size of row +%assign SC 4 ; columns in STATE +%assign SSIZE RSIZE*SC ; size of state + + mov edx, pKey + mov ecx, nr + mov esi,pInpBlk ; input data address + mov edi,pOutBlk ; output data address + + lea eax,[ecx*4] + lea edx,[edx+eax*4] ; AES-128-keys + + LD_ADDR eax, DECODE_DATA + + movdqu xmm0, oword [esi] ; input block + + movdqa xmm7, oword [eax+(GF16_csize-DECODE_DATA)] + + + ;; convert input into the composite GF((2^4)^2) + PTRANSFORM xmm2,xmm0, {eax+(TransFwdLO-DECODE_DATA)},{eax+(TransFwdHI-DECODE_DATA)}, xmm1,xmm3 + + ;; initial whitening + pxor xmm2, oword [edx] + sub edx, SSIZE + + ;; (nr-1) regular rounds + sub ecx,1 + +.decode_round: + ;; InvSubByte() Transformation: + + ;; affine transformation + PTRANSFORM xmm0,xmm2, {eax+(InvAffineLO-DECODE_DATA)},{eax+(InvAffineHI-DECODE_DATA)}, xmm1,xmm3 + pxor xmm0, oword [eax+(InvAffineCnt-DECODE_DATA)] ; H(c), c=0x05 + + ;; split input by low and upper parts + movdqa xmm1, xmm0 + pand xmm0, xmm7 ; upper parts (4 bits) + psrlw xmm1, 4 + pand xmm1, xmm7 ; low parts (4 bits) + + ;; compute multiplicative inverse + PINVERSE_GF16_INV xmm1,xmm0, xmm3,xmm2,xmm4,xmm5 + + ;; InvShiftRows() Transformation: + pshufb xmm0, [eax+(InvShiftRows-DECODE_DATA)] + pshufb xmm1, [eax+(InvShiftRows-DECODE_DATA)] + + ;; InvMixColumn() Transformation: + PLOOKUP_MEM xmm2, xmm0, [eax+(GF16mul_4_2x-DECODE_DATA)] ; mul H(0xE) = 0x24 + pshufb xmm0, [eax+(ColumnROR-DECODE_DATA)] + PLOOKUP_MEM xmm3, xmm1, [eax+(GF16mul_1_6x-DECODE_DATA)] + pshufb xmm1, [eax+(ColumnROR-DECODE_DATA)] + pxor xmm2, xmm3 + + PLOOKUP_MEM xmm3, xmm0, [eax+(GF16mul_C_6x-DECODE_DATA)] ; mul H(0xB) = 0x6C + pshufb xmm0, [eax+(ColumnROR-DECODE_DATA)] + pxor xmm2, xmm3 + PLOOKUP_MEM xmm3, xmm1, [eax+(GF16mul_3_Ax-DECODE_DATA)] + pshufb xmm1, [eax+(ColumnROR-DECODE_DATA)] + pxor xmm2, xmm3 + + PLOOKUP_MEM xmm3, xmm0, [eax+(GF16mul_B_0x-DECODE_DATA)] ; mul H(0xD) = 0x0B + pshufb xmm0, [eax+(ColumnROR-DECODE_DATA)] + pxor xmm2, xmm3 + PLOOKUP_MEM xmm3, xmm1, [eax+(GF16mul_0_Bx-DECODE_DATA)] + pshufb xmm1, [eax+(ColumnROR-DECODE_DATA)] + pxor xmm2, xmm3 + + PLOOKUP_MEM xmm3, xmm0, [eax+(GF16mul_2_4x-DECODE_DATA)] ; mul H(0x9) = 0x42 + pxor xmm2, xmm3 + PLOOKUP_MEM xmm3, xmm1, [eax+(GF16mul_2_6x-DECODE_DATA)] + pxor xmm2, xmm3 + + ;; AddRoundKey() Transformation: + pxor xmm2, oword [edx] + sub edx, SSIZE + + sub ecx,1 + jg .decode_round + + + ;; + ;; the last one is irregular + ;; + + ;; InvSubByte() Transformation: + + ;; affine transformation + PTRANSFORM xmm0,xmm2, {eax+(InvAffineLO-DECODE_DATA)},{eax+(InvAffineHI-DECODE_DATA)}, xmm1,xmm3 + pxor xmm0, oword [eax+(InvAffineCnt-DECODE_DATA)] ; H(c), c=0x05 + + ;; split input by low and upper parts + movdqa xmm1, xmm0 + pand xmm0, xmm7 ; low parts (4 bits) + psrlw xmm1, 4 + pand xmm1, xmm7 ; upper parts (4 bits) + + ;; compute multiplicative inverse + PINVERSE_GF16_INV xmm1,xmm0, xmm3,xmm2,xmm4,xmm5 + + ;; InvShiftRows() Transformation: + psllw xmm1, 4 + por xmm1, xmm0 + pshufb xmm1, [eax+(InvShiftRows-DECODE_DATA)] + + ;; AddRoundKey() Transformation: + pxor xmm1, oword [edx] + sub edx, SSIZE + + ;; convert output into the native GF(2^8) + PTRANSFORM xmm0,xmm1, {eax+(TransInvLO-DECODE_DATA)},{eax+(TransInvHI-DECODE_DATA)}, xmm2, xmm3 + + movdqu oword [edi], xmm0 + REST_GPR + ret +ENDFUNC SafeDecrypt_RIJ128 + +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128safeencv8as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128safeencv8as.asm new file mode 100644 index 0000000..13f1ab0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprij128safeencv8as.asm @@ -0,0 +1,413 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael-128 (AES) cipher functions. +; (It's the special free from Sbox/tables implementation) +; +; Content: +; SafeEncrypt_RIJ128() +; +; History: +; +; Notes. +; The implementation is based on +; isomorphism between native GF(2^8) and composite GF((2^4)^2). +; +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%if (_IPP >= _IPP_V8) + +%macro PTRANSFORM 6.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + %xdefine %%memTransLO %3 + %xdefine %%memTransHI %4 + %xdefine %%tmp %5 + %xdefine %%srcLO %6 + + movdqa %%dst, oword [%%memTransLO] ;; LO transformation + movdqa %%tmp, oword [%%memTransHI] ;; HI transformation + + movdqa %%srcLO, %%src ;; split src: + psrlw %%src, 4 ;; + pand %%srcLO, xmm7 ;; low 4 bits -> srcLO + pand %%src, xmm7 ;; upper 4 bits -> src + + pshufb %%dst, %%srcLO ;; transformation + pshufb %%tmp, %%src + pxor %%dst, %%tmp +%endmacro + + +%macro PLOOKUP_MEM 3.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + %xdefine %%Table %3 + + movdqa %%dst, OWORD %%Table + pshufb %%dst,%%src +%endmacro + + +%macro PREDUCE_MOD15 2.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + + movdqa %%dst, %%src + pcmpgtb %%src, xmm7 + psubb %%dst, %%src +%endmacro + + + +%macro PINVERSE_GF16_FWD 6.nolist + %xdefine %%xmmB %1 + %xdefine %%xmmC %2 + %xdefine %%xmmP %3 + %xdefine %%xmmQ %4 + %xdefine %%xmmD %5 + %xdefine %%xmmT %6 + + PLOOKUP_MEM %%xmmT, %%xmmC, [eax+(GF16_logTbl-ENCODE_DATA)] ;; xmmT = index_of(c) + pxor %%xmmC, %%xmmB + PLOOKUP_MEM %%xmmQ, %%xmmC, [eax+(GF16_logTbl-ENCODE_DATA)] ;; xmmQ = index_of(b xor c) + + PLOOKUP_MEM %%xmmD, %%xmmB, [eax+(GF16_sqr1-ENCODE_DATA)] ;; xmmD = sqr(b)*beta^14 + PLOOKUP_MEM %%xmmP, %%xmmB, [eax+(GF16_logTbl-ENCODE_DATA)] ;; xmmP = index_of(b) + + paddb %%xmmT, %%xmmQ ;; xmmT = index_of(c) + index_of(b xor c) + PREDUCE_MOD15 %%xmmC, %%xmmT ;; + PLOOKUP_MEM %%xmmT, %%xmmC, [eax+(GF16_expTbl-ENCODE_DATA)] ;; c*(b xor c) + + pxor %%xmmD, %%xmmT ;; xmmD = delta = (c*(b xor c)) xor (sqr(b)*beta^14) + PLOOKUP_MEM %%xmmT, %%xmmD, [eax+(GF16_invLog-ENCODE_DATA)] ;; xmmT = index_of( inv(delta) ) + + paddb %%xmmQ, %%xmmT ;; xmmQ = index_of((b xor c) * inv(delta)) + paddb %%xmmP, %%xmmT ;; xmmP = index_of(b * inv(delta)) + PREDUCE_MOD15 %%xmmT, %%xmmQ + PLOOKUP_MEM %%xmmC, %%xmmT, [eax+(GF16_expTbl-ENCODE_DATA)] + PREDUCE_MOD15 %%xmmT, %%xmmP + PLOOKUP_MEM %%xmmB, %%xmmT, [eax+(GF16_expTbl-ENCODE_DATA)] +%endmacro + + + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR + +ENCODE_DATA: + +;; (forward) native GF(2^8) to composite GF((2^4)^2) transformation : {0x01,0x2E,0x49,0x43,0x35,0xD0,0x3D,0xE9} +TransFwdLO \ + DB 000h ;; 000h ;; 0 + DB 001h ;; 001h ;; 1 + DB 02Eh ;; 02Eh ;; 2 + DB 02Fh ;; 02Eh XOR 001h ;; 3 + DB 049h ;; 049h ;; 4 + DB 048h ;; 049h XOR 001h ;; 5 + DB 067h ;; 049h XOR 02Eh ;; 6 + DB 066h ;; 049h XOR 02Eh XOR 001h ;; 7 + DB 043h ;; 043h ;; 8 + DB 042h ;; 043h XOR 001h ;; 9 + DB 06Dh ;; 043h XOR 02Eh ;; a + DB 06Ch ;; 043h XOR 02Eh XOR 001h ;; b + DB 00Ah ;; 043h XOR 049h ;; c + DB 00Bh ;; 043h XOR 049h XOR 001h ;; d + DB 024h ;; 043h XOR 049h XOR 02Eh ;; e + DB 025h ;; 043h XOR 049h XOR 02Eh XOR 001h ;; f +TransFwdHI \ + DB 000h ;; 000h ;; 0 + DB 035h ;; 035h ;; 1 + DB 0D0h ;; 0D0h ;; 2 + DB 0E5h ;; 0D0h XOR 035h ;; 3 + DB 03Dh ;; 03Dh ;; 4 + DB 008h ;; 03Dh XOR 035h ;; 5 + DB 0EDh ;; 03Dh XOR 0D0h ;; 6 + DB 0D8h ;; 03Dh XOR 0D0h XOR 035h ;; 7 + DB 0E9h ;; 0E9h ;; 8 + DB 0DCh ;; 0E9h XOR 035h ;; 9 + DB 039h ;; 0E9h XOR 0D0h ;; a + DB 00Ch ;; 0E9h XOR 0D0h XOR 035h ;; b + DB 0D4h ;; 0E9h XOR 03Dh ;; c + DB 0E1h ;; 0E9h XOR 03Dh XOR 035h ;; d + DB 004h ;; 0E9h XOR 03Dh XOR 0D0h ;; e + DB 031h ;; 0E9h XOR 03Dh XOR 0D0h XOR 035h ;; f + +;; (inverse) composite GF((2^4)^2) to native GF(2^8) transformation : {0x01,0x5C,0xE0,0x50,0x1F,0xEE,0x55,0x6A} +TransInvLO \ + DB 000h ;; 000h ;; 0 + DB 001h ;; 001h ;; 1 + DB 05Ch ;; 05Ch ;; 2 + DB 05Dh ;; 05Ch XOR 001h ;; 3 + DB 0E0h ;; 0E0h ;; 4 + DB 0E1h ;; 0E0h XOR 001h ;; 5 + DB 0BCh ;; 0E0h XOR 05Ch ;; 6 + DB 0BDh ;; 0E0h XOR 05Ch XOR 001h ;; 7 + DB 050h ;; 050h ;; 8 + DB 051h ;; 050h XOR 001h ;; 9 + DB 00Ch ;; 050h XOR 05Ch ;; a + DB 00Dh ;; 050h XOR 05Ch XOR 001h ;; b + DB 0B0h ;; 050h XOR 0E0h ;; c + DB 0B1h ;; 050h XOR 0E0h XOR 001h ;; d + DB 0ECh ;; 050h XOR 0E0h XOR 05Ch ;; e + DB 0EDh ;; 050h XOR 0E0h XOR 05Ch XOR 001h ;; f +TransInvHI \ + DB 000h ;; 000h ;; 0 + DB 01Fh ;; 01Fh ;; 1 + DB 0EEh ;; 0EEh ;; 2 + DB 0F1h ;; 0EEh XOR 01Fh ;; 3 + DB 055h ;; 055h ;; 4 + DB 04Ah ;; 055h XOR 01Fh ;; 5 + DB 0BBh ;; 055h XOR 0EEh ;; 6 + DB 0A4h ;; 055h XOR 0EEh XOR 01Fh ;; 7 + DB 06Ah ;; 06Ah ;; 8 + DB 075h ;; 06Ah XOR 01Fh ;; 9 + DB 084h ;; 06Ah XOR 0EEh ;; a + DB 09Bh ;; 06Ah XOR 0EEh XOR 01Fh ;; b + DB 03Fh ;; 06Ah XOR 055h ;; c + DB 020h ;; 06Ah XOR 055h XOR 01Fh ;; d + DB 0D1h ;; 06Ah XOR 055h XOR 0EEh ;; e + DB 0CEh ;; 06Ah XOR 055h XOR 0EEh XOR 01Fh ;; f + + +GF16_csize DB 00Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh + +;; GF16 elements: +;; 0 1 2 3 4 5 6 7 8 9 A B C D E F +GF16_logTbl \ + DB 0C0h,00h,01h,04h,02h,08h,05h,0Ah,03h,0Eh,09h,07h,06h,0Dh,0Bh,0Ch +GF16_expTbl \ + DB 001h,02h,04h,08h,03h,06h,0Ch,0Bh,05h,0Ah,07h,0Eh,0Fh,0Dh,09h,01h +GF16_sqr1 \ + DB 000h,09h,02h,0Bh,08h,01h,0Ah,03h,06h,0Fh,04h,0Dh,0Eh,07h,0Ch,05h ;; sqr(GF16_element) * beta^14 +GF16_invLog \ + DB 0C0h,00h,0Eh,0Bh,0Dh,07h,0Ah,05h,0Ch,01h,06h,08h,09h,02h,04h,03h +GF16_expTbl_shift \ + DB 010h,020h,040h,080h,030h,060h,0C0h,0B0h,050h,0A0h,070h,0E0h,0F0h,0D0h,090h,010h + +;; affine transformation matrix (forward cipher) : {0x10,0x22,0x55,0x82,0x41,0x34,0x40,0x2A} +FwdAffineLO \ + DB 000h ;; 000h ;; 0 + DB 010h ;; 010h ;; 1 + DB 022h ;; 022h ;; 2 + DB 032h ;; 022h XOR 010h ;; 3 + DB 055h ;; 055h ;; 4 + DB 045h ;; 055h XOR 010h ;; 5 + DB 077h ;; 055h XOR 022h ;; 6 + DB 067h ;; 055h XOR 022h XOR 010h ;; 7 + DB 082h ;; 082h ;; 8 + DB 092h ;; 082h XOR 010h ;; 9 + DB 0A0h ;; 082h XOR 022h ;; a + DB 0B0h ;; 082h XOR 022h XOR 010h ;; b + DB 0D7h ;; 082h XOR 055h ;; c + DB 0C7h ;; 082h XOR 055h XOR 010h ;; d + DB 0F5h ;; 082h XOR 055h XOR 022h ;; e + DB 0E5h ;; 082h XOR 055h XOR 022h XOR 010h ;; f +FwdAffineHI \ + DB 000h ;; 000h ;; 0 + DB 041h ;; 041h ;; 1 + DB 034h ;; 034h ;; 2 + DB 075h ;; 034h XOR 041h ;; 3 + DB 040h ;; 040h ;; 4 + DB 001h ;; 040h XOR 041h ;; 5 + DB 074h ;; 040h XOR 034h ;; 6 + DB 035h ;; 040h XOR 034h XOR 041h ;; 7 + DB 02Ah ;; 02Ah ;; 8 + DB 06Bh ;; 02Ah XOR 041h ;; 9 + DB 01Eh ;; 02Ah XOR 034h ;; a + DB 05Fh ;; 02Ah XOR 034h XOR 041h ;; b + DB 06Ah ;; 02Ah XOR 040h ;; c + DB 02Bh ;; 02Ah XOR 040h XOR 041h ;; d + DB 05Eh ;; 02Ah XOR 040h XOR 034h ;; e + DB 01Fh ;; 02Ah XOR 040h XOR 034hXOR 041h ;; f + +;; affine transformation constant (forward cipher) +FwdAffineCnt \ + DQ 0C2C2C2C2C2C2C2C2h,0C2C2C2C2C2C2C2C2h + +;; shift rows transformation (forward cipher) +FwdShiftRows \ + DB 0,5,10,15,4,9,14,3,8,13,2,7,12,1,6,11 + +;; mix columns transformation (forward cipher) +GF16mul_E_2x \ + DB 000h,02Eh,04Fh,061h,08Dh,0A3h,0C2h,0ECh,039h,017h,076h,058h,0B4h,09Ah,0FBh,0D5h +GF16mul_1_Cx \ + DB 000h,0C1h,0B2h,073h,054h,095h,0E6h,027h,0A8h,069h,01Ah,0DBh,0FCh,03Dh,04Eh,08Fh + +ColumnROR \ + DB 1,2,3,0,5,6,7,4,9,10,11,8,13,14,15,12 + + + +align IPP_ALIGN_FACTOR +;************************************************************* +;* void SafeEncrypt_RIJ128( +;* const Ipp32u* pInpBlk, +;* Ipp32u* pOutBlk, +;* int nr, +;* const Ipp32u* pKeys, +;* const void* Tables); +;* +;************************************************************* + +;; +;; Lib = V8 +;; +IPPASM SafeEncrypt_RIJ128,PUBLIC + USES_GPR esi,edi + +%xdefine pInpBlk [esp + ARG_1 + 0*sizeof(dword)] ; input block address +%xdefine pOutBlk [esp + ARG_1 + 1*sizeof(dword)] ; output block address +%xdefine nr [esp + ARG_1 + 2*sizeof(dword)] ; number of rounds +%xdefine pKey [esp + ARG_1 + 3*sizeof(dword)] ; key material address + +%assign RSIZE sizeof(dword) ; size of row +%assign SC 4 ; columns in STATE +%assign SSIZE RSIZE*SC ; size of state + + mov esi, pInpBlk + mov edi, pOutBlk + mov edx, pKey + mov ecx, nr + + LD_ADDR eax, ENCODE_DATA + + movdqu xmm1, oword [esi] ; input block + + movdqa xmm7, oword [eax+(GF16_csize-ENCODE_DATA)] + + ;; convert input into the composite GF((2^4)^2) + PTRANSFORM xmm0,xmm1, {eax+(TransFwdLO-ENCODE_DATA)},{eax+(TransFwdHI-ENCODE_DATA)}, xmm2,xmm3 + + ;; initial whitening + pxor xmm0, oword [edx] + add edx, SSIZE + + ;; (nr-1) regular rounds + sub ecx,1 + +.encode_round: + ;; SubByte() Transformation: + + ;; split input by low and upper parts + movdqa xmm1, xmm0 + pand xmm0, xmm7 ; low parts (4 bits) + psrlw xmm1, 4 + pand xmm1, xmm7 ; upper parts (4 bits) + + ;; compute multiplicative inverse + PINVERSE_GF16_FWD xmm1,xmm0, xmm3,xmm2,xmm4,xmm5 + + ;; affine transformation + movdqa xmm3, oword [eax+(FwdAffineLO-ENCODE_DATA)] + movdqa xmm2, oword [eax+(FwdAffineHI-ENCODE_DATA)] + movdqa xmm4, oword [eax+(FwdAffineCnt-ENCODE_DATA)] ; H(c), c=0x63 + pshufb xmm3, xmm0 + pshufb xmm2, xmm1 + pxor xmm3, xmm4 + pxor xmm3, xmm2 + + ;; ShiftRows() Transformation: + pshufb xmm3, [eax+(FwdShiftRows-ENCODE_DATA)] + + ;; MixColumn() Transformation: + movdqa xmm1, xmm3 + movdqa xmm2, xmm3 + pxor xmm4, xmm4 + psrlw xmm2, 4 + + pand xmm1, xmm7 ;; a0*(0xE + 0x2*t) + PLOOKUP_MEM xmm0, xmm1, [eax+(GF16mul_E_2x-ENCODE_DATA)] + + pand xmm2, xmm7 ;; a1*(0x2*0x9 + (0x2+0xE)*t) + PLOOKUP_MEM xmm1, xmm2, [eax+(GF16mul_1_Cx-ENCODE_DATA)] + + pxor xmm0, xmm1 + + pshufb xmm3, [eax+(ColumnROR-ENCODE_DATA)] + pxor xmm4, xmm3 + + pshufb xmm3, [eax+(ColumnROR-ENCODE_DATA)] + pxor xmm4, xmm3 + + movdqa xmm2, xmm0 + pshufb xmm2, [eax+(ColumnROR-ENCODE_DATA)] + pxor xmm0, xmm2 + + pshufb xmm3, [eax+(ColumnROR-ENCODE_DATA)] + pxor xmm4, xmm3 + + pxor xmm0, xmm4 + + ;; AddRoundKey() Transformation: + pxor xmm0, oword [edx] + add edx, SSIZE + + sub ecx,1 + jg .encode_round + + + ;; + ;; the last one is irregular + ;; + + ;; SubByte() Transformation: + movdqa xmm1, xmm0 + pand xmm0, xmm7 ; low parts (4 bits) + psrlw xmm1, 4 + pand xmm1, xmm7 ; upper parts (4 bits) + + ;; compute multiplicative inverse + PINVERSE_GF16_FWD xmm1,xmm0, xmm3,xmm2,xmm4,xmm5 + + ;; affine transformation + movdqa xmm3, oword [eax+(FwdAffineLO-ENCODE_DATA)] + movdqa xmm2, oword [eax+(FwdAffineHI-ENCODE_DATA)] + movdqa xmm4, oword [eax+(FwdAffineCnt-ENCODE_DATA)] ; H(c), c=0x63 + pshufb xmm3, xmm0 + pshufb xmm2, xmm1 + pxor xmm3, xmm4 + pxor xmm3, xmm2 + + ;; ShiftRows() Transformation: + pshufb xmm3, [eax+(FwdShiftRows-ENCODE_DATA)] + + ;; AddRoundKey() Transformation: + pxor xmm3, oword [edx] + add edx, SSIZE + + ;; convert output into the native GF(2^8) + PTRANSFORM xmm0,xmm3, {eax+(TransInvLO-ENCODE_DATA)},{eax+(TransInvHI-ENCODE_DATA)}, xmm2, xmm1 + + movdqu oword [edi], xmm0 + REST_GPR + ret +ENDFUNC SafeEncrypt_RIJ128 + +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprijnkeyw7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprijnkeyw7as.asm new file mode 100644 index 0000000..06c9244 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcprijnkeyw7as.asm @@ -0,0 +1,95 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Key Expansion Support +; +; Content: +; SubsDword_8uT() +; +; + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%if (_IPP >= _IPP_W7) + +segment .text align=IPP_ALIGN_FACTOR + + +%xdefine CACHE_LINE_SIZE (64) + +;*************************************************************** +;* Purpose: Mitigation of the Key Expansion procedure +;* +;* Ipp32u Touch_SubsDword_8uT(Ipp32u inp, +;* const Ipp8u* pTbl, +;* int tblBytes) +;*************************************************************** +align IPP_ALIGN_FACTOR +IPPASM Touch_SubsDword_8uT,PUBLIC + USES_GPR esi,edi,ebx + +%xdefine inp [esp + ARG_1 + 0*sizeof(dword)] ; input dword +%xdefine pTbl [esp + ARG_1 + 1*sizeof(dword)] ; Rijndael's S-box +%xdefine tblLen [esp + ARG_1 + 2*sizeof(dword)] ; length of table (bytes) + + mov esi, pTbl ; tbl address and + mov edx, tblLen ; length + xor ecx, ecx +.touch_tbl: + mov eax, [esi+ecx] + add ecx, CACHE_LINE_SIZE + cmp ecx, edx + jl .touch_tbl + + mov edx, inp + + mov eax, edx + and eax, 0FFh ; b[0] + movzx eax, BYTE [esi+eax] + + shr edx, 8 + mov ebx, edx + and ebx, 0FFh ; b[1] + movzx ebx, BYTE [esi+ebx] + shl ebx, 8 + + shr edx, 8 + mov ecx, edx + and ecx, 0FFh ; b[2] + movzx ecx, BYTE [esi+ecx] + shl ecx, 16 + + shr edx, 8 + movzx edx, BYTE [esi+edx] + shl edx, 24 + + or eax, ebx + or eax, ecx + or eax, edx + REST_GPR + ret +ENDFUNC Touch_SubsDword_8uT + +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha1nias.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha1nias.asm new file mode 100644 index 0000000..6ef5fbb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha1nias.asm @@ -0,0 +1,581 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SHA-1 +; +; Content: +; UpdateSHA1ni +; +; + + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SHA1_) +%if (_SHA_NI_ENABLING_ == _FEATURE_ON_) || (_SHA_NI_ENABLING_ == _FEATURE_TICKTOCK_) + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR +CODE_DATA: +UPPER_DWORD_MASK \ + DQ 00000000000000000h, 0ffffffff00000000h +PSHUFFLE_BYTE_FLIP_MASK \ + DB 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 + +align IPP_ALIGN_FACTOR +;***************************************************************************************** +;* Purpose: Update internal digest according to message block +;* +;* void UpdateSHA1ni(DigestSHA1 digest, const Ipp32u* mblk, int mlen, const void* pParam) +;* +;***************************************************************************************** + +%ifndef _VXWORKS + +IPPASM UpdateSHA1ni,PUBLIC + USES_GPR esi,edi,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pDigest [ebp + ARG_1 + 0*sizeof(dword)] ; pointer to the in/out digest +%xdefine pMsg [ebp + ARG_1 + 1*sizeof(dword)] ; pointer to the inp message +%xdefine msgLen [ebp + ARG_1 + 2*sizeof(dword)] ; message length + +%xdefine MBS_SHA1 (64) ; SHA-1 message block length (bytes) + +%xdefine HASH_PTR edi ; 1st arg +%xdefine MSG_PTR esi ; 2nd arg +%xdefine MSG_LEN edx ; 3rd arg + +%xdefine ABCD xmm0 +%xdefine E0 xmm1 ; Need two E's b/c they ping pong +%xdefine E1 xmm2 +%xdefine MSG0 xmm3 +%xdefine MSG1 xmm4 +%xdefine MSG2 xmm5 +%xdefine MSG3 xmm6 +%xdefine SHUF_MASK xmm7 + +; +; stack frame +; +%xdefine abcd_save eax +%xdefine e_save eax+sizeof(oword) +%xdefine frame_size sizeof(oword)+sizeof(oword) + + sub esp, (frame_size+16) + lea eax, [esp+16] + and eax, -16 + + mov MSG_LEN, msgLen ; message length + test MSG_LEN, MSG_LEN + jz .quit + + mov HASH_PTR, pDigest ; digest + mov MSG_PTR, pMsg ; and message pointers + + LD_ADDR ecx, CODE_DATA + +;; load initial hash values + movdqu ABCD, oword [HASH_PTR] + pinsrd E0, dword [HASH_PTR+16], 3 + ;pand E0, oword [UPPER_DWORD_MASK] + pand E0, oword [ecx+(UPPER_DWORD_MASK-CODE_DATA)] + pshufd ABCD, ABCD, 01Bh + + ;movdqa SHUF_MASK, oword [PSHUFFLE_BYTE_FLIP_MASK] + movdqa SHUF_MASK, oword [ecx+(PSHUFFLE_BYTE_FLIP_MASK-CODE_DATA)] + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +.sha1_block_loop: + movdqa oword [abcd_save], ABCD + movdqa oword [e_save], E0 + + ;; rounds 0-3 + movdqu MSG0, oword [MSG_PTR +0*16] + pshufb MSG0, SHUF_MASK + paddd E0, MSG0 + movdqa E1, ABCD + sha1rnds4 ABCD, E0, 0 + ;movdqu oword [rcx+16*0], ABCD + + ;; rounds 4-7 + movdqu MSG1, oword [MSG_PTR +1*16] + pshufb MSG1, SHUF_MASK + sha1nexte E1, MSG1 + movdqa E0, ABCD + sha1rnds4 ABCD, E1, 0 + sha1msg1 MSG0, MSG1 + ;movdqu oword [rcx+16*1], ABCD + + ;; rounds 8-11 + movdqu MSG2, oword [MSG_PTR +2*16] + pshufb MSG2, SHUF_MASK + sha1nexte E0, MSG2 + movdqa E1, ABCD + sha1rnds4 ABCD, E0, 0 + sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + ;movdqu oword [rcx+16*2], ABCD + + ;; rounds 12-15 + movdqu MSG3, oword [MSG_PTR +3*16] + pshufb MSG3, SHUF_MASK + sha1nexte E1, MSG3 + movdqa E0, ABCD + sha1msg2 MSG0, MSG3 + sha1rnds4 ABCD, E1, 0 + sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + ;movdqu oword [rcx+16*3], ABCD + + ;; rounds 16-19 + sha1nexte E0, MSG0 + movdqa E1, ABCD + sha1msg2 MSG1, MSG0 + sha1rnds4 ABCD, E0, 0 + sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + ;movdqu oword [rcx+16*4], ABCD + + ;; rounds 20-23 + sha1nexte E1, MSG1 + movdqa E0, ABCD + sha1msg2 MSG2, MSG1 + sha1rnds4 ABCD, E1, 1 + sha1msg1 MSG0, MSG1 + pxor MSG3, MSG1 + ;movdqu oword [rcx+16*5], ABCD + + ;; rounds 24-27 + sha1nexte E0, MSG2 + movdqa E1, ABCD + sha1msg2 MSG3, MSG2 + sha1rnds4 ABCD, E0, 1 + sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + ;movdqu oword [rcx+16*6], ABCD + + ;; rounds 28-31 + sha1nexte E1, MSG3 + movdqa E0, ABCD + sha1msg2 MSG0, MSG3 + sha1rnds4 ABCD, E1, 1 + sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + ;movdqu oword [rcx+16*7], ABCD + + ;; rounds 32-35 + sha1nexte E0, MSG0 + movdqa E1, ABCD + sha1msg2 MSG1, MSG0 + sha1rnds4 ABCD, E0, 1 + sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + ;movdqu oword [rcx+16*8], ABCD + + ;; rounds 36-39 + sha1nexte E1, MSG1 + movdqa E0, ABCD + sha1msg2 MSG2, MSG1 + sha1rnds4 ABCD, E1, 1 + sha1msg1 MSG0, MSG1 + pxor MSG3, MSG1 + ;movdqu oword [rcx+16*9], ABCD + + ;; rounds 40-43 + sha1nexte E0, MSG2 + movdqa E1, ABCD + sha1msg2 MSG3, MSG2 + sha1rnds4 ABCD, E0, 2 + sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + ;movdqu oword [rcx+16*10], ABCD + + ;; rounds 44-47 + sha1nexte E1, MSG3 + movdqa E0, ABCD + sha1msg2 MSG0, MSG3 + sha1rnds4 ABCD, E1, 2 + sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + ;movdqu oword [rcx+16*11], ABCD + + ;; rounds 48-51 + sha1nexte E0, MSG0 + movdqa E1, ABCD + sha1msg2 MSG1, MSG0 + sha1rnds4 ABCD, E0, 2 + sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + ;movdqu oword [rcx+16*12], ABCD + + ;; rounds 52-55 + sha1nexte E1, MSG1 + movdqa E0, ABCD + sha1msg2 MSG2, MSG1 + sha1rnds4 ABCD, E1, 2 + sha1msg1 MSG0, MSG1 + pxor MSG3, MSG1 + ;movdqu oword [rcx+16*13], ABCD + + ;; rounds 56-59 + sha1nexte E0, MSG2 + movdqa E1, ABCD + sha1msg2 MSG3, MSG2 + sha1rnds4 ABCD, E0, 2 + sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + ;movdqu oword [rcx+16*14], ABCD + + ;; rounds 60-63 + sha1nexte E1, MSG3 + movdqa E0, ABCD + sha1msg2 MSG0, MSG3 + sha1rnds4 ABCD, E1, 3 + sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + ;movdqu oword [rcx+16*15], ABCD + + ;; rounds 64-67 + sha1nexte E0, MSG0 + movdqa E1, ABCD + sha1msg2 MSG1, MSG0 + sha1rnds4 ABCD, E0, 3 + sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + ;movdqu oword [rcx+16*16], ABCD + + ;; rounds 68-71 + sha1nexte E1, MSG1 + movdqa E0, ABCD + sha1msg2 MSG2, MSG1 + sha1rnds4 ABCD, E1, 3 + pxor MSG3, MSG1 + ;movdqu oword [rcx+16*17], ABCD + + ;; rounds 72-75 + sha1nexte E0, MSG2 + movdqa E1, ABCD + sha1msg2 MSG3, MSG2 + sha1rnds4 ABCD, E0, 3 + ;movdqu oword [rcx+16*18], ABCD + + ;; rounds 76-79 + sha1nexte E1, MSG3 + movdqa E0, ABCD + sha1rnds4 ABCD, E1, 3 + ;movdqu oword [rcx+16*19], ABCD + + ;; add current hash values with previously saved + sha1nexte E0, oword [e_save] + paddd ABCD, oword [abcd_save] + + add MSG_PTR, MBS_SHA1 + sub MSG_LEN, MBS_SHA1 + jg .sha1_block_loop + + ;; write hash values back in the correct order + pshufd ABCD, ABCD, 01Bh + movdqu oword [HASH_PTR], ABCD + pextrd dword [HASH_PTR+16], E0, 3 + +.quit: + add esp, (frame_size+16) + REST_GPR + ret +ENDFUNC UpdateSHA1ni + +%else ;; no sha ni support in VxWorks - therefore we temporary use db +IPPASM UpdateSHA1ni,PUBLIC + USES_GPR esi,edi,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pDigest [ebp + ARG_1 + 0*sizeof(dword)] ; pointer to the in/out digest +%xdefine pMsg [ebp + ARG_1 + 1*sizeof(dword)] ; pointer to the inp message +%xdefine msgLen [ebp + ARG_1 + 2*sizeof(dword)] ; message length + +%xdefine MBS_SHA1 (64) ; SHA-1 message block length (bytes) + +%xdefine HASH_PTR edi ; 1st arg +%xdefine MSG_PTR esi ; 2nd arg +%xdefine MSG_LEN edx ; 3rd arg + +%xdefine ABCD xmm0 +%xdefine E0 xmm1 ; Need two E's b/c they ping pong +%xdefine E1 xmm2 +%xdefine MSG0 xmm3 +%xdefine MSG1 xmm4 +%xdefine MSG2 xmm5 +%xdefine MSG3 xmm6 +%xdefine SHUF_MASK xmm7 + +; +; stack frame +; +%xdefine abcd_save eax +%xdefine e_save eax+sizeof(oword) +%xdefine frame_size sizeof(oword)+sizeof(oword) + + sub esp, (frame_size+16) + lea eax, [esp+16] + and eax, -16 + + mov MSG_LEN, msgLen ; message length + test MSG_LEN, MSG_LEN + jz .quit + + mov HASH_PTR, pDigest ; digest + mov MSG_PTR, pMsg ; and message pointers + + LD_ADDR ecx, CODE_DATA + +;; load initial hash values + movdqu ABCD, oword [HASH_PTR] + pinsrd E0, dword [HASH_PTR+16], 3 + ;pand E0, oword [UPPER_DWORD_MASK] + pand E0, oword [ecx+(UPPER_DWORD_MASK-CODE_DATA)] + pshufd ABCD, ABCD, 01Bh + + ;movdqa SHUF_MASK, oword [PSHUFFLE_BYTE_FLIP_MASK] + movdqa SHUF_MASK, oword [ecx+(PSHUFFLE_BYTE_FLIP_MASK-CODE_DATA)] + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +.sha1_block_loop: + movdqa oword [abcd_save], ABCD + movdqa oword [e_save], E0 + + ;; rounds 0-3 + movdqu MSG0, oword [MSG_PTR +0*16] + pshufb MSG0, SHUF_MASK + paddd E0, MSG0 + movdqa E1, ABCD + db 0FH,03AH,0CCH,0C1H,00H ;; sha1rnds4 ABCD, E0, 0 + ;movdqu oword [rcx+16*0], ABCD + + ;; rounds 4-7 + movdqu MSG1, oword [MSG_PTR +1*16] + pshufb MSG1, SHUF_MASK + db 0FH,038H,0C8H,0D4H ;; sha1nexte E1, MSG1 + movdqa E0, ABCD + db 0FH,03AH,0CCH,0C2H,00H ;; sha1rnds4 ABCD, E1, 0 + db 0FH,038H,0C9H,0DCH ;; sha1msg1 MSG0, MSG1 + ;movdqu oword [rcx+16*1], ABCD + + ;; rounds 8-11 + movdqu MSG2, oword [MSG_PTR +2*16] + pshufb MSG2, SHUF_MASK + db 0FH,038H,0C8H,0CDH ;; sha1nexte E0, MSG2 + movdqa E1, ABCD + db 0FH,03AH,0CCH,0C1H,00H ;; sha1rnds4 ABCD, E0, 0 + db 0FH,038H,0C9H,0E5H ;; sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + ;movdqu oword [rcx+16*2], ABCD + + ;; rounds 12-15 + movdqu MSG3, oword [MSG_PTR +3*16] + pshufb MSG3, SHUF_MASK + db 0FH,038H,0C8H,0D6H ;; sha1nexte E1, MSG3 + movdqa E0, ABCD + db 0FH,038H,0CAH,0DEH ;; sha1msg2 MSG0, MSG3 + db 0FH,03AH,0CCH,0C2H,00H ;; sha1rnds4 ABCD, E1, 0 + db 0FH,038H,0C9H,0EEH ;; sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + ;movdqu oword [rcx+16*3], ABCD + + ;; rounds 16-19 + db 0FH,038H,0C8H,0CBH ;; sha1nexte E0, MSG0 + movdqa E1, ABCD + db 0FH,038H,0CAH,0E3H ;; sha1msg2 MSG1, MSG0 + db 0FH,03AH,0CCH,0C1H,00H ;; sha1rnds4 ABCD, E0, 0 + db 0FH,038H,0C9H,0F3H ;; sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + ;movdqu oword [rcx+16*4], ABCD + + ;; rounds 20-23 + db 0FH,038H,0C8H,0D4H ;; sha1nexte E1, MSG1 + movdqa E0, ABCD + db 0FH,038H,0CAH,0ECH ;; sha1msg2 MSG2, MSG1 + db 0FH,03AH,0CCH,0C2H,01H ;; sha1rnds4 ABCD, E1, 1 + db 0FH,038H,0C9H,0DCH ;; sha1msg1 MSG0, MSG1 + pxor MSG3, MSG1 + ;movdqu oword [rcx+16*5], ABCD + + ;; rounds 24-27 + db 0FH,038H,0C8H,0CDH ;; sha1nexte E0, MSG2 + movdqa E1, ABCD + db 0FH,038H,0CAH,0F5H ;; sha1msg2 MSG3, MSG2 + db 0FH,03AH,0CCH,0C1H,01H ;; sha1rnds4 ABCD, E0, 1 + db 0FH,038H,0C9H,0E5H ;; sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + ;movdqu oword [rcx+16*6], ABCD + + ;; rounds 28-31 + db 0FH,038H,0C8H,0D6H ;; sha1nexte E1, MSG3 + movdqa E0, ABCD + db 0FH,038H,0CAH,0DEH ;; sha1msg2 MSG0, MSG3 + db 0FH,03AH,0CCH,0C2H,01H ;; sha1rnds4 ABCD, E1, 1 + db 0FH,038H,0C9H,0EEH ;; sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + ;movdqu oword [rcx+16*7], ABCD + + ;; rounds 32-35 + db 0FH,038H,0C8H,0CBH ;; sha1nexte E0, MSG0 + movdqa E1, ABCD + db 0FH,038H,0CAH,0E3H ;; sha1msg2 MSG1, MSG0 + db 0FH,03AH,0CCH,0C1H,01H ;; sha1rnds4 ABCD, E0, 1 + db 0FH,038H,0C9H,0F3H ;; sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + ;movdqu oword [rcx+16*8], ABCD + + ;; rounds 36-39 + db 0FH,038H,0C8H,0D4H ;; sha1nexte E1, MSG1 + movdqa E0, ABCD + db 0FH,038H,0CAH,0ECH ;; sha1msg2 MSG2, MSG1 + db 0FH,03AH,0CCH,0C2H,01H ;; sha1rnds4 ABCD, E1, 1 + db 0FH,038H,0C9H,0DCH ;; sha1msg1 MSG0, MSG1 + pxor MSG3, MSG1 + ;movdqu oword [rcx+16*9], ABCD + + ;; rounds 40-43 + db 0FH,038H,0C8H,0CDH ;; sha1nexte E0, MSG2 + movdqa E1, ABCD + db 0FH,038H,0CAH,0F5H ;; sha1msg2 MSG3, MSG2 + db 0FH,03AH,0CCH,0C1H,02H ;; sha1rnds4 ABCD, E0, 2 + db 0FH,038H,0C9H,0E5H ;; sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + ;movdqu oword [rcx+16*10], ABCD + + ;; rounds 44-47 + db 0FH,038H,0C8H,0D6H ;; sha1nexte E1, MSG3 + movdqa E0, ABCD + db 0FH,038H,0CAH,0DEH ;; sha1msg2 MSG0, MSG3 + db 0FH,03AH,0CCH,0C2H,02H ;; sha1rnds4 ABCD, E1, 2 + db 0FH,038H,0C9H,0EEH ;; sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + ;movdqu oword [rcx+16*11], ABCD + + ;; rounds 48-51 + db 0FH,038H,0C8H,0CBH ;; sha1nexte E0, MSG0 + movdqa E1, ABCD + db 0FH,038H,0CAH,0E3H ;; sha1msg2 MSG1, MSG0 + db 0FH,03AH,0CCH,0C1H,02H ;; sha1rnds4 ABCD, E0, 2 + db 0FH,038H,0C9H,0F3H ;; sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + ;movdqu oword [rcx+16*12], ABCD + + ;; rounds 52-55 + db 0FH,038H,0C8H,0D4H ;; sha1nexte E1, MSG1 + movdqa E0, ABCD + db 0FH,038H,0CAH,0ECH ;; sha1msg2 MSG2, MSG1 + db 0FH,03AH,0CCH,0C2H,02H ;; sha1rnds4 ABCD, E1, 2 + db 0FH,038H,0C9H,0DCH ;; sha1msg1 MSG0, MSG1 + pxor MSG3, MSG1 + ;movdqu oword [rcx+16*13], ABCD + + ;; rounds 56-59 + db 0FH,038H,0C8H,0CDH ;; sha1nexte E0, MSG2 + movdqa E1, ABCD + db 0FH,038H,0CAH,0F5H ;; sha1msg2 MSG3, MSG2 + db 0FH,03AH,0CCH,0C1H,02H ;; sha1rnds4 ABCD, E0, 2 + db 0FH,038H,0C9H,0E5H ;; sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + ;movdqu oword [rcx+16*14], ABCD + + ;; rounds 60-63 + db 0FH,038H,0C8H,0D6H ;; sha1nexte E1, MSG3 + movdqa E0, ABCD + db 0FH,038H,0CAH,0DEH ;; sha1msg2 MSG0, MSG3 + db 0FH,03AH,0CCH,0C2H,03H ;; sha1rnds4 ABCD, E1, 3 + db 0FH,038H,0C9H,0EEH ;; sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + ;movdqu oword [rcx+16*15], ABCD + + ;; rounds 64-67 + db 0FH,038H,0C8H,0CBH ;; sha1nexte E0, MSG0 + movdqa E1, ABCD + db 0FH,038H,0CAH,0E3H ;; sha1msg2 MSG1, MSG0 + db 0FH,03AH,0CCH,0C1H,03H ;; sha1rnds4 ABCD, E0, 3 + db 0FH,038H,0C9H,0F3H ;; sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + ;movdqu oword [rcx+16*16], ABCD + + ;; rounds 68-71 + db 0FH,038H,0C8H,0D4H ;; sha1nexte E1, MSG1 + movdqa E0, ABCD + db 0FH,038H,0CAH,0ECH ;; sha1msg2 MSG2, MSG1 + db 0FH,03AH,0CCH,0C2H,03H ;; sha1rnds4 ABCD, E1, 3 + pxor MSG3, MSG1 + ;movdqu oword [rcx+16*17], ABCD + + ;; rounds 72-75 + db 0FH,038H,0C8H,0CDH ;; sha1nexte E0, MSG2 + movdqa E1, ABCD + db 0FH,038H,0CAH,0F5H ;; sha1msg2 MSG3, MSG2 + db 0FH,03AH,0CCH,0C1H,03H ;; sha1rnds4 ABCD, E0, 3 + ;movdqu oword [rcx+16*18], ABCD + + ;; rounds 76-79 + db 0FH,038H,0C8H,0D6H ;; sha1nexte E1, MSG3 + movdqa E0, ABCD + db 0FH,03AH,0CCH,0C2H,03H ;; sha1rnds4 ABCD, E1, 3 + ;movdqu oword [rcx+16*19], ABCD + + ;; add current hash values with previously saved + db 0FH,038H,0C8H,048H,10h ;; sha1nexte E0, oword [e_save] + paddd ABCD, oword [abcd_save] + + add MSG_PTR, MBS_SHA1 + sub MSG_LEN, MBS_SHA1 + jg .sha1_block_loop + + ;; write hash values back in the correct order + pshufd ABCD, ABCD, 01Bh + movdqu oword [HASH_PTR], ABCD + pextrd dword [HASH_PTR+16], E0, 3 + +.quit: + add esp, (frame_size+16) + REST_GPR + ret +ENDFUNC UpdateSHA1ni + +%endif ;; VxWorks + +%endif ;; _FEATURE_ON_ / _FEATURE_TICKTOCK_ +%endif ;; _ENABLE_ALG_SHA1_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha1w7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha1w7as.asm new file mode 100644 index 0000000..9d37c75 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha1w7as.asm @@ -0,0 +1,524 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SHA1 +; +; Content: +; UpdateSHA1 +; +; + + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SHA1_) +%if (_SHA_NI_ENABLING_ == _FEATURE_OFF_) || (_SHA_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP >= _IPP_M5) + +;; +;; Magic functions defined in FIPS 180-1 +;; +%macro MAGIC_F0 4.nolist + %xdefine %%regF %1 + %xdefine %%regB %2 + %xdefine %%regC %3 + %xdefine %%regD %4 + + mov %%regF,%%regC + xor %%regF,%%regD + and %%regF,%%regB + xor %%regF,%%regD +%endmacro + + +%macro MAGIC_F1 4.nolist + %xdefine %%regF %1 + %xdefine %%regB %2 + %xdefine %%regC %3 + %xdefine %%regD %4 + + mov %%regF,%%regD + xor %%regF,%%regC + xor %%regF,%%regB +%endmacro + + +%macro MAGIC_F2 5.nolist + %xdefine %%regF %1 + %xdefine %%regB %2 + %xdefine %%regC %3 + %xdefine %%regD %4 + %xdefine %%regT %5 + + mov %%regF,%%regB + mov %%regT,%%regB + or %%regF,%%regC + and %%regT,%%regC + and %%regF,%%regD + or %%regF,%%regT +%endmacro + + +%macro MAGIC_F3 4.nolist + %xdefine %%regF %1 + %xdefine %%regB %2 + %xdefine %%regC %3 + %xdefine %%regD %4 + + MAGIC_F1 %%regF,%%regB,%%regC,%%regD +%endmacro + + +;; +;; single SHA1 step +;; +;; Ipp32u tmp = ROL(A,5) + MAGIC_Fi(B,C,D) + E + W[t] + CNT[i]; +;; E = D; +;; D = C; +;; C = ROL(B,30); +;; B = A; +;; A = tmp; +;; +%macro SHA1_STEP 10.nolist + %xdefine %%regA %1 + %xdefine %%regB %2 + %xdefine %%regC %3 + %xdefine %%regD %4 + %xdefine %%regE %5 + %xdefine %%regT %6 + %xdefine %%regF %7 + %xdefine %%memW %8 + %xdefine %%immCNT %9 + %xdefine %%MAGIC %10 + + add %%regE,%%immCNT + add %%regE,[%%memW] + mov %%regT,%%regA + rol %%regT,5 + add %%regE,%%regT + %%MAGIC %%regF,%%regB,%%regC,%%regD,%%regT ;; FUN = MAGIC_Fi(B,C,D) + rol %%regB,30 + add %%regE,%%regF +%endmacro + + +;; +;; ENDIANNESS +;; +%macro ENDIANNESS 2.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + + %ifnidn %%dst,%%src + mov %%dst,%%src + %endif + bswap %%dst +%endmacro + + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; Following Macros are especially for new implementation of SHA1 +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%macro UPDATE 2-3.nolist + %xdefine %%nr %1 + %xdefine %%regU %2 + %xdefine %%regT %3 + + %ifempty %%regT + mov %%regU,[esp+((%%nr-16)&0Fh)*4] + xor %%regU,[esp+((%%nr-14)&0Fh)*4] + xor %%regU,[esp+((%%nr-8) &0Fh)*4] + xor %%regU,[esp+((%%nr-3) &0Fh)*4] + %else + mov %%regU,[esp+((%%nr-16)&0Fh)*4] + mov %%regT,[esp+((%%nr-14)&0Fh)*4] + xor %%regU,%%regT + mov %%regT,[esp+((%%nr-8) &0Fh)*4] + xor %%regU,%%regT + mov %%regT,[esp+((%%nr-3) &0Fh)*4] + xor %%regU,%%regT + %endif + rol %%regU,1 + mov [esp+(%%nr&0Fh)*4],%%regU +%endmacro + + +;; +;; single SHA1 step +;; +;; Ipp32u tmp = ROL(A,5) + MAGIC_Fi(B,C,D) + E + W[t] + CNT[i]; +;; E = D; +;; D = C; +;; C = ROL(B,30); +;; B = A; +;; A = tmp; +;; +%macro SHA1_RND0 8.nolist + %xdefine %%regA %1 + %xdefine %%regB %2 + %xdefine %%regC %3 + %xdefine %%regD %4 + %xdefine %%regE %5 + %xdefine %%regT %6 + %xdefine %%regF %7 + %xdefine %%nr %8 + + %assign %%immCNT 05A827999h + + MAGIC_F0 %%regF,%%regB,%%regC,%%regD ;; FUN = MAGIC_Fi(B,C,D) + ror %%regB,(32-30) + mov %%regT,%%regA + rol %%regT,5 + add %%regE,[esp+(((%%nr) & 0Fh)*4)] + lea %%regE,[%%regE+%%regF+%%immCNT] + add %%regE,%%regT +%endmacro + + +%macro SHA1_RND1 8.nolist + %xdefine %%regA %1 + %xdefine %%regB %2 + %xdefine %%regC %3 + %xdefine %%regD %4 + %xdefine %%regE %5 + %xdefine %%regT %6 + %xdefine %%regF %7 + %xdefine %%nr %8 + + %assign %%immCNT 06ED9EBA1h + + MAGIC_F1 %%regF,%%regB,%%regC,%%regD ;; FUN = MAGIC_Fi(B,C,D) + ror %%regB,(32-30) + mov %%regT,%%regA + rol %%regT,5 + add %%regE,[esp+(((%%nr)&0Fh)*4)] + lea %%regE,[%%regE+%%regF+%%immCNT] + add %%regE,%%regT +%endmacro + + +%macro SHA1_RND2 8.nolist + %xdefine %%regA %1 + %xdefine %%regB %2 + %xdefine %%regC %3 + %xdefine %%regD %4 + %xdefine %%regE %5 + %xdefine %%regT %6 + %xdefine %%regF %7 + %xdefine %%nr %8 + + %assign %%immCNT 08F1BBCDCh + + MAGIC_F2 %%regF,%%regB,%%regC,%%regD,%%regT ;; FUN = MAGIC_Fi(B,C,D) + ror %%regB,(32-30) + mov %%regT,%%regA + rol %%regT,5 + add %%regE,[esp+(((%%nr)&0Fh)*4)] + lea %%regE,[%%regE+%%regF+%%immCNT] + add %%regE,%%regT +%endmacro + + +%macro SHA1_RND3 8.nolist + %xdefine %%regA %1 + %xdefine %%regB %2 + %xdefine %%regC %3 + %xdefine %%regD %4 + %xdefine %%regE %5 + %xdefine %%regT %6 + %xdefine %%regF %7 + %xdefine %%nr %8 + + %assign %%immCNT 0CA62C1D6h + + MAGIC_F3 %%regF,%%regB,%%regC,%%regD ;; FUN = MAGIC_Fi(B,C,D) + ror %%regB,(32-30) + mov %%regT,%%regA + rol %%regT,5 + add %%regE,[esp+(((%%nr)&0Fh)*4)] + lea %%regE,[%%regE+%%regF+%%immCNT] + add %%regE,%%regT +%endmacro + + + +segment .text align=IPP_ALIGN_FACTOR + + +;***************************************************************************************** +;* Purpose: Update internal digest according to message block +;* +;* void UpdateSHA1(DigestSHA1 digest, const Ipp32u* mblk, int mlen, const void* pParam) +;* +;***************************************************************************************** + +;; +;; Lib = W7,V8 +;; +;; Caller = ippsSHA1Update +;; Caller = ippsSHA1Final +;; Caller = ippsSHA1MessageDigest +;; +;; Caller = ippsHMACSHA1Update +;; Caller = ippsHMACSHA1Final +;; Caller = ippsHMACSHA1MessageDigest +;; +align IPP_ALIGN_FACTOR +IPPASM UpdateSHA1,PUBLIC + USES_GPR esi,edi,ebx,ebp + +%xdefine digest [esp + ARG_1 + 0*sizeof(dword)] ; hash address +%xdefine mblk [esp + ARG_1 + 1*sizeof(dword)] ; buffer address +%xdefine mlen [esp + ARG_1 + 2*sizeof(dword)] ; buffer length +%xdefine pTable [esp + ARG_1 + 3*sizeof(dword)] ; pointer to SHA1 const (dummy) + +%xdefine MBS_SHA1 (64) + +%assign stackSize (16+3) ; stack size + mov eax, pTable ; dummy + mov esi,mblk ; source data address + mov eax,mlen ; data length + mov edi,digest ; hash address + + sub esp,stackSize*4 ; allocate local buffer + mov [esp+stackSize*4-3*4],edi ; save hash address + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +.sha1_block_loop: + mov [esp+stackSize*4-2*4],esi ; save data address + mov [esp+stackSize*4-1*4],eax ; save data length + +;; +;; initialize the first 16 words in the array W (remember about endian) +;; + xor ecx,ecx +.loop1: + mov eax,[esi+ecx*4+0*4] + mov edx,[esi+ecx*4+1*4] + ENDIANNESS eax,eax + ENDIANNESS edx,edx + mov [esp+ecx*4+0*4],eax + mov [esp+ecx*4+1*4],edx + add ecx,2 + cmp ecx,16 + jl .loop1 + +;; +;; init A, B, C, D, E by the internal digest +;; + mov eax,[edi+0*4] ; r2 = digest[0] (A) + mov ebx,[edi+1*4] ; r3 = digest[1] (B) + mov ecx,[edi+2*4] ; r4 = digest[2] (C) + mov edx,[edi+3*4] ; r5 = digest[3] (D) + mov ebp,[edi+4*4] ; r6 = digest[4] (E) + +;; +;; perform 0-79 steps +;; +;; A, B, C, D, E, TMP,FUN, round +;; ----------------------------------- + SHA1_RND0 eax,ebx,ecx,edx,ebp, esi,edi, 0 + UPDATE 16, esi + SHA1_RND0 ebp,eax,ebx,ecx,edx, esi,edi, 1 + UPDATE 17, esi + SHA1_RND0 edx,ebp,eax,ebx,ecx, esi,edi, 2 + UPDATE 18, esi + SHA1_RND0 ecx,edx,ebp,eax,ebx, esi,edi, 3 + UPDATE 19, esi + SHA1_RND0 ebx,ecx,edx,ebp,eax, esi,edi, 4 + UPDATE 20, esi + SHA1_RND0 eax,ebx,ecx,edx,ebp, esi,edi, 5 + UPDATE 21, esi + SHA1_RND0 ebp,eax,ebx,ecx,edx, esi,edi, 6 + UPDATE 22, esi + SHA1_RND0 edx,ebp,eax,ebx,ecx, esi,edi, 7 + UPDATE 23, esi + SHA1_RND0 ecx,edx,ebp,eax,ebx, esi,edi, 8 + UPDATE 24, esi + SHA1_RND0 ebx,ecx,edx,ebp,eax, esi,edi, 9 + UPDATE 25, esi + SHA1_RND0 eax,ebx,ecx,edx,ebp, esi,edi, 10 + UPDATE 26, esi + SHA1_RND0 ebp,eax,ebx,ecx,edx, esi,edi, 11 + UPDATE 27, esi + SHA1_RND0 edx,ebp,eax,ebx,ecx, esi,edi, 12 + UPDATE 28, esi + SHA1_RND0 ecx,edx,ebp,eax,ebx, esi,edi, 13 + UPDATE 29, esi + SHA1_RND0 ebx,ecx,edx,ebp,eax, esi,edi, 14 + UPDATE 30, esi + SHA1_RND0 eax,ebx,ecx,edx,ebp, esi,edi, 15 + UPDATE 31, esi + SHA1_RND0 ebp,eax,ebx,ecx,edx, esi,edi, 16 + UPDATE 32, esi + SHA1_RND0 edx,ebp,eax,ebx,ecx, esi,edi, 17 + UPDATE 33, esi + SHA1_RND0 ecx,edx,ebp,eax,ebx, esi,edi, 18 + UPDATE 34, esi + SHA1_RND0 ebx,ecx,edx,ebp,eax, esi,edi, 19 + UPDATE 35, esi + + SHA1_RND1 eax,ebx,ecx,edx,ebp, esi,edi, 20 + UPDATE 36, esi + SHA1_RND1 ebp,eax,ebx,ecx,edx, esi,edi, 21 + UPDATE 37, esi + SHA1_RND1 edx,ebp,eax,ebx,ecx, esi,edi, 22 + UPDATE 38, esi + SHA1_RND1 ecx,edx,ebp,eax,ebx, esi,edi, 23 + UPDATE 39, esi + SHA1_RND1 ebx,ecx,edx,ebp,eax, esi,edi, 24 + UPDATE 40, esi + SHA1_RND1 eax,ebx,ecx,edx,ebp, esi,edi, 25 + UPDATE 41, esi + SHA1_RND1 ebp,eax,ebx,ecx,edx, esi,edi, 26 + UPDATE 42, esi + SHA1_RND1 edx,ebp,eax,ebx,ecx, esi,edi, 27 + UPDATE 43, esi + SHA1_RND1 ecx,edx,ebp,eax,ebx, esi,edi, 28 + UPDATE 44, esi + SHA1_RND1 ebx,ecx,edx,ebp,eax, esi,edi, 29 + UPDATE 45, esi + SHA1_RND1 eax,ebx,ecx,edx,ebp, esi,edi, 30 + UPDATE 46, esi + SHA1_RND1 ebp,eax,ebx,ecx,edx, esi,edi, 31 + UPDATE 47, esi + SHA1_RND1 edx,ebp,eax,ebx,ecx, esi,edi, 32 + UPDATE 48, esi + SHA1_RND1 ecx,edx,ebp,eax,ebx, esi,edi, 33 + UPDATE 49, esi + SHA1_RND1 ebx,ecx,edx,ebp,eax, esi,edi, 34 + UPDATE 50, esi + SHA1_RND1 eax,ebx,ecx,edx,ebp, esi,edi, 35 + UPDATE 51, esi + SHA1_RND1 ebp,eax,ebx,ecx,edx, esi,edi, 36 + UPDATE 52, esi + SHA1_RND1 edx,ebp,eax,ebx,ecx, esi,edi, 37 + UPDATE 53, esi + SHA1_RND1 ecx,edx,ebp,eax,ebx, esi,edi, 38 + UPDATE 54, esi + SHA1_RND1 ebx,ecx,edx,ebp,eax, esi,edi, 39 + UPDATE 55, esi + + SHA1_RND2 eax,ebx,ecx,edx,ebp, esi,edi, 40 + UPDATE 56, esi + SHA1_RND2 ebp,eax,ebx,ecx,edx, esi,edi, 41 + UPDATE 57, esi + SHA1_RND2 edx,ebp,eax,ebx,ecx, esi,edi, 42 + UPDATE 58, esi + SHA1_RND2 ecx,edx,ebp,eax,ebx, esi,edi, 43 + UPDATE 59, esi + SHA1_RND2 ebx,ecx,edx,ebp,eax, esi,edi, 44 + UPDATE 60, esi + SHA1_RND2 eax,ebx,ecx,edx,ebp, esi,edi, 45 + UPDATE 61, esi + SHA1_RND2 ebp,eax,ebx,ecx,edx, esi,edi, 46 + UPDATE 62, esi + SHA1_RND2 edx,ebp,eax,ebx,ecx, esi,edi, 47 + UPDATE 63, esi + SHA1_RND2 ecx,edx,ebp,eax,ebx, esi,edi, 48 + UPDATE 64, esi + SHA1_RND2 ebx,ecx,edx,ebp,eax, esi,edi, 49 + UPDATE 65, esi + SHA1_RND2 eax,ebx,ecx,edx,ebp, esi,edi, 50 + UPDATE 66, esi + SHA1_RND2 ebp,eax,ebx,ecx,edx, esi,edi, 51 + UPDATE 67, esi + SHA1_RND2 edx,ebp,eax,ebx,ecx, esi,edi, 52 + UPDATE 68, esi + SHA1_RND2 ecx,edx,ebp,eax,ebx, esi,edi, 53 + UPDATE 69, esi + SHA1_RND2 ebx,ecx,edx,ebp,eax, esi,edi, 54 + UPDATE 70, esi + SHA1_RND2 eax,ebx,ecx,edx,ebp, esi,edi, 55 + UPDATE 71, esi + SHA1_RND2 ebp,eax,ebx,ecx,edx, esi,edi, 56 + UPDATE 72, esi + SHA1_RND2 edx,ebp,eax,ebx,ecx, esi,edi, 57 + UPDATE 73, esi + SHA1_RND2 ecx,edx,ebp,eax,ebx, esi,edi, 58 + UPDATE 74, esi + SHA1_RND2 ebx,ecx,edx,ebp,eax, esi,edi, 59 + UPDATE 75, esi + + SHA1_RND3 eax,ebx,ecx,edx,ebp, esi,edi, 60 + UPDATE 76, esi + SHA1_RND3 ebp,eax,ebx,ecx,edx, esi,edi, 61 + UPDATE 77, esi + SHA1_RND3 edx,ebp,eax,ebx,ecx, esi,edi, 62 + UPDATE 78, esi + SHA1_RND3 ecx,edx,ebp,eax,ebx, esi,edi, 63 + UPDATE 79, esi + SHA1_RND3 ebx,ecx,edx,ebp,eax, esi,edi, 64 + SHA1_RND3 eax,ebx,ecx,edx,ebp, esi,edi, 65 + SHA1_RND3 ebp,eax,ebx,ecx,edx, esi,edi, 66 + SHA1_RND3 edx,ebp,eax,ebx,ecx, esi,edi, 67 + SHA1_RND3 ecx,edx,ebp,eax,ebx, esi,edi, 68 + SHA1_RND3 ebx,ecx,edx,ebp,eax, esi,edi, 69 + SHA1_RND3 eax,ebx,ecx,edx,ebp, esi,edi, 70 + SHA1_RND3 ebp,eax,ebx,ecx,edx, esi,edi, 71 + SHA1_RND3 edx,ebp,eax,ebx,ecx, esi,edi, 72 + SHA1_RND3 ecx,edx,ebp,eax,ebx, esi,edi, 73 + SHA1_RND3 ebx,ecx,edx,ebp,eax, esi,edi, 74 + SHA1_RND3 eax,ebx,ecx,edx,ebp, esi,edi, 75 + SHA1_RND3 ebp,eax,ebx,ecx,edx, esi,edi, 76 + SHA1_RND3 edx,ebp,eax,ebx,ecx, esi,edi, 77 + SHA1_RND3 ecx,edx,ebp,eax,ebx, esi,edi, 78 + SHA1_RND3 ebx,ecx,edx,ebp,eax, esi,edi, 79 + +;; +;; update digest +;; + mov edi,[esp+stackSize*4-3*4] ; restore hash address + mov esi,[esp+stackSize*4-2*4] ; restore data address + + add [edi+0*4],eax ; advance digest + mov eax,[esp+stackSize*4-1*4] ; restore data length + add [edi+1*4],ebx + add [edi+2*4],ecx + add [edi+3*4],edx + add [edi+4*4],ebp + + add esi, MBS_SHA1 + sub eax, MBS_SHA1 + jg .sha1_block_loop + + add esp,stackSize*4 ; remove local buffer + REST_GPR + ret +ENDFUNC UpdateSHA1 + +%endif ;; _IPP >= _IPP_M5 +%endif ;; _FEATURE_OFF_ / _FEATURE_TICKTOCK_ +%endif ;; _ENABLE_ALG_SHA1_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha256g9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha256g9as.asm new file mode 100644 index 0000000..9a585dc --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha256g9as.asm @@ -0,0 +1,444 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SHA256 +; +; Content: +; UpdateSHA256 +; +; + + + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SHA256_) +%if (_SHA_NI_ENABLING_ == _FEATURE_OFF_) || (_SHA_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP >= _IPP_G9) + +%xdefine XMM_SHUFB_BSWAP xmm6 +%xdefine W0 xmm0 +%xdefine W4 xmm1 +%xdefine W8 xmm2 +%xdefine W12 xmm3 +%xdefine SIG1 xmm4 +%xdefine SIG0 xmm5 +%xdefine X xmm6 +%xdefine W xmm7 + +%xdefine regTbl ebx + + +;; we are considering x, y, z are polynomials over GF(2) +;; & - multiplication +;; ^ - additive +;; operations + +;; +;; Chj(x,y,z) = (x&y) ^ (~x & z) +;; = (x&y) ^ ((1^x) &z) +;; = (x&y) ^ (z ^ x&z) +;; = x&y ^ z ^ x&z +;; = x&(y^z) ^z +;; +%macro Chj 4.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%Y %3 + %xdefine %%Z %4 + + mov %%F, %%Y + xor %%F, %%Z + and %%F, %%X + xor %%F, %%Z +%endmacro + + +;; +;; Maj(x,y,z) = (x&y) ^ (x&z) ^ (y&z) +;; = (x&y) ^ (x&z) ^ (y&z) ^ (z&z) ^z // note: ((z&z) ^z) = 0 +;; = x&(y^z) ^ z&(y^z) ^z +;; = (x^z)&(y^z) ^z +;; +%macro Maj 4.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%Y %3 + %xdefine %%Z %4 + + mov %%F, %%X + xor %%F, %%Z + xor %%Z, %%Y + and %%F, %%Z + xor %%Z, %%Y + xor %%F, %%Z +%endmacro + + +%macro ROTR 2.nolist + %xdefine %%X %1 + %xdefine %%n %2 + + shrd %%X,%%X, %%n + ;;ror X, n +%endmacro + + +;; +;; Summ0(x) = ROR(x,2) ^ ROR(x,13) ^ ROR(x,22) +;; +%macro Summ0 3.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%T %3 + + mov %%F, %%X + ROTR %%F, 2 + mov %%T, %%X + ROTR %%T, 13 + xor %%F, %%T + ROTR %%T, (22-13) + xor %%F, %%T +%endmacro + + +;; +;; Summ1(x) = ROR(x,6) ^ ROR(x,11) ^ ROR(x,25) +;; +%macro Summ1 3.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%T %3 + + mov %%F, %%X + ROTR %%F, 6 + mov %%T, %%X + ROTR %%T, 11 + xor %%F, %%T + ROTR %%T, (25-11) + xor %%F, %%T +%endmacro + + +;; +;; regular round (i): +;; +;; T1 = h + Sigma1(e) + Ch(e,f,g) + K[i] + W[i] +;; T2 = Sigma0(a) + Maj(a,b,c) +;; h = g +;; g = f +;; f = e +;; e = d + T1 +;; d = c +;; c = b +;; b = a +;; a = T1+T2 +;; +;; or +;; +;; h += Sigma1(e) + Ch(e,f,g) + K[i] + W[i] (==T1) +;; d += h +;; T2 = Sigma0(a) + Maj(a,b,c) +;; h += T2 +;; and following textual shift {a,b,c,d,e,f,g,h} => {h,a,b,c,d,e,f,g} +;; +%macro ROUND 6.nolist + %xdefine %%nr %1 + %xdefine %%hashBuff %2 + %xdefine %%wBuff %3 + %xdefine %%F1 %4 + %xdefine %%F2 %5 + %xdefine %%T1 %6 + ; %xdefine T2 %7 + + Summ1 %%F1, eax, %%T1 + Chj %%F2, eax,{[%%hashBuff+((vF-%%nr)&7)*sizeof(dword)]},{[%%hashBuff+((vG-%%nr)&7)*sizeof(dword)]} + mov eax, [%%hashBuff+((vH-%%nr)&7)*sizeof(dword)] + add eax, %%F1 + add eax, %%F2 + add eax, dword [%%wBuff+(%%nr&3)*sizeof(dword)] + + mov %%F1, dword [%%hashBuff+((vB-%%nr)&7)*sizeof(dword)] + mov %%T1, dword [%%hashBuff+((vC-%%nr)&7)*sizeof(dword)] + Maj %%F2, edx,%%F1, %%T1 + Summ0 %%F1, edx, %%T1 + lea edx, [%%F1+%%F2] + + add edx,eax ; T2+T1 + add eax,[%%hashBuff+((vD-%%nr)&7)*sizeof(dword)] ; T1+d + + mov [%%hashBuff+((vH-%%nr)&7)*sizeof(dword)],edx + mov [%%hashBuff+((vD-%%nr)&7)*sizeof(dword)],eax +%endmacro + + +;; +;; W[i] = Sigma1(W[i-2]) + W[i-7] + Sigma0(W[i-15]) + W[i-16], i=16,..,63 +;; +;;for next rounds 16,17,18 and 19: +;; W[0] <= W[16] = Sigma1(W[14]) + W[ 9] + Sigma0(W[1]) + W[0] +;; W[1] <= W[17] = Sigma1(W[15]) + W[10] + Sigma0(W[2]) + W[1] +;; W[2] <= W[18] = Sigma1(W[ 0]) + W[11] + Sigma0(W[3]) + W[1] +;; W[3] <= W[19] = Sigma1(W[ 1]) + W[12] + Sigma0(W[4]) + W[2] +;; +;; the process is repeated exactly because texual round of W[] +;; +;; Sigma1() and Sigma0() functions are defined as following: +;; Sigma1(X) = ROR(X,17)^ROR(X,19)^SHR(X,10) +;; Sigma0(X) = ROR(X, 7)^ROR(X,18)^SHR(X, 3) +;; +%macro UPDATE_W 8.nolist + %xdefine %%xS %1 + %xdefine %%xS0 %2 + %xdefine %%xS4 %3 + %xdefine %%xS8 %4 + %xdefine %%xS12 %5 + %xdefine %%SIGMA1 %6 + %xdefine %%SIGMA0 %7 + %xdefine %%X %8 + + vpshufd %%X, %%xS12, 11111010b ;; SIGMA1 = {W[15],W[15],W[14],W[14]} + vpsrld %%SIGMA1, %%X, 10 + vpsrlq %%X, %%X, 17 + vpxor %%SIGMA1, %%SIGMA1, %%X + vpsrlq %%X, %%X, (19-17) + vpxor %%SIGMA1, %%SIGMA1, %%X + + vpshufd %%X, %%xS0, 10100101b ;; SIGMA0 = {W[2],W[2],W[1],W[1]} + vpsrld %%SIGMA0, %%X, 3 + vpsrlq %%X, %%X, 7 + vpxor %%SIGMA0, %%SIGMA0, %%X + vpsrlq %%X, %%X, (18-7) + vpxor %%SIGMA0, %%SIGMA0, %%X + + vpshufd %%xS, %%xS0, 01010000b ;; {W[ 1],W[ 1],W[ 0],W[ 0]} + vpaddd %%SIGMA1, %%SIGMA1, %%SIGMA0 + vpshufd %%X, %%xS8, 10100101b ;; {W[10],W[10],W[ 9],W[ 9]} + vpaddd %%xS, %%xS, %%SIGMA1 + vpaddd %%xS, %%xS, %%X + + + vpshufd %%X, %%xS, 10100000b ;; SIGMA1 = {W[1],W[1],W[0],W[0]} + vpsrld %%SIGMA1, %%X, 10 + vpsrlq %%X, %%X, 17 + vpxor %%SIGMA1, %%SIGMA1, %%X + vpsrlq %%X, %%X, (19-17) + vpxor %%SIGMA1, %%SIGMA1, %%X + + vpalignr %%X, %%xS4, %%xS0, (3*sizeof(dword)) ;; SIGMA0 = {W[4],W[4],W[3],W[3]} + vpshufd %%X, %%X, 01010000b + vpsrld %%SIGMA0, %%X, 3 + vpsrlq %%X, %%X, 7 + vpxor %%SIGMA0, %%SIGMA0, %%X + vpsrlq %%X, %%X, (18-7) + vpxor %%SIGMA0, %%SIGMA0, %%X + + vpalignr %%X, %%xS12, %%xS8, (3*sizeof(dword)) ;; {W[14],W[13],W[12],W[11]} + vpshufd %%xS0, %%xS0, 11111010b ;; {W[ 3],W[ 3],W[ 2],W[ 2]} + vpaddd %%SIGMA1, %%SIGMA1, %%SIGMA0 + vpshufd %%X, %%X, 01010000b ;; {W[12],W[12],W[11],W[11]} + vpaddd %%xS0, %%xS0, %%SIGMA1 + vpaddd %%xS0, %%xS0, %%X + + vpshufd %%xS, %%xS, 10001000b ;; {W[1],W[0],W[1],W[0]} + vpshufd %%xS0, %%xS0, 10001000b ;; {W[3],W[2],W[3],W[2]} + vpalignr %%xS0, %%xS0, %%xS, (2*sizeof(dword)) ;; {W[3],W[2],W[1],W[0]} +%endmacro + + + +segment .text align=IPP_ALIGN_FACTOR + + +align IPP_ALIGN_FACTOR +SWP_BYTE: +pByteSwp DB 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; UpdateSHA256(Ipp32u digest[], Ipp8u dataBlock[], int datalen, Ipp32u K_256[]) +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM UpdateSHA256,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pHash [ebp + ARG_1 + 0*sizeof(dword)] ; pointer to hash +%xdefine pData [ebp + ARG_1 + 1*sizeof(dword)] ; pointer to data block +%xdefine dataLen [ebp + ARG_1 + 2*sizeof(dword)] ; data length +%xdefine pTbl [ebp + ARG_1 + 3*sizeof(dword)] ; pointer to the SHA256 const table + +%xdefine MBS_SHA256 (64) + +%assign hSize sizeof(dword)*8 ; size of hash +%assign wSize sizeof(oword) ; W values queue (dwords) +%assign cntSize sizeof(dword) ; local counter + +%assign hashOff 0 ; hash address +%assign wOff hashOff+hSize ; W values offset +%assign cntOff wOff+wSize + +%assign stackSize (hSize+wSize+cntSize) ; stack size + sub esp, stackSize + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +.sha256_block_loop: + + mov eax, pHash ; pointer to the hash + vmovdqu W0, oword [eax] ; load initial hash value + vmovdqu W4, oword [eax+sizeof(oword)] + vmovdqu oword [esp+hashOff], W0 + vmovdqu oword [esp+hashOff+sizeof(oword)*1], W4 + + mov eax, pData ; pointer to the data block + mov regTbl, pTbl ; pointer to SHA256 table (points K_256[] constants) + ;vmovdqa XMM_SHUFB_BSWAP, oword pByteSwp ; load shuffle mask + LD_ADDR ecx, SWP_BYTE + movdqa XMM_SHUFB_BSWAP, oword [ecx+(pByteSwp-SWP_BYTE)] + + vmovdqu W0, oword [eax] ; load buffer content + vmovdqu W4, oword [eax+sizeof(oword)] + vmovdqu W8, oword [eax+sizeof(oword)*2] + vmovdqu W12,oword [eax+sizeof(oword)*3] + +%assign vA 0 +%assign vB 1 +%assign vC 2 +%assign vD 3 +%assign vE 4 +%assign vF 5 +%assign vG 6 +%assign vH 7 + + mov eax, [esp+hashOff+vE*sizeof(dword)] + mov edx, [esp+hashOff+vA*sizeof(dword)] + + ;; perform 0-3 regular rounds + vpshufb W0, W0, XMM_SHUFB_BSWAP ; swap input + vpaddd W, W0, oword [regTbl+sizeof(oword)*0] ; T += K_SHA256[0-3] + vmovdqu oword [esp+wOff], W + ROUND 0, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 1, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 2, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 3, {esp+hashOff},{esp+wOff}, esi,edi,ecx + + ;; perform next 4-7 regular rounds + vpshufb W4, W4, XMM_SHUFB_BSWAP ; swap input + vpaddd W, W4, oword [regTbl+sizeof(oword)*1] ; T += K_SHA256[4-7] + vmovdqu oword [esp+wOff], W + ROUND 4, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 5, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 6, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 7, {esp+hashOff},{esp+wOff}, esi,edi,ecx + + ;; perform next 8-11 regular rounds + vpshufb W8, W8, XMM_SHUFB_BSWAP ; swap input + vpaddd W, W8, oword [regTbl+sizeof(oword)*2] ; T += K_SHA256[8-11] + vmovdqu oword [esp+wOff], W + ROUND 8, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 9, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 10, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 11, {esp+hashOff},{esp+wOff}, esi,edi,ecx + + ;; perform next 12-15 regular rounds + vpshufb W12, W12, XMM_SHUFB_BSWAP ; swap input + vpaddd W, W12, oword [regTbl+sizeof(oword)*3] ; T += K_SHA256[12-15] + vmovdqu oword [esp+wOff], W + ROUND 12, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 13, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 14, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 15, {esp+hashOff},{esp+wOff}, esi,edi,ecx + + mov dword [esp+cntOff], (64-16) ; init counter +.loop_16_63: + add regTbl, sizeof(oword)*4 ; update SHA_256 pointer + + UPDATE_W W, W0, W4, W8, W12, SIG1,SIG0,X ; round: 16*i - 16*i+3 + vpaddd W, W0, oword [regTbl+sizeof(oword)*0] ; T += K_SHA256[16-19] + vmovdqu oword [esp+wOff], W + ROUND 16, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 17, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 18, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 19, {esp+hashOff},{esp+wOff}, esi,edi,ecx + + UPDATE_W W, W4, W8, W12,W0, SIG1,SIG0,X ; round: 20*i 20*i+3 + vpaddd W, W4, oword [regTbl+sizeof(oword)*1] ; T += K_SHA256[20-23] + vmovdqu oword [esp+wOff], W + ROUND 20, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 21, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 22, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 23, {esp+hashOff},{esp+wOff}, esi,edi,ecx + + UPDATE_W W, W8, W12,W0, W4, SIG1,SIG0,X ; round: 24*i - 24*i+3 + vpaddd W, W8, oword [regTbl+sizeof(oword)*2] ; T += K_SHA256[24-27] + vmovdqu oword [esp+wOff], W + ROUND 24, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 25, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 26, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 27, {esp+hashOff},{esp+wOff}, esi,edi,ecx + + UPDATE_W W, W12,W0, W4, W8, SIG1,SIG0,X ; round: 28*i - 28*i+3 + vpaddd W, W12, oword [regTbl+sizeof(oword)*3]; T += K_SHA256[28-31] + vmovdqu oword [esp+wOff], W + ROUND 28, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 29, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 30, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 31, {esp+hashOff},{esp+wOff}, esi,edi,ecx + + sub dword [esp+cntOff], 16 + jg .loop_16_63 + + mov eax, pHash ; pointer to the hash + vmovdqu W0, oword [esp+hashOff] + vmovdqu W4, oword [esp+hashOff+sizeof(oword)*1] + + ; update hash + vmovdqu W, oword [eax] + vpaddd W, W, W0 + vmovdqu oword [eax], W + vmovdqu W, oword [eax+sizeof(oword)] + vpaddd W, W, W4 + vmovdqu oword [eax+sizeof(oword)], W + + add dword pData, MBS_SHA256 + sub dword dataLen, MBS_SHA256 + jg .sha256_block_loop + + add esp, stackSize + REST_GPR + ret +ENDFUNC UpdateSHA256 + +%endif ;; _IPP32E_G9 and above +%endif ;; _FEATURE_OFF_ / _FEATURE_TICKTOCK_ +%endif ;; _ENABLE_ALG_SHA256_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha256nias.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha256nias.asm new file mode 100644 index 0000000..fd6e3e7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha256nias.asm @@ -0,0 +1,617 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SHA-256 +; +; Content: +; UpdateSHA256ni +; +; + + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SHA256_) +%if (_SHA_NI_ENABLING_ == _FEATURE_ON_) || (_SHA_NI_ENABLING_ == _FEATURE_TICKTOCK_) + + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR +CODE_DATA: +PSHUFFLE_BYTE_FLIP_MASK \ + DB 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12 + +align IPP_ALIGN_FACTOR +;***************************************************************************************** +;* Purpose: Update internal digest according to message block +;* +;* void UpdateSHA256ni(DigestSHA256 digest, const Ipp8u* msg, int mlen, const Ipp32u K_256[]) +;* +;***************************************************************************************** + +%ifndef _VXWORKS + +IPPASM UpdateSHA256ni,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pDigest [ebp + ARG_1 + 0*sizeof(dword)] ; pointer to the in/out digest +%xdefine pMsg [ebp + ARG_1 + 1*sizeof(dword)] ; pointer to the inp message +%xdefine msgLen [ebp + ARG_1 + 2*sizeof(dword)] ; message length +%xdefine pTbl [ebp + ARG_1 + 3*sizeof(dword)] ; pointer to SHA256 table of constants + +%xdefine MBS_SHA256 (64) ; SHA-1 message block length (bytes) + +%xdefine HASH_PTR edi ; 1st arg +%xdefine MSG_PTR esi ; 2nd arg +%xdefine MSG_LEN edx ; 3rd arg +%xdefine K256_PTR ebx ; 4rd arg + +%xdefine MSG xmm0 +%xdefine STATE0 xmm1 +%xdefine STATE1 xmm2 +%xdefine MSGTMP0 xmm3 +%xdefine MSGTMP1 xmm4 +%xdefine MSGTMP2 xmm5 +%xdefine MSGTMP3 xmm6 +%xdefine MSGTMP4 xmm7 + +; +; stack frame +; +%xdefine mask_save eax +%xdefine abef_save eax+sizeof(oword) +%xdefine cdgh_save eax+sizeof(oword)*2 +%xdefine frame_size sizeof(oword)+sizeof(oword)+sizeof(oword) + + sub esp, (frame_size+16) + lea eax, [esp+16] + and eax, -16 + + mov MSG_LEN, msgLen ; message length + test MSG_LEN, MSG_LEN + jz .quit + + mov HASH_PTR, pDigest + mov MSG_PTR, pMsg + mov K256_PTR, pTbl + + ;; load input hash value, reorder these appropriately + movdqu STATE0, oword [HASH_PTR+0*sizeof(oword)] + movdqu STATE1, oword [HASH_PTR+1*sizeof(oword)] + + pshufd STATE0, STATE0, 0B1h ; CDAB + pshufd STATE1, STATE1, 01Bh ; EFGH + movdqa MSGTMP4, STATE0 + palignr STATE0, STATE1, 8 ; ABEF + pblendw STATE1, MSGTMP4, 0F0h ; CDGH + + ;; copy byte_flip_mask to stack + mov ecx, 000010203h ;; DB 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12 + mov dword [mask_save], ecx + mov ecx, 004050607h + mov dword [mask_save+1*sizeof(dword)], ecx + mov ecx, 008090a0bh + mov dword [mask_save+2*sizeof(dword)], ecx + mov ecx, 00c0d0e0fh + mov dword [mask_save+3*sizeof(dword)], ecx + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +.sha256_block_loop: + movdqa oword [abef_save], STATE0 ; save for addition after rounds + movdqa oword [cdgh_save], STATE1 + + ;; rounds 0-3 + movdqu MSG, oword [MSG_PTR + 0*sizeof(oword)] + pshufb MSG, [mask_save] + movdqa MSGTMP0, MSG + paddd MSG, oword [K256_PTR + 0*sizeof(oword)] + sha256rnds2 STATE1, STATE0 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + + ;; rounds 4-7 + movdqu MSG, oword [MSG_PTR + 1*sizeof(oword)] + pshufb MSG, [mask_save] + movdqa MSGTMP1, MSG + paddd MSG, oword [K256_PTR + 1*sizeof(oword)] + sha256rnds2 STATE1, STATE0 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP0, MSGTMP1 + + ;; rounds 8-11 + movdqu MSG, oword [MSG_PTR + 2*sizeof(oword)] + pshufb MSG, [mask_save] + movdqa MSGTMP2, MSG + paddd MSG, oword [K256_PTR + 2*sizeof(oword)] + sha256rnds2 STATE1, STATE0 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP1, MSGTMP2 + + ;; rounds 12-15 + movdqu MSG, oword [MSG_PTR + 3*sizeof(oword)] + pshufb MSG, [mask_save] + movdqa MSGTMP3, MSG + paddd MSG, oword [K256_PTR + 3*sizeof(oword)] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP3 + palignr MSGTMP4, MSGTMP2, 4 + paddd MSGTMP0, MSGTMP4 + sha256msg2 MSGTMP0, MSGTMP3 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP2, MSGTMP3 + ;; rounds 16-19 + movdqa MSG, MSGTMP0 + paddd MSG, oword [K256_PTR + 4*sizeof(oword)] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP0 + palignr MSGTMP4, MSGTMP3, 4 + paddd MSGTMP1, MSGTMP4 + sha256msg2 MSGTMP1, MSGTMP0 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP3, MSGTMP0 + + ;; rounds 20-23 + movdqa MSG, MSGTMP1 + paddd MSG, oword [K256_PTR + 5*sizeof(oword)] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP1 + palignr MSGTMP4, MSGTMP0, 4 + paddd MSGTMP2, MSGTMP4 + sha256msg2 MSGTMP2, MSGTMP1 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP0, MSGTMP1 + + ;; rounds 24-27 + movdqa MSG, MSGTMP2 + paddd MSG, oword [K256_PTR + 6*sizeof(oword)] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP2 + palignr MSGTMP4, MSGTMP1, 4 + paddd MSGTMP3, MSGTMP4 + sha256msg2 MSGTMP3, MSGTMP2 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP1, MSGTMP2 + + ;; rounds 28-31 + movdqa MSG, MSGTMP3 + paddd MSG, oword [K256_PTR + 7*sizeof(oword)] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP3 + palignr MSGTMP4, MSGTMP2, 4 + paddd MSGTMP0, MSGTMP4 + sha256msg2 MSGTMP0, MSGTMP3 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP2, MSGTMP3 + + ;; rounds 32-35 + movdqa MSG, MSGTMP0 + paddd MSG, oword [K256_PTR + 8*sizeof(oword)] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP0 + palignr MSGTMP4, MSGTMP3, 4 + paddd MSGTMP1, MSGTMP4 + sha256msg2 MSGTMP1, MSGTMP0 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP3, MSGTMP0 + + ;; rounds 36-39 + movdqa MSG, MSGTMP1 + paddd MSG, oword [K256_PTR + 9*sizeof(oword)] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP1 + palignr MSGTMP4, MSGTMP0, 4 + paddd MSGTMP2, MSGTMP4 + sha256msg2 MSGTMP2, MSGTMP1 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP0, MSGTMP1 + + ;; rounds 40-43 + movdqa MSG, MSGTMP2 + paddd MSG, oword [K256_PTR + 10*sizeof(oword)] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP2 + palignr MSGTMP4, MSGTMP1, 4 + paddd MSGTMP3, MSGTMP4 + sha256msg2 MSGTMP3, MSGTMP2 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP1, MSGTMP2 + + ;; rounds 44-47 + movdqa MSG, MSGTMP3 + paddd MSG, oword [K256_PTR + 11*sizeof(oword)] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP3 + palignr MSGTMP4, MSGTMP2, 4 + paddd MSGTMP0, MSGTMP4 + sha256msg2 MSGTMP0, MSGTMP3 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP2, MSGTMP3 + + ;; rounds 48-51 + movdqa MSG, MSGTMP0 + paddd MSG, oword [K256_PTR + 12*sizeof(oword)] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP0 + palignr MSGTMP4, MSGTMP3, 4 + paddd MSGTMP1, MSGTMP4 + sha256msg2 MSGTMP1, MSGTMP0 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP3, MSGTMP0 + + ;; rounds 52-55 + movdqa MSG, MSGTMP1 + paddd MSG, oword [K256_PTR + 13*sizeof(oword)] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP1 + palignr MSGTMP4, MSGTMP0, 4 + paddd MSGTMP2, MSGTMP4 + sha256msg2 MSGTMP2, MSGTMP1 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + + ;; rounds 56-59 + movdqa MSG, MSGTMP2 + paddd MSG, oword [K256_PTR + 14*sizeof(oword)] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP2 + palignr MSGTMP4, MSGTMP1, 4 + paddd MSGTMP3, MSGTMP4 + sha256msg2 MSGTMP3, MSGTMP2 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + + ;; rounds 60-63 + movdqa MSG, MSGTMP3 + paddd MSG, oword [K256_PTR + 15*sizeof(oword)] + sha256rnds2 STATE1, STATE0 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + + paddd STATE0, oword [abef_save] ; update previously saved hash + paddd STATE1, oword [cdgh_save] + + add MSG_PTR, MBS_SHA256 + sub MSG_LEN, MBS_SHA256 + jg .sha256_block_loop + + ; reorder hash + pshufd STATE0, STATE0, 01Bh ; FEBA + pshufd STATE1, STATE1, 0B1h ; DCHG + movdqa MSGTMP4, STATE0 + pblendw STATE0, STATE1, 0F0h ; DCBA + palignr STATE1, MSGTMP4, 8 ; HGFE + + ; and store it back + movdqu oword [HASH_PTR + 0*sizeof(oword)], STATE0 + movdqu oword [HASH_PTR + 1*sizeof(oword)], STATE1 + +.quit: + add esp, (frame_size+16) + REST_GPR + ret +ENDFUNC UpdateSHA256ni + +%else ;; no sha ni support in VxWorks - therefore we temporary use db + +IPPASM UpdateSHA256ni,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pDigest [ebp + ARG_1 + 0*sizeof(dword)] ; pointer to the in/out digest +%xdefine pMsg [ebp + ARG_1 + 1*sizeof(dword)] ; pointer to the inp message +%xdefine msgLen [ebp + ARG_1 + 2*sizeof(dword)] ; message length +%xdefine pTbl [ebp + ARG_1 + 3*sizeof(dword)] ; pointer to SHA256 table of constants + +%xdefine MBS_SHA256 (64) ; SHA-1 message block length (bytes) + +%xdefine HASH_PTR edi ; 1st arg +%xdefine MSG_PTR esi ; 2nd arg +%xdefine MSG_LEN edx ; 3rd arg +%xdefine K256_PTR ebx ; 4rd arg + +%xdefine MSG xmm0 +%xdefine STATE0 xmm1 +%xdefine STATE1 xmm2 +%xdefine MSGTMP0 xmm3 +%xdefine MSGTMP1 xmm4 +%xdefine MSGTMP2 xmm5 +%xdefine MSGTMP3 xmm6 +%xdefine MSGTMP4 xmm7 + +; +; stack frame +; +%xdefine mask_save eax +%xdefine abef_save eax+sizeof(oword) +%xdefine cdgh_save eax+sizeof(oword)*2 +%xdefine frame_size sizeof(oword)+sizeof(oword)+sizeof(oword) + + sub esp, (frame_size+16) + lea eax, [esp+16] + and eax, -16 + + mov MSG_LEN, msgLen ; message length + test MSG_LEN, MSG_LEN + jz .quit + + mov HASH_PTR, pDigest + mov MSG_PTR, pMsg + mov K256_PTR, pTbl + + ;; load input hash value, reorder these appropriately + movdqu STATE0, oword [HASH_PTR+0*sizeof(oword)] + movdqu STATE1, oword [HASH_PTR+1*sizeof(oword)] + + pshufd STATE0, STATE0, 0B1h ; CDAB + pshufd STATE1, STATE1, 01Bh ; EFGH + movdqa MSGTMP4, STATE0 + palignr STATE0, STATE1, 8 ; ABEF + pblendw STATE1, MSGTMP4, 0F0h ; CDGH + + ;; copy byte_flip_mask to stack + ;movdqa MSGTMP4, oword [PSHUFFLE_BYTE_FLIP_MASK] + LD_ADDR ecx, CODE_DATA + movdqa MSGTMP4, oword [ecx+(PSHUFFLE_BYTE_FLIP_MASK-CODE_DATA)] + movdqa oword [mask_save], MSGTMP4 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +.sha256_block_loop: + movdqa oword [abef_save], STATE0 ; save for addition after rounds + movdqa oword [cdgh_save], STATE1 + + ;; rounds 0-3 + movdqu MSG, oword [MSG_PTR + 0*sizeof(oword)] + pshufb MSG, oword [mask_save] + movdqa MSGTMP0, MSG + paddd MSG, oword [K256_PTR + 0*sizeof(oword)] + db 0FH,038H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + pshufd MSG, MSG, 0Eh + db 0FH,038H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + + ;; rounds 4-7 + movdqu MSG, oword [MSG_PTR + 1*sizeof(oword)] + pshufb MSG, oword [mask_save] + movdqa MSGTMP1, MSG + paddd MSG, oword [K256_PTR + 1*sizeof(oword)] + db 0FH,038H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + pshufd MSG, MSG, 0Eh + db 0FH,038H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,038H,0CCH,0DCH ;; sha256msg1 MSGTMP0, MSGTMP1 + + ;; rounds 8-11 + movdqu MSG, oword [MSG_PTR + 2*sizeof(oword)] + pshufb MSG, oword [mask_save] + movdqa MSGTMP2, MSG + paddd MSG, oword [K256_PTR + 2*sizeof(oword)] + db 0FH,038H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + pshufd MSG, MSG, 0Eh + db 0FH,038H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,038H,0CCH,0E5H ;; sha256msg1 MSGTMP1, MSGTMP2 + + ;; rounds 12-15 + movdqu MSG, oword [MSG_PTR + 3*sizeof(oword)] + pshufb MSG, oword [mask_save] + movdqa MSGTMP3, MSG + paddd MSG, oword [K256_PTR + 3*sizeof(oword)] + db 0FH,038H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP3 + palignr MSGTMP4, MSGTMP2, 4 + paddd MSGTMP0, MSGTMP4 + db 0FH,038H,0CDH,0DEH ;; sha256msg2 MSGTMP0, MSGTMP3 + pshufd MSG, MSG, 0Eh + db 0FH,038H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,038H,0CCH,0EEH ;; sha256msg1 MSGTMP2, MSGTMP3 + ;; rounds 16-19 + movdqa MSG, MSGTMP0 + paddd MSG, oword [K256_PTR + 4*sizeof(oword)] + db 0FH,038H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP0 + palignr MSGTMP4, MSGTMP3, 4 + paddd MSGTMP1, MSGTMP4 + db 0FH,038H,0CDH,0E3H ;; sha256msg2 MSGTMP1, MSGTMP0 + pshufd MSG, MSG, 0Eh + db 0FH,038H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,038H,0CCH,0F3H ;; sha256msg1 MSGTMP3, MSGTMP0 + + ;; rounds 20-23 + movdqa MSG, MSGTMP1 + paddd MSG, oword [K256_PTR + 5*sizeof(oword)] + db 0FH,038H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP1 + palignr MSGTMP4, MSGTMP0, 4 + paddd MSGTMP2, MSGTMP4 + db 0FH,038H,0CDH,0ECH ;; sha256msg2 MSGTMP2, MSGTMP1 + pshufd MSG, MSG, 0Eh + db 0FH,038H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,038H,0CCH,0DCH ;; sha256msg1 MSGTMP0, MSGTMP1 + + ;; rounds 24-27 + movdqa MSG, MSGTMP2 + paddd MSG, oword [K256_PTR + 6*sizeof(oword)] + db 0FH,038H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP2 + palignr MSGTMP4, MSGTMP1, 4 + paddd MSGTMP3, MSGTMP4 + db 0FH,038H,0CDH,0F5H ;; sha256msg2 MSGTMP3, MSGTMP2 + pshufd MSG, MSG, 0Eh + db 0FH,038H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,038H,0CCH,0E5H ;; sha256msg1 MSGTMP1, MSGTMP2 + + ;; rounds 28-31 + movdqa MSG, MSGTMP3 + paddd MSG, oword [K256_PTR + 7*sizeof(oword)] + db 0FH,038H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP3 + palignr MSGTMP4, MSGTMP2, 4 + paddd MSGTMP0, MSGTMP4 + db 0FH,038H,0CDH,0DEH ;; sha256msg2 MSGTMP0, MSGTMP3 + pshufd MSG, MSG, 0Eh + db 0FH,038H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,038H,0CCH,0EEH ;; sha256msg1 MSGTMP2, MSGTMP3 + + ;; rounds 32-35 + movdqa MSG, MSGTMP0 + paddd MSG, oword [K256_PTR + 8*sizeof(oword)] + db 0FH,038H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP0 + palignr MSGTMP4, MSGTMP3, 4 + paddd MSGTMP1, MSGTMP4 + db 0FH,038H,0CDH,0E3H ;; sha256msg2 MSGTMP1, MSGTMP0 + pshufd MSG, MSG, 0Eh + db 0FH,038H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,038H,0CCH,0F3H ;; sha256msg1 MSGTMP3, MSGTMP0 + + ;; rounds 36-39 + movdqa MSG, MSGTMP1 + paddd MSG, oword [K256_PTR + 9*sizeof(oword)] + db 0FH,038H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP1 + palignr MSGTMP4, MSGTMP0, 4 + paddd MSGTMP2, MSGTMP4 + db 0FH,038H,0CDH,0ECH ;; sha256msg2 MSGTMP2, MSGTMP1 + pshufd MSG, MSG, 0Eh + db 0FH,038H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,038H,0CCH,0DCH ;; sha256msg1 MSGTMP0, MSGTMP1 + + ;; rounds 40-43 + movdqa MSG, MSGTMP2 + paddd MSG, oword [K256_PTR + 10*sizeof(oword)] + db 0FH,038H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP2 + palignr MSGTMP4, MSGTMP1, 4 + paddd MSGTMP3, MSGTMP4 + db 0FH,038H,0CDH,0F5H ;; sha256msg2 MSGTMP3, MSGTMP2 + pshufd MSG, MSG, 0Eh + db 0FH,038H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,038H,0CCH,0E5H ;; sha256msg1 MSGTMP1, MSGTMP2 + + ;; rounds 44-47 + movdqa MSG, MSGTMP3 + paddd MSG, oword [K256_PTR + 11*sizeof(oword)] + db 0FH,038H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP3 + palignr MSGTMP4, MSGTMP2, 4 + paddd MSGTMP0, MSGTMP4 + db 0FH,038H,0CDH,0DEH ;; sha256msg2 MSGTMP0, MSGTMP3 + pshufd MSG, MSG, 0Eh + db 0FH,038H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,038H,0CCH,0EEH ;; sha256msg1 MSGTMP2, MSGTMP3 + + ;; rounds 48-51 + movdqa MSG, MSGTMP0 + paddd MSG, oword [K256_PTR + 12*sizeof(oword)] + db 0FH,038H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP0 + palignr MSGTMP4, MSGTMP3, 4 + paddd MSGTMP1, MSGTMP4 + db 0FH,038H,0CDH,0E3H ;; sha256msg2 MSGTMP1, MSGTMP0 + pshufd MSG, MSG, 0Eh + db 0FH,038H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,038H,0CCH,0F3H ;; sha256msg1 MSGTMP3, MSGTMP0 + + ;; rounds 52-55 + movdqa MSG, MSGTMP1 + paddd MSG, oword [K256_PTR + 13*sizeof(oword)] + db 0FH,038H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP1 + palignr MSGTMP4, MSGTMP0, 4 + paddd MSGTMP2, MSGTMP4 + db 0FH,038H,0CDH,0ECH ;; sha256msg2 MSGTMP2, MSGTMP1 + pshufd MSG, MSG, 0Eh + db 0FH,038H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + + ;; rounds 56-59 + movdqa MSG, MSGTMP2 + paddd MSG, oword [K256_PTR + 14*sizeof(oword)] + db 0FH,038H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP2 + palignr MSGTMP4, MSGTMP1, 4 + paddd MSGTMP3, MSGTMP4 + db 0FH,038H,0CDH,0F5H ;; sha256msg2 MSGTMP3, MSGTMP2 + pshufd MSG, MSG, 0Eh + db 0FH,038H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + + ;; rounds 60-63 + movdqa MSG, MSGTMP3 + paddd MSG, oword [K256_PTR + 15*sizeof(oword)] + db 0FH,038H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + pshufd MSG, MSG, 0Eh + db 0FH,038H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + + paddd STATE0, oword [abef_save] ; update previously saved hash + paddd STATE1, oword [cdgh_save] + + add MSG_PTR, MBS_SHA256 + sub MSG_LEN, MBS_SHA256 + jg .sha256_block_loop + + ; reorder hash + pshufd STATE0, STATE0, 01Bh ; FEBA + pshufd STATE1, STATE1, 0B1h ; DCHG + movdqa MSGTMP4, STATE0 + pblendw STATE0, STATE1, 0F0h ; DCBA + palignr STATE1, MSGTMP4, 8 ; HGFE + + ; and store it back + movdqu oword [HASH_PTR + 0*sizeof(oword)], STATE0 + movdqu oword [HASH_PTR + 1*sizeof(oword)], STATE1 + +.quit: + add esp, (frame_size+16) + REST_GPR + ret +ENDFUNC UpdateSHA256ni + +%endif ;; VxWorks + +%endif ;; _FEATURE_ON_ / _FEATURE_TICKTOCK_ +%endif ;; _ENABLE_ALG_SHA256_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha256v8as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha256v8as.asm new file mode 100644 index 0000000..cef8965 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha256v8as.asm @@ -0,0 +1,455 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SHA256 +; +; Content: +; UpdateSHA256 +; +; + + + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SHA256_) +%if (_SHA_NI_ENABLING_ == _FEATURE_OFF_) || (_SHA_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP >= _IPP_V8) && (_IPP < _IPP_G9) + +%xdefine XMM_SHUFB_BSWAP xmm6 +%xdefine W0 xmm0 +%xdefine W4 xmm1 +%xdefine W8 xmm2 +%xdefine W12 xmm3 +%xdefine SIG1 xmm4 +%xdefine SIG0 xmm5 +%xdefine X xmm6 +%xdefine W xmm7 + +%xdefine regTbl ebx + + +;; we are considering x, y, z are polynomials over GF(2) +;; & - multiplication +;; ^ - additive +;; operations + +;; +;; Chj(x,y,z) = (x&y) ^ (~x & z) +;; = (x&y) ^ ((1^x) &z) +;; = (x&y) ^ (z ^ x&z) +;; = x&y ^ z ^ x&z +;; = x&(y^z) ^z +;; +%macro Chj 4.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%Y %3 + %xdefine %%Z %4 + + mov %%F, %%Y + xor %%F, %%Z + and %%F, %%X + xor %%F, %%Z +%endmacro + + +;; +;; Maj(x,y,z) = (x&y) ^ (x&z) ^ (y&z) +;; = (x&y) ^ (x&z) ^ (y&z) ^ (z&z) ^z // note: ((z&z) ^z) = 0 +;; = x&(y^z) ^ z&(y^z) ^z +;; = (x^z)&(y^z) ^z +;; +%macro Maj 4.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%Y %3 + %xdefine %%Z %4 + + mov %%F, %%X + xor %%F, %%Z + xor %%Z, %%Y + and %%F, %%Z + xor %%Z, %%Y + xor %%F, %%Z +%endmacro + + +%macro ROTR 2.nolist + %xdefine %%X %1 + %xdefine %%n %2 + + ;;shrd X,X, n + ror %%X, %%n +%endmacro + + +;; +;; Summ0(x) = ROR(x,2) ^ ROR(x,13) ^ ROR(x,22) +;; +%macro Summ0 3.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%T %3 + + mov %%F, %%X + ROTR %%F, 2 + mov %%T, %%X + ROTR %%T, 13 + xor %%F, %%T + ROTR %%T, (22-13) + xor %%F, %%T +%endmacro + + +;; +;; Summ1(x) = ROR(x,6) ^ ROR(x,11) ^ ROR(x,25) +;; +%macro Summ1 3.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%T %3 + + mov %%F, %%X + ROTR %%F, 6 + mov %%T, %%X + ROTR %%T, 11 + xor %%F, %%T + ROTR %%T, (25-11) + xor %%F, %%T +%endmacro + + +;; +;; regular round (i): +;; +;; T1 = h + Sigma1(e) + Ch(e,f,g) + K[i] + W[i] +;; T2 = Sigma0(a) + Maj(a,b,c) +;; h = g +;; g = f +;; f = e +;; e = d + T1 +;; d = c +;; c = b +;; b = a +;; a = T1+T2 +;; +;; or +;; +;; h += Sigma1(e) + Ch(e,f,g) + K[i] + W[i] (==T1) +;; d += h +;; T2 = Sigma0(a) + Maj(a,b,c) +;; h += T2 +;; and following textual shift {a,b,c,d,e,f,g,h} => {h,a,b,c,d,e,f,g} +;; +%macro ROUND 6.nolist + %xdefine %%nr %1 + %xdefine %%hashBuff %2 + %xdefine %%wBuff %3 + %xdefine %%F1 %4 + %xdefine %%F2 %5 + %xdefine %%T1 %6 + ; %xdefine T2 %7 + + Summ1 %%F1, eax, %%T1 + Chj %%F2, eax,{[%%hashBuff+((vF-%%nr)&7)*sizeof(dword)]},{[%%hashBuff+((vG-%%nr)&7)*sizeof(dword)]} + mov eax, [%%hashBuff+((vH-%%nr)&7)*sizeof(dword)] + add eax, %%F1 + add eax, %%F2 + add eax, dword [%%wBuff+(%%nr&3)*sizeof(dword)] + + mov %%F1, dword [%%hashBuff+((vB-%%nr)&7)*sizeof(dword)] + mov %%T1, dword [%%hashBuff+((vC-%%nr)&7)*sizeof(dword)] + Maj %%F2, edx,%%F1, %%T1 + Summ0 %%F1, edx, %%T1 + lea edx, [%%F1+%%F2] + + add edx,eax ; T2+T1 + add eax,[%%hashBuff+((vD-%%nr)&7)*sizeof(dword)] ; T1+d + + mov [%%hashBuff+((vH-%%nr)&7)*sizeof(dword)],edx + mov [%%hashBuff+((vD-%%nr)&7)*sizeof(dword)],eax +%endmacro + + +;; +;; W[i] = Sigma1(W[i-2]) + W[i-7] + Sigma0(W[i-15]) + W[i-16], i=16,..,63 +;; +;;for next rounds 16,17,18 and 19: +;; W[0] <= W[16] = Sigma1(W[14]) + W[ 9] + Sigma0(W[1]) + W[0] +;; W[1] <= W[17] = Sigma1(W[15]) + W[10] + Sigma0(W[2]) + W[1] +;; W[2] <= W[18] = Sigma1(W[ 0]) + W[11] + Sigma0(W[3]) + W[1] +;; W[3] <= W[19] = Sigma1(W[ 1]) + W[12] + Sigma0(W[4]) + W[2] +;; +;; the process is repeated exactly because texual round of W[] +;; +;; Sigma1() and Sigma0() functions are defined as following: +;; Sigma1(X) = ROR(X,17)^ROR(X,19)^SHR(X,10) +;; Sigma0(X) = ROR(X, 7)^ROR(X,18)^SHR(X, 3) +;; +%macro UPDATE_W 8.nolist + %xdefine %%xS %1 + %xdefine %%xS0 %2 + %xdefine %%xS4 %3 + %xdefine %%xS8 %4 + %xdefine %%xS12 %5 + %xdefine %%SIGMA1 %6 + %xdefine %%SIGMA0 %7 + %xdefine %%X %8 + + pshufd %%SIGMA1, %%xS12, 11111010b ;; SIGMA1 = {W[15],W[15],W[14],W[14]} + movdqa %%X, %%SIGMA1 + psrld %%SIGMA1, 10 + psrlq %%X, 17 + pxor %%SIGMA1, %%X + psrlq %%X, (19-17) + pxor %%SIGMA1, %%X + + pshufd %%SIGMA0, %%xS0, 10100101b ;; SIGMA0 = {W[2],W[2],W[1],W[1]} + movdqa %%X, %%SIGMA0 + psrld %%SIGMA0, 3 + psrlq %%X, 7 + pxor %%SIGMA0, %%X + psrlq %%X, (18-7) + pxor %%SIGMA0, %%X + + pshufd %%xS, %%xS0, 01010000b ;; {W[ 1],W[ 1],W[ 0],W[ 0]} + pshufd %%X, %%xS8, 10100101b ;; {W[10],W[10],W[ 9],W[ 9]} + paddd %%xS, %%SIGMA1 + paddd %%xS, %%SIGMA0 + paddd %%xS, %%X + + + pshufd %%SIGMA1, %%xS, 10100000b ;; SIGMA1 = {W[1],W[1],W[0],W[0]} + movdqa %%X, %%SIGMA1 + psrld %%SIGMA1, 10 + psrlq %%X, 17 + pxor %%SIGMA1, %%X + psrlq %%X, (19-17) + pxor %%SIGMA1, %%X + + movdqa %%SIGMA0, %%xS4 ;; SIGMA0 = {W[4],W[4],W[3],W[3]} + palignr %%SIGMA0, %%xS0, (3*sizeof(dword)) + pshufd %%SIGMA0, %%SIGMA0, 01010000b + movdqa %%X, %%SIGMA0 + psrld %%SIGMA0, 3 + psrlq %%X, 7 + pxor %%SIGMA0, %%X + psrlq %%X, (18-7) + pxor %%SIGMA0, %%X + + movdqa %%X, %%xS12 + palignr %%X, %%xS8, (3*sizeof(dword)) ;; {W[14],W[13],W[12],W[11]} + pshufd %%xS0, %%xS0, 11111010b ;; {W[ 3],W[ 3],W[ 2],W[ 2]} + pshufd %%X, %%X, 01010000b ;; {W[12],W[12],W[11],W[11]} + paddd %%xS0, %%SIGMA1 + paddd %%xS0, %%SIGMA0 + paddd %%xS0, %%X + + pshufd %%xS, %%xS, 10001000b ;; {W[1],W[0],W[1],W[0]} + pshufd %%xS0, %%xS0, 10001000b ;; {W[3],W[2],W[3],W[2]} + palignr %%xS0, %%xS, (2*sizeof(dword)) ;; {W[3],W[2],W[1],W[0]} + movdqa %%xS, %%xS0 +%endmacro + + +segment .text align=IPP_ALIGN_FACTOR + + +align IPP_ALIGN_FACTOR +SWP_BYTE: +pByteSwp DB 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; SHA256(Ipp32u digest[], Ipp8u dataBlock[], int dataLen, Ipp32u K_256[]) +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM UpdateSHA256,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pHash [ebp + ARG_1 + 0*sizeof(dword)] ; pointer to hash +%xdefine pData [ebp + ARG_1 + 1*sizeof(dword)] ; pointer to data block +%xdefine dataLen [ebp + ARG_1 + 2*sizeof(dword)] ; data length +%xdefine pTbl [ebp + ARG_1 + 3*sizeof(dword)] ; pointer to the SHA256 const table + +%xdefine MBS_SHA256 (64) + +%assign hSize sizeof(dword)*8 ; size of hash +%assign wSize sizeof(oword) ; W values queue (dwords) +%assign cntSize sizeof(dword) ; local counter + +%assign hashOff 0 ; hash address +%assign wOff hashOff+hSize ; W values offset +%assign cntOff wOff+wSize + +%assign stackSize (hSize+wSize+cntSize) ; stack size + + sub esp, stackSize + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +.sha256_block_loop: + + mov eax, pHash ; pointer to the hash + movdqu W0, oword [eax] ; load initial hash value + movdqu W4, oword [eax+sizeof(oword)] + movdqu oword [esp+hashOff], W0 + movdqu oword [esp+hashOff+sizeof(oword)*1], W4 + + ;movdqa XMM_SHUFB_BSWAP, oword pByteSwp ; load shuffle mask + LD_ADDR eax, SWP_BYTE + movdqa XMM_SHUFB_BSWAP, oword [eax+(pByteSwp-SWP_BYTE)] + mov eax, pData ; pointer to the data block + mov regTbl, pTbl ; pointer to SHA256 table (points K_256[] constants) + + movdqu W0, oword [eax] ; load buffer content + movdqu W4, oword [eax+sizeof(oword)] + movdqu W8, oword [eax+sizeof(oword)*2] + movdqu W12,oword [eax+sizeof(oword)*3] + +%assign vA 0 +%assign vB 1 +%assign vC 2 +%assign vD 3 +%assign vE 4 +%assign vF 5 +%assign vG 6 +%assign vH 7 + + mov eax, [esp+hashOff+vE*sizeof(dword)] + mov edx, [esp+hashOff+vA*sizeof(dword)] + + ;; perform 0-3 regular rounds + pshufb W0, XMM_SHUFB_BSWAP ; swap input + movdqa W, W0 ; T = W[0-3] + paddd W, oword [regTbl+sizeof(oword)*0] ; T += K_SHA256[0-3] + movdqu oword [esp+wOff], W + ROUND 0, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 1, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 2, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 3, {esp+hashOff},{esp+wOff}, esi,edi,ecx + + ;; perform next 4-7 regular rounds + pshufb W4, XMM_SHUFB_BSWAP ; swap input + movdqa W, W4 ; T = W[4-7] + paddd W, oword [regTbl+sizeof(oword)*1] ; T += K_SHA256[4-7] + movdqu oword [esp+wOff], W + ROUND 4, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 5, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 6, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 7, {esp+hashOff},{esp+wOff}, esi,edi,ecx + + ;; perform next 8-11 regular rounds + pshufb W8, XMM_SHUFB_BSWAP ; swap input + movdqa W, W8 ; T = W[8-11] + paddd W, oword [regTbl+sizeof(oword)*2] ; T += K_SHA256[8-11] + movdqu oword [esp+wOff], W + ROUND 8, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 9, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 10, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 11, {esp+hashOff},{esp+wOff}, esi,edi,ecx + + ;; perform next 12-15 regular rounds + pshufb W12, XMM_SHUFB_BSWAP ; swap input + movdqa W, W12 ; T = W[12-15] + paddd W, oword [regTbl+sizeof(oword)*3] ; T += K_SHA256[12-15] + movdqu oword [esp+wOff], W + ROUND 12, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 13, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 14, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 15, {esp+hashOff},{esp+wOff}, esi,edi,ecx + + mov dword [esp+cntOff], (64-16) ; init counter +.loop_16_63: + add regTbl, sizeof(oword)*4 ; update SHA_256 pointer + + UPDATE_W W, W0, W4, W8, W12, SIG1,SIG0,X ; round: 16*i - 16*i+3 + paddd W, oword [regTbl+sizeof(oword)*0] ; T += K_SHA256[16-19] + movdqu oword [esp+wOff], W + ROUND 16, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 17, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 18, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 19, {esp+hashOff},{esp+wOff}, esi,edi,ecx + + UPDATE_W W, W4, W8, W12,W0, SIG1,SIG0,X ; round: 20*i 20*i+3 + paddd W, oword [regTbl+sizeof(oword)*1] ; T += K_SHA256[20-23] + movdqu oword [esp+wOff], W + ROUND 20, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 21, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 22, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 23, {esp+hashOff},{esp+wOff}, esi,edi,ecx + + UPDATE_W W, W8, W12,W0, W4, SIG1,SIG0,X ; round: 24*i - 24*i+3 + paddd W, oword [regTbl+sizeof(oword)*2] ; T += K_SHA256[24-27] + movdqu oword [esp+wOff], W + ROUND 24, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 25, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 26, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 27, {esp+hashOff},{esp+wOff}, esi,edi,ecx + + UPDATE_W W, W12,W0, W4, W8, SIG1,SIG0,X ; round: 28*i - 28*i+3 + paddd W, oword [regTbl+sizeof(oword)*3] ; T += K_SHA256[28-31] + movdqu oword [esp+wOff], W + ROUND 28, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 29, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 30, {esp+hashOff},{esp+wOff}, esi,edi,ecx + ROUND 31, {esp+hashOff},{esp+wOff}, esi,edi,ecx + + sub dword [esp+cntOff], 16 + jg .loop_16_63 + + mov eax, pHash ; pointer to the hash + movdqu W0, oword [esp+hashOff] + movdqu W4, oword [esp+hashOff+sizeof(oword)*1] + + ; update hash + movdqu W, oword [eax] + paddd W, W0 + movdqu oword [eax], W + movdqu W, oword [eax+sizeof(oword)] + paddd W, W4 + movdqu oword [eax+sizeof(oword)], W + + add dword pData, MBS_SHA256 + sub dword dataLen, MBS_SHA256 + jg .sha256_block_loop + + add esp, stackSize + REST_GPR + ret +ENDFUNC UpdateSHA256 + +%endif ;; (_IPP >= _IPP_V8) && (_IPP < _IPP_G9) +%endif ;; _FEATURE_OFF_ / _FEATURE_TICKTOCK_ +%endif ;; _ENABLE_ALG_SHA256_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha256w7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha256w7as.asm new file mode 100644 index 0000000..b8c76a7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha256w7as.asm @@ -0,0 +1,419 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SHA256 +; +; Content: +; UpdateSHA256 +; +; + + + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SHA256_) +%if (_SHA_NI_ENABLING_ == _FEATURE_OFF_) || (_SHA_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP >= _IPP_M5) && (_IPP < _IPP_V8) + +;; +;; SIG0(x) = ROR32(x, 7) ^ ROR32(x,18) ^ LSR(x, 3) +;; SIG1(x) = ROR32(x,17) ^ ROR32(x,19) ^ LSR(x,10) +;; W[i] = SIG1(W[i-2]) + W[i-7] + SIG0(W[i-15]) + W[i-16], i=16,..,63 +;; +%macro UPDATE 2.nolist + %xdefine %%nr %1 + %xdefine %%wBuff %2 + + mov ebx,[%%wBuff+((%%nr-2) &0Fh)*sizeof(dword)] ; W[i- 2] + ror ebx, 10 + mov ecx,[%%wBuff+((%%nr-15)&0Fh)*sizeof(dword)] ; W[i-15] + ror ecx, 3 + mov esi,[%%wBuff+((%%nr-7) &0Fh)*sizeof(dword)] ; +W[i- 7] + add esi,[%%wBuff+((%%nr-16)&0Fh)*sizeof(dword)] ; +W[i-16] + + mov edi,003FFFFFh ; SIG1() + and edi,ebx + ror ebx,(17-10) + xor edi,ebx + ror ebx,(19-17) + xor edi,ebx + + add edi, esi ; SIG0() +W[i-7] +W[i-16] + + mov esi,1FFFFFFFh ; SIG0() + and esi,ecx + ror ecx,(7-3) + xor esi,ecx + ror ecx,(18-7) + xor esi,ecx + + add edi,esi ; SIG0() +W[i-7] +W[i-16] + SIG1() + mov [%%wBuff+(%%nr&0Fh)*sizeof(dword)],edi +%endmacro + + +; +; SUM1(x) = ROR32(x, 6) ^ ROR32(x,11) ^ ROR32(x,25) +; SUM0(x) = ROR32(x, 2) ^ ROR32(x,13) ^ ROR32(x,22) +; +; CH(x,y,x) = (x & y) ^ (~x & z) +; MAJ(x,y,z) = (x & y) ^ (x & z) ^ (y & z) = (x&y)^((x^y)&z) +; +; T1 = SUM1(E) +CH(E,F,G) +K[i] +W[i] + H +; T2 = SUM0(A) +MAJ(A,B,C) +; +; D += T1 +; H = T1+T2 +; +; eax = E +; edx = A +; +%macro SHA256_STEP 4.nolist + %xdefine %%nr %1 + %xdefine %%hashBuff %2 + %xdefine %%wBuff %3 + %xdefine %%immCnt %4 + + mov esi,[%%hashBuff+((vF-%%nr)&7)*sizeof(dword)] ; init CH() + mov ecx,eax + mov edi,[%%hashBuff+((vG-%%nr)&7)*sizeof(dword)] ; init CH() + mov ebx,eax ; SUM1() + ror eax,6 ; SUM1() + not ecx ; CH() + and esi,ebx ; CH() + ror ebx,11 ; SUM1() + and edi,ecx ; CH() + xor eax,ebx ; SUM1() + ror ebx,(25-11) ; SUM1() + xor esi,edi ; CH() + xor eax,ebx ; SUM1() + add eax,esi ; T1 = SUM1() +CH() + add eax,[%%hashBuff+((vH-%%nr)&7)*sizeof(dword)] ; T1 += h +;; add eax,[wBuff+(nr&0Fh)*sizeof(dword)] ; T1 += w[] + add eax,[%%wBuff] ; T1 += w[] + add eax, %%immCnt ; T1 += K[] + + mov esi,[%%hashBuff+((vB-%%nr)&7)*sizeof(dword)] ; init MAJ() + mov ecx,edx + mov edi,[%%hashBuff+((vC-%%nr)&7)*sizeof(dword)] ; init MAJ() + mov ebx,edx ; SUM0() + and ecx,esi ; MAJ() + ror edx,2 ; SUM0() + xor esi,ebx ; MAJ() + ror ebx,13 ; SUM0() + and esi,edi ; MAJ() + xor edx,ebx ; SUM0() + xor esi,ecx ; MAJ() + ror ebx,(22-13) ; SUM0() + xor edx,ebx ; SUM0() + add edx,esi ; T2 = SUM0()+MAJ() + + add edx,eax ; T2+T1 + add eax,[%%hashBuff+((vD-%%nr)&7)*sizeof(dword)] ; T1+d + + mov [%%hashBuff+((vH-%%nr)&7)*sizeof(dword)],edx + mov [%%hashBuff+((vD-%%nr)&7)*sizeof(dword)],eax +%endmacro + + + +segment .text align=IPP_ALIGN_FACTOR + + + +;***************************************************************************************** +;* Purpose: Update internal digest according to message block +;* +;* void UpdateSHA256(DigestSHA256 digest, const Ipp32u* mblk, int mlen, const void* pParam) +;* +;***************************************************************************************** + +;; +;; Lib = W7 +;; +;; Caller = ippsSHA256Update +;; Caller = ippsSHA256Final +;; Caller = ippsSHA256MessageDigest +;; +;; Caller = ippsSHA224Update +;; Caller = ippsSHA224Final +;; Caller = ippsSHA224MessageDigest +;; +;; Caller = ippsHMACSHA256Update +;; Caller = ippsHMACSHA256Final +;; Caller = ippsHMACSHA256MessageDigest +;; +;; Caller = ippsHMACSHA224Update +;; Caller = ippsHMACSHA224Final +;; Caller = ippsHMACSHA224MessageDigest +;; + +align IPP_ALIGN_FACTOR +IPPASM UpdateSHA256,PUBLIC + USES_GPR esi,edi,ebx,ebp + +%xdefine pHash [esp + ARG_1 + 0*sizeof(dword)] ; pointer to the input/output hash +%xdefine pSrc [esp + ARG_1 + 1*sizeof(dword)] ; input message +%xdefine srcL [esp + ARG_1 + 2*sizeof(dword)] ; message length +%xdefine pParm [esp + ARG_1 + 3*sizeof(dword)] ; dummy parameter + +%xdefine MBS_SHA256 (64) ; SHA256 block data size + +%assign dSize 8 ; size of digest (dwords) +%assign wSize 16 ; W values queue (dwords) + +%assign hashOffset 0 ; hash address +%assign msgOffset hashOffset+sizeof(dword) ; message address offset +%assign lenOffset msgOffset+sizeof(dword) ; message length offset +%assign dOffset lenOffset+sizeof(dword) ; hash buffer offset +%assign wOffset dOffset+dSize*sizeof(dword) ; W values offset + +%assign stackSize 3*sizeof(dword) + (dSize+wSize)*sizeof(dword) + +%assign vA 0 +%assign vB 1 +%assign vC 2 +%assign vD 3 +%assign vE 4 +%assign vF 5 +%assign vG 6 +%assign vH 7 + mov eax, pParm ; dummy + mov edi, pHash ; hash address + mov esi, pSrc ; source data address + mov eax, srcL ; source data length + + sub esp, stackSize ; allocate local buffers + mov [esp+hashOffset], edi; save hash address + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +.sha256_block_loop: + mov [esp+msgOffset], esi ; save message address + mov [esp+lenOffset], eax ; save message length + +;; +;; copy input digest +;; + mov eax, [edi+0*sizeof(dword)] + mov ebx, [edi+1*sizeof(dword)] + mov ecx, [edi+2*sizeof(dword)] + mov edx, [edi+3*sizeof(dword)] + mov [esp+dOffset+vA*sizeof(dword)], eax + mov [esp+dOffset+vB*sizeof(dword)], ebx + mov [esp+dOffset+vC*sizeof(dword)], ecx + mov [esp+dOffset+vD*sizeof(dword)], edx + mov eax, [edi+4*sizeof(dword)] + mov ebx, [edi+5*sizeof(dword)] + mov ecx, [edi+6*sizeof(dword)] + mov edx, [edi+7*sizeof(dword)] + mov [esp+dOffset+vE*sizeof(dword)], eax + mov [esp+dOffset+vF*sizeof(dword)], ebx + mov [esp+dOffset+vG*sizeof(dword)], ecx + mov [esp+dOffset+vH*sizeof(dword)], edx + +;; +;; initialize the first 16 qwords of W array +;; - remember about endian +;; + xor ecx, ecx +.loop1: + mov eax, [esi+ecx*sizeof(dword)] + bswap eax + mov ebx, [esi+ecx*sizeof(dword)+sizeof(dword)] + bswap ebx + + mov [esp+wOffset+ecx*sizeof(dword)], eax + mov [esp+wOffset+ecx*sizeof(dword)+sizeof(dword)], ebx + + add ecx, 2 + cmp ecx, 16 + jl .loop1 + +;; +;; perform 0-64 steps +;; + mov eax, [esp+dOffset+vE*sizeof(dword)] + mov edx, [esp+dOffset+vA*sizeof(dword)] + + + SHA256_STEP 0, {esp+dOffset}, {esp+wOffset+ 0*sizeof(dword)}, 0428A2F98h + UPDATE 16, {esp+wOffset} + SHA256_STEP 1, {esp+dOffset}, {esp+wOffset+ 1*sizeof(dword)}, 071374491h + UPDATE 17, {esp+wOffset} + SHA256_STEP 2, {esp+dOffset}, {esp+wOffset+ 2*sizeof(dword)}, 0B5C0FBCFh + UPDATE 18, {esp+wOffset} + SHA256_STEP 3, {esp+dOffset}, {esp+wOffset+ 3*sizeof(dword)}, 0E9B5DBA5h + UPDATE 19, {esp+wOffset} + SHA256_STEP 4, {esp+dOffset}, {esp+wOffset+ 4*sizeof(dword)}, 03956C25Bh + UPDATE 20, {esp+wOffset} + SHA256_STEP 5, {esp+dOffset}, {esp+wOffset+ 5*sizeof(dword)}, 059F111F1h + UPDATE 21, {esp+wOffset} + SHA256_STEP 6, {esp+dOffset}, {esp+wOffset+ 6*sizeof(dword)}, 0923F82A4h + UPDATE 22, {esp+wOffset} + SHA256_STEP 7, {esp+dOffset}, {esp+wOffset+ 7*sizeof(dword)}, 0AB1C5ED5h + UPDATE 23, {esp+wOffset} + SHA256_STEP 8, {esp+dOffset}, {esp+wOffset+ 8*sizeof(dword)}, 0D807AA98h + UPDATE 24, {esp+wOffset} + SHA256_STEP 9, {esp+dOffset}, {esp+wOffset+ 9*sizeof(dword)}, 012835B01h + UPDATE 25, {esp+wOffset} + SHA256_STEP 10,{esp+dOffset}, {esp+wOffset+10*sizeof(dword)}, 0243185BEh + UPDATE 26, {esp+wOffset} + SHA256_STEP 11, {esp+dOffset}, {esp+wOffset+11*sizeof(dword)}, 0550C7DC3h + UPDATE 27, {esp+wOffset} + SHA256_STEP 12, {esp+dOffset}, {esp+wOffset+12*sizeof(dword)}, 072BE5D74h + UPDATE 28, {esp+wOffset} + SHA256_STEP 13, {esp+dOffset}, {esp+wOffset+13*sizeof(dword)}, 080DEB1FEh + UPDATE 29, {esp+wOffset} + SHA256_STEP 14, {esp+dOffset}, {esp+wOffset+14*sizeof(dword)}, 09BDC06A7h + UPDATE 30, {esp+wOffset} + SHA256_STEP 15, {esp+dOffset}, {esp+wOffset+15*sizeof(dword)}, 0C19BF174h + UPDATE 31, {esp+wOffset} + + SHA256_STEP 16, {esp+dOffset}, {esp+wOffset+ 0*sizeof(dword)}, 0E49B69C1h + UPDATE 32, {esp+wOffset} + SHA256_STEP 17, {esp+dOffset}, {esp+wOffset+ 1*sizeof(dword)}, 0EFBE4786h + UPDATE 33, {esp+wOffset} + SHA256_STEP 18, {esp+dOffset}, {esp+wOffset+ 2*sizeof(dword)}, 00FC19DC6h + UPDATE 34, {esp+wOffset} + SHA256_STEP 19, {esp+dOffset}, {esp+wOffset+ 3*sizeof(dword)}, 0240CA1CCh + UPDATE 35, {esp+wOffset} + SHA256_STEP 20, {esp+dOffset}, {esp+wOffset+ 4*sizeof(dword)}, 02DE92C6Fh + UPDATE 36, {esp+wOffset} + SHA256_STEP 21, {esp+dOffset}, {esp+wOffset+ 5*sizeof(dword)}, 04A7484AAh + UPDATE 37, {esp+wOffset} + SHA256_STEP 22, {esp+dOffset}, {esp+wOffset+ 6*sizeof(dword)}, 05CB0A9DCh + UPDATE 38, {esp+wOffset} + SHA256_STEP 23, {esp+dOffset}, {esp+wOffset+ 7*sizeof(dword)}, 076F988DAh + UPDATE 39, {esp+wOffset} + SHA256_STEP 24, {esp+dOffset}, {esp+wOffset+ 8*sizeof(dword)}, 0983E5152h + UPDATE 40, {esp+wOffset} + SHA256_STEP 25, {esp+dOffset}, {esp+wOffset+ 9*sizeof(dword)}, 0A831C66Dh + UPDATE 41, {esp+wOffset} + SHA256_STEP 26, {esp+dOffset}, {esp+wOffset+10*sizeof(dword)}, 0B00327C8h + UPDATE 42, {esp+wOffset} + SHA256_STEP 27, {esp+dOffset}, {esp+wOffset+11*sizeof(dword)}, 0BF597FC7h + UPDATE 43, {esp+wOffset} + SHA256_STEP 28, {esp+dOffset}, {esp+wOffset+12*sizeof(dword)}, 0C6E00BF3h + UPDATE 44, {esp+wOffset} + SHA256_STEP 29, {esp+dOffset}, {esp+wOffset+13*sizeof(dword)}, 0D5A79147h + UPDATE 45, {esp+wOffset} + SHA256_STEP 30, {esp+dOffset}, {esp+wOffset+14*sizeof(dword)}, 006CA6351h + UPDATE 46, {esp+wOffset} + SHA256_STEP 31, {esp+dOffset}, {esp+wOffset+15*sizeof(dword)}, 014292967h + UPDATE 47, {esp+wOffset} + + SHA256_STEP 32, {esp+dOffset}, {esp+wOffset+ 0*sizeof(dword)}, 027B70A85h + UPDATE 48, {esp+wOffset} + SHA256_STEP 33, {esp+dOffset}, {esp+wOffset+ 1*sizeof(dword)}, 02E1B2138h + UPDATE 49, {esp+wOffset} + SHA256_STEP 34, {esp+dOffset}, {esp+wOffset+ 2*sizeof(dword)}, 04D2C6DFCh + UPDATE 50, {esp+wOffset} + SHA256_STEP 35, {esp+dOffset}, {esp+wOffset+ 3*sizeof(dword)}, 053380D13h + UPDATE 51, {esp+wOffset} + SHA256_STEP 36, {esp+dOffset}, {esp+wOffset+ 4*sizeof(dword)}, 0650A7354h + UPDATE 52, {esp+wOffset} + SHA256_STEP 37, {esp+dOffset}, {esp+wOffset+ 5*sizeof(dword)}, 0766A0ABBh + UPDATE 53, {esp+wOffset} + SHA256_STEP 38, {esp+dOffset}, {esp+wOffset+ 6*sizeof(dword)}, 081C2C92Eh + UPDATE 54, {esp+wOffset} + SHA256_STEP 39, {esp+dOffset}, {esp+wOffset+ 7*sizeof(dword)}, 092722C85h + UPDATE 55, {esp+wOffset} + SHA256_STEP 40, {esp+dOffset}, {esp+wOffset+ 8*sizeof(dword)}, 0A2BFE8A1h + UPDATE 56, {esp+wOffset} + SHA256_STEP 41, {esp+dOffset}, {esp+wOffset+ 9*sizeof(dword)}, 0A81A664Bh + UPDATE 57, {esp+wOffset} + SHA256_STEP 42, {esp+dOffset}, {esp+wOffset+10*sizeof(dword)}, 0C24B8B70h + UPDATE 58, {esp+wOffset} + SHA256_STEP 43, {esp+dOffset}, {esp+wOffset+11*sizeof(dword)}, 0C76C51A3h + UPDATE 59, {esp+wOffset} + SHA256_STEP 44, {esp+dOffset}, {esp+wOffset+12*sizeof(dword)}, 0D192E819h + UPDATE 60, {esp+wOffset} + SHA256_STEP 45, {esp+dOffset}, {esp+wOffset+13*sizeof(dword)}, 0D6990624h + UPDATE 61, {esp+wOffset} + SHA256_STEP 46, {esp+dOffset}, {esp+wOffset+14*sizeof(dword)}, 0F40E3585h + UPDATE 62, {esp+wOffset} + SHA256_STEP 47, {esp+dOffset}, {esp+wOffset+15*sizeof(dword)}, 0106AA070h + UPDATE 63, {esp+wOffset} + + SHA256_STEP 48, {esp+dOffset}, {esp+wOffset+ 0*sizeof(dword)}, 019A4C116h + SHA256_STEP 49, {esp+dOffset}, {esp+wOffset+ 1*sizeof(dword)}, 01E376C08h + SHA256_STEP 50, {esp+dOffset}, {esp+wOffset+ 2*sizeof(dword)}, 02748774Ch + SHA256_STEP 51, {esp+dOffset}, {esp+wOffset+ 3*sizeof(dword)}, 034B0BCB5h + SHA256_STEP 52, {esp+dOffset}, {esp+wOffset+ 4*sizeof(dword)}, 0391C0CB3h + SHA256_STEP 53, {esp+dOffset}, {esp+wOffset+ 5*sizeof(dword)}, 04ED8AA4Ah + SHA256_STEP 54, {esp+dOffset}, {esp+wOffset+ 6*sizeof(dword)}, 05B9CCA4Fh + SHA256_STEP 55, {esp+dOffset}, {esp+wOffset+ 7*sizeof(dword)}, 0682E6FF3h + SHA256_STEP 56, {esp+dOffset}, {esp+wOffset+ 8*sizeof(dword)}, 0748F82EEh + SHA256_STEP 57, {esp+dOffset}, {esp+wOffset+ 9*sizeof(dword)}, 078A5636Fh + SHA256_STEP 58, {esp+dOffset}, {esp+wOffset+10*sizeof(dword)}, 084C87814h + SHA256_STEP 59, {esp+dOffset}, {esp+wOffset+11*sizeof(dword)}, 08CC70208h + SHA256_STEP 60, {esp+dOffset}, {esp+wOffset+12*sizeof(dword)}, 090BEFFFAh + SHA256_STEP 61, {esp+dOffset}, {esp+wOffset+13*sizeof(dword)}, 0A4506CEBh + SHA256_STEP 62, {esp+dOffset}, {esp+wOffset+14*sizeof(dword)}, 0BEF9A3F7h + SHA256_STEP 63, {esp+dOffset}, {esp+wOffset+15*sizeof(dword)}, 0C67178F2h + +; +; update digest +; + mov edi,[esp+hashOffset] + mov esi,[esp+msgOffset] + + mov eax,[esp+dOffset+vA*sizeof(dword)] + mov ebx,[esp+dOffset+vB*sizeof(dword)] + mov ecx,[esp+dOffset+vC*sizeof(dword)] + mov edx,[esp+dOffset+vD*sizeof(dword)] + add [edi+0*sizeof(dword)],eax + add [edi+1*sizeof(dword)],ebx + add [edi+2*sizeof(dword)],ecx + add [edi+3*sizeof(dword)],edx + mov eax,[esp+dOffset+vE*sizeof(dword)] + mov ebx,[esp+dOffset+vF*sizeof(dword)] + mov ecx,[esp+dOffset+vG*sizeof(dword)] + mov edx,[esp+dOffset+vH*sizeof(dword)] + add [edi+4*sizeof(dword)],eax + add [edi+5*sizeof(dword)],ebx + add [edi+6*sizeof(dword)],ecx + add [edi+7*sizeof(dword)],edx + + mov eax,[esp+lenOffset] + add esi, MBS_SHA256 + sub eax, MBS_SHA256 + jg .sha256_block_loop + + add esp,stackSize ; remove local buffers + REST_GPR + ret +ENDFUNC UpdateSHA256 + +%endif ;; (_IPP >= _IPP_M5) && (_IPP < _IPP_V8) +%endif ;; _FEATURE_OFF_ / _FEATURE_TICKTOCK_ +%endif ;; _ENABLE_ALG_SHA256_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha512g9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha512g9as.asm new file mode 100644 index 0000000..96ee3d9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha512g9as.asm @@ -0,0 +1,441 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SHA512 +; +; Content: +; UpdateSHA512 +; +; + + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SHA512_) +%if (_IPP >= _IPP_G9) + +;; +;; ENDIANNESS +;; +%macro ENDIANNESS 2.nolist + %xdefine %%xmm %1 + %xdefine %%masks %2 + + vpshufb %%xmm, %%xmm, %%masks +%endmacro + + +;; +;; Rotate Right +;; +%macro PRORQ 3.nolist + %xdefine %%mm %1 + %xdefine %%nbits %2 + %xdefine %%tmp %3 + + vpsllq %%tmp, %%mm, (64-%%nbits) + vpsrlq %%mm, %%mm, %%nbits + vpor %%mm, %%mm,%%tmp +%endmacro + + + +;; +;; Init and Update W: +;; +;; j = 0-15 +;; W[j] = ENDIANNESS(src) +;; +;; j = 16-79 +;; W[j] = SIGMA1(W[j- 2]) + W[j- 7] +;; +SIGMA0(W[j-15]) + W[j-16] +;; +;; SIGMA0(x) = ROR64(x,1) ^ROR64(x,8) ^LSR64(x,7) +;; SIGMA1(x) = ROR64(x,19)^ROR64(x,61)^LSR64(x,6) +;; +%macro SIGMA0 4.nolist + %xdefine %%sigma %1 + %xdefine %%x %2 + %xdefine %%t1 %3 + %xdefine %%t2 %4 + + vpsrlq %%sigma, %%x, 7 + vmovdqa %%t1, %%x + PRORQ %%x, 1, %%t2 + vpxor %%sigma, %%sigma, %%x + PRORQ %%t1,8, %%t2 + vpxor %%sigma, %%sigma, %%t1 +%endmacro + + +%macro SIGMA1 4.nolist + %xdefine %%sigma %1 + %xdefine %%x %2 + %xdefine %%t1 %3 + %xdefine %%t2 %4 + + vpsrlq %%sigma, %%x, 6 + vmovdqa %%t1, %%x + PRORQ %%x, 19, %%t2 + vpxor %%sigma, %%sigma, %%x + PRORQ %%t1,61, %%t2 + vpxor %%sigma, %%sigma, %%t1 +%endmacro + + + +;; +;; SHA512 step +;; +;; Ipp64u T1 = H + SUM1(E) + CHJ(E,F,G) + K_SHA512[t] + W[t]; +;; Ipp64u T2 = SUM0(A) + MAJ(A,B,C); +;; D+= T1; +;; H = T1 + T2; +;; +;; where +;; SUM1(x) = ROR64(x,14) ^ ROR64(x,18) ^ ROR64(x,41) +;; SUM0(x) = ROR64(x,28) ^ ROR64(x,34) ^ ROR64(x,39) +;; +;; CHJ(x,y,z) = (x & y) ^ (~x & z) => x&(y^z) ^z +;; MAJ(x,y,z) = (x & y) ^ (x & z) ^ (y & z) = (x&y)^((x^y)&z) +;; +;; Input: +;; A,B,C,D,E,F,G,H - 8 digest's values +;; pW - pointer to the W array +;; pK512 - pointer to the constants +;; pBuffer - temporary buffer +;; Output: +;; A,B,C,D*,E,F,G,H* - 8 digest's values (D and H updated) +;; pW - pointer to the W array +;; pK512 - pointer to the constants +;; pBuffer - temporary buffer (changed) +;; +%macro CHJ 5.nolist + %xdefine %%R %1 + %xdefine %%E %2 + %xdefine %%F %3 + %xdefine %%G %4 + %xdefine %%T %5 + + vpxor %%R, %%F,%%G ; R=(f^g) + vpand %%R, %%R,%%E ; R=e & (f^g) + vpxor %%R, %%R,%%G ; R=e & (f^g) ^g +%endmacro + + +%macro MAJ 5.nolist + %xdefine %%R %1 + %xdefine %%A %2 + %xdefine %%B %3 + %xdefine %%C %4 + %xdefine %%T %5 + + vpxor %%T, %%A,%%B ; T=a^b + vpand %%R, %%A,%%B ; R=a&b + vpand %%T, %%T,%%C ; T=(a^b)&c + vpxor %%R, %%R,%%T ; R=(a&b)^((a^b)&c) +%endmacro + + +%macro SUM0 3.nolist + %xdefine %%R %1 + %xdefine %%X %2 + %xdefine %%tmp %3 + + vmovdqa %%R,%%X + PRORQ %%R,28,%%tmp ; R=ROR(X,28) + PRORQ %%X,34,%%tmp ; X=ROR(X,34) + vpxor %%R, %%R,%%X + PRORQ %%X,(39-34),%%tmp ; X=ROR(x,39) + vpxor %%R, %%R,%%X +%endmacro + + +%macro SUM1 3.nolist + %xdefine %%R %1 + %xdefine %%X %2 + %xdefine %%tmp %3 + + vmovdqa %%R,%%X + PRORQ %%R,14,%%tmp ; R=ROR(X,14) + PRORQ %%X,18,%%tmp ; X=ROR(X,18) + vpxor %%R, %%R,%%X + PRORQ %%X,(41-18),%%tmp ; X=ROR(x,41) + vpxor %%R, %%R,%%X +%endmacro + + +%macro SHA512_STEP 11.nolist + %xdefine %%A %1 + %xdefine %%B %2 + %xdefine %%C %3 + %xdefine %%D %4 + %xdefine %%E %5 + %xdefine %%F %6 + %xdefine %%G %7 + %xdefine %%H %8 + %xdefine %%pW %9 + %xdefine %%pK512 %10 + %xdefine %%pBuffer %11 + + vmovdqa oword [%%pBuffer+0*sizeof(oword)],%%E ; save E + vmovdqa oword [%%pBuffer+1*sizeof(oword)],%%A ; save A + + vmovdqa oword [%%pBuffer+2*sizeof(oword)],%%D ; save D + vmovdqa oword [%%pBuffer+3*sizeof(oword)],%%H ; save H + + CHJ %%D,%%E,%%F,%%G, %%H ; t1 = h+CHJ(e,f,g)+pW[]+pK512[] + vmovq %%H, qword [%%pW] + vpaddq %%D, %%D,%%H ; +[pW] + vmovq %%H, qword [%%pK512] + vpaddq %%D, %%D,%%H ; +[pK512] + vpaddq %%D, %%D,oword [%%pBuffer+3*sizeof(oword)] + vmovdqa oword [%%pBuffer+3*sizeof(oword)],%%D ; save t1 + + MAJ %%H,%%A,%%B,%%C, %%D ; t2 = MAJ(a,b,c) + vmovdqa oword [%%pBuffer+4*sizeof(oword)],%%H ; save t2 + + SUM1 %%D,%%E,%%H ; D = SUM1(e) + vpaddq %%D, %%D,oword [%%pBuffer+3*sizeof(oword)]; t1 = h+CHJ(e,f,g)+pW[]+pK512[] + SUM1(e) + + SUM0 %%H,%%A,%%E ; H = SUM0(a) + vpaddq %%H, %%H,oword [%%pBuffer+4*sizeof(oword)]; t2 = MAJ(a,b,c)+SUM0(a) + + vpaddq %%H, %%H,%%D ; h = t1+t2 + vpaddq %%D, %%D,oword [%%pBuffer+2*sizeof(oword)]; d+= t1 + + vmovdqa %%E,oword [%%pBuffer+0*sizeof(oword)] ; restore E + vmovdqa %%A,oword [%%pBuffer+1*sizeof(oword)] ; restore A +%endmacro + + + +segment .text align=IPP_ALIGN_FACTOR + + +%if (_IPP >= _IPP_V8) +align IPP_ALIGN_FACTOR +SWP_BYTE: +pByteSwp DB 7,6,5,4,3,2,1,0, 15,14,13,12,11,10,9,8 +%endif + +;******************************************************************************************* +;* Purpose: Update internal digest according to message block +;* +;* void UpdateSHA512(DigestSHA512 digest, const Ipp64u* mblk, int mlen, const void* pParam) +;* +;******************************************************************************************* + +;; +;; Lib = W7, V8, P8 +;; +;; Caller = ippsSHA512Update +;; Caller = ippsSHA512Final +;; Caller = ippsSHA512MessageDigest +;; +;; Caller = ippsSHA384Update +;; Caller = ippsSHA384Final +;; Caller = ippsSHA384MessageDigest +;; +;; Caller = ippsHMACSHA512Update +;; Caller = ippsHMACSHA512Final +;; Caller = ippsHMACSHA512MessageDigest +;; +;; Caller = ippsHMACSHA384Update +;; Caller = ippsHMACSHA384Final +;; Caller = ippsHMACSHA384MessageDigest +;; +align IPP_ALIGN_FACTOR +IPPASM UpdateSHA512,PUBLIC + USES_GPR esi,edi,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine digest [ebp + ARG_1 + 0*sizeof(dword)] ; digest address +%xdefine mblk [ebp + ARG_1 + 1*sizeof(dword)] ; buffer address +%xdefine mlen [ebp + ARG_1 + 2*sizeof(dword)] ; buffer length +%xdefine pSHA512 [ebp + ARG_1 + 3*sizeof(dword)] ; address of SHA constants + +%xdefine MBS_SHA512 (128) ; SHA512 block data size + +%assign sSize 5 ; size of save area (oword) +%assign dSize 8 ; size of digest (oword) +%assign wSize 80 ; W values queue (qword) +%assign stackSize (sSize*sizeof(oword)+dSize*sizeof(oword)+wSize*sizeof(qword)+sizeof(dword)) ; stack size (bytes) + +%assign sOffset 0 ; save area +%assign dOffset sOffset+sSize*sizeof(oword) ; digest offset +%assign wOffset dOffset+dSize*sizeof(oword) ; W values offset +%assign acualOffset wOffset+wSize*sizeof(qword) ; actual stack size offset + + mov edi,digest ; digest address + mov esi,mblk ; source data address + mov eax,mlen ; source data length + + mov edx, pSHA512 ; table constant address + + sub esp,stackSize ; allocate local buffer (probably unaligned) + mov ecx,esp + and esp,-16 ; 16-byte aligned stack + sub ecx,esp + add ecx,stackSize ; acual stack size (bytes) + mov [esp+acualOffset],ecx + + vmovq xmm0,qword [edi+sizeof(qword)*0] ; A = digest[0] + vmovq xmm1,qword [edi+sizeof(qword)*1] ; B = digest[1] + vmovq xmm2,qword [edi+sizeof(qword)*2] ; C = digest[2] + vmovq xmm3,qword [edi+sizeof(qword)*3] ; D = digest[3] + vmovq xmm4,qword [edi+sizeof(qword)*4] ; E = digest[4] + vmovq xmm5,qword [edi+sizeof(qword)*5] ; F = digest[5] + vmovq xmm6,qword [edi+sizeof(qword)*6] ; G = digest[6] + vmovq xmm7,qword [edi+sizeof(qword)*7] ; H = digest[7] + vmovdqa oword [esp+dOffset+sizeof(oword)*0], xmm0 + vmovdqa oword [esp+dOffset+sizeof(oword)*1], xmm1 + vmovdqa oword [esp+dOffset+sizeof(oword)*2], xmm2 + vmovdqa oword [esp+dOffset+sizeof(oword)*3], xmm3 + vmovdqa oword [esp+dOffset+sizeof(oword)*4], xmm4 + vmovdqa oword [esp+dOffset+sizeof(oword)*5], xmm5 + vmovdqa oword [esp+dOffset+sizeof(oword)*6], xmm6 + vmovdqa oword [esp+dOffset+sizeof(oword)*7], xmm7 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +.sha512_block_loop: + +;; +;; initialize the first 16 qwords in the array W (remember about endian) +;; + ;vmovdqa xmm1, oword pByteSwp ; load shuffle mask + LD_ADDR ecx, SWP_BYTE + movdqa xmm1, oword [ecx+(pByteSwp-SWP_BYTE)] + mov ecx,0 +align IPP_ALIGN_FACTOR +.loop1: + vmovdqu xmm0, oword [esi+ecx*sizeof(qword)] ; swap input + ENDIANNESS xmm0, xmm1 + vmovdqa oword [esp+wOffset+ecx*sizeof(qword)],xmm0 + add ecx,sizeof(oword)/sizeof(qword) + cmp ecx,16 + jl .loop1 + +;; +;; initialize another 80-16 qwords in the array W +;; +align IPP_ALIGN_FACTOR +.loop2: + vmovdqa xmm1,oword [esp+ecx*sizeof(qword)+wOffset- 2*sizeof(qword)] ; xmm1 = W[j-2] + SIGMA1 xmm0,xmm1,xmm2,xmm3 + + vmovdqu xmm5,oword [esp+ecx*sizeof(qword)+wOffset-15*sizeof(qword)] ; xmm5 = W[j-15] + SIGMA0 xmm4,xmm5,xmm6,xmm3 + + vmovdqu xmm7,oword [esp+ecx*sizeof(qword)+wOffset- 7*sizeof(qword)] ; W[j-7] + vpaddq xmm0, xmm0,xmm4 + vpaddq xmm7, xmm7,oword [esp+ecx*sizeof(qword)+wOffset-16*sizeof(qword)] ; W[j-16] + vpaddq xmm0, xmm0,xmm7 + vmovdqa oword [esp+ecx*sizeof(qword)+wOffset],xmm0 + + add ecx,sizeof(oword)/sizeof(qword) + cmp ecx,80 + jl .loop2 + +;; +;; init A,B,C,D,E,F,G,H by the internal digest +;; + vmovdqa xmm0,oword [esp+dOffset+sizeof(oword)*0] ; A = digest[0] + vmovdqa xmm1,oword [esp+dOffset+sizeof(oword)*1] ; B = digest[1] + vmovdqa xmm2,oword [esp+dOffset+sizeof(oword)*2] ; C = digest[2] + vmovdqa xmm3,oword [esp+dOffset+sizeof(oword)*3] ; D = digest[3] + vmovdqa xmm4,oword [esp+dOffset+sizeof(oword)*4] ; E = digest[4] + vmovdqa xmm5,oword [esp+dOffset+sizeof(oword)*5] ; F = digest[5] + vmovdqa xmm6,oword [esp+dOffset+sizeof(oword)*6] ; G = digest[6] + vmovdqa xmm7,oword [esp+dOffset+sizeof(oword)*7] ; H = digest[7] + +;; +;; perform 0-79 steps +;; + xor ecx,ecx +align IPP_ALIGN_FACTOR +.loop3: +;; A, B, C, D, E, F, G, H W[], K[], buffer +;; -------------------------------------------------------------------------------------------------------------------------------------- + SHA512_STEP xmm0,xmm1,xmm2,xmm3,xmm4,xmm5,xmm6,xmm7, {esp+ecx*sizeof(qword)+wOffset+sizeof(qword)*0},{edx+ecx*sizeof(qword)+sizeof(qword)*0}, {esp} + SHA512_STEP xmm7,xmm0,xmm1,xmm2,xmm3,xmm4,xmm5,xmm6, {esp+ecx*sizeof(qword)+wOffset+sizeof(qword)*1},{edx+ecx*sizeof(qword)+sizeof(qword)*1}, {esp} + SHA512_STEP xmm6,xmm7,xmm0,xmm1,xmm2,xmm3,xmm4,xmm5, {esp+ecx*sizeof(qword)+wOffset+sizeof(qword)*2},{edx+ecx*sizeof(qword)+sizeof(qword)*2}, {esp} + SHA512_STEP xmm5,xmm6,xmm7,xmm0,xmm1,xmm2,xmm3,xmm4, {esp+ecx*sizeof(qword)+wOffset+sizeof(qword)*3},{edx+ecx*sizeof(qword)+sizeof(qword)*3}, {esp} + SHA512_STEP xmm4,xmm5,xmm6,xmm7,xmm0,xmm1,xmm2,xmm3, {esp+ecx*sizeof(qword)+wOffset+sizeof(qword)*4},{edx+ecx*sizeof(qword)+sizeof(qword)*4}, {esp} + SHA512_STEP xmm3,xmm4,xmm5,xmm6,xmm7,xmm0,xmm1,xmm2, {esp+ecx*sizeof(qword)+wOffset+sizeof(qword)*5},{edx+ecx*sizeof(qword)+sizeof(qword)*5}, {esp} + SHA512_STEP xmm2,xmm3,xmm4,xmm5,xmm6,xmm7,xmm0,xmm1, {esp+ecx*sizeof(qword)+wOffset+sizeof(qword)*6},{edx+ecx*sizeof(qword)+sizeof(qword)*6}, {esp} + SHA512_STEP xmm1,xmm2,xmm3,xmm4,xmm5,xmm6,xmm7,xmm0, {esp+ecx*sizeof(qword)+wOffset+sizeof(qword)*7},{edx+ecx*sizeof(qword)+sizeof(qword)*7}, {esp} + + add ecx,8 + cmp ecx,80 + jl .loop3 + +;; +;; update digest +;; + vpaddq xmm0, xmm0,oword [esp+dOffset+sizeof(oword)*0] ; A += digest[0] + vpaddq xmm1, xmm1,oword [esp+dOffset+sizeof(oword)*1] ; B += digest[1] + vpaddq xmm2, xmm2,oword [esp+dOffset+sizeof(oword)*2] ; C += digest[2] + vpaddq xmm3, xmm3,oword [esp+dOffset+sizeof(oword)*3] ; D += digest[3] + vpaddq xmm4, xmm4,oword [esp+dOffset+sizeof(oword)*4] ; E += digest[4] + vpaddq xmm5, xmm5,oword [esp+dOffset+sizeof(oword)*5] ; F += digest[5] + vpaddq xmm6, xmm6,oword [esp+dOffset+sizeof(oword)*6] ; G += digest[6] + vpaddq xmm7, xmm7,oword [esp+dOffset+sizeof(oword)*7] ; H += digest[7] + + vmovdqa oword [esp+dOffset+sizeof(oword)*0],xmm0 ; digest[0] = A + vmovdqa oword [esp+dOffset+sizeof(oword)*1],xmm1 ; digest[1] = B + vmovdqa oword [esp+dOffset+sizeof(oword)*2],xmm2 ; digest[2] = C + vmovdqa oword [esp+dOffset+sizeof(oword)*3],xmm3 ; digest[3] = D + vmovdqa oword [esp+dOffset+sizeof(oword)*4],xmm4 ; digest[4] = E + vmovdqa oword [esp+dOffset+sizeof(oword)*5],xmm5 ; digest[5] = F + vmovdqa oword [esp+dOffset+sizeof(oword)*6],xmm6 ; digest[6] = G + vmovdqa oword [esp+dOffset+sizeof(oword)*7],xmm7 ; digest[7] = H + + add esi, MBS_SHA512 + sub eax, MBS_SHA512 + jg .sha512_block_loop + + vmovq qword [edi+sizeof(qword)*0], xmm0 ; A = digest[0] + vmovq qword [edi+sizeof(qword)*1], xmm1 ; B = digest[1] + vmovq qword [edi+sizeof(qword)*2], xmm2 ; C = digest[2] + vmovq qword [edi+sizeof(qword)*3], xmm3 ; D = digest[3] + vmovq qword [edi+sizeof(qword)*4], xmm4 ; E = digest[4] + vmovq qword [edi+sizeof(qword)*5], xmm5 ; F = digest[5] + vmovq qword [edi+sizeof(qword)*6], xmm6 ; G = digest[6] + vmovq qword [edi+sizeof(qword)*7], xmm7 ; H = digest[7] + + add esp,[esp+acualOffset] + REST_GPR + ret +ENDFUNC UpdateSHA512 + +%endif ;; (_IPP >= _IPP_G9) +%endif ;; _ENABLE_ALG_SHA512_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha512w7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha512w7as.asm new file mode 100644 index 0000000..395e06e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsha512w7as.asm @@ -0,0 +1,463 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SHA512 +; +; Content: +; UpdateSHA512 +; +; + + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SHA512_) +%if (_IPP >= _IPP_W7) && (_IPP < _IPP_G9) + +;; +;; ENDIANNESS +;; +%if (_IPP >= _IPP_V8) +%macro ENDIANNESS 2.nolist + %xdefine %%xmm %1 + %xdefine %%masks %2 + + pshufb %%xmm, %%masks +%endmacro + + +%else +%macro ENDIANNESS 2.nolist + %xdefine %%xmm %1 + %xdefine %%tmp %2 + + pshuflw %%xmm,%%xmm, 00011011b + pshufhw %%xmm,%%xmm, 00011011b + movdqa %%tmp,%%xmm + psrlw %%tmp,8 + psllw %%xmm,8 + por %%xmm,%%tmp +%endmacro + +%endif + +;; +;; Rotate Right +;; +%macro PRORQ 3.nolist + %xdefine %%mm %1 + %xdefine %%nbits %2 + %xdefine %%tmp %3 + + movdqa %%tmp,%%mm + psrlq %%mm,%%nbits + psllq %%tmp,(64-%%nbits) + por %%mm,%%tmp +%endmacro + + + +;; +;; Init and Update W: +;; +;; j = 0-15 +;; W[j] = ENDIANNESS(src) +;; +;; j = 16-79 +;; W[j] = SIGMA1(W[j- 2]) + W[j- 7] +;; +SIGMA0(W[j-15]) + W[j-16] +;; +;; SIGMA0(x) = ROR64(x,1) ^ROR64(x,8) ^LSR64(x,7) +;; SIGMA1(x) = ROR64(x,19)^ROR64(x,61)^LSR64(x,6) +;; +%macro SIGMA0 4.nolist + %xdefine %%sigma %1 + %xdefine %%x %2 + %xdefine %%t1 %3 + %xdefine %%t2 %4 + + movdqa %%sigma, %%x + psrlq %%x, 7 + movdqa %%t1,%%sigma + PRORQ %%sigma, 1, %%t2 + pxor %%sigma, %%x + PRORQ %%t1,8, %%t2 + pxor %%sigma, %%t1 +%endmacro + + +%macro SIGMA1 4.nolist + %xdefine %%sigma %1 + %xdefine %%x %2 + %xdefine %%t1 %3 + %xdefine %%t2 %4 + + movdqa %%sigma, %%x + psrlq %%x, 6 + movdqa %%t1,%%sigma + PRORQ %%sigma, 19, %%t2 + pxor %%sigma, %%x + PRORQ %%t1,61, %%t2 + pxor %%sigma, %%t1 +%endmacro + + + +;; +;; SHA512 step +;; +;; Ipp64u T1 = H + SUM1(E) + CHJ(E,F,G) + K_SHA512[t] + W[t]; +;; Ipp64u T2 = SUM0(A) + MAJ(A,B,C); +;; D+= T1; +;; H = T1 + T2; +;; +;; where +;; SUM1(x) = ROR64(x,14) ^ ROR64(x,18) ^ ROR64(x,41) +;; SUM0(x) = ROR64(x,28) ^ ROR64(x,34) ^ ROR64(x,39) +;; +;; CHJ(x,y,z) = (x & y) ^ (~x & z) => x&(y^z) ^z +;; MAJ(x,y,z) = (x & y) ^ (x & z) ^ (y & z) = (x&y)^((x^y)&z) +;; +;; Input: +;; A,B,C,D,E,F,G,H - 8 digest's values +;; pW - pointer to the W array +;; pK512 - pointer to the constants +;; pBuffer - temporary buffer +;; Output: +;; A,B,C,D*,E,F,G,H* - 8 digest's values (D and H updated) +;; pW - pointer to the W array +;; pK512 - pointer to the constants +;; pBuffer - temporary buffer (changed) +;; +%macro CHJ 5.nolist + %xdefine %%R %1 + %xdefine %%E %2 + %xdefine %%F %3 + %xdefine %%G %4 + %xdefine %%T %5 + + movdqa %%R,%%F ; R=f + pxor %%R,%%G ; R=(f^g) + pand %%R,%%E ; R=e & (f^g) + pxor %%R,%%G ; R=e & (f^g) ^g +%endmacro + + +%macro MAJ 5.nolist + %xdefine %%R %1 + %xdefine %%A %2 + %xdefine %%B %3 + %xdefine %%C %4 + %xdefine %%T %5 + + movdqa %%T,%%B ; T=b + movdqa %%R,%%A ; R=a + pxor %%T,%%A ; T=a^b + pand %%R,%%B ; R=a&b + pand %%T,%%C ; T=(a^b)&c + pxor %%R,%%T ; R=(a&b)^((a^b)&c) +%endmacro + + +%macro SUM0 3.nolist + %xdefine %%R %1 + %xdefine %%X %2 + %xdefine %%tmp %3 + + movdqa %%R,%%X + PRORQ %%R,28,%%tmp ; R=ROR(X,28) + PRORQ %%X,34,%%tmp ; X=ROR(X,34) + pxor %%R,%%X + PRORQ %%X,(39-34),%%tmp ; X=ROR(x,39) + pxor %%R,%%X +%endmacro + + +%macro SUM1 3.nolist + %xdefine %%R %1 + %xdefine %%X %2 + %xdefine %%tmp %3 + + movdqa %%R,%%X + PRORQ %%R,14,%%tmp ; R=ROR(X,14) + PRORQ %%X,18,%%tmp ; X=ROR(X,18) + pxor %%R,%%X + PRORQ %%X,(41-18),%%tmp ; X=ROR(x,41) + pxor %%R,%%X +%endmacro + + +%macro SHA512_STEP 11.nolist + %xdefine %%A %1 + %xdefine %%B %2 + %xdefine %%C %3 + %xdefine %%D %4 + %xdefine %%E %5 + %xdefine %%F %6 + %xdefine %%G %7 + %xdefine %%H %8 + %xdefine %%pW %9 + %xdefine %%pK512 %10 + %xdefine %%pBuffer %11 + + movdqa oword [%%pBuffer+0*sizeof(oword)],%%E ; save E + movdqa oword [%%pBuffer+1*sizeof(oword)],%%A ; save A + + movdqa oword [%%pBuffer+2*sizeof(oword)],%%D ; save D + movdqa oword [%%pBuffer+3*sizeof(oword)],%%H ; save H + + CHJ %%D,%%E,%%F,%%G, %%H ; t1 = h+CHJ(e,f,g)+pW[]+pK512[] + movq %%H, qword [%%pW] + paddq %%D, %%H ; +[pW] + movq %%H, qword [%%pK512] + paddq %%D, %%H ; +[pK512] + paddq %%D,oword [%%pBuffer+3*sizeof(oword)] + movdqa oword [%%pBuffer+3*sizeof(oword)],%%D ; save t1 + + MAJ %%H,%%A,%%B,%%C, %%D ; t2 = MAJ(a,b,c) + movdqa oword [%%pBuffer+4*sizeof(oword)],%%H ; save t2 + + SUM1 %%D,%%E,%%H ; D = SUM1(e) + paddq %%D,oword [%%pBuffer+3*sizeof(oword)] ; t1 = h+CHJ(e,f,g)+pW[]+pK512[] + SUM1(e) + + SUM0 %%H,%%A,%%E ; H = SUM0(a) + paddq %%H,oword [%%pBuffer+4*sizeof(oword)] ; t2 = MAJ(a,b,c)+SUM0(a) + + paddq %%H,%%D ; h = t1+t2 + paddq %%D,oword [%%pBuffer+2*sizeof(oword)] ; d+= t1 + + movdqa %%E,oword [%%pBuffer+0*sizeof(oword)] ; restore E + movdqa %%A,oword [%%pBuffer+1*sizeof(oword)] ; restore A +%endmacro + + + +segment .text align=IPP_ALIGN_FACTOR + + +%if (_IPP >= _IPP_V8) +align IPP_ALIGN_FACTOR +SWP_BYTE: +pByteSwp DB 7,6,5,4,3,2,1,0, 15,14,13,12,11,10,9,8 +%endif + +;******************************************************************************************* +;* Purpose: Update internal digest according to message block +;* +;* void UpdateSHA512(DigestSHA512 digest, const Ipp64u* mblk, int mlen, const void* pParam) +;* +;******************************************************************************************* + +;; +;; Lib = W7, V8, P8 +;; +;; Caller = ippsSHA512Update +;; Caller = ippsSHA512Final +;; Caller = ippsSHA512MessageDigest +;; +;; Caller = ippsSHA384Update +;; Caller = ippsSHA384Final +;; Caller = ippsSHA384MessageDigest +;; +;; Caller = ippsHMACSHA512Update +;; Caller = ippsHMACSHA512Final +;; Caller = ippsHMACSHA512MessageDigest +;; +;; Caller = ippsHMACSHA384Update +;; Caller = ippsHMACSHA384Final +;; Caller = ippsHMACSHA384MessageDigest +;; +align IPP_ALIGN_FACTOR +IPPASM UpdateSHA512,PUBLIC + USES_GPR esi,edi + +%xdefine digest [esp + ARG_1 + 0*sizeof(dword)] ; digest address +%xdefine mblk [esp + ARG_1 + 1*sizeof(dword)] ; buffer address +%xdefine mlen [esp + ARG_1 + 2*sizeof(dword)] ; buffer length +%xdefine pSHA512 [esp + ARG_1 + 3*sizeof(dword)] ; address of SHA constants + +%xdefine MBS_SHA512 (128) ; SHA512 block data size + +%assign sSize 5 ; size of save area (oword) +%assign dSize 8 ; size of digest (oword) +%assign wSize 80 ; W values queue (qword) +%assign stackSize (sSize*sizeof(oword)+dSize*sizeof(oword)+wSize*sizeof(qword)+sizeof(dword)) ; stack size (bytes) + +%assign sOffset 0 ; save area +%assign dOffset sOffset+sSize*sizeof(oword) ; digest offset +%assign wOffset dOffset+dSize*sizeof(oword) ; W values offset +%assign acualOffset wOffset+wSize*sizeof(qword) ; actual stack size offset + + mov edi,digest ; digest address + mov esi,mblk ; source data address + mov eax,mlen ; source data length + + mov edx, pSHA512 ; table constant address + + sub esp,stackSize ; allocate local buffer (probably unaligned) + mov ecx,esp + and esp,-16 ; 16-byte aligned stack + sub ecx,esp + add ecx,stackSize ; acual stack size (bytes) + mov [esp+acualOffset],ecx + + movq xmm0,qword [edi+sizeof(qword)*0] ; A = digest[0] + movq xmm1,qword [edi+sizeof(qword)*1] ; B = digest[1] + movq xmm2,qword [edi+sizeof(qword)*2] ; C = digest[2] + movq xmm3,qword [edi+sizeof(qword)*3] ; D = digest[3] + movq xmm4,qword [edi+sizeof(qword)*4] ; E = digest[4] + movq xmm5,qword [edi+sizeof(qword)*5] ; F = digest[5] + movq xmm6,qword [edi+sizeof(qword)*6] ; G = digest[6] + movq xmm7,qword [edi+sizeof(qword)*7] ; H = digest[7] + movdqa oword [esp+dOffset+sizeof(oword)*0], xmm0 + movdqa oword [esp+dOffset+sizeof(oword)*1], xmm1 + movdqa oword [esp+dOffset+sizeof(oword)*2], xmm2 + movdqa oword [esp+dOffset+sizeof(oword)*3], xmm3 + movdqa oword [esp+dOffset+sizeof(oword)*4], xmm4 + movdqa oword [esp+dOffset+sizeof(oword)*5], xmm5 + movdqa oword [esp+dOffset+sizeof(oword)*6], xmm6 + movdqa oword [esp+dOffset+sizeof(oword)*7], xmm7 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +.sha512_block_loop: + +;; +;; initialize the first 16 qwords in the array W (remember about endian) +;; +%if (_IPP >= _IPP_V8) + ;movdqa xmm1, oword pByteSwp ; load shuffle mask + LD_ADDR ecx, SWP_BYTE + movdqa xmm1, oword [ecx+(pByteSwp-SWP_BYTE)] +%endif + mov ecx,0 +align IPP_ALIGN_FACTOR +.loop1: + movdqu xmm0, oword [esi+ecx*sizeof(qword)] ; swap input + ENDIANNESS xmm0, xmm1 + movdqa oword [esp+wOffset+ecx*sizeof(qword)],xmm0 + add ecx,sizeof(oword)/sizeof(qword) + cmp ecx,16 + jl .loop1 + +;; +;; initialize another 80-16 qwords in the array W +;; +align IPP_ALIGN_FACTOR +.loop2: + movdqa xmm1,oword [esp+ecx*sizeof(qword)+wOffset- 2*sizeof(qword)] ; xmm1 = W[j-2] + SIGMA1 xmm0,xmm1,xmm2,xmm3 + + movdqu xmm5,oword [esp+ecx*sizeof(qword)+wOffset-15*sizeof(qword)] ; xmm5 = W[j-15] + SIGMA0 xmm4,xmm5,xmm6,xmm3 + + movdqu xmm7,oword [esp+ecx*sizeof(qword)+wOffset- 7*sizeof(qword)] ; W[j-7] + paddq xmm0,xmm4 + paddq xmm7,oword [esp+ecx*sizeof(qword)+wOffset-16*sizeof(qword)] ; W[j-16] + paddq xmm0,xmm7 + movdqa oword [esp+ecx*sizeof(qword)+wOffset],xmm0 + + add ecx,sizeof(oword)/sizeof(qword) + cmp ecx,80 + jl .loop2 + +;; +;; init A,B,C,D,E,F,G,H by the internal digest +;; + movdqa xmm0,oword [esp+dOffset+sizeof(oword)*0] ; A = digest[0] + movdqa xmm1,oword [esp+dOffset+sizeof(oword)*1] ; B = digest[1] + movdqa xmm2,oword [esp+dOffset+sizeof(oword)*2] ; C = digest[2] + movdqa xmm3,oword [esp+dOffset+sizeof(oword)*3] ; D = digest[3] + movdqa xmm4,oword [esp+dOffset+sizeof(oword)*4] ; E = digest[4] + movdqa xmm5,oword [esp+dOffset+sizeof(oword)*5] ; F = digest[5] + movdqa xmm6,oword [esp+dOffset+sizeof(oword)*6] ; G = digest[6] + movdqa xmm7,oword [esp+dOffset+sizeof(oword)*7] ; H = digest[7] + +;; +;; perform 0-79 steps +;; + xor ecx,ecx +align IPP_ALIGN_FACTOR +.loop3: +;; A, B, C, D, E, F, G, H W[], K[], buffer +;; -------------------------------------------------------------------------------------------------------------------------------------- + SHA512_STEP xmm0,xmm1,xmm2,xmm3,xmm4,xmm5,xmm6,xmm7, {esp+ecx*sizeof(qword)+wOffset+sizeof(qword)*0},{edx+ecx*sizeof(qword)+sizeof(qword)*0}, {esp} + SHA512_STEP xmm7,xmm0,xmm1,xmm2,xmm3,xmm4,xmm5,xmm6, {esp+ecx*sizeof(qword)+wOffset+sizeof(qword)*1},{edx+ecx*sizeof(qword)+sizeof(qword)*1}, {esp} + SHA512_STEP xmm6,xmm7,xmm0,xmm1,xmm2,xmm3,xmm4,xmm5, {esp+ecx*sizeof(qword)+wOffset+sizeof(qword)*2},{edx+ecx*sizeof(qword)+sizeof(qword)*2}, {esp} + SHA512_STEP xmm5,xmm6,xmm7,xmm0,xmm1,xmm2,xmm3,xmm4, {esp+ecx*sizeof(qword)+wOffset+sizeof(qword)*3},{edx+ecx*sizeof(qword)+sizeof(qword)*3}, {esp} + SHA512_STEP xmm4,xmm5,xmm6,xmm7,xmm0,xmm1,xmm2,xmm3, {esp+ecx*sizeof(qword)+wOffset+sizeof(qword)*4},{edx+ecx*sizeof(qword)+sizeof(qword)*4}, {esp} + SHA512_STEP xmm3,xmm4,xmm5,xmm6,xmm7,xmm0,xmm1,xmm2, {esp+ecx*sizeof(qword)+wOffset+sizeof(qword)*5},{edx+ecx*sizeof(qword)+sizeof(qword)*5}, {esp} + SHA512_STEP xmm2,xmm3,xmm4,xmm5,xmm6,xmm7,xmm0,xmm1, {esp+ecx*sizeof(qword)+wOffset+sizeof(qword)*6},{edx+ecx*sizeof(qword)+sizeof(qword)*6}, {esp} + SHA512_STEP xmm1,xmm2,xmm3,xmm4,xmm5,xmm6,xmm7,xmm0, {esp+ecx*sizeof(qword)+wOffset+sizeof(qword)*7},{edx+ecx*sizeof(qword)+sizeof(qword)*7}, {esp} + + add ecx,8 + cmp ecx,80 + jl .loop3 + +;; +;; update digest +;; + paddq xmm0,oword [esp+dOffset+sizeof(oword)*0] ; A += digest[0] + paddq xmm1,oword [esp+dOffset+sizeof(oword)*1] ; B += digest[1] + paddq xmm2,oword [esp+dOffset+sizeof(oword)*2] ; C += digest[2] + paddq xmm3,oword [esp+dOffset+sizeof(oword)*3] ; D += digest[3] + paddq xmm4,oword [esp+dOffset+sizeof(oword)*4] ; E += digest[4] + paddq xmm5,oword [esp+dOffset+sizeof(oword)*5] ; F += digest[5] + paddq xmm6,oword [esp+dOffset+sizeof(oword)*6] ; G += digest[6] + paddq xmm7,oword [esp+dOffset+sizeof(oword)*7] ; H += digest[7] + + movdqa oword [esp+dOffset+sizeof(oword)*0],xmm0 ; digest[0] = A + movdqa oword [esp+dOffset+sizeof(oword)*1],xmm1 ; digest[1] = B + movdqa oword [esp+dOffset+sizeof(oword)*2],xmm2 ; digest[2] = C + movdqa oword [esp+dOffset+sizeof(oword)*3],xmm3 ; digest[3] = D + movdqa oword [esp+dOffset+sizeof(oword)*4],xmm4 ; digest[4] = E + movdqa oword [esp+dOffset+sizeof(oword)*5],xmm5 ; digest[5] = F + movdqa oword [esp+dOffset+sizeof(oword)*6],xmm6 ; digest[6] = G + movdqa oword [esp+dOffset+sizeof(oword)*7],xmm7 ; digest[7] = H + + add esi, MBS_SHA512 + sub eax, MBS_SHA512 + jg .sha512_block_loop + + movq qword [edi+sizeof(qword)*0], xmm0 ; A = digest[0] + movq qword [edi+sizeof(qword)*1], xmm1 ; B = digest[1] + movq qword [edi+sizeof(qword)*2], xmm2 ; C = digest[2] + movq qword [edi+sizeof(qword)*3], xmm3 ; D = digest[3] + movq qword [edi+sizeof(qword)*4], xmm4 ; E = digest[4] + movq qword [edi+sizeof(qword)*5], xmm5 ; F = digest[5] + movq qword [edi+sizeof(qword)*6], xmm6 ; G = digest[6] + movq qword [edi+sizeof(qword)*7], xmm7 ; H = digest[7] + + add esp,[esp+acualOffset] + REST_GPR + ret +ENDFUNC UpdateSHA512 + +%endif ;; (_IPP >= _IPP_W7) && (_IPP < _IPP_G9) +%endif ;; _ENABLE_ALG_SHA512_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsm2arith_mont_slm.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsm2arith_mont_slm.asm new file mode 100644 index 0000000..d3b5b64 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpsm2arith_mont_slm.asm @@ -0,0 +1,880 @@ +;=============================================================================== +; Copyright (C) 2016 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; P256r1 basic arithmetic function +; +; Content: +; sm2_add +; sm2_sub +; sm2_neg +; sm2_div_by_2 +; sm2_mul_mont_slm +; sm2_sqr_mont_slm +; sm2_mred +; sm2_mont_back +; + + + + + + +%include "asmdefs.inc" +%include "ia_emm.inc" + +%if (_IPP >= _IPP_P8) + +segment .text align=IPP_ALIGN_FACTOR + +;; +;; some SM2 constants +;; +sm2_data: +_prime_sm2 DD 0FFFFFFFFh,0FFFFFFFFh,000000000h,0FFFFFFFFh,0FFFFFFFFh,0FFFFFFFFh,0FFFFFFFFh, 0FFFFFFFEh + +%assign LENSM2 (256/32) ; dword's length of operands + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; Ipp32u _add_256(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +;; input: edi = r +;; esi = a +;; ebx = b +;; +;; output: eax = carry = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM _add_256,PRIVATE + ; r = a+b + mov eax, dword [esi] + add eax, dword [ebx] + mov dword [edi], eax + + mov eax, dword [esi+sizeof(dword)] + adc eax, dword [ebx+sizeof(dword)] + mov dword [edi+sizeof(dword)], eax + + mov eax, dword [esi+sizeof(dword)*2] + adc eax, dword [ebx+sizeof(dword)*2] + mov dword [edi+sizeof(dword)*2], eax + + mov eax, dword [esi+sizeof(dword)*3] + adc eax, dword [ebx+sizeof(dword)*3] + mov dword [edi+sizeof(dword)*3], eax + + mov eax, dword [esi+sizeof(dword)*4] + adc eax, dword [ebx+sizeof(dword)*4] + mov dword [edi+sizeof(dword)*4], eax + + mov eax, dword [esi+sizeof(dword)*5] + adc eax, dword [ebx+sizeof(dword)*5] + mov dword [edi+sizeof(dword)*5], eax + + mov eax, dword [esi+sizeof(dword)*6] + adc eax, dword [ebx+sizeof(dword)*6] + mov dword [edi+sizeof(dword)*6], eax + + mov eax, dword [esi+sizeof(dword)*7] + adc eax, dword [ebx+sizeof(dword)*7] + mov dword [edi+sizeof(dword)*7], eax + + mov eax, 0 + adc eax, 0 + ret +ENDFUNC _add_256 + +;; +;; Ipp32u _sub_256(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +;; input: edi = r +;; esi = a +;; ebx = b +;; +;; output: eax = borrow = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM _sub_256,PRIVATE + ; r = a-b + mov eax, dword [esi] + sub eax, dword [ebx] + mov dword [edi], eax + + mov eax, dword [esi+sizeof(dword)] + sbb eax, dword [ebx+sizeof(dword)] + mov dword [edi+sizeof(dword)], eax + + mov eax, dword [esi+sizeof(dword)*2] + sbb eax, dword [ebx+sizeof(dword)*2] + mov dword [edi+sizeof(dword)*2], eax + + mov eax, dword [esi+sizeof(dword)*3] + sbb eax, dword [ebx+sizeof(dword)*3] + mov dword [edi+sizeof(dword)*3], eax + + mov eax, dword [esi+sizeof(dword)*4] + sbb eax, dword [ebx+sizeof(dword)*4] + mov dword [edi+sizeof(dword)*4], eax + + mov eax, dword [esi+sizeof(dword)*5] + sbb eax, dword [ebx+sizeof(dword)*5] + mov dword [edi+sizeof(dword)*5], eax + + mov eax, dword [esi+sizeof(dword)*6] + sbb eax, dword [ebx+sizeof(dword)*6] + mov dword [edi+sizeof(dword)*6], eax + + mov eax, dword [esi+sizeof(dword)*7] + sbb eax, dword [ebx+sizeof(dword)*7] + mov dword [edi+sizeof(dword)*7], eax + + mov eax, 0 + adc eax, 0 + ret +ENDFUNC _sub_256 + +;; +;; Ipp32u _shl_256(Ipp32u* r, const Ipp32u* a) +;; +;; input: edi = r +;; esi = a +;; +;; output: eax = extension = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM _shl_256,PRIVATE + ; r = a<<1 + movdqu xmm0, oword [esi] + movdqu xmm1, oword [esi+sizeof(oword)] + mov eax, dword [esi+(LENSM2-1)*sizeof(dword)] + + movdqa xmm2, xmm0 + psllq xmm0, 1 + psrlq xmm2, 63 + movdqa xmm3, xmm1 + psllq xmm1, 1 + psrlq xmm3, 63 + + palignr xmm3, xmm2, sizeof(qword) + pslldq xmm2, sizeof(qword) + + por xmm1, xmm3 + por xmm0, xmm2 + movdqu oword [edi], xmm0 + movdqu oword [edi+sizeof(oword)], xmm1 + + shr eax, 31 + ret +ENDFUNC _shl_256 + +;; +;; void _shr_256(Ipp32u* r, const Ipp32u* a) +;; +;; input: edi = r +;; esi = a +;; eax = ext +;; output: eax = extension = 0/1 +;; +align IPP_ALIGN_FACTOR +IPPASM _shr_256,PRIVATE + ; r = a>>1 + movd xmm4, eax + movdqu xmm0, oword [esi] + movdqu xmm1, oword [esi+sizeof(oword)] + + psllq xmm4, 63 + movdqa xmm2, xmm0 + psrlq xmm0, 1 + psllq xmm2, 63 + movdqa xmm3, xmm1 + psrlq xmm1, 1 + psllq xmm3, 63 + + palignr xmm4, xmm3, sizeof(qword) + palignr xmm3, xmm2, sizeof(qword) + + por xmm1, xmm4 + por xmm0, xmm3 + movdqu oword [edi], xmm0 + movdqu oword [edi+sizeof(oword)], xmm1 + + ret +ENDFUNC _shr_256 + +;; +;; void cpy_256(Ipp32u* r, const Ipp32u* a) +;; +%macro cpy_256 2.nolist + %xdefine %%pdst %1 + %xdefine %%psrc %2 + + movdqu xmm0, oword [%%psrc] + movdqu xmm1, oword [%%psrc+sizeof(oword)] + movdqu oword [%%pdst], xmm0 + movdqu oword [%%pdst+sizeof(oword)], xmm1 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void sm2_add(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +align IPP_ALIGN_FACTOR +IPPASM sm2_add,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +%xdefine pB [ebp + ARG_1 + 2*sizeof(dword)] ; source B address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LENSM2] +%assign _sp_ _buf_+(LENSM2)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_+sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + mov edi, pR ; pR + mov esi, pA ; pA + mov ebx, pB ; pB + CALL_IPPASM _add_256 ; R = A+B + mov edx, eax + + lea edi, [esp+_buf_] ; T + mov esi, pR ; R + LD_ADDR ebx, sm2_data ; modulus + lea ebx, [ebx+(_prime_sm2-sm2_data)] + CALL_IPPASM _sub_256 ; T = R-modulus + + lea esi,[esp+_buf_] + mov edi, pR + sub edx, eax ; R = T<0? R : T + cmovnz esi, edi + cpy_256 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC sm2_add + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void sm2_sub(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +align IPP_ALIGN_FACTOR +IPPASM sm2_sub,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +%xdefine pB [ebp + ARG_1 + 2*sizeof(dword)] ; source B address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LENSM2] +%assign _sp_ _buf_+(LENSM2)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_+sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + mov edi, pR ; pR + mov esi, pA ; pA + mov ebx, pB ; pB + CALL_IPPASM _sub_256 ; R = A-B + mov edx, eax + + lea edi, [esp+_buf_] ; T + mov esi, pR ; R + LD_ADDR ebx, sm2_data ; modulus + lea ebx, [ebx+(_prime_sm2-sm2_data)] + CALL_IPPASM _add_256 ; T = R+modulus + + lea esi,[esp+_buf_] + mov edi, pR + test edx, edx ; R = T<0? R : T + cmovz esi, edi + cpy_256 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC sm2_sub + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void sm2_neg(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM sm2_neg,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LENSM2] +%assign _sp_ _buf_+(LENSM2)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_+sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + mov edi, pR ; outpur pR + mov esi, pA ; input pA + + ; r = 0-a + mov eax, 0 + sub eax, dword [esi] + mov dword [edi], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)] + mov dword [edi+sizeof(dword)], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*2] + mov dword [edi+sizeof(dword)*2], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*3] + mov dword [edi+sizeof(dword)*3], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*4] + mov dword [edi+sizeof(dword)*4], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*5] + mov dword [edi+sizeof(dword)*5], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*6] + mov dword [edi+sizeof(dword)*6], eax + mov eax, 0 + sbb eax, dword [esi+sizeof(dword)*7] + mov dword [edi+sizeof(dword)*7], eax + sbb edx,edx + + lea edi, [esp+_buf_] ; T + mov esi, pR ; R + LD_ADDR ebx, sm2_data ; modulus + lea ebx, [ebx+(_prime_sm2-sm2_data)] + CALL_IPPASM _add_256 ; T = R+modulus + + lea esi,[esp+_buf_] + mov edi, pR + test edx, edx ; R = T<0? R : T + cmovz esi, edi + cpy_256 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC sm2_neg + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void sm2_mul_by_2(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM sm2_mul_by_2,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LENSM2] +%assign _sp_ _buf_+(LENSM2)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_+sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + lea edi, [esp+_buf_] ; T + mov esi, pA ; pA + CALL_IPPASM _shl_256 ; T = A<<1 + mov edx, eax + + mov esi, edi ; T + mov edi, pR ; R + LD_ADDR ebx, sm2_data ; modulus + lea ebx, [ebx+(_prime_sm2-sm2_data)] + CALL_IPPASM _sub_256 ; R = T-modulus + + sub edx, eax ; R = R<0? T : R + cmovz esi, edi + cpy_256 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC sm2_mul_by_2 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void sm2_mul_by_3(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM sm2_mul_by_3,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _bufT_ 0 ; T buffer[LENSM2] +%assign _bufU_ _bufT_+(LENSM2)*sizeof(dword) ; U buffer[LENSM2] +%assign _mod_ _bufU_+(LENSM2)*sizeof(dword) ; modulus address [1] +%assign _sp_ _mod_+sizeof(dword) ; esp [1] +%assign _frame_ _sp_+sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + LD_ADDR eax, sm2_data ; srore modulus address + lea eax, [eax+(_prime_sm2-sm2_data)] + mov dword [esp+_mod_], eax + + lea edi, [esp+_bufT_] ; T + mov esi, pA ; A + CALL_IPPASM _shl_256 ; T = A<<1 + mov edx, eax + + mov esi, edi ; T + lea edi, [esp+_bufU_] ; U + mov ebx, [esp+_mod_] ; modulus + CALL_IPPASM _sub_256 ; U = T-modulus + + sub edx, eax ; T = U<0? T : U + cmovz esi, edi + cpy_256 edi, esi + + mov esi, edi + mov ebx, pA + CALL_IPPASM _add_256 ; T +=A + mov edx, eax + + mov edi, pR ; R + mov ebx, [esp+_mod_] ; modulus + CALL_IPPASM _sub_256 ; R = T-modulus + + sub edx, eax ; R = T<0? R : T + cmovz esi, edi + cpy_256 edi, esi + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC sm2_mul_by_3 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void sm2_div_by_2(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM sm2_div_by_2,PUBLIC + USES_GPR esi,edi,ebx,ebp + + mov ebp, esp ; save original esp to use it to reach parameters + +%xdefine pR [ebp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [ebp + ARG_1 + 1*sizeof(dword)] ; source A address +; +; stack layout: +; +%assign _buf_ 0 ; buffer[LENSM2] +%assign _sp_ _buf_+(LENSM2)*sizeof(dword) ; esp[1] +%assign _frame_ _sp_+sizeof(dword) ; +16 bytes for alignment + + mov eax, esp ; save esp + sub esp, _frame_ ; allocate frame + and esp, -16 ; provide 16-byte alignment + mov dword [esp+_sp_], eax ; store esp + + lea edi, [esp+_buf_] ; T + mov esi, pA ; A + LD_ADDR ebx, sm2_data ; modulus + lea ebx, [ebx+(_prime_sm2-sm2_data)] + CALL_IPPASM _add_256 ; R = A+modulus + mov edx, 0 + + mov ecx, dword [esi] ; shifted_data = (a[0]&1)? T : A + and ecx, 1 + cmovnz esi, edi + cmovz eax, edx + mov edi, pR + CALL_IPPASM _shr_256 + + mov esp, [esp+_sp_] + REST_GPR + ret +ENDFUNC sm2_div_by_2 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void sm2_mul_mont_slm(Ipp32u* r, const Ipp32u* a, const Ipp32u* b) +;; +align IPP_ALIGN_FACTOR +IPPASM sm2_mul_mont_slm,PUBLIC + USES_GPR ebp,ebx,esi,edi + +%xdefine pR [eax + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [eax + ARG_1 + 1*sizeof(dword)] ; source A address +%xdefine pB [eax + ARG_1 + 2*sizeof(dword)] ; source B address +; +; stack layout: +; +%assign _buf_ 0 +%assign _rp_ _buf_+(LENSM2+1)*sizeof(dword) ; pR +%assign _ap_ _rp_ +sizeof(dword) ; pA +%assign _bp_ _ap_+sizeof(dword) ; pB +%assign _sp_ _bp_+sizeof(dword) ; esp storage +%assign _ssize_ _sp_+sizeof(dword) ; size allocated stack + + mov eax, esp ; save esp + sub esp, _ssize_ ; allocate stack + and esp, -16 ; provide 16-byte stack alignment + mov dword [esp+_sp_], eax ; store original esp + + ; clear buffer + pxor mm0, mm0 + movq qword [esp+_buf_], mm0 + movq qword [esp+_buf_+sizeof(qword)], mm0 + movq qword [esp+_buf_+sizeof(qword)*2], mm0 + movq qword [esp+_buf_+sizeof(qword)*3], mm0 + movd dword [esp+_buf_+sizeof(qword)*4], mm0 + + ; store parameters into the stack + ; note: eax here stores an original esp, so it can be used to reach function parameters + mov edi, pR + mov esi, pA + mov ebp, pB + mov dword [esp+_rp_], edi + mov dword [esp+_ap_], esi + mov dword [esp+_bp_], ebp + + mov edi, LENSM2 + + movd mm1, dword [esi+sizeof(dword)] ; pre load a[1], a[2], a[3], a[4] + movd mm2, dword [esi+sizeof(dword)*2] + movd mm3, dword [esi+sizeof(dword)*3] + movd mm4, dword [esi+sizeof(dword)*4] + +align IPP_ALIGN_FACTOR +.mmul_loop: +; +; i-st pass +; modulus = 2^256 -2^224 +2^192 +2^96 -1 +; [8] [7] [6] [3] [0] +; m0 = 1 +; + movd mm7, edi ; save pass counter + + mov edx, dword [ebp] ; b = b[i] + mov eax, dword [esi] ; a[0] + movd mm0, edx + add ebp, sizeof(dword) + mov dword [esp+_bp_], ebp + + pmuludq mm1, mm0 ; a[1]*b[i] + pmuludq mm2, mm0 ; a[2]*b[i] + + mul edx ; (E:u) = (edx:eax) = a[0]*b[i]+buf[0] + add eax, dword [esp+_buf_] + adc edx, 0 + + pmuludq mm3, mm0 ; a[3]*b[i] + pmuludq mm4, mm0 ; a[4]*b[i] + +; multiplication round 1 - round 4 + movd ecx, mm1 ; p = a[1]*b[i] + E + psrlq mm1, 32 + add ecx, edx + movd edx, mm1 + adc edx, 0 + add ecx, dword [esp+_buf_+sizeof(dword)*1] + movd mm1, dword [esi+sizeof(dword)*5] + adc edx, 0 + + movd ebx, mm2 ; p = a[2]*b[i] + E + psrlq mm2, 32 + add ebx, edx + movd edx, mm2 + adc edx, 0 + add ebx, dword [esp+_buf_+sizeof(dword)*2] + movd mm2, dword [esi+sizeof(dword)*6] + adc edx, 0 + + pmuludq mm1, mm0 ; a[5]*b[i] + pmuludq mm2, mm0 ; a[6]*b[i] + + movd ebp, mm3 ; p = a[3]*b[i] + E + psrlq mm3, 32 + add ebp, edx + movd edx, mm3 + adc edx, 0 + add ebp, dword [esp+_buf_+sizeof(dword)*3] + movd mm3, dword [esi+sizeof(dword)*7] + adc edx, 0 + + movd edi, mm4 ; p = a[4]*b[i] + E + psrlq mm4, 32 + add edi, edx + movd edx, mm4 + adc edx, 0 + add edi, dword [esp+_buf_+sizeof(dword)*4] + adc edx, 0 + + pmuludq mm3, mm0 ; a[7]*b[i] + +;;; and reduction ;;; + ; eax =u0 + + mov dword [esp+_buf_+sizeof(dword)*0], ecx ; copy + + add ebx, eax ; +u0 + mov dword [esp+_buf_+sizeof(dword)*1], ebx + + mov ecx, eax + sbb eax, 0 + sub ebp, eax ; -u0 +cf + mov dword [esp+_buf_+sizeof(dword)*2], ebp + + sbb edi, 0 ; -bf + mov dword [esp+_buf_+sizeof(dword)*3], edi + mov edi, 0 ; save edi = bf + adc edi, 0 + mov eax, ecx + +; multiplication round 5 - round 7 + movd ecx, mm1 ; p = a[5]*b[i] + E + psrlq mm1, 32 + add ecx, edx + movd edx, mm1 + adc edx, 0 + add ecx, dword [esp+_buf_+sizeof(dword)*5] + adc edx, 0 + + movd ebx, mm2 ; p = a[6]*b[i] + E + psrlq mm2, 32 + add ebx, edx + movd edx, mm2 + adc edx, 0 + add ebx, dword [esp+_buf_+sizeof(dword)*6] + adc edx, 0 + + movd ebp, mm3 ; p = a[7]*b[i] + E + psrlq mm3, 32 + add ebp, edx + movd edx, mm3 + adc edx, 0 + add ebp, dword [esp+_buf_+sizeof(dword)*7] + adc edx, 0 + +;;; and reduction ;;; + sub ecx, edi ; -bf + mov dword [esp+_buf_+sizeof(dword)*4], ecx + sbb ebx, 0 ; -bf + mov dword [esp+_buf_+sizeof(dword)*5], ebx + sbb ebp, eax ; -u0+bf + mov dword [esp+_buf_+sizeof(dword)*6], ebp + +; last multiplication round 8 + movd edi, mm7 ; restore pass counter + + sbb eax, 0 ; u0-bf + mov ebx, 0 + + add edx, dword [esp+_buf_+sizeof(dword)*8] + adc ebx, 0 + add edx, eax + mov dword [esp+_buf_+sizeof(dword)*7], edx + adc ebx, 0 + mov dword [esp+_buf_+sizeof(dword)*8], ebx + + sub edi, 1 + movd mm1, dword [esi+sizeof(dword)] ; speculative load a[1], a[2], a[3], a[4] + movd mm2, dword [esi+sizeof(dword)*2] + movd mm3, dword [esi+sizeof(dword)*3] + movd mm4, dword [esi+sizeof(dword)*4] + jz .exit_mmul_loop + + mov ebp, dword [esp+_bp_] ; restore pB + jmp .mmul_loop + +.exit_mmul_loop: + emms + +; final reduction + mov edi, [esp+_rp_] ; result + lea esi, [esp+_buf_] ; buffer + LD_ADDR ebx, sm2_data ; modulus + lea ebx, [ebx+(_prime_sm2-sm2_data)] + CALL_IPPASM _sub_256 + mov edx, dword [esp+_buf_+LENSM2*sizeof(dword)] + sub edx, eax + +; copy + cmovz esi, edi + cpy_256 edi, esi + + mov esp, [esp+_sp_] ; release stack + REST_GPR + ret +ENDFUNC sm2_mul_mont_slm + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void sm2_sqr_mont_slm(Ipp32u* r, const Ipp32u* a) +;; +align IPP_ALIGN_FACTOR +IPPASM sm2_sqr_mont_slm,PUBLIC + USES_GPR esi,edi + +%xdefine pR [esp + ARG_1 + 0*sizeof(dword)] ; product address +%xdefine pA [esp + ARG_1 + 1*sizeof(dword)] ; source A address + + ;; use sm2_mul_mont_slm to compute sqr + mov esi, pA + mov edi, pR + push esi + push esi + push edi + CALL_IPPASM sm2_mul_mont_slm,PUBLIC + add esp, sizeof(dword)*3 + REST_GPR + ret +ENDFUNC sm2_sqr_mont_slm + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; void sm2_mred(Ipp32u* r, Ipp32u* prod) +;; +; modulus = 2^256 -2^224 +2^192 +2^96 -1 +; [8] [7] [6] [3] [0] +; m0 = 1 +; +align IPP_ALIGN_FACTOR +IPPASM sm2_mred,PUBLIC + USES_GPR ebx,esi,edi + +%xdefine pR [esp + ARG_1 + 0*sizeof(dword)] ; reduction address +%xdefine pA [esp + ARG_1 + 1*sizeof(dword)] ; source product address + + ; get parameters: + mov esi, pA + + mov ecx, LENSM2 + xor edx, edx +align IPP_ALIGN_FACTOR +.mred_loop: + mov eax, dword [esi] + + mov ebx, 0 + mov dword [esi], ebx + + mov ebx, dword [esi+sizeof(dword)*2] + add ebx, eax + mov dword [esi+sizeof(dword)*2], ebx + + mov ebx, dword [esi+sizeof(dword)*3] + push eax + sbb eax, 0 + sub ebx, eax + mov dword [esi+sizeof(dword)*3], ebx + pop eax + + mov ebx, dword [esi+sizeof(dword)*4] + sbb ebx, 0 + mov dword [esi+sizeof(dword)*4], ebx + + mov ebx, dword [esi+sizeof(dword)*5] + sbb ebx, 0 + mov dword [esi+sizeof(dword)*5], ebx + + mov ebx, dword [esi+sizeof(dword)*6] + sbb ebx, 0 + mov dword [esi+sizeof(dword)*6], ebx + + mov ebx, dword [esi+sizeof(dword)*7] + sbb ebx, eax + mov dword [esi+sizeof(dword)*7], ebx + + mov ebx, dword [esi+sizeof(dword)*8] + sbb eax, 0 + add eax, edx + mov edx, 0 + adc edx, 0 + add ebx, eax + mov dword [esi+sizeof(dword)*8], ebx + adc edx, 0 + + lea esi, [esi+sizeof(dword)] + sub ecx, 1 + jnz .mred_loop + + ; final reduction + mov edi, pR ; result + LD_ADDR ebx, sm2_data ; addres of the modulus + lea ebx, [ebx+(_prime_sm2-sm2_data)] + CALL_IPPASM _sub_256 + + sub edx, eax + cmovz esi, edi + cpy_256 edi, esi + + REST_GPR + ret +ENDFUNC sm2_mred + +%endif ;; _IPP >= _IPP_P8 + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpvariant.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpvariant.inc new file mode 100644 index 0000000..61f4b8f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpvariant.inc @@ -0,0 +1,133 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; Intel(R) Integrated Performance Primitives +; Cryptographic Primitives (ippcp) +; +; Purpose: +; Define ippCP variant +; +; do not change definitions below! +; + +;; +;; modes of the feature +;; +%assign _FEATURE_OFF_ 0 ;; feature is OFF +%assign _FEATURE_ON_ 1 ;; feature is ON +%assign _FEATURE_TICKTOCK_ 2 ;; dectect is feature OFF/ON + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;_XMM7560_ = 1 +%ifdef _XMM7560_ + %include "pcpvariant_xmm7560.inc" +%endif + +;;_TXT_ACM_ = 1 +%ifdef _TXT_ACM_ + %include "pcpvariant_txt_acm.inc" +%endif +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; +;; it possible to force use of C-version of some implementtaions +;; instead of ASM one +;; +%ifndef _USE_C_cpAdd_BNU_ + %assign _USE_C_cpAdd_BNU_ _FEATURE_OFF_ +%endif + +%ifndef _USE_C_cpSub_BNU_ + %assign _USE_C_cpSub_BNU_ _FEATURE_OFF_ +%endif + +%ifndef _USE_C_cpInc_BNU_ + %assign _USE_C_cpInc_BNU_ _FEATURE_OFF_ +%endif + +%ifndef _USE_C_cpAddMulDgt_BNU_ + %assign _USE_C_cpAddMulDgt_BNU_ _FEATURE_OFF_ +%endif + +%ifndef _USE_C_cpMulAdc_BNU_school_ + %assign _USE_C_cpMulAdc_BNU_school_ _FEATURE_OFF_ +%endif + +%ifndef _USE_C_cpSqrAdc_BNU_school_ + %assign _USE_C_cpSqrAdc_BNU_school_ _FEATURE_OFF_ +%endif + +%ifndef _USE_C_cpMontRedAdc_BNU_ + %assign _USE_C_cpMontRedAdc_BNU_ _FEATURE_OFF_ +%endif + +;; +;; if there is no outside assignment +;; set _SHA_NI_ENABLING_ based on CPU specification +;; +%ifndef _SHA_NI_ENABLING_ + %if (_IPP >= _IPP_P8 ) + %assign _SHA_NI_ENABLING_ _FEATURE_TICKTOCK_ + %else + %assign _SHA_NI_ENABLING_ _FEATURE_OFF_ + %endif +%endif + +;; +;; select Hash algorithm +;; +%ifndef _DISABLE_ALG_SHA1_ + %assign _ENABLE_ALG_SHA1_ _FEATURE_ON_ ;; SHA1 on +%else + %assign _ENABLE_ALG_SHA1_ _FEATURE_OFF_ ;; SHA1 on +%endif + +%ifndef _DISABLE_ALG_SHA256_ + %assign _ENABLE_ALG_SHA256_ _FEATURE_ON_ ;; SHA256 on +%else + %assign _ENABLE_ALG_SHA256_ _FEATURE_OFF_ ;; SHA256 off +%endif + +%ifndef _DISABLE_ALG_SHA521_ + %assign _ENABLE_ALG_SHA512_ _FEATURE_ON_ ;; SHA512 on +%else + %assign _ENABLE_ALG_SHA512_ _FEATURE_OFF_ ;; SHA512 off +%endif + +%ifndef _DISABLE_ALG_MD5_ + %assign _ENABLE_ALG_MD5_ _FEATURE_ON_ ;; MD5 on +%else + %assign _ENABLE_ALG_MD5_ _FEATURE_OFF_ ;; MD5 off +%endif + +%ifndef _DISABLE_ALG_SM3_ + %assign _ENABLE_ALG_SM3_ _FEATURE_ON_ ;; SM3 on +%else + %assign _ENABLE_ALG_SM3_ _FEATURE_OFF_ ;; SM3 off +%endif + + +%assign _ENABLE_KARATSUBA_ 0 ;; not use Karatsuba method for multiplication + +%assign _NUSE 0 ;; do not use use +%assign _USE 1 ;; do use + +;; ~ 5.0 +%assign _USE_NN_MUL_BNU_FS_ _NUSE ;; use/not use NN version of full-size multiplication +%assign _USE_NN_MONTMUL_ _NUSE ;; use/not use NN version of Montgomery multiplication + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpvariant_txt_acm.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpvariant_txt_acm.inc new file mode 100644 index 0000000..ae0fff4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpvariant_txt_acm.inc @@ -0,0 +1,39 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; Intel(R) Integrated Performance Primitives +; Cryptographic Primitives (ippcp) +; +; Purpose: +; Update standard ippCP variant +; +; do not change definitions below! +; + +%ifdef _TXT_ACM_ + +;; +;; HASH algs outside settings +;; +%assign _SHA_NI_ENABLING_ _FEATURE_TICKTOCK_ + +;; +;; select Hash algorithm +;; +%assign _ENABLE_ALG_MD5_ _FEATURE_OFF_ + +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpvariant_xmm7560.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpvariant_xmm7560.inc new file mode 100644 index 0000000..0c3a010 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_ia32/pcpvariant_xmm7560.inc @@ -0,0 +1,45 @@ +;=============================================================================== +; Copyright (C) 2016 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; Intel(R) Integrated Performance Primitives +; Cryptographic Primitives (ippcp) +; +; Purpose: +; Update standard ippCP variant +; for code size optimization +; + +%ifdef _XMM7560_ +%if (_IPP >= _IPP_P8) + +%assign OFF 0 +%assign ON 1 + +;%assign _USE_C_cpAdd_BNU_ ON +;%assign _USE_C_cpSub_BNU_ ON +;%assign _USE_C_cpInc_BNU_ ON +;%assign _USE_C_cpAddMulDgt_BNU_ ON +;%assign _USE_C_cpSubMulDgt_BNU_ ON +%assign _USE_C_cpMulAdc_BNU_school_ ON +%assign _USE_C_cpSqrAdc_BNU_school_ ON +%assign _USE_C_cpMontRedAdc_BNU_ ON + +%assign _DISABLE_ECP_256R1_HARDCODED_BP_TBL_ OFF +%assign _DISABLE_ECP_384R1_HARDCODED_BP_TBL_ OFF + +%endif +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/aes_common.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/aes_common.inc new file mode 100644 index 0000000..f2f793a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/aes_common.inc @@ -0,0 +1,365 @@ +;=============================================================================== +; Copyright (C) 2020 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +%ifndef _AES_COMMON_ASM_ +%define _AES_COMMON_ASM_ + +%include "reg_sizes.inc" + +;; ============================================================================= +;; Generic macro to produce code that executes %%OPCODE instruction +;; on selected number of AES blocks (16 bytes long ) between 0 and 16. +;; All three operands of the instruction come from registers. +;; Note: if 3 blocks are left at the end instruction is produced to operate all +;; 4 blocks (full width of ZMM) + +%macro ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 14 +%define %%NUM_BLOCKS %1 ; [in] numerical value, number of AES blocks (0 to 16) +%define %%OPCODE %2 ; [in] instruction name +%define %%DST0 %3 ; [out] destination ZMM register +%define %%DST1 %4 ; [out] destination ZMM register +%define %%DST2 %5 ; [out] destination ZMM register +%define %%DST3 %6 ; [out] destination ZMM register +%define %%SRC1_0 %7 ; [in] source 1 ZMM register +%define %%SRC1_1 %8 ; [in] source 1 ZMM register +%define %%SRC1_2 %9 ; [in] source 1 ZMM register +%define %%SRC1_3 %10 ; [in] source 1 ZMM register +%define %%SRC2_0 %11 ; [in] source 2 ZMM register +%define %%SRC2_1 %12 ; [in] source 2 ZMM register +%define %%SRC2_2 %13 ; [in] source 2 ZMM register +%define %%SRC2_3 %14 ; [in] source 2 ZMM register + +%assign reg_idx 0 +%assign blocks_left %%NUM_BLOCKS + +%rep (%%NUM_BLOCKS / 4) +%xdefine %%DSTREG %%DST %+ reg_idx +%xdefine %%SRC1REG %%SRC1_ %+ reg_idx +%xdefine %%SRC2REG %%SRC2_ %+ reg_idx + %%OPCODE %%DSTREG, %%SRC1REG, %%SRC2REG +%undef %%DSTREG +%undef %%SRC1REG +%undef %%SRC2REG +%assign reg_idx (reg_idx + 1) +%assign blocks_left (blocks_left - 4) +%endrep + +%xdefine %%DSTREG %%DST %+ reg_idx +%xdefine %%SRC1REG %%SRC1_ %+ reg_idx +%xdefine %%SRC2REG %%SRC2_ %+ reg_idx + +%if blocks_left == 1 + %%OPCODE XWORD(%%DSTREG), XWORD(%%SRC1REG), XWORD(%%SRC2REG) +%elif blocks_left == 2 + %%OPCODE YWORD(%%DSTREG), YWORD(%%SRC1REG), YWORD(%%SRC2REG) +%elif blocks_left == 3 + %%OPCODE %%DSTREG, %%SRC1REG, %%SRC2REG +%endif + +%endmacro + +;; ============================================================================= +;; Loads specified number of AES blocks into ZMM registers +;; %%FLAGS are optional and only affect behavior when 3 trailing blocks are left +;; - if %%FlAGS not provided then exactly 3 blocks are loaded (move and insert) +;; - if "load_4_instead_of_3" option is passed then 4 blocks are loaded +%macro ZMM_LOAD_BLOCKS_0_16 7-8 +%define %%NUM_BLOCKS %1 ; [in] numerical value, number of AES blocks (0 to 16) +%define %%INP %2 ; [in] input data pointer to read from +%define %%DATA_OFFSET %3 ; [in] offset to the output pointer (GP or numerical) +%define %%DST0 %4 ; [out] ZMM register with loaded data +%define %%DST1 %5 ; [out] ZMM register with loaded data +%define %%DST2 %6 ; [out] ZMM register with loaded data +%define %%DST3 %7 ; [out] ZMM register with loaded data +%define %%FLAGS %8 ; [in] optional "load_4_instead_of_3" + +%assign src_offset 0 +%assign dst_idx 0 + +%rep (%%NUM_BLOCKS / 4) +%xdefine %%DSTREG %%DST %+ dst_idx + vmovdqu8 %%DSTREG, [%%INP + %%DATA_OFFSET + src_offset] +%undef %%DSTREG +%assign src_offset (src_offset + 64) +%assign dst_idx (dst_idx + 1) +%endrep + +%assign blocks_left (%%NUM_BLOCKS % 4) +%xdefine %%DSTREG %%DST %+ dst_idx + +%if blocks_left == 1 + vmovdqu8 XWORD(%%DSTREG), [%%INP + %%DATA_OFFSET + src_offset] +%elif blocks_left == 2 + vmovdqu8 YWORD(%%DSTREG), [%%INP + %%DATA_OFFSET + src_offset] +%elif blocks_left == 3 +%ifidn %%FLAGS, load_4_instead_of_3 + vmovdqu8 %%DSTREG, [%%INP + %%DATA_OFFSET + src_offset] +%else + vmovdqu8 YWORD(%%DSTREG), [%%INP + %%DATA_OFFSET + src_offset] + vinserti64x2 %%DSTREG, [%%INP + %%DATA_OFFSET + src_offset + 32], 2 +%endif +%endif + +%endmacro + +;; ============================================================================= +;; Loads specified number of AES blocks into ZMM registers using mask register +;; for the last loaded register (xmm, ymm or zmm). +;; Loads take place at 1 byte granularity. +%macro ZMM_LOAD_MASKED_BLOCKS_0_16 8 +%define %%NUM_BLOCKS %1 ; [in] numerical value, number of AES blocks (0 to 16) +%define %%INP %2 ; [in] input data pointer to read from +%define %%DATA_OFFSET %3 ; [in] offset to the output pointer (GP or numerical) +%define %%DST0 %4 ; [out] ZMM register with loaded data +%define %%DST1 %5 ; [out] ZMM register with loaded data +%define %%DST2 %6 ; [out] ZMM register with loaded data +%define %%DST3 %7 ; [out] ZMM register with loaded data +%define %%MASK %8 ; [in] mask register + +%assign src_offset 0 +%assign dst_idx 0 +%assign blocks_left %%NUM_BLOCKS + +%if %%NUM_BLOCKS > 0 +%rep (((%%NUM_BLOCKS + 3) / 4) - 1) +%xdefine %%DSTREG %%DST %+ dst_idx + vmovdqu8 %%DSTREG, [%%INP + %%DATA_OFFSET + src_offset] +%undef %%DSTREG +%assign src_offset (src_offset + 64) +%assign dst_idx (dst_idx + 1) +%assign blocks_left (blocks_left - 4) +%endrep +%endif ; %if %%NUM_BLOCKS > 0 + +%xdefine %%DSTREG %%DST %+ dst_idx + +%if blocks_left == 1 + vmovdqu8 XWORD(%%DSTREG){%%MASK}{z}, [%%INP + %%DATA_OFFSET + src_offset] +%elif blocks_left == 2 + vmovdqu8 YWORD(%%DSTREG){%%MASK}{z}, [%%INP + %%DATA_OFFSET + src_offset] +%elif (blocks_left == 3 || blocks_left == 4) + vmovdqu8 %%DSTREG{%%MASK}{z}, [%%INP + %%DATA_OFFSET + src_offset] +%endif + +%endmacro + +;; ============================================================================= +;; Stores specified number of AES blocks from ZMM registers +%macro ZMM_STORE_BLOCKS_0_16 7 +%define %%NUM_BLOCKS %1 ; [in] numerical value, number of AES blocks (0 to 16) +%define %%OUTP %2 ; [in] output data pointer to write to +%define %%DATA_OFFSET %3 ; [in] offset to the output pointer (GP or numerical) +%define %%SRC0 %4 ; [in] ZMM register with data to store +%define %%SRC1 %5 ; [in] ZMM register with data to store +%define %%SRC2 %6 ; [in] ZMM register with data to store +%define %%SRC3 %7 ; [in] ZMM register with data to store + +%assign dst_offset 0 +%assign src_idx 0 + +%rep (%%NUM_BLOCKS / 4) +%xdefine %%SRCREG %%SRC %+ src_idx + vmovdqu8 [%%OUTP + %%DATA_OFFSET + dst_offset], %%SRCREG +%undef %%SRCREG +%assign dst_offset (dst_offset + 64) +%assign src_idx (src_idx + 1) +%endrep + +%assign blocks_left (%%NUM_BLOCKS % 4) +%xdefine %%SRCREG %%SRC %+ src_idx + +%if blocks_left == 1 + vmovdqu8 [%%OUTP + %%DATA_OFFSET + dst_offset], XWORD(%%SRCREG) +%elif blocks_left == 2 + vmovdqu8 [%%OUTP + %%DATA_OFFSET + dst_offset], YWORD(%%SRCREG) +%elif blocks_left == 3 + vmovdqu8 [%%OUTP + %%DATA_OFFSET + dst_offset], YWORD(%%SRCREG) + vextracti32x4 [%%OUTP + %%DATA_OFFSET + dst_offset + 32], %%SRCREG, 2 +%endif + +%endmacro + +;; ============================================================================= +;; Stores specified number of AES blocks from ZMM registers with mask register +;; for the last loaded register (xmm, ymm or zmm). +;; Stores take place at 1 byte granularity. +%macro ZMM_STORE_MASKED_BLOCKS_0_16 8 +%define %%NUM_BLOCKS %1 ; [in] numerical value, number of AES blocks (0 to 16) +%define %%OUTP %2 ; [in] output data pointer to write to +%define %%DATA_OFFSET %3 ; [in] offset to the output pointer (GP or numerical) +%define %%SRC0 %4 ; [in] ZMM register with data to store +%define %%SRC1 %5 ; [in] ZMM register with data to store +%define %%SRC2 %6 ; [in] ZMM register with data to store +%define %%SRC3 %7 ; [in] ZMM register with data to store +%define %%MASK %8 ; [in] mask register + +%assign dst_offset 0 +%assign src_idx 0 +%assign blocks_left %%NUM_BLOCKS + +%if %%NUM_BLOCKS > 0 +%rep (((%%NUM_BLOCKS + 3) / 4) - 1) +%xdefine %%SRCREG %%SRC %+ src_idx + vmovdqu8 [%%OUTP + %%DATA_OFFSET + dst_offset], %%SRCREG +%undef %%SRCREG +%assign dst_offset (dst_offset + 64) +%assign src_idx (src_idx + 1) +%assign blocks_left (blocks_left - 4) +%endrep +%endif ; %if %%NUM_BLOCKS > 0 + +%xdefine %%SRCREG %%SRC %+ src_idx + +%if blocks_left == 1 + vmovdqu8 [%%OUTP + %%DATA_OFFSET + dst_offset]{%%MASK}, XWORD(%%SRCREG) +%elif blocks_left == 2 + vmovdqu8 [%%OUTP + %%DATA_OFFSET + dst_offset]{%%MASK}, YWORD(%%SRCREG) +%elif (blocks_left == 3 || blocks_left == 4) + vmovdqu8 [%%OUTP + %%DATA_OFFSET + dst_offset]{%%MASK}, %%SRCREG +%endif + +%endmacro + +;;; =========================================================================== +;;; Handles AES encryption rounds +;;; It handles special cases: the last and first rounds +;;; Optionally, it performs XOR with data after the last AES round. +;;; Uses NROUNDS parameterto check what needs to be done for the current round. +;;; If 3 blocks are trailing then operation on whole ZMM is performed (4 blocks). +%macro ZMM_AESENC_ROUND_BLOCKS_0_16 12 +%define %%L0B0_3 %1 ; [in/out] zmm; blocks 0 to 3 +%define %%L0B4_7 %2 ; [in/out] zmm; blocks 4 to 7 +%define %%L0B8_11 %3 ; [in/out] zmm; blocks 8 to 11 +%define %%L0B12_15 %4 ; [in/out] zmm; blocks 12 to 15 +%define %%KEY %5 ; [in] zmm containing round key +%define %%ROUND %6 ; [in] round number +%define %%D0_3 %7 ; [in] zmm or no_data; plain/cipher text blocks 0-3 +%define %%D4_7 %8 ; [in] zmm or no_data; plain/cipher text blocks 4-7 +%define %%D8_11 %9 ; [in] zmm or no_data; plain/cipher text blocks 8-11 +%define %%D12_15 %10 ; [in] zmm or no_data; plain/cipher text blocks 12-15 +%define %%NUMBL %11 ; [in] number of blocks; numerical value +%define %%NROUNDS %12 ; [in] number of rounds; numerical value + +;;; === first AES round +%if (%%ROUND < 1) + ;; round 0 + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vpxorq, \ + %%L0B0_3, %%L0B4_7, %%L0B8_11, %%L0B12_15, \ + %%L0B0_3, %%L0B4_7, %%L0B8_11, %%L0B12_15, \ + %%KEY, %%KEY, %%KEY, %%KEY +%endif ; ROUND 0 + +;;; === middle AES rounds +%if (%%ROUND >= 1 && %%ROUND <= %%NROUNDS) + ;; rounds 1 to 9/11/13 + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vaesenc, \ + %%L0B0_3, %%L0B4_7, %%L0B8_11, %%L0B12_15, \ + %%L0B0_3, %%L0B4_7, %%L0B8_11, %%L0B12_15, \ + %%KEY, %%KEY, %%KEY, %%KEY +%endif ; rounds 1 to 9/11/13 + +;;; === last AES round +%if (%%ROUND > %%NROUNDS) + ;; the last round - mix enclast with text xor's + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vaesenclast, \ + %%L0B0_3, %%L0B4_7, %%L0B8_11, %%L0B12_15, \ + %%L0B0_3, %%L0B4_7, %%L0B8_11, %%L0B12_15, \ + %%KEY, %%KEY, %%KEY, %%KEY + +;;; === XOR with data +%ifnidn %%D0_3, no_data +%ifnidn %%D4_7, no_data +%ifnidn %%D8_11, no_data +%ifnidn %%D12_15, no_data + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vpxorq, \ + %%L0B0_3, %%L0B4_7, %%L0B8_11, %%L0B12_15, \ + %%L0B0_3, %%L0B4_7, %%L0B8_11, %%L0B12_15, \ + %%D0_3, %%D4_7, %%D8_11, %%D12_15 +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data + +%endif ; The last round + +%endmacro + +;;; =========================================================================== +;;; Handles AES decryption rounds +;;; It handles special cases: the last and first rounds +;;; Optionally, it performs XOR with data after the last AES round. +;;; Uses NROUNDS parameter to check what needs to be done for the current round. +;;; If 3 blocks are trailing then operation on whole ZMM is performed (4 blocks). +%macro ZMM_AESDEC_ROUND_BLOCKS_0_16 12 +%define %%L0B0_3 %1 ; [in/out] zmm; blocks 0 to 3 +%define %%L0B4_7 %2 ; [in/out] zmm; blocks 4 to 7 +%define %%L0B8_11 %3 ; [in/out] zmm; blocks 8 to 11 +%define %%L0B12_15 %4 ; [in/out] zmm; blocks 12 to 15 +%define %%KEY %5 ; [in] zmm containing round key +%define %%ROUND %6 ; [in] round number +%define %%D0_3 %7 ; [in] zmm or no_data; cipher text blocks 0-3 +%define %%D4_7 %8 ; [in] zmm or no_data; cipher text blocks 4-7 +%define %%D8_11 %9 ; [in] zmm or no_data; cipher text blocks 8-11 +%define %%D12_15 %10 ; [in] zmm or no_data; cipher text blocks 12-15 +%define %%NUMBL %11 ; [in] number of blocks; numerical value +%define %%NROUNDS %12 ; [in] number of rounds; numerical value + +;;; === first AES round +%if (%%ROUND < 1) + ;; round 0 + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vpxorq, \ + %%L0B0_3, %%L0B4_7, %%L0B8_11, %%L0B12_15, \ + %%L0B0_3, %%L0B4_7, %%L0B8_11, %%L0B12_15, \ + %%KEY, %%KEY, %%KEY, %%KEY +%endif ; ROUND 0 + +;;; === middle AES rounds +%if (%%ROUND >= 1 && %%ROUND <= %%NROUNDS) + ;; rounds 1 to 9/11/13 + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vaesdec, \ + %%L0B0_3, %%L0B4_7, %%L0B8_11, %%L0B12_15, \ + %%L0B0_3, %%L0B4_7, %%L0B8_11, %%L0B12_15, \ + %%KEY, %%KEY, %%KEY, %%KEY +%endif ; rounds 1 to 9/11/13 + +;;; === last AES round +%if (%%ROUND > %%NROUNDS) + ;; the last round - mix enclast with text xor's + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vaesdeclast, \ + %%L0B0_3, %%L0B4_7, %%L0B8_11, %%L0B12_15, \ + %%L0B0_3, %%L0B4_7, %%L0B8_11, %%L0B12_15, \ + %%KEY, %%KEY, %%KEY, %%KEY + +;;; === XOR with data +%ifnidn %%D0_3, no_data +%ifnidn %%D4_7, no_data +%ifnidn %%D8_11, no_data +%ifnidn %%D12_15, no_data + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vpxorq, \ + %%L0B0_3, %%L0B4_7, %%L0B8_11, %%L0B12_15, \ + %%L0B0_3, %%L0B4_7, %%L0B8_11, %%L0B12_15, \ + %%D0_3, %%D4_7, %%D8_11, %%D12_15 +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data + +%endif ; The last round + +%endmacro + +%endif ;; _AES_COMMON_ASM diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/aes_keyexp_128.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/aes_keyexp_128.asm new file mode 100644 index 0000000..f0a5d19 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/aes_keyexp_128.asm @@ -0,0 +1,103 @@ +;=============================================================================== +; Copyright (C) 2020 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; Routine to do AES key expansion +%include "os.inc" +%define NO_AESNI_RENAME +%include "clear_regs.inc" + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_K0) + +%macro key_expansion_128_avx 0 + ;; Assumes the xmm3 includes all zeros at this point. + vpshufd xmm2, xmm2, 11111111b + vshufps xmm3, xmm3, xmm1, 00010000b + vpxor xmm1, xmm1, xmm3 + vshufps xmm3, xmm3, xmm1, 10001100b + vpxor xmm1, xmm1, xmm3 + vpxor xmm1, xmm1, xmm2 +%endmacro + +%ifdef LINUX +%define KEY rdi +%define EXP_ENC_KEYS rsi +%define EXP_DEC_KEYS rdx +%else +%define KEY rcx +%define EXP_ENC_KEYS rdx +%define EXP_DEC_KEYS r8 +%endif + +section .text + +IPPASM aes_keyexp_128_enc, PUBLIC + + vmovdqu xmm1, [KEY] ; loading the AES key + vmovdqa [EXP_ENC_KEYS + 16*0], xmm1 + vpxor xmm3, xmm3, xmm3 + + vaeskeygenassist xmm2, xmm1, 0x1 ; Generating round key 1 + key_expansion_128_avx + vmovdqa [EXP_ENC_KEYS + 16*1], xmm1 + + vaeskeygenassist xmm2, xmm1, 0x2 ; Generating round key 2 + key_expansion_128_avx + vmovdqa [EXP_ENC_KEYS + 16*2], xmm1 + + vaeskeygenassist xmm2, xmm1, 0x4 ; Generating round key 3 + key_expansion_128_avx + vmovdqa [EXP_ENC_KEYS + 16*3], xmm1 + + vaeskeygenassist xmm2, xmm1, 0x8 ; Generating round key 4 + key_expansion_128_avx + vmovdqa [EXP_ENC_KEYS + 16*4], xmm1 + + vaeskeygenassist xmm2, xmm1, 0x10 ; Generating round key 5 + key_expansion_128_avx + vmovdqa [EXP_ENC_KEYS + 16*5], xmm1 + + vaeskeygenassist xmm2, xmm1, 0x20 ; Generating round key 6 + key_expansion_128_avx + vmovdqa [EXP_ENC_KEYS + 16*6], xmm1 + + vaeskeygenassist xmm2, xmm1, 0x40 ; Generating round key 7 + key_expansion_128_avx + vmovdqa [EXP_ENC_KEYS + 16*7], xmm1 + + vaeskeygenassist xmm2, xmm1, 0x80 ; Generating round key 8 + key_expansion_128_avx + vmovdqa [EXP_ENC_KEYS + 16*8], xmm1 + + vaeskeygenassist xmm2, xmm1, 0x1b ; Generating round key 9 + key_expansion_128_avx + vmovdqa [EXP_ENC_KEYS + 16*9], xmm1 + + vaeskeygenassist xmm2, xmm1, 0x36 ; Generating round key 10 + key_expansion_128_avx + vmovdqa [EXP_ENC_KEYS + 16*10], xmm1 + + clear_scratch_gps_asm + clear_scratch_xmms_avx_asm + + ret + +ENDFUNC aes_keyexp_128_enc + +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/aes_keyexp_192.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/aes_keyexp_192.asm new file mode 100644 index 0000000..b013f54 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/aes_keyexp_192.asm @@ -0,0 +1,128 @@ +;=============================================================================== +; Copyright (C) 2020 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; Routine to do AES key expansion +%include "os.inc" +%define NO_AESNI_RENAME +%include "clear_regs.inc" + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_K0) + +%ifdef LINUX +%define KEY rdi +%define EXP_ENC_KEYS rsi +%define EXP_DEC_KEYS rdx +%else +%define KEY rcx +%define EXP_ENC_KEYS rdx +%define EXP_DEC_KEYS r8 +%endif + +%macro key_expansion_1_192_avx 1 + ;; Assumes the xmm3 includes all zeros at this point. + vpshufd xmm2, xmm2, 11111111b + vshufps xmm3, xmm3, xmm1, 00010000b + vpxor xmm1, xmm1, xmm3 + vshufps xmm3, xmm3, xmm1, 10001100b + vpxor xmm1, xmm1, xmm3 + vpxor xmm1, xmm1, xmm2 + vmovdqu [EXP_ENC_KEYS + %1], xmm1 +%endmacro + +; Calculate w10 and w11 using calculated w9 and known w4-w5 +%macro key_expansion_2_192_avx 1 + vmovdqa xmm5, xmm4 + vpslldq xmm5, xmm5, 4 + vshufps xmm6, xmm6, xmm1, 11110000b + vpxor xmm6, xmm6, xmm5 + vpxor xmm4, xmm4, xmm6 + vpshufd xmm7, xmm4, 00001110b + vmovdqu [EXP_ENC_KEYS + %1], xmm7 +%endmacro + +%macro key_dec_192_avx 1 + vmovdqa xmm0, [EXP_ENC_KEYS + 16 * %1] + vaesimc xmm1, xmm0 + vmovdqa [EXP_DEC_KEYS + 16 * (12 - %1)], xmm1 +%endmacro + +section .text +IPPASM aes_keyexp_192_enc, PUBLIC + +%ifndef LINUX + sub rsp, 16*2 + 8 + vmovdqa [rsp + 0*16], xmm6 + vmovdqa [rsp + 1*16], xmm7 +%endif + + vmovq xmm7, [KEY + 16] ; loading the AES key, 64 bits + vmovq [EXP_ENC_KEYS + 16], xmm7 ; Storing key in memory where all key expansion + vpshufd xmm4, xmm7, 01001111b + vmovdqu xmm1, [KEY] ; loading the AES key, 128 bits + vmovdqu [EXP_ENC_KEYS], xmm1 ; Storing key in memory where all key expansion + + vpxor xmm3, xmm3, xmm3 + vpxor xmm6, xmm6, xmm6 + + vaeskeygenassist xmm2, xmm4, 0x1 ; Complete round key 1 and generate round key 2 + key_expansion_1_192_avx 24 + key_expansion_2_192_avx 40 + + vaeskeygenassist xmm2, xmm4, 0x2 ; Generate round key 3 and part of round key 4 + key_expansion_1_192_avx 48 + key_expansion_2_192_avx 64 + + vaeskeygenassist xmm2, xmm4, 0x4 ; Complete round key 4 and generate round key 5 + key_expansion_1_192_avx 72 + key_expansion_2_192_avx 88 + + vaeskeygenassist xmm2, xmm4, 0x8 ; Generate round key 6 and part of round key 7 + key_expansion_1_192_avx 96 + key_expansion_2_192_avx 112 + + vaeskeygenassist xmm2, xmm4, 0x10 ; Complete round key 7 and generate round key 8 + key_expansion_1_192_avx 120 + key_expansion_2_192_avx 136 + + vaeskeygenassist xmm2, xmm4, 0x20 ; Generate round key 9 and part of round key 10 + key_expansion_1_192_avx 144 + key_expansion_2_192_avx 160 + + vaeskeygenassist xmm2, xmm4, 0x40 ; Complete round key 10 and generate round key 11 + key_expansion_1_192_avx 168 + key_expansion_2_192_avx 184 + + vaeskeygenassist xmm2, xmm4, 0x80 ; Generate round key 12 + key_expansion_1_192_avx 192 + + clear_scratch_gps_asm + clear_scratch_xmms_avx_asm + +%ifndef LINUX + vmovdqa xmm6, [rsp + 0*16] + vmovdqa xmm7, [rsp + 1*16] + add rsp, 16*2 + 8 +%endif + + ret + +ENDFUNC aes_keyexp_192_enc + +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/aes_keyexp_256.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/aes_keyexp_256.asm new file mode 100644 index 0000000..b66340a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/aes_keyexp_256.asm @@ -0,0 +1,131 @@ +;=============================================================================== +; Copyright (C) 2020 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; Routine to do AES key expansion +%include "os.inc" +%define NO_AESNI_RENAME +%include "clear_regs.inc" + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_K0) + +; Uses the f() function of the aeskeygenassist result +%macro key_expansion_256_avx 0 + ;; Assumes the xmm3 includes all zeros at this point. + vpshufd xmm2, xmm2, 11111111b + vshufps xmm3, xmm3, xmm1, 00010000b + vpxor xmm1, xmm1, xmm3 + vshufps xmm3, xmm3, xmm1, 10001100b + vpxor xmm1, xmm1, xmm3 + vpxor xmm1, xmm1, xmm2 +%endmacro + +; Uses the SubWord function of the aeskeygenassist result +%macro key_expansion_256_avx_2 0 + ;; Assumes the xmm3 includes all zeros at this point. + vpshufd xmm2, xmm2, 10101010b + vshufps xmm3, xmm3, xmm4, 00010000b + vpxor xmm4, xmm4, xmm3 + vshufps xmm3, xmm3, xmm4, 10001100b + vpxor xmm4, xmm4, xmm3 + vpxor xmm4, xmm4, xmm2 +%endmacro + +%ifdef LINUX +%define KEY rdi +%define EXP_ENC_KEYS rsi +%define EXP_DEC_KEYS rdx +%else +%define KEY rcx +%define EXP_ENC_KEYS rdx +%define EXP_DEC_KEYS r8 +%endif + +section .text + +IPPASM aes_keyexp_256_enc, PUBLIC + + vmovdqu xmm1, [KEY] ; loading the AES key + vmovdqa [EXP_ENC_KEYS + 16*0], xmm1 + + vmovdqu xmm4, [KEY+16] ; loading the AES key + vmovdqa [EXP_ENC_KEYS + 16*1], xmm4 + + vpxor xmm3, xmm3, xmm3 ; Required for the key_expansion. + + vaeskeygenassist xmm2, xmm4, 0x1 ; Generating round key 2 + key_expansion_256_avx + vmovdqa [EXP_ENC_KEYS + 16*2], xmm1 + + vaeskeygenassist xmm2, xmm1, 0x1 ; Generating round key 3 + key_expansion_256_avx_2 + vmovdqa [EXP_ENC_KEYS + 16*3], xmm4 + + vaeskeygenassist xmm2, xmm4, 0x2 ; Generating round key 4 + key_expansion_256_avx + vmovdqa [EXP_ENC_KEYS + 16*4], xmm1 + + vaeskeygenassist xmm2, xmm1, 0x2 ; Generating round key 5 + key_expansion_256_avx_2 + vmovdqa [EXP_ENC_KEYS + 16*5], xmm4 + + vaeskeygenassist xmm2, xmm4, 0x4 ; Generating round key 6 + key_expansion_256_avx + vmovdqa [EXP_ENC_KEYS + 16*6], xmm1 + + vaeskeygenassist xmm2, xmm1, 0x4 ; Generating round key 7 + key_expansion_256_avx_2 + vmovdqa [EXP_ENC_KEYS + 16*7], xmm4 + + vaeskeygenassist xmm2, xmm4, 0x8 ; Generating round key 8 + key_expansion_256_avx + vmovdqa [EXP_ENC_KEYS + 16*8], xmm1 + + vaeskeygenassist xmm2, xmm1, 0x8 ; Generating round key 9 + key_expansion_256_avx_2 + vmovdqa [EXP_ENC_KEYS + 16*9], xmm4 + + vaeskeygenassist xmm2, xmm4, 0x10 ; Generating round key 10 + key_expansion_256_avx + vmovdqa [EXP_ENC_KEYS + 16*10], xmm1 + + vaeskeygenassist xmm2, xmm1, 0x10 ; Generating round key 11 + key_expansion_256_avx_2 + vmovdqa [EXP_ENC_KEYS + 16*11], xmm4 + + vaeskeygenassist xmm2, xmm4, 0x20 ; Generating round key 12 + key_expansion_256_avx + vmovdqa [EXP_ENC_KEYS + 16*12], xmm1 + + vaeskeygenassist xmm2, xmm1, 0x20 ; Generating round key 13 + key_expansion_256_avx_2 + vmovdqa [EXP_ENC_KEYS + 16*13], xmm4 + + vaeskeygenassist xmm2, xmm4, 0x40 ; Generating round key 14 + key_expansion_256_avx + vmovdqa [EXP_ENC_KEYS + 16*14], xmm1 + + clear_scratch_gps_asm + clear_scratch_xmms_avx_asm + + ret + +ENDFUNC aes_keyexp_256_enc + +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/clear_regs.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/clear_regs.inc new file mode 100644 index 0000000..99dffc2 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/clear_regs.inc @@ -0,0 +1,186 @@ +;=============================================================================== +; Copyright (C) 2020 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +%ifndef _CLEAR_REGS_ASM_ +%define _CLEAR_REGS_ASM_ + +%include "os.inc" + +; +; This macro clears any GP registers passed +; +%macro clear_gps 1-16 +%define %%NUM_REGS %0 +%rep %%NUM_REGS + xor %1, %1 +%rotate 1 +%endrep +%endmacro + +; +; This macro clears any XMM registers passed on SSE +; +%macro clear_xmms_sse 1-16 +%define %%NUM_REGS %0 +%rep %%NUM_REGS + pxor %1, %1 +%rotate 1 +%endrep +%endmacro + +; +; This macro clears any XMM registers passed on AVX +; +%macro clear_xmms_avx 1-16 +%define %%NUM_REGS %0 +%rep %%NUM_REGS + vpxor %1, %1 +%rotate 1 +%endrep +%endmacro + +; +; This macro clears any YMM registers passed +; +%macro clear_ymms 1-16 +%define %%NUM_REGS %0 +%rep %%NUM_REGS + vpxor %1, %1 +%rotate 1 +%endrep +%endmacro + +; +; This macro clears any ZMM registers passed +; +%macro clear_zmms 1-32 +%define %%NUM_REGS %0 +%rep %%NUM_REGS + vpxorq %1, %1 +%rotate 1 +%endrep +%endmacro + +; +; This macro clears all scratch GP registers +; for Windows or Linux +; +%macro clear_scratch_gps_asm 0 + clear_gps rax, rcx, rdx, r8, r9, r10, r11 +%ifdef LINUX + clear_gps rdi, rsi +%endif +%endmacro + +; +; This macro clears all scratch XMM registers on SSE +; +%macro clear_scratch_xmms_sse_asm 0 +%ifdef LINUX +%assign i 0 +%rep 16 + pxor xmm %+ i, xmm %+ i +%assign i (i+1) +%endrep +; On Windows, XMM0-XMM5 registers are scratch registers +%else +%assign i 0 +%rep 6 + pxor xmm %+ i, xmm %+ i +%assign i (i+1) +%endrep +%endif ; LINUX +%endmacro + +; +; This macro clears all scratch XMM registers on AVX +; +%macro clear_scratch_xmms_avx_asm 0 +%ifdef LINUX + vzeroall +; On Windows, XMM0-XMM5 registers are scratch registers +%else +%assign i 0 +%rep 6 + vpxor xmm %+ i, xmm %+ i +%assign i (i+1) +%endrep +%endif ; LINUX +%endmacro + +; +; This macro clears all scratch YMM registers +; +; It should be called before restoring the XMM registers +; for Windows (XMM6-XMM15) +; +%macro clear_scratch_ymms_asm 0 +; On Linux, all YMM registers are scratch registers +%ifdef LINUX + vzeroall +; On Windows, YMM0-YMM5 registers are scratch registers. +; YMM6-YMM15 upper 128 bits are scratch registers too, but +; the lower 128 bits are to be restored after calling these function +; which clears the upper bits too. +%else +%assign i 0 +%rep 6 + vpxor ymm %+ i, ymm %+ i +%assign i (i+1) +%endrep +%endif ; LINUX +%endmacro + +; +; This macro clears all scratch ZMM registers +; +; It should be called before restoring the XMM registers +; for Windows (XMM6-XMM15). YMM registers are used +; on purpose, since XOR'ing YMM registers is faster +; than XOR'ing ZMM registers, and the operation clears +; also the upper 256 bits +; +%macro clear_scratch_zmms_asm 0 +; On Linux, all ZMM registers are scratch registers +%ifdef LINUX + vzeroall + ;; vzeroall only clears the first 16 ZMM registers +%assign i 16 +%rep 16 + vpxorq ymm %+ i, ymm %+ i +%assign i (i+1) +%endrep +; On Windows, ZMM0-ZMM5 and ZMM16-ZMM31 registers are scratch registers. +; ZMM6-ZMM15 upper 384 bits are scratch registers too, but +; the lower 128 bits are to be restored after calling these function +; which clears the upper bits too. +%else +%assign i 0 +%rep 6 + vpxorq ymm %+ i, ymm %+ i +%assign i (i+1) +%endrep + +%assign i 16 +%rep 16 + vpxorq ymm %+ i, ymm %+ i +%assign i (i+1) +%endrep +%endif ; LINUX +%endmacro + +%endif ;; _CLEAR_REGS_ASM diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/cpinitas.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/cpinitas.asm new file mode 100644 index 0000000..33ea996 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/cpinitas.asm @@ -0,0 +1,320 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%assign LOCAL_ALIGN_FACTOR 32 + +%ifdef _IPP_DATA + +segment .text align=LOCAL_ALIGN_FACTOR + +;#################################################################### +;# void cpGetReg( int* buf, int valueEAX, int valueECX ); # +;#################################################################### + +%ifdef WIN32E + %define buf rcx + %define valueEAX edx + %define valueECX r8d +%else + %define buf rdi + %define valueEAX esi + %define valueECX edx +%endif + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC cpGetReg,PUBLIC + push rbx + movsxd r9, valueEAX + movsxd r10, valueECX + mov r11, buf + + mov rax, r9 + mov rcx, r10 + xor ebx, ebx + xor edx, edx + cpuid + mov [r11], eax + mov [r11 + 4], ebx + mov [r11 + 8], ecx + mov [r11 + 12], edx + pop rbx + ret +ENDFUNC cpGetReg + +;################################################### + +; OSXSAVE support, feature information after cpuid(1), ECX, bit 27 ( XGETBV is enabled by OS ) +%assign XSAVEXGETBV_FLAG 8000000h + +; Feature information after XGETBV(ECX=0), EAX, bits 2,1 ( XMM state and YMM state are enabled by OS ) +%assign XGETBV_MASK 06h + +%assign XGETBV_AVX512_MASK 0E0h + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC cp_is_avx_extension,PUBLIC + push rbx + mov eax, 1 + cpuid + xor eax, eax + and ecx, 018000000h + cmp ecx, 018000000h + jne .not_avx + xor ecx, ecx + db 00fh,001h,0d0h ; xgetbv + mov ecx, eax + xor eax, eax + and ecx, XGETBV_MASK + cmp ecx, XGETBV_MASK + jne .not_avx + mov eax, 1 +.not_avx: + pop rbx + ret +ENDFUNC cp_is_avx_extension + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC cp_is_avx512_extension,PUBLIC + push rbx + mov eax, 1 + cpuid + xor eax, eax + and ecx, XSAVEXGETBV_FLAG + cmp ecx, XSAVEXGETBV_FLAG + jne .not_avx512 + xor ecx, ecx + db 00fh,001h,0d0h ; xgetbv + mov ecx, eax + xor eax, eax + and ecx, XGETBV_AVX512_MASK + cmp ecx, XGETBV_AVX512_MASK + jne .not_avx512 + mov eax, 1 +.not_avx512: + pop rbx + ret +ENDFUNC cp_is_avx512_extension + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC cp_issue_avx512_instruction,PUBLIC + db 062h,0f1h,07dh,048h,0efh,0c0h ; vpxord zmm0, zmm0, zmm0 + xor eax, eax + ret +ENDFUNC cp_issue_avx512_instruction + +%ifdef OSXEM64T + extern _ippcpInit +%else + extern ippcpInit +%endif + +;#################################################################### +;# void ippSafeInit( ); # +;#################################################################### + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC ippcpSafeInit,PUBLIC + push rcx + push rdx +%ifdef LINUX32E + push rdi + push rsi +%endif + push r8 + push r9 +%ifdef LINUX32E + %ifdef OSXEM64T + call _ippcpInit + %else + %ifdef IPP_PIC + call ippcpInit wrt ..plt + %else + call ippcpInit + %endif + %endif +%else + call ippcpInit +%endif + pop r9 + pop r8 +%ifdef LINUX32E + pop rsi + pop rdi +%endif + pop rdx + pop rcx + ret +ENDFUNC ippcpSafeInit + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC cp_get_pentium_counter,PUBLIC + rdtsc + sal rdx,32 + or rax,rdx + ret +ENDFUNC cp_get_pentium_counter + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC cpStartTscp,PUBLIC + push rbx + xor rax, rax + cpuid + pop rbx + rdtscp + sal rdx,32 + or rax,rdx + ret +ENDFUNC cpStartTscp + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC cpStopTscp,PUBLIC + rdtscp + sal rdx,32 + or rax,rdx + push rax + push rbx + xor rax, rax + cpuid + pop rbx + pop rax + ret +ENDFUNC cpStopTscp + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC cpStartTsc,PUBLIC + push rbx + xor rax, rax + cpuid + pop rbx + rdtsc + sal rdx,32 + or rax,rdx + ret +ENDFUNC cpStartTsc + +align LOCAL_ALIGN_FACTOR +DECLARE_FUNC cpStopTsc,PUBLIC + rdtsc + sal rdx,32 + or rax,rdx + push rax + push rbx + xor rax, rax + cpuid + pop rbx + pop rax + ret +ENDFUNC cpStopTsc + + +;***************************************** +; int cpGetCacheSize( int* tableCache ); +align LOCAL_ALIGN_FACTOR +%define table rdi +DECLARE_FUNC cpGetCacheSize,PUBLIC +%assign LOCAL_FRAME 16 + USES_GPR rsi, rdi, rbx, rbp + USES_XMM + COMP_ABI 1 + + mov rbp, rsp + xor esi, esi + + mov eax, 2 + cpuid + + cmp al, 1 + jne .GetCacheSize_11 + + test eax, 080000000h + jz .GetCacheSize_00 + xor eax, eax +.GetCacheSize_00: + test ebx, 080000000h + jz .GetCacheSize_01 + xor ebx, ebx +.GetCacheSize_01: + test ecx, 080000000h + jz .GetCacheSize_02 + xor ecx, ecx +.GetCacheSize_02: + test edx, 080000000h + jz .GetCacheSize_03 + xor edx, edx + +.GetCacheSize_03: + test eax, eax + jz .GetCacheSize_04 + mov [rbp], eax + add rbp, 4 + add esi, 3 +.GetCacheSize_04: + test ebx, ebx + jz .GetCacheSize_05 + mov [rbp], ebx + add rbp, 4 + add esi, 4 +.GetCacheSize_05: + test ecx, ecx + jz .GetCacheSize_06 + mov [rbp], ecx + add rbp, 4 + add esi, 4 +.GetCacheSize_06: + test edx, edx + jz .GetCacheSize_07 + mov [rbp], edx + add esi, 4 + +.GetCacheSize_07: + test esi, esi + jz .GetCacheSize_11 + mov eax, -1 +.GetCacheSize_08: + xor edx, edx + add edx, [table] + jz .ExitGetCacheSize00 + add table, 8 + mov ecx, esi +.GetCacheSize_09: + cmp dl, BYTE [rsp + rcx] + je .GetCacheSize_10 + dec ecx + jnz .GetCacheSize_09 + jmp .GetCacheSize_08 + +.GetCacheSize_10: + mov eax, [table - 4] + +.ExitGetCacheSize00: + REST_XMM + REST_GPR + ret + +.GetCacheSize_11: + mov eax, -1 + jmp .ExitGetCacheSize00 +ENDFUNC cpGetCacheSize + +;**************************** + +%endif ; _IPP_DATA diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/emulator.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/emulator.inc new file mode 100644 index 0000000..e0eeebf --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/emulator.inc @@ -0,0 +1,340 @@ +;=============================================================================== +; Copyright (C) 2009 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: EM64T Cryptography Primitive. +; +; +; + +%ifndef _EMULATOR_INC_ +%define _EMULATOR_INC_ + +%macro my_pclmulqdq 3.nolist + %xdefine %%xxDst %1 + %xdefine %%xxSrc %2 + %xdefine %%xxOp %3 + + %if (my_emulator == 0) + pclmulqdq %%xxDst, %%xxSrc, %%xxOp + %else +;; +;; rsp +;; registers +;; +00 => xxDst +;; +16 => xxSrc + + pushf + push rax + push rbx + push rcx + push rdx + push rdi + push rsi + push rbp + push r8 + push r9 + push r10 + push r11 + push r12 + push r13 + push r14 + push r15 + + %assign %%stackSize (sizeof(oword)*2) + sub rsp,%%stackSize + + movdqu oword [rsp+00], %%xxDst ;; save Dst + movdqu oword [rsp+16], %%xxSrc ;; save Src + + lea rcx, [rsp+00] + lea rdx, [rsp+16] + mov r8, %%xxOp + + sub rsp, (sizeof(qword)*3) + call emu_pclmulqdq + add rsp, (sizeof(qword)*3) + + movdqu %%xxDst, oword [rsp+00] ;; return Dst + ;movdqu xxSrc, oword [rsp+16] ;; return Src + add esp, %%stackSize + + pop r15 + pop r14 + pop r13 + pop r12 + pop r11 + pop r10 + pop r9 + pop r8 + pop rbp + pop rsi + pop rdi + pop rdx + pop rcx + pop rbx + pop rax + popf + %endif +%endmacro + +%macro my_aesenc 2.nolist + %xdefine %%xxDst %1 + %xdefine %%xxSrc %2 + + %if (my_emulator == 0) + aesenc %%xxDst, %%xxSrc + %else + pushf + push rax + push rbx + push rcx + push rdx + push rdi + push rsi + push rbp + push r8 + push r9 + push r10 + push r11 + push r12 + push r13 + push r14 + push r15 + + %assign %%stackSize (sizeof(oword)*2) + sub rsp,%%stackSize + + movdqu oword [rsp+00], %%xxDst ;; save Dst + movdqu oword [rsp+16], %%xxSrc ;; save Src + + lea rcx, [rsp+00] + lea rdx, [rsp+16] + + sub rsp, (sizeof(qword)*2) + call emu_aesenc + add rsp, (sizeof(qword)*2) + + movdqu %%xxDst, oword [rsp+00] ;; return Dst + add esp, %%stackSize + + pop r15 + pop r14 + pop r13 + pop r12 + pop r11 + pop r10 + pop r9 + pop r8 + pop rbp + pop rsi + pop rdi + pop rdx + pop rcx + pop rbx + pop rax + popf + %endif +%endmacro + +%macro my_aesenclast 2.nolist + %xdefine %%xxDst %1 + %xdefine %%xxSrc %2 + + %if (my_emulator == 0) + aesenclast %%xxDst, %%xxSrc + %else + pushf + push rax + push rbx + push rcx + push rdx + push rdi + push rsi + push rbp + push r8 + push r9 + push r10 + push r11 + push r12 + push r13 + push r14 + push r15 + + %assign %%stackSize (sizeof(oword)*2) + sub rsp,%%stackSize + + movdqu oword [rsp+00], %%xxDst ;; save Dst + movdqu oword [rsp+16], %%xxSrc ;; save Src + + lea rcx, [rsp+00] + lea rdx, [rsp+16] + + sub rsp, (sizeof(qword)*2) + call emu_aesenclast + add rsp, (sizeof(qword)*2) + + movdqu %%xxDst, oword [rsp+00] ;; return Dst + add esp, %%stackSize + + pop r15 + pop r14 + pop r13 + pop r12 + pop r11 + pop r10 + pop r9 + pop r8 + pop rbp + pop rsi + pop rdi + pop rdx + pop rcx + pop rbx + pop rax + popf + %endif +%endmacro + +%macro my_aesdec 2.nolist + %xdefine %%xxDst %1 + %xdefine %%xxSrc %2 + + %if (my_emulator == 0) + aesdec %%xxDst, %%xxSrc + %else + pushf + push rax + push rbx + push rcx + push rdx + push rdi + push rsi + push rbp + push r8 + push r9 + push r10 + push r11 + push r12 + push r13 + push r14 + push r15 + + %assign %%stackSize (sizeof(oword)*2) + sub rsp,%%stackSize + + movdqu oword [rsp+00], %%xxDst ;; save Dst + movdqu oword [rsp+16], %%xxSrc ;; save Src + + lea rcx, [rsp+00] + lea rdx, [rsp+16] + + sub rsp, (sizeof(qword)*2) + call emu_aesdec + add rsp, (sizeof(qword)*2) + + movdqu %%xxDst, oword [rsp+00] ;; return Dst + add esp, %%stackSize + + pop r15 + pop r14 + pop r13 + pop r12 + pop r11 + pop r10 + pop r9 + pop r8 + pop rbp + pop rsi + pop rdi + pop rdx + pop rcx + pop rbx + pop rax + popf + %endif +%endmacro + +%macro my_aesdeclast 2.nolist + %xdefine %%xxDst %1 + %xdefine %%xxSrc %2 + + %if (my_emulator == 0) + aesenclast %%xxDst, %%xxSrc + %else + pushf + push rax + push rbx + push rcx + push rdx + push rdi + push rsi + push rbp + push r8 + push r9 + push r10 + push r11 + push r12 + push r13 + push r14 + push r15 + + %assign %%stackSize (sizeof(oword)*2) + sub rsp,%%stackSize + + movdqu oword [rsp+00], %%xxDst ;; save Dst + movdqu oword [rsp+16], %%xxSrc ;; save Src + + lea rcx, [rsp+00] + lea rdx, [rsp+16] + + sub rsp, (sizeof(qword)*2) + call emu_aesdeclast + add rsp, (sizeof(qword)*2) + + movdqu %%xxDst, oword [rsp+00] ;; return Dst + add esp, %%stackSize + + pop r15 + pop r14 + pop r13 + pop r12 + pop r11 + pop r10 + pop r9 + pop r8 + pop rbp + pop rsi + pop rdi + pop rdx + pop rcx + pop rbx + pop rax + popf + %endif +%endmacro + +%if (my_emulator != 0) + extern emu_pclmulqdq + extern emu_aesenc + extern emu_aesenclast + extern emu_aesdec + extern emu_aesdeclast +%endif + +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm128_api_vaes_avx512.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm128_api_vaes_avx512.asm new file mode 100644 index 0000000..031cc5a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm128_api_vaes_avx512.asm @@ -0,0 +1,27 @@ +;=============================================================================== +; Copyright (C) 2020 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_K0) + +%define GCM128_MODE 1 +%include "gcm_api_vaes_avx512.inc" +%include "gcm_ippcp_api_vaes_avx512.inc" + +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm128_avx512.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm128_avx512.asm new file mode 100644 index 0000000..1a5f4fb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm128_avx512.asm @@ -0,0 +1,26 @@ +;=============================================================================== +; Copyright (C) 2020 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_K0) + +%define GCM128_MODE 1 +%include "gcm_avx512.inc" + +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm192_api_vaes_avx512.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm192_api_vaes_avx512.asm new file mode 100644 index 0000000..4ba4841 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm192_api_vaes_avx512.asm @@ -0,0 +1,27 @@ +;=============================================================================== +; Copyright (C) 2020 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_K0) + +%define GCM192_MODE 1 +;; single buffer implementation +%include "gcm_api_vaes_avx512.inc" + +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm192_avx512.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm192_avx512.asm new file mode 100644 index 0000000..82c645f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm192_avx512.asm @@ -0,0 +1,26 @@ +;=============================================================================== +; Copyright (C) 2020 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_K0) + +%define GCM192_MODE 1 +%include "gcm_avx512.inc" + +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm256_api_vaes_avx512.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm256_api_vaes_avx512.asm new file mode 100644 index 0000000..5f23906 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm256_api_vaes_avx512.asm @@ -0,0 +1,27 @@ +;=============================================================================== +; Copyright (C) 2020 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_K0) + +%define GCM256_MODE 1 +;; single buffer implementation +%include "gcm_api_vaes_avx512.inc" + +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm256_avx512.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm256_avx512.asm new file mode 100644 index 0000000..8abcd55 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm256_avx512.asm @@ -0,0 +1,26 @@ +;=============================================================================== +; Copyright (C) 2020 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_K0) + +%define GCM256_MODE 1 +%include "gcm_avx512.inc" + +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm_api_vaes_avx512.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm_api_vaes_avx512.inc new file mode 100644 index 0000000..b943ea3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm_api_vaes_avx512.inc @@ -0,0 +1,203 @@ +;=============================================================================== +; Copyright (C) 2020 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +%ifndef _GCM_API_VAES_AVX512_INC_ +%define _GCM_API_VAES_AVX512_INC_ + +%include "gcm_vaes_avx512.inc" + +section .text +default rel + +;; Safe param check is disabled as parameters checking is done outside. +%ifdef SAFE_PARAM +%undef SAFE_PARAM +%endif + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_precomp_128_vaes_avx512 / +; aes_gcm_precomp_192_vaes_avx512 / +; aes_gcm_precomp_256_vaes_avx512 +; (struct gcm_key_data *key_data) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +IPPASM FN_NAME(precomp,_), PUBLIC +;; Parameter is passed through register +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz .exit_precomp +%endif + + FUNC_SAVE + + vpxor xmm6, xmm6 + ENCRYPT_SINGLE_BLOCK arg1, xmm6 ; xmm6 = HashKey + + vpshufb xmm6, [rel SHUF_MASK] + ;;;;;;;;;;;;;;; PRECOMPUTATION of HashKey<<1 mod poly from the HashKey;;;;;;;;;;;;;;; + vmovdqa xmm2, xmm6 + vpsllq xmm6, xmm6, 1 + vpsrlq xmm2, xmm2, 63 + vmovdqa xmm1, xmm2 + vpslldq xmm2, xmm2, 8 + vpsrldq xmm1, xmm1, 8 + vpor xmm6, xmm6, xmm2 + ;reduction + vpshufd xmm2, xmm1, 00100100b + vpcmpeqd xmm2, [rel TWOONE] + vpand xmm2, xmm2, [rel POLY] + vpxor xmm6, xmm6, xmm2 ; xmm6 holds the HashKey<<1 mod poly + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vmovdqu [arg1 + HashKey], xmm6 ; store HashKey<<1 mod poly + + + PRECOMPUTE arg1, xmm6, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm7, xmm8 + + FUNC_RESTORE +.exit_precomp: + + ret + +ENDFUNC FN_NAME(precomp,_) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_enc_128_update_vaes_avx512 / aes_gcm_enc_192_update_vaes_avx512 / +; aes_gcm_enc_256_update_vaes_avx512 +; (const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; u8 *out, +; const u8 *in, +; u64 plaintext_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +IPPASM FN_NAME(enc,_update_), PUBLIC + + FUNC_SAVE + +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz .exit_update_enc + + ;; Check context_data != NULL + cmp arg2, 0 + jz .exit_update_enc + + ;; Check if plaintext_len == 0 + cmp arg5, 0 + jz .exit_update_enc + + ;; Check out != NULL (plaintext_len != 0) + cmp arg3, 0 + jz .exit_update_enc + + ;; Check in != NULL (plaintext_len != 0) + cmp arg4, 0 + jz .exit_update_enc +%endif + GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, ENC, multi_call + +.exit_update_enc: + FUNC_RESTORE + ret + +ENDFUNC FN_NAME(enc,_update_) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_dec_128_update_vaes_avx512 / aes_gcm_dec_192_update_vaes_avx512 / +; aes_gcm_dec_256_update_vaes_avx512 +; (const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; u8 *out, +; const u8 *in, +; u64 plaintext_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +IPPASM FN_NAME(dec,_update_), PUBLIC + FUNC_SAVE + +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz .exit_update_dec + + ;; Check context_data != NULL + cmp arg2, 0 + jz .exit_update_dec + + ;; Check if plaintext_len == 0 + cmp arg5, 0 + jz .exit_update_dec + + ;; Check out != NULL (plaintext_len != 0) + cmp arg3, 0 + jz .exit_update_dec + + ;; Check in != NULL (plaintext_len != 0) + cmp arg4, 0 + jz .exit_update_dec +%endif + + GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, DEC, multi_call + +.exit_update_dec: + FUNC_RESTORE + ret + +ENDFUNC FN_NAME(dec,_update_) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_gettag_128_vaes_avx512 / aes_gcm_gettag_192_vaes_avx512 / +; aes_gcm_gettag_256_vaes_avx512 +; (const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; u8 *auth_tag, +; u64 auth_tag_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +IPPASM FN_NAME(gettag,_), PUBLIC + +;; All parameters are passed through registers +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz .exit_enc_fin + + ;; Check context_data != NULL + cmp arg2, 0 + jz .exit_enc_fin + + ;; Check auth_tag != NULL + cmp arg3, 0 + jz .exit_enc_fin + + ;; Check auth_tag_len == 0 or > 16 + cmp arg4, 0 + jz .exit_enc_fin + + cmp arg4, 16 + ja .exit_enc_fin +%endif + + FUNC_SAVE + GCM_COMPLETE arg1, arg2, arg3, arg4, multi_call, k1, r10, r11, r12 + + FUNC_RESTORE + +.exit_enc_fin: + ret + +ENDFUNC FN_NAME(gettag,_) + +%endif ; _GCM_API_VAES_AVX512_INC_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm_avx512.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm_avx512.inc new file mode 100644 index 0000000..b49cc66 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm_avx512.inc @@ -0,0 +1,3591 @@ +;=============================================================================== +; Copyright (C) 2020 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; Authors: +; Erdinc Ozturk +; Vinodh Gopal +; James Guilford +; Tomasz Kantecki +; +; +; References: +; This code was derived and highly optimized from the code described in paper: +; Vinodh Gopal et. al. Optimized Galois-Counter-Mode Implementation on Intel Architecture Processors. August, 2010 +; The details of the implementation is explained in: +; Erdinc Ozturk et. al. Enabling High-Performance Galois-Counter-Mode on Intel Architecture Processors. October, 2012. +; +; +; +; +; Assumptions: +; +; +; +; iv: +; 0 1 2 3 +; 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | Salt (From the SA) | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | Initialization Vector | +; | (This is the sequence number from IPSec header) | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | 0x1 | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; +; +; +; AAD: +; AAD will be padded with 0 to the next 16byte multiple +; for example, assume AAD is a u32 vector +; +; if AAD is 8 bytes: +; AAD[3] = {A0, A1}; +; padded AAD in xmm register = {A1 A0 0 0} +; +; 0 1 2 3 +; 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | SPI (A1) | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | 32-bit Sequence Number (A0) | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | 0x0 | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; +; AAD Format with 32-bit Sequence Number +; +; if AAD is 12 bytes: +; AAD[3] = {A0, A1, A2}; +; padded AAD in xmm register = {A2 A1 A0 0} +; +; 0 1 2 3 +; 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | SPI (A2) | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | 64-bit Extended Sequence Number {A1,A0} | +; | | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | 0x0 | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; +; AAD Format with 64-bit Extended Sequence Number +; +; +; aadLen: +; Must be a multiple of 4 bytes and from the definition of the spec. +; The code additionally supports any aadLen length. +; +; TLen: +; from the definition of the spec, TLen can only be 8, 12 or 16 bytes. +; +; poly = x^128 + x^127 + x^126 + x^121 + 1 +; throughout the code, one tab and two tab indentations are used. one tab is for GHASH part, two tabs is for AES part. +; + +%ifndef _GCM_AVX512_INC_ +%define _GCM_AVX512_INC_ + +%include "os.inc" +%include "reg_sizes.inc" +%include "clear_regs.inc" +%include "gcm_defines.inc" +%include "gcm_keys_avx512.inc" +%include "memcpy.inc" +%include "aes_common.inc" + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%ifndef GCM128_MODE +%ifndef GCM192_MODE +%ifndef GCM256_MODE +%error "No GCM mode selected for gcm_avx512.asm!" +%endif +%endif +%endif + +;; Decide on AES-GCM key size to compile for +%ifdef GCM128_MODE +%define NROUNDS 9 +%define FN_NAME(x,y) aes_gcm_ %+ x %+ _128 %+ y %+ avx512 +%endif + +%ifdef GCM192_MODE +%define NROUNDS 11 +%define FN_NAME(x,y) aes_gcm_ %+ x %+ _192 %+ y %+ avx512 +%endif + +%ifdef GCM256_MODE +%define NROUNDS 13 +%define FN_NAME(x,y) aes_gcm_ %+ x %+ _256 %+ y %+ avx512 +%endif + +section .text +default rel + +; need to push 4 registers into stack to maintain +%define STACK_OFFSET 8*4 + +%ifidn __OUTPUT_FORMAT__, win64 + %define XMM_STORAGE 16*10 +%else + %define XMM_STORAGE 0 +%endif + +%define TMP2 16*0 ; Temporary storage for AES State 2 (State 1 is stored in an XMM register) +%define TMP3 16*1 ; Temporary storage for AES State 3 +%define TMP4 16*2 ; Temporary storage for AES State 4 +%define TMP5 16*3 ; Temporary storage for AES State 5 +%define TMP6 16*4 ; Temporary storage for AES State 6 +%define TMP7 16*5 ; Temporary storage for AES State 7 +%define TMP8 16*6 ; Temporary storage for AES State 8 +%define LOCAL_STORAGE 16*7 +%define VARIABLE_OFFSET LOCAL_STORAGE + XMM_STORAGE + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Utility Macros +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0) +; Input: A and B (128-bits each, bit-reflected) +; Output: C = A*B*x mod poly, (i.e. >>1 ) +; To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input +; GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro GHASH_MUL 7 +%define %%GH %1 ; 16 Bytes +%define %%HK %2 ; 16 Bytes +%define %%T1 %3 +%define %%T2 %4 +%define %%T3 %5 +%define %%T4 %6 +%define %%T5 %7 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + vpclmulqdq %%T1, %%GH, %%HK, 0x11 ; %%T1 = a1*b1 + vpclmulqdq %%T2, %%GH, %%HK, 0x00 ; %%T2 = a0*b0 + vpclmulqdq %%T3, %%GH, %%HK, 0x01 ; %%T3 = a1*b0 + vpclmulqdq %%GH, %%GH, %%HK, 0x10 ; %%GH = a0*b1 + vpxor %%GH, %%GH, %%T3 + + + vpsrldq %%T3, %%GH, 8 ; shift-R %%GH 2 DWs + vpslldq %%GH, %%GH, 8 ; shift-L %%GH 2 DWs + + vpxor %%T1, %%T1, %%T3 + vpxor %%GH, %%GH, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;first phase of the reduction + vmovdqu %%T3, [rel POLY2] + + vpclmulqdq %%T2, %%T3, %%GH, 0x01 + vpslldq %%T2, %%T2, 8 ; shift-L %%T2 2 DWs + + vpxor %%GH, %%GH, %%T2 ; first phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;second phase of the reduction + vpclmulqdq %%T2, %%T3, %%GH, 0x00 + vpsrldq %%T2, %%T2, 4 ; shift-R %%T2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) + + vpclmulqdq %%GH, %%T3, %%GH, 0x10 + vpslldq %%GH, %%GH, 4 ; shift-L %%GH 1 DW (Shift-L 1-DW to obtain result with no shifts) + + vpxor %%GH, %%GH, %%T2 ; second phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vpxor %%GH, %%GH, %%T1 ; the result is in %%GH +%endmacro + + +; In PRECOMPUTE, the commands filling Hashkey_i_k are not required for avx512 +; functions, but are kept to allow users to switch cpu architectures between calls +; of pre, init, update, and finalize. +%macro PRECOMPUTE 8 +%define %%GDATA %1 +%define %%HK %2 +%define %%T1 %3 +%define %%T2 %4 +%define %%T3 %5 +%define %%T4 %6 +%define %%T5 %7 +%define %%T6 %8 + + ; Haskey_i_k holds XORed values of the low and high parts of the Haskey_i + vmovdqa %%T5, %%HK + + GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^2<<1 mod poly + vmovdqu [%%GDATA + HashKey_2], %%T5 ; [HashKey_2] = HashKey^2<<1 mod poly + + GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^3<<1 mod poly + vmovdqu [%%GDATA + HashKey_3], %%T5 + + GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^4<<1 mod poly + vmovdqu [%%GDATA + HashKey_4], %%T5 + + GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^5<<1 mod poly + vmovdqu [%%GDATA + HashKey_5], %%T5 + + GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^6<<1 mod poly + vmovdqu [%%GDATA + HashKey_6], %%T5 + + GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^7<<1 mod poly + vmovdqu [%%GDATA + HashKey_7], %%T5 + + GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^8<<1 mod poly + vmovdqu [%%GDATA + HashKey_8], %%T5 +%endmacro + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; READ_SMALL_DATA_INPUT: Packs xmm register with data when data input is less than 16 bytes. +; Returns 0 if data has length 0. +; Input: The input data (INPUT), that data's length (LENGTH). +; Output: The packed xmm register (OUTPUT). +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro READ_SMALL_DATA_INPUT 4 +%define %%OUTPUT %1 ; %%OUTPUT is an xmm register +%define %%INPUT %2 +%define %%LENGTH %3 +%define %%TMP1 %4 + + lea %%TMP1, [rel byte_len_to_mask_table] +%ifidn __OUTPUT_FORMAT__, win64 + add %%TMP1, %%LENGTH + add %%TMP1, %%LENGTH + kmovw k1, [%%TMP1] +%else + kmovw k1, [%%TMP1 + %%LENGTH*2] +%endif + vmovdqu8 XWORD(%%OUTPUT){k1}{z}, [%%INPUT] + +%endmacro ; READ_SMALL_DATA_INPUT + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; CALC_AAD_HASH: Calculates the hash of the data which will not be encrypted. +; Input: The input data (A_IN), that data's length (A_LEN), and the hash key (HASH_KEY). +; Output: The hash of the data (AAD_HASH). +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro CALC_AAD_HASH 13 +%define %%A_IN %1 +%define %%A_LEN %2 +%define %%AAD_HASH %3 +%define %%GDATA_KEY %4 +%define %%XTMP0 %5 ; xmm temp reg 5 +%define %%XTMP1 %6 ; xmm temp reg 5 +%define %%XTMP2 %7 +%define %%XTMP3 %8 +%define %%XTMP4 %9 +%define %%XTMP5 %10 ; xmm temp reg 5 +%define %%T1 %11 ; temp reg 1 +%define %%T2 %12 +%define %%T3 %13 + + + mov %%T1, %%A_IN ; T1 = AAD + mov %%T2, %%A_LEN ; T2 = aadLen + vpxor %%AAD_HASH, %%AAD_HASH + +%%_get_AAD_loop128: + cmp %%T2, 128 + jl %%_exit_AAD_loop128 + + vmovdqu %%XTMP0, [%%T1 + 16*0] + vpshufb %%XTMP0, [rel SHUF_MASK] + + vpxor %%XTMP0, %%AAD_HASH + + vmovdqu %%XTMP5, [%%GDATA_KEY + HashKey_8] + vpclmulqdq %%XTMP1, %%XTMP0, %%XTMP5, 0x11 ; %%T1 = a1*b1 + vpclmulqdq %%XTMP2, %%XTMP0, %%XTMP5, 0x00 ; %%T2 = a0*b0 + vpclmulqdq %%XTMP3, %%XTMP0, %%XTMP5, 0x01 ; %%T3 = a1*b0 + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x10 ; %%T4 = a0*b1 + vpxor %%XTMP3, %%XTMP3, %%XTMP4 ; %%T3 = a1*b0 + a0*b1 + +%assign i 1 +%assign j 7 +%rep 7 + vmovdqu %%XTMP0, [%%T1 + 16*i] + vpshufb %%XTMP0, [rel SHUF_MASK] + + vmovdqu %%XTMP5, [%%GDATA_KEY + HashKey_ %+ j] + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x11 ; %%T1 = T1 + a1*b1 + vpxor %%XTMP1, %%XTMP1, %%XTMP4 + + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x00 ; %%T2 = T2 + a0*b0 + vpxor %%XTMP2, %%XTMP2, %%XTMP4 + + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x01 ; %%T3 = T3 + a1*b0 + a0*b1 + vpxor %%XTMP3, %%XTMP3, %%XTMP4 + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x10 + vpxor %%XTMP3, %%XTMP3, %%XTMP4 +%assign i (i + 1) +%assign j (j - 1) +%endrep + + vpslldq %%XTMP4, %%XTMP3, 8 ; shift-L 2 DWs + vpsrldq %%XTMP3, %%XTMP3, 8 ; shift-R 2 DWs + vpxor %%XTMP2, %%XTMP2, %%XTMP4 + vpxor %%XTMP1, %%XTMP1, %%XTMP3 ; accumulate the results in %%T1(M):%%T2(L) + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;first phase of the reduction + vmovdqa %%XTMP5, [rel POLY2] + vpclmulqdq %%XTMP0, %%XTMP5, %%XTMP2, 0x01 + vpslldq %%XTMP0, %%XTMP0, 8 ; shift-L xmm2 2 DWs + vpxor %%XTMP2, %%XTMP2, %%XTMP0 ; first phase of the reduction complete + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;second phase of the reduction + vpclmulqdq %%XTMP3, %%XTMP5, %%XTMP2, 0x00 + vpsrldq %%XTMP3, %%XTMP3, 4 ; shift-R 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) + + vpclmulqdq %%XTMP4, %%XTMP5, %%XTMP2, 0x10 + vpslldq %%XTMP4, %%XTMP4, 4 ; shift-L 1 DW (Shift-L 1-DW to obtain result with no shifts) + + vpxor %%XTMP4, %%XTMP4, %%XTMP3 ; second phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vpxor %%AAD_HASH, %%XTMP1, %%XTMP4 ; the result is in %%T1 + + sub %%T2, 128 + je %%_CALC_AAD_done + + add %%T1, 128 + jmp %%_get_AAD_loop128 + +%%_exit_AAD_loop128: + cmp %%T2, 16 + jl %%_get_small_AAD_block + + ;; calculate hash_key position to start with + mov %%T3, %%T2 + and %%T3, -16 ; 1 to 7 blocks possible here + neg %%T3 + add %%T3, HashKey_1 + 16 + lea %%T3, [%%GDATA_KEY + %%T3] + + vmovdqu %%XTMP0, [%%T1] + vpshufb %%XTMP0, [rel SHUF_MASK] + + vpxor %%XTMP0, %%AAD_HASH + + vmovdqu %%XTMP5, [%%T3] + vpclmulqdq %%XTMP1, %%XTMP0, %%XTMP5, 0x11 ; %%T1 = a1*b1 + vpclmulqdq %%XTMP2, %%XTMP0, %%XTMP5, 0x00 ; %%T2 = a0*b0 + vpclmulqdq %%XTMP3, %%XTMP0, %%XTMP5, 0x01 ; %%T3 = a1*b0 + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x10 ; %%T4 = a0*b1 + vpxor %%XTMP3, %%XTMP3, %%XTMP4 ; %%T3 = a1*b0 + a0*b1 + + add %%T3, 16 ; move to next hashkey + add %%T1, 16 ; move to next data block + sub %%T2, 16 + cmp %%T2, 16 + jl %%_AAD_reduce + +%%_AAD_blocks: + vmovdqu %%XTMP0, [%%T1] + vpshufb %%XTMP0, [rel SHUF_MASK] + + vmovdqu %%XTMP5, [%%T3] + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x11 ; %%T1 = T1 + a1*b1 + vpxor %%XTMP1, %%XTMP1, %%XTMP4 + + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x00 ; %%T2 = T2 + a0*b0 + vpxor %%XTMP2, %%XTMP2, %%XTMP4 + + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x01 ; %%T3 = T3 + a1*b0 + a0*b1 + vpxor %%XTMP3, %%XTMP3, %%XTMP4 + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x10 + vpxor %%XTMP3, %%XTMP3, %%XTMP4 + + add %%T3, 16 ; move to next hashkey + add %%T1, 16 + sub %%T2, 16 + cmp %%T2, 16 + jl %%_AAD_reduce + jmp %%_AAD_blocks + +%%_AAD_reduce: + vpslldq %%XTMP4, %%XTMP3, 8 ; shift-L 2 DWs + vpsrldq %%XTMP3, %%XTMP3, 8 ; shift-R 2 DWs + vpxor %%XTMP2, %%XTMP2, %%XTMP4 + vpxor %%XTMP1, %%XTMP1, %%XTMP3 ; accumulate the results in %%T1(M):%%T2(L) + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;first phase of the reduction + vmovdqa %%XTMP5, [rel POLY2] + vpclmulqdq %%XTMP0, %%XTMP5, %%XTMP2, 0x01 + vpslldq %%XTMP0, %%XTMP0, 8 ; shift-L xmm2 2 DWs + vpxor %%XTMP2, %%XTMP2, %%XTMP0 ; first phase of the reduction complete + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;second phase of the reduction + vpclmulqdq %%XTMP3, %%XTMP5, %%XTMP2, 0x00 + vpsrldq %%XTMP3, %%XTMP3, 4 ; shift-R 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) + + vpclmulqdq %%XTMP4, %%XTMP5, %%XTMP2, 0x10 + vpslldq %%XTMP4, %%XTMP4, 4 ; shift-L 1 DW (Shift-L 1-DW to obtain result with no shifts) + + vpxor %%XTMP4, %%XTMP4, %%XTMP3 ; second phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vpxor %%AAD_HASH, %%XTMP1, %%XTMP4 ; the result is in %%T1 + + or %%T2, %%T2 + je %%_CALC_AAD_done + +%%_get_small_AAD_block: + vmovdqu %%XTMP0, [%%GDATA_KEY + HashKey] + READ_SMALL_DATA_INPUT %%XTMP1, %%T1, %%T2, %%T3 + ;byte-reflect the AAD data + vpshufb %%XTMP1, [rel SHUF_MASK] + vpxor %%AAD_HASH, %%XTMP1 + GHASH_MUL %%AAD_HASH, %%XTMP0, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5 + +%%_CALC_AAD_done: + +%endmacro ; CALC_AAD_HASH + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; PARTIAL_BLOCK: Handles encryption/decryption and the tag partial blocks between update calls. +; Requires the input data be at least 1 byte long. +; Input: gcm_key_data * (GDATA_KEY), gcm_context_data *(GDATA_CTX), input text (PLAIN_CYPH_IN), +; input text length (PLAIN_CYPH_LEN), the current data offset (DATA_OFFSET), +; and whether encoding or decoding (ENC_DEC) +; Output: A cypher of the first partial block (CYPH_PLAIN_OUT), and updated GDATA_CTX +; Clobbers rax, r10, r12, r13, r15, xmm0, xmm1, xmm2, xmm3, xmm5, xmm6, xmm9, xmm10, xmm11, xmm13 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro PARTIAL_BLOCK 8 +%define %%GDATA_KEY %1 +%define %%GDATA_CTX %2 +%define %%CYPH_PLAIN_OUT %3 +%define %%PLAIN_CYPH_IN %4 +%define %%PLAIN_CYPH_LEN %5 +%define %%DATA_OFFSET %6 +%define %%AAD_HASH %7 +%define %%ENC_DEC %8 + + mov r13, [%%GDATA_CTX + PBlockLen] + cmp r13, 0 + je %%_partial_block_done ;Leave Macro if no partial blocks + + cmp %%PLAIN_CYPH_LEN, 16 ;Read in input data without over reading + jl %%_fewer_than_16_bytes + VXLDR xmm1, [%%PLAIN_CYPH_IN] ;If more than 16 bytes of data, just fill the xmm register + jmp %%_data_read + +%%_fewer_than_16_bytes: + lea r10, [%%PLAIN_CYPH_IN] + READ_SMALL_DATA_INPUT xmm1, r10, %%PLAIN_CYPH_LEN, rax + +%%_data_read: ;Finished reading in data + + vmovdqu xmm9, [%%GDATA_CTX + PBlockEncKey] ;xmm9 = my_ctx_data.partial_block_enc_key + vmovdqu xmm13, [%%GDATA_KEY + HashKey] + + lea r12, [rel SHIFT_MASK] + + add r12, r13 ; adjust the shuffle mask pointer to be able to shift r13 bytes (16-r13 is the number of bytes in plaintext mod 16) + vmovdqu xmm2, [r12] ; get the appropriate shuffle mask + vpshufb xmm9, xmm2 ;shift right r13 bytes + +%ifidn %%ENC_DEC, DEC + vmovdqa xmm3, xmm1 +%endif + vpxor xmm9, xmm1 ; Cyphertext XOR E(K, Yn) + + mov r15, %%PLAIN_CYPH_LEN + add r15, r13 + sub r15, 16 ;Set r15 to be the amount of data left in CYPH_PLAIN_IN after filling the block + jge %%_no_extra_mask ;Determine if if partial block is not being filled and shift mask accordingly + sub r12, r15 +%%_no_extra_mask: + + vmovdqu xmm1, [r12 + ALL_F - SHIFT_MASK]; get the appropriate mask to mask out bottom r13 bytes of xmm9 + vpand xmm9, xmm1 ; mask out bottom r13 bytes of xmm9 + +%ifidn %%ENC_DEC, DEC + vpand xmm3, xmm1 + vpshufb xmm3, [rel SHUF_MASK] + vpshufb xmm3, xmm2 + vpxor %%AAD_HASH, xmm3 +%else + vpshufb xmm9, [rel SHUF_MASK] + vpshufb xmm9, xmm2 + vpxor %%AAD_HASH, xmm9 +%endif + cmp r15,0 + jl %%_partial_incomplete + + GHASH_MUL %%AAD_HASH, xmm13, xmm0, xmm10, xmm11, xmm5, xmm6 ;GHASH computation for the last <16 Byte block + xor rax,rax + mov [%%GDATA_CTX + PBlockLen], rax + jmp %%_enc_dec_done +%%_partial_incomplete: +%ifidn __OUTPUT_FORMAT__, win64 + mov rax, %%PLAIN_CYPH_LEN + add [%%GDATA_CTX + PBlockLen], rax +%else + add [%%GDATA_CTX + PBlockLen], %%PLAIN_CYPH_LEN +%endif +%%_enc_dec_done: + vmovdqu [%%GDATA_CTX + AadHash], %%AAD_HASH + +%ifidn %%ENC_DEC, ENC + vpshufb xmm9, [rel SHUF_MASK] ; shuffle xmm9 back to output as ciphertext + vpshufb xmm9, xmm2 +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ; output encrypted Bytes + cmp r15,0 + jl %%_partial_fill + mov r12, r13 + mov r13, 16 + sub r13, r12 ; Set r13 to be the number of bytes to write out + jmp %%_count_set +%%_partial_fill: + mov r13, %%PLAIN_CYPH_LEN +%%_count_set: + lea rax, [rel byte_len_to_mask_table] + kmovw k1, [rax + r13*2] + vmovdqu8 [%%CYPH_PLAIN_OUT + %%DATA_OFFSET]{k1}, xmm9 + add %%DATA_OFFSET, r13 +%%_partial_block_done: +%endmacro ; PARTIAL_BLOCK + + +%macro GHASH_SINGLE_MUL 9 +%define %%GDATA %1 +%define %%HASHKEY %2 +%define %%CIPHER %3 +%define %%STATE_11 %4 +%define %%STATE_00 %5 +%define %%STATE_MID %6 +%define %%T1 %7 +%define %%T2 %8 +%define %%FIRST %9 + + vmovdqu %%T1, [%%GDATA + %%HASHKEY] +%ifidn %%FIRST, first + vpclmulqdq %%STATE_11, %%CIPHER, %%T1, 0x11 ; %%T4 = a1*b1 + vpclmulqdq %%STATE_00, %%CIPHER, %%T1, 0x00 ; %%T4_2 = a0*b0 + vpclmulqdq %%STATE_MID, %%CIPHER, %%T1, 0x01 ; %%T6 = a1*b0 + vpclmulqdq %%T2, %%CIPHER, %%T1, 0x10 ; %%T5 = a0*b1 + vpxor %%STATE_MID, %%STATE_MID, %%T2 +%else + vpclmulqdq %%T2, %%CIPHER, %%T1, 0x11 + vpxor %%STATE_11, %%STATE_11, %%T2 + + vpclmulqdq %%T2, %%CIPHER, %%T1, 0x00 + vpxor %%STATE_00, %%STATE_00, %%T2 + + vpclmulqdq %%T2, %%CIPHER, %%T1, 0x01 + vpxor %%STATE_MID, %%STATE_MID, %%T2 + + vpclmulqdq %%T2, %%CIPHER, %%T1, 0x10 + vpxor %%STATE_MID, %%STATE_MID, %%T2 +%endif + +%endmacro + +; if a = number of total plaintext bytes +; b = floor(a/16) +; %%num_initial_blocks = b mod 8; +; encrypt the initial %%num_initial_blocks blocks and apply ghash on the ciphertext +; %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r14 are used as a pointer only, not modified. +; Updated AAD_HASH is returned in %%T3 + +%macro INITIAL_BLOCKS 23 +%define %%GDATA_KEY %1 +%define %%CYPH_PLAIN_OUT %2 +%define %%PLAIN_CYPH_IN %3 +%define %%LENGTH %4 +%define %%DATA_OFFSET %5 +%define %%num_initial_blocks %6 ; can be 0, 1, 2, 3, 4, 5, 6 or 7 +%define %%T1 %7 +%define %%T2 %8 +%define %%T3 %9 +%define %%T4 %10 +%define %%T5 %11 +%define %%CTR %12 +%define %%XMM1 %13 +%define %%XMM2 %14 +%define %%XMM3 %15 +%define %%XMM4 %16 +%define %%XMM5 %17 +%define %%XMM6 %18 +%define %%XMM7 %19 +%define %%XMM8 %20 +%define %%T6 %21 +%define %%T_key %22 +%define %%ENC_DEC %23 + +%assign i (8-%%num_initial_blocks) + ;; Move AAD_HASH to temp reg + vmovdqu %%T2, %%XMM8 + ;; Start AES for %%num_initial_blocks blocks + ;; vmovdqu %%CTR, [%%GDATA_CTX + CurCount] ; %%CTR = Y0 + +%assign i (9-%%num_initial_blocks) +%rep %%num_initial_blocks + vpaddd %%CTR, %%CTR, [rel ONE] ; INCR Y0 + vmovdqa reg(i), %%CTR + vpshufb reg(i), [rel SHUF_MASK] ; perform a 16Byte swap +%assign i (i+1) +%endrep + +%if(%%num_initial_blocks>0) +vmovdqu %%T_key, [%%GDATA_KEY+16*0] +%assign i (9-%%num_initial_blocks) +%rep %%num_initial_blocks + vpxor reg(i),reg(i),%%T_key +%assign i (i+1) +%endrep + +%assign j 1 +%rep NROUNDS +vmovdqu %%T_key, [%%GDATA_KEY+16*j] +%assign i (9-%%num_initial_blocks) +%rep %%num_initial_blocks + vaesenc reg(i),%%T_key +%assign i (i+1) +%endrep + +%assign j (j+1) +%endrep + + +vmovdqu %%T_key, [%%GDATA_KEY+16*j] +%assign i (9-%%num_initial_blocks) +%rep %%num_initial_blocks + vaesenclast reg(i),%%T_key +%assign i (i+1) +%endrep + +%endif ; %if(%%num_initial_blocks>0) + + + +%assign i (9-%%num_initial_blocks) +%rep %%num_initial_blocks + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] + vpxor reg(i), reg(i), %%T1 + ;; Write back ciphertext for %%num_initial_blocks blocks + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], reg(i) + add %%DATA_OFFSET, 16 + %ifidn %%ENC_DEC, DEC + vmovdqa reg(i), %%T1 + %endif + ;; Prepare ciphertext for GHASH computations + vpshufb reg(i), [rel SHUF_MASK] +%assign i (i+1) +%endrep + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%assign i (9-%%num_initial_blocks) +%if(%%num_initial_blocks>0) + vmovdqa %%T3, reg(i) +%assign i (i+1) +%endif +%if %%num_initial_blocks>1 +%rep %%num_initial_blocks-1 + vmovdqu [rsp + TMP %+ i], reg(i) +%assign i (i+1) +%endrep +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; Prepare 8 counter blocks and perform rounds of AES cipher on + ;; them, load plain/cipher text and store cipher/plain text. + ;; Stitch GHASH computation in between AES rounds. + vpaddd %%XMM1, %%CTR, [rel ONE] ; INCR Y0 + vpaddd %%XMM2, %%CTR, [rel TWO] ; INCR Y0 + vpaddd %%XMM3, %%XMM1, [rel TWO] ; INCR Y0 + vpaddd %%XMM4, %%XMM2, [rel TWO] ; INCR Y0 + vpaddd %%XMM5, %%XMM3, [rel TWO] ; INCR Y0 + vpaddd %%XMM6, %%XMM4, [rel TWO] ; INCR Y0 + vpaddd %%XMM7, %%XMM5, [rel TWO] ; INCR Y0 + vpaddd %%XMM8, %%XMM6, [rel TWO] ; INCR Y0 + vmovdqa %%CTR, %%XMM8 + + vpshufb %%XMM1, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM2, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM3, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM4, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM5, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM6, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM7, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM8, [rel SHUF_MASK] ; perform a 16Byte swap + + vmovdqu %%T_key, [%%GDATA_KEY+16*0] + vpxor %%XMM1, %%XMM1, %%T_key + vpxor %%XMM2, %%XMM2, %%T_key + vpxor %%XMM3, %%XMM3, %%T_key + vpxor %%XMM4, %%XMM4, %%T_key + vpxor %%XMM5, %%XMM5, %%T_key + vpxor %%XMM6, %%XMM6, %%T_key + vpxor %%XMM7, %%XMM7, %%T_key + vpxor %%XMM8, %%XMM8, %%T_key + +%assign i (8-%%num_initial_blocks) +%assign j (9-%%num_initial_blocks) +%assign k (%%num_initial_blocks) + +%define %%T4_2 %%T4 +%if(%%num_initial_blocks>0) + ;; Hash in AES state + ;; T2 - incoming AAD hash + vpxor %%T2, %%T3 + + ;; GDATA, HASHKEY, CIPHER, + ;; STATE_11, STATE_00, STATE_MID, T1, T2 + GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ + %%T1, %%T4, %%T6, %%T5, %%T3, first +%endif + + vmovdqu %%T_key, [%%GDATA_KEY+16*1] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + + vmovdqu %%T_key, [%%GDATA_KEY+16*2] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + +%assign i (i+1) +%assign j (j+1) +%assign k (k-1) +%if(%%num_initial_blocks>1) + ;; GDATA, HASHKEY, CIPHER, + ;; STATE_11, STATE_00, STATE_MID, T1, T2 + vmovdqu %%T2, [rsp + TMP %+ j] + GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ + %%T1, %%T4, %%T6, %%T5, %%T3, not_first +%endif + + vmovdqu %%T_key, [%%GDATA_KEY+16*3] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + + vmovdqu %%T_key, [%%GDATA_KEY+16*4] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + +%assign i (i+1) +%assign j (j+1) +%assign k (k-1) +%if(%%num_initial_blocks>2) + ;; GDATA, HASHKEY, CIPHER, + ;; STATE_11, STATE_00, STATE_MID, T1, T2 + vmovdqu %%T2, [rsp + TMP %+ j] + GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ + %%T1, %%T4, %%T6, %%T5, %%T3, not_first +%endif + +%assign i (i+1) +%assign j (j+1) +%assign k (k-1) +%if(%%num_initial_blocks>3) + ;; GDATA, HASHKEY, CIPHER, + ;; STATE_11, STATE_00, STATE_MID, T1, T2 + vmovdqu %%T2, [rsp + TMP %+ j] + GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ + %%T1, %%T4, %%T6, %%T5, %%T3, not_first +%endif + + vmovdqu %%T_key, [%%GDATA_KEY+16*5] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + + vmovdqu %%T_key, [%%GDATA_KEY+16*6] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + +%assign i (i+1) +%assign j (j+1) +%assign k (k-1) +%if(%%num_initial_blocks>4) + ;; GDATA, HASHKEY, CIPHER, + ;; STATE_11, STATE_00, STATE_MID, T1, T2 + vmovdqu %%T2, [rsp + TMP %+ j] + GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ + %%T1, %%T4, %%T6, %%T5, %%T3, not_first +%endif + + vmovdqu %%T_key, [%%GDATA_KEY+16*7] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + + vmovdqu %%T_key, [%%GDATA_KEY+16*8] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + +%assign i (i+1) +%assign j (j+1) +%assign k (k-1) +%if(%%num_initial_blocks>5) + ;; GDATA, HASHKEY, CIPHER, + ;; STATE_11, STATE_00, STATE_MID, T1, T2 + vmovdqu %%T2, [rsp + TMP %+ j] + GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ + %%T1, %%T4, %%T6, %%T5, %%T3, not_first +%endif + + vmovdqu %%T_key, [%%GDATA_KEY+16*9] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + +%ifndef GCM128_MODE + vmovdqu %%T_key, [%%GDATA_KEY+16*10] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key +%endif + +%assign i (i+1) +%assign j (j+1) +%assign k (k-1) +%if(%%num_initial_blocks>6) + ;; GDATA, HASHKEY, CIPHER, + ;; STATE_11, STATE_00, STATE_MID, T1, T2 + vmovdqu %%T2, [rsp + TMP %+ j] + GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ + %%T1, %%T4, %%T6, %%T5, %%T3, not_first +%endif + +%ifdef GCM128_MODE + vmovdqu %%T_key, [%%GDATA_KEY+16*10] + vaesenclast %%XMM1, %%T_key + vaesenclast %%XMM2, %%T_key + vaesenclast %%XMM3, %%T_key + vaesenclast %%XMM4, %%T_key + vaesenclast %%XMM5, %%T_key + vaesenclast %%XMM6, %%T_key + vaesenclast %%XMM7, %%T_key + vaesenclast %%XMM8, %%T_key +%endif + +%ifdef GCM192_MODE + vmovdqu %%T_key, [%%GDATA_KEY+16*11] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + + vmovdqu %%T_key, [%%GDATA_KEY+16*12] + vaesenclast %%XMM1, %%T_key + vaesenclast %%XMM2, %%T_key + vaesenclast %%XMM3, %%T_key + vaesenclast %%XMM4, %%T_key + vaesenclast %%XMM5, %%T_key + vaesenclast %%XMM6, %%T_key + vaesenclast %%XMM7, %%T_key + vaesenclast %%XMM8, %%T_key +%endif +%ifdef GCM256_MODE + vmovdqu %%T_key, [%%GDATA_KEY+16*11] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + + vmovdqu %%T_key, [%%GDATA_KEY+16*12] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key +%endif + +%assign i (i+1) +%assign j (j+1) +%assign k (k-1) +%if(%%num_initial_blocks>7) + ;; GDATA, HASHKEY, CIPHER, + ;; STATE_11, STATE_00, STATE_MID, T1, T2 + vmovdqu %%T2, [rsp + TMP %+ j] + GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ + %%T1, %%T4, %%T6, %%T5, %%T3, not_first +%endif + +%ifdef GCM256_MODE ; GCM256 + vmovdqu %%T_key, [%%GDATA_KEY+16*13] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + + vmovdqu %%T_key, [%%GDATA_KEY+16*14] + vaesenclast %%XMM1, %%T_key + vaesenclast %%XMM2, %%T_key + vaesenclast %%XMM3, %%T_key + vaesenclast %%XMM4, %%T_key + vaesenclast %%XMM5, %%T_key + vaesenclast %%XMM6, %%T_key + vaesenclast %%XMM7, %%T_key + vaesenclast %%XMM8, %%T_key +%endif ; GCM256 mode + +%if(%%num_initial_blocks>0) + vpsrldq %%T3, %%T6, 8 ; shift-R %%T2 2 DWs + vpslldq %%T6, %%T6, 8 ; shift-L %%T3 2 DWs + vpxor %%T1, %%T1, %%T3 ; accumulate the results in %%T1:%%T4 + vpxor %%T4, %%T6, %%T4 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ; First phase of the reduction + vmovdqu %%T3, [rel POLY2] + + vpclmulqdq %%T2, %%T3, %%T4, 0x01 + vpslldq %%T2, %%T2, 8 ; shift-L xmm2 2 DWs + + ;; First phase of the reduction complete + vpxor %%T4, %%T4, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ; Second phase of the reduction + vpclmulqdq %%T2, %%T3, %%T4, 0x00 + ;; Shift-R xmm2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) + vpsrldq %%T2, %%T2, 4 + + vpclmulqdq %%T4, %%T3, %%T4, 0x10 + ;; Shift-L xmm0 1 DW (Shift-L 1-DW to obtain result with no shifts) + vpslldq %%T4, %%T4, 4 + ;; Second phase of the reduction complete + vpxor %%T4, %%T4, %%T2 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ; The result is in %%T3 + vpxor %%T3, %%T1, %%T4 +%else + ;; The hash should end up in T3 + vmovdqa %%T3, %%T2 +%endif + + ;; Final hash is now in T3 +%if %%num_initial_blocks > 0 + ;; NOTE: obsolete in case %%num_initial_blocks = 0 + sub %%LENGTH, 16*%%num_initial_blocks +%endif + + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*0] + vpxor %%XMM1, %%XMM1, %%T1 + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*0], %%XMM1 + %ifidn %%ENC_DEC, DEC + vmovdqa %%XMM1, %%T1 + %endif + + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*1] + vpxor %%XMM2, %%XMM2, %%T1 + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*1], %%XMM2 + %ifidn %%ENC_DEC, DEC + vmovdqa %%XMM2, %%T1 + %endif + + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*2] + vpxor %%XMM3, %%XMM3, %%T1 + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*2], %%XMM3 + %ifidn %%ENC_DEC, DEC + vmovdqa %%XMM3, %%T1 + %endif + + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*3] + vpxor %%XMM4, %%XMM4, %%T1 + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*3], %%XMM4 + %ifidn %%ENC_DEC, DEC + vmovdqa %%XMM4, %%T1 + %endif + + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*4] + vpxor %%XMM5, %%XMM5, %%T1 + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*4], %%XMM5 + %ifidn %%ENC_DEC, DEC + vmovdqa %%XMM5, %%T1 + %endif + + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*5] + vpxor %%XMM6, %%XMM6, %%T1 + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*5], %%XMM6 + %ifidn %%ENC_DEC, DEC + vmovdqa %%XMM6, %%T1 + %endif + + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*6] + vpxor %%XMM7, %%XMM7, %%T1 + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*6], %%XMM7 + %ifidn %%ENC_DEC, DEC + vmovdqa %%XMM7, %%T1 + %endif + +%if %%num_initial_blocks > 0 + ;; NOTE: 'jl' is never taken for %%num_initial_blocks = 0 + ;; This macro is executed for lenght 128 and up, + ;; zero length is checked in GCM_ENC_DEC. + ;; If the last block is partial then the xor will be done later + ;; in ENCRYPT_FINAL_PARTIAL_BLOCK. + ;; We know it's partial if LENGTH - 16*num_initial_blocks < 128 + cmp %%LENGTH, 128 + jl %%_initial_skip_last_word_write +%endif + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*7] + vpxor %%XMM8, %%XMM8, %%T1 + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*7], %%XMM8 + %ifidn %%ENC_DEC, DEC + vmovdqa %%XMM8, %%T1 + %endif + + ;; Update %%LENGTH with the number of blocks processed + sub %%LENGTH, 16 + add %%DATA_OFFSET, 16 +%%_initial_skip_last_word_write: + sub %%LENGTH, 128-16 + add %%DATA_OFFSET, 128-16 + + vpshufb %%XMM1, [rel SHUF_MASK] ; perform a 16Byte swap + ;; Combine GHASHed value with the corresponding ciphertext + vpxor %%XMM1, %%XMM1, %%T3 + vpshufb %%XMM2, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM3, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM4, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM5, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM6, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM7, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM8, [rel SHUF_MASK] ; perform a 16Byte swap + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%%_initial_blocks_done: + + +%endmacro + +;;; INITIAL_BLOCKS macro with support for a partial final block. +;;; num_initial_blocks is expected to include the partial final block +;;; in the count. +%macro INITIAL_BLOCKS_PARTIAL 25 +%define %%GDATA_KEY %1 +%define %%GDATA_CTX %2 +%define %%CYPH_PLAIN_OUT %3 +%define %%PLAIN_CYPH_IN %4 +%define %%LENGTH %5 +%define %%DATA_OFFSET %6 +%define %%num_initial_blocks %7 ; can be 1, 2, 3, 4, 5, 6 or 7 (not 0) +%define %%T1 %8 +%define %%T2 %9 +%define %%T3 %10 ; [out] hash value +%define %%T4 %11 +%define %%T5 %12 +%define %%CTR %13 +%define %%XMM1 %14 +%define %%XMM2 %15 +%define %%XMM3 %16 +%define %%XMM4 %17 +%define %%XMM5 %18 +%define %%XMM6 %19 +%define %%XMM7 %20 +%define %%XMM8 %21 ; [in] hash value +%define %%T6 %22 +%define %%T_key %23 +%define %%ENC_DEC %24 +%define %%INSTANCE_TYPE %25 + + ;; Move AAD_HASH to temp reg + vmovdqu %%T2, %%XMM8 + +%assign i (9-%%num_initial_blocks) +%rep %%num_initial_blocks + ;; Compute AES counters + vpaddd %%CTR, %%CTR, [rel ONE] ; INCR Y0 + vmovdqa reg(i), %%CTR + vpshufb reg(i), [rel SHUF_MASK] ; perform a 16Byte swap +%assign i (i+1) +%endrep + +vmovdqu %%T_key, [%%GDATA_KEY+16*0] +%assign i (9-%%num_initial_blocks) +%rep %%num_initial_blocks + ; Start AES for %%num_initial_blocks blocks + vpxor reg(i),reg(i),%%T_key +%assign i (i+1) +%endrep + +%assign j 1 +%rep NROUNDS +vmovdqu %%T_key, [%%GDATA_KEY+16*j] +%assign i (9-%%num_initial_blocks) +%rep %%num_initial_blocks + vaesenc reg(i),%%T_key +%assign i (i+1) +%endrep + +%assign j (j+1) +%endrep + + +vmovdqu %%T_key, [%%GDATA_KEY+16*j] +%assign i (9-%%num_initial_blocks) +%rep %%num_initial_blocks + vaesenclast reg(i),%%T_key +%assign i (i+1) +%endrep + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Hash all but the last block of data +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%assign i (9-%%num_initial_blocks) +%rep %%num_initial_blocks-1 + ;; Encrypt the message for all but the last block + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] + vpxor reg(i), reg(i), %%T1 + ;; write back ciphertext for %%num_initial_blocks blocks + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], reg(i) + add %%DATA_OFFSET, 16 +%ifidn %%ENC_DEC, DEC + vmovdqa reg(i), %%T1 +%endif + ;; Prepare ciphertext for GHASH computations + vpshufb reg(i), [rel SHUF_MASK] +%assign i (i+1) +%endrep + +%if %%num_initial_blocks > 1 + ;; The final block of data may be <16B + sub %%LENGTH, 16*(%%num_initial_blocks-1) +%endif + +%if %%num_initial_blocks < 8 + ;; NOTE: the 'jl' is always taken for num_initial_blocks = 8. + ;; This is run in the context of GCM_ENC_DEC_SMALL for length < 128. + cmp %%LENGTH, 16 + jl %%_small_initial_partial_block + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Handle a full length final block - encrypt and hash all blocks +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + sub %%LENGTH, 16 + mov [%%GDATA_CTX + PBlockLen], %%LENGTH + + ;; Encrypt the message + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] + vpxor reg(i), reg(i), %%T1 + ;; write back ciphertext for %%num_initial_blocks blocks + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], reg(i) + add %%DATA_OFFSET, 16 +%ifidn %%ENC_DEC, DEC + vmovdqa reg(i), %%T1 +%endif + ;; Prepare ciphertext for GHASH computations + vpshufb reg(i), [rel SHUF_MASK] + + ;; Hash all of the data +%assign i (8-%%num_initial_blocks) +%assign j (9-%%num_initial_blocks) +%assign k (%%num_initial_blocks) +%assign last_block_to_hash 0 + +%if(%%num_initial_blocks>last_block_to_hash) + ;; Hash in AES state + vpxor %%T2, reg(j) + + ;; T2 - incoming AAD hash + ;; reg(i) holds ciphertext + ;; T5 - hash key + ;; T6 - updated xor + ;; reg(1)/xmm1 should now be available for tmp use + vmovdqu %%T5, [%%GDATA_KEY + HashKey_ %+ k] + vpclmulqdq %%T1, %%T2, %%T5, 0x11 ; %%T4 = a1*b1 + vpclmulqdq %%T4, %%T2, %%T5, 0x00 ; %%T4 = a0*b0 + vpclmulqdq %%T6, %%T2, %%T5, 0x01 ; %%T6 = a1*b0 + vpclmulqdq %%T5, %%T2, %%T5, 0x10 ; %%T5 = a0*b1 + vpxor %%T6, %%T6, %%T5 +%endif + +%assign i (i+1) +%assign j (j+1) +%assign k (k-1) +%assign rep_count (%%num_initial_blocks-1) +%rep rep_count + + vmovdqu %%T5, [%%GDATA_KEY + HashKey_ %+ k] + vpclmulqdq %%T3, reg(j), %%T5, 0x11 + vpxor %%T1, %%T1, %%T3 + + vpclmulqdq %%T3, reg(j), %%T5, 0x00 + vpxor %%T4, %%T4, %%T3 + + vpclmulqdq %%T3, reg(j), %%T5, 0x01 + vpxor %%T6, %%T6, %%T3 + + vpclmulqdq %%T3, reg(j), %%T5, 0x10 + vpxor %%T6, %%T6, %%T3 + +%assign i (i+1) +%assign j (j+1) +%assign k (k-1) +%endrep + + ;; Record that a reduction is needed + mov r12, 1 + + jmp %%_small_initial_compute_hash + + +%endif ; %if %%num_initial_blocks < 8 + +%%_small_initial_partial_block: + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Handle ghash for a <16B final block +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + ;; In this case if it's a single call to encrypt we can + ;; hash all of the data but if it's an init / update / finalize + ;; series of call we need to leave the last block if it's + ;; less than a full block of data. + + mov [%%GDATA_CTX + PBlockLen], %%LENGTH + vmovdqu [%%GDATA_CTX + PBlockEncKey], reg(i) + ;; Handle a partial final block + ;; GDATA, KEY, T1, T2 + ;; r13 - length + ;; LT16 - indicates type of read and that the buffer is less than 16 bytes long + ;; NOTE: could be replaced with %%LENGTH but at this point + ;; %%LENGTH is always less than 16. + ;; No PLAIN_CYPH_LEN argument available in this macro. + ENCRYPT_FINAL_PARTIAL_BLOCK reg(i), %%T1, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, LT16, %%ENC_DEC, %%DATA_OFFSET + vpshufb reg(i), [rel SHUF_MASK] + +%ifidn %%INSTANCE_TYPE, multi_call +%assign i (8-%%num_initial_blocks) +%assign j (9-%%num_initial_blocks) +%assign k (%%num_initial_blocks-1) +%assign last_block_to_hash 1 +%else +%assign i (8-%%num_initial_blocks) +%assign j (9-%%num_initial_blocks) +%assign k (%%num_initial_blocks) +%assign last_block_to_hash 0 +%endif + +%if(%%num_initial_blocks>last_block_to_hash) + ;; Record that a reduction is needed + mov r12, 1 + ;; Hash in AES state + vpxor %%T2, reg(j) + + ;; T2 - incoming AAD hash + ;; reg(i) holds ciphertext + ;; T5 - hash key + ;; T6 - updated xor + ;; reg(1)/xmm1 should now be available for tmp use + vmovdqu %%T5, [%%GDATA_KEY + HashKey_ %+ k] + vpclmulqdq %%T1, %%T2, %%T5, 0x11 ; %%T4 = a1*b1 + vpclmulqdq %%T4, %%T2, %%T5, 0x00 ; %%T4 = a0*b0 + vpclmulqdq %%T6, %%T2, %%T5, 0x01 ; %%T6 = a1*b0 + vpclmulqdq %%T5, %%T2, %%T5, 0x10 ; %%T5 = a0*b1 + vpxor %%T6, %%T6, %%T5 +%else + ;; Record that a reduction is not needed - + ;; In this case no hashes are computed because there + ;; is only one initial block and it is < 16B in length. + xor r12, r12 +%endif + +%assign i (i+1) +%assign j (j+1) +%assign k (k-1) +%ifidn %%INSTANCE_TYPE, multi_call +%assign rep_count (%%num_initial_blocks-2) +%%_multi_call_hash: +%else +%assign rep_count (%%num_initial_blocks-1) +%endif + +%if rep_count < 0 + ;; fix for negative rep_count +%assign rep_count 0 +%endif + +%rep rep_count + + vmovdqu %%T5, [%%GDATA_KEY + HashKey_ %+ k] + vpclmulqdq %%T3, reg(j), %%T5, 0x11 + vpxor %%T1, %%T1, %%T3 + + vpclmulqdq %%T3, reg(j), %%T5, 0x00 + vpxor %%T4, %%T4, %%T3 + + vpclmulqdq %%T3, reg(j), %%T5, 0x01 + vpxor %%T6, %%T6, %%T3 + + vpclmulqdq %%T3, reg(j), %%T5, 0x10 + vpxor %%T6, %%T6, %%T3 + +%assign i (i+1) +%assign j (j+1) +%assign k (k-1) +%endrep + +%%_small_initial_compute_hash: + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Ghash reduction +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%if(%%num_initial_blocks=1) +%ifidn %%INSTANCE_TYPE, multi_call + ;; We only need to check if a reduction is needed if + ;; initial_blocks == 1 and init/update/final is being used. + ;; In this case we may just have a partial block, and that + ;; gets hashed in finalize. + ;; cmp r12, 0 + or r12, r12 + je %%_no_reduction_needed +%endif +%endif + + vpsrldq %%T3, %%T6, 8 ; shift-R %%T2 2 DWs + vpslldq %%T6, %%T6, 8 ; shift-L %%T3 2 DWs + vpxor %%T1, %%T1, %%T3 ; accumulate the results in %%T1:%%T4 + vpxor %%T4, %%T6, %%T4 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; First phase of the reduction + vmovdqu %%T3, [rel POLY2] + + vpclmulqdq %%T2, %%T3, %%T4, 0x01 + ;; shift-L xmm2 2 DWs + vpslldq %%T2, %%T2, 8 + vpxor %%T4, %%T4, %%T2 + + ;; First phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; Second phase of the reduction + + vpclmulqdq %%T2, %%T3, %%T4, 0x00 + ;; Shift-R xmm2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) + vpsrldq %%T2, %%T2, 4 + + vpclmulqdq %%T4, %%T3, %%T4, 0x10 + ;; Shift-L xmm0 1 DW (Shift-L 1-DW to obtain result with no shifts) + vpslldq %%T4, %%T4, 4 + + vpxor %%T4, %%T4, %%T2 + ;; Second phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vpxor %%T3, %%T1, %%T4 + +%ifidn %%INSTANCE_TYPE, multi_call + ;; If using init/update/finalize, we need to xor any partial block data + ;; into the hash. +%if %%num_initial_blocks > 1 + ;; NOTE: for %%num_initial_blocks = 0 the xor never takes place +%if %%num_initial_blocks != 8 + ;; NOTE: for %%num_initial_blocks = 8, %%LENGTH, stored in [PBlockLen] is never zero + cmp qword [%%GDATA_CTX + PBlockLen], 0 + je %%_no_partial_block_xor +%endif ; %%num_initial_blocks != 8 + vpxor %%T3, %%T3, reg(8) +%%_no_partial_block_xor: +%endif ; %%num_initial_blocks > 1 +%endif ; %%INSTANCE_TYPE, multi_call + +%if(%%num_initial_blocks=1) +%ifidn %%INSTANCE_TYPE, multi_call + ;; NOTE: %%_no_reduction_needed case only valid for + ;; multi_call with initial_blocks = 1. + ;; Look for comment above around '_no_reduction_needed' + ;; The jmp below is obsolete as the code will fall through. + + ;; The result is in %%T3 + jmp %%_after_reduction + +%%_no_reduction_needed: + ;; The hash should end up in T3. The only way we should get here is if + ;; there is a partial block of data, so xor that into the hash. + vpxor %%T3, %%T2, reg(8) +%endif ; %%INSTANCE_TYPE = multi_call +%endif ; %%num_initial_blocks=1 + +%%_after_reduction: + ;; Final hash is now in T3 + +%endmacro ; INITIAL_BLOCKS_PARTIAL + + + +; encrypt 8 blocks at a time +; ghash the 8 previously encrypted ciphertext blocks +; %%GDATA (KEY), %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN are used as pointers only, not modified +; %%DATA_OFFSET is the data offset value +%macro GHASH_8_ENCRYPT_8_PARALLEL 23 +%define %%GDATA %1 +%define %%CYPH_PLAIN_OUT %2 +%define %%PLAIN_CYPH_IN %3 +%define %%DATA_OFFSET %4 +%define %%T1 %5 +%define %%T2 %6 +%define %%T3 %7 +%define %%T4 %8 +%define %%T5 %9 +%define %%T6 %10 +%define %%CTR %11 +%define %%XMM1 %12 +%define %%XMM2 %13 +%define %%XMM3 %14 +%define %%XMM4 %15 +%define %%XMM5 %16 +%define %%XMM6 %17 +%define %%XMM7 %18 +%define %%XMM8 %19 +%define %%T7 %20 +%define %%loop_idx %21 +%define %%ENC_DEC %22 +%define %%FULL_PARTIAL %23 + + vmovdqa %%T2, %%XMM1 + vmovdqu [rsp + TMP2], %%XMM2 + vmovdqu [rsp + TMP3], %%XMM3 + vmovdqu [rsp + TMP4], %%XMM4 + vmovdqu [rsp + TMP5], %%XMM5 + vmovdqu [rsp + TMP6], %%XMM6 + vmovdqu [rsp + TMP7], %%XMM7 + vmovdqu [rsp + TMP8], %%XMM8 + +%ifidn %%loop_idx, in_order + vpaddd %%XMM1, %%CTR, [rel ONE] ; INCR CNT + vmovdqu %%T5, [rel TWO] + vpaddd %%XMM2, %%CTR, %%T5 + vpaddd %%XMM3, %%XMM1, %%T5 + vpaddd %%XMM4, %%XMM2, %%T5 + vpaddd %%XMM5, %%XMM3, %%T5 + vpaddd %%XMM6, %%XMM4, %%T5 + vpaddd %%XMM7, %%XMM5, %%T5 + vpaddd %%XMM8, %%XMM6, %%T5 + vmovdqa %%CTR, %%XMM8 + + vmovdqu %%T5, [rel SHUF_MASK] + vpshufb %%XMM1, %%T5 ; perform a 16Byte swap + vpshufb %%XMM2, %%T5 ; perform a 16Byte swap + vpshufb %%XMM3, %%T5 ; perform a 16Byte swap + vpshufb %%XMM4, %%T5 ; perform a 16Byte swap + vpshufb %%XMM5, %%T5 ; perform a 16Byte swap + vpshufb %%XMM6, %%T5 ; perform a 16Byte swap + vpshufb %%XMM7, %%T5 ; perform a 16Byte swap + vpshufb %%XMM8, %%T5 ; perform a 16Byte swap +%else + vpaddd %%XMM1, %%CTR, [rel ONEf] ; INCR CNT + vmovdqu %%T5, [rel TWOf] + vpaddd %%XMM2, %%CTR, %%T5 + vpaddd %%XMM3, %%XMM1, %%T5 + vpaddd %%XMM4, %%XMM2, %%T5 + vpaddd %%XMM5, %%XMM3, %%T5 + vpaddd %%XMM6, %%XMM4, %%T5 + vpaddd %%XMM7, %%XMM5, %%T5 + vpaddd %%XMM8, %%XMM6, %%T5 + vmovdqa %%CTR, %%XMM8 +%endif + + + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T1, [%%GDATA + 16*0] + vpxor %%XMM1, %%XMM1, %%T1 + vpxor %%XMM2, %%XMM2, %%T1 + vpxor %%XMM3, %%XMM3, %%T1 + vpxor %%XMM4, %%XMM4, %%T1 + vpxor %%XMM5, %%XMM5, %%T1 + vpxor %%XMM6, %%XMM6, %%T1 + vpxor %%XMM7, %%XMM7, %%T1 + vpxor %%XMM8, %%XMM8, %%T1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T1, [%%GDATA + 16*1] + vaesenc %%XMM1, %%T1 + vaesenc %%XMM2, %%T1 + vaesenc %%XMM3, %%T1 + vaesenc %%XMM4, %%T1 + vaesenc %%XMM5, %%T1 + vaesenc %%XMM6, %%T1 + vaesenc %%XMM7, %%T1 + vaesenc %%XMM8, %%T1 + + + vmovdqu %%T1, [%%GDATA + 16*2] + vaesenc %%XMM1, %%T1 + vaesenc %%XMM2, %%T1 + vaesenc %%XMM3, %%T1 + vaesenc %%XMM4, %%T1 + vaesenc %%XMM5, %%T1 + vaesenc %%XMM6, %%T1 + vaesenc %%XMM7, %%T1 + vaesenc %%XMM8, %%T1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_8] + vpclmulqdq %%T4, %%T2, %%T5, 0x11 ; %%T4 = a1*b1 + vpclmulqdq %%T7, %%T2, %%T5, 0x00 ; %%T7 = a0*b0 + vpclmulqdq %%T6, %%T2, %%T5, 0x01 ; %%T6 = a1*b0 + vpclmulqdq %%T5, %%T2, %%T5, 0x10 ; %%T5 = a0*b1 + vpxor %%T6, %%T6, %%T5 + + vmovdqu %%T1, [%%GDATA + 16*3] + vaesenc %%XMM1, %%T1 + vaesenc %%XMM2, %%T1 + vaesenc %%XMM3, %%T1 + vaesenc %%XMM4, %%T1 + vaesenc %%XMM5, %%T1 + vaesenc %%XMM6, %%T1 + vaesenc %%XMM7, %%T1 + vaesenc %%XMM8, %%T1 + + vmovdqu %%T1, [rsp + TMP2] + vmovdqu %%T5, [%%GDATA + HashKey_7] + vpclmulqdq %%T3, %%T1, %%T5, 0x11 + vpxor %%T4, %%T4, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x00 + vpxor %%T7, %%T7, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x01 + vpxor %%T6, %%T6, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x10 + vpxor %%T6, %%T6, %%T3 + + vmovdqu %%T1, [%%GDATA + 16*4] + vaesenc %%XMM1, %%T1 + vaesenc %%XMM2, %%T1 + vaesenc %%XMM3, %%T1 + vaesenc %%XMM4, %%T1 + vaesenc %%XMM5, %%T1 + vaesenc %%XMM6, %%T1 + vaesenc %%XMM7, %%T1 + vaesenc %%XMM8, %%T1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vmovdqu %%T1, [rsp + TMP3] + vmovdqu %%T5, [%%GDATA + HashKey_6] + vpclmulqdq %%T3, %%T1, %%T5, 0x11 + vpxor %%T4, %%T4, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x00 + vpxor %%T7, %%T7, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x01 + vpxor %%T6, %%T6, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x10 + vpxor %%T6, %%T6, %%T3 + + vmovdqu %%T1, [%%GDATA + 16*5] + vaesenc %%XMM1, %%T1 + vaesenc %%XMM2, %%T1 + vaesenc %%XMM3, %%T1 + vaesenc %%XMM4, %%T1 + vaesenc %%XMM5, %%T1 + vaesenc %%XMM6, %%T1 + vaesenc %%XMM7, %%T1 + vaesenc %%XMM8, %%T1 + + + vmovdqu %%T1, [rsp + TMP4] + vmovdqu %%T5, [%%GDATA + HashKey_5] + vpclmulqdq %%T3, %%T1, %%T5, 0x11 + vpxor %%T4, %%T4, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x00 + vpxor %%T7, %%T7, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x01 + vpxor %%T6, %%T6, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x10 + vpxor %%T6, %%T6, %%T3 + + vmovdqu %%T1, [%%GDATA + 16*6] + vaesenc %%XMM1, %%T1 + vaesenc %%XMM2, %%T1 + vaesenc %%XMM3, %%T1 + vaesenc %%XMM4, %%T1 + vaesenc %%XMM5, %%T1 + vaesenc %%XMM6, %%T1 + vaesenc %%XMM7, %%T1 + vaesenc %%XMM8, %%T1 + + vmovdqu %%T1, [rsp + TMP5] + vmovdqu %%T5, [%%GDATA + HashKey_4] + vpclmulqdq %%T3, %%T1, %%T5, 0x11 + vpxor %%T4, %%T4, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x00 + vpxor %%T7, %%T7, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x01 + vpxor %%T6, %%T6, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x10 + vpxor %%T6, %%T6, %%T3 + + vmovdqu %%T1, [%%GDATA + 16*7] + vaesenc %%XMM1, %%T1 + vaesenc %%XMM2, %%T1 + vaesenc %%XMM3, %%T1 + vaesenc %%XMM4, %%T1 + vaesenc %%XMM5, %%T1 + vaesenc %%XMM6, %%T1 + vaesenc %%XMM7, %%T1 + vaesenc %%XMM8, %%T1 + + vmovdqu %%T1, [rsp + TMP6] + vmovdqu %%T5, [%%GDATA + HashKey_3] + vpclmulqdq %%T3, %%T1, %%T5, 0x11 + vpxor %%T4, %%T4, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x00 + vpxor %%T7, %%T7, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x01 + vpxor %%T6, %%T6, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x10 + vpxor %%T6, %%T6, %%T3 + + vmovdqu %%T1, [%%GDATA + 16*8] + vaesenc %%XMM1, %%T1 + vaesenc %%XMM2, %%T1 + vaesenc %%XMM3, %%T1 + vaesenc %%XMM4, %%T1 + vaesenc %%XMM5, %%T1 + vaesenc %%XMM6, %%T1 + vaesenc %%XMM7, %%T1 + vaesenc %%XMM8, %%T1 + + vmovdqu %%T1, [rsp + TMP7] + vmovdqu %%T5, [%%GDATA + HashKey_2] + vpclmulqdq %%T3, %%T1, %%T5, 0x11 + vpxor %%T4, %%T4, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x00 + vpxor %%T7, %%T7, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x01 + vpxor %%T6, %%T6, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x10 + vpxor %%T6, %%T6, %%T3 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + 16*9] + vaesenc %%XMM1, %%T5 + vaesenc %%XMM2, %%T5 + vaesenc %%XMM3, %%T5 + vaesenc %%XMM4, %%T5 + vaesenc %%XMM5, %%T5 + vaesenc %%XMM6, %%T5 + vaesenc %%XMM7, %%T5 + vaesenc %%XMM8, %%T5 + + vmovdqu %%T1, [rsp + TMP8] + vmovdqu %%T5, [%%GDATA + HashKey] + + + vpclmulqdq %%T3, %%T1, %%T5, 0x00 + vpxor %%T7, %%T7, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x01 + vpxor %%T6, %%T6, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x10 + vpxor %%T6, %%T6, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x11 + vpxor %%T1, %%T4, %%T3 + + + vmovdqu %%T5, [%%GDATA + 16*10] + %ifndef GCM128_MODE ; GCM192 or GCM256 + vaesenc %%XMM1, %%T5 + vaesenc %%XMM2, %%T5 + vaesenc %%XMM3, %%T5 + vaesenc %%XMM4, %%T5 + vaesenc %%XMM5, %%T5 + vaesenc %%XMM6, %%T5 + vaesenc %%XMM7, %%T5 + vaesenc %%XMM8, %%T5 + + vmovdqu %%T5, [%%GDATA + 16*11] + vaesenc %%XMM1, %%T5 + vaesenc %%XMM2, %%T5 + vaesenc %%XMM3, %%T5 + vaesenc %%XMM4, %%T5 + vaesenc %%XMM5, %%T5 + vaesenc %%XMM6, %%T5 + vaesenc %%XMM7, %%T5 + vaesenc %%XMM8, %%T5 + + vmovdqu %%T5, [%%GDATA + 16*12] +%endif +%ifdef GCM256_MODE + vaesenc %%XMM1, %%T5 + vaesenc %%XMM2, %%T5 + vaesenc %%XMM3, %%T5 + vaesenc %%XMM4, %%T5 + vaesenc %%XMM5, %%T5 + vaesenc %%XMM6, %%T5 + vaesenc %%XMM7, %%T5 + vaesenc %%XMM8, %%T5 + + vmovdqu %%T5, [%%GDATA + 16*13] + vaesenc %%XMM1, %%T5 + vaesenc %%XMM2, %%T5 + vaesenc %%XMM3, %%T5 + vaesenc %%XMM4, %%T5 + vaesenc %%XMM5, %%T5 + vaesenc %%XMM6, %%T5 + vaesenc %%XMM7, %%T5 + vaesenc %%XMM8, %%T5 + + vmovdqu %%T5, [%%GDATA + 16*14] +%endif ; GCM256 + +%assign i 0 +%assign j 1 +%rep 8 + + ;; SNP TBD: This is pretty ugly - consider whether just XORing the + ;; data in after vaesenclast is simpler and performant. Would + ;; also have to ripple it through partial block and ghash_mul_8. +%ifidn %%FULL_PARTIAL, full + %ifdef NT_LD + VXLDR %%T2, [%%PLAIN_CYPH_IN+%%DATA_OFFSET+16*i] + vpxor %%T2, %%T2, %%T5 + %else + vpxor %%T2, %%T5, [%%PLAIN_CYPH_IN+%%DATA_OFFSET+16*i] + %endif + + %ifidn %%ENC_DEC, ENC + vaesenclast reg(j), reg(j), %%T2 + %else + vaesenclast %%T3, reg(j), %%T2 + vpxor reg(j), %%T2, %%T5 + VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*i], %%T3 + %endif + +%else + ; Don't read the final data during partial block processing + %ifdef NT_LD + %if (i<7) + VXLDR %%T2, [%%PLAIN_CYPH_IN+%%DATA_OFFSET+16*i] + vpxor %%T2, %%T2, %%T5 + %else + ;; Stage the key directly in T2 rather than hash it with plaintext + vmovdqu %%T2, %%T5 + %endif + %else + %if (i<7) + vpxor %%T2, %%T5, [%%PLAIN_CYPH_IN+%%DATA_OFFSET+16*i] + %else + ;; Stage the key directly in T2 rather than hash it with plaintext + vmovdqu %%T2, %%T5 + %endif + %endif + + %ifidn %%ENC_DEC, ENC + vaesenclast reg(j), reg(j), %%T2 + %else + %if (i<7) + vaesenclast %%T3, reg(j), %%T2 + vpxor reg(j), %%T2, %%T5 + ;; Do not read the data since it could fault + VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*i], %%T3 + %else + vaesenclast reg(j), reg(j), %%T2 + %endif + %endif +%endif + +%assign i (i+1) +%assign j (j+1) +%endrep + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + + vpslldq %%T3, %%T6, 8 ; shift-L %%T3 2 DWs + vpsrldq %%T6, %%T6, 8 ; shift-R %%T2 2 DWs + vpxor %%T7, %%T7, %%T3 + vpxor %%T1, %%T1, %%T6 ; accumulate the results in %%T1:%%T7 + + + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;first phase of the reduction + vmovdqu %%T3, [rel POLY2] + + vpclmulqdq %%T2, %%T3, %%T7, 0x01 + vpslldq %%T2, %%T2, 8 ; shift-L xmm2 2 DWs + + vpxor %%T7, %%T7, %%T2 ; first phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + %ifidn %%ENC_DEC, ENC + ; Write to the Ciphertext buffer + VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*0], %%XMM1 + VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*1], %%XMM2 + VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*2], %%XMM3 + VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*3], %%XMM4 + VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*4], %%XMM5 + VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*5], %%XMM6 + VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*6], %%XMM7 + %ifidn %%FULL_PARTIAL, full + ;; Avoid writing past the buffer if handling a partial block + VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*7], %%XMM8 + %endif + %endif + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;second phase of the reduction + vpclmulqdq %%T2, %%T3, %%T7, 0x00 + vpsrldq %%T2, %%T2, 4 ; shift-R xmm2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) + + vpclmulqdq %%T4, %%T3, %%T7, 0x10 + vpslldq %%T4, %%T4, 4 ; shift-L xmm0 1 DW (Shift-L 1-DW to obtain result with no shifts) + + vpxor %%T4, %%T4, %%T2 ; second phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vpxor %%T1, %%T1, %%T4 ; the result is in %%T1 + + vpshufb %%XMM1, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM2, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM3, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM4, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM5, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM6, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM7, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM8, [rel SHUF_MASK] ; perform a 16Byte swap + + + vpxor %%XMM1, %%T1 + + +%endmacro ; GHASH_8_ENCRYPT_8_PARALLEL + + +; GHASH the last 4 ciphertext blocks. +%macro GHASH_LAST_8 16 +%define %%GDATA %1 +%define %%T1 %2 +%define %%T2 %3 +%define %%T3 %4 +%define %%T4 %5 +%define %%T5 %6 +%define %%T6 %7 +%define %%T7 %8 +%define %%XMM1 %9 +%define %%XMM2 %10 +%define %%XMM3 %11 +%define %%XMM4 %12 +%define %%XMM5 %13 +%define %%XMM6 %14 +%define %%XMM7 %15 +%define %%XMM8 %16 + + ;; Karatsuba Method + + vmovdqu %%T5, [%%GDATA + HashKey_8] + + vpshufd %%T2, %%XMM1, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM1 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T6, %%XMM1, %%T5, 0x11 + vpclmulqdq %%T7, %%XMM1, %%T5, 0x00 + + vpclmulqdq %%XMM1, %%T2, %%T3, 0x00 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_7] + vpshufd %%T2, %%XMM2, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM2 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM2, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM2, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_6] + vpshufd %%T2, %%XMM3, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM3 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM3, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM3, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_5] + vpshufd %%T2, %%XMM4, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM4 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM4, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM4, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_4] + vpshufd %%T2, %%XMM5, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM5 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM5, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM5, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_3] + vpshufd %%T2, %%XMM6, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM6 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM6, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM6, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_2] + vpshufd %%T2, %%XMM7, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM7 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM7, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM7, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey] + vpshufd %%T2, %%XMM8, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM8 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM8, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM8, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + vpxor %%XMM1, %%XMM1, %%T6 + vpxor %%T2, %%XMM1, %%T7 + + + + + vpslldq %%T4, %%T2, 8 + vpsrldq %%T2, %%T2, 8 + + vpxor %%T7, %%T7, %%T4 + vpxor %%T6, %%T6, %%T2 ; <%%T6:%%T7> holds the result of the accumulated carry-less multiplications + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;first phase of the reduction + vmovdqu %%T3, [rel POLY2] + + vpclmulqdq %%T2, %%T3, %%T7, 0x01 + vpslldq %%T2, %%T2, 8 ; shift-L xmm2 2 DWs + + vpxor %%T7, %%T7, %%T2 ; first phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + + ;second phase of the reduction + vpclmulqdq %%T2, %%T3, %%T7, 0x00 + vpsrldq %%T2, %%T2, 4 ; shift-R %%T2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) + + vpclmulqdq %%T4, %%T3, %%T7, 0x10 + vpslldq %%T4, %%T4, 4 ; shift-L %%T4 1 DW (Shift-L 1-DW to obtain result with no shifts) + + vpxor %%T4, %%T4, %%T2 ; second phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vpxor %%T6, %%T6, %%T4 ; the result is in %%T6 +%endmacro + + +; GHASH the last 4 ciphertext blocks. +%macro GHASH_LAST_7 15 +%define %%GDATA %1 +%define %%T1 %2 +%define %%T2 %3 +%define %%T3 %4 +%define %%T4 %5 +%define %%T5 %6 +%define %%T6 %7 +%define %%T7 %8 +%define %%XMM1 %9 +%define %%XMM2 %10 +%define %%XMM3 %11 +%define %%XMM4 %12 +%define %%XMM5 %13 +%define %%XMM6 %14 +%define %%XMM7 %15 + + ;; Karatsuba Method + + vmovdqu %%T5, [%%GDATA + HashKey_7] + + vpshufd %%T2, %%XMM1, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM1 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T6, %%XMM1, %%T5, 0x11 + vpclmulqdq %%T7, %%XMM1, %%T5, 0x00 + + vpclmulqdq %%XMM1, %%T2, %%T3, 0x00 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_6] + vpshufd %%T2, %%XMM2, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM2 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM2, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM2, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_5] + vpshufd %%T2, %%XMM3, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM3 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM3, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM3, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_4] + vpshufd %%T2, %%XMM4, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM4 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM4, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM4, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_3] + vpshufd %%T2, %%XMM5, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM5 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM5, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM5, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_2] + vpshufd %%T2, %%XMM6, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM6 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM6, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM6, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_1] + vpshufd %%T2, %%XMM7, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM7 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM7, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM7, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vpxor %%XMM1, %%XMM1, %%T6 + vpxor %%T2, %%XMM1, %%T7 + + + + + vpslldq %%T4, %%T2, 8 + vpsrldq %%T2, %%T2, 8 + + vpxor %%T7, %%T7, %%T4 + vpxor %%T6, %%T6, %%T2 ; <%%T6:%%T7> holds the result of the accumulated carry-less multiplications + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;first phase of the reduction + vmovdqu %%T3, [rel POLY2] + + vpclmulqdq %%T2, %%T3, %%T7, 0x01 + vpslldq %%T2, %%T2, 8 ; shift-L xmm2 2 DWs + + vpxor %%T7, %%T7, %%T2 ; first phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + + ;second phase of the reduction + vpclmulqdq %%T2, %%T3, %%T7, 0x00 + vpsrldq %%T2, %%T2, 4 ; shift-R %%T2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) + + vpclmulqdq %%T4, %%T3, %%T7, 0x10 + vpslldq %%T4, %%T4, 4 ; shift-L %%T4 1 DW (Shift-L 1-DW to obtain result with no shifts) + + vpxor %%T4, %%T4, %%T2 ; second phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vpxor %%T6, %%T6, %%T4 ; the result is in %%T6 +%endmacro + + + +;;; Handle encryption of the final partial block +;;; IN: +;;; r13 - Number of bytes to read +;;; MODIFIES: +;;; KEY - Key for encrypting the partial block +;;; SMASHES: +;;; rax, T1 +;;; Note: +;;; PLAIN_CYPH_LEN is unused at this stage. Previously: +;;; it was used to determine if buffer is big enough to do +;;; a 16 byte read & shift. +;;; 'LT16' is passed here only if buffer is known to be smaller +;;; than 16 bytes. +;;; Any other value passed here will result in 16 byte read +;;; code path. +%macro ENCRYPT_FINAL_PARTIAL_BLOCK 7 +%define %%KEY %1 +%define %%T1 %2 +%define %%CYPH_PLAIN_OUT %3 +%define %%PLAIN_CYPH_IN %4 +%define %%PLAIN_CYPH_LEN %5 +%define %%ENC_DEC %6 +%define %%DATA_OFFSET %7 + + ;; %%PLAIN_CYPH_IN + %%DATA_OFFSET + ;; - input data address + ;; r13 - input data length + ;; rax - temp registers + ;; out: + ;; T1 - packed output + ;; k1 - valid byte mask + READ_SMALL_DATA_INPUT %%T1, %%PLAIN_CYPH_IN+%%DATA_OFFSET, r13, rax + + ;; At this point T1 contains the partial block data + ;; Plaintext XOR E(K, Yn) + vpxorq %%KEY, %%KEY, %%T1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; Output r13 Bytes + vmovdqu8 [%%CYPH_PLAIN_OUT + %%DATA_OFFSET]{k1}, %%KEY + +%ifidn %%ENC_DEC, DEC + ;; If decrypt, restore the ciphertext into %%KEY + vmovdqa64 %%KEY, %%T1 +%else + vmovdqu8 %%KEY{k1}{z}, %%KEY +%endif +%endmacro ; ENCRYPT_FINAL_PARTIAL_BLOCK + + + +; Encryption of a single block +%macro ENCRYPT_SINGLE_BLOCK 2 +%define %%GDATA %1 +%define %%XMM0 %2 + + vpxor %%XMM0, %%XMM0, [%%GDATA+16*0] +%assign i 1 +%rep NROUNDS + vaesenc %%XMM0, [%%GDATA+16*i] +%assign i (i+1) +%endrep + vaesenclast %%XMM0, [%%GDATA+16*i] +%endmacro + + +;; Start of Stack Setup + +%macro FUNC_SAVE 0 + ;; Required for Update/GMC_ENC + ;the number of pushes must equal STACK_OFFSET + push r12 + push r13 + push r14 + push r15 + mov r14, rsp + + sub rsp, VARIABLE_OFFSET + and rsp, ~63 + +%ifidn __OUTPUT_FORMAT__, win64 + ; xmm6:xmm15 need to be maintained for Windows + vmovdqu [rsp + LOCAL_STORAGE + 0*16],xmm6 + vmovdqu [rsp + LOCAL_STORAGE + 1*16],xmm7 + vmovdqu [rsp + LOCAL_STORAGE + 2*16],xmm8 + vmovdqu [rsp + LOCAL_STORAGE + 3*16],xmm9 + vmovdqu [rsp + LOCAL_STORAGE + 4*16],xmm10 + vmovdqu [rsp + LOCAL_STORAGE + 5*16],xmm11 + vmovdqu [rsp + LOCAL_STORAGE + 6*16],xmm12 + vmovdqu [rsp + LOCAL_STORAGE + 7*16],xmm13 + vmovdqu [rsp + LOCAL_STORAGE + 8*16],xmm14 + vmovdqu [rsp + LOCAL_STORAGE + 9*16],xmm15 +%endif +%endmacro + + +%macro FUNC_RESTORE 0 + +%ifdef SAFE_DATA + clear_scratch_gps_asm + clear_scratch_zmms_asm +%endif +%ifidn __OUTPUT_FORMAT__, win64 + vmovdqu xmm15, [rsp + LOCAL_STORAGE + 9*16] + vmovdqu xmm14, [rsp + LOCAL_STORAGE + 8*16] + vmovdqu xmm13, [rsp + LOCAL_STORAGE + 7*16] + vmovdqu xmm12, [rsp + LOCAL_STORAGE + 6*16] + vmovdqu xmm11, [rsp + LOCAL_STORAGE + 5*16] + vmovdqu xmm10, [rsp + LOCAL_STORAGE + 4*16] + vmovdqu xmm9, [rsp + LOCAL_STORAGE + 3*16] + vmovdqu xmm8, [rsp + LOCAL_STORAGE + 2*16] + vmovdqu xmm7, [rsp + LOCAL_STORAGE + 1*16] + vmovdqu xmm6, [rsp + LOCAL_STORAGE + 0*16] +%endif +;; Required for Update/GMC_ENC + mov rsp, r14 + pop r15 + pop r14 + pop r13 + pop r12 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; UPDATE_HASH: Calculates the hash of the data which will not be encrypted. +; Input: The input data (A_IN), that data's length (A_LEN), and the hash key (HASH_KEY). +; Output: The hash of the data (AAD_HASH). +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro UPDATE_HASH 13 +%define %%A_IN %1 +%define %%A_LEN %2 +%define %%AAD_HASH %3 +%define %%GDATA_KEY %4 +%define %%XTMP0 %5 ; xmm temp reg 5 +%define %%XTMP1 %6 ; xmm temp reg 5 +%define %%XTMP2 %7 +%define %%XTMP3 %8 +%define %%XTMP4 %9 +%define %%XTMP5 %10 ; xmm temp reg 5 +%define %%T1 %11 ; temp reg 1 +%define %%T2 %12 +%define %%T3 %13 + + + mov %%T1, %%A_IN ; T1 = AAD + mov %%T2, %%A_LEN ; T2 = aadLen + + or %%T2, %%T2 + jz %%_CALC_AAD_done + +%%_get_AAD_loop128: + cmp %%T2, 128 + jl %%_exit_AAD_loop128 + + vmovdqu %%XTMP0, [%%T1 + 16*0] + vpshufb %%XTMP0, [rel SHUF_MASK] + + vpxor %%XTMP0, %%AAD_HASH + + vmovdqu %%XTMP5, [%%GDATA_KEY + HashKey_8] + vpclmulqdq %%XTMP1, %%XTMP0, %%XTMP5, 0x11 ; %%T1 = a1*b1 + vpclmulqdq %%XTMP2, %%XTMP0, %%XTMP5, 0x00 ; %%T2 = a0*b0 + vpclmulqdq %%XTMP3, %%XTMP0, %%XTMP5, 0x01 ; %%T3 = a1*b0 + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x10 ; %%T4 = a0*b1 + vpxor %%XTMP3, %%XTMP3, %%XTMP4 ; %%T3 = a1*b0 + a0*b1 + +%assign i 1 +%assign j 7 +%rep 7 + vmovdqu %%XTMP0, [%%T1 + 16*i] + vpshufb %%XTMP0, [rel SHUF_MASK] + + vmovdqu %%XTMP5, [%%GDATA_KEY + HashKey_ %+ j] + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x11 ; %%T1 = T1 + a1*b1 + vpxor %%XTMP1, %%XTMP1, %%XTMP4 + + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x00 ; %%T2 = T2 + a0*b0 + vpxor %%XTMP2, %%XTMP2, %%XTMP4 + + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x01 ; %%T3 = T3 + a1*b0 + a0*b1 + vpxor %%XTMP3, %%XTMP3, %%XTMP4 + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x10 + vpxor %%XTMP3, %%XTMP3, %%XTMP4 +%assign i (i + 1) +%assign j (j - 1) +%endrep + + vpslldq %%XTMP4, %%XTMP3, 8 ; shift-L 2 DWs + vpsrldq %%XTMP3, %%XTMP3, 8 ; shift-R 2 DWs + vpxor %%XTMP2, %%XTMP2, %%XTMP4 + vpxor %%XTMP1, %%XTMP1, %%XTMP3 ; accumulate the results in %%T1(M):%%T2(L) + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;first phase of the reduction + vmovdqa %%XTMP5, [rel POLY2] + vpclmulqdq %%XTMP0, %%XTMP5, %%XTMP2, 0x01 + vpslldq %%XTMP0, %%XTMP0, 8 ; shift-L xmm2 2 DWs + vpxor %%XTMP2, %%XTMP2, %%XTMP0 ; first phase of the reduction complete + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;second phase of the reduction + vpclmulqdq %%XTMP3, %%XTMP5, %%XTMP2, 0x00 + vpsrldq %%XTMP3, %%XTMP3, 4 ; shift-R 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) + + vpclmulqdq %%XTMP4, %%XTMP5, %%XTMP2, 0x10 + vpslldq %%XTMP4, %%XTMP4, 4 ; shift-L 1 DW (Shift-L 1-DW to obtain result with no shifts) + + vpxor %%XTMP4, %%XTMP4, %%XTMP3 ; second phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vpxor %%AAD_HASH, %%XTMP1, %%XTMP4 ; the result is in %%T1 + + sub %%T2, 128 + je %%_CALC_AAD_done + + add %%T1, 128 + jmp %%_get_AAD_loop128 + +%%_exit_AAD_loop128: + cmp %%T2, 16 + jl %%_get_small_AAD_block + + ;; calculate hash_key position to start with + mov %%T3, %%T2 + and %%T3, -16 ; 1 to 7 blocks possible here + neg %%T3 + add %%T3, HashKey_1 + 16 + lea %%T3, [%%GDATA_KEY + %%T3] + + vmovdqu %%XTMP0, [%%T1] + vpshufb %%XTMP0, [rel SHUF_MASK] + + vpxor %%XTMP0, %%AAD_HASH + + vmovdqu %%XTMP5, [%%T3] + vpclmulqdq %%XTMP1, %%XTMP0, %%XTMP5, 0x11 ; %%T1 = a1*b1 + vpclmulqdq %%XTMP2, %%XTMP0, %%XTMP5, 0x00 ; %%T2 = a0*b0 + vpclmulqdq %%XTMP3, %%XTMP0, %%XTMP5, 0x01 ; %%T3 = a1*b0 + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x10 ; %%T4 = a0*b1 + vpxor %%XTMP3, %%XTMP3, %%XTMP4 ; %%T3 = a1*b0 + a0*b1 + + add %%T3, 16 ; move to next hashkey + add %%T1, 16 ; move to next data block + sub %%T2, 16 + cmp %%T2, 16 + jl %%_AAD_reduce + +%%_AAD_blocks: + vmovdqu %%XTMP0, [%%T1] + vpshufb %%XTMP0, [rel SHUF_MASK] + + vmovdqu %%XTMP5, [%%T3] + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x11 ; %%T1 = T1 + a1*b1 + vpxor %%XTMP1, %%XTMP1, %%XTMP4 + + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x00 ; %%T2 = T2 + a0*b0 + vpxor %%XTMP2, %%XTMP2, %%XTMP4 + + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x01 ; %%T3 = T3 + a1*b0 + a0*b1 + vpxor %%XTMP3, %%XTMP3, %%XTMP4 + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x10 + vpxor %%XTMP3, %%XTMP3, %%XTMP4 + + add %%T3, 16 ; move to next hashkey + add %%T1, 16 + sub %%T2, 16 + cmp %%T2, 16 + jl %%_AAD_reduce + jmp %%_AAD_blocks + +%%_AAD_reduce: + vpslldq %%XTMP4, %%XTMP3, 8 ; shift-L 2 DWs + vpsrldq %%XTMP3, %%XTMP3, 8 ; shift-R 2 DWs + vpxor %%XTMP2, %%XTMP2, %%XTMP4 + vpxor %%XTMP1, %%XTMP1, %%XTMP3 ; accumulate the results in %%T1(M):%%T2(L) + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;first phase of the reduction + vmovdqa %%XTMP5, [rel POLY2] + vpclmulqdq %%XTMP0, %%XTMP5, %%XTMP2, 0x01 + vpslldq %%XTMP0, %%XTMP0, 8 ; shift-L xmm2 2 DWs + vpxor %%XTMP2, %%XTMP2, %%XTMP0 ; first phase of the reduction complete + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;second phase of the reduction + vpclmulqdq %%XTMP3, %%XTMP5, %%XTMP2, 0x00 + vpsrldq %%XTMP3, %%XTMP3, 4 ; shift-R 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) + + vpclmulqdq %%XTMP4, %%XTMP5, %%XTMP2, 0x10 + vpslldq %%XTMP4, %%XTMP4, 4 ; shift-L 1 DW (Shift-L 1-DW to obtain result with no shifts) + + vpxor %%XTMP4, %%XTMP4, %%XTMP3 ; second phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vpxor %%AAD_HASH, %%XTMP1, %%XTMP4 ; the result is in %%T1 + + or %%T2, %%T2 + je %%_CALC_AAD_done + +%%_get_small_AAD_block: + vmovdqu %%XTMP0, [%%GDATA_KEY + HashKey] + READ_SMALL_DATA_INPUT %%XTMP1, %%T1, %%T2, %%T3 + ;byte-reflect the AAD data + vpshufb %%XTMP1, [rel SHUF_MASK] + vpxor %%AAD_HASH, %%XTMP1 + GHASH_MUL %%AAD_HASH, %%XTMP0, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5 + +%%_CALC_AAD_done: + +%endmacro ; UPDATE_HASH + +%macro CALC_J0 14 +%define %%KEY %1 ;; [in] Pointer to GCM KEY structure +%define %%IV %2 ;; [in] Pointer to IV +%define %%IV_LEN %3 ;; [in] IV length +%define %%J0 %4 ;; [out] XMM reg to contain J0 +%define %%TMP0 %5 ;; [clobbered] Temporary GP reg +%define %%TMP1 %6 ;; [clobbered] Temporary GP reg +%define %%TMP2 %7 ;; [clobbered] Temporary GP reg +%define %%XTMP0 %8 ;; [clobbered] Temporary XMM reg +%define %%XTMP1 %9 ;; [clobbered] Temporary XMM reg +%define %%XTMP2 %10 ;; [clobbered] Temporary XMM reg +%define %%XTMP3 %11 ;; [clobbered] Temporary XMM reg +%define %%XTMP4 %12 ;; [clobbered] Temporary XMM reg +%define %%XTMP5 %13 ;; [clobbered] Temporary XMM reg +%define %%IV_GEN_LEN %14 ;; [in] IV general length + + ;; J0 = GHASH(IV || 0s+64 || len(IV)64) + ;; s = 16 * RoundUp(len(IV)/16) - len(IV) */ + + ;; Calculate GHASH of (IV || 0s) + UPDATE_HASH %%IV, %%IV_LEN, %%J0, %%KEY, %%XTMP0, %%XTMP1, %%XTMP2, \ + %%XTMP3, %%XTMP4, %%XTMP5, %%TMP0, %%TMP1, %%TMP2 + + ;; Calculate GHASH of last 16-byte block (0 || len(IV)64) + vmovdqu %%XTMP0, [%%KEY + HashKey] + mov %%TMP2, %%IV_GEN_LEN + shl %%TMP2, 3 ;; IV length in bits + vmovq %%XTMP1, %%TMP2 + vpxor %%J0, %%XTMP1 + GHASH_MUL %%J0, %%XTMP0, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5 + + vpshufb %%J0, [rel SHUF_MASK] ; perform a 16Byte swap +%endmacro + +%macro GCM_ENC_DEC_SMALL 12 +%define %%GDATA_KEY %1 +%define %%GDATA_CTX %2 +%define %%CYPH_PLAIN_OUT %3 +%define %%PLAIN_CYPH_IN %4 +%define %%PLAIN_CYPH_LEN %5 +%define %%ENC_DEC %6 +%define %%DATA_OFFSET %7 +%define %%LENGTH %8 ; assumed r13 +%define %%NUM_BLOCKS %9 +%define %%CTR %10 ; assumed xmm9 +%define %%HASH_OUT %11 ; assumed xmm14 +%define %%INSTANCE_TYPE %12 + + ;; NOTE: the check below is obsolete in current implementation. The check is already done in GCM_ENC_DEC. + ;; cmp %%NUM_BLOCKS, 0 + ;; je %%_small_initial_blocks_encrypted + cmp %%NUM_BLOCKS, 8 + je %%_small_initial_num_blocks_is_8 + cmp %%NUM_BLOCKS, 7 + je %%_small_initial_num_blocks_is_7 + cmp %%NUM_BLOCKS, 6 + je %%_small_initial_num_blocks_is_6 + cmp %%NUM_BLOCKS, 5 + je %%_small_initial_num_blocks_is_5 + cmp %%NUM_BLOCKS, 4 + je %%_small_initial_num_blocks_is_4 + cmp %%NUM_BLOCKS, 3 + je %%_small_initial_num_blocks_is_3 + cmp %%NUM_BLOCKS, 2 + je %%_small_initial_num_blocks_is_2 + + jmp %%_small_initial_num_blocks_is_1 + + +%%_small_initial_num_blocks_is_8: + ;; r13 - %%LENGTH + ;; xmm12 - T1 + ;; xmm13 - T2 + ;; xmm14 - T3 - AAD HASH OUT when not producing 8 AES keys + ;; xmm15 - T4 + ;; xmm11 - T5 + ;; xmm9 - CTR + ;; xmm1 - XMM1 - Cipher + Hash when producing 8 AES keys + ;; xmm2 - XMM2 + ;; xmm3 - XMM3 + ;; xmm4 - XMM4 + ;; xmm5 - XMM5 + ;; xmm6 - XMM6 + ;; xmm7 - XMM7 + ;; xmm8 - XMM8 - AAD HASH IN + ;; xmm10 - T6 + ;; xmm0 - T_key + INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, %%DATA_OFFSET, 8, \ + xmm12, xmm13, %%HASH_OUT, xmm15, xmm11, %%CTR, \ + xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, \ + xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_7: + INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, %%DATA_OFFSET, 7, \ + xmm12, xmm13, %%HASH_OUT, xmm15, xmm11, %%CTR, \ + xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, \ + xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_6: + INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, %%DATA_OFFSET, 6, \ + xmm12, xmm13, %%HASH_OUT, xmm15, xmm11, %%CTR, \ + xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, \ + xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_5: + INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, %%DATA_OFFSET, 5, \ + xmm12, xmm13, %%HASH_OUT, xmm15, xmm11, %%CTR, \ + xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, \ + xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_4: + INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, %%DATA_OFFSET, 4, \ + xmm12, xmm13, %%HASH_OUT, xmm15, xmm11, %%CTR, \ + xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, \ + xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_3: + INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, %%DATA_OFFSET, 3, \ + xmm12, xmm13, %%HASH_OUT, xmm15, xmm11, %%CTR, \ + xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, \ + xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_2: + INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, %%DATA_OFFSET, 2, \ + xmm12, xmm13, %%HASH_OUT, xmm15, xmm11, %%CTR, \ + xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, \ + xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_1: + INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, %%DATA_OFFSET, 1, \ + xmm12, xmm13, %%HASH_OUT, xmm15, xmm11, %%CTR, \ + xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, \ + xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE +%%_small_initial_blocks_encrypted: + +%endmacro ; GCM_ENC_DEC_SMALL + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; GCM_ENC_DEC Encodes/Decodes given data. Assumes that the passed gcm_context_data struct +; has been initialized by GCM_INIT +; Requires the input data be at least 1 byte long because of READ_SMALL_INPUT_DATA. +; Input: gcm_key_data struct* (GDATA_KEY), gcm_context_data *(GDATA_CTX), input text (PLAIN_CYPH_IN), +; input text length (PLAIN_CYPH_LEN) and whether encoding or decoding (ENC_DEC). +; Output: A cypher of the given plain text (CYPH_PLAIN_OUT), and updated GDATA_CTX +; Clobbers rax, r10-r15, and xmm0-xmm15 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro GCM_ENC_DEC 7 +%define %%GDATA_KEY %1 +%define %%GDATA_CTX %2 +%define %%CYPH_PLAIN_OUT %3 +%define %%PLAIN_CYPH_IN %4 +%define %%PLAIN_CYPH_LEN %5 +%define %%ENC_DEC %6 +%define %%INSTANCE_TYPE %7 +%define %%DATA_OFFSET r11 + +; Macro flow: +; calculate the number of 16byte blocks in the message +; process (number of 16byte blocks) mod 8 '%%_initial_num_blocks_is_# .. %%_initial_blocks_encrypted' +; process 8 16 byte blocks at a time until all are done '%%_encrypt_by_8_new .. %%_eight_cipher_left' +; if there is a block of less tahn 16 bytes process it '%%_zero_cipher_left .. %%_multiple_of_16_bytes' + +%ifidn __OUTPUT_FORMAT__, win64 + cmp %%PLAIN_CYPH_LEN, 0 +%else + or %%PLAIN_CYPH_LEN, %%PLAIN_CYPH_LEN +%endif + je %%_enc_dec_done + + xor %%DATA_OFFSET, %%DATA_OFFSET + ;; Update length of data processed +%ifidn __OUTPUT_FORMAT__, win64 + mov rax, %%PLAIN_CYPH_LEN + add [%%GDATA_CTX + InLen], rax +%else + add [%%GDATA_CTX + InLen], %%PLAIN_CYPH_LEN +%endif + vmovdqu xmm13, [%%GDATA_KEY + HashKey] + vmovdqu xmm8, [%%GDATA_CTX + AadHash] + +%ifidn %%INSTANCE_TYPE, multi_call + ;; NOTE: partial block processing makes only sense for multi_call here. + ;; Used for the update flow - if there was a previous partial + ;; block fill the remaining bytes here. + PARTIAL_BLOCK %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%PLAIN_CYPH_LEN, %%DATA_OFFSET, xmm8, %%ENC_DEC +%endif + + ;; lift CTR set from initial_blocks to here +%ifidn %%INSTANCE_TYPE, single_call + vmovdqu xmm9, xmm2 +%else + vmovdqu xmm9, [%%GDATA_CTX + CurCount] +%endif + + ;; Save the amount of data left to process in r10 + mov r13, %%PLAIN_CYPH_LEN +%ifidn %%INSTANCE_TYPE, multi_call + ;; NOTE: %%DATA_OFFSET is zero in single_call case. + ;; Consequently PLAIN_CYPH_LEN will never be zero after + ;; %%DATA_OFFSET subtraction below. + sub r13, %%DATA_OFFSET + + ;; There may be no more data if it was consumed in the partial block. + cmp r13, 0 + je %%_enc_dec_done +%endif ; %%INSTANCE_TYPE, multi_call + mov r10, r13 + + ;; Determine how many blocks to process in INITIAL + mov r12, r13 + shr r12, 4 + and r12, 7 + + ;; Process one additional block in INITIAL if there is a partial block + and r10, 0xf + blsmsk r10, r10 ; Set CF if zero + cmc ; Flip CF + adc r12, 0x0 ; Process an additional INITIAL block if CF set + + ;; Less than 127B will be handled by the small message code, which + ;; can process up to 7 16B blocks. + cmp r13, 128 + jge %%_large_message_path + + GCM_ENC_DEC_SMALL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%PLAIN_CYPH_LEN, %%ENC_DEC, %%DATA_OFFSET, r13, r12, xmm9, xmm14, %%INSTANCE_TYPE + jmp %%_ghash_done + +%%_large_message_path: + and r12, 0x7 ; Still, don't allow 8 INITIAL blocks since this will + ; can be handled by the x8 partial loop. + + cmp r12, 0 + je %%_initial_num_blocks_is_0 + cmp r12, 7 + je %%_initial_num_blocks_is_7 + cmp r12, 6 + je %%_initial_num_blocks_is_6 + cmp r12, 5 + je %%_initial_num_blocks_is_5 + cmp r12, 4 + je %%_initial_num_blocks_is_4 + cmp r12, 3 + je %%_initial_num_blocks_is_3 + cmp r12, 2 + je %%_initial_num_blocks_is_2 + + jmp %%_initial_num_blocks_is_1 + +%%_initial_num_blocks_is_7: + ;; r13 - %%LENGTH + ;; xmm12 - T1 + ;; xmm13 - T2 + ;; xmm14 - T3 - AAD HASH OUT when not producing 8 AES keys + ;; xmm15 - T4 + ;; xmm11 - T5 + ;; xmm9 - CTR + ;; xmm1 - XMM1 - Cipher + Hash when producing 8 AES keys + ;; xmm2 - XMM2 + ;; xmm3 - XMM3 + ;; xmm4 - XMM4 + ;; xmm5 - XMM5 + ;; xmm6 - XMM6 + ;; xmm7 - XMM7 + ;; xmm8 - XMM8 - AAD HASH IN + ;; xmm10 - T6 + ;; xmm0 - T_key + INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 7, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC + jmp %%_initial_blocks_encrypted + +%%_initial_num_blocks_is_6: + INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 6, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC + jmp %%_initial_blocks_encrypted + +%%_initial_num_blocks_is_5: + INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 5, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC + jmp %%_initial_blocks_encrypted + +%%_initial_num_blocks_is_4: + INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 4, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC + jmp %%_initial_blocks_encrypted + +%%_initial_num_blocks_is_3: + INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 3, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC + jmp %%_initial_blocks_encrypted + +%%_initial_num_blocks_is_2: + INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 2, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC + jmp %%_initial_blocks_encrypted + +%%_initial_num_blocks_is_1: + INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 1, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC + jmp %%_initial_blocks_encrypted + +%%_initial_num_blocks_is_0: + INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 0, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC + + +%%_initial_blocks_encrypted: + ;; The entire message was encrypted processed in initial and now need to be hashed + cmp r13, 0 + je %%_encrypt_done + + ;; Encrypt the final <16 byte (partial) block, then hash + cmp r13, 16 + jl %%_encrypt_final_partial + + ;; Process 7 full blocks plus a partial block + cmp r13, 128 + jl %%_encrypt_by_8_partial + + +%%_encrypt_by_8_parallel: + ;; in_order vs. out_order is an optimization to increment the counter without shuffling + ;; it back into little endian. r15d keeps track of when we need to increent in order so + ;; that the carry is handled correctly. + vmovd r15d, xmm9 + and r15d, 255 + vpshufb xmm9, [rel SHUF_MASK] + + +%%_encrypt_by_8_new: + cmp r15d, 255-8 + jg %%_encrypt_by_8 + + + + ;; xmm0 - T1 + ;; xmm10 - T2 + ;; xmm11 - T3 + ;; xmm12 - T4 + ;; xmm13 - T5 + ;; xmm14 - T6 + ;; xmm9 - CTR + ;; xmm1 - XMM1 + ;; xmm2 - XMM2 + ;; xmm3 - XMM3 + ;; xmm4 - XMM4 + ;; xmm5 - XMM5 + ;; xmm6 - XMM6 + ;; xmm7 - XMM7 + ;; xmm8 - XMM8 + ;; xmm15 - T7 + add r15b, 8 + GHASH_8_ENCRYPT_8_PARALLEL %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%DATA_OFFSET, xmm0, xmm10, xmm11, xmm12, xmm13, xmm14, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm15, out_order, %%ENC_DEC, full + add %%DATA_OFFSET, 128 + sub r13, 128 + cmp r13, 128 + jge %%_encrypt_by_8_new + + vpshufb xmm9, [rel SHUF_MASK] + jmp %%_encrypt_by_8_parallel_done + +%%_encrypt_by_8: + vpshufb xmm9, [rel SHUF_MASK] + add r15b, 8 + GHASH_8_ENCRYPT_8_PARALLEL %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%DATA_OFFSET, xmm0, xmm10, xmm11, xmm12, xmm13, xmm14, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm15, in_order, %%ENC_DEC, full + vpshufb xmm9, [rel SHUF_MASK] + add %%DATA_OFFSET, 128 + sub r13, 128 + cmp r13, 128 + jge %%_encrypt_by_8_new + vpshufb xmm9, [rel SHUF_MASK] + + +%%_encrypt_by_8_parallel_done: + ;; Test to see if we need a by 8 with partial block. At this point + ;; bytes remaining should be either zero or between 113-127. + cmp r13, 0 + je %%_encrypt_done + +%%_encrypt_by_8_partial: + ;; Shuffle needed to align key for partial block xor. out_order + ;; is a little faster because it avoids extra shuffles. + ;; TBD: Might need to account for when we don't have room to increment the counter. + + + ;; Process parallel buffers with a final partial block. + GHASH_8_ENCRYPT_8_PARALLEL %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%DATA_OFFSET, xmm0, xmm10, xmm11, xmm12, xmm13, xmm14, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm15, in_order, %%ENC_DEC, partial + + + add %%DATA_OFFSET, 128-16 + sub r13, 128-16 + +%%_encrypt_final_partial: + + vpshufb xmm8, [rel SHUF_MASK] + mov [%%GDATA_CTX + PBlockLen], r13 + vmovdqu [%%GDATA_CTX + PBlockEncKey], xmm8 + + ;; xmm8 - Final encrypted counter - need to hash with partial or full block ciphertext + ;; GDATA, KEY, T1, T2 + ENCRYPT_FINAL_PARTIAL_BLOCK xmm8, xmm0, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%PLAIN_CYPH_LEN, %%ENC_DEC, %%DATA_OFFSET + + vpshufb xmm8, [rel SHUF_MASK] + + +%%_encrypt_done: + + ;; Mapping to macro parameters + ;; IN: + ;; xmm9 contains the counter + ;; xmm1-xmm8 contain the xor'd ciphertext + ;; OUT: + ;; xmm14 contains the final hash + ;; GDATA, T1, T2, T3, T4, T5, T6, T7, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8 +%ifidn %%INSTANCE_TYPE, multi_call + mov r13, [%%GDATA_CTX + PBlockLen] + cmp r13, 0 + jz %%_hash_last_8 + GHASH_LAST_7 %%GDATA_KEY, xmm0, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 + ;; XOR the partial word into the hash + vpxor xmm14, xmm14, xmm8 + jmp %%_ghash_done +%endif +%%_hash_last_8: + GHASH_LAST_8 %%GDATA_KEY, xmm0, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8 + +%%_ghash_done: + vmovdqu [%%GDATA_CTX + CurCount], xmm9 ; my_ctx_data.current_counter = xmm9 + vmovdqu [%%GDATA_CTX + AadHash], xmm14 ; my_ctx_data.aad hash = xmm14 + +%%_enc_dec_done: + + +%endmacro ; GCM_ENC_DEC + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; GCM_COMPLETE Finishes Encryption/Decryption of last partial block after GCM_UPDATE finishes. +; Input: A gcm_key_data * (GDATA_KEY), gcm_context_data (GDATA_CTX) and whether encoding or decoding (ENC_DEC). +; Output: Authorization Tag (AUTH_TAG) and Authorization Tag length (AUTH_TAG_LEN) +; Clobbers rax, r10-r12, and xmm0-xmm2, xmm5-xmm6, xmm9-xmm11, xmm13-xmm15 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro GCM_COMPLETE 6 +%define %%GDATA_KEY %1 +%define %%GDATA_CTX %2 +%define %%AUTH_TAG %3 +%define %%AUTH_TAG_LEN %4 +%define %%ENC_DEC %5 +%define %%INSTANCE_TYPE %6 +%define %%PLAIN_CYPH_LEN rax + + vmovdqu xmm13, [%%GDATA_KEY + HashKey] + ;; Start AES as early as possible + vmovdqu xmm9, [%%GDATA_CTX + OrigIV] ; xmm9 = Y0 + ENCRYPT_SINGLE_BLOCK %%GDATA_KEY, xmm9 ; E(K, Y0) + +%ifidn %%INSTANCE_TYPE, multi_call + ;; If the GCM function is called as a single function call rather + ;; than invoking the individual parts (init, update, finalize) we + ;; can remove a write to read dependency on AadHash. + vmovdqu xmm14, [%%GDATA_CTX + AadHash] + + ;; Encrypt the final partial block. If we did this as a single call then + ;; the partial block was handled in the main GCM_ENC_DEC macro. + mov r12, [%%GDATA_CTX + PBlockLen] + cmp r12, 0 + + je %%_partial_done + + GHASH_MUL xmm14, xmm13, xmm0, xmm10, xmm11, xmm5, xmm6 ;GHASH computation for the last <16 Byte block + vmovdqu [%%GDATA_CTX + AadHash], xmm14 + +%%_partial_done: + +%endif + + mov r12, [%%GDATA_CTX + AadLen] ; r12 = aadLen (number of bytes) + mov %%PLAIN_CYPH_LEN, [%%GDATA_CTX + InLen] + + shl r12, 3 ; convert into number of bits + vmovd xmm15, r12d ; len(A) in xmm15 + + shl %%PLAIN_CYPH_LEN, 3 ; len(C) in bits (*128) + vmovq xmm1, %%PLAIN_CYPH_LEN + vpslldq xmm15, xmm15, 8 ; xmm15 = len(A)|| 0x0000000000000000 + vpxor xmm15, xmm15, xmm1 ; xmm15 = len(A)||len(C) + + vpxor xmm14, xmm15 + GHASH_MUL xmm14, xmm13, xmm0, xmm10, xmm11, xmm5, xmm6 + vpshufb xmm14, [rel SHUF_MASK] ; perform a 16Byte swap + + vpxor xmm9, xmm9, xmm14 + + +%%_return_T: + mov r10, %%AUTH_TAG ; r10 = authTag + mov r11, %%AUTH_TAG_LEN ; r11 = auth_tag_len + + cmp r11, 16 + je %%_T_16 + + cmp r11, 12 + je %%_T_12 + + cmp r11, 8 + je %%_T_8 + + simd_store_avx r10, xmm9, r11, r12, rax + jmp %%_return_T_done +%%_T_8: + vmovq rax, xmm9 + mov [r10], rax + jmp %%_return_T_done +%%_T_12: + vmovq rax, xmm9 + mov [r10], rax + vpsrldq xmm9, xmm9, 8 + vmovd eax, xmm9 + mov [r10 + 8], eax + jmp %%_return_T_done +%%_T_16: + vmovdqu [r10], xmm9 + +%%_return_T_done: + +;; IPPCP: disabled clearing of context fields, as in IPPCP case GetTag may be called +;; several times and it should not stop GCM processing. Context zeroing is done by user. +;; https://www.intel.com/content/www/us/en/docs/ipp-crypto/developer-reference/current/aes-gcm-functions.html +;; %ifdef SAFE_DATA +;; ;; Clear sensitive data from context structure +;; vpxor xmm0, xmm0 +;; vmovdqu [%%GDATA_CTX + AadHash], xmm0 +;; vmovdqu [%%GDATA_CTX + PBlockEncKey], xmm0 +;; %endif +%endmacro ; GCM_COMPLETE + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; GCM_UPDATE_AAD_HASH update AadHash value in gcm_context_data struct to prepare for encoding/decoding. +;;; Input: gcm_key_data * (GDATA_KEY), gcm_context_data *(GDATA_CTX), +;;; Additional Authentication data (A_IN), Additional Data length (A_LEN). +;;; Output: Updated GDATA_CTX with the hash of A_IN (AadHash). +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro GCM_UPDATE_AAD_HASH 7 +%define %%GDATA_KEY %1 ; [in] GCM expanded keys pointer +%define %%GDATA_CTX %2 ; [in] GCM context pointer +%define %%A_IN %3 ; [in] AAD pointer +%define %%A_LEN %4 ; [in] AAD length in bytes +%define %%GPR1 %5 ; temp GPR +%define %%GPR2 %6 ; temp GPR +%define %%GPR3 %7 ; temp GPR + +%define %%AAD_HASH xmm14 + + vmovdqu64 %%AAD_HASH, [%%GDATA_CTX + AadHash] ; load current hash + + UPDATE_HASH %%A_IN, %%A_LEN, %%AAD_HASH, %%GDATA_KEY, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, %%GPR1, %%GPR2, %%GPR3 + + vmovdqu64 [%%GDATA_CTX + AadHash], %%AAD_HASH ; store updated hash + +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; GCM_UPDATE_IV_HASH update orig_IV value in gcm_context_data struct to prepare for encoding/decoding. +;;; Input: gcm_key_data * (GDATA_KEY), gcm_context_data *(GDATA_CTX), +;;; Initialization Vector (IV), Initialization Vector length (IV_LEN). +;;; Output: Updated GDATA_CTX with the hash of A_IN (orig_IV). +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro GCM_UPDATE_IV_HASH 7 +%define %%GDATA_KEY %1 ; [in] GCM expanded keys pointer +%define %%GDATA_CTX %2 ; [in] GCM context pointer +%define %%IV %3 ; [in] AAD pointer +%define %%IV_LEN %4 ; [in] AAD length in bytes +%define %%GPR1 %5 ; temp GPR +%define %%GPR2 %6 ; temp GPR +%define %%GPR3 %7 ; temp GPR + +%define %%IV_HASH xmm14 + + vmovdqu64 %%IV_HASH, [%%GDATA_CTX + OrigIV] ; load current hash + + UPDATE_HASH %%IV, %%IV_LEN, %%IV_HASH, %%GDATA_KEY, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, %%GPR1, %%GPR2, %%GPR3 + + vmovdqu64 [%%GDATA_CTX + OrigIV], %%IV_HASH ; store updated hash + +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; GCM_FINALIZE_IV_HASH finalize updating of AadHash value in gcm_context_data struct +;;; to prepare for encoding/decoding. +;;; Input: gcm_key_data * (GDATA_KEY), gcm_context_data *(GDATA_CTX), +;;; Initialization Vector (IV), Initialization Vector length (IV_LEN). +;;; Output: Updated GDATA_CTX with the hash of A_IN (orig_IV). and initialized other parts of GDATA_CTX. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro GCM_FINALIZE_IV_HASH 7-8 +%define %%GDATA_KEY %1 ; [in] GCM expanded keys pointer +%define %%GDATA_CTX %2 ; [in] GCM context pointer +%define %%IV %3 ; [in] IV pointer +%define %%IV_LEN %4 ; [in] IV length in bytes +%define %%GPR1 %5 ; temp GPR +%define %%GPR2 %6 ; temp GPR +%define %%GPR3 %7 ; temp GPR +%define %%IV_GEN_LEN %8 ; [in] IV general length in bytes + + +%if %0 == 7 ;; IV is 12 bytes + + ;; read 12 IV bytes and pad with 0x00000001 + mov %%GPR2, %%IV + vmovd xmm3, [%%GPR2 + 8] + vpslldq xmm3, 8 + vmovq xmm2, [%%GPR2] + vmovdqa xmm4, [rel ONEf] + vpternlogq xmm2, xmm3, xmm4, 0xfe ; xmm2 = xmm2 or xmm3 or xmm4 + +%else ;; IV is not 12 bytes + + vmovdqu xmm2, [%%GDATA_CTX + OrigIV] + + CALC_J0 %%GDATA_KEY, %%IV, %%IV_LEN, xmm2, r10, r11, r12, xmm0, xmm1, \ + xmm3, xmm4, xmm5, xmm6, %%IV_GEN_LEN + +%endif + vmovdqu [%%GDATA_CTX + OrigIV], xmm2 ; ctx_data.orig_IV = iv + + ;; store IV as counter in LE format + vpshufb xmm2, [rel SHUF_MASK] + vmovdqu [%%GDATA_CTX + CurCount], xmm2 ; ctx_data.current_counter = iv + + +%endmacro + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_precomp_128_avx512 / +; aes_gcm_precomp_192_avx512 / +; aes_gcm_precomp_256_avx512 +; (struct gcm_key_data *key_data) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +IPPASM FN_NAME(precomp,_), PUBLIC +;; Parameter is passed through register +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz exit_precomp +%endif + + push r12 + push r13 + push r14 + push r15 + + mov r14, rsp + + + + sub rsp, VARIABLE_OFFSET + and rsp, ~63 ; align rsp to 64 bytes + +%ifidn __OUTPUT_FORMAT__, win64 + ; only xmm6 needs to be maintained + vmovdqu [rsp + LOCAL_STORAGE + 0*16],xmm6 +%endif + + vpxor xmm6, xmm6 + ENCRYPT_SINGLE_BLOCK arg1, xmm6 ; xmm6 = HashKey + + vpshufb xmm6, [rel SHUF_MASK] + ;;;;;;;;;;;;;;; PRECOMPUTATION of HashKey<<1 mod poly from the HashKey;;;;;;;;;;;;;;; + vmovdqa xmm2, xmm6 + vpsllq xmm6, xmm6, 1 + vpsrlq xmm2, xmm2, 63 + vmovdqa xmm1, xmm2 + vpslldq xmm2, xmm2, 8 + vpsrldq xmm1, xmm1, 8 + vpor xmm6, xmm6, xmm2 + ;reduction + vpshufd xmm2, xmm1, 00100100b + vpcmpeqd xmm2, [rel TWOONE] + vpand xmm2, xmm2, [rel POLY] + vpxor xmm6, xmm6, xmm2 ; xmm6 holds the HashKey<<1 mod poly + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vmovdqu [arg1 + HashKey], xmm6 ; store HashKey<<1 mod poly + + + PRECOMPUTE arg1, xmm6, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5 + +%ifdef SAFE_DATA + clear_scratch_gps_asm + clear_scratch_zmms_asm +%endif +%ifidn __OUTPUT_FORMAT__, win64 + vmovdqu xmm6, [rsp + LOCAL_STORAGE + 0*16] +%endif + mov rsp, r14 + + pop r15 + pop r14 + pop r13 + pop r12 + +.exit_precomp: + ret + +ENDFUNC FN_NAME(precomp,_) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_enc_128_update_avx512 / aes_gcm_enc_192_update_avx512 / +; aes_gcm_enc_256_update_avx512 +; (const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; u8 *out, +; const u8 *in, +; u64 plaintext_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +IPPASM FN_NAME(enc,_update_), PUBLIC + + FUNC_SAVE + +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz exit_update_enc + + ;; Check context_data != NULL + cmp arg2, 0 + jz exit_update_enc + + ;; Check if plaintext_len == 0 + cmp arg5, 0 + jz skip_in_out_check_update_enc + + ;; Check out != NULL (plaintext_len != 0) + cmp arg3, 0 + jz exit_update_enc + + ;; Check in != NULL (plaintext_len != 0) + cmp arg4, 0 + jz exit_update_enc + +.skip_in_out_check_update_enc: +%endif + GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, ENC, multi_call + +.exit_update_enc: + FUNC_RESTORE + + ret + +ENDFUNC FN_NAME(enc,_update_) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_dec_128_update_avx512 / aes_gcm_dec_192_update_avx512 / +; aes_gcm_dec_256_update_avx512 +; (const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; u8 *out, +; const u8 *in, +; u64 plaintext_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +IPPASM FN_NAME(dec,_update_), PUBLIC + + FUNC_SAVE + +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz exit_update_dec + + ;; Check context_data != NULL + cmp arg2, 0 + jz exit_update_dec + + ;; Check if plaintext_len == 0 + cmp arg5, 0 + jz skip_in_out_check_update_dec + + ;; Check out != NULL (plaintext_len != 0) + cmp arg3, 0 + jz exit_update_dec + + ;; Check in != NULL (plaintext_len != 0) + cmp arg4, 0 + jz exit_update_dec + +skip_in_out_check_update_dec: +%endif + + GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, DEC, multi_call + +.exit_update_dec: + FUNC_RESTORE + ret + +ENDFUNC FN_NAME(dec,_update_) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_gettag_128_avx512 / aes_gcm_gettag_192_avx512 / +; aes_gcm_gettag_256_avx512 +; (const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; u8 *auth_tag, +; u64 auth_tag_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +IPPASM FN_NAME(gettag,_), PUBLIC + + FUNC_SAVE + +;; All parameters are passed through registers +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz exit_enc_fin + + ;; Check context_data != NULL + cmp arg2, 0 + jz exit_enc_fin + + ;; Check auth_tag != NULL + cmp arg3, 0 + jz exit_enc_fin + + ;; Check auth_tag_len == 0 or > 16 + cmp arg4, 0 + jz exit_enc_fin + + cmp arg4, 16 + ja exit_enc_fin +%endif + + GCM_COMPLETE arg1, arg2, arg3, arg4, ENC, multi_call + +.exit_enc_fin: + FUNC_RESTORE + ret + +ENDFUNC FN_NAME(gettag,_) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; IV and AAD in disconnected buffers processing +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%ifdef GCM128_MODE + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_aad_hash_update_avx512( +; const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; const u8 *aad, +; const u64 aad_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +IPPASM aes_gcm_aad_hash_update_avx512, PUBLIC + + FUNC_SAVE + + GCM_UPDATE_AAD_HASH arg1, arg2, arg3, arg4, r10, r11, r12 + + FUNC_RESTORE + + ret +ENDFUNC aes_gcm_aad_hash_update_avx512 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_iv_hash_update_avx512( +; const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; const u8 *iv, +; const u64 iv_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +IPPASM aes_gcm_iv_hash_update_avx512, PUBLIC + + FUNC_SAVE + + GCM_UPDATE_IV_HASH arg1, arg2, arg3, arg4, r10, r11, r12 + + FUNC_RESTORE + + ret +ENDFUNC aes_gcm_iv_hash_update_avx512 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_iv_hash_finalize_avx512( +; const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; const u8 *iv, +; const u64 iv_len, +; const u64 iv_general_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +IPPASM aes_gcm_iv_hash_finalize_avx512, PUBLIC + + FUNC_SAVE + + cmp arg5, 12 + je .iv_eq_12 + + GCM_FINALIZE_IV_HASH arg1, arg2, arg3, arg4, r10, r11, r12, arg5 + jmp .skip_iv_eq_12 + +.iv_eq_12: + + GCM_FINALIZE_IV_HASH arg1, arg2, arg3, arg4, r10, r11, r12 +.skip_iv_eq_12: + + FUNC_RESTORE + + ret +ENDFUNC aes_gcm_iv_hash_finalize_avx512 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_gmult_avx512( +; const struct gcm_key_data *key_data, +; u8 *ghash) +; +; Function updates |ghash| value by multiplying it on H^1 key. +; Leaf function. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +IPPASM aes_gcm_gmult_avx512, PUBLIC + + vmovdqu64 xmm1, [arg2] + vmovdqu64 xmm2, [arg1 + HashKey_1] + + GHASH_MUL xmm1, xmm2, xmm3, xmm4, xmm5, xmm16, xmm17 + + vmovdqu64 [arg2], xmm1 + + ret +ENDFUNC aes_gcm_gmult_avx512 + + +%endif + +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm_defines.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm_defines.inc new file mode 100644 index 0000000..6339271 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm_defines.inc @@ -0,0 +1,264 @@ +;=============================================================================== +; Copyright (C) 2020 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +%ifndef GCM_DEFINES_ASM_INCLUDED +%define GCM_DEFINES_ASM_INCLUDED + +; +; Authors: +; Erdinc Ozturk +; Vinodh Gopal +; James Guilford + +section .data +default rel + +align 16 +POLY: dq 0x0000000000000001, 0xC200000000000000 + +align 64 +POLY2: + dq 0x00000001C2000000, 0xC200000000000000 + dq 0x00000001C2000000, 0xC200000000000000 + dq 0x00000001C2000000, 0xC200000000000000 + dq 0x00000001C2000000, 0xC200000000000000 + +align 16 +TWOONE: dq 0x0000000000000001, 0x0000000100000000 + +;;; @note Order of these constants should not change. +;;; More specifically, ALL_F should follow SHIFT_MASK, and ZERO should follow ALL_F +align 64 +SHUF_MASK: + dq 0x08090A0B0C0D0E0F, 0x0001020304050607 + dq 0x08090A0B0C0D0E0F, 0x0001020304050607 + dq 0x08090A0B0C0D0E0F, 0x0001020304050607 + dq 0x08090A0B0C0D0E0F, 0x0001020304050607 + +align 16 +SHIFT_MASK: + dq 0x0706050403020100, 0x0f0e0d0c0b0a0908 + +ALL_F: + dq 0xffffffffffffffff, 0xffffffffffffffff + +ZERO: + dq 0x0000000000000000, 0x0000000000000000 + +align 16 +ONE: + dq 0x0000000000000001, 0x0000000000000000 + +align 16 +TWO: + dq 0x0000000000000002, 0x0000000000000000 + +align 16 +ONEf: + dq 0x0000000000000000, 0x0100000000000000 + +align 16 +TWOf: + dq 0x0000000000000000, 0x0200000000000000 + +align 64 +ddq_add_1234: + dq 0x0000000000000001, 0x0000000000000000 + dq 0x0000000000000002, 0x0000000000000000 + dq 0x0000000000000003, 0x0000000000000000 + dq 0x0000000000000004, 0x0000000000000000 + +align 64 +ddq_add_5678: + dq 0x0000000000000005, 0x0000000000000000 + dq 0x0000000000000006, 0x0000000000000000 + dq 0x0000000000000007, 0x0000000000000000 + dq 0x0000000000000008, 0x0000000000000000 + +align 64 +ddq_add_4444: + dq 0x0000000000000004, 0x0000000000000000 + dq 0x0000000000000004, 0x0000000000000000 + dq 0x0000000000000004, 0x0000000000000000 + dq 0x0000000000000004, 0x0000000000000000 + +align 64 +ddq_add_8888: + dq 0x0000000000000008, 0x0000000000000000 + dq 0x0000000000000008, 0x0000000000000000 + dq 0x0000000000000008, 0x0000000000000000 + dq 0x0000000000000008, 0x0000000000000000 + +align 64 +ddq_addbe_1234: + dq 0x0000000000000000, 0x0100000000000000 + dq 0x0000000000000000, 0x0200000000000000 + dq 0x0000000000000000, 0x0300000000000000 + dq 0x0000000000000000, 0x0400000000000000 + +align 64 +ddq_addbe_5678: + dq 0x0000000000000000, 0x0500000000000000 + dq 0x0000000000000000, 0x0600000000000000 + dq 0x0000000000000000, 0x0700000000000000 + dq 0x0000000000000000, 0x0800000000000000 + +align 64 +ddq_addbe_4444: + dq 0x0000000000000000, 0x0400000000000000 + dq 0x0000000000000000, 0x0400000000000000 + dq 0x0000000000000000, 0x0400000000000000 + dq 0x0000000000000000, 0x0400000000000000 + +align 64 +ddq_addbe_8888: + dq 0x0000000000000000, 0x0800000000000000 + dq 0x0000000000000000, 0x0800000000000000 + dq 0x0000000000000000, 0x0800000000000000 + dq 0x0000000000000000, 0x0800000000000000 + +align 64 +byte_len_to_mask_table: + dw 0x0000, 0x0001, 0x0003, 0x0007, + dw 0x000f, 0x001f, 0x003f, 0x007f, + dw 0x00ff, 0x01ff, 0x03ff, 0x07ff, + dw 0x0fff, 0x1fff, 0x3fff, 0x7fff, + dw 0xffff + +align 64 +byte64_len_to_mask_table: + dq 0x0000000000000000, 0x0000000000000001 + dq 0x0000000000000003, 0x0000000000000007 + dq 0x000000000000000f, 0x000000000000001f + dq 0x000000000000003f, 0x000000000000007f + dq 0x00000000000000ff, 0x00000000000001ff + dq 0x00000000000003ff, 0x00000000000007ff + dq 0x0000000000000fff, 0x0000000000001fff + dq 0x0000000000003fff, 0x0000000000007fff + dq 0x000000000000ffff, 0x000000000001ffff + dq 0x000000000003ffff, 0x000000000007ffff + dq 0x00000000000fffff, 0x00000000001fffff + dq 0x00000000003fffff, 0x00000000007fffff + dq 0x0000000000ffffff, 0x0000000001ffffff + dq 0x0000000003ffffff, 0x0000000007ffffff + dq 0x000000000fffffff, 0x000000001fffffff + dq 0x000000003fffffff, 0x000000007fffffff + dq 0x00000000ffffffff, 0x00000001ffffffff + dq 0x00000003ffffffff, 0x00000007ffffffff + dq 0x0000000fffffffff, 0x0000001fffffffff + dq 0x0000003fffffffff, 0x0000007fffffffff + dq 0x000000ffffffffff, 0x000001ffffffffff + dq 0x000003ffffffffff, 0x000007ffffffffff + dq 0x00000fffffffffff, 0x00001fffffffffff + dq 0x00003fffffffffff, 0x00007fffffffffff + dq 0x0000ffffffffffff, 0x0001ffffffffffff + dq 0x0003ffffffffffff, 0x0007ffffffffffff + dq 0x000fffffffffffff, 0x001fffffffffffff + dq 0x003fffffffffffff, 0x007fffffffffffff + dq 0x00ffffffffffffff, 0x01ffffffffffffff + dq 0x03ffffffffffffff, 0x07ffffffffffffff + dq 0x0fffffffffffffff, 0x1fffffffffffffff + dq 0x3fffffffffffffff, 0x7fffffffffffffff + dq 0xffffffffffffffff + +align 64 +mask_out_top_block: + dq 0xffffffffffffffff, 0xffffffffffffffff + dq 0xffffffffffffffff, 0xffffffffffffffff + dq 0xffffffffffffffff, 0xffffffffffffffff + dq 0x0000000000000000, 0x0000000000000000 + +section .text + +;;define the fields of gcm_context_data struct +;; struct gcm_context_data { +;; // init, update and finalize context data +;; uint8_t aad_hash[GCM_BLOCK_LEN]; +;; uint64_t aad_length; +;; uint64_t in_length; +;; uint8_t partial_block_enc_key[GCM_BLOCK_LEN]; +;; uint8_t orig_IV[GCM_BLOCK_LEN]; +;; uint8_t current_counter[GCM_BLOCK_LEN]; +;; uint64_t partial_block_length; +;; }; + +%define AadHash (16*0) ; store current Hash of data which has been input +%define AadLen (16*1) ; store length of input data which will not be encrypted or decrypted +%define InLen ((16*1)+8); store length of input data which will be encrypted or decrypted +%define PBlockEncKey (16*2) ; encryption key for the partial block at the end of the previous update +%define OrigIV (16*3) ; input IV +%define CurCount (16*4) ; Current counter for generation of encryption key +%define PBlockLen (16*5) ; length of partial block at the end of the previous update + +%define reg(q) xmm %+ q +%define regy(q) ymm %+ q +%define regz(q) zmm %+ q + +%ifdef WIN_ABI + %xdefine arg1 rcx + %xdefine arg2 rdx + %xdefine arg3 r8 + %xdefine arg4 r9 + %xdefine arg5 qword [r14 + STACK_OFFSET + 8*5] + %xdefine arg6 qword [r14 + STACK_OFFSET + 8*6] + %xdefine arg7 qword [r14 + STACK_OFFSET + 8*7] + %xdefine arg8 qword [r14 + STACK_OFFSET + 8*8] + %xdefine arg9 qword [r14 + STACK_OFFSET + 8*9] + %xdefine arg10 qword [r14 + STACK_OFFSET + 8*10] +%else + %xdefine arg1 rdi + %xdefine arg2 rsi + %xdefine arg3 rdx + %xdefine arg4 rcx + %xdefine arg5 r8 + %xdefine arg6 r9 + %xdefine arg7 qword [r14 + STACK_OFFSET + 8*1] + %xdefine arg8 qword [r14 + STACK_OFFSET + 8*2] + %xdefine arg9 qword [r14 + STACK_OFFSET + 8*3] + %xdefine arg10 qword [r14 + STACK_OFFSET + 8*4] +%endif + +%ifdef NT_LDST + %define NT_LD + %define NT_ST +%endif + +;;; Use Non-temporal load/stor +%ifdef NT_LD + %define XLDR movntdqa + %define VXLDR vmovntdqa + %define VX512LDR vmovntdqa +%else + %define XLDR movdqu + %define VXLDR vmovdqu + %define VX512LDR vmovdqu8 +%endif + +;;; Use Non-temporal load/stor +%ifdef NT_ST + %define XSTR movntdq + %define VXSTR vmovntdq + %define VX512STR vmovntdq +%else + %define XSTR movdqu + %define VXSTR vmovdqu + %define VX512STR vmovdqu8 +%endif + +%define SAFE_DATA 1 + +%endif ; GCM_DEFINES_ASM_INCLUDED diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm_ippcp_api_vaes_avx512.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm_ippcp_api_vaes_avx512.inc new file mode 100644 index 0000000..cd84ac0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm_ippcp_api_vaes_avx512.inc @@ -0,0 +1,144 @@ +;=============================================================================== +; Copyright (C) 2020 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +%ifndef _GCM_IPPCP_API_VAES_AVX512_INC_ +%define _GCM_IPPCP_API_VAES_AVX512_INC_ + +%include "gcm_vaes_avx512.inc" + +section .text +default rel + +%ifdef GCM128_MODE + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_iv_hash_update_vaes512( +; const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; const u8 *iv, +; const u64 iv_len); +; +; NB: |iv_len| shall be multiple of 16 bytes (block size). This restriction is handled outside. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +IPPASM aes_gcm_iv_hash_update_vaes512, PUBLIC + FUNC_SAVE + + vmovdqu64 xmm8, [arg2 + OrigIV] ; load current hash + + ;; Calculate GHASH of this segment + CALC_AAD_HASH arg3, arg4, xmm8, arg1, zmm1, zmm2, zmm3, zmm4, zmm5, \ + zmm6, zmm7, zmm9, zmm10, zmm11, zmm12, zmm13, zmm15, \ + zmm16, zmm17, zmm18, zmm19, zmm20, r10, r11, r12, k1 + + vmovdqu64 [arg2 + OrigIV], xmm8 ; store updated hash + + FUNC_RESTORE + + ret +ENDFUNC aes_gcm_iv_hash_update_vaes512 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_iv_hash_finalize_vaes512( +; const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; const u8 *iv, +; const u64 iv_len, +; const u64 iv_general_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +IPPASM aes_gcm_iv_hash_finalize_vaes512, PUBLIC + FUNC_SAVE + + cmp arg5, 12 + jnz .Liv_is_not_12_bytes + + ;; read 12 IV bytes and pad with 0x00000001 + vmovdqa64 xmm2, [rel ONEf] + mov r11, arg3 + mov DWORD(r10), 0x0000_0fff + kmovd k1, DWORD(r10) + vmovdqu8 xmm2{k1}, [r11] ; ctr = IV | 0x1 + + jmp .Liv_compute_done + +.Liv_is_not_12_bytes: + vmovdqu xmm2, [arg2 + OrigIV] + + ;; prepare IV + CALC_J0 arg1, arg3, arg4, xmm2, \ + zmm1, zmm11, zmm3, zmm4, zmm5, zmm6, zmm7, zmm8, zmm9, zmm10, \ + zmm12, zmm13, zmm15, zmm16, zmm17, zmm18, zmm19, zmm20, \ + r10, r11, r12, k1, arg5 + +.Liv_compute_done: + vmovdqu64 [arg2 + OrigIV], xmm2 ; ctx.orig_IV = iv + vpshufb xmm2, [rel SHUF_MASK] + vmovdqu64 [arg2 + CurCount], xmm2 ; ctx.current_counter = iv (LE format) + + FUNC_RESTORE + + ret +ENDFUNC aes_gcm_iv_hash_finalize_vaes512 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_aad_hash_update_vaes512( +; const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; const u8 *aad, +; const u64 aad_len); +; +; NB: This function is always called with |aad_len| that is multiple of 16 bytes (block size). +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +IPPASM aes_gcm_aad_hash_update_vaes512, PUBLIC + FUNC_SAVE + + vmovdqu64 xmm14, [arg2 + AadHash] ; load current hash + + ;; Calculate GHASH of this segment + CALC_AAD_HASH arg3, arg4, xmm14, arg1, zmm1, zmm2, zmm3, zmm4, zmm5, \ + zmm6, zmm7, zmm9, zmm10, zmm11, zmm12, zmm13, zmm15, \ + zmm16, zmm17, zmm18, zmm19, zmm20, r10, r11, r12, k1 + + vmovdqu64 [arg2 + AadHash], xmm14 ; store updated hash + + FUNC_RESTORE + + ret +ENDFUNC aes_gcm_aad_hash_update_vaes512 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_gmult_vaes512( +; struct gcm_context_data *context_data, +; u8 *ghash) +; +; Function updates |ghash| value by multiplying it on H^1 key. +; Leaf function. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +IPPASM aes_gcm_gmult_vaes512, PUBLIC + + vmovdqu64 xmm1, [arg2] + vmovdqu64 xmm2, [arg1 + HashKey_1] + + GHASH_MUL xmm1, xmm2, xmm3, xmm4, xmm5, xmm16, xmm17 + + vmovdqu64 [arg2], xmm1 + + ret +ENDFUNC aes_gcm_gmult_vaes512 + +%endif ; GCM128_MODE + +%endif ; _GCM_IPPCP_API_VAES_AVX512_INC_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm_keys_avx512.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm_keys_avx512.inc new file mode 100644 index 0000000..62dcc86 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm_keys_avx512.inc @@ -0,0 +1,42 @@ +;=============================================================================== +; Copyright (C) 2019 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +%ifndef GCM_KEYS_AVX512_INCLUDED +%define GCM_KEYS_AVX512_INCLUDED + +;; Define the fields of gcm_key_data struct: +;; uint8_t expanded_keys[GCM_ENC_KEY_LEN * GCM_KEY_SETS]; +;; uint8_t shifted_hkey_8[GCM_ENC_KEY_LEN]; // HashKey^8 <<1 mod poly +;; uint8_t shifted_hkey_7[GCM_ENC_KEY_LEN]; // HashKey^7 <<1 mod poly +;; uint8_t shifted_hkey_6[GCM_ENC_KEY_LEN]; // HashKey^6 <<1 mod poly +;; uint8_t shifted_hkey_5[GCM_ENC_KEY_LEN]; // HashKey^5 <<1 mod poly +;; uint8_t shifted_hkey_4[GCM_ENC_KEY_LEN]; // HashKey^4 <<1 mod poly +;; uint8_t shifted_hkey_3[GCM_ENC_KEY_LEN]; // HashKey^3 <<1 mod poly +;; uint8_t shifted_hkey_2[GCM_ENC_KEY_LEN]; // HashKey^2 <<1 mod poly +;; uint8_t shifted_hkey_1[GCM_ENC_KEY_LEN]; // HashKey <<1 mod poly + +%define HashKey_8 (16*15) ; HashKey^8 <<1 mod poly +%define HashKey_7 (16*16) ; HashKey^7 <<1 mod poly +%define HashKey_6 (16*17) ; HashKey^6 <<1 mod poly +%define HashKey_5 (16*18) ; HashKey^5 <<1 mod poly +%define HashKey_4 (16*19) ; HashKey^4 <<1 mod poly +%define HashKey_3 (16*20) ; HashKey^3 <<1 mod poly +%define HashKey_2 (16*21) ; HashKey^2 <<1 mod poly +%define HashKey_1 (16*22) ; HashKey <<1 mod poly +%define HashKey (16*22) ; HashKey <<1 mod poly + +%endif ; GCM_KEYS_AVX512_INCLUDED diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm_keys_vaes_avx512.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm_keys_vaes_avx512.inc new file mode 100644 index 0000000..e23712f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm_keys_vaes_avx512.inc @@ -0,0 +1,221 @@ +;=============================================================================== +; Copyright (C) 2020 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +%ifndef GCM_KEYS_VAES_AVX512_INCLUDED +%define GCM_KEYS_VAES_AVX512_INCLUDED + +;; Define the fields of gcm_key_data struct: +;; uint8_t expanded_keys[GCM_ENC_KEY_LEN * GCM_KEY_SETS]; +;; uint8_t shifted_hkey_9_128[GCM_ENC_KEY_LEN * (128 - 8)]; +;; uint8_t shifted_hkey_8[GCM_ENC_KEY_LEN]; // HashKey^8 <<1 mod poly +;; uint8_t shifted_hkey_7[GCM_ENC_KEY_LEN]; // HashKey^7 <<1 mod poly +;; uint8_t shifted_hkey_6[GCM_ENC_KEY_LEN]; // HashKey^6 <<1 mod poly +;; uint8_t shifted_hkey_5[GCM_ENC_KEY_LEN]; // HashKey^5 <<1 mod poly +;; uint8_t shifted_hkey_4[GCM_ENC_KEY_LEN]; // HashKey^4 <<1 mod poly +;; uint8_t shifted_hkey_3[GCM_ENC_KEY_LEN]; // HashKey^3 <<1 mod poly +;; uint8_t shifted_hkey_2[GCM_ENC_KEY_LEN]; // HashKey^2 <<1 mod poly +;; uint8_t shifted_hkey_1[GCM_ENC_KEY_LEN]; // HashKey <<1 mod poly + +%ifdef GCM_BIG_DATA +;; +;; Key structure holds up to 128 ghash keys +;; +%define HashKey_128 (16*15) ; HashKey^128 <<1 mod poly +%define HashKey_127 (16*16) ; HashKey^127 <<1 mod poly +%define HashKey_126 (16*17) ; HashKey^126 <<1 mod poly +%define HashKey_125 (16*18) ; HashKey^125 <<1 mod poly +%define HashKey_124 (16*19) ; HashKey^124 <<1 mod poly +%define HashKey_123 (16*20) ; HashKey^123 <<1 mod poly +%define HashKey_122 (16*21) ; HashKey^122 <<1 mod poly +%define HashKey_121 (16*22) ; HashKey^121 <<1 mod poly +%define HashKey_120 (16*23) ; HashKey^120 <<1 mod poly +%define HashKey_119 (16*24) ; HashKey^119 <<1 mod poly +%define HashKey_118 (16*25) ; HashKey^118 <<1 mod poly +%define HashKey_117 (16*26) ; HashKey^117 <<1 mod poly +%define HashKey_116 (16*27) ; HashKey^116 <<1 mod poly +%define HashKey_115 (16*28) ; HashKey^115 <<1 mod poly +%define HashKey_114 (16*29) ; HashKey^114 <<1 mod poly +%define HashKey_113 (16*30) ; HashKey^113 <<1 mod poly +%define HashKey_112 (16*31) ; HashKey^112 <<1 mod poly +%define HashKey_111 (16*32) ; HashKey^111 <<1 mod poly +%define HashKey_110 (16*33) ; HashKey^110 <<1 mod poly +%define HashKey_109 (16*34) ; HashKey^109 <<1 mod poly +%define HashKey_108 (16*35) ; HashKey^108 <<1 mod poly +%define HashKey_107 (16*36) ; HashKey^107 <<1 mod poly +%define HashKey_106 (16*37) ; HashKey^106 <<1 mod poly +%define HashKey_105 (16*38) ; HashKey^105 <<1 mod poly +%define HashKey_104 (16*39) ; HashKey^104 <<1 mod poly +%define HashKey_103 (16*40) ; HashKey^103 <<1 mod poly +%define HashKey_102 (16*41) ; HashKey^102 <<1 mod poly +%define HashKey_101 (16*42) ; HashKey^101 <<1 mod poly +%define HashKey_100 (16*43) ; HashKey^100 <<1 mod poly +%define HashKey_99 (16*44) ; HashKey^99 <<1 mod poly +%define HashKey_98 (16*45) ; HashKey^98 <<1 mod poly +%define HashKey_97 (16*46) ; HashKey^97 <<1 mod poly +%define HashKey_96 (16*47) ; HashKey^96 <<1 mod poly +%define HashKey_95 (16*48) ; HashKey^95 <<1 mod poly +%define HashKey_94 (16*49) ; HashKey^94 <<1 mod poly +%define HashKey_93 (16*50) ; HashKey^93 <<1 mod poly +%define HashKey_92 (16*51) ; HashKey^92 <<1 mod poly +%define HashKey_91 (16*52) ; HashKey^91 <<1 mod poly +%define HashKey_90 (16*53) ; HashKey^90 <<1 mod poly +%define HashKey_89 (16*54) ; HashKey^89 <<1 mod poly +%define HashKey_88 (16*55) ; HashKey^88 <<1 mod poly +%define HashKey_87 (16*56) ; HashKey^87 <<1 mod poly +%define HashKey_86 (16*57) ; HashKey^86 <<1 mod poly +%define HashKey_85 (16*58) ; HashKey^85 <<1 mod poly +%define HashKey_84 (16*59) ; HashKey^84 <<1 mod poly +%define HashKey_83 (16*60) ; HashKey^83 <<1 mod poly +%define HashKey_82 (16*61) ; HashKey^82 <<1 mod poly +%define HashKey_81 (16*62) ; HashKey^81 <<1 mod poly +%define HashKey_80 (16*63) ; HashKey^80 <<1 mod poly +%define HashKey_79 (16*64) ; HashKey^79 <<1 mod poly +%define HashKey_78 (16*65) ; HashKey^78 <<1 mod poly +%define HashKey_77 (16*66) ; HashKey^77 <<1 mod poly +%define HashKey_76 (16*67) ; HashKey^76 <<1 mod poly +%define HashKey_75 (16*68) ; HashKey^75 <<1 mod poly +%define HashKey_74 (16*69) ; HashKey^74 <<1 mod poly +%define HashKey_73 (16*70) ; HashKey^73 <<1 mod poly +%define HashKey_72 (16*71) ; HashKey^72 <<1 mod poly +%define HashKey_71 (16*72) ; HashKey^71 <<1 mod poly +%define HashKey_70 (16*73) ; HashKey^70 <<1 mod poly +%define HashKey_69 (16*74) ; HashKey^69 <<1 mod poly +%define HashKey_68 (16*75) ; HashKey^68 <<1 mod poly +%define HashKey_67 (16*76) ; HashKey^67 <<1 mod poly +%define HashKey_66 (16*77) ; HashKey^66 <<1 mod poly +%define HashKey_65 (16*78) ; HashKey^65 <<1 mod poly +%define HashKey_64 (16*79) ; HashKey^64 <<1 mod poly +%define HashKey_63 (16*80) ; HashKey^63 <<1 mod poly +%define HashKey_62 (16*81) ; HashKey^62 <<1 mod poly +%define HashKey_61 (16*82) ; HashKey^61 <<1 mod poly +%define HashKey_60 (16*83) ; HashKey^60 <<1 mod poly +%define HashKey_59 (16*84) ; HashKey^59 <<1 mod poly +%define HashKey_58 (16*85) ; HashKey^58 <<1 mod poly +%define HashKey_57 (16*86) ; HashKey^57 <<1 mod poly +%define HashKey_56 (16*87) ; HashKey^56 <<1 mod poly +%define HashKey_55 (16*88) ; HashKey^55 <<1 mod poly +%define HashKey_54 (16*89) ; HashKey^54 <<1 mod poly +%define HashKey_53 (16*90) ; HashKey^53 <<1 mod poly +%define HashKey_52 (16*91) ; HashKey^52 <<1 mod poly +%define HashKey_51 (16*92) ; HashKey^51 <<1 mod poly +%define HashKey_50 (16*93) ; HashKey^50 <<1 mod poly +%define HashKey_49 (16*94) ; HashKey^49 <<1 mod poly +%define HashKey_48 (16*95) ; HashKey^48 <<1 mod poly +%define HashKey_47 (16*96) ; HashKey^47 <<1 mod poly +%define HashKey_46 (16*97) ; HashKey^46 <<1 mod poly +%define HashKey_45 (16*98) ; HashKey^45 <<1 mod poly +%define HashKey_44 (16*99) ; HashKey^44 <<1 mod poly +%define HashKey_43 (16*100) ; HashKey^43 <<1 mod poly +%define HashKey_42 (16*101) ; HashKey^42 <<1 mod poly +%define HashKey_41 (16*102) ; HashKey^41 <<1 mod poly +%define HashKey_40 (16*103) ; HashKey^40 <<1 mod poly +%define HashKey_39 (16*104) ; HashKey^39 <<1 mod poly +%define HashKey_38 (16*105) ; HashKey^38 <<1 mod poly +%define HashKey_37 (16*106) ; HashKey^37 <<1 mod poly +%define HashKey_36 (16*107) ; HashKey^36 <<1 mod poly +%define HashKey_35 (16*108) ; HashKey^35 <<1 mod poly +%define HashKey_34 (16*109) ; HashKey^34 <<1 mod poly +%define HashKey_33 (16*110) ; HashKey^33 <<1 mod poly +%define HashKey_32 (16*111) ; HashKey^32 <<1 mod poly +%define HashKey_31 (16*112) ; HashKey^31 <<1 mod poly +%define HashKey_30 (16*113) ; HashKey^30 <<1 mod poly +%define HashKey_29 (16*114) ; HashKey^29 <<1 mod poly +%define HashKey_28 (16*115) ; HashKey^28 <<1 mod poly +%define HashKey_27 (16*116) ; HashKey^27 <<1 mod poly +%define HashKey_26 (16*117) ; HashKey^26 <<1 mod poly +%define HashKey_25 (16*118) ; HashKey^25 <<1 mod poly +%define HashKey_24 (16*119) ; HashKey^24 <<1 mod poly +%define HashKey_23 (16*120) ; HashKey^23 <<1 mod poly +%define HashKey_22 (16*121) ; HashKey^22 <<1 mod poly +%define HashKey_21 (16*122) ; HashKey^21 <<1 mod poly +%define HashKey_20 (16*123) ; HashKey^20 <<1 mod poly +%define HashKey_19 (16*124) ; HashKey^19 <<1 mod poly +%define HashKey_18 (16*125) ; HashKey^18 <<1 mod poly +%define HashKey_17 (16*126) ; HashKey^17 <<1 mod poly +%define HashKey_16 (16*127) ; HashKey^16 <<1 mod poly +%define HashKey_15 (16*128) ; HashKey^15 <<1 mod poly +%define HashKey_14 (16*129) ; HashKey^14 <<1 mod poly +%define HashKey_13 (16*130) ; HashKey^13 <<1 mod poly +%define HashKey_12 (16*131) ; HashKey^12 <<1 mod poly +%define HashKey_11 (16*132) ; HashKey^11 <<1 mod poly +%define HashKey_10 (16*133) ; HashKey^10 <<1 mod poly +%define HashKey_9 (16*134) ; HashKey^9 <<1 mod poly +%define HashKey_8 (16*135) ; HashKey^8 <<1 mod poly +%define HashKey_7 (16*136) ; HashKey^7 <<1 mod poly +%define HashKey_6 (16*137) ; HashKey^6 <<1 mod poly +%define HashKey_5 (16*138) ; HashKey^5 <<1 mod poly +%define HashKey_4 (16*139) ; HashKey^4 <<1 mod poly +%define HashKey_3 (16*140) ; HashKey^3 <<1 mod poly +%define HashKey_2 (16*141) ; HashKey^2 <<1 mod poly +%define HashKey_1 (16*142) ; HashKey <<1 mod poly +%define HashKey (16*142) ; HashKey <<1 mod poly +%else +;; +;; Key structure holds up to 48 ghash keys +;; +%define HashKey_48 (16*15) ; HashKey^48 <<1 mod poly +%define HashKey_47 (16*16) ; HashKey^47 <<1 mod poly +%define HashKey_46 (16*17) ; HashKey^46 <<1 mod poly +%define HashKey_45 (16*18) ; HashKey^45 <<1 mod poly +%define HashKey_44 (16*19) ; HashKey^44 <<1 mod poly +%define HashKey_43 (16*20) ; HashKey^43 <<1 mod poly +%define HashKey_42 (16*21) ; HashKey^42 <<1 mod poly +%define HashKey_41 (16*22) ; HashKey^41 <<1 mod poly +%define HashKey_40 (16*23) ; HashKey^40 <<1 mod poly +%define HashKey_39 (16*24) ; HashKey^39 <<1 mod poly +%define HashKey_38 (16*25) ; HashKey^38 <<1 mod poly +%define HashKey_37 (16*26) ; HashKey^37 <<1 mod poly +%define HashKey_36 (16*27) ; HashKey^36 <<1 mod poly +%define HashKey_35 (16*28) ; HashKey^35 <<1 mod poly +%define HashKey_34 (16*29) ; HashKey^34 <<1 mod poly +%define HashKey_33 (16*30) ; HashKey^33 <<1 mod poly +%define HashKey_32 (16*31) ; HashKey^32 <<1 mod poly +%define HashKey_31 (16*32) ; HashKey^31 <<1 mod poly +%define HashKey_30 (16*33) ; HashKey^30 <<1 mod poly +%define HashKey_29 (16*34) ; HashKey^29 <<1 mod poly +%define HashKey_28 (16*35) ; HashKey^28 <<1 mod poly +%define HashKey_27 (16*36) ; HashKey^27 <<1 mod poly +%define HashKey_26 (16*37) ; HashKey^26 <<1 mod poly +%define HashKey_25 (16*38) ; HashKey^25 <<1 mod poly +%define HashKey_24 (16*39) ; HashKey^24 <<1 mod poly +%define HashKey_23 (16*40) ; HashKey^23 <<1 mod poly +%define HashKey_22 (16*41) ; HashKey^22 <<1 mod poly +%define HashKey_21 (16*42) ; HashKey^21 <<1 mod poly +%define HashKey_20 (16*43) ; HashKey^20 <<1 mod poly +%define HashKey_19 (16*44) ; HashKey^19 <<1 mod poly +%define HashKey_18 (16*45) ; HashKey^18 <<1 mod poly +%define HashKey_17 (16*46) ; HashKey^17 <<1 mod poly +%define HashKey_16 (16*47) ; HashKey^16 <<1 mod poly +%define HashKey_15 (16*48) ; HashKey^15 <<1 mod poly +%define HashKey_14 (16*49) ; HashKey^14 <<1 mod poly +%define HashKey_13 (16*50) ; HashKey^13 <<1 mod poly +%define HashKey_12 (16*51) ; HashKey^12 <<1 mod poly +%define HashKey_11 (16*52) ; HashKey^11 <<1 mod poly +%define HashKey_10 (16*53) ; HashKey^10 <<1 mod poly +%define HashKey_9 (16*54) ; HashKey^9 <<1 mod poly +%define HashKey_8 (16*55) ; HashKey^8 <<1 mod poly +%define HashKey_7 (16*56) ; HashKey^7 <<1 mod poly +%define HashKey_6 (16*57) ; HashKey^6 <<1 mod poly +%define HashKey_5 (16*58) ; HashKey^5 <<1 mod poly +%define HashKey_4 (16*59) ; HashKey^4 <<1 mod poly +%define HashKey_3 (16*60) ; HashKey^3 <<1 mod poly +%define HashKey_2 (16*61) ; HashKey^2 <<1 mod poly +%define HashKey_1 (16*62) ; HashKey <<1 mod poly +%define HashKey (16*62) ; HashKey <<1 mod poly +%endif ; !GCM_BIG_DATA + +%endif ; GCM_KEYS_VAES_AVX512_INCLUDED diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm_vaes_avx512.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm_vaes_avx512.inc new file mode 100644 index 0000000..6aca9ca --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/gcm_vaes_avx512.inc @@ -0,0 +1,3559 @@ +;=============================================================================== +; Copyright (C) 2020 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; Authors: +; Erdinc Ozturk +; Vinodh Gopal +; James Guilford +; Tomasz Kantecki +; +; +; References: +; This code was derived and highly optimized from the code described in paper: +; Vinodh Gopal et. al. Optimized Galois-Counter-Mode Implementation on Intel Architecture Processors. August, 2010 +; The details of the implementation is explained in: +; Erdinc Ozturk et. al. Enabling High-Performance Galois-Counter-Mode on Intel Architecture Processors. October, 2012. +; +; +; +; +; Assumptions: +; +; +; +; iv: +; 0 1 2 3 +; 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | Salt (From the SA) | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | Initialization Vector | +; | (This is the sequence number from IPSec header) | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | 0x1 | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; +; +; +; AAD: +; AAD will be padded with 0 to the next 16byte multiple +; for example, assume AAD is a u32 vector +; +; if AAD is 8 bytes: +; AAD[3] = {A0, A1}; +; padded AAD in xmm register = {A1 A0 0 0} +; +; 0 1 2 3 +; 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | SPI (A1) | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | 32-bit Sequence Number (A0) | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | 0x0 | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; +; AAD Format with 32-bit Sequence Number +; +; if AAD is 12 bytes: +; AAD[3] = {A0, A1, A2}; +; padded AAD in xmm register = {A2 A1 A0 0} +; +; 0 1 2 3 +; 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | SPI (A2) | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | 64-bit Extended Sequence Number {A1,A0} | +; | | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | 0x0 | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; +; AAD Format with 64-bit Extended Sequence Number +; +; +; aadLen: +; Must be a multiple of 4 bytes and from the definition of the spec. +; The code additionally supports any aadLen length. +; +; TLen: +; from the definition of the spec, TLen can only be 8, 12 or 16 bytes. +; +; poly = x^128 + x^127 + x^126 + x^121 + 1 +; throughout the code, one tab and two tab indentations are used. one tab is for GHASH part, two tabs is for AES part. +; + +%ifndef _GCM_VAES_AVX512_INC_ +%define _GCM_VAES_AVX512_INC_ + +%include "os.inc" +%include "reg_sizes.inc" +%include "clear_regs.inc" +%include "gcm_defines.inc" +%include "gcm_keys_vaes_avx512.inc" +%include "memcpy.inc" +%include "aes_common.inc" + +%ifndef GCM128_MODE +%ifndef GCM192_MODE +%ifndef GCM256_MODE +%error "No GCM mode selected for gcm_avx512.asm!" +%endif +%endif +%endif + +;; Decide on AES-GCM key size to compile for +%ifdef GCM128_MODE +%define NROUNDS 9 +%define FN_NAME(x,y) aes_gcm_ %+ x %+ _128 %+ y %+ vaes_avx512 +%endif + +%ifdef GCM192_MODE +%define NROUNDS 11 +%define FN_NAME(x,y) aes_gcm_ %+ x %+ _192 %+ y %+ vaes_avx512 +%endif + +%ifdef GCM256_MODE +%define NROUNDS 13 +%define FN_NAME(x,y) aes_gcm_ %+ x %+ _256 %+ y %+ vaes_avx512 +%endif + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Stack frame definition +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%ifidn __OUTPUT_FORMAT__, win64 + %define XMM_STORAGE (10*16) ; space for 10 XMM registers + %define GP_STORAGE ((9*8) + 24) ; space for 9 GP registers + 24 bytes for 64 byte alignment +%else + %define XMM_STORAGE 0 + %define GP_STORAGE (8*8) ; space for 7 GP registers + 1 for alignment +%endif +%define LOCAL_STORAGE (48*16) ; space for up to 48 AES blocks + +;;; sequence is (bottom-up): GP, XMM, local +%define STACK_GP_OFFSET 0 +%define STACK_XMM_OFFSET (STACK_GP_OFFSET + GP_STORAGE) +%define STACK_LOCAL_OFFSET (STACK_XMM_OFFSET + XMM_STORAGE) +%define STACK_FRAME_SIZE (STACK_LOCAL_OFFSET + LOCAL_STORAGE) + +;; for compatibility with stack argument definitions in gcm_defines.asm +%define STACK_OFFSET 0 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Utility Macros +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;;; =========================================================================== +;;; =========================================================================== +;;; Horizontal XOR - 4 x 128bits xored together +%macro VHPXORI4x128 2 +%define %%REG %1 ; [in/out] ZMM with 4x128bits to xor; 128bit output +%define %%TMP %2 ; [clobbered] ZMM temporary register + vextracti64x4 YWORD(%%TMP), %%REG, 1 + vpxorq YWORD(%%REG), YWORD(%%REG), YWORD(%%TMP) + vextracti32x4 XWORD(%%TMP), YWORD(%%REG), 1 + vpxorq XWORD(%%REG), XWORD(%%REG), XWORD(%%TMP) +%endmacro ; VHPXORI4x128 + +;;; =========================================================================== +;;; =========================================================================== +;;; AVX512 reduction macro +%macro VCLMUL_REDUCE 6 +%define %%OUT %1 ; [out] zmm/ymm/xmm: result (must not be %%TMP1 or %%HI128) +%define %%POLY %2 ; [in] zmm/ymm/xmm: polynomial +%define %%HI128 %3 ; [in] zmm/ymm/xmm: high 128b of hash to reduce +%define %%LO128 %4 ; [in] zmm/ymm/xmm: low 128b of hash to reduce +%define %%TMP0 %5 ; [in] zmm/ymm/xmm: temporary register +%define %%TMP1 %6 ; [in] zmm/ymm/xmm: temporary register + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; first phase of the reduction + vpclmulqdq %%TMP0, %%POLY, %%LO128, 0x01 + vpslldq %%TMP0, %%TMP0, 8 ; shift-L 2 DWs + vpxorq %%TMP0, %%LO128, %%TMP0 ; first phase of the reduction complete + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; second phase of the reduction + vpclmulqdq %%TMP1, %%POLY, %%TMP0, 0x00 + vpsrldq %%TMP1, %%TMP1, 4 ; shift-R only 1-DW to obtain 2-DWs shift-R + + vpclmulqdq %%OUT, %%POLY, %%TMP0, 0x10 + vpslldq %%OUT, %%OUT, 4 ; shift-L 1-DW to obtain result with no shifts + + vpternlogq %%OUT, %%TMP1, %%HI128, 0x96 ; OUT/GHASH = OUT xor TMP1 xor HI128 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%endmacro + +;;; =========================================================================== +;;; =========================================================================== +;;; schoolbook multiply of 16 blocks (8 x 16 bytes) +;;; - it is assumed that data read from %%INPTR is already shuffled and +;;; %%INPTR address is 64 byte aligned +;;; - there is an option to pass ready blocks through ZMM registers too. +;;; 4 extra parameters need to passed in such case and 21st argument can be empty +%macro GHASH_16 21-25 +%define %%TYPE %1 ; [in] ghash type: start (xor hash), mid, end (same as mid; no reduction), + ; end_reduce (end with reduction), start_reduce +%define %%GH %2 ; [in/out] ZMM ghash sum: high 128-bits +%define %%GM %3 ; [in/out] ZMM ghash sum: middle 128-bits +%define %%GL %4 ; [in/out] ZMM ghash sum: low 128-bits +%define %%INPTR %5 ; [in] data input pointer +%define %%INOFF %6 ; [in] data input offset +%define %%INDIS %7 ; [in] data input displacement +%define %%HKPTR %8 ; [in] hash key pointer +%define %%HKOFF %9 ; [in] hash key offset +%define %%HKDIS %10 ; [in] hash key displacement +%define %%HASH %11 ; [in/out] ZMM hash value in/out +%define %%ZTMP0 %12 ; [clobbered] temporary ZMM +%define %%ZTMP1 %13 ; [clobbered] temporary ZMM +%define %%ZTMP2 %14 ; [clobbered] temporary ZMM +%define %%ZTMP3 %15 ; [clobbered] temporary ZMM +%define %%ZTMP4 %16 ; [clobbered] temporary ZMM +%define %%ZTMP5 %17 ; [clobbered] temporary ZMM +%define %%ZTMP6 %18 ; [clobbered] temporary ZMM +%define %%ZTMP7 %19 ; [clobbered] temporary ZMM +%define %%ZTMP8 %20 ; [clobbered] temporary ZMM +%define %%ZTMP9 %21 ; [clobbered] temporary ZMM +%define %%DAT0 %22 ; [in] ZMM with 4 blocks of input data (INPTR, INOFF, INDIS unused) +%define %%DAT1 %23 ; [in] ZMM with 4 blocks of input data (INPTR, INOFF, INDIS unused) +%define %%DAT2 %24 ; [in] ZMM with 4 blocks of input data (INPTR, INOFF, INDIS unused) +%define %%DAT3 %25 ; [in] ZMM with 4 blocks of input data (INPTR, INOFF, INDIS unused) + +%assign start_ghash 0 +%assign do_reduction 0 + +%ifidn %%TYPE, start +%assign start_ghash 1 +%endif + +%ifidn %%TYPE, start_reduce +%assign start_ghash 1 +%assign do_reduction 1 +%endif + +%ifidn %%TYPE, end_reduce +%assign do_reduction 1 +%endif + + ;; ghash blocks 0-3 +%if %0 == 21 + vmovdqa64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS] +%else +%xdefine %%ZTMP9 %%DAT0 +%endif + +%if start_ghash != 0 + vpxorq %%ZTMP9, %%HASH +%endif + vmovdqu64 %%ZTMP8, [%%HKPTR + %%HKOFF + %%HKDIS] + vpclmulqdq %%ZTMP0, %%ZTMP9, %%ZTMP8, 0x11 ; T0H = a1*b1 + vpclmulqdq %%ZTMP1, %%ZTMP9, %%ZTMP8, 0x00 ; T0L = a0*b0 + vpclmulqdq %%ZTMP2, %%ZTMP9, %%ZTMP8, 0x01 ; T0M1 = a1*b0 + vpclmulqdq %%ZTMP3, %%ZTMP9, %%ZTMP8, 0x10 ; T0M2 = a0*b1 + ;; ghash blocks 4-7 +%if %0 == 21 + vmovdqa64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS + 64] +%else +%xdefine %%ZTMP9 %%DAT1 +%endif + vmovdqu64 %%ZTMP8, [%%HKPTR + %%HKOFF + %%HKDIS + 64] + vpclmulqdq %%ZTMP4, %%ZTMP9, %%ZTMP8, 0x11 ; T1H = a1*b1 + vpclmulqdq %%ZTMP5, %%ZTMP9, %%ZTMP8, 0x00 ; T1L = a0*b0 + vpclmulqdq %%ZTMP6, %%ZTMP9, %%ZTMP8, 0x01 ; T1M1 = a1*b0 + vpclmulqdq %%ZTMP7, %%ZTMP9, %%ZTMP8, 0x10 ; T1M2 = a0*b1 + ;; update sums +%if start_ghash != 0 + vpxorq %%GM, %%ZTMP2, %%ZTMP6 ; GM = T0M1 + T1M1 + vpxorq %%GH, %%ZTMP0, %%ZTMP4 ; GH = T0H + T1H + vpxorq %%GL, %%ZTMP1, %%ZTMP5 ; GL = T0L + T1L + vpternlogq %%GM, %%ZTMP3, %%ZTMP7, 0x96 ; GM = T0M2 + T1M1 +%else ;; mid, end, end_reduce + vpternlogq %%GM, %%ZTMP2, %%ZTMP6, 0x96 ; GM += T0M1 + T1M1 + vpternlogq %%GH, %%ZTMP0, %%ZTMP4, 0x96 ; GH += T0H + T1H + vpternlogq %%GL, %%ZTMP1, %%ZTMP5, 0x96 ; GL += T0L + T1L + vpternlogq %%GM, %%ZTMP3, %%ZTMP7, 0x96 ; GM += T0M2 + T1M1 +%endif + ;; ghash blocks 8-11 +%if %0 == 21 + vmovdqa64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS + 128] +%else +%xdefine %%ZTMP9 %%DAT2 +%endif + vmovdqu64 %%ZTMP8, [%%HKPTR + %%HKOFF + %%HKDIS + 128] + vpclmulqdq %%ZTMP0, %%ZTMP9, %%ZTMP8, 0x11 ; T0H = a1*b1 + vpclmulqdq %%ZTMP1, %%ZTMP9, %%ZTMP8, 0x00 ; T0L = a0*b0 + vpclmulqdq %%ZTMP2, %%ZTMP9, %%ZTMP8, 0x01 ; T0M1 = a1*b0 + vpclmulqdq %%ZTMP3, %%ZTMP9, %%ZTMP8, 0x10 ; T0M2 = a0*b1 + ;; ghash blocks 12-15 +%if %0 == 21 + vmovdqa64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS + 192] +%else +%xdefine %%ZTMP9 %%DAT3 +%endif + vmovdqu64 %%ZTMP8, [%%HKPTR + %%HKOFF + %%HKDIS + 192] + vpclmulqdq %%ZTMP4, %%ZTMP9, %%ZTMP8, 0x11 ; T1H = a1*b1 + vpclmulqdq %%ZTMP5, %%ZTMP9, %%ZTMP8, 0x00 ; T1L = a0*b0 + vpclmulqdq %%ZTMP6, %%ZTMP9, %%ZTMP8, 0x01 ; T1M1 = a1*b0 + vpclmulqdq %%ZTMP7, %%ZTMP9, %%ZTMP8, 0x10 ; T1M2 = a0*b1 + ;; update sums + vpternlogq %%GM, %%ZTMP2, %%ZTMP6, 0x96 ; GM += T0M1 + T1M1 + vpternlogq %%GH, %%ZTMP0, %%ZTMP4, 0x96 ; GH += T0H + T1H + vpternlogq %%GL, %%ZTMP1, %%ZTMP5, 0x96 ; GL += T0L + T1L + vpternlogq %%GM, %%ZTMP3, %%ZTMP7, 0x96 ; GM += T0M2 + T1M1 + +%if do_reduction != 0 + ;; integrate GM into GH and GL + vpsrldq %%ZTMP0, %%GM, 8 + vpslldq %%ZTMP1, %%GM, 8 + vpxorq %%GH, %%GH, %%ZTMP0 + vpxorq %%GL, %%GL, %%ZTMP1 + + ;; add GH and GL 128-bit words horizontally + VHPXORI4x128 %%GH, %%ZTMP0 + VHPXORI4x128 %%GL, %%ZTMP1 + + ;; reduction + vmovdqa64 XWORD(%%ZTMP2), [rel POLY2] + VCLMUL_REDUCE XWORD(%%HASH), XWORD(%%ZTMP2), \ + XWORD(%%GH), XWORD(%%GL), XWORD(%%ZTMP0), XWORD(%%ZTMP1) +%endif +%endmacro + +;;; =========================================================================== +;;; =========================================================================== +;;; GHASH 1 to 16 blocks of cipher text +;;; - performs reduction at the end +;;; - it doesn't load the data and it assumed it is already loaded and +;;; shuffled +;;; - single_call scenario only +%macro GHASH_1_TO_16 17-20 +%define %%KP %1 ; [in] pointer to expanded keys +%define %%GHASH %2 ; [out] ghash output +%define %%T0H %3 ; [clobbered] temporary ZMM +%define %%T0L %4 ; [clobbered] temporary ZMM +%define %%T0M1 %5 ; [clobbered] temporary ZMM +%define %%T0M2 %6 ; [clobbered] temporary ZMM +%define %%T1H %7 ; [clobbered] temporary ZMM +%define %%T1L %8 ; [clobbered] temporary ZMM +%define %%T1M1 %9 ; [clobbered] temporary ZMM +%define %%T1M2 %10 ; [clobbered] temporary ZMM +%define %%HK %11 ; [clobbered] temporary ZMM +%define %%AAD_HASH_IN %12 ; [in] input hash value +%define %%CIPHER_IN0 %13 ; [in] ZMM with cipher text blocks 0-3 +%define %%CIPHER_IN1 %14 ; [in] ZMM with cipher text blocks 4-7 +%define %%CIPHER_IN2 %15 ; [in] ZMM with cipher text blocks 8-11 +%define %%CIPHER_IN3 %16 ; [in] ZMM with cipher text blocks 12-15 +%define %%NUM_BLOCKS %17 ; [in] numerical value, number of blocks +%define %%GH %18 ; [in] ZMM with hi product part +%define %%GM %19 ; [in] ZMM with mid product part +%define %%GL %20 ; [in] ZMM with lo product part + +%assign hashk HashKey_ %+ %%NUM_BLOCKS + +%if %0 == 17 + vpxorq %%CIPHER_IN0, %%CIPHER_IN0, %%AAD_HASH_IN +%endif + +%if %%NUM_BLOCKS == 16 + + vmovdqu64 %%HK, [%%KP + hashk] + vpclmulqdq %%T0H, %%CIPHER_IN0, %%HK, 0x11 ; H = a1*b1 + vpclmulqdq %%T0L, %%CIPHER_IN0, %%HK, 0x00 ; L = a0*b0 + vpclmulqdq %%T0M1, %%CIPHER_IN0, %%HK, 0x01 ; M1 = a1*b0 + vpclmulqdq %%T0M2, %%CIPHER_IN0, %%HK, 0x10 ; M2 = a0*b1 + vmovdqu64 %%HK, [%%KP + hashk + (1*64)] + vpclmulqdq %%T1H, %%CIPHER_IN1, %%HK, 0x11 ; H = a1*b1 + vpclmulqdq %%T1L, %%CIPHER_IN1, %%HK, 0x00 ; L = a0*b0 + vpclmulqdq %%T1M1, %%CIPHER_IN1, %%HK, 0x01 ; M1 = a1*b0 + vpclmulqdq %%T1M2, %%CIPHER_IN1, %%HK, 0x10 ; M2 = a0*b1 + vmovdqu64 %%HK, [%%KP + hashk + (2*64)] + vpclmulqdq %%CIPHER_IN0, %%CIPHER_IN2, %%HK, 0x11 ; H = a1*b1 + vpclmulqdq %%CIPHER_IN1, %%CIPHER_IN2, %%HK, 0x00 ; L = a0*b0 + vpternlogq %%T0H, %%CIPHER_IN0, %%T1H, 0x96 + vpternlogq %%T0L, %%CIPHER_IN1, %%T1L, 0x96 + vpclmulqdq %%CIPHER_IN0, %%CIPHER_IN2, %%HK, 0x01 ; M1 = a1*b0 + vpclmulqdq %%CIPHER_IN1, %%CIPHER_IN2, %%HK, 0x10 ; M2 = a0*b1 + vpternlogq %%T0M1, %%CIPHER_IN0, %%T1M1, 0x96 + vpternlogq %%T0M2, %%CIPHER_IN1, %%T1M2, 0x96 + vmovdqu64 %%HK, [%%KP + hashk + (3*64)] + vpclmulqdq %%T1H, %%CIPHER_IN3, %%HK, 0x11 ; H = a1*b1 + vpclmulqdq %%T1L, %%CIPHER_IN3, %%HK, 0x00 ; L = a0*b0 + vpclmulqdq %%T1M1, %%CIPHER_IN3, %%HK, 0x01 ; M1 = a1*b0 + vpclmulqdq %%T1M2, %%CIPHER_IN3, %%HK, 0x10 ; M2 = a0*b1 + vpxorq %%T1H, %%T0H, %%T1H + vpxorq %%T1L, %%T0L, %%T1L + vpxorq %%T1M1, %%T0M1, %%T1M1 + vpxorq %%T1M2, %%T0M2, %%T1M2 + +%elif %%NUM_BLOCKS >= 12 + + vmovdqu64 %%HK, [%%KP + hashk] + vpclmulqdq %%T0H, %%CIPHER_IN0, %%HK, 0x11 ; H = a1*b1 + vpclmulqdq %%T0L, %%CIPHER_IN0, %%HK, 0x00 ; L = a0*b0 + vpclmulqdq %%T0M1, %%CIPHER_IN0, %%HK, 0x01 ; M1 = a1*b0 + vpclmulqdq %%T0M2, %%CIPHER_IN0, %%HK, 0x10 ; M2 = a0*b1 + vmovdqu64 %%HK, [%%KP + hashk + (1*64)] + vpclmulqdq %%T1H, %%CIPHER_IN1, %%HK, 0x11 ; H = a1*b1 + vpclmulqdq %%T1L, %%CIPHER_IN1, %%HK, 0x00 ; L = a0*b0 + vpclmulqdq %%T1M1, %%CIPHER_IN1, %%HK, 0x01 ; M1 = a1*b0 + vpclmulqdq %%T1M2, %%CIPHER_IN1, %%HK, 0x10 ; M2 = a0*b1 + vmovdqu64 %%HK, [%%KP + hashk + (2*64)] + vpclmulqdq %%CIPHER_IN0, %%CIPHER_IN2, %%HK, 0x11 ; H = a1*b1 + vpclmulqdq %%CIPHER_IN1, %%CIPHER_IN2, %%HK, 0x00 ; L = a0*b0 + vpternlogq %%T1H, %%CIPHER_IN0, %%T0H, 0x96 + vpternlogq %%T1L, %%CIPHER_IN1, %%T0L, 0x96 + vpclmulqdq %%CIPHER_IN0, %%CIPHER_IN2, %%HK, 0x01 ; M1 = a1*b0 + vpclmulqdq %%CIPHER_IN1, %%CIPHER_IN2, %%HK, 0x10 ; M2 = a0*b1 + vpternlogq %%T1M1, %%CIPHER_IN0, %%T0M1, 0x96 + vpternlogq %%T1M2, %%CIPHER_IN1, %%T0M2, 0x96 + +%elif %%NUM_BLOCKS >= 8 + + vmovdqu64 %%HK, [%%KP + hashk] + vpclmulqdq %%T0H, %%CIPHER_IN0, %%HK, 0x11 ; H = a1*b1 + vpclmulqdq %%T0L, %%CIPHER_IN0, %%HK, 0x00 ; L = a0*b0 + vpclmulqdq %%T0M1, %%CIPHER_IN0, %%HK, 0x01 ; M1 = a1*b0 + vpclmulqdq %%T0M2, %%CIPHER_IN0, %%HK, 0x10 ; M2 = a0*b1 + vmovdqu64 %%HK, [%%KP + hashk + (1*64)] + vpclmulqdq %%T1H, %%CIPHER_IN1, %%HK, 0x11 ; H = a1*b1 + vpclmulqdq %%T1L, %%CIPHER_IN1, %%HK, 0x00 ; L = a0*b0 + vpclmulqdq %%T1M1, %%CIPHER_IN1, %%HK, 0x01 ; M1 = a1*b0 + vpclmulqdq %%T1M2, %%CIPHER_IN1, %%HK, 0x10 ; M2 = a0*b1 + vpxorq %%T1H, %%T0H, %%T1H + vpxorq %%T1L, %%T0L, %%T1L + vpxorq %%T1M1, %%T0M1, %%T1M1 + vpxorq %%T1M2, %%T0M2, %%T1M2 + +%elif %%NUM_BLOCKS >= 4 + + vmovdqu64 %%HK, [%%KP + hashk] + vpclmulqdq %%T1H, %%CIPHER_IN0, %%HK, 0x11 ; H = a1*b1 + vpclmulqdq %%T1L, %%CIPHER_IN0, %%HK, 0x00 ; L = a0*b0 + vpclmulqdq %%T1M1, %%CIPHER_IN0, %%HK, 0x01 ; M1 = a1*b0 + vpclmulqdq %%T1M2, %%CIPHER_IN0, %%HK, 0x10 ; M2 = a0*b1 + +%endif + + ;; T1H/L/M1/M2 - hold current product sums (provided %%NUM_BLOCKS >= 4) +%assign blocks_left (%%NUM_BLOCKS % 4) + +%if blocks_left > 0 + ;; ===================================================== + ;; There are 1, 2 or 3 blocks left to process. + ;; It may also be that they are the only blocks to process. + +;; Set hash key and register index position for the remaining 1 to 3 blocks +%assign hashk HashKey_ %+ blocks_left +%assign reg_idx (%%NUM_BLOCKS / 4) + +%xdefine %%REG_IN %%CIPHER_IN %+ reg_idx + +%if blocks_left == 1 + vmovdqu64 XWORD(%%HK), [%%KP + hashk] + vpclmulqdq XWORD(%%T0M1), XWORD(%%REG_IN), XWORD(%%HK), 0x01 ; M1 = a1*b0 + vpclmulqdq XWORD(%%T0M2), XWORD(%%REG_IN), XWORD(%%HK), 0x10 ; M2 = a0*b1 + vpclmulqdq XWORD(%%T0H), XWORD(%%REG_IN), XWORD(%%HK), 0x11 ; H = a1*b1 + vpclmulqdq XWORD(%%T0L), XWORD(%%REG_IN), XWORD(%%HK), 0x00 ; L = a0*b0 +%elif blocks_left == 2 + vmovdqu64 YWORD(%%HK), [%%KP + hashk] + vpclmulqdq YWORD(%%T0M1), YWORD(%%REG_IN), YWORD(%%HK), 0x01 ; M1 = a1*b0 + vpclmulqdq YWORD(%%T0M2), YWORD(%%REG_IN), YWORD(%%HK), 0x10 ; M2 = a0*b1 + vpclmulqdq YWORD(%%T0H), YWORD(%%REG_IN), YWORD(%%HK), 0x11 ; H = a1*b1 + vpclmulqdq YWORD(%%T0L), YWORD(%%REG_IN), YWORD(%%HK), 0x00 ; L = a0*b0 +%else ; blocks_left == 3 + vmovdqu64 YWORD(%%HK), [%%KP + hashk] + vinserti64x2 %%HK, [%%KP + hashk + 32], 2 + vpclmulqdq %%T0M1, %%REG_IN, %%HK, 0x01 ; M1 = a1*b0 + vpclmulqdq %%T0M2, %%REG_IN, %%HK, 0x10 ; M2 = a0*b1 + vpclmulqdq %%T0H, %%REG_IN, %%HK, 0x11 ; H = a1*b1 + vpclmulqdq %%T0L, %%REG_IN, %%HK, 0x00 ; L = a0*b0 +%endif ; blocks_left + +%undef %%REG_IN + +%if %0 == 20 + ;; *** GH/GM/GL passed as arguments +%if %%NUM_BLOCKS >= 4 + ;; add ghash product sums from the first 4, 8 or 12 blocks + vpxorq %%T0M1, %%T0M1, %%T1M1 + vpternlogq %%T0M2, %%GM, %%T1M2, 0x96 + vpternlogq %%T0H, %%GH, %%T1H, 0x96 + vpternlogq %%T0L, %%GL, %%T1L, 0x96 +%else + vpxorq %%T0M1, %%T0M1, %%GM + vpxorq %%T0H, %%T0H, %%GH + vpxorq %%T0L, %%T0L, %%GL +%endif ;; %%NUM_BLOCKS >= 4 +%else + ;; *** GH/GM/GL NOT passed as arguments +%if %%NUM_BLOCKS >= 4 + ;; add ghash product sums from the first 4, 8 or 12 blocks + vpxorq %%T0M1, %%T0M1, %%T1M1 + vpxorq %%T0M2, %%T0M2, %%T1M2 + vpxorq %%T0H, %%T0H, %%T1H + vpxorq %%T0L, %%T0L, %%T1L +%endif ;; %%NUM_BLOCKS >= 4 +%endif ;; %0 == 20 + + ;; integrate TM into TH and TL + vpxorq %%T0M1, %%T0M1, %%T0M2 + vpsrldq %%T1M1, %%T0M1, 8 + vpslldq %%T1M2, %%T0M1, 8 + vpxorq %%T0H, %%T0H, %%T1M1 + vpxorq %%T0L, %%T0L, %%T1M2 +%else + ;; ===================================================== + ;; number of blocks is 4, 8, 12 or 16 + ;; T1H/L/M1/M2 include product sums not T0H/L/M1/M2 +%if %0 == 20 + ;; *** GH/GM/GL passed as arguments + vpxorq %%T1M1, %%T1M1, %%GM + vpxorq %%T1H, %%T1H, %%GH + vpxorq %%T1L, %%T1L, %%GL +%endif + ;; integrate TM into TH and TL + vpxorq %%T1M1, %%T1M1, %%T1M2 + vpsrldq %%T0M1, %%T1M1, 8 + vpslldq %%T0M2, %%T1M1, 8 + vpxorq %%T0H, %%T1H, %%T0M1 + vpxorq %%T0L, %%T1L, %%T0M2 +%endif ; blocks_left > 0 + + ;; add TH and TL 128-bit words horizontally + VHPXORI4x128 %%T0H, %%T1M1 + VHPXORI4x128 %%T0L, %%T1M2 + + ;; reduction + vmovdqa64 XWORD(%%HK), [rel POLY2] + VCLMUL_REDUCE XWORD(%%GHASH), XWORD(%%HK), \ + XWORD(%%T0H), XWORD(%%T0L), XWORD(%%T0M1), XWORD(%%T0M2) +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0) +;;; Input: A and B (128-bits each, bit-reflected) +;;; Output: C = A*B*x mod poly, (i.e. >>1 ) +;;; To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input +;;; GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro GHASH_MUL 7 +%define %%GH %1 ;; [in/out] xmm/ymm/zmm with multiply operand(s) (128-bits) +%define %%HK %2 ;; [in] xmm/ymm/zmm with hash key value(s) (128-bits) +%define %%T1 %3 ;; [clobbered] xmm/ymm/zmm +%define %%T2 %4 ;; [clobbered] xmm/ymm/zmm +%define %%T3 %5 ;; [clobbered] xmm/ymm/zmm +%define %%T4 %6 ;; [clobbered] xmm/ymm/zmm +%define %%T5 %7 ;; [clobbered] xmm/ymm/zmm + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + vpclmulqdq %%T1, %%GH, %%HK, 0x11 ; %%T1 = a1*b1 + vpclmulqdq %%T2, %%GH, %%HK, 0x00 ; %%T2 = a0*b0 + vpclmulqdq %%T3, %%GH, %%HK, 0x01 ; %%T3 = a1*b0 + vpclmulqdq %%GH, %%GH, %%HK, 0x10 ; %%GH = a0*b1 + vpxorq %%GH, %%GH, %%T3 + + + vpsrldq %%T3, %%GH, 8 ; shift-R %%GH 2 DWs + vpslldq %%GH, %%GH, 8 ; shift-L %%GH 2 DWs + + vpxorq %%T1, %%T1, %%T3 + vpxorq %%GH, %%GH, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;first phase of the reduction + vmovdqu64 %%T3, [rel POLY2] + + vpclmulqdq %%T2, %%T3, %%GH, 0x01 + vpslldq %%T2, %%T2, 8 ; shift-L %%T2 2 DWs + + vpxorq %%GH, %%GH, %%T2 ; first phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;second phase of the reduction + vpclmulqdq %%T2, %%T3, %%GH, 0x00 + vpsrldq %%T2, %%T2, 4 ; shift-R only 1-DW to obtain 2-DWs shift-R + + vpclmulqdq %%GH, %%T3, %%GH, 0x10 + vpslldq %%GH, %%GH, 4 ; Shift-L 1-DW to obtain result with no shifts + + ; second phase of the reduction complete, the result is in %%GH + vpternlogq %%GH, %%T1, %%T2, 0x96 ; GH = GH xor T1 xor T2 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; In PRECOMPUTE, the commands filling Hashkey_i_k are not required for avx512 +;;; functions, but are kept to allow users to switch cpu architectures between calls +;;; of pre, init, update, and finalize. +%macro PRECOMPUTE 10 +%define %%GDATA %1 ;; [in/out] GPR, pointer to GCM key data structure, content updated +%define %%HK %2 ;; [in] xmm, hash key +%define %%T1 %3 ;; [clobbered] xmm +%define %%T2 %4 ;; [clobbered] xmm +%define %%T3 %5 ;; [clobbered] xmm +%define %%T4 %6 ;; [clobbered] xmm +%define %%T5 %7 ;; [clobbered] xmm +%define %%T6 %8 ;; [clobbered] xmm +%define %%T7 %9 ;; [clobbered] xmm +%define %%T8 %10 ;; [clobbered] xmm + +%xdefine %%ZT1 ZWORD(%%T1) +%xdefine %%ZT2 ZWORD(%%T2) +%xdefine %%ZT3 ZWORD(%%T3) +%xdefine %%ZT4 ZWORD(%%T4) +%xdefine %%ZT5 ZWORD(%%T5) +%xdefine %%ZT6 ZWORD(%%T6) +%xdefine %%ZT7 ZWORD(%%T7) +%xdefine %%ZT8 ZWORD(%%T8) + + vmovdqa64 %%T5, %%HK + vinserti64x2 %%ZT7, %%HK, 3 + + ;; calculate HashKey^2<<1 mod poly + GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 + vmovdqu64 [%%GDATA + HashKey_2], %%T5 + vinserti64x2 %%ZT7, %%T5, 2 + + ;; calculate HashKey^3<<1 mod poly + GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 + vmovdqu64 [%%GDATA + HashKey_3], %%T5 + vinserti64x2 %%ZT7, %%T5, 1 + + ;; calculate HashKey^4<<1 mod poly + GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 + vmovdqu64 [%%GDATA + HashKey_4], %%T5 + vinserti64x2 %%ZT7, %%T5, 0 + + ;; switch to 4x128-bit computations now + vshufi64x2 %%ZT5, %%ZT5, %%ZT5, 0x00 ;; broadcast HashKey^4 across all ZT5 + vmovdqa64 %%ZT8, %%ZT7 ;; save HashKey^4 to HashKey^1 in ZT8 + + ;; calculate HashKey^5<<1 mod poly, HashKey^6<<1 mod poly, ... HashKey^8<<1 mod poly + GHASH_MUL %%ZT7, %%ZT5, %%ZT1, %%ZT3, %%ZT4, %%ZT6, %%ZT2 + vmovdqu64 [%%GDATA + HashKey_8], %%ZT7 ;; HashKey^8 to HashKey^5 in ZT7 now + vshufi64x2 %%ZT5, %%ZT7, %%ZT7, 0x00 ;; broadcast HashKey^8 across all ZT5 + + ;; calculate HashKey^9<<1 mod poly, HashKey^10<<1 mod poly, ... HashKey^48<<1 mod poly + ;; use HashKey^8 as multiplier against ZT8 and ZT7 - this allows deeper ooo execution +%assign i 12 +%rep ((48 - 8) / 8) + ;; compute HashKey^(4 + n), HashKey^(3 + n), ... HashKey^(1 + n) + GHASH_MUL %%ZT8, %%ZT5, %%ZT1, %%ZT3, %%ZT4, %%ZT6, %%ZT2 + vmovdqu64 [%%GDATA + HashKey_ %+ i], %%ZT8 +%assign i (i + 4) + + ;; compute HashKey^(8 + n), HashKey^(7 + n), ... HashKey^(5 + n) + GHASH_MUL %%ZT7, %%ZT5, %%ZT1, %%ZT3, %%ZT4, %%ZT6, %%ZT2 + vmovdqu64 [%%GDATA + HashKey_ %+ i], %%ZT7 +%assign i (i + 4) +%endrep +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; READ_SMALL_DATA_INPUT +;;; Packs xmm register with data when data input is less or equal to 16 bytes +;;; Returns 0 if data has length 0 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro READ_SMALL_DATA_INPUT 6 +%define %%OUTPUT %1 ; [out] xmm register +%define %%INPUT %2 ; [in] buffer pointer to read from +%define %%LENGTH %3 ; [in] number of bytes to read +%define %%TMP1 %4 ; [clobbered] +%define %%TMP2 %5 ; [clobbered] +%define %%MASK %6 ; [out] k1 to k7 register to store the partial block mask + + mov DWORD(%%TMP2), 16 + lea %%TMP1, [rel byte_len_to_mask_table] + cmp %%LENGTH, %%TMP2 + cmovb %%TMP2, %%LENGTH +%ifidn __OUTPUT_FORMAT__, win64 + add %%TMP1, %%TMP2 + add %%TMP1, %%TMP2 + kmovw %%MASK, [%%TMP1] +%else + kmovw %%MASK, [%%TMP1 + %%TMP2*2] +%endif + vmovdqu8 %%OUTPUT{%%MASK}{z}, [%%INPUT] +%endmacro ; READ_SMALL_DATA_INPUT + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; CALC_AAD_HASH: Calculates the hash of the data which will not be encrypted. +; Input: The input data (A_IN), that data's length (A_LEN), and the hash key (HASH_KEY). +; Output: The hash of the data (AAD_HASH). +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro CALC_AAD_HASH 26 +%define %%A_IN %1 ; [in] AAD text pointer +%define %%A_LEN %2 ; [in] AAD length +%define %%AAD_HASH %3 ; [in/out] xmm ghash value +%define %%GDATA_KEY %4 ; [in] pointer to keys +%define %%ZT0 %5 ; [clobbered] ZMM register +%define %%ZT1 %6 ; [clobbered] ZMM register +%define %%ZT2 %7 ; [clobbered] ZMM register +%define %%ZT3 %8 ; [clobbered] ZMM register +%define %%ZT4 %9 ; [clobbered] ZMM register +%define %%ZT5 %10 ; [clobbered] ZMM register +%define %%ZT6 %11 ; [clobbered] ZMM register +%define %%ZT7 %12 ; [clobbered] ZMM register +%define %%ZT8 %13 ; [clobbered] ZMM register +%define %%ZT9 %14 ; [clobbered] ZMM register +%define %%ZT10 %15 ; [clobbered] ZMM register +%define %%ZT11 %16 ; [clobbered] ZMM register +%define %%ZT12 %17 ; [clobbered] ZMM register +%define %%ZT13 %18 ; [clobbered] ZMM register +%define %%ZT14 %19 ; [clobbered] ZMM register +%define %%ZT15 %20 ; [clobbered] ZMM register +%define %%ZT16 %21 ; [clobbered] ZMM register +%define %%ZT17 %22 ; [clobbered] ZMM register +%define %%T1 %23 ; [clobbered] GP register +%define %%T2 %24 ; [clobbered] GP register +%define %%T3 %25 ; [clobbered] GP register +%define %%MASKREG %26 ; [clobbered] mask register + +%define %%SHFMSK %%ZT13 + + mov %%T1, %%A_IN ; T1 = AAD + mov %%T2, %%A_LEN ; T2 = aadLen + + or %%T2, %%T2 + jz %%_CALC_AAD_done + + vmovdqa64 %%SHFMSK, [rel SHUF_MASK] + +%%_get_AAD_loop48x16: + cmp %%T2, (48*16) + jl %%_exit_AAD_loop48x16 + + vmovdqu64 %%ZT1, [%%T1 + 64*0] ; Blocks 0-3 + vmovdqu64 %%ZT2, [%%T1 + 64*1] ; Blocks 4-7 + vmovdqu64 %%ZT3, [%%T1 + 64*2] ; Blocks 8-11 + vmovdqu64 %%ZT4, [%%T1 + 64*3] ; Blocks 12-15 + vpshufb %%ZT1, %%SHFMSK + vpshufb %%ZT2, %%SHFMSK + vpshufb %%ZT3, %%SHFMSK + vpshufb %%ZT4, %%SHFMSK + + GHASH_16 start, %%ZT5, %%ZT6, %%ZT7, \ + NO_INPUT_PTR, NO_INPUT_PTR, NO_INPUT_PTR, \ + %%GDATA_KEY, HashKey_48, 0, ZWORD(%%AAD_HASH), \ + %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ + %%ZT14, %%ZT15, %%ZT16, NO_ZMM, \ + %%ZT1, %%ZT2, %%ZT3, %%ZT4 + + vmovdqu64 %%ZT1, [%%T1 + 16*16 + 64*0] ; Blocks 16-19 + vmovdqu64 %%ZT2, [%%T1 + 16*16 + 64*1] ; Blocks 20-23 + vmovdqu64 %%ZT3, [%%T1 + 16*16 + 64*2] ; Blocks 24-27 + vmovdqu64 %%ZT4, [%%T1 + 16*16 + 64*3] ; Blocks 28-31 + vpshufb %%ZT1, %%SHFMSK + vpshufb %%ZT2, %%SHFMSK + vpshufb %%ZT3, %%SHFMSK + vpshufb %%ZT4, %%SHFMSK + + GHASH_16 mid, %%ZT5, %%ZT6, %%ZT7, \ + NO_INPUT_PTR, NO_INPUT_PTR, NO_INPUT_PTR, \ + %%GDATA_KEY, HashKey_32, 0, NO_HASH_IN_OUT, \ + %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ + %%ZT14, %%ZT15, %%ZT16, NO_ZMM, \ + %%ZT1, %%ZT2, %%ZT3, %%ZT4 + + vmovdqu64 %%ZT1, [%%T1 + 32*16 + 64*0] ; Blocks 32-35 + vmovdqu64 %%ZT2, [%%T1 + 32*16 + 64*1] ; Blocks 36-39 + vmovdqu64 %%ZT3, [%%T1 + 32*16 + 64*2] ; Blocks 40-43 + vmovdqu64 %%ZT4, [%%T1 + 32*16 + 64*3] ; Blocks 44-47 + vpshufb %%ZT1, %%SHFMSK + vpshufb %%ZT2, %%SHFMSK + vpshufb %%ZT3, %%SHFMSK + vpshufb %%ZT4, %%SHFMSK + + GHASH_16 end_reduce, %%ZT5, %%ZT6, %%ZT7, \ + NO_INPUT_PTR, NO_INPUT_PTR, NO_INPUT_PTR, \ + %%GDATA_KEY, HashKey_16, 0, ZWORD(%%AAD_HASH), \ + %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ + %%ZT14, %%ZT15, %%ZT16, NO_ZMM, \ + %%ZT1, %%ZT2, %%ZT3, %%ZT4 + + sub %%T2, (48*16) + je %%_CALC_AAD_done + + add %%T1, (48*16) + jmp %%_get_AAD_loop48x16 + +%%_exit_AAD_loop48x16: + ; Less than 48x16 bytes remaining + cmp %%T2, (32*16) + jl %%_less_than_32x16 + + ; Get next 16 blocks + vmovdqu64 %%ZT1, [%%T1 + 64*0] + vmovdqu64 %%ZT2, [%%T1 + 64*1] + vmovdqu64 %%ZT3, [%%T1 + 64*2] + vmovdqu64 %%ZT4, [%%T1 + 64*3] + vpshufb %%ZT1, %%SHFMSK + vpshufb %%ZT2, %%SHFMSK + vpshufb %%ZT3, %%SHFMSK + vpshufb %%ZT4, %%SHFMSK + + GHASH_16 start, %%ZT5, %%ZT6, %%ZT7, \ + NO_INPUT_PTR, NO_INPUT_PTR, NO_INPUT_PTR, \ + %%GDATA_KEY, HashKey_32, 0, ZWORD(%%AAD_HASH), \ + %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ + %%ZT14, %%ZT15, %%ZT16, NO_ZMM, \ + %%ZT1, %%ZT2, %%ZT3, %%ZT4 + + vmovdqu64 %%ZT1, [%%T1 + 16*16 + 64*0] + vmovdqu64 %%ZT2, [%%T1 + 16*16 + 64*1] + vmovdqu64 %%ZT3, [%%T1 + 16*16 + 64*2] + vmovdqu64 %%ZT4, [%%T1 + 16*16 + 64*3] + vpshufb %%ZT1, %%SHFMSK + vpshufb %%ZT2, %%SHFMSK + vpshufb %%ZT3, %%SHFMSK + vpshufb %%ZT4, %%SHFMSK + + GHASH_16 end_reduce, %%ZT5, %%ZT6, %%ZT7, \ + NO_INPUT_PTR, NO_INPUT_PTR, NO_INPUT_PTR, \ + %%GDATA_KEY, HashKey_16, 0, ZWORD(%%AAD_HASH), \ + %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ + %%ZT14, %%ZT15, %%ZT16, NO_ZMM, \ + %%ZT1, %%ZT2, %%ZT3, %%ZT4 + + sub %%T2, (32*16) + je %%_CALC_AAD_done + + add %%T1, (32*16) + jmp %%_less_than_16x16 + +%%_less_than_32x16: + cmp %%T2, (16*16) + jl %%_less_than_16x16 + ; Get next 16 blocks + vmovdqu64 %%ZT1, [%%T1 + 64*0] + vmovdqu64 %%ZT2, [%%T1 + 64*1] + vmovdqu64 %%ZT3, [%%T1 + 64*2] + vmovdqu64 %%ZT4, [%%T1 + 64*3] + vpshufb %%ZT1, %%SHFMSK + vpshufb %%ZT2, %%SHFMSK + vpshufb %%ZT3, %%SHFMSK + vpshufb %%ZT4, %%SHFMSK + + GHASH_16 start_reduce, %%ZT5, %%ZT6, %%ZT7, \ + NO_INPUT_PTR, NO_INPUT_PTR, NO_INPUT_PTR, \ + %%GDATA_KEY, HashKey_16, 0, ZWORD(%%AAD_HASH), \ + %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ + %%ZT14, %%ZT15, %%ZT16, NO_ZMM, \ + %%ZT1, %%ZT2, %%ZT3, %%ZT4 + + sub %%T2, (16*16) + je %%_CALC_AAD_done + + add %%T1, (16*16) + + ; Less than 16x16 bytes remaining +%%_less_than_16x16: + ;; prep mask source address + lea %%T3, [rel byte64_len_to_mask_table] + lea %%T3, [%%T3 + %%T2*8] + + ;; calculate number of blocks to ghash (including partial bytes) + add DWORD(%%T2), 15 + shr DWORD(%%T2), 4 + cmp DWORD(%%T2), 2 + jb %%_AAD_blocks_1 + je %%_AAD_blocks_2 + cmp DWORD(%%T2), 4 + jb %%_AAD_blocks_3 + je %%_AAD_blocks_4 + cmp DWORD(%%T2), 6 + jb %%_AAD_blocks_5 + je %%_AAD_blocks_6 + cmp DWORD(%%T2), 8 + jb %%_AAD_blocks_7 + je %%_AAD_blocks_8 + cmp DWORD(%%T2), 10 + jb %%_AAD_blocks_9 + je %%_AAD_blocks_10 + cmp DWORD(%%T2), 12 + jb %%_AAD_blocks_11 + je %%_AAD_blocks_12 + cmp DWORD(%%T2), 14 + jb %%_AAD_blocks_13 + je %%_AAD_blocks_14 + cmp DWORD(%%T2), 15 + je %%_AAD_blocks_15 + ;; fall through for 16 blocks + + ;; The flow of each of these cases is identical: + ;; - load blocks plain text + ;; - shuffle loaded blocks + ;; - xor in current hash value into block 0 + ;; - perform up multiplications with ghash keys + ;; - jump to reduction code + +%assign I 16 + ;; generate all 16 cases using preprocessor +%rep 16 + +%%_AAD_blocks_ %+ I: +%if I > 12 + sub %%T3, 12 * 16 * 8 +%elif I > 8 + sub %%T3, 8 * 16 * 8 +%elif I > 4 + sub %%T3, 4 * 16 * 8 +%endif + kmovq %%MASKREG, [%%T3] + + ZMM_LOAD_MASKED_BLOCKS_0_16 \ + I, %%T1, 0, \ + %%ZT1, %%ZT2, %%ZT3, %%ZT4, %%MASKREG + + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 \ + I, vpshufb, \ + %%ZT1, %%ZT2, %%ZT3, %%ZT4, \ + %%ZT1, %%ZT2, %%ZT3, %%ZT4, \ + %%SHFMSK, %%SHFMSK, %%SHFMSK, %%SHFMSK + + GHASH_1_TO_16 %%GDATA_KEY, ZWORD(%%AAD_HASH), \ + %%ZT0, %%ZT5, %%ZT6, %%ZT7, %%ZT8, \ + %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ + ZWORD(%%AAD_HASH), %%ZT1, %%ZT2, %%ZT3, %%ZT4, I +%if I > 1 + ;; fall through to CALC_AAD_done in 1 block case + jmp %%_CALC_AAD_done +%endif + +%assign I (I - 1) +%endrep + +%%_CALC_AAD_done: + ;; result in AAD_HASH + +%endmacro ; CALC_AAD_HASH + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; PARTIAL_BLOCK +;;; Handles encryption/decryption and the tag partial blocks between +;;; update calls. +;;; Requires the input data be at least 1 byte long. +;;; Output: +;;; A cipher/plain of the first partial block (CIPH_PLAIN_OUT), +;;; AAD_HASH and updated GDATA_CTX +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro PARTIAL_BLOCK 22 +%define %%GDATA_KEY %1 ; [in] key pointer +%define %%GDATA_CTX %2 ; [in] context pointer +%define %%CIPH_PLAIN_OUT %3 ; [in] output buffer +%define %%PLAIN_CIPH_IN %4 ; [in] input buffer +%define %%PLAIN_CIPH_LEN %5 ; [in] buffer length +%define %%DATA_OFFSET %6 ; [out] data offset (gets set) +%define %%AAD_HASH %7 ; [out] updated GHASH value +%define %%ENC_DEC %8 ; [in] cipher direction +%define %%GPTMP0 %9 ; [clobbered] GP temporary register +%define %%GPTMP1 %10 ; [clobbered] GP temporary register +%define %%GPTMP2 %11 ; [clobbered] GP temporary register +%define %%ZTMP0 %12 ; [clobbered] ZMM temporary register +%define %%ZTMP1 %13 ; [clobbered] ZMM temporary register +%define %%ZTMP2 %14 ; [clobbered] ZMM temporary register +%define %%ZTMP3 %15 ; [clobbered] ZMM temporary register +%define %%ZTMP4 %16 ; [clobbered] ZMM temporary register +%define %%ZTMP5 %17 ; [clobbered] ZMM temporary register +%define %%ZTMP6 %18 ; [clobbered] ZMM temporary register +%define %%ZTMP7 %19 ; [clobbered] ZMM temporary register +%define %%ZTMP8 %20 ; [clobbered] ZMM temporary register +%define %%ZTMP9 %21 ; [clobbered] ZMM temporary register +%define %%MASKREG %22 ; [clobbered] mask temporary register + +%define %%XTMP0 XWORD(%%ZTMP0) +%define %%XTMP1 XWORD(%%ZTMP1) +%define %%XTMP2 XWORD(%%ZTMP2) +%define %%XTMP3 XWORD(%%ZTMP3) +%define %%XTMP4 XWORD(%%ZTMP4) +%define %%XTMP5 XWORD(%%ZTMP5) +%define %%XTMP6 XWORD(%%ZTMP6) +%define %%XTMP7 XWORD(%%ZTMP7) +%define %%XTMP8 XWORD(%%ZTMP8) +%define %%XTMP9 XWORD(%%ZTMP9) + +%define %%LENGTH %%DATA_OFFSET +%define %%IA0 %%GPTMP1 +%define %%IA1 %%GPTMP2 +%define %%IA2 %%GPTMP0 + + ;; if no partial block present then LENGTH/DATA_OFFSET will be set to zero + mov %%LENGTH, [%%GDATA_CTX + PBlockLen] + or %%LENGTH, %%LENGTH + je %%_partial_block_done ;Leave Macro if no partial blocks + + READ_SMALL_DATA_INPUT %%XTMP0, %%PLAIN_CIPH_IN, %%PLAIN_CIPH_LEN, %%IA0, %%IA2, %%MASKREG + + ;; XTMP1 = my_ctx_data.partial_block_enc_key + vmovdqu64 %%XTMP1, [%%GDATA_CTX + PBlockEncKey] + vmovdqu64 %%XTMP2, [%%GDATA_KEY + HashKey] + + ;; adjust the shuffle mask pointer to be able to shift right %%LENGTH bytes + ;; (16 - %%LENGTH) is the number of bytes in plaintext mod 16) + lea %%IA0, [rel SHIFT_MASK] + add %%IA0, %%LENGTH + vmovdqu64 %%XTMP3, [%%IA0] ; shift right shuffle mask + vpshufb %%XTMP1, %%XTMP3 + +%ifidn %%ENC_DEC, DEC + ;; keep copy of cipher text in %%XTMP4 + vmovdqa64 %%XTMP4, %%XTMP0 +%endif + vpxorq %%XTMP1, %%XTMP0 ; Ciphertext XOR E(K, Yn) + + ;; Set %%IA1 to be the amount of data left in CIPH_PLAIN_IN after filling the block + ;; Determine if partial block is not being filled and shift mask accordingly +%ifidn __OUTPUT_FORMAT__, win64 + mov %%IA1, %%PLAIN_CIPH_LEN + add %%IA1, %%LENGTH +%else + lea %%IA1, [%%PLAIN_CIPH_LEN + %%LENGTH] +%endif + sub %%IA1, 16 + jge %%_no_extra_mask + sub %%IA0, %%IA1 +%%_no_extra_mask: + ;; get the appropriate mask to mask out bottom %%LENGTH bytes of %%XTMP1 + ;; - mask out bottom %%LENGTH bytes of %%XTMP1 + vmovdqu64 %%XTMP0, [%%IA0 + ALL_F - SHIFT_MASK] + vpand %%XTMP1, %%XTMP0 + +%ifidn %%ENC_DEC, DEC + vpand %%XTMP4, %%XTMP0 + vpshufb %%XTMP4, [rel SHUF_MASK] + vpshufb %%XTMP4, %%XTMP3 + vpxorq %%AAD_HASH, %%XTMP4 +%else + vpshufb %%XTMP1, [rel SHUF_MASK] + vpshufb %%XTMP1, %%XTMP3 + vpxorq %%AAD_HASH, %%XTMP1 +%endif + cmp %%IA1, 0 + jl %%_partial_incomplete + + ;; GHASH computation for the last <16 Byte block + GHASH_MUL %%AAD_HASH, %%XTMP2, %%XTMP5, %%XTMP6, %%XTMP7, %%XTMP8, %%XTMP9 + + mov qword [%%GDATA_CTX + PBlockLen], 0 + + ;; Set %%LENGTH to be the number of bytes to write out + mov %%IA0, %%LENGTH + mov %%LENGTH, 16 + sub %%LENGTH, %%IA0 + jmp %%_enc_dec_done + +%%_partial_incomplete: +%ifidn __OUTPUT_FORMAT__, win64 + mov %%IA0, %%PLAIN_CIPH_LEN + add [%%GDATA_CTX + PBlockLen], %%IA0 +%else + add [%%GDATA_CTX + PBlockLen], %%PLAIN_CIPH_LEN +%endif + mov %%LENGTH, %%PLAIN_CIPH_LEN + +%%_enc_dec_done: + ;; output encrypted Bytes + + lea %%IA0, [rel byte_len_to_mask_table] + kmovw %%MASKREG, [%%IA0 + %%LENGTH*2] + vmovdqu64 [%%GDATA_CTX + AadHash], %%AAD_HASH + +%ifidn %%ENC_DEC, ENC + ;; shuffle XTMP1 back to output as ciphertext + vpshufb %%XTMP1, [rel SHUF_MASK] + vpshufb %%XTMP1, %%XTMP3 +%endif + vmovdqu8 [%%CIPH_PLAIN_OUT]{%%MASKREG}, %%XTMP1 +%%_partial_block_done: +%endmacro ; PARTIAL_BLOCK + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Ciphers 1 to 16 blocks and prepares them for later GHASH compute operation +%macro INITIAL_BLOCKS_PARTIAL_CIPHER 25 +%define %%GDATA_KEY %1 ; [in] key pointer +%define %%GDATA_CTX %2 ; [in] context pointer +%define %%CIPH_PLAIN_OUT %3 ; [in] text output pointer +%define %%PLAIN_CIPH_IN %4 ; [in] text input pointer +%define %%LENGTH %5 ; [in/clobbered] length in bytes +%define %%DATA_OFFSET %6 ; [in/out] current data offset (updated) +%define %%NUM_BLOCKS %7 ; [in] can only be 1, 2, 3, 4, 5, ..., 15 or 16 (not 0) +%define %%CTR %8 ; [in/out] current counter value +%define %%ENC_DEC %9 ; [in] cipher direction (ENC/DEC) +%define %%INSTANCE_TYPE %10 ; [in] multi_call or single_call +%define %%DAT0 %11 ; [out] ZMM with cipher text shuffled for GHASH +%define %%DAT1 %12 ; [out] ZMM with cipher text shuffled for GHASH +%define %%DAT2 %13 ; [out] ZMM with cipher text shuffled for GHASH +%define %%DAT3 %14 ; [out] ZMM with cipher text shuffled for GHASH +%define %%LAST_CIPHER_BLK %15 ; [out] XMM to put ciphered counter block partially xor'ed with text +%define %%LAST_GHASH_BLK %16 ; [out] XMM to put last cipher text block shuffled for GHASH +%define %%CTR0 %17 ; [clobbered] ZMM temporary +%define %%CTR1 %18 ; [clobbered] ZMM temporary +%define %%CTR2 %19 ; [clobbered] ZMM temporary +%define %%CTR3 %20 ; [clobbered] ZMM temporary +%define %%ZT1 %21 ; [clobbered] ZMM temporary +%define %%IA0 %22 ; [clobbered] GP temporary +%define %%IA1 %23 ; [clobbered] GP temporary +%define %%MASKREG %24 ; [clobbered] mask register +%define %%SHUFMASK %25 ; [out] ZMM loaded with BE/LE shuffle mask + +%if %%NUM_BLOCKS == 1 + vmovdqa64 XWORD(%%SHUFMASK), [rel SHUF_MASK] +%elif %%NUM_BLOCKS == 2 + vmovdqa64 YWORD(%%SHUFMASK), [rel SHUF_MASK] +%else + vmovdqa64 %%SHUFMASK, [rel SHUF_MASK] +%endif + ;; prepare AES counter blocks +%if %%NUM_BLOCKS == 1 + vpaddd XWORD(%%CTR0), %%CTR, [rel ONE] +%elif %%NUM_BLOCKS == 2 + vshufi64x2 YWORD(%%CTR0), YWORD(%%CTR), YWORD(%%CTR), 0 + vpaddd YWORD(%%CTR0), YWORD(%%CTR0), [rel ddq_add_1234] +%else + vshufi64x2 ZWORD(%%CTR), ZWORD(%%CTR), ZWORD(%%CTR), 0 + vpaddd %%CTR0, ZWORD(%%CTR), [rel ddq_add_1234] +%if %%NUM_BLOCKS > 4 + vpaddd %%CTR1, ZWORD(%%CTR), [rel ddq_add_5678] +%endif +%if %%NUM_BLOCKS > 8 + vpaddd %%CTR2, %%CTR0, [rel ddq_add_8888] +%endif +%if %%NUM_BLOCKS > 12 + vpaddd %%CTR3, %%CTR1, [rel ddq_add_8888] +%endif +%endif + + ;; get load/store mask + lea %%IA0, [rel byte64_len_to_mask_table] + mov %%IA1, %%LENGTH +%if %%NUM_BLOCKS > 12 + sub %%IA1, 3 * 64 +%elif %%NUM_BLOCKS > 8 + sub %%IA1, 2 * 64 +%elif %%NUM_BLOCKS > 4 + sub %%IA1, 64 +%endif + kmovq %%MASKREG, [%%IA0 + %%IA1*8] + + ;; extract new counter value + ;; shuffle the counters for AES rounds +%ifidn %%INSTANCE_TYPE, multi_call +%if %%NUM_BLOCKS <= 4 + vextracti32x4 %%CTR, %%CTR0, (%%NUM_BLOCKS - 1) +%elif %%NUM_BLOCKS <= 8 + vextracti32x4 %%CTR, %%CTR1, (%%NUM_BLOCKS - 5) +%elif %%NUM_BLOCKS <= 12 + vextracti32x4 %%CTR, %%CTR2, (%%NUM_BLOCKS - 9) +%else + vextracti32x4 %%CTR, %%CTR3, (%%NUM_BLOCKS - 13) +%endif +%endif + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vpshufb, \ + %%CTR0, %%CTR1, %%CTR2, %%CTR3, \ + %%CTR0, %%CTR1, %%CTR2, %%CTR3, \ + %%SHUFMASK, %%SHUFMASK, %%SHUFMASK, %%SHUFMASK + + ;; load plain/cipher text + ZMM_LOAD_MASKED_BLOCKS_0_16 %%NUM_BLOCKS, %%PLAIN_CIPH_IN, %%DATA_OFFSET, \ + %%DAT0, %%DAT1, %%DAT2, %%DAT3, %%MASKREG + + ;; AES rounds and XOR with plain/cipher text +%assign j 0 +%rep (NROUNDS + 2) + vbroadcastf64x2 %%ZT1, [%%GDATA_KEY + (j * 16)] + ZMM_AESENC_ROUND_BLOCKS_0_16 %%CTR0, %%CTR1, %%CTR2, %%CTR3, \ + %%ZT1, j, \ + %%DAT0, %%DAT1, %%DAT2, %%DAT3, \ + %%NUM_BLOCKS, NROUNDS +%assign j (j + 1) +%endrep + +%ifidn %%INSTANCE_TYPE, multi_call + ;; retrieve the last cipher counter block (partially XOR'ed with text) + ;; - this is needed for partial block cases +%if %%NUM_BLOCKS <= 4 + vextracti32x4 %%LAST_CIPHER_BLK, %%CTR0, (%%NUM_BLOCKS - 1) +%elif %%NUM_BLOCKS <= 8 + vextracti32x4 %%LAST_CIPHER_BLK, %%CTR1, (%%NUM_BLOCKS - 5) +%elif %%NUM_BLOCKS <= 12 + vextracti32x4 %%LAST_CIPHER_BLK, %%CTR2, (%%NUM_BLOCKS - 9) +%else + vextracti32x4 %%LAST_CIPHER_BLK, %%CTR3, (%%NUM_BLOCKS - 13) +%endif +%endif + ;; write cipher/plain text back to output and + ZMM_STORE_MASKED_BLOCKS_0_16 %%NUM_BLOCKS, %%CIPH_PLAIN_OUT, %%DATA_OFFSET, \ + %%CTR0, %%CTR1, %%CTR2, %%CTR3, %%MASKREG + + ;; zero bytes outside the mask before hashing +%if %%NUM_BLOCKS <= 4 + vmovdqu8 %%CTR0{%%MASKREG}{z}, %%CTR0 +%elif %%NUM_BLOCKS <= 8 + vmovdqu8 %%CTR1{%%MASKREG}{z}, %%CTR1 +%elif %%NUM_BLOCKS <= 12 + vmovdqu8 %%CTR2{%%MASKREG}{z}, %%CTR2 +%else + vmovdqu8 %%CTR3{%%MASKREG}{z}, %%CTR3 +%endif + + ;; Shuffle the cipher text blocks for hashing part + ;; ZT5 and ZT6 are expected outputs with blocks for hashing +%ifidn %%ENC_DEC, DEC + ;; Decrypt case + ;; - cipher blocks are in ZT5 & ZT6 + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vpshufb, \ + %%DAT0, %%DAT1, %%DAT2, %%DAT3, \ + %%DAT0, %%DAT1, %%DAT2, %%DAT3, \ + %%SHUFMASK, %%SHUFMASK, %%SHUFMASK, %%SHUFMASK +%else + ;; Encrypt case + ;; - cipher blocks are in CTR0-CTR3 + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vpshufb, \ + %%DAT0, %%DAT1, %%DAT2, %%DAT3, \ + %%CTR0, %%CTR1, %%CTR2, %%CTR3, \ + %%SHUFMASK, %%SHUFMASK, %%SHUFMASK, %%SHUFMASK +%endif ; Encrypt + +%ifidn %%INSTANCE_TYPE, multi_call + ;; Extract the last block for partials and multi_call cases +%if %%NUM_BLOCKS <= 4 + vextracti32x4 %%LAST_GHASH_BLK, %%DAT0, %%NUM_BLOCKS - 1 +%elif %%NUM_BLOCKS <= 8 + vextracti32x4 %%LAST_GHASH_BLK, %%DAT1, %%NUM_BLOCKS - 5 +%elif %%NUM_BLOCKS <= 12 + vextracti32x4 %%LAST_GHASH_BLK, %%DAT2, %%NUM_BLOCKS - 9 +%else + vextracti32x4 %%LAST_GHASH_BLK, %%DAT3, %%NUM_BLOCKS - 13 +%endif +%endif + +%endmacro ; INITIAL_BLOCKS_PARTIAL_CIPHER + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Computes GHASH on 1 to 16 blocks +%macro INITIAL_BLOCKS_PARTIAL_GHASH 22-25 +%define %%GDATA_KEY %1 ; [in] key pointer +%define %%GDATA_CTX %2 ; [in] context pointer +%define %%LENGTH %3 ; [in/clobbered] length in bytes +%define %%NUM_BLOCKS %4 ; [in] can only be 1, 2, 3, 4, 5, ..., 15 or 16 (not 0) +%define %%HASH_IN_OUT %5 ; [in/out] XMM ghash in/out value +%define %%ENC_DEC %6 ; [in] cipher direction (ENC/DEC) +%define %%INSTANCE_TYPE %7 ; [in] multi_call or single_call +%define %%DAT0 %8 ; [in] ZMM with cipher text shuffled for GHASH +%define %%DAT1 %9 ; [in] ZMM with cipher text shuffled for GHASH +%define %%DAT2 %10 ; [in] ZMM with cipher text shuffled for GHASH +%define %%DAT3 %11 ; [in] ZMM with cipher text shuffled for GHASH +%define %%LAST_CIPHER_BLK %12 ; [in] XMM with ciphered counter block partially xor'ed with text +%define %%LAST_GHASH_BLK %13 ; [in] XMM with last cipher text block shuffled for GHASH +%define %%ZT0 %14 ; [clobbered] ZMM temporary +%define %%ZT1 %15 ; [clobbered] ZMM temporary +%define %%ZT2 %16 ; [clobbered] ZMM temporary +%define %%ZT3 %17 ; [clobbered] ZMM temporary +%define %%ZT4 %18 ; [clobbered] ZMM temporary +%define %%ZT5 %19 ; [clobbered] ZMM temporary +%define %%ZT6 %20 ; [clobbered] ZMM temporary +%define %%ZT7 %21 ; [clobbered] ZMM temporary +%define %%ZT8 %22 ; [clobbered] ZMM temporary +%define %%GH %23 ; [in] ZMM with hi product part +%define %%GM %24 ; [in] ZMM with mid prodcut part +%define %%GL %25 ; [in] ZMM with lo product part + + +%ifidn %%INSTANCE_TYPE, single_call + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;;; SINGLE CALL case + ;;; - hash all data including partial block + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%if %0 == 22 + ;; start GHASH compute + GHASH_1_TO_16 %%GDATA_KEY, %%HASH_IN_OUT, \ + %%ZT0, %%ZT1, %%ZT2, %%ZT3, %%ZT4, \ + %%ZT5, %%ZT6, %%ZT7, %%ZT8, ZWORD(%%HASH_IN_OUT), \ + %%DAT0, %%DAT1, %%DAT2, %%DAT3, %%NUM_BLOCKS +%elif %0 == 25 + ;; continue GHASH compute + GHASH_1_TO_16 %%GDATA_KEY, %%HASH_IN_OUT, \ + %%ZT0, %%ZT1, %%ZT2, %%ZT3, %%ZT4, \ + %%ZT5, %%ZT6, %%ZT7, %%ZT8, ZWORD(%%HASH_IN_OUT), \ + %%DAT0, %%DAT1, %%DAT2, %%DAT3, %%NUM_BLOCKS, %%GH, %%GM, %%GL +%endif + +%else + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;;; MULTI CALL (SGL) case + ;;; - hash all but the last partial block of data + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + ;; update data offset +%if %%NUM_BLOCKS > 1 + ;; The final block of data may be <16B + sub %%LENGTH, 16 * (%%NUM_BLOCKS - 1) +%endif + +%if %%NUM_BLOCKS < 16 + ;; NOTE: the 'jl' is always taken for num_initial_blocks = 16. + ;; This is run in the context of GCM_ENC_DEC_SMALL for length < 256. + cmp %%LENGTH, 16 + jl %%_small_initial_partial_block + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;;; Handle a full length final block - encrypt and hash all blocks + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + sub %%LENGTH, 16 + mov qword [%%GDATA_CTX + PBlockLen], 0 + + ;; Hash all of the data +%if %0 == 22 + ;; start GHASH compute + GHASH_1_TO_16 %%GDATA_KEY, %%HASH_IN_OUT, \ + %%ZT0, %%ZT1, %%ZT2, %%ZT3, %%ZT4, \ + %%ZT5, %%ZT6, %%ZT7, %%ZT8, ZWORD(%%HASH_IN_OUT), \ + %%DAT0, %%DAT1, %%DAT2, %%DAT3, %%NUM_BLOCKS +%elif %0 == 25 + ;; continue GHASH compute + GHASH_1_TO_16 %%GDATA_KEY, %%HASH_IN_OUT, \ + %%ZT0, %%ZT1, %%ZT2, %%ZT3, %%ZT4, \ + %%ZT5, %%ZT6, %%ZT7, %%ZT8, ZWORD(%%HASH_IN_OUT), \ + %%DAT0, %%DAT1, %%DAT2, %%DAT3, %%NUM_BLOCKS, %%GH, %%GM, %%GL +%endif + jmp %%_small_initial_compute_done +%endif ; %if %%NUM_BLOCKS < 16 + +%%_small_initial_partial_block: + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;;; Handle ghash for a <16B final block + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + ;; In this case if it's a single call to encrypt we can + ;; hash all of the data but if it's an init / update / finalize + ;; series of call we need to leave the last block if it's + ;; less than a full block of data. + + mov [%%GDATA_CTX + PBlockLen], %%LENGTH + vmovdqu64 [%%GDATA_CTX + PBlockEncKey], %%LAST_CIPHER_BLK + +%assign k (%%NUM_BLOCKS - 1) +%assign last_block_to_hash 1 + +%if (%%NUM_BLOCKS > last_block_to_hash) + + ;; ZT12-ZT20 - temporary registers +%if %0 == 22 + ;; start GHASH compute + GHASH_1_TO_16 %%GDATA_KEY, %%HASH_IN_OUT, \ + %%ZT0, %%ZT1, %%ZT2, %%ZT3, %%ZT4, \ + %%ZT5, %%ZT6, %%ZT7, %%ZT8, ZWORD(%%HASH_IN_OUT), \ + %%DAT0, %%DAT1, %%DAT2, %%DAT3, k +%elif %0 == 25 + ;; continue GHASH compute + GHASH_1_TO_16 %%GDATA_KEY, %%HASH_IN_OUT, \ + %%ZT0, %%ZT1, %%ZT2, %%ZT3, %%ZT4, \ + %%ZT5, %%ZT6, %%ZT7, %%ZT8, ZWORD(%%HASH_IN_OUT), \ + %%DAT0, %%DAT1, %%DAT2, %%DAT3, k, %%GH, %%GM, %%GL +%endif + ;; just fall through no jmp needed +%else + +%if %0 == 25 + ;; Reduction is required in this case. + ;; Integrate GM into GH and GL. + vpsrldq %%ZT0, %%GM, 8 + vpslldq %%ZT1, %%GM, 8 + vpxorq %%GH, %%GH, %%ZT0 + vpxorq %%GL, %%GL, %%ZT1 + + ;; Add GH and GL 128-bit words horizontally + VHPXORI4x128 %%GH, %%ZT0 + VHPXORI4x128 %%GL, %%ZT1 + + ;; 256-bit to 128-bit reduction + vmovdqa64 XWORD(%%ZT0), [rel POLY2] + VCLMUL_REDUCE XWORD(%%HASH_IN_OUT), XWORD(%%ZT0), \ + XWORD(%%GH), XWORD(%%GL), XWORD(%%ZT1), XWORD(%%ZT2) +%endif + ;; Record that a reduction is not needed - + ;; In this case no hashes are computed because there + ;; is only one initial block and it is < 16B in length. + ;; We only need to check if a reduction is needed if + ;; initial_blocks == 1 and init/update/final is being used. + ;; In this case we may just have a partial block, and that + ;; gets hashed in finalize. + + ;; The hash should end up in HASH_IN_OUT. + ;; The only way we should get here is if there is + ;; a partial block of data, so xor that into the hash. + vpxorq %%HASH_IN_OUT, %%HASH_IN_OUT, %%LAST_GHASH_BLK + ;; The result is in %%HASH_IN_OUT + jmp %%_after_reduction +%endif + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; After GHASH reduction +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%%_small_initial_compute_done: + ;; If using init/update/finalize, we need to xor any partial block data + ;; into the hash. +%if %%NUM_BLOCKS > 1 + ;; NOTE: for %%NUM_BLOCKS = 0 the xor never takes place +%if %%NUM_BLOCKS != 16 + ;; NOTE: for %%NUM_BLOCKS = 16, %%LENGTH, stored in [PBlockLen] is never zero + or %%LENGTH, %%LENGTH + je %%_after_reduction +%endif ; %%NUM_BLOCKS != 16 + vpxorq %%HASH_IN_OUT, %%HASH_IN_OUT, %%LAST_GHASH_BLK +%endif ; %%NUM_BLOCKS > 1 + +%%_after_reduction: + +%endif ; %%INSTANCE_TYPE, multi_call + + ;; Final hash is now in HASH_IN_OUT + +%endmacro ; INITIAL_BLOCKS_PARTIAL_GHASH + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; INITIAL_BLOCKS_PARTIAL macro with support for a partial final block. +;;; It may look similar to INITIAL_BLOCKS but its usage is different: +;;; - first encrypts/decrypts required number of blocks and then +;;; ghashes these blocks +;;; - Small packets or left over data chunks (<256 bytes) +;;; - single or multi call +;;; - Remaining data chunks below 256 bytes (multi buffer code) +;;; +;;; num_initial_blocks is expected to include the partial final block +;;; in the count. +%macro INITIAL_BLOCKS_PARTIAL 30 +%define %%GDATA_KEY %1 ; [in] key pointer +%define %%GDATA_CTX %2 ; [in] context pointer +%define %%CIPH_PLAIN_OUT %3 ; [in] text output pointer +%define %%PLAIN_CIPH_IN %4 ; [in] text input pointer +%define %%LENGTH %5 ; [in/clobbered] length in bytes +%define %%DATA_OFFSET %6 ; [in/out] current data offset (updated) +%define %%NUM_BLOCKS %7 ; [in] can only be 1, 2, 3, 4, 5, ..., 15 or 16 (not 0) +%define %%CTR %8 ; [in/out] current counter value +%define %%HASH_IN_OUT %9 ; [in/out] XMM ghash in/out value +%define %%ENC_DEC %10 ; [in] cipher direction (ENC/DEC) +%define %%INSTANCE_TYPE %11 ; [in] multi_call or single_call +%define %%CTR0 %12 ; [clobbered] ZMM temporary +%define %%CTR1 %13 ; [clobbered] ZMM temporary +%define %%CTR2 %14 ; [clobbered] ZMM temporary +%define %%CTR3 %15 ; [clobbered] ZMM temporary +%define %%DAT0 %16 ; [clobbered] ZMM temporary +%define %%DAT1 %17 ; [clobbered] ZMM temporary +%define %%DAT2 %18 ; [clobbered] ZMM temporary +%define %%DAT3 %19 ; [clobbered] ZMM temporary +%define %%LAST_CIPHER_BLK %20 ; [clobbered] ZMM temporary +%define %%LAST_GHASH_BLK %21 ; [clobbered] ZMM temporary +%define %%ZT0 %22 ; [clobbered] ZMM temporary +%define %%ZT1 %23 ; [clobbered] ZMM temporary +%define %%ZT2 %24 ; [clobbered] ZMM temporary +%define %%ZT3 %25 ; [clobbered] ZMM temporary +%define %%ZT4 %26 ; [clobbered] ZMM temporary +%define %%IA0 %27 ; [clobbered] GP temporary +%define %%IA1 %28 ; [clobbered] GP temporary +%define %%MASKREG %29 ; [clobbered] mask register +%define %%SHUFMASK %30 ; [clobbered] ZMM for BE/LE shuffle mask + + INITIAL_BLOCKS_PARTIAL_CIPHER \ + %%GDATA_KEY, %%GDATA_CTX, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, \ + %%LENGTH, %%DATA_OFFSET, %%NUM_BLOCKS, %%CTR, \ + %%ENC_DEC, %%INSTANCE_TYPE, %%DAT0, %%DAT1, %%DAT2, %%DAT3, \ + XWORD(%%LAST_CIPHER_BLK), XWORD(%%LAST_GHASH_BLK), \ + %%CTR0, %%CTR1, %%CTR2, %%CTR3, %%ZT0, \ + %%IA0, %%IA1, %%MASKREG, %%SHUFMASK + + INITIAL_BLOCKS_PARTIAL_GHASH \ + %%GDATA_KEY, %%GDATA_CTX, %%LENGTH, \ + %%NUM_BLOCKS, %%HASH_IN_OUT, %%ENC_DEC, \ + %%INSTANCE_TYPE, %%DAT0, %%DAT1, %%DAT2, %%DAT3, \ + XWORD(%%LAST_CIPHER_BLK), XWORD(%%LAST_GHASH_BLK), \ + %%CTR0, %%CTR1, %%CTR2, %%CTR3, %%ZT0, %%ZT1, \ + %%ZT2, %%ZT3, %%ZT4 + +%endmacro ; INITIAL_BLOCKS_PARTIAL + +;;; =========================================================================== +;;; =========================================================================== +;;; Stitched GHASH of 16 blocks (with reduction) with encryption of N blocks +;;; followed with GHASH of the N blocks. +%macro GHASH_16_ENCRYPT_N_GHASH_N 47 +%define %%GDATA %1 ; [in] key pointer +%define %%GCTX %2 ; [in] context pointer +%define %%CIPH_PLAIN_OUT %3 ; [in] pointer to output buffer +%define %%PLAIN_CIPH_IN %4 ; [in] pointer to input buffer +%define %%DATA_OFFSET %5 ; [in] data offset +%define %%LENGTH %6 ; [in] data length +%define %%CTR_BE %7 ; [in/out] ZMM counter blocks (last 4) in big-endian +%define %%CTR_CHECK %8 ; [in/out] GP with 8-bit counter for overflow check +%define %%HASHKEY_OFFSET %9 ; [in] numerical offset for the highest hash key +%define %%GHASHIN_BLK_OFFSET %10 ; [in] numerical offset for GHASH blocks in +%define %%SHFMSK %11 ; [in] ZMM with byte swap mask for pshufb +%define %%B00_03 %12 ; [clobbered] temporary ZMM +%define %%B04_07 %13 ; [clobbered] temporary ZMM +%define %%B08_11 %14 ; [clobbered] temporary ZMM +%define %%B12_15 %15 ; [clobbered] temporary ZMM +%define %%GH1H_UNUSED %16 ; [clobbered] temporary ZMM +%define %%GH1L %17 ; [clobbered] temporary ZMM +%define %%GH1M %18 ; [clobbered] temporary ZMM +%define %%GH1T %19 ; [clobbered] temporary ZMM +%define %%GH2H %20 ; [clobbered] temporary ZMM +%define %%GH2L %21 ; [clobbered] temporary ZMM +%define %%GH2M %22 ; [clobbered] temporary ZMM +%define %%GH2T %23 ; [clobbered] temporary ZMM +%define %%GH3H %24 ; [clobbered] temporary ZMM +%define %%GH3L %25 ; [clobbered] temporary ZMM +%define %%GH3M %26 ; [clobbered] temporary ZMM +%define %%GH3T %27 ; [clobbered] temporary ZMM +%define %%AESKEY1 %28 ; [clobbered] temporary ZMM +%define %%AESKEY2 %29 ; [clobbered] temporary ZMM +%define %%GHKEY1 %30 ; [clobbered] temporary ZMM +%define %%GHKEY2 %31 ; [clobbered] temporary ZMM +%define %%GHDAT1 %32 ; [clobbered] temporary ZMM +%define %%GHDAT2 %33 ; [clobbered] temporary ZMM +%define %%ZT01 %34 ; [clobbered] temporary ZMM +%define %%ADDBE_4x4 %35 ; [in] ZMM with 4x128bits 4 in big-endian +%define %%ADDBE_1234 %36 ; [in] ZMM with 4x128bits 1, 2, 3 and 4 in big-endian +%define %%GHASH_TYPE %37 ; [in] "start", "start_reduce", "mid", "end_reduce" +%define %%TO_REDUCE_L %38 ; [in] ZMM for low 4x128-bit GHASH sum +%define %%TO_REDUCE_H %39 ; [in] ZMM for hi 4x128-bit GHASH sum +%define %%TO_REDUCE_M %40 ; [in] ZMM for medium 4x128-bit GHASH sum +%define %%ENC_DEC %41 ; [in] cipher direction +%define %%HASH_IN_OUT %42 ; [in/out] XMM ghash in/out value +%define %%IA0 %43 ; [clobbered] GP temporary +%define %%IA1 %44 ; [clobbered] GP temporary +%define %%MASKREG %45 ; [clobbered] mask register +%define %%NUM_BLOCKS %46 ; [in] numerical value with number of blocks to be encrypted/ghashed (1 to 16) +%define %%INSTANCE_TYPE %47 ; [in] multi_call or single_call + +%xdefine %%GH1H %%HASH_IN_OUT ; this is to avoid additional move in do_reduction case + +%define %%LAST_GHASH_BLK %%GH1L +%define %%LAST_CIPHER_BLK %%GH1T + +%define %%RED_POLY %%GH2T +%define %%RED_P1 %%GH2L +%define %%RED_T1 %%GH2H +%define %%RED_T2 %%GH2M + +%define %%DATA1 %%GH3H +%define %%DATA2 %%GH3L +%define %%DATA3 %%GH3M +%define %%DATA4 %%GH3T + +;; do reduction after the 16 blocks ? +%assign do_reduction 0 + +;; is 16 block chunk a start? +%assign is_start 0 + +%ifidn %%GHASH_TYPE, start_reduce +%assign is_start 1 +%assign do_reduction 1 +%endif + +%ifidn %%GHASH_TYPE, start +%assign is_start 1 +%endif + +%ifidn %%GHASH_TYPE, end_reduce +%assign do_reduction 1 +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; - get load/store mask + ;; - load plain/cipher text + ;; get load/store mask + lea %%IA0, [rel byte64_len_to_mask_table] + mov %%IA1, %%LENGTH +%if %%NUM_BLOCKS > 12 + sub %%IA1, 3 * 64 +%elif %%NUM_BLOCKS > 8 + sub %%IA1, 2 * 64 +%elif %%NUM_BLOCKS > 4 + sub %%IA1, 64 +%endif + kmovq %%MASKREG, [%%IA0 + %%IA1*8] + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; prepare counter blocks + + cmp DWORD(%%CTR_CHECK), (256 - %%NUM_BLOCKS) + jae %%_16_blocks_overflow + + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vpaddd, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%CTR_BE, %%B00_03, %%B04_07, %%B08_11, \ + %%ADDBE_1234, %%ADDBE_4x4, %%ADDBE_4x4, %%ADDBE_4x4 + jmp %%_16_blocks_ok + +%%_16_blocks_overflow: + vpshufb %%CTR_BE, %%CTR_BE, %%SHFMSK + vpaddd %%B00_03, %%CTR_BE, [rel ddq_add_1234] +%if %%NUM_BLOCKS > 4 + vmovdqa64 %%B12_15, [rel ddq_add_4444] + vpaddd %%B04_07, %%B00_03, %%B12_15 +%endif +%if %%NUM_BLOCKS > 8 + vpaddd %%B08_11, %%B04_07, %%B12_15 +%endif +%if %%NUM_BLOCKS > 12 + vpaddd %%B12_15, %%B08_11, %%B12_15 +%endif + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vpshufb, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%SHFMSK, %%SHFMSK, %%SHFMSK, %%SHFMSK +%%_16_blocks_ok: + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; - pre-load constants + ;; - add current hash into the 1st block + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 0)] +%if is_start != 0 + vpxorq %%GHDAT1, %%HASH_IN_OUT, [rsp + %%GHASHIN_BLK_OFFSET + (0*64)] +%else + vmovdqa64 %%GHDAT1, [rsp + %%GHASHIN_BLK_OFFSET + (0*64)] +%endif + vmovdqu64 %%GHKEY1, [%%GDATA + %%HASHKEY_OFFSET + (0*64)] + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; save counter for the next round + ;; increment counter overflow check register +%ifidn %%INSTANCE_TYPE, multi_call +%if %%NUM_BLOCKS <= 4 + vextracti32x4 XWORD(%%CTR_BE), %%B00_03, (%%NUM_BLOCKS - 1) +%elif %%NUM_BLOCKS <= 8 + vextracti32x4 XWORD(%%CTR_BE), %%B04_07, (%%NUM_BLOCKS - 5) +%elif %%NUM_BLOCKS <= 12 + vextracti32x4 XWORD(%%CTR_BE), %%B08_11, (%%NUM_BLOCKS - 9) +%else + vextracti32x4 XWORD(%%CTR_BE), %%B12_15, (%%NUM_BLOCKS - 13) +%endif + vshufi64x2 %%CTR_BE, %%CTR_BE, %%CTR_BE, 0000_0000b +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; pre-load constants + vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 1)] + vmovdqu64 %%GHKEY2, [%%GDATA + %%HASHKEY_OFFSET + (1*64)] + vmovdqa64 %%GHDAT2, [rsp + %%GHASHIN_BLK_OFFSET + (1*64)] + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; stitch AES rounds with GHASH + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES round 0 - ARK + + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vpxorq, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%AESKEY1, %%AESKEY1, %%AESKEY1, %%AESKEY1 + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 2)] + + ;;================================================== + ;; GHASH 4 blocks (15 to 12) + vpclmulqdq %%GH1H, %%GHDAT1, %%GHKEY1, 0x11 ; a1*b1 + vpclmulqdq %%GH1L, %%GHDAT1, %%GHKEY1, 0x00 ; a0*b0 + vpclmulqdq %%GH1M, %%GHDAT1, %%GHKEY1, 0x01 ; a1*b0 + vpclmulqdq %%GH1T, %%GHDAT1, %%GHKEY1, 0x10 ; a0*b1 + + vmovdqu64 %%GHKEY1, [%%GDATA + %%HASHKEY_OFFSET + (2*64)] + vmovdqa64 %%GHDAT1, [rsp + %%GHASHIN_BLK_OFFSET + (2*64)] + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES round 1 + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vaesenc, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%AESKEY2, %%AESKEY2, %%AESKEY2, %%AESKEY2 + vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 3)] + + ;; ================================================= + ;; GHASH 4 blocks (11 to 8) + vpclmulqdq %%GH2M, %%GHDAT2, %%GHKEY2, 0x10 ; a0*b1 + vpclmulqdq %%GH2T, %%GHDAT2, %%GHKEY2, 0x01 ; a1*b0 + vpclmulqdq %%GH2H, %%GHDAT2, %%GHKEY2, 0x11 ; a1*b1 + vpclmulqdq %%GH2L, %%GHDAT2, %%GHKEY2, 0x00 ; a0*b0 + + vmovdqu64 %%GHKEY2, [%%GDATA + %%HASHKEY_OFFSET + (3*64)] + vmovdqa64 %%GHDAT2, [rsp + %%GHASHIN_BLK_OFFSET + (3*64)] + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES round 2 + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vaesenc, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%AESKEY1, %%AESKEY1, %%AESKEY1, %%AESKEY1 + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 4)] + + ;; ================================================= + ;; GHASH 4 blocks (7 to 4) + vpclmulqdq %%GH3M, %%GHDAT1, %%GHKEY1, 0x10 ; a0*b1 + vpclmulqdq %%GH3T, %%GHDAT1, %%GHKEY1, 0x01 ; a1*b0 + vpclmulqdq %%GH3H, %%GHDAT1, %%GHKEY1, 0x11 ; a1*b1 + vpclmulqdq %%GH3L, %%GHDAT1, %%GHKEY1, 0x00 ; a0*b0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES rounds 3 + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vaesenc, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%AESKEY2, %%AESKEY2, %%AESKEY2, %%AESKEY2 + vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 5)] + + ;; ================================================= + ;; Gather (XOR) GHASH for 12 blocks + vpternlogq %%GH1H, %%GH2H, %%GH3H, 0x96 + vpternlogq %%GH1L, %%GH2L, %%GH3L, 0x96 + vpternlogq %%GH1T, %%GH2T, %%GH3T, 0x96 + vpternlogq %%GH1M, %%GH2M, %%GH3M, 0x96 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES rounds 4 + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vaesenc, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%AESKEY1, %%AESKEY1, %%AESKEY1, %%AESKEY1 + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 6)] + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; load plain/cipher text + ZMM_LOAD_MASKED_BLOCKS_0_16 %%NUM_BLOCKS, %%PLAIN_CIPH_IN, %%DATA_OFFSET, \ + %%DATA1, %%DATA2, %%DATA3, %%DATA4, %%MASKREG + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES rounds 5 + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vaesenc, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%AESKEY2, %%AESKEY2, %%AESKEY2, %%AESKEY2 + vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 7)] + + ;; ================================================= + ;; GHASH 4 blocks (3 to 0) + vpclmulqdq %%GH2M, %%GHDAT2, %%GHKEY2, 0x10 ; a0*b1 + vpclmulqdq %%GH2T, %%GHDAT2, %%GHKEY2, 0x01 ; a1*b0 + vpclmulqdq %%GH2H, %%GHDAT2, %%GHKEY2, 0x11 ; a1*b1 + vpclmulqdq %%GH2L, %%GHDAT2, %%GHKEY2, 0x00 ; a0*b0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES round 6 + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vaesenc, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%AESKEY1, %%AESKEY1, %%AESKEY1, %%AESKEY1 + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 8)] + + ;; ================================================= + ;; gather GHASH in GH1L (low), GH1H (high), GH1M (mid) + ;; - add GH2[MTLH] to GH1[MTLH] + vpternlogq %%GH1M, %%GH1T, %%GH2T, 0x96 +%if do_reduction != 0 + +%if is_start != 0 + vpxorq %%GH1M, %%GH1M, %%GH2M +%else + vpternlogq %%GH1H, %%TO_REDUCE_H, %%GH2H, 0x96 + vpternlogq %%GH1L, %%TO_REDUCE_L, %%GH2L, 0x96 + vpternlogq %%GH1M, %%TO_REDUCE_M, %%GH2M, 0x96 +%endif + +%else + ;; Update H/M/L hash sums if not carrying reduction +%if is_start != 0 + vpxorq %%TO_REDUCE_H, %%GH1H, %%GH2H + vpxorq %%TO_REDUCE_L, %%GH1L, %%GH2L + vpxorq %%TO_REDUCE_M, %%GH1M, %%GH2M +%else + vpternlogq %%TO_REDUCE_H, %%GH1H, %%GH2H, 0x96 + vpternlogq %%TO_REDUCE_L, %%GH1L, %%GH2L, 0x96 + vpternlogq %%TO_REDUCE_M, %%GH1M, %%GH2M, 0x96 +%endif + +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES round 7 + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vaesenc, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%AESKEY2, %%AESKEY2, %%AESKEY2, %%AESKEY2 + vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 9)] + + ;; ================================================= + ;; prepare mid sum for adding to high & low + ;; load polynomial constant for reduction +%if do_reduction != 0 + vpsrldq %%GH2M, %%GH1M, 8 + vpslldq %%GH1M, %%GH1M, 8 + + vmovdqa64 XWORD(%%RED_POLY), [rel POLY2] +%endif + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES round 8 + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vaesenc, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%AESKEY1, %%AESKEY1, %%AESKEY1, %%AESKEY1 + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 10)] + + ;; ================================================= + ;; Add mid product to high and low +%if do_reduction != 0 +%if is_start != 0 + vpternlogq %%GH1H, %%GH2H, %%GH2M, 0x96 ; TH = TH1 + TH2 + TM>>64 + vpternlogq %%GH1L, %%GH2L, %%GH1M, 0x96 ; TL = TL1 + TL2 + TM<<64 +%else + vpxorq %%GH1H, %%GH1H, %%GH2M ; TH = TH1 + TM>>64 + vpxorq %%GH1L, %%GH1L, %%GH1M ; TL = TL1 + TM<<64 +%endif +%endif + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES round 9 + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vaesenc, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%AESKEY2, %%AESKEY2, %%AESKEY2, %%AESKEY2 + + ;; ================================================= + ;; horizontal xor of low and high 4x128 +%if do_reduction != 0 + VHPXORI4x128 %%GH1H, %%GH2H + VHPXORI4x128 %%GH1L, %%GH2L +%endif + +%if (NROUNDS >= 11) + vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 11)] +%endif + ;; ================================================= + ;; first phase of reduction +%if do_reduction != 0 + vpclmulqdq XWORD(%%RED_P1), XWORD(%%RED_POLY), XWORD(%%GH1L), 0x01 + vpslldq XWORD(%%RED_P1), XWORD(%%RED_P1), 8 ; shift-L 2 DWs + vpxorq XWORD(%%RED_P1), XWORD(%%GH1L), XWORD(%%RED_P1) ; first phase of the reduct +%endif + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES rounds up to 11 (AES192) or 13 (AES256) + ;; AES128 is done +%if (NROUNDS >= 11) + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vaesenc, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%AESKEY1, %%AESKEY1, %%AESKEY1, %%AESKEY1 + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 12)] + + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vaesenc, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%AESKEY2, %%AESKEY2, %%AESKEY2, %%AESKEY2 +%if (NROUNDS == 13) + vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 13)] + + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vaesenc, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%AESKEY1, %%AESKEY1, %%AESKEY1, %%AESKEY1 + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 14)] + + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vaesenc, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%AESKEY2, %%AESKEY2, %%AESKEY2, %%AESKEY2 +%endif ; GCM256 / NROUNDS = 13 (15 including the first and the last) +%endif ; GCM192 / NROUNDS = 11 (13 including the first and the last) + + ;; ================================================= + ;; second phase of the reduction +%if do_reduction != 0 + vpclmulqdq XWORD(%%RED_T1), XWORD(%%RED_POLY), XWORD(%%RED_P1), 0x00 + vpsrldq XWORD(%%RED_T1), XWORD(%%RED_T1), 4 ; shift-R 1-DW to obtain 2-DWs shift-R + + vpclmulqdq XWORD(%%RED_T2), XWORD(%%RED_POLY), XWORD(%%RED_P1), 0x10 + vpslldq XWORD(%%RED_T2), XWORD(%%RED_T2), 4 ; shift-L 1-DW for result without shifts + ;; GH1H = GH1H + RED_T1 + RED_T2 + vpternlogq XWORD(%%GH1H), XWORD(%%RED_T2), XWORD(%%RED_T1), 0x96 +%endif + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; the last AES round + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vaesenclast, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%AESKEY1, %%AESKEY1, %%AESKEY1, %%AESKEY1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; XOR against plain/cipher text + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vpxorq, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%DATA1, %%DATA2, %%DATA3, %%DATA4 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; retrieve the last cipher counter block (partially XOR'ed with text) + ;; - this is needed for partial block cases +%ifidn %%INSTANCE_TYPE, multi_call +%if %%NUM_BLOCKS <= 4 + vextracti32x4 XWORD(%%LAST_CIPHER_BLK), %%B00_03, (%%NUM_BLOCKS - 1) +%elif %%NUM_BLOCKS <= 8 + vextracti32x4 XWORD(%%LAST_CIPHER_BLK), %%B04_07, (%%NUM_BLOCKS - 5) +%elif %%NUM_BLOCKS <= 12 + vextracti32x4 XWORD(%%LAST_CIPHER_BLK), %%B08_11, (%%NUM_BLOCKS - 9) +%else + vextracti32x4 XWORD(%%LAST_CIPHER_BLK), %%B12_15, (%%NUM_BLOCKS - 13) +%endif +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; store cipher/plain text + ZMM_STORE_MASKED_BLOCKS_0_16 %%NUM_BLOCKS, %%CIPH_PLAIN_OUT, %%DATA_OFFSET, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, %%MASKREG + + ;; ================================================= + ;; shuffle cipher text blocks for GHASH computation +%ifidn %%ENC_DEC, ENC + ;; zero bytes outside the mask before hashing +%if %%NUM_BLOCKS <= 4 + vmovdqu8 %%B00_03{%%MASKREG}{z}, %%B00_03 +%elif %%NUM_BLOCKS <= 8 + vmovdqu8 %%B04_07{%%MASKREG}{z}, %%B04_07 +%elif %%NUM_BLOCKS <= 12 + vmovdqu8 %%B08_11{%%MASKREG}{z}, %%B08_11 +%else + vmovdqu8 %%B12_15{%%MASKREG}{z}, %%B12_15 +%endif + + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vpshufb, \ + %%DATA1, %%DATA2, %%DATA3, %%DATA4, \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, \ + %%SHFMSK, %%SHFMSK, %%SHFMSK, %%SHFMSK +%else + ;; zero bytes outside the mask before hashing +%if %%NUM_BLOCKS <= 4 + vmovdqu8 %%DATA1{%%MASKREG}{z}, %%DATA1 +%elif %%NUM_BLOCKS <= 8 + vmovdqu8 %%DATA2{%%MASKREG}{z}, %%DATA2 +%elif %%NUM_BLOCKS <= 12 + vmovdqu8 %%DATA3{%%MASKREG}{z}, %%DATA3 +%else + vmovdqu8 %%DATA4{%%MASKREG}{z}, %%DATA4 +%endif + + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUM_BLOCKS, vpshufb, \ + %%DATA1, %%DATA2, %%DATA3, %%DATA4, \ + %%DATA1, %%DATA2, %%DATA3, %%DATA4, \ + %%SHFMSK, %%SHFMSK, %%SHFMSK, %%SHFMSK +%endif + + ;; ================================================= + ;; Extract the last block for partial / multi_call cases +%ifidn %%INSTANCE_TYPE, multi_call +%if %%NUM_BLOCKS <= 4 + vextracti32x4 XWORD(%%LAST_GHASH_BLK), %%DATA1, %%NUM_BLOCKS - 1 +%elif %%NUM_BLOCKS <= 8 + vextracti32x4 XWORD(%%LAST_GHASH_BLK), %%DATA2, %%NUM_BLOCKS - 5 +%elif %%NUM_BLOCKS <= 12 + vextracti32x4 XWORD(%%LAST_GHASH_BLK), %%DATA3, %%NUM_BLOCKS - 9 +%else + vextracti32x4 XWORD(%%LAST_GHASH_BLK), %%DATA4, %%NUM_BLOCKS - 13 +%endif +%endif + +%if do_reduction != 0 + ;; GH1H holds reduced hash value + ;; - normally do "vmovdqa64 XWORD(%%HASH_IN_OUT), XWORD(%%GH1H)" + ;; - register rename trick obsoletes the above move +%endif + + ;; ================================================= + ;; GHASH last N blocks + ;; - current hash value in HASH_IN_OUT or + ;; product parts in TO_REDUCE_H/M/L + ;; - DATA1-DATA4 include blocks for GHASH + +%if do_reduction == 0 + INITIAL_BLOCKS_PARTIAL_GHASH \ + %%GDATA, %%GCTX, %%LENGTH, \ + %%NUM_BLOCKS, XWORD(%%HASH_IN_OUT), %%ENC_DEC, \ + %%INSTANCE_TYPE, %%DATA1, %%DATA2, %%DATA3, %%DATA4, \ + XWORD(%%LAST_CIPHER_BLK), XWORD(%%LAST_GHASH_BLK), \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, %%GHDAT1, %%GHDAT2, \ + %%AESKEY1, %%AESKEY2, %%GHKEY1, \ + %%TO_REDUCE_H, %%TO_REDUCE_M, %%TO_REDUCE_L +%else + INITIAL_BLOCKS_PARTIAL_GHASH \ + %%GDATA, %%GCTX, %%LENGTH, \ + %%NUM_BLOCKS, XWORD(%%HASH_IN_OUT), %%ENC_DEC, \ + %%INSTANCE_TYPE, %%DATA1, %%DATA2, %%DATA3, %%DATA4, \ + XWORD(%%LAST_CIPHER_BLK), XWORD(%%LAST_GHASH_BLK), \ + %%B00_03, %%B04_07, %%B08_11, %%B12_15, %%GHDAT1, %%GHDAT2, \ + %%AESKEY1, %%AESKEY2, %%GHKEY1 +%endif +%endmacro + +;;; =========================================================================== +;;; =========================================================================== +;;; Stitched GHASH of 16 blocks (with reduction) with encryption of N blocks +;;; followed with GHASH of the N blocks. +%macro GCM_ENC_DEC_LAST 46 +%define %%GDATA %1 ; [in] key pointer +%define %%GCTX %2 ; [in] context pointer +%define %%CIPH_PLAIN_OUT %3 ; [in] pointer to output buffer +%define %%PLAIN_CIPH_IN %4 ; [in] pointer to input buffer +%define %%DATA_OFFSET %5 ; [in] data offset +%define %%LENGTH %6 ; [in/clobbered] data length +%define %%CTR_BE %7 ; [in/out] ZMM counter blocks (last 4) in big-endian +%define %%CTR_CHECK %8 ; [in/out] GP with 8-bit counter for overflow check +%define %%HASHKEY_OFFSET %9 ; [in] numerical offset for the highest hash key +%define %%GHASHIN_BLK_OFFSET %10 ; [in] numerical offset for GHASH blocks in +%define %%SHFMSK %11 ; [in] ZMM with byte swap mask for pshufb +%define %%ZT00 %12 ; [clobbered] temporary ZMM +%define %%ZT01 %13 ; [clobbered] temporary ZMM +%define %%ZT02 %14 ; [clobbered] temporary ZMM +%define %%ZT03 %15 ; [clobbered] temporary ZMM +%define %%ZT04 %16 ; [clobbered] temporary ZMM +%define %%ZT05 %17 ; [clobbered] temporary ZMM +%define %%ZT06 %18 ; [clobbered] temporary ZMM +%define %%ZT07 %19 ; [clobbered] temporary ZMM +%define %%ZT08 %20 ; [clobbered] temporary ZMM +%define %%ZT09 %21 ; [clobbered] temporary ZMM +%define %%ZT10 %22 ; [clobbered] temporary ZMM +%define %%ZT11 %23 ; [clobbered] temporary ZMM +%define %%ZT12 %24 ; [clobbered] temporary ZMM +%define %%ZT13 %25 ; [clobbered] temporary ZMM +%define %%ZT14 %26 ; [clobbered] temporary ZMM +%define %%ZT15 %27 ; [clobbered] temporary ZMM +%define %%ZT16 %28 ; [clobbered] temporary ZMM +%define %%ZT17 %29 ; [clobbered] temporary ZMM +%define %%ZT18 %30 ; [clobbered] temporary ZMM +%define %%ZT19 %31 ; [clobbered] temporary ZMM +%define %%ZT20 %32 ; [clobbered] temporary ZMM +%define %%ZT21 %33 ; [clobbered] temporary ZMM +%define %%ZT22 %34 ; [clobbered] temporary ZMM +%define %%ADDBE_4x4 %35 ; [in] ZMM with 4x128bits 4 in big-endian +%define %%ADDBE_1234 %36 ; [in] ZMM with 4x128bits 1, 2, 3 and 4 in big-endian +%define %%GHASH_TYPE %37 ; [in] "start", "start_reduce", "mid", "end_reduce" +%define %%TO_REDUCE_L %38 ; [in] ZMM for low 4x128-bit GHASH sum +%define %%TO_REDUCE_H %39 ; [in] ZMM for hi 4x128-bit GHASH sum +%define %%TO_REDUCE_M %40 ; [in] ZMM for medium 4x128-bit GHASH sum +%define %%ENC_DEC %41 ; [in] cipher direction +%define %%HASH_IN_OUT %42 ; [in/out] XMM ghash in/out value +%define %%IA0 %43 ; [clobbered] GP temporary +%define %%IA1 %44 ; [clobbered] GP temporary +%define %%MASKREG %45 ; [clobbered] mask register +%define %%INSTANCE_TYPE %46 ; [in] multi_call or single_call + + mov DWORD(%%IA0), DWORD(%%LENGTH) + add DWORD(%%IA0), 15 + shr DWORD(%%IA0), 4 + je %%_last_num_blocks_is_0 + + cmp DWORD(%%IA0), 8 + je %%_last_num_blocks_is_8 + jb %%_last_num_blocks_is_7_1 + + + cmp DWORD(%%IA0), 12 + je %%_last_num_blocks_is_12 + jb %%_last_num_blocks_is_11_9 + + ;; 16, 15, 14 or 13 + cmp DWORD(%%IA0), 15 + je %%_last_num_blocks_is_15 + ja %%_last_num_blocks_is_16 + cmp DWORD(%%IA0), 14 + je %%_last_num_blocks_is_14 + jmp %%_last_num_blocks_is_13 + +%%_last_num_blocks_is_11_9: + ;; 11, 10 or 9 + cmp DWORD(%%IA0), 10 + je %%_last_num_blocks_is_10 + ja %%_last_num_blocks_is_11 + jmp %%_last_num_blocks_is_9 + +%%_last_num_blocks_is_7_1: + cmp DWORD(%%IA0), 4 + je %%_last_num_blocks_is_4 + jb %%_last_num_blocks_is_3_1 + ;; 7, 6 or 5 + cmp DWORD(%%IA0), 6 + ja %%_last_num_blocks_is_7 + je %%_last_num_blocks_is_6 + jmp %%_last_num_blocks_is_5 + +%%_last_num_blocks_is_3_1: + ;; 3, 2 or 1 + cmp DWORD(%%IA0), 2 + ja %%_last_num_blocks_is_3 + je %%_last_num_blocks_is_2 + ;; fall through for `jmp %%_last_num_blocks_is_1` + + ;; Use rep to generate different block size variants + ;; - one block size has to be the first one +%assign num_blocks 1 +%rep 16 +%%_last_num_blocks_is_ %+ num_blocks : + GHASH_16_ENCRYPT_N_GHASH_N \ + %%GDATA, %%GCTX, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, \ + %%DATA_OFFSET, %%LENGTH, %%CTR_BE, %%CTR_CHECK, \ + %%HASHKEY_OFFSET, %%GHASHIN_BLK_OFFSET, %%SHFMSK, \ + %%ZT00, %%ZT01, %%ZT02, %%ZT03, %%ZT04, %%ZT05, %%ZT06, %%ZT07, \ + %%ZT08, %%ZT09, %%ZT10, %%ZT11, %%ZT12, %%ZT13, %%ZT14, %%ZT15, \ + %%ZT16, %%ZT17, %%ZT18, %%ZT19, %%ZT20, %%ZT21, %%ZT22, \ + %%ADDBE_4x4, %%ADDBE_1234, %%GHASH_TYPE, \ + %%TO_REDUCE_L, %%TO_REDUCE_H, %%TO_REDUCE_M, \ + %%ENC_DEC, %%HASH_IN_OUT, %%IA0, %%IA1, %%MASKREG, \ + num_blocks, %%INSTANCE_TYPE + + jmp %%_last_blocks_done +%assign num_blocks (num_blocks + 1) +%endrep + +%%_last_num_blocks_is_0: +;; if there is 0 blocks to cipher then there are only 16 blocks for ghash and reduction +;; - convert mid into end_reduce +;; - convert start into start_reduce +%ifidn %%GHASH_TYPE, mid +%xdefine %%GHASH_TYPE end_reduce +%endif +%ifidn %%GHASH_TYPE, start +%xdefine %%GHASH_TYPE start_reduce +%endif + + GHASH_16 %%GHASH_TYPE, %%TO_REDUCE_H, %%TO_REDUCE_M, %%TO_REDUCE_L, \ + rsp, %%GHASHIN_BLK_OFFSET, 0, %%GDATA, %%HASHKEY_OFFSET, 0, %%HASH_IN_OUT, \ + %%ZT00, %%ZT01, %%ZT02, %%ZT03, %%ZT04, %%ZT05, %%ZT06, %%ZT07, %%ZT08, %%ZT09 + +%%_last_blocks_done: + +%endmacro + + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Main GCM macro stitching cipher with GHASH +;;; - operates on single stream +;;; - encrypts 16 blocks at a time +;;; - ghash the 16 previously encrypted ciphertext blocks +;;; - no partial block or multi_call handling here +%macro GHASH_16_ENCRYPT_16_PARALLEL 42 +%define %%GDATA %1 ; [in] key pointer +%define %%CIPH_PLAIN_OUT %2 ; [in] pointer to output buffer +%define %%PLAIN_CIPH_IN %3 ; [in] pointer to input buffer +%define %%DATA_OFFSET %4 ; [in] data offset +%define %%CTR_BE %5 ; [in/out] ZMM counter blocks (last 4) in big-endian +%define %%CTR_CHECK %6 ; [in/out] GP with 8-bit counter for overflow check +%define %%HASHKEY_OFFSET %7 ; [in] numerical offset for the highest hash key +%define %%AESOUT_BLK_OFFSET %8 ; [in] numerical offset for AES-CTR out +%define %%GHASHIN_BLK_OFFSET %9 ; [in] numerical offset for GHASH blocks in +%define %%SHFMSK %10 ; [in] ZMM with byte swap mask for pshufb +%define %%ZT1 %11 ; [clobbered] temporary ZMM (cipher) +%define %%ZT2 %12 ; [clobbered] temporary ZMM (cipher) +%define %%ZT3 %13 ; [clobbered] temporary ZMM (cipher) +%define %%ZT4 %14 ; [clobbered] temporary ZMM (cipher) +%define %%ZT5 %15 ; [clobbered/out] temporary ZMM or GHASH OUT (final_reduction) +%define %%ZT6 %16 ; [clobbered] temporary ZMM (cipher) +%define %%ZT7 %17 ; [clobbered] temporary ZMM (cipher) +%define %%ZT8 %18 ; [clobbered] temporary ZMM (cipher) +%define %%ZT9 %19 ; [clobbered] temporary ZMM (cipher) +%define %%ZT10 %20 ; [clobbered] temporary ZMM (ghash) +%define %%ZT11 %21 ; [clobbered] temporary ZMM (ghash) +%define %%ZT12 %22 ; [clobbered] temporary ZMM (ghash) +%define %%ZT13 %23 ; [clobbered] temporary ZMM (ghash) +%define %%ZT14 %24 ; [clobbered] temporary ZMM (ghash) +%define %%ZT15 %25 ; [clobbered] temporary ZMM (ghash) +%define %%ZT16 %26 ; [clobbered] temporary ZMM (ghash) +%define %%ZT17 %27 ; [clobbered] temporary ZMM (ghash) +%define %%ZT18 %28 ; [clobbered] temporary ZMM (ghash) +%define %%ZT19 %29 ; [clobbered] temporary ZMM +%define %%ZT20 %30 ; [clobbered] temporary ZMM +%define %%ZT21 %31 ; [clobbered] temporary ZMM +%define %%ZT22 %32 ; [clobbered] temporary ZMM +%define %%ZT23 %33 ; [clobbered] temporary ZMM +%define %%ADDBE_4x4 %34 ; [in] ZMM with 4x128bits 4 in big-endian +%define %%ADDBE_1234 %35 ; [in] ZMM with 4x128bits 1, 2, 3 and 4 in big-endian +%define %%TO_REDUCE_L %36 ; [in/out] ZMM for low 4x128-bit GHASH sum +%define %%TO_REDUCE_H %37 ; [in/out] ZMM for hi 4x128-bit GHASH sum +%define %%TO_REDUCE_M %38 ; [in/out] ZMM for medium 4x128-bit GHASH sum +%define %%DO_REDUCTION %39 ; [in] "no_reduction", "final_reduction", "first_time" +%define %%ENC_DEC %40 ; [in] cipher direction +%define %%DATA_DISPL %41 ; [in] fixed numerical data displacement/offset +%define %%GHASH_IN %42 ; [in] current GHASH value or "no_ghash_in" + +%define %%B00_03 %%ZT1 +%define %%B04_07 %%ZT2 +%define %%B08_11 %%ZT3 +%define %%B12_15 %%ZT4 + +%define %%GH1H %%ZT5 ; @note: do not change this mapping +%define %%GH1L %%ZT6 +%define %%GH1M %%ZT7 +%define %%GH1T %%ZT8 + +%define %%GH2H %%ZT9 +%define %%GH2L %%ZT10 +%define %%GH2M %%ZT11 +%define %%GH2T %%ZT12 + +%define %%RED_POLY %%GH2T +%define %%RED_P1 %%GH2L +%define %%RED_T1 %%GH2H +%define %%RED_T2 %%GH2M + +%define %%GH3H %%ZT13 +%define %%GH3L %%ZT14 +%define %%GH3M %%ZT15 +%define %%GH3T %%ZT16 + +%define %%DATA1 %%ZT13 +%define %%DATA2 %%ZT14 +%define %%DATA3 %%ZT15 +%define %%DATA4 %%ZT16 + +%define %%AESKEY1 %%ZT17 +%define %%AESKEY2 %%ZT18 + +%define %%GHKEY1 %%ZT19 +%define %%GHKEY2 %%ZT20 +%define %%GHDAT1 %%ZT21 +%define %%GHDAT2 %%ZT22 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; prepare counter blocks + + cmp BYTE(%%CTR_CHECK), (256 - 16) + jae %%_16_blocks_overflow + vpaddd %%B00_03, %%CTR_BE, %%ADDBE_1234 + vpaddd %%B04_07, %%B00_03, %%ADDBE_4x4 + vpaddd %%B08_11, %%B04_07, %%ADDBE_4x4 + vpaddd %%B12_15, %%B08_11, %%ADDBE_4x4 + jmp %%_16_blocks_ok +%%_16_blocks_overflow: + vpshufb %%CTR_BE, %%CTR_BE, %%SHFMSK + vmovdqa64 %%B12_15, [rel ddq_add_4444] + vpaddd %%B00_03, %%CTR_BE, [rel ddq_add_1234] + vpaddd %%B04_07, %%B00_03, %%B12_15 + vpaddd %%B08_11, %%B04_07, %%B12_15 + vpaddd %%B12_15, %%B08_11, %%B12_15 + vpshufb %%B00_03, %%SHFMSK + vpshufb %%B04_07, %%SHFMSK + vpshufb %%B08_11, %%SHFMSK + vpshufb %%B12_15, %%SHFMSK +%%_16_blocks_ok: + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; pre-load constants + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 0)] +%ifnidn %%GHASH_IN, no_ghash_in + vpxorq %%GHDAT1, %%GHASH_IN, [rsp + %%GHASHIN_BLK_OFFSET + (0*64)] +%else + vmovdqa64 %%GHDAT1, [rsp + %%GHASHIN_BLK_OFFSET + (0*64)] +%endif + vmovdqu64 %%GHKEY1, [%%GDATA + %%HASHKEY_OFFSET + (0*64)] + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; save counter for the next round + ;; increment counter overflow check register + vshufi64x2 %%CTR_BE, %%B12_15, %%B12_15, 1111_1111b + add BYTE(%%CTR_CHECK), 16 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; pre-load constants + vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 1)] + vmovdqu64 %%GHKEY2, [%%GDATA + %%HASHKEY_OFFSET + (1*64)] + vmovdqa64 %%GHDAT2, [rsp + %%GHASHIN_BLK_OFFSET + (1*64)] + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; stitch AES rounds with GHASH + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES round 0 - ARK + + vpxorq %%B00_03, %%AESKEY1 + vpxorq %%B04_07, %%AESKEY1 + vpxorq %%B08_11, %%AESKEY1 + vpxorq %%B12_15, %%AESKEY1 + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 2)] + + ;;================================================== + ;; GHASH 4 blocks (15 to 12) + vpclmulqdq %%GH1H, %%GHDAT1, %%GHKEY1, 0x11 ; a1*b1 + vpclmulqdq %%GH1L, %%GHDAT1, %%GHKEY1, 0x00 ; a0*b0 + vpclmulqdq %%GH1M, %%GHDAT1, %%GHKEY1, 0x01 ; a1*b0 + vpclmulqdq %%GH1T, %%GHDAT1, %%GHKEY1, 0x10 ; a0*b1 + + vmovdqu64 %%GHKEY1, [%%GDATA + %%HASHKEY_OFFSET + (2*64)] + vmovdqa64 %%GHDAT1, [rsp + %%GHASHIN_BLK_OFFSET + (2*64)] + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES round 1 + vaesenc %%B00_03, %%B00_03, %%AESKEY2 + vaesenc %%B04_07, %%B04_07, %%AESKEY2 + vaesenc %%B08_11, %%B08_11, %%AESKEY2 + vaesenc %%B12_15, %%B12_15, %%AESKEY2 + vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 3)] + + ;; ================================================= + ;; GHASH 4 blocks (11 to 8) + vpclmulqdq %%GH2M, %%GHDAT2, %%GHKEY2, 0x10 ; a0*b1 + vpclmulqdq %%GH2T, %%GHDAT2, %%GHKEY2, 0x01 ; a1*b0 + vpclmulqdq %%GH2H, %%GHDAT2, %%GHKEY2, 0x11 ; a1*b1 + vpclmulqdq %%GH2L, %%GHDAT2, %%GHKEY2, 0x00 ; a0*b0 + + vmovdqu64 %%GHKEY2, [%%GDATA + %%HASHKEY_OFFSET + (3*64)] + vmovdqa64 %%GHDAT2, [rsp + %%GHASHIN_BLK_OFFSET + (3*64)] + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES round 2 + vaesenc %%B00_03, %%B00_03, %%AESKEY1 + vaesenc %%B04_07, %%B04_07, %%AESKEY1 + vaesenc %%B08_11, %%B08_11, %%AESKEY1 + vaesenc %%B12_15, %%B12_15, %%AESKEY1 + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 4)] + + ;; ================================================= + ;; GHASH 4 blocks (7 to 4) + vpclmulqdq %%GH3M, %%GHDAT1, %%GHKEY1, 0x10 ; a0*b1 + vpclmulqdq %%GH3T, %%GHDAT1, %%GHKEY1, 0x01 ; a1*b0 + vpclmulqdq %%GH3H, %%GHDAT1, %%GHKEY1, 0x11 ; a1*b1 + vpclmulqdq %%GH3L, %%GHDAT1, %%GHKEY1, 0x00 ; a0*b0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES rounds 3 + vaesenc %%B00_03, %%B00_03, %%AESKEY2 + vaesenc %%B04_07, %%B04_07, %%AESKEY2 + vaesenc %%B08_11, %%B08_11, %%AESKEY2 + vaesenc %%B12_15, %%B12_15, %%AESKEY2 + vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 5)] + + ;; ================================================= + ;; Gather (XOR) GHASH for 12 blocks + vpternlogq %%GH1H, %%GH2H, %%GH3H, 0x96 + vpternlogq %%GH1L, %%GH2L, %%GH3L, 0x96 + vpternlogq %%GH1T, %%GH2T, %%GH3T, 0x96 + vpternlogq %%GH1M, %%GH2M, %%GH3M, 0x96 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES rounds 4 + vaesenc %%B00_03, %%B00_03, %%AESKEY1 + vaesenc %%B04_07, %%B04_07, %%AESKEY1 + vaesenc %%B08_11, %%B08_11, %%AESKEY1 + vaesenc %%B12_15, %%B12_15, %%AESKEY1 + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 6)] + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; load plain/cipher text (recycle GH3xx registers) + VX512LDR %%DATA1, [%%PLAIN_CIPH_IN + %%DATA_OFFSET + %%DATA_DISPL + (0 * 64)] + VX512LDR %%DATA2, [%%PLAIN_CIPH_IN + %%DATA_OFFSET + %%DATA_DISPL + (1 * 64)] + VX512LDR %%DATA3, [%%PLAIN_CIPH_IN + %%DATA_OFFSET + %%DATA_DISPL + (2 * 64)] + VX512LDR %%DATA4, [%%PLAIN_CIPH_IN + %%DATA_OFFSET + %%DATA_DISPL + (3 * 64)] + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES rounds 5 + vaesenc %%B00_03, %%B00_03, %%AESKEY2 + vaesenc %%B04_07, %%B04_07, %%AESKEY2 + vaesenc %%B08_11, %%B08_11, %%AESKEY2 + vaesenc %%B12_15, %%B12_15, %%AESKEY2 + vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 7)] + + ;; ================================================= + ;; GHASH 4 blocks (3 to 0) + vpclmulqdq %%GH2M, %%GHDAT2, %%GHKEY2, 0x10 ; a0*b1 + vpclmulqdq %%GH2T, %%GHDAT2, %%GHKEY2, 0x01 ; a1*b0 + vpclmulqdq %%GH2H, %%GHDAT2, %%GHKEY2, 0x11 ; a1*b1 + vpclmulqdq %%GH2L, %%GHDAT2, %%GHKEY2, 0x00 ; a0*b0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES round 6 + vaesenc %%B00_03, %%B00_03, %%AESKEY1 + vaesenc %%B04_07, %%B04_07, %%AESKEY1 + vaesenc %%B08_11, %%B08_11, %%AESKEY1 + vaesenc %%B12_15, %%B12_15, %%AESKEY1 + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 8)] + + ;; ================================================= + ;; gather GHASH in GH1L (low) and GH1H (high) +%ifidn %%DO_REDUCTION, first_time + vpternlogq %%GH1M, %%GH1T, %%GH2T, 0x96 ; TM + vpxorq %%TO_REDUCE_M, %%GH1M, %%GH2M ; TM + vpxorq %%TO_REDUCE_H, %%GH1H, %%GH2H ; TH + vpxorq %%TO_REDUCE_L, %%GH1L, %%GH2L ; TL +%endif +%ifidn %%DO_REDUCTION, no_reduction + vpternlogq %%GH1M, %%GH1T, %%GH2T, 0x96 ; TM + vpternlogq %%TO_REDUCE_M, %%GH1M, %%GH2M, 0x96 ; TM + vpternlogq %%TO_REDUCE_H, %%GH1H, %%GH2H, 0x96 ; TH + vpternlogq %%TO_REDUCE_L, %%GH1L, %%GH2L, 0x96 ; TL +%endif +%ifidn %%DO_REDUCTION, final_reduction + ;; phase 1: add mid products together + ;; also load polynomial constant for reduction + vpternlogq %%GH1M, %%GH1T, %%GH2T, 0x96 ; TM + vpternlogq %%GH1M, %%TO_REDUCE_M, %%GH2M, 0x96 + + vpsrldq %%GH2M, %%GH1M, 8 + vpslldq %%GH1M, %%GH1M, 8 + + vmovdqa64 XWORD(%%RED_POLY), [rel POLY2] +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES round 7 + vaesenc %%B00_03, %%B00_03, %%AESKEY2 + vaesenc %%B04_07, %%B04_07, %%AESKEY2 + vaesenc %%B08_11, %%B08_11, %%AESKEY2 + vaesenc %%B12_15, %%B12_15, %%AESKEY2 + vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 9)] + + ;; ================================================= + ;; Add mid product to high and low +%ifidn %%DO_REDUCTION, final_reduction + vpternlogq %%GH1H, %%GH2H, %%GH2M, 0x96 ; TH = TH1 + TH2 + TM>>64 + vpxorq %%GH1H, %%TO_REDUCE_H + vpternlogq %%GH1L, %%GH2L, %%GH1M, 0x96 ; TL = TL1 + TL2 + TM<<64 + vpxorq %%GH1L, %%TO_REDUCE_L +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES round 8 + vaesenc %%B00_03, %%B00_03, %%AESKEY1 + vaesenc %%B04_07, %%B04_07, %%AESKEY1 + vaesenc %%B08_11, %%B08_11, %%AESKEY1 + vaesenc %%B12_15, %%B12_15, %%AESKEY1 + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 10)] + + ;; ================================================= + ;; horizontal xor of low and high 4x128 +%ifidn %%DO_REDUCTION, final_reduction + VHPXORI4x128 %%GH1H, %%GH2H + VHPXORI4x128 %%GH1L, %%GH2L +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES round 9 + vaesenc %%B00_03, %%B00_03, %%AESKEY2 + vaesenc %%B04_07, %%B04_07, %%AESKEY2 + vaesenc %%B08_11, %%B08_11, %%AESKEY2 + vaesenc %%B12_15, %%B12_15, %%AESKEY2 +%if (NROUNDS >= 11) + vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 11)] +%endif + ;; ================================================= + ;; first phase of reduction +%ifidn %%DO_REDUCTION, final_reduction + vpclmulqdq XWORD(%%RED_P1), XWORD(%%RED_POLY), XWORD(%%GH1L), 0x01 + vpslldq XWORD(%%RED_P1), XWORD(%%RED_P1), 8 ; shift-L 2 DWs + vpxorq XWORD(%%RED_P1), XWORD(%%GH1L), XWORD(%%RED_P1) ; first phase of the reduct +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES rounds up to 11 (AES192) or 13 (AES256) + ;; AES128 is done +%if (NROUNDS >= 11) + vaesenc %%B00_03, %%B00_03, %%AESKEY1 + vaesenc %%B04_07, %%B04_07, %%AESKEY1 + vaesenc %%B08_11, %%B08_11, %%AESKEY1 + vaesenc %%B12_15, %%B12_15, %%AESKEY1 + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 12)] + + vaesenc %%B00_03, %%B00_03, %%AESKEY2 + vaesenc %%B04_07, %%B04_07, %%AESKEY2 + vaesenc %%B08_11, %%B08_11, %%AESKEY2 + vaesenc %%B12_15, %%B12_15, %%AESKEY2 +%if (NROUNDS == 13) + vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 13)] + + vaesenc %%B00_03, %%B00_03, %%AESKEY1 + vaesenc %%B04_07, %%B04_07, %%AESKEY1 + vaesenc %%B08_11, %%B08_11, %%AESKEY1 + vaesenc %%B12_15, %%B12_15, %%AESKEY1 + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 14)] + + vaesenc %%B00_03, %%B00_03, %%AESKEY2 + vaesenc %%B04_07, %%B04_07, %%AESKEY2 + vaesenc %%B08_11, %%B08_11, %%AESKEY2 + vaesenc %%B12_15, %%B12_15, %%AESKEY2 +%endif ; GCM256 / NROUNDS = 13 (15 including the first and the last) +%endif ; GCM192 / NROUNDS = 11 (13 including the first and the last) + + ;; ================================================= + ;; second phase of the reduction +%ifidn %%DO_REDUCTION, final_reduction + vpclmulqdq XWORD(%%RED_T1), XWORD(%%RED_POLY), XWORD(%%RED_P1), 0x00 + vpsrldq XWORD(%%RED_T1), XWORD(%%RED_T1), 4 ; shift-R 1-DW to obtain 2-DWs shift-R + + vpclmulqdq XWORD(%%RED_T2), XWORD(%%RED_POLY), XWORD(%%RED_P1), 0x10 + vpslldq XWORD(%%RED_T2), XWORD(%%RED_T2), 4 ; shift-L 1-DW for result without shifts + ;; GH1H = GH1H x RED_T1 x RED_T2 + vpternlogq XWORD(%%GH1H), XWORD(%%RED_T2), XWORD(%%RED_T1), 0x96 +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; the last AES round + vaesenclast %%B00_03, %%B00_03, %%AESKEY1 + vaesenclast %%B04_07, %%B04_07, %%AESKEY1 + vaesenclast %%B08_11, %%B08_11, %%AESKEY1 + vaesenclast %%B12_15, %%B12_15, %%AESKEY1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; XOR against plain/cipher text + vpxorq %%B00_03, %%B00_03, %%DATA1 + vpxorq %%B04_07, %%B04_07, %%DATA2 + vpxorq %%B08_11, %%B08_11, %%DATA3 + vpxorq %%B12_15, %%B12_15, %%DATA4 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; store cipher/plain text + VX512STR [%%CIPH_PLAIN_OUT + %%DATA_OFFSET + %%DATA_DISPL + (0 * 64)], %%B00_03 + VX512STR [%%CIPH_PLAIN_OUT + %%DATA_OFFSET + %%DATA_DISPL + (1 * 64)], %%B04_07 + VX512STR [%%CIPH_PLAIN_OUT + %%DATA_OFFSET + %%DATA_DISPL + (2 * 64)], %%B08_11 + VX512STR [%%CIPH_PLAIN_OUT + %%DATA_OFFSET + %%DATA_DISPL + (3 * 64)], %%B12_15 + + ;; ================================================= + ;; shuffle cipher text blocks for GHASH computation +%ifidn %%ENC_DEC, ENC + vpshufb %%B00_03, %%B00_03, %%SHFMSK + vpshufb %%B04_07, %%B04_07, %%SHFMSK + vpshufb %%B08_11, %%B08_11, %%SHFMSK + vpshufb %%B12_15, %%B12_15, %%SHFMSK +%else + vpshufb %%B00_03, %%DATA1, %%SHFMSK + vpshufb %%B04_07, %%DATA2, %%SHFMSK + vpshufb %%B08_11, %%DATA3, %%SHFMSK + vpshufb %%B12_15, %%DATA4, %%SHFMSK +%endif + + ;; ================================================= + ;; store shuffled cipher text for ghashing + vmovdqa64 [rsp + %%AESOUT_BLK_OFFSET + (0*64)], %%B00_03 + vmovdqa64 [rsp + %%AESOUT_BLK_OFFSET + (1*64)], %%B04_07 + vmovdqa64 [rsp + %%AESOUT_BLK_OFFSET + (2*64)], %%B08_11 + vmovdqa64 [rsp + %%AESOUT_BLK_OFFSET + (3*64)], %%B12_15 + +%ifidn %%DO_REDUCTION, final_reduction + ;; ================================================= + ;; Return GHASH value through %%GH1H +%endif + +%endmacro ; GHASH_16_ENCRYPT_16_PARALLEL + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Encryption of a single block +%macro ENCRYPT_SINGLE_BLOCK 2 +%define %%GDATA %1 +%define %%XMM0 %2 + + vpxorq %%XMM0, %%XMM0, [%%GDATA+16*0] +%assign i 1 +%rep NROUNDS + vaesenc %%XMM0, [%%GDATA+16*i] +%assign i (i+1) +%endrep + vaesenclast %%XMM0, [%%GDATA+16*i] +%endmacro + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Save register content for the caller +%macro FUNC_SAVE 0 + ;; Required for Update/GMC_ENC + ;the number of pushes must equal STACK_OFFSET + mov rax, rsp + + sub rsp, STACK_FRAME_SIZE + and rsp, ~63 + + mov [rsp + STACK_GP_OFFSET + 0*8], r12 + mov [rsp + STACK_GP_OFFSET + 1*8], r13 + mov [rsp + STACK_GP_OFFSET + 2*8], r14 + mov [rsp + STACK_GP_OFFSET + 3*8], r15 + mov [rsp + STACK_GP_OFFSET + 4*8], rax ; stack + mov r14, rax ; r14 is used to retrieve stack args + mov [rsp + STACK_GP_OFFSET + 5*8], rbp + mov [rsp + STACK_GP_OFFSET + 6*8], rbx +%ifidn __OUTPUT_FORMAT__, win64 + mov [rsp + STACK_GP_OFFSET + 7*8], rdi + mov [rsp + STACK_GP_OFFSET + 8*8], rsi +%endif + +%ifidn __OUTPUT_FORMAT__, win64 + ; xmm6:xmm15 need to be maintained for Windows + vmovdqu [rsp + STACK_XMM_OFFSET + 0*16], xmm6 + vmovdqu [rsp + STACK_XMM_OFFSET + 1*16], xmm7 + vmovdqu [rsp + STACK_XMM_OFFSET + 2*16], xmm8 + vmovdqu [rsp + STACK_XMM_OFFSET + 3*16], xmm9 + vmovdqu [rsp + STACK_XMM_OFFSET + 4*16], xmm10 + vmovdqu [rsp + STACK_XMM_OFFSET + 5*16], xmm11 + vmovdqu [rsp + STACK_XMM_OFFSET + 6*16], xmm12 + vmovdqu [rsp + STACK_XMM_OFFSET + 7*16], xmm13 + vmovdqu [rsp + STACK_XMM_OFFSET + 8*16], xmm14 + vmovdqu [rsp + STACK_XMM_OFFSET + 9*16], xmm15 +%endif +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Restore register content for the caller +%macro FUNC_RESTORE 0 + +%ifdef SAFE_DATA + clear_scratch_gps_asm + clear_scratch_zmms_asm +%else + vzeroupper +%endif + +%ifidn __OUTPUT_FORMAT__, win64 + vmovdqu xmm15, [rsp + STACK_XMM_OFFSET + 9*16] + vmovdqu xmm14, [rsp + STACK_XMM_OFFSET + 8*16] + vmovdqu xmm13, [rsp + STACK_XMM_OFFSET + 7*16] + vmovdqu xmm12, [rsp + STACK_XMM_OFFSET + 6*16] + vmovdqu xmm11, [rsp + STACK_XMM_OFFSET + 5*16] + vmovdqu xmm10, [rsp + STACK_XMM_OFFSET + 4*16] + vmovdqu xmm9, [rsp + STACK_XMM_OFFSET + 3*16] + vmovdqu xmm8, [rsp + STACK_XMM_OFFSET + 2*16] + vmovdqu xmm7, [rsp + STACK_XMM_OFFSET + 1*16] + vmovdqu xmm6, [rsp + STACK_XMM_OFFSET + 0*16] +%endif + + ;; Required for Update/GMC_ENC + mov rbp, [rsp + STACK_GP_OFFSET + 5*8] + mov rbx, [rsp + STACK_GP_OFFSET + 6*8] +%ifidn __OUTPUT_FORMAT__, win64 + mov rdi, [rsp + STACK_GP_OFFSET + 7*8] + mov rsi, [rsp + STACK_GP_OFFSET + 8*8] +%endif + mov r12, [rsp + STACK_GP_OFFSET + 0*8] + mov r13, [rsp + STACK_GP_OFFSET + 1*8] + mov r14, [rsp + STACK_GP_OFFSET + 2*8] + mov r15, [rsp + STACK_GP_OFFSET + 3*8] + mov rsp, [rsp + STACK_GP_OFFSET + 4*8] ; stack +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Calculate J0 for cases when IV length is different than 12 bytes +%macro CALC_J0 27 +%define %%KEY %1 ;; [in] Pointer to GCM KEY structure +%define %%IV %2 ;; [in] Pointer to IV +%define %%IV_LEN %3 ;; [in] IV length +%define %%J0 %4 ;; [out] XMM reg to contain J0 +%define %%ZT0 %5 ;; [clobbered] ZMM register +%define %%ZT1 %6 ;; [clobbered] ZMM register +%define %%ZT2 %7 ;; [clobbered] ZMM register +%define %%ZT3 %8 ;; [clobbered] ZMM register +%define %%ZT4 %9 ;; [clobbered] ZMM register +%define %%ZT5 %10 ;; [clobbered] ZMM register +%define %%ZT6 %11 ;; [clobbered] ZMM register +%define %%ZT7 %12 ;; [clobbered] ZMM register +%define %%ZT8 %13 ;; [clobbered] ZMM register +%define %%ZT9 %14 ;; [clobbered] ZMM register +%define %%ZT10 %15 ;; [clobbered] ZMM register +%define %%ZT11 %16 ;; [clobbered] ZMM register +%define %%ZT12 %17 ;; [clobbered] ZMM register +%define %%ZT13 %18 ;; [clobbered] ZMM register +%define %%ZT14 %19 ;; [clobbered] ZMM register +%define %%ZT15 %20 ;; [clobbered] ZMM register +%define %%ZT16 %21 ;; [clobbered] ZMM register +%define %%ZT17 %22 ;; [clobbered] ZMM register +%define %%T1 %23 ;; [clobbered] GP register +%define %%T2 %24 ;; [clobbered] GP register +%define %%T3 %25 ;; [clobbered] GP register +%define %%MASKREG %26 ;; [clobbered] mask register +;; IPPCP: unlike IPsec-MB this macro is called in iv_finalize only that can follow +;; several iv_update calls, so iv_total_len should be an additional external parameter +;; here. +%define %%IV_TOTAL_LEN %27 ;; [in] total len of IV + + ;; J0 = GHASH(IV || 0s+64 || len(IV)64) + ;; s = 16 * RoundUp(len(IV)/16) - len(IV) */ + + ;; Calculate GHASH of (IV || 0s) + ;; vpxor %%J0, %%J0 ;; IPPCP: xor disabled (zeroing is done outside) + CALC_AAD_HASH %%IV, %%IV_LEN, %%J0, %%KEY, %%ZT0, %%ZT1, %%ZT2, %%ZT3, \ + %%ZT4, %%ZT5, %%ZT6, %%ZT7, %%ZT8, %%ZT9, %%ZT10, %%ZT11, \ + %%ZT12, %%ZT13, %%ZT14, %%ZT15, %%ZT16, %%ZT17, \ + %%T1, %%T2, %%T3, %%MASKREG + + ;; Calculate GHASH of last 16-byte block (0 || len(IV)64) + ;; mov %%T1, %%IV_LEN ;; IPPCP: iv total len is external parameter + mov %%T1, %%IV_TOTAL_LEN + shl %%T1, 3 ;; IV length in bits + vmovq XWORD(%%ZT2), %%T1 + + ;; Might need shuffle of ZT2 + vpxorq %%J0, XWORD(%%ZT2), %%J0 + + vmovdqu64 XWORD(%%ZT0), [%%KEY + HashKey_1] + GHASH_MUL %%J0, XWORD(%%ZT0), XWORD(%%ZT1), XWORD(%%ZT2), XWORD(%%ZT3), XWORD(%%ZT4), XWORD(%%ZT5) + + vpshufb %%J0, [rel SHUF_MASK] ; perform a 16Byte swap +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Cipher and ghash of payloads shorter than 256 bytes +;;; - number of blocks in the message comes as argument +;;; - depending on the number of blocks an optimized variant of +;;; INITIAL_BLOCKS_PARTIAL is invoked +%macro GCM_ENC_DEC_SMALL 39 +%define %%GDATA_KEY %1 ; [in] key pointer +%define %%GDATA_CTX %2 ; [in] context pointer +%define %%CIPH_PLAIN_OUT %3 ; [in] output buffer +%define %%PLAIN_CIPH_IN %4 ; [in] input buffer +%define %%PLAIN_CIPH_LEN %5 ; [in] buffer length +%define %%ENC_DEC %6 ; [in] cipher direction +%define %%DATA_OFFSET %7 ; [in] data offset +%define %%LENGTH %8 ; [in] GP data length +%define %%NUM_BLOCKS %9 ; [in] GP number of blocks to process 1 to 16 +%define %%CTR %10 ; [in/out] XMM counter block +%define %%HASH_IN_OUT %11 ; [in/out] XMM GHASH value +%define %%INSTANCE_TYPE %12 ; [in] single or multi call +%define %%ZTMP0 %13 ; [clobbered] ZMM register +%define %%ZTMP1 %14 ; [clobbered] ZMM register +%define %%ZTMP2 %15 ; [clobbered] ZMM register +%define %%ZTMP3 %16 ; [clobbered] ZMM register +%define %%ZTMP4 %17 ; [clobbered] ZMM register +%define %%ZTMP5 %18 ; [clobbered] ZMM register +%define %%ZTMP6 %19 ; [clobbered] ZMM register +%define %%ZTMP7 %20 ; [clobbered] ZMM register +%define %%ZTMP8 %21 ; [clobbered] ZMM register +%define %%ZTMP9 %22 ; [clobbered] ZMM register +%define %%ZTMP10 %23 ; [clobbered] ZMM register +%define %%ZTMP11 %24 ; [clobbered] ZMM register +%define %%ZTMP12 %25 ; [clobbered] ZMM register +%define %%ZTMP13 %26 ; [clobbered] ZMM register +%define %%ZTMP14 %27 ; [clobbered] ZMM register +%define %%ZTMP15 %28 ; [clobbered] ZMM register +%define %%ZTMP16 %29 ; [clobbered] ZMM register +%define %%ZTMP17 %30 ; [clobbered] ZMM register +%define %%ZTMP18 %31 ; [clobbered] ZMM register +%define %%ZTMP19 %32 ; [clobbered] ZMM register +%define %%ZTMP20 %33 ; [clobbered] ZMM register +%define %%ZTMP21 %34 ; [clobbered] ZMM register +%define %%ZTMP22 %35 ; [clobbered] ZMM register +%define %%IA0 %36 ; [clobbered] GP register +%define %%IA1 %37 ; [clobbered] GP register +%define %%MASKREG %38 ; [clobbered] mask register +%define %%SHUFMASK %39 ; [clobbered] ZMM to be used for BE/LE shuffle mask + + cmp DWORD(%%NUM_BLOCKS), 8 + je %%_small_initial_num_blocks_is_8 + jb %%_small_initial_num_blocks_is_7_1 + + cmp DWORD(%%NUM_BLOCKS), 12 + je %%_small_initial_num_blocks_is_12 + jb %%_small_initial_num_blocks_is_11_9 + + ;; 16, 15, 14 or 13 + cmp DWORD(%%NUM_BLOCKS), 15 + ja %%_small_initial_num_blocks_is_16 + je %%_small_initial_num_blocks_is_15 + cmp DWORD(%%NUM_BLOCKS), 14 + je %%_small_initial_num_blocks_is_14 + jmp %%_small_initial_num_blocks_is_13 + +%%_small_initial_num_blocks_is_11_9: + ;; 11, 10 or 9 + cmp DWORD(%%NUM_BLOCKS), 10 + ja %%_small_initial_num_blocks_is_11 + je %%_small_initial_num_blocks_is_10 + jmp %%_small_initial_num_blocks_is_9 + +%%_small_initial_num_blocks_is_7_1: + cmp DWORD(%%NUM_BLOCKS), 4 + je %%_small_initial_num_blocks_is_4 + jb %%_small_initial_num_blocks_is_3_1 + ;; 7, 6 or 5 + cmp DWORD(%%NUM_BLOCKS), 6 + ja %%_small_initial_num_blocks_is_7 + je %%_small_initial_num_blocks_is_6 + jmp %%_small_initial_num_blocks_is_5 + +%%_small_initial_num_blocks_is_3_1: + ;; 3, 2 or 1 + cmp DWORD(%%NUM_BLOCKS), 2 + ja %%_small_initial_num_blocks_is_3 + je %%_small_initial_num_blocks_is_2 + + ;; for %%NUM_BLOCKS == 1, just fall through and no 'jmp' needed + + ;; Use rep to generate different block size variants + ;; - one block size has to be the first one + ;; - ZTMP15 - ZTMP22 are free +%assign num_blocks 1 +%rep 16 +%%_small_initial_num_blocks_is_ %+ num_blocks : + INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CIPH_PLAIN_OUT, \ + %%PLAIN_CIPH_IN, %%LENGTH, %%DATA_OFFSET, num_blocks, \ + %%CTR, %%HASH_IN_OUT, %%ENC_DEC, %%INSTANCE_TYPE, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, \ + %%ZTMP5, %%ZTMP6, %%ZTMP7, %%ZTMP8, %%ZTMP9, \ + %%ZTMP10, %%ZTMP11, %%ZTMP12, %%ZTMP13, %%ZTMP14, \ + %%IA0, %%IA1, %%MASKREG, %%SHUFMASK +%if num_blocks != 16 + jmp %%_small_initial_blocks_encrypted +%endif +%assign num_blocks (num_blocks + 1) +%endrep + +%%_small_initial_blocks_encrypted: + +%endmacro ; GCM_ENC_DEC_SMALL + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; GCM_ENC_DEC Encodes/Decodes given data. Assumes that the passed gcm_context_data struct +; has been initialized by GCM_INIT +; Requires the input data be at least 1 byte long because of READ_SMALL_INPUT_DATA. +; Input: gcm_key_data struct* (GDATA_KEY), gcm_context_data *(GDATA_CTX), input text (PLAIN_CIPH_IN), +; input text length (PLAIN_CIPH_LEN) and whether encoding or decoding (ENC_DEC). +; Output: A cipher of the given plain text (CIPH_PLAIN_OUT), and updated GDATA_CTX +; Clobbers rax, r10-r15, and zmm0-zmm31, k1 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro GCM_ENC_DEC 7 +%define %%GDATA_KEY %1 ; [in] key pointer +%define %%GDATA_CTX %2 ; [in] context pointer +%define %%CIPH_PLAIN_OUT %3 ; [in] output buffer pointer +%define %%PLAIN_CIPH_IN %4 ; [in] input buffer pointer +%define %%PLAIN_CIPH_LEN %5 ; [in] buffer length +%define %%ENC_DEC %6 ; [in] cipher direction +%define %%INSTANCE_TYPE %7 ; [in] 'single_call' or 'multi_call' selection + +%define %%IA0 r10 +%define %%IA1 r12 +%define %%IA2 r13 +%define %%IA3 r15 +%define %%IA4 r11 +%define %%IA5 rax +%define %%IA6 rbx + +%ifidn __OUTPUT_FORMAT__, win64 +%define %%LENGTH %%IA2 +%endif +%define %%CTR_CHECK %%IA3 +%define %%DATA_OFFSET %%IA4 +%define %%HASHK_PTR %%IA6 + +%define %%CTR_BLOCKz zmm2 +%define %%CTR_BLOCKx xmm2 ; hardcoded in GCM_INIT + +%define %%AAD_HASHz zmm14 +%define %%AAD_HASHx xmm14 ; hardcoded in GCM_COMPLETE + +%define %%ZTMP0 zmm0 +%define %%ZTMP1 zmm3 +%define %%ZTMP2 zmm4 +%define %%ZTMP3 zmm5 +%define %%ZTMP4 zmm6 +%define %%ZTMP5 zmm7 +%define %%ZTMP6 zmm10 +%define %%ZTMP7 zmm11 +%define %%ZTMP8 zmm12 +%define %%ZTMP9 zmm13 +%define %%ZTMP10 zmm15 +%define %%ZTMP11 zmm16 +%define %%ZTMP12 zmm17 + +%define %%ZTMP13 zmm19 +%define %%ZTMP14 zmm20 +%define %%ZTMP15 zmm21 +%define %%ZTMP16 zmm30 +%define %%ZTMP17 zmm31 +%define %%ZTMP18 zmm1 +%define %%ZTMP19 zmm18 +%define %%ZTMP20 zmm8 +%define %%ZTMP21 zmm22 +%define %%ZTMP22 zmm23 + +%define %%GH zmm24 +%define %%GL zmm25 +%define %%GM zmm26 +%define %%SHUF_MASK zmm29 + +;;; Unused in the small packet path +%define %%ADDBE_4x4 zmm27 +%define %%ADDBE_1234 zmm28 + +%define %%MASKREG k1 + +;; reduction every 48 blocks, depth 32 blocks +;; @note 48 blocks is the maximum capacity of the stack frame +%assign big_loop_nblocks 48 +%assign big_loop_depth 32 + +;;; Macro flow depending on packet size +;;; - LENGTH <= 16 blocks +;;; - cipher followed by hashing (reduction) +;;; - 16 blocks < LENGTH < 32 blocks +;;; - cipher 16 blocks +;;; - cipher N blocks & hash 16 blocks, hash N blocks (reduction) +;;; - 32 blocks < LENGTH < 48 blocks +;;; - cipher 2 x 16 blocks +;;; - hash 16 blocks +;;; - cipher N blocks & hash 16 blocks, hash N blocks (reduction) +;;; - LENGTH >= 48 blocks +;;; - cipher 2 x 16 blocks +;;; - while (data_to_cipher >= 48 blocks): +;;; - cipher 16 blocks & hash 16 blocks +;;; - cipher 16 blocks & hash 16 blocks +;;; - cipher 16 blocks & hash 16 blocks (reduction) +;;; - if (data_to_cipher >= 32 blocks): +;;; - cipher 16 blocks & hash 16 blocks +;;; - cipher 16 blocks & hash 16 blocks +;;; - hash 16 blocks (reduction) +;;; - cipher N blocks & hash 16 blocks, hash N blocks (reduction) +;;; - elif (data_to_cipher >= 16 blocks): +;;; - cipher 16 blocks & hash 16 blocks +;;; - hash 16 blocks +;;; - cipher N blocks & hash 16 blocks, hash N blocks (reduction) +;;; - else: +;;; - hash 16 blocks +;;; - cipher N blocks & hash 16 blocks, hash N blocks (reduction) + +%ifidn __OUTPUT_FORMAT__, win64 + cmp %%PLAIN_CIPH_LEN, 0 +%else + or %%PLAIN_CIPH_LEN, %%PLAIN_CIPH_LEN +%endif + je %%_enc_dec_done + + ;; Update length of data processed +%ifidn __OUTPUT_FORMAT__, win64 + mov %%IA0, %%PLAIN_CIPH_LEN + add [%%GDATA_CTX + InLen], %%IA0 +%else + add [%%GDATA_CTX + InLen], %%PLAIN_CIPH_LEN +%endif + vmovdqu64 %%AAD_HASHx, [%%GDATA_CTX + AadHash] + +%ifidn %%INSTANCE_TYPE, multi_call + ;; NOTE: partial block processing makes only sense for multi_call here. + ;; Used for the update flow - if there was a previous partial + ;; block fill the remaining bytes here. + PARTIAL_BLOCK %%GDATA_KEY, %%GDATA_CTX, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, \ + %%PLAIN_CIPH_LEN, %%DATA_OFFSET, %%AAD_HASHx, %%ENC_DEC, \ + %%IA0, %%IA1, %%IA2, %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, \ + %%ZTMP5, %%ZTMP6, %%ZTMP7, %%ZTMP8, %%ZTMP9, %%MASKREG +%else + xor %%DATA_OFFSET, %%DATA_OFFSET +%endif + +%ifidn %%INSTANCE_TYPE, single_call + ;; use counter block from GCM_INIT +%else + vmovdqu64 %%CTR_BLOCKx, [%%GDATA_CTX + CurCount] +%endif + + ;; Save the amount of data left to process in %%LENGTH +%ifidn __OUTPUT_FORMAT__, win64 + mov %%LENGTH, %%PLAIN_CIPH_LEN +%else +%define %%LENGTH %%PLAIN_CIPH_LEN ;; PLAIN_CIPH_LEN is a register on linux +%endif + +%ifidn %%INSTANCE_TYPE, multi_call + ;; NOTE: %%DATA_OFFSET is zero in single_call case. + ;; Consequently PLAIN_CIPH_LEN will never be zero after + ;; %%DATA_OFFSET subtraction below. + ;; There may be no more data if it was consumed in the partial block. + sub %%LENGTH, %%DATA_OFFSET + je %%_enc_dec_done +%endif ; %%INSTANCE_TYPE, multi_call + + cmp %%LENGTH, (16 * 16) + jbe %%_message_below_equal_16_blocks + + vmovdqa64 %%SHUF_MASK, [rel SHUF_MASK] + vmovdqa64 %%ADDBE_4x4, [rel ddq_addbe_4444] + vmovdqa64 %%ADDBE_1234, [rel ddq_addbe_1234] + + ;; start the pipeline + ;; - 32 blocks aes-ctr + ;; - 16 blocks ghash + aes-ctr + + ;; set up CTR_CHECK + vmovd DWORD(%%CTR_CHECK), %%CTR_BLOCKx + and DWORD(%%CTR_CHECK), 255 + + ;; in LE format after init, convert to BE + vshufi64x2 %%CTR_BLOCKz, %%CTR_BLOCKz, %%CTR_BLOCKz, 0 + vpshufb %%CTR_BLOCKz, %%CTR_BLOCKz, %%SHUF_MASK + + ;; ==== AES-CTR - first 16 blocks +%assign aesout_offset (STACK_LOCAL_OFFSET + (0 * 16)) +%assign data_in_out_offset 0 + + INITIAL_BLOCKS_16 %%PLAIN_CIPH_IN, %%CIPH_PLAIN_OUT, %%GDATA_KEY, %%DATA_OFFSET, \ + no_ghash, %%CTR_BLOCKz, %%CTR_CHECK, %%ADDBE_4x4, %%ADDBE_1234, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, \ + %%ZTMP5, %%ZTMP6, %%ZTMP7, %%ZTMP8, \ + %%SHUF_MASK, %%ENC_DEC, aesout_offset, data_in_out_offset + + cmp %%LENGTH, (32 * 16) + jb %%_message_below_32_blocks + + ;; ==== AES-CTR - next 16 blocks +%assign aesout_offset (STACK_LOCAL_OFFSET + (16 * 16)) +%assign data_in_out_offset (16 * 16) + + INITIAL_BLOCKS_16 %%PLAIN_CIPH_IN, %%CIPH_PLAIN_OUT, %%GDATA_KEY, %%DATA_OFFSET, \ + no_ghash, %%CTR_BLOCKz, %%CTR_CHECK, %%ADDBE_4x4, %%ADDBE_1234, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, \ + %%ZTMP5, %%ZTMP6, %%ZTMP7, %%ZTMP8, \ + %%SHUF_MASK, %%ENC_DEC, aesout_offset, data_in_out_offset + + add %%DATA_OFFSET, (32 * 16) + sub %%LENGTH, (32 * 16) + + cmp %%LENGTH, (big_loop_nblocks * 16) + jb %%_no_more_big_nblocks + + ;; ==== + ;; ==== AES-CTR + GHASH - 48 blocks loop + ;; ==== +%%_encrypt_big_nblocks: + ;; ==== AES-CTR + GHASH - 16 blocks, start +%assign aesout_offset (STACK_LOCAL_OFFSET + (32 * 16)) +%assign data_in_out_offset (0 * 16) +%assign ghashin_offset (STACK_LOCAL_OFFSET + (0 * 16)) + + GHASH_16_ENCRYPT_16_PARALLEL %%GDATA_KEY, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, %%DATA_OFFSET, \ + %%CTR_BLOCKz, %%CTR_CHECK, \ + HashKey_48, aesout_offset, ghashin_offset, %%SHUF_MASK, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, \ + %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11,\ + %%ZTMP12, %%ZTMP13, %%ZTMP14, %%ZTMP15,\ + %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, \ + %%ZTMP20, %%ZTMP21, %%ZTMP22, \ + %%ADDBE_4x4, %%ADDBE_1234, \ + %%GL, %%GH, %%GM, \ + first_time, %%ENC_DEC, data_in_out_offset, %%AAD_HASHz + + ;; ==== AES-CTR + GHASH - 16 blocks, no reduction +%assign aesout_offset (STACK_LOCAL_OFFSET + (0 * 16)) +%assign data_in_out_offset (16 * 16) +%assign ghashin_offset (STACK_LOCAL_OFFSET + (16 * 16)) + + GHASH_16_ENCRYPT_16_PARALLEL %%GDATA_KEY, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, %%DATA_OFFSET, \ + %%CTR_BLOCKz, %%CTR_CHECK, \ + HashKey_32, aesout_offset, ghashin_offset, %%SHUF_MASK, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, \ + %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11,\ + %%ZTMP12, %%ZTMP13, %%ZTMP14, %%ZTMP15,\ + %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, \ + %%ZTMP20, %%ZTMP21, %%ZTMP22, \ + %%ADDBE_4x4, %%ADDBE_1234, \ + %%GL, %%GH, %%GM, \ + no_reduction, %%ENC_DEC, data_in_out_offset, no_ghash_in + + ;; ==== AES-CTR + GHASH - 16 blocks, reduction +%assign aesout_offset (STACK_LOCAL_OFFSET + (16 * 16)) +%assign data_in_out_offset (32 * 16) +%assign ghashin_offset (STACK_LOCAL_OFFSET + (32 * 16)) + + GHASH_16_ENCRYPT_16_PARALLEL %%GDATA_KEY, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, %%DATA_OFFSET, \ + %%CTR_BLOCKz, %%CTR_CHECK, \ + HashKey_16, aesout_offset, ghashin_offset, %%SHUF_MASK, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, \ + %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11,\ + %%ZTMP12, %%ZTMP13, %%ZTMP14, %%ZTMP15,\ + %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, \ + %%ZTMP20, %%ZTMP21, %%ZTMP22, \ + %%ADDBE_4x4, %%ADDBE_1234, \ + %%GL, %%GH, %%GM, \ + final_reduction, %%ENC_DEC, data_in_out_offset, no_ghash_in + + ;; === xor cipher block 0 with GHASH (ZT4) + vmovdqa64 %%AAD_HASHz, %%ZTMP4 + + add %%DATA_OFFSET, (big_loop_nblocks * 16) + sub %%LENGTH, (big_loop_nblocks * 16) + cmp %%LENGTH, (big_loop_nblocks * 16) + jae %%_encrypt_big_nblocks + +%%_no_more_big_nblocks: + + cmp %%LENGTH, (32 * 16) + jae %%_encrypt_32_blocks + + cmp %%LENGTH, (16 * 16) + jae %%_encrypt_16_blocks + + ;; ===================================================== + ;; ===================================================== + ;; ==== GHASH 1 x 16 blocks + ;; ==== GHASH 1 x 16 blocks (reduction) & encrypt N blocks + ;; ==== then GHASH N blocks +%%_encrypt_0_blocks_ghash_32: + ;; calculate offset to the right hash key +%ifidn %%INSTANCE_TYPE, multi_call + mov DWORD(%%IA0), DWORD(%%LENGTH) +%else + lea DWORD(%%IA0), [DWORD(%%LENGTH) + 15] +%endif + and DWORD(%%IA0), ~15 + mov DWORD(%%HASHK_PTR), HashKey_32 + sub DWORD(%%HASHK_PTR), DWORD(%%IA0) + + ;; ==== GHASH 32 blocks and follow with reduction + GHASH_16 start, %%GH, %%GM, %%GL, rsp, STACK_LOCAL_OFFSET, (0 * 16), %%GDATA_KEY, %%HASHK_PTR, 0, %%AAD_HASHz, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%ZTMP8, %%ZTMP9 + + ;; ==== GHASH 1 x 16 blocks with reduction + cipher and ghash on the reminder +%assign ghashin_offset (STACK_LOCAL_OFFSET + (16 * 16)) + add DWORD(%%HASHK_PTR), (16 * 16) + + GCM_ENC_DEC_LAST \ + %%GDATA_KEY, %%GDATA_CTX, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, \ + %%DATA_OFFSET, %%LENGTH, %%CTR_BLOCKz, %%CTR_CHECK, \ + %%HASHK_PTR, ghashin_offset, %%SHUF_MASK, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ + %%ZTMP7, %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11, %%ZTMP12, %%ZTMP13, \ + %%ZTMP14, %%ZTMP15, %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, %%ZTMP20, \ + %%ZTMP21, %%ZTMP22, \ + %%ADDBE_4x4, %%ADDBE_1234, mid, %%GL, %%GH, %%GM, \ + %%ENC_DEC, %%AAD_HASHz, %%IA0, %%IA5, %%MASKREG, %%INSTANCE_TYPE + +%ifidn %%INSTANCE_TYPE, multi_call + vpshufb %%CTR_BLOCKx, %%CTR_BLOCKx, XWORD(%%SHUF_MASK) +%endif + jmp %%_ghash_done + + ;; ===================================================== + ;; ===================================================== + ;; ==== GHASH & encrypt 1 x 16 blocks + ;; ==== GHASH & encrypt 1 x 16 blocks + ;; ==== GHASH 1 x 16 blocks (reduction) + ;; ==== GHASH 1 x 16 blocks (reduction) & encrypt N blocks + ;; ==== then GHASH N blocks +%%_encrypt_32_blocks: + ;; ==== AES-CTR + GHASH - 16 blocks, start +%assign aesout_offset (STACK_LOCAL_OFFSET + (32 * 16)) +%assign ghashin_offset (STACK_LOCAL_OFFSET + (0 * 16)) +%assign data_in_out_offset (0 * 16) + + GHASH_16_ENCRYPT_16_PARALLEL %%GDATA_KEY, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, %%DATA_OFFSET, \ + %%CTR_BLOCKz, %%CTR_CHECK, \ + HashKey_48, aesout_offset, ghashin_offset, %%SHUF_MASK, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, \ + %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11,\ + %%ZTMP12, %%ZTMP13, %%ZTMP14, %%ZTMP15,\ + %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, \ + %%ZTMP20, %%ZTMP21, %%ZTMP22, \ + %%ADDBE_4x4, %%ADDBE_1234, \ + %%GL, %%GH, %%GM, \ + first_time, %%ENC_DEC, data_in_out_offset, %%AAD_HASHz + + ;; ==== AES-CTR + GHASH - 16 blocks, no reduction +%assign aesout_offset (STACK_LOCAL_OFFSET + (0 * 16)) +%assign ghashin_offset (STACK_LOCAL_OFFSET + (16 * 16)) +%assign data_in_out_offset (16 * 16) + + GHASH_16_ENCRYPT_16_PARALLEL %%GDATA_KEY, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, %%DATA_OFFSET, \ + %%CTR_BLOCKz, %%CTR_CHECK, \ + HashKey_32, aesout_offset, ghashin_offset, %%SHUF_MASK, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, \ + %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11,\ + %%ZTMP12, %%ZTMP13, %%ZTMP14, %%ZTMP15,\ + %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, \ + %%ZTMP20, %%ZTMP21, %%ZTMP22, \ + %%ADDBE_4x4, %%ADDBE_1234, \ + %%GL, %%GH, %%GM, \ + no_reduction, %%ENC_DEC, data_in_out_offset, no_ghash_in + + ;; ==== GHASH 16 blocks with reduction + GHASH_16 end_reduce, %%GH, %%GM, %%GL, rsp, STACK_LOCAL_OFFSET, (32 * 16), %%GDATA_KEY, HashKey_16, 0, %%AAD_HASHz, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%ZTMP8, %%ZTMP9 + + ;; ==== GHASH 1 x 16 blocks with reduction + cipher and ghash on the reminder +%assign ghashin_offset (STACK_LOCAL_OFFSET + (0 * 16)) + + sub %%LENGTH, (32 * 16) + add %%DATA_OFFSET, (32 * 16) + + ;; calculate offset to the right hash key +%ifidn %%INSTANCE_TYPE, multi_call + mov DWORD(%%IA0), DWORD(%%LENGTH) +%else + lea DWORD(%%IA0), [DWORD(%%LENGTH) + 15] +%endif + and DWORD(%%IA0), ~15 + mov DWORD(%%HASHK_PTR), HashKey_16 + sub DWORD(%%HASHK_PTR), DWORD(%%IA0) + + GCM_ENC_DEC_LAST \ + %%GDATA_KEY, %%GDATA_CTX, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, \ + %%DATA_OFFSET, %%LENGTH, %%CTR_BLOCKz, %%CTR_CHECK, \ + %%HASHK_PTR, ghashin_offset, %%SHUF_MASK, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ + %%ZTMP7, %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11, %%ZTMP12, %%ZTMP13, \ + %%ZTMP14, %%ZTMP15, %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, %%ZTMP20, \ + %%ZTMP21, %%ZTMP22, \ + %%ADDBE_4x4, %%ADDBE_1234, start, %%GL, %%GH, %%GM, \ + %%ENC_DEC, %%AAD_HASHz, %%IA0, %%IA5, %%MASKREG, %%INSTANCE_TYPE + +%ifidn %%INSTANCE_TYPE, multi_call + vpshufb %%CTR_BLOCKx, %%CTR_BLOCKx, XWORD(%%SHUF_MASK) +%endif + jmp %%_ghash_done + + ;; ===================================================== + ;; ===================================================== + ;; ==== GHASH & encrypt 16 blocks (done before) + ;; ==== GHASH 1 x 16 blocks + ;; ==== GHASH 1 x 16 blocks (reduction) & encrypt N blocks + ;; ==== then GHASH N blocks +%%_encrypt_16_blocks: + ;; ==== AES-CTR + GHASH - 16 blocks, start +%assign aesout_offset (STACK_LOCAL_OFFSET + (32 * 16)) +%assign ghashin_offset (STACK_LOCAL_OFFSET + (0 * 16)) +%assign data_in_out_offset (0 * 16) + + GHASH_16_ENCRYPT_16_PARALLEL %%GDATA_KEY, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, %%DATA_OFFSET, \ + %%CTR_BLOCKz, %%CTR_CHECK, \ + HashKey_48, aesout_offset, ghashin_offset, %%SHUF_MASK, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, \ + %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11,\ + %%ZTMP12, %%ZTMP13, %%ZTMP14, %%ZTMP15,\ + %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, \ + %%ZTMP20, %%ZTMP21, %%ZTMP22, \ + %%ADDBE_4x4, %%ADDBE_1234, \ + %%GL, %%GH, %%GM, \ + first_time, %%ENC_DEC, data_in_out_offset, %%AAD_HASHz + + ;; ==== GHASH 1 x 16 blocks + GHASH_16 mid, %%GH, %%GM, %%GL, rsp, STACK_LOCAL_OFFSET, (16 * 16), %%GDATA_KEY, HashKey_32, 0, no_hash_input, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%ZTMP8, %%ZTMP9 + + ;; ==== GHASH 1 x 16 blocks with reduction + cipher and ghash on the reminder +%assign ghashin_offset (STACK_LOCAL_OFFSET + (32 * 16)) + + sub %%LENGTH, (16 * 16) + add %%DATA_OFFSET, (16 * 16) + + GCM_ENC_DEC_LAST \ + %%GDATA_KEY, %%GDATA_CTX, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, \ + %%DATA_OFFSET, %%LENGTH, %%CTR_BLOCKz, %%CTR_CHECK, \ + HashKey_16, ghashin_offset, %%SHUF_MASK, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ + %%ZTMP7, %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11, %%ZTMP12, %%ZTMP13, \ + %%ZTMP14, %%ZTMP15, %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, %%ZTMP20, \ + %%ZTMP21, %%ZTMP22, \ + %%ADDBE_4x4, %%ADDBE_1234, end_reduce, %%GL, %%GH, %%GM, \ + %%ENC_DEC, %%AAD_HASHz, %%IA0, %%IA5, %%MASKREG, %%INSTANCE_TYPE + +%ifidn %%INSTANCE_TYPE, multi_call + vpshufb %%CTR_BLOCKx, %%CTR_BLOCKx, XWORD(%%SHUF_MASK) +%endif + jmp %%_ghash_done + +%%_message_below_32_blocks: + ;; 32 > number of blocks > 16 + + sub %%LENGTH, (16 * 16) + add %%DATA_OFFSET, (16 * 16) + +%assign ghashin_offset (STACK_LOCAL_OFFSET + (0 * 16)) + + ;; calculate offset to the right hash key +%ifidn %%INSTANCE_TYPE, multi_call + mov DWORD(%%IA0), DWORD(%%LENGTH) +%else + lea DWORD(%%IA0), [DWORD(%%LENGTH) + 15] +%endif + and DWORD(%%IA0), ~15 + mov DWORD(%%HASHK_PTR), HashKey_16 + sub DWORD(%%HASHK_PTR), DWORD(%%IA0) + + GCM_ENC_DEC_LAST \ + %%GDATA_KEY, %%GDATA_CTX, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, \ + %%DATA_OFFSET, %%LENGTH, %%CTR_BLOCKz, %%CTR_CHECK, \ + %%HASHK_PTR, ghashin_offset, %%SHUF_MASK, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ + %%ZTMP7, %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11, %%ZTMP12, %%ZTMP13, \ + %%ZTMP14, %%ZTMP15, %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, %%ZTMP20, \ + %%ZTMP21, %%ZTMP22, \ + %%ADDBE_4x4, %%ADDBE_1234, start, %%GL, %%GH, %%GM, \ + %%ENC_DEC, %%AAD_HASHz, %%IA0, %%IA5, %%MASKREG, %%INSTANCE_TYPE + +%ifidn %%INSTANCE_TYPE, multi_call + vpshufb %%CTR_BLOCKx, %%CTR_BLOCKx, XWORD(%%SHUF_MASK) +%endif + jmp %%_ghash_done + +%%_message_below_equal_16_blocks: + ;; Determine how many blocks to process + ;; - process one additional block if there is a partial block + mov DWORD(%%IA1), DWORD(%%LENGTH) + add DWORD(%%IA1), 15 + shr DWORD(%%IA1), 4 + ;; %%IA1 can be in the range from 0 to 16 + + GCM_ENC_DEC_SMALL \ + %%GDATA_KEY, %%GDATA_CTX, %%CIPH_PLAIN_OUT, %%PLAIN_CIPH_IN, \ + %%PLAIN_CIPH_LEN, %%ENC_DEC, %%DATA_OFFSET, \ + %%LENGTH, %%IA1, %%CTR_BLOCKx, %%AAD_HASHx, %%INSTANCE_TYPE, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, \ + %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11, \ + %%ZTMP12, %%ZTMP13, %%ZTMP14, %%ZTMP15, \ + %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, \ + %%ZTMP20, %%ZTMP21, %%ZTMP22, \ + %%IA0, %%IA3, %%MASKREG, %%SHUF_MASK + + ;; fall through to exit + +%%_ghash_done: +%ifidn %%INSTANCE_TYPE, multi_call + ;; save the last counter block + vmovdqu64 [%%GDATA_CTX + CurCount], %%CTR_BLOCKx +%endif + vmovdqu64 [%%GDATA_CTX + AadHash], %%AAD_HASHx +%%_enc_dec_done: + +%endmacro ; GCM_ENC_DEC + +;;; =========================================================================== +;;; =========================================================================== +;;; Encrypt/decrypt the initial 16 blocks +%macro INITIAL_BLOCKS_16 22 +%define %%IN %1 ; [in] input buffer +%define %%OUT %2 ; [in] output buffer +%define %%KP %3 ; [in] pointer to expanded keys +%define %%DATA_OFFSET %4 ; [in] data offset +%define %%GHASH %5 ; [in] ZMM with AAD (low 128 bits) +%define %%CTR %6 ; [in] ZMM with CTR BE blocks 4x128 bits +%define %%CTR_CHECK %7 ; [in/out] GPR with counter overflow check +%define %%ADDBE_4x4 %8 ; [in] ZMM 4x128bits with value 4 (big endian) +%define %%ADDBE_1234 %9 ; [in] ZMM 4x128bits with values 1, 2, 3 & 4 (big endian) +%define %%T0 %10 ; [clobered] temporary ZMM register +%define %%T1 %11 ; [clobered] temporary ZMM register +%define %%T2 %12 ; [clobered] temporary ZMM register +%define %%T3 %13 ; [clobered] temporary ZMM register +%define %%T4 %14 ; [clobered] temporary ZMM register +%define %%T5 %15 ; [clobered] temporary ZMM register +%define %%T6 %16 ; [clobered] temporary ZMM register +%define %%T7 %17 ; [clobered] temporary ZMM register +%define %%T8 %18 ; [clobered] temporary ZMM register +%define %%SHUF_MASK %19 ; [in] ZMM with BE/LE shuffle mask +%define %%ENC_DEC %20 ; [in] ENC (encrypt) or DEC (decrypt) selector +%define %%BLK_OFFSET %21 ; [in] stack frame offset to ciphered blocks +%define %%DATA_DISPL %22 ; [in] fixed numerical data displacement/offset + +%define %%B00_03 %%T5 +%define %%B04_07 %%T6 +%define %%B08_11 %%T7 +%define %%B12_15 %%T8 + +%assign stack_offset (%%BLK_OFFSET) + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; prepare counter blocks + + cmp BYTE(%%CTR_CHECK), (256 - 16) + jae %%_next_16_overflow + vpaddd %%B00_03, %%CTR, %%ADDBE_1234 + vpaddd %%B04_07, %%B00_03, %%ADDBE_4x4 + vpaddd %%B08_11, %%B04_07, %%ADDBE_4x4 + vpaddd %%B12_15, %%B08_11, %%ADDBE_4x4 + jmp %%_next_16_ok +%%_next_16_overflow: + vpshufb %%CTR, %%CTR, %%SHUF_MASK + vmovdqa64 %%B12_15, [rel ddq_add_4444] + vpaddd %%B00_03, %%CTR, [rel ddq_add_1234] + vpaddd %%B04_07, %%B00_03, %%B12_15 + vpaddd %%B08_11, %%B04_07, %%B12_15 + vpaddd %%B12_15, %%B08_11, %%B12_15 + vpshufb %%B00_03, %%SHUF_MASK + vpshufb %%B04_07, %%SHUF_MASK + vpshufb %%B08_11, %%SHUF_MASK + vpshufb %%B12_15, %%SHUF_MASK +%%_next_16_ok: + vshufi64x2 %%CTR, %%B12_15, %%B12_15, 1111_1111b + add BYTE(%%CTR_CHECK), 16 + + ;; === load 16 blocks of data + VX512LDR %%T0, [%%IN + %%DATA_OFFSET + %%DATA_DISPL + (64*0)] + VX512LDR %%T1, [%%IN + %%DATA_OFFSET + %%DATA_DISPL + (64*1)] + VX512LDR %%T2, [%%IN + %%DATA_OFFSET + %%DATA_DISPL + (64*2)] + VX512LDR %%T3, [%%IN + %%DATA_OFFSET + %%DATA_DISPL + (64*3)] + + ;; move to AES encryption rounds +%assign i 0 + vbroadcastf64x2 %%T4, [%%KP + (16*i)] + vpxorq %%B00_03, %%B00_03, %%T4 + vpxorq %%B04_07, %%B04_07, %%T4 + vpxorq %%B08_11, %%B08_11, %%T4 + vpxorq %%B12_15, %%B12_15, %%T4 +%assign i (i + 1) + +%rep NROUNDS + vbroadcastf64x2 %%T4, [%%KP + (16*i)] + vaesenc %%B00_03, %%B00_03, %%T4 + vaesenc %%B04_07, %%B04_07, %%T4 + vaesenc %%B08_11, %%B08_11, %%T4 + vaesenc %%B12_15, %%B12_15, %%T4 +%assign i (i + 1) +%endrep + + vbroadcastf64x2 %%T4, [%%KP + (16*i)] + vaesenclast %%B00_03, %%B00_03, %%T4 + vaesenclast %%B04_07, %%B04_07, %%T4 + vaesenclast %%B08_11, %%B08_11, %%T4 + vaesenclast %%B12_15, %%B12_15, %%T4 + + ;; xor against text + vpxorq %%B00_03, %%B00_03, %%T0 + vpxorq %%B04_07, %%B04_07, %%T1 + vpxorq %%B08_11, %%B08_11, %%T2 + vpxorq %%B12_15, %%B12_15, %%T3 + + ;; store + VX512STR [%%OUT + %%DATA_OFFSET + %%DATA_DISPL + (64*0)], %%B00_03 + VX512STR [%%OUT + %%DATA_OFFSET + %%DATA_DISPL + (64*1)], %%B04_07 + VX512STR [%%OUT + %%DATA_OFFSET + %%DATA_DISPL + (64*2)], %%B08_11 + VX512STR [%%OUT + %%DATA_OFFSET + %%DATA_DISPL + (64*3)], %%B12_15 + +%ifidn %%ENC_DEC, DEC + ;; decryption - cipher text needs to go to GHASH phase + vpshufb %%B00_03, %%T0, %%SHUF_MASK + vpshufb %%B04_07, %%T1, %%SHUF_MASK + vpshufb %%B08_11, %%T2, %%SHUF_MASK + vpshufb %%B12_15, %%T3, %%SHUF_MASK +%else + ;; encryption + vpshufb %%B00_03, %%B00_03, %%SHUF_MASK + vpshufb %%B04_07, %%B04_07, %%SHUF_MASK + vpshufb %%B08_11, %%B08_11, %%SHUF_MASK + vpshufb %%B12_15, %%B12_15, %%SHUF_MASK +%endif + +%ifnidn %%GHASH, no_ghash + ;; === xor cipher block 0 with GHASH for the next GHASH round + vpxorq %%B00_03, %%B00_03, %%GHASH +%endif + + vmovdqa64 [rsp + stack_offset + (0 * 64)], %%B00_03 + vmovdqa64 [rsp + stack_offset + (1 * 64)], %%B04_07 + vmovdqa64 [rsp + stack_offset + (2 * 64)], %%B08_11 + vmovdqa64 [rsp + stack_offset + (3 * 64)], %%B12_15 +%endmacro ;INITIAL_BLOCKS_16 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; GCM_COMPLETE Finishes Encryption/Decryption of last partial block after GCM_UPDATE finishes. +; Input: A gcm_key_data * (GDATA_KEY), gcm_context_data (GDATA_CTX). +; Output: Authorization Tag (AUTH_TAG) and Authorization Tag length (AUTH_TAG_LEN) +; Clobbers xmm0-xmm2, xmm5-xmm6, xmm9-xmm11, xmm13-xmm15 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro GCM_COMPLETE 9 +%define %%GDATA_KEY %1 ; [in] GP with pointer to key structure +%define %%GDATA_CTX %2 ; [in] GP with pointer to context structure +%define %%AUTH_TAG %3 ; [in] pointer to store auth tag into (GP or mem) +%define %%AUTH_TAG_LEN %4 ; [in] length in bytes of auth tag (GP or mem) +%define %%INSTANCE_TYPE %5 ; [in] instance type "single_call" vs "multi_call" +%define %%MASKREG %6 ; [clobbered] temporary K register +%define %%IA0 %7 ; [clobbered] temporary GP +%define %%IA1 %8 ; [clobbered] temporary GP +%define %%IA2 %9 ; [clobbered] temporary GP + + ;; @note: xmm14 is hardcoded for hash input in singe_call case + + vmovdqu xmm13, [%%GDATA_KEY + HashKey] + ;; Start AES as early as possible + vmovdqu xmm9, [%%GDATA_CTX + OrigIV] ; xmm9 = Y0 + ENCRYPT_SINGLE_BLOCK %%GDATA_KEY, xmm9 ; E(K, Y0) + +%ifidn %%INSTANCE_TYPE, multi_call + ;; If the GCM function is called as a single function call rather + ;; than invoking the individual parts (init, update, finalize) we + ;; can remove a write to read dependency on AadHash. + vmovdqu xmm14, [%%GDATA_CTX + AadHash] + + ;; Encrypt of the final partial block was already done in the main GCM_ENC_DEC macro. + ;; It may be required to GHASH it now + cmp qword [%%GDATA_CTX + PBlockLen], 0 + je %%_partial_done + + ;; GHASH computation for the last <16 Byte block + GHASH_MUL xmm14, xmm13, xmm0, xmm10, xmm11, xmm5, xmm6 + +%%_partial_done: +%endif + vmovq xmm15, [%%GDATA_CTX + InLen] + vpinsrq xmm15, [%%GDATA_CTX + AadLen], 1 ; xmm15 = len(A)||len(C) + vpsllq xmm15, xmm15, 3 ; convert bytes into bits + + vpxor xmm14, xmm15 + GHASH_MUL xmm14, xmm13, xmm0, xmm10, xmm11, xmm5, xmm6 + vpshufb xmm14, [rel SHUF_MASK] ; perform a 16Byte swap + + vpxor xmm9, xmm9, xmm14 + + ;; xmm9 includes the final TAG + mov %%IA0, %%AUTH_TAG ; r10 = authTag + mov %%IA1, %%AUTH_TAG_LEN ; r11 = auth_tag_len + + cmp %%IA1, 16 + je %%_T_16 + + cmp %%IA1, 12 + je %%_T_12 + + cmp %%IA1, 8 + je %%_T_8 + + lea %%IA2, [rel byte64_len_to_mask_table] + kmovq %%MASKREG, [%%IA2 + %%IA1*8] + vmovdqu8 [%%IA0]{%%MASKREG}, xmm9 + jmp %%_return_T_done +%%_T_8: + vmovq [%%IA0], xmm9 + jmp %%_return_T_done +%%_T_12: + vmovq [%%IA0], xmm9 + vpextrd [%%IA0 + 8], xmm9, 2 + jmp %%_return_T_done +%%_T_16: + vmovdqu [%%IA0], xmm9 + +%%_return_T_done: + +;; IPPCP: disabled clearing of context fields, as in IPPCP case GetTag may be called +;; several times and it should not stop GCM processing. Context zeroing is done by user. +;; https://www.intel.com/content/www/us/en/docs/ipp-crypto/developer-reference/current/aes-gcm-functions.html +;; %ifdef SAFE_DATA +;; ;; Clear sensitive data from context structure +;; vpxor xmm0, xmm0 +;; vmovdqu [%%GDATA_CTX + AadHash], xmm0 +;; %ifidn %%INSTANCE_TYPE, multi_call +;; vmovdqu [%%GDATA_CTX + PBlockEncKey], xmm0 +;; %endif +;; %endif +%endmacro ; GCM_COMPLETE + +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/ia_32e_regs.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/ia_32e_regs.inc new file mode 100644 index 0000000..ee3f4a4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/ia_32e_regs.inc @@ -0,0 +1,125 @@ +;=============================================================================== +; Copyright (C) 2012 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: EM64T Cryptography Primitive. +; +; +; + +;; +;; Just for unify GPRs usage +;; + +%ifndef _IA_32_REGS_INC_ +%define _IA_32_REGS_INC_ + +%define r0 rax ;; 64-bits GPRs +%define r1 rbx +%define r2 rcx +%define r3 rdx +%define r4 rdi +%define r5 rsi +%define r6 rbp +%define r7 rsp + +%define r0d eax ;; 32-bits GPRs +%define r1d ebx +%define r2d ecx +%define r3d edx +%define r4d edi +%define r5d esi +%define r6d ebp +%define r7d esp + +%define raxd eax +%define rbxd ebx +%define rcxd ecx +%define rdxd edx +%define rdid edi +%define rsid esi +%define rbpd ebp + +%define r0w ax ;; 16-bits GPRs +%define r1w bx +%define r2w cx +%define r3w dx +%define r4w di +%define r5w si +%define r6w bp +%define r7w sp + +%define raxw ax +%define rbxw bx +%define rcxw cx +%define rdxw dx +%define rdiw di +%define rsiw si +%define rbpw bp + +%define r0b al ;; 8-bits GPRs +%define r1b bl +%define r2b cl +%define r3b dl +%define r4b dil +%define r5b sil +%define r6b bpl +%define r7b spl + +%define raxb al +%define rbxb bl +%define rcxb cl +%define rdxb dl +%define rdib dil +%define rsib sil +%define rbpb bpl + +%define raxbl al +%define rbxbl bl +%define rcxbl cl +%define rdxbl dl +%define raxbh ah +%define rbxbh bh +%define rcxbh ch +%define rdxbh dh + +;; +;; Register Parameters (depend on used OS) +;; +%ifdef WIN32E + %define rpar1 rcx + %define rpar2 rdx + %define rpar3 r8 + %define rpar4 r9 + %define rpar5 [rsp + ARG_5] + %define rpar6 [rsp + ARG_6] +%endif + +%ifdef LINUX32E + %define rpar1 rdi + %define rpar2 rsi + %define rpar3 rdx + %define rpar4 rcx + %define rpar5 r8 + %define rpar6 r9 +%endif + +;; use GPR implementation everywhere possible +%assign GPR_version 1 + +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/memcpy.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/memcpy.inc new file mode 100644 index 0000000..482e4fa --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/memcpy.inc @@ -0,0 +1,603 @@ +;=============================================================================== +; Copyright (C) 2020 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +%ifndef __MEMCPY_ASM__ +%define __MEMCPY_ASM__ + +%include "reg_sizes.inc" + + +; This section defines a series of macros to copy small to medium amounts +; of data from memory to memory, where the size is variable but limited. +; +; The macros are all called as: +; memcpy DST, SRC, SIZE, TMP0, TMP1, XTMP0, XTMP1, XTMP2, XTMP3 +; with the parameters defined as: +; DST : register: pointer to dst (not modified) +; SRC : register: pointer to src (not modified) +; SIZE : register: length in bytes (not modified) +; TMP0 : 64-bit temp GPR (clobbered) +; TMP1 : 64-bit temp GPR (clobbered) +; XTMP0 : temp XMM (clobbered) +; XTMP1 : temp XMM (clobbered) +; XTMP2 : temp XMM (clobbered) +; XTMP3 : temp XMM (clobbered) +; +; The name indicates the options. The name is of the form: +; memcpy__ +; where: +; is either "sse" or "avx" or "avx2" +; is either "64" or "128" and defines largest value of SIZE +; is blank or "_1". If "_1" then the min SIZE is 1 (otherwise 0) +; is blank or "_ret". If blank, the code falls through. If "ret" +; it does a "ret" at the end +; +; For the avx2 versions, the temp XMM registers need to be YMM registers +; If the SZ is 64, then only two YMM temps are needed, i.e. it is called as: +; memcpy_avx2_64 DST, SRC, SIZE, TMP0, TMP1, YTMP0, YTMP1 +; memcpy_avx2_128 DST, SRC, SIZE, TMP0, TMP1, YTMP0, YTMP1, YTMP2, YTMP3 +; +; For example: +; memcpy_sse_64 : SSE, 0 <= size < 64, falls through +; memcpy_avx_64_1 : AVX1, 1 <= size < 64, falls through +; memcpy_sse_128_ret : SSE, 0 <= size < 128, ends with ret +; mempcy_avx_128_1_ret : AVX1, 1 <= size < 128, ends with ret +; + +%macro memcpy_sse_64 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 64, 0, 0 +%endm + +%macro memcpy_sse_64_1 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 64, 0, 0 +%endm + +%macro memcpy_sse_128 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 128, 0, 0 +%endm + +%macro memcpy_sse_128_1 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 128, 0, 0 +%endm + +%macro memcpy_sse_64_ret 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 64, 1, 0 +%endm + +%macro memcpy_sse_64_1_ret 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 64, 1, 0 +%endm + +%macro memcpy_sse_128_ret 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 128, 1, 0 +%endm + +%macro memcpy_sse_128_1_ret 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 128, 1, 0 +%endm + + +%macro memcpy_sse_16 5 + __memcpy_int %1,%2,%3,%4,%5,,,,, 0, 16, 0, 0 +%endm + +%macro memcpy_sse_16_1 5 + __memcpy_int %1,%2,%3,%4,%5,,,,, 1, 16, 0, 0 +%endm + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%macro memcpy_avx_64 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 64, 0, 1 +%endm + +%macro memcpy_avx_64_1 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 64, 0, 1 +%endm + +%macro memcpy_avx_128 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 128, 0, 1 +%endm + +%macro memcpy_avx_128_1 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 128, 0, 1 +%endm + +%macro memcpy_avx_64_ret 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 64, 1, 1 +%endm + +%macro memcpy_avx_64_1_ret 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 64, 1, 1 +%endm + +%macro memcpy_avx_128_ret 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 128, 1, 1 +%endm + +%macro memcpy_avx_128_1_ret 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 128, 1, 1 +%endm + + +%macro memcpy_avx_16 5 + __memcpy_int %1,%2,%3,%4,%5,,,,, 0, 16, 0, 1 +%endm + +%macro memcpy_avx_16_1 5 + __memcpy_int %1,%2,%3,%4,%5,,,,, 1, 16, 0, 1 +%endm + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%macro memcpy_avx2_64 7 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,--,--, 0, 64, 0, 2 +%endm + +%macro memcpy_avx2_64_1 7 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,--,--, 1, 64, 0, 2 +%endm + +%macro memcpy_avx2_128 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7, %8, %9, 0, 128, 0, 2 +%endm + +%macro memcpy_avx2_128_1 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7, %8, %9, 1, 128, 0, 2 +%endm + +%macro memcpy_avx2_64_ret 7 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,--,--, 0, 64, 1, 2 +%endm + +%macro memcpy_avx2_64_1_ret 7 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,--,--, 1, 64, 1, 2 +%endm + +%macro memcpy_avx2_128_ret 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 128, 1, 2 +%endm + +%macro memcpy_avx2_128_1_ret 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 128, 1, 2 +%endm + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + +%macro __memcpy_int 13 +%define %%DST %1 ; register: pointer to dst (not modified) +%define %%SRC %2 ; register: pointer to src (not modified) +%define %%SIZE %3 ; register: length in bytes (not modified) +%define %%TMP0 %4 ; 64-bit temp GPR (clobbered) +%define %%TMP1 %5 ; 64-bit temp GPR (clobbered) +%define %%XTMP0 %6 ; temp XMM (clobbered) +%define %%XTMP1 %7 ; temp XMM (clobbered) +%define %%XTMP2 %8 ; temp XMM (clobbered) +%define %%XTMP3 %9 ; temp XMM (clobbered) +%define %%NOT0 %10 ; if not 0, then assume size cannot be zero +%define %%MAXSIZE %11 ; 128, 64, etc +%define %%USERET %12 ; if not 0, use "ret" at end +%define %%USEAVX %13 ; 0 = SSE, 1 = AVX1, 2 = AVX2 + +%if (%%USERET != 0) + %define %%DONE ret +%else + %define %%DONE jmp %%end +%endif + +%if (%%USEAVX != 0) + %define %%MOVDQU vmovdqu +%else + %define %%MOVDQU movdqu +%endif + +%if (%%MAXSIZE >= 128) + test %%SIZE, 64 + jz %%lt64 + %if (%%USEAVX >= 2) + %%MOVDQU %%XTMP0, [%%SRC + 0*32] + %%MOVDQU %%XTMP1, [%%SRC + 1*32] + %%MOVDQU %%XTMP2, [%%SRC + %%SIZE - 2*32] + %%MOVDQU %%XTMP3, [%%SRC + %%SIZE - 1*32] + + %%MOVDQU [%%DST + 0*32], %%XTMP0 + %%MOVDQU [%%DST + 1*32], %%XTMP1 + %%MOVDQU [%%DST + %%SIZE - 2*32], %%XTMP2 + %%MOVDQU [%%DST + %%SIZE - 1*32], %%XTMP3 + %else + %%MOVDQU %%XTMP0, [%%SRC + 0*16] + %%MOVDQU %%XTMP1, [%%SRC + 1*16] + %%MOVDQU %%XTMP2, [%%SRC + 2*16] + %%MOVDQU %%XTMP3, [%%SRC + 3*16] + %%MOVDQU [%%DST + 0*16], %%XTMP0 + %%MOVDQU [%%DST + 1*16], %%XTMP1 + %%MOVDQU [%%DST + 2*16], %%XTMP2 + %%MOVDQU [%%DST + 3*16], %%XTMP3 + + %%MOVDQU %%XTMP0, [%%SRC + %%SIZE - 4*16] + %%MOVDQU %%XTMP1, [%%SRC + %%SIZE - 3*16] + %%MOVDQU %%XTMP2, [%%SRC + %%SIZE - 2*16] + %%MOVDQU %%XTMP3, [%%SRC + %%SIZE - 1*16] + %%MOVDQU [%%DST + %%SIZE - 4*16], %%XTMP0 + %%MOVDQU [%%DST + %%SIZE - 3*16], %%XTMP1 + %%MOVDQU [%%DST + %%SIZE - 2*16], %%XTMP2 + %%MOVDQU [%%DST + %%SIZE - 1*16], %%XTMP3 + %endif + %%DONE +%endif + +%if (%%MAXSIZE >= 64) +%%lt64: + test %%SIZE, 32 + jz %%lt32 + %if (%%USEAVX >= 2) + %%MOVDQU %%XTMP0, [%%SRC + 0*32] + %%MOVDQU %%XTMP1, [%%SRC + %%SIZE - 1*32] + %%MOVDQU [%%DST + 0*32], %%XTMP0 + %%MOVDQU [%%DST + %%SIZE - 1*32], %%XTMP1 + %else + %%MOVDQU %%XTMP0, [%%SRC + 0*16] + %%MOVDQU %%XTMP1, [%%SRC + 1*16] + %%MOVDQU %%XTMP2, [%%SRC + %%SIZE - 2*16] + %%MOVDQU %%XTMP3, [%%SRC + %%SIZE - 1*16] + %%MOVDQU [%%DST + 0*16], %%XTMP0 + %%MOVDQU [%%DST + 1*16], %%XTMP1 + %%MOVDQU [%%DST + %%SIZE - 2*16], %%XTMP2 + %%MOVDQU [%%DST + %%SIZE - 1*16], %%XTMP3 + %endif + %%DONE +%endif + +%if (%%MAXSIZE >= 32) +%%lt32: + test %%SIZE, 16 + jz %%lt16 + %if (%%USEAVX >= 2) + %%MOVDQU XWORD(%%XTMP0), [%%SRC + 0*16] + %%MOVDQU XWORD(%%XTMP1), [%%SRC + %%SIZE - 1*16] + %%MOVDQU [%%DST + 0*16], XWORD(%%XTMP0) + %%MOVDQU [%%DST + %%SIZE - 1*16], XWORD(%%XTMP1) + %else + %%MOVDQU %%XTMP0, [%%SRC + 0*16] + %%MOVDQU %%XTMP1, [%%SRC + %%SIZE - 1*16] + %%MOVDQU [%%DST + 0*16], %%XTMP0 + %%MOVDQU [%%DST + %%SIZE - 1*16], %%XTMP1 + %endif + %%DONE +%endif + +%if (%%MAXSIZE >= 16) +%%lt16: + test %%SIZE, 8 + jz %%lt8 + mov %%TMP0, [%%SRC] + mov %%TMP1, [%%SRC + %%SIZE - 8] + mov [%%DST], %%TMP0 + mov [%%DST + %%SIZE - 8], %%TMP1 + %%DONE +%endif + +%if (%%MAXSIZE >= 8) +%%lt8: + test %%SIZE, 4 + jz %%lt4 + mov DWORD(%%TMP0), [%%SRC] + mov DWORD(%%TMP1), [%%SRC + %%SIZE - 4] + mov [%%DST], DWORD(%%TMP0) + mov [%%DST + %%SIZE - 4], DWORD(%%TMP1) + %%DONE +%endif + +%if (%%MAXSIZE >= 4) +%%lt4: + test %%SIZE, 2 + jz %%lt2 + movzx DWORD(%%TMP0), word [%%SRC] + movzx DWORD(%%TMP1), byte [%%SRC + %%SIZE - 1] + mov [%%DST], WORD(%%TMP0) + mov [%%DST + %%SIZE - 1], BYTE(%%TMP1) + %%DONE +%endif + +%%lt2: +%if (%%NOT0 == 0) + test %%SIZE, 1 + jz %%end +%endif + movzx DWORD(%%TMP0), byte [%%SRC] + mov [%%DST], BYTE(%%TMP0) +%%end: +%if (%%USERET != 0) + ret +%endif +%endm + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Utility macro to assist with SIMD shifting +%macro _PSRLDQ 3 +%define %%VEC %1 +%define %%REG %2 +%define %%IMM %3 + +%ifidn %%VEC, SSE + psrldq %%REG, %%IMM +%else + vpsrldq %%REG, %%REG, %%IMM +%endif +%endm + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; This section defines a series of macros to store small to medium amounts +; of data from SIMD registers to memory, where the size is variable but limited. +; +; The macros are all called as: +; memcpy DST, SRC, SIZE, TMP, IDX +; with the parameters defined as: +; DST : register: pointer to dst (not modified) +; SRC : register: src data (clobbered) +; SIZE : register: length in bytes (not modified) +; TMP : 64-bit temp GPR (clobbered) +; IDX : 64-bit GPR to store dst index/offset (clobbered) +; +; The name indicates the options. The name is of the form: +; simd_store_ +; where is the SIMD instruction type e.g. "sse" or "avx" + + +%macro simd_store_sse 5 + __simd_store %1,%2,%3,%4,%5,SSE +%endm + +%macro simd_store_avx 5 + __simd_store %1,%2,%3,%4,%5,AVX +%endm + +%macro simd_store_sse_15 5 + __simd_store %1,%2,%3,%4,%5,SSE,15 +%endm + +%macro simd_store_avx_15 5 + __simd_store %1,%2,%3,%4,%5,AVX,15 +%endm + +%macro __simd_store 6-7 +%define %%DST %1 ; register: pointer to dst (not modified) +%define %%SRC %2 ; register: src data (clobbered) +%define %%SIZE %3 ; register: length in bytes (not modified) +%define %%TMP %4 ; 64-bit temp GPR (clobbered) +%define %%IDX %5 ; 64-bit temp GPR to store dst idx (clobbered) +%define %%SIMDTYPE %6 ; "SSE" or "AVX" +%define %%MAX_LEN %7 ; [optional] maximum length to be stored, default 16 + +%define %%PSRLDQ _PSRLDQ %%SIMDTYPE, + +%ifidn %%SIMDTYPE, SSE + %define %%MOVDQU movdqu + %define %%MOVQ movq +%else + %define %%MOVDQU vmovdqu + %define %%MOVQ vmovq +%endif + +;; determine max byte size for store operation +%if %0 > 6 +%assign max_length_to_store %%MAX_LEN +%else +%assign max_length_to_store 16 +%endif + +%if max_length_to_store > 16 +%error "__simd_store macro invoked with MAX_LEN bigger than 16!" +%endif + + xor %%IDX, %%IDX ; zero idx + +%if max_length_to_store == 16 + test %%SIZE, 16 + jz %%lt16 + %%MOVDQU [%%DST], %%SRC + jmp %%end +%%lt16: +%endif + +%if max_length_to_store >= 8 + test %%SIZE, 8 + jz %%lt8 + %%MOVQ [%%DST + %%IDX], %%SRC + %%PSRLDQ %%SRC, 8 + add %%IDX, 8 +%%lt8: +%endif + + %%MOVQ %%TMP, %%SRC ; use GPR from now on + +%if max_length_to_store >= 4 + test %%SIZE, 4 + jz %%lt4 + mov [%%DST + %%IDX], DWORD(%%TMP) + shr %%TMP, 32 + add %%IDX, 4 +%%lt4: +%endif + + test %%SIZE, 2 + jz %%lt2 + mov [%%DST + %%IDX], WORD(%%TMP) + shr %%TMP, 16 + add %%IDX, 2 +%%lt2: + test %%SIZE, 1 + jz %%end + mov [%%DST + %%IDX], BYTE(%%TMP) +%%end: +%endm + +; This section defines a series of macros to load small to medium amounts +; (from 0 to 16 bytes) of data from memory to SIMD registers, +; where the size is variable but limited. +; +; The macros are all called as: +; simd_load DST, SRC, SIZE +; with the parameters defined as: +; DST : register: destination XMM register +; SRC : register: pointer to src data (not modified) +; SIZE : register: length in bytes (not modified) +; +; The name indicates the options. The name is of the form: +; simd_load__ +; where: +; is either "sse" or "avx" +; is either "15" or "16" and defines largest value of SIZE +; is blank or "_1". If "_1" then the min SIZE is 1 (otherwise 0) +; +; For example: +; simd_load_sse_16 : SSE, 0 <= size <= 16 +; simd_load_avx_15_1 : AVX, 1 <= size <= 15 + +%macro simd_load_sse_15_1 3 + __simd_load %1,%2,%3,0,0,SSE +%endm +%macro simd_load_sse_15 3 + __simd_load %1,%2,%3,1,0,SSE +%endm +%macro simd_load_sse_16_1 3 + __simd_load %1,%2,%3,0,1,SSE +%endm +%macro simd_load_sse_16 3 + __simd_load %1,%2,%3,1,1,SSE +%endm + +%macro simd_load_avx_15_1 3 + __simd_load %1,%2,%3,0,0,AVX +%endm +%macro simd_load_avx_15 3 + __simd_load %1,%2,%3,1,0,AVX +%endm +%macro simd_load_avx_16_1 3 + __simd_load %1,%2,%3,0,1,AVX +%endm +%macro simd_load_avx_16 3 + __simd_load %1,%2,%3,1,1,AVX +%endm + +%macro __simd_load 6 +%define %%DST %1 ; [out] destination XMM register +%define %%SRC %2 ; [in] pointer to src data +%define %%SIZE %3 ; [in] length in bytes (0-16 bytes) +%define %%ACCEPT_0 %4 ; 0 = min length = 1, 1 = min length = 0 +%define %%ACCEPT_16 %5 ; 0 = max length = 15 , 1 = max length = 16 +%define %%SIMDTYPE %6 ; "SSE" or "AVX" + +%ifidn %%SIMDTYPE, SSE + %define %%MOVDQU movdqu + %define %%PINSRB pinsrb + %define %%PINSRQ pinsrq + %define %%PXOR pxor +%else + %define %%MOVDQU vmovdqu + %define %%PINSRB vpinsrb + %define %%PINSRQ vpinsrq + %define %%PXOR vpxor +%endif + +%if (%%ACCEPT_16 != 0) + test %%SIZE, 16 + jz %%_skip_16 + %%MOVDQU %%DST, [%%SRC] + jmp %%end_load + +%%_skip_16: +%endif + %%PXOR %%DST, %%DST ; clear XMM register +%if (%%ACCEPT_0 != 0) + or %%SIZE, %%SIZE + je %%end_load +%endif + cmp %%SIZE, 1 + je %%_size_1 + cmp %%SIZE, 2 + je %%_size_2 + cmp %%SIZE, 3 + je %%_size_3 + cmp %%SIZE, 4 + je %%_size_4 + cmp %%SIZE, 5 + je %%_size_5 + cmp %%SIZE, 6 + je %%_size_6 + cmp %%SIZE, 7 + je %%_size_7 + cmp %%SIZE, 8 + je %%_size_8 + cmp %%SIZE, 9 + je %%_size_9 + cmp %%SIZE, 10 + je %%_size_10 + cmp %%SIZE, 11 + je %%_size_11 + cmp %%SIZE, 12 + je %%_size_12 + cmp %%SIZE, 13 + je %%_size_13 + cmp %%SIZE, 14 + je %%_size_14 + +%%_size_15: + %%PINSRB %%DST, [%%SRC + 14], 14 +%%_size_14: + %%PINSRB %%DST, [%%SRC + 13], 13 +%%_size_13: + %%PINSRB %%DST, [%%SRC + 12], 12 +%%_size_12: + %%PINSRB %%DST, [%%SRC + 11], 11 +%%_size_11: + %%PINSRB %%DST, [%%SRC + 10], 10 +%%_size_10: + %%PINSRB %%DST, [%%SRC + 9], 9 +%%_size_9: + %%PINSRB %%DST, [%%SRC + 8], 8 +%%_size_8: + %%PINSRQ %%DST, [%%SRC], 0 + jmp %%end_load +%%_size_7: + %%PINSRB %%DST, [%%SRC + 6], 6 +%%_size_6: + %%PINSRB %%DST, [%%SRC + 5], 5 +%%_size_5: + %%PINSRB %%DST, [%%SRC + 4], 4 +%%_size_4: + %%PINSRB %%DST, [%%SRC + 3], 3 +%%_size_3: + %%PINSRB %%DST, [%%SRC + 2], 2 +%%_size_2: + %%PINSRB %%DST, [%%SRC + 1], 1 +%%_size_1: + %%PINSRB %%DST, [%%SRC + 0], 0 +%%end_load: +%endm +%endif ; ifndef __MEMCPY_ASM__ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/os.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/os.inc new file mode 100644 index 0000000..38c6586 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/os.inc @@ -0,0 +1,40 @@ +;=============================================================================== +; Copyright (C) 2020 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +%ifndef OS_ASM_FILE +%define OS_ASM_FILE + +%ifndef WIN_ABI +%ifidn __OUTPUT_FORMAT__, win64 +%define WIN_ABI +%endif +%endif + +%ifndef LINUX +%ifidn __OUTPUT_FORMAT__, elf64 +%define LINUX +%endif +%endif + +;; code is the same for linux and macos +%ifndef LINUX +%ifidn __OUTPUT_FORMAT__, macho64 +%define LINUX +%endif +%endif + +%endif ; OS_ASM_FILE diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpaesgcme9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpaesgcme9as.asm new file mode 100644 index 0000000..5cf631d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpaesgcme9as.asm @@ -0,0 +1,931 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; AES-GCM function +; +; Content: +; AesGcmPrecompute_avx() +; AesGcmMulGcm_avx() +; AesGcmAuth_avx() +; AesGcmEnc_avx() +; AesGcmDec_avx() +; + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%assign my_emulator 0; set 1 for emulation +%include "emulator.inc" + + +; sse_clmul_gcm MACRO to implement: Data*HashKey mod (128,127,126,121,0) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%assign REFLECTED_IN_OUT 1 ; GCM input and output are represented in reflected form naturaly +;USE_BYTE_REFLECTED = 0 ; set to 1 %if input data is byte-reflected + + +%macro sse_clmul_gcm_2way 11.nolist + %xdefine %%HK %1 + %xdefine %%HKforKaratsuba %2 + %xdefine %%GH_1 %3 + %xdefine %%tmpX0_1 %4 + %xdefine %%tmpX1_1 %5 + %xdefine %%tmpX2_1 %6 + %xdefine %%GH_2 %7 + %xdefine %%tmpX0_2 %8 + %xdefine %%tmpX1_2 %9 + %xdefine %%tmpX2_2 %10 + %xdefine %%tmpX3_2 %11 + +pshufd %%tmpX2_1, %%GH_1, 01001110b + + ;; Karatsuba Method + movdqa %%tmpX1_1, %%GH_1 + pxor %%tmpX2_1, %%GH_1 +my_pclmulqdq %%GH_1, %%HK, 00h + + pshufd %%tmpX2_2, %%GH_2, 01001110b + + ;; Karatsuba Method + movdqa %%tmpX1_2, %%GH_2 + pxor %%tmpX2_2, %%GH_2 +my_pclmulqdq %%tmpX1_1, %%HK, 11h +my_pclmulqdq %%tmpX2_1, %%HKforKaratsuba, 00h + pxor %%tmpX2_1, %%GH_1 + pxor %%tmpX2_1, %%tmpX1_1 + + pshufd %%tmpX0_1, %%tmpX2_1, 78 + movdqa %%tmpX2_1, %%tmpX0_1 + pand %%tmpX0_1, oword [rel MASK2] + pand %%tmpX2_1, xmm14 + pxor %%GH_1, %%tmpX0_1 + pxor %%tmpX1_1, %%tmpX2_1 + + ;first phase of the reduction + movdqa %%tmpX0_1, %%GH_1 + + psllq %%GH_1, 1 + +my_pclmulqdq %%GH_2, %%HK, 00h + pxor %%GH_1, %%tmpX0_1 + psllq %%GH_1, 5 + pxor %%GH_1, %%tmpX0_1 + psllq %%GH_1, 57 + + pshufd %%tmpX2_1, %%GH_1, 78 + movdqa %%GH_1, %%tmpX2_1 + +my_pclmulqdq %%tmpX1_2, %%HK, 11h + pand %%tmpX2_1, xmm14 + pand %%GH_1, oword [rel MASK2] + pxor %%GH_1, %%tmpX0_1 + pxor %%tmpX1_1, %%tmpX2_1 + + ;second phase of the reduction + movdqa %%tmpX2_1, %%GH_1 + psrlq %%GH_1, 5 + +my_pclmulqdq %%tmpX2_2, %%HKforKaratsuba, 00h + pxor %%GH_1, %%tmpX2_1 + psrlq %%GH_1, 1 + pxor %%GH_1, %%tmpX2_1 + psrlq %%GH_1, 1 + + pxor %%GH_1, %%tmpX2_1 + pxor %%GH_1, %%tmpX1_1 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + pxor %%tmpX2_2, %%GH_2 + pxor %%tmpX2_2, %%tmpX1_2 + + pshufd %%tmpX0_2, %%tmpX2_2, 78 + movdqa %%tmpX2_2, %%tmpX0_2 + pand %%tmpX0_2, oword [rel MASK2] + pand %%tmpX2_2, xmm14 + pxor %%GH_2, %%tmpX0_2 + pxor %%tmpX1_2, %%tmpX2_2 + + ;first phase of the reduction + movdqa %%tmpX0_2, %%GH_2 + movdqa %%tmpX2_2, %%GH_2 + movdqa %%tmpX3_2, %%GH_2 ; move GH_2 into tmpX0_2, tmpX2_2, tmpX3_2 + + psllq %%tmpX0_2, 63 ; packed left shifting << 63 + psllq %%tmpX2_2, 62 ; packed left shifting shift << 62 + psllq %%tmpX3_2, 57 ; packed left shifting shift << 57 + pxor %%tmpX0_2, %%tmpX2_2 ; xor the shifted versions + pxor %%tmpX0_2, %%tmpX3_2 + + movdqa %%tmpX2_2, %%tmpX0_2 + pslldq %%tmpX2_2, 8 ; shift-L tmpX2_2 2 DWs + psrldq %%tmpX0_2, 8 ; shift-R xmm2 2 DWs + + pxor %%GH_2, %%tmpX2_2 ; first phase of the reduction complete + pxor %%tmpX1_2, %%tmpX0_2 ; save the lost MS 1-2-7 bits from first phase + + ;second phase of the reduction + movdqa %%tmpX2_2, %%GH_2 ; move GH_2 into tmpX3_2 + psrlq %%tmpX2_2, 5 ; packed right shifting >> 5 + pxor %%tmpX2_2, %%GH_2 ; xor shifted versions + psrlq %%tmpX2_2, 1 ; packed right shifting >> 1 + pxor %%tmpX2_2, %%GH_2 ; xor shifted versions + psrlq %%tmpX2_2, 1 ; packed right shifting >> 1 + + pxor %%GH_2, %%tmpX2_2 ; second phase of the reduction complete + pxor %%GH_2, %%tmpX1_2 ; the result is in GH_2 +%endmacro + +%macro sse_clmul_gcm 5.nolist + %xdefine %%GH %1 + %xdefine %%HK %2 + %xdefine %%tmpX0 %3 + %xdefine %%tmpX1 %4 + %xdefine %%tmpX2 %5 + +;; GH, HK hold the values for the two operands which are carry-less multiplied + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; Karatsuba Method + ;; + ;; GH = [GH1:GH0] + ;; HK = [HK1:HK0] + ;; + pshufd %%tmpX2, %%GH, 01001110b ;; xmm2 = {GH0:GH1} + pshufd %%tmpX0, %%HK, 01001110b ;; xmm0 = {HK0:HK1} + pxor %%tmpX2, %%GH ;; xmm2 = {GH0+GH1:GH1+GH0} + pxor %%tmpX0, %%HK ;; xmm0 = {HK0+HK1:HK1+HK0} + +my_pclmulqdq %%tmpX2, %%tmpX0,00h ;; tmpX2 = (a1+a0)*(b1+b0) xmm2 = (GH1+GH0)*(HK1+HK0) + movdqa %%tmpX1, %%GH +my_pclmulqdq %%GH, %%HK, 00h ;; GH = a0*b0 GH = GH0*HK0 + pxor %%tmpX0, %%tmpX0 +my_pclmulqdq %%tmpX1, %%HK, 11h ;; tmpX1 = a1*b1 xmm1 = GH1*HK1 + pxor %%tmpX2, %%GH ;; xmm2 = (GH1+GH0)*(HK1+HK0) + GH0*HK0 + pxor %%tmpX2, %%tmpX1 ;; tmpX2 = a0*b1+a1*b0 xmm2 = (GH1+GH0)*(HK1+HK0) + GH0*HK0 + GH1*HK1 = GH0*HK1+GH1*HK0 + + palignr %%tmpX0, %%tmpX2, 8 ;; tmpX0 = {Zeros : HI(a0*b1+a1*b0)} + pslldq %%tmpX2, 8 ;; tmpX2 = {LO(HI(a0*b1+a1*b0)) : Zeros} + pxor %%tmpX1, %%tmpX0 ;; holds the result of the carry-less multiplication of GH by HK + pxor %%GH, %%tmpX2 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;first phase of the reduction: + ; Most( (product_H * g1), 128)) product_H = GH + ; g1 = 2^256/g = g = 1+x+x^2+x^7+x^128 + ; + movdqa %%tmpX0, %%GH ;; copy GH into tmpX0, tmpX2, xmm15 + movdqa %%tmpX2, %%GH + movdqa xmm15, %%GH + + psllq %%tmpX0, 63 ; packed left shifting << 63 + psllq %%tmpX2, 62 ; packed left shifting shift << 62 + psllq xmm15, 57 ; packed left shifting shift << 57 + pxor %%tmpX0, %%tmpX2 ; xor the shifted versions + pxor %%tmpX0, xmm15 + + movdqa %%tmpX2, %%tmpX0 + pslldq %%tmpX2, 8 ; shift-L tmpX2 2 DWs + psrldq %%tmpX0, 8 ; shift-R xmm2 2 DWs + + pxor %%GH, %%tmpX2 ; first phase of the reduction complete + pxor %%tmpX1, %%tmpX0 ; save the lost MS 1-2-7 bits from first phase + + ;second phase of the reduction + movdqa %%tmpX2, %%GH ; move GH into xmm15 + psrlq %%tmpX2, 5 ; packed right shifting >> 5 + pxor %%tmpX2, %%GH ; xor shifted versions + psrlq %%tmpX2, 1 ; packed right shifting >> 1 + pxor %%tmpX2, %%GH ; xor shifted versions + psrlq %%tmpX2, 1 ; packed right shifting >> 1 + + pxor %%GH, %%tmpX2 ; second phase of the reduction complete + pxor %%GH, %%tmpX1 ; the result is in GH +%endmacro + +%if (_IPP32E >= _IPP32E_Y8) + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR +POLY DQ 00000000000000001h,0C200000000000000h ;; 0xC2000000000000000000000000000001 +TWOONE DQ 00000000000000001h,00000000100000000h ;; 0x00000001000000000000000000000001 +SHUF_CONST DQ 008090a0b0c0d0e0fh,00001020304050607h ;; 0x000102030405060708090a0b0c0d0e0f +MASK1 DQ 0ffffffffffffffffh,00000000000000000h ;; 0x0000000000000000ffffffffffffffff +MASK2 DQ 00000000000000000h,0ffffffffffffffffh ;; 0xffffffffffffffff0000000000000000 +INC_1 DQ 1,0 + +%assign sizeof_oword_ (16) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; +;;; void AesGcmPrecomute_avx(Ipp8u* pPrecomutedData, const Ipp8u* pHKey) +;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM AesGcmPrecompute_avx,PUBLIC + USES_GPR rdi,rsi + USES_XMM xmm6,xmm7,xmm8,xmm9,xmm10,xmm11,xmm12,xmm13,xmm14,xmm15 + COMP_ABI 2 + +%xdefine pPrecomData rdi ; (rdi) pointer to the reflected multipliers reflect(hkey),(hkey<<1), (hkey^2)<<1, (hkey^4)<<1, +%xdefine pHKey rsi ; (rsi) pointer to the Hkey value + + movdqu xmm0, oword [rel pHKey] ; xmm0 holds HashKey + pshufb xmm0, [rel SHUF_CONST] + ;movdqu oword [pPrecomData+sizeof_oword_*0], xmm0 + + ; precompute HashKey<<1 mod poly from the HashKey + movdqa xmm4, xmm0 + psllq xmm0, 1 + psrlq xmm4, 63 + movdqa xmm3, xmm4 + pslldq xmm4, 8 + psrldq xmm3, 8 + por xmm0, xmm4 + ;reduction + pshufd xmm4, xmm3, 00100100b + pcmpeqd xmm4, oword [rel TWOONE] ; [TWOONE] = 0x00000001000000000000000000000001 + pand xmm4, oword [rel POLY] + pxor xmm0, xmm4 ; xmm0 holds the HashKey<<1 mod poly + + movdqa xmm1, xmm0 + sse_clmul_gcm xmm1, xmm0, xmm3, xmm4, xmm5 ; xmm1 holds (HashKey^2)<<1 mod poly + + movdqa xmm2, xmm1 + sse_clmul_gcm xmm2, xmm1, xmm3, xmm4, xmm5 ; xmm2 holds (HashKey^4)<<1 mod poly + + movdqu oword [pPrecomData+sizeof_oword_*0], xmm0 + movdqu oword [pPrecomData+sizeof_oword_*1], xmm1 + movdqu oword [pPrecomData+sizeof_oword_*2], xmm2 + + REST_XMM + REST_GPR + ret +ENDFUNC AesGcmPrecompute_avx + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; +;;; void AesGcmMulGcm_avx(Ipp8u* pHash, const Ipp8u* pHKey) +;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM AesGcmMulGcm_avx,PUBLIC + USES_GPR rsi,rdi + USES_XMM xmm15 + COMP_ABI 2 + +%xdefine pHash rdi ; (rdi) pointer to the Hash value +%xdefine pHKey rsi ; (rsi) pointer to the (hkey<<1) value + + movdqa xmm0, oword [rel pHash] + pshufb xmm0, [rel SHUF_CONST] + movdqa xmm1, oword [rel pHKey] + + sse_clmul_gcm xmm0, xmm1, xmm2, xmm3, xmm4 ; xmm0 holds Hash*HKey mod poly + + pshufb xmm0, [rel SHUF_CONST] + movdqa oword [rel pHash], xmm0 + + REST_XMM + REST_GPR + ret +ENDFUNC AesGcmMulGcm_avx + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; +;;; void AesGcmAuth_avx(Ipp8u* pHash, const Ipp8u* pSrc, int len, const Ipp8u* pHKey, const void* pParam +;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM AesGcmAuth_avx,PUBLIC + USES_GPR rsi,rdi + USES_XMM xmm15 + COMP_ABI 5 + +%xdefine pHash rdi ; (rdi) pointer to the Hash value +%xdefine pSrc rsi ; (rsi) pointer to the input data +%xdefine len rdx ; (rdx) length of data (multiplr by AES_DATA_BLOCK) +%xdefine pHKey rcx ; (rcx) pointer to the (hkey<<1) value + +%assign BYTES_PER_BLK (16) + + movdqa xmm0, oword [rel pHash] + pshufb xmm0, [rel SHUF_CONST] + movdqa xmm1, oword [rel pHKey] + + movsxd rdx, edx + +align IPP_ALIGN_FACTOR +.auth_loop: + movdqu xmm2, oword [rel pSrc] ; src[] + pshufb xmm2, [rel SHUF_CONST] + add pSrc, BYTES_PER_BLK + pxor xmm0, xmm2 ; hash ^= src[] + + sse_clmul_gcm xmm0, xmm1, xmm2, xmm3, xmm4 ; xmm0 holds Hash*HKey mod poly + + sub len, BYTES_PER_BLK + jnz .auth_loop + + pshufb xmm0, [rel SHUF_CONST] + movdqa oword [pHash], xmm0 + + REST_XMM + REST_GPR + ret +ENDFUNC AesGcmAuth_avx + +;*************************************************************** +;* Purpose: pipelined AES-GCM encryption +;* +;* void AesGcmEnc_avx(Ipp8u* pDst, +;* const Ipp8u* pSrc, +;* int length, +;* RijnCipher cipher, +;* int nr, +;* const Ipp8u* pRKey, +;* Ipp8u* pGhash, +;* Ipp8u* pCtrValue, +;* Ipp8u* pEncCtrValue, +;* const Ipp8u* pPrecomData) +;*************************************************************** +align IPP_ALIGN_FACTOR +;; +;; Lib = Y8, E9 +;; +;; Caller = ippsRijndael128GCMEncrypt +;; +IPPASM AesGcmEnc_avx,PUBLIC +%assign LOCAL_FRAME (8*16) + USES_GPR rsi,rdi,rbx + USES_XMM xmm6,xmm7,xmm8,xmm9,xmm10,xmm11,xmm12,xmm13,xmm14,xmm15 + COMP_ABI 10 + +%xdefine pDst rdi ; pointer to the encrypted data +%xdefine pSrc rsi ; pointer to the plane data +%xdefine len rdx ; data length in bytes (multiple by BYTES_PER_BLK) +%xdefine cipher rcx ; ciper function (don't need in fact) +%xdefine nr r8d ; number of cipher's rounds +%xdefine pRKey r9 ; pointer to the cipher's round keys +%xdefine pGhash [rsp+ARG_7] ; pointer to the Hash value +%xdefine pCtrValue [rsp+ARG_8] ; pointer to the counter value +%xdefine pEncCtrValue [rsp+ARG_9] ; pointer to the encrypted counter +%xdefine pPrecomData [rsp+ARG_10]; pointer to the precomputed data + +%assign SC (4) +%assign BLKS_PER_LOOP (4) +%assign BYTES_PER_BLK (16) +%assign BYTES_PER_LOOP (BYTES_PER_BLK*BLKS_PER_LOOP) + +;; +;; stack structure: +%assign CNT (0) +%assign ECNT (CNT+sizeof_oword_) +%assign GHASH (ECNT+sizeof_oword_) + +%assign GHASH0 (GHASH) +%assign GHASH1 (GHASH0+sizeof_oword_) +%assign GHASH2 (GHASH1+sizeof_oword_) +%assign GHASH3 (GHASH2+sizeof_oword_) +%assign HKeyKaratsuba (GHASH3+sizeof_oword_) + + mov rax, pCtrValue ; address of the counter + mov rbx, pEncCtrValue ; address of the encrypted counter + mov rcx, pGhash ; address of hash value + movdqa xmm4,oword [rel SHUF_CONST] + + movdqu xmm0, oword [rax] ; counter value + movdqu xmm1, oword [rbx] ; encrypted counter value + movdqu xmm2, oword [rcx] ; hash value + + pshufb xmm0, xmm4 ; convert counter and + movdqa oword [rsp+CNT], xmm0 ; store into the stack + movdqa oword [rsp+ECNT], xmm1 ; store encrypted counter into the stack + + pshufb xmm2, xmm4 ; convert hash value + pxor xmm1, xmm1 + movdqa oword [rsp+GHASH0], xmm2 ; store hash into the stack + movdqa oword [rsp+GHASH1], xmm1 ; + movdqa oword [rsp+GHASH2], xmm1 ; + movdqa oword [rsp+GHASH3], xmm1 ; + + mov rbx, [rsp+ARG_10] ; pointer to the {hk<<1,hk^2<<1,kh^4<<1} multipliers + movdqa xmm10,oword [rbx+sizeof_oword_*2] + pshufd xmm9, xmm10, 01001110b ; xmm9 holds qword-swapped version of (HashKey^4)<<1 mod poly for Karatsuba + pxor xmm9, xmm10 + movdqa oword [rsp+HKeyKaratsuba], xmm9 + + movsxd len, edx ; data length + mov rcx, pRKey ; rcx point to the chipher's round keys + + mov rax, len + and rax, BYTES_PER_LOOP-1 + and len, -BYTES_PER_LOOP + jz .single_block_proc + +;; +;; pipelined processing (4 blocks) +;; +align IPP_ALIGN_FACTOR +.blks4_loop: + ;; + ;; ctr encryption + ;; + movdqa xmm6,oword [rel INC_1] + movdqa xmm5,oword [rel SHUF_CONST] + + movdqa xmm1, xmm0 ; counter+1 + paddd xmm1, xmm6 + movdqa xmm2, xmm1 ; counter+2 + paddd xmm2, xmm6 + movdqa xmm3, xmm2 ; counter+3 + paddd xmm3, xmm6 + movdqa xmm4, xmm3 ; counter+4 + paddd xmm4, xmm6 + movdqa oword [rsp+CNT], xmm4 + + movdqa xmm0, oword [rcx] ; pre-load whitening keys + mov r10, rcx + + pshufb xmm1, xmm5 ; counter, counter+1, counter+2, counter+3 + pshufb xmm2, xmm5 ; ready to be encrypted + pshufb xmm3, xmm5 + pshufb xmm4, xmm5 + + pxor xmm1, xmm0 ; whitening + pxor xmm2, xmm0 + pxor xmm3, xmm0 + pxor xmm4, xmm0 + + movdqa xmm0, oword [r10+16] + add r10, 16 + + mov r11d, nr ; counter depending on key length + sub r11, 1 + +align IPP_ALIGN_FACTOR +.cipher4_loop: +my_aesenc xmm1, xmm0 ; regular round +my_aesenc xmm2, xmm0 +my_aesenc xmm3, xmm0 +my_aesenc xmm4, xmm0 + movdqa xmm0, oword [r10+16] + add r10, 16 + dec r11 + jnz .cipher4_loop +my_aesenclast xmm1, xmm0 +my_aesenclast xmm2, xmm0 +my_aesenclast xmm3, xmm0 +my_aesenclast xmm4, xmm0 + + movdqa xmm0, oword [rsp+ECNT] ; load pre-calculated encrypted counter + movdqa oword [rsp+ECNT], xmm4 ; save encrypted counter+4 + + movdqu xmm4, oword [pSrc+0*BYTES_PER_BLK] ; 4 input blocks + movdqu xmm5, oword [pSrc+1*BYTES_PER_BLK] + movdqu xmm6, oword [pSrc+2*BYTES_PER_BLK] + movdqu xmm7, oword [pSrc+3*BYTES_PER_BLK] + add pSrc, BYTES_PER_LOOP + + pxor xmm0, xmm4 ; ctr encryption + movdqu oword [pDst+0*BYTES_PER_BLK], xmm0 ; store result + pshufb xmm0, [rel SHUF_CONST] ; convert for multiplication and + pxor xmm0, oword [rsp+GHASH0] + + pxor xmm1, xmm5 + movdqu oword [pDst+1*BYTES_PER_BLK], xmm1 + pshufb xmm1, [rel SHUF_CONST] + pxor xmm1, oword [rsp+GHASH1] + + pxor xmm2, xmm6 + movdqu oword [pDst+2*BYTES_PER_BLK], xmm2 + pshufb xmm2, [rel SHUF_CONST] + pxor xmm2, oword [rsp+GHASH2] + + pxor xmm3, xmm7 + movdqu oword [pDst+3*BYTES_PER_BLK], xmm3 + pshufb xmm3, [rel SHUF_CONST] + pxor xmm3, oword [rsp+GHASH3] + + add pDst, BYTES_PER_LOOP + + cmp len, BYTES_PER_LOOP + je .combine_hash + + ;; + ;; update hash value + ;; + movdqa xmm14, oword [rel MASK1] + sse_clmul_gcm_2way xmm10, xmm9, xmm0, xmm4, xmm5, xmm6, xmm1, xmm11, xmm12, xmm13, xmm15 + sse_clmul_gcm_2way xmm10, xmm9, xmm2, xmm4, xmm5, xmm6, xmm3, xmm11, xmm12, xmm13, xmm15 + + movdqa oword [rsp+GHASH0], xmm0 + movdqa oword [rsp+GHASH1], xmm1 + movdqa oword [rsp+GHASH2], xmm2 + movdqa oword [rsp+GHASH3], xmm3 + + sub len, BYTES_PER_LOOP + movdqa xmm0, oword [rsp+CNT] ; next counter value + cmp len, BYTES_PER_LOOP + jge .blks4_loop + +.combine_hash: + movdqa xmm8,oword [rbx] ; hk<<1 + movdqa xmm9,oword [rbx+sizeof_oword_] ; (hk^2)<<1 + + sse_clmul_gcm xmm0, xmm10, xmm6, xmm4, xmm5 ; gHash0 = gHash0 * (HashKey^4)<<1 mod poly + sse_clmul_gcm xmm1, xmm9, xmm6, xmm4, xmm5 ; gHash1 = gHash1 * (HashKey^2)<<1 mod poly + sse_clmul_gcm xmm2, xmm8, xmm6, xmm4, xmm5 ; gHash2 = gHash2 * (HashKey^1)<<1 mod poly + + pxor xmm3, xmm1 + pxor xmm3, xmm2 + sse_clmul_gcm xmm3, xmm8, xmm6, xmm4, xmm5 ; gHash3 = gHash3 * (HashKey)<<1 mod poly + pxor xmm3, xmm0 + movdqa oword [rsp+GHASH0], xmm3 ; store gHash + +;; +;; rest of input processing (1-3 blocks) +;; +.single_block_proc: + test rax, rax + jz .quit + +align IPP_ALIGN_FACTOR +.blk_loop: + movdqa xmm0, oword [rsp+CNT] ; advance counter value + movdqa xmm1, xmm0 + paddd xmm1, oword [rel INC_1] + movdqa oword [rsp+CNT], xmm1 + + movdqa xmm0, oword [rcx] ; pre-load whitening keys + mov r10, rcx + + pshufb xmm1, [rel SHUF_CONST] ; counter is ready to be encrypted + + pxor xmm1, xmm0 ; whitening + + movdqa xmm0, oword [r10+16] + add r10, 16 + + mov r11d, nr ; counter depending on key length + sub r11, 1 + +align IPP_ALIGN_FACTOR +.cipher_loop: +my_aesenc xmm1, xmm0 ; regular round + movdqa xmm0, oword [r10+16] + add r10, 16 + dec r11 + jnz .cipher_loop +my_aesenclast xmm1, xmm0 + + movdqa xmm0, oword [rsp+ECNT] ; load pre-calculated encrypted counter + movdqa oword [rsp+ECNT], xmm1 ; save encrypted counter + + movdqu xmm1, oword [pSrc] ; input block + add pSrc, BYTES_PER_BLK + pxor xmm0, xmm1 ; ctr encryption + movdqu oword [pDst], xmm0 + add pDst, BYTES_PER_BLK + + pshufb xmm0, [rel SHUF_CONST] + pxor xmm0, oword [rsp+GHASH0] + movdqa xmm1, oword [rbx] + sse_clmul_gcm xmm0, xmm1, xmm2, xmm3, xmm4 ; update hash value + movdqa oword [rsp+GHASH0], xmm0 + + sub rax, BYTES_PER_BLK + jg .blk_loop + +;; +;; exit +;; +.quit: + movdqa xmm0, oword [rsp+CNT] ; counter + movdqa xmm1, oword [rsp+ECNT] ; encrypted counter + movdqa xmm2, oword [rsp+GHASH0] ; hash + + mov rax, pCtrValue ; address of the counter + mov rbx, pEncCtrValue ; address of the encrypted counter + mov rcx, pGhash ; address of hash value + + pshufb xmm0, [rel SHUF_CONST] ; convert counter back and + movdqu oword [rax], xmm0 ; store counter into the context + + movdqu oword [rbx], xmm1 ; store encrypted counter into the context + + pshufb xmm2, [rel SHUF_CONST] ; convert hach value back + movdqu oword [rcx], xmm2 ; store hash into the context + + REST_XMM + REST_GPR + ret +ENDFUNC AesGcmEnc_avx + +;*************************************************************** +;* Purpose: pipelined AES-GCM decryption +;* +;* void AesGcmDec_avx(Ipp8u* pDst, +;* const Ipp8u* pSrc,, +;* int length, +;* RijnCipher cipher, +;* int nr, +;* const Ipp8u* pRKey, +;* Ipp8u* pGhash, +;* Ipp8u* pCtrValue, +;* Ipp8u* pEncCtrValue, +;* const Ipp8u* pPrecomData) +;*************************************************************** +align IPP_ALIGN_FACTOR +;; +;; Lib = Y8, E9 +;; +;; Caller = ippsRijndael128GCMDecrypt +;; +IPPASM AesGcmDec_avx,PUBLIC +%assign LOCAL_FRAME (8*16) + USES_GPR rsi,rdi,rbx + USES_XMM xmm6,xmm7,xmm8,xmm9,xmm10,xmm11,xmm12,xmm13,xmm14,xmm15 + COMP_ABI 10 + +%xdefine pDst rdi ; pointer to the encrypted data +%xdefine pSrc rsi ; pointer to the plane data +%xdefine len rdx ; data length in bytes (multiple by BYTES_PER_BLK) +%xdefine cipher rcx ; ciper function (don't need in fact) +%xdefine nr r8d ; number of cipher's rounds +%xdefine pRKey r9 ; pointer to the cipher's round keys +%xdefine pGhash [rsp+ARG_7] ; pointer to the Hash value +%xdefine pCtrValue [rsp+ARG_8] ; pointer to the counter value +%xdefine pEncCtrValue [rsp+ARG_9] ; pointer to the encrypted counter +%xdefine pPrecomData [rsp+ARG_10]; pointer to the precomputed data + +%assign SC (4) +%assign BLKS_PER_LOOP (4) +%assign BYTES_PER_BLK (16) +%assign BYTES_PER_LOOP (BYTES_PER_BLK*BLKS_PER_LOOP) + +;; +;; stack structure: +%assign CNT (0) +%assign ECNT (CNT+sizeof_oword_) +%assign GHASH (ECNT+sizeof_oword_) + +%assign GHASH0 (GHASH) +%assign GHASH1 (GHASH0+sizeof_oword_) +%assign GHASH2 (GHASH1+sizeof_oword_) +%assign GHASH3 (GHASH2+sizeof_oword_) +%assign HKeyKaratsuba (GHASH3+sizeof_oword_) + + mov rax, pCtrValue ; address of the counter + mov rbx, pEncCtrValue ; address of the encrypted counter + mov rcx, pGhash ; address of hash value + movdqa xmm4,oword [rel SHUF_CONST] + + movdqu xmm0, oword [rax] ; counter value + movdqu xmm1, oword [rbx] ; encrypted counter value + movdqu xmm2, oword [rcx] ; hash value + + pshufb xmm0, xmm4 ; convert counter and + movdqa oword [rsp+CNT], xmm0 ; store into the stack + movdqa oword [rsp+ECNT], xmm1 ; store encrypted counter into the stack + + pshufb xmm2, xmm4 ; convert hash value + pxor xmm1, xmm1 + movdqa oword [rsp+GHASH0], xmm2 ; store hash into the stack + movdqa oword [rsp+GHASH1], xmm1 ; + movdqa oword [rsp+GHASH2], xmm1 ; + movdqa oword [rsp+GHASH3], xmm1 ; + + mov rbx, [rsp+ARG_10] ; pointer to the {hk<<1,hk^2<<1,kh^4<<1} multipliers + movdqa xmm10,oword [rbx+sizeof_oword_*2] + pshufd xmm9, xmm10, 01001110b ; xmm0 holds qword-swapped version of (HashKey^4)<<1 mod poly for Karatsuba + pxor xmm9, xmm10 + movdqa oword [rsp+HKeyKaratsuba], xmm9 + + movsxd len, edx ; data length + mov rcx, pRKey ; rcx point to the chipher's round keys + + mov rax, len + and rax, BYTES_PER_LOOP-1 + and len, -BYTES_PER_LOOP + jz .single_block_proc + +;; +;; pipelined processing (4 blocks) +;; +align IPP_ALIGN_FACTOR +.blks4_loop: + ;; + ;; ctr encryption + ;; + movdqa xmm6,oword [rel INC_1] + movdqa xmm5,oword [rel SHUF_CONST] + + movdqa xmm1, xmm0 ; counter+1 + paddd xmm1, oword [rel INC_1] + movdqa xmm2, xmm1 ; counter+2 + paddd xmm2, oword [rel INC_1] + movdqa xmm3, xmm2 ; counter+3 + paddd xmm3, oword [rel INC_1] + movdqa xmm4, xmm3 ; counter+4 + paddd xmm4, oword [rel INC_1] + movdqa oword [rsp+CNT], xmm4 + + movdqa xmm0, oword [rcx] ; pre-load whitening keys + mov r10, rcx + + pshufb xmm1, xmm5 ; counter, counter+1, counter+2, counter+3 + pshufb xmm2, xmm5 ; ready to be encrypted + pshufb xmm3, xmm5 + pshufb xmm4, xmm5 + + pxor xmm1, xmm0 ; whitening + pxor xmm2, xmm0 + pxor xmm3, xmm0 + pxor xmm4, xmm0 + + movdqa xmm0, oword [r10+16] + add r10, 16 + + mov r11d, nr ; counter depending on key length + sub r11, 1 + +align IPP_ALIGN_FACTOR +.cipher4_loop: +my_aesenc xmm1, xmm0 ; regular round +my_aesenc xmm2, xmm0 +my_aesenc xmm3, xmm0 +my_aesenc xmm4, xmm0 + movdqa xmm0, oword [r10+16] + add r10, 16 + dec r11 + jnz .cipher4_loop +my_aesenclast xmm1, xmm0 +my_aesenclast xmm2, xmm0 +my_aesenclast xmm3, xmm0 +my_aesenclast xmm4, xmm0 + + movdqa xmm0, oword [rsp+ECNT] ; load pre-calculated encrypted counter + movdqa oword [rsp+ECNT], xmm4 ; save encrypted counter+4 + + movdqu xmm4, oword [pSrc+0*BYTES_PER_BLK] ; 4 input blocks + movdqu xmm5, oword [pSrc+1*BYTES_PER_BLK] + movdqu xmm6, oword [pSrc+2*BYTES_PER_BLK] + movdqu xmm7, oword [pSrc+3*BYTES_PER_BLK] + add pSrc, BYTES_PER_LOOP + + pxor xmm0, xmm4 ; ctr encryption + movdqu oword [pDst+0*BYTES_PER_BLK], xmm0 ; store result + pshufb xmm4, [rel SHUF_CONST] ; convert for multiplication and + pxor xmm4, oword [rsp+GHASH0] + + pxor xmm1, xmm5 + movdqu oword [pDst+1*BYTES_PER_BLK], xmm1 + pshufb xmm5, [rel SHUF_CONST] + pxor xmm5, oword [rsp+GHASH1] + + pxor xmm2, xmm6 + movdqu oword [pDst+2*BYTES_PER_BLK], xmm2 + pshufb xmm6, [rel SHUF_CONST] + pxor xmm6, oword [rsp+GHASH2] + + pxor xmm3, xmm7 + movdqu oword [pDst+3*BYTES_PER_BLK], xmm3 + pshufb xmm7, [rel SHUF_CONST] + pxor xmm7, oword [rsp+GHASH3] + + add pDst, BYTES_PER_LOOP + + cmp len, BYTES_PER_LOOP + je .combine_hash + + ;; + ;; update hash value + ;; + movdqa xmm14, oword [rel MASK1] + sse_clmul_gcm_2way xmm10, xmm9, xmm4, xmm0, xmm1, xmm2, xmm5, xmm11, xmm12, xmm13, xmm15 + sse_clmul_gcm_2way xmm10, xmm9, xmm6, xmm0, xmm1, xmm2, xmm7, xmm11, xmm12, xmm13, xmm15 + + movdqa oword [rsp+GHASH0], xmm4 + movdqa oword [rsp+GHASH1], xmm5 + movdqa oword [rsp+GHASH2], xmm6 + movdqa oword [rsp+GHASH3], xmm7 + + sub len, BYTES_PER_LOOP + movdqa xmm0, oword [rsp+CNT] ; next counter value + cmp len, BYTES_PER_LOOP + jge .blks4_loop + +.combine_hash: + movdqa xmm8,oword [rbx] ; hk<<1 + movdqa xmm9,oword [rbx+sizeof_oword_] ; (hk^2)<<1 + + sse_clmul_gcm xmm4, xmm10, xmm0, xmm1, xmm2 ; gHash0 = gHash0 * (HashKey^4)<<1 mod poly + sse_clmul_gcm xmm5, xmm9, xmm0, xmm1, xmm2 ; gHash1 = gHash1 * (HashKey^2)<<1 mod poly + sse_clmul_gcm xmm6, xmm8, xmm0, xmm1, xmm2 ; gHash2 = gHash2 * (HashKey^1)<<1 mod poly + + pxor xmm7, xmm5 + pxor xmm7, xmm6 + sse_clmul_gcm xmm7, xmm8, xmm0, xmm1, xmm2 ; gHash3 = gHash3 * (HashKey)<<1 mod poly + pxor xmm7, xmm4 + movdqa oword [rsp+GHASH0], xmm7 ; store gHash + +;; +;; rest of input processing (1-3 blocks) +;; +.single_block_proc: + test rax, rax + jz .quit + +align IPP_ALIGN_FACTOR +.blk_loop: + movdqa xmm0, oword [rsp+CNT] ; advance counter value + movdqa xmm1, xmm0 + paddd xmm1, oword [rel INC_1] + movdqa oword [rsp+CNT], xmm1 + + movdqa xmm0, oword [rcx] ; pre-load whitening keys + mov r10, rcx + + pshufb xmm1, [rel SHUF_CONST] ; counter is ready to be encrypted + + pxor xmm1, xmm0 ; whitening + + movdqa xmm0, oword [r10+16] + add r10, 16 + + mov r11d, nr ; counter depending on key length + sub r11, 1 + +align IPP_ALIGN_FACTOR +.cipher_loop: +my_aesenc xmm1, xmm0 ; regular round + movdqa xmm0, oword [r10+16] + add r10, 16 + dec r11 + jnz .cipher_loop +my_aesenclast xmm1, xmm0 + + movdqa xmm0, oword [rsp+ECNT] ; load pre-calculated encrypted counter + movdqa oword [rsp+ECNT], xmm1 ; save encrypted counter + + movdqu xmm1, oword [pSrc] ; input block + add pSrc, BYTES_PER_BLK + pxor xmm0, xmm1 ; ctr encryption + movdqu oword [pDst], xmm0 + add pDst, BYTES_PER_BLK + + pshufb xmm1, [rel SHUF_CONST] + pxor xmm1, oword [rsp+GHASH0] + movdqa xmm0, oword [rbx] + sse_clmul_gcm xmm1, xmm0, xmm2, xmm3, xmm4 ; update hash value + movdqa oword [rsp+GHASH0], xmm1 + + sub rax, BYTES_PER_BLK + jg .blk_loop + +;; +;; exit +;; +.quit: + movdqa xmm0, oword [rsp+CNT] ; counter + movdqa xmm1, oword [rsp+ECNT] ; encrypted counter + movdqa xmm2, oword [rsp+GHASH0] ; hash + + mov rax, pCtrValue ; address of the counter + mov rbx, pEncCtrValue ; address of the encrypted counter + mov rcx, pGhash ; address of hash value + + pshufb xmm0, [rel SHUF_CONST] ; convert counter back and + movdqu oword [rax], xmm0 ; store counter into the context + + movdqu oword [rbx], xmm1 ; store encrypted counter into the context + + pshufb xmm2, [rel SHUF_CONST] ; convert hach value back + movdqu oword [rcx], xmm2 ; store hash into the context + + REST_XMM + REST_GPR + ret +ENDFUNC AesGcmDec_avx + +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpaesgcmtable2ku8as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpaesgcmtable2ku8as.asm new file mode 100644 index 0000000..283ae0a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpaesgcmtable2ku8as.asm @@ -0,0 +1,418 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Encrypt/Decrypt byte data stream according to Rijndael128 (GCM mode) +; +; Content: +; AesGcmMulGcm_table2K() +; AesGcmAuth_table2K() +; + +%include "asmdefs.inc" +%include "ia_32e.inc" + + +%if (_IPP32E >= _IPP32E_U8) + +segment .text align=IPP_ALIGN_FACTOR + +; +; getAesGcmConst_table_ct provides c-e-t access to pre-computed Ipp16u AesGcmConst_table[256] +; +; input: +; r9: address of the AesGcmConst_table +; rcx: index in the table +; +; output: +; rax +; +; register rcx destoyed +; registers mmx2, mmx3, mmx6, and mmx7 destoyed +; +align IPP_ALIGN_FACTOR +INIT_IDX dw 000h,001h,002h,003h,004h,005h,006h,007h ;; initial search inx = {0:1:2:3:4:5:6:7} +INCR_IDX dw 008h,008h,008h,008h,008h,008h,008h,008h ;; index increment = {8:8:8:8:8:8:8:8} + +align IPP_ALIGN_FACTOR +IPPASM getAesGcmConst_table_ct,PRIVATE + pxor xmm2, xmm2 ;; accumulator xmm2 = 0 + + mov rax, rcx ;; broadcast inx into dword + shl rcx, 16 + or rcx, rax + movq xmm3, rcx + pshufd xmm3, xmm3, 00b ;; search index xmm3 = broadcast(idx) + + movdqa xmm6, xmmword [rel INIT_IDX] ;; current indexes + + xor rax, rax +align IPP_ALIGN_FACTOR +.search_loop: + movdqa xmm7, xmm6 ;; copy current indexes + paddw xmm6, xmmword [rel INCR_IDX] ;; advance current indexes + + pcmpeqw xmm7, xmm3 ;; selection mask + pand xmm7, xmmword [r9+rax*sizeof(word)] ;; mask data + + add rax, 8 + cmp rax, 256 + + por xmm2, xmm7 ;; and accumulate + jl .search_loop + + movdqa xmm3, xmm2 ;; pack result in qword + psrldq xmm2, sizeof(xmmword)/2 + por xmm2, xmm3 + movq rax, xmm2 + + and rcx, 3 ;; select tbl[idx] value + shl rcx, 4 ;; rcx *=16 = sizeof(word)*8 + shr rax, cl + ret +ENDFUNC getAesGcmConst_table_ct + +; +; void AesGcmMulGcm_table2K(Ipp8u* pHash, const Ipp8u* pPrecomputedData, const void* pParam) +; +align IPP_ALIGN_FACTOR +IPPASM AesGcmMulGcm_table2K,PUBLIC + USES_GPR rsi,rdi,rbx + USES_XMM xmm6,xmm7 + COMP_ABI 3 + + movdqu xmm0, [rdi] ; hash value + mov r8, rsi ; precomputed data pointer + + mov r9, rdx ; pointer to the fixed table (AesGcmConst_table) + + movd ebx, xmm0 ; ebx = hash.0 + mov eax, 0f0f0f0f0h + and eax, ebx ; eax = 4 x 4_bits + shl ebx, 4 + and ebx, 0f0f0f0f0h ; ebx = 4 x 4_bits (another) + movzx ecx, ah + movdqa xmm5, oword [r8+1024+rcx] + movzx ecx, al + movdqa xmm4, oword [r8+1024+rcx] + shr eax, 16 + movzx ecx, ah + movdqa xmm3, oword [r8+1024+rcx] + movzx ecx, al + movdqa xmm2, oword [r8+1024+rcx] + + psrldq xmm0, 4 ; shift xmm0 + movd eax, xmm0 ; eax = hash[1] + and eax, 0f0f0f0f0h ; eax = 4 x 4_bits + + movzx ecx, bh + pxor xmm5, oword [r8+ (1-1)*256 + rcx] + movzx ecx, bl + pxor xmm4, oword [r8+ (1-1)*256 + rcx] + shr ebx, 16 + movzx ecx, bh + pxor xmm3, oword [r8+ (1-1)*256 + rcx] + movzx ecx, bl + pxor xmm2, oword [r8+ (1-1)*256 + rcx] + + movd ebx, xmm0 ; ebx = hash[1] + shl ebx, 4 ; another 4 x 4_bits + and ebx, 0f0f0f0f0h + + movzx ecx, ah + pxor xmm5, oword [r8+1024+ 1*256 + rcx] + movzx ecx, al + pxor xmm4, oword [r8+1024+ 1*256 + rcx] + shr eax, 16 + movzx ecx, ah + pxor xmm3, oword [r8+1024+ 1*256 + rcx] + movzx ecx, al + pxor xmm2, oword [r8+1024+ 1*256 + rcx] + psrldq xmm0, 4 + + movd eax, xmm0 ; eax = hash.2 + and eax, 0f0f0f0f0h + + movzx ecx, bh + pxor xmm5, oword [r8+ (2-1)*256 + rcx] + movzx ecx, bl + pxor xmm4, oword [r8+ (2-1)*256 + rcx] + shr ebx, 16 + movzx ecx, bh + pxor xmm3, oword [r8+ (2-1)*256 + rcx] + movzx ecx, bl + pxor xmm2, oword [r8+ (2-1)*256 + rcx] + + movd ebx, xmm0 + shl ebx, 4 + and ebx, 0f0f0f0f0h + + movzx ecx, ah + pxor xmm5, oword [r8+1024+ 2*256 + rcx] + movzx ecx, al + pxor xmm4, oword [r8+1024+ 2*256 + rcx] + shr eax, 16 + movzx ecx, ah + pxor xmm3, oword [r8+1024+ 2*256 + rcx] + movzx ecx, al + pxor xmm2, oword [r8+1024+ 2*256 + rcx] + + psrldq xmm0, 4 + movd eax, xmm0 ; eax = hash.3 + and eax, 0f0f0f0f0h + + movzx ecx, bh + pxor xmm5, oword [r8+ (3-1)*256 + rcx] + movzx ecx, bl + pxor xmm4, oword [r8+ (3-1)*256 + rcx] + shr ebx, 16 + movzx ecx, bh + pxor xmm3, oword [r8+ (3-1)*256 + rcx] + movzx ecx, bl + pxor xmm2, oword [r8+ (3-1)*256 + rcx] + + movd ebx, xmm0 + shl ebx, 4 + and ebx, 0f0f0f0f0h + + movzx ecx, ah + pxor xmm5, oword [r8+1024+ 3*256 + rcx] + movzx ecx, al + pxor xmm4, oword [r8+1024+ 3*256 + rcx] + shr eax, 16 + movzx ecx, ah + pxor xmm3, oword [r8+1024+ 3*256 + rcx] + movzx ecx, al + pxor xmm2, oword [r8+1024+ 3*256 + rcx] + + movzx ecx, bh + pxor xmm5, oword [r8+ 3*256 + rcx] + movzx ecx, bl + pxor xmm4, oword [r8+ 3*256 + rcx] + shr ebx, 16 + movzx ecx, bh + pxor xmm3, oword [r8+ 3*256 + rcx] + movzx ecx, bl + pxor xmm2, oword [r8+ 3*256 + rcx] + + movdqa xmm0, xmm3 + pslldq xmm3, 1 + pxor xmm2, xmm3 + movdqa xmm1, xmm2 + pslldq xmm2, 1 + pxor xmm5, xmm2 + psrldq xmm0, 15 + + movd ecx, xmm0 + CALL_IPPASM getAesGcmConst_table_ct ;;movzx eax, word [r9 + rcx*sizeof(word)] + shl eax, 8 + movdqa xmm0, xmm5 + pslldq xmm5, 1 + pxor xmm4, xmm5 + psrldq xmm1, 15 + movd ecx, xmm1 + mov rbx, rax ;;xor ax, word [r9 + rcx*sizeof(word)] + CALL_IPPASM getAesGcmConst_table_ct ;; + xor rax, rbx ;; + shl eax, 8 + psrldq xmm0, 15 + movd ecx, xmm0 + mov rbx, rax ;;xor ax, word [r9 + rcx*sizeof(word)] + CALL_IPPASM getAesGcmConst_table_ct ;; + xor rax, rbx ;; + movd xmm0, eax + pxor xmm0, xmm4 + + movdqu oword [rdi], xmm0 ; store hash value + + REST_XMM + REST_GPR + ret +ENDFUNC AesGcmMulGcm_table2K + + +; +; void AesGcmAuth_table2K(Ipp8u* pHash, const Ipp8u* pSrc, int len, const Ipp8u* pPrecomputedData, const void* pParam) +; +align IPP_ALIGN_FACTOR +IPPASM AesGcmAuth_table2K,PUBLIC + USES_GPR rsi,rdi,rbx + USES_XMM xmm6,xmm7 + COMP_ABI 5 + + mov r9, r8 ; pointer to the fixed table (pParam) + + movdqu xmm0, [rdi] ; hash value + mov r8, rcx ; precomputed data pointer + +align IPP_ALIGN_FACTOR +.auth_loop: + movdqu xmm4, [rsi] ; get src[] + pxor xmm0, xmm4 ; hash ^= src[] + + movd ebx, xmm0 ; ebx = hash.0 + mov eax, 0f0f0f0f0h + and eax, ebx ; eax = 4 x 4_bits + shl ebx, 4 + and ebx, 0f0f0f0f0h ; ebx = 4 x 4_bits (another) + movzx ecx, ah + movdqa xmm5, oword [r8+1024+rcx] + movzx ecx, al + movdqa xmm4, oword [r8+1024+rcx] + shr eax, 16 + movzx ecx, ah + movdqa xmm3, oword [r8+1024+rcx] + movzx ecx, al + movdqa xmm2, oword [r8+1024+rcx] + + psrldq xmm0, 4 ; shift xmm0 + movd eax, xmm0 ; eax = hash[1] + and eax, 0f0f0f0f0h ; eax = 4 x 4_bits + + movzx ecx, bh + pxor xmm5, oword [r8+ (1-1)*256 + rcx] + movzx ecx, bl + pxor xmm4, oword [r8+ (1-1)*256 + rcx] + shr ebx, 16 + movzx ecx, bh + pxor xmm3, oword [r8+ (1-1)*256 + rcx] + movzx ecx, bl + pxor xmm2, oword [r8+ (1-1)*256 + rcx] + + movd ebx, xmm0 ; ebx = hash[1] + shl ebx, 4 ; another 4 x 4_bits + and ebx, 0f0f0f0f0h + + movzx ecx, ah + pxor xmm5, oword [r8+1024+ 1*256 + rcx] + movzx ecx, al + pxor xmm4, oword [r8+1024+ 1*256 + rcx] + shr eax, 16 + movzx ecx, ah + pxor xmm3, oword [r8+1024+ 1*256 + rcx] + movzx ecx, al + pxor xmm2, oword [r8+1024+ 1*256 + rcx] + + psrldq xmm0, 4 + movd eax, xmm0 ; eax = hash[2] + and eax, 0f0f0f0f0h + + movzx ecx, bh + pxor xmm5, oword [r8+ (2-1)*256 + rcx] + movzx ecx, bl + pxor xmm4, oword [r8+ (2-1)*256 + rcx] + shr ebx, 16 + movzx ecx, bh + pxor xmm3, oword [r8+ (2-1)*256 + rcx] + movzx ecx, bl + pxor xmm2, oword [r8+ (2-1)*256 + rcx] + + movd ebx, xmm0 + shl ebx, 4 + and ebx, 0f0f0f0f0h + + movzx ecx, ah + pxor xmm5, oword [r8+1024+ 2*256 + rcx] + movzx ecx, al + pxor xmm4, oword [r8+1024+ 2*256 + rcx] + shr eax, 16 + movzx ecx, ah + pxor xmm3, oword [r8+1024+ 2*256 + rcx] + movzx ecx, al + pxor xmm2, oword [r8+1024+ 2*256 + rcx] + + psrldq xmm0, 4 + movd eax, xmm0 ; eax = hash[3] + and eax, 0f0f0f0f0h + + movzx ecx, bh + pxor xmm5, oword [r8+ (3-1)*256 + rcx] + movzx ecx, bl + pxor xmm4, oword [r8+ (3-1)*256 + rcx] + shr ebx, 16 + movzx ecx, bh + pxor xmm3, oword [r8+ (3-1)*256 + rcx] + movzx ecx, bl + pxor xmm2, oword [r8+ (3-1)*256 + rcx] + + movd ebx, xmm0 + shl ebx, 4 + and ebx, 0f0f0f0f0h + + movzx ecx, ah + pxor xmm5, oword [r8+1024+ 3*256 + rcx] + movzx ecx, al + pxor xmm4, oword [r8+1024+ 3*256 + rcx] + shr eax, 16 + movzx ecx, ah + pxor xmm3, oword [r8+1024+ 3*256 + rcx] + movzx ecx, al + pxor xmm2, oword [r8+1024+ 3*256 + rcx] + + movzx ecx, bh + pxor xmm5, oword [r8+ 3*256 + rcx] + movzx ecx, bl + pxor xmm4, oword [r8+ 3*256 + rcx] + shr ebx, 16 + movzx ecx, bh + pxor xmm3, oword [r8+ 3*256 + rcx] + movzx ecx, bl + pxor xmm2, oword [r8+ 3*256 + rcx] + + movdqa xmm0, xmm3 + pslldq xmm3, 1 + pxor xmm2, xmm3 + movdqa xmm1, xmm2 + pslldq xmm2, 1 + pxor xmm5, xmm2 + psrldq xmm0, 15 + + movd ecx, xmm0 + CALL_IPPASM getAesGcmConst_table_ct ;;movzx eax, word [r9 + rcx*sizeof(word)] + shl eax, 8 + movdqa xmm0, xmm5 + pslldq xmm5, 1 + pxor xmm4, xmm5 + psrldq xmm1, 15 + movd ecx, xmm1 + mov rbx, rax ;;xor ax, word [r9 + rcx*sizeof(word)] + CALL_IPPASM getAesGcmConst_table_ct ;; + xor rax, rbx ;; + shl eax, 8 + psrldq xmm0, 15 + movd ecx, xmm0 + mov rbx, rax ;;xor ax, word [r9 + rcx*sizeof(word)] + CALL_IPPASM getAesGcmConst_table_ct ;; + xor rax, rbx ;; + movd xmm0, eax + pxor xmm0, xmm4 + + add rsi, sizeof(oword) ; advance src address + sub rdx, sizeof(oword) ; decrease counter + jnz .auth_loop ; process next block + + movdqu oword [rdi], xmm0 ; store hash value + + REST_XMM + REST_GPR + ret +ENDFUNC AesGcmAuth_table2K + +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnuaddm7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnuaddm7as.asm new file mode 100644 index 0000000..1a2672c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnuaddm7as.asm @@ -0,0 +1,327 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Big Number Operations +; +; Content: +; cpAdd_BNU() +; +; + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_M7) + +segment .text align=IPP_ALIGN_FACTOR + +;************************************************************* +;* Ipp64u cpAdd_BNU(Ipp64u* pDst, +;* const Ipp64u* pSrc1, +;* const Ipp64u* pSrc2, +;* int len) +;* returns carry +;************************************************************* +align IPP_ALIGN_FACTOR +IPPASM cpAdd_BNU,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi + USES_XMM + COMP_ABI 4 + +; rdi = pDst +; rsi = pSrcA +; rdx = pSrcB +; rcx = len + + movsxd rcx, ecx ; unsigned length + xor rax, rax + + cmp rcx, 2 + jge .ADD_GE2 + +;********** lenSrcA == 1 ************************************* + add rax, rax + mov r8, qword [rsi] ; rsi = a + adc r8, qword [rdx] ; r8 = a+b = s + mov qword [rdi], r8 ; save s + sbb rax, rax ; + jmp .FINAL + +;********** lenSrcA == 1 END ******************************** + +.ADD_GE2: + jg .ADD_GT2 + +;********** lenSrcA == 2 ************************************* + add rax, rax + mov r8, qword [rsi] ; r8 = a0 + adc r8, qword [rdx] ; r8 = a0+b0 = s0 + mov r9, qword [rsi+8] ; r9 = a1 + adc r9, qword [rdx+8] ; r9 = a1+b1 = s1 + mov qword [rdi], r8 ; save s0 + mov qword [rdi+8], r9 ; save s1 + sbb rax, rax ; rax = carry + jmp .FINAL + +;********** lenSrcA == 2 END ********************************* + +.ADD_GT2: + cmp rcx, 4 + jge .ADD_GE4 + +;********** lenSrcA == 3 ************************************* + add rax, rax + mov r8, qword [rsi] ; r8 = a0 + adc r8, qword [rdx] ; r8 = a0+b0 = s0 + mov r9, qword [rsi+8] ; r9 = a1 + adc r9, qword [rdx+8] ; r9 = a1+b1 = s1 + mov r10, qword [rsi+16] ; r10 = a2 + adc r10, qword [rdx+16] ; r10 = a2+b2 = s2 + mov qword [rdi], r8 ; save s0 + mov qword [rdi+8], r9 ; save s1 + mov qword [rdi+16], r10 ; save s2 + sbb rax, rax ; rax = carry + jmp .FINAL + +;********** lenSrcA == 3 END ********************************* + +.ADD_GE4: + jg .ADD_GT4 + +;********** lenSrcA == 4 ************************************* + add rax, rax + mov r8, qword [rsi] ; r8 = a0 + adc r8, qword [rdx] ; r8 = a0+b0 = s0 + mov r9, qword [rsi+8] ; r9 = a1 + adc r9, qword [rdx+8] ; r9 = a1+b1 = s1 + mov r10, qword [rsi+16] ; r10 = a2 + adc r10, qword [rdx+16] ; r10 = a2+b2 = s2 + mov r11, qword [rsi+24] ; r11 = a3 + adc r11, qword [rdx+24] ; r11 = a3+b3 = s3 + mov qword [rdi], r8 ; save s0 + mov qword [rdi+8], r9 ; save s1 + mov qword [rdi+16], r10 ; save s2 + mov qword [rdi+24], r11 ; save s2 + sbb rax, rax ; rax = carry + jmp .FINAL + +;********** lenSrcA == 4 END ********************************* + +.ADD_GT4: + cmp rcx, 6 + jge .ADD_GE6 + +;********** lenSrcA == 5 ************************************* + add rax, rax + mov r8, qword [rsi] ; r8 = a0 + adc r8, qword [rdx] ; r8 = a0+b0 = s0 + mov r9, qword [rsi+8] ; r9 = a1 + adc r9, qword [rdx+8] ; r9 = a1+b1 = s1 + mov r10, qword [rsi+16] ; r10 = a2 + adc r10, qword [rdx+16] ; r10 = a2+b2 = s2 + mov r11, qword [rsi+24] ; r11 = a3 + adc r11, qword [rdx+24] ; r11 = a3+b3 = s3 + mov rcx, qword [rsi+32] ; rcx = a4 + adc rcx, qword [rdx+32] ; rcx = a4+b4 = s4 + mov qword [rdi], r8 ; save s0 + mov qword [rdi+8], r9 ; save s1 + mov qword [rdi+16], r10 ; save s2 + mov qword [rdi+24], r11 ; save s3 + mov qword [rdi+32], rcx ; save s4 + sbb rax, rax ; rax = carry + jmp .FINAL + +;********** lenSrcA == 5 END ********************************* + +.ADD_GE6: + jg .ADD_GT6 + +;********** lenSrcA == 6 ************************************* + add rax, rax + mov r8, qword [rsi] ; r8 = a0 + adc r8, qword [rdx] ; r8 = a0+b0 = s0 + mov r9, qword [rsi+8] ; r9 = a1 + adc r9, qword [rdx+8] ; r9 = a1+b1 = s1 + mov r10, qword [rsi+16] ; r10 = a2 + adc r10, qword [rdx+16] ; r10 = a2+b2 = s2 + mov r11, qword [rsi+24] ; r11 = a3 + adc r11, qword [rdx+24] ; r11 = a3+b3 = s3 + mov rcx, qword [rsi+32] ; rcx = a4 + adc rcx, qword [rdx+32] ; rcx = a4+b4 = s4 + mov rsi, qword [rsi+40] ; rsi = a5 + adc rsi, qword [rdx+40] ; rsi = a5+b5 = s5 + mov qword [rdi], r8 ; save s0 + mov qword [rdi+8], r9 ; save s1 + mov qword [rdi+16], r10 ; save s2 + mov qword [rdi+24], r11 ; save s3 + mov qword [rdi+32], rcx ; save s4 + mov qword [rdi+40], rsi ; save s5 + sbb rax, rax ; rax = carry + jmp .FINAL + +;********** lenSrcA == 6 END ********************************* + +.ADD_GT6: + cmp rcx, 8 + jge .ADD_GE8 + +.ADD_EQ7: +;********** lenSrcA == 7 ************************************* + add rax, rax + mov r8, qword [rsi] ; r8 = a0 + adc r8, qword [rdx] ; r8 = a0+b0 = s0 + mov r9, qword [rsi+8] ; r9 = a1 + adc r9, qword [rdx+8] ; r9 = a1+b1 = s1 + mov r10, qword [rsi+16] ; r10 = a2 + adc r10, qword [rdx+16] ; r10 = a2+b2 = s2 + mov r11, qword [rsi+24] ; r11 = a3 + adc r11, qword [rdx+24] ; r11 = a3+b3 = s3 + mov rcx, qword [rsi+32] ; rcx = a4 + adc rcx, qword [rdx+32] ; rcx = a4+b4 = s4 + mov qword [rdi], r8 ; save s0 + mov r8, qword [rsi+40] ; r8 = a5 + adc r8, qword [rdx+40] ; r8 = a5+b5 = s5 + mov rsi, qword [rsi+48] ; rsi = a6 + adc rsi, qword [rdx+48] ; rsi = a6+b6 = s6 + mov qword [rdi+8], r9 ; save s1 + mov qword [rdi+16], r10 ; save s2 + mov qword [rdi+24], r11 ; save s3 + mov qword [rdi+32], rcx ; save s4 + mov qword [rdi+40], r8 ; save s5 + mov qword [rdi+48], rsi ; save s6 + sbb rax, rax ; rax = carry + jmp .FINAL + +;********** lenSrcA == 7 END ********************************* + + +.ADD_GE8: + jg .ADD_GT8 + +;********** lenSrcA == 8 ************************************* + add rax, rax + mov r8, qword [rsi] ; r8 = a0 + adc r8, qword [rdx] ; r8 = a0+b0 = s0 + mov r9, qword [rsi+8] ; r9 = a1 + adc r9, qword [rdx+8] ; r9 = a1+b1 = s1 + mov r10, qword [rsi+16] ; r10 = a2 + adc r10, qword [rdx+16] ; r10 = a2+b2 = s2 + mov r11, qword [rsi+24] ; r11 = a3 + adc r11, qword [rdx+24] ; r11 = a3+b3 = s3 + mov rcx, qword [rsi+32] ; rcx = a4 + adc rcx, qword [rdx+32] ; rcx = a4+b4 = s4 + mov qword [rdi], r8 ; save s0 + mov r8, qword [rsi+40] ; r8 = a5 + adc r8, qword [rdx+40] ; r8 = a5+b5 = s5 + mov qword [rdi+8], r9 ; save s1 + mov r9, qword [rsi+48] ; r9 = a7 + adc r9, qword [rdx+48] ; r9 = a7+b7 = s7 + mov rsi, qword [rsi+56] ; rsi = a6 + adc rsi, qword [rdx+56] ; rsi = a6+b6 = s6 + mov qword [rdi+16], r10 ; save s2 + mov qword [rdi+24], r11 ; save s3 + mov qword [rdi+32], rcx ; save s4 + mov qword [rdi+40], r8 ; save s5 + mov qword [rdi+48], r9 ; save s6 + mov qword [rdi+56], rsi ; save s7 + sbb rax, rax ; rax = carry + jmp .FINAL + +;********** lenSrcA == 8 END ********************************* + + +;********** lenSrcA > 8 ************************************* + +.ADD_GT8: + mov r8, rax + mov rax, rcx ; rax = len + and rcx, 3 ; + xor rcx, rax ; + lea rsi, [rsi+8*rcx] ; + lea rdx, [rdx+8*rcx] ; + lea rdi, [rdi+8*rcx] ; + neg rcx + add r8, r8 + jmp .ADD_GLOOP + +align IPP_ALIGN_FACTOR +.ADD_GLOOP: + mov r8, qword [rsi+8*rcx] ; r8 = a0 + mov r9, qword [rsi+8*rcx+8] ; r9 = a1 + mov r10, qword [rsi+8*rcx+16] ; r10 = a2 + mov r11, qword [rsi+8*rcx+24] ; r11 = a3 + adc r8, qword [rdx+8*rcx] ; r8 = a0+b0 = r0 + adc r9, qword [rdx+8*rcx+8] ; r9 = a1+b1 = r1 + adc r10, qword [rdx+8*rcx+16] ; r10 = a2+b2 = r2 + adc r11, qword [rdx+8*rcx+24] ; r11 = a3+b3 = r3 + mov qword [rdi+8*rcx], r8 ; + mov qword [rdi+8*rcx+8], r9 ; + mov qword [rdi+8*rcx+16], r10 ; + mov qword [rdi+8*rcx+24], r11 ; + lea rcx, [rcx+4] + jrcxz .ADD_LLAST0 + jmp .ADD_GLOOP + +.ADD_LLAST0: + sbb rcx, rcx + and rax, 3 + jz .FIN0 + +.ADD_LLOOP: + test rax, 2 + jz .ADD_LLAST1 + + add rcx, rcx + mov r8, qword [rsi] ; r8 = a0 + mov r9, qword [rsi+8] ; r9 = a1 + adc r8, qword [rdx] ; r8 = a0+b0 = r0 + adc r9, qword [rdx+8] ; r9 = a1+b1 = r1 + mov qword [rdi], r8 ; + mov qword [rdi+8], r9 ; + sbb rcx, rcx + test rax, 1 + jz .FIN0 + + add rsi, 16 + add rdx, 16 + add rdi, 16 + +.ADD_LLAST1: + add rcx, rcx + mov r8, qword [rsi] ; r8 = a0 + adc r8, qword [rdx] ; r8 = a0+b0 = r0 + mov qword [rdi], r8 ; + sbb rcx, rcx + +.FIN0: + mov rax, rcx + +;******************* .FINAL *********************************************************** + +.FINAL: + neg rax + REST_XMM + REST_GPR + ret +ENDFUNC cpAdd_BNU + +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnudivm7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnudivm7as.asm new file mode 100644 index 0000000..5afe156 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnudivm7as.asm @@ -0,0 +1,476 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Big Number Operations +; +; Content: +; cpDiv_BNU32() +; +; + +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "ia_32e_regs.inc" + +%if (_IPP32E >= _IPP32E_M7) + +;; +;; FIX_BNU returns actual length of BNU +;; +;; input +;; rSrc points BNU +;; rLen initial BNU size +;; +;; output +;; rSrc points BNU +;; rLen actual BNU size +;; +%macro FIX_BNU 3.nolist + %xdefine %%rSrc %1 + %xdefine %%rLen %2 + %xdefine %%tmp %3 + +%%fix_bnu_loop: + mov %%tmp%+d,[%%rSrc+%%rLen*4-4] ;; value + test %%tmp%+d,%%tmp%+d ;; test BNU component + jnz %%fix_bnu_quit + sub %%rLen,1 + jg %%fix_bnu_loop + add %%rLen,1 +%%fix_bnu_quit: +%endmacro + + +;; +;; Number of Leaging Zeros in32-bit value +;; +%macro NLZ32u 2.nolist + %xdefine %%rNlz %1 + %xdefine %%rVal %2 + + mov %%rNlz, 32 + test %%rVal,%%rVal + jz %%nlz_quit + + xor %%rNlz,%%rNlz +%%nlz16: + test %%rVal,0FFFF0000h + jnz %%nlz8 + shl %%rVal,16 + add %%rNlz,16 +%%nlz8: + test %%rVal,0FF000000h + jnz %%nlz4 + shl %%rVal,8 + add %%rNlz,8 +%%nlz4: + test %%rVal,0F0000000h + jnz %%nlz2 + shl %%rVal,4 + add %%rNlz,4 +%%nlz2: + test %%rVal,0C0000000h + jnz %%nlz1 + shl %%rVal,2 + add %%rNlz,2 +%%nlz1: + test %%rVal,080000000h + jnz %%nlz_quit + add %%rNlz,1 +%%nlz_quit: +%endmacro + + +;; +;; (Logical) Shift BNU Left and Right +;; +;; Input: +;; rDst source/destination address +;; rLen length (dwords) of BNU +;; CL left shift +;; rTmpH scratch +;; rTmpL scratch +;; Note +;; rDst don't changes +;; rLen changes +;; +%macro SHL_BNU_I 4.nolist + %xdefine %%rDst %1 + %xdefine %%rLen %2 + %xdefine %%rTmpH %3 + %xdefine %%rTmpL %4 + + mov %%rTmpH%+d,[%%rDst+%%rLen*4-4] + sub %%rLen,1 + jz %%shl_bnu_quit +%%shl_bnu_loop: + mov %%rTmpL%+d,[%%rDst+%%rLen*4-4] + shld %%rTmpH%+d,%%rTmpL%+d, CL + mov [%%rDst+%%rLen*4],%%rTmpH%+d + mov %%rTmpH%+d,%%rTmpL%+d + sub %%rLen,1 + jg %%shl_bnu_loop +%%shl_bnu_quit: + shl %%rTmpH%+d,CL + mov [%%rDst],%%rTmpH%+d +%endmacro + + +%macro SHR_BNU_I 4.nolist + %xdefine %%rDst %1 + %xdefine %%rLen %2 + %xdefine %%rTmpH %3 + %xdefine %%rTmpL %4 + + push %%rDst + mov %%rTmpL%+d,[%%rDst] + sub %%rLen,1 + jz %%shr_bnu_quit +%%shr_bnu_loop: + mov %%rTmpH%+d,[%%rDst+4] + shrd %%rTmpL%+d,%%rTmpH%+d, CL + mov [%%rDst],%%rTmpL%+d + add %%rDst,4 + mov %%rTmpL%+d,%%rTmpH%+d + sub %%rLen,1 + jg %%shr_bnu_loop +%%shr_bnu_quit: + shr %%rTmpL%+d,CL + mov [%%rDst],%%rTmpL%+d + pop %%rDst +%endmacro + + +;; +;; Multuply BNU by 32-bit digit and Subtract +;; +;; input +;; rSrc points source BNU +;; rDgt 32-bit digit value +;; rDst points accumulator (resultant) BNU +;; rLen BNU size +;; rIdx (scratch) source/target index +;; rExp (scratch) expansion +;; +;; output +;; rDgt 32-bit expansion +;; +;; Note +;; rdx and rax used in mul instruction, +;; this mean any macro argument may be neither rax nor rdx +;; +%macro xMDC_BNU_D32 6.nolist + %xdefine %%rSrc %1 + %xdefine %%rDgt %2 + %xdefine %%rDst %3 + %xdefine %%rLen %4 + %xdefine %%rIdx %5 + %xdefine %%rExp %6 + + xor %%rIdx,%%rIdx ;; index = 0 + xor %%rExp,%%rExp ;; carry = 0 + + sub %%rLen,2 ;; test length + jl %%mdc_short + +%%mdc_loop: + mov rax, [%%rSrc+%%rIdx] ;; a = src[i] + mul %%rDgt ;; x = a*w + + add rax, %%rExp ;; x += borrow + adc rdx,0 + + xor %%rExp,%%rExp ;; zero extension of r + sub [%%rDst+%%rIdx],rax ;; dst[] -= x + sbb %%rExp,rdx + + neg %%rExp ;; update borrow + + add %%rIdx,2*4 ;; advance index + sub %%rLen,2 ;; decrease counter + jge %%mdc_loop ;; continue + + add %%rLen,2 + jz %%mdc_quit + +%%mdc_short: + mov eax, [%%rSrc+%%rIdx] ;; a = src[i] + mul %%rDgt%+d ;; x = a*w + + add eax, %%rExp%+d ;; x += borrow + adc edx,0 + + xor %%rExp%+d, %%rExp%+d ;; zero extension of r + sub [%%rDst+%%rIdx],eax ;; dst[] -= x + sbb %%rExp%+d,edx + + neg %%rExp%+d ;; update borrow + +%%mdc_quit: + mov %%rDgt,%%rExp ;; return borrow +%endmacro + + +;; +;; xADD_BNU add BNUs +;; +;; input +;; rDst points resultant BNU +;; rSrc1 points source BNU +;; rSrc2 points source BNU +;; rLen BNU size +;; rIdx source, resultant index +;; +;; output +;; rCarry carry value (byte) +;; +%macro xADD_BNU 8.nolist + %xdefine %%rDst %1 + %xdefine %%rCarry %2 + %xdefine %%rSrc1 %3 + %xdefine %%rSrc2 %4 + %xdefine %%rLen %5 + %xdefine %%rIdx %6 + %xdefine %%tmp1 %7 + %xdefine %%tmp2 %8 + + xor %%rCarry,%%rCarry ;; carry=0 + xor %%rIdx,%%rIdx ;; index=0 + + sub %%rLen,2 ;; test BNU size + jl %%short_bnu + + clc ;; CF=0 +%%add_bnu_loop: + mov %%tmp1,[%%rSrc1+%%rIdx*8] ;; src1[] + mov %%tmp2,[%%rSrc2+%%rIdx*8] ;; src2[] + adc %%tmp1,%%tmp2 ;; x = src1[]+src[2]+CF + mov [%%rDst+%%rIdx*8],%%tmp1 ;; dst[] = x + + inc %%rIdx ;; advance index + dec %%rLen ;; decrease length + dec %%rLen + jge %%add_bnu_loop ;; continue + setc %%rCarry%+b ;; save CF + + add %%rIdx,%%rIdx ;; restore ordinal index + add %%rLen,2 ;; restore length + jz %%add_bnu_exit + +%%short_bnu: + shr %%rCarry%+d,1 ;; restore CF + mov %%tmp1%+d,[%%rSrc1+%%rIdx*4] ;; src1[] + mov %%tmp2%+d,[%%rSrc2+%%rIdx*4] ;; src2[] + adc %%tmp1%+d,%%tmp2%+d ;; x = src1[]-src[2]-CF + mov [%%rDst+%%rIdx*4],%%tmp1%+d ;; dst[] = x + setc %%rCarry%+b ;; save CF + add %%rIdx,1 ;; advance index +%%add_bnu_exit: +%endmacro + + + +segment .text align=IPP_ALIGN_FACTOR + + +;************************************************************* +;* Ipp32u cpDiv_BNU32(Ipp32u* pQ, int* sizeQ, +;* Ipp32u* pX, int sizeX, +;* Ipp32u* pY, int sizeY) +;* +;************************************************************* +align IPP_ALIGN_FACTOR +IPPASM cpDiv_BNU32,PUBLIC +%assign LOCAL_FRAME 4*8 + USES_GPR rsi,rdi,rbx,rbp,r12,r13,r14,r15 + USES_XMM + COMP_ABI 6 + +; rdi = pQ ; address of quotient +; rsi = sizeQ ; address of quotient length +; rdx = pX ; address of denominator (remaider) +; rcx = sizeX ; length (dwords) of denominator +; r8 = pY ; address of divired +; r9 = sizeY ; length (dwords) of length + + movsxd rcx, ecx ; length + movsxd r9, r9d + +; make sure denominator and divider are fixed + FIX_BNU rdx,rcx, rax + FIX_BNU r8, r9, rax + + mov r10, rdx ; save pX + mov r11, rcx ; save sizeX + +; +; special case: sizeX < sizeY +; +.spec_case1: + cmp rcx, r9 + jae .spec_case2 + + test rdi, rdi ; %if quotient was requested + jz .spec_case1_quit + mov DWORD [rdi], 0 ; pQ[0] = 0 + mov DWORD [rsi], 1 ; sizeQ = 1 +.spec_case1_quit: + mov rax, rcx ; remainder length address + REST_XMM + REST_GPR + ret + +; +; special case: 1 == sizeY +; +.spec_case2: + cmp r9, 1 + jnz .common_case + + mov ebx, [r8] ; divider = pY[0] + xor edx,edx ; init remaider + +.spec_case2_loop: + mov eax,[r10+r11*4-4] + div ebx ; (edx,eax)/pY[0] + + test rdi, rdi ; store %if quotient requested + je .spec_case2_cont + mov [rdi+r11*4-4],eax + +.spec_case2_cont: + sub r11,1 + jg .spec_case2_loop + + test rdi, rdi ; %if quotient was requested + je .spec_case2_quit + FIX_BNU rdi,rcx, rax ; fix quotient + mov DWORD [rsi], ecx ; store quotient length +.spec_case2_quit: + mov DWORD [r10],edx ; pX[0] = remainder value + mov rax, dword 1 + REST_XMM + REST_GPR + ret + +; +; common case +; +.common_case: + xor eax,eax ; expand denominator + mov [r10+r11*4], eax ; by zero + + mov eax,[r8+r9*4-4] ; get divider's + NLZ32u ecx,eax ; factor + + test ecx,ecx ; test normalization factor + jz .division ; and + mov r15,r9 ; normalize + SHL_BNU_I r8,r15, r12,r13 ; divider + lea r15,[r11+1] ; and + SHL_BNU_I r10,r15, r12,r13 ; denominator + +; compute quotation digit-by-digit +.division: + mov ebx,[r8+r9*4-4] ; yHi - the most significant divider digit pY[ySize-1] + + mov [rsp], r10 ; save pX + mov [rsp+8], r11 ; save sizeX + + sub r11, r9 ; estimate length of quotient = (sizeX-sizeY+1) + mov [rsp+16], r11 ; (will use for loop counter) + + lea r10, [r10+r11*4] ; points current denominator position + +.division_loop: + mov rax,[r10+r9*4-4] ; tmp = (pX[xSize],pX[xSize-1]) + xor rdx,rdx ; estimate quotation digit: + div rbx ; (rax) q = tmp/yHi + ; (rdx) r = tmp%yHi + mov r12,rax + mov r13,rdx + + mov ebp,[r8+r9*4-8] ; next significant divider digit pY[ySize-2] +.tune_loop: + mov r15,0FFFFFFFF00000000h + and r15,rax ; %if q >= base .tune q value + jne .tune + mul rbp ; (rax) A = q*pY[ySize-2] + mov r14,r13 + shl r14,32 ; (rdx) B = QWORD(r,pX[xSize-2]) + mov edx,[r10+r9*4-8] + or rdx,r14 + cmp rax,rdx ; %if A>B .tune q value + jbe .mul_and_sub +.tune: + sub r12,1 ; q -= 1 + add r13d,ebx ; r += yHi + mov rax,r12 + jnc .tune_loop + +.mul_and_sub: + mov r15,r9 ; multiplay and subtract + mov ebp,r12d + xMDC_BNU_D32 r8, rbp, r10,r15, r13,r14 + sub [r10+r9*4],ebp ; extend = (pX[i+sizeY] -= extend); + + jnc .store_duotation + sub r12d,1 + mov r15,r9 + xADD_BNU r10,rax, r10,r8,r15, r13,r14,rdx + add [r10+r9*4],eax ; pX[i+sizeY] += extend; + +.store_duotation: + test rdi, rdi + jz .cont_division_loop + mov DWORD [rdi+r11*4],r12d + +.cont_division_loop: + sub r10,4 + sub r11,1 + jge .division_loop + + mov r10,[rsp] ; restore pX + mov r11,[rsp+8] ; restore sizeX + + test ecx,ecx ; test normalization factor + jz .store_results ; and + mov r15,r9 ; de-normalize + SHR_BNU_I r8,r15, r12,r13 ; divider + mov r15,r11 ; and + SHR_BNU_I r10,r15, r12,r13 ; remainder + +.store_results: + test rdi, rdi + jz .quit + mov rcx,[rsp+16] ; restore quotient length + add rcx,1 + FIX_BNU rdi, rcx, rax ; fix quotient + mov DWORD [rsi], ecx + +.quit: + FIX_BNU r10,r11, rax ; fix remainder + mov rax, r11 + REST_XMM + REST_GPR + ret +ENDFUNC cpDiv_BNU32 + +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnuincm7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnuincm7as.asm new file mode 100644 index 0000000..704793d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnuincm7as.asm @@ -0,0 +1,168 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Big Number Operations +; +; Content: +; cpInc_BNU() +; cpDec_BNU() +; +; + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_M7) + +segment .text align=IPP_ALIGN_FACTOR + + +;************************************************************* +;* Ipp64u cpInc_BNU(Ipp64u* pDst, +;* const Ipp64u* pSrc, int len, +;* Ipp64u increment) +;* returns carry +;************************************************************* +align IPP_ALIGN_FACTOR +IPPASM cpInc_BNU,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi + USES_XMM + COMP_ABI 4 + +; rdi = pDst +; rsi = pSrc +; rdx = len +; rcx = increment + + movsxd rdx, edx ; length + + mov r8, qword [rsi] ; r[0] = r[0]+increment + add r8, rcx + mov qword [rdi], r8 + + lea rsi, [rsi+rdx*sizeof(qword)] + lea rdi, [rdi+rdx*sizeof(qword)] + lea rcx, [rdx*sizeof(qword)] + + sbb rax, rax ; save cf + neg rcx ; rcx = negative length (bytes) + add rcx, sizeof(qword) + jrcxz .exit + add rax, rax ; restore cf + jnc .copy + +align IPP_ALIGN_FACTOR +.inc_loop: + mov r8, qword [rsi+rcx] + adc r8, 0 + mov qword [rdi+rcx], r8 + lea rcx, [rcx+sizeof(qword)] + jrcxz .exit_loop + jnc .exit_loop + jmp .inc_loop +.exit_loop: + sbb rax, rax ; save cf + +.copy: + cmp rsi, rdi + jz .exit + jrcxz .exit +.copy_loop: + mov r8, qword [rsi+rcx] + mov qword [rdi+rcx], r8 + add rcx, sizeof(qword) + jnz .copy_loop + +.exit: + neg rax + REST_XMM + REST_GPR + ret +ENDFUNC cpInc_BNU + + + +;************************************************************* +;* Ipp64u cpDec_BNU(Ipp64u* pDst, +;* const Ipp64u* pSrc, int len, +;* Ipp64u increment) +;* returns borrow +;************************************************************* +align IPP_ALIGN_FACTOR +IPPASM cpDec_BNU,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi + USES_XMM + COMP_ABI 4 + +; rdi = pDst +; rsi = pSrc +; rdx = len +; rcx = increment + + movsxd rdx, edx ; length + + mov r8, qword [rsi] ; r[0] = r[0]+increment + sub r8, rcx + mov qword [rdi], r8 + + lea rsi, [rsi+rdx*sizeof(qword)] + lea rdi, [rdi+rdx*sizeof(qword)] + lea rcx, [rdx*sizeof(qword)] + + sbb rax, rax ; save cf + neg rcx ; rcx = negative length (bytes) + add rcx, sizeof(qword) + jrcxz .exit + add rax, rax ; restore cf + jnc .copy + +align IPP_ALIGN_FACTOR +.inc_loop: + mov r8, qword [rsi+rcx] + sbb r8, 0 + mov qword [rdi+rcx], r8 + lea rcx, [rcx+sizeof(qword)] + jrcxz .exit_loop + jnc .exit_loop + jmp .inc_loop +.exit_loop: + sbb rax, rax ; save cf + +.copy: + cmp rsi, rdi + jz .exit + jrcxz .exit +.copy_loop: + mov r8, qword [rsi+rcx] + mov qword [rdi+rcx], r8 + add rcx, sizeof(qword) + jnz .copy_loop + +.exit: + neg rax + REST_XMM + REST_GPR + ret +ENDFUNC cpDec_BNU + +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnum7.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnum7.inc new file mode 100644 index 0000000..3b25b77 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnum7.inc @@ -0,0 +1,691 @@ +;=============================================================================== +; Copyright (C) 2013 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Big Number macros +; +; Content: +; SWAP +; EXPAND_BNU +; +; COPY_BNU +; FIX_BNU CMP_BNU +; +; ADD_BNU SUB_BNU +; ECARRY EBORROW +; +; MUL_BNU_D32 MAC_BNU_D32 +; MUL_BNU_D64 MAC_BNU_D64 +; +; MAC MAC2 +; +; ADD_FIX_BNU SUB_FIX_BNU +; + +;; +;; Swap values or addresses +;; +%macro SWAP 4.nolist + %xdefine %%cc %1 + %xdefine %%dst %2 + %xdefine %%src %3 + %xdefine %%tmp %4 + + cmov&%%cc %%tmp,%%src + cmov&%%cc %%src,%%dst + cmov&%%cc %%dst,%%tmp +%endmacro + +;; +;; EXPAND_BNU expands BNU by zero %if one has odd size +;; +%macro EXPAND_BNU 2.nolist + %xdefine %%rBNU %1 + %xdefine %%rLen %2 + + test %%rLen,1 + jz %%expand_quit + mov DWORD [%%rBNU+%%rLen*4],0 + add %%rLen,1 +%%expand_quit: +%endmacro + +;; +;; COPY_BNU +;; +%macro COPY_BNU 5.nolist + %xdefine %%rSrc %1 + %xdefine %%rDst %2 + %xdefine %%rLen %3 + %xdefine %%rIdx %4 + %xdefine %%rTmp %5 + + xor %%rIdx,%%rIdx +%%copy_bnu: + mov %%rTmp&d,[%%rSrc+%%rIdx*4] + mov [%%rDst+%%rIdx*4],%%rTmp&d + add %%rIdx,1 + cmp %%rIdx,%%rLen + jl %%copy_bnu +%endmacro + +;; +;; FIX_BNU returns actual length of BNU +;; +;; input +;; rSrc points BNU +;; rLen initial BNU size +;; +;; output +;; rSrc points BNU +;; rLen actual BNU size +;; +%macro FIX_BNU 3.nolist + %xdefine %%rSrc %1 + %xdefine %%rLen %2 + %xdefine %%tmp %3 + +%%fix_bnu_loop: + mov %%tmp&d,[%%rSrc+%%rLen*4-4] ;; value + test %%tmp&d,%%tmp&d ;; test BNU component + jnz %%fix_bnu_quit + sub %%rLen,1 + jg %%fix_bnu_loop + add %%rLen,1 +%%fix_bnu_quit: +%endmacro + +;; +;; CMP_BNU comare BNUs +;; +;; input +;; rSrc1 points BNU1 +;; rSrc2 points BNU2 +;; rLen size of BNUs +;; +;; output +;; rCode -1/0/1 +;; +%macro CMP_BNU 4.nolist + %xdefine %%rCode %1 + %xdefine %%rSrc1 %2 + %xdefine %%rSrc2 %3 + %xdefine %%rLen %4 + +%%cmp_bnu_loop: + mov %%rCode&d,[%%rSrc1+%%rLen*4-4] ;; src1[] + cmp %%rCode&d,[%%rSrc2+%%rLen*4-4] ;; src1[] ~ src2[] + jnz %%cmp_bnu_quit ;; src1[] != src2[] + sub %%rLen,1 + jg %%cmp_bnu_loop +%%cmp_bnu_quit: + seta %%rCode&b ;; rCode = (src1[]>src2[])? 1:0 + setb %%rLen&b ;; rLen = (src1[]= _IPP32E_M7) + +segment .text align=IPP_ALIGN_FACTOR + + +;************************************************************* +; Ipp64u cpAddMulDgt_BNU(Ipp64u* pDst, +; const Ipp64u* pSrcA, +; int len, +; Ipp64u B ) +;************************************************************* +align IPP_ALIGN_FACTOR +IPPASM cpAddMulDgt_BNU,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbx,rsi,rdi,r11,r12 + USES_XMM + COMP_ABI 4 + +; rdi = pDst +; rsi = pSrc +; rdx = len +; rcx = B + +%xdefine B0 rcx ; b + +%xdefine T0 r8 ; temporary +%xdefine T1 r9 +%xdefine T2 r10 +%xdefine T3 r11 + +%xdefine idx rbx ; index +%xdefine rDst rdi +%xdefine rSrc rsi + + mov edx, edx ; unsigned length + + mov rax, qword [rsi] + cmp rdx, 1 + jnz .general_case + + mul rcx + add qword [rdi], rax + adc rdx, 0 + mov rax, rdx + REST_XMM + REST_GPR + ret + +.general_case: + lea rSrc, [rSrc+rdx*sizeof(qword)-sizeof(qword)*5] + lea rDst, [rDst+rdx*sizeof(qword)-sizeof(qword)*5] + mov idx, dword 5 + sub idx, rdx ; negative counter -(len-5) + + mul rcx ; {T1:T0} = a[0]*B + mov T0, rax + mov rax, qword [rSrc+idx*sizeof(qword)+sizeof(qword)] + mov T1, rdx + + cmp idx, 0 + jge .skip_muladd_loop4 + +align IPP_ALIGN_FACTOR +.muladd_loop4: + mul rcx ; a[4*i+1]*B + xor T2, T2 + add qword [rDst+idx*sizeof(qword)], T0 + adc T1, rax + mov rax, qword [rSrc+idx*sizeof(qword)+sizeof(qword)*2] + adc T2, rdx + + mul rcx ; a[4*i+2]*B + xor T3, T3 + add qword [rDst+idx*sizeof(qword)+sizeof(qword)], T1 + adc T2, rax + mov rax, qword [rSrc+idx*sizeof(qword)+sizeof(qword)*3] + adc T3, rdx + + mul rcx ; a[4*i+3]*B + xor T0, T0 + add qword [rDst+idx*sizeof(qword)+sizeof(qword)*2], T2 + adc T3, rax + mov rax, qword [rSrc+idx*sizeof(qword)+sizeof(qword)*4] + adc T0, rdx + + mul rcx ; a[4*i+4]*B + xor T1, T1 + add qword [rDst+idx*sizeof(qword)+sizeof(qword)*3], T3 + adc T0, rax + mov rax, qword [rSrc+idx*sizeof(qword)+sizeof(qword)*5] + adc T1, rdx + + add idx, 4 + jnc .muladd_loop4 + +.skip_muladd_loop4: + mul rcx + xor T2, T2 + add qword [rDst+idx*sizeof(qword)], T0 + adc T1, rax + adc T2, rdx + + cmp idx, 2 + ja .fin_mul1x4n_2 ; idx=3 + jz .fin_mul1x4n_3 ; idx=2 + jp .fin_mul1x4n_4 ; idx=1 + ; .fin_mul1x4n_1 ; idx=0 + +.fin_mul1x4n_1: + mov rax, qword [rSrc+idx*sizeof(qword)+sizeof(qword)*2] + mul rcx + xor T3, T3 + add qword [rDst+idx*sizeof(qword)+sizeof(qword)], T1 + adc T2, rax + mov rax, qword [rSrc+idx*sizeof(qword)+sizeof(qword)*3] + adc T3, rdx + + mul rcx + xor T0, T0 + add qword [rDst+idx*sizeof(qword)+sizeof(qword)*2], T2 + adc T3, rax + mov rax, qword [rSrc+idx*sizeof(qword)+sizeof(qword)*4] + adc T0, rdx + + mul rcx + xor T1, T1 + add qword [rDst+idx*sizeof(qword)+sizeof(qword)*3], T3 + adc T0, rax + adc rdx, 0 + add qword [rDst+idx*sizeof(qword)+sizeof(qword)*4], T0 + adc rdx, 0 + mov rax, rdx + jmp .exit + +.fin_mul1x4n_4: + mov rax, qword [rSrc+idx*sizeof(qword)+sizeof(qword)*2] + mul rcx + xor T3, T3 + add qword [rDst+idx*sizeof(qword)+sizeof(qword)], T1 + adc T2, rax + mov rax, qword [rSrc+idx*sizeof(qword)+sizeof(qword)*3] + adc T3, rdx + + mul rcx + xor T0, T0 + add qword [rDst+idx*sizeof(qword)+sizeof(qword)*2], T2 + adc T3, rax + adc rdx, 0 + add qword [rDst+idx*sizeof(qword)+sizeof(qword)*3], T3 + adc rdx, 0 + mov rax, rdx + jmp .exit + +.fin_mul1x4n_3: + mov rax, qword [rSrc+idx*sizeof(qword)+sizeof(qword)*2] + mul rcx + xor T3, T3 + add qword [rDst+idx*sizeof(qword)+sizeof(qword)], T1 + adc T2, rax + adc rdx, 0 + add qword [rDst+idx*sizeof(qword)+sizeof(qword)*2], T2 + adc rdx, 0 + mov rax, rdx + jmp .exit + +.fin_mul1x4n_2: + add qword [rDst+idx*sizeof(qword)+sizeof(qword)], T1 + adc T2, 0 + mov rax, T2 + +.exit: + REST_XMM + REST_GPR + ret +ENDFUNC cpAddMulDgt_BNU + +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumul.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumul.inc new file mode 100644 index 0000000..481702f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumul.inc @@ -0,0 +1,666 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Low level Big Number multiplication Support +; +; + +%ifndef _PCPBNUMUL_INC_ +%assign _PCPBNUMUL_INC_ 1 + +%include "pcpbnumul_basic.inc" + +%macro CLEAR 2.nolist + %xdefine %%rPtr %1 + %xdefine %%rLen %2 + +%%L_1: + mov qword [%%rPtr], rax + add %%rPtr, sizeof(qword) + sub %%rLen, 1 + jnz %%L_1 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; (8*n)x(8*m) multiplier +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_8Nx8M,PRIVATE + push rbx ;; nsB + push rdi ;; pR + push rsi ;; pA + push rdx ;; nsA + +;;; +;;; init +;;; +.mul_loopA: + push rdx ;; nsA + call mla_8x8 + add rdi, sizeof(qword)*8 ; adv pR + add rsi, sizeof(qword)*8 ; adv pA + pop rdx ; nsA-- + sub rdx, 8 + jnz .mul_loopA + + mov qword [rdi+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*2],r10 + mov qword [rdi+sizeof(qword)*3],r11 + mov qword [rdi+sizeof(qword)*4],r12 + mov qword [rdi+sizeof(qword)*5],r13 + mov qword [rdi+sizeof(qword)*6],r14 + mov qword [rdi+sizeof(qword)*7],r15 + + jmp .mla_entry + +.mla_loopB: + push rbx ;; nsB + + push rdi ;; pR + push rsi ;; pA + push rdx ;; nsA + + xor rax, rax ; c-flag + push rax + + mov r8, qword [rdi+sizeof(qword)*0] + mov r9, qword [rdi+sizeof(qword)*1] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + +.loopA: + push rdx ; nsA + call mla_8x8 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + pop rdx ; nsA-- + sub rdx, 8 + jz .exit_loopA + + pop rax ; restore c-flag + neg rax + op_reg_mem adc, r8, qword [rdi+sizeof(qword)*0], rax + op_reg_mem adc, r9, qword [rdi+sizeof(qword)*1], rax + op_reg_mem adc, r10,qword [rdi+sizeof(qword)*2], rax + op_reg_mem adc, r11,qword [rdi+sizeof(qword)*3], rax + op_reg_mem adc, r12,qword [rdi+sizeof(qword)*4], rax + op_reg_mem adc, r13,qword [rdi+sizeof(qword)*5], rax + op_reg_mem adc, r14,qword [rdi+sizeof(qword)*6], rax + op_reg_mem adc, r15,qword [rdi+sizeof(qword)*7], rax + sbb rax, rax ; save c-flag + push rax + jmp .loopA + +.exit_loopA: + pop rax ; restore c-flag + neg rax + adc r8, 0 + mov qword [rdi+sizeof(qword)*0], r8 + adc r9, 0 + mov qword [rdi+sizeof(qword)*1], r9 + adc r10,0 + mov qword [rdi+sizeof(qword)*2],r10 + adc r11,0 + mov qword [rdi+sizeof(qword)*3],r11 + adc r12,0 + mov qword [rdi+sizeof(qword)*4],r12 + adc r13,0 + mov qword [rdi+sizeof(qword)*5],r13 + adc r14,0 + mov qword [rdi+sizeof(qword)*6],r14 + adc r15,0 + mov qword [rdi+sizeof(qword)*7],r15 + +.mla_entry: + pop rdx ; restore nsA + pop rsi ; restore pA + pop rdi + add rdi, sizeof(qword)*8 + + add rcx, sizeof(qword)*8 + pop rbx ;; nsB-- + sub rbx, 8 + jnz .mla_loopB + ret +ENDFUNC mul_8Nx8M + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; simplest general case N x M multiplier +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_simple,PRIVATE + xor rax, rax ; carry = 0 + + mov r11, rdx + cmp r11, rbx + jge .ms_mla_entry + SWAP r11, rbx + SWAP rsi, rcx + jmp .ms_mla_entry + +.ms_loopB: + push rbx ;; nsB + push rdi ;; pR + push rsi ;; pA + push r11 ;; nsA + push rax ;; save previous pass carry + + mov rdx, qword [rcx] ; pB[] + xor r10, r10 ; extension +.ms_loopA: + gsmulx r9, r8, qword [rsi] + add rdi, sizeof(qword) + add rsi, sizeof(qword) + add r8, r10 + adc r9, 0 + add r8, qword [rdi-sizeof(qword)] + adc r9, 0 + mov qword [rdi-sizeof(qword)], r8 + mov r10, r9 + sub r11, 1 + jnz .ms_loopA + + pop rax ; restore carry + shr rax, 1 + + adc r10, qword [rdi] + mov qword [rdi], r10 + adc rax, 0 ; save carry + + pop r11 ; restore nsA + pop rsi ; restore pA + pop rdi ; restore pR + pop rbx ; nsB + + add rdi, sizeof(qword) + add rcx, sizeof(qword) + +.ms_mla_entry: + sub rbx, 1 + jnc .ms_loopB + ret +ENDFUNC mla_simple + + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; general case N x M multiplier +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_NxM,PRIVATE +; +; stack struct +; +%assign nsB_cnt 0 ; rest of B operand +%assign ptrR nsB_cnt+sizeof(qword) ; current product pointer +%assign ptrA ptrR+sizeof(qword) ; ponter to A operand (pA) +%assign nsA ptrA+sizeof(qword) ; length of A operand (nsA) +%assign nsA_cnt nsA+sizeof(qword) ; rest of A operand +%assign carry nsA_cnt+sizeof(qword) ; carry +%assign tailproc carry+sizeof(qword) ; mla_8xn procedure +%assign stack_mem tailproc+sizeof(qword) ; size of stack allocation + + sub rsp, stack_mem ; allocate stack + + cmp rbx, 8 + jge .regular_entry + cmp rdx, 8 + jge .irregular_entry + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; degradated case +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ; clear product + mov r8, rdx + add r8, rbx + mov rbp, rdi + xor rax, rax + CLEAR rbp, r8 + + call mla_simple + jmp .quit + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; irregular init +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +.irregular_entry: + mov [rsp+nsB_cnt], rbx + mov [rsp+nsA], rdx + mov [rsp+nsA_cnt], rdx + + GET_EP rax, mla_8xl_tail, rbx, rbp ; tail procedure (mla_8xn) address + mov [rsp+tailproc], rax + + jmp .irr_init_entry + +.irr_init_loop: + mov [rsp+nsA_cnt], rdx ; save A counter + call rax + + mov rbx, [rsp+nsB_cnt] + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*0], r8 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*1], r9 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*2], r10 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*3], r11 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*4], r12 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*5], r13 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*6], r14 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*7], r15 + + add rsi, sizeof(qword)*8 + add rdi, sizeof(qword)*8 + + xor r8, r8 + xor r9, r9 + xor r10,r10 + xor r11,r11 + xor r12,r12 + xor r13,r13 + xor r14,r14 + xor r15,r15 + + ; load data depending on nsB length + mov r8, qword [rdi] + cmp rbx, 1 + jz .continue + mov r9, qword [rdi+sizeof(qword)] + cmp rbx, 2 + jz .continue + mov r10, qword [rdi+sizeof(qword)*2] + cmp rbx, 3 + jz .continue + mov r11, qword [rdi+sizeof(qword)*3] + cmp rbx, 4 + jz .continue + mov r12, qword [rdi+sizeof(qword)*4] + cmp rbx, 5 + jz .continue + mov r13, qword [rdi+sizeof(qword)*5] + cmp rbx, 6 + jz .continue + mov r14, qword [rdi+sizeof(qword)*6] +.continue: + mov rdx, [rsp+nsA_cnt] ; nsA + +.irr_init_entry: + sub rdx, 8 + mov rax, [rsp+tailproc] + jnc .irr_init_loop + + add rdx, 8 + jz .quit + + ; clear uninitialized rest of product + lea rbp, [rdi+rbx*sizeof(qword)] + xor rax, rax + CLEAR rbp, rdx + + mov rdx, [rsp+nsA_cnt] + call mla_simple + jmp .quit + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; regular ep +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +.regular_entry: + sub rbx, 8 + xor rax, rax + mov [rsp+nsB_cnt], rbx + mov [rsp+ptrR], rdi + mov [rsp+ptrA], rsi + mov [rsp+nsA], rdx + mov [rsp+nsA_cnt], rdx + mov [rsp+carry], rax + + mov rbp, rdx ; n = nsA %8 + and rbp, 7 + GET_EP rax, mla_8xl_tail, rbp ; tail procedure (mla_8xn) address + mov [rsp+tailproc], rax + +;; +;; regular init +;; + sub rdx, 8 ;; nsA counter +.init_loopA: + mov [rsp+nsA_cnt], rdx ; nsA + call mla_8x8 + mov rdx, [rsp+nsA_cnt] + add rdi, sizeof(qword)*8 ; adv ptrR + add rsi, sizeof(qword)*8 ; adv ptrA + sub rdx, 8 ; nsA -= 8 + jnc .init_loopA + + add rdx, 8 + jz .init_complete + + mov [rsp+nsA_cnt], rdx + + mov rax, [rsp+tailproc] + SWAP rcx, rsi + call rax + SWAP rcx, rsi + + mov rdx, [rsp+nsA_cnt] + lea rdi, [rdi+rdx*sizeof(qword)] + +.init_complete: + mov qword [rdi+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*2],r10 + mov qword [rdi+sizeof(qword)*3],r11 + mov qword [rdi+sizeof(qword)*4],r12 + mov qword [rdi+sizeof(qword)*5],r13 + mov qword [rdi+sizeof(qword)*6],r14 + mov qword [rdi+sizeof(qword)*7],r15 + + jmp .mla_entry + +;; +;; regular mla passes +;; +.mla_loopB: + mov [rsp+nsB_cnt], rbx ; update B conter + mov [rsp+ptrR], rdi ; update current product pointer + + xor rax, rax ; init carry + mov [rsp+carry], rax + + mov r8, qword [rdi+sizeof(qword)*0] + mov r9, qword [rdi+sizeof(qword)*1] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + sub rdx, 8 +.loopA: + mov [rsp+nsA_cnt], rdx ; save A counter + call mla_8x8 + mov rdx, [rsp+nsA_cnt] ; A counter + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + sub rdx, 8 ; nsA -= 8 + jc .exit_loopA + + mov rax, [rsp+carry] ; restore carry + shr rax, 1 + op_reg_mem adc, r8, qword [rdi+sizeof(qword)*0], rbx + op_reg_mem adc, r9, qword [rdi+sizeof(qword)*1], rbx + op_reg_mem adc, r10,qword [rdi+sizeof(qword)*2], rbx + op_reg_mem adc, r11,qword [rdi+sizeof(qword)*3], rbx + op_reg_mem adc, r12,qword [rdi+sizeof(qword)*4], rbx + op_reg_mem adc, r13,qword [rdi+sizeof(qword)*5], rbx + op_reg_mem adc, r14,qword [rdi+sizeof(qword)*6], rbx + op_reg_mem adc, r15,qword [rdi+sizeof(qword)*7], rbx + adc rax, 0 ; save carry + mov [rsp+carry], rax + jmp .loopA + +.exit_loopA: + add rdx, 8 + jz .complete_reg_loopB + + mov [rsp+nsA_cnt], rdx + + ; put zeros + xor rax, rax +.put_zero: + mov qword [rdi+rdx*sizeof(qword)], rax + add rdx,1 + cmp rdx, 8 + jl .put_zero + + mov rax, [rsp+carry] ; restore carry + shr rax, 1 + op_reg_mem adc, r8, [rdi+sizeof(qword)*0], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*1], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*2], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*3], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*4], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*5], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*6], rbx + op_reg_mem adc, r15,[rdi+sizeof(qword)*7], rbx + adc rax, 0 ; save carry + mov [rsp+carry], rax + + mov rax, [rsp+tailproc] + SWAP rcx, rsi + call rax + SWAP rcx, rsi + + mov rdx, [rsp+nsA_cnt] ; restore nsA + lea rdi, [rdi+rdx*sizeof(qword)] + + mov rax, [rsp+carry] ; restore carry + shr rax, 1 + + dec rdx + jz .mt_1 + dec rdx + jz .mt_2 + dec rdx + jz .mt_3 + dec rdx + jz .mt_4 + dec rdx + jz .mt_5 + dec rdx + jz .mt_6 +.mt_7: adc r9, 0 +.mt_6: adc r10,0 +.mt_5: adc r11,0 +.mt_4: adc r12,0 +.mt_3: adc r13,0 +.mt_2: adc r14,0 +.mt_1: adc r15,0 + mov qword [rdi+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*2],r10 + mov qword [rdi+sizeof(qword)*3],r11 + mov qword [rdi+sizeof(qword)*4],r12 + mov qword [rdi+sizeof(qword)*5],r13 + mov qword [rdi+sizeof(qword)*6],r14 + mov qword [rdi+sizeof(qword)*7],r15 + jmp .mla_entry + +.complete_reg_loopB: + mov rax, [rsp+carry] ; restore carry + add r8, rax + adc r9, 0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + adc r14,0 + adc r15,0 + mov qword [rdi+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*2],r10 + mov qword [rdi+sizeof(qword)*3],r11 + mov qword [rdi+sizeof(qword)*4],r12 + mov qword [rdi+sizeof(qword)*5],r13 + mov qword [rdi+sizeof(qword)*6],r14 + mov qword [rdi+sizeof(qword)*7],r15 + +.mla_entry: + ; restore: + mov rbx, [rsp+nsB_cnt] ; - B counter + mov rdi, [rsp+ptrR] ; - ptrR + mov rdx, [rsp+nsA] ; - nsA + mov rsi, [rsp+ptrA] ; - pA + + add rcx, sizeof(qword)*8 ; adv pB + add rdi, sizeof(qword)*8 ; adv ptrR + sub rbx, 8 ; nsB -= 8 + jnc .mla_loopB + + add rbx, 8 + jz .quit + +;; +;; final mla pass +;; + mov [rsp+nsB_cnt], rbx + + GET_EP rax, mla_8xl_tail, rbx, rbp ; tail procedure (mla_8xn) address + mov [rsp+tailproc], rax + + ; clear uninitialized rest of product + lea rbp, [rdi+rdx*sizeof(qword)] + xor rax, rax + CLEAR rbp, rbx + + xor rax, rax ; init carry + mov [rsp+carry], rax + + sub rdx, 8 +.tail_loopA: + mov r8, qword [rdi] + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + mov [rsp+nsA_cnt], rdx ; save A counter + mov rax, [rsp+tailproc] + call rax + +.entry_tail_loopA: + mov rax, [rsp+carry] ; restore carry + shr rax, 1 + adc r8, 0 + adc r9, 0 + adc r10, 0 + adc r11, 0 + adc r12, 0 + adc r13, 0 + adc r14, 0 + adc r15, 0 + adc rax, 0 + + mov rbx, [rsp+nsB_cnt] ; nsB + mov rbp, rbx + dec rbp + jz .tt_1 + dec rbp + jz .tt_2 + dec rbp + jz .tt_3 + dec rbp + jz .tt_4 + dec rbp + jz .tt_5 + dec rbp + jz .tt_6 +.tt_7: op_reg_mem adc, r9, [rdi+rbx*sizeof(qword)+sizeof(qword)*1], rbp +.tt_6: op_reg_mem adc, r10,[rdi+rbx*sizeof(qword)+sizeof(qword)*2], rbp +.tt_5: op_reg_mem adc, r11,[rdi+rbx*sizeof(qword)+sizeof(qword)*3], rbp +.tt_4: op_reg_mem adc, r12,[rdi+rbx*sizeof(qword)+sizeof(qword)*4], rbp +.tt_3: op_reg_mem adc, r13,[rdi+rbx*sizeof(qword)+sizeof(qword)*5], rbp +.tt_2: op_reg_mem adc, r14,[rdi+rbx*sizeof(qword)+sizeof(qword)*6], rbp +.tt_1: op_reg_mem adc, r15,[rdi+rbx*sizeof(qword)+sizeof(qword)*7], rbp + adc rax, 0 + mov [rsp+carry], rax ; update carry + + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*0], r8 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*1], r9 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*2], r10 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*3], r11 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*4], r12 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*5], r13 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*6], r14 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*7], r15 + + mov rdx, [rsp+nsA_cnt] ; A counter + add rsi, sizeof(qword)*8 ; adv pA + add rdi, sizeof(qword)*8 ; adv pR + sub rdx, 8 ; nsA -= 8 + jnc .tail_loopA + + add rdx, 8 + jz .quit + + ; carry propagation + mov rax, [rsp+carry] + mov rbp, rbx + + dec rbp + mov r8, qword [rdi+rbx*sizeof(qword)+sizeof(qword)*0] + add r8, rax + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*0], r8 + jz .simple + dec rbp + mov r9, qword [rdi+rbx*sizeof(qword)+sizeof(qword)*1] + adc r9, 0 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*1], r9 + jz .simple + dec rbp + mov r10,qword [rdi+rbx*sizeof(qword)+sizeof(qword)*2] + adc r10, 0 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*2], r10 + jz .simple + dec rbp + mov r11,qword [rdi+rbx*sizeof(qword)+sizeof(qword)*3] + adc r11, 0 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*3], r11 + jz .simple + dec rbp + mov r12,qword [rdi+rbx*sizeof(qword)+sizeof(qword)*4] + adc r12, 0 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*4], r12 + jz .simple + dec rbp + mov r13,qword [rdi+rbx*sizeof(qword)+sizeof(qword)*5] + adc r13, 0 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*5], r13 + jz .simple + dec rbp + mov r14,qword [rdi+rbx*sizeof(qword)+sizeof(qword)*6] + adc r14, 0 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*6], r14 + +.simple: + call mla_simple + + +.quit: + add rsp, stack_mem ; release stack + ret +ENDFUNC mul_NxM + + +%endif ;; _PCPBNUMUL_INC_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumul_basic.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumul_basic.inc new file mode 100644 index 0000000..71f9b41 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumul_basic.inc @@ -0,0 +1,947 @@ +;=============================================================================== +; Copyright (C) 2013 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Low level Big Number multiplication Support +; +; + +%ifndef _PCPBNUMUL_BASIC_INC_ +%assign _PCPBNUMUL_BASIC_INC_ 1 + +%include "pcpmulx.inc" +%include "pcpbnumul_fix.inc" + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; short size (1-8 qwords) operand mla and mul operations +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; +;; 1x1 multiplication +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_1x1,PRIVATE + mov rdx, qword [rcx] + MLA_FIX 1, rdi, rsi, rbx,rbp, r8 + ret +ENDFUNC mla_1x1 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_1x1,PRIVATE + mov rdx,qword [rcx] + gsmulx rdx, rax, qword [rsi] + mov qword [rdi], rax + mov qword [rdi+sizeof(qword)], rdx + ret +ENDFUNC mul_1x1 + + +;; +;; 2x2 multiplication +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_2x2,PRIVATE + mov rdx, qword [rcx] + MLA_FIX 2, rdi, rsi, rbx,rbp, r8,r9 + mov rdx, qword [rcx+sizeof(qword)] + MLA_FIX 2, {rdi+sizeof(qword)}, rsi, rbx,rbp, r8,r9 + ret +ENDFUNC mla_2x2 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_2x2,PRIVATE + call mla_2x2 + mov qword [rdi+sizeof(qword)*2], r8 + mov qword [rdi+sizeof(qword)*3], r9 + ret +ENDFUNC mul_2x2 + + + +;; +;; 3x3 multiplication +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_3x3,PRIVATE + mov rdx, qword [rcx] + MLA_FIX 3, rdi, rsi, rbx,rbp, r8,r9,r10 + mov rdx, qword [rcx+sizeof(qword)] + MLA_FIX 3, {rdi+sizeof(qword)}, rsi, rbx,rbp, r8,r9,r10 + mov rdx, qword [rcx+sizeof(qword)*2] + MLA_FIX 3, {rdi+sizeof(qword)*2}, rsi, rbx,rbp, r8,r9,r10 + ret +ENDFUNC mla_3x3 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_3x3,PRIVATE + call mla_3x3 + mov qword [rdi+sizeof(qword)*3], r8 + mov qword [rdi+sizeof(qword)*4], r9 + mov qword [rdi+sizeof(qword)*5], r10 + ret +ENDFUNC mul_3x3 + + + +;; +;; 4x{2,4} multiplication +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_4x2,PRIVATE + mov rdx, qword [rcx] + MLA_FIX 4, rdi, rsi, rbx,rbp, r8,r9,r10,r11 + mov rdx, qword [rcx+sizeof(qword)] + MLA_FIX 4, {rdi+sizeof(qword)}, rsi, rbx,rbp, r8,r9,r10,r11 + ret +ENDFUNC mla_4x2 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_4x4,PRIVATE + call mla_4x2 + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_4x2 + sub rdi, sizeof(qword)*2 + sub rcx, sizeof(qword)*2 + ret +ENDFUNC mla_4x4 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_4x4,PRIVATE + call mla_4x4 + mov qword [rdi+sizeof(qword)*4], r8 + mov qword [rdi+sizeof(qword)*5], r9 + mov qword [rdi+sizeof(qword)*6], r10 + mov qword [rdi+sizeof(qword)*7], r11 + ret +ENDFUNC mul_4x4 + + + +;; +;; 5x{2,5} multiplication +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_5x2,PRIVATE + mov rdx, [rcx+sizeof(qword)*0] + MLA_FIX 5, rdi, rsi, rbx,rbp, r8,r9,r10,r11,r12 + mov rdx, [rcx+sizeof(qword)*1] + MLA_FIX 5, {rdi+sizeof(qword)}, rsi, rbx,rbp, r8,r9,r10,r11,r12 + ret +ENDFUNC mla_5x2 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_5x5,PRIVATE + mov rdx, [rcx] + MLA_FIX 5, rdi, rsi, rbx,rbp, r8,r9,r10,r11,r12 + + add rdi, sizeof(qword) + add rcx, sizeof(qword) + call mla_5x2 + + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_5x2 + + sub rdi, sizeof(qword)*3 + sub rcx, sizeof(qword)*3 + ret +ENDFUNC mla_5x5 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_5x5,PRIVATE + call mla_5x5 + mov qword [rdi+sizeof(qword)*5], r8 + mov qword [rdi+sizeof(qword)*6], r9 + mov qword [rdi+sizeof(qword)*7], r10 + mov qword [rdi+sizeof(qword)*8], r11 + mov qword [rdi+sizeof(qword)*9], r12 + ret +ENDFUNC mul_5x5 + + + +;; +;; 6x{2,6} multiplication +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_6x2,PRIVATE + mov rdx, [rcx+sizeof(qword)*0] + MLA_FIX 6, rdi, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13 + mov rdx, [rcx+sizeof(qword)*1] + MLA_FIX 6, {rdi+sizeof(qword)}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13 + ret +ENDFUNC mla_6x2 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_6x6,PRIVATE + call mla_6x2 + + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_6x2 + + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_6x2 + + sub rdi, sizeof(qword)*4 + sub rcx, sizeof(qword)*4 + ret +ENDFUNC mla_6x6 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_6x6,PRIVATE + call mla_6x6 + mov qword [rdi+sizeof(qword)*6], r8 + mov qword [rdi+sizeof(qword)*7], r9 + mov qword [rdi+sizeof(qword)*8], r10 + mov qword [rdi+sizeof(qword)*9], r11 + mov qword [rdi+sizeof(qword)*10],r12 + mov qword [rdi+sizeof(qword)*11],r13 + ret +ENDFUNC mul_6x6 + + + +;; +;; 7x{2,7} multiplication +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_7x2,PRIVATE + mov rdx, [rcx+sizeof(qword)*0] + MLA_FIX 7, rdi, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14 + mov rdx, [rcx+sizeof(qword)*1] + MLA_FIX 7, {rdi+sizeof(qword)}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14 + ret +ENDFUNC mla_7x2 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_7x7,PRIVATE + mov rdx, [rcx] + MLA_FIX 7, rdi, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14 + + add rdi, sizeof(qword) + add rcx, sizeof(qword) + call mla_7x2 + + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_7x2 + + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_7x2 + + sub rdi, sizeof(qword)*5 + sub rcx, sizeof(qword)*5 + ret +ENDFUNC mla_7x7 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_7x7,PRIVATE + call mla_7x7 + mov qword [rdi+sizeof(qword)*7], r8 + mov qword [rdi+sizeof(qword)*8], r9 + mov qword [rdi+sizeof(qword)*9], r10 + mov qword [rdi+sizeof(qword)*10],r11 + mov qword [rdi+sizeof(qword)*11],r12 + mov qword [rdi+sizeof(qword)*12],r13 + mov qword [rdi+sizeof(qword)*13],r14 + ret +ENDFUNC mul_7x7 + + + +;; +;; 8x{1,2,3,4,5,6,7,8} multiplication +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_8x1,PRIVATE + mov rdx, [rcx] + MLA_FIX 8, rdi, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + ret +ENDFUNC mla_8x1 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_8x2,PRIVATE + mov rdx, [rcx+sizeof(qword)*0] + MLA_FIX 8, rdi, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*1] + MLA_FIX 8, {rdi+sizeof(qword)}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + ret +ENDFUNC mla_8x2 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_8x3,PRIVATE + call mla_8x1 + add rdi, sizeof(qword) + add rcx, sizeof(qword) + call mla_8x2 + sub rdi, sizeof(qword) + sub rcx, sizeof(qword) + ret +ENDFUNC mla_8x3 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_8x4,PRIVATE + call mla_8x2 + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x2 + sub rdi, sizeof(qword)*2 + sub rcx, sizeof(qword)*2 + ret +ENDFUNC mla_8x4 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_8x5,PRIVATE + call mla_8x1 + add rdi, sizeof(qword) + add rcx, sizeof(qword) + call mla_8x4 + sub rdi, sizeof(qword) + sub rcx, sizeof(qword) + ret +ENDFUNC mla_8x5 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_8x6,PRIVATE + call mla_8x2 + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x4 + sub rdi, sizeof(qword)*2 + sub rcx, sizeof(qword)*2 + ret +ENDFUNC mla_8x6 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_8x7,PRIVATE + call mla_8x1 + add rdi, sizeof(qword) + add rcx, sizeof(qword) + call mla_8x6 + sub rdi, sizeof(qword) + sub rcx, sizeof(qword) + ret +ENDFUNC mla_8x7 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_8x8,PRIVATE + call mla_8x2 + + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x2 + + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x2 + + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x2 + + sub rdi, sizeof(qword)*6 + sub rcx, sizeof(qword)*6 + ret +ENDFUNC mla_8x8 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_8x8,PRIVATE + call mla_8x8 + mov qword [rdi+sizeof(qword)*8], r8 + mov qword [rdi+sizeof(qword)*9], r9 + mov qword [rdi+sizeof(qword)*10],r10 + mov qword [rdi+sizeof(qword)*11],r11 + mov qword [rdi+sizeof(qword)*12],r12 + mov qword [rdi+sizeof(qword)*13],r13 + mov qword [rdi+sizeof(qword)*14],r14 + mov qword [rdi+sizeof(qword)*15],r15 + ret +ENDFUNC mul_8x8 + + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; middle size (9-16 qwords) operand mla and mul operations +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_9x9,PRIVATE + call mla_8x8 + + mov rdx, [rcx+sizeof(qword)*8] + MLA_FIX 8, {rdi+sizeof(qword)*8}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + push r15 + + mov rdx, [rsi+sizeof(qword)*8] + mov r15, [rdi+sizeof(qword)*8] + MLA_FIX 8, {rdi+sizeof(qword)*8}, rcx, rbx,rbp, r15,r8,r9,r10,r11,r12,r13,r14 + mov qword [rdi+sizeof(qword)*9], r15 + + gsmulx r15, rbp, [rcx+sizeof(qword)*8] + pop rax + add r14, rax + adc r15,0 + add r14, rbp + adc r15, 0 + + mov qword [rdi+sizeof(qword)*10],r8 + mov qword [rdi+sizeof(qword)*11],r9 + mov qword [rdi+sizeof(qword)*12],r10 + mov qword [rdi+sizeof(qword)*13],r11 + mov qword [rdi+sizeof(qword)*14],r12 + mov qword [rdi+sizeof(qword)*15],r13 + mov qword [rdi+sizeof(qword)*16],r14 + mov qword [rdi+sizeof(qword)*17],r15 + ret +ENDFUNC mul_9x9 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_10x10,PRIVATE +; A0*B0 + call mla_8x8 + add rdi, sizeof(qword)*8 + +; + A0*B1 + add rcx, sizeof(qword)*8 + call mla_8x2 + push r15 + push r14 + +; + A1*B0 + add rsi, sizeof(qword)*8 + sub rcx, sizeof(qword)*8 + SWAP rcx, rsi + + mov r15, r13 + mov r14, r12 + mov r13, r11 + mov r12, r10 + mov r11, r9 + mov r10, r8 + mov r9, qword [rdi+sizeof(qword)*1] + mov r8, qword [rdi+sizeof(qword)*0] + call mla_8x2 + + mov qword [rdi+sizeof(qword)*2],r8 + mov qword [rdi+sizeof(qword)*3],r9 + mov qword [rdi+sizeof(qword)*4],r10 + mov qword [rdi+sizeof(qword)*5],r11 + mov qword [rdi+sizeof(qword)*6],r12 + mov qword [rdi+sizeof(qword)*7],r13 + add rdi, sizeof(qword)*8 + + xor r10, r10 + pop r8 + pop r9 + add r8, r14 + adc r9, r15 + adc r10,0 + +; + A1*B1 + SWAP rcx, rsi + add rcx, sizeof(qword)*8 + call mla_2x2 + add rdi, sizeof(qword)*2 + + add r8, r10 + adc r9, 0 + mov qword [rdi+sizeof(qword)*0],r8 + mov qword [rdi+sizeof(qword)*1],r9 + ret +ENDFUNC mul_10x10 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_11x11,PRIVATE +; A0*B0 + call mla_8x8 + add rdi, sizeof(qword)*8 + +; + A0*B1 + add rcx, sizeof(qword)*8 + call mla_8x3 + push r15 + push r14 + push r13 + +; + A1*B0 + add rsi, sizeof(qword)*8 + sub rcx, sizeof(qword)*8 + SWAP rcx, rsi + + mov r15, r12 + mov r14, r11 + mov r13, r10 + mov r12, r9 + mov r11, r8 + mov r10,qword [rdi+sizeof(qword)*2] + mov r9, qword [rdi+sizeof(qword)*1] + mov r8, qword [rdi+sizeof(qword)*0] + call mla_8x3 + + mov qword [rdi+sizeof(qword)*3],r8 + mov qword [rdi+sizeof(qword)*4],r9 + mov qword [rdi+sizeof(qword)*5],r10 + mov qword [rdi+sizeof(qword)*6],r11 + mov qword [rdi+sizeof(qword)*7],r12 + add rdi, sizeof(qword)*8 + + xor r11, r11 + pop r8 + pop r9 + pop r10 + add r8, r13 + adc r9, r14 + adc r10,r15 + adc r11,0 + +; + A1*B1 + SWAP rcx, rsi + add rcx, sizeof(qword)*8 + call mla_3x3 + add rdi, sizeof(qword)*3 + + add r8, r11 + adc r9, 0 + adc r10,0 + mov qword [rdi+sizeof(qword)*0],r8 + mov qword [rdi+sizeof(qword)*1],r9 + mov qword [rdi+sizeof(qword)*2],r10 + ret +ENDFUNC mul_11x11 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_12x12,PRIVATE +; A0*B0 + call mla_8x8 + add rdi, sizeof(qword)*8 + +; + A0*B1 + add rcx, sizeof(qword)*8 + call mla_8x4 + push r15 + push r14 + push r13 + push r12 + +; + A1*B0 + add rsi, sizeof(qword)*8 + sub rcx, sizeof(qword)*8 + SWAP rcx, rsi + + mov r15, r11 + mov r14, r10 + mov r13, r9 + mov r12, r8 + mov r11,qword [rdi+sizeof(qword)*3] + mov r10,qword [rdi+sizeof(qword)*2] + mov r9, qword [rdi+sizeof(qword)*1] + mov r8, qword [rdi+sizeof(qword)*0] + call mla_8x4 + + mov qword [rdi+sizeof(qword)*4],r8 + mov qword [rdi+sizeof(qword)*5],r9 + mov qword [rdi+sizeof(qword)*6],r10 + mov qword [rdi+sizeof(qword)*7],r11 + add rdi, sizeof(qword)*8 + + xor rax, rax + pop r8 + pop r9 + pop r10 + pop r11 + add r8, r12 + adc r9, r13 + adc r10,r14 + adc r11,r15 + adc rax,0 + push rax + +; + A1*B1 + SWAP rcx, rsi + add rcx, sizeof(qword)*8 + call mla_4x4 + add rdi, sizeof(qword)*4 + + pop rax + add r8, rax + adc r9, 0 + adc r10,0 + adc r11,0 + mov qword [rdi+sizeof(qword)*0],r8 + mov qword [rdi+sizeof(qword)*1],r9 + mov qword [rdi+sizeof(qword)*2],r10 + mov qword [rdi+sizeof(qword)*3],r11 + ret +ENDFUNC mul_12x12 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_13x13,PRIVATE +; A0*B0 + call mla_8x8 + add rdi, sizeof(qword)*8 + +; + A0*B1 + add rcx, sizeof(qword)*8 + call mla_8x5 + push r15 + push r14 + push r13 + push r12 + push r11 + +; + A1*B0 + add rsi, sizeof(qword)*8 + sub rcx, sizeof(qword)*8 + SWAP rcx, rsi + + mov r15, r10 + mov r14, r9 + mov r13, r8 + mov r12,qword [rdi+sizeof(qword)*4] + mov r11,qword [rdi+sizeof(qword)*3] + mov r10,qword [rdi+sizeof(qword)*2] + mov r9, qword [rdi+sizeof(qword)*1] + mov r8, qword [rdi+sizeof(qword)*0] + call mla_8x5 + + mov qword [rdi+sizeof(qword)*5], r8 + mov qword [rdi+sizeof(qword)*6], r9 + mov qword [rdi+sizeof(qword)*7], r10 + add rdi, sizeof(qword)*8 + + xor rax, rax + pop r8 + pop r9 + pop r10 + pop rbx + pop rbp + add r8, r11 + adc r9, r12 + adc r10,r13 + adc rbx,r14 + adc rbp,r15 + adc rax,0 + push rax + + mov r11, rbx + mov r12, rbp + +; + A1*B1 + SWAP rcx, rsi + add rcx, sizeof(qword)*8 + call mla_5x5 + add rdi, sizeof(qword)*5 + + pop rax + add r8, rax + adc r9, 0 + adc r10,0 + adc r11,0 + adc r12,0 + mov qword [rdi+sizeof(qword)*0],r8 + mov qword [rdi+sizeof(qword)*1],r9 + mov qword [rdi+sizeof(qword)*2],r10 + mov qword [rdi+sizeof(qword)*3],r11 + mov qword [rdi+sizeof(qword)*4],r12 + ret +ENDFUNC mul_13x13 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_14x14,PRIVATE + call mla_7x7 + + add rdi, sizeof(qword)*7 + add rsi, sizeof(qword)*7 + call mla_7x7 + mov qword [rdi+sizeof(qword)*7], r8 + mov qword [rdi+sizeof(qword)*8], r9 + mov qword [rdi+sizeof(qword)*9], r10 + mov qword [rdi+sizeof(qword)*10],r11 + mov qword [rdi+sizeof(qword)*11],r12 + mov qword [rdi+sizeof(qword)*12],r13 + mov qword [rdi+sizeof(qword)*13],r14 + + mov r8, qword [rdi+sizeof(qword)*0] + mov r9, qword [rdi+sizeof(qword)*1] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + + add rcx, sizeof(qword)*7 + sub rsi, sizeof(qword)*7 + call mla_7x7 + + xor rdx, rdx + op_reg_mem add, r8, qword [rdi+sizeof(qword)*7], rax + op_reg_mem adc, r9, qword [rdi+sizeof(qword)*8], rax + op_reg_mem adc, r10,qword [rdi+sizeof(qword)*9], rax + op_reg_mem adc, r11,qword [rdi+sizeof(qword)*10],rax + op_reg_mem adc, r12,qword [rdi+sizeof(qword)*11],rax + op_reg_mem adc, r13,qword [rdi+sizeof(qword)*12],rax + op_reg_mem adc, r14,qword [rdi+sizeof(qword)*13],rax + adc rdx, 0 + push rdx + + add rdi, sizeof(qword)*7 + add rsi, sizeof(qword)*7 + call mla_7x7 + + sub rdi, sizeof(qword)*14 + sub rsi, sizeof(qword)*7 + sub rcx, sizeof(qword)*7 + + pop rdx + add r8, rdx + adc r9, 0 + adc r10, 0 + adc r11, 0 + adc r12, 0 + adc r13, 0 + adc r14, 0 + + mov qword [rdi+sizeof(qword)*21],r8 + mov qword [rdi+sizeof(qword)*22],r9 + mov qword [rdi+sizeof(qword)*23],r10 + mov qword [rdi+sizeof(qword)*24],r11 + mov qword [rdi+sizeof(qword)*25],r12 + mov qword [rdi+sizeof(qword)*26],r13 + mov qword [rdi+sizeof(qword)*27],r14 + ret +ENDFUNC mul_14x14 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_15x15,PRIVATE +; A0*B0 + call mla_8x8 + add rdi, sizeof(qword)*8 + +; + A0*B1 + add rcx, sizeof(qword)*8 + call mla_8x7 + mov qword [rdi+sizeof(qword)*7], r8 + mov qword [rdi+sizeof(qword)*8], r9 + mov qword [rdi+sizeof(qword)*9], r10 + mov qword [rdi+sizeof(qword)*10],r11 + mov qword [rdi+sizeof(qword)*11],r12 + mov qword [rdi+sizeof(qword)*12],r13 + mov qword [rdi+sizeof(qword)*13],r14 + mov qword [rdi+sizeof(qword)*14],r15 + +; + A1*B0 + add rsi, sizeof(qword)*8 + sub rcx, sizeof(qword)*8 + SWAP rcx, rsi + + mov r8, qword [rdi+sizeof(qword)*0] + mov r9, qword [rdi+sizeof(qword)*1] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + call mla_8x7 + + mov qword [rdi+sizeof(qword)*7], r8 + add rdi, sizeof(qword)*8 + +; + A1*B1 + SWAP rcx, rsi + add rcx, sizeof(qword)*8 + + mov r8, r9 + mov r9, r10 + mov r10,r11 + mov r11,r12 + mov r12,r13 + mov r13,r14 + mov r14,r15 + xor rdx, rdx + op_reg_mem add, r8, qword [rdi+sizeof(qword)*0], rax + op_reg_mem adc, r9, qword [rdi+sizeof(qword)*1], rax + op_reg_mem adc, r10,qword [rdi+sizeof(qword)*2], rax + op_reg_mem adc, r11,qword [rdi+sizeof(qword)*3], rax + op_reg_mem adc, r12,qword [rdi+sizeof(qword)*4], rax + op_reg_mem adc, r13,qword [rdi+sizeof(qword)*5], rax + op_reg_mem adc, r14,qword [rdi+sizeof(qword)*6], rax + adc rdx, 0 + push rdx + + call mla_7x7 + add rdi, sizeof(qword)*7 + + pop rax + add r8, rax + adc r9, 0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + adc r14,0 + mov qword [rdi+sizeof(qword)*0],r8 + mov qword [rdi+sizeof(qword)*1],r9 + mov qword [rdi+sizeof(qword)*2],r10 + mov qword [rdi+sizeof(qword)*3],r11 + mov qword [rdi+sizeof(qword)*4],r12 + mov qword [rdi+sizeof(qword)*5],r13 + mov qword [rdi+sizeof(qword)*6],r14 + ret +ENDFUNC mul_15x15 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_16x16,PRIVATE + call mla_8x8 + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + call mla_8x8 + mov qword [rdi+sizeof(qword)*8], r8 + mov qword [rdi+sizeof(qword)*9], r9 + mov qword [rdi+sizeof(qword)*10],r10 + mov qword [rdi+sizeof(qword)*11],r11 + mov qword [rdi+sizeof(qword)*12],r12 + mov qword [rdi+sizeof(qword)*13],r13 + mov qword [rdi+sizeof(qword)*14],r14 + mov qword [rdi+sizeof(qword)*15],r15 + + mov r8, qword [rdi+sizeof(qword)*0] + mov r9, qword [rdi+sizeof(qword)*1] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + add rcx, sizeof(qword)*8 + sub rsi, sizeof(qword)*8 + call mla_8x8 + + xor rdx, rdx + op_reg_mem add, r8, qword [rdi+sizeof(qword)*8], rax + op_reg_mem adc, r9, qword [rdi+sizeof(qword)*9], rax + op_reg_mem adc, r10,qword [rdi+sizeof(qword)*10],rax + op_reg_mem adc, r11,qword [rdi+sizeof(qword)*11],rax + op_reg_mem adc, r12,qword [rdi+sizeof(qword)*12],rax + op_reg_mem adc, r13,qword [rdi+sizeof(qword)*13],rax + op_reg_mem adc, r14,qword [rdi+sizeof(qword)*14],rax + op_reg_mem adc, r15,qword [rdi+sizeof(qword)*15],rax + adc rdx, 0 + push rdx + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + call mla_8x8 + + sub rdi, sizeof(qword)*16 + sub rsi, sizeof(qword)*8 + sub rcx, sizeof(qword)*8 + + pop rdx + add r8, rdx + adc r9, 0 + adc r10, 0 + adc r11, 0 + adc r12, 0 + adc r13, 0 + adc r14, 0 + adc r15, 0 + + mov qword [rdi+sizeof(qword)*24],r8 + mov qword [rdi+sizeof(qword)*25],r9 + mov qword [rdi+sizeof(qword)*26],r10 + mov qword [rdi+sizeof(qword)*27],r11 + mov qword [rdi+sizeof(qword)*28],r12 + mov qword [rdi+sizeof(qword)*29],r13 + mov qword [rdi+sizeof(qword)*30],r14 + mov qword [rdi+sizeof(qword)*31],r15 + ret +ENDFUNC mul_16x16 + + +mul_lxl_basic DQ mul_1x1 - mul_lxl_basic + DQ mul_2x2 - mul_lxl_basic + DQ mul_3x3 - mul_lxl_basic + DQ mul_4x4 - mul_lxl_basic + DQ mul_5x5 - mul_lxl_basic + DQ mul_6x6 - mul_lxl_basic + DQ mul_7x7 - mul_lxl_basic + DQ mul_8x8 - mul_lxl_basic + DQ mul_9x9 - mul_lxl_basic + DQ mul_10x10-mul_lxl_basic + DQ mul_11x11-mul_lxl_basic + DQ mul_12x12-mul_lxl_basic + DQ mul_13x13-mul_lxl_basic + DQ mul_14x14-mul_lxl_basic + DQ mul_15x15-mul_lxl_basic + DQ mul_16x16-mul_lxl_basic + +mla_lxl_short DQ mla_1x1 - mla_lxl_short + DQ mla_2x2 - mla_lxl_short + DQ mla_3x3 - mla_lxl_short + DQ mla_4x4 - mla_lxl_short + DQ mla_5x5 - mla_lxl_short + DQ mla_6x6 - mla_lxl_short + DQ mla_7x7 - mla_lxl_short + +mla_8xl_tail DQ mla_8x1 - mla_8xl_tail + DQ mla_8x2 - mla_8xl_tail + DQ mla_8x3 - mla_8xl_tail + DQ mla_8x4 - mla_8xl_tail + DQ mla_8x5 - mla_8xl_tail + DQ mla_8x6 - mla_8xl_tail + DQ mla_8x7 - mla_8xl_tail + +%endif ;; _PCPBNUMUL_BASIC_INC_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumul_fix.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumul_fix.inc new file mode 100644 index 0000000..3093f2e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumul_fix.inc @@ -0,0 +1,406 @@ +;=============================================================================== +; Copyright (C) 2013 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Low level Big Number multiplication Support +; +; + +%ifndef _PCPBNUMUL_FIX_ADCX_INC_ +%assign _PCPBNUMUL_FIX_ADCX_INC_ 1 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; macro used to replace op reg, mem instructions +%macro SWAP 2.nolist + %xdefine %%r1 %1 + %xdefine %%r2 %2 + + xor %%r1, %%r2 + xor %%r2, %%r1 + xor %%r1, %%r2 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; macro used extract proc ep from the table +%macro GET_EP 3-4.nolist + %xdefine %%regEp %1 + %xdefine %%Table %2 + %xdefine %%regIdx %3 + %xdefine %%tmp %4 + + lea %%regEp, [rel %%Table] + %ifnempty %%tmp + mov %%tmp, [%%regEp+%%regIdx*sizeof(qword)-sizeof(qword)] + add %%regEp, %%tmp + %else + mov %%regIdx, [%%regEp+%%regIdx*sizeof(qword)-sizeof(qword)] + add %%regEp, %%regIdx + %endif +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; op DST, mem_src +%macro op_reg_mem 4.nolist + %xdefine %%opCode %1 + %xdefine %%dst %2 + %xdefine %%mem_src %3 + %xdefine %%tmp %4 + + mov %%tmp, %%mem_src + %%opCode %%dst, %%tmp +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; MEM_DST = (op src1, mem_src2) +%macro op_mem_reg_mem 5.nolist + %xdefine %%opCode %1 + %xdefine %%mem_dst %2 + %xdefine %%src1 %3 + %xdefine %%mem_src2 %4 + %xdefine %%tmp %5 + + op_reg_mem %%opCode, %%src1, %%mem_src2, %%tmp + mov %%mem_dst, %%src1 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; MEM_DST = op mem_src1, mem_src2 +%macro op_mem_mem 5.nolist + %xdefine %%opCode %1 + %xdefine %%mem_dst %2 + %xdefine %%mem_src1 %3 + %xdefine %%mem_src2 %4 + %xdefine %%tmp %5 + + mov %%tmp, %%mem_src1 + %%opCode %%tmp, %%mem_src2 + mov %%mem_dst, %%tmp +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;;; Inputs: pDst: Destination (upto 1024 bits, 16 qwords) +;;; pA: Multiplicand (upto 512 bits, 8 qwords) +;;; +;;; +;;; Uses registers: +;;; rdx - implicit multiplicand +;;; (HIP:LOP)- 64x64-bit product (mulx) +;;; X7,,X0 - high result +;;; +%macro MUL_FIX 12-13.nolist + %xdefine %%N %1 + %xdefine %%pDst %2 + %xdefine %%pA %3 + %xdefine %%HIP %4 + %xdefine %%LOP %5 + %xdefine %%X0 %6 + %xdefine %%X1 %7 + %xdefine %%X2 %8 + %xdefine %%X3 %9 + %xdefine %%X4 %10 + %xdefine %%X5 %11 + %xdefine %%X6 %12 + %xdefine %%X7 %13 + + gsmulx %%X0, %%LOP, [%%pA] ; (X0:LO) = a[0]*b + %ifnempty %%pDst + mov [%%pDst], %%LOP + %endif + + %if %%N > 1 + gsmulx %%X1, %%HIP, [%%pA+sizeof(qword)] ; (X1:LO) = a[1]*b + add %%X0, %%HIP + %if %%N == 2 + adc %%X1, 0 + %endif + + %if %%N > 2 + gsmulx %%X2, %%LOP, [%%pA+sizeof(qword)*2] ; (X2:LO) = a[2]*b + adc %%X1, %%LOP + %if %%N == 3 + adc %%X2, 0 + %endif + + %if %%N > 3 + gsmulx %%X3, %%HIP, [%%pA+sizeof(qword)*3] ; (X3:LO) = a[3]*b + adc %%X2, %%HIP + %if %%N == 4 + adc %%X3, 0 + %endif + + %if %%N > 4 + gsmulx %%X4, %%LOP, [%%pA+sizeof(qword)*4] ; (X4:LO) = a[4]*b + adc %%X3, %%LOP + %if %%N == 5 + adc %%X4, 0 + %endif + + %if %%N > 5 + gsmulx %%X5, %%HIP, [%%pA+sizeof(qword)*5] ; (X5:LO) = a[5]*b + adc %%X4, %%HIP + %if %%N == 6 + adc %%X5, 0 + %endif + + %if %%N > 6 + gsmulx %%X6, %%LOP, [%%pA+sizeof(qword)*6] ; (X6:LO) = a[6]*b + adc %%X5, %%LOP + %if %%N == 7 + adc %%X6, 0 + %endif + + %if %%N > 7 + gsmulx %%X7, %%HIP, [%%pA+sizeof(qword)*7] ; (X7:LO) = a[7]*b + adc %%X6, %%HIP + adc %%X7, 0 + %endif + %endif + %endif + %endif + %endif + %endif + %endif +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; {x7,x6,x5,x4,x3,x2,x1,x0},DCT[0] = {x7,x6,x5,x4,x3,x2,x1,x0} + A[7:0]*B[i] +;; +;; uses rax, rdx +;; rdx - implicit multiplicand B[i] (should to be preloaded) +;; rax - temporary product +;; (HIP:LOP)- 64x64-bit product (mulx) +;; X7,,X0 - high result +;; +%macro MLA_FIX 6-13.nolist + %xdefine %%N %1 + %xdefine %%pDst %2 + %xdefine %%pA %3 + %xdefine %%HIP %4 + %xdefine %%LOP %5 + %xdefine %%X0 %6 + %xdefine %%X1 %7 + %xdefine %%X2 %8 + %xdefine %%X3 %9 + %xdefine %%X4 %10 + %xdefine %%X5 %11 + %xdefine %%X6 %12 + %xdefine %%X7 %13 + + gsmulx %%HIP, rax, [%%pA] ; (HI:rax) = a[0]*b[i] + add rax, %%X0 + adc %%HIP, 0 + %ifnempty %%pDst + mov [%%pDst], rax + %endif + %if %%N == 1 + mov %%X0, %%HIP + %endif + + %if %%N > 1 + gsmulx %%LOP,%%X0, [%%pA+sizeof(qword)] ; (LO:X0) = a[1]*b[i] + add %%X0, %%X1 + adc %%LOP, 0 + add %%X0, %%HIP + adc %%LOP, 0 + %if %%N == 2 + mov %%X1, %%LOP + %endif + + %if %%N > 2 + gsmulx %%HIP,%%X1, [%%pA+sizeof(qword)*2] ; (HI:X1) = a[2]*b[i] + add %%X1, %%X2 + adc %%HIP, 0 + add %%X1, %%LOP + adc %%HIP, 0 + %if %%N == 3 + mov %%X2, %%HIP + %endif + + %if %%N > 3 + gsmulx %%LOP,%%X2, [%%pA+sizeof(qword)*3] ; (LO,X2) = a[3]*b[i] + add %%X2, %%X3 + adc %%LOP, 0 + add %%X2, %%HIP + adc %%LOP, 0 + %if %%N == 4 + mov %%X3, %%LOP + %endif + + %if %%N > 4 + gsmulx %%HIP,%%X3, [%%pA+sizeof(qword)*4] ; (HI:X3) = a[4]*b[i] + add %%X3, %%X4 + adc %%HIP, 0 + add %%X3, %%LOP + adc %%HIP, 0 + %if %%N == 5 + mov %%X4, %%HIP + %endif + + %if %%N > 5 + gsmulx %%LOP,%%X4, [%%pA+sizeof(qword)*5] ; (LO:X4) = a[5]*b[i] + add %%X4, %%X5 + adc %%LOP, 0 + add %%X4, %%HIP + adc %%LOP, 0 + %if %%N == 6 + mov %%X5, %%LOP + %endif + + %if %%N > 6 + gsmulx %%HIP,%%X5, [%%pA+sizeof(qword)*6] ; (HI:X5) = a[6]*b[i] + add %%X5, %%X6 + adc %%HIP, 0 + add %%X5, %%LOP + adc %%HIP, 0 + %if %%N == 7 + mov %%X6, %%HIP + %endif + + %if %%N > 7 + gsmulx %%LOP, %%X6, [%%pA+sizeof(qword)*7] ; (LO:X6) = a[7]*b[i] + add %%X6, %%X7 + adc %%LOP, 0 + add %%X6, %%HIP + adc %%LOP, 0 + mov %%X7, %%LOP + %endif + %endif + %endif + %endif + %endif + %endif + %endif +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Inputs: pDst: Destination (upto 1024 bits, 16 qwords) +;;; pA: Multiplicand (upto 512 bits, 8 qwords) +;;; pB: Multiplicand (upto 512 bits, 8 qwords) +;;; +;;; +;;; Uses registers: +;;; rdx - implicit multiplicand +;;; (HIP:LOP)- 64x64-bit product (mulx) +;;; X0,,X7 - high result +;;; +%macro MUL_NxN 7-14.nolist + %xdefine %%N %1 + %xdefine %%pDst %2 + %xdefine %%pA %3 + %xdefine %%pB %4 + %xdefine %%HIP %5 + %xdefine %%LOP %6 + %xdefine %%X0 %7 + %xdefine %%X1 %8 + %xdefine %%X2 %9 + %xdefine %%X3 %10 + %xdefine %%X4 %11 + %xdefine %%X5 %12 + %xdefine %%X6 %13 + %xdefine %%X7 %14 + + mov rdx, [%%pB] + MUL_FIX {%%N},%%pDst,{%%pA},{%%HIP},{%%LOP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} + + %if %%N > 1 + mov rdx, [%%pB+sizeof(qword)*1] + MLA_FIX {%%N},{%%pDst+sizeof(qword)*1},{%%pA},{%%HIP},{%%LOP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} + %endif + + %if %%N > 2 + mov rdx, [%%pB+sizeof(qword)*2] + MLA_FIX {%%N},{%%pDst+sizeof(qword)*2},{%%pA},{%%HIP},{%%LOP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} + %endif + + %if %%N > 3 + mov rdx, [%%pB+sizeof(qword)*3] + MLA_FIX {%%N},{%%pDst+sizeof(qword)*3},{%%pA},{%%HIP},{%%LOP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} + %endif + + %if %%N > 4 + mov rdx, [%%pB+sizeof(qword)*4] + MLA_FIX {%%N},{%%pDst+sizeof(qword)*4},{%%pA},{%%HIP},{%%LOP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} + %endif + + %if %%N > 5 + mov rdx, [%%pB+sizeof(qword)*5] + MLA_FIX {%%N},{%%pDst+sizeof(qword)*5},{%%pA},{%%HIP},{%%LOP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} + %endif + + %if %%N > 6 + mov rdx, [%%pB+sizeof(qword)*6] + MLA_FIX {%%N},{%%pDst+sizeof(qword)*6},{%%pA},{%%HIP},{%%LOP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} + %endif + + %if %%N > 7 + mov rdx, [%%pB+sizeof(qword)*7] + MLA_FIX {%%N},{%%pDst+sizeof(qword)*7},{%%pA},{%%HIP},{%%LOP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} + %endif + + %if %%N > 7 + mov qword [%%pDst+sizeof(qword)*8], %%X0 + mov qword [%%pDst+sizeof(qword)*9], %%X1 + mov qword [%%pDst+sizeof(qword)*10], %%X2 + mov qword [%%pDst+sizeof(qword)*11], %%X3 + mov qword [%%pDst+sizeof(qword)*12], %%X4 + mov qword [%%pDst+sizeof(qword)*13], %%X5 + mov qword [%%pDst+sizeof(qword)*14], %%X6 + mov qword [%%pDst+sizeof(qword)*15], %%X7 + %elif %%N > 6 + mov qword [%%pDst+sizeof(qword)*7], %%X0 + mov qword [%%pDst+sizeof(qword)*8], %%X1 + mov qword [%%pDst+sizeof(qword)*9], %%X2 + mov qword [%%pDst+sizeof(qword)*10], %%X3 + mov qword [%%pDst+sizeof(qword)*11], %%X4 + mov qword [%%pDst+sizeof(qword)*12], %%X5 + mov qword [%%pDst+sizeof(qword)*13], %%X6 + %elif %%N > 5 + mov qword [%%pDst+sizeof(qword)*6], %%X0 + mov qword [%%pDst+sizeof(qword)*7], %%X1 + mov qword [%%pDst+sizeof(qword)*8], %%X2 + mov qword [%%pDst+sizeof(qword)*9], %%X3 + mov qword [%%pDst+sizeof(qword)*10], %%X4 + mov qword [%%pDst+sizeof(qword)*11], %%X5 + %elif %%N > 4 + mov qword [%%pDst+sizeof(qword)*5], %%X0 + mov qword [%%pDst+sizeof(qword)*6], %%X1 + mov qword [%%pDst+sizeof(qword)*7], %%X2 + mov qword [%%pDst+sizeof(qword)*8], %%X3 + mov qword [%%pDst+sizeof(qword)*9], %%X4 + %elif %%N > 3 + mov qword [%%pDst+sizeof(qword)*4], %%X0 + mov qword [%%pDst+sizeof(qword)*5], %%X1 + mov qword [%%pDst+sizeof(qword)*6], %%X2 + mov qword [%%pDst+sizeof(qword)*7], %%X3 + %elif %%N > 2 + mov qword [%%pDst+sizeof(qword)*3], %%X0 + mov qword [%%pDst+sizeof(qword)*4], %%X1 + mov qword [%%pDst+sizeof(qword)*5], %%X2 + %elif %%N > 1 + mov qword [%%pDst+sizeof(qword)*2], %%X0 + mov qword [%%pDst+sizeof(qword)*3], %%X1 + %else + mov qword [%%pDst+sizeof(qword)*1], %%X0 + %endif ;; N==2 + +%endmacro + +%endif ;; _PCPBNUMUL_FIX_ADCX_INC_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumulpp.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumulpp.inc new file mode 100644 index 0000000..5e53d37 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumulpp.inc @@ -0,0 +1,666 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Low level Big Number multiplication Support +; +; + +%ifndef _PCPBNUMUL_INC_ +%assign _PCPBNUMUL_INC_ 1 + +%include "pcpbnumulpp_basic.inc" + +%macro CLEAR 2.nolist + %xdefine %%rPtr %1 + %xdefine %%rLen %2 + +%%L_1: + mov qword [%%rPtr], rax + add %%rPtr, sizeof(qword) + sub %%rLen, 1 + jnz %%L_1 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; (8*n)x(8*m) multiplier +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_8Nx8M_adcox,PRIVATE + push rbx ;; nsB + push rdi ;; pR + push rsi ;; pA + push rdx ;; nsA + +;;; +;;; init +;;; +.mul_loopA: + push rdx ;; nsA + call mla_8x8 + add rdi, sizeof(qword)*8 ; adv pR + add rsi, sizeof(qword)*8 ; adv pA + pop rdx ; nsA-- + sub rdx, 8 + jnz .mul_loopA + + mov qword [rdi+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*2],r10 + mov qword [rdi+sizeof(qword)*3],r11 + mov qword [rdi+sizeof(qword)*4],r12 + mov qword [rdi+sizeof(qword)*5],r13 + mov qword [rdi+sizeof(qword)*6],r14 + mov qword [rdi+sizeof(qword)*7],r15 + + jmp .mla_entry + +.mla_loopB: + push rbx ;; nsB + + push rdi ;; pR + push rsi ;; pA + push rdx ;; nsA + + xor rax, rax ; c-flag + push rax + + mov r8, qword [rdi+sizeof(qword)*0] + mov r9, qword [rdi+sizeof(qword)*1] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + +.loopA: + push rdx ; nsA + call mla_8x8 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + pop rdx ; nsA-- + sub rdx, 8 + jz .exit_loopA + + pop rax ; restore c-flag + neg rax + op_reg_mem adc, r8, qword [rdi+sizeof(qword)*0], rax + op_reg_mem adc, r9, qword [rdi+sizeof(qword)*1], rax + op_reg_mem adc, r10,qword [rdi+sizeof(qword)*2], rax + op_reg_mem adc, r11,qword [rdi+sizeof(qword)*3], rax + op_reg_mem adc, r12,qword [rdi+sizeof(qword)*4], rax + op_reg_mem adc, r13,qword [rdi+sizeof(qword)*5], rax + op_reg_mem adc, r14,qword [rdi+sizeof(qword)*6], rax + op_reg_mem adc, r15,qword [rdi+sizeof(qword)*7], rax + sbb rax, rax ; save c-flag + push rax + jmp .loopA + +.exit_loopA: + pop rax ; restore c-flag + neg rax + adc r8, 0 + mov qword [rdi+sizeof(qword)*0], r8 + adc r9, 0 + mov qword [rdi+sizeof(qword)*1], r9 + adc r10,0 + mov qword [rdi+sizeof(qword)*2],r10 + adc r11,0 + mov qword [rdi+sizeof(qword)*3],r11 + adc r12,0 + mov qword [rdi+sizeof(qword)*4],r12 + adc r13,0 + mov qword [rdi+sizeof(qword)*5],r13 + adc r14,0 + mov qword [rdi+sizeof(qword)*6],r14 + adc r15,0 + mov qword [rdi+sizeof(qword)*7],r15 + +.mla_entry: + pop rdx ; restore nsA + pop rsi ; restore pA + pop rdi + add rdi, sizeof(qword)*8 + + add rcx, sizeof(qword)*8 + pop rbx ;; nsB-- + sub rbx, 8 + jnz .mla_loopB + ret +ENDFUNC mul_8Nx8M_adcox + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; simplest general case N x M multiplier +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_simple,PRIVATE + xor rax, rax ; carry = 0 + + mov r11, rdx + cmp r11, rbx + jge .ms_mla_entry + SWAP r11, rbx + SWAP rsi, rcx + jmp .ms_mla_entry + +.ms_loopB: + push rbx ;; nsB + push rdi ;; pR + push rsi ;; pA + push r11 ;; nsA + push rax ;; save previous pass carry + + mov rdx, qword [rcx] ; pB[] + xor r10, r10 ; extension +.ms_loopA: + gsmulx r9, r8, qword [rsi] + add rdi, sizeof(qword) + add rsi, sizeof(qword) + add r8, r10 + adc r9, 0 + add r8, qword [rdi-sizeof(qword)] + adc r9, 0 + mov qword [rdi-sizeof(qword)], r8 + mov r10, r9 + sub r11, 1 + jnz .ms_loopA + + pop rax ; restore carry + shr rax, 1 + + adc r10, qword [rdi] + mov qword [rdi], r10 + adc rax, 0 ; save carry + + pop r11 ; restore nsA + pop rsi ; restore pA + pop rdi ; restore pR + pop rbx ; nsB + + add rdi, sizeof(qword) + add rcx, sizeof(qword) + +.ms_mla_entry: + sub rbx, 1 + jnc .ms_loopB + ret +ENDFUNC mla_simple + + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; general case N x M multiplier +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_NxM_adcox,PRIVATE +; +; stack struct +; +%assign nsB_cnt 0 ; rest of B operand +%assign ptrR nsB_cnt+sizeof(qword) ; current product pointer +%assign ptrA ptrR+sizeof(qword) ; ponter to A operand (pA) +%assign nsA ptrA+sizeof(qword) ; length of A operand (nsA) +%assign nsA_cnt nsA+sizeof(qword) ; rest of A operand +%assign carry nsA_cnt+sizeof(qword) ; carry +%assign tailproc carry+sizeof(qword) ; mla_8xn procedure +%assign stack_mem tailproc+sizeof(qword) ; size of stack allocation + + sub rsp, stack_mem ; allocate stack + + cmp rbx, 8 + jge .regular_entry + cmp rdx, 8 + jge .irregular_entry + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; degradated case +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ; clear product + mov r8, rdx + add r8, rbx + mov rbp, rdi + xor rax, rax + CLEAR rbp, r8 + + call mla_simple + jmp .quit + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; irregular init +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +.irregular_entry: + mov [rsp+nsB_cnt], rbx + mov [rsp+nsA], rdx + mov [rsp+nsA_cnt], rdx + + GET_EP rax, mla_8xl_tail, rbx, rbp ; tail procedure (mla_8xn) address + mov [rsp+tailproc], rax + + jmp .irr_init_entry + +.irr_init_loop: + mov [rsp+nsA_cnt], rdx ; save A counter + call rax + + mov rbx, [rsp+nsB_cnt] + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*0], r8 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*1], r9 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*2], r10 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*3], r11 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*4], r12 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*5], r13 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*6], r14 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*7], r15 + + add rsi, sizeof(qword)*8 + add rdi, sizeof(qword)*8 + + xor r8, r8 + xor r9, r9 + xor r10,r10 + xor r11,r11 + xor r12,r12 + xor r13,r13 + xor r14,r14 + xor r15,r15 + + ; load data depending on nsB length + mov r8, qword [rdi] + cmp rbx, 1 + jz .continue + mov r9, qword [rdi+sizeof(qword)] + cmp rbx, 2 + jz .continue + mov r10, qword [rdi+sizeof(qword)*2] + cmp rbx, 3 + jz .continue + mov r11, qword [rdi+sizeof(qword)*3] + cmp rbx, 4 + jz .continue + mov r12, qword [rdi+sizeof(qword)*4] + cmp rbx, 5 + jz .continue + mov r13, qword [rdi+sizeof(qword)*5] + cmp rbx, 6 + jz .continue + mov r14, qword [rdi+sizeof(qword)*6] +.continue: + mov rdx, [rsp+nsA_cnt] ; nsA + +.irr_init_entry: + sub rdx, 8 + mov rax, [rsp+tailproc] + jnc .irr_init_loop + + add rdx, 8 + jz .quit + + ; clear uninitialized rest of product + lea rbp, [rdi+rbx*sizeof(qword)] + xor rax, rax + CLEAR rbp, rdx + + mov rdx, [rsp+nsA_cnt] + call mla_simple + jmp .quit + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; regular ep +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +.regular_entry: + sub rbx, 8 + xor rax, rax + mov [rsp+nsB_cnt], rbx + mov [rsp+ptrR], rdi + mov [rsp+ptrA], rsi + mov [rsp+nsA], rdx + mov [rsp+nsA_cnt], rdx + mov [rsp+carry], rax + + mov rbp, rdx ; n = nsA %8 + and rbp, 7 + GET_EP rax, mla_8xl_tail, rbp ; tail procedure (mla_8xn) address + mov [rsp+tailproc], rax + +;; +;; regular init +;; + sub rdx, 8 ;; nsA counter +.init_loopA: + mov [rsp+nsA_cnt], rdx ; nsA + call mla_8x8 + mov rdx, [rsp+nsA_cnt] + add rdi, sizeof(qword)*8 ; adv ptrR + add rsi, sizeof(qword)*8 ; adv ptrA + sub rdx, 8 ; nsA -= 8 + jnc .init_loopA + + add rdx, 8 + jz .init_complete + + mov [rsp+nsA_cnt], rdx + + mov rax, [rsp+tailproc] + SWAP rcx, rsi + call rax + SWAP rcx, rsi + + mov rdx, [rsp+nsA_cnt] + lea rdi, [rdi+rdx*sizeof(qword)] + +.init_complete: + mov qword [rdi+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*2],r10 + mov qword [rdi+sizeof(qword)*3],r11 + mov qword [rdi+sizeof(qword)*4],r12 + mov qword [rdi+sizeof(qword)*5],r13 + mov qword [rdi+sizeof(qword)*6],r14 + mov qword [rdi+sizeof(qword)*7],r15 + + jmp .mla_entry + +;; +;; regular mla passes +;; +.mla_loopB: + mov [rsp+nsB_cnt], rbx ; update B conter + mov [rsp+ptrR], rdi ; update current product pointer + + xor rax, rax ; init carry + mov [rsp+carry], rax + + mov r8, qword [rdi+sizeof(qword)*0] + mov r9, qword [rdi+sizeof(qword)*1] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + sub rdx, 8 +.loopA: + mov [rsp+nsA_cnt], rdx ; save A counter + call mla_8x8 + mov rdx, [rsp+nsA_cnt] ; A counter + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + sub rdx, 8 ; nsA -= 8 + jc .exit_loopA + + mov rax, [rsp+carry] ; restore carry + shr rax, 1 + op_reg_mem adc, r8, qword [rdi+sizeof(qword)*0], rbx + op_reg_mem adc, r9, qword [rdi+sizeof(qword)*1], rbx + op_reg_mem adc, r10,qword [rdi+sizeof(qword)*2], rbx + op_reg_mem adc, r11,qword [rdi+sizeof(qword)*3], rbx + op_reg_mem adc, r12,qword [rdi+sizeof(qword)*4], rbx + op_reg_mem adc, r13,qword [rdi+sizeof(qword)*5], rbx + op_reg_mem adc, r14,qword [rdi+sizeof(qword)*6], rbx + op_reg_mem adc, r15,qword [rdi+sizeof(qword)*7], rbx + adc rax, 0 ; save carry + mov [rsp+carry], rax + jmp .loopA + +.exit_loopA: + add rdx, 8 + jz .complete_reg_loopB + + mov [rsp+nsA_cnt], rdx + + ; put zeros + xor rax, rax +.put_zero: + mov qword [rdi+rdx*sizeof(qword)], rax + add rdx,1 + cmp rdx, 8 + jl .put_zero + + mov rax, [rsp+carry] ; restore carry + shr rax, 1 + op_reg_mem adc, r8, [rdi+sizeof(qword)*0], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*1], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*2], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*3], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*4], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*5], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*6], rbx + op_reg_mem adc, r15,[rdi+sizeof(qword)*7], rbx + adc rax, 0 ; save carry + mov [rsp+carry], rax + + mov rax, [rsp+tailproc] + SWAP rcx, rsi + call rax + SWAP rcx, rsi + + mov rdx, [rsp+nsA_cnt] ; restore nsA + lea rdi, [rdi+rdx*sizeof(qword)] + + mov rax, [rsp+carry] ; restore carry + shr rax, 1 + + dec rdx + jz .mt_1 + dec rdx + jz .mt_2 + dec rdx + jz .mt_3 + dec rdx + jz .mt_4 + dec rdx + jz .mt_5 + dec rdx + jz .mt_6 +.mt_7: adc r9, 0 +.mt_6: adc r10,0 +.mt_5: adc r11,0 +.mt_4: adc r12,0 +.mt_3: adc r13,0 +.mt_2: adc r14,0 +.mt_1: adc r15,0 + mov qword [rdi+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*2],r10 + mov qword [rdi+sizeof(qword)*3],r11 + mov qword [rdi+sizeof(qword)*4],r12 + mov qword [rdi+sizeof(qword)*5],r13 + mov qword [rdi+sizeof(qword)*6],r14 + mov qword [rdi+sizeof(qword)*7],r15 + jmp .mla_entry + +.complete_reg_loopB: + mov rax, [rsp+carry] ; restore carry + add r8, rax + adc r9, 0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + adc r14,0 + adc r15,0 + mov qword [rdi+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*2],r10 + mov qword [rdi+sizeof(qword)*3],r11 + mov qword [rdi+sizeof(qword)*4],r12 + mov qword [rdi+sizeof(qword)*5],r13 + mov qword [rdi+sizeof(qword)*6],r14 + mov qword [rdi+sizeof(qword)*7],r15 + +.mla_entry: + ; restore: + mov rbx, [rsp+nsB_cnt] ; - B counter + mov rdi, [rsp+ptrR] ; - ptrR + mov rdx, [rsp+nsA] ; - nsA + mov rsi, [rsp+ptrA] ; - pA + + add rcx, sizeof(qword)*8 ; adv pB + add rdi, sizeof(qword)*8 ; adv ptrR + sub rbx, 8 ; nsB -= 8 + jnc .mla_loopB + + add rbx, 8 + jz .quit + +;; +;; final mla pass +;; + mov [rsp+nsB_cnt], rbx + + GET_EP rax, mla_8xl_tail, rbx, rbp ; tail procedure (mla_8xn) address + mov [rsp+tailproc], rax + + ; clear uninitialized rest of product + lea rbp, [rdi+rdx*sizeof(qword)] + xor rax, rax + CLEAR rbp, rbx + + xor rax, rax ; init carry + mov [rsp+carry], rax + + sub rdx, 8 +.tail_loopA: + mov r8, qword [rdi] + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + mov [rsp+nsA_cnt], rdx ; save A counter + mov rax, [rsp+tailproc] + call rax + +.entry_tail_loopA: + mov rax, [rsp+carry] ; restore carry + shr rax, 1 + adc r8, 0 + adc r9, 0 + adc r10, 0 + adc r11, 0 + adc r12, 0 + adc r13, 0 + adc r14, 0 + adc r15, 0 + adc rax, 0 + + mov rbx, [rsp+nsB_cnt] ; nsB + mov rbp, rbx + dec rbp + jz .tt_1 + dec rbp + jz .tt_2 + dec rbp + jz .tt_3 + dec rbp + jz .tt_4 + dec rbp + jz .tt_5 + dec rbp + jz .tt_6 +.tt_7: op_reg_mem adc, r9, [rdi+rbx*sizeof(qword)+sizeof(qword)*1], rbp +.tt_6: op_reg_mem adc, r10,[rdi+rbx*sizeof(qword)+sizeof(qword)*2], rbp +.tt_5: op_reg_mem adc, r11,[rdi+rbx*sizeof(qword)+sizeof(qword)*3], rbp +.tt_4: op_reg_mem adc, r12,[rdi+rbx*sizeof(qword)+sizeof(qword)*4], rbp +.tt_3: op_reg_mem adc, r13,[rdi+rbx*sizeof(qword)+sizeof(qword)*5], rbp +.tt_2: op_reg_mem adc, r14,[rdi+rbx*sizeof(qword)+sizeof(qword)*6], rbp +.tt_1: op_reg_mem adc, r15,[rdi+rbx*sizeof(qword)+sizeof(qword)*7], rbp + adc rax, 0 + mov [rsp+carry], rax ; update carry + + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*0], r8 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*1], r9 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*2], r10 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*3], r11 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*4], r12 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*5], r13 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*6], r14 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*7], r15 + + mov rdx, [rsp+nsA_cnt] ; A counter + add rsi, sizeof(qword)*8 ; adv pA + add rdi, sizeof(qword)*8 ; adv pR + sub rdx, 8 ; nsA -= 8 + jnc .tail_loopA + + add rdx, 8 + jz .quit + + ; carry propagation + mov rax, [rsp+carry] + mov rbp, rbx + + dec rbp + mov r8, qword [rdi+rbx*sizeof(qword)+sizeof(qword)*0] + add r8, rax + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*0], r8 + jz .simple + dec rbp + mov r9, qword [rdi+rbx*sizeof(qword)+sizeof(qword)*1] + adc r9, 0 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*1], r9 + jz .simple + dec rbp + mov r10,qword [rdi+rbx*sizeof(qword)+sizeof(qword)*2] + adc r10, 0 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*2], r10 + jz .simple + dec rbp + mov r11,qword [rdi+rbx*sizeof(qword)+sizeof(qword)*3] + adc r11, 0 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*3], r11 + jz .simple + dec rbp + mov r12,qword [rdi+rbx*sizeof(qword)+sizeof(qword)*4] + adc r12, 0 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*4], r12 + jz .simple + dec rbp + mov r13,qword [rdi+rbx*sizeof(qword)+sizeof(qword)*5] + adc r13, 0 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*5], r13 + jz .simple + dec rbp + mov r14,qword [rdi+rbx*sizeof(qword)+sizeof(qword)*6] + adc r14, 0 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*6], r14 + +.simple: + call mla_simple + + +.quit: + add rsp, stack_mem ; release stack + ret +ENDFUNC mul_NxM_adcox + + +%endif ;; _PCPBNUMUL_INC_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumulpp_basic.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumulpp_basic.inc new file mode 100644 index 0000000..d89375f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumulpp_basic.inc @@ -0,0 +1,924 @@ +;=============================================================================== +; Copyright (C) 2013 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Low level Big Number multiplication Support +; +; + +%ifndef _PCPBNUMUL_BASIC_INC_ +%assign _PCPBNUMUL_BASIC_INC_ 1 + +%include "pcpmulx.inc" +%include "pcpbnumulpp_fix.inc" + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; short size (1-8 qwords) operand mla and mul operations +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; +;; 1x1 multiplication +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_1x1,PRIVATE + mov rdx, qword [rcx] + MLA_FIX 1, rdi, rsi, rbx,rbp, r8 + ret +ENDFUNC mla_1x1 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_1x1,PRIVATE + mov rdx,qword [rcx] + gsmulx rdx, rax, qword [rsi] + mov qword [rdi], rax + mov qword [rdi+sizeof(qword)], rdx + ret +ENDFUNC mul_1x1 + + + +;; +;; 2x2 multiplication +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_2x2,PRIVATE + mov rdx, qword [rcx] + MLA_FIX 2, rdi, rsi, rbx,rbp, r8,r9 + mov rdx, qword [rcx+sizeof(qword)] + MLA_FIX 2, {rdi+sizeof(qword)}, rsi, rbx,rbp, r8,r9 + ret +ENDFUNC mla_2x2 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_2x2,PRIVATE + call mla_2x2 + mov qword [rdi+sizeof(qword)*2], r8 + mov qword [rdi+sizeof(qword)*3], r9 + ret +ENDFUNC mul_2x2 + + + +;; +;; 3x3 multiplication +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_3x3,PRIVATE + mov rdx, qword [rcx] + MLA_FIX 3, rdi, rsi, rbx,rbp, r8,r9,r10 + mov rdx, qword [rcx+sizeof(qword)] + MLA_FIX 3, {rdi+sizeof(qword)}, rsi, rbx,rbp, r8,r9,r10 + mov rdx, qword [rcx+sizeof(qword)*2] + MLA_FIX 3, {rdi+sizeof(qword)*2}, rsi, rbx,rbp, r8,r9,r10 + ret +ENDFUNC mla_3x3 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_3x3,PRIVATE + call mla_3x3 + mov qword [rdi+sizeof(qword)*3], r8 + mov qword [rdi+sizeof(qword)*4], r9 + mov qword [rdi+sizeof(qword)*5], r10 + ret +ENDFUNC mul_3x3 + + + +;; +;; 4x{2,4} multiplication +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_4x4,PRIVATE + mov rdx, qword [rcx] + MLA_FIX 4, rdi, rsi, rbx,rbp, r8,r9,r10,r11 + mov rdx, qword [rcx+sizeof(qword)] + MLA_FIX 4, {rdi+sizeof(qword)}, rsi, rbx,rbp, r8,r9,r10,r11 + mov rdx, qword [rcx+sizeof(qword)*2] + MLA_FIX 4, {rdi+sizeof(qword)*2}, rsi, rbx,rbp, r8,r9,r10,r11 + mov rdx, qword [rcx+sizeof(qword)*3] + MLA_FIX 4, {rdi+sizeof(qword)*3}, rsi, rbx,rbp, r8,r9,r10,r11 + ret +ENDFUNC mla_4x4 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_4x4,PRIVATE + call mla_4x4 + mov qword [rdi+sizeof(qword)*4], r8 + mov qword [rdi+sizeof(qword)*5], r9 + mov qword [rdi+sizeof(qword)*6], r10 + mov qword [rdi+sizeof(qword)*7], r11 + ret +ENDFUNC mul_4x4 + + + +;; +;; 5x{2,5} multiplication +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_5x5,PRIVATE + mov rdx, [rcx+sizeof(qword)*0] + MLA_FIX 5, rdi, rsi, rbx,rbp, r8,r9,r10,r11,r12 + mov rdx, [rcx+sizeof(qword)*1] + MLA_FIX 5, {rdi+sizeof(qword)*1}, rsi, rbx,rbp, r8,r9,r10,r11,r12 + mov rdx, [rcx+sizeof(qword)*2] + MLA_FIX 5, {rdi+sizeof(qword)*2}, rsi, rbx,rbp, r8,r9,r10,r11,r12 + mov rdx, [rcx+sizeof(qword)*3] + MLA_FIX 5, {rdi+sizeof(qword)*3}, rsi, rbx,rbp, r8,r9,r10,r11,r12 + mov rdx, [rcx+sizeof(qword)*4] + MLA_FIX 5, {rdi+sizeof(qword)*4}, rsi, rbx,rbp, r8,r9,r10,r11,r12 + ret +ENDFUNC mla_5x5 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_5x5,PRIVATE + call mla_5x5 + mov qword [rdi+sizeof(qword)*5], r8 + mov qword [rdi+sizeof(qword)*6], r9 + mov qword [rdi+sizeof(qword)*7], r10 + mov qword [rdi+sizeof(qword)*8], r11 + mov qword [rdi+sizeof(qword)*9], r12 + ret +ENDFUNC mul_5x5 + + + +;; +;; 6x{2,6} multiplication +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_6x6,PRIVATE + mov rdx, [rcx+sizeof(qword)*0] + MLA_FIX 6, {rdi+sizeof(qword)*0}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13 + mov rdx, [rcx+sizeof(qword)*1] + MLA_FIX 6, {rdi+sizeof(qword)*1}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13 + mov rdx, [rcx+sizeof(qword)*2] + MLA_FIX 6, {rdi+sizeof(qword)*2}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13 + mov rdx, [rcx+sizeof(qword)*3] + MLA_FIX 6, {rdi+sizeof(qword)*3}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13 + mov rdx, [rcx+sizeof(qword)*4] + MLA_FIX 6, {rdi+sizeof(qword)*4}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13 + mov rdx, [rcx+sizeof(qword)*5] + MLA_FIX 6, {rdi+sizeof(qword)*5}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13 + ret +ENDFUNC mla_6x6 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_6x6,PRIVATE + call mla_6x6 + mov qword [rdi+sizeof(qword)*6], r8 + mov qword [rdi+sizeof(qword)*7], r9 + mov qword [rdi+sizeof(qword)*8], r10 + mov qword [rdi+sizeof(qword)*9], r11 + mov qword [rdi+sizeof(qword)*10],r12 + mov qword [rdi+sizeof(qword)*11],r13 + ret +ENDFUNC mul_6x6 + + + +;; +;; 7x{2,7} multiplication +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_7x7,PRIVATE + mov rdx, [rcx+sizeof(qword)*0] + MLA_FIX 7, {rdi+sizeof(qword)*0}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14 + mov rdx, [rcx+sizeof(qword)*1] + MLA_FIX 7, {rdi+sizeof(qword)*1}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14 + mov rdx, [rcx+sizeof(qword)*2] + MLA_FIX 7, {rdi+sizeof(qword)*2}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14 + mov rdx, [rcx+sizeof(qword)*3] + MLA_FIX 7, {rdi+sizeof(qword)*3}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14 + mov rdx, [rcx+sizeof(qword)*4] + MLA_FIX 7, {rdi+sizeof(qword)*4}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14 + mov rdx, [rcx+sizeof(qword)*5] + MLA_FIX 7, {rdi+sizeof(qword)*5}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14 + mov rdx, [rcx+sizeof(qword)*6] + MLA_FIX 7, {rdi+sizeof(qword)*6}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14 + ret +ENDFUNC mla_7x7 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_7x7,PRIVATE + call mla_7x7 + mov qword [rdi+sizeof(qword)*7], r8 + mov qword [rdi+sizeof(qword)*8], r9 + mov qword [rdi+sizeof(qword)*9], r10 + mov qword [rdi+sizeof(qword)*10],r11 + mov qword [rdi+sizeof(qword)*11],r12 + mov qword [rdi+sizeof(qword)*12],r13 + mov qword [rdi+sizeof(qword)*13],r14 + ret +ENDFUNC mul_7x7 + + + +;; +;; 8x{1,2,3,4,5,6,7,8} multiplication +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_8x1,PRIVATE + mov rdx, [rcx] + MLA_FIX 8, rdi, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + ret +ENDFUNC mla_8x1 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_8x2,PRIVATE + mov rdx, [rcx+sizeof(qword)*0] + MLA_FIX 8, {rdi+sizeof(qword)*0}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*1] + MLA_FIX 8, {rdi+sizeof(qword)*1}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + ret +ENDFUNC mla_8x2 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_8x3,PRIVATE + mov rdx, [rcx+sizeof(qword)*0] + MLA_FIX 8, {rdi+sizeof(qword)*0}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*1] + MLA_FIX 8, {rdi+sizeof(qword)*1}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*2] + MLA_FIX 8, {rdi+sizeof(qword)*2}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + ret +ENDFUNC mla_8x3 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_8x4,PRIVATE + mov rdx, [rcx+sizeof(qword)*0] + MLA_FIX 8, {rdi+sizeof(qword)*0}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*1] + MLA_FIX 8, {rdi+sizeof(qword)*1}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*2] + MLA_FIX 8, {rdi+sizeof(qword)*2}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*3] + MLA_FIX 8, {rdi+sizeof(qword)*3}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + ret +ENDFUNC mla_8x4 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_8x5,PRIVATE + mov rdx, [rcx+sizeof(qword)*0] + MLA_FIX 8, {rdi+sizeof(qword)*0}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*1] + MLA_FIX 8, {rdi+sizeof(qword)*1}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*2] + MLA_FIX 8, {rdi+sizeof(qword)*2}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*3] + MLA_FIX 8, {rdi+sizeof(qword)*3}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*4] + MLA_FIX 8, {rdi+sizeof(qword)*4}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + ret +ENDFUNC mla_8x5 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_8x6,PRIVATE + mov rdx, [rcx+sizeof(qword)*0] + MLA_FIX 8, {rdi+sizeof(qword)*0}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*1] + MLA_FIX 8, {rdi+sizeof(qword)*1}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*2] + MLA_FIX 8, {rdi+sizeof(qword)*2}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*3] + MLA_FIX 8, {rdi+sizeof(qword)*3}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*4] + MLA_FIX 8, {rdi+sizeof(qword)*4}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*5] + MLA_FIX 8, {rdi+sizeof(qword)*5}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + ret +ENDFUNC mla_8x6 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_8x7,PRIVATE + mov rdx, [rcx+sizeof(qword)*0] + MLA_FIX 8, {rdi+sizeof(qword)*0}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*1] + MLA_FIX 8, {rdi+sizeof(qword)*1}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*2] + MLA_FIX 8, {rdi+sizeof(qword)*2}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*3] + MLA_FIX 8, {rdi+sizeof(qword)*3}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*4] + MLA_FIX 8, {rdi+sizeof(qword)*4}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*5] + MLA_FIX 8, {rdi+sizeof(qword)*5}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*6] + MLA_FIX 8, {rdi+sizeof(qword)*6}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + ret +ENDFUNC mla_8x7 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mla_8x8,PRIVATE + mov rdx, [rcx+sizeof(qword)*0] + MLA_FIX 8, {rdi+sizeof(qword)*0}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*1] + MLA_FIX 8, {rdi+sizeof(qword)*1}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*2] + MLA_FIX 8, {rdi+sizeof(qword)*2}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*3] + MLA_FIX 8, {rdi+sizeof(qword)*3}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*4] + MLA_FIX 8, {rdi+sizeof(qword)*4}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*5] + MLA_FIX 8, {rdi+sizeof(qword)*5}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*6] + MLA_FIX 8, {rdi+sizeof(qword)*6}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + mov rdx, [rcx+sizeof(qword)*7] + MLA_FIX 8, {rdi+sizeof(qword)*7}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + ret +ENDFUNC mla_8x8 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_8x8,PRIVATE + call mla_8x8 + mov qword [rdi+sizeof(qword)*8], r8 + mov qword [rdi+sizeof(qword)*9], r9 + mov qword [rdi+sizeof(qword)*10],r10 + mov qword [rdi+sizeof(qword)*11],r11 + mov qword [rdi+sizeof(qword)*12],r12 + mov qword [rdi+sizeof(qword)*13],r13 + mov qword [rdi+sizeof(qword)*14],r14 + mov qword [rdi+sizeof(qword)*15],r15 + ret +ENDFUNC mul_8x8 + + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; middle size (9-16 qwords) operand mla and mul operations +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_9x9,PRIVATE + call mla_8x8 + + mov rdx, [rcx+sizeof(qword)*8] + MLA_FIX 8, {rdi+sizeof(qword)*8}, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + push r15 + + mov rdx, [rsi+sizeof(qword)*8] + mov r15, [rdi+sizeof(qword)*8] + MLA_FIX 8, {rdi+sizeof(qword)*8}, rcx, rbx,rbp, r15,r8,r9,r10,r11,r12,r13,r14 + mov qword [rdi+sizeof(qword)*9], r15 + + gsmulx r15, rbp, [rcx+sizeof(qword)*8] + pop rax + add r14, rax + adc r15,0 + add r14, rbp + adc r15, 0 + + mov qword [rdi+sizeof(qword)*10],r8 + mov qword [rdi+sizeof(qword)*11],r9 + mov qword [rdi+sizeof(qword)*12],r10 + mov qword [rdi+sizeof(qword)*13],r11 + mov qword [rdi+sizeof(qword)*14],r12 + mov qword [rdi+sizeof(qword)*15],r13 + mov qword [rdi+sizeof(qword)*16],r14 + mov qword [rdi+sizeof(qword)*17],r15 + ret +ENDFUNC mul_9x9 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_10x10,PRIVATE +; A0*B0 + call mla_8x8 + add rdi, sizeof(qword)*8 + +; + A0*B1 + add rcx, sizeof(qword)*8 + call mla_8x2 + push r15 + push r14 + +; + A1*B0 + add rsi, sizeof(qword)*8 + sub rcx, sizeof(qword)*8 + SWAP rcx, rsi + + mov r15, r13 + mov r14, r12 + mov r13, r11 + mov r12, r10 + mov r11, r9 + mov r10, r8 + mov r9, qword [rdi+sizeof(qword)*1] + mov r8, qword [rdi+sizeof(qword)*0] + call mla_8x2 + + mov qword [rdi+sizeof(qword)*2],r8 + mov qword [rdi+sizeof(qword)*3],r9 + mov qword [rdi+sizeof(qword)*4],r10 + mov qword [rdi+sizeof(qword)*5],r11 + mov qword [rdi+sizeof(qword)*6],r12 + mov qword [rdi+sizeof(qword)*7],r13 + add rdi, sizeof(qword)*8 + + xor r10, r10 + pop r8 + pop r9 + add r8, r14 + adc r9, r15 + adc r10,0 + +; + A1*B1 + SWAP rcx, rsi + add rcx, sizeof(qword)*8 + call mla_2x2 + add rdi, sizeof(qword)*2 + + add r8, r10 + adc r9, 0 + mov qword [rdi+sizeof(qword)*0],r8 + mov qword [rdi+sizeof(qword)*1],r9 + ret +ENDFUNC mul_10x10 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_11x11,PRIVATE +; A0*B0 + call mla_8x8 + add rdi, sizeof(qword)*8 + +; + A0*B1 + add rcx, sizeof(qword)*8 + call mla_8x3 + push r15 + push r14 + push r13 + +; + A1*B0 + add rsi, sizeof(qword)*8 + sub rcx, sizeof(qword)*8 + SWAP rcx, rsi + + mov r15, r12 + mov r14, r11 + mov r13, r10 + mov r12, r9 + mov r11, r8 + mov r10,qword [rdi+sizeof(qword)*2] + mov r9, qword [rdi+sizeof(qword)*1] + mov r8, qword [rdi+sizeof(qword)*0] + call mla_8x3 + + mov qword [rdi+sizeof(qword)*3],r8 + mov qword [rdi+sizeof(qword)*4],r9 + mov qword [rdi+sizeof(qword)*5],r10 + mov qword [rdi+sizeof(qword)*6],r11 + mov qword [rdi+sizeof(qword)*7],r12 + add rdi, sizeof(qword)*8 + + xor r11, r11 + pop r8 + pop r9 + pop r10 + add r8, r13 + adc r9, r14 + adc r10,r15 + adc r11,0 + +; + A1*B1 + SWAP rcx, rsi + add rcx, sizeof(qword)*8 + call mla_3x3 + add rdi, sizeof(qword)*3 + + add r8, r11 + adc r9, 0 + adc r10,0 + mov qword [rdi+sizeof(qword)*0],r8 + mov qword [rdi+sizeof(qword)*1],r9 + mov qword [rdi+sizeof(qword)*2],r10 + ret +ENDFUNC mul_11x11 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_12x12,PRIVATE +; A0*B0 + call mla_8x8 + add rdi, sizeof(qword)*8 + +; + A0*B1 + add rcx, sizeof(qword)*8 + call mla_8x4 + push r15 + push r14 + push r13 + push r12 + +; + A1*B0 + add rsi, sizeof(qword)*8 + sub rcx, sizeof(qword)*8 + SWAP rcx, rsi + + mov r15, r11 + mov r14, r10 + mov r13, r9 + mov r12, r8 + mov r11,qword [rdi+sizeof(qword)*3] + mov r10,qword [rdi+sizeof(qword)*2] + mov r9, qword [rdi+sizeof(qword)*1] + mov r8, qword [rdi+sizeof(qword)*0] + call mla_8x4 + + mov qword [rdi+sizeof(qword)*4],r8 + mov qword [rdi+sizeof(qword)*5],r9 + mov qword [rdi+sizeof(qword)*6],r10 + mov qword [rdi+sizeof(qword)*7],r11 + add rdi, sizeof(qword)*8 + + xor rax, rax + pop r8 + pop r9 + pop r10 + pop r11 + add r8, r12 + adc r9, r13 + adc r10,r14 + adc r11,r15 + adc rax,0 + push rax + +; + A1*B1 + SWAP rcx, rsi + add rcx, sizeof(qword)*8 + call mla_4x4 + add rdi, sizeof(qword)*4 + + pop rax + add r8, rax + adc r9, 0 + adc r10,0 + adc r11,0 + mov qword [rdi+sizeof(qword)*0],r8 + mov qword [rdi+sizeof(qword)*1],r9 + mov qword [rdi+sizeof(qword)*2],r10 + mov qword [rdi+sizeof(qword)*3],r11 + ret +ENDFUNC mul_12x12 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_13x13,PRIVATE +; A0*B0 + call mla_8x8 + add rdi, sizeof(qword)*8 + +; + A0*B1 + add rcx, sizeof(qword)*8 + call mla_8x5 + push r15 + push r14 + push r13 + push r12 + push r11 + +; + A1*B0 + add rsi, sizeof(qword)*8 + sub rcx, sizeof(qword)*8 + SWAP rcx, rsi + + mov r15, r10 + mov r14, r9 + mov r13, r8 + mov r12,qword [rdi+sizeof(qword)*4] + mov r11,qword [rdi+sizeof(qword)*3] + mov r10,qword [rdi+sizeof(qword)*2] + mov r9, qword [rdi+sizeof(qword)*1] + mov r8, qword [rdi+sizeof(qword)*0] + call mla_8x5 + + mov qword [rdi+sizeof(qword)*5], r8 + mov qword [rdi+sizeof(qword)*6], r9 + mov qword [rdi+sizeof(qword)*7], r10 + add rdi, sizeof(qword)*8 + + xor rax, rax + pop r8 + pop r9 + pop r10 + pop rbx + pop rbp + add r8, r11 + adc r9, r12 + adc r10,r13 + adc rbx,r14 + adc rbp,r15 + adc rax,0 + push rax + + mov r11, rbx + mov r12, rbp + +; + A1*B1 + SWAP rcx, rsi + add rcx, sizeof(qword)*8 + call mla_5x5 + add rdi, sizeof(qword)*5 + + pop rax + add r8, rax + adc r9, 0 + adc r10,0 + adc r11,0 + adc r12,0 + mov qword [rdi+sizeof(qword)*0],r8 + mov qword [rdi+sizeof(qword)*1],r9 + mov qword [rdi+sizeof(qword)*2],r10 + mov qword [rdi+sizeof(qword)*3],r11 + mov qword [rdi+sizeof(qword)*4],r12 + ret +ENDFUNC mul_13x13 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_14x14,PRIVATE + call mla_7x7 + + add rdi, sizeof(qword)*7 + add rsi, sizeof(qword)*7 + call mla_7x7 + mov qword [rdi+sizeof(qword)*7], r8 + mov qword [rdi+sizeof(qword)*8], r9 + mov qword [rdi+sizeof(qword)*9], r10 + mov qword [rdi+sizeof(qword)*10],r11 + mov qword [rdi+sizeof(qword)*11],r12 + mov qword [rdi+sizeof(qword)*12],r13 + mov qword [rdi+sizeof(qword)*13],r14 + + mov r8, qword [rdi+sizeof(qword)*0] + mov r9, qword [rdi+sizeof(qword)*1] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + + add rcx, sizeof(qword)*7 + sub rsi, sizeof(qword)*7 + call mla_7x7 + + xor rdx, rdx + op_reg_mem add, r8, qword [rdi+sizeof(qword)*7], rax + op_reg_mem adc, r9, qword [rdi+sizeof(qword)*8], rax + op_reg_mem adc, r10,qword [rdi+sizeof(qword)*9], rax + op_reg_mem adc, r11,qword [rdi+sizeof(qword)*10],rax + op_reg_mem adc, r12,qword [rdi+sizeof(qword)*11],rax + op_reg_mem adc, r13,qword [rdi+sizeof(qword)*12],rax + op_reg_mem adc, r14,qword [rdi+sizeof(qword)*13],rax + adc rdx, 0 + push rdx + + add rdi, sizeof(qword)*7 + add rsi, sizeof(qword)*7 + call mla_7x7 + + sub rdi, sizeof(qword)*14 + sub rsi, sizeof(qword)*7 + sub rcx, sizeof(qword)*7 + + pop rdx + add r8, rdx + adc r9, 0 + adc r10, 0 + adc r11, 0 + adc r12, 0 + adc r13, 0 + adc r14, 0 + + mov qword [rdi+sizeof(qword)*21],r8 + mov qword [rdi+sizeof(qword)*22],r9 + mov qword [rdi+sizeof(qword)*23],r10 + mov qword [rdi+sizeof(qword)*24],r11 + mov qword [rdi+sizeof(qword)*25],r12 + mov qword [rdi+sizeof(qword)*26],r13 + mov qword [rdi+sizeof(qword)*27],r14 + ret +ENDFUNC mul_14x14 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_15x15,PRIVATE +; A0*B0 + call mla_8x8 + add rdi, sizeof(qword)*8 + +; + A0*B1 + add rcx, sizeof(qword)*8 + call mla_8x7 + mov qword [rdi+sizeof(qword)*7], r8 + mov qword [rdi+sizeof(qword)*8], r9 + mov qword [rdi+sizeof(qword)*9], r10 + mov qword [rdi+sizeof(qword)*10],r11 + mov qword [rdi+sizeof(qword)*11],r12 + mov qword [rdi+sizeof(qword)*12],r13 + mov qword [rdi+sizeof(qword)*13],r14 + mov qword [rdi+sizeof(qword)*14],r15 + +; + A1*B0 + add rsi, sizeof(qword)*8 + sub rcx, sizeof(qword)*8 + SWAP rcx, rsi + + mov r8, qword [rdi+sizeof(qword)*0] + mov r9, qword [rdi+sizeof(qword)*1] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + call mla_8x7 + + mov qword [rdi+sizeof(qword)*7], r8 + add rdi, sizeof(qword)*8 + +; + A1*B1 + SWAP rcx, rsi + add rcx, sizeof(qword)*8 + + mov r8, r9 + mov r9, r10 + mov r10,r11 + mov r11,r12 + mov r12,r13 + mov r13,r14 + mov r14,r15 + xor rdx, rdx + op_reg_mem add, r8, qword [rdi+sizeof(qword)*0], rax + op_reg_mem adc, r9, qword [rdi+sizeof(qword)*1], rax + op_reg_mem adc, r10,qword [rdi+sizeof(qword)*2], rax + op_reg_mem adc, r11,qword [rdi+sizeof(qword)*3], rax + op_reg_mem adc, r12,qword [rdi+sizeof(qword)*4], rax + op_reg_mem adc, r13,qword [rdi+sizeof(qword)*5], rax + op_reg_mem adc, r14,qword [rdi+sizeof(qword)*6], rax + adc rdx, 0 + push rdx + + call mla_7x7 + add rdi, sizeof(qword)*7 + + pop rax + add r8, rax + adc r9, 0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + adc r14,0 + mov qword [rdi+sizeof(qword)*0],r8 + mov qword [rdi+sizeof(qword)*1],r9 + mov qword [rdi+sizeof(qword)*2],r10 + mov qword [rdi+sizeof(qword)*3],r11 + mov qword [rdi+sizeof(qword)*4],r12 + mov qword [rdi+sizeof(qword)*5],r13 + mov qword [rdi+sizeof(qword)*6],r14 + ret +ENDFUNC mul_15x15 + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mul_16x16,PRIVATE + call mla_8x8 + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + call mla_8x8 + mov qword [rdi+sizeof(qword)*8], r8 + mov qword [rdi+sizeof(qword)*9], r9 + mov qword [rdi+sizeof(qword)*10],r10 + mov qword [rdi+sizeof(qword)*11],r11 + mov qword [rdi+sizeof(qword)*12],r12 + mov qword [rdi+sizeof(qword)*13],r13 + mov qword [rdi+sizeof(qword)*14],r14 + mov qword [rdi+sizeof(qword)*15],r15 + + mov r8, qword [rdi+sizeof(qword)*0] + mov r9, qword [rdi+sizeof(qword)*1] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + add rcx, sizeof(qword)*8 + sub rsi, sizeof(qword)*8 + call mla_8x8 + + xor rdx, rdx + op_reg_mem add, r8, qword [rdi+sizeof(qword)*8], rax + op_reg_mem adc, r9, qword [rdi+sizeof(qword)*9], rax + op_reg_mem adc, r10,qword [rdi+sizeof(qword)*10],rax + op_reg_mem adc, r11,qword [rdi+sizeof(qword)*11],rax + op_reg_mem adc, r12,qword [rdi+sizeof(qword)*12],rax + op_reg_mem adc, r13,qword [rdi+sizeof(qword)*13],rax + op_reg_mem adc, r14,qword [rdi+sizeof(qword)*14],rax + op_reg_mem adc, r15,qword [rdi+sizeof(qword)*15],rax + adc rdx, 0 + push rdx + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + call mla_8x8 + + sub rdi, sizeof(qword)*16 + sub rsi, sizeof(qword)*8 + sub rcx, sizeof(qword)*8 + + pop rdx + add r8, rdx + adc r9, 0 + adc r10, 0 + adc r11, 0 + adc r12, 0 + adc r13, 0 + adc r14, 0 + adc r15, 0 + + mov qword [rdi+sizeof(qword)*24],r8 + mov qword [rdi+sizeof(qword)*25],r9 + mov qword [rdi+sizeof(qword)*26],r10 + mov qword [rdi+sizeof(qword)*27],r11 + mov qword [rdi+sizeof(qword)*28],r12 + mov qword [rdi+sizeof(qword)*29],r13 + mov qword [rdi+sizeof(qword)*30],r14 + mov qword [rdi+sizeof(qword)*31],r15 + ret +ENDFUNC mul_16x16 + + +mul_lxl_basic DQ mul_1x1 - mul_lxl_basic + DQ mul_2x2 - mul_lxl_basic + DQ mul_3x3 - mul_lxl_basic + DQ mul_4x4 - mul_lxl_basic + DQ mul_5x5 - mul_lxl_basic + DQ mul_6x6 - mul_lxl_basic + DQ mul_7x7 - mul_lxl_basic + DQ mul_8x8 - mul_lxl_basic + DQ mul_9x9 - mul_lxl_basic + DQ mul_10x10-mul_lxl_basic + DQ mul_11x11-mul_lxl_basic + DQ mul_12x12-mul_lxl_basic + DQ mul_13x13-mul_lxl_basic + DQ mul_14x14-mul_lxl_basic + DQ mul_15x15-mul_lxl_basic + DQ mul_16x16-mul_lxl_basic + +mla_lxl_short DQ mla_1x1 - mla_lxl_short + DQ mla_2x2 - mla_lxl_short + DQ mla_3x3 - mla_lxl_short + DQ mla_4x4 - mla_lxl_short + DQ mla_5x5 - mla_lxl_short + DQ mla_6x6 - mla_lxl_short + DQ mla_7x7 - mla_lxl_short + +mla_8xl_tail DQ mla_8x1 - mla_8xl_tail + DQ mla_8x2 - mla_8xl_tail + DQ mla_8x3 - mla_8xl_tail + DQ mla_8x4 - mla_8xl_tail + DQ mla_8x5 - mla_8xl_tail + DQ mla_8x6 - mla_8xl_tail + DQ mla_8x7 - mla_8xl_tail + +%endif ;; _PCPBNUMUL_BASIC_INC_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumulpp_fix.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumulpp_fix.inc new file mode 100644 index 0000000..62f35d8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumulpp_fix.inc @@ -0,0 +1,427 @@ +;=============================================================================== +; Copyright (C) 2013 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Low level Big Number multiplication Support +; +; + +%ifndef _PCPBNUMUL_FIX_ADCX_INC_ +%assign _PCPBNUMUL_FIX_ADCX_INC_ 1 + +%ifdef _ADCX_ADOX_ + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; macro used to replace op reg, mem instructions +%macro SWAP 2.nolist + %xdefine %%r1 %1 + %xdefine %%r2 %2 + + xor %%r1, %%r2 + xor %%r2, %%r1 + xor %%r1, %%r2 +%endmacro + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; macro used extract proc ep from the table +%macro GET_EP 3-4.nolist + %xdefine %%regEp %1 + %xdefine %%Table %2 + %xdefine %%regIdx %3 + %xdefine %%tmp %4 + + lea %%regEp, [rel %%Table] + %ifnempty %%tmp + mov %%tmp, [%%regEp+%%regIdx*sizeof(qword)-sizeof(qword)] + add %%regEp, %%tmp + %else + mov %%regIdx, [%%regEp+%%regIdx*sizeof(qword)-sizeof(qword)] + add %%regEp, %%regIdx + %endif +%endmacro + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; op DST, mem_src +%macro op_reg_mem 4.nolist + %xdefine %%opCode %1 + %xdefine %%dst %2 + %xdefine %%mem_src %3 + %xdefine %%tmp %4 + + mov %%tmp, %%mem_src + %%opCode %%dst, %%tmp +%endmacro + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; MEM_DST = (op src1, mem_src2) +%macro op_mem_reg_mem 5.nolist + %xdefine %%opCode %1 + %xdefine %%mem_dst %2 + %xdefine %%src1 %3 + %xdefine %%mem_src2 %4 + %xdefine %%tmp %5 + + op_reg_mem %%opCode, %%src1, %%mem_src2, %%tmp + mov %%mem_dst, %%src1 +%endmacro + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; MEM_DST = op mem_src1, mem_src2 +%macro op_mem_mem 5.nolist + %xdefine %%opCode %1 + %xdefine %%mem_dst %2 + %xdefine %%mem_src1 %3 + %xdefine %%mem_src2 %4 + %xdefine %%tmp %5 + + mov %%tmp, %%mem_src1 + %%opCode %%tmp, %%mem_src2 + mov %%mem_dst, %%tmp +%endmacro + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;;; Inputs: pDst: Destination (upto 1024 bits, 16 qwords) +;;; pA: Multiplicand (upto 512 bits, 8 qwords) +;;; +;;; +;;; Uses registers: +;;; rdx - implicit multiplicand +;;; (HIP:LOP)- 64x64-bit product (mulx) +;;; X7,,X0 - high result +;;; +%macro MUL_FIX 12-13.nolist + %xdefine %%N %1 + %xdefine %%pDst %2 + %xdefine %%pA %3 + %xdefine %%HIP %4 + %xdefine %%LOP %5 + %xdefine %%X0 %6 + %xdefine %%X1 %7 + %xdefine %%X2 %8 + %xdefine %%X3 %9 + %xdefine %%X4 %10 + %xdefine %%X5 %11 + %xdefine %%X6 %12 + %xdefine %%X7 %13 + + gsmulx %%X0, %%LOP, [%%pA] ; (X0:LO) = a[0]*b + %ifnempty %%pDst + mov [%%pDst], %%LOP + %endif + + %if %%N > 1 + gsmulx %%X1, %%HIP, [%%pA+sizeof(qword)] ; (X1:LO) = a[1]*b + add %%X0, %%HIP + %if %%N == 2 + adc %%X1, 0 + %endif + %endif + + %if %%N > 2 + gsmulx %%X2, %%LOP, [%%pA+sizeof(qword)*2] ; (X2:LO) = a[2]*b + adc %%X1, %%LOP + %if %%N == 3 + adc %%X2, 0 + %endif + %endif + + %if %%N > 3 + gsmulx %%X3, %%HIP, [%%pA+sizeof(qword)*3] ; (X3:LO) = a[3]*b + adc %%X2, %%HIP + %if %%N == 4 + adc %%X3, 0 + %endif + %endif + + %if %%N > 4 + gsmulx %%X4, %%LOP, [%%pA+sizeof(qword)*4] ; (X4:LO) = a[4]*b + adc %%X3, %%LOP + %if %%N == 5 + adc %%X4, 0 + %endif + %endif + + %if %%N > 5 + gsmulx %%X5, %%HIP, [%%pA+sizeof(qword)*5] ; (X5:LO) = a[5]*b + adc %%X4, %%HIP + %if %%N == 6 + adc %%X5, 0 + %endif + %endif + + %if %%N > 6 + gsmulx %%X6, %%LOP, [%%pA+sizeof(qword)*6] ; (X6:LO) = a[6]*b + adc %%X5, %%LOP + %if %%N == 7 + adc %%X6, 0 + %endif + %endif + + %if %%N > 7 + gsmulx %%X7, %%HIP, [%%pA+sizeof(qword)*7] ; (X7:LO) = a[7]*b + adc %%X6, %%HIP + adc %%X7, 0 + %endif +%endmacro + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; {x7,x6,x5,x4,x3,x2,x1,x0},DCT[0] = {x7,x6,x5,x4,x3,x2,x1,x0} + A[7:0]*B[i] +;; +;; uses rax, rdx +;; rdx - implicit multiplicand B[i] (should to be preloaded) +;; rax - temporary product +;; (HIP:LOP)- 64x64-bit product (mulx) +;; X7,,X0 - high result +;; +%macro MLA_FIX 6-13.nolist + %xdefine %%N %1 + %xdefine %%pDst %2 + %xdefine %%pA %3 + %xdefine %%HIP %4 + %xdefine %%LOP %5 + %xdefine %%X0 %6 + %xdefine %%X1 %7 + %xdefine %%X2 %8 + %xdefine %%X3 %9 + %xdefine %%X4 %10 + %xdefine %%X5 %11 + %xdefine %%X6 %12 + %xdefine %%X7 %13 + + xor rax, rax + + gsmulx %%HIP, %%LOP, [%%pA] ; (HI:LO) = a[0]*b[i] + gsadox %%X0, %%LOP + %ifnempty %%pDst + mov [%%pDst], %%X0 + %endif + + %if %%N == 1 + gsadox %%HIP, rax + mov %%X0, %%HIP + %else + + gsadcx %%X1, %%HIP + + gsmulx %%LOP,%%X0, [%%pA+sizeof(qword)] ; (LO:X0) = a[1]*b[i] + gsadox %%X0, %%X1 + + %if %%N == 2 + gsadcx %%LOP, rax + gsadox %%LOP, rax + mov %%X1, %%LOP + %else + + gsadcx %%X2, %%LOP + + gsmulx %%HIP,%%X1, [%%pA+sizeof(qword)*2] ; (HI:X1) = a[2]*b[i] + gsadox %%X1, %%X2 + + %if %%N == 3 + gsadcx %%HIP, rax + gsadox %%HIP, rax + mov %%X2, %%HIP + %else + + gsadcx %%X3, %%HIP + + gsmulx %%LOP,%%X2, [%%pA+sizeof(qword)*3] ; (LO,X2) = a[3]*b[i] + gsadox %%X2, %%X3 + + %if %%N == 4 + gsadcx %%LOP, rax + gsadox %%LOP, rax + mov %%X3, %%LOP + %else + + gsadcx %%X4, %%LOP + + gsmulx %%HIP,%%X3, [%%pA+sizeof(qword)*4] ; (HI:X3) = a[4]*b[i] + gsadox %%X3, %%X4 + + %if %%N == 5 + gsadcx %%HIP, rax + gsadox %%HIP, rax + mov %%X4, %%HIP + %else + + gsadcx %%X5, %%HIP + + gsmulx %%LOP,%%X4, [%%pA+sizeof(qword)*5] ; (LO:X4) = a[5]*b[i] + gsadox %%X4, %%X5 + + %if %%N == 6 + gsadcx %%LOP, rax + gsadox %%LOP, rax + mov %%X5, %%LOP + %else + + gsadcx %%X6, %%LOP + + gsmulx %%HIP,%%X5, [%%pA+sizeof(qword)*6] ; (HI:X5) = a[6]*b[i] + gsadox %%X5, %%X6 + + %if %%N == 7 + gsadcx %%HIP, rax + gsadox %%HIP, rax + mov %%X6, %%HIP + %else + + gsadcx %%HIP,%%X7 + + gsmulx %%X7, %%X6, [%%pA+sizeof(qword)*7] ; (LO:X6) = a[7]*b[i] + gsadox %%X6, %%HIP + gsadcx %%X7, rax + gsadox %%X7, rax + + %endif + %endif + %endif + %endif + %endif + %endif + %endif +%endmacro + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Inputs: pDst: Destination (upto 1024 bits, 16 qwords) +;;; pA: Multiplicand (upto 512 bits, 8 qwords) +;;; pB: Multiplicand (upto 512 bits, 8 qwords) +;;; +;;; +;;; Uses registers: +;;; rdx - implicit multiplicand +;;; (HIP:LOP)- 64x64-bit product (mulx) +;;; X0,,X7 - high result +;;; +%macro MUL_NxN 7-14.nolist + %xdefine %%N %1 + %xdefine %%pDst %2 + %xdefine %%pA %3 + %xdefine %%pB %4 + %xdefine %%HIP %5 + %xdefine %%LOP %6 + %xdefine %%X0 %7 + %xdefine %%X1 %8 + %xdefine %%X2 %9 + %xdefine %%X3 %10 + %xdefine %%X4 %11 + %xdefine %%X5 %12 + %xdefine %%X6 %13 + %xdefine %%X7 %14 + + mov rdx, [%%pB] + MUL_FIX {%%N},{%%pDst},{%%pA},{%%HIP},{%%LOP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} + + %if %%N > 1 + mov rdx, [%%pB+sizeof(qword)*1] + MLA_FIX {%%N},{%%pDst+sizeof(qword)*1},{%%pA},{%%HIP},{%%LOP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} + %endif + + %if %%N > 2 + mov rdx,[%%pB+sizeof(qword)*2] + MLA_FIX {%%N},{%%pDst+sizeof(qword)*2},{%%pA},{%%HIP},{%%LOP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} + %endif + + %if %%N > 3 + mov rdx,[%%pB+sizeof(qword)*3] + MLA_FIX {%%N},{%%pDst+sizeof(qword)*3},{%%pA},{%%HIP},{%%LOP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} + %endif + + %if %%N > 4 + mov rdx,[%%pB+sizeof(qword)*4] + MLA_FIX {%%N},{%%pDst+sizeof(qword)*4},{%%pA},{%%HIP},{%%LOP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} + %endif + + %if %%N > 5 + mov rdx,[%%pB+sizeof(qword)*5] + MLA_FIX {%%N},{%%pDst+sizeof(qword)*5},{%%pA},{%%HIP},{%%LOP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} + %endif + + %if %%N > 6 + mov rdx,[%%pB+sizeof(qword)*6] + MLA_FIX {%%N},{%%pDst+sizeof(qword)*6},{%%pA},{%%HIP},{%%LOP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} + %endif + + %if %%N > 7 + mov rdx,[%%pB+sizeof(qword)*7] + MLA_FIX {%%N},{%%pDst+sizeof(qword)*7},{%%pA},{%%HIP},{%%LOP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} + %endif + + %if %%N > 7 + mov qword [%%pDst+sizeof(qword)*8], %%X0 + mov qword [%%pDst+sizeof(qword)*9], %%X1 + mov qword [%%pDst+sizeof(qword)*10], %%X2 + mov qword [%%pDst+sizeof(qword)*11], %%X3 + mov qword [%%pDst+sizeof(qword)*12], %%X4 + mov qword [%%pDst+sizeof(qword)*13], %%X5 + mov qword [%%pDst+sizeof(qword)*14], %%X6 + mov qword [%%pDst+sizeof(qword)*15], %%X7 + %elif %%N > 6 + mov qword [%%pDst+sizeof(qword)*7], %%X0 + mov qword [%%pDst+sizeof(qword)*8], %%X1 + mov qword [%%pDst+sizeof(qword)*9], %%X2 + mov qword [%%pDst+sizeof(qword)*10], %%X3 + mov qword [%%pDst+sizeof(qword)*11], %%X4 + mov qword [%%pDst+sizeof(qword)*12], %%X5 + mov qword [%%pDst+sizeof(qword)*13], %%X6 + %elif %%N > 5 + mov qword [%%pDst+sizeof(qword)*6], %%X0 + mov qword [%%pDst+sizeof(qword)*7], %%X1 + mov qword [%%pDst+sizeof(qword)*8], %%X2 + mov qword [%%pDst+sizeof(qword)*9], %%X3 + mov qword [%%pDst+sizeof(qword)*10], %%X4 + mov qword [%%pDst+sizeof(qword)*11], %%X5 + %elif %%N > 4 + mov qword [%%pDst+sizeof(qword)*5], %%X0 + mov qword [%%pDst+sizeof(qword)*6], %%X1 + mov qword [%%pDst+sizeof(qword)*7], %%X2 + mov qword [%%pDst+sizeof(qword)*8], %%X3 + mov qword [%%pDst+sizeof(qword)*9], %%X4 + %elif %%N > 3 + mov qword [%%pDst+sizeof(qword)*4], %%X0 + mov qword [%%pDst+sizeof(qword)*5], %%X1 + mov qword [%%pDst+sizeof(qword)*6], %%X2 + mov qword [%%pDst+sizeof(qword)*7], %%X3 + %elif %%N > 2 + mov qword [%%pDst+sizeof(qword)*3], %%X0 + mov qword [%%pDst+sizeof(qword)*4], %%X1 + mov qword [%%pDst+sizeof(qword)*5], %%X2 + %elif %%N > 1 + mov qword [%%pDst+sizeof(qword)*2], %%X0 + mov qword [%%pDst+sizeof(qword)*3], %%X1 + %else + mov qword [%%pDst+sizeof(qword)*1], %%X0 + %endif ;; N==2 + +%endmacro + + +%endif ;; _ADCX_ADOX_ + +%endif ;; _PCPBNUMUL_FIX_ADCX_INC_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumulschool.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumulschool.inc new file mode 100644 index 0000000..b461d95 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumulschool.inc @@ -0,0 +1,1327 @@ +;=============================================================================== +; Copyright (C) 2010 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; BNU multiplication support +; + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; fixed multiplier macros +;; + +;; +;; {x0,x7,x6,x5,x4,x3,x2,x1},DCT[0] = OP*SRC2[] + {x7,x6,x5,x4,x3,x2,x1,x0} +;; +;; uses rax, rdx +;; OP=SRC1[i] should to be preloaded +;; TMP is destoyed +;; +%macro MLA_FIX 6-13.nolist + %xdefine %%N %1 + %xdefine %%pDst %2 + %xdefine %%SRC2 %3 + %xdefine %%OP %4 + %xdefine %%TMP %5 + %xdefine %%X7 %6 + %xdefine %%X6 %7 + %xdefine %%X5 %8 + %xdefine %%X4 %9 + %xdefine %%X3 %10 + %xdefine %%X2 %11 + %xdefine %%X1 %12 + %xdefine %%X0 %13 + + mov rax, [%%SRC2+8*0] + mul %%OP ; rdx:rax = OP * [] + add %%X0, rax + adc rdx, 0 + mov %%pDst, %%X0 + mov %%TMP, rdx + + %if %%N > 1 + mov rax, [%%SRC2+8*1] + mul %%OP ; rdx:rax = OP * [] + add %%X1, rax + adc rdx, 0 + add %%X1, %%TMP + adc rdx, 0 + %if %%N == 2 + mov %%X2, rdx + %else + mov %%TMP, rdx + %endif + + %if %%N > 2 + mov rax, [%%SRC2+8*2] + mul %%OP ; rdx:rax = OP * [] + add %%X2, rax + adc rdx, 0 + add %%X2, %%TMP + adc rdx, 0 + %if %%N == 3 + mov %%X3, rdx + %else + mov %%TMP, rdx + %endif + + %if %%N > 3 + mov rax, [%%SRC2+8*3] + mul %%OP ; rdx:rax = OP * [] + add %%X3, rax + adc rdx, 0 + add %%X3, %%TMP + adc rdx, 0 + %if %%N == 4 + mov %%X4, rdx + %else + mov %%TMP, rdx + %endif + + %if %%N > 4 + mov rax, [%%SRC2+8*4] + mul %%OP ; rdx:rax = OP * [] + add %%X4, rax + adc rdx, 0 + add %%X4, %%TMP + adc rdx, 0 + %if %%N == 5 + mov %%X5, rdx + %else + mov %%TMP, rdx + %endif + + %if %%N > 5 + mov rax, [%%SRC2+8*5] + mul %%OP ; rdx:rax = OP * [] + add %%X5, rax + adc rdx, 0 + add %%X5, %%TMP + adc rdx, 0 + %if %%N == 6 + mov %%X6, rdx + %else + mov %%TMP, rdx + %endif + + %if %%N > 6 + mov rax, [%%SRC2+8*6] + mul %%OP ; rdx:rax = OP * [] + add %%X6, rax + adc rdx, 0 + add %%X6, %%TMP + adc rdx, 0 + %if %%N == 7 + mov %%X7, rdx + %else + mov %%TMP, rdx + %endif + + %if %%N > 7 + mov rax, [%%SRC2+8*7] + mul %%OP ; rdx:rax = OP * [] + add %%X7, rax + adc rdx, 0 + add %%X7, %%TMP + adc rdx, 0 + mov %%X0, rdx + %endif + %endif + %endif + %endif + %endif + %endif + %endif +%endmacro + + +;;; Inputs: pDst: Destination (1024 bits, 16 qwords) +;;; pA: Multiplicand (512 bits, 8 qwords) +;;; pB: Multiplicand (512 bits, 8 qwords) +;;; +;;; +;;; Uses registers rax, rdx, args and rbp, rbx +;;; B operand in [pB] and also in x7...x0 +%macro MUL_NxN 7-14.nolist + %xdefine %%N %1 + %xdefine %%pDst %2 + %xdefine %%pA %3 + %xdefine %%pB %4 + %xdefine %%OP %5 + %xdefine %%TMP %6 + %xdefine %%X7 %7 + %xdefine %%X6 %8 + %xdefine %%X5 %9 + %xdefine %%X4 %10 + %xdefine %%X3 %11 + %xdefine %%X2 %12 + %xdefine %%X1 %13 + %xdefine %%X0 %14 + +mov %%OP, [%%pA+8*0] + + mov rax, %%X0 + mul %%OP ; rdx:rax = OP * [] + mov [%%pDst+8*0], rax + mov %%X0, rdx + + %if %%N > 1 + mov rax, %%X1 + mul %%OP ; rdx:rax = OP * [] + add %%X0, rax + adc rdx, 0 + mov %%X1, rdx + %endif + + %if %%N > 2 + mov rax, %%X2 + mul %%OP ; rdx:rax = OP * [] + add %%X1, rax + adc rdx, 0 + mov %%X2, rdx + %endif + + %if %%N > 3 + mov rax, %%X3 + mul %%OP ; rdx:rax = OP * [] + add %%X2, rax + adc rdx, 0 + mov %%X3, rdx + %endif + + %if %%N > 4 + mov rax, %%X4 + mul %%OP ; rdx:rax = OP * [] + add %%X3, rax + adc rdx, 0 + mov %%X4, rdx + %endif + + %if %%N > 5 + mov rax, %%X5 + mul %%OP ; rdx:rax = OP * [] + add %%X4, rax + adc rdx, 0 + mov %%X5, rdx + %endif + + %if %%N > 6 + mov rax, %%X6 + mul %%OP ; rdx:rax = OP * [] + add %%X5, rax + adc rdx, 0 + mov %%X6, rdx + %endif + + %if %%N > 7 + mov rax, %%X7 + mul %%OP ; rdx:rax = OP * [] + add %%X6, rax + adc rdx, 0 + mov %%X7, rdx + %endif + + %if %%N > 1 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ; {X0,X7,X6,X5,X4,X3,X2,X1},pDst[1] = pA[1]*pB[] + {X7,X6,X5,X4,X3,X2,X1,X0} + mov %%OP, [%%pA+sizeof(qword)*1] + MLA_FIX {%%N},[%%pDst+sizeof(qword)*1],{%%pB},{%%OP},{%%TMP},{%%X7},{%%X6},{%%X5},{%%X4},{%%X3},{%%X2},{%%X1},{%%X0} + %endif + + %if %%N > 2 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ; {X1,X0,X7,X6,X5,X4,X3,X2},pDst[2] = pA[2]*pB[] + {X0,X7,X6,X5,X4,X3,X2,X1} + mov %%OP, [%%pA+sizeof(qword)*2] + MLA_FIX {%%N},[%%pDst+sizeof(qword)*2],{%%pB},{%%OP},{%%TMP},{%%X0},{%%X7},{%%X6},{%%X5},{%%X4},{%%X3},{%%X2},{%%X1} + %endif + + %if %%N > 3 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ; {X2,X1,X0,X7,X6,X5,X4,X3},pDst[3] = pA[3]*pB[] + {X1,X0,X7,X6,X5,X4,X3,X2} + mov %%OP, [%%pA+sizeof(qword)*3] + MLA_FIX {%%N},[%%pDst+sizeof(qword)*3],{%%pB},{%%OP},{%%TMP},{%%X1},{%%X0},{%%X7},{%%X6},{%%X5},{%%X4},{%%X3},{%%X2} + %endif + + %if %%N > 4 + ; {X3,X2,X1,X0,X7,X6,X5,X4},pDst[4] = pA[4]*pB[] + {X2,X1,X0,X7,X6,X5,X4,X3} + mov %%OP, [%%pA+sizeof(qword)*4] + MLA_FIX {%%N},[%%pDst+sizeof(qword)*4],{%%pB},{%%OP},{%%TMP},{%%X2},{%%X1},{%%X0},{%%X7},{%%X6},{%%X5},{%%X4},{%%X3} + %endif + + %if %%N > 5 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ; {X4,X3,X2,X1,X0,X7,X6,X5},pDst[5] = pA[5]*pB[] + {X3,X2,X1,X0,X7,X6,X5,X4} + mov %%OP, [%%pA+sizeof(qword)*5] + MLA_FIX {%%N},[%%pDst+sizeof(qword)*5],{%%pB},{%%OP},{%%TMP},{%%X3},{%%X2},{%%X1},{%%X0},{%%X7},{%%X6},{%%X5},{%%X4} + %endif + + %if %%N > 6 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ; {X5,X4,X3,X2,X1,X0,X7,X6},pDst[6] = pA[6]*pB[] + {X4,X3,X2,X1,X0,X7,X6,X5} + mov %%OP, [%%pA+sizeof(qword)*6] + MLA_FIX {%%N},[%%pDst+sizeof(qword)*6],{%%pB},{%%OP},{%%TMP},{%%X4},{%%X3},{%%X2},{%%X1},{%%X0},{%%X7},{%%X6},{%%X5} + %endif + + %if %%N > 7 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ; {X6,X5,X4,X3,X2,X1,X0,X7},pDst[7] = pA[7]*pB[] + {X5,X4,X3,X2,X1,X0,X7,X6} + mov %%OP, [%%pA+sizeof(qword)*7] + MLA_FIX {%%N},[%%pDst+sizeof(qword)*7],{%%pB},{%%OP},{%%TMP},{%%X5},{%%X4},{%%X3},{%%X2},{%%X1},{%%X0},{%%X7},{%%X6} + %endif + + %if %%N > 7 + mov qword [%%pDst+sizeof(qword)*8], %%X7 + mov qword [%%pDst+sizeof(qword)*9], %%X0 + mov qword [%%pDst+sizeof(qword)*10], %%X1 + mov qword [%%pDst+sizeof(qword)*11], %%X2 + mov qword [%%pDst+sizeof(qword)*12], %%X3 + mov qword [%%pDst+sizeof(qword)*13], %%X4 + mov qword [%%pDst+sizeof(qword)*14], %%X5 + mov qword [%%pDst+sizeof(qword)*15], %%X6 + %elif %%N > 6 + mov qword [%%pDst+sizeof(qword)*7], %%X6 + mov qword [%%pDst+sizeof(qword)*8], %%X7 + mov qword [%%pDst+sizeof(qword)*9], %%X0 + mov qword [%%pDst+sizeof(qword)*10], %%X1 + mov qword [%%pDst+sizeof(qword)*11], %%X2 + mov qword [%%pDst+sizeof(qword)*12], %%X3 + mov qword [%%pDst+sizeof(qword)*13], %%X4 + %elif %%N > 5 + mov qword [%%pDst+sizeof(qword)*6], %%X5 + mov qword [%%pDst+sizeof(qword)*7], %%X6 + mov qword [%%pDst+sizeof(qword)*8], %%X7 + mov qword [%%pDst+sizeof(qword)*9], %%X0 + mov qword [%%pDst+sizeof(qword)*10], %%X1 + mov qword [%%pDst+sizeof(qword)*11], %%X2 + %elif %%N > 4 + mov qword [%%pDst+sizeof(qword)*5], %%X4 + mov qword [%%pDst+sizeof(qword)*6], %%X5 + mov qword [%%pDst+sizeof(qword)*7], %%X6 + mov qword [%%pDst+sizeof(qword)*8], %%X7 + mov qword [%%pDst+sizeof(qword)*9], %%X0 + %elif %%N > 3 + mov qword [%%pDst+sizeof(qword)*4], %%X3 + mov qword [%%pDst+sizeof(qword)*5], %%X4 + mov qword [%%pDst+sizeof(qword)*6], %%X5 + mov qword [%%pDst+sizeof(qword)*7], %%X6 + %elif %%N > 2 + mov qword [%%pDst+sizeof(qword)*3], %%X2 + mov qword [%%pDst+sizeof(qword)*4], %%X3 + mov qword [%%pDst+sizeof(qword)*5], %%X4 + %elif %%N > 1 + mov qword [%%pDst+sizeof(qword)*2], %%X1 + mov qword [%%pDst+sizeof(qword)*3], %%X2 + %else + mov qword [%%pDst+sizeof(qword)*1], %%X0 + %endif ;; N==2 + +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; MULx1 genaral-case multiplier macros +;; + +;; dst = src * B (body) +%macro MULx1 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%B %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + align 16 +%%L_1: + mul %%B + xor %%T1, %%T1 + add %%T0, rax + mov qword [%%rDst+%%idx*sizeof(qword)], %%T0 + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)] + adc %%T1, rdx + + mul %%B + xor %%T2, %%T2 + add %%T1, rax + mov qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)], %%T1 + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*2] + adc %%T2, rdx + + mul %%B + xor %%T3, %%T3 + add %%T2, rax + mov qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)*2], %%T2 + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*3] + adc %%T3, rdx + + mul %%B + xor %%T0, %%T0 + add %%T3, rax + mov qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)*3], %%T3 + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*4] + adc %%T0, rdx + + add %%idx, 4 + jnc %%L_1 +%endmacro + + +;; dst = src * B epilogue (srcLen=4*n+4) +%macro MULx1_4N_4_ELOG 7.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%B %3 + %xdefine %%T0 %4 + %xdefine %%T1 %5 + %xdefine %%T2 %6 + %xdefine %%T3 %7 + + mul %%B + xor %%T1, %%T1 + add %%T0, rax + mov qword [%%rDst], %%T0 + mov rax, qword [%%rSrc+sizeof(qword)] + adc %%T1, rdx + + mul %%B + xor %%T2, %%T2 + add %%T1, rax + mov qword [%%rDst+sizeof(qword)], %%T1 + mov rax, qword [%%rSrc+sizeof(qword)*2] + adc %%T2, rdx + + mul %%B + xor %%T3, %%T3 + add %%T2, rax + mov qword [%%rDst+sizeof(qword)*2], %%T2 + mov rax, qword [%%rSrc+sizeof(qword)*3] + adc %%T3, rdx + + mul %%B + mov idx, qword [rsp+counterA] + add %%T3, rax + mov qword [%%rDst+sizeof(qword)*3], %%T3 + mov rax, qword [%%rSrc+idx*sizeof(qword)] + adc rdx, 0 + ;mov rbp, rax + + mov qword [%%rDst+sizeof(qword)*4], rdx + add %%rDst, sizeof(qword) +%endmacro + + +;; dst = src * B epilogue (srcLen=4*n+3) +%macro MULx1_4N_3_ELOG 7.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%B %3 + %xdefine %%T0 %4 + %xdefine %%T1 %5 + %xdefine %%T2 %6 + %xdefine %%T3 %7 + + mul %%B + xor %%T1, %%T1 + add %%T0, rax + mov qword [%%rDst+sizeof(qword)], %%T0 + mov rax, qword [%%rSrc+sizeof(qword)*2] + adc %%T1, rdx + + mul %%B + xor %%T2, %%T2 + add %%T1, rax + mov qword [%%rDst+sizeof(qword)*2], %%T1 + mov rax, qword [%%rSrc+sizeof(qword)*3] + adc %%T2, rdx + + mul %%B + mov idx, qword [rsp+counterA] + add %%T2, rax + mov qword [%%rDst+sizeof(qword)*3], %%T2 + mov rax, qword [%%rSrc+idx*sizeof(qword)] + adc rdx, 0 + ;mov rbp, rax + + mov qword [%%rDst+sizeof(qword)*4], rdx + add %%rDst, sizeof(qword) +%endmacro + + +;; dst = src * B epilogue (srcLen=4*n+2) +%macro MULx1_4N_2_ELOG 7.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%B %3 + %xdefine %%T0 %4 + %xdefine %%T1 %5 + %xdefine %%T2 %6 + %xdefine %%T3 %7 + + mul %%B + xor %%T1, %%T1 + add %%T0, rax + mov qword [%%rDst+sizeof(qword)*2], %%T0 + mov rax, qword [%%rSrc+sizeof(qword)*3] + adc %%T1, rdx + + mul %%B + mov idx, qword [rsp+counterA] + add %%T1, rax + mov qword [%%rDst+sizeof(qword)*3], %%T1 + mov rax, qword [%%rSrc+idx*sizeof(qword)] + adc rdx, 0 + ;mov rbp, rax + + mov qword [%%rDst+sizeof(qword)*4], rdx + add %%rDst, sizeof(qword) +%endmacro + + +;; dst = src * B epilogue (srcLen=4*n+1) +%macro MULx1_4N_1_ELOG 7.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%B %3 + %xdefine %%T0 %4 + %xdefine %%T1 %5 + %xdefine %%T2 %6 + %xdefine %%T3 %7 + + mul %%B + mov idx, qword [rsp+counterA] + add %%T0, rax + mov qword [%%rDst+sizeof(qword)*3], %%T0 + mov rax, qword [%%rSrc+idx*sizeof(qword)] + adc rdx, 0 + ;mov rbp, rax + + mov qword [%%rDst+sizeof(qword)*4], rdx + add %%rDst, sizeof(qword) +%endmacro + + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; MULx2 genaral-case multiplier macros +;; + +; dst = src*{B1:B0} +%macro MULx2 9.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%B0 %4 + %xdefine %%B1 %5 + %xdefine %%T0 %6 + %xdefine %%T1 %7 + %xdefine %%T2 %8 + %xdefine %%T3 %9 + + align IPP_ALIGN_FACTOR +%%L_1: + mul %%B1 ; {T2:T1} += a[i]*B1 + xor %%T3, %%T3 + add %%T1, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)] ; a[i+1] + adc %%T2, rdx + + mul %%B0 ; {T3:T2:T1} += a[i+1]*B0 + mov qword [%%rDst+%%idx*sizeof(qword)], %%T0 + add %%T1, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)] ; a[i+1] + adc %%T2, rdx + adc %%T3, 0 + + mul %%B1 ; {T3:T2} += a[i+1]*B1 + xor %%T0, %%T0 + add %%T2, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*2] ; a[i+2] + adc %%T3, rdx + + mul %%B0 ; {T0:T3:T2} += a[i+2]*B0 + mov qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)], %%T1 + add %%T2, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*2] ; a[i+2] + adc %%T3, rdx + adc %%T0, 0 + + mul %%B1 ; {T0:T3} += a[i+2]*B1 + xor %%T1, %%T1 + add %%T3, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*3] ; a[i+3] + adc %%T0, rdx + + mul %%B0 ; {T1:T0:T3} += a[i+3]*B0 + mov qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)*2], %%T2 + add %%T3, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*3] ; a[i+3] + adc %%T0, rdx + adc %%T1, 0 + + mul %%B1 ; {T1:T0} += a[i+3]*B1 + xor %%T2, %%T2 + add %%T0, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*4] ; a[i+4] + adc %%T1, rdx + + mul %%B0 ; {T2:T1:T0} += a[i+4]*B0 + mov qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)*3], %%T3 + add %%T0, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*4] ; a[i+4] + adc %%T1, rdx + adc %%T2, 0 + + add %%idx, 4 + jnc %%L_1 +%endmacro + + +;; dst = src * {B1:B0} epilogue (srcLen=4*n+1) +%macro MULx2_4N_1_ELOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%B0 %3 + %xdefine %%B1 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mul %%B1 ; {T2:T1} += a[lenA-1]*B1 + add %%rDst, sizeof(qword)*2 + mov idx, [rsp+counterA] + mov qword [%%rDst+sizeof(qword)*3-sizeof(qword)*2], %%T0 + add %%T1, rax + mov rax, qword [%%rSrc+idx*sizeof(qword)] + adc rdx, %%T2 + + mov qword [%%rDst+sizeof(qword)*4-sizeof(qword)*2], %%T1 + mov qword [%%rDst+sizeof(qword)*5-sizeof(qword)*2], rdx +%endmacro + + +;; dst = src * {B1:B0} epilogue (srcLen=4*n+2) +%macro MULx2_4N_2_ELOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%B0 %3 + %xdefine %%B1 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mul %%B1 ; {T2:T1} += a[lenA-2]*B1 + xor %%T3, %%T3 + add %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[lenA-1] + adc %%T2, rdx + + mul %%B0 ; {T3:T2:T1} += a[LenA-1]*B0 + mov qword [%%rDst+sizeof(qword)*2], %%T0 + add %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[lenA-1] + adc %%T2, rdx + adc %%T3, 0 + + mul %%B1 ; {T3:T2} += a[lenA-1]*B1 + add %%rDst, sizeof(qword)*2 + mov idx, [rsp+counterA] + mov qword [%%rDst+sizeof(qword)*3-sizeof(qword)*2], %%T1 + add %%T2, rax + mov rax, qword [%%rSrc+idx*sizeof(qword)] + adc rdx, %%T3 + + mov qword [%%rDst+sizeof(qword)*4-sizeof(qword)*2], %%T2 + mov qword [%%rDst+sizeof(qword)*5-sizeof(qword)*2], rdx +%endmacro + + +;; dst = src * {B1:B0} epilogue (srcLen=4*n+3) +%macro MULx2_4N_3_ELOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%B0 %3 + %xdefine %%B1 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mul %%B1 ; {T2:T1} += a[lenA-3]*B1 + xor %%T3, %%T3 + add %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a[lenA-2] + adc %%T2, rdx + + mul %%B0 ; {T3:T2:T1} += a[LenA-2]*B0 + mov qword [%%rDst+sizeof(qword)], %%T0 + add %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a[lenA-2] + adc %%T2, rdx + adc %%T3, 0 + + mul %%B1 ; {T3:T2} += a[lenA-2]*B1 + xor %%T0, %%T0 + add %%T2, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[lenA-1] + adc %%T3, rdx + + mul %%B0 ; {T0:T3:T2} += a[lenA-1]*B0 + mov qword [%%rDst+sizeof(qword)*2], %%T1 + add %%T2, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[lenA-1] + adc %%T3, rdx + adc %%T0, 0 + + mul %%B1 ; {T0:T3} += a[lenA-1]*B1 + add %%rDst, sizeof(qword)*2 + mov idx, [rsp+counterA] + mov qword [%%rDst+sizeof(qword)*3-sizeof(qword)*2], %%T2 + add %%T3, rax + mov rax, qword [%%rSrc+idx*sizeof(qword)] + adc rdx, %%T0 + + mov qword [%%rDst+sizeof(qword)*4-sizeof(qword)*2], %%T3 + mov qword [%%rDst+sizeof(qword)*5-sizeof(qword)*2], rdx +%endmacro + + +;; dst = src * {B1:B0} epilogue (srcLen=4*n+4) +%macro MULx2_4N_4_ELOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%B0 %3 + %xdefine %%B1 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mul %%B1 ; {T2:T1} += a[lenA-4]*B1 + xor %%T3, %%T3 + add %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)] ; a[lenA-3] + adc %%T2, rdx + + mul %%B0 ; {T3:T2:T1} += a[lenA-3]*B0 + mov qword [%%rDst], %%T0 + add %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)] ; a[lenA-3] + adc %%T2, rdx + adc %%T3, 0 + + mul %%B1 ; {T3:T2} += a[lenA-3]*B1 + xor %%T0, %%T0 + add %%T2, rax + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a{lenA-2] + adc %%T3, rdx + + mul %%B0 ; {T0:T3:T2} += a[aLen-2]*B0 + mov qword [%%rDst+sizeof(qword)], %%T1 + add %%T2, rax + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a{lenA-2] + adc %%T3, rdx + adc %%T0, 0 + + mul %%B1 ; {T0:T3} += a[lenA-2]*B1 + xor %%T1, %%T1 + add %%T3, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[lenA-1] + adc %%T0, rdx + + mul %%B0 ; {T1:T0:T3} += a[lenA-1]*B0 + mov qword [%%rDst+sizeof(qword)*2], %%T2 + add %%T3, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[lenA-1] + adc %%T0, rdx + adc %%T1, 0 + + mul %%B1 ; {T1:T0} += a[lenA-1]*B1 + add %%rDst, sizeof(qword)*2 + mov idx, [rsp+counterA] + mov qword [%%rDst+sizeof(qword)*3-sizeof(qword)*2], %%T3 + add %%T0, rax + mov rax, qword [%%rSrc+idx*sizeof(qword)] + adc rdx, %%T1 + + mov qword [%%rDst+sizeof(qword)*4-sizeof(qword)*2], %%T0 + mov qword [%%rDst+sizeof(qword)*5-sizeof(qword)*2], rdx +%endmacro + + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; MLAx2 genaral-case multiplier macros +;; + +%macro MLAx2_PLOG 7.nolist + %xdefine %%B0 %1 + %xdefine %%B1 %2 + %xdefine %%dataB %3 + %xdefine %%T0 %4 + %xdefine %%T1 %5 + %xdefine %%T2 %6 + %xdefine %%T3 %7 + + mov %%B0, qword [%%dataB] + mul %%B0 ; {T2:T1:T0} = a[0]*b[i] + mov %%B1, qword [%%dataB+sizeof(qword)] + xor %%T2, %%T2 + mov %%T0, rax + mov rax, qword [rSrc+idx*sizeof(qword)] ; a[i] + mov %%T1, rdx +%endmacro + + + +%if (_IPP32E <= _IPP32E_Y8) +;; +;; Pre- Sandy Bridge specific code +;; + +%macro MLAx2 9.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%B0 %4 + %xdefine %%B1 %5 + %xdefine %%T0 %6 + %xdefine %%T1 %7 + %xdefine %%T2 %8 + %xdefine %%T3 %9 + +align IPP_ALIGN_FACTOR +%%L_1: + mul %%B1 ; {T2:T1} += a[i]*B1 + xor %%T3, %%T3 + add %%T1, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)] ; a[i+1] + adc %%T2, rdx + + mul %%B0 ; {T3:T2:T1} += a[i+1]*B0 + r[i] + add %%T0, qword [%%rDst+%%idx*sizeof(qword)] + adc %%T1, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)] ; a[i+1] + adc %%T2, rdx + mov qword [%%rDst+%%idx*sizeof(qword)], %%T0 + adc %%T3, 0 + + mul %%B1 ; {T3:T2} += a[i+1]*B1 + xor %%T0, %%T0 + add %%T2, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*2] ; a[i+2] + adc %%T3, rdx + + mul %%B0 ; {T0:T3:T2} += a[i+2]*B0 + r[i+1] + add %%T1, qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)] + adc %%T2, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*2] ; a[i+2] + adc %%T3, rdx + mov qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)], %%T1 + adc %%T0, 0 + + mul %%B1 ; {T0:T3} += a[i+2]*B1 + xor %%T1, %%T1 + add %%T3, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*3] ; a[i+3] + adc %%T0, rdx + + mul %%B0 ; {T1:T0:T3} += a[i+3]*B0 + r[i+2] + add %%T2, qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)*2] + adc %%T3, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*3] ; a[i+3] + adc %%T0, rdx + mov qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)*2], %%T2 + adc %%T1, 0 + + mul %%B1 ; {T1:T0} += a[i+3]*B1 + xor %%T2, %%T2 + add %%T0, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*4] ; a[i+4] + adc %%T1, rdx + + mul %%B0 ; {T2:T1:T0} += a[i+4]*B0 + r[i+3] + add %%T3, qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)*3] + adc %%T0, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*4] ; a[i+4] + adc %%T1, rdx + mov qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)*3], %%T3 + adc %%T2, 0 + + add %%idx, 4 + jnc %%L_1 +%endmacro + + +;; dst = + src * {B1:B0} epilogue (srcLen=4*n+1) +%macro MLAx2_4N_1_ELOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%B0 %3 + %xdefine %%B1 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mul %%B1 ; {T2:T1} += a[lenA-1]*B1 + r[lenA-1] + add %%rDst, sizeof(qword)*2 + mov idx, [rsp+counterA] + add %%T0, qword [%%rDst+sizeof(qword)*3-sizeof(qword)*2] + mov qword [%%rDst+sizeof(qword)*3-sizeof(qword)*2], %%T0 + adc %%T1, rax + mov rax, qword [%%rSrc+idx*sizeof(qword)] + adc rdx, %%T2 + + mov qword [%%rDst+sizeof(qword)*4-sizeof(qword)*2], %%T1 + mov qword [%%rDst+sizeof(qword)*5-sizeof(qword)*2], rdx +%endmacro + + +;; dst = dst + src * {B1:B0} epilogue (srcLen=4*n+2) +%macro MLAx2_4N_2_ELOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%B0 %3 + %xdefine %%B1 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mul %%B1 ; {T2:T1} += a[lenA-2]*B1 + xor %%T3, %%T3 + add %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[lenA-1] + adc %%T2, rdx + + mul %%B0 ; {T3:T2:T1} += a[lenA-1]*B0 + r[lenA-2] + add %%T0, qword [%%rDst+sizeof(qword)*2] + adc %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[lenA-1] + adc %%T2, rdx + mov qword [%%rDst+sizeof(qword)*2], %%T0 + adc %%T3, 0 + + mul %%B1 ; {T3:T2} += a[lenA-1]*B1 + r[lenA-1] + add %%rDst, sizeof(qword)*2 + mov idx, [rsp+counterA] + add %%T1, qword [%%rDst+sizeof(qword)*3-sizeof(qword)*2] + mov qword [%%rDst+sizeof(qword)*3-sizeof(qword)*2], %%T1 + adc %%T2, rax + mov rax, qword [%%rSrc+idx*sizeof(qword)] + adc rdx, %%T3 + + mov qword [%%rDst+sizeof(qword)*4-sizeof(qword)*2], %%T2 + mov qword [%%rDst+sizeof(qword)*5-sizeof(qword)*2], rdx +%endmacro + + +;; dst = + src * {B1:B0} epilogue (srcLen=4*n+3) +%macro MLAx2_4N_3_ELOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%B0 %3 + %xdefine %%B1 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mul %%B1 ; {T2:T1} += a[lenA-3]*B1 + xor %%T3, %%T3 + add %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a[lenA-2] + adc %%T2, rdx + + mul %%B0 ; {T3:T2:T1} += a[LenA-2]*B0 + r[len-3] + add %%T0, qword [%%rDst+sizeof(qword)] + adc %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a[lenA-2] + adc %%T2, rdx + mov qword [%%rDst+sizeof(qword)], %%T0 + adc %%T3, 0 + + mul %%B1 ; {T3:T2} += a[lenA-2]*B1 + xor %%T0, %%T0 + add %%T2, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[lenA-1] + adc %%T3, rdx + + mul %%B0 ; {T0:T3:T2} += a[lenA-1]*B0 + r[lenA-2] + add %%T1, qword [%%rDst+sizeof(qword)*2] + adc %%T2, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[lenA-1] + adc %%T3, rdx + mov qword [%%rDst+sizeof(qword)*2], %%T1 + adc %%T0, 0 + + mul %%B1 ; {T0:T3} += a[lenA-1]*B1 + r[lenA-1] + add %%rDst, sizeof(qword)*2 + mov idx, [rsp+counterA] + add %%T2, qword [%%rDst+sizeof(qword)*3-sizeof(qword)*2] + mov qword [%%rDst+sizeof(qword)*3-sizeof(qword)*2], %%T2 + adc %%T3, rax + mov rax, qword [%%rSrc+idx*sizeof(qword)] + adc rdx, %%T0 + + mov qword [%%rDst+sizeof(qword)*4-sizeof(qword)*2], %%T3 + mov qword [%%rDst+sizeof(qword)*5-sizeof(qword)*2], rdx +%endmacro + + +;; dst = + src * {B1:B0} epilogue (srcLen=4*n+4) +%macro MLAx2_4N_4_ELOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%B0 %3 + %xdefine %%B1 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mul %%B1 ; {T2:T1} += a[lenA-4]*B1 + xor %%T3, %%T3 + add %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)] ; a[lenA-3] + adc %%T2, rdx + + mul %%B0 ; {T3:T2:T1} += a[lenA-3]*B0 + r[lenA-4] + add %%T0, qword [%%rDst] + adc %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)] ; a[lenA-3] + adc %%T2, rdx + mov qword [%%rDst], %%T0 + adc %%T3, 0 + + mul %%B1 ; {T3:T2} += a[lenA-3]*B1 + xor %%T0, %%T0 + add %%T2, rax + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a{lenA-2] + adc %%T3, rdx + + mul %%B0 ; {T0:T3:T2} += a[aLen-2]*B0 + r[lenA-3] + add %%T1, qword [%%rDst+sizeof(qword)] + adc %%T2, rax + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a{lenA-2] + adc %%T3, rdx + mov qword [%%rDst+sizeof(qword)], %%T1 + adc %%T0, 0 + + mul %%B1 ; {T0:T3} += a[lenA-2]*B1 + xor %%T1, %%T1 + add %%T3, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[lenA-1] + adc %%T0, rdx + + mul %%B0 ; {T1:T0:T3} += a[lenA-1]*B0 + r[lenA-2] + add %%T2, qword [%%rDst+sizeof(qword)*2] + adc %%T3, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[lenA-1] + adc %%T0, rdx + mov qword [%%rDst+sizeof(qword)*2], %%T2 + adc %%T1, 0 + + mul %%B1 ; {T1:T0} += a[lenA-1]*B1 + r[lenA-1] + add %%rDst, sizeof(qword)*2 + mov idx, [rsp+counterA] + add %%T3, qword [%%rDst+sizeof(qword)*3-sizeof(qword)*2] + mov qword [%%rDst+sizeof(qword)*3-sizeof(qword)*2], %%T3 + adc %%T0, rax + mov rax, qword [%%rSrc+idx*sizeof(qword)] + adc rdx, %%T1 + + mov qword [%%rDst+sizeof(qword)*4-sizeof(qword)*2], %%T0 + mov qword [%%rDst+sizeof(qword)*5-sizeof(qword)*2], rdx +%endmacro + +%endif + +%if (_IPP32E >= _IPP32E_E9) +;; +;; Sandy Bridge specific code +;; +%macro MLAx2 9.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%B0 %4 + %xdefine %%B1 %5 + %xdefine %%T0 %6 + %xdefine %%T1 %7 + %xdefine %%T2 %8 + %xdefine %%T3 %9 + +align IPP_ALIGN_FACTOR +%%L_1: + mul %%B1 ; {T2:T1} += a[i]*B1 + xor %%T3, %%T3 + add %%T1, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)] ; a[i+1] + adc %%T2, rdx + + mul %%B0 ; {T3:T2:T1} += a[i+1]*B0 + r[i] + add %%T0, qword [%%rDst+%%idx*sizeof(qword)] + mov qword [%%rDst+%%idx*sizeof(qword)], %%T0 + adc %%T1, rax + adc %%T2, rdx + adc %%T3, 0 + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)] ; a[i+1] + + mul %%B1 ; {T3:T2} += a[i+1]*B1 + xor %%T0, %%T0 + add %%T2, rax + adc %%T3, rdx + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*2] ; a[i+2] + + mul %%B0 ; {T0:T3:T2} += a[i+2]*B0 + r[i+1] + add %%T1, qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)] + mov qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)], %%T1 + adc %%T2, rax + adc %%T3, rdx + adc %%T0, 0 + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*2] ; a[i+2] + + mul %%B1 ; {T0:T3} += a[i+2]*B1 + xor %%T1, %%T1 + add %%T3, rax + adc %%T0, rdx + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*3] ; a[i+3] + + mul %%B0 ; {T1:T0:T3} += a[i+3]*B0 + r[i+2] + add %%T2, qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)*2] + mov qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)*2], %%T2 + adc %%T3, rax + adc %%T0, rdx + adc %%T1, 0 + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*3] ; a[i+3] + + mul %%B1 ; {T1:T0} += a[i+3]*B1 + xor %%T2, %%T2 + add %%T0, rax + adc %%T1, rdx + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*4] ; a[i+4] + + mul %%B0 ; {T2:T1:T0} += a[i+4]*B0 + r[i+3] + add %%T3, qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)*3] + mov qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)*3], %%T3 + adc %%T0, rax + adc %%T1, rdx + adc %%T2, 0 + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*4] ; a[i+4] + + add %%idx, 4 + jnc %%L_1 +%endmacro + + +;; dst = + src * {B1:B0} epilogue (srcLen=4*n+1) +%macro MLAx2_4N_1_ELOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%B0 %3 + %xdefine %%B1 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mul %%B1 ; {T2:T1} += a[lenA-1]*B1 + r[lenA-1] + add %%rDst, sizeof(qword)*2 + mov idx, [rsp+counterA] + add %%T0, qword [%%rDst+sizeof(qword)*3-sizeof(qword)*2] + mov qword [%%rDst+sizeof(qword)*3-sizeof(qword)*2], %%T0 + adc %%T1, rax + adc rdx, %%T2 + mov rax, qword [%%rSrc+idx*sizeof(qword)] + + mov qword [%%rDst+sizeof(qword)*4-sizeof(qword)*2], %%T1 + mov qword [%%rDst+sizeof(qword)*5-sizeof(qword)*2], rdx +%endmacro + + +;; dst = dst + src * {B1:B0} epilogue (srcLen=4*n+2) +%macro MLAx2_4N_2_ELOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%B0 %3 + %xdefine %%B1 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mul %%B1 ; {T2:T1} += a[lenA-2]*B1 + xor %%T3, %%T3 + add %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[lenA-1] + adc %%T2, rdx + + mul %%B0 ; {T3:T2:T1} += a[lenA-1]*B0 + r[lenA-2] + add %%T0, qword [%%rDst+sizeof(qword)*2] + mov qword [%%rDst+sizeof(qword)*2], %%T0 + adc %%T1, rax + adc %%T2, rdx + adc %%T3, 0 + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[lenA-1] + + mul %%B1 ; {T3:T2} += a[lenA-1]*B1 + r[lenA-1] + add %%rDst, sizeof(qword)*2 + mov idx, [rsp+counterA] + add %%T1, qword [%%rDst+sizeof(qword)*3-sizeof(qword)*2] + mov qword [%%rDst+sizeof(qword)*3-sizeof(qword)*2], %%T1 + adc %%T2, rax + adc rdx, %%T3 + mov rax, qword [%%rSrc+idx*sizeof(qword)] + + mov qword [%%rDst+sizeof(qword)*4-sizeof(qword)*2], %%T2 + mov qword [%%rDst+sizeof(qword)*5-sizeof(qword)*2], rdx +%endmacro + + +;; dst = + src * {B1:B0} epilogue (srcLen=4*n+3) +%macro MLAx2_4N_3_ELOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%B0 %3 + %xdefine %%B1 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mul %%B1 ; {T2:T1} += a[lenA-3]*B1 + xor %%T3, %%T3 + add %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a[lenA-2] + adc %%T2, rdx + + mul %%B0 ; {T3:T2:T1} += a[LenA-2]*B0 + r[len-3] + add %%T0, qword [%%rDst+sizeof(qword)] + mov qword [%%rDst+sizeof(qword)], %%T0 + adc %%T1, rax + adc %%T2, rdx + adc %%T3, 0 + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a[lenA-2] + + mul %%B1 ; {T3:T2} += a[lenA-2]*B1 + xor %%T0, %%T0 + add %%T2, rax + adc %%T3, rdx + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[lenA-1] + + mul %%B0 ; {T0:T3:T2} += a[lenA-1]*B0 + r[lenA-2] + add %%T1, qword [%%rDst+sizeof(qword)*2] + mov qword [%%rDst+sizeof(qword)*2], %%T1 + adc %%T2, rax + adc %%T3, rdx + adc %%T0, 0 + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[lenA-1] + + mul %%B1 ; {T0:T3} += a[lenA-1]*B1 + r[lenA-1] + add %%rDst, sizeof(qword)*2 + mov idx, [rsp+counterA] + add %%T2, qword [%%rDst+sizeof(qword)*3-sizeof(qword)*2] + mov qword [%%rDst+sizeof(qword)*3-sizeof(qword)*2], %%T2 + adc %%T3, rax + adc rdx, %%T0 + mov rax, qword [%%rSrc+idx*sizeof(qword)] + + mov qword [%%rDst+sizeof(qword)*4-sizeof(qword)*2], %%T3 + mov qword [%%rDst+sizeof(qword)*5-sizeof(qword)*2], rdx +%endmacro + + +;; dst = + src * {B1:B0} epilogue (srcLen=4*n+4) +%macro MLAx2_4N_4_ELOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%B0 %3 + %xdefine %%B1 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mul %%B1 ; {T2:T1} += a[lenA-4]*B1 + xor %%T3, %%T3 + add %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)] ; a[lenA-3] + adc %%T2, rdx + + mul %%B0 ; {T3:T2:T1} += a[lenA-3]*B0 + r[lenA-4] + add %%T0, qword [%%rDst] + mov qword [%%rDst], %%T0 + adc %%T1, rax + adc %%T2, rdx + adc %%T3, 0 + mov rax, qword [%%rSrc+sizeof(qword)] ; a[lenA-3] + + mul %%B1 ; {T3:T2} += a[lenA-3]*B1 + xor %%T0, %%T0 + add %%T2, rax + adc %%T3, rdx + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a{lenA-2] + + mul %%B0 ; {T0:T3:T2} += a[aLen-2]*B0 + r[lenA-3] + add %%T1, qword [%%rDst+sizeof(qword)] + mov qword [%%rDst+sizeof(qword)], %%T1 + adc %%T2, rax + adc %%T3, rdx + adc %%T0, 0 + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a{lenA-2] + + mul %%B1 ; {T0:T3} += a[lenA-2]*B1 + xor %%T1, %%T1 + add %%T3, rax + adc %%T0, rdx + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[lenA-1] + + mul %%B0 ; {T1:T0:T3} += a[lenA-1]*B0 + r[lenA-2] + add %%T2, qword [%%rDst+sizeof(qword)*2] + mov qword [%%rDst+sizeof(qword)*2], %%T2 + adc %%T3, rax + adc %%T0, rdx + adc %%T1, 0 + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[lenA-1] + + mul %%B1 ; {T1:T0} += a[lenA-1]*B1 + r[lenA-1] + add %%rDst, sizeof(qword)*2 + mov idx, [rsp+counterA] + add %%T3, qword [%%rDst+sizeof(qword)*3-sizeof(qword)*2] + mov qword [%%rDst+sizeof(qword)*3-sizeof(qword)*2], %%T3 + adc %%T0, rax + adc rdx, %%T1 + mov rax, qword [%%rSrc+idx*sizeof(qword)] + + mov qword [%%rDst+sizeof(qword)*4-sizeof(qword)*2], %%T0 + mov qword [%%rDst+sizeof(qword)*5-sizeof(qword)*2], rdx +%endmacro + +%endif +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumulschoolm7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumulschoolm7as.asm new file mode 100644 index 0000000..f6f6a2c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumulschoolm7as.asm @@ -0,0 +1,379 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Big Number Operations +; +; Content: +; cpMulAdc_BNU_school() +; +; + +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpbnumulschool.inc" +%include "pcpvariant.inc" + +%if (_ADCOX_NI_ENABLING_ == _FEATURE_OFF_) || (_ADCOX_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_M7) && (_IPP32E < _IPP32E_L9) + + +segment .text align=IPP_ALIGN_FACTOR + + +;************************************************************* +;* Ipp64u cpMulAdc_BNU_school(Ipp64u* pR; +;* const Ipp64u* pA, int aSize, +;* const Ipp64u* pB, int bSize) +;* returns pR[aSize+bSize] +;* +;************************************************************* +align IPP_ALIGN_FACTOR +IPPASM cpMulAdc_BNU_school,PUBLIC +%assign LOCAL_FRAME (1*sizeof(qword)) + USES_GPR rbx,rbp,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 5 + +; rdi = pDst +; rsi = pSrcA +; edx = lenA +; rcx = pSrcB +; r8d = lenB + +;; +;; stack structure: +;;counterB = (0) +;;counterA = (counterB+sizeof(qword)) +%assign counterA (0) + + + cmp edx, r8d + jl .general_case_mul_entry + jg .general_case_mul +%if (_IPP32E < _IPP32E_E9) + cmp edx, 4 +%else + cmp edx, 8 +%endif + jg .general_case_mul + +%if (_IPP32E >= _IPP32E_E9) + cmp edx, 4 + jg .more_then_4 +%endif + + cmp edx, 3 + ja .mul_4x4 + jz .mul_3x3 + jp .mul_2x2 + ; mul_1x1 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; fixed-size multipliers (1-4) +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +.mul_1x1: + mov rax, qword [rsi] + mul qword [rcx] + mov qword [rdi], rax + mov qword [rdi+sizeof(qword)], rdx + mov rax, qword [rdi+sizeof(qword)*1] + REST_XMM + REST_GPR + ret + +align IPP_ALIGN_FACTOR +.mul_2x2: + mov r8, [rcx] + mov r9, [rcx+sizeof(qword)*1] + MUL_NxN 2, rdi, rsi, rcx, rbx, rbp, r15, r14, r13, r12, r11, r10, r9, r8 + mov rax, qword [rdi+sizeof(qword)*3] + REST_XMM + REST_GPR + ret + +align IPP_ALIGN_FACTOR +.mul_3x3: + mov r8, [rcx] + mov r9, [rcx+sizeof(qword)*1] + mov r10,[rcx+sizeof(qword)*2] + MUL_NxN 3, rdi, rsi, rcx, rbx, rbp, r15, r14, r13, r12, r11, r10, r9, r8 + mov rax, qword [rdi+sizeof(qword)*5] + REST_XMM + REST_GPR + ret + +align IPP_ALIGN_FACTOR +.mul_4x4: + mov r8, [rcx] + mov r9, [rcx+sizeof(qword)*1] + mov r10,[rcx+sizeof(qword)*2] + mov r11,[rcx+sizeof(qword)*3] + MUL_NxN 4, rdi, rsi, rcx, rbx, rbp, r15, r14, r13, r12, r11, r10, r9, r8 + mov rax, qword [rdi+sizeof(qword)*7] + REST_XMM + REST_GPR + ret + +%if (_IPP32E >= _IPP32E_E9) +.more_then_4: + cmp edx, 7 + ja .mul_8x8 + jz .mul_7x7 + jp .mul_6x6 + ; mul_5x5 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; fixed-size multipliers (5-8) +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +.mul_5x5: + mov r8, [rcx] + mov r9, [rcx+sizeof(qword)*1] + mov r10,[rcx+sizeof(qword)*2] + mov r11,[rcx+sizeof(qword)*3] + mov r12,[rcx+sizeof(qword)*4] + MUL_NxN 5, rdi, rsi, rcx, rbx, rbp, r15, r14, r13, r12, r11, r10, r9, r8 + mov rax, qword [rdi+sizeof(qword)*9] + REST_XMM + REST_GPR + ret + +align IPP_ALIGN_FACTOR +.mul_6x6: + mov r8, [rcx] + mov r9, [rcx+sizeof(qword)*1] + mov r10,[rcx+sizeof(qword)*2] + mov r11,[rcx+sizeof(qword)*3] + mov r12,[rcx+sizeof(qword)*4] + mov r13,[rcx+sizeof(qword)*5] + MUL_NxN 6, rdi, rsi, rcx, rbx, rbp, r15, r14, r13, r12, r11, r10, r9, r8 + mov rax, qword [rdi+sizeof(qword)*11] + REST_XMM + REST_GPR + ret + +align IPP_ALIGN_FACTOR +.mul_7x7: + mov r8, [rcx] + mov r9, [rcx+sizeof(qword)*1] + mov r10,[rcx+sizeof(qword)*2] + mov r11,[rcx+sizeof(qword)*3] + mov r12,[rcx+sizeof(qword)*4] + mov r13,[rcx+sizeof(qword)*5] + mov r14,[rcx+sizeof(qword)*6] + MUL_NxN 7, rdi, rsi, rcx, rbx, rbp, r15, r14, r13, r12, r11, r10, r9, r8 + mov rax, qword [rdi+sizeof(qword)*13] + REST_XMM + REST_GPR + ret + +align IPP_ALIGN_FACTOR +.mul_8x8: + mov r8, [rcx] + mov r9, [rcx+sizeof(qword)*1] + mov r10,[rcx+sizeof(qword)*2] + mov r11,[rcx+sizeof(qword)*3] + mov r12,[rcx+sizeof(qword)*4] + mov r13,[rcx+sizeof(qword)*5] + mov r14,[rcx+sizeof(qword)*6] + mov r15,[rcx+sizeof(qword)*7] + MUL_NxN 8, rdi, rsi, rcx, rbx, rbp, r15, r14, r13, r12, r11, r10, r9, r8 + mov rax, qword [rdi+sizeof(qword)*15] + REST_XMM + REST_GPR + ret +%endif + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; general case multiplier +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +.general_case_mul_entry: + ; swap operands %if lenA < lenB then exchange operands + xor rsi, rcx + xor edx, r8d + xor rcx, rsi + xor r8d, edx + xor rsi, rcx + xor edx, r8d + +%xdefine B0 r10 ; b[i], b[i+1] +%xdefine B1 r11 + +%xdefine T0 r12 ; temporary +%xdefine T1 r13 +%xdefine T2 r14 +%xdefine T3 r15 + +%xdefine idx rbx ; index +%xdefine rDst rdi +%xdefine rSrc rsi + +align IPP_ALIGN_FACTOR +.general_case_mul: + movsxd rdx, edx ; expand length + movsxd r8, r8d + + lea rdi, [rdi+rdx*sizeof(qword)-sizeof(qword)*4] ; rdi = &R[lenA-4] + lea rsi, [rsi+rdx*sizeof(qword)-sizeof(qword)*4] ; rsi = &A[lenA-4] + + mov idx, dword 4 ; negative + sub idx, rdx ; A-counter + mov qword [rsp+counterA], idx + + mov rax, qword [rsi+idx*sizeof(qword)] ; a[0] + mov B0, qword [rcx] ; b[0] + test r8, 1 + jz .init_even_B + +;********** lenSrcB = 2*n+ 1 (multiply only) ********************* +.init_odd_B: + xor T0, T0 + cmp idx, 0 + jge .skip_mul1 + + MULx1 rdi, rsi, idx, B0, T0, T1, T2, T3 + +.skip_mul1: + cmp idx, 2 + ja .fin_mul1x4n_1 ; idx=3 + jz .fin_mul1x4n_2 ; idx=2 + jp .fin_mul1x4n_3 ; idx=1 + ; fin_mul1x4n_4 ; idx=0 + +.fin_mul1x4n_4: + MULx1_4N_4_ELOG rdi, rsi, B0, T0,T1,T2,T3 + add rcx, sizeof(qword) + add r8, 1 + jmp .mla2x4n_4 +.fin_mul1x4n_3: + MULx1_4N_3_ELOG rdi, rsi, B0, T0,T1,T2,T3 + add rcx, sizeof(qword) + add r8, 1 + jmp .mla2x4n_3 +.fin_mul1x4n_2: + MULx1_4N_2_ELOG rdi, rsi, B0, T0,T1,T2,T3 + add rcx, sizeof(qword) + add r8, 1 + jmp .mla2x4n_2 +.fin_mul1x4n_1: + MULx1_4N_1_ELOG rdi, rsi, B0, T0,T1,T2,T3 + add rcx, sizeof(qword) + add r8, 1 + jmp .mla2x4n_1 + + +;********** lenSrcB = 2*n (multiply only) ************************ +.init_even_B: + mov rbp, rax + mul B0 ; {T2:T1:T0} = a[0]*B0 + mov B1, qword [rcx+sizeof(qword)] + xor T2, T2 + mov T0, rax + mov rax, rbp ; restore a[0] + mov T1, rdx + + cmp idx, 0 + jge .skip_mul_nx2 + + MULx2 rdi, rsi, idx, B0,B1, T0,T1,T2,T3 + +.skip_mul_nx2: + cmp idx, 2 + ja .fin_mul2x4n_1 ; idx=3 + jz .fin_mul2x4n_2 ; idx=2 + jp .fin_mul2x4n_3 ; idx=1 + ; fin_mul2x4n_4 ; idx=0 + +.fin_mul2x4n_4: + MULx2_4N_4_ELOG rdi, rsi, B0,B1, T0,T1,T2,T3 + add rcx, sizeof(qword)*2 +align IPP_ALIGN_FACTOR +.mla2x4n_4: + sub r8, 2 + jz .quit + MLAx2_PLOG B0,B1, rcx, T0,T1,T2,T3 + cmp idx, 0 + jz .skip_mla_x2 + MLAx2 rdi, rsi, idx, B0,B1, T0,T1,T2,T3 +.skip_mla_x2: + MLAx2_4N_4_ELOG rdi, rsi, B0,B1, T0,T1,T2,T3 + add rcx, sizeof(qword)*2 + jmp .mla2x4n_4 + +.fin_mul2x4n_3: + MULx2_4N_3_ELOG rdi, rsi, B0,B1, T0,T1,T2,T3 + add rcx, sizeof(qword)*2 +align IPP_ALIGN_FACTOR +.mla2x4n_3: + sub r8, 2 + jz .quit + MLAx2_PLOG B0,B1, rcx, T0,T1,T2,T3 + MLAx2 rdi, rsi, idx, B0,B1, T0,T1,T2,T3 + MLAx2_4N_3_ELOG rdi, rsi, B0,B1, T0,T1,T2,T3 + add rcx, sizeof(qword)*2 + jmp .mla2x4n_3 + +.fin_mul2x4n_2: + MULx2_4N_2_ELOG rdi, rsi, B0,B1, T0,T1,T2,T3 + add rcx, sizeof(qword)*2 +align IPP_ALIGN_FACTOR +.mla2x4n_2: + sub r8, 2 + jz .quit + MLAx2_PLOG B0,B1, rcx, T0,T1,T2,T3 + MLAx2 rdi, rsi, idx, B0,B1, T0,T1,T2,T3 + MLAx2_4N_2_ELOG rdi, rsi, B0,B1, T0,T1,T2,T3 + add rcx, sizeof(qword)*2 + jmp .mla2x4n_2 + +.fin_mul2x4n_1: + MULx2_4N_1_ELOG rdi, rsi, B0,B1, T0,T1,T2,T3 + add rcx, sizeof(qword)*2 +align IPP_ALIGN_FACTOR +.mla2x4n_1: + sub r8, 2 + jz .quit + MLAx2_PLOG B0,B1, rcx, T0,T1,T2,T3 + MLAx2 rdi, rsi, idx, B0,B1, T0,T1,T2,T3 + MLAx2_4N_1_ELOG rdi, rsi, B0,B1, T0,T1,T2,T3 + add rcx, sizeof(qword)*2 + jmp .mla2x4n_1 + +.quit: + mov rax, rdx + + REST_XMM + REST_GPR + ret +ENDFUNC cpMulAdc_BNU_school + +%endif + +%endif ;; _ADCOX_NI_ENABLING_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumulschoolsrvl9.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumulschoolsrvl9.asm new file mode 100644 index 0000000..5c495bf --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumulschoolsrvl9.asm @@ -0,0 +1,308 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Big Number Multiplicative Operations +; +; Content: +; cpMulAdc_BNU_school() +; cpSqrAdc_BNU_school() +; cpMontRedAdc_BNU() +; +; Implementation is using mulx and adcx/adox instruvtions +; +; + +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_ADCOX_NI_ENABLING_ == _FEATURE_ON_) || (_ADCOX_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_L9) + +%assign _xEMULATION_ 1 + +segment .text align=IPP_ALIGN_FACTOR + + +%include "pcpbnumul.inc" +%include "pcpbnusqr.inc" +%include "pcpmred.inc" + +;************************************************************* +;* Ipp64u cpMulAdc_BNU_school(Ipp64u* pR; +;* const Ipp64u* pA, int aSize, +;* const Ipp64u* pB, int bSize) +;* +;************************************************************* +align IPP_ALIGN_FACTOR +IPPASM cpMulAdc_BNU_school,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbx,rbp,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 5 + +; rdi = pR +; rsi = pA +; edx = nsA +; rcx = pB +; r8d = nsB + + movsxd rdx, edx ; expand length + movsxd rbx, r8d + + xor r8, r8 ; clear scratch + xor r9, r9 + xor r10, r10 + xor r11, r11 + xor r12, r12 + xor r13, r13 + xor r14, r14 + xor r15, r15 + + cmp rdx, rbx + jl .swap_operans ; nsA < nsB + jg .test_8N_case ; test %if nsA=8*N and nsB=8*M + + cmp rdx, 16 + jg .test_8N_case + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; short nsA==nsB (1,..,16) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + cmp rdx, 4 + jg .more_then_4 + + cmp edx, 3 + ja .mul_4_4 + jz .mul_3_3 + jp .mul_2_2 + ; mul_1_1 + +.mul_1_1: + MUL_NxN 1, rdi, rsi, rcx, rbx,rbp, r8 + jmp .quit +.mul_2_2: + MUL_NxN 2, rdi, rsi, rcx, rbx,rbp, r8,r9 + jmp .quit +.mul_3_3: + MUL_NxN 3, rdi, rsi, rcx, rbx,rbp, r8,r9,r10 + jmp .quit +.mul_4_4: + MUL_NxN 4, rdi, rsi, rcx, rbx,rbp, r8,r9,r10,r11 + jmp .quit + +.more_then_4: + GET_EP rax, mul_lxl_basic, rdx, rbp + call rax + jmp .quit + +.swap_operans: + SWAP rsi, rcx ; swap operands + SWAP rdx, rbx + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; 8*N x 8*M case multiplier +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +.test_8N_case: + mov rax, rdx + or rax, rbx + and rax, 7 + jnz .general_mul + + CALL_FUNC mul_8Nx8M + jmp .quit + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; general case multiplier +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +.general_mul: + CALL_FUNC mul_NxM + jmp .quit + +.quit: + REST_XMM + REST_GPR + ret +ENDFUNC cpMulAdc_BNU_school + +;************************************************************* +;* +;* Ipp64u cpSqrAdc_BNU_school(Ipp64u* pR; +;* const Ipp64u* pA, int aSize) +;* +;************************************************************* +align IPP_ALIGN_FACTOR +IPPASM cpSqrAdc_BNU_school,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbx,rbp,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 3 + + movsxd rdx, edx ; expand length + + xor r8, r8 ; clear scratch + xor r9, r9 + xor r10, r10 + xor r11, r11 + xor r12, r12 + xor r13, r13 + xor r14, r14 + xor r15, r15 + + cmp rdx, 16 + jg .test_8N_case + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; short nsA (1,..,16) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + GET_EP rax, sqr_l_basic, rdx, rbp + call rax + jmp .quit + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; 8N case squarer +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +.test_8N_case: + test rdx, 7 + jnz .general_sqr + + CALL_FUNC sqr_8N + jmp .quit + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; general case squarer +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +.general_sqr: + CALL_FUNC sqr_N + +.quit: + REST_XMM + REST_GPR + ret +ENDFUNC cpSqrAdc_BNU_school + +;************************************************************* +;* +;* Ipp64u cpMontRedAdc_BNU(Ipp64u* pR; +;* Ipp64u* pProduct, +;* const Ipp64u* pModulus, int mSize, +;* Ipp64u m) +;************************************************************* +align IPP_ALIGN_FACTOR +IPPASM cpMontRedAdc_BNU,PUBLIC +%assign LOCAL_FRAME (0) + USES_GPR rbx,rbp,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 5 +;pR (rdi) address of the reduction +;pProduct (rsi) address of the temporary product +;pModulus (rdx) address of the modulus +;mSize (rcx) size of the modulus +;m0 (r8) montgomery helper (m') + + mov r15, rdi ; store reduction address + + ; reload parameters for future convinience: + mov rdi, rsi ; rdi = temporary product buffer + mov rsi, rdx ; rsi = modulus + movsxd rdx, ecx ; rdx = length of modulus + + cmp rdx, 16 + ja .test_8N_case ; length of modulus >16 + + cmp rdx, 4 + ja .above4 ; length of modulus 4,..,16 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; short modulus (1,..,4) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + cmp rdx, 3 + ja .red_4 + jz .red_3 + jp .red_2 + ; red_1 + +.red_1: + mov r9, qword [rdi+sizeof(qword)*0] + MRED_FIX 1, r15, rdi, rsi, r8, rbp,rbx, r9 + jmp .quit + +.red_2: + mov r9, qword [rdi+sizeof(qword)*0] + mov r10, qword [rdi+sizeof(qword)*1] + MRED_FIX 2, r15, rdi, rsi, r8, rbp,rbx, r9,r10 + jmp .quit + +.red_3: + mov r9, qword [rdi+sizeof(qword)*0] + mov r10, qword [rdi+sizeof(qword)*1] + mov r11, qword [rdi+sizeof(qword)*2] + MRED_FIX 3, r15, rdi, rsi, r8, rbp,rbx, r9,r10,r11 + jmp .quit + +.red_4: + mov r9, qword [rdi+sizeof(qword)*0] + mov r10, qword [rdi+sizeof(qword)*1] + mov r11, qword [rdi+sizeof(qword)*2] + mov r12, qword [rdi+sizeof(qword)*3] + MRED_FIX 4, r15, rdi, rsi, r8, rbp,rbx, r9,r10,r11,r12 + jmp .quit + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; short modulus (5,..,16) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +.above4: + mov rbp, rdx + sub rbp, 4 + GET_EP rax, mred_short, rbp ; mred procedure + + call rax + jmp .quit + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; 8N case squarer +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +.test_8N_case: + test rdx, 7 + jnz .general_case + + CALL_FUNC mred_8N + jmp .quit + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; general case modulus +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +.general_case: + CALL_FUNC mred_N + +.quit: + REST_XMM + REST_GPR + ret +ENDFUNC cpMontRedAdc_BNU + +%endif + +%endif ;; _ADCOX_NI_ENABLING_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumulschoolsrvl9pp.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumulschoolsrvl9pp.asm new file mode 100644 index 0000000..00f47c9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnumulschoolsrvl9pp.asm @@ -0,0 +1,307 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Big Number Multiplicative Operations +; +; Content: +; cpMulAdx_BNU_school() +; cpSqrAdx_BNU_school() +; cpMontRedAdx_BNU() +; +; Implementation is using mulx and adcx/adox instruvtions +; +; + +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_ADCOX_NI_ENABLING_ == _FEATURE_ON_) || (_ADCOX_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_L9) + +%assign _xEMULATION_ 1 +%assign _ADCX_ADOX_ 1 + +segment .text align=IPP_ALIGN_FACTOR + +%include "pcpbnumulpp.inc" +%include "pcpbnusqrpp.inc" +%include "pcpmredpp.inc" + +;************************************************************* +;* Ipp64u cpMulAdx_BNU_school(Ipp64u* pR; +;* const Ipp64u* pA, int aSize, +;* const Ipp64u* pB, int bSize) +;************************************************************* +align IPP_ALIGN_FACTOR +IPPASM cpMulAdx_BNU_school,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbx,rbp,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 5 + +; rdi = pR +; rsi = pA +; edx = nsA +; rcx = pB +; r8d = nsB + + movsxd rdx, edx ; expand length + movsxd rbx, r8d + + xor r8, r8 ; clear scratch + xor r9, r9 + xor r10, r10 + xor r11, r11 + xor r12, r12 + xor r13, r13 + xor r14, r14 + xor r15, r15 + + cmp rdx, rbx + jl .swap_operans ; nsA < nsB + jg .test_8N_case ; test %if nsA=8*N and nsB=8*M + + cmp rdx, 16 + jg .test_8N_case + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; short nsA==nsB (1,..,16) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + cmp rdx, 4 + jg .more_then_4 + + cmp edx, 3 + ja .mul_4_4 + jz .mul_3_3 + jp .mul_2_2 + ; mul_1_1 + +.mul_1_1: + MUL_NxN 1, rdi, rsi, rcx, rbx, rbp, r8 + jmp .quit +.mul_2_2: + MUL_NxN 2, rdi, rsi, rcx, rbx, rbp, r8, r9 + jmp .quit +.mul_3_3: + MUL_NxN 3, rdi, rsi, rcx, rbx, rbp, r8, r9, r10 + jmp .quit +.mul_4_4: + MUL_NxN 4, rdi, rsi, rcx, rbx, rbp, r8, r9, r10, r11 + jmp .quit + +.more_then_4: + GET_EP rax, mul_lxl_basic, rdx, rbp + call rax + jmp .quit + +.swap_operans: + SWAP rsi, rcx ; swap operands + SWAP rdx, rbx + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; 8*N x 8*M case multiplier +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +.test_8N_case: + mov rax, rdx + or rax, rbx + and rax, 7 + jnz .general_mul + + CALL_FUNC mul_8Nx8M_adcox + jmp .quit + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; general case multiplier +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +.general_mul: + CALL_FUNC mul_NxM_adcox + jmp .quit + +.quit: + REST_XMM + REST_GPR + ret +ENDFUNC cpMulAdx_BNU_school + +;************************************************************* +;* +;* Ipp64u cpSqrAdx_BNU_school(Ipp64u* pR; +;* const Ipp64u* pA, int aSize) +;* +;************************************************************* +align IPP_ALIGN_FACTOR +IPPASM cpSqrAdx_BNU_school,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbx,rbp,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 3 + + movsxd rdx, edx ; expand length + + xor r8, r8 ; clear scratch + xor r9, r9 + xor r10, r10 + xor r11, r11 + xor r12, r12 + xor r13, r13 + xor r14, r14 + xor r15, r15 + + cmp rdx, 16 + jg .test_8N_case + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; short nsA (1,..,16) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + GET_EP rax, sqr_l_basic, rdx, rbp + call rax + jmp .quit + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; 8N case squarer +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +.test_8N_case: + test rdx, 7 + jnz .general_sqr + + CALL_FUNC sqr_8N_adcox + jmp .quit + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; general case squarer +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +.general_sqr: + CALL_FUNC sqr_N_adcox + +.quit: + REST_XMM + REST_GPR + ret +ENDFUNC cpSqrAdx_BNU_school + +;************************************************************* +;* +;* Ipp64u cpMontRedAdx_BNU(Ipp64u* pR; +;* Ipp64u* pProduct, +;* const Ipp64u* pModulus, int mSize, +;* Ipp64u m) +;************************************************************* +align IPP_ALIGN_FACTOR +IPPASM cpMontRedAdx_BNU,PUBLIC +%assign LOCAL_FRAME (0) + USES_GPR rbx,rbp,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 5 +;pR (rdi) address of the reduction +;pProduct (rsi) address of the temporary product +;pModulus (rdx) address of the modulus +;mSize (rcx) size of the modulus +;m0 (r8) montgomery helper (m') + + mov r15, rdi ; store reduction address + + ; reload parameters for future convinience: + mov rdi, rsi ; rdi = temporary product buffer + mov rsi, rdx ; rsi = modulus + movsxd rdx, ecx ; rdx = length of modulus + + cmp rdx, 16 + ja .test_8N_case ; length of modulus >16 + + cmp rdx, 4 + ja .above4 ; length of modulus 4,..,16 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; short modulus (1,..,4) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + cmp rdx, 3 + ja .red_4 + jz .red_3 + jp .red_2 + ; red_1 + +.red_1: + mov r9, qword [rdi+sizeof(qword)*0] + MRED_FIX 1, r15, rdi, rsi, r8, rbp,rbx, r9 + jmp .quit + +.red_2: + mov r9, qword [rdi+sizeof(qword)*0] + mov r10, qword [rdi+sizeof(qword)*1] + MRED_FIX 2, r15, rdi, rsi, r8, rbp,rbx, r9,r10 + jmp .quit + +.red_3: + mov r9, qword [rdi+sizeof(qword)*0] + mov r10, qword [rdi+sizeof(qword)*1] + mov r11, qword [rdi+sizeof(qword)*2] + MRED_FIX 3, r15, rdi, rsi, r8, rbp,rbx, r9,r10,r11 + jmp .quit + +.red_4: + mov r9, qword [rdi+sizeof(qword)*0] + mov r10, qword [rdi+sizeof(qword)*1] + mov r11, qword [rdi+sizeof(qword)*2] + mov r12, qword [rdi+sizeof(qword)*3] + MRED_FIX 4, r15, rdi, rsi, r8, rbp,rbx, r9,r10,r11,r12 + jmp .quit + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; short modulus (5,..,16) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +.above4: + mov rbp, rdx + sub rbp, 4 + GET_EP rax, mred_short, rbp ; mred procedure + + call rax + jmp .quit + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; 8N case squarer +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +.test_8N_case: + test rdx, 7 + jnz .general_case + + CALL_FUNC mred_8N_adcox + jmp .quit + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; general case modulus +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +.general_case: + CALL_FUNC mred_N_adcox + +.quit: + REST_XMM + REST_GPR + ret +ENDFUNC cpMontRedAdx_BNU + +%endif + +%endif ;; _ADCOX_NI_ENABLING_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnusqr.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnusqr.inc new file mode 100644 index 0000000..45f4a9e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnusqr.inc @@ -0,0 +1,474 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Low level Big Number squaring Support +; +; + +%ifndef _PCPBNUSQR_INC_ +%assign _PCPBNUSQR_INC_ 1 + +%include "pcpmulx.inc" +%include "pcpbnusqr_basic.inc" + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; (8*n) squarer +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_8N,PRIVATE + push rdi ; save diagonal loop parameters + push rsi + push rdx + + push rdi ; save initial triangle product parameters + push rsi + push rdx +; +; init upper triangle product +; + push rdx + call sqr8_triangle + pop rdx + + mov qword [rdi+sizeof(qword)*7], r15 + xor r15, r15 + add rdi, sizeof(qword)*8 + + sub rdx, 8 + + mov rcx, rsi + add rsi, sizeof(qword)*8 +.initLoop: + push rdx + call mla_8x8 + pop rdx + add rsi, sizeof(qword)*8 + add rdi, sizeof(qword)*8 + sub rdx, 8 + jnz .initLoop + + mov qword [rdi+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*2],r10 + mov qword [rdi+sizeof(qword)*3],r11 + mov qword [rdi+sizeof(qword)*4],r12 + mov qword [rdi+sizeof(qword)*5],r13 + mov qword [rdi+sizeof(qword)*6],r14 + mov qword [rdi+sizeof(qword)*7],r15 + jmp .update_Triangle + +; +; update upper triangle product +; +.outerLoop: + push rdi ; update triangle product parameters + push rsi + push rdx + + xor rax, rax ; c-flag + push rax + + mov r8, qword [rdi+sizeof(qword)*0] + mov r9, qword [rdi+sizeof(qword)*1] + mov r10, qword [rdi+sizeof(qword)*2] + mov r11, qword [rdi+sizeof(qword)*3] + mov r12, qword [rdi+sizeof(qword)*4] + mov r13, qword [rdi+sizeof(qword)*5] + mov r14, qword [rdi+sizeof(qword)*6] + mov r15, qword [rdi+sizeof(qword)*7] + +.innerLoop_entry: + push rdx + call sqr8_triangle + pop rdx + + mov qword [rdi+sizeof(qword)*7], r15 + xor r15, r15 + add rdi, sizeof(qword)*8 + + sub rdx, 8 + jz .skipInnerLoop + + mov rcx, rsi + add rsi, sizeof(qword)*8 +.innerLoop: + pop rax ; restore c-flag + neg rax + op_reg_mem adc, r8, qword [rdi+sizeof(qword)*0], rax + op_reg_mem adc, r9, qword [rdi+sizeof(qword)*1], rax + op_reg_mem adc, r10, qword [rdi+sizeof(qword)*2], rax + op_reg_mem adc, r11, qword [rdi+sizeof(qword)*3], rax + op_reg_mem adc, r12, qword [rdi+sizeof(qword)*4], rax + op_reg_mem adc, r13, qword [rdi+sizeof(qword)*5], rax + op_reg_mem adc, r14, qword [rdi+sizeof(qword)*6], rax + op_reg_mem adc, r15, qword [rdi+sizeof(qword)*7], rax + sbb rax, rax ; save c-flag + push rax + + push rdx + call mla_8x8 + pop rdx + add rsi, sizeof(qword)*8 + add rdi, sizeof(qword)*8 + sub rdx, 8 + jnz .innerLoop + +.skipInnerLoop: + pop rax ; restore c-flag + neg rax + adc r8, 0 + mov qword [rdi+sizeof(qword)*0], r8 + adc r9, 0 + mov qword [rdi+sizeof(qword)*1], r9 + adc r10,0 + mov qword [rdi+sizeof(qword)*2],r10 + adc r11,0 + mov qword [rdi+sizeof(qword)*3],r11 + adc r12,0 + mov qword [rdi+sizeof(qword)*4],r12 + adc r13,0 + mov qword [rdi+sizeof(qword)*5],r13 + adc r14,0 + mov qword [rdi+sizeof(qword)*6],r14 + adc r15,0 + mov qword [rdi+sizeof(qword)*7],r15 + +.update_Triangle: + pop rdx + pop rsi + pop rdi + add rsi, sizeof(qword)*8 + add rdi, sizeof(qword)*(8*2) + sub rdx, 8 + jnz .outerLoop + +; +; add diagonal terms +; + pop rcx + pop rsi + pop rdi + xor rbx, rbx +.update_loop: + call add_diag_4 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*4 + sub rcx, 4 + jnz .update_loop + ret +ENDFUNC sqr_8N + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; general case N>16 squarer +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_N,PRIVATE + push rdi ; save diagonal loop parameters + push rsi + push rdx + + push rdi ; save initial triangle product parameters + push rsi + push rdx + + mov rbp, rdx + and rbp, 7 + GET_EP rax, mla_8xl_tail, rbp ; get tail procedure + push rax + +; +; init upper triangle product +; + sub rdx, 8 + + push rdx + call sqr8_triangle + pop rdx + + mov qword [rdi+sizeof(qword)*7], r15 + add rdi, sizeof(qword)*8 + xor r15, r15 + + mov rcx, rsi + add rsi, sizeof(qword)*8 + sub rdx, 8 + +.initLoop: + push rdx + call mla_8x8 + pop rdx + add rsi, sizeof(qword)*8 + add rdi, sizeof(qword)*8 + sub rdx, 8 + jnc .initLoop + + add rdx, 8 +; +; tail +; + SWAP rsi, rcx + mov rax, [rsp] ; procedure + push rdx + call rax + pop rdx + lea rdi, [rdi+rdx*sizeof(qword)] + + mov qword [rdi+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*2],r10 + mov qword [rdi+sizeof(qword)*3],r11 + mov qword [rdi+sizeof(qword)*4],r12 + mov qword [rdi+sizeof(qword)*5],r13 + mov qword [rdi+sizeof(qword)*6],r14 + mov qword [rdi+sizeof(qword)*7],r15 + jmp .update_Triangle + +; +; update upper triangle product +; +.outerLoop: + push rdi ; update triangle product parameters + push rsi + push rdx + push rax ; tail procedure + + xor rax, rax ; c-flag + push rax + + mov r8, qword [rdi+sizeof(qword)*0] + mov r9, qword [rdi+sizeof(qword)*1] + mov r10, qword [rdi+sizeof(qword)*2] + mov r11, qword [rdi+sizeof(qword)*3] + mov r12, qword [rdi+sizeof(qword)*4] + mov r13, qword [rdi+sizeof(qword)*5] + mov r14, qword [rdi+sizeof(qword)*6] + mov r15, qword [rdi+sizeof(qword)*7] + + sub rdx, 8 + + push rdx + call sqr8_triangle + pop rdx + + mov qword [rdi+sizeof(qword)*7], r15 + add rdi, sizeof(qword)*8 + xor r15, r15 + + mov rcx, rsi + add rsi, sizeof(qword)*8 + sub rdx, 8 + +.innerLoop: + pop rax ; restore c-flag + neg rax + op_reg_mem adc, r8, qword [rdi+sizeof(qword)*0], rax + op_reg_mem adc, r9, qword [rdi+sizeof(qword)*1], rax + op_reg_mem adc, r10, qword [rdi+sizeof(qword)*2], rax + op_reg_mem adc, r11, qword [rdi+sizeof(qword)*3], rax + op_reg_mem adc, r12, qword [rdi+sizeof(qword)*4], rax + op_reg_mem adc, r13, qword [rdi+sizeof(qword)*5], rax + op_reg_mem adc, r14, qword [rdi+sizeof(qword)*6], rax + op_reg_mem adc, r15, qword [rdi+sizeof(qword)*7], rax + sbb rax, rax ; save c-flag + push rax + + push rdx + call mla_8x8 + pop rdx + + add rsi, sizeof(qword)*8 + add rdi, sizeof(qword)*8 + sub rdx, 8 + jnc .innerLoop + + add rdx, 8 +; +; tail +; + ; clear in advance + pxor xmm0, xmm0 + movdqu xmmword [rdi+rdx*sizeof(qword)], xmm0 + movdqu xmmword [rdi+rdx*sizeof(qword)+sizeof(qword)*2], xmm0 + movdqu xmmword [rdi+rdx*sizeof(qword)+sizeof(qword)*4], xmm0 + movdqu xmmword [rdi+rdx*sizeof(qword)+sizeof(qword)*6], xmm0 + + ; updates registers before mla operation + pop rax ; restore c-flag + neg rax + op_reg_mem adc, r8, qword [rdi+sizeof(qword)*0], rax + op_reg_mem adc, r9, qword [rdi+sizeof(qword)*1], rax + op_reg_mem adc, r10, qword [rdi+sizeof(qword)*2], rax + op_reg_mem adc, r11, qword [rdi+sizeof(qword)*3], rax + op_reg_mem adc, r12, qword [rdi+sizeof(qword)*4], rax + op_reg_mem adc, r13, qword [rdi+sizeof(qword)*5], rax + op_reg_mem adc, r14, qword [rdi+sizeof(qword)*6], rax + op_reg_mem adc, r15, qword [rdi+sizeof(qword)*7], rax + + ; store carry for future + sbb rax, rax + neg rax + mov qword [rdi+sizeof(qword)*8], rax + + ; mla_8xn operation + SWAP rsi, rcx + mov rax, [rsp] ; procedure + push rdx + call rax + pop rdx + lea rdi, [rdi+rdx*sizeof(qword)] + + ; updates registers before store + xor rax, rax + mov rax, qword [rdi+sizeof(qword)*0] + add r8, rax + mov qword [rdi+sizeof(qword)*0], r8 + mov rax, qword [rdi+sizeof(qword)*1] + adc r9, rax + mov qword [rdi+sizeof(qword)*1], r9 + mov rax, qword [rdi+sizeof(qword)*2] + adc r10, rax + mov qword [rdi+sizeof(qword)*2], r10 + mov rax, qword [rdi+sizeof(qword)*3] + adc r11, rax + mov qword [rdi+sizeof(qword)*3], r11 + mov rax, qword [rdi+sizeof(qword)*4] + adc r12, rax + mov qword [rdi+sizeof(qword)*4], r12 + mov rax, qword [rdi+sizeof(qword)*5] + adc r13, rax + mov qword [rdi+sizeof(qword)*5], r13 + mov rax, qword [rdi+sizeof(qword)*6] + adc r14, rax + mov qword [rdi+sizeof(qword)*6], r14 + mov rax, qword [rdi+sizeof(qword)*7] + adc r15, rax + mov qword [rdi+sizeof(qword)*7], r15 + +.update_Triangle: + pop rax ; tail procedure + pop rdx + pop rsi + pop rdi + add rsi, sizeof(qword)*8 + add rdi, sizeof(qword)*(8*2) + sub rdx, 8 + cmp rdx, 16 + jg .outerLoop + +; +; tail +; + mov rbp, rdx + sub rbp, 8 + GET_EP rax, sqrN_triangle, rbp ; get triangle proc + + sub rsp, sizeof(qword)*32 + push rdi + push rdx + + mov r8, qword [rdi+sizeof(qword)*0] + mov r9, qword [rdi+sizeof(qword)*1] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + lea rdi, [rsp+sizeof(qword)*2] + call rax + + mov rsi, rdi + pop rdx + pop rdi + + ; copy 8 terms + movdqu xmm0, xmmword [rsi] + movdqu xmm1, xmmword [rsi+sizeof(qword)*2] + movdqu xmm2, xmmword [rsi+sizeof(qword)*4] + movdqu xmm3, xmmword [rsi+sizeof(qword)*6] + add rsi, sizeof(qword)*8 + movdqu xmmword [rdi], xmm0 + movdqu xmmword [rdi+sizeof(qword)*2], xmm1 + movdqu xmmword [rdi+sizeof(qword)*4], xmm2 + movdqu xmmword [rdi+sizeof(qword)*6], xmm3 + add rdi, sizeof(qword)*8 + + ; update rdx-8 terms + lea rax, [rdx-8] + xor rbx, rbx +.update1: + mov r8, qword [rsi] + mov r9, qword [rdi] + add rsi, sizeof(qword) + neg rbx + adc r8, r9 + sbb rbx, rbx + mov qword [rdi], r8 + add rdi, sizeof(qword) + sub rax, 1 + jg .update1 + + ; update rdx terms +.update2: + mov r8, qword [rsi] + add rsi, sizeof(qword) + neg rbx + adc r8, 0 + sbb rbx, rbx + mov qword [rdi], r8 + add rdi, sizeof(qword) + sub rdx, 1 + jg .update2 + + add rsp, sizeof(qword)*32 + +; +; add diagonal terms +; +.add_diagonals: + pop rcx + pop rsi + pop rdi + sub rcx, 4 + xor rbx, rbx +.add_diagonal_loop: + call add_diag_4 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*4 + sub rcx, 4 + jnc .add_diagonal_loop + + add rcx, 4 + jz .quit +.add_diagonal_rest: + ADD_DIAG 1, rdi, rsi + add rdi, sizeof(qword)*2 + add rsi, sizeof(qword) + sub rcx, 1 + jnz .add_diagonal_rest + +.quit: + ret +ENDFUNC sqr_N + + +%endif ;; _PCPBNUSQR_INC_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnusqr_basic.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnusqr_basic.inc new file mode 100644 index 0000000..1474f72 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnusqr_basic.inc @@ -0,0 +1,2003 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Low level Big Number squaring Support +; +; + +%ifndef _PCPBNSQR_BASIC_ADCX_INC_ +%assign _PCPBNSQR_BASIC_ADCX_INC_ 1 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; Fixed-size (1-8 qwords) square operations +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; acc:a1 = src * mem + a1 +%macro MULADD1x 4.nolist + %xdefine %%acc %1 + %xdefine %%a1 %2 + %xdefine %%src %3 + %xdefine %%mem %4 + +%ifnidni %%src,rdx + mov rdx, %%src +%endif +gsmulx %%acc, rax, %%mem + add %%a1, rax + adc %%acc, 0 +%endmacro + +; acc:a1 = src * mem + a1 + acc +%macro MULADDx 5.nolist + %xdefine %%acc %1 + %xdefine %%a1 %2 + %xdefine %%src %3 + %xdefine %%mem %4 + %xdefine %%H %5 + +%ifnidni %%src,rdx + mov rdx, %%src +%endif + +gsmulx %%H,rax, %%mem + add %%a1, rax + adc %%H, 0 + add %%a1, %%acc + adc %%H, 0 + mov %%acc, %%H +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; assignment +;; +%xdefine pDst rdi +%xdefine pSrc rsi +%xdefine pA rsi + +%xdefine A rcx +%xdefine x0 r8 +%xdefine x1 r9 +%xdefine x2 r10 +%xdefine x3 r11 +%xdefine x4 r12 +%xdefine x5 r13 +%xdefine x6 r14 +%xdefine x7 r15 +%xdefine x8 rbx +%xdefine t0 rbp + + +;; +;; 1*qword squarer +;; +%macro SQR_64 2.nolist + %xdefine %%pDst %1 + %xdefine %%pA %2 + + mov rdx, qword [%%pA] +gsmulx rdx, rax, rdx + mov qword [%%pDst], rax + mov qword [%%pDst+sizeof(qword)], rdx +%endmacro + +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_1,PRIVATE + SQR_64 pDst, pA + ret +ENDFUNC sqr_1 + + + +;; +;; 2*qword squarer +;; +%macro SQR_128 13.nolist + %xdefine %%pDst %1 + %xdefine %%pA %2 + %xdefine %%x7 %3 + %xdefine %%x6 %4 + %xdefine %%x5 %5 + %xdefine %%x4 %6 + %xdefine %%x3 %7 + %xdefine %%x2 %8 + %xdefine %%x1 %9 + %xdefine %%x0 %10 + %xdefine %%A %11 + %xdefine %%x8 %12 + %xdefine %%t0 %13 + + mov rdx, qword [%%pA] +gsmulx %%x1,%%x0,qword [rsi+sizeof(qword)*1] ; a[0]*a[1] + +gsmulx %%x3,%%x2,rdx ; a[0]^2 + mov rdx, qword [%%pA+sizeof(qword)] ; a[1] +gsmulx rdx,rax,rdx ; a[1]^2 + + xor %%A, %%A + add %%x0, %%x0 + adc %%x1, %%x1 + adc %%A, 0 + + mov qword [%%pDst+sizeof(qword)*0], %%x2 + add %%x3, %%x0 + mov qword [%%pDst+sizeof(qword)*1], %%x3 + adc rax, %%x1 + mov qword [%%pDst+sizeof(qword)*2], rax + adc rdx, %%A + mov qword [%%pDst+sizeof(qword)*3], rdx +%endmacro + +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_2,PRIVATE + SQR_128 pDst, pA, x7, x6, x5, x4, x3, x2, x1, x0, A, x8, t0 + ret +ENDFUNC sqr_2 + + + +;; +;; 3*qword squarer +;; +%macro SQR_192 13.nolist + %xdefine %%pDst %1 + %xdefine %%pA %2 + %xdefine %%x7 %3 + %xdefine %%x6 %4 + %xdefine %%x5 %5 + %xdefine %%x4 %6 + %xdefine %%x3 %7 + %xdefine %%x2 %8 + %xdefine %%x1 %9 + %xdefine %%x0 %10 + %xdefine %%A %11 + %xdefine %%x8 %12 + %xdefine %%t0 %13 + + mov rdx, [%%pA] +gsmulx %%x1,%%x0, [%%pA+sizeof(qword)*1] + MULADD1x %%x2, %%x1, rdx, [%%pA+sizeof(qword)*2] + + mov rdx, qword [%%pA+sizeof(qword)*1] +gsmulx %%x3, rax, qword [%%pA+sizeof(qword)*2] + add %%x2, rax + adc %%x3, 0 + + mov rdx, qword [%%pA+ sizeof(qword)*0] + + xor %%A, %%A + add %%x0, %%x0 + adc %%x1, %%x1 + adc %%x2, %%x2 + adc %%x3, %%x3 + adc %%A, %%A + + ;; add sqr(a[0]),...,sqr(a[2]) +gsmulx rax, rdx, rdx ; a[0]^2 + mov [%%pDst+sizeof(qword)*0], rdx + mov rdx, [%%pA+sizeof(qword)*1] + add %%x0, rax + mov [%%pDst+sizeof(qword)*1], %%x0 + +gsmulx rax, rdx, rdx ; a[1]^2 + adc %%x1, rdx + mov [%%pDst+sizeof(qword)*2], %%x1 + mov rdx, [%%pA+sizeof(qword)*2] + adc %%x2, rax + mov [%%pDst+sizeof(qword)*3], %%x2 + +gsmulx rax, rdx, rdx ; a[2]^2 + adc %%x3, rdx + mov [%%pDst+sizeof(qword)*4], %%x3 + adc %%A, rax + mov [%%pDst+sizeof(qword)*5], %%A +%endmacro + +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_3,PRIVATE + SQR_192 pDst, pA, x7, x6, x5, x4, x3, x2, x1, x0, A, x8, t0 + ret +ENDFUNC sqr_3 + + + +;; +;; 4*qword squarer +;; +%macro SQR_256 13.nolist + %xdefine %%pDst %1 + %xdefine %%pA %2 + %xdefine %%x7 %3 + %xdefine %%x6 %4 + %xdefine %%x5 %5 + %xdefine %%x4 %6 + %xdefine %%x3 %7 + %xdefine %%x2 %8 + %xdefine %%x1 %9 + %xdefine %%x0 %10 + %xdefine %%A %11 + %xdefine %%x8 %12 + %xdefine %%t0 %13 + + mov rdx, [%%pA] +gsmulx %%x1,%%x0, [%%pA+sizeof(qword)*1] + MULADD1x %%x2, %%x1, rdx, [%%pA+sizeof(qword)*2] + MULADD1x %%x3, %%x2, rdx, [%%pA+sizeof(qword)*3] + + mov rdx, qword [%%pA+sizeof(qword)*1] +gsmulx %%A, rax, qword [%%pA+sizeof(qword)*2] + xor %%x4, %%x4 + add %%x2, rax + adc %%x3, %%A + adc %%x4, 0 +gsmulx %%A, rax, qword [%%pA+sizeof(qword)*3] + add %%x3, rax + adc %%x4, %%A + + mov rdx, qword [%%pA+sizeof(qword)*2] +gsmulx %%x5, rax, qword [%%pA+sizeof(qword)*3] + add %%x4, rax + adc %%x5, 0 + + mov rdx, [%%pA+sizeof(qword)*0] + + ;; --- double x0...x5 + xor %%A, %%A + add %%x0, %%x0 + adc %%x1, %%x1 + adc %%x2, %%x2 + adc %%x3, %%x3 + adc %%x4, %%x4 + adc %%x5, %%x5 + adc %%A, 0 + + ;; add sqr(a[0]),...,sqr(a[3]) +gsmulx rax, rdx, rdx ; a[0]^2 + mov [%%pDst+sizeof(qword)*0], rdx + mov rdx, [%%pA+sizeof(qword)*1] + add %%x0, rax + mov [%%pDst+sizeof(qword)*1], %%x0 + +gsmulx rax, rdx, rdx ; a[1]^2 + adc %%x1, rdx + mov [%%pDst+sizeof(qword)*2], %%x1 + mov rdx, [%%pA+sizeof(qword)*2] + adc %%x2, rax + mov [%%pDst+sizeof(qword)*3], %%x2 + +gsmulx rax, rdx, rdx ; a[2]^2 + adc %%x3, rdx + mov [%%pDst+sizeof(qword)*4], %%x3 + mov rdx, [%%pA+sizeof(qword)*3] + adc %%x4, rax + mov [%%pDst+sizeof(qword)*5], %%x4 + +gsmulx rax, rdx, rdx ; a[3]^2 + adc %%x5, rdx + mov [%%pDst+sizeof(qword)*6], %%x5 + adc %%A, rax + mov [%%pDst+sizeof(qword)*7], %%A +%endmacro + +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_4,PRIVATE + SQR_256 pDst, pA, x7, x6, x5, x4, x3, x2, x1, x0, A, x8, t0 + ret +ENDFUNC sqr_4 + + + +;; +;; 5*qword squarer +;; +%macro SQR_320 13.nolist + %xdefine %%pDst %1 + %xdefine %%pA %2 + %xdefine %%x7 %3 + %xdefine %%x6 %4 + %xdefine %%x5 %5 + %xdefine %%x4 %6 + %xdefine %%x3 %7 + %xdefine %%x2 %8 + %xdefine %%x1 %9 + %xdefine %%x0 %10 + %xdefine %%A %11 + %xdefine %%x8 %12 + %xdefine %%t0 %13 + +mov rdx, [%%pA] +gsmulx %%x1,%%x0, [%%pA+sizeof(qword)*1] + MULADD1x %%x2, %%x1, rdx, [%%pA+sizeof(qword)*2] + MULADD1x %%x3, %%x2, rdx, [%%pA+sizeof(qword)*3] + MULADD1x %%x4, %%x3, rdx, [%%pA+sizeof(qword)*4] + + mov rdx, [%%pA + sizeof(qword)*1] +gsmulx %%t0, rax, [%%pA+sizeof(qword)*2] + add %%x2, rax + adc %%t0, 0 + MULADDx %%t0, %%x3, rdx, [%%pA+sizeof(qword)*3], %%A + MULADDx %%t0, %%x4, rdx, [%%pA+sizeof(qword)*4], %%A + mov %%x5, %%t0 + + mov rdx, qword [%%pA+sizeof(qword)*2] +gsmulx %%A, rax, qword [%%pA+sizeof(qword)*3] + xor %%x6, %%x6 + add %%x4, rax + adc %%x5, %%A + adc %%x6, 0 +gsmulx %%A, rax, qword [%%pA+sizeof(qword)*4] + add %%x5, rax + adc %%x6, %%A + + mov rdx, qword [%%pA+sizeof(qword)*3] +gsmulx %%x7, rax, qword [%%pA+sizeof(qword)*4] + add %%x6, rax + adc %%x7, 0 + + mov rdx, [%%pA+sizeof(qword)*0] + + ;; --- double x0...x5 + xor %%A, %%A + add %%x0, %%x0 + adc %%x1, %%x1 + adc %%x2, %%x2 + adc %%x3, %%x3 + adc %%x4, %%x4 + adc %%x5, %%x5 + adc %%x6, %%x6 + adc %%x7, %%x7 + adc %%A, 0 + + ;; add sqr(a[0]),...,sqr(a[4]) +gsmulx rax, rdx, rdx ; a[0]^2 + mov [%%pDst+sizeof(qword)*0], rdx + mov rdx, [%%pA+sizeof(qword)*1] + add %%x0, rax + mov [%%pDst+sizeof(qword)*1], %%x0 + +gsmulx rax, rdx, rdx ; a[1]^2 + adc %%x1, rdx + mov [%%pDst+sizeof(qword)*2], %%x1 + mov rdx, [%%pA+sizeof(qword)*2] + adc %%x2, rax + mov [%%pDst+sizeof(qword)*3], %%x2 + +gsmulx rax, rdx, rdx ; a[2]^2 + adc %%x3, rdx + mov [%%pDst+sizeof(qword)*4], %%x3 + mov rdx, [%%pA+sizeof(qword)*3] + adc %%x4, rax + mov [%%pDst+sizeof(qword)*5], %%x4 + +gsmulx rax, rdx, rdx ; a[3]^2 + adc %%x5, rdx + mov [%%pDst+sizeof(qword)*6], %%x5 + mov rdx, [%%pA+sizeof(qword)*4] + adc %%x6, rax + mov [%%pDst+sizeof(qword)*7], %%x6 + +gsmulx rax, rdx, rdx ; a[4]^2 + adc %%x7, rdx + mov [%%pDst+sizeof(qword)*8], %%x7 + mov rdx, [%%pA+sizeof(qword)*5] + adc %%A, rax + mov [%%pDst+sizeof(qword)*9], %%A +%endmacro + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_5,PRIVATE + SQR_320 pDst, pA, x7, x6, x5, x4, x3, x2, x1, x0, A, x8, t0 + ret +ENDFUNC sqr_5 + + + +;; +;; 6*qword squarer +;; +%macro SQR_384 13.nolist + %xdefine %%pDst %1 + %xdefine %%pA %2 + %xdefine %%x7 %3 + %xdefine %%x6 %4 + %xdefine %%x5 %5 + %xdefine %%x4 %6 + %xdefine %%x3 %7 + %xdefine %%x2 %8 + %xdefine %%x1 %9 + %xdefine %%x0 %10 + %xdefine %%A %11 + %xdefine %%x8 %12 + %xdefine %%t0 %13 + +mov rdx, [%%pA] +gsmulx %%x1,%%x0, [%%pA+sizeof(qword)*1] + MULADD1x %%x2, %%x1, rdx, [%%pA+sizeof(qword)*2] + MULADD1x %%x3, %%x2, rdx, [%%pA+sizeof(qword)*3] + MULADD1x %%x4, %%x3, rdx, [%%pA+sizeof(qword)*4] + MULADD1x %%x5, %%x4, rdx, [%%pA+sizeof(qword)*5] + + mov rdx, [%%pA + sizeof(qword)*1] +gsmulx %%t0, rax, [%%pA+sizeof(qword)*2] + add %%x2, rax + adc %%t0, 0 + MULADDx %%t0, %%x3, rdx, [%%pA+sizeof(qword)*3], %%A + MULADDx %%t0, %%x4, rdx, [%%pA+sizeof(qword)*4], %%A + MULADDx %%t0, %%x5, rdx, [%%pA+sizeof(qword)*5], %%A + mov %%x6, %%t0 + + mov rdx, [%%pA+sizeof(qword)*2] +gsmulx %%t0, rax, [%%pA+sizeof(qword)*3] + add %%x4, rax + adc %%t0, 0 + MULADDx %%t0, %%x5, rdx, [%%pA+sizeof(qword)*4], %%A + MULADDx %%t0, %%x6, rdx, [%%pA+sizeof(qword)*5], %%A + mov %%x7, %%t0 + + mov rdx, qword [%%pA+sizeof(qword)*3] +gsmulx %%A, rax, qword [%%pA+sizeof(qword)*4] + xor %%x8, %%x8 + add %%x6, rax + adc %%x7, %%A + adc %%x8, 0 +gsmulx %%A, rax, qword [%%pA+sizeof(qword)*5] + add %%x7, rax + adc %%x8, %%A + + mov rdx, qword [%%pA+sizeof(qword)*4] +gsmulx %%t0, rax, qword [%%pA+sizeof(qword)*5] + add %%x8, rax + adc %%t0, 0 + + mov rdx, [%%pA+sizeof(qword)*0] + + ;; --- double x0...x7,x8,t0 + xor %%A, %%A + add %%x0, %%x0 + adc %%x1, %%x1 + adc %%x2, %%x2 + adc %%x3, %%x3 + adc %%x4, %%x4 + adc %%x5, %%x5 + adc %%x6, %%x6 + adc %%x7, %%x7 + adc %%x8, %%x8 + adc %%t0, %%t0 + adc %%A, 0 + + ;; add sqr(a[0]),...,sqr(a[5]) +gsmulx rax, rdx, rdx ; a[0]^2 + mov [%%pDst+sizeof(qword)*0], rdx + mov rdx, [%%pA+sizeof(qword)*1] + add %%x0, rax + mov [%%pDst+sizeof(qword)*1], %%x0 + +gsmulx rax, rdx, rdx ; a[1]^2 + adc %%x1, rdx + mov [%%pDst+sizeof(qword)*2], %%x1 + mov rdx, [%%pA+sizeof(qword)*2] + adc %%x2, rax + mov [%%pDst+sizeof(qword)*3], %%x2 + +gsmulx rax, rdx, rdx ; a[2]^2 + adc %%x3, rdx + mov [%%pDst+sizeof(qword)*4], %%x3 + mov rdx, [%%pA+sizeof(qword)*3] + adc %%x4, rax + mov [%%pDst+sizeof(qword)*5], %%x4 + +gsmulx rax, rdx, rdx ; a[3]^2 + adc %%x5, rdx + mov [%%pDst+sizeof(qword)*6], %%x5 + mov rdx, [%%pA+sizeof(qword)*4] + adc %%x6, rax + mov [%%pDst+sizeof(qword)*7], %%x6 + +gsmulx rax, rdx, rdx ; a[4]^2 + adc %%x7, rdx + mov [%%pDst+sizeof(qword)*8], %%x7 + mov rdx, [%%pA+sizeof(qword)*5] + adc %%x8, rax + mov [%%pDst+sizeof(qword)*9], %%x8 + +gsmulx rax, rdx, rdx ; a[5]^2 + adc %%t0, rdx + mov [%%pDst+sizeof(qword)*10], %%t0 + adc %%A, rax + mov [%%pDst+sizeof(qword)*11], %%A +%endmacro + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_6,PRIVATE + SQR_384 pDst, pA, x7, x6, x5, x4, x3, x2, x1, x0, A, x8, t0 + ret +ENDFUNC sqr_6 + + + +;; +;; 7*qword squarer +;; +%macro SQR_448 13.nolist + %xdefine %%pDst %1 + %xdefine %%pA %2 + %xdefine %%x7 %3 + %xdefine %%x6 %4 + %xdefine %%x5 %5 + %xdefine %%x4 %6 + %xdefine %%x3 %7 + %xdefine %%x2 %8 + %xdefine %%x1 %9 + %xdefine %%x0 %10 + %xdefine %%A %11 + %xdefine %%x8 %12 + %xdefine %%t0 %13 + + ;; ------------------ + ;; first pass 01...06 + ;; ------------------ + mov rdx, [%%pA] +gsmulx %%x1,%%x0, [%%pA+sizeof(qword)*1] + MULADD1x %%x2, %%x1, rdx, [%%pA+sizeof(qword)*2] + MULADD1x %%x3, %%x2, rdx, [%%pA+sizeof(qword)*3] + MULADD1x %%x4, %%x3, rdx, [%%pA+sizeof(qword)*4] + MULADD1x %%x5, %%x4, rdx, [%%pA+sizeof(qword)*5] + MULADD1x %%x6, %%x5, rdx, [%%pA+sizeof(qword)*6] + + ;; ------------------ + ;; second pass 12...16 + ;; ------------------ + mov rdx, [%%pA + sizeof(qword)*1] +gsmulx %%t0, rax, [%%pA+sizeof(qword)*2] + add %%x2, rax + adc %%t0, 0 + + MULADDx %%t0, %%x3, rdx, [%%pA+sizeof(qword)*3], %%A + MULADDx %%t0, %%x4, rdx, [%%pA+sizeof(qword)*4], %%A + MULADDx %%t0, %%x5, rdx, [%%pA+sizeof(qword)*5], %%A + MULADDx %%t0, %%x6, rdx, [%%pA+sizeof(qword)*6], %%A + mov %%x7, %%t0 + + ;; ------------------ + ;; third pass 23...25 + ;; ------------------ + mov rdx, [%%pA+sizeof(qword)*2] +gsmulx %%t0, rax, [%%pA+sizeof(qword)*3] + add %%x4, rax + adc %%t0, 0 + + xor %%x8, %%x8 + MULADDx %%t0, %%x5, rdx, [%%pA+sizeof(qword)*4], %%A + MULADDx %%t0, %%x6, rdx, [%%pA+sizeof(qword)*5], %%A + add %%x7, %%t0 + adc %%x8, 0 + + ;; ------------------ + ;; fourth pass 34 + ;; ------------------ + mov rax, [%%pA+sizeof(qword)*3] + mul qword [%%pA+sizeof(qword)*4] + add %%x6, rax + adc rdx, 0 + add %%x7, rdx + adc %%x8, 0 + + mov rdx, [%%pA+sizeof(qword)*0] + + ;; --- double x0...x6 + xor %%A, %%A + add %%x0, %%x0 + adc %%x1, %%x1 + adc %%x2, %%x2 + adc %%x3, %%x3 + adc %%x4, %%x4 + adc %%x5, %%x5 + adc %%x6, %%x6 + adc %%A, 0 + + ;; add sqr(a[0]),...,sqr(a[3]) +gsmulx rax, rdx, rdx ; a[0]^2 + mov [%%pDst+sizeof(qword)*0], rdx + mov rdx, [%%pA+sizeof(qword)*1] + add %%x0, rax + mov [%%pDst+sizeof(qword)*1], %%x0 + +gsmulx rax, rdx, rdx ; a[1]^2 + adc %%x1, rdx + mov [%%pDst+sizeof(qword)*2], %%x1 + mov rdx, [%%pA+sizeof(qword)*2] + adc %%x2, rax + mov [%%pDst+sizeof(qword)*3], %%x2 + +gsmulx rax, rdx, rdx ; a[2]^2 + adc %%x3, rdx + mov [%%pDst+sizeof(qword)*4], %%x3 + mov rdx, [%%pA+sizeof(qword)*3] + adc %%x4, rax + mov [%%pDst+sizeof(qword)*5], %%x4 + +gsmulx rax, rdx, rdx ; a[3]^2 + adc %%x5, rdx + mov [%%pDst+sizeof(qword)*6], %%x5 + adc %%x6, rax + mov [%%pDst+sizeof(qword)*7], %%x6 + adc %%A, 0 + + ;; ------------------ + ;; third pass complete 26 + ;; ------------------ + mov rax, [%%pA+sizeof(qword)*2] + xor %%x0, %%x0 + + mul qword [%%pA+sizeof(qword)*6] + add %%x7, rax + adc rdx, 0 + add %%x8, rdx + adc %%x0, 0 + + ;; ------------------ + ;; forth pass complete 35...36 + ;; ------------------ + mov rdx, [%%pA+sizeof(qword)*3] +gsmulx %%x6, rax, [%%pA+sizeof(qword)*5] + add %%x7, rax + adc %%x8, %%x6 + adc %%x0, 0 +gsmulx %%x6, rax, [%%pA+sizeof(qword)*6] + add %%x8, rax + adc %%x0, %%x6 + + ;; ------------------ + ;; fifth pass 45...46 + ;; ------------------ + mov rdx, [%%pA+sizeof(qword)*4] +gsmulx %%x6, rax, [%%pA+sizeof(qword)*5] + xor %%x1, %%x1 + add %%x8, rax + adc %%x0, %%x6 + adc %%x1, 0 +gsmulx %%x6, rax, [%%pA+sizeof(qword)*6] + add %%x0, rax + adc %%x1, %%x6 + + ;; ------------------ + ;; six pass 56 + ;; ------------------ + mov rax, [%%pA+sizeof(qword)*5] + xor %%x2, %%x2 + mul qword [%%pA + sizeof(qword)*6] + add %%x1, rax + adc %%x2, rdx + + mov rdx, [%%pA+sizeof(qword)*4] + + ;; --- double x7, x8, x0, x1, x2 + xor %%x3, %%x3 + add %%x7, %%x7 + adc %%x8, %%x8 + adc %%x0, %%x0 + adc %%x1, %%x1 + adc %%x2, %%x2 + adc %%x3, 0 + +gsmulx rax, rdx, rdx ; a[4]^2 + add rdx, %%A + adc rax, 0 + add %%x7, rdx + mov rdx, [%%pA+sizeof(qword)*5] + mov [%%pDst+sizeof(qword)*8], %%x7 + adc %%x8, rax + mov [%%pDst+sizeof(qword)*9], %%x8 + +gsmulx rax, rdx, rdx ; a[5]^2 + adc %%x0, rdx + mov [%%pDst+sizeof(qword)*10], %%x0 + mov rdx, [%%pA+sizeof(qword)*6] + adc %%x1, rax + mov [%%pDst+sizeof(qword)*11], %%x1 + +gsmulx rax, rdx, rdx ; a[6]^2 + adc %%x2, rdx + mov [%%pDst+sizeof(qword)*12], %%x2 + adc %%x3, rax + mov [%%pDst+sizeof(qword)*13], %%x3 +%endmacro + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_7,PRIVATE + SQR_448 pDst, pA, x7, x6, x5, x4, x3, x2, x1, x0, A, x8, t0 + ret +ENDFUNC sqr_7 + + + +;; +;; 8*qword squarer +;; +%macro SQR_512 13.nolist + %xdefine %%pDst %1 + %xdefine %%pA %2 + %xdefine %%x7 %3 + %xdefine %%x6 %4 + %xdefine %%x5 %5 + %xdefine %%x4 %6 + %xdefine %%x3 %7 + %xdefine %%x2 %8 + %xdefine %%x1 %9 + %xdefine %%x0 %10 + %xdefine %%A %11 + %xdefine %%x8 %12 + %xdefine %%t0 %13 + +;; ---------------------------------- + ;; first pass a[0]*a[1],...,a[0]*a[7] + ;; ---------------------------------- + mov rdx, [%%pA] +gsmulx %%x1,%%x0, [%%pA+sizeof(qword)*1] + MULADD1x %%x2, %%x1, rdx, [%%pA+sizeof(qword)*2] + MULADD1x %%x3, %%x2, rdx, [%%pA+sizeof(qword)*3] + MULADD1x %%x4, %%x3, rdx, [%%pA+sizeof(qword)*4] + MULADD1x %%x5, %%x4, rdx, [%%pA+sizeof(qword)*5] + MULADD1x %%x6, %%x5, rdx, [%%pA+sizeof(qword)*6] + MULADD1x %%x7, %%x6, rdx, [%%pA+sizeof(qword)*7] + + ;; ----------------------------------- + ;; second pass a[1]*a[2],...,a[1]*a[6] + ;; ----------------------------------- + mov rdx, [%%pA + sizeof(qword)*1] +gsmulx %%t0, rax, [%%pA+sizeof(qword)*2] + add %%x2, rax + adc %%t0, 0 + xor %%x8, %%x8 + + MULADDx %%t0, %%x3, rdx, [%%pA+sizeof(qword)*3], %%A + MULADDx %%t0, %%x4, rdx, [%%pA+sizeof(qword)*4], %%A + MULADDx %%t0, %%x5, rdx, [%%pA+sizeof(qword)*5], %%A + MULADDx %%t0, %%x6, rdx, [%%pA+sizeof(qword)*6], %%A + add %%x7, %%t0 + adc %%x8, 0 + + ;; ---------------------------------- + ;; third pass a[2]*a[3],...,a[2]*a[5] + ;; ---------------------------------- + mov rdx, [%%pA+sizeof(qword)*2] +gsmulx %%t0, rax, [%%pA+sizeof(qword)*3] + add %%x4, rax + adc %%t0, 0 + + MULADDx %%t0, %%x5, rdx, [%%pA+sizeof(qword)*4], %%A + MULADDx %%t0, %%x6, rdx, [%%pA+sizeof(qword)*5], %%A + add %%x7, %%t0 + adc %%x8, 0 + + ;; --------------------- + ;; fourth pass a[3]*a[4] + ;; --------------------- + mov rax, [%%pA+sizeof(qword)*3] + mul qword [%%pA+sizeof(qword)*4] + add %%x6, rax + adc rdx, 0 + add %%x7, rdx + adc %%x8, 0 + + mov rdx, [%%pA+sizeof(qword)*0] + + ;; double x0,...,x6 + xor %%A, %%A + add %%x0, %%x0 + adc %%x1, %%x1 + adc %%x2, %%x2 + adc %%x3, %%x3 + adc %%x4, %%x4 + adc %%x5, %%x5 + adc %%x6, %%x6 + adc %%A, 0 + + ;; add sqr(a[0]),...,sqr(a[3]) +gsmulx rax, rdx, rdx ; a[0]^2 + mov [%%pDst+sizeof(qword)*0], rdx + mov rdx, [%%pA+sizeof(qword)*1] + add %%x0, rax + mov [%%pDst+sizeof(qword)*1], %%x0 + +gsmulx rax, rdx, rdx ; a[1]^2 + adc %%x1, rdx + mov [%%pDst+sizeof(qword)*2], %%x1 + mov rdx, [%%pA+sizeof(qword)*2] + adc %%x2, rax + mov [%%pDst+sizeof(qword)*3], %%x2 + +gsmulx rax, rdx, rdx ; a[2]^2 + adc %%x3, rdx + mov [%%pDst+sizeof(qword)*4], %%x3 + mov rdx, [%%pA+sizeof(qword)*3] + adc %%x4, rax + mov [%%pDst+sizeof(qword)*5], %%x4 + +gsmulx rax, rdx, rdx ; a[3]^2 + adc %%x5, rdx + mov [%%pDst+sizeof(qword)*6], %%x5 + adc %%x6, rax + mov [%%pDst+sizeof(qword)*7], %%x6 + adc %%A, 0 + + ;; ---------------------------- + ;; second pass (cont) a[1]*a[7] + ;; ---------------------------- + mov rax, [%%pA+sizeof(qword)*1] + xor %%x0, %%x0 + mul qword [%%pA+sizeof(qword)*7] + add %%x7, rax + adc rdx, 0 + add %%x8, rdx + adc %%x0, 0 + + ;; ----------------------------------------- + ;; third pass (cont) a[2]*a[6],...,a[2]*a[7] + ;; ----------------------------------------- + mov rdx, [%%pA+sizeof(qword)*2] +gsmulx %%x6, rax, [%%pA+sizeof(qword)*6] + add %%x7, rax + adc %%x8, %%x6 + adc %%x0, 0 +gsmulx %%x6, rax, [%%pA+sizeof(qword)*7] + xor %%x1, %%x1 + add %%x8, rax + adc %%x0, %%x6 + adc %%x1, 0 + + ;; ----------------------------------- + ;; fourth pass a[3]*a[5],...,a[3]*a[7] + ;; ----------------------------------- + mov rdx, [%%pA+sizeof(qword)*3] +gsmulx %%x6, rax, [%%pA+sizeof(qword)*5] + add %%x7, rax + adc %%x6, 0 + add %%x8, %%x6 + adc %%x0, 0 + adc %%x1, 0 +gsmulx %%x6, rax, [%%pA+sizeof(qword)*6] + add %%x8, rax + adc %%x6, 0 + add %%x0, %%x6 + adc %%x1, 0 +gsmulx %%x6, rax, [%%pA+sizeof(qword)*7] + add %%x0, rax + adc %%x6, 0 + add %%x1, %%x6 + ;; carry out should be 0 + + ;; ----------------------------------- + ;; fifth pass a[4]*a[5],...,a[4]*a[7] + ;; ----------------------------------- + mov rdx, [%%pA+sizeof(qword)*4] + +gsmulx %%x2, rax, [%%pA+sizeof(qword)*5] + add %%x8, rax + adc %%x2, 0 + + MULADDx %%x2, %%x0, rdx, [%%pA+sizeof(qword)*6], %%x6 + MULADDx %%x2, %%x1, rdx, [%%pA+sizeof(qword)*7], %%x6 + + ;; ----------------------------------------------------------- + ;; sixth pass a[5]*a[6],...,a[5]*a[7] & seventh pass a[6]*a[7] + ;; ----------------------------------------------------------- + mov rdx, [%%pA+sizeof(qword)*5] + +gsmulx %%x3, rax, [%%pA+sizeof(qword)*6] + add %%x1, rax + adc %%x3, 0 + + MULADDx %%x3, %%x2, rdx, [%%pA+sizeof(qword)*7], %%x6 + + mov rax, [%%pA+sizeof(qword)*6] + mul qword [%%pA+sizeof(qword)*7] + add %%x3, rax + adc rdx, 0 + mov %%x4, rdx + + mov rdx, [%%pA+sizeof(qword)*4] + + ;; --- double x7, x8, x0, ..., x4 + xor %%x5, %%x5 + add %%x7, %%x7 + adc %%x8, %%x8 + adc %%x0, %%x0 + adc %%x1, %%x1 + adc %%x2, %%x2 + adc %%x3, %%x3 + adc %%x4, %%x4 + adc %%x5, 0 + + ;; add sqr(a[4]),...,sqr(a[7]) +gsmulx rax, rdx, rdx ; a[4]^2 + add rdx, %%A + adc rax, 0 + add %%x7, rdx + mov rdx, [%%pA+sizeof(qword)*5] + mov [%%pDst+sizeof(qword)*8], %%x7 + adc %%x8, rax + mov [%%pDst+sizeof(qword)*9], %%x8 + +gsmulx rax, rdx, rdx ; a[5]^2 + adc %%x0, rdx + mov [%%pDst+sizeof(qword)*10], %%x0 + mov rdx, [%%pA+sizeof(qword)*6] + adc %%x1, rax + mov [%%pDst+sizeof(qword)*11], %%x1 + +gsmulx rax, rdx, rdx ; a[6]^2 + adc %%x2, rdx + mov [%%pDst+sizeof(qword)*12], %%x2 + mov rdx, [%%pA+sizeof(qword)*7] + adc %%x3, rax + mov [%%pDst+sizeof(qword)*13], %%x3 + +gsmulx rax, rdx, rdx ; a[7]^2 + adc %%x4, rdx + mov [%%pDst+sizeof(qword)*14], %%x4 + adc %%x5, rax + mov [%%pDst+sizeof(qword)*15], %%x5 +%endmacro + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_8,PRIVATE + SQR_512 pDst, pA, x7, x6, x5, x4, x3, x2, x1, x0, A, x8, t0 + ret +ENDFUNC sqr_8 + + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; SQR_512_TRIANGLE_STEP +; executes single line of upper tpiangle +; +; Inp:partial sum: x7 x6 x5 x4 x3 x2 x1 x0 +; rdx * pA[] p p p p p p p p p +; Out: x0 x7 x6 x5 x4 x3 x2 x1 +; [dst] + +%macro SQR_512_TRIANGLE_STEP 14.nolist + %xdefine %%HEAD_X %1 + %xdefine %%X7 %2 + %xdefine %%X6 %3 + %xdefine %%X5 %4 + %xdefine %%X4 %5 + %xdefine %%X3 %6 + %xdefine %%X2 %7 + %xdefine %%X1 %8 + %xdefine %%X0 %9 + %xdefine %%TAIL_X %10 + %xdefine %%pDst %11 + %xdefine %%pA %12 + %xdefine %%TMP1 %13 + %xdefine %%TMP2 %14 + +%ifnempty %%X0 +gsmulx %%TMP1, rax, [%%pA+sizeof(qword)*0] ; TMP1:rax = rdx * pA[0] + add %%X0, rax + adc %%TMP1, 0 +%endif + +%ifnempty %%TAIL_X + mov %%pDst, %%TAIL_X +%endif + +%ifnempty %%X1 +gsmulx %%TMP2, rax, [%%pA+sizeof(qword)*1] ; TMP2:rax = rdx * pA[1] + add %%X1, rax + adc %%TMP2, 0 +%ifnempty %%X0 + add %%X1, %%TMP1 + adc %%TMP2, 0 +%endif +%endif + +%ifnempty %%X2 +gsmulx %%TMP1, rax, [%%pA+sizeof(qword)*2] ; TMP1:rax = rdx * pA[2] + add %%X2, rax + adc %%TMP1, 0 +%ifnempty %%X1 + add %%X2, %%TMP2 + adc %%TMP1, 0 +%endif +%endif + +%ifnempty %%X3 +gsmulx %%TMP2, rax, [%%pA+sizeof(qword)*3] ; TMP2:rax = rdx * pA[3] + add %%X3, rax + adc %%TMP2, 0 +%ifnempty %%X2 + add %%X3, %%TMP1 + adc %%TMP2, 0 +%endif +%endif + +%ifnempty %%X4 +gsmulx %%TMP1, rax, [%%pA+sizeof(qword)*4] ; TMP1:rax = rdx * pA[4] + add %%X4, rax + adc %%TMP1, 0 +%ifnempty %%X3 + add %%X4, %%TMP2 + adc %%TMP1, 0 +%endif +%endif + +%ifnempty %%X5 +gsmulx %%TMP2, rax, [%%pA+sizeof(qword)*5] ; TMP2:rax = rdx * pA[5] + add %%X5, rax + adc %%TMP2, 0 +%ifnempty %%X4 + add %%X5, %%TMP1 + adc %%TMP2, 0 +%endif +%endif + +%ifnempty %%X6 +gsmulx %%TMP1, rax, [%%pA+sizeof(qword)*6] ; TMP1:rax = rdx * pA[6] + add %%X6, rax + adc %%TMP1, 0 +%ifnempty %%X5 + add %%X6, %%TMP2 + adc %%TMP1, 0 +%endif +%endif + +%ifnempty %%X7 +gsmulx %%HEAD_X, rax, [%%pA+sizeof(qword)*7] ; X0:rax = rdx * pA[7] + add %%X7, rax + adc %%HEAD_X, 0 +%ifnempty %%X6 + add %%X7, %%TMP1 + adc %%HEAD_X, 0 +%endif +%endif +%endmacro + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; square and add diagonal terms + +%macro ADD_DIAG 3.nolist + %xdefine %%N %1 + %xdefine %%pDst %2 + %xdefine %%pSrc %3 + +mov r8, qword [%%pDst+sizeof(qword)*0] + mov r9, qword [%%pDst+sizeof(qword)*1] +%if %%N > 1 + mov r10,qword [%%pDst+sizeof(qword)*2] + mov r11,qword [%%pDst+sizeof(qword)*3] +%if %%N > 2 + mov r12,qword [%%pDst+sizeof(qword)*4] + mov r13,qword [%%pDst+sizeof(qword)*5] +%if %%N > 3 + mov r14,qword [%%pDst+sizeof(qword)*6] + mov r15,qword [%%pDst+sizeof(qword)*7] +%endif +%endif +%endif + + xor rbp, rbp + add r8, r8 + adc r9, r9 +%if %%N > 1 + adc r10,r10 + adc r11,r11 +%if %%N > 2 + adc r12,r12 + adc r13,r13 +%if %%N > 3 + adc r14,r14 + adc r15,r15 +%endif +%endif +%endif + adc rbp, 0 + + mov rdx, [%%pSrc+sizeof(qword)*0] +gsmulx rdx, rax, rdx + add rax, rbx + adc rdx, 0 + add r8, rax + mov rbx, rbp + adc r9, rdx + +%if %%N > 1 + mov rdx, [%%pSrc+sizeof(qword)*1] +gsmulx rdx, rax, rdx + adc r10,rax + adc r11,rdx + +%if %%N > 2 + mov rdx, [%%pSrc+sizeof(qword)*2] +gsmulx rdx, rax, rdx + adc r12,rax + adc r13,rdx + +%if %%N > 3 + mov rdx, [%%pSrc+sizeof(qword)*3] +gsmulx rdx, rax, rdx + adc r14,rax + adc r15,rdx +%endif +%endif +%endif + + adc rbx, 0 + mov qword [%%pDst+sizeof(qword)*0], r8 + mov qword [%%pDst+sizeof(qword)*1], r9 +%if %%N > 1 + mov qword [%%pDst+sizeof(qword)*2], r10 + mov qword [%%pDst+sizeof(qword)*3], r11 +%if %%N > 2 + mov qword [%%pDst+sizeof(qword)*4], r12 + mov qword [%%pDst+sizeof(qword)*5], r13 +%if %%N > 3 + mov qword [%%pDst+sizeof(qword)*6], r14 + mov qword [%%pDst+sizeof(qword)*7], r15 +%endif +%endif +%endif +%endmacro + + + +;; rbp = local carry +;; rbx = global carry +align IPP_ALIGN_FACTOR +DECLARE_FUNC add_diag_4,PRIVATE + ADD_DIAG 4, rdi, rsi + ret +ENDFUNC add_diag_4 + + + +;; +;; 8*qword triangle +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr8_triangle,PRIVATE + ;; A[0]*A[1..7] + mov rdx, [rsi+sizeof(qword)*0] + SQR_512_TRIANGLE_STEP r8, r15,r14,r13,r12,r11,r10, r9, , r8, [rdi+sizeof(qword)*0],rsi, rbx,rbp + + ;; A[1]*A[2..7] + mov rdx, [rsi+sizeof(qword)*1] + SQR_512_TRIANGLE_STEP r9, r8,r15,r14,r13,r12,r11, , , r9, [rdi+sizeof(qword)*1],rsi, rbx,rbp + + ;; A[2]*A[3..7] + mov rdx, [rsi+sizeof(qword)*2] + SQR_512_TRIANGLE_STEP r10, r9, r8,r15,r14,r13, , , , r10, [rdi+sizeof(qword)*2],rsi, rbx,rbp + + ;; A[3]*A[4..7] + mov rdx, [rsi+sizeof(qword)*3] + SQR_512_TRIANGLE_STEP r11, r10, r9, r8,r15, , , , , r11, [rdi+sizeof(qword)*3],rsi, rbx,rbp + + ;; A[4]*A[5..7] + mov rdx, [rsi+sizeof(qword)*4] + SQR_512_TRIANGLE_STEP r12, r11,r10, r9, , , , , , r12, [rdi+sizeof(qword)*4],rsi, rbx,rbp + + ;; A[5]*A[6..7] + mov rdx, [rsi+sizeof(qword)*5] + SQR_512_TRIANGLE_STEP r13, r12,r11, , , , , , , r13, [rdi+sizeof(qword)*5],rsi, rbx,rbp + + ;; A[6]*A[7] + mov rdx, [rsi+sizeof(qword)*6] + SQR_512_TRIANGLE_STEP r14, r13, , , , , , , , r14, [rdi+sizeof(qword)*6],rsi, rbx,rbp + ret +ENDFUNC sqr8_triangle + + + +;; +;; 9*qword squarer +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_9,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + xor r15, r15 + call mla_8x1 + + mov qword [rdi+sizeof(qword)*(1+0)], r8 + mov qword [rdi+sizeof(qword)*(1+1)], r9 + mov qword [rdi+sizeof(qword)*(1+2)], r10 + mov qword [rdi+sizeof(qword)*(1+3)], r11 + mov qword [rdi+sizeof(qword)*(1+4)], r12 + mov qword [rdi+sizeof(qword)*(1+5)], r13 + mov qword [rdi+sizeof(qword)*(1+6)], r14 + mov qword [rdi+sizeof(qword)*(1+7)], r15 + + xor rbx, rbx + mov qword [rdi+sizeof(qword)*(1+8)], rbx + + sub rdi, sizeof(qword)*8 + + call add_diag_4 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*4 + call add_diag_4 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*4 + ADD_DIAG 1, rdi, rsi + + sub rsi, sizeof(qword)*8 + sub rdi, sizeof(qword)*16 + ret +ENDFUNC sqr_9 + + + +;; +;; 10*qword squarer +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_10,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + xor r15, r15 + call mla_8x2 + + mov qword [rdi+sizeof(qword)*(2+0)], r8 + mov qword [rdi+sizeof(qword)*(2+1)], r9 + mov qword [rdi+sizeof(qword)*(2+2)], r10 + mov qword [rdi+sizeof(qword)*(2+3)], r11 + mov qword [rdi+sizeof(qword)*(2+4)], r12 + mov qword [rdi+sizeof(qword)*(2+5)], r13 + mov qword [rdi+sizeof(qword)*(2+6)], r14 + + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15, , , , , , , , , , {rsi+sizeof(qword)*2}, rbx,rbp + + xor rbx, rbx + mov qword [rdi+sizeof(qword)*(2+7)], r15 + mov qword [rdi+sizeof(qword)*(2+8)], r8 + mov qword [rdi+sizeof(qword)*(2+9)], rbx + + sub rdi, sizeof(qword)*8 + + call add_diag_4 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*4 + call add_diag_4 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*4 + ADD_DIAG 2, rdi, rsi + + sub rsi, sizeof(qword)*8 + sub rdi, sizeof(qword)*16 + ret +ENDFUNC sqr_10 + + +;; +;; 10*qword squarer +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_11,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + xor r15, r15 + call mla_8x1 + add rdi, sizeof(qword)*1 + add rcx, sizeof(qword)*1 + call mla_8x2 + sub rdi, sizeof(qword)*1 + sub rcx, sizeof(qword)*1 + + mov qword [rdi+sizeof(qword)*(3+0)], r8 + mov qword [rdi+sizeof(qword)*(3+1)], r9 + mov qword [rdi+sizeof(qword)*(3+2)], r10 + mov qword [rdi+sizeof(qword)*(3+3)], r11 + mov qword [rdi+sizeof(qword)*(3+4)], r12 + + +; SQR_512_TRIANGLE_STEP H, p7, p6, p5, p4, p3, p2, p1, p0, T, ,, rbx,rbp +; ------------------------------ + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15,r14, , , , , , , r13,[rdi+sizeof(qword)*(3+5)],{rsi+sizeof(qword)*3}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*9] + SQR_512_TRIANGLE_STEP r9, r8, , , , , , , , r14,[rdi+sizeof(qword)*(3+6)],{rsi+sizeof(qword)*3}, rbx,rbp + + xor rbx, rbx + mov qword [rdi+sizeof(qword)*(3+7)], r15 + mov qword [rdi+sizeof(qword)*(3+8)], r8 + mov qword [rdi+sizeof(qword)*(3+9)], r9 + mov qword [rdi+sizeof(qword)*(3+10)],rbx + + sub rdi, sizeof(qword)*8 + + call add_diag_4 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*4 + call add_diag_4 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*4 + ADD_DIAG 3, rdi, rsi + + sub rsi, sizeof(qword)*(4*2) + sub rdi, sizeof(qword)*(8*2) + ret +ENDFUNC sqr_11 + + +;; +;; 12*qword squarer +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_12,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + xor r15, r15 + call mla_8x2 + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x2 + sub rdi, sizeof(qword)*2 + sub rcx, sizeof(qword)*2 + + mov qword [rdi+sizeof(qword)*(4+0)], r8 + mov qword [rdi+sizeof(qword)*(4+1)], r9 + mov qword [rdi+sizeof(qword)*(4+2)], r10 + mov qword [rdi+sizeof(qword)*(4+3)], r11 + +; SQR_512_TRIANGLE_STEP H, p7, p6, p5, p4, p3, p2, p1, p0, T, ,, rbx,rbp +; ------------------------------ + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15,r14,r13, , , , , , r12,[rdi+sizeof(qword)*(4+4)],{rsi+sizeof(qword)*4}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*9] + SQR_512_TRIANGLE_STEP r9, r8,r15, , , , , , , r13,[rdi+sizeof(qword)*(4+5)],{rsi+sizeof(qword)*4}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*10] + SQR_512_TRIANGLE_STEP r10, r9, , , , , , , , r14,[rdi+sizeof(qword)*(4+6)],{rsi+sizeof(qword)*4}, rbx,rbp + + xor rbx, rbx + mov qword [rdi+sizeof(qword)*(4+7)], r15 + mov qword [rdi+sizeof(qword)*(4+8)], r8 + mov qword [rdi+sizeof(qword)*(4+9)], r9 + mov qword [rdi+sizeof(qword)*(4+10)],r10 + mov qword [rdi+sizeof(qword)*(4+11)],rbx + + sub rdi, sizeof(qword)*8 + + call add_diag_4 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*4 + call add_diag_4 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*4 + call add_diag_4 + + sub rsi, sizeof(qword)*(4*2) + sub rdi, sizeof(qword)*(8*2) + ret +ENDFUNC sqr_12 + + +;; +;; 13*qword squarer +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_13,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + xor r15, r15 + call mla_8x1 + add rdi, sizeof(qword) + add rcx, sizeof(qword) + call mla_8x2 + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x2 + sub rdi, sizeof(qword)*3 + sub rcx, sizeof(qword)*3 + + mov qword [rdi+sizeof(qword)*(5+0)], r8 + mov qword [rdi+sizeof(qword)*(5+1)], r9 + mov qword [rdi+sizeof(qword)*(5+2)], r10 + +; SQR_512_TRIANGLE_STEP H, p7, p6, p5, p4, p3, p2, p1, p0, T, ,, rbx,rbp +; ------------------------------ + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15,r14,r13,r12, , , , , r11,[rdi+sizeof(qword)*(5+3)],{rsi+sizeof(qword)*5}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*9] + SQR_512_TRIANGLE_STEP r9, r8,r15,r14, , , , , , r12,[rdi+sizeof(qword)*(5+4)],{rsi+sizeof(qword)*5}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*10] + SQR_512_TRIANGLE_STEP r10, r9, r8, , , , , , , r13,[rdi+sizeof(qword)*(5+5)],{rsi+sizeof(qword)*5}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*11] + SQR_512_TRIANGLE_STEP r11, r10, , , , , , , , r14,[rdi+sizeof(qword)*(5+6)],{rsi+sizeof(qword)*5}, rbx,rbp + + xor rbx, rbx + mov qword [rdi+sizeof(qword)*(5+7)], r15 + mov qword [rdi+sizeof(qword)*(5+8)], r8 + mov qword [rdi+sizeof(qword)*(5+9)], r9 + mov qword [rdi+sizeof(qword)*(5+10)],r10 + mov qword [rdi+sizeof(qword)*(5+11)],r11 + mov qword [rdi+sizeof(qword)*(5+12)],rbx + + sub rdi, sizeof(qword)*8 + + call add_diag_4 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*4 + call add_diag_4 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*4 + call add_diag_4 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*4 + ADD_DIAG 1, rdi, rsi + + sub rsi, sizeof(qword)*(4*3) + sub rdi, sizeof(qword)*(8*3) + ret +ENDFUNC sqr_13 + + +;; +;; 14*qword squarer +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_14,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + xor r15, r15 + call mla_8x2 + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x2 + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x2 + sub rdi, sizeof(qword)*4 + sub rcx, sizeof(qword)*4 + + mov qword [rdi+sizeof(qword)*(6+0)], r8 + mov qword [rdi+sizeof(qword)*(6+1)], r9 + +; SQR_512_TRIANGLE_STEP H, p7, p6, p5, p4, p3, p2, p1, p0, T, ,, rbx,rbp +; ------------------------------ + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15,r14,r13,r12,r11, , , , r10,[rdi+sizeof(qword)*(6+2)],{rsi+sizeof(qword)*6}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*9] + SQR_512_TRIANGLE_STEP r9, r8,r15,r14,r13, , , , , r11,[rdi+sizeof(qword)*(6+3)],{rsi+sizeof(qword)*6}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*10] + SQR_512_TRIANGLE_STEP r10, r9, r8,r15, , , , , , r12,[rdi+sizeof(qword)*(6+4)],{rsi+sizeof(qword)*6}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*11] + SQR_512_TRIANGLE_STEP r11, r10, r9, , , , , , , r13,[rdi+sizeof(qword)*(6+5)],{rsi+sizeof(qword)*6}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*12] + SQR_512_TRIANGLE_STEP r12, r11, , , , , , , , r14,[rdi+sizeof(qword)*(6+6)],{rsi+sizeof(qword)*6}, rbx,rbp + + xor rbx, rbx + mov qword [rdi+sizeof(qword)*(6+7)], r15 + mov qword [rdi+sizeof(qword)*(6+8)], r8 + mov qword [rdi+sizeof(qword)*(6+9)], r9 + mov qword [rdi+sizeof(qword)*(6+10)],r10 + mov qword [rdi+sizeof(qword)*(6+11)],r11 + mov qword [rdi+sizeof(qword)*(6+12)],r12 + mov qword [rdi+sizeof(qword)*(6+13)],rbx + + sub rdi, sizeof(qword)*8 + + call add_diag_4 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*4 + call add_diag_4 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*4 + call add_diag_4 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*4 + ADD_DIAG 2, rdi, rsi + + sub rsi, sizeof(qword)*(4*3) + sub rdi, sizeof(qword)*(8*3) + ret +ENDFUNC sqr_14 + + +;; +;; 15*qword squarer +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_15,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + xor r15, r15 + call mla_8x1 + add rdi, sizeof(qword) + add rcx, sizeof(qword) + call mla_8x2 + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x2 + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x2 + sub rdi, sizeof(qword)*5 + sub rcx, sizeof(qword)*5 + + mov qword [rdi+sizeof(qword)*(7+0)], r8 + +; SQR_512_TRIANGLE_STEP H, p7, p6, p5, p4, p3, p2, p1, p0, T, ,, rbx,rbp +; ------------------------------ + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15,r14,r13,r12,r11,r10, , , r9, [rdi+sizeof(qword)*(7+1)],{rsi+sizeof(qword)*7}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*9] + SQR_512_TRIANGLE_STEP r9, r8,r15,r14,r13,r12, , , , r10,[rdi+sizeof(qword)*(7+2)],{rsi+sizeof(qword)*7}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*10] + SQR_512_TRIANGLE_STEP r10, r9, r8,r15,r14, , , , , r11,[rdi+sizeof(qword)*(7+3)],{rsi+sizeof(qword)*7}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*11] + SQR_512_TRIANGLE_STEP r11, r10, r9, r8, , , , , , r12,[rdi+sizeof(qword)*(7+4)],{rsi+sizeof(qword)*7}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*12] + SQR_512_TRIANGLE_STEP r12, r11,r10, , , , , , , r13,[rdi+sizeof(qword)*(7+5)],{rsi+sizeof(qword)*7}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*13] + SQR_512_TRIANGLE_STEP r13, r12, , , , , , , , r14,[rdi+sizeof(qword)*(7+6)],{rsi+sizeof(qword)*7}, rbx,rbp + + xor rbx, rbx + mov qword [rdi+sizeof(qword)*(7+7)], r15 + mov qword [rdi+sizeof(qword)*(7+8)], r8 + mov qword [rdi+sizeof(qword)*(7+9)], r9 + mov qword [rdi+sizeof(qword)*(7+10)],r10 + mov qword [rdi+sizeof(qword)*(7+11)],r11 + mov qword [rdi+sizeof(qword)*(7+12)],r12 + mov qword [rdi+sizeof(qword)*(7+13)],r13 + mov qword [rdi+sizeof(qword)*(7+14)],rbx + + sub rdi, sizeof(qword)*8 + + call add_diag_4 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*4 + call add_diag_4 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*4 + call add_diag_4 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*4 + ADD_DIAG 3, rdi, rsi + + sub rsi, sizeof(qword)*(4*3) + sub rdi, sizeof(qword)*(8*3) + ret +ENDFUNC sqr_15 + + +;; +;; 16*qword squarer +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_16,PRIVATE + call sqr8_triangle + + mov qword [rdi+sizeof(qword)*7], r15 + + mov rcx, rsi + add rsi, sizeof(qword)*8 + add rdi, sizeof(qword)*8 + xor r15, r15 + call mla_8x2 + + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x2 + + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x2 + + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x2 + + sub rdi, sizeof(qword)*6 + sub rcx, sizeof(qword)*6 + + add rdi, sizeof(qword)*8 + call sqr8_triangle + + xor rbx, rbx + mov qword [rdi+sizeof(qword)*7], r15 + mov qword [rdi+sizeof(qword)*8], r8 + mov qword [rdi+sizeof(qword)*9], r9 + mov qword [rdi+sizeof(qword)*10],r10 + mov qword [rdi+sizeof(qword)*11],r11 + mov qword [rdi+sizeof(qword)*12],r12 + mov qword [rdi+sizeof(qword)*13],r13 + mov qword [rdi+sizeof(qword)*14],r14 + mov qword [rdi+sizeof(qword)*15],rbx + + sub rsi, sizeof(qword)*8 + sub rdi, sizeof(qword)*16 + + call add_diag_4 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*4 + call add_diag_4 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*4 + call add_diag_4 + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*4 + call add_diag_4 + + sub rsi, sizeof(qword)*12 + sub rdi, sizeof(qword)*24 + ret +ENDFUNC sqr_16 + + + +;; +;; 9*qword triangle +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr9_triangle,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + xor r15, r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + call mla_8x1 + + xor rax, rax + mov qword [rdi+sizeof(qword)*(1+0)], r8 + mov qword [rdi+sizeof(qword)*(1+1)], r9 + mov qword [rdi+sizeof(qword)*(1+2)], r10 + mov qword [rdi+sizeof(qword)*(1+3)], r11 + mov qword [rdi+sizeof(qword)*(1+4)], r12 + mov qword [rdi+sizeof(qword)*(1+5)], r13 + mov qword [rdi+sizeof(qword)*(1+6)], r14 + mov qword [rdi+sizeof(qword)*(1+7)], r15 + mov qword [rdi+sizeof(qword)*(1+8)], rax + + sub rdi, sizeof(qword)*8 + ret +ENDFUNC sqr9_triangle + + +;; +;; 10*qword triangle +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr10_triangle,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + xor r15, r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + call mla_8x2 + + mov qword [rdi+sizeof(qword)*(2+0)], r8 + mov qword [rdi+sizeof(qword)*(2+1)], r9 + mov qword [rdi+sizeof(qword)*(2+2)], r10 + mov qword [rdi+sizeof(qword)*(2+3)], r11 + mov qword [rdi+sizeof(qword)*(2+4)], r12 + mov qword [rdi+sizeof(qword)*(2+5)], r13 + mov qword [rdi+sizeof(qword)*(2+6)], r14 + + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15, , , , , , , , , , {rsi+sizeof(qword)*2}, rbx,rbp + + xor rax, rax + mov qword [rdi+sizeof(qword)*(2+7)], r15 + mov qword [rdi+sizeof(qword)*(2+8)], r8 + mov qword [rdi+sizeof(qword)*(2+9)], rax + + sub rdi, sizeof(qword)*8 + ret +ENDFUNC sqr10_triangle + + +;; +;; 11*qword triangle +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr11_triangle,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + xor r15, r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + call mla_8x3 + + mov qword [rdi+sizeof(qword)*(3+0)], r8 + mov qword [rdi+sizeof(qword)*(3+1)], r9 + mov qword [rdi+sizeof(qword)*(3+2)], r10 + mov qword [rdi+sizeof(qword)*(3+3)], r11 + mov qword [rdi+sizeof(qword)*(3+4)], r12 + +; SQR_512_TRIANGLE_STEP H, p7, p6, p5, p4, p3, p2, p1, p0, T, ,, rbx,rbp +; ------------------------------ + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15,r14, , , , , , , r13,[rdi+sizeof(qword)*(3+5)],{rsi+sizeof(qword)*3}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*9] + SQR_512_TRIANGLE_STEP r9, r8, , , , , , , , r14,[rdi+sizeof(qword)*(3+6)],{rsi+sizeof(qword)*3}, rbx,rbp + + xor rax, rax + mov qword [rdi+sizeof(qword)*(3+7)], r15 + mov qword [rdi+sizeof(qword)*(3+8)], r8 + mov qword [rdi+sizeof(qword)*(3+9)], r9 + mov qword [rdi+sizeof(qword)*(3+10)],rax + + sub rdi, sizeof(qword)*8 + ret +ENDFUNC sqr11_triangle + + +;; +;; 12*qword triangle +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr12_triangle,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + xor r15, r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + call mla_8x4 + + mov qword [rdi+sizeof(qword)*(4+0)], r8 + mov qword [rdi+sizeof(qword)*(4+1)], r9 + mov qword [rdi+sizeof(qword)*(4+2)], r10 + mov qword [rdi+sizeof(qword)*(4+3)], r11 + +; SQR_512_TRIANGLE_STEP H, p7, p6, p5, p4, p3, p2, p1, p0, T, ,, rbx,rbp +; ------------------------------ + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15,r14,r13, , , , , , r12,[rdi+sizeof(qword)*(4+4)],{rsi+sizeof(qword)*4}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*9] + SQR_512_TRIANGLE_STEP r9, r8,r15, , , , , , , r13,[rdi+sizeof(qword)*(4+5)],{rsi+sizeof(qword)*4}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*10] + SQR_512_TRIANGLE_STEP r10, r9, , , , , , , , r14,[rdi+sizeof(qword)*(4+6)],{rsi+sizeof(qword)*4}, rbx,rbp + + xor rax, rax + mov qword [rdi+sizeof(qword)*(4+7)], r15 + mov qword [rdi+sizeof(qword)*(4+8)], r8 + mov qword [rdi+sizeof(qword)*(4+9)], r9 + mov qword [rdi+sizeof(qword)*(4+10)],r10 + mov qword [rdi+sizeof(qword)*(4+11)],rax + + sub rdi, sizeof(qword)*8 + ret +ENDFUNC sqr12_triangle + + +;; +;; 13*qword triangle +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr13_triangle,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + xor r15, r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + call mla_8x5 + + mov qword [rdi+sizeof(qword)*(5+0)], r8 + mov qword [rdi+sizeof(qword)*(5+1)], r9 + mov qword [rdi+sizeof(qword)*(5+2)], r10 + +; SQR_512_TRIANGLE_STEP H, p7, p6, p5, p4, p3, p2, p1, p0, T, ,, rbx,rbp +; ------------------------------ + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15,r14,r13,r12, , , , , r11,[rdi+sizeof(qword)*(5+3)],{rsi+sizeof(qword)*5}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*9] + SQR_512_TRIANGLE_STEP r9, r8,r15,r14, , , , , , r12,[rdi+sizeof(qword)*(5+4)],{rsi+sizeof(qword)*5}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*10] + SQR_512_TRIANGLE_STEP r10, r9, r8, , , , , , , r13,[rdi+sizeof(qword)*(5+5)],{rsi+sizeof(qword)*5}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*11] + SQR_512_TRIANGLE_STEP r11, r10, , , , , , , , r14,[rdi+sizeof(qword)*(5+6)],{rsi+sizeof(qword)*5}, rbx,rbp + + xor rax, rax + mov qword [rdi+sizeof(qword)*(5+7)], r15 + mov qword [rdi+sizeof(qword)*(5+8)], r8 + mov qword [rdi+sizeof(qword)*(5+9)], r9 + mov qword [rdi+sizeof(qword)*(5+10)],r10 + mov qword [rdi+sizeof(qword)*(5+11)],r11 + mov qword [rdi+sizeof(qword)*(5+12)],rax + + sub rdi, sizeof(qword)*8 + ret +ENDFUNC sqr13_triangle + + +;; +;; 14*qword triangle +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr14_triangle,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + xor r15, r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + call mla_8x6 + + mov qword [rdi+sizeof(qword)*(6+0)], r8 + mov qword [rdi+sizeof(qword)*(6+1)], r9 + +; SQR_512_TRIANGLE_STEP H, p7, p6, p5, p4, p3, p2, p1, p0, T, ,, rbx,rbp +; ------------------------------ + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15,r14,r13,r12,r11, , , , r10,[rdi+sizeof(qword)*(6+2)],{rsi+sizeof(qword)*6}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*9] + SQR_512_TRIANGLE_STEP r9, r8,r15,r14,r13, , , , , r11,[rdi+sizeof(qword)*(6+3)],{rsi+sizeof(qword)*6}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*10] + SQR_512_TRIANGLE_STEP r10, r9, r8,r15, , , , , , r12,[rdi+sizeof(qword)*(6+4)],{rsi+sizeof(qword)*6}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*11] + SQR_512_TRIANGLE_STEP r11, r10, r9, , , , , , , r13,[rdi+sizeof(qword)*(6+5)],{rsi+sizeof(qword)*6}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*12] + SQR_512_TRIANGLE_STEP r12, r11, , , , , , , , r14,[rdi+sizeof(qword)*(6+6)],{rsi+sizeof(qword)*6}, rbx,rbp + + xor rax, rax + mov qword [rdi+sizeof(qword)*(6+7)], r15 + mov qword [rdi+sizeof(qword)*(6+8)], r8 + mov qword [rdi+sizeof(qword)*(6+9)], r9 + mov qword [rdi+sizeof(qword)*(6+10)],r10 + mov qword [rdi+sizeof(qword)*(6+11)],r11 + mov qword [rdi+sizeof(qword)*(6+12)],r12 + mov qword [rdi+sizeof(qword)*(6+13)],rax + + sub rdi, sizeof(qword)*8 + ret +ENDFUNC sqr14_triangle + + +;; +;; 15*qword triangle +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr15_triangle,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + xor r15, r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + call mla_8x7 + + mov qword [rdi+sizeof(qword)*(7+0)], r8 + +; SQR_512_TRIANGLE_STEP H, p7, p6, p5, p4, p3, p2, p1, p0, T, ,, rbx,rbp +; ------------------------------ + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15,r14,r13,r12,r11,r10, , , r9, [rdi+sizeof(qword)*(7+1)],{rsi+sizeof(qword)*7}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*9] + SQR_512_TRIANGLE_STEP r9, r8,r15,r14,r13,r12, , , , r10,[rdi+sizeof(qword)*(7+2)],{rsi+sizeof(qword)*7}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*10] + SQR_512_TRIANGLE_STEP r10, r9, r8,r15,r14, , , , , r11,[rdi+sizeof(qword)*(7+3)],{rsi+sizeof(qword)*7}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*11] + SQR_512_TRIANGLE_STEP r11, r10, r9, r8, , , , , , r12,[rdi+sizeof(qword)*(7+4)],{rsi+sizeof(qword)*7}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*12] + SQR_512_TRIANGLE_STEP r12, r11,r10, , , , , , , r13,[rdi+sizeof(qword)*(7+5)],{rsi+sizeof(qword)*7}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*13] + SQR_512_TRIANGLE_STEP r13, r12, , , , , , , , r14,[rdi+sizeof(qword)*(7+6)],{rsi+sizeof(qword)*7}, rbx,rbp + + xor rax, rax + mov qword [rdi+sizeof(qword)*(7+7)], r15 + mov qword [rdi+sizeof(qword)*(7+8)], r8 + mov qword [rdi+sizeof(qword)*(7+9)], r9 + mov qword [rdi+sizeof(qword)*(7+10)],r10 + mov qword [rdi+sizeof(qword)*(7+11)],r11 + mov qword [rdi+sizeof(qword)*(7+12)],r12 + mov qword [rdi+sizeof(qword)*(7+13)],r13 + mov qword [rdi+sizeof(qword)*(7+14)],rax + + sub rdi, sizeof(qword)*8 + ret +ENDFUNC sqr15_triangle + + +;; +;; 16*qword triangle +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr16_triangle,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + xor r15, r15 + + mov rcx, rsi + add rsi, sizeof(qword)*8 + add rdi, sizeof(qword)*8 + call mla_8x8 + + add rdi, sizeof(qword)*8 + call sqr8_triangle + + xor rax, rax + mov qword [rdi+sizeof(qword)*7], r15 + mov qword [rdi+sizeof(qword)*8], r8 + mov qword [rdi+sizeof(qword)*9], r9 + mov qword [rdi+sizeof(qword)*10],r10 + mov qword [rdi+sizeof(qword)*11],r11 + mov qword [rdi+sizeof(qword)*12],r12 + mov qword [rdi+sizeof(qword)*13],r13 + mov qword [rdi+sizeof(qword)*14],r14 + mov qword [rdi+sizeof(qword)*15],rax + + sub rsi, sizeof(qword)*8 + sub rdi, sizeof(qword)*16 + ret +ENDFUNC sqr16_triangle + + +sqr_l_basic dq sqr_1 - sqr_l_basic + dq sqr_2 - sqr_l_basic + dq sqr_3 - sqr_l_basic + dq sqr_4 - sqr_l_basic + dq sqr_5 - sqr_l_basic + dq sqr_6 - sqr_l_basic + dq sqr_7 - sqr_l_basic + dq sqr_8 - sqr_l_basic + dq sqr_9 - sqr_l_basic + dq sqr_10- sqr_l_basic + dq sqr_11- sqr_l_basic + dq sqr_12- sqr_l_basic + dq sqr_13- sqr_l_basic + dq sqr_14- sqr_l_basic + dq sqr_15- sqr_l_basic + dq sqr_16- sqr_l_basic + +sqrN_triangle dq sqr9_triangle - sqrN_triangle + dq sqr10_triangle - sqrN_triangle + dq sqr11_triangle - sqrN_triangle + dq sqr12_triangle - sqrN_triangle + dq sqr13_triangle - sqrN_triangle + dq sqr14_triangle - sqrN_triangle + dq sqr15_triangle - sqrN_triangle + dq sqr16_triangle - sqrN_triangle + +%endif ;; _PCPBNSQR_BASIC_ADCX_INC_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnusqrpp.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnusqrpp.inc new file mode 100644 index 0000000..ef73b5b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnusqrpp.inc @@ -0,0 +1,452 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Low level Big Number squaring Support +; +; + +%ifndef _PCPBNUSQR_INC_ +%assign _PCPBNUSQR_INC_ 1 + +%include "pcpmulx.inc" +%include "pcpbnusqrpp_basic.inc" + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; (8*n) squarer +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_8N_adcox,PRIVATE + push rdi ; save diagonal loop parameters + push rsi + push rdx + + push rdi ; save initial triangle product parameters + push rsi + push rdx +; +; init upper triangle product +; + push rdx + call sqr8_triangle + pop rdx + + mov qword [rdi+sizeof(qword)*7], r15 + xor r15, r15 + add rdi, sizeof(qword)*8 + + sub rdx, 8 + + mov rcx, rsi + add rsi, sizeof(qword)*8 +.initLoop: + push rdx + call mla_8x8 + pop rdx + add rsi, sizeof(qword)*8 + add rdi, sizeof(qword)*8 + sub rdx, 8 + jnz .initLoop + + mov qword [rdi+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*2],r10 + mov qword [rdi+sizeof(qword)*3],r11 + mov qword [rdi+sizeof(qword)*4],r12 + mov qword [rdi+sizeof(qword)*5],r13 + mov qword [rdi+sizeof(qword)*6],r14 + mov qword [rdi+sizeof(qword)*7],r15 + jmp .update_Triangle + +; +; update upper triangle product +; +.outerLoop: + push rdi ; update triangle product parameters + push rsi + push rdx + + xor rax, rax ; c-flag + push rax + + mov r8, qword [rdi+sizeof(qword)*0] + mov r9, qword [rdi+sizeof(qword)*1] + mov r10, qword [rdi+sizeof(qword)*2] + mov r11, qword [rdi+sizeof(qword)*3] + mov r12, qword [rdi+sizeof(qword)*4] + mov r13, qword [rdi+sizeof(qword)*5] + mov r14, qword [rdi+sizeof(qword)*6] + mov r15, qword [rdi+sizeof(qword)*7] + +.innerLoop_entry: + push rdx + call sqr8_triangle + pop rdx + + mov qword [rdi+sizeof(qword)*7], r15 + xor r15, r15 + add rdi, sizeof(qword)*8 + + sub rdx, 8 + jz .skipInnerLoop + + mov rcx, rsi + add rsi, sizeof(qword)*8 +.innerLoop: + pop rax ; restore c-flag + neg rax + op_reg_mem adc, r8, qword [rdi+sizeof(qword)*0], rax + op_reg_mem adc, r9, qword [rdi+sizeof(qword)*1], rax + op_reg_mem adc, r10, qword [rdi+sizeof(qword)*2], rax + op_reg_mem adc, r11, qword [rdi+sizeof(qword)*3], rax + op_reg_mem adc, r12, qword [rdi+sizeof(qword)*4], rax + op_reg_mem adc, r13, qword [rdi+sizeof(qword)*5], rax + op_reg_mem adc, r14, qword [rdi+sizeof(qword)*6], rax + op_reg_mem adc, r15, qword [rdi+sizeof(qword)*7], rax + sbb rax, rax ; save c-flag + push rax + + push rdx + call mla_8x8 + pop rdx + add rsi, sizeof(qword)*8 + add rdi, sizeof(qword)*8 + sub rdx, 8 + jnz .innerLoop + +.skipInnerLoop: + pop rax ; restore c-flag + neg rax + adc r8, 0 + mov qword [rdi+sizeof(qword)*0], r8 + adc r9, 0 + mov qword [rdi+sizeof(qword)*1], r9 + adc r10,0 + mov qword [rdi+sizeof(qword)*2],r10 + adc r11,0 + mov qword [rdi+sizeof(qword)*3],r11 + adc r12,0 + mov qword [rdi+sizeof(qword)*4],r12 + adc r13,0 + mov qword [rdi+sizeof(qword)*5],r13 + adc r14,0 + mov qword [rdi+sizeof(qword)*6],r14 + adc r15,0 + mov qword [rdi+sizeof(qword)*7],r15 + +.update_Triangle: + pop rdx + pop rsi + pop rdi + add rsi, sizeof(qword)*8 + add rdi, sizeof(qword)*(8*2) + sub rdx, 8 + jnz .outerLoop + +; +; add diagonal terms +; + pop rcx + pop rsi + pop rdi + call finalize + ret +ENDFUNC sqr_8N_adcox + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; general case N>16 squarer +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_N_adcox,PRIVATE + push rdi ; save diagonal loop parameters + push rsi + push rdx + + push rdi ; save initial triangle product parameters + push rsi + push rdx + + mov rbp, rdx + and rbp, 7 + GET_EP rax, mla_8xl_tail, rbp ; get tail procedure + push rax + +; +; init upper triangle product +; + sub rdx, 8 + + push rdx + call sqr8_triangle + pop rdx + + mov qword [rdi+sizeof(qword)*7], r15 + add rdi, sizeof(qword)*8 + xor r15, r15 + + mov rcx, rsi + add rsi, sizeof(qword)*8 + sub rdx, 8 + +.initLoop: + push rdx + call mla_8x8 + pop rdx + add rsi, sizeof(qword)*8 + add rdi, sizeof(qword)*8 + sub rdx, 8 + jnc .initLoop + + add rdx, 8 +; +; tail +; + SWAP rsi, rcx + mov rax, [rsp] ; procedure + push rdx + call rax + pop rdx + lea rdi, [rdi+rdx*sizeof(qword)] + + mov qword [rdi+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*2],r10 + mov qword [rdi+sizeof(qword)*3],r11 + mov qword [rdi+sizeof(qword)*4],r12 + mov qword [rdi+sizeof(qword)*5],r13 + mov qword [rdi+sizeof(qword)*6],r14 + mov qword [rdi+sizeof(qword)*7],r15 + jmp .update_Triangle + +; +; update upper triangle product +; +.outerLoop: + push rdi ; update triangle product parameters + push rsi + push rdx + push rax ; tail procedure + + xor rax, rax ; c-flag + push rax + + mov r8, qword [rdi+sizeof(qword)*0] + mov r9, qword [rdi+sizeof(qword)*1] + mov r10, qword [rdi+sizeof(qword)*2] + mov r11, qword [rdi+sizeof(qword)*3] + mov r12, qword [rdi+sizeof(qword)*4] + mov r13, qword [rdi+sizeof(qword)*5] + mov r14, qword [rdi+sizeof(qword)*6] + mov r15, qword [rdi+sizeof(qword)*7] + + sub rdx, 8 + + push rdx + call sqr8_triangle + pop rdx + + mov qword [rdi+sizeof(qword)*7], r15 + add rdi, sizeof(qword)*8 + xor r15, r15 + + mov rcx, rsi + add rsi, sizeof(qword)*8 + sub rdx, 8 + +.innerLoop: + pop rax ; restore c-flag + neg rax + op_reg_mem adc, r8, qword [rdi+sizeof(qword)*0], rax + op_reg_mem adc, r9, qword [rdi+sizeof(qword)*1], rax + op_reg_mem adc, r10, qword [rdi+sizeof(qword)*2], rax + op_reg_mem adc, r11, qword [rdi+sizeof(qword)*3], rax + op_reg_mem adc, r12, qword [rdi+sizeof(qword)*4], rax + op_reg_mem adc, r13, qword [rdi+sizeof(qword)*5], rax + op_reg_mem adc, r14, qword [rdi+sizeof(qword)*6], rax + op_reg_mem adc, r15, qword [rdi+sizeof(qword)*7], rax + sbb rax, rax ; save c-flag + push rax + + push rdx + call mla_8x8 + pop rdx + + add rsi, sizeof(qword)*8 + add rdi, sizeof(qword)*8 + sub rdx, 8 + jnc .innerLoop + + add rdx, 8 +; +; tail +; + ; clear in advance + pxor xmm0, xmm0 + movdqu xmmword [rdi+rdx*sizeof(qword)], xmm0 + movdqu xmmword [rdi+rdx*sizeof(qword)+sizeof(qword)*2], xmm0 + movdqu xmmword [rdi+rdx*sizeof(qword)+sizeof(qword)*4], xmm0 + movdqu xmmword [rdi+rdx*sizeof(qword)+sizeof(qword)*6], xmm0 + + ; updates registers before mla operation + pop rax ; restore c-flag + neg rax + op_reg_mem adc, r8, qword [rdi+sizeof(qword)*0], rax + op_reg_mem adc, r9, qword [rdi+sizeof(qword)*1], rax + op_reg_mem adc, r10, qword [rdi+sizeof(qword)*2], rax + op_reg_mem adc, r11, qword [rdi+sizeof(qword)*3], rax + op_reg_mem adc, r12, qword [rdi+sizeof(qword)*4], rax + op_reg_mem adc, r13, qword [rdi+sizeof(qword)*5], rax + op_reg_mem adc, r14, qword [rdi+sizeof(qword)*6], rax + op_reg_mem adc, r15, qword [rdi+sizeof(qword)*7], rax + + ; store carry for future + sbb rax, rax + neg rax + mov qword [rdi+sizeof(qword)*8], rax + + ; mla_8xn operation + SWAP rsi, rcx + mov rax, [rsp] ; procedure + push rdx + call rax + pop rdx + lea rdi, [rdi+rdx*sizeof(qword)] + + ; updates registers before store + xor rax, rax + mov rax, qword [rdi+sizeof(qword)*0] + add r8, rax + mov qword [rdi+sizeof(qword)*0], r8 + mov rax, qword [rdi+sizeof(qword)*1] + adc r9, rax + mov qword [rdi+sizeof(qword)*1], r9 + mov rax, qword [rdi+sizeof(qword)*2] + adc r10, rax + mov qword [rdi+sizeof(qword)*2], r10 + mov rax, qword [rdi+sizeof(qword)*3] + adc r11, rax + mov qword [rdi+sizeof(qword)*3], r11 + mov rax, qword [rdi+sizeof(qword)*4] + adc r12, rax + mov qword [rdi+sizeof(qword)*4], r12 + mov rax, qword [rdi+sizeof(qword)*5] + adc r13, rax + mov qword [rdi+sizeof(qword)*5], r13 + mov rax, qword [rdi+sizeof(qword)*6] + adc r14, rax + mov qword [rdi+sizeof(qword)*6], r14 + mov rax, qword [rdi+sizeof(qword)*7] + adc r15, rax + mov qword [rdi+sizeof(qword)*7], r15 + +.update_Triangle: + pop rax ; tail procedure + pop rdx + pop rsi + pop rdi + add rsi, sizeof(qword)*8 + add rdi, sizeof(qword)*(8*2) + sub rdx, 8 + cmp rdx, 16 + jg .outerLoop + +; +; tail +; + mov rbp, rdx + sub rbp, 8 + GET_EP rax, sqrN_triangle, rbp ; get triangle proc + + sub rsp, sizeof(qword)*32 + push rdi + push rdx + + mov r8, qword [rdi+sizeof(qword)*0] + mov r9, qword [rdi+sizeof(qword)*1] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + lea rdi, [rsp+sizeof(qword)*2] + call rax + + mov rsi, rdi + pop rdx + pop rdi + + ; copy 8 terms + movdqu xmm0, xmmword [rsi] + movdqu xmm1, xmmword [rsi+sizeof(qword)*2] + movdqu xmm2, xmmword [rsi+sizeof(qword)*4] + movdqu xmm3, xmmword [rsi+sizeof(qword)*6] + add rsi, sizeof(qword)*8 + movdqu xmmword [rdi], xmm0 + movdqu xmmword [rdi+sizeof(qword)*2], xmm1 + movdqu xmmword [rdi+sizeof(qword)*4], xmm2 + movdqu xmmword [rdi+sizeof(qword)*6], xmm3 + add rdi, sizeof(qword)*8 + + ; update rdx-8 terms + lea rax, [rdx-8] + xor rbx, rbx +.update1: + mov r8, qword [rsi] + mov r9, qword [rdi] + add rsi, sizeof(qword) + neg rbx + adc r8, r9 + sbb rbx, rbx + mov qword [rdi], r8 + add rdi, sizeof(qword) + sub rax, 1 + jg .update1 + + ; update rdx terms +.update2: + mov r8, qword [rsi] + add rsi, sizeof(qword) + neg rbx + adc r8, 0 + sbb rbx, rbx + mov qword [rdi], r8 + add rdi, sizeof(qword) + sub rdx, 1 + jg .update2 + + add rsp, sizeof(qword)*32 + +; +; add diagonal terms +; +.add_diagonals: + pop rcx + pop rsi + pop rdi + call finalize + +.quit: + ret +ENDFUNC sqr_N_adcox + + +%endif ;; _PCPBNUSQR_INC_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnusqrpp_basic.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnusqrpp_basic.inc new file mode 100644 index 0000000..c1764c7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnusqrpp_basic.inc @@ -0,0 +1,1800 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Low level Big Number squaring Support +; +; + +%ifndef _PCPBNSQR_BASIC_ADCX_INC_ +%assign _PCPBNSQR_BASIC_ADCX_INC_ 1 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; Fixed-size (1-8 qwords) square operations +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; +;; 1*qword squarer +;; +%macro SQR_64 2.nolist + %xdefine %%pDst %1 + %xdefine %%pA %2 + + mov rdx, qword [%%pA] +gsmulx rdx, rax, rdx + mov qword [%%pDst], rax + mov qword [%%pDst+sizeof(qword)], rdx +%endmacro + +align IPP_ALIGN_FACTOR +sqr_1: + SQR_64 rdi, rsi + ret + + +;; +;; 2*qword squarer +;; +%macro SQR_128 13.nolist + %xdefine %%pDst %1 + %xdefine %%pA %2 + %xdefine %%x7 %3 + %xdefine %%x6 %4 + %xdefine %%x5 %5 + %xdefine %%x4 %6 + %xdefine %%x3 %7 + %xdefine %%x2 %8 + %xdefine %%x1 %9 + %xdefine %%x0 %10 + %xdefine %%A %11 + %xdefine %%x8 %12 + %xdefine %%t0 %13 + + mov rdx, qword [%%pA+sizeof(qword)] + gsmulx rax, %%x1, qword [%%pA] ; (rax:x1) = a[0]*a[1] + gsmulx %%x3, %%x2, rdx ; (x3 :x2) = a[1]^2 + mov rdx, qword [%%pA] ; a[0] + gsmulx rdx, %%x0, rdx ; (rdx:x0) = a[0]^2 + + add %%x1, %%x1 ; (rax:x1) = a[0]*a[1]*2 + adc rax,rax + adc %%x3, 0 + + mov qword [%%pDst+sizeof(qword)*0], %%x0 + add %%x1, rdx + mov qword [%%pDst+sizeof(qword)*1], %%x1 + adc %%x2, rax + mov qword [%%pDst+sizeof(qword)*2], %%x2 + adc %%x3, 0 + mov qword [%%pDst+sizeof(qword)*3], %%x3 +%endmacro + +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_2,PRIVATE + SQR_128 rdi, rsi, r15,r14,r13,r12,r11,r10,r9,r8, rcx, rbx, rbp + ret +ENDFUNC sqr_2 + + + +;; +;; 3*qword squarer +;; +%macro SQR_192 13.nolist + %xdefine %%pDst %1 + %xdefine %%pA %2 + %xdefine %%x7 %3 + %xdefine %%x6 %4 + %xdefine %%x5 %5 + %xdefine %%x4 %6 + %xdefine %%x3 %7 + %xdefine %%x2 %8 + %xdefine %%x1 %9 + %xdefine %%x0 %10 + %xdefine %%A %11 + %xdefine %%x8 %12 + %xdefine %%t0 %13 + + mov rdx, [%%pA] ;; a[0]*{a[1],a[2]} + gsmulx %%x1, %%x0, [%%pA+sizeof(qword)*1] + gsmulx %%x2, rax,[%%pA+sizeof(qword)*2] + add %%x1, rax + adc %%x2, 0 + mov rdx, [%%pA+sizeof(qword)*1] ;; a[1]*a[2] + gsmulx %%x3, rax, [%%pA+sizeof(qword)*2] + add %%x2, rax + adc %%x3, 0 + + xor %%A, %%A + ;; square a[0],a[1],a[2] and add double x0,x1,x2,x3 + mov rdx, [%%pA+sizeof(qword)*0] + gsmulx rdx, rax, rdx + mov qword [%%pDst], rax + gsadcx rdx, %%x0 + gsadox rdx, %%x0 + mov qword [%%pDst+sizeof(qword)], rdx + + mov rdx, [%%pA+sizeof(qword)*1] + gsmulx rdx, rax, rdx + gsadcx rax, %%x1 + gsadox rax, %%x1 + mov qword [%%pDst+sizeof(qword)*2], rax + gsadcx rdx, %%x2 + gsadox rdx, %%x2 + mov qword [%%pDst+sizeof(qword)*3], rdx + + mov rdx, [%%pA+sizeof(qword)*2] + gsmulx rdx, rax, rdx + gsadcx rax, %%x3 + gsadox rax, %%x3 + mov qword [%%pDst+sizeof(qword)*4], rax + gsadcx rdx, %%A + gsadox rdx, %%A + mov qword [%%pDst+sizeof(qword)*5], rdx +%endmacro + +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_3,PRIVATE + SQR_192 rdi,rsi, r15,r14,r13,r12,r11,r10,r9,r8, rcx,rbx,rbp + ret +ENDFUNC sqr_3 + + + +;; +;; 4*qword squarer +;; +%macro SQR_256 13.nolist + %xdefine %%pDst %1 + %xdefine %%pA %2 + %xdefine %%x7 %3 + %xdefine %%x6 %4 + %xdefine %%x5 %5 + %xdefine %%x4 %6 + %xdefine %%x3 %7 + %xdefine %%x2 %8 + %xdefine %%x1 %9 + %xdefine %%x0 %10 + %xdefine %%A %11 + %xdefine %%x8 %12 + %xdefine %%t0 %13 + + ;; 1-st pass a[0]*{a[1],a[2],a[3]} + mov rdx, [%%pA] + gsmulx %%x1, %%x0, [%%pA+sizeof(qword)*1] + xor %%A, %%A + gsmulx %%x2, rax,[%%pA+sizeof(qword)*2] + add %%x1, rax + gsmulx %%x3, rax,[%%pA+sizeof(qword)*3] + adc %%x2, rax + adc %%x3, 0 + + ;; 2-nd pass a[1]*{a[2],a[3]} + mov rdx, [%%pA+sizeof(qword)*1] + gsmulx %%t0, rax, [%%pA+sizeof(qword)*2] + gsadcx %%x2, rax + gsadox %%x3, %%t0 + gsmulx %%x4, rax, [%%pA+sizeof(qword)*3] + gsadcx %%x3, rax + gsadox %%x4, %%A + adc %%x4, 0 + + ;; 3-rd pass a[2]*a[3] + mov rdx, [%%pA+sizeof(qword)*2] + gsmulx %%x5, rax, [%%pA+sizeof(qword)*3] + add %%x4, rax + adc %%x5, 0 + + ;; square a[0],...,a[3] and add double x0,...,,x5 + mov rdx, [%%pA+sizeof(qword)*0] + gsmulx rdx, rax, rdx + mov qword [%%pDst], rax + gsadcx rdx, %%x0 + gsadox rdx, %%x0 + mov qword [%%pDst+sizeof(qword)], rdx + + mov rdx, [%%pA+sizeof(qword)*1] + gsmulx rdx, rax, rdx + gsadcx rax, %%x1 + gsadox rax, %%x1 + mov qword [%%pDst+sizeof(qword)*2], rax + gsadcx rdx, %%x2 + gsadox rdx, %%x2 + mov qword [%%pDst+sizeof(qword)*3], rdx + + mov rdx, [%%pA+sizeof(qword)*2] + gsmulx rdx, rax, rdx + gsadcx rax, %%x3 + gsadox rax, %%x3 + mov qword [%%pDst+sizeof(qword)*4], rax + gsadcx rdx, %%x4 + gsadox rdx, %%x4 + mov qword [%%pDst+sizeof(qword)*5], rdx + + mov rdx, [%%pA+sizeof(qword)*3] + gsmulx rdx, rax, rdx + gsadcx rax, %%x5 + gsadox rax, %%x5 + mov qword [%%pDst+sizeof(qword)*6], rax + gsadcx rdx, %%A + gsadox rdx, %%A + mov qword [%%pDst+sizeof(qword)*7], rdx +%endmacro + +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_4,PRIVATE + SQR_256 rdi,rsi, r15,r14,r13,r12,r11,r10,r9,r8, rcx,rbx,rbp + ret +ENDFUNC sqr_4 + + + +;; +;; 5*qword squarer +;; +%macro SQR_320 13.nolist + %xdefine %%pDst %1 + %xdefine %%pA %2 + %xdefine %%x7 %3 + %xdefine %%x6 %4 + %xdefine %%x5 %5 + %xdefine %%x4 %6 + %xdefine %%x3 %7 + %xdefine %%x2 %8 + %xdefine %%x1 %9 + %xdefine %%x0 %10 + %xdefine %%A %11 + %xdefine %%x8 %12 + %xdefine %%t0 %13 + + ;; 1-st pass a[0]*{a[1],...,a[4]} + mov rdx, [%%pA] + gsmulx %%x1, %%x0, [%%pA+sizeof(qword)*1] + xor %%A, %%A + gsmulx %%x2, rax,[%%pA+sizeof(qword)*2] + add %%x1, rax + gsmulx %%x3, rax,[%%pA+sizeof(qword)*3] + adc %%x2, rax + gsmulx %%x4, rax,[%%pA+sizeof(qword)*4] + adc %%x3, rax + adc %%x4, 0 + + ;; 2-nd pass a[1]*{a[2],...,a[4]} + mov rdx, [%%pA+sizeof(qword)*1] + gsmulx %%t0, rax, [%%pA+sizeof(qword)*2] + gsadcx %%x2, rax + gsadox %%x3, %%t0 + gsmulx %%t0, rax, [%%pA+sizeof(qword)*3] + gsadcx %%x3, rax + gsadox %%x4, %%t0 + gsmulx %%x5, rax, [%%pA+sizeof(qword)*4] + gsadcx %%x4, rax + gsadox %%x5, %%A + adc %%x5, 0 + + ;; 3-rd pass a[2]*{a[3],a[4]} + mov rdx, [%%pA+sizeof(qword)*2] + gsmulx %%t0, rax, [%%pA+sizeof(qword)*3] + gsadcx %%x4, rax + gsadox %%x5, %%t0 + gsmulx %%x6, rax, [%%pA+sizeof(qword)*4] + gsadcx %%x5, rax + gsadox %%x6, %%A + adc %%x6, 0 + + ;; 4-th pass a[3]*a[4] + mov rdx, [%%pA+sizeof(qword)*3] + gsmulx %%x7, rax, [%%pA+sizeof(qword)*4] + add %%x6, rax + adc %%x7, 0 + + ;; square a[0],...,a[4] and add double x0,...,,x7 + mov rdx, [%%pA+sizeof(qword)*0] + gsmulx rdx, rax, rdx + mov qword [%%pDst], rax + gsadcx rdx, %%x0 + gsadox rdx, %%x0 + mov qword [%%pDst+sizeof(qword)], rdx + + mov rdx, [%%pA+sizeof(qword)*1] + gsmulx rdx, rax, rdx + gsadcx rax, %%x1 + gsadox rax, %%x1 + mov qword [%%pDst+sizeof(qword)*2], rax + gsadcx rdx, %%x2 + gsadox rdx, %%x2 + mov qword [%%pDst+sizeof(qword)*3], rdx + + mov rdx, [%%pA+sizeof(qword)*2] + gsmulx rdx, rax, rdx + gsadcx rax, %%x3 + gsadox rax, %%x3 + mov qword [%%pDst+sizeof(qword)*4], rax + gsadcx rdx, %%x4 + gsadox rdx, %%x4 + mov qword [%%pDst+sizeof(qword)*5], rdx + + mov rdx, [%%pA+sizeof(qword)*3] + gsmulx rdx, rax, rdx + gsadcx rax, %%x5 + gsadox rax, %%x5 + mov qword [%%pDst+sizeof(qword)*6], rax + gsadcx rdx, %%x6 + gsadox rdx, %%x6 + mov qword [%%pDst+sizeof(qword)*7], rdx + + mov rdx, [%%pA+sizeof(qword)*4] + gsmulx rdx, rax, rdx + gsadcx rax, %%x7 + gsadox rax, %%x7 + mov qword [%%pDst+sizeof(qword)*8], rax + gsadcx rdx, %%A + gsadox rdx, %%A + mov qword [%%pDst+sizeof(qword)*9], rdx +%endmacro + +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_5,PRIVATE + SQR_320 rdi,rsi, r15,r14,r13,r12,r11,r10,r9,r8, rcx,rbx,rbp + ret +ENDFUNC sqr_5 + + + +;; +;; 6*qword squarer +;; +%macro SQR_384 13.nolist + %xdefine %%pDst %1 + %xdefine %%pA %2 + %xdefine %%x7 %3 + %xdefine %%x6 %4 + %xdefine %%x5 %5 + %xdefine %%x4 %6 + %xdefine %%x3 %7 + %xdefine %%x2 %8 + %xdefine %%x1 %9 + %xdefine %%x0 %10 + %xdefine %%A %11 + %xdefine %%x8 %12 + %xdefine %%t0 %13 + + ;; 1-st pass a[0]*{a[1],...,a[5]} + mov rdx, [%%pA] + gsmulx %%x1, %%x0, [%%pA+sizeof(qword)*1] + xor %%A, %%A + gsmulx %%x2, rax,[%%pA+sizeof(qword)*2] + add %%x1, rax + gsmulx %%x3, rax,[%%pA+sizeof(qword)*3] + adc %%x2, rax + gsmulx %%x4, rax,[%%pA+sizeof(qword)*4] + adc %%x3, rax + gsmulx %%x5, rax,[%%pA+sizeof(qword)*5] + adc %%x4, rax + adc %%x5, 0 + + ;; 2-nd pass a[1]*{a[2],...,a[5]} + mov rdx, [%%pA+sizeof(qword)*1] + gsmulx %%t0, rax, [%%pA+sizeof(qword)*2] + gsadcx %%x2, rax + gsadox %%x3, %%t0 + gsmulx %%t0, rax, [%%pA+sizeof(qword)*3] + gsadcx %%x3, rax + gsadox %%x4, %%t0 + gsmulx %%t0, rax, [%%pA+sizeof(qword)*4] + gsadcx %%x4, rax + gsadox %%x5, %%t0 + gsmulx %%x6, rax, [%%pA+sizeof(qword)*5] + gsadcx %%x5, rax + gsadox %%x6, %%A + adc %%x6, 0 + + ;; 3-rd pass a[2]*{a[3],a[4],a[5]} + mov rdx, [%%pA+sizeof(qword)*2] + gsmulx %%t0, rax, [%%pA+sizeof(qword)*3] + gsadcx %%x4, rax + gsadox %%x5, %%t0 + gsmulx %%t0, rax, [%%pA+sizeof(qword)*4] + gsadcx %%x5, rax + gsadox %%x6, %%t0 + gsmulx %%x7, rax, [%%pA+sizeof(qword)*5] + gsadcx %%x6, rax + gsadox %%x7, %%A + adc %%x7, 0 + + ;; 4-th pass a[3]*{a[4],a[5]} + mov rdx, [%%pA+sizeof(qword)*3] + gsmulx %%t0, rax, [%%pA+sizeof(qword)*4] + gsadcx %%x6, rax + gsadox %%x7, %%t0 + gsmulx %%x8, rax, [%%pA+sizeof(qword)*5] + gsadcx %%x7, rax + gsadox %%x8, %%A + adc %%x8, 0 + + ;; 5-th pass a[4]*a[5] + mov rdx, [%%pA+sizeof(qword)*4] + gsmulx %%t0, rax, [%%pA+sizeof(qword)*5] + add %%x8, rax + adc %%t0, 0 + + ;; square a[0],...,a[5] and add double x0,...,x7,x8,t0 + mov rdx, [%%pA+sizeof(qword)*0] + gsmulx rdx, rax, rdx + mov qword [%%pDst], rax + gsadcx rdx, %%x0 + gsadox rdx, %%x0 + mov qword [%%pDst+sizeof(qword)], rdx + + mov rdx, [%%pA+sizeof(qword)*1] + gsmulx rdx, rax, rdx + gsadcx rax, %%x1 + gsadox rax, %%x1 + mov qword [%%pDst+sizeof(qword)*2], rax + gsadcx rdx, %%x2 + gsadox rdx, %%x2 + mov qword [%%pDst+sizeof(qword)*3], rdx + + mov rdx, [%%pA+sizeof(qword)*2] + gsmulx rdx, rax, rdx + gsadcx rax, %%x3 + gsadox rax, %%x3 + mov qword [%%pDst+sizeof(qword)*4], rax + gsadcx rdx, %%x4 + gsadox rdx, %%x4 + mov qword [%%pDst+sizeof(qword)*5], rdx + + mov rdx, [%%pA+sizeof(qword)*3] + gsmulx rdx, rax, rdx + gsadcx rax, %%x5 + gsadox rax, %%x5 + mov qword [%%pDst+sizeof(qword)*6], rax + gsadcx rdx, %%x6 + gsadox rdx, %%x6 + mov qword [%%pDst+sizeof(qword)*7], rdx + + mov rdx, [%%pA+sizeof(qword)*4] + gsmulx rdx, rax, rdx + gsadcx rax, %%x7 + gsadox rax, %%x7 + mov qword [%%pDst+sizeof(qword)*8], rax + gsadcx rdx, %%x8 + gsadox rdx, %%x8 + mov qword [%%pDst+sizeof(qword)*9], rdx + + mov rdx, [%%pA+sizeof(qword)*5] + gsmulx rdx, rax, rdx + gsadcx rax, %%t0 + gsadox rax, %%t0 + mov qword [%%pDst+sizeof(qword)*10], rax + gsadcx rdx, %%A + gsadox rdx, %%A + mov qword [%%pDst+sizeof(qword)*11], rdx +%endmacro + +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_6,PRIVATE + SQR_384 rdi,rsi, r15,r14,r13,r12,r11,r10,r9,r8, rcx,rbx,rbp + ret +ENDFUNC sqr_6 + + + +;; +;; 7*qword squarer +;; +%macro SQR_448 13.nolist + %xdefine %%pDst %1 + %xdefine %%pA %2 + %xdefine %%x7 %3 + %xdefine %%x6 %4 + %xdefine %%x5 %5 + %xdefine %%x4 %6 + %xdefine %%x3 %7 + %xdefine %%x2 %8 + %xdefine %%x1 %9 + %xdefine %%x0 %10 + %xdefine %%A %11 + %xdefine %%x8 %12 + %xdefine %%t0 %13 + + ;; 1-st pass a[0]*{a[1],...,a[6]} + mov rdx, [%%pA] + gsmulx %%x1, %%x0, [%%pA+sizeof(qword)*1] + xor %%A, %%A + gsmulx %%x2, rax,[%%pA+sizeof(qword)*2] + add %%x1, rax + gsmulx %%x3, rax,[%%pA+sizeof(qword)*3] + adc %%x2, rax + gsmulx %%x4, rax,[%%pA+sizeof(qword)*4] + adc %%x3, rax + gsmulx %%x5, rax,[%%pA+sizeof(qword)*5] + adc %%x4, rax + gsmulx %%x6, rax,[%%pA+sizeof(qword)*6] + adc %%x5, rax + adc %%x6, 0 + + mov [%%pDst+sizeof(qword)*1], %%x0 + mov [%%pDst+sizeof(qword)*2], %%x1 + + ;; 2-nd pass a[1]*{a[2],...,a[6]} + mov rdx, [%%pA+sizeof(qword)*1] + gsmulx %%t0, rax, [%%pA+sizeof(qword)*2] + gsadcx %%x2, rax + gsadox %%x3, %%t0 + gsmulx %%t0, rax, [%%pA+sizeof(qword)*3] + gsadcx %%x3, rax + gsadox %%x4, %%t0 + gsmulx %%t0, rax, [%%pA+sizeof(qword)*4] + gsadcx %%x4, rax + gsadox %%x5, %%t0 + gsmulx %%t0, rax, [%%pA+sizeof(qword)*5] + gsadcx %%x5, rax + gsadox %%x6, %%t0 + gsmulx %%x7, rax, [%%pA+sizeof(qword)*6] + gsadcx %%x6, rax + gsadox %%x7, %%A + adc %%x7, 0 + + mov [%%pDst+sizeof(qword)*3], %%x2 + + ;; 3-rd pass a[2]*{a[3],...,a[6]} + mov rdx, [%%pA+sizeof(qword)*2] + gsmulx %%t0, rax, [%%pA+sizeof(qword)*3] + gsadcx %%x4, rax + gsadox %%x5, %%t0 + gsmulx %%t0, rax, [%%pA+sizeof(qword)*4] + gsadcx %%x5, rax + gsadox %%x6, %%t0 + gsmulx %%t0, rax, [%%pA+sizeof(qword)*5] + gsadcx %%x6, rax + gsadox %%x7, %%t0 + gsmulx %%x8, rax, [%%pA+sizeof(qword)*6] + gsadcx %%x7, rax + gsadox %%x8, %%A + adc %%x8, 0 + + ;; 4-rd pass a[3]*{a[4],...,a[6]} + mov rdx, [%%pA+sizeof(qword)*3] + gsmulx %%t0, rax, [%%pA+sizeof(qword)*4] + gsadcx %%x6, rax + gsadox %%x7, %%t0 + gsmulx %%t0, rax, [%%pA+sizeof(qword)*5] + gsadcx %%x7, rax + gsadox %%x8, %%t0 + gsmulx %%x0, rax, [%%pA+sizeof(qword)*6] + gsadcx %%x8, rax + gsadox %%x0, %%A + adc %%x0, 0 + + ;; 5-rd pass a[4]*{a[5],a[6]} + mov rdx, [%%pA+sizeof(qword)*4] + gsmulx %%t0, rax, [%%pA+sizeof(qword)*5] + gsadcx %%x8, rax + gsadox %%x0, %%t0 + gsmulx %%x1, rax, [%%pA+sizeof(qword)*6] + gsadcx %%x0, rax + gsadox %%x1, %%A + adc %%x1, 0 + + ;; 6-rd pass a[5]*a[6] + mov rdx, [%%pA+sizeof(qword)*5] + gsmulx %%x2, rax, [%%pA+sizeof(qword)*6] + add %%x1, rax + adc %%x2, 0 + + ;; square a[0],...,a[5] and add double + mov rdx, [%%pA+sizeof(qword)*0] + gsmulx rdx, rax, rdx + mov qword [%%pDst], rax + gsadcx rdx, qword [%%pDst+sizeof(qword)] + gsadox rdx, qword [%%pDst+sizeof(qword)] + mov qword [%%pDst+sizeof(qword)], rdx + + mov rdx, [%%pA+sizeof(qword)*1] + gsmulx rdx, rax, rdx + gsadcx rax, qword [%%pDst+sizeof(qword)*2] + gsadox rax, qword [%%pDst+sizeof(qword)*2] + mov qword [%%pDst+sizeof(qword)*2], rax + gsadcx rdx, qword [%%pDst+sizeof(qword)*3] + gsadox rdx, qword [%%pDst+sizeof(qword)*3] + mov qword [%%pDst+sizeof(qword)*3], rdx + + mov rdx, [%%pA+sizeof(qword)*2] + gsmulx rdx, rax, rdx + gsadcx rax, %%x3 + gsadox rax, %%x3 + mov qword [%%pDst+sizeof(qword)*4], rax + gsadcx rdx, %%x4 + gsadox rdx, %%x4 + mov qword [%%pDst+sizeof(qword)*5], rdx + + mov rdx, [%%pA+sizeof(qword)*3] + gsmulx rdx, rax, rdx + gsadcx rax, %%x5 + gsadox rax, %%x5 + mov qword [%%pDst+sizeof(qword)*6], rax + gsadcx rdx, %%x6 + gsadox rdx, %%x6 + mov qword [%%pDst+sizeof(qword)*7], rdx + + mov rdx, [%%pA+sizeof(qword)*4] + gsmulx rdx, rax, rdx + gsadcx rax, %%x7 + gsadox rax, %%x7 + mov qword [%%pDst+sizeof(qword)*8], rax + gsadcx rdx, %%x8 + gsadox rdx, %%x8 + mov qword [%%pDst+sizeof(qword)*9], rdx + + mov rdx, [%%pA+sizeof(qword)*5] + gsmulx rdx, rax, rdx + gsadcx rax, %%x0 + gsadox rax, %%x0 + mov qword [%%pDst+sizeof(qword)*10], rax + gsadcx rdx, %%x1 + gsadox rdx, %%x1 + mov qword [%%pDst+sizeof(qword)*11], rdx + + mov rdx, [%%pA+sizeof(qword)*6] + gsmulx rdx, rax, rdx + gsadcx rax, %%x2 + gsadox rax, %%x2 + mov qword [%%pDst+sizeof(qword)*12], rax + gsadcx rdx, %%A + gsadox rdx, %%A + mov qword [%%pDst+sizeof(qword)*13], rdx +%endmacro + +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_7,PRIVATE + SQR_448 rdi,rsi, r15,r14,r13,r12,r11,r10,r9,r8, rcx,rbx,rbp + ret +ENDFUNC sqr_7 + + + +;; +;; 8*qword squarer +;; +%macro SQR_512 13.nolist + %xdefine %%pDst %1 + %xdefine %%pA %2 + %xdefine %%x7 %3 + %xdefine %%x6 %4 + %xdefine %%x5 %5 + %xdefine %%x4 %6 + %xdefine %%x3 %7 + %xdefine %%x2 %8 + %xdefine %%x1 %9 + %xdefine %%x0 %10 + %xdefine %%A %11 + %xdefine %%x8 %12 + %xdefine %%t0 %13 + + ;; 1-st pass a[0]*{a[1],...,a[7]} + mov rdx, [%%pA] + gsmulx %%x1, %%x0, [%%pA+sizeof(qword)*1] + xor %%A, %%A + gsmulx %%x2, rax,[%%pA+sizeof(qword)*2] + add %%x1, rax + gsmulx %%x3, rax,[%%pA+sizeof(qword)*3] + adc %%x2, rax + gsmulx %%x4, rax,[%%pA+sizeof(qword)*4] + adc %%x3, rax + gsmulx %%x5, rax,[%%pA+sizeof(qword)*5] + adc %%x4, rax + gsmulx %%x6, rax,[%%pA+sizeof(qword)*6] + adc %%x5, rax + gsmulx %%x7, rax,[%%pA+sizeof(qword)*7] + adc %%x6, rax + adc %%x7, 0 + + mov [%%pDst+sizeof(qword)*1], %%x0 + mov [%%pDst+sizeof(qword)*2], %%x1 + + ;; 2-nd pass a[1]*{a[2],...,a[7]} + mov rdx, [%%pA+sizeof(qword)*1] + gsmulx %%t0, rax, [%%pA+sizeof(qword)*2] + gsadcx %%x2, rax + gsadox %%x3, %%t0 + gsmulx %%t0, rax, [%%pA+sizeof(qword)*3] + gsadcx %%x3, rax + gsadox %%x4, %%t0 + gsmulx %%t0, rax, [%%pA+sizeof(qword)*4] + gsadcx %%x4, rax + gsadox %%x5, %%t0 + gsmulx %%t0, rax, [%%pA+sizeof(qword)*5] + gsadcx %%x5, rax + gsadox %%x6, %%t0 + gsmulx %%t0, rax, [%%pA+sizeof(qword)*6] + gsadcx %%x6, rax + gsadox %%x7, %%t0 + gsmulx %%x8, rax, [%%pA+sizeof(qword)*7] + gsadcx %%x7, rax + gsadox %%x8, %%A + adc %%x8, 0 + + mov [%%pDst+sizeof(qword)*3], %%x2 + mov [%%pDst+sizeof(qword)*4], %%x3 + + ;; 3-rd pass a[2]*{a[3],...,a[7]} + mov rdx, [%%pA+sizeof(qword)*2] + gsmulx %%t0, rax, [%%pA+sizeof(qword)*3] + gsadcx %%x4, rax + gsadox %%x5, %%t0 + gsmulx %%t0, rax, [%%pA+sizeof(qword)*4] + gsadcx %%x5, rax + gsadox %%x6, %%t0 + gsmulx %%t0, rax, [%%pA+sizeof(qword)*5] + gsadcx %%x6, rax + gsadox %%x7, %%t0 + gsmulx %%t0, rax, [%%pA+sizeof(qword)*6] + gsadcx %%x7, rax + gsadox %%x8, %%t0 + gsmulx %%x0, rax, [%%pA+sizeof(qword)*7] + gsadcx %%x8, rax + gsadox %%x0, %%A + adc %%x0, 0 + + ;; 4-rd pass a[3]*{a[4],...,a[7]} + mov rdx, [%%pA+sizeof(qword)*3] + gsmulx %%t0, rax, [%%pA+sizeof(qword)*4] + gsadcx %%x6, rax + gsadox %%x7, %%t0 + gsmulx %%t0, rax, [%%pA+sizeof(qword)*5] + gsadcx %%x7, rax + gsadox %%x8, %%t0 + gsmulx %%t0, rax, [%%pA+sizeof(qword)*6] + gsadcx %%x8, rax + gsadox %%x0, %%t0 + gsmulx %%x1, rax, [%%pA+sizeof(qword)*7] + gsadcx %%x0, rax + gsadox %%x1, %%A + adc %%x1, 0 + + ;; 5-rd pass a[4]*{a[5],...,a[7]} + mov rdx, [%%pA+sizeof(qword)*4] + gsmulx %%t0, rax, [%%pA+sizeof(qword)*5] + gsadcx %%x8, rax + gsadox %%x0, %%t0 + gsmulx %%t0, rax, [%%pA+sizeof(qword)*6] + gsadcx %%x0, rax + gsadox %%x1, %%t0 + gsmulx %%x2, rax, [%%pA+sizeof(qword)*7] + gsadcx %%x1, rax + gsadox %%x2, %%A + adc %%x2, 0 + + ;; 6-rd pass a[5]*{a[6],a[7]} + mov rdx, [%%pA+sizeof(qword)*5] + gsmulx %%t0, rax, [%%pA+sizeof(qword)*6] + gsadcx %%x1, rax + gsadox %%x2, %%t0 + gsmulx %%x3, rax, [%%pA+sizeof(qword)*7] + gsadcx %%x2, rax + gsadox %%x3, %%A + adc %%x3, 0 + + ;; 7-rd pass a[6]*a[7] + mov rdx, [%%pA+sizeof(qword)*6] + gsmulx %%t0, rax, [%%pA+sizeof(qword)*7] + add %%x3, rax + adc %%t0, 0 + + ;; square a[0],...,a[7] and add double + mov rdx, [%%pA+sizeof(qword)*0] + gsmulx rdx, rax, rdx + mov qword [%%pDst], rax + gsadcx rdx, qword [%%pDst+sizeof(qword)] + gsadox rdx, qword [%%pDst+sizeof(qword)] + mov qword [%%pDst+sizeof(qword)], rdx + + mov rdx, [%%pA+sizeof(qword)*1] + gsmulx rdx, rax, rdx + gsadcx rax, qword [%%pDst+sizeof(qword)*2] + gsadox rax, qword [%%pDst+sizeof(qword)*2] + mov qword [%%pDst+sizeof(qword)*2], rax + gsadcx rdx, qword [%%pDst+sizeof(qword)*3] + gsadox rdx, qword [%%pDst+sizeof(qword)*3] + mov qword [%%pDst+sizeof(qword)*3], rdx + + mov rdx, [%%pA+sizeof(qword)*2] + gsmulx rdx, rax, rdx + gsadcx rax, qword [%%pDst+sizeof(qword)*4] + gsadox rax, qword [%%pDst+sizeof(qword)*4] + mov qword [%%pDst+sizeof(qword)*4], rax + gsadcx rdx, %%x4 + gsadox rdx, %%x4 + mov qword [%%pDst+sizeof(qword)*5], rdx + + mov rdx, [%%pA+sizeof(qword)*3] + gsmulx rdx, rax, rdx + gsadcx rax, %%x5 + gsadox rax, %%x5 + mov qword [%%pDst+sizeof(qword)*6], rax + gsadcx rdx, %%x6 + gsadox rdx, %%x6 + mov qword [%%pDst+sizeof(qword)*7], rdx + + mov rdx, [%%pA+sizeof(qword)*4] + gsmulx rdx, rax, rdx + gsadcx rax, %%x7 + gsadox rax, %%x7 + mov qword [%%pDst+sizeof(qword)*8], rax + gsadcx rdx, %%x8 + gsadox rdx, %%x8 + mov qword [%%pDst+sizeof(qword)*9], rdx + + mov rdx, [%%pA+sizeof(qword)*5] + gsmulx rdx, rax, rdx + gsadcx rax, %%x0 + gsadox rax, %%x0 + mov qword [%%pDst+sizeof(qword)*10], rax + gsadcx rdx, %%x1 + gsadox rdx, %%x1 + mov qword [%%pDst+sizeof(qword)*11], rdx + + mov rdx, [%%pA+sizeof(qword)*6] + gsmulx rdx, rax, rdx + gsadcx rax, %%x2 + gsadox rax, %%x2 + mov qword [%%pDst+sizeof(qword)*12], rax + gsadcx rdx, %%x3 + gsadox rdx, %%x3 + mov qword [%%pDst+sizeof(qword)*13], rdx + + mov rdx, [%%pA+sizeof(qword)*7] + gsmulx rdx, rax, rdx + gsadcx rax, %%t0 + gsadox rax, %%t0 + mov qword [%%pDst+sizeof(qword)*14], rax + gsadcx rdx, %%A + gsadox rdx, %%A + mov qword [%%pDst+sizeof(qword)*15], rdx +%endmacro + +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_8,PRIVATE + SQR_512 rdi, rsi, r15,r14,r13,r12,r11,r10,r9,r8, rcx, rbx, rbp + ret +ENDFUNC sqr_8 + + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; SQR_512_TRIANGLE_STEP +; executes single line of upper triangle +; +; Inp:partial sum: x7 x6 x5 x4 x3 x2 x1 x0 +; rdx * pA[] p p p p p p p p p +; Out: x0 x7 x6 x5 x4 x3 x2 x1 +; [dst] + +%macro SQR_512_TRIANGLE_STEP 14.nolist + %xdefine %%HEAD_X %1 + %xdefine %%X7 %2 + %xdefine %%X6 %3 + %xdefine %%X5 %4 + %xdefine %%X4 %5 + %xdefine %%X3 %6 + %xdefine %%X2 %7 + %xdefine %%X1 %8 + %xdefine %%X0 %9 + %xdefine %%TAIL_X %10 + %xdefine %%pDst %11 + %xdefine %%pA %12 + %xdefine %%TMP1 %13 + %xdefine %%TMP2 %14 + + xor rax, rax + +%ifnempty %%X0 +gsmulx %%TMP2,%%TMP1, [%%pA+sizeof(qword)*0] ; TMP2:TMP1 = rdx * pA[0] +gsadcx %%X0, %%TMP1 +gsadox %%X1, %%TMP2 +%endif + +%ifnempty %%TAIL_X + mov %%pDst, %%TAIL_X +%endif + +%ifnempty %%X1 +gsmulx %%TMP2,%%TMP1, [%%pA+sizeof(qword)*1] ; TMP2:TMP1 = rdx * pA[1] +gsadcx %%X1, %%TMP1 +gsadox %%X2, %%TMP2 +%endif + +%ifnempty %%X2 +gsmulx %%TMP2,%%TMP1, [%%pA+sizeof(qword)*2] ; TMP2:TMP1 = rdx * pA[2] +gsadcx %%X2, %%TMP1 +gsadox %%X3, %%TMP2 +%endif + +%ifnempty %%X3 +gsmulx %%TMP2,%%TMP1, [%%pA+sizeof(qword)*3] ; TMP2:TMP1 = rdx * pA[3] +gsadcx %%X3, %%TMP1 +gsadox %%X4, %%TMP2 +%endif + +%ifnempty %%X4 +gsmulx %%TMP2,%%TMP1, [%%pA+sizeof(qword)*4] ; TMP2:TMP1 = rdx * pA[4] +gsadcx %%X4, %%TMP1 +gsadox %%X5, %%TMP2 +%endif + +%ifnempty %%X5 +gsmulx %%TMP2,%%TMP1, [%%pA+sizeof(qword)*5] ; TMP2:TMP1 = rdx * pA[5] +gsadcx %%X5, %%TMP1 +gsadox %%X6, %%TMP2 +%endif + +%ifnempty %%X6 +gsmulx %%TMP2,%%TMP1, [%%pA+sizeof(qword)*6] ; TMP2:TMP1 = rdx * pA[6] +gsadcx %%X6, %%TMP1 +gsadox %%X7, %%TMP2 +%endif + +%ifnempty %%X7 +gsmulx %%HEAD_X,%%TMP1, [%%pA+sizeof(qword)*7] ; X0:TMP1 = rdx * pA[7] +gsadcx %%X7, %%TMP1 +gsadox %%HEAD_X, rax + adc %%HEAD_X, 0 +%endif +%endmacro + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; square and add diagonal terms +; + +;; rdi = dst +;; rsi = src +;; rcx = src lengh +%macro FINALIZE_FIX 3.nolist + %xdefine %%N %1 + %xdefine %%pDst %2 + %xdefine %%pSrc %3 + + xor rax, rax + mov rdx, qword [%%pSrc] + gsmulx rdx, rax, rdx + mov qword [%%pDst], rax + gsadcx rdx, qword [%%pDst+sizeof(qword)] + gsadox rdx, qword [%%pDst+sizeof(qword)] + mov qword [rdi+sizeof(qword)], rdx + + %assign %%i 1 + %rep (%%N-2) + mov rdx, qword [%%pSrc+%%i*sizeof(qword)] + gsmulx rdx, rax, rdx + gsadcx rax, qword [%%pDst+(2*%%i)*sizeof(qword)] + gsadox rax, qword [%%pDst+(2*%%i)*sizeof(qword)] + mov qword [%%pDst+(2*%%i)*sizeof(qword)], rax + gsadcx rdx, qword [%%pDst+(2*%%i+1)*sizeof(qword)] + gsadox rdx, qword [%%pDst+(2*%%i+1)*sizeof(qword)] + mov qword [rdi+(2*%%i+1)*sizeof(qword)], rdx + %assign %%i %%i+1 + %endrep + + mov rdx, qword [%%pSrc+(%%N-1)*sizeof(qword)] + gsmulx rdx, rax, rdx + gsadcx rax, qword [%%pDst+(2*%%N-2)*sizeof(qword)] + gsadox rax, qword [%%pDst+(2*%%N-2)*sizeof(qword)] + mov qword [%%pDst+(2*%%N-2)*sizeof(qword)], rax + mov rax, dword 0 + gsadcx rdx, rax + gsadox rdx, rax + mov qword [rdi+(2*%%N-1)*sizeof(qword)], rdx +%endmacro + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC finalize,PRIVATE + push rcx + xor rax, rax + + mov rdx, qword [rsi] + gsmulx rdx, rax, rdx + lea rsi, [rsi+sizeof(qword)] + mov qword [rdi], rax + gsadcx rdx, qword [rdi+sizeof(qword)] + gsadox rdx, qword [rdi+sizeof(qword)] + mov qword [rdi+sizeof(qword)], rdx + lea rdi, [rdi+sizeof(qword)*2] + lea rcx, [rcx-2] + +.next_sqr: + mov rdx, qword [rsi] + gsmulx rdx, rax, rdx + lea rsi, [rsi+sizeof(qword)] + gsadcx rax, qword [rdi] + gsadox rax, qword [rdi] + mov qword [rdi], rax + gsadcx rdx, qword [rdi+sizeof(qword)] + gsadox rdx, qword [rdi+sizeof(qword)] + mov qword [rdi+sizeof(qword)], rdx + lea rdi, [rdi+sizeof(qword)*2] + lea rcx, [rcx-1] + jrcxz .last_sqr + jmp .next_sqr + +.last_sqr: + mov rdx, qword [rsi] + gsmulx rdx, rax, rdx + gsadcx rax, qword [rdi] + gsadox rax, qword [rdi] + mov qword [rdi], rax + mov rax, dword 0 + gsadcx rdx, rax + gsadox rdx, rax + mov qword [rdi+sizeof(qword)], rdx + + pop rcx + lea rax, [rcx*sizeof(qword)-sizeof(qword)] + sub rsi, rax + sub rdi, rax + sub rdi, rax + ret +ENDFUNC finalize + + +;; +;; 8*qword triangle +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr8_triangle,PRIVATE + ;; A[0]*A[1..7] + mov rdx, [rsi+sizeof(qword)*0] + SQR_512_TRIANGLE_STEP r8, r15,r14,r13,r12,r11,r10, r9, , r8, [rdi+sizeof(qword)*0],rsi, rbx,rbp + + ;; A[1]*A[2..7] + mov rdx, [rsi+sizeof(qword)*1] + SQR_512_TRIANGLE_STEP r9, r8,r15,r14,r13,r12,r11, , , r9, [rdi+sizeof(qword)*1],rsi, rbx,rbp + + ;; A[2]*A[3..7] + mov rdx, [rsi+sizeof(qword)*2] + SQR_512_TRIANGLE_STEP r10, r9, r8,r15,r14,r13, , , , r10, [rdi+sizeof(qword)*2],rsi, rbx,rbp + + ;; A[3]*A[4..7] + mov rdx, [rsi+sizeof(qword)*3] + SQR_512_TRIANGLE_STEP r11, r10, r9, r8,r15, , , , , r11, [rdi+sizeof(qword)*3],rsi, rbx,rbp + + ;; A[4]*A[5..7] + mov rdx, [rsi+sizeof(qword)*4] + SQR_512_TRIANGLE_STEP r12, r11,r10, r9, , , , , , r12, [rdi+sizeof(qword)*4],rsi, rbx,rbp + + ;; A[5]*A[6..7] + mov rdx, [rsi+sizeof(qword)*5] + SQR_512_TRIANGLE_STEP r13, r12,r11, , , , , , , r13, [rdi+sizeof(qword)*5],rsi, rbx,rbp + + ;; A[6]*A[7] + mov rdx, [rsi+sizeof(qword)*6] + SQR_512_TRIANGLE_STEP r14, r13, , , , , , , , r14, [rdi+sizeof(qword)*6],rsi, rbx,rbp + ret +ENDFUNC sqr8_triangle + + + +;; +;; 9*qword squarer +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_9,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + xor r15, r15 + call mla_8x1 + + mov qword [rdi+sizeof(qword)*(1+0)], r8 + mov qword [rdi+sizeof(qword)*(1+1)], r9 + mov qword [rdi+sizeof(qword)*(1+2)], r10 + mov qword [rdi+sizeof(qword)*(1+3)], r11 + mov qword [rdi+sizeof(qword)*(1+4)], r12 + mov qword [rdi+sizeof(qword)*(1+5)], r13 + mov qword [rdi+sizeof(qword)*(1+6)], r14 + mov qword [rdi+sizeof(qword)*(1+7)], r15 + + xor rbx, rbx + mov qword [rdi+sizeof(qword)*(1+8)], rbx + + sub rdi, sizeof(qword)*8 + + FINALIZE_FIX 9, rdi, rsi + ret +ENDFUNC sqr_9 + + +;; +;; 10*qword squarer +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_10,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + xor r15, r15 + call mla_8x2 + + mov qword [rdi+sizeof(qword)*(2+0)], r8 + mov qword [rdi+sizeof(qword)*(2+1)], r9 + mov qword [rdi+sizeof(qword)*(2+2)], r10 + mov qword [rdi+sizeof(qword)*(2+3)], r11 + mov qword [rdi+sizeof(qword)*(2+4)], r12 + mov qword [rdi+sizeof(qword)*(2+5)], r13 + mov qword [rdi+sizeof(qword)*(2+6)], r14 + + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15, , , , , , , , , , {rsi+sizeof(qword)*2}, rbx,rbp + + xor rbx, rbx + mov qword [rdi+sizeof(qword)*(2+7)], r15 + mov qword [rdi+sizeof(qword)*(2+8)], r8 + mov qword [rdi+sizeof(qword)*(2+9)], rbx + + sub rdi, sizeof(qword)*8 + + FINALIZE_FIX 10, rdi, rsi + ret +ENDFUNC sqr_10 + + +;; +;; 10*qword squarer +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_11,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + xor r15, r15 + call mla_8x1 + add rdi, sizeof(qword)*1 + add rcx, sizeof(qword)*1 + call mla_8x2 + sub rdi, sizeof(qword)*1 + sub rcx, sizeof(qword)*1 + + mov qword [rdi+sizeof(qword)*(3+0)], r8 + mov qword [rdi+sizeof(qword)*(3+1)], r9 + mov qword [rdi+sizeof(qword)*(3+2)], r10 + mov qword [rdi+sizeof(qword)*(3+3)], r11 + mov qword [rdi+sizeof(qword)*(3+4)], r12 + + +; SQR_512_TRIANGLE_STEP H, p7, p6, p5, p4, p3, p2, p1, p0, T, ,, rbx,rbp +; ------------------------------ + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15,r14, , , , , , , r13,[rdi+sizeof(qword)*(3+5)],{rsi+sizeof(qword)*3}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*9] + SQR_512_TRIANGLE_STEP r9, r8, , , , , , , , r14,[rdi+sizeof(qword)*(3+6)],{rsi+sizeof(qword)*3}, rbx,rbp + + xor rbx, rbx + mov qword [rdi+sizeof(qword)*(3+7)], r15 + mov qword [rdi+sizeof(qword)*(3+8)], r8 + mov qword [rdi+sizeof(qword)*(3+9)], r9 + mov qword [rdi+sizeof(qword)*(3+10)],rbx + + sub rdi, sizeof(qword)*8 + + FINALIZE_FIX 11, rdi, rsi + ret +ENDFUNC sqr_11 + + +;; +;; 12*qword squarer +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_12,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + xor r15, r15 + call mla_8x2 + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x2 + sub rdi, sizeof(qword)*2 + sub rcx, sizeof(qword)*2 + + mov qword [rdi+sizeof(qword)*(4+0)], r8 + mov qword [rdi+sizeof(qword)*(4+1)], r9 + mov qword [rdi+sizeof(qword)*(4+2)], r10 + mov qword [rdi+sizeof(qword)*(4+3)], r11 + +; SQR_512_TRIANGLE_STEP H, p7, p6, p5, p4, p3, p2, p1, p0, T, ,, rbx,rbp +; ------------------------------ + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15,r14,r13, , , , , , r12,[rdi+sizeof(qword)*(4+4)],{rsi+sizeof(qword)*4}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*9] + SQR_512_TRIANGLE_STEP r9, r8,r15, , , , , , , r13,[rdi+sizeof(qword)*(4+5)],{rsi+sizeof(qword)*4}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*10] + SQR_512_TRIANGLE_STEP r10, r9, , , , , , , , r14,[rdi+sizeof(qword)*(4+6)],{rsi+sizeof(qword)*4}, rbx,rbp + + xor rbx, rbx + mov qword [rdi+sizeof(qword)*(4+7)], r15 + mov qword [rdi+sizeof(qword)*(4+8)], r8 + mov qword [rdi+sizeof(qword)*(4+9)], r9 + mov qword [rdi+sizeof(qword)*(4+10)],r10 + mov qword [rdi+sizeof(qword)*(4+11)],rbx + + sub rdi, sizeof(qword)*8 + + FINALIZE_FIX 12, rdi, rsi + ret +ENDFUNC sqr_12 + + +;; +;; 13*qword squarer +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_13,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + xor r15, r15 + call mla_8x1 + add rdi, sizeof(qword) + add rcx, sizeof(qword) + call mla_8x2 + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x2 + sub rdi, sizeof(qword)*3 + sub rcx, sizeof(qword)*3 + + mov qword [rdi+sizeof(qword)*(5+0)], r8 + mov qword [rdi+sizeof(qword)*(5+1)], r9 + mov qword [rdi+sizeof(qword)*(5+2)], r10 + +; SQR_512_TRIANGLE_STEP H, p7, p6, p5, p4, p3, p2, p1, p0, T, ,, rbx,rbp +; ------------------------------ + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15,r14,r13,r12, , , , , r11,[rdi+sizeof(qword)*(5+3)],{rsi+sizeof(qword)*5}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*9] + SQR_512_TRIANGLE_STEP r9, r8,r15,r14, , , , , , r12,[rdi+sizeof(qword)*(5+4)],{rsi+sizeof(qword)*5}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*10] + SQR_512_TRIANGLE_STEP r10, r9, r8, , , , , , , r13,[rdi+sizeof(qword)*(5+5)],{rsi+sizeof(qword)*5}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*11] + SQR_512_TRIANGLE_STEP r11, r10, , , , , , , , r14,[rdi+sizeof(qword)*(5+6)],{rsi+sizeof(qword)*5}, rbx,rbp + + xor rbx, rbx + mov qword [rdi+sizeof(qword)*(5+7)], r15 + mov qword [rdi+sizeof(qword)*(5+8)], r8 + mov qword [rdi+sizeof(qword)*(5+9)], r9 + mov qword [rdi+sizeof(qword)*(5+10)],r10 + mov qword [rdi+sizeof(qword)*(5+11)],r11 + mov qword [rdi+sizeof(qword)*(5+12)],rbx + + sub rdi, sizeof(qword)*8 + + FINALIZE_FIX 13, rdi, rsi + ret +ENDFUNC sqr_13 + + +;; +;; 14*qword squarer +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_14,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + xor r15, r15 + call mla_8x2 + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x2 + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x2 + sub rdi, sizeof(qword)*4 + sub rcx, sizeof(qword)*4 + + mov qword [rdi+sizeof(qword)*(6+0)], r8 + mov qword [rdi+sizeof(qword)*(6+1)], r9 + +; SQR_512_TRIANGLE_STEP H, p7, p6, p5, p4, p3, p2, p1, p0, T, ,, rbx,rbp +; ------------------------------ + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15,r14,r13,r12,r11, , , , r10,[rdi+sizeof(qword)*(6+2)],{rsi+sizeof(qword)*6}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*9] + SQR_512_TRIANGLE_STEP r9, r8,r15,r14,r13, , , , , r11,[rdi+sizeof(qword)*(6+3)],{rsi+sizeof(qword)*6}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*10] + SQR_512_TRIANGLE_STEP r10, r9, r8,r15, , , , , , r12,[rdi+sizeof(qword)*(6+4)],{rsi+sizeof(qword)*6}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*11] + SQR_512_TRIANGLE_STEP r11, r10, r9, , , , , , , r13,[rdi+sizeof(qword)*(6+5)],{rsi+sizeof(qword)*6}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*12] + SQR_512_TRIANGLE_STEP r12, r11, , , , , , , , r14,[rdi+sizeof(qword)*(6+6)],{rsi+sizeof(qword)*6}, rbx,rbp + + xor rbx, rbx + mov qword [rdi+sizeof(qword)*(6+7)], r15 + mov qword [rdi+sizeof(qword)*(6+8)], r8 + mov qword [rdi+sizeof(qword)*(6+9)], r9 + mov qword [rdi+sizeof(qword)*(6+10)],r10 + mov qword [rdi+sizeof(qword)*(6+11)],r11 + mov qword [rdi+sizeof(qword)*(6+12)],r12 + mov qword [rdi+sizeof(qword)*(6+13)],rbx + + sub rdi, sizeof(qword)*8 + + FINALIZE_FIX 14, rdi, rsi + ret +ENDFUNC sqr_14 + + +;; +;; 15*qword squarer +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_15,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + xor r15, r15 + call mla_8x1 + add rdi, sizeof(qword) + add rcx, sizeof(qword) + call mla_8x2 + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x2 + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x2 + sub rdi, sizeof(qword)*5 + sub rcx, sizeof(qword)*5 + + mov qword [rdi+sizeof(qword)*(7+0)], r8 + +; SQR_512_TRIANGLE_STEP H, p7, p6, p5, p4, p3, p2, p1, p0, T, ,, rbx,rbp +; ------------------------------ + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15,r14,r13,r12,r11,r10, , , r9, [rdi+sizeof(qword)*(7+1)],{rsi+sizeof(qword)*7}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*9] + SQR_512_TRIANGLE_STEP r9, r8,r15,r14,r13,r12, , , , r10,[rdi+sizeof(qword)*(7+2)],{rsi+sizeof(qword)*7}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*10] + SQR_512_TRIANGLE_STEP r10, r9, r8,r15,r14, , , , , r11,[rdi+sizeof(qword)*(7+3)],{rsi+sizeof(qword)*7}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*11] + SQR_512_TRIANGLE_STEP r11, r10, r9, r8, , , , , , r12,[rdi+sizeof(qword)*(7+4)],{rsi+sizeof(qword)*7}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*12] + SQR_512_TRIANGLE_STEP r12, r11,r10, , , , , , , r13,[rdi+sizeof(qword)*(7+5)],{rsi+sizeof(qword)*7}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*13] + SQR_512_TRIANGLE_STEP r13, r12, , , , , , , , r14,[rdi+sizeof(qword)*(7+6)],{rsi+sizeof(qword)*7}, rbx,rbp + + xor rbx, rbx + mov qword [rdi+sizeof(qword)*(7+7)], r15 + mov qword [rdi+sizeof(qword)*(7+8)], r8 + mov qword [rdi+sizeof(qword)*(7+9)], r9 + mov qword [rdi+sizeof(qword)*(7+10)],r10 + mov qword [rdi+sizeof(qword)*(7+11)],r11 + mov qword [rdi+sizeof(qword)*(7+12)],r12 + mov qword [rdi+sizeof(qword)*(7+13)],r13 + mov qword [rdi+sizeof(qword)*(7+14)],rbx + + sub rdi, sizeof(qword)*8 + + FINALIZE_FIX 15, rdi, rsi + ret +ENDFUNC sqr_15 + + +;; +;; 16*qword squarer +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr_16,PRIVATE + call sqr8_triangle + + mov qword [rdi+sizeof(qword)*7], r15 + + mov rcx, rsi + add rsi, sizeof(qword)*8 + add rdi, sizeof(qword)*8 + xor r15, r15 + call mla_8x2 + + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x2 + + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x2 + + add rdi, sizeof(qword)*2 + add rcx, sizeof(qword)*2 + call mla_8x2 + + sub rdi, sizeof(qword)*6 + sub rcx, sizeof(qword)*6 + + add rdi, sizeof(qword)*8 + call sqr8_triangle + + xor rbx, rbx + mov qword [rdi+sizeof(qword)*7], r15 + mov qword [rdi+sizeof(qword)*8], r8 + mov qword [rdi+sizeof(qword)*9], r9 + mov qword [rdi+sizeof(qword)*10],r10 + mov qword [rdi+sizeof(qword)*11],r11 + mov qword [rdi+sizeof(qword)*12],r12 + mov qword [rdi+sizeof(qword)*13],r13 + mov qword [rdi+sizeof(qword)*14],r14 + mov qword [rdi+sizeof(qword)*15],rbx + + sub rsi, sizeof(qword)*8 + sub rdi, sizeof(qword)*16 + + FINALIZE_FIX 16, rdi, rsi + ret +ENDFUNC sqr_16 + + + +;; +;; 9*qword triangle +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr9_triangle,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + xor r15, r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + call mla_8x1 + + xor rax, rax + mov qword [rdi+sizeof(qword)*(1+0)], r8 + mov qword [rdi+sizeof(qword)*(1+1)], r9 + mov qword [rdi+sizeof(qword)*(1+2)], r10 + mov qword [rdi+sizeof(qword)*(1+3)], r11 + mov qword [rdi+sizeof(qword)*(1+4)], r12 + mov qword [rdi+sizeof(qword)*(1+5)], r13 + mov qword [rdi+sizeof(qword)*(1+6)], r14 + mov qword [rdi+sizeof(qword)*(1+7)], r15 + mov qword [rdi+sizeof(qword)*(1+8)], rax + + sub rdi, sizeof(qword)*8 + ret +ENDFUNC sqr9_triangle + + +;; +;; 10*qword triangle +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr10_triangle,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + xor r15, r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + call mla_8x2 + + mov qword [rdi+sizeof(qword)*(2+0)], r8 + mov qword [rdi+sizeof(qword)*(2+1)], r9 + mov qword [rdi+sizeof(qword)*(2+2)], r10 + mov qword [rdi+sizeof(qword)*(2+3)], r11 + mov qword [rdi+sizeof(qword)*(2+4)], r12 + mov qword [rdi+sizeof(qword)*(2+5)], r13 + mov qword [rdi+sizeof(qword)*(2+6)], r14 + + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15, , , , , , , , , , {rsi+sizeof(qword)*2}, rbx,rbp + + xor rax, rax + mov qword [rdi+sizeof(qword)*(2+7)], r15 + mov qword [rdi+sizeof(qword)*(2+8)], r8 + mov qword [rdi+sizeof(qword)*(2+9)], rax + + sub rdi, sizeof(qword)*8 + ret +ENDFUNC sqr10_triangle + + +;; +;; 11*qword triangle +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr11_triangle,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + xor r15, r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + call mla_8x3 + + mov qword [rdi+sizeof(qword)*(3+0)], r8 + mov qword [rdi+sizeof(qword)*(3+1)], r9 + mov qword [rdi+sizeof(qword)*(3+2)], r10 + mov qword [rdi+sizeof(qword)*(3+3)], r11 + mov qword [rdi+sizeof(qword)*(3+4)], r12 + +; SQR_512_TRIANGLE_STEP H, p7, p6, p5, p4, p3, p2, p1, p0, T, ,, rbx,rbp +; ------------------------------ + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15,r14, , , , , , , r13,[rdi+sizeof(qword)*(3+5)],{rsi+sizeof(qword)*3}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*9] + SQR_512_TRIANGLE_STEP r9, r8, , , , , , , , r14,[rdi+sizeof(qword)*(3+6)],{rsi+sizeof(qword)*3}, rbx,rbp + + xor rax, rax + mov qword [rdi+sizeof(qword)*(3+7)], r15 + mov qword [rdi+sizeof(qword)*(3+8)], r8 + mov qword [rdi+sizeof(qword)*(3+9)], r9 + mov qword [rdi+sizeof(qword)*(3+10)],rax + + sub rdi, sizeof(qword)*8 + ret +ENDFUNC sqr11_triangle + + +;; +;; 12*qword triangle +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr12_triangle,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + xor r15, r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + call mla_8x4 + + mov qword [rdi+sizeof(qword)*(4+0)], r8 + mov qword [rdi+sizeof(qword)*(4+1)], r9 + mov qword [rdi+sizeof(qword)*(4+2)], r10 + mov qword [rdi+sizeof(qword)*(4+3)], r11 + +; SQR_512_TRIANGLE_STEP H, p7, p6, p5, p4, p3, p2, p1, p0, T, ,, rbx,rbp +; ------------------------------ + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15,r14,r13, , , , , , r12,[rdi+sizeof(qword)*(4+4)],{rsi+sizeof(qword)*4}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*9] + SQR_512_TRIANGLE_STEP r9, r8,r15, , , , , , , r13,[rdi+sizeof(qword)*(4+5)],{rsi+sizeof(qword)*4}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*10] + SQR_512_TRIANGLE_STEP r10, r9, , , , , , , , r14,[rdi+sizeof(qword)*(4+6)],{rsi+sizeof(qword)*4}, rbx,rbp + + xor rax, rax + mov qword [rdi+sizeof(qword)*(4+7)], r15 + mov qword [rdi+sizeof(qword)*(4+8)], r8 + mov qword [rdi+sizeof(qword)*(4+9)], r9 + mov qword [rdi+sizeof(qword)*(4+10)],r10 + mov qword [rdi+sizeof(qword)*(4+11)],rax + + sub rdi, sizeof(qword)*8 + ret +ENDFUNC sqr12_triangle + + +;; +;; 13*qword triangle +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr13_triangle,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + xor r15, r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + call mla_8x5 + + mov qword [rdi+sizeof(qword)*(5+0)], r8 + mov qword [rdi+sizeof(qword)*(5+1)], r9 + mov qword [rdi+sizeof(qword)*(5+2)], r10 + +; SQR_512_TRIANGLE_STEP H, p7, p6, p5, p4, p3, p2, p1, p0, T, ,, rbx,rbp +; ------------------------------ + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15,r14,r13,r12, , , , , r11,[rdi+sizeof(qword)*(5+3)],{rsi+sizeof(qword)*5}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*9] + SQR_512_TRIANGLE_STEP r9, r8,r15,r14, , , , , , r12,[rdi+sizeof(qword)*(5+4)],{rsi+sizeof(qword)*5}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*10] + SQR_512_TRIANGLE_STEP r10, r9, r8, , , , , , , r13,[rdi+sizeof(qword)*(5+5)],{rsi+sizeof(qword)*5}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*11] + SQR_512_TRIANGLE_STEP r11, r10, , , , , , , , r14,[rdi+sizeof(qword)*(5+6)],{rsi+sizeof(qword)*5}, rbx,rbp + + xor rax, rax + mov qword [rdi+sizeof(qword)*(5+7)], r15 + mov qword [rdi+sizeof(qword)*(5+8)], r8 + mov qword [rdi+sizeof(qword)*(5+9)], r9 + mov qword [rdi+sizeof(qword)*(5+10)],r10 + mov qword [rdi+sizeof(qword)*(5+11)],r11 + mov qword [rdi+sizeof(qword)*(5+12)],rax + + sub rdi, sizeof(qword)*8 + ret +ENDFUNC sqr13_triangle + + +;; +;; 14*qword triangle +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr14_triangle,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + xor r15, r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + call mla_8x6 + + mov qword [rdi+sizeof(qword)*(6+0)], r8 + mov qword [rdi+sizeof(qword)*(6+1)], r9 + +; SQR_512_TRIANGLE_STEP H, p7, p6, p5, p4, p3, p2, p1, p0, T, ,, rbx,rbp +; ------------------------------ + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15,r14,r13,r12,r11, , , , r10,[rdi+sizeof(qword)*(6+2)],{rsi+sizeof(qword)*6}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*9] + SQR_512_TRIANGLE_STEP r9, r8,r15,r14,r13, , , , , r11,[rdi+sizeof(qword)*(6+3)],{rsi+sizeof(qword)*6}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*10] + SQR_512_TRIANGLE_STEP r10, r9, r8,r15, , , , , , r12,[rdi+sizeof(qword)*(6+4)],{rsi+sizeof(qword)*6}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*11] + SQR_512_TRIANGLE_STEP r11, r10, r9, , , , , , , r13,[rdi+sizeof(qword)*(6+5)],{rsi+sizeof(qword)*6}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*12] + SQR_512_TRIANGLE_STEP r12, r11, , , , , , , , r14,[rdi+sizeof(qword)*(6+6)],{rsi+sizeof(qword)*6}, rbx,rbp + + xor rax, rax + mov qword [rdi+sizeof(qword)*(6+7)], r15 + mov qword [rdi+sizeof(qword)*(6+8)], r8 + mov qword [rdi+sizeof(qword)*(6+9)], r9 + mov qword [rdi+sizeof(qword)*(6+10)],r10 + mov qword [rdi+sizeof(qword)*(6+11)],r11 + mov qword [rdi+sizeof(qword)*(6+12)],r12 + mov qword [rdi+sizeof(qword)*(6+13)],rax + + sub rdi, sizeof(qword)*8 + ret +ENDFUNC sqr14_triangle + + +;; +;; 15*qword triangle +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr15_triangle,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + xor r15, r15 + + lea rcx, [rsi+sizeof(qword)*8] + add rdi, sizeof(qword)*8 + call mla_8x7 + + mov qword [rdi+sizeof(qword)*(7+0)], r8 + +; SQR_512_TRIANGLE_STEP H, p7, p6, p5, p4, p3, p2, p1, p0, T, ,, rbx,rbp +; ------------------------------ + mov rdx, qword [rsi+sizeof(qword)*8] + SQR_512_TRIANGLE_STEP r8, r15,r14,r13,r12,r11,r10, , , r9, [rdi+sizeof(qword)*(7+1)],{rsi+sizeof(qword)*7}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*9] + SQR_512_TRIANGLE_STEP r9, r8,r15,r14,r13,r12, , , , r10,[rdi+sizeof(qword)*(7+2)],{rsi+sizeof(qword)*7}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*10] + SQR_512_TRIANGLE_STEP r10, r9, r8,r15,r14, , , , , r11,[rdi+sizeof(qword)*(7+3)],{rsi+sizeof(qword)*7}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*11] + SQR_512_TRIANGLE_STEP r11, r10, r9, r8, , , , , , r12,[rdi+sizeof(qword)*(7+4)],{rsi+sizeof(qword)*7}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*12] + SQR_512_TRIANGLE_STEP r12, r11,r10, , , , , , , r13,[rdi+sizeof(qword)*(7+5)],{rsi+sizeof(qword)*7}, rbx,rbp + mov rdx, qword [rsi+sizeof(qword)*13] + SQR_512_TRIANGLE_STEP r13, r12, , , , , , , , r14,[rdi+sizeof(qword)*(7+6)],{rsi+sizeof(qword)*7}, rbx,rbp + + xor rax, rax + mov qword [rdi+sizeof(qword)*(7+7)], r15 + mov qword [rdi+sizeof(qword)*(7+8)], r8 + mov qword [rdi+sizeof(qword)*(7+9)], r9 + mov qword [rdi+sizeof(qword)*(7+10)],r10 + mov qword [rdi+sizeof(qword)*(7+11)],r11 + mov qword [rdi+sizeof(qword)*(7+12)],r12 + mov qword [rdi+sizeof(qword)*(7+13)],r13 + mov qword [rdi+sizeof(qword)*(7+14)],rax + + sub rdi, sizeof(qword)*8 + ret +ENDFUNC sqr15_triangle + + +;; +;; 16*qword triangle +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC sqr16_triangle,PRIVATE + call sqr8_triangle + mov qword [rdi+sizeof(qword)*7], r15 + xor r15, r15 + + mov rcx, rsi + add rsi, sizeof(qword)*8 + add rdi, sizeof(qword)*8 + call mla_8x8 + + add rdi, sizeof(qword)*8 + call sqr8_triangle + + xor rax, rax + mov qword [rdi+sizeof(qword)*7], r15 + mov qword [rdi+sizeof(qword)*8], r8 + mov qword [rdi+sizeof(qword)*9], r9 + mov qword [rdi+sizeof(qword)*10],r10 + mov qword [rdi+sizeof(qword)*11],r11 + mov qword [rdi+sizeof(qword)*12],r12 + mov qword [rdi+sizeof(qword)*13],r13 + mov qword [rdi+sizeof(qword)*14],r14 + mov qword [rdi+sizeof(qword)*15],rax + + sub rsi, sizeof(qword)*8 + sub rdi, sizeof(qword)*16 + ret +ENDFUNC sqr16_triangle + + + +sqr_l_basic DQ sqr_1 - sqr_l_basic + DQ sqr_2 - sqr_l_basic + DQ sqr_3 - sqr_l_basic + DQ sqr_4 - sqr_l_basic + DQ sqr_5 - sqr_l_basic + DQ sqr_6 - sqr_l_basic + DQ sqr_7 - sqr_l_basic + DQ sqr_8 - sqr_l_basic + DQ sqr_9 - sqr_l_basic + DQ sqr_10- sqr_l_basic + DQ sqr_11- sqr_l_basic + DQ sqr_12- sqr_l_basic + DQ sqr_13- sqr_l_basic + DQ sqr_14- sqr_l_basic + DQ sqr_15- sqr_l_basic + DQ sqr_16- sqr_l_basic + +sqrN_triangle DQ sqr9_triangle - sqrN_triangle + DQ sqr10_triangle - sqrN_triangle + DQ sqr11_triangle - sqrN_triangle + DQ sqr12_triangle - sqrN_triangle + DQ sqr13_triangle - sqrN_triangle + DQ sqr14_triangle - sqrN_triangle + DQ sqr15_triangle - sqrN_triangle + DQ sqr16_triangle - sqrN_triangle + +%endif ;; _PCPBNSQR_BASIC_ADCX_INC_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnusqrschool.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnusqrschool.inc new file mode 100644 index 0000000..cb45b44 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnusqrschool.inc @@ -0,0 +1,283 @@ +;=============================================================================== +; Copyright (C) 2010 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; BNU squaring support +; +; + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; MULx1 genaral-case squarer macros +;; + +;; dst = src * B epilogue (srcLen=4*n+3) +%macro sMULx1_4N_3_ELOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%update_idx %3 + %xdefine %%B %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mul %%B + xor %%T1, %%T1 + add %%T0, rax + mov qword [%%rDst+sizeof(qword)], %%T0 + mov rax, qword [%%rSrc+sizeof(qword)*2] + adc %%T1, rdx + + mul %%B + xor %%T2, %%T2 + add %%T1, rax + mov qword [%%rDst+sizeof(qword)*2], %%T1 + mov rax, qword [%%rSrc+sizeof(qword)*3] + adc %%T2, rdx + + mul %%B + %%update_idx + add %%T2, rax + mov qword [%%rDst+sizeof(qword)*3], %%T2 + ;mov rax, qword [rSrc+idx*sizeof(qword)] + adc rdx, 0 + + mov qword [%%rDst+sizeof(qword)*4], rdx + add %%rDst, sizeof(qword) +%endmacro + +;; dst = src * B epilogue (srcLen=4*n+1) +%macro sMULx1_4N_1_ELOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%update_idx %3 + %xdefine %%B %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mul %%B + %%update_idx + add %%T0, rax + mov qword [%%rDst+sizeof(qword)*3], %%T0 + ;mov rax, qword [rSrc+idx*sizeof(qword)] + adc rdx, 0 + + mov qword [%%rDst+sizeof(qword)*4], rdx + add %%rDst, sizeof(qword) +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; MULx2 genaral-case multiplier macros +;; + +;; dst = src * {B1:B0} epilogue (srcLen=4*n+1) +%macro sMULx2_4N_1_ELOG 9.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%update_idx %3 + %xdefine %%B0 %4 + %xdefine %%B1 %5 + %xdefine %%T0 %6 + %xdefine %%T1 %7 + %xdefine %%T2 %8 + %xdefine %%T3 %9 + + mul %%B1 ; {T2:T1} += a[lenA-1]*B1 + ;add rDst, sizeof(qword)*2 + %%update_idx + mov qword [%%rDst+sizeof(qword)*3], %%T0 + add %%T1, rax + ;mov rax, qword [rSrc+idx*sizeof(qword)] + adc rdx, %%T2 + + mov qword [%%rDst+sizeof(qword)*4], %%T1 + mov qword [%%rDst+sizeof(qword)*5], rdx +%endmacro + +;; dst = src * {B1:B0} epilogue (srcLen=4*n+3) +%macro sMULx2_4N_3_ELOG 9.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%update_idx %3 + %xdefine %%B0 %4 + %xdefine %%B1 %5 + %xdefine %%T0 %6 + %xdefine %%T1 %7 + %xdefine %%T2 %8 + %xdefine %%T3 %9 + + mul %%B1 ; {T2:T1} += a[lenA-3]*B1 + xor %%T3, %%T3 + add %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a[lenA-2] + adc %%T2, rdx + + mul %%B0 ; {T3:T2:T1} += a[LenA-2]*B0 + mov qword [%%rDst+sizeof(qword)], %%T0 + add %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a[lenA-2] + adc %%T2, rdx + adc %%T3, 0 + + mul %%B1 ; {T3:T2} += a[lenA-2]*B1 + xor %%T0, %%T0 + add %%T2, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[lenA-1] + adc %%T3, rdx + + mul %%B0 ; {T0:T3:T2} += a[lenA-1]*B0 + mov qword [%%rDst+sizeof(qword)*2], %%T1 + add %%T2, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[lenA-1] + adc %%T3, rdx + adc %%T0, 0 + + mul %%B1 ; {T0:T3} += a[lenA-1]*B1 + ;add rDst, sizeof(qword)*2 + %%update_idx + mov qword [%%rDst+sizeof(qword)*3], %%T2 + add %%T3, rax + ;mov rax, qword [rSrc+idx*sizeof(qword)] + adc rdx, %%T0 + + mov qword [%%rDst+sizeof(qword)*4], %%T3 + mov qword [%%rDst+sizeof(qword)*5], rdx +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; MLAx2 genaral-case multiplier macros +;; + +;; +;; B0 = rSrc[-2] +;; B1 = rSrc[-1] +;; inp_vector = rSrc +;; out_vector = rDst +%macro sMLAx2_PLOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%B0 %3 + %xdefine %%B1 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mov %%B0, qword [%%rSrc-2*sizeof(qword)] ; preload a[-2] + mov %%B1, qword [%%rSrc-sizeof(qword)] ; and a[i-1] + + mov rax, %%B1 + mul %%B0 ; a[-2]*a[i-1] + xor %%T0, %%T0 + + add qword [%%rDst-sizeof(qword)], rax + mov rax, qword [%%rSrc] ; a[i] + adc %%T0, rdx + + mul %%B0 ; B0*a[i] + xor %%T1, %%T1 + xor %%T2, %%T2 + add %%T0, rax + mov rax, qword [%%rSrc] ; a[i] + adc %%T1, rdx +%endmacro + +;; dst = + src * {B1:B0} epilogue (srcLen=4*n+1) +%macro sMLAx2_4N_1_ELOG 9.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%update_idx %3 + %xdefine %%B0 %4 + %xdefine %%B1 %5 + %xdefine %%T0 %6 + %xdefine %%T1 %7 + %xdefine %%T2 %8 + %xdefine %%T3 %9 + + mul %%B1 ; {T2:T1} += a[lenA-1]*B1 + r[lenA-1] + ;add rDst, sizeof(qword)*2 + %%update_idx + add %%T0, qword [%%rDst+sizeof(qword)*3] + mov qword [%%rDst+sizeof(qword)*3], %%T0 + adc %%T1, rax + adc rdx, %%T2 + ;mov rax, qword [rSrc+idx*sizeof(qword)] + + mov qword [%%rDst+sizeof(qword)*4], %%T1 + mov qword [%%rDst+sizeof(qword)*5], rdx +%endmacro + +;; dst = + src * {B1:B0} epilogue (srcLen=4*n+3) +%macro sMLAx2_4N_3_ELOG 9.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%update_idx %3 + %xdefine %%B0 %4 + %xdefine %%B1 %5 + %xdefine %%T0 %6 + %xdefine %%T1 %7 + %xdefine %%T2 %8 + %xdefine %%T3 %9 + + mul %%B1 ; {T2:T1} += a[lenA-3]*B1 + xor %%T3, %%T3 + add %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a[lenA-2] + adc %%T2, rdx + + mul %%B0 ; {T3:T2:T1} += a[LenA-2]*B0 + r[len-3] + add %%T0, qword [%%rDst+sizeof(qword)] + mov qword [%%rDst+sizeof(qword)], %%T0 + adc %%T1, rax + adc %%T2, rdx + adc %%T3, 0 + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a[lenA-2] + + mul %%B1 ; {T3:T2} += a[lenA-2]*B1 + xor %%T0, %%T0 + add %%T2, rax + adc %%T3, rdx + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[lenA-1] + + mul %%B0 ; {T0:T3:T2} += a[lenA-1]*B0 + r[lenA-2] + add %%T1, qword [%%rDst+sizeof(qword)*2] + mov qword [%%rDst+sizeof(qword)*2], %%T1 + adc %%T2, rax + adc %%T3, rdx + adc %%T0, 0 + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[lenA-1] + + mul %%B1 ; {T0:T3} += a[lenA-1]*B1 + r[lenA-1] + ;add rDst, sizeof(qword)*2 + %%update_idx + add %%T2, qword [%%rDst+sizeof(qword)*3] + mov qword [%%rDst+sizeof(qword)*3], %%T2 + adc %%T3, rax + adc rdx, %%T0 + ;mov rax, qword [rSrc+idx*sizeof(qword)] + + mov qword [%%rDst+sizeof(qword)*4], %%T3 + mov qword [%%rDst+sizeof(qword)*5], rdx +%endmacro + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnusqrschoolm7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnusqrschoolm7as.asm new file mode 100644 index 0000000..aeb8acb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnusqrschoolm7as.asm @@ -0,0 +1,1343 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Big Number Operations +; +; Content: +; cpSqrAdc_BNU_school() +; +; + +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpbnumulschool.inc" +%include "pcpbnusqrschool.inc" +%include "pcpvariant.inc" + +%if (_ADCOX_NI_ENABLING_ == _FEATURE_OFF_) || (_ADCOX_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_M7) && (_IPP32E < _IPP32E_L9) + +; acc:a1 = src * mem + a1 +; clobbers rax, rdx +%macro MULADD0 4.nolist + %xdefine %%acc %1 + %xdefine %%a1 %2 + %xdefine %%src %3 + %xdefine %%mem %4 + + mov rax, %%mem + mul %%src + xor %%acc, %%acc + add %%a1, rax + adc %%acc, rdx +%endmacro + + +; acc:a1 = src * mem + a1 + acc +; clobbers rax, rdx +%macro MULADD 4.nolist + %xdefine %%acc %1 + %xdefine %%a1 %2 + %xdefine %%src %3 + %xdefine %%mem %4 + + mov rax, %%mem + mul %%src + add %%a1, rax + adc rdx, 0 + add %%a1, %%acc + adc rdx, 0 + mov %%acc, rdx +%endmacro + + +; acc:a1 = src * mem + a1 +; clobbers rax, rdx +%macro MULADD1 4.nolist + %xdefine %%acc %1 + %xdefine %%a1 %2 + %xdefine %%src %3 + %xdefine %%mem %4 + + mov rax, %%mem + mul %%src + add %%a1, rax + adc rdx, 0 + mov %%acc, rdx +%endmacro + + +; Macro to allow us to do an adc as an adc_0/add pair +%macro adc2 2.nolist + %xdefine %%a1 %1 + %xdefine %%a2 %2 + +%if 1 + adc %%a1, %%a2 +%else + adc %%a2, 0 + add %%a1, %%a2 +%endif +%endmacro + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; This version does the first half of the off-diagonal terms, then adds +; in the first half of the diagonal terms, then does the second half. +; This is to avoid the writing and reading of the off-diagonal terms to mem. +; Due to extra overhead in the arithmetic, this may or may not be faster +; than the simple version (which follows) +; +; Input in memory [pA] and also in x7...x0 +; Uses all argument registers plus rax and rdx +; +%macro SQR_512 13.nolist + %xdefine %%pDst %1 + %xdefine %%pA %2 + %xdefine %%x7 %3 + %xdefine %%x6 %4 + %xdefine %%x5 %5 + %xdefine %%x4 %6 + %xdefine %%x3 %7 + %xdefine %%x2 %8 + %xdefine %%x1 %9 + %xdefine %%x0 %10 + %xdefine %%A %11 + %xdefine %%x8 %12 + %xdefine %%t0 %13 + + ;; ------------------ + ;; first pass 01...07 + ;; ------------------ + mov %%A, %%x0 + + mov rax, %%x1 + mul %%A + mov %%x0, rax + mov %%x1, rdx + + MULADD1 %%x2, %%x1, %%A, %%x2 + MULADD1 %%x3, %%x2, %%A, %%x3 + MULADD1 %%x4, %%x3, %%A, %%x4 + MULADD1 %%x5, %%x4, %%A, %%x5 + MULADD1 %%x6, %%x5, %%A, %%x6 + MULADD1 %%x7, %%x6, %%A, %%x7 + + ;; ------------------ + ;; second pass 12...16 + ;; ------------------ + mov %%A, [%%pA + 8*1] + xor %%x8, %%x8 + + mov rax, [%%pA + 8*2] + mul %%A + add %%x2, rax + adc rdx, 0 + mov %%t0, rdx + + MULADD %%t0, %%x3, %%A, [%%pA + 8*3] + MULADD %%t0, %%x4, %%A, [%%pA + 8*4] + MULADD %%t0, %%x5, %%A, [%%pA + 8*5] + MULADD %%t0, %%x6, %%A, [%%pA + 8*6] + add %%x7, %%t0 + adc %%x8, 0 + + ;; ------------------ + ;; third pass 23...25 + ;; ------------------ + mov %%A, [%%pA + 8*2] + + mov rax, [%%pA + 8*3] + mul %%A + add %%x4, rax + adc rdx, 0 + mov %%t0, rdx + + MULADD %%t0, %%x5, %%A, [%%pA + 8*4] + MULADD %%t0, %%x6, %%A, [%%pA + 8*5] + add %%x7, %%t0 + adc %%x8, 0 + + ;; ------------------ + ;; fourth pass 34 + ;; ------------------ + mov rax, [%%pA + 8*3] + mul qword [%%pA + 8*4] + add %%x6, rax + adc rdx, 0 + add %%x7, rdx + adc %%x8, 0 + + ;; --- double x0...x6 + xor %%A, %%A + add %%x0, %%x0 + adc %%x1, %%x1 + adc %%x2, %%x2 + adc %%x3, %%x3 + adc %%x4, %%x4 + adc %%x5, %%x5 + adc %%x6, %%x6 + adc %%A, 0 + + mov rax, [%%pA + 8*0] + mul rax + mov [%%pDst + 8*0], rax + mov %%t0, rdx + + mov rax, [%%pA + 8*1] + mul rax + add %%x0, %%t0 + adc %%x1, rax + mov [%%pDst + 8*1], %%x0 + adc rdx, 0 + mov [%%pDst + 8*2], %%x1 + mov %%t0, rdx + mov rax, [%%pA + 8*2] + mul rax + mov %%x0, rax + mov %%x1, rdx + mov rax, [%%pA + 8*3] + mul rax + add %%x2, %%t0 + adc %%x3, %%x0 + mov [%%pDst + 8*3], %%x2 + adc2 %%x4, %%x1 + mov [%%pDst + 8*4], %%x3 + adc %%x5, rax + mov [%%pDst + 8*5], %%x4 + adc2 %%x6, rdx + mov [%%pDst + 8*6], %%x5 + adc %%A, 0 + mov [%%pDst + 8*7], %%x6 + + ;; ------------------ + ;; second pass 17 + ;; ------------------ + mov rax, [%%pA + 8*1] + xor %%x0, %%x0 + + mul qword [%%pA + 8*7] + add %%x7, rax + adc rdx, 0 + add %%x8, rdx + adc %%x0, 0 + + ;; ------------------ + ;; third pass 26...27 + ;; ------------------ + mov %%x6, [%%pA + 8*2] + + mov rax, [%%pA + 8*6] + mul %%x6 + add %%x7, rax + adc rdx, 0 + add %%x8, rdx + adc %%x0, 0 + + mov rax, [%%pA + 8*7] + xor %%x1, %%x1 + mul %%x6 + add %%x8, rax + adc rdx, 0 + add %%x0, rdx + adc %%x1, 0 + + ;; ------------------ + ;; fourth pass 35...37 + ;; ------------------ + mov %%x6, [%%pA + 8*3] + + mov rax, [%%pA + 8*5] + mul %%x6 + add %%x7, rax + adc rdx, 0 + add %%x8, rdx + adc %%x0, 0 + adc %%x1, 0 + + mov rax, [%%pA + 8*6] + mul %%x6 + add %%x8, rax + adc rdx, 0 + add %%x0, rdx + adc %%x1, 0 + + mov rax, [%%pA + 8*7] + mul %%x6 + add %%x0, rax + adc rdx, 0 + add %%x1, rdx + ;; carry out should be 0 + + ;; ------------------ + ;; f%ifth pass 45...47 + ;; ------------------ + mov %%x6, [%%pA + 8*4] + + mov rax, [%%pA + 8*5] + mul %%x6 + add %%x8, rax + adc rdx, 0 + mov %%x2, rdx + + MULADD %%x2, %%x0, %%x6, [%%pA + 8*6] + MULADD %%x2, %%x1, %%x6, [%%pA + 8*7] + + ;; ------------------ + ;; sixth pass 56...57 & seventh pass 67 + ;; ------------------ + mov %%x6, [%%pA + 8*5] + + mov rax, [%%pA + 8*6] + mul %%x6 + add %%x1, rax + adc rdx, 0 + mov %%x3, rdx + + MULADD %%x3, %%x2, %%x6, [%%pA + 8*7] + + mov rax, [%%pA + 8*6] + mul qword [%%pA + 8*7] + add %%x3, rax + adc rdx, 0 + mov %%x4, rdx + + ;; --- double x7, x8, x0, ..., x4 + xor %%x5, %%x5 + add %%x7, %%x7 + adc %%x8, %%x8 + adc %%x0, %%x0 + adc %%x1, %%x1 + adc %%x2, %%x2 + adc %%x3, %%x3 + adc %%x4, %%x4 + adc %%x5, 0 + + mov rax, [%%pA + 8*4] + mul rax + add rax, %%A + adc rdx, 0 + add rax, %%x7 + adc rdx, 0 + mov [%%pDst + 8*8], rax + mov %%A, rdx + mov rax, [%%pA + 8*5] + mul rax + add %%x8, %%A + adc %%x0, rax + mov [%%pDst + 8*9], %%x8 + adc rdx, 0 + mov [%%pDst + 8*10], %%x0 + mov %%A, rdx + mov rax, [%%pA + 8*6] + mul rax + mov %%x7, rax + mov %%x8, rdx + mov rax, [%%pA + 8*7] + mul rax + add %%x1, %%A + adc %%x2, %%x7 + mov [%%pDst + 8*11], %%x1 + adc2 %%x3, %%x8 + mov [%%pDst + 8*12], %%x2 + adc %%x4, rax + mov [%%pDst + 8*13], %%x3 + adc2 %%x5, rdx + mov [%%pDst + 8*14], %%x4 + mov [%%pDst + 8*15], %%x5 +%endmacro + + +%macro SQR_448 13.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%x7 %3 + %xdefine %%x6 %4 + %xdefine %%x5 %5 + %xdefine %%x4 %6 + %xdefine %%x3 %7 + %xdefine %%x2 %8 + %xdefine %%x1 %9 + %xdefine %%x0 %10 + %xdefine %%A %11 + %xdefine %%x8 %12 + %xdefine %%t0 %13 + + ;; ------------------ + ;; first pass 01...06 + ;; ------------------ + mov %%A, %%x0 + + mov rax, %%x1 + mul %%A + mov %%x0, rax + mov %%x1, rdx + + MULADD1 %%x2, %%x1, %%A, %%x2 + MULADD1 %%x3, %%x2, %%A, %%x3 + MULADD1 %%x4, %%x3, %%A, %%x4 + MULADD1 %%x5, %%x4, %%A, %%x5 + MULADD1 %%x6, %%x5, %%A, %%x6 + + ;; ------------------ + ;; second pass 12...16 + ;; ------------------ + mov %%A, [%%rSrc + sizeof(qword)*1] + mov rax, [%%rSrc+ sizeof(qword)*2] + mul %%A + add %%x2, rax + adc rdx, 0 + mov %%t0, rdx + MULADD %%t0, %%x3, %%A, [%%rSrc + sizeof(qword)*3] + MULADD %%t0, %%x4, %%A, [%%rSrc + sizeof(qword)*4] + MULADD %%t0, %%x5, %%A, [%%rSrc + sizeof(qword)*5] + MULADD %%t0, %%x6, %%A, [%%rSrc + sizeof(qword)*6] + mov %%x7, %%t0 + + ;; ------------------ + ;; third pass 23...25 + ;; ------------------ + mov %%A, [%%rSrc + sizeof(qword)*2] + xor %%x8, %%x8 + mov rax, [%%rSrc + sizeof(qword)*3] + mul %%A + add %%x4, rax + adc rdx, 0 + mov %%t0, rdx + MULADD %%t0, %%x5, %%A, [%%rSrc+ sizeof(qword)*4] + MULADD %%t0, %%x6, %%A, [%%rSrc+ sizeof(qword)*5] + add %%x7, %%t0 + adc %%x8, 0 + + ;; ------------------ + ;; fourth pass 34 + ;; ------------------ + mov rax, [%%rSrc + sizeof(qword)*3] + mul qword [%%rSrc + sizeof(qword)*4] + add %%x6, rax + adc rdx, 0 + add %%x7, rdx + adc %%x8, 0 + + + mov rax, [%%rSrc + sizeof(qword)*0] + + ;; --- double x0...x6 + xor %%A, %%A + add %%x0, %%x0 + adc %%x1, %%x1 + adc %%x2, %%x2 + adc %%x3, %%x3 + adc %%x4, %%x4 + adc %%x5, %%x5 + adc %%x6, %%x6 + adc %%A, 0 + + mul rax ; a[0]^2 + mov [%%rDst + sizeof(qword)*0], rax + mov rax, [%%rSrc + sizeof(qword)*1] + mov %%t0, rdx + + mul rax ; a[1]^2 + add %%x0, %%t0 + adc %%x1, rax + mov rax, [%%rSrc + sizeof(qword)*2] + mov [%%rDst + sizeof(qword)*1], %%x0 + adc rdx, 0 + mov [%%rDst + sizeof(qword)*2], %%x1 + mov %%t0, rdx + + mul rax ; a[2]^2 + add %%x2, %%t0 + adc %%x3, rax + mov rax, [%%rSrc + sizeof(qword)*3] + mov [%%rDst + sizeof(qword)*3], %%x2 + adc rdx, 0 + mov [%%rDst + sizeof(qword)*4], %%x3 + mov %%t0, rdx + + mul rax ; a[3]^2 + add %%x4, %%t0 + adc %%x5, rax + mov [%%rDst + sizeof(qword)*5], %%x4 + adc %%x6, rdx + mov [%%rDst + sizeof(qword)*6], %%x5 + adc %%A, 0 + mov [%%rDst + sizeof(qword)*7], %%x6 + + ;; ------------------ + ;; third pass complete 26 + ;; ------------------ + mov rax, [%%rSrc + sizeof(qword)*2] + xor %%x0, %%x0 + + mul qword [%%rSrc + sizeof(qword)*6] + add %%x7, rax + adc rdx, 0 + add %%x8, rdx + adc %%x0, 0 + + ;; ------------------ + ;; forth pass complete 35...36 + ;; ------------------ + mov %%x6, [%%rSrc + sizeof(qword)*3] + mov rax, [%%rSrc+ sizeof(qword)*5] + mul %%x6 + add %%x7, rax + mov rax, [%%rSrc + sizeof(qword)*6] + adc %%x8, rdx + adc %%x0, 0 + mul %%x6 + add %%x8, rax + adc %%x0, rdx + + ;; ------------------ + ;; f%ifth pass 45...46 + ;; ------------------ + mov %%x6, [%%rSrc + sizeof(qword)*4] + xor %%x1, %%x1 + mov rax, [%%rSrc + sizeof(qword)*5] + mul %%x6 + add %%x8, rax + mov rax, [%%rSrc + sizeof(qword)*6] + adc %%x0, rdx + adc %%x1, 0 + mul %%x6 + add %%x0, rax + adc %%x1, rdx + + ;; ------------------ + ;; six pass 56 + ;; ------------------ + mov %%x6, [%%rSrc + sizeof(qword)*5] + xor %%x2, %%x2 + mov rax, [%%rSrc + sizeof(qword)*6] + mul %%x6 + add %%x1, rax + adc %%x2, rdx + + mov rax, [%%rSrc + sizeof(qword)*4] + + ;; --- double x7, x8, x0, x1, x2 + xor %%x3, %%x3 + add %%x7, %%x7 + adc %%x8, %%x8 + adc %%x0, %%x0 + adc %%x1, %%x1 + adc %%x2, %%x2 + adc %%x3, 0 + + mul rax ; a[4]^2 + add %%x7, %%A + adc rdx, 0 + xor %%A, %%A + add %%x7, rax + mov rax, [%%rSrc + sizeof(qword)*5] + adc %%x8, rdx + mov [%%rDst+ sizeof(qword)*8], %%x7 + adc %%A, 0 + mov [%%rDst + sizeof(qword)*9], %%x8 + + mul rax ; a[5]^2 + add %%x0, %%A + adc rdx, 0 + xor %%A, %%A + add %%x0, rax + mov rax, [%%rSrc + sizeof(qword)*6] + adc %%x1, rdx + mov [%%rDst + sizeof(qword)*10], %%x0 + adc %%A, 0 + mov [%%rDst + sizeof(qword)*11], %%x1 + + mul rax ; a[6]^2 + add %%x2, %%A + adc rdx, 0 + add %%x2, rax + adc rdx, %%x3 + mov [%%rDst + sizeof(qword)*12], %%x2 + mov [%%rDst + sizeof(qword)*13], rdx +%endmacro + + +%macro SQR_384 13.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%x7 %3 + %xdefine %%x6 %4 + %xdefine %%x5 %5 + %xdefine %%x4 %6 + %xdefine %%x3 %7 + %xdefine %%x2 %8 + %xdefine %%x1 %9 + %xdefine %%x0 %10 + %xdefine %%A %11 + %xdefine %%x8 %12 + %xdefine %%t0 %13 + + mov %%A, %%x0 + mov rax, %%x1 + mul %%A + mov %%x0, rax + mov %%x1, rdx + MULADD1 %%x2, %%x1, %%A, %%x2 + MULADD1 %%x3, %%x2, %%A, %%x3 + MULADD1 %%x4, %%x3, %%A, %%x4 + MULADD1 %%x5, %%x4, %%A, %%x5 + + mov %%A, qword [%%rSrc+ sizeof(qword)*1] + mov rax, qword [%%rSrc+ sizeof(qword)*2] + mul %%A + add %%x2, rax + adc rdx, 0 + mov %%t0, rdx + MULADD %%t0, %%x3, %%A, [%%rSrc + sizeof(qword)*3] + MULADD %%t0, %%x4, %%A, [%%rSrc + sizeof(qword)*4] + MULADD %%t0, %%x5, %%A, [%%rSrc + sizeof(qword)*5] + mov %%x6, %%t0 + + mov %%A, qword [%%rSrc+ sizeof(qword)*2] + mov rax, qword [%%rSrc+ sizeof(qword)*3] + mul %%A + add %%x4, rax + adc rdx, 0 + mov %%t0, rdx + MULADD %%t0, %%x5, %%A, [%%rSrc + sizeof(qword)*4] + MULADD %%t0, %%x6, %%A, [%%rSrc + sizeof(qword)*5] + mov %%x7, %%t0 + + mov %%A, qword [%%rSrc+ sizeof(qword)*3] + mov rax, qword [%%rSrc+ sizeof(qword)*4] + mul %%A + xor %%x8, %%x8 + add %%x6, rax + mov rax, qword [%%rSrc+ sizeof(qword)*5] + adc %%x7, rdx + adc %%x8, 0 + mul %%A + mov %%A, qword [%%rSrc+ sizeof(qword)*4] + add %%x7, rax + mov rax, qword [%%rSrc+ sizeof(qword)*5] + adc %%x8, rdx + + mul %%A + xor %%t0, %%t0 + add %%x8, rax + adc %%t0, rdx + + mov rax, [%%rSrc + sizeof(qword)*0] + + ;; --- double x0...x7,x8,t0 + xor %%A, %%A + add %%x0, %%x0 + adc %%x1, %%x1 + adc %%x2, %%x2 + adc %%x3, %%x3 + adc %%x4, %%x4 + adc %%x5, %%x5 + adc %%x6, %%x6 + adc %%x7, %%x7 + adc %%x8, %%x8 + adc %%t0, %%t0 + adc %%A, 0 + mov qword [rsp], %%A + + mul rax + mov [%%rDst + sizeof(qword)*0], rax + mov rax, [%%rSrc + sizeof(qword)*1] ; a[1] + mov %%A, rdx + + mul rax + add %%x0, %%A + adc %%x1, rax + mov rax, [%%rSrc + sizeof(qword)*2] ; a[2] + mov [%%rDst + sizeof(qword)*1], %%x0 + adc rdx, 0 + mov [%%rDst + sizeof(qword)*2], %%x1 + mov %%A, rdx + + mul rax + add %%x2, %%A + adc %%x3, rax + mov rax, [%%rSrc + sizeof(qword)*3] ; a[3] + mov [%%rDst + sizeof(qword)*3], %%x2 + adc rdx, 0 + mov [%%rDst + sizeof(qword)*4], %%x3 + mov %%A, rdx + + mul rax + add %%x4, %%A + adc %%x5, rax + mov rax, [%%rSrc + sizeof(qword)*4] ; a[4] + mov [%%rDst + sizeof(qword)*5], %%x4 + adc rdx, 0 + mov [%%rDst + sizeof(qword)*6], %%x5 + mov %%A, rdx + + mul rax + add %%x6, %%A + adc %%x7, rax + mov rax, [%%rSrc + sizeof(qword)*5] ; a[5] + mov [%%rDst + sizeof(qword)*7], %%x6 + adc rdx, 0 + mov [%%rDst + sizeof(qword)*8], %%x7 + mov %%A, rdx + + mul rax + add %%x8, %%A + adc %%t0, rax + mov [%%rDst + sizeof(qword)*9], %%x8 + adc rdx, qword [rsp] + mov [%%rDst + sizeof(qword)*10], %%t0 + mov [%%rDst + sizeof(qword)*11], rdx +%endmacro + + +%macro SQR_320 13.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%x7 %3 + %xdefine %%x6 %4 + %xdefine %%x5 %5 + %xdefine %%x4 %6 + %xdefine %%x3 %7 + %xdefine %%x2 %8 + %xdefine %%x1 %9 + %xdefine %%x0 %10 + %xdefine %%A %11 + %xdefine %%x8 %12 + %xdefine %%t0 %13 + + mov %%A, %%x0 + mov rax, %%x1 + mul %%A + mov %%x0, rax + mov %%x1, rdx + MULADD1 %%x2, %%x1, %%A, %%x2 + MULADD1 %%x3, %%x2, %%A, %%x3 + MULADD1 %%x4, %%x3, %%A, %%x4 + + mov %%A, qword [%%rSrc+ sizeof(qword)*1] + mov rax, qword [%%rSrc+ sizeof(qword)*2] + mul %%A + add %%x2, rax + adc rdx, 0 + mov %%t0, rdx + MULADD %%t0, %%x3, %%A, [%%rSrc + sizeof(qword)*3] + MULADD %%t0, %%x4, %%A, [%%rSrc + sizeof(qword)*4] + mov %%x5, %%t0 + + mov %%A, qword [%%rSrc+ sizeof(qword)*2] + mov rax, qword [%%rSrc+ sizeof(qword)*3] + mul %%A + xor %%x6, %%x6 + add %%x4, rax + mov rax, qword [%%rSrc+ sizeof(qword)*4] + adc %%x5, rdx + adc %%x6, 0 + mul %%A + mov %%A, qword [%%rSrc+ sizeof(qword)*3] + add %%x5, rax + mov rax, qword [%%rSrc+ sizeof(qword)*4] + adc %%x6, rdx + + mul %%A + xor %%x7, %%x7 + add %%x6, rax + adc %%x7, rdx + + mov rax, [%%rSrc + sizeof(qword)*0] + + ;; --- double x0...x5 + xor %%A, %%A + add %%x0, %%x0 + adc %%x1, %%x1 + adc %%x2, %%x2 + adc %%x3, %%x3 + adc %%x4, %%x4 + adc %%x5, %%x5 + adc %%x6, %%x6 + adc %%x7, %%x7 + adc %%A, 0 + + mul rax + mov [%%rDst + sizeof(qword)*0], rax + mov rax, [%%rSrc + sizeof(qword)*1] + mov %%t0, rdx + + mul rax + add %%x0, %%t0 + adc %%x1, rax + mov rax, [%%rSrc + sizeof(qword)*2] + mov [%%rDst + sizeof(qword)*1], %%x0 + adc rdx, 0 + mov [%%rDst + sizeof(qword)*2], %%x1 + mov %%t0, rdx + + mul rax + add %%x2, %%t0 + adc %%x3, rax + mov rax, [%%rSrc + sizeof(qword)*3] + mov [%%rDst + sizeof(qword)*3], %%x2 + adc rdx, 0 + mov [%%rDst + sizeof(qword)*4], %%x3 + mov %%t0, rdx + + mul rax + add %%x4, %%t0 + adc %%x5, rax + mov rax, [%%rSrc + sizeof(qword)*4] + mov [%%rDst + sizeof(qword)*5], %%x4 + adc rdx, 0 + mov [%%rDst + sizeof(qword)*6], %%x5 + mov %%t0, rdx + + mul rax + add %%x6, %%t0 + adc %%x7, rax + mov [%%rDst + sizeof(qword)*7], %%x6 + adc rdx, 0 + mov [%%rDst + sizeof(qword)*8], %%x7 + add rdx, %%A + mov [%%rDst + sizeof(qword)*9], rdx +%endmacro + + +%macro SQR_256 13.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%x7 %3 + %xdefine %%x6 %4 + %xdefine %%x5 %5 + %xdefine %%x4 %6 + %xdefine %%x3 %7 + %xdefine %%x2 %8 + %xdefine %%x1 %9 + %xdefine %%x0 %10 + %xdefine %%A %11 + %xdefine %%x8 %12 + %xdefine %%t0 %13 + + ;; ------------------ + ;; first pass 01...03 + ;; ------------------ + mov %%A, %%x0 + mov rax, %%x1 + mul %%A + mov %%x0, rax + mov %%x1, rdx + MULADD1 %%x2, %%x1, %%A, %%x2 + MULADD1 %%x3, %%x2, %%A, %%x3 + + ;; ------------------ + ;; second pass 12, 13 + ;; ------------------ + mov %%A, qword [%%rSrc+ sizeof(qword)*1] + mov rax, qword [%%rSrc+ sizeof(qword)*2] + mul %%A + xor %%x4, %%x4 + add %%x2, rax + mov rax, qword [%%rSrc+ sizeof(qword)*3] + adc %%x3, rdx + adc %%x4, 0 + + mul %%A + mov %%A, qword [%%rSrc+ sizeof(qword)*2] + add %%x3, rax + mov rax, qword [%%rSrc+ sizeof(qword)*3] + adc %%x4, rdx + + ;; ------------------ + ;; third pass 23 + ;; ------------------ + mul %%A + xor %%x5, %%x5 + add %%x4, rax + adc %%x5, rdx + + mov rax, [%%rSrc + sizeof(qword)*0] + + ;; --- double x0...x5 + xor %%A, %%A + add %%x0, %%x0 + adc %%x1, %%x1 + adc %%x2, %%x2 + adc %%x3, %%x3 + adc %%x4, %%x4 + adc %%x5, %%x5 + adc %%A, 0 + + mul rax + mov [%%rDst + sizeof(qword)*0], rax + mov rax, [%%rSrc + sizeof(qword)*1] + mov %%t0, rdx + + mul rax + add %%x0, %%t0 + adc %%x1, rax + mov rax, [%%rSrc + sizeof(qword)*2] + mov [%%rDst + sizeof(qword)*1], %%x0 + adc rdx, 0 + mov [%%rDst + sizeof(qword)*2], %%x1 + mov %%t0, rdx + + mul rax + add %%x2, %%t0 + adc %%x3, rax + mov rax, [%%rSrc + sizeof(qword)*3] + mov [%%rDst + sizeof(qword)*3], %%x2 + adc rdx, 0 + mov [%%rDst + sizeof(qword)*4], %%x3 + mov %%t0, rdx + + mul rax + add %%x4, %%t0 + adc %%x5, rax + mov [%%rDst + sizeof(qword)*5], %%x4 + adc rdx, 0 + mov [%%rDst + sizeof(qword)*6], %%x5 + add rdx, %%A + mov [%%rDst + sizeof(qword)*7], rdx +%endmacro + + +%macro SQR_192 13.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%x7 %3 + %xdefine %%x6 %4 + %xdefine %%x5 %5 + %xdefine %%x4 %6 + %xdefine %%x3 %7 + %xdefine %%x2 %8 + %xdefine %%x1 %9 + %xdefine %%x0 %10 + %xdefine %%A %11 + %xdefine %%x8 %12 + %xdefine %%t0 %13 + + mov %%A, %%x0 + + mov rax, %%x1 + mul %%A + mov %%x0, rax + mov %%x1, rdx + + MULADD1 %%x2, %%x1, %%A, %%x2 + + mov rax, qword [%%rSrc+ sizeof(qword)*1] + mul qword [%%rSrc+ sizeof(qword)*2] + xor %%x3, %%x3 + add %%x2, rax + adc %%x3, rdx + + xor %%A, %%A + add %%x0, %%x0 + adc %%x1, %%x1 + adc %%x2, %%x2 + adc %%x3, %%x3 + adc %%A, %%A + + mov rax, qword [%%rSrc+ sizeof(qword)*0] + mul rax + mov %%x4, rax + mov %%x5, rdx + + mov rax, qword [%%rSrc+ sizeof(qword)*1] + mul rax + mov %%x6, rax + mov %%x7, rdx + + mov rax, qword [%%rSrc+ sizeof(qword)*2] + mul rax + + mov qword [%%rDst+sizeof(qword)*0], %%x4 + add %%x5, %%x0 + mov qword [%%rDst+sizeof(qword)*1], %%x5 + adc %%x6, %%x1 + mov qword [%%rDst+sizeof(qword)*2], %%x6 + adc %%x7, %%x2 + mov qword [%%rDst+sizeof(qword)*3], %%x7 + adc rax, %%x3 + mov qword [%%rDst+sizeof(qword)*4], rax + adc rdx, %%A + mov qword [%%rDst+sizeof(qword)*5], rdx +%endmacro + +segment .text align=IPP_ALIGN_FACTOR + +;************************************************************* +;* Ipp64u cpSqrAdc_BNU_school(Ipp64u* pR; +;* const Ipp64u* pA, int aSize) +;* returns pR[aSize+aSize-1] +;* +;************************************************************* +align IPP_ALIGN_FACTOR +IPPASM cpSqrAdc_BNU_school,PUBLIC +%assign LOCAL_FRAME (3*sizeof(qword)) + USES_GPR rbx,rbp,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 3 + +;; +;; assignment +%xdefine rDst rdi +%xdefine rSrc rsi +%xdefine srcL rdx + +%xdefine A rcx +%xdefine x0 r8 +%xdefine x1 r9 +%xdefine x2 r10 +%xdefine x3 r11 +%xdefine x4 r12 +%xdefine x5 r13 +%xdefine x6 r14 +%xdefine x7 r15 +%xdefine x8 rbx +%xdefine t0 rbp + + cmp edx, 4 + jg .more_then_4 + + cmp edx, 3 + jg .SQR4 + je .SQR3 + jp .SQR2 + +align IPP_ALIGN_FACTOR +.SQR1: ;; len=1 + mov rax, qword [rsi] ; eax = a + mul rax ; eax = rL edx = rH + mov qword [rdi], rax ; + mov rax, rdx ; + mov qword [rdi+8], rdx ; + mov rax, rdx + REST_XMM + REST_GPR + ret + +align IPP_ALIGN_FACTOR +.SQR2: ;; len=2 + mov rax, qword [rsi] ; a[0] + mul qword [rsi+sizeof(qword)*1] ; a[0]*a[1] + xor A, A + mov x2, rax + mov x3, rdx + + mov rax, qword [rsi] ; a[0] + mul rax ; (x1:x0) = a[1]^2 + + add x2, x2 ; (A:x3:x2) = 2*a[0]*a[1] + adc x3, x3 + adc A, 0 + + mov x0, rax + mov x1, rdx + + mov rax, qword [rsi+sizeof(qword)*1]; a[1] + mul rax ; (rdx:rax) = a[1]^2 + + mov qword [rdi+sizeof(qword)*0], x0 + add x1, x2 + mov qword [rdi+sizeof(qword)*1], x1 + adc rax, x3 + mov qword [rdi+sizeof(qword)*2], rax + adc rdx, A + mov qword [rdi+sizeof(qword)*3], rdx + mov rax, rdx + REST_XMM + REST_GPR + ret + +align IPP_ALIGN_FACTOR +.SQR3: ;; len=3 + mov x0, [rSrc+ sizeof(qword)*0] + mov x1, [rSrc+ sizeof(qword)*1] + mov x2, [rSrc+ sizeof(qword)*2] + SQR_192 rDst, rSrc, x7, x6, x5, x4, x3, x2, x1, x0, A, x8, t0 + mov rax, rdx + REST_XMM + REST_GPR + ret + +align IPP_ALIGN_FACTOR +.SQR4: ;; len=4 + mov x0, [rSrc+ sizeof(qword)*0] + mov x1, [rSrc+ sizeof(qword)*1] + mov x2, [rSrc+ sizeof(qword)*2] + mov x3, [rSrc+ sizeof(qword)*3] + SQR_256 rDst, rSrc, x7, x6, x5, x4, x3, x2, x1, x0, A, x8, t0 + mov rax, rdx + REST_XMM + REST_GPR + ret + +align IPP_ALIGN_FACTOR +.more_then_4: + cmp edx, 8 + jg .general_case + + cmp edx, 7 + jg .SQR8 + je .SQR7 + jp .SQR6 + +align IPP_ALIGN_FACTOR +.SQR5: ;; len=5 + mov x0, [rSrc+ sizeof(qword)*0] + mov x1, [rSrc+ sizeof(qword)*1] + mov x2, [rSrc+ sizeof(qword)*2] + mov x3, [rSrc+ sizeof(qword)*3] + mov x4, [rSrc+ sizeof(qword)*4] + SQR_320 rDst, rSrc, x7, x6, x5, x4, x3, x2, x1, x0, A, x8, t0 + mov rax, rdx + REST_XMM + REST_GPR + ret + +align IPP_ALIGN_FACTOR +.SQR6: ;; len=6 + mov x0, [rSrc+ sizeof(qword)*0] + mov x1, [rSrc+ sizeof(qword)*1] + mov x2, [rSrc+ sizeof(qword)*2] + mov x3, [rSrc+ sizeof(qword)*3] + mov x4, [rSrc+ sizeof(qword)*4] + mov x5, [rSrc+ sizeof(qword)*5] + SQR_384 rDst, rSrc, x7, x6, x5, x4, x3, x2, x1, x0, A, x8, t0 + mov rax, rdx + REST_XMM + REST_GPR + ret + +align IPP_ALIGN_FACTOR +.SQR7: ;; len=7 + mov x0, [rSrc+ sizeof(qword)*0] + mov x1, [rSrc+ sizeof(qword)*1] + mov x2, [rSrc+ sizeof(qword)*2] + mov x3, [rSrc+ sizeof(qword)*3] + mov x4, [rSrc+ sizeof(qword)*4] + mov x5, [rSrc+ sizeof(qword)*5] + mov x6, [rSrc+ sizeof(qword)*6] + SQR_448 rDst, rSrc, x7, x6, x5, x4, x3, x2, x1, x0, A, x8, t0 + mov rax, rdx + REST_XMM + REST_GPR + ret + +align IPP_ALIGN_FACTOR +.SQR8: ;; len=8 + mov x0, [rSrc+ sizeof(qword)*0] + mov x1, [rSrc+ sizeof(qword)*1] + mov x2, [rSrc+ sizeof(qword)*2] + mov x3, [rSrc+ sizeof(qword)*3] + mov x4, [rSrc+ sizeof(qword)*4] + mov x5, [rSrc+ sizeof(qword)*5] + mov x6, [rSrc+ sizeof(qword)*6] + mov x7, [rSrc+ sizeof(qword)*7] + SQR_512 rDst, rSrc, x7, x6, x5, x4, x3, x2, x1, x0, A, x8, t0 + mov rax, rdx + REST_XMM + REST_GPR + ret + + + +;********** lenSrcA > 8 ************************************** +align IPP_ALIGN_FACTOR +.general_case: + +;; +;; stack structure: +%assign pDst 0 +%assign pSrc pDst+sizeof(qword) +%assign len pSrc+sizeof(qword) + +;; +;; assignment +%xdefine B0 r10 ; a[i], a[i+1] +%xdefine B1 r11 + +%xdefine T0 r12 ; temporary +%xdefine T1 r13 +%xdefine T2 r14 +%xdefine T3 r15 + +%xdefine idx rcx ; indexs +%xdefine idxt rbx + +%xdefine rSrc rsi +%xdefine rDst rdi + + movsxd rdx, edx ; expand length + mov [rsp+pDst], rdi ; save parameters + mov [rsp+pSrc], rsi + mov [rsp+len], rdx + + mov r8, rdx + mov rax, dword 2 + mov rbx, dword 1 + test r8, 1 + cmove rax, rbx ; delta = len&1? 2:1 + + sub rdx, rax ; len -= delta + lea rsi, [rsi+rax*sizeof(qword)] ; A' = A+delta + lea rdi, [rdi+rax*sizeof(qword)] ; R' = R+delta + lea rsi, [rsi+rdx*sizeof(qword)-4*sizeof(qword)] ; rsi = &A'[len-4] + lea rdi, [rdi+rdx*sizeof(qword)-4*sizeof(qword)] ; rdi = &R'[len-4] + + mov idx, dword 4 ; init + sub idx, rdx ; negative index + + test r8, 1 + jnz .init_odd_len_operation + +;********** odd number of passes (multiply only) ********************* +.init_even_len_operation: + mov B0, qword [rsi+idx*sizeof(qword)-sizeof(qword)] ; B0 = a[0] + mov rax, qword [rsi+idx*sizeof(qword)] ; a[1] + xor T0, T0 + cmp idx, 0 + jge .skip_mul1 + + mov idxt, idx + MULx1 rdi, rsi, idxt, B0, T0, T1, T2, T3 + +.skip_mul1: + cmp idxt, 1 + jne .fin_mulx1_3 ; idx=3 (effective len=4n+1) + ; .fin_mulx1_1 ; idx=1 (effective len=4n+3) + +.fin_mulx1_1: + sMULx1_4N_3_ELOG rdi, rsi, {add idx,2}, B0, T0,T1,T2,T3 + jmp .odd_pass_pairs +.fin_mulx1_3: + sMULx1_4N_1_ELOG rdi, rsi, {add idx,2}, B0, T0,T1,T2,T3 + jmp .even_pass_pairs + + +;********** even number of passes (multiply only) ********************* +.init_odd_len_operation: + mov B0, qword [rsi+idx*sizeof(qword)-2*sizeof(qword)] ; a[0] and a[1] + mov B1, qword [rsi+idx*sizeof(qword)-sizeof(qword)] + + mov rax, B1 ; a[0]*a[1] + mul B0 + + mov qword [rdi+idx*sizeof(qword)-sizeof(qword)], rax + mov rax, qword [rsi+idx*sizeof(qword)] ; a[2] + mov T0, rdx + + mul B0 ; a[0]*a[2] + xor T1, T1 + xor T2, T2 + add T0, rax + mov rax, qword [rsi+idx*sizeof(qword)] ; a[2] + adc T1, rdx + + cmp idx, 0 + jge .skip_mul_nx2 + + mov idxt, idx + MULx2 rdi, rsi, idxt, B0,B1, T0,T1,T2,T3 + +.skip_mul_nx2: + cmp idxt, 1 + jnz .fin_mul2x_3 ; idx=3 (effective len=4n+3) + ; .fin_mul2x_1 ; idx=1 (effective len=4n+1) + +.fin_mul2x_1: + sMULx2_4N_3_ELOG rdi, rsi, {add idx,2}, B0,B1, T0,T1,T2,T3 + add rdi, 2*sizeof(qword) + jmp .odd_pass_pairs +.fin_mul2x_3: + sMULx2_4N_1_ELOG rdi, rsi, {add idx,2}, B0,B1, T0,T1,T2,T3 + add rdi, 2*sizeof(qword) + +align IPP_ALIGN_FACTOR +.even_pass_pairs: + sMLAx2_PLOG {rdi+idx*sizeof(qword)}, {rsi+idx*sizeof(qword)}, B0,B1, T0,T1,T2,T3 + cmp idx, 0 + jge .skip1 + mov idxt, idx + MLAx2 rdi, rsi, idxt, B0,B1, T0,T1,T2,T3 +.skip1: + sMLAx2_4N_3_ELOG rdi, rsi, {add idx,2}, B0,B1, T0,T1,T2,T3 + add rdi, 2*sizeof(qword) + +.odd_pass_pairs: + sMLAx2_PLOG {rdi+idx*sizeof(qword)}, {rsi+idx*sizeof(qword)}, B0,B1, T0,T1,T2,T3 + cmp idx, 0 + jge .skip2 + mov idxt, idx + MLAx2 rdi, rsi, idxt, B0,B1, T0,T1,T2,T3 +.skip2: + sMLAx2_4N_1_ELOG rdi, rsi, {add idx,2}, B0,B1, T0,T1,T2,T3 + add rdi, 2*sizeof(qword) + + cmp idx, 4 + jl .even_pass_pairs + + +.add_diag: + mov rdi, [rsp+pDst] ; restore parameters + mov rsi, [rsp+pSrc] + mov rcx, [rsp+len] + + xor idxt, idxt + xor T0, T0 + xor T1, T1 + lea rax, [rdi+rcx*sizeof(qword)] + lea rsi, [rsi+rcx*sizeof(qword)] + mov qword [rdi], T0 ; r[0] = 0 + mov qword [rax+rcx*sizeof(qword)-sizeof(qword)], T0 ; r[2*len-1] = 0 + neg rcx + +align IPP_ALIGN_FACTOR +.add_diag_loop: + mov rax, qword [rsi+rcx*sizeof(qword)] ; a[i] + mul rax + mov T2, qword [rdi] ; r[2*i] + mov T3, qword [rdi+sizeof(qword)] ; r[2*i+1] + add T0, 1 + adc T2, T2 + adc T3, T3 + sbb T0, T0 + add T1, 1 + adc T2, rax + adc T3, rdx + sbb T1, T1 + mov qword [rdi], T2 + mov qword [rdi+sizeof(qword)], T3 + add rdi, sizeof(qword)*2 + add rcx, 1 + jnz .add_diag_loop + + mov rax, T3 ; r[2*len-1] + REST_XMM + REST_GPR + ret +ENDFUNC cpSqrAdc_BNU_school + + +%endif + +%endif ;; _ADCOX_NI_ENABLING_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnusubm7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnusubm7as.asm new file mode 100644 index 0000000..7c19b1e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpbnusubm7as.asm @@ -0,0 +1,330 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Big Number Operations +; +; Content: +; cpSub_BNU() +; +; + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_M7) + +segment .text align=IPP_ALIGN_FACTOR + + +;************************************************************* +;* Ipp64u cpSub_BNU(Ipp64u* pDst, +;* const Ipp64u* pSrc1, +;* const Ipp64u* pSrc2, +;* int len) +;* +;* returns borrow >= 0 +;************************************************************* +align IPP_ALIGN_FACTOR +IPPASM cpSub_BNU,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi + USES_XMM + COMP_ABI 4 + +; rdi = pDst +; rsi = pSrcA +; rdx = pSrcB +; rcx = len + + movsxd rcx, ecx ; unsigned length + xor rax, rax + + cmp rcx, 2 + jge .SUB_GE2 + +;********** lenSrcA == 1 ************************************* + add rax, rax + mov r8, qword [rsi] ; rsi = a + sbb r8, qword [rdx] ; r8 = a-b = s + mov qword [rdi], r8 ; save s + sbb rax, rax ; + jmp .FINAL + +;********** lenSrcA == 1 END ******************************** + +.SUB_GE2: + jg .SUB_GT2 + +;********** lenSrcA == 2 ************************************* + add rax, rax + mov r8, qword [rsi] ; r8 = a0 + sbb r8, qword [rdx] ; r8 = a0-b0 = s0 + mov r9, qword [rsi+8] ; r9 = a1 + sbb r9, qword [rdx+8] ; r9 = a1-b1 = s1 + mov qword [rdi], r8 ; save s0 + mov qword [rdi+8], r9 ; save s1 + sbb rax, rax ; rax = borrow + jmp .FINAL + +;********** lenSrcA == 2 END ********************************* + +.SUB_GT2: + cmp rcx, 4 + jge .SUB_GE4 + +;********** lenSrcA == 3 ************************************* + add rax, rax + mov r8, qword [rsi] ; r8 = a0 + sbb r8, qword [rdx] ; r8 = a0-b0 = s0 + mov r9, qword [rsi+8] ; r9 = a1 + sbb r9, qword [rdx+8] ; r9 = a1-b1 = s1 + mov r10, qword [rsi+16] ; r10 = a2 + sbb r10, qword [rdx+16] ; r10 = a2-b2 = s2 + mov qword [rdi], r8 ; save s0 + mov qword [rdi+8], r9 ; save s1 + mov qword [rdi+16], r10 ; save s2 + sbb rax, rax ; rax = borrow + jmp .FINAL + +;********** lenSrcA == 3 END ********************************* + +.SUB_GE4: + jg .SUB_GT4 + +;********** lenSrcA == 4 ************************************* + add rax, rax + mov r8, qword [rsi] ; r8 = a0 + sbb r8, qword [rdx] ; r8 = a0-b0 = s0 + mov r9, qword [rsi+8] ; r9 = a1 + sbb r9, qword [rdx+8] ; r9 = a1-b1 = s1 + mov r10, qword [rsi+16] ; r10 = a2 + sbb r10, qword [rdx+16] ; r10 = a2-b2 = s2 + mov r11, qword [rsi+24] ; r11 = a3 + sbb r11, qword [rdx+24] ; r11 = a3-b3 = s3 + mov qword [rdi], r8 ; save s0 + mov qword [rdi+8], r9 ; save s1 + mov qword [rdi+16], r10 ; save s2 + mov qword [rdi+24], r11 ; save s2 + sbb rax, rax ; rax = borrow + jmp .FINAL + +;********** lenSrcA == 4 END ********************************* + +.SUB_GT4: + cmp rcx, 6 + jge .SUB_GE6 + +;********** lenSrcA == 5 ************************************* + add rax, rax + mov r8, qword [rsi] ; r8 = a0 + sbb r8, qword [rdx] ; r8 = a0-b0 = s0 + mov r9, qword [rsi+8] ; r9 = a1 + sbb r9, qword [rdx+8] ; r9 = a1-b1 = s1 + mov r10, qword [rsi+16] ; r10 = a2 + sbb r10, qword [rdx+16] ; r10 = a2-b2 = s2 + mov r11, qword [rsi+24] ; r11 = a3 + sbb r11, qword [rdx+24] ; r11 = a3-b3 = s3 + mov rcx, qword [rsi+32] ; rcx = a4 + sbb rcx, qword [rdx+32] ; rcx = a4-b4 = s4 + mov qword [rdi], r8 ; save s0 + mov qword [rdi+8], r9 ; save s1 + mov qword [rdi+16], r10 ; save s2 + mov qword [rdi+24], r11 ; save s3 + mov qword [rdi+32], rcx ; save s4 + sbb rax, rax ; rax = borrow + jmp .FINAL + +;********** lenSrcA == 5 END ********************************* + +.SUB_GE6: + jg .SUB_GT6 + +;********** lenSrcA == 6 ************************************* + add rax, rax + mov r8, qword [rsi] ; r8 = a0 + sbb r8, qword [rdx] ; r8 = a0-b0 = s0 + mov r9, qword [rsi+8] ; r9 = a1 + sbb r9, qword [rdx+8] ; r9 = a1-b1 = s1 + mov r10, qword [rsi+16] ; r10 = a2 + sbb r10, qword [rdx+16] ; r10 = a2-b2 = s2 + mov r11, qword [rsi+24] ; r11 = a3 + sbb r11, qword [rdx+24] ; r11 = a3-b3 = s3 + mov rcx, qword [rsi+32] ; rcx = a4 + sbb rcx, qword [rdx+32] ; rcx = a4-b4 = s4 + mov rsi, qword [rsi+40] ; rsi = a5 + sbb rsi, qword [rdx+40] ; rsi = a5-b5 = s5 + mov qword [rdi], r8 ; save s0 + mov qword [rdi+8], r9 ; save s1 + mov qword [rdi+16], r10 ; save s2 + mov qword [rdi+24], r11 ; save s3 + mov qword [rdi+32], rcx ; save s4 + mov qword [rdi+40], rsi ; save s5 + sbb rax, rax ; rax = borrow + jmp .FINAL + +;********** lenSrcA == 6 END ********************************* + +.SUB_GT6: + cmp rcx, 8 + jge .SUB_GE8 + +.SUB_EQ7: +;********** lenSrcA == 7 ************************************* + add rax, rax + mov r8, qword [rsi] ; r8 = a0 + sbb r8, qword [rdx] ; r8 = a0-b0 = s0 + mov r9, qword [rsi+8] ; r9 = a1 + sbb r9, qword [rdx+8] ; r9 = a1-b1 = s1 + mov r10, qword [rsi+16] ; r10 = a2 + sbb r10, qword [rdx+16] ; r10 = a2-b2 = s2 + mov r11, qword [rsi+24] ; r11 = a3 + sbb r11, qword [rdx+24] ; r11 = a3-b3 = s3 + mov rcx, qword [rsi+32] ; rcx = a4 + sbb rcx, qword [rdx+32] ; rcx = a4-b4 = s4 + mov qword [rdi], r8 ; save s0 + mov r8, qword [rsi+40] ; r8 = a5 + sbb r8, qword [rdx+40] ; r8 = a5-b5 = s5 + mov rsi, qword [rsi+48] ; rsi = a6 + sbb rsi, qword [rdx+48] ; rsi = a6-b6 = s6 + mov qword [rdi+8], r9 ; save s1 + mov qword [rdi+16], r10 ; save s2 + mov qword [rdi+24], r11 ; save s3 + mov qword [rdi+32], rcx ; save s4 + mov qword [rdi+40], r8 ; save s5 + mov qword [rdi+48], rsi ; save s6 + sbb rax, rax ; rax = borrow + jmp .FINAL + +;********** lenSrcA == 7 END ********************************* + + +.SUB_GE8: + jg .SUB_GT8 + +;********** lenSrcA == 8 ************************************* + add rax, rax + mov r8, qword [rsi] ; r8 = a0 + sbb r8, qword [rdx] ; r8 = a0-b0 = s0 + mov r9, qword [rsi+8] ; r9 = a1 + sbb r9, qword [rdx+8] ; r9 = a1-b1 = s1 + mov r10, qword [rsi+16] ; r10 = a2 + sbb r10, qword [rdx+16] ; r10 = a2-b2 = s2 + mov r11, qword [rsi+24] ; r11 = a3 + sbb r11, qword [rdx+24] ; r11 = a3-b3 = s3 + mov rcx, qword [rsi+32] ; rcx = a4 + sbb rcx, qword [rdx+32] ; rcx = a4-b4 = s4 + mov qword [rdi], r8 ; save s0 + mov r8, qword [rsi+40] ; r8 = a5 + sbb r8, qword [rdx+40] ; r8 = a5-b5 = s5 + mov qword [rdi+8], r9 ; save s1 + mov r9, qword [rsi+48] ; r9 = a7 + sbb r9, qword [rdx+48] ; r9 = a7-b7 = s7 + mov rsi, qword [rsi+56] ; rsi = a6 + sbb rsi, qword [rdx+56] ; rsi = a6-b6 = s6 + mov qword [rdi+16], r10 ; save s2 + mov qword [rdi+24], r11 ; save s3 + mov qword [rdi+32], rcx ; save s4 + mov qword [rdi+40], r8 ; save s5 + mov qword [rdi+48], r9 ; save s6 + mov qword [rdi+56], rsi ; save s7 + sbb rax, rax ; rax = borrow + jmp .FINAL + +;********** lenSrcA == 8 END ********************************* + + +;********** lenSrcA > 8 ************************************* + +.SUB_GT8: + mov r8, rax + mov rax, rcx ; rax = len + and rcx, 3 ; + xor rcx, rax ; + lea rsi, [rsi+8*rcx] ; + lea rdx, [rdx+8*rcx] ; + lea rdi, [rdi+8*rcx] ; + neg rcx + add r8, r8 + jmp .SUB_GLOOP + +align IPP_ALIGN_FACTOR +.SUB_GLOOP: + mov r8, qword [rsi+8*rcx] ; r8 = a0 + mov r9, qword [rsi+8*rcx+8] ; r9 = a1 + mov r10, qword [rsi+8*rcx+16] ; r10 = a2 + mov r11, qword [rsi+8*rcx+24] ; r11 = a3 + sbb r8, qword [rdx+8*rcx] ; r8 = a0+b0 = r0 + sbb r9, qword [rdx+8*rcx+8] ; r9 = a1+b1 = r1 + sbb r10, qword [rdx+8*rcx+16] ; r10 = a2+b2 = r2 + sbb r11, qword [rdx+8*rcx+24] ; r11 = a3+b3 = r3 + mov qword [rdi+8*rcx], r8 ; + mov qword [rdi+8*rcx+8], r9 ; + mov qword [rdi+8*rcx+16], r10 ; + mov qword [rdi+8*rcx+24], r11 ; + lea rcx, [rcx+4] + jrcxz .SUB_LLAST0 + jmp .SUB_GLOOP + +.SUB_LLAST0: + sbb rcx, rcx + and rax, 3 + jz .FIN0 + +.SUB_LLOOP: + test rax, 2 + jz .SUB_LLAST1 + + add rcx, rcx + mov r8, qword [rsi] ; r8 = a0 + mov r9, qword [rsi+8] ; r9 = a1 + sbb r8, qword [rdx] ; r8 = a0+b0 = r0 + sbb r9, qword [rdx+8] ; r9 = a1+b1 = r1 + mov qword [rdi], r8 ; + mov qword [rdi+8], r9 ; + sbb rcx, rcx + test rax, 1 + jz .FIN0 + + add rsi, 16 + add rdx, 16 + add rdi, 16 + +.SUB_LLAST1: + add rcx, rcx + mov r8, qword [rsi] ; r8 = a0 + sbb r8, qword [rdx] ; r8 = a0+b0 = r0 + mov qword [rdi], r8 ; + sbb rcx, rcx + +.FIN0: + mov rax, rcx + +;******************* .FINAL *********************************************************** + +.FINAL: + neg rax + REST_XMM + REST_GPR + ret +ENDFUNC cpSub_BNU + + +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpdelay.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpdelay.asm new file mode 100644 index 0000000..db1d31b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpdelay.asm @@ -0,0 +1,52 @@ +;=============================================================================== +; Copyright 2022 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the "License"); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, software +; distributed under the License is distributed on an "AS IS" BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions and +; limitations under the License. +;=============================================================================== + +; +; +; Purpose: Delay function, intel64 version +; +; Content: +; _ippcpDelay() +; + +%include "asmdefs.inc" +%include "ia_32e.inc" + +segment .text align=IPP_ALIGN_FACTOR + +;*************************************************************** +;* Purpose: delay +;* +;* void _ippcpDelay(Ipp32u value) +;* +;* Caller = cpAESRandomNoise +;*************************************************************** +align IPP_ALIGN_FACTOR +IPPASM _ippcpDelay,PUBLIC + COMP_ABI 1 + +;; edi - delay value + + mov ecx,edi + +.Loop: + mov eax,ecx + dec ecx + test eax,eax + jnz .Loop + + ret +ENDFUNC _ippcpDelay diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmd5m7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmd5m7as.asm new file mode 100644 index 0000000..dcaa194 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmd5m7as.asm @@ -0,0 +1,320 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to MD5 +; (derived from the RSA Data Security, Inc. MD5 Message-Digest Algorithm) +; +; Content: +; UpdateMD5 +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_MD5_) +%if (_IPP32E >= _IPP32E_M7) + + +;; +;; Magic functions defined in RFC 1321 +;; +%macro MAGIC_F 4.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%Y %3 + %xdefine %%Z %4 + + mov %%F,%%Z + xor %%F,%%Y + and %%F,%%X + xor %%F,%%Z +%endmacro + + +%macro MAGIC_G 4.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%Y %3 + %xdefine %%Z %4 + + MAGIC_F %%F,%%Z,%%X,%%Y +%endmacro + + +%macro MAGIC_H 4.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%Y %3 + %xdefine %%Z %4 + + mov %%F,%%Z + xor %%F,%%Y + xor %%F,%%X +%endmacro + + +%macro MAGIC_I 4.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%Y %3 + %xdefine %%Z %4 + + mov %%F,%%Z + not %%F + or %%F,%%X + xor %%F,%%Y +%endmacro + + +%macro ROT_L 2.nolist + %xdefine %%r %1 + %xdefine %%nbits %2 + + %if (_IPP32E >= _IPP32E_L9) + rorx %%r,%%r,(32-%%nbits) + %elif (_IPP32E >= _IPP32E_Y8) + shld %%r,%%r,%%nbits + %else + rol %%r,%%nbits + %endif +%endmacro + + +;; +;; single MD5 step +;; +;; A = B +ROL32((A +MAGIC(B,C,D) +data +const), nrot) +;; +%macro xMD5_STEP 10.nolist + %xdefine %%MAGIC_FUN %1 + %xdefine %%A %2 + %xdefine %%B %3 + %xdefine %%C %4 + %xdefine %%D %5 + %xdefine %%FUN %6 + %xdefine %%TMP %7 + %xdefine %%data %8 + %xdefine %%MD5const %9 + %xdefine %%nrot %10 + + mov %%TMP,[%%data] +; lea A,[TMP+A+MD5const] + add %%A, %%MD5const + add %%A, %%TMP + %%MAGIC_FUN %%FUN, %%B,%%C,%%D + add %%A,%%FUN + rol %%A,%%nrot + add %%A,%%B +%endmacro + + +%macro MD5_STEP 10.nolist + %xdefine %%MAGIC_FUN %1 + %xdefine %%A %2 + %xdefine %%B %3 + %xdefine %%C %4 + %xdefine %%D %5 + %xdefine %%FUN %6 + %xdefine %%TMP %7 + %xdefine %%data %8 + %xdefine %%MD5const %9 + %xdefine %%nrot %10 + + mov %%TMP,[%%data] + add %%A, %%MD5const + add %%A, %%TMP + %%MAGIC_FUN %%FUN, %%B,%%C,%%D + add %%A,%%FUN + ROT_L %%A,%%nrot + add %%A,%%B +%endmacro + + +segment .text align=IPP_ALIGN_FACTOR + + +;***************************************************************************************** +;* Purpose: Update internal digest according to message block +;* +;* void UpdateMD5(DigestMD5digest, const Ipp32u* mblk, int mlen, const void* pParam) +;* +;***************************************************************************************** + +;; +;; MD5 left rotations (number of bits) +;; +%assign rot11 7 +%assign rot12 12 +%assign rot13 17 +%assign rot14 22 +%assign rot21 5 +%assign rot22 9 +%assign rot23 14 +%assign rot24 20 +%assign rot31 4 +%assign rot32 11 +%assign rot33 16 +%assign rot34 23 +%assign rot41 6 +%assign rot42 10 +%assign rot43 15 +%assign rot44 21 + +align IPP_ALIGN_FACTOR + +;; +;; Lib = M7 +;; +;; Caller = ippsMD5Update +;; Caller = ippsMD5Final +;; Caller = ippsMD5MessageDigest +;; +;; Caller = ippsHMACMD5Update +;; Caller = ippsHMACMD5Final +;; Caller = ippsHMACMD5MessageDigest +;; + +align IPP_ALIGN_FACTOR +IPPASM UpdateMD5,PUBLIC + USES_GPR rbx,rsi,rdi,r12 + USES_XMM + COMP_ABI 4 + +;; rdi = hash +;; rsi = data buffer +;; rdx = buffer length +;; rcx = address of MD5 constants + +%assign MBS_MD5 64 + + movsxd r12, edx + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +.md5_block_loop: +; prefetcht0 [rsi+64] + +;; +;; init A, B, C, D by the internal digest +;; + mov r8d, [rdi+0*4] ; r8d = digest[0] (A) + mov r9d, [rdi+1*4] ; r9d = digest[1] (B) + mov r10d,[rdi+2*4] ; r10d = digest[2] (C) + mov r11d,[rdi+3*4] ; r11d = digest[3] (D) + +;; +;; perform 0-63 steps +;; +;; MAGIC A, B, C, D, FUN,TMP, pData, cnt, nrot +;; ------------------------------------------------------------------- + MD5_STEP MAGIC_F, r8d, r9d, r10d,r11d, edx,ecx, {rsi+ 0*4}, 0d76aa478h, rot11 + MD5_STEP MAGIC_F, r11d,r8d, r9d, r10d, edx,ecx, {rsi+ 1*4}, 0e8c7b756h, rot12 + MD5_STEP MAGIC_F, r10d,r11d,r8d, r9d, edx,ecx, {rsi+ 2*4}, 0242070dbh, rot13 + MD5_STEP MAGIC_F, r9d, r10d,r11d,r8d, edx,ecx, {rsi+ 3*4}, 0c1bdceeeh, rot14 + MD5_STEP MAGIC_F, r8d, r9d, r10d,r11d, edx,ecx, {rsi+ 4*4}, 0f57c0fafh, rot11 + MD5_STEP MAGIC_F, r11d,r8d, r9d, r10d, edx,ecx, {rsi+ 5*4}, 04787c62ah, rot12 + MD5_STEP MAGIC_F, r10d,r11d,r8d, r9d, edx,ecx, {rsi+ 6*4}, 0a8304613h, rot13 + MD5_STEP MAGIC_F, r9d, r10d,r11d,r8d, edx,ecx, {rsi+ 7*4}, 0fd469501h, rot14 + MD5_STEP MAGIC_F, r8d, r9d, r10d,r11d, edx,ecx, {rsi+ 8*4}, 0698098d8h, rot11 + MD5_STEP MAGIC_F, r11d,r8d, r9d, r10d, edx,ecx, {rsi+ 9*4}, 08b44f7afh, rot12 + MD5_STEP MAGIC_F, r10d,r11d,r8d, r9d, edx,ecx, {rsi+10*4}, 0ffff5bb1h, rot13 + MD5_STEP MAGIC_F, r9d, r10d,r11d,r8d, edx,ecx, {rsi+11*4}, 0895cd7beh, rot14 + MD5_STEP MAGIC_F, r8d, r9d, r10d,r11d, edx,ecx, {rsi+12*4}, 06b901122h, rot11 + MD5_STEP MAGIC_F, r11d,r8d, r9d, r10d, edx,ecx, {rsi+13*4}, 0fd987193h, rot12 + MD5_STEP MAGIC_F, r10d,r11d,r8d, r9d, edx,ecx, {rsi+14*4}, 0a679438eh, rot13 + MD5_STEP MAGIC_F, r9d, r10d,r11d,r8d, edx,ecx, {rsi+15*4}, 049b40821h, rot14 + + MD5_STEP MAGIC_G, r8d, r9d, r10d,r11d, edx,ecx, {rsi+ 1*4}, 0f61e2562h, rot21 + MD5_STEP MAGIC_G, r11d,r8d, r9d, r10d, edx,ecx, {rsi+ 6*4}, 0c040b340h, rot22 + MD5_STEP MAGIC_G, r10d,r11d,r8d, r9d, edx,ecx, {rsi+11*4}, 0265e5a51h, rot23 + MD5_STEP MAGIC_G, r9d, r10d,r11d,r8d, edx,ecx, {rsi+ 0*4}, 0e9b6c7aah, rot24 + MD5_STEP MAGIC_G, r8d, r9d, r10d,r11d, edx,ecx, {rsi+ 5*4}, 0d62f105dh, rot21 + MD5_STEP MAGIC_G, r11d,r8d, r9d, r10d, edx,ecx, {rsi+10*4}, 002441453h, rot22 + MD5_STEP MAGIC_G, r10d,r11d,r8d, r9d, edx,ecx, {rsi+15*4}, 0d8a1e681h, rot23 + MD5_STEP MAGIC_G, r9d, r10d,r11d,r8d, edx,ecx, {rsi+ 4*4}, 0e7d3fbc8h, rot24 + MD5_STEP MAGIC_G, r8d, r9d, r10d,r11d, edx,ecx, {rsi+ 9*4}, 021e1cde6h, rot21 + MD5_STEP MAGIC_G, r11d,r8d, r9d, r10d, edx,ecx, {rsi+14*4}, 0c33707d6h, rot22 + MD5_STEP MAGIC_G, r10d,r11d,r8d, r9d, edx,ecx, {rsi+ 3*4}, 0f4d50d87h, rot23 + MD5_STEP MAGIC_G, r9d, r10d,r11d,r8d, edx,ecx, {rsi+ 8*4}, 0455a14edh, rot24 + MD5_STEP MAGIC_G, r8d, r9d, r10d,r11d, edx,ecx, {rsi+13*4}, 0a9e3e905h, rot21 + MD5_STEP MAGIC_G, r11d,r8d, r9d, r10d, edx,ecx, {rsi+ 2*4}, 0fcefa3f8h, rot22 + MD5_STEP MAGIC_G, r10d,r11d,r8d, r9d, edx,ecx, {rsi+ 7*4}, 0676f02d9h, rot23 + MD5_STEP MAGIC_G, r9d, r10d,r11d,r8d, edx,ecx, {rsi+12*4}, 08d2a4c8ah, rot24 + + MD5_STEP MAGIC_H, r8d, r9d, r10d,r11d, edx,ecx, {rsi+ 5*4}, 0fffa3942h, rot31 + MD5_STEP MAGIC_H, r11d,r8d, r9d, r10d, edx,ecx, {rsi+ 8*4}, 08771f681h, rot32 + MD5_STEP MAGIC_H, r10d,r11d,r8d, r9d, edx,ecx, {rsi+11*4}, 06d9d6122h, rot33 + MD5_STEP MAGIC_H, r9d, r10d,r11d,r8d, edx,ecx, {rsi+14*4}, 0fde5380ch, rot34 + MD5_STEP MAGIC_H, r8d, r9d, r10d,r11d, edx,ecx, {rsi+ 1*4}, 0a4beea44h, rot31 + MD5_STEP MAGIC_H, r11d,r8d, r9d, r10d, edx,ecx, {rsi+ 4*4}, 04bdecfa9h, rot32 + MD5_STEP MAGIC_H, r10d,r11d,r8d, r9d, edx,ecx, {rsi+ 7*4}, 0f6bb4b60h, rot33 + MD5_STEP MAGIC_H, r9d, r10d,r11d,r8d, edx,ecx, {rsi+10*4}, 0bebfbc70h, rot34 + MD5_STEP MAGIC_H, r8d, r9d, r10d,r11d, edx,ecx, {rsi+13*4}, 0289b7ec6h, rot31 + MD5_STEP MAGIC_H, r11d,r8d, r9d, r10d, edx,ecx, {rsi+ 0*4}, 0eaa127fah, rot32 + MD5_STEP MAGIC_H, r10d,r11d,r8d, r9d, edx,ecx, {rsi+ 3*4}, 0d4ef3085h, rot33 + MD5_STEP MAGIC_H, r9d, r10d,r11d,r8d, edx,ecx, {rsi+ 6*4}, 004881d05h, rot34 + MD5_STEP MAGIC_H, r8d, r9d, r10d,r11d, edx,ecx, {rsi+ 9*4}, 0d9d4d039h, rot31 + MD5_STEP MAGIC_H, r11d,r8d, r9d, r10d, edx,ecx, {rsi+12*4}, 0e6db99e5h, rot32 + MD5_STEP MAGIC_H, r10d,r11d,r8d, r9d, edx,ecx, {rsi+15*4}, 01fa27cf8h, rot33 + MD5_STEP MAGIC_H, r9d, r10d,r11d,r8d, edx,ecx, {rsi+ 2*4}, 0c4ac5665h, rot34 + + MD5_STEP MAGIC_I, r8d, r9d, r10d,r11d, edx,ecx, {rsi+ 0*4}, 0f4292244h, rot41 + MD5_STEP MAGIC_I, r11d,r8d, r9d, r10d, edx,ecx, {rsi+ 7*4}, 0432aff97h, rot42 + MD5_STEP MAGIC_I, r10d,r11d,r8d, r9d, edx,ecx, {rsi+14*4}, 0ab9423a7h, rot43 + MD5_STEP MAGIC_I, r9d, r10d,r11d,r8d, edx,ecx, {rsi+ 5*4}, 0fc93a039h, rot44 + MD5_STEP MAGIC_I, r8d, r9d, r10d,r11d, edx,ecx, {rsi+12*4}, 0655b59c3h, rot41 + MD5_STEP MAGIC_I, r11d,r8d, r9d, r10d, edx,ecx, {rsi+ 3*4}, 08f0ccc92h, rot42 + MD5_STEP MAGIC_I, r10d,r11d,r8d, r9d, edx,ecx, {rsi+10*4}, 0ffeff47dh, rot43 + MD5_STEP MAGIC_I, r9d, r10d,r11d,r8d, edx,ecx, {rsi+ 1*4}, 085845dd1h, rot44 + MD5_STEP MAGIC_I, r8d, r9d, r10d,r11d, edx,ecx, {rsi+ 8*4}, 06fa87e4fh, rot41 + MD5_STEP MAGIC_I, r11d,r8d, r9d, r10d, edx,ecx, {rsi+15*4}, 0fe2ce6e0h, rot42 + MD5_STEP MAGIC_I, r10d,r11d,r8d, r9d, edx,ecx, {rsi+ 6*4}, 0a3014314h, rot43 + MD5_STEP MAGIC_I, r9d, r10d,r11d,r8d, edx,ecx, {rsi+13*4}, 04e0811a1h, rot44 + MD5_STEP MAGIC_I, r8d, r9d, r10d,r11d, edx,ecx, {rsi+ 4*4}, 0f7537e82h, rot41 + MD5_STEP MAGIC_I, r11d,r8d, r9d, r10d, edx,ecx, {rsi+11*4}, 0bd3af235h, rot42 + MD5_STEP MAGIC_I, r10d,r11d,r8d, r9d, edx,ecx, {rsi+ 2*4}, 02ad7d2bbh, rot43 + MD5_STEP MAGIC_I, r9d, r10d,r11d,r8d, edx,ecx, {rsi+ 9*4}, 0eb86d391h, rot44 + +;; +;; update digest +;; + add [rdi+0*4],r8d ; advance digest + add [rdi+1*4],r9d + add [rdi+2*4],r10d + add [rdi+3*4],r11d + + add rsi, MBS_MD5 + sub r12, MBS_MD5 + jg .md5_block_loop + + REST_XMM + REST_GPR + ret +ENDFUNC UpdateMD5 + + +%endif ;; _IPP32E >= _IPP32E_M7 +%endif ;; _ENABLE_ALG_MD5_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmontmul1024_avx2as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmontmul1024_avx2as.asm new file mode 100644 index 0000000..9752c18 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmontmul1024_avx2as.asm @@ -0,0 +1,378 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Big Number Operations +; +; Content: +; cpMontMul1024_avx2() +; + +%include "asmdefs.inc" +%include "ia_32e.inc" +;include pcpbnumulschool.inc +;include pcpvariant.inc + +%if (_IPP32E >= _IPP32E_L9) + +segment .text align=IPP_ALIGN_FACTOR + +%assign DIGIT_BITS 27 +%assign DIGIT_MASK (1 << DIGIT_BITS) -1 + +;************************************************************* +;* void cpMontMul1024_avx2(Ipp64u* pR, +;* const Ipp64u* pA, +;* const Ipp64u* pB, +;* const Ipp64u* pModulo, int mSize, +;* Ipp64u m0) +;************************************************************* +align IPP_ALIGN_FACTOR +IPPASM cpMontMul1024_avx2,PUBLIC +%assign LOCAL_FRAME sizeof(ymmword) + USES_GPR rsi,rdi,rbx,rbp,r12,r13,r14 + USES_XMM_AVX ymm6,ymm7,ymm8,ymm9,ymm10,ymm11,ymm12,ymm13,ymm14 + COMP_ABI 6 + + mov rbp,rdx ; pointer to B operand + movsxd r8, r8d ; redLen value counter + +;; clear results and buffers + vzeroall + +;; expands A and M operands + vmovdqu ymmword [rsi+r8*sizeof(qword)], ymm0 + vmovdqu ymmword [rcx+r8*sizeof(qword)], ymm0 + + xor r10, r10 ; ac0 = 0 + + vmovdqu ymmword [rsp], ymm0 ; {r3:r2:r1;R0} = 0 + +align IPP_ALIGN_FACTOR +;; +;; process b[] by quadruples (b[j], b[j+1], b[j+2] and b[j+3]) per pass +;; +.loop4_B: + mov rbx, qword [rbp] ; rbx = b[j] + vpbroadcastq ymm0, qword [rbp] + + mov r10, rbx ; ac0 = pa[0]*b[j]+pr[0] + imul r10, qword [rsi] + add r10, qword [rsp] + mov rdx, r10 ; y0 = (ac0*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + mov r11, rbx ; ac1 = pa[1]*b[j]+pr[1] + imul r11, qword [rsi+sizeof(qword)] + add r11, qword [rsp+sizeof(qword)] + mov r12, rbx ; ac2 = pa[2]*b[j]+pr[2] + imul r12, qword [rsi+sizeof(qword)*2] + add r12, qword [rsp+sizeof(qword)*2] + mov r13, rbx ; ac3 = pa[3]*b[j]+pr[3] + imul r13, qword [rsi+sizeof(qword)*3] + add r13, qword [rsp+sizeof(qword)*3] + vmovd xmm11, edx + vpbroadcastq ymm11, xmm11 + + vpmuludq ymm12, ymm0, ymmword [rsi+sizeof(ymmword)] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm0, ymmword [rsi+sizeof(ymmword)*2] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm14, ymm0, ymmword [rsi+sizeof(ymmword)*3] + vpaddq ymm3, ymm3, ymm14 + vpmuludq ymm12, ymm0, ymmword [rsi+sizeof(ymmword)*4] + vpaddq ymm4, ymm4, ymm12 + vpmuludq ymm13, ymm0, ymmword [rsi+sizeof(ymmword)*5] + vpaddq ymm5, ymm5, ymm13 + vpmuludq ymm14, ymm0, ymmword [rsi+sizeof(ymmword)*6] + vpaddq ymm6, ymm6, ymm14 + vpmuludq ymm12, ymm0, ymmword [rsi+sizeof(ymmword)*7] + vpaddq ymm7, ymm7, ymm12 + vpmuludq ymm13, ymm0, ymmword [rsi+sizeof(ymmword)*8] + vpaddq ymm8, ymm8, ymm13 + vpmuludq ymm14, ymm0, ymmword [rsi+sizeof(ymmword)*9] + vpaddq ymm9, ymm9, ymm14 + + mov rax, rdx ; ac0 += pn[0]*y0 + imul rax, qword [rcx] + add r10, rax + shr r10, DIGIT_BITS + mov rax, rdx ; ac1 += pn[1]*y0 + imul rax, qword [rcx+sizeof(qword)] + add r11, rax + add r11, r10 + mov rax, rdx ; ac2 += pn[2]*y0 + imul rax, qword [rcx+sizeof(qword)*2] + add r12, rax + mov rax, rdx ; ac3 += pn[3]*y0 + imul rax, qword [rcx+sizeof(qword)*3] + add r13, rax + + vpmuludq ymm12, ymm11, ymmword [rcx+sizeof(ymmword)] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm11, ymmword [rcx+sizeof(ymmword)*2] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm14, ymm11, ymmword [rcx+sizeof(ymmword)*3] + vpaddq ymm3, ymm3, ymm14 + vpmuludq ymm12, ymm11, ymmword [rcx+sizeof(ymmword)*4] + vpaddq ymm4, ymm4, ymm12 + vpmuludq ymm13, ymm11, ymmword [rcx+sizeof(ymmword)*5] + vpaddq ymm5, ymm5, ymm13 + vpmuludq ymm14, ymm11, ymmword [rcx+sizeof(ymmword)*6] + vpaddq ymm6, ymm6, ymm14 + vpmuludq ymm12, ymm11, ymmword [rcx+sizeof(ymmword)*7] + vpaddq ymm7, ymm7, ymm12 + vpmuludq ymm13, ymm11, ymmword [rcx+sizeof(ymmword)*8] + vpaddq ymm8, ymm8, ymm13 + vpmuludq ymm14, ymm11, ymmword [rcx+sizeof(ymmword)*9] + vpaddq ymm9, ymm9, ymm14 +;; ------------------------------------------------------------ + + mov rbx, qword [rbp+sizeof(qword)] ; rbx = b[j+1] + vpbroadcastq ymm0, qword [rbp+sizeof(qword)] + mov rax, rbx ; ac1 += pa[0]*b[j+1] + imul rax, qword [rsi] + add r11, rax + mov rdx, r11 ; y1 = (ac1*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + mov rax, rbx ; ac2 += pa[1]*b[j+1] + imul rax, qword [rsi+sizeof(qword)] + add r12, rax + mov rax, rbx ; ac3 += pa[2]*b[j+1] + imul rax, qword [rsi+sizeof(qword)*2] + add r13, rax + vmovd xmm11, edx + vpbroadcastq ymm11, xmm11 + + vpmuludq ymm12, ymm0, ymmword [rsi+sizeof(ymmword)-sizeof(qword)] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm0, ymmword [rsi+sizeof(ymmword)*2-sizeof(qword)] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm14, ymm0, ymmword [rsi+sizeof(ymmword)*3-sizeof(qword)] + vpaddq ymm3, ymm3, ymm14 + vpmuludq ymm12, ymm0, ymmword [rsi+sizeof(ymmword)*4-sizeof(qword)] + vpaddq ymm4, ymm4, ymm12 + vpmuludq ymm13, ymm0, ymmword [rsi+sizeof(ymmword)*5-sizeof(qword)] + vpaddq ymm5, ymm5, ymm13 + vpmuludq ymm14, ymm0, ymmword [rsi+sizeof(ymmword)*6-sizeof(qword)] + vpaddq ymm6, ymm6, ymm14 + vpmuludq ymm12, ymm0, ymmword [rsi+sizeof(ymmword)*7-sizeof(qword)] + vpaddq ymm7, ymm7, ymm12 + vpmuludq ymm13, ymm0, ymmword [rsi+sizeof(ymmword)*8-sizeof(qword)] + vpaddq ymm8, ymm8, ymm13 + vpmuludq ymm14, ymm0, ymmword [rsi+sizeof(ymmword)*9-sizeof(qword)] + vpaddq ymm9, ymm9, ymm14 + + mov rax, rdx ; ac1 += pn[0]*y1 + imul rax, qword [rcx] + add r11, rax + shr r11, DIGIT_BITS + mov rax, rdx ; ac2 += pn[1]*y1 + imul rax, qword [rcx+sizeof(qword)] + add r12, rax + add r12, r11 + mov rax, rdx ; ac3 += pn[2]*y1 + imul rax, qword [rcx+sizeof(qword)*2] + add r13, rax + + vpmuludq ymm12, ymm11, ymmword [rcx+sizeof(ymmword)-sizeof(qword)] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm11, ymmword [rcx+sizeof(ymmword)*2-sizeof(qword)] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm14, ymm11, ymmword [rcx+sizeof(ymmword)*3-sizeof(qword)] + vpaddq ymm3, ymm3, ymm14 + vpmuludq ymm12, ymm11, ymmword [rcx+sizeof(ymmword)*4-sizeof(qword)] + vpaddq ymm4, ymm4, ymm12 + vpmuludq ymm13, ymm11, ymmword [rcx+sizeof(ymmword)*5-sizeof(qword)] + vpaddq ymm5, ymm5, ymm13 + vpmuludq ymm14, ymm11, ymmword [rcx+sizeof(ymmword)*6-sizeof(qword)] + vpaddq ymm6, ymm6, ymm14 + vpmuludq ymm12, ymm11, ymmword [rcx+sizeof(ymmword)*7-sizeof(qword)] + vpaddq ymm7, ymm7, ymm12 + vpmuludq ymm13, ymm11, ymmword [rcx+sizeof(ymmword)*8-sizeof(qword)] + vpaddq ymm8, ymm8, ymm13 + vpmuludq ymm14, ymm11, ymmword [rcx+sizeof(ymmword)*9-sizeof(qword)] + vpaddq ymm9, ymm9, ymm14 + sub r8, 2 + jz .exit_loop_B +;; ------------------------------------------------------------ + + mov rbx, qword [rbp+sizeof(qword)*2] ; rbx = b[j+2] + vpbroadcastq ymm0, qword [rbp+sizeof(qword)*2] + mov rax, rbx ; ac2 += pa[0]*b[j+2] + imul rax, qword [rsi] + add r12, rax + mov rdx, r12 ; y2 = (ac2*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + mov rax, rbx ; ac2 += pa[0]*b[j+2] + imul rax, qword [rsi+sizeof(qword)] + add r13, rax + vmovd xmm11, edx + vpbroadcastq ymm11, xmm11 + + vpmuludq ymm12, ymm0, ymmword [rsi+sizeof(ymmword)-sizeof(qword)*2] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm0, ymmword [rsi+sizeof(ymmword)*2-sizeof(qword)*2] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm14, ymm0, ymmword [rsi+sizeof(ymmword)*3-sizeof(qword)*2] + vpaddq ymm3, ymm3, ymm14 + vpmuludq ymm12, ymm0, ymmword [rsi+sizeof(ymmword)*4-sizeof(qword)*2] + vpaddq ymm4, ymm4, ymm12 + vpmuludq ymm13, ymm0, ymmword [rsi+sizeof(ymmword)*5-sizeof(qword)*2] + vpaddq ymm5, ymm5, ymm13 + vpmuludq ymm14, ymm0, ymmword [rsi+sizeof(ymmword)*6-sizeof(qword)*2] + vpaddq ymm6, ymm6, ymm14 + vpmuludq ymm12, ymm0, ymmword [rsi+sizeof(ymmword)*7-sizeof(qword)*2] + vpaddq ymm7, ymm7, ymm12 + vpmuludq ymm13, ymm0, ymmword [rsi+sizeof(ymmword)*8-sizeof(qword)*2] + vpaddq ymm8, ymm8, ymm13 + vpmuludq ymm14, ymm0, ymmword [rsi+sizeof(ymmword)*9-sizeof(qword)*2] + vpaddq ymm9, ymm9, ymm14 + + mov rax, rdx ; ac2 += pn[0]*y2 + imul rax, qword [rcx] + add r12, rax + shr r12, DIGIT_BITS + mov rax, rdx ; ac3 += pn[1]*y2 + imul rax, qword [rcx+sizeof(qword)] + add r13, rax + add r13, r12 + + vpmuludq ymm12, ymm11, ymmword [rcx+sizeof(ymmword)-sizeof(qword)*2] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm11, ymmword [rcx+sizeof(ymmword)*2-sizeof(qword)*2] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm14, ymm11, ymmword [rcx+sizeof(ymmword)*3-sizeof(qword)*2] + vpaddq ymm3, ymm3, ymm14 + vpmuludq ymm12, ymm11, ymmword [rcx+sizeof(ymmword)*4-sizeof(qword)*2] + vpaddq ymm4, ymm4, ymm12 + vpmuludq ymm13, ymm11, ymmword [rcx+sizeof(ymmword)*5-sizeof(qword)*2] + vpaddq ymm5, ymm5, ymm13 + vpmuludq ymm14, ymm11, ymmword [rcx+sizeof(ymmword)*6-sizeof(qword)*2] + vpaddq ymm6, ymm6, ymm14 + vpmuludq ymm12, ymm11, ymmword [rcx+sizeof(ymmword)*7-sizeof(qword)*2] + vpaddq ymm7, ymm7, ymm12 + vpmuludq ymm13, ymm11, ymmword [rcx+sizeof(ymmword)*8-sizeof(qword)*2] + vpaddq ymm8, ymm8, ymm13 + vpmuludq ymm14, ymm11, ymmword [rcx+sizeof(ymmword)*9-sizeof(qword)*2] + vpaddq ymm9, ymm9, ymm14 +;; ------------------------------------------------------------ + + mov rbx, qword [rbp+sizeof(qword)*3] ; rbx = b[j+3] + vpbroadcastq ymm0, qword [rbp+sizeof(qword)*3] + imul rbx, qword [rsi] ; ac3 += pa[0]*b[j+3] + add r13, rbx + mov rdx, r13 ; y3 = (ac3*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + vmovd xmm11, edx + vpbroadcastq ymm11, xmm11 + + vpmuludq ymm12, ymm0, ymmword [rsi+sizeof(ymmword)-sizeof(qword)*3] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm0, ymmword [rsi+sizeof(ymmword)*2-sizeof(qword)*3] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm14, ymm0, ymmword [rsi+sizeof(ymmword)*3-sizeof(qword)*3] + vpaddq ymm3, ymm3, ymm14 + vpmuludq ymm12, ymm0, ymmword [rsi+sizeof(ymmword)*4-sizeof(qword)*3] + vpaddq ymm4, ymm4, ymm12 + vpmuludq ymm13, ymm0, ymmword [rsi+sizeof(ymmword)*5-sizeof(qword)*3] + vpaddq ymm5, ymm5, ymm13 + vpmuludq ymm14, ymm0, ymmword [rsi+sizeof(ymmword)*6-sizeof(qword)*3] + vpaddq ymm6, ymm6, ymm14 + vpmuludq ymm12, ymm0, ymmword [rsi+sizeof(ymmword)*7-sizeof(qword)*3] + vpaddq ymm7, ymm7, ymm12 + vpmuludq ymm13, ymm0, ymmword [rsi+sizeof(ymmword)*8-sizeof(qword)*3] + vpaddq ymm8, ymm8, ymm13 + vpmuludq ymm14, ymm0, ymmword [rsi+sizeof(ymmword)*9-sizeof(qword)*3] + vpaddq ymm9, ymm9, ymm14 + vpmuludq ymm10, ymm0, ymmword [rsi+sizeof(ymmword)*10-sizeof(qword)*3] + + imul rdx, qword [rcx] ; ac3 += pn[0]*y3 + add r13, rdx + shr r13, DIGIT_BITS + vmovq xmm14, r13 + + vpmuludq ymm12, ymm11, ymmword [rcx+sizeof(ymmword)-sizeof(qword)*3] + vpaddq ymm1, ymm1, ymm12 + vpaddq ymm1, ymm1, ymm14 + vmovdqu ymmword [rsp], ymm1 + vpmuludq ymm13, ymm11, ymmword [rcx+sizeof(ymmword)*2-sizeof(qword)*3] + vpaddq ymm1, ymm2, ymm13 + vpmuludq ymm14, ymm11, ymmword [rcx+sizeof(ymmword)*3-sizeof(qword)*3] + vpaddq ymm2, ymm3, ymm14 + vpmuludq ymm12, ymm11, ymmword [rcx+sizeof(ymmword)*4-sizeof(qword)*3] + vpaddq ymm3, ymm4, ymm12 + vpmuludq ymm13, ymm11, ymmword [rcx+sizeof(ymmword)*5-sizeof(qword)*3] + vpaddq ymm4, ymm5, ymm13 + vpmuludq ymm14, ymm11, ymmword [rcx+sizeof(ymmword)*6-sizeof(qword)*3] + vpaddq ymm5, ymm6, ymm14 + vpmuludq ymm12, ymm11, ymmword [rcx+sizeof(ymmword)*7-sizeof(qword)*3] + vpaddq ymm6, ymm7, ymm12 + vpmuludq ymm13, ymm11, ymmword [rcx+sizeof(ymmword)*8-sizeof(qword)*3] + vpaddq ymm7, ymm8, ymm13 + vpmuludq ymm14, ymm11, ymmword [rcx+sizeof(ymmword)*9-sizeof(qword)*3] + vpaddq ymm8, ymm9, ymm14 + vpmuludq ymm12, ymm11, ymmword [rcx+sizeof(ymmword)*10-sizeof(qword)*3] + vpaddq ymm9, ymm10, ymm12 +;; ------------------------------------------------------------ + + add rbp, sizeof(qword)*4 + sub r8, 2 + jnz .loop4_B + +.exit_loop_B: + mov qword [rdi], r12 + mov qword [rdi+sizeof(qword)], r13 + vmovdqu ymmword [rdi+sizeof(qword)*2], ymm1 + vmovdqu ymmword [rdi+sizeof(qword)*2+sizeof(ymmword)], ymm2 + vmovdqu ymmword [rdi+sizeof(qword)*2+sizeof(ymmword)*2], ymm3 + vmovdqu ymmword [rdi+sizeof(qword)*2+sizeof(ymmword)*3], ymm4 + vmovdqu ymmword [rdi+sizeof(qword)*2+sizeof(ymmword)*4], ymm5 + vmovdqu ymmword [rdi+sizeof(qword)*2+sizeof(ymmword)*5], ymm6 + vmovdqu ymmword [rdi+sizeof(qword)*2+sizeof(ymmword)*6], ymm7 + vmovdqu ymmword [rdi+sizeof(qword)*2+sizeof(ymmword)*7], ymm8 + vmovdqu ymmword [rdi+sizeof(qword)*2+sizeof(ymmword)*8], ymm9 + + ;; + ;; normalize + ;; + mov r8, dword 38 + xor rax, rax +.norm_loop: + add rax, qword [rdi] + add rdi, sizeof(qword) + mov rdx, dword DIGIT_MASK + and rdx, rax + shr rax, DIGIT_BITS + mov qword [rdi-sizeof(qword)], rdx + sub r8, 1 + jg .norm_loop + mov qword [rdi], rax + + REST_XMM_AVX + REST_GPR + ret +ENDFUNC cpMontMul1024_avx2 + + +%endif ; _IPP32E_L9 + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmontmul_avx2as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmontmul_avx2as.asm new file mode 100644 index 0000000..c104077 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmontmul_avx2as.asm @@ -0,0 +1,1834 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Big Number Operations +; +; Content: +; cpMontMul4n_avx2() +; cpMontMul4n1_avx2() +; cpMontMul4n2_avx2() +; cpMontMul4n3_avx2() +; + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_L9) + +segment .text align=IPP_ALIGN_FACTOR + + +%assign DIGIT_BITS 27 +%assign DIGIT_MASK (1 << DIGIT_BITS) -1 + +;************************************************************* +;* void cpMontMul4n_avx2(Ipp64u* pR, +;* const Ipp64u* pA, +;* const Ipp64u* pB, +;* const Ipp64u* pModulo, int mSize, +;* Ipp64u m0, +;* Ipp64u* pBuffer) +;************************************************************* +align IPP_ALIGN_FACTOR +IPPASM cpMontMul4n_avx2,PUBLIC +%assign LOCAL_FRAME sizeof(qword)*5 + USES_GPR rsi,rdi,rbx,rbp,r12,r13,r14 + USES_XMM_AVX ymm6,ymm7,ymm8,ymm9,ymm10,ymm11,ymm12,ymm13 + COMP_ABI 7 + + mov rbp,rdx ; pointer to B operand + movsxd r8, r8d ; redLen value counter + +; +; stack struct +; +%assign pR 0 ; pointer to result +%assign pResult pR+sizeof(qword) ; pointer to buffer +%assign pA pResult+sizeof(qword) ; pointer to A operand +%assign pM pA+sizeof(qword) ; pointer to modulus +%assign redLen pM+sizeof(qword) ; length + + mov qword [rsp+pR], rdi ; save pointer to destination + mov qword [rsp+pA], rsi ; A operand + mov qword [rsp+pM], rcx ; modulus + mov qword [rsp+redLen], r8 ; modulus length + + mov rdi, qword [rsp+ARG_7] ; buffer + mov [rsp+pResult], rdi + + vpxor ymm0, ymm0, ymm0 + xor rax, rax + +;; expands A and M operands + vmovdqu ymmword [rsi+r8*sizeof(qword)], ymm0 + vmovdqu ymmword [rcx+r8*sizeof(qword)], ymm0 + +;; clear result buffer of (redLen+4) qword length + mov r14, r8 +align IPP_ALIGN_FACTOR +.clearLoop: + vmovdqu ymmword [rdi], ymm0 + add rdi, sizeof(ymmword) + sub r14, sizeof(ymmword)/sizeof(qword) + jg .clearLoop + vmovdqu ymmword [rdi], ymm0 + + lea r14, [r8+sizeof(ymmword)/sizeof(qword)-1] ; a_counter = (redLen+3) & (-4) + and r14,-(sizeof(ymmword)/sizeof(qword)) + +align IPP_ALIGN_FACTOR +;; +;; process b[] by quadruples (b[j], b[j+1], b[j+2] and b[j+3]) per pass +;; +.loop4_B: + sub r8, sizeof(ymmword)/sizeof(qword) + jl .exit_loop4_B + + mov rbx, qword [rbp] ; rbx = b[j] + vpbroadcastq ymm4, qword [rbp] + + mov rdi, qword [rsp+pResult] ; restore pointer to destination + mov rsi, qword [rsp+pA] ; A operand + mov rcx, qword [rsp+pM] ; modulus + + mov r10, qword [rdi] + mov r11, qword [rdi+sizeof(qword)] + mov r12, qword [rdi+sizeof(qword)*2] + mov r13, qword [rdi+sizeof(qword)*3] + + mov rax, rbx ; ac0 = pa[0]*b[j]+pr[0] + imul rax, qword [rsi] + add r10, rax + mov rdx, r10 ; y0 = (ac0*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + mov rax, rbx ; ac1 = pa[1]*b[j]+pr[1] + imul rax, qword [rsi+sizeof(qword)] + add r11, rax + mov rax, rbx ; ac2 = pa[2]*b[j]+pr[2] + imul rax, qword [rsi+sizeof(qword)*2] + add r12, rax + mov rax, rbx ; ac3 = pa[3]*b[j]+pr[3] + imul rax, qword [rsi+sizeof(qword)*3] + add r13, rax + vmovd xmm8, edx + vpbroadcastq ymm8, xmm8 + + mov rax, rdx ; ac0 += pn[0]*y0 + imul rax, qword [rcx] + add r10, rax + shr r10, DIGIT_BITS + mov rax, rdx ; ac1 += pn[1]*y0 + imul rax, qword [rcx+sizeof(qword)] + add r11, rax + add r11, r10 + mov rax, rdx ; ac2 += pn[2]*y0 + imul rax, qword [rcx+sizeof(qword)*2] + add r12, rax + mov rax, rdx ; ac3 += pn[3]*y0 + imul rax, qword [rcx+sizeof(qword)*3] + add r13, rax + + + mov rbx, qword [rbp+sizeof(qword)] ; rbx = b[j+1] + vpbroadcastq ymm5, qword [rbp+sizeof(qword)] + mov rax, rbx ; ac1 += pa[0]*b[j+1] + imul rax, qword [rsi] + add r11, rax + mov rdx, r11 ; y1 = (ac1*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + mov rax, rbx ; ac2 += pa[1]*b[j+1] + imul rax, qword [rsi+sizeof(qword)] + add r12, rax + mov rax, rbx ; ac3 += pa[2]*b[j+1] + imul rax, qword [rsi+sizeof(qword)*2] + add r13, rax + vmovd xmm9, edx + vpbroadcastq ymm9, xmm9 + + mov rax, rdx ; ac1 += pn[0]*y1 + imul rax, qword [rcx] + add r11, rax + shr r11, DIGIT_BITS + mov rax, rdx ; ac2 += pn[1]*y1 + imul rax, qword [rcx+sizeof(qword)] + add r12, rax + add r12, r11 + mov rax, rdx ; ac3 += pn[2]*y1 + imul rax, qword [rcx+sizeof(qword)*2] + add r13, rax + + + mov rbx, qword [rbp+sizeof(qword)*2] ; rbx = b[j+2] + vpbroadcastq ymm6, qword [rbp+sizeof(qword)*2] + mov rax, rbx ; ac2 += pa[0]*b[j+2] + imul rax, qword [rsi] + add r12, rax + mov rdx, r12 ; y2 = (ac2*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + mov rax, rbx ; ac2 += pa[0]*b[j+2] + imul rax, qword [rsi+sizeof(qword)] + add r13, rax + vmovd xmm10, edx + vpbroadcastq ymm10, xmm10 + + mov rax, rdx ; ac2 += pn[0]*y2 + imul rax, qword [rcx] + add r12, rax + shr r12, DIGIT_BITS + mov rax, rdx ; ac3 += pn[1]*y2 + imul rax, qword [rcx+sizeof(qword)] + add r13, rax + add r13, r12 + + mov rbx, qword [rbp+sizeof(qword)*3] ; rbx = b[j+3] + vpbroadcastq ymm7, qword [rbp+sizeof(qword)*3] + imul rbx, qword [rsi] ; ac3 += pa[0]*b[j+3] + add r13, rbx + mov rdx, r13 ; y3 = (ac3*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + vmovd xmm11, edx + vpbroadcastq ymm11, xmm11 + + imul rdx, qword [rcx] ; ac3 += pn[0]*y3 + add r13, rdx + shr r13, DIGIT_BITS + vmovq xmm0, r13 + + vpaddq ymm0, ymm0, ymmword [rdi+sizeof(ymmword)] + vmovdqu ymmword [rdi+sizeof(ymmword)], ymm0 + + add rbp, sizeof(qword)*4 + add rsi, sizeof(qword)*4 + add rcx, sizeof(qword)*4 + add rdi, sizeof(ymmword) + + mov r11, r14 ; init a_counter + + ;; (vb0) ymm4 = {b0:b0:b0:b0} + ;; (vb1) ymm5 = {b1:b1:b1:b1} + ;; (vb2) ymm6 = {b2:b2:b2:b2} + ;; (vb3) ymm7 = {b3:b3:b3:b3} + + ;; (vy0) ymm8 = {y0:y0:y0:y0} + ;; (vy1) ymm9 = {y1:y1:y1:y1} + ;; (vy2) ymm10= {y2:y2:y2:y2} + ;; (vy3) ymm11= {y3:y3:y3:y3} + + sub r11, sizeof(ymmword)/sizeof(qword) + +align IPP_ALIGN_FACTOR +.loop16_A: + sub r11, 4*sizeof(ymmword)/sizeof(qword) + jl .exit_loop16_A + + vmovdqu ymm0, ymmword [rdi] ; r0 + vmovdqu ymm1, ymmword [rdi+sizeof(ymmword)] ; r1 + vmovdqu ymm2, ymmword [rdi+sizeof(ymmword)*2] ; r2 + vmovdqu ymm3, ymmword [rdi+sizeof(ymmword)*3] ; r3 + + vpmuludq ymm12, ymm4, ymmword [rsi] ; r0 += vb0 * (__m256i*)(pa)[j] + vy0 * (__m256i*)(pn)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx] + vpaddq ymm0, ymm0, ymm13 + vpmuludq ymm12, ymm4, ymmword [rsi+sizeof(ymmword)] ; r1 += vb0 * (__m256i*)(pa)[j+1] + vy0 * (__m256i*)(pn)[j+1] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx+sizeof(ymmword)] + vpaddq ymm1, ymm1, ymm13 + vpmuludq ymm12, ymm4, ymmword [rsi+sizeof(ymmword)*2] ; r2 += vb0 * (__m256i*)(pa)[j+2] + vy0 * (__m256i*)(pn)[j+2] + vpaddq ymm2, ymm2, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx+sizeof(ymmword)*2] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm12, ymm4, ymmword [rsi+sizeof(ymmword)*3] ; r3 += vb0 * (__m256i*)(pa)[j+3] + vy0 * (__m256i*)(pn)[j+3] + vpaddq ymm3, ymm3, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx+sizeof(ymmword)*3] + vpaddq ymm3, ymm3, ymm13 + + vpmuludq ymm12, ymm5, ymmword [rsi-sizeof(qword)] ; r0 += vb1 * (__m256i*)(pa-1)[j] + vy1 * (__m256i*)(pn-1)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm9, ymmword [rcx-sizeof(qword)] + vpaddq ymm0, ymm0, ymm13 + vpmuludq ymm12, ymm5, ymmword [rsi-sizeof(qword)+sizeof(ymmword)] ; r1 += vb1 * (__m256i*)(pa-1)[j+1] + vy1 * (__m256i*)(pn-1)[j+1] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm9, ymmword [rcx-sizeof(qword)+sizeof(ymmword)] + vpaddq ymm1, ymm1, ymm13 + vpmuludq ymm12, ymm5, ymmword [rsi-sizeof(qword)+sizeof(ymmword)*2] ; r2 += vb1 * (__m256i*)(pa-1)[j+2] + vy1 * (__m256i*)(pn-1)[j+2] + vpaddq ymm2, ymm2, ymm12 + vpmuludq ymm13, ymm9, ymmword [rcx-sizeof(qword)+sizeof(ymmword)*2] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm12, ymm5, ymmword [rsi-sizeof(qword)+sizeof(ymmword)*3] ; r3 += vb1 * (__m256i*)(pa-1)[j+3] + vy1 * (__m256i*)(pn-1)[j+3] + vpaddq ymm3, ymm3, ymm12 + vpmuludq ymm13, ymm9, ymmword [rcx-sizeof(qword)+sizeof(ymmword)*3] + vpaddq ymm3, ymm3, ymm13 + + vpmuludq ymm12, ymm6, ymmword [rsi-sizeof(qword)*2] ; r0 += vb2 * (__m256i*)(pa-2)[j] + vy2 * (__m256i*)(pn-2)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2] + vpaddq ymm0, ymm0, ymm13 + vpmuludq ymm12, ymm6, ymmword [rsi-sizeof(qword)*2+sizeof(ymmword)] ; r1 += vb2 * (__m256i*)(pa-2)[j+1] + vy2 * (__m256i*)(pn-2)[j+1] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2+sizeof(ymmword)] + vpaddq ymm1, ymm1, ymm13 + vpmuludq ymm12, ymm6, ymmword [rsi-sizeof(qword)*2+sizeof(ymmword)*2] ; r2 += vb2 * (__m256i*)(pa-2)[j+2] + vy2 * (__m256i*)(pn-2)[j+2] + vpaddq ymm2, ymm2, ymm12 + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2+sizeof(ymmword)*2] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm12, ymm6, ymmword [rsi-sizeof(qword)*2+sizeof(ymmword)*3] ; r3 += vb2 * (__m256i*)(pa-2)[j+3] + vy2 * (__m256i*)(pn-2)[j+3] + vpaddq ymm3, ymm3, ymm12 + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2+sizeof(ymmword)*3] + vpaddq ymm3, ymm3, ymm13 + + vpmuludq ymm12, ymm7, ymmword [rsi-sizeof(qword)*3] ; r0 += vb3 * (__m256i*)(pa-3)[j] + vy3 * (__m256i*)(pn-3)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm11,ymmword [rcx-sizeof(qword)*3] + vpaddq ymm0, ymm0, ymm13 + vpmuludq ymm12, ymm7, ymmword [rsi-sizeof(qword)*3+sizeof(ymmword)] ; r1 += vb3 * (__m256i*)(pa-3)[j+1] + vy3 * (__m256i*)(pn-3)[j+1] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm11,ymmword [rcx-sizeof(qword)*3+sizeof(ymmword)] + vpaddq ymm1, ymm1, ymm13 + vpmuludq ymm12, ymm7, ymmword [rsi-sizeof(qword)*3+sizeof(ymmword)*2] ; r2 += vb3 * (__m256i*)(pa-3)[j+2] + vy3 * (__m256i*)(pn-3)[j+2] + vpaddq ymm2, ymm2, ymm12 + vpmuludq ymm13, ymm11,ymmword [rcx-sizeof(qword)*3+sizeof(ymmword)*2] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm12, ymm7, ymmword [rsi-sizeof(qword)*3+sizeof(ymmword)*3] ; r3 += vb3 * (__m256i*)(pa-3)[j+3] + vy3 * (__m256i*)(pn-3)[j+3] + vpaddq ymm3, ymm3, ymm12 + vpmuludq ymm13, ymm11,ymmword [rcx-sizeof(qword)*3+sizeof(ymmword)*3] + vpaddq ymm3, ymm3, ymm13 + + vmovdqu ymmword [rdi-sizeof(ymmword)], ymm0 + vmovdqu ymmword [rdi-sizeof(ymmword)+sizeof(ymmword)], ymm1 + vmovdqu ymmword [rdi-sizeof(ymmword)+sizeof(ymmword)*2], ymm2 + vmovdqu ymmword [rdi-sizeof(ymmword)+sizeof(ymmword)*3], ymm3 + + add rdi, sizeof(ymmword)*4 + add rsi, sizeof(ymmword)*4 + add rcx, sizeof(ymmword)*4 + jmp .loop16_A + +.exit_loop16_A: + add r11, 4*(sizeof(ymmword)/sizeof(qword)) + jz .exitA + +.loop4_A: + sub r11, sizeof(ymmword)/sizeof(qword) + jl .exitA + + vmovdqu ymm0, ymmword [rdi] ; r0 + + vpmuludq ymm12, ymm4, ymmword [rsi] ; r0 += vb0 * (__m256i*)(pa)[j] + vy0 * (__m256i*)(pn)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx] + vpaddq ymm0, ymm0, ymm13 + + vpmuludq ymm1, ymm5, ymmword [rsi-sizeof(qword)] ; r0 += vb1 * (__m256i*)(pa-1)[j] + vy1 * (__m256i*)(pn-1)[j] + vpaddq ymm0, ymm0, ymm1 + vpmuludq ymm2, ymm9, ymmword [rcx-sizeof(qword)] + vpaddq ymm0, ymm0, ymm2 + + vpmuludq ymm12, ymm6, ymmword [rsi-sizeof(qword)*2] ; r0 += vb2 * (__m256i*)(pa-2)[j] + vy2 * (__m256i*)(pn-2)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2] + vpaddq ymm0, ymm0, ymm13 + + vpmuludq ymm1, ymm7, ymmword [rsi-sizeof(qword)*3] ; r0 += vb3 * (__m256i*)(pa-3)[j] + vy3 * (__m256i*)(pn-3)[j] + vpaddq ymm0, ymm0, ymm1 + vpmuludq ymm2, ymm11,ymmword [rcx-sizeof(qword)*3] + vpaddq ymm0, ymm0, ymm2 + + vmovdqu ymmword [rdi-sizeof(ymmword)], ymm0 + + add rdi, sizeof(ymmword) + add rsi, sizeof(ymmword) + add rcx, sizeof(ymmword) + jmp .loop4_A + +.exitA: + vpmuludq ymm1, ymm5, ymmword [rsi-sizeof(qword)] ; r1 += vb1 * (__m256i*)(pa-1)[j] + vy1 * (__m256i*)(pn-1)[j] + vpmuludq ymm13, ymm9, ymmword [rcx-sizeof(qword)] + vpaddq ymm1, ymm1, ymm13 + + vpmuludq ymm2, ymm6, ymmword [rsi-sizeof(qword)*2] ; r2 += vb2 * (__m256i*)(pa-2)[j] + vy2 * (__m256i*)(pn-2)[j] + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2] + vpaddq ymm2, ymm2, ymm13 + + vpmuludq ymm3, ymm7, ymmword [rsi-sizeof(qword)*3] ; r3 += vb3 * (__m256i*)(pa-3)[j] + vy3 * (__m256i*)(pn-3)[j] + vpmuludq ymm13, ymm11,ymmword [rcx-sizeof(qword)*3] + vpaddq ymm3, ymm3, ymm13 + + vpaddq ymm1, ymm1, ymm2 + vpaddq ymm1, ymm1, ymm3 + vmovdqu ymmword [rdi-sizeof(ymmword)], ymm1 + + jmp .loop4_B + +.exit_loop4_B: + ;; + ;; normalize + ;; + mov rdi, qword [rsp+pR] + mov rsi, qword [rsp+pResult] + mov r8, qword [rsp+redLen] + xor rax, rax +.norm_loop: + add rax, qword [rsi] + add rsi, sizeof(qword) + mov rdx, dword DIGIT_MASK + and rdx, rax + shr rax, DIGIT_BITS + mov qword [rdi], rdx + add rdi, sizeof(qword) + sub r8, 1 + jg .norm_loop + mov qword [rdi], rax + + REST_XMM_AVX + REST_GPR + ret +ENDFUNC cpMontMul4n_avx2 + + +;************************************************************* +;* void cpMontMul4n1_avx2(Ipp64u* pR, +;* const Ipp64u* pA, +;* const Ipp64u* pB, +;* const Ipp64u* pModulo, int mSize, +;* Ipp64u m0, +;* Ipp64u* pBuffer) +;************************************************************* +align IPP_ALIGN_FACTOR +IPPASM cpMontMul4n1_avx2,PUBLIC +%assign LOCAL_FRAME sizeof(qword)*5 + USES_GPR rsi,rdi,rbx,rbp,r12,r13,r14 + USES_XMM_AVX ymm6,ymm7,ymm8,ymm9,ymm10,ymm11,ymm12,ymm13 + COMP_ABI 7 + + mov rbp,rdx ; pointer to B operand + movsxd r8, r8d ; redLen value counter + +; +; stack struct +; +%xdefine pR 0 ; pointer to result +%xdefine pResult pR+sizeof(qword) ; pointer to buffer +%xdefine pA pResult+sizeof(qword) ; pointer to A operand +%xdefine pM pA+sizeof(qword) ; pointer to modulus +%xdefine redLen pM+sizeof(qword) ; length + + mov qword [rsp+pR], rdi ; save pointer to destination + mov qword [rsp+pA], rsi ; A operand + mov qword [rsp+pM], rcx ; modulus + mov qword [rsp+redLen], r8 ; modulus length + + mov rdi, qword [rsp+ARG_7] ; buffer + mov [rsp+pResult], rdi + + vpxor ymm0, ymm0, ymm0 + xor rax, rax + +;; expands A and M operands + vmovdqu ymmword [rsi+r8*sizeof(qword)], ymm0 + vmovdqu ymmword [rcx+r8*sizeof(qword)], ymm0 + +;; clear result buffer of (redLen+4) qword length + mov r14, r8 +align IPP_ALIGN_FACTOR +.clearLoop: + vmovdqu ymmword [rdi], ymm0 + add rdi, sizeof(ymmword) + sub r14, sizeof(ymmword)/sizeof(qword) + jg .clearLoop + mov qword [rdi], rax + + lea r14, [r8+sizeof(ymmword)/sizeof(qword)-1] ; a_counter = (redLen+3) & (-4) + and r14,-(sizeof(ymmword)/sizeof(qword)) + +align IPP_ALIGN_FACTOR +;; +;; process b[] by quadruples (b[j], b[j+1], b[j+2] and b[j+3]) per pass +;; +.loop4_B: + sub r8, sizeof(ymmword)/sizeof(qword) + jl .exit_loop4_B + + mov rbx, qword [rbp] ; rbx = b[j] + vpbroadcastq ymm4, qword [rbp] + + mov rdi, qword [rsp+pResult] ; restore pointer to destination + mov rsi, qword [rsp+pA] ; A operand + mov rcx, qword [rsp+pM] ; modulus + + mov r10, qword [rdi] + mov r11, qword [rdi+sizeof(qword)] + mov r12, qword [rdi+sizeof(qword)*2] + mov r13, qword [rdi+sizeof(qword)*3] + + mov rax, rbx ; ac0 = pa[0]*b[j]+pr[0] + imul rax, qword [rsi] + add r10, rax + mov rdx, r10 ; y0 = (ac0*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + mov rax, rbx ; ac1 = pa[1]*b[j]+pr[1] + imul rax, qword [rsi+sizeof(qword)] + add r11, rax + mov rax, rbx ; ac2 = pa[2]*b[j]+pr[2] + imul rax, qword [rsi+sizeof(qword)*2] + add r12, rax + mov rax, rbx ; ac3 = pa[3]*b[j]+pr[3] + imul rax, qword [rsi+sizeof(qword)*3] + add r13, rax + vmovd xmm8, edx + vpbroadcastq ymm8, xmm8 + + mov rax, rdx ; ac0 += pn[0]*y0 + imul rax, qword [rcx] + add r10, rax + shr r10, DIGIT_BITS + mov rax, rdx ; ac1 += pn[1]*y0 + imul rax, qword [rcx+sizeof(qword)] + add r11, rax + add r11, r10 + mov rax, rdx ; ac2 += pn[2]*y0 + imul rax, qword [rcx+sizeof(qword)*2] + add r12, rax + mov rax, rdx ; ac3 += pn[3]*y0 + imul rax, qword [rcx+sizeof(qword)*3] + add r13, rax + + + mov rbx, qword [rbp+sizeof(qword)] ; rbx = b[j+1] + vpbroadcastq ymm5, qword [rbp+sizeof(qword)] + mov rax, rbx ; ac1 += pa[0]*b[j+1] + imul rax, qword [rsi] + add r11, rax + mov rdx, r11 ; y1 = (ac1*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + mov rax, rbx ; ac2 += pa[1]*b[j+1] + imul rax, qword [rsi+sizeof(qword)] + add r12, rax + mov rax, rbx ; ac3 += pa[2]*b[j+1] + imul rax, qword [rsi+sizeof(qword)*2] + add r13, rax + vmovd xmm9, edx + vpbroadcastq ymm9, xmm9 + + mov rax, rdx ; ac1 += pn[0]*y1 + imul rax, qword [rcx] + add r11, rax + shr r11, DIGIT_BITS + mov rax, rdx ; ac2 += pn[1]*y1 + imul rax, qword [rcx+sizeof(qword)] + add r12, rax + add r12, r11 + mov rax, rdx ; ac3 += pn[2]*y1 + imul rax, qword [rcx+sizeof(qword)*2] + add r13, rax + + + mov rbx, qword [rbp+sizeof(qword)*2] ; rbx = b[j+2] + vpbroadcastq ymm6, qword [rbp+sizeof(qword)*2] + mov rax, rbx ; ac2 += pa[0]*b[j+2] + imul rax, qword [rsi] + add r12, rax + mov rdx, r12 ; y2 = (ac2*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + mov rax, rbx ; ac2 += pa[0]*b[j+2] + imul rax, qword [rsi+sizeof(qword)] + add r13, rax + vmovd xmm10, edx + vpbroadcastq ymm10, xmm10 + + mov rax, rdx ; ac2 += pn[0]*y2 + imul rax, qword [rcx] + add r12, rax + shr r12, DIGIT_BITS + mov rax, rdx ; ac3 += pn[1]*y2 + imul rax, qword [rcx+sizeof(qword)] + add r13, rax + add r13, r12 + + mov rbx, qword [rbp+sizeof(qword)*3] ; rbx = b[j+3] + vpbroadcastq ymm7, qword [rbp+sizeof(qword)*3] + imul rbx, qword [rsi] ; ac3 += pa[0]*b[j+3] + add r13, rbx + mov rdx, r13 ; y3 = (ac3*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + vmovd xmm11, edx + vpbroadcastq ymm11, xmm11 + + imul rdx, qword [rcx] ; ac3 += pn[0]*y3 + add r13, rdx + shr r13, DIGIT_BITS + vmovq xmm0, r13 + + vpaddq ymm0, ymm0, ymmword [rdi+sizeof(ymmword)] + vmovdqu ymmword [rdi+sizeof(ymmword)], ymm0 + + add rbp, sizeof(qword)*4 + add rsi, sizeof(qword)*4 + add rcx, sizeof(qword)*4 + add rdi, sizeof(ymmword) + + mov r11, r14 ; init a_counter + + ;; (vb0) ymm4 = {b0:b0:b0:b0} + ;; (vb1) ymm5 = {b1:b1:b1:b1} + ;; (vb2) ymm6 = {b2:b2:b2:b2} + ;; (vb3) ymm7 = {b3:b3:b3:b3} + + ;; (vy0) ymm8 = {y0:y0:y0:y0} + ;; (vy1) ymm9 = {y1:y1:y1:y1} + ;; (vy2) ymm10= {y2:y2:y2:y2} + ;; (vy3) ymm11= {y3:y3:y3:y3} + + sub r11, sizeof(ymmword)/sizeof(qword) + +align IPP_ALIGN_FACTOR +.loop16_A: + sub r11, 4*sizeof(ymmword)/sizeof(qword) + jl .exit_loop16_A + + vmovdqu ymm0, ymmword [rdi] ; r0 + vmovdqu ymm1, ymmword [rdi+sizeof(ymmword)] ; r1 + vmovdqu ymm2, ymmword [rdi+sizeof(ymmword)*2] ; r2 + vmovdqu ymm3, ymmword [rdi+sizeof(ymmword)*3] ; r3 + + vpmuludq ymm12, ymm4, ymmword [rsi] ; r0 += vb0 * (__m256i*)(pa)[j] + vy0 * (__m256i*)(pn)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx] + vpaddq ymm0, ymm0, ymm13 + vpmuludq ymm12, ymm4, ymmword [rsi+sizeof(ymmword)] ; r1 += vb0 * (__m256i*)(pa)[j+1] + vy0 * (__m256i*)(pn)[j+1] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx+sizeof(ymmword)] + vpaddq ymm1, ymm1, ymm13 + vpmuludq ymm12, ymm4, ymmword [rsi+sizeof(ymmword)*2] ; r2 += vb0 * (__m256i*)(pa)[j+2] + vy0 * (__m256i*)(pn)[j+2] + vpaddq ymm2, ymm2, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx+sizeof(ymmword)*2] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm12, ymm4, ymmword [rsi+sizeof(ymmword)*3] ; r3 += vb0 * (__m256i*)(pa)[j+3] + vy0 * (__m256i*)(pn)[j+3] + vpaddq ymm3, ymm3, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx+sizeof(ymmword)*3] + vpaddq ymm3, ymm3, ymm13 + + vpmuludq ymm12, ymm5, ymmword [rsi-sizeof(qword)] ; r0 += vb1 * (__m256i*)(pa-1)[j] + vy1 * (__m256i*)(pn-1)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm9, ymmword [rcx-sizeof(qword)] + vpaddq ymm0, ymm0, ymm13 + vpmuludq ymm12, ymm5, ymmword [rsi-sizeof(qword)+sizeof(ymmword)] ; r1 += vb1 * (__m256i*)(pa-1)[j+1] + vy1 * (__m256i*)(pn-1)[j+1] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm9, ymmword [rcx-sizeof(qword)+sizeof(ymmword)] + vpaddq ymm1, ymm1, ymm13 + vpmuludq ymm12, ymm5, ymmword [rsi-sizeof(qword)+sizeof(ymmword)*2] ; r2 += vb1 * (__m256i*)(pa-1)[j+2] + vy1 * (__m256i*)(pn-1)[j+2] + vpaddq ymm2, ymm2, ymm12 + vpmuludq ymm13, ymm9, ymmword [rcx-sizeof(qword)+sizeof(ymmword)*2] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm12, ymm5, ymmword [rsi-sizeof(qword)+sizeof(ymmword)*3] ; r3 += vb1 * (__m256i*)(pa-1)[j+3] + vy1 * (__m256i*)(pn-1)[j+3] + vpaddq ymm3, ymm3, ymm12 + vpmuludq ymm13, ymm9, ymmword [rcx-sizeof(qword)+sizeof(ymmword)*3] + vpaddq ymm3, ymm3, ymm13 + + vpmuludq ymm12, ymm6, ymmword [rsi-sizeof(qword)*2] ; r0 += vb2 * (__m256i*)(pa-2)[j] + vy2 * (__m256i*)(pn-2)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2] + vpaddq ymm0, ymm0, ymm13 + vpmuludq ymm12, ymm6, ymmword [rsi-sizeof(qword)*2+sizeof(ymmword)] ; r1 += vb2 * (__m256i*)(pa-2)[j+1] + vy2 * (__m256i*)(pn-2)[j+1] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2+sizeof(ymmword)] + vpaddq ymm1, ymm1, ymm13 + vpmuludq ymm12, ymm6, ymmword [rsi-sizeof(qword)*2+sizeof(ymmword)*2] ; r2 += vb2 * (__m256i*)(pa-2)[j+2] + vy2 * (__m256i*)(pn-2)[j+2] + vpaddq ymm2, ymm2, ymm12 + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2+sizeof(ymmword)*2] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm12, ymm6, ymmword [rsi-sizeof(qword)*2+sizeof(ymmword)*3] ; r3 += vb2 * (__m256i*)(pa-2)[j+3] + vy2 * (__m256i*)(pn-2)[j+3] + vpaddq ymm3, ymm3, ymm12 + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2+sizeof(ymmword)*3] + vpaddq ymm3, ymm3, ymm13 + + vpmuludq ymm12, ymm7, ymmword [rsi-sizeof(qword)*3] ; r0 += vb3 * (__m256i*)(pa-3)[j] + vy3 * (__m256i*)(pn-3)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm11,ymmword [rcx-sizeof(qword)*3] + vpaddq ymm0, ymm0, ymm13 + vpmuludq ymm12, ymm7, ymmword [rsi-sizeof(qword)*3+sizeof(ymmword)] ; r1 += vb3 * (__m256i*)(pa-3)[j+1] + vy3 * (__m256i*)(pn-3)[j+1] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm11,ymmword [rcx-sizeof(qword)*3+sizeof(ymmword)] + vpaddq ymm1, ymm1, ymm13 + vpmuludq ymm12, ymm7, ymmword [rsi-sizeof(qword)*3+sizeof(ymmword)*2] ; r2 += vb3 * (__m256i*)(pa-3)[j+2] + vy3 * (__m256i*)(pn-3)[j+2] + vpaddq ymm2, ymm2, ymm12 + vpmuludq ymm13, ymm11,ymmword [rcx-sizeof(qword)*3+sizeof(ymmword)*2] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm12, ymm7, ymmword [rsi-sizeof(qword)*3+sizeof(ymmword)*3] ; r3 += vb3 * (__m256i*)(pa-3)[j+3] + vy3 * (__m256i*)(pn-3)[j+3] + vpaddq ymm3, ymm3, ymm12 + vpmuludq ymm13, ymm11,ymmword [rcx-sizeof(qword)*3+sizeof(ymmword)*3] + vpaddq ymm3, ymm3, ymm13 + + vmovdqu ymmword [rdi-sizeof(ymmword)], ymm0 + vmovdqu ymmword [rdi-sizeof(ymmword)+sizeof(ymmword)], ymm1 + vmovdqu ymmword [rdi-sizeof(ymmword)+sizeof(ymmword)*2], ymm2 + vmovdqu ymmword [rdi-sizeof(ymmword)+sizeof(ymmword)*3], ymm3 + + add rdi, sizeof(ymmword)*4 + add rsi, sizeof(ymmword)*4 + add rcx, sizeof(ymmword)*4 + jmp .loop16_A + +.exit_loop16_A: + add r11, 4*(sizeof(ymmword)/sizeof(qword)) + jz .exitA + +.loop4_A: + sub r11, sizeof(ymmword)/sizeof(qword) + jl .exitA + + vmovdqu ymm0, ymmword [rdi] ; r0 + + vpmuludq ymm12, ymm4, ymmword [rsi] ; r0 += vb0 * (__m256i*)(pa)[j] + vy0 * (__m256i*)(pn)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx] + vpaddq ymm0, ymm0, ymm13 + + vpmuludq ymm1, ymm5, ymmword [rsi-sizeof(qword)] ; r0 += vb1 * (__m256i*)(pa-1)[j] + vy1 * (__m256i*)(pn-1)[j] + vpaddq ymm0, ymm0, ymm1 + vpmuludq ymm2, ymm9, ymmword [rcx-sizeof(qword)] + vpaddq ymm0, ymm0, ymm2 + + vpmuludq ymm12, ymm6, ymmword [rsi-sizeof(qword)*2] ; r0 += vb2 * (__m256i*)(pa-2)[j] + vy2 * (__m256i*)(pn-2)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2] + vpaddq ymm0, ymm0, ymm13 + + vpmuludq ymm1, ymm7, ymmword [rsi-sizeof(qword)*3] ; r0 += vb3 * (__m256i*)(pa-3)[j] + vy3 * (__m256i*)(pn-3)[j] + vpaddq ymm0, ymm0, ymm1 + vpmuludq ymm2, ymm11,ymmword [rcx-sizeof(qword)*3] + vpaddq ymm0, ymm0, ymm2 + + vmovdqu ymmword [rdi-sizeof(ymmword)], ymm0 + + add rdi, sizeof(ymmword) + add rsi, sizeof(ymmword) + add rcx, sizeof(ymmword) + jmp .loop4_A + +.exitA: + jmp .loop4_B + +;; +;; process latest b[redLen-1] +;; +.exit_loop4_B: + mov rbx, qword [rbp] ; rbx = b[redLen-1] + vpbroadcastq ymm4, qword [rbp] + + mov rdi, qword [rsp+pResult] ; restore pointer to destination + mov rsi, qword [rsp+pA] ; A operand + mov rcx, qword [rsp+pM] ; modulus + + mov r10, qword [rdi] + mov r11, qword [rdi+sizeof(qword)] + mov r12, qword [rdi+sizeof(qword)*2] + mov r13, qword [rdi+sizeof(qword)*3] + + mov rax, rbx ; ac0 = pa[0]*b[j]+pr[0] + imul rax, qword [rsi] + add r10, rax + mov rdx, r10 ; y0 = (ac0*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + mov rax, rbx ; ac1 = pa[1]*b[j]+pr[1] + imul rax, qword [rsi+sizeof(qword)] + add r11, rax + mov rax, rbx ; ac2 = pa[2]*b[j]+pr[2] + imul rax, qword [rsi+sizeof(qword)*2] + add r12, rax + mov rax, rbx ; ac3 = pa[3]*b[j]+pr[3] + imul rax, qword [rsi+sizeof(qword)*3] + add r13, rax + vmovd xmm8, edx + vpbroadcastq ymm8, xmm8 + + mov rax, rdx ; ac0 += pn[0]*y0 + imul rax, qword [rcx] + add r10, rax + shr r10, DIGIT_BITS + mov rax, rdx ; ac1 += pn[1]*y0 + imul rax, qword [rcx+sizeof(qword)] + add r11, rax + add r11, r10 + mov rax, rdx ; ac2 += pn[2]*y0 + imul rax, qword [rcx+sizeof(qword)*2] + add r12, rax + mov rax, rdx ; ac3 += pn[3]*y0 + imul rax, qword [rcx+sizeof(qword)*3] + add r13, rax + + mov qword [rdi], r11 ; pr[0] = ac1 + mov qword [rdi+sizeof(qword)], r12 ; pr[1] = ac2 + mov qword [rdi+sizeof(qword)*2], r13 ; pr[2] = ac3 + + add rbp, sizeof(qword)*4 + add rsi, sizeof(qword)*4 + add rcx, sizeof(qword)*4 + add rdi, sizeof(ymmword) + + mov r11, r14 ; init a_counter + sub r11, sizeof(ymmword)/sizeof(qword) + +align IPP_ALIGN_FACTOR +.rem_loop4_A: + sub r11, sizeof(ymmword)/sizeof(qword) + jl .exit_rem_loop4_A + + vmovdqu ymm0, ymmword [rdi] ; r0 + + vpmuludq ymm12, ymm4, ymmword [rsi] ; r0 += vb0 * (__m256i*)(pa)[j] + vy0 * (__m256i*)(pn)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx] + vpaddq ymm0, ymm0, ymm13 + + vmovdqu ymmword [rdi+3*sizeof(qword)-sizeof(ymmword)], ymm0 + + add rdi, sizeof(ymmword) + add rsi, sizeof(ymmword) + add rcx, sizeof(ymmword) + jmp .rem_loop4_A + +.exit_rem_loop4_A: + + ;; + ;; normalize + ;; + mov rdi, qword [rsp+pR] + mov rsi, qword [rsp+pResult] + mov r8, qword [rsp+redLen] + xor rax, rax +.norm_loop: + add rax, qword [rsi] + add rsi, sizeof(qword) + mov rdx, dword DIGIT_MASK + and rdx, rax + shr rax, DIGIT_BITS + mov qword [rdi], rdx + add rdi, sizeof(qword) + sub r8, 1 + jg .norm_loop + mov qword [rdi], rax + + REST_XMM_AVX + REST_GPR + ret +ENDFUNC cpMontMul4n1_avx2 + + +;************************************************************* +;* void cpMontMul4n2_avx2(Ipp64u* pR, +;* const Ipp64u* pA, +;* const Ipp64u* pB, +;* const Ipp64u* pModulo, int mSize, +;* Ipp64u m0, +;* Ipp64u* pBuffer) +;************************************************************* +align IPP_ALIGN_FACTOR +IPPASM cpMontMul4n2_avx2,PUBLIC +%assign LOCAL_FRAME sizeof(qword)*5 + USES_GPR rsi,rdi,rbx,rbp,r12,r13,r14 + USES_XMM_AVX ymm6,ymm7,ymm8,ymm9,ymm10,ymm11,ymm12,ymm13 + COMP_ABI 7 + + mov rbp,rdx ; pointer to B operand + movsxd r8, r8d ; redLen value counter + +; +; stack struct +; +%xdefine pR 0 ; pointer to result +%xdefine pResult pR+sizeof(qword) ; pointer to buffer +%xdefine pA pResult+sizeof(qword) ; pointer to A operand +%xdefine pM pA+sizeof(qword) ; pointer to modulus +%xdefine redLen pM+sizeof(qword) ; length + + mov qword [rsp+pR], rdi ; save pointer to destination + mov qword [rsp+pA], rsi ; A operand + mov qword [rsp+pM], rcx ; modulus + mov qword [rsp+redLen], r8 ; modulus length + + mov rdi, qword [rsp+ARG_7] ; buffer + mov [rsp+pResult], rdi + + vpxor ymm0, ymm0, ymm0 + xor rax, rax + +;; expands A and M operands + vmovdqu ymmword [rsi+r8*sizeof(qword)], ymm0 + vmovdqu ymmword [rcx+r8*sizeof(qword)], ymm0 + +;; clear result buffer of (redLen+4) qword length + mov r14, r8 +align IPP_ALIGN_FACTOR +.clearLoop: + vmovdqu ymmword [rdi], ymm0 + add rdi, sizeof(ymmword) + sub r14, sizeof(ymmword)/sizeof(qword) + jg .clearLoop + vmovdqu xmmword [rdi], xmm0 + + lea r14, [r8+sizeof(ymmword)/sizeof(qword)-1] ; a_counter = (redLen+3) & (-4) + and r14,-(sizeof(ymmword)/sizeof(qword)) + +align IPP_ALIGN_FACTOR +;; +;; process b[] by quadruples (b[j], b[j+1], b[j+2] and b[j+3]) per pass +;; +.loop4_B: + sub r8, sizeof(ymmword)/sizeof(qword) + jl .exit_loop4_B + + mov rbx, qword [rbp] ; rbx = b[j] + vpbroadcastq ymm4, qword [rbp] + + mov rdi, qword [rsp+pResult] ; restore pointer to destination + mov rsi, qword [rsp+pA] ; A operand + mov rcx, qword [rsp+pM] ; modulus + + mov r10, qword [rdi] + mov r11, qword [rdi+sizeof(qword)] + mov r12, qword [rdi+sizeof(qword)*2] + mov r13, qword [rdi+sizeof(qword)*3] + + mov rax, rbx ; ac0 = pa[0]*b[j]+pr[0] + imul rax, qword [rsi] + add r10, rax + mov rdx, r10 ; y0 = (ac0*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + mov rax, rbx ; ac1 = pa[1]*b[j]+pr[1] + imul rax, qword [rsi+sizeof(qword)] + add r11, rax + mov rax, rbx ; ac2 = pa[2]*b[j]+pr[2] + imul rax, qword [rsi+sizeof(qword)*2] + add r12, rax + mov rax, rbx ; ac3 = pa[3]*b[j]+pr[3] + imul rax, qword [rsi+sizeof(qword)*3] + add r13, rax + vmovd xmm8, edx + vpbroadcastq ymm8, xmm8 + + mov rax, rdx ; ac0 += pn[0]*y0 + imul rax, qword [rcx] + add r10, rax + shr r10, DIGIT_BITS + mov rax, rdx ; ac1 += pn[1]*y0 + imul rax, qword [rcx+sizeof(qword)] + add r11, rax + add r11, r10 + mov rax, rdx ; ac2 += pn[2]*y0 + imul rax, qword [rcx+sizeof(qword)*2] + add r12, rax + mov rax, rdx ; ac3 += pn[3]*y0 + imul rax, qword [rcx+sizeof(qword)*3] + add r13, rax + + + mov rbx, qword [rbp+sizeof(qword)] ; rbx = b[j+1] + vpbroadcastq ymm5, qword [rbp+sizeof(qword)] + mov rax, rbx ; ac1 += pa[0]*b[j+1] + imul rax, qword [rsi] + add r11, rax + mov rdx, r11 ; y1 = (ac1*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + mov rax, rbx ; ac2 += pa[1]*b[j+1] + imul rax, qword [rsi+sizeof(qword)] + add r12, rax + mov rax, rbx ; ac3 += pa[2]*b[j+1] + imul rax, qword [rsi+sizeof(qword)*2] + add r13, rax + vmovd xmm9, edx + vpbroadcastq ymm9, xmm9 + + mov rax, rdx ; ac1 += pn[0]*y1 + imul rax, qword [rcx] + add r11, rax + shr r11, DIGIT_BITS + mov rax, rdx ; ac2 += pn[1]*y1 + imul rax, qword [rcx+sizeof(qword)] + add r12, rax + add r12, r11 + mov rax, rdx ; ac3 += pn[2]*y1 + imul rax, qword [rcx+sizeof(qword)*2] + add r13, rax + + + mov rbx, qword [rbp+sizeof(qword)*2] ; rbx = b[j+2] + vpbroadcastq ymm6, qword [rbp+sizeof(qword)*2] + mov rax, rbx ; ac2 += pa[0]*b[j+2] + imul rax, qword [rsi] + add r12, rax + mov rdx, r12 ; y2 = (ac2*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + mov rax, rbx ; ac2 += pa[0]*b[j+2] + imul rax, qword [rsi+sizeof(qword)] + add r13, rax + vmovd xmm10, edx + vpbroadcastq ymm10, xmm10 + + mov rax, rdx ; ac2 += pn[0]*y2 + imul rax, qword [rcx] + add r12, rax + shr r12, DIGIT_BITS + mov rax, rdx ; ac3 += pn[1]*y2 + imul rax, qword [rcx+sizeof(qword)] + add r13, rax + add r13, r12 + + mov rbx, qword [rbp+sizeof(qword)*3] ; rbx = b[j+3] + vpbroadcastq ymm7, qword [rbp+sizeof(qword)*3] + imul rbx, qword [rsi] ; ac3 += pa[0]*b[j+3] + add r13, rbx + mov rdx, r13 ; y3 = (ac3*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + vmovd xmm11, edx + vpbroadcastq ymm11, xmm11 + + imul rdx, qword [rcx] ; ac3 += pn[0]*y3 + add r13, rdx + shr r13, DIGIT_BITS + vmovq xmm0, r13 + + vpaddq ymm0, ymm0, ymmword [rdi+sizeof(ymmword)] + vmovdqu ymmword [rdi+sizeof(ymmword)], ymm0 + + add rbp, sizeof(qword)*4 + add rsi, sizeof(qword)*4 + add rcx, sizeof(qword)*4 + add rdi, sizeof(ymmword) + + mov r11, r14 ; init a_counter + + ;; (vb0) ymm4 = {b0:b0:b0:b0} + ;; (vb1) ymm5 = {b1:b1:b1:b1} + ;; (vb2) ymm6 = {b2:b2:b2:b2} + ;; (vb3) ymm7 = {b3:b3:b3:b3} + + ;; (vy0) ymm8 = {y0:y0:y0:y0} + ;; (vy1) ymm9 = {y1:y1:y1:y1} + ;; (vy2) ymm10= {y2:y2:y2:y2} + ;; (vy3) ymm11= {y3:y3:y3:y3} + + sub r11, sizeof(ymmword)/sizeof(qword) + +align IPP_ALIGN_FACTOR +.loop16_A: + sub r11, 4*sizeof(ymmword)/sizeof(qword) + jl .exit_loop16_A + + vmovdqu ymm0, ymmword [rdi] ; r0 + vmovdqu ymm1, ymmword [rdi+sizeof(ymmword)] ; r1 + vmovdqu ymm2, ymmword [rdi+sizeof(ymmword)*2] ; r2 + vmovdqu ymm3, ymmword [rdi+sizeof(ymmword)*3] ; r3 + + vpmuludq ymm12, ymm4, ymmword [rsi] ; r0 += vb0 * (__m256i*)(pa)[j] + vy0 * (__m256i*)(pn)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx] + vpaddq ymm0, ymm0, ymm13 + vpmuludq ymm12, ymm4, ymmword [rsi+sizeof(ymmword)] ; r1 += vb0 * (__m256i*)(pa)[j+1] + vy0 * (__m256i*)(pn)[j+1] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx+sizeof(ymmword)] + vpaddq ymm1, ymm1, ymm13 + vpmuludq ymm12, ymm4, ymmword [rsi+sizeof(ymmword)*2] ; r2 += vb0 * (__m256i*)(pa)[j+2] + vy0 * (__m256i*)(pn)[j+2] + vpaddq ymm2, ymm2, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx+sizeof(ymmword)*2] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm12, ymm4, ymmword [rsi+sizeof(ymmword)*3] ; r3 += vb0 * (__m256i*)(pa)[j+3] + vy0 * (__m256i*)(pn)[j+3] + vpaddq ymm3, ymm3, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx+sizeof(ymmword)*3] + vpaddq ymm3, ymm3, ymm13 + + vpmuludq ymm12, ymm5, ymmword [rsi-sizeof(qword)] ; r0 += vb1 * (__m256i*)(pa-1)[j] + vy1 * (__m256i*)(pn-1)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm9, ymmword [rcx-sizeof(qword)] + vpaddq ymm0, ymm0, ymm13 + vpmuludq ymm12, ymm5, ymmword [rsi-sizeof(qword)+sizeof(ymmword)] ; r1 += vb1 * (__m256i*)(pa-1)[j+1] + vy1 * (__m256i*)(pn-1)[j+1] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm9, ymmword [rcx-sizeof(qword)+sizeof(ymmword)] + vpaddq ymm1, ymm1, ymm13 + vpmuludq ymm12, ymm5, ymmword [rsi-sizeof(qword)+sizeof(ymmword)*2] ; r2 += vb1 * (__m256i*)(pa-1)[j+2] + vy1 * (__m256i*)(pn-1)[j+2] + vpaddq ymm2, ymm2, ymm12 + vpmuludq ymm13, ymm9, ymmword [rcx-sizeof(qword)+sizeof(ymmword)*2] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm12, ymm5, ymmword [rsi-sizeof(qword)+sizeof(ymmword)*3] ; r3 += vb1 * (__m256i*)(pa-1)[j+3] + vy1 * (__m256i*)(pn-1)[j+3] + vpaddq ymm3, ymm3, ymm12 + vpmuludq ymm13, ymm9, ymmword [rcx-sizeof(qword)+sizeof(ymmword)*3] + vpaddq ymm3, ymm3, ymm13 + + vpmuludq ymm12, ymm6, ymmword [rsi-sizeof(qword)*2] ; r0 += vb2 * (__m256i*)(pa-2)[j] + vy2 * (__m256i*)(pn-2)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2] + vpaddq ymm0, ymm0, ymm13 + vpmuludq ymm12, ymm6, ymmword [rsi-sizeof(qword)*2+sizeof(ymmword)] ; r1 += vb2 * (__m256i*)(pa-2)[j+1] + vy2 * (__m256i*)(pn-2)[j+1] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2+sizeof(ymmword)] + vpaddq ymm1, ymm1, ymm13 + vpmuludq ymm12, ymm6, ymmword [rsi-sizeof(qword)*2+sizeof(ymmword)*2] ; r2 += vb2 * (__m256i*)(pa-2)[j+2] + vy2 * (__m256i*)(pn-2)[j+2] + vpaddq ymm2, ymm2, ymm12 + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2+sizeof(ymmword)*2] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm12, ymm6, ymmword [rsi-sizeof(qword)*2+sizeof(ymmword)*3] ; r3 += vb2 * (__m256i*)(pa-2)[j+3] + vy2 * (__m256i*)(pn-2)[j+3] + vpaddq ymm3, ymm3, ymm12 + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2+sizeof(ymmword)*3] + vpaddq ymm3, ymm3, ymm13 + + vpmuludq ymm12, ymm7, ymmword [rsi-sizeof(qword)*3] ; r0 += vb3 * (__m256i*)(pa-3)[j] + vy3 * (__m256i*)(pn-3)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm11,ymmword [rcx-sizeof(qword)*3] + vpaddq ymm0, ymm0, ymm13 + vpmuludq ymm12, ymm7, ymmword [rsi-sizeof(qword)*3+sizeof(ymmword)] ; r1 += vb3 * (__m256i*)(pa-3)[j+1] + vy3 * (__m256i*)(pn-3)[j+1] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm11,ymmword [rcx-sizeof(qword)*3+sizeof(ymmword)] + vpaddq ymm1, ymm1, ymm13 + vpmuludq ymm12, ymm7, ymmword [rsi-sizeof(qword)*3+sizeof(ymmword)*2] ; r2 += vb3 * (__m256i*)(pa-3)[j+2] + vy3 * (__m256i*)(pn-3)[j+2] + vpaddq ymm2, ymm2, ymm12 + vpmuludq ymm13, ymm11,ymmword [rcx-sizeof(qword)*3+sizeof(ymmword)*2] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm12, ymm7, ymmword [rsi-sizeof(qword)*3+sizeof(ymmword)*3] ; r3 += vb3 * (__m256i*)(pa-3)[j+3] + vy3 * (__m256i*)(pn-3)[j+3] + vpaddq ymm3, ymm3, ymm12 + vpmuludq ymm13, ymm11,ymmword [rcx-sizeof(qword)*3+sizeof(ymmword)*3] + vpaddq ymm3, ymm3, ymm13 + + vmovdqu ymmword [rdi-sizeof(ymmword)], ymm0 + vmovdqu ymmword [rdi-sizeof(ymmword)+sizeof(ymmword)], ymm1 + vmovdqu ymmword [rdi-sizeof(ymmword)+sizeof(ymmword)*2], ymm2 + vmovdqu ymmword [rdi-sizeof(ymmword)+sizeof(ymmword)*3], ymm3 + + add rdi, sizeof(ymmword)*4 + add rsi, sizeof(ymmword)*4 + add rcx, sizeof(ymmword)*4 + jmp .loop16_A + +.exit_loop16_A: + add r11, 4*(sizeof(ymmword)/sizeof(qword)) + jz .exitA + +.loop4_A: + sub r11, sizeof(ymmword)/sizeof(qword) + jl .exitA + + vmovdqu ymm0, ymmword [rdi] ; r0 + + vpmuludq ymm12, ymm4, ymmword [rsi] ; r0 += vb0 * (__m256i*)(pa)[j] + vy0 * (__m256i*)(pn)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx] + vpaddq ymm0, ymm0, ymm13 + + vpmuludq ymm1, ymm5, ymmword [rsi-sizeof(qword)] ; r0 += vb1 * (__m256i*)(pa-1)[j] + vy1 * (__m256i*)(pn-1)[j] + vpaddq ymm0, ymm0, ymm1 + vpmuludq ymm2, ymm9, ymmword [rcx-sizeof(qword)] + vpaddq ymm0, ymm0, ymm2 + + vpmuludq ymm12, ymm6, ymmword [rsi-sizeof(qword)*2] ; r0 += vb2 * (__m256i*)(pa-2)[j] + vy2 * (__m256i*)(pn-2)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2] + vpaddq ymm0, ymm0, ymm13 + + vpmuludq ymm1, ymm7, ymmword [rsi-sizeof(qword)*3] ; r0 += vb3 * (__m256i*)(pa-3)[j] + vy3 * (__m256i*)(pn-3)[j] + vpaddq ymm0, ymm0, ymm1 + vpmuludq ymm2, ymm11,ymmword [rcx-sizeof(qword)*3] + vpaddq ymm0, ymm0, ymm2 + + vmovdqu ymmword [rdi-sizeof(ymmword)], ymm0 + + add rdi, sizeof(ymmword) + add rsi, sizeof(ymmword) + add rcx, sizeof(ymmword) + jmp .loop4_A + +.exitA: + vpmuludq ymm0, ymm7, ymmword [rsi-sizeof(qword)*3] ; r0 += vb3 * (__m256i*)(pa-3)[j] + vy3 * (__m256i*)(pn-3)[j] + vpmuludq ymm1, ymm11,ymmword [rcx-sizeof(qword)*3] + vpaddq ymm0, ymm0, ymm1 + vmovdqu ymmword [rdi-sizeof(ymmword)], ymm0 + + jmp .loop4_B + +;; +;; process latest b[redLen-2] and b[redLen-1] +;; +.exit_loop4_B: + mov rbx, qword [rbp] ; rbx = b[redLen-2] + vpbroadcastq ymm4, qword [rbp] + + mov rdi, qword [rsp+pResult] ; restore pointer to destination + mov rsi, qword [rsp+pA] ; A operand + mov rcx, qword [rsp+pM] ; modulus + + mov r10, qword [rdi] + mov r11, qword [rdi+sizeof(qword)] + mov r12, qword [rdi+sizeof(qword)*2] + mov r13, qword [rdi+sizeof(qword)*3] + + mov rax, rbx ; ac0 = pa[0]*b[j]+pr[0] + imul rax, qword [rsi] + add r10, rax + mov rdx, r10 ; y0 = (ac0*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + mov rax, rbx ; ac1 = pa[1]*b[j]+pr[1] + imul rax, qword [rsi+sizeof(qword)] + add r11, rax + mov rax, rbx ; ac2 = pa[2]*b[j]+pr[2] + imul rax, qword [rsi+sizeof(qword)*2] + add r12, rax + mov rax, rbx ; ac3 = pa[3]*b[j]+pr[3] + imul rax, qword [rsi+sizeof(qword)*3] + add r13, rax + vmovd xmm8, edx + vpbroadcastq ymm8, xmm8 + + mov rax, rdx ; ac0 += pn[0]*y0 + imul rax, qword [rcx] + add r10, rax + shr r10, DIGIT_BITS + mov rax, rdx ; ac1 += pn[1]*y0 + imul rax, qword [rcx+sizeof(qword)] + add r11, rax + add r11, r10 + mov rax, rdx ; ac2 += pn[2]*y0 + imul rax, qword [rcx+sizeof(qword)*2] + add r12, rax + mov rax, rdx ; ac3 += pn[3]*y0 + imul rax, qword [rcx+sizeof(qword)*3] + add r13, rax + + + mov rbx, qword [rbp+sizeof(qword)] ; rbx = b[j+1] + vpbroadcastq ymm5, qword [rbp+sizeof(qword)] + mov rax, rbx ; ac1 += pa[0]*b[j+1] + imul rax, qword [rsi] + add r11, rax + mov rdx, r11 ; y1 = (ac1*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + mov rax, rbx ; ac2 += pa[1]*b[j+1] + imul rax, qword [rsi+sizeof(qword)] + add r12, rax + mov rax, rbx ; ac3 += pa[2]*b[j+1] + imul rax, qword [rsi+sizeof(qword)*2] + add r13, rax + vmovd xmm9, edx + vpbroadcastq ymm9, xmm9 + + mov rax, rdx ; ac1 += pn[0]*y1 + imul rax, qword [rcx] + add r11, rax + shr r11, DIGIT_BITS + mov rax, rdx ; ac2 += pn[1]*y1 + imul rax, qword [rcx+sizeof(qword)] + add r12, rax + add r12, r11 + mov rax, rdx ; ac3 += pn[2]*y1 + imul rax, qword [rcx+sizeof(qword)*2] + add r13, rax + + mov qword [rdi], r12 ; pr[0] = ac2 + mov qword [rdi+sizeof(qword)], r13 ; pr[1] = ac3 + + add rbp, sizeof(qword)*4 + add rsi, sizeof(qword)*4 + add rcx, sizeof(qword)*4 + add rdi, sizeof(ymmword) + + mov r11, r14 ; init a_counter + sub r11, sizeof(ymmword)/sizeof(qword) + +align IPP_ALIGN_FACTOR +.rem_loop4_A: + sub r11, sizeof(ymmword)/sizeof(qword) + jl .exit_rem_loop4_A + + vmovdqu ymm0, ymmword [rdi] ; r0 + + vpmuludq ymm12, ymm4, ymmword [rsi] ; r0 += vb0 * (__m256i*)(pa)[j] + vy0 * (__m256i*)(pn)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx] + vpaddq ymm0, ymm0, ymm13 + + vpmuludq ymm12, ymm5, ymmword [rsi-sizeof(qword)] ; r0 += vb1 * (__m256i*)(pa-1)[j] + vy1 * (__m256i*)(pn-1)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm9, ymmword [rcx-sizeof(qword)] + vpaddq ymm0, ymm0, ymm13 + + vmovdqu ymmword [rdi+sizeof(qword)*2-sizeof(ymmword)], ymm0 + + add rdi, sizeof(ymmword) + add rsi, sizeof(ymmword) + add rcx, sizeof(ymmword) + jmp .rem_loop4_A + +.exit_rem_loop4_A: + + ;; + ;; normalize + ;; + mov rdi, qword [rsp+pR] + mov rsi, qword [rsp+pResult] + mov r8, qword [rsp+redLen] + xor rax, rax +.norm_loop: + add rax, qword [rsi] + add rsi, sizeof(qword) + mov rdx, dword DIGIT_MASK + and rdx, rax + shr rax, DIGIT_BITS + mov qword [rdi], rdx + add rdi, sizeof(qword) + sub r8, 1 + jg .norm_loop + mov qword [rdi], rax + + REST_XMM_AVX + REST_GPR + ret +ENDFUNC cpMontMul4n2_avx2 + + +;************************************************************* +;* void cpMontMul4n3_avx2(Ipp64u* pR, +;* const Ipp64u* pA, +;* const Ipp64u* pB, +;* const Ipp64u* pModulo, int mSize, +;* Ipp64u m0, +;* Ipp64u* pBuffer) +;************************************************************* +align IPP_ALIGN_FACTOR +IPPASM cpMontMul4n3_avx2,PUBLIC +%assign LOCAL_FRAME sizeof(qword)*5 + USES_GPR rsi,rdi,rbx,rbp,r12,r13,r14 + USES_XMM_AVX ymm6,ymm7,ymm8,ymm9,ymm10,ymm11,ymm12,ymm13 + COMP_ABI 7 + + mov rbp,rdx ; pointer to B operand + movsxd r8, r8d ; redLen value counter + +; +; stack struct +; +%xdefine pR 0 ; pointer to result +%xdefine pResult pR+sizeof(qword) ; pointer to buffer +%xdefine pA pResult+sizeof(qword) ; pointer to A operand +%xdefine pM pA+sizeof(qword) ; pointer to modulus +%xdefine redLen pM+sizeof(qword) ; length + + mov qword [rsp+pR], rdi ; save pointer to destination + mov qword [rsp+pA], rsi ; A operand + mov qword [rsp+pM], rcx ; modulus + mov qword [rsp+redLen], r8 ; modulus length + + mov rdi, qword [rsp+ARG_7] ; buffer + mov [rsp+pResult], rdi + + vpxor ymm0, ymm0, ymm0 + xor rax, rax + +;; expands A and M operands + vmovdqu ymmword [rsi+r8*sizeof(qword)], ymm0 + vmovdqu ymmword [rcx+r8*sizeof(qword)], ymm0 + +;; clear result buffer of (redLen+4) qword length + mov r14, r8 +align IPP_ALIGN_FACTOR +.clearLoop: + vmovdqu ymmword [rdi], ymm0 + add rdi, sizeof(ymmword) + sub r14, sizeof(ymmword)/sizeof(qword) + jg .clearLoop + vmovdqu xmmword [rdi], xmm0 + mov qword [rdi+sizeof(xmmword)], rax + + lea r14, [r8+sizeof(ymmword)/sizeof(qword)-1] ; a_counter = (redLen+3) & (-4) + and r14,-(sizeof(ymmword)/sizeof(qword)) + +align IPP_ALIGN_FACTOR +;; +;; process b[] by quadruples (b[j], b[j+1], b[j+2] and b[j+3]) per pass +;; +.loop4_B: + sub r8, sizeof(ymmword)/sizeof(qword) + jl .exit_loop4_B + + mov rbx, qword [rbp] ; rbx = b[j] + vpbroadcastq ymm4, qword [rbp] + + mov rdi, qword [rsp+pResult] ; restore pointer to destination + mov rsi, qword [rsp+pA] ; A operand + mov rcx, qword [rsp+pM] ; modulus + + mov r10, qword [rdi] + mov r11, qword [rdi+sizeof(qword)] + mov r12, qword [rdi+sizeof(qword)*2] + mov r13, qword [rdi+sizeof(qword)*3] + + mov rax, rbx ; ac0 = pa[0]*b[j]+pr[0] + imul rax, qword [rsi] + add r10, rax + mov rdx, r10 ; y0 = (ac0*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + mov rax, rbx ; ac1 = pa[1]*b[j]+pr[1] + imul rax, qword [rsi+sizeof(qword)] + add r11, rax + mov rax, rbx ; ac2 = pa[2]*b[j]+pr[2] + imul rax, qword [rsi+sizeof(qword)*2] + add r12, rax + mov rax, rbx ; ac3 = pa[3]*b[j]+pr[3] + imul rax, qword [rsi+sizeof(qword)*3] + add r13, rax + vmovd xmm8, edx + vpbroadcastq ymm8, xmm8 + + mov rax, rdx ; ac0 += pn[0]*y0 + imul rax, qword [rcx] + add r10, rax + shr r10, DIGIT_BITS + mov rax, rdx ; ac1 += pn[1]*y0 + imul rax, qword [rcx+sizeof(qword)] + add r11, rax + add r11, r10 + mov rax, rdx ; ac2 += pn[2]*y0 + imul rax, qword [rcx+sizeof(qword)*2] + add r12, rax + mov rax, rdx ; ac3 += pn[3]*y0 + imul rax, qword [rcx+sizeof(qword)*3] + add r13, rax + + + mov rbx, qword [rbp+sizeof(qword)] ; rbx = b[j+1] + vpbroadcastq ymm5, qword [rbp+sizeof(qword)] + mov rax, rbx ; ac1 += pa[0]*b[j+1] + imul rax, qword [rsi] + add r11, rax + mov rdx, r11 ; y1 = (ac1*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + mov rax, rbx ; ac2 += pa[1]*b[j+1] + imul rax, qword [rsi+sizeof(qword)] + add r12, rax + mov rax, rbx ; ac3 += pa[2]*b[j+1] + imul rax, qword [rsi+sizeof(qword)*2] + add r13, rax + vmovd xmm9, edx + vpbroadcastq ymm9, xmm9 + + mov rax, rdx ; ac1 += pn[0]*y1 + imul rax, qword [rcx] + add r11, rax + shr r11, DIGIT_BITS + mov rax, rdx ; ac2 += pn[1]*y1 + imul rax, qword [rcx+sizeof(qword)] + add r12, rax + add r12, r11 + mov rax, rdx ; ac3 += pn[2]*y1 + imul rax, qword [rcx+sizeof(qword)*2] + add r13, rax + + + mov rbx, qword [rbp+sizeof(qword)*2] ; rbx = b[j+2] + vpbroadcastq ymm6, qword [rbp+sizeof(qword)*2] + mov rax, rbx ; ac2 += pa[0]*b[j+2] + imul rax, qword [rsi] + add r12, rax + mov rdx, r12 ; y2 = (ac2*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + mov rax, rbx ; ac2 += pa[0]*b[j+2] + imul rax, qword [rsi+sizeof(qword)] + add r13, rax + vmovd xmm10, edx + vpbroadcastq ymm10, xmm10 + + mov rax, rdx ; ac2 += pn[0]*y2 + imul rax, qword [rcx] + add r12, rax + shr r12, DIGIT_BITS + mov rax, rdx ; ac3 += pn[1]*y2 + imul rax, qword [rcx+sizeof(qword)] + add r13, rax + add r13, r12 + + mov rbx, qword [rbp+sizeof(qword)*3] ; rbx = b[j+3] + vpbroadcastq ymm7, qword [rbp+sizeof(qword)*3] + imul rbx, qword [rsi] ; ac3 += pa[0]*b[j+3] + add r13, rbx + mov rdx, r13 ; y3 = (ac3*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + vmovd xmm11, edx + vpbroadcastq ymm11, xmm11 + + imul rdx, qword [rcx] ; ac3 += pn[0]*y3 + add r13, rdx + shr r13, DIGIT_BITS + vmovq xmm0, r13 + + vpaddq ymm0, ymm0, ymmword [rdi+sizeof(ymmword)] + vmovdqu ymmword [rdi+sizeof(ymmword)], ymm0 + + add rbp, sizeof(qword)*4 + add rsi, sizeof(qword)*4 + add rcx, sizeof(qword)*4 + add rdi, sizeof(ymmword) + + mov r11, r14 ; init a_counter + + ;; (vb0) ymm4 = {b0:b0:b0:b0} + ;; (vb1) ymm5 = {b1:b1:b1:b1} + ;; (vb2) ymm6 = {b2:b2:b2:b2} + ;; (vb3) ymm7 = {b3:b3:b3:b3} + + ;; (vy0) ymm8 = {y0:y0:y0:y0} + ;; (vy1) ymm9 = {y1:y1:y1:y1} + ;; (vy2) ymm10= {y2:y2:y2:y2} + ;; (vy3) ymm11= {y3:y3:y3:y3} + + sub r11, sizeof(ymmword)/sizeof(qword) + +align IPP_ALIGN_FACTOR +.loop16_A: + sub r11, 4*sizeof(ymmword)/sizeof(qword) + jl .exit_loop16_A + + vmovdqu ymm0, ymmword [rdi] ; r0 + vmovdqu ymm1, ymmword [rdi+sizeof(ymmword)] ; r1 + vmovdqu ymm2, ymmword [rdi+sizeof(ymmword)*2] ; r2 + vmovdqu ymm3, ymmword [rdi+sizeof(ymmword)*3] ; r3 + + vpmuludq ymm12, ymm4, ymmword [rsi] ; r0 += vb0 * (__m256i*)(pa)[j] + vy0 * (__m256i*)(pn)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx] + vpaddq ymm0, ymm0, ymm13 + vpmuludq ymm12, ymm4, ymmword [rsi+sizeof(ymmword)] ; r1 += vb0 * (__m256i*)(pa)[j+1] + vy0 * (__m256i*)(pn)[j+1] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx+sizeof(ymmword)] + vpaddq ymm1, ymm1, ymm13 + vpmuludq ymm12, ymm4, ymmword [rsi+sizeof(ymmword)*2] ; r2 += vb0 * (__m256i*)(pa)[j+2] + vy0 * (__m256i*)(pn)[j+2] + vpaddq ymm2, ymm2, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx+sizeof(ymmword)*2] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm12, ymm4, ymmword [rsi+sizeof(ymmword)*3] ; r3 += vb0 * (__m256i*)(pa)[j+3] + vy0 * (__m256i*)(pn)[j+3] + vpaddq ymm3, ymm3, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx+sizeof(ymmword)*3] + vpaddq ymm3, ymm3, ymm13 + + vpmuludq ymm12, ymm5, ymmword [rsi-sizeof(qword)] ; r0 += vb1 * (__m256i*)(pa-1)[j] + vy1 * (__m256i*)(pn-1)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm9, ymmword [rcx-sizeof(qword)] + vpaddq ymm0, ymm0, ymm13 + vpmuludq ymm12, ymm5, ymmword [rsi-sizeof(qword)+sizeof(ymmword)] ; r1 += vb1 * (__m256i*)(pa-1)[j+1] + vy1 * (__m256i*)(pn-1)[j+1] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm9, ymmword [rcx-sizeof(qword)+sizeof(ymmword)] + vpaddq ymm1, ymm1, ymm13 + vpmuludq ymm12, ymm5, ymmword [rsi-sizeof(qword)+sizeof(ymmword)*2] ; r2 += vb1 * (__m256i*)(pa-1)[j+2] + vy1 * (__m256i*)(pn-1)[j+2] + vpaddq ymm2, ymm2, ymm12 + vpmuludq ymm13, ymm9, ymmword [rcx-sizeof(qword)+sizeof(ymmword)*2] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm12, ymm5, ymmword [rsi-sizeof(qword)+sizeof(ymmword)*3] ; r3 += vb1 * (__m256i*)(pa-1)[j+3] + vy1 * (__m256i*)(pn-1)[j+3] + vpaddq ymm3, ymm3, ymm12 + vpmuludq ymm13, ymm9, ymmword [rcx-sizeof(qword)+sizeof(ymmword)*3] + vpaddq ymm3, ymm3, ymm13 + + vpmuludq ymm12, ymm6, ymmword [rsi-sizeof(qword)*2] ; r0 += vb2 * (__m256i*)(pa-2)[j] + vy2 * (__m256i*)(pn-2)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2] + vpaddq ymm0, ymm0, ymm13 + vpmuludq ymm12, ymm6, ymmword [rsi-sizeof(qword)*2+sizeof(ymmword)] ; r1 += vb2 * (__m256i*)(pa-2)[j+1] + vy2 * (__m256i*)(pn-2)[j+1] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2+sizeof(ymmword)] + vpaddq ymm1, ymm1, ymm13 + vpmuludq ymm12, ymm6, ymmword [rsi-sizeof(qword)*2+sizeof(ymmword)*2] ; r2 += vb2 * (__m256i*)(pa-2)[j+2] + vy2 * (__m256i*)(pn-2)[j+2] + vpaddq ymm2, ymm2, ymm12 + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2+sizeof(ymmword)*2] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm12, ymm6, ymmword [rsi-sizeof(qword)*2+sizeof(ymmword)*3] ; r3 += vb2 * (__m256i*)(pa-2)[j+3] + vy2 * (__m256i*)(pn-2)[j+3] + vpaddq ymm3, ymm3, ymm12 + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2+sizeof(ymmword)*3] + vpaddq ymm3, ymm3, ymm13 + + vpmuludq ymm12, ymm7, ymmword [rsi-sizeof(qword)*3] ; r0 += vb3 * (__m256i*)(pa-3)[j] + vy3 * (__m256i*)(pn-3)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm11,ymmword [rcx-sizeof(qword)*3] + vpaddq ymm0, ymm0, ymm13 + vpmuludq ymm12, ymm7, ymmword [rsi-sizeof(qword)*3+sizeof(ymmword)] ; r1 += vb3 * (__m256i*)(pa-3)[j+1] + vy3 * (__m256i*)(pn-3)[j+1] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm11,ymmword [rcx-sizeof(qword)*3+sizeof(ymmword)] + vpaddq ymm1, ymm1, ymm13 + vpmuludq ymm12, ymm7, ymmword [rsi-sizeof(qword)*3+sizeof(ymmword)*2] ; r2 += vb3 * (__m256i*)(pa-3)[j+2] + vy3 * (__m256i*)(pn-3)[j+2] + vpaddq ymm2, ymm2, ymm12 + vpmuludq ymm13, ymm11,ymmword [rcx-sizeof(qword)*3+sizeof(ymmword)*2] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm12, ymm7, ymmword [rsi-sizeof(qword)*3+sizeof(ymmword)*3] ; r3 += vb3 * (__m256i*)(pa-3)[j+3] + vy3 * (__m256i*)(pn-3)[j+3] + vpaddq ymm3, ymm3, ymm12 + vpmuludq ymm13, ymm11,ymmword [rcx-sizeof(qword)*3+sizeof(ymmword)*3] + vpaddq ymm3, ymm3, ymm13 + + vmovdqu ymmword [rdi-sizeof(ymmword)], ymm0 + vmovdqu ymmword [rdi-sizeof(ymmword)+sizeof(ymmword)], ymm1 + vmovdqu ymmword [rdi-sizeof(ymmword)+sizeof(ymmword)*2], ymm2 + vmovdqu ymmword [rdi-sizeof(ymmword)+sizeof(ymmword)*3], ymm3 + + add rdi, sizeof(ymmword)*4 + add rsi, sizeof(ymmword)*4 + add rcx, sizeof(ymmword)*4 + jmp .loop16_A + +.exit_loop16_A: + add r11, 4*(sizeof(ymmword)/sizeof(qword)) + jz .exitA + +.loop4_A: + sub r11, sizeof(ymmword)/sizeof(qword) + jl .exitA + + vmovdqu ymm0, ymmword [rdi] ; r0 + + vpmuludq ymm12, ymm4, ymmword [rsi] ; r0 += vb0 * (__m256i*)(pa)[j] + vy0 * (__m256i*)(pn)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx] + vpaddq ymm0, ymm0, ymm13 + + vpmuludq ymm1, ymm5, ymmword [rsi-sizeof(qword)] ; r0 += vb1 * (__m256i*)(pa-1)[j] + vy1 * (__m256i*)(pn-1)[j] + vpaddq ymm0, ymm0, ymm1 + vpmuludq ymm2, ymm9, ymmword [rcx-sizeof(qword)] + vpaddq ymm0, ymm0, ymm2 + + vpmuludq ymm12, ymm6, ymmword [rsi-sizeof(qword)*2] ; r0 += vb2 * (__m256i*)(pa-2)[j] + vy2 * (__m256i*)(pn-2)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2] + vpaddq ymm0, ymm0, ymm13 + + vpmuludq ymm1, ymm7, ymmword [rsi-sizeof(qword)*3] ; r0 += vb3 * (__m256i*)(pa-3)[j] + vy3 * (__m256i*)(pn-3)[j] + vpaddq ymm0, ymm0, ymm1 + vpmuludq ymm2, ymm11,ymmword [rcx-sizeof(qword)*3] + vpaddq ymm0, ymm0, ymm2 + + vmovdqu ymmword [rdi-sizeof(ymmword)], ymm0 + + add rdi, sizeof(ymmword) + add rsi, sizeof(ymmword) + add rcx, sizeof(ymmword) + jmp .loop4_A + +.exitA: + vpmuludq ymm2, ymm6, ymmword [rsi-sizeof(qword)*2] ; r2 += vb3 * (__m256i*)(pa-2)[j] + vy2 * (__m256i*)(pn-2)[j] + vpmuludq ymm12, ymm10,ymmword [rcx-sizeof(qword)*2] + vpaddq ymm2, ymm2, ymm12 + + vpmuludq ymm3, ymm7, ymmword [rsi-sizeof(qword)*3] ; r3 += vb3 * (__m256i*)(pa-3)[j] + vy3 * (__m256i*)(pn-3)[j] + vpmuludq ymm13, ymm11,ymmword [rcx-sizeof(qword)*3] + vpaddq ymm3, ymm3, ymm13 + + vpaddq ymm2, ymm2, ymm3 + vmovdqu ymmword [rdi-sizeof(ymmword)], ymm2 + + jmp .loop4_B + +;; +;; process latest b[redLen-3], b[redLen-2] and b[redLen-1] +;; +.exit_loop4_B: + mov rbx, qword [rbp] ; rbx = b[redLen-3] + vpbroadcastq ymm4, qword [rbp] + + mov rdi, qword [rsp+pResult] ; restore pointer to destination + mov rsi, qword [rsp+pA] ; A operand + mov rcx, qword [rsp+pM] ; modulus + + mov r10, qword [rdi] + mov r11, qword [rdi+sizeof(qword)] + mov r12, qword [rdi+sizeof(qword)*2] + mov r13, qword [rdi+sizeof(qword)*3] + + mov rax, rbx ; ac0 = pa[0]*b[j]+pr[0] + imul rax, qword [rsi] + add r10, rax + mov rdx, r10 ; y0 = (ac0*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + mov rax, rbx ; ac1 = pa[1]*b[j]+pr[1] + imul rax, qword [rsi+sizeof(qword)] + add r11, rax + mov rax, rbx ; ac2 = pa[2]*b[j]+pr[2] + imul rax, qword [rsi+sizeof(qword)*2] + add r12, rax + mov rax, rbx ; ac3 = pa[3]*b[j]+pr[3] + imul rax, qword [rsi+sizeof(qword)*3] + add r13, rax + vmovd xmm8, edx + vpbroadcastq ymm8, xmm8 + + mov rax, rdx ; ac0 += pn[0]*y0 + imul rax, qword [rcx] + add r10, rax + shr r10, DIGIT_BITS + mov rax, rdx ; ac1 += pn[1]*y0 + imul rax, qword [rcx+sizeof(qword)] + add r11, rax + add r11, r10 + mov rax, rdx ; ac2 += pn[2]*y0 + imul rax, qword [rcx+sizeof(qword)*2] + add r12, rax + mov rax, rdx ; ac3 += pn[3]*y0 + imul rax, qword [rcx+sizeof(qword)*3] + add r13, rax + + + mov rbx, qword [rbp+sizeof(qword)] ; rbx = b[redLen-2] + vpbroadcastq ymm5, qword [rbp+sizeof(qword)] + mov rax, rbx ; ac1 += pa[0]*b[j+1] + imul rax, qword [rsi] + add r11, rax + mov rdx, r11 ; y1 = (ac1*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + mov rax, rbx ; ac2 += pa[1]*b[j+1] + imul rax, qword [rsi+sizeof(qword)] + add r12, rax + mov rax, rbx ; ac3 += pa[2]*b[j+1] + imul rax, qword [rsi+sizeof(qword)*2] + add r13, rax + vmovd xmm9, edx + vpbroadcastq ymm9, xmm9 + + mov rax, rdx ; ac1 += pn[0]*y1 + imul rax, qword [rcx] + add r11, rax + shr r11, DIGIT_BITS + mov rax, rdx ; ac2 += pn[1]*y1 + imul rax, qword [rcx+sizeof(qword)] + add r12, rax + add r12, r11 + mov rax, rdx ; ac3 += pn[2]*y1 + imul rax, qword [rcx+sizeof(qword)*2] + add r13, rax + + + mov rbx, qword [rbp+sizeof(qword)*2] ; rbx = b[redLen-1] + vpbroadcastq ymm6, qword [rbp+sizeof(qword)*2] + mov rax, rbx ; ac2 += pa[0]*b[j+2] + imul rax, qword [rsi] + add r12, rax + mov rdx, r12 ; y2 = (ac2*m0) & DIGIT_MASK + imul edx, r9d + and edx, DIGIT_MASK + mov rax, rbx ; ac2 += pa[0]*b[j+2] + imul rax, qword [rsi+sizeof(qword)] + add r13, rax + vmovd xmm10, edx + vpbroadcastq ymm10, xmm10 + + mov rax, rdx ; ac2 += pn[0]*y2 + imul rax, qword [rcx] + add r12, rax + shr r12, DIGIT_BITS + mov rax, rdx ; ac3 += pn[1]*y2 + imul rax, qword [rcx+sizeof(qword)] + add r13, rax + add r13, r12 + + mov qword [rdi], r13 ; pr[0] = ac3 + + add rbp, sizeof(qword)*4 + add rsi, sizeof(qword)*4 + add rcx, sizeof(qword)*4 + add rdi, sizeof(ymmword) + + mov r11, r14 ; init a_counter + sub r11, sizeof(ymmword)/sizeof(qword) + +align IPP_ALIGN_FACTOR +.rem_loop4_A: + sub r11, sizeof(ymmword)/sizeof(qword) + jl .exit_rem_loop4_A + + vmovdqu ymm0, ymmword [rdi] ; r0 + + vpmuludq ymm12, ymm4, ymmword [rsi] ; r0 += vb0 * (__m256i*)(pa)[j] + vy0 * (__m256i*)(pn)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm8, ymmword [rcx] + vpaddq ymm0, ymm0, ymm13 + + vpmuludq ymm12, ymm5, ymmword [rsi-sizeof(qword)] ; r0 += vb1 * (__m256i*)(pa-1)[j] + vy1 * (__m256i*)(pn-1)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm9, ymmword [rcx-sizeof(qword)] + vpaddq ymm0, ymm0, ymm13 + + vpmuludq ymm12, ymm6, ymmword [rsi-sizeof(qword)*2]; r0 += vb2 * (__m256i*)(pa-2)[j] + vy2 * (__m256i*)(pn-2)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2] + vpaddq ymm0, ymm0, ymm13 + + vmovdqu ymmword [rdi+sizeof(qword)-sizeof(ymmword)], ymm0 + + add rdi, sizeof(ymmword) + add rsi, sizeof(ymmword) + add rcx, sizeof(ymmword) + jmp .rem_loop4_A + +.exit_rem_loop4_A: + vmovdqu ymm0, ymmword [rdi] ; r0 + + vpmuludq ymm12, ymm6, ymmword [rsi-sizeof(qword)*2]; r0 += vb2 * (__m256i*)(pa-2)[j] + vy2 * (__m256i*)(pn-2)[j] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm10,ymmword [rcx-sizeof(qword)*2] + vpaddq ymm0, ymm0, ymm13 + + vmovdqu ymmword [rdi+sizeof(qword)-sizeof(ymmword)], ymm0 + + ;; + ;; normalize + ;; + mov rdi, qword [rsp+pR] + mov rsi, qword [rsp+pResult] + mov r8, qword [rsp+redLen] + xor rax, rax +.norm_loop: + add rax, qword [rsi] + add rsi, sizeof(qword) + mov rdx, dword DIGIT_MASK + and rdx, rax + shr rax, DIGIT_BITS + mov qword [rdi], rdx + add rdi, sizeof(qword) + sub r8, 1 + jg .norm_loop + mov qword [rdi], rax + + REST_XMM_AVX + REST_GPR + ret +ENDFUNC cpMontMul4n3_avx2 + + +%endif ; _IPP32E_L9 + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmontreductionm7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmontreductionm7as.asm new file mode 100644 index 0000000..9e361fd --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmontreductionm7as.asm @@ -0,0 +1,1549 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Big Number Operations +; +; Content: +; cpMontRedAdc_BNU() +; +; +; History: +; Extra reduction (R=A-M) has been added to perform MontReduction safe +; +; + +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpbnumulschool.inc" +%include "pcpvariant.inc" + +%if (_ADCOX_NI_ENABLING_ == _FEATURE_OFF_) || (_ADCOX_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_M7) && (_IPP32E < _IPP32E_L9) + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; fixed size reduction +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%macro MREDUCE_FIX_STEP 12.nolist + %xdefine %%mSize %1 + %xdefine %%X7 %2 + %xdefine %%X6 %3 + %xdefine %%X5 %4 + %xdefine %%X4 %5 + %xdefine %%X3 %6 + %xdefine %%X2 %7 + %xdefine %%X1 %8 + %xdefine %%X0 %9 + %xdefine %%rSrc %10 + %xdefine %%U %11 + %xdefine %%rCarry %12 + +%if %%mSize > 0 + mul %%U + xor %%rCarry, %%rCarry + add %%X0, rax + +%if %%mSize > 1 + mov rax, qword [%%rSrc+sizeof(qword)] + adc %%rCarry, rdx + mul %%U + add %%X1, %%rCarry + adc rdx, 0 + xor %%rCarry, %%rCarry + add %%X1, rax + +%if %%mSize > 2 + mov rax, qword [%%rSrc+sizeof(qword)*2] + adc %%rCarry, rdx + mul %%U + add %%X2, %%rCarry + adc rdx, 0 + xor %%rCarry, %%rCarry + add %%X2, rax + +%if %%mSize > 3 + mov rax, qword [%%rSrc+sizeof(qword)*3] + adc %%rCarry, rdx + mul %%U + add %%X3, %%rCarry + adc rdx, 0 + xor %%rCarry, %%rCarry + add %%X3, rax +%if %%mSize == 4 + adc %%X4, rdx + adc %%rCarry, 0 + add %%X4, qword [rsp+carry] +%endif +%endif ;; mSize==4 + +%if %%mSize == 3 + adc %%X3, rdx + adc %%rCarry, 0 + add %%X3, qword [rsp+carry] +%endif +%endif ;; mSize==3 + +%if %%mSize == 2 + adc %%X2, rdx + adc %%rCarry, 0 + add %%X2, qword [rsp+carry] +%endif +%endif ;; mSize==2 + +%if %%mSize == 1 + adc %%X1, rdx + adc %%rCarry, 0 + add %%X1, qword [rsp+carry] +%endif +%endif ;; mSize==1 + + adc %%rCarry, 0 + mov qword [rsp+carry], %%rCarry +%endmacro + +%macro MREDUCE_FIX 14.nolist + %xdefine %%mSize %1 + %xdefine %%rDst %2 + %xdefine %%rSrc %3 + %xdefine %%M0 %4 + %xdefine %%X7 %5 + %xdefine %%X6 %6 + %xdefine %%X5 %7 + %xdefine %%X4 %8 + %xdefine %%X3 %9 + %xdefine %%X2 %10 + %xdefine %%X1 %11 + %xdefine %%X0 %12 + %xdefine %%U %13 + %xdefine %%rCarry %14 + +%if %%mSize > 0 + mov %%U, %%X0 + imul %%U, %%M0 + mov rax, qword [%%rSrc] + MREDUCE_FIX_STEP %%mSize, %%X7,%%X6,%%X5,%%X4,%%X3,%%X2,%%X1,%%X0, %%rSrc, %%U, %%rCarry + +%if %%mSize > 1 + mov %%U, %%X1 + imul %%U, %%M0 + mov rax, qword [%%rSrc] + MREDUCE_FIX_STEP %%mSize, %%X0,%%X7,%%X6,%%X5,%%X4,%%X3,%%X2,%%X1, %%rSrc, %%U, %%rCarry + +%if %%mSize > 2 + mov %%U, %%X2 + imul %%U, %%M0 + mov rax, qword [%%rSrc] + MREDUCE_FIX_STEP %%mSize, %%X1,%%X0,%%X7,%%X6,%%X5,%%X4,%%X3,%%X2, %%rSrc, %%U, %%rCarry + +%if %%mSize > 3 ; mSize == 4 + mov %%U, %%X3 + imul %%U, %%M0 + mov rax, qword [%%rSrc] + MREDUCE_FIX_STEP %%mSize, %%X2,%%X1,%%X0,%%X7,%%X6,%%X5,%%X4,%%X3, %%rSrc, %%U, %%rCarry + + mov %%X0, %%X4 ; {X3:X2:X1:X0} = {X7:X6:X5:X4} + sub %%X4, qword [%%rSrc] ; {X7:X6:X5:X4}-= modulus + mov %%X1, %%X5 + sbb %%X5, qword [%%rSrc+sizeof(qword)] + mov %%X2, %%X6 + sbb %%X6, qword [%%rSrc+sizeof(qword)*2] + mov %%X3, %%X7 + sbb %%X7, qword [%%rSrc+sizeof(qword)*3] + sbb %%rCarry, 0 + + cmovc %%X4, %%X0 ; dst = borrow? {X3:X2:X1:X0} : {X7:X6:X5:X4} + mov qword [%%rDst], %%X4 + cmovc %%X5, %%X1 + mov qword [%%rDst+sizeof(qword)], %%X5 + cmovc %%X6, %%X2 + mov qword [%%rDst+sizeof(qword)*2], %%X6 + cmovc %%X7, %%X3 + mov qword [%%rDst+sizeof(qword)*3], %%X7 +%endif +%endif +%endif +%endif + +%if %%mSize == 3 + mov %%X0, %%X3 ; {X2:X1:X0} = {X5:X4:X3} + sub %%X3, qword [%%rSrc] ; {X5:X4:X3}-= modulus + mov %%X1, %%X4 + sbb %%X4, qword [%%rSrc+sizeof(qword)] + mov %%X2, %%X5 + sbb %%X5, qword [%%rSrc+sizeof(qword)*2] + sbb %%rCarry, 0 + + cmovc %%X3, %%X0 ; dst = borrow? {X2:X1:X0} : {X5:X4:X3} + mov qword [%%rDst], %%X3 + cmovc %%X4, %%X1 + mov qword [%%rDst+sizeof(qword)], %%X4 + cmovc %%X5, %%X2 + mov qword [%%rDst+sizeof(qword)*2], %%X5 +%endif ; mSize==3 + +%if %%mSize == 2 + mov %%X0, %%X2 ; {X1:X0} = {X3:X2} + sub %%X2, qword [%%rSrc] ; {X3:X2}-= modulus + mov %%X1, %%X3 + sbb %%X3, qword [%%rSrc+sizeof(qword)] + sbb %%rCarry, 0 + + cmovc %%X2, %%X0 ; dst = borrow? {X1:X0} : {X3:X2} + mov qword [%%rDst], %%X2 + cmovc %%X3, %%X1 + mov qword [%%rDst+sizeof(qword)], %%X3 +%endif ; mSize==2 + +%if %%mSize == 1 + mov %%X0, %%X1 ; X1 = X0 + sub %%X1, qword [%%rSrc] ; X1 -= modulus + sbb %%rCarry, 0 + + cmovc %%X1, %%X0 ; dst = borrow? X0 : X1 + mov qword [%%rDst], %%X1 +%endif ; mSize==1 +%endmacro + +%if (_IPP32E <= _IPP32E_Y8) +;; +;; Pre- Sandy Brige specific code +;; + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; product = + modulus * U0 + +;; main body: product = + modulus * U0 +%macro MLAx1 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%U0 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + ALIGN IPP_ALIGN_FACTOR +%%L_1: + mul %%U0 + xor %%T2, %%T2 + add %%T0, qword [%%rDst+%%idx*sizeof(qword)-sizeof(qword)] + adc %%T1, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)] + adc %%T2, rdx + mov qword [%%rDst+%%idx*sizeof(qword)-sizeof(qword)], %%T0 + + mul %%U0 + xor %%T3, %%T3 + add %%T1, qword [%%rDst+%%idx*sizeof(qword)] + adc %%T2, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*2] + adc %%T3, rdx + mov qword [%%rDst+%%idx*sizeof(qword)], %%T1 + + mul %%U0 + xor %%T0, %%T0 + add %%T2, qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)] + adc %%T3, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*3] + adc %%T0, rdx + mov qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)], %%T2 + + mul %%U0 + xor %%T1, %%T1 + add %%T3, qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)*2] + adc %%T0, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*4] + adc %%T1, rdx + mov qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)*2], %%T3 + + add %%idx, 4 + jnc %%L_1 +%endmacro + +;; elipogue: product = modulus * U0, (srcLen=4*n+1) +%macro MM_MLAx1_4N_1_ELOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%U0 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mov rax, qword [%%rSrc+sizeof(qword)] + mov %%idx, qword [rsp+counter] + + mul %%U0 + xor %%T3, %%T3 + add %%T1, qword [%%rDst] + adc %%T2, rax + mov rax, qword [%%rSrc+sizeof(qword)*2] + adc %%T3, rdx + mov qword [%%rDst], %%T1 + + mul %%U0 + xor %%T0, %%T0 + add %%T2, qword [%%rDst+sizeof(qword)] + adc %%T3, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] + adc %%T0, rdx + mov qword [%%rDst+sizeof(qword)], %%T2 + + mul %%U0 + add %%T3, qword [%%rDst+sizeof(qword)*2] + adc %%T0, rax + adc rdx, 0 + xor rax, rax + mov qword [%%rDst+sizeof(qword)*2], %%T3 + + add %%T0, qword [%%rDst+sizeof(qword)*3] + adc rdx, qword [%%rDst+sizeof(qword)*4] + adc rax, 0 + mov qword [%%rDst+sizeof(qword)*3], %%T0 + mov qword [%%rDst+sizeof(qword)*4], rdx + mov qword [rsp+carry], rax + + add %%rDst, sizeof(qword) +%endmacro + +;; elipogue: product = modulus * U0, (srcLen=4*n+4) +%macro MM_MLAx1_4N_4_ELOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%U0 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mov rax, qword [%%rSrc+sizeof(qword)*2] + mov %%idx, qword [rsp+counter] + + mul %%U0 + xor %%T3, %%T0 + add %%T1, qword [%%rDst+sizeof(qword)] + adc %%T2, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] + adc %%T3, rdx + mov qword [%%rDst+sizeof(qword)], %%T1 + + mul %%U0 + add %%T2, qword [%%rDst+sizeof(qword)*2] + adc %%T3, rax + adc rdx, 0 + xor rax, rax + mov qword [%%rDst+sizeof(qword)*2], %%T2 + + add %%T3, qword [%%rDst+sizeof(qword)*3] + adc rdx, qword [%%rDst+sizeof(qword)*4] + adc rax, 0 + mov qword [%%rDst+sizeof(qword)*3], %%T3 + mov qword [%%rDst+sizeof(qword)*4], rdx + mov qword [rsp+carry], rax + + add %%rDst, sizeof(qword) +%endmacro + +;; elipogue: product = modulus * U0, (srcLen=4*n+3) +%macro MM_MLAx1_4N_3_ELOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%U0 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mov rax, qword [%%rSrc+sizeof(qword)*3] + mov %%idx, qword [rsp+counter] + + mul %%U0 + add %%T1, qword [%%rDst+sizeof(qword)*2] + adc %%T2, rax + adc rdx, 0 + xor rax, rax + mov qword [%%rDst+sizeof(qword)*2], %%T1 + + add %%T2, qword [%%rDst+sizeof(qword)*3] + adc rdx, qword [%%rDst+sizeof(qword)*4] + adc rax, 0 + mov qword [%%rDst+sizeof(qword)*3], %%T2 + mov qword [%%rDst+sizeof(qword)*4], rdx + mov qword [rsp+carry], rax + + add %%rDst, sizeof(qword) +%endmacro + +;; elipogue: product = modulus * U0, (srcLen=4*n+2) +%macro MM_MLAx1_4N_2_ELOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%U0 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mov %%idx, qword [rsp+counter] + xor rax, rax + + add %%T1, qword [%%rDst+sizeof(qword)*3] + adc %%T2, qword [%%rDst+sizeof(qword)*4] + adc rax, 0 + mov qword [%%rDst+sizeof(qword)*3], %%T1 + mov qword [%%rDst+sizeof(qword)*4], %%T2 + mov qword [rsp+carry], rax + + add %%rDst, sizeof(qword) +%endmacro + +%endif + + +%if (_IPP32E >= _IPP32E_E9) +;; +;; Sandy Brige specific code +;; +%macro MLAx1 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%U0 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + ALIGN IPP_ALIGN_FACTOR +%%L_1: + mul %%U0 + xor %%T2, %%T2 + add %%T0, qword [%%rDst+%%idx*sizeof(qword)-sizeof(qword)] + mov qword [%%rDst+%%idx*sizeof(qword)-sizeof(qword)], %%T0 + adc %%T1, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)] + adc %%T2, rdx + + mul %%U0 + xor %%T3, %%T3 + add %%T1, qword [%%rDst+%%idx*sizeof(qword)] + mov qword [%%rDst+%%idx*sizeof(qword)], %%T1 + adc %%T2, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*2] + adc %%T3, rdx + + mul %%U0 + xor %%T0, %%T0 + add %%T2, qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)] + mov qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)], %%T2 + adc %%T3, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*3] + adc %%T0, rdx + + mul %%U0 + xor %%T1, %%T1 + add %%T3, qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)*2] + mov qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)*2], %%T3 + adc %%T0, rax + mov rax, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)*4] + adc %%T1, rdx + + add %%idx, 4 + jnc %%L_1 +%endmacro + +%macro MM_MLAx1_4N_1_ELOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%U0 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mov rax, qword [%%rSrc+sizeof(qword)] + mov %%idx, qword [rsp+counter] + + mul %%U0 + xor %%T3, %%T3 + add %%T1, qword [%%rDst] + mov qword [%%rDst], %%T1 + adc %%T2, rax + mov rax, qword [%%rSrc+sizeof(qword)*2] + adc %%T3, rdx + + mul %%U0 + xor %%T0, %%T0 + add %%T2, qword [%%rDst+sizeof(qword)] + mov qword [%%rDst+sizeof(qword)], %%T2 + adc %%T3, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] + adc %%T0, rdx + + mul %%U0 + add %%T3, qword [%%rDst+sizeof(qword)*2] + mov qword [%%rDst+sizeof(qword)*2], %%T3 + adc %%T0, rax + adc rdx, 0 + xor rax, rax + + add %%T0, qword [%%rDst+sizeof(qword)*3] + adc rdx, qword [%%rDst+sizeof(qword)*4] + adc rax, 0 + mov qword [%%rDst+sizeof(qword)*3], %%T0 + mov qword [%%rDst+sizeof(qword)*4], rdx + mov qword [rsp+carry], rax + + add %%rDst, sizeof(qword) +%endmacro + +%macro MM_MLAx1_4N_4_ELOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%U0 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mov rax, qword [%%rSrc+sizeof(qword)*2] + mov %%idx, qword [rsp+counter] + + mul %%U0 + xor %%T3, %%T0 + add %%T1, qword [%%rDst+sizeof(qword)] + mov qword [%%rDst+sizeof(qword)], %%T1 + adc %%T2, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] + adc %%T3, rdx + + mul %%U0 + add %%T2, qword [%%rDst+sizeof(qword)*2] + mov qword [%%rDst+sizeof(qword)*2], %%T2 + adc %%T3, rax + adc rdx, 0 + xor rax, rax + + add %%T3, qword [%%rDst+sizeof(qword)*3] + adc rdx, qword [%%rDst+sizeof(qword)*4] + adc rax, 0 + mov qword [%%rDst+sizeof(qword)*3], %%T3 + mov qword [%%rDst+sizeof(qword)*4], rdx + mov qword [rsp+carry], rax + + add %%rDst, sizeof(qword) +%endmacro + +%macro MM_MLAx1_4N_3_ELOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%U0 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mov rax, qword [%%rSrc+sizeof(qword)*3] + mov %%idx, qword [rsp+counter] + + mul %%U0 + add %%T1, qword [%%rDst+sizeof(qword)*2] + mov qword [%%rDst+sizeof(qword)*2], %%T1 + adc %%T2, rax + adc rdx, 0 + xor rax, rax + + add %%T2, qword [%%rDst+sizeof(qword)*3] + adc rdx, qword [%%rDst+sizeof(qword)*4] + adc rax, 0 + mov qword [%%rDst+sizeof(qword)*3], %%T2 + mov qword [%%rDst+sizeof(qword)*4], rdx + mov qword [rsp+carry], rax + + add %%rDst, sizeof(qword) +%endmacro + +%macro MM_MLAx1_4N_2_ELOG 8.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%U0 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + + mov %%idx, qword [rsp+counter] + xor rax, rax + + add %%T1, qword [%%rDst+sizeof(qword)*3] + adc %%T2, qword [%%rDst+sizeof(qword)*4] + adc rax, 0 + mov qword [%%rDst+sizeof(qword)*3], %%T1 + mov qword [%%rDst+sizeof(qword)*4], %%T2 + mov qword [rsp+carry], rax + + add %%rDst, sizeof(qword) +%endmacro + +%endif + +; +; prologue: +; pre-compute: +; - u0 = product[0]*m0 +; - u1 = (product[1] + LO(modulo[1]*u0) + HI(modulo[0]*u0) carry(product[0]+LO(modulo[0]*u0)))*m0 +; +%macro MMx2_PLOG 10.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%m0 %4 + %xdefine %%U0 %5 + %xdefine %%U1 %6 + %xdefine %%T0 %7 + %xdefine %%T1 %8 + %xdefine %%T2 %9 + %xdefine %%T3 %10 + + mov %%U0, qword [%%rDst+%%idx*sizeof(qword)] ; product[0] + imul %%U0, %%m0 ; u0 = product[0]*m0 + + mov rax, qword [%%rSrc+%%idx*sizeof(qword)] ; modulo[0]*u0 + mul %%U0 + + mov %%U1, qword [%%rSrc+%%idx*sizeof(qword)+sizeof(qword)] ; modulo[1]*u0 + imul %%U1, %%U0 + + mov %%T0, rax ; save modulo[0]*u0 + mov %%T1, rdx + + add rax, qword [rdi+%%idx*sizeof(qword)] ; a[1] = product[1] + LO(modulo[1]*u0) + adc rdx, qword [rdi+%%idx*sizeof(qword)+sizeof(qword)] ; + HI(modulo[0]*u0) + add %%U1, rdx ; + carry(product[0]+LO(modulo[0]*u0)) + imul %%U1, %%m0 ; u1 = a[1]*m0 + + mov rax, qword [%%rSrc+%%idx*sizeof(qword)] + xor %%T2, %%T2 +%endmacro + +%if (_IPP32E <= _IPP32E_Y8) +;; +;; Pre- Sandy Brige specific code +;; +%macro MM_MLAx2_4N_1_ELOG 9.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%U0 %4 + %xdefine %%U1 %5 + %xdefine %%T0 %6 + %xdefine %%T1 %7 + %xdefine %%T2 %8 + %xdefine %%T3 %9 + + mul %%U1 ; {T2:T1} += modulus[mSize-1]*U1 + r[mSize-1] + mov %%idx, qword [rsp+counter] + add %%T0, qword [%%rDst+sizeof(qword)*3] + adc %%T1, rax + mov qword [%%rDst+sizeof(qword)*3], %%T0 + adc %%T2, rdx + xor rax, rax + + add %%T1, qword [%%rDst+sizeof(qword)*4] + adc %%T2, qword [%%rDst+sizeof(qword)*5] + adc rax, 0 + add %%T1, qword [rsp+carry] + adc %%T2, 0 + adc rax, 0 + mov qword [%%rDst+sizeof(qword)*4], %%T1 + mov qword [%%rDst+sizeof(qword)*5], %%T2 + mov qword [rsp+carry], rax + + add %%rDst, sizeof(qword)*2 +%endmacro + +%macro MM_MLAx2_4N_2_ELOG 9.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%U0 %4 + %xdefine %%U1 %5 + %xdefine %%T0 %6 + %xdefine %%T1 %7 + %xdefine %%T2 %8 + %xdefine %%T3 %9 + + mul %%U1 ; {T2:T1} += a[mSize-2]*U1 + xor %%T3, %%T3 + add %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[mSize-1] + adc %%T2, rdx + + mul %%U0 ; {T3:T2:T1} += a[mSize-1]*U0 + r[mSize-2] + add %%T0, qword [%%rDst+sizeof(qword)*2] + adc %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[mSize-1] + adc %%T2, rdx + mov qword [%%rDst+sizeof(qword)*2], %%T0 + adc %%T3, 0 + + mul %%U1 ; {T3:T2} += a[mSize-1]*B1 + r[mSize-1] + mov %%idx, qword [rsp+counter] + add %%T1, qword [%%rDst+sizeof(qword)*3] + adc %%T2, rax + mov qword [%%rDst+sizeof(qword)*3], %%T1 + adc %%T3, rdx + xor rax, rax + + add %%T2, qword [%%rDst+sizeof(qword)*4] + adc %%T3, qword [%%rDst+sizeof(qword)*5] + adc rax, 0 + add %%T2, qword [rsp+carry] + adc %%T3, 0 + adc rax, 0 + mov qword [%%rDst+sizeof(qword)*4], %%T2 + mov qword [%%rDst+sizeof(qword)*5], %%T3 + mov qword [rsp+carry], rax + + add %%rDst, sizeof(qword)*2 +%endmacro + +%macro MM_MLAx2_4N_3_ELOG 9.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%U0 %4 + %xdefine %%U1 %5 + %xdefine %%T0 %6 + %xdefine %%T1 %7 + %xdefine %%T2 %8 + %xdefine %%T3 %9 + + mul %%U1 ; {T2:T1} += a[mSize-3]*U1 + xor %%T3, %%T3 + add %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a[mSize-2] + adc %%T2, rdx + + mul %%U0 ; {T3:T2:T1} += a[mSize-2]*U0 + r[mSize-3] + add %%T0, qword [%%rDst+sizeof(qword)] + adc %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a[mSize-2] + adc %%T2, rdx + mov qword [%%rDst+sizeof(qword)], %%T0 + adc %%T3, 0 + + mul %%U1 ; {T3:T2} += a[mSize-2]*U1 + xor %%T0, %%T0 + add %%T2, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[mSize-1] + adc %%T3, rdx + + mul %%U0 ; {T0:T3:T2} += a[mSize-1]*U0 + r[mSize-2] + add %%T1, qword [rdi+sizeof(qword)*2] + adc %%T2, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[mSize-1] + adc %%T3, rdx + mov qword [rdi+sizeof(qword)*2], %%T1 + adc %%T0, 0 + + mul %%U1 ; {T0:T3} += a[mSize-1]*U1 + r[mSize-1] + mov %%idx, qword [rsp+counter] + add %%T2, qword [rdi+sizeof(qword)*3] + adc %%T3, rax + mov qword [rdi+sizeof(qword)*3], %%T2 + adc %%T0, rdx + xor rax, rax + + add %%T3, qword [rdi+sizeof(qword)*4] + adc %%T0, qword [rdi+sizeof(qword)*5] + adc rax, 0 + add %%T3, qword [rsp+carry] + adc %%T0, 0 + adc rax, 0 + mov qword [rdi+sizeof(qword)*4], %%T3 + mov qword [rdi+sizeof(qword)*5], %%T0 + mov qword [rsp+carry], rax + + add %%rDst, sizeof(qword)*2 +%endmacro + +%macro MM_MLAx2_4N_4_ELOG 9.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%U0 %4 + %xdefine %%U1 %5 + %xdefine %%T0 %6 + %xdefine %%T1 %7 + %xdefine %%T2 %8 + %xdefine %%T3 %9 + + mul %%U1 ; {T2:T1} += a[mSize-4]*U1 + xor %%T3, %%T3 + add %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)] ; a[lenA-3] + adc %%T2, rdx + + mul %%U0 ; {T3:T2:T1} += a[mSize-3]*U0 + r[mSize-4] + add %%T0, qword [%%rDst] + adc %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)] ; a[mSize-3] + adc %%T2, rdx + mov qword [%%rDst], %%T0 + adc %%T3, 0 + + mul %%U1 ; {T3:T2} += a[mSize-3]*U1 + xor %%T0, %%T0 + add %%T2, rax + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a[mSize-2] + adc %%T3, rdx + + mul %%U0 ; {T0:T3:T2} += a[mSize-2]*U0 + r[mSize-3] + add %%T1, qword [%%rDst+sizeof(qword)] + adc %%T2, rax + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a{lenA-2] + adc %%T3, rdx + mov qword [%%rDst+sizeof(qword)], %%T1 + adc %%T0, 0 + + mul %%U1 ; {T0:T3} += a[mSize-2]*U1 + xor %%T1, %%T1 + add %%T3, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[mSize-1] + adc %%T0, rdx + + mul %%U0 ; {T1:T0:T3} += a[mSize-1]*U0 + r[mSize-2] + add %%T2, qword [%%rDst+sizeof(qword)*2] + adc %%T3, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[mSize-1] + adc %%T0, rdx + mov qword [%%rDst+sizeof(qword)*2], %%T2 + adc %%T1, 0 + + mul %%U1 ; {T1:T0} += a[mSize-1]*U1 + r[mSize-1] + mov %%idx, qword [rsp+counter] + add %%T3, qword [%%rDst+sizeof(qword)*3] + adc %%T0, rax + mov qword [%%rDst+sizeof(qword)*3], %%T3 + adc %%T1, rdx + xor rax, rax + + add %%T0, qword [%%rDst+sizeof(qword)*4] + adc %%T1, qword [%%rDst+sizeof(qword)*5] + adc rax, 0 + add %%T0, qword [rsp+carry] + adc %%T1, 0 + adc rax, 0 + mov qword [%%rDst+sizeof(qword)*4], %%T0 + mov qword [%%rDst+sizeof(qword)*5], %%T1 + mov qword [rsp+carry], rax + + add %%rDst, sizeof(qword)*2 +%endmacro + +%endif + +%if (_IPP32E >= _IPP32E_E9) +;; +;; Sandy Brige specific code +;; +%macro MM_MLAx2_4N_1_ELOG 9.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%U0 %4 + %xdefine %%U1 %5 + %xdefine %%T0 %6 + %xdefine %%T1 %7 + %xdefine %%T2 %8 + %xdefine %%T3 %9 + + mul %%U1 ; {T2:T1} += modulus[mSize-1]*U1 + r[mSize-1] + mov %%idx, qword [rsp+counter] + add %%T0, qword [%%rDst+sizeof(qword)*3] + mov qword [%%rDst+sizeof(qword)*3], %%T0 + adc %%T1, rax + adc %%T2, rdx + xor rax, rax + + add %%T1, qword [%%rDst+sizeof(qword)*4] + adc %%T2, qword [%%rDst+sizeof(qword)*5] + adc rax, 0 + add %%T1, qword [rsp+carry] + adc %%T2, 0 + adc rax, 0 + mov qword [%%rDst+sizeof(qword)*4], %%T1 + mov qword [%%rDst+sizeof(qword)*5], %%T2 + mov qword [rsp+carry], rax + + add %%rDst, sizeof(qword)*2 +%endmacro + +%macro MM_MLAx2_4N_2_ELOG 9.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%U0 %4 + %xdefine %%U1 %5 + %xdefine %%T0 %6 + %xdefine %%T1 %7 + %xdefine %%T2 %8 + %xdefine %%T3 %9 + + mul %%U1 ; {T2:T1} += a[mSize-2]*U1 + xor %%T3, %%T3 + add %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[mSize-1] + adc %%T2, rdx + + mul %%U0 ; {T3:T2:T1} += a[mSize-1]*U0 + r[mSize-2] + add %%T0, qword [%%rDst+sizeof(qword)*2] + mov qword [%%rDst+sizeof(qword)*2], %%T0 + adc %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[mSize-1] + adc %%T2, rdx + adc %%T3, 0 + + mul %%U1 ; {T3:T2} += a[mSize-1]*B1 + r[mSize-1] + mov %%idx, qword [rsp+counter] + add %%T1, qword [%%rDst+sizeof(qword)*3] + mov qword [%%rDst+sizeof(qword)*3], %%T1 + adc %%T2, rax + adc %%T3, rdx + xor rax, rax + + add %%T2, qword [%%rDst+sizeof(qword)*4] + adc %%T3, qword [%%rDst+sizeof(qword)*5] + adc rax, 0 + add %%T2, qword [rsp+carry] + adc %%T3, 0 + adc rax, 0 + mov qword [%%rDst+sizeof(qword)*4], %%T2 + mov qword [%%rDst+sizeof(qword)*5], %%T3 + mov qword [rsp+carry], rax + + add %%rDst, sizeof(qword)*2 +%endmacro + +%macro MM_MLAx2_4N_3_ELOG 9.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%U0 %4 + %xdefine %%U1 %5 + %xdefine %%T0 %6 + %xdefine %%T1 %7 + %xdefine %%T2 %8 + %xdefine %%T3 %9 + + mul %%U1 ; {T2:T1} += a[mSize-3]*U1 + xor %%T3, %%T3 + add %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a[mSize-2] + adc %%T2, rdx + + mul %%U0 ; {T3:T2:T1} += a[mSize-2]*U0 + r[mSize-3] + add %%T0, qword [%%rDst+sizeof(qword)] + mov qword [%%rDst+sizeof(qword)], %%T0 + adc %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a[mSize-2] + adc %%T2, rdx + adc %%T3, 0 + + mul %%U1 ; {T3:T2} += a[mSize-2]*U1 + xor %%T0, %%T0 + add %%T2, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[mSize-1] + adc %%T3, rdx + + mul %%U0 ; {T0:T3:T2} += a[mSize-1]*U0 + r[mSize-2] + add %%T1, qword [rdi+sizeof(qword)*2] + mov qword [rdi+sizeof(qword)*2], %%T1 + adc %%T2, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[mSize-1] + adc %%T3, rdx + adc %%T0, 0 + + mul %%U1 ; {T0:T3} += a[mSize-1]*U1 + r[mSize-1] + mov %%idx, qword [rsp+counter] + add %%T2, qword [rdi+sizeof(qword)*3] + mov qword [rdi+sizeof(qword)*3], %%T2 + adc %%T3, rax + adc %%T0, rdx + xor rax, rax + + add %%T3, qword [rdi+sizeof(qword)*4] + adc %%T0, qword [rdi+sizeof(qword)*5] + adc rax, 0 + add %%T3, qword [rsp+carry] + adc %%T0, 0 + adc rax, 0 + mov qword [rdi+sizeof(qword)*4], %%T3 + mov qword [rdi+sizeof(qword)*5], %%T0 + mov qword [rsp+carry], rax + + add %%rDst, sizeof(qword)*2 +%endmacro + +%macro MM_MLAx2_4N_4_ELOG 9.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc %2 + %xdefine %%idx %3 + %xdefine %%U0 %4 + %xdefine %%U1 %5 + %xdefine %%T0 %6 + %xdefine %%T1 %7 + %xdefine %%T2 %8 + %xdefine %%T3 %9 + + mul %%U1 ; {T2:T1} += a[mSize-4]*U1 + xor %%T3, %%T3 + add %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)] ; a[lenA-3] + adc %%T2, rdx + + mul %%U0 ; {T3:T2:T1} += a[mSize-3]*U0 + r[mSize-4] + add %%T0, qword [%%rDst] + mov qword [%%rDst], %%T0 + adc %%T1, rax + mov rax, qword [%%rSrc+sizeof(qword)] ; a[mSize-3] + adc %%T2, rdx + adc %%T3, 0 + + mul %%U1 ; {T3:T2} += a[mSize-3]*U1 + xor %%T0, %%T0 + add %%T2, rax + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a[mSize-2] + adc %%T3, rdx + + mul %%U0 ; {T0:T3:T2} += a[mSize-2]*U0 + r[mSize-3] + add %%T1, qword [%%rDst+sizeof(qword)] + mov qword [%%rDst+sizeof(qword)], %%T1 + adc %%T2, rax + mov rax, qword [%%rSrc+sizeof(qword)*2] ; a{lenA-2] + adc %%T3, rdx + adc %%T0, 0 + + mul %%U1 ; {T0:T3} += a[mSize-2]*U1 + xor %%T1, %%T1 + add %%T3, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[mSize-1] + adc %%T0, rdx + + mul %%U0 ; {T1:T0:T3} += a[mSize-1]*U0 + r[mSize-2] + add %%T2, qword [%%rDst+sizeof(qword)*2] + mov qword [%%rDst+sizeof(qword)*2], %%T2 + adc %%T3, rax + mov rax, qword [%%rSrc+sizeof(qword)*3] ; a[mSize-1] + adc %%T0, rdx + adc %%T1, 0 + + mul %%U1 ; {T1:T0} += a[mSize-1]*U1 + r[mSize-1] + mov %%idx, qword [rsp+counter] + add %%T3, qword [%%rDst+sizeof(qword)*3] + mov qword [%%rDst+sizeof(qword)*3], %%T3 + adc %%T0, rax + adc %%T1, rdx + xor rax, rax + + add %%T0, qword [%%rDst+sizeof(qword)*4] + adc %%T1, qword [%%rDst+sizeof(qword)*5] + adc rax, 0 + add %%T0, qword [rsp+carry] + adc %%T1, 0 + adc rax, 0 + mov qword [%%rDst+sizeof(qword)*4], %%T0 + mov qword [%%rDst+sizeof(qword)*5], %%T1 + mov qword [rsp+carry], rax + + add %%rDst, sizeof(qword)*2 +%endmacro + +%endif + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro SBB_x4 9.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc1 %2 + %xdefine %%rSrc2 %3 + %xdefine %%idx %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + %xdefine %%rcf %9 + + ALIGN IPP_ALIGN_FACTOR +%%L_1: + add %%rcf, %%rcf ; restore cf + + mov %%T0, qword [%%rSrc1+%%idx*sizeof(qword)] + sbb %%T0, qword [%%rSrc2+%%idx*sizeof(qword)] + mov qword [%%rDst+%%idx*sizeof(qword)], %%T0 + + mov %%T1, qword [%%rSrc1+%%idx*sizeof(qword)+sizeof(qword)] + sbb %%T1, qword [%%rSrc2+%%idx*sizeof(qword)+sizeof(qword)] + mov qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)], %%T1 + + mov %%T2, qword [%%rSrc1+%%idx*sizeof(qword)+sizeof(qword)*2] + sbb %%T2, qword [%%rSrc2+%%idx*sizeof(qword)+sizeof(qword)*2] + mov qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)*2], %%T2 + + mov %%T3, qword [%%rSrc1+%%idx*sizeof(qword)+sizeof(qword)*3] + sbb %%T3, qword [%%rSrc2+%%idx*sizeof(qword)+sizeof(qword)*3] + mov qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)*3], %%T3 + + sbb %%rcf, %%rcf ; save cf + add %%idx, 4 + jnc %%L_1 +%endmacro + +%macro SBB_TAIL 9.nolist + %xdefine %%N %1 + %xdefine %%rDst %2 + %xdefine %%rSrc1 %3 + %xdefine %%rSrc2 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + %xdefine %%rcf %9 + + add %%rcf, %%rcf ; restore cf + mov idx, qword [rsp+counter] ; restore counter + +%if %%N > 3 + mov %%T0, qword [%%rSrc1] + sbb %%T0, qword [%%rSrc2] + mov qword [%%rDst], %%T0 +%endif + +%if %%N > 2 + mov %%T1, qword [%%rSrc1+sizeof(qword)] + sbb %%T1, qword [%%rSrc2+sizeof(qword)] + mov qword [%%rDst+sizeof(qword)], %%T1 +%endif + +%if %%N > 1 + mov %%T2, qword [%%rSrc1+sizeof(qword)*2] + sbb %%T2, qword [%%rSrc2+sizeof(qword)*2] + mov qword [%%rDst+sizeof(qword)*2], %%T2 +%endif + +%if %%N > 0 + mov %%T3, qword [%%rSrc1+sizeof(qword)*3] + sbb %%T3, qword [%%rSrc2+sizeof(qword)*3] + mov qword [%%rDst+sizeof(qword)*3], %%T3 +%endif + + sbb rax, 0 + sbb %%rcf, %%rcf ; set cf +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; dst[] = cf? src1[] : src2[] +%macro CMOV_x4 9.nolist + %xdefine %%rDst %1 + %xdefine %%rSrc1 %2 + %xdefine %%rSrc2 %3 + %xdefine %%idx %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + %xdefine %%rcf %9 + + mov %%T3, %%rcf ; copy cf + ALIGN IPP_ALIGN_FACTOR +%%L_1: + add %%T3, %%T3 ; restore cf + + mov %%T0, qword [%%rSrc2+%%idx*sizeof(qword)] + mov %%T1, qword [%%rSrc1+%%idx*sizeof(qword)] + mov %%T2, qword [%%rSrc2+%%idx*sizeof(qword)+sizeof(qword)] + mov %%T3, qword [%%rSrc1+%%idx*sizeof(qword)+sizeof(qword)] + cmovc %%T0, %%T1 + mov qword [%%rDst+%%idx*sizeof(qword)], %%T0 + cmovc %%T2, %%T3 + mov qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)], %%T2 + + mov %%T0, qword [%%rSrc2+%%idx*sizeof(qword)+sizeof(qword)*2] + mov %%T1, qword [%%rSrc1+%%idx*sizeof(qword)+sizeof(qword)*2] + mov %%T2, qword [%%rSrc2+%%idx*sizeof(qword)+sizeof(qword)*3] + mov %%T3, qword [%%rSrc1+%%idx*sizeof(qword)+sizeof(qword)*3] + cmovc %%T0, %%T1 + mov qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)*2], %%T0 + cmovc %%T2, %%T3 + mov qword [%%rDst+%%idx*sizeof(qword)+sizeof(qword)*3], %%T2 + + mov %%T3, %%rcf ; copy cf + add %%idx, 4 + jnc %%L_1 +%endmacro + +%macro CMOV_TAIL 9.nolist + %xdefine %%N %1 + %xdefine %%rDst %2 + %xdefine %%rSrc1 %3 + %xdefine %%rSrc2 %4 + %xdefine %%T0 %5 + %xdefine %%T1 %6 + %xdefine %%T2 %7 + %xdefine %%T3 %8 + %xdefine %%rcf %9 + + add %%rcf, %%rcf ; restore cf + mov idx, qword [rsp+counter] ; restore counter + +%if %%N > 3 + mov %%T0, qword [%%rSrc2] + mov %%T1, qword [%%rSrc1] + cmovc %%T0, %%T1 + mov qword [%%rDst], %%T0 +%endif + +%if %%N > 2 + mov %%T2, qword [%%rSrc2+sizeof(qword)] + mov %%T3, qword [%%rSrc1+sizeof(qword)] + cmovc %%T2, %%T3 + mov qword [%%rDst+sizeof(qword)], %%T2 +%endif + +%if %%N > 1 + mov %%T0, qword [%%rSrc2+sizeof(qword)*2] + mov %%T1, qword [%%rSrc1+sizeof(qword)*2] + cmovc %%T0, %%T1 + mov qword [%%rDst+sizeof(qword)*2], %%T0 +%endif + +%if %%N > 0 + mov %%T2, qword [%%rSrc2+sizeof(qword)*3] + mov %%T3, qword [%%rSrc1+sizeof(qword)*3] + cmovc %%T2, %%T3 + mov qword [%%rDst+sizeof(qword)*3], %%T2 +%endif +%endmacro + +segment .text align=IPP_ALIGN_FACTOR + + +;************************************************************* +;* void cpMontRedAdc_BNU(Ipp64u* pR, +;* Ipp64u* pBuffer, +;* const Ipp64u* pModulo, int mSize, Ipp64u m0) +;* +;************************************************************* + +;; +;; Lib = M7 +;; +;; Caller = ippsMontMul +;; +align IPP_ALIGN_FACTOR +IPPASM cpMontRedAdc_BNU,PUBLIC +%assign LOCAL_FRAME (1+1+1+1)*sizeof(qword) + USES_GPR rbx,rbp,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 5 + +;pR (rdi) address of the reduction +;pBuffer (rsi) address of the temporary product +;pModulo (rdx) address of the modulus +;mSize (rcx) size of the modulus +;m0 (r8) helper + +;; +;; stack structure: +%assign pR (0) ; reduction address +%assign mSize (pR+sizeof(qword)) ; modulus length (qwords) +%assign carry (mSize+sizeof(qword)) ; carry from previous MLAx1 or MLAx2 operation +%assign counter (carry+sizeof(qword)) ; counter = 4-mSize + + mov qword [rsp+carry], 0 ; clear carry + movsxd rcx, ecx ; expand modulus length + mov r15, r8 ; helper m0 + + cmp rcx, 5 + jge .general_case + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; reducer of the fixed sizes +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + cmp rcx, 3 + ja .mSize_4 ; rcx=4 + jz .mSize_3 ; rcx=3 + jp .mSize_2 ; rcx=2 + ; mSize_1 ; rcx=1 + +%xdefine X0 r8 +%xdefine X1 r9 +%xdefine X2 r10 +%xdefine X3 r11 +%xdefine X4 r12 +%xdefine X5 r13 +%xdefine X6 r14 +%xdefine X7 rcx + +align IPP_ALIGN_FACTOR +.mSize_1: + mov X0, qword [rsi] + mov X1, qword [rsi+sizeof(qword)] + mov rsi, rdx + ; rDst,rSrc, U, M0, T0 + MREDUCE_FIX 1, rdi, rsi, r15, X7, X6, X5, X4, X3, X2, X1,X0, rbp,rbx + jmp .quit + +align IPP_ALIGN_FACTOR +.mSize_2: + mov X0, qword [rsi] + mov X1, qword [rsi+sizeof(qword)] + mov X2, qword [rsi+sizeof(qword)*2] + mov X3, qword [rsi+sizeof(qword)*3] + mov rsi, rdx + MREDUCE_FIX 2, rdi, rsi, r15, X7, X6, X5, X4, X3, X2, X1,X0, rbp,rbx + jmp .quit + +align IPP_ALIGN_FACTOR +.mSize_3: + mov X0, qword [rsi] + mov X1, qword [rsi+sizeof(qword)] + mov X2, qword [rsi+sizeof(qword)*2] + mov X3, qword [rsi+sizeof(qword)*3] + mov X4, qword [rsi+sizeof(qword)*4] + mov X5, qword [rsi+sizeof(qword)*5] + mov rsi, rdx + MREDUCE_FIX 3, rdi, rsi, r15, X7, X6, X5, X4, X3, X2, X1,X0, rbp,rbx + jmp .quit + +align IPP_ALIGN_FACTOR +.mSize_4: + mov X0, qword [rsi] + mov X1, qword [rsi+sizeof(qword)] + mov X2, qword [rsi+sizeof(qword)*2] + mov X3, qword [rsi+sizeof(qword)*3] + mov X4, qword [rsi+sizeof(qword)*4] + mov X5, qword [rsi+sizeof(qword)*5] + mov X6, qword [rsi+sizeof(qword)*6] + mov X7, qword [rsi+sizeof(qword)*7] + mov rsi, rdx + MREDUCE_FIX 4, rdi, rsi, r15, X7, X6, X5, X4, X3, X2, X1,X0, rbp,rbx + jmp .quit + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; general case reducer +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%xdefine U0 r9 ; u0, u1 +%xdefine U1 r10 + +%xdefine T0 r11 ; temporary +%xdefine T1 r12 +%xdefine T2 r13 +%xdefine T3 r14 + +%xdefine idx rbx ; index +%xdefine rDst rdi +%xdefine rSrc rsi + +align IPP_ALIGN_FACTOR +.general_case: + lea rdi, [rdi+rcx*sizeof(qword)-sizeof(qword)*4] ; save pR+mSzie-4 + mov qword [rsp+pR], rdi ; save pR + ;mov qword [rsp+mSize], rcx ; save length of modulus + + mov rdi, rsi ; rdi = pBuffer + mov rsi, rdx ; rsi = pModulo + + lea rdi, [rdi+rcx*sizeof(qword)-sizeof(qword)*4] ; rdi = pBuffer + lea rsi, [rsi+rcx*sizeof(qword)-sizeof(qword)*4] ; rsi = pModulus + + mov idx, dword 4 ; init negative counter + sub idx, rcx + mov qword [rsp+counter], idx + mov rdx, dword 3 + and rdx, rcx + + test rcx,1 + jz .even_len_Modulus + +; +; modulus of odd length +; +.odd_len_Modulus: + mov U0, qword [rDst+idx*sizeof(qword)] ; pBuffer[0] + imul U0, r15 ; u0 = pBuffer[0]*m0 + + mov rax, qword [rSrc+idx*sizeof(qword)] ; pModulo[0] + + mul U0 ; prologue + mov T0, rax + mov rax, qword [rSrc+idx*sizeof(qword)+sizeof(qword)] + mov T1, rdx + + add idx, 1 + jz .skip_mlax1 + + MLAx1 rdi, rsi, idx, U0, T0,T1,T2,T3 ; pBuffer[] += pModulo[]*u + +.skip_mlax1: + mul U0 + xor T2, T2 + add qword [rDst+idx*sizeof(qword)-sizeof(qword)], T0 + adc T1, rax + adc T2, rdx + + cmp idx, 2 + ja .fin_mla1x4n_2 ; idx=3 + jz .fin_mla1x4n_3 ; idx=2 + jp .fin_mla1x4n_4 ; idx=1 + ; fin_mla1x4n_1 ; idx=0 + +.fin_mla1x4n_1: + MM_MLAx1_4N_1_ELOG rdi, rsi, idx, U0, T0,T1,T2,T3 ; [rsp+carry] = rax = cf = pBuffer[]+pModulo[]*u + sub rcx, 1 + jmp .mla2x4n_1 + +.fin_mla1x4n_4: + MM_MLAx1_4N_4_ELOG rdi, rsi, idx, U0, T0,T1,T2,T3 ; [rsp+carry] = rax = cf = pBuffer[]+pModulo[]*u + sub rcx, 1 + jmp .mla2x4n_4 + +.fin_mla1x4n_3: + MM_MLAx1_4N_3_ELOG rdi, rsi, idx, U0, T0,T1,T2,T3 ; [rsp+carry] = rax = cf = pBuffer[]+pModulo[]*u + sub rcx, 1 + jmp .mla2x4n_3 + +.fin_mla1x4n_2: + MM_MLAx1_4N_2_ELOG rdi, rsi, idx, U0, T0,T1,T2,T3 ; [rsp+carry] = rax = cf = pBuffer[]+pModulo[]*u + sub rcx, 1 + jmp .mla2x4n_2 + +align IPP_ALIGN_FACTOR +; +; modulus of even length +; +.even_len_Modulus: + xor rax, rax ; clear carry + cmp rdx, 2 + ja .mla2x4n_1 ; rdx=1 + jz .mla2x4n_2 ; rdx=2 + jp .mla2x4n_3 ; rdx=3 + ; mla2x4n_4 ; rdx=0 + +align IPP_ALIGN_FACTOR +.mla2x4n_4: + MMx2_PLOG rdi, rsi, idx, r15, U0,U1,T0,T1,T2,T3 ; pre-compute u0 and u1 + MLAx2 rdi, rsi, idx, U0,U1, T0,T1,T2,T3 ; rax = cf = product[]+modulo[]*{u1:u0} + MM_MLAx2_4N_4_ELOG rdi, rsi, idx, U0,U1, T0,T1,T2,T3 ; [rsp+carry] = rax = cf + sub rcx, 2 + jnz .mla2x4n_4 + + ; (borrow, pR) = (carry,product) - modulo + mov rdx, qword [rsp+pR] + xor rcx, rcx + SBB_x4 rdx, rdi, rsi, idx, T0,T1,T2,T3, rcx + SBB_TAIL 4, rdx, rdi, rsi, T0,T1,T2,T3, rcx + + ; pR = borrow? product : pR + CMOV_x4 rdx, rdi, rdx, idx, T0,T1,T2,T3, rcx + CMOV_TAIL 4, rdx, rdi, rdx, T0,T1,T2,T3, rcx + jmp .quit + +align IPP_ALIGN_FACTOR +.mla2x4n_3: + MMx2_PLOG rdi, rsi, idx, r15, U0,U1,T0,T1,T2,T3 ; pre-compute u0 and u1 + MLAx2 rdi, rsi, idx, U0,U1, T0,T1,T2,T3 ; rax = cf = product[]+modulo[]*{u1:u0} + MM_MLAx2_4N_3_ELOG rdi, rsi, idx, U0,U1, T0,T1,T2,T3 ; [rsp+carry] = rax = cf + sub rcx, 2 + jnz .mla2x4n_3 + + ; (borrow, pR) = (carry,product) - modulo + mov rdx, qword [rsp+pR] + xor rcx, rcx + SBB_x4 rdx, rdi, rsi, idx, T0,T1,T2,T3, rcx + SBB_TAIL 3, rdx, rdi, rsi, T0,T1,T2,T3, rcx + + ; pR = borrow? product : pR + CMOV_x4 rdx, rdi, rdx, idx, T0,T1,T2,T3, rcx + CMOV_TAIL 3, rdx, rdi, rdx, T0,T1,T2,T3, rcx + jmp .quit + +align IPP_ALIGN_FACTOR +.mla2x4n_2: + MMx2_PLOG rdi, rsi, idx, r15, U0,U1,T0,T1,T2,T3 ; pre-compute u0 and u1 + MLAx2 rdi, rsi, idx, U0,U1, T0,T1,T2,T3 ; rax = cf = product[]+modulo[]*{u1:u0} + MM_MLAx2_4N_2_ELOG rdi, rsi, idx, U0,U1, T0,T1,T2,T3 ; [rsp+carry] = rax = cf + sub rcx, 2 + jnz .mla2x4n_2 + + ; (borrow, pR) = (carry,product) - modulo + mov rdx, qword [rsp+pR] + xor rcx, rcx + SBB_x4 rdx, rdi, rsi, idx, T0,T1,T2,T3, rcx + SBB_TAIL 2, rdx, rdi, rsi, T0,T1,T2,T3, rcx + + ; pR = borrow? product : pR + CMOV_x4 rdx, rdi, rdx, idx, T0,T1,T2,T3, rcx + CMOV_TAIL 2, rdx, rdi, rdx, T0,T1,T2,T3, rcx + jmp .quit + + +align IPP_ALIGN_FACTOR +.mla2x4n_1: + MMx2_PLOG rdi, rsi, idx, r15, U0,U1,T0,T1,T2,T3 ; pre-compute u0 and u1 + MLAx2 rdi, rsi, idx, U0,U1, T0,T1,T2,T3 ; rax = cf = product[]+modulo[]*{u1:u0} + MM_MLAx2_4N_1_ELOG rdi, rsi, idx, U0,U1, T0,T1,T2,T3 ; [rsp+carry] = rax = cf + sub rcx, 2 + jnz .mla2x4n_1 + + ; (borrow, pR) = (carry,product) - modulo + mov rdx, qword [rsp+pR] + xor rcx, rcx + SBB_x4 rdx, rdi, rsi, idx, T0,T1,T2,T3, rcx + SBB_TAIL 1, rdx, rdi, rsi, T0,T1,T2,T3, rcx + + ; pR = borrow? product : pR + CMOV_x4 rdx, rdi, rdx, idx, T0,T1,T2,T3, rcx + CMOV_TAIL 1, rdx, rdi, rdx, T0,T1,T2,T3, rcx + +.quit: + REST_XMM + REST_GPR + ret +ENDFUNC cpMontRedAdc_BNU + +%endif + +%endif ;; _ADCOX_NI_ENABLING_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmontsqr1024_avx2as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmontsqr1024_avx2as.asm new file mode 100644 index 0000000..cf5f2f6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmontsqr1024_avx2as.asm @@ -0,0 +1,895 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Big Number Operations +; +; Content: +; cpMontSqr1024_avx2() +; + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_L9) + +segment .text align=IPP_ALIGN_FACTOR + + +%assign DIGIT_BITS 27 +%assign DIGIT_MASK (1 << DIGIT_BITS) -1 + +;************************************************************* +;* void cpMontSqr1024_avx2(Ipp64u* pR, +;* const Ipp64u* pA, +;* const Ipp64u* pModulus, int mSize, +;* Ipp64u k0, +;* Ipp64u* pBuffer) +;************************************************************* + +align IPP_ALIGN_FACTOR +IPPASM cpMontSqr1024_avx2,PUBLIC +%assign LOCAL_FRAME sizeof(qword)*7 + USES_GPR rsi,rdi,rbx,rbp,r12,r13,r14 + USES_XMM_AVX ymm6,ymm7,ymm8,ymm9,ymm10,ymm11,ymm12,ymm13,ymm14 + COMP_ABI 6 + + movsxd rcx, ecx ; redLen value counter + vpxor ymm11, ymm11, ymm11 + +;; expands A and M operands + vmovdqu ymmword [rsi+rcx*sizeof(qword)], ymm11 + vmovdqu ymmword [rdx+rcx*sizeof(qword)], ymm11 + +; +; stack struct +; +%assign pResult 0 ; pointer to result +%assign pA pResult+sizeof(qword) ; pointer to A operand +%assign pM pA+sizeof(qword) ; pointer to modulus +%assign redLen pM+sizeof(qword) ; length +%assign m0 redLen+sizeof(qword) ; m0 value +%assign pA2 m0+sizeof(qword) ; pointer to buffer (contains doubled input (A*2) and temporary result (A^2) ) +%assign pAxA pA2+sizeof(qword) ; pointer to temporary result (A^2) + + mov qword [rsp+pResult], rdi ; store pointer to result + mov qword [rsp+pA], rsi ; store pointer to input A + mov qword [rsp+pM], rdx ; store pointer to modulus + mov qword [rsp+redLen], rcx ; store redLen + mov qword [rsp+m0], r8 ; store m0 value + + mov rcx, dword 40 + + mov rdi, r9 + mov qword [rsp+pAxA], rdi ; pointer to temporary result (low A^2) + lea rbx, [rdi+rcx*sizeof(qword)] ; pointer to temporary result (high A^2) + lea r9, [rbx+rcx*sizeof(qword)] ; pointer to doubled input (A*2) + mov qword [rsp+pA2], r9 + + mov rax, rsi + mov rcx, dword sizeof(ymmword)/sizeof(qword) + +;; doubling input + vmovdqu ymm0, ymmword [rsi] + vmovdqu ymm1, ymmword [rsi+sizeof(ymmword)] + vmovdqu ymm2, ymmword [rsi+sizeof(ymmword)*2] + vmovdqu ymm3, ymmword [rsi+sizeof(ymmword)*3] + vmovdqu ymm4, ymmword [rsi+sizeof(ymmword)*4] + vmovdqu ymm5, ymmword [rsi+sizeof(ymmword)*5] + vmovdqu ymm6, ymmword [rsi+sizeof(ymmword)*6] + vmovdqu ymm7, ymmword [rsi+sizeof(ymmword)*7] + vmovdqu ymm8, ymmword [rsi+sizeof(ymmword)*8] + vmovdqu ymm9, ymmword [rsi+sizeof(ymmword)*9] + + vmovdqu ymmword [r9], ymm0 + vpbroadcastq ymm10, qword [rax] ; ymm10 = {a0:a0:a0:a0} + vpaddq ymm1, ymm1, ymm1 + vmovdqu ymmword [r9+sizeof(ymmword)], ymm1 + vpaddq ymm2, ymm2, ymm2 + vmovdqu ymmword [r9+sizeof(ymmword)*2], ymm2 + vpaddq ymm3, ymm3, ymm3 + vmovdqu ymmword [r9+sizeof(ymmword)*3], ymm3 + vpaddq ymm4, ymm4, ymm4 + vmovdqu ymmword [r9+sizeof(ymmword)*4], ymm4 + vpaddq ymm5, ymm5, ymm5 + vmovdqu ymmword [r9+sizeof(ymmword)*5], ymm5 + vpaddq ymm6, ymm6, ymm6 + vmovdqu ymmword [r9+sizeof(ymmword)*6], ymm6 + vpaddq ymm7, ymm7, ymm7 + vmovdqu ymmword [r9+sizeof(ymmword)*7], ymm7 + vpaddq ymm8, ymm8, ymm8 + vmovdqu ymmword [r9+sizeof(ymmword)*8], ymm8 + vpaddq ymm9, ymm9, ymm9 + vmovdqu ymmword [r9+sizeof(ymmword)*9], ymm9 + +;; +;; squaring +;; + vpmuludq ymm0, ymm10, ymmword [rsi] + vpbroadcastq ymm14, qword [rax+sizeof(qword)*4] ; ymm14 = {a4:a4:a4:a4} + vmovdqu ymmword [rbx], ymm11 + vpmuludq ymm1, ymm10, ymmword [r9+sizeof(ymmword)] + vmovdqu ymmword [rbx+sizeof(ymmword)], ymm11 + vpmuludq ymm2, ymm10, ymmword [r9+sizeof(ymmword)*2] + vmovdqu ymmword [rbx+sizeof(ymmword)*2], ymm11 + vpmuludq ymm3, ymm10, ymmword [r9+sizeof(ymmword)*3] + vmovdqu ymmword [rbx+sizeof(ymmword)*3], ymm11 + vpmuludq ymm4, ymm10, ymmword [r9+sizeof(ymmword)*4] + vmovdqu ymmword [rbx+sizeof(ymmword)*4], ymm11 + vpmuludq ymm5, ymm10, ymmword [r9+sizeof(ymmword)*5] + vmovdqu ymmword [rbx+sizeof(ymmword)*5], ymm11 + vpmuludq ymm6, ymm10, ymmword [r9+sizeof(ymmword)*6] + vmovdqu ymmword [rbx+sizeof(ymmword)*6], ymm11 + vpmuludq ymm7, ymm10, ymmword [r9+sizeof(ymmword)*7] + vmovdqu ymmword [rbx+sizeof(ymmword)*7], ymm11 + vpmuludq ymm8, ymm10, ymmword [r9+sizeof(ymmword)*8] + vmovdqu ymmword [rbx+sizeof(ymmword)*8], ymm11 + vpmuludq ymm9, ymm10, ymmword [r9+sizeof(ymmword)*9] + vmovdqu ymmword [rbx+sizeof(ymmword)*9], ymm11 + + jmp .sqr1024_ep + +align IPP_ALIGN_FACTOR +.sqr1024_loop4: + vpmuludq ymm0, ymm10, ymmword [rsi] + vpbroadcastq ymm14, qword [rax+sizeof(qword)*4] ; ymm14 = {a4:a4:a4:a4} + vpaddq ymm0, ymm0, ymmword [rdi] + + vpmuludq ymm1, ymm10, ymmword [r9+sizeof(ymmword)] + vpaddq ymm1, ymm1, ymmword [rdi+sizeof(ymmword)] + + vpmuludq ymm2, ymm10, ymmword [r9+sizeof(ymmword)*2] + vpaddq ymm2, ymm2, ymmword [rdi+sizeof(ymmword)*2] + + vpmuludq ymm3, ymm10, ymmword [r9+sizeof(ymmword)*3] + vpaddq ymm3, ymm3, ymmword [rdi+sizeof(ymmword)*3] + + vpmuludq ymm4, ymm10, ymmword [r9+sizeof(ymmword)*4] + vpaddq ymm4, ymm4, ymmword [rdi+sizeof(ymmword)*4] + + vpmuludq ymm5, ymm10, ymmword [r9+sizeof(ymmword)*5] + vpaddq ymm5, ymm5, ymmword [rdi+sizeof(ymmword)*5] + + vpmuludq ymm6, ymm10, ymmword [r9+sizeof(ymmword)*6] + vpaddq ymm6, ymm6, ymmword [rdi+sizeof(ymmword)*6] + + vpmuludq ymm7, ymm10, ymmword [r9+sizeof(ymmword)*7] + vpaddq ymm7, ymm7, ymmword [rdi+sizeof(ymmword)*7] + + vpmuludq ymm8, ymm10, ymmword [r9+sizeof(ymmword)*8] + vpaddq ymm8, ymm8, ymmword [rdi+sizeof(ymmword)*8] + + vpmuludq ymm9, ymm10, ymmword [r9+sizeof(ymmword)*9] + vpaddq ymm9, ymm9, ymmword [rdi+sizeof(ymmword)*9] + +.sqr1024_ep: + vmovdqu ymmword [rdi], ymm0 + vmovdqu ymmword [rdi+sizeof(ymmword)], ymm1 + + vpmuludq ymm11, ymm14, ymmword [rsi+sizeof(ymmword)] + vpbroadcastq ymm10, qword [rax+sizeof(qword)*8] ; ymm10 = {a8:a8:a8:a8} + vpaddq ymm2, ymm2, ymm11 + vpmuludq ymm12, ymm14, ymmword [r9+sizeof(ymmword)*2] + vpaddq ymm3, ymm3, ymm12 + vpmuludq ymm13, ymm14, ymmword [r9+sizeof(ymmword)*3] + vpaddq ymm4, ymm4, ymm13 + vpmuludq ymm11, ymm14, ymmword [r9+sizeof(ymmword)*4] + vpaddq ymm5, ymm5, ymm11 + vpmuludq ymm12, ymm14, ymmword [r9+sizeof(ymmword)*5] + vpaddq ymm6, ymm6, ymm12 + vpmuludq ymm13, ymm14, ymmword [r9+sizeof(ymmword)*6] + vpaddq ymm7, ymm7, ymm13 + vpmuludq ymm11, ymm14, ymmword [r9+sizeof(ymmword)*7] + vpaddq ymm8, ymm8, ymm11 + vpmuludq ymm12, ymm14, ymmword [r9+sizeof(ymmword)*8] + vpaddq ymm9, ymm9, ymm12 + vpmuludq ymm0, ymm14, ymmword [r9+sizeof(ymmword)*9] + vpaddq ymm0, ymm0, ymmword [rbx] + + vmovdqu ymmword [rdi+sizeof(ymmword)*2], ymm2 + vmovdqu ymmword [rdi+sizeof(ymmword)*3], ymm3 + + vpmuludq ymm11, ymm10, ymmword [rsi+sizeof(ymmword)*2] + vpbroadcastq ymm14, qword [rax+sizeof(qword)*12] ; ymm14 = {a12:a12:a12:a12} + vpaddq ymm4, ymm4, ymm11 + vpmuludq ymm12, ymm10, ymmword [r9+sizeof(ymmword)*3] + vpaddq ymm5, ymm5, ymm12 + vpmuludq ymm13, ymm10, ymmword [r9+sizeof(ymmword)*4] + vpaddq ymm6, ymm6, ymm13 + vpmuludq ymm11, ymm10, ymmword [r9+sizeof(ymmword)*5] + vpaddq ymm7, ymm7, ymm11 + vpmuludq ymm12, ymm10, ymmword [r9+sizeof(ymmword)*6] + vpaddq ymm8, ymm8, ymm12 + vpmuludq ymm13, ymm10, ymmword [r9+sizeof(ymmword)*7] + vpaddq ymm9, ymm9, ymm13 + vpmuludq ymm11, ymm10, ymmword [r9+sizeof(ymmword)*8] + vpaddq ymm0, ymm0, ymm11 + vpmuludq ymm1, ymm10, ymmword [r9+sizeof(ymmword)*9] + vpaddq ymm1, ymm1, ymmword [rbx+sizeof(ymmword)] + + vmovdqu ymmword [rdi+sizeof(ymmword)*4], ymm4 + vmovdqu ymmword [rdi+sizeof(ymmword)*5], ymm5 + + vpmuludq ymm11, ymm14, ymmword [rsi+sizeof(ymmword)*3] + vpbroadcastq ymm10, qword [rax+sizeof(qword)*16] ; ymm10 = {a16:a16:a16:a16} + vpaddq ymm6, ymm6, ymm11 + vpmuludq ymm12, ymm14, ymmword [r9+sizeof(ymmword)*4] + vpaddq ymm7, ymm7, ymm12 + vpmuludq ymm13, ymm14, ymmword [r9+sizeof(ymmword)*5] + vpaddq ymm8, ymm8, ymm13 + vpmuludq ymm11, ymm14, ymmword [r9+sizeof(ymmword)*6] + vpaddq ymm9, ymm9, ymm11 + vpmuludq ymm12, ymm14, ymmword [r9+sizeof(ymmword)*7] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm14, ymmword [r9+sizeof(ymmword)*8] + vpaddq ymm1, ymm1, ymm13 + vpmuludq ymm2, ymm14, ymmword [r9+sizeof(ymmword)*9] + vpaddq ymm2, ymm2, ymmword [rbx+sizeof(ymmword)*2] + + vmovdqu ymmword [rdi+sizeof(ymmword)*6], ymm6 + vmovdqu ymmword [rdi+sizeof(ymmword)*7], ymm7 + + vpmuludq ymm11, ymm10, ymmword [rsi+sizeof(ymmword)*4] + vpbroadcastq ymm14, qword [rax+sizeof(qword)*20] ; ymm14 = {a20:a20:a20:a20} + vpaddq ymm8, ymm8, ymm11 + vpmuludq ymm12, ymm10, ymmword [r9+sizeof(ymmword)*5] + vpaddq ymm9, ymm9, ymm12 + vpmuludq ymm13, ymm10, ymmword [r9+sizeof(ymmword)*6] + vpaddq ymm0, ymm0, ymm13 + vpmuludq ymm11, ymm10, ymmword [r9+sizeof(ymmword)*7] + vpaddq ymm1, ymm1, ymm11 + vpmuludq ymm12, ymm10, ymmword [r9+sizeof(ymmword)*8] + vpaddq ymm2, ymm2, ymm12 + vpmuludq ymm3, ymm10, ymmword [r9+sizeof(ymmword)*9] + vpaddq ymm3, ymm3, ymmword [rbx+sizeof(ymmword)*3] + + vmovdqu ymmword [rdi+sizeof(ymmword)*8], ymm8 + vmovdqu ymmword [rdi+sizeof(ymmword)*9], ymm9 + + vpmuludq ymm11, ymm14, ymmword [rsi+sizeof(ymmword)*5] + vpbroadcastq ymm10, qword [rax+sizeof(qword)*24] ; ymm10 = {a24:a24:a24:a24} + vpaddq ymm0, ymm0, ymm11 + vpmuludq ymm12, ymm14, ymmword [r9+sizeof(ymmword)*6] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm14, ymmword [r9+sizeof(ymmword)*7] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm11, ymm14, ymmword [r9+sizeof(ymmword)*8] + vpaddq ymm3, ymm3, ymm11 + vpmuludq ymm4, ymm14, ymmword [r9+sizeof(ymmword)*9] + vpaddq ymm4, ymm4, ymmword [rbx+sizeof(ymmword)*4] + + vmovdqu ymmword [rdi+sizeof(ymmword)*10], ymm0 + vmovdqu ymmword [rdi+sizeof(ymmword)*11], ymm1 + + vpmuludq ymm11, ymm10, ymmword [rsi+sizeof(ymmword)*6] + vpbroadcastq ymm14, qword [rax+sizeof(qword)*28] ; ymm14 = {a28:a28:a28:a28} + vpaddq ymm2, ymm2, ymm11 + vpmuludq ymm12, ymm10, ymmword [r9+sizeof(ymmword)*7] + vpaddq ymm3, ymm3, ymm12 + vpmuludq ymm13, ymm10, ymmword [r9+sizeof(ymmword)*8] + vpaddq ymm4, ymm4, ymm13 + vpmuludq ymm5, ymm10, ymmword [r9+sizeof(ymmword)*9] + vpaddq ymm5, ymm5, ymmword [rbx+sizeof(ymmword)*5] + + vmovdqu ymmword [rdi+sizeof(ymmword)*12], ymm2 + vmovdqu ymmword [rdi+sizeof(ymmword)*13], ymm3 + + vpmuludq ymm11, ymm14, ymmword [rsi+sizeof(ymmword)*7] + vpbroadcastq ymm10, qword [rax+sizeof(qword)*32] ; ymm10 = {a32:a32:a32:a32} + vpaddq ymm4, ymm4, ymm11 + vpmuludq ymm12, ymm14, ymmword [r9+sizeof(ymmword)*8] + vpaddq ymm5, ymm5, ymm12 + vpmuludq ymm6, ymm14, ymmword [r9+sizeof(ymmword)*9] + vpaddq ymm6, ymm6, ymmword [rbx+sizeof(ymmword)*6] + + vmovdqu ymmword [rdi+sizeof(ymmword)*14], ymm4 + vmovdqu ymmword [rdi+sizeof(ymmword)*15], ymm5 + + vpmuludq ymm11, ymm10, ymmword [rsi+sizeof(ymmword)*8] + vpbroadcastq ymm14, qword [rax+sizeof(qword)*36] ; ymm14 = {a36:a36:a36:a36} + vpaddq ymm6, ymm6, ymm11 + vpmuludq ymm7, ymm10, ymmword [r9+sizeof(ymmword)*9] + vpaddq ymm7, ymm7, ymmword [rbx+sizeof(ymmword)*7] + + vpmuludq ymm8, ymm14, ymmword [rsi+sizeof(ymmword)*9] + vpbroadcastq ymm10, qword [rax+sizeof(qword)] ; ymm10 = {a[1/2/3]:a[1/2/3]:a[1/2/3]:a[1/2/3]} + vpaddq ymm8, ymm8, ymmword [rbx+sizeof(ymmword)*8] + + vmovdqu ymmword [rdi+sizeof(ymmword)*16], ymm6 + vmovdqu ymmword [rdi+sizeof(ymmword)*17], ymm7 + vmovdqu ymmword [rdi+sizeof(ymmword)*18], ymm8 + + add rdi, sizeof(qword) + add rbx, sizeof(qword) + add rax, sizeof(qword) + sub rcx, 1 + jnz .sqr1024_loop4 + +;; +;; reduction +;; + mov rdi, qword [rsp+pAxA] ; restore pointer to temporary result (low A^2) + mov rcx, qword [rsp+pM] ; restore pointer to modulus + mov r8, qword [rsp+m0] ; restore m0 value + mov r9, dword 38 ; modulus length + + mov r10, qword [rdi] ; load low part of temporary result + mov r11, qword [rdi+sizeof(qword)] ; + mov r12, qword [rdi+sizeof(qword)*2] ; + mov r13, qword [rdi+sizeof(qword)*3] ; + + mov rdx, r10 ; y0 = (ac0*k0) & DIGIT_MASK + imul edx, r8d + and edx, DIGIT_MASK + vmovd xmm10, edx + + vmovdqu ymm1, ymmword [rdi+sizeof(ymmword)*1] ; load other data + vmovdqu ymm2, ymmword [rdi+sizeof(ymmword)*2] ; + vmovdqu ymm3, ymmword [rdi+sizeof(ymmword)*3] ; + vmovdqu ymm4, ymmword [rdi+sizeof(ymmword)*4] ; + vmovdqu ymm5, ymmword [rdi+sizeof(ymmword)*5] ; + vmovdqu ymm6, ymmword [rdi+sizeof(ymmword)*6] ; + + mov rax, rdx ; ac0 += pn[0]*y0 + imul rax, qword [rcx] + add r10, rax + + vpbroadcastq ymm10, xmm10 + + vmovdqu ymm7, ymmword [rdi+sizeof(ymmword)*7] ; load other data + vmovdqu ymm8, ymmword [rdi+sizeof(ymmword)*8] ; + vmovdqu ymm9, ymmword [rdi+sizeof(ymmword)*9] ; + + mov rax, rdx ; ac1 += pn[1]*y0 + imul rax, qword [rcx+sizeof(qword)] + add r11, rax + + mov rax, rdx ; ac2 += pn[2]*y0 + imul rax, qword [rcx+sizeof(qword)*2] + add r12, rax + + shr r10, DIGIT_BITS ; updtae ac1 + + imul rdx, qword [rcx+sizeof(qword)*3] ; ac3 += pn[3]*y0 + add r13, rdx + add r11, r10 ; updtae ac1 + + mov rdx, r11 ; y1 = (ac1*k0) & DIGIT_MASK + imul edx, r8d + and rdx, DIGIT_MASK + +align IPP_ALIGN_FACTOR +.reduction_loop: + vmovd xmm11, edx + vpbroadcastq ymm11, xmm11 + + vpmuludq ymm14, ymm10, ymmword [rcx+sizeof(ymmword)] + mov rax, rdx ; ac1 += pn[0]*y1 + imul rax, qword [rcx] + vpaddq ymm1, ymm1, ymm14 + vpmuludq ymm14, ymm10, ymmword [rcx+sizeof(ymmword)*2] + vpaddq ymm2, ymm2, ymm14 + vpmuludq ymm14, ymm10, ymmword [rcx+sizeof(ymmword)*3] + vpaddq ymm3, ymm3, ymm14 + + vpmuludq ymm14, ymm10, ymmword [rcx+sizeof(ymmword)*4] + add r11, rax + shr r11, DIGIT_BITS ; update ac2 + vpaddq ymm4, ymm4, ymm14 + + vpmuludq ymm14, ymm10, ymmword [rcx+sizeof(ymmword)*5] + mov rax, rdx ; ac2 += pn[1]*y1 + imul rax, qword [rcx+sizeof(qword)] + vpaddq ymm5, ymm5, ymm14 + add r12, rax + + vpmuludq ymm14, ymm10, ymmword [rcx+sizeof(ymmword)*6] + imul rdx, qword [rcx+sizeof(qword)*2] ; ac3 += pn[2]*y1 + add r12, r11 + vpaddq ymm6, ymm6, ymm14 + add r13, rdx + + vpmuludq ymm14, ymm10, ymmword [rcx+sizeof(ymmword)*7] + mov rdx, r12 ; y2 = (ac2*m0) & DIGIT_MASK + imul edx, r8d + vpaddq ymm7, ymm7, ymm14 + vpmuludq ymm14, ymm10, ymmword [rcx+sizeof(ymmword)*8] + vpaddq ymm8, ymm8, ymm14 + and rdx, DIGIT_MASK + + vpmuludq ymm14, ymm10, ymmword [rcx+sizeof(ymmword)*9] + vpaddq ymm9, ymm9, ymm14 +;; ------------------------------------------------------------ + + vmovd xmm12, edx + vpbroadcastq ymm12, xmm12 + + vpmuludq ymm14, ymm11, ymmword [rcx+sizeof(ymmword)-sizeof(qword)] + mov rax, rdx ; ac2 += pn[0]*y2 + imul rax, qword [rcx] + vpaddq ymm1, ymm1, ymm14 + + vpmuludq ymm14, ymm11, ymmword [rcx+sizeof(ymmword)*2-sizeof(qword)] + vpaddq ymm2, ymm2, ymm14 + + vpmuludq ymm14, ymm11, ymmword [rcx+sizeof(ymmword)*3-sizeof(qword)] + vpaddq ymm3, ymm3, ymm14 + vpmuludq ymm14, ymm11, ymmword [rcx+sizeof(ymmword)*4-sizeof(qword)] + vpaddq ymm4, ymm4, ymm14 + add rax, r12 + shr rax, DIGIT_BITS ; update ac3 + + + vpmuludq ymm14, ymm11, ymmword [rcx+sizeof(ymmword)*5-sizeof(qword)] + imul rdx, qword [rcx+sizeof(qword)] ; ac3 += pn[1]*y2 + vpaddq ymm5, ymm5, ymm14 + vpmuludq ymm14, ymm11, ymmword [rcx+sizeof(ymmword)*6-sizeof(qword)] + vpaddq ymm6, ymm6, ymm14 + vpmuludq ymm14, ymm11, ymmword [rcx+sizeof(ymmword)*7-sizeof(qword)] + vpaddq ymm7, ymm7, ymm14 + add rdx, r13 + add rdx, rax ; update ac3 + + vpmuludq ymm14, ymm11, ymmword [rcx+sizeof(ymmword)*8-sizeof(qword)] + vpaddq ymm8, ymm8, ymm14 + vpmuludq ymm14, ymm11, ymmword [rcx+sizeof(ymmword)*9-sizeof(qword)] + vpaddq ymm9, ymm9, ymm14 + + sub r9, 2 + jz .exit_reduction_loop +;; ------------------------------------------------------------ + + vpmuludq ymm14, ymm12, ymmword [rcx+sizeof(ymmword)-sizeof(qword)*2] + mov r13, rdx ; y3 = (ac3*m0) & DIGIT_MASK + imul edx, r8d + vpaddq ymm1, ymm1, ymm14 + vpmuludq ymm14, ymm12, ymmword [rcx+sizeof(ymmword)*2-sizeof(qword)*2] + vpaddq ymm2, ymm2, ymm14 + and rdx, DIGIT_MASK + + vpmuludq ymm14, ymm12, ymmword [rcx+sizeof(ymmword)*3-sizeof(qword)*2] + vmovd xmm13, edx + vpaddq ymm3, ymm3, ymm14 + vpmuludq ymm14, ymm12, ymmword [rcx+sizeof(ymmword)*4-sizeof(qword)*2] + vpbroadcastq ymm13, xmm13 + vpaddq ymm4, ymm4, ymm14 + + vpmuludq ymm14, ymm12, ymmword [rcx+sizeof(ymmword)*5-sizeof(qword)*2] + imul rdx, qword [rcx] ; ac3 += pn[0]*y3 + vpaddq ymm5, ymm5, ymm14 + vpmuludq ymm14, ymm12, ymmword [rcx+sizeof(ymmword)*6-sizeof(qword)*2] + vpaddq ymm6, ymm6, ymm14 + add r13, rdx + + vpmuludq ymm14, ymm12, ymmword [rcx+sizeof(ymmword)*7-sizeof(qword)*2] + vpaddq ymm7, ymm7, ymm14 + shr r13, DIGIT_BITS + vmovq xmm0, r13 + + vpmuludq ymm14, ymm13, ymmword [rcx+sizeof(ymmword)-sizeof(qword)*3] + vpaddq ymm1, ymm1, ymm0 + vpaddq ymm1, ymm1, ymm14 + vmovdqu ymmword [rdi], ymm1 + + vpmuludq ymm14, ymm12, ymmword [rcx+sizeof(ymmword)*8-sizeof(qword)*2] + vpaddq ymm8, ymm8, ymm14 + + vpmuludq ymm14, ymm12, ymmword [rcx+sizeof(ymmword)*9-sizeof(qword)*2] + vpaddq ymm9, ymm9, ymm14 +;; ------------------------------------------------------------ + + vmovq rdx, xmm1 ; y0 = (ac0*k0) & DIGIT_MASK + imul edx, r8d + and edx, DIGIT_MASK + + vmovq r10, xmm1 ; update lowest part of temporary result + mov r11, qword [rdi+sizeof(qword)] ; + mov r12, qword [rdi+sizeof(qword)*2] ; + mov r13, qword [rdi+sizeof(qword)*3] ; + + vmovd xmm10, edx + vpbroadcastq ymm10, xmm10 + + vpmuludq ymm14, ymm13, ymmword [rcx+sizeof(ymmword)*2-sizeof(qword)*3] + mov rax, rdx ; ac0 += pn[0]*y0 + imul rax, qword [rcx] + vpaddq ymm1, ymm2, ymm14 + vpmuludq ymm14, ymm13, ymmword [rcx+sizeof(ymmword)*3-sizeof(qword)*3] + add r10, rax + vpaddq ymm2, ymm3, ymm14 + shr r10, DIGIT_BITS ; updtae ac1 + + vpmuludq ymm14, ymm13, ymmword [rcx+sizeof(ymmword)*4-sizeof(qword)*3] + mov rax, rdx ; ac1 += pn[1]*y0 + imul rax, qword [rcx+sizeof(qword)] + vpaddq ymm3, ymm4, ymm14 + add r11, rax + + vpmuludq ymm14, ymm13, ymmword [rcx+sizeof(ymmword)*5-sizeof(qword)*3] + mov rax, rdx ; ac2 += pn[2]*y0 + imul rax, qword [rcx+sizeof(qword)*2] + vpaddq ymm4, ymm5, ymm14 + add r12, rax + + vpmuludq ymm14, ymm13, ymmword [rcx+sizeof(ymmword)*6-sizeof(qword)*3] + imul rdx, qword [rcx+sizeof(qword)*3] ; ac3 += pn[3]*y0 + vpaddq ymm5, ymm6, ymm14 + add r13, rdx + add r11, r10 + + vpmuludq ymm14, ymm13, ymmword [rcx+sizeof(ymmword)*7-sizeof(qword)*3] + mov rdx, r11 ; y1 = (ac1*k0) & DIGIT_MASK + imul edx, r8d + vpaddq ymm6, ymm7, ymm14 + and rdx, DIGIT_MASK + + vpmuludq ymm14, ymm13, ymmword [rcx+sizeof(ymmword)*8-sizeof(qword)*3] + vpaddq ymm7, ymm8, ymm14 + + vpmuludq ymm14, ymm13, ymmword [rcx+sizeof(ymmword)*9-sizeof(qword)*3] + vpaddq ymm8, ymm9, ymm14 + + vpmuludq ymm14, ymm13, ymmword [rcx+sizeof(ymmword)*10-sizeof(qword)*3] + vpaddq ymm9, ymm14, ymmword [rdi+sizeof(ymmword)*10] + + add rdi, sizeof(qword)*4 + sub r9, 2 + jnz .reduction_loop + +.exit_reduction_loop: + mov rdi, qword [rsp+pResult] ; restore pointer to result + mov qword [rdi], r12 + mov qword [rdi+sizeof(qword)], r13 + vmovdqu ymmword [rdi+sizeof(ymmword)-sizeof(qword)*2], ymm1 + vmovdqu ymmword [rdi+sizeof(ymmword)*2-sizeof(qword)*2], ymm2 + vmovdqu ymmword [rdi+sizeof(ymmword)*3-sizeof(qword)*2], ymm3 + vmovdqu ymmword [rdi+sizeof(ymmword)*4-sizeof(qword)*2], ymm4 + vmovdqu ymmword [rdi+sizeof(ymmword)*5-sizeof(qword)*2], ymm5 + vmovdqu ymmword [rdi+sizeof(ymmword)*6-sizeof(qword)*2], ymm6 + vmovdqu ymmword [rdi+sizeof(ymmword)*7-sizeof(qword)*2], ymm7 + vmovdqu ymmword [rdi+sizeof(ymmword)*8-sizeof(qword)*2], ymm8 + vmovdqu ymmword [rdi+sizeof(ymmword)*9-sizeof(qword)*2], ymm9 + +;; +;; normalization +;; + mov r9, dword 38 + xor rax, rax +.norm_loop: + add rax, qword [rdi] + add rdi, sizeof(qword) + mov rdx, dword DIGIT_MASK + and rdx, rax + shr rax, DIGIT_BITS + mov qword [rdi-sizeof(qword)], rdx + sub r9, 1 + jg .norm_loop + mov qword [rdi], rax + + REST_XMM_AVX + REST_GPR + ret +ENDFUNC cpMontSqr1024_avx2 + +;************************************************************* +;* void cpSqr1024_avx2(Ipp64u* pR, +;* const Ipp64u* pA, int nsA, +;* Ipp64u* pBuffer) +;************************************************************* + +align IPP_ALIGN_FACTOR +IPPASM cpSqr1024_avx2,PUBLIC +%assign LOCAL_FRAME sizeof(qword)*5 + USES_GPR rsi,rdi,rbx + USES_XMM_AVX ymm6,ymm7,ymm8,ymm9,ymm10,ymm11,ymm12,ymm13,ymm14 + COMP_ABI 4 + + movsxd rdx, edx ; redLen value counter + vpxor ymm11, ymm11, ymm11 + +;; expands A operand + vmovdqu ymmword [rsi+rdx*sizeof(qword)], ymm11 + +; +; stack struct +; +%assign pResult 0 ; pointer to result +%assign pA pResult+sizeof(qword) ; pointer to A operand +%assign redLen pA+sizeof(qword) ; length +%assign pA2 redLen+sizeof(qword) ; pointer to buffer (contains doubled input (A*2) and temporary result (A^2) ) +%assign pAxA pA2+sizeof(qword) ; pointer to temporary result (A^2) + + mov qword [rsp+pResult], rdi ; store pointer to result + mov qword [rsp+pA], rsi ; store pointer to input A + mov qword [rsp+redLen], rdx ; store redLen + + mov rdx, dword 40 + + mov rdi, rcx ; pointer to buffer + mov qword [rsp+pAxA], rdi ; pointer to temporary result (low A^2) + lea rbx, [rdi+rdx*sizeof(qword)] ; pointer to temporary result (high A^2) + lea r9, [rbx+rdx*sizeof(qword)] ; pointer to doubled input (A*2) + mov qword [rsp+pA2], r9 + + mov rax, rsi + mov rcx, dword sizeof(ymmword)/sizeof(qword) + +;; doubling input + vmovdqu ymm0, ymmword [rsi] + vmovdqu ymm1, ymmword [rsi+sizeof(ymmword)] + vmovdqu ymm2, ymmword [rsi+sizeof(ymmword)*2] + vmovdqu ymm3, ymmword [rsi+sizeof(ymmword)*3] + vmovdqu ymm4, ymmword [rsi+sizeof(ymmword)*4] + vmovdqu ymm5, ymmword [rsi+sizeof(ymmword)*5] + vmovdqu ymm6, ymmword [rsi+sizeof(ymmword)*6] + vmovdqu ymm7, ymmword [rsi+sizeof(ymmword)*7] + vmovdqu ymm8, ymmword [rsi+sizeof(ymmword)*8] + vmovdqu ymm9, ymmword [rsi+sizeof(ymmword)*9] + + vmovdqu ymmword [r9], ymm0 + vpbroadcastq ymm10, qword [rax] ; ymm10 = {a0:a0:a0:a0} + vpaddq ymm1, ymm1, ymm1 + vmovdqu ymmword [r9+sizeof(ymmword)], ymm1 + vpaddq ymm2, ymm2, ymm2 + vmovdqu ymmword [r9+sizeof(ymmword)*2], ymm2 + vpaddq ymm3, ymm3, ymm3 + vmovdqu ymmword [r9+sizeof(ymmword)*3], ymm3 + vpaddq ymm4, ymm4, ymm4 + vmovdqu ymmword [r9+sizeof(ymmword)*4], ymm4 + vpaddq ymm5, ymm5, ymm5 + vmovdqu ymmword [r9+sizeof(ymmword)*5], ymm5 + vpaddq ymm6, ymm6, ymm6 + vmovdqu ymmword [r9+sizeof(ymmword)*6], ymm6 + vpaddq ymm7, ymm7, ymm7 + vmovdqu ymmword [r9+sizeof(ymmword)*7], ymm7 + vpaddq ymm8, ymm8, ymm8 + vmovdqu ymmword [r9+sizeof(ymmword)*8], ymm8 + vpaddq ymm9, ymm9, ymm9 + vmovdqu ymmword [r9+sizeof(ymmword)*9], ymm9 + +;; +;; squaring +;; + vpmuludq ymm0, ymm10, ymmword [rsi] + vpbroadcastq ymm14, qword [rax+sizeof(qword)*4] ; ymm14 = {a4:a4:a4:a4} + vmovdqu ymmword [rbx], ymm11 + vpmuludq ymm1, ymm10, ymmword [r9+sizeof(ymmword)] + vmovdqu ymmword [rbx+sizeof(ymmword)], ymm11 + vpmuludq ymm2, ymm10, ymmword [r9+sizeof(ymmword)*2] + vmovdqu ymmword [rbx+sizeof(ymmword)*2], ymm11 + vpmuludq ymm3, ymm10, ymmword [r9+sizeof(ymmword)*3] + vmovdqu ymmword [rbx+sizeof(ymmword)*3], ymm11 + vpmuludq ymm4, ymm10, ymmword [r9+sizeof(ymmword)*4] + vmovdqu ymmword [rbx+sizeof(ymmword)*4], ymm11 + vpmuludq ymm5, ymm10, ymmword [r9+sizeof(ymmword)*5] + vmovdqu ymmword [rbx+sizeof(ymmword)*5], ymm11 + vpmuludq ymm6, ymm10, ymmword [r9+sizeof(ymmword)*6] + vmovdqu ymmword [rbx+sizeof(ymmword)*6], ymm11 + vpmuludq ymm7, ymm10, ymmword [r9+sizeof(ymmword)*7] + vmovdqu ymmword [rbx+sizeof(ymmword)*7], ymm11 + vpmuludq ymm8, ymm10, ymmword [r9+sizeof(ymmword)*8] + vmovdqu ymmword [rbx+sizeof(ymmword)*8], ymm11 + vpmuludq ymm9, ymm10, ymmword [r9+sizeof(ymmword)*9] + vmovdqu ymmword [rbx+sizeof(ymmword)*9], ymm11 + + jmp .sqr1024_ep + +align IPP_ALIGN_FACTOR +.sqr1024_loop4: + vpmuludq ymm0, ymm10, ymmword [rsi] + vpbroadcastq ymm14, qword [rax+sizeof(qword)*4] ; ymm14 = {a4:a4:a4:a4} + vpaddq ymm0, ymm0, ymmword [rdi] + + vpmuludq ymm1, ymm10, ymmword [r9+sizeof(ymmword)] + vpaddq ymm1, ymm1, ymmword [rdi+sizeof(ymmword)] + + vpmuludq ymm2, ymm10, ymmword [r9+sizeof(ymmword)*2] + vpaddq ymm2, ymm2, ymmword [rdi+sizeof(ymmword)*2] + + vpmuludq ymm3, ymm10, ymmword [r9+sizeof(ymmword)*3] + vpaddq ymm3, ymm3, ymmword [rdi+sizeof(ymmword)*3] + + vpmuludq ymm4, ymm10, ymmword [r9+sizeof(ymmword)*4] + vpaddq ymm4, ymm4, ymmword [rdi+sizeof(ymmword)*4] + + vpmuludq ymm5, ymm10, ymmword [r9+sizeof(ymmword)*5] + vpaddq ymm5, ymm5, ymmword [rdi+sizeof(ymmword)*5] + + vpmuludq ymm6, ymm10, ymmword [r9+sizeof(ymmword)*6] + vpaddq ymm6, ymm6, ymmword [rdi+sizeof(ymmword)*6] + + vpmuludq ymm7, ymm10, ymmword [r9+sizeof(ymmword)*7] + vpaddq ymm7, ymm7, ymmword [rdi+sizeof(ymmword)*7] + + vpmuludq ymm8, ymm10, ymmword [r9+sizeof(ymmword)*8] + vpaddq ymm8, ymm8, ymmword [rdi+sizeof(ymmword)*8] + + vpmuludq ymm9, ymm10, ymmword [r9+sizeof(ymmword)*9] + vpaddq ymm9, ymm9, ymmword [rdi+sizeof(ymmword)*9] + +.sqr1024_ep: + vmovdqu ymmword [rdi], ymm0 + vmovdqu ymmword [rdi+sizeof(ymmword)], ymm1 + + vpmuludq ymm11, ymm14, ymmword [rsi+sizeof(ymmword)] + vpbroadcastq ymm10, qword [rax+sizeof(qword)*8] ; ymm10 = {a8:a8:a8:a8} + vpaddq ymm2, ymm2, ymm11 + vpmuludq ymm12, ymm14, ymmword [r9+sizeof(ymmword)*2] + vpaddq ymm3, ymm3, ymm12 + vpmuludq ymm13, ymm14, ymmword [r9+sizeof(ymmword)*3] + vpaddq ymm4, ymm4, ymm13 + vpmuludq ymm11, ymm14, ymmword [r9+sizeof(ymmword)*4] + vpaddq ymm5, ymm5, ymm11 + vpmuludq ymm12, ymm14, ymmword [r9+sizeof(ymmword)*5] + vpaddq ymm6, ymm6, ymm12 + vpmuludq ymm13, ymm14, ymmword [r9+sizeof(ymmword)*6] + vpaddq ymm7, ymm7, ymm13 + vpmuludq ymm11, ymm14, ymmword [r9+sizeof(ymmword)*7] + vpaddq ymm8, ymm8, ymm11 + vpmuludq ymm12, ymm14, ymmword [r9+sizeof(ymmword)*8] + vpaddq ymm9, ymm9, ymm12 + vpmuludq ymm0, ymm14, ymmword [r9+sizeof(ymmword)*9] + vpaddq ymm0, ymm0, ymmword [rbx] + + vmovdqu ymmword [rdi+sizeof(ymmword)*2], ymm2 + vmovdqu ymmword [rdi+sizeof(ymmword)*3], ymm3 + + vpmuludq ymm11, ymm10, ymmword [rsi+sizeof(ymmword)*2] + vpbroadcastq ymm14, qword [rax+sizeof(qword)*12] ; ymm14 = {a12:a12:a12:a12} + vpaddq ymm4, ymm4, ymm11 + vpmuludq ymm12, ymm10, ymmword [r9+sizeof(ymmword)*3] + vpaddq ymm5, ymm5, ymm12 + vpmuludq ymm13, ymm10, ymmword [r9+sizeof(ymmword)*4] + vpaddq ymm6, ymm6, ymm13 + vpmuludq ymm11, ymm10, ymmword [r9+sizeof(ymmword)*5] + vpaddq ymm7, ymm7, ymm11 + vpmuludq ymm12, ymm10, ymmword [r9+sizeof(ymmword)*6] + vpaddq ymm8, ymm8, ymm12 + vpmuludq ymm13, ymm10, ymmword [r9+sizeof(ymmword)*7] + vpaddq ymm9, ymm9, ymm13 + vpmuludq ymm11, ymm10, ymmword [r9+sizeof(ymmword)*8] + vpaddq ymm0, ymm0, ymm11 + vpmuludq ymm1, ymm10, ymmword [r9+sizeof(ymmword)*9] + vpaddq ymm1, ymm1, ymmword [rbx+sizeof(ymmword)] + + vmovdqu ymmword [rdi+sizeof(ymmword)*4], ymm4 + vmovdqu ymmword [rdi+sizeof(ymmword)*5], ymm5 + + vpmuludq ymm11, ymm14, ymmword [rsi+sizeof(ymmword)*3] + vpbroadcastq ymm10, qword [rax+sizeof(qword)*16] ; ymm10 = {a16:a16:a16:a16} + vpaddq ymm6, ymm6, ymm11 + vpmuludq ymm12, ymm14, ymmword [r9+sizeof(ymmword)*4] + vpaddq ymm7, ymm7, ymm12 + vpmuludq ymm13, ymm14, ymmword [r9+sizeof(ymmword)*5] + vpaddq ymm8, ymm8, ymm13 + vpmuludq ymm11, ymm14, ymmword [r9+sizeof(ymmword)*6] + vpaddq ymm9, ymm9, ymm11 + vpmuludq ymm12, ymm14, ymmword [r9+sizeof(ymmword)*7] + vpaddq ymm0, ymm0, ymm12 + vpmuludq ymm13, ymm14, ymmword [r9+sizeof(ymmword)*8] + vpaddq ymm1, ymm1, ymm13 + vpmuludq ymm2, ymm14, ymmword [r9+sizeof(ymmword)*9] + vpaddq ymm2, ymm2, ymmword [rbx+sizeof(ymmword)*2] + + vmovdqu ymmword [rdi+sizeof(ymmword)*6], ymm6 + vmovdqu ymmword [rdi+sizeof(ymmword)*7], ymm7 + + vpmuludq ymm11, ymm10, ymmword [rsi+sizeof(ymmword)*4] + vpbroadcastq ymm14, qword [rax+sizeof(qword)*20] ; ymm14 = {a20:a20:a20:a20} + vpaddq ymm8, ymm8, ymm11 + vpmuludq ymm12, ymm10, ymmword [r9+sizeof(ymmword)*5] + vpaddq ymm9, ymm9, ymm12 + vpmuludq ymm13, ymm10, ymmword [r9+sizeof(ymmword)*6] + vpaddq ymm0, ymm0, ymm13 + vpmuludq ymm11, ymm10, ymmword [r9+sizeof(ymmword)*7] + vpaddq ymm1, ymm1, ymm11 + vpmuludq ymm12, ymm10, ymmword [r9+sizeof(ymmword)*8] + vpaddq ymm2, ymm2, ymm12 + vpmuludq ymm3, ymm10, ymmword [r9+sizeof(ymmword)*9] + vpaddq ymm3, ymm3, ymmword [rbx+sizeof(ymmword)*3] + + vmovdqu ymmword [rdi+sizeof(ymmword)*8], ymm8 + vmovdqu ymmword [rdi+sizeof(ymmword)*9], ymm9 + + vpmuludq ymm11, ymm14, ymmword [rsi+sizeof(ymmword)*5] + vpbroadcastq ymm10, qword [rax+sizeof(qword)*24] ; ymm10 = {a24:a24:a24:a24} + vpaddq ymm0, ymm0, ymm11 + vpmuludq ymm12, ymm14, ymmword [r9+sizeof(ymmword)*6] + vpaddq ymm1, ymm1, ymm12 + vpmuludq ymm13, ymm14, ymmword [r9+sizeof(ymmword)*7] + vpaddq ymm2, ymm2, ymm13 + vpmuludq ymm11, ymm14, ymmword [r9+sizeof(ymmword)*8] + vpaddq ymm3, ymm3, ymm11 + vpmuludq ymm4, ymm14, ymmword [r9+sizeof(ymmword)*9] + vpaddq ymm4, ymm4, ymmword [rbx+sizeof(ymmword)*4] + + vmovdqu ymmword [rdi+sizeof(ymmword)*10], ymm0 + vmovdqu ymmword [rdi+sizeof(ymmword)*11], ymm1 + + vpmuludq ymm11, ymm10, ymmword [rsi+sizeof(ymmword)*6] + vpbroadcastq ymm14, qword [rax+sizeof(qword)*28] ; ymm14 = {a28:a28:a28:a28} + vpaddq ymm2, ymm2, ymm11 + vpmuludq ymm12, ymm10, ymmword [r9+sizeof(ymmword)*7] + vpaddq ymm3, ymm3, ymm12 + vpmuludq ymm13, ymm10, ymmword [r9+sizeof(ymmword)*8] + vpaddq ymm4, ymm4, ymm13 + vpmuludq ymm5, ymm10, ymmword [r9+sizeof(ymmword)*9] + vpaddq ymm5, ymm5, ymmword [rbx+sizeof(ymmword)*5] + + vmovdqu ymmword [rdi+sizeof(ymmword)*12], ymm2 + vmovdqu ymmword [rdi+sizeof(ymmword)*13], ymm3 + + vpmuludq ymm11, ymm14, ymmword [rsi+sizeof(ymmword)*7] + vpbroadcastq ymm10, qword [rax+sizeof(qword)*32] ; ymm10 = {a32:a32:a32:a32} + vpaddq ymm4, ymm4, ymm11 + vpmuludq ymm12, ymm14, ymmword [r9+sizeof(ymmword)*8] + vpaddq ymm5, ymm5, ymm12 + vpmuludq ymm6, ymm14, ymmword [r9+sizeof(ymmword)*9] + vpaddq ymm6, ymm6, ymmword [rbx+sizeof(ymmword)*6] + + vmovdqu ymmword [rdi+sizeof(ymmword)*14], ymm4 + vmovdqu ymmword [rdi+sizeof(ymmword)*15], ymm5 + + vpmuludq ymm11, ymm10, ymmword [rsi+sizeof(ymmword)*8] + vpbroadcastq ymm14, qword [rax+sizeof(qword)*36] ; ymm14 = {a36:a36:a36:a36} + vpaddq ymm6, ymm6, ymm11 + vpmuludq ymm7, ymm10, ymmword [r9+sizeof(ymmword)*9] + vpaddq ymm7, ymm7, ymmword [rbx+sizeof(ymmword)*7] + + vpmuludq ymm8, ymm14, ymmword [rsi+sizeof(ymmword)*9] + vpbroadcastq ymm10, qword [rax+sizeof(qword)] ; ymm10 = {a[1/2/3]:a[1/2/3]:a[1/2/3]:a[1/2/3]} + vpaddq ymm8, ymm8, ymmword [rbx+sizeof(ymmword)*8] + + vmovdqu ymmword [rdi+sizeof(ymmword)*16], ymm6 + vmovdqu ymmword [rdi+sizeof(ymmword)*17], ymm7 + vmovdqu ymmword [rdi+sizeof(ymmword)*18], ymm8 + + add rdi, sizeof(qword) + add rbx, sizeof(qword) + add rax, sizeof(qword) + sub rcx, 1 + jnz .sqr1024_loop4 + + +;; +;; normalization +;; + mov rsi, qword [rsp+pAxA] + mov rdi, qword [rsp+pResult] + mov r9, dword 38*2 + xor rax, rax +.norm_loop: + add rax, qword [rsi] + add rsi, sizeof(qword) + mov rdx, dword DIGIT_MASK + and rdx, rax + shr rax, DIGIT_BITS + mov qword [rdi], rdx + add rdi, sizeof(qword) + sub r9, 1 + jg .norm_loop + mov qword [rdi], rax + + REST_XMM_AVX + REST_GPR + ret +ENDFUNC cpSqr1024_avx2 + +%endif ; _IPP32E_L9 + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmontsqr_avx2as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmontsqr_avx2as.asm new file mode 100644 index 0000000..d825b60 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmontsqr_avx2as.asm @@ -0,0 +1,802 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Big Number Operations +; +; Content: +; cpSqr_avx2() +; cpMontRed_avx2() +; + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_L9) + +segment .text align=IPP_ALIGN_FACTOR + + +%assign DIGIT_BITS 27 +%assign DIGIT_MASK (1 << DIGIT_BITS) -1 + +;************************************************************* +;* void cpSqr_avx2(Ipp64u* pR, +;* const Ipp64u* pA, int aSize, +;* Ipp64u* pBuffer) +;************************************************************* +align IPP_ALIGN_FACTOR +IPPASM cpSqr_avx2,PUBLIC +%assign LOCAL_FRAME sizeof(qword)*6 + USES_GPR rsi,rdi,rbx,rbp,r12,r13 + USES_XMM_AVX ymm6,ymm7,ymm8,ymm9,ymm10,ymm11,ymm12,ymm13,ymm14 + COMP_ABI 4 + + movsxd rdx, edx ; redLen value counter + vpxor ymm11, ymm11, ymm11 + +;; expands A + vmovdqu ymmword [rsi+rdx*sizeof(qword)], ymm11 ; size of buffer = redLen+4 + +; +; stack struct +; +%assign pResult 0 ; pointer to result +%assign pA pResult+sizeof(qword) ; pointer to A operand +%assign redLen pA+sizeof(qword) ; length +%assign len4 redLen+sizeof(qword) ; x4 length +%assign pA2 len4+sizeof(qword) ; pointer to buffer (contains doubled input (A*2) and temporary result (A^2) ) +%assign pAxA pA2+sizeof(qword) ; pointer to temporary result (A^2) + + lea rax, [rdx+3] ; len4 = ((redLen+3) &(-4))/4 + and rax, -4 ; multiple 4 length of operand (in ymmwords) + shr rax, 2 + + mov qword [rsp+pResult], rdi ; store pointer to result + mov qword [rsp+pA], rsi ; store pointer to input A + mov qword [rsp+redLen], rdx ; store redLen + mov qword [rsp+len4], rax ; store len4 + + mov rdi, rcx ; pointer to word buffer + shl rax, 2 + mov qword [rsp+pAxA], rdi ; pointer to temporary result (low A^2) + lea rbx, [rdi+rax*sizeof(qword)] ; pointer to temporary result (high A^2) + lea r9, [rbx+rax*sizeof(qword)] ; pointer to doubled input (A*2) + mov qword [rsp+pA2], r9 + +;; clear temporary result and double input +.clr_dbl_loop: + vmovdqu ymm0, ymmword [rsi] + vpaddq ymm0, ymm0, ymm0 + vmovdqu ymmword [rdi], ymm11 + vmovdqu ymmword [rbx], ymm11 + vmovdqu ymmword [r9], ymm0 + add rsi, sizeof(ymmword) + add rdi, sizeof(ymmword) + add rbx, sizeof(ymmword) + add r9, sizeof(ymmword) + sub rax, sizeof(ymmword)/sizeof(qword) + jg .clr_dbl_loop + +;; +;; squaring +;; + mov rsi, qword [rsp+pA] ; restore A + mov rdi, qword [rsp+pAxA] ; restore temp result buffer + mov r9, qword [rsp+pA2] ; restore temp double buffer + + mov rax, rsi ; tsrc = src + sizeof(qword)*n, n=0,1,2,3 + mov rcx, dword 4 +align IPP_ALIGN_FACTOR +.sqr_offset_loop: ; for(n=0; n<4; n++) + + mov r10, qword [rsp+len4] + mov r12, dword 3 + and r12, r10 ; init vflg = len4%4 + lea r11, [r10-(4*2-1)] ; init inner_loop counter (hcnt = len4 -7) + shr r10, 2 ; init outer_loop counter (vcnt = len4/4 -1) + sub r10, 1 ; + + push r9 ; (optr_d256) + push rsi ; (optr_s256) + push rdi ; (optr_r256) + push rax ; (optr_a256) + +align IPP_ALIGN_FACTOR +.sqr_outer_loop: ; for(i=0; i 1 + mov rdx, %%M0 + imul rdx, %%X0 + MLA_FIX {%%mSize},{},{%%rModulus},{%%TMPH},{%%TMP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} +%endif +%if %%mSize > 2 + mov rdx, %%M0 + imul rdx, %%X0 + MLA_FIX {%%mSize},{},{%%rModulus},{%%TMPH},{%%TMP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} +%endif +%if %%mSize > 3 + mov rdx, %%M0 + imul rdx, %%X0 + MLA_FIX {%%mSize},{},{%%rModulus},{%%TMPH},{%%TMP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} +%endif +%if %%mSize > 4 + mov rdx, %%M0 + imul rdx, %%X0 + MLA_FIX {%%mSize},{},{%%rModulus},{%%TMPH},{%%TMP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} +%endif +%if %%mSize > 5 + mov rdx, %%M0 + imul rdx, %%X0 + MLA_FIX {%%mSize},{},{%%rModulus},{%%TMPH},{%%TMP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} +%endif +%if %%mSize > 6 + mov rdx, %%M0 + imul rdx, %%X0 + MLA_FIX {%%mSize},{},{%%rModulus},{%%TMPH},{%%TMP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} +%endif +%if %%mSize > 7 + mov rdx, %%M0 + imul rdx, %%X0 + MLA_FIX {%%mSize},{},{%%rModulus},{%%TMPH},{%%TMP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} +%endif + + xor %%TMP, %%TMP + add %%X0, qword [%%rProduct+sizeof(qword)*(%%mSize+0)] + mov qword [%%rProduct+sizeof(qword)*(%%mSize+0)], %%X0 +%if %%mSize > 1 + adc %%X1, qword [%%rProduct+sizeof(qword)*(%%mSize+1)] + mov qword [%%rProduct+sizeof(qword)*(%%mSize+1)], %%X1 +%endif +%if %%mSize > 2 + adc %%X2, qword [%%rProduct+sizeof(qword)*(%%mSize+2)] + mov qword [%%rProduct+sizeof(qword)*(%%mSize+2)], %%X2 +%endif +%if %%mSize > 3 + adc %%X3, qword [%%rProduct+sizeof(qword)*(%%mSize+3)] + mov qword [%%rProduct+sizeof(qword)*(%%mSize+3)], %%X3 +%endif +%if %%mSize > 4 + adc %%X4, qword [%%rProduct+sizeof(qword)*(%%mSize+4)] + mov qword [%%rProduct+sizeof(qword)*(%%mSize+4)], %%X4 +%endif +%if %%mSize > 5 + adc %%X5, qword [%%rProduct+sizeof(qword)*(%%mSize+5)] + mov qword [%%rProduct+sizeof(qword)*(%%mSize+5)], %%X5 +%endif +%if %%mSize > 6 + adc %%X6, qword [%%rProduct+sizeof(qword)*(%%mSize+6)] + mov qword [%%rProduct+sizeof(qword)*(%%mSize+6)], %%X6 +%endif +%if %%mSize > 7 + adc %%X7, qword [%%rProduct+sizeof(qword)*(%%mSize+7)] + mov qword [%%rProduct+sizeof(qword)*(%%mSize+7)], %%X7 +%endif + adc %%TMP, 0 + + sub %%X0, qword [%%rModulus+sizeof(qword)*0] +%if %%mSize > 1 + sbb %%X1, qword [%%rModulus+sizeof(qword)*1] +%endif +%if %%mSize > 2 + sbb %%X2, qword [%%rModulus+sizeof(qword)*2] +%endif +%if %%mSize > 3 + sbb %%X3, qword [%%rModulus+sizeof(qword)*3] +%endif +%if %%mSize > 4 + sbb %%X4, qword [%%rModulus+sizeof(qword)*4] +%endif +%if %%mSize > 5 + sbb %%X5, qword [%%rModulus+sizeof(qword)*5] +%endif +%if %%mSize > 6 + sbb %%X6, qword [%%rModulus+sizeof(qword)*6] +%endif +%if %%mSize > 7 + sbb %%X7, qword [%%rModulus+sizeof(qword)*7] +%endif + sbb %%TMP, 0 + + mov rax, qword [%%rProduct+sizeof(qword)*(%%mSize+0)] + cmovae rax, %%X0 + mov qword [%%rRed+sizeof(qword)*0], rax +%if %%mSize > 1 + mov rax, qword [%%rProduct+sizeof(qword)*(%%mSize+1)] + cmovae rax, %%X1 + mov qword [%%rRed+sizeof(qword)*1], rax +%endif +%if %%mSize > 2 + mov rax, qword [%%rProduct+sizeof(qword)*(%%mSize+2)] + cmovae rax, %%X2 + mov qword [%%rRed+sizeof(qword)*2], rax +%endif +%if %%mSize > 3 + mov rax, qword [%%rProduct+sizeof(qword)*(%%mSize+3)] + cmovae rax, %%X3 + mov qword [%%rRed+sizeof(qword)*3], rax +%endif +%if %%mSize > 4 + mov rax, qword [%%rProduct+sizeof(qword)*(%%mSize+4)] + cmovae rax, %%X4 + mov qword [%%rRed+sizeof(qword)*4], rax +%endif +%if %%mSize > 5 + mov rax, qword [%%rProduct+sizeof(qword)*(%%mSize+5)] + cmovae rax, %%X5 + mov qword [%%rRed+sizeof(qword)*5], rax +%endif +%if %%mSize > 6 + mov rax, qword [%%rProduct+sizeof(qword)*(%%mSize+6)] + cmovae rax, %%X6 + mov qword [%%rRed+sizeof(qword)*6], rax +%endif +%if %%mSize > 7 + mov rax, qword [%%rProduct+sizeof(qword)*(%%mSize+7)] + cmovae rax, %%X7 + mov qword [%%rRed+sizeof(qword)*7], rax +%endif + +%endmacro + +align IPP_ALIGN_FACTOR +DECLARE_FUNC sub_N,PRIVATE + xor rax, rax ; cf = 0 +.sub_next: + lea rdi, [rdi+sizeof(qword)] + mov r8, qword [rsi] + mov r9, qword [rcx] + lea rsi, [rsi+sizeof(qword)] + lea rcx, [rcx+sizeof(qword)] + sbb r8, r9 + mov qword [rdi-sizeof(qword)], r8 + dec rdx + jnz .sub_next + adc rax, 0 + ret +ENDFUNC sub_N + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC copy_ae_N,PRIVATE + lea rdi, [rdi+sizeof(qword)] + mov r8, qword [rsi] ; src1[] + mov r9, qword [rcx] ; src2[] + lea rsi, [rsi+sizeof(qword)] + lea rcx, [rcx+sizeof(qword)] + cmovae r8, r9 + mov qword [rdi-sizeof(qword)], r8 + dec rdx + jnz copy_ae_N + ret +ENDFUNC copy_ae_N + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; mredN_start procedures +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred1_start,PRIVATE + MLA_FIX 1,, rsi, rbx,rbp, r8 + ret +ENDFUNC mred1_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred2_start,PRIVATE + MLA_FIX 2,, rsi, rbx,rbp, r8,r9 + ret +ENDFUNC mred2_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred3_start,PRIVATE + MLA_FIX 3,, rsi, rbx,rbp, r8,r9,r10 + ret +ENDFUNC mred3_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred4_start,PRIVATE + MLA_FIX 4,, rsi, rbx,rbp, r8,r9,r10,r11 + ret +ENDFUNC mred4_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred5_start,PRIVATE + MLA_FIX 5,, rsi, rbx,rbp, r8,r9,r10,r11,r12 + ret +ENDFUNC mred5_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred6_start,PRIVATE + MLA_FIX 6,, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13 + ret +ENDFUNC mred6_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred7_start,PRIVATE + MLA_FIX 7,, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14 + ret +ENDFUNC mred7_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred8_start,PRIVATE + MLA_FIX 8,, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + ret +ENDFUNC mred8_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred8x1_start,PRIVATE + push rdx ; save m' + + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*0], rdx + call mred8_start + mov [rdi], rax + + pop rdx ; resrore m' + ret +ENDFUNC mred8x1_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred8x2_start,PRIVATE + push rdx ; save m' + + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*0], rdx + call mred8_start + mov [rdi], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*1], rdx + call mred8_start + mov [rdi+sizeof(qword)*1], rax + + pop rdx ; resrore m' + ret +ENDFUNC mred8x2_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred8x3_start,PRIVATE + push rdx ; save m' + + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*0], rdx + call mred8_start + mov [rdi], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*1], rdx + call mred8_start + mov [rdi+sizeof(qword)*1], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*2], rdx + call mred8_start + mov [rdi+sizeof(qword)*2], rax + + pop rdx ; resrore m' + ret +ENDFUNC mred8x3_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred8x4_start,PRIVATE + push rdx ; save m' + + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*0], rdx + call mred8_start + mov [rdi], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*1], rdx + call mred8_start + mov [rdi+sizeof(qword)*1], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*2], rdx + call mred8_start + mov [rdi+sizeof(qword)*2], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*3], rdx + call mred8_start + mov [rdi+sizeof(qword)*3], rax + + pop rdx ; resrore m' + ret +ENDFUNC mred8x4_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred8x5_start,PRIVATE + push rdx ; save m' + + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*0], rdx + call mred8_start + mov [rdi], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*1], rdx + call mred8_start + mov [rdi+sizeof(qword)*1], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*2], rdx + call mred8_start + mov [rdi+sizeof(qword)*2], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*3], rdx + call mred8_start + mov [rdi+sizeof(qword)*3], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*4], rdx + call mred8_start + mov [rdi+sizeof(qword)*4], rax + + pop rdx ; resrore m' + ret +ENDFUNC mred8x5_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred8x6_start,PRIVATE + push rdx ; save m' + + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*0], rdx + call mred8_start + mov [rdi], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*1], rdx + call mred8_start + mov [rdi+sizeof(qword)*1], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*2], rdx + call mred8_start + mov [rdi+sizeof(qword)*2], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*3], rdx + call mred8_start + mov [rdi+sizeof(qword)*3], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*4], rdx + call mred8_start + mov [rdi+sizeof(qword)*4], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*5], rdx + call mred8_start + mov [rdi+sizeof(qword)*5], rax + + pop rdx ; resrore m' + ret +ENDFUNC mred8x6_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred8x7_start,PRIVATE + push rdx ; save m' + + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*0], rdx + call mred8_start + mov [rdi], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*1], rdx + call mred8_start + mov [rdi+sizeof(qword)*1], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*2], rdx + call mred8_start + mov [rdi+sizeof(qword)*2], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*3], rdx + call mred8_start + mov [rdi+sizeof(qword)*3], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*4], rdx + call mred8_start + mov [rdi+sizeof(qword)*4], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*5], rdx + call mred8_start + mov [rdi+sizeof(qword)*5], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*6], rdx + call mred8_start + mov [rdi+sizeof(qword)*6], rax + + pop rdx ; resrore m' + ret +ENDFUNC mred8x7_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred8x8_start,PRIVATE + push rdx ; save m' + + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*0], rdx + call mred8_start + mov [rdi], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*1], rdx + call mred8_start + mov [rdi+sizeof(qword)*1], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*2], rdx + call mred8_start + mov [rdi+sizeof(qword)*2], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*3], rdx + call mred8_start + mov [rdi+sizeof(qword)*3], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*4], rdx + call mred8_start + mov [rdi+sizeof(qword)*4], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*5], rdx + call mred8_start + mov [rdi+sizeof(qword)*5], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*6], rdx + call mred8_start + mov [rdi+sizeof(qword)*6], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*7], rdx + call mred8_start + mov [rdi+sizeof(qword)*7], rax + + pop rdx ; resrore m' + ret +ENDFUNC mred8x8_start + + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; rdi - temporary product buffer/temporary reduction +;; rsi - modulus +;; r8 - m' +;; r15 - target reduction address +;; + +;; +;; 1*qword modulus length +;; + +;; +;; 2*qword modulus length +;; + +;; +;; 3*qword modulus length +;; + +;; +;; 4*qword modulus length +;; + +;; +;; 5*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_5,PRIVATE +%assign MSIZE 5 + push r8 ; m' + + ; load low half of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + + ; ui = x[i]*m' + ; reduction += ui*modulus + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred5_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred5_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred5_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred5_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred5_start + + pop rax ; remove m' + + ; finalize cf, reduction += U0*modulus operation + xor rax, rax + op_reg_mem add, r8, [rdi+sizeof(qword)*(MSIZE+0)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+0)], r8 + op_reg_mem adc, r9, [rdi+sizeof(qword)*(MSIZE+1)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+1)], r9 + op_reg_mem adc, r10,[rdi+sizeof(qword)*(MSIZE+2)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+2)], r10 + op_reg_mem adc, r11,[rdi+sizeof(qword)*(MSIZE+3)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+3)], r11 + op_reg_mem adc, r12,[rdi+sizeof(qword)*(MSIZE+4)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+4)], r12 + adc rax, 0 + + ; reduction -= modulus + op_reg_mem sub, r8, [rsi+sizeof(qword)*0], rbx + op_reg_mem sbb, r9, [rsi+sizeof(qword)*1], rbx + op_reg_mem sbb, r10,[rsi+sizeof(qword)*2], rbx + op_reg_mem sbb, r11,[rsi+sizeof(qword)*3], rbx + op_reg_mem sbb, r12,[rsi+sizeof(qword)*4], rbx + sbb rax, 0 + + ; copy under cf [r15] = cf? [r15] : {r8-r12} + mov rax, qword [rdi+sizeof(qword)*(MSIZE+0)] + mov rbx, qword [rdi+sizeof(qword)*(MSIZE+1)] + mov rcx, qword [rdi+sizeof(qword)*(MSIZE+2)] + mov rdx, qword [rdi+sizeof(qword)*(MSIZE+3)] + mov rbp, qword [rdi+sizeof(qword)*(MSIZE+4)] + cmovae rax, r8 + cmovae rbx, r9 + cmovae rcx, r10 + cmovae rdx, r11 + cmovae rbp, r12 + mov qword [r15+sizeof(qword)*0], rax + mov qword [r15+sizeof(qword)*1], rbx + mov qword [r15+sizeof(qword)*2], rcx + mov qword [r15+sizeof(qword)*3], rdx + mov qword [r15+sizeof(qword)*4], rbp + ret +ENDFUNC mred_5 + + +;; +;; 6*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_6,PRIVATE +%assign MSIZE 6 + push r8 ; m' + + ; load low half of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + + ; ui = x[i]*m' + ; reduction += ui*modulus + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred6_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred6_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred6_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred6_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred6_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred6_start + + pop rax ; remove m' + + ; finalize cf, reduction += U0*modulus operation + xor rax, rax + op_reg_mem add, r8, [rdi+sizeof(qword)*(MSIZE+0)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+0)], r8 + op_reg_mem adc, r9, [rdi+sizeof(qword)*(MSIZE+1)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+1)], r9 + op_reg_mem adc, r10,[rdi+sizeof(qword)*(MSIZE+2)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+2)], r10 + op_reg_mem adc, r11,[rdi+sizeof(qword)*(MSIZE+3)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+3)], r11 + op_reg_mem adc, r12,[rdi+sizeof(qword)*(MSIZE+4)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+4)], r12 + op_reg_mem adc, r13,[rdi+sizeof(qword)*(MSIZE+5)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+5)], r13 + adc rax, 0 + + ; reduction -= modulus + op_reg_mem sub, r8, [rsi+sizeof(qword)*0], rbx + op_reg_mem sbb, r9, [rsi+sizeof(qword)*1], rbx + op_reg_mem sbb, r10,[rsi+sizeof(qword)*2], rbx + op_reg_mem sbb, r11,[rsi+sizeof(qword)*3], rbx + op_reg_mem sbb, r12,[rsi+sizeof(qword)*4], rbx + op_reg_mem sbb, r13,[rsi+sizeof(qword)*5], rbx + sbb rax, 0 + + ; copy under cf [r15] = cf? [r15] : {r8-r13} + mov rax, qword [rdi+sizeof(qword)*(MSIZE+0)] + mov rbx, qword [rdi+sizeof(qword)*(MSIZE+1)] + mov rcx, qword [rdi+sizeof(qword)*(MSIZE+2)] + mov rdx, qword [rdi+sizeof(qword)*(MSIZE+3)] + mov rbp, qword [rdi+sizeof(qword)*(MSIZE+4)] + mov rsi, qword [rdi+sizeof(qword)*(MSIZE+5)] + cmovae rax, r8 + cmovae rbx, r9 + cmovae rcx, r10 + cmovae rdx, r11 + cmovae rbp, r12 + cmovae rsi, r13 + mov qword [r15+sizeof(qword)*0], rax + mov qword [r15+sizeof(qword)*1], rbx + mov qword [r15+sizeof(qword)*2], rcx + mov qword [r15+sizeof(qword)*3], rdx + mov qword [r15+sizeof(qword)*4], rbp + mov qword [r15+sizeof(qword)*5], rsi + ret +ENDFUNC mred_6 + + +;; +;; 7*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_7,PRIVATE +%assign MSIZE 7 + push r8 ; m' + + ; load low half of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + + ; ui = x[i]*m' + ; reduction += ui*modulus + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred7_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred7_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred7_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred7_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred7_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred7_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred7_start + + pop rax ; remove m' + + ; finalize cf, reduction += U0*modulus operation + xor rax, rax + op_reg_mem add, r8, [rdi+sizeof(qword)*(MSIZE+0)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+0)], r8 + op_reg_mem adc, r9, [rdi+sizeof(qword)*(MSIZE+1)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+1)], r9 + op_reg_mem adc, r10,[rdi+sizeof(qword)*(MSIZE+2)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+2)], r10 + op_reg_mem adc, r11,[rdi+sizeof(qword)*(MSIZE+3)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+3)], r11 + op_reg_mem adc, r12,[rdi+sizeof(qword)*(MSIZE+4)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+4)], r12 + op_reg_mem adc, r13,[rdi+sizeof(qword)*(MSIZE+5)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+5)], r13 + op_reg_mem adc, r14,[rdi+sizeof(qword)*(MSIZE+6)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+6)], r14 + adc rax, 0 + + ; reduction -= modulus + op_reg_mem sub, r8, [rsi+sizeof(qword)*0], rbx + op_reg_mem sbb, r9, [rsi+sizeof(qword)*1], rbx + op_reg_mem sbb, r10,[rsi+sizeof(qword)*2], rbx + op_reg_mem sbb, r11,[rsi+sizeof(qword)*3], rbx + op_reg_mem sbb, r12,[rsi+sizeof(qword)*4], rbx + op_reg_mem sbb, r13,[rsi+sizeof(qword)*5], rbx + op_reg_mem sbb, r14,[rsi+sizeof(qword)*6], rbx + sbb rax, 0 + + ; copy under cf [r15] = cf? [r15] : {r8-r14} + mov rax, qword [rdi+sizeof(qword)*(MSIZE+0)] + mov rbx, qword [rdi+sizeof(qword)*(MSIZE+1)] + mov rcx, qword [rdi+sizeof(qword)*(MSIZE+2)] + mov rdx, qword [rdi+sizeof(qword)*(MSIZE+3)] + mov rbp, qword [rdi+sizeof(qword)*(MSIZE+4)] + mov rsi, qword [rdi+sizeof(qword)*(MSIZE+5)] + mov rdi, qword [rdi+sizeof(qword)*(MSIZE+6)] + cmovae rax, r8 + cmovae rbx, r9 + cmovae rcx, r10 + cmovae rdx, r11 + cmovae rbp, r12 + cmovae rsi, r13 + cmovae rdi, r14 + mov qword [r15+sizeof(qword)*0], rax + mov qword [r15+sizeof(qword)*1], rbx + mov qword [r15+sizeof(qword)*2], rcx + mov qword [r15+sizeof(qword)*3], rdx + mov qword [r15+sizeof(qword)*4], rbp + mov qword [r15+sizeof(qword)*5], rsi + mov qword [r15+sizeof(qword)*6], rdi + ret +ENDFUNC mred_7 + + +;; +;; 8*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_8,PRIVATE +%assign MSIZE 8 + push r15 ; save reduction address + push r8 ; m' + + ; load low half of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + ; ui = x[i]*m' + ; reduction += ui*modulus + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred8_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred8_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred8_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred8_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred8_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred8_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred8_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred8_start + + pop rax ; remove m' + + ; finalize cf, reduction += U0*modulus operation + xor rax, rax + op_reg_mem add, r8, [rdi+sizeof(qword)*(MSIZE+0)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+0)], r8 + op_reg_mem adc, r9, [rdi+sizeof(qword)*(MSIZE+1)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+1)], r9 + op_reg_mem adc, r10,[rdi+sizeof(qword)*(MSIZE+2)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+2)], r10 + op_reg_mem adc, r11,[rdi+sizeof(qword)*(MSIZE+3)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+3)], r11 + op_reg_mem adc, r12,[rdi+sizeof(qword)*(MSIZE+4)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+4)], r12 + op_reg_mem adc, r13,[rdi+sizeof(qword)*(MSIZE+5)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+5)], r13 + op_reg_mem adc, r14,[rdi+sizeof(qword)*(MSIZE+6)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+6)], r14 + op_reg_mem adc, r15,[rdi+sizeof(qword)*(MSIZE+7)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+7)], r15 + adc rax, 0 + + ; reduction -= modulus + op_reg_mem sub, r8, [rsi+sizeof(qword)*0], rbx + op_reg_mem sbb, r9, [rsi+sizeof(qword)*1], rbx + op_reg_mem sbb, r10,[rsi+sizeof(qword)*2], rbx + op_reg_mem sbb, r11,[rsi+sizeof(qword)*3], rbx + op_reg_mem sbb, r12,[rsi+sizeof(qword)*4], rbx + op_reg_mem sbb, r13,[rsi+sizeof(qword)*5], rbx + op_reg_mem sbb, r14,[rsi+sizeof(qword)*6], rbx + op_reg_mem sbb, r15,[rsi+sizeof(qword)*7], rbx + sbb rax, 0 + + pop rsi ; address of reduction + ; copy under cf [rsi] = cf? [rdi] : {r8-r15} + mov rax, qword [rdi+sizeof(qword)*(MSIZE+0)] + mov rbx, qword [rdi+sizeof(qword)*(MSIZE+1)] + mov rcx, qword [rdi+sizeof(qword)*(MSIZE+2)] + mov rdx, qword [rdi+sizeof(qword)*(MSIZE+3)] + cmovae rax, r8 + cmovae rbx, r9 + cmovae rcx, r10 + cmovae rdx, r11 + mov qword [rsi+sizeof(qword)*0], rax + mov qword [rsi+sizeof(qword)*1], rbx + mov qword [rsi+sizeof(qword)*2], rcx + mov qword [rsi+sizeof(qword)*3], rdx + + mov rax, qword [rdi+sizeof(qword)*(MSIZE+4)] + mov rbx, qword [rdi+sizeof(qword)*(MSIZE+5)] + mov rcx, qword [rdi+sizeof(qword)*(MSIZE+6)] + mov rdx, qword [rdi+sizeof(qword)*(MSIZE+7)] + cmovae rax, r12 + cmovae rbx, r13 + cmovae rcx, r14 + cmovae rdx, r15 + mov qword [rsi+sizeof(qword)*4], rax + mov qword [rsi+sizeof(qword)*5], rbx + mov qword [rsi+sizeof(qword)*6], rcx + mov qword [rsi+sizeof(qword)*7], rdx + ret +ENDFUNC mred_8 + + +;; +;; 9*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_9,PRIVATE +%assign MSIZE 9 + push r15 ; save reduction address + + sub rsp, sizeof(qword)*8 ; allocate U space + mov rcx, rsp + + push r8 ; save m' + mov rdx, r8 + +;; +;; init pass +;; + + ; load low part of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + call mred8x8_start + + xor rax, rax ; init carryLCL + op_reg_mem add, r8, [rdi+sizeof(qword)*(8+0)], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(8+1)], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(8+2)], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(8+3)], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(8+4)], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(8+5)], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*(8+6)], rbx + op_reg_mem adc, r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + SWAP rcx, rsi + call mla_8x1 + SWAP rcx, rsi + + pop rax + shr rax, 1 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], r13 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], r14 + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], r15, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], rbx + adc rax, 0 + push rax ; store carryGBL + +;; +;; last pass +;; + sub rsi, sizeof(qword)*8 + + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + mov rdx, [rsp+sizeof(qword)] + call mred8x1_start + + xor rax, rax ; init carryLCL + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], r13 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], r14 + mov r8, r15 + add r8, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7] + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + call mla_1x1 + + pop rax ; restore carryLCL + shr rax, 1 + op_reg_mem adc, r8, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], rbx + adc rax, 0 + + pop rbx ; carryGBL + add r8, rbx + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + adc rax, 0 ; update carryGBL + + + pop rcx ; remove m' + add rsp, sizeof(qword)*8 ; release U space + + lea rcx, [rsi-sizeof(qword)*8] ; restore modulus + lea rsi, [rdi+sizeof(qword)*(MSIZE-8)-sizeof(qword)*8] ; restore buffer + pop rdi ; restore reduction + + mov rbx, rax ; save carryGBL + + mov rdx, dword MSIZE + call sub_N ; reduction = buffer - modulus + sub rbx, rax ; rbx = borrow + + sub rdi, sizeof(qword)*MSIZE ; reduction + sub rsi, sizeof(qword)*MSIZE ; buffer (src1) + mov rcx, rdi ; reduction (src2) + mov rdx, dword MSIZE + shr rbx,1 ; restore cf + call copy_ae_N ; copy under cf, reduction = cf? buffer : reduction + + ret +ENDFUNC mred_9 + + +;; +;; 10*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_10,PRIVATE +%assign MSIZE 10 + push r15 ; save reduction address + + sub rsp, sizeof(qword)*8 ; allocate U space + mov rcx, rsp + + push r8 ; save m' + mov rdx, r8 + +;; +;; init pass +;; + + ; load low part of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + call mred8x8_start + + xor rax, rax ; init carryLCL + op_reg_mem add, r8, [rdi+sizeof(qword)*(8+0)], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(8+1)], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(8+2)], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(8+3)], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(8+4)], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(8+5)], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*(8+6)], rbx + op_reg_mem adc, r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + SWAP rcx, rsi + call mla_8x2 + SWAP rcx, rsi + + pop rax + shr rax, 1 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], r13 + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], r14, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], r15, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], rbx + adc rax, 0 + push rax ; store carryGBL + +;; +;; last pass +;; + sub rsi, sizeof(qword)*8 + + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + mov rdx, [rsp+sizeof(qword)] + call mred8x2_start + + xor rax, rax ; init carryLCL + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], r13 + mov r8, r14 + mov r9, r15 + add r8, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6] + adc r9, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7] + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + call mla_2x2 + + pop rax ; restore carryLCL + shr rax, 1 + op_reg_mem adc, r8, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], rbx + adc rax, 0 + + pop rbx ; carryGBL + add r8, rbx + adc r9, 0 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + adc rax, 0 ; update carryGBL + + + pop rcx ; remove m' + add rsp, sizeof(qword)*8 ; release U space + + lea rcx, [rsi-sizeof(qword)*8] ; restore modulus + lea rsi, [rdi+sizeof(qword)*(MSIZE-8)-sizeof(qword)*8] ; restore buffer + pop rdi ; restore reduction + + mov rbx, rax ; save carryGBL + + mov rdx, dword MSIZE + call sub_N ; reduction = buffer - modulus + sub rbx, rax ; rbx = borrow + + sub rdi, sizeof(qword)*MSIZE ; reduction + sub rsi, sizeof(qword)*MSIZE ; buffer (src1) + mov rcx, rdi ; reduction (src2) + mov rdx, dword MSIZE + shr rbx,1 ; restore cf + call copy_ae_N ; copy under cf, reduction = cf? buffer : reduction + + ret +ENDFUNC mred_10 + + +;; +;; 11*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_11,PRIVATE +%assign MSIZE 11 + push r15 ; save reduction address + + sub rsp, sizeof(qword)*8 ; allocate U space + mov rcx, rsp + + push r8 ; save m' + mov rdx, r8 + +;; +;; init pass +;; + + ; load low part of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + call mred8x8_start + + xor rax, rax ; init carryLCL + op_reg_mem add, r8, [rdi+sizeof(qword)*(8+0)], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(8+1)], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(8+2)], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(8+3)], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(8+4)], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(8+5)], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*(8+6)], rbx + op_reg_mem adc, r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + SWAP rcx, rsi + call mla_8x3 + SWAP rcx, rsi + + pop rax + shr rax, 1 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12 + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], r13, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], r14, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], r15, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], rbx + adc rax, 0 + push rax ; store carryGBL + +;; +;; last pass +;; + sub rsi, sizeof(qword)*8 + + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + mov rdx, [rsp+sizeof(qword)] + call mred8x3_start + + xor rax, rax ; init carryLCL + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12 + mov r8, r13 + mov r9, r14 + mov r10,r15 + add r8, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5] + adc r9, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6] + adc r10,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7] + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + call mla_3x3 + + pop rax ; restore carryLCL + shr rax, 1 + op_reg_mem adc, r8, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], rbx + adc rax, 0 + + pop rbx ; carryGBL + add r8, rbx + adc r9, 0 + adc r10,0 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + adc rax, 0 ; update carryGBL + + + pop rcx ; remove m' + add rsp, sizeof(qword)*8 ; release U space + + lea rcx, [rsi-sizeof(qword)*8] ; restore modulus + lea rsi, [rdi+sizeof(qword)*(MSIZE-8)-sizeof(qword)*8] ; restore buffer + pop rdi ; restore reduction + + mov rbx, rax ; save carryGBL + + mov rdx, dword MSIZE + call sub_N ; reduction = buffer - modulus + sub rbx, rax ; rbx = borrow + + sub rdi, sizeof(qword)*MSIZE ; reduction + sub rsi, sizeof(qword)*MSIZE ; buffer (src1) + mov rcx, rdi ; reduction (src2) + mov rdx, dword MSIZE + shr rbx,1 ; restore cf + call copy_ae_N ; copy under cf, reduction = cf? buffer : reduction + + ret +ENDFUNC mred_11 + + +;; +;; 12*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_12,PRIVATE +%assign MSIZE 12 + push r15 ; save reduction address + + sub rsp, sizeof(qword)*8 ; allocate U space + mov rcx, rsp + + push r8 ; save m' + mov rdx, r8 + +;; +;; init pass +;; + + ; load low part of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + call mred8x8_start + + xor rax, rax ; init carryLCL + op_reg_mem add, r8, [rdi+sizeof(qword)*(8+0)], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(8+1)], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(8+2)], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(8+3)], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(8+4)], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(8+5)], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*(8+6)], rbx + op_reg_mem adc, r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + SWAP rcx, rsi + call mla_8x4 + SWAP rcx, rsi + + pop rax + shr rax, 1 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], r13, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], r14, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], r15, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], rbx + adc rax, 0 + push rax ; store carryGBL + +;; +;; last pass +;; + sub rsi, sizeof(qword)*8 + + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + mov rdx, [rsp+sizeof(qword)] + call mred8x4_start + + xor rax, rax ; init carryLCL + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + mov r8, r12 + mov r9, r13 + mov r10,r14 + mov r11,r15 + add r8, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4] + adc r9, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5] + adc r10,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6] + adc r11,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7] + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + call mla_4x4 + + pop rax ; restore carryLCL + shr rax, 1 + op_reg_mem adc, r8, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], rbx + adc rax, 0 + + pop rbx ; carryGBL + add r8, rbx + adc r9, 0 + adc r10,0 + adc r11,0 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + adc rax, 0 ; update carryGBL + + + pop rcx ; remove m' + add rsp, sizeof(qword)*8 ; release U space + + lea rcx, [rsi-sizeof(qword)*8] ; restore modulus + lea rsi, [rdi+sizeof(qword)*(MSIZE-8)-sizeof(qword)*8] ; restore buffer + pop rdi ; restore reduction + + mov rbx, rax ; save carryGBL + + mov rdx, dword MSIZE + call sub_N ; reduction = buffer - modulus + sub rbx, rax ; rbx = borrow + + sub rdi, sizeof(qword)*MSIZE ; reduction + sub rsi, sizeof(qword)*MSIZE ; buffer (src1) + mov rcx, rdi ; reduction (src2) + mov rdx, dword MSIZE + shr rbx,1 ; restore cf + call copy_ae_N ; copy under cf, reduction = cf? buffer : reduction + + ret +ENDFUNC mred_12 + + +;; +;; 13*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_13,PRIVATE +%assign MSIZE 13 + push r15 ; save reduction address + + sub rsp, sizeof(qword)*8 ; allocate U space + mov rcx, rsp + + push r8 ; save m' + mov rdx, r8 + +;; +;; init pass +;; + + ; load low part of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + call mred8x8_start + + xor rax, rax ; init carryLCL + op_reg_mem add, r8, [rdi+sizeof(qword)*(8+0)], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(8+1)], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(8+2)], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(8+3)], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(8+4)], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(8+5)], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*(8+6)], rbx + op_reg_mem adc, r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + SWAP rcx, rsi + call mla_8x5 + SWAP rcx, rsi + + pop rax + shr rax, 1 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], r13, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], r14, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], r15, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], rbx + adc rax, 0 + push rax ; store carryGBL + +;; +;; last pass +;; + sub rsi, sizeof(qword)*8 + + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + mov rdx, [rsp+sizeof(qword)] + call mred8x5_start + + xor rax, rax ; init carryLCL + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov r8, r11 + mov r9, r12 + mov r10,r13 + mov r11,r14 + mov r12,r15 + add r8, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3] + adc r9, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4] + adc r10,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5] + adc r11,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6] + adc r12,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7] + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + call mla_5x5 + + pop rax ; restore carryLCL + shr rax, 1 + op_reg_mem adc, r8, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], rbx + adc rax, 0 + + pop rbx ; carryGBL + add r8, rbx + adc r9, 0 + adc r10,0 + adc r11,0 + adc r12,0 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12 + adc rax, 0 ; update carryGBL + + + pop rcx ; remove m' + add rsp, sizeof(qword)*8 ; release U space + + lea rcx, [rsi-sizeof(qword)*8] ; restore modulus + lea rsi, [rdi+sizeof(qword)*(MSIZE-8)-sizeof(qword)*8] ; restore buffer + pop rdi ; restore reduction + + mov rbx, rax ; save carryGBL + + mov rdx, dword MSIZE + call sub_N ; reduction = buffer - modulus + sub rbx, rax ; rbx = borrow + + sub rdi, sizeof(qword)*MSIZE ; reduction + sub rsi, sizeof(qword)*MSIZE ; buffer (src1) + mov rcx, rdi ; reduction (src2) + mov rdx, dword MSIZE + shr rbx,1 ; restore cf + call copy_ae_N ; copy under cf, reduction = cf? buffer : reduction + + ret +ENDFUNC mred_13 + + +;; +;; 14*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_14,PRIVATE +%assign MSIZE 14 + push r15 ; save reduction address + + sub rsp, sizeof(qword)*8 ; allocate U space + mov rcx, rsp + + push r8 ; save m' + mov rdx, r8 + +;; +;; init pass +;; + + ; load low part of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + call mred8x8_start + + xor rax, rax ; init carryLCL + op_reg_mem add, r8, [rdi+sizeof(qword)*(8+0)], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(8+1)], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(8+2)], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(8+3)], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(8+4)], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(8+5)], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*(8+6)], rbx + op_reg_mem adc, r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + SWAP rcx, rsi + call mla_8x6 + SWAP rcx, rsi + + pop rax + shr rax, 1 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], r13, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], r14, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], r15, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], rbx + adc rax, 0 + push rax ; store carryGBL + +;; +;; last pass +;; + sub rsi, sizeof(qword)*8 + + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + mov rdx, [rsp+sizeof(qword)] + call mred8x6_start + + xor rax, rax ; init carryLCL + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov r8, r10 + mov r9, r11 + mov r10,r12 + mov r11,r13 + mov r12,r14 + mov r13,r15 + add r8, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2] + adc r9, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3] + adc r10,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4] + adc r11,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5] + adc r12,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6] + adc r13,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7] + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + call mla_6x6 + + pop rax ; restore carryLCL + shr rax, 1 + op_reg_mem adc, r8, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], rbx + adc rax, 0 + + pop rbx ; carryGBL + add r8, rbx + adc r9, 0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], r13 + adc rax, 0 ; update carryGBL + + + pop rcx ; remove m' + add rsp, sizeof(qword)*8 ; release U space + + lea rcx, [rsi-sizeof(qword)*8] ; restore modulus + lea rsi, [rdi+sizeof(qword)*(MSIZE-8)-sizeof(qword)*8] ; restore buffer + pop rdi ; restore reduction + + mov rbx, rax ; save carryGBL + + mov rdx, dword MSIZE + call sub_N ; reduction = buffer - modulus + sub rbx, rax ; rbx = borrow + + sub rdi, sizeof(qword)*MSIZE ; reduction + sub rsi, sizeof(qword)*MSIZE ; buffer (src1) + mov rcx, rdi ; reduction (src2) + mov rdx, dword MSIZE + shr rbx,1 ; restore cf + call copy_ae_N ; copy under cf, reduction = cf? buffer : reduction + + ret +ENDFUNC mred_14 + + +;; +;; 15*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_15,PRIVATE +%assign MSIZE 15 + push r15 ; save reduction address + + sub rsp, sizeof(qword)*8 ; allocate U space + mov rcx, rsp + + push r8 ; save m' + mov rdx, r8 + +;; +;; init pass +;; + + ; load low part of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + call mred8x8_start + + xor rax, rax ; init carryLCL + op_reg_mem add, r8, [rdi+sizeof(qword)*(8+0)], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(8+1)], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(8+2)], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(8+3)], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(8+4)], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(8+5)], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*(8+6)], rbx + op_reg_mem adc, r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + SWAP rcx, rsi + call mla_8x7 + SWAP rcx, rsi + + pop rax + shr rax, 1 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], r13, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], r14, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], r15, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], rbx + adc rax, 0 + push rax ; store carryGBL + +;; +;; last pass +;; + sub rsi, sizeof(qword)*8 + + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + mov rdx, [rsp+sizeof(qword)] + call mred8x7_start + + xor rax, rax ; init carryLCL + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov r8, r9 + mov r9, r10 + mov r10,r11 + mov r11,r12 + mov r12,r13 + mov r13,r14 + mov r14,r15 + add r8, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1] + adc r9, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2] + adc r10,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3] + adc r11,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4] + adc r12,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5] + adc r13,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6] + adc r14,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7] + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + call mla_7x7 + + pop rax ; restore carryLCL + shr rax, 1 + op_reg_mem adc, r8, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], rbx + adc rax, 0 + + pop rbx ; carryGBL + add r8, rbx + adc r9, 0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + adc r14,0 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], r13 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], r14 + adc rax, 0 ; update carryGBL + + + pop rcx ; remove m' + add rsp, sizeof(qword)*8 ; release U space + + lea rcx, [rsi-sizeof(qword)*8] ; restore modulus + lea rsi, [rdi+sizeof(qword)*(MSIZE-8)-sizeof(qword)*8] ; restore buffer + pop rdi ; restore reduction + + mov rbx, rax ; save carryGBL + + mov rdx, dword MSIZE + call sub_N ; reduction = buffer - modulus + sub rbx, rax ; rbx = borrow + + sub rdi, sizeof(qword)*MSIZE ; reduction + sub rsi, sizeof(qword)*MSIZE ; buffer (src1) + mov rcx, rdi ; reduction (src2) + mov rdx, dword MSIZE + shr rbx,1 ; restore cf + call copy_ae_N ; copy under cf, reduction = cf? buffer : reduction + + ret +ENDFUNC mred_15 + + +;; +;; 16*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_16,PRIVATE +%assign MSIZE 16 + push r15 ; save reduction address + + sub rsp, sizeof(qword)*8 ; allocate U space + mov rcx, rsp + + mov rdx, r8 ; copy m' + + ; load low part of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + call mred8x8_start + + xor rax, rax + op_reg_mem add, r8, [rdi+sizeof(qword)*(8+0)], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(8+1)], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(8+2)], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(8+3)], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(8+4)], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(8+5)], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*(8+6)], rbx + op_reg_mem adc, r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax + + add rsi, sizeof(qword)*8 + add rdi, sizeof(qword)*8 + + push rdx + call mla_8x8 + pop rdx + + pop rax + shr rax, 1 + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+0)], r8, [rdi+sizeof(qword)*(8+0)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+1)], r9, [rdi+sizeof(qword)*(8+1)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+2)], r10,[rdi+sizeof(qword)*(8+2)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+3)], r11,[rdi+sizeof(qword)*(8+3)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+4)], r12,[rdi+sizeof(qword)*(8+4)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+5)], r13,[rdi+sizeof(qword)*(8+5)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+6)], r14,[rdi+sizeof(qword)*(8+6)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+7)], r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax + + sub rsi, sizeof(qword)*8 + + mov r8, qword [rdi] + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + call mred8x8_start + + xor rax, rax + op_reg_mem add, r8, [rdi+sizeof(qword)*(8+0)], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(8+1)], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(8+2)], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(8+3)], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(8+4)], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(8+5)], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*(8+6)], rbx + op_reg_mem adc, r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + call mla_8x8 + + sub rsi, sizeof(qword)*8 + + pop rax + shr rax, 1 + op_reg_mem adc, r8, [rdi+sizeof(qword)*(8+0)], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(8+1)], rbx + op_reg_mem adc, r10, [rdi+sizeof(qword)*(8+2)], rbx + op_reg_mem adc, r11, [rdi+sizeof(qword)*(8+3)], rbx + op_reg_mem adc, r12, [rdi+sizeof(qword)*(8+4)], rbx + op_reg_mem adc, r13, [rdi+sizeof(qword)*(8+5)], rbx + op_reg_mem adc, r14, [rdi+sizeof(qword)*(8+6)], rbx + op_reg_mem adc, r15, [rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + + pop rbx + add r8, rbx + adc r9, 0 + adc r10, 0 + adc r11, 0 + adc r12, 0 + adc r13, 0 + adc r14, 0 + adc r15, 0 + adc rax, 0 + + mov qword [rdi+sizeof(qword)*(8+0)], r8 + mov qword [rdi+sizeof(qword)*(8+1)], r9 + mov qword [rdi+sizeof(qword)*(8+2)], r10 + mov qword [rdi+sizeof(qword)*(8+3)], r11 + mov qword [rdi+sizeof(qword)*(8+4)], r12 + mov qword [rdi+sizeof(qword)*(8+5)], r13 + mov qword [rdi+sizeof(qword)*(8+6)], r14 + mov qword [rdi+sizeof(qword)*(8+7)], r15 + + add rsp, sizeof(qword)*8 ; release U space + + pop rbp + + ; reduction -= modulus + op_mem_mem sub, [rbp+sizeof(qword)*0 ], [rdi+sizeof(qword)*0 ], [rsi+sizeof(qword)*0 ], rbx + op_mem_mem sbb, [rbp+sizeof(qword)*1 ], [rdi+sizeof(qword)*1 ], [rsi+sizeof(qword)*1 ], rbx + op_mem_mem sbb, [rbp+sizeof(qword)*2 ], [rdi+sizeof(qword)*2 ], [rsi+sizeof(qword)*2 ], rbx + op_mem_mem sbb, [rbp+sizeof(qword)*3 ], [rdi+sizeof(qword)*3 ], [rsi+sizeof(qword)*3 ], rbx + op_mem_mem sbb, [rbp+sizeof(qword)*4 ], [rdi+sizeof(qword)*4 ], [rsi+sizeof(qword)*4 ], rbx + op_mem_mem sbb, [rbp+sizeof(qword)*5 ], [rdi+sizeof(qword)*5 ], [rsi+sizeof(qword)*5 ], rbx + op_mem_mem sbb, [rbp+sizeof(qword)*6 ], [rdi+sizeof(qword)*6 ], [rsi+sizeof(qword)*6 ], rbx + op_mem_mem sbb, [rbp+sizeof(qword)*7 ], [rdi+sizeof(qword)*7 ], [rsi+sizeof(qword)*7 ], rbx + op_reg_mem sbb, r8, [rsi+sizeof(qword)*8 ], rbx + op_reg_mem sbb, r9, [rsi+sizeof(qword)*9 ], rbx + op_reg_mem sbb, r10, [rsi+sizeof(qword)*10], rbx + op_reg_mem sbb, r11, [rsi+sizeof(qword)*11], rbx + op_reg_mem sbb, r12, [rsi+sizeof(qword)*12], rbx + op_reg_mem sbb, r13, [rsi+sizeof(qword)*13], rbx + op_reg_mem sbb, r14, [rsi+sizeof(qword)*14], rbx + op_reg_mem sbb, r15, [rsi+sizeof(qword)*15], rbx + sbb rax, 0 + + ; copy under cf [rbp] = cf? [rdi] : [rbp] + mov rax, qword [rdi+sizeof(qword)*8 ] + mov rbx, qword [rdi+sizeof(qword)*9 ] + mov rcx, qword [rdi+sizeof(qword)*10] + mov rdx, qword [rdi+sizeof(qword)*11] + cmovae rax, r8 + cmovae rbx, r9 + cmovae rcx, r10 + cmovae rdx, r11 + mov qword [rbp+sizeof(qword)*8 ], rax + mov qword [rbp+sizeof(qword)*9 ], rbx + mov qword [rbp+sizeof(qword)*10], rcx + mov qword [rbp+sizeof(qword)*11], rdx + + mov rax, qword [rdi+sizeof(qword)*12] + mov rbx, qword [rdi+sizeof(qword)*13] + mov rcx, qword [rdi+sizeof(qword)*14] + mov rdx, qword [rdi+sizeof(qword)*15] + cmovae rax, r12 + cmovae rbx, r13 + cmovae rcx, r14 + cmovae rdx, r15 + mov qword [rbp+sizeof(qword)*12], rax + mov qword [rbp+sizeof(qword)*13], rbx + mov qword [rbp+sizeof(qword)*14], rcx + mov qword [rbp+sizeof(qword)*15], rdx + + mov r8, qword [rbp+sizeof(qword)*0] + mov r9, qword [rbp+sizeof(qword)*1] + mov r10, qword [rbp+sizeof(qword)*2] + mov r11, qword [rbp+sizeof(qword)*3] + mov r12, qword [rbp+sizeof(qword)*4] + mov r13, qword [rbp+sizeof(qword)*5] + mov r14, qword [rbp+sizeof(qword)*6] + mov r15, qword [rbp+sizeof(qword)*7] + + mov rax, qword [rdi+sizeof(qword)*0] + mov rbx, qword [rdi+sizeof(qword)*1] + mov rcx, qword [rdi+sizeof(qword)*2] + mov rdx, qword [rdi+sizeof(qword)*3] + cmovae rax, r8 + cmovae rbx, r9 + cmovae rcx, r10 + cmovae rdx, r11 + mov qword [rbp+sizeof(qword)*0], rax + mov qword [rbp+sizeof(qword)*1], rbx + mov qword [rbp+sizeof(qword)*2], rcx + mov qword [rbp+sizeof(qword)*3], rdx + + mov rax, qword [rdi+sizeof(qword)*4] + mov rbx, qword [rdi+sizeof(qword)*5] + mov rcx, qword [rdi+sizeof(qword)*6] + mov rdx, qword [rdi+sizeof(qword)*7] + cmovae rax, r12 + cmovae rbx, r13 + cmovae rcx, r14 + cmovae rdx, r15 + mov qword [rbp+sizeof(qword)*4], rax + mov qword [rbp+sizeof(qword)*5], rbx + mov qword [rbp+sizeof(qword)*6], rcx + mov qword [rbp+sizeof(qword)*7], rdx + + ret +ENDFUNC mred_16 + + + +mred_short DQ mred_5 - mred_short + DQ mred_6 - mred_short + DQ mred_7 - mred_short + DQ mred_8 - mred_short + DQ mred_9 - mred_short + DQ mred_10 - mred_short + DQ mred_11 - mred_short + DQ mred_12 - mred_short + DQ mred_13 - mred_short + DQ mred_14 - mred_short + DQ mred_15 - mred_short + DQ mred_16 - mred_short + +mred8x_start DQ mred8x1_start - mred8x_start + DQ mred8x2_start - mred8x_start + DQ mred8x3_start - mred8x_start + DQ mred8x4_start - mred8x_start + DQ mred8x5_start - mred8x_start + DQ mred8x6_start - mred8x_start + DQ mred8x7_start - mred8x_start diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmredpp.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmredpp.inc new file mode 100644 index 0000000..4a2bf7b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmredpp.inc @@ -0,0 +1,571 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Low level Big Number reduction Support +; +; + +%ifndef _PCPMRED_INC_ +%assign _PCPMRED_INC_ 1 + +%include "pcpmredpp_basic.inc" + +; +; r15 - reduction +; rdi - buffer +; rsi - modulus +; rdx - modulus legth +; r8 - m' +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_8N_adcox,PRIVATE + push r15 ; save reduction address + + sub rsp, sizeof(qword)*8 ; allocate U space + mov rcx, rsp + + mov rbx, rdx ; number of passes == modulus length + + xor rax, rax ; init global carry + +.passLoop: + ; save: + push rdi ; - buffer address + push rsi ; - modulus address + push rdx ; - modulus length + push r8 ; - m' + push rbx ; - rest of passes + push rax ; - global carry + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ; + ; signgle pass + ; + push rdx ; init modulus counter + mov rdx, r8 ; copy m' + + ; load low part of the product into r8,r9,r10,r11,r12,r13,r14,r15 + mov r8, qword [rdi] + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + call mred8x8_start + pop rdx ; restore modulus counter + + xor rax, rax + op_reg_mem add, r8, [rdi+sizeof(qword)*(8+0)], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(8+1)], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(8+2)], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(8+3)], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(8+4)], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(8+5)], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*(8+6)], rbx + op_reg_mem adc, r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax + + jmp .entryInnerLoop + +.innerLoop: + push rdx ; save modulus counter + call mla_8x8 + pop rdx ; restore modulus counter + + pop rax + shr rax, 1 + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+0)], r8, [rdi+sizeof(qword)*(8+0)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+1)], r9, [rdi+sizeof(qword)*(8+1)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+2)], r10,[rdi+sizeof(qword)*(8+2)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+3)], r11,[rdi+sizeof(qword)*(8+3)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+4)], r12,[rdi+sizeof(qword)*(8+4)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+5)], r13,[rdi+sizeof(qword)*(8+5)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+6)], r14,[rdi+sizeof(qword)*(8+6)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+7)], r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax + +.entryInnerLoop: + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + sub rdx, 8 + jg .innerLoop + + ; single pass completion + pop rax ; new global carry + + pop rbx ; prev global carry + add r8, rbx + adc r9, 0 + adc r10, 0 + adc r11, 0 + adc r12, 0 + adc r13, 0 + adc r14, 0 + adc r15, 0 + adc rax, 0 ; update global carry + + mov qword [rdi+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*3], r11 + mov qword [rdi+sizeof(qword)*4], r12 + mov qword [rdi+sizeof(qword)*5], r13 + mov qword [rdi+sizeof(qword)*6], r14 + mov qword [rdi+sizeof(qword)*7], r15 + ; + ; end of single pass + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + ; restore: + pop rbx ; - rest of passes + pop r8 ; - m' + pop rdx ; - modulus length + pop rsi ; - modulus address + pop rdi ; - buffer address + add rdi, sizeof(qword)*8 + sub rbx, 8 + jg .passLoop + + add rsp, sizeof(qword)*8 ; release U space + + mov r14, rdx ; save modulus length + lea r15, [rdx*sizeof(qword)]; save modulus length (bytes) + + mov rbx, rax ; global carry + mov rcx, rsi ; modulus + mov rsi, rdi ; buffer + pop rdi ; reduction address + + call sub_N ; reduction = buffer - modulus + sub rbx, rax ; rbx = borrow + + mov rdx, r14 ; length + sub rdi, r15 ; reduction + sub rsi, r15 ; buffer (src1) + mov rcx, rdi ; reduction (src2) + shr rbx,1 ; restore cf + + call copy_ae_N ; copy under cf, reduction = cf? buffer : reduction + + ret +ENDFUNC mred_8N_adcox + + +; +; r15 - reduction +; rdi - buffer +; rsi - modulus +; rdx - modulus legth +; r8 - m' +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_N_adcox,PRIVATE + push r15 ; save reduction address + + sub rsp, sizeof(qword)*8 ; allocate U space + mov rcx, rsp + + mov rbx, rdx ; number of passes counter == (modulus_length -8) + sub rbx, 8 + + xor rax, rax ; init carryGBL + + mov r15, dword 7 ; n = modulus_len%8 + and r15, rdx + GET_EP rbp, mla_8xl_tail, r15 ; tail procedure (mla_8xn) address + +.passLoop: + ; save: + push rdi ; - buffer address + push rsi ; - modulus address + push rdx ; - modulus length + push r8 ; - m' + push rbx ; - rest of passes + push rax ; - carryGBL + push rbp ; - mla_8xn procedure + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; regular pass +;; + sub rdx, 8 ; init modulus counter + push rdx + mov rdx, r8 ; copy m' + + ; load low part of the product into r8,r9,r10,r11,r12,r13,r14,r15 + mov r8, qword [rdi] + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + call mred8x8_start + pop rdx ; restore modulus counter + + xor rax, rax ; init carryLCL + push rax + + jmp .entryInnerLoop + +.innerLoop: + push rdx ; save modulus counter + call mla_8x8 + pop rdx ; restore modulus counter + +.entryInnerLoop: + pop rax ; restore carryLCL + shr rax, 1 + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+0)], r8, [rdi+sizeof(qword)*(8+0)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+1)], r9, [rdi+sizeof(qword)*(8+1)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+2)], r10,[rdi+sizeof(qword)*(8+2)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+3)], r11,[rdi+sizeof(qword)*(8+3)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+4)], r12,[rdi+sizeof(qword)*(8+4)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+5)], r13,[rdi+sizeof(qword)*(8+5)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+6)], r14,[rdi+sizeof(qword)*(8+6)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+7)], r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax ; update store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + sub rdx, 8 + jnc .innerLoop + + add rdx, 8 + jz .complete_regular_pass + + ; tail of single pass + mov rax, [rsp+sizeof(qword)] ; procedure address + + SWAP rcx, rsi + push rdx + call rax + pop rdx + SWAP rcx, rsi + + lea rdi, [rdi+rdx*sizeof(qword)] + pop rax ; restore carryLCL + shr rax, 1 + mov rbx, rdx + + dec rbx + jz .mt_1 + dec rbx + jz .mt_2 + dec rbx + jz .mt_3 + dec rbx + jz .mt_4 + dec rbx + jz .mt_5 + dec rbx + jz .mt_6 +.mt_7: op_reg_mem adc, r9, [rdi+sizeof(qword)*1], rbx +.mt_6: op_reg_mem adc, r10,[rdi+sizeof(qword)*2], rbx +.mt_5: op_reg_mem adc, r11,[rdi+sizeof(qword)*3], rbx +.mt_4: op_reg_mem adc, r12,[rdi+sizeof(qword)*4], rbx +.mt_3: op_reg_mem adc, r13,[rdi+sizeof(qword)*5], rbx +.mt_2: op_reg_mem adc, r14,[rdi+sizeof(qword)*6], rbx +.mt_1: op_reg_mem adc, r15,[rdi+sizeof(qword)*7], rbx + + adc rax, 0 ; update carryLCL + push rax + + ; single pass completion +.complete_regular_pass: + pop rax ; carryLCL + pop rbp ; mla_8xn procedure + + pop rbx ; carryGBL + add r8, rbx + adc r9, 0 + adc r10, 0 + adc r11, 0 + adc r12, 0 + adc r13, 0 + adc r14, 0 + adc r15, 0 + adc rax, 0 ; update carryGBL + + mov qword [rdi+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*3], r11 + mov qword [rdi+sizeof(qword)*4], r12 + mov qword [rdi+sizeof(qword)*5], r13 + mov qword [rdi+sizeof(qword)*6], r14 + mov qword [rdi+sizeof(qword)*7], r15 +;; +;; end of regular pass +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + ; restore: + pop rbx ; - rest of passes + pop r8 ; - m' + pop rdx ; - modulus length + pop rsi ; - modulus address + pop rdi ; - buffer address + add rdi, sizeof(qword)*8 + sub rbx, 8 + jnc .passLoop + + add rbx, 8 + jz .complete_reduction + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; tail pass of reduction +;; + ; save: + push rdi ; - buffer address + push rsi ; - modulus address + push rdx ; - modulus length + push r8 ; - m' + push rbx ; - rest of passes + push rax ; - carry + push rbp ; - mla_8xn procedure + + sub rdx, 8 ; init modulus counter + push rdx + mov rdx, r8 ; copy m' + + ; load low part of the product into r8,r9,r10,r11,r12,r13,r14,r15 + mov r8, qword [rdi] + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + GET_EP rax, mred8x_start, rbx, rbp ; procedure mred8xn_start address + + call rax + pop rdx ; restore modulus counter + + xor rax, rax ; init carryCLC + push rax + + jmp .entryTailLoop + +.tailLoop: + mov r8, qword [rdi] + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + mov rax, [rsp+sizeof(qword)] ; procedure address + push rdx ; save modulus counter + call rax + pop rdx ; restore modulus counter + +.entryTailLoop: + pop rax ; restore carryLCL + shr rax, 1 + adc r8, 0 + adc r9, 0 + adc r10, 0 + adc r11, 0 + adc r12, 0 + adc r13, 0 + adc r14, 0 + adc r15, 0 + adc rax, 0 ; update carryGBL + + mov rbx, [rsp+sizeof(qword)*2] ; modulus_len%7 + cmp rbx, 1 + jz .tt_1 + cmp rbx, 2 + jz .tt_2 + cmp rbx, 3 + jz .tt_3 + cmp rbx, 4 + jz .tt_4 + cmp rbx, 5 + jz .tt_5 + cmp rbx, 6 + jz .tt_6 +.tt_7: op_reg_mem adc, r9, [rdi+rbx*sizeof(qword)+sizeof(qword)*1], rbp +.tt_6: op_reg_mem adc, r10,[rdi+rbx*sizeof(qword)+sizeof(qword)*2], rbp +.tt_5: op_reg_mem adc, r11,[rdi+rbx*sizeof(qword)+sizeof(qword)*3], rbp +.tt_4: op_reg_mem adc, r12,[rdi+rbx*sizeof(qword)+sizeof(qword)*4], rbp +.tt_3: op_reg_mem adc, r13,[rdi+rbx*sizeof(qword)+sizeof(qword)*5], rbp +.tt_2: op_reg_mem adc, r14,[rdi+rbx*sizeof(qword)+sizeof(qword)*6], rbp +.tt_1: op_reg_mem adc, r15,[rdi+rbx*sizeof(qword)+sizeof(qword)*7], rbp + + adc rax, 0 + push rax ; update store carryLCL + + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*0], r8 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*1], r9 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*2], r10 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*3], r11 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*4], r12 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*5], r13 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*6], r14 + mov qword [rdi+rbx*sizeof(qword)+sizeof(qword)*7], r15 + + add rsi, sizeof(qword)*8 + add rdi, sizeof(qword)*8 + sub rdx, 8 + jnc .tailLoop + + add rdx, 8 + + ; load data depending on length of tail + mov rbx, rdx + + mov r8, qword [rdi] + dec rbx + jz .get_tail_proc + mov r9, qword [rdi+sizeof(qword)] + dec rbx + jz .get_tail_proc + mov r10, qword [rdi+sizeof(qword)*2] + dec rbx + jz .get_tail_proc + mov r11, qword [rdi+sizeof(qword)*3] + dec rbx + jz .get_tail_proc + mov r12, qword [rdi+sizeof(qword)*4] + dec rbx + jz .get_tail_proc + mov r13, qword [rdi+sizeof(qword)*5] + dec rbx + jz .get_tail_proc + mov r14, qword [rdi+sizeof(qword)*6] + +.get_tail_proc: + ; multiply +; GET_EP rax, mla_8xl_tail, rdx, rbp ; tail procedure mla_8xn address + GET_EP rax, mla_lxl_short, rdx, rbp ; tail procedure mla_lxl address + + push rdx + call rax ; mla_nxn + pop rdx + + lea rdi, [rdi+rdx*sizeof(qword)] + + ; accumulate carryLCL and update hight product above + pop rax + shr rax, 1 + mov rbx, rdx + op_reg_mem adc, r8, [rdi+sizeof(qword)*0], rbp + dec rbx + jz .add_carry1 + op_reg_mem adc, r9, [rdi+sizeof(qword)*1], rbp + dec rbx + jz .add_carry1 + op_reg_mem adc, r10,[rdi+sizeof(qword)*2], rbp + dec rbx + jz .add_carry1 + op_reg_mem adc, r11,[rdi+sizeof(qword)*3], rbp + dec rbx + jz .add_carry1 + op_reg_mem adc, r12,[rdi+sizeof(qword)*4], rbp + dec rbx + jz .add_carry1 + op_reg_mem adc, r13,[rdi+sizeof(qword)*5], rbp + dec rbx + jz .add_carry1 + op_reg_mem adc, r14,[rdi+sizeof(qword)*6], rbp +.add_carry1: + adc rax, 0 + + pop rbp ; mul_8xn procedure + + ; accumulate carryGBL and store hight product above + pop rbx ; carryGBL + add r8, rbx + mov qword [rdi+sizeof(qword)*0], r8 + dec rdx + jz .add_carry2 + adc r9, 0 + mov qword [rdi+sizeof(qword)*1], r9 + dec rdx + jz .add_carry2 + adc r10, 0 + mov qword [rdi+sizeof(qword)*2], r10 + dec rdx + jz .add_carry2 + adc r11, 0 + mov qword [rdi+sizeof(qword)*3], r11 + dec rdx + jz .add_carry2 + adc r12, 0 + mov qword [rdi+sizeof(qword)*4], r12 + dec rdx + jz .add_carry2 + adc r13, 0 + mov qword [rdi+sizeof(qword)*5], r13 + dec rdx + jz .add_carry2 + adc r14, 0 + mov qword [rdi+sizeof(qword)*6], r14 +.add_carry2: + adc rax, 0 ; update carryGBL + + + ; release stack + pop rbx ; - passes counter + pop r8 ; - m' + pop rdx ; - modulus length + pop rsi ; - modulus + pop rdi ; - buffer + lea rdi, [rdi+rbx*sizeof(qword)] + +.complete_reduction: + add rsp, sizeof(qword)*8 ; release U space + + mov r14, rdx ; save modulus length + lea r15, [rdx*sizeof(qword)]; save modulus length (bytes) + + mov rbx, rax ; carry + mov rcx, rsi ; modulus + mov rsi, rdi ; buffer + pop rdi ; reduction address + + call sub_N ; reduction = buffer - modulus + sub rbx, rax ; rbx = borrow + + mov rdx, r14 ; length + sub rdi, r15 ; reduction + sub rsi, r15 ; buffer (src1) + mov rcx, rdi ; reduction (src2) + shr rbx,1 ; restore cf + + call copy_ae_N ; copy under cf, reduction = cf? buffer : reduction + + ret +ENDFUNC mred_N_adcox + + +%endif ;; _PCPMRED_INC_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmredpp_basic.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmredpp_basic.inc new file mode 100644 index 0000000..8e8b163 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmredpp_basic.inc @@ -0,0 +1,2193 @@ +;=============================================================================== +; Copyright (C) 2013 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Low level Big Number reduction Support +; +; + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; Fixed-size Montgomery reduction +;; + +; +; X7,X6,X5,X4,X3,X2,X1,X0 contains already preloaded (low) product +; +%macro MRED_FIX 8-15.nolist + %xdefine %%mSize %1 + %xdefine %%rRed %2 + %xdefine %%rProduct %3 + %xdefine %%rModulus %4 + %xdefine %%M0 %5 + %xdefine %%TMPH %6 + %xdefine %%TMP %7 + %xdefine %%X0 %8 + %xdefine %%X1 %9 + %xdefine %%X2 %10 + %xdefine %%X3 %11 + %xdefine %%X4 %12 + %xdefine %%X5 %13 + %xdefine %%X6 %14 + %xdefine %%X7 %15 + + mov rdx, %%M0 + imul rdx, %%X0 + MLA_FIX {%%mSize},{},{%%rModulus},{%%TMPH},{%%TMP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} +%if %%mSize > 1 + mov rdx, %%M0 + imul rdx, %%X0 + MLA_FIX {%%mSize},{}, {%%rModulus}, {%%TMPH},{%%TMP}, {%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} +%if %%mSize > 2 + mov rdx, %%M0 + imul rdx, %%X0 + MLA_FIX {%%mSize},{},{%%rModulus},{%%TMPH},{%%TMP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} +%if %%mSize > 3 + mov rdx, %%M0 + imul rdx, %%X0 + MLA_FIX {%%mSize},{}, {%%rModulus}, {%%TMPH},{%%TMP}, {%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} +%if %%mSize > 4 + mov rdx, %%M0 + imul rdx, %%X0 + MLA_FIX {%%mSize},{},{%%rModulus},{%%TMPH},{%%TMP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} +%if %%mSize > 5 + mov rdx, %%M0 + imul rdx, %%X0 + MLA_FIX {%%mSize},{}, {%%rModulus}, {%%TMPH},{%%TMP}, {%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} +%if %%mSize > 6 + mov rdx, %%M0 + imul rdx, %%X0 + MLA_FIX {%%mSize},{},{%%rModulus},{%%TMPH},{%%TMP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} +%if %%mSize > 7 + mov rdx, %%M0 + imul rdx, %%X0 + MLA_FIX {%%mSize},{},{%%rModulus},{%%TMPH},{%%TMP},{%%X0},{%%X1},{%%X2},{%%X3},{%%X4},{%%X5},{%%X6},{%%X7} +%endif +%endif +%endif +%endif +%endif +%endif +%endif + + xor %%TMP, %%TMP + add %%X0, qword [%%rProduct+sizeof(qword)*(%%mSize+0)] + mov qword [%%rProduct+sizeof(qword)*(%%mSize+0)], %%X0 +%if %%mSize > 1 + adc %%X1, qword [%%rProduct+sizeof(qword)*(%%mSize+1)] + mov qword [%%rProduct+sizeof(qword)*(%%mSize+1)], %%X1 +%if %%mSize > 2 + adc %%X2, qword [%%rProduct+sizeof(qword)*(%%mSize+2)] + mov qword [%%rProduct+sizeof(qword)*(%%mSize+2)], %%X2 +%if %%mSize > 3 + adc %%X3, qword [%%rProduct+sizeof(qword)*(%%mSize+3)] + mov qword [%%rProduct+sizeof(qword)*(%%mSize+3)], %%X3 +%if %%mSize > 4 + adc %%X4, qword [%%rProduct+sizeof(qword)*(%%mSize+4)] + mov qword [%%rProduct+sizeof(qword)*(%%mSize+4)], %%X4 +%if %%mSize > 5 + adc %%X5, qword [%%rProduct+sizeof(qword)*(%%mSize+5)] + mov qword [%%rProduct+sizeof(qword)*(%%mSize+5)], %%X5 +%if %%mSize > 6 + adc %%X6, qword [%%rProduct+sizeof(qword)*(%%mSize+6)] + mov qword [%%rProduct+sizeof(qword)*(%%mSize+6)], %%X6 +%if %%mSize > 7 + adc %%X7, qword [%%rProduct+sizeof(qword)*(%%mSize+7)] + mov qword [%%rProduct+sizeof(qword)*(%%mSize+7)], %%X7 +%endif +%endif +%endif +%endif +%endif +%endif +%endif + adc %%TMP, 0 + + sub %%X0, qword [%%rModulus+sizeof(qword)*0] +%if %%mSize > 1 + sbb %%X1, qword [%%rModulus+sizeof(qword)*1] +%if %%mSize > 2 + sbb %%X2, qword [%%rModulus+sizeof(qword)*2] +%if %%mSize > 3 + sbb %%X3, qword [%%rModulus+sizeof(qword)*3] +%if %%mSize > 4 + sbb %%X4, qword [%%rModulus+sizeof(qword)*4] +%if %%mSize > 5 + sbb %%X5, qword [%%rModulus+sizeof(qword)*5] +%if %%mSize > 6 + sbb %%X6, qword [%%rModulus+sizeof(qword)*6] +%if %%mSize > 7 + sbb %%X7, qword [%%rModulus+sizeof(qword)*7] +%endif +%endif +%endif +%endif +%endif +%endif +%endif + sbb %%TMP, 0 + + mov rax, qword [%%rProduct+sizeof(qword)*(%%mSize+0)] + cmovae rax, %%X0 + mov qword [%%rRed+sizeof(qword)*0], rax +%if %%mSize > 1 + mov rax, qword [%%rProduct+sizeof(qword)*(%%mSize+1)] + cmovae rax, %%X1 + mov qword [%%rRed+sizeof(qword)*1], rax +%if %%mSize > 2 + mov rax, qword [%%rProduct+sizeof(qword)*(%%mSize+2)] + cmovae rax, %%X2 + mov qword [%%rRed+sizeof(qword)*2], rax +%if %%mSize > 3 + mov rax, qword [%%rProduct+sizeof(qword)*(%%mSize+3)] + cmovae rax, %%X3 + mov qword [%%rRed+sizeof(qword)*3], rax +%if %%mSize > 4 + mov rax, qword [%%rProduct+sizeof(qword)*(%%mSize+4)] + cmovae rax, %%X4 + mov qword [%%rRed+sizeof(qword)*4], rax +%if %%mSize > 5 + mov rax, qword [%%rProduct+sizeof(qword)*(%%mSize+5)] + cmovae rax, %%X5 + mov qword [%%rRed+sizeof(qword)*5], rax +%if %%mSize > 6 + mov rax, qword [%%rProduct+sizeof(qword)*(%%mSize+6)] + cmovae rax, %%X6 + mov qword [%%rRed+sizeof(qword)*6], rax +%if %%mSize > 7 + mov rax, qword [%%rProduct+sizeof(qword)*(%%mSize+7)] + cmovae rax, %%X7 + mov qword [%%rRed+sizeof(qword)*7], rax +%endif +%endif +%endif +%endif +%endif +%endif +%endif +%endmacro + +align IPP_ALIGN_FACTOR +DECLARE_FUNC sub_N,PRIVATE + xor rax, rax ; cf = 0 +.sub_next: + lea rdi, [rdi+sizeof(qword)] + mov r8, qword [rsi] + mov r9, qword [rcx] + lea rsi, [rsi+sizeof(qword)] + lea rcx, [rcx+sizeof(qword)] + sbb r8, r9 + mov qword [rdi-sizeof(qword)], r8 + dec rdx + jnz .sub_next + adc rax, 0 + ret +ENDFUNC sub_N + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC copy_ae_N,PRIVATE + lea rdi, [rdi+sizeof(qword)] + mov r8, qword [rsi] ; src1[] + mov r9, qword [rcx] ; src2[] + lea rsi, [rsi+sizeof(qword)] + lea rcx, [rcx+sizeof(qword)] + cmovae r8, r9 + mov qword [rdi-sizeof(qword)], r8 + dec rdx + jnz copy_ae_N + ret +ENDFUNC copy_ae_N + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; mredN_start procedures +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred1_start,PRIVATE + MLA_FIX 1,, rsi, rbx,rbp, r8 + ret +ENDFUNC mred1_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred2_start,PRIVATE + MLA_FIX 2,, rsi, rbx,rbp, r8,r9 + ret +ENDFUNC mred2_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred3_start,PRIVATE + MLA_FIX 3,, rsi, rbx,rbp, r8,r9,r10 + ret +ENDFUNC mred3_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred4_start,PRIVATE + MLA_FIX 4,, rsi, rbx,rbp, r8,r9,r10,r11 + ret +ENDFUNC mred4_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred5_start,PRIVATE + MLA_FIX 5,, rsi, rbx,rbp, r8,r9,r10,r11,r12 + ret +ENDFUNC mred5_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred6_start,PRIVATE + MLA_FIX 6,, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13 + ret +ENDFUNC mred6_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred7_start,PRIVATE + MLA_FIX 7,, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14 + ret +ENDFUNC mred7_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred8_start,PRIVATE + MLA_FIX 8,, rsi, rbx,rbp, r8,r9,r10,r11,r12,r13,r14,r15 + ret +ENDFUNC mred8_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred8x1_start,PRIVATE + push rdx ; save m' + + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*0], rdx + call mred8_start + mov [rdi], rax + + pop rdx ; resrore m' + ret +ENDFUNC mred8x1_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred8x2_start,PRIVATE + push rdx ; save m' + + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*0], rdx + call mred8_start + mov [rdi], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*1], rdx + call mred8_start + mov [rdi+sizeof(qword)*1], rax + + pop rdx ; resrore m' + ret +ENDFUNC mred8x2_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred8x3_start,PRIVATE + push rdx ; save m' + + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*0], rdx + call mred8_start + mov [rdi], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*1], rdx + call mred8_start + mov [rdi+sizeof(qword)*1], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*2], rdx + call mred8_start + mov [rdi+sizeof(qword)*2], rax + + pop rdx ; resrore m' + ret +ENDFUNC mred8x3_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred8x4_start,PRIVATE + push rdx ; save m' + + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*0], rdx + call mred8_start + mov [rdi], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*1], rdx + call mred8_start + mov [rdi+sizeof(qword)*1], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*2], rdx + call mred8_start + mov [rdi+sizeof(qword)*2], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*3], rdx + call mred8_start + mov [rdi+sizeof(qword)*3], rax + + pop rdx ; resrore m' + ret +ENDFUNC mred8x4_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred8x5_start,PRIVATE + push rdx ; save m' + + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*0], rdx + call mred8_start + mov [rdi], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*1], rdx + call mred8_start + mov [rdi+sizeof(qword)*1], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*2], rdx + call mred8_start + mov [rdi+sizeof(qword)*2], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*3], rdx + call mred8_start + mov [rdi+sizeof(qword)*3], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*4], rdx + call mred8_start + mov [rdi+sizeof(qword)*4], rax + + pop rdx ; resrore m' + ret +ENDFUNC mred8x5_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred8x6_start,PRIVATE + push rdx ; save m' + + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*0], rdx + call mred8_start + mov [rdi], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*1], rdx + call mred8_start + mov [rdi+sizeof(qword)*1], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*2], rdx + call mred8_start + mov [rdi+sizeof(qword)*2], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*3], rdx + call mred8_start + mov [rdi+sizeof(qword)*3], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*4], rdx + call mred8_start + mov [rdi+sizeof(qword)*4], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*5], rdx + call mred8_start + mov [rdi+sizeof(qword)*5], rax + + pop rdx ; resrore m' + ret +ENDFUNC mred8x6_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred8x7_start,PRIVATE + push rdx ; save m' + + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*0], rdx + call mred8_start + mov [rdi], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*1], rdx + call mred8_start + mov [rdi+sizeof(qword)*1], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*2], rdx + call mred8_start + mov [rdi+sizeof(qword)*2], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*3], rdx + call mred8_start + mov [rdi+sizeof(qword)*3], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*4], rdx + call mred8_start + mov [rdi+sizeof(qword)*4], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*5], rdx + call mred8_start + mov [rdi+sizeof(qword)*5], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*6], rdx + call mred8_start + mov [rdi+sizeof(qword)*6], rax + + pop rdx ; resrore m' + ret +ENDFUNC mred8x7_start + + +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred8x8_start,PRIVATE + push rdx ; save m' + + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*0], rdx + call mred8_start + mov [rdi], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*1], rdx + call mred8_start + mov [rdi+sizeof(qword)*1], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*2], rdx + call mred8_start + mov [rdi+sizeof(qword)*2], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*3], rdx + call mred8_start + mov [rdi+sizeof(qword)*3], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*4], rdx + call mred8_start + mov [rdi+sizeof(qword)*4], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*5], rdx + call mred8_start + mov [rdi+sizeof(qword)*5], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*6], rdx + call mred8_start + mov [rdi+sizeof(qword)*6], rax + + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + mov qword [rcx+sizeof(qword)*7], rdx + call mred8_start + mov [rdi+sizeof(qword)*7], rax + + pop rdx ; resrore m' + ret +ENDFUNC mred8x8_start + + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; rdi - temporary product buffer/temporary reduction +;; rsi - modulus +;; r8 - m' +;; r15 - target reduction address +;; + +;; +;; 1*qword modulus length +;; + +;; +;; 2*qword modulus length +;; + +;; +;; 3*qword modulus length +;; + +;; +;; 4*qword modulus length +;; + +;; +;; 5*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_5,PRIVATE +%assign MSIZE 5 + push r8 ; m' + + ; load low half of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + + ; ui = x[i]*m' + ; reduction += ui*modulus + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred5_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred5_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred5_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred5_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred5_start + + pop rax ; remove m' + + ; finalize cf, reduction += U0*modulus operation + xor rax, rax + op_reg_mem add, r8, [rdi+sizeof(qword)*(MSIZE+0)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+0)], r8 + op_reg_mem adc, r9, [rdi+sizeof(qword)*(MSIZE+1)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+1)], r9 + op_reg_mem adc, r10,[rdi+sizeof(qword)*(MSIZE+2)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+2)], r10 + op_reg_mem adc, r11,[rdi+sizeof(qword)*(MSIZE+3)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+3)], r11 + op_reg_mem adc, r12,[rdi+sizeof(qword)*(MSIZE+4)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+4)], r12 + adc rax, 0 + + ; reduction -= modulus + op_reg_mem sub, r8, [rsi+sizeof(qword)*0], rbx + op_reg_mem sbb, r9, [rsi+sizeof(qword)*1], rbx + op_reg_mem sbb, r10,[rsi+sizeof(qword)*2], rbx + op_reg_mem sbb, r11,[rsi+sizeof(qword)*3], rbx + op_reg_mem sbb, r12,[rsi+sizeof(qword)*4], rbx + sbb rax, 0 + + ; copy under cf [r15] = cf? [r15] : {r8-r12} + mov rax, qword [rdi+sizeof(qword)*(MSIZE+0)] + mov rbx, qword [rdi+sizeof(qword)*(MSIZE+1)] + mov rcx, qword [rdi+sizeof(qword)*(MSIZE+2)] + mov rdx, qword [rdi+sizeof(qword)*(MSIZE+3)] + mov rbp, qword [rdi+sizeof(qword)*(MSIZE+4)] + cmovae rax, r8 + cmovae rbx, r9 + cmovae rcx, r10 + cmovae rdx, r11 + cmovae rbp, r12 + mov qword [r15+sizeof(qword)*0], rax + mov qword [r15+sizeof(qword)*1], rbx + mov qword [r15+sizeof(qword)*2], rcx + mov qword [r15+sizeof(qword)*3], rdx + mov qword [r15+sizeof(qword)*4], rbp + ret +ENDFUNC mred_5 + + +;; +;; 6*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_6,PRIVATE +%assign MSIZE 6 + push r8 ; m' + + ; load low half of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + + ; ui = x[i]*m' + ; reduction += ui*modulus + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred6_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred6_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred6_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred6_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred6_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred6_start + + pop rax ; remove m' + + ; finalize cf, reduction += U0*modulus operation + xor rax, rax + op_reg_mem add, r8, [rdi+sizeof(qword)*(MSIZE+0)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+0)], r8 + op_reg_mem adc, r9, [rdi+sizeof(qword)*(MSIZE+1)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+1)], r9 + op_reg_mem adc, r10,[rdi+sizeof(qword)*(MSIZE+2)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+2)], r10 + op_reg_mem adc, r11,[rdi+sizeof(qword)*(MSIZE+3)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+3)], r11 + op_reg_mem adc, r12,[rdi+sizeof(qword)*(MSIZE+4)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+4)], r12 + op_reg_mem adc, r13,[rdi+sizeof(qword)*(MSIZE+5)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+5)], r13 + adc rax, 0 + + ; reduction -= modulus + op_reg_mem sub, r8, [rsi+sizeof(qword)*0], rbx + op_reg_mem sbb, r9, [rsi+sizeof(qword)*1], rbx + op_reg_mem sbb, r10,[rsi+sizeof(qword)*2], rbx + op_reg_mem sbb, r11,[rsi+sizeof(qword)*3], rbx + op_reg_mem sbb, r12,[rsi+sizeof(qword)*4], rbx + op_reg_mem sbb, r13,[rsi+sizeof(qword)*5], rbx + sbb rax, 0 + + ; copy under cf [r15] = cf? [r15] : {r8-r13} + mov rax, qword [rdi+sizeof(qword)*(MSIZE+0)] + mov rbx, qword [rdi+sizeof(qword)*(MSIZE+1)] + mov rcx, qword [rdi+sizeof(qword)*(MSIZE+2)] + mov rdx, qword [rdi+sizeof(qword)*(MSIZE+3)] + mov rbp, qword [rdi+sizeof(qword)*(MSIZE+4)] + mov rsi, qword [rdi+sizeof(qword)*(MSIZE+5)] + cmovae rax, r8 + cmovae rbx, r9 + cmovae rcx, r10 + cmovae rdx, r11 + cmovae rbp, r12 + cmovae rsi, r13 + mov qword [r15+sizeof(qword)*0], rax + mov qword [r15+sizeof(qword)*1], rbx + mov qword [r15+sizeof(qword)*2], rcx + mov qword [r15+sizeof(qword)*3], rdx + mov qword [r15+sizeof(qword)*4], rbp + mov qword [r15+sizeof(qword)*5], rsi + ret +ENDFUNC mred_6 + + +;; +;; 7*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_7,PRIVATE +%assign MSIZE 7 + push r8 ; m' + + ; load low half of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + + ; ui = x[i]*m' + ; reduction += ui*modulus + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred7_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred7_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred7_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred7_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred7_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred7_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred7_start + + pop rax ; remove m' + + ; finalize cf, reduction += U0*modulus operation + xor rax, rax + op_reg_mem add, r8, [rdi+sizeof(qword)*(MSIZE+0)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+0)], r8 + op_reg_mem adc, r9, [rdi+sizeof(qword)*(MSIZE+1)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+1)], r9 + op_reg_mem adc, r10,[rdi+sizeof(qword)*(MSIZE+2)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+2)], r10 + op_reg_mem adc, r11,[rdi+sizeof(qword)*(MSIZE+3)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+3)], r11 + op_reg_mem adc, r12,[rdi+sizeof(qword)*(MSIZE+4)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+4)], r12 + op_reg_mem adc, r13,[rdi+sizeof(qword)*(MSIZE+5)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+5)], r13 + op_reg_mem adc, r14,[rdi+sizeof(qword)*(MSIZE+6)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+6)], r14 + adc rax, 0 + + ; reduction -= modulus + op_reg_mem sub, r8, [rsi+sizeof(qword)*0], rbx + op_reg_mem sbb, r9, [rsi+sizeof(qword)*1], rbx + op_reg_mem sbb, r10,[rsi+sizeof(qword)*2], rbx + op_reg_mem sbb, r11,[rsi+sizeof(qword)*3], rbx + op_reg_mem sbb, r12,[rsi+sizeof(qword)*4], rbx + op_reg_mem sbb, r13,[rsi+sizeof(qword)*5], rbx + op_reg_mem sbb, r14,[rsi+sizeof(qword)*6], rbx + sbb rax, 0 + + ; copy under cf [r15] = cf? [r15] : {r8-r14} + mov rax, qword [rdi+sizeof(qword)*(MSIZE+0)] + mov rbx, qword [rdi+sizeof(qword)*(MSIZE+1)] + mov rcx, qword [rdi+sizeof(qword)*(MSIZE+2)] + mov rdx, qword [rdi+sizeof(qword)*(MSIZE+3)] + mov rbp, qword [rdi+sizeof(qword)*(MSIZE+4)] + mov rsi, qword [rdi+sizeof(qword)*(MSIZE+5)] + mov rdi, qword [rdi+sizeof(qword)*(MSIZE+6)] + cmovae rax, r8 + cmovae rbx, r9 + cmovae rcx, r10 + cmovae rdx, r11 + cmovae rbp, r12 + cmovae rsi, r13 + cmovae rdi, r14 + mov qword [r15+sizeof(qword)*0], rax + mov qword [r15+sizeof(qword)*1], rbx + mov qword [r15+sizeof(qword)*2], rcx + mov qword [r15+sizeof(qword)*3], rdx + mov qword [r15+sizeof(qword)*4], rbp + mov qword [r15+sizeof(qword)*5], rsi + mov qword [r15+sizeof(qword)*6], rdi + ret +ENDFUNC mred_7 + + +;; +;; 8*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_8,PRIVATE +%assign MSIZE 8 + push r15 ; save reduction address + push r8 ; m' + + ; load low half of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + ; ui = x[i]*m' + ; reduction += ui*modulus + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred8_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred8_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred8_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred8_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred8_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred8_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred8_start + mov rdx, [rsp] + gsmulx rbx, rdx, r8 + call mred8_start + + pop rax ; remove m' + + ; finalize cf, reduction += U0*modulus operation + xor rax, rax + op_reg_mem add, r8, [rdi+sizeof(qword)*(MSIZE+0)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+0)], r8 + op_reg_mem adc, r9, [rdi+sizeof(qword)*(MSIZE+1)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+1)], r9 + op_reg_mem adc, r10,[rdi+sizeof(qword)*(MSIZE+2)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+2)], r10 + op_reg_mem adc, r11,[rdi+sizeof(qword)*(MSIZE+3)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+3)], r11 + op_reg_mem adc, r12,[rdi+sizeof(qword)*(MSIZE+4)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+4)], r12 + op_reg_mem adc, r13,[rdi+sizeof(qword)*(MSIZE+5)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+5)], r13 + op_reg_mem adc, r14,[rdi+sizeof(qword)*(MSIZE+6)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+6)], r14 + op_reg_mem adc, r15,[rdi+sizeof(qword)*(MSIZE+7)], rbx + mov qword [rdi+sizeof(qword)*(MSIZE+7)], r15 + adc rax, 0 + + ; reduction -= modulus + op_reg_mem sub, r8, [rsi+sizeof(qword)*0], rbx + op_reg_mem sbb, r9, [rsi+sizeof(qword)*1], rbx + op_reg_mem sbb, r10,[rsi+sizeof(qword)*2], rbx + op_reg_mem sbb, r11,[rsi+sizeof(qword)*3], rbx + op_reg_mem sbb, r12,[rsi+sizeof(qword)*4], rbx + op_reg_mem sbb, r13,[rsi+sizeof(qword)*5], rbx + op_reg_mem sbb, r14,[rsi+sizeof(qword)*6], rbx + op_reg_mem sbb, r15,[rsi+sizeof(qword)*7], rbx + sbb rax, 0 + + pop rsi ; address of reduction + ; copy under cf [rsi] = cf? [rdi] : {r8-r15} + mov rax, qword [rdi+sizeof(qword)*(MSIZE+0)] + mov rbx, qword [rdi+sizeof(qword)*(MSIZE+1)] + mov rcx, qword [rdi+sizeof(qword)*(MSIZE+2)] + mov rdx, qword [rdi+sizeof(qword)*(MSIZE+3)] + cmovae rax, r8 + cmovae rbx, r9 + cmovae rcx, r10 + cmovae rdx, r11 + mov qword [rsi+sizeof(qword)*0], rax + mov qword [rsi+sizeof(qword)*1], rbx + mov qword [rsi+sizeof(qword)*2], rcx + mov qword [rsi+sizeof(qword)*3], rdx + + mov rax, qword [rdi+sizeof(qword)*(MSIZE+4)] + mov rbx, qword [rdi+sizeof(qword)*(MSIZE+5)] + mov rcx, qword [rdi+sizeof(qword)*(MSIZE+6)] + mov rdx, qword [rdi+sizeof(qword)*(MSIZE+7)] + cmovae rax, r12 + cmovae rbx, r13 + cmovae rcx, r14 + cmovae rdx, r15 + mov qword [rsi+sizeof(qword)*4], rax + mov qword [rsi+sizeof(qword)*5], rbx + mov qword [rsi+sizeof(qword)*6], rcx + mov qword [rsi+sizeof(qword)*7], rdx + ret +ENDFUNC mred_8 + + +;; +;; 9*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_9,PRIVATE +%assign MSIZE 9 + push r15 ; save reduction address + + sub rsp, sizeof(qword)*8 ; allocate U space + mov rcx, rsp + + push r8 ; save m' + mov rdx, r8 + +;; +;; init pass +;; + + ; load low part of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + call mred8x8_start + + xor rax, rax ; init carryLCL + op_reg_mem add, r8, [rdi+sizeof(qword)*(8+0)], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(8+1)], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(8+2)], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(8+3)], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(8+4)], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(8+5)], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*(8+6)], rbx + op_reg_mem adc, r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + SWAP rcx, rsi + call mla_8x1 + SWAP rcx, rsi + + pop rax + shr rax, 1 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], r13 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], r14 + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], r15, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], rbx + adc rax, 0 + push rax ; store carryGBL + +;; +;; last pass +;; + sub rsi, sizeof(qword)*8 + + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + mov rdx, [rsp+sizeof(qword)] + call mred8x1_start + + xor rax, rax ; init carryLCL + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], r13 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], r14 + mov r8, r15 + add r8, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7] + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + call mla_1x1 + + pop rax ; restore carryLCL + shr rax, 1 + op_reg_mem adc, r8, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], rbx + adc rax, 0 + + pop rbx ; carryGBL + add r8, rbx + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + adc rax, 0 ; update carryGBL + + + pop rcx ; remove m' + add rsp, sizeof(qword)*8 ; release U space + + lea rcx, [rsi-sizeof(qword)*8] ; restore modulus + lea rsi, [rdi+sizeof(qword)*(MSIZE-8)-sizeof(qword)*8] ; restore buffer + pop rdi ; restore reduction + + mov rbx, rax ; save carryGBL + + mov rdx, dword MSIZE + call sub_N ; reduction = buffer - modulus + sub rbx, rax ; rbx = borrow + + sub rdi, sizeof(qword)*MSIZE ; reduction + sub rsi, sizeof(qword)*MSIZE ; buffer (src1) + mov rcx, rdi ; reduction (src2) + mov rdx, dword MSIZE + shr rbx,1 ; restore cf + call copy_ae_N ; copy under cf, reduction = cf? buffer : reduction + + ret +ENDFUNC mred_9 + + +;; +;; 10*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_10,PRIVATE +%assign MSIZE 10 + push r15 ; save reduction address + + sub rsp, sizeof(qword)*8 ; allocate U space + mov rcx, rsp + + push r8 ; save m' + mov rdx, r8 + +;; +;; init pass +;; + + ; load low part of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + call mred8x8_start + + xor rax, rax ; init carryLCL + op_reg_mem add, r8, [rdi+sizeof(qword)*(8+0)], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(8+1)], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(8+2)], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(8+3)], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(8+4)], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(8+5)], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*(8+6)], rbx + op_reg_mem adc, r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + SWAP rcx, rsi + call mla_8x2 + SWAP rcx, rsi + + pop rax + shr rax, 1 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], r13 + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], r14, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], r15, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], rbx + adc rax, 0 + push rax ; store carryGBL + +;; +;; last pass +;; + sub rsi, sizeof(qword)*8 + + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + mov rdx, [rsp+sizeof(qword)] + call mred8x2_start + + xor rax, rax ; init carryLCL + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], r13 + mov r8, r14 + mov r9, r15 + add r8, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6] + adc r9, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7] + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + call mla_2x2 + + pop rax ; restore carryLCL + shr rax, 1 + op_reg_mem adc, r8, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], rbx + adc rax, 0 + + pop rbx ; carryGBL + add r8, rbx + adc r9, 0 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + adc rax, 0 ; update carryGBL + + + pop rcx ; remove m' + add rsp, sizeof(qword)*8 ; release U space + + lea rcx, [rsi-sizeof(qword)*8] ; restore modulus + lea rsi, [rdi+sizeof(qword)*(MSIZE-8)-sizeof(qword)*8] ; restore buffer + pop rdi ; restore reduction + + mov rbx, rax ; save carryGBL + + mov rdx, dword MSIZE + call sub_N ; reduction = buffer - modulus + sub rbx, rax ; rbx = borrow + + sub rdi, sizeof(qword)*MSIZE ; reduction + sub rsi, sizeof(qword)*MSIZE ; buffer (src1) + mov rcx, rdi ; reduction (src2) + mov rdx, dword MSIZE + shr rbx,1 ; restore cf + call copy_ae_N ; copy under cf, reduction = cf? buffer : reduction + + ret +ENDFUNC mred_10 + + +;; +;; 11*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_11,PRIVATE +%assign MSIZE 11 + push r15 ; save reduction address + + sub rsp, sizeof(qword)*8 ; allocate U space + mov rcx, rsp + + push r8 ; save m' + mov rdx, r8 + +;; +;; init pass +;; + + ; load low part of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + call mred8x8_start + + xor rax, rax ; init carryLCL + op_reg_mem add, r8, [rdi+sizeof(qword)*(8+0)], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(8+1)], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(8+2)], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(8+3)], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(8+4)], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(8+5)], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*(8+6)], rbx + op_reg_mem adc, r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + SWAP rcx, rsi + call mla_8x3 + SWAP rcx, rsi + + pop rax + shr rax, 1 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12 + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], r13, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], r14, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], r15, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], rbx + adc rax, 0 + push rax ; store carryGBL + +;; +;; last pass +;; + sub rsi, sizeof(qword)*8 + + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + mov rdx, [rsp+sizeof(qword)] + call mred8x3_start + + xor rax, rax ; init carryLCL + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12 + mov r8, r13 + mov r9, r14 + mov r10,r15 + add r8, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5] + adc r9, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6] + adc r10,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7] + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + call mla_3x3 + + pop rax ; restore carryLCL + shr rax, 1 + op_reg_mem adc, r8, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], rbx + adc rax, 0 + + pop rbx ; carryGBL + add r8, rbx + adc r9, 0 + adc r10,0 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + adc rax, 0 ; update carryGBL + + + pop rcx ; remove m' + add rsp, sizeof(qword)*8 ; release U space + + lea rcx, [rsi-sizeof(qword)*8] ; restore modulus + lea rsi, [rdi+sizeof(qword)*(MSIZE-8)-sizeof(qword)*8] ; restore buffer + pop rdi ; restore reduction + + mov rbx, rax ; save carryGBL + + mov rdx, dword MSIZE + call sub_N ; reduction = buffer - modulus + sub rbx, rax ; rbx = borrow + + sub rdi, sizeof(qword)*MSIZE ; reduction + sub rsi, sizeof(qword)*MSIZE ; buffer (src1) + mov rcx, rdi ; reduction (src2) + mov rdx, dword MSIZE + shr rbx,1 ; restore cf + call copy_ae_N ; copy under cf, reduction = cf? buffer : reduction + + ret +ENDFUNC mred_11 + + +;; +;; 12*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_12,PRIVATE +%assign MSIZE 12 + push r15 ; save reduction address + + sub rsp, sizeof(qword)*8 ; allocate U space + mov rcx, rsp + + push r8 ; save m' + mov rdx, r8 + +;; +;; init pass +;; + + ; load low part of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + call mred8x8_start + + xor rax, rax ; init carryLCL + op_reg_mem add, r8, [rdi+sizeof(qword)*(8+0)], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(8+1)], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(8+2)], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(8+3)], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(8+4)], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(8+5)], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*(8+6)], rbx + op_reg_mem adc, r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + SWAP rcx, rsi + call mla_8x4 + SWAP rcx, rsi + + pop rax + shr rax, 1 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], r13, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], r14, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], r15, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], rbx + adc rax, 0 + push rax ; store carryGBL + +;; +;; last pass +;; + sub rsi, sizeof(qword)*8 + + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + mov rdx, [rsp+sizeof(qword)] + call mred8x4_start + + xor rax, rax ; init carryLCL + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + mov r8, r12 + mov r9, r13 + mov r10,r14 + mov r11,r15 + add r8, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4] + adc r9, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5] + adc r10,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6] + adc r11,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7] + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + call mla_4x4 + + pop rax ; restore carryLCL + shr rax, 1 + op_reg_mem adc, r8, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], rbx + adc rax, 0 + + pop rbx ; carryGBL + add r8, rbx + adc r9, 0 + adc r10,0 + adc r11,0 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + adc rax, 0 ; update carryGBL + + + pop rcx ; remove m' + add rsp, sizeof(qword)*8 ; release U space + + lea rcx, [rsi-sizeof(qword)*8] ; restore modulus + lea rsi, [rdi+sizeof(qword)*(MSIZE-8)-sizeof(qword)*8] ; restore buffer + pop rdi ; restore reduction + + mov rbx, rax ; save carryGBL + + mov rdx, dword MSIZE + call sub_N ; reduction = buffer - modulus + sub rbx, rax ; rbx = borrow + + sub rdi, sizeof(qword)*MSIZE ; reduction + sub rsi, sizeof(qword)*MSIZE ; buffer (src1) + mov rcx, rdi ; reduction (src2) + mov rdx, dword MSIZE + shr rbx,1 ; restore cf + call copy_ae_N ; copy under cf, reduction = cf? buffer : reduction + + ret +ENDFUNC mred_12 + + +;; +;; 13*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_13,PRIVATE +%assign MSIZE 13 + push r15 ; save reduction address + + sub rsp, sizeof(qword)*8 ; allocate U space + mov rcx, rsp + + push r8 ; save m' + mov rdx, r8 + +;; +;; init pass +;; + + ; load low part of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + call mred8x8_start + + xor rax, rax ; init carryLCL + op_reg_mem add, r8, [rdi+sizeof(qword)*(8+0)], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(8+1)], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(8+2)], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(8+3)], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(8+4)], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(8+5)], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*(8+6)], rbx + op_reg_mem adc, r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + SWAP rcx, rsi + call mla_8x5 + SWAP rcx, rsi + + pop rax + shr rax, 1 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], r13, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], r14, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], r15, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], rbx + adc rax, 0 + push rax ; store carryGBL + +;; +;; last pass +;; + sub rsi, sizeof(qword)*8 + + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + mov rdx, [rsp+sizeof(qword)] + call mred8x5_start + + xor rax, rax ; init carryLCL + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov r8, r11 + mov r9, r12 + mov r10,r13 + mov r11,r14 + mov r12,r15 + add r8, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3] + adc r9, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4] + adc r10,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5] + adc r11,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6] + adc r12,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7] + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + call mla_5x5 + + pop rax ; restore carryLCL + shr rax, 1 + op_reg_mem adc, r8, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], rbx + adc rax, 0 + + pop rbx ; carryGBL + add r8, rbx + adc r9, 0 + adc r10,0 + adc r11,0 + adc r12,0 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12 + adc rax, 0 ; update carryGBL + + + pop rcx ; remove m' + add rsp, sizeof(qword)*8 ; release U space + + lea rcx, [rsi-sizeof(qword)*8] ; restore modulus + lea rsi, [rdi+sizeof(qword)*(MSIZE-8)-sizeof(qword)*8] ; restore buffer + pop rdi ; restore reduction + + mov rbx, rax ; save carryGBL + + mov rdx, dword MSIZE + call sub_N ; reduction = buffer - modulus + sub rbx, rax ; rbx = borrow + + sub rdi, sizeof(qword)*MSIZE ; reduction + sub rsi, sizeof(qword)*MSIZE ; buffer (src1) + mov rcx, rdi ; reduction (src2) + mov rdx, dword MSIZE + shr rbx,1 ; restore cf + call copy_ae_N ; copy under cf, reduction = cf? buffer : reduction + + ret +ENDFUNC mred_13 + + +;; +;; 14*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_14,PRIVATE +%assign MSIZE 14 + push r15 ; save reduction address + + sub rsp, sizeof(qword)*8 ; allocate U space + mov rcx, rsp + + push r8 ; save m' + mov rdx, r8 + +;; +;; init pass +;; + + ; load low part of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + call mred8x8_start + + xor rax, rax ; init carryLCL + op_reg_mem add, r8, [rdi+sizeof(qword)*(8+0)], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(8+1)], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(8+2)], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(8+3)], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(8+4)], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(8+5)], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*(8+6)], rbx + op_reg_mem adc, r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + SWAP rcx, rsi + call mla_8x6 + SWAP rcx, rsi + + pop rax + shr rax, 1 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], r13, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], r14, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], r15, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], rbx + adc rax, 0 + push rax ; store carryGBL + +;; +;; last pass +;; + sub rsi, sizeof(qword)*8 + + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + mov rdx, [rsp+sizeof(qword)] + call mred8x6_start + + xor rax, rax ; init carryLCL + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov r8, r10 + mov r9, r11 + mov r10,r12 + mov r11,r13 + mov r12,r14 + mov r13,r15 + add r8, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2] + adc r9, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3] + adc r10,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4] + adc r11,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5] + adc r12,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6] + adc r13,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7] + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + call mla_6x6 + + pop rax ; restore carryLCL + shr rax, 1 + op_reg_mem adc, r8, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], rbx + adc rax, 0 + + pop rbx ; carryGBL + add r8, rbx + adc r9, 0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], r13 + adc rax, 0 ; update carryGBL + + + pop rcx ; remove m' + add rsp, sizeof(qword)*8 ; release U space + + lea rcx, [rsi-sizeof(qword)*8] ; restore modulus + lea rsi, [rdi+sizeof(qword)*(MSIZE-8)-sizeof(qword)*8] ; restore buffer + pop rdi ; restore reduction + + mov rbx, rax ; save carryGBL + + mov rdx, dword MSIZE + call sub_N ; reduction = buffer - modulus + sub rbx, rax ; rbx = borrow + + sub rdi, sizeof(qword)*MSIZE ; reduction + sub rsi, sizeof(qword)*MSIZE ; buffer (src1) + mov rcx, rdi ; reduction (src2) + mov rdx, dword MSIZE + shr rbx,1 ; restore cf + call copy_ae_N ; copy under cf, reduction = cf? buffer : reduction + + ret +ENDFUNC mred_14 + + +;; +;; 15*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_15,PRIVATE +%assign MSIZE 15 + push r15 ; save reduction address + + sub rsp, sizeof(qword)*8 ; allocate U space + mov rcx, rsp + + push r8 ; save m' + mov rdx, r8 + +;; +;; init pass +;; + + ; load low part of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + call mred8x8_start + + xor rax, rax ; init carryLCL + op_reg_mem add, r8, [rdi+sizeof(qword)*(8+0)], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(8+1)], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(8+2)], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(8+3)], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(8+4)], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(8+5)], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*(8+6)], rbx + op_reg_mem adc, r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + SWAP rcx, rsi + call mla_8x7 + SWAP rcx, rsi + + pop rax + shr rax, 1 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], r13, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], r14, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], r15, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7], rbx + adc rax, 0 + push rax ; store carryGBL + +;; +;; last pass +;; + sub rsi, sizeof(qword)*8 + + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + mov rdx, [rsp+sizeof(qword)] + call mred8x7_start + + xor rax, rax ; init carryLCL + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov r8, r9 + mov r9, r10 + mov r10,r11 + mov r11,r12 + mov r12,r13 + mov r13,r14 + mov r14,r15 + add r8, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1] + adc r9, qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2] + adc r10,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3] + adc r11,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4] + adc r12,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5] + adc r13,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6] + adc r14,qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*7] + adc rax, 0 + push rax ; store carryLCL + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + call mla_7x7 + + pop rax ; restore carryLCL + shr rax, 1 + op_reg_mem adc, r8, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], rbx + adc rax, 0 + + pop rbx ; carryGBL + add r8, rbx + adc r9, 0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + adc r14,0 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*0], r8 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*1], r9 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*2], r10 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*3], r11 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*4], r12 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*5], r13 + mov qword [rdi+sizeof(qword)*(MSIZE-8)+sizeof(qword)*6], r14 + adc rax, 0 ; update carryGBL + + + pop rcx ; remove m' + add rsp, sizeof(qword)*8 ; release U space + + lea rcx, [rsi-sizeof(qword)*8] ; restore modulus + lea rsi, [rdi+sizeof(qword)*(MSIZE-8)-sizeof(qword)*8] ; restore buffer + pop rdi ; restore reduction + + mov rbx, rax ; save carryGBL + + mov rdx, dword MSIZE + call sub_N ; reduction = buffer - modulus + sub rbx, rax ; rbx = borrow + + sub rdi, sizeof(qword)*MSIZE ; reduction + sub rsi, sizeof(qword)*MSIZE ; buffer (src1) + mov rcx, rdi ; reduction (src2) + mov rdx, dword MSIZE + shr rbx,1 ; restore cf + call copy_ae_N ; copy under cf, reduction = cf? buffer : reduction + + ret +ENDFUNC mred_15 + + +;; +;; 16*qword modulus length +;; +align IPP_ALIGN_FACTOR +DECLARE_FUNC mred_16,PRIVATE +%assign MSIZE 16 + push r15 ; save reduction address + + sub rsp, sizeof(qword)*8 ; allocate U space + mov rcx, rsp + + mov rdx, r8 ; copy m' + + ; load low part of the product + mov r8, qword [rdi] ; x0,x1,x2,x3,x4,x5,x6,x7 + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + call mred8x8_start + + xor rax, rax + op_reg_mem add, r8, [rdi+sizeof(qword)*(8+0)], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(8+1)], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(8+2)], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(8+3)], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(8+4)], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(8+5)], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*(8+6)], rbx + op_reg_mem adc, r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax + + add rsi, sizeof(qword)*8 + add rdi, sizeof(qword)*8 + + push rdx + call mla_8x8 + pop rdx + + pop rax + shr rax, 1 + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+0)], r8, [rdi+sizeof(qword)*(8+0)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+1)], r9, [rdi+sizeof(qword)*(8+1)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+2)], r10,[rdi+sizeof(qword)*(8+2)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+3)], r11,[rdi+sizeof(qword)*(8+3)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+4)], r12,[rdi+sizeof(qword)*(8+4)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+5)], r13,[rdi+sizeof(qword)*(8+5)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+6)], r14,[rdi+sizeof(qword)*(8+6)], rbx + op_mem_reg_mem adc, [rdi+sizeof(qword)*(8+7)], r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax + + sub rsi, sizeof(qword)*8 + + mov r8, qword [rdi] + mov r9, qword [rdi+sizeof(qword)] + mov r10,qword [rdi+sizeof(qword)*2] + mov r11,qword [rdi+sizeof(qword)*3] + mov r12,qword [rdi+sizeof(qword)*4] + mov r13,qword [rdi+sizeof(qword)*5] + mov r14,qword [rdi+sizeof(qword)*6] + mov r15,qword [rdi+sizeof(qword)*7] + + call mred8x8_start + + xor rax, rax + op_reg_mem add, r8, [rdi+sizeof(qword)*(8+0)], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(8+1)], rbx + op_reg_mem adc, r10,[rdi+sizeof(qword)*(8+2)], rbx + op_reg_mem adc, r11,[rdi+sizeof(qword)*(8+3)], rbx + op_reg_mem adc, r12,[rdi+sizeof(qword)*(8+4)], rbx + op_reg_mem adc, r13,[rdi+sizeof(qword)*(8+5)], rbx + op_reg_mem adc, r14,[rdi+sizeof(qword)*(8+6)], rbx + op_reg_mem adc, r15,[rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + push rax + + add rdi, sizeof(qword)*8 + add rsi, sizeof(qword)*8 + + call mla_8x8 + + sub rsi, sizeof(qword)*8 + + pop rax + shr rax, 1 + op_reg_mem adc, r8, [rdi+sizeof(qword)*(8+0)], rbx + op_reg_mem adc, r9, [rdi+sizeof(qword)*(8+1)], rbx + op_reg_mem adc, r10, [rdi+sizeof(qword)*(8+2)], rbx + op_reg_mem adc, r11, [rdi+sizeof(qword)*(8+3)], rbx + op_reg_mem adc, r12, [rdi+sizeof(qword)*(8+4)], rbx + op_reg_mem adc, r13, [rdi+sizeof(qword)*(8+5)], rbx + op_reg_mem adc, r14, [rdi+sizeof(qword)*(8+6)], rbx + op_reg_mem adc, r15, [rdi+sizeof(qword)*(8+7)], rbx + adc rax, 0 + + pop rbx + add r8, rbx + adc r9, 0 + adc r10, 0 + adc r11, 0 + adc r12, 0 + adc r13, 0 + adc r14, 0 + adc r15, 0 + adc rax, 0 + + mov qword [rdi+sizeof(qword)*(8+0)], r8 + mov qword [rdi+sizeof(qword)*(8+1)], r9 + mov qword [rdi+sizeof(qword)*(8+2)], r10 + mov qword [rdi+sizeof(qword)*(8+3)], r11 + mov qword [rdi+sizeof(qword)*(8+4)], r12 + mov qword [rdi+sizeof(qword)*(8+5)], r13 + mov qword [rdi+sizeof(qword)*(8+6)], r14 + mov qword [rdi+sizeof(qword)*(8+7)], r15 + + add rsp, sizeof(qword)*8 ; release U space + + pop rbp + + ; reduction -= modulus + op_mem_mem sub, [rbp+sizeof(qword)*0 ], [rdi+sizeof(qword)*0 ], [rsi+sizeof(qword)*0 ], rbx + op_mem_mem sbb, [rbp+sizeof(qword)*1 ], [rdi+sizeof(qword)*1 ], [rsi+sizeof(qword)*1 ], rbx + op_mem_mem sbb, [rbp+sizeof(qword)*2 ], [rdi+sizeof(qword)*2 ], [rsi+sizeof(qword)*2 ], rbx + op_mem_mem sbb, [rbp+sizeof(qword)*3 ], [rdi+sizeof(qword)*3 ], [rsi+sizeof(qword)*3 ], rbx + op_mem_mem sbb, [rbp+sizeof(qword)*4 ], [rdi+sizeof(qword)*4 ], [rsi+sizeof(qword)*4 ], rbx + op_mem_mem sbb, [rbp+sizeof(qword)*5 ], [rdi+sizeof(qword)*5 ], [rsi+sizeof(qword)*5 ], rbx + op_mem_mem sbb, [rbp+sizeof(qword)*6 ], [rdi+sizeof(qword)*6 ], [rsi+sizeof(qword)*6 ], rbx + op_mem_mem sbb, [rbp+sizeof(qword)*7 ], [rdi+sizeof(qword)*7 ], [rsi+sizeof(qword)*7 ], rbx + op_reg_mem sbb, r8, [rsi+sizeof(qword)*8 ], rbx + op_reg_mem sbb, r9, [rsi+sizeof(qword)*9 ], rbx + op_reg_mem sbb, r10, [rsi+sizeof(qword)*10], rbx + op_reg_mem sbb, r11, [rsi+sizeof(qword)*11], rbx + op_reg_mem sbb, r12, [rsi+sizeof(qword)*12], rbx + op_reg_mem sbb, r13, [rsi+sizeof(qword)*13], rbx + op_reg_mem sbb, r14, [rsi+sizeof(qword)*14], rbx + op_reg_mem sbb, r15, [rsi+sizeof(qword)*15], rbx + sbb rax, 0 + + ; copy under cf [rbp] = cf? [rdi] : [rbp] + mov rax, qword [rdi+sizeof(qword)*8 ] + mov rbx, qword [rdi+sizeof(qword)*9 ] + mov rcx, qword [rdi+sizeof(qword)*10] + mov rdx, qword [rdi+sizeof(qword)*11] + cmovae rax, r8 + cmovae rbx, r9 + cmovae rcx, r10 + cmovae rdx, r11 + mov qword [rbp+sizeof(qword)*8 ], rax + mov qword [rbp+sizeof(qword)*9 ], rbx + mov qword [rbp+sizeof(qword)*10], rcx + mov qword [rbp+sizeof(qword)*11], rdx + + mov rax, qword [rdi+sizeof(qword)*12] + mov rbx, qword [rdi+sizeof(qword)*13] + mov rcx, qword [rdi+sizeof(qword)*14] + mov rdx, qword [rdi+sizeof(qword)*15] + cmovae rax, r12 + cmovae rbx, r13 + cmovae rcx, r14 + cmovae rdx, r15 + mov qword [rbp+sizeof(qword)*12], rax + mov qword [rbp+sizeof(qword)*13], rbx + mov qword [rbp+sizeof(qword)*14], rcx + mov qword [rbp+sizeof(qword)*15], rdx + + mov r8, qword [rbp+sizeof(qword)*0] + mov r9, qword [rbp+sizeof(qword)*1] + mov r10, qword [rbp+sizeof(qword)*2] + mov r11, qword [rbp+sizeof(qword)*3] + mov r12, qword [rbp+sizeof(qword)*4] + mov r13, qword [rbp+sizeof(qword)*5] + mov r14, qword [rbp+sizeof(qword)*6] + mov r15, qword [rbp+sizeof(qword)*7] + + mov rax, qword [rdi+sizeof(qword)*0] + mov rbx, qword [rdi+sizeof(qword)*1] + mov rcx, qword [rdi+sizeof(qword)*2] + mov rdx, qword [rdi+sizeof(qword)*3] + cmovae rax, r8 + cmovae rbx, r9 + cmovae rcx, r10 + cmovae rdx, r11 + mov qword [rbp+sizeof(qword)*0], rax + mov qword [rbp+sizeof(qword)*1], rbx + mov qword [rbp+sizeof(qword)*2], rcx + mov qword [rbp+sizeof(qword)*3], rdx + + mov rax, qword [rdi+sizeof(qword)*4] + mov rbx, qword [rdi+sizeof(qword)*5] + mov rcx, qword [rdi+sizeof(qword)*6] + mov rdx, qword [rdi+sizeof(qword)*7] + cmovae rax, r12 + cmovae rbx, r13 + cmovae rcx, r14 + cmovae rdx, r15 + mov qword [rbp+sizeof(qword)*4], rax + mov qword [rbp+sizeof(qword)*5], rbx + mov qword [rbp+sizeof(qword)*6], rcx + mov qword [rbp+sizeof(qword)*7], rdx + + ret +ENDFUNC mred_16 + + + +mred_short DQ mred_5 - mred_short + DQ mred_6 - mred_short + DQ mred_7 - mred_short + DQ mred_8 - mred_short + DQ mred_9 - mred_short + DQ mred_10 - mred_short + DQ mred_11 - mred_short + DQ mred_12 - mred_short + DQ mred_13 - mred_short + DQ mred_14 - mred_short + DQ mred_15 - mred_short + DQ mred_16 - mred_short + +mred8x_start DQ mred8x1_start - mred8x_start + DQ mred8x2_start - mred8x_start + DQ mred8x3_start - mred8x_start + DQ mred8x4_start - mred8x_start + DQ mred8x5_start - mred8x_start + DQ mred8x6_start - mred8x_start + DQ mred8x7_start - mred8x_start diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmulx.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmulx.inc new file mode 100644 index 0000000..f3689a6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpmulx.inc @@ -0,0 +1,154 @@ +;=============================================================================== +; Copyright (C) 2013 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: EM64T Cryptography Primitive. +; Emulation of Intel(R) instructions MULX, ADCX, ADOX (for debug only) +; +; +%ifndef _PCPMULX_INC_ +%assign _PCPMULX_INC_ 1 + +%ifndef _EMULATION_ +%macro gsmulx 3.nolist + %xdefine %%resH %1 + %xdefine %%resL %2 + %xdefine %%src %3 + + mulx %%resH,%%resL,%%src +%endmacro + +%endif + +%ifdef _EMULATION_ +%macro gsmulx 3.nolist + %xdefine %%resH %1 + %xdefine %%resL %2 + %xdefine %%src %3 + + pushf ;; store flags + + sub rsp, sizeof(qword)*4 + mov [rsp-sizeof(qword)*3], rax ;; store RAX + mov [rsp-sizeof(qword)*2], rdx ;; store RDX + mov rax,rdx + mov rdx, %%src + + mul rdx + + mov [rsp-sizeof(qword)*1], rax ;; store Low product + mov [rsp-sizeof(qword)*0], rdx ;; store Hig product + + mov rax, [rsp-sizeof(qword)*3] ;; re-store RAX + mov rdx, [rsp-sizeof(qword)*2] ;; re-store RDX + mov %%resL, [rsp-sizeof(qword)*1];; load Low product + mov %%resH, [rsp-sizeof(qword)*0];; load Hig product + add rsp, sizeof(qword)*4 + + popf ;; re-store flags +%endmacro + +%endif + +%ifndef _EMULATION_ +%macro gsadcx 2.nolist + %xdefine %%rdst %1 + %xdefine %%rsrc %2 + + adcx %%rdst, %%rsrc +%endmacro + +%endif + +%ifdef _EMULATION_ +%macro gsadcx 2.nolist + %xdefine %%rdst %1 + %xdefine %%src %2 + + push %%rdst ;; slot for result + push rax ;; save rax + pushfq ;; flags before adc + + adc %%rdst, %%src + mov [rsp+2*sizeof(qword)], %%rdst + + pushfq ;; rsrc = flags after operation + pop rax + and rax, 1 ;; cf after operation + and qword [rsp], (-2) ;; clear cf before operation + or [rsp], rax ;; new psw + popfq + + pop rax + pop %%rdst +%endmacro + +%endif + +%ifndef _EMULATION_ +%macro gsadox 2.nolist + %xdefine %%rdst %1 + %xdefine %%rsrc %2 + + adox %%rdst, %%rsrc +%endmacro + +%endif + +%ifdef _EMULATION_ +%macro gsadox 2.nolist + %xdefine %%rdst %1 + %xdefine %%src %2 + + push %%rdst + push rax ;; save rax + + pushfq ;; rax = flags before adc + mov rax, [rsp] + and rax, 800h ;; of + xor [rsp], rax ;; clear of + + shr rax, 11 ;; mov of to cf position + push rax ;; new psw + popfq + +%ifidni %%src,rax + mov rax, [rsp+sizeof(qword)] +%endif +%ifidni %%rdst,rax + mov %%rdst, [rsp+2*sizeof(qword)] +%endif + + adc %%rdst, %%src + mov [rsp+2*sizeof(qword)], %%rdst + + pushfq ;; rsrc = flags after operation + pop rax + and rax, 1 ;; cf after operation + + shl rax, 11 ;; mov cf into of position + or [rsp], rax ;; new psw + popfq + + pop rax + pop %%rdst +%endmacro + +%endif + +%endif ;; _PCPMULX_INC_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpp192r1funcs_montas.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpp192r1funcs_montas.asm new file mode 100644 index 0000000..a47deb3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpp192r1funcs_montas.asm @@ -0,0 +1,1030 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; secp p192r1 specific implementation +; + + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if _IPP32E >= _IPP32E_M7 + +%assign _xEMULATION_ 1 +%assign _ADCX_ADOX_ 1 + +segment .text align=IPP_ALIGN_FACTOR + + +align IPP_ALIGN_FACTOR + +;; The p192r1 polynomial +Lpoly DQ 0FFFFFFFFFFFFFFFFh,0FFFFFFFFFFFFFFFEh,0FFFFFFFFFFFFFFFFh + +;; 2^(192*2) mod P precomputed for p192r1 polynomial +LRR DQ 00000000000000001h,00000000000000002h,00000000000000001h + +LOne DD 1,1,1,1,1,1,1,1 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p192r1_mul_by_2(uint64_t res[3], uint64_t a[3]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p192r1_mul_by_2,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12 + USES_XMM + COMP_ABI 2 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 + + xor t3, t3 + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + + shld t3, a2, 1 + shld a2, a1, 1 + shld a1, a0, 1 + shl a0, 1 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb t3, 0 + + cmovz a0, t0 + cmovz a1, t1 + cmovz a2, t2 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + + REST_XMM + REST_GPR + ret +ENDFUNC p192r1_mul_by_2 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p192r1_div_by_2(uint64_t res[3], uint64_t a[3]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p192r1_div_by_2,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13 + USES_XMM + COMP_ABI 2 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + + xor t3, t3 + xor r13, r13 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + + add t0, qword [rel Lpoly+sizeof(qword)*0] + adc t1, qword [rel Lpoly+sizeof(qword)*1] + adc t2, qword [rel Lpoly+sizeof(qword)*2] + adc t3, 0 + test a0, 1 + + cmovnz a0, t0 + cmovnz a1, t1 + cmovnz a2, t2 + cmovnz r13,t3 + + shrd a0, a1, 1 + shrd a1, a2, 1 + shrd a2, r13,1 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + + REST_XMM + REST_GPR + ret +ENDFUNC p192r1_div_by_2 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p192r1_mul_by_3(uint64_t res[3], uint64_t a[3]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p192r1_mul_by_3,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12 + USES_XMM + COMP_ABI 2 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 + + xor t3, t3 + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + + shld t3, a2, 1 + shld a2, a1, 1 + shld a1, a0, 1 + shl a0, 1 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb t3, 0 + + cmovz a0, t0 + cmovz a1, t1 + cmovz a2, t2 + + xor t3, t3 + add a0, qword [rsi+sizeof(qword)*0] + adc a1, qword [rsi+sizeof(qword)*1] + adc a2, qword [rsi+sizeof(qword)*2] + adc t3, 0 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb t3, 0 + + cmovz a0, t0 + cmovz a1, t1 + cmovz a2, t2 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + + REST_XMM + REST_GPR + ret +ENDFUNC p192r1_mul_by_3 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p192r1_add(uint64_t res[3], uint64_t a[3], uint64_t b[3]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p192r1_add,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12 + USES_XMM + COMP_ABI 3 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 + + xor t3, t3 + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + + add a0, qword [rdx+sizeof(qword)*0] + adc a1, qword [rdx+sizeof(qword)*1] + adc a2, qword [rdx+sizeof(qword)*2] + adc t3, 0 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb t3, 0 + + cmovz a0, t0 + cmovz a1, t1 + cmovz a2, t2 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + + REST_XMM + REST_GPR + ret +ENDFUNC p192r1_add + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p192r1_sub(uint64_t res[3], uint64_t a[3], uint64_t b[3]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p192r1_sub,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12 + USES_XMM + COMP_ABI 3 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 + + xor t3, t3 + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + + sub a0, qword [rdx+sizeof(qword)*0] + sbb a1, qword [rdx+sizeof(qword)*1] + sbb a2, qword [rdx+sizeof(qword)*2] + sbb t3, 0 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + + add t0, qword [rel Lpoly+sizeof(qword)*0] + adc t1, qword [rel Lpoly+sizeof(qword)*1] + adc t2, qword [rel Lpoly+sizeof(qword)*2] + test t3, t3 + + cmovnz a0, t0 + cmovnz a1, t1 + cmovnz a2, t2 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + + REST_XMM + REST_GPR + ret +ENDFUNC p192r1_sub + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p192r1_neg(uint64_t res[3], uint64_t a[3]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p192r1_neg,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12 + USES_XMM + COMP_ABI 2 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 + + xor t3, t3 + + xor a0, a0 + xor a1, a1 + xor a2, a2 + + sub a0, qword [rsi+sizeof(qword)*0] + sbb a1, qword [rsi+sizeof(qword)*1] + sbb a2, qword [rsi+sizeof(qword)*2] + sbb t3, 0 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + + add t0, qword [rel Lpoly+sizeof(qword)*0] + adc t1, qword [rel Lpoly+sizeof(qword)*1] + adc t2, qword [rel Lpoly+sizeof(qword)*2] + test t3, t3 + + cmovnz a0, t0 + cmovnz a1, t1 + cmovnz a2, t2 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + + REST_XMM + REST_GPR + ret +ENDFUNC p192r1_neg + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p192r1_mul_montl(uint64_t res[3], uint64_t a[3], uint64_t b[3]); +; void p192r1_mul_montx(uint64_t res[3], uint64_t a[3], uint64_t b[3]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; +; product = {p3,p2,p1,p0}, p4=0 +; product += 2^192*p0 -2^64*p0 -p0 +; +; on entry p4=0 +; on exit p0=0 +; +%macro p192r1_mul_redstep 5.nolist + %xdefine %%p4 %1 + %xdefine %%p3 %2 + %xdefine %%p2 %3 + %xdefine %%p1 %4 + %xdefine %%p0 %5 + + sub %%p1, %%p0 + sbb %%p2, 0 + sbb %%p0, 0 + add %%p3, %%p0 + adc %%p4, 0 + xor %%p0, %%p0 +%endmacro + +align IPP_ALIGN_FACTOR +p192r1_mmull: + +%xdefine acc0 r8 +%xdefine acc1 r9 +%xdefine acc2 r10 +%xdefine acc3 r11 +%xdefine acc4 r12 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx + +; rdi assumed as result +%xdefine aPtr rsi +%xdefine bPtr rbx + + xor acc4, acc4 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[0] + mov rax, qword [bPtr+sizeof(qword)*0] + mul qword [aPtr+sizeof(qword)*0] + mov acc0, rax + mov acc1, rdx + + mov rax, qword [bPtr+sizeof(qword)*0] + mul qword [aPtr+sizeof(qword)*1] + add acc1, rax + adc rdx, 0 + mov acc2, rdx + + mov rax, qword [bPtr+sizeof(qword)*0] + mul qword [aPtr+sizeof(qword)*2] + add acc2, rax + adc rdx, 0 + mov acc3, rdx + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 0 + p192r1_mul_redstep acc4,acc3,acc2,acc1,acc0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[1] + mov rax, qword [bPtr+sizeof(qword)*1] + mul qword [aPtr+sizeof(qword)*0] + add acc1, rax + adc rdx, 0 + mov rcx, rdx + + mov rax, qword [bPtr+sizeof(qword)*1] + mul qword [aPtr+sizeof(qword)*1] + add acc2, rcx + adc rdx, 0 + add acc2, rax + adc rdx, 0 + mov rcx, rdx + + mov rax, qword [bPtr+sizeof(qword)*1] + mul qword [aPtr+sizeof(qword)*2] + add acc3, rcx + adc rdx, 0 + add acc3, rax + adc acc4, rdx + adc acc0, 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 1 + p192r1_mul_redstep acc0,acc4,acc3,acc2,acc1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[2] + mov rax, qword [bPtr+sizeof(qword)*2] + mul qword [aPtr+sizeof(qword)*0] + add acc2, rax + adc rdx, 0 + mov rcx, rdx + + mov rax, qword [bPtr+sizeof(qword)*2] + mul qword [aPtr+sizeof(qword)*1] + add acc3, rcx + adc rdx, 0 + add acc3, rax + adc rdx, 0 + mov rcx, rdx + + mov rax, qword [bPtr+sizeof(qword)*2] + mul qword [aPtr+sizeof(qword)*2] + add acc4, rcx + adc rdx, 0 + add acc4, rax + adc acc0, rdx + adc acc1, 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 2 (final) + p192r1_mul_redstep acc1,acc0,acc4,acc3,acc2 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov t0, acc3 ;; copy reducted result + mov t1, acc4 + mov t2, acc0 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] ;; test %if it exceeds prime value + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb acc1, 0 + + cmovnc acc3, t0 + cmovnc acc4, t1 + cmovnc acc0, t2 + + mov qword [rdi+sizeof(qword)*0], acc3 + mov qword [rdi+sizeof(qword)*1], acc4 + mov qword [rdi+sizeof(qword)*2], acc0 + + ret + +%if (_IPP32E >= _IPP32E_L9) +align IPP_ALIGN_FACTOR +p192r1_mmulx: + +%xdefine acc0 r8 +%xdefine acc1 r9 +%xdefine acc2 r10 +%xdefine acc3 r11 +%xdefine acc4 r12 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx + +; rdi assumed as result +%xdefine aPtr rsi +%xdefine bPtr rbx + + xor acc4, acc4 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[0] + xor rdx, rdx + mov rdx, qword [bPtr+sizeof(qword)*0] + mulx acc1,acc0, qword [aPtr+sizeof(qword)*0] + mulx acc2,t2, qword [aPtr+sizeof(qword)*1] + add acc1,t2 + mulx acc3,t2, qword [aPtr+sizeof(qword)*2] + adc acc2,t2 + adc acc3,0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 0 + p192r1_mul_redstep acc4,acc3,acc2,acc1,acc0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[1] + mov rdx, qword [bPtr+sizeof(qword)*1] + + mulx t0, t2, qword [aPtr+sizeof(qword)*0] + adcx acc1, t2 + adox acc2, t0 + + mulx t0, t2, qword [aPtr+sizeof(qword)*1] + adcx acc2, t2 + adox acc3, t0 + + mulx t0, t2, qword [aPtr+sizeof(qword)*2] + adcx acc3, t2 + adox acc4, t0 + + adcx acc4, acc0 + adox acc0, acc0 + adc acc0, 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 1 + p192r1_mul_redstep acc0,acc4,acc3,acc2,acc1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[2] + mov rdx, qword [bPtr+sizeof(qword)*2] + + mulx t0, t2, qword [aPtr+sizeof(qword)*0] + adcx acc2, t2 + adox acc3, t0 + + mulx t0, t2, qword [aPtr+sizeof(qword)*1] + adcx acc3, t2 + adox acc4, t0 + + mulx t0, t2, qword [aPtr+sizeof(qword)*2] + adcx acc4, t2 + adox acc0, t0 + + adcx acc0, acc1 + adox acc1, acc1 + adc acc1, 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 2 (final) + p192r1_mul_redstep acc1,acc0,acc4,acc3,acc2 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov t0, acc3 ;; copy reducted result + mov t1, acc4 + mov t2, acc0 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] ;; test %if it exceeds prime value + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb acc1, 0 + + cmovnc acc3, t0 + cmovnc acc4, t1 + cmovnc acc0, t2 + + mov qword [rdi+sizeof(qword)*0], acc3 + mov qword [rdi+sizeof(qword)*1], acc4 + mov qword [rdi+sizeof(qword)*2], acc0 + + ret +%endif + +align IPP_ALIGN_FACTOR +IPPASM p192r1_mul_montl,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbx,rsi,rdi,r12 + USES_XMM + COMP_ABI 3 + +%xdefine bPtr rbx + + mov bPtr, rdx + call p192r1_mmull + + REST_XMM + REST_GPR + ret +ENDFUNC p192r1_mul_montl + +%if _IPP32E >= _IPP32E_L9 +align IPP_ALIGN_FACTOR +IPPASM p192r1_mul_montx,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbx,rsi,rdi,r12 + USES_XMM + COMP_ABI 3 + +%xdefine bPtr rbx + + mov bPtr, rdx + call p192r1_mmulx + + REST_XMM + REST_GPR + ret +ENDFUNC p192r1_mul_montx + +%endif ;; _IPP32E_L9 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p192r1_to_mont(uint64_t res[3], uint64_t a[3]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p192r1_to_mont,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbx,rsi,rdi,r12 + USES_XMM + COMP_ABI 2 + + lea rbx, [rel LRR] + call p192r1_mmull + REST_XMM + REST_GPR + ret +ENDFUNC p192r1_to_mont + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p192r1_sqr_montl(uint64_t res[3], uint64_t a[3]); +; void p192r1_sqr_montx(uint64_t res[3], uint64_t a[3]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; on entry e = expasion (previous step) +; on exit p0= expasion (next step) +; +%macro p192r1_prod_redstep 5.nolist + %xdefine %%e %1 + %xdefine %%p3 %2 + %xdefine %%p2 %3 + %xdefine %%p1 %4 + %xdefine %%p0 %5 + + sub %%p1, %%p0 + sbb %%p2, 0 + sbb %%p0, 0 + add %%p3, %%p0 + mov %%p0, dword 0 + adc %%p0, 0 + + %ifnempty %%e + add %%p3, %%e + adc %%p0, 0 + %endif +%endmacro + +align IPP_ALIGN_FACTOR +IPPASM p192r1_sqr_montl,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13 + USES_XMM + COMP_ABI 2 + +%xdefine acc0 r8 +%xdefine acc1 r9 +%xdefine acc2 r10 +%xdefine acc3 r11 +%xdefine acc4 r12 +%xdefine acc5 r13 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov t2, qword [aPtr+sizeof(qword)*0] + mov rax,qword [aPtr+sizeof(qword)*1] + mul t2 + mov acc1, rax + mov acc2, rdx + mov rax,qword [aPtr+sizeof(qword)*2] + mul t2 + add acc2, rax + adc rdx, 0 + mov acc3, rdx + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov t2, qword [aPtr+sizeof(qword)*1] + mov rax,qword [aPtr+sizeof(qword)*2] + mul t2 + add acc3, rax + adc rdx, 0 + mov acc4, rdx + + xor acc5, acc5 + shld acc5, acc4, 1 + shld acc4, acc3, 1 + shld acc3, acc2, 1 + shld acc2, acc1, 1 + shl acc1, 1 + + mov rax,qword [aPtr+sizeof(qword)*0] + mul rax + mov acc0, rax + add acc1, rdx + adc acc2, 0 + mov rax,qword [aPtr+sizeof(qword)*1] + mul rax + add acc2, rax + adc acc3, rdx + adc acc4, 0 + mov rax,qword [aPtr+sizeof(qword)*2] + mul rax + add acc4, rax + adc acc5, rdx + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + p192r1_prod_redstep ,acc3,acc2,acc1,acc0 + p192r1_prod_redstep acc0,acc4,acc3,acc2,acc1 + p192r1_prod_redstep acc1,acc5,acc4,acc3,acc2 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + mov t0, acc3 + mov t1, acc4 + mov t2, acc5 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb acc2, 0 + + cmovnc acc3, t0 + cmovnc acc4, t1 + cmovnc acc5, t2 + + mov qword [rdi+sizeof(qword)*0], acc3 + mov qword [rdi+sizeof(qword)*1], acc4 + mov qword [rdi+sizeof(qword)*2], acc5 + + REST_XMM + REST_GPR + ret +ENDFUNC p192r1_sqr_montl + +%if _IPP32E >= _IPP32E_L9 +align IPP_ALIGN_FACTOR +IPPASM p192r1_sqr_montx,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbp,rbx,rsi,rdi,r12,r13 + USES_XMM + COMP_ABI 2 + +%xdefine acc0 r8 +%xdefine acc1 r9 +%xdefine acc2 r10 +%xdefine acc3 r11 +%xdefine acc4 r12 +%xdefine acc5 r13 + +%xdefine t0 rcx +%xdefine t1 rax +%xdefine t2 rdx + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov rdx, qword [aPtr+sizeof(qword)*0] + mulx acc2, acc1, qword [aPtr+sizeof(qword)*1] + mulx acc3, t0, qword [aPtr+sizeof(qword)*2] + add acc2, t0 + adc acc3, 0 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov rdx, qword [aPtr+sizeof(qword)*1] + mulx acc4, t0, qword [aPtr+sizeof(qword)*2] + add acc3, t0 + adc acc4, 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + xor acc5, acc5 + shld acc5, acc4, 1 + shld acc4, acc3, 1 + shld acc3, acc2, 1 + shld acc2, acc1, 1 + shl acc1, 1 + + xor acc0, acc0 + mov rdx, qword [aPtr+sizeof(qword)*0] + mulx t1, acc0, rdx + adcx acc1, t1 + mov rdx, qword [aPtr+sizeof(qword)*1] + mulx t1, t0, rdx + adcx acc2, t0 + adcx acc3, t1 + mov rdx, qword [aPtr+sizeof(qword)*2] + mulx t1, t0, rdx + adcx acc4, t0 + adcx acc5, t1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + p192r1_prod_redstep ,acc3,acc2,acc1,acc0 + p192r1_prod_redstep acc0,acc4,acc3,acc2,acc1 + p192r1_prod_redstep acc1,acc5,acc4,acc3,acc2 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + mov t0, acc3 + mov t1, acc4 + mov t2, acc5 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb acc2, 0 + + cmovnc acc3, t0 + cmovnc acc4, t1 + cmovnc acc5, t2 + + mov qword [rdi+sizeof(qword)*0], acc3 + mov qword [rdi+sizeof(qword)*1], acc4 + mov qword [rdi+sizeof(qword)*2], acc5 + + REST_XMM + REST_GPR + ret +ENDFUNC p192r1_sqr_montx + +%endif ;; _IPP32E_L9 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p192r1_mont_back(uint64_t res[3], uint64_t a[3]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p192r1_mont_back,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12 + USES_XMM + COMP_ABI 2 + +%xdefine acc0 r8 +%xdefine acc1 r9 +%xdefine acc2 r10 +%xdefine acc3 r11 +%xdefine acc4 r12 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 rsi + + mov acc2, qword [rsi+sizeof(qword)*0] + mov acc3, qword [rsi+sizeof(qword)*1] + mov acc4, qword [rsi+sizeof(qword)*2] + xor acc0, acc0 + xor acc1, acc1 + + p192r1_mul_redstep acc1,acc0,acc4,acc3,acc2 + p192r1_mul_redstep acc2,acc1,acc0,acc4,acc3 + p192r1_mul_redstep acc3,acc2,acc1,acc0,acc4 + + mov t0, acc0 + mov t1, acc1 + mov t2, acc2 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb acc4, 0 + + cmovnc acc0, t0 + cmovnc acc1, t1 + cmovnc acc2, t2 + + mov qword [rdi+sizeof(qword)*0], acc0 + mov qword [rdi+sizeof(qword)*1], acc1 + mov qword [rdi+sizeof(qword)*2], acc2 + + REST_XMM + REST_GPR + ret +ENDFUNC p192r1_mont_back + + +%ifndef _DISABLE_ECP_192R1_HARDCODED_BP_TBL_ +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p192r1_select_ap_w7(AF_POINT *val, const AF_POINT *in_t, int index); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p192r1_select_ap_w7,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi + USES_XMM xmm6,xmm7,xmm8,xmm9 + COMP_ABI 3 + +%xdefine val rdi +%xdefine in_t rsi +%xdefine idx edx + +%xdefine ONE xmm0 +%xdefine INDEX xmm1 + +%xdefine Ra xmm2 +%xdefine Rb xmm3 +%xdefine Rc xmm4 + +%xdefine T0a xmm5 +%xdefine T0b xmm6 +%xdefine T0c xmm7 + +%xdefine M0 xmm8 +%xdefine TMP0 xmm9 + + movdqa ONE, oword [rel LOne] + + pxor Ra, Ra + pxor Rb, Rb + pxor Rc, Rc + + movdqa M0, ONE + + movd INDEX, idx + pshufd INDEX, INDEX, 0 + + ; Skip index = 0, is implicictly infty -> load with offset -1 + mov rcx, dword 64 +.select_loop_sse_w7: + movdqa TMP0, M0 + pcmpeqd TMP0, INDEX + paddd M0, ONE + + movdqa T0a, oword [in_t+sizeof(oword)*0] + movdqa T0b, oword [in_t+sizeof(oword)*1] + movdqa T0c, oword [in_t+sizeof(oword)*2] + add in_t, sizeof(oword)*3 + + pand T0a, TMP0 + pand T0b, TMP0 + pand T0c, TMP0 + + por Ra, T0a + por Rb, T0b + por Rc, T0c + dec rcx + jnz .select_loop_sse_w7 + + movdqu oword [val+sizeof(oword)*0], Ra + movdqu oword [val+sizeof(oword)*1], Rb + movdqu oword [val+sizeof(oword)*2], Rc + + REST_XMM + REST_GPR + ret +ENDFUNC p192r1_select_ap_w7 + +%endif + +%endif ;; _IPP32E_M7 + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpp224r1funcs_montas.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpp224r1funcs_montas.asm new file mode 100644 index 0000000..ac2dba6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpp224r1funcs_montas.asm @@ -0,0 +1,1066 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; secp p224r1 specific implementation +; + + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_M7) + +%assign _xEMULATION_ 1 +%assign _ADCX_ADOX_ 1 + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR + +;; The p224r1 polynomial +Lpoly DQ 00000000000000001h,0ffffffff00000000h,0ffffffffffffffffh,000000000ffffffffh + +;; mont(1) +;; ffffffff00000000 ffffffffffffffff 0000000000000000 0000000000000000 + +;; 2^(2*224) mod P precomputed for p224r1 polynomial +LRR DQ 0ffffffff00000001h,0ffffffff00000000h,0fffffffe00000000h,000000000ffffffffh + +LOne DD 1,1,1,1,1,1,1,1 +LTwo DD 2,2,2,2,2,2,2,2 +LThree DD 3,3,3,3,3,3,3,3 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p224r1_mul_by_2(uint64_t res[4], uint64_t a[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p224r1_mul_by_2,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13 + USES_XMM + COMP_ABI 2 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 +%xdefine t4 r13 + + xor t4, t4 + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + + shld t4, a3, 1 + shld a3, a2, 1 + shld a2, a1, 1 + shld a1, a0, 1 + shl a0, 1 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + mov t3, a3 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb t3, qword [rel Lpoly+sizeof(qword)*3] + sbb t4, 0 + + cmovz a0, t0 + cmovz a1, t1 + cmovz a2, t2 + cmovz a3, t3 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + + REST_XMM + REST_GPR + ret +ENDFUNC p224r1_mul_by_2 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p224r1_div_by_2(uint64_t res[4], uint64_t a[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p224r1_div_by_2,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13,r14 + USES_XMM + COMP_ABI 2 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 +%xdefine t4 r13 + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + + xor t4, t4 + xor r14, r14 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + mov t3, a3 + + add t0, qword [rel Lpoly+sizeof(qword)*0] + adc t1, qword [rel Lpoly+sizeof(qword)*1] + adc t2, qword [rel Lpoly+sizeof(qword)*2] + adc t3, qword [rel Lpoly+sizeof(qword)*3] + adc t4, 0 + test a0, 1 + + cmovnz a0, t0 + cmovnz a1, t1 + cmovnz a2, t2 + cmovnz a3, t3 + cmovnz r14,t4 + + shrd a0, a1, 1 + shrd a1, a2, 1 + shrd a2, a3, 1 + shrd a3, r14,1 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + + REST_XMM + REST_GPR + ret +ENDFUNC p224r1_div_by_2 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p224r1_mul_by_3(uint64_t res[4], uint64_t a[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p224r1_mul_by_3,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13 + USES_XMM + COMP_ABI 2 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 +%xdefine t4 r13 + + xor t4, t4 + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + + shld t4, a3, 1 + shld a3, a2, 1 + shld a2, a1, 1 + shld a1, a0, 1 + shl a0, 1 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + mov t3, a3 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb t3, qword [rel Lpoly+sizeof(qword)*3] + sbb t4, 0 + + cmovz a0, t0 + cmovz a1, t1 + cmovz a2, t2 + cmovz a3, t3 + + xor t4, t4 + add a0, qword [rsi+sizeof(qword)*0] + adc a1, qword [rsi+sizeof(qword)*1] + adc a2, qword [rsi+sizeof(qword)*2] + adc a3, qword [rsi+sizeof(qword)*3] + adc t4, 0 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + mov t3, a3 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb t3, qword [rel Lpoly+sizeof(qword)*3] + sbb t4, 0 + + cmovz a0, t0 + cmovz a1, t1 + cmovz a2, t2 + cmovz a3, t3 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + + REST_XMM + REST_GPR + ret +ENDFUNC p224r1_mul_by_3 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p224r1_add(uint64_t res[4], uint64_t a[4], uint64_t b[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p224r1_add,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13 + USES_XMM + COMP_ABI 3 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 +%xdefine t4 r13 + + xor t4, t4 + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + + add a0, qword [rdx+sizeof(qword)*0] + adc a1, qword [rdx+sizeof(qword)*1] + adc a2, qword [rdx+sizeof(qword)*2] + adc a3, qword [rdx+sizeof(qword)*3] + adc t4, 0 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + mov t3, a3 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb t3, qword [rel Lpoly+sizeof(qword)*3] + sbb t4, 0 + + cmovz a0, t0 + cmovz a1, t1 + cmovz a2, t2 + cmovz a3, t3 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + + REST_XMM + REST_GPR + ret +ENDFUNC p224r1_add + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p224r1_sub(uint64_t res[4], uint64_t a[4], uint64_t b[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p224r1_sub,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13 + USES_XMM + COMP_ABI 3 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 +%xdefine t4 r13 + + xor t4, t4 + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + + sub a0, qword [rdx+sizeof(qword)*0] + sbb a1, qword [rdx+sizeof(qword)*1] + sbb a2, qword [rdx+sizeof(qword)*2] + sbb a3, qword [rdx+sizeof(qword)*3] + sbb t4, 0 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + mov t3, a3 + + add t0, qword [rel Lpoly+sizeof(qword)*0] + adc t1, qword [rel Lpoly+sizeof(qword)*1] + adc t2, qword [rel Lpoly+sizeof(qword)*2] + adc t3, qword [rel Lpoly+sizeof(qword)*3] + test t4, t4 + + cmovnz a0, t0 + cmovnz a1, t1 + cmovnz a2, t2 + cmovnz a3, t3 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + + REST_XMM + REST_GPR + ret +ENDFUNC p224r1_sub + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p224r1_neg(uint64_t res[4], uint64_t a[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p224r1_neg,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13 + USES_XMM + COMP_ABI 2 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 +%xdefine t4 r13 + + xor t4, t4 + + xor a0, a0 + xor a1, a1 + xor a2, a2 + xor a3, a3 + + sub a0, qword [rsi+sizeof(qword)*0] + sbb a1, qword [rsi+sizeof(qword)*1] + sbb a2, qword [rsi+sizeof(qword)*2] + sbb a3, qword [rsi+sizeof(qword)*3] + sbb t4, 0 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + mov t3, a3 + + add t0, qword [rel Lpoly+sizeof(qword)*0] + adc t1, qword [rel Lpoly+sizeof(qword)*1] + adc t2, qword [rel Lpoly+sizeof(qword)*2] + adc t3, qword [rel Lpoly+sizeof(qword)*3] + test t4, t4 + + cmovnz a0, t0 + cmovnz a1, t1 + cmovnz a2, t2 + cmovnz a3, t3 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + + REST_XMM + REST_GPR + ret +ENDFUNC p224r1_neg + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p224r1_mul_montl(uint64_t res[4], uint64_t a[4], uint64_t b[4]); +; void p224r1_mul_montx(uint64_t res[4], uint64_t a[4], uint64_t b[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; was working on GFp functionality the problem (in reduction spep) has been found +;; 1) "sqr" impementation has been changed by "mul" +;; 2) fortunately "mont_back" stay as is because of operand zero extensioned + +; on entry p5=0 +; on exit p0=0 +; +%macro p224r1_prod_redstep 5.nolist + %xdefine %%p4 %1 + %xdefine %%p3 %2 + %xdefine %%p2 %3 + %xdefine %%p1 %4 + %xdefine %%p0 %5 + + neg %%p0 + mov t2, %%p0 + mov t3, %%p0 + xor t0, t0 + xor t1, t1 + shr t3, 32 + shl t2, 32 + sub t0, t2 + sbb t1, t3 + sbb t2, 0 + sbb t3, 0 + + neg %%p0 + adc %%p1, t0 + adc %%p2, t1 + adc %%p3, t2 + adc %%p4, t3 +%endmacro + +align IPP_ALIGN_FACTOR +p224r1_mmull: + +%xdefine acc0 r8 +%xdefine acc1 r9 +%xdefine acc2 r10 +%xdefine acc3 r11 +%xdefine acc4 r12 +%xdefine acc5 r13 +%xdefine acc6 r14 +%xdefine acc7 r15 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 rbp +%xdefine t4 rbx + +; rdi assumed as result +%xdefine aPtr rsi +%xdefine bPtr rbx + + xor acc5, acc5 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[0] + mov rax, qword [bPtr+sizeof(qword)*0] + mul qword [aPtr+sizeof(qword)*0] + mov acc0, rax + mov acc1, rdx + + mov rax, qword [bPtr+sizeof(qword)*0] + mul qword [aPtr+sizeof(qword)*1] + add acc1, rax + adc rdx, 0 + mov acc2, rdx + + mov rax, qword [bPtr+sizeof(qword)*0] + mul qword [aPtr+sizeof(qword)*2] + add acc2, rax + adc rdx, 0 + mov acc3, rdx + + mov rax, qword [bPtr+sizeof(qword)*0] + mul qword [aPtr+sizeof(qword)*3] + add acc3, rax + adc rdx, 0 + mov acc4, rdx + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 0 + p224r1_prod_redstep acc4,acc3,acc2,acc1,acc0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[1] + mov rax, qword [bPtr+sizeof(qword)*1] + mul qword [aPtr+sizeof(qword)*0] + add acc1, rax + adc rdx, 0 + mov rcx, rdx + + mov rax, qword [bPtr+sizeof(qword)*1] + mul qword [aPtr+sizeof(qword)*1] + add acc2, rcx + adc rdx, 0 + add acc2, rax + adc rdx, 0 + mov rcx, rdx + + mov rax, qword [bPtr+sizeof(qword)*1] + mul qword [aPtr+sizeof(qword)*2] + add acc3, rcx + adc rdx, 0 + add acc3, rax + adc rdx, 0 + mov rcx, rdx + + mov rax, qword [bPtr+sizeof(qword)*1] + mul qword [aPtr+sizeof(qword)*3] + add acc4, rcx + adc rdx, 0 + add acc4, rax + adc rdx, 0 + mov acc5, rdx + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 1 + p224r1_prod_redstep acc5,acc4,acc3,acc2,acc1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[2] + mov rax, qword [bPtr+sizeof(qword)*2] + mul qword [aPtr+sizeof(qword)*0] + add acc2, rax + adc rdx, 0 + mov rcx, rdx + + mov rax, qword [bPtr+sizeof(qword)*2] + mul qword [aPtr+sizeof(qword)*1] + add acc3, rcx + adc rdx, 0 + add acc3, rax + adc rdx, 0 + mov rcx, rdx + + mov rax, qword [bPtr+sizeof(qword)*2] + mul qword [aPtr+sizeof(qword)*2] + add acc4, rcx + adc rdx, 0 + add acc4, rax + adc rdx, 0 + mov rcx, rdx + + mov rax, qword [bPtr+sizeof(qword)*2] + mul qword [aPtr+sizeof(qword)*3] + add acc5, rcx + adc rdx, 0 + add acc5, rax + adc rdx, 0 + mov acc6, rdx + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 2 + p224r1_prod_redstep acc6,acc5,acc4,acc3,acc2 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[3] + mov rax, qword [bPtr+sizeof(qword)*3] + mul qword [aPtr+sizeof(qword)*0] + add acc3, rax + adc rdx, 0 + mov rcx, rdx + + mov rax, qword [bPtr+sizeof(qword)*3] + mul qword [aPtr+sizeof(qword)*1] + add acc4, rcx + adc rdx, 0 + add acc4, rax + adc rdx, 0 + mov rcx, rdx + + mov rax, qword [bPtr+sizeof(qword)*3] + mul qword [aPtr+sizeof(qword)*2] + add acc5, rcx + adc rdx, 0 + add acc5, rax + adc rdx, 0 + mov rcx, rdx + + mov rax, qword [bPtr+sizeof(qword)*3] + mul qword [aPtr+sizeof(qword)*3] + add acc6, rcx + adc rdx, 0 + add acc6, rax + adc rdx, 0 + mov acc7, rdx + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 3 (final) + p224r1_prod_redstep acc7,acc6,acc5,acc4,acc3 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + mov t0, qword [rel Lpoly+sizeof(qword)*0] + mov t1, qword [rel Lpoly+sizeof(qword)*1] + mov t2, qword [rel Lpoly+sizeof(qword)*2] + mov t3, qword [rel Lpoly+sizeof(qword)*3] + + mov acc0, acc4 ;; copy reducted result + mov acc1, acc5 + mov acc2, acc6 + mov acc3, acc7 + + sub acc4, t0 ;; test %if it exceeds prime value + sbb acc5, t1 + sbb acc6, t2 + sbb acc7, t3 + + cmovc acc4, acc0 + cmovc acc5, acc1 + cmovc acc6, acc2 + cmovc acc7, acc3 + + mov qword [rdi+sizeof(qword)*0], acc4 + mov qword [rdi+sizeof(qword)*1], acc5 + mov qword [rdi+sizeof(qword)*2], acc6 + mov qword [rdi+sizeof(qword)*3], acc7 + + ret + +%if _IPP32E >= _IPP32E_L9 +align IPP_ALIGN_FACTOR +p224r1_mmulx: + +%xdefine acc0 r8 +%xdefine acc1 r9 +%xdefine acc2 r10 +%xdefine acc3 r11 +%xdefine acc4 r12 +%xdefine acc5 r13 +%xdefine acc6 r14 +%xdefine acc7 r15 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 rbp +%xdefine t4 rbx + +; rdi assumed as result +%xdefine aPtr rsi +%xdefine bPtr rbx + + xor acc5, acc5 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[0] + mov rdx, qword [bPtr+sizeof(qword)*0] + mulx acc1,acc0, qword [aPtr+sizeof(qword)*0] + mulx acc2,t2, qword [aPtr+sizeof(qword)*1] + add acc1,t2 + mulx acc3,t2, qword [aPtr+sizeof(qword)*2] + adc acc2,t2 + mulx acc4,t2, qword [aPtr+sizeof(qword)*3] + adc acc3,t2 + adc acc4,0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 0 + p224r1_prod_redstep acc4,acc3,acc2,acc1,acc0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[1] + mov rdx, qword [bPtr+sizeof(qword)*1] + xor t0, t0 + + mulx t3, t2, qword [aPtr+sizeof(qword)*0] + adcx acc1, t2 + adox acc2, t3 + + mulx t3, t2, qword [aPtr+sizeof(qword)*1] + adcx acc2, t2 + adox acc3, t3 + + mulx t3, t2, qword [aPtr+sizeof(qword)*2] + adcx acc3, t2 + adox acc4, t3 + + mulx acc5, t2, qword [aPtr+sizeof(qword)*3] + adcx acc4, t2 + adox acc5, t0 + adc acc5, 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 1 + p224r1_prod_redstep acc5,acc4,acc3,acc2,acc1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[2] + mov rdx, qword [bPtr+sizeof(qword)*2] + xor t0, t0 + + mulx t3, t2, qword [aPtr+sizeof(qword)*0] + adcx acc2, t2 + adox acc3, t3 + + mulx t3, t2, qword [aPtr+sizeof(qword)*1] + adcx acc3, t2 + adox acc4, t3 + + mulx t3, t2, qword [aPtr+sizeof(qword)*2] + adcx acc4, t2 + adox acc5, t3 + + mulx acc6, t2, qword [aPtr+sizeof(qword)*3] + adcx acc5, t2 + adox acc6, t0 + adc acc6, 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 2 + p224r1_prod_redstep acc6,acc5,acc4,acc3,acc2 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[3] + mov rdx, qword [bPtr+sizeof(qword)*3] + xor t0, t0 + + mulx t3, t2, qword [aPtr+sizeof(qword)*0] + adcx acc3, t2 + adox acc4, t3 + + mulx t3, t2, qword [aPtr+sizeof(qword)*1] + adcx acc4, t2 + adox acc5, t3 + + mulx t3, t2, qword [aPtr+sizeof(qword)*2] + adcx acc5, t2 + adox acc6, t3 + + mulx acc7, t2, qword [aPtr+sizeof(qword)*3] + adcx acc6, t2 + adox acc7, t0 + adc acc7, 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 3 (final) + p224r1_prod_redstep acc7,acc6,acc5,acc4,acc3 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + mov t0, qword [rel Lpoly+sizeof(qword)*0] + mov t1, qword [rel Lpoly+sizeof(qword)*1] + mov t2, qword [rel Lpoly+sizeof(qword)*2] + mov t3, qword [rel Lpoly+sizeof(qword)*3] + + mov acc0, acc4 ;; copy reducted result + mov acc1, acc5 + mov acc2, acc6 + mov acc3, acc7 + + sub acc4, t0 ;; test %if it exceeds prime value + sbb acc5, t1 + sbb acc6, t2 + sbb acc7, t3 + + cmovc acc4, acc0 + cmovc acc5, acc1 + cmovc acc6, acc2 + cmovc acc7, acc3 + + mov qword [rdi+sizeof(qword)*0], acc4 + mov qword [rdi+sizeof(qword)*1], acc5 + mov qword [rdi+sizeof(qword)*2], acc6 + mov qword [rdi+sizeof(qword)*3], acc7 + + ret +%endif + +IPPASM p224r1_mul_montl,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbp,rbx,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 3 + +%xdefine bPtr rbx + + mov bPtr, rdx + call p224r1_mmull + + REST_XMM + REST_GPR + ret +ENDFUNC p224r1_mul_montl + +%if _IPP32E >= _IPP32E_L9 +align IPP_ALIGN_FACTOR +IPPASM p224r1_mul_montx,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbp,rbx,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 3 + +%xdefine bPtr rbx + + mov bPtr, rdx + call p224r1_mmulx + + REST_XMM + REST_GPR + ret +ENDFUNC p224r1_mul_montx + +%endif ;; _IPP32E_L9 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p224r1_to_mont(uint64_t res[4], uint64_t a[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p224r1_to_mont,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbp,rbx,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 2 + + lea rbx, [rel LRR] + call p224r1_mmull + REST_XMM + REST_GPR + ret +ENDFUNC p224r1_to_mont + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p224r1_sqr_montl(uint64_t res[4], uint64_t a[4]); +; void p224r1_sqr_montx(uint64_t res[4], uint64_t a[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p224r1_sqr_montl,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbp,rbx,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 2 + + mov rbx, rsi + call p224r1_mmull + + REST_XMM + REST_GPR + ret +ENDFUNC p224r1_sqr_montl + +%if _IPP32E >= _IPP32E_L9 +align IPP_ALIGN_FACTOR +IPPASM p224r1_sqr_montx,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbp,rbx,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 2 + + mov rbx, rsi + call p224r1_mmulx + + REST_XMM + REST_GPR + ret +ENDFUNC p224r1_sqr_montx + +%endif ;; _IPP32E_L9 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p224r1_mont_back(uint64_t res[4], uint64_t a[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p224r1_mont_back,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 2 + +%xdefine acc0 r8 +%xdefine acc1 r9 +%xdefine acc2 r10 +%xdefine acc3 r11 +%xdefine acc4 r12 +%xdefine acc5 r13 +%xdefine acc6 r14 +%xdefine acc7 r15 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 rsi + + mov acc0, qword [rsi+sizeof(qword)*0] + mov acc1, qword [rsi+sizeof(qword)*1] + mov acc2, qword [rsi+sizeof(qword)*2] + mov acc3, qword [rsi+sizeof(qword)*3] + xor acc4, acc4 + xor acc5, acc5 + xor acc6, acc6 + xor acc7, acc7 + + p224r1_prod_redstep acc4,acc3,acc2,acc1,acc0 + p224r1_prod_redstep acc5,acc4,acc3,acc2,acc1 + p224r1_prod_redstep acc6,acc5,acc4,acc3,acc2 + p224r1_prod_redstep acc7,acc6,acc5,acc4,acc3 + + mov acc0, acc4 + mov acc1, acc5 + mov acc2, acc6 + mov acc3, acc7 + + sub acc4, qword [rel Lpoly+sizeof(qword)*0] + sbb acc5, qword [rel Lpoly+sizeof(qword)*1] + sbb acc6, qword [rel Lpoly+sizeof(qword)*2] + sbb acc7, qword [rel Lpoly+sizeof(qword)*3] + + cmovc acc4, acc0 + cmovc acc5, acc1 + cmovc acc6, acc2 + cmovc acc7, acc3 + + mov qword [rdi+sizeof(qword)*0], acc4 + mov qword [rdi+sizeof(qword)*1], acc5 + mov qword [rdi+sizeof(qword)*2], acc6 + mov qword [rdi+sizeof(qword)*3], acc7 + + REST_XMM + REST_GPR + ret +ENDFUNC p224r1_mont_back + +%ifndef _DISABLE_ECP_224R1_HARDCODED_BP_TBL_ +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p224r1_select_ap_w7(AF_POINT *val, const AF_POINT *in_t, int index); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p224r1_select_ap_w7,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13 + USES_XMM xmm6,xmm7,xmm8,xmm9,xmm10,xmm11,xmm12,xmm13,xmm14,xmm15 + COMP_ABI 3 + +%xdefine val rdi +%xdefine in_t rsi +%xdefine idx edx + +%xdefine ONE xmm0 +%xdefine INDEX xmm1 + +%xdefine Ra xmm2 +%xdefine Rb xmm3 +%xdefine Rc xmm4 +%xdefine Rd xmm5 + +%xdefine M0 xmm8 +%xdefine T0a xmm9 +%xdefine T0b xmm10 +%xdefine T0c xmm11 +%xdefine T0d xmm12 +%xdefine TMP0 xmm15 + + movdqa ONE, oword [rel LOne] + + pxor Ra, Ra + pxor Rb, Rb + pxor Rc, Rc + pxor Rd, Rd + + movdqa M0, ONE + + movd INDEX, idx + pshufd INDEX, INDEX, 0 + + ; Skip index = 0, is implicictly infty -> load with offset -1 + mov rcx, dword 64 +.select_loop_sse_w7: + movdqa TMP0, M0 + pcmpeqd TMP0, INDEX + paddd M0, ONE + + movdqa T0a, oword [in_t+sizeof(oword)*0] + movdqa T0b, oword [in_t+sizeof(oword)*1] + movdqa T0c, oword [in_t+sizeof(oword)*2] + movdqa T0d, oword [in_t+sizeof(oword)*3] + add in_t, sizeof(oword)*4 + + pand T0a, TMP0 + pand T0b, TMP0 + pand T0c, TMP0 + pand T0d, TMP0 + + por Ra, T0a + por Rb, T0b + por Rc, T0c + por Rd, T0d + dec rcx + jnz .select_loop_sse_w7 + + movdqu oword [val+sizeof(oword)*0], Ra + movdqu oword [val+sizeof(oword)*1], Rb + movdqu oword [val+sizeof(oword)*2], Rc + movdqu oword [val+sizeof(oword)*3], Rd + + REST_XMM + REST_GPR + ret +ENDFUNC p224r1_select_ap_w7 + +%endif + +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpp256funcs_montas.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpp256funcs_montas.asm new file mode 100644 index 0000000..40675b7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpp256funcs_montas.asm @@ -0,0 +1,677 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; 256-bit modular arithmetic +; + +; +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_M7) + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; Ipp64u* gf256_add(Ipp64u res[4], Ipp64u a[4], Ipp64u b[4], Ipp64u poly[4]) +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM gf256_add,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13,r14 + USES_XMM + COMP_ABI 4 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 r12 +%xdefine t3 r13 +%xdefine t4 r14 + + xor t4, t4 + + mov a0, qword [rsi+sizeof(qword)*0] ; a[] = A + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + + add a0, qword [rdx+sizeof(qword)*0] ; a[] = a[]+b[] + adc a1, qword [rdx+sizeof(qword)*1] + adc a2, qword [rdx+sizeof(qword)*2] + adc a3, qword [rdx+sizeof(qword)*3] + adc t4, 0 + + mov t0, a0 ; save result t[] = a[] + mov t1, a1 + mov t2, a2 + mov t3, a3 + + sub t0, qword [rcx+sizeof(qword)*0] ; t[] = a[]+b[] -poly[] + sbb t1, qword [rcx+sizeof(qword)*1] + sbb t2, qword [rcx+sizeof(qword)*2] + sbb t3, qword [rcx+sizeof(qword)*3] + sbb t4, 0 + + cmovz a0, t0 ; A = a:t + cmovz a1, t1 + cmovz a2, t2 + cmovz a3, t3 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + + mov rax, rdi ; return pointer to result + + REST_XMM + REST_GPR + ret +ENDFUNC gf256_add + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; Ipp64u* gf256_sub(Ipp64u res[4], Ipp64u a[4], Ipp64u b[4], Ipp64u poly[4]) +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM gf256_sub,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13,r14 + USES_XMM + COMP_ABI 4 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 r12 +%xdefine t3 r13 +%xdefine t4 r14 + + xor t4, t4 + + mov a0, qword [rsi+sizeof(qword)*0] ; a[] = A + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + + sub a0, qword [rdx+sizeof(qword)*0] ; a[] = a[]-b[] + sbb a1, qword [rdx+sizeof(qword)*1] + sbb a2, qword [rdx+sizeof(qword)*2] + sbb a3, qword [rdx+sizeof(qword)*3] + sbb t4, 0 + + mov t0, a0 ; save result t[] = a[] + mov t1, a1 + mov t2, a2 + mov t3, a3 + + add t0, qword [rcx+sizeof(qword)*0] ; t[] = a[]-b[] +poly[] + adc t1, qword [rcx+sizeof(qword)*1] + adc t2, qword [rcx+sizeof(qword)*2] + adc t3, qword [rcx+sizeof(qword)*3] + test t4, t4 + + cmovnz a0, t0 ; A = a:t + cmovnz a1, t1 + cmovnz a2, t2 + cmovnz a3, t3 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + + mov rax, rdi ; return pointer to result + + REST_XMM + REST_GPR + ret +ENDFUNC gf256_sub + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; Ipp64u* gf256_neg(Ipp64u res[4], Ipp64u a[4], Ipp64u poly[4]) +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM gf256_neg,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13,r14 + USES_XMM + COMP_ABI 3 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 + +%xdefine t0 rax +%xdefine t1 rcx +%xdefine t2 r12 +%xdefine t3 r13 +%xdefine t4 r14 + + xor t4, t4 + + xor a0, a0 ; a[] = 0; + xor a1, a1 + xor a2, a2 + xor a3, a3 + + sub a0, qword [rsi+sizeof(qword)*0] ; a[] = -a[] + sbb a1, qword [rsi+sizeof(qword)*1] + sbb a2, qword [rsi+sizeof(qword)*2] + sbb a3, qword [rsi+sizeof(qword)*3] + sbb t4, 0 + + mov t0, a0 ; t[] = a[] + mov t1, a1 + mov t2, a2 + mov t3, a3 + + add t0, qword [rdx+sizeof(qword)*0] ; t[] = -a[] +poly[] + adc t1, qword [rdx+sizeof(qword)*1] + adc t2, qword [rdx+sizeof(qword)*2] + adc t3, qword [rdx+sizeof(qword)*3] + test t4, t4 + + cmovnz a0, t0 ; A = a:t + cmovnz a1, t1 + cmovnz a2, t2 + cmovnz a3, t3 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + + mov rax, rdi ; return pointer to result + + REST_XMM + REST_GPR + ret +ENDFUNC gf256_neg + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; Ipp64u* gf256_div2(Ipp64u res[4], Ipp64u a[4], Ipp64u poly[4]) +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM gf256_div2,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13,r14 + USES_XMM + COMP_ABI 3 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 + +%xdefine t0 rax +%xdefine t1 rcx +%xdefine t2 r12 +%xdefine t3 r13 +%xdefine t4 r14 + + mov a0, qword [rsi+sizeof(qword)*0] ; a[] = A + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + + xor t4, t4 + xor rsi, rsi + + mov t0, a0 ; t[] = A + mov t1, a1 + mov t2, a2 + mov t3, a3 + + add t0, qword [rdx+sizeof(qword)*0] ; t[] += poly[] + adc t1, qword [rdx+sizeof(qword)*1] + adc t2, qword [rdx+sizeof(qword)*2] + adc t3, qword [rdx+sizeof(qword)*3] + adc t4, 0 + + test a0, 1 ; %if (A[0] & 1) + cmovnz a0, t0 ; a[] = t[] + cmovnz a1, t1 + cmovnz a2, t2 + cmovnz a3, t3 + cmovnz rsi,t4 + + shrd a0, a1, 1 ; a[] /= 2 + shrd a1, a2, 1 + shrd a2, a3, 1 + shrd a3, rsi,1 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + + mov rax, rdi ; return pointer to result + + REST_XMM + REST_GPR + ret +ENDFUNC gf256_div2 + +;; +;; GF256 multiplicative operations +;; +%macro MUL_4x1 8.nolist + %xdefine %%dm %1 + %xdefine %%ACC4 %2 + %xdefine %%ACC3 %3 + %xdefine %%ACC2 %4 + %xdefine %%ACC1 %5 + %xdefine %%ACC0 %6 + %xdefine %%aPtr %7 + %xdefine %%B %8 + + mov rax, qword [%%aPtr+sizeof(qword)*0] + mul %%B + mov %%ACC0,rax + mov %%ACC1,rdx + imul %%dm, %%ACC0 + + mov rax, qword [%%aPtr+sizeof(qword)*1] + mul %%B + add %%ACC1,rax + adc rdx, 0 + mov %%ACC2,rdx + + mov rax, qword [%%aPtr+sizeof(qword)*2] + mul %%B + add %%ACC2,rax + adc rdx, 0 + mov %%ACC3,rdx + + mov rax, qword [%%aPtr+sizeof(qword)*3] + mul %%B + add %%ACC3,rax + adc rdx, 0 + mov %%ACC4,rdx +%endmacro + +%macro MLA_4x1 10.nolist + %xdefine %%dm %1 + %xdefine %%ACC5 %2 + %xdefine %%ACC4 %3 + %xdefine %%ACC3 %4 + %xdefine %%ACC2 %5 + %xdefine %%ACC1 %6 + %xdefine %%ACC0 %7 + %xdefine %%aPtr %8 + %xdefine %%B %9 + %xdefine %%TMP %10 + + mov rax, qword [%%aPtr+sizeof(qword)*0] + mul %%B + add %%ACC0,rax + adc rdx, 0 + mov %%TMP, rdx + imul %%dm, %%ACC0 + + mov rax, qword [%%aPtr+sizeof(qword)*1] + mul %%B + add %%ACC1,%%TMP + adc rdx, 0 + add %%ACC1,rax + adc rdx, 0 + mov %%TMP, rdx + + mov rax, qword [%%aPtr+sizeof(qword)*2] + mul %%B + add %%ACC2,%%TMP + adc rdx, 0 + add %%ACC2,rax + adc rdx, 0 + mov %%TMP, rdx + + mov rax, qword [%%aPtr+sizeof(qword)*3] + mul %%B + add %%ACC3,%%TMP + adc rdx, 0 + add %%ACC3,rax + adc %%ACC4,rdx + adc %%ACC5,0 +%endmacro + +%macro MRED_4x1 8.nolist + %xdefine %%ACC5 %1 + %xdefine %%ACC4 %2 + %xdefine %%ACC3 %3 + %xdefine %%ACC2 %4 + %xdefine %%ACC1 %5 + %xdefine %%ACC0 %6 + %xdefine %%mPtr %7 + %xdefine %%dm %8 + + mov rax, qword [%%mPtr+sizeof(qword)*0] + mul %%dm + add %%ACC0,rax + adc rdx, 0 + mov %%ACC0,rdx + + mov rax, qword [%%mPtr+sizeof(qword)*1] + mul %%dm + add %%ACC1,%%ACC0 + adc rdx, 0 + add %%ACC1,rax + adc rdx, 0 + mov %%ACC0,rdx + + mov rax, qword [%%mPtr+sizeof(qword)*2] + mul %%dm + add %%ACC2,%%ACC0 + adc rdx, 0 + add %%ACC2,rax + adc rdx, 0 + mov %%ACC0,rdx + + mov rax, qword [%%mPtr+sizeof(qword)*3] + mul %%dm + add %%ACC3,%%ACC0 + adc rdx, 0 + add %%ACC3,rax + adc rdx, 0 + add %%ACC4,rdx + + adc %%ACC5,0 + xor %%ACC0,%%ACC0 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; Ipp64u* gf256_mulm(Ipp64u res[4], Ipp64u a[4], Ipp64u b[4], Ipp64u poly[4], Ipp64u m0) +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM gf256_mulm,PUBLIC +%assign LOCAL_FRAME 2*sizeof(qword) + USES_GPR rsi,rdi,rbx,rbp,r12,r13,r14,r15 + USES_XMM + COMP_ABI 5 + +;; stack structure: +%assign res 0 +%assign m0 res+sizeof(qword) + + mov qword [rsp+res], rdi ; save result + mov rbx, rdx ; rbx = bPtr + mov rdi, rcx ; rcx = mPtr + mov qword [rsp+m0], r8 ; save m0 + +%xdefine acc0 r8 +%xdefine acc1 r9 +%xdefine acc2 r10 +%xdefine acc3 r11 +%xdefine acc4 r12 +%xdefine acc5 r13 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; P = A[] * b[0] + mov r14, qword [rbx+sizeof(qword)*0] + mov r15, qword [rsp+m0] + MUL_4x1 r15, acc4,acc3,acc2,acc1,acc0, rsi, r14 + + xor acc5,acc5 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step + MRED_4x1 acc5,acc4,acc3,acc2,acc1,acc0, rdi, r15 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; P+= A[] * b[1] + mov r14, qword [rbx+sizeof(qword)*1] + mov r15, qword [rsp+m0] + MLA_4x1 r15, acc0,acc5,acc4,acc3,acc2,acc1, rsi, r14, rcx + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step + MRED_4x1 acc0,acc5,acc4,acc3,acc2,acc1, rdi, r15 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; P+= A[] * b[2] + mov r14, qword [rbx+sizeof(qword)*2] + mov r15, qword [rsp+m0] + MLA_4x1 r15, acc1,acc0,acc5,acc4,acc3,acc2, rsi, r14, rcx + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step + MRED_4x1 acc1,acc0,acc5,acc4,acc3,acc2, rdi, r15 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; P+= A[] * b[3] + mov r14, qword [rbx+sizeof(qword)*3] + mov r15, qword [rsp+m0] + MLA_4x1 r15, acc2,acc1,acc0,acc5,acc4,acc3, rsi, r14, rcx + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step + MRED_4x1 acc2,acc1,acc0,acc5,acc4,acc3, rdi, r15 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; final reduction + mov rsi, qword [rsp+res] ;; restore result address + + mov rax, acc4 ;; copy temporary result + mov rbx, acc5 + mov rcx, acc0 + mov rdx, acc1 + + sub rax, qword [rdi+sizeof(qword)*0] ;; subtract modulus + sbb rbx, qword [rdi+sizeof(qword)*1] + sbb rcx, qword [rdi+sizeof(qword)*2] + sbb rdx, qword [rdi+sizeof(qword)*3] + sbb acc2, 0 + + cmovnc acc4, rax + cmovnc acc5, rbx + cmovnc acc0, rcx + cmovnc acc1, rdx + + mov qword [rsi+sizeof(qword)*0], acc4 + mov qword [rsi+sizeof(qword)*1], acc5 + mov qword [rsi+sizeof(qword)*2], acc0 + mov qword [rsi+sizeof(qword)*3], acc1 + + mov rax, rsi ; return pointer to result + + REST_XMM + REST_GPR + ret +ENDFUNC gf256_mulm + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; Ipp64u* gf256_sqrm(Ipp64u res[4], Ipp64u a[4], Ipp64u poly[4], Ipp64u m0) +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM gf256_sqrm,PUBLIC +%assign LOCAL_FRAME 2*sizeof(qword) + USES_GPR rsi,rdi,rbx,rbp,r12,r13,r14,r15 + USES_XMM + COMP_ABI 4 + +;; stack structure: +%assign res 0 +%assign m0 res+sizeof(qword) + + mov qword [rsp+res], rdi ; save result + mov rdi, rdx ; rdi = mPtr + mov qword [rsp+m0], rcx ; save m0 + +%xdefine acc0 r8 +%xdefine acc1 r9 +%xdefine acc2 r10 +%xdefine acc3 r11 +%xdefine acc4 r12 +%xdefine acc5 r13 +%xdefine acc6 r14 +%xdefine acc7 r15 + +%xdefine t0 rcx +%xdefine t1 rbp +%xdefine t2 rbx +%xdefine t3 rdx +%xdefine t4 rax + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; diagonal + mov t2, qword [rsi+sizeof(qword)*0] + mov rax,qword [rsi+sizeof(qword)*1] + mul t2 + mov acc1, rax + mov acc2, rdx + mov rax,qword [rsi+sizeof(qword)*2] + mul t2 + add acc2, rax + adc rdx, 0 + mov acc3, rdx + mov rax,qword [rsi+sizeof(qword)*3] + mul t2 + add acc3, rax + adc rdx, 0 + mov acc4, rdx + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov t2, qword [rsi+sizeof(qword)*1] + mov rax,qword [rsi+sizeof(qword)*2] + mul t2 + add acc3, rax + adc rdx, 0 + mov t1, rdx + mov rax,qword [rsi+sizeof(qword)*3] + mul t2 + add acc4, rax + adc rdx, 0 + add acc4, t1 + adc rdx, 0 + mov acc5, rdx + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov t2, qword [rsi+sizeof(qword)*2] + mov rax,qword [rsi+sizeof(qword)*3] + mul t2 + add acc5, rax + adc rdx, 0 + mov acc6, rdx + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; doubling + xor acc7, acc7 + shld acc7, acc6, 1 + shld acc6, acc5, 1 + shld acc5, acc4, 1 + shld acc4, acc3, 1 + shld acc3, acc2, 1 + shld acc2, acc1, 1 + shl acc1, 1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; add sruares + mov rax,qword [rsi+sizeof(qword)*0] + mul rax + mov acc0, rax + add acc1, rdx + adc acc2, 0 + mov rax,qword [rsi+sizeof(qword)*1] + mul rax + add acc2, rax + adc acc3, rdx + adc acc4, 0 + mov rax,qword [rsi+sizeof(qword)*2] + mul rax + add acc4, rax + adc acc5, rdx + adc acc6, 0 + mov rax,qword [rsi+sizeof(qword)*3] + mul rax + add acc6, rax + adc acc7, rdx + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction + mov rcx, qword [rsp+m0] ; restore m0 + imul rcx, acc0 + MRED_4x1 acc5,acc4,acc3,acc2,acc1,acc0, rdi, rcx + + mov rcx, qword [rsp+m0] ; restore m0 + imul rcx, acc1 + MRED_4x1 acc6,acc5,acc4,acc3,acc2,acc1, rdi, rcx + + mov rcx, qword [rsp+m0] ; restore m0 + imul rcx, acc2 + MRED_4x1 acc7,acc6,acc5,acc4,acc3,acc2, rdi, rcx + + mov rcx, qword [rsp+m0] ; restore m0 + imul rcx, acc3 + MRED_4x1 acc0,acc7,acc6,acc5,acc4,acc3, rdi, rcx + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; final reduction + mov rsi, qword [rsp+res] ;; restore result address + + mov rax, acc4 ;; copy temporary result + mov rbx, acc5 + mov rcx, acc6 + mov rdx, acc7 + + sub rax, qword [rdi+sizeof(qword)*0] ;; subtract modulus + sbb rbx, qword [rdi+sizeof(qword)*1] + sbb rcx, qword [rdi+sizeof(qword)*2] + sbb rdx, qword [rdi+sizeof(qword)*3] + sbb acc0, 0 + + cmovnc acc4, rax + cmovnc acc5, rbx + cmovnc acc6, rcx + cmovnc acc7, rdx + + mov qword [rsi+sizeof(qword)*0], acc4 + mov qword [rsi+sizeof(qword)*1], acc5 + mov qword [rsi+sizeof(qword)*2], acc6 + mov qword [rsi+sizeof(qword)*3], acc7 + + mov rax, rsi ; return pointer to result + + REST_XMM + REST_GPR + ret +ENDFUNC gf256_sqrm + +%endif ;; _IPP32E_M7 + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpp256r1funcs_montas.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpp256r1funcs_montas.asm new file mode 100644 index 0000000..5d01202 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpp256r1funcs_montas.asm @@ -0,0 +1,1316 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; secp p256r1 specific implementation +; + + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_M7) + +%assign _xEMULATION_ 1 +%assign _ADCX_ADOX_ 1 + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR + +;; The p256r1 polynomial +Lpoly DQ 0FFFFFFFFFFFFFFFFh,000000000FFFFFFFFh,00000000000000000h,0FFFFFFFF00000001h + +;; 2^512 mod P precomputed for p256r1 polynomial +LRR DQ 00000000000000003h,0fffffffbffffffffh,0fffffffffffffffeh,000000004fffffffdh + +LOne DD 1,1,1,1,1,1,1,1 +LTwo DD 2,2,2,2,2,2,2,2 +LThree DD 3,3,3,3,3,3,3,3 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p256r1_mul_by_2(uint64_t res[4], uint64_t a[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p256r1_mul_by_2,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13 + USES_XMM + COMP_ABI 2 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 +%xdefine t4 r13 + + xor t4, t4 + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + + shld t4, a3, 1 + shld a3, a2, 1 + shld a2, a1, 1 + shld a1, a0, 1 + shl a0, 1 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + mov t3, a3 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb t3, qword [rel Lpoly+sizeof(qword)*3] + sbb t4, 0 + + cmovz a0, t0 + cmovz a1, t1 + cmovz a2, t2 + cmovz a3, t3 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + + REST_XMM + REST_GPR + ret +ENDFUNC p256r1_mul_by_2 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p256r1_div_by_2(uint64_t res[4], uint64_t a[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p256r1_div_by_2,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13,r14 + USES_XMM + COMP_ABI 2 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 +%xdefine t4 r13 + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + + xor t4, t4 + xor r14, r14 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + mov t3, a3 + + add t0, qword [rel Lpoly+sizeof(qword)*0] + adc t1, qword [rel Lpoly+sizeof(qword)*1] + adc t2, qword [rel Lpoly+sizeof(qword)*2] + adc t3, qword [rel Lpoly+sizeof(qword)*3] + adc t4, 0 + test a0, 1 + + cmovnz a0, t0 + cmovnz a1, t1 + cmovnz a2, t2 + cmovnz a3, t3 + cmovnz r14,t4 + + shrd a0, a1, 1 + shrd a1, a2, 1 + shrd a2, a3, 1 + shrd a3, r14,1 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + + REST_XMM + REST_GPR + ret +ENDFUNC p256r1_div_by_2 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p256r1_mul_by_3(uint64_t res[4], uint64_t a[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p256r1_mul_by_3,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13 + USES_XMM + COMP_ABI 2 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 +%xdefine t4 r13 + + xor t4, t4 + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + + shld t4, a3, 1 + shld a3, a2, 1 + shld a2, a1, 1 + shld a1, a0, 1 + shl a0, 1 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + mov t3, a3 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb t3, qword [rel Lpoly+sizeof(qword)*3] + sbb t4, 0 + + cmovz a0, t0 + cmovz a1, t1 + cmovz a2, t2 + cmovz a3, t3 + + xor t4, t4 + add a0, qword [rsi+sizeof(qword)*0] + adc a1, qword [rsi+sizeof(qword)*1] + adc a2, qword [rsi+sizeof(qword)*2] + adc a3, qword [rsi+sizeof(qword)*3] + adc t4, 0 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + mov t3, a3 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb t3, qword [rel Lpoly+sizeof(qword)*3] + sbb t4, 0 + + cmovz a0, t0 + cmovz a1, t1 + cmovz a2, t2 + cmovz a3, t3 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + + REST_XMM + REST_GPR + ret +ENDFUNC p256r1_mul_by_3 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p256r1_add(uint64_t res[4], uint64_t a[4], uint64_t b[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p256r1_add,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13 + USES_XMM + COMP_ABI 3 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 +%xdefine t4 r13 + + xor t4, t4 + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + + add a0, qword [rdx+sizeof(qword)*0] + adc a1, qword [rdx+sizeof(qword)*1] + adc a2, qword [rdx+sizeof(qword)*2] + adc a3, qword [rdx+sizeof(qword)*3] + adc t4, 0 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + mov t3, a3 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb t3, qword [rel Lpoly+sizeof(qword)*3] + sbb t4, 0 + + cmovz a0, t0 + cmovz a1, t1 + cmovz a2, t2 + cmovz a3, t3 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + + REST_XMM + REST_GPR + ret +ENDFUNC p256r1_add + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p256r1_sub(uint64_t res[4], uint64_t a[4], uint64_t b[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p256r1_sub,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13 + USES_XMM + COMP_ABI 3 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 +%xdefine t4 r13 + + xor t4, t4 + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + + sub a0, qword [rdx+sizeof(qword)*0] + sbb a1, qword [rdx+sizeof(qword)*1] + sbb a2, qword [rdx+sizeof(qword)*2] + sbb a3, qword [rdx+sizeof(qword)*3] + sbb t4, 0 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + mov t3, a3 + + add t0, qword [rel Lpoly+sizeof(qword)*0] + adc t1, qword [rel Lpoly+sizeof(qword)*1] + adc t2, qword [rel Lpoly+sizeof(qword)*2] + adc t3, qword [rel Lpoly+sizeof(qword)*3] + test t4, t4 + + cmovnz a0, t0 + cmovnz a1, t1 + cmovnz a2, t2 + cmovnz a3, t3 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + + REST_XMM + REST_GPR + ret +ENDFUNC p256r1_sub + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p256r1_neg(uint64_t res[4], uint64_t a[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p256r1_neg,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13 + USES_XMM + COMP_ABI 2 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 +%xdefine t4 r13 + + xor t4, t4 + + xor a0, a0 + xor a1, a1 + xor a2, a2 + xor a3, a3 + + sub a0, qword [rsi+sizeof(qword)*0] + sbb a1, qword [rsi+sizeof(qword)*1] + sbb a2, qword [rsi+sizeof(qword)*2] + sbb a3, qword [rsi+sizeof(qword)*3] + sbb t4, 0 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + mov t3, a3 + + add t0, qword [rel Lpoly+sizeof(qword)*0] + adc t1, qword [rel Lpoly+sizeof(qword)*1] + adc t2, qword [rel Lpoly+sizeof(qword)*2] + adc t3, qword [rel Lpoly+sizeof(qword)*3] + test t4, t4 + + cmovnz a0, t0 + cmovnz a1, t1 + cmovnz a2, t2 + cmovnz a3, t3 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + + REST_XMM + REST_GPR + ret +ENDFUNC p256r1_neg + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p256r1_mul_montl(uint64_t res[4], uint64_t a[4], uint64_t b[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; on entry p5=0 +; on exit p0=0 +; +%macro p256r1_mul_redstep 6.nolist + %xdefine %%p5 %1 + %xdefine %%p4 %2 + %xdefine %%p3 %3 + %xdefine %%p2 %4 + %xdefine %%p1 %5 + %xdefine %%p0 %6 + + mov t0, %%p0 + shl t0, 32 + mov t1, %%p0 + shr t1, 32 ;; (t1:t0) = p0*2^32 + + mov t2, %%p0 + mov t3, %%p0 + xor %%p0, %%p0 + sub t2, t0 + sbb t3, t1 ;; (t3:t2) = (p0*2^64+p0) - p0*2^32 + + add %%p1, t0 ;; (p2:p1) += (t1:t0) + adc %%p2, t1 + adc %%p3, t2 ;; (p4:p3) += (t3:t2) + adc %%p4, t3 + adc %%p5, 0 ;; extension = {0/1} +%endmacro + +align IPP_ALIGN_FACTOR +p256r1_mmull: + +%xdefine acc0 r8 +%xdefine acc1 r9 +%xdefine acc2 r10 +%xdefine acc3 r11 +%xdefine acc4 r12 +%xdefine acc5 r13 +%xdefine acc6 r14 +%xdefine acc7 r15 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 rbp +%xdefine t4 rbx + +; rdi assumed as result +%xdefine aPtr rsi +%xdefine bPtr rbx + + xor acc5, acc5 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[0] + mov rax, qword [bPtr+sizeof(qword)*0] + mul qword [aPtr+sizeof(qword)*0] + mov acc0, rax + mov acc1, rdx + + mov rax, qword [bPtr+sizeof(qword)*0] + mul qword [aPtr+sizeof(qword)*1] + add acc1, rax + adc rdx, 0 + mov acc2, rdx + + mov rax, qword [bPtr+sizeof(qword)*0] + mul qword [aPtr+sizeof(qword)*2] + add acc2, rax + adc rdx, 0 + mov acc3, rdx + + mov rax, qword [bPtr+sizeof(qword)*0] + mul qword [aPtr+sizeof(qword)*3] + add acc3, rax + adc rdx, 0 + mov acc4, rdx + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 0 + p256r1_mul_redstep acc5,acc4,acc3,acc2,acc1,acc0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[1] + mov rax, qword [bPtr+sizeof(qword)*1] + mul qword [aPtr+sizeof(qword)*0] + add acc1, rax + adc rdx, 0 + mov rcx, rdx + + mov rax, qword [bPtr+sizeof(qword)*1] + mul qword [aPtr+sizeof(qword)*1] + add acc2, rcx + adc rdx, 0 + add acc2, rax + adc rdx, 0 + mov rcx, rdx + + mov rax, qword [bPtr+sizeof(qword)*1] + mul qword [aPtr+sizeof(qword)*2] + add acc3, rcx + adc rdx, 0 + add acc3, rax + adc rdx, 0 + mov rcx, rdx + + mov rax, qword [bPtr+sizeof(qword)*1] + mul qword [aPtr+sizeof(qword)*3] + add acc4, rcx + adc rdx, 0 + add acc4, rax + adc acc5, rdx + adc acc0, 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 1 + p256r1_mul_redstep acc0,acc5,acc4,acc3,acc2,acc1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[2] + mov rax, qword [bPtr+sizeof(qword)*2] + mul qword [aPtr+sizeof(qword)*0] + add acc2, rax + adc rdx, 0 + mov rcx, rdx + + mov rax, qword [bPtr+sizeof(qword)*2] + mul qword [aPtr+sizeof(qword)*1] + add acc3, rcx + adc rdx, 0 + add acc3, rax + adc rdx, 0 + mov rcx, rdx + + mov rax, qword [bPtr+sizeof(qword)*2] + mul qword [aPtr+sizeof(qword)*2] + add acc4, rcx + adc rdx, 0 + add acc4, rax + adc rdx, 0 + mov rcx, rdx + + mov rax, qword [bPtr+sizeof(qword)*2] + mul qword [aPtr+sizeof(qword)*3] + add acc5, rcx + adc rdx, 0 + add acc5, rax + adc acc0, rdx + adc acc1, 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 2 + p256r1_mul_redstep acc1,acc0,acc5,acc4,acc3,acc2 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[3] + mov rax, qword [bPtr+sizeof(qword)*3] + mul qword [aPtr+sizeof(qword)*0] + add acc3, rax + adc rdx, 0 + mov rcx, rdx + + mov rax, qword [bPtr+sizeof(qword)*3] + mul qword [aPtr+sizeof(qword)*1] + add acc4, rcx + adc rdx, 0 + add acc4, rax + adc rdx, 0 + mov rcx, rdx + + mov rax, qword [bPtr+sizeof(qword)*3] + mul qword [aPtr+sizeof(qword)*2] + add acc5, rcx + adc rdx, 0 + add acc5, rax + adc rdx, 0 + mov rcx, rdx + + mov rax, qword [bPtr+sizeof(qword)*3] + mul qword [aPtr+sizeof(qword)*3] + add acc0, rcx + adc rdx, 0 + add acc0, rax + adc acc1, rdx + adc acc2, 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 3 (final) + p256r1_mul_redstep acc2,acc1,acc0,acc5,acc4,acc3 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov t0, qword [rel Lpoly+sizeof(qword)*0] + mov t1, qword [rel Lpoly+sizeof(qword)*1] + mov t2, qword [rel Lpoly+sizeof(qword)*2] + mov t3, qword [rel Lpoly+sizeof(qword)*3] + + mov t4, acc4 ;; copy reducted result + mov acc3, acc5 + mov acc6, acc0 + mov acc7, acc1 + + sub t4, t0 ;; test %if it exceeds prime value + sbb acc3, t1 + sbb acc6, t2 + sbb acc7, t3 + sbb acc2, 0 + + cmovnc acc4, t4 + cmovnc acc5, acc3 + cmovnc acc0, acc6 + cmovnc acc1, acc7 + + mov qword [rdi+sizeof(qword)*0], acc4 + mov qword [rdi+sizeof(qword)*1], acc5 + mov qword [rdi+sizeof(qword)*2], acc0 + mov qword [rdi+sizeof(qword)*3], acc1 + + ret + +align IPP_ALIGN_FACTOR +IPPASM p256r1_mul_montl,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbp,rbx,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 3 + +%xdefine bPtr rbx + + mov bPtr, rdx + call p256r1_mmull + + REST_XMM + REST_GPR + ret +ENDFUNC p256r1_mul_montl + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p256r1_to_mont(uint64_t res[4], uint64_t a[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p256r1_to_mont,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbp,rbx,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 2 + + lea rbx, [rel LRR] + call p256r1_mmull + REST_XMM + REST_GPR + ret +ENDFUNC p256r1_to_mont + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p256r1_mul_montx(uint64_t res[4], uint64_t a[4], uint64_t b[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%if _IPP32E >= _IPP32E_L9 +align IPP_ALIGN_FACTOR +p256r1_mmulx: + +%xdefine acc0 r8 +%xdefine acc1 r9 +%xdefine acc2 r10 +%xdefine acc3 r11 +%xdefine acc4 r12 +%xdefine acc5 r13 +%xdefine acc6 r14 +%xdefine acc7 r15 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 rbp +%xdefine t4 rbx + +; rdi assumed as result +%xdefine aPtr rsi +%xdefine bPtr rbx + + xor acc5, acc5 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[0] + xor rdx, rdx + mov rdx, qword [bPtr+sizeof(qword)*0] + mulx acc1,acc0, qword [aPtr+sizeof(qword)*0] + mulx acc2,t2, qword [aPtr+sizeof(qword)*1] + add acc1,t2 + mulx acc3,t2, qword [aPtr+sizeof(qword)*2] + adc acc2,t2 + mulx acc4,t2, qword [aPtr+sizeof(qword)*3] + adc acc3,t2 + adc acc4,0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 0 + p256r1_mul_redstep acc5,acc4,acc3,acc2,acc1,acc0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[1] + mov rdx, qword [bPtr+sizeof(qword)*1] + + mulx t3, t2, qword [aPtr+sizeof(qword)*0] + adcx acc1, t2 + adox acc2, t3 + + mulx t3, t2, qword [aPtr+sizeof(qword)*1] + adcx acc2, t2 + adox acc3, t3 + + mulx t3, t2, qword [aPtr+sizeof(qword)*2] + adcx acc3, t2 + adox acc4, t3 + + mulx t3, t2, qword [aPtr+sizeof(qword)*3] + adcx acc4, t2 + adox acc5, t3 + + adcx acc5, acc0 + adox acc0, acc0 + adc acc0, 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 1 + p256r1_mul_redstep acc0,acc5,acc4,acc3,acc2,acc1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[2] + mov rdx, qword [bPtr+sizeof(qword)*2] + + mulx t3, t2, qword [aPtr+sizeof(qword)*0] + adcx acc2, t2 + adox acc3, t3 + + mulx t3, t2, qword [aPtr+sizeof(qword)*1] + adcx acc3, t2 + adox acc4, t3 + + mulx t3, t2, qword [aPtr+sizeof(qword)*2] + adcx acc4, t2 + adox acc5, t3 + + mulx t3, t2, qword [aPtr+sizeof(qword)*3] + adcx acc5, t2 + adox acc0, t3 + + adcx acc0, acc1 + adox acc1, acc1 + + adc acc1, 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 2 + p256r1_mul_redstep acc1,acc0,acc5,acc4,acc3,acc2 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[3] + mov rdx, qword [bPtr+sizeof(qword)*3] + + mulx t3, t2, qword [aPtr+sizeof(qword)*0] + adcx acc3, t2 + adox acc4, t3 + + mulx t3, t2, qword [aPtr+sizeof(qword)*1] + adcx acc4, t2 + adox acc5, t3 + + mulx t3, t2, qword [aPtr+sizeof(qword)*2] + adcx acc5, t2 + adox acc0, t3 + + mulx t3, t2, qword [aPtr+sizeof(qword)*3] + adcx acc0, t2 + adox acc1, t3 + + adcx acc1, acc2 + adox acc2, acc2 + adc acc2, 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 3 (final) + p256r1_mul_redstep acc2,acc1,acc0,acc5,acc4,acc3 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov t0, qword [rel Lpoly+sizeof(qword)*0] + mov t1, qword [rel Lpoly+sizeof(qword)*1] + mov t2, qword [rel Lpoly+sizeof(qword)*2] + mov t3, qword [rel Lpoly+sizeof(qword)*3] + + mov t4, acc4 ;; copy reducted result + mov acc3, acc5 + mov acc6, acc0 + mov acc7, acc1 + + sub t4, t0 ;; test %if it exceeds prime value + sbb acc3, t1 + sbb acc6, t2 + sbb acc7, t3 + sbb acc2, 0 + + cmovnc acc4, t4 + cmovnc acc5, acc3 + cmovnc acc0, acc6 + cmovnc acc1, acc7 + + mov qword [rdi+sizeof(qword)*0], acc4 + mov qword [rdi+sizeof(qword)*1], acc5 + mov qword [rdi+sizeof(qword)*2], acc0 + mov qword [rdi+sizeof(qword)*3], acc1 + + ret +%endif + +%if _IPP32E >= _IPP32E_L9 +align IPP_ALIGN_FACTOR +IPPASM p256r1_mul_montx,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbp,rbx,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 3 + +%xdefine bPtr rbx + + mov bPtr, rdx + call p256r1_mmulx + + REST_XMM + REST_GPR + ret +ENDFUNC p256r1_mul_montx + +%endif ;; _IPP32E_L9 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p256r1_sqr_montl(uint64_t res[4], uint64_t a[4]); +; void p256r1_sqr_montx(uint64_t res[4], uint64_t a[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; on entry e = expasion (previous step) +; on exit p0= expasion (next step) +; +%macro p256r1_prod_redstep 6.nolist + %xdefine %%e %1 + %xdefine %%p4 %2 + %xdefine %%p3 %3 + %xdefine %%p2 %4 + %xdefine %%p1 %5 + %xdefine %%p0 %6 + + mov t0, %%p0 + shl t0, 32 + mov t1, %%p0 + shr t1, 32 ;; (t1:t0) = p0*2^32 + + mov t2, %%p0 + mov t3, %%p0 + xor %%p0, %%p0 + sub t2, t0 + sbb t3, t1 ;; (t3:t2) = (p0*2^64+p0) - p0*2^32 + + add %%p1, t0 ;; (p2:p1) += (t1:t0) + adc %%p2, t1 + adc %%p3, t2 ;; (p4:p3) += (t3:t2) + adc %%p4, t3 + adc %%p0, 0 ;; extension = {0/1} + + %ifnempty %%e + add %%p4, %%e + adc %%p0, 0 + %endif + +%endmacro + +align IPP_ALIGN_FACTOR +IPPASM p256r1_sqr_montl,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbp,rbx,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 2 + +%xdefine acc0 r8 +%xdefine acc1 r9 +%xdefine acc2 r10 +%xdefine acc3 r11 +%xdefine acc4 r12 +%xdefine acc5 r13 +%xdefine acc6 r14 +%xdefine acc7 r15 + +%xdefine t0 rcx +%xdefine t1 rbp +%xdefine t2 rbx +%xdefine t3 rdx +%xdefine t4 rax + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov t2, qword [aPtr+sizeof(qword)*0] + mov rax,qword [aPtr+sizeof(qword)*1] + mul t2 + mov acc1, rax + mov acc2, rdx + mov rax,qword [aPtr+sizeof(qword)*2] + mul t2 + add acc2, rax + adc rdx, 0 + mov acc3, rdx + mov rax,qword [aPtr+sizeof(qword)*3] + mul t2 + add acc3, rax + adc rdx, 0 + mov acc4, rdx + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov t2, qword [aPtr+sizeof(qword)*1] + mov rax,qword [aPtr+sizeof(qword)*2] + mul t2 + add acc3, rax + adc rdx, 0 + mov t1, rdx + mov rax,qword [aPtr+sizeof(qword)*3] + mul t2 + add acc4, rax + adc rdx, 0 + add acc4, t1 + adc rdx, 0 + mov acc5, rdx + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov t2, qword [aPtr+sizeof(qword)*2] + mov rax,qword [aPtr+sizeof(qword)*3] + mul t2 + add acc5, rax + adc rdx, 0 + mov acc6, rdx + + xor acc7, acc7 + shld acc7, acc6, 1 + shld acc6, acc5, 1 + shld acc5, acc4, 1 + shld acc4, acc3, 1 + shld acc3, acc2, 1 + shld acc2, acc1, 1 + shl acc1, 1 + + mov rax,qword [aPtr+sizeof(qword)*0] + mul rax + mov acc0, rax + add acc1, rdx + adc acc2, 0 + mov rax,qword [aPtr+sizeof(qword)*1] + mul rax + add acc2, rax + adc acc3, rdx + adc acc4, 0 + mov rax,qword [aPtr+sizeof(qword)*2] + mul rax + add acc4, rax + adc acc5, rdx + adc acc6, 0 + mov rax,qword [aPtr+sizeof(qword)*3] + mul rax + add acc6, rax + adc acc7, rdx + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + p256r1_prod_redstep ,acc4,acc3,acc2,acc1,acc0 + p256r1_prod_redstep acc0,acc5,acc4,acc3,acc2,acc1 + p256r1_prod_redstep acc1,acc6,acc5,acc4,acc3,acc2 + p256r1_prod_redstep acc2,acc7,acc6,acc5,acc4,acc3 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + mov t0, qword [rel Lpoly+sizeof(qword)*0] + mov t1, qword [rel Lpoly+sizeof(qword)*1] + mov t2, qword [rel Lpoly+sizeof(qword)*2] + mov t3, qword [rel Lpoly+sizeof(qword)*3] + + mov t4, acc4 + mov acc0, acc5 + mov acc1, acc6 + mov acc2, acc7 + + sub t4, t0 + sbb acc0, t1 + sbb acc1, t2 + sbb acc2, t3 + sbb acc3, 0 + + cmovnc acc4, t4 + cmovnc acc5, acc0 + cmovnc acc6, acc1 + cmovnc acc7, acc2 + + mov qword [rdi+sizeof(qword)*0], acc4 + mov qword [rdi+sizeof(qword)*1], acc5 + mov qword [rdi+sizeof(qword)*2], acc6 + mov qword [rdi+sizeof(qword)*3], acc7 + + REST_XMM + REST_GPR + ret +ENDFUNC p256r1_sqr_montl + +%if _IPP32E >= _IPP32E_L9 +align IPP_ALIGN_FACTOR +IPPASM p256r1_sqr_montx,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbp,rbx,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 2 + +%xdefine acc0 r8 +%xdefine acc1 r9 +%xdefine acc2 r10 +%xdefine acc3 r11 +%xdefine acc4 r12 +%xdefine acc5 r13 +%xdefine acc6 r14 +%xdefine acc7 r15 + +%xdefine t0 rcx +%xdefine t1 rbp +%xdefine t2 rbx +%xdefine t3 rdx +%xdefine t4 rax + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov rdx, qword [aPtr+sizeof(qword)*0] + mulx acc2, acc1, qword [aPtr+sizeof(qword)*1] + mulx acc3, t0, qword [aPtr+sizeof(qword)*2] + add acc2, t0 + mulx acc4, t0, qword [aPtr+sizeof(qword)*3] + adc acc3, t0 + adc acc4, 0 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov rdx, qword [aPtr+sizeof(qword)*1] + xor acc5, acc5 + mulx t1, t0, qword [aPtr+sizeof(qword)*2] + adcx acc3, t0 + adox acc4, t1 + mulx t1, t0, qword [aPtr+sizeof(qword)*3] + adcx acc4, t0 + adox acc5, t1 + adc acc5, 0 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov rdx, qword [aPtr+sizeof(qword)*2] + mulx acc6, t0, qword [aPtr+sizeof(qword)*3] + add acc5, t0 + adc acc6, 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + xor acc7, acc7 + shld acc7, acc6, 1 + shld acc6, acc5, 1 + shld acc5, acc4, 1 + shld acc4, acc3, 1 + shld acc3, acc2, 1 + shld acc2, acc1, 1 + shl acc1, 1 + + xor acc0, acc0 + mov rdx, qword [aPtr+sizeof(qword)*0] + mulx t1, acc0, rdx + adcx acc1, t1 + mov rdx, qword [aPtr+sizeof(qword)*1] + mulx t1, t0, rdx + adcx acc2, t0 + adcx acc3, t1 + mov rdx, qword [aPtr+sizeof(qword)*2] + mulx t1, t0, rdx + adcx acc4, t0 + adcx acc5, t1 + mov rdx, qword [aPtr+sizeof(qword)*3] + mulx t1, t0, rdx + adcx acc6, t0 + adcx acc7, t1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + p256r1_prod_redstep ,acc4,acc3,acc2,acc1,acc0 + p256r1_prod_redstep acc0,acc5,acc4,acc3,acc2,acc1 + p256r1_prod_redstep acc1,acc6,acc5,acc4,acc3,acc2 + p256r1_prod_redstep acc2,acc7,acc6,acc5,acc4,acc3 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + mov t0, qword [rel Lpoly+sizeof(qword)*0] + mov t1, qword [rel Lpoly+sizeof(qword)*1] + mov t2, qword [rel Lpoly+sizeof(qword)*2] + mov t3, qword [rel Lpoly+sizeof(qword)*3] + + mov t4, acc4 + mov acc0, acc5 + mov acc1, acc6 + mov acc2, acc7 + + sub t4, t0 + sbb acc0, t1 + sbb acc1, t2 + sbb acc2, t3 + sbb acc3, 0 + + cmovnc acc4, t4 + cmovnc acc5, acc0 + cmovnc acc6, acc1 + cmovnc acc7, acc2 + + mov qword [rdi+sizeof(qword)*0], acc4 + mov qword [rdi+sizeof(qword)*1], acc5 + mov qword [rdi+sizeof(qword)*2], acc6 + mov qword [rdi+sizeof(qword)*3], acc7 + + REST_XMM + REST_GPR + ret +ENDFUNC p256r1_sqr_montx + +%endif ;; _IPP32E_L9 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p256r1_mont_back(uint64_t res[4], uint64_t a[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p256r1_mont_back,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13 + USES_XMM + COMP_ABI 2 + +%xdefine acc0 r8 +%xdefine acc1 r9 +%xdefine acc2 r10 +%xdefine acc3 r11 +%xdefine acc4 r12 +%xdefine acc5 r13 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 rsi + + mov acc2, qword [rsi+sizeof(qword)*0] + mov acc3, qword [rsi+sizeof(qword)*1] + mov acc4, qword [rsi+sizeof(qword)*2] + mov acc5, qword [rsi+sizeof(qword)*3] + xor acc0, acc0 + xor acc1, acc1 + + p256r1_mul_redstep acc1,acc0,acc5,acc4,acc3,acc2 + p256r1_mul_redstep acc2,acc1,acc0,acc5,acc4,acc3 + p256r1_mul_redstep acc3,acc2,acc1,acc0,acc5,acc4 + p256r1_mul_redstep acc4,acc3,acc2,acc1,acc0,acc5 + + mov t0, acc0 + mov t1, acc1 + mov t2, acc2 + mov t3, acc3 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb t3, qword [rel Lpoly+sizeof(qword)*3] + sbb acc4, 0 + + cmovnc acc0, t0 + cmovnc acc1, t1 + cmovnc acc2, t2 + cmovnc acc3, t3 + + mov qword [rdi+sizeof(qword)*0], acc0 + mov qword [rdi+sizeof(qword)*1], acc1 + mov qword [rdi+sizeof(qword)*2], acc2 + mov qword [rdi+sizeof(qword)*3], acc3 + + REST_XMM + REST_GPR + ret +ENDFUNC p256r1_mont_back + +;;%if _IPP32E < _IPP32E_L9 +%ifndef _DISABLE_ECP_256R1_HARDCODED_BP_TBL_ +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p256r1_select_ap_w7(AF_POINT *val, const AF_POINT *in_t, int index); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p256r1_select_ap_w7,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13 + USES_XMM xmm6,xmm7,xmm8,xmm9,xmm10,xmm11,xmm12,xmm13,xmm14,xmm15 + COMP_ABI 3 + +%xdefine val rdi +%xdefine in_t rsi +%xdefine idx edx + +%xdefine ONE xmm0 +%xdefine INDEX xmm1 + +%xdefine Ra xmm2 +%xdefine Rb xmm3 +%xdefine Rc xmm4 +%xdefine Rd xmm5 + +%xdefine M0 xmm8 +%xdefine T0a xmm9 +%xdefine T0b xmm10 +%xdefine T0c xmm11 +%xdefine T0d xmm12 +%xdefine TMP0 xmm15 + + movdqa ONE, oword [rel LOne] + + pxor Ra, Ra + pxor Rb, Rb + pxor Rc, Rc + pxor Rd, Rd + + movdqa M0, ONE + + movd INDEX, idx + pshufd INDEX, INDEX, 0 + + ; Skip index = 0, is implicictly infty -> load with offset -1 + mov rcx, dword 64 +.select_loop_sse_w7: + movdqa TMP0, M0 + pcmpeqd TMP0, INDEX + paddd M0, ONE + + movdqa T0a, oword [in_t+sizeof(oword)*0] + movdqa T0b, oword [in_t+sizeof(oword)*1] + movdqa T0c, oword [in_t+sizeof(oword)*2] + movdqa T0d, oword [in_t+sizeof(oword)*3] + add in_t, sizeof(oword)*4 + + pand T0a, TMP0 + pand T0b, TMP0 + pand T0c, TMP0 + pand T0d, TMP0 + + por Ra, T0a + por Rb, T0b + por Rc, T0c + por Rd, T0d + dec rcx + jnz .select_loop_sse_w7 + + movdqu oword [val+sizeof(oword)*0], Ra + movdqu oword [val+sizeof(oword)*1], Rb + movdqu oword [val+sizeof(oword)*2], Rc + movdqu oword [val+sizeof(oword)*3], Rd + + REST_XMM + REST_GPR + ret +ENDFUNC p256r1_select_ap_w7 + +%endif +;;%endif ;; _IPP32E < _IPP32E_L9 + +%endif ;; _IPP32E_M7 + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpp384r1funcs_montas.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpp384r1funcs_montas.asm new file mode 100644 index 0000000..f312782 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpp384r1funcs_montas.asm @@ -0,0 +1,843 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; secp p384r1 specific implementation +; + + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_M7) + +%assign _xEMULATION_ 1 +%assign _ADCX_ADOX_ 1 + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR + +;; The p384r1 polynomial +Lpoly DQ 000000000ffffffffh,0ffffffff00000000h,0fffffffffffffffeh + DQ 0ffffffffffffffffh,0ffffffffffffffffh,0ffffffffffffffffh + +;; 2^(2*384) mod P precomputed for p384r1 polynomial +;LRR DQ 0fffffffe00000001h,00000000200000000h,0fffffffe00000000h +; DQ 00000000200000000h,00000000000000001h,00000000000000000h + +LOne DD 1,1,1,1,1,1,1,1 +LTwo DD 2,2,2,2,2,2,2,2 +LThree DD 3,3,3,3,3,3,3,3 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p384r1_mul_by_2(uint64_t res[6], uint64_t a[6]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p384r1_mul_by_2,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12 + USES_XMM + COMP_ABI 2 + +%xdefine a0 rax +%xdefine a1 rcx +%xdefine a2 rdx +%xdefine a3 r8 +%xdefine a4 r9 +%xdefine a5 r10 +%xdefine ex r11 + +%xdefine t r12 + + xor ex, ex + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + mov a4, qword [rsi+sizeof(qword)*4] + mov a5, qword [rsi+sizeof(qword)*5] + + shld ex, a5, 1 + shld a5, a4, 1 + mov qword [rdi+sizeof(qword)*5], a5 + shld a4, a3, 1 + mov qword [rdi+sizeof(qword)*4], a4 + shld a3, a2, 1 + mov qword [rdi+sizeof(qword)*3], a3 + shld a2, a1, 1 + mov qword [rdi+sizeof(qword)*2], a2 + shld a1, a0, 1 + mov qword [rdi+sizeof(qword)*1], a1 + shl a0, 1 + mov qword [rdi+sizeof(qword)*0], a0 + + sub a0, qword [rel Lpoly+sizeof(qword)*0] + sbb a1, qword [rel Lpoly+sizeof(qword)*1] + sbb a2, qword [rel Lpoly+sizeof(qword)*2] + sbb a3, qword [rel Lpoly+sizeof(qword)*3] + sbb a4, qword [rel Lpoly+sizeof(qword)*4] + sbb a5, qword [rel Lpoly+sizeof(qword)*5] + sbb ex, 0 + + mov t, qword [rdi+sizeof(qword)*0] + cmovnz a0, t + mov t, qword [rdi+sizeof(qword)*1] + cmovnz a1, t + mov t, qword [rdi+sizeof(qword)*2] + cmovnz a2, t + mov t, qword [rdi+sizeof(qword)*3] + cmovnz a3, t + mov t, qword [rdi+sizeof(qword)*4] + cmovnz a4, t + mov t, qword [rdi+sizeof(qword)*5] + cmovnz a5, t + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + mov qword [rdi+sizeof(qword)*4], a4 + mov qword [rdi+sizeof(qword)*5], a5 + + REST_XMM + REST_GPR + ret +ENDFUNC p384r1_mul_by_2 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p384r1_div_by_2(uint64_t res[6], uint64_t a[6]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p384r1_div_by_2,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12 + USES_XMM + COMP_ABI 2 + +%xdefine a0 rax +%xdefine a1 rcx +%xdefine a2 rdx +%xdefine a3 r8 +%xdefine a4 r9 +%xdefine a5 r10 +%xdefine ex r11 + +%xdefine t r12 + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + mov a4, qword [rsi+sizeof(qword)*4] + mov a5, qword [rsi+sizeof(qword)*5] + + xor t, t + xor ex, ex + add a0, qword [rel Lpoly+sizeof(qword)*0] + adc a1, qword [rel Lpoly+sizeof(qword)*1] + adc a2, qword [rel Lpoly+sizeof(qword)*2] + adc a3, qword [rel Lpoly+sizeof(qword)*3] + adc a4, qword [rel Lpoly+sizeof(qword)*4] + adc a5, qword [rel Lpoly+sizeof(qword)*5] + adc ex, 0 + + test a0, 1 + cmovnz ex, t + mov t, qword [rsi+sizeof(qword)*0] + cmovnz a0, t + mov t, qword [rsi+sizeof(qword)*1] + cmovnz a1, t + mov t, qword [rsi+sizeof(qword)*2] + cmovnz a2, t + mov t, qword [rsi+sizeof(qword)*3] + cmovnz a3, t + mov t, qword [rsi+sizeof(qword)*4] + cmovnz a4, t + mov t, qword [rsi+sizeof(qword)*5] + cmovnz a5, t + + shrd a0, a1, 1 + shrd a1, a2, 1 + shrd a2, a3, 1 + shrd a3, a4, 1 + shrd a4, a5, 1 + shrd a5, ex, 1 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + mov qword [rdi+sizeof(qword)*4], a4 + mov qword [rdi+sizeof(qword)*5], a5 + + REST_XMM + REST_GPR + ret +ENDFUNC p384r1_div_by_2 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p384r1_mul_by_3(uint64_t res[6], uint64_t a[6]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p384r1_mul_by_3,PUBLIC +%assign LOCAL_FRAME sizeof(qword)*6 + USES_GPR rsi,rdi,r12,r13 + USES_XMM + COMP_ABI 2 + +%xdefine a0 rax +%xdefine a1 rcx +%xdefine a2 rdx +%xdefine a3 r8 +%xdefine a4 r9 +%xdefine a5 r10 +%xdefine ex r11 + +%xdefine t r12 + + xor ex, ex + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + mov a4, qword [rsi+sizeof(qword)*4] + mov a5, qword [rsi+sizeof(qword)*5] + + shld ex, a5, 1 + shld a5, a4, 1 + mov qword [rsp+sizeof(qword)*5], a5 + shld a4, a3, 1 + mov qword [rsp+sizeof(qword)*4], a4 + shld a3, a2, 1 + mov qword [rsp+sizeof(qword)*3], a3 + shld a2, a1, 1 + mov qword [rsp+sizeof(qword)*2], a2 + shld a1, a0, 1 + mov qword [rsp+sizeof(qword)*1], a1 + shl a0, 1 + mov qword [rsp+sizeof(qword)*0], a0 + + sub a0, qword [rel Lpoly+sizeof(qword)*0] + sbb a1, qword [rel Lpoly+sizeof(qword)*1] + sbb a2, qword [rel Lpoly+sizeof(qword)*2] + sbb a3, qword [rel Lpoly+sizeof(qword)*3] + sbb a4, qword [rel Lpoly+sizeof(qword)*4] + sbb a5, qword [rel Lpoly+sizeof(qword)*5] + sbb ex, 0 + + mov t, qword [rsp+0*sizeof(qword)] + cmovb a0, t + mov t, qword [rsp+1*sizeof(qword)] + cmovb a1, t + mov t, qword [rsp+2*sizeof(qword)] + cmovb a2, t + mov t, qword [rsp+3*sizeof(qword)] + cmovb a3, t + mov t, qword [rsp+4*sizeof(qword)] + cmovb a4, t + mov t, qword [rsp+5*sizeof(qword)] + cmovb a5, t + + xor ex, ex + add a0, qword [rsi+sizeof(qword)*0] + mov qword [rsp+sizeof(qword)*0], a0 + adc a1, qword [rsi+sizeof(qword)*1] + mov qword [rsp+sizeof(qword)*1], a1 + adc a2, qword [rsi+sizeof(qword)*2] + mov qword [rsp+sizeof(qword)*2], a2 + adc a3, qword [rsi+sizeof(qword)*3] + mov qword [rsp+sizeof(qword)*3], a3 + adc a4, qword [rsi+sizeof(qword)*4] + mov qword [rsp+sizeof(qword)*4], a4 + adc a5, qword [rsi+sizeof(qword)*5] + mov qword [rsp+sizeof(qword)*5], a5 + adc ex, 0 + + sub a0, qword [rel Lpoly+sizeof(qword)*0] + sbb a1, qword [rel Lpoly+sizeof(qword)*1] + sbb a2, qword [rel Lpoly+sizeof(qword)*2] + sbb a3, qword [rel Lpoly+sizeof(qword)*3] + sbb a4, qword [rel Lpoly+sizeof(qword)*4] + sbb a5, qword [rel Lpoly+sizeof(qword)*5] + sbb ex, 0 + + mov t, qword [rsp+sizeof(qword)*0] + cmovb a0, t + mov t, qword [rsp+sizeof(qword)*1] + cmovb a1, t + mov t, qword [rsp+sizeof(qword)*2] + cmovb a2, t + mov t, qword [rsp+sizeof(qword)*3] + cmovb a3, t + mov t, qword [rsp+sizeof(qword)*4] + cmovb a4, t + mov t, qword [rsp+sizeof(qword)*5] + cmovb a5, t + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + mov qword [rdi+sizeof(qword)*4], a4 + mov qword [rdi+sizeof(qword)*5], a5 + + REST_XMM + REST_GPR + ret +ENDFUNC p384r1_mul_by_3 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p384r1_add(uint64_t res[6], uint64_t a[6], uint64_t b[6]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p384r1_add,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbx,rsi,rdi,r12 + USES_XMM + COMP_ABI 3 + +%xdefine a0 rax +%xdefine a1 rcx +%xdefine a2 rbx +%xdefine a3 r8 +%xdefine a4 r9 +%xdefine a5 r10 +%xdefine ex r11 + +%xdefine t r12 + + xor ex, ex + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + mov a4, qword [rsi+sizeof(qword)*4] + mov a5, qword [rsi+sizeof(qword)*5] + + add a0, qword [rdx+sizeof(qword)*0] + adc a1, qword [rdx+sizeof(qword)*1] + adc a2, qword [rdx+sizeof(qword)*2] + adc a3, qword [rdx+sizeof(qword)*3] + adc a4, qword [rdx+sizeof(qword)*4] + adc a5, qword [rdx+sizeof(qword)*5] + adc ex, 0 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + mov qword [rdi+sizeof(qword)*4], a4 + mov qword [rdi+sizeof(qword)*5], a5 + + sub a0, qword [rel Lpoly+sizeof(qword)*0] + sbb a1, qword [rel Lpoly+sizeof(qword)*1] + sbb a2, qword [rel Lpoly+sizeof(qword)*2] + sbb a3, qword [rel Lpoly+sizeof(qword)*3] + sbb a4, qword [rel Lpoly+sizeof(qword)*4] + sbb a5, qword [rel Lpoly+sizeof(qword)*5] + sbb ex, 0 + + mov t, qword [rdi+sizeof(qword)*0] + cmovb a0, t + mov t, qword [rdi+sizeof(qword)*1] + cmovb a1, t + mov t, qword [rdi+sizeof(qword)*2] + cmovb a2, t + mov t, qword [rdi+sizeof(qword)*3] + cmovb a3, t + mov t, qword [rdi+sizeof(qword)*4] + cmovb a4, t + mov t, qword [rdi+sizeof(qword)*5] + cmovb a5, t + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + mov qword [rdi+sizeof(qword)*4], a4 + mov qword [rdi+sizeof(qword)*5], a5 + + REST_XMM + REST_GPR + ret +ENDFUNC p384r1_add + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p384r1_sub(uint64_t res[6], uint64_t a[6], uint64_t b[6]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p384r1_sub,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbx,rsi,rdi,r12 + USES_XMM + COMP_ABI 3 + +%xdefine a0 rax +%xdefine a1 rcx +%xdefine a2 rbx +%xdefine a3 r8 +%xdefine a4 r9 +%xdefine a5 r10 +%xdefine ex r11 + +%xdefine t r12 + + xor ex, ex + + mov a0, qword [rsi+sizeof(qword)*0] ; a + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + mov a4, qword [rsi+sizeof(qword)*4] + mov a5, qword [rsi+sizeof(qword)*5] + + sub a0, qword [rdx+sizeof(qword)*0] ; a-b + sbb a1, qword [rdx+sizeof(qword)*1] + sbb a2, qword [rdx+sizeof(qword)*2] + sbb a3, qword [rdx+sizeof(qword)*3] + sbb a4, qword [rdx+sizeof(qword)*4] + sbb a5, qword [rdx+sizeof(qword)*5] + + sbb ex, 0 ; ex = a>=b? 0 : -1 + + mov qword [rdi+sizeof(qword)*0], a0 ; store (a-b) + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + mov qword [rdi+sizeof(qword)*4], a4 + mov qword [rdi+sizeof(qword)*5], a5 + + add a0, qword [rel Lpoly+sizeof(qword)*0] ; (a-b) +poly + adc a1, qword [rel Lpoly+sizeof(qword)*1] + adc a2, qword [rel Lpoly+sizeof(qword)*2] + adc a3, qword [rel Lpoly+sizeof(qword)*3] + adc a4, qword [rel Lpoly+sizeof(qword)*4] + adc a5, qword [rel Lpoly+sizeof(qword)*5] + + test ex, ex ; r = (ex)? ((a-b)+poly) : (a-b) + + mov t, qword [rdi+sizeof(qword)*0] + cmovz a0, t + mov t, qword [rdi+sizeof(qword)*1] + cmovz a1, t + mov t, qword [rdi+sizeof(qword)*2] + cmovz a2, t + mov t, qword [rdi+sizeof(qword)*3] + cmovz a3, t + mov t, qword [rdi+sizeof(qword)*4] + cmovz a4, t + mov t, qword [rdi+sizeof(qword)*5] + cmovz a5, t + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + mov qword [rdi+sizeof(qword)*4], a4 + mov qword [rdi+sizeof(qword)*5], a5 + + REST_XMM + REST_GPR + ret +ENDFUNC p384r1_sub + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p384r1_neg(uint64_t res[6], uint64_t a[6]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p384r1_neg,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12 + USES_XMM + COMP_ABI 2 + +%xdefine a0 rax +%xdefine a1 rcx +%xdefine a2 rdx +%xdefine a3 r8 +%xdefine a4 r9 +%xdefine a5 r10 +%xdefine ex r11 + +%xdefine t r12 + + xor ex, ex + + xor a0, a0 + xor a1, a1 + xor a2, a2 + xor a3, a3 + xor a4, a4 + xor a5, a5 + + sub a0, qword [rsi+sizeof(qword)*0] + sbb a1, qword [rsi+sizeof(qword)*1] + sbb a2, qword [rsi+sizeof(qword)*2] + sbb a3, qword [rsi+sizeof(qword)*3] + sbb a4, qword [rsi+sizeof(qword)*4] + sbb a5, qword [rsi+sizeof(qword)*5] + sbb ex, 0 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + mov qword [rdi+sizeof(qword)*4], a4 + mov qword [rdi+sizeof(qword)*5], a5 + + add a0, qword [rel Lpoly+sizeof(qword)*0] + adc a1, qword [rel Lpoly+sizeof(qword)*1] + adc a2, qword [rel Lpoly+sizeof(qword)*2] + adc a3, qword [rel Lpoly+sizeof(qword)*3] + adc a4, qword [rel Lpoly+sizeof(qword)*4] + adc a5, qword [rel Lpoly+sizeof(qword)*5] + test ex, ex + + mov t, qword [rdi+sizeof(qword)*0] + cmovz a0, t + mov t, qword [rdi+sizeof(qword)*1] + cmovz a1, t + mov t, qword [rdi+sizeof(qword)*2] + cmovz a2, t + mov t, qword [rdi+sizeof(qword)*3] + cmovz a3, t + mov t, qword [rdi+sizeof(qword)*4] + cmovz a4, t + mov t, qword [rdi+sizeof(qword)*5] + cmovz a5, t + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + mov qword [rdi+sizeof(qword)*4], a4 + mov qword [rdi+sizeof(qword)*5], a5 + + REST_XMM + REST_GPR + ret +ENDFUNC p384r1_neg + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; projective point selector +; +; void p384r1_mred(Ipp464u* res, Ipp64u product); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; +; mred_step does single Montgomery reduction step -- computes {a0:a1:a2:a3:a4:a5:a6} += u*M[] +; where u = a0*m0 = a0*(2^32+1) and +; M[] = (2^32-1) + (2^64-2^32)*2^64 + (2^64-2)*2^64*2 + (2^64-1)*2^64*3 + (2^64-1)*2^64*4 + (2^64-1)*2^64*5 +; +; u*M[] = u*(2^32-1) + u*(2^64-2^32)*2^64 + u*(2^64-2)*2^64*2 + u*(2^64-1)*2^64*3 + u*(2^64-1)*2^64*4 + u*(2^64-1)*2^64*5 +; = u*2^32 -u +; +u*2^128 -u*2^32*2^64 +; +u*2^192 -u*2^128 -u*2^128 +; +u*2^256 -u*2^192 +; +u*2^320 -u*2^256 +; +u*2^384 -u*2^320 +; +; = (u*2^32 -u) + u*2^384 +; -u*2^32*2^64 -u*2^128 +; +; = (u*2^32 -u) + u*B^5 +; -(u*2^32 +u*B)*B, where B=2^64 is the target radix +; +; so update of {a0:a1:a2:a3:a4:a5:a6} consisns of 2 steps: +; {a0:a1:a2:a3:a4:a5:a6} += {(u*2^32 -u):0:0:0:0:0:u} and +; {a0:a1:a2:a3:a4:a5:a6} -= {0:(u*2^32 +u*B):0:0:0:0:0} +; +%macro mred_step 9.nolist + %xdefine %%a6 %1 + %xdefine %%a5 %2 + %xdefine %%a4 %3 + %xdefine %%a3 %4 + %xdefine %%a2 %5 + %xdefine %%a1 %6 + %xdefine %%a0 %7 + %xdefine %%t2 %8 + %xdefine %%t1 %9 + + mov rax, %%a0 ;; u = (m0*a0) mod 2^64= ((2^32+1)*a0) mod 2^64 + shl rax, 32 + add rax, %%a0 + + mov %%t2, rax ;; (t2:t1) = u*2^32, store + shr %%t2, (64-32) + push %%t2 + mov %%t1, rax + shl %%t1, 32 + push %%t1 + + sub %%t1, rax ;; {t2:t1} = (2^32 -1)*u + sbb %%t2, 0 + + ;; {a0:a1:a2:a3:a4:a5:a6} += (u*2^32 -u) + u*2^64*6 + add %%a0, %%t1 ;; {a0:a1} += (u*2^32 -u) + mov %%t1, rdx ;; t1 = carry_inp + mov rdx, 0 + adc %%a1, %%t2 + adc %%a2, 0 + adc %%a3, 0 + adc %%a4, 0 + adc %%a5, 0 + adc %%a6, rax ;; a6 += u + cf + adc rdx, 0 ;; rdx = carry_out + + add %%a6, %%t1 ;; add input carry + adc rdx, 0 ;; rdx = carry_out + + pop %%t1 ;; restore {t2:t1} = u*2^32 + pop %%t2 + + add %%t2, rax ;; {t1:t2} = u*2^32 +u*2^64 + mov rax,0 + adc rax,0 + + ;; {a1:a2:a3:a4:a5:a6} -= (u*2^32 + u) + sub %%a1, %%t1 + sbb %%a2, %%t2 + sbb %%a3, rax + sbb %%a4, 0 + sbb %%a5, 0 + sbb %%a6, 0 + + sbb rdx, 0 ;; update carry_out +%endmacro + +align IPP_ALIGN_FACTOR +IPPASM p384r1_mred,PUBLIC + + USES_GPR rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 2 + +;; rdi = result +;; rsi = product buffer + + xor rdx, rdx + mov r8, qword [rsi] + mov r9, qword [rsi+sizeof(qword)] + mov r10, qword [rsi+sizeof(qword)*2] + mov r11, qword [rsi+sizeof(qword)*3] + mov r12, qword [rsi+sizeof(qword)*4] + mov r13, qword [rsi+sizeof(qword)*5] + mov r14, qword [rsi+sizeof(qword)*6] + mred_step r14,r13,r12,r11,r10,r9,r8, r15,rcx + ;mov qword [rdi+sizeof(qword)*0], r9 + ;mov qword [rdi+sizeof(qword)*1], r10 + ;mov qword [rdi+sizeof(qword)*2], r11 + ;mov qword [rdi+sizeof(qword)*3], r12 + ;mov qword [rdi+sizeof(qword)*4], r13 + ;mov qword [rdi+sizeof(qword)*5], r14 + + mov r8, qword [rsi+sizeof(qword)*7] + mred_step r8,r14,r13,r12,r11,r10,r9, r15,rcx + ;mov qword [rdi+sizeof(qword)*0], r10 + ;mov qword [rdi+sizeof(qword)*1], r11 + ;mov qword [rdi+sizeof(qword)*2], r12 + ;mov qword [rdi+sizeof(qword)*3], r13 + ;mov qword [rdi+sizeof(qword)*4], r14 + ;mov qword [rdi+sizeof(qword)*5], r8 + + mov r9, qword [rsi+sizeof(qword)*8] + mred_step r9,r8,r14,r13,r12,r11,r10, r15,rcx + ;mov qword [rdi+sizeof(qword)*0], r11 + ;mov qword [rdi+sizeof(qword)*1], r12 + ;mov qword [rdi+sizeof(qword)*2], r13 + ;mov qword [rdi+sizeof(qword)*3], r14 + ;mov qword [rdi+sizeof(qword)*4], r8 + ;mov qword [rdi+sizeof(qword)*5], r9 + + mov r10, qword [rsi+sizeof(qword)*9] + mred_step r10,r9,r8,r14,r13,r12,r11, r15,rcx + ;mov qword [rdi+sizeof(qword)*0], r12 + ;mov qword [rdi+sizeof(qword)*1], r13 + ;mov qword [rdi+sizeof(qword)*2], r14 + ;mov qword [rdi+sizeof(qword)*3], r8 + ;mov qword [rdi+sizeof(qword)*4], r9 + ;mov qword [rdi+sizeof(qword)*5], r10 + + mov r11, qword [rsi+sizeof(qword)*10] + mred_step r11,r10,r9,r8,r14,r13,r12, r15,rcx + ;mov qword [rdi+sizeof(qword)*0], r13 + ;mov qword [rdi+sizeof(qword)*1], r14 + ;mov qword [rdi+sizeof(qword)*2], r8 + ;mov qword [rdi+sizeof(qword)*3], r9 + ;mov qword [rdi+sizeof(qword)*4], r10 + ;mov qword [rdi+sizeof(qword)*5], r11 + + mov r12, qword [rsi+sizeof(qword)*11] + mred_step r12,r11,r10,r9,r8,r14,r13, r15,rcx ; {r12,r11,r10,r9,r8,r14} - result + mov qword [rdi+sizeof(qword)*0], r14 + mov qword [rdi+sizeof(qword)*1], r8 + mov qword [rdi+sizeof(qword)*2], r9 + mov qword [rdi+sizeof(qword)*3], r10 + mov qword [rdi+sizeof(qword)*4], r11 + mov qword [rdi+sizeof(qword)*5], r12 + + sub r14, qword [rel Lpoly+sizeof(qword)*0] + sbb r8, qword [rel Lpoly+sizeof(qword)*1] + sbb r9, qword [rel Lpoly+sizeof(qword)*2] + sbb r10, qword [rel Lpoly+sizeof(qword)*3] + sbb r11, qword [rel Lpoly+sizeof(qword)*4] + sbb r12, qword [rel Lpoly+sizeof(qword)*5] + sbb rdx, 0 + + mov rax, qword [rdi+sizeof(qword)*0] + cmovnz r14, rax + mov rax, qword [rdi+sizeof(qword)*1] + cmovnz r8, rax + mov rax, qword [rdi+sizeof(qword)*2] + cmovnz r9, rax + mov rax, qword [rdi+sizeof(qword)*3] + cmovnz r10, rax + mov rax, qword [rdi+sizeof(qword)*4] + cmovnz r11, rax + mov rax, qword [rdi+sizeof(qword)*5] + cmovnz r12, rax + + mov qword [rdi+sizeof(qword)*0], r14 + mov qword [rdi+sizeof(qword)*1], r8 + mov qword [rdi+sizeof(qword)*2], r9 + mov qword [rdi+sizeof(qword)*3], r10 + mov qword [rdi+sizeof(qword)*4], r11 + mov qword [rdi+sizeof(qword)*5], r12 + + REST_XMM + REST_GPR + ret +ENDFUNC p384r1_mred + +%ifndef _DISABLE_ECP_384R1_HARDCODED_BP_TBL_ +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; affine point selector +; +; void p384r1_select_ap_w5(AF_POINT *val, const AF_POINT *tbl, int idx); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p384r1_select_ap_w5,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13 + USES_XMM xmm6,xmm7,xmm8,xmm9,xmm10,xmm11,xmm12,xmm13,xmm14 + COMP_ABI 3 + +%xdefine val rdi +%xdefine in_t rsi +%xdefine idx edx + +%xdefine Xa xmm0 +%xdefine Xb xmm1 +%xdefine Xc xmm2 +%xdefine Ya xmm3 +%xdefine Yb xmm4 +%xdefine Yc xmm5 + +%xdefine TXa xmm6 +%xdefine TXb xmm7 +%xdefine TXc xmm8 +%xdefine TYa xmm9 +%xdefine TYb xmm10 +%xdefine TYc xmm11 + +%xdefine REQ_IDX xmm12 +%xdefine CUR_IDX xmm13 +%xdefine MASKDATA xmm14 + + movdqa CUR_IDX, oword [rel LOne] + + movd REQ_IDX, idx + pshufd REQ_IDX, REQ_IDX, 0 + + pxor Xa, Xa + pxor Xb, Xb + pxor Xc, Xc + pxor Ya, Ya + pxor Yb, Yb + pxor Yc, Yc + + ; Skip index = 0, is implicictly infty -> load with offset -1 + mov rcx, dword 16 +.select_loop: + movdqa MASKDATA, CUR_IDX ; MASK = CUR_IDX==REQ_IDX? 0xFF : 0x00 + pcmpeqd MASKDATA, REQ_IDX ; + paddd CUR_IDX, oword [rel LOne] + + movdqa TXa, oword [in_t+sizeof(oword)*0] + movdqa TXb, oword [in_t+sizeof(oword)*1] + movdqa TXc, oword [in_t+sizeof(oword)*2] + movdqa TYa, oword [in_t+sizeof(oword)*3] + movdqa TYb, oword [in_t+sizeof(oword)*4] + movdqa TYc, oword [in_t+sizeof(oword)*5] + add in_t, sizeof(oword)*6 + + pand TXa, MASKDATA + pand TXb, MASKDATA + pand TXc, MASKDATA + pand TYa, MASKDATA + pand TYb, MASKDATA + pand TYc, MASKDATA + + por Xa, TXa + por Xb, TXb + por Xc, TXc + por Ya, TYa + por Yb, TYb + por Yc, TYc + + dec rcx + jnz .select_loop + + movdqu oword [val+sizeof(oword)*0], Xa + movdqu oword [val+sizeof(oword)*1], Xb + movdqu oword [val+sizeof(oword)*2], Xc + movdqu oword [val+sizeof(oword)*3], Ya + movdqu oword [val+sizeof(oword)*4], Yb + movdqu oword [val+sizeof(oword)*5], Yc + + REST_XMM + REST_GPR + ret +ENDFUNC p384r1_select_ap_w5 + +%endif + +%endif ;; _IPP32E_M7 + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpp521r1funcs_montas.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpp521r1funcs_montas.asm new file mode 100644 index 0000000..1aba2bd --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpp521r1funcs_montas.asm @@ -0,0 +1,963 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; secp p521r1 specific implementation +; + + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_M7) + +%assign _xEMULATION_ 1 +%assign _ADCX_ADOX_ 1 + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR + +LOne DD 1,1,1,1,1,1,1,1 +LTwo DD 2,2,2,2,2,2,2,2 +LThree DD 3,3,3,3,3,3,3,3 + +;; The p521r1 polynomial +Lpoly DQ 0ffffffffffffffffh,0ffffffffffffffffh,0ffffffffffffffffh + DQ 0ffffffffffffffffh,0ffffffffffffffffh,0ffffffffffffffffh + DQ 0ffffffffffffffffh,0ffffffffffffffffh,000000000000001ffh + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p521r1_mul_by_2(uint64_t res[9], uint64_t a[9]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p521r1_mul_by_2,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 2 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 +%xdefine a4 r12 +%xdefine a5 r13 +%xdefine a6 r14 +%xdefine a7 r15 +%xdefine a8 rax +%xdefine ex rcx + +%xdefine t rdx + + xor ex, ex + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + mov a4, qword [rsi+sizeof(qword)*4] + mov a5, qword [rsi+sizeof(qword)*5] + mov a6, qword [rsi+sizeof(qword)*6] + mov a7, qword [rsi+sizeof(qword)*7] + mov a8, qword [rsi+sizeof(qword)*8] + + shld ex, a8, 1 + shld a8, a7, 1 + shld a7, a6, 1 + shld a6, a5, 1 + shld a5, a4, 1 + shld a4, a3, 1 + shld a3, a2, 1 + shld a2, a1, 1 + shld a1, a0, 1 + shl a0, 1 + + mov qword [rdi+sizeof(qword)*8], a8 + mov qword [rdi+sizeof(qword)*7], a7 + mov qword [rdi+sizeof(qword)*6], a6 + mov qword [rdi+sizeof(qword)*5], a5 + mov qword [rdi+sizeof(qword)*4], a4 + mov qword [rdi+sizeof(qword)*3], a3 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*0], a0 + + sub a0, qword [rel Lpoly+sizeof(qword)*0] + sbb a1, qword [rel Lpoly+sizeof(qword)*1] + sbb a2, qword [rel Lpoly+sizeof(qword)*2] + sbb a3, qword [rel Lpoly+sizeof(qword)*3] + sbb a4, qword [rel Lpoly+sizeof(qword)*4] + sbb a5, qword [rel Lpoly+sizeof(qword)*5] + sbb a6, qword [rel Lpoly+sizeof(qword)*6] + sbb a7, qword [rel Lpoly+sizeof(qword)*7] + sbb a8, qword [rel Lpoly+sizeof(qword)*8] + sbb ex, 0 + + mov t, qword [rdi+sizeof(qword)*0] + cmovnz a0, t + mov t, qword [rdi+sizeof(qword)*1] + cmovnz a1, t + mov t, qword [rdi+sizeof(qword)*2] + cmovnz a2, t + mov t, qword [rdi+sizeof(qword)*3] + cmovnz a3, t + mov t, qword [rdi+sizeof(qword)*4] + cmovnz a4, t + mov t, qword [rdi+sizeof(qword)*5] + cmovnz a5, t + mov t, qword [rdi+sizeof(qword)*6] + cmovnz a6, t + mov t, qword [rdi+sizeof(qword)*7] + cmovnz a7, t + mov t, qword [rdi+sizeof(qword)*8] + cmovnz a8, t + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + mov qword [rdi+sizeof(qword)*4], a4 + mov qword [rdi+sizeof(qword)*5], a5 + mov qword [rdi+sizeof(qword)*6], a6 + mov qword [rdi+sizeof(qword)*7], a7 + mov qword [rdi+sizeof(qword)*8], a8 + + REST_XMM + REST_GPR + ret +ENDFUNC p521r1_mul_by_2 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p521r1_div_by_2(uint64_t res[9], uint64_t a[9]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p521r1_div_by_2,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 2 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 +%xdefine a4 r12 +%xdefine a5 r13 +%xdefine a6 r14 +%xdefine a7 r15 +%xdefine a8 rax +%xdefine ex rcx + +%xdefine t rdx + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + mov a4, qword [rsi+sizeof(qword)*4] + mov a5, qword [rsi+sizeof(qword)*5] + mov a6, qword [rsi+sizeof(qword)*6] + mov a7, qword [rsi+sizeof(qword)*7] + mov a8, qword [rsi+sizeof(qword)*8] + + xor t, t + xor ex, ex + add a0, qword [rel Lpoly+sizeof(qword)*0] + adc a1, qword [rel Lpoly+sizeof(qword)*1] + adc a2, qword [rel Lpoly+sizeof(qword)*2] + adc a3, qword [rel Lpoly+sizeof(qword)*3] + adc a4, qword [rel Lpoly+sizeof(qword)*4] + adc a5, qword [rel Lpoly+sizeof(qword)*5] + adc a6, qword [rel Lpoly+sizeof(qword)*6] + adc a7, qword [rel Lpoly+sizeof(qword)*7] + adc a8, qword [rel Lpoly+sizeof(qword)*8] + adc ex, 0 + + test a0, 1 + cmovnz ex, t + mov t, qword [rsi+sizeof(qword)*0] + cmovnz a0, t + mov t, qword [rsi+sizeof(qword)*1] + cmovnz a1, t + mov t, qword [rsi+sizeof(qword)*2] + cmovnz a2, t + mov t, qword [rsi+sizeof(qword)*3] + cmovnz a3, t + mov t, qword [rsi+sizeof(qword)*4] + cmovnz a4, t + mov t, qword [rsi+sizeof(qword)*5] + cmovnz a5, t + mov t, qword [rsi+sizeof(qword)*6] + cmovnz a6, t + mov t, qword [rsi+sizeof(qword)*7] + cmovnz a7, t + mov t, qword [rsi+sizeof(qword)*8] + cmovnz a8, t + + shrd a0, a1, 1 + shrd a1, a2, 1 + shrd a2, a3, 1 + shrd a3, a4, 1 + shrd a4, a5, 1 + shrd a5, a6, 1 + shrd a6, a7, 1 + shrd a7, a8, 1 + shrd a8, ex, 1 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + mov qword [rdi+sizeof(qword)*4], a4 + mov qword [rdi+sizeof(qword)*5], a5 + mov qword [rdi+sizeof(qword)*6], a6 + mov qword [rdi+sizeof(qword)*7], a7 + mov qword [rdi+sizeof(qword)*8], a8 + + REST_XMM + REST_GPR + ret +ENDFUNC p521r1_div_by_2 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p521r1_mul_by_3(uint64_t res[9], uint64_t a[9]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p521r1_mul_by_3,PUBLIC +%assign LOCAL_FRAME sizeof(qword)*9 + USES_GPR rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 2 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 +%xdefine a4 r12 +%xdefine a5 r13 +%xdefine a6 r14 +%xdefine a7 r15 +%xdefine a8 rax +%xdefine ex rcx + +%xdefine t rdx + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + mov a4, qword [rsi+sizeof(qword)*4] + mov a5, qword [rsi+sizeof(qword)*5] + mov a6, qword [rsi+sizeof(qword)*6] + mov a7, qword [rsi+sizeof(qword)*7] + mov a8, qword [rsi+sizeof(qword)*8] + + xor ex, ex + shld ex, a8, 1 + shld a8, a7, 1 + shld a7, a6, 1 + shld a6, a5, 1 + shld a5, a4, 1 + shld a4, a3, 1 + shld a3, a2, 1 + shld a2, a1, 1 + shld a1, a0, 1 + shl a0, 1 + + mov qword [rsp+sizeof(qword)*8], a8 + mov qword [rsp+sizeof(qword)*7], a7 + mov qword [rsp+sizeof(qword)*6], a6 + mov qword [rsp+sizeof(qword)*5], a5 + mov qword [rsp+sizeof(qword)*4], a4 + mov qword [rsp+sizeof(qword)*3], a3 + mov qword [rsp+sizeof(qword)*2], a2 + mov qword [rsp+sizeof(qword)*1], a1 + mov qword [rsp+sizeof(qword)*0], a0 + + sub a0, qword [rel Lpoly+sizeof(qword)*0] + sbb a1, qword [rel Lpoly+sizeof(qword)*1] + sbb a2, qword [rel Lpoly+sizeof(qword)*2] + sbb a3, qword [rel Lpoly+sizeof(qword)*3] + sbb a4, qword [rel Lpoly+sizeof(qword)*4] + sbb a5, qword [rel Lpoly+sizeof(qword)*5] + sbb a6, qword [rel Lpoly+sizeof(qword)*6] + sbb a7, qword [rel Lpoly+sizeof(qword)*7] + sbb a8, qword [rel Lpoly+sizeof(qword)*8] + sbb ex, 0 + + mov t, qword [rsp+sizeof(qword)*0] + cmovb a0, t + mov t, qword [rsp+sizeof(qword)*1] + cmovb a1, t + mov t, qword [rsp+sizeof(qword)*2] + cmovb a2, t + mov t, qword [rsp+sizeof(qword)*3] + cmovb a3, t + mov t, qword [rsp+sizeof(qword)*4] + cmovb a4, t + mov t, qword [rsp+sizeof(qword)*5] + cmovb a5, t + mov t, qword [rsp+sizeof(qword)*6] + cmovb a6, t + mov t, qword [rsp+sizeof(qword)*7] + cmovb a7, t + mov t, qword [rsp+sizeof(qword)*8] + cmovb a8, t + + xor ex, ex + add a0, qword [rsi+sizeof(qword)*0] + adc a1, qword [rsi+sizeof(qword)*1] + adc a2, qword [rsi+sizeof(qword)*2] + adc a3, qword [rsi+sizeof(qword)*3] + adc a4, qword [rsi+sizeof(qword)*4] + adc a5, qword [rsi+sizeof(qword)*5] + adc a6, qword [rsi+sizeof(qword)*6] + adc a7, qword [rsi+sizeof(qword)*7] + adc a8, qword [rsi+sizeof(qword)*8] + adc ex, 0 + + mov qword [rsp+sizeof(qword)*0], a0 + mov qword [rsp+sizeof(qword)*1], a1 + mov qword [rsp+sizeof(qword)*2], a2 + mov qword [rsp+sizeof(qword)*3], a3 + mov qword [rsp+sizeof(qword)*4], a4 + mov qword [rsp+sizeof(qword)*5], a5 + mov qword [rsp+sizeof(qword)*6], a6 + mov qword [rsp+sizeof(qword)*7], a7 + mov qword [rsp+sizeof(qword)*8], a8 + + sub a0, qword [rel Lpoly+sizeof(qword)*0] + sbb a1, qword [rel Lpoly+sizeof(qword)*1] + sbb a2, qword [rel Lpoly+sizeof(qword)*2] + sbb a3, qword [rel Lpoly+sizeof(qword)*3] + sbb a4, qword [rel Lpoly+sizeof(qword)*4] + sbb a5, qword [rel Lpoly+sizeof(qword)*5] + sbb a6, qword [rel Lpoly+sizeof(qword)*6] + sbb a7, qword [rel Lpoly+sizeof(qword)*7] + sbb a8, qword [rel Lpoly+sizeof(qword)*8] + sbb ex, 0 + + mov t, qword [rsp+sizeof(qword)*0] + cmovb a0, t + mov t, qword [rsp+sizeof(qword)*1] + cmovb a1, t + mov t, qword [rsp+sizeof(qword)*2] + cmovb a2, t + mov t, qword [rsp+sizeof(qword)*3] + cmovb a3, t + mov t, qword [rsp+sizeof(qword)*4] + cmovb a4, t + mov t, qword [rsp+sizeof(qword)*5] + cmovb a5, t + mov t, qword [rsp+sizeof(qword)*6] + cmovb a6, t + mov t, qword [rsp+sizeof(qword)*7] + cmovb a7, t + mov t, qword [rsp+sizeof(qword)*8] + cmovb a8, t + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + mov qword [rdi+sizeof(qword)*4], a4 + mov qword [rdi+sizeof(qword)*5], a5 + mov qword [rdi+sizeof(qword)*6], a6 + mov qword [rdi+sizeof(qword)*7], a7 + mov qword [rdi+sizeof(qword)*8], a8 + + REST_XMM + REST_GPR + ret +ENDFUNC p521r1_mul_by_3 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p521r1_add(uint64_t res[9], uint64_t a[9], uint64_t b[9]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p521r1_add,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbx,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 3 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 +%xdefine a4 r12 +%xdefine a5 r13 +%xdefine a6 r14 +%xdefine a7 r15 +%xdefine a8 rax +%xdefine ex rcx + +%xdefine t rdx + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + mov a4, qword [rsi+sizeof(qword)*4] + mov a5, qword [rsi+sizeof(qword)*5] + mov a6, qword [rsi+sizeof(qword)*6] + mov a7, qword [rsi+sizeof(qword)*7] + mov a8, qword [rsi+sizeof(qword)*8] + + xor ex, ex + add a0, qword [rdx+sizeof(qword)*0] + adc a1, qword [rdx+sizeof(qword)*1] + adc a2, qword [rdx+sizeof(qword)*2] + adc a3, qword [rdx+sizeof(qword)*3] + adc a4, qword [rdx+sizeof(qword)*4] + adc a5, qword [rdx+sizeof(qword)*5] + adc a6, qword [rdx+sizeof(qword)*6] + adc a7, qword [rdx+sizeof(qword)*7] + adc a8, qword [rdx+sizeof(qword)*8] + adc ex, 0 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + mov qword [rdi+sizeof(qword)*4], a4 + mov qword [rdi+sizeof(qword)*5], a5 + mov qword [rdi+sizeof(qword)*6], a6 + mov qword [rdi+sizeof(qword)*7], a7 + mov qword [rdi+sizeof(qword)*8], a8 + + sub a0, qword [rel Lpoly+sizeof(qword)*0] + sbb a1, qword [rel Lpoly+sizeof(qword)*1] + sbb a2, qword [rel Lpoly+sizeof(qword)*2] + sbb a3, qword [rel Lpoly+sizeof(qword)*3] + sbb a4, qword [rel Lpoly+sizeof(qword)*4] + sbb a5, qword [rel Lpoly+sizeof(qword)*5] + sbb a6, qword [rel Lpoly+sizeof(qword)*6] + sbb a7, qword [rel Lpoly+sizeof(qword)*7] + sbb a8, qword [rel Lpoly+sizeof(qword)*8] + sbb ex, 0 + + mov t, qword [rdi+sizeof(qword)*0] + cmovb a0, t + mov t, qword [rdi+sizeof(qword)*1] + cmovb a1, t + mov t, qword [rdi+sizeof(qword)*2] + cmovb a2, t + mov t, qword [rdi+sizeof(qword)*3] + cmovb a3, t + mov t, qword [rdi+sizeof(qword)*4] + cmovb a4, t + mov t, qword [rdi+sizeof(qword)*5] + cmovb a5, t + mov t, qword [rdi+sizeof(qword)*6] + cmovb a6, t + mov t, qword [rdi+sizeof(qword)*7] + cmovb a7, t + mov t, qword [rdi+sizeof(qword)*8] + cmovb a8, t + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + mov qword [rdi+sizeof(qword)*4], a4 + mov qword [rdi+sizeof(qword)*5], a5 + mov qword [rdi+sizeof(qword)*6], a6 + mov qword [rdi+sizeof(qword)*7], a7 + mov qword [rdi+sizeof(qword)*8], a8 + + REST_XMM + REST_GPR + ret +ENDFUNC p521r1_add + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p521r1_sub(uint64_t res[9], uint64_t a[9], uint64_t b[9]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p521r1_sub,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbx,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 3 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 +%xdefine a4 r12 +%xdefine a5 r13 +%xdefine a6 r14 +%xdefine a7 r15 +%xdefine a8 rax +%xdefine ex rcx + +%xdefine t rdx + + mov a0, qword [rsi+sizeof(qword)*0] ; a + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + mov a4, qword [rsi+sizeof(qword)*4] + mov a5, qword [rsi+sizeof(qword)*5] + mov a6, qword [rsi+sizeof(qword)*6] + mov a7, qword [rsi+sizeof(qword)*7] + mov a8, qword [rsi+sizeof(qword)*8] + + xor ex, ex + sub a0, qword [rdx+sizeof(qword)*0] ; a-b + sbb a1, qword [rdx+sizeof(qword)*1] + sbb a2, qword [rdx+sizeof(qword)*2] + sbb a3, qword [rdx+sizeof(qword)*3] + sbb a4, qword [rdx+sizeof(qword)*4] + sbb a5, qword [rdx+sizeof(qword)*5] + sbb a6, qword [rdx+sizeof(qword)*6] + sbb a7, qword [rdx+sizeof(qword)*7] + sbb a8, qword [rdx+sizeof(qword)*8] + sbb ex, 0 ; ex = a>=b? 0 : -1 + + mov qword [rdi+sizeof(qword)*0], a0 ; store (a-b) + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + mov qword [rdi+sizeof(qword)*4], a4 + mov qword [rdi+sizeof(qword)*5], a5 + mov qword [rdi+sizeof(qword)*6], a6 + mov qword [rdi+sizeof(qword)*7], a7 + mov qword [rdi+sizeof(qword)*8], a8 + + add a0, qword [rel Lpoly+sizeof(qword)*0] ; (a-b) +poly + adc a1, qword [rel Lpoly+sizeof(qword)*1] + adc a2, qword [rel Lpoly+sizeof(qword)*2] + adc a3, qword [rel Lpoly+sizeof(qword)*3] + adc a4, qword [rel Lpoly+sizeof(qword)*4] + adc a5, qword [rel Lpoly+sizeof(qword)*5] + adc a6, qword [rel Lpoly+sizeof(qword)*6] + adc a7, qword [rel Lpoly+sizeof(qword)*7] + adc a8, qword [rel Lpoly+sizeof(qword)*8] + + test ex, ex ; r = (ex)? ((a-b)+poly) : (a-b) + + mov t, qword [rdi+sizeof(qword)*0] + cmovz a0, t + mov t, qword [rdi+sizeof(qword)*1] + cmovz a1, t + mov t, qword [rdi+sizeof(qword)*2] + cmovz a2, t + mov t, qword [rdi+sizeof(qword)*3] + cmovz a3, t + mov t, qword [rdi+sizeof(qword)*4] + cmovz a4, t + mov t, qword [rdi+sizeof(qword)*5] + cmovz a5, t + mov t, qword [rdi+sizeof(qword)*6] + cmovz a6, t + mov t, qword [rdi+sizeof(qword)*7] + cmovz a7, t + mov t, qword [rdi+sizeof(qword)*8] + cmovz a8, t + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + mov qword [rdi+sizeof(qword)*4], a4 + mov qword [rdi+sizeof(qword)*5], a5 + mov qword [rdi+sizeof(qword)*6], a6 + mov qword [rdi+sizeof(qword)*7], a7 + mov qword [rdi+sizeof(qword)*8], a8 + + REST_XMM + REST_GPR + ret +ENDFUNC p521r1_sub + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void p521r1_neg(uint64_t res[9], uint64_t a[9]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p521r1_neg,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 2 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 +%xdefine a4 r12 +%xdefine a5 r13 +%xdefine a6 r14 +%xdefine a7 r15 +%xdefine a8 rax +%xdefine ex rcx + +%xdefine t rdx + + xor a0, a0 + xor a1, a1 + xor a2, a2 + xor a3, a3 + xor a4, a4 + xor a5, a5 + xor a6, a6 + xor a7, a7 + xor a8, a8 + + xor ex, ex + sub a0, qword [rsi+sizeof(qword)*0] + sbb a1, qword [rsi+sizeof(qword)*1] + sbb a2, qword [rsi+sizeof(qword)*2] + sbb a3, qword [rsi+sizeof(qword)*3] + sbb a4, qword [rsi+sizeof(qword)*4] + sbb a5, qword [rsi+sizeof(qword)*5] + sbb a6, qword [rsi+sizeof(qword)*6] + sbb a7, qword [rsi+sizeof(qword)*7] + sbb a8, qword [rsi+sizeof(qword)*8] + sbb ex, 0 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + mov qword [rdi+sizeof(qword)*4], a4 + mov qword [rdi+sizeof(qword)*5], a5 + mov qword [rdi+sizeof(qword)*6], a6 + mov qword [rdi+sizeof(qword)*7], a7 + mov qword [rdi+sizeof(qword)*8], a8 + + add a0, qword [rel Lpoly+sizeof(qword)*0] + adc a1, qword [rel Lpoly+sizeof(qword)*1] + adc a2, qword [rel Lpoly+sizeof(qword)*2] + adc a3, qword [rel Lpoly+sizeof(qword)*3] + adc a4, qword [rel Lpoly+sizeof(qword)*4] + adc a5, qword [rel Lpoly+sizeof(qword)*5] + adc a6, qword [rel Lpoly+sizeof(qword)*6] + adc a7, qword [rel Lpoly+sizeof(qword)*7] + adc a8, qword [rel Lpoly+sizeof(qword)*8] + test ex, ex + + mov t, qword [rdi+sizeof(qword)*0] + cmovz a0, t + mov t, qword [rdi+sizeof(qword)*1] + cmovz a1, t + mov t, qword [rdi+sizeof(qword)*2] + cmovz a2, t + mov t, qword [rdi+sizeof(qword)*3] + cmovz a3, t + mov t, qword [rdi+sizeof(qword)*4] + cmovz a4, t + mov t, qword [rdi+sizeof(qword)*5] + cmovz a5, t + mov t, qword [rdi+sizeof(qword)*6] + cmovz a6, t + mov t, qword [rdi+sizeof(qword)*7] + cmovz a7, t + mov t, qword [rdi+sizeof(qword)*8] + cmovz a8, t + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + mov qword [rdi+sizeof(qword)*4], a4 + mov qword [rdi+sizeof(qword)*5], a5 + mov qword [rdi+sizeof(qword)*6], a6 + mov qword [rdi+sizeof(qword)*7], a7 + mov qword [rdi+sizeof(qword)*8], a8 + + REST_XMM + REST_GPR + ret +ENDFUNC p521r1_neg + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; p521r1_mred(uint64_t res[9], const uint64_t product[9*2]); +; +; modulus = 2^521 -1 +; [17] [0] +; m0 = 1 +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%macro mred_step 12.nolist + %xdefine %%EX %1 + %xdefine %%X8 %2 + %xdefine %%X7 %3 + %xdefine %%X6 %4 + %xdefine %%X5 %5 + %xdefine %%X4 %6 + %xdefine %%X3 %7 + %xdefine %%X2 %8 + %xdefine %%X1 %9 + %xdefine %%X0 %10 + %xdefine %%T %11 + %xdefine %%CF %12 + + mov %%T, %%X0 ;; u0 = X0 + shr %%T, (64-(521-512)) ;; (T:X0) = u0<<9 + shl %%X0, (521-512) + add %%T, %%CF + + add %%X8, %%X0 + adc %%EX, %%T + mov %%CF, dword 0 + adc %%CF, 0 +%endmacro + +align IPP_ALIGN_FACTOR +IPPASM p521r1_mred,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbx,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 2 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 +%xdefine a4 r12 +%xdefine a5 r13 +%xdefine a6 r14 +%xdefine a7 r15 +%xdefine a8 rax +%xdefine ex rcx + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + mov a4, qword [rsi+sizeof(qword)*4] + mov a5, qword [rsi+sizeof(qword)*5] + mov a6, qword [rsi+sizeof(qword)*6] + mov a7, qword [rsi+sizeof(qword)*7] + mov a8, qword [rsi+sizeof(qword)*8] + mov ex, qword [rsi+sizeof(qword)*9] + +%xdefine t rbx +%xdefine carry rdx + + xor carry, carry + mred_step ex,a8,a7,a6,a5,a4,a3,a2,a1,a0, t,carry ;; step 0 {rcx,rax,r15,r14,r13,r12,r11,r10,r9,r8} -> {rcx,rax,r15,r14,r13,r12,r11,r10,r9,--} + + mov a0, qword [rsi+sizeof(qword)*10] + mred_step a0,ex,a8,a7,a6,a5,a4,a3,a2,a1, t,carry ;; step 1 {r8,rcx,rax,r15,r14,r13,r12,r11,r10,r9} -> {r8,rcx,rax,r15,r14,r13,r12,r11,r10,--} + + mov a1, qword [rsi+sizeof(qword)*11] + mred_step a1,a0,ex,a8,a7,a6,a5,a4,a3,a2, t,carry ;; step 2 {r9,r8,rcx,rax,r15,r14,r13,r12,r11,r10} -> {r9,r8,rcx,rax,r15,r14,r13,r12,r11, --} + + mov a2, qword [rsi+sizeof(qword)*12] + mred_step a2,a1,a0,ex,a8,a7,a6,a5,a4,a3, t,carry ;; step 3 {r10,r9,r8,rcx,rax,r15,r14,r13,r12,r11} -> {r10,r9,r8,rcx,rax,r15,r14,r13,r12, --} + + mov a3, qword [rsi+sizeof(qword)*13] + mred_step a3,a2,a1,a0,ex,a8,a7,a6,a5,a4, t,carry ;; step 4 {r11,r10,r9,r8,rcx,rax,r15,r14,r13,r12} -> {r11,r10,r9,r8,rcx,rax,r15,r14,r13, --} + + mov a4, qword [rsi+sizeof(qword)*14] + mred_step a4,a3,a2,a1,a0,ex,a8,a7,a6,a5, t,carry ;; step 5 {r12,r11,r10,r9,r8,rcx,rax,r15,r14,r13} -> {r12,r11,r10,r9,r8,rcx,rax,r15,r14, --} + + mov a5, qword [rsi+sizeof(qword)*15] + mred_step a5,a4,a3,a2,a1,a0,ex,a8,a7,a6, t,carry ;; step 6 {r13,r12,r11,r10,r9,r8,rcx,rax,r15,r14} -> {r13,r12,r11,r10,r9,r8,rcx,rax,r15, --} + + mov a6, qword [rsi+sizeof(qword)*16] + mred_step a6,a5,a4,a3,a2,a1,a0,ex,a8,a7, t,carry ;; step 7 {r14,r13,r12,r11,r10,r9,r8,rcx,rax,r15} -> {r14,r13,r12,r11,r10,r9,r8,rcx,rax, --} + + mov a7, qword [rsi+sizeof(qword)*17] + mred_step a7,a6,a5,a4,a3,a2,a1,a0,ex,a8, t,carry ;; step 8 {r15,r14,r13,r12,r11,r10,r9,r8,rcx,rax} -> {r15,r14,r13,r12,r11,r10,r9,r8,rcx, --} + + ;; temporary result: a8,a7,a6,a5,a4,a3,a2,a1,a0,ex + mov qword [rdi+sizeof(qword)*0], ex + mov qword [rdi+sizeof(qword)*1], a0 + mov qword [rdi+sizeof(qword)*2], a1 + mov qword [rdi+sizeof(qword)*3], a2 + mov qword [rdi+sizeof(qword)*4], a3 + mov qword [rdi+sizeof(qword)*5], a4 + mov qword [rdi+sizeof(qword)*6], a5 + mov qword [rdi+sizeof(qword)*7], a6 + mov qword [rdi+sizeof(qword)*8], a7 + + ;; sub modulus + sub ex, qword [rel Lpoly+sizeof(qword)*0] + sbb a0, qword [rel Lpoly+sizeof(qword)*1] + sbb a1, qword [rel Lpoly+sizeof(qword)*2] + sbb a2, qword [rel Lpoly+sizeof(qword)*3] + sbb a3, qword [rel Lpoly+sizeof(qword)*4] + sbb a4, qword [rel Lpoly+sizeof(qword)*5] + sbb a5, qword [rel Lpoly+sizeof(qword)*6] + sbb a6, qword [rel Lpoly+sizeof(qword)*7] + sbb a7, qword [rel Lpoly+sizeof(qword)*8] + + ;; masked copy + mov t, qword [rdi+sizeof(qword)*0] + cmovb ex, t + mov t, qword [rdi+sizeof(qword)*1] + cmovb a0, t + mov t, qword [rdi+sizeof(qword)*2] + cmovb a1, t + mov t, qword [rdi+sizeof(qword)*3] + cmovb a2, t + mov t, qword [rdi+sizeof(qword)*4] + cmovb a3, t + mov t, qword [rdi+sizeof(qword)*5] + cmovb a4, t + mov t, qword [rdi+sizeof(qword)*6] + cmovb a5, t + mov t, qword [rdi+sizeof(qword)*7] + cmovb a6, t + mov t, qword [rdi+sizeof(qword)*8] + cmovb a7, t + + ;; store result: a7,a6,a5,a4,a3,a2,a1,a0,ex + mov qword [rdi+sizeof(qword)*0], ex + mov qword [rdi+sizeof(qword)*1], a0 + mov qword [rdi+sizeof(qword)*2], a1 + mov qword [rdi+sizeof(qword)*3], a2 + mov qword [rdi+sizeof(qword)*4], a3 + mov qword [rdi+sizeof(qword)*5], a4 + mov qword [rdi+sizeof(qword)*6], a5 + mov qword [rdi+sizeof(qword)*7], a6 + mov qword [rdi+sizeof(qword)*8], a7 + + REST_XMM + REST_GPR + ret +ENDFUNC p521r1_mred + +%ifndef _DISABLE_ECP_521R1_HARDCODED_BP_TBL_ +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; affine point selector +; +; void p521r1_select_ap_w5(AF_POINT *val, const AF_POINT *tbl, int idx); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM p521r1_select_ap_w5,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13 + USES_XMM xmm6,xmm7,xmm8,xmm9,xmm10,xmm11,xmm14 + COMP_ABI 3 + +%xdefine val rdi +%xdefine in_t rsi +%xdefine idx edx + +%xdefine xyz0 xmm0 +%xdefine xyz1 xmm1 +%xdefine xyz2 xmm2 +%xdefine xyz3 xmm3 +%xdefine xyz4 xmm4 +%xdefine xyz5 xmm5 +%xdefine xyz6 xmm6 +%xdefine xyz7 xmm7 +%xdefine xyz8 xmm8 + +%xdefine REQ_IDX xmm9 +%xdefine CUR_IDX xmm10 +%xdefine MASKDATA xmm11 +%xdefine TMP xmm12 + + movdqa CUR_IDX, oword [rel LOne] + + movd REQ_IDX, idx + pshufd REQ_IDX, REQ_IDX, 0 + + pxor xyz0, xyz0 + pxor xyz1, xyz1 + pxor xyz2, xyz2 + pxor xyz3, xyz3 + pxor xyz4, xyz4 + pxor xyz5, xyz5 + pxor xyz6, xyz6 + pxor xyz7, xyz7 + pxor xyz8, xyz8 + + ; Skip index = 0, is implicictly infty -> load with offset -1 + mov rcx, dword 16 +.select_loop: + movdqa MASKDATA, CUR_IDX ; MASK = CUR_IDX==REQ_IDX? 0xFF : 0x00 + pcmpeqd MASKDATA, REQ_IDX ; + paddd CUR_IDX, oword [rel LOne] + + movdqa TMP, oword [in_t+sizeof(oword)*0] + pand TMP, MASKDATA + por xyz0, TMP + movdqa TMP, oword [in_t+sizeof(oword)*1] + pand TMP, MASKDATA + por xyz1, TMP + movdqa TMP, oword [in_t+sizeof(oword)*2] + pand TMP, MASKDATA + por xyz2, TMP + movdqa TMP, oword [in_t+sizeof(oword)*3] + pand TMP, MASKDATA + por xyz3, TMP + movdqa TMP, oword [in_t+sizeof(oword)*4] + pand TMP, MASKDATA + por xyz4, TMP + movdqa TMP, oword [in_t+sizeof(oword)*5] + pand TMP, MASKDATA + por xyz5, TMP + movdqa TMP, oword [in_t+sizeof(oword)*6] + pand TMP, MASKDATA + por xyz6, TMP + movdqa TMP, oword [in_t+sizeof(oword)*7] + pand TMP, MASKDATA + por xyz7, TMP + movdqa TMP, oword [in_t+sizeof(oword)*8] + pand TMP, MASKDATA + por xyz8, TMP + + add in_t, sizeof(qword)*(9*2) + dec rcx + jnz .select_loop + + movdqu oword [val+sizeof(oword)*0], xyz0 + movdqu oword [val+sizeof(oword)*1], xyz1 + movdqu oword [val+sizeof(oword)*2], xyz2 + movdqu oword [val+sizeof(oword)*3], xyz3 + movdqu oword [val+sizeof(oword)*4], xyz4 + movdqu oword [val+sizeof(oword)*5], xyz5 + movdqu oword [val+sizeof(oword)*6], xyz6 + movdqu oword [val+sizeof(oword)*7], xyz7 + movdqu oword [val+sizeof(oword)*8], xyz8 + + REST_XMM + REST_GPR + ret +ENDFUNC p521r1_select_ap_w5 + +%endif + +%endif ;; _IPP32E_M7 + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcppurgeblkm7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcppurgeblkm7as.asm new file mode 100644 index 0000000..b02619a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcppurgeblkm7as.asm @@ -0,0 +1,78 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Purge block +; +; +; + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_M7) + +segment .text align=IPP_ALIGN_FACTOR + + +;*************************************************************** +;* Purpose: Clear memory block +;* +;* void PurgeBlock(Ipp8u *pDst, int len) +;* +;*************************************************************** + +;; +;; Lib = M7 +;; +align IPP_ALIGN_FACTOR +IPPASM PurgeBlock,PUBLIC + USES_GPR rsi,rdi + USES_XMM + COMP_ABI 2 +;; rdi: pDst: BYTE, ; mem being clear +;; rsi: len: DWORD ; length + + movsxd rcx, esi ; store stream length + xor rax, rax + sub rcx, sizeof(qword) + jl .test_purge +.purge8: + mov qword [rdi], rax ; clear + add rdi, sizeof(qword) + sub rcx, sizeof(qword) + jge .purge8 + +.test_purge: + add rcx, sizeof(qword) + jz .quit +.purge1: + mov byte [rdi], al + add rdi, sizeof(byte) + sub rcx, sizeof(byte) + jg .purge1 + +.quit: + REST_XMM + REST_GPR + ret +ENDFUNC PurgeBlock + +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprc4m7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprc4m7as.asm new file mode 100644 index 0000000..4991757 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprc4m7as.asm @@ -0,0 +1,114 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; ARCFour +; +; Content: +; ARCFourKernel() +; +; + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_M7) + +segment .text align=IPP_ALIGN_FACTOR + + +;*************************************************************** +;* Purpose: RC4 kernel +;* +;* void ARCFourProcessData(const Ipp8u *pSrc, Ipp8u *pDst, int len, +;* IppsARCFourState* pCtx) +;* +;*************************************************************** + +;; +;; Lib = M7 +;; +;; Caller = ippsARCFourEncrypt +;; Caller = ippsARCFourDecrypt +;; +align IPP_ALIGN_FACTOR +IPPASM ARCFourProcessData,PUBLIC + USES_GPR rsi,rdi,rbx,rbp + USES_XMM + COMP_ABI 4 +;; rdi: pSrc: BYTE, ; input stream +;; rsi: pDst: BYTE, ; output stream +;; rdx: len: DWORD, ; stream length +;; rcx: pCtx: BYTE ; context + + movsxd r8, edx + test r8, r8 ; test length + mov rbp, rcx ; copy pointer context + jz .quit + + movzx rax, byte [rbp+4] ; extract x + movzx rbx, byte [rbp+8] ; extract y + + lea rbp, [rbp+12] ; sbox + + add rax,1 ; x = (x+1)&0xFF + movzx rax, al + movzx rcx, byte [rbp+rax*4] ; tx = S[x] + +;; +;; main code +;; +align IPP_ALIGN_FACTOR +.main_loop: + add rbx, rcx ; y = (x+tx)&0xFF + movzx rbx, bl + add rdi, 1 + add rsi, 1 + movzx rdx, byte [rbp+rbx*4] ; ty = S[y] + + mov dword [rbp+rbx*4],ecx ; S[y] = tx + add rcx, rdx ; tmp_idx = (tx+ty)&0xFF + movzx rcx, cl + mov dword [rbp+rax*4],edx ; S[x] = ty + + mov dl, byte [rbp+rcx*4] ; byte of gamma + add rax, 1 ; next x = (x+1)&0xFF + movzx rax, al + + xor dl,byte [rdi-1] ; gamma ^= src + sub r8, 1 + movzx rcx, byte [rbp+rax*4] ; next tx = S[x] + mov byte [rsi-1],dl ; store result + jne .main_loop + + lea rbp, [rbp-12] ; pointer to context + + sub rax, 1 ; actual new x counter + movzx rax, al + mov dword [rbp+4], eax ; update x conter + mov dword [rbp+8], ebx ; updtae y counter + +.quit: + REST_XMM + REST_GPR + ret +ENDFUNC ARCFourProcessData + +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128ccme9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128ccme9as.asm new file mode 100644 index 0000000..66ff4b5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128ccme9as.asm @@ -0,0 +1,289 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Inverse Cipher function +; +; Content: +; AuthEncrypt_RIJ128_AES_NI() +; DecryptAuth_RIJ128_AES_NI() +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_AES_NI_ENABLING_ == _FEATURE_ON_) || (_AES_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_Y8) + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR +u128_str DB 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 +increment DQ 1,0 + +;*************************************************************** +;* Purpose: Authenticate and Encrypt +;* +;* void AuthEncrypt_RIJ128_AES_NI(Ipp8u* outBlk, +;* const Ipp8u* inpBlk, +;* int nr, +;* const Ipp8u* pRKey, +;* Ipp32u len, +;* Ipp8u* pLocalState) +;* inp localCtx: +;* MAC +;* CTRi +;* CTRi mask +;* +;* out localCtx: +;* new MAC +;* S = enc(CTRi) +;*************************************************************** + +;; +;; Lib = Y8 +;; +;; Caller = ippsAES_CCMEncrypt +;; +align IPP_ALIGN_FACTOR +IPPASM AuthEncrypt_RIJ128_AES_NI,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,rbx + USES_XMM xmm6,xmm7 + COMP_ABI 6 +;; rdi: pInpBlk: BYTE, ; input blocks address +;; rsi: pOutBlk: BYTE, ; output blocks address +;; rdx: nr: DWORD, ; number of rounds +;; rcx pKey: BYTE, ; key material address +;; r8d length: DWORD, ; length (bytes) +;; r9 pLocCtx: BYTE ; pointer to the localState + +%assign BYTES_PER_BLK (16) + + movdqa xmm0, oword [r9] ; MAC + movdqa xmm2, oword [r9+sizeof(oword)] ; CTRi block + movdqa xmm1, oword [r9+sizeof(oword)*2] ; CTR mask + + movdqa xmm7, oword [rel u128_str] + + pshufb xmm2, xmm7 ; CTRi block (LE) + pshufb xmm1, xmm7 ; CTR mask + + movdqa xmm3, xmm1 + pandn xmm3, xmm2 ; CTR block template + pand xmm2, xmm1 ; CTR value + + mov r8d, r8d ; expand length + movsxd rdx, edx ; expand number of rounds + + lea rdx, [rdx*4] ; nrCounter = -nr*16 + lea rdx, [rdx*4] ; pKey += nr*16 + lea rcx, [rcx+rdx] + neg rdx + mov rbx, rdx + +align IPP_ALIGN_FACTOR +;; +;; block-by-block processing +;; +.blk_loop: + movdqu xmm4, oword [rdi] ; input block src[i] + pxor xmm0, xmm4 ; MAC ^= src[i] + + movdqa xmm5, xmm3 + paddq xmm2, oword [rel increment] ; advance counter bits + pand xmm2, xmm1 ; and mask them + por xmm5, xmm2 + pshufb xmm5, xmm7 ; CTRi (BE) + + movdqa xmm6, oword [rcx+rdx] ; keys for whitening + add rdx, 16 + + pxor xmm5, xmm6 ; whitening (CTRi) + pxor xmm0, xmm6 ; whitening (MAC) + + movdqa xmm6, oword [rcx+rdx] ; pre load operation's keys + +align IPP_ALIGN_FACTOR +.cipher_loop: + aesenc xmm5, xmm6 ; regular round (CTRi) + aesenc xmm0, xmm6 ; regular round (MAC) + movdqa xmm6, oword [rcx+rdx+16] + add rdx, 16 + jnz .cipher_loop + aesenclast xmm5, xmm6 ; irregular round (CTRi) + aesenclast xmm0, xmm6 ; irregular round (MAC) + + pxor xmm4, xmm5 ; dst[i] = src[i] ^ ENC(CTRi) + movdqu oword [rsi], xmm4 + + mov rdx, rbx + add rsi, BYTES_PER_BLK + add rdi, BYTES_PER_BLK + sub r8, BYTES_PER_BLK + jnz .blk_loop + + movdqu oword [r9], xmm0 ; update MAC value + movdqu oword [r9+sizeof(oword)], xmm5 ; update ENC(Ctri) + + pxor xmm6, xmm6 + + REST_XMM + REST_GPR + ret +ENDFUNC AuthEncrypt_RIJ128_AES_NI + +;*************************************************************** +;* Purpose: Decrypt and Authenticate +;* +;* void DecryptAuth_RIJ128_AES_NI(Ipp8u* outBlk, +;* const Ipp8u* inpBlk, +;* int nr, +;* const Ipp8u* pRKey, +;* Ipp32u len, +;* Ipp8u* pLocalState) +;* inp localCtx: +;* MAC +;* CTRi +;* CTRi mask +;* +;* out localCtx: +;* new MAC +;* S = enc(CTRi) +;*************************************************************** + +;; +;; Lib = Y8 +;; +;; Caller = ippsAES_CCMDecrypt +;; +align IPP_ALIGN_FACTOR +IPPASM DecryptAuth_RIJ128_AES_NI,PUBLIC +%assign LOCAL_FRAME sizeof(qword) + USES_GPR rsi,rdi,rbx + USES_XMM xmm6,xmm7 + COMP_ABI 6 +;; rdi: pInpBlk: BYTE, ; input blocks address +;; rsi: pOutBlk: BYTE, ; output blocks address +;; rdx: nr: DWORD, ; number of rounds +;; rcx pKey: BYTE, ; key material address +;; r8d length: DWORD, ; length (bytes) +;; r9 pLocCtx: BYTE ; pointer to the localState + +%assign BYTES_PER_BLK (16) + + movdqa xmm0, oword [r9] ; MAC + movdqa xmm2, oword [r9+sizeof(oword)] ; CTRi block + movdqa xmm1, oword [r9+sizeof(oword)*2] ; CTR mask + + movdqa xmm7, oword [rel u128_str] + + pshufb xmm2, xmm7 ; CTRi block (LE) + pshufb xmm1, xmm7 ; CTR mask + + movdqa xmm3, xmm1 + pandn xmm3, xmm2 ; CTR block template + pand xmm2, xmm1 ; CTR value + + mov r8d, r8d ; expand length + movsxd rdx, edx ; expand number of rounds + + lea rdx, [rdx*4] ; nrCounter = -nr*16 + lea rdx, [rdx*4] ; pKey += nr*16 + lea rcx, [rcx+rdx] + neg rdx + mov rbx, rdx + +align IPP_ALIGN_FACTOR +;; +;; block-by-block processing +;; +.blk_loop: + + ;;;;;;;;;;;;;;;;; + ;; decryption + ;;;;;;;;;;;;;;;;; + movdqu xmm4, oword [rdi] ; input block src[i] + + movdqa xmm5, xmm3 + paddq xmm2, oword [rel increment] ; advance counter bits + pand xmm2, xmm1 ; and mask them + por xmm5, xmm2 + pshufb xmm5, xmm7 ; CTRi (BE) + + movdqa xmm6, oword [rcx+rdx] ; keys for whitening + add rdx, 16 + + pxor xmm5, xmm6 ; whitening (CTRi) + movdqa xmm6, oword [rcx+rdx] ; pre load operation's keys + +align IPP_ALIGN_FACTOR +.cipher_loop: + aesenc xmm5, xmm6 ; regular round (CTRi) + movdqa xmm6, oword [rcx+rdx+16] + add rdx, 16 + jnz .cipher_loop + aesenclast xmm5, xmm6 ; irregular round (CTRi) + + pxor xmm4, xmm5 ; dst[i] = src[i] ^ ENC(CTRi) + movdqu oword [rsi], xmm4 + + ;;;;;;;;;;;;;;;;; + ;; update MAC + ;;;;;;;;;;;;;;;;; + mov rdx, rbx + + movdqa xmm6, oword [rcx+rdx] ; keys for whitening + add rdx, 16 + + pxor xmm0, xmm4 ; MAC ^= dst[i] + pxor xmm0, xmm6 ; whitening (MAC) + + movdqa xmm6, oword [rcx+rdx] ; pre load operation's keys + +align IPP_ALIGN_FACTOR +.auth_loop: + aesenc xmm0, xmm6 ; regular round (MAC) + movdqa xmm6, oword [rcx+rdx+16] + add rdx, 16 + jnz .auth_loop + aesenclast xmm0, xmm6 ; irregular round (MAC) + + + mov rdx, rbx + add rsi, BYTES_PER_BLK + add rdi, BYTES_PER_BLK + sub r8, BYTES_PER_BLK + jnz .blk_loop + + movdqu oword [r9], xmm0 ; update MAC value + movdqu oword [r9+sizeof(oword)], xmm6 ; update ENC(Ctri) + + pxor xmm6, xmm6 + + REST_XMM + REST_GPR + ret +ENDFUNC DecryptAuth_RIJ128_AES_NI + +%endif +%endif ;; _AES_NI_ENABLING_ + + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128cmace9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128cmace9as.asm new file mode 100644 index 0000000..97a55f2 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128cmace9as.asm @@ -0,0 +1,114 @@ +;=============================================================================== +; Copyright (C) 2018 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Cipher function +; +; Content: +; cpAESCMAC_Update_AES_NI() +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "ia_32e_regs.inc" +%include "pcpvariant.inc" + +%if (_AES_NI_ENABLING_ == _FEATURE_ON_) || (_AES_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_Y8) + +segment .text align=IPP_ALIGN_FACTOR + + +;*************************************************************** +;* Purpose: AES-CMAC update +;* +;* void cpAESCMAC_Update_AES_NI(Ipp8u* digest, +;* const Ipp8u* input, +;* int inpLen, +;* int nr, +;* const Ipp32u* pRKey) +;*************************************************************** + +;; +;; Lib = Y8 +;; +;; Caller = ippsAES_CMACUpdate +;; +align IPP_ALIGN_FACTOR +IPPASM cpAESCMAC_Update_AES_NI,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi + USES_XMM + COMP_ABI 5 +;; rdi: pDigest: DWORD, ; input blocks address +;; rsi: pInput: DWORD, ; output blocks address +;; rdx: length: DWORD, ; lenght in bytes (multiple 16) +;; rcx: nr: DWORD ; number of rounds +;; r8: pRKey: DWORD ; pointer to keys + +%xdefine SC (4) +%assign BYTES_PER_BLK (16) + + movsxd rdx, edx ; input length + movdqu xmm0, oword [rdi] ; digest + +align IPP_ALIGN_FACTOR +;; +;; pseudo-pipelined processing +;; +.blks_loop: + movdqu xmm1, oword [rsi] ; input block + + movdqa xmm4, oword [r8] + mov r9, r8 ; save pointer to the key material + + pxor xmm0, xmm1 ; digest ^ src[] + + pxor xmm0, xmm4 ; whitening + + movdqa xmm4, oword [r9+16] + add r9, 16 + + mov r10, rcx ; counter depending on key length + sub r10, 1 +align IPP_ALIGN_FACTOR +.cipher_loop: + aesenc xmm0, xmm4 ; regular round + movdqa xmm4, oword [r9+16] + add r9, 16 + dec r10 + jnz .cipher_loop + aesenclast xmm0, xmm4 ; irregular round + + add rsi, BYTES_PER_BLK ; advance pointers + sub rdx, BYTES_PER_BLK ; decrease counter + jnz .blks_loop + + pxor xmm4, xmm4 + movdqu oword [rdi], xmm0 ; store updated digest digest + + REST_XMM + REST_GPR + ret +ENDFUNC cpAESCMAC_Update_AES_NI + +%endif + +%endif ;; _AES_NI_ENABLING_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128decryptcbcpipee9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128decryptcbcpipee9as.asm new file mode 100644 index 0000000..3327197 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128decryptcbcpipee9as.asm @@ -0,0 +1,342 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Inverse Cipher function +; +; Content: +; DecryptCBC_RIJ128pipe_AES_NI() +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_AES_NI_ENABLING_ == _FEATURE_ON_) || (_AES_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_Y8) + +segment .text align=IPP_ALIGN_FACTOR + + +;*************************************************************** +;* Purpose: pipelined RIJ128 CBC decryption +;* +;* void DecryptCBC_RIJ128pipe_AES_NI(const Ipp32u* inpBlk, +;* Ipp32u* outBlk, +;* int nr, +;* const Ipp32u* pRKey, +;* int len, +;* const Ipp8u* pIV) +;*************************************************************** + +;; +;; Lib = Y8 +;; +;; Caller = ippsAESDecryptCBC +;; + +%assign AES_BLOCK (16) + +align IPP_ALIGN_FACTOR +IPPASM DecryptCBC_RIJ128pipe_AES_NI,PUBLIC +%assign LOCAL_FRAME ((1+8)*AES_BLOCK) + USES_GPR rsi,rdi,rbx + USES_XMM xmm15 + COMP_ABI 6 +;; rdi: pInpBlk: DWORD, ; input blocks address +;; rsi: pOutBlk: DWORD, ; output blocks address +;; rdx: nr: DWORD, ; number of rounds +;; rcx pKey: DWORD ; key material address +;; r8d length: DWORD ; length (bytes) +;; r9 pIV BYTE ; pointer to the IV + +%xdefine SC (4) + + movdqu xmm15, oword [r9] ; IV + movsxd r8, r8d ; processed length + + lea rax,[rdx*SC] ; keys offset + + cmp r8, (4*AES_BLOCK) + jl .short123_input + cmp r8, (16*AES_BLOCK) + jle .block4x + +;; +;; 8-blocks processing +;; + mov rbx, rsp ; rbx points on cipher's data in stack + + sub rsp, sizeof(oword)*4 ; allocate stack and xmm registers + movdqa oword [rsp+0*sizeof(oword)], xmm6 + movdqa oword [rsp+1*sizeof(oword)], xmm7 + movdqa oword [rsp+2*sizeof(oword)], xmm8 + movdqa oword [rsp+3*sizeof(oword)], xmm9 + + sub r8, (8*AES_BLOCK) + +align IPP_ALIGN_FACTOR +.blk8_loop: + lea r9,[rcx+rax*sizeof(dword)-AES_BLOCK]; pointer to the key material + + movdqu xmm0, oword [rdi+0*AES_BLOCK] ; get input blocks + movdqu xmm1, oword [rdi+1*AES_BLOCK] + movdqu xmm2, oword [rdi+2*AES_BLOCK] + movdqu xmm3, oword [rdi+3*AES_BLOCK] + movdqu xmm6, oword [rdi+4*AES_BLOCK] + movdqu xmm7, oword [rdi+5*AES_BLOCK] + movdqu xmm8, oword [rdi+6*AES_BLOCK] + movdqu xmm9, oword [rdi+7*AES_BLOCK] + + movdqa xmm5, oword [r9+AES_BLOCK] ; whitening keys + movdqa xmm4, oword [r9] ; pre load operation's keys + + movdqa oword [rbx+1*AES_BLOCK], xmm0 ; save input into the stack + pxor xmm0, xmm5 ; and do whitening + movdqa oword [rbx+2*AES_BLOCK], xmm1 + pxor xmm1, xmm5 + movdqa oword [rbx+3*AES_BLOCK], xmm2 + pxor xmm2, xmm5 + movdqa oword [rbx+4*AES_BLOCK], xmm3 + pxor xmm3, xmm5 + movdqa oword [rbx+5*AES_BLOCK], xmm6 + pxor xmm6, xmm5 + movdqa oword [rbx+6*AES_BLOCK], xmm7 + pxor xmm7, xmm5 + movdqa oword [rbx+7*AES_BLOCK], xmm8 + pxor xmm8, xmm5 + movdqa oword [rbx+8*AES_BLOCK], xmm9 + pxor xmm9, xmm5 + + movdqa xmm5, oword [r9-AES_BLOCK] ; pre load operation's keys + sub r9, (2*AES_BLOCK) + + lea r10, [rdx-2] ; counter = nrounds-2 +align IPP_ALIGN_FACTOR +.cipher_loop8: + aesdec xmm0, xmm4 ; regular round + aesdec xmm1, xmm4 + aesdec xmm2, xmm4 + aesdec xmm3, xmm4 + aesdec xmm6, xmm4 + aesdec xmm7, xmm4 + aesdec xmm8, xmm4 + aesdec xmm9, xmm4 + movdqa xmm4, oword [r9] ; pre load operation's keys + + aesdec xmm0, xmm5 ; regular round + aesdec xmm1, xmm5 + aesdec xmm2, xmm5 + aesdec xmm3, xmm5 + aesdec xmm6, xmm5 + aesdec xmm7, xmm5 + aesdec xmm8, xmm5 + aesdec xmm9, xmm5 + movdqa xmm5, oword [r9-AES_BLOCK] ; pre load operation's keys + sub r9, (2*AES_BLOCK) + + sub r10, 2 + jnz .cipher_loop8 + + aesdec xmm0, xmm4 ; regular round + aesdec xmm1, xmm4 + aesdec xmm2, xmm4 + aesdec xmm3, xmm4 + aesdec xmm6, xmm4 + aesdec xmm7, xmm4 + aesdec xmm8, xmm4 + aesdec xmm9, xmm4 + + aesdeclast xmm0, xmm5 ; irregular round, ^IV, and store result + pxor xmm0, xmm15 + movdqu oword [rsi+0*AES_BLOCK], xmm0 + aesdeclast xmm1, xmm5 + pxor xmm1, oword [rbx+1*AES_BLOCK] + movdqu oword [rsi+1*AES_BLOCK], xmm1 + aesdeclast xmm2, xmm5 + pxor xmm2, oword [rbx+2*AES_BLOCK] + movdqu oword [rsi+2*AES_BLOCK], xmm2 + aesdeclast xmm3, xmm5 + pxor xmm3, oword [rbx+3*AES_BLOCK] + movdqu oword [rsi+3*AES_BLOCK], xmm3 + aesdeclast xmm6, xmm5 + pxor xmm6, oword [rbx+4*AES_BLOCK] + movdqu oword [rsi+4*AES_BLOCK], xmm6 + aesdeclast xmm7, xmm5 + pxor xmm7, oword [rbx+5*AES_BLOCK] + movdqu oword [rsi+5*AES_BLOCK], xmm7 + aesdeclast xmm8, xmm5 + pxor xmm8, oword [rbx+6*AES_BLOCK] + movdqu oword [rsi+6*AES_BLOCK], xmm8 + aesdeclast xmm9, xmm5 + pxor xmm9, oword [rbx+7*AES_BLOCK] + movdqu oword [rsi+7*AES_BLOCK], xmm9 + + movdqa xmm15,oword [rbx+8*AES_BLOCK] ; update IV + + add rsi, (8*AES_BLOCK) + add rdi, (8*AES_BLOCK) + sub r8, (8*AES_BLOCK) + jge .blk8_loop + + movdqa xmm6, oword [rsp+0*sizeof(oword)] ; restore xmm registers + movdqa xmm7, oword [rsp+1*sizeof(oword)] + movdqa xmm8, oword [rsp+2*sizeof(oword)] + movdqa xmm9, oword [rsp+3*sizeof(oword)] + add rsp, sizeof(oword)*4 ; and release stack + + add r8, (8*AES_BLOCK) + jz .quit + +;; +;; test %if 4-blocks processing alaivalbe +;; +.block4x: + cmp r8, (4*AES_BLOCK) + jl .short123_input + sub r8, (4*AES_BLOCK) + +align IPP_ALIGN_FACTOR +.blk4_loop: + lea r9,[rcx+rax*sizeof(dword)-AES_BLOCK]; pointer to the key material + + movdqu xmm0, oword [rdi+0*AES_BLOCK] ; get input blocks + movdqu xmm1, oword [rdi+1*AES_BLOCK] + movdqu xmm2, oword [rdi+2*AES_BLOCK] + movdqu xmm3, oword [rdi+3*AES_BLOCK] + + movdqa xmm5, oword [r9+AES_BLOCK] ; whitening keys + movdqa xmm4, oword [r9] ; pre load operation's keys + + movdqa oword [rsp+1*AES_BLOCK], xmm0 ; save input into the stack + pxor xmm0, xmm5 ; and do whitening + movdqa oword [rsp+2*AES_BLOCK], xmm1 + pxor xmm1, xmm5 + movdqa oword [rsp+3*AES_BLOCK], xmm2 + pxor xmm2, xmm5 + movdqa oword [rsp+4*AES_BLOCK], xmm3 + pxor xmm3, xmm5 + + movdqa xmm5, oword [r9-AES_BLOCK] ; pre load operation's keys + sub r9, (2*AES_BLOCK) + + lea r10, [rdx-2] ; counter = nrounds-2 +align IPP_ALIGN_FACTOR +.cipher_loop4: + aesdec xmm0, xmm4 ; regular round + aesdec xmm1, xmm4 + aesdec xmm2, xmm4 + aesdec xmm3, xmm4 + movdqa xmm4, oword [r9] ; pre load operation's keys + + aesdec xmm0, xmm5 ; regular round + aesdec xmm1, xmm5 + aesdec xmm2, xmm5 + aesdec xmm3, xmm5 + movdqa xmm5, oword [r9-AES_BLOCK] ; pre load operation's keys + sub r9, (2*AES_BLOCK) + + sub r10, 2 + jnz .cipher_loop4 + + aesdec xmm0, xmm4 ; regular round + aesdec xmm1, xmm4 + aesdec xmm2, xmm4 + aesdec xmm3, xmm4 + + aesdeclast xmm0, xmm5 ; irregular round, ^IV, and store result + pxor xmm0, xmm15 + movdqu oword [rsi+0*AES_BLOCK], xmm0 + aesdeclast xmm1, xmm5 + pxor xmm1, oword [rsp+1*AES_BLOCK] + movdqu oword [rsi+1*AES_BLOCK], xmm1 + aesdeclast xmm2, xmm5 + pxor xmm2, oword [rsp+2*AES_BLOCK] + movdqu oword [rsi+2*AES_BLOCK], xmm2 + aesdeclast xmm3, xmm5 + pxor xmm3, oword [rsp+3*AES_BLOCK] + movdqu oword [rsi+3*AES_BLOCK], xmm3 + + movdqa xmm15,oword [rsp+4*AES_BLOCK] ; update IV + + add rsi, (4*AES_BLOCK) + add rdi, (4*AES_BLOCK) + sub r8, (4*AES_BLOCK) + jge .blk4_loop + + add r8, (4*AES_BLOCK) + jz .quit + +;; +;; block-by-block processing +;; +.short123_input: + lea r9,[rcx+rax*sizeof(dword)] ; pointer to the key material (whitening) + +align IPP_ALIGN_FACTOR +.single_blk_loop: + movdqu xmm0, oword [rdi] ; get input block + add rdi, AES_BLOCK + movdqa xmm1, xmm0 ; and save as IV for future + + pxor xmm0, oword [r9] ; whitening + + cmp rdx,12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + +.key_256_s: + aesdec xmm0, oword [rcx+9*SC*4+4*SC*4] + aesdec xmm0, oword [rcx+9*SC*4+3*SC*4] +.key_192_s: + aesdec xmm0, oword [rcx+9*SC*4+2*SC*4] + aesdec xmm0, oword [rcx+9*SC*4+1*SC*4] +.key_128_s: + aesdec xmm0, oword [rcx+9*SC*4-0*SC*4] + aesdec xmm0, oword [rcx+9*SC*4-1*SC*4] + aesdec xmm0, oword [rcx+9*SC*4-2*SC*4] + aesdec xmm0, oword [rcx+9*SC*4-3*SC*4] + aesdec xmm0, oword [rcx+9*SC*4-4*SC*4] + aesdec xmm0, oword [rcx+9*SC*4-5*SC*4] + aesdec xmm0, oword [rcx+9*SC*4-6*SC*4] + aesdec xmm0, oword [rcx+9*SC*4-7*SC*4] + aesdec xmm0, oword [rcx+9*SC*4-8*SC*4] + aesdeclast xmm0, oword [rcx+9*SC*4-9*SC*4] + + pxor xmm0, xmm15 ; add IV + movdqu oword [rsi], xmm0 ; and save output blocl + add rsi, AES_BLOCK + + sub r8, AES_BLOCK + movdqa xmm15, xmm1 ; update IV + + jnz .single_blk_loop + +.quit: + pxor xmm4, xmm4 + pxor xmm5, xmm5 + + REST_XMM + REST_GPR + ret +ENDFUNC DecryptCBC_RIJ128pipe_AES_NI + +%endif ;; _IPP32E >= _IPP32E_Y8 +%endif ;; _AES_NI_ENABLING_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128decryptcfbpipee9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128decryptcfbpipee9as.asm new file mode 100644 index 0000000..4a7461f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128decryptcfbpipee9as.asm @@ -0,0 +1,562 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Inverse Cipher function +; +; Content: +; Decrypt_RIJ128_AES_NI() +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_AES_NI_ENABLING_ == _FEATURE_ON_) || (_AES_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_Y8) + + +%macro COPY_8U 4.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + %xdefine %%limit %3 + %xdefine %%tmp %4 + + xor rcx, rcx +%%next_byte: + mov %%tmp, byte [%%src+rcx] + mov byte [%%dst+rcx], %%tmp + add rcx, 1 + cmp rcx, %%limit + jl %%next_byte +%endmacro + +%macro COPY_32U 4.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + %xdefine %%limit %3 + %xdefine %%tmp %4 + + xor rcx, rcx +%%next_dword: + mov %%tmp, dword [%%src+rcx] + mov dword [%%dst+rcx], %%tmp + add rcx, 4 + cmp rcx, %%limit + jl %%next_dword +%endmacro + +%macro COPY_128U 4.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + %xdefine %%limit %3 + %xdefine %%tmp %4 + + xor rcx, rcx +%%next_oword: + movdqu %%tmp, oword [%%src+rcx] + movdqu oword [%%dst+rcx], %%tmp + add rcx, 16 + cmp rcx, %%limit + jl %%next_oword +%endmacro + +segment .text align=IPP_ALIGN_FACTOR + + +;*************************************************************** +;* Purpose: pipelined RIJ128 CFB decryption +;* +;* void DecryptCFB_RIJ128pipe_AES_NI(const Ipp32u* inpBlk, +;* Ipp32u* outBlk, +;* int nr, +;* const Ipp32u* pRKey, +;* int cfbBlks, +;* int cfbSize, +;* const Ipp8u* pIV) +;*************************************************************** + +;; +;; Lib = Y8 +;; +;; Caller = ippsRijndael128DecryptCFB +;; +align IPP_ALIGN_FACTOR +IPPASM DecryptCFB_RIJ128pipe_AES_NI,PUBLIC +%assign LOCAL_FRAME (1+4+4)*16 + USES_GPR rsi,rdi,r13,r14,r15 + USES_XMM xmm6,xmm7 + COMP_ABI 7 +;; rdi: pInpBlk: DWORD, ; input blocks address +;; rsi: pOutBlk: DWORD, ; output blocks address +;; rdx: nr: DWORD, ; number of rounds +;; rcx pKey: DWORD ; key material address +;; r8d cfbBlks: DWORD ; length of stream in cfbSize +;; r9d cfbSize: DWORD ; cfb blk size +;; [rsp+ARG_7] pIV BYTE ; pointer to the IV + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) + + mov rax, [rsp+ARG_7] ; IV address + movdqu xmm4, oword [rax] ; get IV + movdqa oword [rsp+0*16], xmm4 ; into the stack + + mov r13, rdi + mov r14, rsi + mov r15, rcx + + movsxd r8, r8d ; length of stream + movsxd r9, r9d ; cfb blk size + + sub r8, BLKS_PER_LOOP + jl .short_input + +;; +;; pipelined processing +;; + lea r10, [r9*BLKS_PER_LOOP] +.blks_loop: + COPY_32U {rsp+16}, r13, r10, r11d ; move 4 input blocks to stack + + movdqa xmm4, oword [r15] + + lea r10, [r9+r9*2] + movdqa xmm0, oword [rsp] ; get encoded blocks + movdqu xmm1, oword [rsp+r9] + movdqu xmm2, oword [rsp+r9*2] + movdqu xmm3, oword [rsp+r10] + + mov r10, r15 ; set pointer to the key material + + pxor xmm0, xmm4 ; whitening + pxor xmm1, xmm4 + pxor xmm2, xmm4 + pxor xmm3, xmm4 + + movdqa xmm4, oword [r10+16] ; pre load operation's keys + add r10, 16 + + mov r11, rdx ; counter depending on key length + sub r11, 1 +.cipher_loop: + aesenc xmm0, xmm4 ; regular round + aesenc xmm1, xmm4 + aesenc xmm2, xmm4 + aesenc xmm3, xmm4 + movdqa xmm4, oword [r10+16]; pre load operation's keys + add r10, 16 + dec r11 + jnz .cipher_loop + + aesenclast xmm0, xmm4 ; irregular round and IV + aesenclast xmm1, xmm4 + aesenclast xmm2, xmm4 + aesenclast xmm3, xmm4 + + lea r10, [r9+r9*2] ; get src blocks from the stack + movdqa xmm4, oword [rsp+16] + movdqu xmm5, oword [rsp+16+r9] + movdqu xmm6, oword [rsp+16+r9*2] + movdqu xmm7, oword [rsp+16+r10] + + pxor xmm0, xmm4 ; xor src + movdqa oword [rsp+5*16],xmm0;and store into the stack + pxor xmm1, xmm5 + movdqu oword [rsp+5*16+r9], xmm1 + pxor xmm2, xmm6 + movdqu oword [rsp+5*16+r9*2], xmm2 + pxor xmm3, xmm7 + movdqu oword [rsp+5*16+r10], xmm3 + + lea r10, [r9*BLKS_PER_LOOP] + ;COPY_8U r14, {rsp+5*16}, r10 ; move 4 blocks to output + COPY_32U r14, {rsp+5*16}, r10, r11d ; move 4 blocks to output + + movdqu xmm0, oword [rsp+r10]; update IV + movdqu oword [rsp], xmm0 + + add r13, r10 + add r14, r10 + sub r8, BLKS_PER_LOOP + jge .blks_loop + +;; +;; block-by-block processing +;; +.short_input: + add r8, BLKS_PER_LOOP + jz .quit + + lea r10, [r9*2] + lea r11, [r9+r9*2] + cmp r8, 2 + cmovl r10, r9 + cmovg r10, r11 + COPY_8U {rsp+16}, r13, r10, al ; move recent input blocks to stack + + ; get actual address of key material: pRKeys += (nr-9) * SC + lea rax,[rdx*4] + lea rax, [r15+rax*4-9*(SC)*4] ; AES-128 round keys + + xor r11, r11 ; index +.single_blk_loop: + movdqu xmm0, oword [rsp+r11] ; get encoded block + + pxor xmm0, oword [r15] ; whitening + + cmp rdx,12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + +.key_256_s: + aesenc xmm0, oword [rax-4*4*SC] + aesenc xmm0, oword [rax-3*4*SC] +.key_192_s: + aesenc xmm0, oword [rax-2*4*SC] + aesenc xmm0, oword [rax-1*4*SC] +.key_128_s: + aesenc xmm0, oword [rax+0*4*SC] + aesenc xmm0, oword [rax+1*4*SC] + aesenc xmm0, oword [rax+2*4*SC] + aesenc xmm0, oword [rax+3*4*SC] + aesenc xmm0, oword [rax+4*4*SC] + aesenc xmm0, oword [rax+5*4*SC] + aesenc xmm0, oword [rax+6*4*SC] + aesenc xmm0, oword [rax+7*4*SC] + aesenc xmm0, oword [rax+8*4*SC] + aesenclast xmm0, oword [rax+9*4*SC] + + movdqu xmm1, oword [rsp+r11+16] ; get input block from the stack + pxor xmm0, xmm1 ; xor src + movdqu oword [rsp+5*16+r11], xmm0 ; and save output + + add r11, r9 + dec r8 + jnz .single_blk_loop + + COPY_8U r14, {rsp+5*16}, r10, al ; copy rest output from the stack + +.quit: + REST_XMM + REST_GPR + ret +ENDFUNC DecryptCFB_RIJ128pipe_AES_NI + +align IPP_ALIGN_FACTOR +IPPASM DecryptCFB32_RIJ128pipe_AES_NI,PUBLIC +%assign LOCAL_FRAME (1+4+4)*16 + USES_GPR rsi,rdi,r13,r14,r15 + USES_XMM xmm6,xmm7 + COMP_ABI 7 +;; rdi: pInpBlk: DWORD, ; input blocks address +;; rsi: pOutBlk: DWORD, ; output blocks address +;; rdx: nr: DWORD, ; number of rounds +;; rcx pKey: DWORD ; key material address +;; r8d cfbBlks: DWORD ; length of stream in cfbSize +;; r9d cfbSize: DWORD ; cfb blk size (4 bytes multible) +;; [rsp+ARG_7] pIV BYTE ; pointer to the IV + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) + + mov rax, [rsp+ARG_7] ; IV address + movdqu xmm4, oword [rax] ; get IV + movdqa oword [rsp+0*16], xmm4 ; into the stack + + mov r13, rdi + mov r14, rsi + mov r15, rcx + + movsxd r8, r8d ; length of stream + movsxd r9, r9d ; cfb blk size + + sub r8, BLKS_PER_LOOP + jl .short_input + +;; +;; pipelined processing +;; + lea r10, [r9*BLKS_PER_LOOP] +.blks_loop: + COPY_128U {rsp+16}, r13, r10, xmm0 ; move 4 input blocks to stack + + movdqa xmm4, oword [r15] + + lea r10, [r9+r9*2] + movdqa xmm0, oword [rsp] ; get encoded blocks + movdqu xmm1, oword [rsp+r9] + movdqu xmm2, oword [rsp+r9*2] + movdqu xmm3, oword [rsp+r10] + + mov r10, r15 ; set pointer to the key material + + pxor xmm0, xmm4 ; whitening + pxor xmm1, xmm4 + pxor xmm2, xmm4 + pxor xmm3, xmm4 + + movdqa xmm4, oword [r10+16] ; pre load operation's keys + add r10, 16 + + mov r11, rdx ; counter depending on key length + sub r11, 1 +.cipher_loop: + aesenc xmm0, xmm4 ; regular round + aesenc xmm1, xmm4 + aesenc xmm2, xmm4 + aesenc xmm3, xmm4 + movdqa xmm4, oword [r10+16]; pre load operation's keys + add r10, 16 + dec r11 + jnz .cipher_loop + + aesenclast xmm0, xmm4 ; irregular round and IV + aesenclast xmm1, xmm4 + aesenclast xmm2, xmm4 + aesenclast xmm3, xmm4 + + lea r10, [r9+r9*2] ; get src blocks from the stack + movdqa xmm4, oword [rsp+16] + movdqu xmm5, oword [rsp+16+r9] + movdqu xmm6, oword [rsp+16+r9*2] + movdqu xmm7, oword [rsp+16+r10] + + pxor xmm0, xmm4 ; xor src + movdqa oword [rsp+5*16],xmm0;and store into the stack + pxor xmm1, xmm5 + movdqu oword [rsp+5*16+r9], xmm1 + pxor xmm2, xmm6 + movdqu oword [rsp+5*16+r9*2], xmm2 + pxor xmm3, xmm7 + movdqu oword [rsp+5*16+r10], xmm3 + + lea r10, [r9*BLKS_PER_LOOP] + COPY_128U r14, {rsp+5*16}, r10, xmm0 ; move 4 blocks to output + + movdqu xmm0, oword [rsp+r10] ; update IV + movdqu oword [rsp], xmm0 + + add r13, r10 + add r14, r10 + sub r8, BLKS_PER_LOOP + jge .blks_loop + +;; +;; block-by-block processing +;; +.short_input: + add r8, BLKS_PER_LOOP + jz .quit + + lea r10, [r9*2] + lea r11, [r9+r9*2] + cmp r8, 2 + cmovl r10, r9 + cmovg r10, r11 + COPY_32U {rsp+16}, r13, r10, eax ; move recent input blocks to stack + + ; get actual address of key material: pRKeys += (nr-9) * SC + lea rax,[rdx*4] + lea rax, [r15+rax*4-9*(SC)*4] ; AES-128 round keys + + xor r11, r11 ; index +.single_blk_loop: + movdqu xmm0, oword [rsp+r11] ; get encoded block + + pxor xmm0, oword [r15] ; whitening + + cmp rdx,12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + +.key_256_s: + aesenc xmm0, oword [rax-4*4*SC] + aesenc xmm0, oword [rax-3*4*SC] +.key_192_s: + aesenc xmm0, oword [rax-2*4*SC] + aesenc xmm0, oword [rax-1*4*SC] +.key_128_s: + aesenc xmm0, oword [rax+0*4*SC] + aesenc xmm0, oword [rax+1*4*SC] + aesenc xmm0, oword [rax+2*4*SC] + aesenc xmm0, oword [rax+3*4*SC] + aesenc xmm0, oword [rax+4*4*SC] + aesenc xmm0, oword [rax+5*4*SC] + aesenc xmm0, oword [rax+6*4*SC] + aesenc xmm0, oword [rax+7*4*SC] + aesenc xmm0, oword [rax+8*4*SC] + aesenclast xmm0, oword [rax+9*4*SC] + + movdqu xmm1, oword [rsp+r11+16] ; get input block from the stack + pxor xmm0, xmm1 ; xor src + movdqu oword [rsp+5*16+r11], xmm0 ; and save output + + add r11, r9 + dec r8 + jnz .single_blk_loop + + COPY_32U r14, {rsp+5*16}, r10, eax ; copy rest output from the stack + +.quit: + REST_XMM + REST_GPR + ret +ENDFUNC DecryptCFB32_RIJ128pipe_AES_NI + +;; +;; Lib = Y8 +;; +;; Caller = ippsRijndael128DecryptCFB +;; +align IPP_ALIGN_FACTOR +IPPASM DecryptCFB128_RIJ128pipe_AES_NI,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi + USES_XMM xmm6,xmm7 + COMP_ABI 6 +;; rdi: pInpBlk: DWORD, ; input blocks address +;; rsi: pOutBlk: DWORD, ; output blocks address +;; rdx: nr: DWORD, ; number of rounds +;; rcx pKey: DWORD ; key material address +;; r8d lenBytes: DWORD ; length of stream in bytes +;; r9 pIV BYTE ; pointer to the IV + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) +%assign BYTES_PER_BLK (16) +%assign BYTES_PER_LOOP (BYTES_PER_BLK*BLKS_PER_LOOP) + + movdqu xmm0, oword [r9] ; get IV + + movsxd r8, r8d ; length of the stream + sub r8, BYTES_PER_LOOP + jl .short_input + +;; +;; pipelined processing +;; +.blks_loop: + movdqa xmm7, oword [rcx] ; get initial key material + mov r10, rcx ; set pointer to the key material + + movdqu xmm1, oword [rdi+0*BYTES_PER_BLK] ; get another encoded cblocks + movdqu xmm2, oword [rdi+1*BYTES_PER_BLK] + movdqu xmm3, oword [rdi+2*BYTES_PER_BLK] + + pxor xmm0, xmm7 ; whitening + pxor xmm1, xmm7 + pxor xmm2, xmm7 + pxor xmm3, xmm7 + + movdqa xmm7, oword [r10+16] ; pre load operation's keys + add r10, 16 + + mov r11, rdx ; counter depending on key length + sub r11, 1 +.cipher_loop: + aesenc xmm0, xmm7 ; regular round + aesenc xmm1, xmm7 + aesenc xmm2, xmm7 + aesenc xmm3, xmm7 + movdqa xmm7, oword [r10+16] ; pre load operation's keys + add r10, 16 + dec r11 + jnz .cipher_loop + + aesenclast xmm0, xmm7 ; irregular round and IV + movdqu xmm4, oword [rdi+0*BYTES_PER_BLK] ; 4 input blocks + aesenclast xmm1, xmm7 + movdqu xmm5, oword [rdi+1*BYTES_PER_BLK] + aesenclast xmm2, xmm7 + movdqu xmm6, oword [rdi+2*BYTES_PER_BLK] + aesenclast xmm3, xmm7 + movdqu xmm7, oword [rdi+3*BYTES_PER_BLK] + add rdi, BYTES_PER_LOOP + + pxor xmm0, xmm4 ; 4 output blocks + movdqu oword [rsi+0*16], xmm0 + pxor xmm1, xmm5 + movdqu oword [rsi+1*16], xmm1 + pxor xmm2, xmm6 + movdqu oword [rsi+2*16], xmm2 + pxor xmm3, xmm7 + movdqu oword [rsi+3*16], xmm3 + add rsi, BYTES_PER_LOOP + + movdqa xmm0, xmm7 ; update IV + sub r8, BYTES_PER_LOOP + jge .blks_loop + +;; +;; block-by-block processing +;; +.short_input: + add r8, BYTES_PER_LOOP + jz .quit + + ; get actual address of key material: pRKeys += (nr-9) * SC + lea rax, [rdx*4] + lea rax, [rcx+rax*4-9*(SC)*4] ; AES-128 round keys + +.single_blk_loop: + pxor xmm0, oword [rcx] ; whitening + + cmp rdx,12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + +.key_256_s: + aesenc xmm0, oword [rax-4*4*SC] + aesenc xmm0, oword [rax-3*4*SC] +.key_192_s: + aesenc xmm0, oword [rax-2*4*SC] + aesenc xmm0, oword [rax-1*4*SC] +.key_128_s: + aesenc xmm0, oword [rax+0*4*SC] + aesenc xmm0, oword [rax+1*4*SC] + aesenc xmm0, oword [rax+2*4*SC] + aesenc xmm0, oword [rax+3*4*SC] + aesenc xmm0, oword [rax+4*4*SC] + aesenc xmm0, oword [rax+5*4*SC] + aesenc xmm0, oword [rax+6*4*SC] + aesenc xmm0, oword [rax+7*4*SC] + aesenc xmm0, oword [rax+8*4*SC] + aesenclast xmm0, oword [rax+9*4*SC] + + movdqu xmm1, oword [rdi] ; input block from the stream + add rdi, BYTES_PER_BLK + pxor xmm0, xmm1 ; xor src + movdqu oword [rsi], xmm0 ; and save output + add rsi, BYTES_PER_BLK + + movdqa xmm0, xmm1 ; update IV + sub r8, BYTES_PER_BLK + jnz .single_blk_loop + +.quit: + REST_XMM + REST_GPR + ret +ENDFUNC DecryptCFB128_RIJ128pipe_AES_NI + +%endif + +%endif ;; _AES_NI_ENABLING_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128decrypte9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128decrypte9as.asm new file mode 100644 index 0000000..8e3a94c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128decrypte9as.asm @@ -0,0 +1,115 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Inverse Cipher function +; +; Content: +; Decrypt_RIJ128_AES_NI() +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_AES_NI_ENABLING_ == _FEATURE_ON_) || (_AES_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_Y8) + + +segment .text align=IPP_ALIGN_FACTOR + + + +;*************************************************************** +;* Purpose: single block RIJ128 Inverse Cipher +;* +;* void Decrypt_RIJ128_AES_NI(const Ipp32u* inpBlk, +;* Ipp32u* outBlk, +;* int nr, +;* const Ipp32u* pRKey, +;* const Ipp32u Tables[][256]) +;*************************************************************** + +;; +;; Lib = Y8 +;; +;; Caller = ippsRijndael128DecryptECB +;; Caller = ippsRijndael128DecryptCBC +;; +align IPP_ALIGN_FACTOR +IPPASM Decrypt_RIJ128_AES_NI,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi + USES_XMM + COMP_ABI 4 +;; rdi: pInpBlk: DWORD, ; input block address +;; rsi: pOutBlk: DWORD, ; output block address +;; rdx: nr: DWORD, ; number of rounds +;; rcx pKey: DWORD ; key material address + +%xdefine SC (4) + + + lea rax,[rdx*SC] + lea rax,[rax*4] + + movdqu xmm0, oword [rdi] ; input block + + ;;whitening + pxor xmm0, oword [rcx+rax] + + cmp rdx,12 ; switch according to number of rounds + jl .key_128 + jz .key_192 + + ;; + ;; regular rounds + ;; +.key_256: + aesdec xmm0,oword [rcx+9*SC*4+4*SC*4] + aesdec xmm0,oword [rcx+9*SC*4+3*SC*4] +.key_192: + aesdec xmm0,oword [rcx+9*SC*4+2*SC*4] + aesdec xmm0,oword [rcx+9*SC*4+1*SC*4] +.key_128: + aesdec xmm0,oword [rcx+9*SC*4-0*SC*4] + aesdec xmm0,oword [rcx+9*SC*4-1*SC*4] + aesdec xmm0,oword [rcx+9*SC*4-2*SC*4] + aesdec xmm0,oword [rcx+9*SC*4-3*SC*4] + aesdec xmm0,oword [rcx+9*SC*4-4*SC*4] + aesdec xmm0,oword [rcx+9*SC*4-5*SC*4] + aesdec xmm0,oword [rcx+9*SC*4-6*SC*4] + aesdec xmm0,oword [rcx+9*SC*4-7*SC*4] + aesdec xmm0,oword [rcx+9*SC*4-8*SC*4] + ;; + ;; last rounds + ;; + aesdeclast xmm0,oword [rcx+9*SC*4-9*SC*4] + + movdqu oword [rsi], xmm0 ; output block + + REST_XMM + REST_GPR + ret +ENDFUNC Decrypt_RIJ128_AES_NI + +%endif + +%endif ;; _AES_NI_ENABLING_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128decryptecbpipee9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128decryptecbpipee9as.asm new file mode 100644 index 0000000..9ac77b5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128decryptecbpipee9as.asm @@ -0,0 +1,180 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Inverse Cipher function +; +; Content: +; DecryptECB_RIJ128pipe_AES_NI() +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_AES_NI_ENABLING_ == _FEATURE_ON_) || (_AES_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_Y8) + + +segment .text align=IPP_ALIGN_FACTOR + +;*************************************************************** +;* Purpose: pipelined RIJ128 ECB decryption +;* +;* void DecryptECB_RIJ128pipe_AES_NI(const Ipp32u* inpBlk, +;* Ipp32u* outBlk, +;* int nr, +;* const Ipp32u* pRKey, +;* int len) +;*************************************************************** + +;; +;; Lib = Y8 +;; +;; Caller = ippsAESDecryptECB +;; +align IPP_ALIGN_FACTOR +IPPASM DecryptECB_RIJ128pipe_AES_NI,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi + USES_XMM + COMP_ABI 5 +;; rdi: pInpBlk: DWORD, ; input blocks address +;; rsi: pOutBlk: DWORD, ; output blocks address +;; rdx: nr: DWORD, ; number of rounds +;; rcx pKey: DWORD ; key material address +;; r8d length: DWORD ; length (bytes) + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) +%assign BYTES_PER_BLK (16) +%assign BYTES_PER_LOOP (BYTES_PER_BLK*BLKS_PER_LOOP) + + lea rax,[rdx*SC] ; keys offset + + movsxd r8, r8d + sub r8, BYTES_PER_LOOP + jl .short_input + +;; +;; pipelined processing +;; +;ALIGN IPP_ALIGN_FACTOR +.blks_loop: + lea r9,[rcx+rax*4] ; set pointer to the key material + movdqa xmm4, oword [r9] ; keys for whitening + sub r9, 16 + + movdqu xmm0, oword [rdi+0*BYTES_PER_BLK] ; get input blocks + movdqu xmm1, oword [rdi+1*BYTES_PER_BLK] + movdqu xmm2, oword [rdi+2*BYTES_PER_BLK] + movdqu xmm3, oword [rdi+3*BYTES_PER_BLK] + add rdi, BYTES_PER_LOOP + + pxor xmm0, xmm4 ; whitening + pxor xmm1, xmm4 + pxor xmm2, xmm4 + pxor xmm3, xmm4 + + movdqa xmm4, oword [r9] ; pre load operation's keys + sub r9, 16 + + mov r10, rdx ; counter depending on key length + sub r10, 1 +;ALIGN IPP_ALIGN_FACTOR +.cipher_loop: + aesdec xmm0, xmm4 ; regular round + aesdec xmm1, xmm4 + aesdec xmm2, xmm4 + aesdec xmm3, xmm4 + movdqa xmm4, oword [r9] ; pre load operation's keys + sub r9, 16 + dec r10 + jnz .cipher_loop + + aesdeclast xmm0, xmm4 ; irregular round + movdqu oword [rsi+0*BYTES_PER_BLK], xmm0 ; store output blocks + + aesdeclast xmm1, xmm4 + movdqu oword [rsi+1*BYTES_PER_BLK], xmm1 + + aesdeclast xmm2, xmm4 + movdqu oword [rsi+2*BYTES_PER_BLK], xmm2 + + aesdeclast xmm3, xmm4 + movdqu oword [rsi+3*BYTES_PER_BLK], xmm3 + + add rsi, BYTES_PER_LOOP + sub r8, BYTES_PER_LOOP + jge .blks_loop + +;; +;; block-by-block processing +;; +.short_input: + add r8, BYTES_PER_LOOP + jz .quit + + lea r9,[rcx+rax*4] ; set pointer to the key material +align IPP_ALIGN_FACTOR +.single_blk_loop: + movdqu xmm0, oword [rdi] ; get input block + add rdi, BYTES_PER_BLK + pxor xmm0, oword [r9] ; whitening + + cmp rdx,12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + +.key_256_s: + aesdec xmm0, oword [rcx+9*SC*4+4*SC*4] + aesdec xmm0, oword [rcx+9*SC*4+3*SC*4] +.key_192_s: + aesdec xmm0, oword [rcx+9*SC*4+2*SC*4] + aesdec xmm0, oword [rcx+9*SC*4+1*SC*4] +.key_128_s: + aesdec xmm0, oword [rcx+9*SC*4-0*SC*4] + aesdec xmm0, oword [rcx+9*SC*4-1*SC*4] + aesdec xmm0, oword [rcx+9*SC*4-2*SC*4] + aesdec xmm0, oword [rcx+9*SC*4-3*SC*4] + aesdec xmm0, oword [rcx+9*SC*4-4*SC*4] + aesdec xmm0, oword [rcx+9*SC*4-5*SC*4] + aesdec xmm0, oword [rcx+9*SC*4-6*SC*4] + aesdec xmm0, oword [rcx+9*SC*4-7*SC*4] + aesdec xmm0, oword [rcx+9*SC*4-8*SC*4] + aesdeclast xmm0, oword [rcx+9*SC*4-9*SC*4] + + movdqu oword [rsi], xmm0 ; save output block + add rsi, BYTES_PER_BLK + + sub r8, BYTES_PER_BLK + jnz .single_blk_loop + +.quit: + pxor xmm4, xmm4 + + REST_XMM + REST_GPR + ret +ENDFUNC DecryptECB_RIJ128pipe_AES_NI + +%endif + +%endif ;; _AES_NI_ENABLING_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128decryptxtse9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128decryptxtse9as.asm new file mode 100644 index 0000000..372eef8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128decryptxtse9as.asm @@ -0,0 +1,377 @@ +;=============================================================================== +; Copyright (C) 2016 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; AES functions +; +; Content: +; cpAESEncryptXTS_AES_NI() +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_AES_NI_ENABLING_ == _FEATURE_ON_) || (_AES_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_Y8) + + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR + +ALPHA_MUL_CNT dq 00000000000000087h, 00000000000000001h + + +;*************************************************************** +;* Purpose: AES-XTS encryption +;* +;* void cpAESDecryptXTS_AES_NI(Ipp8u* outBlk, +;* const Ipp8u* inpBlk, +;* int length, +;* const Ipp8u* pRKey, +;* int nr, +;* Ipp8u* pTweak) +;*************************************************************** + +;; +;; key = ks[0] +;; mul_cnt = {0x0000000000000001:0x0000000000000087} +;; returns: +;; ktwk = twk^key +;; twk = twk*alpha +;; twk2= twk2 *2 - auxillary +;; +%macro OUTER_MUL_X 6.nolist + %xdefine %%ktwk %1 + %xdefine %%twk %2 + %xdefine %%key %3 + %xdefine %%mul_cnt %4 + %xdefine %%twk2 %5 + %xdefine %%mask %6 + + movdqa %%mask, %%twk2 + paddd %%twk2, %%twk2 + movdqa %%ktwk, %%twk + psrad %%mask, 31 + paddq %%twk, %%twk + pand %%mask, %%mul_cnt + pxor %%ktwk, %%key + pxor %%twk, %%mask +%endmacro + +%macro LAST_OUTER_MUL_X 5.nolist + %xdefine %%ktwk %1 + %xdefine %%twk %2 + %xdefine %%key %3 + %xdefine %%mul_cnt %4 + %xdefine %%twk2 %5 + + movdqa %%ktwk, %%twk + psrad %%twk2, 31 + paddq %%twk, %%twk + pand %%twk2, %%mul_cnt + pxor %%ktwk, %%key + pxor %%twk, %%twk2 +%endmacro + +%macro INNER_MUL_X 6.nolist + %xdefine %%ktwk %1 + %xdefine %%twk %2 + %xdefine %%key %3 + %xdefine %%mul_cnt %4 + %xdefine %%twk2 %5 + %xdefine %%mask %6 + + movdqa %%mask, %%twk2 + paddd %%twk2, %%twk2 + psrad %%mask, 31 + paddq %%twk, %%twk + pand %%mask, %%mul_cnt + pxor %%twk, %%mask +%ifnidn %%key,%%ktwk + movdqa %%key, %%ktwk +%endif + pxor %%ktwk, %%twk +%endmacro + +%macro LAST_INNER_MUL_X 3.nolist + %xdefine %%twk %1 + %xdefine %%mul_cnt %2 + %xdefine %%twk2 %3 + + psrad %%twk2, 31 + paddq %%twk, %%twk + pand %%twk2, %%mul_cnt + pxor %%twk, %%twk2 +%endmacro + +align IPP_ALIGN_FACTOR +IPPASM cpAESDecryptXTS_AES_NI,PUBLIC +%assign LOCAL_FRAME sizeof(oword)*6 + USES_GPR rsi,rdi + USES_XMM xmm6,xmm7,xmm8,xmm9,xmm10,xmm11,xmm12,xmm13,xmm14,xmm15 + COMP_ABI 6 +;; rdi: pOutBlk: BYTE ; pointer to the output bloak +;; rsi: pInpBlk: BYTE ; pointer to the input block +;; edx: nBlocks DWORD ; number of blocks +;; rcx: pKey: BYTE ; key material address +;; r8d: nr: DWORD ; number of rounds +;; r9 pTweak: BYTE ; pointer to the input/outpout (ciphertext) tweak + +%assign AES_BLOCK (16) + + movsxd r8, r8d ; number of cipher rounds + + movdqu xmm15, xmmword [r9] ; input tweak value + + shl r8, 4 ; key schedule length (bytes) + + movdqa xmm0, xmmword [rcx+r8] ; key[0] + movdqa xmm8, xmmword [rel ALPHA_MUL_CNT] ; mul constant + + pshufd xmm9, xmm15, 5Fh ; {twk[1]:twk[1]:twk[3]:twk[3]} - auxillary value + + movsxd rdx, edx ; number of blocks being processing + + ;; compute: + ;; - ktwk[i] = twk[i]^key[0] + ;; - twk[i+1] = twk[i]*alpha^(i+1), i=0..5 + OUTER_MUL_X xmm10, xmm15, xmm0, xmm8, xmm9, xmm14 + OUTER_MUL_X xmm11, xmm15, xmm0, xmm8, xmm9, xmm14 + OUTER_MUL_X xmm12, xmm15, xmm0, xmm8, xmm9, xmm14 + OUTER_MUL_X xmm13, xmm15, xmm0, xmm8, xmm9, xmm14 + LAST_OUTER_MUL_X xmm14, xmm15, xmm0, xmm8, xmm9 + + movdqa xmm8, xmm15 ; save tweak for next iteration + pxor xmm15, xmm0 ; add key[0] + + sub rdx, 6 ; test input length + jc .short_input + +;; +;; blocks processing +;; +align IPP_ALIGN_FACTOR +.blks_loop: + pxor xmm0, xmmword [rcx] ; key[0]^key[last] + + lea rax, [r8-6*AES_BLOCK] ; cipher_loop counter + lea r10, [rcx+6*AES_BLOCK] ; mov key pointer down + + movdqu xmm2, xmmword [rsi] ; src[0] - src[7] + movdqu xmm3, xmmword [rsi+AES_BLOCK] + movdqu xmm4, xmmword [rsi+2*AES_BLOCK] + movdqu xmm5, xmmword [rsi+3*AES_BLOCK] + movdqu xmm6, xmmword [rsi+4*AES_BLOCK] + movdqu xmm7, xmmword [rsi+5*AES_BLOCK] + + movdqa xmm1, xmmword [rcx+r8-1*AES_BLOCK] ; key[1] + + pxor xmm2, xmm10 ; src[] ^twk[] ^ key[0] + pxor xmm3, xmm11 + pxor xmm4, xmm12 + pxor xmm5, xmm13 + pxor xmm6, xmm14 + pxor xmm7, xmm15 + + pxor xmm10, xmm0 ; store twk[] ^ key[last] + pxor xmm11, xmm0 + pxor xmm12, xmm0 + pxor xmm13, xmm0 + pxor xmm14, xmm0 + pxor xmm15, xmm0 + movdqa xmm0, xmmword [rcx+r8-2*AES_BLOCK] ; key[2] + + movdqa xmmword [rsp+0*AES_BLOCK], xmm10 + movdqa xmmword [rsp+1*AES_BLOCK], xmm11 + movdqa xmmword [rsp+2*AES_BLOCK], xmm12 + movdqa xmmword [rsp+3*AES_BLOCK], xmm13 + movdqa xmmword [rsp+4*AES_BLOCK], xmm14 + movdqa xmmword [rsp+5*AES_BLOCK], xmm15 + add rsi, 6*AES_BLOCK + +align IPP_ALIGN_FACTOR +.cipher_loop: + sub rax, 2*AES_BLOCK ; dec loop counter + aesdec xmm2, xmm1 ; regular rounds 3 - (last-2) + aesdec xmm3, xmm1 + aesdec xmm4, xmm1 + aesdec xmm5, xmm1 + aesdec xmm6, xmm1 + aesdec xmm7, xmm1 + movdqa xmm1, xmmword [r10+rax-1*AES_BLOCK] + aesdec xmm2, xmm0 + aesdec xmm3, xmm0 + aesdec xmm4, xmm0 + aesdec xmm5, xmm0 + aesdec xmm6, xmm0 + aesdec xmm7, xmm0 + movdqa xmm0, xmmword [r10+rax-2*AES_BLOCK] + jnz .cipher_loop + + movdqa xmm10, xmmword [rcx+r8] ; key[0] + + movdqa xmm15, xmm8 ; restore tweak value + pshufd xmm9, xmm8, 5Fh ; {twk[1]:twk[1]:twk[3]:twk[3]} - auxillary value + movdqa xmm8, xmmword [rel ALPHA_MUL_CNT] ; mul constant + + ; + ; last 6 rounds (5 regular rounds + irregular) + ; merged together with next tweaks computation + ; + aesdec xmm2, xmm1 + aesdec xmm3, xmm1 + aesdec xmm4, xmm1 + aesdec xmm5, xmm1 + aesdec xmm6, xmm1 + aesdec xmm7, xmm1 + movdqa xmm1, xmmword [rcx+3*AES_BLOCK] + + INNER_MUL_X xmm10, xmm15, xmm11, xmm8, xmm9, xmm14 + + aesdec xmm2, xmm0 + aesdec xmm3, xmm0 + aesdec xmm4, xmm0 + aesdec xmm5, xmm0 + aesdec xmm6, xmm0 + aesdec xmm7, xmm0 + movdqa xmm0, xmmword [rcx+2*AES_BLOCK] + + INNER_MUL_X xmm11, xmm15, xmm12, xmm8, xmm9, xmm14 + + aesdec xmm2, xmm1 + aesdec xmm3, xmm1 + aesdec xmm4, xmm1 + aesdec xmm5, xmm1 + aesdec xmm6, xmm1 + aesdec xmm7, xmm1 + movdqa xmm1, xmmword [rcx+1*AES_BLOCK] + + INNER_MUL_X xmm12, xmm15, xmm13, xmm8, xmm9, xmm14 + + aesdec xmm2, xmm0 + aesdec xmm3, xmm0 + aesdec xmm4, xmm0 + aesdec xmm5, xmm0 + aesdec xmm6, xmm0 + aesdec xmm7, xmm0 + + INNER_MUL_X xmm13, xmm15, xmm14, xmm8, xmm9, xmm14 + + aesdec xmm2, xmm1 + aesdec xmm3, xmm1 + aesdec xmm4, xmm1 + aesdec xmm5, xmm1 + aesdec xmm6, xmm1 + aesdec xmm7, xmm1 + + INNER_MUL_X xmm14, xmm15, xmm0, xmm8, xmm9, xmm0 + + aesdeclast xmm2, xmmword [rsp] ; final irregular round + aesdeclast xmm3, xmmword [rsp+1*AES_BLOCK] + aesdeclast xmm4, xmmword [rsp+2*AES_BLOCK] + aesdeclast xmm5, xmmword [rsp+3*AES_BLOCK] + aesdeclast xmm6, xmmword [rsp+4*AES_BLOCK] + aesdeclast xmm7, xmmword [rsp+5*AES_BLOCK] + + LAST_INNER_MUL_X xmm15, xmm8, xmm9 + movdqa xmm8, xmm15 ; save tweak for next iteration + pxor xmm15, xmm0 ; add key[0] + + movdqu xmmword [rdi+0*AES_BLOCK], xmm2 ; store output blocks + movdqu xmmword [rdi+1*AES_BLOCK], xmm3 + movdqu xmmword [rdi+2*AES_BLOCK], xmm4 + movdqu xmmword [rdi+3*AES_BLOCK], xmm5 + movdqu xmmword [rdi+4*AES_BLOCK], xmm6 + movdqu xmmword [rdi+5*AES_BLOCK], xmm7 + add rdi, 6*AES_BLOCK + + sub rdx, 6 + jnc .blks_loop + +.short_input: + add rdx, 6 + jz .quit + + movdqa xmmword [rsp+0*AES_BLOCK], xmm10 ; save pre-computed twae + movdqa xmmword [rsp+1*AES_BLOCK], xmm11 + movdqa xmmword [rsp+2*AES_BLOCK], xmm12 + movdqa xmmword [rsp+3*AES_BLOCK], xmm13 + movdqa xmmword [rsp+4*AES_BLOCK], xmm14 + movdqa xmmword [rsp+5*AES_BLOCK], xmm15 + +;; +;; block-by-block processing +;; + movdqa xmm0, xmmword [rcx+r8] ; key[0] + pxor xmm0, xmmword [rcx] ; key[0] ^ key[last] + + xor rax, rax + +.single_blk_loop: + movdqu xmm2, xmmword [rsi] ; input block + movdqa xmm1, xmmword [rsp+rax] ; tweak ^ key[0] + add rsi, AES_BLOCK + + pxor xmm2, xmm1 ; src[] ^tweak ^ key[0] + pxor xmm1, xmm0 ; tweak ^ key[lasl] + cmp r8, 12*16 ; switch according to number of rounds + jl .key_128_s + +.key_256_s: + aesdec xmm2, xmmword [rcx+9*AES_BLOCK+4*AES_BLOCK] + aesdec xmm2, xmmword [rcx+9*AES_BLOCK+3*AES_BLOCK] + aesdec xmm2, xmmword [rcx+9*AES_BLOCK+2*AES_BLOCK] + aesdec xmm2, xmmword [rcx+9*AES_BLOCK+1*AES_BLOCK] +.key_128_s: + aesdec xmm2, xmmword [rcx+9*AES_BLOCK-0*AES_BLOCK] + aesdec xmm2, xmmword [rcx+9*AES_BLOCK-1*AES_BLOCK] + aesdec xmm2, xmmword [rcx+9*AES_BLOCK-2*AES_BLOCK] + aesdec xmm2, xmmword [rcx+9*AES_BLOCK-3*AES_BLOCK] + aesdec xmm2, xmmword [rcx+9*AES_BLOCK-4*AES_BLOCK] + aesdec xmm2, xmmword [rcx+9*AES_BLOCK-5*AES_BLOCK] + aesdec xmm2, xmmword [rcx+9*AES_BLOCK-6*AES_BLOCK] + aesdec xmm2, xmmword [rcx+9*AES_BLOCK-7*AES_BLOCK] + aesdec xmm2, xmmword [rcx+9*AES_BLOCK-8*AES_BLOCK] + aesdeclast xmm2, xmm1 + + movdqu xmmword [rdi], xmm2 ; output block + add rdi, AES_BLOCK + add rax, AES_BLOCK + sub rdx, 1 + jnz .single_blk_loop + + movdqa xmm10, xmmword [rsp+rax] ; tweak ^ key[0] + +.quit: + pxor xmm10, xmmword [rcx+r8] ; remove key[0] + movdqu xmmword [r9], xmm10 ; and save tweak value + + pxor xmm0, xmm0 + pxor xmm1, xmm1 + + REST_XMM + REST_GPR + ret +ENDFUNC cpAESDecryptXTS_AES_NI + +%endif ;; _AES_NI_ENABLING_ +%endif ;;_IPP32E_Y8 + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encryptcbce9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encryptcbce9as.asm new file mode 100644 index 0000000..a41f3c2 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encryptcbce9as.asm @@ -0,0 +1,118 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Cipher function +; +; Content: +; EncryptCBC_RIJ128_AES_NI() +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "ia_32e_regs.inc" +%include "pcpvariant.inc" + +%if (_AES_NI_ENABLING_ == _FEATURE_ON_) || (_AES_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_Y8) + +segment .text align=IPP_ALIGN_FACTOR + + +;*************************************************************** +;* Purpose: RIJ128 CBC encryption +;* +;* void EncryptCBC_RIJ128_AES_NI(const Ipp32u* inpBlk, +;* Ipp32u* outBlk, +;* int nr, +;* const Ipp32u* pRKey, +;* int len, +;* const Ipp8u* pIV) +;*************************************************************** + +;; +;; Lib = Y8 +;; +;; Caller = ippsAESEncryptCBC +;; +align IPP_ALIGN_FACTOR +IPPASM EncryptCBC_RIJ128_AES_NI,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi + USES_XMM + COMP_ABI 6 +;; rdi: pInpBlk: DWORD, ; input blocks address +;; rsi: pOutBlk: DWORD, ; output blocks address +;; rdx: nr: DWORD, ; number of rounds +;; rcx pKey: DWORD ; key material address +;; r8d length: DWORD ; length (bytes) +;; r9 pIV: DWORD ; IV address + +%xdefine SC (4) +%assign BYTES_PER_BLK (16) + + movsxd r8, r8d ; input length + movdqu xmm0, oword [r9] ; IV + +align IPP_ALIGN_FACTOR +;; +;; pseudo-pipelined processing +;; +.blks_loop: + movdqu xmm1, oword [rdi] ; input block + + movdqa xmm4, oword [rcx] + mov r9, rcx ; set pointer to the key material + + pxor xmm0, xmm1 ; src[] ^ iv + + pxor xmm0, xmm4 ; whitening + + movdqa xmm4, oword [r9+16] + add r9, 16 + + mov r10, rdx ; counter depending on key length + sub r10, 1 +align IPP_ALIGN_FACTOR +.cipher_loop: + aesenc xmm0, xmm4 ; regular round + movdqa xmm4, oword [r9+16] + add r9, 16 + dec r10 + jnz .cipher_loop + aesenclast xmm0, xmm4 ; irregular round + + movdqu oword [rsi], xmm0 ; store output block + + add rdi, BYTES_PER_BLK ; advance pointers + add rsi, BYTES_PER_BLK + sub r8, BYTES_PER_BLK ; decrease counter + jnz .blks_loop + + pxor xmm4, xmm4 + + REST_XMM + REST_GPR + ret +ENDFUNC EncryptCBC_RIJ128_AES_NI + +%endif + +%endif ;; _AES_NI_ENABLING_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encryptcfbe9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encryptcfbe9as.asm new file mode 100644 index 0000000..429778e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encryptcfbe9as.asm @@ -0,0 +1,354 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; Purpose: Cryptography Primitive. +; Rijndael Inverse Cipher function +; +; Content: +; Encrypt_RIJ128_AES_NI() +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_AES_NI_ENABLING_ == _FEATURE_ON_) || (_AES_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_Y8) + + +%macro COPY_8U 4.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + %xdefine %%limit %3 + %xdefine %%tmp %4 + + xor rcx, rcx +%%next_byte: + mov %%tmp, byte [%%src+rcx] + mov byte [%%dst+rcx], %%tmp + add rcx, 1 + cmp rcx, %%limit + jl %%next_byte +%endmacro + +%macro COPY_32U 4.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + %xdefine %%limit %3 + %xdefine %%tmp %4 + + xor rcx, rcx +%%next_dword: + mov %%tmp, dword [%%src+rcx] + mov dword [%%dst+rcx], %%tmp + add rcx, 4 + cmp rcx, %%limit + jl %%next_dword +%endmacro + +%macro COPY_128U 4.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + %xdefine %%limit %3 + %xdefine %%tmp %4 + + xor rcx, rcx +%%next_oword: + movdqu %%tmp, oword [%%src+rcx] + movdqu oword [%%dst+rcx], %%tmp + add rcx, 16 + cmp rcx, %%limit + jl %%next_oword +%endmacro + +segment .text align=IPP_ALIGN_FACTOR + + +;*************************************************************** +;* Purpose: RIJ128 CFB encryption +;* +;* void EncryptCFB_RIJ128_AES_NI(const Ipp32u* inpBlk, +;* Ipp32u* outBlk, +;* int nr, +;* const Ipp32u* pRKey, +;* int cfbBlks, +;* int cfbSize, +;* const Ipp8u* pIV) +;*************************************************************** + +;; +;; Lib = Y8 +;; +;; Caller = ippsAESEncryptCFB +;; +align IPP_ALIGN_FACTOR +IPPASM EncryptCFB_RIJ128_AES_NI,PUBLIC +%assign LOCAL_FRAME (1+4+4)*16 + USES_GPR rsi,rdi,r12,r15 + USES_XMM + COMP_ABI 7 +;; rdi: pInpBlk: DWORD, ; input blocks address +;; rsi: pOutBlk: DWORD, ; output blocks address +;; rdx: nr: DWORD, ; number of rounds +;; rcx pKey: DWORD ; key material address +;; r8d cfbBlks: DWORD ; length of stream in bytes +;; r9d cfbSize: DWORD ; cfb blk size +;; [rsp+ARG_7] pIV BYTE ; pointer to the IV + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) + + mov rax, [rsp+ARG_7] ; IV address + movdqu xmm4, oword [rax] ; get IV + movdqa oword [rsp+0*16], xmm4 ; into the stack + + movsxd r8, r8d ; length of stream + movsxd r9, r9d ; cfb blk size + + mov r15, rcx ; save key material address + + ; get actual address of key material: pRKeys += (nr-9) * SC + lea rax,[rdx*4] + lea rax, [r15+rax*4-9*(SC)*4] ; AES-128 round keys + +;; +;; processing +;; + lea r10, [r9*BLKS_PER_LOOP] ; 4 cfb block +.blks_loop: + cmp r8, r10 + cmovl r10, r8 + COPY_8U {rsp+5*16}, rdi, r10, r11b ; move 1-4 input blocks to stack + + mov r12, r10 ; copy length to be processed + xor r11, r11 ; index +.single_blk: + movdqu xmm0, oword [rsp+r11] ; get processing blocks + + pxor xmm0, oword [r15] ; whitening + + cmp rdx,12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + ; do encryption +.key_256_s: + aesenc xmm0, oword [rax-4*4*SC] + aesenc xmm0, oword [rax-3*4*SC] +.key_192_s: + aesenc xmm0, oword [rax-2*4*SC] + aesenc xmm0, oword [rax-1*4*SC] +.key_128_s: + aesenc xmm0, oword [rax+0*4*SC] + aesenc xmm0, oword [rax+1*4*SC] + aesenc xmm0, oword [rax+2*4*SC] + aesenc xmm0, oword [rax+3*4*SC] + aesenc xmm0, oword [rax+4*4*SC] + aesenc xmm0, oword [rax+5*4*SC] + aesenc xmm0, oword [rax+6*4*SC] + aesenc xmm0, oword [rax+7*4*SC] + aesenc xmm0, oword [rax+8*4*SC] + aesenclast xmm0, oword [rax+9*4*SC] + + movdqu xmm1, oword [rsp+5*16+r11] ; get src blocks from the stack + pxor xmm0, xmm1 ; xor src + movdqu oword [rsp+1*16+r11],xmm0 ;and store into the stack + + add r11, r9 ; advance index + sub r12, r9 ; decrease length + jg .single_blk + + COPY_8U rsi, {rsp+1*16}, r10, r11b ; move 1-4 blocks to output + + movdqu xmm0, oword [rsp+r10]; update IV + movdqa oword [rsp], xmm0 + + add rdi, r10 + add rsi, r10 + sub r8, r10 + jg .blks_loop + + REST_XMM + REST_GPR + ret +ENDFUNC EncryptCFB_RIJ128_AES_NI + +align IPP_ALIGN_FACTOR +IPPASM EncryptCFB32_RIJ128_AES_NI,PUBLIC +%assign LOCAL_FRAME (1+4+4)*16 + USES_GPR rsi,rdi,r12,r15 + USES_XMM + COMP_ABI 7 +;; rdi: pInpBlk: DWORD, ; input blocks address +;; rsi: pOutBlk: DWORD, ; output blocks address +;; rdx: nr: DWORD, ; number of rounds +;; rcx pKey: DWORD ; key material address +;; r8d cfbBlks: DWORD ; length of stream in bytes +;; r9d cfbSize: DWORD ; cfb blk size +;; [rsp+ARG_7] pIV BYTE ; pointer to the IV + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) + + mov rax, [rsp+ARG_7] ; IV address + movdqu xmm4, oword [rax] ; get IV + movdqa oword [rsp+0*16], xmm4 ; into the stack + + movsxd r8, r8d ; length of stream + movsxd r9, r9d ; cfb blk size + + mov r15, rcx ; save key material address + + ; get actual address of key material: pRKeys += (nr-9) * SC + lea rax,[rdx*4] + lea rax, [r15+rax*4-9*(SC)*4] ; AES-128 round keys + +;; +;; processing +;; + lea r10, [r9*BLKS_PER_LOOP] ; 4 cfb block +.blks_loop: + cmp r8, r10 + cmovl r10, r8 + COPY_32U {rsp+5*16}, rdi, r10, r11d ; move 1-4 input blocks to stack + + mov r12, r10 ; copy length to be processed + xor r11, r11 ; index +.single_blk: + movdqu xmm0, oword [rsp+r11] ; get processing blocks + + pxor xmm0, oword [r15] ; whitening + + cmp rdx,12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + ; do encryption +.key_256_s: + aesenc xmm0, oword [rax-4*4*SC] + aesenc xmm0, oword [rax-3*4*SC] +.key_192_s: + aesenc xmm0, oword [rax-2*4*SC] + aesenc xmm0, oword [rax-1*4*SC] +.key_128_s: + aesenc xmm0, oword [rax+0*4*SC] + aesenc xmm0, oword [rax+1*4*SC] + aesenc xmm0, oword [rax+2*4*SC] + aesenc xmm0, oword [rax+3*4*SC] + aesenc xmm0, oword [rax+4*4*SC] + aesenc xmm0, oword [rax+5*4*SC] + aesenc xmm0, oword [rax+6*4*SC] + aesenc xmm0, oword [rax+7*4*SC] + aesenc xmm0, oword [rax+8*4*SC] + aesenclast xmm0, oword [rax+9*4*SC] + + movdqu xmm1, oword [rsp+5*16+r11] ; get src blocks from the stack + pxor xmm0, xmm1 ; xor src + movdqu oword [rsp+1*16+r11],xmm0 ;and store into the stack + + add r11, r9 ; advance index + sub r12, r9 ; decrease length + jg .single_blk + + COPY_32U rsi, {rsp+1*16}, r10, r11d ; move 1-4 blocks to output + + movdqu xmm0, oword [rsp+r10]; update IV + movdqa oword [rsp], xmm0 + + add rdi, r10 + add rsi, r10 + sub r8, r10 + jg .blks_loop + + REST_XMM + REST_GPR + ret +ENDFUNC EncryptCFB32_RIJ128_AES_NI + +align IPP_ALIGN_FACTOR +IPPASM EncryptCFB128_RIJ128_AES_NI,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi + USES_XMM + COMP_ABI 6 +;; rdi: pInpBlk: DWORD, ; input blocks address +;; rsi: pOutBlk: DWORD, ; output blocks address +;; rdx: nr: DWORD, ; number of rounds +;; rcx pKey: DWORD ; key material address +;; r8d cfbBlks: DWORD ; length of stream in bytes +;; r9d pIV: BYTE ; pointer to the IV + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) + + movdqu xmm0, oword [r9] ; get IV + + movsxd r8, r8d ; length of stream + movsxd r9, r9d ; cfb blk size + + ; get actual address of key material: pRKeys += (nr-9) * SC + lea rax,[rdx*4] + lea rax, [rcx+rax*4-9*(SC)*4] ; AES-128 round keys + +;; +;; processing +;; +.blks_loop: + pxor xmm0, oword [rcx] ; whitening + + movdqu xmm1, oword [rdi] ; input blocks + + + cmp rdx,12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + ; do encryption +.key_256_s: + aesenc xmm0, oword [rax-4*4*SC] + aesenc xmm0, oword [rax-3*4*SC] +.key_192_s: + aesenc xmm0, oword [rax-2*4*SC] + aesenc xmm0, oword [rax-1*4*SC] +.key_128_s: + aesenc xmm0, oword [rax+0*4*SC] + aesenc xmm0, oword [rax+1*4*SC] + aesenc xmm0, oword [rax+2*4*SC] + aesenc xmm0, oword [rax+3*4*SC] + aesenc xmm0, oword [rax+4*4*SC] + aesenc xmm0, oword [rax+5*4*SC] + aesenc xmm0, oword [rax+6*4*SC] + aesenc xmm0, oword [rax+7*4*SC] + aesenc xmm0, oword [rax+8*4*SC] + aesenclast xmm0, oword [rax+9*4*SC] + + pxor xmm0, xmm1 ; xor src + movdqu oword [rsi],xmm0 ;and store into the dst + + add rdi, 16 + add rsi, 16 + sub r8, 16 + jg .blks_loop + + REST_XMM + REST_GPR + ret +ENDFUNC EncryptCFB128_RIJ128_AES_NI + +%endif + +%endif ;; _AES_NI_ENABLING_ + + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encryptctr128pipee9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encryptctr128pipee9as.asm new file mode 100644 index 0000000..c8f9a95 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encryptctr128pipee9as.asm @@ -0,0 +1,457 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Cipher function +; +; Content: +; EncryptCTR_RIJ128pipe_AES_NI() +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "ia_32e_regs.inc" +%include "pcpvariant.inc" + +%if (_AES_NI_ENABLING_ == _FEATURE_ON_) || (_AES_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_Y8) + +segment .text align=IPP_ALIGN_FACTOR + + +;*************************************************************** +;* Purpose: pipelined RIJ128 CTR128 encryption/decryption +;* +;* +;* void EncryptStreamCTR32_AES_NI(const Ipp8u* inpBlk, +;* Ipp8u* outBlk, +;* int nr, +;* const Ipp8u* pRKey, +;* int length, +;* Ipp8u* pIV) +;*************************************************************** +%assign AES_BLOCK (16) + +align IPP_ALIGN_FACTOR +IPPASM EncryptStreamCTR32_AES_NI,PUBLIC +%assign LOCAL_FRAME sizeof(oword)*8 + USES_GPR rsi,rdi,rbx + USES_XMM xmm6,xmm7,xmm8,xmm9 + COMP_ABI 6 +;; rdi: pInpBlk: BYTE ; pointer to the input +;; rsi: pOutBlk: BYTE ; pointer to the output +;; rdx: nr: DWORD ; number of rounds +;; rcx pKey: BYTE ; key material address +;; r8d length: DWORD ; length of the input +;; r9 pIV : BYTE ; pointer to the iv (BE string counter representation) + +%xdefine SC (4) + + movdqu xmm0, oword [r9] ; input IV + movsxd r8, r8d ; length + + lea r11, [r8+AES_BLOCK-1] ; r11 = blocks = ceil(len/AES_BLOCK) + shr r11, 4 + + mov rax, qword [r9+sizeof(qword)] ; input IV + mov rbx, qword [r9] + bswap rax + bswap rbx + mov r10d, eax ; save ctr32 + add rax, r11 ; +blocks + adc rbx, 0 + bswap rax + bswap rbx + mov qword [r9+sizeof(qword)], rax ; update IV + mov qword [r9], rbx + + mov r11d, dword [rcx+AES_BLOCK-sizeof(dword)] ; get whitening keys corresponding to ctr32 + + pxor xmm0, oword [rcx] ; apply whitening keys + add rcx, AES_BLOCK ; mov pointer to the round keys + movdqa oword [rsp+0*AES_BLOCK], xmm0 ; store unchanged part of ctr128 + movdqa oword [rsp+1*AES_BLOCK], xmm0 + movdqa oword [rsp+2*AES_BLOCK], xmm0 + movdqa oword [rsp+3*AES_BLOCK], xmm0 + movdqa oword [rsp+4*AES_BLOCK], xmm0 + movdqa oword [rsp+5*AES_BLOCK], xmm0 + movdqa oword [rsp+6*AES_BLOCK], xmm0 + movdqa oword [rsp+7*AES_BLOCK], xmm0 + + mov r9, rsp + cmp r8, AES_BLOCK ; test %if single block processed + jle .short123_input + + movdqa xmm1, xmm0 + movdqa xmm2, xmm0 + movdqa xmm3, xmm0 + lea ebx, [r10d+1] ; init ctr32+1 + lea eax, [r10d+2] ; init ctr32+2 + lea r9d, [r10d+3] ; init ctr32+3 + bswap ebx + bswap eax + bswap r9d + xor ebx, r11d + pinsrd xmm1, ebx, 3 + xor eax, r11d + pinsrd xmm2, eax, 3 + xor r9d, r11d + pinsrd xmm3, r9d, 3 + + movdqa oword [rsp+1*AES_BLOCK], xmm1 + movdqa oword [rsp+2*AES_BLOCK], xmm2 + movdqa oword [rsp+3*AES_BLOCK], xmm3 + + mov r9, rsp + + movdqa xmm4, oword [rcx+0*AES_BLOCK] ; pre load operation's keys + movdqa xmm5, oword [rcx+1*AES_BLOCK] + + cmp r8, (4*AES_BLOCK) ; test %if 1-2-3 blocks processed + jl .short123_input + jz .short_input + + lea eax, [r10d+4] ; init ctr32+4 + lea ebx, [r10d+5] ; init ctr32+5 + bswap eax + bswap ebx + xor ebx, r11d + xor eax, r11d + mov dword [rsp+4*AES_BLOCK+AES_BLOCK-sizeof(dword)], eax + mov dword [rsp+5*AES_BLOCK+AES_BLOCK-sizeof(dword)], ebx + + lea eax, [r10d+6] ; init ctr32+6 + lea ebx, [r10d+7] ; init ctr32+7 + bswap eax + bswap ebx + xor eax, r11d + xor ebx, r11d + mov dword [rsp+6*AES_BLOCK+AES_BLOCK-sizeof(dword)], eax + mov dword [rsp+7*AES_BLOCK+AES_BLOCK-sizeof(dword)], ebx + + cmp r8, (8*AES_BLOCK) + jl .short_input + +;; +;; 8-blocks processing alaivalbe +;; + sub rsp, sizeof(oword)*4 ; save xmm registers + movdqa oword [rsp+0*sizeof(oword)], xmm10 + movdqa oword [rsp+1*sizeof(oword)], xmm11 + movdqa oword [rsp+2*sizeof(oword)], xmm12 + movdqa oword [rsp+3*sizeof(oword)], xmm13 + + push rcx ; store pointer to the key material + push rdx ; store number of rounds + sub r8, (8*AES_BLOCK) + +align IPP_ALIGN_FACTOR +.blk8_loop: +; movdqa xmm4, oword [rcx+0*AES_BLOCK] ; pre load operation's keys +; movdqa xmm5, oword [rcx+1*AES_BLOCK] + add rcx, (2*AES_BLOCK) + + add r10d, 8 ; next counter value + sub rdx, 4 ; rounds -= 4 + + movdqa xmm6, oword [r9+4*AES_BLOCK] ; load current ctr32 + movdqa xmm7, oword [r9+5*AES_BLOCK] + movdqa xmm8, oword [r9+6*AES_BLOCK] + movdqa xmm9, oword [r9+7*AES_BLOCK] + + mov eax, r10d ; improve next 2 ctr32 values in advance + aesenc xmm0, xmm4 + lea ebx, [r10d+1] + aesenc xmm1, xmm4 + bswap eax + aesenc xmm2, xmm4 + bswap ebx + aesenc xmm3, xmm4 + xor eax, r11d + aesenc xmm6, xmm4 + xor ebx, r11d + aesenc xmm7, xmm4 + mov dword [r9+0*AES_BLOCK+AES_BLOCK-sizeof(dword)], eax + aesenc xmm8, xmm4 + mov dword [r9+1*AES_BLOCK+AES_BLOCK-sizeof(dword)], ebx + aesenc xmm9, xmm4 + movdqa xmm4, oword [rcx] ; pre load operation's keys + + lea eax, [r10d+2] ; improve next 2 ctr32 values in advance + aesenc xmm0, xmm5 + lea ebx, [r10d+3] + aesenc xmm1, xmm5 + bswap eax + aesenc xmm2, xmm5 + bswap ebx + aesenc xmm3, xmm5 + xor eax, r11d + aesenc xmm6, xmm5 + xor ebx, r11d + aesenc xmm7, xmm5 + mov dword [r9+2*AES_BLOCK+AES_BLOCK-sizeof(dword)], eax + aesenc xmm8, xmm5 + mov dword [r9+3*AES_BLOCK+AES_BLOCK-sizeof(dword)], ebx + aesenc xmm9, xmm5 + movdqa xmm5, oword [rcx+AES_BLOCK] ; pre load operation's keys + +align IPP_ALIGN_FACTOR +.cipher_loop: + add rcx, (2*AES_BLOCK) + sub rdx, 2 + + aesenc xmm0, xmm4 + aesenc xmm1, xmm4 + aesenc xmm2, xmm4 + aesenc xmm3, xmm4 + aesenc xmm6, xmm4 + aesenc xmm7, xmm4 + aesenc xmm8, xmm4 + aesenc xmm9, xmm4 + movdqa xmm4, oword [rcx] ; pre load operation's keys + + aesenc xmm0, xmm5 + aesenc xmm1, xmm5 + aesenc xmm2, xmm5 + aesenc xmm3, xmm5 + aesenc xmm6, xmm5 + aesenc xmm7, xmm5 + aesenc xmm8, xmm5 + aesenc xmm9, xmm5 + movdqa xmm5, oword [rcx+AES_BLOCK] ; pre load operation's keys + jnz .cipher_loop + + lea eax, [r10d+4] ; improve next 2 ctr32 values in advance + aesenc xmm0, xmm4 + lea ebx, [r10d+5] + aesenc xmm1, xmm4 + bswap eax + aesenc xmm2, xmm4 + bswap ebx + aesenc xmm3, xmm4 + xor eax, r11d + aesenc xmm6, xmm4 + xor ebx, r11d + aesenc xmm7, xmm4 + mov dword [r9+4*AES_BLOCK+AES_BLOCK-sizeof(dword)], eax + aesenc xmm8, xmm4 + mov dword [r9+5*AES_BLOCK+AES_BLOCK-sizeof(dword)], ebx + aesenc xmm9, xmm4 + + lea eax, [r10d+6] ; improve next 2 ctr32 values in advance + aesenclast xmm0, xmm5 + lea ebx, [r10d+7] + aesenclast xmm1, xmm5 + bswap eax + aesenclast xmm2, xmm5 + bswap ebx + aesenclast xmm3, xmm5 + xor eax, r11d + aesenclast xmm6, xmm5 + xor ebx, r11d + aesenclast xmm7, xmm5 + mov dword [r9+6*AES_BLOCK+AES_BLOCK-sizeof(dword)], eax + aesenclast xmm8, xmm5 + mov dword [r9+7*AES_BLOCK+AES_BLOCK-sizeof(dword)], ebx + aesenclast xmm9, xmm5 + + movdqu xmm10, oword [rdi+0*AES_BLOCK] + movdqu xmm11, oword [rdi+1*AES_BLOCK] + movdqu xmm12, oword [rdi+2*AES_BLOCK] + movdqu xmm13, oword [rdi+3*AES_BLOCK] + pxor xmm0, xmm10 + pxor xmm1, xmm11 + pxor xmm2, xmm12 + pxor xmm3, xmm13 + movdqu oword [rsi+0*AES_BLOCK], xmm0 + movdqu oword [rsi+1*AES_BLOCK], xmm1 + movdqu oword [rsi+2*AES_BLOCK], xmm2 + movdqu oword [rsi+3*AES_BLOCK], xmm3 + movdqu xmm10, oword [rdi+4*AES_BLOCK] + movdqu xmm11, oword [rdi+5*AES_BLOCK] + movdqu xmm12, oword [rdi+6*AES_BLOCK] + movdqu xmm13, oword [rdi+7*AES_BLOCK] + pxor xmm6, xmm10 + pxor xmm7, xmm11 + pxor xmm8, xmm12 + pxor xmm9, xmm13 + movdqu oword [rsi+4*AES_BLOCK], xmm6 + movdqu oword [rsi+5*AES_BLOCK], xmm7 + movdqu oword [rsi+6*AES_BLOCK], xmm8 + movdqu oword [rsi+7*AES_BLOCK], xmm9 + + mov rcx, qword [rsp+sizeof(qword)] ; restore pointer to the key material + mov rdx, qword [rsp] ; restore number of rounds + + movdqa xmm0, oword [r9+0*AES_BLOCK] ; load next current ctr32 + movdqa xmm1, oword [r9+1*AES_BLOCK] + movdqa xmm2, oword [r9+2*AES_BLOCK] + movdqa xmm3, oword [r9+3*AES_BLOCK] + + movdqa xmm4, oword [rcx+0*AES_BLOCK] ; pre load operation's keys + movdqa xmm5, oword [rcx+1*AES_BLOCK] ; pre load operation's keys + + add rdi, 8*AES_BLOCK + add rsi, 8*AES_BLOCK + sub r8, 8*AES_BLOCK + jge .blk8_loop + + pop rdx + pop rcx + + movdqa xmm10,oword [rsp+0*sizeof(oword)] ; resrore xmm registers + movdqa xmm11,oword [rsp+1*sizeof(oword)] + movdqa xmm12,oword [rsp+2*sizeof(oword)] + movdqa xmm13,oword [rsp+3*sizeof(oword)] + add rsp, sizeof(oword)*4 ; release stack + + add r8, 8*AES_BLOCK + jz .quit + +align IPP_ALIGN_FACTOR +.short_input: +;; +;; test %if 4-blocks processing alaivalbe +;; + cmp r8, (4*AES_BLOCK) + jl .short123_input + + mov rbx, rcx ; pointer to the key material + lea r10, [rdx-2] ; rounds -= 2 + +align IPP_ALIGN_FACTOR +.cipher_loop4: + add rbx, (2*AES_BLOCK) + sub r10, 2 + + aesenc xmm0, xmm4 ; regular round + aesenc xmm1, xmm4 + aesenc xmm2, xmm4 + aesenc xmm3, xmm4 + movdqa xmm4, oword [rbx] ; pre load operation's keys + + aesenc xmm0, xmm5 + aesenc xmm1, xmm5 + aesenc xmm2, xmm5 + aesenc xmm3, xmm5 + movdqa xmm5, oword [rbx+AES_BLOCK] ; pre load operation's keys + jnz .cipher_loop4 + + movdqu xmm6, oword [rdi+0*AES_BLOCK] + movdqu xmm7, oword [rdi+1*AES_BLOCK] + movdqu xmm8, oword [rdi+2*AES_BLOCK] + movdqu xmm9, oword [rdi+3*AES_BLOCK] + add rdi, (4*AES_BLOCK) + + aesenc xmm0, xmm4 + aesenc xmm1, xmm4 + aesenc xmm2, xmm4 + aesenc xmm3, xmm4 + + aesenclast xmm0, xmm5 + aesenclast xmm1, xmm5 + aesenclast xmm2, xmm5 + aesenclast xmm3, xmm5 + + pxor xmm0, xmm6 + movdqu oword [rsi+0*AES_BLOCK], xmm0 + pxor xmm1, xmm7 + movdqu oword [rsi+1*AES_BLOCK], xmm1 + pxor xmm2, xmm8 + movdqu oword [rsi+2*AES_BLOCK], xmm2 + pxor xmm3, xmm9 + movdqu oword [rsi+3*AES_BLOCK], xmm3 + add rsi, (4*AES_BLOCK) + + add r9, (4*AES_BLOCK) + + sub r8, (4*AES_BLOCK) + jz .quit + +;; +;; block-by-block processing +;; +.short123_input: + ; get actual address of key material: pRKeys += (nr-9) * SC + lea rbx,[rdx*4] + lea rbx, [rcx+rbx*4-10*(SC)*4] ; AES-128 round keys + +.single_blk: + movdqa xmm0, oword [r9] ; counter from the stack + add r9, AES_BLOCK + + cmp rdx,12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + +.key_256_s: + aesenc xmm0,oword [rbx-4*4*SC] + aesenc xmm0,oword [rbx-3*4*SC] +.key_192_s: + aesenc xmm0,oword [rbx-2*4*SC] + aesenc xmm0,oword [rbx-1*4*SC] +.key_128_s: + aesenc xmm0,oword [rbx+0*4*SC] + aesenc xmm0,oword [rbx+1*4*SC] + aesenc xmm0,oword [rbx+2*4*SC] + aesenc xmm0,oword [rbx+3*4*SC] + aesenc xmm0,oword [rbx+4*4*SC] + aesenc xmm0,oword [rbx+5*4*SC] + aesenc xmm0,oword [rbx+6*4*SC] + aesenc xmm0,oword [rbx+7*4*SC] + aesenc xmm0,oword [rbx+8*4*SC] + aesenclast xmm0,oword [rbx+9*4*SC] + + cmp r8, AES_BLOCK ; test %if partial bloak + jl .partial_block + + movdqu xmm1, oword [rdi] ; input data block + add rdi, AES_BLOCK + pxor xmm0, xmm1 ; output block + movdqu oword [rsi], xmm0 ; save output block + add rsi, AES_BLOCK + + sub r8, AES_BLOCK + jz .quit + jmp .single_blk + +.partial_block: + pextrb eax, xmm0, 0 + psrldq xmm0, 1 + movzx edx, byte [rdi] + inc rdi + xor rax, rdx + mov byte [rsi], al + inc rsi + dec r8 + jnz .partial_block + +.quit: + pxor xmm4, xmm4 + pxor xmm5, xmm5 + + REST_XMM + REST_GPR + ret +ENDFUNC EncryptStreamCTR32_AES_NI + +%endif ;; _IPP32E_Y8 +%endif ;; _AES_NI_ENABLING_ + + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encryptctrpipee9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encryptctrpipee9as.asm new file mode 100644 index 0000000..a9128f7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encryptctrpipee9as.asm @@ -0,0 +1,271 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Cipher function +; +; Content: +; EncryptCTR_RIJ128pipe_AES_NI() +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "ia_32e_regs.inc" +%include "pcpvariant.inc" + +%if (_AES_NI_ENABLING_ == _FEATURE_ON_) || (_AES_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_Y8) + +segment .text align=IPP_ALIGN_FACTOR + + +align IPP_ALIGN_FACTOR +u128_str DB 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 + +;*************************************************************** +;* Purpose: pipelined RIJ128 CTR encryption/decryption +;* +;* void EncryptCTR_RIJ128pipe_AES_NI(const Ipp32u* inpBlk, +;* Ipp32u* outBlk, +;* int nr, +;* const Ipp32u* pRKey, +;* int length, +;* Ipp8u* pCtrValue, +;* Ipp8u* pCtrBitMask) +;*************************************************************** + +;; +;; Lib = Y8 +;; +;; Caller = ippsAESEncryptCTR +;; +align IPP_ALIGN_FACTOR +IPPASM EncryptCTR_RIJ128pipe_AES_NI,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,rbx + USES_XMM xmm6,xmm7,xmm8,xmm9 + COMP_ABI 7 +;; rdi: pInpBlk: DWORD, ; pointer to the input +;; rsi: pOutBlk: DWORD, ; pointer to the output +;; rdx: nr: DWORD, ; number of rounds +;; rcx pKey: DWORD ; key material address +;; r8d length: DWORD ; length of the input +;; r9 pCtrValue: BYTED ; pointer to the Counter +;; [rsp+ARG_7] pCtrBitMask: BYTE ; pointer to the Counter Bit Mask + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) +%assign BYTES_PER_BLK (16) +%assign BYTES_PER_LOOP (BYTES_PER_BLK*BLKS_PER_LOOP) + + mov rax, [rsp+ARG_7] + movdqu xmm8, oword [rax] ; counter bit mask + + movdqu xmm0, oword [r9] ; initial counter + movdqa xmm9, xmm8 + pandn xmm9, xmm0 ; counter template + + ;; + ;; init counter + ;; + mov rbx, qword [r9] ; initial counter (BE) + mov rax, qword [r9+8] + bswap rbx + bswap rax + + movsxd r8, r8d + sub r8, BYTES_PER_LOOP + jl .short_input + +;; +;; pipelined processing +;; +.blks_loop: + movdqa xmm4,oword [rel u128_str] + + pinsrq xmm0, rax, 0 ; get counter value + pinsrq xmm0, rbx, 1 + pshufb xmm0, xmm4 ; convert int the octet string + pand xmm0, xmm8 ; select counter bits + por xmm0, xmm9 ; add unchanged bits + + add rax, 1 + adc rbx, 0 + pinsrq xmm1, rax, 0 + pinsrq xmm1, rbx, 1 + pshufb xmm1, xmm4 + pand xmm1, xmm8 + por xmm1, xmm9 + + add rax, 1 + adc rbx, 0 + pinsrq xmm2, rax, 0 + pinsrq xmm2, rbx, 1 + pshufb xmm2, xmm4 + pand xmm2, xmm8 + por xmm2, xmm9 + + add rax, 1 + adc rbx, 0 + pinsrq xmm3, rax, 0 + pinsrq xmm3, rbx, 1 + pshufb xmm3, xmm4 + pand xmm3, xmm8 + por xmm3, xmm9 + + movdqa xmm4, oword [rcx] + mov r10, rcx ; set pointer to the key material + + pxor xmm0, xmm4 ; whitening + pxor xmm1, xmm4 + pxor xmm2, xmm4 + pxor xmm3, xmm4 + + movdqa xmm4, oword [r10+16] + add r10, 16 + + mov r11, rdx ; counter depending on key length + sub r11, 1 +.cipher_loop: + aesenc xmm0, xmm4 ; regular round + aesenc xmm1, xmm4 + aesenc xmm2, xmm4 + aesenc xmm3, xmm4 + movdqa xmm4, oword [r10+16] + add r10, 16 + dec r11 + jnz .cipher_loop + + aesenclast xmm0, xmm4 ; irregular round + aesenclast xmm1, xmm4 + aesenclast xmm2, xmm4 + aesenclast xmm3, xmm4 + + movdqu xmm4, oword [rdi+0*BYTES_PER_BLK] ; 4 input blocks + movdqu xmm5, oword [rdi+1*BYTES_PER_BLK] + movdqu xmm6, oword [rdi+2*BYTES_PER_BLK] + movdqu xmm7, oword [rdi+3*BYTES_PER_BLK] + add rdi, BYTES_PER_LOOP + + pxor xmm0, xmm4 ; 4 output blocks + movdqu oword [rsi+0*BYTES_PER_BLK], xmm0 + pxor xmm1, xmm5 + movdqu oword [rsi+1*BYTES_PER_BLK], xmm1 + pxor xmm2, xmm6 + movdqu oword [rsi+2*BYTES_PER_BLK], xmm2 + pxor xmm3, xmm7 + movdqu oword [rsi+3*BYTES_PER_BLK], xmm3 + + add rax, 1 ; advance counter + adc rbx, 0 + + add rsi, BYTES_PER_LOOP + + sub r8, BYTES_PER_LOOP + jge .blks_loop + +;; +;; block-by-block processing +;; +.short_input: + add r8, BYTES_PER_LOOP + jz .quit + + ; get actual address of key material: pRKeys += (nr-9) * SC + lea r10,[rdx*4] + lea r10, [rcx+r10*4-9*(SC)*4] ; AES-128 round keys + +.single_blk_loop: + pinsrq xmm0, rax, 0 ; get counter value + pinsrq xmm0, rbx, 1 + pshufb xmm0, [rel u128_str] ; convert int the octet string + pand xmm0, xmm8 ; select counter bits + por xmm0, xmm9 ; add unchanged bits + + pxor xmm0, oword [rcx] ; whitening + + cmp rdx,12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + +.key_256_s: + aesenc xmm0,oword [r10-4*4*SC] + aesenc xmm0,oword [r10-3*4*SC] +.key_192_s: + aesenc xmm0,oword [r10-2*4*SC] + aesenc xmm0,oword [r10-1*4*SC] +.key_128_s: + aesenc xmm0,oword [r10+0*4*SC] + aesenc xmm0,oword [r10+1*4*SC] + aesenc xmm0,oword [r10+2*4*SC] + aesenc xmm0,oword [r10+3*4*SC] + aesenc xmm0,oword [r10+4*4*SC] + aesenc xmm0,oword [r10+5*4*SC] + aesenc xmm0,oword [r10+6*4*SC] + aesenc xmm0,oword [r10+7*4*SC] + aesenc xmm0,oword [r10+8*4*SC] + aesenclast xmm0,oword [r10+9*4*SC] + + add rax, 1 ; update counter + adc rbx, 0 + + sub r8, BYTES_PER_BLK + jl .partial_block + + movdqu xmm4, oword [rdi] ; input block + pxor xmm0, xmm4 ; output block + movdqu oword [rsi], xmm0 ; save output block + + add rdi, BYTES_PER_BLK + add rsi, BYTES_PER_BLK + cmp r8, 0 + jz .quit + jmp .single_blk_loop + +.partial_block: + add r8, BYTES_PER_BLK + +.partial_block_loop: + pextrb r10d, xmm0, 0 + psrldq xmm0, 1 + movzx r11d, byte [rdi] + xor r10, r11 + mov byte [rsi], r10b + inc rdi + inc rsi + dec r8 + jnz .partial_block_loop + +.quit: + pinsrq xmm0, rax, 0 ; get counter value + pinsrq xmm0, rbx, 1 + pshufb xmm0, [rel u128_str] ; convert int the octet string + pand xmm0, xmm8 ; select counter bits + por xmm0, xmm9 ; add unchanged bits + movdqu oword [r9], xmm0 ; return updated counter + REST_XMM + REST_GPR + ret +ENDFUNC EncryptCTR_RIJ128pipe_AES_NI + +%endif + +%endif ;; _AES_NI_ENABLING_ + + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encrypte9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encrypte9as.asm new file mode 100644 index 0000000..229f43e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encrypte9as.asm @@ -0,0 +1,122 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Cipher function +; +; Content: +; Encrypt_RIJ128_AES_NI() +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "ia_32e_regs.inc" +%include "pcpvariant.inc" + +%if (_AES_NI_ENABLING_ == _FEATURE_ON_) || (_AES_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_Y8) + +segment .text align=IPP_ALIGN_FACTOR + + +;*************************************************************** +;* Purpose: single block RIJ128 Cipher +;* +;* void Encrypt_RIJ128_AES_NI(const Ipp32u* inpBlk, +;* Ipp32u* outBlk, +;* int nr, +;* const Ipp32u* pRKey, +;* const Ipp32u Tables[][256]) +;*************************************************************** + +;; +;; Lib = Y8 +;; +;; Caller = ippsRijndael128EncryptECB +;; Caller = ippsRijndael128EncryptCBC +;; Caller = ippsRijndael128EncryptCFB +;; Caller = ippsRijndael128DecryptCFB +;; +;; Caller = ippsDAARijndael128Update +;; Caller = ippsDAARijndael128Final +;; Caller = ippsDAARijndael128MessageDigest +;; +align IPP_ALIGN_FACTOR +IPPASM Encrypt_RIJ128_AES_NI,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi + USES_XMM + COMP_ABI 4 + +;; rdi: pInpBlk: DWORD, ; input block address +;; rsi: pOutBlk: DWORD, ; output block address +;; rdx: nr: DWORD, ; number of rounds +;; rcx pKey: DWORD ; key material address + +%xdefine SC (4) + + + movdqu xmm0, oword [rdi] ; input block + + ;;whitening + pxor xmm0, oword [rcx] + + ; get actual address of key material: pRKeys += (nr-9) * SC + lea rax,[rdx*4] + lea rcx,[rcx+rax*4-9*(SC)*4] ; AES-128-keys + + cmp rdx,12 ; switch according to number of rounds + jl .key_128 + jz .key_192 + + ;; + ;; regular rounds + ;; +.key_256: + aesenc xmm0,oword [rcx-4*4*SC] + aesenc xmm0,oword [rcx-3*4*SC] +.key_192: + aesenc xmm0,oword [rcx-2*4*SC] + aesenc xmm0,oword [rcx-1*4*SC] +.key_128: + aesenc xmm0,oword [rcx+0*4*SC] + aesenc xmm0,oword [rcx+1*4*SC] + aesenc xmm0,oword [rcx+2*4*SC] + aesenc xmm0,oword [rcx+3*4*SC] + aesenc xmm0,oword [rcx+4*4*SC] + aesenc xmm0,oword [rcx+5*4*SC] + aesenc xmm0,oword [rcx+6*4*SC] + aesenc xmm0,oword [rcx+7*4*SC] + aesenc xmm0,oword [rcx+8*4*SC] + ;; + ;; last rounds + ;; + aesenclast xmm0,oword [rcx+9*4*SC] + + movdqu oword [rsi], xmm0 ; output block + + REST_XMM + REST_GPR + ret +ENDFUNC Encrypt_RIJ128_AES_NI + +%endif + +%endif ;; _AES_NI_ENABLING_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encryptecbpipee9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encryptecbpipee9as.asm new file mode 100644 index 0000000..0042619 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encryptecbpipee9as.asm @@ -0,0 +1,179 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Cipher function +; +; Content: +; EncryptECB_RIJ128pipe_AES_NI() +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "ia_32e_regs.inc" +%include "pcpvariant.inc" + +%if (_AES_NI_ENABLING_ == _FEATURE_ON_) || (_AES_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_Y8) + +segment .text align=IPP_ALIGN_FACTOR + + +;*************************************************************** +;* Purpose: pipelined RIJ128 ECB encryption +;* +;* void EncryptECB_RIJ128pipe_AES_NI(const Ipp32u* inpBlk, +;* Ipp32u* outBlk, +;* int nr, +;* const Ipp32u* pRKey, +;* int len) +;*************************************************************** + +;; +;; Lib = Y8 +;; +;; Caller = ippsAESEncryptECB +;; +align IPP_ALIGN_FACTOR +IPPASM EncryptECB_RIJ128pipe_AES_NI,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi + USES_XMM + COMP_ABI 5 +;; rdi: pInpBlk: DWORD, ; input blocks address +;; rsi: pOutBlk: DWORD, ; output blocks address +;; rdx: nr: DWORD, ; number of rounds +;; rcx pKey: DWORD ; key material address +;; r8d length: DWORD ; length (bytes) + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) +%assign BYTES_PER_BLK (16) +%assign BYTES_PER_LOOP (BYTES_PER_BLK*BLKS_PER_LOOP) + + movsxd r8, r8d + sub r8, BYTES_PER_LOOP + jl .short_input + +;; +;; pipelined processing +;; +align IPP_ALIGN_FACTOR +.blks_loop: + movdqa xmm4, oword [rcx] + mov r9, rcx ; set pointer to the key material + + movdqu xmm0, oword [rdi+0*16] ; get input blocks + movdqu xmm1, oword [rdi+1*16] + movdqu xmm2, oword [rdi+2*16] + movdqu xmm3, oword [rdi+3*16] + add rdi, BYTES_PER_LOOP + + pxor xmm0, xmm4 ; whitening + pxor xmm1, xmm4 + pxor xmm2, xmm4 + pxor xmm3, xmm4 + + movdqa xmm4, oword [r9+16] + add r9, 16 + + mov r10, rdx ; counter depending on key length + sub r10, 1 +align IPP_ALIGN_FACTOR +.cipher_loop: + aesenc xmm0, xmm4 ; regular round + aesenc xmm1, xmm4 + aesenc xmm2, xmm4 + aesenc xmm3, xmm4 + movdqa xmm4, oword [r9+16] + add r9, 16 + dec r10 + jnz .cipher_loop + + aesenclast xmm0, xmm4 ; irregular round + movdqu oword [rsi+0*16], xmm0 ; store output blocks + aesenclast xmm1, xmm4 + movdqu oword [rsi+1*16], xmm1 + aesenclast xmm2, xmm4 + movdqu oword [rsi+2*16], xmm2 + aesenclast xmm3, xmm4 + movdqu oword [rsi+3*16], xmm3 + add rsi, BYTES_PER_LOOP + + sub r8, BYTES_PER_LOOP + jge .blks_loop + +;; +;; block-by-block processing +;; +.short_input: + add r8, BYTES_PER_LOOP + jz .quit + + ; get actual address of key material: pRKeys += (nr-9) * SC + lea rax,[rdx*4] + lea r9, [rcx+rax*4-9*(SC)*4] ; AES-128 round keys + +align IPP_ALIGN_FACTOR +.single_blk_loop: + movdqu xmm0, oword [rdi] ; get input block + add rdi, BYTES_PER_BLK + pxor xmm0, oword [rcx] ; whitening + + cmp rdx,12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + +.key_256_s: + aesenc xmm0,oword [r9-4*4*SC] + aesenc xmm0,oword [r9-3*4*SC] +.key_192_s: + aesenc xmm0,oword [r9-2*4*SC] + aesenc xmm0,oword [r9-1*4*SC] +.key_128_s: + aesenc xmm0,oword [r9+0*4*SC] + aesenc xmm0,oword [r9+1*4*SC] + aesenc xmm0,oword [r9+2*4*SC] + aesenc xmm0,oword [r9+3*4*SC] + aesenc xmm0,oword [r9+4*4*SC] + aesenc xmm0,oword [r9+5*4*SC] + aesenc xmm0,oword [r9+6*4*SC] + aesenc xmm0,oword [r9+7*4*SC] + aesenc xmm0,oword [r9+8*4*SC] + aesenclast xmm0,oword [r9+9*4*SC] + + movdqu oword [rsi], xmm0 ; save output block + add rsi, BYTES_PER_BLK + + sub r8, BYTES_PER_BLK + jnz .single_blk_loop + +.quit: + pxor xmm4, xmm4 + + REST_XMM + REST_GPR + ret +ENDFUNC EncryptECB_RIJ128pipe_AES_NI + +%endif + +%endif ;; _AES_NI_ENABLING_ + + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encryptofbe9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encryptofbe9as.asm new file mode 100644 index 0000000..b268007 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encryptofbe9as.asm @@ -0,0 +1,247 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Inverse Cipher function +; +; Content: +; Encrypt_RIJ128_AES_NI() +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_AES_NI_ENABLING_ == _FEATURE_ON_) || (_AES_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_Y8) + + +%macro COPY_8U 4.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + %xdefine %%limit %3 + %xdefine %%tmp %4 + + xor rcx, rcx +%%next_byte: + mov %%tmp, byte [%%src+rcx] + mov byte [%%dst+rcx], %%tmp + add rcx, 1 + cmp rcx, %%limit + jl %%next_byte +%endmacro + +segment .text align=IPP_ALIGN_FACTOR + + +;*************************************************************** +;* Purpose: RIJ128 OFB encryption +;* +;* void EncryptOFB_RIJ128_AES_NI(const Ipp32u* inpBlk, +;* Ipp32u* outBlk, +;* int nr, +;* const Ipp32u* pRKey, +;* int length, +;* int ofbBlks, +;* const Ipp8u* pIV) +;*************************************************************** + +;; +;; Lib = Y8 +;; +;; Caller = ippsRijndael128DecryptCFB +;; +align IPP_ALIGN_FACTOR +IPPASM EncryptOFB_RIJ128_AES_NI,PUBLIC +%assign LOCAL_FRAME (1+1+4+4)*16 + USES_GPR rsi,rdi,r12,r15 + USES_XMM + COMP_ABI 7 +;; rdi: pInpBlk: DWORD, ; input blocks address +;; rsi: pOutBlk: DWORD, ; output blocks address +;; rdx: nr: DWORD, ; number of rounds +;; rcx pKey: DWORD ; key material address +;; r8d cfbBlks: DWORD ; length of stream in bytes +;; r9d cfbSize: DWORD ; cfb blk size +;; [rsp+ARG_7] pIV BYTE ; pointer to the IV + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) + +%xdefine tmpInp [rsp] +%xdefine tmpOut [rsp+16*1] +%xdefine locDst [rsp+16*2] +%xdefine locSrc [rsp+16*6] + + mov rax, [rsp+ARG_7] ; IV address + movdqu xmm0, oword [rax] ; get IV + movdqa oword [rsp+0*16], xmm0 ; into the stack + + movsxd r8, r8d ; length of stream + movsxd r9, r9d ; cfb blk size + + mov r15, rcx ; save key material address + + ; get actual address of key material: pRKeys += (nr-9) * SC + lea rax,[rdx*4] + lea rax, [r15+rax*4-9*(SC)*4] ; AES-128 round keys + +;; +;; processing +;; + lea r10, [r9*BLKS_PER_LOOP] ; 4 cfb block + +align IPP_ALIGN_FACTOR +.blks_loop: + cmp r8, r10 + cmovl r10, r8 + COPY_8U {rsp+6*16}, rdi, r10, r11b ; move 1-4 input blocks to stack + + mov r12, r10 ; copy length to be processed + xor r11, r11 ; index + +align IPP_ALIGN_FACTOR +.single_blk: + pxor xmm0, oword [r15] ; whitening + + cmp rdx,12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + ; do encryption +.key_256_s: + aesenc xmm0, oword [rax-4*4*SC] + aesenc xmm0, oword [rax-3*4*SC] +.key_192_s: + aesenc xmm0, oword [rax-2*4*SC] + aesenc xmm0, oword [rax-1*4*SC] +.key_128_s: + aesenc xmm0, oword [rax+0*4*SC] + aesenc xmm0, oword [rax+1*4*SC] + aesenc xmm0, oword [rax+2*4*SC] + aesenc xmm0, oword [rax+3*4*SC] + aesenc xmm0, oword [rax+4*4*SC] + aesenc xmm0, oword [rax+5*4*SC] + aesenc xmm0, oword [rax+6*4*SC] + aesenc xmm0, oword [rax+7*4*SC] + aesenc xmm0, oword [rax+8*4*SC] + aesenclast xmm0, oword [rax+9*4*SC] + movdqa oword [rsp+1*16], xmm0 ; save chipher output + + movdqu xmm1, oword [rsp+6*16+r11] ; get src blocks from the stack + pxor xmm1, xmm0 ; xor src + movdqu oword [rsp+2*16+r11],xmm1 ;and store into the stack + + movdqu xmm0, oword [rsp+r9] ; update chiper input (IV) + movdqa oword [rsp], xmm0 + + add r11, r9 ; advance index + sub r12, r9 ; decrease length + jg .single_blk + + COPY_8U rsi, {rsp+2*16}, r10, r11b ; move 1-4 blocks to output + + add rdi, r10 + add rsi, r10 + sub r8, r10 + jg .blks_loop + + mov rax, [rsp+ARG_7] ; IV address + movdqa xmm0, oword [rsp] ; update IV before return + movdqu oword [rax], xmm0 + + REST_XMM + REST_GPR + ret +ENDFUNC EncryptOFB_RIJ128_AES_NI + +align IPP_ALIGN_FACTOR +IPPASM EncryptOFB128_RIJ128_AES_NI,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi + USES_XMM + COMP_ABI 6 +;; rdi: pInpBlk: DWORD, ; input blocks address +;; rsi: pOutBlk: DWORD, ; output blocks address +;; rdx: nr: DWORD, ; number of rounds +;; rcx pKey: DWORD ; key material address +;; r8d cfbBlks: DWORD ; length of stream in bytes +;; r9 pIV: BYTE ; pointer to the IV + +%xdefine SC (4) +%assign BLKS_PER_LOOP (4) + + movdqu xmm0, oword [r9] ; get IV + + movsxd r8, r8d ; length of stream + + ; get actual address of key material: pRKeys += (nr-9) * SC + lea rax,[rdx*4] + lea rax, [rcx+rax*4-9*(SC)*4] ; AES-128 round keys + +;; +;; processing +;; +.blks_loop: + pxor xmm0, oword [rcx] ; whitening + + movdqu xmm1, oword [rdi] ; input blocks + + + cmp rdx,12 ; switch according to number of rounds + jl .key_128_s + jz .key_192_s + ; do encryption +.key_256_s: + aesenc xmm0, oword [rax-4*4*SC] + aesenc xmm0, oword [rax-3*4*SC] +.key_192_s: + aesenc xmm0, oword [rax-2*4*SC] + aesenc xmm0, oword [rax-1*4*SC] +.key_128_s: + aesenc xmm0, oword [rax+0*4*SC] + aesenc xmm0, oword [rax+1*4*SC] + aesenc xmm0, oword [rax+2*4*SC] + aesenc xmm0, oword [rax+3*4*SC] + aesenc xmm0, oword [rax+4*4*SC] + aesenc xmm0, oword [rax+5*4*SC] + aesenc xmm0, oword [rax+6*4*SC] + aesenc xmm0, oword [rax+7*4*SC] + aesenc xmm0, oword [rax+8*4*SC] + aesenclast xmm0, oword [rax+9*4*SC] + + pxor xmm1, xmm0 ; xor src + movdqu oword [rsi],xmm1 ;and store into the dst + + add rdi, 16 + add rsi, 16 + sub r8, 16 + jg .blks_loop + + movdqu oword [r9], xmm0 ; update IV before return + + REST_XMM + REST_GPR + ret +ENDFUNC EncryptOFB128_RIJ128_AES_NI + +%endif + +%endif ;; _AES_NI_ENABLING_ + + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encryptxtse9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encryptxtse9as.asm new file mode 100644 index 0000000..6032ff8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128encryptxtse9as.asm @@ -0,0 +1,377 @@ +;=============================================================================== +; Copyright (C) 2016 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; AES functions +; +; Content: +; cpAESEncryptXTS_AES_NI() +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_AES_NI_ENABLING_ == _FEATURE_ON_) || (_AES_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_Y8) + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR + +ALPHA_MUL_CNT dq 00000000000000087h, 00000000000000001h + + +;*************************************************************** +;* Purpose: AES-XTS encryption +;* +;* void cpAESEncryptXTS_AES_NI(Ipp8u* outBlk, +;* const Ipp8u* inpBlk, +;* int length, +;* const Ipp8u* pRKey, +;* int nr, +;* Ipp8u* pTweak) +;*************************************************************** + +;; +;; key = ks[0] +;; mul_cnt = {0x0000000000000001:0x0000000000000087} +;; returns: +;; ktwk = twk^key +;; twk = twk*alpha +;; twk2= twk2 *2 - auxillary +;; +%macro OUTER_MUL_X 6.nolist + %xdefine %%ktwk %1 + %xdefine %%twk %2 + %xdefine %%key %3 + %xdefine %%mul_cnt %4 + %xdefine %%twk2 %5 + %xdefine %%mask %6 + + movdqa %%mask, %%twk2 + paddd %%twk2, %%twk2 + movdqa %%ktwk, %%twk + psrad %%mask, 31 + paddq %%twk, %%twk + pand %%mask, %%mul_cnt + pxor %%ktwk, %%key + pxor %%twk, %%mask +%endmacro + +%macro LAST_OUTER_MUL_X 5.nolist + %xdefine %%ktwk %1 + %xdefine %%twk %2 + %xdefine %%key %3 + %xdefine %%mul_cnt %4 + %xdefine %%twk2 %5 + + movdqa %%ktwk, %%twk + psrad %%twk2, 31 + paddq %%twk, %%twk + pand %%twk2, %%mul_cnt + pxor %%ktwk, %%key + pxor %%twk, %%twk2 +%endmacro + +%macro INNER_MUL_X 6.nolist + %xdefine %%ktwk %1 + %xdefine %%twk %2 + %xdefine %%key %3 + %xdefine %%mul_cnt %4 + %xdefine %%twk2 %5 + %xdefine %%mask %6 + + movdqa %%mask, %%twk2 + paddd %%twk2, %%twk2 + psrad %%mask, 31 + paddq %%twk, %%twk + pand %%mask, %%mul_cnt + pxor %%twk, %%mask +%ifnidn %%key,%%ktwk + movdqa %%key, %%ktwk +%endif + pxor %%ktwk, %%twk +%endmacro + +%macro LAST_INNER_MUL_X 3.nolist + %xdefine %%twk %1 + %xdefine %%mul_cnt %2 + %xdefine %%twk2 %3 + + psrad %%twk2, 31 + paddq %%twk, %%twk + pand %%twk2, %%mul_cnt + pxor %%twk, %%twk2 +%endmacro + +align IPP_ALIGN_FACTOR +IPPASM cpAESEncryptXTS_AES_NI,PUBLIC +%assign LOCAL_FRAME sizeof(oword)*6 + USES_GPR rsi,rdi + USES_XMM xmm6,xmm7,xmm8,xmm9,xmm10,xmm11,xmm12,xmm13,xmm14,xmm15 + COMP_ABI 6 +;; rdi: pOutBlk: BYTE ; pointer to the output bloak +;; rsi: pInpBlk: BYTE ; pointer to the input block +;; edx: nBlocks DWORD ; number of blocks +;; rcx: pKey: BYTE ; key material address +;; r8d: nr: DWORD ; number of rounds +;; r9 pTweak: BYTE ; pointer to the input/outpout (ciphertext) tweak + +%assign AES_BLOCK (16) + + movsxd r8, r8d ; number of cipher rounds + + movdqu xmm15, xmmword [r9] ; input tweak value + + shl r8, 4 ; key schedule length (bytes) + + movdqa xmm0, xmmword [rcx] ; key[0] + movdqa xmm8, xmmword [rel ALPHA_MUL_CNT] ; mul constant + + pshufd xmm9, xmm15, 5Fh ; {twk[1]:twk[1]:twk[3]:twk[3]} - auxillary value + + movsxd rdx, edx ; number of blocks being processing + + ;; compute: + ;; - ktwk[i] = twk[i]^key[0] + ;; - twk[i+1] = twk[i]*alpha^(i+1), i=0..5 + OUTER_MUL_X xmm10, xmm15, xmm0, xmm8, xmm9, xmm14 + OUTER_MUL_X xmm11, xmm15, xmm0, xmm8, xmm9, xmm14 + OUTER_MUL_X xmm12, xmm15, xmm0, xmm8, xmm9, xmm14 + OUTER_MUL_X xmm13, xmm15, xmm0, xmm8, xmm9, xmm14 + LAST_OUTER_MUL_X xmm14, xmm15, xmm0, xmm8, xmm9 + + movdqa xmm8, xmm15 ; save tweak for next iteration + pxor xmm15, xmm0 ; add key[0] + + sub rdx, 6 ; test input length + jc .short_input + +;; +;; blocks processing +;; +align IPP_ALIGN_FACTOR +.blks_loop: + pxor xmm0, xmmword [rcx+r8] ; key[0]^key[last] + + lea rax, [r8-6*AES_BLOCK] ; cipher_loop counter + lea r10, [rcx+rax+AES_BLOCK] ; mov key pointer down + neg rax + + movdqu xmm2, xmmword [rsi] ; src[0] - src[7] + movdqu xmm3, xmmword [rsi+AES_BLOCK] + movdqu xmm4, xmmword [rsi+2*AES_BLOCK] + movdqu xmm5, xmmword [rsi+3*AES_BLOCK] + movdqu xmm6, xmmword [rsi+4*AES_BLOCK] + movdqu xmm7, xmmword [rsi+5*AES_BLOCK] + + movdqa xmm1, xmmword [rcx+1*AES_BLOCK] ; key[1] + + pxor xmm2, xmm10 ; src[] ^twk[] ^ key[0] + pxor xmm3, xmm11 + pxor xmm4, xmm12 + pxor xmm5, xmm13 + pxor xmm6, xmm14 + pxor xmm7, xmm15 + + pxor xmm10, xmm0 ; store twk[] ^ key[last] + pxor xmm11, xmm0 + pxor xmm12, xmm0 + pxor xmm13, xmm0 + pxor xmm14, xmm0 + pxor xmm15, xmm0 + movdqa xmm0, xmmword [rcx+2*AES_BLOCK] ; key[2] + + movdqa xmmword [rsp+0*AES_BLOCK], xmm10 + movdqa xmmword [rsp+1*AES_BLOCK], xmm11 + movdqa xmmword [rsp+2*AES_BLOCK], xmm12 + movdqa xmmword [rsp+3*AES_BLOCK], xmm13 + movdqa xmmword [rsp+4*AES_BLOCK], xmm14 + movdqa xmmword [rsp+5*AES_BLOCK], xmm15 + add rsi, 6*AES_BLOCK + +align IPP_ALIGN_FACTOR +.cipher_loop: + add rax, 2*AES_BLOCK ; inc loop counter + aesenc xmm2, xmm1 ; regular rounds 3 - (last-2) + aesenc xmm3, xmm1 + aesenc xmm4, xmm1 + aesenc xmm5, xmm1 + aesenc xmm6, xmm1 + aesenc xmm7, xmm1 + movdqa xmm1, xmmword [r10+rax] + aesenc xmm2, xmm0 + aesenc xmm3, xmm0 + aesenc xmm4, xmm0 + aesenc xmm5, xmm0 + aesenc xmm6, xmm0 + aesenc xmm7, xmm0 + movdqa xmm0, xmmword [r10+rax+AES_BLOCK] + jnz .cipher_loop + + movdqa xmm10, xmmword [rcx] ; key[0] + + movdqa xmm15, xmm8 ; restore tweak value + pshufd xmm9, xmm8, 5Fh ; {twk[1]:twk[1]:twk[3]:twk[3]} - auxillary value + movdqa xmm8, xmmword [rel ALPHA_MUL_CNT] ; mul constant + + ; + ; last 6 rounds (5 regular rounds + irregular) + ; merged together with next tweaks computation + ; + aesenc xmm2, xmm1 + aesenc xmm3, xmm1 + aesenc xmm4, xmm1 + aesenc xmm5, xmm1 + aesenc xmm6, xmm1 + aesenc xmm7, xmm1 + movdqa xmm1, xmmword [rcx+r8-3*AES_BLOCK] + + INNER_MUL_X xmm10, xmm15, xmm11, xmm8, xmm9, xmm14 + + aesenc xmm2, xmm0 + aesenc xmm3, xmm0 + aesenc xmm4, xmm0 + aesenc xmm5, xmm0 + aesenc xmm6, xmm0 + aesenc xmm7, xmm0 + movdqa xmm0, xmmword [rcx+r8-2*AES_BLOCK] + + INNER_MUL_X xmm11, xmm15, xmm12, xmm8, xmm9, xmm14 + + aesenc xmm2, xmm1 + aesenc xmm3, xmm1 + aesenc xmm4, xmm1 + aesenc xmm5, xmm1 + aesenc xmm6, xmm1 + aesenc xmm7, xmm1 + movdqa xmm1, xmmword [rcx+r8-1*AES_BLOCK] + + INNER_MUL_X xmm12, xmm15, xmm13, xmm8, xmm9, xmm14 + + aesenc xmm2, xmm0 + aesenc xmm3, xmm0 + aesenc xmm4, xmm0 + aesenc xmm5, xmm0 + aesenc xmm6, xmm0 + aesenc xmm7, xmm0 + + INNER_MUL_X xmm13, xmm15, xmm14, xmm8, xmm9, xmm14 + + aesenc xmm2, xmm1 + aesenc xmm3, xmm1 + aesenc xmm4, xmm1 + aesenc xmm5, xmm1 + aesenc xmm6, xmm1 + aesenc xmm7, xmm1 + + INNER_MUL_X xmm14, xmm15, xmm0, xmm8, xmm9, xmm0 + + aesenclast xmm2, xmmword [rsp] ; final irregular round + aesenclast xmm3, xmmword [rsp+1*AES_BLOCK] + aesenclast xmm4, xmmword [rsp+2*AES_BLOCK] + aesenclast xmm5, xmmword [rsp+3*AES_BLOCK] + aesenclast xmm6, xmmword [rsp+4*AES_BLOCK] + aesenclast xmm7, xmmword [rsp+5*AES_BLOCK] + + LAST_INNER_MUL_X xmm15, xmm8, xmm9 + movdqa xmm8, xmm15 ; save tweak for next iteration + pxor xmm15, xmm0 ; add key[0] + + movdqu xmmword [rdi+0*AES_BLOCK], xmm2 ; store output blocks + movdqu xmmword [rdi+1*AES_BLOCK], xmm3 + movdqu xmmword [rdi+2*AES_BLOCK], xmm4 + movdqu xmmword [rdi+3*AES_BLOCK], xmm5 + movdqu xmmword [rdi+4*AES_BLOCK], xmm6 + movdqu xmmword [rdi+5*AES_BLOCK], xmm7 + add rdi, 6*AES_BLOCK + + sub rdx, 6 + jnc .blks_loop + +.short_input: + add rdx, 6 + jz .quit + + movdqa xmmword [rsp+0*AES_BLOCK], xmm10 ; save pre-computed twae + movdqa xmmword [rsp+1*AES_BLOCK], xmm11 + movdqa xmmword [rsp+2*AES_BLOCK], xmm12 + movdqa xmmword [rsp+3*AES_BLOCK], xmm13 + movdqa xmmword [rsp+4*AES_BLOCK], xmm14 + movdqa xmmword [rsp+5*AES_BLOCK], xmm15 + +;; +;; block-by-block processing +;; + movdqa xmm0, xmmword [rcx] ; key[0] + pxor xmm0, xmmword [rcx+r8] ; key[0] ^ key[last] + + lea r10, [rcx+r8-9*AES_BLOCK] ; r10 = &key[nr-9] + xor rax, rax + +.single_blk_loop: + movdqu xmm2, xmmword [rsi] ; input block + movdqa xmm1, xmmword [rsp+rax] ; tweak ^ key[0] + add rsi, AES_BLOCK + + pxor xmm2, xmm1 ; src[] ^tweak ^ key[0] + pxor xmm1, xmm0 ; tweak ^ key[lasl] + cmp r8, 12*16 ; switch according to number of rounds + jl .key_128_s + +.key_256_s: + aesenc xmm2, xmmword [r10-4*AES_BLOCK] + aesenc xmm2, xmmword [r10-3*AES_BLOCK] + aesenc xmm2, xmmword [r10-2*AES_BLOCK] + aesenc xmm2, xmmword [r10-1*AES_BLOCK] +.key_128_s: + aesenc xmm2, xmmword [r10+0*AES_BLOCK] + aesenc xmm2, xmmword [r10+1*AES_BLOCK] + aesenc xmm2, xmmword [r10+2*AES_BLOCK] + aesenc xmm2, xmmword [r10+3*AES_BLOCK] + aesenc xmm2, xmmword [r10+4*AES_BLOCK] + aesenc xmm2, xmmword [r10+5*AES_BLOCK] + aesenc xmm2, xmmword [r10+6*AES_BLOCK] + aesenc xmm2, xmmword [r10+7*AES_BLOCK] + aesenc xmm2, xmmword [r10+8*AES_BLOCK] + aesenclast xmm2, xmm1 + + movdqu xmmword [rdi], xmm2 ; output block + add rdi, AES_BLOCK + add rax, AES_BLOCK + sub rdx, 1 + jnz .single_blk_loop + + movdqa xmm10, xmmword [rsp+rax] ; tweak ^ key[0] + +.quit: + pxor xmm10, xmmword [rcx] ; remove key[0] + movdqu xmmword [r9], xmm10 ; and save tweak value + + pxor xmm1, xmm1 + + REST_XMM + REST_GPR + ret +ENDFUNC cpAESEncryptXTS_AES_NI + +%endif ;; _AES_NI_ENABLING_ +%endif ;;_IPP32E_Y8 + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128safedecu8as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128safedecu8as.asm new file mode 100644 index 0000000..3864a5f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128safedecu8as.asm @@ -0,0 +1,410 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael-128 (AES) cipher functions. +; (It's the special free from Sbox/tables implementation) +; +; Content: +; SafeDecrypt_RIJ128() +; +; History: +; +; Notes. +; The implementation is based on +; isomorphism between native GF(2^8) and composite GF((2^4)^2). +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "ia_32e_regs.inc" +%include "pcpvariant.inc" + +%if (_AES_NI_ENABLING_ == _FEATURE_OFF_) || (_AES_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_U8) + +%macro PTRANSFORM 6.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + %xdefine %%memTransLO %3 + %xdefine %%memTransHI %4 + %xdefine %%tmp %5 + %xdefine %%srcLO %6 + + movdqa %%dst, oword [rel %%memTransLO] ;; LO transformation + movdqa %%tmp, oword [rel %%memTransHI] ;; HI transformation + + movdqa %%srcLO, %%src ;; split src: + psrlw %%src, 4 ;; + pand %%srcLO, xmm7 ;; low 4 bits -> srcLO + pand %%src, xmm7 ;; upper 4 bits -> src + + pshufb %%dst, %%srcLO ;; transformation + pshufb %%tmp, %%src + pxor %%dst, %%tmp +%endmacro + +%macro PLOOKUP_MEM 3.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + %xdefine %%Table %3 + + movdqa %%dst, OWORD [rel %%Table] + pshufb %%dst,%%src +%endmacro + +%macro PREDUCE_MOD15 2.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + + movdqa %%dst, %%src + pcmpgtb %%src, xmm7 + psubb %%dst, %%src +%endmacro + +%macro PINVERSE_GF16_INV 6.nolist + %xdefine %%xmmB %1 + %xdefine %%xmmC %2 + %xdefine %%xmmP %3 + %xdefine %%xmmQ %4 + %xdefine %%xmmD %5 + %xdefine %%xmmT %6 + + PLOOKUP_MEM %%xmmT, %%xmmC, GF16_logTbl ;; xmmT = index_of(c) + pxor %%xmmC, %%xmmB + PLOOKUP_MEM %%xmmQ, %%xmmC, GF16_logTbl ;; xmmQ = index_of(b xor c) + + PLOOKUP_MEM %%xmmD, %%xmmB, GF16_sqr1 ;; xmmD = sqr(b)*beta^14 + PLOOKUP_MEM %%xmmP, %%xmmB, GF16_logTbl ;; xmmP = index_of(b) + + paddb %%xmmT, %%xmmQ ;; xmmT = index_of(c) + index_of(b xor c) + PREDUCE_MOD15 %%xmmC, %%xmmT ;; + PLOOKUP_MEM %%xmmT, %%xmmC, GF16_expTbl ;; c*(b xor c) + + pxor %%xmmD, %%xmmT ;; xmmD = delta = (c*(b xor c)) xor (sqr(b)*beta^14) + PLOOKUP_MEM %%xmmT, %%xmmD, GF16_invLog ;; xmmT = index_of( inv(delta) ) + + paddb %%xmmQ, %%xmmT ;; xmmQ = index_of((b xor c) * inv(delta)) + paddb %%xmmP, %%xmmT ;; xmmP = index_of(b * inv(delta)) + PREDUCE_MOD15 %%xmmT, %%xmmQ + PLOOKUP_MEM %%xmmC, %%xmmT, GF16_expTbl + PREDUCE_MOD15 %%xmmT, %%xmmP + PLOOKUP_MEM %%xmmB, %%xmmT, GF16_expTbl +%endmacro + +segment .text align=IPP_ALIGN_FACTOR + + +align IPP_ALIGN_FACTOR + +DECODE_DATA: + +;; (forward) native GF(2^8) to composite GF((2^4)^2) transformation : {0x01,0x2E,0x49,0x43,0x35,0xD0,0x3D,0xE9} +TransFwdLO \ + DB 000h ;; 000h ;; 0 + DB 001h ;; 001h ;; 1 + DB 02Eh ;; 02Eh ;; 2 + DB 02Fh ;; 02Eh XOR 001h ;; 3 + DB 049h ;; 049h ;; 4 + DB 048h ;; 049h XOR 001h ;; 5 + DB 067h ;; 049h XOR 02Eh ;; 6 + DB 066h ;; 049h XOR 02Eh XOR 001h ;; 7 + DB 043h ;; 043h ;; 8 + DB 042h ;; 043h XOR 001h ;; 9 + DB 06Dh ;; 043h XOR 02Eh ;; a + DB 06Ch ;; 043h XOR 02Eh XOR 001h ;; b + DB 00Ah ;; 043h XOR 049h ;; c + DB 00Bh ;; 043h XOR 049h XOR 001h ;; d + DB 024h ;; 043h XOR 049h XOR 02Eh ;; e + DB 025h ;; 043h XOR 049h XOR 02Eh XOR 001h ;; f +TransFwdHI \ + DB 000h ;; 000h ;; 0 + DB 035h ;; 035h ;; 1 + DB 0D0h ;; 0D0h ;; 2 + DB 0E5h ;; 0D0h XOR 035h ;; 3 + DB 03Dh ;; 03Dh ;; 4 + DB 008h ;; 03Dh XOR 035h ;; 5 + DB 0EDh ;; 03Dh XOR 0D0h ;; 6 + DB 0D8h ;; 03Dh XOR 0D0h XOR 035h ;; 7 + DB 0E9h ;; 0E9h ;; 8 + DB 0DCh ;; 0E9h XOR 035h ;; 9 + DB 039h ;; 0E9h XOR 0D0h ;; a + DB 00Ch ;; 0E9h XOR 0D0h XOR 035h ;; b + DB 0D4h ;; 0E9h XOR 03Dh ;; c + DB 0E1h ;; 0E9h XOR 03Dh XOR 035h ;; d + DB 004h ;; 0E9h XOR 03Dh XOR 0D0h ;; e + DB 031h ;; 0E9h XOR 03Dh XOR 0D0h XOR 035h ;; f + +;; (inverse) composite GF((2^4)^2) to native GF(2^8) transformation : {0x01,0x5C,0xE0,0x50,0x1F,0xEE,0x55,0x6A} +TransInvLO \ + DB 000h ;; 000h ;; 0 + DB 001h ;; 001h ;; 1 + DB 05Ch ;; 05Ch ;; 2 + DB 05Dh ;; 05Ch XOR 001h ;; 3 + DB 0E0h ;; 0E0h ;; 4 + DB 0E1h ;; 0E0h XOR 001h ;; 5 + DB 0BCh ;; 0E0h XOR 05Ch ;; 6 + DB 0BDh ;; 0E0h XOR 05Ch XOR 001h ;; 7 + DB 050h ;; 050h ;; 8 + DB 051h ;; 050h XOR 001h ;; 9 + DB 00Ch ;; 050h XOR 05Ch ;; a + DB 00Dh ;; 050h XOR 05Ch XOR 001h ;; b + DB 0B0h ;; 050h XOR 0E0h ;; c + DB 0B1h ;; 050h XOR 0E0h XOR 001h ;; d + DB 0ECh ;; 050h XOR 0E0h XOR 05Ch ;; e + DB 0EDh ;; 050h XOR 0E0h XOR 05Ch XOR 001h ;; f +TransInvHI \ + DB 000h ;; 000h ;; 0 + DB 01Fh ;; 01Fh ;; 1 + DB 0EEh ;; 0EEh ;; 2 + DB 0F1h ;; 0EEh XOR 01Fh ;; 3 + DB 055h ;; 055h ;; 4 + DB 04Ah ;; 055h XOR 01Fh ;; 5 + DB 0BBh ;; 055h XOR 0EEh ;; 6 + DB 0A4h ;; 055h XOR 0EEh XOR 01Fh ;; 7 + DB 06Ah ;; 06Ah ;; 8 + DB 075h ;; 06Ah XOR 01Fh ;; 9 + DB 084h ;; 06Ah XOR 0EEh ;; a + DB 09Bh ;; 06Ah XOR 0EEh XOR 01Fh ;; b + DB 03Fh ;; 06Ah XOR 055h ;; c + DB 020h ;; 06Ah XOR 055h XOR 01Fh ;; d + DB 0D1h ;; 06Ah XOR 055h XOR 0EEh ;; e + DB 0CEh ;; 06Ah XOR 055h XOR 0EEh XOR 01Fh ;; f + + +GF16_csize DB 00Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh + +;; GF16 elements: +;; 0 1 2 3 4 5 6 7 8 9 A B C D E F +GF16_logTbl \ + DB 0C0h,00h,01h,04h,02h,08h,05h,0Ah,03h,0Eh,09h,07h,06h,0Dh,0Bh,0Ch +GF16_expTbl \ + DB 001h,02h,04h,08h,03h,06h,0Ch,0Bh,05h,0Ah,07h,0Eh,0Fh,0Dh,09h,01h +GF16_sqr1 \ + DB 000h,09h,02h,0Bh,08h,01h,0Ah,03h,06h,0Fh,04h,0Dh,0Eh,07h,0Ch,05h ;; sqr(GF16_element) * beta^14 +GF16_invLog \ + DB 0C0h,00h,0Eh,0Bh,0Dh,07h,0Ah,05h,0Ch,01h,06h,08h,09h,02h,04h,03h + +;; affine transformation matrix (inverse cipher) : {0x50,0x36,0x15,0x82,0x01,0x34,0x40,0x3E} +InvAffineLO \ + DB 000h ;; 000h ;; 0 + DB 050h ;; 050h ;; 1 + DB 036h ;; 036h ;; 2 + DB 066h ;; 036h XOR 050h ;; 3 + DB 015h ;; 015h ;; 4 + DB 045h ;; 015h XOR 050h ;; 5 + DB 023h ;; 015h XOR 036h ;; 6 + DB 073h ;; 015h XOR 036h XOR 050h ;; 7 + DB 082h ;; 082h ;; 8 + DB 0D2h ;; 082h XOR 050h ;; 9 + DB 0B4h ;; 082h XOR 036h ;; a + DB 0E4h ;; 082h XOR 036h XOR 050h ;; b + DB 097h ;; 082h XOR 015h ;; c + DB 0C7h ;; 082h XOR 015h XOR 050h ;; d + DB 0A1h ;; 082h XOR 015h XOR 036h ;; e + DB 0F1h ;; 082h XOR 015h XOR 036h XOR 050h ;; f +InvAffineHI \ + DB 000h ;; 000h ;; 0 + DB 001h ;; 001h ;; 1 + DB 034h ;; 034h ;; 2 + DB 035h ;; 034h XOR 001h ;; 3 + DB 040h ;; 040h ;; 4 + DB 041h ;; 040h XOR 001h ;; 5 + DB 074h ;; 040h XOR 034h ;; 6 + DB 075h ;; 040h XOR 034h XOR 001h ;; 7 + DB 03Eh ;; 03Eh ;; 8 + DB 03Fh ;; 03Eh XOR 001h ;; 9 + DB 00Ah ;; 03Eh XOR 034h ;; a + DB 00Bh ;; 03Eh XOR 034h XOR 001h ;; b + DB 07Eh ;; 03Eh XOR 040h ;; c + DB 07Fh ;; 03Eh XOR 040h XOR 001h ;; d + DB 04Ah ;; 03Eh XOR 040h XOR 034h ;; e + DB 04Bh ;; 03Eh XOR 040h XOR 034h XOR 001h ;; f + +;; affine transformation constant (inverse cipher) +InvAffineCnt \ + DQ 04848484848484848h,04848484848484848h + +;; shift rows transformation (inverse cipher) +InvShiftRows \ + DB 0,13,10,7,4,1,14,11,8,5,2,15,12,9,6,3 + +;; mix columns transformation (inverse cipher) +GF16mul_4_2x \ + DB 000h,024h,048h,06Ch,083h,0A7h,0CBh,0EFh,036h,012h,07Eh,05Ah,0B5h,091h,0FDh,0D9h ;; *(4+2x) +GF16mul_1_6x \ + DB 000h,061h,0C2h,0A3h,0B4h,0D5h,076h,017h,058h,039h,09Ah,0FBh,0ECh,08Dh,02Eh,04Fh ;; *(1+6x) + +GF16mul_C_6x \ + DB 000h,06Ch,0CBh,0A7h,0B5h,0D9h,07Eh,012h,05Ah,036h,091h,0FDh,0EFh,083h,024h,048h ;; *(C+6x) +GF16mul_3_Ax \ + DB 000h,0A3h,076h,0D5h,0ECh,04Fh,09Ah,039h,0FBh,058h,08Dh,02Eh,017h,0B4h,061h,0C2h ;; *(3+Ax) + +GF16mul_B_0x \ + DB 000h,00Bh,005h,00Eh,00Ah,001h,00Fh,004h,007h,00Ch,002h,009h,00Dh,006h,008h,003h ;; *(B+0x) +GF16mul_0_Bx \ + DB 000h,0B0h,050h,0E0h,0A0h,010h,0F0h,040h,070h,0C0h,020h,090h,0D0h,060h,080h,030h ;; *(0+Bx) + +GF16mul_2_4x \ + DB 000h,042h,084h,0C6h,038h,07Ah,0BCh,0FEh,063h,021h,0E7h,0A5h,05Bh,019h,0DFh,09Dh ;; *(2+4x) +GF16mul_2_6x \ + DB 000h,062h,0C4h,0A6h,0B8h,0DAh,07Ch,01Eh,053h,031h,097h,0F5h,0EBh,089h,02Fh,04Dh ;; *(2+6x) + +ColumnROR \ + DB 1,2,3,0,5,6,7,4,9,10,11,8,13,14,15,12 + + +align IPP_ALIGN_FACTOR +;************************************************************* +;* void SafeDecrypt_RIJ128(const Ipp8u* pInpBlk, +;* Ipp8u* pOutBlk, +;* int nr, +;* const Ipp8u* pKeys, +;* const void* Tables) +;************************************************************* + +;; +;; Lib = U8 +;; +IPPASM SafeDecrypt_RIJ128,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi + USES_XMM xmm6,xmm7 + COMP_ABI 5 +;; rdi: pInpBlk: DWORD, ; input block address +;; rsi: pOutBlk: DWORD, ; output block address +;; rdx: nr: DWORD, ; number of rounds +;; rcx: pKey: DWORD ; key material address + +%assign RSIZE sizeof(dword) ; size of row +%assign SC 4 ; columns in STATE +%assign SSIZE RSIZE*SC ; size of state + + lea rax,[rdx*4] + lea rcx,[rcx+rax*4] ; AES-128-keys + + movdqu xmm0, oword [rdi] ; input block + + movdqa xmm7, oword [rel GF16_csize] + + ;; convert input into the composite GF((2^4)^2) + PTRANSFORM xmm2, xmm0, TransFwdLO,TransFwdHI, xmm1, xmm3 + + ;; initial whitening + pxor xmm2, oword [rcx] + sub rcx, SSIZE + + ;; (nr-1) regular rounds + sub rdx,1 + +.decode_round: + ;; InvSubByte() Transformation: + + ;; affine transformation + PTRANSFORM xmm0, xmm2, InvAffineLO,InvAffineHI, xmm1, xmm3 + pxor xmm0, oword [rel InvAffineCnt] ; H(c), c=0x05 + + ;; split input by low and upper parts + movdqa xmm1, xmm0 + pand xmm0, xmm7 ; low parts (4 bits) + psrlw xmm1, 4 + pand xmm1, xmm7 ; upper parts (4 bits) + + ;; compute multiplicative inverse + PINVERSE_GF16_INV xmm1,xmm0, xmm3,xmm2,xmm4,xmm5 + + ;; InvShiftRows() Transformation: + pshufb xmm0, [rel InvShiftRows] + pshufb xmm1, [rel InvShiftRows] + + ;; InvMixColumn() Transformation: + PLOOKUP_MEM xmm2, xmm0, GF16mul_4_2x ; mul H(0xE) = 0x24 + pshufb xmm0, [rel ColumnROR] + PLOOKUP_MEM xmm3, xmm1, GF16mul_1_6x + pshufb xmm1, [rel ColumnROR] + pxor xmm2, xmm3 + + PLOOKUP_MEM xmm3, xmm0, GF16mul_C_6x ; mul H(0xB) = 0x6C + pshufb xmm0, [rel ColumnROR] + pxor xmm2, xmm3 + PLOOKUP_MEM xmm3, xmm1, GF16mul_3_Ax + pshufb xmm1, [rel ColumnROR] + pxor xmm2, xmm3 + + PLOOKUP_MEM xmm3, xmm0, GF16mul_B_0x ; mul H(0xD) = 0x0B + pshufb xmm0, [rel ColumnROR] + pxor xmm2, xmm3 + PLOOKUP_MEM xmm3, xmm1, GF16mul_0_Bx + pshufb xmm1, [rel ColumnROR] + pxor xmm2, xmm3 + + PLOOKUP_MEM xmm3, xmm0, GF16mul_2_4x ; mul H(0x9) = 0x42 + pxor xmm2, xmm3 + PLOOKUP_MEM xmm3, xmm1, GF16mul_2_6x + pxor xmm2, xmm3 + + ;; AddRoundKey() Transformation: + pxor xmm2, oword [rcx] + sub rcx, SSIZE + + sub rdx,1 + jg .decode_round + + + ;; + ;; the last one is irregular + ;; + + ;; InvSubByte() Transformation + + ;; affine transformation + PTRANSFORM xmm0, xmm2, InvAffineLO,InvAffineHI, xmm1, xmm3 + pxor xmm0, oword [rel InvAffineCnt] ; H(c), c=0x05 + + ;; split input by low and upper parts + movdqa xmm1, xmm0 + pand xmm0, xmm7 ; low parts (4 bits) + psrlw xmm1, 4 + pand xmm1, xmm7 ; upper parts (4 bits) + + ;; compute multiplicative inverse + PINVERSE_GF16_INV xmm1,xmm0, xmm3,xmm2,xmm4,xmm5 + + ;; InvShiftRows() Transformation: + psllw xmm1, 4 + por xmm1, xmm0 + pshufb xmm1, [rel InvShiftRows] + + ;; AddRoundKey() Transformation: + pxor xmm1, oword [rcx] + + ;; convert output into the native GF(2^8) + PTRANSFORM xmm0,xmm1, TransInvLO,TransInvHI, xmm2, xmm3 + + movdqu oword [rsi], xmm0 + + REST_XMM + REST_GPR + ret +ENDFUNC SafeDecrypt_RIJ128 + +%endif + +%endif ;; _AES_NI_ENABLING_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128safeencu8as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128safeencu8as.asm new file mode 100644 index 0000000..0de8597 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprij128safeencu8as.asm @@ -0,0 +1,426 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael-128 (AES) cipher functions. +; (It's the special free from Sbox/tables implementation) +; +; Content: +; SafeEncrypt_RIJ128() +; +; History: +; +; Notes. +; The implementation is based on +; isomorphism between native GF(2^8) and composite GF((2^4)^2). +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "ia_32e_regs.inc" +%include "pcpvariant.inc" + +%if (_AES_NI_ENABLING_ == _FEATURE_OFF_) || (_AES_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_U8) + +%macro PTRANSFORM 6.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + %xdefine %%memTransLO %3 + %xdefine %%memTransHI %4 + %xdefine %%tmp %5 + %xdefine %%srcLO %6 + + movdqa %%dst, oword [rel %%memTransLO] ;; LO transformation + movdqa %%tmp, oword [rel %%memTransHI] ;; HI transformation + + movdqa %%srcLO, %%src ;; split src: + psrlw %%src, 4 ;; + pand %%srcLO, xmm7 ;; low 4 bits -> srcLO + pand %%src, xmm7 ;; upper 4 bits -> src + + pshufb %%dst, %%srcLO ;; transformation + pshufb %%tmp, %%src + pxor %%dst, %%tmp +%endmacro + +%macro PLOOKUP_MEM 3.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + %xdefine %%Table %3 + + movdqa %%dst, OWORD [rel %%Table] + pshufb %%dst,%%src +%endmacro + +%macro PREDUCE_MOD15 2.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + + movdqa %%dst, %%src + pcmpgtb %%src, xmm7 + psubb %%dst, %%src +%endmacro + +%macro PINVERSE_GF16_FWD 6.nolist + %xdefine %%xmmB %1 + %xdefine %%xmmC %2 + %xdefine %%xmmP %3 + %xdefine %%xmmQ %4 + %xdefine %%xmmD %5 + %xdefine %%xmmT %6 + + PLOOKUP_MEM %%xmmT, %%xmmC, GF16_logTbl ;; xmmT = index_of(c) + pxor %%xmmC, %%xmmB + PLOOKUP_MEM %%xmmQ, %%xmmC, GF16_logTbl ;; xmmQ = index_of(b xor c) + + PLOOKUP_MEM %%xmmD, %%xmmB, GF16_sqr1 ;; xmmD = sqr(b)*beta^14 + PLOOKUP_MEM %%xmmP, %%xmmB, GF16_logTbl ;; xmmP = index_of(b) + + paddb %%xmmT, %%xmmQ ;; xmmT = index_of(c) + index_of(b xor c) + PREDUCE_MOD15 %%xmmC, %%xmmT ;; + PLOOKUP_MEM %%xmmT, %%xmmC, GF16_expTbl ;; c*(b xor c) + + pxor %%xmmD, %%xmmT ;; xmmD = delta = (c*(b xor c)) xor (sqr(b)*beta^14) + PLOOKUP_MEM %%xmmT, %%xmmD, GF16_invLog ;; xmmT = index_of( inv(delta) ) + + paddb %%xmmQ, %%xmmT ;; xmmQ = index_of((b xor c) * inv(delta)) + paddb %%xmmP, %%xmmT ;; xmmP = index_of(b * inv(delta)) + PREDUCE_MOD15 %%xmmT, %%xmmQ + PLOOKUP_MEM %%xmmC, %%xmmT, GF16_expTbl + PREDUCE_MOD15 %%xmmT, %%xmmP + PLOOKUP_MEM %%xmmB, %%xmmT, GF16_expTbl +%endmacro + +segment .text align=IPP_ALIGN_FACTOR + + +align IPP_ALIGN_FACTOR + +ENCODE_DATA: + +;; (forward) native GF(2^8) to composite GF((2^4)^2) transformation : {0x01,0x2E,0x49,0x43,0x35,0xD0,0x3D,0xE9} +TransFwdLO \ + DB 000h ;; 000h ;; 0 + DB 001h ;; 001h ;; 1 + DB 02Eh ;; 02Eh ;; 2 + DB 02Fh ;; 02Eh XOR 001h ;; 3 + DB 049h ;; 049h ;; 4 + DB 048h ;; 049h XOR 001h ;; 5 + DB 067h ;; 049h XOR 02Eh ;; 6 + DB 066h ;; 049h XOR 02Eh XOR 001h ;; 7 + DB 043h ;; 043h ;; 8 + DB 042h ;; 043h XOR 001h ;; 9 + DB 06Dh ;; 043h XOR 02Eh ;; a + DB 06Ch ;; 043h XOR 02Eh XOR 001h ;; b + DB 00Ah ;; 043h XOR 049h ;; c + DB 00Bh ;; 043h XOR 049h XOR 001h ;; d + DB 024h ;; 043h XOR 049h XOR 02Eh ;; e + DB 025h ;; 043h XOR 049h XOR 02Eh XOR 001h ;; f +TransFwdHI \ + DB 000h ;; 000h ;; 0 + DB 035h ;; 035h ;; 1 + DB 0D0h ;; 0D0h ;; 2 + DB 0E5h ;; 0D0h XOR 035h ;; 3 + DB 03Dh ;; 03Dh ;; 4 + DB 008h ;; 03Dh XOR 035h ;; 5 + DB 0EDh ;; 03Dh XOR 0D0h ;; 6 + DB 0D8h ;; 03Dh XOR 0D0h XOR 035h ;; 7 + DB 0E9h ;; 0E9h ;; 8 + DB 0DCh ;; 0E9h XOR 035h ;; 9 + DB 039h ;; 0E9h XOR 0D0h ;; a + DB 00Ch ;; 0E9h XOR 0D0h XOR 035h ;; b + DB 0D4h ;; 0E9h XOR 03Dh ;; c + DB 0E1h ;; 0E9h XOR 03Dh XOR 035h ;; d + DB 004h ;; 0E9h XOR 03Dh XOR 0D0h ;; e + DB 031h ;; 0E9h XOR 03Dh XOR 0D0h XOR 035h ;; f + +;; (inverse) composite GF((2^4)^2) to native GF(2^8) transformation : {0x01,0x5C,0xE0,0x50,0x1F,0xEE,0x55,0x6A} +TransInvLO \ + DB 000h ;; 000h ;; 0 + DB 001h ;; 001h ;; 1 + DB 05Ch ;; 05Ch ;; 2 + DB 05Dh ;; 05Ch XOR 001h ;; 3 + DB 0E0h ;; 0E0h ;; 4 + DB 0E1h ;; 0E0h XOR 001h ;; 5 + DB 0BCh ;; 0E0h XOR 05Ch ;; 6 + DB 0BDh ;; 0E0h XOR 05Ch XOR 001h ;; 7 + DB 050h ;; 050h ;; 8 + DB 051h ;; 050h XOR 001h ;; 9 + DB 00Ch ;; 050h XOR 05Ch ;; a + DB 00Dh ;; 050h XOR 05Ch XOR 001h ;; b + DB 0B0h ;; 050h XOR 0E0h ;; c + DB 0B1h ;; 050h XOR 0E0h XOR 001h ;; d + DB 0ECh ;; 050h XOR 0E0h XOR 05Ch ;; e + DB 0EDh ;; 050h XOR 0E0h XOR 05Ch XOR 001h ;; f +TransInvHI \ + DB 000h ;; 000h ;; 0 + DB 01Fh ;; 01Fh ;; 1 + DB 0EEh ;; 0EEh ;; 2 + DB 0F1h ;; 0EEh XOR 01Fh ;; 3 + DB 055h ;; 055h ;; 4 + DB 04Ah ;; 055h XOR 01Fh ;; 5 + DB 0BBh ;; 055h XOR 0EEh ;; 6 + DB 0A4h ;; 055h XOR 0EEh XOR 01Fh ;; 7 + DB 06Ah ;; 06Ah ;; 8 + DB 075h ;; 06Ah XOR 01Fh ;; 9 + DB 084h ;; 06Ah XOR 0EEh ;; a + DB 09Bh ;; 06Ah XOR 0EEh XOR 01Fh ;; b + DB 03Fh ;; 06Ah XOR 055h ;; c + DB 020h ;; 06Ah XOR 055h XOR 01Fh ;; d + DB 0D1h ;; 06Ah XOR 055h XOR 0EEh ;; e + DB 0CEh ;; 06Ah XOR 055h XOR 0EEh XOR 01Fh ;; f + + +GF16_csize DB 00Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh,0Fh + +;; GF16 elements: +;; 0 1 2 3 4 5 6 7 8 9 A B C D E F +GF16_logTbl \ + DB 0C0h,00h,01h,04h,02h,08h,05h,0Ah,03h,0Eh,09h,07h,06h,0Dh,0Bh,0Ch +GF16_expTbl \ + DB 001h,02h,04h,08h,03h,06h,0Ch,0Bh,05h,0Ah,07h,0Eh,0Fh,0Dh,09h,01h +GF16_sqr1 \ + DB 000h,09h,02h,0Bh,08h,01h,0Ah,03h,06h,0Fh,04h,0Dh,0Eh,07h,0Ch,05h ;; sqr(GF16_element) * beta^14 +GF16_invLog \ + DB 0C0h,00h,0Eh,0Bh,0Dh,07h,0Ah,05h,0Ch,01h,06h,08h,09h,02h,04h,03h +GF16_expTbl_shift \ + DB 010h,020h,040h,080h,030h,060h,0C0h,0B0h,050h,0A0h,070h,0E0h,0F0h,0D0h,090h,010h + +;; affine transformation matrix (forward cipher) : {0x10,0x22,0x55,0x82,0x41,0x34,0x40,0x2A} +FwdAffineLO \ + DB 000h ;; 000h ;; 0 + DB 010h ;; 010h ;; 1 + DB 022h ;; 022h ;; 2 + DB 032h ;; 022h XOR 010h ;; 3 + DB 055h ;; 055h ;; 4 + DB 045h ;; 055h XOR 010h ;; 5 + DB 077h ;; 055h XOR 022h ;; 6 + DB 067h ;; 055h XOR 022h XOR 010h ;; 7 + DB 082h ;; 082h ;; 8 + DB 092h ;; 082h XOR 010h ;; 9 + DB 0A0h ;; 082h XOR 022h ;; a + DB 0B0h ;; 082h XOR 022h XOR 010h ;; b + DB 0D7h ;; 082h XOR 055h ;; c + DB 0C7h ;; 082h XOR 055h XOR 010h ;; d + DB 0F5h ;; 082h XOR 055h XOR 022h ;; e + DB 0E5h ;; 082h XOR 055h XOR 022h XOR 010h ;; f +FwdAffineHI \ + DB 000h ;; 000h ;; 0 + DB 041h ;; 041h ;; 1 + DB 034h ;; 034h ;; 2 + DB 075h ;; 034h XOR 041h ;; 3 + DB 040h ;; 040h ;; 4 + DB 001h ;; 040h XOR 041h ;; 5 + DB 074h ;; 040h XOR 034h ;; 6 + DB 035h ;; 040h XOR 034h XOR 041h ;; 7 + DB 02Ah ;; 02Ah ;; 8 + DB 06Bh ;; 02Ah XOR 041h ;; 9 + DB 01Eh ;; 02Ah XOR 034h ;; a + DB 05Fh ;; 02Ah XOR 034h XOR 041h ;; b + DB 06Ah ;; 02Ah XOR 040h ;; c + DB 02Bh ;; 02Ah XOR 040h XOR 041h ;; d + DB 05Eh ;; 02Ah XOR 040h XOR 034h ;; e + DB 01Fh ;; 02Ah XOR 040h XOR 034h XOR 041h ;; f + +;; affine transformation constant (forward cipher) +FwdAffineCnt \ + DQ 0C2C2C2C2C2C2C2C2h,0C2C2C2C2C2C2C2C2h + +;; shift rows transformation (forward cipher) +FwdShiftRows \ + DB 0,5,10,15,4,9,14,3,8,13,2,7,12,1,6,11 + +;; mix columns transformation (forward cipher) +GF16mul_E_2x \ + DB 000h,02Eh,04Fh,061h,08Dh,0A3h,0C2h,0ECh,039h,017h,076h,058h,0B4h,09Ah,0FBh,0D5h +GF16mul_1_Cx \ + DB 000h,0C1h,0B2h,073h,054h,095h,0E6h,027h,0A8h,069h,01Ah,0DBh,0FCh,03Dh,04Eh,08Fh + +ColumnROR \ + DB 1,2,3,0,5,6,7,4,9,10,11,8,13,14,15,12 + + +;************************************************************* +; convert GF(2^128) -> GF((2^4)^2) +;************************************************************* +align IPP_ALIGN_FACTOR +IPPASM TransformNative2Composite,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi + USES_XMM xmm7 + COMP_ABI 2 +;; rdi: pOutBlk: DWORD, ; output block address +;; rsi: pInpBlk: DWORD, ; input block address + + movdqu xmm0, oword [rsi] ; input block + movdqa xmm7, oword [rel GF16_csize] + + ;; convert input into the composite GF((2^4)^2) + PTRANSFORM xmm1, xmm0, TransFwdLO,TransFwdHI, xmm2, xmm3 + + movdqu oword [rdi], xmm1 ; output block + + REST_XMM + REST_GPR + ret +ENDFUNC TransformNative2Composite + +;************************************************************* +;* void SafeEncrypt_RIJ128(const Ipp8u* pInpBlk, +;* Ipp8u* pOutBlk, +;* int nr, +;* const Ipp8u* pKeys, +;* const void* Tables) +;************************************************************* +align IPP_ALIGN_FACTOR +;; +;; Lib = U8 +;; +IPPASM SafeEncrypt_RIJ128,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi + USES_XMM xmm6,xmm7 + COMP_ABI 5 +;; rdi: pInpBlk: DWORD, ; input block address +;; rsi: pOutBlk: DWORD, ; output block address +;; rdx: nr: DWORD, ; number of rounds +;; rcx: pKey: DWORD ; key material address + +%assign RSIZE sizeof(dword) ; size of row +%assign SC 4 ; columns in STATE +%assign SSIZE RSIZE*SC ; size of state + + movdqu xmm1, oword [rdi] ; input block + + movdqa xmm7, oword [rel GF16_csize] + + ;; convert input into the composite GF((2^4)^2) + PTRANSFORM xmm0, xmm1, TransFwdLO,TransFwdHI, xmm2, xmm3 + + ;; initial whitening + pxor xmm0, oword [rcx] + add rcx, SSIZE + + ;; (nr-1) regular rounds + sub rdx,1 + +.encode_round: + ;; SubByte() Transformation: + + ;; split input by low and upper parts + movdqa xmm1, xmm0 + pand xmm0, xmm7 ; low parts (4 bits) + psrlw xmm1, 4 + pand xmm1, xmm7 ; upper parts (4 bits) + + ;; compute multiplicative inverse + PINVERSE_GF16_FWD xmm1,xmm0, xmm3,xmm2,xmm4,xmm5 + + ;; affine transformation + movdqa xmm3, oword [rel FwdAffineLO] + movdqa xmm2, oword [rel FwdAffineHI] + movdqa xmm4, oword [rel FwdAffineCnt] ; H(c), c=0x63 + pshufb xmm3, xmm0 + pshufb xmm2, xmm1 + pxor xmm3, xmm4 + pxor xmm3, xmm2 + + ;; ShiftRows() Transformation: + pshufb xmm3, [rel FwdShiftRows] + + ;; MixColumn() Transformation: + movdqa xmm1, xmm3 + movdqa xmm2, xmm3 + pxor xmm4, xmm4 + psrlw xmm2, 4 + + pand xmm1, xmm7 ;; a0*(0xE + 0x2*t) + PLOOKUP_MEM xmm0, xmm1, GF16mul_E_2x + + pand xmm2, xmm7 ;; a1*(0x2*0x9 + (0x2+0xE)*t) + PLOOKUP_MEM xmm1, xmm2, GF16mul_1_Cx + + pxor xmm0, xmm1 + + pshufb xmm3, [rel ColumnROR] + pxor xmm4, xmm3 + + pshufb xmm3, [rel ColumnROR] + pxor xmm4, xmm3 + + movdqa xmm2, xmm0 + pshufb xmm2, [rel ColumnROR] + pxor xmm0, xmm2 + + pshufb xmm3, [rel ColumnROR] + pxor xmm4, xmm3 + + pxor xmm0, xmm4 + + ;; AddRoundKey() Transformation: + pxor xmm0, oword [rcx] + add rcx, SSIZE + + sub rdx,1 + jg .encode_round + + + ;; + ;; the last one is irregular + ;; + + ;; SubByte() Transformation: + movdqa xmm1, xmm0 + pand xmm0, xmm7 ; low parts (4 bits) + psrlw xmm1, 4 + pand xmm1, xmm7 ; upper parts (4 bits) + + ;; compute multiplicative inverse + PINVERSE_GF16_FWD xmm1,xmm0, xmm3,xmm2,xmm4,xmm5 + + ;; affine transformation + movdqa xmm3, oword [rel FwdAffineLO] + movdqa xmm2, oword [rel FwdAffineHI] + movdqa xmm4, oword [rel FwdAffineCnt] ; H(c), c=0x63 + pshufb xmm3, xmm0 + pshufb xmm2, xmm1 + pxor xmm3, xmm4 + pxor xmm3, xmm2 + + ;; ShiftRows() Transformation: + pshufb xmm3, [rel FwdShiftRows] + + ;; AddRoundKey() Transformation: + pxor xmm3, oword [rcx] + + ;; convert output into the native GF(2^8) + PTRANSFORM xmm0,xmm3, TransInvLO,TransInvHI, xmm2,xmm1 + + movdqu oword [rsi], xmm0 + + REST_XMM + REST_GPR + ret +ENDFUNC SafeEncrypt_RIJ128 + +%endif + +%endif ;; _AES_NI_ENABLING_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprijnkeym7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprijnkeym7as.asm new file mode 100644 index 0000000..d37a86f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcprijnkeym7as.asm @@ -0,0 +1,91 @@ +;=============================================================================== +; Copyright (C) 2014 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Rijndael Key Expansion Support +; +; Content: +; SubsDword_8uT() +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_M7) + +segment .text align=IPP_ALIGN_FACTOR + +%xdefine CACHE_LINE_SIZE (64) + +;*************************************************************** +;* Purpose: Mitigation of the Key Expansion procedure +;* +;* Ipp32u Touch_SubsDword_8uT(Ipp32u inp, +;* const Ipp8u* pTbl, +;* int tblBytes) +;*************************************************************** +align IPP_ALIGN_FACTOR +IPPASM Touch_SubsDword_8uT,PUBLIC + USES_GPR rsi,rdi + USES_XMM + COMP_ABI 3 +;; rdi: inp: DWORD, ; input dword +;; rsi: pTbl: BYTE, ; Rijndael's S-box +;; edx tblLen: DWORD ; length of table (bytes) + + movsxd r8, edx ; length + xor rcx, rcx +.touch_tbl: + mov rax, [rsi+rcx] + add rcx, CACHE_LINE_SIZE + cmp rcx, r8 + jl .touch_tbl + + mov rax, rdi + and rax, 0FFh ; b[0] + movzx rax, BYTE [rsi+rax] + + shr rdi, 8 + mov r9, rdi + and r9, 0FFh ; b[1] + movzx r9, BYTE [rsi+r9] + shl r9, 8 + + shr rdi, 8 + mov rcx, rdi + and rcx, 0FFh ; b[2] + movzx rcx, BYTE [rsi+rcx] + shl rcx, 16 + + shr rdi, 8 + mov rdx, rdi + and rdx, 0FFh ; b[3] + movzx rdx, BYTE [rsi+rdx] + shl rdx, 24 + + or rax, r9 + or rax, rcx + or rax, rdx + REST_XMM + REST_GPR + ret +ENDFUNC Touch_SubsDword_8uT + +%endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha1e9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha1e9as.asm new file mode 100644 index 0000000..ca71d33 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha1e9as.asm @@ -0,0 +1,510 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SHA1 +; +; Content: +; UpdateSHA1 +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SHA1_) +%if (_SHA_NI_ENABLING_ == _FEATURE_OFF_) || (_SHA_NI_ENABLING_ == _FEATURE_TICKTOCK_) +;;%if (_IPP32E >= _IPP32E_E9 ) +%if (_IPP32E == _IPP32E_E9 ) + +;; +;; SHA1 constants K[i] +%xdefine SHA1_K1 (05a827999h) +%xdefine SHA1_K2 (06ed9eba1h) +%xdefine SHA1_K3 (08f1bbcdch) +%xdefine SHA1_K4 (0ca62c1d6h) + +;; +;; Magic functions defined in FIPS 180-1 +;; +;; F1, F2, F3 and F4 assumes, that +;; - T1 returns function value +;; - T2 is the temporary +;; + +%macro F1 3.nolist + %xdefine %%B %1 + %xdefine %%C %2 + %xdefine %%D %3 + + mov T1,%%C + xor T1,%%D + and T1,%%B + xor T1,%%D +%endmacro + +%macro F2 3.nolist + %xdefine %%B %1 + %xdefine %%C %2 + %xdefine %%D %3 + + mov T1,%%D + xor T1,%%C + xor T1,%%B +%endmacro + +%macro F3 3.nolist + %xdefine %%B %1 + %xdefine %%C %2 + %xdefine %%D %3 + + mov T1,%%C + mov T2,%%B + or T1,%%B + and T2,%%C + and T1,%%D + or T1,T2 +%endmacro + +%macro F4 3.nolist + %xdefine %%B %1 + %xdefine %%C %2 + %xdefine %%D %3 + + F2 %%B,%%C,%%D +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; +;; rotations +;; + +%macro ROL_5 1.nolist + %xdefine %%x %1 + + shld %%x,%%x, 5 +%endmacro + +%macro ROL_30 1.nolist + %xdefine %%x %1 + + shld %%x,%%x, 30 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; +;; textual rotation of W array +;; +%macro ROTATE_W 0.nolist + %xdefine W_minus_32 W_minus_28 + %xdefine W_minus_28 W_minus_24 + %xdefine W_minus_24 W_minus_20 + %xdefine W_minus_20 W_minus_16 + %xdefine W_minus_16 W_minus_12 + %xdefine W_minus_12 W_minus_08 + %xdefine W_minus_08 W_minus_04 + %xdefine W_minus_04 W + %xdefine W W_minus_32 +%endmacro + +;; +;; SHA1 update round: +;; - F1 magic is used (and imbedded into the macros directly) +;; - 16 bytes of input are swapped +;; +%macro SHA1_UPDATE_RND_F1_BSWAP 7.nolist + %xdefine %%A %1 + %xdefine %%B %2 + %xdefine %%C %3 + %xdefine %%D %4 + %xdefine %%E %5 + %xdefine %%nr %6 + %xdefine %%Wchunk %7 + + vpshufb W, %%Wchunk, XMM_SHUFB_BSWAP + vpaddd %%Wchunk, W, oword [K_XMM] + vmovdqa oword [rsp + (%%nr & 15)*4], %%Wchunk + mov T1,%%C ; F1 + mov T2,%%A + xor T1,%%D ; F1 + and T1,%%B ; F1 + ROL_5 T2 + xor T1,%%D ; F1 + add %%E, T2 + ROL_30 %%B + add T1, dword [rsp + (%%nr & 15)*4] + add %%E,T1 + + ROTATE_W +%endmacro + +;; +;; SHA1 update round: +;; - F1 magic is used (and imbedded into the macros directly) +;; +%macro SHA1_UPDATE_RND_F1 6.nolist + %xdefine %%A %1 + %xdefine %%B %2 + %xdefine %%C %3 + %xdefine %%D %4 + %xdefine %%E %5 + %xdefine %%nr %6 + + mov T1,%%C ; F1 + mov T2,%%A + xor T1,%%D ; F1 + ROL_5 T2 + and T1,%%B ; F1 + xor T1,%%D ; F1 + add %%E, T2 + ROL_30 %%B + add T1, dword [rsp + (%%nr & 15)*4] + add %%E,T1 +%endmacro + +;; +;; update W +;; +%macro W_CALC 1.nolist + %xdefine %%nr %1 + + %assign %%W_CALC_ahead 8 + + %assign %%i (%%nr + %%W_CALC_ahead) + + %if (%%i < 20) + %xdefine K_XMM K_BASE + %elif (%%i < 40) + %xdefine K_XMM K_BASE+16 + %elif (%%i < 60) + %xdefine K_XMM K_BASE+32 + %else + %xdefine K_XMM K_BASE+48 + %endif + + %if (%%i < 32) + %if ((%%i & 3) == 0) ;; just scheduling to interleave with ALUs + vpalignr W, W_minus_12, W_minus_16, 8 ; w[i-14] + vpsrldq W_TMP, W_minus_04, 4 ; w[i-3] + vpxor W, W, W_minus_08 + %elif ((%%i & 3) == 1) + vpxor W_TMP, W_TMP, W_minus_16 + vpxor W, W, W_TMP + vpslldq W_TMP2, W, 12 + %elif ((%%i & 3) == 2) + vpslld W_TMP, W, 1 + vpsrld W, W, 31 + vpor W_TMP, W_TMP, W + vpslld W, W_TMP2, 2 + vpsrld W_TMP2, W_TMP2, 30 + %elif ((%%i & 3) == 3) + vpxor W_TMP, W_TMP, W + vpxor W, W_TMP, W_TMP2 + vpaddd W_TMP, W, oword [K_XMM] + vmovdqa oword [rsp + ((%%i & (~3)) & 15)*4],W_TMP + + ROTATE_W + %endif + +;; %elif (i < 83) + %elif (%%i < 80) + %if ((%%i & 3) == 0) ;; scheduling to interleave with ALUs + vpalignr W_TMP, W_minus_04, W_minus_08, 8 + vpxor W, W, W_minus_28 ;; W == W_minus_32 + %elif ((%%i & 3) == 1) + vpxor W_TMP, W_TMP, W_minus_16 + vpxor W, W, W_TMP + %elif ((%%i & 3) == 2) + vpslld W_TMP, W, 2 + vpsrld W, W, 30 + vpor W, W_TMP, W + %elif ((%%i & 3) == 3) + vpaddd W_TMP, W, oword [K_XMM] + vmovdqa oword [rsp + ((%%i & (~3)) & 15)*4],W_TMP + + ROTATE_W + %endif + + %endif +%endmacro + +;; +;; Regular hash update +;; +%macro SHA1_UPDATE_REGULAR 7.nolist + %xdefine %%A %1 + %xdefine %%B %2 + %xdefine %%C %3 + %xdefine %%D %4 + %xdefine %%E %5 + %xdefine %%nr %6 + %xdefine %%MagiF %7 + + W_CALC %%nr + + add %%E, dword [rsp + (%%nr & 15)*4] + %%MagiF %%B,%%C,%%D + add %%D, dword [rsp +((%%nr+1) & 15)*4] + ROL_30 %%B + mov T3,%%A + add %%E, T1 + ROL_5 T3 + add T3, %%E + mov %%E, T3 + + W_CALC %%nr+1 + + ROL_5 T3 + add %%D,T3 + %%MagiF %%A,%%B,%%C + add %%D, T1 + ROL_30 %%A + +; write: %1, %2 +; rotate: %1<=%4, %2<=%5, %3<=%1, %4<=%2, %5<=%3 +%endmacro + +;; update hash macro +%macro UPDATE_HASH 2.nolist + %xdefine %%hash0 %1 + %xdefine %%hashAdd %2 + + add %%hashAdd, %%hash0 + mov %%hash0, %%hashAdd +%endmacro + +segment .text align=IPP_ALIGN_FACTOR + + +align IPP_ALIGN_FACTOR + +K_XMM_AR dd SHA1_K1, SHA1_K1, SHA1_K1, SHA1_K1 + dd SHA1_K2, SHA1_K2, SHA1_K2, SHA1_K2 + dd SHA1_K3, SHA1_K3, SHA1_K3, SHA1_K3 + dd SHA1_K4, SHA1_K4, SHA1_K4, SHA1_K4 + +shuffle_mask DD 00010203h + DD 04050607h + DD 08090a0bh + DD 0c0d0e0fh + +;***************************************************************************************** +;* Purpose: Update internal digest according to message block +;* +;* void UpdateSHA1(DigestSHA1 digest, const Ipp32u* mblk, int mlen, const void* pParam) +;* +;***************************************************************************************** + +;; +;; Lib = Y8 +;; +;; Caller = ippsSHA1Update +;; Caller = ippsSHA1Final +;; Caller = ippsSHA1MessageDigest +;; +;; Caller = ippsHMACSHA1Update +;; Caller = ippsHMACSHA1Final +;; Caller = ippsHMACSHA1MessageDigest +;; + +;; assign hash values to GPU registers +%xdefine A ecx +%xdefine B eax +%xdefine C edx +%xdefine D r8d +%xdefine E r9d + +;; temporary +%xdefine T1 r10d +%xdefine T2 r11d +%xdefine T3 r13d +%xdefine T4 r13d + +%xdefine W_TMP xmm0 +%xdefine W_TMP2 xmm1 + +%xdefine W0 xmm2 +%xdefine W4 xmm3 +%xdefine W8 xmm4 +%xdefine W12 xmm5 +%xdefine W16 xmm6 +%xdefine W20 xmm7 +%xdefine W24 xmm8 +%xdefine W28 xmm9 + +;; endianness swap constant +%xdefine XMM_SHUFB_BSWAP xmm10 + +;; K_BASE contains [K_XMM_AR] address +%xdefine K_BASE r12 + + +align IPP_ALIGN_FACTOR +IPPASM UpdateSHA1,PUBLIC +%assign LOCAL_FRAME (16*4) + USES_GPR rdi,rsi,r12,r13,r14 + USES_XMM_AVX xmm6,xmm7,xmm8,xmm9,xmm10 + COMP_ABI 4 + +;; +;; rdi = digest ptr +;; rsi = data block ptr +;; rdx = data length +;; rcx = dummy + +%xdefine MBS_SHA1 (64) + + movsxd r14, edx + + movdqa XMM_SHUFB_BSWAP, oword [rel shuffle_mask] ; load shuffle mask + lea K_BASE, [rel K_XMM_AR] ; SHA1 const array address + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +.sha1_block_loop: + mov A, dword [rdi] ; load initial hash value + mov B, dword [rdi+4] + mov C, dword [rdi+8] + mov D, dword [rdi+12] + mov E, dword [rdi+16] + + movdqu W28, oword [rsi] ; load buffer content + movdqu W24, oword [rsi+16] + movdqu W20, oword [rsi+32] + movdqu W16, oword [rsi+48] + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;;SHA1_MAIN_BODY + +%xdefine W W0 +%xdefine W_minus_04 W4 +%xdefine W_minus_08 W8 +%xdefine W_minus_12 W12 +%xdefine W_minus_16 W16 +%xdefine W_minus_20 W20 +%xdefine W_minus_24 W24 +%xdefine W_minus_28 W28 +%xdefine W_minus_32 W + + ;; assignment +%xdefine K_XMM K_BASE + +;;F textequ + SHA1_UPDATE_RND_F1_BSWAP A,B,C,D,E, 0, W28 + SHA1_UPDATE_RND_F1 E,A,B,C,D, 1 + SHA1_UPDATE_RND_F1 D,E,A,B,C, 2 + SHA1_UPDATE_RND_F1 C,D,E,A,B, 3 + + SHA1_UPDATE_RND_F1_BSWAP B,C,D,E,A, 4, W24 + SHA1_UPDATE_RND_F1 A,B,C,D,E, 5 + SHA1_UPDATE_RND_F1 E,A,B,C,D, 6 + SHA1_UPDATE_RND_F1 D,E,A,B,C, 7 + + SHA1_UPDATE_RND_F1_BSWAP C,D,E,A,B, 8, W20 + SHA1_UPDATE_RND_F1 B,C,D,E,A, 9 + SHA1_UPDATE_RND_F1 A,B,C,D,E, 10 + SHA1_UPDATE_RND_F1 E,A,B,C,D, 11 + + SHA1_UPDATE_RND_F1_BSWAP D,E,A,B,C, 12, W16 + + W_CALC 8 + W_CALC 9 + W_CALC 10 + + SHA1_UPDATE_RND_F1 C,D,E,A,B, 13 + + W_CALC 11 + W_CALC 12 + + SHA1_UPDATE_RND_F1 B,C,D,E,A, 14 + + W_CALC 13 + W_CALC 14 + W_CALC 15 + + SHA1_UPDATE_RND_F1 A,B,C,D,E, 15 + + SHA1_UPDATE_REGULAR E,A,B,C,D,16, F1 + SHA1_UPDATE_REGULAR C,D,E,A,B,18, F1 + +;;F textequ + SHA1_UPDATE_REGULAR A,B,C,D,E,20, F2 + SHA1_UPDATE_REGULAR D,E,A,B,C,22, F2 + SHA1_UPDATE_REGULAR B,C,D,E,A,24, F2 + SHA1_UPDATE_REGULAR E,A,B,C,D,26, F2 + SHA1_UPDATE_REGULAR C,D,E,A,B,28, F2 + + SHA1_UPDATE_REGULAR A,B,C,D,E,30, F2 + SHA1_UPDATE_REGULAR D,E,A,B,C,32, F2 + SHA1_UPDATE_REGULAR B,C,D,E,A,34, F2 + SHA1_UPDATE_REGULAR E,A,B,C,D,36, F2 + SHA1_UPDATE_REGULAR C,D,E,A,B,38, F2 + +;;F textequ + SHA1_UPDATE_REGULAR A,B,C,D,E,40, F3 + SHA1_UPDATE_REGULAR D,E,A,B,C,42, F3 + SHA1_UPDATE_REGULAR B,C,D,E,A,44, F3 + SHA1_UPDATE_REGULAR E,A,B,C,D,46, F3 + SHA1_UPDATE_REGULAR C,D,E,A,B,48, F3 + + SHA1_UPDATE_REGULAR A,B,C,D,E,50, F3 + SHA1_UPDATE_REGULAR D,E,A,B,C,52, F3 + SHA1_UPDATE_REGULAR B,C,D,E,A,54, F3 + SHA1_UPDATE_REGULAR E,A,B,C,D,56, F3 + SHA1_UPDATE_REGULAR C,D,E,A,B,58, F3 + +;;F textequ + SHA1_UPDATE_REGULAR A,B,C,D,E,60, F4 + SHA1_UPDATE_REGULAR D,E,A,B,C,62, F4 + SHA1_UPDATE_REGULAR B,C,D,E,A,64, F4 + SHA1_UPDATE_REGULAR E,A,B,C,D,66, F4 + SHA1_UPDATE_REGULAR C,D,E,A,B,68, F4 + + SHA1_UPDATE_REGULAR A,B,C,D,E,70, F4 + SHA1_UPDATE_REGULAR D,E,A,B,C,72, F4 + SHA1_UPDATE_REGULAR B,C,D,E,A,74, F4 + SHA1_UPDATE_REGULAR E,A,B,C,D,76, F4 + SHA1_UPDATE_REGULAR C,D,E,A,B,78, F4 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + UPDATE_HASH dword [rdi], A + UPDATE_HASH dword [rdi+4], B + UPDATE_HASH dword [rdi+8], C + UPDATE_HASH dword [rdi+12],D + UPDATE_HASH dword [rdi+16],E + + add rsi, MBS_SHA1 + sub r14, MBS_SHA1 + jg .sha1_block_loop + + REST_XMM_AVX + REST_GPR + ret +ENDFUNC UpdateSHA1 + +%endif ;; _IPP32E >= _IPP32E_E9 +%endif ;; _FEATURE_OFF_ / _FEATURE_TICKTOCK_ +%endif ;; _ENABLE_ALG_SHA1_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha1l9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha1l9as.asm new file mode 100644 index 0000000..94dd60f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha1l9as.asm @@ -0,0 +1,788 @@ +;=============================================================================== +; Copyright (C) 2017 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SHA1 +; +; Content: +; UpdateSHA1 +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SHA1_) +%if (_SHA_NI_ENABLING_ == _FEATURE_OFF_) || (_SHA_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_L9 ) + + +;; +;; assignments +;; +%xdefine hA eax ;; hash values into GPU registers +%xdefine F ebp +%xdefine hB ebx +%xdefine hC ecx +%xdefine hD edx +%xdefine hE r8d + +%xdefine T1 r10d ;; SHA1 round computation (temporary) +%xdefine T2 r11d + +%xdefine W00 ymm2 ;; W values into YMM registers +%xdefine W04 ymm3 +%xdefine W08 ymm4 +%xdefine W12 ymm5 +%xdefine W16 ymm6 +%xdefine W20 ymm7 +%xdefine W24 ymm8 +%xdefine W28 ymm9 + +%xdefine W16L xmm6 +%xdefine W20L xmm7 +%xdefine W24L xmm8 +%xdefine W28L xmm9 + +%xdefine WTMP1 ymm0 ;; msg schedulling computation (temporary) +%xdefine WTMP2 ymm1 +%xdefine WTMP3 ymm10 + +%xdefine YMM_SHUFB ymm11 ;; byte swap constant +%xdefine YMM_K ymm12 ;; sha1 round constant value + +%xdefine F_PTR r13 ;; frame ptr/data block ptr +;;W_PTR textequ r12 ;; frame ptr/data block ptr + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; +;; textual rotation of W array +;; +%macro ROTATE_W 0.nolist + %xdefine W_minus_32 W_minus_28 + %xdefine W_minus_28 W_minus_24 + %xdefine W_minus_24 W_minus_20 + %xdefine W_minus_20 W_minus_16 + %xdefine W_minus_16 W_minus_12 + %xdefine W_minus_12 W_minus_08 + %xdefine W_minus_08 W_minus_04 + %xdefine W_minus_04 W + %xdefine W W_minus_32 +%endmacro + +;; +;; msg schedulling for initial 00-15 sha1 rounds: +;; - byte swap input +;; - add sha1 round constant +%macro W_CALC_00_15 2.nolist + %xdefine %%nr %1 + %xdefine %%Wchunk %2 + + vpshufb W, %%Wchunk, YMM_SHUFB + vpaddd %%Wchunk, W, YMM_K + vmovdqa ymmword [F_PTR + (%%nr)*sizeof(dword)*2], %%Wchunk + + ROTATE_W +%endmacro + +;; +;; msg schedulling for other 16-79 sha1 rounds: +;; +%macro W_CALC 1.nolist + %xdefine %%rndw %1 + + %if (%%rndw < 32) + %if ((%%rndw & 3) == 0) ;; scheduling to interleave with ALUs + vpalignr W, W_minus_12, W_minus_16, 8 ;; w[t-14] + vpsrldq WTMP1, W_minus_04, 4 ;; w[t-3] + vpxor W, W, W_minus_16 + vpxor WTMP1, WTMP1, W_minus_08 + %elif ((%%rndw & 3) == 1) + vpxor W, W, WTMP1 + vpsrld WTMP1, W, 31 + vpslldq WTMP2, W, 12 + vpaddd W, W, W + %elif ((%%rndw & 3) == 2) + vpsrld WTMP3, WTMP2, 30 + vpxor W, W, WTMP1 + vpslld WTMP2, WTMP2, 2 + vpxor W, W, WTMP3 + %elif ((%%rndw & 3) == 3) + vpxor W, W, WTMP2 + ;;vpaddd WTMP1, W, ymmword [K_SHA1_PTR] + vpaddd WTMP1, W, YMM_K + vmovdqa ymmword [F_PTR+4*sizeof(ymmword)+((%%rndw & 15)/4)*sizeof(ymmword)],WTMP1 + ROTATE_W + %endif + + %elif (%%rndw < 80) + %if ((%%rndw & 3) == 0) ;; scheduling to interleave with ALUs + vpalignr WTMP1, W_minus_04, W_minus_08, 8 + vpxor W, W, W_minus_28 ;; W == W_minus_32 + %elif ((%%rndw & 3) == 1) + vpxor W, W, W_minus_16 + vpxor W, W, WTMP1 + %elif ((%%rndw & 3) == 2) + vpslld WTMP1, W, 2 + vpsrld W, W, 30 + %elif ((%%rndw & 3) == 3) + vpxor W, WTMP1, W + ;;vpaddd WTMP1, W, ymmword [K_SHA1_PTR] + vpaddd WTMP1, W, YMM_K + vmovdqa ymmword [F_PTR+4*sizeof(ymmword)+((%%rndw & 15)/4)*sizeof(ymmword)],WTMP1 + ROTATE_W + %endif + + %endif +%endmacro + +;; +;; update hash macro +;; +%macro UPDATE_HASH 2.nolist + %xdefine %%hashMem %1 + %xdefine %%hash %2 + + add %%hash, %%hashMem + mov %%hashMem, %%hash +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; +;; textual rotation of HASH arguments +;; +%macro ROTATE_H 0.nolist + %xdefine %%_X hE + %xdefine hE hD + %xdefine hD hC + %xdefine hC hB + %xdefine hB F + %xdefine F hA + %xdefine hA %%_X +%endmacro + +;; +;; SHA1 rounds 0 - 19 +;; on entry: +;; - a, f(), b', c, d, e values +;; where: f() = F(b,c,d) = (b&c) & (~b&d) already pre-computed for this round +;; b' = rotl(b,30) already pre-computed for the next round +;; note: +;; %if nr==19 the f(b,c,d)=b^c^d precomputed +;; +%macro SHA1_ROUND_00_19 1.nolist + %xdefine %%nr %1 + + ;; hE+=W[nr]+K + add hE, dword [F_PTR+((%%nr & 15)/4)*sizeof(ymmword)+(%%nr & 3)*sizeof(dword)] + +%if (((%%nr+1) & 15) == 0) + add F_PTR, (16/4)*sizeof(ymmword) +%endif + +%if (%%nr < 19) + andn T1,hA,hC ;; ~hB&hC (next round) +%endif + add hE, F ;; hE += F() + rorx T2,hA, 27 ;; hA<<<5 + rorx F, hA, 2 ;; hB<<<30 (next round) +%if (%%nr < 19) + and hA, hB ;; hB&hC (next round) +%endif + add hE, T2 ;; hE += (hA<<<5) +%if (%%nr < 19) + xor hA, T1 ;; F() = (hB&hC)^(~hB&hC) (next round) +%else + xor hA, hB ;; F() = hB^hC^hD next round + xor hA, hC +%endif + + ROTATE_H +%endmacro + +;; +;; SHA1 rounds 20 - 39 +;; on entry: +;; - a, f(), b', c, d, e values +;; where: f() = F(b,c,d) = b^c^d already pre-computed for this round +;; b' = rotl(b,30) already pre-computed for the next round +;; +;; note: +;; %if nr==39 the f(b,c,d)=(b^c)&(c^d) precomputed +;; +%macro SHA1_ROUND_20_39 1.nolist + %xdefine %%nr %1 + + ;; hE+=W[nr]+K + add hE, dword [F_PTR+((%%nr & 15)/4)*sizeof(ymmword)+(%%nr & 3)*sizeof(dword)] + +%if (((%%nr+1) & 15) == 0) + add F_PTR, (16/4)*sizeof(ymmword) +%endif + + add hE, F ;; hE += F() + rorx T2,hA, 27 ;; hA<<<5 + rorx F, hA, 2 ;; hB<<<30 (next round) + xor hA, hB ;; hB^hC (next round) + add hE, T2 ;; hE += (hA<<<5) +%if (%%nr < 39) + xor hA, hC ;; F() = hB^hC^hD (next round) +%else + mov T1, hB ;; hC^hD (next round) + xor T1, hC ;; + and hA, T1 ;; (hB^hC)&(hC^hD) +%endif + + ROTATE_H +%endmacro + +;; +;; SHA1 rounds 40 - 59 +;; on entry: +;; - a, f(), b', c, d, e values +;; where: f() = (b&c)^(c&d) already pre-computed (part of F()) for this round +;; b' = rotl(b,30) already pre-computed for the next round +;; +;; F(b,c,d) = (b&c)^(b&d)^(c&d) +;; +;; note, using GF(2): arithmetic +;; F(b,c,d) = (b&c)^(b&d)^(c&d) ~ bc+bd+cd +;; =(b+c)(c+d) +c^2 = (b+c)(c+d) +c +;; ~ ((b^c)&(c^d)) ^c +;; direct substitution: +;; (b+c)(c+d) = bc + bd + c^2 + cd, but c^2 = c +;; +%macro SHA1_ROUND_40_59 1.nolist + %xdefine %%nr %1 + + ;; hE+=W[nr]+K + add hE, dword [F_PTR+((%%nr & 15)/4)*sizeof(ymmword)+(%%nr & 3)*sizeof(dword)] + +%if (((%%nr+1) & 15) == 0) + add F_PTR, (16/4)*sizeof(ymmword) +%endif + + xor F, hC ;; F() = ((b^c)&(c^d)) ^c +%if(%%nr < 59) + mov T1, hB ;; hC^hD (next round) + xor T1, hC ;; +%endif + add hE, F ;; hE += F() + rorx T2,hA, 27 ;; hA<<<5 + rorx F, hA, 2 ;; hB<<<30 (next round) + xor hA, hB ;; hB^hC (next round) + add hE, T2 ;; hE += (hA<<<5) +%if(%%nr < 59) + and hA, T1 ;; (hB^hC)&(hC^hD) (next round) +%else + xor hA, hC ;; (hB^hC^hD) (next round) +%endif + + ROTATE_H +%endmacro + +;; +;; SHA1 rounds 60 - 79 +;; on entry: +;; - a, f(), b', c, d, e values +;; where: f() = F(b,c,d) = b^c^d already pre-computed for this round +;; b' = rotl(b,30) already pre-computed for the next round +;; +%macro SHA1_ROUND_60_79 1.nolist + %xdefine %%nr %1 + + ;; hE+=W[nr]+K + add hE, dword [F_PTR+((%%nr & 15)/4)*sizeof(ymmword)+(%%nr & 3)*sizeof(dword)] + +%if (((%%nr+1) & 15) == 0) + add F_PTR, (16/4)*sizeof(ymmword) +%endif + + add hE, F ;; hE += F() + rorx T2,hA, 27 ;; hA<<<5 +%if (%%nr < 79) + rorx F, hA, 2 ;; hB<<<30 (next round) + xor hA, hB ;; hB^hC (next round) +%endif + add hE, T2 ;; hE += (hA<<<5) +%if (%%nr < 79) + xor hA, hC ;; F() = hB^hC^hD (next round) +%endif + + ROTATE_H +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +segment .text align=IPP_ALIGN_FACTOR + + +align IPP_ALIGN_FACTOR + +SHA1_YMM_K dd 05a827999h, 05a827999h, 05a827999h, 05a827999h, 05a827999h, 05a827999h, 05a827999h, 05a827999h + dd 06ed9eba1h, 06ed9eba1h, 06ed9eba1h, 06ed9eba1h, 06ed9eba1h, 06ed9eba1h, 06ed9eba1h, 06ed9eba1h + dd 08f1bbcdch, 08f1bbcdch, 08f1bbcdch, 08f1bbcdch, 08f1bbcdch, 08f1bbcdch, 08f1bbcdch, 08f1bbcdch + dd 0ca62c1d6h, 0ca62c1d6h, 0ca62c1d6h, 0ca62c1d6h, 0ca62c1d6h, 0ca62c1d6h, 0ca62c1d6h, 0ca62c1d6h + +SHA1_YMM_BF dd 00010203h,04050607h,08090a0bh,0c0d0e0fh + dd 00010203h,04050607h,08090a0bh,0c0d0e0fh + + +;***************************************************************************************** +;* Purpose: Update internal digest according to message block +;* +;* void UpdateSHA1(DigestSHA1 digest, const Ipp32u* mblk, int mlen, const void* pParam) +;* +;***************************************************************************************** +align IPP_ALIGN_FACTOR +IPPASM UpdateSHA1,PUBLIC +%assign LOCAL_FRAME (sizeof(dword)*80*2) + USES_GPR rdi,rsi,rbp,rbx,r12,r13,r14,r15 + USES_XMM_AVX xmm6,xmm7,xmm8,xmm9,xmm10,xmm11,xmm12 + COMP_ABI 4 + +;; rdi = hash ptr +;; rsi = data block ptr +;; rdx = data length in bytes +;; rcx = dummy + +%xdefine MBS_SHA1 (64) + + mov r15, rsp ; store orifinal rsp + and rsp, -IPP_ALIGN_FACTOR ; 32-byte aligned stack + + movsxd r14, edx ; input length in bytes + + vmovdqa YMM_SHUFB, [rel SHA1_YMM_BF] ; load byte shuffler + + mov hA, dword [rdi] ; load initial hash value + mov F, dword [rdi+sizeof(dword)] + mov hC, dword [rdi+2*sizeof(dword)] + mov hD, dword [rdi+3*sizeof(dword)] + mov hE, dword [rdi+4*sizeof(dword)] + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data 2 block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; +;; assignment: +;; - W00,...,W28 are fixed +;; - W_minus_04,...,W_minus_32 are rorarted +;; - W corresponds to W[t] +;; +%xdefine W W00 +%xdefine W_minus_04 W04 +%xdefine W_minus_08 W08 +%xdefine W_minus_12 W12 +%xdefine W_minus_16 W16 +%xdefine W_minus_20 W20 +%xdefine W_minus_24 W24 +%xdefine W_minus_28 W28 +%xdefine W_minus_32 W + +align IPP_ALIGN_FACTOR +.sha1_block2_loop: + lea F_PTR, [rsi+MBS_SHA1] ; next block + + cmp r14, MBS_SHA1 ; %if single block processed + cmovbe F_PTR, rsi ; use the same data block address + + ;; + ;; load data block and merge next data block + ;; + vmovdqa YMM_K, [rel SHA1_YMM_K] ; pre-load sha1 constant + + vmovdqu W28L, xmmword [rsi] ; load data block + vmovdqu W24L, xmmword [rsi+1*sizeof(xmmword)] + vmovdqu W20L, xmmword [rsi+2*sizeof(xmmword)] + vmovdqu W16L, xmmword [rsi+3*sizeof(xmmword)] + + vinserti128 W28, W28, xmmword [F_PTR], 1 ; merge next data block + vinserti128 W24, W24, xmmword [F_PTR+1*sizeof(xmmword)], 1 + vinserti128 W20, W20, xmmword [F_PTR+2*sizeof(xmmword)], 1 + vinserti128 W16, W16, xmmword [F_PTR+3*sizeof(xmmword)], 1 + + mov F_PTR, rsp ;; set local data pointer + + W_CALC_00_15 0, W28 ;; msg scheduling for rounds 00 .. 15 + W_CALC_00_15 4, W24 ;; + W_CALC_00_15 8, W20 ;; msg scheduling for rounds 08 .. 15 + W_CALC_00_15 12, W16 ;; + + rorx hB, F, 2 ;; pre-compute (b<<<30) next round + andn T1,F, hD ;; pre-compute F1(F,hC,hD) = hB&hC ^(~hB&hD) + and F, hC + xor F, T1 + + W_CALC 16 ;; msg schedilling ahead 16 rounds + SHA1_ROUND_00_19 0 ;; sha1 round + + W_CALC 17 + SHA1_ROUND_00_19 1 + + W_CALC 18 + SHA1_ROUND_00_19 2 + + W_CALC 19 + SHA1_ROUND_00_19 3 + + ; pre-load sha1 constant + vmovdqa YMM_K, [rel SHA1_YMM_K+sizeof(ymmword)] + W_CALC 20 + SHA1_ROUND_00_19 4 + + W_CALC 21 + SHA1_ROUND_00_19 5 + + W_CALC 22 + SHA1_ROUND_00_19 6 + + W_CALC 23 + SHA1_ROUND_00_19 7 + + W_CALC 24 + SHA1_ROUND_00_19 8 + + W_CALC 25 + SHA1_ROUND_00_19 9 + + W_CALC 26 + SHA1_ROUND_00_19 10 + + W_CALC 27 + SHA1_ROUND_00_19 11 + + W_CALC 28 + SHA1_ROUND_00_19 12 + + W_CALC 29 + SHA1_ROUND_00_19 13 + + W_CALC 30 + SHA1_ROUND_00_19 14 + + W_CALC 31 + SHA1_ROUND_00_19 15 + + W_CALC 32 + SHA1_ROUND_00_19 16 + + W_CALC 33 + SHA1_ROUND_00_19 17 + + W_CALC 34 + SHA1_ROUND_00_19 18 + + W_CALC 35 + SHA1_ROUND_00_19 19 + + W_CALC 36 + SHA1_ROUND_20_39 20 + + W_CALC 37 + SHA1_ROUND_20_39 21 + + W_CALC 38 + SHA1_ROUND_20_39 22 + + W_CALC 39 + SHA1_ROUND_20_39 23 + + ; pre-load sha1 constant + vmovdqa YMM_K, [rel SHA1_YMM_K+2*sizeof(ymmword)] + W_CALC 40 + SHA1_ROUND_20_39 24 + + W_CALC 41 + SHA1_ROUND_20_39 25 + + W_CALC 42 + SHA1_ROUND_20_39 26 + + W_CALC 43 + SHA1_ROUND_20_39 27 + + W_CALC 44 + SHA1_ROUND_20_39 28 + + W_CALC 45 + SHA1_ROUND_20_39 29 + + W_CALC 46 + SHA1_ROUND_20_39 30 + + W_CALC 47 + SHA1_ROUND_20_39 31 + + W_CALC 48 + SHA1_ROUND_20_39 32 + + W_CALC 49 + SHA1_ROUND_20_39 33 + + W_CALC 50 + SHA1_ROUND_20_39 34 + + W_CALC 51 + SHA1_ROUND_20_39 35 + + W_CALC 52 + SHA1_ROUND_20_39 36 + + W_CALC 53 + SHA1_ROUND_20_39 37 + + W_CALC 54 + SHA1_ROUND_20_39 38 + + W_CALC 55 + SHA1_ROUND_20_39 39 + + W_CALC 56 + SHA1_ROUND_40_59 40 + + W_CALC 57 + SHA1_ROUND_40_59 41 + + W_CALC 58 + SHA1_ROUND_40_59 42 + + W_CALC 59 + SHA1_ROUND_40_59 43 + + ; pre-load sha1 constant + vmovdqa YMM_K, [rel SHA1_YMM_K+3*sizeof(ymmword)] + W_CALC 60 + SHA1_ROUND_40_59 44 + + W_CALC 61 + SHA1_ROUND_40_59 45 + + W_CALC 62 + SHA1_ROUND_40_59 46 + + W_CALC 63 + SHA1_ROUND_40_59 47 + + W_CALC 64 + SHA1_ROUND_40_59 48 + + W_CALC 65 + SHA1_ROUND_40_59 49 + + W_CALC 66 + SHA1_ROUND_40_59 50 + + W_CALC 67 + SHA1_ROUND_40_59 51 + + W_CALC 68 + SHA1_ROUND_40_59 52 + + W_CALC 69 + SHA1_ROUND_40_59 53 + + W_CALC 70 + SHA1_ROUND_40_59 54 + + W_CALC 71 + SHA1_ROUND_40_59 55 + + W_CALC 72 + SHA1_ROUND_40_59 56 + + W_CALC 73 + SHA1_ROUND_40_59 57 + + W_CALC 74 + SHA1_ROUND_40_59 58 + + W_CALC 75 + SHA1_ROUND_40_59 59 + + W_CALC 76 + SHA1_ROUND_60_79 60 + + W_CALC 77 + SHA1_ROUND_60_79 61 + + W_CALC 78 + SHA1_ROUND_60_79 62 + + W_CALC 79 + SHA1_ROUND_60_79 63 + + SHA1_ROUND_60_79 64 + SHA1_ROUND_60_79 65 + SHA1_ROUND_60_79 66 + SHA1_ROUND_60_79 67 + SHA1_ROUND_60_79 68 + SHA1_ROUND_60_79 69 + SHA1_ROUND_60_79 70 + SHA1_ROUND_60_79 71 + SHA1_ROUND_60_79 72 + SHA1_ROUND_60_79 73 + SHA1_ROUND_60_79 74 + SHA1_ROUND_60_79 75 + SHA1_ROUND_60_79 76 + SHA1_ROUND_60_79 77 + SHA1_ROUND_60_79 78 + SHA1_ROUND_60_79 79 + + lea F_PTR, [rsp+sizeof(xmmword)] ;; set local data pointer + + ;; update hash values by 1-st data block + UPDATE_HASH dword [rdi], hA + UPDATE_HASH dword [rdi+4], F + UPDATE_HASH dword [rdi+8], hC + UPDATE_HASH dword [rdi+12],hD + UPDATE_HASH dword [rdi+16],hE + + cmp r14, MBS_SHA1*2 + jl .done + + rorx hB, F, 2 ;; pre-compute (b<<<30) next round + andn T1,F, hD ;; pre-compute F1(F,hC,hD) = hB&hC ^(~hB&hD) + and F, hC + xor F, T1 + + SHA1_ROUND_00_19 0 + SHA1_ROUND_00_19 1 + SHA1_ROUND_00_19 2 + SHA1_ROUND_00_19 3 + SHA1_ROUND_00_19 4 + SHA1_ROUND_00_19 5 + SHA1_ROUND_00_19 6 + SHA1_ROUND_00_19 7 + SHA1_ROUND_00_19 8 + SHA1_ROUND_00_19 9 + SHA1_ROUND_00_19 10 + SHA1_ROUND_00_19 11 + SHA1_ROUND_00_19 12 + SHA1_ROUND_00_19 13 + SHA1_ROUND_00_19 14 + SHA1_ROUND_00_19 15 + SHA1_ROUND_00_19 16 + SHA1_ROUND_00_19 17 + SHA1_ROUND_00_19 18 + SHA1_ROUND_00_19 19 + + SHA1_ROUND_20_39 20 + SHA1_ROUND_20_39 21 + SHA1_ROUND_20_39 22 + SHA1_ROUND_20_39 23 + SHA1_ROUND_20_39 24 + SHA1_ROUND_20_39 25 + SHA1_ROUND_20_39 26 + SHA1_ROUND_20_39 27 + SHA1_ROUND_20_39 28 + SHA1_ROUND_20_39 29 + SHA1_ROUND_20_39 30 + SHA1_ROUND_20_39 31 + SHA1_ROUND_20_39 32 + SHA1_ROUND_20_39 33 + SHA1_ROUND_20_39 34 + SHA1_ROUND_20_39 35 + SHA1_ROUND_20_39 36 + SHA1_ROUND_20_39 37 + SHA1_ROUND_20_39 38 + SHA1_ROUND_20_39 39 + + SHA1_ROUND_40_59 40 + SHA1_ROUND_40_59 41 + SHA1_ROUND_40_59 42 + SHA1_ROUND_40_59 43 + SHA1_ROUND_40_59 44 + SHA1_ROUND_40_59 45 + SHA1_ROUND_40_59 46 + SHA1_ROUND_40_59 47 + SHA1_ROUND_40_59 48 + SHA1_ROUND_40_59 49 + SHA1_ROUND_40_59 50 + SHA1_ROUND_40_59 51 + SHA1_ROUND_40_59 52 + SHA1_ROUND_40_59 53 + SHA1_ROUND_40_59 54 + SHA1_ROUND_40_59 55 + SHA1_ROUND_40_59 56 + SHA1_ROUND_40_59 57 + SHA1_ROUND_40_59 58 + SHA1_ROUND_40_59 59 + + SHA1_ROUND_60_79 60 + SHA1_ROUND_60_79 61 + SHA1_ROUND_60_79 62 + SHA1_ROUND_60_79 63 + SHA1_ROUND_60_79 64 + SHA1_ROUND_60_79 65 + SHA1_ROUND_60_79 66 + SHA1_ROUND_60_79 67 + SHA1_ROUND_60_79 68 + SHA1_ROUND_60_79 69 + SHA1_ROUND_60_79 70 + SHA1_ROUND_60_79 71 + SHA1_ROUND_60_79 72 + SHA1_ROUND_60_79 73 + SHA1_ROUND_60_79 74 + SHA1_ROUND_60_79 75 + SHA1_ROUND_60_79 76 + SHA1_ROUND_60_79 77 + SHA1_ROUND_60_79 78 + SHA1_ROUND_60_79 79 + + ;; update hash values by 2-nd data block + UPDATE_HASH dword [rdi], hA + UPDATE_HASH dword [rdi+4], F + UPDATE_HASH dword [rdi+8], hC + UPDATE_HASH dword [rdi+12],hD + UPDATE_HASH dword [rdi+16],hE + + ;; unfortunately 2*80%6 != 0 + ;; and so need to re-order hA,F,hB,hC,hD,hE values + ;; to match the code generated for 1-st block processing + mov hB, hD ; re-order data physically + mov hD, hA + mov T1, hE + mov hE, F + mov F, hC + mov hC, T1 + + ROTATE_H ; re-order data logically + ROTATE_H ; twice, because 6 -(2*80%6) = 2 + + add rsi, MBS_SHA1*2 + sub r14, MBS_SHA1*2 + jg .sha1_block2_loop + +.done: + mov rsp, r15 + REST_XMM_AVX + REST_GPR + ret +ENDFUNC UpdateSHA1 + +%endif ;; _IPP32E >= _IPP32E_L9 +%endif ;; _FEATURE_OFF_ / _FEATURE_TICKTOCK_ +%endif ;; _ENABLE_ALG_SHA1_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha1m7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha1m7as.asm new file mode 100644 index 0000000..2a003e9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha1m7as.asm @@ -0,0 +1,503 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SHA1 +; +; Content: +; UpdateSHA1 +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SHA1_) +%if (_SHA_NI_ENABLING_ == _FEATURE_OFF_) || (_SHA_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_M7) && (_IPP32E < _IPP32E_U8 ) + + +;; +;; Magic functions defined in FIPS 180-1 +;; +%macro MAGIC_F0 4-5.nolist + %xdefine %%regF %1 + %xdefine %%regB %2 + %xdefine %%regC %3 + %xdefine %%regD %4 + %xdefine %%regT %5 + + mov %%regF,%%regC + xor %%regF,%%regD + and %%regF,%%regB + xor %%regF,%%regD +%endmacro + +%macro MAGIC_F1 4-5.nolist + %xdefine %%regF %1 + %xdefine %%regB %2 + %xdefine %%regC %3 + %xdefine %%regD %4 + %xdefine %%regT %5 + + mov %%regF,%%regD + xor %%regF,%%regC + xor %%regF,%%regB +%endmacro + +%macro MAGIC_F2 5.nolist + %xdefine %%regF %1 + %xdefine %%regB %2 + %xdefine %%regC %3 + %xdefine %%regD %4 + %xdefine %%regT %5 + + mov %%regF,%%regB + mov %%regT,%%regB + or %%regF,%%regC + and %%regT,%%regC + and %%regF,%%regD + or %%regF,%%regT +%endmacro + +%macro MAGIC_F3 4-5.nolist + %xdefine %%regF %1 + %xdefine %%regB %2 + %xdefine %%regC %3 + %xdefine %%regD %4 + %xdefine %%regT %5 + + MAGIC_F1 {%%regF},{%%regB},{%%regC},{%%regD},{%%regT} +%endmacro + +;; +;; single SHA1 step +;; +;; Ipp32u tmp = ROL(A,5) + MAGIC_Fi(B,C,D) + E + W[t] + CNT[i]; +;; E = D; +;; D = C; +;; C = ROL(B,30); +;; B = A; +;; A = tmp; +;; +%macro SHA1_STEP 10.nolist + %xdefine %%regA %1 + %xdefine %%regB %2 + %xdefine %%regC %3 + %xdefine %%regD %4 + %xdefine %%regE %5 + %xdefine %%regT %6 + %xdefine %%regF %7 + %xdefine %%memW %8 + %xdefine %%immCNT %9 + %xdefine %%MAGIC %10 + + add %%regE,%%immCNT + add %%regE,[%%memW] + mov %%regT,%%regA + rol %%regT,5 + add %%regE,%%regT + %%MAGIC %%regF,%%regB,%%regC,%%regD,%%regT ;; FUN = MAGIC_Fi(B,C,D) + rol %%regB,30 + add %%regE,%%regF +%endmacro + +%macro SHA1_RND0 8.nolist + %xdefine %%regA %1 + %xdefine %%regB %2 + %xdefine %%regC %3 + %xdefine %%regD %4 + %xdefine %%regE %5 + %xdefine %%regT %6 + %xdefine %%regF %7 + %xdefine %%nr %8 + + %assign %%immCNT 05A827999h + mov r13d,%%immCNT + MAGIC_F0 %%regF,%%regB,%%regC,%%regD ;; FUN = MAGIC_Fi(B,C,D) + ror %%regB,(32-30) + mov %%regT,%%regA + rol %%regT,5 + add %%regE,[rsp+(((%%nr) & 0Fh)*4)] +; lea regE,[regE+regF+immCNT] ; substituted with 2 adds because of gnu as bug + add r13d, %%regF + add %%regE, r13d + add %%regE,%%regT +%endmacro + +%macro SHA1_RND1 8.nolist + %xdefine %%regA %1 + %xdefine %%regB %2 + %xdefine %%regC %3 + %xdefine %%regD %4 + %xdefine %%regE %5 + %xdefine %%regT %6 + %xdefine %%regF %7 + %xdefine %%nr %8 + + %assign %%immCNT 06ED9EBA1h + mov r13d,%%immCNT + MAGIC_F1 {%%regF},{%%regB},{%%regC},{%%regD} ;; FUN = MAGIC_Fi(B,C,D) + ror %%regB,(32-30) + mov %%regT,%%regA + rol %%regT,5 + add %%regE,[rsp+(((%%nr) & 0Fh)*4)] +; lea regE,[regE+regF+immCNT] ; substituted with 2 adds because of gnu as bug + add r13d, %%regF + add %%regE, r13d + add %%regE,%%regT +%endmacro + +%macro SHA1_RND2 8.nolist + %xdefine %%regA %1 + %xdefine %%regB %2 + %xdefine %%regC %3 + %xdefine %%regD %4 + %xdefine %%regE %5 + %xdefine %%regT %6 + %xdefine %%regF %7 + %xdefine %%nr %8 + + %ifndef _VXWORKS + %assign %%immCNT 08F1BBCDCh + %else + %assign %%immCNT -1894007588 + %endif + mov r13d,%%immCNT + MAGIC_F2 %%regF,%%regB,%%regC,%%regD,%%regT ;; FUN = MAGIC_Fi(B,C,D) + ror %%regB,(32-30) + mov %%regT,%%regA + rol %%regT,5 + add %%regE,[rsp+(((%%nr) & 0Fh)*4)] +; lea regE,[regE+regF+immCNT] ; substituted with 2 adds because of gnu as bug + add r13d, %%regF + add %%regE, r13d + add %%regE,%%regT +%endmacro + +%macro SHA1_RND3 8.nolist + %xdefine %%regA %1 + %xdefine %%regB %2 + %xdefine %%regC %3 + %xdefine %%regD %4 + %xdefine %%regE %5 + %xdefine %%regT %6 + %xdefine %%regF %7 + %xdefine %%nr %8 + + %ifndef _VXWORKS + %assign %%immCNT 0CA62C1D6h + %else + %assign %%immCNT -899497514 + %endif + mov r13d,%%immCNT + MAGIC_F3 {%%regF},{%%regB},{%%regC},{%%regD} ;; FUN = MAGIC_Fi(B,C,D) + ror %%regB,(32-30) + mov %%regT,%%regA + rol %%regT,5 + add %%regE,[rsp+(((%%nr) & 0Fh)*4)] +; lea regE,[regE+regF+immCNT] ; substituted with 2 adds because of gnu as bug + add r13d, %%regF + add %%regE, r13d + add %%regE,%%regT +%endmacro + +;; +;; ENDIANNESS +;; +%macro ENDIANNESS 2.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + + %ifnidn %%dst,%%src + mov %%dst,%%src + %endif + bswap %%dst +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; Following Macros are especially for new implementation of SHA1 +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro UPDATE 2-3.nolist + %xdefine %%nr %1 + %xdefine %%regU %2 + %xdefine %%regT %3 + + %ifempty %%regT + mov %%regU,[rsp+((%%nr-16) & 0Fh)*4] + xor %%regU,[rsp+((%%nr-14) & 0Fh)*4] + xor %%regU,[rsp+((%%nr-8) & 0Fh)*4] + xor %%regU,[rsp+((%%nr-3) & 0Fh)*4] + %else + mov %%regU,[rsp+((%%nr-16) & 0Fh)*4] + mov %%regT,[rsp+((%%nr-14) & 0Fh)*4] + xor %%regU,%%regT + mov %%regT,[rsp+((%%nr-8) & 0Fh)*4] + xor %%regU,%%regT + mov %%regT,[rsp+((%%nr-3) & 0Fh)*4] + xor %%regU,%%regT + %endif + rol %%regU,1 + mov [rsp+(%%nr & 0Fh)*4],%%regU +%endmacro + +segment .text align=IPP_ALIGN_FACTOR + + +;***************************************************************************************** +;* Purpose: Update internal digest according to message block +;* +;* void UpdateSHA1(DigestSHA1 digest, const Ipp32u* mblk, int mlen, const void* pParam) +;* +;***************************************************************************************** + +;; +;; Lib = M7 +;; +;; Caller = ippsSHA1Update +;; Caller = ippsSHA1Final +;; Caller = ippsSHA1MessageDigest +;; +;; Caller = ippsHMACSHA1Update +;; Caller = ippsHMACSHA1Final +;; Caller = ippsHMACSHA1MessageDigest +;; + +align IPP_ALIGN_FACTOR +IPPASM UpdateSHA1,PUBLIC +%assign LOCAL_FRAME 16*4 + USES_GPR rbx,rsi,rdi,r8,r9,r10,r11,r12,r13 + USES_XMM + COMP_ABI 4 + +%xdefine MBS_SHA1 (64) + + movsxd rdx, edx + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +.sha1_block_loop: + +;; +;; init A, B, C, D, E by the internal digest +;; + mov r8d, [rdi+0*4] ; r8d = digest[0] (A) + mov r9d, [rdi+1*4] ; r9d = digest[1] (B) + mov r10d,[rdi+2*4] ; r10d= digest[2] (C) + mov r11d,[rdi+3*4] ; r11d= digest[3] (D) + mov r12d,[rdi+4*4] ; r12d= digest[4] (E) + +;; +;; initialize the first 16 words in the array W (remember about endian) +;; + xor rcx,rcx +.loop1: + mov eax,[rsi+rcx*4+0*4] + ENDIANNESS eax,eax + mov [rsp+rcx*4+0*4],eax + + mov ebx,[rsi+rcx*4+1*4] + ENDIANNESS ebx,ebx + mov [rsp+rcx*4+1*4],ebx + + add rcx,2 + cmp rcx,16 + jl .loop1 + +;; +;; perform 0-79 steps +;; +;; A, B, C, D, E, TMP,FUN, round +;; ----------------------------------- + SHA1_RND0 r8d,r9d,r10d,r11d,r12d, ecx,ebx, 0 + UPDATE 16, eax + SHA1_RND0 r12d,r8d,r9d,r10d,r11d, ecx,ebx, 1 + UPDATE 17, eax + SHA1_RND0 r11d,r12d,r8d,r9d,r10d, ecx,ebx, 2 + UPDATE 18, eax + SHA1_RND0 r10d,r11d,r12d,r8d,r9d, ecx,ebx, 3 + UPDATE 19, eax + SHA1_RND0 r9d,r10d,r11d,r12d,r8d, ecx,ebx, 4 + UPDATE 20, eax + SHA1_RND0 r8d,r9d,r10d,r11d,r12d, ecx,ebx, 5 + UPDATE 21, eax + SHA1_RND0 r12d,r8d,r9d,r10d,r11d, ecx,ebx, 6 + UPDATE 22, eax + SHA1_RND0 r11d,r12d,r8d,r9d,r10d, ecx,ebx, 7 + UPDATE 23, eax + SHA1_RND0 r10d,r11d,r12d,r8d,r9d, ecx,ebx, 8 + UPDATE 24, eax + SHA1_RND0 r9d,r10d,r11d,r12d,r8d, ecx,ebx, 9 + UPDATE 25, eax + SHA1_RND0 r8d,r9d,r10d,r11d,r12d, ecx,ebx, 10 + UPDATE 26, eax + SHA1_RND0 r12d,r8d,r9d,r10d,r11d, ecx,ebx, 11 + UPDATE 27, eax + SHA1_RND0 r11d,r12d,r8d,r9d,r10d, ecx,ebx, 12 + UPDATE 28, eax + SHA1_RND0 r10d,r11d,r12d,r8d,r9d, ecx,ebx, 13 + UPDATE 29, eax + SHA1_RND0 r9d,r10d,r11d,r12d,r8d, ecx,ebx, 14 + UPDATE 30, eax + SHA1_RND0 r8d,r9d,r10d,r11d,r12d, ecx,ebx, 15 + UPDATE 31, eax + SHA1_RND0 r12d,r8d,r9d,r10d,r11d, ecx,ebx, 16 + UPDATE 32, eax + SHA1_RND0 r11d,r12d,r8d,r9d,r10d, ecx,ebx, 17 + UPDATE 33, eax + SHA1_RND0 r10d,r11d,r12d,r8d,r9d, ecx,ebx, 18 + UPDATE 34, eax + SHA1_RND0 r9d,r10d,r11d,r12d,r8d, ecx,ebx, 19 + UPDATE 35, eax + + SHA1_RND1 r8d,r9d,r10d,r11d,r12d, ecx,ebx, 20 + UPDATE 36, eax + SHA1_RND1 r12d,r8d,r9d,r10d,r11d, ecx,ebx, 21 + UPDATE 37, eax + SHA1_RND1 r11d,r12d,r8d,r9d,r10d, ecx,ebx, 22 + UPDATE 38, eax + SHA1_RND1 r10d,r11d,r12d,r8d,r9d, ecx,ebx, 23 + UPDATE 39, eax + SHA1_RND1 r9d,r10d,r11d,r12d,r8d, ecx,ebx, 24 + UPDATE 40, eax + SHA1_RND1 r8d,r9d,r10d,r11d,r12d, ecx,ebx, 25 + UPDATE 41, eax + SHA1_RND1 r12d,r8d,r9d,r10d,r11d, ecx,ebx, 26 + UPDATE 42, eax + SHA1_RND1 r11d,r12d,r8d,r9d,r10d, ecx,ebx, 27 + UPDATE 43, eax + SHA1_RND1 r10d,r11d,r12d,r8d,r9d, ecx,ebx, 28 + UPDATE 44, eax + SHA1_RND1 r9d,r10d,r11d,r12d,r8d, ecx,ebx, 29 + UPDATE 45, eax + SHA1_RND1 r8d,r9d,r10d,r11d,r12d, ecx,ebx, 30 + UPDATE 46, eax + SHA1_RND1 r12d,r8d,r9d,r10d,r11d, ecx,ebx, 31 + UPDATE 47, eax + SHA1_RND1 r11d,r12d,r8d,r9d,r10d, ecx,ebx, 32 + UPDATE 48, eax + SHA1_RND1 r10d,r11d,r12d,r8d,r9d, ecx,ebx, 33 + UPDATE 49, eax + SHA1_RND1 r9d,r10d,r11d,r12d,r8d, ecx,ebx, 34 + UPDATE 50, eax + SHA1_RND1 r8d,r9d,r10d,r11d,r12d, ecx,ebx, 35 + UPDATE 51, eax + SHA1_RND1 r12d,r8d,r9d,r10d,r11d, ecx,ebx, 36 + UPDATE 52, eax + SHA1_RND1 r11d,r12d,r8d,r9d,r10d, ecx,ebx, 37 + UPDATE 53, eax + SHA1_RND1 r10d,r11d,r12d,r8d,r9d, ecx,ebx, 38 + UPDATE 54, eax + SHA1_RND1 r9d,r10d,r11d,r12d,r8d, ecx,ebx, 39 + UPDATE 55, eax + + SHA1_RND2 r8d,r9d,r10d,r11d,r12d, ecx,ebx, 40 + UPDATE 56, eax + SHA1_RND2 r12d,r8d,r9d,r10d,r11d, ecx,ebx, 41 + UPDATE 57, eax + SHA1_RND2 r11d,r12d,r8d,r9d,r10d, ecx,ebx, 42 + UPDATE 58, eax + SHA1_RND2 r10d,r11d,r12d,r8d,r9d, ecx,ebx, 43 + UPDATE 59, eax + SHA1_RND2 r9d,r10d,r11d,r12d,r8d, ecx,ebx, 44 + UPDATE 60, eax + SHA1_RND2 r8d,r9d,r10d,r11d,r12d, ecx,ebx, 45 + UPDATE 61, eax + SHA1_RND2 r12d,r8d,r9d,r10d,r11d, ecx,ebx, 46 + UPDATE 62, eax + SHA1_RND2 r11d,r12d,r8d,r9d,r10d, ecx,ebx, 47 + UPDATE 63, eax + SHA1_RND2 r10d,r11d,r12d,r8d,r9d, ecx,ebx, 48 + UPDATE 64, eax + SHA1_RND2 r9d,r10d,r11d,r12d,r8d, ecx,ebx, 49 + UPDATE 65, eax + SHA1_RND2 r8d,r9d,r10d,r11d,r12d, ecx,ebx, 50 + UPDATE 66, eax + SHA1_RND2 r12d,r8d,r9d,r10d,r11d, ecx,ebx, 51 + UPDATE 67, eax + SHA1_RND2 r11d,r12d,r8d,r9d,r10d, ecx,ebx, 52 + UPDATE 68, eax + SHA1_RND2 r10d,r11d,r12d,r8d,r9d, ecx,ebx, 53 + UPDATE 69, eax + SHA1_RND2 r9d,r10d,r11d,r12d,r8d, ecx,ebx, 54 + UPDATE 70, eax + SHA1_RND2 r8d,r9d,r10d,r11d,r12d, ecx,ebx, 55 + UPDATE 71, eax + SHA1_RND2 r12d,r8d,r9d,r10d,r11d, ecx,ebx, 56 + UPDATE 72, eax + SHA1_RND2 r11d,r12d,r8d,r9d,r10d, ecx,ebx, 57 + UPDATE 73, eax + SHA1_RND2 r10d,r11d,r12d,r8d,r9d, ecx,ebx, 58 + UPDATE 74, eax + SHA1_RND2 r9d,r10d,r11d,r12d,r8d, ecx,ebx, 59 + UPDATE 75, eax + + SHA1_RND3 r8d,r9d,r10d,r11d,r12d, ecx,ebx, 60 + UPDATE 76, eax + SHA1_RND3 r12d,r8d,r9d,r10d,r11d, ecx,ebx, 61 + UPDATE 77, eax + SHA1_RND3 r11d,r12d,r8d,r9d,r10d, ecx,ebx, 62 + UPDATE 78, eax + SHA1_RND3 r10d,r11d,r12d,r8d,r9d, ecx,ebx, 63 + UPDATE 79, eax + SHA1_RND3 r9d,r10d,r11d,r12d,r8d, ecx,ebx, 64 + SHA1_RND3 r8d,r9d,r10d,r11d,r12d, ecx,ebx, 65 + SHA1_RND3 r12d,r8d,r9d,r10d,r11d, ecx,ebx, 66 + SHA1_RND3 r11d,r12d,r8d,r9d,r10d, ecx,ebx, 67 + SHA1_RND3 r10d,r11d,r12d,r8d,r9d, ecx,ebx, 68 + SHA1_RND3 r9d,r10d,r11d,r12d,r8d, ecx,ebx, 69 + SHA1_RND3 r8d,r9d,r10d,r11d,r12d, ecx,ebx, 70 + SHA1_RND3 r12d,r8d,r9d,r10d,r11d, ecx,ebx, 71 + SHA1_RND3 r11d,r12d,r8d,r9d,r10d, ecx,ebx, 72 + SHA1_RND3 r10d,r11d,r12d,r8d,r9d, ecx,ebx, 73 + SHA1_RND3 r9d,r10d,r11d,r12d,r8d, ecx,ebx, 74 + SHA1_RND3 r8d,r9d,r10d,r11d,r12d, ecx,ebx, 75 + SHA1_RND3 r12d,r8d,r9d,r10d,r11d, ecx,ebx, 76 + SHA1_RND3 r11d,r12d,r8d,r9d,r10d, ecx,ebx, 77 + SHA1_RND3 r10d,r11d,r12d,r8d,r9d, ecx,ebx, 78 + SHA1_RND3 r9d,r10d,r11d,r12d,r8d, ecx,ebx, 79 + +;; +;; update digest +;; + add [rdi+0*4],r8d ; advance digest + add [rdi+1*4],r9d + add [rdi+2*4],r10d + add [rdi+3*4],r11d + add [rdi+4*4],r12d + + add rsi, MBS_SHA1 + sub rdx, MBS_SHA1 + jg .sha1_block_loop + + REST_XMM + REST_GPR + ret +ENDFUNC UpdateSHA1 + +%endif ;; (_IPP32E >= _IPP32E_M7) AND (_IPP32E < _IPP32E_U8 ) +%endif ;; _FEATURE_OFF_ / _FEATURE_TICKTOCK_ +%endif ;; _ENABLE_ALG_SHA1_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha1nias.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha1nias.asm new file mode 100644 index 0000000..8ed02fa --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha1nias.asm @@ -0,0 +1,549 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SHA-1 +; +; Content: +; UpdateSHA1ni +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SHA1_) +%if (_SHA_NI_ENABLING_ == _FEATURE_ON_) || (_SHA_NI_ENABLING_ == _FEATURE_TICKTOCK_) +;;%if (_IPP32E >= _IPP32E_Y8 ) + +segment .text align=IPP_ALIGN_FACTOR + + +align IPP_ALIGN_FACTOR + +UPPER_DWORD_MASK \ + DQ 00000000000000000h, 0ffffffff00000000h +PSHUFFLE_BYTE_FLIP_MASK \ + DB 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 + +align IPP_ALIGN_FACTOR +;***************************************************************************************** +;* Purpose: Update internal digest according to message block +;* +;* void UpdateSHA1ni(DigestSHA1 digest, const Ipp32u* mblk, int mlen, const void* pParam) +;* +;***************************************************************************************** + +%ifndef _VXWORKS + +IPPASM UpdateSHA1ni,PUBLIC +%assign LOCAL_FRAME 16*2 + USES_GPR rsi,rdi + USES_XMM xmm6,xmm7 + COMP_ABI 4 + +%xdefine MBS_SHA1 (64) ; SHA-1 message block length (bytes) + +%xdefine HASH_PTR rdi ; 1st arg +%xdefine MSG_PTR rsi ; 2nd arg +%xdefine MSG_LEN rdx ; 3rd arg + +%xdefine ABCD xmm0 +%xdefine E0 xmm1 ; Need two E's b/c they ping pong +%xdefine E1 xmm2 +%xdefine MSG0 xmm3 +%xdefine MSG1 xmm4 +%xdefine MSG2 xmm5 +%xdefine MSG3 xmm6 +%xdefine SHUF_MASK xmm7 + +; +; stack frame +; +%xdefine abcd_save rsp +%xdefine e_save rsp+16 + + movsxd MSG_LEN, edx ; expand mLen + test MSG_LEN, MSG_LEN + jz .quit + +;; load initial hash values + movdqu ABCD, oword [HASH_PTR] + pinsrd E0, dword [HASH_PTR+16], 3 + pand E0, oword [rel UPPER_DWORD_MASK] + pshufd ABCD, ABCD, 01Bh + + movdqa SHUF_MASK, oword [rel PSHUFFLE_BYTE_FLIP_MASK] + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +.sha1_block_loop: + movdqa oword [abcd_save], ABCD + movdqa oword [e_save], E0 + + ;; rounds 0-3 + movdqu MSG0, oword [MSG_PTR +0*16] + pshufb MSG0, SHUF_MASK + paddd E0, MSG0 + movdqa E1, ABCD + sha1rnds4 ABCD, E0, 0 + ;movdqu oword [rcx+16*0], ABCD + + ;; rounds 4-7 + movdqu MSG1, oword [MSG_PTR +1*16] + pshufb MSG1, SHUF_MASK + sha1nexte E1, MSG1 + movdqa E0, ABCD + sha1rnds4 ABCD, E1, 0 + sha1msg1 MSG0, MSG1 + ;movdqu oword [rcx+16*1], ABCD + + ;; rounds 8-11 + movdqu MSG2, oword [MSG_PTR +2*16] + pshufb MSG2, SHUF_MASK + sha1nexte E0, MSG2 + movdqa E1, ABCD + sha1rnds4 ABCD, E0, 0 + sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + ;movdqu oword [rcx+16*2], ABCD + + ;; rounds 12-15 + movdqu MSG3, oword [MSG_PTR +3*16] + pshufb MSG3, SHUF_MASK + sha1nexte E1, MSG3 + movdqa E0, ABCD + sha1msg2 MSG0, MSG3 + sha1rnds4 ABCD, E1, 0 + sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + ;movdqu oword [rcx+16*3], ABCD + + ;; rounds 16-19 + sha1nexte E0, MSG0 + movdqa E1, ABCD + sha1msg2 MSG1, MSG0 + sha1rnds4 ABCD, E0, 0 + sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + ;movdqu oword [rcx+16*4], ABCD + + ;; rounds 20-23 + sha1nexte E1, MSG1 + movdqa E0, ABCD + sha1msg2 MSG2, MSG1 + sha1rnds4 ABCD, E1, 1 + sha1msg1 MSG0, MSG1 + pxor MSG3, MSG1 + ;movdqu oword [rcx+16*5], ABCD + + ;; rounds 24-27 + sha1nexte E0, MSG2 + movdqa E1, ABCD + sha1msg2 MSG3, MSG2 + sha1rnds4 ABCD, E0, 1 + sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + ;movdqu oword [rcx+16*6], ABCD + + ;; rounds 28-31 + sha1nexte E1, MSG3 + movdqa E0, ABCD + sha1msg2 MSG0, MSG3 + sha1rnds4 ABCD, E1, 1 + sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + ;movdqu oword [rcx+16*7], ABCD + + ;; rounds 32-35 + sha1nexte E0, MSG0 + movdqa E1, ABCD + sha1msg2 MSG1, MSG0 + sha1rnds4 ABCD, E0, 1 + sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + ;movdqu oword [rcx+16*8], ABCD + + ;; rounds 36-39 + sha1nexte E1, MSG1 + movdqa E0, ABCD + sha1msg2 MSG2, MSG1 + sha1rnds4 ABCD, E1, 1 + sha1msg1 MSG0, MSG1 + pxor MSG3, MSG1 + ;movdqu oword [rcx+16*9], ABCD + + ;; rounds 40-43 + sha1nexte E0, MSG2 + movdqa E1, ABCD + sha1msg2 MSG3, MSG2 + sha1rnds4 ABCD, E0, 2 + sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + ;movdqu oword [rcx+16*10], ABCD + + ;; rounds 44-47 + sha1nexte E1, MSG3 + movdqa E0, ABCD + sha1msg2 MSG0, MSG3 + sha1rnds4 ABCD, E1, 2 + sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + ;movdqu oword [rcx+16*11], ABCD + + ;; rounds 48-51 + sha1nexte E0, MSG0 + movdqa E1, ABCD + sha1msg2 MSG1, MSG0 + sha1rnds4 ABCD, E0, 2 + sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + ;movdqu oword [rcx+16*12], ABCD + + ;; rounds 52-55 + sha1nexte E1, MSG1 + movdqa E0, ABCD + sha1msg2 MSG2, MSG1 + sha1rnds4 ABCD, E1, 2 + sha1msg1 MSG0, MSG1 + pxor MSG3, MSG1 + ;movdqu oword [rcx+16*13], ABCD + + ;; rounds 56-59 + sha1nexte E0, MSG2 + movdqa E1, ABCD + sha1msg2 MSG3, MSG2 + sha1rnds4 ABCD, E0, 2 + sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + ;movdqu oword [rcx+16*14], ABCD + + ;; rounds 60-63 + sha1nexte E1, MSG3 + movdqa E0, ABCD + sha1msg2 MSG0, MSG3 + sha1rnds4 ABCD, E1, 3 + sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + ;movdqu oword [rcx+16*15], ABCD + + ;; rounds 64-67 + sha1nexte E0, MSG0 + movdqa E1, ABCD + sha1msg2 MSG1, MSG0 + sha1rnds4 ABCD, E0, 3 + sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + ;movdqu oword [rcx+16*16], ABCD + + ;; rounds 68-71 + sha1nexte E1, MSG1 + movdqa E0, ABCD + sha1msg2 MSG2, MSG1 + sha1rnds4 ABCD, E1, 3 + pxor MSG3, MSG1 + ;movdqu oword [rcx+16*17], ABCD + + ;; rounds 72-75 + sha1nexte E0, MSG2 + movdqa E1, ABCD + sha1msg2 MSG3, MSG2 + sha1rnds4 ABCD, E0, 3 + ;movdqu oword [rcx+16*18], ABCD + + ;; rounds 76-79 + sha1nexte E1, MSG3 + movdqa E0, ABCD + sha1rnds4 ABCD, E1, 3 + ;movdqu oword [rcx+16*19], ABCD + + ;; add current hash values with previously saved + sha1nexte E0, oword [e_save] + paddd ABCD, oword [abcd_save] + + add MSG_PTR, MBS_SHA1 + sub MSG_LEN, MBS_SHA1 + jg .sha1_block_loop + + ;; write hash values back in the correct order + pshufd ABCD, ABCD, 01Bh + movdqu oword [HASH_PTR], ABCD + pextrd dword [HASH_PTR+16], E0, 3 + +.quit: + REST_XMM + REST_GPR + ret +ENDFUNC UpdateSHA1ni + +%else ;; no sha ni support in VxWorks - therefore we temporary use db +IPPASM UpdateSHA1ni,PUBLIC +%assign LOCAL_FRAME 16*2 + USES_GPR rsi,rdi + USES_XMM xmm6,xmm7 + COMP_ABI 4 + +%xdefine MBS_SHA1 (64) ; SHA-1 message block length (bytes) + +%xdefine HASH_PTR rdi ; 1st arg +%xdefine MSG_PTR rsi ; 2nd arg +%xdefine MSG_LEN rdx ; 3rd arg + +%xdefine ABCD xmm0 +%xdefine E0 xmm1 ; Need two E's b/c they ping pong +%xdefine E1 xmm2 +%xdefine MSG0 xmm3 +%xdefine MSG1 xmm4 +%xdefine MSG2 xmm5 +%xdefine MSG3 xmm6 +%xdefine SHUF_MASK xmm7 + +; +; stack frame +; +%xdefine abcd_save rsp +%xdefine e_save rsp+16 + + movsxd MSG_LEN, edx ; expand mLen + test MSG_LEN, MSG_LEN + jz .quit + +;; load initial hash values + movdqu ABCD, oword [HASH_PTR] + pinsrd E0, dword [HASH_PTR+16], 3 + pand E0, oword [rel UPPER_DWORD_MASK] + pshufd ABCD, ABCD, 01Bh + + movdqa SHUF_MASK, oword [rel PSHUFFLE_BYTE_FLIP_MASK] + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +.sha1_block_loop: + movdqa oword [abcd_save], ABCD + movdqa oword [e_save], E0 + + ;; rounds 0-3 + movdqu MSG0, oword [MSG_PTR +0*16] + pshufb MSG0, SHUF_MASK + paddd E0, MSG0 + movdqa E1, ABCD + db 0FH,3AH,0CCH,0C1H,00H ;;sha1rnds4 ABCD, E0, 0 + ;movdqu oword [rcx+16*0], ABCD + + ;; rounds 4-7 + movdqu MSG1, oword [MSG_PTR +1*16] + pshufb MSG1, SHUF_MASK + db 0FH,38H,0C8H,0D4H ;;sha1nexte E1, MSG1 + movdqa E0, ABCD + db 0FH,3AH,0CCH,0C2H,00H ;;sha1rnds4 ABCD, E1, 0 + db 0FH,38H,0C9H,0DCH ;;sha1msg1 MSG0, MSG1 + ;movdqu oword [rcx+16*1], ABCD + + ;; rounds 8-11 + movdqu MSG2, oword [MSG_PTR +2*16] + pshufb MSG2, SHUF_MASK + db 0FH,38H,0C8H,0CDH ;;sha1nexte E0, MSG2 + movdqa E1, ABCD + db 0Fh,3Ah,0CCh,0C1h,00h ;;sha1rnds4 ABCD, E0, 0 + db 0Fh,38h,0C9h,0E5h ;;sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + ;movdqu oword [rcx+16*2], ABCD + + ;; rounds 12-15 + movdqu MSG3, oword [MSG_PTR +3*16] + pshufb MSG3, SHUF_MASK + db 0Fh,38H,0C8h,0D6h ;;sha1nexte E1, MSG3 + movdqa E0, ABCD + db 0Fh,38H,0CAh,0DEh ;;sha1msg2 MSG0, MSG3 + db 0Fh,3AH,0CCh,0C2h,00h ;;sha1rnds4 ABCD, E1, 0 + db 0Fh,38H,0C9h,0EEh ;;sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + ;movdqu oword [rcx+16*3], ABCD + + ;; rounds 16-19 + db 0Fh,38H,0C8h,0CBh ;;sha1nexte E0, MSG0 + movdqa E1, ABCD + db 0Fh,38H,0CAh,0E3h ;;sha1msg2 MSG1, MSG0 + db 0Fh,3AH,0CCh,0C1h,00h ;;sha1rnds4 ABCD, E0, 0 + db 0Fh,38H,0C9h,0F3h ;;sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + ;movdqu oword [rcx+16*4], ABCD + + ;; rounds 20-23 + db 0FH,38H,0C8h,0D4h ;;sha1nexte E1, MSG1 + movdqa E0, ABCD + db 0FH,38H,0CAh,0ECH ;;sha1msg2 MSG2, MSG1 + db 0FH,3AH,0CCH,0C2H,01H ;;sha1rnds4 ABCD, E1, 1 + db 0FH,38H,0C9h,0DCh ;;sha1msg1 MSG0, MSG1 + pxor MSG3, MSG1 + ;movdqu oword [rcx+16*5], ABCD + + ;; rounds 24-27 + db 0FH,38H,0C8h,0CDh ;;sha1nexte E0, MSG2 + movdqa E1, ABCD + db 0FH,38H,0CAh,0F5h ;;sha1msg2 MSG3, MSG2 + db 0FH,3AH,0CCh,0C1h,01h ;;sha1rnds4 ABCD, E0, 1 + db 0FH,38H,0C9h,0E5h ;;sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + ;movdqu oword [rcx+16*6], ABCD + + ;; rounds 28-31 + db 0FH,38H,0C8H,0D6h ;;sha1nexte E1, MSG3 + movdqa E0, ABCD + db 0FH,38H,0CAH,0DEh ;;sha1msg2 MSG0, MSG3 + db 0FH,3AH,0CCH,0C2H,01h ;;sha1rnds4 ABCD, E1, 1 + db 0FH,38H,0C9H,0EEh ;;sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + ;movdqu oword [rcx+16*7], ABCD + + ;; rounds 32-35 + db 0FH,38H,0C8H,0CBh ;;sha1nexte E0, MSG0 + movdqa E1, ABCD + db 0FH,38H,0CAH,0E3h ;;sha1msg2 MSG1, MSG0 + db 0FH,3AH,0CCH,0C1H,01h ;;sha1rnds4 ABCD, E0, 1 + db 0FH,38H,0C9H,0F3h ;;sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + ;movdqu oword [rcx+16*8], ABCD + + ;; rounds 36-39 + db 0FH,38H,0C8H,0D4h ;;sha1nexte E1, MSG1 + movdqa E0, ABCD + db 0FH,38H,0CAH,0ECh ;;sha1msg2 MSG2, MSG1 + db 0FH,3AH,0CCH,0C2H,01h ;;sha1rnds4 ABCD, E1, 1 + db 0FH,38H,0C9H,0DCh ;;sha1msg1 MSG0, MSG1 + pxor MSG3, MSG1 + ;movdqu oword [rcx+16*9], ABCD + + ;; rounds 40-43 + db 0FH,38H,0C8H,0CDh ;;sha1nexte E0, MSG2 + movdqa E1, ABCD + db 0FH,38H,0CAH,0F5h ;;sha1msg2 MSG3, MSG2 + db 0FH,3AH,0CCH,0C1H,02h ;;sha1rnds4 ABCD, E0, 2 + db 0FH,38H,0C9H,0E5h ;;sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + ;movdqu oword [rcx+16*10], ABCD + + ;; rounds 44-47 + db 0FH,38H,0C8H,0D6h ;;sha1nexte E1, MSG3 + movdqa E0, ABCD + db 0FH,38H,0CAH,0DEh ;;sha1msg2 MSG0, MSG3 + db 0FH,3AH,0CCH,0C2h,02h ;;sha1rnds4 ABCD, E1, 2 + db 0FH,38H,0C9H,0EEh ;;sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + ;movdqu oword [rcx+16*11], ABCD + + ;; rounds 48-51 + db 0Fh,38H,0C8h,0CBh ;;sha1nexte E0, MSG0 + movdqa E1, ABCD + db 0Fh,38H,0CAH,0E3h ;;sha1msg2 MSG1, MSG0 + db 0Fh,3AH,0CCH,0C1H,02h ;;sha1rnds4 ABCD, E0, 2 + db 0Fh,38H,0C9H,0F3h ;;sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + ;movdqu oword [rcx+16*12], ABCD + + ;; rounds 52-55 + db 0Fh,38H,0C8h,0D4h ;;sha1nexte E1, MSG1 + movdqa E0, ABCD + db 0Fh,38H,0CAH,0ECh ;;sha1msg2 MSG2, MSG1 + db 0Fh,3AH,0CCH,0C2H,02h ;;sha1rnds4 ABCD, E1, 2 + db 0Fh,38H,0C9H,0DCh ;;sha1msg1 MSG0, MSG1 + pxor MSG3, MSG1 + ;movdqu oword [rcx+16*13], ABCD + + ;; rounds 56-59 + db 0Fh,38H,0C8H,0CDh ;;sha1nexte E0, MSG2 + movdqa E1, ABCD + db 0Fh,38H,0CAH,0F5h ;;sha1msg2 MSG3, MSG2 + db 0Fh,3AH,0CCH,0C1H,02h ;;sha1rnds4 ABCD, E0, 2 + db 0Fh,38H,0C9H,0E5h ;;sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + ;movdqu oword [rcx+16*14], ABCD + + ;; rounds 60-63 + db 0Fh,38H,0C8H,0D6h ;;sha1nexte E1, MSG3 + movdqa E0, ABCD + db 0Fh,38H,0CAH,0DEh ;;sha1msg2 MSG0, MSG3 + db 0Fh,3AH,0CCH,0C2H,03h ;;sha1rnds4 ABCD, E1, 3 + db 0Fh,38H,0C9H,0EEh ;;sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + ;movdqu oword [rcx+16*15], ABCD + + ;; rounds 64-67 + db 0Fh,38H,0C8H,0CBh ;;sha1nexte E0, MSG0 + movdqa E1, ABCD + db 0Fh,38H,0CAH,0E3h ;;sha1msg2 MSG1, MSG0 + db 0Fh,3AH,0CCH,0C1H,03h ;;sha1rnds4 ABCD, E0, 3 + db 0Fh,38H,0C9H,0F3h ;;sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + ;movdqu oword [rcx+16*16], ABCD + + ;; rounds 68-71 + db 0Fh,38H,0C8h,0D4h ;;sha1nexte E1, MSG1 + movdqa E0, ABCD + db 0Fh,38H,0CAh,0ECh ;;sha1msg2 MSG2, MSG1 + db 0Fh,3AH,0CCh,0C2h,03h ;;sha1rnds4 ABCD, E1, 3 + pxor MSG3, MSG1 + ;movdqu oword [rcx+16*17], ABCD + + ;; rounds 72-75 + db 0Fh,38H,0C8h,0CDh ;;sha1nexte E0, MSG2 + movdqa E1, ABCD + db 0Fh,38H,0CAh,0F5h ;;sha1msg2 MSG3, MSG2 + db 0Fh,3AH,0CCh,0C1h,03h ;;sha1rnds4 ABCD, E0, 3 + ;movdqu oword [rcx+16*18], ABCD + + ;; rounds 76-79 + db 0Fh,38H,0C8h,0D6h ;;sha1nexte E1, MSG3 + movdqa E0, ABCD + db 0Fh,3AH,0CCh,0C2h,03h ;;sha1rnds4 ABCD, E1, 3 + ;movdqu oword [rcx+16*19], ABCD + + ;; add current hash values with previously saved + db 0Fh,38H,0C8h,4Ch,24h,10h ;;sha1nexte E0, oword [e_save] + paddd ABCD, oword [abcd_save] + + add MSG_PTR, MBS_SHA1 + sub MSG_LEN, MBS_SHA1 + jg .sha1_block_loop + + ;; write hash values back in the correct order + pshufd ABCD, ABCD, 01Bh + movdqu oword [HASH_PTR], ABCD + pextrd dword [HASH_PTR+16], E0, 3 + +.quit: + REST_XMM + REST_GPR + ret +ENDFUNC UpdateSHA1ni + +%endif ;; VxWorks + +;;%endif ;; (_IPP32E >= _IPP32E_Y8) +%endif ;; _FEATURE_ON_ / _FEATURE_TICKTOCK_ +%endif ;; _ENABLE_ALG_SHA1_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha1u8as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha1u8as.asm new file mode 100644 index 0000000..014f145 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha1u8as.asm @@ -0,0 +1,518 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SHA1 +; +; Content: +; UpdateSHA1 +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SHA1_) +%if (_SHA_NI_ENABLING_ == _FEATURE_OFF_) || (_SHA_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if ((_IPP32E >= _IPP32E_U8 ) && (_IPP32E < _IPP32E_E9 )) || (_IPP32E == _IPP32E_N8 ) + +;; +;; SHA1 constants K[i] +%xdefine SHA1_K1 (05a827999h) +%xdefine SHA1_K2 (06ed9eba1h) +%xdefine SHA1_K3 (08f1bbcdch) +%xdefine SHA1_K4 (0ca62c1d6h) + +;; +;; Magic functions defined in FIPS 180-1 +;; +;; F1, F2, F3 and F4 assumes, that +;; - T1 returns function value +;; - T2 is the temporary +;; + +%macro F1 3.nolist + %xdefine %%B %1 + %xdefine %%C %2 + %xdefine %%D %3 + + mov T1,%%C + xor T1,%%D + and T1,%%B + xor T1,%%D +%endmacro + +%macro F2 3.nolist + %xdefine %%B %1 + %xdefine %%C %2 + %xdefine %%D %3 + + mov T1,%%D + xor T1,%%C + xor T1,%%B +%endmacro + +%macro F3 3.nolist + %xdefine %%B %1 + %xdefine %%C %2 + %xdefine %%D %3 + + mov T1,%%C + mov T2,%%B + or T1,%%B + and T2,%%C + and T1,%%D + or T1,T2 +%endmacro + +%macro F4 3.nolist + %xdefine %%B %1 + %xdefine %%C %2 + %xdefine %%D %3 + + F2 %%B,%%C,%%D +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; +;; rotations +;; + +%macro ROL_5 1.nolist + %xdefine %%x %1 + + rol %%x, 5 +%endmacro + +%macro ROL_30 1.nolist + %xdefine %%x %1 + + rol %%x, 30 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; +;; textual rotation of W array +;; +%macro ROTATE_W 0.nolist + %xdefine W_minus_32 W_minus_28 + %xdefine W_minus_28 W_minus_24 + %xdefine W_minus_24 W_minus_20 + %xdefine W_minus_20 W_minus_16 + %xdefine W_minus_16 W_minus_12 + %xdefine W_minus_12 W_minus_08 + %xdefine W_minus_08 W_minus_04 + %xdefine W_minus_04 W + %xdefine W W_minus_32 +%endmacro + +;; +;; SHA1 update round: +;; - F1 magic is used (and imbedded into the macros directly) +;; - 16 bytes of input are swapped +;; +%macro SHA1_UPDATE_RND_F1_BSWAP 7.nolist + %xdefine %%A %1 + %xdefine %%B %2 + %xdefine %%C %3 + %xdefine %%D %4 + %xdefine %%E %5 + %xdefine %%nr %6 + %xdefine %%Wchunk %7 + + pshufb %%Wchunk, XMM_SHUFB_BSWAP + movdqa W, %%Wchunk + paddd %%Wchunk, oword [K_XMM] + movdqa oword [rsp + (%%nr & 15)*4], %%Wchunk + mov T1,%%C ; F1 + mov T2,%%A + xor T1,%%D ; F1 + and T1,%%B ; F1 + ROL_5 T2 + xor T1,%%D ; F1 + add %%E, T2 + ROL_30 %%B + add T1, dword [rsp + (%%nr & 15)*4] + add %%E,T1 + + ROTATE_W +%endmacro + +;; +;; SHA1 update round: +;; - F1 magic is used (and imbedded into the macros directly) +;; +%macro SHA1_UPDATE_RND_F1 6.nolist + %xdefine %%A %1 + %xdefine %%B %2 + %xdefine %%C %3 + %xdefine %%D %4 + %xdefine %%E %5 + %xdefine %%nr %6 + + mov T1,%%C ; F1 + mov T2,%%A + xor T1,%%D ; F1 + ROL_5 T2 + and T1,%%B ; F1 + xor T1,%%D ; F1 + add %%E, T2 + ROL_30 %%B + add T1, dword [rsp + (%%nr & 15)*4] + add %%E,T1 +%endmacro + +;; +;; update W +;; +%macro W_CALC 1.nolist + %xdefine %%nr %1 + + %assign %%W_CALC_ahead 8 + + %assign %%i (%%nr + %%W_CALC_ahead) + + %if (%%i < 20) + %xdefine K_XMM K_BASE + %elif (%%i < 40) + %xdefine K_XMM K_BASE+16 + %elif (%%i < 60) + %xdefine K_XMM K_BASE+32 + %else + %xdefine K_XMM K_BASE+48 + %endif + + %if (%%i < 32) + %if ((%%i & 3) == 0) ;; just scheduling to interleave with ALUs + movdqa W, W_minus_12 + palignr W, W_minus_16, 8 ; w[i-14] + + movdqa W_TMP, W_minus_04 + psrldq W_TMP, 4 ; w[i-3] + + pxor W, W_minus_08 + %elif ((%%i & 3) == 1) + pxor W_TMP, W_minus_16 + pxor W, W_TMP + + movdqa W_TMP2, W + movdqa W_TMP, W + pslldq W_TMP2, 12 + %elif ((%%i & 3) == 2) + psrld W, 31 + pslld W_TMP, 1 + por W_TMP, W + + movdqa W, W_TMP2 + psrld W_TMP2, 30 + pslld W, 2 + %elif ((%%i & 3) == 3) + pxor W_TMP, W + pxor W_TMP, W_TMP2 + movdqa W, W_TMP + + paddd W_TMP, oword [K_XMM] + movdqa oword [rsp + ((%%i & (~3)) & 15)*4],W_TMP + + ROTATE_W + %endif + + %elif (%%i < 83) + %if ((%%i & 3) == 0) ;; scheduling to interleave with ALUs + movdqa W_TMP, W_minus_04 + pxor W, W_minus_28 ;; W == W_minus_32 + palignr W_TMP, W_minus_08, 8 + %elif ((%%i & 3) == 1) + pxor W, W_minus_16 + pxor W, W_TMP + movdqa W_TMP, W + %elif ((%%i & 3) == 2) + psrld W, 30 + pslld W_TMP, 2 + por W_TMP, W + %elif ((%%i & 3) == 3) + movdqa W, W_TMP + paddd W_TMP, oword [K_XMM] + movdqa oword [rsp + ((%%i & (~3)) & 15)*4],W_TMP + + ROTATE_W + %endif + + %endif +%endmacro + +;; +;; Regular hash update +;; +%macro SHA1_UPDATE_REGULAR 7.nolist + %xdefine %%A %1 + %xdefine %%B %2 + %xdefine %%C %3 + %xdefine %%D %4 + %xdefine %%E %5 + %xdefine %%nr %6 + %xdefine %%MagiF %7 + + W_CALC %%nr + + add %%E, dword [rsp + (%%nr & 15)*4] + %%MagiF %%B,%%C,%%D + add %%D, dword [rsp +((%%nr+1) & 15)*4] + ROL_30 %%B + mov T3,%%A + add %%E, T1 + ROL_5 T3 + add T3, %%E + mov %%E, T3 + + W_CALC %%nr+1 + + ROL_5 T3 + add %%D,T3 + %%MagiF %%A,%%B,%%C + add %%D, T1 + ROL_30 %%A + +; write: %1, %2 +; rotate: %1<=%4, %2<=%5, %3<=%1, %4<=%2, %5<=%3 +%endmacro + +;; update hash macro +%macro UPDATE_HASH 2.nolist + %xdefine %%hash0 %1 + %xdefine %%hashAdd %2 + + add %%hashAdd, %%hash0 + mov %%hash0, %%hashAdd +%endmacro + +segment .text align=IPP_ALIGN_FACTOR + + +align IPP_ALIGN_FACTOR + +K_XMM_AR dd SHA1_K1, SHA1_K1, SHA1_K1, SHA1_K1 + dd SHA1_K2, SHA1_K2, SHA1_K2, SHA1_K2 + dd SHA1_K3, SHA1_K3, SHA1_K3, SHA1_K3 + dd SHA1_K4, SHA1_K4, SHA1_K4, SHA1_K4 + +shuffle_mask DD 00010203h + DD 04050607h + DD 08090a0bh + DD 0c0d0e0fh + + +;***************************************************************************************** +;* Purpose: Update internal digest according to message block +;* +;* void UpdateSHA1(DigestSHA1 digest, const Ipp32u* mblk, int mlen, const void* pParam) +;* +;***************************************************************************************** + +;; +;; Lib = U8, N8 +;; +;; Caller = ippsSHA1Update +;; Caller = ippsSHA1Final +;; Caller = ippsSHA1MessageDigest +;; +;; Caller = ippsHMACSHA1Update +;; Caller = ippsHMACSHA1Final +;; Caller = ippsHMACSHA1MessageDigest +;; + +;; assign hash values to GPU registers +%xdefine A ecx +%xdefine B eax +%xdefine C edx +%xdefine D r8d +%xdefine E r9d + +;; temporary +%xdefine T1 r10d +%xdefine T2 r11d +%xdefine T3 r13d +%xdefine T4 r13d + +%xdefine W_TMP xmm0 +%xdefine W_TMP2 xmm1 + +%xdefine W0 xmm2 +%xdefine W4 xmm3 +%xdefine W8 xmm4 +%xdefine W12 xmm5 +%xdefine W16 xmm6 +%xdefine W20 xmm7 +%xdefine W24 xmm8 +%xdefine W28 xmm9 + +;; endianness swap constant +%xdefine XMM_SHUFB_BSWAP xmm10 + +;; K_BASE contains K_XMM_AR address +%xdefine K_BASE r12 + + +align IPP_ALIGN_FACTOR +IPPASM UpdateSHA1,PUBLIC +%assign LOCAL_FRAME (16*4) + USES_GPR rdi,rsi,r12,r13,r14 + USES_XMM xmm6,xmm7,xmm8,xmm9,xmm10 + COMP_ABI 4 + +%xdefine MBS_SHA1 (64) + + movsxd r14, edx + + movdqa XMM_SHUFB_BSWAP, oword [rel shuffle_mask] ; load shuffle mask + lea K_BASE, [rel K_XMM_AR] ; SHA1 const array address + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +.sha1_block_loop: + mov A, dword [rdi] ; load initial hash value + mov B, dword [rdi+4] + mov C, dword [rdi+8] + mov D, dword [rdi+12] + mov E, dword [rdi+16] + + movdqu W28, oword [rsi] ; load buffer content + movdqu W24, oword [rsi+16] + movdqu W20, oword [rsi+32] + movdqu W16, oword [rsi+48] + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;;SHA1_MAIN_BODY + +%xdefine W W0 +%xdefine W_minus_04 W4 +%xdefine W_minus_08 W8 +%xdefine W_minus_12 W12 +%xdefine W_minus_16 W16 +%xdefine W_minus_20 W20 +%xdefine W_minus_24 W24 +%xdefine W_minus_28 W28 +%xdefine W_minus_32 W + + ;; assignment +%xdefine K_XMM K_BASE + +;;F textequ + SHA1_UPDATE_RND_F1_BSWAP A,B,C,D,E, 0, W28 + SHA1_UPDATE_RND_F1 E,A,B,C,D, 1 + SHA1_UPDATE_RND_F1 D,E,A,B,C, 2 + SHA1_UPDATE_RND_F1 C,D,E,A,B, 3 + + SHA1_UPDATE_RND_F1_BSWAP B,C,D,E,A, 4, W24 + SHA1_UPDATE_RND_F1 A,B,C,D,E, 5 + SHA1_UPDATE_RND_F1 E,A,B,C,D, 6 + SHA1_UPDATE_RND_F1 D,E,A,B,C, 7 + + SHA1_UPDATE_RND_F1_BSWAP C,D,E,A,B, 8, W20 + SHA1_UPDATE_RND_F1 B,C,D,E,A, 9 + SHA1_UPDATE_RND_F1 A,B,C,D,E, 10 + SHA1_UPDATE_RND_F1 E,A,B,C,D, 11 + + SHA1_UPDATE_RND_F1_BSWAP D,E,A,B,C, 12, W16 + + W_CALC 8 + W_CALC 9 + W_CALC 10 + + SHA1_UPDATE_RND_F1 C,D,E,A,B, 13 + + W_CALC 11 + W_CALC 12 + + SHA1_UPDATE_RND_F1 B,C,D,E,A, 14 + + W_CALC 13 + W_CALC 14 + W_CALC 15 + + SHA1_UPDATE_RND_F1 A,B,C,D,E, 15 + + SHA1_UPDATE_REGULAR E,A,B,C,D,16, F1 + SHA1_UPDATE_REGULAR C,D,E,A,B,18, F1 + +;;F textequ + SHA1_UPDATE_REGULAR A,B,C,D,E,20, F2 + SHA1_UPDATE_REGULAR D,E,A,B,C,22, F2 + SHA1_UPDATE_REGULAR B,C,D,E,A,24, F2 + SHA1_UPDATE_REGULAR E,A,B,C,D,26, F2 + SHA1_UPDATE_REGULAR C,D,E,A,B,28, F2 + + SHA1_UPDATE_REGULAR A,B,C,D,E,30, F2 + SHA1_UPDATE_REGULAR D,E,A,B,C,32, F2 + SHA1_UPDATE_REGULAR B,C,D,E,A,34, F2 + SHA1_UPDATE_REGULAR E,A,B,C,D,36, F2 + SHA1_UPDATE_REGULAR C,D,E,A,B,38, F2 + +;;F textequ + SHA1_UPDATE_REGULAR A,B,C,D,E,40, F3 + SHA1_UPDATE_REGULAR D,E,A,B,C,42, F3 + SHA1_UPDATE_REGULAR B,C,D,E,A,44, F3 + SHA1_UPDATE_REGULAR E,A,B,C,D,46, F3 + SHA1_UPDATE_REGULAR C,D,E,A,B,48, F3 + + SHA1_UPDATE_REGULAR A,B,C,D,E,50, F3 + SHA1_UPDATE_REGULAR D,E,A,B,C,52, F3 + SHA1_UPDATE_REGULAR B,C,D,E,A,54, F3 + SHA1_UPDATE_REGULAR E,A,B,C,D,56, F3 + SHA1_UPDATE_REGULAR C,D,E,A,B,58, F3 + +;;F textequ + SHA1_UPDATE_REGULAR A,B,C,D,E,60, F4 + SHA1_UPDATE_REGULAR D,E,A,B,C,62, F4 + SHA1_UPDATE_REGULAR B,C,D,E,A,64, F4 + SHA1_UPDATE_REGULAR E,A,B,C,D,66, F4 + SHA1_UPDATE_REGULAR C,D,E,A,B,68, F4 + + SHA1_UPDATE_REGULAR A,B,C,D,E,70, F4 + SHA1_UPDATE_REGULAR D,E,A,B,C,72, F4 + SHA1_UPDATE_REGULAR B,C,D,E,A,74, F4 + SHA1_UPDATE_REGULAR E,A,B,C,D,76, F4 + SHA1_UPDATE_REGULAR C,D,E,A,B,78, F4 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + UPDATE_HASH dword [rdi], A + UPDATE_HASH dword [rdi+4], B + UPDATE_HASH dword [rdi+8], C + UPDATE_HASH dword [rdi+12],D + UPDATE_HASH dword [rdi+16],E + + add rsi, MBS_SHA1 + sub r14, MBS_SHA1 + jg .sha1_block_loop + + REST_XMM + REST_GPR + ret +ENDFUNC UpdateSHA1 + +%endif ;; ((_IPP32E >= _IPP32E_U8 ) AND (_IPP32E < _IPP32E_E9 )) OR (_IPP32E == _IPP32E_N8 ) +%endif ;; _FEATURE_OFF_ / _FEATURE_TICKTOCK_ +%endif ;; _ENABLE_ALG_SHA1_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha256e9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha256e9as.asm new file mode 100644 index 0000000..cb99c59 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha256e9as.asm @@ -0,0 +1,541 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SHA256 +; +; Content: +; UpdateSHA256 +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SHA256_) +%if (_SHA_NI_ENABLING_ == _FEATURE_OFF_) || (_SHA_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E == _IPP32E_E9 ) + +%xdefine W0 xmm0 +%xdefine W4 xmm1 +%xdefine W8 xmm2 +%xdefine W12 xmm3 +%xdefine SIG1 xmm4 +%xdefine SIG0 xmm5 +%xdefine X xmm6 +%xdefine W xmm7 +%xdefine XMM_SHUFB_BSWAP xmm6 + +;; assign hash values to GPU registers +%xdefine A eax +%xdefine B ebx +%xdefine C ecx +%xdefine D edx +%xdefine E r8d +%xdefine F r9d +%xdefine G r10d +%xdefine H r11d +%xdefine T1 r12d +%xdefine T2 r13d +%xdefine T3 r14d +%xdefine T4 r15d + +;; we are considering x, y, z are polynomials over GF(2) +;; & - multiplication +;; ^ - additive +;; operations + +;; +;; Chj(x,y,z) = (x&y) ^ (~x & z) +;; = (x&y) ^ ((1^x) &z) +;; = (x&y) ^ (z ^ x&z) +;; = x&y ^ z ^ x&z +;; = x&(y^z) ^z +;; +%macro Chj 4.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%Y %3 + %xdefine %%Z %4 + + mov %%F, %%Y + xor %%F, %%Z + and %%F, %%X + xor %%F, %%Z +%endmacro + +;; +;; Maj(x,y,z) = (x&y) ^ (x&z) ^ (y&z) +;; = (x&y) ^ (x&z) ^ (y&z) ^ (z&z) ^z // note: ((z&z) ^z) = 0 +;; = x&(y^z) ^ z&(y^z) ^z +;; = (x^z)&(y^z) ^z +;; +%macro Maj 4.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%Y %3 + %xdefine %%Z %4 + + mov %%F, %%X + xor %%F, %%Z + xor %%Y, %%Z + and %%F, %%Y + xor %%Y, %%Z + xor %%F, %%Z +%endmacro + +%macro ROTR 2.nolist + %xdefine %%X %1 + %xdefine %%n %2 + + shrd %%X,%%X, %%n +%endmacro + +;; +;; Summ0(x) = ROR(x,2) ^ ROR(x,13) ^ ROR(x,22) +;; +%macro Summ0 2.nolist + %xdefine %%F %1 + %xdefine %%X %2 + + mov %%F, %%X + ROTR %%F, (22-13) + xor %%F, %%X + ROTR %%F, (13-2) + xor %%F, %%X + ROTR %%F, 2 +%endmacro + +;; +;; Summ1(x) = ROR(x,6) ^ ROR(x,11) ^ ROR(x,25) +;; +%macro Summ1 2.nolist + %xdefine %%F %1 + %xdefine %%X %2 + + mov %%F, %%X + ROTR %%F, (25-11) + xor %%F, %%X + ROTR %%F, (11-6) + xor %%F, %%X + ROTR %%F, 6 +%endmacro + +;; +;; regular round (i): +;; +;; T1 = h + Sigma1(e) + Ch(e,f,g) + K[i] + W[i] +;; T2 = Sigma0(a) + Maj(a,b,c) +;; h = g +;; g = f +;; f = e +;; e = d + T1 +;; d = c +;; c = b +;; b = a +;; a = T1+T2 +;; +;; or +;; +;; h += Sigma1(e) + Ch(e,f,g) + K[i] + W[i] (==T1) +;; d += h +;; T2 = Sigma0(a) + Maj(a,b,c) +;; h += T2 +;; and following textual shift {a,b,c,d,e,f,g,h} => {h,a,b,c,d,e,f,g} +;; +%macro ROUND 11.nolist + %xdefine %%nr %1 + %xdefine %%A %2 + %xdefine %%B %3 + %xdefine %%C %4 + %xdefine %%D %5 + %xdefine %%E %6 + %xdefine %%F %7 + %xdefine %%G %8 + %xdefine %%H %9 + %xdefine %%T1 %10 + %xdefine %%T2 %11 + + add %%H, dword [rsp+(%%nr & 3)*sizeof(dword)] + Summ1 %%T1, %%E + Chj %%T2, %%E,%%F,%%G + add %%H, %%T1 + add %%H, %%T2 + + add %%D, %%H + + Summ0 %%T1, %%A + Maj %%T2, %%A,%%B,%%C + add %%H, %%T1 + add %%H, %%T2 +%endmacro + +%macro ROUND_v1 12.nolist + %xdefine %%nr %1 + %xdefine %%A %2 + %xdefine %%B %3 + %xdefine %%C %4 + %xdefine %%D %5 + %xdefine %%E %6 + %xdefine %%F %7 + %xdefine %%G %8 + %xdefine %%H %9 + %xdefine %%T1 %10 + %xdefine %%T2 %11 + %xdefine %%T3 %12 + + Summ1 %%T1, %%E ;; S1 + Chj %%T2, %%E,%%F,%%G ;; CH + add %%H, dword [rsp+(%%nr & 3)*sizeof(dword)] + add %%H, %%T1 + add %%H, %%T2 + + Maj %%T1, %%A,%%B,%%C ;; MAJ + Summ0 %%T2, %%A ;; S0 + + add %%D, %%H + + add %%H, %%T1 + add %%H, %%T2 +%endmacro + +;; W[i] = Sigma1(W[i-2]) + W[i-7] + Sigma0(W[i-15]) + W[i-16], i=16,..,63 +;; +;;for next rounds 16,17,18 and 19: +;; W[0] <= W[16] = Sigma1(W[14]) + W[ 9] + Sigma0(W[1]) + W[0] +;; W[1] <= W[17] = Sigma1(W[15]) + W[10] + Sigma0(W[2]) + W[1] +;; W[2] <= W[18] = Sigma1(W[ 0]) + W[11] + Sigma0(W[3]) + W[2] +;; W[3] <= W[19] = Sigma1(W[ 1]) + W[12] + Sigma0(W[4]) + W[3] +;; +;; the process is repeated exactly because texual round of W[] +;; +;; Sigma1() and Sigma0() functions are defined as following: +;; Sigma1(X) = ROR(X,17)^ROR(X,19)^SHR(X,10) +;; Sigma0(X) = ROR(X, 7)^ROR(X,18)^SHR(X, 3) +;; +%macro UPDATE_W 8.nolist + %xdefine %%xS0 %1 + %xdefine %%xS4 %2 + %xdefine %%xS8 %3 + %xdefine %%xS12 %4 + %xdefine %%SIGMA %5 + %xdefine %%xS %6 + %xdefine %%xT %7 + %xdefine %%phase %8 + +%if %%phase == 0 + ;; SIGMA0 + vpalignr %%xS, %%xS4, %%xS0, 4 ;; xS = {W04,W03,W02,W01} + vpsrld %%SIGMA, %%xS, 3 ;; SIGMA0 = SHR({W04,W03,W02,W01},3) + vpsrld %%xT, %%xS, 7 ;; SIGMA0 ^= SHR({W04,W03,W02,W01},7) + vpxor %%SIGMA, %%SIGMA, %%xT + vpsrld %%xT, %%xS, 18 ;; SIGMA0 ^= SHR({W04,W03,W02,W01},18) + vpxor %%SIGMA, %%SIGMA, %%xT + vpslld %%xT, %%xS, (32-18) ;; SIGMA0 ^= SHL({W04,W03,W02,W01},32-18) + vpxor %%SIGMA, %%SIGMA, %%xT + vpslld %%xT, %%xS, (32-7) ;; SIGMA0 ^= SHL({W04,W03,W02,W01},32-7) + vpxor %%SIGMA, %%SIGMA, %%xT +%endif +%if %%phase == 1 + vpalignr %%xS, %%xS12,%%xS8, 4 ;; xS = {W12,W11,W10,W09} + vpaddd %%xS0, %%xS0, %%xS ;; xS0 = xS0 + {W12,W11,W10,W09} + SIGMA0 + vpaddd %%xS0, %%xS0, %%SIGMA +%endif +%if %%phase == 2 + ;; SIGMA1 (low) + vpshufd %%xS, %%xS12, 11111010b ;; xS = {W[15],W[15],W[14],W[14]} + vpsrld %%SIGMA, %%xS, 10 ;; SIGMA1 = SHR({W[15],W[15],W[14],W[14]},10) + vpsrlq %%xT, %%xS, 17 ;; xT = SHR({W[15],W[15],W[14],W[14]},17) + vpsrlq %%xS, %%xS, 19 ;; xS = SHR({W[15],W[15],W[14],W[14]},19) + vpxor %%SIGMA, %%SIGMA, %%xT + vpxor %%SIGMA, %%SIGMA, %%xS ;; SIGMA1 ^= xT ^ xS + vpshufb %%SIGMA, %%SIGMA, [rel SHUFD_ZZ10] ;; SIGMA1 = {zzz,zzz,SIGMA1.1,SIGMA1.0} + + vpaddd %%xS0, %%xS0, %%SIGMA ;; {???,???,new W01,new W00} = {??,??,W17,W16} +%endif +%if %%phase == 3 + ;; SIGMA1 (hight) + vpshufd %%xS, %%xS0, 01010000b ;; xS = {W17,W17,W16,W16} + vpsrld %%SIGMA, %%xS, 10 ;; SIGMA1 = SHR({W[17],W[17],W[16],W[16]},10) + vpsrlq %%xT, %%xS, 17 ;; xT = ROTR({W[17],W[17],W[16],W[16]},17)- low 32 rotation in fact + vpsrlq %%xS, %%xS, 19 ;; xS = RORR({W[17],W[17],W[16],W[16]},19) - low 32 rotation in fact + vpxor %%SIGMA, %%SIGMA, %%xT + vpxor %%SIGMA, %%SIGMA, %%xS ;; SIGMA1 ^= xT ^ xS + vpshufb %%SIGMA, %%SIGMA, [rel SHUFD_32ZZ] ;; SIGMA1 = {SIGMA1.3,SIGMA1.2,zzz,zzz} + + vpaddd %%xS0, %%xS0, %%SIGMA ;; {new W03, new W02,new W01,new W00} = {W19,W18,W17,W16} +%endif +%endmacro + +segment .text align=IPP_ALIGN_FACTOR + + +align IPP_ALIGN_FACTOR +SHUFB_BSWAP DB 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12 +SHUFD_ZZ10 DB 0,1,2,3, 8,9,10,11, 0ffh,0ffh,0ffh,0ffh,0ffh,0ffh,0ffh,0ffh +SHUFD_32ZZ DB 0ffh,0ffh,0ffh,0ffh,0ffh,0ffh,0ffh,0ffh, 0,1,2,3, 8,9,10,11 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; UpdateSHA256(Ipp32u digest[], Ipp8u dataBlock[], int datalen, Ipp32u K_256[]) +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +align IPP_ALIGN_FACTOR +IPPASM UpdateSHA256,PUBLIC +%assign LOCAL_FRAME sizeof(oword)+sizeof(qword) + USES_GPR rbx,rsi,rdi,rbp,r12,r13,r14,r15 + USES_XMM xmm6,xmm7 + COMP_ABI 4 +;; +;; rdi = pointer to the updated hash +;; rsi = pointer to the data block +;; rdx = data block length +;; rcx = pointer to the SHA_256 constant +;; + +%xdefine MBS_SHA256 (64) + + movsxd rdx, edx + mov qword [rsp+sizeof(oword)], rdx ; save length of buffer + + mov rbp, rcx ; rbp points K_256[] constants + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +.sha256_block_loop: + vmovdqu W0, oword [rsi] ; load buffer content and swap + vpshufb W0, W0, [rel SHUFB_BSWAP] + vmovdqu W4, oword [rsi+sizeof(oword)] + vpshufb W4, W4, [rel SHUFB_BSWAP] + vmovdqu W8, oword [rsi+sizeof(oword)*2] + vpshufb W8, W8, [rel SHUFB_BSWAP] + vmovdqu W12,oword [rsi+sizeof(oword)*3] + vpshufb W12,W12, [rel SHUFB_BSWAP] + + mov A, dword [rdi] ; load initial hash value + mov B, dword [rdi+sizeof(dword)] + mov C, dword [rdi+sizeof(dword)*2] + mov D, dword [rdi+sizeof(dword)*3] + mov E, dword [rdi+sizeof(dword)*4] + mov F, dword [rdi+sizeof(dword)*5] + mov G, dword [rdi+sizeof(dword)*6] + mov H, dword [rdi+sizeof(dword)*7] + + ;; perform 0-3 regular rounds + vpaddd W, W0, oword [rbp+sizeof(oword)*0] ; T += K_SHA256[0-3] + vmovdqa oword [rsp], W + UPDATE_W W0,W4,W8,W12, SIG1,W,X, 0 ; update for round: 16-19 + ROUND 0, A,B,C,D,E,F,G,H, T1,T2 + UPDATE_W W0,W4,W8,W12, SIG1,W,X, 1 + ROUND 1, H,A,B,C,D,E,F,G, T1,T2 + UPDATE_W W0,W4,W8,W12, SIG1,W,X, 2 + ROUND 2, G,H,A,B,C,D,E,F, T1,T2 + UPDATE_W W0,W4,W8,W12, SIG1,W,X, 3 + ROUND 3, F,G,H,A,B,C,D,E, T1,T2 + + ;; perform next 4-7 regular rounds + vpaddd W, W4, oword [rbp+sizeof(oword)*1] ; T += K_SHA256[4-7] + vmovdqa oword [rsp], W + UPDATE_W W4,W8,W12,W0, SIG1,W,X, 0 ; update for round: 20-23 + ROUND 4, E,F,G,H,A,B,C,D, T1,T2 + UPDATE_W W4,W8,W12,W0, SIG1,W,X, 1 + ROUND 5, D,E,F,G,H,A,B,C, T1,T2 + UPDATE_W W4,W8,W12,W0, SIG1,W,X, 2 + ROUND 6, C,D,E,F,G,H,A,B, T1,T2 + UPDATE_W W4,W8,W12,W0, SIG1,W,X, 3 + ROUND 7, B,C,D,E,F,G,H,A, T1,T2 + + ;; perform next 8-11 regular rounds + vpaddd W, W8, oword [rbp+sizeof(oword)*2] ; T += K_SHA256[8-11] + vmovdqa oword [rsp], W + UPDATE_W W8,W12,W0,W4, SIG1,W,X, 0 ; update for round: 24-27 + ROUND 8, A,B,C,D,E,F,G,H, T1,T2 + UPDATE_W W8,W12,W0,W4, SIG1,W,X, 1 + ROUND 9, H,A,B,C,D,E,F,G, T1,T2 + UPDATE_W W8,W12,W0,W4, SIG1,W,X, 2 + ROUND 10, G,H,A,B,C,D,E,F, T1,T2 + UPDATE_W W8,W12,W0,W4, SIG1,W,X, 3 + ROUND 11, F,G,H,A,B,C,D,E, T1,T2 + + ;; perform next 12-15 regular rounds + vpaddd W, W12, oword [rbp+sizeof(oword)*3] ; T += K_SHA256[12-15] + vmovdqa oword [rsp], W + UPDATE_W W12,W0,W4,W8, SIG1,W,X, 0 ; update for round: 28-31 + ROUND 12, E,F,G,H,A,B,C,D, T1,T2 + UPDATE_W W12,W0,W4,W8, SIG1,W,X, 1 + ROUND 13, D,E,F,G,H,A,B,C, T1,T2 + UPDATE_W W12,W0,W4,W8, SIG1,W,X, 2 + ROUND 14, C,D,E,F,G,H,A,B, T1,T2 + UPDATE_W W12,W0,W4,W8, SIG1,W,X, 3 + ROUND 15, B,C,D,E,F,G,H,A, T1,T2 + + ;; perform next 16-19 regular rounds + vpaddd W, W0, oword [rbp+sizeof(oword)*4] ; T += K_SHA256[16-19] + vmovdqa oword [rsp], W + UPDATE_W W0,W4,W8,W12, SIG1,W,X, 0 ; update for round: 32-35 + ROUND 16, A,B,C,D,E,F,G,H, T1,T2 + UPDATE_W W0,W4,W8,W12, SIG1,W,X, 1 + ROUND 17, H,A,B,C,D,E,F,G, T1,T2 + UPDATE_W W0,W4,W8,W12, SIG1,W,X, 2 + ROUND 18, G,H,A,B,C,D,E,F, T1,T2 + UPDATE_W W0,W4,W8,W12, SIG1,W,X, 3 + ROUND 19, F,G,H,A,B,C,D,E, T1,T2 + + ;; perform next 20-23 regular rounds + vpaddd W, W4, oword [rbp+sizeof(oword)*5] ; T += K_SHA256[20-23] + vmovdqa oword [rsp], W + UPDATE_W W4,W8,W12,W0, SIG1,W,X, 0 ; update for round: 36-39 + ROUND 20, E,F,G,H,A,B,C,D, T1,T2 + UPDATE_W W4,W8,W12,W0, SIG1,W,X, 1 + ROUND 21, D,E,F,G,H,A,B,C, T1,T2 + UPDATE_W W4,W8,W12,W0, SIG1,W,X, 2 + ROUND 22, C,D,E,F,G,H,A,B, T1,T2 + UPDATE_W W4,W8,W12,W0, SIG1,W,X, 3 + ROUND 23, B,C,D,E,F,G,H,A, T1,T2 + + ;; perform next 24-27 regular rounds + vpaddd W, W8, oword [rbp+sizeof(oword)*6] ; T += K_SHA256[24-27] + vmovdqa oword [rsp], W + UPDATE_W W8,W12,W0,W4, SIG1,W,X, 0 ; update for round: 40-43 + ROUND 24, A,B,C,D,E,F,G,H, T1,T2 + UPDATE_W W8,W12,W0,W4, SIG1,W,X, 1 + ROUND 25, H,A,B,C,D,E,F,G, T1,T2 + UPDATE_W W8,W12,W0,W4, SIG1,W,X, 2 + ROUND 26, G,H,A,B,C,D,E,F, T1,T2 + UPDATE_W W8,W12,W0,W4, SIG1,W,X, 3 + ROUND 27, F,G,H,A,B,C,D,E, T1,T2 + + ;; perform next 28-31 regular rounds + vpaddd W, W12, oword [rbp+sizeof(oword)*7] ; T += K_SHA256[28-31] + vmovdqa oword [rsp], W + UPDATE_W W12,W0,W4,W8, SIG1,W,X, 0 ; update for round: 44-47 + ROUND 28, E,F,G,H,A,B,C,D, T1,T2 + UPDATE_W W12,W0,W4,W8, SIG1,W,X, 1 + ROUND 29, D,E,F,G,H,A,B,C, T1,T2 + UPDATE_W W12,W0,W4,W8, SIG1,W,X, 2 + ROUND 30, C,D,E,F,G,H,A,B, T1,T2 + UPDATE_W W12,W0,W4,W8, SIG1,W,X, 3 + ROUND 31, B,C,D,E,F,G,H,A, T1,T2 + + ;; perform next 32-35 regular rounds + vpaddd W, W0, oword [rbp+sizeof(oword)*8] ; T += K_SHA256[32-35] + vmovdqa oword [rsp], W + UPDATE_W W0,W4,W8,W12, SIG1,W,X, 0 ; update for round: 48-51 + ROUND 32, A,B,C,D,E,F,G,H, T1,T2 + UPDATE_W W0,W4,W8,W12, SIG1,W,X, 1 + ROUND 33, H,A,B,C,D,E,F,G, T1,T2 + UPDATE_W W0,W4,W8,W12, SIG1,W,X, 2 + ROUND 34, G,H,A,B,C,D,E,F, T1,T2 + UPDATE_W W0,W4,W8,W12, SIG1,W,X, 3 + ROUND 35, F,G,H,A,B,C,D,E, T1,T2 + + ;; perform next 36-39 regular rounds + vpaddd W, W4, oword [rbp+sizeof(oword)*9] ; T += K_SHA256[36-39] + vmovdqa oword [rsp], W + UPDATE_W W4,W8,W12,W0, SIG1,W,X, 0 ; update for round: 52-55 + ROUND 36, E,F,G,H,A,B,C,D, T1,T2 + UPDATE_W W4,W8,W12,W0, SIG1,W,X, 1 + ROUND 37, D,E,F,G,H,A,B,C, T1,T2 + UPDATE_W W4,W8,W12,W0, SIG1,W,X, 2 + ROUND 38, C,D,E,F,G,H,A,B, T1,T2 + UPDATE_W W4,W8,W12,W0, SIG1,W,X, 3 + ROUND 39, B,C,D,E,F,G,H,A, T1,T2 + + ;; perform next 40-43 regular rounds + vpaddd W, W8, oword [rbp+sizeof(oword)*10] ; T += K_SHA256[40-43] + vmovdqa oword [rsp], W + UPDATE_W W8,W12,W0,W4, SIG1,W,X, 0 ; update for round: 56-59 + ROUND 40, A,B,C,D,E,F,G,H, T1,T2 + UPDATE_W W8,W12,W0,W4, SIG1,W,X, 1 + ROUND 41, H,A,B,C,D,E,F,G, T1,T2 + UPDATE_W W8,W12,W0,W4, SIG1,W,X, 2 + ROUND 42, G,H,A,B,C,D,E,F, T1,T2 + UPDATE_W W8,W12,W0,W4, SIG1,W,X, 3 + ROUND 43, F,G,H,A,B,C,D,E, T1,T2 + + ;; perform next 44-47 regular rounds + vpaddd W, W12, oword [rbp+sizeof(oword)*11] ; T += K_SHA256[44-47] + vmovdqa oword [rsp], W + UPDATE_W W12,W0,W4,W8, SIG1,W,X, 0 ; update for round: 60-63 + ROUND 44, E,F,G,H,A,B,C,D, T1,T2 + UPDATE_W W12,W0,W4,W8, SIG1,W,X, 1 + ROUND 45, D,E,F,G,H,A,B,C, T1,T2 + UPDATE_W W12,W0,W4,W8, SIG1,W,X, 2 + ROUND 46, C,D,E,F,G,H,A,B, T1,T2 + UPDATE_W W12,W0,W4,W8, SIG1,W,X, 3 + ROUND 47, B,C,D,E,F,G,H,A, T1,T2 + + ;; perform next 48-51 regular rounds + vpaddd W, W0, oword [rbp+sizeof(oword)*12] ; T += K_SHA256[48-51] + vmovdqa oword [rsp], W + ROUND 48, A,B,C,D,E,F,G,H, T1,T2 + ROUND 49, H,A,B,C,D,E,F,G, T1,T2 + ROUND 50, G,H,A,B,C,D,E,F, T1,T2 + ROUND 51, F,G,H,A,B,C,D,E, T1,T2 + + ;; perform next 52-55 regular rounds + vpaddd W, W4, oword [rbp+sizeof(oword)*13] ; T += K_SHA256[52-55] + vmovdqa oword [rsp], W + ROUND 52, E,F,G,H,A,B,C,D, T1,T2 + ROUND 53, D,E,F,G,H,A,B,C, T1,T2 + ROUND 54, C,D,E,F,G,H,A,B, T1,T2 + ROUND 55, B,C,D,E,F,G,H,A, T1,T2 + + ;; perform next 56-59 regular rounds + vpaddd W, W8, oword [rbp+sizeof(oword)*14] ; T += K_SHA256[56-59] + vmovdqa oword [rsp], W + ROUND 56, A,B,C,D,E,F,G,H, T1,T2 + ROUND 57, H,A,B,C,D,E,F,G, T1,T2 + ROUND 58, G,H,A,B,C,D,E,F, T1,T2 + ROUND 59, F,G,H,A,B,C,D,E, T1,T2 + + ;; perform next 60-63 regular rounds + vpaddd W, W12, oword [rbp+sizeof(oword)*15] ; T += K_SHA256[60-63] + vmovdqa oword [rsp], W + ROUND 60, E,F,G,H,A,B,C,D, T1,T2 + ROUND 61, D,E,F,G,H,A,B,C, T1,T2 + ROUND 62, C,D,E,F,G,H,A,B, T1,T2 + ROUND 63, B,C,D,E,F,G,H,A, T1,T2 + + add dword [rdi], A ; update shash + add dword [rdi+sizeof(dword)*1], B + add dword [rdi+sizeof(dword)*2], C + add dword [rdi+sizeof(dword)*3], D + add dword [rdi+sizeof(dword)*4], E + add dword [rdi+sizeof(dword)*5], F + add dword [rdi+sizeof(dword)*6], G + add dword [rdi+sizeof(dword)*7], H + + add rsi, MBS_SHA256 + sub qword [rsp+sizeof(oword)], MBS_SHA256 + jg .sha256_block_loop + + REST_XMM + REST_GPR + ret +ENDFUNC UpdateSHA256 + +%endif ;; _IPP32E_E9 and above +%endif ;; _FEATURE_OFF_ / _FEATURE_TICKTOCK_ +%endif ;; _ENABLE_ALG_SHA256_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha256l9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha256l9as.asm new file mode 100644 index 0000000..7f2ade9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha256l9as.asm @@ -0,0 +1,629 @@ +;=============================================================================== +; Copyright (C) 2017 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SHA256 +; +; Content: +; UpdateSHA256 +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "ia_32e_regs.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SHA256_) +%if (_SHA_NI_ENABLING_ == _FEATURE_OFF_) || (_SHA_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_L9 ) + +;; +;; assignments +;; +%xdefine hA eax ;; hash values into GPU registers +%xdefine hB ebx +%xdefine hC ecx +%xdefine hD edx +%xdefine hE r8d +%xdefine hF r9d +%xdefine hG r10d +%xdefine hH r11d + +%xdefine T1 r12d ;; scratch +%xdefine T2 r13d +%xdefine T3 r14d +%xdefine T4 r15d +%xdefine T5 edi + +%xdefine W0 ymm0 ;; W values into YMM registers +%xdefine W1 ymm1 +%xdefine W2 ymm2 +%xdefine W3 ymm3 + +%xdefine yT1 ymm4 ;; scratch +%xdefine yT2 ymm5 +%xdefine yT3 ymm6 +%xdefine yT4 ymm7 + +%xdefine W0L xmm0 +%xdefine W1L xmm1 +%xdefine W2L xmm2 +%xdefine W3L xmm3 + +%xdefine YMM_zzBA ymm8 ;; byte swap constant +%xdefine YMM_DCzz ymm9 ;; byte swap constant +%xdefine YMM_SHUFB_BSWAP ymm10 ;; byte swap constant + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; +;; textual rotation of W args +;; +%macro ROTATE_W 0.nolist + %xdefine %%_X W0 + %xdefine W0 W1 + %xdefine W1 W2 + %xdefine W2 W3 + %xdefine W3 %%_X +%endmacro + +;; +;; textual rotation of HASH arguments +;; +%macro ROTATE_H 0.nolist + %xdefine %%_X hH + %xdefine hH hG + %xdefine hG hF + %xdefine hF hE + %xdefine hE hD + %xdefine hD hC + %xdefine hC hB + %xdefine hB hA + %xdefine hA %%_X +%endmacro + +%macro ROTATE_T4_T5 0.nolist + %xdefine %%T T4 + %xdefine T4 T5 + %xdefine T5 %%T +%endmacro + +;; +;; compute next 4 W[t], W[t+1], W[t+2] and W[t+3], t=16,...63 +;; (see pcpsha256e9as.asm for details) +%macro UPDATE_W 5.nolist + %xdefine %%nr %1 + %xdefine %%W0 %2 + %xdefine %%W1 %3 + %xdefine %%W2 %4 + %xdefine %%W3 %5 + + %assign %%W_AHEAD 16 + + vpalignr yT3,%%W1,%%W0,4 + vpalignr yT2,%%W3,%%W2,4 + vpsrld yT1,yT3,7 + vpaddd %%W0,%%W0,yT2 + vpsrld yT2,yT3,3 + vpslld yT4,yT3,14 + vpxor yT3,yT2,yT1 + vpshufd yT2,%%W3,250 + vpsrld yT1,yT1,11 + vpxor yT3,yT3,yT4 + vpslld yT4,yT4,11 + vpxor yT3,yT3,yT1 + vpsrld yT1,yT2,10 + vpxor yT3,yT3,yT4 + vpsrlq yT2,yT2,17 + vpaddd %%W0,%%W0,yT3 + vpxor yT1,yT1,yT2 + vpsrlq yT2,yT2,2 + vpxor yT1,yT1,yT2 + vpshufb yT1,yT1,YMM_zzBA + vpaddd %%W0,%%W0,yT1 + vpshufd yT2,%%W0,80 + vpsrld yT1,yT2,10 + vpsrlq yT2,yT2,17 + vpxor yT1,yT1,yT2 + vpsrlq yT2,yT2,2 + vpxor yT1,yT1,yT2 + vpshufb yT1,yT1,YMM_DCzz + vpaddd %%W0,%%W0,yT1 + vpaddd yT1,%%W0,YMMWORD [rbp+(%%nr/4)*sizeof(ymmword)] + vmovdqa YMMWORD [rsi+(%%W_AHEAD/4)*sizeof(ymmword)+(%%nr/4)*sizeof(ymmword)],yT1 +%endmacro + +;; +;; regular round (i): +;; +;; T1 = h + Sum1(e) + Ch(e,f,g) + K[i] + W[i] +;; T2 = Sum0(a) + Maj(a,b,c) +;; h = g +;; g = f +;; f = e +;; e = d + T1 +;; d = c +;; c = b +;; b = a +;; a = T1+T2 +;; +;; sum1(e) = (e>>>25)^(e>>>11)^(e>>>6) +;; sum0(a) = (a>>>13)^(a>>>22)^(a>>>2) +;; ch(e,f,g) = (e&f)^(~e^g) +;; maj(a,b,m)= (a&b)^(a&c)^(b&c) +;; +;; note: +;; 1) U + ch(e,f,g) = U + (e&f) & (~e&g) +;; 2) maj(a,b,c)= (a&b)^(a&c)^(b&c) = (a^b)&(b^c) ^b +;; to make sure both are correct - use GF(2) arith instead of logic +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; or +;; X = sum0(a[i-1]) computed on prev round +;; a[i] += X +;; h[i] += (K[i]+W[i]) + sum1(e[i]) + ch(e[i],f[i],g[i]) or +;; h[i] += (K[i]+W[i]) + sum1(e[i]) + (e[i]&f[i]) + (~e[i]&g[i]) -- helps break dependencies +;; d[i] += h[i] +;; h[i] += maj(a[i],b[i],c[i]) +;; and following textual shift +;; {a[i+1],b[i+1],c[i+1],d[i+1],e[i+1],f[i+1],g[i+1],h[i+1]} <= {h[i],a[i],b[i],c[i],d[i],e[i],f[i],g[i]} +;; +;; on entry: +;; - T1 = f +;; - T3 = sum0{a[i-1]) +;; - T5 = b&c +%macro SHA256_ROUND 9.nolist + %xdefine %%nr %1 + %xdefine %%hA %2 + %xdefine %%hB %3 + %xdefine %%hC %4 + %xdefine %%hD %5 + %xdefine %%hE %6 + %xdefine %%hF %7 + %xdefine %%hG %8 + %xdefine %%hH %9 + + add %%hH, dword [rsi+(%%nr/4)*sizeof(ymmword)+(%%nr & 3)*sizeof(dword)] ;; h += (k[t]+w[t]) + and T1, %%hE ;; ch(e,f,g): (f&e) + rorx T2, %%hE, 25 ;; sum1(e): e>>>25 + rorx T4, %%hE, 11 ;; sum1(e): e>>>11 + add %%hA, T3 ;; complete computation a += sum0(a[t-1]) + add %%hH, T1 ;; h += (k[t]+w[t]) + (f&e) + andn T1, %%hE, %%hG ;; ch(e,f,g): (~e&g) + xor T2, T4 ;; sum1(e): (e>>>25)^(e>>>11) + rorx T3, %%hE, 6 ;; sum1(e): e>>>6 + add %%hH, T1 ;; h += (k[t]+w[t]) + (f&e) + (~e&g) + xor T2, T3 ;; sum1(e) = (e>>>25)^(e>>>11)^(e>>>6) + mov T4, %%hA ;; maj(a,b,c): a + rorx T1, %%hA, 22 ;; sum0(a): a>>>22 + add %%hH, T2 ;; h += (k[t]+w[t]) +(f&e) +(~e&g) +sig1(e) + xor T4, %%hB ;; maj(a,b,c): (a^b) + rorx T3, %%hA, 13 ;; sum0(a): a>>>13 + rorx T2, %%hA, 2 ;; sum0(a): a>>>2 + add %%hD, %%hH ;; d += h + and T5, T4 ;; maj(a,b,c): (b^c)&(a^b) + xor T3, T1 ;; sum0(a): (a>>>13)^(a>>>22) + xor T5, %%hB ;; maj(a,b,c) = (b^c)&(a^b)^b = (a&b)^(a&c)^(b&c) + xor T3, T2 ;; sum0(a): = (a>>>13)^(a>>>22)^(a>>>2) + add %%hH, T5 ;; h += (k[t]+w[t]) +(f&e) +(~e&g) +sig1(e) +maj(a,b,c) + mov T1, %%hE ;; T1 = f (next round) + ROTATE_T4_T5 ;; T5 = (b^c) (next round) +%endmacro + +;; +;; does 4 regular rounds and computes next 4 W values +;; (just 4 instances of SHA256_ROUND merged together woth UPDATE_W) +;; +%macro SHA256_4ROUND_SHED 1.nolist + %xdefine %%round %1 + + %assign %%W_AHEAD 16 +vpalignr yT3,W1,W0,4 + %assign %%nr %%round + add hH, dword [rsi+(%%nr/4)*sizeof(ymmword)+(%%nr & 3)*sizeof(dword)] + and T1, hE + rorx T2, hE, 25 +vpalignr yT2,W3,W2,4 + rorx T4, hE, 11 + add hA, T3 + add hH, T1 +vpsrld yT1,yT3,7 + andn T1, hE, hG + xor T2, T4 + rorx T3, hE, 6 +vpaddd W0,W0,yT2 + add hH, T1 + xor T2, T3 + mov T4, hA +vpsrld yT2,yT3,3 + rorx T1, hA, 22 + add hH, T2 + xor T4, hB +vpslld yT4,yT3,14 + rorx T3, hA, 13 + rorx T2, hA, 2 + add hD, hH +vpxor yT3,yT2,yT1 + and T5, T4 + xor T3, T1 + xor T5, hB +vpshufd yT2,W3,250 + xor T3, T2 + add hH, T5 + mov T1, hE + ROTATE_T4_T5 + ROTATE_H + +vpsrld yT1,yT1,11 + %assign %%nr %%nr+1 + add hH, dword [rsi+(%%nr/4)*sizeof(ymmword)+(%%nr & 3)*sizeof(dword)] + and T1, hE + rorx T2, hE, 25 +vpxor yT3,yT3,yT4 + rorx T4, hE, 11 + add hA, T3 + add hH, T1 +vpslld yT4,yT4,11 + andn T1, hE, hG + xor T2, T4 + rorx T3, hE, 6 +vpxor yT3,yT3,yT1 + add hH, T1 + xor T2, T3 + mov T4, hA +vpsrld yT1,yT2,10 + rorx T1, hA, 22 + add hH, T2 + xor T4, hB +vpxor yT3,yT3,yT4 + rorx T3, hA, 13 + rorx T2, hA, 2 + add hD, hH +vpsrlq yT2,yT2,17 + and T5, T4 + xor T3, T1 + xor T5, hB +vpaddd W0,W0,yT3 + xor T3, T2 + add hH, T5 + mov T1, hE + ROTATE_T4_T5 + ROTATE_H + +vpxor yT1,yT1,yT2 + %assign %%nr %%nr+1 + add hH, dword [rsi+(%%nr/4)*sizeof(ymmword)+(%%nr & 3)*sizeof(dword)] + and T1, hE + rorx T2, hE, 25 +vpsrlq yT2,yT2,2 + rorx T4, hE, 11 + add hA, T3 + add hH, T1 +vpxor yT1,yT1,yT2 + andn T1, hE, hG + xor T2, T4 + rorx T3, hE, 6 +vpshufb yT1,yT1,YMM_zzBA + add hH, T1 + xor T2, T3 + mov T4, hA +vpaddd W0,W0,yT1 + rorx T1, hA, 22 + add hH, T2 + xor T4, hB +vpshufd yT2,W0,80 + rorx T3, hA, 13 + rorx T2, hA, 2 + add hD, hH +vpsrld yT1,yT2,10 + and T5, T4 + xor T3, T1 + xor T5, hB +vpsrlq yT2,yT2,17 + xor T3, T2 + add hH, T5 + mov T1, hE + ROTATE_T4_T5 + ROTATE_H + +vpxor yT1,yT1,yT2 + %assign %%nr %%nr+1 + add hH, dword [rsi+(%%nr/4)*sizeof(ymmword)+(%%nr & 3)*sizeof(dword)] + and T1, hE + rorx T2, hE, 25 +vpsrlq yT2,yT2,2 + rorx T4, hE, 11 + add hA, T3 + add hH, T1 +vpxor yT1,yT1,yT2 + andn T1, hE, hG + xor T2, T4 + rorx T3, hE, 6 +vpshufb yT1,yT1,YMM_DCzz + add hH, T1 + xor T2, T3 + mov T4, hA +vpaddd W0,W0,yT1 + rorx T1, hA, 22 + add hH, T2 + xor T4, hB +vpaddd yT1,W0,YMMWORD [rbp+(%%nr/4)*sizeof(ymmword)] + rorx T3, hA, 13 + rorx T2, hA, 2 + add hD, hH + and T5, T4 + xor T3, T1 + xor T5, hB +vmovdqa YMMWORD [rsi+(%%W_AHEAD/4)*sizeof(ymmword)+(%%round/4)*sizeof(ymmword)],yT1 + xor T3, T2 + add hH, T5 + mov T1, hE + ROTATE_T4_T5 + ROTATE_H + + ROTATE_W +%endmacro + +;; +;; update hash +;; +%macro UPDATE_HASH 2.nolist + %xdefine %%hashMem %1 + %xdefine %%hash %2 + + add %%hash, %%hashMem + mov %%hashMem, %%hash +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +segment .text align=IPP_ALIGN_FACTOR + + +align IPP_ALIGN_FACTOR + +SHA256_YMM_K dd 0428a2f98h, 071374491h, 0b5c0fbcfh, 0e9b5dba5h, 0428a2f98h, 071374491h, 0b5c0fbcfh, 0e9b5dba5h + dd 03956c25bh, 059f111f1h, 0923f82a4h, 0ab1c5ed5h, 03956c25bh, 059f111f1h, 0923f82a4h, 0ab1c5ed5h + dd 0d807aa98h, 012835b01h, 0243185beh, 0550c7dc3h, 0d807aa98h, 012835b01h, 0243185beh, 0550c7dc3h + dd 072be5d74h, 080deb1feh, 09bdc06a7h, 0c19bf174h, 072be5d74h, 080deb1feh, 09bdc06a7h, 0c19bf174h + dd 0e49b69c1h, 0efbe4786h, 00fc19dc6h, 0240ca1cch, 0e49b69c1h, 0efbe4786h, 00fc19dc6h, 0240ca1cch + dd 02de92c6fh, 04a7484aah, 05cb0a9dch, 076f988dah, 02de92c6fh, 04a7484aah, 05cb0a9dch, 076f988dah + dd 0983e5152h, 0a831c66dh, 0b00327c8h, 0bf597fc7h, 0983e5152h, 0a831c66dh, 0b00327c8h, 0bf597fc7h + dd 0c6e00bf3h, 0d5a79147h, 006ca6351h, 014292967h, 0c6e00bf3h, 0d5a79147h, 006ca6351h, 014292967h + dd 027b70a85h, 02e1b2138h, 04d2c6dfch, 053380d13h, 027b70a85h, 02e1b2138h, 04d2c6dfch, 053380d13h + dd 0650a7354h, 0766a0abbh, 081c2c92eh, 092722c85h, 0650a7354h, 0766a0abbh, 081c2c92eh, 092722c85h + dd 0a2bfe8a1h, 0a81a664bh, 0c24b8b70h, 0c76c51a3h, 0a2bfe8a1h, 0a81a664bh, 0c24b8b70h, 0c76c51a3h + dd 0d192e819h, 0d6990624h, 0f40e3585h, 0106aa070h, 0d192e819h, 0d6990624h, 0f40e3585h, 0106aa070h + dd 019a4c116h, 01e376c08h, 02748774ch, 034b0bcb5h, 019a4c116h, 01e376c08h, 02748774ch, 034b0bcb5h + dd 0391c0cb3h, 04ed8aa4ah, 05b9cca4fh, 0682e6ff3h, 0391c0cb3h, 04ed8aa4ah, 05b9cca4fh, 0682e6ff3h + dd 0748f82eeh, 078a5636fh, 084c87814h, 08cc70208h, 0748f82eeh, 078a5636fh, 084c87814h, 08cc70208h + dd 090befffah, 0a4506cebh, 0bef9a3f7h, 0c67178f2h, 090befffah, 0a4506cebh, 0bef9a3f7h, 0c67178f2h + +SHA256_YMM_BF dd 000010203h, 004050607h, 008090a0bh, 00c0d0e0fh, 000010203h, 004050607h, 008090a0bh, 00c0d0e0fh + +SHA256_DCzz db 0ffh,0ffh,0ffh,0ffh,0ffh,0ffh,0ffh,0ffh, 0,1,2,3, 8,9,10,11 + db 0ffh,0ffh,0ffh,0ffh,0ffh,0ffh,0ffh,0ffh, 0,1,2,3, 8,9,10,11 +SHA256_zzBA db 0,1,2,3, 8,9,10,11, 0ffh,0ffh,0ffh,0ffh,0ffh,0ffh,0ffh,0ffh + db 0,1,2,3, 8,9,10,11, 0ffh,0ffh,0ffh,0ffh,0ffh,0ffh,0ffh,0ffh + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; UpdateSHA256(Ipp32u digest[], Ipp8u dataBlock[], int datalen, Ipp32u K_256[]) +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +align IPP_ALIGN_FACTOR +IPPASM UpdateSHA256,PUBLIC +%assign LOCAL_FRAME (sizeof(qword)*4 + sizeof(dword)*64*2) + USES_GPR rbx,rsi,rdi,rbp,rbx,r12,r13,r14,r15 + USES_XMM_AVX xmm6,xmm7,xmm8,xmm9,xmm10 + COMP_ABI 4 +;; +;; rdi = pointer to the updated hash +;; rsi = pointer to the data block +;; rdx = data block length +;; rcx = pointer to the SHA_256 constant (ignored) +;; + +%xdefine MBS_SHA256 (64) + +;; +;; stack structure: +;; +%assign _block 0 ;; block address +%assign _hash _block+sizeof(qword) ;; hash address +%assign _len _hash+sizeof(qword) ;; rest of processed data +%assign _frame _len+sizeof(qword) ;; rsp before alignment +%assign _dataW _frame+sizeof(qword) ;; W[t] values + + + mov r15, rsp ; store orifinal rsp + and rsp, -IPP_ALIGN_FACTOR ; 32-byte aligned stack + + movsxd r14, edx ; input length in bytes + + mov qword [rsp+_hash], rdi ; store hash address + mov qword [rsp+_len], r14 ; store length + mov qword [rsp+_frame], r15 ; store rsp + + mov hA, dword [rdi] ; load initial hash value + mov hB, dword [rdi+1*sizeof(dword)] + mov hC, dword [rdi+2*sizeof(dword)] + mov hD, dword [rdi+3*sizeof(dword)] + mov hE, dword [rdi+4*sizeof(dword)] + mov hF, dword [rdi+5*sizeof(dword)] + mov hG, dword [rdi+6*sizeof(dword)] + mov hH, dword [rdi+7*sizeof(dword)] + + vmovdqa YMM_SHUFB_BSWAP, ymmword [rel SHA256_YMM_BF] ; load byte shuffler + + vmovdqa YMM_zzBA, ymmword [rel SHA256_zzBA] ; load byte shuffler (zzBA) + vmovdqa YMM_DCzz, ymmword [rel SHA256_DCzz] ; load byte shuffler (DCzz) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data 2 block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +align IPP_ALIGN_FACTOR +.sha256_block2_loop: + lea r12, [rsi+MBS_SHA256] ; next block + + cmp r14, MBS_SHA256 ; %if single block processed + cmovbe r12, rsi ; use the same data block address + lea rbp, [rel SHA256_YMM_K] ; to SHA256 consts + + vmovdqu W0L, xmmword [rsi] ; load data block + vmovdqu W1L, xmmword [rsi+1*sizeof(xmmword)] + vmovdqu W2L, xmmword [rsi+2*sizeof(xmmword)] + vmovdqu W3L, xmmword [rsi+3*sizeof(xmmword)] + + vinserti128 W0, W0, xmmword [r12], 1 ; merge next data block + vinserti128 W1, W1, xmmword [r12+1*sizeof(xmmword)], 1 + vinserti128 W2, W2, xmmword [r12+2*sizeof(xmmword)], 1 + vinserti128 W3, W3, xmmword [r12+3*sizeof(xmmword)], 1 + + vpshufb W0, W0, YMM_SHUFB_BSWAP + vpshufb W1, W1, YMM_SHUFB_BSWAP + vpshufb W2, W2, YMM_SHUFB_BSWAP + vpshufb W3, W3, YMM_SHUFB_BSWAP + vpaddd yT1, W0, ymmword [rbp] + vpaddd yT2, W1, ymmword [rbp+1*sizeof(ymmword)] + vpaddd yT3, W2, ymmword [rbp+2*sizeof(ymmword)] + vpaddd yT4, W3, ymmword [rbp+3*sizeof(ymmword)] + add rbp, 4*sizeof(ymmword) + vmovdqa ymmword [rsp+_dataW], yT1 + vmovdqa ymmword [rsp+_dataW+1*sizeof(ymmword)], yT2 + vmovdqa ymmword [rsp+_dataW+2*sizeof(ymmword)], yT3 + vmovdqa ymmword [rsp+_dataW+3*sizeof(ymmword)], yT4 + + mov T5, hB ; T5 = b^c + xor T3, T3 ; T3 = 0 + mov T1, hF ; T1 = f + xor T5, hC + + mov qword [rsp+_block], rsi ; store block addres + lea rsi, [rsp+_dataW] ; use rsi as stack pointer + +align IPP_ALIGN_FACTOR +.block1_shed_proc: + SHA256_4ROUND_SHED 0 + SHA256_4ROUND_SHED 4 + SHA256_4ROUND_SHED 8 + SHA256_4ROUND_SHED 12 + + add rsi, 4*sizeof(ymmword) + add rbp, 4*sizeof(ymmword) + + ;; and repeat + cmp dword [rbp-sizeof(dword)],0c67178f2h + jne .block1_shed_proc + + ;; the rest 16 rounds + SHA256_ROUND 0, hA,hB,hC,hD,hE,hF,hG,hH + SHA256_ROUND 1, hH,hA,hB,hC,hD,hE,hF,hG + SHA256_ROUND 2, hG,hH,hA,hB,hC,hD,hE,hF + SHA256_ROUND 3, hF,hG,hH,hA,hB,hC,hD,hE + SHA256_ROUND 4, hE,hF,hG,hH,hA,hB,hC,hD + SHA256_ROUND 5, hD,hE,hF,hG,hH,hA,hB,hC + SHA256_ROUND 6, hC,hD,hE,hF,hG,hH,hA,hB + SHA256_ROUND 7, hB,hC,hD,hE,hF,hG,hH,hA + SHA256_ROUND 8, hA,hB,hC,hD,hE,hF,hG,hH + SHA256_ROUND 9, hH,hA,hB,hC,hD,hE,hF,hG + SHA256_ROUND 10, hG,hH,hA,hB,hC,hD,hE,hF + SHA256_ROUND 11, hF,hG,hH,hA,hB,hC,hD,hE + SHA256_ROUND 12, hE,hF,hG,hH,hA,hB,hC,hD + SHA256_ROUND 13, hD,hE,hF,hG,hH,hA,hB,hC + SHA256_ROUND 14, hC,hD,hE,hF,hG,hH,hA,hB + SHA256_ROUND 15, hB,hC,hD,hE,hF,hG,hH,hA + add hA, T3 + + sub rsi, (16-4)*sizeof(ymmword) ; restore stack to W + + mov rdi, qword [rsp+_hash] ; restore hash pointer + mov r14, qword [rsp+_len] ; restore data length + + ;; update hash values by 1-st data block + UPDATE_HASH dword [rdi], hA + UPDATE_HASH dword [rdi+1*sizeof(dword)], hB + UPDATE_HASH dword [rdi+2*sizeof(dword)], hC + UPDATE_HASH dword [rdi+3*sizeof(dword)], hD + UPDATE_HASH dword [rdi+4*sizeof(dword)], hE + UPDATE_HASH dword [rdi+5*sizeof(dword)], hF + UPDATE_HASH dword [rdi+6*sizeof(dword)], hG + UPDATE_HASH dword [rdi+7*sizeof(dword)], hH + + cmp r14, MBS_SHA256*2 + jl .done + + ;; do 64 rounds for the next block + add rsi, 4*sizeof(dword) ; restore stack to next block W + lea rbp, [rsi+16*sizeof(ymmword)] ; use rbp for loop limiter + + mov T5, hB ; T5 = b^c + xor T3, T3 ; T3 = 0 + mov T1, hF ; T1 = f + xor T5, hC + +align IPP_ALIGN_FACTOR +.block2_proc: + SHA256_ROUND 0, hA,hB,hC,hD,hE,hF,hG,hH + SHA256_ROUND 1, hH,hA,hB,hC,hD,hE,hF,hG + SHA256_ROUND 2, hG,hH,hA,hB,hC,hD,hE,hF + SHA256_ROUND 3, hF,hG,hH,hA,hB,hC,hD,hE + SHA256_ROUND 4, hE,hF,hG,hH,hA,hB,hC,hD + SHA256_ROUND 5, hD,hE,hF,hG,hH,hA,hB,hC + SHA256_ROUND 6, hC,hD,hE,hF,hG,hH,hA,hB + SHA256_ROUND 7, hB,hC,hD,hE,hF,hG,hH,hA + add rsi, 2*sizeof(ymmword) + cmp rsi, rbp + jb .block2_proc + add hA, T3 + + mov rdi, qword [rsp+_hash] ; restore hash pointer + mov r14, qword [rsp+_len] ; restore data length + + ;; update hash values by 2-nd data block + UPDATE_HASH dword [rdi], hA + UPDATE_HASH dword [rdi+1*sizeof(dword)], hB + UPDATE_HASH dword [rdi+2*sizeof(dword)], hC + UPDATE_HASH dword [rdi+3*sizeof(dword)], hD + UPDATE_HASH dword [rdi+4*sizeof(dword)], hE + UPDATE_HASH dword [rdi+5*sizeof(dword)], hF + UPDATE_HASH dword [rdi+6*sizeof(dword)], hG + UPDATE_HASH dword [rdi+7*sizeof(dword)], hH + + mov rsi, qword [rsp+_block] ; restore block addres + add rsi, MBS_SHA256*2 ; move data pointer + sub r14, MBS_SHA256*2 ; update data length + mov qword [rsp+_len], r14 + jg .sha256_block2_loop + +.done: + mov rsp, qword [rsp+_frame] + REST_XMM_AVX + REST_GPR + ret +ENDFUNC UpdateSHA256 + +%endif ;; _IPP32E_L9 and above +%endif ;; _FEATURE_OFF_ / _FEATURE_TICKTOCK_ +%endif ;; _ENABLE_ALG_SHA256_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha256m7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha256m7as.asm new file mode 100644 index 0000000..36287e3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha256m7as.asm @@ -0,0 +1,416 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SHA256 +; +; Content: +; UpdateSHA256 +; + +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "ia_32e_regs.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SHA256_) +%if (_SHA_NI_ENABLING_ == _FEATURE_OFF_) || (_SHA_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_M7) && (_IPP32E < _IPP32E_U8 ) + + +;; +;; ENDIANNESS +;; +%macro ENDIANNESS 2.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + + %ifnidn %%dst,%%src + mov %%dst,%%src + %endif + bswap %%dst +%endmacro + +;; +;; single SHA256 step +;; +;; Ipp32u T1 = H + SUM1(E) + CH(E,F,G) + K_SHA256[t] + W[t]; +;; Ipp32u T2 = SUM0(A) + MAJ(A,B,C); +;; D+= T1; +;; H = T1 + T2; +;; +;; where +;; SUM1(x) = ROR(x,6) ^ ROR(x,11) ^ ROR(x,25) +;; SUM0(x) = ROR(x,2) ^ ROR(x,13) ^ ROR(x,22) +;; +;; CH(x,y,z) = (x & y) ^ (~x & z) +;; MAJ(x,y,z) = (x & y) ^ (x & z) ^ (y & z) = (x&y)^((x^y)&z) +;; +%macro SHA256_STEP_2 13.nolist + %xdefine %%regA %1 + %xdefine %%regB %2 + %xdefine %%regC %3 + %xdefine %%regD %4 + %xdefine %%regE %5 + %xdefine %%regF %6 + %xdefine %%regG %7 + %xdefine %%regH %8 + %xdefine %%regT %9 + %xdefine %%regF1 %10 + %xdefine %%regF2 %11 + %xdefine %%memW %12 + %xdefine %%immCNT %13 + + ;; update H (start) + add %%regH%+d,%%immCNT ;; H += W[t]+K_SHA256[t] + add %%regH%+d,[%%memW] + + ;; compute T = SUM1(E) + CH(E,F,G) + mov %%regF1%+d,%%regE%+d ;; SUM1() = E + mov %%regF2%+d,%%regE%+d ;; CH() = E + ror %%regF1%+d,6 ;; ROR(E,6) + mov %%regT%+d, %%regE%+d + push %%regE + not %%regF2%+d ;; ~E + ror %%regE%+d, 11 ;; ROR(E,11) + and %%regT%+d, %%regF%+d ;; E&F + and %%regF2%+d,%%regG%+d ;;~E&G + xor %%regF1%+d,%%regE%+d ;; ROR(E,6)^ROR(E,11) + ror %%regE%+d, 14 ;; ROR(E,25) + xor %%regF2%+d,%%regT%+d ;; CH() = (E&F)&(~E&G) + xor %%regF1%+d,%%regE%+d ;; SUM1() = ROR(E,6)^ROR(E,11)^ROR(E,25) + pop %%regE ;; repair E + lea %%regT, [%%regF1+%%regF2] + + ;; update H (continue) + add %%regH%+d, %%regT%+d + + ;; update D + add %%regD%+d, %%regH%+d + + ;; compute T = SUM0(A) + MAJ(A,B,C) + mov %%regF1%+d,%%regA%+d ;; SUM0() = A + mov %%regF2%+d,%%regA%+d ;; MAJ() = A + ror %%regF1%+d,2 ;; ROR(A,2) + mov %%regT%+d, %%regA%+d + push %%regA + xor %%regF2%+d,%%regB%+d ;; A^B + ror %%regA%+d, 13 ;; ROR(A,13) + and %%regT%+d, %%regB%+d ;; A&B + and %%regF2%+d,%%regC%+d ;; (A^B)&C + xor %%regF1%+d,%%regA%+d ;; ROR(A,2)^ROR(A,13) + ror %%regA%+d, 9 ;; ROR(A,22) + xor %%regF2%+d,%%regT%+d ;; MAJ() = (A^B)^((A^B)&C) + xor %%regF1%+d,%%regA%+d ;; SUM0() = ROR(A,2)^ROR(A,13)^ROR(A,22) + pop %%regA ;; repair A + lea %%regT, [%%regF1+%%regF2] + + ;; update H (finish) + add %%regH%+d, %%regT%+d +%endmacro + +;; +;; update W[] +;; +;; W[j] = SIG1(W[j- 2]) + W[j- 7] +;; +SIG0(W[j-15]) + W[j-16] +;; +;; SIG0(x) = ROR(x,7) ^ROR(x,18) ^LSR(x,3) +;; SIG1(x) = ROR(x,17)^ROR(x,19) ^LSR(x,10) +;; +%macro UPDATE_2 5.nolist + %xdefine %%nr %1 + %xdefine %%sig0 %2 + %xdefine %%sig1 %3 + %xdefine %%W15 %4 + %xdefine %%W2 %5 + + mov %%sig0, [rsp+((%%nr-15) & 0Fh)*4] ;; W[j-15] + mov %%sig1, [rsp+((%%nr-2) & 0Fh)*4] ;; W[j-2] + shr %%sig0, 3 + shr %%sig1, 10 + mov %%W15, [rsp+((%%nr-15) & 0Fh)*4] ;; W[j-15] + mov %%W2, [rsp+((%%nr-2) & 0Fh)*4] ;; W[j-2] + ror %%W15, 7 + ror %%W2, 17 + xor %%sig0, %%W15 ;; SIG0 = LSR(W[j-15], 3) + xor %%sig1, %%W2 ;; SIG1 = LSR(W[j-2], 10) + ror %%W15, 11 ;; ROR(W[j-15], 18) + ror %%W2, 2 ;; ROR(W[j-2, 19) + xor %%sig0, %%W15 + xor %%sig1, %%W2 + + add %%sig0, [rsp+((%%nr-16) & 0Fh)*4] ;;SIG0 += W[j-16] + add %%sig1, [rsp+((%%nr-7) & 0Fh)*4] ;;SIG1 += W[j-7] + add %%sig0, %%sig1 + mov [rsp+((%%nr-16) & 0Fh)*4], %%sig0 +%endmacro + +segment .text align=IPP_ALIGN_FACTOR + + +;****************************************************************************************** +;* Purpose: Update internal digest according to message block +;* +;* void UpdateSHA256(DigestSHA256 digest, const Ipp32u* mblk, int mlen, const void* pParam) +;* +;****************************************************************************************** + +;; +;; Lib = M7 +;; +;; Caller = ippsSHA256Update +;; Caller = ippsSHA256Final +;; Caller = ippsSHA256MessageDigest +;; +;; Caller = ippsSHA224Update +;; Caller = ippsSHA224Final +;; Caller = ippsSHA224MessageDigest +;; +;; Caller = ippsHMACSHA256Update +;; Caller = ippsHMACSHA256Final +;; Caller = ippsHMACSHA256MessageDigest +;; +;; Caller = ippsHMACSHA224Update +;; Caller = ippsHMACSHA224Final +;; Caller = ippsHMACSHA224MessageDigest +;; + + +%if (_IPP32E >= _IPP32E_U8) +align IPP_ALIGN_FACTOR +pByteSwp DB 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12 +%endif + +align IPP_ALIGN_FACTOR +IPPASM UpdateSHA256,PUBLIC +%assign LOCAL_FRAME (16*sizeof(dword) + sizeof(qword)) + USES_GPR rbx,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 4 + +;; rdi = digest +;; rsi = data buffer +;; rdx = buffer len +;; rcx = dummy parameter + +%xdefine MBS_SHA256 (64) + + movsxd rdx, edx + mov qword [rsp+16*sizeof(dword)], rdx ; save length of buffer + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +.sha256_block_loop: + +;; +;; initialize the first 16 words in the array W (remember about endian) +;; +%if (_IPP32E >= _IPP32E_U8) + movdqa xmm4, oword [rel pByteSwp] + movdqu xmm0, oword [rsi+0*16] + pshufb xmm0, xmm4 + movdqa oword [rsp+0*16], xmm0 + movdqu xmm1, oword [rsi+1*16] + pshufb xmm1, xmm4 + movdqa oword [rsp+1*16], xmm1 + movdqu xmm2, oword [rsi+2*16] + pshufb xmm2, xmm4 + movdqa oword [rsp+2*16], xmm2 + movdqu xmm3, oword [rsi+3*16] + pshufb xmm3, xmm4 + movdqa oword [rsp+3*16], xmm3 +%else + xor rcx,rcx +.loop1: + mov r8d,[rsi+rcx*4+0*4] + ENDIANNESS r8d,r8d + mov [rsp+rcx*4+0*4],r8d + + mov r9d,[rsi+rcx*4+1*4] + ENDIANNESS r9d,r9d + mov [rsp+rcx*4+1*4],r9d + + add rcx,2 + cmp rcx,16 + jl .loop1 +%endif + +;; +;; init A, B, C, D, E, F, G, H by the internal digest +;; + mov r8d, [rdi+0*4] ; r8d = digest[0] (A) + mov r9d, [rdi+1*4] ; r9d = digest[1] (B) + mov r10d,[rdi+2*4] ; r10d= digest[2] (C) + mov r11d,[rdi+3*4] ; r11d= digest[3] (D) + mov r12d,[rdi+4*4] ; r12d= digest[4] (E) + mov r13d,[rdi+5*4] ; r13d= digest[5] (F) + mov r14d,[rdi+6*4] ; r14d= digest[6] (G) + mov r15d,[rdi+7*4] ; r15d= digest[7] (H) + +;; +;; perform 0-64 steps +;; +;; A, B, C, D, E, F, G, H T, F1, F2, W[], K256 +;; ---------------------------------------------------------------------------- + SHA256_STEP_2 r8, r9, r10,r11,r12,r13,r14,r15, rbx,rcx,rdx, {rsp+ 0*4}, 0428A2F98h + UPDATE_2 16, eax,ebx, ecx,edx + SHA256_STEP_2 r15,r8, r9, r10,r11,r12,r13,r14, rbx,rcx,rdx, {rsp+ 1*4}, 071374491h + UPDATE_2 17, eax,ebx, ecx,edx + SHA256_STEP_2 r14,r15,r8, r9, r10,r11,r12,r13, rbx,rcx,rdx, {rsp+ 2*4}, 0B5C0FBCFh + UPDATE_2 18, eax,ebx, ecx,edx + SHA256_STEP_2 r13,r14,r15,r8, r9, r10,r11,r12, rbx,rcx,rdx, {rsp+ 3*4}, 0E9B5DBA5h + UPDATE_2 19, eax,ebx, ecx,edx + SHA256_STEP_2 r12,r13,r14,r15,r8, r9, r10,r11, rbx,rcx,rdx, {rsp+ 4*4}, 03956C25Bh + UPDATE_2 20, eax,ebx, ecx,edx + SHA256_STEP_2 r11,r12,r13,r14,r15,r8, r9, r10, rbx,rcx,rdx, {rsp+ 5*4}, 059F111F1h + UPDATE_2 21, eax,ebx, ecx,edx + SHA256_STEP_2 r10,r11,r12,r13,r14,r15,r8, r9, rbx,rcx,rdx, {rsp+ 6*4}, 0923F82A4h + UPDATE_2 22, eax,ebx, ecx,edx + SHA256_STEP_2 r9, r10,r11,r12,r13,r14,r15,r8, rbx,rcx,rdx, {rsp+ 7*4}, 0AB1C5ED5h + UPDATE_2 23, eax,ebx, ecx,edx + SHA256_STEP_2 r8, r9, r10,r11,r12,r13,r14,r15, rbx,rcx,rdx, {rsp+ 8*4}, 0D807AA98h + UPDATE_2 24, eax,ebx, ecx,edx + SHA256_STEP_2 r15,r8, r9, r10,r11,r12,r13,r14, rbx,rcx,rdx, {rsp+ 9*4}, 012835B01h + UPDATE_2 25, eax,ebx, ecx,edx + SHA256_STEP_2 r14,r15,r8, r9, r10,r11,r12,r13, rbx,rcx,rdx, {rsp+10*4}, 0243185BEh + UPDATE_2 26, eax,ebx, ecx,edx + SHA256_STEP_2 r13,r14,r15,r8, r9, r10,r11,r12, rbx,rcx,rdx, {rsp+11*4}, 0550C7DC3h + UPDATE_2 27, eax,ebx, ecx,edx + SHA256_STEP_2 r12,r13,r14,r15,r8, r9, r10,r11, rbx,rcx,rdx, {rsp+12*4}, 072BE5D74h + UPDATE_2 28, eax,ebx, ecx,edx + SHA256_STEP_2 r11,r12,r13,r14,r15,r8, r9, r10, rbx,rcx,rdx, {rsp+13*4}, 080DEB1FEh + UPDATE_2 29, eax,ebx, ecx,edx + SHA256_STEP_2 r10,r11,r12,r13,r14,r15,r8, r9, rbx,rcx,rdx, {rsp+14*4}, 09BDC06A7h + UPDATE_2 30, eax,ebx, ecx,edx + SHA256_STEP_2 r9, r10,r11,r12,r13,r14,r15,r8, rbx,rcx,rdx, {rsp+15*4}, 0C19BF174h + UPDATE_2 31, eax,ebx, ecx,edx + + SHA256_STEP_2 r8, r9, r10,r11,r12,r13,r14,r15, rbx,rcx,rdx, {rsp+ 0*4}, 0E49B69C1h + UPDATE_2 32, eax,ebx, ecx,edx + SHA256_STEP_2 r15,r8, r9, r10,r11,r12,r13,r14, rbx,rcx,rdx, {rsp+ 1*4}, 0EFBE4786h + UPDATE_2 33, eax,ebx, ecx,edx + SHA256_STEP_2 r14,r15,r8, r9, r10,r11,r12,r13, rbx,rcx,rdx, {rsp+ 2*4}, 00FC19DC6h + UPDATE_2 34, eax,ebx, ecx,edx + SHA256_STEP_2 r13,r14,r15,r8, r9, r10,r11,r12, rbx,rcx,rdx, {rsp+ 3*4}, 0240CA1CCh + UPDATE_2 35, eax,ebx, ecx,edx + SHA256_STEP_2 r12,r13,r14,r15,r8, r9, r10,r11, rbx,rcx,rdx, {rsp+ 4*4}, 02DE92C6Fh + UPDATE_2 36, eax,ebx, ecx,edx + SHA256_STEP_2 r11,r12,r13,r14,r15,r8, r9, r10, rbx,rcx,rdx, {rsp+ 5*4}, 04A7484AAh + UPDATE_2 37, eax,ebx, ecx,edx + SHA256_STEP_2 r10,r11,r12,r13,r14,r15,r8, r9, rbx,rcx,rdx, {rsp+ 6*4}, 05CB0A9DCh + UPDATE_2 38, eax,ebx, ecx,edx + SHA256_STEP_2 r9, r10,r11,r12,r13,r14,r15,r8, rbx,rcx,rdx, {rsp+ 7*4}, 076F988DAh + UPDATE_2 39, eax,ebx, ecx,edx + SHA256_STEP_2 r8, r9, r10,r11,r12,r13,r14,r15, rbx,rcx,rdx, {rsp+ 8*4}, 0983E5152h + UPDATE_2 40, eax,ebx, ecx,edx + SHA256_STEP_2 r15,r8, r9, r10,r11,r12,r13,r14, rbx,rcx,rdx, {rsp+ 9*4}, 0A831C66Dh + UPDATE_2 41, eax,ebx, ecx,edx + SHA256_STEP_2 r14,r15,r8, r9, r10,r11,r12,r13, rbx,rcx,rdx, {rsp+10*4}, 0B00327C8h + UPDATE_2 42, eax,ebx, ecx,edx + SHA256_STEP_2 r13,r14,r15,r8, r9, r10,r11,r12, rbx,rcx,rdx, {rsp+11*4}, 0BF597FC7h + UPDATE_2 43, eax,ebx, ecx,edx + SHA256_STEP_2 r12,r13,r14,r15,r8, r9, r10,r11, rbx,rcx,rdx, {rsp+12*4}, 0C6E00BF3h + UPDATE_2 44, eax,ebx, ecx,edx + SHA256_STEP_2 r11,r12,r13,r14,r15,r8, r9, r10, rbx,rcx,rdx, {rsp+13*4}, 0D5A79147h + UPDATE_2 45, eax,ebx, ecx,edx + SHA256_STEP_2 r10,r11,r12,r13,r14,r15,r8, r9, rbx,rcx,rdx, {rsp+14*4}, 006CA6351h + UPDATE_2 46, eax,ebx, ecx,edx + SHA256_STEP_2 r9, r10,r11,r12,r13,r14,r15,r8, rbx,rcx,rdx, {rsp+15*4}, 014292967h + UPDATE_2 47, eax,ebx, ecx,edx + + SHA256_STEP_2 r8, r9, r10,r11,r12,r13,r14,r15, rbx,rcx,rdx, {rsp+ 0*4}, 027B70A85h + UPDATE_2 48, eax,ebx, ecx,edx + SHA256_STEP_2 r15,r8, r9, r10,r11,r12,r13,r14, rbx,rcx,rdx, {rsp+ 1*4}, 02E1B2138h + UPDATE_2 49, eax,ebx, ecx,edx + SHA256_STEP_2 r14,r15,r8, r9, r10,r11,r12,r13, rbx,rcx,rdx, {rsp+ 2*4}, 04D2C6DFCh + UPDATE_2 50, eax,ebx, ecx,edx + SHA256_STEP_2 r13,r14,r15,r8, r9, r10,r11,r12, rbx,rcx,rdx, {rsp+ 3*4}, 053380D13h + UPDATE_2 51, eax,ebx, ecx,edx + SHA256_STEP_2 r12,r13,r14,r15,r8, r9, r10,r11, rbx,rcx,rdx, {rsp+ 4*4}, 0650A7354h + UPDATE_2 52, eax,ebx, ecx,edx + SHA256_STEP_2 r11,r12,r13,r14,r15,r8, r9, r10, rbx,rcx,rdx, {rsp+ 5*4}, 0766A0ABBh + UPDATE_2 53, eax,ebx, ecx,edx + SHA256_STEP_2 r10,r11,r12,r13,r14,r15,r8, r9, rbx,rcx,rdx, {rsp+ 6*4}, 081C2C92Eh + UPDATE_2 54, eax,ebx, ecx,edx + SHA256_STEP_2 r9, r10,r11,r12,r13,r14,r15,r8, rbx,rcx,rdx, {rsp+ 7*4}, 092722C85h + UPDATE_2 55, eax,ebx, ecx,edx + SHA256_STEP_2 r8, r9, r10,r11,r12,r13,r14,r15, rbx,rcx,rdx, {rsp+ 8*4}, 0A2BFE8A1h + UPDATE_2 56, eax,ebx, ecx,edx + SHA256_STEP_2 r15,r8, r9, r10,r11,r12,r13,r14, rbx,rcx,rdx, {rsp+ 9*4}, 0A81A664Bh + UPDATE_2 57, eax,ebx, ecx,edx + SHA256_STEP_2 r14,r15,r8, r9, r10,r11,r12,r13, rbx,rcx,rdx, {rsp+10*4}, 0C24B8B70h + UPDATE_2 58, eax,ebx, ecx,edx + SHA256_STEP_2 r13,r14,r15,r8, r9, r10,r11,r12, rbx,rcx,rdx, {rsp+11*4}, 0C76C51A3h + UPDATE_2 59, eax,ebx, ecx,edx + SHA256_STEP_2 r12,r13,r14,r15,r8, r9, r10,r11, rbx,rcx,rdx, {rsp+12*4}, 0D192E819h + UPDATE_2 60, eax,ebx, ecx,edx + SHA256_STEP_2 r11,r12,r13,r14,r15,r8, r9, r10, rbx,rcx,rdx, {rsp+13*4}, 0D6990624h + UPDATE_2 61, eax,ebx, ecx,edx + SHA256_STEP_2 r10,r11,r12,r13,r14,r15,r8, r9, rbx,rcx,rdx, {rsp+14*4}, 0F40E3585h + UPDATE_2 62, eax,ebx, ecx,edx + SHA256_STEP_2 r9, r10,r11,r12,r13,r14,r15,r8, rbx,rcx,rdx, {rsp+15*4}, 0106AA070h + UPDATE_2 63, eax,ebx, ecx,edx + + SHA256_STEP_2 r8, r9, r10,r11,r12,r13,r14,r15, rbx,rcx,rdx, {rsp+ 0*4}, 019A4C116h + SHA256_STEP_2 r15,r8, r9, r10,r11,r12,r13,r14, rbx,rcx,rdx, {rsp+ 1*4}, 01E376C08h + SHA256_STEP_2 r14,r15,r8, r9, r10,r11,r12,r13, rbx,rcx,rdx, {rsp+ 2*4}, 02748774Ch + SHA256_STEP_2 r13,r14,r15,r8, r9, r10,r11,r12, rbx,rcx,rdx, {rsp+ 3*4}, 034B0BCB5h + SHA256_STEP_2 r12,r13,r14,r15,r8, r9, r10,r11, rbx,rcx,rdx, {rsp+ 4*4}, 0391C0CB3h + SHA256_STEP_2 r11,r12,r13,r14,r15,r8, r9, r10, rbx,rcx,rdx, {rsp+ 5*4}, 04ED8AA4Ah + SHA256_STEP_2 r10,r11,r12,r13,r14,r15,r8, r9, rbx,rcx,rdx, {rsp+ 6*4}, 05B9CCA4Fh + SHA256_STEP_2 r9, r10,r11,r12,r13,r14,r15,r8, rbx,rcx,rdx, {rsp+ 7*4}, 0682E6FF3h + SHA256_STEP_2 r8, r9, r10,r11,r12,r13,r14,r15, rbx,rcx,rdx, {rsp+ 8*4}, 0748F82EEh + SHA256_STEP_2 r15,r8, r9, r10,r11,r12,r13,r14, rbx,rcx,rdx, {rsp+ 9*4}, 078A5636Fh + SHA256_STEP_2 r14,r15,r8, r9, r10,r11,r12,r13, rbx,rcx,rdx, {rsp+10*4}, 084C87814h + SHA256_STEP_2 r13,r14,r15,r8, r9, r10,r11,r12, rbx,rcx,rdx, {rsp+11*4}, 08CC70208h + SHA256_STEP_2 r12,r13,r14,r15,r8, r9, r10,r11, rbx,rcx,rdx, {rsp+12*4}, 090BEFFFAh + SHA256_STEP_2 r11,r12,r13,r14,r15,r8, r9, r10, rbx,rcx,rdx, {rsp+13*4}, 0A4506CEBh + SHA256_STEP_2 r10,r11,r12,r13,r14,r15,r8, r9, rbx,rcx,rdx, {rsp+14*4}, 0BEF9A3F7h + SHA256_STEP_2 r9, r10,r11,r12,r13,r14,r15,r8, rbx,rcx,rdx, {rsp+15*4}, 0C67178F2h + +;; +;; update digest +;; + add [rdi+0*4],r8d + add [rdi+1*4],r9d + add [rdi+2*4],r10d + add [rdi+3*4],r11d + add [rdi+4*4],r12d + add [rdi+5*4],r13d + add [rdi+6*4],r14d + add [rdi+7*4],r15d + + add rsi, MBS_SHA256 + sub qword [rsp+16*sizeof(dword)], MBS_SHA256 + jg .sha256_block_loop + + REST_XMM + REST_GPR + ret +ENDFUNC UpdateSHA256 + +%endif ;; (_IPP32E >= _IPP32E_M7) AND (_IPP32E < _IPP32E_U8 ) +%endif ;; _FEATURE_OFF_ / _FEATURE_TICKTOCK_ +%endif ;; _ENABLE_ALG_SHA256_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha256nias.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha256nias.asm new file mode 100644 index 0000000..3c8928c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha256nias.asm @@ -0,0 +1,569 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SHA-256 +; +; Content: +; UpdateSHA256ni +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SHA256_) +%if (_SHA_NI_ENABLING_ == _FEATURE_ON_) || (_SHA_NI_ENABLING_ == _FEATURE_TICKTOCK_) +;;%if (_IPP32E >= _IPP32E_Y8 ) + +segment .text align=IPP_ALIGN_FACTOR + + +align IPP_ALIGN_FACTOR +PSHUFFLE_BYTE_FLIP_MASK \ + DB 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12 + +align IPP_ALIGN_FACTOR +;***************************************************************************************** +;* Purpose: Update internal digest according to message block +;* +;* void UpdateSHA256ni(DigestSHA256 digest, const Ipp8u* msg, int mlen, const void* pParam) +;* +;***************************************************************************************** +%ifndef _VXWORKS + +IPPASM UpdateSHA256ni,PUBLIC +%assign LOCAL_FRAME 16*2 + USES_GPR rsi,rdi + USES_XMM xmm6,xmm7,xmm8,xmm9,xmm10 + COMP_ABI 4 + +%xdefine MBS_SHA256 (64) ; SHA-256 message block length (bytes) + +%xdefine HASH_PTR rdi ; 1st arg +%xdefine MSG_PTR rsi ; 2nd arg +%xdefine MSG_LEN rdx ; 3rd arg +%xdefine K256_PTR rcx ; 4th arg + +%xdefine MSG xmm0 +%xdefine STATE0 xmm1 +%xdefine STATE1 xmm2 +%xdefine MSGTMP0 xmm3 +%xdefine MSGTMP1 xmm4 +%xdefine MSGTMP2 xmm5 +%xdefine MSGTMP3 xmm6 +%xdefine MSGTMP4 xmm7 + +%xdefine SHUF_MASK xmm8 + +%xdefine ABEF_SAVE xmm9 +%xdefine CDGH_SAVE xmm10 + + movsxd MSG_LEN, edx ; expand mLen + test MSG_LEN, MSG_LEN + jz .quit + + ;; load input hash value, reorder these appropriately + movdqu STATE0, oword [HASH_PTR+0*16] + movdqu STATE1, oword [HASH_PTR+1*16] + + pshufd STATE0, STATE0, 0B1h ; CDAB + pshufd STATE1, STATE1, 01Bh ; EFGH + movdqa MSGTMP4, STATE0 + palignr STATE0, STATE1, 8 ; ABEF + pblendw STATE1, MSGTMP4, 0F0h ; CDGH + + movdqa SHUF_MASK, oword [rel PSHUFFLE_BYTE_FLIP_MASK] + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +.sha256_block_loop: + movdqa ABEF_SAVE, STATE0 ; save for addition after rounds + movdqa CDGH_SAVE, STATE1 + + ;; rounds 0-3 + movdqu MSG, oword [MSG_PTR + 0*16] + pshufb MSG, SHUF_MASK + movdqa MSGTMP0, MSG + paddd MSG, oword [K256_PTR + 0*16] + sha256rnds2 STATE1, STATE0 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + + ;; rounds 4-7 + movdqu MSG, oword [MSG_PTR + 1*16] + pshufb MSG, SHUF_MASK + movdqa MSGTMP1, MSG + paddd MSG, oword [K256_PTR + 1*16] + sha256rnds2 STATE1, STATE0 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP0, MSGTMP1 + + ;; rounds 8-11 + movdqu MSG, oword [MSG_PTR + 2*16] + pshufb MSG, SHUF_MASK + movdqa MSGTMP2, MSG + paddd MSG, oword [K256_PTR + 2*16] + sha256rnds2 STATE1, STATE0 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP1, MSGTMP2 + + ;; rounds 12-15 + movdqu MSG, oword [MSG_PTR + 3*16] + pshufb MSG, SHUF_MASK + movdqa MSGTMP3, MSG + paddd MSG, oword [K256_PTR + 3*16] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP3 + palignr MSGTMP4, MSGTMP2, 4 + paddd MSGTMP0, MSGTMP4 + sha256msg2 MSGTMP0, MSGTMP3 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP2, MSGTMP3 + ;; rounds 16-19 + movdqa MSG, MSGTMP0 + paddd MSG, oword [K256_PTR + 4*16] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP0 + palignr MSGTMP4, MSGTMP3, 4 + paddd MSGTMP1, MSGTMP4 + sha256msg2 MSGTMP1, MSGTMP0 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP3, MSGTMP0 + + ;; rounds 20-23 + movdqa MSG, MSGTMP1 + paddd MSG, oword [K256_PTR + 5*16] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP1 + palignr MSGTMP4, MSGTMP0, 4 + paddd MSGTMP2, MSGTMP4 + sha256msg2 MSGTMP2, MSGTMP1 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP0, MSGTMP1 + + ;; rounds 24-27 + movdqa MSG, MSGTMP2 + paddd MSG, oword [K256_PTR + 6*16] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP2 + palignr MSGTMP4, MSGTMP1, 4 + paddd MSGTMP3, MSGTMP4 + sha256msg2 MSGTMP3, MSGTMP2 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP1, MSGTMP2 + + ;; rounds 28-31 + movdqa MSG, MSGTMP3 + paddd MSG, oword [K256_PTR + 7*16] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP3 + palignr MSGTMP4, MSGTMP2, 4 + paddd MSGTMP0, MSGTMP4 + sha256msg2 MSGTMP0, MSGTMP3 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP2, MSGTMP3 + + ;; rounds 32-35 + movdqa MSG, MSGTMP0 + paddd MSG, oword [K256_PTR + 8*16] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP0 + palignr MSGTMP4, MSGTMP3, 4 + paddd MSGTMP1, MSGTMP4 + sha256msg2 MSGTMP1, MSGTMP0 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP3, MSGTMP0 + + ;; rounds 36-39 + movdqa MSG, MSGTMP1 + paddd MSG, oword [K256_PTR + 9*16] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP1 + palignr MSGTMP4, MSGTMP0, 4 + paddd MSGTMP2, MSGTMP4 + sha256msg2 MSGTMP2, MSGTMP1 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP0, MSGTMP1 + + ;; rounds 40-43 + movdqa MSG, MSGTMP2 + paddd MSG, oword [K256_PTR + 10*16] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP2 + palignr MSGTMP4, MSGTMP1, 4 + paddd MSGTMP3, MSGTMP4 + sha256msg2 MSGTMP3, MSGTMP2 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP1, MSGTMP2 + + ;; rounds 44-47 + movdqa MSG, MSGTMP3 + paddd MSG, oword [K256_PTR + 11*16] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP3 + palignr MSGTMP4, MSGTMP2, 4 + paddd MSGTMP0, MSGTMP4 + sha256msg2 MSGTMP0, MSGTMP3 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP2, MSGTMP3 + + ;; rounds 48-51 + movdqa MSG, MSGTMP0 + paddd MSG, oword [K256_PTR + 12*16] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP0 + palignr MSGTMP4, MSGTMP3, 4 + paddd MSGTMP1, MSGTMP4 + sha256msg2 MSGTMP1, MSGTMP0 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + sha256msg1 MSGTMP3, MSGTMP0 + + ;; rounds 52-55 + movdqa MSG, MSGTMP1 + paddd MSG, oword [K256_PTR + 13*16] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP1 + palignr MSGTMP4, MSGTMP0, 4 + paddd MSGTMP2, MSGTMP4 + sha256msg2 MSGTMP2, MSGTMP1 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + + ;; rounds 56-59 + movdqa MSG, MSGTMP2 + paddd MSG, oword [K256_PTR + 14*16] + sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP2 + palignr MSGTMP4, MSGTMP1, 4 + paddd MSGTMP3, MSGTMP4 + sha256msg2 MSGTMP3, MSGTMP2 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + + ;; rounds 60-63 + movdqa MSG, MSGTMP3 + paddd MSG, oword [K256_PTR + 15*16] + sha256rnds2 STATE1, STATE0 + pshufd MSG, MSG, 0Eh + sha256rnds2 STATE0, STATE1 + + paddd STATE0, ABEF_SAVE ; update previously saved hash + paddd STATE1, CDGH_SAVE + + add MSG_PTR, MBS_SHA256 + sub MSG_LEN, MBS_SHA256 + jg .sha256_block_loop + + ; reorder hash + pshufd STATE0, STATE0, 01Bh ; FEBA + pshufd STATE1, STATE1, 0B1h ; DCHG + movdqa MSGTMP4, STATE0 + pblendw STATE0, STATE1, 0F0h ; DCBA + palignr STATE1, MSGTMP4, 8 ; HGFE + + ; and store it back + movdqu oword [HASH_PTR + 0*16], STATE0 + movdqu oword [HASH_PTR + 1*16], STATE1 + +.quit: + REST_XMM + REST_GPR + ret +ENDFUNC UpdateSHA256ni + +%else ;; no sha ni support in VxWorks - therefore we temporary use db +IPPASM UpdateSHA256ni,PUBLIC +%assign LOCAL_FRAME 16*2 + USES_GPR rsi,rdi + USES_XMM xmm6,xmm7,xmm8,xmm9,xmm10 + COMP_ABI 4 + +%xdefine MBS_SHA256 (64) ; SHA-256 message block length (bytes) + +%xdefine HASH_PTR rdi ; 1st arg +%xdefine MSG_PTR rsi ; 2nd arg +%xdefine MSG_LEN rdx ; 3rd arg +%xdefine K256_PTR rcx ; 4th arg + +%xdefine MSG xmm0 +%xdefine STATE0 xmm1 +%xdefine STATE1 xmm2 +%xdefine MSGTMP0 xmm3 +%xdefine MSGTMP1 xmm4 +%xdefine MSGTMP2 xmm5 +%xdefine MSGTMP3 xmm6 +%xdefine MSGTMP4 xmm7 + +%xdefine SHUF_MASK xmm8 + +%xdefine ABEF_SAVE xmm9 +%xdefine CDGH_SAVE xmm10 + + movsxd MSG_LEN, edx ; expand mLen + test MSG_LEN, MSG_LEN + jz .quit + + ;; load input hash value, reorder these appropriately + movdqu STATE0, oword [HASH_PTR+0*16] + movdqu STATE1, oword [HASH_PTR+1*16] + + pshufd STATE0, STATE0, 0B1h ; CDAB + pshufd STATE1, STATE1, 01Bh ; EFGH + movdqa MSGTMP4, STATE0 + palignr STATE0, STATE1, 8 ; ABEF + pblendw STATE1, MSGTMP4, 0F0h ; CDGH + + movdqa SHUF_MASK, oword [rel PSHUFFLE_BYTE_FLIP_MASK] + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +.sha256_block_loop: + movdqa ABEF_SAVE, STATE0 ; save for addition after rounds + movdqa CDGH_SAVE, STATE1 + + ;; rounds 0-3 + movdqu MSG, oword [MSG_PTR + 0*16] + pshufb MSG, SHUF_MASK + movdqa MSGTMP0, MSG + paddd MSG, oword [K256_PTR + 0*16] + db 0FH,38H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + pshufd MSG, MSG, 0Eh + db 0FH,38H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + + ;; rounds 4-7 + movdqu MSG, oword [MSG_PTR + 1*16] + pshufb MSG, SHUF_MASK + movdqa MSGTMP1, MSG + paddd MSG, oword [K256_PTR + 1*16] + db 0FH,38H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + pshufd MSG, MSG, 0Eh + db 0FH,38H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,38H,0CCH,0DCH ;; sha256msg1 MSGTMP0, MSGTMP1 + + ;; rounds 8-11 + movdqu MSG, oword [MSG_PTR + 2*16] + pshufb MSG, SHUF_MASK + movdqa MSGTMP2, MSG + paddd MSG, oword [K256_PTR + 2*16] + db 0FH,38H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + pshufd MSG, MSG, 0Eh + db 0FH,38H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,38H,0CCH,0E5H ;; sha256msg1 MSGTMP1, MSGTMP2 + + ;; rounds 12-15 + movdqu MSG, oword [MSG_PTR + 3*16] + pshufb MSG, SHUF_MASK + movdqa MSGTMP3, MSG + paddd MSG, oword [K256_PTR + 3*16] + db 0FH,38H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP3 + palignr MSGTMP4, MSGTMP2, 4 + paddd MSGTMP0, MSGTMP4 + db 0FH,38H,0CDH,0DEH ;; sha256msg2 MSGTMP0, MSGTMP3 + pshufd MSG, MSG, 0Eh + db 0FH,38H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,38H,0CCH,0EEH ;; sha256msg1 MSGTMP2, MSGTMP3 + ;; rounds 16-19 + movdqa MSG, MSGTMP0 + paddd MSG, oword [K256_PTR + 4*16] + db 0FH,38H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP0 + palignr MSGTMP4, MSGTMP3, 4 + paddd MSGTMP1, MSGTMP4 + db 0FH,38H,0CDH,0E3H ;; sha256msg2 MSGTMP1, MSGTMP0 + pshufd MSG, MSG, 0Eh + db 0FH,38H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,38H,0CCH,0F3H ;; sha256msg1 MSGTMP3, MSGTMP0 + + ;; rounds 20-23 + movdqa MSG, MSGTMP1 + paddd MSG, oword [K256_PTR + 5*16] + db 0FH,38H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP1 + palignr MSGTMP4, MSGTMP0, 4 + paddd MSGTMP2, MSGTMP4 + db 0FH,38H,0CDH,0ECH ;; sha256msg2 MSGTMP2, MSGTMP1 + pshufd MSG, MSG, 0Eh + db 0FH,38H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,38H,0CCH,0DCH ;; sha256msg1 MSGTMP0, MSGTMP1 + + ;; rounds 24-27 + movdqa MSG, MSGTMP2 + paddd MSG, oword [K256_PTR + 6*16] + db 0FH,38H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP2 + palignr MSGTMP4, MSGTMP1, 4 + paddd MSGTMP3, MSGTMP4 + db 0FH,38H,0CDH,0F5H ;; sha256msg2 MSGTMP3, MSGTMP2 + pshufd MSG, MSG, 0Eh + db 0FH,38H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,38H,0CCH,0E5H ;; sha256msg1 MSGTMP1, MSGTMP2 + + ;; rounds 28-31 + movdqa MSG, MSGTMP3 + paddd MSG, oword [K256_PTR + 7*16] + db 0FH,38H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP3 + palignr MSGTMP4, MSGTMP2, 4 + paddd MSGTMP0, MSGTMP4 + db 0FH,38H,0CDH,0DEH ;; sha256msg2 MSGTMP0, MSGTMP3 + pshufd MSG, MSG, 0Eh + db 0FH,38H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,38H,0CCH,0EEH ;; sha256msg1 MSGTMP2, MSGTMP3 + + ;; rounds 32-35 + movdqa MSG, MSGTMP0 + paddd MSG, oword [K256_PTR + 8*16] + db 0FH,38H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP0 + palignr MSGTMP4, MSGTMP3, 4 + paddd MSGTMP1, MSGTMP4 + db 0FH,38H,0CDH,0E3H ;; sha256msg2 MSGTMP1, MSGTMP0 + pshufd MSG, MSG, 0Eh + db 0FH,38H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,38H,0CCH,0F3H ;; sha256msg1 MSGTMP3, MSGTMP0 + + ;; rounds 36-39 + movdqa MSG, MSGTMP1 + paddd MSG, oword [K256_PTR + 9*16] + db 0FH,38H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP1 + palignr MSGTMP4, MSGTMP0, 4 + paddd MSGTMP2, MSGTMP4 + db 0FH,38H,0CDH,0ECH ;; sha256msg2 MSGTMP2, MSGTMP1 + pshufd MSG, MSG, 0Eh + db 0FH,38H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,38H,0CCH,0DCH ;; sha256msg1 MSGTMP0, MSGTMP1 + + ;; rounds 40-43 + movdqa MSG, MSGTMP2 + paddd MSG, oword [K256_PTR + 10*16] + db 0FH,38H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP2 + palignr MSGTMP4, MSGTMP1, 4 + paddd MSGTMP3, MSGTMP4 + db 0FH,38H,0CDH,0F5H ;; sha256msg2 MSGTMP3, MSGTMP2 + pshufd MSG, MSG, 0Eh + db 0FH,38H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,38H,0CCH,0E5H ;; sha256msg1 MSGTMP1, MSGTMP2 + + ;; rounds 44-47 + movdqa MSG, MSGTMP3 + paddd MSG, oword [K256_PTR + 11*16] + db 0FH,38H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP3 + palignr MSGTMP4, MSGTMP2, 4 + paddd MSGTMP0, MSGTMP4 + db 0FH,38H,0CDH,0DEH ;; sha256msg2 MSGTMP0, MSGTMP3 + pshufd MSG, MSG, 0Eh + db 0FH,38H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,38H,0CCH,0EEH ;; sha256msg1 MSGTMP2, MSGTMP3 + + ;; rounds 48-51 + movdqa MSG, MSGTMP0 + paddd MSG, oword [K256_PTR + 12*16] + db 0FH,38H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP0 + palignr MSGTMP4, MSGTMP3, 4 + paddd MSGTMP1, MSGTMP4 + db 0FH,38H,0CDH,0E3H ;; sha256msg2 MSGTMP1, MSGTMP0 + pshufd MSG, MSG, 0Eh + db 0FH,38H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + db 0FH,38H,0CCH,0F3H ;; sha256msg1 MSGTMP3, MSGTMP0 + + ;; rounds 52-55 + movdqa MSG, MSGTMP1 + paddd MSG, oword [K256_PTR + 13*16] + db 0FH,38H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP1 + palignr MSGTMP4, MSGTMP0, 4 + paddd MSGTMP2, MSGTMP4 + db 0FH,38H,0CDH,0ECH ;; sha256msg2 MSGTMP2, MSGTMP1 + pshufd MSG, MSG, 0Eh + db 0FH,38H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + + ;; rounds 56-59 + movdqa MSG, MSGTMP2 + paddd MSG, oword [K256_PTR + 14*16] + db 0FH,38H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + movdqa MSGTMP4, MSGTMP2 + palignr MSGTMP4, MSGTMP1, 4 + paddd MSGTMP3, MSGTMP4 + db 0FH,38H,0CDH,0F5H ;; sha256msg2 MSGTMP3, MSGTMP2 + pshufd MSG, MSG, 0Eh + db 0FH,38H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + + ;; rounds 60-63 + movdqa MSG, MSGTMP3 + paddd MSG, oword [K256_PTR + 15*16] + db 0FH,38H,0CBH,0D1H ;; sha256rnds2 STATE1, STATE0 + pshufd MSG, MSG, 0Eh + db 0FH,38H,0CBH,0CAH ;; sha256rnds2 STATE0, STATE1 + + paddd STATE0, ABEF_SAVE ; update previously saved hash + paddd STATE1, CDGH_SAVE + + add MSG_PTR, MBS_SHA256 + sub MSG_LEN, MBS_SHA256 + jg .sha256_block_loop + + ; reorder hash + pshufd STATE0, STATE0, 01Bh ; FEBA + pshufd STATE1, STATE1, 0B1h ; DCHG + movdqa MSGTMP4, STATE0 + pblendw STATE0, STATE1, 0F0h ; DCBA + palignr STATE1, MSGTMP4, 8 ; HGFE + + ; and store it back + movdqu oword [HASH_PTR + 0*16], STATE0 + movdqu oword [HASH_PTR + 1*16], STATE1 + +.quit: + REST_XMM + REST_GPR + ret +ENDFUNC UpdateSHA256ni + +%endif ;; VxWorks + +;;%endif ;; (_IPP32E >= _IPP32E_Y8) +%endif ;; _FEATURE_ON_ / _FEATURE_TICKTOCK_ +%endif ;; _ENABLE_ALG_SHA256_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha256u8as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha256u8as.asm new file mode 100644 index 0000000..3880b5b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha256u8as.asm @@ -0,0 +1,488 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SHA256 +; +; Content: +; UpdateSHA256 +; +; + +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "ia_32e_regs.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SHA256_) +%if (_SHA_NI_ENABLING_ == _FEATURE_OFF_) || (_SHA_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if ((_IPP32E >= _IPP32E_U8 ) && (_IPP32E < _IPP32E_E9 )) || (_IPP32E == _IPP32E_N8 ) + + +%xdefine XMM_SHUFB_BSWAP xmm6 +%xdefine W0 xmm0 +%xdefine W4 xmm1 +%xdefine W8 xmm2 +%xdefine W12 xmm3 +%xdefine SIG1 xmm4 +%xdefine SIG0 xmm5 +%xdefine X xmm6 +%xdefine W xmm7 + +;; assign hash values to GPU registers +%xdefine A eax +%xdefine B ebx +%xdefine C edx +%xdefine D r8d +%xdefine E r9d +%xdefine F r10d +%xdefine G r11d +%xdefine H r12d +%xdefine T1 r13d +%xdefine T2 r14d +%xdefine T3 r15d + +;; we are considering x, y, z are polynomials over GF(2) +;; & - multiplication +;; ^ - additive +;; operations + +;; +;; Chj(x,y,z) = (x&y) ^ (~x & z) +;; = (x&y) ^ ((1^x) &z) +;; = (x&y) ^ (z ^ x&z) +;; = x&y ^ z ^ x&z +;; = x&(y^z) ^z +;; +%macro Chj 4.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%Y %3 + %xdefine %%Z %4 + + mov %%F, %%Y + xor %%F, %%Z + and %%F, %%X + xor %%F, %%Z +%endmacro + +;; +;; Maj(x,y,z) = (x&y) ^ (x&z) ^ (y&z) +;; = (x&y) ^ (x&z) ^ (y&z) ^ (z&z) ^z // note: ((z&z) ^z) = 0 +;; = x&(y^z) ^ z&(y^z) ^z +;; = (x^z)&(y^z) ^z +;; +%macro Maj 4.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%Y %3 + %xdefine %%Z %4 + + mov %%F, %%X + xor %%F, %%Z + xor %%Z, %%Y + and %%F, %%Z + xor %%Z, %%Y + xor %%F, %%Z +%endmacro + +%macro ROTR 2.nolist + %xdefine %%X %1 + %xdefine %%n %2 + + ;;shrd X,X, n + ror %%X, %%n +%endmacro + +;; +;; Summ0(x) = ROR(x,2) ^ ROR(x,13) ^ ROR(x,22) +;; +%macro Summ0 3.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%T %3 + + mov %%F, %%X + ROTR %%F, 2 + mov %%T, %%X + ROTR %%T, 13 + xor %%F, %%T + ROTR %%T, (22-13) + xor %%F, %%T +%endmacro + +;; +;; Summ1(x) = ROR(x,6) ^ ROR(x,11) ^ ROR(x,25) +;; +%macro Summ1 3.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%T %3 + + mov %%F, %%X + ROTR %%F, 6 + mov %%T, %%X + ROTR %%T, 11 + xor %%F, %%T + ROTR %%T, (25-11) + xor %%F, %%T +%endmacro + +;; +;; regular round (i): +;; +;; T1 = h + Sigma1(e) + Ch(e,f,g) + K[i] + W[i] +;; T2 = Sigma0(a) + Maj(a,b,c) +;; h = g +;; g = f +;; f = e +;; e = d + T1 +;; d = c +;; c = b +;; b = a +;; a = T1+T2 +;; +;; or +;; +;; h += Sigma1(e) + Ch(e,f,g) + K[i] + W[i] (==T1) +;; d += h +;; T2 = Sigma0(a) + Maj(a,b,c) +;; h += T2 +;; and following textual shift {a,b,c,d,e,f,g,h} => {h,a,b,c,d,e,f,g} +;; +%macro ROUND 11.nolist + %xdefine %%nr %1 + %xdefine %%A %2 + %xdefine %%B %3 + %xdefine %%C %4 + %xdefine %%D %5 + %xdefine %%E %6 + %xdefine %%F %7 + %xdefine %%G %8 + %xdefine %%H %9 + %xdefine %%T1 %10 + %xdefine %%T2 %11 + + add %%H, dword [rsp+(%%nr & 3)*sizeof(dword)] + Summ1 %%T1, %%E, %%T2 + Chj %%T2, %%E,%%F,%%G + add %%H, %%T1 + add %%H, %%T2 + + add %%D, %%H + + Summ0 %%T1, %%A, %%T2 + Maj %%T2, %%A,%%B,%%C + add %%H, %%T1 + add %%H, %%T2 +%endmacro + +;; W[i] = Sigma1(W[i-2]) + W[i-7] + Sigma0(W[i-15]) + W[i-16], i=16,..,63 +;; +;;for next rounds 16,17,18 and 19: +;; W[0] <= W[16] = Sigma1(W[14]) + W[ 9] + Sigma0(W[1]) + W[0] +;; W[1] <= W[17] = Sigma1(W[15]) + W[10] + Sigma0(W[2]) + W[1] +;; W[2] <= W[18] = Sigma1(W[ 0]) + W[11] + Sigma0(W[3]) + W[1] +;; W[3] <= W[19] = Sigma1(W[ 1]) + W[12] + Sigma0(W[4]) + W[2] +;; +;; the process is repeated exactly because texual round of W[] +;; +;; Sigma1() and Sigma0() functions are defined as following: +;; Sigma1(X) = ROR(X,17)^ROR(X,19)^SHR(X,10) +;; Sigma0(X) = ROR(X, 7)^ROR(X,18)^SHR(X, 3) +;; +%macro UPDATE_W 8.nolist + %xdefine %%xS %1 + %xdefine %%xS0 %2 + %xdefine %%xS4 %3 + %xdefine %%xS8 %4 + %xdefine %%xS12 %5 + %xdefine %%SIGMA1 %6 + %xdefine %%SIGMA0 %7 + %xdefine %%X %8 + + pshufd %%SIGMA1, %%xS12, 11111010b ;; SIGMA1 = {W[15],W[15],W[14],W[14]} + movdqa %%X, %%SIGMA1 + psrld %%SIGMA1, 10 + psrlq %%X, 17 + pxor %%SIGMA1, %%X + psrlq %%X, (19-17) + pxor %%SIGMA1, %%X + + pshufd %%SIGMA0, %%xS0, 10100101b ;; SIGMA0 = {W[2],W[2],W[1],W[1]} + movdqa %%X, %%SIGMA0 + psrld %%SIGMA0, 3 + psrlq %%X, 7 + pxor %%SIGMA0, %%X + psrlq %%X, (18-7) + pxor %%SIGMA0, %%X + + pshufd %%xS, %%xS0, 01010000b ;; {W[ 1],W[ 1],W[ 0],W[ 0]} + pshufd %%X, %%xS8, 10100101b ;; {W[10],W[10],W[ 9],W[ 9]} + paddd %%xS, %%SIGMA1 + paddd %%xS, %%SIGMA0 + paddd %%xS, %%X + + + pshufd %%SIGMA1, %%xS, 10100000b ;; SIGMA1 = {W[1],W[1],W[0],W[0]} + movdqa %%X, %%SIGMA1 + psrld %%SIGMA1, 10 + psrlq %%X, 17 + pxor %%SIGMA1, %%X + psrlq %%X, (19-17) + pxor %%SIGMA1, %%X + + movdqa %%SIGMA0, %%xS4 ;; SIGMA0 = {W[4],W[4],W[3],W[3]} + palignr %%SIGMA0, %%xS0, (3*sizeof(dword)) + pshufd %%SIGMA0, %%SIGMA0, 01010000b + movdqa %%X, %%SIGMA0 + psrld %%SIGMA0, 3 + psrlq %%X, 7 + pxor %%SIGMA0, %%X + psrlq %%X, (18-7) + pxor %%SIGMA0, %%X + + movdqa %%X, %%xS12 + palignr %%X, %%xS8, (3*sizeof(dword)) ;; {W[14],W[13],W[12],W[11]} + pshufd %%xS0, %%xS0, 11111010b ;; {W[ 3],W[ 3],W[ 2],W[ 2]} + pshufd %%X, %%X, 01010000b ;; {W[12],W[12],W[11],W[11]} + paddd %%xS0, %%SIGMA1 + paddd %%xS0, %%SIGMA0 + paddd %%xS0, %%X + + pshufd %%xS, %%xS, 10001000b ;; {W[1],W[0],W[1],W[0]} + pshufd %%xS0, %%xS0, 10001000b ;; {W[3],W[2],W[3],W[2]} + palignr %%xS0, %%xS, (2*sizeof(dword)) ;; {W[3],W[2],W[1],W[0]} + movdqa %%xS, %%xS0 +%endmacro + +segment .text align=IPP_ALIGN_FACTOR + + +align IPP_ALIGN_FACTOR +pByteSwp DB 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; UpdateSHA256(Ipp32u digest[], Ipp8u dataBlock[], int datalen, Ipp32u K_256[]) +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +align IPP_ALIGN_FACTOR +IPPASM UpdateSHA256,PUBLIC +%assign LOCAL_FRAME sizeof(oword)+sizeof(qword) + USES_GPR rbx,rsi,rdi,rbp,r12,r13,r14,r15 + USES_XMM xmm6,xmm7 + COMP_ABI 4 +;; +;; rdi = pointer to the updated hash +;; rsi = pointer to the data block +;; rdx = data block length +;; rcx = pointer to the SHA_256 constant +;; + +%xdefine MBS_SHA256 (64) + + movsxd rdx, edx + mov qword [rsp+sizeof(oword)], rdx ; save length of buffer + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +.sha256_block_loop: + + movdqu W0, oword [rsi] ; load buffer content + movdqu W4, oword [rsi+sizeof(oword)] + movdqu W8, oword [rsi+sizeof(oword)*2] + movdqu W12,oword [rsi+sizeof(oword)*3] + + mov A, dword [rdi] ; load initial hash value + mov B, dword [rdi+sizeof(dword)] + mov C, dword [rdi+sizeof(dword)*2] + mov D, dword [rdi+sizeof(dword)*3] + mov E, dword [rdi+sizeof(dword)*4] + mov F, dword [rdi+sizeof(dword)*5] + mov G, dword [rdi+sizeof(dword)*6] + mov H, dword [rdi+sizeof(dword)*7] + + movdqa XMM_SHUFB_BSWAP, oword [rel pByteSwp] ; load shuffle mask + + ; rcx points K_256[] constants + + ;; perform 0-3 regular rounds + pshufb W0, XMM_SHUFB_BSWAP ; swap input + movdqa W, W0 ; T = W[0-3] + paddd W, oword [rcx+sizeof(oword)*0] ; T += K_SHA256[0-3] + movdqa oword [rsp], W + ROUND 0, A,B,C,D,E,F,G,H, T1,T2 + ROUND 1, H,A,B,C,D,E,F,G, T1,T2 + ROUND 2, G,H,A,B,C,D,E,F, T1,T2 + ROUND 3, F,G,H,A,B,C,D,E, T1,T2 + + ;; perform next 4-7 regular rounds + pshufb W4, XMM_SHUFB_BSWAP ; swap input + movdqa W, W4 ; T = W[4-7] + paddd W, oword [rcx+sizeof(oword)*1] ; T += K_SHA256[4-7] + movdqa oword [rsp], W + ROUND 4, E,F,G,H,A,B,C,D, T1,T2 + ROUND 5, D,E,F,G,H,A,B,C, T1,T2 + ROUND 6, C,D,E,F,G,H,A,B, T1,T2 + ROUND 7, B,C,D,E,F,G,H,A, T1,T2 + + ;; perform next 8-11 regular rounds + pshufb W8, XMM_SHUFB_BSWAP ; swap input + movdqa W, W8 ; T = W[8-11] + paddd W, oword [rcx+sizeof(oword)*2] ; T += K_SHA256[8-11] + movdqa oword [rsp], W + ROUND 8, A,B,C,D,E,F,G,H, T1,T2 + ROUND 9, H,A,B,C,D,E,F,G, T1,T2 + ROUND 10, G,H,A,B,C,D,E,F, T1,T2 + ROUND 11, F,G,H,A,B,C,D,E, T1,T2 + + ;; perform next 12-15 regular rounds + pshufb W12, XMM_SHUFB_BSWAP ; swap input + movdqa W, W12 ; T = W[12-15] + paddd W, oword [rcx+sizeof(oword)*3] ; T += K_SHA256[12-15] + movdqa oword [rsp], W + ROUND 12, E,F,G,H,A,B,C,D, T1,T2 + ROUND 13, D,E,F,G,H,A,B,C, T1,T2 + ROUND 14, C,D,E,F,G,H,A,B, T1,T2 + ROUND 15, B,C,D,E,F,G,H,A, T1,T2 + + UPDATE_W W, W0, W4, W8, W12, SIG1,SIG0,X ; round: 16-19 + paddd W, oword [rcx+sizeof(oword)*4] ; T += K_SHA256[16-19] + movdqa oword [rsp], W + ROUND 16, A,B,C,D,E,F,G,H, T1,T2 + ROUND 17, H,A,B,C,D,E,F,G, T1,T2 + ROUND 18, G,H,A,B,C,D,E,F, T1,T2 + ROUND 19, F,G,H,A,B,C,D,E, T1,T2 + + UPDATE_W W, W4, W8, W12,W0, SIG1,SIG0,X ; round: 20-23 + paddd W, oword [rcx+sizeof(oword)*5] ; T += K_SHA256[20-23] + movdqa oword [rsp], W + ROUND 20, E,F,G,H,A,B,C,D, T1,T2 + ROUND 21, D,E,F,G,H,A,B,C, T1,T2 + ROUND 22, C,D,E,F,G,H,A,B, T1,T2 + ROUND 23, B,C,D,E,F,G,H,A, T1,T2 + + UPDATE_W W, W8, W12,W0, W4, SIG1,SIG0,X ; round: 24-27 + paddd W, oword [rcx+sizeof(oword)*6] ; T += K_SHA256[24-27] + movdqa oword [rsp], W + ROUND 24, A,B,C,D,E,F,G,H, T1,T2 + ROUND 25, H,A,B,C,D,E,F,G, T1,T2 + ROUND 26, G,H,A,B,C,D,E,F, T1,T2 + ROUND 27, F,G,H,A,B,C,D,E, T1,T2 + + UPDATE_W W, W12,W0, W4, W8, SIG1,SIG0,X ; round: 28-31 + paddd W, oword [rcx+sizeof(oword)*7] ; T += K_SHA256[28-31] + movdqa oword [rsp], W + ROUND 28, E,F,G,H,A,B,C,D, T1,T2 + ROUND 29, D,E,F,G,H,A,B,C, T1,T2 + ROUND 30, C,D,E,F,G,H,A,B, T1,T2 + ROUND 31, B,C,D,E,F,G,H,A, T1,T2 + + UPDATE_W W, W0, W4, W8, W12, SIG1,SIG0,X ; round: 32-35 + paddd W, oword [rcx+sizeof(oword)*8] ; T += K_SHA256[32-35] + movdqa oword [rsp], W + ROUND 32, A,B,C,D,E,F,G,H, T1,T2 + ROUND 33, H,A,B,C,D,E,F,G, T1,T2 + ROUND 34, G,H,A,B,C,D,E,F, T1,T2 + ROUND 35, F,G,H,A,B,C,D,E, T1,T2 + + UPDATE_W W, W4, W8, W12,W0, SIG1,SIG0,X ; round: 36-39 + paddd W, oword [rcx+sizeof(oword)*9] ; T += K_SHA256[36-39] + movdqa oword [rsp], W + ROUND 36, E,F,G,H,A,B,C,D, T1,T2 + ROUND 37, D,E,F,G,H,A,B,C, T1,T2 + ROUND 38, C,D,E,F,G,H,A,B, T1,T2 + ROUND 39, B,C,D,E,F,G,H,A, T1,T2 + + UPDATE_W W, W8, W12,W0, W4, SIG1,SIG0,X ; round: 40-43 + paddd W, oword [rcx+sizeof(oword)*10] ; T += K_SHA256[40-43] + movdqa oword [rsp], W + ROUND 40, A,B,C,D,E,F,G,H, T1,T2 + ROUND 41, H,A,B,C,D,E,F,G, T1,T2 + ROUND 42, G,H,A,B,C,D,E,F, T1,T2 + ROUND 43, F,G,H,A,B,C,D,E, T1,T2 + + UPDATE_W W, W12,W0, W4, W8, SIG1,SIG0,X ; round: 44-47 + paddd W, oword [rcx+sizeof(oword)*11] ; T += K_SHA256[44-47] + movdqa oword [rsp], W + ROUND 44, E,F,G,H,A,B,C,D, T1,T2 + ROUND 45, D,E,F,G,H,A,B,C, T1,T2 + ROUND 46, C,D,E,F,G,H,A,B, T1,T2 + ROUND 47, B,C,D,E,F,G,H,A, T1,T2 + + UPDATE_W W, W0, W4, W8, W12, SIG1,SIG0,X ; round: 48-51 + paddd W, oword [rcx+sizeof(oword)*12] ; T += K_SHA256[48-51] + movdqa oword [rsp], W + ROUND 48, A,B,C,D,E,F,G,H, T1,T2 + ROUND 49, H,A,B,C,D,E,F,G, T1,T2 + ROUND 50, G,H,A,B,C,D,E,F, T1,T2 + ROUND 51, F,G,H,A,B,C,D,E, T1,T2 + + UPDATE_W W, W4, W8, W12,W0, SIG1,SIG0,X ; round: 52-55 + paddd W, oword [rcx+sizeof(oword)*13] ; T += K_SHA256[52-55] + movdqa oword [rsp], W + ROUND 52, E,F,G,H,A,B,C,D, T1,T2 + ROUND 53, D,E,F,G,H,A,B,C, T1,T2 + ROUND 54, C,D,E,F,G,H,A,B, T1,T2 + ROUND 55, B,C,D,E,F,G,H,A, T1,T2 + + UPDATE_W W, W8, W12,W0, W4, SIG1,SIG0,X ; round: 56-59 + paddd W, oword [rcx+sizeof(oword)*14] ; T += K_SHA256[56-59] + movdqa oword [rsp], W + ROUND 56, A,B,C,D,E,F,G,H, T1,T2 + ROUND 57, H,A,B,C,D,E,F,G, T1,T2 + ROUND 58, G,H,A,B,C,D,E,F, T1,T2 + ROUND 59, F,G,H,A,B,C,D,E, T1,T2 + + UPDATE_W W, W12,W0, W4, W8, SIG1,SIG0,X ; round: 60-63 + paddd W, oword [rcx+sizeof(oword)*15] ; T += K_SHA256[60-63] + movdqa oword [rsp], W + ROUND 60, E,F,G,H,A,B,C,D, T1,T2 + ROUND 61, D,E,F,G,H,A,B,C, T1,T2 + ROUND 62, C,D,E,F,G,H,A,B, T1,T2 + ROUND 63, B,C,D,E,F,G,H,A, T1,T2 + + add dword [rdi], A ; update hash + add dword [rdi+sizeof(dword)*1], B + add dword [rdi+sizeof(dword)*2], C + add dword [rdi+sizeof(dword)*3], D + add dword [rdi+sizeof(dword)*4], E + add dword [rdi+sizeof(dword)*5], F + add dword [rdi+sizeof(dword)*6], G + add dword [rdi+sizeof(dword)*7], H + + add rsi, MBS_SHA256 + sub qword [rsp+sizeof(oword)], MBS_SHA256 + jg .sha256_block_loop + + REST_XMM + REST_GPR + ret +ENDFUNC UpdateSHA256 + +%endif ;; ((_IPP32E >= _IPP32E_U8 ) AND (_IPP32E < _IPP32E_E9 )) OR (_IPP32E == _IPP32E_N8 ) +%endif ;; _FEATURE_OFF_ / _FEATURE_TICKTOCK_ +%endif ;; _ENABLE_ALG_SHA256_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha512e9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha512e9as.asm new file mode 100644 index 0000000..8c9d5fd --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha512e9as.asm @@ -0,0 +1,732 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SHA512 +; +; Content: +; UpdateSHA256 +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SHA512_) +;;%if (_IPP32E >= _IPP32E_E9) +%if (_IPP32E == _IPP32E_E9 ) + +%xdefine W0 xmm0 +%xdefine W1 xmm1 +%xdefine W2 xmm2 +%xdefine W3 xmm3 +%xdefine W4 xmm4 +%xdefine W5 xmm5 +%xdefine W6 xmm6 +%xdefine W7 xmm7 +%xdefine xT0 xmm8 +%xdefine xT1 xmm9 +%xdefine xT2 xmm10 +%xdefine SIGMA xmm11 + +;; assign hash values to GPU registers +%xdefine A r8 +%xdefine B r9 +%xdefine C r10 +%xdefine D r11 +%xdefine E r12 +%xdefine F r13 +%xdefine G r14 +%xdefine H r15 +%xdefine T1 rax +%xdefine T2 rbx +%xdefine T0 rbp +%xdefine KK_SHA512 rcx + +%macro ROTATE_H 0.nolist + %xdefine %%_TMP H + %xdefine H G + %xdefine G F + %xdefine F E + %xdefine E D + %xdefine D C + %xdefine C B + %xdefine B A + %xdefine A %%_TMP +%endmacro + +%macro ROTATE_W 0.nolist + %xdefine %%DUMMY W0 + %xdefine W0 W1 + %xdefine W1 W2 + %xdefine W2 W3 + %xdefine W3 W4 + %xdefine W4 W5 + %xdefine W5 W6 + %xdefine W6 W7 + %xdefine W7 %%DUMMY +%endmacro + +%macro ROR64 2.nolist + %xdefine %%r %1 + %xdefine %%nbits %2 + + %if _IPP32E >= _IPP32E_L9 + rorx %%r,%%r,%%nbits + %elif _IPP32E >= _IPP32E_Y8 + shld %%r,%%r,(64-%%nbits) + %else + ror %%r,%%nbits + %endif +%endmacro + +;; +;; CHJ(x,y,z) = (x & y) ^ (~x & z) +;; +;; CHJ(x,y,z) = (x & y) ^ (~x & z) +;; = (x&y) ^ ((1^x) &z) +;; = (x&y) ^ (z ^ x&z) +;; = x&y ^ z ^ x&z +;; = x&(y^z) ^z +;; +;; => CHJ(E,F,G) = ((F^G) & E) ^G +;; +%macro CHJ 4.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%Y %3 + %xdefine %%Z %4 + + mov %%F, %%Y + xor %%F, %%Z + and %%F, %%X + xor %%F, %%Z +%endmacro + +;; +;; MAJ(x,y,z) = (x & y) ^ (x & z) ^ (y & z) +;; +;; MAJ(x,y,z) = (x&y) ^ (x&z) ^ (y&z) +;; = (x&y) ^ (x&z) ^ (y&z) ^ (z&z) ^z // note: ((z&z) ^z) = 0 +;; = x&(y^z) ^ z&(y^z) ^z +;; = (x^z)&(y^z) ^z +;; +;; => MAJ(A,B,C) = ((A^C) & B) ^ (A&C) +;; +%macro MAJ 4.nolist + %xdefine %%F %1 + %xdefine %%X %2 + %xdefine %%Y %3 + %xdefine %%Z %4 + + mov %%F, %%X + xor %%F, %%Z ;; maj = x^z + xor %%Y, %%Z ;; y ^= z + and %%F, %%Y ;; maj = (x^z)&(y^z) + xor %%F, %%Z ;; maj = (x^z)&(y^z) ^z + xor %%Y, %%Z ;; restore y +%endmacro + +;; +;; SUM1(x) = ROR64(x,14) ^ ROR64(x,18) ^ ROR64(x,41) +;; +;;=> SUM1(x) = ROR((ROR((ROR(x, 23) ^x), 4) ^x), 14) +;; +%macro SUM1 2.nolist + %xdefine %%F %1 + %xdefine %%X %2 + + mov %%F, %%X + ROR64 %%F, 23 + xor %%F, %%X + ROR64 %%F, 4 + xor %%F, %%X + ROR64 %%F, 14 +%endmacro + +;; +;; SUM0(x) = ROR64(x,28) ^ ROR64(x,34) ^ ROR64(x,39) +;; +;; => SUM0(x) = ROR((ROR((ROR(x, 5) ^x), 6) ^x), 28) +;; +%macro SUM0 2.nolist + %xdefine %%F %1 + %xdefine %%X %2 + + mov %%F, %%X + ROR64 %%F, 5 + xor %%F, %%X + ROR64 %%F, 6 + xor %%F, %%X + ROR64 %%F, 28 +%endmacro + +;; regular SHA512 step +;; +;; Ipp64u T1 = H + SUM1(E) + CHJ(E,F,G) + K_SHA512[t] + W[t]; +;; Ipp64u T2 = SUM0(A) + MAJ(A,B,C); +;; D+= T1; +;; H = T1 + T2; +;; +;; where +;; SUM1(x) = ROR64(x,14) ^ ROR64(x,18) ^ ROR64(x,41) +;; SUM0(x) = ROR64(x,28) ^ ROR64(x,34) ^ ROR64(x,39) +;; +;; CHJ(x,y,z) = (x & y) ^ (~x & z) +;; MAJ(x,y,z) = (x & y) ^ (x & z) ^ (y & z) = (x&y)^((x^y)&z) +;; +%macro SHA512_STEP_v0 1.nolist + %xdefine %%nr %1 + + add H, qword [rsp+(%%nr & 1)*sizeof(qword)] + SUM1 T0, E + CHJ T1, E,F,G + add H, T1 + add H, T0 + + add D, H + + SUM0 T0, A + MAJ T1, A,B,C + add H, T1 + add H, T0 + + ROTATE_H +%endmacro + +%macro SHA512_2step 9.nolist + %xdefine %%nr %1 + %xdefine %%A %2 + %xdefine %%B %3 + %xdefine %%C %4 + %xdefine %%D %5 + %xdefine %%E %6 + %xdefine %%F %7 + %xdefine %%G %8 + %xdefine %%H %9 + + mov T0, %%E ;; T0: SUM1(E) + ROR64 T0, 23 ;; T0: SUM1(E) + add %%H, qword [rsp+(%%nr & 1)*sizeof(qword)] + mov T1, %%F ;; T1: CHJ(E,F,G) + xor T0, %%E ;; T0: SUM1(E) + ROR64 T0, 4 ;; T0: SUM1(E) + xor T1, %%G ;; T1: CHJ(E,G,G) + and T1, %%E ;; T1: CHJ(E,G,G) + xor T0, %%E ;; T0: SUM1(E) + ROR64 T0,14 ;; T0: SUM1(E) + xor T1, %%G ;; T1: CHJ(E,G,G) + add %%H, T1 + add %%H, T0 ;; H += SUM1(E) + CHJ(E,F,G) + K_SHA512[t] + W[t] + + add %%D, %%H + + mov T0, %%A ;; T0: SUM0(A) + ROR64 T0, 5 ;; T0: SUM0(A) + mov T1, %%A ;; T1: MAJ(A,B,C) + xor T1, %%C ;; T1: MAJ(A,B,C) + xor T0, %%A ;; T0: SUM0(A) + ROR64 T0, 6 ;; T0: SUM0(A) + xor %%B, %%C ;; T1: MAJ(A,B,C) + and T1, %%B ;; T1: MAJ(A,B,C) + xor T0, %%A ;; T0: SUM0(A) + ROR64 T0,28 ;; T0: SUM0(A) + xor %%B, %%C ;; T1: MAJ(A,B,C) + xor T1, %%C ;; T1: MAJ(A,B,C) + + add %%H, T0 + add %%H, T1 + + ;ROTATE_H + + mov T0, %%D ;; T0: SUM1(E) + ROR64 T0, 23 ;; T0: SUM1(E) + add %%G, qword [rsp+((%%nr+1) & 1)*sizeof(qword)] + mov T1, %%E ;; T1: CHJ(E,F,G) + xor T0, %%D ;; T0: SUM1(E) + ROR64 T0, 4 ;; T0: SUM1(E) + xor T1, %%F ;; T1: CHJ(E,G,G) + and T1, %%D ;; T1: CHJ(E,G,G) + xor T0, %%D ;; T0: SUM1(E) + ROR64 T0,14 ;; T0: SUM1(E) + xor T1, %%F ;; T1: CHJ(E,G,G) + add %%G, T1 + add %%G, T0 ;; H += SUM1(E) + CHJ(E,F,G) + K_SHA512[t] + W[t] + + add %%C, %%G + + mov T0, %%H ;; T0: SUM0(A) + ROR64 T0, 5 ;; T0: SUM0(A) + mov T1, %%H ;; T1: MAJ(A,B,C) + xor T1, %%B ;; T1: MAJ(A,B,C) + xor T0, %%H ;; T0: SUM0(A) + ROR64 T0, 6 ;; T0: SUM0(A) + xor %%A, %%B ;; T1: MAJ(A,B,C) + and T1, %%A ;; T1: MAJ(A,B,C) + xor T0, %%H ;; T0: SUM0(A) + ROR64 T0,28 ;; T0: SUM0(A) + xor %%A, %%B ;; T1: MAJ(A,B,C) + xor T1, %%B ;; T1: MAJ(A,B,C) + + add %%G, T0 + add %%G, T1 + + ;ROTATE_H +%endmacro + +;; +;; update W[] +;; +;; W[j] = SIG1(W[j- 2]) + W[j- 7] +;; +SIG0(W[j-15]) + W[j-16] +;; +;; SIG0(x) = ROR(x,1) ^ROR(x,8) ^LSR(x,7) +;; SIG1(x) = ROR(x,19)^ROR(x,61) ^LSR(x,6) +;; +%macro SHA512_2Wupdate 0.nolist + vpalignr xT1, W5, W4, 8 ;; xT1 = W[t-7] + vpalignr xT0, W1, W0, 8 ;; xT0 = W[t-15] + vpaddq W0, W0, xT1 ;; W0 = W0 + W[t-7] + + vpsrlq SIGMA, W7, 6 ;; SIG1: W[t-2]>>6 + vpsrlq xT1, W7,61 ;; SIG1: W[t-2]>>61 + vpsllq xT2, W7,(64-61) ;; SIG1: W[t-2]<<(64-61) + vpxor SIGMA, SIGMA, xT1 + vpxor SIGMA, SIGMA, xT2 + vpsrlq xT1, W7,19 ;; SIG1: W[t-2]>>19 + vpsllq xT2, W7,(64-19) ;; SIG1: W[t-2]<<(64-19) + vpxor SIGMA, SIGMA, xT1 + vpxor SIGMA, SIGMA, xT2 + vpaddq W0, W0, SIGMA ;; W0 = W0 + W[t-7] + SIG1(W[t-2]) + + vpsrlq SIGMA, xT0, 7 ;; SIG0: W[t-15]>>7 + vpsrlq xT1, xT0, 1 ;; SIG0: W[t-15]>>1 + vpsllq xT2, xT0,(64-1) ;; SIG0: W[t-15]<<(64-1) + vpxor SIGMA, SIGMA, xT1 + vpxor SIGMA, SIGMA, xT2 + vpsrlq xT1, xT0, 8 ;; SIG0: W[t-15]>>8 + vpsllq xT2, xT0,(64-8) ;; SIG0: W[t-15]<<(64-8) + vpxor SIGMA, SIGMA, xT1 + vpxor SIGMA, SIGMA, xT2 + vpaddq W0, W0, SIGMA ;; W0 = W0 + W[t-7] + SIG1(W[t-2]) +SIG0(W[t-15]) + + ROTATE_W +%endmacro + +%macro SHA512_2step_2Wupdate 9.nolist + %xdefine %%nr %1 + %xdefine %%A %2 + %xdefine %%B %3 + %xdefine %%C %4 + %xdefine %%D %5 + %xdefine %%E %6 + %xdefine %%F %7 + %xdefine %%G %8 + %xdefine %%H %9 + + add %%H, qword [rsp+(%%nr & 1)*sizeof(qword)] + + vpalignr xT1, W5, W4, 8 ;; xT1 = W[t-7] + + mov T0, %%E ;; T0: SUM1(E) + ROR64 T0, 23 ;; T0: SUM1(E) + + vpalignr xT0, W1, W0, 8 ;; xT0 = W[t-15] + + mov T1, %%F ;; T1: CHJ(E,F,G) + xor T0, %%E ;; T0: SUM1(E) + + vpaddq W0, W0, xT1 ;; W0 = W0 + W[t-7] + + ROR64 T0, 4 ;; T0: SUM1(E) + xor T1, %%G ;; T1: CHJ(E,G,G) + + vpsrlq SIGMA, W7, 6 ;; SIG1: W[t-2]>>6 + + and T1, %%E ;; T1: CHJ(E,G,G) + xor T0, %%E ;; T0: SUM1(E) + ROR64 T0,14 ;; T0: SUM1(E) + + vpsrlq xT1, W7,61 ;; SIG1: W[t-2]>>61 + + xor T1, %%G ;; T1: CHJ(E,G,G) + add %%H, T1 + add %%H, T0 ;; H += SUM1(E) + CHJ(E,F,G) + K_SHA512[t] + W[t] + + vpsllq xT2, W7,(64-61) ;; SIG1: W[t-2]<<(64-61) + + add %%D, %%H + mov T0, %%A ;; T0: SUM0(A) + ROR64 T0, 5 ;; T0: SUM0(A) + + vpxor SIGMA, SIGMA, xT1 + + mov T1, %%A ;; T1: MAJ(A,B,C) + xor T1, %%C ;; T1: MAJ(A,B,C) + + vpxor SIGMA, SIGMA, xT2 + + xor T0, %%A ;; T0: SUM0(A) + ROR64 T0, 6 ;; T0: SUM0(A) + + vpsrlq xT1, W7,19 ;; SIG1: W[t-2]>>19 + + xor %%B, %%C ;; T1: MAJ(A,B,C) + and T1, %%B ;; T1: MAJ(A,B,C) + xor T0, %%A ;; T0: SUM0(A) + + vpsllq xT2, W7,(64-19) ;; SIG1: W[t-2]<<(64-19) + + ROR64 T0,28 ;; T0: SUM0(A) + xor %%B, %%C ;; T1: MAJ(A,B,C) + xor T1, %%C ;; T1: MAJ(A,B,C) + + vpxor SIGMA, SIGMA, xT1 + add %%H, T0 + add %%H, T1 + ;ROTATE_H + + vpxor SIGMA, SIGMA, xT2 + + mov T0, %%D ;; T0: SUM1(E) + ROR64 T0, 23 ;; T0: SUM1(E) + + vpaddq W0, W0, SIGMA ;; W0 = W0 + W[t-7] + SIG1(W[t-2]) + + add %%G, qword [rsp+((%%nr+1) & 1)*sizeof(qword)] + mov T1, %%E ;; T1: CHJ(E,F,G) + + vpsrlq SIGMA, xT0, 7 ;; SIG0: W[t-15]>>7 + + xor T0, %%D ;; T0: SUM1(E) + ROR64 T0, 4 ;; T0: SUM1(E) + xor T1, %%F ;; T1: CHJ(E,G,G) + + vpsrlq xT1, xT0, 1 ;; SIG0: W[t-15]>>1 + + and T1, %%D ;; T1: CHJ(E,G,G) + xor T0, %%D ;; T0: SUM1(E) + ROR64 T0,14 ;; T0: SUM1(E) + + vpsllq xT2, xT0,(64-1) ;; SIG0: W[t-15]<<(64-1) + + xor T1, %%F ;; T1: CHJ(E,G,G) + add %%G, T1 + add %%G, T0 ;; H += SUM1(E) + CHJ(E,F,G) + K_SHA512[t] + W[t] + + vpxor SIGMA, SIGMA, xT1 + + add %%C, %%G + mov T0, %%H ;; T0: SUM0(A) + + vpxor SIGMA, SIGMA, xT2 + + ROR64 T0, 5 ;; T0: SUM0(A) + mov T1, %%H ;; T1: MAJ(A,B,C) + + vpsrlq xT1, xT0, 8 ;; SIG0: W[t-15]>>8 + + xor T1, %%B ;; T1: MAJ(A,B,C) + xor T0, %%H ;; T0: SUM0(A) + ROR64 T0, 6 ;; T0: SUM0(A) + + vpsllq xT2, xT0,(64-8) ;; SIG0: W[t-15]<<(64-8) + + xor %%A, %%B ;; T1: MAJ(A,B,C) + and T1, %%A ;; T1: MAJ(A,B,C) + xor T0, %%H ;; T0: SUM0(A) + + vpxor SIGMA, SIGMA, xT1 + + ROR64 T0,28 ;; T0: SUM0(A) + xor %%A, %%B ;; T1: MAJ(A,B,C) + + vpxor SIGMA, SIGMA, xT2 + xor T1, %%B ;; T1: MAJ(A,B,C) + add %%G, T0 + + vpaddq W0, W0, SIGMA ;; W0 = W0 + W[t-7] + SIG1(W[t-2]) +SIG0(W[t-15]) + add %%G, T1 + + ;ROTATE_H + ROTATE_W +%endmacro + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR +SHUFB_BSWAP DB 7,6,5,4,3,2,1,0, 15,14,13,12,11,10,9,8 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; UpdateSHA512(Ipp64u digest[], Ipp8u dataBlock[], int datalen, Ipp64u K_512[]) +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +align IPP_ALIGN_FACTOR +IPPASM UpdateSHA512,PUBLIC +%assign LOCAL_FRAME sizeof(oword)+sizeof(qword) + USES_GPR rbx,rsi,rdi,rbp,r12,r13,r14,r15 + USES_XMM xmm6,xmm7,xmm8,xmm9,xmm10,xmm11 + COMP_ABI 4 +;; +;; rdi = pointer to the updated hash +;; rsi = pointer to the data block +;; rdx = data block length +;; rcx = pointer to the SHA_512 constant +;; + +%xdefine MBS_SHA512 (128) + + vmovdqa xT1, oword [rel SHUFB_BSWAP] + movsxd rdx, edx + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +.sha512_block_loop: +;; +;; initialize the first 16 qwords in the array W (remember about endian) +;; + vmovdqu W0, oword [rsi] ; load buffer content and swap + vpshufb W0, W0, xT1 + vmovdqu W1, oword [rsi+sizeof(oword)*1]; + vpshufb W1, W1, xT1 + vmovdqu W2, oword [rsi+sizeof(oword)*2]; + vpshufb W2, W2, xT1 + vmovdqu W3, oword [rsi+sizeof(oword)*3]; + vpshufb W3, W3, xT1 + vmovdqu W4, oword [rsi+sizeof(oword)*4]; + vpshufb W4, W4, xT1 + vmovdqu W5, oword [rsi+sizeof(oword)*5]; + vpshufb W5, W5, xT1 + vmovdqu W6, oword [rsi+sizeof(oword)*6]; + vpshufb W6, W6, xT1 + vmovdqu W7, oword [rsi+sizeof(oword)*7]; + vpshufb W7, W7, xT1 + + mov A, qword [rdi] ; load initial hash value + mov B, qword [rdi+sizeof(qword)] + mov C, qword [rdi+sizeof(qword)*2] + mov D, qword [rdi+sizeof(qword)*3] + mov E, qword [rdi+sizeof(qword)*4] + mov F, qword [rdi+sizeof(qword)*5] + mov G, qword [rdi+sizeof(qword)*6] + mov H, qword [rdi+sizeof(qword)*7] + + ;; perform 0- 9 rounds + vpaddq xT1, W0, oword [KK_SHA512+ 0*sizeof(qword)] ; T += K_SHA512[0-1] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 0, A,B,C,D,E,F,G,H + + vpaddq xT1, W0, oword [KK_SHA512+ 2*sizeof(qword)] ; T += K_SHA512[2-3] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 2, G,H,A,B,C,D,E,F + + vpaddq xT1, W0, oword [KK_SHA512+ 4*sizeof(qword)] ; T += K_SHA512[4-5] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 4, E,F,G,H,A,B,C,D + + vpaddq xT1, W0, oword [KK_SHA512+ 6*sizeof(qword)] ; T += K_SHA512[6-7] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 6, C,D,E,F,G,H,A,B + + vpaddq xT1, W0, oword [KK_SHA512+ 8*sizeof(qword)] ; T += K_SHA512[8-9] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 8, A,B,C,D,E,F,G,H + + ;; perform 10-19 rounds + vpaddq xT1, W0, oword [KK_SHA512+10*sizeof(qword)] ; T += K_SHA512[10-11] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 10, G,H,A,B,C,D,E,F + + vpaddq xT1, W0, oword [KK_SHA512+12*sizeof(qword)] ; T += K_SHA512[12-13] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 12, E,F,G,H,A,B,C,D + + vpaddq xT1, W0, oword [KK_SHA512+14*sizeof(qword)] ; T += K_SHA512[14-15] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 14, C,D,E,F,G,H,A,B + + vpaddq xT1, W0, oword [KK_SHA512+16*sizeof(qword)] ; T += K_SHA512[16-17] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 16, A,B,C,D,E,F,G,H + + vpaddq xT1, W0, oword [KK_SHA512+18*sizeof(qword)] ; T += K_SHA512[18-19] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 18, G,H,A,B,C,D,E,F + + ;; perform 20-29 rounds + vpaddq xT1, W0, oword [KK_SHA512+20*sizeof(qword)] ; T += K_SHA512[20-21] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 20, E,F,G,H,A,B,C,D + + vpaddq xT1, W0, oword [KK_SHA512+22*sizeof(qword)] ; T += K_SHA512[22-23] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 22, C,D,E,F,G,H,A,B + + vpaddq xT1, W0, oword [KK_SHA512+24*sizeof(qword)] ; T += K_SHA512[24-25] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 24, A,B,C,D,E,F,G,H + + vpaddq xT1, W0, oword [KK_SHA512+26*sizeof(qword)] ; T += K_SHA512[26-27] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 26, G,H,A,B,C,D,E,F + + vpaddq xT1, W0, oword [KK_SHA512+28*sizeof(qword)] ; T += K_SHA512[28-29] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 28, E,F,G,H,A,B,C,D + + ;; perform 30-39 rounds + vpaddq xT1, W0, oword [KK_SHA512+30*sizeof(qword)] ; T += K_SHA512[30-31] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 30, C,D,E,F,G,H,A,B + + vpaddq xT1, W0, oword [KK_SHA512+32*sizeof(qword)] ; T += K_SHA512[32-33] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 32, A,B,C,D,E,F,G,H + + vpaddq xT1, W0, oword [KK_SHA512+34*sizeof(qword)] ; T += K_SHA512[34-35] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 34, G,H,A,B,C,D,E,F + + vpaddq xT1, W0, oword [KK_SHA512+36*sizeof(qword)] ; T += K_SHA512[36-37] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 36, E,F,G,H,A,B,C,D + + vpaddq xT1, W0, oword [KK_SHA512+38*sizeof(qword)] ; T += K_SHA512[38-39] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 38, C,D,E,F,G,H,A,B + + ;; perform 40-49 rounds + vpaddq xT1, W0, oword [KK_SHA512+40*sizeof(qword)] ; T += K_SHA512[40-41] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 40, A,B,C,D,E,F,G,H + + vpaddq xT1, W0, oword [KK_SHA512+42*sizeof(qword)] ; T += K_SHA512[42-43] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 42, G,H,A,B,C,D,E,F + + vpaddq xT1, W0, oword [KK_SHA512+44*sizeof(qword)] ; T += K_SHA512[44-45] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 44, E,F,G,H,A,B,C,D + + vpaddq xT1, W0, oword [KK_SHA512+46*sizeof(qword)] ; T += K_SHA512[46-47] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 46, C,D,E,F,G,H,A,B + + vpaddq xT1, W0, oword [KK_SHA512+48*sizeof(qword)] ; T += K_SHA512[48-49] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 48, A,B,C,D,E,F,G,H + + ;; perform 50-59 rounds + vpaddq xT1, W0, oword [KK_SHA512+50*sizeof(qword)] ; T += K_SHA512[50-51] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 50, G,H,A,B,C,D,E,F + + vpaddq xT1, W0, oword [KK_SHA512+52*sizeof(qword)] ; T += K_SHA512[52-53] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 52, E,F,G,H,A,B,C,D + + vpaddq xT1, W0, oword [KK_SHA512+54*sizeof(qword)] ; T += K_SHA512[54-55] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 54, C,D,E,F,G,H,A,B + + vpaddq xT1, W0, oword [KK_SHA512+56*sizeof(qword)] ; T += K_SHA512[56-57] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 56, A,B,C,D,E,F,G,H + + vpaddq xT1, W0, oword [KK_SHA512+58*sizeof(qword)] ; T += K_SHA512[58-59] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 58, G,H,A,B,C,D,E,F + + ;; perform 60-69 rounds + vpaddq xT1, W0, oword [KK_SHA512+60*sizeof(qword)] ; T += K_SHA512[60-61] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 60, E,F,G,H,A,B,C,D + + vpaddq xT1, W0, oword [KK_SHA512+62*sizeof(qword)] ; T += K_SHA512[62-63] + vmovdqa oword [rsp], xT1 + SHA512_2step_2Wupdate 62, C,D,E,F,G,H,A,B + + vpaddq xT1, W0, oword [KK_SHA512+64*sizeof(qword)] ; T += K_SHA512[64-65] + vmovdqa oword [rsp], xT1 + SHA512_2step 64, A,B,C,D,E,F,G,H + ROTATE_W + + vpaddq xT1, W0, oword [KK_SHA512+66*sizeof(qword)] ; T += K_SHA512[66-67] + vmovdqa oword [rsp], xT1 + SHA512_2step 66, G,H,A,B,C,D,E,F + ROTATE_W + + vpaddq xT1, W0, oword [KK_SHA512+68*sizeof(qword)] ; T += K_SHA512[68-69] + vmovdqa oword [rsp], xT1 + SHA512_2step 68, E,F,G,H,A,B,C,D + ROTATE_W + + ;; perform 70-79 rounds + vpaddq xT1, W0, oword [KK_SHA512+70*sizeof(qword)] ; T += K_SHA512[70-71] + vmovdqa oword [rsp], xT1 + SHA512_2step 70, C,D,E,F,G,H,A,B + ROTATE_W + + vpaddq xT1, W0, oword [KK_SHA512+72*sizeof(qword)] ; T += K_SHA512[72-73] + vmovdqa oword [rsp], xT1 + SHA512_2step 72, A,B,C,D,E,F,G,H + ROTATE_W + + vpaddq xT1, W0, oword [KK_SHA512+74*sizeof(qword)] ; T += K_SHA512[74-75] + vmovdqa oword [rsp], xT1 + SHA512_2step 74, G,H,A,B,C,D,E,F + ROTATE_W + + vpaddq xT1, W0, oword [KK_SHA512+76*sizeof(qword)] ; T += K_SHA512[76-77] + vmovdqa oword [rsp], xT1 + SHA512_2step 76, E,F,G,H,A,B,C,D + ROTATE_W + + vpaddq xT1, W0, oword [KK_SHA512+78*sizeof(qword)] ; T += K_SHA512[78-79] + vmovdqa oword [rsp], xT1 + SHA512_2step 78, C,D,E,F,G,H,A,B + ROTATE_W + + add qword [rdi], A ; update shash + add qword [rdi+sizeof(qword)*1], B + add qword [rdi+sizeof(qword)*2], C + add qword [rdi+sizeof(qword)*3], D + add qword [rdi+sizeof(qword)*4], E + add qword [rdi+sizeof(qword)*5], F + add qword [rdi+sizeof(qword)*6], G + add qword [rdi+sizeof(qword)*7], H + + vmovdqa xT1, oword [rel SHUFB_BSWAP] + add rsi, MBS_SHA512 + sub rdx, MBS_SHA512 + jg .sha512_block_loop + + REST_XMM + REST_GPR + ret +ENDFUNC UpdateSHA512 + +%endif ;; (_IPP32E >= _IPP32E_E9 ) +%endif ;; _ENABLE_ALG_SHA512_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha512l9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha512l9as.asm new file mode 100644 index 0000000..5cde357 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha512l9as.asm @@ -0,0 +1,635 @@ +;=============================================================================== +; Copyright (C) 2017 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SHA512 +; +; Content: +; UpdateSHA512 +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "ia_32e_regs.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SHA512_) +%if (_SHA_NI_ENABLING_ == _FEATURE_OFF_) || (_SHA_NI_ENABLING_ == _FEATURE_TICKTOCK_) +%if (_IPP32E >= _IPP32E_L9 ) + +;; +;; assignments +;; +%xdefine hA rax ;; hash values into GPU registers +%xdefine hB rbx +%xdefine hC rcx +%xdefine hD rdx +%xdefine hE r8 +%xdefine hF r9 +%xdefine hG r10 +%xdefine hH r11 + +%xdefine T1 r12 ;; scratch +%xdefine T2 r13 +%xdefine T3 r14 +%xdefine T4 r15 +%xdefine T5 rdi + +%xdefine W0 ymm0 ;; W values into YMM registers +%xdefine W1 ymm1 +%xdefine W2 ymm2 +%xdefine W3 ymm3 +%xdefine W4 ymm4 +%xdefine W5 ymm5 +%xdefine W6 ymm6 +%xdefine W7 ymm7 + +%xdefine yT1 ymm8 ;; scratch +%xdefine yT2 ymm9 +%xdefine yT3 ymm10 +%xdefine yT4 ymm11 + +%xdefine W0L xmm0 +%xdefine W1L xmm1 +%xdefine W2L xmm2 +%xdefine W3L xmm3 +%xdefine W4L xmm4 +%xdefine W5L xmm5 +%xdefine W6L xmm6 +%xdefine W7L xmm7 + +%xdefine YMM_SHUFB_BSWAP ymm12 ;; byte swap constant + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; +;; textual rotation of W args +;; +%macro ROTATE_W 0.nolist + %xdefine %%_X W0 + %xdefine W0 W1 + %xdefine W1 W2 + %xdefine W2 W3 + %xdefine W3 W4 + %xdefine W4 W5 + %xdefine W5 W6 + %xdefine W6 W7 + %xdefine W7 %%_X +%endmacro + +;; +;; textual rotation of HASH arguments +;; +%macro ROTATE_H 0.nolist + %xdefine %%_X hH + %xdefine hH hG + %xdefine hG hF + %xdefine hF hE + %xdefine hE hD + %xdefine hD hC + %xdefine hC hB + %xdefine hB hA + %xdefine hA %%_X + %endmacro + +%macro ROTATE_T4_T5 0.nolist + %xdefine %%T T4 + %xdefine T4 T5 + %xdefine T5 %%T +%endmacro + +;; +;; update next 2 W[t] and W[t+1], t=16,...79 +;; +;; W[j] = sig1(W[t- 2]) + W[t- 7] +;; +sig0(W[t-15]) + W[t-16] +;; +;; sig0(x) = ROR(x,1) ^ROR(x,8) ^SHR(x,7) +;; = SHR(x,7) +;; ^SHR(x,1) ^SHL(x,63) +;; ^SHR(x,8) ^SHL(x,56) +;; sig1(x) = ROR(x,19)^ROR(x,61) ^SHR(x,6) +;; = SHR(x,6) +;; ^SHR(x,19) ^SHL(x,45) +;; ^SHR(x,61) ^SHL(x,3) +;; +%macro UPDATE_W 9.nolist + %xdefine %%nr %1 + %xdefine %%W0 %2 + %xdefine %%W1 %3 + %xdefine %%W2 %4 + %xdefine %%W3 %5 + %xdefine %%W4 %6 + %xdefine %%W5 %7 + %xdefine %%W6 %8 + %xdefine %%W7 %9 + + %assign %%W_AHEAD 16 + + vpalignr yT1,%%W1,%%W0,sizeof(qword) ;; T1 = {w2:w1} args sig0() + vpalignr yT4,%%W5,%%W4,sizeof(qword) ;; T4 = {w10:w9} additive + vpsrlq yT3,yT1,1 ;; sig0(): T3 = {w2:w1}>>1 + vpaddq %%W0,%%W0,yT4 ;; W0 += {w10:w9} + vpsrlq yT4,yT1,7 ;; sig0(): {w2:w1}>>7 + vpsllq yT2,yT1,56 ;; sig0(): {w2:w1}<<(64-8) + vpxor yT1,yT4,yT3 ;; sig0(): {w2:w1}>>7 ^ {w2:w1}>>1 + vpsrlq yT3,yT3, 7 ;; sig0(): {w2:w1}>>1>>7 = {w2:w1}>>8 + vpxor yT1,yT1,yT2 ;; sig0(): {w2:w1}>>7 ^ {w2:w1}>>1 ^ {w2:w1}<<(64-8) + vpsllq yT2,yT2, 7 ;; sig0(): {w2:w1}<<(64-8)<<7 = {w2:w1}<<63 + vpxor yT1,yT1,yT3 ;; sig0(): {w2:w1}>>7 ^ {w2:w1}>>1 ^ {w2:w1}<<(64-8) ^ {w2:w1}>>8 + vpsrlq yT4,%%W7, 6 ;; sig1(): {w15:w14}>>6 + vpxor yT1,yT1,yT2 ;; sig0()= {w2:w1}>>7 + ;; ^{w2:w1}>>1 ^ {w2:w1}<<63 + ;; ^{w2:w1}<<(64-8) ^ {w2:w1}>>8 + vpsllq yT3,%%W7, 3 ;; sig1(): {w15:w14}<<3 + vpaddq %%W0,%%W0,yT1 ;; W0 += sig0() + vpsrlq yT2,%%W7, 19 ;; sig1(): {w15:w14}>>19 + vpxor yT4,yT4,yT3 ;; sig1(): {w15:w14}>>6 ^ {w15:w14}<<3 + vpsllq yT3,yT3, 42 ;; sig1(): {w15:w14}<<3<<42 = {w15:w14}<<45 + vpxor yT4,yT4,yT2 ;; sig1(): {w15:w14}>>6 ^ {w15:w14}<<3 ^ {w15:w14}>>19 + vpsrlq yT2,yT2,42 ;; sig1(): {w15:w14}>>19>>42= {w15:w14}>>61 + vpxor yT4,yT4,yT3 ;; sig1(): {w15:w14}>>6 ^ {w15:w14}<<3 ^ {w15:w14}>>19 ^ {w15:w14}<<45 + vpxor yT4,yT4,yT2 ;; sig1()= {w15:w14}>>6 + ;; ^{w15:w14}<<3 ^ {w15:w14}>>61 + ;; ^{w15:w14}>>19 ^ {w15:w14}<<45 + vpaddq %%W0,%%W0,yT4 ;; W0 += sig1() + vpaddq yT3,%%W0,YMMWORD [rbp+(%%nr/2)*sizeof(ymmword)] + vmovdqa YMMWORD [rsi+(%%W_AHEAD/2)*sizeof(ymmword)+(%%nr/2)*sizeof(ymmword)],yT3 + ROTATE_W +%endmacro + +;; +;; regular round (i): +;; +;; T1 = h + Sum1(e) + Ch(e,f,g) + K[i] + W[i] +;; T2 = Sum0(a) + Maj(a,b,c) +;; h = g +;; g = f +;; f = e +;; e = d + T1 +;; d = c +;; c = b +;; b = a +;; a = T1+T2 +;; +;; sum1(e) = (e>>>14)^(e>>>18)^(e>>>41) +;; sum0(a) = (a>>>28)^(a>>>34)^(a>>>39) +;; ch(e,f,g) = (e&f)^(~e^g) +;; maj(a,b,m)= (a&b)^(a&c)^(b&c) +;; +;; note: +;; 1) U + ch(e,f,g) = U + (e&f) & (~e&g) +;; 2) maj(a,b,c)= (a&b)^(a&c)^(b&c) = (a^b)&(b^c) ^b +;; to make sure both are correct - use GF(2) arith instead of logic +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; or +;; X = sum0(a[i-1]) computed on prev round +;; a[i] += X +;; h[i] += (K[i]+W[i]) + sum1(e[i]) + ch(e[i],f[i],g[i]) or +;; h[i] += (K[i]+W[i]) + sum1(e[i]) + (e[i]&f[i]) + (~e[i]&g[i]) -- helps break dependencies +;; d[i] += h[i] +;; h[i] += maj(a[i],b[i],c[i]) +;; and following textual shift +;; {a[i+1],b[i+1],c[i+1],d[i+1],e[i+1],f[i+1],g[i+1],h[i+1]} <= {h[i],a[i],b[i],c[i],d[i],e[i],f[i],g[i]} +;; +;; on entry: +;; - T1 = f +;; - T3 = sum0{a[i-1]) +;; - T5 = b&c +%macro SHA512_ROUND 9.nolist + %xdefine %%nr %1 + %xdefine %%hA %2 + %xdefine %%hB %3 + %xdefine %%hC %4 + %xdefine %%hD %5 + %xdefine %%hE %6 + %xdefine %%hF %7 + %xdefine %%hG %8 + %xdefine %%hH %9 + + add %%hH, qword [rsi+(%%nr/2)*sizeof(ymmword)+(%%nr & 1)*sizeof(qword)] ;; h += (k[t]+w[t]) + and T1, %%hE ;; ch(e,f,g): (f&e) + rorx T2, %%hE, 41 ;; sum1(e): e>>>41 + rorx T4, %%hE, 18 ;; sum1(e): e>>>18 + add %%hA, T3 ;; complete computation a += sum0(a[t-1]) + add %%hH, T1 ;; h += (k[t]+w[t]) + (f&e) + andn T1, %%hE, %%hG ;; ch(e,f,g): (~e&g) + xor T2, T4 ;; sum1(e): (e>>>41)^(e>>>18) + rorx T3, %%hE, 14 ;; sum1(e): e>>>14 + add %%hH, T1 ;; h += (k[t]+w[t]) + (f&e) + (~e&g) + xor T2, T3 ;; sum1(e) = (e>>>41)^(e>>>18)^(e>>>14) + mov T4, %%hA ;; maj(a,b,c): a + rorx T1, %%hA, 39 ;; sum0(a): a>>>39 + add %%hH, T2 ;; h += (k[t]+w[t]) +(f&e) +(~e&g) +sig1(e) + xor T4, %%hB ;; maj(a,b,c): (a^b) + rorx T3, %%hA, 34 ;; sum0(a): a>>>34 + rorx T2, %%hA, 28 ;; sum0(a): a>>>28 + add %%hD, %%hH ;; d += h + and T5, T4 ;; maj(a,b,c): (b^c)&(a^b) + xor T3, T1 ;; sum0(a): (a>>>13)^(a>>>22) + xor T5, %%hB ;; maj(a,b,c) = (b^c)&(a^b)^b = (a&b)^(a&c)^(b&c) + xor T3, T2 ;; sum0(a): = (a>>>13)^(a>>>22)^(a>>>2) + add %%hH, T5 ;; h += (k[t]+w[t]) +(f&e) +(~e&g) +sig1(e) +maj(a,b,c) + mov T1, %%hE ;; T1 = f (next round) + ROTATE_T4_T5 ;; T5 = (b^c) (next round) +%endmacro + +;; +;; does 2 regular rounds and computes next 2 W values +;; (just 2 instances of SHA512_ROUND merged together woth UPDATE_W) +;; +%macro SHA512_2ROUND_SHED 1.nolist + %xdefine %%round %1 + + %assign %%W_AHEAD 16 + %assign %%nr %%round + +vpalignr yT1,W1,W0,sizeof(qword) + add hH, qword [rsi+(%%nr/2)*sizeof(ymmword)+(%%nr & 1)*sizeof(qword)] + and T1, hE +vpalignr yT4,W5,W4,sizeof(qword) + rorx T2, hE, 41 + rorx T4, hE, 18 +vpsrlq yT3,yT1,1 + add hA, T3 + add hH, T1 +vpaddq W0,W0,yT4 + andn T1, hE, hG + xor T2, T4 +vpsrlq yT4,yT1,7 + rorx T3, hE, 14 + add hH, T1 +vpsllq yT2,yT1,56 + xor T2, T3 + mov T4, hA +vpxor yT1,yT4,yT3 + rorx T1, hA, 39 + add hH, T2 +vpsrlq yT3,yT3, 7 + xor T4, hB + rorx T3, hA, 34 +vpxor yT1,yT1,yT2 + rorx T2, hA, 28 + add hD, hH +vpsllq yT2,yT2, 7 + and T5, T4 + xor T3, T1 +vpxor yT1,yT1,yT3 + xor T5, hB + xor T3, T2 +vpsrlq yT4,W7, 6 + add hH, T5 + mov T1, hE +vpxor yT1,yT1,yT2 + ROTATE_T4_T5 + ROTATE_H + + %assign %%nr %%nr+1 + add hH, qword [rsi+(%%nr/2)*sizeof(ymmword)+(%%nr & 1)*sizeof(qword)] + and T1, hE +vpsllq yT3,W7, 3 + rorx T2, hE, 41 + rorx T4, hE, 18 +vpaddq W0,W0,yT1 + add hA, T3 + add hH, T1 +vpsrlq yT2,W7, 19 + andn T1, hE, hG + xor T2, T4 +vpxor yT4,yT4,yT3 + rorx T3, hE, 14 + add hH, T1 +vpsllq yT3,yT3, 42 + xor T2, T3 + mov T4, hA +vpxor yT4,yT4,yT2 + rorx T1, hA, 39 + add hH, T2 +vpsrlq yT2,yT2,42 + xor T4, hB + rorx T3, hA, 34 +vpxor yT4,yT4,yT3 + rorx T2, hA, 28 + add hD, hH +vpxor yT4,yT4,yT2 + and T5, T4 + xor T3, T1 +vpaddq W0,W0,yT4 + xor T5, hB + xor T3, T2 +vpaddq yT3,W0,YMMWORD [rbp+(%%nr/2)*sizeof(ymmword)] + add hH, T5 + mov T1, hE +vmovdqa YMMWORD [rsi+(%%W_AHEAD/2)*sizeof(ymmword)+(%%nr/2)*sizeof(ymmword)],yT3 + ROTATE_T4_T5 + ROTATE_H + ROTATE_W +%endmacro + +;; +;; update hash +;; +%macro UPDATE_HASH 2.nolist + %xdefine %%hashMem %1 + %xdefine %%hash %2 + + add %%hash, %%hashMem + mov %%hashMem, %%hash +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +segment .text align=IPP_ALIGN_FACTOR + + +align IPP_ALIGN_FACTOR + +SHA512_YMM_K dq 0428a2f98d728ae22h, 07137449123ef65cdh, 0428a2f98d728ae22h, 07137449123ef65cdh + dq 0b5c0fbcfec4d3b2fh, 0e9b5dba58189dbbch, 0b5c0fbcfec4d3b2fh, 0e9b5dba58189dbbch + dq 03956c25bf348b538h, 059f111f1b605d019h, 03956c25bf348b538h, 059f111f1b605d019h + dq 0923f82a4af194f9bh, 0ab1c5ed5da6d8118h, 0923f82a4af194f9bh, 0ab1c5ed5da6d8118h + dq 0d807aa98a3030242h, 012835b0145706fbeh, 0d807aa98a3030242h, 012835b0145706fbeh + dq 0243185be4ee4b28ch, 0550c7dc3d5ffb4e2h, 0243185be4ee4b28ch, 0550c7dc3d5ffb4e2h + dq 072be5d74f27b896fh, 080deb1fe3b1696b1h, 072be5d74f27b896fh, 080deb1fe3b1696b1h + dq 09bdc06a725c71235h, 0c19bf174cf692694h, 09bdc06a725c71235h, 0c19bf174cf692694h + dq 0e49b69c19ef14ad2h, 0efbe4786384f25e3h, 0e49b69c19ef14ad2h, 0efbe4786384f25e3h + dq 00fc19dc68b8cd5b5h, 0240ca1cc77ac9c65h, 00fc19dc68b8cd5b5h, 0240ca1cc77ac9c65h + dq 02de92c6f592b0275h, 04a7484aa6ea6e483h, 02de92c6f592b0275h, 04a7484aa6ea6e483h + dq 05cb0a9dcbd41fbd4h, 076f988da831153b5h, 05cb0a9dcbd41fbd4h, 076f988da831153b5h + dq 0983e5152ee66dfabh, 0a831c66d2db43210h, 0983e5152ee66dfabh, 0a831c66d2db43210h + dq 0b00327c898fb213fh, 0bf597fc7beef0ee4h, 0b00327c898fb213fh, 0bf597fc7beef0ee4h + dq 0c6e00bf33da88fc2h, 0d5a79147930aa725h, 0c6e00bf33da88fc2h, 0d5a79147930aa725h + dq 006ca6351e003826fh, 0142929670a0e6e70h, 006ca6351e003826fh, 0142929670a0e6e70h + dq 027b70a8546d22ffch, 02e1b21385c26c926h, 027b70a8546d22ffch, 02e1b21385c26c926h + dq 04d2c6dfc5ac42aedh, 053380d139d95b3dfh, 04d2c6dfc5ac42aedh, 053380d139d95b3dfh + dq 0650a73548baf63deh, 0766a0abb3c77b2a8h, 0650a73548baf63deh, 0766a0abb3c77b2a8h + dq 081c2c92e47edaee6h, 092722c851482353bh, 081c2c92e47edaee6h, 092722c851482353bh + dq 0a2bfe8a14cf10364h, 0a81a664bbc423001h, 0a2bfe8a14cf10364h, 0a81a664bbc423001h + dq 0c24b8b70d0f89791h, 0c76c51a30654be30h, 0c24b8b70d0f89791h, 0c76c51a30654be30h + dq 0d192e819d6ef5218h, 0d69906245565a910h, 0d192e819d6ef5218h, 0d69906245565a910h + dq 0f40e35855771202ah, 0106aa07032bbd1b8h, 0f40e35855771202ah, 0106aa07032bbd1b8h + dq 019a4c116b8d2d0c8h, 01e376c085141ab53h, 019a4c116b8d2d0c8h, 01e376c085141ab53h + dq 02748774cdf8eeb99h, 034b0bcb5e19b48a8h, 02748774cdf8eeb99h, 034b0bcb5e19b48a8h + dq 0391c0cb3c5c95a63h, 04ed8aa4ae3418acbh, 0391c0cb3c5c95a63h, 04ed8aa4ae3418acbh + dq 05b9cca4f7763e373h, 0682e6ff3d6b2b8a3h, 05b9cca4f7763e373h, 0682e6ff3d6b2b8a3h + dq 0748f82ee5defb2fch, 078a5636f43172f60h, 0748f82ee5defb2fch, 078a5636f43172f60h + dq 084c87814a1f0ab72h, 08cc702081a6439ech, 084c87814a1f0ab72h, 08cc702081a6439ech + dq 090befffa23631e28h, 0a4506cebde82bde9h, 090befffa23631e28h, 0a4506cebde82bde9h + dq 0bef9a3f7b2c67915h, 0c67178f2e372532bh, 0bef9a3f7b2c67915h, 0c67178f2e372532bh + dq 0ca273eceea26619ch, 0d186b8c721c0c207h, 0ca273eceea26619ch, 0d186b8c721c0c207h + dq 0eada7dd6cde0eb1eh, 0f57d4f7fee6ed178h, 0eada7dd6cde0eb1eh, 0f57d4f7fee6ed178h + dq 006f067aa72176fbah, 00a637dc5a2c898a6h, 006f067aa72176fbah, 00a637dc5a2c898a6h + dq 0113f9804bef90daeh, 01b710b35131c471bh, 0113f9804bef90daeh, 01b710b35131c471bh + dq 028db77f523047d84h, 032caab7b40c72493h, 028db77f523047d84h, 032caab7b40c72493h + dq 03c9ebe0a15c9bebch, 0431d67c49c100d4ch, 03c9ebe0a15c9bebch, 0431d67c49c100d4ch + dq 04cc5d4becb3e42b6h, 0597f299cfc657e2ah, 04cc5d4becb3e42b6h, 0597f299cfc657e2ah + dq 05fcb6fab3ad6faech, 06c44198c4a475817h, 05fcb6fab3ad6faech, 06c44198c4a475817h + +SHA512_YMM_BF dq 00001020304050607h, 008090a0b0c0d0e0fh, 00001020304050607h, 008090a0b0c0d0e0fh + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; UpdateSHA512(Ipp64u digest[], Ipp8u dataBlock[], int datalen, Ipp64u K_512[]) +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +align IPP_ALIGN_FACTOR +IPPASM UpdateSHA512,PUBLIC +%assign LOCAL_FRAME (sizeof(qword)*4 + sizeof(qword)*80*2) + USES_GPR rbx,rsi,rdi,rbp,rbx,r12,r13,r14,r15 + USES_XMM_AVX xmm6,xmm7,xmm8,xmm9,xmm10,xmm11,xmm12 + COMP_ABI 4 +;; +;; rdi = pointer to the updated hash +;; rsi = pointer to the data block +;; rdx = data block length +;; rcx = pointer to the SHA_512 constant (ignored) +;; + +%xdefine MBS_SHA512 (128) + +;; +;; stack structure: +;; +%assign _block 0 ;; block address +%assign _hash _block+sizeof(qword) ;; hash address +%assign _len _hash+sizeof(qword) ;; rest of processed data +%assign _frame _len+sizeof(qword) ;; rsp before alignment +%assign _dataW _frame+sizeof(qword) ;; W[t] values + + mov r15, rsp ; store orifinal rsp + and rsp, -IPP_ALIGN_FACTOR ; 32-byte aligned stack + + movsxd r14, edx ; input length in bytes + + mov qword [rsp+_hash], rdi ; store hash address + mov qword [rsp+_len], r14 ; store length + mov qword [rsp+_frame], r15 ; store rsp + + vmovdqa YMM_SHUFB_BSWAP, ymmword [rel SHA512_YMM_BF] ; load byte shuffler + + mov hA, qword [rdi] ; load initial hash value + mov hB, qword [rdi+1*sizeof(qword)] + mov hC, qword [rdi+2*sizeof(qword)] + mov hD, qword [rdi+3*sizeof(qword)] + mov hE, qword [rdi+4*sizeof(qword)] + mov hF, qword [rdi+5*sizeof(qword)] + mov hG, qword [rdi+6*sizeof(qword)] + mov hH, qword [rdi+7*sizeof(qword)] + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data 2 block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +align IPP_ALIGN_FACTOR +.sha512_block2_loop: + lea r12, [rsi+MBS_SHA512] ; next block + + cmp r14, MBS_SHA512 ; %if single block processed + cmovbe r12, rsi ; use the same data block address + + lea rbp, [rel SHA512_YMM_K] ; to SHA512 consts + + vmovdqu W0L, xmmword [rsi] ; load data block + vmovdqu W1L, xmmword [rsi+1*sizeof(xmmword)] + vmovdqu W2L, xmmword [rsi+2*sizeof(xmmword)] + vmovdqu W3L, xmmword [rsi+3*sizeof(xmmword)] + vmovdqu W4L, xmmword [rsi+4*sizeof(xmmword)] + vmovdqu W5L, xmmword [rsi+5*sizeof(xmmword)] + vmovdqu W6L, xmmword [rsi+6*sizeof(xmmword)] + vmovdqu W7L, xmmword [rsi+7*sizeof(xmmword)] + + vinserti128 W0, W0, xmmword [r12], 1 ; merge next data block + vinserti128 W1, W1, xmmword [r12+1*sizeof(xmmword)], 1 + vinserti128 W2, W2, xmmword [r12+2*sizeof(xmmword)], 1 + vinserti128 W3, W3, xmmword [r12+3*sizeof(xmmword)], 1 + vinserti128 W4, W4, xmmword [r12+4*sizeof(xmmword)], 1 + vinserti128 W5, W5, xmmword [r12+5*sizeof(xmmword)], 1 + vinserti128 W6, W6, xmmword [r12+6*sizeof(xmmword)], 1 + vinserti128 W7, W7, xmmword [r12+7*sizeof(xmmword)], 1 + + vpshufb W0, W0, YMM_SHUFB_BSWAP + vpshufb W1, W1, YMM_SHUFB_BSWAP + vpshufb W2, W2, YMM_SHUFB_BSWAP + vpshufb W3, W3, YMM_SHUFB_BSWAP + vpshufb W4, W4, YMM_SHUFB_BSWAP + vpshufb W5, W5, YMM_SHUFB_BSWAP + vpshufb W6, W6, YMM_SHUFB_BSWAP + vpshufb W7, W7, YMM_SHUFB_BSWAP + + vpaddq yT1, W0, ymmword [rbp] + vmovdqa ymmword [rsp+_dataW], yT1 + vpaddq yT2, W1, ymmword [rbp+1*sizeof(ymmword)] + vmovdqa ymmword [rsp+_dataW+1*sizeof(ymmword)], yT2 + vpaddq yT3, W2, ymmword [rbp+2*sizeof(ymmword)] + vmovdqa ymmword [rsp+_dataW+2*sizeof(ymmword)], yT3 + vpaddq yT4, W3, ymmword [rbp+3*sizeof(ymmword)] + vmovdqa ymmword [rsp+_dataW+3*sizeof(ymmword)], yT4 + + vpaddq yT1, W4, ymmword [rbp+4*sizeof(ymmword)] + vmovdqa ymmword [rsp+_dataW+4*sizeof(ymmword)], yT1 + vpaddq yT2, W5, ymmword [rbp+5*sizeof(ymmword)] + vmovdqa ymmword [rsp+_dataW+5*sizeof(ymmword)], yT2 + vpaddq yT3, W6, ymmword [rbp+6*sizeof(ymmword)] + vmovdqa ymmword [rsp+_dataW+6*sizeof(ymmword)], yT3 + vpaddq yT4, W7, ymmword [rbp+7*sizeof(ymmword)] + vmovdqa ymmword [rsp+_dataW+7*sizeof(ymmword)], yT4 + + add rbp, 8*sizeof(ymmword) + + mov T5, hB ; T5 = b^c + xor T3, T3 ; T3 = 0 + mov T1, hF ; T1 = f + xor T5, hC + + mov qword [rsp+_block], rsi ; store block addres + lea rsi, [rsp+_dataW] ; use rsi as stack pointer + +align IPP_ALIGN_FACTOR +.block1_shed_proc: + SHA512_2ROUND_SHED 0 + SHA512_2ROUND_SHED 2 + SHA512_2ROUND_SHED 4 + SHA512_2ROUND_SHED 6 + SHA512_2ROUND_SHED 8 + SHA512_2ROUND_SHED 10 + SHA512_2ROUND_SHED 12 + SHA512_2ROUND_SHED 14 + + add rsi, 8*sizeof(ymmword) + add rbp, 8*sizeof(ymmword) + + ;; and repeat + cmp dword [rbp-sizeof(qword)],04a475817h + jne .block1_shed_proc + + ;; the rest 16 rounds + SHA512_ROUND 0, hA,hB,hC,hD,hE,hF,hG,hH + SHA512_ROUND 1, hH,hA,hB,hC,hD,hE,hF,hG + SHA512_ROUND 2, hG,hH,hA,hB,hC,hD,hE,hF + SHA512_ROUND 3, hF,hG,hH,hA,hB,hC,hD,hE + SHA512_ROUND 4, hE,hF,hG,hH,hA,hB,hC,hD + SHA512_ROUND 5, hD,hE,hF,hG,hH,hA,hB,hC + SHA512_ROUND 6, hC,hD,hE,hF,hG,hH,hA,hB + SHA512_ROUND 7, hB,hC,hD,hE,hF,hG,hH,hA + SHA512_ROUND 8, hA,hB,hC,hD,hE,hF,hG,hH + SHA512_ROUND 9, hH,hA,hB,hC,hD,hE,hF,hG + SHA512_ROUND 10, hG,hH,hA,hB,hC,hD,hE,hF + SHA512_ROUND 11, hF,hG,hH,hA,hB,hC,hD,hE + SHA512_ROUND 12, hE,hF,hG,hH,hA,hB,hC,hD + SHA512_ROUND 13, hD,hE,hF,hG,hH,hA,hB,hC + SHA512_ROUND 14, hC,hD,hE,hF,hG,hH,hA,hB + SHA512_ROUND 15, hB,hC,hD,hE,hF,hG,hH,hA + add hA, T3 + + sub rsi, ((80-16)/2)*sizeof(ymmword) ; restore stack to W + + mov rdi, qword [rsi+_hash-_dataW] ; restore hash pointer + mov r14, qword [rsi+_len-_dataW] ; restore data length + + ;; update hash values by 1-st data block + UPDATE_HASH qword [rdi], hA + UPDATE_HASH qword [rdi+1*sizeof(qword)], hB + UPDATE_HASH qword [rdi+2*sizeof(qword)], hC + UPDATE_HASH qword [rdi+3*sizeof(qword)], hD + UPDATE_HASH qword [rdi+4*sizeof(qword)], hE + UPDATE_HASH qword [rdi+5*sizeof(qword)], hF + UPDATE_HASH qword [rdi+6*sizeof(qword)], hG + UPDATE_HASH qword [rdi+7*sizeof(qword)], hH + + cmp r14, MBS_SHA512*2 + jl .done + + ;; do 80 rounds for the next block + add rsi, 2*sizeof(qword) ; restore stack to next block W + lea rbp, [rsi+40*sizeof(ymmword)] ; use rbp for loop limiter + + mov T5, hB ; T5 = b^c + xor T3, T3 ; T3 = 0 + mov T1, hF ; T1 = f + xor T5, hC + +align IPP_ALIGN_FACTOR +.block2_proc: + SHA512_ROUND 0, hA,hB,hC,hD,hE,hF,hG,hH + SHA512_ROUND 1, hH,hA,hB,hC,hD,hE,hF,hG + SHA512_ROUND 2, hG,hH,hA,hB,hC,hD,hE,hF + SHA512_ROUND 3, hF,hG,hH,hA,hB,hC,hD,hE + SHA512_ROUND 4, hE,hF,hG,hH,hA,hB,hC,hD + SHA512_ROUND 5, hD,hE,hF,hG,hH,hA,hB,hC + SHA512_ROUND 6, hC,hD,hE,hF,hG,hH,hA,hB + SHA512_ROUND 7, hB,hC,hD,hE,hF,hG,hH,hA + add rsi, 4*sizeof(ymmword) + cmp rsi, rbp + jb .block2_proc + add hA, T3 + + mov rdi, qword [rsp+_hash] ; restore hash pointer + mov r14, qword [rsp+_len] ; restore data length + + ;; update hash values by 1-st data block + UPDATE_HASH qword [rdi], hA + UPDATE_HASH qword [rdi+1*sizeof(qword)], hB + UPDATE_HASH qword [rdi+2*sizeof(qword)], hC + UPDATE_HASH qword [rdi+3*sizeof(qword)], hD + UPDATE_HASH qword [rdi+4*sizeof(qword)], hE + UPDATE_HASH qword [rdi+5*sizeof(qword)], hF + UPDATE_HASH qword [rdi+6*sizeof(qword)], hG + UPDATE_HASH qword [rdi+7*sizeof(qword)], hH + + mov rsi, qword [rsp+_block] ; restore block addres + add rsi, MBS_SHA512*2 ; move data pointer + sub r14, MBS_SHA512*2 ; update data length + mov qword [rsp+_len], r14 + jg .sha512_block2_loop + +.done: + mov rsp, qword [rsp+_frame] + REST_XMM_AVX + REST_GPR + ret +ENDFUNC UpdateSHA512 + +%endif ;; _IPP32E_L9 and above +%endif ;; _FEATURE_OFF_ / _FEATURE_TICKTOCK_ +%endif ;; _ENABLE_ALG_SHA512_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha512m7as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha512m7as.asm new file mode 100644 index 0000000..c38c38a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsha512m7as.asm @@ -0,0 +1,490 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SHA512 +; +; Content: +; UpdateSHA512 +; +; +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SHA512_) +%if (_IPP32E >= _IPP32E_M7) && (_IPP32E < _IPP32E_E9 ) + + +;; +;; ENDIANNESS +;; +%macro ENDIANNESS 2.nolist + %xdefine %%dst %1 + %xdefine %%src %2 + + %ifnidn %%dst,%%src + mov %%dst,%%src + %endif + bswap %%dst +%endmacro + +%macro ROT_R 2.nolist + %xdefine %%r %1 + %xdefine %%nbits %2 + + %if _IPP32E >= _IPP32E_L9 + rorx %%r,%%r,%%nbits + ;; %elif _IPP32E >= _IPP32E_Y8 + ;; shrd %%r,%%r,%%nbits + %else + ror %%r,%%nbits + %endif +%endmacro + +;; +;; SHA512 step +;; +;; Ipp64u T1 = H + SUM1(E) + CH(E,F,G) + K_SHA512[t] + W[t]; +;; Ipp64u T2 = SUM0(A) + MAJ(A,B,C); +;; D+= T1; +;; H = T1 + T2; +;; +;; where +;; SUM1(x) = ROR64(x,14) ^ ROR64(x,18) ^ ROR64(x,41) +;; SUM0(x) = ROR64(x,28) ^ ROR64(x,34) ^ ROR64(x,39) +;; +;; CHJ(x,y,z) = (x & y) ^ (~x & z) +;; MAJ(x,y,z) = (x & y) ^ (x & z) ^ (y & z) = (x&y)^((x^y)&z) +;; +%macro SHA512_STEP_2 13.nolist + %xdefine %%regA %1 + %xdefine %%regB %2 + %xdefine %%regC %3 + %xdefine %%regD %4 + %xdefine %%regE %5 + %xdefine %%regF %6 + %xdefine %%regG %7 + %xdefine %%regH %8 + %xdefine %%regT %9 + %xdefine %%regF1 %10 + %xdefine %%regF2 %11 + %xdefine %%memW %12 + %xdefine %%memCNT %13 + + ;; update H (start) + add %%regH,[%%memCNT] ;; H += W[t]+K_SHA512[t] + add %%regH,[%%memW] + + ;; compute T = SUM1(E) + CH(E,F,G) + mov %%regF1,%%regE ;; SUM1() = E + mov %%regF2,%%regE ;; CH() = E + ROT_R %%regF1,14 ;; ROR(E,14) + mov %%regT, %%regE + push %%regE + not %%regF2 ;; ~E + ROT_R %%regE, 18 ;; ROR(E,18) + and %%regT, %%regF ;; E&F + and %%regF2,%%regG ;;~E&G + xor %%regF1,%%regE ;; ROR(E,14)^ROR(E,18) + ROT_R %%regE, 23 ;; ROR(E,41) + xor %%regF2,%%regT ;; CH() = (E&F)&(~E&G) + xor %%regF1,%%regE ;; SUM1() = ROR(E,14)^ROR(E,18)^ROR(E,41) + pop %%regE ;; repair E + lea %%regT, [%%regF1+%%regF2] + + ;; update H (continue) + add %%regH, %%regT + + ;; update D + add %%regD, %%regH + + ;; compute T = SUM0(A) + MAJ(A,B,C) + mov %%regF1,%%regA ;; SUM0() = A + mov %%regF2,%%regA ;; MAJ() = A + ROT_R %%regF1,28 ;; ROR(A,28) + mov %%regT, %%regA + push %%regA + xor %%regF2,%%regB ;; A^B + ROT_R %%regA, 34 ;; ROR(A,34) + and %%regT, %%regB ;; A&B + and %%regF2,%%regC ;; (A^B)&C + xor %%regF1,%%regA ;; ROR(A,2)^ROR(A,13) + ROT_R %%regA, 5 ;; ROR(A,39) + xor %%regF2,%%regT ;; MAJ() = (A^B)^((A^B)&C) + xor %%regF1,%%regA ;; SUM0() = ROR(A,28)^ROR(A,34)^ROR(A,39) + pop %%regA ;; repair A + lea %%regT, [%%regF1+%%regF2] + + ;; update H (finish) + add %%regH, %%regT +%endmacro + +;; +;; update W[] +;; +;; W[j] = SIG1(W[j- 2]) + W[j- 7] +;; +SIG0(W[j-15]) + W[j-16] +;; +;; SIG0(x) = ROR(x,1) ^ROR(x,8) ^LSR(x,7) +;; SIG1(x) = ROR(x,19)^ROR(x,61) ^LSR(x,6) +;; +%macro UPDATE_2 5.nolist + %xdefine %%nr %1 + %xdefine %%sig0 %2 + %xdefine %%sig1 %3 + %xdefine %%W15 %4 + %xdefine %%W2 %5 + + mov %%sig0, [rsp+((%%nr-15) & 0Fh)*8] ;; W[j-15] + mov %%sig1, [rsp+((%%nr-2) & 0Fh)*8] ;; W[j-2] + shr %%sig0, 7 ;; LSR(W[j-15], 7) + shr %%sig1, 6 ;; LSR(W[j-2], 6) + + mov %%W15, [rsp+((%%nr-15) & 0Fh)*8] ;; W[j-15] + mov %%W2, [rsp+((%%nr-2) & 0Fh)*8] ;; W[j-2] + ROT_R %%W15, 1 ;; ROR(W[j-15], 1) + ROT_R %%W2, 19 ;; ROR(W[j-2], 19) + xor %%sig0, %%W15 ;; SIG0 ^= ROR(W[j-15], 1) + xor %%sig1, %%W2 ;; SIG1 ^= ROR(W[j-2], 19) + + ROT_R %%W15, 7 ;; ROR(W[j-15], 8) + ROT_R %%W2, 42 ;; ROR(W[j-2], 61) + xor %%sig0, %%W15 ;; SIG0 ^= ROR(W[j-15], 8) + xor %%sig1, %%W2 ;; SIG1 ^= ROR(W[j-2], 61) + + add %%sig0, [rsp+((%%nr-16) & 0Fh)*8] ;; SIG0 += W[j-16] + add %%sig1, [rsp+((%%nr-7) & 0Fh)*8] ;; SIG1 += W[j-7] + add %%sig0, %%sig1 + mov [rsp+((%%nr-16) & 0Fh)*8], %%sig0 +%endmacro + +segment .text align=IPP_ALIGN_FACTOR + + + +;****************************************************************************************** +;* Purpose: Update internal digest according to message block +;* +;* void UpdateSHA512(DigestSHA512 digest, const Ipp32u* mblk, int mlen, const void* pParam) +;* +;****************************************************************************************** + +;; +;; Lib = M7 +;; +;; Caller = ippsSHA512Update +;; Caller = ippsSHA512Final +;; Caller = ippsSHA512MessageDigest +;; +;; Caller = ippsSHA384Update +;; Caller = ippsSHA384Final +;; Caller = ippsSHA384MessageDigest +;; +;; Caller = ippsHMACSHA512Update +;; Caller = ippsHMACSHA512Final +;; Caller = ippsHMACSHA512MessageDigest +;; +;; Caller = ippsHMACSHA384Update +;; Caller = ippsHMACSHA384Final +;; Caller = ippsHMACSHA384MessageDigest +;; + +%xdefine KK_SHA512 rbp + +%if _IPP32E >= _IPP32E_U8 +align IPP_ALIGN_FACTOR +pByteSwp DB 7,6,5,4,3,2,1,0, 15,14,13,12,11,10,9,8 +%endif + +align IPP_ALIGN_FACTOR +IPPASM UpdateSHA512,PUBLIC +%assign LOCAL_FRAME (16*sizeof(qword)+sizeof(qword)) + USES_GPR rbx,rsi,rdi,r12,r13,r14,r15,rbp + USES_XMM + COMP_ABI 4 + +;; rdi = hash +;; rsi = data buffer +;; rdx = buffer length +;; rcx = address of SHA512 constants + +%xdefine MBS_SHA512 (128) + + movsxd rdx, edx + mov qword [rsp+16*sizeof(qword)], rdx ; save length of buffer + + mov KK_SHA512, rcx + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process next data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +align IPP_ALIGN_FACTOR +.sha512_block_loop: + +;; +;; initialize the first 16 qwords in the array W (remember about endian) +;; +%if (_IPP32E >= _IPP32E_U8) + movdqu xmm4, oword [rel pByteSwp] + + movdqu xmm0, oword [rsi+0*16] + pshufb xmm0, xmm4 + movdqa oword [rsp+0*16], xmm0 + movdqu xmm1, oword [rsi+1*16] + pshufb xmm1, xmm4 + movdqa oword [rsp+1*16], xmm1 + movdqu xmm2, oword [rsi+2*16] + pshufb xmm2, xmm4 + movdqa oword [rsp+2*16], xmm2 + movdqu xmm3, oword [rsi+3*16] + pshufb xmm3, xmm4 + movdqa oword [rsp+3*16], xmm3 + + movdqu xmm0, oword [rsi+4*16] + pshufb xmm0, xmm4 + movdqa oword [rsp+4*16], xmm0 + movdqu xmm1, oword [rsi+5*16] + pshufb xmm1, xmm4 + movdqa oword [rsp+5*16], xmm1 + movdqu xmm2, oword [rsi+6*16] + pshufb xmm2, xmm4 + movdqa oword [rsp+6*16], xmm2 + movdqu xmm3, oword [rsi+7*16] + pshufb xmm3, xmm4 + movdqa oword [rsp+7*16], xmm3 +%else + xor rcx,rcx + +align IPP_ALIGN_FACTOR +.loop1: + mov r8,[rsi+rcx*8+0*8] + ENDIANNESS r8,r8 + mov [rsp+rcx*8+0*8],r8 + + mov r9,[rsi+rcx*8+1*8] + ENDIANNESS r9,r9 + mov [rsp+rcx*8+1*8],r9 + + add rcx,2 + cmp rcx,16 + jl .loop1 +%endif + +;; +;; init A,B,C,D,E,F,G,H by the internal digest +;; +;; +;; init A, B, C, D, E, F, G, H by the internal digest +;; + mov r8, [rdi+0*8] ; r8 = digest[0] (A) + mov r9, [rdi+1*8] ; r9 = digest[1] (B) + mov r10,[rdi+2*8] ; r10= digest[2] (C) + mov r11,[rdi+3*8] ; r11= digest[3] (D) + mov r12,[rdi+4*8] ; r12= digest[4] (E) + mov r13,[rdi+5*8] ; r13= digest[5] (F) + mov r14,[rdi+6*8] ; r14= digest[6] (G) + mov r15,[rdi+7*8] ; r15= digest[7] (H) + +;; +;; perform 0-79 steps +;; +;; A, B, C, D, E, F, G, H W[], K[] +;; ------------------------------------------------------------------------- + SHA512_STEP_2 r8, r9, r10,r11,r12,r13,r14,r15, rbx,rcx,rdx, {rsp+ 0*8}, {KK_SHA512+ 0*8} + UPDATE_2 16, rax,rbx, rcx,rdx + SHA512_STEP_2 r15,r8, r9, r10,r11,r12,r13,r14, rbx,rcx,rdx, {rsp+ 1*8}, {KK_SHA512+ 1*8} + UPDATE_2 17, rax,rbx, rcx,rdx + SHA512_STEP_2 r14,r15,r8, r9, r10,r11,r12,r13, rbx,rcx,rdx, {rsp+ 2*8}, {KK_SHA512+ 2*8} + UPDATE_2 18, rax,rbx, rcx,rdx + SHA512_STEP_2 r13,r14,r15,r8, r9, r10,r11,r12, rbx,rcx,rdx, {rsp+ 3*8}, {KK_SHA512+ 3*8} + UPDATE_2 19, rax,rbx, rcx,rdx + SHA512_STEP_2 r12,r13,r14,r15,r8, r9, r10,r11, rbx,rcx,rdx, {rsp+ 4*8}, {KK_SHA512+ 4*8} + UPDATE_2 20, rax,rbx, rcx,rdx + SHA512_STEP_2 r11,r12,r13,r14,r15,r8, r9, r10, rbx,rcx,rdx, {rsp+ 5*8}, {KK_SHA512+ 5*8} + UPDATE_2 21, rax,rbx, rcx,rdx + SHA512_STEP_2 r10,r11,r12,r13,r14,r15,r8, r9, rbx,rcx,rdx, {rsp+ 6*8}, {KK_SHA512+ 6*8} + UPDATE_2 22, rax,rbx, rcx,rdx + SHA512_STEP_2 r9, r10,r11,r12,r13,r14,r15,r8, rbx,rcx,rdx, {rsp+ 7*8}, {KK_SHA512+ 7*8} + UPDATE_2 23, rax,rbx, rcx,rdx + + SHA512_STEP_2 r8, r9, r10,r11,r12,r13,r14,r15, rbx,rcx,rdx, {rsp+ 8*8}, {KK_SHA512+ 8*8} + UPDATE_2 24, rax,rbx, rcx,rdx + SHA512_STEP_2 r15,r8, r9, r10,r11,r12,r13,r14, rbx,rcx,rdx, {rsp+ 9*8}, {KK_SHA512+ 9*8} + UPDATE_2 25, rax,rbx, rcx,rdx + SHA512_STEP_2 r14,r15,r8, r9, r10,r11,r12,r13, rbx,rcx,rdx, {rsp+10*8}, {KK_SHA512+10*8} + UPDATE_2 26, rax,rbx, rcx,rdx + SHA512_STEP_2 r13,r14,r15,r8, r9, r10,r11,r12, rbx,rcx,rdx, {rsp+11*8}, {KK_SHA512+11*8} + UPDATE_2 27, rax,rbx, rcx,rdx + SHA512_STEP_2 r12,r13,r14,r15,r8, r9, r10,r11, rbx,rcx,rdx, {rsp+12*8}, {KK_SHA512+12*8} + UPDATE_2 28, rax,rbx, rcx,rdx + SHA512_STEP_2 r11,r12,r13,r14,r15,r8, r9, r10, rbx,rcx,rdx, {rsp+13*8}, {KK_SHA512+13*8} + UPDATE_2 29, rax,rbx, rcx,rdx + SHA512_STEP_2 r10,r11,r12,r13,r14,r15,r8, r9, rbx,rcx,rdx, {rsp+14*8}, {KK_SHA512+14*8} + UPDATE_2 30, rax,rbx, rcx,rdx + SHA512_STEP_2 r9, r10,r11,r12,r13,r14,r15,r8, rbx,rcx,rdx, {rsp+15*8}, {KK_SHA512+15*8} + UPDATE_2 31, rax,rbx, rcx,rdx + + SHA512_STEP_2 r8, r9, r10,r11,r12,r13,r14,r15, rbx,rcx,rdx, {rsp+ 0*8}, {KK_SHA512+16*8} + UPDATE_2 32, rax,rbx, rcx,rdx + SHA512_STEP_2 r15,r8, r9, r10,r11,r12,r13,r14, rbx,rcx,rdx, {rsp+ 1*8}, {KK_SHA512+17*8} + UPDATE_2 33, rax,rbx, rcx,rdx + SHA512_STEP_2 r14,r15,r8, r9, r10,r11,r12,r13, rbx,rcx,rdx, {rsp+ 2*8}, {KK_SHA512+18*8} + UPDATE_2 34, rax,rbx, rcx,rdx + SHA512_STEP_2 r13,r14,r15,r8, r9, r10,r11,r12, rbx,rcx,rdx, {rsp+ 3*8}, {KK_SHA512+19*8} + UPDATE_2 35, rax,rbx, rcx,rdx + SHA512_STEP_2 r12,r13,r14,r15,r8, r9, r10,r11, rbx,rcx,rdx, {rsp+ 4*8}, {KK_SHA512+20*8} + UPDATE_2 36, rax,rbx, rcx,rdx + SHA512_STEP_2 r11,r12,r13,r14,r15,r8, r9, r10, rbx,rcx,rdx, {rsp+ 5*8}, {KK_SHA512+21*8} + UPDATE_2 37, rax,rbx, rcx,rdx + SHA512_STEP_2 r10,r11,r12,r13,r14,r15,r8, r9, rbx,rcx,rdx, {rsp+ 6*8}, {KK_SHA512+22*8} + UPDATE_2 38, rax,rbx, rcx,rdx + SHA512_STEP_2 r9, r10,r11,r12,r13,r14,r15,r8, rbx,rcx,rdx, {rsp+ 7*8}, {KK_SHA512+23*8} + UPDATE_2 39, rax,rbx, rcx,rdx + + SHA512_STEP_2 r8, r9, r10,r11,r12,r13,r14,r15, rbx,rcx,rdx, {rsp+ 8*8}, {KK_SHA512+24*8} + UPDATE_2 40, rax,rbx, rcx,rdx + SHA512_STEP_2 r15,r8, r9, r10,r11,r12,r13,r14, rbx,rcx,rdx, {rsp+ 9*8}, {KK_SHA512+25*8} + UPDATE_2 41, rax,rbx, rcx,rdx + SHA512_STEP_2 r14,r15,r8, r9, r10,r11,r12,r13, rbx,rcx,rdx, {rsp+10*8}, {KK_SHA512+26*8} + UPDATE_2 42, rax,rbx, rcx,rdx + SHA512_STEP_2 r13,r14,r15,r8, r9, r10,r11,r12, rbx,rcx,rdx, {rsp+11*8}, {KK_SHA512+27*8} + UPDATE_2 43, rax,rbx, rcx,rdx + SHA512_STEP_2 r12,r13,r14,r15,r8, r9, r10,r11, rbx,rcx,rdx, {rsp+12*8}, {KK_SHA512+28*8} + UPDATE_2 44, rax,rbx, rcx,rdx + SHA512_STEP_2 r11,r12,r13,r14,r15,r8, r9, r10, rbx,rcx,rdx, {rsp+13*8}, {KK_SHA512+29*8} + UPDATE_2 45, rax,rbx, rcx,rdx + SHA512_STEP_2 r10,r11,r12,r13,r14,r15,r8, r9, rbx,rcx,rdx, {rsp+14*8}, {KK_SHA512+30*8} + UPDATE_2 46, rax,rbx, rcx,rdx + SHA512_STEP_2 r9, r10,r11,r12,r13,r14,r15,r8, rbx,rcx,rdx, {rsp+15*8}, {KK_SHA512+31*8} + UPDATE_2 47, rax,rbx, rcx,rdx + + SHA512_STEP_2 r8, r9, r10,r11,r12,r13,r14,r15, rbx,rcx,rdx, {rsp+ 0*8}, {KK_SHA512+32*8} + UPDATE_2 48, rax,rbx, rcx,rdx + SHA512_STEP_2 r15,r8, r9, r10,r11,r12,r13,r14, rbx,rcx,rdx, {rsp+ 1*8}, {KK_SHA512+33*8} + UPDATE_2 49, rax,rbx, rcx,rdx + SHA512_STEP_2 r14,r15,r8, r9, r10,r11,r12,r13, rbx,rcx,rdx, {rsp+ 2*8}, {KK_SHA512+34*8} + UPDATE_2 50, rax,rbx, rcx,rdx + SHA512_STEP_2 r13,r14,r15,r8, r9, r10,r11,r12, rbx,rcx,rdx, {rsp+ 3*8}, {KK_SHA512+35*8} + UPDATE_2 51, rax,rbx, rcx,rdx + SHA512_STEP_2 r12,r13,r14,r15,r8, r9, r10,r11, rbx,rcx,rdx, {rsp+ 4*8}, {KK_SHA512+36*8} + UPDATE_2 52, rax,rbx, rcx,rdx + SHA512_STEP_2 r11,r12,r13,r14,r15,r8, r9, r10, rbx,rcx,rdx, {rsp+ 5*8}, {KK_SHA512+37*8} + UPDATE_2 53, rax,rbx, rcx,rdx + SHA512_STEP_2 r10,r11,r12,r13,r14,r15,r8, r9, rbx,rcx,rdx, {rsp+ 6*8}, {KK_SHA512+38*8} + UPDATE_2 54, rax,rbx, rcx,rdx + SHA512_STEP_2 r9, r10,r11,r12,r13,r14,r15,r8, rbx,rcx,rdx, {rsp+ 7*8}, {KK_SHA512+39*8} + UPDATE_2 55, rax,rbx, rcx,rdx + + SHA512_STEP_2 r8, r9, r10,r11,r12,r13,r14,r15, rbx,rcx,rdx, {rsp+ 8*8}, {KK_SHA512+40*8} + UPDATE_2 56, rax,rbx, rcx,rdx + SHA512_STEP_2 r15,r8, r9, r10,r11,r12,r13,r14, rbx,rcx,rdx, {rsp+ 9*8}, {KK_SHA512+41*8} + UPDATE_2 57, rax,rbx, rcx,rdx + SHA512_STEP_2 r14,r15,r8, r9, r10,r11,r12,r13, rbx,rcx,rdx, {rsp+10*8}, {KK_SHA512+42*8} + UPDATE_2 58, rax,rbx, rcx,rdx + SHA512_STEP_2 r13,r14,r15,r8, r9, r10,r11,r12, rbx,rcx,rdx, {rsp+11*8}, {KK_SHA512+43*8} + UPDATE_2 59, rax,rbx, rcx,rdx + SHA512_STEP_2 r12,r13,r14,r15,r8, r9, r10,r11, rbx,rcx,rdx, {rsp+12*8}, {KK_SHA512+44*8} + UPDATE_2 60, rax,rbx, rcx,rdx + SHA512_STEP_2 r11,r12,r13,r14,r15,r8, r9, r10, rbx,rcx,rdx, {rsp+13*8}, {KK_SHA512+45*8} + UPDATE_2 61, rax,rbx, rcx,rdx + SHA512_STEP_2 r10,r11,r12,r13,r14,r15,r8, r9, rbx,rcx,rdx, {rsp+14*8}, {KK_SHA512+46*8} + UPDATE_2 62, rax,rbx, rcx,rdx + SHA512_STEP_2 r9, r10,r11,r12,r13,r14,r15,r8, rbx,rcx,rdx, {rsp+15*8}, {KK_SHA512+47*8} + UPDATE_2 63, rax,rbx, rcx,rdx + + SHA512_STEP_2 r8, r9, r10,r11,r12,r13,r14,r15, rbx,rcx,rdx, {rsp+ 0*8}, {KK_SHA512+48*8} + UPDATE_2 64, rax,rbx, rcx,rdx + SHA512_STEP_2 r15,r8, r9, r10,r11,r12,r13,r14, rbx,rcx,rdx, {rsp+ 1*8}, {KK_SHA512+49*8} + UPDATE_2 65, rax,rbx, rcx,rdx + SHA512_STEP_2 r14,r15,r8, r9, r10,r11,r12,r13, rbx,rcx,rdx, {rsp+ 2*8}, {KK_SHA512+50*8} + UPDATE_2 66, rax,rbx, rcx,rdx + SHA512_STEP_2 r13,r14,r15,r8, r9, r10,r11,r12, rbx,rcx,rdx, {rsp+ 3*8}, {KK_SHA512+51*8} + UPDATE_2 67, rax,rbx, rcx,rdx + SHA512_STEP_2 r12,r13,r14,r15,r8, r9, r10,r11, rbx,rcx,rdx, {rsp+ 4*8}, {KK_SHA512+52*8} + UPDATE_2 68, rax,rbx, rcx,rdx + SHA512_STEP_2 r11,r12,r13,r14,r15,r8, r9, r10, rbx,rcx,rdx, {rsp+ 5*8}, {KK_SHA512+53*8} + UPDATE_2 69, rax,rbx, rcx,rdx + SHA512_STEP_2 r10,r11,r12,r13,r14,r15,r8, r9, rbx,rcx,rdx, {rsp+ 6*8}, {KK_SHA512+54*8} + UPDATE_2 70, rax,rbx, rcx,rdx + SHA512_STEP_2 r9, r10,r11,r12,r13,r14,r15,r8, rbx,rcx,rdx, {rsp+ 7*8}, {KK_SHA512+55*8} + UPDATE_2 71, rax,rbx, rcx,rdx + + SHA512_STEP_2 r8, r9, r10,r11,r12,r13,r14,r15, rbx,rcx,rdx, {rsp+ 8*8}, {KK_SHA512+56*8} + UPDATE_2 72, rax,rbx, rcx,rdx + SHA512_STEP_2 r15,r8, r9, r10,r11,r12,r13,r14, rbx,rcx,rdx, {rsp+ 9*8}, {KK_SHA512+57*8} + UPDATE_2 73, rax,rbx, rcx,rdx + SHA512_STEP_2 r14,r15,r8, r9, r10,r11,r12,r13, rbx,rcx,rdx, {rsp+10*8}, {KK_SHA512+58*8} + UPDATE_2 74, rax,rbx, rcx,rdx + SHA512_STEP_2 r13,r14,r15,r8, r9, r10,r11,r12, rbx,rcx,rdx, {rsp+11*8}, {KK_SHA512+59*8} + UPDATE_2 75, rax,rbx, rcx,rdx + SHA512_STEP_2 r12,r13,r14,r15,r8, r9, r10,r11, rbx,rcx,rdx, {rsp+12*8}, {KK_SHA512+60*8} + UPDATE_2 76, rax,rbx, rcx,rdx + SHA512_STEP_2 r11,r12,r13,r14,r15,r8, r9, r10, rbx,rcx,rdx, {rsp+13*8}, {KK_SHA512+61*8} + UPDATE_2 77, rax,rbx, rcx,rdx + SHA512_STEP_2 r10,r11,r12,r13,r14,r15,r8, r9, rbx,rcx,rdx, {rsp+14*8}, {KK_SHA512+62*8} + UPDATE_2 78, rax,rbx, rcx,rdx + SHA512_STEP_2 r9, r10,r11,r12,r13,r14,r15,r8, rbx,rcx,rdx, {rsp+15*8}, {KK_SHA512+63*8} + UPDATE_2 79, rax,rbx, rcx,rdx + + SHA512_STEP_2 r8, r9, r10,r11,r12,r13,r14,r15, rbx,rcx,rdx, {rsp+ 0*8}, {KK_SHA512+64*8} + SHA512_STEP_2 r15,r8, r9, r10,r11,r12,r13,r14, rbx,rcx,rdx, {rsp+ 1*8}, {KK_SHA512+65*8} + SHA512_STEP_2 r14,r15,r8, r9, r10,r11,r12,r13, rbx,rcx,rdx, {rsp+ 2*8}, {KK_SHA512+66*8} + SHA512_STEP_2 r13,r14,r15,r8, r9, r10,r11,r12, rbx,rcx,rdx, {rsp+ 3*8}, {KK_SHA512+67*8} + SHA512_STEP_2 r12,r13,r14,r15,r8, r9, r10,r11, rbx,rcx,rdx, {rsp+ 4*8}, {KK_SHA512+68*8} + SHA512_STEP_2 r11,r12,r13,r14,r15,r8, r9, r10, rbx,rcx,rdx, {rsp+ 5*8}, {KK_SHA512+69*8} + SHA512_STEP_2 r10,r11,r12,r13,r14,r15,r8, r9, rbx,rcx,rdx, {rsp+ 6*8}, {KK_SHA512+70*8} + SHA512_STEP_2 r9, r10,r11,r12,r13,r14,r15,r8, rbx,rcx,rdx, {rsp+ 7*8}, {KK_SHA512+71*8} + + SHA512_STEP_2 r8, r9, r10,r11,r12,r13,r14,r15, rbx,rcx,rdx, {rsp+ 8*8}, {KK_SHA512+72*8} + SHA512_STEP_2 r15,r8, r9, r10,r11,r12,r13,r14, rbx,rcx,rdx, {rsp+ 9*8}, {KK_SHA512+73*8} + SHA512_STEP_2 r14,r15,r8, r9, r10,r11,r12,r13, rbx,rcx,rdx, {rsp+10*8}, {KK_SHA512+74*8} + SHA512_STEP_2 r13,r14,r15,r8, r9, r10,r11,r12, rbx,rcx,rdx, {rsp+11*8}, {KK_SHA512+75*8} + SHA512_STEP_2 r12,r13,r14,r15,r8, r9, r10,r11, rbx,rcx,rdx, {rsp+12*8}, {KK_SHA512+76*8} + SHA512_STEP_2 r11,r12,r13,r14,r15,r8, r9, r10, rbx,rcx,rdx, {rsp+13*8}, {KK_SHA512+77*8} + SHA512_STEP_2 r10,r11,r12,r13,r14,r15,r8, r9, rbx,rcx,rdx, {rsp+14*8}, {KK_SHA512+78*8} + SHA512_STEP_2 r9, r10,r11,r12,r13,r14,r15,r8, rbx,rcx,rdx, {rsp+15*8}, {KK_SHA512+79*8} + +;; +;; update digest +;; + add [rdi+0*8],r8 + add [rdi+1*8],r9 + add [rdi+2*8],r10 + add [rdi+3*8],r11 + add [rdi+4*8],r12 + add [rdi+5*8],r13 + add [rdi+6*8],r14 + add [rdi+7*8],r15 + + add rsi, MBS_SHA512 + sub qword [rsp+16*sizeof(qword)], MBS_SHA512 + jg .sha512_block_loop + + REST_XMM + REST_GPR + ret +ENDFUNC UpdateSHA512 + +%endif ;; (_IPP32E >= _IPP32E_M7) AND (_IPP32E < _IPP32E_E9 ) +%endif ;; _ENABLE_ALG_SHA512_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsm2pfuncs_montas.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsm2pfuncs_montas.asm new file mode 100644 index 0000000..329138a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsm2pfuncs_montas.asm @@ -0,0 +1,1241 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; SM2 specific implementation +; + +%include "asmdefs.inc" +%include "ia_32e.inc" + +%if (_IPP32E >= _IPP32E_M7) + +%assign _xEMULATION_ 1 +%assign _ADCX_ADOX_ 1 + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR + +;; The SM2 polynomial +Lpoly DQ 0FFFFFFFFFFFFFFFFh,0FFFFFFFF00000000h,0FFFFFFFFFFFFFFFFh,0FFFFFFFEFFFFFFFFh + +;; 2^512 mod P precomputed for SM2 polynomial +LRR DQ 00000000200000003h,000000002ffffffffh,00000000100000001h,00000000400000002h + +LOne DD 1,1,1,1,1,1,1,1 +LTwo DD 2,2,2,2,2,2,2,2 +LThree DD 3,3,3,3,3,3,3,3 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void sm2_mul_by_2(uint64_t res[4], uint64_t a[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM sm2_mul_by_2,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13 + USES_XMM + COMP_ABI 2 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 +%xdefine t4 r13 + + xor t4, t4 + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + + shld t4, a3, 1 + shld a3, a2, 1 + shld a2, a1, 1 + shld a1, a0, 1 + shl a0, 1 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + mov t3, a3 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb t3, qword [rel Lpoly+sizeof(qword)*3] + sbb t4, 0 + + cmovz a0, t0 + cmovz a1, t1 + cmovz a2, t2 + cmovz a3, t3 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + + REST_XMM + REST_GPR + ret +ENDFUNC sm2_mul_by_2 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void sm2_div_by_2(uint64_t res[4], uint64_t a[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM sm2_div_by_2,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13,r14 + USES_XMM + COMP_ABI 2 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 +%xdefine t4 r13 + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + + xor t4, t4 + xor r14, r14 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + mov t3, a3 + + add t0, qword [rel Lpoly+sizeof(qword)*0] + adc t1, qword [rel Lpoly+sizeof(qword)*1] + adc t2, qword [rel Lpoly+sizeof(qword)*2] + adc t3, qword [rel Lpoly+sizeof(qword)*3] + adc t4, 0 + test a0, 1 + + cmovnz a0, t0 + cmovnz a1, t1 + cmovnz a2, t2 + cmovnz a3, t3 + cmovnz r14,t4 + + shrd a0, a1, 1 + shrd a1, a2, 1 + shrd a2, a3, 1 + shrd a3, r14,1 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + + REST_XMM + REST_GPR + ret +ENDFUNC sm2_div_by_2 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void sm2_mul_by_3(uint64_t res[4], uint64_t a[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM sm2_mul_by_3,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13 + USES_XMM + COMP_ABI 2 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 +%xdefine t4 r13 + + xor t4, t4 + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + + shld t4, a3, 1 + shld a3, a2, 1 + shld a2, a1, 1 + shld a1, a0, 1 + shl a0, 1 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + mov t3, a3 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb t3, qword [rel Lpoly+sizeof(qword)*3] + sbb t4, 0 + + cmovz a0, t0 + cmovz a1, t1 + cmovz a2, t2 + cmovz a3, t3 + + xor t4, t4 + add a0, qword [rsi+sizeof(qword)*0] + adc a1, qword [rsi+sizeof(qword)*1] + adc a2, qword [rsi+sizeof(qword)*2] + adc a3, qword [rsi+sizeof(qword)*3] + adc t4, 0 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + mov t3, a3 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb t3, qword [rel Lpoly+sizeof(qword)*3] + sbb t4, 0 + + cmovz a0, t0 + cmovz a1, t1 + cmovz a2, t2 + cmovz a3, t3 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + + REST_XMM + REST_GPR + ret +ENDFUNC sm2_mul_by_3 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void sm2_add(uint64_t res[4], uint64_t a[4], uint64_t b[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM sm2_add,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13 + USES_XMM + COMP_ABI 3 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 +%xdefine t4 r13 + + xor t4, t4 + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + + add a0, qword [rdx+sizeof(qword)*0] + adc a1, qword [rdx+sizeof(qword)*1] + adc a2, qword [rdx+sizeof(qword)*2] + adc a3, qword [rdx+sizeof(qword)*3] + adc t4, 0 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + mov t3, a3 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb t3, qword [rel Lpoly+sizeof(qword)*3] + sbb t4, 0 + + cmovz a0, t0 + cmovz a1, t1 + cmovz a2, t2 + cmovz a3, t3 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + + REST_XMM + REST_GPR + ret +ENDFUNC sm2_add + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void sm2_sub(uint64_t res[4], uint64_t a[4], uint64_t b[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM sm2_sub,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13 + USES_XMM + COMP_ABI 3 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 +%xdefine t4 r13 + + xor t4, t4 + + mov a0, qword [rsi+sizeof(qword)*0] + mov a1, qword [rsi+sizeof(qword)*1] + mov a2, qword [rsi+sizeof(qword)*2] + mov a3, qword [rsi+sizeof(qword)*3] + + sub a0, qword [rdx+sizeof(qword)*0] + sbb a1, qword [rdx+sizeof(qword)*1] + sbb a2, qword [rdx+sizeof(qword)*2] + sbb a3, qword [rdx+sizeof(qword)*3] + sbb t4, 0 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + mov t3, a3 + + add t0, qword [rel Lpoly+sizeof(qword)*0] + adc t1, qword [rel Lpoly+sizeof(qword)*1] + adc t2, qword [rel Lpoly+sizeof(qword)*2] + adc t3, qword [rel Lpoly+sizeof(qword)*3] + test t4, t4 + + cmovnz a0, t0 + cmovnz a1, t1 + cmovnz a2, t2 + cmovnz a3, t3 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + + REST_XMM + REST_GPR + ret +ENDFUNC sm2_sub + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void sm2_neg(uint64_t res[4], uint64_t a[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM sm2_neg,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rsi,rdi,r12,r13 + USES_XMM + COMP_ABI 2 + +%xdefine a0 r8 +%xdefine a1 r9 +%xdefine a2 r10 +%xdefine a3 r11 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t2 rcx +%xdefine t3 r12 +%xdefine t4 r13 + + xor t4, t4 + + xor a0, a0 + xor a1, a1 + xor a2, a2 + xor a3, a3 + + sub a0, qword [rsi+sizeof(qword)*0] + sbb a1, qword [rsi+sizeof(qword)*1] + sbb a2, qword [rsi+sizeof(qword)*2] + sbb a3, qword [rsi+sizeof(qword)*3] + sbb t4, 0 + + mov t0, a0 + mov t1, a1 + mov t2, a2 + mov t3, a3 + + add t0, qword [rel Lpoly+sizeof(qword)*0] + adc t1, qword [rel Lpoly+sizeof(qword)*1] + adc t2, qword [rel Lpoly+sizeof(qword)*2] + adc t3, qword [rel Lpoly+sizeof(qword)*3] + test t4, t4 + + cmovnz a0, t0 + cmovnz a1, t1 + cmovnz a2, t2 + cmovnz a3, t3 + + mov qword [rdi+sizeof(qword)*0], a0 + mov qword [rdi+sizeof(qword)*1], a1 + mov qword [rdi+sizeof(qword)*2], a2 + mov qword [rdi+sizeof(qword)*3], a3 + + REST_XMM + REST_GPR + ret +ENDFUNC sm2_neg + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void sm2_mul_montl(uint64_t res[4], uint64_t a[4], uint64_t b[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; on entry p5=0 +; on exit p0=0, p5=0/1 +; +%macro sm2_mul_redstep 6.nolist + %xdefine %%p5 %1 + %xdefine %%p4 %2 + %xdefine %%p3 %3 + %xdefine %%p2 %4 + %xdefine %%p1 %5 + %xdefine %%p0 %6 + + mov acc6, %%p0 ;; (u1:u0) = p0*2^32 + shl acc6, 32 + mov acc7, %%p0 + shr acc7, 32 + + mov t0, %%p0 + mov t4, %%p0 + xor t1, t1 + xor t3, t3 + + sub t0, acc6 ;; (t3:t2:t1:t0) = p0*2^256 -p0*2^224 +p0*2^64 -p0*2^96 + sbb t1, acc7 + sbb t3, acc6 + sbb t4, acc7 + + add %%p1, t0 ;; (p5:p4:p3:p2:p1) -= (t3:t2:t1:t0) + adc %%p2, t1 + adc %%p3, t3 + adc %%p4, t4 + adc %%p5, 0 ;; p5 extension + xor %%p0, %%p0 ;; p0 = 0; +%endmacro + +align IPP_ALIGN_FACTOR +sm2_mmull: + +%xdefine acc0 r8 +%xdefine acc1 r9 +%xdefine acc2 r10 +%xdefine acc3 r11 +%xdefine acc4 r12 +%xdefine acc5 r13 +%xdefine acc6 r14 +%xdefine acc7 r15 + +%xdefine t0 rcx +%xdefine t1 rbp +%xdefine t2 rbx +%xdefine t3 rdx +%xdefine t4 rax +%xdefine t5 r14 + +; rdi assumed as result +%xdefine aPtr rsi +%xdefine bPtr rbx + + xor acc5, acc5 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[0] + mov rax, qword [bPtr+sizeof(qword)*0] + mul qword [aPtr+sizeof(qword)*0] + mov acc0, rax + mov acc1, rdx + + mov rax, qword [bPtr+sizeof(qword)*0] + mul qword [aPtr+sizeof(qword)*1] + add acc1, rax + adc rdx, 0 + mov acc2, rdx + + mov rax, qword [bPtr+sizeof(qword)*0] + mul qword [aPtr+sizeof(qword)*2] + add acc2, rax + adc rdx, 0 + mov acc3, rdx + + mov rax, qword [bPtr+sizeof(qword)*0] + mul qword [aPtr+sizeof(qword)*3] + add acc3, rax + adc rdx, 0 + mov acc4, rdx + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; 1-st reduction step + sm2_mul_redstep acc5,acc4,acc3,acc2,acc1,acc0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[1] + mov rax, qword [bPtr+sizeof(qword)*1] + mul qword [aPtr+sizeof(qword)*0] + add acc1, rax + adc rdx, 0 + mov t0, rdx + + mov rax, qword [bPtr+sizeof(qword)*1] + mul qword [aPtr+sizeof(qword)*1] + add acc2, t0 + adc rdx, 0 + add acc2, rax + adc rdx, 0 + mov t0, rdx + + mov rax, qword [bPtr+sizeof(qword)*1] + mul qword [aPtr+sizeof(qword)*2] + add acc3, t0 + adc rdx, 0 + add acc3, rax + adc rdx, 0 + mov t0, rdx + + mov rax, qword [bPtr+sizeof(qword)*1] + mul qword [aPtr+sizeof(qword)*3] + add acc4, t0 + adc rdx, 0 + add acc4, rax + adc acc5, rdx + adc acc0, 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; 2-ndt reduction step + sm2_mul_redstep acc0,acc5,acc4,acc3,acc2,acc1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[2] + mov rax, qword [bPtr+sizeof(qword)*2] + mul qword [aPtr+sizeof(qword)*0] + add acc2, rax + adc rdx, 0 + mov t0, rdx + + mov rax, qword [bPtr+sizeof(qword)*2] + mul qword [aPtr+sizeof(qword)*1] + add acc3, t0 + adc rdx, 0 + add acc3, rax + adc rdx, 0 + mov t0, rdx + + mov rax, qword [bPtr+sizeof(qword)*2] + mul qword [aPtr+sizeof(qword)*2] + add acc4, t0 + adc rdx, 0 + add acc4, rax + adc rdx, 0 + mov t0, rdx + + mov rax, qword [bPtr+sizeof(qword)*2] + mul qword [aPtr+sizeof(qword)*3] + add acc5, t0 + adc rdx, 0 + add acc5, rax + adc acc0, rdx + adc acc1, 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; 3-rd reduction step + sm2_mul_redstep acc1,acc0,acc5,acc4,acc3,acc2 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[3] + mov rax, qword [bPtr+sizeof(qword)*3] + mul qword [aPtr+sizeof(qword)*0] + add acc3, rax + adc rdx, 0 + mov t0, rdx + + mov rax, qword [bPtr+sizeof(qword)*3] + mul qword [aPtr+sizeof(qword)*1] + add acc4, t0 + adc rdx, 0 + add acc4, rax + adc rdx, 0 + mov t0, rdx + + mov rax, qword [bPtr+sizeof(qword)*3] + mul qword [aPtr+sizeof(qword)*2] + add acc5, t0 + adc rdx, 0 + add acc5, rax + adc rdx, 0 + mov t0, rdx + + mov rax, qword [bPtr+sizeof(qword)*3] + mul qword [aPtr+sizeof(qword)*3] + add acc0, t0 + adc rdx, 0 + add acc0, rax + adc acc1, rdx + adc acc2, 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; final reduction step + sm2_mul_redstep acc2,acc1,acc0,acc5,acc4,acc3 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov t0, qword [rel Lpoly+sizeof(qword)*0] + mov t1, qword [rel Lpoly+sizeof(qword)*1] + mov t2, qword [rel Lpoly+sizeof(qword)*2] + mov t3, qword [rel Lpoly+sizeof(qword)*3] + + mov t4, acc4 + mov acc3, acc5 + mov acc6, acc0 + mov acc7, acc1 + + sub t4, t0 ;; test %if it exceeds prime value + sbb acc3, t1 + sbb acc6, t2 + sbb acc7, t3 + sbb acc2, 0 + + cmovnc acc4, t4 + cmovnc acc5, acc3 + cmovnc acc0, acc6 + cmovnc acc1, acc7 + + mov qword [rdi+sizeof(qword)*0], acc4 + mov qword [rdi+sizeof(qword)*1], acc5 + mov qword [rdi+sizeof(qword)*2], acc0 + mov qword [rdi+sizeof(qword)*3], acc1 + + ret + +align IPP_ALIGN_FACTOR +IPPASM sm2_mul_montl,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbp,rbx,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 3 + +%xdefine bPtr rbx + + mov bPtr, rdx + call sm2_mmull + + REST_XMM + REST_GPR + ret +ENDFUNC sm2_mul_montl + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void sm2_to_mont(uint64_t res[4], uint64_t a[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM sm2_to_mont,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbp,rbx,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 2 + + lea rbx, [rel LRR] + call sm2_mmull + REST_XMM + REST_GPR + ret +ENDFUNC sm2_to_mont + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void sm2_mul_montx(uint64_t res[4], uint64_t a[4], uint64_t b[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%if (_IPP32E >= _IPP32E_L9) +align IPP_ALIGN_FACTOR +sm2_mmulx: + +%xdefine acc0 r8 +%xdefine acc1 r9 +%xdefine acc2 r10 +%xdefine acc3 r11 +%xdefine acc4 r12 +%xdefine acc5 r13 +%xdefine acc6 r14 +%xdefine acc7 r15 + +%xdefine t0 rax +%xdefine t1 rdx +%xdefine t3 rcx +%xdefine t4 rbp +%xdefine t2 rbx + +; rdi assumed as result +%xdefine aPtr rsi +%xdefine bPtr rbx + + xor acc5, acc5 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[0] + xor rdx, rdx + mov rdx, qword [bPtr+sizeof(qword)*0] + mulx acc1,acc0, qword [aPtr+sizeof(qword)*0] + mulx acc2,t3, qword [aPtr+sizeof(qword)*1] + add acc1,t3 + mulx acc3,t3, qword [aPtr+sizeof(qword)*2] + adc acc2,t3 + mulx acc4,t3, qword [aPtr+sizeof(qword)*3] + adc acc3,t3 + adc acc4,0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 0 + sm2_mul_redstep acc5,acc4,acc3,acc2,acc1,acc0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[1] + mov rdx, qword [bPtr+sizeof(qword)*1] + + mulx t4, t3, qword [aPtr+sizeof(qword)*0] + adcx acc1, t3 + adox acc2, t4 + + mulx t4, t3, qword [aPtr+sizeof(qword)*1] + adcx acc2, t3 + adox acc3, t4 + + mulx t4, t3, qword [aPtr+sizeof(qword)*2] + adcx acc3, t3 + adox acc4, t4 + + mulx t4, t3, qword [aPtr+sizeof(qword)*3] + adcx acc4, t3 + adox acc5, t4 + + adcx acc5, acc0 + adox acc0, acc0 + adc acc0, 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 1 + sm2_mul_redstep acc0,acc5,acc4,acc3,acc2,acc1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[2] + mov rdx, qword [bPtr+sizeof(qword)*2] + + mulx t4, t3, qword [aPtr+sizeof(qword)*0] + adcx acc2, t3 + adox acc3, t4 + + mulx t4, t3, qword [aPtr+sizeof(qword)*1] + adcx acc3, t3 + adox acc4, t4 + + mulx t4, t3, qword [aPtr+sizeof(qword)*2] + adcx acc4, t3 + adox acc5, t4 + + mulx t4, t3, qword [aPtr+sizeof(qword)*3] + adcx acc5, t3 + adox acc0, t4 + + adcx acc0, acc1 + adox acc1, acc1 + + adc acc1, 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 2 + sm2_mul_redstep acc1,acc0,acc5,acc4,acc3,acc2 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; * b[3] + mov rdx, qword [bPtr+sizeof(qword)*3] + + mulx t4, t3, qword [aPtr+sizeof(qword)*0] + adcx acc3, t3 + adox acc4, t4 + + mulx t4, t3, qword [aPtr+sizeof(qword)*1] + adcx acc4, t3 + adox acc5, t4 + + mulx t4, t3, qword [aPtr+sizeof(qword)*2] + adcx acc5, t3 + adox acc0, t4 + + mulx t4, t3, qword [aPtr+sizeof(qword)*3] + adcx acc0, t3 + adox acc1, t4 + + adcx acc1, acc2 + adox acc2, acc2 + adc acc2, 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; reduction step 3 (final) + sm2_mul_redstep acc2,acc1,acc0,acc5,acc4,acc3 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov t0, qword [rel Lpoly+sizeof(qword)*0] + mov t1, qword [rel Lpoly+sizeof(qword)*1] + mov t2, qword [rel Lpoly+sizeof(qword)*2] + mov t3, qword [rel Lpoly+sizeof(qword)*3] + + mov t4, acc4 ;; copy reducted result + mov acc3, acc5 + mov acc6, acc0 + mov acc7, acc1 + + sub t4, t0 ;; test %if it exceeds prime value + sbb acc3, t1 + sbb acc6, t2 + sbb acc7, t3 + sbb acc2, 0 + + cmovnc acc4, t4 + cmovnc acc5, acc3 + cmovnc acc0, acc6 + cmovnc acc1, acc7 + + mov qword [rdi+sizeof(qword)*0], acc4 + mov qword [rdi+sizeof(qword)*1], acc5 + mov qword [rdi+sizeof(qword)*2], acc0 + mov qword [rdi+sizeof(qword)*3], acc1 + + ret +%endif + +%if _IPP32E >= _IPP32E_L9 +align IPP_ALIGN_FACTOR +IPPASM sm2_mul_montx,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbp,rbx,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 3 + +%xdefine bPtr rbx + + mov bPtr, rdx + call sm2_mmulx + + REST_XMM + REST_GPR + ret +ENDFUNC sm2_mul_montx + +%endif ;; _IPP32E_L9 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void sm2_sqr_montl(uint64_t res[4], uint64_t a[4]); +; void sm2_sqr_montx(uint64_t res[4], uint64_t a[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%macro sm2_prod_redstep 6.nolist + %xdefine %%e %1 + %xdefine %%p4 %2 + %xdefine %%p3 %3 + %xdefine %%p2 %4 + %xdefine %%p1 %5 + %xdefine %%p0 %6 + + mov t4, %%p0 + mov t0, %%p0 + mov t3, %%p0 + xor t1, t1 + xor t2, t2 + shl %%p0, 32 + shr t4, 32 ;; (t4:p0) = p0*2^32 + + sub t0, %%p0 ;; (t3:t2:t1:t0) = p0*2^256 -p0*2^224 +p0*2^64 -p0*2^96 + sbb t1, t4 + sbb t2, %%p0 + sbb t3, t4 + + xor %%p0, %%p0 + add %%p1, t0 ;; (p5:p4:p3:p2:p1) -= (t3:t2:t1:t0) + adc %%p2, t1 + adc %%p3, t2 + adc %%p4, t3 + adc %%p0, 0 + + %ifnempty %%e + add %%p4, %%e + adc %%p0, 0 + %endif + +%endmacro + +align IPP_ALIGN_FACTOR +IPPASM sm2_sqr_montl,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbp,rbx,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 2 + +%xdefine acc0 r8 +%xdefine acc1 r9 +%xdefine acc2 r10 +%xdefine acc3 r11 +%xdefine acc4 r12 +%xdefine acc5 r13 +%xdefine acc6 r14 +%xdefine acc7 r15 + +%xdefine t0 rcx +%xdefine t1 rbp +%xdefine t2 rbx +%xdefine t3 rdx +%xdefine t4 rax + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov t2, qword [aPtr+sizeof(qword)*0] + mov rax,qword [aPtr+sizeof(qword)*1] + mul t2 + mov acc1, rax + mov acc2, rdx + mov rax,qword [aPtr+sizeof(qword)*2] + mul t2 + add acc2, rax + adc rdx, 0 + mov acc3, rdx + mov rax,qword [aPtr+sizeof(qword)*3] + mul t2 + add acc3, rax + adc rdx, 0 + mov acc4, rdx + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov t2, qword [aPtr+sizeof(qword)*1] + mov rax,qword [aPtr+sizeof(qword)*2] + mul t2 + add acc3, rax + adc rdx, 0 + mov t1, rdx + mov rax,qword [aPtr+sizeof(qword)*3] + mul t2 + add acc4, rax + adc rdx, 0 + add acc4, t1 + adc rdx, 0 + mov acc5, rdx + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov t2, qword [aPtr+sizeof(qword)*2] + mov rax,qword [aPtr+sizeof(qword)*3] + mul t2 + add acc5, rax + adc rdx, 0 + mov acc6, rdx + + xor acc7, acc7 + shld acc7, acc6, 1 + shld acc6, acc5, 1 + shld acc5, acc4, 1 + shld acc4, acc3, 1 + shld acc3, acc2, 1 + shld acc2, acc1, 1 + shl acc1, 1 + + mov rax,qword [aPtr+sizeof(qword)*0] + mul rax + mov acc0, rax + add acc1, rdx + adc acc2, 0 + mov rax,qword [aPtr+sizeof(qword)*1] + mul rax + add acc2, rax + adc acc3, rdx + adc acc4, 0 + mov rax,qword [aPtr+sizeof(qword)*2] + mul rax + add acc4, rax + adc acc5, rdx + adc acc6, 0 + mov rax,qword [aPtr+sizeof(qword)*3] + mul rax + add acc6, rax + adc acc7, rdx + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + sm2_prod_redstep ,acc4,acc3,acc2,acc1,acc0 + sm2_prod_redstep acc0,acc5,acc4,acc3,acc2,acc1 + sm2_prod_redstep acc1,acc6,acc5,acc4,acc3,acc2 + sm2_prod_redstep acc2,acc7,acc6,acc5,acc4,acc3 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + mov t0, qword [rel Lpoly+sizeof(qword)*0] + mov t1, qword [rel Lpoly+sizeof(qword)*1] + mov t2, qword [rel Lpoly+sizeof(qword)*2] + mov t3, qword [rel Lpoly+sizeof(qword)*3] + + mov t4, acc4 + mov acc0, acc5 + mov acc1, acc6 + mov acc2, acc7 + + sub t4, t0 + sbb acc0, t1 + sbb acc1, t2 + sbb acc2, t3 + sbb acc3, 0 + + cmovnc acc4, t4 + cmovnc acc5, acc0 + cmovnc acc6, acc1 + cmovnc acc7, acc2 + + mov qword [rdi+sizeof(qword)*0], acc4 + mov qword [rdi+sizeof(qword)*1], acc5 + mov qword [rdi+sizeof(qword)*2], acc6 + mov qword [rdi+sizeof(qword)*3], acc7 + + REST_XMM + REST_GPR + ret +ENDFUNC sm2_sqr_montl + +%if (_IPP32E >= _IPP32E_L9) +align IPP_ALIGN_FACTOR +IPPASM sm2_sqr_montx,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbp,rbx,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 2 + +%xdefine acc0 r8 +%xdefine acc1 r9 +%xdefine acc2 r10 +%xdefine acc3 r11 +%xdefine acc4 r12 +%xdefine acc5 r13 +%xdefine acc6 r14 +%xdefine acc7 r15 + +%xdefine t0 rcx +%xdefine t1 rbp +%xdefine t2 rbx +%xdefine t3 rdx +%xdefine t4 rax + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov rdx, qword [aPtr+sizeof(qword)*0] + mulx acc2, acc1, qword [aPtr+sizeof(qword)*1] + mulx acc3, t0, qword [aPtr+sizeof(qword)*2] + add acc2, t0 + mulx acc4, t0, qword [aPtr+sizeof(qword)*3] + adc acc3, t0 + adc acc4, 0 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov rdx, qword [aPtr+sizeof(qword)*1] + xor acc5, acc5 + mulx t1, t0, qword [aPtr+sizeof(qword)*2] + adcx acc3, t0 + adox acc4, t1 + mulx t1, t0, qword [aPtr+sizeof(qword)*3] + adcx acc4, t0 + adox acc5, t1 + adc acc5, 0 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov rdx, qword [aPtr+sizeof(qword)*2] + mulx acc6, t0, qword [aPtr+sizeof(qword)*3] + add acc5, t0 + adc acc6, 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + xor acc7, acc7 + shld acc7, acc6, 1 + shld acc6, acc5, 1 + shld acc5, acc4, 1 + shld acc4, acc3, 1 + shld acc3, acc2, 1 + shld acc2, acc1, 1 + shl acc1, 1 + + xor acc0, acc0 + mov rdx, qword [aPtr+sizeof(qword)*0] + mulx t1, acc0, rdx + adcx acc1, t1 + mov rdx, qword [aPtr+sizeof(qword)*1] + mulx t1, t0, rdx + adcx acc2, t0 + adcx acc3, t1 + mov rdx, qword [aPtr+sizeof(qword)*2] + mulx t1, t0, rdx + adcx acc4, t0 + adcx acc5, t1 + mov rdx, qword [aPtr+sizeof(qword)*3] + mulx t1, t0, rdx + adcx acc6, t0 + adcx acc7, t1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + sm2_prod_redstep ,acc4,acc3,acc2,acc1,acc0 + sm2_prod_redstep acc0,acc5,acc4,acc3,acc2,acc1 + sm2_prod_redstep acc1,acc6,acc5,acc4,acc3,acc2 + sm2_prod_redstep acc2,acc7,acc6,acc5,acc4,acc3 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + mov t0, qword [rel Lpoly+sizeof(qword)*0] + mov t1, qword [rel Lpoly+sizeof(qword)*1] + mov t2, qword [rel Lpoly+sizeof(qword)*2] + mov t3, qword [rel Lpoly+sizeof(qword)*3] + + mov t4, acc4 + mov acc0, acc5 + mov acc1, acc6 + mov acc2, acc7 + + sub t4, t0 + sbb acc0, t1 + sbb acc1, t2 + sbb acc2, t3 + sbb acc3, 0 + + cmovnc acc4, t4 + cmovnc acc5, acc0 + cmovnc acc6, acc1 + cmovnc acc7, acc2 + + mov qword [rdi+sizeof(qword)*0], acc4 + mov qword [rdi+sizeof(qword)*1], acc5 + mov qword [rdi+sizeof(qword)*2], acc6 + mov qword [rdi+sizeof(qword)*3], acc7 + + REST_XMM + REST_GPR + ret +ENDFUNC sm2_sqr_montx + +%endif ;; _IPP32E_L9 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; void sm2_mont_back(uint64_t res[4], uint64_t a[4]); +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +IPPASM sm2_mont_back,PUBLIC +%assign LOCAL_FRAME 0 + USES_GPR rbp,rbx,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 2 + +%xdefine acc0 r8 +%xdefine acc1 r9 +%xdefine acc2 r10 +%xdefine acc3 r11 +%xdefine acc4 r12 +%xdefine acc5 r13 +%xdefine acc6 r14 +%xdefine acc7 r15 + +%xdefine t0 rcx +%xdefine t1 rbp +%xdefine t2 rbx +%xdefine t3 rdx +%xdefine t4 rax + + mov acc2, qword [rsi+sizeof(qword)*0] + mov acc3, qword [rsi+sizeof(qword)*1] + mov acc4, qword [rsi+sizeof(qword)*2] + mov acc5, qword [rsi+sizeof(qword)*3] + xor acc0, acc0 + xor acc1, acc1 + + sm2_mul_redstep acc1,acc0,acc5,acc4,acc3,acc2 + sm2_mul_redstep acc2,acc1,acc0,acc5,acc4,acc3 + sm2_mul_redstep acc3,acc2,acc1,acc0,acc5,acc4 + sm2_mul_redstep acc4,acc3,acc2,acc1,acc0,acc5 + + mov t0, acc0 + mov t1, acc1 + mov t2, acc2 + mov t3, acc3 + + sub t0, qword [rel Lpoly+sizeof(qword)*0] + sbb t1, qword [rel Lpoly+sizeof(qword)*1] + sbb t2, qword [rel Lpoly+sizeof(qword)*2] + sbb t3, qword [rel Lpoly+sizeof(qword)*3] + sbb acc4, 0 + + cmovnc acc0, t0 + cmovnc acc1, t1 + cmovnc acc2, t2 + cmovnc acc3, t3 + + mov qword [rdi+sizeof(qword)*0], acc0 + mov qword [rdi+sizeof(qword)*1], acc1 + mov qword [rdi+sizeof(qword)*2], acc2 + mov qword [rdi+sizeof(qword)*3], acc3 + + REST_XMM + REST_GPR + ret +ENDFUNC sm2_mont_back + +%endif ;; _IPP32E>=_IPP32E_M7 + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsm3e9as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsm3e9as.asm new file mode 100644 index 0000000..30924ad --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsm3e9as.asm @@ -0,0 +1,874 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SM3 +; +; Content: +; UpdateSM3 +; + +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SM3_) +%if (_IPP32E >= _IPP32E_E9) + +%xdefine A r8d +%xdefine B r9d +%xdefine C r10d +%xdefine D r11d +%xdefine E r12d +%xdefine F r13d +%xdefine G r14d +%xdefine H r15d + +%xdefine hPtr rdi +%xdefine mPtr rsi +%xdefine kPtr rcx + +%xdefine t0 eax +%xdefine t1 ebx +%xdefine t2 ebp +%xdefine t3 edx + +%xdefine W0 xmm0 +%xdefine W1 xmm1 +%xdefine W2 xmm2 +%xdefine W3 xmm3 + +%xdefine xT0 xmm4 +%xdefine xT1 xmm5 +%xdefine xTr xmm6 +%xdefine xWZZZ xmm7 +%xdefine xBCAST xmm8 +%xdefine xROL8 xmm9 + + +%macro ROTATE_H 0.nolist + %xdefine %%DUMMY H + %xdefine H G + %xdefine G F + %xdefine F E + %xdefine E D + %xdefine D C + %xdefine C B + %xdefine B A + %xdefine A %%DUMMY +%endmacro + +%macro ROUND_00_15 1.nolist + %xdefine %%nr %1 + + mov t0, A + mov t1, [kPtr+%%nr*sizeof(dword)] + shld t0, t0, 12 ; t0 = t + add t1, t0 + add t1, E + shld t1, t1, 7 ; t1 = SS1 + xor t0, t1 ; t0 = SS2 + + add t1, H ; t1: TT2 = SS1 + h + add t0, D ; t0: TT1 = SS2 + d + + add t1, [rsp+(%%nr & 3)*sizeof(dword)] ; t1: TT2 = SS1 + h +w[nr] + add t0, [rsp+(%%nr & 3)*sizeof(dword)+sizeof(oword)] ; t0: TT1 = SS2 + d +(w[nr]^w[nr+4]) + + mov t2, B + mov t3, F + + xor t2, C + xor t3, G + + xor t2, A ; t2 = FF(a,b,c) = a^b^c + xor t3, E ; t3 = GG(e,f,g) = e^f^g + + add t1, t3 ; t1: TT2 = SS1 +h +w[nr] +GG + add t0, t2 ; t0: TT1 = SS2 +d +(w[nr]^w[nr+4]) +FF + + shld B, B, 9 + shld F, F, 19 + + mov H, t0 + + mov D, t1 + shld D, D, 8 + xor D, t1 + shld D, D, 9 + xor D, t1 + + ROTATE_H +%endmacro + +%macro ROUND_16_63 9.nolist + %xdefine %%nr %1 + %xdefine %%A %2 + %xdefine %%B %3 + %xdefine %%C %4 + %xdefine %%D %5 + %xdefine %%E %6 + %xdefine %%F %7 + %xdefine %%G %8 + %xdefine %%H %9 + + mov t0, %%A + mov t1, [kPtr+%%nr*sizeof(dword)] + shld t0, t0, 12 ; t0 = t + add t1, t0 + add t1, %%E + shld t1, t1, 7 ; t1 = SS1 + xor t0, t1 ; t0 = SS2 + + add t1, %%H ; t1: TT2 = SS1 + h + add t0, %%D ; t0: TT1 = SS2 + d + + add t1, [rsp+(%%nr & 3)*sizeof(dword)] ; t1: TT2 = SS1 + h +w[nr] + add t0, [rsp+(%%nr & 3)*sizeof(dword)+sizeof(oword)] ; t0: TT1 = SS2 + d +(w[nr]^w[nr+4]) + + mov t3, %%B + mov t2, %%B + and t3, %%C + xor t2, %%C + and t2, %%A + add t2, t3 + + mov t3, %%F + xor t3, %%G + and t3, %%E + xor t3, %%G + + add t0, t2 ; t0: TT1 = SS2 +d +(w[nr]^w[nr+4]) +FF + add t1, t3 ; t1: TT2 = SS1 +h +w[nr] +GG + + shld %%B, %%B, 9 + shld %%F, %%F, 19 + + mov %%H, t0 + + mov %%D, t1 + shld %%D, %%D, 8 + xor %%D, t1 + shld %%D, %%D, 9 + xor %%D, t1 + + ;ROTATE_H +%endmacro + +;; +;; Y = ROL32(X,nbits) +%macro xmmROL32 3.nolist + %xdefine %%Y %1 + %xdefine %%X %2 + %xdefine %%nbits %3 + +%if %%nbits != 8 + vpslld xTr,%%X, %%nbits + vpsrld %%Y, %%X, (32-%%nbits) + vpxor %%Y, %%Y, xTr +%else + ;;vpshufb Y, X, oword [rel rol_32_8] + vpshufb %%Y, %%X, xROL8 +%endif +%endmacro + +%macro xmmBCAST 2.nolist + %xdefine %%Y %1 + %xdefine %%X %2 + + ;;vpshufb Y, X, oword [rel bcast] + vpshufb %%Y, %%X, xBCAST +%endmacro + +;; +;; w[j] = P1(w[t-16]^w[t-9]^ROL32(w[t-3],15)) ^ ROL32(w[t-13],7) ^ w[t-6] +;; +%macro ROTATE_W 0.nolist + %xdefine %%DUMMY W0 + %xdefine W0 W1 + %xdefine W1 W2 + %xdefine W2 W3 + %xdefine W3 %%DUMMY +%endmacro + +%macro QSCHED 0.nolist + vpalignr xT0, W1, W0, sizeof(dword)*3 ;; T0 = {w6,w5,w4,w3} + xmmROL32 xT0, xT0, 7 ;; T0 = ROL32({w6,w5,w4,w3}, 7) + + ;; compute P1() argument + vpsrldq xT1, W3, sizeof(dword) ;; T1 = {Z,w15,w14,w13} + xmmROL32 xT1, xT1, 15 ;; T1 = ROL32({Z,w15,w14,w13}, 15) + vpxor W0, W0, xT1 ;; W0 = {w3,w2,w1,w0} ^ ROL32({Z,w15,w14,w13}, 15) + vpalignr xT1, W2, W1, sizeof(dword)*3 ;; T1 = {w10,w9,w8,w7} + vpxor W0, W0, xT1 ;; W0 = {w3,w2,w1,w0} ^ ROL32({Z,w15,w14,w13}, 15) ^ {w10,w9,w8,w7} + + ;; compute W0 = P1(W0), P1(x) = x ^ rol32(x,15) ^ rol(x,23) + xmmROL32 xT1, W0, 8 + vpxor xT1, xT1, W0 + xmmROL32 xT1, xT1, 15 + vpxor W0, W0, xT1 + + vpalignr xT1, W3, W2, sizeof(dword)*2 ;; T1 = {w13,w12,w11,w10} + + vpxor W0, W0, xT0 + vpxor W0, W0, xT1 + + ;; compute P1(rol(w16,15)) + xmmBCAST xT0, W0 ;; T0 = {w16,w16,w16,rw16} + vpsrlq xT0, xT0, (32-15) ;; T0 = {??,rol(w16,15),??,rol(w16,15)} + xmmBCAST xT0, xT0 ;; T0 = {rol(w16,15),rol(w16,15),rol(w16,15),rol(w16,15)} + vpsllq xT1, xT0, 15 + vpxor xT0, xT0, xT1 + vpsllq xT1, xT1, (23-15) + vpxor xT0, xT0, xT1 + ;;vpshufb xT0, xT0, [rel wzzz] ;; T0 = {P1(rol(w16,15)),zz,zz,zz} + vpshufb xT0, xT0, xWZZZ ;; T0 = {P1(rol(w16,15)),zz,zz,zz} + + vpxor W0, W0, xT0 + ROTATE_W +%endmacro + +%macro QSCHED_W_QROUND_00_15 9.nolist + %xdefine %%nr %1 + %xdefine %%A %2 + %xdefine %%B %3 + %xdefine %%C %4 + %xdefine %%D %5 + %xdefine %%E %6 + %xdefine %%F %7 + %xdefine %%G %8 + %xdefine %%H %9 + + vpalignr xT0, W1, W0, sizeof(dword)*3 ;; T0 = {w6,w5,w4,w3} + xmmROL32 xT0, xT0, 7 ;; T0 = ROL32({w6,w5,w4,w3}, 7) + + %assign %%t %%nr + ;ROUND_00_15 t + mov t0, %%A + mov t1, [kPtr+%%t*sizeof(dword)] + shld t0, t0, 12 ; t0 = t + add t1, t0 + add t1, %%E + shld t1, t1, 7 ; t1 = SS1 + xor t0, t1 ; t0 = SS2 + + add t1, %%H ; t1: TT2 = SS1 + h + add t0, %%D ; t0: TT1 = SS2 + d + + add t1, [rsp+(%%t & 3)*sizeof(dword)] ; t1: TT2 = SS1 + h +w[nr] + add t0, [rsp+(%%t & 3)*sizeof(dword)+sizeof(oword)] ; t0: TT1 = SS2 + d +(w[nr]^w[nr+4]) + + vpsrldq xT1, W3, sizeof(dword) ;; T1 = {Z,w15,w14,w13} + xmmROL32 xT1, xT1, 15 ;; T1 = ROL32({Z,w15,w14,w13}, 15) + + mov t2, %%B + mov t3, %%F + + xor t2, %%C + xor t3, %%G + + xor t2, %%A ; t2 = FF(a,b,c) = a^b^c + xor t3, %%E ; t3 = GG(e,f,g) = e^f^g + + add t1, t3 ; t1: TT2 = SS1 +h +w[nr] +GG + add t0, t2 ; t0: TT1 = SS2 +d +(w[nr]^w[nr+4]) +FF + + shld %%B, %%B, 9 + shld %%F, %%F, 19 + + mov %%H, t0 + + vpxor W0, W0, xT1 ;; W0 = {w3,w2,w1,w0} ^ ROL32({Z,w15,w14,w13}, 15) + vpalignr xT1, W2, W1, sizeof(dword)*3 ;; T1 = {w10,w9,w8,w7} + vpxor W0, W0, xT1 ;; W0 = {w3,w2,w1,w0} ^ ROL32({Z,w15,w14,w13}, 15) ^ {w10,w9,w8,w7} + + mov %%D, t1 + shld %%D, %%D, 8 + xor %%D, t1 + shld %%D, %%D, 9 + xor %%D, t1 + + ;ROTATE_H + ;;;;;;;;;;;;;;; + + %assign %%t %%t+1 + ;ROUND_00_15 t + mov t0, %%H + mov t1, [kPtr+%%t*sizeof(dword)] + shld t0, t0, 12 ; t0 = t + add t1, t0 + add t1, %%D + shld t1, t1, 7 ; t1 = SS1 + xor t0, t1 ; t0 = SS2 + + xmmROL32 xT1, W0, 8 + vpxor xT1, xT1, W0 + + add t1, %%G ; t1: TT2 = SS1 + h + add t0, %%C ; t0: TT1 = SS2 + d + + add t1, [rsp+(%%t & 3)*sizeof(dword)] ; t1: TT2 = SS1 + h +w[nr] + add t0, [rsp+(%%t & 3)*sizeof(dword)+sizeof(oword)] ; t0: TT1 = SS2 + d +(w[nr]^w[nr+4]) + + xmmROL32 xT1, xT1, 15 + vpxor W0, W0, xT1 + + mov t2, %%A + mov t3, %%E + + xor t2, %%B + xor t3, %%F + + xor t2, %%H ; t2 = FF(a,b,c) = a^b^c + xor t3, %%D ; t3 = GG(e,f,g) = e^f^g + + add t1, t3 ; t1: TT2 = SS1 +h +w[nr] +GG + add t0, t2 ; t0: TT1 = SS2 +d +(w[nr]^w[nr+4]) +FF + + shld %%A, %%A, 9 + shld %%E, %%E, 19 + + mov %%G, t0 + + mov %%C, t1 + shld %%C, %%C, 8 + xor %%C, t1 + shld %%C, %%C, 9 + xor %%C, t1 + + ;ROTATE_H + ;;;;;;;;;;;;;;; + + vpalignr xT1, W3, W2, sizeof(dword)*2 ;; T1 = {w13,w12,w11,w10} + vpxor W0, W0, xT0 + vpxor W0, W0, xT1 + + %assign %%t %%t+1 + ;ROUND_00_15 t + mov t0, %%G + mov t1, [kPtr+%%t*sizeof(dword)] + shld t0, t0, 12 ; t0 = t + add t1, t0 + add t1, %%C + shld t1, t1, 7 ; t1 = SS1 + xor t0, t1 ; t0 = SS2 + + add t1, %%F ; t1: TT2 = SS1 + h + add t0, %%B ; t0: TT1 = SS2 + d + + xmmBCAST xT0, W0 ;; T0 = {w16,w16,w16,rw16} + vpsrlq xT0, xT0, (32-15) ;; T0 = {??,rol(w16,15),??,rol(w16,15)} + + add t1, [rsp+(%%t & 3)*sizeof(dword)] ; t1: TT2 = SS1 + h +w[nr] + add t0, [rsp+(%%t & 3)*sizeof(dword)+sizeof(oword)] ; t0: TT1 = SS2 + d +(w[nr]^w[nr+4]) + + mov t2, %%H + mov t3, %%D + + xor t2, %%A + xor t3, %%E + + xor t2, %%G ; t2 = FF(a,b,c) = a^b^c + xor t3, %%C ; t3 = GG(e,f,g) = e^f^g + + add t1, t3 ; t1: TT2 = SS1 +h +w[nr] +GG + add t0, t2 ; t0: TT1 = SS2 +d +(w[nr]^w[nr+4]) +FF + + shld %%H, %%H, 9 + shld %%D, %%D, 19 + + xmmBCAST xT0, xT0 ;; T0 = {rol(w16,15),rol(w16,15),rol(w16,15),rol(w16,15)} + vpsllq xT1, xT0, 15 + + mov %%F, t0 + + mov %%B, t1 + shld %%B, %%B, 8 + xor %%B, t1 + shld %%B, %%B, 9 + xor %%B, t1 + + ;ROTATE_H + ;;;;;;;;;;;;;;; + + %assign %%t %%t+1 + ;ROUND_00_15 t + mov t0, %%F + mov t1, [kPtr+%%t*sizeof(dword)] + shld t0, t0, 12 ; t0 = t + add t1, t0 + add t1, %%B + shld t1, t1, 7 ; t1 = SS1 + xor t0, t1 ; t0 = SS2 + + vpxor xT0, xT0, xT1 + vpsllq xT1, xT1, (23-15) + + add t1, %%E ; t1: TT2 = SS1 + h + add t0, %%A ; t0: TT1 = SS2 + d + + add t1, [rsp+(%%t & 3)*sizeof(dword)] ; t1: TT2 = SS1 + h +w[nr] + add t0, [rsp+(%%t & 3)*sizeof(dword)+sizeof(oword)] ; t0: TT1 = SS2 + d +(w[nr]^w[nr+4]) + + mov t2, %%G + mov t3, %%C + + xor t2, %%H + xor t3, %%D + + vpxor xT0, xT0, xT1 + ;;vpshufb xT0, xT0, [rel wzzz] ;; T0 = {P1(rol(w16,15)),zz,zz,zz} + vpshufb xT0, xT0, xWZZZ ;; T0 = {P1(rol(w16,15)),zz,zz,zz} + + xor t2, %%F ; t2 = FF(a,b,c) = a^b^c + xor t3, %%B ; t3 = GG(e,f,g) = e^f^g + + add t1, t3 ; t1: TT2 = SS1 +h +w[nr] +GG + add t0, t2 ; t0: TT1 = SS2 +d +(w[nr]^w[nr+4]) +FF + + shld %%G, %%G, 9 + shld %%C, %%C, 19 + + vpxor W0, W0, xT0 + + mov %%E, t0 + + mov %%A, t1 + shld %%A, %%A, 8 + xor %%A, t1 + shld %%A, %%A, 9 + xor %%A, t1 + + ;ROTATE_H + ROTATE_W + ;;;;;;;;;;;;;;; +%endmacro + +%macro QSCHED_W_QROUND_16_63 9.nolist + %xdefine %%nr %1 + %xdefine %%A %2 + %xdefine %%B %3 + %xdefine %%C %4 + %xdefine %%D %5 + %xdefine %%E %6 + %xdefine %%F %7 + %xdefine %%G %8 + %xdefine %%H %9 + + vpalignr xT0, W1, W0, sizeof(dword)*3 ;; T0 = {w6,w5,w4,w3} + xmmROL32 xT0, xT0, 7 ;; T0 = ROL32({w6,w5,w4,w3}, 7) + + %assign %%t %%nr + ;ROUND_16_63 t + mov t0, %%A + mov t1, [kPtr+%%t*sizeof(dword)] + shld t0, t0, 12 ; t0 = t + add t1, t0 + add t1, %%E + shld t1, t1, 7 ; t1 = SS1 + xor t0, t1 ; t0 = SS2 + + add t1, %%H ; t1: TT2 = SS1 + h + add t0, %%D ; t0: TT1 = SS2 + d + + add t1, [rsp+(%%t & 3)*sizeof(dword)] ; t1: TT2 = SS1 + h +w[nr] + add t0, [rsp+(%%t & 3)*sizeof(dword)+sizeof(oword)] ; t0: TT1 = SS2 + d +(w[nr]^w[nr+4]) + + vpsrldq xT1, W3, sizeof(dword) ;; T1 = {Z,w15,w14,w13} + xmmROL32 xT1, xT1, 15 ;; T1 = ROL32({Z,w15,w14,w13}, 15) + + mov t3, %%B + mov t2, %%B + and t3, %%C + xor t2, %%C + and t2, %%A + add t2, t3 + + mov t3, %%F + xor t3, %%G + and t3, %%E + xor t3, %%G + + add t0, t2 ; t0: TT1 = SS2 +d +(w[nr]^w[nr+4]) +FF + add t1, t3 ; t1: TT2 = SS1 +h +w[nr] +GG + + shld %%B, %%B, 9 + shld %%F, %%F, 19 + + mov %%H, t0 + + vpxor W0, W0, xT1 ;; W0 = {w3,w2,w1,w0} ^ ROL32({Z,w15,w14,w13}, 15) + vpalignr xT1, W2, W1, sizeof(dword)*3 ;; T1 = {w10,w9,w8,w7} + vpxor W0, W0, xT1 ;; W0 = {w3,w2,w1,w0} ^ ROL32({Z,w15,w14,w13}, 15) ^ {w10,w9,w8,w7} + + mov %%D, t1 + shld %%D, %%D, 8 + xor %%D, t1 + shld %%D, %%D, 9 + xor %%D, t1 + + ;ROTATE_H + ;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + %assign %%t %%t + 1 + ;ROUND_16_63 t + mov t0, %%H + mov t1, [kPtr+%%t*sizeof(dword)] + shld t0, t0, 12 ; t0 = t + add t1, t0 + add t1, %%D + shld t1, t1, 7 ; t1 = SS1 + xor t0, t1 ; t0 = SS2 + + xmmROL32 xT1, W0, 8 + vpxor xT1, xT1, W0 + + add t1, %%G ; t1: TT2 = SS1 + h + add t0, %%C ; t0: TT1 = SS2 + d + + add t1, [rsp+(%%t & 3)*sizeof(dword)] ; t1: TT2 = SS1 + h +w[nr] + add t0, [rsp+(%%t & 3)*sizeof(dword)+sizeof(oword)] ; t0: TT1 = SS2 + d +(w[nr]^w[nr+4]) + + xmmROL32 xT1, xT1, 15 + vpxor W0, W0, xT1 + + mov t3, %%A + mov t2, %%A + and t3, %%B + xor t2, %%B + and t2, %%H + add t2, t3 + + mov t3, %%E + xor t3, %%F + and t3, %%D + xor t3, %%F + + add t0, t2 ; t0: TT1 = SS2 +d +(w[nr]^w[nr+4]) +FF + add t1, t3 ; t1: TT2 = SS1 +h +w[nr] +GG + + shld %%A, %%A, 9 + shld %%E, %%E, 19 + + mov %%G, t0 + + mov %%C, t1 + shld %%C, %%C, 8 + xor %%C, t1 + shld %%C, %%C, 9 + xor %%C, t1 + + ;ROTATE_H + ;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + vpalignr xT1, W3, W2, sizeof(dword)*2 ;; T1 = {w13,w12,w11,w10} + vpxor W0, W0, xT0 + vpxor W0, W0, xT1 + + %assign %%t %%t + 1 + ;ROUND_16_63 t + mov t0, %%G + mov t1, [kPtr+%%t*sizeof(dword)] + shld t0, t0, 12 ; t0 = t + add t1, t0 + add t1, %%C + shld t1, t1, 7 ; t1 = SS1 + xor t0, t1 ; t0 = SS2 + + add t1, %%F ; t1: TT2 = SS1 + h + add t0, %%B ; t0: TT1 = SS2 + d + + xmmBCAST xT0, W0 ;; T0 = {w16,w16,w16,rw16} + vpsrlq xT0, xT0, (32-15) ;; T0 = {??,rol(w16,15),??,rol(w16,15)} + + add t1, [rsp+(%%t & 3)*sizeof(dword)] ; t1: TT2 = SS1 + h +w[nr] + add t0, [rsp+(%%t & 3)*sizeof(dword)+sizeof(oword)] ; t0: TT1 = SS2 + d +(w[nr]^w[nr+4]) + + mov t3, %%H + mov t2, %%H + and t3, %%A + xor t2, %%A + and t2, %%G + add t2, t3 + + mov t3, %%D + xor t3, %%E + and t3, %%C + xor t3, %%E + + add t0, t2 ; t0: TT1 = SS2 +d +(w[nr]^w[nr+4]) +FF + add t1, t3 ; t1: TT2 = SS1 +h +w[nr] +GG + + shld %%H, %%H, 9 + shld %%D, %%D, 19 + + xmmBCAST xT0, xT0 ;; T0 = {rol(w16,15),rol(w16,15),rol(w16,15),rol(w16,15)} + vpsllq xT1, xT0, 15 + + mov %%F, t0 + + mov %%B, t1 + shld %%B, %%B, 8 + xor %%B, t1 + shld %%B, %%B, 9 + xor %%B, t1 + + ;ROTATE_H + ;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + %assign %%t %%t + 1 + ;ROUND_16_63 t + mov t0, %%F + mov t1, [kPtr+%%t*sizeof(dword)] + shld t0, t0, 12 ; t0 = t + add t1, t0 + add t1, %%B + shld t1, t1, 7 ; t1 = SS1 + xor t0, t1 ; t0 = SS2 + + vpxor xT0, xT0, xT1 + vpsllq xT1, xT1, (23-15) + + add t1, %%E ; t1: TT2 = SS1 + h + add t0, %%A ; t0: TT1 = SS2 + d + + add t1, [rsp+(%%t & 3)*sizeof(dword)] ; t1: TT2 = SS1 + h +w[nr] + add t0, [rsp+(%%t & 3)*sizeof(dword)+sizeof(oword)] ; t0: TT1 = SS2 + d +(w[nr]^w[nr+4]) + + mov t3, %%G + mov t2, %%G + and t3, %%H + xor t2, %%H + and t2, %%F + add t2, t3 + + vpxor xT0, xT0, xT1 + ;;vpshufb xT0, xT0, [rel wzzz] ;; T0 = {P1(rol(w16,15)),zz,zz,zz} + vpshufb xT0, xT0, xWZZZ ;; T0 = {P1(rol(w16,15)),zz,zz,zz} + + mov t3, %%C + xor t3, %%D + and t3, %%B + xor t3, %%D + + add t0, t2 ; t0: TT1 = SS2 +d +(w[nr]^w[nr+4]) +FF + add t1, t3 ; t1: TT2 = SS1 +h +w[nr] +GG + + shld %%G, %%G, 9 + shld %%C, %%C, 19 + + vpxor W0, W0, xT0 + + mov %%E, t0 + + mov %%A, t1 + shld %%A, %%A, 8 + xor %%A, t1 + shld %%A, %%A, 9 + xor %%A, t1 + + ;ROTATE_H + ROTATE_W + ;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%endmacro + +segment .text align=IPP_ALIGN_FACTOR + + +align IPP_ALIGN_FACTOR + +bswap128 DB 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12 +rol_32_8 DB 3,0,1,2, 7,4,5,6, 11,8,9,10, 15,12,13,14 +bcast DB 0,1,2,3, 0,1,2,3, 0,1,2,3, 0,1,2,3 +wzzz DB 80h,80h,80h,80h, 80h,80h,80h,80h, 80h,80h,80h,80h,12,13,14,15 + +;******************************************************************** +;* void UpdateSM3(Ipp32u* hash, +; const Ipp8u* msg, int msgLen, +; const Ipp32u* K_SM3) +;******************************************************************** +align IPP_ALIGN_FACTOR +IPPASM UpdateSM3,PUBLIC +%assign LOCAL_FRAME sizeof(oword)*2+sizeof(qword)*2 + USES_GPR rbp,rbx,rsi,rdi,r12,r13,r14,r15 + USES_XMM xmm6,xmm7,xmm8,xmm9 + COMP_ABI 4 + +;; rdi = hash +;; rsi = data buffer +;; rdx = data buffer length (bytes) +;; rcx = address of SM3 constants + +;; stack structure: +%assign _wj 0 ; W[t+3]:W[t+2]:W[t+1]:W[t] +%assign _hash _wj+sizeof(oword)*2 ; hash pointer +%assign _len _hash+sizeof(qword) ; msg length + +%xdefine MBS_SM3 (64) + + movsxd rdx, edx + + movdqa xT0, oword [rel bswap128] ; swap byte + movdqa xWZZZ, oword [rel wzzz] ; shufle constant + movdqa xBCAST, oword [rel bcast] + movdqa xROL8, oword [rel rol_32_8] + + mov qword [rsp+_hash], rdi ; save hash pointer + mov qword [rsp+_len], rdx ; save msg length + +align IPP_ALIGN_FACTOR +.sm3_loop: + ; read data block (64 bytes) + vmovdqu W0, oword [mPtr+sizeof(oword)*0] + vpshufb W0, W0, xT0 + vmovdqu W1, oword [mPtr+sizeof(oword)*1] + vpshufb W1, W1, xT0 + vmovdqu W2, oword [mPtr+sizeof(oword)*2] + vpshufb W2, W2, xT0 + vmovdqu W3, oword [mPtr+sizeof(oword)*3] + vpshufb W3, W3, xT0 + add mPtr, MBS_SM3 + + mov A, [hPtr+0*sizeof(dword)] ; input hash value + mov B, [hPtr+1*sizeof(dword)] + mov C, [hPtr+2*sizeof(dword)] + mov D, [hPtr+3*sizeof(dword)] + mov E, [hPtr+4*sizeof(dword)] + mov F, [hPtr+5*sizeof(dword)] + mov G, [hPtr+6*sizeof(dword)] + mov H, [hPtr+7*sizeof(dword)] + + + vmovdqa oword [rsp+_wj], W0 + vpxor xT0, W0, W1 + vmovdqa oword [rsp+_wj+sizeof(oword)], xT0 + QSCHED_W_QROUND_00_15 0, A,B,C,D,E,F,G,H + + vmovdqa oword [rsp+_wj], W0 + vpxor xT0, W0, W1 + vmovdqa oword [rsp+_wj+sizeof(oword)], xT0 + QSCHED_W_QROUND_00_15 4, E,F,G,H,A,B,C,D + + vmovdqa oword [rsp+_wj], W0 + vpxor xT0, W0, W1 + vmovdqa oword [rsp+_wj+sizeof(oword)], xT0 + QSCHED_W_QROUND_00_15 8, A,B,C,D,E,F,G,H + + vmovdqa oword [rsp+_wj], W0 + vpxor xT0, W0, W1 + vmovdqa oword [rsp+_wj+sizeof(oword)], xT0 + QSCHED_W_QROUND_00_15 12, E,F,G,H,A,B,C,D + + vmovdqa oword [rsp+_wj], W0 + vpxor xT0, W0, W1 + vmovdqa oword [rsp+_wj+sizeof(oword)], xT0 + QSCHED_W_QROUND_16_63 16, A,B,C,D,E,F,G,H + + vmovdqa oword [rsp+_wj], W0 + vpxor xT0, W0, W1 + vmovdqa oword [rsp+_wj+sizeof(oword)], xT0 + QSCHED_W_QROUND_16_63 20, E,F,G,H,A,B,C,D + + vmovdqa oword [rsp+_wj], W0 + vpxor xT0, W0, W1 + vmovdqa oword [rsp+_wj+sizeof(oword)], xT0 + QSCHED_W_QROUND_16_63 24, A,B,C,D,E,F,G,H + + vmovdqa oword [rsp+_wj], W0 + vpxor xT0, W0, W1 + vmovdqa oword [rsp+_wj+sizeof(oword)], xT0 + QSCHED_W_QROUND_16_63 28, E,F,G,H,A,B,C,D + + vmovdqa oword [rsp+_wj], W0 + vpxor xT0, W0, W1 + vmovdqa oword [rsp+_wj+sizeof(oword)], xT0 + QSCHED_W_QROUND_16_63 32, A,B,C,D,E,F,G,H + + vmovdqa oword [rsp+_wj], W0 + vpxor xT0, W0, W1 + vmovdqa oword [rsp+_wj+sizeof(oword)], xT0 + QSCHED_W_QROUND_16_63 36, E,F,G,H,A,B,C,D + + vmovdqa oword [rsp+_wj], W0 + vpxor xT0, W0, W1 + vmovdqa oword [rsp+_wj+sizeof(oword)], xT0 + QSCHED_W_QROUND_16_63 40, A,B,C,D,E,F,G,H + + vmovdqa oword [rsp+_wj], W0 + vpxor xT0, W0, W1 + vmovdqa oword [rsp+_wj+sizeof(oword)], xT0 + QSCHED_W_QROUND_16_63 44, E,F,G,H,A,B,C,D + + vmovdqa oword [rsp+_wj], W0 + vpxor xT0, W0, W1 + vmovdqa oword [rsp+_wj+sizeof(oword)], xT0 + QSCHED_W_QROUND_16_63 48, A,B,C,D,E,F,G,H + + vmovdqa oword [rsp+_wj], W0 + vpxor xT0, W0, W1 + vmovdqa oword [rsp+_wj+sizeof(oword)], xT0 + ROTATE_W + ROUND_16_63 52, E,F,G,H,A,B,C,D + ROUND_16_63 53, D,E,F,G,H,A,B,C + ROUND_16_63 54, C,D,E,F,G,H,A,B + ROUND_16_63 55, B,C,D,E,F,G,H,A + + vmovdqa oword [rsp+_wj], W0 + vpxor xT0, W0, W1 + vmovdqa oword [rsp+_wj+sizeof(oword)], xT0 + ROTATE_W + ROUND_16_63 56, A,B,C,D,E,F,G,H + ROUND_16_63 57, H,A,B,C,D,E,F,G + ROUND_16_63 58, G,H,A,B,C,D,E,F + ROUND_16_63 59, F,G,H,A,B,C,D,E + + vmovdqa oword [rsp+_wj], W0 + vpxor xT0, W0, W1 + vmovdqa oword [rsp+_wj+sizeof(oword)], xT0 + ROTATE_W + ROUND_16_63 60, E,F,G,H,A,B,C,D + ROUND_16_63 61, D,E,F,G,H,A,B,C + ROUND_16_63 62, C,D,E,F,G,H,A,B + ROUND_16_63 63, B,C,D,E,F,G,H,A + + mov hPtr, qword [rsp+_hash] ; restore hash pointer + + xor [hPtr+0*sizeof(dword)], A ; update hash + xor [hPtr+1*sizeof(dword)], B + xor [hPtr+2*sizeof(dword)], C + xor [hPtr+3*sizeof(dword)], D + xor [hPtr+4*sizeof(dword)], E + xor [hPtr+5*sizeof(dword)], F + xor [hPtr+6*sizeof(dword)], G + xor [hPtr+7*sizeof(dword)], H + + movdqa xT0, oword [rel bswap128] ; swap byte + + sub qword [rsp+_len], MBS_SM3 + cmp qword [rsp+_len], MBS_SM3 + jge .sm3_loop + + REST_XMM + REST_GPR + ret +ENDFUNC UpdateSM3 + +%endif ;; _IPP32E >= _IPP32E_E9 +%endif ;; _ENABLE_ALG_SM3_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsm3u8as.asm b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsm3u8as.asm new file mode 100644 index 0000000..813501c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpsm3u8as.asm @@ -0,0 +1,364 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; +; Purpose: Cryptography Primitive. +; Message block processing according to SM3 +; +; Content: +; UpdateSM3 +; + +%include "asmdefs.inc" +%include "ia_32e.inc" +%include "pcpvariant.inc" + +%if (_ENABLE_ALG_SM3_) +%if ((_IPP32E >= _IPP32E_U8 ) && (_IPP32E < _IPP32E_E9 )) || (_IPP32E == _IPP32E_N8 ) + +%xdefine a r8d +%xdefine b r9d +%xdefine c r10d +%xdefine d r11d +%xdefine e r12d +%xdefine f r13d +%xdefine g r14d +%xdefine h r15d + +%xdefine hPtr rdi +%xdefine mPtr rsi +%xdefine kPtr rcx + +%xdefine t0 eax +%xdefine t1 ebx +%xdefine t2 ebp +%xdefine t3 edx + +%xdefine ctr hPtr + +%if (_IPP32E >= _IPP32E_U8 ) + +%macro ROTL 2.nolist + %xdefine %%x %1 + %xdefine %%n %2 + + shld %%x,%%x, %%n +%endmacro + +%else +%macro ROTL 2.nolist + %xdefine %%x %1 + %xdefine %%n %2 + + rol %%x, %%n +%endmacro + +%endif + +%macro ROUND_00_15 9.nolist + %xdefine %%a %1 + %xdefine %%b %2 + %xdefine %%c %3 + %xdefine %%d %4 + %xdefine %%e %5 + %xdefine %%f %6 + %xdefine %%g %7 + %xdefine %%h %8 + %xdefine %%i %9 + + mov t0, %%a + mov t1, [kPtr+ctr*sizeof(dword)+%%i*sizeof(dword)] + ROTL t0, 12 ; t0 = t + add t1, t0 + add t1, %%e + ROTL t1, 7 ; t1 = SS1 + xor t0, t1 ; t0 = SS2 + + mov t2, %%b + mov t3, %%f + + xor t2, %%c + xor t3, %%g + + xor t2, %%a ; t2 = FF + xor t3, %%e ; t3 = GG + + add t0, t2 ; t0 = SS2 + FF + add t1, t3 ; t1 = SS1 + GG + + add t0, %%d ; t0 = SS2 + FF + d + add t1, %%h ; t1 = SS1 + GG + h + + mov t2, [rsp+ctr*sizeof(dword)+%%i*sizeof(dword)] ; t2 = w[i] + add t1, t2 ; t1 = TT2 + xor t2, [rsp+ctr*sizeof(dword)+(%%i+4)*sizeof(dword)] ; t2 = w[i] ^ w[i+4] + + add t0, t2 ; t0 = TT1 + + ROTL %%b, 9 + ROTL %%f, 19 + mov %%h, t0 + + mov %%d, t1 + ROTL %%d, 8 + xor %%d, t1 + ROTL %%d, 9 + xor %%d, t1 +%endmacro + +%macro ROUND_16_63 9.nolist + %xdefine %%a %1 + %xdefine %%b %2 + %xdefine %%c %3 + %xdefine %%d %4 + %xdefine %%e %5 + %xdefine %%f %6 + %xdefine %%g %7 + %xdefine %%h %8 + %xdefine %%i %9 + + mov t0, %%a + mov t1, [kPtr+ctr*sizeof(dword)+%%i*sizeof(dword)] + ROTL t0, 12 + add t1, t0 + add t1, %%e + ROTL t1, 7 ;; t1 = SS1 + xor t0, t1 ;; t0 = SS2 + + mov t3, %%b + mov t2, %%b + and t3, %%c + xor t2, %%c + and t2, %%a + add t2, t3 + + mov t3, %%f + xor t3, %%g + and t3, %%e + xor t3, %%g + + add t0, t2 ;; t0 = SS2 + FF + add t1, t3 ;; t1 = SS1 + GG + + add t0, %%d ;; t0 = SS2 + FF + d + add t1, %%h ;; t1 = SS1 + GG + h + + mov t2, [rsp+ctr*sizeof(dword)+%%i*sizeof(dword)] ;; t2 = w[i] + add t1, t2 ;; t1 = TT2 + xor t2, [rsp+ctr*sizeof(dword)+(%%i+4)*sizeof(dword)] ;; t2 = w[i] ^ w[i+4] + + add t0, t2 ;; t0 = TT1 + + ROTL %%b, 9 + ROTL %%f, 19 + mov %%h, t0 + + mov %%d, t1 + ROTL %%d, 8 + xor %%d, t1 + ROTL %%d, 9 + xor %%d, t1 +%endmacro + +%macro SCHED_16_67 1.nolist + %xdefine %%i %1 + + mov t2, [rsp+ctr*sizeof(dword)+(%%i-3)*sizeof(dword)] ;; t2 = w[i-3] + ROTL t2, 15 ;; t2 = rol(t2,15) + xor t2, [rsp+ctr*sizeof(dword)+(%%i-9)*sizeof(dword)] ;; t2 ^= w[i-9] + xor t2, [rsp+ctr*sizeof(dword)+(%%i-16)*sizeof(dword)] ;; t2 ^= w[i-16] + + mov t3, t2 + ROTL t3, 8 + xor t3, t2 + ROTL t3, 15 + xor t3, t2 ;; t3 = P1 + + mov t2, [rsp+ctr*sizeof(dword)+(%%i-13)*sizeof(dword)] + xor t3, [rsp+ctr*sizeof(dword)+(%%i-6)*sizeof(dword)] + ROTL t2, 7 + xor t3, t2 + mov [rsp+ctr*sizeof(dword)+%%i*sizeof(dword)], t3 +%endmacro + +segment .text align=IPP_ALIGN_FACTOR + +align IPP_ALIGN_FACTOR + +bswap128 DB 3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12 + +;******************************************************************** +;* void UpdateSM3(uint32_t hash[8], +; const uint32_t msg[16], int msgLen, +; const uint32_t* K_SM3) +;******************************************************************** +align IPP_ALIGN_FACTOR +IPPASM UpdateSM3,PUBLIC +%assign LOCAL_FRAME 68*sizeof(dword)+3*sizeof(qword) + USES_GPR rbp,rbx,rsi,rdi,r12,r13,r14,r15 + USES_XMM + COMP_ABI 4 + +;; rdi = hash +;; rsi = data buffer +;; rdx = data buffer length (bytes) +;; rcx = address of SM3 constants + +;; stack structure: +%assign _w 0 ; msg extension W0, ..., W67 +%assign _hash _w+68*sizeof(dword) ; hash +%assign _msg _hash+sizeof(qword) ; msg pointer +%assign _len _msg+sizeof(qword) ; msg length + +%xdefine MBS_SM3 (64) + + movsxd rdx, edx + + mov qword [rsp+_hash], hPtr ; save hash pointer + mov qword [rsp+_len], rdx ; save msg length + + mov a, [hPtr+0*sizeof(dword)] ; input hash value + mov b, [hPtr+1*sizeof(dword)] + mov c, [hPtr+2*sizeof(dword)] + mov d, [hPtr+3*sizeof(dword)] + mov e, [hPtr+4*sizeof(dword)] + mov f, [hPtr+5*sizeof(dword)] + mov g, [hPtr+6*sizeof(dword)] + mov h, [hPtr+7*sizeof(dword)] + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; process data block +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +align IPP_ALIGN_FACTOR +.main_loop: + xor ctr, ctr ; round + + movdqu xmm0, oword [mPtr+0*sizeof(oword)] + movdqu xmm1, oword [mPtr+1*sizeof(oword)] + movdqu xmm2, oword [mPtr+2*sizeof(oword)] + movdqu xmm3, oword [mPtr+3*sizeof(oword)] + add mPtr, MBS_SM3 + mov qword [rsp+_msg], mPtr ; save msg pointer + + pshufb xmm0, [rel bswap128] + pshufb xmm1, [rel bswap128] + pshufb xmm2, [rel bswap128] + pshufb xmm3, [rel bswap128] + + movdqu oword [rsp+0*sizeof(oword)], xmm0 + movdqu oword [rsp+1*sizeof(oword)], xmm1 + movdqu oword [rsp+2*sizeof(oword)], xmm2 + movdqu oword [rsp+3*sizeof(oword)], xmm3 + +align IPP_ALIGN_FACTOR +.rounds_00_15: + SCHED_16_67 16 + ROUND_00_15 a,b,c,d,e,f,g,h,0 + SCHED_16_67 17 + ROUND_00_15 h,a,b,c,d,e,f,g,1 + SCHED_16_67 18 + ROUND_00_15 g,h,a,b,c,d,e,f,2 + SCHED_16_67 19 + ROUND_00_15 f,g,h,a,b,c,d,e,3 + SCHED_16_67 20 + ROUND_00_15 e,f,g,h,a,b,c,d,4 + SCHED_16_67 21 + ROUND_00_15 d,e,f,g,h,a,b,c,5 + SCHED_16_67 22 + ROUND_00_15 c,d,e,f,g,h,a,b,6 + SCHED_16_67 23 + ROUND_00_15 b,c,d,e,f,g,h,a,7 + add ctr, 8 + cmp ctr, 16 + jne .rounds_00_15 + +align IPP_ALIGN_FACTOR +.rounds_16_47: + SCHED_16_67 16 + ROUND_16_63 a,b,c,d,e,f,g,h,0 + SCHED_16_67 17 + ROUND_16_63 h,a,b,c,d,e,f,g,1 + SCHED_16_67 18 + ROUND_16_63 g,h,a,b,c,d,e,f,2 + SCHED_16_67 19 + ROUND_16_63 f,g,h,a,b,c,d,e,3 + SCHED_16_67 20 + ROUND_16_63 e,f,g,h,a,b,c,d,4 + SCHED_16_67 21 + ROUND_16_63 d,e,f,g,h,a,b,c,5 + SCHED_16_67 22 + ROUND_16_63 c,d,e,f,g,h,a,b,6 + SCHED_16_67 23 + ROUND_16_63 b,c,d,e,f,g,h,a,7 + add ctr, 8 + cmp ctr, 48 + jne .rounds_16_47 + + SCHED_16_67 16 + SCHED_16_67 17 + SCHED_16_67 18 + SCHED_16_67 19 + +align IPP_ALIGN_FACTOR +.rounds_48_63: + ROUND_16_63 a,b,c,d,e,f,g,h,0 + ROUND_16_63 h,a,b,c,d,e,f,g,1 + ROUND_16_63 g,h,a,b,c,d,e,f,2 + ROUND_16_63 f,g,h,a,b,c,d,e,3 + ROUND_16_63 e,f,g,h,a,b,c,d,4 + ROUND_16_63 d,e,f,g,h,a,b,c,5 + ROUND_16_63 c,d,e,f,g,h,a,b,6 + ROUND_16_63 b,c,d,e,f,g,h,a,7 + add ctr, 8 + cmp ctr, 64 + jne .rounds_48_63 + + mov hPtr, [rsp+_hash] + mov mPtr, [rsp+_msg] + + xor a, [hPtr+0*sizeof(dword)] + xor b, [hPtr+1*sizeof(dword)] + xor c, [hPtr+2*sizeof(dword)] + xor d, [hPtr+3*sizeof(dword)] + xor e, [hPtr+4*sizeof(dword)] + xor f, [hPtr+5*sizeof(dword)] + xor g, [hPtr+6*sizeof(dword)] + xor h, [hPtr+7*sizeof(dword)] + + mov [hPtr+0*sizeof(dword)], a + mov [hPtr+1*sizeof(dword)], b + mov [hPtr+2*sizeof(dword)], c + mov [hPtr+3*sizeof(dword)], d + mov [hPtr+4*sizeof(dword)], e + mov [hPtr+5*sizeof(dword)], f + mov [hPtr+6*sizeof(dword)], g + mov [hPtr+7*sizeof(dword)], h + + sub qword [rsp+_len], MBS_SM3 + jne .main_loop + + REST_XMM + REST_GPR + ret +ENDFUNC UpdateSM3 + +%endif ;; ((_IPP32E >= _IPP32E_U8 ) AND (_IPP32E < _IPP32E_E9 )) OR (_IPP32E == _IPP32E_N8 ) +%endif ;; _ENABLE_ALG_SM3_ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpvariant.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpvariant.inc new file mode 100644 index 0000000..a1685e1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpvariant.inc @@ -0,0 +1,202 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; Intel(R) Integrated Performance Primitives +; Cryptographic Primitives (ippcp) +; +; Purpose: +; Define ippCP variant +; +; do not changes in definitions below! +; + +;; +;; modes of the feature +;; +%assign _FEATURE_OFF_ 0 ;; feature is OFF +%assign _FEATURE_ON_ 1 ;; feature is ON +%assign _FEATURE_TICKTOCK_ 2 ;; dectect is feature OFF/ON + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; %define _XMM7560_ 1 +%ifdef _XMM7560_ +%include "pcpvariant_xmm7560.inc" +%endif + +; %define _TXT_ACM_ 1 +%ifdef _TXT_ACM_ +%include "pcpvariant_txt_acm.inc" +%endif +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; +;; it possible to force use of C-version of some implementtaions +;; instead of ASM one +;; +%ifndef _USE_C_cpAdd_BNU_ + %assign _USE_C_cpAdd_BNU_ _FEATURE_OFF_ +%endif + +%ifndef _USE_C_cpSub_BNU_ + %assign _USE_C_cpSub_BNU_ _FEATURE_OFF_ +%endif + +%ifndef _USE_C_cpInc_BNU_ + %assign _USE_C_cpInc_BNU_ _FEATURE_OFF_ +%endif + +%ifndef _USE_C_cpAddMulDgt_BNU_ + %assign _USE_C_cpAddMulDgt_BNU_ _FEATURE_OFF_ +%endif + +%ifndef _USE_C_cpMulAdc_BNU_school_ + %assign _USE_C_cpMulAdc_BNU_school_ _FEATURE_OFF_ +%endif + +%ifndef _USE_C_cpSqrAdc_BNU_school_ + %assign _USE_C_cpMulSqr_BNU_school_ _FEATURE_OFF_ +%endif + +%ifndef _USE_C_cpMontRedAdc_BNU_ + %assign _USE_C_cpMontRedAdc_BNU_ _FEATURE_OFF_ +%endif + +;; +;; set _AES_NI_ENABLING_ +;; +%ifdef _IPP_AES_NI_ + %if (_IPP_AES_NI_ == 0) + %assign _AES_NI_ENABLING_ _FEATURE_OFF_ + %elif (_IPP_AES_NI_ == 1) + %assign _AES_NI_ENABLING_ _FEATURE_ON_ + %else + %error + %endif +%else + %if (_IPP32E >= _IPP32E_Y8) + %assign _AES_NI_ENABLING_ _FEATURE_TICKTOCK_ + %else + %assign _AES_NI_ENABLING_ _FEATURE_OFF_ + %endif +%endif + +;; +;; if there is no outside assignment +;; set _SHA_NI_ENABLING_ based on CPU specification +;; +%ifndef _SHA_NI_ENABLING_ + %if (_IPP32E >= _IPP32E_Y8 ) + %assign _SHA_NI_ENABLING_ _FEATURE_TICKTOCK_ + %else + %assign _SHA_NI_ENABLING_ _FEATURE_OFF_ + %endif +%endif + +;; +;; set _ADCOX_NI_ENABLING_ +;; +%ifdef _IPP_ADCX_NI_ + %if (_IPP_ADCX_NI_ == 0) + %assign _ADCOX_NI_ENABLING_ _FEATURE_OFF_ + %elif (_IPP_ADCX_NI_ == 1) + %assign _ADCOX_NI_ENABLING_ _FEATURE_ON_ + %else + %error + %endif +%else + %if (_IPP32E >= _IPP32E_L9) + %assign _ADCOX_NI_ENABLING_ _FEATURE_TICKTOCK_ + %else + %assign _ADCOX_NI_ENABLING_ _FEATURE_OFF_ + %endif +%endif + + +;; +;; select Hash algorithm +;; +%ifndef _DISABLE_ALG_SHA1_ + %assign _ENABLE_ALG_SHA1_ _FEATURE_ON_ ;; SHA1 on +%else + %assign _ENABLE_ALG_SHA1_ _FEATURE_OFF_ ;; SHA1 on +%endif + +%ifndef _DISABLE_ALG_SHA256_ + %assign _ENABLE_ALG_SHA256_ _FEATURE_ON_ ;; SHA256 on +%else + %assign _ENABLE_ALG_SHA256_ _FEATURE_OFF_ ;; SHA256 off +%endif + +%ifndef _DISABLE_ALG_SHA521_ + %assign _ENABLE_ALG_SHA512_ _FEATURE_ON_ ;; SHA512 on +%else + %assign _ENABLE_ALG_SHA512_ _FEATURE_OFF_ ;; SHA512 off +%endif + +%ifndef _DISABLE_ALG_MD5_ + %assign _ENABLE_ALG_MD5_ _FEATURE_ON_ ;; MD5 on +%else + %assign _ENABLE_ALG_MD5_ _FEATURE_OFF_ ;; MD5 off +%endif + +%ifndef _DISABLE_ALG_SM3_ + %assign _ENABLE_ALG_SM3_ _FEATURE_ON_ ;; SM3 on +%else + %assign _ENABLE_ALG_SM3_ _FEATURE_OFF_ ;; SM3 off +%endif + +;; +;; BN arithmetic +;; +%assign _ENABLE_KARATSUBA_ _FEATURE_OFF_ ;; not use Karatsuba method for multiplication + +;; +;; EC specific +;; +%assign _ECP_IMPL_NONE_ 0 +%assign _ECP_IMPL_ARBIRTRARY_ 1 +%assign _ECP_IMPL_SPECIFIC_ 2 +%assign _ECP_IMPL_MFM_ 3 + +%ifndef _ECP_128_ + %assign _ECP_128_ _ECP_IMPL_SPECIFIC_ +%endif + +%ifndef _ECP_192_ + %assign _ECP_192_ _ECP_IMPL_MFM_ +%endif + +%ifndef _ECP_224_ + %assign _ECP_224_ _ECP_IMPL_MFM_ +%endif + +%ifndef _ECP_256_ + %assign _ECP_256_ _ECP_IMPL_MFM_ +%endif + +%ifndef _ECP_384_ + %assign _ECP_384_ _ECP_IMPL_MFM_ +%endif + +%ifndef _ECP_521_ + %assign _ECP_521_ _ECP_IMPL_MFM_ +%endif + +%ifndef _ECP_SM2_ + %assign _ECP_SM2_ _ECP_IMPL_MFM_ +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpvariant_txt_acm.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpvariant_txt_acm.inc new file mode 100644 index 0000000..9045fd4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/pcpvariant_txt_acm.inc @@ -0,0 +1,40 @@ +;=============================================================================== +; Copyright (C) 2015 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; +; Intel(R) Integrated Performance Primitives +; Cryptographic Primitives (ippcp) +; +; Purpose: +; Update standard ippCP variant +; +; do not changes in definitions below! +; + +%ifdef _TXT_ACM_ + +;; +;; HASH algs outside settings +;; +%assign _SHA_NI_ENABLING_ _FEATURE_TICKTOCK_ + +;; +;; select Hash algorithm +;; +; %assign _ENABLE_ALG_MD5_ _FEATURE_OFF_ + +%endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/reg_sizes.inc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/reg_sizes.inc new file mode 100644 index 0000000..4297963 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/asm_intel64/reg_sizes.inc @@ -0,0 +1,290 @@ +;=============================================================================== +; Copyright (C) 2020 Intel Corporation +; +; Licensed under the Apache License, Version 2.0 (the 'License'); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, +; software distributed under the License is distributed on an 'AS IS' BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions +; and limitations under the License. +; +;=============================================================================== + +; define d and w variants for registers + +%ifndef _REG_SIZES_ASM_ +%define _REG_SIZES_ASM_ + +%define raxd eax +%define raxw ax +%define raxb al + +%define rbxd ebx +%define rbxw bx +%define rbxb bl + +%define rcxd ecx +%define rcxw cx +%define rcxb cl + +%define rdxd edx +%define rdxw dx +%define rdxb dl + +%define rsid esi +%define rsiw si +%define rsib sil + +%define rdid edi +%define rdiw di +%define rdib dil + +%define rbpd ebp +%define rbpw bp +%define rbpb bpl + +%define zmm0x xmm0 +%define zmm1x xmm1 +%define zmm2x xmm2 +%define zmm3x xmm3 +%define zmm4x xmm4 +%define zmm5x xmm5 +%define zmm6x xmm6 +%define zmm7x xmm7 +%define zmm8x xmm8 +%define zmm9x xmm9 +%define zmm10x xmm10 +%define zmm11x xmm11 +%define zmm12x xmm12 +%define zmm13x xmm13 +%define zmm14x xmm14 +%define zmm15x xmm15 +%define zmm16x xmm16 +%define zmm17x xmm17 +%define zmm18x xmm18 +%define zmm19x xmm19 +%define zmm20x xmm20 +%define zmm21x xmm21 +%define zmm22x xmm22 +%define zmm23x xmm23 +%define zmm24x xmm24 +%define zmm25x xmm25 +%define zmm26x xmm26 +%define zmm27x xmm27 +%define zmm28x xmm28 +%define zmm29x xmm29 +%define zmm30x xmm30 +%define zmm31x xmm31 + +%define ymm0x xmm0 +%define ymm1x xmm1 +%define ymm2x xmm2 +%define ymm3x xmm3 +%define ymm4x xmm4 +%define ymm5x xmm5 +%define ymm6x xmm6 +%define ymm7x xmm7 +%define ymm8x xmm8 +%define ymm9x xmm9 +%define ymm10x xmm10 +%define ymm11x xmm11 +%define ymm12x xmm12 +%define ymm13x xmm13 +%define ymm14x xmm14 +%define ymm15x xmm15 +%define ymm16x xmm16 +%define ymm17x xmm17 +%define ymm18x xmm18 +%define ymm19x xmm19 +%define ymm20x xmm20 +%define ymm21x xmm21 +%define ymm22x xmm22 +%define ymm23x xmm23 +%define ymm24x xmm24 +%define ymm25x xmm25 +%define ymm26x xmm26 +%define ymm27x xmm27 +%define ymm28x xmm28 +%define ymm29x xmm29 +%define ymm30x xmm30 +%define ymm31x xmm31 + +%define xmm0x xmm0 +%define xmm1x xmm1 +%define xmm2x xmm2 +%define xmm3x xmm3 +%define xmm4x xmm4 +%define xmm5x xmm5 +%define xmm6x xmm6 +%define xmm7x xmm7 +%define xmm8x xmm8 +%define xmm9x xmm9 +%define xmm10x xmm10 +%define xmm11x xmm11 +%define xmm12x xmm12 +%define xmm13x xmm13 +%define xmm14x xmm14 +%define xmm15x xmm15 +%define xmm16x xmm16 +%define xmm17x xmm17 +%define xmm18x xmm18 +%define xmm19x xmm19 +%define xmm20x xmm20 +%define xmm21x xmm21 +%define xmm22x xmm22 +%define xmm23x xmm23 +%define xmm24x xmm24 +%define xmm25x xmm25 +%define xmm26x xmm26 +%define xmm27x xmm27 +%define xmm28x xmm28 +%define xmm29x xmm29 +%define xmm30x xmm30 +%define xmm31x xmm31 + +%define zmm0y ymm0 +%define zmm1y ymm1 +%define zmm2y ymm2 +%define zmm3y ymm3 +%define zmm4y ymm4 +%define zmm5y ymm5 +%define zmm6y ymm6 +%define zmm7y ymm7 +%define zmm8y ymm8 +%define zmm9y ymm9 +%define zmm10y ymm10 +%define zmm11y ymm11 +%define zmm12y ymm12 +%define zmm13y ymm13 +%define zmm14y ymm14 +%define zmm15y ymm15 +%define zmm16y ymm16 +%define zmm17y ymm17 +%define zmm18y ymm18 +%define zmm19y ymm19 +%define zmm20y ymm20 +%define zmm21y ymm21 +%define zmm22y ymm22 +%define zmm23y ymm23 +%define zmm24y ymm24 +%define zmm25y ymm25 +%define zmm26y ymm26 +%define zmm27y ymm27 +%define zmm28y ymm28 +%define zmm29y ymm29 +%define zmm30y ymm30 +%define zmm31y ymm31 + +%define xmm0y ymm0 +%define xmm1y ymm1 +%define xmm2y ymm2 +%define xmm3y ymm3 +%define xmm4y ymm4 +%define xmm5y ymm5 +%define xmm6y ymm6 +%define xmm7y ymm7 +%define xmm8y ymm8 +%define xmm9y ymm9 +%define xmm10y ymm10 +%define xmm11y ymm11 +%define xmm12y ymm12 +%define xmm13y ymm13 +%define xmm14y ymm14 +%define xmm15y ymm15 +%define xmm16y ymm16 +%define xmm17y ymm17 +%define xmm18y ymm18 +%define xmm19y ymm19 +%define xmm20y ymm20 +%define xmm21y ymm21 +%define xmm22y ymm22 +%define xmm23y ymm23 +%define xmm24y ymm24 +%define xmm25y ymm25 +%define xmm26y ymm26 +%define xmm27y ymm27 +%define xmm28y ymm28 +%define xmm29y ymm29 +%define xmm30y ymm30 +%define xmm31y ymm31 + +%define xmm0z zmm0 +%define xmm1z zmm1 +%define xmm2z zmm2 +%define xmm3z zmm3 +%define xmm4z zmm4 +%define xmm5z zmm5 +%define xmm6z zmm6 +%define xmm7z zmm7 +%define xmm8z zmm8 +%define xmm9z zmm9 +%define xmm10z zmm10 +%define xmm11z zmm11 +%define xmm12z zmm12 +%define xmm13z zmm13 +%define xmm14z zmm14 +%define xmm15z zmm15 +%define xmm16z zmm16 +%define xmm17z zmm17 +%define xmm18z zmm18 +%define xmm19z zmm19 +%define xmm20z zmm20 +%define xmm21z zmm21 +%define xmm22z zmm22 +%define xmm23z zmm23 +%define xmm24z zmm24 +%define xmm25z zmm25 +%define xmm26z zmm26 +%define xmm27z zmm27 +%define xmm28z zmm28 +%define xmm29z zmm29 +%define xmm30z zmm30 +%define xmm31z zmm31 + +%define ymm0z zmm0 +%define ymm1z zmm1 +%define ymm2z zmm2 +%define ymm3z zmm3 +%define ymm4z zmm4 +%define ymm5z zmm5 +%define ymm6z zmm6 +%define ymm7z zmm7 +%define ymm8z zmm8 +%define ymm9z zmm9 +%define ymm10z zmm10 +%define ymm11z zmm11 +%define ymm12z zmm12 +%define ymm13z zmm13 +%define ymm14z zmm14 +%define ymm15z zmm15 +%define ymm16z zmm16 +%define ymm17z zmm17 +%define ymm18z zmm18 +%define ymm19z zmm19 +%define ymm20z zmm20 +%define ymm21z zmm21 +%define ymm22z zmm22 +%define ymm23z zmm23 +%define ymm24z zmm24 +%define ymm25z zmm25 +%define ymm26z zmm26 +%define ymm27z zmm27 +%define ymm28z zmm28 +%define ymm29z zmm29 +%define ymm30z zmm30 +%define ymm31z zmm31 + +%define DWORD(reg) reg %+ d +%define WORD(reg) reg %+ w +%define BYTE(reg) reg %+ b + +%define XWORD(reg) reg %+ x +%define YWORD(reg) reg %+ y +%define ZWORD(reg) reg %+ z + +%endif ;; _REG_SIZES_ASM_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/cpinit.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/cpinit.c new file mode 100644 index 0000000..1cde86d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/cpinit.c @@ -0,0 +1,475 @@ +/******************************************************************************* +* Copyright (C) 2001 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +// +// Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) +// + +#include "owndefs.h" +#include "ippcpdefs.h" +#include "ippcp.h" +#include "dispatcher.h" + +#if defined( _IPP_DATA ) + +static Ipp64u cpFeatures = 0; +static Ipp64u cpFeaturesMask = 0; + +static int cpGetFeatures( Ipp64u* pFeaturesMask ); + +extern IPP_OWN_DECL (void, cpGetReg, ( int* buf, int valEAX, int valECX )) +extern IPP_OWN_DECL (int, cp_is_avx_extension, ( void )) +extern IPP_OWN_DECL (int, cp_is_avx512_extension, ( void )) +extern IPP_OWN_DECL (int, cp_issue_avx512_instruction, ( void )) + +IppStatus owncpSetCpuFeaturesAndIdx( Ipp64u cpuFeatures, int* index ); + +IPPFUN( Ipp64u, ippcpGetEnabledCpuFeatures, ( void )) +{ + return cpFeaturesMask; +} + +/*===================================================================*/ +IPPFUN( IppStatus, ippcpGetCpuFeatures, ( Ipp64u* pFeaturesMask )) +{ + IPP_BAD_PTR1_RET( pFeaturesMask ) + { + if( 0 != cpFeatures){ + *pFeaturesMask = cpFeatures;// & cpFeaturesMask; + } else { + int ret = cpGetFeatures( pFeaturesMask ); + if( !ret ) return ippStsNotSupportedCpu; + } + return ippStsNoErr; + } +} + +/*===================================================================*/ + +int cpGetFeature( Ipp64u Feature ) +{ + // We provide users the ability to build their own custom build of 1cpu Intel® IPP Cryptography library + // and turn CPU features on at compile-time if they are sure these features available on the target systems. + #if (!defined(_MERGED_BLD) && defined(IPPCP_CUSTOM_BUILD)) + int if_feature_enabled = ((IPP_CUSTOM_ENABLED_FEATURES & Feature) == Feature); + return if_feature_enabled; + #else + if( 0 == cpFeaturesMask ) { + Ipp64u loc_features; + // set up cpFeaturesMask and cpFeatures for the proper work of tick-tok in 1cpu libraries + ippcpGetCpuFeatures(&loc_features); + } + + if((cpFeaturesMask & Feature) == Feature) { + return 1; + } else { + return 0; + } + #endif +} + +/*===================================================================*/ +#define BIT00 0x00000001 +#define BIT01 0x00000002 +#define BIT02 0x00000004 +#define BIT03 0x00000008 +#define BIT04 0x00000010 +#define BIT05 0x00000020 +#define BIT06 0x00000040 +#define BIT07 0x00000080 +#define BIT08 0x00000100 +#define BIT09 0x00000200 +#define BIT10 0x00000400 +#define BIT11 0x00000800 +#define BIT12 0x00001000 +#define BIT13 0x00002000 +#define BIT14 0x00004000 +#define BIT15 0x00008000 +#define BIT16 0x00010000 +#define BIT17 0x00020000 +#define BIT18 0x00040000 +#define BIT19 0x00080000 +#define BIT20 0x00100000 +#define BIT21 0x00200000 +#define BIT22 0x00400000 +#define BIT23 0x00800000 +#define BIT24 0x01000000 +#define BIT25 0x02000000 +#define BIT26 0x04000000 +#define BIT27 0x08000000 +#define BIT28 0x10000000 +#define BIT29 0x20000000 +#define BIT30 0x40000000 +#define BIT31 0x80000000 + + +static int cpGetFeatures( Ipp64u* pFeaturesMask ) +{ + Ipp32u buf[4]; + Ipp32u eax_, ebx_, ecx_, edx_, tmp; + Ipp64u mask; + int flgFMA=0, flgINT=0, flgGPR=0; // for avx2 + Ipp32u idBaseMax, idExtdMax; + + cpGetReg((int*)buf, 0, 0); //get max value for basic info. + idBaseMax = buf[0]; + cpGetReg((int*)buf, (Ipp32s)0x80000000, 0); //get max value for extended info. + idExtdMax = buf[0]; + + cpGetReg( (int*)buf, 1, 0 ); + eax_ = (Ipp32u)buf[0]; + ecx_ = (Ipp32u)buf[2]; + edx_ = (Ipp32u)buf[3]; + mask = 0; + if( edx_ & BIT23 ) mask |= ippCPUID_MMX; // edx[23] - MMX(TM) Technology + if( edx_ & BIT25 ) mask |= ippCPUID_SSE; // edx[25] - Intel® Streaming SIMD Extensions (Intel® SSE) + if( edx_ & BIT26 ) mask |= ippCPUID_SSE2; // edx[26] - Intel® Streaming SIMD Extensions 2 (Intel® SSE2) + if( ecx_ & BIT00 ) mask |= ippCPUID_SSE3; // ecx[0] - Intel® Streaming SIMD Extensions 3 (Intel® SSE3) (formerly codenamed Prescott) + if( ecx_ & BIT09 ) mask |= ippCPUID_SSSE3; // ecx[9] - Supplemental Streaming SIMD Extensions 3 (SSSE3) (formerly codenamed Merom) + if( ecx_ & BIT22 ) mask |= ippCPUID_MOVBE; // ecx[22] - Intel® instruction MOVBE (Intel Atom® processor) + if( ecx_ & BIT19 ) mask |= ippCPUID_SSE41; // ecx[19] - Intel® Streaming SIMD Extensions 4.1 (Intel® SSE4.1) (formerly codenamed Penryn) + if( ecx_ & BIT20 ) mask |= ippCPUID_SSE42; // ecx[20] - Intel® Streaming SIMD Extensions 4.2 (Intel® SSE4.2) (formerly codenamed Nenalem) + if( ecx_ & BIT28 ) mask |= ippCPUID_AVX; // ecx[28] - Intel® Advanced Vector Extensions (Intel® AVX) (formerly codenamed Sandy Bridge) + if(( ecx_ & 0x18000000 ) == 0x18000000 ){ + tmp = (Ipp32u)cp_is_avx_extension(); + if( tmp & BIT00 ) mask |= ippAVX_ENABLEDBYOS; // Intel® AVX is supported by OS + } + if( ecx_ & BIT25 ) mask |= ippCPUID_AES; // ecx[25] - Intel® AES New Instructions + if( ecx_ & BIT01 ) mask |= ippCPUID_CLMUL; // ecx[1] - Intel® instruction PCLMULQDQ + if( ecx_ & BIT30 ) mask |= ippCPUID_RDRAND; // ecx[30] - Intel® instruction RDRRAND + if( ecx_ & BIT29 ) mask |= ippCPUID_F16C; // ecx[29] - Intel® instruction F16C + // Intel® AVX2 instructions extention: only if 3 features are enabled at once: + // FMA, Intel® AVX 256 int & GPR BMI (bit-manipulation); + if( ecx_ & BIT12 ) flgFMA = 1; else flgFMA = 0; // ecx[12] - FMA 128 & 256 bit + if( idBaseMax >= 7 ){ // get CPUID.eax = 7 + cpGetReg( (int*)buf, 0x7, 0 ); + ebx_ = (Ipp32u)buf[1]; + ecx_ = (Ipp32u)buf[2]; + edx_ = (Ipp32u)buf[3]; + if( ebx_ & BIT05 ) flgINT = 1; + else flgINT = 0; //ebx[5], Intel® Advanced Vector Extensions 2 (Intel® AVX2) (int 256bits) + // ebx[3] - enabled ANDN, BEXTR, BLSI, BLSMK, BLSR, TZCNT + // ebx[8] - enabled BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX, SHRX + if(( ebx_ & BIT03 )&&( ebx_ & BIT08 )) flgGPR = 1; + else flgGPR = 0; // VEX-encoded GPR instructions (GPR BMI) + // Intel® architecture formerly codenamed Broadwell instructions extention + if( ebx_ & BIT19 ) mask |= ippCPUID_ADCOX; // eax[0x7] -->> ebx:: Bit 19: Intel® instructions ADOX/ADCX + if( ebx_ & BIT18 ) mask |= ippCPUID_RDSEED; // eax[0x7] -->> ebx:: Bit 18: Intel® instruction RDSEED + if( ebx_ & BIT29 ) mask |= ippCPUID_SHA; // eax[0x7] -->> ebx:: Bit 29: Intel® Secure Hash Algorithm Extensions + + if( ecx_ & BIT09 ) mask |= ippCPUID_AVX2VAES; // ecx[09] - Intel® Vector AES instruction set + if( ecx_ & BIT10 ) mask |= ippCPUID_AVX2VCLMUL; // ecx[10] - Intel® instruction VPCLMULQDQ + + // Intel® Advanced Vector Extensions 512 (Intel® AVX-512) extention + if( ebx_ & BIT16 ) mask |= ippCPUID_AVX512F; // ebx[16] - Intel® AVX-512 Foundation + if( ebx_ & BIT26 ) mask |= ippCPUID_AVX512PF; // ebx[26] - Intel® AVX-512 Pre Fetch Instructions (PFI) + if( ebx_ & BIT27 ) mask |= ippCPUID_AVX512ER; // ebx[27] - Intel® AVX-512 Exponential and Reciprocal Instructions (ERI) + if( ebx_ & BIT28 ) mask |= ippCPUID_AVX512CD; // ebx[28] - Intel® AVX-512 Conflict Detection + if( ebx_ & BIT17 ) mask |= ippCPUID_AVX512DQ; // ebx[17] - Intel® AVX-512 Dword & Quadword + if( ebx_ & BIT30 ) mask |= ippCPUID_AVX512BW; // ebx[30] - Intel® AVX-512 Byte & Word + if( ebx_ & BIT31 ) mask |= ippCPUID_AVX512VL; // ebx[31] - Intel® AVX-512 Vector Length Extensions (VLE) + if( ecx_ & BIT01 ) mask |= ippCPUID_AVX512VBMI; // ecx[01] - Intel® AVX-512 Vector Bit Manipulation Instructions + if( ecx_ & BIT06 ) mask |= ippCPUID_AVX512VBMI2; // ecx[06] - Intel® AVX-512 Vector Bit Manipulation Instructions 2 + if( edx_ & BIT02 ) mask |= ippCPUID_AVX512_4VNNIW; // edx[02] - Intel® AVX-512 Vector instructions for deep learning enhanced word variable precision + if( edx_ & BIT03 ) mask |= ippCPUID_AVX512_4FMADDPS; // edx[03] - Intel® AVX-512 Vector instructions for deep learning floating-point single precision + // bitwise OR between ippCPUID_MPX & ippCPUID_AVX flags can be used to define that arch is GE than formerly codenamed Skylake + if( ebx_ & BIT14 ) mask |= ippCPUID_MPX; // ebx[14] - Intel® Memory Protection Extensions (Intel® MPX) + if( ebx_ & BIT21 ) mask |= ippCPUID_AVX512IFMA; // ebx[21] - Intel® AVX-512 IFMA PMADD52 + + if (ecx_ & BIT08) mask |= ippCPUID_AVX512GFNI; // test bit ecx[08] + if (ecx_ & BIT09) mask |= ippCPUID_AVX512VAES; // test bit ecx[09] + if (ecx_ & BIT10) mask |= ippCPUID_AVX512VCLMUL; // test bit ecx[10] + + if (mask & ippCPUID_AVX512F) { + /* test if Intel® AVX-512 is supported by OS */ + if (cp_is_avx512_extension()) + mask |= ippAVX512_ENABLEDBYOS; + + #if defined(OSXEM64T) + else { + /* submit some avx512f instructions, returns 0 */ + mask |= cp_issue_avx512_instruction(); + /* and check OS support again */ + if (cp_is_avx512_extension()) + mask |= ippAVX512_ENABLEDBYOS; + } + #endif + } + } + mask = ( flgFMA && flgINT && flgGPR ) ? (mask | ippCPUID_AVX2) : mask; // to separate Intel® AVX2 flags here + + if( idExtdMax >= 0x80000001 ){ // get CPUID.eax=0x80000001 + cpGetReg( (int*)buf, (Ipp32s)0x80000001, 0 ); + ecx_ = (Ipp32u)buf[2]; + // Intel® architecture formerly codenamed Broadwell instructions extention + if( ecx_ & BIT08 ) mask |= ippCPUID_PREFETCHW; // eax[0x80000001] -->> ecx:: Bit 8: Intel® instruction PREFETCHW + } + // Intel® architecture formerly codenamed Knights Corner + if(((( eax_ << 20 ) >> 24 ) ^ 0xb1 ) == 0 ){ + mask = mask | ippCPUID_KNC; + } + cpFeatures = mask; + cpFeaturesMask = mask; /* all CPU features are enabled by default */ + *pFeaturesMask = cpFeatures; + return 1; /* if somebody need to check for cpuid support - do it at the top of function and return 0 if it's not supported */ +} + +int ippcpJumpIndexForMergedLibs = -1; +static int cpthreads_omp_of_n_ipp = 1; + +IPPFUN( int, ippcpGetEnabledNumThreads,( void )) +{ + return cpthreads_omp_of_n_ipp; +} + + +#define AVX3X_FEATURES ( ippCPUID_AVX512F|ippCPUID_AVX512CD|ippCPUID_AVX512VL|ippCPUID_AVX512BW|ippCPUID_AVX512DQ ) +#define AVX3M_FEATURES ( ippCPUID_AVX512F|ippCPUID_AVX512CD|ippCPUID_AVX512PF|ippCPUID_AVX512ER ) +// AVX3X_FEATURES means Intel® Xeon® processor +// AVX3M_FEATURES means Intel® Many Integrated Core Architecture +#define AVX3I_FEATURES ( AVX3X_FEATURES| \ + ippCPUID_SHA|ippCPUID_AVX512VBMI|ippCPUID_AVX512VBMI2| \ + ippCPUID_AVX512IFMA| \ + ippCPUID_AVX512GFNI|ippCPUID_AVX512VAES|ippCPUID_AVX512VCLMUL) + + +IppStatus owncpFeaturesToIdx( Ipp64u* cpuFeatures, int* index ) +{ + IppStatus ownStatus = ippStsNoErr; + Ipp64u mask = 0; + + *index = 0; + + if(( AVX3I_FEATURES == ( *cpuFeatures & AVX3I_FEATURES ))&& + ( ippAVX512_ENABLEDBYOS & cpFeatures )){ /* Intel® architecture formerly codenamed Icelake ia32=S0, x64=K0 */ + mask = AVX3I_MSK; + *index = LIB_AVX3I; + } else + if(( AVX3X_FEATURES == ( *cpuFeatures & AVX3X_FEATURES ))&& + ( ippAVX512_ENABLEDBYOS & cpFeatures )){ /* Intel® architecture formerly codenamed Skylake ia32=S0, x64=K0 */ + mask = AVX3X_MSK; + *index = LIB_AVX3X; + } else + if(( AVX3M_FEATURES == ( *cpuFeatures & AVX3M_FEATURES ))&& + ( ippAVX512_ENABLEDBYOS & cpFeatures )){ /* Intel® architecture formerly codenamed Knights Landing ia32=i0, x64=N0 */ + mask = AVX3M_MSK; + *index = LIB_AVX3M; + } else + if(( ippCPUID_AVX2 == ( *cpuFeatures & ippCPUID_AVX2 ))&& + ( ippAVX_ENABLEDBYOS & cpFeatures )){ /* Intel® architecture formerly codenamed Haswell ia32=H9, x64=L9 */ + mask = AVX2_MSK; + *index = LIB_AVX2; + } else + if(( ippCPUID_AVX == ( *cpuFeatures & ippCPUID_AVX ))&& + ( ippAVX_ENABLEDBYOS & cpFeatures )){ /* Intel® architecture formerly codenamed Sandy Bridge ia32=G9, x64=E9 */ + mask = AVX_MSK; + *index = LIB_AVX; + } else + if( ippCPUID_SSE42 == ( *cpuFeatures & ippCPUID_SSE42 )){ /* Intel® microarchitecture code name Nehalem or Intel® architecture formerly codenamed Westmer = Intel® architecture formerly codenamed Penryn + Intel® SSE4.2 + ?Intel® instruction PCLMULQDQ + ?(Intel® AES New Instructions) + ?(Intel® Secure Hash Algorithm Extensions) */ + mask = SSE42_MSK; /* or new Intel Atom® processor formerly codenamed Silvermont */ + *index = LIB_SSE42; + } else + if( ippCPUID_SSE41 == ( *cpuFeatures & ippCPUID_SSE41 )){ /* Intel® architecture formerly codenamed Penryn ia32=P8, x64=Y8 */ + mask = SSE41_MSK; + *index = LIB_SSE41; + } else + if( ippCPUID_MOVBE == ( *cpuFeatures & ippCPUID_MOVBE )) { /* Intel Atom® processor formerly codenamed Silverthorne ia32=S8, x64=N8 */ + mask = ATOM_MSK; + *index = LIB_ATOM; + } else + if( ippCPUID_SSSE3 == ( *cpuFeatures & ippCPUID_SSSE3 )) { /* Intel® architecture formerly codenamed Merom ia32=V8, x64=U8 (letters etymology is unknown) */ + mask = SSSE3_MSK; + *index = LIB_SSSE3; + } else + if( ippCPUID_SSE3 == ( *cpuFeatures & ippCPUID_SSE3 )) { /* Intel® architecture formerly codenamed Prescott ia32=W7, x64=M7 */ + mask = SSE3_MSK; + *index = LIB_SSE3; + } else + if( ippCPUID_SSE2 == ( *cpuFeatures & ippCPUID_SSE2 )) { /* Intel® architecture formerly codenamed Willamette ia32=W7, x64=PX */ + mask = SSE2_MSK; + *index = LIB_SSE2; + } else + if( ippCPUID_SSE == ( *cpuFeatures & ippCPUID_SSE )) { /* Intel® Pentium® processor III ia32=PX only */ + mask = SSE_MSK; + *index = LIB_SSE; +#if (defined( WIN32E ) || defined( LINUX32E ) || defined( OSXEM64T )) && !(defined( _ARCH_LRB2 )) + ownStatus = ippStsNotSupportedCpu; /* the lowest CPU supported by Intel IPP Cryptography must at least support Intel® SSE2 for x64 */ +#endif + } else /* not supported, PX dispatched */ + { + mask = MMX_MSK; + *index = LIB_MMX; + ownStatus = ippStsNotSupportedCpu; /* the lowest CPU supported by Intel IPP Cryptography must at least support Intel® SSE for ia32 or Intel® SSE2 for x64 */ + } + + if(( mask != ( *cpuFeatures & mask ))&&( ownStatus == ippStsNoErr )) + ownStatus = ippStsFeaturesCombination; /* warning if combination of features is incomplete */ + *cpuFeatures |= mask; + return ownStatus; +} + + +IPPFUN(IppStatus, ippcpSetNumThreads, (int numThr)) +{ + IppStatus status = ippStsNoErr; + + IPP_UNREFERENCED_PARAMETER(numThr); + status = ippStsNoOperation; + + return status; +} + +IPPFUN(IppStatus, ippcpGetNumThreads, (int* pNumThr)) +{ + IppStatus status = ippStsNoErr; + IPP_BAD_PTR1_RET(pNumThr) + + *pNumThr = 1; + status = ippStsNoOperation; + + return status; +} + +IPPFUN( IppStatus, ippcpInit,( void )) +{ + Ipp64u cpuFeatures; + + ippcpGetCpuFeatures( &cpuFeatures ); + return ippcpSetCpuFeatures( cpuFeatures ); +} + + +IPPFUN( IppStatus, ippcpSetCpuFeatures,( Ipp64u cpuFeatures )) +{ + IppStatus ownStatus; + int index = 0; + + ownStatus = owncpSetCpuFeaturesAndIdx( cpuFeatures, &index ); + ippcpJumpIndexForMergedLibs = index; + cpFeaturesMask = cpuFeatures; + return ownStatus; +} + +IppStatus owncpSetCpuFeaturesAndIdx(Ipp64u cpuFeatures, int* index) +{ + Ipp64u tmp; + IppStatus tmpStatus; + *index = 0; + + if (ippCPUID_NOCHECK & cpuFeatures) { + // if NOCHECK is set - static variable cpFeatures is initialized unconditionally and real CPU features from CPUID are ignored; + // the one who uses this method of initialization must understand what and why it does and the possible unpredictable consequences. + // the only one known purpose for this approach - environments where CPUID instruction is disabled (for example Intel® Software Guard Extensions). + cpuFeatures &= (IPP_MAX_64U ^ ippCPUID_NOCHECK); + cpFeatures = cpuFeatures; + } + else + /* read cpu features unconditionally */ + cpGetFeatures(&tmp); + + tmpStatus = owncpFeaturesToIdx(&cpuFeatures, index); + cpFeaturesMask = cpuFeatures; + + return tmpStatus; +} + +static struct { + int sts; + const char *msg; +} ippcpMsg[] = { +/* ippStatus */ +/* -9999 */ ippStsCpuNotSupportedErr, "ippStsCpuNotSupportedErr: The target CPU is not supported", +/* -9702 */ MSG_NO_SHARED, "No shared libraries were found in the Waterfall procedure", +/* -9701 */ MSG_NO_DLL, "No DLLs were found in the Waterfall procedure", +/* -9700 */ MSG_LOAD_DLL_ERR, "Error at loading of %s library", +/* -1017 */ ippStsInvalidPoint, "ippStsInvalidPoint ECC: Invalid point (out of EC)", +/* -1016 */ ippStsQuadraticNonResidueErr, "ippStsQuadraticNonResidueErr: SQRT operation on quadratic non-residue value", +/* -1015 */ ippStsPointAtInfinity, "ippStsPointAtInfinity: Point at infinity is detected", +/* -1014 */ ippStsOFBSizeErr, "ippStsOFBSizeErr: Incorrect value for crypto OFB block size", +/* -1013 */ ippStsIncompleteContextErr, "ippStsIncompleteContextErr: Crypto: set up of context is not complete", +/* -1012 */ ippStsCTRSizeErr, "ippStsCTRSizeErr: Incorrect value for crypto CTR block size", +/* -1011 */ ippStsEphemeralKeyErr, "ippStsEphemeralKeyErr: ECC: Invalid ephemeral key", +/* -1010 */ ippStsMessageErr, "ippStsMessageErr: ECC: Invalid message digest", +/* -1009 */ ippStsShareKeyErr, "ippStsShareKeyErr: ECC: Invalid share key", +/* -1008 */ ippStsInvalidPrivateKey, "ippStsInvalidPrivateKey ECC: Invalid private key", +/* -1007 */ ippStsOutOfECErr, "ippStsOutOfECErr: ECC: Point out of EC", +/* -1006 */ ippStsECCInvalidFlagErr, "ippStsECCInvalidFlagErr: ECC: Invalid Flag", +/* -1005 */ ippStsUnderRunErr, "ippStsUnderRunErr: Error in data under run", +/* -1004 */ ippStsPaddingErr, "ippStsPaddingErr: Detected padding error indicates the possible data corruption", +/* -1003 */ ippStsCFBSizeErr, "ippStsCFBSizeErr: Incorrect value for crypto CFB block size", +/* -1002 */ ippStsPaddingSchemeErr, "ippStsPaddingSchemeErr: Invalid padding scheme", +/* -1001 */ ippStsBadModulusErr, "ippStsBadModulusErr: Bad modulus caused a failure in module inversion", +/* -216 */ ippStsUnknownStatusCodeErr, "ippStsUnknownStatusCodeErr: Unknown status code", +/* -221 */ ippStsLoadDynErr, "ippStsLoadDynErr: Error when loading the dynamic library", +/* -15 */ ippStsLengthErr, "ippStsLengthErr: Incorrect value for string length", +/* -14 */ ippStsNotSupportedModeErr, "ippStsNotSupportedModeErr: The requested mode is currently not supported", +/* -13 */ ippStsContextMatchErr, "ippStsContextMatchErr: Context parameter does not match the operation", +/* -12 */ ippStsScaleRangeErr, "ippStsScaleRangeErr: Scale bounds are out of range", +/* -11 */ ippStsOutOfRangeErr, "ippStsOutOfRangeErr: Argument is out of range, or point is outside the image", +/* -10 */ ippStsDivByZeroErr, "ippStsDivByZeroErr: An attempt to divide by zero", +/* -9 */ ippStsMemAllocErr, "ippStsMemAllocErr: Memory allocated for the operation is not enough", +/* -8 */ ippStsNullPtrErr, "ippStsNullPtrErr: Null pointer error", +/* -7 */ ippStsRangeErr, "ippStsRangeErr: Incorrect values for bounds: the lower bound is greater than the upper bound", +/* -6 */ ippStsSizeErr, "ippStsSizeErr: Incorrect value for data size", +/* -5 */ ippStsBadArgErr, "ippStsBadArgErr: Incorrect arg/param of the function", +/* -4 */ ippStsNoMemErr, "ippStsNoMemErr: Not enough memory for the operation", +/* -2 */ ippStsErr, "ippStsErr: Unknown/unspecified error, -2", +/* 0 */ ippStsNoErr, "ippStsNoErr: No errors", +/* 1 */ ippStsNoOperation, "ippStsNoOperation: No operation has been executed", +/* 2 */ ippStsDivByZero, "ippStsDivByZero: Zero value(s) for the divisor in the Div function", +/* 25 */ ippStsInsufficientEntropy, "ippStsInsufficientEntropy: Generation of the prime/key failed due to insufficient entropy in the random seed and stimulus bit string", +/* 36 */ ippStsNotSupportedCpu, "The CPU is not supported", +/* 51 */ ippStsFeaturesCombination, "Wrong combination of features", +/* 53 */ ippStsMbWarning, "ippStsMbWarning: Error(s) in statuses array", +}; + +/* ///////////////////////////////////////////////////////////////////////////// +// Name: ippcpGetStatusString +// Purpose: transformation of a code of a status Intel IPP Cryptography to string +// Returns: +// Parameters: +// StsCode Intel IPP Cryptography status code +// +// Notes: not necessary to release the returned string +*/ +IPPFUN( const char*, ippcpGetStatusString, ( IppStatus StsCode ) ) +{ + unsigned int i; + for( i=0; i:$ENV{ROOT}/compiler/include $ENV{ROOT}/compiler/include/icc> + $<$>:${CMAKE_SYSTEM_INCLUDE_PATH}> + $<$,$>:$ENV{INCLUDE}> +) + +add_subdirectory(src) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/Readme.md b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/Readme.md new file mode 100644 index 0000000..ae3e07e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/Readme.md @@ -0,0 +1,135 @@ +# Crypto Multi-buffer Library + +Currently, the library provides optimized version of the following algorithms: +1. RSA, ECDSA, ECDH, x25519 multi-buffer algorithms based on Intel® Advanced Vector Extensions 512 (Intel® AVX-512) integer fused multiply-add (IFMA) operations. This CPU feature is introduced with Intel® Microarchitecture Code Named Ice Lake. +2. SM4 based on Intel(R) Advanced Vector Extensions 512 (Intel(R) AVX-512) GFNI instructions. +3. SM3 based on Intel® Advanced Vector Extensions 512 (Intel® AVX-512) instructions. + +## Multiple Buffers Processing Overview + +Some of the cryptography algorithms cannot be parallelized due to their specificity. For example, the RSA algorithm consists of big-number (multi-digit) arithmetic: multi-precision modular multiplications and squaring operations. + +Digits of multi-digit numbers are dependent because of carry propagation during arithmetic operations. Therefore, single RSA computation based on general purpose mul/adc instructions is not efficient. + +The way to get high-performance implementations of such cryptographic algorithms is a parallel processing using *single instruction, multiple data* (SIMD) architecture. The usage model implies presence of several independent and homogeneous (encryption or decryption) requests for RSA operations, using of SIMD leads to performance improvement. + +Multi-buffer processing collects up to eight RSA operations. Each request is computed independently from the others, so it is possible to process several packets at the same time. + +This library consists of highly-optimized kernels taking advantage of Intel’s multi-buffer processing and Intel® AVX-512 instruction set. + +## Software Requirements + +### Common + +- CMake\* 3.10 or higher +- The Netwide Assembler (NASM) 2.14\* +- OpenSSL\* 3.0.8 or higher + +### Linux* OS + +- Intel® C++ Compiler Classic 2021.9 for Linux\* OS +- GCC 8.3 +- GCC 9.1 +- GCC 10.1 +- GCC 11.1 +- Clang 9.0 +- Clang 12.0 +- GNU binutils 2.32 + +### Windows* OS + +- Intel® C++ Compiler Classic 2021.9 for Windows\* OS +- Microsoft Visual C++ Compiler\* version 19.24 provided by Microsoft Visual Studio\* 2019 version 16.4 +- Microsoft Visual C++ Compiler\* version 19.30 provided by Microsoft Visual Studio\* 2022 version 17.0 +> **NOTE:** [CMake\*](https://cmake.org/download) 3.21 or higher is required to build using Microsoft Visual Studio\* 2022. + +### macOS* + +- Intel® C++ Compiler Classic 2021.9 for macOS\* + +## Installation + +You can install the Crypto Multi-buffer library in two different ways: +1. Installation to the default directories. + > **Note**: To run this installation type, you need to have appropriate permissions to write to the installation directory. + + Default locations (default values of `CMAKE_INSTALL_PREFIX`): + - Unix: /usr/local + - Windows: C:\Program Files\crypto_mb or C:\Program Files (x86)\crypto_mb + + To begin installation, run the command below in the project folder that was specified with `-B`: + ``` bash + make install + ``` + > **Note**: Installation requires write permissions to the build directory to create the installation manifest file. If it is not feasible in your setup, copy the build to the local directory and change permissions accordingly. + +2. Installation to your own directory. + If you want to install the library not by default paths, specify the `CMAKE_INSTALL_PREFIX` variable at the configuration step, for example: + ``` bash + cmake . -B"../build" -DCMAKE_INSTALL_PREFIX=path/to/libcrypto_mb/installation + ``` + +You can find the installed files in: + +``` bash + +├── ${CMAKE_INSTALL_PREFIX} +   ├── include + | └── crypto_mb + | ├── cpu_features.h +   │   ├── defs.h +   │   ├── ec_nistp256.h +   │   ├── ec_nistp384.h +   │   ├── ec_nistp521.h +   │   ├── ec_sm2.h +   │   ├── ed25519.h +   │   ├── exp.h +   │   ├── rsa.h +   │   ├── sm3.h +   │   ├── sm4_ccm.h +   │   ├── sm4_gcm.h +   │   ├── sm4.h +   │   ├── status.h + | ├── version.h +   │   └── x25519.h +   └── lib + └── libcrypto_mb.so +``` + +## How to Build + +1. Clone the repository from GitHub\* as follows: + + ``` bash + git clone --recursive https://github.com/intel/ipp-crypto + ``` + and navigate to the `sources/ippcp/crypto_mb` folder. +2. Set the environment variables for one of the supported C/C++ compilers. + + *Example for Intel® Compiler:* + + ```bash + source /opt/intel/bin/compilervars.sh intel64 + ``` + + For details, refer to the [Intel® C++ Compiler Developer Guide and Reference](https://www.intel.com/content/www/us/en/docs/cpp-compiler/developer-guide-reference/current/specifying-the-location-of-compiler-components.html) + +3. Run CMake on the command line. Use `-B` to specify path to the resulting project. +4. Go to the project folder that was specified with `-B` and run `make` to build the library (`crypto_mb` target). + +### Building with Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) + +The Crypto Multi-buffer library will be built automatically with Intel® IPP Cryptography if optimization for Intel® Microarchitecture Code Named Ice Lake is available. For more information see [Intel IPP Cryptography Build Instructions](../../../BUILD.md) + +### CMake Build Options + +- Use `OPENSSL_INCLUDE_DIR`, `OPENSSL_LIBRARIES` and `OPENSSL_ROOT_DIR` to override path to OpenSSL\*: + + ``` bash + cmake . -B"../build" + -DOPENSSL_INCLUDE_DIR=/path/to/openssl/include + -DOPENSSL_LIBRARIES=/path/to/openssl/lib + -DOPENSSL_ROOT_DIR=/path/to/openssl/installation/dir + ``` + +- Set `-DOPENSSL_USE_STATIC_LIBS=TRUE` if static OpenSSL libraries are preferred. diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/cpu_features.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/cpu_features.h new file mode 100644 index 0000000..577f81d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/cpu_features.h @@ -0,0 +1,113 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef CPU_FEATURES_H +#define CPU_FEATURES_H + +#include + +/* definition of cpu features */ +#define mbcpCPUID_MMX 0x00000001LL /* Intel® architecture with MMX(TM) technology supported */ +#define mbcpCPUID_SSE 0x00000002LL /* Intel® Streaming SIMD Extensions (Intel® SSE) instruction set */ +#define mbcpCPUID_SSE2 0x00000004LL /* Intel® Streaming SIMD Extensions 2 (Intel® SSE2) instruction set */ +#define mbcpCPUID_SSE3 0x00000008LL /* Intel® Streaming SIMD Extensions 3 (Intel® SSE3) instruction set */ +#define mbcpCPUID_SSSE3 0x00000010LL /* Supplemental Streaming SIMD Extensions 3 (SSSE3) instruction set */ +#define mbcpCPUID_MOVBE 0x00000020LL /* Intel® instruction MOVBE */ +#define mbcpCPUID_SSE41 0x00000040LL /* Intel® Streaming SIMD Extensions 4.1 (Intel® SSE4.1) instruction set */ +#define mbcpCPUID_SSE42 0x00000080LL /* Intel® Streaming SIMD Extensions 4.2 (Intel® SSE4.2) instruction set */ +#define mbcpCPUID_AVX 0x00000100LL /* Intel® Advanced Vector Extensions (Intel® AVX) instruction set */ +#define mbcpAVX_ENABLEDBYOS 0x00000200LL /* Intel® Advanced Vector Extensions (Intel® AVX) instruction set is supported by OS */ +#define mbcpCPUID_AES 0x00000400LL /* AES */ +#define mbcpCPUID_CLMUL 0x00000800LL /* Intel® instruction PCLMULQDQ */ +#define mbcpCPUID_ABR 0x00001000LL /* Reserved */ +#define mbcpCPUID_RDRAND 0x00002000LL /* Intel® instruction RDRAND */ +#define mbcpCPUID_F16C 0x00004000LL /* Intel® instruction F16C */ +#define mbcpCPUID_AVX2 0x00008000LL /* Intel® Advanced Vector Extensions 2 (Intel® AVX2) */ +#define mbcpCPUID_ADX 0x00010000LL /* Intel® instructions ADOX/ADCX */ +#define mbcpCPUID_RDSEED 0x00020000LL /* Intel® instruction RDSEED */ +#define mbcpCPUID_PREFETCHW 0x00040000LL /* Intel® instruction PREFETCHW */ +#define mbcpCPUID_SHA 0x00080000LL /* Intel® Secure Hash Algorithm Extensions (Intel® SHA Extensions) */ +#define mbcpCPUID_AVX512F 0x00100000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) Foundation instruction set */ +#define mbcpCPUID_AVX512CD 0x00200000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) CD instruction set */ +#define mbcpCPUID_AVX512ER 0x00400000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) ER instruction set */ +#define mbcpCPUID_AVX512PF 0x00800000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) PF instruction set */ +#define mbcpCPUID_AVX512BW 0x01000000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) BW instruction set */ +#define mbcpCPUID_AVX512DQ 0x02000000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) DQ instruction set */ +#define mbcpCPUID_AVX512VL 0x04000000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) VL instruction set */ +#define mbcpCPUID_AVX512VBMI 0x08000000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) Bit Manipulation instructions */ +#define mbcpCPUID_MPX 0x10000000LL /* Intel® Memory Protection Extensions (Intel® MPX) */ +#define mbcpCPUID_AVX512_4FMADDPS 0x20000000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) DL floating-point single precision */ +#define mbcpCPUID_AVX512_4VNNIW 0x40000000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) DL enhanced word variable precision */ +#define mbcpCPUID_KNC 0x80000000LL /* Intel® Xeon® Phi(TM) Coprocessor */ +#define mbcpCPUID_AVX512IFMA 0x100000000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) IFMA (PMADD52) instruction set */ +#define mbcpAVX512_ENABLEDBYOS 0x200000000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) is supported by OS */ +#define mbcpCPUID_AVX512GFNI 0x400000000LL /* GFNI */ +#define mbcpCPUID_AVX512VAES 0x800000000LL /* VAES */ +#define mbcpCPUID_AVX512VCLMUL 0x1000000000LL /* VCLMUL */ +#define mbcpCPUID_AVX512VBMI2 0x2000000000LL /* Intel® Advanced Vector Extensions 512 (Intel® AVX-512) Bit Manipulation instructions 2 */ +#define mbcpCPUID_BMI1 0x4000000000LL /* BMI1 */ +#define mbcpCPUID_BMI2 0x8000000000LL /* BMI2 */ + +/* map cpu features */ +EXTERN_C int64u mbx_get_cpu_features(void); + +/* check if crypto_mb is applicable */ +EXTERN_C int mbx_is_crypto_mb_applicable(int64u cpu_features); + +/* supported algorithm */ +enum MBX_ALGO { + MBX_ALGO_RSA_1K, + MBX_ALGO_RSA_2K, + MBX_ALGO_RSA_3K, + MBX_ALGO_RSA_4K, + MBX_ALGO_X25519, + MBX_ALGO_EC_NIST_P256, + MBX_ALGO_ECDHE_NIST_P256 = MBX_ALGO_EC_NIST_P256, + MBX_ALGO_ECDSA_NIST_P256 = MBX_ALGO_EC_NIST_P256, + MBX_ALGO_EC_NIST_P384, + MBX_ALGO_ECDHE_NIST_P384 = MBX_ALGO_EC_NIST_P384, + MBX_ALGO_ECDSA_NIST_P384 = MBX_ALGO_EC_NIST_P384, + MBX_ALGO_EC_NIST_P521, + MBX_ALGO_ECDHE_NIST_P521 = MBX_ALGO_EC_NIST_P521, + MBX_ALGO_ECDSA_NIST_P521 = MBX_ALGO_EC_NIST_P521, + MBX_ALGO_EC_SM2, + MBX_ALGO_ECDHE_SM2 = MBX_ALGO_EC_SM2, + MBX_ALGO_ECDSA_SM2 = MBX_ALGO_EC_SM2, + MBX_ALGO_SM3, + MBX_ALGO_SM4, + MBX_ALGO_ECB_SM4 = MBX_ALGO_SM4, + MBX_ALGO_CBC_SM4 = MBX_ALGO_SM4, + MBX_ALGO_CTR_SM4 = MBX_ALGO_SM4, + MBX_ALGO_OFB_SM4 = MBX_ALGO_SM4, + MBX_ALGO_OFB128_SM4 = MBX_ALGO_SM4, +}; + +/* multi-buffer width implemented by library */ +enum MBX_WIDTH { + MBX_WIDTH_MB8 = 8, + MBX_WIDTH_MB16 = 16, + MBX_WIDTH_ANY = (1 << 16) - 1 +}; + +typedef int64u MBX_ALGO_INFO; + +/* check if algorithm is supported on current platform + * returns: multi-buffer width mask or 0 if algorithm not supported +*/ +EXTERN_C MBX_ALGO_INFO mbx_get_algo_info(enum MBX_ALGO algo); + +#endif /* CPU_FEATURES_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/defs.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/defs.h new file mode 100644 index 0000000..aa5a234 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/defs.h @@ -0,0 +1,68 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef DEFS_H +#define DEFS_H + +/* data types */ +typedef unsigned char int8u; +typedef unsigned short int16u; +typedef unsigned int int32u; +typedef unsigned long long int64u; + +#ifndef NULL + #define NULL ((void *)0) +#endif + +/* alignment & inline */ +#if defined(__GNUC__) + #if !defined(__ALIGN64) + #define __ALIGN64 __attribute__((aligned(64))) + #endif + + #if !defined(__INLINE) + #define __INLINE static __inline__ + #endif + + #if !defined(__NOINLINE) + #define __NOINLINE __attribute__((noinline)) + #endif +#else + #if !defined(__ALIGN64) + #define __ALIGN64 __declspec(align(64)) + #endif + + #if !defined(__INLINE) + #define __INLINE static __forceinline + #endif + + #if !defined(__NOINLINE) + #define __NOINLINE __declspec(noinline) + #endif +#endif + + +/* externals */ +#undef EXTERN_C + +#ifdef __cplusplus + #define EXTERN_C extern "C" +#else + #define EXTERN_C +#endif + +#endif /* DEFS_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/ec_nistp256.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/ec_nistp256.h new file mode 100644 index 0000000..2ffa7c5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/ec_nistp256.h @@ -0,0 +1,187 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef EC_NISTP256_H +#define EC_NISTP256_H + +#include +#include + +#ifndef BN_OPENSSL_DISABLE + #include + #include +#endif // BN_OPENSSL_DISABLE + + +/* +// ECDHE +*/ + +/* +// Computes public key +// pa_pubx[] array of pointers to the public keys X-coordinates +// pa_puby[] array of pointers to the public keys Y-coordinates +// pa_pubz[] array of pointers to the public keys Z-coordinates +// pa_skey[] array of pointers to the private keys +// pBuffer pointer to the scratch buffer +// +// Note: +// output public key is represented by (X:Y:Z) projective Jacobian coordinates +*/ +#ifndef BN_OPENSSL_DISABLE +EXTERN_C mbx_status mbx_nistp256_ecpublic_key_ssl_mb8(BIGNUM* pa_pubx[8], + BIGNUM* pa_puby[8], + BIGNUM* pa_pubz[8], + const BIGNUM* const pa_skey[8], + int8u* pBuffer); +#endif // BN_OPENSSL_DISABLE + + +EXTERN_C mbx_status mbx_nistp256_ecpublic_key_mb8(int64u* pa_pubx[8], + int64u* pa_puby[8], + int64u* pa_pubz[8], + const int64u* const pa_skey[8], + int8u* pBuffer); +/* +// Computes shared key +// pa_shared_key[] array of pointers to the shared keys +// pa_skey[] array of pointers to the own (ephemeral) private keys +// pa_pubx[] array of pointers to the party's public keys X-coordinates +// pa_puby[] array of pointers to the party's public keys Y-coordinates +// pa_pubz[] array of pointers to the party's public keys Z-coordinates +// pBuffer pointer to the scratch buffer +// +// Note: +// input party's public key is represented by (X:Y:Z) projective Jacobian coordinates +*/ +#ifndef BN_OPENSSL_DISABLE +EXTERN_C mbx_status mbx_nistp256_ecdh_ssl_mb8(int8u* pa_shared_key[8], + const BIGNUM* const pa_skey[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer); +#endif // BN_OPENSSL_DISABLE + + +EXTERN_C mbx_status mbx_nistp256_ecdh_mb8(int8u* pa_shared_key[8], + const int64u* const pa_skey[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer); + + +/* +// ECDSA signature generation +*/ + +/* +// Pre-computes ECDSA signature +// pa_inv_eph_skey[] array of pointers to the inversion of signer's ephemeral private keys +// pa_sign_rp[] array of pointers to the pre-computed r-components of the signatures +// pa_eph_skey[] array of pointers to the signer's ephemeral private keys +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp256_ecdsa_sign_setup_mb8(int64u* pa_inv_eph_skey[8], + int64u* pa_sign_rp[8], + const int64u* const pa_eph_skey[8], + int8u* pBuffer); +/* +// computes ECDSA signature +// +// pa_sign_pr[] array of pointers to the r-components of the signatures +// pa_sign_ps[] array of pointers to the s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// pa_sign_rp[] array of pointers to the pre-computed r-components of the signatures +// pa_inv_eph_skey[] array of pointers to the inversion of signer's ephemeral private keys +// pa_reg_skey[] array of pointers to the regular signer's ephemeral private keys +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp256_ecdsa_sign_complete_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_sgn_rp[8], + const int64u* const pa_inv_eph_skey[8], + const int64u* const pa_reg_skey[8], + int8u* pBuffer); +/* +// Computes ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// pa_eph_skey[] array of pointers to the signer's ephemeral private keys +// pa_reg_skey[] array of pointers to the signer's regular private keys +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp256_ecdsa_sign_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_eph_skey[8], + const int64u* const pa_reg_skey[8], + int8u* pBuffer); + +/* +// Verifies ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_msg[] array of pointers to the messages that have been signed +// pa_pubx[] array of pointers to the signer's public keys X-coordinates +// pa_puby[] array of pointers to the signer's public keys Y-coordinates +// pa_pubz[] array of pointers to the signer's public keys Z-coordinates (or NULL, if affine coordinate requested) +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp256_ecdsa_verify_mb8(const int8u* const pa_sign_r[8], + const int8u* const pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer); +/* +// OpenSSL's specific similar APIs +*/ +#ifndef BN_OPENSSL_DISABLE +EXTERN_C mbx_status mbx_nistp256_ecdsa_sign_setup_ssl_mb8(BIGNUM* pa_inv_eph_skey[8], + BIGNUM* pa_sign_pr[8], + const BIGNUM* const pa_eph_skey[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_nistp256_ecdsa_sign_complete_ssl_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const BIGNUM* const pa_sgn_rp[8], + const BIGNUM* const pa_inv_eph_skey[8], + const BIGNUM* const pa_reg_skey[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_nistp256_ecdsa_sign_ssl_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const BIGNUM* const pa_eph_skey[8], + const BIGNUM* const pa_reg_skey[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_nistp256_ecdsa_verify_ssl_mb8(const ECDSA_SIG* const pa_sig[8], + const int8u* const pa_msg[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer); + +#endif // BN_OPENSSL_DISABLE +#endif /* EC_NISTP256_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/ec_nistp384.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/ec_nistp384.h new file mode 100644 index 0000000..4c3fa98 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/ec_nistp384.h @@ -0,0 +1,186 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef EC_NISTP384_H +#define EC_NISTP384_H + +#include +#include + +#ifndef BN_OPENSSL_DISABLE + #include + #include +#endif // BN_OPENSSL_DISABLE + + +/* +// ECDHE +*/ + +/* +// Computes public key +// pa_pubx[] array of pointers to the public keys X-coordinates +// pa_puby[] array of pointers to the public keys Y-coordinates +// pa_pubz[] array of pointers to the public keys Z-coordinates +// pa_skey[] array of pointers to the private keys +// pBuffer pointer to the scratch buffer +// +// Note: +// output public key is represented by (X:Y:Z) projective Jacobian coordinates +*/ +#ifndef BN_OPENSSL_DISABLE +EXTERN_C mbx_status mbx_nistp384_ecpublic_key_ssl_mb8(BIGNUM* pa_pubx[8], + BIGNUM* pa_puby[8], + BIGNUM* pa_pubz[8], + const BIGNUM* const pa_skey[8], + int8u* pBuffer); +#endif // BN_OPENSSL_DISABLE + + +EXTERN_C mbx_status mbx_nistp384_ecpublic_key_mb8(int64u* pa_pubx[8], + int64u* pa_puby[8], + int64u* pa_pubz[8], + const int64u* const pa_skey[8], + int8u* pBuffer); +/* +// Computes shared key +// pa_shared_key[] array of pointers to the shared keys +// pa_skey[] array of pointers to the own (ephemeral) private keys +// pa_pubx[] array of pointers to the party's public keys X-coordinates +// pa_puby[] array of pointers to the party's public keys Y-coordinates +// pa_pubz[] array of pointers to the party's public keys Z-coordinates +// pBuffer pointer to the scratch buffer +// +// Note: +// input party's public key is represented by (X:Y:Z) projective Jacobian coordinates +*/ +#ifndef BN_OPENSSL_DISABLE +EXTERN_C mbx_status mbx_nistp384_ecdh_ssl_mb8(int8u* pa_shared_key[8], + const BIGNUM* const pa_skey[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer); +#endif // BN_OPENSSL_DISABLE + + +EXTERN_C mbx_status mbx_nistp384_ecdh_mb8(int8u* pa_shared_key[8], + const int64u* const pa_skey[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer); + + +/* +// ECDSA signature generation +*/ + +/* +// Pre-computes ECDSA signature +// pa_inv_eph_skey[] array of pointers to the inversion of signer's ephemeral private keys +// pa_sign_rp[] array of pointers to the pre-computed r-components of the signatures +// pa_eph_skey[] array of pointers to the signer's ephemeral private keys +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp384_ecdsa_sign_setup_mb8(int64u* pa_inv_eph_skey[8], + int64u* pa_sign_rp[8], + const int64u* const pa_eph_skey[8], + int8u* pBuffer); +/* +// computes ECDSA signature +// +// pa_sign_pr[] array of pointers to the r-components of the signatures +// pa_sign_ps[] array of pointers to the s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// pa_sign_rp[] array of pointers to the pre-computed r-components of the signatures +// pa_inv_eph_skey[] array of pointers to the inversion of signer's ephemeral private keys +// pa_reg_skey[] array of pointers to the regular signer's ephemeral private keys +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp384_ecdsa_sign_complete_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_sgn_rp[8], + const int64u* const pa_inv_eph_skey[8], + const int64u* const pa_reg_skey[8], + int8u* pBuffer); +/* +// Computes ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// pa_eph_skey[] array of pointers to the signer's ephemeral private keys +// pa_reg_skey[] array of pointers to the signer's regular private keys +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp384_ecdsa_sign_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_eph_skey[8], + const int64u* const pa_reg_skey[8], + int8u* pBuffer); + +/* +// Verifies ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_msg[] array of pointers to the messages that have been signed +// pa_pubx[] array of pointers to the signer's public keys X-coordinates +// pa_puby[] array of pointers to the signer's public keys Y-coordinates +// pa_pubz[] array of pointers to the signer's public keys Z-coordinates (or NULL, if affine coordinate requested) +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp384_ecdsa_verify_mb8(const int8u* const pa_sign_r[8], + const int8u* const pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer); +/* +// OpenSSL's specific similar APIs +*/ +#ifndef BN_OPENSSL_DISABLE +EXTERN_C mbx_status mbx_nistp384_ecdsa_sign_setup_ssl_mb8(BIGNUM* pa_inv_eph_skey[8], + BIGNUM* pa_sign_pr[8], + const BIGNUM* const pa_eph_skey[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_nistp384_ecdsa_sign_complete_ssl_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const BIGNUM* const pa_sgn_rp[8], + const BIGNUM* const pa_inv_eph_skey[8], + const BIGNUM* const pa_reg_skey[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_nistp384_ecdsa_sign_ssl_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const BIGNUM* const pa_eph_skey[8], + const BIGNUM* const pa_reg_skey[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_nistp384_ecdsa_verify_ssl_mb8(const ECDSA_SIG* const pa_sign[8], + const int8u* const pa_msg[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer); +#endif // BN_OPENSSL_DISABLE +#endif /* EC_NISTP384_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/ec_nistp521.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/ec_nistp521.h new file mode 100644 index 0000000..1e938c3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/ec_nistp521.h @@ -0,0 +1,187 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef EC_NISTP521_H +#define EC_NISTP521_H + +#include +#include + +#ifndef BN_OPENSSL_DISABLE + #include + #include +#endif // BN_OPENSSL_DISABLE + + +/* +// ECDHE +*/ + +/* +// Computes public key +// pa_pubx[] array of pointers to the public keys X-coordinates +// pa_puby[] array of pointers to the public keys Y-coordinates +// pa_pubz[] array of pointers to the public keys Z-coordinates +// pa_skey[] array of pointers to the private keys +// pBuffer pointer to the scratch buffer +// +// Note: +// output public key is represented by (X:Y:Z) projective Jacobian coordinates +*/ +#ifndef BN_OPENSSL_DISABLE +EXTERN_C mbx_status mbx_nistp521_ecpublic_key_ssl_mb8(BIGNUM* pa_pubx[8], + BIGNUM* pa_puby[8], + BIGNUM* pa_pubz[8], + const BIGNUM* const pa_skey[8], + int8u* pBuffer); +#endif // BN_OPENSSL_DISABLE + + +EXTERN_C mbx_status mbx_nistp521_ecpublic_key_mb8(int64u* pa_pubx[8], + int64u* pa_puby[8], + int64u* pa_pubz[8], + const int64u* const pa_skey[8], + int8u* pBuffer); +/* +// Computes shared key +// pa_shared_key[] array of pointers to the shared keys +// pa_skey[] array of pointers to the own (ephemeral) private keys +// pa_pubx[] array of pointers to the party's public keys X-coordinates +// pa_puby[] array of pointers to the party's public keys Y-coordinates +// pa_pubz[] array of pointers to the party's public keys Z-coordinates +// pBuffer pointer to the scratch buffer +// +// Note: +// input party's public key is represented by (X:Y:Z) projective Jacobian coordinates +*/ +#ifndef BN_OPENSSL_DISABLE +EXTERN_C mbx_status mbx_nistp521_ecdh_ssl_mb8(int8u* pa_shared_key[8], + const BIGNUM* const pa_skey[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer); +#endif // BN_OPENSSL_DISABLE + + +EXTERN_C mbx_status mbx_nistp521_ecdh_mb8(int8u* pa_shared_key[8], + const int64u* const pa_skey[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer); + + +/* +// ECDSA signature generation +*/ + +/* +// Pre-computes ECDSA signature +// pa_inv_eph_skey[] array of pointers to the inversion of signer's ephemeral private keys +// pa_sign_rp[] array of pointers to the pre-computed r-components of the signatures +// pa_eph_skey[] array of pointers to the signer's ephemeral private keys +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp521_ecdsa_sign_setup_mb8(int64u* pa_inv_eph_skey[8], + int64u* pa_sign_rp[8], + const int64u* const pa_eph_skey[8], + int8u* pBuffer); +/* +// computes ECDSA signature +// +// pa_sign_pr[] array of pointers to the r-components of the signatures +// pa_sign_ps[] array of pointers to the s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// pa_sign_rp[] array of pointers to the pre-computed r-components of the signatures +// pa_inv_eph_skey[] array of pointers to the inversion of signer's ephemeral private keys +// pa_reg_skey[] array of pointers to the regular signer's ephemeral private keys +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp521_ecdsa_sign_complete_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_sgn_rp[8], + const int64u* const pa_inv_eph_skey[8], + const int64u* const pa_reg_skey[8], + int8u* pBuffer); +/* +// Computes ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// pa_eph_skey[] array of pointers to the signer's ephemeral private keys +// pa_reg_skey[] array of pointers to the signer's regular private keys +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp521_ecdsa_sign_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_eph_skey[8], + const int64u* const pa_reg_skey[8], + int8u* pBuffer); + +/* +// Verifies ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// pa_pubx[] array of pointers to the public keys X-coordinates +// pa_puby[] array of pointers to the public keys Y-coordinates +// pa_pubz[] array of pointers to the public keys Z-coordinates +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_nistp521_ecdsa_verify_mb8(const int8u* const pa_sign_r[8], + const int8u* const pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer); +/* +// OpenSSL's specific similar APIs +*/ +#ifndef BN_OPENSSL_DISABLE +EXTERN_C mbx_status mbx_nistp521_ecdsa_sign_setup_ssl_mb8(BIGNUM* pa_inv_eph_skey[8], + BIGNUM* pa_sign_pr[8], + const BIGNUM* const pa_eph_skey[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_nistp521_ecdsa_sign_complete_ssl_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const BIGNUM* const pa_sgn_rp[8], + const BIGNUM* const pa_inv_eph_skey[8], + const BIGNUM* const pa_reg_skey[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_nistp521_ecdsa_sign_ssl_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const BIGNUM* const pa_eph_skey[8], + const BIGNUM* const pa_reg_skey[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_nistp521_ecdsa_verify_ssl_mb8(const ECDSA_SIG* const pa_sig[8], + const int8u* const pa_msg[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer); + +#endif // BN_OPENSSL_DISABLE +#endif /* EC_NISTP521_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/ec_sm2.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/ec_sm2.h new file mode 100644 index 0000000..43d6f97 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/ec_sm2.h @@ -0,0 +1,154 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef EC_SM2_H +#define EC_SM2_H + +#include +#include + +#ifndef BN_OPENSSL_DISABLE + #include + #include +#endif // BN_OPENSSL_DISABLE + +/* +// Computes public key +// pa_pubx[] array of pointers to the public keys X-coordinates +// pa_puby[] array of pointers to the public keys Y-coordinates +// pa_pubz[] array of pointers to the public keys Z-coordinates (or NULL, if affine coordinate requested) +// pa_skey[] array of pointers to the private keys +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_sm2_ecpublic_key_mb8(int64u* pa_pubx[8], + int64u* pa_puby[8], + int64u* pa_pubz[8], + const int64u* const pa_skey[8], + int8u* pBuffer); + +/* +// Computes shared key +// pa_shared_key[] array of pointers to the shared keys +// pa_skey[] array of pointers to the own (ephemeral) private keys +// pa_pubx[] array of pointers to the party's public keys X-coordinates +// pa_puby[] array of pointers to the party's public keys Y-coordinates +// pa_pubz[] array of pointers to the party's public keys Z-coordinates (or NULL, if affine coordinate requested) +// pBuffer pointer to the scratch buffer +// +// Note: +// This function implements ECDHE over SM2 curve according to IEEE 1363-2000 standard. +*/ +EXTERN_C mbx_status mbx_sm2_ecdh_mb8(int8u* pa_shared_key[8], + const int64u* const pa_skey[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer); + +/* +// Computes SM2 ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_user_id[] array of pointers to the users ID +// user_id_len[] array of users ID length +// pa_msg[] array of pointers to the messages are being signed +// msg_len[] array of messages length +// pa_eph_skey[] array of pointers to the signer's ephemeral private keys +// pa_reg_skey[] array of pointers to the signer's regular private keys +// pa_pubx[] array of pointers to the party's public keys X-coordinates +// pa_puby[] array of pointers to the party's public keys Y-coordinates +// pa_pubz[] array of pointers to the party's public keys Z-coordinates (or NULL, if affine coordinate requested) +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_sm2_ecdsa_sign_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_user_id[8], + const int user_id_len[8], + const int8u* const pa_msg[8], + const int msg_len[8], + const int64u* const pa_eph_skey[8], + const int64u* const pa_reg_skey[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer); + +/* +// Verifies SM2 ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_user_id[] array of pointers to the users ID +// user_id_len[] array of users ID length +// pa_msg[] array of pointers to the messages are being signed +// msg_len[] array of messages length +// pa_pubx[] array of pointers to the signer's public keys X-coordinates +// pa_puby[] array of pointers to the signer's public keys Y-coordinates +// pa_pubz[] array of pointers to the signer's public keys Z-coordinates (or NULL, if affine coordinate requested) +// pBuffer pointer to the scratch buffer +*/ +EXTERN_C mbx_status mbx_sm2_ecdsa_verify_mb8(const int8u* const pa_sign_r[8], + const int8u* const pa_sign_s[8], + const int8u* const pa_user_id[8], + const int user_id_len[8], + const int8u* const pa_msg[8], + const int msg_len[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer); + +/* +// OpenSSL's specific similar APIs +*/ +#ifndef BN_OPENSSL_DISABLE +EXTERN_C mbx_status mbx_sm2_ecpublic_key_ssl_mb8(BIGNUM* pa_pubx[8], + BIGNUM* pa_puby[8], + BIGNUM* pa_pubz[8], + const BIGNUM* const pa_skey[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_sm2_ecdh_ssl_mb8(int8u* pa_shared_key[8], + const BIGNUM* const pa_skey[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_sm2_ecdsa_sign_ssl_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_user_id[8], + const int user_id_len[8], + const int8u* const pa_msg[8], + const int msg_len[8], + const BIGNUM* const pa_eph_skey[8], + const BIGNUM* const pa_reg_skey[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer); + +EXTERN_C mbx_status mbx_sm2_ecdsa_verify_ssl_mb8(const ECDSA_SIG* const pa_sig[8], + const int8u* const pa_user_id[8], + const int user_id_len[8], + const int8u* const pa_msg[8], + const int msg_len[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer); +#endif // BN_OPENSSL_DISABLE +#endif /* EC_SM2_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/ed25519.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/ed25519.h new file mode 100644 index 0000000..8cc737d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/ed25519.h @@ -0,0 +1,66 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef ED25519_H +#define ED25519_H + +#include +#include + +typedef int8u ed25519_sign_component[32]; +typedef ed25519_sign_component ed25519_sign[2]; + +typedef int8u ed25519_public_key[32]; +typedef int8u ed25519_private_key[32]; + +/* +// Computes ed25519 public key +// pa_public_key[] array of pointers to the public keys +// pa_private_key[] array of pointers to the public keys Y-coordinates +*/ +EXTERN_C mbx_status mbx_ed25519_public_key_mb8(ed25519_public_key* pa_public_key[8], + const ed25519_private_key* const pa_private_key[8]); + +/* +// Computes ed25519 signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// msgLen[] lengths of the messages are being signed +// pa_private_key[] array of pointers to the signer's private keys +// pa_public_key[] array of pointers to the signer's public keys +*/ +EXTERN_C mbx_status mbx_ed25519_sign_mb8(ed25519_sign_component* pa_sign_r[8], + ed25519_sign_component* pa_sign_s[8], + const int8u* const pa_msg[8], const int32u msgLen[8], + const ed25519_private_key* const pa_private_key[8], + const ed25519_public_key* const pa_public_key[8]); + +/* +// Verifies ed25519 signature +// pa_sign_r[] array of pointers to the r-components of the verified signatures +// pa_sign_s[] array of pointers to the s-components of the verified signatures +// pa_msg[] array of pointers to the signed messages +// msgLen[] array of signed messages lengths +// pa_public_key[] array of pointers to the signer's public keys +*/ +EXTERN_C mbx_status mbx_ed25519_verify_mb8(const ed25519_sign_component* const pa_sign_r[8], + const ed25519_sign_component* const pa_sign_s[8], + const int8u* const pa_msg[8], const int32u msgLen[8], + const ed25519_public_key* const pa_public_key[8]); + +#endif /* ED25519_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/exp.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/exp.h new file mode 100644 index 0000000..b5a76b8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/exp.h @@ -0,0 +1,59 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef EXP_H +#define EXP_H + +#include +#include + + +/* size of scratch buffer */ +EXTERN_C int mbx_exp_BufferSize(int modulusBits); + +/* exp operation */ +EXTERN_C mbx_status mbx_exp1024_mb8(int64u* const out_pa[8], + const int64u* const base_pa[8], + const int64u* const exp_pa[8], int exp_bits, + const int64u* const mod_pa[8], int mod_bits, + int8u* pBuffer, int bufferLen); + +EXTERN_C mbx_status mbx_exp2048_mb8(int64u* const out_pa[8], + const int64u* const base_pa[8], + const int64u* const exp_pa[8], int exp_bits, + const int64u* const mod_pa[8], int mod_bits, + int8u* pBuffer, int bufferLen); + +EXTERN_C mbx_status mbx_exp3072_mb8(int64u* const out_pa[8], + const int64u* const base_pa[8], + const int64u* const exp_pa[8], int exp_bits, + const int64u* const mod_pa[8], int mod_bits, + int8u* pBuffer, int bufferLen); + +EXTERN_C mbx_status mbx_exp4096_mb8(int64u* const out_pa[8], + const int64u* const base_pa[8], + const int64u* const exp_pa[8], int exp_bits, + const int64u* const mod_pa[8], int mod_bits, + int8u* pBuffer, int bufferLen); + +EXTERN_C mbx_status mbx_exp_mb8(int64u* const out_pa[8], + const int64u* const base_pa[8], + const int64u* const exp_pa[8], int exp_bits, + const int64u* const mod_pa[8], int mod_bits, + int8u* pBuffer, int bufferLen); + +#endif /* EXP_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/rsa.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/rsa.h new file mode 100644 index 0000000..f71f2d5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/rsa.h @@ -0,0 +1,104 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef RSA_H +#define RSA_H + +#include +#include + +#ifndef BN_OPENSSL_DISABLE +#include + +EXTERN_C mbx_status mbx_rsa_public_ssl_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const e_pa[8], + const BIGNUM* const n_pa[8], + int expected_rsa_bitsize); + +EXTERN_C mbx_status mbx_rsa_private_ssl_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const d_pa[8], + const BIGNUM* const n_pa[8], + int expected_rsa_bitsize); + +EXTERN_C mbx_status mbx_rsa_private_crt_ssl_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const p_pa[8], + const BIGNUM* const q_pa[8], + const BIGNUM* const dp_pa[8], + const BIGNUM* const dq_pa[8], + const BIGNUM* const iq_pa[8], + int expected_rsa_bitsize); +#endif /* BN_OPENSSL_DISABLE */ + + +/* +// rsa cp methods +*/ +typedef struct _ifma_rsa_method mbx_RSA_Method; + +/* rsa public key opertaion */ +EXTERN_C const mbx_RSA_Method* mbx_RSA1K_pub65537_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA2K_pub65537_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA3K_pub65537_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA4K_pub65537_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA_pub65537_Method(int rsaBitsize); + +/* rsa private key opertaion */ +EXTERN_C const mbx_RSA_Method* mbx_RSA1K_private_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA2K_private_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA3K_private_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA4K_private_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA_private_Method(int rsaBitsize); + +/* rsa private key opertaion (ctr) */ +EXTERN_C const mbx_RSA_Method* mbx_RSA1K_private_crt_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA2K_private_crt_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA3K_private_crt_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA4K_private_crt_Method(void); +EXTERN_C const mbx_RSA_Method* mbx_RSA_private_crt_Method(int rsaBitsize); + +EXTERN_C int mbx_RSA_Method_BufSize(const mbx_RSA_Method* m); + + +EXTERN_C mbx_status mbx_rsa_public_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const int64u* const n_pa[8], + int rsaBitlen, + const mbx_RSA_Method* m, + int8u* pBuffer); + +EXTERN_C mbx_status mbx_rsa_private_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const int64u* const d_pa[8], + const int64u* const n_pa[8], + int rsaBitlen, + const mbx_RSA_Method* m, + int8u* pBuffer); + +EXTERN_C mbx_status mbx_rsa_private_crt_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const int64u* const p_pa[8], + const int64u* const q_pa[8], + const int64u* const dp_pa[8], + const int64u* const dq_pa[8], + const int64u* const iq_pa[8], + int rsaBitlen, + const mbx_RSA_Method* m, + int8u* pBuffer); +#endif /* RSA_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/sm3.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/sm3.h new file mode 100644 index 0000000..8f7f7d8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/sm3.h @@ -0,0 +1,60 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + + +#ifndef SM3_H +#define SM3_H + +#include +#include + +#define SM3_SIZE_IN_BITS (256) /* sm3 size in bits */ +#define SM3_SIZE_IN_WORDS (SM3_SIZE_IN_BITS/(sizeof(int32u)*8)) /* sm3 hash size in words */ +#define SM3_MSG_BLOCK_SIZE (64) /* messge block size */ + +#define SM3_NUM_BUFFERS (16) /* max number of buffers in sm3 multi-buffer */ + +/* +// sm3 context for mb16 +*/ + +typedef int32u sm3_hash_mb[SM3_SIZE_IN_WORDS][SM3_NUM_BUFFERS]; /* sm3 hash value in multi-buffer format */ + +struct _sm3_context_mb16 { + int msg_buff_idx[SM3_NUM_BUFFERS]; /* buffer entry */ + int64u msg_len[SM3_NUM_BUFFERS]; /* message length */ + int8u msg_buffer[SM3_NUM_BUFFERS][SM3_MSG_BLOCK_SIZE]; /* buffer */ + __ALIGN64 + sm3_hash_mb msg_hash; /* intermediate hash */ +}; + +typedef struct _sm3_context_mb16 SM3_CTX_mb16; + +EXTERN_C mbx_status16 mbx_sm3_init_mb16(SM3_CTX_mb16* p_state); + +EXTERN_C mbx_status16 mbx_sm3_update_mb16(const int8u* const msg_pa[16], + int len[16], + SM3_CTX_mb16* p_state); + +EXTERN_C mbx_status16 mbx_sm3_final_mb16(int8u* hash_pa[16], + SM3_CTX_mb16* p_state); + +EXTERN_C mbx_status16 mbx_sm3_msg_digest_mb16(const int8u* const msg_pa[16], + int len[16], + int8u* hash_pa[16]); + +#endif /* SM3_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/sm4.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/sm4.h new file mode 100644 index 0000000..144fb6f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/sm4.h @@ -0,0 +1,59 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + + +#ifndef SM4_H +#define SM4_H + +#include +#include + +#define SM4_LINES (16) /* Max number of buffers */ +#define SM4_BLOCK_SIZE (16) /* SM4 data block size (bytes) */ +#define SM4_KEY_SIZE (16) /* SM4 key size (bytes) */ +#define SM4_ROUNDS (32) /* SM4 number of rounds */ +#define SM4_XTS_MAX_SIZE ((1 << 20) * SM4_BLOCK_SIZE) /* SM4 max buffer size (bytes) */ + +typedef int8u sm4_key[SM4_KEY_SIZE]; +typedef int8u sm4_xts_key[SM4_KEY_SIZE*2]; +typedef int32u mbx_sm4_key_schedule[SM4_ROUNDS][SM4_LINES]; + +EXTERN_C mbx_status16 mbx_sm4_set_key_mb16(mbx_sm4_key_schedule* key_sched, const sm4_key* pa_key[SM4_LINES]); +EXTERN_C mbx_status16 mbx_sm4_xts_set_keys_mb16(mbx_sm4_key_schedule* key_sched1, mbx_sm4_key_schedule* key_sched2, const sm4_xts_key* pa_key[SM4_LINES]); + +EXTERN_C mbx_status16 mbx_sm4_encrypt_ecb_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched); +EXTERN_C mbx_status16 mbx_sm4_decrypt_ecb_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched); + +EXTERN_C mbx_status16 mbx_sm4_encrypt_cbc_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, const int8u* pa_iv[SM4_LINES]); +EXTERN_C mbx_status16 mbx_sm4_decrypt_cbc_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, const int8u* pa_iv[SM4_LINES]); + +EXTERN_C mbx_status16 mbx_sm4_encrypt_ctr128_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, int8u* pa_ctr[SM4_LINES]); +EXTERN_C mbx_status16 mbx_sm4_decrypt_ctr128_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, int8u* pa_ctr[SM4_LINES]); + +EXTERN_C mbx_status16 mbx_sm4_encrypt_ofb_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, int8u* pa_iv[SM4_LINES]); +EXTERN_C mbx_status16 mbx_sm4_decrypt_ofb_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, int8u* pa_iv[SM4_LINES]); + +EXTERN_C mbx_status16 mbx_sm4_encrypt_cfb128_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, const int8u* pa_iv[SM4_LINES]); +EXTERN_C mbx_status16 mbx_sm4_decrypt_cfb128_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, const int8u* pa_iv[SM4_LINES]); + +EXTERN_C mbx_status16 mbx_sm4_xts_encrypt_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], + const mbx_sm4_key_schedule* key_sched1, const mbx_sm4_key_schedule* key_sched2, + const int8u* pa_tweak[SM4_LINES]); +EXTERN_C mbx_status16 mbx_sm4_xts_decrypt_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], + const mbx_sm4_key_schedule* key_sched1, const mbx_sm4_key_schedule* key_sched2, + const int8u* pa_tweak[SM4_LINES]); +#endif /* SM4_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/sm4_ccm.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/sm4_ccm.h new file mode 100644 index 0000000..45128b7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/sm4_ccm.h @@ -0,0 +1,143 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#ifndef SM4_CCM_H +#define SM4_CCM_H + +#include + +#include + +#define MIN_CCM_IV_LENGTH 7 +#define MAX_CCM_IV_LENGTH 13 +#define MIN_CCM_TAG_LENGTH 4 +#define MAX_CCM_TAG_LENGTH 16 +#define MAX_CCM_AAD_LENGTH 65280 /* 2^16 - 2^8 */ + +#define SM4_CCM_CONTEXT_BUFFER_SLOT_TYPE int64u + +#define SM4_CCM_CONTEXT_BUFFER_SLOT_SIZE_BYTES (sizeof(SM4_CCM_CONTEXT_BUFFER_SLOT_TYPE)) +#define SM4_CCM_CONTEXT_BUFFER_SIZE_BYTES ((SM4_LINES * SM4_BLOCK_SIZE) / SM4_CCM_CONTEXT_BUFFER_SLOT_SIZE_BYTES) + +/* +// Enum to control call sequence +// +// Valid call sequence: +// +// 1) mbx_sm4_ccm_init_mb16 +// 2) mbx_sm4_ccm_update_aad_mb16 – optional +// 3) mbx_sm4_ccm_encrypt_mb16/mbx_sm4_ccm_decrypt_mb16 – optional, can be called as many times as necessary +// 4) mbx_sm4_ccm_get_tag_mb16 +// +// Call sequence restrictions: +// +// * mbx_sm4_ccm_get_tag_mb16 can be called after mbx_sm4_ccm_init_mb16 has been called. +// * functions at steps 2-3 can be called as many times as needed to process payload while this functions processes buffers +// with full blocks (Blocks of 16 bytes size) or empty buffers and length of processed payload is not overflowed. +// * if functions at steps 2-3 called to process a partial block, it can’t be called again. +// * if mbx_sm4_ccm_encrypt_mb16 or mbx_sm4_ccm_decrypt_mb16 was called, mbx_sm4_ccm_update_aad_mb16 can’t be called. +// * if mbx_sm4_ccm_encrypt_mb16 was called, mbx_sm4_ccm_decrypt_mb16 can’t be called. +// * if mbx_sm4_ccm_decrypt_mb16 was called, mbx_sm4_ccm_encrypt_mb16 can’t be called. +*/ +typedef enum { sm4_ccm_update_aad = 0xF0A1, sm4_ccm_start_encdec, sm4_ccm_enc, sm4_ccm_dec, sm4_ccm_get_tag } sm4_ccm_state; + +struct _sm4_ccm_context_mb16 { + int64u msg_len[SM4_LINES]; /* Message length (in bytes) of all lines */ + int64u total_processed_len[SM4_LINES]; /* Total processed plaintext/ciphertext length (in bytes) of all lines */ + int tag_len[SM4_LINES]; /* Tag length (in bytes) of all lines */ + int iv_len[SM4_LINES]; /* Total IV length (in bytes) of all lines */ + __m128i ctr0[SM4_LINES]; /* CTR0 content */ + __m128i ctr[SM4_LINES]; /* CTR content */ + __m128i hash[SM4_LINES]; /* hash value accumulator for AAD and TXT processing */ + + mbx_sm4_key_schedule key_sched; /* SM4 key schedule */ + sm4_ccm_state state; /* call sequence state */ +}; + +typedef struct _sm4_ccm_context_mb16 SM4_CCM_CTX_mb16; + +/* + * Initializes SM4-CCM context. + * + * @param[in] pa_key Array of key pointers + * @param[in] pa_iv Array of IV pointers + * @param[in] iv_len Array of IV lengths + * @param[in] tag_len Array of authentication tag lengths + * @param[in] msg_len Array of total message lengths + * @param[in/out] p_context SM4-CCM context + * + * @return Bitmask of operation status + */ +EXTERN_C mbx_status16 mbx_sm4_ccm_init_mb16(const sm4_key *const pa_key[SM4_LINES], + const int8u *const pa_iv[SM4_LINES], + const int iv_len[SM4_LINES], + const int tag_len[SM4_LINES], + const int64u msg_len[SM4_LINES], + SM4_CCM_CTX_mb16 *p_context); +/* + * Digests additional authenticated data (AAD) for 16 buffers + * + * @param[in] pa_aad Array of AAD pointers + * @param[in] aad_len Array of AAD lengths + * @param[in/out] p_context SM4-CCM context + * + * @return Bitmask of operation status + */ +EXTERN_C mbx_status16 mbx_sm4_ccm_update_aad_mb16(const int8u *const pa_aad[SM4_LINES], + const int aad_len[SM4_LINES], + SM4_CCM_CTX_mb16 *p_context); +/* + * Retrieves authentication tag for 16 buffers + * + * @param[out] pa_tag Array of authentication tag pointers + * @param[in] tag_len Array of tag lengths + * @param[in/out] p_context SM4-CCM context + * + * @return Bitmask of operation status + */ +EXTERN_C mbx_status16 mbx_sm4_ccm_get_tag_mb16(int8u *pa_tag[SM4_LINES], + const int tag_len[SM4_LINES], + SM4_CCM_CTX_mb16 *p_context); +/* + * Encrypts 16 buffers with SM4-CCM. + * + * @param[out] pa_out Array of ciphertext pointers + * @param[in] pa_in Array of plaintext pointers + * @param[in] in_len Array of plaintext lengths + * @param[in/out] p_context SM4-CCM context + * + * @return Bitmask of operation status + */ +EXTERN_C mbx_status16 mbx_sm4_ccm_encrypt_mb16(int8u *pa_out[SM4_LINES], + const int8u *const pa_in[SM4_LINES], + const int in_len[SM4_LINES], + SM4_CCM_CTX_mb16 *p_context); +/* + * Decrypts 16 buffers with SM4-CCM. + * + * @param[out] pa_out Array of plaintext pointers + * @param[in] pa_in Array of ciphertext pointers + * @param[in] in_len Array of ciphertext lengths + * @param[in/out] p_context SM4-CCM context + * + * @return Bitmask of operation status + */ +EXTERN_C mbx_status16 mbx_sm4_ccm_decrypt_mb16(int8u *pa_out[SM4_LINES], + const int8u *const pa_in[SM4_LINES], + const int in_len[SM4_LINES], + SM4_CCM_CTX_mb16 *p_context); +#endif /* SM4_CCM_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/sm4_gcm.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/sm4_gcm.h new file mode 100644 index 0000000..9dede98 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/sm4_gcm.h @@ -0,0 +1,113 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#ifndef SM4_GCM_H +#define SM4_GCM_H + +#include + +#include + +#define SM4_GCM_CONTEXT_BUFFER_SLOT_TYPE int64u + +#define SM4_GCM_CONTEXT_BUFFER_SLOT_SIZE_BYTES (sizeof(SM4_GCM_CONTEXT_BUFFER_SLOT_TYPE)) +#define SM4_GCM_CONTEXT_BUFFER_SIZE_BYTES ((SM4_LINES * SM4_BLOCK_SIZE) / SM4_GCM_CONTEXT_BUFFER_SLOT_SIZE_BYTES) + +#define SM4_GCM_HASHKEY_PWR_NUM 8 + +/* +// Enum to control call sequence +// +// Valid call sequence: +// +// 1) mbx_sm4_gcm_init_mb16 +// 2) mbx_sm4_gcm_update_iv_mb16 – optional, can be called as many times as necessary +// 3) mbx_sm4_gcm_update_aad_mb16 – optional, can be called as many times as necessary +// 4) mbx_sm4_gcm_encrypt_mb16/mbx_sm4_gcm_decrypt_mb16 – optional, can be called as many times as necessary +// 5) mbx_sm4_gcm_get_tag_mb16 +// +// Call sequence restrictions: +// +// * mbx_sm4_gcm_get_tag_mb16 can be called after IV is fully processed. +// IV is fully processed if buffer with partial block (Block of less than 16 bytes size) was processed or if mbx_sm4_gcm_update_aad_mb16 was called +// * functions at steps 2-4 can be called as many times as needed to process payload while this functions processes buffers +// with full blocks (Blocks of 16 bytes size) or empty buffers and length of processed payload is not overflowed. +// * if functions at steps 2-4 called to process a partial block, it can’t be called again. +// * if mbx_sm4_gcm_update_aad_mb16 was called, mbx_sm4_gcm_update_iv_mb16 can’t be called. +// * if mbx_sm4_gcm_encrypt_mb16 or mbx_sm4_gcm_decrypt_mb16 was called, mbx_sm4_gcm_update_aad_mb16 and mbx_sm4_gcm_update_iv_mb16 can’t be called. +// * if mbx_sm4_gcm_encrypt_mb16 was called, mbx_sm4_gcm_decrypt_mb16 can’t be called. +// * if mbx_sm4_gcm_decrypt_mb16 was called, mbx_sm4_gcm_encrypt_mb16 can’t be called. +*/ +typedef enum { sm4_gcm_update_iv = 0xF0A1, sm4_gcm_update_aad, sm4_gcm_start_encdec, sm4_gcm_enc, sm4_gcm_dec, sm4_gcm_get_tag } sm4_gcm_state; + +struct _sm4_gcm_context_mb16 { + __m128i hashkey[SM4_GCM_HASHKEY_PWR_NUM][SM4_LINES]; /* Set of hashkeys for ghash computation */ + __m128i j0[SM4_LINES]; /* J0 value accumulator for IV processing */ + __m128i ghash[SM4_LINES]; /* ghash value accumulator for AAD and TXT processing */ + __m128i ctr[SM4_LINES]; /* counter for gctr encryption */ + + /* + // buffer to store IV, AAD and TXT length in bytes + // + // this buffer is used to store IV length to compute J0 block + // and reused to store AAD and TXT length to compute ghash + // + // length is stored as follow: + // + // J0 computation: + // [64 bits with IV len (buffer 0)] + // [64 bits with IV len (buffer 1)] + // .. + // [64 bits with IV len (buffer SM4_LINES-1)] + // + // Only half of buffer is used for J0 computation + // + // ghash computation: + // [64 bits with AAD len (buffer 0)][64 bits with TXT len (buffer 0)] + // [64 bits with AAD len (buffer 1)][64 bits with TXT len (buffer 1)] + // .. + // [64 bits with AAD len (buffer SM4_LINES-1)][64 bits with TXT len (buffer SM4_LINES-1)] + // + */ + int64u len[SM4_LINES * 2]; + + mbx_sm4_key_schedule key_sched; /* SM4 key schedule */ + sm4_gcm_state state; /* call sequence state */ +}; + +typedef struct _sm4_gcm_context_mb16 SM4_GCM_CTX_mb16; + +EXTERN_C mbx_status16 mbx_sm4_gcm_init_mb16(const sm4_key *const pa_key[SM4_LINES], + const int8u *const pa_iv[SM4_LINES], + const int iv_len[SM4_LINES], + SM4_GCM_CTX_mb16 *p_context); + +EXTERN_C mbx_status16 mbx_sm4_gcm_update_iv_mb16(const int8u *const pa_iv[SM4_LINES], const int iv_len[SM4_LINES], SM4_GCM_CTX_mb16 *p_state); +EXTERN_C mbx_status16 mbx_sm4_gcm_update_aad_mb16(const int8u *const pa_aad[SM4_LINES], const int aad_len[SM4_LINES], SM4_GCM_CTX_mb16 *p_state); + +EXTERN_C mbx_status16 mbx_sm4_gcm_encrypt_mb16(int8u *pa_out[SM4_LINES], + const int8u *const pa_in[SM4_LINES], + const int in_len[SM4_LINES], + SM4_GCM_CTX_mb16 *p_context); +EXTERN_C mbx_status16 mbx_sm4_gcm_decrypt_mb16(int8u *pa_out[SM4_LINES], + const int8u *const pa_in[SM4_LINES], + const int in_len[SM4_LINES], + SM4_GCM_CTX_mb16 *p_context); + +EXTERN_C mbx_status16 mbx_sm4_gcm_get_tag_mb16(int8u *pa_tag[SM4_LINES], const int tag_len[SM4_LINES], SM4_GCM_CTX_mb16 *p_context); + +#endif /* SM4_GCM_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/status.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/status.h new file mode 100644 index 0000000..add3b5c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/status.h @@ -0,0 +1,117 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef STATUS_H +#define STATUS_H + +#include + +typedef int32u mbx_status; +typedef int64u mbx_status16; + +// error statuses and manipulators +#define MBX_STATUS_OK (0) +#define MBX_STATUS_MISMATCH_PARAM_ERR (1) +#define MBX_STATUS_NULL_PARAM_ERR (2) +#define MBX_STATUS_LOW_ORDER_ERR (4) +#define MBX_STATUS_SIGNATURE_ERR (8) + +__INLINE mbx_status MBX_SET_STS(mbx_status status, int numb, mbx_status sttVal) +{ + numb &= 7; /* 0 <= numb < 8 */ + status &= (mbx_status)(~(0xF << (numb*4))); + return status |= (sttVal & 0xF) << (numb*4); +} + +__INLINE mbx_status MBX_GET_STS(mbx_status status, int numb) +{ + return (status >>(numb*4)) & 0xF; +} +__INLINE mbx_status MBX_SET_STS_ALL(mbx_status stsVal) +{ + return (stsVal<<4*7) | (stsVal<<4*6) | (stsVal<<4*5) | (stsVal<<4*4) | (stsVal<<4*3) | (stsVal<<4*2) | (stsVal<<4*1) | stsVal; +} + +__INLINE mbx_status MBX_SET_STS_BY_MASK(mbx_status status, int8u mask, mbx_status sttVal) +{ + int numb; + + for(numb=0; numb<8; numb++) { + mbx_status buf_stt = (0 - ((mask>>numb) &1)) & sttVal; + status = MBX_SET_STS(status, numb, buf_stt); + } + return status; +} + +__INLINE int MBX_IS_ANY_OK_STS(mbx_status status) +{ + int ret = MBX_STATUS_OK==MBX_GET_STS(status, 0) + || MBX_STATUS_OK==MBX_GET_STS(status, 1) + || MBX_STATUS_OK==MBX_GET_STS(status, 2) + || MBX_STATUS_OK==MBX_GET_STS(status, 3) + || MBX_STATUS_OK==MBX_GET_STS(status, 4) + || MBX_STATUS_OK==MBX_GET_STS(status, 5) + || MBX_STATUS_OK==MBX_GET_STS(status, 6) + || MBX_STATUS_OK==MBX_GET_STS(status, 7); + return ret; +} + +/* +// Helpers for 64-bit status mbx_status16 +*/ + +/* Accessors for the low and high part of 64-bit status */ +__INLINE mbx_status MBX_GET_HIGH_PART_STS16(mbx_status16 status16) +{ + return ((mbx_status)(((mbx_status16)(status16) >> 32) & 0xFFFFFFFF)); +} + +__INLINE mbx_status MBX_GET_LOW_PART_STS16(mbx_status16 status16) +{ + return ((mbx_status)(status16)); +} + +__INLINE mbx_status16 MBX_SET_STS16_ALL(mbx_status16 stsVal) +{ + return (stsVal<<4*15) | (stsVal<<4*14) | (stsVal<<4*13) | (stsVal<<4*12) | (stsVal<<4*11) | (stsVal<<4*10) | (stsVal<<4*9) | (stsVal<<4*8) | \ + (stsVal<<4*7) | (stsVal<<4*6) | (stsVal<<4*5) | (stsVal<<4*4) | (stsVal<<4*3) | (stsVal<<4*2) | (stsVal<<4*1) | stsVal; +} + +__INLINE mbx_status16 MBX_SET_STS16(mbx_status16 status, int numb, mbx_status16 sttVal) +{ + numb &= 15; /* 0 <= numb < 16 */ + status &= (mbx_status16)(~((int64u)0xF << (numb*4))); + return status |= (sttVal & 0xF) << (numb*4); +} + +__INLINE mbx_status16 MBX_SET_STS16_BY_MASK(mbx_status16 status, int16u mask, mbx_status16 sttVal) +{ + int numb; + for (numb = 0; numb < 16; numb++) { + mbx_status16 buf_stt = (0 - ((mask >> numb) & 1)) & sttVal; + status = MBX_SET_STS16(status, numb, buf_stt); + } + return status; +} + +__INLINE int MBX_IS_ANY_OK_STS16(mbx_status16 status) +{ + return MBX_IS_ANY_OK_STS(MBX_GET_HIGH_PART_STS16(status)) || \ + MBX_IS_ANY_OK_STS(MBX_GET_LOW_PART_STS16(status)); +} + +#endif /* STATUS_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/version.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/version.h new file mode 100644 index 0000000..8e6c863 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/version.h @@ -0,0 +1,45 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef VERSION_H +#define VERSION_H + +#include + +/* crypto_mb name & version */ +#define MBX_LIB_NAME() "crypto_mb" +#define MBX_VER_MAJOR 1 +#define MBX_VER_MINOR 0 +#define MBX_VER_REV 9 + +/* major interface version */ +#define MBX_INTERFACE_VERSION_MAJOR 11 +/* minor interface version */ +#define MBX_INTERFACE_VERSION_MINOR 9 + +typedef struct { + int major; /* e.g. 1 */ + int minor; /* e.g. 2 */ + int revision; /* e.g. 3 */ + const char* name; /* e,g. "crypto_mb" */ + const char* buildDate; /* e.g. "Oct 28 2019" */ + const char* strVersion;/* e.g. "crypto_mb (ver 1.2.3 Oct 28 2019)" */ +} mbxVersion; + +EXTERN_C const mbxVersion* mbx_getversion(void); + +#endif /* VERSION_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/x25519.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/x25519.h new file mode 100644 index 0000000..ab0231f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/crypto_mb/x25519.h @@ -0,0 +1,31 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef X25519_H +#define X25519_H + +#include +#include + +EXTERN_C mbx_status mbx_x25519_public_key_mb8(int8u* const pa_public_key[8], + const int8u* const pa_private_key[8]); + +EXTERN_C mbx_status mbx_x25519_mb8(int8u* const pa_shared_key[8], + const int8u* const pa_private_key[8], + const int8u* const pa_public_key[8]); + +#endif /* X25519_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/common/crypto_mb_ver.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/common/crypto_mb_ver.h new file mode 100644 index 0000000..f1c1e1e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/common/crypto_mb_ver.h @@ -0,0 +1,46 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +#define STR2(x) #x +#define STR(x) STR2(x) + +#define MBX_LIB_LONGNAME() "Crypto multi-buffer" +#define MBX_LIB_SHORTNAME() "crypto_mb" + +#ifndef MBX_BASE_VERSION +#define MBX_BASE_VERSION() MBX_VER_MAJOR,MBX_VER_MINOR,MBX_VER_REV +#endif + +#define MBX_BUILD() 1043 +#define MBX_VERSION() MBX_BASE_VERSION(),MBX_BUILD() + +#ifndef MBX_STR_VERSION +#define MBX_STR_VERSION() STR(MBX_VER_MAJOR) "." STR(MBX_VER_MINOR) "." STR(MBX_VER_REV) " (" STR(MBX_INTERFACE_VERSION_MAJOR) "." STR(MBX_INTERFACE_VERSION_MINOR) ")" +#endif + +#ifndef CRYPTO_MB_STR_VERSION + #ifdef MBX_REVISION + #define CRYPTO_MB_STR_VERSION() MBX_STR_VERSION() " (r" STR( MBX_REVISION ) ")" + #else + #define CRYPTO_MB_STR_VERSION() MBX_STR_VERSION() " (-)" + #endif +#endif + + +/* ////////////////////////////// End of file /////////////////////////////// */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/common/ifma_cvt52.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/common/ifma_cvt52.h new file mode 100644 index 0000000..5a81378 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/common/ifma_cvt52.h @@ -0,0 +1,43 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_CVT52_H +#define IFMA_CVT52_H + +#ifndef BN_OPENSSL_DISABLE + #include +#endif + +#include + +// from 8 buffers regular (radix2^64) to mb8 redundant (radix 2^52) representation +EXTERN_C int8u ifma_BNU_to_mb8(int64u out_mb8[][8], const int64u* const bn[8], int bitLen); +EXTERN_C int8u ifma_HexStr8_to_mb8(int64u out_mb8[][8], const int8u* const pStr[8], int bitLen); +#ifndef BN_OPENSSL_DISABLE +EXTERN_C int8u ifma_BN_to_mb8(int64u res[][8], const BIGNUM* const bn[8], int bitLen); +#endif + +// from 8 buffers mb8 redundant (radix 2^52) to regular (radix2^64) representation +EXTERN_C int8u ifma_mb8_to_BNU(int64u* const out_bn[8], const int64u inp_mb8[][8], const int bitLen); +EXTERN_C int8u ifma_mb8_to_HexStr8(int8u* const pStr[8], const int64u inp_mb8[][8], int bitLen); + +EXTERN_C int8u ifma_BNU_transpose_copy(int64u out_mb8[][8], const int64u* const inp[8], int bitLen); +#ifndef BN_OPENSSL_DISABLE +EXTERN_C int8u ifma_BN_transpose_copy(int64u out_mb8[][8], const BIGNUM* const inp[8], int bitLen); +#endif /* BN_OPENSSL_DISABLE */ + +#endif /* IFMA_CVT52_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/common/ifma_defs.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/common/ifma_defs.h new file mode 100644 index 0000000..a9ae832 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/common/ifma_defs.h @@ -0,0 +1,84 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_DEFS_H +#define IFMA_DEFS_H + +#include + +/* define DLL_EXPORT */ +#if defined(__GNUC__) || defined(__CLANG__) + #define DLL_PUBLIC __attribute__ ((visibility ("default"))) + #define DLL_LOCAL __attribute__ ((visibility ("hidden"))) +#else + #define DLL_PUBLIC + #define DLL_LOCAL +#endif + +/* define optimization on/off VS2019 (>= 19.27) */ +#if defined(_MSC_VER) && (_MSC_VER >= 1927) && !defined(__INTEL_COMPILER) && !defined(__INTEL_LLVM_COMPILER) + #define OPTIMIZE_OFF_VS19 _Pragma("optimize( \"\", off )") + #define OPTIMIZE_ON_VS19 _Pragma("optimize( \"\", on )") +#else + #define OPTIMIZE_OFF_VS19 + #define OPTIMIZE_ON_VS19 +#endif + +/* define SIMD_LEN if not set (Default is 512 bit AVX) */ +#ifndef SIMD_LEN + #define SIMD_LEN 512 +#endif + +#if ((SIMD_LEN != 512) && (SIMD_LEN != 256)) + #error "Incorrect SIMD length" +#endif + +/* internal function names */ +#if (SIMD_LEN == 512) + #define FUNC_SUFFIX mb8 + #define MB_FUNC_NAME(name) name ## mb8 +#else + #define FUNC_SUFFIX mb4 + #define MB_FUNC_NAME(name) name ## mb4 +#endif + +#define SIMD_TYPE(LEN) typedef __m ## LEN ## i U64; + +/* max internal data bitsize */ +#define IFMA_MAX_BITSIZE (4096) + +/* internal radix definition */ +#define DIGIT_SIZE (52) +#define DIGIT_BASE ((int64u)1< + #include + + #if (SIMD_LEN == 512) + SIMD_TYPE(512) + typedef __mmask8 __mb_mask; + + #define SIMD_LEN 512 + #define SIMD_BYTES (SIMD_LEN/8) + #define MB_WIDTH (SIMD_LEN/64) + + __INLINE U64 loadu64(const void *p) { + return _mm512_loadu_si512((U64*)p); + } + + __INLINE U64 loadstream64(const void *p) { + return _mm512_stream_load_si512 ((U64*)p); + } + + __INLINE void storeu64(const void *p, U64 v) { + _mm512_storeu_si512((U64*)p, v); + } + + #define mask_mov64 _mm512_mask_mov_epi64 + #define set64 _mm512_set1_epi64 + + __INLINE U64 fma52lo(U64 a, U64 b, U64 c) { + return _mm512_madd52lo_epu64(a, b, c); + } + + __INLINE U64 fma52hi(U64 a, U64 b, U64 c) { + return _mm512_madd52hi_epu64(a, b, c); + } + + __INLINE U64 mul52lo(U64 b, U64 c) { + return _mm512_madd52lo_epu64(_mm512_setzero_si512(), b, c); + } + + #ifdef __GNUC__ + // memory ops intrinsics - force load from original buffer + #define _mm512_madd52lo_epu64_(r, a, b, c, o) {\ + r=a; \ + __asm__ ( "vpmadd52luq " #o "(%2), %1, %0" : "+x" (r): "x" (b), "r" (c) ); \ + } + + #define _mm512_madd52hi_epu64_(r, a, b, c, o) {\ + r=a; \ + __asm__ ( "vpmadd52huq " #o "(%2), %1, %0" : "+x" (r): "x" (b), "r" (c) ); \ + } + + __INLINE U64 select64(__mb_mask k, U64 v, U64 *d) { + __asm__("vmovdqu64 %2, %%zmm0 \n" + "vpblendmq %%zmm0, %0, %0 %{%1%} \n" + : "+v"(v) + : "Yk"(k), "m"(*d) + : "zmm0"); + return v; + } + + #else + // Use IFMA instrinsics for all other compilers + #define _mm512_madd52lo_epu64_(r, a, b, c, o) {\ + r=fma52lo(a, b, _mm512_loadu_si512((U64*)(((char*)c)+o))); \ + } + + #define _mm512_madd52hi_epu64_(r, a, b, c, o) {\ + r=fma52hi(a, b, _mm512_loadu_si512((U64*)(((char*)c)+o))); \ + } + + #pragma optimize("", off) + __INLINE U64 select64(__mb_mask k, U64 v, U64 *d) { + return _mm512_mask_blend_epi64(k, v, _mm512_load_si512(d)); + } + + #pragma optimize("", on) + #endif + + #define fma52lo_mem(r, a, b, c, o) _mm512_madd52lo_epu64_(r, a, b, c, o) // gres + #define fma52hi_mem(r, a, b, c, o) _mm512_madd52hi_epu64_(r, a, b, c, o) // gres + + __INLINE U64 add64(U64 a, U64 b) { + return _mm512_add_epi64(a, b); + } + + __INLINE U64 sub64(U64 a, U64 b) { + return _mm512_sub_epi64(a, b); + } + + __INLINE U64 get_zero64() { + return _mm512_setzero_si512(); + } + + __INLINE void set_zero64(U64 *a) { + *a = _mm512_xor_si512(*a, *a); + } + + __INLINE U64 set1(unsigned long long a) { + return _mm512_set1_epi64((long long)a); + } + + __INLINE U64 srli64(U64 a, int s) { + return _mm512_srli_epi64(a, s); + } + + #define srai64 _mm512_srai_epi64 + #define slli64 _mm512_slli_epi64 + + __INLINE U64 and64_const(U64 a, unsigned long long mask) { + return _mm512_and_epi64(a, _mm512_set1_epi64((long long)mask)); + } + + __INLINE U64 and64(U64 a, U64 mask) { + return _mm512_and_epi64(a, mask); + } + + #define or64 _mm512_or_epi64 + #define xor64 _mm512_xor_epi64 + #define cmp64_mask _mm512_cmp_epi64_mask + #define cmpeq16_mask _mm512_cmpeq_epi16_mask + #define cmpeq64_mask _mm512_cmpeq_epi64_mask + + // Mask operations + #define mask_blend64 _mm512_mask_blend_epi64 + #define mask_add64 _mm512_mask_add_epi64 + #define mask_sub64 _mm512_mask_sub_epi64 + #define maskz_sub64 _mm512_maskz_sub_epi64 + + __INLINE __mb_mask is_zero(U64* p, int len) { + U64 Z = p[0]; + for(int i = 1; i < len; i++) { + Z = or64(Z, p[i]); + } + + return cmpeq64_mask(Z, get_zero64()); + } + + #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && !defined(__INTEL_LLVM_COMPILER) // for MSVC + #define mask_xor(m1,m2) (__mb_mask)(_mm512_kxor((m1),(m2))) + #else + #define mask_xor _kxor_mask8 + #endif + + #define get_mask(a) (a) + #define get_mask_value(a) (a) + + #define TRANSPOSE_8xI64x8(X0_, X1_ ,X2_ ,X3_ ,X4_ ,X5_ ,X6_ ,X7_) {\ + __m512i X01L = _mm512_unpacklo_epi64(X0_, X1_); \ + __m512i X23L = _mm512_unpacklo_epi64(X2_, X3_); \ + __m512i X45L = _mm512_unpacklo_epi64(X4_, X5_); \ + __m512i X67L = _mm512_unpacklo_epi64(X6_, X7_); \ + \ + __m512i X01H = _mm512_unpackhi_epi64(X0_, X1_); \ + __m512i X23H = _mm512_unpackhi_epi64(X2_, X3_); \ + __m512i X45H = _mm512_unpackhi_epi64(X4_, X5_); \ + __m512i X67H = _mm512_unpackhi_epi64(X6_, X7_); \ + \ + __m512i X4567L, X0123L, X4567H, X0123H; \ + X4567L = _mm512_shuffle_i64x2(X45L, X67L, 0b01000100 ); \ + X0_ = _mm512_mask_shuffle_i64x2(X01L, 0b11111100, X23L, X4567L, 0b10000000 ); \ + X2_ = _mm512_mask_shuffle_i64x2(X23L, 0b11110011, X01L, X4567L, 0b11010001 ); \ + \ + X0123L = _mm512_shuffle_i64x2(X01L, X23L, 0b11101110 ); \ + X4_ = _mm512_mask_shuffle_i64x2(X45L, 0b11001111, X0123L, X67L, 0b10001000 ); \ + X6_ = _mm512_mask_shuffle_i64x2(X67L, 0b00111111, X0123L, X45L, 0b10111101 ); \ + \ + X4567H = _mm512_shuffle_i64x2(X45H, X67H, 0b01000100 ); \ + X1_ = _mm512_mask_shuffle_i64x2(X01H, 0b11111100, X23H, X4567H, 0b10000000 ); \ + X3_ = _mm512_mask_shuffle_i64x2(X23H, 0b11110011, X01H, X4567H, 0b11010001 ); \ + \ + X0123H = _mm512_shuffle_i64x2(X01H, X23H, 0b11101110 ); \ + X5_ = _mm512_mask_shuffle_i64x2(X45H, 0b11001111, X0123H, X67H, 0b10001000 ); \ + X7_ = _mm512_mask_shuffle_i64x2(X67H, 0b00111111, X0123H, X45H, 0b10111101 ); \ + } + + #else + #error "Incorrect SIMD length" + #endif // SIMD_LEN + +#endif // IFMA_MATH_H diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/common/mem_fns.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/common/mem_fns.h new file mode 100644 index 0000000..fda962b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/common/mem_fns.h @@ -0,0 +1,36 @@ +/******************************************************************************* +* Copyright (C) 2021-2022 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* + * Auxiliary functions to set and copy memory + */ +__INLINE void CopyBlock(const void* pSrc, void* pDst, int numBytes) +{ + const int8u* s = (int8u*)pSrc; + int8u* d = (int8u*)pDst; + int k; + for (k = 0; k < numBytes; k++) + d[k] = s[k]; +} + +__INLINE void PadBlock(int8u paddingByte, void* pDst, int numBytes) +{ + int8u* d = (int8u*)pDst; + int k; + for (k = 0; k < numBytes; k++) + d[k] = paddingByte; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_arith_p256.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_arith_p256.h new file mode 100644 index 0000000..0968e52 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_arith_p256.h @@ -0,0 +1,173 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_ARITH_P256_H +#define IFMA_ARITH_P256_H + +#include +#include + +/* underlying prime's size */ +#define P256_BITSIZE (256) + +/* lengths of FF elements */ +#define P256_LEN52 NUMBER_OF_DIGITS(P256_BITSIZE,DIGIT_SIZE) +#define P256_LEN64 NUMBER_OF_DIGITS(P256_BITSIZE,64) + +__ALIGN64 static const int64u ones[P256_LEN52][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(1) }, + { REP8_DECL(0) }, + { REP8_DECL(0) }, + { REP8_DECL(0) }, + { REP8_DECL(0) } +}; + +static const int64u VMASK52[sizeof(U64)/sizeof(int64u)] = { + REP8_DECL(DIGIT_MASK) +}; + +#define NORM_LSHIFTR(R, I, J) \ + R##J = add64(R##J, srli64(R##I, DIGIT_SIZE)); \ + R##I = and64(R##I, loadu64(VMASK52)); + +#define NORM_ASHIFTR(R, I, J) \ + R##J = add64(R##J, srai64(R##I, DIGIT_SIZE)); \ + R##I = and64(R##I, loadu64(VMASK52)); + + +/* set FE to zero */ +__INLINE void MB_FUNC_NAME(zero_FE256_)(U64 T[]) +{ + T[0] = T[1] = T[2] = T[3] = T[4] = get_zero64(); +} + +/* check if FE is zero */ +__INLINE __mb_mask MB_FUNC_NAME(is_zero_FE256_)(const U64 T[]) +{ + U64 Z = or64(or64(T[0], T[1]), or64(or64(T[2], T[3]), T[4])); + return cmpeq64_mask(Z, get_zero64()); +} + +__INLINE U64 cmov_U64(U64 a, U64 b, __mb_mask kmask) +{ return mask_mov64 (a, kmask, b); } + +/* move field element */ +__INLINE void MB_FUNC_NAME(mov_FE256_)(U64 r[], const U64 a[]) +{ + r[0] = a[0]; + r[1] = a[1]; + r[2] = a[2]; + r[3] = a[3]; + r[4] = a[4]; +} + +/* move coodinate using mask: R = k? A : B */ +__INLINE void MB_FUNC_NAME(mask_mov_FE256_)(U64 R[], const U64 B[], __mb_mask k, const U64 A[]) +{ + R[0] = mask_mov64(B[0], k, A[0]); + R[1] = mask_mov64(B[1], k, A[1]); + R[2] = mask_mov64(B[2], k, A[2]); + R[3] = mask_mov64(B[3], k, A[3]); + R[4] = mask_mov64(B[4], k, A[4]); +} + +__INLINE void MB_FUNC_NAME(secure_mask_mov_FE256_)(U64 R[], U64 B[], __mb_mask k, const U64 A[]) +{ + R[0] = select64(k, B[0], (U64*)(&A[0])); + R[1] = select64(k, B[1], (U64*)(&A[1])); + R[2] = select64(k, B[2], (U64*)(&A[2])); + R[3] = select64(k, B[3], (U64*)(&A[3])); + R[4] = select64(k, B[4], (U64*)(&A[4])); +} + +/* compare two FE */ +__INLINE __mb_mask MB_FUNC_NAME(cmp_lt_FE256_)(const U64 A[], const U64 B[]) +{ + /* r = a - b */ + U64 r0 = sub64(A[0], B[0]); + U64 r1 = sub64(A[1], B[1]); + U64 r2 = sub64(A[2], B[2]); + U64 r3 = sub64(A[3], B[3]); + U64 r4 = sub64(A[4], B[4]); + + /* normalize r0 – r4 */ + NORM_ASHIFTR(r, 0, 1) + NORM_ASHIFTR(r, 1, 2) + NORM_ASHIFTR(r, 2, 3) + NORM_ASHIFTR(r, 3, 4) + + /* return mask LT */ + return cmp64_mask(r4, get_zero64(), _MM_CMPINT_LT); +} + +__INLINE __mb_mask MB_FUNC_NAME(cmp_eq_FE256_)(const U64 A[], const U64 B[]) +{ + __ALIGN64 U64 msg[P256_LEN52]; + + msg[0] = xor64(A[0], B[0]); + msg[1] = xor64(A[1], B[1]); + msg[2] = xor64(A[2], B[2]); + msg[3] = xor64(A[3], B[3]); + msg[4] = xor64(A[4], B[4]); + + return MB_FUNC_NAME(is_zero_FE256_)(msg); +} + +/* General 256-bit operations */ +EXTERN_C void MB_FUNC_NAME(ifma_amm52x5_)(U64 R[], const U64 inpA[], const U64 inpB[], const U64 inpM[], const int64u* k0_mb); +EXTERN_C void MB_FUNC_NAME(ifma_ams52x5_)(U64 r[], const U64 a[], const U64 m[], const int64u* k0_mb); +EXTERN_C void MB_FUNC_NAME(ifma_add52x5_)(U64 R[], const U64 A[], const U64 B[], const U64 M[]); +EXTERN_C void MB_FUNC_NAME(ifma_sub52x5_)(U64 R[], const U64 A[], const U64 B[], const U64 M[]); +EXTERN_C void MB_FUNC_NAME(ifma_neg52x5_)(U64 R[], const U64 A[], const U64 M[]); + +/* Specialized operations over NIST P256 */ +EXTERN_C void MB_FUNC_NAME(ifma_tomont52_p256_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_frommont52_p256_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_ams52_p256_)(U64 r[], const U64 va[]); +EXTERN_C void MB_FUNC_NAME(ifma_amm52_p256_)(U64 r[], const U64 va[], const U64 vb[]); +EXTERN_C void MB_FUNC_NAME(ifma_aminv52_p256_)(U64 r[], const U64 z[]); +EXTERN_C void MB_FUNC_NAME(ifma_add52_p256_)(U64 r[], const U64 a[], const U64 b[]); +EXTERN_C void MB_FUNC_NAME(ifma_sub52_p256_)(U64 r[], const U64 a[], const U64 b[]); +EXTERN_C void MB_FUNC_NAME(ifma_neg52_p256_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_double52_p256_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_tripple52_p256_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_half52_p256_)(U64 r[], const U64 a[]); + +EXTERN_C void MB_FUNC_NAME(ifma_ams52_p256_dual_)(U64 r0[], U64 r1[], + const U64 inp0[], const U64 inp1[]); +EXTERN_C void MB_FUNC_NAME(ifma_amm52_p256_dual_)(U64 r0[], U64 r1[], + const U64 inp0A[], const U64 inp0B[], + const U64 inp1A[], const U64 inp1B[]); + +EXTERN_C __mb_mask MB_FUNC_NAME(ifma_cmp_lt_p256_)(const U64 a[]); +EXTERN_C __mb_mask MB_FUNC_NAME(ifma_check_range_p256_)(const U64 a[]); + +/* Specialized operations over EC NIST-P256 order */ +EXTERN_C U64* MB_FUNC_NAME(ifma_n256_)(void); +EXTERN_C void MB_FUNC_NAME(ifma_tomont52_n256_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_frommont52_n256_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_ams52_n256_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_amm52_n256_)(U64 r[], const U64 a[], const U64 b[]); +EXTERN_C void MB_FUNC_NAME(ifma_aminv52_n256_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_add52_n256_)(U64 r[], const U64 a[], const U64 b[]); +EXTERN_C void MB_FUNC_NAME(ifma_sub52_n256_)(U64 r[], const U64 a[], const U64 b[]); +EXTERN_C void MB_FUNC_NAME(ifma_neg52_n256_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_fastred52_pn256_)(U64 r[], const U64 a[]); +EXTERN_C __mb_mask MB_FUNC_NAME(ifma_cmp_lt_n256_)(const U64 a[]); +EXTERN_C __mb_mask MB_FUNC_NAME(ifma_check_range_n256_)(const U64 a[]); + +#endif /* IFMA_ARITH_P256_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_arith_p384.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_arith_p384.h new file mode 100644 index 0000000..2098c18 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_arith_p384.h @@ -0,0 +1,182 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_ARITH_P384_H +#define IFMA_ARITH_P384_H + +#include +#include + +/* underlying prime's size */ +#define P384_BITSIZE (384) + +/* lengths of FF elements */ +#define P384_LEN52 NUMBER_OF_DIGITS(P384_BITSIZE,DIGIT_SIZE) +#define P384_LEN64 NUMBER_OF_DIGITS(P384_BITSIZE,64) + +__ALIGN64 static const int64u ones[P384_LEN52][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(1) }, + { REP8_DECL(0) }, + { REP8_DECL(0) }, + { REP8_DECL(0) }, + { REP8_DECL(0) }, + { REP8_DECL(0) }, + { REP8_DECL(0) }, + { REP8_DECL(0) } +}; + + +static const int64u VMASK52[sizeof(U64)/sizeof(int64u)] = { + REP8_DECL(DIGIT_MASK) +}; + +#define NORM_LSHIFTR(R, I, J) \ + R##J = add64(R##J, srli64(R##I, DIGIT_SIZE)); \ + R##I = and64(R##I, loadu64(VMASK52)); + +#define NORM_ASHIFTR(R, I, J) \ + R##J = add64(R##J, srai64(R##I, DIGIT_SIZE)); \ + R##I = and64(R##I, loadu64(VMASK52)); + + +/* set FE to zero */ +__INLINE void MB_FUNC_NAME(zero_FE384_)(U64 T[]) +{ + T[0] = T[1] = T[2] = T[3] = T[4] = T[5] = T[6] = T[7] = get_zero64(); +} + +/* check if FE is zero */ +__INLINE __mb_mask MB_FUNC_NAME(is_zero_FE384_)(const U64 T[]) +{ + //U64 Z = or64(or64(or64(or64(or64(or64(or64(T[0], T[1]), T[2]), T[3]), T[4]), T[5]), T[6]), T[7]); + U64 Z = or64(or64(or64(T[0], T[1]), or64(T[2], T[3])), or64(or64(T[4], T[5]), or64(T[6], T[7]))); + return cmpeq64_mask(Z, get_zero64()); +} + +__INLINE U64 cmov_U64(U64 a, U64 b, __mb_mask kmask) +{ return mask_mov64 (a, kmask, b); } + +/* move field element */ +__INLINE void MB_FUNC_NAME(mov_FE384_)(U64 r[], const U64 a[]) +{ + r[0] = a[0]; + r[1] = a[1]; + r[2] = a[2]; + r[3] = a[3]; + r[4] = a[4]; + r[5] = a[5]; + r[6] = a[6]; + r[7] = a[7]; +} + +/* move coodinate using mask: R = k? A : B */ +__INLINE void MB_FUNC_NAME(mask_mov_FE384_)(U64 R[], const U64 B[], __mb_mask k, const U64 A[]) +{ + R[0] = mask_mov64(B[0], k, A[0]); + R[1] = mask_mov64(B[1], k, A[1]); + R[2] = mask_mov64(B[2], k, A[2]); + R[3] = mask_mov64(B[3], k, A[3]); + R[4] = mask_mov64(B[4], k, A[4]); + R[5] = mask_mov64(B[5], k, A[5]); + R[6] = mask_mov64(B[6], k, A[6]); + R[7] = mask_mov64(B[7], k, A[7]); +} + +__INLINE void MB_FUNC_NAME(secure_mask_mov_FE384_)(U64 R[], U64 B[], __mb_mask k, const U64 A[]) +{ + R[0] = select64(k, B[0], (U64*)(&A[0])); + R[1] = select64(k, B[1], (U64*)(&A[1])); + R[2] = select64(k, B[2], (U64*)(&A[2])); + R[3] = select64(k, B[3], (U64*)(&A[3])); + R[4] = select64(k, B[4], (U64*)(&A[4])); + R[5] = select64(k, B[5], (U64*)(&A[5])); + R[6] = select64(k, B[6], (U64*)(&A[6])); + R[7] = select64(k, B[7], (U64*)(&A[7])); +} + +__INLINE __mb_mask MB_FUNC_NAME(cmp_lt_FE384_)(const U64 A[], const U64 B[]) +{ + /* r = a - b */ + U64 r0 = sub64(A[0], B[0]); + U64 r1 = sub64(A[1], B[1]); + U64 r2 = sub64(A[2], B[2]); + U64 r3 = sub64(A[3], B[3]); + U64 r4 = sub64(A[4], B[4]); + U64 r5 = sub64(A[5], B[5]); + U64 r6 = sub64(A[6], B[6]); + U64 r7 = sub64(A[7], B[7]); + + + /* normalize r0 – r7 */ + NORM_ASHIFTR(r, 0, 1) + NORM_ASHIFTR(r, 1, 2) + NORM_ASHIFTR(r, 2, 3) + NORM_ASHIFTR(r, 3, 4) + NORM_ASHIFTR(r, 4, 5) + NORM_ASHIFTR(r, 5, 6) + NORM_ASHIFTR(r, 6, 7) + + /* return mask LT */ + return cmp64_mask(r7, get_zero64(), _MM_CMPINT_LT); +} + +/* compare two FE */ +__INLINE __mb_mask MB_FUNC_NAME(cmp_eq_FE384_)(const U64 A[], const U64 B[]) +{ + U64 T[P384_LEN52]; + T[0] = xor64(A[0], B[0]); + T[1] = xor64(A[1], B[1]); + T[2] = xor64(A[2], B[2]); + T[3] = xor64(A[3], B[3]); + T[4] = xor64(A[4], B[4]); + T[5] = xor64(A[5], B[5]); + T[6] = xor64(A[6], B[6]); + T[7] = xor64(A[7], B[7]); + + return MB_FUNC_NAME(is_zero_FE384_)(T); +} + +/* Specialized operations over NIST P384 */ +EXTERN_C void MB_FUNC_NAME(ifma_tomont52_p384_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_frommont52_p384_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_ams52_p384_)(U64 r[], const U64 va[]); +EXTERN_C void MB_FUNC_NAME(ifma_amm52_p384_)(U64 r[], const U64 va[], const U64 vb[]); +EXTERN_C void MB_FUNC_NAME(ifma_aminv52_p384_)(U64 r[], const U64 z[]); +EXTERN_C void MB_FUNC_NAME(ifma_add52_p384_)(U64 r[], const U64 a[], const U64 b[]); +EXTERN_C void MB_FUNC_NAME(ifma_sub52_p384_)(U64 r[], const U64 a[], const U64 b[]); +EXTERN_C void MB_FUNC_NAME(ifma_neg52_p384_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_double52_p384_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_tripple52_p384_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_half52_p384_)(U64 r[], const U64 a[]); +EXTERN_C __mb_mask MB_FUNC_NAME(ifma_cmp_lt_p384)(const U64 A[]); +EXTERN_C __mb_mask MB_FUNC_NAME(ifma_check_range_p384_)(const U64 a[]); + +/* Specialized operations over EC NIST-P384 order */ +EXTERN_C U64* MB_FUNC_NAME(ifma_n384_)(void); +EXTERN_C void MB_FUNC_NAME(ifma_tomont52_n384_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_frommont52_n384_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_ams52_n384_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_amm52_n384_)(U64 r[], const U64 a[], const U64 b[]); +EXTERN_C void MB_FUNC_NAME(ifma_aminv52_n384_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_add52_n384_)(U64 r[], const U64 a[], const U64 b[]); +EXTERN_C void MB_FUNC_NAME(ifma_sub52_n384_)(U64 r[], const U64 a[], const U64 b[]); +EXTERN_C void MB_FUNC_NAME(ifma_neg52_n384_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_fastred52_pn384_)(U64 r[], const U64 a[]); +EXTERN_C __mb_mask MB_FUNC_NAME(ifma_cmp_lt_n384)(const U64 A[]); +EXTERN_C __mb_mask MB_FUNC_NAME(ifma_check_range_n384_)(const U64 a[]); + +#endif /* IFMA_ARITH_P384_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_arith_p521.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_arith_p521.h new file mode 100644 index 0000000..063c2e0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_arith_p521.h @@ -0,0 +1,201 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_ARITH_P521_H +#define IFMA_ARITH_P521_H + +#include +#include + +/* underlying prime's size */ +#define P521_BITSIZE (521) + +/* lengths of FF elements */ +#define P521_LEN52 NUMBER_OF_DIGITS(P521_BITSIZE,DIGIT_SIZE) +#define P521_LEN64 NUMBER_OF_DIGITS(P521_BITSIZE,64) + +__ALIGN64 static const int64u ones[P521_LEN52][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(1) }, + { REP8_DECL(0) }, + { REP8_DECL(0) }, + { REP8_DECL(0) }, + { REP8_DECL(0) }, + { REP8_DECL(0) }, + { REP8_DECL(0) }, + { REP8_DECL(0) }, + { REP8_DECL(0) }, + { REP8_DECL(0) }, + { REP8_DECL(0) } +}; + + +static const int64u VMASK52[sizeof(U64)/sizeof(int64u)] = { + REP8_DECL(DIGIT_MASK) +}; + +#define NORM_LSHIFTR(R, I, J) \ + R##J = add64(R##J, srli64(R##I, DIGIT_SIZE)); \ + R##I = and64(R##I, loadu64(VMASK52)); + +#define NORM_ASHIFTR(R, I, J) \ + R##J = add64(R##J, srai64(R##I, DIGIT_SIZE)); \ + R##I = and64(R##I, loadu64(VMASK52)); + + +/* set FE to zero */ +__INLINE void MB_FUNC_NAME(zero_FE521_)(U64 T[]) +{ + T[0] = T[1] = T[2] = T[3] = T[4] = T[5] = T[6] = T[7] = T[8] = T[9] = T[10] = get_zero64(); +} + +/* check if FE is zero */ +__INLINE __mb_mask MB_FUNC_NAME(is_zero_FE521_)(const U64 T[]) +{ + U64 Z = or64(or64(or64(or64(T[0], T[1]), or64(T[2], T[3])), or64(or64(T[4], T[5]), or64(T[6], T[7]))), or64(or64(T[8], T[9]), T[10])); + return cmpeq64_mask(Z, get_zero64()); +} + +__INLINE U64 cmov_U64(U64 a, U64 b, __mb_mask kmask) +{ return mask_mov64 (a, kmask, b); } + +/* move field element */ +__INLINE void MB_FUNC_NAME(mov_FE521_)(U64 r[], const U64 a[]) +{ + r[0] = a[0]; + r[1] = a[1]; + r[2] = a[2]; + r[3] = a[3]; + r[4] = a[4]; + r[5] = a[5]; + r[6] = a[6]; + r[7] = a[7]; + r[8] = a[8]; + r[9] = a[9]; + r[10]= a[10]; +} + +/* move coodinate using mask: R = k? A : B */ +__INLINE void MB_FUNC_NAME(mask_mov_FE521_)(U64 R[], const U64 B[], __mb_mask k, const U64 A[]) +{ + R[0] = mask_mov64(B[0], k, A[0]); + R[1] = mask_mov64(B[1], k, A[1]); + R[2] = mask_mov64(B[2], k, A[2]); + R[3] = mask_mov64(B[3], k, A[3]); + R[4] = mask_mov64(B[4], k, A[4]); + R[5] = mask_mov64(B[5], k, A[5]); + R[6] = mask_mov64(B[6], k, A[6]); + R[7] = mask_mov64(B[7], k, A[7]); + R[8] = mask_mov64(B[8], k, A[8]); + R[9] = mask_mov64(B[9], k, A[9]); + R[10]= mask_mov64(B[10],k, A[10]); +} + +__INLINE void MB_FUNC_NAME(secure_mask_mov_FE521_)(U64 R[], U64 B[], __mb_mask k, const U64 A[]) +{ + R[0] = select64(k, B[0], (U64*)(&A[0])); + R[1] = select64(k, B[1], (U64*)(&A[1])); + R[2] = select64(k, B[2], (U64*)(&A[2])); + R[3] = select64(k, B[3], (U64*)(&A[3])); + R[4] = select64(k, B[4], (U64*)(&A[4])); + R[5] = select64(k, B[5], (U64*)(&A[5])); + R[6] = select64(k, B[6], (U64*)(&A[6])); + R[7] = select64(k, B[7], (U64*)(&A[7])); + R[8] = select64(k, B[8], (U64*)(&A[8])); + R[9] = select64(k, B[9], (U64*)(&A[9])); + R[10]= select64(k,B[10], (U64*)(&A[10])); +} + +__INLINE __mb_mask MB_FUNC_NAME(cmp_lt_FE521_)(const U64 A[], const U64 B[]) +{ + /* r = a - b */ + U64 r0 = sub64(A[0], B[0]); + U64 r1 = sub64(A[1], B[1]); + U64 r2 = sub64(A[2], B[2]); + U64 r3 = sub64(A[3], B[3]); + U64 r4 = sub64(A[4], B[4]); + U64 r5 = sub64(A[5], B[5]); + U64 r6 = sub64(A[6], B[6]); + U64 r7 = sub64(A[7], B[7]); + U64 r8 = sub64(A[8], B[8]); + U64 r9 = sub64(A[9], B[9]); + U64 r10= sub64(A[10],B[10]); + + /* normalize r0 – r10 */ + NORM_ASHIFTR(r, 0, 1) + NORM_ASHIFTR(r, 1, 2) + NORM_ASHIFTR(r, 2, 3) + NORM_ASHIFTR(r, 3, 4) + NORM_ASHIFTR(r, 4, 5) + NORM_ASHIFTR(r, 5, 6) + NORM_ASHIFTR(r, 6, 7) + NORM_ASHIFTR(r, 7, 8) + NORM_ASHIFTR(r, 8, 9) + NORM_ASHIFTR(r, 9,10) + + /* return mask LT */ + return cmp64_mask(r10, get_zero64(), _MM_CMPINT_LT); +} + +__INLINE __mb_mask MB_FUNC_NAME(cmp_eq_FE521_)(const U64 A[], const U64 B[]) +{ + U64 T[P521_LEN52]; + + T[0] = xor64(A[0], B[0]); + T[1] = xor64(A[1], B[1]); + T[2] = xor64(A[2], B[2]); + T[3] = xor64(A[3], B[3]); + T[4] = xor64(A[4], B[4]); + T[5] = xor64(A[5], B[5]); + T[6] = xor64(A[6], B[6]); + T[7] = xor64(A[7], B[7]); + T[8] = xor64(A[8], B[8]); + T[9] = xor64(A[9], B[9]); + T[10] = xor64(A[10], B[10]); + + return MB_FUNC_NAME(is_zero_FE521_)(T); +} + +/* Specialized operations over NIST P521 */ +EXTERN_C void MB_FUNC_NAME(ifma_tomont52_p521_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_frommont52_p521_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_ams52_p521_)(U64 r[], const U64 va[]); +EXTERN_C void MB_FUNC_NAME(ifma_amm52_p521_)(U64 r[], const U64 va[], const U64 vb[]); +EXTERN_C void MB_FUNC_NAME(ifma_aminv52_p521_)(U64 r[], const U64 z[]); +EXTERN_C void MB_FUNC_NAME(ifma_add52_p521_)(U64 r[], const U64 a[], const U64 b[]); +EXTERN_C void MB_FUNC_NAME(ifma_sub52_p521_)(U64 r[], const U64 a[], const U64 b[]); +EXTERN_C void MB_FUNC_NAME(ifma_neg52_p521_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_double52_p521_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_tripple52_p521_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_half52_p521_)(U64 r[], const U64 a[]); +EXTERN_C __mb_mask MB_FUNC_NAME(ifma_cmp_lt_p521_)(const U64 a[]); +EXTERN_C __mb_mask MB_FUNC_NAME(ifma_check_range_p521_)(const U64 a[]); + +/* Specialized operations over EC NIST-P521 order */ +EXTERN_C U64* MB_FUNC_NAME(ifma_n521_)(void); +EXTERN_C void MB_FUNC_NAME(ifma_tomont52_n521_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_frommont52_n521_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_ams52_n521_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_amm52_n521_)(U64 r[], const U64 a[], const U64 b[]); +EXTERN_C void MB_FUNC_NAME(ifma_aminv52_n521_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_add52_n521_)(U64 r[], const U64 a[], const U64 b[]); +EXTERN_C void MB_FUNC_NAME(ifma_sub52_n521_)(U64 r[], const U64 a[], const U64 b[]); +EXTERN_C void MB_FUNC_NAME(ifma_neg52_n521_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_fastred52_pn521_)(U64 r[], const U64 a[]); +EXTERN_C __mb_mask MB_FUNC_NAME(ifma_cmp_lt_n521_)(const U64 a[]); +EXTERN_C __mb_mask MB_FUNC_NAME(ifma_check_range_n521_)(const U64 a[]); + +#endif /* IFMA_ARITH_P521_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecpoint_p256.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecpoint_p256.h new file mode 100644 index 0000000..73fd2d6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecpoint_p256.h @@ -0,0 +1,95 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_ECPOINT_P256_H +#define IFMA_ECPOINT_P256_H + +#include + +typedef struct { + U64 X[P256_LEN52]; + U64 Y[P256_LEN52]; + U64 Z[P256_LEN52]; +} P256_POINT; + +typedef struct { + U64 x[P256_LEN52]; + U64 y[P256_LEN52]; +} P256_POINT_AFFINE; + +typedef struct { + int64u x[P256_LEN52]; + int64u y[P256_LEN52]; +} SINGLE_P256_POINT_AFFINE; + + + +/* check if coodinate is zero */ +__INLINE __mb_mask MB_FUNC_NAME(is_zero_point_cordinate_)(const U64 T[]) +{ + return MB_FUNC_NAME(is_zero_FE256_)(T); +} + +/* set point to infinity */ +__INLINE void MB_FUNC_NAME(set_point_to_infinity_)(P256_POINT* r) +{ + r->X[0] = r->X[1] = r->X[2] = r->X[3] = r->X[4] = get_zero64(); + r->Y[0] = r->Y[1] = r->Y[2] = r->Y[3] = r->Y[4] = get_zero64(); + r->Z[0] = r->Z[1] = r->Z[2] = r->Z[3] = r->Z[4] = get_zero64(); +} + +/* set point to infinity by mask */ +__INLINE void MB_FUNC_NAME(mask_set_point_to_infinity_)(P256_POINT* r, __mb_mask mask) +{ + U64 zeros = get_zero64(); + + r->X[0] = mask_mov64(r->X[0], mask, zeros); + r->X[1] = mask_mov64(r->X[1], mask, zeros); + r->X[2] = mask_mov64(r->X[2], mask, zeros); + r->X[3] = mask_mov64(r->X[3], mask, zeros); + r->X[4] = mask_mov64(r->X[4], mask, zeros); + + r->Y[0] = mask_mov64(r->Y[0], mask, zeros); + r->Y[1] = mask_mov64(r->Y[1], mask, zeros); + r->Y[2] = mask_mov64(r->Y[2], mask, zeros); + r->Y[3] = mask_mov64(r->Y[3], mask, zeros); + r->Y[4] = mask_mov64(r->Y[4], mask, zeros); + + r->Z[0] = mask_mov64(r->Z[0], mask, zeros); + r->Z[1] = mask_mov64(r->Z[1], mask, zeros); + r->Z[2] = mask_mov64(r->Z[2], mask, zeros); + r->Z[3] = mask_mov64(r->Z[3], mask, zeros); + r->Z[4] = mask_mov64(r->Z[4], mask, zeros); +} + +/* set affine point to infinity */ +__INLINE void MB_FUNC_NAME(set_point_affine_to_infinity_)(P256_POINT_AFFINE* r) +{ + r->x[0] = r->x[1] = r->x[2] = r->x[3] = r->x[4] = get_zero64(); + r->y[0] = r->y[1] = r->y[2] = r->y[3] = r->y[4] = get_zero64(); +} + +EXTERN_C void MB_FUNC_NAME(ifma_ec_nistp256_dbl_point_)(P256_POINT* r, const P256_POINT* p); +EXTERN_C void MB_FUNC_NAME(ifma_ec_nistp256_add_point_)(P256_POINT* r, const P256_POINT* p, const P256_POINT* q); +EXTERN_C void MB_FUNC_NAME(ifma_ec_nistp256_add_point_affine_)(P256_POINT* r, const P256_POINT* p, const P256_POINT_AFFINE* q); +EXTERN_C void MB_FUNC_NAME(ifma_ec_nistp256_mul_point_)(P256_POINT* r, const P256_POINT* p, const U64* scalar); +EXTERN_C void MB_FUNC_NAME(ifma_ec_nistp256_mul_pointbase_)(P256_POINT* r, const U64* scalar); +EXTERN_C void MB_FUNC_NAME(get_nistp256_ec_affine_coords_)(U64 x[], U64 y[], const P256_POINT* P); +EXTERN_C const U64* MB_FUNC_NAME(ifma_ec_nistp256_coord_one_)(void); +EXTERN_C __mb_mask MB_FUNC_NAME(ifma_is_on_curve_p256_)(const P256_POINT* p, int use_jproj_coords); + +#endif /* IFMA_ECPOINT_P256_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecpoint_p384.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecpoint_p384.h new file mode 100644 index 0000000..5434065 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecpoint_p384.h @@ -0,0 +1,104 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_ECPOINT_P384_H +#define IFMA_ECPOINT_P384_H + +#include + +typedef struct { + U64 X[P384_LEN52]; + U64 Y[P384_LEN52]; + U64 Z[P384_LEN52]; +} P384_POINT; + +typedef struct { + U64 x[P384_LEN52]; + U64 y[P384_LEN52]; +} P384_POINT_AFFINE; + +typedef struct { + int64u x[P384_LEN52]; + int64u y[P384_LEN52]; +} SINGLE_P384_POINT_AFFINE; + + + +/* check if coodinate is zero */ +__INLINE __mb_mask MB_FUNC_NAME(is_zero_point_cordinate_)(const U64 T[]) +{ + return MB_FUNC_NAME(is_zero_FE384_)(T); +} + +/* set point to infinity */ +__INLINE void MB_FUNC_NAME(set_point_to_infinity_)(P384_POINT* r) +{ + r->X[0] = r->X[1] = r->X[2] = r->X[3] = r->X[4] = r->X[5] = r->X[6] = r->X[7] = get_zero64(); + r->Y[0] = r->Y[1] = r->Y[2] = r->Y[3] = r->Y[4] = r->Y[5] = r->Y[6] = r->Y[7] = get_zero64(); + r->Z[0] = r->Z[1] = r->Z[2] = r->Z[3] = r->Z[4] = r->Z[5] = r->Z[6] = r->Z[7] = get_zero64(); +} + +/* set point to infinity by mask */ +__INLINE void MB_FUNC_NAME(mask_set_point_to_infinity_)(P384_POINT* r, __mb_mask mask) +{ + U64 zeros = get_zero64(); + + r->X[0] = mask_mov64(r->X[0], mask, zeros); + r->X[1] = mask_mov64(r->X[1], mask, zeros); + r->X[2] = mask_mov64(r->X[2], mask, zeros); + r->X[3] = mask_mov64(r->X[3], mask, zeros); + r->X[4] = mask_mov64(r->X[4], mask, zeros); + r->X[5] = mask_mov64(r->X[5], mask, zeros); + r->X[6] = mask_mov64(r->X[6], mask, zeros); + r->X[7] = mask_mov64(r->X[7], mask, zeros); + + r->Y[0] = mask_mov64(r->Y[0], mask, zeros); + r->Y[1] = mask_mov64(r->Y[1], mask, zeros); + r->Y[2] = mask_mov64(r->Y[2], mask, zeros); + r->Y[3] = mask_mov64(r->Y[3], mask, zeros); + r->Y[4] = mask_mov64(r->Y[4], mask, zeros); + r->Y[5] = mask_mov64(r->Y[5], mask, zeros); + r->Y[6] = mask_mov64(r->Y[6], mask, zeros); + r->Y[7] = mask_mov64(r->Y[7], mask, zeros); + + r->Z[0] = mask_mov64(r->Z[0], mask, zeros); + r->Z[1] = mask_mov64(r->Z[1], mask, zeros); + r->Z[2] = mask_mov64(r->Z[2], mask, zeros); + r->Z[3] = mask_mov64(r->Z[3], mask, zeros); + r->Z[4] = mask_mov64(r->Z[4], mask, zeros); + r->Z[5] = mask_mov64(r->Z[5], mask, zeros); + r->Z[6] = mask_mov64(r->Z[6], mask, zeros); + r->Z[7] = mask_mov64(r->Z[7], mask, zeros); +} + +/* set affine point to infinity */ +__INLINE void MB_FUNC_NAME(set_point_affine_to_infinity_)(P384_POINT_AFFINE* r) +{ + r->x[0] = r->x[1] = r->x[2] = r->x[3] = r->x[4] = r->x[5] = r->x[6] = r->x[7] = get_zero64(); + r->y[0] = r->y[1] = r->y[2] = r->y[3] = r->y[4] = r->y[5] = r->y[6] = r->y[7] = get_zero64(); +} + +EXTERN_C void MB_FUNC_NAME(ifma_ec_nistp384_dbl_point_)(P384_POINT* r, const P384_POINT* p); +EXTERN_C void MB_FUNC_NAME(ifma_ec_nistp384_add_point_)(P384_POINT* r, const P384_POINT* p, const P384_POINT* q); +EXTERN_C void MB_FUNC_NAME(ifma_ec_nistp384_add_point_affine_)(P384_POINT* r, const P384_POINT* p, const P384_POINT_AFFINE* q); +EXTERN_C void MB_FUNC_NAME(ifma_ec_nistp384_mul_point_)(P384_POINT* r, const P384_POINT* p, const U64* scalar); +EXTERN_C void MB_FUNC_NAME(ifma_ec_nistp384_mul_pointbase_)(P384_POINT* r, const U64* scalar); +EXTERN_C void MB_FUNC_NAME(get_nistp384_ec_affine_coords_)(U64 x[], U64 y[], const P384_POINT* P); +EXTERN_C const U64* MB_FUNC_NAME(ifma_ec_nistp384_coord_one_)(void); +EXTERN_C __mb_mask MB_FUNC_NAME(ifma_is_on_curve_p384_)(const P384_POINT* p, int use_jproj_coords); + +#endif /* IFMA_ECPOINT_P384_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecpoint_p521.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecpoint_p521.h new file mode 100644 index 0000000..30757e7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecpoint_p521.h @@ -0,0 +1,112 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_ECPOINT_P521_H +#define IFMA_ECPOINT_P521_H + +#include + +typedef struct { + U64 X[P521_LEN52]; + U64 Y[P521_LEN52]; + U64 Z[P521_LEN52]; +} P521_POINT; + +typedef struct { + U64 x[P521_LEN52]; + U64 y[P521_LEN52]; +} P521_POINT_AFFINE; + +typedef struct { + int64u x[P521_LEN52]; + int64u y[P521_LEN52]; +} SINGLE_P521_POINT_AFFINE; + + + +/* check if coodinate is zero */ +__INLINE __mb_mask MB_FUNC_NAME(is_zero_point_cordinate_)(const U64 T[]) +{ + return MB_FUNC_NAME(is_zero_FE521_)(T); +} + +/* set point to infinity */ +__INLINE void MB_FUNC_NAME(set_point_to_infinity_)(P521_POINT* r) +{ + r->X[0] = r->X[1] = r->X[2] = r->X[3] = r->X[4] = r->X[5] = r->X[6] = r->X[7] = r->X[8] = r->X[9] = r->X[10] = get_zero64(); + r->Y[0] = r->Y[1] = r->Y[2] = r->Y[3] = r->Y[4] = r->Y[5] = r->Y[6] = r->Y[7] = r->Y[8] = r->Y[9] = r->Y[10] = get_zero64(); + r->Z[0] = r->Z[1] = r->Z[2] = r->Z[3] = r->Z[4] = r->Z[5] = r->Z[6] = r->Z[7] = r->Z[8] = r->Z[9] = r->Z[10] = get_zero64(); +} + +/* set point to infinity by mask */ +__INLINE void MB_FUNC_NAME(mask_set_point_to_infinity_)(P521_POINT* r, __mb_mask mask) +{ + U64 zeros = get_zero64(); + + r->X[0] = mask_mov64(r->X[0], mask, zeros); + r->X[1] = mask_mov64(r->X[1], mask, zeros); + r->X[2] = mask_mov64(r->X[2], mask, zeros); + r->X[3] = mask_mov64(r->X[3], mask, zeros); + r->X[4] = mask_mov64(r->X[4], mask, zeros); + r->X[5] = mask_mov64(r->X[5], mask, zeros); + r->X[6] = mask_mov64(r->X[6], mask, zeros); + r->X[7] = mask_mov64(r->X[7], mask, zeros); + r->X[8] = mask_mov64(r->X[8], mask, zeros); + r->X[9] = mask_mov64(r->X[9], mask, zeros); + r->X[10]= mask_mov64(r->X[10],mask, zeros); + + r->Y[0] = mask_mov64(r->Y[0], mask, zeros); + r->Y[1] = mask_mov64(r->Y[1], mask, zeros); + r->Y[2] = mask_mov64(r->Y[2], mask, zeros); + r->Y[3] = mask_mov64(r->Y[3], mask, zeros); + r->Y[4] = mask_mov64(r->Y[4], mask, zeros); + r->Y[5] = mask_mov64(r->Y[5], mask, zeros); + r->Y[6] = mask_mov64(r->Y[6], mask, zeros); + r->Y[7] = mask_mov64(r->Y[7], mask, zeros); + r->Y[8] = mask_mov64(r->Y[8], mask, zeros); + r->Y[9] = mask_mov64(r->Y[9], mask, zeros); + r->Y[10]= mask_mov64(r->Y[10],mask, zeros); + + r->Z[0] = mask_mov64(r->Z[0], mask, zeros); + r->Z[1] = mask_mov64(r->Z[1], mask, zeros); + r->Z[2] = mask_mov64(r->Z[2], mask, zeros); + r->Z[3] = mask_mov64(r->Z[3], mask, zeros); + r->Z[4] = mask_mov64(r->Z[4], mask, zeros); + r->Z[5] = mask_mov64(r->Z[5], mask, zeros); + r->Z[6] = mask_mov64(r->Z[6], mask, zeros); + r->Z[7] = mask_mov64(r->Z[7], mask, zeros); + r->Z[8] = mask_mov64(r->Z[8], mask, zeros); + r->Z[9] = mask_mov64(r->Z[9], mask, zeros); + r->Z[10]= mask_mov64(r->Z[10],mask, zeros); +} + +/* set affine point to infinity */ +__INLINE void MB_FUNC_NAME(set_point_affine_to_infinity_)(P521_POINT_AFFINE* r) +{ + r->x[0] = r->x[1] = r->x[2] = r->x[3] = r->x[4] = r->x[5] = r->x[6] = r->x[7] = r->x[8] = r->x[9] = r->x[10] = get_zero64(); + r->y[0] = r->y[1] = r->y[2] = r->y[3] = r->y[4] = r->y[5] = r->y[6] = r->y[7] = r->y[8] = r->y[9] = r->y[10] = get_zero64(); +} + +EXTERN_C void MB_FUNC_NAME(ifma_ec_nistp521_dbl_point_)(P521_POINT* r, const P521_POINT* p); +EXTERN_C void MB_FUNC_NAME(ifma_ec_nistp521_add_point_)(P521_POINT* r, const P521_POINT* p, const P521_POINT* q); +EXTERN_C void MB_FUNC_NAME(ifma_ec_nistp521_add_point_affine_)(P521_POINT* r, const P521_POINT* p, const P521_POINT_AFFINE* q); +EXTERN_C void MB_FUNC_NAME(ifma_ec_nistp521_mul_point_)(P521_POINT* r, const P521_POINT* p, const U64* scalar); +EXTERN_C void MB_FUNC_NAME(ifma_ec_nistp521_mul_pointbase_)(P521_POINT* r, const U64* scalar); +EXTERN_C void MB_FUNC_NAME(get_nistp521_ec_affine_coords_)(U64 x[], U64 y[], const P521_POINT* P); +EXTERN_C __mb_mask MB_FUNC_NAME(ifma_is_on_curve_p521_)(const P521_POINT* p, int use_jproj_coords); + +#endif /* IFMA_ECPOINT_P521_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecprecomp4_p256.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecprecomp4_p256.h new file mode 100644 index 0000000..3406c32 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecprecomp4_p256.h @@ -0,0 +1,682 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_ECPRECOMP4_P256_H +#define IFMA_ECPRECOMP4_P256_H + +#include + +#define MUL_BASEPOINT_WIN_SIZE (4) + +#define BP_WIN_SIZE MUL_BASEPOINT_WIN_SIZE +#define BP_N_SLOTS NUMBER_OF_DIGITS(P256_BITSIZE+1,BP_WIN_SIZE) +#define BP_N_ENTRY (1<<(BP_WIN_SIZE-1)) + +__ALIGN64 static SINGLE_P256_POINT_AFFINE ifma_ec_nistp256_bp_precomp[][BP_N_ENTRY] = { +{/* digit=0 [{1,2,3,..,}]*([2^0]*G) */ + {{0x00030d418a9143c1,0x000c4fedb60179e7,0x00062251075ba95f,0x0005c669fb732b77,0x00008905f76b5375}, {0x0005357ce95560a8,0x00043a19e45cddf2,0x00021f3258b4ab8e,0x000d8552e88688dd,0x0000571ff18a5885}}, + {{0x00046d410ddd64df,0x0000b433827d8500,0x0001490d9aa6ae3c,0x000a3a832205038d,0x00006bb32e52dcf3}, {0x00048d361bee1a57,0x000b7b236ff82f36,0x000042dbe152cd7c,0x000a3aa9a8fb0e92,0x00008c577517a5b8}}, + {{0x0003f904eebc1272,0x0009e87d81fbffac,0x000cbbc98b027f84,0x00047e46ad77dd87,0x00006936a3fd6ff7}, {0x0005c1fc983a7ebd,0x000c3861fe1ab04c,0x0002ee98e583e47a,0x000c06a88208311a,0x00005f06a2ab587c}}, + {{0x000b50d46918dcc5,0x000d7623c17374b0,0x000100af24650a6e,0x00076abcdaacace8,0x000077362f591b01}, {0x000f24ce4cbaba68,0x00017ad6f4472d96,0x000ddd22e1762847,0x000862eb6c36dee5,0x00004b14c39cc5ab}}, + {{0x0008aaec45c61f5c,0x0009d4b9537dbe1b,0x00076c20c90ec649,0x0003c7d41cb5aad0,0x0000907960649052}, {0x0009b4ae7ba4f107,0x000f75eb882beb30,0x0007a1f6873c568e,0x000915c540a9877e,0x00003a076bb9dd1e}}, + {{0x00047373e77664a1,0x000f246cee3e4039,0x00017a3ad55ae744,0x000673c50a961a5b,0x00003074b5964213}, {0x0006220d377e44ba,0x00030dff14b593d3,0x000639f11299c2b5,0x00075f5424d44cef,0x00004c9916dea07f}}, + {{0x000354ea0173b4f1,0x0003c23c00f70746,0x00023bb082bd2021,0x000e03e43eaab50c,0x00003ba5119d3123}, {0x000d0303f5b9d4de,0x00017da67bdd2847,0x000c941956742f2f,0x0008670f933bdc77,0x0000aedd9164e240}}, + {{0x0004cd19499a78fb,0x0004bf9b345527f1,0x0002cfc6b462ab5c,0x00030cdf90f02af0,0x0000763891f62652}, {0x000a3a9532d49775,0x000d7f9eba15f59d,0x00060bbf021e3327,0x000f75c23c7b84be,0x00006ec12f2c706d}}, +}, +{/* digit=1 [{1,2,3,..,}]*([2^4]*G) */ + {{0x0000b650bc6fb805,0x0004effe2e6b808b,0x00083f5495882e07,0x00072385ef2f7c2c,0x00004d63c80e103b}, {0x0001bd652a23f9b6,0x0008eb0b6587f2f1,0x000580e9e3670c31,0x00021ff5c4623bb1,0x00004edf7b261efe}}, + {{0x000fccfc5e3a3d83,0x000c1079dfbfd8c5,0x000ad0197befd904,0x0002a48c6d6a58fe,0x0000922707799553}, {0x0003e6ddbef42f56,0x0003e80a990809e2,0x0009a2e407e449b6,0x0002a41b969c1aad,0x0000231d792f591c}}, + {{0x000413077adc612c,0x0008fbd803a06b34,0x0008805bda749652,0x0003ac5a1baaa76d,0x0000840390307034}, {0x0009f66175adff18,0x000b37d8c5b739f5,0x0009d75e30b26d7f,0x000cc22875f5ce52,0x00005efc7e9c1325}}, + {{0x00096f305968b809,0x0002789f73b9db6d,0x000c61e01380a091,0x0008c6eda70b83c2,0x00005fb8394e69b3}, {0x000651280edfe2f2,0x00096faeaf829a3c,0x000424bf88f726bb,0x0009706010a4a078,0x000096720442e844}}, + {{0x000dc0752b3e584c,0x0007f5f86a2c492b,0x0007de57335ff9aa,0x0003296074213db2,0x00000bfffc512638}, {0x000c22a1d49c6052,0x00059320ebfb2429,0x00020f4281b037d7,0x00066032b6a17392,0x0000995919cc4cd2}}, + {{0x0008d2333e12b701,0x000b09dd329b802b,0x000bc354d6d490a4,0x000a0d04f356cc6a,0x00001eddf7fe0a0d}, {0x0008328d87fd1d83,0x000ccd0258131e20,0x00029bca2fd2f4f8,0x000b70d3b48cc47c,0x0000f2a78b3541a2}}, + {{0x00005760a99cbca7,0x000594a428f26a95,0x00032ba0c94e258f,0x000166d5ab5a4d78,0x00001704d00f938c}, {0x000ab0ef88b8b706,0x000e600eb207db97,0x000367d8056feb92,0x000a41870352687d,0x00005000c2527973}}, + {{0x000cb817a2ad62aa,0x00090c62ff5463c5,0x000ad9db57ef2b6b,0x0006169749bba4b3,0x0000d311f2ce6d5a}, {0x0008087c2ff3b6df,0x0002467834ffb77a,0x000d6b138b46feaf,0x00018808aa266d75,0x0000a38d321dc008}}, +}, +{/* digit=2 [{1,2,3,..,}]*([2^8]*G) */ + {{0x0008ffa696946fc7,0x000849cba56d486d,0x000f35a1550fbc6d,0x00062c0e3d423e90,0x0000c3da19630dd9}, {0x000fdb03cfd5d8b3,0x0002589dfca5e673,0x0002305aa0704b7c,0x000e53c6ce581ff5,0x000099d49ebc14d5}}, + {{0x0008110399492969,0x000aa61db1b544e3,0x0006eaff55b63827,0x00028fae5323ed20,0x000042370d3521f4}, {0x000af2ee0d985a17,0x000b0239846df2ca,0x0006312f8192cc64,0x0001080c0b8f47ae,0x0000dc61f9206620}}, + {{0x000a53eed4c37174,0x000efd0ed2a335d6,0x000543aa59f8240c,0x0004b44c0d4d05e5,0x00005d5bbfc1d33b}, {0x000cc73137fd28e3,0x000f973b3ffdfa04,0x000f51ef2862ac6e,0x0005a2103ff9f531,0x00004d5e0fcec73f}}, + {{0x00081e144cc3adda,0x0005e7be82cf4f70,0x000dd6472d5ffa1d,0x000862e9890b6c0e,0x0000da26e1aded17}, {0x000271563483caaf,0x00083f6077fd276f,0x000466e3ce6924cd,0x000d1e15a7fe980a,0x00001c794b1a1902}}, + {{0x000385c08369a907,0x000aa90eb4f833b2,0x0008eac802990c59,0x000014119a6145c6,0x0000a786d629ec4a}, {0x000adbe20ac3a8dd,0x00008aba2d3033fa,0x000a4f56531a2178,0x000fba509d2742db,0x0000b2ce9e425aa0}}, + {{0x0006bf38bd7aff18,0x000a9d81b146b315,0x00028a9151b5ee4c,0x00099dfba1ac41d6,0x0000f3a8f9d7d896}, {0x000b9c9a0748be7d,0x0004d92e621f7329,0x00010a8371d391c9,0x000435151e6b214d,0x0000255f53b1947b}}, + {{0x00058d474a861082,0x000fce4c5d900c4a,0x0006d4c80f8048a8,0x000e60c3c7c924e8,0x00008c889de256a1}, {0x000662eb214a040f,0x000747e1034757e2,0x000ac748ae8c48e9,0x0006f19774286280,0x00001c24023086b0}}, + {{0x00024b9eb7926b83,0x00039dbe55093d2b,0x000dd640bbff88cb,0x000d45a0f399afe4,0x0000c5fe1305f76e}, {0x00062f43764fb3df,0x00074151b62d6f35,0x0009ce5f37b5af31,0x00090ee5bd0bc7d7,0x0000daf6b21dc668}}, +}, +{/* digit=3 [{1,2,3,..,}]*([2^12]*G) */ + {{0x000b0e5ab4b35a24,0x0001b5eeaacf6772,0x0005b95801d8b600,0x0001da328f7ce479,0x0000a20ed2a81fb8}, {0x0005cd44fec01e63,0x000c77ff50ad9f68,0x0002d97fd3ed7ddc,0x0004f9160fd2640c,0x0000a2414271b82f}}, + {{0x0009862d5d721d5b,0x0002abd3a1828000,0x000a2cda40c3357a,0x00008477f3a83b7a,0x000058ae74fa6f83}, {0x0001a812e6dad6be,0x0001143d6c5b2a91,0x00096c4d8de28605,0x00024d6bdccc41f9,0x00007312ec0eae1e}}, + {{0x00068feaae6ee702,0x00053602b0c96faf,0x00094052a78f4cc1,0x000d805e3321a86e,0x0000fb3a0d6934d5}, {0x0008f3bb25a43ba1,0x000109ee2951f3b9,0x000b0612a30bf803,0x0001d06ffee43321,0x00002f775e43eb82}}, + {{0x00012e76e6485b37,0x000b071c52f8f8d1,0x0004a2f6d4d3e24d,0x000550d8e3ee4168,0x0000161957d91d95}, {0x0001283cdb12a6c3,0x0001fe50e1641963,0x00066cc73bf3fa88,0x000c38c6254b6331,0x0000aefa7aedee8c}}, + {{0x000008037a929a9e,0x000d39fec6bd46f7,0x000ab8b6265601a8,0x0000ce737f5edc12,0x0000497cd963e599}, {0x000387f9aa5b2f93,0x000f37b78fe82fcd,0x0005d5e30e5faa3f,0x0007ebebcf538d29,0x0000a573239d813a}}, + {{0x000286bd17c24093,0x000333264d9be9f5,0x000f6880f2c4e479,0x000e42e77042b117,0x00009b7e2c87ce1e}, {0x000e722d096f4a9c,0x000afed5e697cec8,0x0002319116861aec,0x00037a02d153f06c,0x0000c2f42b8e8905}}, + {{0x000cb51d0a917b49,0x0002b899f93133e2,0x000a2f6b6c2cfa3f,0x0002b722c94f4be9,0x0000707b1820ca16}, {0x000a172d5f8b10d2,0x00052fd4542ab602,0x000996992fd30783,0x0005e1cef226dddd,0x000021fa98a0b0a1}}, + {{0x000fe623b36f9fd2,0x0003dde19fc079b0,0x0008482ef26543b2,0x000824f36e64a095,0x00003f63771bb095}, {0x000d596b6a1142e5,0x0005e35aac0b14cf,0x000081dd55ea6aac,0x00012a36a0e8bdf3,0x0000fb89d79503dc}}, +}, +{/* digit=4 [{1,2,3,..,}]*([2^16]*G) */ + {{0x00065fce3779ee34,0x000d7d495d9e0f01,0x000284e7ae00e7f9,0x000218dfa4efa220,0x0000564bade87ac6}, {0x000312ac4708e8e4,0x000b671e9adf90e6,0x000684b9f4f5725f,0x000415a95f55ae3d,0x00007f7ccb15e94b}}, + {{0x000890361a341c13,0x000fdcfd61423617,0x00033316c3604dc5,0x000921d22295eb85,0x0000dbde4ac74af2}, {0x000fc5d1c7eef696,0x0005714f4fa1898a,0x0003c21ca5889680,0x00030aa500216020,0x0000f0d1f30a0ef7}}, + {{0x0008b1dbe7a2af37,0x0008dfb74a72bd9b,0x000879697ec51caa,0x0007d549937a4b63,0x0000c9a9d215c268}, {0x000e44f6ef5f0145,0x0002990c69001773,0x00042161e8abcf41,0x000f29e87bd02281,0x000003937564cb6f}}, + {{0x000770977f7195ad,0x0006ddeb838ffabf,0x0004f012d8ec8616,0x000b3f1a1285a8bb,0x000068835046a3ea}, {0x00024f8309004c28,0x000593ffe95eee5d,0x000223ea4a96e4b7,0x000a528cdffe12bd,0x0000f5c2ee636739}}, + {{0x000333959145a65f,0x00080a4063373d61,0x0008a52a0cd9bc36,0x00058f92d11be32d,0x00006877b2887a1c}, {0x000819bf5cbdb258,0x00085e090249837a,0x000990e5f2a4fd1d,0x00011ae22a7de774,0x000040fa5a0f9455}}, + {{0x00090b36b0cf82eb,0x00057615b5e7e58e,0x0009c145a6438d24,0x0001ca57b1f8fc66,0x00000d8b2dae6f1e}, {0x000dadbd9184c4d2,0x0005d93d997654d5,0x000147d473dbb18d,0x000608ea3e0f56d1,0x0000afa8c8dc0a48}}, + {{0x0008c07e3533d77b,0x00097e341c9926e0,0x0002dc4edd7222e6,0x000cf7ed60ec3d8d,0x0000dfe0d902c476}, {0x0009ab61d056605f,0x000596a8551f1fe5,0x000fb8d8ca9ea9df,0x000b0f9489941e47,0x0000eb874ec3a7f1}}, + {{0x0006aa9bd7638026,0x000005303da1ed40,0x000e62ec4c21486a,0x00033e01ae291ec7,0x000022a04933f993}, {0x0000c9dbb7a8ee0d,0x000b9c01aedb7fd8,0x000be74ecdc2ed3b,0x00071e65c35a1208,0x0000540cb1b169f6}}, +}, +{/* digit=5 [{1,2,3,..,}]*([2^20]*G) */ + {{0x000746a584c5e205,0x000169dc7035a7a8,0x000548c9b267e4ea,0x0002f3093a15cfb9,0x0000e6e21359bd01}, {0x000cc6a8c8f936e6,0x000455c241dcdf31,0x0005efb868af84d0,0x0002cb03990a6f34,0x0000fef4e6219b96}}, + {{0x0008f09257226088,0x000a931cf5c6f636,0x000b4f7ac131260d,0x000828c0eb353bfa,0x00005c78880b7eee}, {0x00081ffc3bdf24eb,0x000b45c3c5a84c15,0x0004e6f405bff75c,0x0000c985e8c83fa1,0x000081d1c0fb295e}}, + {{0x000e23d442a8ad11,0x000cf6b9c164f2ef,0x0000aa5e5c3816a7,0x000e6599df2d8bdc,0x000091ae46f220a8}, {0x0007f8700611c5bf,0x00070f1099488366,0x00069595283171ed,0x000a1243a2ecf8ca,0x0000a4a73efe48d1}}, + {{0x0007cc8f43a730f8,0x000bb3ab590efcde,0x00003240be89b6f3,0x0005db4823f529ad,0x00002b79aff18bea}, {0x0002856962fe5de3,0x000b30c591f3568f,0x00028a8580c590ad,0x000f4befc74a144a,0x0000b662498e3203}}, + {{0x0004ed082dd1b6ad,0x000797b703af48fc,0x0005d6aaa5783a13,0x000d425463cb9a00,0x000031ec55d406ec}, {0x000d33f8e9a76414,0x000cc98d9e7a9f8e,0x000887493625453e,0x00056663beade4ec,0x000042b80509a795}}, + {{0x000cf0d6c39765a2,0x000d8c3cca0b91e3,0x000953b50a2db3ac,0x000f1a088f2f08cb,0x0000414582cef43c}, {0x0008bbc60eee9a8a,0x0001d29aa0428dec,0x00032f5d554c79f0,0x00015f381cd5ec65,0x0000672303b6f82e}}, + {{0x000582d3bfab839b,0x00037f8adade46df,0x0007a1bc392474e0,0x00097886a7766a14,0x00006940f54bdc0f}, {0x0008ef2f2759f255,0x00095719f4c64473,0x00050c3459dd9578,0x0000d4d859b7f407,0x0000e788bf302218}}, + {{0x000afa8719c05631,0x000cac5fc79f376a,0x000750cd3cd8ad2d,0x00008e203fdb9fcb,0x00004ff052f5418b}, {0x00084cf3e2d65208,0x0007944ed509f750,0x000f25b987ebdf0f,0x000837743bf0f2d3,0x00006ad71d02354d}}, +}, +{/* digit=6 [{1,2,3,..,}]*([2^24]*G) */ + {{0x000c8c4868af75df,0x000e55c8c7ead9d0,0x00081ecb0d7325cf,0x0004ecbb471996cc,0x0000f5d55f451182}, {0x00045411977a0ee8,0x0004f22038c6be31,0x0004bb4955085c4c,0x000081ad5335bff9,0x000094ad8a748e2a}}, + {{0x00034b22c11bb373,0x000dbd4c74a35402,0x000c5f25d2d0366d,0x000142c9a968daee,0x0000660106897b63}, {0x0006d2c68d7b6d44,0x0008cc84294207cd,0x00068b1eea8f74f0,0x000ee4a275140477,0x0000b5f7e8a3e62a}}, + {{0x00059b21994ef202,0x0009438ae318d1e0,0x0006990102a653b6,0x00084a50d5eb582f,0x000079739f729f5f}, {0x000663c8b799336a,0x000c803c37eb5da4,0x000dbfb2dfdfdf14,0x000f9a92d8a9dca1,0x0000b40cff117d48}}, + {{0x0009f0b879fbbed4,0x00069a9d1869f236,0x000766f450ff0ae8,0x0000fc1251d75956,0x0000984d8c06be8d}, {0x00095a6d21008f03,0x00000a1a1c497ecc,0x0006c50f329bd54a,0x0002517b9828c5d2,0x00002c0087c81d0d}}, + {{0x00090abfbaf50a55,0x000b184e0750f617,0x00076b005df55e76,0x000dc79c516da7f1,0x000075553bbca2dd}, {0x0007ca3553afa736,0x000bed55c25137c8,0x0003e5d35315f3ff,0x000f288846442aaf,0x00001b91149c495f}}, + {{0x000eb6662b5f3af3,0x0000dabb373447fe,0x000f35cb1cefab56,0x000eb9149de60e19,0x00009f8db14457f0}, {0x000cc5b3c61bfd60,0x000d41216703ffae,0x0004e1cc2a5a4d41,0x0009537f8fabed22,0x0000d5a8186871ad}}, + {{0x0003a4956f908239,0x000fe41d777b4bdf,0x0008bf760ba0f507,0x000b01791d71c3f3,0x0000633d5102b625}, {0x000b743b8c9de617,0x0003ede7472003ec,0x000ce1cb2b475125,0x0002ef2f9defc974,0x000074a4f6a70bd3}}, + {{0x0008ea601799a527,0x0001486d2952190d,0x000ff2a7ca20cec4,0x000d36c062ffb27f,0x000041b32e5e9f19}, {0x00081814eb57d471,0x000406aef06bf80d,0x000ecb5887a2d0ed,0x000f5af9735fb01c,0x0000641caaad6061}}, +}, +{/* digit=7 [{1,2,3,..,}]*([2^28]*G) */ + {{0x000824f20151427a,0x0005f24302067f99,0x000112357206828b,0x0004ec0a9097d7e1,0x0000cf9a2f2a9e41}, {0x000c9da279153564,0x0006c01efee3dbda,0x000b288e27e0734b,0x00009c14fab5bbd2,0x0000c630fc5362dd}}, + {{0x000107a1ac2703b2,0x00084bc857b58537,0x000daccd1b49258d,0x00052937df14debc,0x00004ab68d7e4ae8}, {0x000b5d4734e59d08,0x00084495cc807ed8,0x0001db9b35f8740c,0x0005be04aedd5a29,0x00000b360f8cfb99}}, + {{0x0005f5d5fa067d1d,0x000ec668960cae91,0x0008edaac4134b57,0x000435ed3656d6a4,0x0000ac1e3e5cc1d7}, {0x000f869d81fbb26c,0x000bf26c33d4674f,0x0004203e8449ed3e,0x000f49c5138705d9,0x0000cde538c7eeb6}}, + {{0x000c68da61a76fa3,0x000d9a1554dc55d5,0x0003b279c598b441,0x000efca39923b977,0x00003331d3c66bf9}, {0x000848e298de399d,0x0006d1a27f562d4c,0x000b8ab70cfdb8e7,0x0009b9c4c855ea57,0x0000cdb9daf3f787}}, + {{0x000f8c2019f2a596,0x00036b4fbc747bdf,0x0009173ddb3ce5bb,0x0004398a907f688a,0x0000cd3d0d3f5a75}, {0x000c4d6efed021c1,0x00005a77339a92ec,0x00088c64a09a9f9b,0x00015877ca6b1571,0x00000c2996854899}}, + {{0x000a229ed6e82ef1,0x000355ebaf4e5859,0x000ad67ae16f338e,0x000bb3fcd313875e,0x0000c73d22864ef0}, {0x000513174a5c8c7d,0x000faf69ad6a4cb5,0x00066f87e01cd296,0x000320d04d00dde9,0x000096fe447db7b0}}, + {{0x000c06e88fbd3813,0x00042c35a493342a,0x000f1bbcd02cd4a8,0x000d4cb8fa89de54,0x000041d63675575e}, {0x00057fbd238202b9,0x000a1984ead9ebe3,0x000436ea0600b4d1,0x00051b335c9f4452,0x00006fe0a3a33707}}, + {{0x00007367f636a38c,0x00064e76d5cb4c4f,0x000b68b8b9f943fb,0x000a1ef03510baa8,0x0000246780b5ed07}, {0x00014156d549fc2b,0x0000b07781ca3c05,0x000d95413c2953f3,0x0002e2e55e2c69d8,0x0000300fadd2bd28}}, +}, +{/* digit=8 [{1,2,3,..,}]*([2^32]*G) */ + {{0x00086024147519ad,0x000b56b372f02028,0x00085ebc8d0981ea,0x0008e8d9d4a7caa7,0x0000953c50eabdf5}, {0x00061ccfd590f8f8,0x000ac4e6c9179d63,0x000eb64cf72e9626,0x0008f2ffd9611022,0x000063ebb7f1eb28}}, + {{0x0007cf5678a31b0f,0x000d4998b620877b,0x0000fb396d50301a,0x0002a5834257c5c0,0x00009fb18a0f4e67}, {0x000d8ebe8758851b,0x0005ad99ba44ff8b,0x000fd93b71e64e4c,0x000b8b9b8eaedf7d,0x0000a2f2a98b4e76}}, + {{0x00007e0e90fb21e5,0x00006ba7fca1a18f,0x000cd67b500fd2b8,0x0007f6d0387f2795,0x0000b89a4e823970}, {0x000ad3f894407ce5,0x00041c2261328f83,0x00006c13ba0025b9,0x000025779563c7f9,0x0000f548f319e7bb}}, + {{0x000d3a7c35d87949,0x00077356bae50ee6,0x0003322fd042e655,0x0009670f59698d64,0x0000379ae15e0a61}, {0x000ae62fcc9981ea,0x0000cd2934c664b9,0x0004e65ebaed3d63,0x0004278454b3025e,0x0000b09f64899950}}, + {{0x000d3d331b85f091,0x000a988ae64ac1b3,0x000ec50fd0f45354,0x00034f98b626d32f,0x0000bdcfbd4f8288}, {0x0002866cd5225391,0x0002710f7ab3e45a,0x00005f293fa9d473,0x000597c8c1d6b4c9,0x00000ac80474461b}}, + {{0x00015366d91cd2c0,0x000adaa3f0e4e2c8,0x00041e08340a2bee,0x000347fb167a5924,0x000004675e9e9240}, {0x000aaff840e446e0,0x000fea308f727848,0x0009bfad99f9f258,0x000af650f1289963,0x0000939ae63205c0}}, + {{0x00075146fc627e20,0x000591573a51bbb1,0x0008243d5a0569bc,0x000692a7016d9e35,0x0000dac0c56ac1d6}, {0x00033b5da590d5fc,0x00031e8174919938,0x000bf75d0a806780,0x000cfaa5b4f2124d,0x0000c9602338cf80}}, + {{0x0003a1222248acc7,0x000ec264e366b208,0x000fdee281f6ec0e,0x000bb4e659b7045a,0x0000a823a4156430}, {0x0002a04e1900a791,0x000ab9ee65762459,0x0005ea54acde09d4,0x0005a742b6463f4b,0x0000efe9ed3e3ca6}}, +}, +{/* digit=9 [{1,2,3,..,}]*([2^36]*G) */ + {{0x0006dbe305406dd9,0x000f4d5d1957e27a,0x0007d4d8f8eb7dc7,0x000de4654a687638,0x0000c47940a57762}, {0x0005b5d99b307781,0x00065e793682be4d,0x000c740e325380c5,0x0004ae502d37f3da,0x000040deabe2566e}}, + {{0x0006126c49a861ec,0x0005214f0d06eaee,0x0009bfc17024f3b6,0x00038091a3f1e8c6,0x00003c3a8ea67686}, {0x000752cb103d4c8d,0x0002c218b36b3400,0x00051504a02bc461,0x000bf9f67f75eb76,0x00006848b57a02ae}}, + {{0x00081db1782269b6,0x0008c597e5509583,0x000385153ae34bf7,0x0000485b5c60645f,0x0000f0e96b043088}, {0x000021577884456e,0x000b89310ea7bf6a,0x000fad2deb3b5688,0x000d4c37c9429504,0x0000020f0e5f7896}}, + {{0x000428dbbe5a1a9a,0x000e9126bd67cca4,0x0001058268187fd5,0x00019f6036973a48,0x000039b666458bd6}, {0x000deef2d65a8087,0x000f24636b196d42,0x0005d564c4969044,0x0000778611ee47dd,0x0000b2f3a4a42873}}, + {{0x000d8dd0f82b2148,0x00097103cbc603b0,0x000d79e19460c34f,0x0007f8732e5c0318,0x0000b8888bb28411}, {0x00037dcc07226779,0x00088c1c0f278f3c,0x000f7a0c610d21be,0x0000e0447c8468e0,0x0000bf022143decc}}, + {{0x0004160b7fe7b6e0,0x000a400a3fb29755,0x00028ca1e7d16189,0x0008ccd73e9beae3,0x0000dd04b97e793d}, {0x0003c9b506db8cc0,0x000ecf38814ca9c8,0x0004b45e65cd47aa,0x000a8426fc430db6,0x000079b5499d818e}}, + {{0x000d21ae0ac29416,0x000462d3193703b5,0x000c992d0279b025,0x0001f2d307c052ca,0x0000aa7cb934fa8b}, {0x00025800d37c7a50,0x0007342d54225a18,0x000d2ef9213380c3,0x0003c692ac2d66d5,0x000035a70c9030c6}}, + {{0x000b78571ba18615,0x000c80c8f93d5109,0x00033bb9348b22d5,0x000d0898fa84a786,0x00003fba6baaaebb}, {0x0007df3e5eea7d82,0x000648ca71587ff2,0x0006f1a05521c879,0x000ee499d5133bce,0x0000d50cd541d0eb}}, +}, +{/* digit=10 [{1,2,3,..,}]*([2^40]*G) */ + {{0x0006d65533ef2177,0x000453ca2e87889f,0x0002b41677158c7e,0x00057f8b670dfbdc,0x00005910a01f44c2}, {0x000bf07cf88577d2,0x0000c45e2acef336,0x000a23d852224525,0x000f580ed92e8d7c,0x00009f8be4c4b812}}, + {{0x000b2452133ffd9d,0x0000b30f1a20fbb9,0x000a1f52a39a8b2f,0x000df7784bc97dd5,0x00006aebf57740ed}, {0x0007acb76ccdac60,0x000c1586ff273225,0x000de7dd1af4d36e,0x000c168eaa8863f8,0x0000045d5cf88647}}, + {{0x000414351facc618,0x000d668a25bcc51e,0x000f872edbaf2647,0x0006590f5271a00f,0x0000f32ef99bd2d9}, {0x000488c7593cbd4c,0x000c42b82fabca12,0x000eb3f16ed266c5,0x000fe24a2f78ad14,0x000034049490d47a}}, + {{0x000d574c005979da,0x0001ca40e350a6f3,0x000e2ecf9c2072b4,0x00044e5ca5c1568d,0x00008c8bf5c45153}, {0x000e555114df14a7,0x000d8dc5ec6b97ae,0x000a85418d4374a4,0x000f78054cc28f2c,0x00001cb9e2843c41}}, + {{0x0006702094704963,0x000dbbd2381509c1,0x000dd4398a489a5e,0x00069694dde4648e,0x0000ca7b94ab0111}, {0x0005d682ad636a41,0x0004f8dc5f1e3c38,0x000a219436702702,0x00069dfc1965deaf,0x00008666e16710be}}, + {{0x000fd350369c8e10,0x0002b9dc843b6792,0x00002e2ab9271aa6,0x0003838711a4b14d,0x00002b2a3e27ee1a}, {0x000e35f0e2b379be,0x000bf652ab25b226,0x0005601063d3de39,0x000876cca6d4c93b,0x0000ced0cf5a95bd}}, + {{0x000b4ca2a604b3b7,0x0003ca61676245be,0x0008b806e56f6518,0x000480852f5a7097,0x0000aa3978781dc4}, {0x000ac2a0e01fabc4,0x0004e37d99f9e13f,0x000211ffe7c6ee8a,0x0003eae51384ee05,0x0000ff6976d5bc9d}}, + {{0x000507903605c397,0x000e3142c96c8910,0x000923684f0843d9,0x0008938374493416,0x000032caa306a0a2}, {0x000c27061160170e,0x000b637fbaa3b2e8,0x000eda3acc32788c,0x000e0659cd818ea6,0x00002e9423a7e2b2}}, +}, +{/* digit=11 [{1,2,3,..,}]*([2^44]*G) */ + {{0x00075455922ac1c3,0x000c752b3f638df2,0x000de57c4a7b3ef5,0x00008b5e77b21471,0x00001682c10b34c0}, {0x00024f04bd55d319,0x000587b61c71c768,0x000a5089db6d1c08,0x000d3ea1db0903c2,0x0000c092172a84e5}}, + {{0x0005035ea6c39976,0x000a62610bef5ace,0x00080dd3954259aa,0x000a398f18bb3f3c,0x0000910b95bbfc3f}, {0x000f51043e09aee6,0x000f47675665fce2,0x00072db61ced56c9,0x000e68b0e265acd8,0x0000982812f0e9fc}}, + {{0x000b690768fccfce,0x000cd835b362ca2e,0x000fdfccef402d37,0x00098f2efac0d0e2,0x0000fc9cdf09638d}, {0x0002b72d1669a8bc,0x000b9774ccbd2af1,0x00034870e33c536b,0x000ac970b21909fb,0x000038fa2f83df25}}, + {{0x0003d931341ed7a4,0x000c67b59d49b8fa,0x000b8c4a44223272,0x0002e3fdcb194783,0x0000e413c022d130}, {0x0009127e17e44ceb,0x000483b3adfb6d99,0x000aa96caee86bf7,0x00047d46902fe625,0x000073540e595aae}}, + {{0x0005e75b2c69dbcc,0x000843c3da6c7bfc,0x0009102713aa77a2,0x000c551e0df03cca,0x0000bd5ca4b3806d}, {0x00058076db476cb9,0x0001cf37a31ee1ca,0x0001af416fde15d6,0x000db5649af520f4,0x00006c5c5b20d342}}, + {{0x000ef6c872b4a606,0x0003e613521bcc50,0x0003e15d1ab2a34a,0x000511d9c5c19098,0x00001dde5dfb9905}, {0x000f6219f2275f33,0x0006151d894be417,0x000b0bdaa0750c8b,0x0009bd45b04ab978,0x0000bfd9fd475858}}, + {{0x000e02adb22b94b8,0x000921eaa45bc792,0x0001e1c63993d8ae,0x00088a0aad6cd3cd,0x00009529ca845ce6}, {0x000e3aae572a2530,0x000802a21efb2cce,0x000430358e02b643,0x000504a7091b6ec9,0x00006d1b1fa9d7db}}, + {{0x000bfabaf95894c5,0x0006d76b2241aafc,0x000dda48b7b9bdc0,0x0004df9af983625b,0x0000977faf2f3fcb}, {0x00042ef052c4b5b7,0x0000967591f0bed0,0x000f24ec79fe87f7,0x000f1b589c73ca22,0x0000d37fa9f564a9}}, +}, +{/* digit=12 [{1,2,3,..,}]*([2^48]*G) */ + {{0x00064880a750c0f3,0x00031e548e83cc7a,0x000110f0539bacfe,0x0005880d418c760c,0x0000e4daa4ce1f11}, {0x000e7b55ffc69ff6,0x000c320531272733,0x00022df9446f147b,0x000b7c285b2434d7,0x0000a444f6646fc6}}, + {{0x000465ac3f16ea83,0x000d82f1d11c7a1a,0x00068a172115a461,0x0006981767dd956c,0x0000392f2ec013a4}, {0x0009ccde526cdc7f,0x000b32292b81c7a9,0x000d391988e537fd,0x00052c86d8cf69a6,0x0000fc5ff4414468}}, + {{0x00016f4bdaedfbdc,0x000fc6746ced6d0b,0x0004b3e1723fd325,0x0004c7cbfb1d2fff,0x00007f2ec2dc19c1}, {0x00032f245104b0d2,0x000b8dea2b7e3e08,0x000fbfb0f5f00daf,0x000cda09e5cf6699,0x000064f972381827}}, + {{0x0004f7ea90567e6d,0x0006e6ae5cb797b1,0x00010903d513257b,0x000723b5454a3c9f,0x00008d2c9ae39bc3}, {0x00093246b29cb44f,0x000c87c8cbac38da,0x000918e42b540a21,0x00014dabbfe43501,0x0000ffa707b46c36}}, + {{0x000a2f3e30bc27ff,0x0009c08365116eb1,0x00065ab0ee5f0c05,0x000bbc5d741bbf49,0x0000eec41cb73464}, {0x000705f99d0b09f8,0x0006742da5fa1aca,0x00052b931c5d6cc5,0x0008d7c9964eddcc,0x0000ae596164884d}}, + {{0x000e3f1d4e353b7b,0x00043f46b0a00ce4,0x0004b73fd062d8a1,0x000ffcb408d5ab57,0x0000c41d1ca83273}, {0x000e1e76be77800a,0x0007256550313538,0x0009b331a71fe8b3,0x000f727cd916216b,0x0000d825d0c5b388}}, + {{0x000b57b39f8868a8,0x0003f5cc69aff634,0x000d5496ee27f4fd,0x00007f247e58cbd0,0x0000a2679405323e}, {0x0009b72fa30f349b,0x00090696d134c61a,0x00080a6d194c9d9c,0x000994c92beca858,0x0000dcc46465f039}}, + {{0x000e05b1cb76219d,0x0000a1567e7e56c2,0x000c4c9100ec0bf9,0x0004d917076f8661,0x000067b085c8abc0}, {0x00004595e93a96a9,0x000a6bdc249a9fb9,0x000dd0bb77526c1e,0x0006947d44d367ec,0x000053999182dc0d}}, +}, +{/* digit=13 [{1,2,3,..,}]*([2^52]*G) */ + {{0x0009167ceca9754a,0x0005ab7939a083f4,0x0003fd0bf426d2cf,0x0004e18555e35572,0x000096e6d0764f14}, {0x000a8dd87880e616,0x00058508e4d54768,0x000b65e151554381,0x000f9fa9d7e772b1,0x00003439dd70c302}}, + {{0x0003145983c38b5e,0x000b837abc8b859d,0x000ff7be6b14f176,0x000a594793fb9dca,0x0000be5a56015a66}, {0x0001dcd9f87dc596,0x00039bdbf5607cec,0x000eb32577c595cd,0x0005fcfb543b2226,0x0000908064724c93}}, + {{0x000688eadd70482e,0x00099b4a4e8a6aac,0x0008a6eef708de92,0x000c4295b6dd7375,0x0000a4bf353525b3}, {0x0001f2c87912868c,0x00052f09297a1004,0x000f3860ab1b1be9,0x000f4a59ae23c5a9,0x00004f0f83a115dc}}, + {{0x00052381d531696e,0x000fa8cdde69b934,0x00086afc757201bf,0x000ea7fde922519a,0x000030438969d35c}, {0x000c1e18555970de,0x00084535935e7608,0x0002ea38b8267dfa,0x0008b4f4c60a5732,0x00000bf7978604ef}}, + {{0x000049f16a66e918,0x00007a1b0e0dd730,0x0004c28eae97f282,0x000c61c131e00330,0x000020ab732d26ba}, {0x0009ef9287144234,0x000a6db10cb2b2ac,0x00086a4cc54ecfff,0x000d494781476ef8,0x0000b2c87b61b2f8}}, + {{0x0001d4286d2e0f86,0x0001ae8a9fd56ada,0x0008c1b49e592012,0x000afea2c936af70,0x00000f30fee8b4bf}, {0x000ad06858e6a61a,0x00069fd374d06637,0x00088defbce4c776,0x000b6599d54b2d71,0x00008c9d251956a6}}, + {{0x000c0965f5206989,0x000db4f7b8d90e6e,0x0005a68b9640631f,0x000f4e02fd34fca3,0x0000c5a4b66dd40c}, {0x00054bf80b6783d1,0x000e2a320a109494,0x0000a39b280e701f,0x0007db7d1a564a1a,0x0000436d53d42058}}, + {{0x000ea68c094dbb56,0x000e7968d4106233,0x000b3002db77d062,0x000d57de719bbc58,0x00008e7dd3d9dc49}, {0x0005740013a5e585,0x0006ec9e3c1b8d82,0x00099b6ab2131174,0x0008f1bcb0a2a77c,0x0000c48a3b412f88}}, +}, +{/* digit=14 [{1,2,3,..,}]*([2^56]*G) */ + {{0x0003e91991724f36,0x000bd9cbd686c791,0x000d4fc1e5eda799,0x000d547db595c763,0x0000b63b80c0c4fe}, {0x000fc697e5fb5166,0x000a70f1c9646ea0,0x000a92ca5737708b,0x00067a3628745f11,0x00001f37958fa869}}, + {{0x0009b2caa6650728,0x00046fd324ef9af3,0x00027bd3178322fa,0x000aafbd153394c3,0x00001d5f271b129d}, {0x0000c42f48027f5b,0x000bd536e717c72e,0x000369d0faa40cdb,0x0004e6445a657a2d,0x000003bbfc59a7f7}}, + {{0x000c4180d738ded3,0x0000b0de572946a8,0x000a816756f1a5bb,0x0003d4c10230b98b,0x00002c6f30c412b3}, {0x000129dd8fffb620,0x0007b459bf057559,0x0003b67766a281b4,0x00073a77c1bd3afa,0x0000709b38078299}}, + {{0x000b232a3326505c,0x00022e1d41bf8c26,0x000e32afa38d6927,0x000a864459453eff,0x0000e8143ae3cb3e}, {0x000c1fa7e6ab6665,0x000fd2286264932e,0x00036f8ed6cd2d22,0x0005baf59a46fe67,0x00000bf0d00eeca8}}, + {{0x0005852877a21ec5,0x0006bf537a940b82,0x000a9a6a2300414a,0x000bffef1cba4021,0x00000824eeec6943}, {0x000fcecf83cba5d5,0x000843b4f3c0a0db,0x000f24dd7f953814,0x0009dd1174416248,0x0000322d64e34fb0}}, + {{0x00073843d9325f3b,0x00004371cb845744,0x0001e36c5a9bef2d,0x000f71c7d2188ba6,0x0000bd6a7d87602d}, {0x000a9028f61bc0b8,0x000ceed0b6a1ba3a,0x0006e8298f49085e,0x00001d0bc625d6ae,0x000032b0b1e22e9c}}, + {{0x000c447f1f0ced18,0x00031492dd2ba337,0x000a08efa800cc79,0x00041dcb93151dbe,0x000020cf3f95e0a7}, {0x00082dc1c0f7d133,0x000054dde6caff19,0x000f96ee3ef92196,0x0000c6ead7d97245,0x000019c8dbe59dea}}, + {{0x00038717b82b99b6,0x000ce70eb624d3ea,0x00095d46675922d4,0x0003462f66ec543b,0x00006e673cd1ee1e}, {0x00067c4b5f2b89a4,0x0005e90e5cd36afe,0x0000a2ada3de9c1e,0x00023b4c278bb631,0x000020fa3844bdb3}}, +}, +{/* digit=15 [{1,2,3,..,}]*([2^60]*G) */ + {{0x00096796424c49b9,0x0007d7c241c9646f,0x000f68b49f888dfe,0x000f20512d4b9324,0x0000a6b62d93571d}, {0x000b26d179483cb7,0x00022511fae281b4,0x0003aa51f666f963,0x000d166281b3e4d5,0x0000f96a765ef3db}}, + {{0x000d37c051af62b9,0x000a7bf944968553,0x000d59aa1e9a998e,0x00081350844f9fb0,0x000083fd55976afb}, {0x000c0ca65d698049,0x000ddea5ff2d9670,0x000d8623b732b22d,0x00078247640ba95f,0x0000f61916436351}}, + {{0x000b4e0bdefdd4f1,0x0005e366e401f167,0x0003bbec06995846,0x000c214aa368aba7,0x000021487098b240}, {0x000323318969006d,0x000e11fe53d1378c,0x0000c4361cb4d73c,0x00012a8f50a80e13,0x000067f59524ef52}}, + {{0x00081088cad38c0b,0x000fbbd68ae2332f,0x0008e27a3471b7e8,0x000b0ca6ac3fb20d,0x000054660dbc36b4}, {0x0001e11a6fd8de44,0x000a637799ef123a,0x0006ac17c44dbffe,0x000cef0540b977ce,0x000095173a8ef60a}}, + {{0x00037434573eab0b,0x000b21ac6031eb44,0x000dd9afd11570df,0x00023147d9b45b44,0x00008066addd2067}, {0x0002ad8f8a3f0b44,0x0009a0ace2a215f9,0x000b38b809e0e489,0x0000527dcd0aadfa,0x00006506ae957020}}, + {{0x00069f78fca399da,0x00016207bb63429a,0x00088f582fe9e27d,0x000f6e4c655ed687,0x0000426d7494db75}, {0x0005c02ca81c66da,0x00070531d4251869,0x000ff48ba84fb8d2,0x0002469a3a8956de,0x0000f1d0d57166d2}}, + {{0x0009565352c4b5c2,0x0001390bc3e25a05,0x0006f9f5f4926153,0x000a9b609f7521f6,0x0000baef6bfe70a4}, {0x000fa6509ed3561b,0x0002e84b230ce7e6,0x0004cdc691137023,0x0004157151659bd0,0x0000db83c64a007d}}, + {{0x000284d391c2a82a,0x0002758308e89ebb,0x000f1edcabcdd486,0x0006c7606f16ec83,0x000013e2c38095dc}, {0x00056f04a057a87a,0x000006b48f982ab7,0x000651c44a876550,0x000e01a252face68,0x000052b540c81765}}, +}, +{/* digit=16 [{1,2,3,..,}]*([2^64]*G) */ + {{0x0002fc516a0d2bb2,0x000bfa6234994f92,0x000c62c8b0d5cc16,0x00067f7241cf3a57,0x0000f5e69621d1b6}, {0x000c70bf5a01797f,0x000c709561925c15,0x0001fdb523d20b44,0x000f7a14911b3707,0x0000648f9177d6f0}}, + {{0x000c8b8fac61d9a1,0x0002d3c6fe8a027c,0x000bff5037d25e06,0x0002f7d08805bfe5,0x00003271e6c7ff63}, {0x000a6c0232f76a55,0x000d201ef42655dc,0x0000a51788957c32,0x0001739e728bcba1,0x0000ea60412062c5}}, + {{0x000914bb5def9961,0x0003133dd1e74090,0x0003d5e761cb69c8,0x000012b1e9c1d39b,0x0000f3338ee0ccf6}, {0x0005d0d2f5378a8d,0x00065f00cd21b1e9,0x0005fe290acf4c2c,0x0008ad9e984240eb,0x000066c038df4808}}, + {{0x000462bb4d8bc50a,0x0006091957709ad5,0x000412a68181c0b1,0x00048c4bd4fe1c78,0x0000e0341bd60dff}, {0x00045cf7003e8666,0x000a2a24a41bb6bc,0x0004c24c2f11a6de,0x000b67f407151ad0,0x00002c9d27e3a5b7}}, + {{0x0002b30614c09004,0x00017d00c24bd499,0x000c4bfa1da98d12,0x0004bc3f534dc87e,0x0000a5ff67477dc3}, {0x00096b81d7ea1d7e,0x0002a0a6d20868c1,0x000cbbd6e38cf289,0x0005b61d56cd09e3,0x0000c72e27f2205a}}, + {{0x0005719a8afd30bb,0x0007da826dce3286,0x000a8fbe08679832,0x000ad32f04e891c4,0x0000b6b6e1c9bf56}, {0x0005b11471f1ff0f,0x0008ce15baf00a69,0x00096c43ed76c338,0x000157118edb95be,0x00002beaaf580794}}, + {{0x0007932b88756ddc,0x0002317e3e61e8b9,0x000e1c4a4ed4e865,0x000c0e02dd14993e,0x00000aaee18197f8}, {0x000edb96c168af3a,0x000f139ae87515c4,0x000adb4366563c7b,0x000ac00dfadb6f20,0x0000d55e8ca3a042}}, + {{0x000b12e523b8bf60,0x000b8f910c1b0a50,0x0001675888009eb5,0x000abdf535af824a,0x0000f835f9cfb2a2}, {0x00029312afceb620,0x000a169d383ff59b,0x000ac02b0c797df2,0x0000caeb3f5fb066,0x000029d4c6fdaa2d}}, +}, +{/* digit=17 [{1,2,3,..,}]*([2^68]*G) */ + {{0x0002010f5b343bcf,0x000a02f142fe58af,0x0005f4bdf0f2e400,0x000aa84483bfdea8,0x00000b1d093f3bfe}, {0x0001b95c70816030,0x00093dba10972ea0,0x00038f3a6e943e4c,0x00063647be92adb4,0x00000bb7742e5bf6}}, + {{0x000481b7bfe71786,0x000e05405868b674,0x0008c867d4e1deba,0x0000e9a61b2821c4,0x00009c15b35b13b3}, {0x0001666368710884,0x000cd220b1ff3b4a,0x0003d9f4de5e29f5,0x0006750b82bb3523,0x0000e07633358cdc}}, + {{0x000c39731815e696,0x00067cdd28023a63,0x000b4f6af6df9cbd,0x000977ec47ed4a15,0x00002009d82cac0f}, {0x00080d28b898fc75,0x0009dc17c91f664d,0x000ae660972f1eed,0x0008954e84d3bc7a,0x00008c7c19578376}}, + {{0x000f5c7a3e6fced0,0x0005f45fbdeb0d53,0x000339a70e8cbbdd,0x000b81f85c01df13,0x0000ff71880142ce}, {0x0008774bd70437a2,0x00019a0bda6a4c4e,0x0008bd26e5fb3289,0x000521fcdbebd2f1,0x0000f9526f123a9d}}, + {{0x000c905a8271d7e9,0x0004c8e5810ba752,0x000925aeb4735dfa,0x000853518a44ee5d,0x0000708697fa3c8a}, {0x000d540bfcc9a0be,0x000c0574d4038377,0x00060a8a67b27e01,0x0008be5d3d180ccf,0x000048ef153d1c29}}, + {{0x0002e8c0aeaa3c09,0x000e1be85b0a313a,0x000a97d3a89f46d9,0x000cba42889ebd2d,0x0000484026aa03f8}, {0x0009f8a87d83b5f4,0x000671ef02955215,0x000c01acfdb2220d,0x000fe37da0746e03,0x0000f97b2a754419}}, + {{0x0007ac2c880e5cac,0x000dcd11c450ccbf,0x00077a6bf8299dbe,0x000792fb27d11b0f,0x0000601630c1edce}, {0x000b9fb79e7f8ea9,0x000764367288db73,0x00035571be448bba,0x0001a265b6416fb0,0x000081b8f5e34889}}, + {{0x000305192c4d6840,0x00057612efcd40ce,0x0009cae208b04d72,0x00056cb9dcda366f,0x0000edc4d24f0588}, {0x000e6bf854279005,0x00058c09dfea64f2,0x0009bf26c3de8129,0x0005a9841b448737,0x00000b62c6dbdf13}}, +}, +{/* digit=18 [{1,2,3,..,}]*([2^72]*G) */ + {{0x000fb5ed005832ae,0x000ab1042e4f0db2,0x00070f8ca5f5efd3,0x0009cbac4ffdc6ed,0x00004645d0c952da}, {0x000f58bc9001d1f8,0x000bce1172059596,0x00098a08452c8f0b,0x0009de7d4aa0d2e3,0x000015bfe3a904f4}}, + {{0x0003acce548b37b2,0x000264d4054954eb,0x000341b4fb38e754,0x0007fa6c3daa517b,0x0000f6928ec890bf}, {0x000b32386ce6c413,0x0004e0adadcd0496,0x000b5faf901be1c5,0x000985904e67e74b,0x0000cbaf679115c9}}, + {{0x000226ad7ab9a2d8,0x000cfdfae958524d,0x00051d8c29c00090,0x00062c8ba5f53987,0x0000afcbcddbab82}, {0x0002729e99d043b6,0x000b4ebc943a5739,0x000862935ef51263,0x00017b3feace9320,0x000039efc04106c8}}, + {{0x000be7d341d81dcf,0x000842148379e839,0x000026eadcddb688,0x000c5dea6211a1f7,0x00003b25760e4d1c}, {0x000c8f6a7a73ae65,0x000e11d5b48340cf,0x000a50ebc83879a5,0x0008fa75acb1ed41,0x00009a60cc88c07d}}, + {{0x0008d4ac3b819900,0x00029e0cc8fedec9,0x000b427b91cb8372,0x00066cfe0b0491d2,0x0000f2386ace983a}, {0x0004d1eb32912137,0x000de9a62ae4930c,0x0003e89e3a2f82b2,0x000c7f07233853f9,0x0000f8063ac81777}}, + {{0x000c97c593710000,0x00007f759c18604a,0x000db6b65e1c48c7,0x0004953f62ecc5a5,0x0000a78b17338a21}, {0x000819dbcc8ad945,0x0006889c34006be1,0x000b4840a70dc04f,0x0001bff62557b4a6,0x000044c6adeb0bd2}}, + {{0x00007cf02ff6072b,0x0008dad98cdc36e6,0x000f56609a47d2ca,0x000da00f471d1ef5,0x0000cf86624a264a}, {0x0000687aa9e5cb6d,0x000147401c6cb70c,0x000a61435c98124f,0x000ea5b189635fd4,0x000028fb8b079d98}}, + {{0x00030b665c7322db,0x000103c1b3fb4395,0x00072f685cf12cc0,0x00091d170b018601,0x0000915ee22cb583}, {0x000f03ba317db248,0x000897b8ffc49afd,0x000d3d05087dec65,0x0000e6ff46597be4,0x00000a1c1ed80650}}, +}, +{/* digit=19 [{1,2,3,..,}]*([2^76]*G) */ + {{0x000a7b397acf4ec2,0x00003ea8b6403e22,0x0009692850426c40,0x0006703e3295a64e,0x00002aabc59c6a45}, {0x000714c5f5942bc4,0x000dba3182edb929,0x0004152ba9a6168b,0x000367e216a66510,0x00006908d03f6926}}, + {{0x000d8745a1251fb6,0x0008672725c7a9f5,0x000ffe89e967747a,0x00035db95c33e531,0x000009d211049649}, {0x0006ca82fe122271,0x0005f426469dcafd,0x00093183caf9b5b9,0x000fef1e9ee04c56,0x0000084a333d8146}}, + {{0x000b88210395755c,0x00039ec1df80ce06,0x000f55e96117ce63,0x000d1e3efae513ef,0x0000f36cba7bd7fe}, {0x000eca9a40ebf884,0x000f73d37e127340,0x000bbf9ffe6ec1bc,0x00005e8a51b64e86,0x0000e0dbb58cb40e}}, + {{0x0009933aed1d1f71,0x0003405630909664,0x0002e39cf566eaff,0x000124245057f0ad,0x000048ff65b2f832}, {0x00089d4cf94cf0dc,0x0003920c58b3042e,0x00061aa0d319bec8,0x0007ac6a26762653,0x000086fa3034fbc8}}, + {{0x0007b9c7ea2ee345,0x000bb9cc3a71359d,0x0001ea37a3fd0d94,0x000c876b53c31c3a,0x000033425fb1f818}, {0x00099c3810156e0e,0x000350c164487cd1,0x0006425420ea020e,0x0005ea0557ba094a,0x0000657e7e87465f}}, + {{0x000d2ab5c8b06d5c,0x00023e4eac46fc83,0x0006f7779b1a785a,0x000504f99315bc84,0x0000f31d817af9ea}, {0x000fe6a15d7dc85d,0x0003e4016b332391,0x0001cb4c72f132b0,0x0005a059547fe318,0x0000b66d8a735015}}, + {{0x0000e8b593d070f7,0x0005e255625d59cd,0x0007a03994375751,0x000eeae51fdda75b,0x0000bb6e6b09dec1}, {0x000b662334c0922f,0x000e1cf41b79729b,0x000f324023df631d,0x000c8711abf3c578,0x0000cb4666d8cd33}}, + {{0x000d7e1adc1696fc,0x00024acd72d06b66,0x0001b743598ebe59,0x0005eba5f24550cc,0x0000e23139474b9a}, {0x00022d4db067df91,0x0005baff9b00234a,0x00000c9c198dda09,0x0006950bbc75a061,0x0000560a9c8a39cf}}, +}, +{/* digit=20 [{1,2,3,..,}]*([2^80]*G) */ + {{0x0000f1cf1c367cab,0x000b190fbc7de405,0x000a110329bc85a9,0x0003a8f373c4a2e1,0x000064232b85d039}, {0x0007eb0167dad297,0x000124b78ab2f557,0x00029348b1604f30,0x0003419baa94afe8,0x00007fbd8ddb1654}}, + {{0x0004802fcf0a7fd8,0x0008d488b01e31f1,0x00052b49842fd078,0x000200f1d78d6d99,0x0000eb572d987ac5}, {0x000a44c4d194a88b,0x00090a017e66e0a2,0x00088aefcd2b63fd,0x000a10c8efc6c8f8,0x000076f6bdafa881}}, + {{0x000932c68af43eec,0x000db03d00bda2f7,0x000b061f55502468,0x0005ad25dc978f2f,0x00009a1904ae8c81}, {0x000538d470c56a4f,0x000e293d8cedd3af,0x000108ef3159abc5,0x0001773a37245f20,0x0000a17081f123f7}}, + {{0x000a9b2b4b4b67c8,0x0000680206041fe2,0x0008058d8c1d10df,0x000fbb1d64abfcbc,0x0000943b9b2f12a0}, {0x000d9143b3def048,0x0001cce775ff90ee,0x000bc904085ab3aa,0x000dfae05fd4ca7b,0x0000b34a56562c75}}, + {{0x000acf88e2f7d902,0x000757be32cd5c18,0x000eb5ee9fdbf33d,0x000114ea085cd7d2,0x0000d702cfbd3201}, {0x000ebdb85c88ce89,0x000b8e01d617b6e0,0x0007333ac23a3ce3,0x000b6aa041618e56,0x0000dd0fd8fa57ed}}, + {{0x000610798fa7aaac,0x00043073aa4eb2b2,0x000d6b19b41209ee,0x000caf31570359f2,0x0000be6868dbc577}, {0x0004bdc32c04dd3e,0x000defeee397186c,0x00086c0cfa6c35fa,0x000fe1d4a1b312f0,0x00000a5ccc7b9461}}, + {{0x000f3a36fa6110cc,0x00013b93561f516f,0x00057522b74fb1eb,0x000dc5ac0c904784,0x0000fd321052bb8b}, {0x00084a2cc80ad571,0x000576a9b6372d68,0x000f4e8cd7c27fc3,0x0002f02461baedad,0x0000d56251a71724}}, + {{0x00011a9431dd80e8,0x0009f3306cd9b840,0x000b3b730eb7c7cc,0x0003d2a0fadd29d1,0x00003858b5c7e37b}, {0x000d193b6251d5c7,0x0002a352d952bf4c,0x000fbc0511cca1fd,0x000636566157a490,0x0000990a638f9b98}}, +}, +{/* digit=21 [{1,2,3,..,}]*([2^84]*G) */ + {{0x000692a87dec0e16,0x000d97b39d00e5aa,0x000cfa0b5010ded8,0x000a281b1b80c854,0x00006beb87700f8e}, {0x000f5313476cd0e8,0x0005308d394950d7,0x000479fc6a63d0e6,0x0007419a09eea953,0x00002ae98927499e}}, + {{0x000b9105ca7d8669,0x0001aadb3b34ab58,0x000eac0bc582967e,0x000af4f9ae4447cc,0x000019c667d0bf56}, {0x00017b160f5dcd7b,0x000f2dcaadbc9aec,0x0003467f5ec697b9,0x00032e5b98f34146,0x0000187f1f859671}}, + {{0x0007a1d214aeb181,0x000c641432f790fe,0x00091a0c41506af3,0x000bc3ab5565f9e5,0x00000d41a77c44f1}, {0x00065e4a84bde96b,0x0008420a6a1ca09d,0x0007f9ce742f060d,0x00039eb52a3bfdf2,0x00006bdb65ceb3d7}}, + {{0x000dcb6ec7fae9f1,0x0004dfb66e5aeb5d,0x000445d52995f271,0x000620cee95d8e69,0x0000b6c2d4619e27}, {0x0001c318129d7161,0x0000f958c1aa3262,0x000f4af63b03909f,0x000df67c468ef91a,0x000062c42a00ba5c}}, + {{0x0002343753b9371c,0x00099f1f9cd72f68,0x00045db9629cab45,0x000998971623abb2,0x0000507db09ffd79}, {0x000f652af036c326,0x0007a5018e5c4e2e,0x0008be35086f0cc7,0x000327610a73d4ab,0x0000519b397de826}}, + {{0x0005eef9c053df72,0x00079300ea6fe8cb,0x00049cffb8de25b3,0x0009bbbb03fa92c8,0x000042e43a808416}, {0x00051f4dd6f958ec,0x000f34445a8de4fa,0x0000d89496925a77,0x00039026e72a50e9,0x000066648e3eb1f6}}, + {{0x0001957173e460c5,0x0004e0704590b2ab,0x0001c71621bbbce7,0x00065d70a90dbddb,0x000005e399e65cdd}, {0x0004dcb57797ab7b,0x0009ba2ca8e86843,0x0003336c160ad35b,0x0000149bfdb1e0de,0x0000bef99ec88b39}}, + {{0x00096f31711ebec8,0x000f4e98fdc46c3b,0x000b4411f2da40f1,0x000bb6399774d357,0x00007c8bdf495b65}, {0x00089e3c2eef12d5,0x000aec7471f3da3a,0x00012c594de95bb9,0x00056b100f225bd8,0x00004907c5d7b75a}}, +}, +{/* digit=22 [{1,2,3,..,}]*([2^88]*G) */ + {{0x0001db6f79588c00,0x0009b55768cca80d,0x00054438afa52fc6,0x000a4f0b4df1ae7f,0x0000cadd1a7f9b46}, {0x000a6b31803dd6fc,0x000495eaae35b40e,0x0002e4e16488e4fa,0x000c97df047d5538,0x00009b5b7e0ef6e0}}, + {{0x00038b67c4a658ad,0x000870e72182c127,0x00098e44fb3c4763,0x00085e6b77be4687,0x0000c047df2e7a7f}, {0x000d4c55e59d92d3,0x0005b8e64d8d2439,0x000ca9b16cedca47,0x000dfe7724cd0d87,0x00005e4fd59d5540}}, + {{0x0000e0683a7337bd,0x00042fecf2494b7d,0x000a2b71f1e3416d,0x00026c54840eff66,0x00000d9a50b837cc}, {0x00081506fe28ef7e,0x000543324c7fe219,0x0009b52633cc5ef1,0x000474420f345576,0x00002ade2f2810bf}}, + {{0x000344f3a29467a3,0x0009951eba6d9894,0x000e5c2f2de81e94,0x0007b3aaea066ba5,0x0000fc8a61438c8c}, {0x000f88f06d0de9fa,0x00049b75ce0a7adf,0x000bc87d5bbc11cf,0x000de1ffbb7accfb,0x00001458e271badf}}, + {{0x000c8c7dacddb7dc,0x0003be1edcad03b6,0x00008063392ed500,0x0001cde0e46c2f54,0x0000d37663e06dec}, {0x00084c5f365b7cc3,0x000ab79bb95d3969,0x000b1d3c1294e3a2,0x0000f56aa17d7727,0x0000ffd3cfb14944}}, + {{0x00093e3abb830d13,0x00022c2c5270041c,0x0004b259d2ad2353,0x0008577efd1be2ee,0x0000ef267774eadd}, {0x000f7039b0d7d86e,0x000c9b7e6f202af8,0x000c8e29580f5af2,0x0009f5d5fa1d3cce,0x000073f3903d68f0}}, + {{0x0009d11399f9cf33,0x000dee3c43942667,0x00098daf178e7a48,0x000ea2d8722dea0d,0x00007e7ed58b0030}, {0x0001ad43c8aae72e,0x0004cc729695f373,0x000c283527878be9,0x000b608a643affbb,0x0000f8b801c58759}}, + {{0x0003668e039c2560,0x000b7c17fd5d1cb4,0x000aa062b5f26fb8,0x000f04eee426af79,0x000072002d0d78fb}, {0x000a237e84fb7e3e,0x00002c82133d4c9c,0x0007e4181b401d8a,0x000151caa525926d,0x0000943083453dbb}}, +}, +{/* digit=23 [{1,2,3,..,}]*([2^92]*G) */ + {{0x000da31be24319a2,0x000bc095a8e7f92d,0x00078218503f7d28,0x000dbc852fe84098,0x000076ddafe49c24}, {0x00054961d7a64eb7,0x00090f1dbe4280cd,0x00038d2d5e436088,0x000035bf81a87784,0x0000e4d52a8f5169}}, + {{0x000fd9d615faa8f2,0x0000768554ed7b15,0x000a448828fa1eb4,0x000f325bb4447e7a,0x0000bb2d0d1229ff}, {0x0002a646caa6d2f2,0x000e02e7351b075e,0x000506c628eb879d,0x0004dc9cd5624e9a,0x000018eaef0c87e2}}, + {{0x00047d83d30a2c5c,0x000e378a81dc1fe6,0x0001a4a9b0857f77,0x0003f451d5a33413,0x00000a94af9e9d39}, {0x0005c0bdaa6ec1a9,0x0002f8d2d7edbc3a,0x000614797ba9fe49,0x0005332b4335b4bb,0x000091c4d6902f83}}, + {{0x00068cee978a1d35,0x00032ab92d0477b8,0x000a5b862e3a68b3,0x00041d0102979487,0x0000f0606c38a61d}, {0x000be276f9326f11,0x0004b6fe3c2e2814,0x000df73512f521c1,0x000e4407464d7dac,0x00000f5f9d3877f7}}, + {{0x0004ce43d34a2e3f,0x000dc43b5d611fd8,0x0009186c7ee3759c,0x000259995bc78c61,0x000019c380abbb97}, {0x00021aade744b1f6,0x000000f8056bc0be,0x0003efe11a7d222b,0x00025314be6157b2,0x0000fab2b4f6cd68}}, + {{0x0003878a4d32282f,0x000c58020ae7b6e3,0x000a9b750e36e029,0x000818f05847fb37,0x0000876812da29e3}, {0x000138ed23a17f08,0x000070b3950e84ad,0x000d67ae06d7b448,0x000af65fa8aef42f,0x0000d3eea24d2333}}, + {{0x000a522b99a72cb0,0x0007876180162101,0x000f3653e06de6e6,0x00054a5ff8c7cde6,0x0000a821ab5c7a67}, {0x000a52b7cb0b5a2b,0x000c190487907e3f,0x000ce053aa7fb121,0x0009af6a72502006,0x0000490a31fb4e92}}, + {{0x0001244c5f95cd80,0x0000f4ab95f4b06b,0x000e5836dda8c8af,0x000ffc1bae59c2b9,0x00007d51e7e3acff}, {0x0005e6ac2ccbcda6,0x000f2528c3e001e1,0x0009fead43bc1923,0x000710e3324577a4,0x00001a1b8848aa7a}}, +}, +{/* digit=24 [{1,2,3,..,}]*([2^96]*G) */ + {{0x000ee31b0e63d348,0x000229e54fab4fe7,0x000e7b5a4f460057,0x00083140493334d5,0x0000589fb9286d54}, {0x000f5cc6583553ae,0x000a025649e5aa70,0x0000446520879094,0x000c4eec90450710,0x0000bb0696de2541}}, + {{0x0001a3ea2dee7a63,0x000c434b2284758c,0x000aba6addcde2f3,0x0000a77ba445d24e,0x00005aaf668a6cee}, {0x00004a9e5aa049a6,0x000d31103e847e0b,0x000afecc3e74083a,0x000f7a5eb183ce40,0x0000b89dea04a043}}, + {{0x0000e0399375235e,0x0006d9917970b99f,0x0004ec0677614c84,0x0005201ec93ce952,0x000040e7bf97b122}, {0x0000631ee4c47744,0x0009fb04914cb567,0x0009dd2266f03847,0x0001f8896e9429dc,0x00003489b6ccc57c}}, + {{0x0009d23fe67ba665,0x000043cf2f340e29,0x000fcf9139145076,0x000ddaa45b5ea997,0x0000be00843dbd7d}, {0x0003e05d53ff04d3,0x00032de91ef7358c,0x0009ec1a0bf7ccdc,0x0009977d684dbfb6,0x000067e7cf2b01fd}}, + {{0x000d227cc2338fb4,0x000950e2615346ff,0x0001a007689ff6fa,0x0003af7e57077933,0x00003d241c546e1f}, {0x000b97dde9b62a31,0x000490ae30eafdcd,0x000bddf7d6a06e98,0x0003c4b9bf16804f,0x000070471a2e3616}}, + {{0x000a8ae3113655ef,0x000a67b83180ff5b,0x000e0eabefa2c6e2,0x000a962c48271977,0x00009f3c556237fe}, {0x0007022a42581cb1,0x000208f710e3340f,0x0002e5aa8e1de0bc,0x000940de640adef6,0x00006b2389159428}}, + {{0x00019e455950cc3a,0x000bb6b66bb83616,0x000ac6d84c71d665,0x0007e34a034b34af,0x0000987f83385e4c}, {0x00027727a79a6a7a,0x0007426d6c23a074,0x000167e1056e5d01,0x00084dde50b97638,0x0000a6c81f0888aa}}, + {{0x000f3b7b0dc85957,0x00082f1d9f2e0ca1,0x000dd82a727de460,0x000447aaf3bf39ba,0x00009356a79d5862}, {0x0002345f5f9a0529,0x0008839a42f9c060,0x0004d40fc1a8b0f8,0x000368253eee4284,0x00003b0bfe5de5b6}}, +}, +{/* digit=25 [{1,2,3,..,}]*([2^100]*G) */ + {{0x0003a5dc8de610b8,0x000ae7e223ce0f89,0x000ad6dc5e8c515f,0x00028ef774bfa64e,0x00009d20f96125c7}, {0x0000966098583ce0,0x000493f2a7d77a1e,0x000304d4aa2eedb9,0x00082d1b2820974c,0x0000842e3dac0772}}, + {{0x0000e9d74cd06ff8,0x000f2ca3eeaca101,0x00063aa2b9c17c7d,0x0004fef4c86cd380,0x0000595c4b3f3461}, {0x00000ca990f62ccd,0x0002fa0c3be5a3de,0x0008ce9f5d9bed21,0x000443a886078adf,0x0000db27ce42cd44}}, + {{0x00097befc15aa1ee,0x0007d54b07455a30,0x0009a5f1240d1254,0x000ee57bad470651,0x0000d03f7188439d}, {0x000bb6c4a02c4997,0x000d5ffe71d20794,0x0003adcaff725083,0x00030fbcad75190f,0x0000f68ea1cb3729}}, + {{0x000581d26ee83821,0x0005259d638e9c7c,0x00028ae3dcf17dcc,0x000047de8273abb7,0x0000d1129270821f}, {0x000847750491a746,0x00019de0dfb91149,0x000a435ab687fa76,0x000e3ecc2580227e,0x0000b8bdb94f1ce7}}, + {{0x0006d8af7f91d0f2,0x0001882a57289c80,0x000d767543b61b0f,0x0004c62640032d94,0x000073eb5de67d83}, {0x000abf77b4e4d538,0x000f5e4017772988,0x0005071b3b7ce66b,0x000a981fba6b3271,0x00002413c252d3a1}}, + {{0x0007b7e3cc8ac855,0x00078d02753b7553,0x00037df2f8d725f5,0x00031dad05ff64b7,0x00005fe871346d25}, {0x00004a96ab6b01c8,0x0008fcd9372457ce,0x00086699b69a02a8,0x000231cf82ac35cf,0x0000242d3ae1cb4b}}, + {{0x00035269be47be0e,0x0007eb28fea169c4,0x0006c67e5323b7dd,0x000e461a5538ba3a,0x0000f921d70fd378}, {0x00061fc3c4b880ea,0x000df8940a67f929,0x000f0ff393f6f914,0x000f9c0990eb0afe,0x00006c2921090eef}}, + {{0x00003a553fb2b560,0x00074e057f78b23a,0x000e490d96ce141e,0x000e75796525c389,0x0000bc95725a31a7}, {0x00067911220fd06a,0x000ba08b0bd61ec5,0x000ebeba9716e3a3,0x00066f91cd6bf7e8,0x00007326ca75ee6b}}, +}, +{/* digit=26 [{1,2,3,..,}]*([2^104]*G) */ + {{0x000c982cf7d62d27,0x000cb3ba815020d3,0x000763f9e1f36e29,0x000006d8ae0bf092,0x0000a527e6b8d3a7}, {0x0009097581a85e38,0x000f5c158be5b4a8,0x0007d726e1f1a520,0x000862798db37d16,0x0000802786e9113e}}, + {{0x000149e36f09ab05,0x0009fa10bb5befb2,0x000e2099803f163c,0x000bab8029704506,0x00006f0af006b5a3}, {0x000cfec70880e0de,0x000ede3d913f7af4,0x000ceb4bd7332a66,0x000f5452e6c84a7e,0x0000dc4a79b7c228}}, + {{0x00094d1f4c6b6ec2,0x0006f8b3cd9bf6e8,0x000117fbf526b082,0x000bf553f952a812,0x0000be864b031945}, {0x0008ea542099b646,0x0009a7548ce286f1,0x0005c1c9c2770b28,0x0006c337390f2829,0x000072e6a442b520}}, + {{0x0007dd0c55c44969,0x000695bbabd2c37c,0x000d7f363a6a9635,0x0001decb7e63f2ad,0x0000dce3782be73f}, {0x000a16ab2b91f71a,0x0002bba0163ce1e5,0x000e515ade448982,0x000ecf52759c32f6,0x00005e2f1f92615e}}, + {{0x000e2c847c643671,0x000f35af4ec0cacc,0x00034042c6a496b9,0x0002ea1a0836f360,0x00004a1f3901b6c6}, {0x00093633ef1f5404,0x000a32a76d93e7fa,0x000eae451d323b30,0x000f87bfeec8b50f,0x0000eafc172fd04e}}, + {{0x0009be7abded5516,0x000868b744107451,0x00010d9a903d358b,0x0002b6ed00b10b0e,0x0000392b0b188da5}, {0x000a2980b75c904f,0x000db8f7f96c6744,0x0002cf932c305b0a,0x0006c9142e421d18,0x00006fc5d518e463}}, + {{0x0005a51b3e59b897,0x0004d133a1c9e443,0x0000bee591361395,0x0001e417f4697344,0x000014710f870c40}, {0x0004bced6c446c96,0x00060c4d5368c0cf,0x00068fc37e0aa7fd,0x0000577e5d811afc,0x00001febd7317c2a}}, + {{0x00047c9d64cc78c4,0x000b5b6cb27b7958,0x0008022ab6c50621,0x000a1cc7099bf8df,0x00008f862ec004ed}, {0x00032ede1603c166,0x000efc9a9450d127,0x00029b4fc19a80e0,0x00051582257f54b4,0x00006d3b2c6a5460}}, +}, +{/* digit=27 [{1,2,3,..,}]*([2^108]*G) */ + {{0x000b7bc453cadd69,0x00072c0bc1f88de2,0x000abd3af203900a,0x000ffb2cd86e47a6,0x000011cac131502e}, {0x0000242ec965469f,0x000139e0017e2d55,0x0009798850e9f769,0x0001ee733f078f65,0x0000b87d44a3cf75}}, + {{0x000e2a2d551ee106,0x0008127e09a66066,0x00001148d87a8f1d,0x0003fda0d08bab2c,0x0000da8e4f1a24f3}, {0x00017f0cf9a4e718,0x000fbbf5cb19466d,0x00062ecc0ff50200,0x000ac45ccf97d8d0,0x00000c0d9b001d80}}, + {{0x00045ff1d7aadabe,0x000ff5f6a67c1a04,0x000cfb26f65d3825,0x0001d58e62fb0891,0x0000f1e0fa63c7d9}, {0x000c7ba33db72cdc,0x00093a7c74b247e7,0x0000a503c017cbc0,0x000a417c931590f5,0x0000ac54f61216ba}}, + {{0x0007ceb1bf4581ca,0x00060ca7b1669885,0x0009722ace635e18,0x000006878ddd2265,0x00000903c4cbdb68}, {0x000458948f214029,0x0004296abda2366e,0x00040319031b49c1,0x0003fda29c4b09e0,0x00007197ca4629f4}}, + {{0x000dd5d4b07e2b19,0x000afd9ea2217173,0x00005ab14d144c4c,0x0008f158b04ea411,0x00002dda5438e80d}, {0x0002fa8cf03dce66,0x0004ba22cffce998,0x000ad88c48b5ea96,0x00095c97f4ea7f3f,0x00002db773eca5ba}}, + {{0x0003fb4820357c72,0x000e4f1458ad18bd,0x000b44aa1992039a,0x0002817a1df3c525,0x0000d7803580d3d5}, {0x0007e4dc77ad4d4a,0x000859df4fc458cf,0x00071205ed49a799,0x0004a9a465a8b51d,0x00000ee0ea704925}}, + {{0x000e6e89c92b2350,0x000e6b3993a14baa,0x0003dd031a73bbd0,0x00081cd06d60ec69,0x00003cab91b71568}, {0x000862f1db3574b2,0x000544bb061ad615,0x000181e06485b018,0x00075707434988a0,0x0000cd61ad4e1c0c}}, + {{0x000de1cf3480d4af,0x0006cc8acf1a03e2,0x000295a9cf0d8edc,0x00097d023e330368,0x0000add5f69b546a}, {0x00097ad96f8acb1a,0x0004c71bdae28955,0x000dd43f4bddd49d,0x00041976fcd52821,0x00005a4541306191}}, +}, +{/* digit=28 [{1,2,3,..,}]*([2^112]*G) */ + {{0x000b6bfc360e25a8,0x0004875a1a788ce9,0x0001732f4e642519,0x00057a1dc756a848,0x00003c0440fd432b}, {0x000b3f1d720281f6,0x000e7135e051c670,0x000052be72205910,0x000a397ed14b0edb,0x000097b3d282568e}}, + {{0x0009b9afb3ff9ed0,0x000b17f6515c2e59,0x0004da44928c2e0a,0x0004521cbee4fd47,0x000071279a44f364}, {0x000ff6601fbe8556,0x0007ffda51c497ab,0x000597c0b3ee394e,0x00034ab90385f667,0x0000e9fccc7027ee}}, + {{0x000de9314092ebb4,0x000d028e240c0b89,0x000d2f064f17256b,0x000b148f89a7f393,0x0000f57841f21ed3}, {0x0004405e708d8553,0x0001d3f1c3d04ee1,0x000d7eed5856aae7,0x00027098e5424fbd,0x0000333e4efa3ab4}}, + {{0x0007adedda492f81,0x000a682972053bc7,0x000931b4cc11a3ae,0x0004bb4e89a3e734,0x00007512e2eaf569}, {0x00049f3177bf8b6c,0x000948c7ff3e5dc3,0x00011145d232ea4b,0x0009c2dc4f9d16f5,0x0000cf109a3f3b37}}, + {{0x0007a88a1f258972,0x000f81b5d4d8e75e,0x000f3ed5c7ac6961,0x000dfbc3e1077308,0x000008a54ec2a892}, {0x0006e1978660710a,0x0006837df2c8be82,0x000704da50cf70a9,0x0003fca18a7340ed,0x00003eeb9a9a8ca3}}, + {{0x0006233169bca968,0x0003ada6aafb49d9,0x000c2fa9404d286d,0x000fb3409606eca0,0x0000869d0d5a3ff0}, {0x00037e5d0150d651,0x0003140c14c9a999,0x0008e2d49a92e250,0x000e2b556bf94510,0x000052a733ab2f59}}, + {{0x0003d588434a920a,0x0002c22103c5b432,0x0008dbf9ac0af8e9,0x000df1c67518ef93,0x0000184307423a9c}, {0x00094aa5447ab801,0x0005b75a3d61350a,0x000411a9ee5e5a32,0x0000c564ba507f68,0x00000581fc1694f7}}, + {{0x000857080eb24a90,0x000d488e0cfd60e2,0x00059cdb87bedfb4,0x0000a9721ebbd7c2,0x0000b0da855bc639}, {0x00004dbde314c709,0x000bdc32e8462b4d,0x00062fc9ecdbf1fb,0x000ab6a3833eabb1,0x0000939b48c40dd3}}, +}, +{/* digit=29 [{1,2,3,..,}]*([2^116]*G) */ + {{0x0002c1f711b0eb9e,0x0007980ab9549689,0x0000792dbb905f2c,0x000125cce26309a2,0x0000c8ac9b3e684e}, {0x000d8b6b40a24474,0x00015fe3fb24486a,0x0008e3b3f60121fc,0x0003941626fccf1a,0x0000e568622aad1f}}, + {{0x00046c64a8a3d626,0x000fc47743d25a4b,0x000f7e4338469c4c,0x000848cbb3a13d88,0x00002b23a1061be5}, {0x00096b4a63d1a4c4,0x000a3d183f3ee835,0x000afb01c454e7fe,0x000638243fce6117,0x0000e65e5e65c4c3}}, + {{0x00077176add85450,0x000672c49b66e5db,0x000421d771b71cb6,0x000fead856073968,0x00003840fe883e3a}, {0x000dad51ec699775,0x0008e07f6726b391,0x000ca160cae243fb,0x0005f4788ac87be8,0x0000174cced9ce35}}, + {{0x0007eb6cbc613e57,0x0004d97ea61cc1e1,0x0007eded533131d5,0x00011abf69d39eaf,0x00003c2f4354e6af}, {0x0002493a4a375fa7,0x000c4833c5c24ca5,0x0006e71cf5f06787,0x000666114e091f3e,0x00006451f57fb746}}, + {{0x000ab8495fe1347f,0x00087f24503c5ee6,0x00086dd6bab0f6c3,0x000ef4907e3ffb44,0x000000b6c757002f}, {0x000f9a6a78629992,0x000c9ed89e2648bf,0x0008419eb85e5a06,0x000f1666d311af3d,0x00004f3ad7854733}}, + {{0x00003fc6c68d6876,0x000cbff052c75e3e,0x000d16e7a3e732c3,0x00074692d0efa66e,0x00003d92b27165bb}, {0x00082badd44867cb,0x000e48c081b8ffcd,0x0006c8785a71b4a9,0x0002cfbc1676a773,0x0000e2c06174d893}}, + {{0x0006d282bcffbc46,0x0006a6eadb7a5337,0x00035ae69708817a,0x000fde0ff50e05cd,0x00003b5fb75d4bc7}, {0x000e953e7fe08c4a,0x00045583ca1871c9,0x000e81c5cb4d8bfd,0x0001383e8d788245,0x00005f5e93d80474}}, + {{0x000bdef694db7e04,0x000779fcddc680f9,0x000b8dce1edca878,0x000ba111981c3403,0x0000274dcf1b0e10}, {0x00043b86def6d1a2,0x0000cbdb1866f727,0x0000c6f58d25b167,0x0007f5a4491e8c05,0x0000be2b2aba7fbd}}, +}, +{/* digit=30 [{1,2,3,..,}]*([2^120]*G) */ + {{0x0005c9dd111f8ec7,0x000d47c4e7603e0e,0x000392a51bcc33f8,0x00092d002f9a91bd,0x0000da4a7963132e}, {0x0000ae30bb1151be,0x000722e322511a0b,0x0004e9e7854febac,0x0000b80a3a508269,0x000058ffec2c4fe4}}, + {{0x000349d29c4120ba,0x000f20d0d915fbb8,0x00010ba519f94391,0x00091124074fa754,0x000066adbf6b50a5}, {0x000543c34bfca38e,0x000fd9e1ccfcc164,0x00020219ce0f2755,0x000979b9da0f53e8,0x00008234499a6b49}}, + {{0x000c513516e19e45,0x000775c4d5937b23,0x000e71ef656e2e84,0x0004c54f727d735c,0x0000b6304a7479a4}, {0x000a7363ab7e433f,0x00000e742f836638,0x0007fc19f1adea47,0x000697f054b8545b,0x0000935381baa1d0}}, + {{0x0004f9d918e49366,0x000652513982b550,0x0004d9cb965035ef,0x000508a553a0c26f,0x0000cb10d571ea85}, {0x00057b7a242da112,0x000d472b726848d9,0x00002a96b16a4d3d,0x00063b1d7e637c85,0x00007c7032b930d4}}, + {{0x0006b7d5846426f7,0x0008b47d441d5536,0x0006fbf48e7d09e8,0x000d7ce10b404d73,0x0000fa003d15784b}, {0x000614f17fd95965,0x0000e5cb98db25f7,0x00083a76a49e0e0a,0x0000f7dc65957b2e,0x0000d40da8e1ddbe}}, + {{0x0008bb4a595939dd,0x00085874021737f6,0x000ad76120355647,0x00095ebe740e7c84,0x000089bc84460446}, {0x000da5d85a9184d6,0x000b3fc0b074f7f3,0x0008a888e562563b,0x000e7ba6d2e6aaf8,0x000012d8643761fb}}, + {{0x000bba354530bb24,0x00078b0869ea9fb3,0x000431163bde3ef7,0x000a3549bc90460b,0x0000d03d7d324819}, {0x0004f9e43b6a782b,0x0006ec88a68633ae,0x000ffedd9216db30,0x00083fe1dd88e000,0x0000280da9fc2bd4}}, + {{0x00086913e538cd75,0x000c3e08ad53458f,0x0005d15ffa7001f6,0x0005dd02b8c6e6bf,0x000048234a451121}, {0x0009d2d3d5b40458,0x0005ca904190ff5a,0x000607f8bb0ffeeb,0x000729d5a3aca448,0x0000cbd665cb0a06}}, +}, +{/* digit=31 [{1,2,3,..,}]*([2^124]*G) */ + {{0x000a8f833f6746cb,0x00054ea990cac7f3,0x000ddb0a921e46f6,0x000554e15fd5c5ca,0x0000d41f01728614}, {0x0004434426ffb589,0x000584dbc204346f,0x000969b7f8055943,0x00039a63dd20fe5a,0x0000d59e9577899a}}, + {{0x0000ba9b733aa5fc,0x000b305af2353c2f,0x000ac82a5dece47c,0x00018a38e3f715a2,0x000097ba641e203f}, {0x000550409c110608,0x0004c6af512dc3af,0x000f2814656ea2c0,0x0004947ac28daff3,0x00007fab43b159ef}}, + {{0x000115c775d6ece0,0x000be8c0e78def4f,0x0005cfc8169d2e3b,0x00088bb0264ef114,0x0000a41e9fa1b697}, {0x00033be909a1f0b1,0x0001fae76b300d92,0x00032bb69150a845,0x00084ada33753706,0x00005f7b3cfba255}}, + {{0x0001641d4c5105f3,0x000e3d7fbd650989,0x000e6bdb01ae80f8,0x0008606d67225fbe,0x0000b433b59afc4d}, {0x0006db693e856387,0x000273e9862f44e6,0x0005c32ecf7b5925,0x000f506b78515766,0x000002fefd81e362}}, + {{0x000a9141339609ae,0x000d5e37eabd6eb4,0x0008c8d9c2b627de,0x0001e40a4083d172,0x000070814d5318f2}, {0x0005b5717398d14c,0x0004c003f6c94cb0,0x0008292759d37d25,0x000e4eb0577af760,0x0000b4a9a9b867d7}}, + {{0x0006a5c97290293f,0x000be388acbffe75,0x000916bdabf04a19,0x000bec0bbbb9cf5e,0x0000489527481f93}, {0x0007ec32a5923d75,0x000bade0c370dee0,0x00019d8fcc7bc949,0x00093f6d51217504,0x00004f5d4768fdcc}}, + {{0x00005f7d13fb27da,0x0002cc7195c0c20f,0x000fc56c5c05b30d,0x00082b0335cf1832,0x0000e65bcd402b3a}, {0x000aab8630d99eaf,0x0005f62cec6ccbf6,0x000eed2f1164be81,0x000d0c7d41819d2f,0x0000cdc5907fb91b}}, + {{0x000475d0fefb0c3a,0x000aa6d7c35d3754,0x0003798a4d48fb56,0x0008e60070b63336,0x0000e89f3d32fdb9}, {0x00089c86363d14cb,0x0000b7abd27d970b,0x000d5a0218981752,0x000aedebf7d47444,0x00003083bb07ac72}}, +}, +{/* digit=32 [{1,2,3,..,}]*([2^128]*G) */ + {{0x000c244bfe209256,0x00032fdce86762a8,0x00038706391c19ac,0x0004f5fa96a5d5dd,0x00001d587d481d32}, {0x00073a2a37173eaf,0x000763778b65e876,0x000bab43e2384800,0x000fbd20f8441e05,0x0000a11fe133621e}}, + {{0x00049feb8a24a205,0x0001a52ca53f23f9,0x000fb485317ebfed,0x00005d4b691bbebc,0x0000617ff6bb278a}, {0x00034c5e3c99ebdb,0x000d6784156a241b,0x0005d67dffc64242,0x0000109206482f69,0x0000967ce0f9e27c}}, + {{0x00034a3b23358342,0x000a70ef6860c0f7,0x000e2bb0d9526205,0x0003faab8be71704,0x0000418871e22f38}, {0x00076814082c1576,0x000fc9c20073d717,0x00087e728cc914ac,0x0005fd9186c1ebe5,0x0000fdb3c22c1bcd}}, + {{0x0000f0441c23fa36,0x000061989a2eb448,0x000a29ca7b4712eb,0x00028bdccbba0f93,0x0000e205c1536194}, {0x0007957b36416860,0x000d45ac8b4e90db,0x0004e03500432691,0x00051707a759acf6,0x0000514d89c9c972}}, + {{0x0004c1c2cf9d7c1c,0x000a2e95e5abcc7c,0x000ae170c1320886,0x000661fb7b9056be,0x00008a5b2519bc0d}, {0x0001432c11d23031,0x00020f03769f4ed8,0x0005398287da6691,0x000d022ac7a5fd84,0x00004dada944bccd}}, + {{0x000f7aaf0dcbc49e,0x000890bbb45b7bb4,0x0002ca2e57de551f,0x0006eeefd0f3e49f,0x0000ce58709ff5c7}, {0x0000edd167d79ae1,0x0002ea7d7ec13292,0x00030af91039df8a,0x000b59e46206c0bb,0x0000ff5e2f532676}}, + {{0x0000651cbae2f701,0x000583aaa8eb51b9,0x0001df499efc4bc0,0x0007a57ecd8689dd,0x0000aee99a832f36}, {0x00085b9ae8274c57,0x00050d30b39c95d4,0x000c1ef816c14d44,0x0002ed4afea90bbc,0x0000c5f317b1459a}}, + {{0x0002c6bc4fe3c395,0x00031c7bebdfe3b2,0x000693459ba4a815,0x000b11a23ab6b725,0x00003bc377064922}, {0x000c8ab5afc60db8,0x0004a0b9f2a34645,0x0000fc507aa02235,0x0002e6d2a2954cce,0x0000c2731bbfce1c}}, +}, +{/* digit=33 [{1,2,3,..,}]*([2^132]*G) */ + {{0x00091c2e48fb8898,0x0002fb8a9d066a70,0x00082a0e226882c1,0x00052d224986631b,0x000044ed736b5181}, {0x000476fd86e27c75,0x0009b4afefdc282f,0x00019e34da04edac,0x00078b3b256ebc61,0x00006a413e95787d}}, + {{0x000061d5a74be506,0x00047ea16ff582ee,0x000bfc8a2e41781c,0x000e2d80b0c81e99,0x000024f4d696b547}, {0x000545dbdcc9ae4f,0x0005509b1e8e3a83,0x000c935392573dbb,0x000797582960c4a6,0x000001059ae4ae18}}, + {{0x0001e274d559d96d,0x0002e8db6c013815,0x0009921af4f18c0d,0x00002879a3aa836f,0x0000beab27c5c046}, {0x0009eaa7040bf3b5,0x0004c614b091242b,0x0004baf5d39c479e,0x000944e38ede2b0e,0x0000bb192b840a53}}, + {{0x0009f973112795f7,0x0007284e6ee1715c,0x000b66bcde824443,0x000bede5cb4858ec,0x0000c1367361baff}, {0x00015955dbec38e2,0x000c188ad1535466,0x000e0952f51c0782,0x000fa87ba4c53ac6,0x00007e6782a3b21d}}, + {{0x000c251ec5d7f652,0x0006703940877d89,0x000691ab30c8f561,0x000b21409e1cfcf0,0x0000a0300c009b20}, {0x0002fadb114faf4d,0x0008821bf5d1bf53,0x000fc36de328fc0b,0x000f5fed51f93c3b,0x0000989050f4a4e5}}, + {{0x000903d4ed2dbc25,0x00082c3b2d83682f,0x0007e93350eba59c,0x0006d73e9dc84d9c,0x0000f9b21b05eb22}, {0x000d394af267bae5,0x00056e2e15aee33b,0x0008ec500aa86cc2,0x000657ff0bf67d6a,0x0000846aa4549630}}, + {{0x000ba38e7e0c2780,0x000e588b2e6f6786,0x0005fee3a09bf87c,0x000394723b702246,0x00008b8411464682}, {0x0002ce029e646294,0x000f8ca78e430eb5,0x00054a991adb60e8,0x000fe4e0dd7062b6,0x0000281d429069a6}}, + {{0x0009740e2c2bf152,0x000589e99704feb0,0x000fbc565627a220,0x000de8cc8d73d0c2,0x000023eed8fe20c8}, {0x0002583a8363b49a,0x000929c2b0a61ee3,0x000dbc85c1a0b6cb,0x0001aba9f7c3d290,0x00008dfbb97bef4c}}, +}, +{/* digit=34 [{1,2,3,..,}]*([2^136]*G) */ + {{0x000236e846e364f6,0x000c7ea50ca0c16c,0x00026b86d7f33527,0x00070c6481077509,0x0000c2a36096598e}, {0x0005e52f024e9245,0x00044db4afcaa675,0x000831790e0fa07a,0x0000d5c5c3ce7d66,0x0000b4ef350f6cbb}}, + {{0x0004c050f15dde91,0x0007fd5f2b820521,0x000e82b62a47a76a,0x0005eeab254d3062,0x00001a05fe04ec95}, {0x000f46e9d529b36f,0x00009f9e3df67eaf,0x00031769855ab130,0x0007acd463e37199,0x0000d251439bcda4}}, + {{0x0007598a9d82abfd,0x000b16c170f5e2a3,0x00066b0875f188cc,0x000ad9b168220050,0x0000a22c21397155}, {0x0005d3afbddb4799,0x0003dd715b99151e,0x00097cb2e4b606b8,0x000b65ba73b54bf9,0x0000a1bfe43cecd8}}, + {{0x00022f3dbfb894e2,0x0006ae274b18e131,0x00058aadfbe9b79f,0x00035165a49de5ca,0x0000495775831487}, {0x000ef61bb9390993,0x0009f6d13694111d,0x000fc253b1d6a974,0x00015e1474b4ced3,0x0000a1485e67c5db}}, + {{0x000dab61430c9ab7,0x0002b238e9975afd,0x0008042ae0bdd41d,0x0004cb8094743041,0x00001f9addb3dddc}, {0x000c016c52dd907b,0x000c79e2047f7090,0x0001011a6d9bdf44,0x000c7836f1fe801b,0x000063accbd89acd}}, + {{0x000acab4baef62e3,0x0002785b91e87817,0x000e576109f5a220,0x000e036666ebe66c,0x00002ad31f4273bf}, {0x00030a425bcf4d6c,0x0002915056e66283,0x000332156ea95059,0x0002d699811c89e1,0x000089cf1ff4c11b}}, + {{0x000337ac0b7eff35,0x000e75e48b3c0ad7,0x000f13a5f8552225,0x000cbe96f78b0c73,0x0000e70062ed2349}, {0x0005048e7073969a,0x0009233cb3d26b8d,0x000caa20f392d2a2,0x0007074e4f727c4e,0x0000068c99ecccde}}, + {{0x00086ec1ed66f181,0x000bc61fce43ebde,0x000bed74d225d906,0x000ab74cab07d6e8,0x00006e4617f37855}, {0x000aaddb2fbc3dd3,0x000f5aeddf5b6568,0x000cf2fadedb5484,0x000699578f20e86d,0x0000516497c915f5}}, +}, +{/* digit=35 [{1,2,3,..,}]*([2^140]*G) */ + {{0x0003fecfa181e695,0x0000e0d69a98ef0a,0x000eab95d9ea02f8,0x00002162e9cf8e66,0x000020f2beb74720}, {0x000540a1df843618,0x0000f1fa6d5d621c,0x000f5f6ff1203772,0x000ef2ee3c7b510f,0x000017a069c2bb2b}}, + {{0x0002fb6b294cda6a,0x000519039f348357,0x0005cbb216ce9bf7,0x0000d980e012f009,0x00000aecc1c7063f}, {0x0001c3af02909e50,0x000f48ce9cdc57c2,0x000e336f8c7d59ec,0x0005f42732b8448a,0x000056e37233f4f8}}, + {{0x000b53189e800ca4,0x0006d45208fd8a10,0x00014ba3750fe0c1,0x000acc5e43c0d3b7,0x000027d200e74189}, {0x000e24fe616e2c00,0x0008ee1854c105de,0x000342a739c25f4c,0x0009524d3222a58f,0x0000807804fa027c}}, + {{0x000653a4f0d56f34,0x00078a28b805c222,0x00073434b961e404,0x000a18ec03f8b04a,0x0000c966787eb712}, {0x0006c42864fee422,0x000a3b0ece5ccc19,0x00031c159c1be93d,0x000655887d9f22c1,0x0000bb6d593fce45}}, + {{0x0009ec9b809b7ceb,0x000b32c72c2c22c4,0x000a0bf368a41486,0x000c68d13b9420fe,0x00003d36eea566da}, {0x000c08a328cc987f,0x000b4a3264616fdd,0x00010dbba0a3bcd2,0x0004c38103c49dd8,0x00009d81a293b78a}}, + {{0x00065ade4d559419,0x000da03840873de8,0x000f18b9bdedafa5,0x000267df414abb4e,0x0000ee9ea438aee5}, {0x000aa1637a55a4a5,0x0003b15f93b9260f,0x0009c3598eb19a51,0x00078e01d7ebd29e,0x000023fc56d69321}}, + {{0x000070cb98fe684f,0x0009224a1458501d,0x000bc6b3fd60fbe9,0x0007cab45761c892,0x00005384859ee6f2}, {0x00071f7b59e763bd,0x00088b5a8e5e4b02,0x000a482923d4606a,0x0004454eda5d9b05,0x0000a7731d1b6fec}}, + {{0x000369390d458714,0x000fc6166d8da3e3,0x000a90403e976403,0x00063775c3368289,0x0000bd17983b2f1d}, {0x000679ed5d2c53a7,0x00088dcf3b87a616,0x0006a694e5ec4bcd,0x0007e53e6d7613b6,0x0000460fc7753fc2}}, +}, +{/* digit=36 [{1,2,3,..,}]*([2^144]*G) */ + {{0x0001fe1c63c4962c,0x0008d81fdb258053,0x0004c2b6b50541e8,0x000fca1c1291a1fd,0x00000693a1866df4}, {0x000604e0117f203b,0x00025a99b8d0b2c4,0x000212c44245f196,0x0002a7fedc20aac6,0x00001ed4e57020f5}}, + {{0x0000fb6700a1acd0,0x0003fd999681b556,0x000b4e1bae823fd7,0x0000a3da915d1f6c,0x0000d0301186ebe0}, {0x000b0c989fca8cdb,0x000b49da0e0b744f,0x00031d76f970d01d,0x0009695ad8c56479,0x000015737c0a659b}}, + {{0x0007a9c6bdf22da8,0x000f10dc82df18f3,0x000703651efbc432,0x0001a5452cef8e5d,0x00002887ba159988}, {0x0009ddab920ec1de,0x00030c3e8d3b7cec,0x000a88747d0d7e8c,0x000534645bc3954c,0x0000deaa2e17fd53}}, + {{0x000384ece53c2d04,0x000d1e4606daa12b,0x000ec12b0779d897,0x0001ad653e47b073,0x000062dbbba9756f}, {0x00009f2cafe37b68,0x000f6cce2e1769fe,0x000f607fd273d1eb,0x000c250ac1d5383c,0x0000035f7ff92e10}}, + {{0x000cc0f296c9005d,0x0009ab0aebdbb7d4,0x0000ec8d44b9094f,0x00010111bf10f1c0,0x0000807b1c5a667c}, {0x0002cdfbe7133835,0x000e31142ba1a941,0x0000a6bdc435e063,0x000ab9484c15ecaf,0x000092c2460e2a3d}}, + {{0x0002d5a2093c22a9,0x000145703aedca44,0x0003287b6ebd0bd3,0x0008b9a08f2afd65,0x0000bb88bac9d1bc}, {0x000853875c1e3b2d,0x000ffa11447cfbaf,0x0005c4c8dbd2ac94,0x000207586d816cea,0x0000c3aa800f8dc3}}, + {{0x000690016e23e9d2,0x000b87cc41e19365,0x000d6245ccb220c6,0x000a6a936b20c369,0x0000d63c348d62e9}, {0x0003e19cdc0bcb5a,0x000eef601b98a347,0x000e346e470f18b3,0x000a63ead7a2c7cd,0x0000e9f6ec45d3aa}}, + {{0x00034c77e6c55202,0x000fbcb9ea58854d,0x00086666dc27df9e,0x000a85205f2369d6,0x00009d1febf2417a}, {0x000819e93470afed,0x000912a27f9e9846,0x0001e65043e6a966,0x00080954d008a2e3,0x0000ba7ced06cb76}}, +}, +{/* digit=37 [{1,2,3,..,}]*([2^148]*G) */ + {{0x000f541338d6e434,0x00030541d5ccecaf,0x000bc88ca56f7dd7,0x0002c375d426de96,0x00008d94f6bded3a}, {0x000a3bb2ef8279cf,0x000a1b1867f26354,0x000225151d575465,0x0000d7ff99b0ff95,0x00003e19d89e9450}}, + {{0x000f6ce51efb3108,0x0004158df5be0d0d,0x000158e59cb5b2eb,0x00033656459e2936,0x00002aae2b99466e}, {0x0008a39411aa636f,0x0004e4c0a933fb65,0x000f026b77152ecc,0x00011f010c758a49,0x00004837f98bb093}}, + {{0x000f364b71698f58,0x00021f7b605e7807,0x0003b2cbb6ba418d,0x00086f7d20b00fa0,0x000083eca385a543}, {0x000e43ff3437f24a,0x0002048bb33cff0b,0x0009df765e910b43,0x0006f6a963a12832,0x0000c1dd5575e2fe}}, + {{0x000523a626332d5d,0x00028561bb44994f,0x000845ea27bc3883,0x000089305ed4b03d,0x000039d3ee292a1f}, {0x000fdd3e7676b0dc,0x000f3b7060176561,0x00064f9a8620e35f,0x0001f676ce424ff2,0x00004c341a09a268}}, + {{0x000861cf405ff065,0x000cf86e828b1c30,0x0006933fcebac86b,0x0009479791a97163,0x00000e7c2becaeee}, {0x000a095fa90d7679,0x000b5670ab7bc3d4,0x0007b056dae60eb7,0x0002a987633a6439,0x00003a21f33a0501}}, + {{0x0001f22369b87ada,0x000892fca556857c,0x000b064663c00e5d,0x000af17ad74cab90,0x00007112386f50fa}, {0x000e1986d9bd5f51,0x000849c3463f7435,0x0007bd4b22dcc7e3,0x000f31cc7df748ca,0x00003cd4c08adec2}}, + {{0x000e46022caf46b4,0x000f5a96fe4f5936,0x0008f474e6a45dd8,0x000f15b7925434b9,0x000014104124053e}, {0x0008d1241de97bff,0x0000cd80bef471cf,0x000db0037b8547b6,0x000dfec47d3970c4,0x00001bcd329eef20}}, + {{0x00031769bb816483,0x0005353120d000f8,0x000cabc62d69eb48,0x000cb1a17d75f44c,0x00004a07f82e749f}, {0x000f787bbfb55541,0x000052e283f82c3a,0x0009213a0b06ed4d,0x0007b44722889fa1,0x000062b085eecf3c}}, +}, +{/* digit=38 [{1,2,3,..,}]*([2^152]*G) */ + {{0x0000d2f7189e71f4,0x00081ecf91e73267,0x000757a21c643874,0x000ce4d5758e57db,0x000027d09f8690a9}, {0x000308f38384a7a9,0x000420732b99846a,0x000845819aac3acb,0x000e030e94100917,0x00005cba11237ce5}}, + {{0x000851aaaca5e9b7,0x0000e6713b9797b7,0x0000a61f6518aa52,0x000b68c357e8c715,0x0000842e7e35c2c2}, {0x000af656868a5489,0x00025068fc818dff,0x000917733d963bd8,0x000327d4da5c8b65,0x000027091000b247}}, + {{0x0001e48a105fc8eb,0x0009789ba48c37a0,0x0001c2180769d754,0x000387108c6fe1d5,0x0000032dd3467bd1}, {0x00026db020b0aa69,0x000f7664c73c9538,0x0000cf95d05137e7,0x00028a366302c466,0x00009004e1242cef}}, + {{0x000c9a7d298c241d,0x000986807cfd214b,0x00064eadbe3b697b,0x0009c51f1c780245,0x0000de8cdd084814}, {0x000f0a75a4d2604d,0x000e9c1538af946b,0x0005b1fcc27154d7,0x000f81c5cc9230de,0x000088519ea36864}}, + {{0x000e4f796ea6ca1f,0x00092f7928711013,0x000658d45567cdc2,0x000e97bdb728705c,0x00007c1ff4bbe600}, {0x00086574b6cad39a,0x00041a20b428a1ba,0x000e6fdfb3d58d63,0x00060d20011cdea2,0x0000832367b1b189}}, + {{0x0008c9f0e4938f75,0x00098c83719e4761,0x0001a64cc58d47d6,0x000f66874c1a23f4,0x0000d28e06905829}, {0x0007529210466f66,0x000f66a64ef8d8d3,0x000ce6a7a2af1152,0x0002d6f5d4485c19,0x0000d0bd2f65648e}}, + {{0x000032af416448df,0x000ffc76d9711ecc,0x0000b6eae4a7e8c0,0x000593954f9805b9,0x0000d0b15333bed0}, {0x0001848d98b5ca3e,0x000c1039b3ef89f7,0x0007bda2ed01fe5f,0x000e406481332e62,0x000067cecd885073}}, + {{0x000dd1a7cb1282c7,0x00064e46973ab828,0x00008d6b2a08d762,0x0003f2fbaf8d40e7,0x00002571fa1bdaeb}, {0x000732ff22dfd98c,0x00064087108d85b1,0x00088207a87ab01a,0x000754eaaafea859,0x0000cc832f929f00}}, +}, +{/* digit=39 [{1,2,3,..,}]*([2^156]*G) */ + {{0x0001185ca8d9d1ae,0x000cf987ded2488f,0x000c46124adf2c77,0x0005f37f3039f060,0x00005d70b7651e09}, {0x00086506260e70ff,0x00070750d10582d5,0x000bac36439d75ea,0x0003289cf3d0b175,0x00003a7564e11d01}}, + {{0x000a43e3fcd3efca,0x000418088e9ab24a,0x0003d46eadd26c03,0x000a6f05ef4dc9bd,0x00002f99d592a4c6}, {0x000d3552f1da46cd,0x000d4afacdd1ddab,0x000d4057872c3f8c,0x000b94090c4eee92,0x000028bb4209a623}}, + {{0x00092d2e7417ce17,0x0001270ee7f52427,0x00067a41eff42bc7,0x000a57aff4dc6d5c,0x00007709b7b90882}, {0x000731dbe217f2cb,0x000cabb721773554,0x0001dd0592af2a8c,0x000476a8eee76959,0x0000b2930c9fbba6}}, + {{0x000dddaf176f2c08,0x0004625726581e6a,0x000342ffb01ca460,0x0008d58a404ded85,0x0000cf60f96c4183}, {0x000691cc9071c4aa,0x0003944428039bbc,0x0009c0d81fd58874,0x000f7ef7101c8580,0x00007fb754d2c456}}, + {{0x000a1d5af71013f7,0x00049bedc9466af7,0x0007370a0e68216e,0x0001cc84cba30bd2,0x0000981afbff7042}, {0x0006a679449f0e1f,0x000d1a47edae0249,0x000feca2286cfc4b,0x0008fa4073c936b1,0x00005694612f3f8f}}, + {{0x000d6715bde48f8d,0x00025189bc7dbcad,0x00009ee8ac970387,0x000ff78d45299ec7,0x00001287ee3545aa}, {0x0008874db1dbf1fd,0x000ac90c88d67d1f,0x000368313ea46588,0x0003ad90ba649a84,0x00005fdcbcf30d54}}, + {{0x000e9959890272db,0x00098e713a10cf3d,0x0008227b875f3432,0x000dc7ae13479fe2,0x00008561eaaaefac}, {0x00097a08332aafd7,0x000503809b62a6a2,0x00063036f9b0d8bb,0x000da862fa1cfd0c,0x0000a16eb562d64b}}, + {{0x0008e8a2ac13e274,0x000f0eb1a9f5f7e4,0x0001f0a624494f6d,0x0008f0adbf84eb98,0x00009badc3293643}, {0x000a541004f7571b,0x00002f1c94ee50be,0x00027bc31bac67d1,0x000e27753d73a1b7,0x00003d01cf2e0686}}, +}, +{/* digit=40 [{1,2,3,..,}]*([2^160]*G) */ + {{0x000e50f6d3549cf8,0x000f7acd665ed433,0x00011fcb46f33696,0x00085fe95bfdacce,0x000010ee2532f7c9}, {0x0000fe17159bb2cd,0x000da58b357b6545,0x0009fea72f7dfbeb,0x0007445b057e74d6,0x0000485717b62731}}, + {{0x00042e8ee36860ce,0x000c6113c22d896c,0x000104213daf04df,0x0004e93adbb7b744,0x00005fd5fa1ffd39}, {0x0005d941a4e0551e,0x000d38d101516823,0x0009845236772cfb,0x000a97476071e309,0x00004e879df3a56b}}, + {{0x0000aa9b898fd522,0x00079e9af1a76c8d,0x0004f03f82fb38a5,0x000c6fc1f2b9a93b,0x0000b1aad44e3f0c}, {0x000332e7cf2c0846,0x000ea367d26d58b5,0x0006e4a8d1c57d96,0x000b69c297eabdfa,0x00005a947eeaa0e2}}, + {{0x000afb0285b94916,0x0007be4c705eaaaf,0x000d9caab01a0be8,0x00033f9f1d4f5d2a,0x0000e349a4b237a2}, {0x00012464a1c6a163,0x0005f9383260cf1c,0x0006d5471d99e6b6,0x00089bba3d43665f,0x00006974d052f8cc}}, + {{0x000b616fdd5b8549,0x0007c728719ff535,0x000921cad592549c,0x000ef85231468606,0x00008c8ce34c11b1}, {0x00037e7e9090b363,0x0008dbf7bbb728b9,0x000d8797467fc3ab,0x0003fde2337097a9,0x0000e5adca22970e}}, + {{0x00049a1cfe89d80f,0x000cea9c8371c26c,0x000d066d2b42c026,0x0003edda6c013ada,0x0000b8f722946a4f}, {0x00079ecd850935b3,0x000ca631e1b308b5,0x00019853434c1a74,0x000f259b5fe596ac,0x00009ff21f711f24}}, + {{0x00068a7b3f85ff0f,0x000c2a888044cdcc,0x000dbe894acd21cd,0x0000d3c6719b2e05,0x0000ae1d3d97b826}, {0x000dece8a1c5d92d,0x00040c52077eedfe,0x000dd13edbca01a9,0x000aacf085549c16,0x0000c5c3baf195eb}}, + {{0x0009e148f9290579,0x000630c853df27f2,0x000e9c5ce7a64ae0,0x0002a4956cd18358,0x0000d9cce836ed09}, {0x00059796e93b7c7b,0x000181bb9e27cc6e,0x0009e29a0e1e4709,0x000644070b3083aa,0x0000f181a75e785e}}, +}, +{/* digit=41 [{1,2,3,..,}]*([2^164]*G) */ + {{0x000a13f1402b9d0c,0x00026c7bc863d3b3,0x0008c3e6e573441c,0x00057d8b301ec457,0x000026fc9c4cadaf}, {0x0001bfd7493cea35,0x000ecaf8145696e7,0x0008c608fd05d4b3,0x0002768aca2a8a6a,0x00003ef07f65725b}}, + {{0x000c9d646ac49d20,0x000b83137aa9a6b5,0x000225a3842c77c0,0x00090724d000fc68,0x0000f63cfc82fe1e}, {0x000b01bc6441f959,0x00095c8e448f22d1,0x0007fb1ba7d38f71,0x0008df0b33fa5f78,0x00004dcfda1a9015}}, + {{0x000ed45526f09fd2,0x000c6128240a057f,0x0002bfd8de8a4f10,0x000a317332efc4ff,0x000014e77a0dd35a}, {0x0006d7314faa40ec,0x000b41e5f1863289,0x000a1813e767867e,0x00079509adf8f117,0x0000b6cda7914741}}, + {{0x000de10296c36eff,0x000192c4da77211c,0x0007836da7ee8967,0x00060ac617d270a5,0x00000cd9c328cb75}, {0x000cbf7e455fe908,0x000afe7334f301fd,0x0007de4ec3fb53cb,0x000fcff81e2ea44e,0x0000adab3ad8b384}}, + {{0x000d47d53b618c0e,0x000b8a2279231c6b,0x00092d964c424f46,0x000bf19303ffdedd,0x0000971287951b5a}, {0x000a632f815561dd,0x000503c055d18f48,0x00025684f85f48ff,0x000cc2522a142775,0x00000d841a137360}}, + {{0x0008a2b599ff0f94,0x00074104fc6b0177,0x000694ff368a923d,0x000f121bfa44dfda,0x0000f7199dc37667}, {0x0008ff6e46f2a79c,0x000d29f8131dc06d,0x000b4ce7c08b5dea,0x000c3d42519a59ab,0x00004f710bd742ae}}, + {{0x000a90f0e0b040d2,0x000225ff897fb228,0x000fa6122baf02d8,0x0005570aac79e600,0x00004828817ae36f}, {0x0001d31113ec3567,0x000da5eff1f8b952,0x000d417159e48861,0x0001b7baa1d412e0,0x00001f86203c3f13}}, + {{0x000368b4ed80940f,0x00058a6fcedd3014,0x00097579f67e6d05,0x0007f58c208c49ca,0x0000e3d7a8292359}, {0x00032027e096ae27,0x0006b4b393665e20,0x000dcdffcb1f3e1e,0x000e82b6da26f32f,0x00009422f1dd097b}}, +}, +{/* digit=42 [{1,2,3,..,}]*([2^168]*G) */ + {{0x0002cfb9db3b3818,0x000e54df0a4b263a,0x00004e61f9c3a2de,0x000324f28d06e97d,0x0000b1adfbcc2449}, {0x000d9397e053a1bd,0x000696daf7076ec1,0x0000ac7abee2be5c,0x000173b0ba1e1481,0x0000d2ae779c530f}}, + {{0x000d97a205b9d8b0,0x0004056756d40435,0x000f8210e6eb8f06,0x0009ead5e88a8bb6,0x000070ef12dec9fd}, {0x00095053bcc876ae,0x0007c7404ce34d84,0x000a1db5e12a7533,0x0005acf22b49e1b8,0x0000c1f2051f4bfa}}, + {{0x000eb79b6828f364,0x0007c1bd5b9eadba,0x000844b0c9d7a025,0x000fc9ada01e0d1e,0x0000b625175c87ed}, {0x0009fdd9669b6210,0x0006f6f87b981410,0x0000df6bc88a2ca5,0x0003f9fe2eb78817,0x0000cea06f4ffa47}}, + {{0x00081b5c4e83d33c,0x00089efd488b43ed,0x000eb4d0fd9f3587,0x000393564a620f9d,0x00006927bdc6c6a7}, {0x0008df79f9e0f036,0x000e9cd7e1a945c2,0x000a348f12868661,0x0008e01cf4e8d0ff,0x0000bd4c28499853}}, + {{0x000a091289a8619a,0x000fc671b1732618,0x00090c632ef796e5,0x0008fac64e46e590,0x000038062d4be66f}, {0x0004a200573274eb,0x0003f92713946c74,0x000dc0e20d07b67e,0x0005a6891223b26b,0x0000e2d93f29b0a0}}, + {{0x0002e533f36d1411,0x00043dfca442f23e,0x0007c023ae84bb3d,0x000c3ba804a48d6b,0x0000e16a8fa86431}, {0x00052adddd472e03,0x0006dd1ee1271b54,0x000a275997d405ee,0x000b3520fc6f1dff,0x000051ac53cef391}}, + {{0x00014b84444896b8,0x000f794027fb7efa,0x00084487d64974d2,0x00089b6fdcd0e8de,0x0000c45b260ab489}, {0x000bbc2d84634875,0x0006efbc476ca8fc,0x000f443c0d1b2b3f,0x00039bd1d005b7c8,0x000018f2e6790c01}}, + {{0x0006e8c06d75fc1f,0x00064249a89f5603,0x00045e7dd2dcf7bb,0x0002a691dd1d3de2,0x0000578dc4cdbd6e}, {0x0008903df2ce7a06,0x00083c39afac4c02,0x0006404abaee3628,0x0008187c847c3114,0x0000304c0d904e97}}, +}, +{/* digit=43 [{1,2,3,..,}]*([2^172]*G) */ + {{0x0008b33070d3aab6,0x000b3a2cd5e5e4ac,0x000fc91732643672,0x00013ef2eff79b1c,0x000065ca49bf0a7c}, {0x000da59b3efb9983,0x000cd52f13415a8d,0x000f9a5308a5b922,0x0004d77e9ebbab3c,0x00005986e7c256da}}, + {{0x000cf4fda79e5acb,0x0009d630215f534a,0x00085756e68b83b3,0x000cb1ac748b2ed0,0x0000031725995d37}, {0x000841ac5ccc2c46,0x000add9d50696735,0x0001754bd7d7dc96,0x000dd54147e410fd,0x00005296e953399d}}, + {{0x000b3edf0290a8fd,0x0005fb47c387831a,0x0004efb4fcae8196,0x00010ddad7dece18,0x0000cfc53b417491}, {0x000f23c4cb632f9e,0x0005d91f80676698,0x00084180ac42a1ad,0x00026ed116a81d62,0x0000bedf5f9c9013}}, + {{0x000c4cdb30cfb3a6,0x00010c9db4c8d7e0,0x000c8d9df6d09b8c,0x00066ce0ba1a4207,0x0000fd495f77c52c}, {0x000169f275264daf,0x0005f57d8362fb0e,0x000ad722280c2b74,0x000c7afdd987f749,0x0000dc229b03398e}}, + {{0x000fe5a59b0ff628,0x0002c094d0d454a6,0x000437f1d25ec81a,0x0004dc7cfd834e33,0x0000e98378c26760}, {0x0007dd6f48485983,0x000f32fda36a5313,0x00074df4687f2c5b,0x00005640dc1c27ef,0x0000ebf428f3a86a}}, + {{0x000ac7ad0d350a3a,0x0008705a1f2f6713,0x00025404684f6ebf,0x0006f677a1495816,0x0000fa5e9a54ef40}, {0x000af7c1cdd69b63,0x00002ee86474f97b,0x00027f62a64081a3,0x00052cbb9f7f0912,0x0000e47f1dc64856}}, + {{0x0005e7fb7f01d83d,0x000a75ccf5cb6f97,0x00070930f5f1f860,0x00087852702eba8b,0x00008186df7fb5cc}, {0x0005da01720468f0,0x000100464c808c06,0x00077e1ca4247726,0x00042fd8c4bbbec2,0x00004aaea1766ba6}}, + {{0x000ed8452666a58c,0x000026a9c3c2b0d1,0x0009064084bcb6e0,0x0003ff7c57411c26,0x0000fc20755d3556}, {0x0001c505294dba30,0x00068b7dd31ea08b,0x0001eca74a30ba28,0x0002b9d70ba90e99,0x000094e142ce762c}}, +}, +{/* digit=44 [{1,2,3,..,}]*([2^176]*G) */ + {{0x000783e979f39254,0x000a6f4c89a7b81d,0x0001bf7fa1efd130,0x000a9e125c2144fd,0x0000b2969045b265}, {0x0009634b9db65b69,0x000173599d8aed8e,0x0003563f335c82e3,0x0008ab4aa7a54f40,0x0000df088ad922c3}}, + {{0x0004f124237b64b8,0x000963ecfd078d08,0x000845dd8688ebe9,0x000324d7b8a70cf6,0x000008fc59cdda4a}, {0x0002b2ba3585862b,0x00053df29386a903,0x0001ec29bb66825d,0x000dc805a5a8db43,0x0000b143a98ea1e8}}, + {{0x000ba24f111661eb,0x00040105eb049e93,0x00024b578edced48,0x000068e6dc9ba1f4,0x0000f8f66b8983e9}, {0x0004df4d7ed8216f,0x00069e2cbecf872d,0x000e73754bf07f37,0x0007075281d89998,0x0000ec85fbc7aab8}}, + {{0x000bf22765fa7d04,0x000fdd6a537013b5,0x00080db9859805be,0x000ce327a5e29d42,0x0000f53916fb76b1}, {0x000f61f33ddf6269,0x000e1085d103714f,0x000809ee34206238,0x000b1c8c50d4b7e5,0x000099f450e15f8e}}, + {{0x000be731a3a93bc2,0x0005821adc1a82ee,0x000030efd42bbf46,0x0007bba10b6fa4ef,0x000047aa4c7a7b09}, {0x000c632f60c77da5,0x000a7223523e8b8d,0x0004579cf6ffbc26,0x0000f654f6ff1134,0x0000825653ce8025}}, + {{0x00067ba4a493b31c,0x0001dbf7f0264bf3,0x00095914b54f20a5,0x0006abf696e06297,0x0000ddab96e4bf23}, {0x000c70aed25ea138,0x000b01cbbbe74ff2,0x0008544c5fa1d09e,0x00031708fc8c8746,0x000047a670de96b3}}, + {{0x000c595d314e7bc2,0x000b267899ededa6,0x0001ed5d32ee7464,0x000612fcef423c0a,0x000017e76ea89cc7}, {0x000ce1fe7cda917f,0x000a9a893f1627cc,0x000c74f6b12d8016,0x000e60ccd6de849f,0x0000a5817e3e3144}}, + {{0x0004cc5ac751e7b7,0x00028d4211bdb79d,0x000de4fc693f9647,0x0000641c72d3d2c8,0x0000b69cbf64f44f}, {0x0000ca2f4bf94e19,0x0008612894e23da9,0x00017d60b1a5325f,0x000b5c7a437f6c79,0x0000be7048726c9c}}, +}, +{/* digit=45 [{1,2,3,..,}]*([2^180]*G) */ + {{0x0009976e1337c262,0x000db73d68e5949c,0x000b768d96faadeb,0x0000697e158614f1,0x00002dfa557bcc4f}, {0x000da17be93c6d61,0x00019504f5b9ccd6,0x000694da124866c6,0x0008c61121353c8d,0x0000c6ca5801140b}}, + {{0x0004a7dd4b79bb87,0x000ae2c878c8f160,0x00047b8e8aee806f,0x000053c4144f118d,0x00002edf52c049f9}, {0x000a84e2127015a3,0x00006cb7cef3ebfc,0x0006deec89051d0c,0x000d7456e8fe5829,0x00003b2818871010}}, + {{0x00060ed9aed9f40e,0x0000732a8c99bd56,0x000c371ea70ca6ad,0x00009ce4978bfb95,0x00005464d0e50031}, {0x0002fdfd9e535ef8,0x000718c9185b1af3,0x000b42488abf57ea,0x0006fa6d7a741712,0x0000e0296a869728}}, + {{0x0009383171b445fe,0x0002a131ad4c0107,0x0003987e89bcf21e,0x000c8eacdfe205c9,0x000063f4153a92e8}, {0x00062a930add43df,0x0002d980f05a7294,0x00006e96862ebb14,0x0006b05f3954e53b,0x0000e1d75ae142cf}}, + {{0x000416e1f017d5eb,0x0005c674e99b8b57,0x000f488a03753339,0x00095dbe6d94c0e8,0x000093a787b8c16f}, {0x00051a2dcc99ccc0,0x00039aa47c1dc3ac,0x000dfd8d5c134b41,0x0001edf28fcdafaf,0x0000d57bd8e10b83}}, + {{0x000fbccf0bcfc46a,0x000e15cffee69276,0x0005d915b3a822ac,0x00076b928ed2fec7,0x0000145c11463594}, {0x00081538be17bcdc,0x0002ea6c3d8ff61a,0x00016c82f01e867c,0x0009af9634e15d65,0x00001437bd32948b}}, + {{0x000d2006c19d4c7d,0x000711b1e976d2fc,0x000f237e8a0f3c43,0x000bb120545ff694,0x0000d10ec4090bf8}, {0x000696cac7cd3e1f,0x000b6f24bfe64f89,0x000af7706ed3714e,0x000c31463eb1d85f,0x0000cbd604eb027c}}, + {{0x000c6c7af8685c8b,0x000d7f8f01aa5f95,0x00074692ad4c1c8c,0x000068144bbe3225,0x0000800347984a4a}, {0x000c6e52eca3cdb7,0x000b7c04d3997c8f,0x0002bc5cfea1db16,0x0003d2405bc82e8f,0x000063d518064479}}, +}, +{/* digit=46 [{1,2,3,..,}]*([2^184]*G) */ + {{0x000963f4c8303203,0x0000603203e3f3b7,0x000327afb842c7aa,0x0009b67f22ca0ae7,0x00008e13092c6760}, {0x000fb62757558f1a,0x000157eca8c173b8,0x0003316273cc3e83,0x00023444174474f6,0x000077989cb63c40}}, + {{0x00017f4b0166f7a3,0x00079eec74e6ae83,0x000874bfdfbd3e3f,0x0003a3cdb516ace0,0x0000d846019f681f}, {0x000ee5c7c1620b02,0x000d0b63c5010b12,0x00068c51eba68b4d,0x000b5b8c03cd3266,0x0000a6279f76e0bc}}, + {{0x000b8b0b796d2197,0x000ec4741dd9b32c,0x000edf6f5c3e95f4,0x000b8e1721212568,0x0000a03aee512b9c}, {0x000c376f53a89aa8,0x0001148a28dc0cd3,0x0002ab04f0d8af9b,0x00002d4f86a3f490,0x0000aacb62aff420}}, + {{0x000139f8f5fcda83,0x00048dee5bfdfd8e,0x0003f9f77f3e558c,0x000969a76cbaf4e3,0x0000a4c97a4a1771}, {0x000e84bf6dce6a73,0x0005e3e6c2d1da27,0x00059a6e9ff373d9,0x00062cc115193cd7,0x0000f9b702593d22}}, + {{0x000ae6c252bd4796,0x0009b2b5848f9cb0,0x000c9766305e0f88,0x00025c18f6d2b2a5,0x0000f6e149c21622}, {0x000235cde601a897,0x00088373be1fe602,0x000471827d17bbe9,0x0001165af49a5ba8,0x0000e1a0a8588aaa}}, + {{0x0006fea87baa6279,0x0001672aa6801253,0x0001e5dc958c1fec,0x0001b8dc29b63760,0x0000e3c3c1d6e9e0}, {0x000127b2bcfe0b0f,0x00013a12f50defc8,0x00079b3973510710,0x000f207ccd6cb148,0x0000792f805e8a82}}, + {{0x000572235e6fc06c,0x000e4b3e13d58b1e,0x0008a73723477728,0x000289550c294daa,0x00000291d43fbfa5}, {0x000bc67cec5a196b,0x0003ac2e8a7cc6c8,0x0006e1c51deeb31e,0x0001560a93e244fb,0x00009f8b71bde28e}}, + {{0x00084911a335cc88,0x0009ea5913e48c31,0x000b32919563459b,0x0005ac9b920d61c7,0x000005ab8b720242}, {0x00012da8d006086c,0x0009fcf5c0fd2ac5,0x0002138d76ca4846,0x000442efea51d8ac,0x0000b647545f44cd}}, +}, +{/* digit=47 [{1,2,3,..,}]*([2^188]*G) */ + {{0x00053e453544774f,0x000b4adba2bc5110,0x000e371f5834d0ec,0x0003bb5215d7f7ba,0x0000cfd57c05c866}, {0x000383dd6901b1d3,0x000485587dc3ded2,0x000625f623b49fbb,0x000762cd44a08d07,0x0000ee4d65bcde9b}}, + {{0x000137d0d63d1fae,0x000122a9d89f64e5,0x000436309658fc05,0x000a606889487450,0x00009ae30f9b598d}, {0x00010d1818baf918,0x00060b6a0c202ed7,0x0001a6b44e27e9e0,0x0007db9e28dcfb1c,0x000083acb6556ac5}}, + {{0x0002e774e6daae2d,0x0002ce0a19bced7f,0x000ae677e7b3ae0e,0x00011f03293f8a91,0x0000363b0cc15c86}, {0x0001ccf309ae93b1,0x00072920cae1be1d,0x0008edf01a3f80be,0x000c909aacba7449,0x0000e6d2a4ac2f5a}}, + {{0x000728dc2c6ff707,0x000f55dc22358735,0x000e277f979d6122,0x000cc6b3f5d00319,0x0000ee84e264ded8}, {0x000afb063cd880a6,0x0005d574af6091a8,0x000de7f423f3ea7c,0x000151acfcdc8402,0x00002d07930131aa}}, + {{0x000f5aabeccefb5d,0x00086621d7c740fd,0x000b576c1cf56ede,0x000026e632a9ce52,0x00003403ae96a6f6}, {0x000050de8785a64c,0x0006d682652e660a,0x000bcbe0210f3d64,0x00015cb8b25edf4f,0x00009710fdf74f93}}, + {{0x0004e438a5807cef,0x0002f4109a7e8e1b,0x000d59ddaad28389,0x000092d30cc9cbaf,0x000065f36c72d8d8}, {0x000469ea60d32b2d,0x000a6e8191c8df31,0x0005bdeb5ee93df4,0x000a27cc1017c535,0x000026231865616a}}, + {{0x000ade73245980e9,0x0006f1067200d655,0x000136be1a6f5965,0x00087954fc23bedb,0x0000f246cdd3f13d}, {0x0003117f961ac0ef,0x0004fbdb9e1ac2b9,0x000693bd1c8a741b,0x00001d92ede2466c,0x0000cde6b502dd17}}, + {{0x00083f9dec31a21e,0x00028ad9d573b02c,0x0007be365988c8b2,0x00034d73e983aea5,0x0000968734e446f8}, {0x000ea8f5da6309bd,0x0003f1f1ce169137,0x00044092110f3a62,0x0001b4a82a9ea2ca,0x0000f94739f2b46f}}, +}, +{/* digit=48 [{1,2,3,..,}]*([2^192]*G) */ + {{0x000410ef4f8b16a8,0x000e447b266a56f8,0x0009c87c197241af,0x000b1a8a406b8e6d,0x000003f3e034d42a}, {0x00009a804dbec69c,0x00067bbad05f7f03,0x0008e197fa83b85f,0x000dc106097273ad,0x0000097440f1067a}}, + {{0x0007fa0b311898c7,0x00045d0eac653f74,0x00014d0bce2a272e,0x000ee2dbba5851f9,0x0000a1a966134a43}, {0x00067cea1c8cde9a,0x0008d271abe3e5a3,0x0001615cd9d958ba,0x0000b053ff7eb63d,0x00002280dcf95ae2}}, + {{0x00044a43794f8dcb,0x0009983c5c362663,0x0009d10a0dcca923,0x000df27d6b6bbf3f,0x0000320c5cb31d9b}, {0x00028ff47b50a951,0x0001bef03371620e,0x000100153933e3b0,0x0008d6e081bf8599,0x000083be9a0d3a8c}}, + {{0x00085c341dca5663,0x000aa8622aa3b6c1,0x0001b6dfb7de7fed,0x00028869e84d9290,0x00000a02b0eac4ad}, {0x0001daa2fd3cf36d,0x00070f89e59fc7c8,0x000496733d131954,0x00012ae2be8184cd,0x00005f449ec63d34}}, + {{0x0000fabe085116b3,0x0005572853102547,0x000bfd52f04a4337,0x000c741e39187ee2,0x00006166b44ad9eb}, {0x000433cfd4b322cf,0x0006ca79ab5192ad,0x000db15eb726aa81,0x000e63096eacd8c1,0x0000af71e91f476b}}, + {{0x0009bdac97e65165,0x00007230f49ed74e,0x00074ea498877936,0x0005a256ec1de31e,0x000081dcee58fb64}, {0x00023918f483f148,0x000c5137d13bbaef,0x000743a426d2dddf,0x000e66d4cde50ed2,0x00009a34fc664d97}}, + {{0x000d2e949dee1686,0x0002de2af23972cf,0x00094066a1ae0522,0x000412a09e75be1d,0x0000cca31c798abf}, {0x000d61d9bc499082,0x000cd5e2bc1eb50b,0x0006f83ac4a9b4a8,0x000b28cb6cc5f794,0x00007da93fd0bffa}}, + {{0x000ec644cd8f64c3,0x000ff79d7b51c492,0x000c7525658a2d78,0x000016dced1fc51f,0x0000e658aedbf433}, {0x000942e05da59eb6,0x0002addc37220b61,0x0002e7f87ba3d60a,0x000b6e1c311cd174,0x0000473ffef56b01}}, +}, +{/* digit=49 [{1,2,3,..,}]*([2^196]*G) */ + {{0x000604f692ac542f,0x0000327b91d38303,0x000aaf9bdf079ffe,0x0004fa29f63e6315,0x000099ee566e1f34}, {0x000661fd62191997,0x0006648ce41c8a1d,0x00074d9048c883bc,0x000b1aa065118f3c,0x000013889ee7faf8}}, + {{0x0003f8f81a1b3bed,0x0004fe2764a0972b,0x000c4f5f74f3ce14,0x00085b12d0f1cc28,0x0000eee0c0e97f39}, {0x000adc0d39e25c35,0x00007467a0807df4,0x000cf5a584061982,0x0005fff40ebc9361,0x000027729a6922ad}}, + {{0x0000937b1b76ba6f,0x00045d2026dcca6c,0x000d9ae0a1a2eab8,0x000025c1715e1519,0x00001ad919aaac4a}, {0x000dfb807ea7b0ef,0x000e4ed9eb8935b3,0x0006d08abedf5496,0x0007309932e5ff2d,0x0000314874f15bd2}}, + {{0x0006a753f73f449b,0x0007dd44fc79efb2,0x000c0dc4d1d1c94f,0x0000cf99f0fbc53b,0x0000747ea0be698a}, {0x000c3fe228d291e6,0x0004e3c129d65218,0x000acc51635b804b,0x0006689ac859b8d1,0x0000c10697df5d6e}}, + {{0x000438f0876fd4e6,0x000723d2f383c38e,0x0000934cb45f0c30,0x0006edc03cc2ecb1,0x0000a8f24398c9d4}, {0x000431b65ccde7b6,0x0007c7e76a6ff16b,0x0003484d741e2cd1,0x00044a59c8cf8f4e,0x00004426efde3152}}, + {{0x0008e44fc94dea3f,0x000eead6a0b01c0a,0x000113cef34c8cdb,0x000ff9a19c384004,0x0000d32fba505490}, {0x00090f6795dcfb75,0x000333588baf58d1,0x0001fc1c0fef01b0,0x000ac94e6d1d63ca,0x00003173f9740a41}}, + {{0x000402aba16f73bc,0x0003ccf9b9fc2b1d,0x0006ef7bf2fb3101,0x0007446d51e60e44,0x0000731021c791e1}, {0x00047244fee99d47,0x00068ac5c1ea9d3b,0x000ea9af74bca48b,0x00083a00f5f514bb,0x000051f55a6074c2}}, + {{0x000251acb452fdbb,0x0004a0f306506e30,0x0003548d931ee696,0x000f5b00b3e50893,0x00008949a50a4b0e}, {0x00083263c88f3bd1,0x0000cb1d9989208b,0x000d4df03ab147c3,0x0000c5dd6515fd44,0x00007a12f75f72eb}}, +}, +{/* digit=50 [{1,2,3,..,}]*([2^200]*G) */ + {{0x0004f7881fdad909,0x00057d2cf6ab2591,0x000054de5cf638f5,0x000350290bc03fcc,0x000032811a7a8b06}, {0x000b3309bbd11ff0,0x000fb40449742f00,0x00051d26676108a6,0x0000c1801bb9e0a8,0x0000dd099bebf899}}, + {{0x000dd8a58d6cd461,0x00057e6634d214c6,0x0001bc3289cb633b,0x0007e5b1305047f8,0x00002ede0e236a17}, {0x000ca62065a6f4f9,0x000cd7be487b332c,0x00047ed1cc3a47ec,0x000b13e41eb1870f,0x00009e66e5977598}}, + {{0x000a6777b0ac93d1,0x000d68f5e0d7ebd6,0x000f5492ba6e37b0,0x000f3a1516c09676,0x0000e4bf888aac05}, {0x0002ce04df0ba2b4,0x000d1062341bcdb4,0x000acac20935d5cf,0x00000e4a30333382,0x000029438c49198b}}, + {{0x00038be67e573e06,0x0008e084c44bfb28,0x000c1c2c505891db,0x00044b3131137396,0x0000aebfa4039584}, {0x000dce9e56e55c19,0x00029caa46d0ac9c,0x0001fe8eb7148ced,0x000f4c9e10c7efb6,0x0000fd835db8f97c}}, + {{0x0006f56c17706169,0x000d79da9a2d6c62,0x000730e455351909,0x000a79558e6825a3,0x0000d8c8bc093ef0}, {0x00078b6056becfd1,0x00039090b36d543f,0x0004432f933f1325,0x00050272ad499779,0x0000386493c5721f}}, + {{0x000eefa5abea82a6,0x000933fe62d43794,0x0001ef6068dc611b,0x000e6909f1af3728,0x0000af546c899839}, {0x00078c7c977ec238,0x000c3d5c05766255,0x000d1a4c0a8de294,0x0004d462ddaf0f7c,0x0000243fc70cf95f}}, + {{0x000f400b008733a5,0x000d012e1f57e566,0x000509cd0cba0697,0x000d8c4537c2b240,0x0000f989c69a7353}, {0x000c9724c3c2b2fb,0x00084f031fa87dbe,0x000d5d11f90e02fa,0x000dfc44d15c53cf,0x00003404faef8314}}, + {{0x000a109081e9387c,0x0006cc935828a36d,0x00040b015fb9780d,0x0006fa15940332e5,0x00009d7b51ca0f46}, {0x000cd41d6d9f6711,0x0008a1a2ac17faad,0x000201e5fba6c1e2,0x00062af66a7833ed,0x00009d9971a090f4}}, +}, +{/* digit=51 [{1,2,3,..,}]*([2^204]*G) */ + {{0x000f462060b5f619,0x0003ebd057c2f431,0x000e1bf65a56f46b,0x0001fea48dca6c47,0x0000a38783ed1bcf}, {0x00033a9da710718f,0x00063e0aeaf67a5d,0x00029d1875a77998,0x000732da87314d2d,0x0000a0edc3fb687d}}, + {{0x0004849cb198ac73,0x000cdf2646651c89,0x000200678a884a93,0x0004e5fda964ef9b,0x0000c351b8730983}, {0x000ef9fe2c4b44b8,0x0006b326790cafb2,0x00002264a580f6c4,0x0004e2384805210b,0x0000ba6f9e2c2a19}}, + {{0x0006ab65eb03c0ee,0x000392bc3fde499b,0x0003a80d2f19b795,0x00019ec86b5b9c6e,0x000043775094d428}, {0x0003650bb3ee8a3e,0x000bd132075fc166,0x000d834f675eb14f,0x000ffcc8ccc9067a,0x0000a6a2475c6e92}}, + {{0x00034abd3c095f18,0x000e64b76d7139d9,0x0003e698404b261b,0x000b109d2e6970e7,0x000079fb23bde5fc}, {0x0006c72dfd754907,0x0004f1bcf1c11150,0x0005e70073a97d08,0x0002a6d3201d82bf,0x0000f0ac52fe9823}}, + {{0x00020ee4b049136e,0x0001556a4613cb4d,0x000e081288b63bf1,0x000b153221aef670,0x000062d8c522acb6}, {0x0004a67379e7896c,0x00020afd7fa571f6,0x00041ba6ab25237a,0x000e7e3077bd9838,0x00004ac0244fcd16}}, + {{0x00001fb319d76820,0x00090a982feeb251,0x00061b344b029312,0x0001fa51c1c9b902,0x0000e008c5bbfd37}, {0x000dd1c0278ca331,0x0006d5aa53b1d866,0x00013a2cf666f76a,0x000836d5cfb77960,0x0000d3a1aadb3521}}, + {{0x00004ff50f75f9cd,0x0002ae752b223c56,0x0009a11181d8eddf,0x000d7a3ef074dd3c,0x00000ffc1739cb86}, {0x000ece3037d90f29,0x0005d055856cabd1,0x0004c6dafe3f307d,0x00099fb22f93287e,0x000002aac66c3487}}, + {{0x000131a567193ec5,0x000a95f6e70b76b4,0x0001eebddaf3c305,0x0008314587bd3903,0x0000709def8c1bbe}, {0x00099830eb2b6692,0x000b675b70295705,0x00064ac164d80ce1,0x0003ab638a7da803,0x0000f431d23de1c8}}, +}, +{/* digit=52 [{1,2,3,..,}]*([2^208]*G) */ + {{0x000bc15adf7cccff,0x0005efa1e1b075d9,0x0009bc17e81a3e5d,0x000d429c39e44424,0x000037dccb37ea7f}, {0x0004873907fba12d,0x00097a372904da65,0x00083a6c535daa6d,0x0005be3564cfc662,0x000009fa4f71a939}}, + {{0x0007080eb6b242d6,0x0002db71e246832d,0x00031139dd30bd02,0x000e531027991bbe,0x00008797e91a62e4}, {0x000e20a6b4e185ab,0x000d92d9b707423f,0x000f7811b82f2c67,0x00095c75c817684c,0x0000d53005eb45bb}}, + {{0x000a29e5cfe5c487,0x000e115ee096c51a,0x00049a68a82c020a,0x0003550848ad8275,0x0000933d489d0471}, {0x0008d2e67c51e574,0x000a99944afc0499,0x000fadac60f64020,0x00092c3a299fe1a7,0x00000c73ff49aefe}}, + {{0x00049be9d8e68fd9,0x00028b044320e5f6,0x000c33398db0f053,0x000fae66fde9b3e0,0x00002f4209bf6c8c}, {0x000afcc1a739d4b6,0x000f428ab8dee9d1,0x000c6f1d009aea75,0x000aa4b4375fb5ea,0x0000420b560d08f7}}, + {{0x000ffc75488771a4,0x00013f2f2191bf44,0x000f86a42cb76e3f,0x000d9a0197bde394,0x00005c25bb9b0641}, {0x0009e31f88ce6dce,0x000c6bb7ac7dd8a2,0x000670cc7be2becf,0x00043233094214b5,0x000090a8fd640af8}}, + {{0x000fd5696f37750f,0x0006a1507ff22d1a,0x0006543ed25dda55,0x0009103b95fd4c00,0x00003c778da923c3}, {0x000f4463b04938d5,0x00062eef947b84cc,0x000e325b53d9dded,0x00055d6ed83735da,0x0000ba0f75d49214}}, + {{0x0009b8b4ebd3f02d,0x000cb6b770ea0ecf,0x000a213cea47acd9,0x0008cec3b84a6a2d,0x0000760871c23e7c}, {0x00058e536e530d74,0x0004d912ad517a5f,0x000a0252a7abc52a,0x0007423ad43db02e,0x000098b00ed0176b}}, + {{0x000499c6254dc41f,0x00038a837e7e9eae,0x0000524a77e29392,0x0005f184aec08c09,0x000082b921a7d6f5}, {0x000962e1402cec5e,0x00071a2f30e7493c,0x000b879cb9f17ca1,0x00045edcd783e8e9,0x0000a3d8c153a6f1}}, +}, +{/* digit=53 [{1,2,3,..,}]*([2^212]*G) */ + {{0x00046e60ebcf7262,0x00014231470e103c,0x0007c21094482b83,0x0006ef4f6dfaca48,0x0000e0ace9782e66}, {0x000a9d31f8d1f420,0x0001574944d23246,0x0007f334b1b1e83f,0x000d8113dfa63aa5,0x0000cf8daed9f025}}, + {{0x0001f0d1e935abb5,0x0003c54de37a85de,0x0009cebb5defd10b,0x0004be68d9e39236,0x00004d5ef9bc6132}, {0x00041ba74f17e266,0x000818c1dde44d63,0x000d918fdc0a0e3c,0x000a1346d7758187,0x0000687601562ca3}}, + {{0x000180c207674f17,0x0006c3ae8fdbbc19,0x000aeb71e112e09a,0x0001c7296675546a,0x00009432af25101b}, {0x000558fde2ddec64,0x000f1357753fd5eb,0x000e1158a81392d1,0x00099167a76b973a,0x000016fbbff8a899}}, + {{0x00032c7904fc3fa4,0x000587e3636aee73,0x00091d9aa14a23f4,0x000540838659c3f0,0x0000a995e5df12d8}, {0x0003becf3a5598a7,0x000741eaa99520a5,0x00004e03c56534b1,0x0002682ed3dca4bf,0x000016c563b48d56}}, + {{0x0006e892f3b26e7d,0x00000a8752476d95,0x00082dda3f470986,0x0002ef6ad1517924,0x000064110e3d17d8}, {0x0008d2cfad414e41,0x00081ed02b241492,0x000821bf12b155f5,0x0005da381a141bcb,0x00002e3c7705f81f}}, + {{0x00044ea308780f21,0x00012845f5e4dd59,0x00024d7a3dc8de76,0x00011e5beaba7d76,0x0000e709afd404df}, {0x0004376021704560,0x000ac8f94b649536,0x00080ca68bf204b3,0x0005744e53af7c56,0x0000526074ae0c67}}, + {{0x000a63e7882f14f1,0x0004f7c6cadce29f,0x00082bed0c9f6dc3,0x00052c36f22d6fb8,0x0000a45755be118e}, {0x0007c277c4608cf7,0x0001e68012c29f2c,0x000729b0e7ccbdf3,0x000dbf8cb0aedd61,0x0000ca2ca9f67d75}}, + {{0x000dea7e0f222c2d,0x000ba2e651425043,0x00016cd30309d42a,0x000eebc4fe9ddd92,0x00006539c7ddf87f}, {0x000a57c432ac7d72,0x0000127fda1003c5,0x0000698de72692cf,0x0003b1cc28c85f28,0x0000331fb469ec28}}, +}, +{/* digit=54 [{1,2,3,..,}]*([2^216]*G) */ + {{0x0008eeae457a4773,0x000bbe6ddc05a015,0x000c41671d19857d,0x000d588326522418,0x0000ffdfc7e6c2c0}, {0x000525426ee7cda3,0x0009af02c3a83a3a,0x0003bbfc8341b086,0x0006917023bf4272,0x0000d15002a44452}}, + {{0x000324c85edfa308,0x000407d4f3da5ef7,0x000b50c862597655,0x00096bb52f5bc0dc,0x0000f6927b0c832a}, {0x000e1ba55f2f94c5,0x0008e44b45fad08e,0x000aa455d6a996f9,0x0001f79133cb8da8,0x0000d0721ecc58dc}}, + {{0x000e7e9262a35393,0x000d3670d59ef3ca,0x000c5e1b978a49d1,0x000c1c07de0f63c1,0x0000072c30c99cb7}, {0x0008a5277c850e61,0x000f0f6a3de61d27,0x0002ca7ad84f15f8,0x0004b836a8bb4559,0x0000912e3eef4d42}}, + {{0x000a92079e5fb67e,0x000a90aa725e6ba7,0x000f5d837e1331fe,0x000e207080ccf57d,0x00004cae01e5ff72}, {0x0003ee60412a77db,0x000c2f449025d924,0x000ef5a3106ff7ca,0x0007a80e75f7cd23,0x0000c957822bddef}}, + {{0x0008086365e668b9,0x000a1abda5fbdc98,0x0005f1fbeada8dcd,0x000fc32c146b4c25,0x0000cfcde2a5f34c}, {0x000453e7e85d1e48,0x0009792358b5acbb,0x0000823ff9ca0967,0x00011d95fc2d9624,0x0000d65adf78c11d}}, + {{0x000230cb0ce1c552,0x0004ebbfb6078cf7,0x00016363b5b534d0,0x000e82ce1ef1130e,0x00007e0aa7ad4999}, {0x000ac2d79362c410,0x000091bb6cb0ce1d,0x00023df2467920c9,0x000f281e648d6322,0x0000f7d9eefe32e8}}, + {{0x00057f10296f4fd3,0x0003ba51b4367755,0x0009508051dca76a,0x0007f1c3e98f60fb,0x00001ff32eab31cf}, {0x0007bf18d2c714bf,0x00083e9d2aca643e,0x000dc2d2364b5c33,0x000b9ab9fd9ccc6a,0x0000c2397edbc721}}, + {{0x000f39afa8338349,0x0002163285626943,0x00070fc102295172,0x000e6cf1d63dd541,0x0000f5fa5903ecc2}, {0x0008725e77d9a3b0,0x000a6384ebe0b66c,0x00045e24a11235ce,0x0003b106a8c11858,0x0000137b286ebd09}}, +}, +{/* digit=55 [{1,2,3,..,}]*([2^220]*G) */ + {{0x0007d6ac42bd6d29,0x00082b1f96aedb56,0x00043b28e6df8646,0x00023f7efe5b1a48,0x000061bbb05f379b}, {0x000f5f070a6a26b8,0x000cb28e6e39b6ca,0x0005fc8d370686c0,0x000dc900da06cf89,0x000004d88113363f}}, + {{0x0009ce74462007dd,0x00047cb5f5b763b9,0x0005edde7b8ab48a,0x000fd9cec673d2f5,0x00001567f755cfae}, {0x0001b6b0887bcec5,0x000e9178f3c24638,0x0006266cb694497c,0x0004130e6525e31e,0x0000931de26b97d6}}, + {{0x0009da11f17a34c7,0x0008b35a145614e4,0x00050363b5420ab3,0x000b6e476372412f,0x0000b15d62433fab}, {0x00040b1e274e49c4,0x000456b1860aa0ef,0x000afe5a45cf5074,0x000e9a96583fbf66,0x00004240511347e3}}, + {{0x00055021a93507a4,0x00074d3c06cf142b,0x000ec3f40b4cd118,0x0003c29f70e76a91,0x000084e81ad8e755}, {0x00087b5272e9d6ec,0x000506ff514a830f,0x000192a8eea1c93e,0x000359a7cc2adcc4,0x000077e27e302f45}}, + {{0x0008b1e48ac28403,0x0004bcba9477b535,0x000946b431831129,0x000819aa58f990a6,0x0000098baf9cab41}, {0x000c1584198da522,0x000bf46bfd1b66c4,0x00036a908ab4fc17,0x0008380f0a4c3cbf,0x0000ae9e34b78cf7}}, + {{0x00040ec0ccced589,0x000b7da44f9845eb,0x0001812c625cd4b9,0x000650b3e0645887,0x00009f80d55a6cef}, {0x00040c9ce6dc1532,0x0007b86655215713,0x00007014d138d511,0x000b918cdb45bc4e,0x0000f34bb38a4b60}}, + {{0x00099e84a34f239b,0x000c090402d54174,0x0003aa83215fdb83,0x000db1075f46bf43,0x000061e15b013215}, {0x00059d4a127f89a5,0x000bb7e816daaabe,0x00018b6925d541e0,0x0000265aba0659a6,0x0000532773367266}}, + {{0x0009cf2d0c051995,0x0002aae784548cda,0x00072a182502fbc2,0x00037270bda9dff5,0x0000f9b71b8b158b}, {0x0003a592b82dd077,0x00052523032ee0f3,0x000505a327630273,0x00009f0fe1a721c4,0x0000b6e3e8367964}}, +}, +{/* digit=56 [{1,2,3,..,}]*([2^224]*G) */ + {{0x0007bc035d0b34a2,0x000b6327c0a7e341,0x0000362d1440b386,0x0009436fb7262dac,0x0000c41114d00cdf}, {0x000cef1ad95a0b12,0x000847d543622ba5,0x000e486c9c09b37a,0x00029706d6cdd201,0x00000477abf62ff9}}, + {{0x000dcb3292a92874,0x000637b092c7a004,0x0006c0605ddc15cf,0x0007afc83a846480,0x0000a68df707db99}, {0x0004e4505bf7dd0a,0x0008eccf7f8c9c13,0x000b5f8afa4e63d3,0x0001cc06e6517f41,0x0000a8b93434d7bc}}, + {{0x00035b51e706ad97,0x000453a9ebdf126f,0x000608d90b99cebb,0x000858375389afbf,0x00006113c5036c89}, {0x0008eb097e2b5aa3,0x000c33b9130480de,0x000cc066c7e1022c,0x0009000bdab6056c,0x00003cbb144e2edf}}, + {{0x00064717af715d2e,0x0003f0134a96c417,0x0001ec956e2f7f59,0x0003034c1873efa4,0x00004e7b4f757821}, {0x000ff9788d5374a6,0x000320823d5be5c8,0x000ee8fe22b915e6,0x0006518a6bc755b2,0x0000657624d47112}}, + {{0x000f101dace5aca9,0x000181a6a267157a,0x0009c8609c4fdbcf,0x000a654addf340c4,0x00007e49f5379604}, {0x000e790937e2ad5c,0x0007726e17f19be8,0x000bbc0dc846e250,0x0006d57f38007a0b,0x0000f036040711e1}}, + {{0x00000e07442f1d58,0x0000e6e0e3abd6f8,0x000c64047475607d,0x00083d02807f16b7,0x0000858e1e427498}, {0x000120b8231ee10a,0x000ac38a1ece5859,0x000aa73a41b80e7e,0x0003ac2b72525ac6,0x00007cdea3e24442}}, + {{0x000c007f8ae7c38d,0x000b6d7401925ed0,0x000e36db36db07a5,0x000045ee5e9c2a5f,0x00005b9d57b46e95}, {0x00032e78eba20f2f,0x000e81b9a35254ac,0x00098a658ef11ca8,0x000666405e373eff,0x0000fe5a101723eb}}, + {{0x0007b11e51732d26,0x0007c538fc0e5747,0x00039eec5dfd6eb2,0x000c56fc43b0cc3b,0x0000af127792b36c}, {0x000852d06c425aef,0x000b6c221b9b70b0,0x000826d9c6df92f8,0x0009c27c8d4f9ece,0x000059aba7ca4935}}, +}, +{/* digit=57 [{1,2,3,..,}]*([2^228]*G) */ + {{0x0002c2e421d3aa42,0x000cc84fa840c37e,0x00054e41cf926407,0x000643f8abc03d14,0x00006605ecd5f7af}, {0x00041a6d6a5eabf5,0x0003d16b668e2423,0x0000101021edb84f,0x000d8c8836edb804,0x0000b337ce7e45e1}}, + {{0x0006b86d23ddc820,0x000c7e0143f04c07,0x0007af2c503fd344,0x000a4fa95362ff31,0x0000add3db7e18b7}, {0x0003e3f8260e01bc,0x000494a1cc919c67,0x000f2e433fbeb49e,0x0001ead1351bf292,0x0000755e7ed45114}}, + {{0x000e368da9f3804c,0x000de164349c349a,0x00062baa5470f07f,0x000df3152f4cc985,0x000074a9e86eb290}, {0x000aa3543471a24c,0x000df8194511d3a1,0x000dcd44d239446b,0x00082cfec2dd0081,0x0000a3d7f10842ac}}, + {{0x000db47f23206d55,0x000fcd2601522bf5,0x0008ff89a2f6d341,0x000457c7b876533f,0x0000157c30c878fa}, {0x000c5c52d4fb936f,0x000bf6518cdc7517,0x000847a64ef22f7a,0x000a88eeb483e6bf,0x0000508455982e0f}}, + {{0x000c2a69b583160e,0x0003d4e59194b418,0x00083e3ffbe74fcd,0x00029b1178eeaa3c,0x0000051f896c296f}, {0x000523806ceb84af,0x000df111fe6bd023,0x0000455455ace48c,0x00014550e43a491c,0x00003fa86decd522}}, + {{0x00016fa908ec5b55,0x000d501ab12d9596,0x00082ae8a882d661,0x000f5e749f608243,0x0000cdf92ebad133}, {0x000cc425ef6c9c1e,0x0001864d84ee98ce,0x00025b8c4b52d668,0x000e87be285ed86f,0x0000b80cdc828deb}}, + {{0x00094c1ec222ba0e,0x000e723e5d482d97,0x000e4846bc3dff42,0x0001736a7cd5700f,0x0000fc5b114df135}, {0x000b25bc6b05e85d,0x0008954cd0772630,0x000d8b89d0a6d302,0x000c26e4f1f54f32,0x0000e3baff2e627f}}, + {{0x00059d8df7304d44,0x000bbf210e8eab96,0x0003fbd60b71bcf1,0x0004de69a2438bd7,0x0000595cd1f9d11b}, {0x000329a4835859dc,0x000cbdbb6e569c0d,0x000928a4e4a0f0d2,0x00015406038e5edf,0x000094296224f5ad}}, +}, +{/* digit=58 [{1,2,3,..,}]*([2^232]*G) */ + {{0x0003462f23f2d925,0x000d10b940789121,0x0006cde206cab71b,0x0004bc1bdd0a6317,0x00004c9b20d3e4d5}, {0x000d8aa9f2ac02f8,0x0006a06eedb03cd2,0x00008643403f8e61,0x000db947f68e1693,0x000031469c612dd3}}, + {{0x000a1583ae9c1bde,0x0008037ce2407aa7,0x000ab38b4e0af6d9,0x0008ca054342d928,0x00008b75007ea1c9}, {0x00086afe02358f2b,0x00063a921228efce,0x0001c67fc31b8b85,0x000d58552a19120a,0x00004069ea593aea}}, + {{0x00090cde36d07576,0x000179a293824a90,0x000b48ddff722d7b,0x000f439b7fb04c04,0x000028ad2a84be16}, {0x000bfb520226040e,0x00007104b6c4cd3f,0x00003c1886c34ecb,0x000aaf50c0754ec9,0x0000c336b090d23c}}, + {{0x000439e558df0194,0x000a6c712b279f51,0x000185a24230da4b,0x000f50118919e355,0x0000dcefcddc4b78}, {0x0000fb2a47d4c5ab,0x000f030e009ea7d9,0x000eed27355ac9ab,0x000faf4d2fc35974,0x000072d824d8bea8}}, + {{0x000f923cbb13d1b0,0x00025bfb9bfed213,0x0001144a998799f4,0x0005ee1ae8ddc970,0x0000b8b3bb64c559}, {0x000ef2e3ecebb211,0x000b2671f9a70ea9,0x0006f1d1f17cb6c4,0x00027637ef464f72,0x000071b94847943a}}, + {{0x0008552de7e5c194,0x000981c0256c779b,0x000d4743dfab2860,0x00093b24f58eeeab,0x0000e8ef838bb6cc}, {0x0000d264cb1bf3d1,0x000963dedf61ee65,0x000b70ced4c1f9d0,0x000e1e9ef7c9d7bf,0x0000ec0507e2641d}}, + {{0x0001109a607419d5,0x00026b6bca80c994,0x000c431f3faa71e6,0x000479e4158c1307,0x000094abebce92bc}, {0x000a691eb78399f1,0x00052f42cba46dfe,0x0007c04f048aafb3,0x00091addcd65af07,0x0000a29a366f8844}}, + {{0x000b2b5ef7d92893,0x0007e97f015a549d,0x0000493b62480d4a,0x00033131d5590bc4,0x0000a55b52e9f780}, {0x0008115309eadb09,0x000a02e5c62540eb,0x0006a3d5adea7de5,0x000d60d4d631f0cc,0x0000d5e9d7d23e8d}}, +}, +{/* digit=59 [{1,2,3,..,}]*([2^236]*G) */ + {{0x00060411e84e0e5b,0x0002fea34c931968,0x00073a732a5db84d,0x0007c049d5bb1970,0x00008d2fe571bcfd}, {0x0005f36f3eb82fab,0x000c4dff8b584577,0x00074c1108cb20cc,0x0008996659b65f83,0x00008b4a422e30c7}}, + {{0x000cda3af2ebc2fd,0x000cfb4efe24c4f4,0x000cd10b1a0af843,0x000e0383b857c19c,0x0000dc9d1ec614d3}, {0x000c8bb62771deb1,0x000a81c5aa817bde,0x0002391ae829277a,0x0004ca6af18dd683,0x0000740f316d71a8}}, + {{0x0000910cab91f1eb,0x00039d1cd2162d50,0x000d02252bedd9e4,0x00017a2634b74fed,0x0000d60f8e1c2586}, {0x000537b9e05614a8,0x000667af5fc5d8c7,0x0002bd926fd26c76,0x000fc78660b58158,0x000070192452cf07}}, + {{0x000e99aeeaf8c495,0x000d1e24d7288928,0x0002b156cee7aa73,0x000a1cfc5007c2e7,0x0000fcf57c63d408}, {0x0009e39b6057604e,0x0000e2868bbf9f71,0x000103e2d7d343c0,0x000ea14cca254b7e,0x00006eb38aad131b}}, + {{0x000e3520a981b0d0,0x0003bd1a41a40ba4,0x0009fab9c1c354cb,0x0008d51aabaa3adf,0x0000701a7d153c41}, {0x0007cefdcf2b9218,0x00033cf48061dd1a,0x00025cce66ceef0b,0x000e339083b598de,0x000090a54c8690a5}}, + {{0x00018db4f6d01b11,0x00027f11e8a04057,0x000591a3be73c6bc,0x0005319c11bb8ca0,0x00002d09a5a1acc4}, {0x00074eee7de13f4c,0x0005444fd682cbf1,0x00048af70177e2be,0x000b7ba5f574cb1c,0x0000e5966935961c}}, + {{0x000ed6c048752a14,0x000e601341b4c59e,0x000c6b09241f2702,0x000b232e35903b9d,0x0000291aba85f5b5}, {0x000aa70a653d61da,0x0003af2eb51e8173,0x000b93f8fd1b648d,0x0004fd91b7ce065a,0x000055408ef39e2f}}, + {{0x000624f8be762b49,0x0000758e3413b33e,0x000d805fa2a9ee4d,0x000fd7068e636967,0x0000848949c0db8b}, {0x000d7e5d23a84178,0x000d73e29da55308,0x000ee471f892f3b1,0x000089495c139e3d,0x0000631594e5757e}}, +}, +{/* digit=60 [{1,2,3,..,}]*([2^240]*G) */ + {{0x000e2ea1f095615b,0x000664e68c331083,0x0008818be0a28ad7,0x0000ccbbfc02523d,0x0000585113ba3585}, {0x0005f0b30df8aa1c,0x000b8ab7e3ac7d93,0x0002f00cbaddda07,0x000f6bd2c3429955,0x000033ed1dee909d}}, + {{0x0000d483e07113c9,0x0008ed8b63ae2dc4,0x000684c2b6e4a5d3,0x00026bc582a94b79,0x000032b33d4f22da}, {0x000f6510dbbf08dd,0x000894c23a52f534,0x0005bdc9b211d07c,0x0005573eeece0fee,0x0000f178169c7015}}, + {{0x000905a83cdd60ed,0x0004d1170184abe7,0x00023642a50602fb,0x000aff989886cdb0,0x0000568d09176e1f}, {0x00022c70259217fd,0x0008f43141e45b19,0x00095f86e93831cd,0x0008280fca35870c,0x0000ec2057b268ae}}, + {{0x0008925913cc16df,0x000cf1a26f5a568f,0x000f499ae18bc5b6,0x000e83efa413bef5,0x00008835dedb3f0a}, {0x0000bd865a40ab05,0x0008c94b377eb6e6,0x000084a696559643,0x000de06cd8562592,0x0000ce433b99f23e}}, + {{0x000523d42e06189e,0x00006e3aff13860d,0x000b20650bf07794,0x00000c2b616dcac1,0x000066dd6d201313}, {0x000fd67ff99abde3,0x000097aac50dd4a0,0x00046b2d7c990355,0x0002aed22ecf8b7c,0x0000333b1e86abf9}}, + {{0x00065e784d6365d8,0x000f0f759fb8c0da,0x000e81930bcb7443,0x0008aab5c712b17a,0x00000428dffcc6e0}, {0x000afefa4faf8433,0x000dcfa9855ff19d,0x0003ac7ceced8538,0x00071df0ac409cbe,0x000058c1fb6b82da}}, + {{0x000def7be42a5821,0x000055046be6efec,0x000e8dba9d3fc608,0x0001ffb9af13c809,0x0000e6c984774149}, {0x0004925d30c31f70,0x000aac2a21223b57,0x0000859e7b7eb72b,0x000942776a0dacef,0x00006fec31421900}}, + {{0x00094b07e50122b3,0x000b1af07ca53247,0x0003fc97bdd744f8,0x000d9d00a12f08d6,0x00009650f1aa6626}, {0x00047f71fa38477c,0x000914dc124f101b,0x0006eb58a3d815f1,0x0008865569ae95b2,0x00003cde18955fb1}}, +}, +{/* digit=61 [{1,2,3,..,}]*([2^244]*G) */ + {{0x0007a952f41deff1,0x000ad63b89b702b3,0x0003ff9510e44a59,0x000af4573257dc14,0x00009c02205e752b}, {0x0003069c4b7d692e,0x00031d1502ac46c2,0x0002208462e6392c,0x000b628057b1a21b,0x000051ff946ec1b5}}, + {{0x000cb51566c5c43e,0x00085597f0466e85,0x00094d94acff9c91,0x00027cb354e90c49,0x00000a3933301479}, {0x000fac10dc1eb2bf,0x00013ff319fa8427,0x00096527488cfd8c,0x000745f2d4e68401,0x0000a2e067e57aaa}}, + {{0x000857c5b0f7bd45,0x0008c08ea1cdb9dc,0x00084c7a96990c2c,0x000a7834730b83b9,0x000052723d33ab18}, {0x000c2e2919ba0f93,0x00091bf408909752,0x000d98212075a3bd,0x0004c841e52a04a6,0x0000fb6607acf18a}}, + {{0x0002a7f3e5f9f11f,0x0009e6cb3b8eb6d9,0x000f800bd9afe153,0x000e185d1a6dd7dd,0x00006c13cc1baf17}, {0x000c58e325fc3ee3,0x0000731dc3b215f6,0x000a3d3e77109540,0x000e2ce68e7c07af,0x0000f8417a1c4c7a}}, + {{0x00075216ce400bb1,0x0007baf07d99f47b,0x000ce62e0f72919f,0x00000e85b86e0600,0x00001872baf9fcfd}, {0x00021eb211f7dc69,0x0005c4ebd6f6049b,0x0002d78dab8900e5,0x000d9fec38cea416,0x0000a586c9e9bfa3}}, + {{0x0004772813b230d7,0x000ea7344427ec23,0x0007fc56a634d0f5,0x000f76a1548ab1d7,0x0000fab17513e06a}, {0x00010a74f7c4f830,0x0004220a67d9b62c,0x0001209a0a7d2edc,0x0009f01c40417092,0x0000b9815a0face5}}, + {{0x00083e151c3ebe53,0x0007a25d7be8ad38,0x0008ea8c9db14d5c,0x0006aaf3e4491155,0x0000a68529f6f45c}, {0x000a1dc149f75b88,0x0000879c7cb2eb18,0x00057a94e9b8946a,0x000859a7ad2a1911,0x00004b14f469106f}}, + {{0x000589b319540c33,0x00097283d6f82842,0x000ae9fcb18490f5,0x000ba072731f84da,0x0000db6d960f3683}, {0x00063bb146110697,0x000e9788bf05c85c,0x0007460d2b19436a,0x000db1205459df34,0x00003f6e095511a7}}, +}, +{/* digit=62 [{1,2,3,..,}]*([2^248]*G) */ + {{0x000a3c8ee3c76cb3,0x0003a32a1f6ef306,0x00063e9563cf1162,0x000c26b6d5ab6468,0x0000b8a4cbe8c005}, {0x00029a59ce6bb278,0x000184d4b16fdcd5,0x00023798dc4afaa5,0x000fab30624a2679,0x00005e56df6eb307}}, + {{0x0000fef4e4ca4631,0x00072566cc63b233,0x000780900bcef728,0x00027dc161d2cacf,0x000035dc5396b548}, {0x000052e27bf1bc68,0x000f87dfa06c638f,0x0003321da10a224e,0x000c8f6973586d6d,0x0000b0c5738a6152}}, + {{0x00095959884aaf7a,0x000267b348a68968,0x000147c87b1959be,0x0001f7f6250e573c,0x0000e0efb3b7d0c6}, {0x000745eca8c325e7,0x00067cff3f70ed00,0x0009ad41d3c91169,0x0007ef03acbc6531,0x0000b01a02160b1c}}, + {{0x000ba6b23a5d8961,0x00056fe4364e9910,0x00033c6771fe19e3,0x000fd05e1da8c39a,0x00005b4488b39fd9}, {0x00092541a1f22bff,0x000fbb8163e81f43,0x000e5658e920a8a6,0x00039a4fd1b24907,0x00002c4f79da6ec8}}, + {{0x00043b5224c08dca,0x0003e1b50c912621,0x000a8c84f2bbb09b,0x000ca8216ed709ac,0x00006210d9e52850}, {0x000f67a09cb54d69,0x000fc00919a46d8d,0x00013285791eef6d,0x00028b00f613810f,0x0000acede4888d50}}, + {{0x000a0691416a6a5e,0x000863ef881c84ce,0x000038a5d8f860c7,0x00006661311f8a38,0x000078c2ec1dc612}, {0x0002e815ad735813,0x000029604097494d,0x000612cbab4cc9e0,0x00039ecf558aecf3,0x00005beef7ace36c}}, + {{0x0001446de6736294,0x000e303c2d2145e2,0x000c868c757f7aa1,0x00067660e99b7f98,0x0000e42f66dcb641}, {0x00084dc910778965,0x000f72c9885b6028,0x0009a5187a0d690c,0x0007eeaeb4da333b,0x0000f789598653c8}}, + {{0x000619c76497ee80,0x0006c717370e8b5c,0x000cf68e15d2b0ac,0x00079298204cb64f,0x0000bdec21162bc6}, {0x000ccefa63b10110,0x0007e0de1ac56973,0x0000e0c8bf9e3fa9,0x000cb45efb693e3d,0x000037248e9d2d4f}}, +}, +{/* digit=63 [{1,2,3,..,}]*([2^252]*G) */ + {{0x0002dc91ec34f9e7,0x0004c38106038080,0x0000cb4f3d8772d3,0x000128cf06d66c53,0x0000be5ed0e3475c}, {0x0003c1931e82b100,0x0007c9ff6b4ccb9e,0x000a1b45ec63d285,0x000bcab92118c692,0x0000aec44147285b}}, + {{0x0009ae71e29a3efb,0x000f9c93302efc18,0x000aae10ecbe906e,0x0009f820107914ce,0x00007a23f35668e1}, {0x00075c2efd2119d3,0x000eccadc9c8e9d8,0x000a1711303198c6,0x00003835591bf64d,0x0000cf0bbf86d443}}, + {{0x0005bb72b7247593,0x000182d4c63aae48,0x0007d6f2c945353e,0x00010952159d07de,0x000089caef37ec5b}, {0x000bb53db65ef147,0x000e6d99de434a8e,0x000f2405f2dc2cb7,0x0008a3116fa3ed83,0x00003429bba31420}}, + {{0x000d590b01e6e274,0x000da180b2dcb618,0x000aea4a9047e2cc,0x0003a491b299b504,0x000012c9e1edfa40}, {0x0008a36794075521,0x0006e332b8e388d2,0x00068de1949c5013,0x000b972a1b6fcce6,0x000078851bc85122}}, + {{0x0003752fb85fa4ca,0x000d983c8ce9b1e1,0x000f74daed61257c,0x000bbb343da670d2,0x000035aa2405f846}, {0x000235d4421fc835,0x00007363473b5e74,0x0004aa158f6df8ee,0x00022de4d7f52a3c,0x00000d05aabebc6d}}, + {{0x000e735a64785f45,0x000b0f29cd078c56,0x000e35067bc56637,0x00027003b2bb803e,0x00000235a102c919}, {0x000b6d8f2c4aa658,0x00010396023b191a,0x000f805bac347583,0x00080f00400ba5f0,0x0000881065bdec0f}}, + {{0x000e522cc1b5e838,0x0001060b8bfbc370,0x000b256dfde2d4ad,0x0009972d364df067,0x0000f12502f60138}, {0x000a0dc7783920a0,0x000dc0bc866a503f,0x000064ba6e80014a,0x000ba53f89b744d3,0x00003511dcdcba5d}}, + {{0x000d46d95a7b1a29,0x0005ac6341fb197d,0x0004c2ece9c4e7ad,0x000f89b26eca2948,0x0000211e48a6e7f4}, {0x0007f6ec78ef1f42,0x000fe65745861499,0x0003eede82b2c090,0x000017f7286a6e1c,0x00005f92e472f60e}}, +}, +{/* digit=64 [{1,2,3,..,}]*([2^256]*G) */ + {{0x00070af3aeac968f,0x0008d4b63266b4e3,0x000ac5664e4f7fee,0x000cbec4acd4c2e3,0x00008910bd3beb38}, {0x000e50cc9c0726e3,0x0009a97b40bf1c3a,0x0005a5a1b1530956,0x0004cd40884b7ffd,0x0000890896b1f831}}, + {{0x0003c9f82e4c6346,0x0003da4464f85ced,0x0001dca258efb831,0x00012b8706381b7a,0x0000cd15a3cba2a4}, {0x000a8fdbfcd8fb51,0x000f5e54cd229347,0x000d8932f31db2ee,0x0001afb4aeb11ef8,0x00001e7c1ed44441}}, + {{0x00084f5903fa2711,0x0002a9da921e9968,0x000b01e54e6da0fd,0x00014e96f2f2695d,0x0000ee3e9bd78762}, {0x000181ce27a94979,0x0003fe215e04a26e,0x0002cabca36d254e,0x000613b2f32a6c25,0x0000948148810b57}}, + {{0x000b43a43228d831,0x00003ad63f99ab41,0x000a5122924ae1c3,0x0002b47e525f1a46,0x00004af860fdd26d}, {0x000ef613f714aa18,0x000d6b78795ed6ba,0x000a9d694f51865a,0x00052753e21fcee6,0x00002ceb1de0a37b}}, + {{0x0005bfd2f9fd51a3,0x0002181b97f74a66,0x00036ce507f2f1fe,0x000ded9ad05d69ad,0x000014fc2a4b44f4}, {0x0003d8cb55fc5c6d,0x0007efb1e23dd559,0x000453ccee3510ce,0x00063129b7be6937,0x00003541b7a39fae}}, + {{0x0006525eca445df4,0x0001ecdfa4c69929,0x000a6d3bcf1af24f,0x000cc7b5b4eb61eb,0x0000560910cd8972}, {0x0001c32093eaa327,0x00090d3c67bb5475,0x0008711100183134,0x000a7dcbd90ce62d,0x00005fc863ac38ba}}, + {{0x0008a4176a9f05d0,0x000b9011d488711b,0x00048a65e06ca4e4,0x000894543bc62ba2,0x000017535ffc9290}, {0x00084ce406851d75,0x000f40e960b4840b,0x00028fd34afa3acd,0x00092c5c3394af71,0x00004eb4d5b7ac0f}}, + {{0x000e87355dbd4b3d,0x00079639bbb1db09,0x0006519621f87992,0x000573e83e47e51c,0x00004ef0fb7943fb}, {0x000b9d8f1bfb12a4,0x00082e5e8b7227d3,0x0007b90146ab877e,0x000b644eebdc9d15,0x0000c2110057aa5c}}, +} +}; + +#endif /* IFMA_ECPRECOMP4_P256_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecprecomp4_p384.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecprecomp4_p384.h new file mode 100644 index 0000000..e79cf27 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecprecomp4_p384.h @@ -0,0 +1,1002 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_ECPRECOMP4_P384_H +#define IFMA_ECPRECOMP4_P384_H + +#include + +#define MUL_BASEPOINT_WIN_SIZE (4) + +#define BP_WIN_SIZE MUL_BASEPOINT_WIN_SIZE +#define BP_N_SLOTS NUMBER_OF_DIGITS(P384_BITSIZE+1,BP_WIN_SIZE) +#define BP_N_ENTRY (1<<(BP_WIN_SIZE-1)) + +__ALIGN64 static SINGLE_P384_POINT_AFFINE ifma_ec_nistp384_bp_precomp[][BP_N_ENTRY] = { +{/* digit=0 [{1,2,3,..,}]*([2^0]*G) */ + {{0x000607664d3aadc2,0x000fa3dd07565fc8,0x000e1e26a4ee117b,0x0003afc541b4d6e6,0x000459a30eff879c,0x0004ede2b6454868,0x000513812ff72361,0x00000000000299e1}, {0x000af93c2b78abc2,0x0006e23043dad1f8,0x000d385481a72d55,0x000e7562e83b050c,0x000968f4ffd98bad,0x00069a840c6c3521,0x0005e9dd80022639,0x000000000005a15c}}, + {{0x000a271bdb93b776,0x00066c8229e549ca,0x000a0046a4ddbf0b,0x000e6f0ff9d48a26,0x0005f0687f503504,0x000e4b506da82149,0x0000c39c90a4fd2d,0x0000000000042746}, {0x000777e3e34947f7,0x000cf42ea84624df,0x000c322ca0a5f414,0x000f18bdc588259c,0x00015172bad915e4,0x000b0e68409f1fe4,0x0000c2070d430900,0x00000000000123df}}, + {{0x0008420bd283fe68,0x0000405e4dbe5ef5,0x0007d2a868c2d376,0x00034e9a170ccf19,0x0002d51c6c3e6b20,0x0003aa4703a48d73,0x0003ace36f7e2d26,0x00000000000e7c1c}, {0x000a7b5b465465fc,0x000697e28482179f,0x00092befa3c13549,0x00063c04ef67446d,0x000ad2e1d0b41326,0x00002b33968012d5,0x0002aff6db68b151,0x0000000000098329}}, + {{0x0007a84fcfeee6dd,0x000c00aae84771bc,0x00004833a9bdf308,0x000153b0aecac470,0x0004736400ad2e4f,0x00085d979078358d,0x000228fb40f647d6,0x0000000000034179}, {0x00059b3d50946875,0x000f354f3e8e74aa,0x0007e02066cc9331,0x00061a34c542ad23,0x00030418c6938e3e,0x00020017d147162d,0x000319e607b9e338,0x00000000000303df}}, + {{0x0006ca2ee1bb26b1,0x00017bb595eb9873,0x000340e77dae425b,0x000b1b5293c703ca,0x0005aacc05e67f1e,0x000e8e4c660db2cf,0x000ffbc676b987e2,0x000000000001d178}, {0x0002304b4db1c9d6,0x0003c2b694ba022c,0x0000733804c0a50f,0x0001b3101c35b997,0x000f982c041180b6,0x000de236d4b237fa,0x0004a3e6c5944024,0x00000000000e209e}}, + {{0x0003f94fc482e189,0x000c37eb5c930b8d,0x000fa7363cfe5622,0x000930f580d57f38,0x00061bdf6015ec52,0x00002d33b2a33f66,0x000c404f0f6a962b,0x00000000000f0430}, {0x000a60b1c9962152,0x000203f62b16dde1,0x000d30e7f024d36f,0x000bffcb79e33b13,0x00061b546f058bd4,0x00021559a93c9e5f,0x000eba586d8ededf,0x00000000000af2a9}}, + {{0x000c82aa932c9f81,0x00032df13b9d0ea3,0x000012e0e11a7414,0x000dcf8211faa95e,0x0001659753ed731e,0x000b2df555f4215d,0x00025bf893db589d,0x000000000001c890}, {0x000c8f68d90a16b6,0x0002f0996b21f9df,0x000c5d608c816e33,0x000d76f50130844e,0x000401fff78065aa,0x0003b07060ff37c0,0x000b3ef57f238e69,0x00000000000af6c9}}, + {{0x0006da365db4f184,0x000fe23f60a057fe,0x000be85f6a0c5049,0x0002e7193d7e30ff,0x00064f3ddb371c5c,0x000b664231d9aebd,0x0009b11c7b5fe116,0x00000000000349cd}, {0x0008ec6d3c0c6dd7,0x0005e0d2cfe83aa5,0x000f7a290df3f1cc,0x00054bf7d8686e4b,0x0003a42dbba27017,0x0008ecf0ee992326,0x000f617008d943c4,0x000000000000d27e}}, +}, +{/* digit=1 [{1,2,3,..,}]*([2^4]*G) */ + {{0x000b56e7a10896aa,0x00082da6e8a7edb2,0x000a339205afd669,0x00065517917652b3,0x000a2887d5ff37cf,0x000bdc3fa317b63e,0x000aa137065f5313,0x00000000000435ab}, {0x000e15a5659db481,0x0008e9b21615f8a0,0x0000a5926b88aaa4,0x00071dc154d41105,0x000bc88ee1489148,0x0002d1967b333bdf,0x00051351c305c6a7,0x0000000000081ef2}}, + {{0x0007c0045dd2a4a0,0x00075ad852b872e3,0x000b4af19266ca1e,0x000b9fcc651b1dd6,0x000612e8dc871896,0x00031cfb0ba8953d,0x000d793a9865baa6,0x00000000000626b3}, {0x000b7328c510ad93,0x000901148bc71a36,0x0008838d56b7b5d9,0x0005f9e9448fd096,0x00000a2377b67731,0x0005b4ff04bcb06f,0x00099f73b42725b3,0x00000000000aabca}}, + {{0x00077d8466e4794b,0x000763ccf806a4c7,0x00041b06944ed785,0x0004e06ea52bef99,0x00053a7d2f3c4f50,0x0003a1bc940d01ff,0x00040062e5c5d3e7,0x00000000000ae7d7}, {0x0000b071271e42a1,0x000923d30625e38e,0x000c9ea33ece8520,0x000a10d04bab9856,0x00009da2a2ca5c3c,0x000c9462c2605ca0,0x00058348eab00eb5,0x000000000002624a}}, + {{0x000c57a24f5d5ab6,0x000eded4de4f83f2,0x0004d9f2c578d7f4,0x0003d30f8a0580de,0x000dca57b7bde04b,0x000e44d56a309199,0x000e5cfc8e87cf3b,0x00000000000d1b30}, {0x000dc1d0888b708b,0x0008cb213c69fa81,0x00085d35b9791d2c,0x000bbbf1090fede9,0x000c301fe259fe51,0x000cd3fe86d97cab,0x000a513ee127895e,0x000000000009404c}}, + {{0x0004d8911e8568cc,0x000c5194924b48e8,0x00026b2f852cc83a,0x0006428b12136094,0x000351fea1dc4906,0x00015ace6dd2ec6d,0x00024620fe8c27a7,0x00000000000a4463}, {0x0003c328530abb42,0x000b900c213bba9c,0x000bf43a5f2c2e1e,0x000903c6484641de,0x000a1378e68fbc7a,0x000cd8ae42413063,0x0006f9b960b5efee,0x000000000000614a}}, + {{0x0001dedb8bf3dccc,0x000a0dc529384912,0x000bc9fafda07c1d,0x000597e52ce08f71,0x000581998af2ee21,0x00041eb4226de4c6,0x000b96cb4aa43c97,0x0000000000039c18}, {0x000a9ce2b257fb6c,0x000a566e1d5da261,0x0002d61f72303077,0x000396f305ee4f10,0x0000c831254b8545,0x0005680b8f9d19ea,0x0004a4cee0842f5d,0x000000000005a443}}, + {{0x00007168b4a67147,0x000a12e206547853,0x0001c6a852120cb6,0x00009d5504c8129c,0x000c9710b70b2b56,0x000296a52fb25e37,0x000dce83f2fd2cd8,0x0000000000062f45}, {0x0000b128f82bb944,0x000a8a818b9bda93,0x000ed2d611039805,0x000e43a2ec76a180,0x0007caa846883e7e,0x000182141473e687,0x00004db9a19eb57c,0x0000000000045ed2}}, + {{0x0001e64ba6661cc4,0x000cee484fd9edbf,0x0005b5c2b4988114,0x000449e7c1c3984b,0x000118eb5195c0dd,0x0007a16d2f313389,0x00020a0336aab877,0x00000000000c2417}, {0x0001e21e239dcab2,0x00034678db970845,0x000627331787ffae,0x000a0e4a022c7a44,0x000434a02a6b5d85,0x000791ce3b01f1e0,0x000c5b2657eedda5,0x00000000000a277a}}, +}, +{/* digit=2 [{1,2,3,..,}]*([2^8]*G) */ + {{0x000fa21fa335ab82,0x000a49a7a5b41c7a,0x000300862e13765b,0x000438e3d9f0e627,0x0009e328c2e27539,0x000cbf891013c671,0x000d287f4a706ccc,0x00000000000735a2}, {0x000a7119424dd00b,0x0004246694eeffb4,0x000059afb703b483,0x000ed8b423d47e45,0x000bf44c91809d54,0x000e9b3848075a8c,0x000c75d4f5b184ab,0x0000000000041abd}}, + {{0x00093e732cc6e06c,0x000c65e2cb07faab,0x0006c10c7767a2e6,0x000c53fd4de1f262,0x000c838f7169a296,0x0008a6ce7d408060,0x00067168e19d7b2e,0x0000000000094b58}, {0x000136755dca2adc,0x0000293d02a07640,0x000ed9dfab92ca5c,0x00069f51aa3bc4ef,0x0000dd09b1426aa0,0x0002e59450e44fbf,0x0006ace264f34383,0x000000000001fc16}}, + {{0x0001b41eba2511e6,0x0003e9ee4f521f6d,0x0005af7a840c9880,0x000396db7edb07d1,0x000c2e8290630d5f,0x0003495da09b3457,0x0009b8f1d28188f8,0x00000000000cce55}, {0x000f6b035c499b66,0x0005617cbaf577ca,0x0007eb3582ad9848,0x00008995145b8fd9,0x00081a33b1a72982,0x0005149e992cb5da,0x0004c0ca49fe334c,0x000000000001772b}}, + {{0x000b80038e0f9767,0x0006756ad758212d,0x00066af19dfc2941,0x000c6ffe2c8b0369,0x0007fcd7336b85f2,0x000a46acd55c6d35,0x000ac7b1ecc56d22,0x0000000000036277}, {0x000330b02f145871,0x000c1a4ed11e8d27,0x000297add7ae640e,0x000ba45266158ab0,0x000d89e0dff05fda,0x0006b02d06f0b27f,0x0006e132ef7ae2eb,0x00000000000cc1b4}}, + {{0x0008162061985fbc,0x00005c112733b3ba,0x00062ae17de90bd5,0x0008e01810097859,0x0002bfe16c4fbb7d,0x000d9f8107640a3e,0x0005d74e34813ec1,0x000000000008d260}, {0x00078cdfc58ed763,0x000f72a544cd81e6,0x000e167259300b75,0x00057bacec18a7f0,0x000511b882d69e61,0x000f86563a555fc9,0x00096e4305a4dd04,0x000000000001d0fd}}, + {{0x00008da90a96090a,0x00032f04145e8229,0x000a916fb6ff9132,0x000ba4e12aa299fb,0x000991b3b5179ffc,0x00081c747cc5ec24,0x0003eb9edcd4616f,0x0000000000077a88}, {0x000a4909883002a4,0x0008b9b0bab581a7,0x000e659d0317cb87,0x000d81e438a9d43f,0x000e25ca8b3cfe8c,0x000bc720cf40e2b5,0x0006a34254030067,0x000000000006b244}}, + {{0x00099b58a43c6d42,0x0005180e1cd16205,0x000e96620312fe6d,0x00019d509ddce071,0x000e70c4b03267a0,0x0003ba57e52573e3,0x0004f716d253e14d,0x0000000000016250}, {0x0003e944594baca0,0x00013a237bbf8f9b,0x000a642b05f4171f,0x000531a1f384daed,0x0003981251654b13,0x0002dccc139067f3,0x0007b5e98fb14167,0x00000000000a75e2}}, + {{0x0009f542630002ea,0x00044e65245ce93e,0x00012350ea59da7e,0x000c121bad2c8070,0x0002060fcf245677,0x00078cccac52dec3,0x0006fb78d070675d,0x000000000001bc8b}, {0x000ac9684403d046,0x000b5c5cb86bea72,0x00053d522dc955a1,0x000cdf2c92a70d83,0x0001f53cd2a1fbb7,0x0004f11395a9ff1f,0x0009f1fdbe6b7a98,0x00000000000a470a}}, +}, +{/* digit=3 [{1,2,3,..,}]*([2^12]*G) */ + {{0x0006766eb19e084c,0x00028eb06571b5db,0x000430cbda13e4c9,0x000966726eed225a,0x00046100e387a185,0x0006298d18d9e56e,0x000ad0470506b9dc,0x00000000000f3350}, {0x0009595e79f27f3a,0x0006683eb62a798b,0x000ae3d2069c14b3,0x000e880e1bd4a82e,0x000fcaf3b3fcb089,0x000ffd65cd4d1e70,0x000ebbd0b1ec6395,0x000000000009b184}}, + {{0x000d72326a677bdc,0x0001bd4277730e1a,0x0004e8c2adc8ef98,0x00046b099f1867d4,0x0002602dd4cc6b07,0x0000811201ec73d7,0x000f2d27fae51538,0x000000000002f8b2}, {0x000e28e4b1971c05,0x0001bb924af64246,0x0005d0fd898e9387,0x000e9ae068565acc,0x0005a9a4f1464e88,0x00093f7348a3dbd2,0x0003bcdb4a3e483f,0x000000000008f1d3}}, + {{0x000da9f02128c46a,0x00049d1de964bafd,0x0007f571d595c1ce,0x00055af0de9eb074,0x0005a60289bfbc4b,0x000392c619f11b99,0x0004fc3e59000c52,0x000000000005ccff}, {0x000c017748720be7,0x00064b28b306ba1a,0x0007e101bd3e41b0,0x000542ce3f824faa,0x00022f52b71c59b0,0x000c6d26370f097b,0x000e5b4483b72604,0x0000000000034d93}}, + {{0x000b2e0b9f0415b8,0x000c7721bb8359b7,0x000a5f46c16031df,0x000a789348538714,0x0007af598c4f9cc9,0x0006f27c878b604a,0x000ba5d370375e47,0x000000000000b15c}, {0x00021b9613cec089,0x000662bcfd9a4c03,0x000e3ea0c45f94ee,0x0006464a211b19f3,0x00019990b504b05a,0x0004951d3ce059d4,0x0007b0011c5f87d9,0x0000000000000d9c}}, + {{0x0002533a1c8fbed1,0x000ce64e84c28804,0x000338cbe4f167c9,0x000d9ed9fbf23cc9,0x000f5b93118bb77e,0x0006255cf155fd45,0x0008941e9f6d7d9c,0x00000000000c4f64}, {0x00008205c725e2b1,0x000154bc3a502a87,0x00030c3fbf39b6ac,0x0005548d3c862428,0x0004030f713cc7df,0x000785cbf9dbfc08,0x000637f3623326ad,0x00000000000dd3ad}}, + {{0x00053a3eba12bfa4,0x000ced8b8b37a274,0x000ff25533a7ef36,0x000684bd17d58a93,0x0002032fda408ac7,0x0004b49645f9557e,0x0001097fe128e6ed,0x00000000000ed02e}, {0x000f765f56c35dab,0x0008c0052d88eb68,0x000256931b154329,0x00010798446a4f6a,0x0007c99ad35fbf46,0x0000906073bc4391,0x0008aada18234dda,0x000000000005164f}}, + {{0x000f715095892612,0x0004e02c16c7865e,0x0009e82bb73222b5,0x000bbc0795486af0,0x00070427332d3abe,0x0005d3cabad858cc,0x00019c9a1d4b6aa3,0x00000000000e208c}, {0x000d5b54420318c5,0x0000afcc14276eea,0x0008e6c4a86b5358,0x0007cb4e7706b5bf,0x000e479e2c750027,0x0007ad688c01ed42,0x000626ff1759604d,0x00000000000c045f}}, + {{0x000e0b227c3a04b7,0x00029f365419f1ee,0x00001db5dec2705b,0x000c165c41880aa9,0x0007f9712fbd8a91,0x000c556783eb27a9,0x0009cfa6587aec76,0x0000000000002cd7}, {0x000e78bc85d2b5fc,0x000fefc878f9c549,0x000d411713959cf3,0x00084d8caf6df5e8,0x0002aabcde7509a1,0x000de597ad32bf23,0x000858f601d0de03,0x00000000000c5da2}}, +}, +{/* digit=4 [{1,2,3,..,}]*([2^16]*G) */ + {{0x000f068a28ea9470,0x0008bb6029961859,0x0007d86ade910602,0x000693b4df3e5b1b,0x0008a0c3e35782db,0x000b2f577b513148,0x000cc3bfb01ff3cc,0x0000000000027a72}, {0x0000fdf0e7cd346e,0x000f626170927fbd,0x000aa1bbda6cc535,0x0006a634c872d772,0x0000b14d9c9f0bec,0x0006c7778a0a7cc9,0x000a4c8a32d2c44f,0x000000000003b889}}, + {{0x000f462aea173d82,0x000a4860ef793767,0x000a7a5856850902,0x00083662ee7f523d,0x000f54122af0322c,0x000bb2d8058ccd95,0x0005777454880c2b,0x0000000000086d8d}, {0x000038487c4c8fb9,0x00042d5a3057c6dd,0x000955643c37ff31,0x000c99ec3c512f97,0x0006556d891e26aa,0x0009f6112c3eac03,0x0007e9866c3aa7bf,0x00000000000c144f}}, + {{0x00000b161de71555,0x000aead0d24c7983,0x0006a55d94bbb854,0x00034ff7655aa29d,0x00057a5e217ea551,0x00021b295a3d1038,0x00036dfbb9eeb53a,0x00000000000c84a1}, {0x000aac3258d9db81,0x000087579398db29,0x000fa470f6fa27aa,0x0002e1e464522581,0x0005479d8f2c99b3,0x0000b80ef99d5495,0x00050bc2a8a6a193,0x00000000000656c8}}, + {{0x000f81f2532800c4,0x00045171898aa3c4,0x000ea2712f9cc33b,0x000835ffdb2c1bad,0x0001591f5aafbc0e,0x000272c6a4ee3028,0x00068afd71de3bcc,0x000000000006e93f}, {0x000145dbf5847f9b,0x000bc35ee08038de,0x0001b04c30c2d081,0x0007ff5957b2ff76,0x000e5ec029c049f3,0x000324c12315d8e7,0x00057833230602ef,0x00000000000966b2}}, + {{0x0003e43e11c6c113,0x0002e86283e21e81,0x0009c3e50b313030,0x0009b1bb9784a9a5,0x000ea0c0acb57d02,0x0007c682b7c7798b,0x0002b041241c716d,0x000000000001d33c}, {0x00079a15b39e351d,0x000dc5d469ca181c,0x000ce825406e72f2,0x0004cc2a13cf4ce5,0x00069e3ce2793d05,0x0004beafc13bd216,0x00087e01bc70e68a,0x000000000008aba0}}, + {{0x0009cf16a3c4418b,0x0005884aa863e012,0x00089c47322b55de,0x0003206b5c399b2c,0x00073cc109bd553f,0x000384088775b921,0x0003cf25c01263fe,0x00000000000d5f74}, {0x00057c2efedc75c2,0x000933d69705ce0b,0x000359bbe99d9a50,0x000ab1a2626cebe5,0x000285b1afe80198,0x0007e6efdaf8320f,0x000bb6b9c0968ce1,0x0000000000090215}}, + {{0x00066543cf4fd691,0x0000d3ee52d8e909,0x00094816ee49cd7e,0x00095c61881a757d,0x0009c13e370735ce,0x000d2d3f60a8cf9d,0x000c0de71258d548,0x000000000000bbe7}, {0x000476d4cb00031d,0x000bfbd6496e1309,0x000c1c69b8768cb6,0x000501358cfdfb53,0x000b59275b4acbe8,0x000f722ba655c902,0x000aad7e0ff05b20,0x0000000000042b17}}, + {{0x000b76411fcb09e7,0x00066da643272cd3,0x0006802b1cc8eac2,0x0005a7c35b43943a,0x00084606bbf22386,0x00059fb9a6ac0158,0x0001a59660ab7215,0x000000000003ce2f}, {0x00083d9ad8b4f172,0x0006e62af29aaa08,0x00060fa06813a370,0x00029b744c110388,0x0001d36c2571ee9f,0x000b552b7b2a19cf,0x0003d4b87d88e265,0x00000000000beb25}}, +}, +{/* digit=5 [{1,2,3,..,}]*([2^20]*G) */ + {{0x0002b4a5d3a4b643,0x0000231bdb4829ee,0x0006d713dc69c388,0x00042469b6fc64eb,0x000a15786b8c18c0,0x00063da4a0dcc15f,0x0000fb81acdb068e,0x00000000000dada7}, {0x0008c1ca45ab6321,0x0009146361afe98f,0x0001d88fcfcf648c,0x000b61b615694e72,0x0001872951d2f389,0x0003badc77036def,0x0008d340fd3bdad9,0x00000000000380a6}}, + {{0x000d66b2c4fdf4c0,0x0007ac5cf8997090,0x000a08d2a2626f49,0x000681e307b254af,0x000775cd94d78139,0x000684954cc87fb5,0x00099190e7027478,0x0000000000095ceb}, {0x0004649153ee63bb,0x0006891bbc0ab337,0x0001f845e221f84c,0x0003d704b93d45fb,0x0004f541da1f1cb8,0x0007ffd10e229902,0x000b9a3eef7ce14b,0x00000000000b3fc1}}, + {{0x00067cf1182bf349,0x0007f23c1f744697,0x000288faa5d1b184,0x0002c9d5afd1bfd3,0x0004f89d76fea4f8,0x000702f80a3d1e9a,0x00089d964f705075,0x00000000000f56c0}, {0x000478c2e0092e1f,0x00012e205fa8ede0,0x0002998cd8ea4a27,0x0001004356d43961,0x0001fdbbdfde331a,0x000a00e7158b7890,0x000a30a52a15979b,0x0000000000055497}}, + {{0x000d05d101bb2988,0x00097e5004e8001a,0x000e96c63ff7fdd5,0x00050d1bd869f77c,0x000de7ebea2c405f,0x0007620baecffa54,0x000ff43354dc22d8,0x00000000000b1c01}, {0x000d9286dd26e577,0x000c2d9370076043,0x00025722a20b989f,0x00076273291e5c62,0x0007f0a7ca55c0de,0x000592a305cfebd8,0x000ce4de1162809e,0x00000000000a78eb}}, + {{0x000153343d6d3c05,0x000a15562a856338,0x00041dfd1ca25266,0x000317409c75b8cc,0x000124923f80c19f,0x0005b291e21c7cc3,0x000b05e63fe47f8f,0x000000000000dc08}, {0x000a81ce4831b7d3,0x0001558ae455ea5d,0x0000f05c04530b31,0x000c97f3ee3989cc,0x0005f38759ae42c7,0x000f46899c1b03af,0x000c7d1a673c75bc,0x000000000008d508}}, + {{0x000fc69642676f12,0x0008d1e9b23b9bca,0x000626ac6d6d75ba,0x00000fe59b7721d0,0x000c9e2f4cebd4cc,0x0000af70ed5c36f9,0x000799deca06bac9,0x00000000000416ee}, {0x000affe525098c8a,0x000df0d7afe1b4a6,0x000083fa6f5ecd29,0x0003d6ee6eaed183,0x0002496087e011e4,0x000a3a66e5baf860,0x000677f833634fb1,0x0000000000079398}}, + {{0x000c39f67e66a95e,0x0005b76512d1b46e,0x0009e5603da596ca,0x0003aa8025a5f25a,0x00095cbd2a40d1c7,0x0008d61c62aba192,0x0000f3b53cadc3c8,0x0000000000009829}, {0x000894e8b1d3bb8b,0x0003a72800ecafd7,0x0003c62dea4f99fb,0x00092a7282ba9d23,0x0002bd5f1bb214bf,0x0007c6c969062967,0x000601362f68eba9,0x000000000007ea9d}}, + {{0x0004d04ff62d3721,0x00071e141e762de4,0x000fa0d592d3e0eb,0x000cde496131447a,0x0005cb6c2ef746e6,0x0002f9fd80458a5d,0x000457b774763453,0x0000000000016544}, {0x000adb5cae252cf8,0x0005abfacb4de24e,0x000e9db72a61c26c,0x0003220f22d92e51,0x0006557232589b54,0x000ddb8bcaa4590f,0x0007bdc7b6730e01,0x0000000000069e1e}}, +}, +{/* digit=6 [{1,2,3,..,}]*([2^24]*G) */ + {{0x000e98d9a0583230,0x000f77d27d71f312,0x000823b17edc1a45,0x000afeb6075b2d00,0x0006d93a06f7418c,0x000d001b9c0d691e,0x0007b9c16a95259d,0x00000000000026f1}, {0x000b72219cfa1dea,0x000984c1041afdb4,0x00056e257be49c48,0x000efd62c1758e9c,0x0007e6c3229a8d08,0x0002d89249cc6f20,0x000fe7ec69e90208,0x00000000000f331d}}, + {{0x00061c722c01a99c,0x000518ec4335f7fe,0x000df3425d366c49,0x0006de001141ab62,0x000c2eca98a13ff9,0x000fdf648b21acc2,0x000e8b6154849010,0x00000000000d1403}, {0x00097be7041b8df0,0x0008ff1a35f306b8,0x000b8cedd3e80c1c,0x0006c077fc9a752f,0x000e420d48a089c2,0x000fe2e738d2535f,0x000f3980ec5ddd52,0x0000000000071704}}, + {{0x000259172095dfcf,0x000a020aa15d95a2,0x000d85bd292d185e,0x00005caef579e8f8,0x000b325981bfe2f2,0x000438be8ad27e38,0x0000d9087b8284c3,0x0000000000042236}, {0x00091bc7ac277af8,0x000bb87cdf5accaa,0x000de0f7da8c4a28,0x00040c1891046669,0x000c9c1578e8a712,0x00050ffa2eb5a175,0x000a28bd66910ad1,0x0000000000011459}}, + {{0x000f920077501dce,0x00091498808ed4f0,0x000dc6c59ac5d089,0x00025f176c6b964a,0x000bac474261796e,0x000a460c11aced64,0x000e48a62470fc29,0x000000000005e751}, {0x000842d2c145f36f,0x0007acc00053aac5,0x000ca1b81e5b854d,0x000cc2e3f9ca178e,0x000a0b80d1b0ddac,0x000642225ad33f34,0x00061b6a76df9364,0x00000000000778e7}}, + {{0x0009eb69fa5f1bc6,0x0008ed30302342c1,0x000e3ef7d69039f0,0x0009b575c4630f26,0x00008d098d745364,0x0007f5cbc60197fc,0x000efc9c295d5464,0x00000000000c0813}, {0x0001c5999be2ce7e,0x000e7f6e08007370,0x000b6019bc473a63,0x000e08e11d9b388e,0x000a5db61c657af3,0x000b4dc4d073ec38,0x00082a9b480cb89d,0x00000000000372fa}}, + {{0x0003179049382c1e,0x00069dffb03ae77d,0x0002e9528cdd6bd1,0x0002521b19fe0db8,0x000f3d5c7fee4c26,0x000e68e1e0ec1e54,0x0008a62856510b05,0x00000000000dc80b}, {0x000ac17897e6fc5d,0x0000680a509308c8,0x0000b4dde3197e47,0x00012ee28235c538,0x0008301f9653ca61,0x0001fcdf8d28a0eb,0x0005f322b11e26e1,0x00000000000e4d73}}, + {{0x0002a6a91ebadb85,0x0006346a2a08bc3b,0x00020e574ba891b2,0x00056a2b3df9fbc8,0x000eb121d51228c5,0x0003bc86c81d3161,0x0000d14e27ce0b12,0x00000000000bfb24}, {0x00072db1b86f039c,0x0005986edb71958e,0x00011b5a99c9a865,0x000c6d8067f5870b,0x00033fe8e5322f6b,0x000a7997ff558b88,0x000b9be9433e2321,0x0000000000087e53}}, + {{0x00084d7bdfada95f,0x0008b9c66a32b0d1,0x0004ace9a2f6c763,0x000fab721e716f0c,0x000b96ed74e68c6c,0x000110c8332c8fcc,0x000efe475890dd0b,0x000000000005cb4e}, {0x0003947e05207b63,0x000d29d7b89a68e0,0x00001e2e33262bf2,0x000b55bca7a7d527,0x000655d04585c3f0,0x00057acc5a6e56a0,0x00039e818e221c42,0x00000000000fcb8d}}, +}, +{/* digit=7 [{1,2,3,..,}]*([2^28]*G) */ + {{0x0007398749666d45,0x0009c0a74da828f4,0x00001ff782080bc1,0x00026bb57c2f5ad1,0x0002845d45e4896c,0x0009a7d36981e2f7,0x000e8fca152b877e,0x000000000007b582}, {0x000b1649b1810a70,0x000c3ea3b9bd9987,0x000d4cd2bb2df2fb,0x000fc5d5748a6550,0x0007622665eed346,0x000f16e277ac2f21,0x000dc8bb5efe7fb6,0x00000000000644c9}}, + {{0x000db9a336c7d7d8,0x0003598d0898164f,0x00065860354f4784,0x00018287cfc13dbd,0x000c8655a658651b,0x000c91b712d606e4,0x000090ba64d3c563,0x00000000000b82a5}, {0x000726397fcaaf5f,0x0006c2d1dff024ae,0x00092238373e43a8,0x000ee6b0ea1fe022,0x000cd5c1273c1ac2,0x000e603e7c100b60,0x000dfb4496084cea,0x0000000000077c2f}}, + {{0x00064d07c56a20fe,0x00000a93fec079c4,0x000155e36a436889,0x00045a5cce5662fd,0x000f83a9a4a9c00b,0x000bbeb632e8a0a7,0x00080f6e0cdebbc0,0x0000000000063ccd}, {0x0008f36be2f62c1e,0x00061fc10fa07d22,0x000b3e653f03a3be,0x0009cc66bf53af92,0x0000f10bb6c9fda6,0x0007625e1474b744,0x0003cbcda9db3b1e,0x000000000001dc7f}}, + {{0x0006fec7b896d97b,0x0007de8e32259b22,0x00051ccb0af3cd54,0x000a4219f42edba4,0x000d0d411d4df147,0x000014bb46d4bc00,0x00066fa1a13a2770,0x00000000000fa101}, {0x000b6039e0c4cc34,0x000d8b2a1dfa62b6,0x000ae98992614f2f,0x000a3a2f88c7359e,0x0008347726a08409,0x000507bb9071f383,0x000167d18a551c27,0x00000000000b359b}}, + {{0x000fae4c55d4b2c3,0x0000aeaaaf45fd46,0x000aa7e37459675f,0x0009b673fe123f1e,0x000dd8fd0129989b,0x0004982a4e2ca56d,0x000ec777d6d0cd62,0x0000000000071e1f}, {0x0001c6bdd9bc3a7f,0x000043e9a049f5c5,0x0006deb929a38a20,0x00008e24fed8f86f,0x000ce199e8dbac2b,0x0009cc964a1d1357,0x00063b7cf06ec8e9,0x00000000000d85ec}}, + {{0x000ba68a3fc0bcb2,0x0004e7d111c66c1b,0x000d9aa66fbcd347,0x000730c6db857e9e,0x0009f4b46d124cd8,0x0008472dc3c9c03e,0x0001bbd42f0242a7,0x0000000000026084}, {0x000aac1b65a94c0d,0x000ea6332b11a21d,0x000acbe9385d6783,0x00028eee7e8944ac,0x00005ab28372402f,0x0005e1ff33d1bab5,0x0007296944e82cad,0x00000000000e8c75}}, + {{0x00058e168fe9a81d,0x00043a151dcbb9f9,0x0002eed94828803a,0x000fc00604d46e1d,0x000572f3e28c947a,0x000b1cd1dc3c9d57,0x000a45ce4c1cbd14,0x00000000000f80de}, {0x000d8f65d998669e,0x0003c50920f39bce,0x000b6be78ee5193f,0x0008ba13f798e332,0x0006c5edde471997,0x000714a1e1294aaa,0x00003c280002c2be,0x00000000000f2126}}, + {{0x000493dde1b54616,0x0002ea44f6ef79f3,0x000c2b67fffeca1c,0x0004ed80eaf66728,0x0008181514a2cb0e,0x0002927ea2bf485f,0x00064574670e180a,0x0000000000012c14}, {0x000339b9a314b3a8,0x000724068c073875,0x0004212e0016a517,0x000651d698b28177,0x00096da14fa8391b,0x000a578b1f310d16,0x000ad7a089be6bd8,0x0000000000044389}}, +}, +{/* digit=8 [{1,2,3,..,}]*([2^32]*G) */ + {{0x00024e7304503422,0x000f0ba86aec16bc,0x0007f0cf87c57f69,0x000ff0789df2f808,0x000d97a773d58978,0x0003f35f685750cf,0x0008c9806bb730fc,0x00000000000fed86}, {0x0006b0ff06192aaf,0x000eadc0fcde080e,0x00055bc2901e7a1e,0x0007d028d3ad6cd9,0x000997293550fefb,0x0005cfbba5c652b5,0x000d2232e12942ed,0x0000000000098800}}, + {{0x000418b23a7be4e3,0x000cb162cdf33f48,0x000c8d04be100c6b,0x000d114454eb977c,0x0008dea38a674478,0x00035728a8ce403d,0x000504d459633b74,0x00000000000a63b0}, {0x00010a5f9fdcafdf,0x000c40c74066a938,0x000e6c61b766c71e,0x000b588a99690ede,0x000c3ad775623398,0x000bb60ee4949517,0x000becf9824f09cb,0x0000000000085660}}, + {{0x00083bb80ede991f,0x000a02daddb11952,0x000f09f6c4b181d6,0x000e82721a6aa89b,0x0007467506deb73c,0x0008d8daa1091958,0x000dfd0927724c42,0x000000000007c17a}, {0x0009a9bb30e43182,0x0003a8518dab18e9,0x000594c3465b3913,0x000c37f89e7a3983,0x0008e273f6f35943,0x000143d228e63f5e,0x00028ec6d0352b83,0x00000000000ebd16}}, + {{0x000731dadf48f7e1,0x000a14074ee26b83,0x00088243bc9a29c8,0x000d53972cecb4c8,0x00079a7dd9c4aa01,0x000c787cf4b0cf12,0x00053f3e3e3f165b,0x00000000000942de}, {0x000bfa5d149fa2b1,0x000010cc6971941d,0x0007bdd5c6a1acca,0x000c1e292314a097,0x000614a1adcb9fed,0x00062b86a7547d22,0x000b7d405561a486,0x00000000000f5480}}, + {{0x000be69f3af05d97,0x00082c4e59f2ff48,0x000865d4a01ec6bd,0x0000d824464bbbbd,0x000016f9540dfbc8,0x000595b5d3bacfa2,0x00080d1954efb613,0x000000000007a5cd}, {0x0009c6dbd9d7e6b6,0x000ac926a54cf784,0x000f3366624e7b07,0x000167ccb5c8d4c7,0x000ff9a21ce20677,0x0002df8cdc994d22,0x0009083a25ff6b42,0x00000000000f68e9}}, + {{0x000a093607905265,0x000ede544b89ab7b,0x00017731e314dedd,0x0000da69a73104b6,0x00067274b105a6a9,0x000c61bb65c26021,0x0005068f9705cf60,0x000000000003c4d1}, {0x000ed5677f9dc5c6,0x00020ab5a27accb8,0x000e0bb2ed27cc25,0x00036d15a36afae3,0x00095c455916e68f,0x000b5d1fa79004b1,0x00048916ffe6249b,0x0000000000049338}}, + {{0x000a7603914a9a59,0x0000b941be86e102,0x0001a6f35b551149,0x00095b469d75ec8a,0x0003db0d4374658f,0x000fc77053fa79d5,0x00012885da635c6a,0x000000000008e7e8}, {0x000f4285c3e56baf,0x00002558cfa8eed1,0x0000effdf411ca89,0x00098b96a32e8849,0x000e3c45ce1a104f,0x00085de0268237ef,0x000de35c820dc22d,0x000000000007459e}}, + {{0x000a78ec870a31bb,0x00036923d0b44369,0x0005db7ea085ecd1,0x000be009cf5b87e5,0x000d1d61103d1656,0x00065239313a37d0,0x000ed81d705880fb,0x00000000000ed712}, {0x000ff1a5976c303e,0x0006f15ad02e6160,0x00077114865ad858,0x000376cba2b3ffe8,0x000f9745443c56aa,0x000903660c3be2b2,0x00092d47c8a870eb,0x000000000006c6c1}}, +}, +{/* digit=9 [{1,2,3,..,}]*([2^36]*G) */ + {{0x000b550138d02bd3,0x00038148bd39cbc2,0x000f6b4c6038c07a,0x000fbe2ce5484157,0x000c87fdde9ff397,0x000e9c179441e5c2,0x000c716366b49ffe,0x000000000002938d}, {0x0008a64bcbf3adf9,0x000d026d450f9f8a,0x00015da756f71781,0x000bf4d298fd8771,0x0007544768b65f68,0x000491267e86df04,0x00071a40b69a32f8,0x00000000000f917c}}, + {{0x0007f3a58f523dab,0x000a7a66c70349a5,0x000f8ae356d6f09f,0x0003e96b5ab54115,0x000c7c57d123dee3,0x000d6ad37d068929,0x0001780839a208f1,0x00000000000123f8}, {0x000f3c2b5c9dfc15,0x000f4b3e5e52449d,0x00055ba373af8955,0x0000ab7389f2dd3e,0x0005890bba6f513a,0x00066bf093197a14,0x00072261add75b6f,0x000000000004eef1}}, + {{0x000fbdf154b15bac,0x00063810b6ab3193,0x0006da8c3809a3ef,0x00038dd898977511,0x000e7a336c9a3cf8,0x0006f89c03e391e4,0x000e227014833717,0x00000000000bc4f1}, {0x000e7d4400e0ab41,0x0006b32b104f92b2,0x000d1a7a3b67e3fa,0x0000437bf178ac12,0x0005c99370d5b831,0x0002b93a8722299f,0x0007190a493cf033,0x00000000000a420f}}, + {{0x00046acf9a0ee15c,0x0006a21feb7fb87b,0x000579369777bef5,0x000557624b04e704,0x0006342cb0ad03a6,0x0004f64262531f18,0x0003ea088c4d54a2,0x0000000000006a87}, {0x000f1f11e0fca837,0x000d5dbe0253ef23,0x000bcfdbd73eb554,0x000368173e65902b,0x0002ccbfa504eaaf,0x0000e163e71f1fab,0x000f3bb7b845224b,0x000000000003c779}}, + {{0x000cf36036019ecd,0x00029cd7b3c4286b,0x0005e1ca08cbdeb9,0x000bcbd24ef5c386,0x000ce579c309eb66,0x000f6c9007edcc21,0x000c2c7b19d49116,0x00000000000b6317}, {0x000aad793c4e52a3,0x000e7554ba553558,0x0002315e3b514170,0x000e33bffda4032e,0x00082306675c3d1f,0x0000c91e75dfec47,0x000879be59305e00,0x0000000000025a6b}}, + {{0x0002c6fd041a12fc,0x0006aa35802f5d21,0x0000c3d459456256,0x000991d472b9d211,0x0006a2f8e875261a,0x0009b6d63d81a1ed,0x000758942f213a69,0x000000000000ae57}, {0x00067bfe08ea2ebd,0x0007061191c82b48,0x0000611a48f73652,0x0003e86525112224,0x000d30dabb91abe2,0x000d2742466dd967,0x00005077650c597e,0x00000000000ab25a}}, + {{0x000abb01ee5e0194,0x000bca624ab366b4,0x0009dc413b0af513,0x0009c4273aa694c3,0x0009779288abe822,0x000575e0e0cc3102,0x00003ef8eff30f57,0x000000000007d528}, {0x00093a51fb5fbbe1,0x0002f32d87e548f1,0x0004001c13dfb44c,0x000b8dd16c6e6274,0x000c2c140452aa2c,0x0003031b1add098a,0x000543d25f285d2d,0x0000000000075b59}}, + {{0x00032a5061a42b94,0x000dc520b0bb8a42,0x000466f1305a432b,0x000c73a73c239760,0x0009783aabba85c1,0x0004631556e4dec4,0x00017b69f0c69bb0,0x000000000009c97b}, {0x000fbbef3e8375b4,0x000b155af24a9074,0x000991d9ad3481f7,0x000283d708671c48,0x00035fd9001a4034,0x0002eaf3b200ddab,0x0006c4e45f28e434,0x000000000001ba93}}, +}, +{/* digit=10 [{1,2,3,..,}]*([2^40]*G) */ + {{0x000e8393cd68c757,0x000b2b083ba6a1e9,0x0004638d474c7417,0x0007a21fc82dc041,0x000a9d3679d89536,0x0009724c0227be26,0x000c0fc70f6d6c7e,0x00000000000f9ebe}, {0x00075bdec21bc5d4,0x000b029dde03dcdd,0x000a669d8fc534ff,0x00090c90f602f4cb,0x000849722bc4daf5,0x0009b22b617c5288,0x000b90a8df99f008,0x00000000000e59b9}}, + {{0x00015e6442d15d01,0x000dc6f5775290ef,0x000cdd79298e58a8,0x000842778b96c6d8,0x00022f59350519a1,0x0007209d6a674f99,0x000fff5abeeec46b,0x0000000000047cf5}, {0x0009d3497d146805,0x000ede24509b7378,0x000ed2fba1e0b34e,0x000af595761e8e3f,0x0008d420a2887f7d,0x0000ff696ed5cfcb,0x000c8f365b29eb7a,0x0000000000099a1a}}, + {{0x000785db50fa1164,0x000694936c6a0393,0x0005ce545ed4b2d3,0x000e8b45714f2c6a,0x00023f5ac25a03ae,0x000b33794139bd69,0x000ba96a2e42bab5,0x000000000003ff7b}, {0x00034248c56f7e51,0x00088b61d8643327,0x0008d647e582cbe4,0x0000e1472eb77fae,0x00013b99c6356211,0x00074c9f23d5b96f,0x000250956ecf64f6,0x00000000000ef2ba}}, + {{0x000a8baf84131eb9,0x000019ee1ec3a29b,0x000d9f684960ce84,0x000737588102ac15,0x0009c08527f432b9,0x000e3dfbedd296cf,0x000c4fb74f8145fa,0x000000000006cd7c}, {0x000debad8e4205ae,0x00062a0f2fe7a043,0x00094ce3fc7d23aa,0x000f2d40eeb90a7f,0x0000be4de6846e7d,0x000dd06bce2f46e2,0x0009f6cd28feba3f,0x00000000000e6d6d}}, + {{0x0002283f4c1e03dc,0x000bc246ffcb6b34,0x000a382150ba305c,0x0003ae2250e66766,0x0000924ce5fab4b4,0x000d8c77695c1b5f,0x0009d02555795beb,0x00000000000acd9d}, {0x000cf0d26acc1b8f,0x00088e1d74aa6321,0x00035822f91490d5,0x000df2795af56df1,0x000fb331b6f4df74,0x00059e13724b10c5,0x0007f2b0a6df9a65,0x00000000000c0663}}, + {{0x000c55f77d493f59,0x000089ad73168775,0x000791ccc3015317,0x0006c2d30b3a5f4a,0x0007c89723d59e94,0x00031f6077bc4ced,0x00034179f514a1bd,0x000000000003a274}, {0x0007950f4645c0c2,0x000e07eb010278e1,0x000a3d29cb5ab91d,0x000760f35be21cba,0x000b7c793331718d,0x00030d29eba58160,0x00003afbc0ce1f8d,0x00000000000c6f4b}}, + {{0x000986e6462b5c9b,0x0000cbd8c0867ee8,0x000db80962770b4f,0x00012de024593896,0x000fdef840b687ed,0x0000b56e13f7d98f,0x000e8771eee0cb5f,0x00000000000d8d9b}, {0x000f5c1c38b9eff3,0x000c1e6b50b5a5f4,0x000fada267894657,0x0001bd17cb1f9925,0x000d4ff11827418b,0x000042c63607818e,0x000ae3e630d93a9b,0x000000000008c779}}, + {{0x0000de60ecec558f,0x000bb35d474260aa,0x0007deb342712d19,0x00015e22e91bf5f3,0x000cc08b6b1abd6a,0x000b97de8e366a84,0x000f29759c122f55,0x0000000000008a03}, {0x0005b54173576b1f,0x00000dcc9fca2774,0x00073c06ae128d8b,0x00039029b59cd052,0x00006f5e5bd4deae,0x000099f4df532ede,0x00005284fbeeb936,0x00000000000088d2}}, +}, +{/* digit=11 [{1,2,3,..,}]*([2^44]*G) */ + {{0x000b3d633d0721cb,0x000732ba8c78fe5a,0x00016e2c1c57f816,0x000f36a2fc2451f3,0x0008bb91e1e36842,0x000ead762fc5c955,0x000556035d1dfcc3,0x00000000000031e5}, {0x000b4359fe8646d5,0x000383af0cc803c6,0x00070f15b8bb97ea,0x000de0a6ade1d137,0x000d93b2dcb580c3,0x000a2214de8c3a5b,0x00048de3adbd7c90,0x0000000000011929}}, + {{0x000e8783f9b6f97e,0x0009b65026296c0d,0x00086ba77888a60e,0x00063a460c8bbf8b,0x00078b2a71206237,0x0005497e7fa8f5ad,0x000618fd744bdf08,0x000000000002ba35}, {0x0004df87b45c7eff,0x000870cebfe9d444,0x000c034f12ddb3df,0x00017a3fcf19627f,0x0007c2f112616558,0x000f2c85030ab44f,0x0000c3bb001c9ddd,0x0000000000007326}}, + {{0x000e365e9f55b0a4,0x000aeb08fb116bd9,0x0002cf623c1f798f,0x000fc7b6f9549671,0x000f76bd243c73ae,0x00009d5a8c0fb886,0x000049871eacc5ce,0x00000000000e773f}, {0x0001eb3732cb8726,0x000aa92945c840e9,0x000022c04533de34,0x000bc1d0509d7400,0x00010f1af1754762,0x0008c160f15cf97f,0x000f0c1f85569532,0x000000000003b439}}, + {{0x0004f7c9bedca76f,0x0006dfa7d1236235,0x000a7e4930642e7b,0x0007288beb1282c5,0x000a07fee8a99ea2,0x00070fee91c069ef,0x0006fa5749c7b558,0x00000000000afcec}, {0x00048441716f41a1,0x00064a3f8f1b0daf,0x000b8af2f805e4cc,0x00029a9b59dc06f1,0x000b98a92c387533,0x0002b4662fa8e5f5,0x0006c66b6f46fd3c,0x00000000000ec04c}}, + {{0x00054b9f6efe8494,0x0005eaa16c27a15a,0x000106292d7b104e,0x000d193aae87c9d3,0x0009916d634e7ae2,0x000a65b4b125ab45,0x000e2202ded714cf,0x000000000004e212}, {0x0009494225bd1826,0x000c097c48a1862b,0x000bf9e4c3ff8573,0x000b77b2652f5018,0x000d078efd386fe8,0x000cb82991daa602,0x00062635885364db,0x00000000000b8240}}, + {{0x000f5a3697f1c244,0x0000e0430af76c1b,0x000f0e87f66ce63d,0x000905f12e919108,0x000012db9e14e1a7,0x000baeeac1c689b7,0x0003196bdd3dc90a,0x00000000000504f0}, {0x000e18cdf6373284,0x0003c874afd60b16,0x000a978150da10ac,0x000eee1ebf4aab2c,0x000c1aa49fe60d33,0x000217cfda3631ca,0x000e770d8340fbf2,0x00000000000423b7}}, + {{0x000b3813851ecc4b,0x0001df8c07372826,0x000ea9f99e2d35f1,0x000faf1a6305a291,0x0007f3e0f93d2b97,0x000aeb8c15bc61f6,0x00024b7238583cd7,0x0000000000039f5f}, {0x0002b746db300ac6,0x000a11cc8b467be6,0x000e46954d17b55e,0x0005f95ba2641ae4,0x0002ce9d565b1b9a,0x000eedc6287a0c36,0x00003d07fb51b2e1,0x00000000000a9739}}, + {{0x000d77fe5e566bbd,0x0001978a53b5a370,0x00081dca6fe505a1,0x000f427019a6f8e7,0x0006dc3ad0ba3520,0x000745b7cde6fcad,0x0002dcfec96e4f79,0x00000000000b133f}, {0x000924a225ecf745,0x0000c50088a2b006,0x000c145291ebead7,0x00032ff23ae4b9d3,0x000e85246712f213,0x0000b515e8cbc659,0x0008b727fa9c8df5,0x00000000000494ac}}, +}, +{/* digit=12 [{1,2,3,..,}]*([2^48]*G) */ + {{0x000ff6bf222c5c9c,0x000322986475308d,0x000309c5ef927cbf,0x000d6b4216ab9acb,0x0007be12d76a038c,0x000347bdb3df9b7b,0x00048913f4d9785f,0x0000000000013e94}, {0x000466717b5c75f3,0x0000a5f8e796eab2,0x000d6af2aad3919a,0x0005d8ad10740b88,0x000b5337dee6254b,0x000f02247c38b8ef,0x000c4cf688c2e194,0x000000000006c25b}}, + {{0x000272cd3b35df41,0x000d936c9dbbec27,0x00026ae70fa619c9,0x0008db696a8f9f19,0x00056b01e6bc1ab3,0x000fc4adae031d23,0x0004e410466ae68a,0x00000000000ed9c4}, {0x0005ea962547af52,0x000cb61272c12a27,0x000f929706a5a2ac,0x0007a910ecc49eb8,0x000ccbe84d5cf4c4,0x000e497d7eb95dfe,0x000ce443f3b71c8e,0x000000000004c6fe}}, + {{0x0002d9d94d551889,0x0006182e5d818574,0x000101531df0c231,0x00044261daa2e22b,0x0000e46f32576b02,0x00069db38b86a358,0x00027eacf145bd76,0x000000000004df27}, {0x000d2ba752047cd9,0x000c203d9391e25b,0x0007c9592434b2d4,0x0007845ec38fa9ac,0x000a265ad6bbefb7,0x00054a1b2dd40660,0x000499a22d988618,0x00000000000737ea}}, + {{0x000ef1248ca55f15,0x00028e323ed0c422,0x0001736a7d35b006,0x0002f06e8d68e4e9,0x000ad0742e5d9c09,0x000d3df92d8f5555,0x000eabe2d175bf00,0x000000000004f71a}, {0x000a6a143a42cf09,0x000c6d1762d7229e,0x000840a2cbe90735,0x000cb4c6281f2a74,0x00003603e53a2caa,0x000fecf29635ba47,0x00036194a9811d49,0x00000000000466bf}}, + {{0x0009fc85048451fa,0x000fd4737236d065,0x0005b89cfa755eca,0x00070306da6e06f0,0x0006f3838f569da9,0x00043188730279bd,0x0005d0fb328c8b94,0x00000000000be90e}, {0x000859016f87df1b,0x000843334a6711d3,0x00078d74e5890358,0x0007b6e38904b738,0x000296b588a53493,0x000577ae391e227c,0x000c7da599b21544,0x00000000000214a0}}, + {{0x000fcc62fe159c27,0x000c9e63fbb0fb71,0x0007ab3cd12c8947,0x00030677afd4bf85,0x000dfd37120d5cea,0x000d718a74494e39,0x0005fb8c572c7249,0x000000000005fa30}, {0x0005abf2e0c1181d,0x00074751c217ee1b,0x0005917c5a26a520,0x0006e6efe7a64872,0x0000f53b0e479a99,0x0005fd5931a4f6d1,0x0009ee651390ecd1,0x00000000000739ee}}, + {{0x000e27c9677a2151,0x000ef0b6d37446aa,0x0007f13e8f2bd87e,0x000c94fa109847d5,0x000044944c7712bb,0x0005a31b874c0d53,0x0005920d280b18eb,0x000000000007ef42}, {0x000c07ddca373d80,0x0004ef11030c77be,0x000075bee798eeea,0x0002d013d22f1b04,0x000cd93ee54dd5e0,0x00041d4b1b6d66c9,0x0006ed80b4154faa,0x00000000000acf8f}}, + {{0x0004ca485f1804e7,0x000ad0f05710ab2a,0x00002b0d41da0420,0x000a67a46b8d0e2d,0x000c698b78cc137d,0x0008a9393454b89f,0x000e69f2a6e1de25,0x0000000000016488}, {0x000b96b954a8287c,0x0003d7c6c5501c10,0x000fb63222050457,0x000e30e92f152478,0x000327e70a0a4b9d,0x00014936309d4ca9,0x0001379c8b16340d,0x00000000000ce642}}, +}, +{/* digit=13 [{1,2,3,..,}]*([2^52]*G) */ + {{0x0003c432161be476,0x000c0f8a8499b505,0x000715248b87d78c,0x0001b1d515e1328e,0x000ba941e788b85e,0x0005dd8d888a2636,0x000350a045241d2b,0x00000000000332f0}, {0x0008eeabc026bdc0,0x000f796c4b204e16,0x000ce54b1f342310,0x0003fc6d00b602a1,0x000e89aa3b796fc3,0x000d4dd0007a914e,0x00095635353eb7a4,0x00000000000673e8}}, + {{0x00027e0f6ecb7465,0x00040f36e83987c1,0x0002e0c806d929c2,0x00007464efc5b0d5,0x000ad316c43436ab,0x000ccf839b211e59,0x000a072515ec9f16,0x0000000000003dc6}, {0x000ec4a69e8d5661,0x00017842b727527c,0x00065c4526d40261,0x000711ccef5255e9,0x00075108cb92967d,0x0001b9740cd3bfb0,0x000308e50c0d8aec,0x000000000001a9e6}}, + {{0x0002078cea733c1a,0x0005b283eec25eca,0x00036d44d991d5b4,0x00083b827ad302e8,0x0002bde3fdd0269a,0x00030b3a6225f2f1,0x000043046fcf3801,0x00000000000c3ed9}, {0x00066dedc2439ae4,0x000eff870f14cae6,0x000680b39cf67cb8,0x000d5f4847be7732,0x0003d0ed73a0f3e1,0x000b3babba949822,0x000f706933ccf014,0x0000000000037f08}}, + {{0x000ba839c8cf3524,0x000ed1afa6aa5579,0x000f9ef0d2ddddd9,0x000920b5d36da502,0x0009291e774f07fb,0x0000d87a8144d51f,0x0001a026c2c134f4,0x00000000000a932f}, {0x0003544f31a7b78f,0x0009935bb2a42294,0x000ea47969f6664d,0x000cabfaa1838ed8,0x000fe2855f1f5c40,0x000525934ea0c05f,0x000fd4931ebb02fd,0x0000000000016246}}, + {{0x000bd623cb7fe067,0x00038ecfabd26775,0x0008c3832c0a3527,0x00064ccfe2691ca7,0x00058347566acf4e,0x000b8c733e3889e2,0x000f5748da354885,0x000000000009d9c9}, {0x000592cc5e5c9fb4,0x0000e8c26a8d609c,0x0000459f168ec210,0x000a70a3f8db9f92,0x0000e758213e181c,0x000b2653e25aa645,0x00003aa4898f9169,0x00000000000ccd83}}, + {{0x000fac468f3d59db,0x00099517d13b0e90,0x000cf2490366b8de,0x00026866d752aaab,0x0001cab026676e2e,0x0003163c395da9fa,0x0003c9be8d91ad42,0x00000000000a6995}, {0x0007a57ea4ee9030,0x0006ef728b1d231f,0x00013aa7ae93d8f6,0x00000a82e75d9c30,0x000e3ad09def97a9,0x0008a2be8136c6b1,0x0004474bab14a6dd,0x0000000000025cf1}}, + {{0x000e1d30e97eb41c,0x000b411b59f3da92,0x00020acfcec12d54,0x000f0a1936595900,0x000cca0b1b0e5cad,0x000274a5e8fef04f,0x00027ebcb4d9fb0c,0x00000000000ba784}, {0x0008e71ae4477c5e,0x000fbc49f0bc478a,0x000ac96d890c62e2,0x0009e583f796b820,0x000b17964262200e,0x000db00395bbea92,0x0002ba86b3c15756,0x00000000000ead48}}, + {{0x0007642e08638534,0x000b5cd92906c650,0x000ae6db49a06b5d,0x00029781fdc19156,0x000c269d611e0d69,0x00065b00a45d01a8,0x000388e7bd1e7096,0x000000000009bcaa}, {0x0007591cdd6ae97d,0x000ed8f189e87506,0x000ccf1d10959a9b,0x000bf16c633b1123,0x0009d6f1dac8ca65,0x000c3381dc6adc9c,0x000d120df2c293a3,0x000000000008388c}}, +}, +{/* digit=14 [{1,2,3,..,}]*([2^56]*G) */ + {{0x000653d60a9d872c,0x0003bffdbd0eb2dc,0x00061fddaf39f568,0x000cead35e7a3a29,0x000c67833028c11a,0x0004c79764998cf8,0x000a1c8f3a346b84,0x00000000000db9b1}, {0x000642a471b701b9,0x000628735dabc47d,0x0005300f39a216a2,0x000dd49d267a01b0,0x0003ffa20d117c0a,0x000ab2d4a2b46c91,0x000080f2acef26d8,0x000000000003d926}}, + {{0x000ba70a22083784,0x00084e9d2a98a2f2,0x00091072e4da23c2,0x000325dceaf86ae5,0x00088f161525f399,0x000211b9d03b17c8,0x000a48d8ac35c984,0x000000000009050c}, {0x000bbfa19d5ef891,0x0006ba818c44b2c9,0x0005e1b560830da0,0x000af35f8715b052,0x00099d8829a9633a,0x000a820f15463e1b,0x00075db18df52d84,0x00000000000d0966}}, + {{0x000ae853a945611a,0x000bed54e2c7c031,0x0009b2bf77ae12b1,0x00005e8e60f7f5a6,0x0008427483adcb41,0x000bff383705db30,0x0006e9f73ba98bf2,0x000000000000220e}, {0x000231b48f25af77,0x000e8c01c46b84a3,0x0004ae472f3bd7a2,0x000dc0bfa3403e6d,0x0007d2202896f738,0x000882e5e098bc68,0x000b13ec0c217a5f,0x00000000000a1f4e}}, + {{0x0009208204f520a2,0x000ed2615c78254f,0x0002b7a2ee7b484b,0x0001d771c84a04b5,0x000fcb9f9e349c3b,0x0004f7846b6fc203,0x000248bed65464c6,0x00000000000eb004}, {0x00040455a574f8cd,0x0003f5c017726a4e,0x000b0a7d5066828d,0x0006666aceb0e3ac,0x0008fc046f0ab78c,0x000aa959518d3c18,0x0004e87f3e2f301b,0x000000000008a284}}, + {{0x0008a96d207a2eb2,0x000e3c899eba3614,0x0002ec9689146ad2,0x0008629da55568ed,0x00082b1dc1d9607c,0x000c001ff6b54722,0x000b8523232311c9,0x00000000000488f8}, {0x0000aca655c2cd82,0x000723867ac9a850,0x00092fe2557b4773,0x000c647a74488fff,0x000fbe0876407398,0x00019a571f6ed920,0x000aeb72beddc763,0x000000000006a244}}, + {{0x0006fe658431a6d6,0x00064b7da79c5a1c,0x000fc6b2b6576354,0x000b7b54aa36d18e,0x0008ed0e30913481,0x00093074c6efeaf8,0x000654eb017bddc9,0x00000000000529dd}, {0x00089f1ff5fdf666,0x000fcf5177230c70,0x000373317732e646,0x00082d34ca267426,0x0005adcd1650194d,0x0007758b7eaeffe1,0x0008194dcec3d9af,0x000000000004cc2c}}, + {{0x0003e55601cd21ee,0x0004794bdc7f4a7b,0x00080eb7c8f212fa,0x000dab0d654cb574,0x00037a49195627e2,0x000d5d0991d4e1e5,0x000de7a1ef570c31,0x00000000000295f3}, {0x000c78902e5817a8,0x000198681b00d89c,0x00061b3376c1d033,0x000e90c6a1b57484,0x000c1222e5544324,0x000d53dd044f9324,0x000ef0e30ba10fff,0x00000000000b48ee}}, + {{0x0004a14fee68295a,0x00041a349bb65da9,0x0006f09eba200d68,0x000d891c18d37516,0x000dae8d2bfba6e1,0x000b330789985aa4,0x000e948baec9ae31,0x0000000000098750}, {0x000e6cd5311f8630,0x0009c0d834bf8a5b,0x000c536623c88198,0x000faaa0c51d098d,0x0002b887a10f0c22,0x0008bed323240404,0x00066231f6a61424,0x000000000001ce0d}}, +}, +{/* digit=15 [{1,2,3,..,}]*([2^60]*G) */ + {{0x000c2532e44da930,0x0009eac6dbdf6097,0x0009dd0474f3e9a9,0x0004de3dc28e1b18,0x0002a66477111669,0x000b08b4b2d039cf,0x0008cea13bbe6fc5,0x0000000000010ebd}, {0x0006e345ee3db6d1,0x000c2cd4720862b2,0x000fcd0516567c14,0x0006303929bfa29c,0x0003818249bfb3f6,0x0007f7cd97c378e4,0x000a1676068c8084,0x00000000000987eb}}, + {{0x000e55f593b72e2d,0x0008c1238e0971aa,0x00081a4c08ea4523,0x00054e7f74c5c514,0x00028805813b4512,0x0005e7a6e238b16d,0x0000bc23987b2892,0x000000000002a1a1}, {0x000c0e8fa1a83cf0,0x00010944c784c643,0x000f1939fa2e366e,0x000ab2aead457171,0x000631c042056bf1,0x0000b8881d5f8f0e,0x0005a8ac062526a1,0x00000000000fe550}}, + {{0x000d9597e54c99bb,0x00076cbbd8575ded,0x000019092c8277a0,0x00056202e6b72a58,0x000e7c024443e5cb,0x0005368f35738ea0,0x00044f8ed06170b5,0x000000000001aeed}, {0x0001a7ce730f30e4,0x0003a3d90a6b26fd,0x000480ba5c428a59,0x00093dcc5612aec9,0x000f99c1bacc0890,0x00043eaadc272ef6,0x00089147db3b43dc,0x000000000000832d}}, + {{0x0001abaa498687bf,0x0001bac92ee14ee5,0x0002ef494748554e,0x000ca876e7a32661,0x000a8456c9af29a7,0x00054326f0f2b7e7,0x000ec87471824e97,0x00000000000364d2}, {0x0007a1e32547416e,0x000f386d8aacd172,0x000735a9921c3b5f,0x000fc881d79f7eb2,0x00040040547d9805,0x000b4e90b377ac3a,0x00081bd39215d461,0x000000000004c5fd}}, + {{0x000bf8e11ac3f3fa,0x000df9ffe5562f1b,0x0000905bb11344f5,0x000f981cbefa3c77,0x0003667bfd643039,0x00040e3df2ab0688,0x0009ca9007a25743,0x000000000005a3a4}, {0x000a64031de3bcfa,0x000fd9c7be629ab9,0x0003fbe0cdc1be9c,0x000ff39e7380dbb1,0x00059facc3b8ffe7,0x000ae422c620bb9b,0x000c432ddcb8cd31,0x00000000000d12c3}}, + {{0x00072db65199d9d3,0x000d703621d34a54,0x0008bb1d8b92a619,0x00000f66ca2933b1,0x0002de1494a2886d,0x000dcdf82d0a2238,0x000832b8656c4d0b,0x00000000000c4a58}, {0x0005f2f5aceb0154,0x0002fa982c1aa34d,0x0006b9fce5077d2d,0x0008de431390c6e9,0x00018c4fe595fc26,0x00090d82fea4160e,0x0009076c427a6367,0x00000000000fc519}}, + {{0x000775c688e983e3,0x0003e8c0749464df,0x000f192d78daad34,0x000e96904bb2048e,0x0004d6dc8b606cda,0x000438bbc6dec959,0x0005a58d26586954,0x000000000001b0e9}, {0x0001e04544fa17f3,0x0005e8189f1141bd,0x000c131e8abc17dc,0x00075a227f8ec1ab,0x0000607e37397757,0x000793e4e7a12524,0x000af4afae84e74f,0x000000000005bf5b}}, + {{0x00039549b6d366af,0x0000aa10292b850d,0x000fff80cff6798a,0x000f18f73ad7a1fd,0x000c5897b11f7a36,0x000664c618b2c7b0,0x00022cf7b9a272cb,0x000000000008f81e}, {0x000e957ffad5963b,0x000663b99b210935,0x000ea3abc7ab4283,0x000175d001db74bf,0x00067159cb8a3af1,0x000de4601526f084,0x000eac6a3c6e1feb,0x000000000008c232}}, +}, +{/* digit=16 [{1,2,3,..,}]*([2^64]*G) */ + {{0x0004c667c92f7888,0x000aaa54768d9e88,0x000d1397d0aa7f52,0x000f203faef6863d,0x0004bd7471b3774d,0x0007de2e9f795a03,0x000cfff0958718b4,0x00000000000e1160}, {0x0005c7ba07a0ffd6,0x0005186ded97af9a,0x0008fa18cb4fab4b,0x0008920424e84590,0x0004eecf8b2b0558,0x00068a6fa3745591,0x00019fe7d3df1fb9,0x00000000000bad07}}, + {{0x000b4e8b4136f0e4,0x000ae2566021f579,0x000cbf2ef6760188,0x00047a5d9f51b6a3,0x000df8efa64634e0,0x0001f584f0b50d91,0x00091cc2bbcbb297,0x000000000000907c}, {0x000fd43610d5f812,0x0000ca7ebeb0dd65,0x0008a7b6eef1e9c1,0x0009a073f5c962ab,0x00053ff74a2fc04d,0x000f0749d95c155e,0x0005d0923c65a53d,0x0000000000027ae3}}, + {{0x000cd3b33afd62e7,0x0003c4d37c266037,0x00042b261375e38f,0x000a2e928ca9d674,0x000a79beb236566d,0x000f801e7a9771c1,0x000358af6b97a976,0x0000000000071259}, {0x00004ab4fe03d3c3,0x000bcc23a5e31cf5,0x000c506466ff69e3,0x000233b1911ccf3a,0x000cfa3b3ace3f3a,0x0004c5f5c93e4664,0x0003c04bdc14832d,0x000000000006abf1}}, + {{0x000832aefb763bd9,0x000f0d5469c7af17,0x0000d7bc962f1b04,0x000ca21a16caa7b0,0x0002e6cc7f39f881,0x000378723221de18,0x00066010d61ab531,0x00000000000520c9}, {0x000ed27c50cc42c5,0x000ac145214ccbc9,0x000ac441f327ce66,0x00059e30cb1fe6a2,0x000a3e299fe79fce,0x000af78cf2e6e77f,0x0001593a0cf652d5,0x000000000003e852}}, + {{0x00087e649fa97c56,0x00029ebbe18b74d8,0x000417a0e476f2ee,0x0005a2a0b24f97ba,0x000262fc61243ddf,0x00003b73af9a400c,0x000fde9ad9b0bc6b,0x00000000000153c8}, {0x0006f3a6ac904b01,0x000f41b6477d9542,0x0008ea414bce433e,0x0007128953069569,0x000db9e775794428,0x00005e0d14b5db01,0x0002f5237edf0bde,0x0000000000085533}}, + {{0x000e220415482512,0x000190791dfa1ca2,0x0007078d88a1aeff,0x0005b77ccedf4f34,0x000d5d965c0549f2,0x0009e672705170cc,0x00017637d9521bd4,0x0000000000086a00}, {0x000e53f005b29758,0x000b5dab44493664,0x0005df02ede5bef3,0x0000c3e4c2506ba1,0x00065ec6f1324366,0x000ac7691b2fb261,0x000eb5ccc4221a99,0x00000000000a576d}}, + {{0x0006b1405a6d6a62,0x0001e3a17f86cedc,0x000ffc614b14d0e0,0x000a3f0927935e2a,0x000ac04d4fb0a86e,0x0007694212f43742,0x00012f32a30bce38,0x00000000000d8d39}, {0x00091bf3c81523e6,0x000ed34154b18f9f,0x0001725c2ac49b1c,0x000be0f9f7a13c1b,0x000a531736e526cf,0x000cfa2c250d50b3,0x000e5f81849773b2,0x000000000003ba8a}}, + {{0x0003089126b94126,0x000854a87307686d,0x000fced9f497ff98,0x000042f427ea198a,0x0001ed259219c1d3,0x0004850cadc59211,0x000e8d0abdbd1623,0x0000000000043b6f}, {0x0004352ff19d9aba,0x00069d3c79d16450,0x00090120dc8bc8d6,0x000535d5a98bf664,0x000dab2cdb2ba736,0x000af89eeacb3b7d,0x00053237d3743ada,0x000000000000b348}}, +}, +{/* digit=17 [{1,2,3,..,}]*([2^68]*G) */ + {{0x0006e0a86f5827e3,0x000e1a68295d9bd5,0x00032e6b7dcbdf31,0x000a0cffe0c3df09,0x000b3cd00a1a8deb,0x0000f90885b4d037,0x000ef7e9edc429aa,0x000000000005847d}, {0x00025d7642f87bf0,0x000f49739d03ced7,0x000f63949ff1cd98,0x00034ff1759060ab,0x0009a7e94dbbdc3d,0x000e8b7f3029e9aa,0x0006f42a3cdfa0f7,0x00000000000bbd8f}}, + {{0x000178a4d4ad7d87,0x00040cd8c2ec862e,0x0003f7b9dc665542,0x00086d4db83e1ac5,0x0002bb549bb5e123,0x00007c0f19a79203,0x00008eaf81cba628,0x00000000000abf20}, {0x000001ed540ad4e0,0x0000a9d7a72302a9,0x0007110b90df5c50,0x00024daec9005170,0x00037193eb3047c3,0x00015c655408cd0d,0x000c228f0bcce2d3,0x00000000000869d6}}, + {{0x000e5354c88a38dc,0x000717192a26df13,0x00059c532d5dc6ab,0x000195620aeab120,0x00057c2328498938,0x00079bc39ebe39dd,0x0000925787970e4f,0x00000000000143e4}, {0x0006e01b286241f3,0x0001e5f60303562b,0x000cf202635ec6b7,0x000b3a2f1856e619,0x000bbf77d65c7ec4,0x00011058fbef7dc4,0x000945b444d50670,0x00000000000c33cb}}, + {{0x0000e30b66c01c80,0x000354b5e753d742,0x000c1ca06540fdb4,0x000d75f42fa905c4,0x000ac33e462b4034,0x0008de3bb89b85ed,0x00039131f413c305,0x000000000006ba75}, {0x00046bc0a659bd1e,0x000573f50020190e,0x0006ea7059f1f5a4,0x000aa10efa2adddf,0x000e1a6566aa7297,0x000b4f2143e8ccdb,0x000ae74dfda07fd7,0x000000000007cb1c}}, + {{0x0008cb34259001d3,0x0001d49d1cfcc46a,0x0008f80188e0d2b9,0x000d285d2ab0a996,0x00071df5b6cb8cd5,0x00003fadd9c5cff6,0x000c0019f4095a30,0x00000000000aa463}, {0x0002c2d4a3652c8f,0x00087f86673b013d,0x000814cd905b85f4,0x00075b44cc2cd5ed,0x0000b0342517b376,0x000817cd771d5262,0x0006183af1f31657,0x0000000000079199}}, + {{0x000066ce8afe9a22,0x000815600fedffd0,0x000fa276d4e1e61e,0x00062674cb08e4e2,0x00052e9ab52d8f12,0x0005686adfab0c1f,0x000e4e090b1ac94c,0x00000000000bb10d}, {0x000c301d650cbaa8,0x0003b3383f48546f,0x00045006d2b6b386,0x00092fe500058f01,0x0007f4ce508ac5ab,0x0007a5166f03c7c9,0x000d66525ac2cea7,0x00000000000996ef}}, + {{0x0002ee0f55870dbe,0x000f522daed9eb3c,0x0008bd7a619de4ba,0x0007daa46ec27562,0x0004df0f7a076593,0x0006036c14c58eeb,0x000aa5eb972393fa,0x000000000004d17f}, {0x0005a44cb91701e0,0x000a3de29c31e831,0x0004699fe7072908,0x000952e2c563ddbf,0x000e5d6b0b57ac00,0x000c4728203a767f,0x000d56e858fdf3a6,0x00000000000a5213}}, + {{0x000b5ed7b705b7ba,0x0002631796c22d0c,0x00099ed65d60e408,0x000cb716cb0c301b,0x0007d1702ca94ff0,0x000a02a90dd0cdbd,0x000625470a26c8e1,0x00000000000d7054}, {0x000218ab60af89bd,0x000e0fadc2e8673f,0x0000890003c99982,0x0005ebb2951a8fe7,0x000364197a76042b,0x0009c1b5de4af9d0,0x0002b1dd2c6bb324,0x00000000000f0193}}, +}, +{/* digit=18 [{1,2,3,..,}]*([2^72]*G) */ + {{0x0004ea16a709d85d,0x000329ded9b0ccd4,0x0002e9bda4d583bd,0x000aaf5393937290,0x0002a438413e94f3,0x0003c7de32213686,0x000540449286da37,0x0000000000029aa1}, {0x0001d592acb2cf64,0x000ef1bd13e4055d,0x0004681cce9d2c6b,0x000558f6bcd0cb2e,0x000e9e610369d43b,0x0006e0651f5757c0,0x0000aa15c80b23c1,0x0000000000000182}}, + {{0x0004d40f93bd737d,0x0008c10a8aabc8f9,0x000d5b1177a72237,0x000df1a076945a2e,0x000c03861f02a009,0x000866a3cf869152,0x000cbbc405226e9f,0x000000000008b41e}, {0x000cb1b314f36d3a,0x000bab5c9ed101e9,0x0005d83bc9ae2498,0x000f22e57589279a,0x000d7fe2d6aa1a9e,0x00073dbfa83ef607,0x000066f2da845434,0x0000000000057879}}, + {{0x000f91047abd3166,0x000593df98ea92d5,0x0002ec06cefe6ff8,0x000d1b5ea6d69771,0x00097e25ffbcc9cb,0x000a6280e3f9f231,0x000713bca8e0567d,0x0000000000035cda}, {0x000fe70b79caf906,0x0003946ef321cfa3,0x000b4a657c54ec43,0x0003b4b51de40452,0x000cccd9ba8d37b7,0x000f7045b5bccf3c,0x000dcca2ca080dd0,0x0000000000068cf4}}, + {{0x000d265e2cd7626e,0x0007e3499b44b9a9,0x000ad7706fd7669e,0x0000406c1f517d80,0x000b82a174e83014,0x0002c9b3ad4b3c8e,0x0006f4c8835b13af,0x0000000000044371}, {0x000fa90f05b33562,0x000c2336c4f4b966,0x000385e7d9f805d4,0x000a2dc828c34d3a,0x000c34d0d1f76ffb,0x000a4fbce257a345,0x00049fbc1eff056c,0x0000000000007683}}, + {{0x000db1540add72ad,0x0009e5ebdc79cc93,0x000baaf845cf7653,0x000efe79c8b7f7df,0x000ac469aa5793f6,0x00033c455b2f1e40,0x00012a882f0e9434,0x00000000000aea07}, {0x00030f4a7dec8786,0x000e9b13578c31bc,0x0001af154c9f03fd,0x000a2d854e09133c,0x000cc6ad853283ac,0x00004f75cae03943,0x000c37ce0a619171,0x00000000000f9838}}, + {{0x0004bc7eb2d5ec5c,0x0007e9db114e2be8,0x000c421a1f0f05da,0x000016bd2f3080f0,0x000fb9b7203704df,0x0004fe2ab0cb3f7c,0x000ec1adb877c781,0x0000000000037761}, {0x00081bf9c70c3c38,0x0003cb5d6068d8ec,0x0006b75a387ed168,0x000aa5457d36d422,0x0007423cc8cb39b4,0x000a6566dec1de46,0x000f55280e02dafd,0x000000000003ca55}}, + {{0x000e1cebdad8aedb,0x0006b340d77c218d,0x000b7391085bcda5,0x0005b028b74d32db,0x000a3cc1117e3e91,0x0006c5939ae7101c,0x000caa3c36152b52,0x00000000000d3ec0}, {0x0001039f162ada74,0x0004534de058e1ef,0x0005d23a486e5201,0x00030efc214ac0e4,0x000cc7100acb27c0,0x0009097b72a216d1,0x000aaf3b73004330,0x000000000007afd8}}, + {{0x000af1b588df6219,0x0003a5d7afe99881,0x000b20bd69a8d3f2,0x000067dc65234936,0x000527557f5de39a,0x0000e9f9d3458527,0x0002d8756b8d8693,0x00000000000040d5}, {0x0003845ac0bf2beb,0x000a9606eb60a92a,0x000238ad785ef768,0x0005748e068d2949,0x00022fce158da621,0x000183c8f73fbdca,0x000c989cfee07dcb,0x00000000000a5e03}}, +}, +{/* digit=19 [{1,2,3,..,}]*([2^76]*G) */ + {{0x0000e52cf2e9b4ad,0x0003f86230933fe6,0x000fe37b1c8aab75,0x00086ba6f595ed40,0x000de2284a5801e1,0x0009e5e25d3291c3,0x000f5303dee2311b,0x0000000000015cc9}, {0x000f15883981ad15,0x000444fba15675de,0x0008ff144b8a465b,0x0001eb92532d015f,0x00084e7455de8690,0x000f811c94396fd7,0x000b372fbcea8fbc,0x00000000000ae952}}, + {{0x0007195bc501819f,0x000f06bd2c54c87b,0x000c8e0d99fb53f8,0x0002654e3eee748a,0x00039598d90727f5,0x000cb371256f2058,0x000c5b5f91c2d027,0x00000000000a3e33}, {0x0002b15fc69c25d9,0x0003e248490a9884,0x000516ef1dee14b6,0x0008b3d74f7d38ca,0x00002fb602142013,0x000033d4eae791f6,0x000f176b4fb300bd,0x00000000000bdfd1}}, + {{0x000558e7c27dd125,0x0004d9bd2b68cfc0,0x0001783c86b4626f,0x000f35989586da1d,0x000afd0944ca67e7,0x000569b5abc04c8a,0x00046eb8db5424f0,0x000000000000cc55}, {0x0008ec9be17ec8dc,0x0009e8f0ef9decbe,0x000cdc094579a394,0x000eb644e7329b8f,0x0007e6896b042ec1,0x00018b6bf3e0e7f3,0x000fa6ad2bff11c1,0x0000000000040f93}}, + {{0x000fa586c442030a,0x000a30c3d2055f9d,0x000061ff065411bd,0x000a98b1fbf56cae,0x000f396e31cd1b68,0x00082c03b56fd85f,0x000917d2ca584443,0x000000000007d3ef}, {0x000572f96fe58801,0x000c74129a730d28,0x000ce4a000ce9071,0x0004180653f3b241,0x00049ee066bd9e85,0x0002ea5fee65a1a1,0x000e2e3420084e36,0x00000000000c7911}}, + {{0x00079bb68a21bf45,0x000b04c3f4f487b8,0x000f5bdc807ffd5c,0x0009bad1844be875,0x000e272afc140fbd,0x00040a7386b3b80e,0x000a37d727b0b6aa,0x000000000004d809}, {0x0001886f07e4e9c3,0x0004c09fb0640689,0x000e568c8f4da447,0x000ff8ad73a211ba,0x0001d9a6b97da7e5,0x0009e8f56ccdbf62,0x000a0938448e2cc8,0x000000000002126d}}, + {{0x000bfb3f1befeb61,0x0004c43f0ae67505,0x0007b68cdb6c3beb,0x0006c25dbb4492ec,0x000884bff97fd5be,0x0002da2e27efc9ee,0x000c3254fbd9d706,0x00000000000c6d2a}, {0x000d04af3852eaac,0x0003e14cf6dc0935,0x0006b6f8f3aa240c,0x0006543d9a16800c,0x0009beb1e28dd056,0x000e658339cc2ade,0x000cb2cbfeb45038,0x00000000000d0f5a}}, + {{0x00014927529fad7a,0x000714f10299a19b,0x0007bc14eebc4178,0x00018bc0cbede41f,0x000e6b03ac7d024b,0x0005213be7883c01,0x000540a7b99e741c,0x0000000000085a97}, {0x0007979358dabc76,0x000060b6fe59caa3,0x000a32d2efcf0218,0x000969d6ad1891a4,0x0003ee7761dccaa0,0x000b12539c47afe1,0x0005f4cae681fcdf,0x0000000000002ea6}}, + {{0x00033b53cfd1d032,0x000865e930d64f49,0x000e6882e13322aa,0x0004f7efa08bd4a8,0x0006a6e10f64cfac,0x0007ac162277e845,0x00068b41be306740,0x0000000000052a9b}, {0x000cf81e872685ed,0x000e28d8d216a150,0x000406cac9d54222,0x0004102c60d54c43,0x000028815187ad9a,0x0003b8e59b09420f,0x000ba1881179c60b,0x000000000005b09a}}, +}, +{/* digit=20 [{1,2,3,..,}]*([2^80]*G) */ + {{0x0002bb0ebd1ffa8d,0x0000c22313ded9b3,0x000c76c37673c0ae,0x000eeaea76ec3812,0x00060275ad3643d6,0x000095bdaa165513,0x0004b0e5c1b65adf,0x00000000000367c4}, {0x000f3876ae1976b5,0x0007de2b419c1bd2,0x0005d5344cebbb0b,0x000e51fa2baff060,0x00006269b5d0b5fb,0x000e8d667a0594d7,0x0006e70b07b70523,0x0000000000063e01}}, + {{0x00082243f0134f99,0x000d9022aa09c335,0x0001e1b9a8966dc5,0x00044c5b38b417ba,0x000b436451ec3173,0x000b1c87629a74cd,0x000ffdd898eb6ca8,0x00000000000b74ee}, {0x000187e3675bea14,0x000410d9eab63bb0,0x000dd59e68360fb6,0x00094ce14dbff2ff,0x00092706528037c8,0x00029a0cd3704d34,0x000c4f9ee435d8ba,0x0000000000009c11}}, + {{0x000414827bc8962c,0x000552fde4893802,0x00000bdd0aabfbb8,0x0008a5b09456ed5c,0x000925797c6c157b,0x000c14c0673606a7,0x0001caa9d0f47c33,0x00000000000faf97}, {0x00082453cb3879aa,0x000ec6e763c509a5,0x0006f45dc5dfe238,0x0007ecbc49e9efce,0x0007b38f2b59d95b,0x000c397db0c31792,0x000bc797912b53b3,0x0000000000045c7a}}, + {{0x0002600fe807b0d9,0x000578b3269a1894,0x000cd6a11a8caa23,0x000c714e428865d4,0x000ffdaba094bc5a,0x000d657f2531dd17,0x000bbf86d2405718,0x000000000002f99d}, {0x00063a9cad22b4ec,0x000d1c428a80ebb2,0x000111bbf67dd9f1,0x000691922a5d2566,0x000318a18586a752,0x000f633b3bfe6392,0x00029828a0c772fc,0x000000000003f3c5}}, + {{0x000cdc4baa08bf36,0x00052cd81bdce7a2,0x0009dbe6198554f1,0x000228bf8ebe39d1,0x000eb2cdd8524cb0,0x000e9b3af496a9dc,0x000f8a0f115a1ad0,0x00000000000d8eee}, {0x000de9efc567fd4a,0x0009b5232a514417,0x000762e720496fa2,0x00018b08994b4e8e,0x000809097a52289d,0x0004621d0b4e346a,0x000ab8c641510f32,0x0000000000095a41}}, + {{0x000c5913c30a847d,0x00004c8c9b0adae6,0x000c6088d1ec2259,0x0007946e62c508f6,0x0006e9fe2321b03b,0x000c1d1366042592,0x000c9c73b10bba80,0x000000000009d218}, {0x000e72b1483512ee,0x0006d6ed0c8eed83,0x0009abfdad8b62ec,0x000096ab9e96167e,0x000cc473e3773078,0x0003a3bc8b28f0e6,0x000796b44e499568,0x00000000000d3523}}, + {{0x000ec81c53fdc5cd,0x00095261578c7817,0x000dba0eec8c348b,0x00066b414c803900,0x0005faa7cc399932,0x0009b7e27bacad75,0x000533418c4defd6,0x00000000000ae751}, {0x000b26e534337f55,0x000f36f3bd3298fb,0x000bd59fe7197151,0x0004f73dc4c54ecd,0x000d44d3d74eb716,0x0009ea511ca66290,0x000c49f77c62424c,0x000000000001f714}}, + {{0x000343bd88e025df,0x000b893a62e7c716,0x0003b75e6fa4a448,0x0003cbb10cc95456,0x00062f37154c265e,0x000f2e08b28cceb0,0x0000916b79b0713b,0x000000000003ab39}, {0x00046ae52480b7d8,0x000a750a8d04cede,0x000f3c798fdbfa72,0x000336ac21cc62f0,0x00036fc004503db4,0x0004b2d954747f0a,0x0007a7067cbf1c54,0x00000000000fd2be}}, +}, +{/* digit=21 [{1,2,3,..,}]*([2^84]*G) */ + {{0x000f8f632d3f21bf,0x000880233e423a7e,0x000fc3a5113e93d2,0x0009cef6349e35be,0x0003ded2ec542ca0,0x000dded3d3b70afc,0x000ee0c813474d52,0x0000000000012f00}, {0x0002c304534f52c2,0x00002a0908f763fa,0x000cccb3aba1eb57,0x00010c134b6a8e10,0x00056e7c163e84ad,0x00057e0c36f37fd9,0x0001a27caae8c8d0,0x0000000000055372}}, + {{0x00086fa76217d20b,0x00015a354c1de108,0x000a55c4648be905,0x00010999d532bbe4,0x000f520117516766,0x00059e35eb7c2f4c,0x000b7b6945d34ff6,0x00000000000a1303}, {0x000495f21c8085bb,0x00028a200ddba08f,0x0003caf84b1e53dc,0x000445faa82b46fc,0x000cfccedc1b3018,0x00053e29fc3e6a3a,0x000547e86ae87033,0x00000000000fd7e8}}, + {{0x000683184a93e477,0x000a4728c8e14128,0x0003d8bece755c7e,0x000c2da4796df734,0x000e00a55efc3016,0x0006b1cb8f6d79e4,0x0005a94fced26952,0x0000000000091f16}, {0x00031165825c7f6e,0x000cf528b857275f,0x000d8cb6dce0ced4,0x0003520e07bb990b,0x000f4ea8ae6b4f90,0x000b735bb07ddb97,0x000d4a29c01e70b3,0x000000000005000d}}, + {{0x000f814f4e7d6929,0x0005e306b63e3665,0x00098d89bb61e0d2,0x00068898bdeddea9,0x000d67a329ac36bb,0x00092ccce9331655,0x0009b414c7fe26ba,0x00000000000be051}, {0x00042824d37928be,0x000830dc39dbc42d,0x00083d595f84e0c5,0x0008818fa8682bba,0x000db60aad97c7f7,0x0002c64cc43396ea,0x00043e751784d7ff,0x00000000000866af}}, + {{0x0007b94e1c1e3bf9,0x000a53742b61de85,0x0004d0debb63955e,0x0006787abce34016,0x000ea4cb41b4f52c,0x000ca817e4749711,0x0000487ce0dfb03f,0x0000000000096e85}, {0x000c4c6d7efe5c7c,0x000b6cd65250dc1f,0x0007ec2aec96d815,0x0003cb2b4dd561a2,0x000abb5379f57352,0x000a847ba15ba773,0x0002decc5e62d6c0,0x0000000000004d85}}, + {{0x0003450b4a8e849f,0x00036f0a69e68100,0x000d9356a0504fb6,0x0003ab7cd14af1d4,0x000d558d555d773a,0x000e63dd6c0b88ed,0x000db3eb12d197d2,0x00000000000bcd9c}, {0x000ed15d1fefd8b7,0x000fa06432985766,0x00052d6c3d3a4a47,0x000f6164171b9bc6,0x0009bbfcf90abd28,0x00036084d3778163,0x000a751f93898f3a,0x00000000000dd00c}}, + {{0x0000e82da9d478f7,0x000ae3b8d8e48b47,0x0000534e09405090,0x000430c2aad8c684,0x0006b82c16731d72,0x0000fb41de8c5d50,0x00034f989978868a,0x000000000000f812}, {0x000d9518cbfe70fc,0x000f1a0c941ce78c,0x000c323a8cf0aba1,0x000e296101f89af8,0x0003ca6a3227aa7d,0x00064ec0c4cadd8e,0x000db43b4db51f27,0x0000000000009256}}, + {{0x000e28a122d0f32b,0x00022b9207dbc085,0x000dce5b9cc6b932,0x0004e5534ba150b8,0x0001dfec99724992,0x0006f870d2935ebf,0x00085c34bf5e94b7,0x000000000004c203}, {0x000bdbb9ed40b2b9,0x000c3c78ac1719f8,0x0008866adbfa0f7d,0x0007ee4cfb4b8bb6,0x0002cd8724b676c6,0x0000bc1efecfaefb,0x0001b5e9bc3d6734,0x00000000000ca554}}, +}, +{/* digit=22 [{1,2,3,..,}]*([2^88]*G) */ + {{0x0004ccdc054842bd,0x000704f293478100,0x0000ca74cad5c267,0x000c94f64d55f6f2,0x000d725c51251bf7,0x000b6a0914489b57,0x0000fa3aa4d43ab1,0x00000000000cf7a6}, {0x0007fef943134dbd,0x00038aa918a4ce7d,0x000eee4d56ef907f,0x00000a16812a03d0,0x00070b0a1e4bed0e,0x00028386bb09acf2,0x000321c73a41f7ac,0x00000000000f4cd1}}, + {{0x000b2804db21ede5,0x000fd10a53b8f9a2,0x00080087f23a6e5f,0x000513fabef2b974,0x000cd3f802fd740d,0x0000627afff6cc47,0x0002dc1be6825beb,0x000000000005886c}, {0x0002ebc44567c313,0x000afb6cfb2c6b07,0x000e1dddbc404a5d,0x00016f53a5a485ba,0x0002e7b2223e7a1e,0x000a13a9b16b60b9,0x0001a2332f33d836,0x00000000000876cd}}, + {{0x000bcac1cf1b2eda,0x00095802f495c1f2,0x000e55ffc912d44b,0x000b294e9b3f0f6d,0x000033cbdcd1eaf3,0x00083fce582cc760,0x0004cc1a5642278e,0x00000000000ffa0e}, {0x000dada72c71a95f,0x000ed7d93e9766ea,0x00015d2d9c24e57f,0x000dd79eafe5f361,0x0000fcaafabbcb53,0x000053efecbbed8f,0x000b7a570472705f,0x000000000001ebfe}}, + {{0x0005f3669cd44c3c,0x00025a896d28dae1,0x0002e7c6820a84bb,0x000fe8c06fb15cd6,0x00061cdffdad1bba,0x000119b3a2daab29,0x0005a3984defc8e1,0x00000000000de2a2}, {0x0000f914a19c745f,0x000b3a7f54ecdb35,0x00041f75def22ce4,0x0000efc6a3b94437,0x00076785163b6913,0x0002b1d0a5d17f12,0x00093cc019911b17,0x000000000007e3e0}}, + {{0x00049fc81e69d5e0,0x00085d307d4279b4,0x00039a33441f2443,0x00078371531d6263,0x0008b42639ecfff7,0x00099fb76fdd61b0,0x000f0eca9462e7be,0x000000000007c3f2}, {0x0008f656f3641bfa,0x000f85c07066bcc1,0x0008bd883c4e6617,0x0001387d93ff4937,0x000b864eed19b6d3,0x000fad6382fd7d67,0x000fe667b707ff5a,0x00000000000d0206}}, + {{0x000bb568a2a2f3e3,0x000a9235455295a9,0x0008072539522964,0x0006d10c75720aab,0x0009f6a0df88ac83,0x0005a9c96ea2c538,0x0007bedfbf31519e,0x000000000008cad4}, {0x000d6b001091ce3f,0x00036dbd5de982e7,0x000e935fa43e76d4,0x000fef6324ba9235,0x000e8f14a2f5a2bd,0x000880ade8b0f981,0x000d4a590de1c45f,0x00000000000107f3}}, + {{0x00091e8b5ebad75d,0x00009b03d47621b9,0x000da50c79b05bb6,0x000ca72548723c56,0x0007c944e3849da9,0x000947973a12e40d,0x0001ea403dbcf2cc,0x00000000000c6ef4}, {0x000f1cf0966fef48,0x000d74ccc2856b9a,0x00011bbeae1e30d2,0x0007f7e43ee1576e,0x000a0a894961701a,0x000396f6e7f061c1,0x000ca956059fb88f,0x000000000001a91f}}, + {{0x0000a27b9e3336b7,0x0000be185bdc20d5,0x0008a8bb1575f81e,0x0003397c3fe5695a,0x000f61f6a04f1f6f,0x00090a92b6c9f3a5,0x0004543c0f9d4bb3,0x000000000004793b}, {0x00055e696b3d9202,0x000d991ad0c33a93,0x000c4d967cf46a4f,0x000d90565d53c83a,0x00034176007dfa24,0x000d1623cd63e583,0x000345741089fd2c,0x000000000002685d}}, +}, +{/* digit=23 [{1,2,3,..,}]*([2^92]*G) */ + {{0x000883b867d2fbb5,0x000271d51015761f,0x000afd0328ae7e6f,0x0007e2fc6dde9e2b,0x000937d5bb1f2ea0,0x000cc6e28ceed9ad,0x000fc43121994b98,0x0000000000067ad8}, {0x000d53a60b81c5b7,0x000ca34707fb3d11,0x0005e1e4ff2b5765,0x0009b8d81ef6cb57,0x000b1223a9a1fe93,0x00074cf375c51e8a,0x0008e18f6d7993db,0x0000000000097280}}, + {{0x00067952d0358e02,0x0005b0db3322730d,0x000098edfd5f000a,0x00085380e8a38fae,0x0005769360d86efd,0x000f27ef5c1812ea,0x0002e8505723dc76,0x00000000000f35af}, {0x000764970d5cb557,0x000849f7bb7fecc3,0x00052323da5066c3,0x0007549773eaf7f9,0x000489d51bda0118,0x0000ccde605bbbba,0x00087d52cabb0664,0x00000000000e7ff3}}, + {{0x000fa071b96c7c7e,0x0009c70187dfb0a2,0x0000f19decc5bd43,0x0008b816a6c0d616,0x000d6a27e97b3018,0x000a389aaeaeda67,0x0002734a5192826b,0x00000000000a2bf1}, {0x000832bf68c7b455,0x0000c8ff10656bdb,0x0008bc56c7023546,0x000a3e11dc7aed54,0x000b38853d61e34b,0x000cbf658c457048,0x000edde89825db1c,0x000000000006b255}}, + {{0x00067336ad6a8789,0x000d0c377b3730db,0x00063e31dca6f13f,0x000f2adf39b68951,0x0006bd6da22ed9c2,0x00009678e1bbff10,0x000e783f7e5b8e7e,0x000000000007ed3e}, {0x000a0e3ab40df97f,0x00071689e6b30fb8,0x0000b670f8712d68,0x000277edda78c55a,0x000cefb65cb717d3,0x000a30f0ce686cac,0x00071e758aa1ab19,0x00000000000b11f0}}, + {{0x0007d3d2024290f7,0x0000c0bef2d25159,0x000b908d7c494e4d,0x00017cedd4af7aeb,0x000bc8e88ca42c24,0x00020d15452b5b3b,0x000d4bc00328c44f,0x0000000000030af5}, {0x00053687c2073cc9,0x000dee8bacbeb2cf,0x000bf613b57931f9,0x0001da61d9e70314,0x000c0e70a331b67d,0x000e325772ddefcb,0x0009797d09bc6d61,0x000000000005b52e}}, + {{0x0000d7a5438e4dd1,0x000b5d40f36e5e3e,0x000702a395a7941d,0x00094d7cc51dbaf1,0x000f0edf8bf9e340,0x000b5e16520f93f0,0x0003a8534b75aa23,0x00000000000dc1b7}, {0x000196b771cc1d7e,0x000b09dd48c40ca1,0x000664851f0b3766,0x000f8a2a9ffa0db3,0x0004d94ed9bd4212,0x00069cb198d5236c,0x000dfc81bec489f1,0x000000000006104e}}, + {{0x00076550d0174653,0x0008d9cb21b56311,0x000fe59ad5e35eee,0x000a67100d7b34c9,0x0001a6c1b0c3e3f3,0x0003c3e31d4bcb3d,0x000be86ebd26fb9f,0x000000000002dc79}, {0x000582f4b4dfea09,0x0002ae7cf9a0a1f3,0x000c37b50dcbdab8,0x000320e69e1f08bd,0x000b946f83d78b0e,0x000a399ad2a8caf9,0x0000866daff0ba98,0x0000000000038ed6}}, + {{0x000af81ab2009a09,0x000fc16f3708ae33,0x000f381b02290f16,0x00015b227bb71a8b,0x0004162f8b5597a6,0x0001e3fb9a8b9a86,0x000ff680ee8362d9,0x00000000000f83a4}, {0x0005ffc830c362b9,0x0009507a7873a90a,0x0008c2637b53ccc0,0x000683d860d60ba4,0x00071f87d32a427f,0x00050319dea99592,0x0009ceb2ac69faa0,0x00000000000d2d0b}}, +}, +{/* digit=24 [{1,2,3,..,}]*([2^96]*G) */ + {{0x000fbfb66a502f43,0x0008324480c57881,0x000d6a55d7a45e41,0x00002c3273e2bc82,0x00053ef1ed4c7350,0x0004948e88c42e9c,0x000632028babf67f,0x000000000008a978}, {0x0008edce917c4930,0x0001bf5f13a4625c,0x000e9dd4dc6a263a,0x0006bcc37232768a,0x0001576e8c1830f3,0x000bcb766c5317b3,0x0004dcef1d57a69b,0x00000000000b3e3d}}, + {{0x000aa97e11eb2f75,0x0004f53ebbaaea28,0x000f59d2921161ee,0x000f25d340986b44,0x000e3365312a3c15,0x000641f96c5925d2,0x0008e4c35d3ee251,0x0000000000098412}, {0x00032040e2f19c59,0x00029b41a21a75a1,0x0005c4225a472860,0x000f4c0faf5663dd,0x000a0a62c9dc4ffc,0x0002d60c5889d285,0x000be50908665acb,0x000000000000a131}}, + {{0x0006f87a21caa381,0x000a6318feb4c006,0x000ed4b6b2c3b939,0x000786d5b2386621,0x0005a722c5911e4a,0x00006cb0188e0430,0x00036438eb062af4,0x00000000000e7216}, {0x0003e9f2fb9e0c73,0x00093450c1d28f54,0x00023904331c181f,0x000a53a6f6c06813,0x000a25eac032fe46,0x000c497224553199,0x000a2941fa659aff,0x00000000000bbcb7}}, + {{0x000a3dcccc791733,0x000a8c0de60f43c9,0x0004454fef6e3630,0x0002aa8e734e2e0a,0x0003ef773db6f412,0x0002d71c508f40f6,0x0008250a7e9484ce,0x0000000000078a3f}, {0x0005a99063a64539,0x000ef0cac7a5b50d,0x000064e61e079dfa,0x0009a90d3e181458,0x000a0aadf689ecf1,0x000f0acf70d1b062,0x0002b942299f1ff3,0x000000000005ac25}}, + {{0x000b4c356075ce0d,0x000a8a4bfe150fb5,0x000907dbaf6f1c40,0x0008260a0ee708f9,0x0004abc9d5f541e2,0x000aae99eff4fdff,0x0004d96ed92241ff,0x00000000000c04fe}, {0x000731763369357f,0x000a8f10a173e2a4,0x00065723921ac826,0x000823f8ed02e85d,0x00042e522dc1d2a0,0x000c939fdffa78cf,0x000bba07041e4600,0x000000000003a9a8}}, + {{0x0008012ba5de4e7d,0x000c7a1fc20f7144,0x000540a292e6e434,0x00081c28e2e8e16b,0x000a4c4562342e55,0x000e31ad4912c80f,0x0008117fd61fb9e9,0x000000000001c19e}, {0x000de815bbe3ce38,0x0003b6a7ab2cb6d7,0x000aeaa95e1796a8,0x0006a85ab06af4e7,0x00044866378d33f7,0x000826f8cda387e5,0x000cb6bb71deb856,0x00000000000eb64e}}, + {{0x000d7e72d8af89da,0x000ba0e93593424a,0x000584230569d130,0x0003dba8e15d8645,0x0001cab64644ab77,0x000fff0619cea4a5,0x0006a4516754d72e,0x00000000000cd3a3}, {0x000ee6af7eb6909f,0x000c6c7d352abc75,0x00007221eaa0649d,0x000ed979199938f5,0x000acd6f34958a8a,0x0005f93eeeb06789,0x000851c51d776677,0x000000000000a8af}}, + {{0x000b089742823cef,0x000c544bfd166199,0x0008c58b08cf9fc3,0x000cf81c39366838,0x00068b08680fe50f,0x00032aab0c9b4eb0,0x000ddaf90882c528,0x00000000000ecbf5}, {0x0004bc1447794cdc,0x000f8fd4d51c6fd3,0x000cc355ce034b68,0x00053dbfb187c34f,0x0006f8ed037fb729,0x0007bad45166f7f2,0x000808d2b1077a09,0x00000000000790dd}}, +}, +{/* digit=25 [{1,2,3,..,}]*([2^100]*G) */ + {{0x000273ed4a6f3c1a,0x00092a05c751c876,0x000bdb5d554c8320,0x000e26cf4c98fee4,0x0009e7b3a7c79c56,0x000cc16466084f8b,0x0000bca0b042f6de,0x00000000000fbf3a}, {0x000078eb34f9801e,0x0006694524c2c1cd,0x0008ad2d5e6f66b7,0x0002d6e7ce1f9f62,0x0007fc0c34fa1879,0x0000a16938ebc2fd,0x000019032a9f4178,0x0000000000056a60}}, + {{0x00009be19d72cca6,0x0009ea4b957d1c55,0x000c88712c99979c,0x0005e80a271d9e14,0x00044b9367e51157,0x00079779f0fff68d,0x000f68f55ba67322,0x00000000000d4d68}, {0x000ebae305d14369,0x000b401474ab1533,0x0007e0884a0d1ea6,0x0003a8336111d25d,0x000fab531cef6390,0x0002ed86737342a5,0x0007e1d3c93fe770,0x000000000006279f}}, + {{0x000ed609fa3c6148,0x000beb398290bf69,0x000b5366c9b47f2a,0x000aa2956560b89f,0x000c50e6647b3e71,0x000be42da80817aa,0x000a4e35c833ada0,0x00000000000f1bab}, {0x000a4dc6e615148d,0x0009ac57be644e77,0x000de3f8f9ac6cc1,0x00077310defd9095,0x000e899cba568300,0x000effb5a92fe9bc,0x0004bf48450ec4bc,0x00000000000f2f5f}}, + {{0x000e1a6a67c12c1b,0x000e7dccf68bc63e,0x000af5107f8ecb57,0x000e199254dd3e41,0x0006ddce5c84e43a,0x0000b4258e8526e2,0x0004c452bad815ea,0x0000000000009457}, {0x000fc838b5157d7f,0x000e1a5362fcad0f,0x0003a5c9b577c868,0x00012b5eeffc2cdf,0x000b14de4e5b0f93,0x000233fa55475657,0x000ec2746e67d4f0,0x0000000000035471}}, + {{0x000afd6c233d63fa,0x0009cf55edabb6d7,0x000b353b7e2a9194,0x000822b9bf171614,0x0003afe808dcf53e,0x000272f554a5b3b9,0x0009a1590bbbded7,0x00000000000eaea7}, {0x000101d093edb837,0x000b5d7042bea6a3,0x0006a766d7a462c5,0x000e69031078aa66,0x00087e37bc8bd8d1,0x0000089759e837f1,0x0008679558ff4f85,0x00000000000421fe}}, + {{0x00076617121c3ac1,0x000a13503d9370d0,0x0006e1d369d53e18,0x00001ef5b52e0f07,0x0002f955b5fa67f1,0x0000e0569b6b5b4c,0x0003efb5f03d5388,0x00000000000c9939}, {0x000c81fd72768469,0x000a690755bd5e4b,0x000324dbfd474da1,0x000fd519e9ce792e,0x000396ccfa14326f,0x0004a22de1b772d7,0x000342652710f487,0x00000000000db210}}, + {{0x0008597293bd01e8,0x000dfaa6489def01,0x000dc630321a7c29,0x00043a95fa2efed7,0x0003720e1a8c4d89,0x000a4f8a3c9baae5,0x00006bc055444b95,0x00000000000a7c12}, {0x00017016dee8b7c9,0x000dfd97765b5926,0x00050b911df827d2,0x000c35af1503b16b,0x000fa9fa21f8115c,0x000fed1f11dd5359,0x000679197c32996d,0x00000000000f77f2}}, + {{0x0004d7575fc73ce2,0x000ce58d6998db75,0x000d11d5d661c62a,0x0002a0165739fcf5,0x000822f7a073420b,0x000042f005c5db30,0x0003d9016c547805,0x00000000000a1241}, {0x0008aac8eaa9ff4f,0x000fe8ceb2d6ffe8,0x000f806ab6efe1ae,0x000d012339c146c9,0x000628ac05552638,0x000d46e6302fbb7b,0x00088e7fcb0c8162,0x00000000000066d0}}, +}, +{/* digit=26 [{1,2,3,..,}]*([2^104]*G) */ + {{0x00029831f2b8ed10,0x0004917e3d0b77e8,0x00062b9d59b96ed5,0x0000261d0bb1849e,0x000ec71a3bbef8ef,0x0002a88ae9b804ce,0x000b9e00b7d17154,0x00000000000e9097}, {0x0001f74d24b8094d,0x0000cdbad9f12075,0x000917cb364517bb,0x000834fb83debf54,0x000d637e449ba8a4,0x000b82664c3ee226,0x0000e8e3070d93ca,0x00000000000b3732}}, + {{0x000cd133b578c566,0x0004bb506b7b86f8,0x000a189da5b723c8,0x00029ad3789281ee,0x0001f9af456164b7,0x000f9186069ae8f8,0x0000fb007befe156,0x00000000000edc25}, {0x0003b5e8dc3a9219,0x000ab1c16d7598b7,0x0000fdec987e0e87,0x0007da936881f89e,0x000103a5b1ea5dc9,0x000153a0faddb603,0x000ee27eb515b13e,0x000000000002b4a1}}, + {{0x000bcceb06f924be,0x000d4e6b739c7071,0x0004a4fc7ff778b8,0x0003acc1b4b45493,0x0008f6bad44a14e3,0x000020972d6704b2,0x00033543cd7e5ddf,0x00000000000e0d6c}, {0x0008ad0a64850139,0x000b67d395c3ef6a,0x00040e40560ce9c8,0x0009f21efe4a5757,0x0004f55ea727b626,0x000450c823f97b37,0x0005d5b689b4de42,0x000000000004f5ae}}, + {{0x000bd2a72ae0f683,0x000d7e7365f9e264,0x0009dec60e7c539c,0x00089ac8e611fc3c,0x0005699c227dbfb3,0x000f2ef17ce778fd,0x000ed4dc1f47b63f,0x0000000000002672}, {0x00013b4caed73e7f,0x000a551c63805fa1,0x0002b72b6fe7e133,0x0003d13e00e12a38,0x0000bf8df325375b,0x000f5f21a91f9f75,0x000584f1ea0e421d,0x0000000000032c60}}, + {{0x000c6a3d385d54a6,0x000a26d720050d7c,0x00008151ff431078,0x000aee6f1c89ce3d,0x000b57b2b79d5d1d,0x000c7ce823f7644b,0x000b0bb1063f92fc,0x00000000000c15a9}, {0x0004b892e4ed0e8c,0x0007865661a5290d,0x000a266171fca5c3,0x000dc5d42ecede66,0x000c8eb0689e0661,0x000d7ff77be586ca,0x000c3bc86641a02d,0x00000000000b9ef4}}, + {{0x0006b862a25bd60b,0x0005903b07fb53e1,0x0000603f2bb26df9,0x0006449356376454,0x0002a55bcadf272e,0x00071c6af2b6ad1c,0x00080686c527765c,0x00000000000c1678}, {0x000fb64422cb7cb4,0x000660cae829453e,0x000b68619a56abb9,0x000f8337513dfb65,0x000e1b351dda504d,0x000af7cd04260ee8,0x0000fda473c5d9db,0x0000000000071e39}}, + {{0x000c98e3f8d5e928,0x000c217544e9789d,0x00047053e368c072,0x0000e3937af8cf8c,0x0009a2f308e2e146,0x0005060e98b4aa2f,0x000a7dafa2bdbc3a,0x00000000000f84dd}, {0x0004961384232a8d,0x000d53b256b55aeb,0x00063be03fa9b44c,0x00010d13c8e52f06,0x000b38865772c405,0x000416f1062c1542,0x000eb54755de7ebf,0x0000000000097aa2}}, + {{0x0009115b2961da7d,0x000f01d9aa9f9fd6,0x0004c642e6501980,0x000d6a17f167b479,0x00085bd9153d7ebe,0x000682ba6324c13f,0x0006e2e9ea5b734c,0x000000000007e3ff}, {0x000ac4814edd792c,0x000361ed2b04fb2c,0x0001100e4a13806f,0x000a5d3d68b8ec70,0x000d1c6ce0d2afce,0x00019410c21dc058,0x0005340043de75e7,0x00000000000e15cf}}, +}, +{/* digit=27 [{1,2,3,..,}]*([2^108]*G) */ + {{0x000e0e09fdbcad95,0x0004ef7aaa73197d,0x000fdf5aa02c7c2e,0x00017372e0c286f0,0x00025da3472da1a4,0x00076f2a23b66850,0x0007d4bc0d116b75,0x00000000000a36a2}, {0x000b11735059e67b,0x0001ecdb3f4744cf,0x00033adc74dab02a,0x00011b969d4f1723,0x00066cca4ef7c70d,0x000c6afdfdf96d13,0x0004da570693db2f,0x0000000000056750}}, + {{0x00080567673bd755,0x000df02c6b607d3c,0x00025bebf3a2d95d,0x000b01d4f57eb0c9,0x0004d750e515af5a,0x00089ad6859935b9,0x000eff32721b408f,0x00000000000a7e3c}, {0x0009a7a317218bf0,0x00010bd462fd8d81,0x000035ae54257b2d,0x000da619263fa61b,0x000eb6488eb34162,0x0005680c42ed1f2d,0x0007b0b0bd37a872,0x0000000000029ec2}}, + {{0x000d73368a1e2f2d,0x00047170f8a745d8,0x00056c4ca9228944,0x000bd224b08921af,0x000de6caca3a5f65,0x000cb9d5b1017ae3,0x00099b613f36b626,0x00000000000ed19d}, {0x00092b03ebebeebf,0x000c752c915a4d7d,0x00088621f044eee8,0x000f7c206a51a132,0x00066e0197ce2cd7,0x00071b04cfed60db,0x000b134e5b2bb7d9,0x000000000002f45a}}, + {{0x0009fb6e89b64981,0x00034fb09e755c21,0x0005804661454c99,0x0005b01cdb1f4f66,0x0008f1dd1cf0451c,0x000687e41b02f906,0x000bd3d4765e7c0c,0x00000000000d1967}, {0x000f135de5248a08,0x000f406a0e4ec0b1,0x000b70c2868ad046,0x000d01ac651d8073,0x00030618a96cc1e9,0x0004ad8ce5e6ebd8,0x00090f9ce1aacec5,0x000000000009953f}}, + {{0x000f6e431dc5b814,0x00044fa4b66978a6,0x00032f1c66e89ca4,0x000ba60986c85001,0x0004dae014e110aa,0x000bc0e4649bdfff,0x000d5fe2a810bd0a,0x0000000000062d1d}, {0x0007dc411c936920,0x0002c15d2f2c7072,0x00075b1f6234ac4c,0x000b6f94c545f5b7,0x0002f99241e840c6,0x0007082935875b6b,0x000344fa3e88a9d6,0x00000000000ad6d9}}, + {{0x000edb2b4857e570,0x000fba8d7901c33b,0x000ae7d92e8b3979,0x000d7c504e4bce1a,0x000e5ab0696fb905,0x000db9cd5dfbf9c4,0x0000fe0f4e6e2ab6,0x00000000000482e4}, {0x000aa1f98e714cdb,0x000225ccd9b418dc,0x00058c553b3cc2f6,0x0000dbe59065754a,0x000a461c1620cebe,0x000e541a74d052d3,0x0002d8396bac422f,0x0000000000070bf3}}, + {{0x000a0c1fea68e228,0x0005003c88a847c6,0x0008f73e5d2fc7c6,0x0003d393870d90fd,0x0004f2fdb976d90a,0x000312a302fd78d1,0x000f2f9472a6066a,0x000000000003484a}, {0x000f5efbe46efdf7,0x0008c7cf25950c82,0x0006be05e61c118c,0x0009379e563a2bde,0x0004b46e93d85d47,0x00037285ce87b50a,0x0007e69128fc11f1,0x000000000007c67d}}, + {{0x0006a0e99182cb3e,0x00059da2b072529a,0x0000a5cae364c3e2,0x000e35a5b132756b,0x0004fadb6907c721,0x00083f546e5695e8,0x0008aebc5a3bf4c2,0x00000000000dde12}, {0x000842d7630a9d87,0x0000d179c7fa6028,0x00016ae518569923,0x00083bea16bad558,0x000e9137ecd8af7e,0x00030d1ab6f41231,0x000ee8ac87543d8f,0x00000000000b1ee0}}, +}, +{/* digit=28 [{1,2,3,..,}]*([2^112]*G) */ + {{0x00021880433d0094,0x000fe359cbfa01b1,0x000ccfcddd1c5f17,0x000e90f630cbcb5a,0x00006ddbf2382fd5,0x000f753e62b0f613,0x0000165070970a3a,0x0000000000041727}, {0x000724192d1373c9,0x0003e2eb15b3b70b,0x000786ed962decdc,0x00074aee930a75c6,0x0006849e77270d43,0x0006a19ff3cd3604,0x000c049ac3117e33,0x00000000000433dd}}, + {{0x000b49aee05ae3b9,0x0005d985dcb4303d,0x000d13447c8994a2,0x0007f3055fbf1d6c,0x000f43fcd98a4059,0x000ec99cab6d166b,0x0003705932570915,0x00000000000c5bdd}, {0x00050b2df8b6890e,0x000dac18f782613c,0x000d7472d5468bfb,0x000a89df478e56c7,0x000864d290535d50,0x000cf7e0e242c2f4,0x000d40cda12c6161,0x00000000000ac8d1}}, + {{0x0001c7858f5f5373,0x000027a9a9dd7275,0x000baba450a60383,0x000267b6726c3253,0x000d7f841afb1550,0x00004643012cfb2f,0x0006cba41b19052f,0x00000000000b7d78}, {0x0007d7e12176321a,0x000667b61b39c512,0x00091b7379479d0c,0x000efb84a3489052,0x000cf2e864b965d6,0x00096cdb2f377862,0x0000f63c94000dab,0x000000000008efef}}, + {{0x0001ea0f696fd259,0x000ff8c558000585,0x0007b878d3ed6d3e,0x000d75ceb7a35cc0,0x000cf0a93110acf0,0x000c50c850fbdce4,0x000d6f82b2d13a9c,0x00000000000cef70}, {0x000ce46086dea462,0x0001e1cc9be2dbe9,0x00043757ada5e365,0x000479f195a532b9,0x000a1ab3605c52bd,0x00024c1e0769fe6c,0x000d7aba6a63e48a,0x0000000000099da5}}, + {{0x000ad5fa3c1cbaf9,0x00066df2a6cec962,0x0003dc76a0f4995b,0x0007e4c0de8a6aa7,0x0004be7b8f5d7c8a,0x000da3e7e5c806b9,0x0008e5ccca1c714f,0x00000000000cff78}, {0x00056778e2279cec,0x0006e17a081a8743,0x0007133643f614b8,0x00090549e4cfac8a,0x00029d53a2000d7c,0x000977a8b6fb74c6,0x0007c465b9ed6d5f,0x000000000000ba2e}}, + {{0x000a6aa57649a837,0x0000f8435e295b6f,0x0009b668270d6cf9,0x0006e6fece5cb161,0x00095336f7ffe2fe,0x0008ee2676f249b0,0x00017b5801feb990,0x00000000000fc8f2}, {0x00006b2a4f1b5cde,0x00053fc1adc3e51d,0x000e0bc8cae40bdd,0x0005404cc1d9b726,0x0001a1f2c5a44afb,0x00008f19575cabd6,0x000f492bd797e1cd,0x00000000000cbea0}}, + {{0x0008ce5f4471b983,0x000d8e9759aa9a33,0x0001c65f7f2a0258,0x00059aa2c80bb617,0x000230c9b328e49a,0x0004ebffcdb89e21,0x0008e4f42b9adb00,0x0000000000047967}, {0x000c42c02e645b6d,0x000f20dde5b0e690,0x000d9a21c0cf9036,0x0007a5cd8c8285c5,0x00071d1562069f8d,0x000e126bcd7cde59,0x00060bdbe765d7a8,0x00000000000e3f4e}}, + {{0x000bcd16afac5b0a,0x00041509abccab5b,0x000ba67813ffa4bc,0x00071bd640bc2f7a,0x00078f546c681ae3,0x0008b8cbfa155c29,0x00048b8858cadcc0,0x000000000001d969}, {0x000da5c95714aa4d,0x00010919cb2262bc,0x00085c89980779aa,0x0000f1ab4abe5612,0x00069cb75bd125dd,0x000a778e4c6bbe63,0x0004b3bb367cf947,0x00000000000bde52}}, +}, +{/* digit=29 [{1,2,3,..,}]*([2^116]*G) */ + {{0x000545d771f3d008,0x00076c6307398a47,0x00054915c4cf9c72,0x00026aae3a993222,0x0006da9308d18e01,0x00068e5050fbfdf7,0x0008bd2163ed6b61,0x000000000007500d}, {0x000f06ed5b023878,0x000d3f571595fffb,0x000ccfd2dfb01965,0x0005f7e53d84c661,0x0009025029da6b22,0x000d89aed9c03126,0x00015054c1edfa17,0x000000000006b435}}, + {{0x0007c3fb4d5bcba6,0x0005b009ef80a60c,0x00017966ca95c6c2,0x00055ed685add960,0x000672c7dac8ab44,0x00072f16818f323f,0x000a99e7009790e3,0x00000000000067be}, {0x000c5bcf19664dda,0x00043f15bdbcc383,0x0007e49d4e4912df,0x0002c304bdbec36f,0x000d1b3ac6c72fd6,0x0002938898b3ea93,0x000f8bce1c8e5c9c,0x0000000000011564}}, + {{0x00098a86761cd6cf,0x0008477faac22471,0x0009e917079e4be8,0x0001dd45fbd35ab8,0x000a45d6d40e31ad,0x000749ef67e0de9d,0x000cf2a16c5f1939,0x0000000000085691}, {0x00046f38fcf548ca,0x000e7156536ac506,0x000b8e250c84b9f9,0x0001eacd35088fde,0x000538fb995c165a,0x00093b3a5ce8a733,0x000e7934d0d33132,0x000000000008b570}}, + {{0x0002b3fc481df360,0x0006d6a3a232752d,0x0000509b75e73d18,0x0000c3322a82d404,0x000a14cbc8351703,0x000da272724bf18e,0x000d4a319dc4cab2,0x000000000006d020}, {0x000398eafd6e92bb,0x00066aeebdd89825,0x000a73a882a68cbb,0x0002978595f361eb,0x000c1ae8fa3cfc8a,0x000341575e60dd96,0x000e4819c2109aa5,0x0000000000006a0e}}, + {{0x000065dd44a81443,0x00083443deaaabf2,0x000c83e66800298b,0x00018ded51d6acd9,0x00017d4ec9bef889,0x000c48a6948c9b2f,0x000f5478b41104df,0x0000000000078827}, {0x0003f535562e2f60,0x0004674ed7948c07,0x0001ccc08940c103,0x000449b0d9fe252c,0x0006c14e825a6fe6,0x000690f0031743d0,0x000748d9d4ad8c08,0x000000000000e65f}}, + {{0x000eaf196980dd99,0x0009692501ff821f,0x00072b4a2cc10b2a,0x0009a05e53038473,0x000c2543d1a995b9,0x00080877130a72fd,0x0001456648126b11,0x00000000000eefb8}, {0x00041f6a86207b43,0x000d8d9a87f836b7,0x00016f9bd17dd926,0x000e118c40122e4c,0x000f853b291e3341,0x000565bade513567,0x000f77d4f1bbb73b,0x00000000000a7658}}, + {{0x0007b4a08d152bb9,0x0007cbab37a630ed,0x00077618410cb7bb,0x000086c1d4a8f5be,0x00046581428269e1,0x00076cc7d3a87384,0x0003a0b642dc1626,0x0000000000097e42}, {0x0008ca2aba5729e1,0x0005b66ac1b365bb,0x000e67e449b0ae7e,0x000debbadcc68242,0x0004b1536fa18f47,0x0009e546144e9bf6,0x000b901cfdd50493,0x0000000000043bb3}}, + {{0x0005a0a5974046e0,0x000112e32f89636a,0x000495f55b511bd6,0x000b1ad8f8328866,0x0008adcbd933278d,0x00064f863271d907,0x0003652208fae34a,0x0000000000039c89}, {0x0000e167567c2b6c,0x00048cb46f2725d5,0x00086f7986cb61ca,0x000d316edd504188,0x0004cabe36ed3421,0x0002efcdab1d8a06,0x0003eedb13e56036,0x00000000000c71eb}}, +}, +{/* digit=30 [{1,2,3,..,}]*([2^120]*G) */ + {{0x0000e8a3d453ef19,0x000752af8ed809c8,0x000017d0798a8632,0x000726f782193578,0x0001b87254c44c0e,0x000e7691a8c1962a,0x0002ee30796a71c9,0x00000000000a75a1}, {0x000c339094f215d4,0x0003e535f42c17eb,0x000b753210a29b6e,0x000ef35e39a7a591,0x0002d459b91ab7f1,0x000fd429da2789ae,0x000f57eadbca7f02,0x0000000000065290}}, + {{0x000ffd02d3a50b87,0x00027c085500177e,0x00023ee786608759,0x0001d964318e861c,0x000604fe9b85dda7,0x0005e7e2001d3d39,0x00081cda4bc065e2,0x00000000000e076c}, {0x000171ac92e482e8,0x000095b9f82189f0,0x000cf8881039863b,0x00083e4d8dd159bf,0x000720b18043f526,0x000a0d8f5ca9c888,0x0005c473c040fa08,0x0000000000017952}}, + {{0x00023e634e793b49,0x0000c37ed2be4ce8,0x000e92823d3628c0,0x000ad8ae77c2f00c,0x000a44de16a8b061,0x000e490ffe87e1a9,0x0003eddf4f57c87e,0x0000000000000599}, {0x00036b9f1b67eda0,0x00020e1036387a16,0x000cdc81b1b14889,0x00020d15ab42f920,0x000dac0ff03359cb,0x000c1e7f4a738a18,0x0006e0da501a2e2a,0x0000000000084d8a}}, + {{0x0000efaf35b1418f,0x000173a8289046f2,0x0004fa8840c897aa,0x000898ae5fa19d2f,0x00065e1c5fa4c574,0x000390fc20b13dbb,0x000187f11343ba7c,0x00000000000d7d32}, {0x000b8b2ed2cc734d,0x00011c92cb1bab11,0x000e6307a3aa4fd9,0x000dfe5672f2af7d,0x000430932441da69,0x000af02d69c62d7b,0x000c22165672ad94,0x00000000000cde81}}, + {{0x0006bff8295a2913,0x000d91d27efcde72,0x000bdb7b59692a46,0x000fb83caa26d18c,0x0003b82babbe99a3,0x00083dd60d27e613,0x000cb861030dfdd7,0x0000000000073c78}, {0x000a3caf05842de3,0x0002d8707a2cf633,0x000f47297bf43775,0x000d412a2b176b8c,0x000fc72021017c3f,0x000a6d536d5b52e2,0x000ec024a63030f0,0x0000000000004648}}, + {{0x00024dd131928645,0x00063d45e3501026,0x000cd2a97f7d8b99,0x000a088e302483ae,0x00082c2485c01532,0x0009cdbe63475e8b,0x00073808f9ea5696,0x00000000000253cd}, {0x0002a544f2d5917a,0x000581cf323a3b9c,0x00023a0e49f58c76,0x000e84fc06f3d0ad,0x000e39efe6d99a31,0x000ac4decd326b86,0x00058277e3e1df14,0x00000000000f3e0c}}, + {{0x000611e8121168e1,0x0008967477cdc1e4,0x0003ef00563660fb,0x00060b9917eec666,0x0004d66c5c1d31fd,0x0009508baacd95bd,0x0002432e0551f3bf,0x00000000000688cb}, {0x000033e51fe09e4b,0x000448c039740256,0x0004cddb41207f6c,0x00025e144db62afe,0x000bc4a030da1918,0x0000815043ee8aca,0x000cd08ab154a894,0x00000000000486c9}}, + {{0x0009bbd8fc757c25,0x00003e92b56bf065,0x0003d138d6f390b1,0x000d0ef50f5c483a,0x000611fa89fe7754,0x0004ea7d8850a9ef,0x000a2e97d74b1bba,0x00000000000aab7b}, {0x00042e268ab251b7,0x000af06f30ab067f,0x000bbd0bcb9d997c,0x0000bca7a2e053e3,0x0008dcf0e14c4758,0x000f553108579559,0x000e18c3596781d6,0x000000000004c9b7}}, +}, +{/* digit=31 [{1,2,3,..,}]*([2^124]*G) */ + {{0x00056db726934b05,0x000d28db8b78ca20,0x000efbe1df76bc8b,0x0000f022dde2e3c8,0x00038cbf406c67e5,0x000f7ff602e2461f,0x00068832182781e1,0x00000000000e30e2}, {0x00008c8748c4086f,0x0000895adc204b38,0x00054339345edf7d,0x000f060b73417379,0x000fd128d46cd5ca,0x000fc1e04ed93187,0x0000a13f2819cb20,0x000000000002b7f7}}, + {{0x000424a4cdc9ef0a,0x0009fd74d09c0410,0x000c23c8eb2570ac,0x000cfdce132b9412,0x000c843d3c66db1c,0x000cb3ef4a0e5309,0x0001771fbd03a5f9,0x0000000000000ea5}, {0x0001bb0b6df68f60,0x000fa1ebf5a155d8,0x00046a120ff8039f,0x000c37aa0d34d161,0x00050fca43af3256,0x000841bdeee40efa,0x000a0bc299bbd4b9,0x000000000003bef4}}, + {{0x0001fdb0e87d4c69,0x0008f5bbf8bfdcec,0x0002c8b1b68f641d,0x000718bfd3fb74be,0x00015a085196abd2,0x000dba2ea03c0150,0x0008bc147474dc4d,0x00000000000eaa2c}, {0x000a67e6cbd2f408,0x000e9bc3dc5c2f94,0x00045e933f00eed5,0x000e98325d341410,0x000406fcbc8e0276,0x00024ebab9372694,0x00050b8c4b7f2ca3,0x000000000007dc3a}}, + {{0x000547a274927d68,0x00075e01295fd654,0x0003c43252dc8b6c,0x000689f9eaf8eb82,0x0005c622acc52ec9,0x000757f2a327b96c,0x00006eae918f81b2,0x000000000004fd66}, {0x000f52e65907ef2f,0x000a109bb7fcdbf4,0x0007ab72ec41ee7a,0x0006f125fb475125,0x00018e399520df2a,0x000b1c3a2be88faa,0x00081f0166d57e3f,0x00000000000e525f}}, + {{0x0005132af5b85725,0x0006e0c64cec623a,0x000d73b377e49971,0x00043ae5a8027b17,0x0007d0a7406ac27a,0x0002bfeed1c5729c,0x000357445fc7d34e,0x00000000000809bc}, {0x000fe2c3577b04e1,0x000a30598626b6a7,0x00059236250da683,0x0009b314294839a7,0x00093ab9bc662151,0x000ab597466282da,0x00012e84c422c25c,0x0000000000071c0f}}, + {{0x00044dd5af953087,0x000b3f24fab9aa7e,0x0007e53a0eea7298,0x00070e733ac4fb68,0x0002c8a6976575e3,0x00030aa0ed8d6164,0x000637f057f10a5e,0x000000000004f1e8}, {0x000514c4bb67bcdc,0x0005e887f06b2b4b,0x00032e66a34587bd,0x000f7eb6e6483a19,0x000f475d4bfd4319,0x000811fc744f1aaf,0x000f8ba8ee73b50f,0x0000000000082916}}, + {{0x000b0ecbb3371c13,0x0009596495c1e57a,0x0002216cb93b15bb,0x000c858fe6e84ffb,0x000b0b189be26806,0x00058a7685c215e8,0x000202ac61577e80,0x0000000000088e0d}, {0x0004f2a99a53c49a,0x000de70322e164ce,0x000973bc42c42efc,0x0001f014d701ab8e,0x0004a61df25f8863,0x000f6dbeb889f10d,0x000168affec5a9bb,0x00000000000934ee}}, + {{0x0006f789ccd0fed7,0x0003cdefe3a7abd6,0x000569da64526056,0x000c26a73d1f9b54,0x0008237ae8b77366,0x00047825935d5d71,0x000ee33efb20feb4,0x00000000000aa545}, {0x000d70972e2560bb,0x00081750edd05bef,0x000da581b51c4635,0x0000f370a9e29dc7,0x0006bfbec7f616e6,0x00047e1439cf0a13,0x0001a9b430a34b2e,0x00000000000c6cdd}}, +}, +{/* digit=32 [{1,2,3,..,}]*([2^128]*G) */ + {{0x00056f0266d7e788,0x00065f186d6bc61b,0x000e03ac4fb95d1d,0x000432edfe64dc3e,0x0002d795ea57c9e7,0x00045af2bd9fc483,0x000517f4ffdb81c8,0x000000000002b670}, {0x0004d8950582c931,0x0007aa7c1be04d23,0x00045b4daadb356b,0x0008aba03f2ef6dc,0x000420dc701d62ff,0x000f1011960ecaac,0x000fd09f4b559f4f,0x000000000003cd54}}, + {{0x000973c51356e0d2,0x000aa31954a56667,0x00047c018911cdb5,0x0003ec5e37ef1d2d,0x000575bc3b668c43,0x0009cd98bcd0f203,0x0002a33723f14524,0x00000000000ea3b4}, {0x000998df93db2ed6,0x0000ff607fee05d9,0x0005662b349502ed,0x0005d74cea5417fc,0x000a10fca22bd47b,0x000635e8b6891940,0x000fef5cea41332b,0x00000000000b5934}}, + {{0x0000bd5aa7cff3d9,0x0003c365a4a426e6,0x00069eea0aae5d6c,0x000c8b4b7205a704,0x000357df815ca330,0x000a926744858eab,0x0005d76e522afaf1,0x00000000000f4136}, {0x0001a59021cca096,0x0002a00dd3461384,0x0008629f956ac671,0x000103f7c082301c,0x00076b092b71f427,0x000037a3314a2a18,0x000e14210f632130,0x0000000000039340}}, + {{0x00025fc00f86f9f5,0x000b5b1f4fead323,0x000e624d9ef69f8a,0x0007efe2e28aa2a7,0x0009b3151d6748b4,0x0006a346070dad2e,0x000af787a8178b43,0x00000000000801f7}, {0x000701583b4bfabc,0x00034fab462e36e3,0x000ddd2177064fe4,0x0006b3fe66812831,0x000a6a3fdc1db469,0x000e37dac3bd9910,0x000fb34409128449,0x00000000000cf605}}, + {{0x00081f556f82acd9,0x0006f59470e4953f,0x000963a40f813b02,0x0003c1e0fb30ecd3,0x0003b62892f14761,0x000eefa161fbeffa,0x000719ddfeef49f8,0x00000000000daeff}, {0x0004929822ef7d6f,0x000dcff1872cd89c,0x0009edd2dba5c5c9,0x0003a387695218a2,0x000720802b8852a6,0x000f4a473a0d413a,0x000a4a233f1f3118,0x00000000000c9d7d}}, + {{0x0006ae71b9b2b784,0x000ca86737912907,0x000e250552a02dec,0x000185ee92c712de,0x0006c201e3272efe,0x000f6b0788b908f8,0x000bf33ab5528894,0x00000000000ce0a5}, {0x000abf1c0844ab25,0x000185eab37ac7f5,0x000c6bca8c37680c,0x000edff304f1cf97,0x0008f6fd3e90f3a3,0x00016e5ed8992ecd,0x0005c4ff24d7c69f,0x00000000000def9a}}, + {{0x00054a31e739d50e,0x0001e2edaaaa9c57,0x000574cc3b6825a1,0x000fc42bcb161908,0x00046709d8513542,0x000af4ce06c04ec2,0x000e2e5fe9768ea9,0x00000000000eb6eb}, {0x0005126ebc807472,0x000c87b7da4abcae,0x000d07139021766d,0x0001c51fd563c816,0x000926c775e17f69,0x000b58889ea990d6,0x000c2c2cd96f1321,0x000000000000f1aa}}, + {{0x000e697116bc35a7,0x0002057edf71ec8d,0x00063f77e7793641,0x000ba1aa6934160d,0x000c5e639a54b37a,0x000bce957d45b3d2,0x0007f362c6b9ad70,0x000000000005d7e8}, {0x000a8127f71a285b,0x00017b02169816f9,0x000c7a7939056bf2,0x000692b478e4b92a,0x0009be3fe25a15aa,0x000bdd4a8dd67cf2,0x00027af1ef75b006,0x0000000000041df6}}, +}, +{/* digit=33 [{1,2,3,..,}]*([2^132]*G) */ + {{0x000455d88f7fc60e,0x00091a58a27c4fad,0x000ee65aa7e47d3c,0x000826600079a263,0x000f2606caf52431,0x000cb28c8113f6fc,0x000aa6f6ff2be316,0x000000000003c17c}, {0x000c05f668a9dc76,0x000798ea577e0148,0x000c912137590c65,0x000eff4592a57ef2,0x000c5e3f67b24b28,0x00000890276e1f87,0x000b7e40d7a676fb,0x000000000004b6e6}}, + {{0x0007f95d338f5fb6,0x0000076e27519dec,0x000fefb9477deeec,0x0002dfb71fba6625,0x00085e0886af5832,0x000ee544c228aec0,0x0006398d83b6276a,0x00000000000e29f9}, {0x0004808391599977,0x0006d1ba4cfe02e0,0x0000441e3d4b3e92,0x000ed58cee95d92c,0x000abe5355407e0e,0x000a9a1f42d29282,0x000c82866638b607,0x000000000006ab8d}}, + {{0x000c43ee7094e067,0x0000b599ece9688c,0x00025bcbe937baac,0x0005ba86373ccef5,0x0003c64f72612afc,0x0004ce1abe455683,0x000a77f29ad54041,0x000000000003ee82}, {0x000e0b1548890be5,0x00094fa26eaa0940,0x000c96b0a3e57442,0x000ade54e1c94bd7,0x000e20b0ddbe9570,0x0006eff6ff4a86d8,0x0007a4967ab653dc,0x00000000000e8bab}}, + {{0x000de87dcf2f3d8e,0x00056a16c79cee01,0x000db47cfbc9e301,0x0006aa0c2f47e69c,0x000f2c9b4914b8bd,0x0007242a8277d590,0x000512b6d1c81009,0x0000000000045f75}, {0x00044d25d22dbf16,0x000f02176162aa27,0x0001167bcefd598c,0x000c0d1a503aee8c,0x00057af4dd78fcf8,0x000b43be45d1f94a,0x00071b8f0bc1a62a,0x00000000000ba9e0}}, + {{0x000adf4d6bef5a3d,0x000a19a2dc376622,0x000b08bb2e394780,0x000e5cf7c0765207,0x000c4063c3538e70,0x000baec46c0dbb4a,0x0001ad3f4550566a,0x00000000000f9356}, {0x000e15a92e3f32ae,0x00055d0bc6d91fc6,0x00092d1d7bd54520,0x00033424b80c021f,0x0007af5e458c62a7,0x000bfde586e1d546,0x00052c6922574f70,0x00000000000d19f0}}, + {{0x00077ca19b6937db,0x000c3bf87f30d9c2,0x000f3d29bdf61e62,0x00032bd50ec0ba46,0x000ae60d1e72d5fd,0x0004ff2d37de8fe0,0x0007e452d77fe0e1,0x0000000000055ca4}, {0x000fd784e6c9781f,0x0000339e3d19e6d1,0x00045fa26fa441a0,0x0000c7afe92c6d07,0x000869a829043bd7,0x0006d1d9ea3fbc06,0x00024be263f00ed9,0x0000000000075c0c}}, + {{0x0002619f4e7dcc16,0x00090a1d1a8f0893,0x000f4151f0e91898,0x000c39b84ee41fc5,0x0002e4a019099db2,0x000a66c12ad292a0,0x0000f6ee6b26ddfc,0x00000000000f04f7}, {0x000e18e3a8161ac5,0x0007f01e66788b92,0x00099db080ad62ab,0x000e4542595d5a6c,0x000bdce965cc3e7c,0x000ec7931f894a64,0x0005be12e46952d9,0x00000000000fb65b}}, + {{0x000d91fc53fa82b7,0x0007d705bfe3760e,0x0006eb6118c41a96,0x00077a9bc4567977,0x000617081a2f811e,0x000e1640e4f52c4e,0x0001d787405d819f,0x0000000000070771}, {0x00085581b6dc5925,0x00062dc6fff82580,0x000af03b55b43c90,0x00066bea9cf3eae3,0x000e87139b8b0ecb,0x00042b4d77418372,0x00032e6aaccf295e,0x0000000000031fc9}}, +}, +{/* digit=34 [{1,2,3,..,}]*([2^136]*G) */ + {{0x000f65dda78f831c,0x00073fa3b4c8e10f,0x0006b8117d5cb12e,0x00039562e8c4d8cc,0x0005cf89935b06aa,0x000fa071e4981c3e,0x0003bebdbd0c4745,0x0000000000049607}, {0x00004bb91c157448,0x0007009a7298688d,0x000867eb798cb22b,0x000b5b3988f2781c,0x000d73b1719d9a64,0x000d2e8076ac9440,0x00042b58ad54c7e3,0x000000000008f067}}, + {{0x000489635dd868b5,0x000ef339521774bb,0x0001f606d4b5774d,0x000d61e0a285bd5e,0x000e71171b1c1084,0x0009b29f93935a84,0x0009bd8ac2433cf2,0x000000000006dd1e}, {0x000f576ac5f0cc26,0x000f788da0477c71,0x0007313f812b64cc,0x000a25f9d5b19e1d,0x000b6a27fa79a792,0x000a16a8e9ee015c,0x000e67ea3bf8b57b,0x00000000000c15fd}}, + {{0x000f3b091fcd53e3,0x000c537f50e43695,0x00003782e79d52fb,0x000af85e1d111511,0x000f6785ae1c3916,0x0006ada8cf56e852,0x0005b2bf8c72adae,0x00000000000e1328}, {0x000cbe0d0bda153b,0x000e35327920cb17,0x000e7daa3650306d,0x00028573caf37928,0x000b550ffe1ca713,0x0005d4e4fb15ab34,0x000f9d818980666a,0x000000000002f854}}, + {{0x0005d9265089916f,0x000013b623150146,0x0007ede4f0fb2f49,0x000c31c56471100e,0x000b1bae8d2a4bc2,0x000f1f76a4a0e73b,0x0006cabfc0770a8d,0x00000000000a7bb1}, {0x00039cb13d9c4b7d,0x000f42afe5b1cb58,0x0008a2ce9b2dade4,0x000a333a1af2a824,0x0005dafed6cd97ca,0x000c3b2e393cb92c,0x00095609553e7e92,0x0000000000061af2}}, + {{0x000ea55edca2a058,0x0003f5559dd3109c,0x000f7ba60d37c6c8,0x0000fabcaf57d0dc,0x000578742ab60d84,0x0009c1c8e9625866,0x000167c85482ad40,0x00000000000adaa6}, {0x000208d39ae67d2a,0x000dcec26ad9971a,0x00067a35ccc54689,0x000a7fc2539986bb,0x0006a23dee41340a,0x0001a9837d767487,0x000da9a948a9292e,0x0000000000071438}}, + {{0x000778a60c6bd7ad,0x0007f39038863eb8,0x000e85efa03ef35f,0x000682ef3d8310f4,0x0008f412315338aa,0x00026310bb52d41e,0x0006924fbef3dd70,0x00000000000f6ff5}, {0x000ba50eddf2b7fe,0x000b63831c6b1cbc,0x0003bc6684c6c5e0,0x0006abc6516bba59,0x0005446d35a876d2,0x00012a8d0c237f9a,0x00015bb2b16c0ff0,0x000000000000ee03}}, + {{0x00014182dba86bd2,0x0009c817d77b02b0,0x0006420950654aca,0x0004107da68b7691,0x000dbebc4c4dd3ac,0x0003d39e96904bcd,0x000950b0d2103ca5,0x0000000000030a5e}, {0x00028ff31a9f909b,0x000c7b092568034b,0x000262a60542e8eb,0x000ab34c15855ae5,0x00063017194f2389,0x00046b838c14dfd9,0x0006fc420e071911,0x000000000008fb4b}}, + {{0x0009c78614d38ab4,0x000d813722ab0651,0x00088626a03970e0,0x000f467f76fdaf74,0x000912ddfd9ad3d0,0x000bdefd37ce072f,0x000315ce918a5747,0x00000000000750e5}, {0x000fa00e65975639,0x000cd08bbb20dcda,0x000822e7b86b49be,0x0005c21ca865ba6a,0x00002e6e8e6fc8fb,0x000608b60ee6e41e,0x0005cdd00ae6214c,0x000000000006ff68}}, +}, +{/* digit=35 [{1,2,3,..,}]*([2^140]*G) */ + {{0x000cd00e20fc1fe8,0x0002bbbccce39826,0x0009c6cbc7ade77c,0x00035a5d2252fdaf,0x000954e7dd499eae,0x0005100fda8f4f20,0x000727a56d72a629,0x0000000000056767}, {0x000898f026420cbb,0x000adbf60b247e57,0x000b35db15577b1e,0x0007ad4b93dab9cc,0x000022d71f39c2a6,0x000304db218cd0ed,0x00082104380c425f,0x000000000006729c}}, + {{0x0000345995afd46f,0x0009dca07923b790,0x000129149d2f3565,0x00079e83cb025114,0x0005beb383ef41e3,0x00076dd0dfabac00,0x00012724d12d9a10,0x00000000000b208f}, {0x000d58cd6475c579,0x000359cc38a604c8,0x000857e410d47fbc,0x00060d98ac219eff,0x000faf284c806f63,0x0009e366a1edaab3,0x0007269c3b528101,0x00000000000bc9e9}}, + {{0x000a386526319283,0x0008d26bd07d697c,0x0006e6a9c305333a,0x000c5466798b96c3,0x00081d3f3859d831,0x0001b9ec71d6b410,0x0001a9501d38ec99,0x00000000000d7843}, {0x0008f62f7d623e0d,0x000dd8b85baf6942,0x0003f90bead64135,0x000c1107acdcf58c,0x000c848d5842efc7,0x000c7fdacd9af415,0x00019b6b5a06bc0a,0x00000000000a3443}}, + {{0x000517cc17c08a8e,0x000a28410d82975c,0x0000ae8bc362b8a4,0x0000d0d18c253486,0x00007eb035a3ae46,0x0001144145d0279a,0x000f7987e7c1289a,0x00000000000c0744}, {0x000ccad1ad801112,0x00072b7b2b4f054d,0x0007f502703051c0,0x0007395b51ee6864,0x000bf6122422124a,0x000f1a7fff2937c4,0x00032eb4ec133207,0x00000000000f8860}}, + {{0x000daf15d19f8632,0x000794c0d78053af,0x000ade8ac0a0ca73,0x000c453e4b57e236,0x0005f4172b285217,0x000f999f8b4669e3,0x000d41509a3cd049,0x0000000000087c69}, {0x000ec5ba18211916,0x00032e14d01e8346,0x000c499a14031eb7,0x000bff270dc2bf04,0x000bc01864000e17,0x000a4dd3446560b7,0x000d06e28c7b9c49,0x000000000000f325}}, + {{0x000161555383d319,0x000d0dd4eb3d0283,0x000dca801a8bb250,0x000285ddc1973c7e,0x000aa0046b981200,0x00019a7fcfd342be,0x000fe5d191734912,0x00000000000b7caf}, {0x000a5ee9f805422f,0x00057ea5a0c4d360,0x0004c8206c9d4abd,0x00016dc6046eeeb1,0x000ecc8e64b15b2a,0x0007fadda1755e1f,0x000ec251e4946e9e,0x00000000000fcbf4}}, + {{0x00024bd4d62de244,0x0001d46d7088801b,0x0002ae4b99b01f02,0x0008fca4fbea2725,0x0005bd80a9dfe494,0x000871c717d6c776,0x0005701b470ea6f3,0x000000000008330a}, {0x0004db08d904e89f,0x0009bdaecddc0ed5,0x000f26cdb2e08463,0x000cd84caeeef145,0x000fc685a35656c7,0x000d93ddcc60c910,0x000895e68bc5c59d,0x00000000000feb64}}, + {{0x0001bb5474d7445b,0x0008af877e8c6483,0x000948d44b23fa45,0x000269f2b004cfe9,0x000795450f8b19ff,0x000a7d4588adc5d7,0x000fdc688ce8bce2,0x0000000000097bd7}, {0x00043a2684d27187,0x0007c1b9f4ad5bd1,0x000b255e8d74e0bd,0x0001f7e71d83d86e,0x000d25ffc219abee,0x00073f553e693c76,0x000a551c9a84afc8,0x0000000000066d77}}, +}, +{/* digit=36 [{1,2,3,..,}]*([2^144]*G) */ + {{0x000e4a9a609d4f93,0x000f05584cbb3289,0x0002a9b59e61225b,0x000d8267df2d43de,0x00000109e8014126,0x000172f1cdd5bbbf,0x0000d985b92ee338,0x00000000000f3143}, {0x000ac2d50dd03701,0x000b11e059a07dd0,0x000eb68a6d1ce296,0x0000751560f20e77,0x000e7aaf3a9ad622,0x000bae14ea59489a,0x0002497b70e2f664,0x0000000000076d08}}, + {{0x0001e7720f69ad57,0x0004baff47822226,0x00011f4e0e8fa5c8,0x0005f8d2ef696aa0,0x000b4fa94fcabeb8,0x000d41a438ce5baa,0x0007720a32f96200,0x0000000000074f6e}, {0x0005b3ae79f59da9,0x000a8be0221aef2d,0x0005793443a4452f,0x00085d6a7e49f3a4,0x0009cb5d3c6378ff,0x000eb65f56300658,0x000bdfe8e4383596,0x00000000000ff8ad}}, + {{0x000ec83ab52a5d83,0x0006cf6fd26cb934,0x000ee1af72ab19a2,0x000788be372817f8,0x000ae8c90c31694a,0x000aa0ce585f3dc1,0x0002238c90c6bf15,0x00000000000f01bb}, {0x00043ef688d7f41f,0x00075bdae01dd56c,0x000cf79ad5ecb5f0,0x000ff4ec2548f251,0x0008d12802a750dd,0x0004d924054a4986,0x000d1acd922fb640,0x00000000000957f6}}, + {{0x0006a17488184bb0,0x0004d082ea61c88d,0x0004b5a68e9c5821,0x000df962e54f5d00,0x0006e39dc6ab7d33,0x000179b13340b0d3,0x00088cd97a8b848d,0x00000000000288d3}, {0x000bcf17110adf56,0x000462237e507d00,0x000fa28fb932260c,0x0008bcb42be74594,0x0004de2176b6645f,0x000e2f09cce2b0f5,0x000001af570a09d1,0x0000000000057fdc}}, + {{0x00009414087acee6,0x000fc7dd81467f3f,0x0007997c546fe735,0x0004e832d2e9fbfb,0x00061c9d9c7ca2ae,0x000f27f140f127f6,0x00029ef06c5a57e6,0x000000000001f303}, {0x000b431a1206d910,0x00037a1b2b82f82b,0x0007312610c2e220,0x000cb8b039f12bbb,0x000902b92d6b1cf0,0x000f649e606ce433,0x0006eccb5e826869,0x00000000000e9309}}, + {{0x000f672480aedb22,0x000c79d21c740037,0x0001a34fe6cb5f73,0x000b1a3aae5d701b,0x000891978d1557f8,0x000b85b1e477e4f8,0x000fb42913ffb40f,0x0000000000058489}, {0x00047efe2cde596c,0x000a60e1ab266dc3,0x00073cc3adaf7198,0x000ccb657ab8d871,0x0006547f64bdf578,0x0003e497e339fd79,0x000706904693943f,0x000000000001d864}}, + {{0x0009da8454e1979d,0x00030da92e482d7a,0x000b666249139a91,0x000ebb3639c78714,0x0009d77aac3c8763,0x0002769a43bf7177,0x000eda92a5420d3f,0x00000000000d092e}, {0x000352bd3a0254a9,0x000b5d7bd8d8f8e8,0x0009446f6266c164,0x000fa3ed6b0cf872,0x00040490a771a5ea,0x000ff37225255afc,0x000cdbd40bb0fd15,0x0000000000079294}}, + {{0x0005908307814aa6,0x00009158bfe27b53,0x000df35ce707d624,0x000153b8e71aca19,0x000171b11643f07b,0x0004e2f905e67698,0x00010977b7dd7dd0,0x00000000000f0dcf}, {0x000bd23edf0138e0,0x00071d3bab2a41f1,0x000facf3129d6b1f,0x00099afc6f53cda2,0x000f628039e454fe,0x00083aa16071f2a4,0x0006f9f1f44181b1,0x000000000005010f}}, +}, +{/* digit=37 [{1,2,3,..,}]*([2^148]*G) */ + {{0x000b28d4d03ca18c,0x000a0b1e36500cc3,0x000e44ebafd13e2f,0x0001d2ca4bfdcedc,0x000fb29cdcfebf45,0x000514871d210892,0x0001e35b12bcd894,0x000000000001809b}, {0x000c86d0fbcc5e1a,0x000ce09b243105c8,0x00037f6b6be14182,0x000a16b5de334b63,0x00020b116076cef0,0x000ae2bf4fe0bd1e,0x00093754e4b48289,0x0000000000068c8a}}, + {{0x0007e2c503fdc14e,0x00049309b1eb33af,0x000b5e3acfac1d05,0x000b81ebf8a894ca,0x000e0d29329387c2,0x000bf371427a40bc,0x0008957f4c315be4,0x0000000000001236}, {0x0002d22bfd1d81d3,0x000f4319c88f7e2d,0x000189361ce75d22,0x000a05df0dd13811,0x00024acba9fafcc2,0x00027313ec8d55b0,0x00094a871358de59,0x0000000000017ce2}}, + {{0x0002c99cf8ff48b8,0x0002410777188c8a,0x00068c821fc35883,0x000b515120380e77,0x000fe13577d1261a,0x000862db1453c858,0x000e1cb1f6bb58f3,0x00000000000b9529}, {0x000338bb0bed7b45,0x0008003f63a416f7,0x00081208dbc793e6,0x000048b756e5af2e,0x00031c984bef8423,0x000a00f3e4d978ed,0x0009e7a06242995b,0x000000000004f4d1}}, + {{0x000e08308825a639,0x000e8d3797831773,0x000f79843c567224,0x000a6a611a4e33a2,0x00005328043a2ff7,0x000e9f7dd904f86f,0x000904e29d31c012,0x00000000000c0a51}, {0x0002dae695c951e1,0x00024070c2696563,0x00060638255bc0fb,0x0000d0c12c8a1f65,0x0001ab352a835b97,0x0005eb0c7572aaaa,0x00059963df45a90c,0x00000000000d4977}}, + {{0x000fe53cf0bc7d5d,0x00073e35daffd3d8,0x00005ab0f149d24a,0x0005a40f009a48b1,0x000d308a81c693f0,0x000885426a4a801f,0x000f0575e5dc467e,0x000000000004629f}, {0x000c6c457ceba67d,0x000c6356b879b5ee,0x00084a5c7d7e4e3e,0x000856eec2dd55b5,0x000880f0c80411c9,0x00001b21720f0443,0x0007b8c0b5e3e218,0x000000000006ea09}}, + {{0x00042a359a9c02d9,0x0006601c2df8ca11,0x0001ea46897d0b3b,0x000341c8360fa6b2,0x0002c52bb2d6e198,0x0002efba5e67f809,0x00032af944dc63a0,0x000000000002c123}, {0x000d5d58228e0e7d,0x000b239684f6c863,0x000f4f910b494aa3,0x0008eb646594725d,0x0005793c32ddb7fb,0x000f94b55bb4f5f0,0x0002773ef3c33845,0x0000000000009eac}}, + {{0x000bc0df5aa0ebff,0x00046efd26dca17d,0x000d31eadaba6e9d,0x0008a89e1830a96b,0x00013e039a029f10,0x0006fbb3b7e8e368,0x0002f11747b3e925,0x00000000000abb3b}, {0x00023cb577b95e94,0x000e2cac5818c280,0x000b36c4a5c24e15,0x000fd4d7a5485367,0x0009aa3645074081,0x0001a5f81fe2d8e7,0x000db4e86ce00ea8,0x0000000000077a9b}}, + {{0x000a3bf39d563e4f,0x000f02c5b0e421f4,0x000bae31a917643d,0x00085959aa907285,0x000af658699bace4,0x0003b18e632be886,0x000667ce75d6c6da,0x0000000000069caf}, {0x000af371b713c401,0x000f0c17c66ce4f4,0x0002f4e783050dba,0x0000041623db4f0b,0x0002c74762e1ceb8,0x00071c52fe75615b,0x0002ecade8a54386,0x00000000000cacaf}}, +}, +{/* digit=38 [{1,2,3,..,}]*([2^152]*G) */ + {{0x000ea5d38989c046,0x000c5a933aaf71af,0x00084b51a5d47afc,0x000dff8854de4972,0x000b247bec1525a9,0x00061e58da8b31d9,0x000707468a25c846,0x00000000000786a0}, {0x000d126f8d197fbf,0x000be282db8ca2e4,0x0000d8a3ccd2e3a9,0x0000faaeaeda06f0,0x0009add94b47a2c4,0x0000690766963292,0x00092cc72354f6b0,0x000000000007878e}}, + {{0x000456ff7fbc201b,0x00083854b0583e19,0x0005b9f9d05986b8,0x00093894c32fc71b,0x000c9ec8f90dec82,0x000c7b9c0882fad4,0x0005e52d39990dc6,0x00000000000d71a2}, {0x0003262dabc3b450,0x000866b852e64a5a,0x000281968ae95022,0x00011545857f0497,0x000c1ccb9bc83700,0x0002bb853746621f,0x0004ed97e44e6361,0x00000000000758da}}, + {{0x000e66dca895aedb,0x0007e58775856e71,0x0003edbdf1471e8b,0x0002e3da62265d35,0x000672d98b0886a1,0x0001e9c858ec4278,0x0004ceb9da8016f6,0x0000000000080099}, {0x0007a34c46f751da,0x0004f63d0878c9c9,0x00077928ee65f2b7,0x0009e126a1c1efae,0x000eebd0497a780b,0x00065231eeed68d0,0x000127bfeee3d292,0x0000000000080e03}}, + {{0x000369b381ff008e,0x00068d25f6507829,0x000435b503d33f46,0x000031108b9b08bf,0x000f3fe92e910b36,0x000189ddc16477af,0x00038f6e5f6cb103,0x00000000000c698a}, {0x000953f049518733,0x000e102a092187fe,0x000b4e74068daf16,0x0005eac8fd6b76cb,0x000611ef96b455a1,0x000e433b51e37ec3,0x0004c3d1b3b3fc30,0x0000000000051d17}}, + {{0x000c3efd55e9f108,0x0004ee67813dd55d,0x000e8b95d557829a,0x0007b634c8cf1647,0x000fa3556b4674ea,0x000db03dff1bde0b,0x0006b45343f260c1,0x00000000000e0a1d}, {0x000557bdaa85b25c,0x000b5af56bed0543,0x0009694c640e2d2a,0x000c5892c72fa801,0x0006a49486e504e0,0x000f78943812e259,0x000431d5e0bddb2e,0x00000000000acdba}}, + {{0x0005b2c1b41a7e30,0x000e8b6e95494a98,0x000e156b8fa7f1c3,0x00012ee2183ce113,0x0000cc01d1434741,0x000d0d25f4180ec4,0x00016ddc5f8f7b8d,0x00000000000974a6}, {0x00054a8b6ee62d01,0x00072b9a7f0a96a9,0x0007a0f1f81abc8f,0x0000b82bc5671a8c,0x00000466ffaf50eb,0x000fdc348fa58667,0x000299a75ff5aab9,0x000000000007f784}}, + {{0x00064348b9a55592,0x0008f24b18ccf351,0x00046d73b67eefc8,0x000977c532d340c4,0x000191002448043a,0x000960397a2de526,0x00034e1e11027870,0x000000000000164e}, {0x00056330f30da4d6,0x00014cc5f57288f2,0x000974f0c19f8ace,0x00020963266aaedd,0x0002c3ccd59f3b15,0x000a6dd5dfaea30f,0x00035a1da2e1fbc9,0x00000000000ceda8}}, + {{0x0007996f48325ebf,0x0004f7914213d709,0x000270acf84435ed,0x000e5a126a34238c,0x0000701ce8c76eda,0x000656e02e566bf5,0x000fbf3562e87555,0x00000000000b4e8e}, {0x0008d6657de7885e,0x0004a5f10a503e86,0x0006a10baa0b8f13,0x000fc25cc2f2e415,0x000caf5718c149d1,0x000d6b890e973bb1,0x000f129b8825dad2,0x00000000000e2f00}}, +}, +{/* digit=39 [{1,2,3,..,}]*([2^156]*G) */ + {{0x0009cd8cd3edcb0c,0x00022e37211bdab0,0x000bfe0383f52218,0x0007e26a1b9f8b57,0x000d7d7f72d5fdcd,0x00049c9205641e45,0x0002a15377c1bec4,0x00000000000efc7b}, {0x0007344da1d40eaf,0x000b6c657a9ff3f7,0x000acc6777a25729,0x000d1bcd020eaa96,0x000a3f6860c76bfc,0x000c7c80617534b0,0x00056b8ce5722284,0x000000000002bd74}}, + {{0x00070f4fca1b2907,0x000d2aed02b6a844,0x00048854f708389d,0x0006ec4654e7c314,0x0003e7034bfd8222,0x000d6b555008ac00,0x000d44e343c5407f,0x000000000001b429}, {0x0003fda90bb8f0f9,0x000bce0702a33908,0x0003c08ba27edf85,0x00072f6d46524015,0x000d35f600437e6d,0x0003e8cc4d92655e,0x000b40a0dbaac627,0x000000000003c3ff}}, + {{0x00098d832fcb2cab,0x00078bc06ecab0e1,0x0004e7cd0ece1448,0x0006fa0453c94bf2,0x0003ed1a6731a6fc,0x000c3f1fb5460f94,0x000d4eeb11656a4e,0x00000000000ff1e7}, {0x000efd2eb43b2558,0x0009059526466dba,0x0007bc3cfc024713,0x000588824fd2ce63,0x00039f2c29257bdf,0x000e97f3013df0c8,0x0006411bf621e6de,0x00000000000ebea6}}, + {{0x0003b015d907e90a,0x0004e5906a35b43f,0x000f32388b4d1260,0x000ac9136f847648,0x000bbe0f1dd365c8,0x000d26c21f73b3de,0x000ae740358868a2,0x0000000000076792}, {0x000fc16ec80c7bf1,0x0008d3ecea1669e7,0x000b1ce4e42f6130,0x00091090e0062443,0x0006dd94681b6db8,0x0005a3de2d29106e,0x0006ccb40b8694a8,0x00000000000936b8}}, + {{0x000fa93eb46c6ed1,0x0009f28d33e792c1,0x000af8e666ab1b38,0x0009f3bce683c5c2,0x00098371fe755a74,0x000712c1d717629d,0x0001aa5e828fc057,0x000000000007e4c6}, {0x00082c4505e4cd17,0x000035d927bad55e,0x000bbbc997dd1436,0x00099a398591dc25,0x000a4836664c560a,0x000d79298c885fe8,0x0001d7d18acd4226,0x00000000000185df}}, + {{0x000755a507c76d63,0x000c69d8a925b591,0x000ef5ac5d730610,0x000ca6ddfb534b8b,0x000c6dd78a324f53,0x000a146d54e64874,0x000201336e5b46c2,0x0000000000098395}, {0x0001a5ef82624226,0x0004ca4c095220d8,0x000ae0a7c3b4840e,0x0002f64c36286ed0,0x0003c3ebb08c0ff9,0x000f00c3057b1b90,0x00036ec6bdc9b665,0x000000000009a46d}}, + {{0x000c3639d049078f,0x00048d67dc92ab51,0x000e52783edc1242,0x0009baa8b87c0b05,0x000052760ef4b6f0,0x00047bf855a8a903,0x000e742e2ae75610,0x0000000000085bd4}, {0x00097e1bd11078ed,0x00093ced11ff7661,0x000e3244d9aa20e1,0x00088ff24b3e912a,0x000a1afd219683d4,0x0000d21286e166fd,0x00039c66a912114f,0x0000000000005c9f}}, + {{0x00070dd4c4f3a35e,0x0006d913f92069a9,0x000aaa1c8b2107ab,0x0004e2c787c35959,0x000fe7b7b7ddefab,0x000e28e6aa55d465,0x0007bc921aaad834,0x0000000000012d6a}, {0x000d81dd493823d1,0x00072109803c417d,0x00095b2d5e30421d,0x0008bdb99c5bb670,0x0006bc7c2da71a8c,0x000927eef1cd1c2b,0x00041050189c975f,0x00000000000229f9}}, +}, +{/* digit=40 [{1,2,3,..,}]*([2^160]*G) */ + {{0x00086c2a1c05c1ca,0x000a911a8fde5d4c,0x0008768c091692e8,0x000c275c74dfe82d,0x000c38373a506818,0x000e5f88f2b0294a,0x00083a584c4061e8,0x0000000000073423}, {0x0003f61270e03ada,0x0002d263895b3203,0x00007b3b0d15f74f,0x00059da84da7f0d6,0x000a924a09f21443,0x0009ad83576e3095,0x0002af612986e3d6,0x0000000000039212}}, + {{0x000c342a3198c068,0x0005d3ee0a31c35b,0x0003fc2cd3c80f60,0x0004f4ff0fbfa963,0x000efafea65a90a6,0x000f4b9f9513f054,0x000590796ba7479c,0x00000000000932a9}, {0x00037547e06b6b4e,0x0005d93af8a64743,0x000922627b827de1,0x00071e9c1a46c909,0x000f4c9bca7223bc,0x0008cea30000643a,0x000ec2b6d967c784,0x0000000000073312}}, + {{0x000ecb2626cc0655,0x000eafeb707ee40d,0x0001edb8dca9d02f,0x00016459a0d32bae,0x000a1ffb926626de,0x000578b4a9a2ff38,0x0004434e5ad96d8f,0x00000000000883e7}, {0x00041045bc252635,0x0002d1d8eecaa72c,0x000edd696d444309,0x0002faed758ae41f,0x00004a85d867e49b,0x0008cba866a9a229,0x0009822fb89dcee6,0x000000000007f09a}}, + {{0x0009f15eb1bbb5c7,0x000d952ec99556bc,0x00015795c8a4dc35,0x000337c6dcc7816c,0x0009791f7cf1e881,0x000885a42e4e7b6d,0x000e41faa717aa59,0x00000000000f9c01}, {0x000171c4ffe4a6bc,0x0000ccf208d57a05,0x00042714fd20944a,0x000871b264ce04b2,0x0006cd42026a7261,0x0009f99fe66be4a8,0x000e2bc5397b7782,0x0000000000024578}}, + {{0x000b87e03bf622cc,0x0002fadc79436506,0x0000be1fdf9a8888,0x000aaa40406c1296,0x000c8b658d3cef9f,0x000435baff4e6388,0x000c997262beb41e,0x00000000000fdaea}, {0x000ba84e0b8f458e,0x00045d359f7d8428,0x00016b951ae31b5d,0x000f03229498ba51,0x000a85bf35adb18c,0x000aa4bc8c67388c,0x00027a8a7a6aa262,0x0000000000072f46}}, + {{0x0001e4a772f0f152,0x000520733667a853,0x000ae90cf08b2488,0x0009b27a7bc26604,0x0008eb4f0c7a7ca2,0x000f276309fefa69,0x0001337b6f351301,0x00000000000fcfb1}, {0x00022648d86fc4cc,0x000979928d9cf841,0x0009da8838c5ffbb,0x0001acbe041bef47,0x000af8d998547fbd,0x0005a0dc57d25159,0x0009332ec3a7d8db,0x0000000000087e3e}}, + {{0x00088fc716a5b4c1,0x000423fb812eeafe,0x000a6b9f65779527,0x00041838fbefbe46,0x0004a4b24a05e572,0x0000c0a432b7d49f,0x0001db23b138d071,0x00000000000a85ec}, {0x00095de9d8768d19,0x0006d7cc1f3d657e,0x000b6e219733e2a8,0x00023128f56981d2,0x000eb4080a011bda,0x000999b7f68663ea,0x00080c8dd7ba7e9f,0x00000000000b6865}}, + {{0x000b2ea58bc5b2ba,0x00008ba8418f2c05,0x000ab2d74f8719f3,0x00038d7bce1e82e0,0x000eb397915b4e64,0x0004cf6599e489b3,0x000affa9baea9ffb,0x0000000000042ef4}, {0x0001f2cfd470aeb9,0x000eeae3fb532a9f,0x0001d5334d1ddf3c,0x0008e28defe047ca,0x0009d24e60fe8972,0x0000f710c63a8c67,0x000d3cadacbf9247,0x000000000005e198}}, +}, +{/* digit=41 [{1,2,3,..,}]*([2^164]*G) */ + {{0x0008597cdef10944,0x000f4294d29542ec,0x00058a58394c4343,0x0007f9158f95038b,0x000164420c94fa1a,0x0003caea8b137981,0x0003402b686e1e09,0x00000000000f9e1c}, {0x00006da6276d2e4f,0x000cacd1beecf39b,0x000bd69a9fc92254,0x000c7192dfc2b165,0x000dcee7e854cecb,0x000635cc82cdb955,0x0007d39fefc321f2,0x000000000002936f}}, + {{0x000f165eb19500c6,0x000f54962c020b57,0x000116eb855d7c76,0x0009a79dd189c401,0x000d6ce517a77854,0x000d6bb9747675cd,0x0000295102294e32,0x000000000006ed1a}, {0x000926ef8530f97d,0x000353efb54e82c3,0x0002eec5a292f22f,0x000c2948967fb050,0x0009be04e909955b,0x0001efad6373615c,0x00097af1fcf82011,0x000000000006fd2e}}, + {{0x000445389cf3da63,0x0002b85e1e4ab383,0x0004d8f24163478b,0x0004da1a717e0356,0x00080f1bf3dfc0b8,0x000a6fd41f1a63b2,0x000cab47b74201ab,0x000000000003518f}, {0x000a17bf41f156b8,0x00031e12511fae1c,0x0005a421fc7a58f4,0x000fb10db5665c05,0x0003b99f4d5a20e0,0x000b7dbceb5448a3,0x000b87cb1847ad46,0x00000000000fdcad}}, + {{0x00040c75d27c479b,0x000823f2b5bd3e20,0x000298ef45cbe8a6,0x000f2db94256fe1f,0x0009318ddb03532e,0x00024c8e1acbfc45,0x0003db2375f9fd5f,0x0000000000037078}, {0x0002335b7531e78b,0x000251bd461e7e12,0x000e223c32f4b08c,0x000777d8845f315e,0x000697fc92c7c9f8,0x00092954081aed29,0x000fe09f2f8d7949,0x00000000000ff5eb}}, + {{0x0004fbb34790cc07,0x000a4397049f47ff,0x00000e7e84e96f9b,0x0005ec1e7862c0cf,0x000834350bf1ce6d,0x000ec8d7417a98db,0x000aa86ceccd8030,0x00000000000bcab4}, {0x00032ba3b5c44605,0x000c04378b1fdff3,0x0002189e90f61242,0x0000faa8d60f86df,0x000a3573271abddf,0x0008f8032987583d,0x0003f6b0afe4ec4b,0x00000000000b69d0}}, + {{0x000f8df0532bb051,0x000c5bd9a66d6010,0x000c7470f77e12e7,0x00051dfef38e9b37,0x0005754fb3fba751,0x000069b72a3348ae,0x000faf1218fa8f13,0x000000000000835d}, {0x0007a4c220543f04,0x000c02121a98ebed,0x0005ec4ceabea5d8,0x000a4be1a1eb6eea,0x0002adeb3ae51b23,0x000ea2b45b5c48b9,0x0008eaeebd305ded,0x00000000000c3719}}, + {{0x00055d2a529f419b,0x00062f73471b59c9,0x0003ecf010a8f9e8,0x000497dae082cef5,0x0005c3e563bc57cb,0x000125f95dfcbf2b,0x00001922149c0fd1,0x00000000000425bc}, {0x000c9ac134356e8b,0x0000e049476de7aa,0x0008c62ed440f124,0x000c62d1256424f6,0x000541bc66fa56c5,0x0006ada140a118ee,0x0008c50d8e9ff829,0x000000000000134f}}, + {{0x000b2c8bea306a01,0x000b1d0960bd7257,0x0002ca4efc9832f7,0x000cc77a98509175,0x0001e5f81554975c,0x000cff2c8b41bd53,0x000be3bdf8ab57c8,0x00000000000f5822}, {0x000ad597c5610627,0x000ae1f0ac0e6f99,0x000855b71125b2a5,0x0008b82e3cc86ac3,0x000d2b2beaee22bd,0x0002617ffc43bde8,0x0000968168781e41,0x00000000000b7ee7}}, +}, +{/* digit=42 [{1,2,3,..,}]*([2^168]*G) */ + {{0x000956a7f54f949d,0x00074086945790c5,0x000771ce236fcb82,0x000b0c6000575064,0x00080b09adeb2c04,0x00081474f468be6e,0x000cce2f3bf32b6f,0x00000000000a712c}, {0x000c24bf8b416a6b,0x0004c292cee41c19,0x0008e149d7276386,0x000a66156f47b2ac,0x0005840b5d1be54d,0x0006e8cf62ca7683,0x00053eb52adb6a8a,0x00000000000dade1}}, + {{0x00026c98ae834331,0x000b365f7d2c0d6e,0x0004cfcca31f7dbe,0x000634c986436f32,0x000133356165e268,0x000c7957d6334d8d,0x000983f17164269e,0x00000000000b8093}, {0x0008676a1037657d,0x000d9edb1fe5cc2a,0x0001cd1de0787da3,0x0005f4da691b6657,0x0001e1e2727e746e,0x00027b0296117129,0x000197bb4aa8f16f,0x000000000007f42c}}, + {{0x000a6e4b3ca52862,0x000cf2af8bdfa5a6,0x000675c2d00d6d96,0x000d5ced7046d2e5,0x0007d545fd33d57f,0x00061ffd75ea025f,0x000242e2ccb6f431,0x0000000000009406}, {0x000194e235777423,0x00019f3536d60805,0x0005fe57dd0b2a05,0x000b06a5cc554450,0x000f9a9e2a66fd15,0x0003dfd0261b0feb,0x00051fdc3c057665,0x000000000000a8ab}}, + {{0x00014e15511a3745,0x00067fe19901abc4,0x0000c6f09a808e87,0x000012556c4ce5cb,0x0005938c89ab92fd,0x000a587b123172d6,0x000c50a71f8a33aa,0x00000000000b55c9}, {0x0009dec34d6b29ab,0x00056ec005f6a241,0x000b67510d45fff0,0x000d67f9e26361fb,0x000321389c2598a4,0x000ffbcee7f0a2b2,0x000888d158820795,0x000000000009f36a}}, + {{0x00065f3227de5d3c,0x0001e5ffec1d7642,0x000dceb00d947f3b,0x000844c649c850b5,0x000610fa337dfbe3,0x00080773d450263e,0x00066c44f7c8f402,0x00000000000e8969}, {0x0009e0325576575b,0x000a42587f475435,0x0002c020daa3c5c5,0x0002f667071543b1,0x000e06577b749e90,0x0000bee1303398aa,0x000cec030926d691,0x00000000000ffa92}}, + {{0x000298023770357c,0x000dae4ee3345cdf,0x000943bb20278bd9,0x000ca667ce118490,0x000dcb69c7ead817,0x000680afeda222cd,0x000ca874ac74709d,0x00000000000d9596}, {0x0002187a0dd3835c,0x00082dff57da7e9a,0x0006d7aa1a2ea94a,0x000ba28b7fbc1b01,0x000c13d4ed0f71c0,0x0003a2ef260faab1,0x000dff6fbe3567ea,0x000000000004e577}}, + {{0x000b9f236346df15,0x000f89b758b50c99,0x00075b638b06df34,0x000b3c39ee88a784,0x0001105e1ec04669,0x00088ea133f36a67,0x000aadd0f30e5d9e,0x00000000000baecb}, {0x000d89dcc4fedb34,0x000d3ec65be650a4,0x00093dd7d659e073,0x0005d29942ce52c4,0x0007d3ce28d4f719,0x0007fc9041220187,0x00055e9c962aa1a9,0x0000000000065c5a}}, + {{0x00095e9750ada30e,0x00094eccb421e7ff,0x000dfa5cb406ba75,0x000da9e05a53972f,0x00007bc99fead344,0x000f77bf53a8035e,0x00078ae0214485c0,0x00000000000e54df}, {0x00053ac771ed9aaf,0x00027def45af5dcd,0x00024afc33d18821,0x0007db1c337e0181,0x0006b84671d5a9b1,0x0007d696d026a4f6,0x00022a606142343b,0x0000000000081cfd}}, +}, +{/* digit=43 [{1,2,3,..,}]*([2^172]*G) */ + {{0x000d2a53b2756567,0x000f3feb984b33c0,0x0004e95d9895327d,0x000f97e3ca0a9a03,0x000dacc000ac177e,0x00040f51a76d4796,0x000810bad0fa6eb1,0x0000000000006ebc}, {0x000c76db6b103c54,0x00004f89eb78f367,0x000ad3bb031162c2,0x0009cfbb25e9d1c3,0x000fb4e7aef3e2b2,0x00067e8459388ea8,0x0006d7ee606c12e2,0x000000000009f580}}, + {{0x0008f59540514451,0x00028cf947c06046,0x00062665be2e4fbd,0x00015c05f1835ca3,0x0002d8c79d90d001,0x000b5f791df08415,0x0008cdc0c3a6846d,0x00000000000749b1}, {0x0000ce2ed12d25c3,0x000d8d314b7d4a22,0x000dfaf9b4508139,0x0005af06855a9438,0x000ee9d02ab997ed,0x00038a5d0ea84ae9,0x000eabb87e903432,0x00000000000650a0}}, + {{0x000c8f034ea36274,0x00088357540ab419,0x0000f5e32c760e57,0x00073ac14d7fed37,0x000fd186ee3d33d2,0x000a7b1d9fef6b9f,0x000bc2a94c207101,0x0000000000050bc8}, {0x00013f9560f76983,0x000e5c68959c999a,0x000056367228b52b,0x000a4fce6861310f,0x0000c828330f7a73,0x000f74fdb19bcac9,0x0002f473e3b66f7e,0x0000000000052d8f}}, + {{0x0008b2d06a3248b0,0x0008e58225208828,0x00095368e614a61f,0x000e45562fe86dbb,0x00082dc5b2f11224,0x000096555a8c8a47,0x0004528a957b8dc3,0x00000000000b1591}, {0x00063a60dfca11b8,0x000161e563c57eb9,0x00024a9e6e7bbb16,0x0002de99462e7a31,0x0001c2489214f8f6,0x00074639c6b3dfe9,0x0004b56c1d8fc421,0x00000000000ef88d}}, + {{0x00071f1bee09bff3,0x0004a6eda384edeb,0x0001d457a27a9c42,0x000e6af10e01b58c,0x000ae1175185c441,0x0000dfcc6ac962bc,0x0001b472d34d676f,0x00000000000a9412}, {0x000f38c043747e7d,0x000ad77dea14649d,0x00065b52576e79ef,0x000c09dd5d17db20,0x000db45a6e94625a,0x0001f8caf579edea,0x000b8ea659282a84,0x00000000000c11a7}}, + {{0x000f9bf364d3e9eb,0x000407d0850b3dae,0x000e31cf311bd53b,0x000f1f891cb75575,0x00089d3b6afe68ce,0x0003b46e427a6ba7,0x000059f220f9c1f8,0x00000000000f526e}, {0x0007512e159b9fa5,0x00094edf3cccac66,0x0003e3258b2c0e34,0x000d9e98cc78d3e7,0x000b5fa48469cc09,0x000686001391bc40,0x00073a22bcdcd522,0x000000000007c599}}, + {{0x0002ae0971dbb084,0x000bd9de8555bad1,0x000c9834d42cb891,0x0000285de654b80c,0x0008802be80bf17f,0x000d823bf33998d8,0x0006b64095923d5a,0x00000000000e848f}, {0x000fcca2a1ce91f8,0x00015c6e95017db4,0x00048a3538b7e8ae,0x0001be7a70558759,0x0008441da4770c3f,0x00048d4c2b652671,0x0008deaba3b06f9c,0x00000000000086f1}}, + {{0x000d473962fbc397,0x000b15a4a5ce3b56,0x0003f34c786123b8,0x000388e4efe9a313,0x000aef9d074bd459,0x000ab9cb03a45c11,0x000566f68ab50b93,0x00000000000ecd9a}, {0x0008303c4d104a42,0x000f0bfb79b7f7f4,0x000a0520b1df9755,0x000aa21877390f14,0x0005e314cfb54f31,0x0008eab122de0c64,0x0004f656d904f623,0x00000000000ccb4d}}, +}, +{/* digit=44 [{1,2,3,..,}]*([2^176]*G) */ + {{0x0007aa3a213de299,0x000c93eb83a8707f,0x000eb1f52edb04b6,0x000c77a53abe4e9c,0x000ebb9feb8257b8,0x0006f922e0a14673,0x00026cc0a6cbf7f2,0x00000000000a32e5}, {0x000ec20f9c4de649,0x00049ca417e66df1,0x0009b8c741987bef,0x0000e7a62e135de2,0x000ea7ee82c72bae,0x000776c74962bdca,0x000feb573d7f6ae5,0x000000000006ffbe}}, + {{0x000a70e677b6f831,0x000b9cbc0bb18f0d,0x00037297c3884bb7,0x000b350d38064428,0x000c397a7602f62c,0x000987d82e4d2f1c,0x0009faa54bd48e43,0x000000000004f0a1}, {0x000a1b4997cc48fa,0x00005fa0c9a44d6a,0x00012729fa8ce7f0,0x0002c5b9cbd8d343,0x000813d979568e24,0x000da55fa1671eb0,0x000ce5ccae388f40,0x00000000000f376d}}, + {{0x000f917fe9a26ae0,0x0003ae54d92cd055,0x000e0e9599c7454d,0x0007fd2849f6143a,0x000dacd0202d7c90,0x00092d19abb11dd2,0x000677a4913a701d,0x00000000000cf610}, {0x000f3626f0aa0d9f,0x000a3fff1e1d462d,0x0009edea6495252f,0x000f33a92cabf724,0x000ce329d1adcafc,0x0009a0e4cd571e13,0x000f867626ad237a,0x00000000000130d7}}, + {{0x000f8196d845d3e6,0x0002f8a89daef137,0x0004ece7e9ffa3ea,0x00023775b80bb4b5,0x0002a45b362648d9,0x0005bfe910a587c5,0x0007d7daff503cc7,0x00000000000116d0}, {0x000600a28d48ec00,0x000916b5a471517d,0x0001f4eebe019105,0x0006cfc595abf8dc,0x0009ed0391ce8f07,0x000fcabd3c9de4ce,0x0009edaaaad03ae1,0x0000000000087b19}}, + {{0x000b51557a9fd4ff,0x00034b0641d0941f,0x00049c97e60548cf,0x0005b4b00d5ec6f7,0x00015569d89ad12a,0x000ac72089be1a11,0x000df19c0566deff,0x000000000007034e}, {0x00000817b5f1bb5e,0x000e5636aeb6adf8,0x0006c2164b0bbfac,0x000f898734e9d301,0x00095ccdc6bcf4e2,0x0008a4f28daf7421,0x0002c39d39249f60,0x000000000008820e}}, + {{0x000d9387d2303055,0x000ac620248118e0,0x00080c838dc206ab,0x000eb38d7f033fdf,0x000232646d6f86b3,0x000dae596ee3226b,0x0008e58c4825f6f1,0x00000000000a5bcb}, {0x0000edaef1eabcc8,0x0006f904a53484a2,0x0007fa1deed27103,0x000329f21d45f8ad,0x000a06605546af6a,0x000a93d14f20ad88,0x000174cf7a0e9619,0x0000000000091c97}}, + {{0x000ae272638371ad,0x0005559edd263abb,0x000509e662e63add,0x0000d304e7f07169,0x000119b88200bf4a,0x000da801e36fc0e4,0x0002c791db560240,0x000000000002c72c}, {0x0006387b3ebbf52c,0x0000dfe960c2596c,0x00093166fd89300b,0x00080febccc1576b,0x0005c3c88f475a64,0x0007bd8ec72f4e5e,0x00055e224e7e749a,0x0000000000076314}}, + {{0x00073d61157ae785,0x000ab42c9fad4fc2,0x0000e5d3107f2d93,0x000c7fdb149854b8,0x00002fc359eb0cf4,0x000ec86034a7d900,0x000e149ff0c3ea29,0x000000000009b24e}, {0x000512a0eb94d71b,0x000e0638c80999de,0x000012d24a63feaf,0x000e0f8a07ea5482,0x0002aecdec3f2fb1,0x00074025b1e580d3,0x000164bbf895730f,0x00000000000dd529}}, +}, +{/* digit=45 [{1,2,3,..,}]*([2^180]*G) */ + {{0x00051b4cfaf372e5,0x000b9fa2db519927,0x000d4edc529b8ffb,0x000af1d605917201,0x000e782a09939f88,0x000c710d31c5fea5,0x000365e0fb15884e,0x00000000000d32ce}, {0x00054592dfab29b3,0x000d7d9f896aa46a,0x000ce02cec631ef0,0x000992dca6e73436,0x000a6fd0c08b1b76,0x0000f5c2376338fc,0x00035bc3ea4c65a0,0x00000000000b316b}}, + {{0x0001e6b975664a84,0x000aa14f962e38f4,0x000c4a3248d485a4,0x00011c079e777bf3,0x000b4657c31b2c0e,0x000494de3a1705ef,0x00071802688fd23c,0x00000000000412a8}, {0x000b58909626d6fa,0x00004878fc7ace41,0x000bfc58da0a4094,0x0003c4c704b70c17,0x00087323c0087c81,0x000089f1a98553da,0x0002914f63cec663,0x000000000009655d}}, + {{0x000f1be887dbd011,0x0008384e5541ec05,0x0002bbc7edd0f227,0x0003ad69beaad3d1,0x0000f51c2714e0a4,0x000ea5b0c97dd182,0x0007caac2b4cb457,0x00000000000d1176}, {0x000327be0b446dbc,0x000e318ccf36cb48,0x000fe4f5fd270bff,0x000ce43d8d292a8d,0x0008d58c4ee79811,0x0005c65a772c5fc6,0x0006695bc0f0bed3,0x00000000000cbbf9}}, + {{0x00002ac6c24f6061,0x0003810586ea68e3,0x0007603f1b64b158,0x000c4d7dfbfa352d,0x00021e1c2291a42f,0x00011760abf38c38,0x0004d746e40ef60e,0x00000000000dcb97}, {0x000c4f8f01f54098,0x000f1755b105ba56,0x0001f9ffa1baf4e7,0x000d0b00945db608,0x000cf2809e1ca630,0x0006c95c5a160ac9,0x00074f38fc1113dc,0x000000000005d525}}, + {{0x0001da84872cc14a,0x0009a7eba3da615c,0x000558d3935c6438,0x0007d982b93d8b2c,0x000c7cad1f758c91,0x000ff150aca8fc6a,0x0007fa5f581f19a0,0x00000000000e0832}, {0x0005d538d28d5c50,0x00038c774f18716a,0x00051c30fd1c0854,0x0006e9b8ad72b112,0x000b917986cfce03,0x00025cf9b463f9eb,0x000feabe51632813,0x00000000000dd7e5}}, + {{0x000710a35bbc6ad8,0x00005a4e9f29eaf6,0x000a92c5e19e2d59,0x00084e42993359de,0x000f224d5aa30e21,0x000132c484f96ce7,0x000f5f0862e2003a,0x000000000000f015}, {0x00066db2ab4fc1aa,0x0007cab9d51492bc,0x0009b538d3bdb7a7,0x000e671c1fd96e3a,0x000e71a703b24865,0x0002add107baf4db,0x000d3083dd6cf914,0x0000000000098461}}, + {{0x000b2da1393d8e42,0x00051714c1d1ba41,0x000ef78ff03cd88b,0x000ea3a6951ac80b,0x0000ac00c8377f23,0x00024cc1b5c59929,0x00062bf6efa2b3bf,0x000000000001e844}, {0x0006a668e721edeb,0x000069bda627d119,0x000d91ed1a995ffb,0x0007089c3b94ec3c,0x000e3031699ad1ee,0x0002b2453f75dba6,0x000ed48ff75f7924,0x00000000000289bf}}, + {{0x000ac44be8741dd1,0x0002fcda68a7d811,0x000fb56aeb52d290,0x000e2dadce20b92a,0x000cfc69dca6483b,0x0004f98917de1601,0x000a564bec17aaac,0x000000000008d479}, {0x0009255137ea7d35,0x00025c623cb8d743,0x0009f513ea4e4bb1,0x000b7c030dcde621,0x00073a1733fdda9b,0x0002fac31f84ea32,0x000449d7afb6c3e8,0x00000000000d3897}}, +}, +{/* digit=46 [{1,2,3,..,}]*([2^184]*G) */ + {{0x0006c80cb0000ec5,0x0007da189f30f16a,0x000d675bfc196669,0x0009ec37d8da76e8,0x0006c1ea7c10307c,0x0008c62d4b3e1d00,0x000b3ac15e20b3b8,0x000000000000bff3}, {0x000b01f1748ea86b,0x000e29e330eb12b6,0x000af2a26953e630,0x0003cb002e1eb2af,0x000bf525e4d4157d,0x000a3dff1638f297,0x000051a20f833234,0x0000000000045a9c}}, + {{0x000b35a3033b3940,0x00007fe9fdde8b8e,0x0004a1bfc8bd5420,0x00049acddde6f6e9,0x00097e54356ca653,0x00009f73cc53c29c,0x000277ee15ad9457,0x00000000000e5429}, {0x000bdd741f2769e6,0x000d6f52035cdb19,0x000b835933b3195b,0x000b0ceca319bd4b,0x000b6951fd8d26e0,0x000c34d6f4e7eb67,0x000b59ac3a6f4395,0x0000000000000f60}}, + {{0x000d7763c7959780,0x000c02c47010c514,0x000f6a495cc56b87,0x000be509d930f6e7,0x000d5f56cf045c8a,0x0002f1fc16bcf875,0x0007101f6456c006,0x000000000005304d}, {0x0001668b6ef47661,0x0006dd76452e46c9,0x0004266da10fc06f,0x0009fc89021bde74,0x000f4babfae0b5ae,0x000a61fd6505c6b9,0x00025a99d943c17e,0x0000000000059bf1}}, + {{0x0003de8a392493e6,0x0009cbd10b8bfc25,0x00082fa94d1d5f3e,0x000be5ec0c907818,0x00071ad167ce9a18,0x0008c1c563677f4c,0x000275ccb254e2a6,0x00000000000e2c4d}, {0x000250541623e5c3,0x0003f44958cb1bf9,0x000db2f9dd62ce34,0x000a767a2ffdbd52,0x0009b0b6c22d7445,0x000c92b2e0e789c9,0x0007823ff8b6565d,0x00000000000eca98}}, + {{0x0000ad4fec1eb621,0x000c36618fcad673,0x0002540e2f8dc71a,0x00039947d7ce5530,0x00095257f24b90ad,0x00098768cbf8c458,0x0008305a94992020,0x00000000000bb6d8}, {0x0007503283325be4,0x0007dfae8f1616ec,0x000fc3e0aeb8a2a8,0x0001ea9139ea0507,0x00012e39e1d8a72b,0x0009c7bc282229a9,0x0008254153bf3e47,0x00000000000c5541}}, + {{0x000605d76cba7718,0x000a6fadf9be90b6,0x000490316f096fe8,0x00024ec2f9953940,0x000861203303cbad,0x00089a4be6236a26,0x000a82e4bafc3365,0x000000000005e23f}, {0x000642137e1da447,0x0005c8ccbc576c76,0x0003f63011c9e098,0x0002d5841df8dd26,0x000914d31fcde6bc,0x00026010bec24e1d,0x0002f3acaaf13efc,0x00000000000e01b9}}, + {{0x0005639fc34e5a27,0x000c7c52789bb2f8,0x0006d4ce23fe7231,0x000f95aacafbbfb9,0x000dc7f6eb6d8b6d,0x000b4c9d737afdcb,0x00045357775bdd6c,0x0000000000011007}, {0x000bddf07c5f1b9e,0x000e3903e5ba1399,0x000d7d2fc919a9a5,0x00018932d7ac9e4f,0x000543ce66a8046d,0x000956410e2fe9d1,0x000f7244b5beb4d4,0x00000000000bb147}}, + {{0x0006e03c443812f7,0x0005c6f6a6104456,0x00098182647d3e84,0x0009a6ab51989e5d,0x000f68f5b16842fd,0x000f7fe671b60ce0,0x0005a897324d8756,0x0000000000067681}, {0x00061f269664533a,0x0002a265ee993d1f,0x000a90ce6a02c969,0x000232334b4adb0c,0x000e1e5a8d18e909,0x0004f6456ddcd233,0x000d9b3dc5b27c5f,0x000000000007f421}}, +}, +{/* digit=47 [{1,2,3,..,}]*([2^188]*G) */ + {{0x000b170252588bc8,0x00099bfc40c30a63,0x0001ef23d6587c46,0x000b54dc027511d3,0x0004cf3484ce4fd7,0x0009beea5f479928,0x000280655ab81106,0x000000000007392e}, {0x000148f913baced2,0x00084522e7b403b5,0x000493599cdac0b8,0x000b95877f3913fd,0x00067dd525149cf1,0x00008cbca3e06b92,0x000529992e920e29,0x000000000006d6ed}}, + {{0x00097dff92ad4838,0x0001e8a46c9112d1,0x0000711ed277a798,0x00064f15cf8e4ca2,0x000a01cb3488d4ba,0x000bd7ded01b3908,0x000ae169b1fa5d38,0x0000000000018b2e}, {0x000f3694f52a1f22,0x00019619324bdbe0,0x000d1d925851b48d,0x000f42e925b3f6ab,0x00095f7e4d11a397,0x0000d9f0132169a4,0x0002e5c0fa9a548d,0x0000000000089d8f}}, + {{0x0000115baf0f7c2d,0x00040240239ae483,0x000c7482d351827f,0x0007151f58ec53e4,0x000cc080d59ff9ab,0x000f782f4d397862,0x000d5873eee88536,0x000000000006c1d8}, {0x000c16042c611b2f,0x000f258bed3e5b15,0x00036e097964eba0,0x000c8ab8a482af89,0x0009e746a8549044,0x000a0548e0065858,0x000a492538d1b926,0x0000000000007cdd}}, + {{0x000178ddffec9175,0x0004c9c0d5230baa,0x0004bce21493d0f7,0x00090dd985154559,0x000c8dbfb46a67df,0x000757ce3223e8b9,0x000296f39529a36d,0x00000000000b4648}, {0x00000562605aa919,0x000dc330749e8973,0x000a3ffdafa653e5,0x0000b3494083aa87,0x000321e321c68c32,0x0003e78921161f5a,0x000ccc0980deedde,0x000000000006ad76}}, + {{0x0003725fda5a777d,0x00086af9a69e965a,0x000a3534516a8b8a,0x000a77f3e52375ce,0x0009a019a5932dff,0x000238091ac65569,0x000085d402f5c4df,0x00000000000d3518}, {0x0004a37271e8fafd,0x00007dcee2db4b54,0x0006c1d813edf12e,0x0005b6121bc49990,0x000e68b9808d9cb1,0x000d6ac5b40b34f9,0x000b8a98de63590b,0x0000000000039766}}, + {{0x00069c8c3d7c7657,0x0005171191261c8a,0x000244a0eba69bbd,0x000344bdda57f44c,0x000ac4f0cfd2ad4b,0x000543efd674b758,0x000d063bc058a077,0x0000000000056618}, {0x000a82ca14a01b7a,0x0009d95107c74391,0x000a3c4cfae47f34,0x000af35e3f1d63cf,0x000643ab87265dbe,0x00056c6fd012029c,0x000e304f588a4ea2,0x000000000003e5d2}}, + {{0x00081d2046b48f0f,0x00043847622b5217,0x000c2a7014a5d0be,0x0009da7b82c435cd,0x00025b73e01114da,0x0008b37b399c8c43,0x000ddab978fe55ec,0x00000000000337d6}, {0x00034bf111412925,0x00071e0d4ffda16d,0x0003fc3275d2e3f4,0x00062872913cddbb,0x0004f67405be2a7d,0x000a31060229afd0,0x0009d6e372202e49,0x000000000009fc21}}, + {{0x0001a0d511e4c022,0x0009173fa3508062,0x000e92c1603f0953,0x000549f58493d985,0x000adc79f602f64a,0x000512b84d9ceae0,0x0001516569e37bd1,0x00000000000151c9}, {0x000c6addaeefed36,0x0004c075678c2066,0x00015cc88eb8c3b8,0x000dca3a57fb96a6,0x0000223dc3ce6334,0x00011e2770ed9082,0x0008e274f9c3aebd,0x0000000000079c0b}}, +}, +{/* digit=48 [{1,2,3,..,}]*([2^192]*G) */ + {{0x000688b6fc0935b1,0x000f5378205dd339,0x0000b901357b7bc3,0x000c06c682e00f2c,0x0003114d5423dbce,0x00052463ef2a145c,0x000b0aa01d98747a,0x000000000007d717}, {0x000d2bf78a72f39e,0x000d29653bc4f4a5,0x00051b32471fd3a0,0x00043dcaf8e3f402,0x0000e86fe16ef779,0x0009ffdcf70774a1,0x0005c96b62e6f1bf,0x0000000000058874}}, + {{0x000b3ac410563249,0x000bc2a5f8ecef60,0x000af14f01d834e5,0x0001cedc59c4301d,0x00010111d9989de3,0x0000d5b951e0f40b,0x000ab8d29d229f96,0x00000000000d1dab}, {0x00033bacd39b8f1d,0x000bd7b225cc8ccc,0x0003c9f7b44c8f47,0x00052a1f5fb06b38,0x000f842b9081009b,0x0002725128a575d3,0x000cb7fddb48afe9,0x000000000000b452}}, + {{0x000dcfd459bff4dd,0x00050ae10069e26c,0x000e9f25bee973af,0x000caf27ebaad0bb,0x00073dd6119cbbe4,0x000fceefe5907bf3,0x000c7e0a723dff9d,0x00000000000f7cff}, {0x0002a3a44c0ca01e,0x000d17bc95fa21e6,0x000c0e71f388ad82,0x0007ecd27b3335bd,0x00027b8d7d49316a,0x00019058fbf08e67,0x0009ea4b209f93c6,0x0000000000059d8f}}, + {{0x000cfbdc0726f5f2,0x000ba167ec88a4a8,0x00009c64d249271a,0x000e2443877e6342,0x000603462cb310f2,0x0003afcee6321bf2,0x000dcd1dbd10ee9d,0x000000000002ca17}, {0x000667ac9826886c,0x000d509465265738,0x000151279a7a2541,0x000b0f95e1c59136,0x0001757d3a630043,0x000def1e0a09b94d,0x000d41533956529f,0x00000000000d4fed}}, + {{0x000c29ae93761a8a,0x0008a3459097559c,0x000f79e8fee087bc,0x0009a286ec406ef0,0x0006fee5454dcc93,0x000257f708d21427,0x00085e66a0e1a56b,0x00000000000006fb}, {0x0006c4387ea9f222,0x0000ac44f9df22bc,0x000c5644721083d5,0x000a8224fca1a819,0x0008f3ed85bf5894,0x000899b5b8586e41,0x000371d494dfb202,0x00000000000ecb8e}}, + {{0x00094e8a6ed08358,0x000cf690c0cfe457,0x00092e98638a5e98,0x0008042204d98a6b,0x0005bad2eb082250,0x0005823eec87a97f,0x0003f6d307c59ed2,0x000000000000df8b}, {0x0002a9b6fd1bc660,0x000e9280ae343343,0x00077184e86c10ae,0x0000e5a24d04e396,0x0007309830fcea93,0x000afeebc0269d9a,0x00002d41dc8f0ae4,0x00000000000f14ee}}, + {{0x000ef795d8c64486,0x00026efca7f7acdf,0x000411f7c32b0a9c,0x000b87fe57d08e0a,0x000a7d9a1967c9ea,0x000c2749248c01c2,0x00063911c97ed97d,0x000000000001cf1f}, {0x000b6334379af438,0x000d1541f8c0d49b,0x0006f782375b1fd6,0x000935ba6be190e1,0x000764494b4e9806,0x0003c00b6ec6c5de,0x000f15f04e2d4cb8,0x00000000000c0b84}}, + {{0x000c9d81a70917bd,0x000702e75a26e455,0x0005bf4870175e47,0x000d057ee7d8e4ba,0x000d8994f44953df,0x000538367b959110,0x00029cb4a16596bb,0x00000000000ef82f}, {0x000ea0c85d7d05e6,0x000305a7642ffe63,0x000b9d7a5d2d391f,0x000b23803a4184c4,0x00020f7fcd62c7ea,0x000a8a0c660c67ef,0x0002b041e05799df,0x0000000000004d35}}, +}, +{/* digit=49 [{1,2,3,..,}]*([2^196]*G) */ + {{0x000ffb708a3b5e85,0x00030c97c01eab92,0x0007510b2d7953a7,0x000ce807fa7d3c2a,0x000e81060874dba1,0x0007eeead69e6f96,0x000f3d6e3e0df74d,0x00000000000b0c87}, {0x000317c5146f214f,0x00028c55ae3dbb43,0x00014b4be1d3dc49,0x0008c591de7860a7,0x00066e546731a600,0x0001e45d48202f8b,0x00015a652f2d07aa,0x000000000006df54}}, + {{0x0007d6371007dea2,0x00049041c706cbe5,0x000dcf6b55c23258,0x000cd27839e9d5ae,0x0003bf3c4c067dc4,0x000b7bd22dfc9db8,0x0002da85b8094138,0x00000000000d0f4c}, {0x0000b16d9a334a33,0x00092d7b340062c0,0x0002bb5502deaa2f,0x000b2c2752366864,0x00010113a85fa340,0x000b327045ddd055,0x0002ff7dfc7ab29c,0x00000000000dabf2}}, + {{0x000a8373e5c690f8,0x000ca2fbce9bdf20,0x00049076d1995e9c,0x00045939f4cbaf1b,0x00092574a3bd48ea,0x00092c39a56c5400,0x00034384a39630e7,0x00000000000eef81}, {0x000361503c11fa79,0x00095f996760edb5,0x000c1bbc8ea81e13,0x00012e6966d70279,0x00052e6f7c63a0ca,0x000d13ead92a6d5d,0x00068146809d269b,0x0000000000067aac}}, + {{0x000cf4e4cd35d7a3,0x000a13fc9b3cedea,0x000ac33c871e844f,0x000a58afe1ad536c,0x0008cb39149f2003,0x000edf470cec4be3,0x0005194d578c99bd,0x000000000003a356}, {0x0001980c5865f55f,0x000607a762f2732a,0x0003ce874c8a141c,0x000817f270c508e9,0x00049fdb29c8dc0e,0x0002711d35a7be20,0x0000c2fa1a0be3cb,0x000000000003786a}}, + {{0x00065cdb1cddc024,0x000c41d0af6b5128,0x0006106e0f532684,0x0001951b1ea8fc4c,0x0004b1fae4826764,0x00023477bc0b9006,0x0009ce7012642f66,0x000000000005bf01}, {0x00001d44438309f3,0x0007fca1f46757f4,0x0004d56451db59bb,0x0004ef2d3868de95,0x0001044e0c189c03,0x0005e38c30533d92,0x000053ba6cf14ecb,0x00000000000509d7}}, + {{0x000af6aee4d4a85d,0x000c0e164268de02,0x0001633ba7cb9816,0x000979478ab17f45,0x000e0179ed0e734f,0x000a2686746d468d,0x00085d7e68f006df,0x00000000000e3d04}, {0x00073699ad94d8f6,0x000c30913a1d74ab,0x000e2aa1b6d33ea2,0x000a79e49eadd08e,0x00017dd8f954eeb1,0x000bb26d0433f5e4,0x000e2970a6281430,0x00000000000ff5e8}}, + {{0x000235f9cfa08aab,0x000eb6a352b56ce9,0x0002152b2e478d04,0x0007c7a240e6dc62,0x0002d313b4a9ee1f,0x00001a40585d5be6,0x000d5a1522c5d25c,0x00000000000960af}, {0x000459bf66d63a1d,0x000e4a3cb77f327b,0x0001a15093d3f2d2,0x0000c7d3b93fa9e1,0x00013c0383ad8409,0x000f7a220c77f1ee,0x000bfc461c93b776,0x0000000000004ac0}}, + {{0x000fd3f75cd14c88,0x0002a3c7d6b63ea7,0x0002b345f341120e,0x0005d20aa0eaa1ec,0x000fc0eab4908ed1,0x000d9f260e944ad2,0x000ba371525aa1f6,0x0000000000016146}, {0x000bd29ab6e83fdb,0x00068f94019075db,0x0002a02a4fd970a1,0x0001c37cab1060af,0x0000c8cac96f6a4e,0x0002466ec357fe4d,0x000e7097a8b8ab6a,0x000000000009c01b}}, +}, +{/* digit=50 [{1,2,3,..,}]*([2^200]*G) */ + {{0x0007eae876f30205,0x000645b0b5d68b38,0x0002f6471178cf56,0x0000a4a404a3458c,0x00059f467b6072ad,0x0006348091de8e25,0x000178a4b3570590,0x00000000000706f0}, {0x0007cba07f8d2545,0x0006d588d21aac4c,0x0001bb1a8ee3a06e,0x000e73d241bcd915,0x00022facc7ccf4e7,0x00025d2a0b8d8a1d,0x000608483c35a71d,0x000000000001ef9f}}, + {{0x0009cd91f152b14c,0x00034a704015f319,0x000a64fabfbdef40,0x000301f2ccb94180,0x00046f00d8aa697f,0x00038a0173ee8776,0x0005432b5afaf881,0x00000000000832d7}, {0x0002183eafee3abb,0x000d627c27ce1884,0x000735007191c91b,0x0005ac75b752008f,0x0001e84fe5f192dc,0x0005929cecf382e0,0x000ffa90e034197d,0x0000000000015ca3}}, + {{0x000596896506329d,0x00058cb51f038efe,0x00073c05f41ddada,0x000fafab41fe1a74,0x000da719f25493c8,0x0004f5cde6297701,0x0005426e9165bc64,0x000000000000c11c}, {0x000368f61fe7d95a,0x00098a2564809894,0x000e829acda88407,0x000592622b1d1be2,0x00026ecdd041286a,0x0009f952486a3d75,0x000b0f4b867e0a64,0x000000000000629c}}, + {{0x000259f3facaa9bd,0x0001d11dd860d21a,0x000b8c19c604b970,0x000aff635c019302,0x0001e3a4a900d4f8,0x00078c8ba96a727b,0x0007c41426daffde,0x000000000008d152}, {0x0001e6f4fd354295,0x0004a0c0d5233cfd,0x00066c04a38eba93,0x000bee43d914fb41,0x0008f3ba26a64828,0x0004eb26f8324ea3,0x0007bf027590f3a9,0x00000000000acd95}}, + {{0x000a71b96f713d9b,0x00013f4f668435ae,0x0008fef0f35f5919,0x000e86e7365712f9,0x00088a822bc0f607,0x0001299b3d588229,0x000b1a2cfbd63ac6,0x0000000000067167}, {0x0006f5a47be411d6,0x000b0750f673f622,0x00032c38df6a058a,0x0005bd169123c758,0x0006eab99b375e6d,0x000aec6a36a93d1b,0x0008186ef4f7e68c,0x00000000000cf3ed}}, + {{0x0006410726f50135,0x000fd959353be170,0x000b4de98d5dc91d,0x00026f799d7a4f4a,0x000e52fe4b656a48,0x000038573ab146ec,0x000e8494fc21d735,0x00000000000f4d56}, {0x0006901ebf8c490f,0x00093e15ca04c71d,0x000ef178dcf47997,0x00079244f21a9114,0x0009dcc76132ef7e,0x000e890482eecb7e,0x0002c55b484745db,0x000000000006e43a}}, + {{0x000b8d876ab51a4b,0x0001af92b3072f8e,0x000d8f5d67f2d2e3,0x000d5edc578e3a39,0x00029587fa22e51b,0x0002eba85efda70d,0x000530cfec17089b,0x00000000000af7ba}, {0x0004893a5eb2bed8,0x000bb5ac155ae396,0x0009a3394a2b6335,0x00086c2c38718a82,0x0003d63745b7280e,0x0008a79aa9d12de7,0x000bf70e8ea855bf,0x00000000000bd705}}, + {{0x000260c123f30563,0x000c53ede2484b68,0x000620a80e97a435,0x0009e93962a667bd,0x000b130f2cea5606,0x000366a66c931266,0x0003b14bd6a6fca7,0x00000000000aa5ac}, {0x0004e3f2adddce7d,0x00044a025d0ef29e,0x00075ab6560ff06a,0x000927f2b3057f30,0x000a1499f8844809,0x000b9a653b001c10,0x000d05309d141c30,0x00000000000bf659}}, +}, +{/* digit=51 [{1,2,3,..,}]*([2^204]*G) */ + {{0x000d68f9d41abcc8,0x00016a6c328ffdb0,0x000797038aa63e5d,0x0007d39063de7eb8,0x000710daf9bd691e,0x0008b5d7a998df4e,0x0004b8c7085b9e71,0x0000000000016b3d}, {0x00019da01ecaa2a1,0x000494dfce693daf,0x0007011a8e84696a,0x00004bf4491fb345,0x00014552451c2c19,0x0005e5e407c1bf11,0x0003726562cc2c3c,0x00000000000fe0e4}}, + {{0x00073ecab0b13cfe,0x0002484c3630b425,0x0005d7cee5256fab,0x000125ff61af001c,0x000fbfea35255abd,0x000e0cb6e69bca56,0x0005a6384af19900,0x00000000000d0047}, {0x0001438f80a7fcba,0x00090edafde48dd9,0x000a30b2135b9aaa,0x000f97a6c8ffcca5,0x0003e5a9cc5cf14c,0x000104e054d6cec2,0x0007c1b0d678f88a,0x000000000009fb52}}, + {{0x000cf7bdfab400e3,0x0009e8618ffa6a37,0x00041539f0cbda9a,0x0000d744f61edff9,0x0009eb7a476f5b1b,0x0002ee99b33df67d,0x0002cc7f1a767ea9,0x000000000002223a}, {0x0004324c9cd0a9f1,0x000f616f376f2586,0x000c0794e16b9222,0x000516ca58765df0,0x00062260ed6b8dc1,0x0004b29ba8934082,0x000bba060eb0afcf,0x000000000001d252}}, + {{0x000bb25431b5857e,0x000160d6cadbf906,0x000790df51943fb8,0x0003c734ab19507e,0x0005660086b33b43,0x000eb0f434fc5340,0x00058d770ae903a1,0x00000000000f6b5b}, {0x000dc8bc8bb4bdd2,0x000d6e206e14147d,0x00079b5341d6e69b,0x000ca2f47449e081,0x000639fef8e1cfbf,0x0006fc80cebaef25,0x00061b959e37b8c1,0x00000000000e911d}}, + {{0x0006b0541df5b61f,0x000b0014a0907c72,0x00060742ec5c6420,0x0007c4dc999acc04,0x00075e7ab1e3dbbb,0x0008fb11e01b7710,0x0005c2a33fefbfcd,0x00000000000c0b8a}, {0x0004467ba7747f4b,0x000ec774dc2669d0,0x0007562d1fca7010,0x000dc694d9c84626,0x000a1e772c5f5ac4,0x00083fe91bf6e002,0x000dce4922120e0b,0x000000000003efba}}, + {{0x000263fbc0d157fd,0x0005bc483d4b1827,0x00037dfadf4ae121,0x000df6fcbab1fb10,0x0000f6cdfc0c5165,0x0004320fceb28437,0x000e80ab565c0099,0x0000000000062133}, {0x0007401414422436,0x000ee7850cda5472,0x0007bd0ba094b0ec,0x000805cc2c82eddf,0x0006cf244539d14b,0x000dbe92dcb5468b,0x000f1e97d43ee825,0x0000000000089fb8}}, + {{0x000a494639f26e1a,0x000b5421afbe0092,0x0004dc6db28bf654,0x000cf4db1a2705ad,0x0006d128bcd556ca,0x000191ad86a3a413,0x000d242411c4b866,0x0000000000015b45}, {0x000845fec1268b55,0x000a74cc82459052,0x000da0992b42bc3f,0x000bb1c69f6298e1,0x00031933fdb59c88,0x0001308c3fe567b0,0x000dfa8a6aea7188,0x00000000000a2f0a}}, + {{0x0006a4d1fcc1ad65,0x000d2d7bf2ac5a3e,0x0005e362c26e5671,0x000abaa9afa97632,0x0004b62b36ab162f,0x0006a84e97b7f166,0x00043d77b9f79729,0x000000000007dbd8}, {0x000519c3add29e33,0x000fcc6e9c1e11fa,0x000ac380a63f4305,0x000a93d3bfa90c04,0x0009d050e46afa7f,0x000c5625655846fe,0x000a65473b9a0d35,0x000000000002b656}}, +}, +{/* digit=52 [{1,2,3,..,}]*([2^208]*G) */ + {{0x000f0e1b25d4e4dc,0x000a6372798f002b,0x000537a488c42515,0x000f9a98a1e25677,0x000ce70391c85e64,0x000f024585870254,0x000bed2def81a341,0x000000000001d087}, {0x0002a1629bafa8b1,0x0006d3557d07cb43,0x0009543a3877e0bb,0x000c5675f73ba510,0x000b9c7c670608c8,0x0009850725309050,0x000e962ab67da3bd,0x00000000000e5df4}}, + {{0x00057ab93a62b1b3,0x0004b7be81fb0ec2,0x000d385405273506,0x00040e27a8d16791,0x000ad520811ebb3d,0x000f65d231806c67,0x0003d7add4bb6686,0x00000000000e20e2}, {0x0008a96c64700a7f,0x00088208b470000d,0x000907fb1a1c5c32,0x00064f8121c37e26,0x0001a598efbbbd39,0x000eef966d35ef30,0x000e46bd76a276c5,0x000000000000af64}}, + {{0x0007b8ba2901e630,0x000573f40494a69d,0x0001d7e86c246f17,0x0003360c9e634b1f,0x00096ab166bbacc3,0x000fdb67e6cf72ff,0x000736477d8f2db9,0x00000000000cb644}, {0x0000821b2e82caf8,0x0007549454e1ad4f,0x000486f6c48cff7d,0x0005f0be8e7b06ec,0x00047dc40b498042,0x000ed620b862df52,0x000a648ca7d7c812,0x000000000003c45b}}, + {{0x000d7620f273aa67,0x000e1169474a1e10,0x000056ab42590c74,0x000de922ede425c9,0x000a6df8a8908589,0x000a8b8e350e03fe,0x00091a0d8a5c1c4b,0x000000000003fffb}, {0x00026981fafa18b5,0x000f721cf05437b0,0x0007e513859293da,0x0007eaed0962c826,0x0004213f6004c323,0x000148b6b43d6ac3,0x00080a45e619b2d4,0x00000000000ea5fb}}, + {{0x00014768f5f99aa5,0x00067314ea217285,0x000017c8fd29a716,0x000fb46a63ea8fc5,0x000890d84e5b0902,0x000e49b8a925a6dc,0x000be5e2e74f9c14,0x0000000000007d45}, {0x000cae18673b6270,0x000667b768d075ba,0x00089b2a5deeff6b,0x000223360d5b216a,0x00080a7386f475db,0x000c47746b132b67,0x00031d7f933fd580,0x00000000000fbaa6}}, + {{0x000ee5b308cc45e3,0x0002faba967ac481,0x000d29b2fe96bd68,0x000c601ef5f681db,0x000ffddc580fe033,0x0007572d85c34f77,0x0004d0d30f5b66f3,0x00000000000da20d}, {0x000030e8bcc549d9,0x0002cdb2310273e6,0x000ec784d2efa81a,0x000a33f7899cd7f5,0x000fda29c3821cce,0x000e14ecf0f4e0a7,0x000839c6d7c5f32b,0x00000000000d9caa}}, + {{0x0003fe55b28c2fec,0x0007ba884edf1601,0x000775572e4af6c1,0x00004152d7852a27,0x0007f26efb4c66d0,0x00022f8cb34732d7,0x000ff518b3ef8e29,0x0000000000018bcc}, {0x000ec4cab3e21461,0x00004a219cb1deb6,0x000868a49e96a154,0x00099e1d90760ec0,0x00078a94df2ef0af,0x000058f89e6fe194,0x000d9764b5dfcd04,0x0000000000023d21}}, + {{0x000d7944d758c20d,0x000209a8580a957a,0x000f955204f37a28,0x0000970be07f7827,0x0000712f7b7cb4da,0x0006a7b970ac2a26,0x000f62c9b8ee8443,0x0000000000011fb2}, {0x000ff5f68230c1a3,0x0002a5daabed96f3,0x0003bdf181469c85,0x000b5b7d96cfb8e8,0x000344d9e84382d6,0x000e7da3c4d7d0d9,0x000253fa2a9ea991,0x00000000000d531b}}, +}, +{/* digit=53 [{1,2,3,..,}]*([2^212]*G) */ + {{0x0003b2a8a65e5b7e,0x000424cc41f278dc,0x000bf1d7ec4af5a5,0x00066640fa1ca255,0x000b91e5edaf7053,0x000d3de14eeb40f3,0x000c43cdf98235f1,0x00000000000ff018}, {0x000927ebce051283,0x00074aa3228e6dee,0x00043f750dae9462,0x0000425650b2dab8,0x00026d875f1790e9,0x000e8a46ee4a8cad,0x000fb5c212029c9c,0x000000000005ed7c}}, + {{0x0007539a8f390740,0x0008eadb5966f40b,0x000e0b7342eb902f,0x00073e244693a961,0x00055982bbd3a76e,0x0002ca13214da743,0x000e7646e982cd5d,0x0000000000024938}, {0x000cf856b36cb844,0x00029749206b2571,0x0006030c0c47215a,0x000d1567025bb7cf,0x000e19555c9ebee3,0x0001639bae23f0e1,0x000bd00dec383775,0x0000000000005d43}}, + {{0x0002c235491635a5,0x000e4e4e52e86121,0x000459dc25e36e9d,0x00051bfb49f2b393,0x000b3f8097cf73b9,0x0008fbf057b6cb7d,0x0002119dfb8d0b32,0x00000000000ebce6}, {0x000c890c36814c6b,0x00007a31a15235a7,0x0009c26d4a535440,0x000834e6b5638766,0x000d10ee5a281d22,0x000aee4eafd91b30,0x000a763d7a282d59,0x0000000000073300}}, + {{0x000f6efbfb5bea3a,0x000f878b5c14b0f6,0x0005485b973e6dbd,0x0002ab209e1759fd,0x000db4b2f68cffa2,0x0005be7f45a86263,0x000f6d71e77c516b,0x0000000000019844}, {0x000dc7fe7c7337ad,0x0009d2519d0058c1,0x000edd3b9e6ca5d6,0x00074a685b3c2a9b,0x000fc294f4492c6d,0x00069fb469306f68,0x000886552e77c22c,0x0000000000010bb9}}, + {{0x0001ae09a4f32c66,0x0007beba7daac862,0x000767fe0f73dc31,0x00018f885bdbc832,0x000094d43909985c,0x0009e108b86555ff,0x000313b0b1b2b653,0x00000000000c0bf1}, {0x0003b62754d457e4,0x00021bd4777c10d3,0x0007d7f58d2fb40a,0x00057374a27f1ddc,0x000eeaaa58ab85bc,0x00076fac29a8ae24,0x000377161cb2f5e8,0x000000000006636e}}, + {{0x0001f89428b5c457,0x0007f1674b959a73,0x000b96ebf7106c2e,0x000a32dc67c36488,0x000368d720a63962,0x00057a5b24949617,0x000c0f4e81df85a9,0x0000000000053123}, {0x0003624d70103a1a,0x0003f5091dd340e1,0x0000c9fe10861f33,0x00020f52c119dbe8,0x0006c94d609a5e77,0x000dccd1fd584ae7,0x000c6e476c63ba86,0x0000000000032508}}, + {{0x000df9bca60288d5,0x000016bbf77cab44,0x0000fa9d18796041,0x000eb1a2b9febf8d,0x0001a25330ce357f,0x00091799874240a8,0x000f5c7a9ab575b4,0x00000000000eda3e}, {0x000c7149276e2420,0x00036360410d2e37,0x0006d4d0d5e12db0,0x000b466cc381b581,0x0008247a49047bae,0x000c58130024a98b,0x0006d26e70b4c3e3,0x00000000000ae8a5}}, + {{0x000d9a7dd453995c,0x000393313a9d4705,0x000fd95bba01fcaa,0x000ef915e4dd5cea,0x0003c565dd67c0fd,0x000ed05ac902a2a9,0x000ae9d8eba4dc7f,0x00000000000e157d}, {0x00019071237f3ae4,0x00006d655d0b3ced,0x000513db82a990cd,0x000525a0652872b6,0x000fe68c0ddb5b7e,0x0001cb31caf7968e,0x00071e2ec02930f5,0x00000000000f2be0}}, +}, +{/* digit=54 [{1,2,3,..,}]*([2^216]*G) */ + {{0x0003b3ac56ccd2a3,0x000649b23ab4e3e0,0x000d023509576972,0x0009e51e798edf99,0x0009307675c7dbe9,0x0008c0fb63854744,0x00037223ffaf5562,0x000000000001698c}, {0x000420dd9073adb8,0x000d039f45a56f2d,0x00011e9a2cdfa00e,0x000079e4af138fd7,0x000a2ee4ecc02a89,0x000bbf92fb86371e,0x000c51076d256a06,0x00000000000ae3c4}}, + {{0x000340cb6908d50e,0x00092ba2e95430b3,0x000660e7e985a29e,0x000b95145bdc19ee,0x000e382e77bdf94d,0x00020b29a951d00a,0x0001f19940a5fbb2,0x0000000000058fc9}, {0x000d804932dbc0b5,0x000be682e42eaaa2,0x000400a2efd4aee0,0x000810016294d055,0x00032e326d68be15,0x000e64fceaea13fe,0x000a8ac0dfe1ef15,0x00000000000b8237}}, + {{0x0004480f8fce3f16,0x000a7e59b80017bf,0x0006c7396aa46dc8,0x000172a5af5b47f5,0x0000160d7e8d8799,0x000f9a549f72c978,0x00044a1d1ce972b2,0x000000000004857b}, {0x000d15fb2758caea,0x000542545bdd6f77,0x000984fe91e9b1e2,0x000343a4e23c0644,0x00091d1fd9cd5a60,0x00070b5b3986779f,0x0005a35bd5611b35,0x00000000000f9d76}}, + {{0x000b72123cb1cd13,0x000e76ee65a0886b,0x00081e4e332045b1,0x000cc382876e523b,0x0006d3bf53aac4a2,0x0007f290cab7aba2,0x0005bd5bf00871db,0x000000000001ee6d}, {0x000bcea869ddbc32,0x000334cafb21874b,0x000bcaaf9d600f4d,0x000785520b281cd0,0x000e64d9c65ea1fe,0x000a0e67be457198,0x00068aa3d1a6d0c5,0x0000000000090cc4}}, + {{0x000450a44e08b4d9,0x00014cb0a365753f,0x000b82633a02b2b6,0x000210997ed887af,0x000f30d9b2970b85,0x000fb9c745fec3e1,0x0007854ce4149f10,0x000000000008cbff}, {0x000dc4bd785a06f1,0x0009f81b0d7b3b6a,0x000116390fc1ac37,0x00021de2b841eb88,0x000ad83c22e6aec2,0x000affe9162c7d86,0x00081d5504dcf885,0x00000000000f4454}}, + {{0x000578651af84c0a,0x000e0d4ee3f7f52b,0x000cec289c787837,0x000ee1363ebab5bb,0x0007005ec2374c0b,0x0002fb00670e32d7,0x000899302fc73dc5,0x000000000008f159}, {0x000ba114d96a8216,0x0009d42a5478e2d1,0x000e66d84b639b08,0x0004970c8378f0e8,0x00058e2c86c5042c,0x000c7c76770c1957,0x0003a861a95e6884,0x00000000000d6fb4}}, + {{0x0000a2299e18ff96,0x0001ceaf237a8503,0x0006d80455ecbada,0x000fa473f251ad61,0x0006d828578e5fbf,0x000e118adc40570f,0x0005485cc65c0dd4,0x000000000005da48}, {0x00073e60bf0732eb,0x0000fe27fc2e7307,0x0009067267d2e6a8,0x0002fa55e27fb12d,0x000810003fae35a3,0x0009800c17fcfd72,0x0002e6c74b50a3f4,0x00000000000dbafb}}, + {{0x0002a6bfc8996b96,0x000bd0c62fd2c8ba,0x000a840806b7cf85,0x000933dcef3f9e43,0x000d9889ffa276b0,0x0003c88d251b1ec2,0x00052f9e84b2ba9a,0x000000000001913e}, {0x000a4507b899f92f,0x000e6bafc5e94164,0x0002238654296051,0x000cc41bed171099,0x00036c7a41c84e9b,0x0005369cd0db5b73,0x000934d4be07a779,0x000000000007bd3a}}, +}, +{/* digit=55 [{1,2,3,..,}]*([2^220]*G) */ + {{0x0006d08f59277dc6,0x0008a3f2eff5384f,0x00049c170a3dfb6a,0x000b18a0dde190dc,0x000da26b0409af10,0x000b1d944f491b98,0x00054166080782a2,0x0000000000097e8c}, {0x000baebab71369f0,0x000d1fdbfc5f5495,0x000a70804cb1f0f5,0x000263857645ef4f,0x0006a02583638e5d,0x0005d250331bcfda,0x000285f5330ab8d3,0x00000000000c7cab}}, + {{0x0004a7ee3780eead,0x0001ef16938f4dd4,0x0005af2b9dcbcc11,0x00095530b6490d71,0x0001a28a296a2d50,0x00026415c8432fef,0x0008656f254dd08d,0x00000000000d50c2}, {0x0005457026e64224,0x0003c4f5bc4553f7,0x0006183dc27db1b2,0x000dd6e65e593411,0x0002a56dc2eabab8,0x000a90e05676baca,0x000da038eea06bea,0x00000000000174ba}}, + {{0x00021d43da6aa865,0x0005de6a19dcb664,0x0006a4c857b63184,0x0009b9fc6455613f,0x0004a7390d0eb4d8,0x000ea135a6cb0fe4,0x000982ade197a459,0x0000000000020680}, {0x000776554c3cb5c6,0x000b803db9be90e0,0x000e56e339783849,0x000e8d4753c196c6,0x00000b7c6ff544de,0x000a1b14259adcc7,0x0004f2c6260ec24c,0x0000000000046cbd}}, + {{0x000c69e90d279f7b,0x00051a35e411c1f8,0x000aa4eec7d05943,0x000859e89a66f2be,0x000e0def8ecd7c7c,0x0004947b79908c37,0x000ce88274124e34,0x00000000000568b0}, {0x000eb0436a41e1a9,0x00070e52919611c1,0x000a98c568a44a8e,0x00039e156bd3a7e1,0x0006268f856260fb,0x000fd8293e56a34d,0x000fcbb3a1fe1613,0x0000000000067537}}, + {{0x000d9811d879790e,0x000da8a15d6fdcb9,0x0006c38fcc4a52b8,0x000e38c55bbe733d,0x00051ff94a9d7a7e,0x000585ab5eff146a,0x000a5de572d13edd,0x0000000000006491}, {0x000a7bbc541e4f72,0x000a29c84e1c4d63,0x000bbb62e6c0b1d5,0x000cd9b385f06c82,0x00077a7759c3db12,0x0003bd7060c93eb9,0x000d50f1f5b0fe68,0x0000000000085ec3}}, + {{0x000e87011f7cd75e,0x0006e89e48d8ba73,0x0007fdc53e3e2631,0x00033d7302c0daa2,0x000a048eefe360f0,0x000a7224415e4578,0x0009cdfd6dec89b0,0x0000000000030948}, {0x0003345fd128739e,0x000ad7cdcc2a0188,0x0002b63966c3b413,0x0000a455812b560a,0x00052ca31d8ca630,0x0003a5b5a8fa5c41,0x0004e036aa3c234f,0x00000000000c86cf}}, + {{0x0004ff5664ce36b2,0x0005e9a0e15351cb,0x00019cbdd0d2f66a,0x00059eafb29777cc,0x000ae354cafdc170,0x00007c3717e40e5f,0x0009459cf594054d,0x00000000000a71c3}, {0x0007429ea783b1e9,0x0003469309e95af4,0x0004f55088c266f7,0x0004070e25823b6e,0x000d0bc27359f216,0x000925094ead851b,0x000f4e3d21bfe8b0,0x0000000000034a97}}, + {{0x000a4c18541d03ec,0x0004ad927282fbf3,0x0005c034c274cf2b,0x000207f450db7135,0x000423e16d9558b9,0x0000e349cae95338,0x0002bc4f10c6d4e6,0x00000000000feb12}, {0x000eced76985b33a,0x0002f22548cd1c2d,0x0005b37b87399908,0x000f912b6167b3cc,0x00027902d2baa1c6,0x000de34ba6967bab,0x00025eebbe0b0836,0x000000000004b796}}, +}, +{/* digit=56 [{1,2,3,..,}]*([2^224]*G) */ + {{0x000e99ecf706c6bf,0x0005c9e857f32800,0x0001e880e21c15d7,0x0008d68ff4f65674,0x0005ac339148f853,0x000dfc12f35380f1,0x00093efef0bfdd5d,0x000000000001387d}, {0x0009274bbe5eb9e6,0x000aa618ce77c94f,0x000ef0d12ae1c332,0x000f06e00dc0da6a,0x000e07603cc724ea,0x0006963c7049113b,0x0003005cf489088f,0x00000000000ede4a}}, + {{0x000abae3c29bb132,0x000af77e486f79a6,0x000ea167f51170e1,0x00028ab7df36628c,0x00016704dcd6322b,0x0009a35672d14d13,0x0003b6d359977af2,0x00000000000ec96d}, {0x00053212afaa74ef,0x000f0fd6e400a371,0x0003860e13fc28c5,0x0001c7d9b8533afb,0x00028de66eb862d8,0x0006784eeefa638c,0x0002237405a9d7e8,0x00000000000a6c22}}, + {{0x000fc1e6b9032350,0x000a46909994e4c6,0x0006261c6638f0ac,0x000ca05884aacaa5,0x000996995a981505,0x0002c000ee4b6530,0x000b0930e00a5ed0,0x00000000000236e7}, {0x0005ec99c1d0db26,0x0002ce696f09d532,0x0002f7914e3f9268,0x000a7b401e1e2a4e,0x00069d2d025aa9ad,0x0004ffeb19630acb,0x0004f69fab2c6ed8,0x00000000000ab758}}, + {{0x000c87e27d06e6af,0x00073f9b2dba43cc,0x000cbbdd7e7ab099,0x000a4f33b8104eed,0x0000e4e1896e7692,0x000d2aa365da885b,0x000bcac2a30fec73,0x0000000000086f60}, {0x000adfed330d989a,0x00086bc8bf16d541,0x000f4b7104707db4,0x000e2f37e35610a9,0x000482f9d71c8e79,0x000e62733981139f,0x000061f8997ec424,0x00000000000a3518}}, + {{0x000efb4736bf4182,0x0003f6cf0e6ef64b,0x000b24ffed39dfee,0x0007783856111dce,0x0000c0e9f2b00277,0x0007fe5073a1d36f,0x0008f1fcf4f6365e,0x000000000007fa7c}, {0x000ce2543c17ec02,0x0005509a02de874d,0x000cd3e25ee5e59f,0x000a9654b7f4e35d,0x0009805b58bd7211,0x00057860ca6b2ba7,0x000d58418302c209,0x00000000000f99f9}}, + {{0x000634f7fb73c6b6,0x0002e4e6ef40fcf1,0x000c701a714f0702,0x0003403fd41d144d,0x0007e0774c37a4f0,0x000c7484a3a50717,0x00066b078e8c568e,0x00000000000fbb3f}, {0x0000fb0d6daae4e9,0x0002c169c9474ce5,0x00027d6aef77ce07,0x000968508303114b,0x0008fad0def23e8e,0x000c1c8da7a9797b,0x0007210ad14404ef,0x0000000000021ced}}, + {{0x000169a6e51baf05,0x00088fde0d1b3e6e,0x0008e5407b7aa0be,0x000ad79c9eb9de48,0x000b0ffbcdac16d3,0x00020287c2707ec8,0x00055ad7e6750fa2,0x000000000009c2e1}, {0x000dcbd856a04522,0x000e43018c309307,0x000def4e0648d266,0x00023aecf15a4af5,0x000cc1b8cca01aa3,0x00043a969f085d69,0x00043047a3eaccb1,0x00000000000f3a98}}, + {{0x000270a279eabd20,0x000d5c7e9ddef0ef,0x00060c66b8938b7d,0x000746db239bb82a,0x000b28ea13416bc0,0x0001309b0c811a8e,0x000b345f714ca71d,0x00000000000d4eb9}, {0x000f50441ed062cb,0x00091e0e5afdcc03,0x0009d20438aad877,0x0001e7b843343663,0x000a0eed1116670e,0x0009bbd50c8c38f9,0x00095af914fae261,0x0000000000051c19}}, +}, +{/* digit=57 [{1,2,3,..,}]*([2^228]*G) */ + {{0x0001b38ab493e121,0x0005bde849cd1240,0x000576b3d2c358dc,0x0009e3dabe92fbab,0x00043324900a3fbd,0x00020904e785414d,0x000ba8daead1abde,0x00000000000aa5f1}, {0x0002d0438c4bd099,0x0002fd60a4f2ce26,0x000593174efc1656,0x000c78934efa243c,0x000f216a8d8c163d,0x00001617b3067dcc,0x00051e116b6534a9,0x00000000000bbabe}}, + {{0x0004b6e85f0076cc,0x000c1929454f6549,0x000021b9b8ac3fe0,0x000a7c5ee25c0b0a,0x000f2e752295f5b0,0x000acac687d3372f,0x000e3cd6dadc7d6e,0x00000000000a96a8}, {0x0006465e062c14dc,0x00030ea831db66b4,0x0000548165c8c6c9,0x00017e572e3c00c5,0x000d2a5fb6ba5ff8,0x000392476b022e25,0x0005a0b611c5bcbb,0x0000000000019048}}, + {{0x0000dddcc280d252,0x000d5efda99d90b9,0x0002988f9d0202d2,0x0006cd1ad14ac705,0x00031f4138808b9e,0x000dd7fb91239ee3,0x000b12d98e93d993,0x00000000000894fc}, {0x000d9440883321ae,0x000d433a92019c9f,0x000ee20fd3f674ff,0x00051280d0a320b4,0x000b4b607b538450,0x000228c2ec20551d,0x00025c6e63c766ea,0x00000000000ac48f}}, + {{0x000ea4ad3f5b0bfd,0x000140372678d84b,0x0008ab3dd6009aeb,0x000bca4b79594c43,0x000baf3b75cfebae,0x0001e09c6e587850,0x0004cd534183ac2c,0x00000000000c0820}, {0x00012c542116a023,0x000a7dac2cf06c18,0x000f5e79e9f15f10,0x00009f490b0f6c27,0x0006c2c62207f6f5,0x000ff18873ffc3cd,0x000c6fbb21eb1132,0x00000000000e62cc}}, + {{0x000a11bec64a35cd,0x0004f1ca74a30c77,0x000de3a654c55d5a,0x00049038d6b4b005,0x00002f7906ee3709,0x000452bbb12ba86e,0x000039aa76f77adc,0x00000000000f9837}, {0x0000782bfd430ed7,0x0001841fe306f87d,0x000ce68ff3cdd73a,0x00024ccaa7d44b2c,0x0005d86900f9cffa,0x0003ecfa022bae39,0x000980e7a138782e,0x0000000000086d28}}, + {{0x000489915b0c1e42,0x0000991b8b685879,0x0005ba3b38e17597,0x000d69ea7d9931a2,0x000c7632a26bcdb0,0x000ba170f3c8441d,0x0008adf11a365c62,0x0000000000034e74}, {0x000d3f6d2f87f536,0x0009d523ca2d7db2,0x000b5fe1b40ff204,0x000c7771d07308bd,0x0007c291c2ef71af,0x000b40d1773588d7,0x00015629baa3b0aa,0x00000000000eea9f}}, + {{0x000a95a8a249bd36,0x000f90ae9143a26d,0x000709bd167b8510,0x000f7a7f3b4b882d,0x000de22d9b9923df,0x0004f02b1b178e73,0x000c2fa83861afaa,0x00000000000b9064}, {0x000de7d4573c6d34,0x00022142eb294574,0x0006f55c30205aad,0x000717fe649a8b70,0x000cad53c9bbd589,0x000ecd8f7a925c47,0x000142c339b11a09,0x000000000001bbf1}}, + {{0x000abd60f6eb49c8,0x0008c406a7e201fa,0x0007246dc14c8322,0x0002eff020748efc,0x000af39b58dbd440,0x0008cfb047827442,0x00078f77e3f2768d,0x00000000000e45a9}, {0x000ab42dc779cb3d,0x000229db0829881a,0x0005eb7284cde06b,0x000ce47b82775f69,0x000f63910016e434,0x00047792fe84995e,0x000eb9a35e8b971e,0x000000000007c6aa}}, +}, +{/* digit=58 [{1,2,3,..,}]*([2^232]*G) */ + {{0x00097b7667d86ea7,0x0001b1fa064cf475,0x00026db64fb0c148,0x00002a1fa9d94539,0x000bdc6bd7eada81,0x0002f6044786aeca,0x000208caf91e3bca,0x000000000008573f}, {0x00036746e95246de,0x0006cd309fce8dbb,0x0001300c9068d932,0x0001ae0f3d530575,0x00000d1fd61e5779,0x0005ebfa626b053f,0x00097991c962c004,0x00000000000076ed}}, + {{0x000013c3d6e02921,0x000c449f2499410e,0x000e2ab53501cdc4,0x0009103d5e91bc0f,0x000bcb404f68897b,0x000f7fc0263db1ef,0x0000af70efd9d842,0x00000000000c6a23}, {0x000a0390300406c6,0x000308d5b199ee1a,0x0009868ea4fe37e1,0x0003d34504dd889f,0x0001f823686fde58,0x000fc73dc0375600,0x000f42f19ab86f95,0x0000000000093f12}}, + {{0x000b8a18e1e24b49,0x000143f896ca9186,0x000d7ce3f5b4c07a,0x0001632600e4e2b2,0x000f69e702d5d074,0x000e0df2b87c7db5,0x000fba1f5a3b8053,0x00000000000f443d}, {0x0005af3bc7c98ae2,0x000deeb90e22f972,0x00070953899b58fe,0x0008299a5aa335b5,0x000d4197b32afb1e,0x00055918ed78504c,0x000c720e7a79cc67,0x00000000000883b1}}, + {{0x0004af5925d66db3,0x0008dbf66baa58e7,0x0001c386a0ca25fc,0x00032abeaaa466ce,0x0007e5f2733b80bc,0x000531afa605b789,0x000369e9e7e3a1a2,0x000000000003deb8}, {0x000174c1d570e84f,0x0008d36212ea2dd8,0x0008479c4475fe18,0x0004b31444e9ea02,0x000169530c1befa5,0x00079bdd19c2229a,0x000cc368feb9854b,0x000000000008b984}}, + {{0x000682b315cae64a,0x000981df8d98c41e,0x0009fabd7bca288f,0x00064f91703e1431,0x000e9de0ab4ca54d,0x000dc4e0fa12198b,0x00091f160e06241d,0x0000000000066958}, {0x000b5c1e9cafc463,0x0006f808565e66f7,0x000a9467f76914c8,0x000cd934bac17690,0x000ca5be965f6682,0x000a38e9062e7ac2,0x00016ed33f6f8ad7,0x000000000005b8b5}}, + {{0x000509d8741fea0a,0x000c1f041e421362,0x000c0d8899228fbc,0x00093751128ed62b,0x000acec6eae64fea,0x000a65dae041a1c1,0x0008e81f32359415,0x000000000000154e}, {0x000a9eb331e84371,0x000ec309e9f286a4,0x000765936bc21528,0x0004dd5692420c27,0x000ec4fa1a6a7bb5,0x000779e77193a41a,0x000b8c35f5f3b43d,0x00000000000046eb}}, + {{0x000b5769d7b9ec0d,0x000aa3a5a366dc99,0x0005d073ce2a60d9,0x000e4aafe5355bee,0x0003ced676663e16,0x0003440036d8bac0,0x00020c403eb33ed9,0x00000000000333ab}, {0x00094bfc36e2db30,0x000739fce19869f6,0x000435b17be8c513,0x0009611921a58e5d,0x000620d5c61a8e68,0x000c81b4e8f5f115,0x000779b17f612fad,0x00000000000e562f}}, + {{0x000d75f385d1b0f1,0x000f0d6a4d25bfe8,0x0007b705bfa0d54e,0x00059731cdedfc0f,0x000603d6502f9420,0x0005ce8c80c4e385,0x0000b7981a4fc5e1,0x000000000007aca3}, {0x000ea22cf2bcfc17,0x0009ef2037ef684f,0x0007e5010cdb37dd,0x000a23c71f3e4e4b,0x0009f47b504b9c98,0x0003233aaa73c8b8,0x000f68b9e33f5402,0x00000000000f92c9}}, +}, +{/* digit=59 [{1,2,3,..,}]*([2^236]*G) */ + {{0x000a9e3a73909533,0x00090ba03ba3b07a,0x00090d7a3c9c5a5a,0x0008cfe4f0f60b35,0x000e6fcccd96f96b,0x0009dd17ab908d77,0x0000487208ef7de7,0x000000000003ec3d}, {0x0006af6a704d4f0e,0x000d09c5ad2d9a11,0x000e77d5943d9764,0x0009449470eb938b,0x000bee7d772fac99,0x000b7ad09faaf27f,0x0000a9fbe402abd0,0x0000000000057db0}}, + {{0x000ca62ea2a4a457,0x0001b10c082a59d1,0x000bdd4313beafb9,0x000935b4cb291a7f,0x000313e9ce08785d,0x000f6f1c4fc2ae15,0x00024c3146fabf4d,0x00000000000c87dd}, {0x000f74ecc24bd4cf,0x0000385fdd8765b9,0x000dc418405d512e,0x00005013e7e0297e,0x000fb92df904c81d,0x0005ddccbb56ddd1,0x000f1d4612df9f29,0x000000000000e27c}}, + {{0x0002069dae7548ac,0x0002986d9b05f69f,0x000025ad33463063,0x0008e7c27d9d64d3,0x000aba04e6ad9b6e,0x0008abbbe79bd66b,0x000d0477e4d0b082,0x00000000000cc540}, {0x000bdbdb8d90e2e4,0x00067f5d8a46ef90,0x00078bfd47c637af,0x000b852563fe6b52,0x000997cdd04fb93b,0x0007de47d06bb3ae,0x00061c07f011d48b,0x00000000000de78f}}, + {{0x000eb7904785958b,0x0007d7830460f8a0,0x000cf49e72cbbaa0,0x000e2aa307d9c790,0x000b5aad8b6c73be,0x000848db51b02af1,0x0004765e31882703,0x00000000000f230f}, {0x000b79128735694b,0x000bd4ea6535cc84,0x00008e33135971e8,0x0007b332fd33cac2,0x0001c566914f4c19,0x000952d3b24c7b0b,0x0005f2956ce3c371,0x00000000000fabae}}, + {{0x0004121555388dc8,0x0008b8a95e5c1a8a,0x000cd5dbd7b0133e,0x000df6acda40bc0b,0x00058234471d1859,0x0003c0100f9097aa,0x00067caddc0c9b78,0x00000000000feb70}, {0x000ce59ade26052d,0x000a6e3805fd0a27,0x000626ac8acaae6d,0x00099921943a0a1d,0x00091dfe627ea459,0x00006515fb47f061,0x00073e313d4f09da,0x000000000001f54a}}, + {{0x000f6dd7137bd615,0x0007e0db87bc3c53,0x0002eda89127238c,0x000e9c4a3fd2a60d,0x0001c9f017e5ea81,0x000c75768190c9c7,0x000de621864180bc,0x0000000000043dbc}, {0x000fb5dd8276da18,0x000de5b090c70dac,0x000cd9057894e345,0x000d268a918bf24b,0x0008ae204f49fef3,0x000d7c356e10c52c,0x000a17f4be898d86,0x000000000002fdb5}}, + {{0x0009bfc4aad5cc44,0x000a20079c69c96d,0x0008a5713957941d,0x00086660139685a2,0x0004946fbeddb8d1,0x0001aace408590ca,0x0001b67fecd9b964,0x000000000002936a}, {0x000267114a49f247,0x00065925b6235aee,0x00055e3538cd2015,0x000663d8c6fb34ac,0x000c2fc1c10d971e,0x00042b822543146d,0x00024d6e4053c706,0x00000000000492e5}}, + {{0x000d356fb82e9b9f,0x000beca9872e0a14,0x000f54683a031c30,0x0007cb84e05b2811,0x0006fa234d4596b1,0x00035a7d89798714,0x000384ba78949ebb,0x000000000006fc59}, {0x0001361f78e02fa8,0x00081a303e549f81,0x00064be08532a2fa,0x0002de8ee7220467,0x000563e27035e57e,0x000d2fe6fa05c106,0x000aadaa38e86602,0x00000000000ee2f6}}, +}, +{/* digit=60 [{1,2,3,..,}]*([2^240]*G) */ + {{0x00044971cf281b0a,0x00052c0426b768f1,0x000ef3f4445c186e,0x00012e3172c0d3e8,0x0006ee75473731d3,0x000a7ee615f49fde,0x0005fb895530f06d,0x00000000000e8b3a}, {0x0006345afdc270e9,0x00019fd14973443f,0x000f6896912e434e,0x000ae07653908d03,0x0006ba02a278e2ba,0x00021b8f8c3d0143,0x000297a0d0222e7b,0x00000000000d7ec1}}, + {{0x000653659b1a252c,0x000514f120aa7478,0x000c72dfe03d7757,0x000ac5ecfe5f7a92,0x0009bbf3cec6c96b,0x000361cd5d4e73d5,0x00044ced8d233560,0x00000000000562f4}, {0x00045d3e2b7ac684,0x00022bd37d3cf9b9,0x000cb601f2d0a968,0x000535a3d2f41ee1,0x000ee8b1743e7e35,0x0005a27650353b52,0x0008b831d89dfd7b,0x000000000008d9ea}}, + {{0x000718fb55d90569,0x000b306a67bd2493,0x0001471031374c3f,0x0005d5197bc62d32,0x000924c51874ee0b,0x000a1a0d552b1703,0x0000acfed1f42382,0x00000000000db627}, {0x0006189cf7edbc97,0x0006c36be4a9b658,0x000680236e8f5c91,0x00036d3b8f8074cb,0x0009718545c6c174,0x000757d213bb4d39,0x0003668e1ea3555c,0x000000000008c474}}, + {{0x00063be615177c6f,0x0002773457010af5,0x0001ce08b2f26f1c,0x0000e8c9c25fe5be,0x000182dd0485705b,0x000ac280540f36ea,0x000b923d55bc8527,0x00000000000ad921}, {0x000b6da293461f09,0x0009551586cd4c76,0x00086171a05efa67,0x000605e84f0abcb8,0x0003772dd0dabb4e,0x0004e1d41354ef8e,0x0004917f1a8f795e,0x00000000000de5d8}}, + {{0x000beb4ebddc46f4,0x00073ec72c64fb0c,0x0005bac2d9d9096a,0x00022001819efb1b,0x00068cdde8703c5e,0x000a87aeedf5ab6d,0x000f1975a44e9d92,0x00000000000bcf77}, {0x0009407ed3c226cf,0x0005e8191efbc92d,0x00064a74c9c1339d,0x000e58265cf242d2,0x000180b1d17be62b,0x000de59a9ae99a3b,0x000ce248cbb44692,0x000000000002dcb3}}, + {{0x000a48783de6cfb4,0x0006bf3899558552,0x0009d51bfff43e77,0x0004fd32df8d1a75,0x000376d3fbbf0b1c,0x000fd52bcf16bcc2,0x0001f0d5888916f4,0x00000000000d5cde}, {0x000f03d1ac917a2c,0x000ae764ffffd280,0x000af8be538ef59b,0x0004762ccd57b860,0x00032935106234f6,0x000c642f32233a4c,0x000f34df076095d9,0x0000000000059f0d}}, + {{0x00010c66eff8425e,0x000379580cdfaafe,0x000d1f7ccb185b5d,0x0005f77c327f3e8d,0x000c35353c5f5d3d,0x000258eb105d5339,0x000f79c56fb5fe5c,0x00000000000edce1}, {0x0005bd6f7b6e122d,0x0007cab7aa141541,0x0008987b379beb7f,0x0001491458d9e533,0x000caa7f0f31e124,0x000fda7abdd2448c,0x000a4dec58d3c7f0,0x00000000000c91bb}}, + {{0x0002f037fabc6138,0x000b73bd258d77ca,0x0006aa4d0ec1d1f3,0x0002512e3f966a14,0x0007709d0c2d5b43,0x000658259338bfca,0x000023d142cc1049,0x00000000000636b8}, {0x0007458ca547abc1,0x000cda9ef9400a80,0x000ad926836a9402,0x00063c55cb644887,0x00011cea475bfd2f,0x00067a25fbae949b,0x000a6aa45446031e,0x00000000000dc6a7}}, +}, +{/* digit=61 [{1,2,3,..,}]*([2^244]*G) */ + {{0x00085fa16820f665,0x0009fd699ea2d24e,0x0004f1772a862ed3,0x0004bad66a8b35ba,0x00024233fccb4660,0x000d3cf0c0c779b6,0x0007af578458acbf,0x0000000000096bf5}, {0x0008a325d9d68d07,0x00045a9724244e54,0x0005b4b1e20150f9,0x000a5b8c6be8c159,0x000c774d62c40980,0x000bde24b6230e3e,0x000204da1467d84f,0x00000000000cc862}}, + {{0x000b4d1a75edabf3,0x0007567c51633fd8,0x00020dc66cdc521c,0x000c8dc9ee450d03,0x0008b41a3e2f77fc,0x000bf06898dd2b31,0x000464df6a935e93,0x00000000000a92e5}, {0x0001e3ee6beb3c9c,0x000e449afcd9ef46,0x00031a4b44405106,0x0008ad2c7ea7810a,0x000e550822b2cdbd,0x000606adcfe61571,0x000110744a4f9386,0x00000000000d9d4e}}, + {{0x00006ff4ac15d783,0x0007ef1084276ccf,0x0009d3b1212d957e,0x000dcf5bfb4283a4,0x000db74017eb3752,0x00078fcf8f6b2214,0x00039afc1cdf7245,0x0000000000012265}, {0x000c5dc1b7858cd2,0x000cdcc0796680d4,0x000e05b222bc5975,0x0003a9a504cf7d65,0x0003c93ed5932027,0x000303f1b0c7b7e5,0x0006a4aaf9c36866,0x00000000000cb013}}, + {{0x000bfa5cdf24bf96,0x000411fef389c07d,0x00022753fd218088,0x0000a1437f04a344,0x0009d0169369bd77,0x000377cc3c7438e2,0x000a4f6b265742e2,0x00000000000c369f}, {0x000bb384dc3d9a84,0x00060dfdbcf462e9,0x000d3f52e65bb342,0x000a0b82a9c483da,0x00042de432285574,0x000d1fabe0563fe9,0x00096658ca0e8aea,0x0000000000066023}}, + {{0x000afbbacd3ede36,0x00007746325d090f,0x00094f8b4a38ccef,0x0001c2866c3931a7,0x000a783a73e7d9f2,0x000d82d13c12880e,0x00010e382e1ce28b,0x00000000000ac023}, {0x000fda6b09a40144,0x000b69802d06233d,0x000de8140422422b,0x000367efd4cf75a0,0x0000e8f2f6ed38b4,0x000e72ff4765cdee,0x00070ae0b4d72b35,0x00000000000947d4}}, + {{0x000a35bb9d72eb2a,0x000c5383bda07268,0x00038c9d09f99c2f,0x000717d369f39c03,0x00011a5a39006f3c,0x000ec6c2b1bb593d,0x000202d0f07ecc2a,0x000000000004240b}, {0x00083c5449860db8,0x0001935342f6c7b8,0x0009a1541ab519cd,0x000eb09ccb6a888b,0x000785aa42c5fcd6,0x0004e5895abb7a6f,0x000582952f8824aa,0x000000000005c406}}, + {{0x0001d7b0f8433a5d,0x0004359d6e052cda,0x000fea341e325461,0x000d07d7907cc890,0x000dc4ce5d800459,0x00004f40267d720d,0x0002a83262028eed,0x00000000000d7881}, {0x00055f59d844fe29,0x000fcf735fd6cf7f,0x0001c0c0179cc733,0x00006e8e19a43f29,0x000f19592b76328c,0x000b836f7b97ef65,0x000a9981325f3db8,0x00000000000d6e6c}}, + {{0x000a67318a4b19fa,0x000a63667a71faf0,0x0007be6235b29837,0x000535efc62f7919,0x000b389faf7fe084,0x00071b7f65bc1652,0x00070340cf51683a,0x00000000000d4f39}, {0x0002b576e30c499d,0x00099823e7549478,0x00032769bfa306a4,0x000ee027225b31ad,0x0006fc282f165639,0x0009f61ae7533bc8,0x000803710009d2c6,0x00000000000bf65e}}, +}, +{/* digit=62 [{1,2,3,..,}]*([2^248]*G) */ + {{0x0006c5f4c042c4cd,0x0001ceb29e44bf59,0x0004c11cc5ce653f,0x00004943d2bf689a,0x000a47428dd2d09c,0x000aafac83ab7799,0x0006e0dc558d6be9,0x0000000000087f9f}, {0x00056bc34f65dad0,0x000c793842bcd3a9,0x000241a2ffbfced6,0x00052687e6d47b5b,0x000a4c37eeee1645,0x000c412ceab304b7,0x0002c8dbb3d4e13f,0x00000000000a726a}}, + {{0x00059e97675084b3,0x0008d79d88dffddc,0x00002dc16c994a53,0x000000fce7d606f0,0x000e2fa27fd3b528,0x0009436afc773557,0x0004c755b53dd3e1,0x00000000000e10b6}, {0x00077a15a41de95f,0x000bfe5832664b2b,0x000d15e689d49c17,0x000bf537af3e3dd9,0x000cf47d298c7b93,0x00012136994fafa2,0x000ff694c2ff9a21,0x0000000000033468}}, + {{0x0001b740495480b0,0x0003f91e8c991baf,0x000b4c043871430f,0x0003f6dd095f5b94,0x000cddf3cad27c5e,0x00057aacfed7522f,0x000f5180bb87056c,0x000000000000cc5f}, {0x0006cac5a2f35aa0,0x000e0964def7e61e,0x000e006a84529a7b,0x000192584de66a22,0x000e075f07c5cc75,0x000eade939acaf7f,0x00058f6505c2f81e,0x000000000000e3cf}}, + {{0x0001fb7f00850a1a,0x0001dbe45d81a1aa,0x0000ee7897985bd0,0x000a516b078ea895,0x0001b446476463c5,0x0001e52329f3efe4,0x00022084580e2410,0x00000000000c8ae8}, {0x000539dde4d08a27,0x00077bff4077d088,0x000c1e715cd7b849,0x000eef207e1c03eb,0x00096e654d584df4,0x0006f15964ab3d03,0x000cd5b20056cd08,0x000000000008acd7}}, + {{0x0008f4a5a8f26e12,0x000538caa623c979,0x000e7d0ec3691999,0x000f856ee285bfe4,0x0006d0674ecd089a,0x0006f661277b461f,0x0009f8baa9d9b38d,0x0000000000059066}, {0x000460fe25d6fd2a,0x000d2b164340c38e,0x0003981c9e27c186,0x0003b9176e4346f2,0x00017caad5fc73c0,0x00087803d6a678eb,0x000a94103ff0790a,0x00000000000f7430}}, + {{0x000dd3174fcc39a0,0x00094517075af0fd,0x000fd65c623f7ba2,0x000e9493b86f6398,0x00017f296bf4320c,0x0001082765115657,0x0009000919607a96,0x000000000002f035}, {0x0004b29394d28cac,0x0008f5d13de7c5ae,0x000ea9b2719ceec5,0x0008089c0f58697a,0x00030ca7ca20f1b2,0x0001e1be52adcd1a,0x000ba096c07f42c2,0x00000000000baa0a}}, + {{0x000dc0265a01c54c,0x000233027c169336,0x0003342d9c6b202a,0x0006cd31b8b0179e,0x000a6dd5a4a7e6eb,0x000c82f110d2d27e,0x00002241682c0007,0x0000000000081075}, {0x000f1169b3430f67,0x00019f3903f514b2,0x00091dfa21d2d176,0x0005eed470bd3b32,0x0007e85a62931b62,0x0005ad640a46398a,0x0003a58ff6ef8c80,0x0000000000095bfa}}, + {{0x0006e424a5742fd1,0x0003a8cd6c9f7239,0x000180484ef81e04,0x000b0f589b8ad2c9,0x00070d9c999d9c0d,0x000f84ab4692a8db,0x00006ca407fd03c9,0x00000000000cc9a9}, {0x000b5d757bfbcaf2,0x000014813a7654ce,0x0008615f305ee56c,0x000f2934fa0a23f1,0x0007cb1f8d7aca6b,0x000d10c531dfa3af,0x000c1328036f5498,0x000000000000e012}}, +}, +{/* digit=63 [{1,2,3,..,}]*([2^252]*G) */ + {{0x00034e0d2aae29e9,0x00091a53f1a10241,0x0004dd23936a5886,0x000ed8532976d137,0x00065ee692d130ef,0x0009e7cc1cf5ada1,0x0004723ceda69d25,0x000000000009baab}, {0x00095b8627836d36,0x000237ee5baef4bf,0x000e9b1caaa769fb,0x0008ddacfdaff633,0x0009cec076939e5c,0x000fb509d575c8db,0x0006979afc8ae0fd,0x0000000000085651}}, + {{0x0000da11d13ab853,0x00042a7342f9446b,0x00039f5ef3dbc527,0x0008bf07471c3856,0x000130827f3450e4,0x000779c23729e5ab,0x000d5817199191ea,0x000000000009fa9f}, {0x00077a9b9dc5fca3,0x000827d5799369d5,0x0003bee46a6f5cc0,0x0007d51e85417260,0x000dec720355253f,0x000a16268107a793,0x00031b1c14d0566a,0x000000000008bbb2}}, + {{0x00016132445f3e21,0x0009ef5689c5dce5,0x0006397102e2e73d,0x00079d012eaa5340,0x0006d9271e941af8,0x000e63c34dba775e,0x00024b60f8507310,0x00000000000a686a}, {0x0006624c82f5c532,0x000dded5ca8f992c,0x000b2edebcb9407c,0x00039426d90a7d51,0x000bccd37e76e2ee,0x000412e0fe5b3e4c,0x0000489013bd08f1,0x00000000000ce999}}, + {{0x00009cf56d1d974c,0x0005561d0e01b86e,0x00046be97f98120d,0x000e224b362902dd,0x00029e0a5d16d4a4,0x000e580e945923c1,0x0009544fc2bdd495,0x00000000000a8c3d}, {0x0004daa7a4671003,0x000a1aeff11352c5,0x0005a1c5b6c0120c,0x0007d3eab8f9f31a,0x00066d566c158090,0x0002e6fa7af2c121,0x00003a0082daba95,0x000000000002df9e}}, + {{0x000fc79dfc540dc4,0x0004091b379c535c,0x00090bc691de574f,0x0004208176b3b828,0x0008aaba5cd3d0f4,0x000794f9fd18c211,0x0008944aed5c9770,0x00000000000f4c33}, {0x000946afeb1a61f9,0x000b9b8e67989163,0x0000523ed92edf12,0x000ef39ab5573985,0x0004b71031051b7e,0x000ac89e4175e393,0x000db8943abf4a82,0x0000000000057ffc}}, + {{0x00066f02c7c0f325,0x000f899c2b4cc9e8,0x000f2b3e2c2a45ff,0x000b66b5a352b7b1,0x00043ec757ed067b,0x000d36adc3c7d6fc,0x0003a42f69291cb9,0x000000000007c914}, {0x0004d853fbc3268f,0x00006768c3c3f0a6,0x000dd7feada4c10c,0x0004b025299ea1bb,0x00091a2d4005d86e,0x00017fa60be46b7e,0x000fbeaf865a1693,0x00000000000cdce2}}, + {{0x000297083b5951f8,0x0002b66fcba529da,0x000a6f9544c17490,0x000b97241b4305fd,0x000e25b6af89a371,0x0005f9e8cb2d858c,0x000ef8bab07d5327,0x00000000000d525b}, {0x00073848e93e22c2,0x00084ebf79e3bb83,0x0007a079aa9da5d4,0x0008b66cba1a65a3,0x00038b1eb8b67dbd,0x000a6436a88ea401,0x000e33210ac1b38f,0x000000000003df40}}, + {{0x0000b48bcae58bd8,0x000c45f79f0de54f,0x000db929ac4cc6a4,0x000a68e11571c5c9,0x000e2474faf7d631,0x0002a924a6d072b4,0x000d7a5e043a6d86,0x00000000000b0fc8}, {0x000fb0fbc0056e2d,0x0008bf54bb7f2f12,0x00058e2455c41895,0x0009f4c5909ec050,0x0005abd164608145,0x000b1c69ed28cda2,0x0006f4bb676d018d,0x00000000000ac503}}, +}, +{/* digit=64 [{1,2,3,..,}]*([2^256]*G) */ + {{0x0005d534448865a4,0x00079dbe04c30004,0x000de820bf917110,0x000b54670911457d,0x000b3f7a1757dbf8,0x0007bd62c9e683b5,0x00097e7b89a08a9c,0x000000000000b3fc}, {0x000ba720a0047774,0x000980ac9abfb9ba,0x00007bd7b6be79d8,0x000c335d9deed984,0x000f72603e080aa6,0x000783bb2e270580,0x0009ae70857a946c,0x00000000000caa92}}, + {{0x00008763d38fe0f6,0x00001f071c986bc1,0x000413c627ede21a,0x000b12a61483bc2d,0x00001caf6dd6845e,0x000c21b9217471d8,0x00036d7b603616dd,0x00000000000b9925}, {0x000a996617818c3e,0x0007f433f065211f,0x000b56653ac9464b,0x0007386a9adf49b1,0x00092cb2944fdf61,0x000d9c0764bfeb39,0x000576bf288427b3,0x00000000000965b4}}, + {{0x000b333ccbe322d0,0x000fbce23a199d59,0x000074dc302e3830,0x00000121d15a0063,0x0005f4355a1fc720,0x0007f780715fde73,0x0006913849761f60,0x0000000000018204}, {0x00074bfa03274297,0x0002575756e4ea63,0x0009809c9584a1de,0x00032763d4ce3dc3,0x000ef1d66e006312,0x0008829e6a769d2f,0x0006ef8624ddbac2,0x00000000000d2df0}}, + {{0x000f606b2db32f32,0x000b59ec11596f9e,0x0004a5f37de5fafc,0x00094e131fca111c,0x000896076f45acf7,0x000732588438b917,0x0000daad71035b51,0x00000000000a5d91}, {0x000d85cad07f3bba,0x000a7efaad0e82d9,0x000f829c6ffe7ff7,0x0002426474ce3273,0x000a2aa270da9940,0x0008fa6ebd53b687,0x00041196c8bb78d7,0x000000000006e699}}, + {{0x00044c83de3d28f8,0x0008b2dfc39e611a,0x00099024ec940ff4,0x0002123313aa4788,0x000b9c6acbd11cf1,0x000d54177785b025,0x0001eab4aeb5b8e4,0x00000000000d943c}, {0x00041945f056ffdc,0x000097892db846ab,0x0000f6f8e04921ff,0x00013c61cbd3232b,0x000700b6c34ebdee,0x0003de32b873e0a7,0x000302b279505ae2,0x00000000000c5a03}}, + {{0x000395d621a49e0a,0x000c7430b669fc05,0x00093c4a448f25c5,0x000730c4c33493e3,0x000fca00ed3901ad,0x0004a55168c882c1,0x00027dab2e9c89d2,0x00000000000f5d03}, {0x00020231a1107ea0,0x000f946488bd7b79,0x000d5a9d2b183f55,0x0000e90aecfac7d9,0x000fac6a9e8cf9f6,0x0008bcda84afd1f5,0x00099857a20bf8c9,0x000000000008b46b}}, + {{0x0005f4b3119ad50d,0x0006c1f2f6c1efc1,0x000be8ee71be11e0,0x000e4a6569c090bd,0x000adb3986d020dc,0x000f553eae4f7401,0x000bff1389799485,0x0000000000009026}, {0x0003665df6023a4c,0x000b388067aa6ec6,0x00069e9d017576cb,0x0005019459bf26a1,0x00058b6e76f68416,0x0000d2434ec125bd,0x0000b0d636f9321f,0x000000000008e05e}}, + {{0x00021074c1c07346,0x00068fe1f11ac76d,0x000a3e93fcce998e,0x000015a3c5babf98,0x000d5929ea0a99d0,0x000fb7b7e7e795b4,0x0002e8c9cf68331d,0x00000000000a6499}, {0x000716aaedb25dc3,0x0000d0b7e0dd7aad,0x00064fd90a09f648,0x0003cf034f02b979,0x000a5662f0c86402,0x000058ccaf7dd410,0x0001eda3bb6088a1,0x000000000004e780}}, +}, +{/* digit=65 [{1,2,3,..,}]*([2^260]*G) */ + {{0x0006b5199cfd1ff1,0x0009d140a0f9f409,0x0004971fd020b4cb,0x0007c2304a1f742e,0x0001195ba34e78ff,0x0007f78b4c0568f5,0x000a53e97183603b,0x00000000000f9efa}, {0x0009e6e1cb2c1b4d,0x000bae924d2c4efd,0x0004b415d4c5ceca,0x000e73f6ee37e106,0x000a5e5a1dde4b12,0x000cd64161836fdf,0x0004d87f1b92259d,0x0000000000067754}}, + {{0x000befa999003a02,0x0000e279fd119411,0x000606c204c4310e,0x0007da4da44105e5,0x000a28223fe1d8f5,0x0006f2d2eb1814b7,0x000e06cf2fd241e0,0x0000000000001dfd}, {0x0001563b8cdc2810,0x000a4876a31af711,0x000bd72037bc4e78,0x000d608493a6a0aa,0x0008a88c03e75117,0x000916897dcec808,0x000c57eae1d352ea,0x000000000006e8b2}}, + {{0x0001a9c4fdcf93d1,0x000650254486c9e5,0x0006796f28c8ff02,0x00058ba000f54926,0x000934009fb6fb58,0x0002bde301315bdd,0x000358b18a8e0ac4,0x00000000000f1070}, {0x000b8de4d767059a,0x0004ebeb1db7458b,0x000305d015a22913,0x00014d4e4122b217,0x0002aabccc7b1522,0x000d07571d3e673f,0x000f1794c50f64ae,0x00000000000e66b4}}, + {{0x0004bef0d3847d2e,0x000aa09f8bb05816,0x000388d5b381065f,0x000c7b6076e13ec8,0x00035d5ac07f26eb,0x000ab69e6bda0b55,0x0001fabcb8132248,0x000000000001c0f2}, {0x0006ceb771ee0889,0x0007b7a466528564,0x000e55b024527048,0x00011864c1d7cb8d,0x000f2d08130185dd,0x0005ea0f0096f849,0x0009b2b4f503dd8a,0x00000000000d1bf6}}, + {{0x000f0c6218b7c4e9,0x0008540336b12c41,0x000f74c446fc6c56,0x000048d9c841f0d0,0x00051d617f50c337,0x00017d3794ce6d02,0x00024300fef21981,0x00000000000f95be}, {0x000f7d33fe9f8bcc,0x000ec98de119d3a3,0x000e778f8a8c16b8,0x0005b720f9678bed,0x000d334ace309412,0x000e86e04fc5b57c,0x00003909486527b7,0x000000000006e552}}, + {{0x0001aa0f2e0127d7,0x0007b4ab7a22bd4a,0x00047f417a4172fd,0x0009f95078de336c,0x0006f786924520f5,0x000f1d96ffd28fa3,0x000f0f1de42581cb,0x00000000000366e1}, {0x000e183b180aaf06,0x000febc65fa9d0a4,0x000739e25cf57814,0x0004f3a4a0822a93,0x000bfd05a0aa0638,0x0005ee3ce1c81332,0x000d00bfb12361d9,0x0000000000042016}}, + {{0x000f0b32c63695dc,0x000143b75c45b10b,0x00037d16d187f9f5,0x000227df8e0a114c,0x000e73ddba245450,0x000f246455e69f1a,0x0007412007b80e57,0x000000000003d159}, {0x0007528fbc79a394,0x000ffcbe54d2888b,0x0004e47907e4e2fb,0x00070594cc39f745,0x00032b1b8da9e19f,0x000f68d2680f00ab,0x000829b765aaf973,0x00000000000e993f}}, + {{0x0000b5faa7527283,0x000530a159f61ee6,0x0008e1f9dcf0213d,0x000b8ee1661a2405,0x000ec95c41b36d1a,0x00051eb56cae7f40,0x000250bc3d20407c,0x00000000000e8754}, {0x000ae69a0a837ff0,0x0000281561056151,0x00032df2869a92d9,0x000c1bf6af2a00cc,0x00042d0a56c0abdb,0x000c8959ee9a9425,0x0002f34774a7b77b,0x0000000000019cef}}, +}, +{/* digit=66 [{1,2,3,..,}]*([2^264]*G) */ + {{0x00035e2ab65ec55c,0x000b3114a25c78e3,0x00036712a123ad50,0x00068411e7bd7df9,0x000ad6da54dbc49b,0x0000da3215b0359f,0x00087ea6e6e5f93f,0x00000000000d640a}, {0x0003244eb630ddc0,0x000f7c1a4f6cdf83,0x0008137a92bebef0,0x000eb8e3c0a631d4,0x000db756445ff44c,0x000c8880e0205b11,0x0000d304844e845b,0x00000000000b85e0}}, + {{0x0005c0fc3837219a,0x000cee76894c3764,0x000faaa07cdf021d,0x000fcfbe55cd1e6b,0x00023a7ad5b9566d,0x00087a4ef5421a9f,0x000423a005838a46,0x0000000000023a2c}, {0x000326b6922fa665,0x000ed4b780011f85,0x00024bae2459585b,0x00069e937ced8f3d,0x000d8c1e1eaa2b5f,0x00089cc4d306b621,0x0007748acbbe71d3,0x00000000000f4ab7}}, + {{0x000e8b5d217d418f,0x0002ee557e7c707f,0x000dffc8ed3b58d7,0x000efa4b42b3ab4b,0x000d8e53b32db9b0,0x000b1bfbd9d71dc4,0x000d3fe3a93e4e11,0x000000000009901b}, {0x0009bf805b79f71c,0x000b35f3a5304e02,0x0007d4e3dff75554,0x0007a7bef469e919,0x000278f160dcf415,0x0009189a9f03282f,0x000f2197e4b7f4c8,0x00000000000a967e}}, + {{0x00083fb3cef3edcd,0x000642df6f242046,0x00054e87770387bc,0x000232b372bcc88c,0x00086e428cc59e80,0x000a1b76326ba13b,0x0005f32526ef1f13,0x00000000000dc97c}, {0x0007320a0cda3969,0x000344954867fb10,0x0004f1baa6436a30,0x000fa69be143027a,0x0003cf54f28b7e39,0x00097c2da1232946,0x000099cf0991dc75,0x0000000000052e07}}, + {{0x0004ba256e3a80b8,0x0005fbdfea74874f,0x00090a65af244c4b,0x0005675aaba39901,0x0006a12be348d5a6,0x00018ac5d2250648,0x0006f9766dd428e8,0x00000000000fff9b}, {0x000ff89f1b07248b,0x000174b3b10f3ab0,0x000ac70bef37c1ca,0x00011ed5c9e36bef,0x000364cdfcdd2a61,0x000462f54a99a302,0x000b5fdbfbbe7a59,0x00000000000cc266}}, + {{0x00081f8725c117c2,0x00037dc9daeefc8d,0x0009cda216a5b4ef,0x0002271ca53d53df,0x00054d1aa50b206d,0x000001e99b054633,0x000aa452bca91088,0x0000000000017fb7}, {0x0007b7c6cf80a17f,0x0007afff3af9472e,0x0007c0765e5ecd82,0x00004be24753989d,0x0006e90950c6aac0,0x0008efe5b4a5de2b,0x0009f9b46af0ab73,0x00000000000db445}}, + {{0x000c5196b2aa222f,0x000d15bdb6d3f8f0,0x000389884e011601,0x00008a96ddd9e23e,0x0008467e7577ab50,0x0007555edac1e974,0x000d57e74e35b601,0x000000000009d04f}, {0x00080785cb147efa,0x000908585c6fc59b,0x00035ae9e9b63afd,0x000d1e80b684dd21,0x0009edbaddcc2739,0x000c57660e1ba788,0x00089a464f42059d,0x00000000000a7cf2}}, + {{0x0002dd131270b84a,0x000dd412f64a3e09,0x000b9c94f3cfd9dd,0x000cdc2c2e964e0f,0x000d0011cb01fbe6,0x000228f23e9a3b1f,0x0009a16c30762dcf,0x00000000000be919}, {0x000e7d10046b4ea1,0x000b4732711dfdf5,0x0008160cd8dd88e3,0x0008ba0c6eceba7a,0x0001f3c3d31d8ee0,0x000716948b171153,0x000add65060b633c,0x000000000002ff2c}}, +}, +{/* digit=67 [{1,2,3,..,}]*([2^268]*G) */ + {{0x00000725d23401df,0x000096f178dbeb92,0x000498bead595449,0x000b4b459b46611d,0x0007c72b4a58a6e8,0x0005985adafce826,0x00073321142175a4,0x00000000000e649d}, {0x000052db6dbc2445,0x0003a6bf42fe0182,0x000384cfd9aea017,0x000a58acd0291983,0x0009215492e1b0c8,0x000b5a5f0a73ff32,0x00049b895c545eb1,0x000000000006fcaf}}, + {{0x000a5a0bc1e856f0,0x000f28baef5de481,0x000a8b075c84a181,0x00058d754c8267b1,0x0005ccf703932685,0x0008ee021f924f79,0x000a4be3763f30f6,0x000000000005c01b}, {0x0006ee7749647d88,0x0008acc01a3e928c,0x000c22c3ac36bfb8,0x000512e6c45e3401,0x00084ef433f61ab3,0x00021f5afa978fe4,0x000a4023e2ea018e,0x0000000000011524}}, + {{0x000ecc04426e192d,0x000c692a7fbcd69c,0x000df4111d9bb7a6,0x000db2d02a8feb62,0x0002728cecbe8e45,0x000837662176c0cb,0x00082c33fcfbb0d1,0x00000000000d6f28}, {0x000d967a59df021f,0x000bc75a0f344b04,0x0001f4de7fbff391,0x00090cefa0453b03,0x000ff54c96fbf4ea,0x000f77afc5108858,0x0002fc86b46b5731,0x000000000006bf7e}}, + {{0x000215c3f055a1c5,0x0002fe1d42d93e03,0x000c50c3e0aa63b9,0x00098e783686967e,0x000474a30dbf5f66,0x000d20230bb5e159,0x000f8bef86bb5bcf,0x00000000000403b8}, {0x0001c2443b169c63,0x000fbf249aac89ec,0x000e2f941f78369d,0x000960cefca9a90a,0x00088f449c4b3b80,0x000cfe09ef28e338,0x000d260cdfc61bf0,0x000000000008b22c}}, + {{0x0001733d4c0dd30d,0x000af7013d596371,0x00017f590b1fdd3f,0x000898e35915f523,0x00039fd967d4bba7,0x00073558f9fbe5f1,0x000bea0aba8344d1,0x00000000000ffeb6}, {0x0009792191976aee,0x0004e928f68cbbf7,0x0007210e163722d8,0x000c4e06abb0ac9a,0x000a6895762709b5,0x0004045a401ef3f8,0x000ac355dbe9f79f,0x00000000000ef178}}, + {{0x00094c375a95d7b3,0x000dd7f89c2c3b20,0x000a45fd88318202,0x000e48b12d9f811e,0x000dc7a43dd0be70,0x00018ca3a703ac1b,0x000012a4309a2c21,0x0000000000001943}, {0x0009ce1f49259e54,0x000e6d9597d8737f,0x0001541e69df2d37,0x000760df1561aaa4,0x000e2d91ee39fcf8,0x000bbcb6de1e0306,0x00074699979031e0,0x00000000000cfcc8}}, + {{0x0005490cc3cc0b27,0x0002a4073f24615c,0x0003e0a87e36ac6d,0x00006c18fcec6a28,0x0005d75a73f10abb,0x0008c94883a7d129,0x00074165fd8700f1,0x00000000000732f7}, {0x00049117079dd0b3,0x00098d15d8c801e0,0x0009200404155fcf,0x00007c7120e12665,0x0001f42cc9fd9816,0x000a888cf0e486b1,0x000249e606c16c01,0x000000000001ea23}}, + {{0x00014f7d981a63eb,0x000cb923e948b882,0x0000fdde16bb34e4,0x000011a6df27debf,0x0001d57f4ca2345a,0x0004196e9ba6784c,0x00026df01b370311,0x000000000001aab4}, {0x000a2240acace9dc,0x000d82ffdc977a4c,0x000a7e87839c540c,0x000c01216f09c1f8,0x000e5b9cc0b65ca1,0x000150569612021d,0x0003ea910e10e95e,0x00000000000e2ab9}}, +}, +{/* digit=68 [{1,2,3,..,}]*([2^272]*G) */ + {{0x0008093fcd51c0da,0x000aaac58c9dfd06,0x00005aab38065215,0x0004afc2cc4252aa,0x000397c2bdf932ef,0x00049e19a08bd48b,0x000496ac8a7803a8,0x0000000000075c31}, {0x000fef0d3072e592,0x00057733dc7dec06,0x000d0f9063e4be72,0x0005b0847be21a8a,0x00055ebf426f6d9c,0x0004e9e5bfab0fdc,0x000089d60748caf9,0x0000000000069366}}, + {{0x0000e5d6509e91ef,0x00048b06c17a3a05,0x0001e37973be9551,0x000ae990038aeb31,0x000b58b402ca2440,0x00077732a83bf711,0x000932b8763e00b5,0x00000000000f651a}, {0x000cf91f09ef177f,0x00070be02ab9cf6b,0x0005ba59eb97ec90,0x00072f64283f8100,0x00072365da5e7ed2,0x000dc6beb098cf05,0x000d72d90e6f1805,0x00000000000cf7b9}}, + {{0x000bed14c2e5c1d5,0x0009f89b2edba20f,0x000638f30da8440c,0x000d9ce8943fce4f,0x00045a2ff9961dde,0x0006e87feaaf07ff,0x00008c69e60f92c9,0x00000000000b9ee8}, {0x000d99ec41a8cc2a,0x0005279d6d8c67c9,0x000b28385a21a71a,0x0005267350b7fc9c,0x000d8a2abab0c8a6,0x0004faf7ce9fcb46,0x000c8a57c3bfc62c,0x00000000000a8207}}, + {{0x000f000e371891b6,0x000b56f101762b79,0x0004acb33b163eb0,0x000e2aaac1dc8274,0x0009f835a1b62c58,0x0009915a451410e8,0x00048d981333a762,0x000000000000c141}, {0x0009810640a6c340,0x000184d20b3d37fe,0x000e4eb008d53e3a,0x000b68c2a2645f81,0x0007add082eaa664,0x0005ff2067a6bc85,0x000fcce7467dc63e,0x0000000000004e2d}}, + {{0x0000cfcf5d6b5960,0x000cc57fe04dac14,0x000998e82d998b8f,0x000bd40e0a341ea4,0x000b9c24fdba5128,0x000c1dea81adf3cb,0x000ef2b1ce8520f4,0x00000000000db22e}, {0x000911256ee6f617,0x000b2e9f05f28713,0x0000043d1376f349,0x00052d3f63bfade4,0x00006cbcc395a2ad,0x000714bd81b43575,0x000ca6fd1d7e0666,0x00000000000a40ea}}, + {{0x0004ca398c06d768,0x000c23d0efae2846,0x000878540a1ede95,0x0005a253f00eeef2,0x000cd86161cea5cb,0x0004a389ac2f9258,0x000159d2b0865f5c,0x000000000009b1f2}, {0x0008e174cdba2f0b,0x000245a758a60599,0x00087073a8d80dcb,0x0005a4949d301c92,0x00061c4bb89b8865,0x00060259c129647e,0x0007d106f0665ec3,0x0000000000006619}}, + {{0x000cae6e407127c8,0x000070262c90bff7,0x000ca8e9829543e7,0x000f8fb1577634a8,0x0004597668ad6514,0x000ff6ecf4678d63,0x000dfc90213b637c,0x000000000007518c}, {0x0005a2999ad1c0e5,0x00071d006db8f897,0x0004562511312669,0x0003aeadf6356731,0x000f1a366456acc7,0x000e73aeaaeaa4fa,0x000a480df51aea1c,0x000000000009a8e4}}, + {{0x0001074c67a33fda,0x00086a23545689bd,0x0009b9de484075b5,0x000247b132c568a3,0x000dcc4760bbab0c,0x000ae821b129a5c9,0x000f8303eba46726,0x000000000003df1c}, {0x000f0c5101b30f29,0x0009f010813cfb68,0x000cc25f999a807c,0x00020b31fe4b6007,0x000398dd32399073,0x000abb34cda3bfeb,0x0005f123f1ed1641,0x00000000000946f8}}, +}, +{/* digit=69 [{1,2,3,..,}]*([2^276]*G) */ + {{0x000261be9524972c,0x0005728524830014,0x000a9e4b0c851ae7,0x000fa9900e4f78b6,0x000b54a3f7a33a66,0x00065a79afd8a65b,0x00069a505d3f6319,0x0000000000018914}, {0x000675265cae6514,0x0008278367cbbd6a,0x000c906352414281,0x000b3f6f5af173f1,0x000a5ca36597e41b,0x00099f79557cb03c,0x000ef127f86cb87b,0x00000000000ad4dc}}, + {{0x00092527cce78878,0x000b138a49bf4890,0x00076c4f31c0490e,0x0009eca01b2f7c2e,0x00005be8359f5f70,0x000646b3fc479a65,0x00011b6b6a22bfd6,0x00000000000cc5b7}, {0x000c9aa1253a31c2,0x000d18e7dbc638f0,0x00085601e946c9b1,0x0007e04030271eb4,0x0008e22fb3c72d3a,0x000e0d46cb08b597,0x000915860d62789b,0x000000000001d49a}}, + {{0x000e171e53bdf97e,0x0008556d65155b11,0x00013206465d7fcc,0x000049f10dfaeae2,0x000ae30b70b8ef9e,0x000c6a56219750b3,0x00002611ed860015,0x0000000000012097}, {0x000714d4a4467bbb,0x000e279559d04c7c,0x000363f176594d50,0x0007323ae8fb3c53,0x0009a4111f88d6fd,0x00010a683834639c,0x000e9afc69a029a3,0x00000000000255f2}}, + {{0x0006e6e51e1dd6f5,0x0009766797f569dc,0x0007480d93f86f88,0x000f4a531adb034d,0x000e5d185cfc18ee,0x00053c27bcf1cb5e,0x000569f596a59bbd,0x0000000000069002}, {0x000a4105949da385,0x00053d9433e78e1e,0x00022ac847a15a5d,0x000d7ed65d68ece1,0x0009fb2aded8233c,0x0009b750e201bbe0,0x000f25987f4975ac,0x00000000000337f7}}, + {{0x0009f8de8126d12d,0x000626fb6b7bb82e,0x000f3dc00034e80a,0x00089d8316ff5318,0x00079894d65f98ad,0x0002801704800ee1,0x000537034ea3c448,0x00000000000c30be}, {0x00066423e58a8102,0x0009243c2d27d6b9,0x000c30559bd3f060,0x00040105eaef29b7,0x00017b678fc76545,0x0003012701d8f07a,0x00036554bc6f737b,0x00000000000e9473}}, + {{0x0005ee6b65e3f046,0x0002a3561b3fe3d4,0x0003adfe7b57a7b4,0x000831347fb79d76,0x0001c48002ed4374,0x00044dcb0a7f497e,0x000e0686221cce2c,0x0000000000043785}, {0x000bdab4ad1119ad,0x000279ee9061f323,0x000458b83a9d33ec,0x000e85b76f7f52c7,0x00090fd6c65f1d8d,0x0001f8a3939a713c,0x00040af74ca06771,0x000000000008ebc6}}, + {{0x000c37bd5d97c8e0,0x000841942c2ff6fd,0x000f5c9ed6475a4f,0x00052b5771a5972c,0x0004363c2048f3c1,0x000db8da24cfb1e7,0x00043dcd2ce8249d,0x00000000000a5ee4}, {0x00044b387eb3689d,0x000606fa84b3cea2,0x00069e100b1b94b7,0x00071368e13869e1,0x000c96f0f8a12e45,0x000d93dc039ac6a0,0x000ec9e24458ac42,0x000000000005f60b}}, + {{0x0001937d698b9991,0x000c7514cc1dc470,0x0007f6d12f9a74ce,0x00022ac2fc9230fb,0x000d92e23e21f0e4,0x0006d1ac0aa50a1a,0x00049370ac46d867,0x00000000000f9f54}, {0x0006e32302426e7c,0x0002059a6b86a840,0x0004338952e5938c,0x000cff13192968cc,0x00040b342dd8d3d9,0x0001da44113e700b,0x000760fd5dc7bb3b,0x000000000002c883}}, +}, +{/* digit=70 [{1,2,3,..,}]*([2^280]*G) */ + {{0x00067944f366244c,0x000d59e9af31428e,0x0009cbe9f6c4942e,0x000bd92582864947,0x0009b2e09205204a,0x00056b1017995988,0x000f260c727a3dfe,0x000000000008b657}, {0x000fac647b56af7e,0x0004cde35d954514,0x000e44c1af2c53f1,0x000261f36019feec,0x000966511d8ffa03,0x0001148b9afda42d,0x0003e1f63211dd82,0x000000000006f13a}}, + {{0x000fe40d0b19b4c9,0x0006a3997900ec16,0x000abf1980c5bfc2,0x0005b0a661f3c579,0x0008aab1292b71bf,0x000d993f244b1338,0x000966cbe80cb9d4,0x000000000002203f}, {0x000fa9a95e9148b0,0x00053bc82a651a1e,0x0005a004b90c2827,0x000c7e7502f0003e,0x000056a38f73a388,0x00087d9b09c704ef,0x0001f393cde8a734,0x00000000000ec11a}}, + {{0x0004e6a4ec3efae0,0x000e047fe47986cd,0x0007fd532d6e5353,0x000fa8b9a8671744,0x0000514bf471b76c,0x000dcf1fc9bc87bf,0x000fa837392dce71,0x000000000003ad1e}, {0x0001e59290757ebf,0x000cebbea5a33841,0x00075b7402f28c87,0x0004ecd9b1665aa2,0x0006e329df225b3d,0x000aa808de7b0df2,0x000faf18fb438e0b,0x00000000000dd516}}, + {{0x000d8d8f4a6b2b66,0x0002a63f27a255cd,0x0003341f52773d72,0x0007cd965f9ce38b,0x0000635ba3005d31,0x000343662162c92e,0x000ac85259f64ffe,0x000000000008e614}, {0x000a1c59580d5fe2,0x000e397fb55baa90,0x000365cc03ff132a,0x0007325780618255,0x00086c1e6306a57e,0x00067b14c892f6b3,0x0008c5f12e4c0723,0x00000000000c83a4}}, + {{0x00040eada4657ebb,0x000e31e6f9a95314,0x000a04269b290326,0x00056256e8b41991,0x000b7a4a53f9365d,0x0002b9b16e1b9c53,0x00070155dc1d50e3,0x00000000000bb6d5}, {0x000b5b1121311bd1,0x000984c270249c99,0x000f7b0846375c38,0x0005b610689902bd,0x000fa4cfc5a819a9,0x000e5b424dd5706b,0x000a87d33bdb7314,0x00000000000e69eb}}, + {{0x000fd0482abbdf6b,0x0004f6897401cbf6,0x000652cc2d40814f,0x000b32939e2b43ba,0x0001bf809331e511,0x00058f8f43952286,0x000727815f6c1dc9,0x00000000000c213e}, {0x0009a038bd421fe7,0x0008ceb09ae59065,0x000d25d81924ad76,0x00015a2a20f86c7b,0x00069aa308078f48,0x000758878748a176,0x0003d7f1f46a211b,0x000000000006f6fc}}, + {{0x0005626ca6bcb860,0x000ac75ba2f9acbe,0x000c234135494b93,0x0009e0b610d080fd,0x000333a99c2f3c33,0x000580aff614ac29,0x000a3aa5e23af2aa,0x00000000000e1fdb}, {0x000ca5f5b322ee30,0x00065aa603365011,0x0001b05f57ab1134,0x000524456330a336,0x000b9e025aa3a2b2,0x000a0d5cfc396b5b,0x00083babbf77faf8,0x00000000000ea31c}}, + {{0x00047a49edb97340,0x0002d300b04831fb,0x0008ca060cb164c9,0x000c5420cc10a74f,0x0004eff9d01017d9,0x000aa12857f637e8,0x000114d732aa1e27,0x00000000000e2a77}, {0x0008fc5f031be351,0x000d262acf1585bd,0x0001b0dfbfbd395e,0x000c5d22e6eb2db2,0x00028d18263d88be,0x000bbd6acaec2720,0x0004c04b687353e4,0x000000000000ff7e}}, +}, +{/* digit=71 [{1,2,3,..,}]*([2^284]*G) */ + {{0x0006ecc79d0fef75,0x000dfa6e19d09c24,0x000097d1dea89821,0x00080950ee527a58,0x000b7c4278a6ef73,0x0000798ff78b7174,0x0000cac133d867e7,0x000000000009e923}, {0x000f7954ffb5e165,0x000877431eeef1ad,0x000728c869881eba,0x000306a8d6af3f83,0x0000bbec076af7a4,0x000257ca3633bb5a,0x0006636d07623e7a,0x0000000000021c00}}, + {{0x000f8db3d9e25bfd,0x000ed35ae891ae18,0x00094f42c82c0dd3,0x000b3cae63745c4f,0x0005f218d139c2da,0x000f3b0e255b8c18,0x0003fa6ab039c889,0x00000000000c644c}, {0x0008fe1e4d75d65d,0x000d9c8496f1ff0b,0x00010bc25fdaddb7,0x0008392921149191,0x000fef73f3455c67,0x000fcfb22e962e52,0x0009113818baf354,0x000000000009d4bc}}, + {{0x000af695e08e48a8,0x000683a35938698b,0x0003114a51ff94f2,0x0000f82899d204cf,0x0009158780b5fcce,0x0004edd4f4ca4158,0x000c34c28f1bb73c,0x000000000007a6ae}, {0x000b281e555c9d40,0x000676d4783df7ee,0x000d1f8a08acdccd,0x000c2bcdf6d7f923,0x00000f356ca3b7b8,0x000b8e964f0a8d47,0x0000e1229894bfde,0x00000000000d3aa7}}, + {{0x000d000279a1a838,0x000b77f1f985b888,0x000a43ce9b20f9f6,0x000930a5188fcf70,0x000ba3a82904b40a,0x0001510273fad79a,0x00017b7bee8d22f3,0x00000000000c2c3e}, {0x0003fdcf0848373a,0x000041fbe06e92b8,0x000ae48de4a30594,0x00090ec2d9cf7a56,0x000a9bd062c5fff2,0x000d353815d0deda,0x000eda080344abd7,0x00000000000f5cf9}}, + {{0x00012d81fb28e752,0x00018093526223ac,0x000d23e13369699e,0x000d26ee94c9abed,0x0003a20b52679a62,0x0003d5fa4425b86b,0x000fc68164f9c8e7,0x000000000003b12c}, {0x0006c5fe565db2c2,0x000be6cc34bad21b,0x0002901d6245306f,0x000f50970e4963cb,0x000cf0813b31f80f,0x000f1ab8ee403ccc,0x000b92a72ab5f584,0x0000000000070f15}}, + {{0x000e213ed814f6f4,0x00034170098ddc99,0x000e9c62ee04edbc,0x000b04b8660d7749,0x000b5152ad1e35b1,0x0008e92aa8b4b6c4,0x000f4e0b0bd1c24a,0x0000000000088172}, {0x0007445379f53f7e,0x000d513a76898c7b,0x000f4e73336bcd42,0x0004f973dc17e72a,0x00060a7124b24b39,0x0008a372c1b0cd16,0x00003b7549880cce,0x00000000000e7126}}, + {{0x00055aa45f74a004,0x0008dc14a3abb979,0x0004a411c66e9f88,0x000f1828a4be08cc,0x0009deca5a89e49b,0x000fbb36be6e7bf8,0x000e57a7326cf88e,0x00000000000c7f6a}, {0x000da2212d19f21a,0x0005dceced37b724,0x00062d313930a591,0x0005b3fb62a62e4c,0x000793e5c0e1a7e6,0x000b8d77382cb79a,0x0002f014c9a99461,0x000000000000b05f}}, + {{0x000a2dd7deb1dde9,0x000a831d31f799b5,0x0008e986349103b0,0x000362e52f04d685,0x000921c7b6511793,0x0006f47f65808e70,0x00040533fe9123ed,0x0000000000058f71}, {0x000d5dbe8f1956b4,0x00061821d30451f3,0x000e7b4e166f399a,0x000db145b2048aa2,0x000ed7811330932d,0x0008fd85f17d989f,0x000434032ae4cb12,0x000000000007d1ef}}, +}, +{/* digit=72 [{1,2,3,..,}]*([2^288]*G) */ + {{0x000acb4203cc71b2,0x000a9070d34e112c,0x00047b523d821ba6,0x000f9b95eb8ad292,0x0006a6a45dda269a,0x000df41e0dde0a03,0x000f2aa18b528e63,0x00000000000a260d}, {0x000c22d6a6aefb01,0x000d424a67709f9b,0x0002a9971343d570,0x000e2a1968444330,0x000d131a6cf073c1,0x000e794543660558,0x000dcdb0289c832e,0x00000000000c6d8d}}, + {{0x000d86ee6f475d5b,0x000d30ed1082f1a9,0x000a6711ccf28680,0x000ef0eece773534,0x00064bf0c426a35f,0x00011b2fb76adaa4,0x0004d3fbab929aa0,0x000000000009d8cc}, {0x000bb954fd395b2c,0x000dabe6d7f2076f,0x000a4cbdfe911e02,0x000cba5106ceaac2,0x000a45258697c7e2,0x0005945b0fe94528,0x000f7a7109b17d07,0x00000000000cae17}}, + {{0x000ce6be22a12c0b,0x0008a6855029961b,0x000a3c801158be2e,0x000d2b4a8e92e70b,0x000f0cc9fca9f601,0x000bc2e96fd02799,0x0009b889f99ca013,0x00000000000ff9db}, {0x0006b1bd37f91ceb,0x000616ae70849faf,0x0005d65bd4da44b0,0x00092a4e0945e8f9,0x0002ecea36f6394d,0x00018a069ddab675,0x000860650b74d60d,0x00000000000ad650}}, + {{0x000bf5b4a4380ef4,0x0001cfc1faedb8cc,0x000e6ba03d77f4c0,0x00085456416defe8,0x0000c58653e7f004,0x000f6f99e75de777,0x000c07dd2dcbebe2,0x00000000000d159a}, {0x000c9af1998a9b17,0x000a045dd713c4a8,0x000e1c3678ddf502,0x000756b6b962b38d,0x000bbffd3f0c3f73,0x00008ebae3cb9f53,0x000dbad723c40b7f,0x000000000000c3cd}}, + {{0x00003107748ef382,0x00058a804b2f0156,0x000b319b886fb1eb,0x000f4fd353970be7,0x000712854a9131ba,0x0009713983481c8b,0x00094ab0424eecf3,0x000000000000f4ed}, {0x000648ea129bd87a,0x000a992b74ad013d,0x000692f5d2444d08,0x000b28070a5f0c93,0x000b9f5101ca6741,0x000d0f81e46c12cf,0x000288e7014d7901,0x00000000000d758c}}, + {{0x0000c39f81600214,0x000686a565269228,0x000c17ee858626b0,0x00011a7d8c6ddb92,0x000b2332347b0bda,0x0009dff48b7ebfaf,0x000c83cf791881e9,0x00000000000dc357}, {0x00082704ffef2257,0x000fe2a264f8831e,0x000c9352584eb7a7,0x0008260bb47d4188,0x000c3d8170326d47,0x00005f901544de4e,0x00063cac7150a9dc,0x0000000000075c09}}, + {{0x000bcfd07a0f22c4,0x0007c58805ca858d,0x000b1c2b2351e4ce,0x000c9a1349cfed00,0x00017800a980f9cb,0x000847e9d67bc7b4,0x000c48e7e6dbc0f6,0x00000000000af379}, {0x00047d63263e04cc,0x0003c80b6fc048b9,0x000808eb22fe8f7f,0x0001ed14e9dc24bc,0x00073fab87e6d2d1,0x000c630ae48d74f1,0x00020a1feca5af50,0x0000000000062d06}}, + {{0x000e56ad2bfbbdbc,0x0008ff6377e3bcab,0x000f30b92835c3f0,0x000ac42c21c0cf1e,0x0009857d8edb7f85,0x000b77ed67a3e31c,0x00010b2eb1076327,0x0000000000038dae}, {0x00026ce2d0dbce6d,0x000dced7c6fb0aad,0x00066cec0d07ac1c,0x00035504569e7309,0x0007bedae3ab4dc4,0x00027e463aa82fb9,0x000a1106d37bef4f,0x00000000000f0c35}}, +}, +{/* digit=73 [{1,2,3,..,}]*([2^292]*G) */ + {{0x00011f784b3a0046,0x000066edf5561b45,0x000c8578d54224c1,0x000ae56d8d66d29a,0x000ae9d58eb27699,0x000f9176d5f81602,0x0003b1d0c04874fa,0x0000000000052315}, {0x00067980492adf03,0x000309690930fad0,0x000146ea47ef3788,0x000725873bcba90f,0x000de902e3292d0e,0x000bc27b957efe72,0x000c093fcd598676,0x0000000000093940}}, + {{0x000cdd973668c530,0x00077c68b641e912,0x00064e1ee190b62b,0x0008cbad89ce6688,0x00006c269d8eeda9,0x000f3402315f84dc,0x00034af0eb685ecb,0x00000000000c0326}, {0x000bc909db075650,0x000562e3d7fff094,0x00012e59576050a3,0x000abcfa3e050e0c,0x0007c001dcd9f02c,0x0006d2d52ccd2da6,0x00041f50aa372306,0x000000000007224a}}, + {{0x000ec1f1185b9762,0x0007e4b1f8698c94,0x000d0af36ed765ea,0x00049f56536e156a,0x000109c95a75ba1c,0x0006d9af131c3163,0x0008b7268a308f0a,0x00000000000fa9ca}, {0x000f13b9ac944487,0x000b1c3b1cbccb86,0x000446bb4e60be0c,0x000c7b334396850c,0x000a048a69431b4e,0x0005820bf42f21d5,0x000a580bf948f55b,0x00000000000b48dc}}, + {{0x000b86ea9a29c966,0x000cd04418b58cb8,0x0002d1612a6d4e73,0x000e22c7c65a6da4,0x00051dbeffe45f87,0x0003cc05caac7106,0x0002c4ff619d64bd,0x00000000000e65c9}, {0x0002126bd4b5ea52,0x000d123af2b21062,0x000e0691b9e9f16b,0x0007efac6478c561,0x000af6cd140ec34a,0x000a0db2e57cde95,0x000b7664b74575f5,0x0000000000075d7f}}, + {{0x00078c972345257c,0x00058a9532cb266c,0x0002b89ba64976de,0x000067545b1733a4,0x0004cc6e65dd74ea,0x0006ff2aa7f37f02,0x00079e6612b1425c,0x0000000000030374}, {0x000f1ac7f7aa2f0c,0x00097e512bca4fe9,0x0003049963042b95,0x000bd29a259d6515,0x00077373034f5b97,0x0005aa640215eb65,0x0008b779da0fc488,0x000000000007e435}}, + {{0x00041294c6ef7a53,0x00023ce3cc51372d,0x0009af7f675a386a,0x000d981efc9f71b0,0x000a20bea9baa9e6,0x000ef7c1fc01ded6,0x000a5cc4b7e6424f,0x00000000000f9dec}, {0x000954a122b474b3,0x0002aa0a43d0a3de,0x000a0d55c109a592,0x000a173b5d57bbf3,0x00056299978197ce,0x000ca2bb13c3f30f,0x0002e069a7820a8e,0x000000000000ff69}}, + {{0x000deeb8201c384c,0x00029aca3f998176,0x0001642e9c9fe922,0x000dbd63d82750be,0x0002faa3400be031,0x0005fd54bfb2552a,0x00030374e9b5d365,0x0000000000035894}, {0x000839a12661f3f2,0x000a15a575a4221a,0x00056e3aabcc078b,0x000ada596e1e1175,0x000d0f305dc3ca6f,0x0008d7af5126e9af,0x00069391da8d7305,0x00000000000d8353}}, + {{0x000cb9cfccb5f12c,0x0002c28405061c1b,0x000211e4805b5eb0,0x000944405547f7c3,0x000d3441e472789d,0x00044ce877fa445e,0x000d14feb198254a,0x00000000000129ae}, {0x00073247d16dc8ed,0x0008622b05de2de2,0x000590692ea55dfa,0x00052caa83f7f094,0x0006cefaa757fba2,0x00017d8060f6ad6e,0x0007ade1ad187165,0x000000000008e9d9}}, +}, +{/* digit=74 [{1,2,3,..,}]*([2^296]*G) */ + {{0x000c5ac4350713f2,0x00062cbb991946e6,0x00025875bf6be408,0x00024aab6ab6c041,0x0000bc41f4b7d50b,0x0005496cf419d281,0x000ac0e2e88d7a29,0x00000000000f2457}, {0x000a357ad05dad34,0x0008e838e4a3533c,0x00068ef45f1df4fb,0x000ab1c7a5c5fdd4,0x000dde34cb8d9fa3,0x0009105fe324798c,0x000249bfa5a9d088,0x00000000000fd0df}}, + {{0x000b286ea52474ce,0x000419974759bb2f,0x00039b1e3f8aef8e,0x000c58ec64941cb4,0x000a01d055758afe,0x000448db52d7c57f,0x000440c07c74cc5c,0x0000000000001bb1}, {0x0007977c86a9c43f,0x0003893162d96796,0x000ee52587841e45,0x000c57e6af18dedc,0x000f174a30894f28,0x000dd626536faed8,0x000d7b702dbd64a3,0x000000000006adc0}}, + {{0x0002a70e7d991630,0x000854981be688ec,0x000de0f53a95f8f1,0x000e05074cff00d2,0x0001b52c7f9c3168,0x0001c7986111150a,0x0002ad0db2ed84a5,0x00000000000e6127}, {0x000c16b3e4d1fe2a,0x000897443d935440,0x0002c51d6c8aaceb,0x000ef4e866b5e41e,0x0005b965a67071e0,0x000e1aa7705c57a4,0x000e3285bdb58c70,0x000000000009df3c}}, + {{0x0009a11de31a0e3c,0x000c6c41e4aed12a,0x000342590d5103b5,0x000ab7b8ed812d38,0x00081afb58b8f5ea,0x000efcc3a7801fbb,0x000b91be5cdba671,0x00000000000cd10c}, {0x000761b96524aa0d,0x00013882e821e20b,0x0008d1b11d20578f,0x000de2bd8a0f0393,0x000c857fd3a03afe,0x00060f767e20a117,0x000da1ebaa2aa430,0x000000000002b9d1}}, + {{0x000d9fea4e493f3b,0x000e070e478fb44f,0x0000254b6a02edec,0x0002b005be36ea75,0x000f6717e06aaec8,0x0008f197782618b8,0x0000d12db1f6d400,0x0000000000016b39}, {0x000ddfa794ec9644,0x000bafe86fd4d041,0x0006bc15de0d2bf6,0x00023e70c0491593,0x0005ed4562356cbb,0x000d836efd05be7e,0x000604601374069b,0x0000000000032e5f}}, + {{0x0001ebab46405b61,0x000f009db138cd2b,0x0009fd7df23c53c0,0x00047794feebab9a,0x00017fdb1e710bb6,0x000f1f65dfce4aee,0x0006cd7d5c0c61a7,0x000000000008d02d}, {0x000621234d968fad,0x0005c7cac0485224,0x0002b8dbf87d2555,0x0009a59db0aa864d,0x0008ff94f8b5b587,0x000c97f8e75f391b,0x000232a6c994ae49,0x00000000000d690b}}, + {{0x000e6486f79bfa96,0x000e507c60c742b4,0x000d5aa8cb9b07e8,0x0000a09dcba89982,0x000a802ce50ca8b0,0x0002144b0a77f1f3,0x0000db6c4050df2a,0x00000000000ab1b1}, {0x000a93fc6ce00720,0x000e89025d46fd65,0x000c1e4037fdb9f5,0x0002d629ae2d5361,0x0009eff0f6a9bcad,0x0005fa5a07a15282,0x00039ce87345cd92,0x000000000000e840}}, + {{0x000cab548e246919,0x000d15017e1454d5,0x0005f67186d3e9b6,0x000059dfa98234b5,0x00099985af982541,0x0005b7b1c4bf344d,0x0001ad39737ed67b,0x0000000000087c41}, {0x00069e3ff4c10920,0x000022fad8cecc3b,0x0008e45c000ca718,0x000d856404043f8d,0x0005863927a11a53,0x0008921216c1f07a,0x000c8f38d3a4f4b6,0x0000000000001976}}, +}, +{/* digit=75 [{1,2,3,..,}]*([2^300]*G) */ + {{0x000dd4906efe62a1,0x00063bcc8842253e,0x0007a247757fdab1,0x000a1ef2571d6b1e,0x0001739a956e745d,0x000bffc16f01c292,0x00072ceffd8065e4,0x00000000000f36fe}, {0x000478592361b14b,0x0003ff49e90d1ebe,0x0001338e12b047f8,0x000b583b7240d91d,0x000c03f8387fcb78,0x00000f2d92e1a7c3,0x000a8416566c2232,0x00000000000aaf80}}, + {{0x000f32d335de93bd,0x0002d7a1a522aee8,0x000b425f9e81403f,0x0004abe3d4ac4abe,0x000337f996e841bb,0x000727761c52950b,0x0007d201d991e679,0x00000000000978fd}, {0x0006e2e2907ff054,0x0007486bad5e5c68,0x00010431aa78725c,0x000eb7705757453a,0x00093939df7758ff,0x00078ea0fbb18612,0x0001d3bfdd394a6a,0x000000000006e33e}}, + {{0x000c3ccc7d56f63d,0x00076db1cffd2381,0x0002131b488cf4cf,0x0001e5aa3c1db85a,0x0001ac55d391a418,0x000d270d0e5183e5,0x0005462793d5efef,0x00000000000c0631}, {0x000c80c1d49264dc,0x00008ecb40a58378,0x000aa60ec18fee5f,0x00014ab6c5830fab,0x000ca03f201ea89b,0x0008c54f64dc477d,0x00066e5604f5dac8,0x000000000002acfc}}, + {{0x0003e9c742a5516d,0x0009da7f29531cb5,0x0007fe865f9eba30,0x0004ce9efae348cc,0x000cac52758dd2f5,0x0002961a45acb931,0x000fe1287b3e18d2,0x00000000000748f8}, {0x000f0554bb131f76,0x0001d1b62b340cc5,0x000b08ad2a8e1d6a,0x0008dadbe8da8486,0x000954f18276bad7,0x0004bfdad85fa710,0x000237e2cc9d287c,0x000000000002c75f}}, + {{0x0004d2c2f1f927d3,0x00033283dd7ef2a5,0x000a91f40054b9d9,0x000c095870282aba,0x0008f4e36324ba08,0x0004cecf2b02b00d,0x00024e53aaddc060,0x0000000000084ddd}, {0x000a889fa3214142,0x000ca34bc00a03f8,0x0000c263c00349e0,0x000eaaacef68f74d,0x00023c0293515228,0x00042e3b740a790d,0x0001e0e0e9af5c84,0x00000000000a9f14}}, + {{0x00087ba6916bcaca,0x0005bcdcdd7d6721,0x000b58247a460e0a,0x000c569c00ce5c40,0x000fc589d0c75923,0x000fa6fe6099c17f,0x0008c40335eeea89,0x00000000000a4e86}, {0x000a7415516e07c0,0x00024b7037325a9f,0x000944c467e9c1bd,0x00032efd5e26d28a,0x0008de919cbedf31,0x000bbb18d7ed994b,0x00050812df67cd6f,0x000000000006baff}}, + {{0x000c3f58ee594ee5,0x0008b72f4949e0b4,0x0001e5bef192dfaf,0x0004726092a58066,0x00010f5cdb7271b0,0x0004bfc49ad5c3c1,0x000be7d4951f9e55,0x000000000006131e}, {0x0009f77a3840f92f,0x0009237da59f303d,0x000ff0c06b034503,0x000962a4d89fa660,0x000e71d824f5d020,0x000050e312610c61,0x000bfb9c917da73c,0x00000000000e6e7e}}, + {{0x0003a9eb85c489f9,0x0003af2ec9ef5122,0x000d827990bab108,0x000b2387d2ac6e41,0x00020318a3b790e2,0x00084a7a0cf682b1,0x000d057fe76089ea,0x0000000000016546}, {0x000bc6c0a0421345,0x00041f987118eca7,0x00079f0cde4a8da9,0x0007774888a7e8d0,0x000a63ec5831544c,0x0002fe985bd2647b,0x0003e8b479cea3fd,0x0000000000028d16}}, +}, +{/* digit=76 [{1,2,3,..,}]*([2^304]*G) */ + {{0x0005b08a8a2a82b4,0x000fa04baa17598e,0x0006a0661405c62f,0x000031dc6cd61096,0x000030bf87a5c4a6,0x000e497ccba2534d,0x0006657ebc6e2131,0x00000000000851fd}, {0x00018d257f3e221b,0x000e69ecae010ee2,0x0009788f9b15d390,0x000a9c22fa527837,0x0001236d442e39e8,0x000c289ce74c9380,0x0009cf8b21ba23b1,0x0000000000025c76}}, + {{0x000aafc618669c84,0x00037761e10e2b1a,0x000c03aea1a1a99e,0x0000ff43dbc6fcaf,0x0006cbe0ba8aa087,0x000f1ff6a9765128,0x0005b647d46075c0,0x000000000003abeb}, {0x000f7f59840e13b0,0x000af1234c80e297,0x0000d9f885a37dab,0x0002d41d82ab780a,0x000fde426e50399a,0x0006dde778a03afa,0x0005eca2bcb109ca,0x000000000000618f}}, + {{0x000adbd12143606f,0x0003e0d27500f7d7,0x000cde21714cb370,0x00078f8610763e40,0x000f47ef08feee75,0x0007d04c06e56078,0x000aa3a8d03a7f04,0x0000000000027cc8}, {0x000677d27af9cf4f,0x000056b08eb37c00,0x000a4a2b5d3fdd3b,0x000869ae3f10ba99,0x0003e500e8466f9b,0x000171b1a3bfb983,0x0000456d975557b9,0x0000000000018fa0}}, + {{0x000e348941e625d9,0x000fc15d47e52aa7,0x000dc7038ccf5369,0x0006a3070cddb11e,0x00032b5a8db75122,0x000f37d6563a253d,0x000b91e8be4d1fbe,0x00000000000c0920}, {0x00045423a83becc9,0x0008308b713a8319,0x000ebb9d576c8cba,0x0005877917ee393e,0x00091528cd12a897,0x000daf6aa294792e,0x0009b94512dc8c37,0x00000000000197a9}}, + {{0x000e5d8cfc9c3b49,0x000e0cfebe026cd6,0x00010748e4a52d81,0x0009d16725ebd099,0x00007a584e8d00f8,0x0001cab5894c6f2b,0x0003c8095ac9cc91,0x000000000009c407}, {0x000462695d0ed83f,0x00056265919b06a1,0x0003cfa033d1cd2a,0x00097f1815a70aa4,0x00001c5fd35a6db5,0x000d52a8bdd9e4a1,0x000fab38b8e35112,0x000000000002ef20}}, + {{0x000003aa77fd8f2e,0x000b43b21dc1edb9,0x000008b1e2380339,0x000e5d30da87d664,0x000f0cd999a0f536,0x000e08f5d4ce10ed,0x0007e68949181450,0x0000000000052664}, {0x000bcdd23ee4cc2b,0x000a3cb207ee8e61,0x0002ac9dd73c4f6c,0x00040d225e0cf154,0x0001067fa0527d49,0x0008b2e21b688b31,0x000eab89308326a9,0x0000000000072311}}, + {{0x000c6c4de58ef59c,0x000db2a6280408d6,0x000f4e9cd796cbfa,0x000daa3c647fcba8,0x000f4952d3a44a2c,0x000e2e2c3e1a9e91,0x00040ebc1b3d8011,0x0000000000018e43}, {0x00017ee00926dad8,0x000cceb8696ff13d,0x000436379cf7b867,0x000d6ce6c1e905c6,0x00091ed2e8cba471,0x0007c8c914e13d60,0x00084e52951509c7,0x00000000000e2348}}, + {{0x000a3b1514eda4ba,0x000ac37be5d3f53e,0x000cab6203ce338b,0x00024b59d1569e7a,0x0009b4e293ab165a,0x000b7c0a4254aaf5,0x000b183c751fbd6f,0x000000000005018c}, {0x0000fe7bbd1e6c72,0x000eaacb3b8965e4,0x000a2f2ab6db69d7,0x0007258621df6d82,0x0003185f0e2a245d,0x000ddbd812af0e28,0x00002a1e7edc818f,0x00000000000c538d}}, +}, +{/* digit=77 [{1,2,3,..,}]*([2^308]*G) */ + {{0x00042da90721f8f9,0x0009f6858b6747d2,0x00082ecd0813dd9f,0x000674e29d9194d6,0x0006dcb30a51324e,0x0008aa5492dc6fdf,0x000a923f3ecb7752,0x0000000000023ffa}, {0x000c7344981b01f4,0x00090d8efcbd58f8,0x000d0144b46e312e,0x000b0b74cf4a1708,0x000fbbf335fd04f2,0x000928c36bc0bb49,0x000d45c6b6e5bd5a,0x00000000000d893d}}, + {{0x00016a2799bedc34,0x00022fb083c2a5e5,0x0000296a1d9c8cea,0x00030e747a81b965,0x000658a987376e41,0x0002829c9cd704dc,0x000d63842ae32959,0x00000000000be20d}, {0x000f531daa9bdad7,0x00053abee8e552a9,0x00078232300aac8e,0x00039b2467112ba1,0x000fbe341038781f,0x000a2f8be6f06058,0x00066e5effdca512,0x00000000000af344}}, + {{0x000afa3ee32c886c,0x0006d3933b4c0742,0x000439c40f3f936d,0x0003c0f0b7e019e4,0x000fa48715d652fd,0x00039be537f8e5dd,0x000cdc0879c5defa,0x0000000000056f01}, {0x0005bfa2d160b7fa,0x0004671b2dcba45c,0x000e0bb6fffdcb60,0x0004846640b2d753,0x0000049a2cbe6af9,0x0002ab0f178a1071,0x00043841a1ce67ac,0x00000000000a76d8}}, + {{0x000a317691072fc1,0x00025ab86639e0b1,0x0006a85f7a13b8bd,0x0004418d763f0bbd,0x000211a9802cb573,0x000269b8ad2cc332,0x000355e41d7db881,0x000000000002ecfa}, {0x0004e57bc1e94af4,0x0006159d6912442e,0x000356e842631b03,0x0008b6b8a7397395,0x00091cc83ae38447,0x0007ccd97e86ad7d,0x0004c6bc9b208479,0x0000000000069494}}, + {{0x0006d3d1d84bf486,0x000a4bbb6725a843,0x000b1260c5ec574a,0x000dd0664bcb2d64,0x000a780d5c1bab65,0x000a61e6e755e848,0x000b4b84e6f686a3,0x00000000000bd100}, {0x000cfe4a11d7ea4f,0x000a3b34fdf84517,0x000d9afd2967c440,0x000c881f9a125f0d,0x00064aeadf8e3c2c,0x0006bf0c2b1fcce5,0x0001bebdfc54c0c4,0x0000000000009d28}}, + {{0x000af84d84069bc1,0x00010e1aa844a91f,0x000628bd439b9e84,0x000aba65ddcb5d25,0x000a38009c6a3025,0x000261ee5ac8cdab,0x000430b9ee07b46e,0x0000000000006ab8}, {0x0003196ba81697a7,0x000207a36b461d0a,0x0005a884d3b0fe10,0x000f469341275a1d,0x000f1c7cf858ae4e,0x00092af3df7cb393,0x00017683c00eafd1,0x00000000000a1e15}}, + {{0x000475d9329b61bc,0x00078687d0980c72,0x000d109af3ad73e4,0x00007fabbcde5b3c,0x00016ed103a56414,0x000a6ee2212ec9d2,0x000c11f858c7851f,0x00000000000818f2}, {0x000247f6c002433a,0x000a04720d42a19d,0x000f64099b42a8aa,0x000f25533723e4ef,0x0003d3260b6299af,0x000391c33d6c67ce,0x00038163a6fc9d86,0x0000000000031de2}}, + {{0x00092324c0c2dff4,0x0004f585edee2c89,0x000820f42406ab00,0x000924fd85ad3203,0x00019124bffa3bd7,0x000c628682e7d8f9,0x000f5d885397c044,0x000000000007fda9}, {0x00035c11900b0d2c,0x000e6d302582e314,0x0003e1f066fcf73e,0x0006b5b63653c06e,0x000d9c70dad4b021,0x000bba93d8f08a1a,0x0002b7ccf014aaff,0x00000000000a33f1}}, +}, +{/* digit=78 [{1,2,3,..,}]*([2^312]*G) */ + {{0x000cb198f774dfa5,0x000dd1e347819af2,0x00020411ce747fbb,0x0005ccc28af85bb9,0x000b3ae9ba11fc18,0x000bdb313c923d78,0x000b877d494d7ea7,0x00000000000af8f8}, {0x0004fe8c8323652c,0x000bdc178ccc093c,0x00006a94572bacfb,0x00047cc80a50372b,0x0000e6f901976087,0x000c2f2168147aa9,0x000b514683d4c978,0x00000000000552f3}}, + {{0x000f8017d10c6b96,0x0002229d81537326,0x0007f1980aaaed23,0x000ce6e8783d1fe4,0x000df0e3c126aa4a,0x000ca4b719c14ebd,0x000f350eb1c08d6e,0x000000000001c91d}, {0x00013bd5d85a7c75,0x00064db8119bb0fc,0x000fdf5036f03e4b,0x000916835436f82c,0x000b9b18d212053c,0x000bc3497eb41ccb,0x000d8193b3fb43b1,0x000000000008c1ce}}, + {{0x0007ad7a8ff88fe5,0x00031521f4af6aed,0x00033eaf4c7af196,0x0009ab7ae0377807,0x000db65f73ffad09,0x0006059b9541cadc,0x000f8253430463a8,0x00000000000043e9}, {0x00061f938516beb2,0x0001bd515f4c75f1,0x000d0a9d767932f1,0x00053567ef1b9007,0x000f1295b5fe7bed,0x0009176278782b45,0x000d9fab54ebaa03,0x000000000008787e}}, + {{0x0006f626b572390e,0x000905255508b5c5,0x000fa6a60258070c,0x0005eea3095205c4,0x00002c470ef7d976,0x00040bf0afea2ed7,0x000a16f5e8a0fc9c,0x00000000000e4137}, {0x00055209e40da33c,0x00003b917537047e,0x00003ab2893cdb40,0x000bce9023854871,0x0008a8bc3a94bea6,0x000c450444620840,0x00015164a87a2a3a,0x000000000007df70}}, + {{0x000cf2f10b0520cd,0x00004c38d07270c5,0x000e4dc64fc6cd1a,0x0001d8f300e953df,0x00040a7dce63abde,0x000425847b98b23d,0x00061b63e51543fc,0x0000000000043a21}, {0x000d54e8ecf42bd9,0x000ee2650a1d5920,0x00086c043a07e5a3,0x000b7048bd9bd78b,0x000df89e86bdc15d,0x000579426bad1dba,0x0005f8d7d4845024,0x00000000000287bf}}, + {{0x0006ca98107f85ef,0x000615fb05303f18,0x00091aa7ab0bf32f,0x0000fa47d92132d0,0x0008994564bf7a0a,0x00029e066e993faf,0x000574bf30252517,0x00000000000d4260}, {0x000990cf5caa4f19,0x00009f1eecc3d948,0x000f8dbc962efd6e,0x0001cf1d1dd88736,0x0003b6f1aae115fd,0x0009344b97ce882a,0x0009ca7efb7ec6da,0x00000000000cc5b5}}, + {{0x0003c28364de2038,0x000c48bcbf6cdce6,0x000e355114722287,0x000febf92ebe1bcf,0x00019af24c2e4ee8,0x000a861d57847829,0x0007ec618760b094,0x000000000006bae2}, {0x000de113e64c1ae9,0x000fc77c174b9e18,0x00055d1ba670f4b5,0x0004f6a58a0062c2,0x00047781acd1db9e,0x0006d86b03480c9b,0x000ef52d7c628ef5,0x00000000000b3e2e}}, + {{0x000ade0c1eab3d7e,0x0000fee9e25d91a6,0x00060f30210199eb,0x0001157d2d91300a,0x000fffcbe50fd2fc,0x0001d376ee7597e2,0x0005624be3814fe1,0x00000000000dbf14}, {0x000860a341c4b797,0x0005638a107c07ee,0x00045c1dbdbe39f3,0x0004e09caebd481c,0x000bf9ace60d1801,0x0001cb8e7b28c331,0x00065ee8fba04f21,0x00000000000e42dc}}, +}, +{/* digit=79 [{1,2,3,..,}]*([2^316]*G) */ + {{0x00060f98795275e8,0x0009d009dc2f44a9,0x000392d1d1132d95,0x000faa2ec5d91fec,0x0000f160afcbc89b,0x0007566b5461be9a,0x0009a2fdd084bb7d,0x00000000000e4809}, {0x000f69da99c8ac01,0x000bd1b461fc964c,0x00012ac0cf0d2b27,0x000332d9f09a206f,0x00036941d6ef3246,0x00087f5f6bde4c0c,0x00040044ef03fdf3,0x0000000000057b63}}, + {{0x000b480bf51dac8f,0x000ea7e485158a07,0x0002c9e9ee8b46dc,0x0000dafa6fbabfcb,0x00038f09fff720de,0x00006994f04f0158,0x000d335a0562976a,0x000000000006e3cc}, {0x000d89d62ca77de2,0x00045c087ef9c0e4,0x000f0235509703e4,0x0004814d916ee937,0x0008d23d140b4fc0,0x00062028f3369c97,0x000adbb0fcd70f16,0x00000000000e1e28}}, + {{0x0003a6684cfed7ee,0x00094e3ccfe8dea6,0x000d721274c869f9,0x000e2cbb605c4274,0x0005b0d809923f13,0x0009913815ee1239,0x000b6e4dcaea94b9,0x000000000008c4c4}, {0x000ec88cc6307d53,0x0004bd7aad54ab94,0x000859bdd379b119,0x0001eee68048fd9e,0x000fd57225fc6eb3,0x0008dfd3f693842d,0x000cf13e62cd85a8,0x00000000000d5462}}, + {{0x000df1c3e7de2ba1,0x000ff99b5f14f615,0x000c00e1f8e35232,0x0003900ff19bf251,0x0000260925ede749,0x000ea1eafc82e5b8,0x00021e3398d340c0,0x0000000000012871}, {0x000896ddafdbd954,0x00046ea6dfb0ad7a,0x00039aa9f60646cc,0x00078bf1201299a2,0x000d32e5cebce266,0x000181e32b5369e4,0x000add2304eac86d,0x000000000003d364}}, + {{0x000f357cfbf02e91,0x000a2f55f868d743,0x000049d92d19de08,0x000ff9bbb5ba194e,0x000957724703bec0,0x000e0e4fbf7325c4,0x000d7c5ab8f06323,0x00000000000ecb0f}, {0x000a0f97049bea9a,0x00069cc72e8da9e4,0x000c7a0fc237673e,0x00046e64252763c4,0x000eb8d81de3bfa6,0x000772f007a769ef,0x000a5ef5ec70031e,0x000000000002729a}}, + {{0x000cc742f063ab99,0x0003b51f19bb30d3,0x000878f139d45b8c,0x000de2f987112cb9,0x0008759de2cb5f7c,0x00049bd98dc51d94,0x0001fd9d05c410ed,0x0000000000036434}, {0x00020d69ae77fca8,0x000a9d7869d68233,0x00007a8daf0bfe29,0x0008180c25910747,0x000a6ebc4ebc7f25,0x000da1d687b90fc7,0x0004935565390f3d,0x00000000000a44e4}}, + {{0x000116432be8131b,0x000ebb4a2742da3f,0x0008bb7a8fda7aaf,0x000adccdd7fcf4fc,0x000337d4fcc6b15f,0x0001c7ab70b1fa2b,0x000cc9242c7b26f0,0x0000000000050574}, {0x000069eaa2593652,0x000b76ee50f421b3,0x000a5035e1a2b7ad,0x0003af18420501c6,0x000b1b6e7d2a04d1,0x000516f80c22e092,0x0005b5a393be7ee9,0x00000000000b80bb}}, + {{0x0003f2ad59508de2,0x000bd9e41b9b6b62,0x0006d3eb906556a9,0x0000c8cf272346c8,0x0005def5749367e5,0x0002676fcc98a5ef,0x000659c2dea8afa0,0x000000000006ace4}, {0x000f1f5cd12db7d7,0x00090c2b707aa093,0x000d4cda421d1577,0x0000fa2a778c20c3,0x000d57af166b5b58,0x00078ce62272c3a1,0x000087e47a64a9ca,0x0000000000071d35}}, +}, +{/* digit=80 [{1,2,3,..,}]*([2^320]*G) */ + {{0x000e3ee1ae74a706,0x00069c20fb9106bc,0x0005cf0e923045cf,0x000bc6a5cc0aa89b,0x000b4fb9f01e12bc,0x000ae1b895279c6a,0x00003cbc8e178911,0x00000000000c2900}, {0x0006736dff18cbd8,0x000579949d6eb7ac,0x000931ae3b95e19f,0x0006ee2dd2275c0b,0x0008c419f1d568de,0x0002b5a40a9e4fff,0x00096dbc759ca4a5,0x0000000000063b09}}, + {{0x000eafd50b503ff0,0x00058f682715078f,0x000b7ec2b450662f,0x0008f55c51916c40,0x000b61ee081fa46c,0x00064872a98c3524,0x000faa5ab7f2c2ad,0x000000000007e555}, {0x000425ebf2c1673e,0x0003c802e0d22c20,0x0004e9f77f229a7f,0x00019087af20d270,0x00016be50784b56d,0x00072b8518d619e2,0x00076e10fdbb7213,0x0000000000049355}}, + {{0x000df2627e9f5583,0x000e073c57fde593,0x000f9da9fd5339d4,0x00071f1cfb9821f2,0x0009cf6a1d68d25c,0x000d24e649427e20,0x0008b5bf3c3916ac,0x00000000000bda7b}, {0x0001a88dfb057584,0x000e2e7c5f7c7d49,0x00071c4b5c378478,0x000c472e66b277de,0x0007b2816f1e818d,0x000383c6d4413fd4,0x00053740f262af48,0x000000000004f190}}, + {{0x000aa901e2868cf5,0x00009d97a9b136f1,0x000f9d37eb7a3b6c,0x00060c2548188194,0x00061d59be44c571,0x000f2b2474dbdeab,0x000b33b93a9dad81,0x00000000000ecbca}, {0x000a766b5fedf0a6,0x000ce5e1ce82823b,0x000997bad589b344,0x000273a2d82b0df8,0x0006ac8ff52fe716,0x000ea7ccdb017f5f,0x00074b7563e79970,0x000000000003f0e6}}, + {{0x000fda8d824654de,0x00033487edc06864,0x00024946c59b50d3,0x0002bea10fdba0ce,0x00052c3dbf337fc6,0x0007d2a46c836bbc,0x000f714fdfb5c34f,0x00000000000dca0d}, {0x00024bebc7614440,0x0007730a0767544c,0x00067fffd43f5d03,0x000d49dfacc13dc4,0x000aae1a261ad307,0x000d98650148d673,0x000fdee008f95b52,0x000000000009f558}}, + {{0x000015df21e067c3,0x0003dd084564b8db,0x000d0b4c4a73b501,0x0004bf63bc7206d7,0x000ffc3d577224bd,0x000c4924fdd37b4e,0x00096e8156d6f635,0x000000000001d1d3}, {0x000ca9d7511fd3e1,0x0009d977b3b3982e,0x000fc4c09646cd37,0x000f9c4ae811d70c,0x000ca06c7f0de581,0x00049d87fe5441d5,0x0009bb0275c92b19,0x0000000000078046}}, + {{0x000cdb7d87948ad3,0x0009117cd6af6151,0x000e60b294f18979,0x0005b1d788224330,0x00031b9745257e49,0x000f3ce21fed338e,0x000896527743d386,0x00000000000b515c}, {0x000e2d7dc921023b,0x000d19fde7038e85,0x000c1b9c9ba2d99f,0x000600b36ff74dd6,0x00048302c3c95d46,0x00044b5eb8ea74b0,0x000402d560f570f7,0x00000000000fe762}}, + {{0x000cf97b63daba80,0x000eea7f8501424f,0x000f397f945052f2,0x0006b8a2ad5053d9,0x0003fbcebe43c658,0x000054f1688c09a4,0x00047fde7f2a4a20,0x00000000000bbbb1}, {0x00041659e06114fc,0x00075087e48f29cd,0x0009df479d6378c0,0x00002ff7f27cac87,0x000c37b1f6e18ae8,0x0006d1deab01a131,0x000f803f0d4c2e9a,0x00000000000e7cee}}, +}, +{/* digit=81 [{1,2,3,..,}]*([2^324]*G) */ + {{0x000643c3f46ade04,0x0007aaa03b473d97,0x00091126d04a500a,0x000ef86f37b57bd6,0x000b2a6371d2b286,0x00077ccbd95b797e,0x000a6da489ef8940,0x000000000008e99c}, {0x0007980fdcec94f4,0x000845cf9e236463,0x0000b4681fa85d18,0x00023b6ba7ba4781,0x000757de30bf3295,0x00036ca01d406ab8,0x000ac5aa10e4a215,0x00000000000dfa7a}}, + {{0x000146df48d94f2a,0x000a9762a5eebd68,0x000728ae94bf3482,0x00034aeaa4f964b4,0x000ed6526f010fd5,0x000d71d72cf46648,0x0008eabc62a54a18,0x000000000004f848}, {0x000d9dd392598cb3,0x000a262c40de70e6,0x00099451ce04b75f,0x000d7466ab190976,0x000b2dd1ad3d3a51,0x000f88078f07bea4,0x00035d2a0ec4672e,0x00000000000664e4}}, + {{0x0002f7cfce391389,0x000d017c563bd4e7,0x0006d61076868226,0x00049ac4cd1bfc15,0x0000d4e53d3b591d,0x000079a6ed446551,0x000eb4ca95ccf6ad,0x00000000000b5ea6}, {0x00099522fba1cf25,0x000cd600ccc9ba8e,0x000c729701bdaf25,0x000edc569c6c6e81,0x000a586abcdc5f27,0x000dbc5b6b3fc0f4,0x000fe0dbfdf3f985,0x000000000004ea80}}, + {{0x000e7847d76787ae,0x00083ee6e7005dda,0x000646f5e7ea8d16,0x000df7dfc45111c8,0x0001875c23c8373a,0x000039d490c7535b,0x000a062a7e04f897,0x000000000009afd1}, {0x000cbb06099a2183,0x000fb049dd385881,0x0008a6ac539cc7cc,0x00057b9925498c29,0x0002da25daa3dce0,0x000decb7d7b9c9bd,0x0006ef6d0b70a3d4,0x0000000000003df7}}, + {{0x000405f3ffe95388,0x0004eff54e8bb8ae,0x000ad0d572466c0c,0x0005bc4ed1fae2ed,0x0009ea983db1b226,0x000851454a5e23c2,0x0005064af5f55909,0x0000000000036eb4}, {0x00062710e23e547f,0x000844c72b2d0dfe,0x0003b7abb313e4c9,0x0008ad4e19c9c269,0x00053e8b8004b1aa,0x000c747fdb9cfea9,0x000e4107782d788a,0x00000000000ce9d9}}, + {{0x000aeca3f9d9a2f2,0x000f8779905b5deb,0x00064a28a88ba6a4,0x000f0aa679096dc3,0x0000f558913cce9a,0x00024a2e2fdf04f1,0x000b3505300cb06f,0x000000000005d581}, {0x000679a2a74abdaf,0x000aa855c8de8c32,0x0000f6e26bb556cb,0x0002a4197740d667,0x000744c9360d6761,0x0005cc8807588a52,0x000188928ac91032,0x00000000000acdd3}}, + {{0x000ac6fa45a00390,0x000933774af3674c,0x000bcfd34179ae6d,0x000205ec11875399,0x00018799c22ede45,0x000b0842ef3b1144,0x000d64fd53f681e8,0x0000000000066d36}, {0x0007de25fd86d3c2,0x0009ca2c3391c37b,0x000760d08855c7c1,0x000fa4a6ab8ab4bf,0x000bd841a6234df1,0x0000b189a2b0cd34,0x000da752441d79b1,0x00000000000fa025}}, + {{0x0009e3af5a1f4313,0x0000c3c6c56185fb,0x00083406c1a4afdd,0x0009c90ae2fb830d,0x0003ec31a504b325,0x0006230bdf7cc6e4,0x000e3a83bb13c612,0x00000000000ff1da}, {0x000c261cfaa4903f,0x000010cc6c1a44f0,0x00029f13aa21ef15,0x000c75fe604a58e6,0x00012491f872d454,0x000e116a5ceadfd9,0x0004816c5575da9c,0x00000000000b24a4}}, +}, +{/* digit=82 [{1,2,3,..,}]*([2^328]*G) */ + {{0x00087efd2e620105,0x00045c5ad791e85f,0x000885fe6de88f7b,0x0009e4998544bc47,0x0005d273c360673b,0x000a75a8c934d8f5,0x000c977fb48d0768,0x000000000005e0fa}, {0x000ee7923fcfedee,0x0000dbe01655fb13,0x0001ed409cde6661,0x00030de533251391,0x0001de9f3fc574f8,0x000e01fa5a724033,0x0004b496b25206a7,0x0000000000007eda}}, + {{0x000a5e409b41a55f,0x000283e6e9380f21,0x000bd6398225ced2,0x000c8b4b843ec071,0x0002af8a23f1f2d7,0x0007923350055d88,0x000e74e848010ed3,0x0000000000055e08}, {0x000839aa213ff6af,0x0003d956ea8e93ca,0x000299a04653a35e,0x0009271cca1b5491,0x0001f1d0b7f15f7b,0x0009963fbd41d228,0x000cd8b187047b5c,0x0000000000002536}}, + {{0x000a16fe10008718,0x000fd7138a03f4b5,0x0008b9afc3d67514,0x00041fd2a7cbcc17,0x0005f6073ecba29c,0x00004fa086014349,0x00053298b46d3805,0x00000000000401f9}, {0x000b314f178aa1e6,0x0009808cc4abc63d,0x0005190f1545e910,0x000c8faa75d9af1b,0x000fc2e5592d6eb2,0x000468e666f50198,0x000fc3773b9d55c0,0x000000000001a2a9}}, + {{0x0004cdf21adad97a,0x00008e51a956ff23,0x0002a26288680976,0x0000fa2f974d2cb8,0x00074c6a78dec616,0x000c89d11e00474f,0x000133ee916e510a,0x00000000000f826f}, {0x0002a0e3be0ad710,0x000103fc65d3ecf2,0x00089ef785c9b19a,0x000a82e0b9196b29,0x0000a572b6f55237,0x000bed2bc4975e68,0x00025e41ea8b92b9,0x0000000000098268}}, + {{0x000bc901bf39d0d3,0x000fae352fb9c164,0x0000b33959aa5988,0x000ce6fafcb7de1c,0x0004f694cad0f9ff,0x00061d6110e574cf,0x0005d8adb4671cb7,0x00000000000c8aa0}, {0x000b2a78a9127846,0x000a987cd7872df6,0x00010a813ed5d7a6,0x000531ec2cb43e23,0x0004b9e6b89f1c41,0x000e5d4d262fc310,0x0001dd7f406a0dc4,0x00000000000df3f6}}, + {{0x0005300bc3a99814,0x0008c06f721d246f,0x0003df2a36c6f8d6,0x0006b7a5a44ffdab,0x000915764390c5ca,0x000747d7dc53e328,0x0001227f16917459,0x0000000000061f79}, {0x000965ef8eb0e108,0x000ae099f4e3a08f,0x0009b651d265b539,0x00043576b72f3285,0x000cc33695bd270a,0x0002249029d6bb26,0x00051f60f9202126,0x00000000000e1b0a}}, + {{0x000bf8353cd33783,0x00086265d4eff002,0x00010092c6e845e0,0x0005c996be6ce254,0x00096b6056eec399,0x000a007266fd43e5,0x000fb1d328c28f28,0x00000000000f719a}, {0x000f1e2a3462b4c4,0x000fb75dcabd2f96,0x0006c939c482a7c5,0x000a522d764ab7e8,0x000dcf8a487267c6,0x000810a449458c96,0x000786af1af61b78,0x0000000000073902}}, + {{0x000b7c441c7f3df5,0x000a88a390dca834,0x0004f92ea9440c60,0x000ea69bc5e82f15,0x000d4628d6f177bb,0x00068e24071f02e2,0x0000ee6260790cf9,0x0000000000066527}, {0x0001df8e5c25d23a,0x000f233639544be6,0x000c37cf5bac2371,0x0000f867807ed557,0x00019dfa452c6906,0x000d27c0d1b0ac01,0x00090cbbdb25fe06,0x00000000000a60ef}}, +}, +{/* digit=83 [{1,2,3,..,}]*([2^332]*G) */ + {{0x000957058557b81a,0x000dc1877dfd042b,0x000fe0adb79e32df,0x0009536c78e295f6,0x000374a04cc59d08,0x0007f75edcbb5d55,0x00078ce366600682,0x0000000000061e73}, {0x000dd46741146ae1,0x0000a74170170ac0,0x00052de550198353,0x00030f8ac1c4a0fc,0x000d3cd92824d75a,0x0005e86d4d7e7be0,0x0005ef5ea0abdb1b,0x00000000000b3b61}}, + {{0x0003f4e13d621f8b,0x0005a618ca06b4bf,0x0002813dfa928efe,0x00072cca3b6ce7e1,0x000e5f14bb579813,0x00073abbbb44a68e,0x000c0afc1167e963,0x00000000000767d4}, {0x000277deb88a4f9a,0x00077c6dcdfeb3e6,0x000664e1fd3b5c4b,0x00001aabfcbf2f89,0x000a67aa82774775,0x000aab9cd0be7228,0x000b4a5de7b8331d,0x00000000000262fe}}, + {{0x000cce85ebd04b7f,0x0007f7a9623b5682,0x000f8d813f260bd0,0x000dec9e0f8e530f,0x00091d65dcd4f8a5,0x00039541c88bcbe2,0x00097a09fe380367,0x00000000000e716a}, {0x000184fc8c1d72cf,0x0003bbd8e34de924,0x000a19b51363e270,0x000d905351941206,0x0006f7a3c4830183,0x0003557396345099,0x0002f718283cea01,0x000000000009aaa7}}, + {{0x000a56eb624d0cf2,0x0006a7b7da0287f4,0x00087c61d1ede60c,0x000d83c51f3dbd9c,0x000fa91d085ac056,0x00035262de2f4d46,0x00029e31759c9ceb,0x000000000000c9dd}, {0x0009a41b82bb0c12,0x000e8108cf9ba936,0x0004fc5060b7103d,0x00016b273d372b32,0x0007d5415e53f698,0x0006b8b0e126575a,0x000a5e546bb4c130,0x000000000004d54e}}, + {{0x0002c932a7c5abec,0x000a881dd074d9bd,0x000f76c734b204ea,0x000e1f0352b58f62,0x00029c390eefb5af,0x0006a8d236d2d948,0x000be5d073f9a0f3,0x00000000000466eb}, {0x000118cb3dd8d87a,0x0004d968c04a5c76,0x00007c8586d1cdac,0x00096bb15c0970b9,0x000370ab8dda98e1,0x000039fa92af28ea,0x000ff6b912fbda7a,0x00000000000cbd02}}, + {{0x000e45ea08b7c4d8,0x0001d5bcf28785c2,0x000fc63d7c5e8cf8,0x0005de6707a7138f,0x000cfb2041a48868,0x000db3060a77646d,0x0006908608695289,0x00000000000e27a1}, {0x0000f0b051870cd1,0x000794148d936eb2,0x000a4e50dd50fda5,0x0002d4db0391d14a,0x000462b0373f9d72,0x000a9992c5683112,0x000b8487b0363be9,0x000000000008853d}}, + {{0x0002c578f8f683b4,0x000576c8e5a2524a,0x0007522f4370ef9a,0x0004aad50ad6f3a6,0x00051f107e269b01,0x00051bf732a3933c,0x0006df4b04395cc9,0x00000000000b0ef5}, {0x00017d2e75e44b21,0x000cd28c9fcdca32,0x00061a5012b72649,0x000f58e53ace23e2,0x000efe688f02b546,0x00091406dee5fec4,0x0004d9d8b43f6d2e,0x0000000000026176}}, + {{0x0002d2ad73a4790a,0x00034daa7fcc985a,0x0000276056234a54,0x00018c59894edceb,0x000da9ac29a7fbf9,0x000372f2c470c45e,0x000d3fdfd44f6fbc,0x00000000000a1e38}, {0x00009dd8fe96aa5f,0x0003923d2400aaad,0x000c441c1c12013d,0x000488148baa4ac3,0x000ebc26e6b0c76a,0x000ef566273227e4,0x0002bbe732edcfe8,0x000000000000c566}}, +}, +{/* digit=84 [{1,2,3,..,}]*([2^336]*G) */ + {{0x000d5d97020dd4a2,0x0005087c7dd7d118,0x000d6556f04f9110,0x00048789fff00158,0x000b909eaf1abcbc,0x000443a788ffcef8,0x000fed9905f4eef1,0x00000000000e15cb}, {0x000831894b4ea50c,0x0006cca2969ec57b,0x000ae2b1ba048b54,0x000fab787ea6e3a2,0x000ff98c4fe567f7,0x0009cb91e0d0ee10,0x000ab4c646879ac1,0x000000000007d1d7}}, + {{0x00092725c84d0435,0x00013cfbcbc49b56,0x000166c387130162,0x000e05622322f91f,0x000a7662ae23735d,0x0002be7a6d2590bd,0x00017263d468fed8,0x00000000000695ea}, {0x0005986e2611645f,0x00050c50f4940fa9,0x0003729def8820a9,0x000cd041fd257785,0x00044a3d0808680c,0x000684cba25ddac4,0x00088841d8b738c4,0x0000000000053963}}, + {{0x0003a8028963f3e3,0x00076b05cb83426c,0x00074bc4cf79f053,0x000b5aaefa9e4f0b,0x000d9bd075ecf023,0x000a41bfbb8061e5,0x000d967d2ba50f1a,0x0000000000090865}, {0x0004ae64b7cf3e16,0x000d97f221a788f4,0x00000b032e3ccff2,0x000eb6af6ab15418,0x0004dc87cd93085f,0x00039dbf6fd14102,0x0003643e196fdb32,0x00000000000dbcd5}}, + {{0x000d4ecde2fce0e7,0x000891466c9f4fd3,0x000b63a0ee692739,0x0008c75b58655519,0x0004e65862cc291e,0x0009971671ddb715,0x000c19285153bc2b,0x00000000000954b6}, {0x00039e1aad688b1d,0x000e0985d4505697,0x000559cd349baa64,0x00098dafa5fe5e6a,0x0006f90f231d39e0,0x000afd0b53fc2604,0x000c18de5d5ced32,0x0000000000060c09}}, + {{0x000b1d0586854e54,0x000cf3720b17188f,0x0005eebb9c78a9a7,0x0006e315b54eefca,0x000472b0edaa03c4,0x00077476e387e466,0x000259fc59b03dee,0x00000000000607a7}, {0x000e6b356394b5a7,0x0008b1478bceddaf,0x000e3cdb4ff0323d,0x0002f72cf272ee0a,0x00038bd885cd7127,0x0006ba31c19e3a4a,0x000b369af6415b37,0x00000000000807b2}}, + {{0x0008b39051ba2cdd,0x00030dbfcfa7593b,0x00015a46addd3783,0x000851d67a19b610,0x000431ae1a67cc6c,0x000d1e135ace88bf,0x00090a74554a6193,0x0000000000042118}, {0x000bd87496a1468e,0x000277cb3268907e,0x000047feb42207f2,0x0002c21766e72f3c,0x000a880642e30a67,0x0004d2b3624bb718,0x0004ab425dc41d18,0x0000000000089102}}, + {{0x000caa1ac9c2c636,0x000673180789b5d1,0x000d386a756090c3,0x000b895e99e922ea,0x00003d0bbe807951,0x000fe79c0987ea25,0x000d2a6d2f49f0e2,0x00000000000c2b18}, {0x000ed8226be989ea,0x000d8707798f36cd,0x000ebf8dc541b02a,0x0001e32bc4479412,0x000938a845f346cf,0x0005e5d924dfc145,0x000c3f210083fe45,0x00000000000a1fed}}, + {{0x000faea0d2bc54ec,0x000a972fc8198080,0x00029736c6c3c3d0,0x00098bbd99e30373,0x000a0efddfca3691,0x0009147416b68390,0x0008078c35f3e4f0,0x00000000000ca7d7}, {0x0005a76575b0ee6c,0x000e9564d991dc4d,0x0003cf033490be81,0x0000dab5d635893d,0x0006f944e49a51f3,0x000233bc00427e34,0x00093e1e0bf1b56a,0x00000000000617bf}}, +}, +{/* digit=85 [{1,2,3,..,}]*([2^340]*G) */ + {{0x000129009060318c,0x00056d23658c8428,0x0004596f83a71a5a,0x00010ec082210f1b,0x0001bfc364906dab,0x0004f2d14fb1add9,0x000443eda8b02f9a,0x0000000000056560}, {0x0004169ffc0d0413,0x000986c01479d686,0x000d5173dbdf44ef,0x0003c1d718883983,0x000e13c34fd24dcf,0x0000c15ac87a9c04,0x0005d2fe0e08ec51,0x00000000000c0f49}}, + {{0x00099e97b82e73d1,0x0005db097b2c50cb,0x00071fe923d57f3e,0x00066e9420996453,0x000eff7290736382,0x0008c394043d059d,0x000925eebb5fe114,0x00000000000daa8e}, {0x000995963e49eb67,0x00007f43c78d7e6f,0x000a34a7c03b2d6b,0x000a7d66b6363576,0x000be096a954cdd5,0x000afa2fc8b70a2e,0x00095a011d5419d0,0x0000000000004fb0}}, + {{0x00033bb61172fb0c,0x00008eb05c51603f,0x0000d435c61a818f,0x00073cd489576f54,0x0000535a8d57cfd5,0x0001436c4e653538,0x00022d1394731467,0x000000000005f0a1}, {0x0007992fb047bfc7,0x00099aecc7ec110e,0x000e8ab91f3e896f,0x000f6523d4221aba,0x000f2851996bdf96,0x000cfb67efef5649,0x00005246fb9f26d5,0x00000000000ef5c4}}, + {{0x00016ebee78befa2,0x0003188e7ac8d2f0,0x000b37c50c491499,0x000d52918be419b3,0x0004c057621c3b96,0x000e57e597e75e35,0x000463cb1b709f5c,0x00000000000844f2}, {0x00053157dde1a349,0x0006d53608198549,0x0000450e27f360c3,0x000d348e04114157,0x00004d73eeae6dd0,0x0000ed85e5b28e95,0x000b580843923269,0x000000000006da14}}, + {{0x000ce0f0a89b6284,0x000735cbdc4424ba,0x0003b0cbdca1bfb5,0x0006886c118d02b1,0x000675aaf27658a2,0x0005f6ec187ff74e,0x000d32b133be95f6,0x000000000004b1b8}, {0x000fda7aaf38358a,0x0001bdd8a8ef25a8,0x000eba5b65bc2a4f,0x000a6db26623fb02,0x0000bae15453d525,0x000f89fc9f2368d4,0x000a3e55d6a84d84,0x0000000000086021}}, + {{0x0009f5565cd739fd,0x000f210520b26d6e,0x0004ac2940ef5ddb,0x0008ef88948f78bf,0x000550ad452e1246,0x000c4847040a9d27,0x000ace5e382347c1,0x0000000000033e73}, {0x000c8b4278956db1,0x000437d83d02e97a,0x000af99c625baa1b,0x000d448885679e6a,0x0005fc95919c6716,0x000c2194a0e12bdd,0x000d2d7da4420492,0x0000000000015ffc}}, + {{0x000c11f07976d090,0x00025bd048bdc85d,0x0003c8a142cbe0e8,0x000a758a985ac100,0x000cf2c7ace940aa,0x000ec347d6039bcd,0x000ab7712092cc6a,0x000000000006b5ac}, {0x0003db66ec59bab1,0x000551ebcf80c829,0x000fffebfb9d4dbf,0x0003d1ad0b610f09,0x000498c28ac73fdd,0x00059750fdd3f6f9,0x0001ac650b77943a,0x000000000002d399}}, + {{0x0001cfefd0dbceda,0x000826cbd2756691,0x000e925943cf3919,0x000aa0be4c58c7c2,0x000f0e488784177e,0x000916b0f603551e,0x0002e2c8eba131df,0x0000000000015973}, {0x00008eec1292bc0e,0x0000355acca849df,0x000e7c404ec832b8,0x0002c703bd3b202e,0x00056ddd8eba162a,0x000b1d93d4c5e5d2,0x000dcf66e7844a77,0x00000000000110b9}}, +}, +{/* digit=86 [{1,2,3,..,}]*([2^344]*G) */ + {{0x0003ab751954f075,0x000f91b66faabe09,0x0001714e51539902,0x000f3a0a675f7c8e,0x000f30313a711a82,0x000aea9e682884a2,0x00005d7ac5d7b058,0x00000000000cd5ff}, {0x000d5d715b7b74ff,0x000287c29638d05f,0x0006736db974b38e,0x0003c47a17ae3a7c,0x00077009e38ae85e,0x000f9c52e91b107c,0x0008a0f3b777d8f1,0x0000000000011b68}}, + {{0x00072d048b012b70,0x0009a4ae3d232353,0x000ebed55756fa98,0x000c769ec62fd6b0,0x000f62a4720cba73,0x000c1f491d586ba7,0x0005716497cd140c,0x000000000007b2ac}, {0x0007af894008277c,0x000b9a65eabb5e68,0x000cb737865439ae,0x0001b84231457d7c,0x0005901645c525b3,0x000f7b656cab62f7,0x00095c2377d74db2,0x000000000002d33c}}, + {{0x000e54e4ebfecf4f,0x000b310105dfc241,0x0000ded90576b5a5,0x00068324e80fc085,0x000287dc7da6122e,0x0004728cf3b26e76,0x0001297bc183de6d,0x000000000008bcdc}, {0x000bafda0190b71b,0x0005d8b995cec24c,0x00099629f2c641a0,0x0009e6b4da5b77d0,0x00096caf9161612e,0x000f45eb68ec048a,0x0000e9e3628ee2c7,0x00000000000d8565}}, + {{0x000dc37cf3c258c7,0x0005aaae2f447f93,0x000c6f7663c30e3d,0x000b2f482c1f372f,0x000d351a5f7f3262,0x0006c75a85521feb,0x000fb8f8ec919091,0x000000000002728d}, {0x000f483180d24d43,0x0005f5dd4ff4f0a1,0x0000b042bcddd9b8,0x0005b98ba8b777b7,0x0006fb7f8409318b,0x000fd31dbd971d42,0x000347ed1465e8b0,0x0000000000000f66}}, + {{0x000f5311360afd40,0x000bacea0374a33b,0x000feb5ded889fab,0x00002361bb01f474,0x000a8c328b8bf6ce,0x00053d6302a5b28b,0x0005d86991b1d8b2,0x0000000000085945}, {0x0008eeb3e3866a93,0x00024f5c6e141989,0x000231ed9c304f2e,0x000032f67e76ece0,0x0002338980594eb4,0x000001b765bf14aa,0x0001340804a7c00e,0x00000000000734cd}}, + {{0x00036d45a9f2195d,0x0007e5ba5288b70f,0x000e413923c56371,0x000a997602f3c65a,0x000fe08c0223c1a9,0x0007e6f1dc5c7512,0x00059748a19c3c36,0x00000000000fb241}, {0x0001d161a9f86145,0x000879f674a5f0bc,0x0001526754b42988,0x0008c4303d6f13ed,0x000d917433fb5aeb,0x000f534900fed575,0x000616e4a5ef9a59,0x00000000000f315e}}, + {{0x0008596b7b07e015,0x0009c7059c585ee9,0x000cfabffa6395f2,0x000d9318d9633cd0,0x000f37bda14896dc,0x0008964dcb2abc44,0x00076ce31adb3feb,0x00000000000e3168}, {0x000ed4ab82ecfd95,0x000363173288028b,0x000f24a3fd6c4552,0x00034c91d6f69add,0x00022c34e118b5b3,0x0005984ede613e56,0x0003a18bbdbf5c5f,0x00000000000cf4f6}}, + {{0x0008f4401a1165b4,0x000b4315ad7a4644,0x000672c1b06e4df0,0x00097a8baa564733,0x000446edcfcbe12f,0x0001a968ef263db4,0x00087547cf91d53c,0x00000000000c15db}, {0x0007f5c515f16ba5,0x000345d35e53a1e8,0x000a90f359724b01,0x00011ea246da3d37,0x000653f068205d3d,0x00010c00fc1ddfcc,0x0008c78169d71166,0x000000000004bddc}}, +}, +{/* digit=87 [{1,2,3,..,}]*([2^348]*G) */ + {{0x0005418902e018d8,0x000328002f4583b6,0x00029b160f5eca39,0x000d112fb93f735b,0x0002196e3084a8ce,0x000e8c74427cb629,0x0005ff72395bdd77,0x00000000000ee71f}, {0x00030cc165e06d5c,0x000ca7cfc14d95b5,0x000ac1b9673d9545,0x0000129d213738ef,0x0001bc0b5ea366e5,0x000a067007a905d9,0x00082192cb630afc,0x00000000000bf3a0}}, + {{0x0009c76f27cbedd6,0x00086e96c4aebbe2,0x00087447d6551831,0x000d2f632f9151d3,0x0004302b99e2f86a,0x000fd317105daf87,0x000c624299dbfa14,0x00000000000812cf}, {0x000ccd383b8a542c,0x0001b42e615367ce,0x000e792323a5de78,0x0006c70548fffa38,0x00077b6db825c34c,0x000f2989e1fbed77,0x0003ee850bded44e,0x0000000000081546}}, + {{0x0008bd3d7d2dac19,0x000ce352e14371a1,0x000574a96d5757aa,0x000d9395a7b7719e,0x000b8544328b64d5,0x000f9c5934d5197b,0x00045e5220626522,0x0000000000074d6a}, {0x000ed277c567a2e7,0x00003f52c9eeb86b,0x00037cef0ab9cbea,0x000b3bfc9ed39349,0x0007a14c3d70e606,0x000db5876fc5046d,0x0007a181cd053e5e,0x00000000000a3034}}, + {{0x00026a12e217941d,0x0008a9decf2164e6,0x000546598e5e9913,0x0009ce8aeb36b93a,0x000158fb4dc8d564,0x0002d60cfab9f77d,0x00061966b11fb6b5,0x00000000000eaaa4}, {0x00098700891e3d12,0x000dbf3522a998ce,0x00034cf7624bc215,0x00097c625e387237,0x00072d4595ee2679,0x000e5456ced5047b,0x0006feaaa41e5f55,0x0000000000078cfc}}, + {{0x000abf51b7538110,0x000353fd75579f7b,0x000019e5c13ce4a6,0x00076854d208bb77,0x000ac1c9512f4c82,0x00081e9f3941aeb9,0x0003396ce0bed5a1,0x000000000003744a}, {0x000bd7c923f4230a,0x0006180eaff7a041,0x000dbb73984381c7,0x0007c1b0f8e7fd8b,0x000aaf499630f16e,0x00080a16856bdddd,0x00012be8c112df11,0x000000000002987f}}, + {{0x000b02027989cfc0,0x000a60ce9ab12f61,0x000973d1a5ee6cd8,0x000b3b69a1753a9f,0x00002bae5685f031,0x000d3c06632160ba,0x000f5cdfde9ae80c,0x00000000000a180c}, {0x000f0a3aff152330,0x00023e2c194158b5,0x000e1481b10c0c49,0x000e7d12ea20322e,0x00007968c7ff67ea,0x00027bab93eb507b,0x0006eaeb300ff9ce,0x0000000000097575}}, + {{0x000b71b2ec924484,0x00007325de655ef5,0x00027d8f4ae5da5e,0x00026ad89e4c34bb,0x0000ec5a615fa909,0x000c770ac8e61adb,0x000f8a5a2233d43f,0x00000000000cbb23}, {0x0003b01804f61225,0x000a4a7e6c5861cf,0x0004fcae81492249,0x00040be697e7dc09,0x000eb6f29135b0f3,0x000e89899783a7d0,0x000f55412469b007,0x000000000001bfaa}}, + {{0x0003c3de6d71b673,0x000cca438634e69c,0x000f00cb40863203,0x000dedfac40dd56f,0x000e4956a5de2ab4,0x00013f84d758e95e,0x00016cfc11e39451,0x000000000006059f}, {0x0008f87a862c83c5,0x00038fd8e2236750,0x000df4ebd7b9092c,0x000b1538ea13b455,0x0003013d382702ae,0x0009ca201de1275d,0x000351470a7b7e65,0x000000000005c77b}}, +}, +{/* digit=88 [{1,2,3,..,}]*([2^352]*G) */ + {{0x000bd58c2fd385c4,0x000d8281f6e58982,0x000c3afee2ff7056,0x000a41afd89abd8e,0x00007984feefe29f,0x000d20a64fcb0b0b,0x000cd50b0928a6d9,0x000000000006979c}, {0x0008ab27cda5c7bd,0x0001cce9c34c7521,0x0006dc0b027875db,0x000635250946a5c3,0x0006f39f53b9d464,0x000bc8b64b09a97c,0x0000d61d47bc20dc,0x00000000000d458b}}, + {{0x0004b16a9cc79c48,0x00075763e36638c9,0x000ff772bf788245,0x00011a8c66b40e9f,0x0008f384b70862ab,0x0001978760624469,0x000837e7cdf3bd69,0x00000000000064f8}, {0x000fbc5f9be69c3f,0x0007895900a21f89,0x00053a9326cbd6b2,0x000fbbddd3e6e471,0x0006baa2e2a03f65,0x00085282484f52e0,0x00032c1dc462a8e3,0x000000000007e4c0}}, + {{0x0008c150b5ec2626,0x00094cc580ea6853,0x000526b13f535e43,0x000604fb23480cfc,0x000a344146898665,0x00010a94595787cb,0x000c78425d7c6f4d,0x00000000000b2f1c}, {0x000940b59f5f9db6,0x000f455da8884e6f,0x000468b788890b3d,0x00081d7d99e417f5,0x000abf28fba2c648,0x0004eff801eeba5a,0x000fb720feb7b350,0x0000000000050deb}}, + {{0x0002dafb4e000df5,0x0009720ebf79c9aa,0x000a041b02faa426,0x00007e78d630d2b3,0x0007fa605dcb016c,0x000d0470520021d5,0x0007e66190f3e942,0x000000000008974b}, {0x000bb7ed0e0135bf,0x0003b6710da6c4cb,0x00011bda556d9709,0x0004b8ba583089a0,0x0004ea7dbcdd192c,0x000df1097171ab8c,0x0000ed715f60818c,0x00000000000205d1}}, + {{0x000c0f09863d151a,0x0004b4a6226f970c,0x0004a88f8872d167,0x0002e60a1193cac8,0x000543dda270b44e,0x000d647382ce6393,0x0006751a9e8a2138,0x000000000009d843}, {0x000abb28a3b6891b,0x0008a98a1222e3ef,0x000341bb8ccbdd0d,0x0005be5555088026,0x00017b38f0648047,0x000e249f5c39ccd9,0x000b74ea31304de8,0x000000000004d42d}}, + {{0x000330dc4e7217de,0x0009c39a689bbd9a,0x00001ce7a86200c3,0x000108a8d29457b6,0x000014ed4b4dcf33,0x00015625f312612e,0x00063bcbb21f3451,0x00000000000303a3}, {0x0009f7756d9dff06,0x00004aeb0c8c0639,0x000d964999e9958d,0x0002604683b37dcb,0x0007862b08477a02,0x000d69390fa4ced7,0x0003145a49bdc136,0x00000000000c0215}}, + {{0x000155abc265f191,0x000f0544874b6521,0x0007a036ebfb6d29,0x000afd631411ba1e,0x0000466d6415b303,0x000d4c2de30047aa,0x0007b13a1b676594,0x000000000002860c}, {0x0008a20d9745d768,0x00046fe140a72d14,0x0007d03c948ac2bf,0x0006df9f46f144f8,0x000bc0defbc46c9f,0x00024c075f7e7b95,0x000c39fbc9a96978,0x00000000000d7773}}, + {{0x000f0ec4d363b0b7,0x000bdd2db4d34f56,0x0005740dc154bacd,0x00065cc723a57c02,0x0007d8ded62c4475,0x0002f1a9ed98c359,0x0003b20aeac6d9de,0x000000000000a98d}, {0x0004a99c9e88ef97,0x00034d9708f06642,0x0001bd570d037e70,0x00032cf49cda0113,0x000a858467e24993,0x000748e8e546df74,0x0000738b84a55093,0x0000000000006f09}}, +}, +{/* digit=89 [{1,2,3,..,}]*([2^356]*G) */ + {{0x000f37fe6b03f897,0x00052c0a40cabe70,0x00091f1b94fec55b,0x000898c340187426,0x000636d9e57a8625,0x0008aa21169c9f3f,0x00016869d9d7f337,0x00000000000e7dca}, {0x000214154225907a,0x000ba24d49aebb77,0x000fbddce2036f3c,0x0001d01576f533f5,0x00099ef82ece667c,0x00007c14d372eee1,0x0007f6577723c0c2,0x000000000000eed3}}, + {{0x000850ebff25b818,0x00013c61db976bce,0x000a1c9cf36381eb,0x000b8ca060b1adcb,0x000188e2c178ce89,0x0003bdd3c41db448,0x00042cba6339f392,0x00000000000d29db}, {0x000bbd1437baf649,0x000f53521116eaf1,0x0005d6c8cce9ea0c,0x000c984bd79fe5d8,0x000e9d45eee49357,0x00088e01f2eda73b,0x0008731a50c59f62,0x0000000000075018}}, + {{0x00038b1bc1ced6c1,0x0004df323953ab33,0x000a2512b2fa2401,0x000dfd9bd32cb8d7,0x00037ecf35382937,0x00039941507c4e56,0x000a06b573960eb6,0x00000000000dd1a4}, {0x000a33b92253abc1,0x00050c625f400562,0x000bec6e4c3f3a23,0x0001e5b6220f24df,0x000827d01c6f66c5,0x00012372d9f97d75,0x00004860b1572404,0x0000000000004d55}}, + {{0x000a3b570044f6d8,0x0005fc552c6c6093,0x000f9e99da8c0559,0x000375c19610b0cd,0x000bb051dad5c9ed,0x000556643f0e7b4e,0x0001d87267a304c6,0x00000000000c96dc}, {0x000cd649696f60dc,0x000dbf0ed5f9a8b8,0x00051f0009075842,0x00066f4af5a4b4c2,0x000d20644cf2ef5d,0x000ae23c00b5bed3,0x000b66f7e4543a75,0x0000000000041325}}, + {{0x0006798502ee1353,0x0006c04ef7ad5a7b,0x000c6878548d78d6,0x0008a6a47591d88b,0x00046902edb49ae3,0x0002b143a27d9125,0x000df65dae3d8381,0x0000000000093a2f}, {0x00091ef1ecb7e486,0x0000807a00388858,0x000fb3b7ca4398f1,0x000c17172b1bcba8,0x00032d0033c73fd6,0x00016c28aa8d70f9,0x000ad79ea9eea329,0x00000000000423ed}}, + {{0x000a6b031af68d65,0x000b59949b33d112,0x00063134f5d066ef,0x00071e788e17f2bf,0x0002da9c088188ce,0x000d8c9e9d851baf,0x0005b6e869c5ba86,0x00000000000a0142}, {0x0009ab9cc7c38dc7,0x000eceae8b1c5d3f,0x000b0c8e79b3cc01,0x0002910c374bb97b,0x0007054874831494,0x000bc6ee13e13c45,0x00047be0e6fad435,0x00000000000b54d2}}, + {{0x000e68e47f9ea217,0x0003c9c85e50a3a1,0x000b0543d8520966,0x000ce81807f33dba,0x000a3db81e06cb78,0x000638d709d337d8,0x000babe223eae472,0x000000000006bf2e}, {0x0006763908bcca20,0x000662804b59c92a,0x000900c614fc9957,0x000ca1e7bc6949c1,0x00008f051155321e,0x000539f40bf2906c,0x000808b802a3289c,0x00000000000073cd}}, + {{0x000fc3dc90267bca,0x000956d34bdf61b1,0x000bd7a38090ba35,0x00051135a3bfdd10,0x0005f1296bdd61d0,0x0002e9a7c4abab57,0x00003d72da68494d,0x000000000008d11f}, {0x000319b6e5281755,0x0002247811121597,0x000526a929004138,0x0003ec67521898f7,0x000996c1da75ec29,0x000f2c7b3f0cf026,0x000af8b0dbd4c380,0x0000000000034e4b}}, +}, +{/* digit=90 [{1,2,3,..,}]*([2^360]*G) */ + {{0x000c4e9ab57885e4,0x000f3c32730e8279,0x00033a83277668a3,0x000fdbf3f5cd8478,0x00067272d4dd2bac,0x000b7577645f3641,0x000be2dd813a795e,0x0000000000050997}, {0x0005df7d4526b8ac,0x00040da054e2b966,0x00068890ccefdc5e,0x0008f31026116a38,0x0000dcd85e914c42,0x0003adc2d372af9a,0x0009e6fda6e90367,0x00000000000a8db5}}, + {{0x00029349f348fe26,0x0004251a033a4db8,0x000e0f2d8a80c6e4,0x000ce49a42a266f1,0x000b82c4d11e92ee,0x000bae08a87e037d,0x00078c875be1416f,0x00000000000c5394}, {0x000064a55c345b79,0x0001651f8b907a36,0x0000a11c59759db6,0x000bd666c51b9c32,0x0004ee565de82c74,0x00082c8c3635d3d2,0x00093c6bd6566389,0x00000000000daf6a}}, + {{0x0007ac96ab52d7dc,0x000dd68fe359d36e,0x000f3dea99f83698,0x00052c3fc704935e,0x000952e4e22d0ec6,0x0003e3ada1eeff1f,0x000871ba6777cb08,0x000000000008befc}, {0x0007324fc4458b0a,0x000e34ede689e853,0x000cf3cdc3eb9d08,0x00080517ab83c288,0x00052c8c1f48e0f0,0x000241aac1f3d5f2,0x0007e057991607af,0x00000000000b8a33}}, + {{0x000ca2047d1f8f40,0x00004179c59cea3e,0x000880cb97b10e4b,0x0005cfdba0cf3857,0x000b3a8fbafadda7,0x000384e0b94081a5,0x000a45b91de90a33,0x0000000000027aa2}, {0x00058a31cd3d8717,0x000220747bcc094c,0x0000191551c0619d,0x000e3e9722484f1a,0x000da62d1dda9c1c,0x000963bce13a7ee5,0x00021b77fd79343a,0x0000000000083d2f}}, + {{0x0009d92935034280,0x0008b3dbb3fa61cd,0x000a3f5ecd8961d8,0x0006574793ce8041,0x000783dbce4f35ac,0x0003cf6b0b2697dc,0x000164e35c5bf2e1,0x00000000000b0c4a}, {0x0008df1996e7f4a8,0x00038aa49090842d,0x0003b655f6623523,0x000e96c54d504e4c,0x0001b73310a3f646,0x000bde5dad74e754,0x000ef7ead7159618,0x00000000000aa09a}}, + {{0x0005ac9c05f620d2,0x000df609deb16279,0x000a25447b2b8795,0x0003132b378305ce,0x000869275f5e4a9f,0x000b089b5f101799,0x0000c514be746761,0x000000000000c81b}, {0x000e2dd3d7fa135b,0x00030ca90a9bf94b,0x0006097de4edabc4,0x000ac9620c71989d,0x000a390aedd01b25,0x000d8cd39b971b61,0x00011a995214d779,0x0000000000065c6e}}, + {{0x0003855cea5ad9ec,0x000d4a8393a23731,0x0006e8588fe37236,0x0004e0255a202691,0x0002a446bcc6d882,0x00051499b9dd9a27,0x000535c929cded53,0x00000000000cfe76}, {0x000bc0f428a70f0d,0x0009626f3d7d9b38,0x000a6acd906c92a5,0x000cc06759e6ddb7,0x0004e302b89191fd,0x00007e1f7ac2e190,0x000b4aad25c645c2,0x00000000000ab3de}}, + {{0x000338669b2cb8c7,0x00071a13f19b3741,0x00031ac1789cc21d,0x0000c997044a6f4f,0x0008dca1075c00ec,0x00020ab0caf5665f,0x000effded5ca3f06,0x00000000000a896d}, {0x000378285debab1d,0x000a5032ab2b2a9f,0x000438bbbc4fc5ea,0x000b6f725133304f,0x0001977a45672286,0x000abae4f0d16d53,0x00003b00a66036f1,0x0000000000095f01}}, +}, +{/* digit=91 [{1,2,3,..,}]*([2^364]*G) */ + {{0x000925dc8a89e47c,0x00066f2cdae310b9,0x0002591b317d8ec6,0x000f25ad750b29bd,0x0005dba15f3e9d14,0x0009fe2dd931b9c6,0x00043334db169ebf,0x0000000000052663}, {0x000c577396a26bac,0x000968859a4f76bc,0x0005596f973fb63a,0x0008d6a67ac4db88,0x0002be25d7557463,0x000db10ee5c8ea7b,0x000cd38a0f0a8738,0x00000000000e890b}}, + {{0x000bf56a7de9f2ff,0x00022883533aee6b,0x0008916bc1e75ddf,0x0005fb546d0fcf42,0x000a4ab05da78f0b,0x0007b66147e6b0a0,0x000cf2cd91a869c6,0x000000000006c6f7}, {0x000593da575680e9,0x000d7c072a1067a9,0x0006a216ef99f7a9,0x000192c8f0ce9469,0x000fd11494417378,0x000ccf45adf2e7b9,0x0005684fa17cb614,0x0000000000045f03}}, + {{0x000725d8721dd338,0x000930f4ca77fc15,0x000b6681d170fa2e,0x000625b4ef805afc,0x000fc015c5864873,0x0001a8c93bddba22,0x000f427091b50aa4,0x000000000004c005}, {0x00044f31df11a4ec,0x0009ff43d3745415,0x00007d532af636b7,0x000a10c82770690b,0x000b77513f003efc,0x000c3af9bd0c25d7,0x000adac0015cc12d,0x0000000000094c6c}}, + {{0x000e8ca221c6ffc6,0x0008d8b70e94f6d7,0x000aee261a6327ec,0x00019ba4d3ef22f4,0x000a1999f948e222,0x000fe73027f8a712,0x000af025575e96ab,0x00000000000564a1}, {0x000b25cff3d9db0b,0x000b1045e9c658aa,0x0003561802f8c969,0x000b825c3db161be,0x000aba33a906dd23,0x0000e41f205fb173,0x000d1d9a8b5ecb87,0x00000000000ccc30}}, + {{0x0002aea737bc7de5,0x000c2bed7d94c6a0,0x00033bc161f07397,0x0006af6106834bff,0x00044c4f74486541,0x0001dfc06dade8a1,0x0007bc62227a1d88,0x000000000008dc2b}, {0x000d41c50320c146,0x0001c4f1154170fa,0x0005645ed0c5caa2,0x00053aba0f1b9617,0x000d48b0ec8e98cc,0x00072d0342f68011,0x000e384bbc34679c,0x000000000009c576}}, + {{0x000919b377dae55c,0x000a5d9243926709,0x000605401369e5f3,0x000feee0e35f201a,0x0003196f23f3a1bc,0x000c1d068d25be5f,0x00050b298c67f2ff,0x000000000000c3d9}, {0x000d303412559962,0x000fa5e15ab9975c,0x000533c7cf964ba3,0x0002ec947a7e2696,0x00022ac22e49a9f8,0x00006f090f02af9a,0x000cb0dfb3103fa7,0x000000000003cf8d}}, + {{0x000298e2791b8ead,0x000259f5f44f9ed5,0x000ce416b16e73bb,0x000f258f9627f9cc,0x000938fd51bdd7e8,0x0006a4a7922499dc,0x000250a9d7497c84,0x00000000000026ae}, {0x000e3b88b9c7c3db,0x0009084d47d19214,0x000f52469be04308,0x000138806a52e316,0x00027f08953f9b2f,0x000fcac083fc8da0,0x0003fc22d074d292,0x00000000000701d7}}, + {{0x000b913b1d418177,0x000cbb7b856256b6,0x000f3717023b8633,0x000ddd5c91b8ffd9,0x000155d38511b4eb,0x0000da525ef815ae,0x000bbd98587d551b,0x0000000000034a9e}, {0x0005ef8ce15a684b,0x000b8844811fa3d0,0x000d70ffbd583fff,0x0008bb4f623789be,0x000e404980584b26,0x0002b435ab26ae5b,0x000eb3b47a5ded3c,0x000000000008fcec}}, +}, +{/* digit=92 [{1,2,3,..,}]*([2^368]*G) */ + {{0x00069e40601a7dfa,0x000cb70765682efa,0x0003c3be2bc6c542,0x0005ef0a6db6169b,0x00012032d5992a93,0x0006f13160029276,0x000f51b5babb2d3a,0x00000000000db26a}, {0x00039e460e4f3b3d,0x0000200c3401304d,0x0003ff9e293a0443,0x000e2ed0f9b36565,0x000934031d18a1bb,0x000fe1224e17a5d4,0x000cf5e3661047f8,0x00000000000623c6}}, + {{0x000ad46943acea12,0x000a859367582797,0x00066c45ce5e5faf,0x000351af6a27741d,0x00087f929e0d5d96,0x0003f1afbeab94db,0x0001dd865ba01104,0x000000000005fb63}, {0x00040d1f1e090717,0x00038192e065294f,0x0002fb37089941d8,0x000228db7cae5f66,0x000d6a828b037bcc,0x000f301aa02eaecc,0x000a077c48a2ea91,0x00000000000f5eb1}}, + {{0x0000bb651b864105,0x000154dd3bd8408c,0x0001daf9340de0f6,0x0006998ba0668966,0x000cad713bbd1ad4,0x000ca4fa5b7c679a,0x000a1cef9389aff7,0x00000000000d68b6}, {0x0009051030865994,0x000093acfff601ae,0x000275b28bea6bfe,0x0006441c66ad734b,0x00042fd3eb165e2f,0x0004e211749f144c,0x0001a3bd7f22082e,0x0000000000041791}}, + {{0x0002b76266c69181,0x0004edf0ae8df2d5,0x000906801183c79d,0x00017ca1dd286917,0x0009709b678ede37,0x0000acdc60c1db87,0x00003a0288959a40,0x000000000005ca2d}, {0x000fd72975fb9eb5,0x00084e52534b365b,0x000f103241b3e4c6,0x00034f873eece315,0x000d642a67991a9a,0x000a7e8a80fb0c7b,0x000238375cc0cfea,0x00000000000d00ec}}, + {{0x0005a38c81bac32b,0x000bdb67d88fb498,0x000506df3f19aea7,0x000433ede7910791,0x000a20ec085cc26e,0x0000eefb30ed8fc0,0x0008c22684a901b3,0x00000000000e855e}, {0x0003801582a535c0,0x000a23d31d6c91c3,0x00014e8637446682,0x000380e755f1af17,0x000a6c985ecfd938,0x00065084e6a8e434,0x000ec653bec9c1cb,0x0000000000070309}}, + {{0x000a18c1c3a6f1f6,0x000f49a2461c2364,0x0005b210149e5bc7,0x000bfea0ae15fd0a,0x00027ac4d98f2265,0x000831902d817b35,0x0001fe9788aa4511,0x00000000000d4616}, {0x0004b65395df8642,0x0003d2a7cac2f471,0x0003af0e90c12844,0x00048d211070e2f3,0x000ab769926571a6,0x000aea1556050d8b,0x000411cf8f24a040,0x000000000003522b}}, + {{0x000a6d728de1a709,0x0002e45d4ed63b83,0x000803bc165cd24c,0x00029022b267f598,0x000fc8446afe76be,0x0004d6f50791c22a,0x00077ed6ce8b8859,0x00000000000e3f1f}, {0x0000a258fb32c514,0x0006d1a72b16f0c7,0x0009dc6ac4083dba,0x000743e1cad2e785,0x00041e0e640d6a1c,0x00074648529ff0a5,0x0001fb4cd519be4b,0x000000000004584f}}, + {{0x00051eecfb6439ff,0x000619ca8cc9cbbc,0x0007f10adb0b792f,0x0005d53059b2bbb3,0x0000730e5dc37211,0x0001d89988fe7ac3,0x0007a54e67ef6984,0x00000000000c8ed3}, {0x00004a045edf3ac0,0x0005e48164b3dc80,0x0006b3cd6d953b04,0x0004643beed38ce9,0x000fa3e30b3db7fd,0x000ddd484993cfa9,0x00075ddcc5e7af01,0x0000000000068401}}, +}, +{/* digit=93 [{1,2,3,..,}]*([2^372]*G) */ + {{0x0007c9efed4b7222,0x000ab79768e9e648,0x000c0eb72891fdba,0x000821bbe0c67d8d,0x000a6909fdfcc194,0x000ccdda7537a6a4,0x00026cae6d705195,0x0000000000092b39}, {0x0004e2be194fc116,0x000ab2c5dd6af51d,0x000bd2ad4ffd8821,0x0007388397a3bd3a,0x000a8baf59c2dd3c,0x0000ec9589d176c6,0x000315d5c6219e38,0x0000000000054e8e}}, + {{0x00045f27ccc6232a,0x000560683349d797,0x0004a272721dca0a,0x000181050663b9f6,0x00099733dac647c7,0x0002137c5f327d14,0x0001c703f55e4a29,0x00000000000da59e}, {0x000b58d5c8ca43e3,0x000c7baa8b4dce4f,0x000e015119c73eca,0x0008c1db0322dfa0,0x0007fd69954f99a8,0x000a29889394440a,0x00064a9060473da5,0x000000000002f048}}, + {{0x000df3b38c2a0a79,0x0000190762c1ecbf,0x000c4019f47729e5,0x000849f5c47c645a,0x00056d72e7487420,0x000e11773b082319,0x00050eaeac683b8e,0x00000000000fb5c5}, {0x000722990680a985,0x0003e6d986d69658,0x000da73ac5c10442,0x000136fd94d52235,0x0009ed5b01c6d13e,0x0008fcdcc1aa7541,0x0005ee7c2014b140,0x00000000000cffde}}, + {{0x0009c28bf944531a,0x000bbc7d0abb1f4a,0x000c2054201bd117,0x000defca80767df6,0x000871820908eb67,0x00074d19a6fa174d,0x00081a3215df4856,0x00000000000960a0}, {0x000602066fc6cc5c,0x0006993ed71807f6,0x0002dfba1c3883c8,0x000d56efe787e592,0x00098fe8b96f0909,0x000ab68115ae74fc,0x000e0c8fc342c435,0x000000000000b991}}, + {{0x00047f2399bf97b6,0x00021c600dd569b8,0x000917d84fb94235,0x0000a27de43c1d70,0x000e616bdb162d7b,0x00029795321bf3a9,0x0008b233ce767c96,0x000000000009e439}, {0x000800714c3c1216,0x000934069cced410,0x000f1cffed0b3d13,0x000bf89c53942369,0x0009acb24e389981,0x000da473507d9c07,0x000cd9d2e5e904a2,0x000000000002f1b6}}, + {{0x000ee38595e6d6e3,0x0003bfeecc6e7a5f,0x00087ab2bdd62b28,0x0004e20e81879cac,0x00089c4a27124845,0x0000749c3d0ea9cb,0x000f93ba8b6f3354,0x00000000000e971d}, {0x0002b66fa9a77846,0x000b64c2a1b129de,0x0005071339da6099,0x0000b53435739873,0x000a57040c00e6c6,0x00091a7b4b9f56a6,0x0002466de6489f62,0x0000000000053bc3}}, + {{0x000554330abff5cf,0x000ae828a9dbe5c5,0x000263a5672326a0,0x000428089faa5816,0x00002dba2a1d1558,0x000197ae62da0922,0x000abba225f631df,0x000000000008f4eb}, {0x0005dcc2ee38818c,0x0001bbbadc9b2cb5,0x000ab544fb33427b,0x0006cc2759413f76,0x000121dd1222072a,0x000ac2c737b39ea4,0x000296d6baf9e196,0x0000000000075d46}}, + {{0x000b2448cce42a48,0x0001555671128874,0x0000a30717d307ff,0x000207ce99ca16c1,0x00057c7a26b97e21,0x000c3ab00785fa10,0x00059f33c28658c2,0x00000000000d79cd}, {0x0004e804722ed471,0x0006f9a674db4469,0x000c9ef186db33b8,0x000722166bf2c6d5,0x000abd4c3791990e,0x000696ee1ba29aff,0x0006a5d98510cf20,0x00000000000f93d2}}, +}, +{/* digit=94 [{1,2,3,..,}]*([2^376]*G) */ + {{0x000bbb4af21bdc49,0x00065e7cf5babca3,0x0003f75a29c95b64,0x000531d04fa84740,0x00045780efda9c1a,0x00062cb5a399e848,0x000f3b6e9c12be4f,0x0000000000094a5d}, {0x000aeeb34b3f3b9c,0x0006d3c15fe11c69,0x0009e05f576f256d,0x0006da4be294c912,0x000aa99b83983ef7,0x0001b9f6a5cf21d5,0x0007bf50679cf875,0x0000000000054d0b}}, + {{0x0009b2dd4ee443f9,0x000fcc84f9171b4d,0x000ea8b580b8da51,0x000753ba05068eb1,0x0004077777efcdc4,0x0004cf44aa177ad1,0x00044e0b7f54eb7e,0x0000000000018601}, {0x000becbcecff7dd7,0x0002b1279ed4c55b,0x000e10b5af306b99,0x00058fab8cdb6cf3,0x0004a0c7614aebc9,0x0002c6d8ebc4f93f,0x00038ac5bd6cde7b,0x000000000008d65f}}, + {{0x0000f0922779f009,0x000bb4da7d63a943,0x00025a4f402efdea,0x000001668841d1ef,0x0005b10366f61e8c,0x000b121e90688f48,0x000c1e38ee34b005,0x0000000000014e0d}, {0x000a8eaa6c54d224,0x000e683c1d44e6fa,0x000050a1a6ad5a9a,0x000274453715349d,0x000dc5ca18f66b73,0x000ef86e35065bdc,0x000f5d677313cf2d,0x0000000000033ebf}}, + {{0x000a3669bab3aa8c,0x0001fffec66743ba,0x000af6b2298cee7c,0x0001365ee61bc95f,0x000e8948b6e23181,0x0001644be358bbd6,0x0007a71cd9c342e3,0x0000000000060a8a}, {0x000569b8813054fe,0x0000ae065519f224,0x000901053fc569ee,0x000c03b8517ae8b7,0x000173750d6e8957,0x000aee6d7c96a040,0x0001144eb2e364b2,0x00000000000ed099}}, + {{0x0006635997173f0f,0x0003b71714896339,0x00025444ec94256f,0x0005f33fe56d0905,0x000638a1b2efb078,0x0000fc15add43e0f,0x000d362df35cfb36,0x0000000000090b3e}, {0x000659873829fac7,0x0000bd4970bdd7ef,0x000e315a3dccca14,0x0008a3e4a43c4f3e,0x000fd9cb4808c99e,0x000c3d0948013609,0x000223f3fef0eaec,0x000000000008642d}}, + {{0x0005a811304eaea0,0x00018ffe3104b37f,0x000808bc57214341,0x000ed42cca2f1506,0x0002c421c69415aa,0x000b87b368618aa0,0x000d5f4d6a091dfa,0x000000000000e786}, {0x0009341357580c61,0x000ee0be97746d4b,0x000cd6467b049086,0x000823b7a70d4224,0x000809de860f6094,0x000cf60205928791,0x00086e7f7fca496d,0x000000000005f460}}, + {{0x0007768861c39ce8,0x00007b509f7028f0,0x000fc09189281be1,0x000832a7f07f3291,0x000d8fbdcb83853d,0x000a4650d074fdec,0x00072080ed8de9d6,0x00000000000a5d68}, {0x00079ea50cb0600f,0x0009ae66666d224c,0x0008a45c66bc5d53,0x00099510da620fbe,0x000fdf866f777e2d,0x000045beb901145f,0x000008442a37e5db,0x00000000000beb1b}}, + {{0x00048a2dbd606668,0x000c9bb39a179bed,0x0009439710e05c24,0x000abe3b91f14833,0x000fbd39a2bc6be6,0x0001d813ef972804,0x000d4a06737e54f9,0x000000000001a87c}, {0x000d0c4b5d0bc923,0x000b2bf88b2e5982,0x00052c33a491bb9f,0x000aedfa58af0431,0x0006ac0511b07a93,0x0007e2c198853cdf,0x000d8a9d7f416d06,0x00000000000f9671}}, +}, +{/* digit=95 [{1,2,3,..,}]*([2^380]*G) */ + {{0x000f6957a24c76d1,0x000222e7d0a157df,0x0001697f0073308f,0x000ddc63eb317f9f,0x00015adf71d715f1,0x000858bc3f027507,0x00071a2ce33c2eee,0x00000000000da73b}, {0x00035cb76cad67cc,0x000de9ef6a709ffc,0x000c7c7ed1727cfd,0x000f5a67502de655,0x0001a47019739f5b,0x000ea7b24d11122a,0x0002e182cfaac2a2,0x000000000000a458}}, + {{0x0007ab134ebd334b,0x0005196a1e74f032,0x00014e69e2323c83,0x0009c4fbe35109cc,0x00004521d7e61244,0x000d002931fad8b3,0x000229c85eb23d57,0x0000000000035f68}, {0x0000d113aed4dbde,0x0000c7c5b86677f2,0x000189b64d0338ba,0x0000456757cd5717,0x000e3a1c866b067b,0x000ba88c0eaab81b,0x0007dc72a6af75bb,0x000000000000ef56}}, + {{0x000ac045af51c5bb,0x00010597a26ff058,0x00059bcfafa87a4b,0x000aff65c27f5f6d,0x0005d8d544e60b06,0x000275c32ce5ab66,0x000e7c92f031be99,0x00000000000f42e0}, {0x000c8a905adb28dd,0x000175bb28b05c35,0x00031ae347df4e7d,0x000d299f93fb7dcb,0x00086e2ccad9e73f,0x0004d4f57fcd999a,0x000c9bb8c8a6df33,0x000000000009a2ac}}, + {{0x00030bea7fcf0ecc,0x0006a3afdee26fd5,0x000cc01988c78a70,0x0008ee3ba4a33026,0x000a2a883b7f340c,0x000d7412b1a6ea51,0x000b6e64f27976d5,0x0000000000091251}, {0x0008bb3d89325d7a,0x000d25f3f8f40ad2,0x000216e9116c139b,0x00073d6ad61b2cbc,0x0005d542676dde3b,0x0007d712bf08398d,0x000023d373931e8e,0x000000000007f96e}}, + {{0x0004e4e1b942add8,0x000e15adf3d9957d,0x0005ba50de0860ba,0x000ce33a82d3736d,0x000e718c8aa3f9fb,0x0006ccf69f307823,0x00065b860578cf41,0x000000000001ef84}, {0x0009d5f6cc7a9ceb,0x000939ee0cf97011,0x0002f3cbdb7c8fc5,0x0009a8dc09da90cb,0x000dcbf3ccb8f99b,0x000a626321f521c2,0x0009da6bf6694891,0x00000000000854fe}}, + {{0x0000917c5016c853,0x00049e44e3f85fb2,0x0000e1793eed8bac,0x000254501c4e171e,0x000cce52defb1004,0x00063100ac12df68,0x00035fb1fae2fbf3,0x0000000000035732}, {0x0007edc8c1da40d3,0x000c58d4deb655bb,0x000bfb14f6d9d28f,0x00085835a4e118f4,0x0001308772e93249,0x0004e48765abfa96,0x0007b241c7611ff5,0x000000000001f586}}, + {{0x000782cfada548c3,0x0001e463031709b7,0x0005afd4d0d5166c,0x00097fff172c7d05,0x000735b193cfd8e4,0x0009df4bc7f7009d,0x000e376b9fd3f7f1,0x00000000000b46f1}, {0x0004c800fedd3a70,0x0000987c6eaa8c8c,0x000dff8047ad562a,0x0001042539eab96e,0x000779b8f5cc2a12,0x0007840dc29a6d5d,0x00030f33803a10b7,0x0000000000011a6a}}, + {{0x0000457a60ad991e,0x0005e3c90a59f250,0x0005eeda9345d42a,0x000885c1a0d58e29,0x000d6816b0099aca,0x000e4718f77b9b6d,0x000c458e5c12139e,0x000000000004e4ea}, {0x0005f9c85e9bfc5c,0x00086c00c3568eed,0x000b5b4b0eba61f9,0x000ffc75eaf3eab0,0x000d42a87e32cba1,0x0007a882f5f99092,0x0005122e7b49c76f,0x0000000000029040}}, +}, +{/* digit=96 [{1,2,3,..,}]*([2^384]*G) */ + {{0x000dbfc10083fe9c,0x000bfcf17f9dc084,0x0005e1e364b063e9,0x00054ffe437b29cb,0x0009e27ee9e2a694,0x0003af03b6628d78,0x000d6256e3b975ee,0x000000000002f532}, {0x000e1d6aa4b057a7,0x00043cae15213418,0x000003dc9b2b9ddc,0x000a959fa4d82608,0x00055ae17566902b,0x000c82eb4bad700d,0x000ea716b2c5dc14,0x0000000000036708}}, + {{0x000e41b4a9c3409a,0x000849cbd62cd8bb,0x00047e8c2d4de38c,0x000bbe98f886eca3,0x00049b432990b7f3,0x00030035684860e8,0x00029ed19a3bafa4,0x000000000000e209}, {0x000ed539231869f6,0x00098f15ff660294,0x00050e66a84ebdc7,0x000de8d2955612f1,0x0006ec97a4a53ec6,0x0000d15fe2d95b4e,0x00024d868731f0f9,0x00000000000aefae}}, + {{0x000253f48e5fab58,0x0006889f97859df6,0x000dee3f7bba228a,0x0001d4ea62a5c827,0x000ea3045da6826e,0x000d3d237ecba3e4,0x000358beed1da058,0x00000000000b8b41}, {0x000293271a9d4293,0x000d2730e7ac94c3,0x000f438703e7096b,0x0004cace8c1c6462,0x0001055d39de9ba9,0x000b71db3ec7c382,0x000a11c89705a82f,0x00000000000781b6}}, + {{0x000606caa5686427,0x000cdc1951fb0886,0x0000daff401d6319,0x00009bb2754f1d38,0x000e0e0e2bf19831,0x0005a3da40f7db33,0x000197348c4e1937,0x00000000000840a6}, {0x0008d7f45b2f9769,0x000d11687a735d22,0x000434d1e1dee1ff,0x000773c0aa9e79d1,0x0009f56de9654a25,0x000dda7af656f6e2,0x00063fdf84666df0,0x000000000003f9e4}}, + {{0x000a9c9951735edd,0x0006afe9f90d3ad9,0x000f97a0b45a07d7,0x0007d6ec36e3b706,0x000f580c8e6dd513,0x000bab45c3ea55f4,0x000d5819398b33e1,0x00000000000aee76}, {0x000d9c115c469d59,0x00071e72c8214ce4,0x0006b3ef7ac6abe3,0x0001ac198f9553cb,0x000a1c98ad467fa5,0x0000ebbb2019ac4e,0x0000a68942e16e01,0x00000000000bed3c}}, + {{0x000f13cdd62e69a0,0x000461a814bf695a,0x000e4323f4986584,0x0001bfa2a4dd16f8,0x000a76fb9f2b33c9,0x000b1f1114af4d9c,0x000b45a23635ef75,0x00000000000a0891}, {0x000bef0dcdd6b903,0x0005f5d1b87ba5f6,0x0005d7ac1d6bdae8,0x000cb4543d4ef435,0x00064aa784ea70bd,0x00070e4220ed12d3,0x000483009cd34901,0x00000000000280cf}}, + {{0x000bddf9aa596a1d,0x0008952c04b79c0e,0x00099a2770f3fa4a,0x00044bf1911b3184,0x0006bb897d318407,0x000e9de5dd13f080,0x00052b376dc8b81d,0x00000000000c996b}, {0x00047f465159b51f,0x00041d91e47b224a,0x0009b71ad19e642b,0x000c167eface7572,0x0001d4805ed6a441,0x0003fd6654eb9588,0x0005778fc93daf3f,0x00000000000cc570}}, + {{0x0000b16f46a35f8c,0x00065a630c20c4e5,0x0001f4362772ed03,0x000aca10c0dec6cd,0x000ba9e2f55428c8,0x000bbb1705d34bb5,0x000f6e8e81b4f732,0x000000000008363b}, {0x000ca7950547e910,0x000969603fe028be,0x00047954fea1ddef,0x000bb8efc191d12e,0x0005dba97347c0da,0x000656aaaf0e463b,0x000cf0b7f7c207a8,0x000000000003f08d}}, +} +}; + +#endif /* IFMA_ECPRECOMP4_P384_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecprecomp4_p521.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecprecomp4_p521.h new file mode 100644 index 0000000..13433c2 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecprecomp4_p521.h @@ -0,0 +1,1342 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_ECPRECOMP4_P521_H +#define IFMA_ECPRECOMP4_P521_H + +#include + +#define MUL_BASEPOINT_WIN_SIZE (4) + +#define BP_WIN_SIZE MUL_BASEPOINT_WIN_SIZE +#define BP_N_SLOTS NUMBER_OF_DIGITS(P521_BITSIZE+1,BP_WIN_SIZE) +#define BP_N_ENTRY (1<<(BP_WIN_SIZE-1)) + +__ALIGN64 static SINGLE_P521_POINT_AFFINE ifma_ec_nistp521_bp_precomp[][BP_N_ENTRY] = { +{/* digit=0 [{1,2,3,..,}]*([2^0]*G) */ + {{0x00031a16381adc10,0x000f3f18e172deb3,0x000e0c2b5214dfcb,0x00017fd46f19a459,0x000ac947f0ee093d,0x000d50a5af3bf7f3,0x0001457b035a69ed,0x00009c829fda90fc,0x00011cada214e324,0x000274e6cf1f65b3,0x0000000000000000}, {0x000460e4a5a9e268,0x000f4a3b4fe8b328,0x0004351396120445,0x000fd683b09a9e38,0x000132062a85c809,0x00064bf7394caf7a,0x000d7de8b939f331,0x000224abcda2340b,0x000163e8deccc7aa,0x000de0022e452fda,0x0000000000000001}}, + {{0x00090cf08640909d,0x000f1c99dd36bc1e,0x000b26b07ecb3fa1,0x000ae2d7a0e797d1,0x000aa83d508251d1,0x000bd9d9026d377a,0x000372a82ebb4df4,0x0003cd8e66031a96,0x000a461413a3a019,0x000f3f3417e59440,0x0000000000000001}, {0x0003d2ee331fe1b6,0x0001ab6b30fa0d81,0x0004af6e07a7b8df,0x000d19247a757e5f,0x0008fb5c9c9bfd4c,0x000dd9f1bbef4f92,0x00090d14c836216d,0x00083e26d4bba055,0x0007769f85ae35a8,0x0009338053f9f677,0x0000000000000001}}, + {{0x000e9cf4d4910f78,0x0008ce976f1bd6be,0x0004316197502d2c,0x000acff10dd75a48,0x00019028ed35e8b5,0x0008d69f8b251d24,0x000d6bd0896bd46e,0x00072d891ecd5cf2,0x0005acaca3cda953,0x00048caeec8eb532,0x0000000000000001}, {0x000cfa6c0ee5f7e9,0x000c4650f7436072,0x000de49d2c8212fa,0x000f61e3867882e4,0x000bad816ad6768e,0x00061716ea67c6e2,0x0007c558fd1aae77,0x000fd4154e818be9,0x000655c0a7978aab,0x00016eeccbcfc363,0x0000000000000001}}, + {{0x0008d6d77d92b8ab,0x00043a09438c8179,0x0002d8472d2f17de,0x0003c5783350ea81,0x00083a8745c474f8,0x0006432cf1257f1e,0x00062eaaaa0e9e75,0x00048e2ff9cd7e03,0x0003e483866e30e4,0x00010261aa5a41a4,0x0000000000000000}, {0x000a0825be109849,0x000fa3fe1a372686,0x00078234ce8ecf10,0x0004adc2f75dbfd7,0x000c2a029127ba85,0x00093cf941f2a5d1,0x000731ff178cc83f,0x00077b7371970dad,0x000585a55db2a90d,0x000ce95b39f00bc7,0x0000000000000001}}, + {{0x000194afcf14a49e,0x0005a84b7647983c,0x000f36c498b9c6ad,0x0009bf3cd194ebf0,0x0000a11b8897f578,0x00021c1e0636af18,0x00081ed5c78bbd67,0x00077eda9f869267,0x000e027585fbd2cb,0x000219639ede19c8,0x0000000000000001}, {0x000d6f9bbc6f7598,0x000e61f46f584865,0x00092b9aa7bfc0b9,0x000e7affbce8f803,0x00079ba188aa0108,0x0003ddb44be48396,0x000ec0be4d01a384,0x000f4743970028f6,0x000a54089488e6c7,0x0000eb764515b988,0x0000000000000000}}, + {{0x000fb915a75b36d6,0x00098df6fbc9035c,0x000ab2bf9c05711b,0x000a98df4617b374,0x000b9ca70393d11c,0x00092fde650b0a9f,0x0000a8356f25580a,0x00084bbfeb8e79cc,0x000a24068cab11e9,0x00090ca9977f9a7c,0x0000000000000000}, {0x000f780956b43319,0x000666bc2c6a278b,0x0005aae506d6f0f5,0x00013a79101ee3dc,0x000464efcb64c26f,0x000b655b96872b32,0x000205493100d454,0x000db9ed2d404739,0x000a371d8889555d,0x0007ac35716e9382,0x0000000000000000}}, + {{0x00015b574766756d,0x000756c4140b766a,0x000a87ee130cd00e,0x000e71dde237ca9f,0x0004c6c64d36f986,0x000ec61846855fe3,0x0000c69617b88a62,0x0000747aa4191478,0x00005839d062f917,0x000fb1a3775b2fed,0x0000000000000001}, {0x0008f4b46df66eaa,0x0005c5e48292928d,0x000952eef7e3dae3,0x0008e70d2fcf3b38,0x0004f15ca91d1a2c,0x000ab5e87949e6f6,0x000edecc51365ef2,0x0001681412786eb8,0x0001ceb423c5ae2c,0x0001508868ec18bd,0x0000000000000001}}, + {{0x0000208b103ed8c0,0x000cf52a553c6734,0x000dad37a0202c37,0x00046bcf0d5ab144,0x000a4f845acc60de,0x0007adff52dc2bcf,0x000c51d82fc1314c,0x000ec54d801f0545,0x0005808712dea714,0x000fb931541a41cb,0x0000000000000000}, {0x00058cc64475550b,0x000b21788f8bc00e,0x000a004a389a9c56,0x0002e2bc34cf9dd4,0x000da5ff85d06f83,0x0008c4f4e0552c88,0x00041ef30833bd47,0x0006f4f16038ada8,0x0003c429dcd227c7,0x000e2410247ed5b7,0x0000000000000001}}, +}, +{/* digit=1 [{1,2,3,..,}]*([2^4]*G) */ + {{0x000f45f44362a272,0x00062847c0d42017,0x0004003ee6e299d3,0x000c2b186f86dfae,0x000005072cb2ed3c,0x00094a1ff8c430e5,0x000197c69058c452,0x000b30c97e9f9eeb,0x000563a1d859543f,0x0005682eed4bed13,0x0000000000000000}, {0x0001e8dcd0f160b5,0x0001397c4de39d5f,0x00069468c3199a97,0x000eccc4294f1802,0x000a3d505983ad64,0x0000e0c1709cc6da,0x000e51864a5b5c16,0x000261dc006a8763,0x0009e9f34b9099af,0x000b800fe38a58c6,0x0000000000000001}}, + {{0x00059ae8d65b4828,0x0008ad5c3b72b54f,0x000ae6b62e3c123c,0x000d2e8fca4e7ba3,0x000cb7633eb4deb5,0x000b750251aee39a,0x000d3f677dcf2f5e,0x000b32a703401c6d,0x000a5d0ad9a1f1f0,0x000d234a21b83d7c,0x0000000000000001}, {0x00011dec3fd34650,0x00054d4c8a50ab55,0x000b2d5cb0b1b4ae,0x0005079b6fe6cc64,0x000eb5cdba6f0e3f,0x000ef10266dc0c66,0x000ef80e32e16ebb,0x000c5faff80cb3e0,0x000f9f041347fbde,0x00055c088b1af3af,0x0000000000000000}}, + {{0x000fecd778ec8555,0x00013eb0956c3cd6,0x0005e90a57516438,0x0007f84773320ba9,0x000dfd66f6e10d88,0x000a16d04c079fb7,0x0009e6452c01f397,0x0006b339081619a5,0x00076316466573eb,0x000a315640ac72e5,0x0000000000000001}, {0x000e34d296b0561e,0x000ac0c9e92b6a35,0x000402420e573d8c,0x000d871e142cb792,0x000bf1bdbe0a1707,0x000d7310a6b250ec,0x000894a7b366170d,0x000fe4a684f16643,0x00025d9fae0bb793,0x000c96f6d42adcf6,0x0000000000000000}}, + {{0x0006ae6a85f73844,0x000ee1c671907ba1,0x000014d027ea0da9,0x0001f3800efad55c,0x0001bb9d3e0160b0,0x00038df5e9e2f7ed,0x0000a5ce67c43969,0x000dd40c305dd65b,0x000c97f61533f5ed,0x00097668a79ebe01,0x0000000000000000}, {0x0009d235c3ae123a,0x0008b82c52fc6fad,0x000edd0329eea2d7,0x0007cac021e0e0d2,0x000a5887a53dcf1d,0x0005d60b2dd0a846,0x00031dd1b4f43d5c,0x00064597d8ee0e86,0x0002bdcac36cdd8e,0x000902b9d50810be,0x0000000000000000}}, + {{0x0002ff15cc9efbda,0x000d0adea1bf15e4,0x000712003c28f70c,0x0009acbb183fe29d,0x000e35ff46058b74,0x0000e6fbe9505c1f,0x000949d4ed7c586f,0x0006bc96db22c518,0x000e9fa6b4caa767,0x0006b3b723ba4dae,0x0000000000000000}, {0x000d4a1dbb01ce30,0x0002733373d1589b,0x000695b9bd79abea,0x00024c417444789b,0x000a3e366b3687f5,0x00018eccb8e87edb,0x0007fa4355949d4f,0x0003b1b5225855c5,0x00040d600e111ea7,0x0002ca9912224327,0x0000000000000000}}, + {{0x0001ccaa77955b13,0x000980cc45066e57,0x00005f6f02b3f9b1,0x000f381f6757639e,0x000d14e03775ce84,0x000770c3535ef348,0x0000c573845ff7f4,0x0007edbd50365fb5,0x000ae96135b16a31,0x000b1cbda8a1cae0,0x0000000000000000}, {0x000fb5af8ac6debc,0x000ccace499e422e,0x000f85f9a34dea5d,0x00099ffe8fc76f8d,0x000700f62f621078,0x00048a20afbdb94b,0x000353ffe99ecf56,0x0009d5421253436c,0x000d53ffd3fcac92,0x000092a9413337e3,0x0000000000000000}}, + {{0x000cc6739bc22034,0x0002d46c578a3cfb,0x00074dd1918f2158,0x000820e4ee32d9c8,0x000ee4769c451119,0x000a52dc0b788cd9,0x000ea08ad1dce239,0x000e6c51ce30fbf6,0x000c6808b739646f,0x00014612aa8ed807,0x0000000000000000}, {0x00084f57ce386096,0x00001bd8dd33a19d,0x0006d6cf7616b3bf,0x000aa0ec8a31fd3f,0x0000f33cd7d20cf3,0x000d8c7b490a7ae4,0x0003afd25187398f,0x0001d4b82d520c24,0x00008bc63a1c99f6,0x000da20702f937af,0x0000000000000000}}, + {{0x0005f3504967c9f8,0x00081372b9b7d4f0,0x00090711a09dabb5,0x00052fc0a79713f3,0x0008ebea8efb840b,0x000ba724a4b43b13,0x00089257bae703ec,0x000c5de16b6e9669,0x000a9811fd41e4b4,0x000c0fc1b18e0380,0x0000000000000001}, {0x0009bdb977d41aa0,0x00084a0964efd898,0x000049a3954725e1,0x0002567309871a1a,0x000d6462734e1923,0x0004d24ffa586e4c,0x000a7d5e1d8d7ce5,0x000f69f3efbcb30d,0x0007de3d3416e700,0x00041ee729987fef,0x0000000000000001}}, +}, +{/* digit=2 [{1,2,3,..,}]*([2^8]*G) */ + {{0x000757d0283c0ad6,0x000745c834df0056,0x000e5caf285c0d9c,0x000faea391f23599,0x000232d4e48a9620,0x0009bdc7a7b74615,0x0002fd4f47934e59,0x000c4f65ada3d4dc,0x0000798974c81e39,0x00061064a2c57e3c,0x0000000000000001}, {0x000a38d40ea9cd04,0x00060922a435ef3f,0x000a826a53bc247d,0x000e94d33d8a1866,0x000de75ac695cced,0x000bcb2e7b2b9c71,0x00016e1d52b9aa78,0x000540f2da2b1a83,0x000881db4e2a0769,0x000e217e4c0ddd49,0x0000000000000001}}, + {{0x0008396a04a1a9b5,0x00071e1d4dea3fc9,0x000f1b1b60428524,0x000875279e270a42,0x00031e46c1327bff,0x000c05c823fe0222,0x0001988e4c1b07ef,0x000346e86dbfa458,0x000ea14d7c3803e0,0x0002698c2f4163f3,0x0000000000000001}, {0x0004df73f1c64b3b,0x000bae3f77cba047,0x0009fac52f482f0e,0x00046303eabe2a5c,0x000605a86c7774d0,0x0006157561d8716f,0x0006dae76cfe4cf1,0x0006f10528e0564b,0x0008d8ad69113bb2,0x000c2f933ccc8b87,0x0000000000000000}}, + {{0x000963cd234bdeab,0x0005b961fb152043,0x000192d80595fa4f,0x00021e095276439a,0x000ab3cdd8200acb,0x000b977cb97a7564,0x000f1ac5e95df534,0x000082ed91f8da63,0x0005eeb0ff149253,0x000e11676ac3e35f,0x0000000000000001}, {0x0001cccbb8a0782c,0x0008ef764987a4d5,0x000621e7649cd3df,0x0006ca705ffe9912,0x000d887d63f6769b,0x000a2f7ad40e5963,0x00074e3b9fc2e426,0x000bd597aa3dacd3,0x0000fea916df09cd,0x000c4db4a1b5ffeb,0x0000000000000000}}, + {{0x0006312ffff6fab5,0x0006b5b394c36d7c,0x000ae9f8123d8c52,0x0007a4616b7fb3e1,0x000a92d9f22f9728,0x00095d4a0fd21b31,0x0002d23d7cbfded8,0x000b5c10574881ff,0x000e2bd04e830bd0,0x000fd69dfeb7774f,0x0000000000000000}, {0x000b243fcf68f023,0x00066b7e7441cd83,0x0005c91b009a23e1,0x000f85c785f70865,0x0002122e7768c122,0x000fb751856db403,0x0001836d6df94b82,0x000098df3edc80b3,0x000298e9aeea7ce8,0x0006e6048ecb9605,0x0000000000000001}}, + {{0x000cad0b80b053c4,0x000883158e365644,0x0001acd6f66175a4,0x00078eba00f9062c,0x0000f302446bdc8b,0x0004ffcc5c874a3b,0x0001cc9e542baabd,0x000ca8d66f56fe73,0x000fec6e656e27fc,0x0000e12576ad29e3,0x0000000000000000}, {0x000f2114ad6a5008,0x000ff37b5e9ff077,0x000fe609c9e85ddc,0x000612f68cb97937,0x00035cc97418d8cb,0x000e6da8506bf5e6,0x0001561e4122e23c,0x00026ba1b08bd010,0x0007eb4a708cbd94,0x0000c9d309deaf41,0x0000000000000001}}, + {{0x0001bba765fa9bcf,0x0003babdc21b3ff6,0x00025b55117dbb2e,0x000cd85081bffb4d,0x00081cb8cbfa61b2,0x000244b920db891a,0x000db06e819df932,0x000532ca257bcc77,0x000ade96bb81f698,0x000552846b3d2e5e,0x0000000000000000}, {0x000cc4965acbbc52,0x00019fc3317e41d4,0x000a394d289bf4a3,0x000dfcc926b6b451,0x00099fdc4795a3bb,0x000fbdeb87e6005a,0x000e9db4adbdc0b3,0x00050e680b14c0a6,0x000a1f2811642efe,0x000396ef97cb540f,0x0000000000000000}}, + {{0x0005f3ef0a7bd1f0,0x000acae8e898fb4a,0x000ba953fee10eac,0x000ac34b13e74541,0x000d80a4f317458b,0x000699b6bc22b700,0x0009d659b3752d47,0x000972beec5644da,0x00015e14a99a228f,0x0008731fa64678ce,0x0000000000000000}, {0x00076f9ee24a5ff4,0x0001761a3d49abfd,0x000099dc50257cde,0x000182905bef244d,0x000f586d9a90e6d1,0x0007ebb4cb4857b6,0x00070d79d0a32ad0,0x0004bfe6c4dca5d6,0x0009e03cd5b7b143,0x00028c04a0538f51,0x0000000000000001}}, + {{0x000c6fc14a74584c,0x000402292021e48e,0x0009500ecd0a9680,0x00002339ed719b16,0x000ebb81e8a19412,0x00040e8e4db85440,0x0002a313f6a53c2d,0x00082796c5c684a1,0x000636765497c008,0x00020c751837b791,0x0000000000000000}, {0x000740897b89a933,0x000f39feb6c7cfd4,0x0006675504305fd0,0x000708d724da0165,0x0003fcde5846c915,0x000cbcc847c7bb1c,0x00035875d5c58a40,0x000f531dd999d009,0x000ff3f98178ab52,0x000a7c4485d31888,0x0000000000000000}}, +}, +{/* digit=3 [{1,2,3,..,}]*([2^12]*G) */ + {{0x000bc092fe2354e8,0x000850a2f27fd64d,0x0002ad51407fff03,0x000808402ffc14aa,0x0007fbe516b67c4b,0x000f027098449910,0x0009af3715688b40,0x000dbddce7795e2c,0x0008a5dc626ec8f7,0x00032acc9e1305cc,0x0000000000000001}, {0x00014a5956e30ed0,0x000921ce664e13cd,0x000b7485d5a678ff,0x0001d65fed6fe685,0x000152b7d0453dd6,0x0001e48dc7a066d9,0x00012560c3395f08,0x000d6053e587c1cb,0x00076afca630f2cd,0x000814f0d70553c7,0x0000000000000001}}, + {{0x000a19c3686b1f78,0x0009545540d3da61,0x000fed5fc9dde90b,0x0009be8908cbe546,0x0002b931292ec657,0x000e0b221531c8bf,0x0003dcf64709233d,0x0006a91e2913f0e3,0x0003880d89929920,0x000d07ab37b02493,0x0000000000000000}, {0x000b1d587081c0df,0x00062b5f29f3ee6e,0x00013755e246f468,0x000c2e51e2652ae3,0x00046ba6a65e2952,0x000fd1b792013b94,0x00049175e7bffb43,0x000166af7dd896a1,0x0003d0d5f68a4101,0x0003a34ff29cf955,0x0000000000000000}}, + {{0x000b9e5f2e5de279,0x0006be9e4c557ee8,0x000c510883da6331,0x00016eedfba63955,0x000b55eba66cb586,0x000d93dd071f901b,0x0000d11e4c33f467,0x00029c2288bdd752,0x000f22d4f3c9b728,0x000416f979cce9a3,0x0000000000000000}, {0x000f91fa66b3408c,0x0009041780ab3969,0x0001e17f9e99f2b3,0x0002825a0408a22e,0x00013e814b39af10,0x00017c70c14077db,0x000fd91116e8d047,0x00025157bba11642,0x0003d53fd072760c,0x000130f596860d22,0x0000000000000001}}, + {{0x000ddddf5d1e5c64,0x000b39631d236577,0x0005fc5e812c4b6d,0x000ec807d1cccab0,0x0002c8729f1a1c38,0x000999e4061629e9,0x00088f56b4c00d1c,0x000c3cac8f29781d,0x000b02141cce3380,0x000920c7e0e0cc16,0x0000000000000001}, {0x000234580d88382b,0x000b0ad02da7d076,0x000cc82cf5ae2d27,0x0008a15c3adad7f2,0x0004d7009305d2c0,0x000e9e632a55fa7b,0x00011560b55b693d,0x0008b565732e2a82,0x000f0adb63788cf9,0x00038e2d1f605489,0x0000000000000001}}, + {{0x00007e33611831c8,0x00062ce163c826e6,0x0001c1873d3e07b5,0x000df7813a79a532,0x0007451a6bebf65d,0x000789f014abc3f1,0x000cc2aa01512853,0x000bc25e7cee2985,0x000dd32f61a13543,0x000f9b061a57a2a5,0x0000000000000000}, {0x0003734e520ae3c4,0x00041544fe045295,0x000321b2f9da98d0,0x0004ac066f5c4118,0x0004f9ed8d4d338c,0x000c2ea3577c3f4a,0x000a775eee973782,0x000efa3a3176188c,0x000aa0ec53f7823f,0x0009b227eec20996,0x0000000000000000}}, + {{0x000cab486125bb6b,0x000ce02028378ff0,0x0005e625f7d1c380,0x000dc7cec93bc7c0,0x000d0c3b6e1a5a67,0x0001da033baa95ac,0x000e0b126ba3c688,0x0005b900f71e6919,0x000cfb69d4bcab68,0x00090d9f859cecf3,0x0000000000000001}, {0x00030307c7f8906c,0x0006bf1f14afae02,0x00001d0c8e70d7c3,0x00031fcdd396a945,0x00035972de152221,0x0001149c59aa6c9c,0x000e743c53f3251e,0x00072a17186bca64,0x000bae52281f6178,0x000c98982b2d35fe,0x0000000000000001}}, + {{0x000abaada33bf5eb,0x000334cbe9475074,0x000503921efaf87b,0x0001a3119b05ca55,0x000a4fa919940136,0x000a8bd8f158f3c3,0x0002e855587cab4c,0x000fd133003309bc,0x000a468f055eab49,0x0008de65f435935f,0x0000000000000001}, {0x0005a479ed62ca37,0x00046fb6c0237009,0x000c6558be89daa3,0x000e86cd9c606b6e,0x000dcb5426f2b48f,0x0002744bfa702fd9,0x000e9ceab7372571,0x000cd8bef4768a91,0x000d361ea2d4f5d1,0x0000ca847b5f94d5,0x0000000000000001}}, + {{0x000c6a7b65b21bde,0x000fc723a29c73b0,0x000392643c39c3ea,0x0000b213f81be3c4,0x000e3ec734fa388c,0x000b26d37a33b98a,0x000332e230742689,0x000e28354ec1687a,0x0000d4b7e6935b64,0x000ba79d55aecff6,0x0000000000000000}, {0x000073362910afa1,0x0007fb8bcd336bd6,0x000b6c7a7845b5f6,0x000017305633e845,0x00076a907be72df6,0x000e65734d2814a5,0x000f113c7084b86f,0x000cd7bad9f20758,0x000f6af2a5030c22,0x0001647ff1cabc3e,0x0000000000000001}}, +}, +{/* digit=4 [{1,2,3,..,}]*([2^16]*G) */ + {{0x00084cce9eb37269,0x000406ac65525f61,0x000c9acc4f25051a,0x0007bdd2651c4a44,0x00059571fa6bdb63,0x000cf1489d2ae9ce,0x000a821f56bdf324,0x0000e5fa827f61b0,0x00046a2449dcea62,0x000c947027c9ed4b,0x0000000000000001}, {0x00095f1c50d4d450,0x0002c227a410cd04,0x000bc9ba135ee643,0x0004257073536858,0x0000b7e39c350531,0x00016eeb65d0616e,0x000e949a694a0693,0x00069aba0dc455bb,0x000d36d721f9d7b7,0x000a041dcb7a1d32,0x0000000000000000}}, + {{0x00030d330fc15e51,0x0006a88312448f9b,0x00027c12fd1499ca,0x000b765eaf5a132e,0x0008d01b2d2a5c3f,0x000e3517c807951a,0x000936a97c68ed6c,0x000f8cdd161ce67d,0x0005d9876ad5eb28,0x0009976496ac4a79,0x0000000000000001}, {0x000d912524de7c0e,0x0001e66e4dff627f,0x000a96a9194e4460,0x000ccae884a673b1,0x0005d06054966f81,0x00032269452eba8c,0x000ba7677e70b535,0x000298891e5c17de,0x000f9a70e2fe55a9,0x0004d48b39032dcc,0x0000000000000001}}, + {{0x0009dc9a72f4ff50,0x000df54e86b3f74b,0x000b7fc672cd56a4,0x000ac313c91daa4c,0x00053d8b04fac047,0x000047ffb771df8b,0x000a8ad48cf7c44d,0x0008bf663542e196,0x000aa68b0ea4fed6,0x000483dbd49e0b45,0x0000000000000001}, {0x0007d603e389e5cb,0x000ee233664de2d7,0x000994f96855ef7d,0x000c5bf8c8ab10b1,0x000c2f5ab3d235e3,0x000bff37afff2ae5,0x00050de9d0fd0f4d,0x000ca6d91d5250db,0x00042da0be2c950f,0x0001c70ec3836fa7,0x0000000000000000}}, + {{0x0009b222a53c1578,0x000733b1bb114a22,0x000887f6c13ff59f,0x000d5dfb2679cded,0x0001fd35dec8bbba,0x0000930770ea94d4,0x0007da8d4f0a6019,0x000d2142901c2ad0,0x0002aaa8648f142e,0x000f42252e455969,0x0000000000000001}, {0x0004f335e4753950,0x00010578c42f0d9b,0x000fda89975c2716,0x000761372c49b195,0x000583ac76051357,0x000cd0c4d54de0d0,0x000c35f47ffa549f,0x000731f21817e11b,0x000ac2b103f57a56,0x000284cde0cd7146,0x0000000000000001}}, + {{0x000bb2a3bcba0504,0x00052359d22ba169,0x000ee4d727c18bc1,0x000338aababfd9ca,0x000cae35505124c8,0x000599b6e8a9cc3c,0x0003c6415386807e,0x00043919da2fc5ab,0x0001a4c6fd2ee43d,0x0007be38ead93480,0x0000000000000000}, {0x0008c66b564a97d4,0x0002177834d44e8b,0x000690ef30774807,0x0007151d926feb1c,0x0003fbe2f1f3454c,0x00048ce8e6456bd0,0x000270c04a6964dc,0x000fe8febbc7afec,0x0000f159a483b3a5,0x000aca96cb139ad3,0x0000000000000001}}, + {{0x000c0e9763d8a013,0x000baa65d7b1d37f,0x000608a4b87c8c06,0x0008c2592e527b8c,0x000aacc19bb3aa2d,0x000ce5b0adb09308,0x000e0f42458761d4,0x0002d73d4f707a6e,0x0003867f8d791c44,0x000c943ba7a1a60d,0x0000000000000001}, {0x0007ffca3e51b076,0x000d23467af3d90e,0x0009427b9fa60c44,0x00054ce0e4a16358,0x0001655e4129aaff,0x000befd5ea275c28,0x0000ce27c03c7fcc,0x0000c97ca421b716,0x00069ee6f84bb35f,0x0007ec35e0436eea,0x0000000000000000}}, + {{0x00033b8c99430421,0x000e3aa65723117d,0x0001482e2ca3fcee,0x0006dfdb52560262,0x00036a105a9eb6d9,0x000c0fd8b7bdc41e,0x000c58ba2f2edd58,0x000c050043d8b271,0x0009966a34a51907,0x000aee0fa52e13a7,0x0000000000000000}, {0x000c2d7066d5fc91,0x0000d462accbe2da,0x0008397028d0b78e,0x000b525e2c9d107f,0x0003dfedd5666711,0x00083957250c9620,0x0006d0f2be094638,0x00026dd96c8ff985,0x00098fe829c7a670,0x000dacfc430b6d43,0x0000000000000000}}, + {{0x000aba8bae3f0048,0x0006fc5f421abd9f,0x00094ac402ce8227,0x0005bead91f2efc8,0x000d28241f32e7d5,0x0008bce170cc1090,0x00050cb19f59df3e,0x00034ac35c2de273,0x0003cf90c3e6cfc4,0x0002a784bc2847d1,0x0000000000000001}, {0x0003f87f754f1aa3,0x0003382713cbe9fd,0x00034163c334fd8d,0x0004cbe346cada61,0x000dd6aa94a54721,0x0007b9235830a042,0x000500be120acf2f,0x000d30c3e8d009be,0x000225e2751dc7f0,0x000714b7edd06e6f,0x0000000000000000}}, +}, +{/* digit=5 [{1,2,3,..,}]*([2^20]*G) */ + {{0x000c1256fe47d13c,0x0007012fd1181020,0x000c4a469315aa78,0x000b1163ea26a86c,0x000e4be00b905056,0x0002f1dad4a0ac68,0x000e2d8c19c57695,0x0000bbc11daec6fd,0x0003baf9c6293f81,0x000d375056fba03a,0x0000000000000000}, {0x00073f08bbfc9af7,0x0006c14cc716b559,0x000b5b613b18efce,0x000f005d64d3ad94,0x00034ba83b800248,0x0009ee4cf2a375eb,0x0007d29413af2a4c,0x000525ea872268a2,0x000c082bd8d12fde,0x0006fa2d233189c9,0x0000000000000000}}, + {{0x000f1bef2c1b123b,0x000c9ca73fb5cd85,0x0001a80d76a111a8,0x00025f888d3b7461,0x0003f7765b87f2e3,0x0002e36012c8ad9e,0x0009dc42c7cf6c49,0x000d7d5db366bf5a,0x0005359f96228a81,0x000772725123cd91,0x0000000000000001}, {0x0006c7a0e2cfcba5,0x00097fa38cc5da8b,0x000b43bb38eee14f,0x000f15c0770c4afd,0x00093138850f3aa0,0x000658cf7e3953b9,0x0007c8bb70f07792,0x0000d78fd38c1d44,0x00023ebe4681177a,0x000c9d704ca7518e,0x0000000000000001}}, + {{0x0008fa7e4527a9d3,0x0004db4e9fda74ba,0x000404855f433494,0x000f130f65201753,0x000d719a9846d31d,0x000c651ab9661cb9,0x000b653c04c2995b,0x00031b2c3fb591c2,0x0000b91d21b65fb3,0x000c1f9233b624c9,0x0000000000000001}, {0x000eac108061f9eb,0x000a2e86d9cc5afc,0x0000e2ec8f94cdd0,0x00040675309b7d38,0x000320d2223f6f2c,0x0003be480dc1e34e,0x0007b72b364f62ba,0x00063595753dec52,0x000283d90f6639f0,0x0001d567ed0c354e,0x0000000000000001}}, + {{0x00067e2e3d478324,0x000b9d2b33e93756,0x0005cc9c7d0711cf,0x000aa7c2edf0adb9,0x000b6610b704fc5a,0x000107368e770150,0x000cc4ef9af2a471,0x000afe1e566d06e6,0x000a67146814dd0c,0x000fd36c67f6637c,0x0000000000000000}, {0x000b744b33ca6a46,0x000a2ad960d19dec,0x00099ff41dbc0bcf,0x000977ca933b28a6,0x000a7951faf63b97,0x0005168f23ca3752,0x00097d901e0f16b1,0x000105f55f964ea3,0x0003c0d403b374a5,0x000d43e408ed3a81,0x0000000000000001}}, + {{0x0007586c50a55f86,0x00026583c230d093,0x0009c8f1eaf61062,0x0009876910419f67,0x0006e8d67dbad2c6,0x000c3c184d440783,0x000753899fd2f814,0x00037825fefa52a3,0x000f0758545a721e,0x000498a4b823d5ff,0x0000000000000000}, {0x000e376ebd4ed258,0x00004d6a23fbe496,0x000b69ec3505f765,0x0008fe6b545afc26,0x0000e87ed2073fb2,0x0006145047af95f2,0x00053f84d27cd1ba,0x000fa35d865dc4cc,0x0007b711a9ee96b7,0x0008ec430aefdeb0,0x0000000000000000}}, + {{0x000354ba169146af,0x0008c79fdb88cac7,0x0009f85e2efdc64a,0x0001012d7f3a69d0,0x00017d2bed232563,0x0004dfd89cfd4d1f,0x0008288e64d46be0,0x00089f8bf20fd559,0x0001d08641f269d1,0x0009fc333e29ffc1,0x0000000000000001}, {0x000c7dab1b832059,0x000223bbef8e949c,0x0007f10fed75c714,0x000647b0bb61d266,0x000b8e823dbf309c,0x000601c5a1f58db2,0x0009c023a71fa3e4,0x000a0b5cbdd6344f,0x000df6a6577b11f1,0x00027e6eb12db5f8,0x0000000000000001}}, + {{0x000b94e9f2c64c2a,0x000dff9c4cc3460d,0x000339e03c0646b9,0x000ca76c7ae26f18,0x000612ba1712f64d,0x0006950e5f2c8040,0x000569eb5bf0fae1,0x0006185858b613d1,0x00080b1245b35ba8,0x000d9a3c93740668,0x0000000000000000}, {0x0005f1c44e1c9646,0x00061096f83044ee,0x000e69176fe10924,0x000a78875cfb2614,0x00007825516a8324,0x000065d69cfbad90,0x0001f873d71727bc,0x000185c81c530662,0x000c1471ae21856f,0x00047e68582e4e3a,0x0000000000000001}}, + {{0x000ce1c67b68f07c,0x000dfa5469124c9a,0x000ccd6db2024f3d,0x0002fc6faadd52b4,0x0005124af076574a,0x000510591517b271,0x0000081118a106bb,0x0007cffda2d67e24,0x000b3b39fc6925ec,0x00012037b374e288,0x0000000000000000}, {0x000d91b81541ec51,0x000c413a683e17ef,0x000952a60eda72b7,0x000b61d80495c130,0x000f6bf06a5749c4,0x000c7cbd39872c4b,0x0001a82e01cb7ce0,0x0001726d7547989f,0x000742de944906b4,0x00009f2e02ff3752,0x0000000000000001}}, +}, +{/* digit=6 [{1,2,3,..,}]*([2^24]*G) */ + {{0x0000872f1cba7983,0x000a9c28a369cd44,0x00007ce63b6f5daf,0x0009c12dc12c54cd,0x000a00a67eec84f7,0x000bc7a4edee7a34,0x000ea82ff4e7f63d,0x000950ab492940f7,0x000953027dc3353f,0x000be21b37a3c6cd,0x0000000000000001}, {0x0005f758ab77d6e7,0x000bab416ce18f4c,0x000b9e45e70a1adf,0x00038074a53a6ae6,0x0005919bbc5eaf8e,0x0006580a40639d33,0x00033f83c3446f86,0x0009f20b5de7abe3,0x000a42c48703c063,0x000ed659dbbfadd4,0x0000000000000000}}, + {{0x00025b49cc7d0744,0x000d5ca66a4107d0,0x00009a278704a0b9,0x00027e26383562d7,0x000c28bfb3e1fb8b,0x0006132452cd156b,0x000d9a249ca82ac0,0x0005a22d3a0e92d3,0x000b19aafd34655b,0x00048d36ff20dea2,0x0000000000000000}, {0x000f6b3025835a5b,0x0008531143727c86,0x000c5c003858192a,0x0001da9352b2fd4b,0x000d25e79d62bdaf,0x0003c74903de7c27,0x000ad4718fc47302,0x00063e48286ed6b3,0x0005e2dbee1fb246,0x000857558feb16de,0x0000000000000001}}, + {{0x0009f56e96a543bc,0x000d31c6bb399ae7,0x00031b99e11a37bd,0x0006c0e42fdb7dca,0x00074aad0723ad0f,0x000aae97e4f8e5ac,0x0000fdee33ad0efe,0x000675078a78e14a,0x000909b38b03996b,0x0007469ca609f2ac,0x0000000000000000}, {0x000466dcc50132b4,0x000eddd95df5a1fb,0x000a8d9a82cb91aa,0x000178685d056d78,0x000f44580cbb5024,0x000b8404125b7ea5,0x000c959397a21279,0x000b3c397b77ec41,0x00050358651ee34e,0x000febe48525c5d2,0x0000000000000001}}, + {{0x0003aa997f7f37af,0x000bd0399f83a0f3,0x000f152344eac6ee,0x0003cc5d1e36ba58,0x00058bcb7b01bb23,0x000e6e01c3b95ccb,0x0003f3cbd70b3470,0x0005471f142928b7,0x000f33fe8192a8fa,0x00067751b82a4272,0x0000000000000000}, {0x000ea1f801a1be23,0x0000e029d67e2091,0x000355df0fe9219d,0x000069ec86128187,0x000e9196ceb57b88,0x000b831fc0c5d6f3,0x000b18561a90e72c,0x0006875cf51476cc,0x0009d0fdc775daca,0x000556f11fb0d65e,0x0000000000000000}}, + {{0x000d755a99415514,0x00085952e4dc0b72,0x000ddaa98f7198b9,0x0002cf0597f96150,0x000e7060ed10994b,0x0005df6128c96445,0x000ea5cf35637a95,0x000b47cf25cfbeee,0x0003e924da7f8ade,0x000d5d59754f0973,0x0000000000000000}, {0x0007794536974a33,0x000bee39625b4887,0x000c916eff917626,0x000c143d7b2fe99a,0x0002b42a1f214f15,0x000a92c52323d0ac,0x000fb97a09db1674,0x00097d2bc99da798,0x000062e5a6c7b18b,0x000498e32a3a4e9d,0x0000000000000000}}, + {{0x0006f0528c6a99de,0x0005f78365a5404f,0x000a6130da7d607f,0x000ce8f385d457a1,0x000127a71db8c282,0x00075b1780f22e8e,0x0005f271ed981d16,0x0000879b71615445,0x0003f2395d4c8d2d,0x00002ffbb885eae5,0x0000000000000001}, {0x0004e8a699830814,0x000f60c1a29f03dd,0x000620f1843d15d6,0x0005370351aa6497,0x0003f3bdc3d85b92,0x0005713630b15aa8,0x000b25038bafd76d,0x000aa324c45046f5,0x0008ff854c7ef0e3,0x000f873074cd9a42,0x0000000000000001}}, + {{0x000a5c1655f0c965,0x000ad8760b75d0c0,0x0002d3b7a7dc8ea4,0x000d9d864941a528,0x000eb47e8ca867c6,0x0001a8847066dfb5,0x000249722cfbbbd6,0x000d270c27ad4dec,0x000e74307c896550,0x000812fcfddc8f52,0x0000000000000001}, {0x0009df50d9240b18,0x000a5431583e94ff,0x00044e5e649470bb,0x000156c0cde5a76d,0x000ad9f50c93b508,0x00099e86567b7ad7,0x000e7618f9dede32,0x000b84978d6f1f46,0x000aba70858d56a1,0x000626f5aa274be1,0x0000000000000001}}, + {{0x00038150eda6a189,0x000d5498edde21db,0x00010e8c4a802c3e,0x000d570892441512,0x000a6f9b009f95d3,0x000c80e954785d69,0x0008dbfa197859ea,0x000f9a954ad6166d,0x0005360545cac81c,0x00042d01aaa83bd5,0x0000000000000001}, {0x0008a0342c362ba4,0x0008202cf6246957,0x0008c4987d0b8dd8,0x000becfec2b48e0f,0x000a0d5c621a63b1,0x000668f2b6a90343,0x0008bd4fe098700f,0x0008bd8fa8829c08,0x0007948513f0936a,0x000733ae92e72067,0x0000000000000000}}, +}, +{/* digit=7 [{1,2,3,..,}]*([2^28]*G) */ + {{0x000f8517a4c96e5e,0x000b0e039e54c124,0x0001ec83d522549e,0x0008ead8eb02627a,0x000ef95771e9618a,0x0004f3a6b4497c2c,0x00055a41c4917c7d,0x000be8babd80080b,0x000548c7990585b7,0x000c64dabe54a06c,0x0000000000000001}, {0x000269bac978d382,0x000c6bdf9b98b02b,0x0000247e075c4ed8,0x00030716d8e1a299,0x0003a019c94edbc8,0x000c1665fb3458f3,0x000e700cd4d82fa9,0x000a2326e507ff63,0x000452b8341c9bbf,0x0000dfc9214a3f5b,0x0000000000000001}}, + {{0x0001109faf34910d,0x00006e16fb098733,0x000f23db2247dc5b,0x000f53ef62c76a0d,0x00018f68431633f2,0x0003cbe6a9413972,0x000253b6d05ffc26,0x00047ed103a8b14a,0x00089d6938ee0b45,0x000f4f3310d7c6f3,0x0000000000000001}, {0x000ae5ea5e61f0fe,0x000803eba63d9ee3,0x000ff5d7d44281cc,0x0006da525a289454,0x000688b5bf3e7fa2,0x0006c91ab88b9a08,0x0009907565a2877f,0x0001737c021210a3,0x00031db551223d6a,0x00041419f277aae1,0x0000000000000000}}, + {{0x000ab41a098c92a7,0x00072fa6811333e6,0x000851f03fd8fe3b,0x0008e34493163c43,0x00028174f0009cb6,0x000db83d4e177695,0x0004d1bfc734499f,0x000c56272647cc55,0x000d38b2b770d722,0x000fcea5c023f6ae,0x0000000000000000}, {0x000d52df57e5ca52,0x00065ec98ce0bb77,0x0002140495f37162,0x000d03a5ffc6d632,0x000f0d0061a4f743,0x000af61ada3e0094,0x0004b70091a92992,0x0003fe83591d84a0,0x0008b7e8b2e59f37,0x0006892e459b7561,0x0000000000000001}}, + {{0x0005e28d077485e2,0x00020e97a1a0b220,0x0006679b5d8662c1,0x0000da8bd256806d,0x000925177882d56f,0x000fd447d9b3a0c2,0x000077e15eb4fc81,0x0000f15d98b7d44b,0x00021ac9944ac5bb,0x00096438b1a53927,0x0000000000000001}, {0x00015ec8db3161c2,0x000f397c19309926,0x000de0771a46014e,0x00075e027929bb3e,0x0009dcd3fad002d9,0x0008c634c3d387bd,0x0009c3f4ebf24793,0x000a2c2da2b50807,0x000e85151e2b2209,0x0005f4bb80ad2ed2,0x0000000000000000}}, + {{0x000d84a6577d1d88,0x000209b195d9875f,0x0007dc25315ab09e,0x00041e59650e28f1,0x000f81b07fd77f04,0x000191d3288a8ccb,0x0001838964e3273f,0x00014aa2bd5786f8,0x000d924c29c83190,0x000d240eca2dec17,0x0000000000000001}, {0x000c94cf500620a8,0x00017a01ff5bfc59,0x0001d4396ff8b7cb,0x0004fb6ee69226c5,0x000565facfed55e4,0x000fa0b30de43edc,0x000b20ea8612b643,0x00056d93e66dab8d,0x00098f9a9e46bfc5,0x000537cf26d7b09b,0x0000000000000001}}, + {{0x000ab3bbacf312ed,0x00003114377687ca,0x000ecbe360e1fdea,0x0000336e644349d4,0x000ce9701c97498e,0x000b970f76e36640,0x000d18aeb034f110,0x000d8c04737b96e3,0x000b828ff89d3a4a,0x000ceb091b279c08,0x0000000000000000}, {0x00002832632c0b79,0x000f06083c30b6b6,0x000d6d46a5784fec,0x00018a1a822c2ef2,0x000ffd16846c810e,0x000859c7f32abfb3,0x000410206687e5a6,0x0001a4731948bcab,0x0009310e425eab64,0x0006640ebb114d39,0x0000000000000000}}, + {{0x000759039c7a9d17,0x000c2d4dd47b8758,0x000c1f2cd883e5fd,0x0007f9cf12cfd22e,0x000b330c8103c8b6,0x000fef902907b0c0,0x0008540db26c476e,0x000e726a64472792,0x000b090ef761ac1d,0x000f96107e57dd18,0x0000000000000001}, {0x0004650aeff40f19,0x000f9022a822976b,0x000a5007e542efc8,0x0007e6336d263df3,0x000d3e6b563a86e0,0x000fc14e677c3dd9,0x0001920cf4b81cd1,0x0001cedaf8056af9,0x0006ebfcbc28b238,0x000af30ce3975f89,0x0000000000000001}}, + {{0x00090c95c3d0b9c7,0x000844b8f686b48f,0x000a746779417d91,0x000bd2f6b498e1e2,0x0002adf2e61e9354,0x0000db08d1aea1ed,0x000a497f347f08db,0x000c7871378402cb,0x000753c735de0850,0x0001f9ed5b707951,0x0000000000000000}, {0x000d045e45f819b7,0x00024591e83186d4,0x0002f07b1add4180,0x00067074059bab68,0x000ea2b0f371c1d4,0x0000e5f219b0e391,0x000f844f22e68c28,0x000679b7f960b058,0x0002502036ca9899,0x00028a1f63acdbd1,0x0000000000000000}}, +}, +{/* digit=8 [{1,2,3,..,}]*([2^32]*G) */ + {{0x00023b6569acb75b,0x0008bff580463d22,0x000bd3d896f7a55f,0x000bc35819d6f3c8,0x000fcf8dfdd32dc1,0x0007477a57a2786c,0x000a3b0ae5589b65,0x000806de5eae9f47,0x000782fcd93f2178,0x00096dd9479ee32c,0x0000000000000001}, {0x00029435cbeebc5d,0x000d8352a35de209,0x0004ce809af5b11d,0x000a76cf080ab8c3,0x0002250478b1ce25,0x00003b3ff7216bfb,0x0004133d87f47621,0x000a4132748099b3,0x00029b68995f627a,0x0006fb62b3cf30a7,0x0000000000000000}}, + {{0x0009267d43d108b4,0x000b5b0396f4d947,0x000b1e5f7f088692,0x0009c3c1fd9437e3,0x0006cb9b9b7fa950,0x000a37b6c115a75e,0x00062d05b42f650b,0x000d5b3cf510dbbc,0x000a49bf06d5fe62,0x000049824e6593c6,0x0000000000000000}, {0x0007f70f36b1fb53,0x0004a3eb94fc53cc,0x00082c0e60f6a2b3,0x0002888aea4bc9d4,0x000d37ab9ed318f4,0x000a505ae0c0fe3e,0x00082c9e94fad3b1,0x000efa0f31285453,0x000bd4be69ae4fb8,0x000003a516fe8844,0x0000000000000001}}, + {{0x0005e8088591d8c2,0x000c6e24bb4a7ab3,0x000968fd3709e47e,0x000d55e3e7d95582,0x000f8783f2538474,0x0003b454e19d35c0,0x000724c4578fbe58,0x000b98f51326114b,0x00090d99b97ee546,0x000691801229b8f1,0x0000000000000000}, {0x000e4a0a2aa86765,0x000aa08b19b4b67e,0x000c220cb11d4f5f,0x000c7d426ad0862b,0x0002dd7a63508ce1,0x000aae47773236a7,0x000cce936cda4ace,0x0000808ba7a0be3f,0x000c14a5b972c384,0x000887cda4655c79,0x0000000000000001}}, + {{0x0007786aa39096d4,0x000a4070017bccec,0x000926b1ae026576,0x000026855e810d17,0x00039def24d306c0,0x0008ac55c7d826c4,0x000b521e40f66cc7,0x000ae26697c9fbf9,0x000a75caf01aaaf8,0x0002a9d094aec346,0x0000000000000000}, {0x000a161f4c1b67b7,0x000e72cc3c7e5f99,0x000fefa749b88977,0x00063737d6e24cf4,0x000ed4607a910568,0x00017686854dcfe4,0x00011acac7bc09c0,0x000cc731394d9ec2,0x0009e4dd8ac76909,0x000bc7246fb612d5,0x0000000000000000}}, + {{0x000d5c2ad970ea88,0x0000eb7dbc22b4be,0x00070d534e1a9e33,0x0004b0eab21921c0,0x00071b3a90f4b6c3,0x00015ce8387fd7ac,0x000ec28efad61dad,0x000f792985d577d5,0x0009ac313d175e05,0x0002d86582a5f995,0x0000000000000000}, {0x000258f797f0af53,0x0006198f53a67607,0x0008e97661c802c3,0x000001f410140442,0x0003c270534a9c94,0x000a5333d3d840fe,0x00076ed16f99ddeb,0x00021b94ff80e532,0x0002761460189f70,0x00007bce48c909c6,0x0000000000000001}}, + {{0x00018e1640843ac0,0x000d435c11720f14,0x00057cebd3881c75,0x00023104876137be,0x000da25ae87bc97f,0x000052fc640f68c5,0x00027372b9f14882,0x000b4e854fcb3017,0x0009d3406b0f1a39,0x000e653db1df6886,0x0000000000000001}, {0x000c416fe50a1e7a,0x000c29b7a99148a5,0x000d660bcfe2e7c6,0x000630309213716a,0x0004b63e38c0d1fe,0x00033bb86d3923a4,0x00056213e592fda9,0x00080b360f884315,0x000db3b21dcd9fe7,0x000617bc0a7836eb,0x0000000000000000}}, + {{0x00035cd63c7c9de3,0x000a91ffca828c02,0x000b3e377bfdd504,0x000496682e625999,0x000b8af1c462d519,0x0009570d96676ec9,0x000bf815f3869cb1,0x000b476a8b4fba76,0x00057597ce654eda,0x000e2235375a2127,0x0000000000000001}, {0x000e9252adfeeb9b,0x000707b233b4a650,0x000ab158faa771d2,0x000df78fbc7a8536,0x000be88e5009a864,0x000abbc52635e311,0x0008769bd53cfa60,0x000305e36a566f93,0x0000e758e66271c0,0x000b10e12e9cdd0e,0x0000000000000000}}, + {{0x00092f38031c9da3,0x000f6f0dd54d0455,0x000298fe24144d05,0x0006274ae2d191b2,0x000909113fc1ead4,0x000dea702ccc54c5,0x00008c5763ea8c10,0x000635441e7274f2,0x00006b2f2578c573,0x000c3b8134369537,0x0000000000000000}, {0x000ada464a85cfcd,0x0006064c2605a588,0x000004bbf31d5464,0x00010805ca04e18d,0x00067998cccf9032,0x0008398a6f896278,0x000dacf8d2faed14,0x0002de01ea7eca85,0x000eb7d829278474,0x000eb89fe5859b0f,0x0000000000000000}}, +}, +{/* digit=9 [{1,2,3,..,}]*([2^36]*G) */ + {{0x000a73aaf5f01aa9,0x0009abe5e08d9505,0x0009fd8bcd318e91,0x000915895b386f06,0x0004a0b5abd07d68,0x00039901b007bc24,0x000ff05c36936913,0x0003179dcef7684c,0x000125196e69b799,0x000a2823db7308b0,0x0000000000000001}, {0x0008e83616d0d063,0x0003bf0c66e267f8,0x0006d375b822f9f9,0x000dc02109656c0d,0x000530c1fc16b6d5,0x0006bf6541ba81db,0x000443cb4caf94b8,0x000a87f59d3c1a6d,0x0002afcd60e10fe1,0x000a088d7e1995b7,0x0000000000000000}}, + {{0x0005b8daa50517e7,0x000dbed46676160a,0x00016e7a7e78dcdb,0x0008e1fc1b0f5e9c,0x000933cb9503e917,0x0006f78b36f2294c,0x0006ec3420442aae,0x000c01e17c7ffb2d,0x0007293b87d30fdd,0x00092836bed4739c,0x0000000000000001}, {0x00075943e2ebfd04,0x0007b784cf37b849,0x000645eaa02ab285,0x0000b0ace12a0d65,0x000ff267c93aa3d3,0x000fa6954230be3e,0x00001e1903aa7ed6,0x000c311ae0c82cdb,0x0005f0bd3c5056d4,0x00022094d54a9827,0x0000000000000001}}, + {{0x0001c0660a4f2bbd,0x000c3d2fe061f455,0x000a2b6dd7091d40,0x000bf7437cc4883d,0x000f258011ddabd2,0x0001bdaba7aee852,0x000195b39f463ce7,0x000efb33b881bd7c,0x0007d695aede085f,0x000380f8fa2471bf,0x0000000000000001}, {0x0006eeab983554b4,0x000f138a6f5a6dc9,0x000b29f154f07f71,0x0007d1c7242d7a9e,0x00042f3c7f1afb70,0x000af2896b44aee8,0x000b55601f569990,0x00061e36ed07c5c5,0x0005430facca2678,0x00016ea78eb21424,0x0000000000000000}}, + {{0x000aeaaabe68e207,0x00098b6eaee3177d,0x000f5c1267a6e353,0x00092ae2d01bbbbb,0x000bf3af1a4f4a2b,0x000532088e5cdfa4,0x000c8ee638a58f55,0x000bbce5e7aa2a0e,0x000a1644879c3f50,0x00025e378387c246,0x0000000000000001}, {0x00069ee8884e88e1,0x00090f8ec2984b67,0x0001f9e47bd62fea,0x000f9867e7522665,0x000cb916cb0493d9,0x000106987f5be2cc,0x000869c71156e1c1,0x0005011576bd7bad,0x000b76a6e945007a,0x0000aec18de41f38,0x0000000000000001}}, + {{0x0007c59949c6f67a,0x000c897a229d2368,0x000118e3e788aa53,0x000794e3080e973b,0x000815a91046f15c,0x000ac04ed530c0f2,0x00078a4fe35eb925,0x0001fe18e7070c54,0x000f409ce7d9c57a,0x0008aafcd17edd4b,0x0000000000000001}, {0x0002850fe8174557,0x00037e234694753d,0x000c528ec7d2c45c,0x000bdd41f346792a,0x00032ca964be2a36,0x000fef3e7911d679,0x0000101c72673a03,0x000653d6358ef0bc,0x000f874e2b9caaba,0x00045ebbcc31882b,0x0000000000000001}}, + {{0x00090baa505384e0,0x000993148d7d8454,0x000d948464cae379,0x000f46feac4f8ace,0x0007268d1b7c8dda,0x0000d4770e6c39b6,0x000c1955967ac571,0x00052098b74830a2,0x0008d17fe1904773,0x000191e5071ced85,0x0000000000000000}, {0x000b4f003b56551f,0x0000872a7e419344,0x0001262c90a767f8,0x00015a9791770fa4,0x0001a4bc5606e253,0x000a665041696dd3,0x000da6d7ec137466,0x000cac07c562bd01,0x000c4a92e59de1e2,0x0007068f7d29994b,0x0000000000000000}}, + {{0x000172698b05a9bf,0x0000570dde3c4b9d,0x0004dad7e2ab8a86,0x000cc4fe376d08bb,0x000df2d396e50709,0x0008138716b38f91,0x0004e9a6a45d9577,0x000feda477817a06,0x0007062a4f79bf83,0x000ca12aff8e0103,0x0000000000000000}, {0x000748577bd0ea3b,0x00068ca23c2d1e49,0x00079fc968bdb9b4,0x000359089d5d5807,0x00037ae7478afb4f,0x000f6ac1a17ffb5a,0x00076fb0ad6d0956,0x0003cc73da4935d0,0x000b38ddd4e896c8,0x00091fcd942654f0,0x0000000000000000}}, + {{0x000d06bf88e947b1,0x000a0ca717a0accc,0x000b428946c6ceb8,0x000992652bd2594e,0x00041806882365c1,0x0004f2f3523a8241,0x0002e27b634a60c7,0x0006f4680f2b83ce,0x0005e159f3342680,0x00053718c76ca148,0x0000000000000001}, {0x00063dec4e0c9dd7,0x000cb7d74bb74e1f,0x0004a5f09446f95c,0x000f4555c75d0662,0x00028d6e7583fd36,0x00083fa30323cc2b,0x00010012c0c49bb7,0x0009907cd3d64f77,0x000c79e69ac90f89,0x0006c12cb352bde2,0x0000000000000001}}, +}, +{/* digit=10 [{1,2,3,..,}]*([2^40]*G) */ + {{0x000bc2a7fe05ab80,0x0001a246f49f7a93,0x00091072db7cc0b4,0x00011f97d30ac7ec,0x000169de685e7537,0x0008a2ae9aef0acc,0x000e3c46bded4c6b,0x000604789cdc4497,0x000a082feb2b2ea2,0x00078fe890cba4fe,0x0000000000000001}, {0x000edbc5c5076824,0x000298a472aca466,0x000a80660505ad15,0x0000c1cc1b16ca97,0x000dda2937bbf38b,0x0002545fac4d56c3,0x00090f07e35494b7,0x0000903e9ca74b57,0x000b43111b0c8bc4,0x0005465923f9e9c4,0x0000000000000000}}, + {{0x0008b52bfb73ff4e,0x000a4d6410d489c1,0x000ae318bb528530,0x0004b71f3f7d7028,0x000970b21b686b53,0x00001ea2751f5995,0x0001322663379ddf,0x000e77c11915800a,0x000261c476db4bea,0x000ae8e89c22a1e3,0x0000000000000000}, {0x000e16965a7b1dfa,0x0009afd6426762fc,0x000532ec13f2e53d,0x000fe4131e801ed5,0x000c86963e2f9e9e,0x000f46e5098aecc3,0x000faa2b47801a35,0x00043b6f3406c3d8,0x000b57971349f37b,0x0007d749ef39f4ad,0x0000000000000000}}, + {{0x000dcaceb126ca58,0x00046c5a73dec3f9,0x0000e0ffb30d8879,0x00026935e7e62831,0x000f5e074c83b841,0x0008b04291158a7d,0x000f72a08eaada21,0x000a40d05438fa20,0x00005e6ac9aa8c4a,0x000b92b755928484,0x0000000000000000}, {0x00081326c74e90da,0x000cdbf1a037a879,0x0009887e29013ab2,0x000ffd6598bd3d86,0x0003b3c803a95fec,0x000e4c50722fd839,0x0004e70129b699c3,0x0000fa72cdd65e3c,0x0000ccbba65f24da,0x0008975325682f6c,0x0000000000000001}}, + {{0x0008d8231d21b98b,0x000f46eede07ff72,0x000c57dc8bc52ba3,0x00096a93bbd28782,0x000609e0a7a0e49d,0x0009fbe4aa495765,0x0004c5f79dbfb8ae,0x0005e1789960cccb,0x000b74da3d25ebfd,0x000c2e56df642b09,0x0000000000000001}, {0x0007057a2be83a30,0x00026ba759ce4dd1,0x00057744ef0ab9d2,0x000c9ee2b7115a63,0x00000d77f24c2ddd,0x000142ea1adfee89,0x0000f3fa9d5346a2,0x0007a84ecd7e6d50,0x00035caeb7a1527e,0x000c8110e6262dae,0x0000000000000001}}, + {{0x000457989aea7c3e,0x000a16bd6196a6cd,0x0006f76c2cbdd85c,0x0005840dcfd847e9,0x000ec8ea001b3aa2,0x000898be24444e27,0x000397d8a0c53dda,0x0003fa5f98a5b3e4,0x0008e197364ea986,0x000bbd922c6bbfbe,0x0000000000000000}, {0x00064b7db5084e37,0x0008c954735a1906,0x0007371d65a99056,0x000bdaf052b4c902,0x0009ec2cc9668600,0x000adac668469729,0x0002f7ceb4949cfb,0x0001eda14ca03327,0x0009270923989fbe,0x0002a80f1714c9d9,0x0000000000000001}}, + {{0x000da9fadd2d2e35,0x000ebe39d9c06e00,0x0007878b1c269b9b,0x00045c0350e16aa4,0x000d604f7fb31a05,0x000233dc435d57a4,0x0004a59629977c5d,0x000caa747e5387e5,0x0006980680cca577,0x000c7f3aa2473480,0x0000000000000000}, {0x0006ecf72ff177c1,0x000ea087d84398f4,0x000f8e3dbad5b5e3,0x000793b7c29bdf29,0x000d48b4ad4a2c86,0x0004cf9d25337e0d,0x000be01c858ea723,0x000fe90a676bb282,0x000306f507590c7b,0x000915155053c47d,0x0000000000000001}}, + {{0x000311b42bb970cd,0x00056fa08e6727fa,0x0006285b2f7609fc,0x000807c307ce1a3e,0x000d8c9c1df6de94,0x00070b979619a317,0x000fdde052a3379d,0x000baa7d20b3870e,0x0006e443d8f7406d,0x000355511beafebb,0x0000000000000000}, {0x0002e82c90634dd6,0x0002249d51e499d6,0x000615ca9d89995a,0x00097ac7d99162d9,0x0009551034000ab8,0x00070ca9d924f35e,0x000b526853be7dfb,0x000638dc8c8aff11,0x00031fb01463b8a6,0x0004e7b55e740433,0x0000000000000000}}, + {{0x0008d4f13a03485a,0x0004c5dd3ccf1850,0x000d8ab1776f2552,0x000a54a5e0bf0c9e,0x000e81226e24a017,0x0000e5b1ecb9626b,0x0006acb8b7b3bc5b,0x000414da0130f24c,0x000b2f1d94673605,0x0003ae73af8b9c7d,0x0000000000000001}, {0x0001a4baf806025a,0x00068cf7b99ef6b1,0x000b70549901e9bb,0x000cc5d4a5ca0071,0x000b555009f7be57,0x0008dbaab6501aba,0x0005c582d17b21b0,0x000d9921c7bacde3,0x00013fb4b9991c48,0x000d72c6f664c93f,0x0000000000000000}}, +}, +{/* digit=11 [{1,2,3,..,}]*([2^44]*G) */ + {{0x00000e09c15f3e5c,0x000f97f45b409fb9,0x000ff4a01087837b,0x000d204b2bcafcbc,0x0000a0da165ec6b8,0x00011716978423a6,0x000f2f7f8295351b,0x00081f58e2d13eb1,0x000ed84592b66922,0x00025f5f9819aebb,0x0000000000000001}, {0x0007ea077d668278,0x000b9653ee2ff77b,0x00098e8334b5b359,0x0005210487baabe1,0x0001a95a5886c85a,0x00009649f4c23788,0x00056127f95c6f68,0x000aed6c6419d339,0x000be49aa657d29f,0x000da67ae0b376a5,0x0000000000000001}}, + {{0x000fb32bacbbde73,0x0004f08a721e3545,0x000584020b45c467,0x000fd3a284a774fc,0x00004477afffeada,0x0006a2c4ec266e10,0x00066dc326c6652d,0x00070b3a65b94280,0x00055b8104c7d5c7,0x000d8c4b6af237e3,0x0000000000000000}, {0x0000b97b7ca1cffd,0x000435de1358221c,0x000876cab38cc7ac,0x00054cdc8f30b09e,0x0000ccb3a4f5aec6,0x0002ac30ca26a9da,0x00011038e2a6fa3b,0x0004545c20a577ee,0x000bff8e2f50fb14,0x00093158359a6d97,0x0000000000000000}}, + {{0x000b8fa7bdc8384d,0x000f2859e636e718,0x000ed301cc9c465c,0x000006a5b105b2b7,0x0007075698827d1b,0x000fbe7e34565b30,0x0006110c56ecfa0a,0x000be3136324ecec,0x000a684fbfe59f06,0x00056bc39bf313cf,0x0000000000000001}, {0x0008543e894e8a16,0x0006d339681c386b,0x0001fc0aac97005d,0x000c6eaa38aa9f79,0x000a90d6089956c5,0x0005ec33374fdc05,0x0007a4eaa1631484,0x0008df64f8ad9e1c,0x0007f44336b984af,0x000859e7ad53b8dd,0x0000000000000000}}, + {{0x000c8ba9ccf89d24,0x000874ddf8d1b9b2,0x00027291ffd7f24e,0x0008bd9d563287c7,0x00065d01bdb48d02,0x0001b99b973b0c12,0x000050d618319b97,0x00038420d531f686,0x000d7c201c411c3a,0x000d97468eb84cae,0x0000000000000000}, {0x000eb2fc05bf609a,0x00073e1dab9da1f6,0x00049847c3ac275b,0x0007880554d322f9,0x000fedd0cd2b7f05,0x00085bb3e74958ea,0x000fcd8d9061a481,0x0006f9ac370d5c6d,0x0004cb188a021786,0x00057aa132c3b5f5,0x0000000000000000}}, + {{0x000a082d83368da6,0x0009c04ba3a92f9d,0x00018e366fb4a189,0x000dca28f8a4aa8f,0x0008c98efdf41b12,0x000e7f6ce11df724,0x0003850179926b22,0x000c0fa0bae8aa29,0x0007fa3acc87191b,0x0008222f18a88f29,0x0000000000000001}, {0x0009dda00370c8d5,0x000720c0497d2a02,0x000cdb5a6b3dc411,0x00032f0daa73c7c9,0x000505d3784e14b1,0x000d7b77b06edb5d,0x000c1f94b45f2c9d,0x0003193f38ea4c8e,0x0001706f6af3bbf0,0x000f0df22ed99a0b,0x0000000000000001}}, + {{0x00097825d9e0b2bb,0x00096d1340276af1,0x000d82fe6324bbcc,0x0000475ecad6233b,0x0009a0cd8d04ac29,0x000e8e067d738cce,0x0004317aa038ad08,0x0009a7ce55aad83e,0x0006a1887d5e91f4,0x000a135efeae9285,0x0000000000000001}, {0x000fa0b6a035b085,0x0005853d153ead9b,0x000ca7f6fb4ef7bc,0x000bfbb30b798e2a,0x0006653595cf1f8a,0x000774e7d1791820,0x00048df862d39281,0x0002db1e40868b45,0x000153b336e38fc5,0x00052ce2e4b80e72,0x0000000000000001}}, + {{0x000306c918831eac,0x000de7b8556b1913,0x000c579621fb2237,0x0001fa1088e011f5,0x0007511558db4265,0x000c63131310e896,0x00029cec356a91fc,0x0004b85a2f85b7d1,0x0006bc4404e3b5cc,0x000ac5c4fcc3a6fc,0x0000000000000001}, {0x000498b1e054105b,0x000818b7c971dcbb,0x0006cf66f8ee2eea,0x000fa05b49db517f,0x00012073a35daeb3,0x000d5bb674f202b9,0x00032d411842792d,0x00076cdcc487632e,0x0006568c96ca8dd3,0x000ecdfe1c986f51,0x0000000000000000}}, + {{0x000595043dc23233,0x0008d9e1b752f3f5,0x000f6e2b3641b931,0x00064e0f5c02bb70,0x000169d8f287038d,0x00062f3a1b075424,0x00001bf3b8c6755a,0x000a2b642127d597,0x000c17f0c20fbe8b,0x0005263410177dfa,0x0000000000000000}, {0x00008cc560c65efb,0x000ef6706807502b,0x0007a1e8980e532c,0x0007963729a4a8b8,0x0007ccb3a4f193b6,0x0002e07ae80043db,0x0005be0346fea839,0x0009ef33f7a00da3,0x0001ea778cb41f4e,0x00096ebb760e7727,0x0000000000000001}}, +}, +{/* digit=12 [{1,2,3,..,}]*([2^48]*G) */ + {{0x0009ba658c1f9460,0x000b18b48e1df3f7,0x0005fc03a103eb15,0x0001ad263bed592a,0x0000a127b78a3359,0x0000337c7b07e9d8,0x000d2a0349dd74fc,0x0003b1a807c5364e,0x000d92cca588d420,0x000d9e772a1716ec,0x0000000000000001}, {0x000fc1df3f66f295,0x00015742d25980f6,0x00036f0fdb08922f,0x0001fe47a583206a,0x00001c73f88168cc,0x0001b777671d2798,0x00068317ac8979ce,0x0002a98b48363dba,0x000f36b7460d4015,0x000da1c3d46c62c7,0x0000000000000001}}, + {{0x0001fb046e7fab2a,0x0000e190213473d6,0x000cbb6e9b84f9db,0x0008fb8a36fcff78,0x000c47cd5e9d16aa,0x000c2601e9337a00,0x000713efe8445d72,0x00030681fd15bbab,0x00051cff90b2dd23,0x00024900ab444b21,0x0000000000000000}, {0x000de9a880ca8289,0x000e3bcb8ede5206,0x000e1369e32209ab,0x00036516b711e224,0x00025533569db531,0x000419656e59d965,0x000ee21f2d680255,0x0001d59bb004326e,0x000bb722c073cca7,0x000866aa784f931c,0x0000000000000001}}, + {{0x000d262bb3ca8da3,0x00045288c7563397,0x000d10c59451797d,0x0004e594575ac7f4,0x000311006db0bd6b,0x0004d6485bec56e6,0x00089e23442953a9,0x000610efbe7ae598,0x000f1a0f166d5639,0x0004e5f91e0ed7d5,0x0000000000000001}, {0x000aca95d51d383f,0x0000d23ae1d67e17,0x0001b867b62738a3,0x000907e6a529f72f,0x00059335e44e6986,0x0001d5fbd59c1e58,0x00018393e6789776,0x000e5bc878acd1b0,0x00059b26328cd9f3,0x000f3d83306eef44,0x0000000000000000}}, + {{0x0009313593b1d240,0x0003b3672b4b0a51,0x00078ea42e614acc,0x000784cd22fe0a9a,0x000a6c20faf43e72,0x000e49f3038f9c3e,0x000fb914c50987c5,0x00000c76e9b912d1,0x000dc2b7b96a89b9,0x0008de238b29a074,0x0000000000000000}, {0x00068ea377031f72,0x000e9606adb168ca,0x000e58dde885ecba,0x000177424d422e92,0x0008aa609937ceba,0x000f30fc81145199,0x000c9f99eba807b8,0x000e200db6e7a724,0x000db2dc7651c126,0x00009cb58e38f0c9,0x0000000000000001}}, + {{0x0002dda36b654cc9,0x0008acdfcec83248,0x000164e7c3c6f06e,0x000ff27341f98b3f,0x000b145595e7208c,0x000f8fbf6e4d7416,0x000a519d6e691f26,0x00082c10e5fd7be7,0x000b1af42c760524,0x000f36238c97dbcb,0x0000000000000000}, {0x000a3b8e5e8a7251,0x0006fabc0a713664,0x0004fe41fe20cb31,0x0004067dc2b00d32,0x0004fa30d7f12a4e,0x0003f79c759905ca,0x000ef9b323da1724,0x000357ea84efa0af,0x00083350488b839e,0x000408f03e5cf22c,0x0000000000000001}}, + {{0x000e616991835801,0x00028f75d4e1a857,0x00031b04bb8869ad,0x000fb7737091410d,0x000cdd4fad2e164a,0x0004c7cf900934ac,0x0009b433f8948c75,0x00028687541974f6,0x0008f0af013f8d2a,0x0000b59eac62d398,0x0000000000000000}, {0x0000c23b25d034e1,0x0007d927e7c611fb,0x0007596a05e54d05,0x000f8995596eb4b3,0x0005b26be4690555,0x000ae1277dbebe99,0x0006bc99c064aa3f,0x0001128fda3fad13,0x000cd244392bf3eb,0x000a481bf8fac406,0x0000000000000001}}, + {{0x000d8f6c1e9aa431,0x000e3d32dbe1679d,0x0006391e1828c205,0x00026f450a2821e3,0x000a00cf3fc9e32a,0x000a2fc5a7a744f6,0x0006b70facb792c6,0x000e8edf92ab0b29,0x000d53faf8f11ba4,0x000b1e87f01ff5b4,0x0000000000000000}, {0x000e6fdf30e29338,0x00014840cefab311,0x000c4e092d034ea0,0x000be36bee8c5b8d,0x0001f39e2788cf85,0x0008f32c467a702e,0x0005e353cc3b7917,0x000f137377576e06,0x0000db55d1c5fcce,0x000d16e9cd523a0e,0x0000000000000001}}, + {{0x000e8870b81dfb8c,0x0006908ea654a671,0x000c3eb3660a23dd,0x000daf70673dbdf6,0x000c19bbf5d38a5d,0x000fe1371d1e7af5,0x000e30bcc1eff610,0x000f1308bdd31572,0x0000db70b20ce33c,0x000e036ab6b3edc6,0x0000000000000000}, {0x000357b86d4f22a6,0x000b893ce6e16bae,0x000a3849b8d94e06,0x000e1675b6058ad8,0x000ed6add0f99ace,0x0003cd380c39df12,0x000e2335c645ff14,0x000994a0f6180481,0x0005a52a4c84b4bf,0x000e9849a710f480,0x0000000000000000}}, +}, +{/* digit=13 [{1,2,3,..,}]*([2^52]*G) */ + {{0x0002ea2996fba378,0x000cf933ae1a32c4,0x000b43b79b9d0dda,0x0002636c30561bd9,0x000f9122413700ad,0x0005a779a0d830de,0x000580fda5f65618,0x00067e785d8628b8,0x000ce8b188bafa8c,0x0006d2c75df63d48,0x0000000000000001}, {0x00090afd2d7e01b7,0x000e57c72b304c5a,0x00040d7dec21b4b2,0x00094cfde0f45d07,0x00010aabbfa713eb,0x0007fa8b4fae1b3f,0x00047d2b080d24d3,0x0001142abdb36f64,0x000470df72045350,0x00047e76e433f8fd,0x0000000000000001}}, + {{0x000081cbd682dd91,0x000d2c1433b543ad,0x00094641d2488d8c,0x00036e702da0394f,0x0008248288ca4d8e,0x000112c8a6461fe7,0x0004a486f063613a,0x000f77efb66bb862,0x0006e8d41511d90f,0x0009a1ce80969401,0x0000000000000000}, {0x000feced91fc3935,0x0002e83ecdac7136,0x000ee8e2857921f4,0x000ef9bbe82b293d,0x000bd182b25ab2c3,0x00097ad819ac32f4,0x000916b74b598de2,0x0004d5e666a5dd15,0x0007be0b151456a2,0x000b794dc25c5c44,0x0000000000000000}}, + {{0x00087c377c23c5aa,0x000d33ca314119f6,0x0002512d0943eacf,0x000f9fd69c0e1850,0x000b7c3c6ea7ee55,0x0006291556c20685,0x00053374868b07c6,0x0009f9f339d7b589,0x000d6855b9238a10,0x000491ac6af37f75,0x0000000000000000}, {0x000b5d5b2f49812c,0x000855e7603bff6e,0x0008f73b087f7552,0x0005c0adc19b7320,0x000d355df5442e8f,0x000a4b8876b6aeab,0x000a7378dc2b22b3,0x000c26f89265f8bc,0x0006645f23dbb040,0x000f38b09ab1bbfb,0x0000000000000001}}, + {{0x00011ca0d8fcc245,0x0008ea92267e1494,0x000bbfcc2abb385c,0x00029656bfd56d29,0x000075f2180a734b,0x000dc3400006f728,0x000f754023104376,0x00021bae73e6854a,0x000a8d2ddbc75324,0x0003d711770a3406,0x0000000000000001}, {0x000476594b8d6365,0x000aedeb8cb49714,0x000c86324ad6ba99,0x00016428c49863ca,0x0002a2e5cfc3d8a3,0x0009adc3e0cb62d8,0x000eff79e5f3fda7,0x000eb4f990b6cadd,0x000b0e410ae15a98,0x00000aedf394c7b9,0x0000000000000001}}, + {{0x00021de1984e2e65,0x000603254887ed1d,0x00087e7535512fa1,0x0006f4ddde7dd7d2,0x000fd9cd7f2faac0,0x00009274ea570744,0x000b8cb9c92e3b10,0x000294aa3ad7752d,0x0001e25f444d43f5,0x000d6a378f2a4af0,0x0000000000000000}, {0x00007294901179cf,0x000fc01f2357da9c,0x0003c6543ad3aafc,0x0004db8aa97c6078,0x000dd86b1f882854,0x000e842c5d55c6a1,0x000e30c9aff1c67b,0x000d71107c0af546,0x00097038444a87de,0x000d8c2bb6354272,0x0000000000000001}}, + {{0x00086f90f323a729,0x00082963dacbf625,0x0004056ab9d5b07c,0x0001d30212e37a0e,0x00065da9a3d1070f,0x000445a33677e429,0x000c39a6aee3c700,0x0006b968d6cd0279,0x00098776415f9a3e,0x0007bd59dc1684b7,0x0000000000000000}, {0x0005cafc1fb833a8,0x000b18657b2cb0ad,0x000af58b0bf123c6,0x000acc1e7ceb4334,0x000e5575f0e10ca6,0x000d22916c7bc194,0x00097f735880fd80,0x000604a86980d5b3,0x0002f218f0f8d8fa,0x0000806dbeec3ed6,0x0000000000000000}}, + {{0x000b6c1669c31b94,0x000b37bb5eb33b20,0x000f2f86abcd3194,0x000093b3893ef395,0x000d0b48537fd7dc,0x0006863b5109af48,0x000cb076c2fca9e7,0x000fc785f088e6f5,0x000c88facfd552fb,0x0001054316685a3b,0x0000000000000001}, {0x000b6af5d1972479,0x000ccf45d30e213b,0x000db2b3881288b7,0x0008f7bff9008ba2,0x000607009e5be8cf,0x000e10b176da0a78,0x000522cf433a33d0,0x000090924737e4d8,0x000cc0a11c296771,0x0001fd3715d11b75,0x0000000000000001}}, + {{0x000c391c2f2938b1,0x0006b396d1c5f420,0x0006bb17f5eeaef7,0x0006a57b7feb16a1,0x00026cc8015523f1,0x000ded6e6d4aadf1,0x000f602e23393c9c,0x000e2c8dbcb36848,0x00011e23c49f3a9a,0x000730c0c1ebfaf8,0x0000000000000000}, {0x0001b88cdd5da561,0x0002fcb4c22029af,0x0009624d6d5aa7f2,0x000db935bb120735,0x00019a8308449416,0x000467f9f185fd32,0x00057d8b4d3e00fc,0x000e187052a8a69d,0x0009e66380528c91,0x0007a42a603bc9b7,0x0000000000000001}}, +}, +{/* digit=14 [{1,2,3,..,}]*([2^56]*G) */ + {{0x00026efe1474fe09,0x00029da3ad38a0ca,0x000ec34abeaf5cd5,0x000847ac94808b1e,0x0001587ade96127c,0x000a43fa8cfa6df2,0x000bb39bcfdb5ad6,0x0005dd4d0c9f947f,0x000772a4ebca687c,0x0009227d79e215e8,0x0000000000000000}, {0x000926e1c81cb032,0x000ffdb04fbc5abf,0x00034707ba5b9c12,0x000a347c4ee8c89b,0x00072367a152d81a,0x0004511a3a4cd565,0x000b8f1a66429397,0x000260ea13e9d0e3,0x000a19a28ee14ab4,0x000d5dea76ba4c81,0x0000000000000001}}, + {{0x000f1e1b3fe2f1a3,0x0006da87421ab9af,0x000a3305ebdbbfcc,0x000cb7784b75a8e1,0x0006a4410056f8f4,0x000ff65612b5abdc,0x0004b1cd83f32f54,0x000fb989d25121c4,0x00014abed80a7bb1,0x0005ba8f200e1152,0x0000000000000001}, {0x000fb8525d63a07f,0x0002a4f5f23c02f4,0x000405911d9aa8e0,0x000dae0345abb8b1,0x0007b4834d14e7a6,0x000b31fdc5462195,0x0005dca7cbf9b75e,0x0001ee84304e26ee,0x0006a2c7d37349cc,0x000fc85a34c3afcc,0x0000000000000001}}, + {{0x000cefac8c58a5d9,0x000939b188506808,0x0005985dd6ee8422,0x00094a64ab14cf0e,0x0001ac27af983cda,0x00024f6eafd12785,0x00025d8bab20f8ff,0x0004a549d9c6da3b,0x000f18f37ed810bd,0x000655f630e4c95b,0x0000000000000000}, {0x00018594e51ad76d,0x000d8952697460ae,0x000aec56660f8de9,0x00093a39294777cd,0x000bdf7dc98fde3a,0x0000c53dc363fcc0,0x00091985d2c2708c,0x00093692d05055da,0x000c4d312ebcde24,0x0000c18d0017f5cd,0x0000000000000000}}, + {{0x000a3f86e5f8496f,0x0007f32df7ec8ea1,0x000d8551991f1f8f,0x0000f4eeb16ec397,0x000f5ebe5be1abc8,0x000f8233b8a1e6cb,0x00037675c403702a,0x000cbf97ecb04148,0x0006555682899a5c,0x0000280720d39958,0x0000000000000000}, {0x000312054dc27af9,0x00074dd550df7288,0x000193eb1e2f87e2,0x00073656a715c43f,0x0006ecb67dce2977,0x000aacb5db8a585c,0x000d92a6332fcd10,0x000cdeebccba4f16,0x00036c8da2b8001a,0x0007817b62765789,0x0000000000000000}}, + {{0x0002d6d084e03d59,0x000c244d84ecf179,0x000c93fa7f1e0110,0x000f695cc72b1bb4,0x000921730f1b2908,0x000b0b36b38d0bc6,0x00029c0e4bf469cd,0x000c1d41428da1db,0x000d1253d7a577f2,0x00006a23b655222c,0x0000000000000000}, {0x000ba5fbd7ebe31e,0x0005209808ec8aa4,0x00049718327a5383,0x000bb2492c210a5f,0x0002eef53e1dbdc5,0x0009d3c1717e38e0,0x000d4877b41e983c,0x00012d8aedea3a07,0x0007e058b6c0e3ba,0x000fd022c8be6da1,0x0000000000000001}}, + {{0x000870ab98aeb123,0x000044a353002cf5,0x0006150f34fa12da,0x0006eea20086b83e,0x000a0a2cdf13169e,0x00028616b25e80e0,0x000c5982d13e0cc5,0x00019702e01a4a67,0x000b60ef183d6e66,0x000a1f6f9172f815,0x0000000000000001}, {0x0002b57766386476,0x0001a0e6acc5477b,0x000ba422b2405581,0x00090991a9873020,0x00045310acf2c8c9,0x0008701ea796459d,0x0008c83917c30ec7,0x0009db51be44d168,0x000514c3bb42ce9e,0x00048d0b03fd870b,0x0000000000000001}}, + {{0x0007edbd640c9bb6,0x0001c6f483d72a01,0x00058a225c5b0849,0x000e8697568a7e71,0x00022821bf73d7fd,0x000c765e3aef4bc0,0x0001d2e8d1daf2fe,0x000772d486e7b59a,0x000595f951edfc03,0x000ffff1683f882a,0x0000000000000000}, {0x000fc53814c4cc13,0x00014196f30cc555,0x00076a3af64c6ce2,0x0009bfff339f5668,0x000ffe438adb5544,0x000aa59ae8f3c48d,0x0006c57ce59b5441,0x000eb7bdc7b7c0fd,0x0003b8e1d8e51d10,0x000d6a6427d57897,0x0000000000000001}}, + {{0x00005519264c8b35,0x00066ff272a08c1d,0x000142d545c343f7,0x000ef117b8bd86e9,0x0001c60c69c66860,0x000b54e53cb6de93,0x0000c9b9924f2f51,0x00030b949095878c,0x00016f5f1fba7e2a,0x0005817da79c3a69,0x0000000000000000}, {0x0006ad6babd55997,0x000be6d551de11e0,0x0006c45d4c33b3cb,0x0009e3dfcc4aa553,0x000821bb5c238e3c,0x000dfc012d05a1e3,0x000650684d8d4638,0x000805b7e2413b85,0x000718949cdcfd8e,0x0002efc1a85e6627,0x0000000000000001}}, +}, +{/* digit=15 [{1,2,3,..,}]*([2^60]*G) */ + {{0x000f6a9bd09d8e58,0x00004bdca0a6cc0a,0x000d9e6d336fe5f9,0x000c9d8bd87d0339,0x0003f4d463bab3b8,0x000203e46dfb629c,0x0000ef34ea62ed4c,0x000564035458998a,0x00069592c6278328,0x00081b3c56ebb377,0x0000000000000001}, {0x0009a17aab96cc87,0x000f8ed51ce44125,0x000c62b1c658666d,0x000b6999437c7966,0x0008f0fecb36474d,0x000f725b1f7c6099,0x000396c71fdafc21,0x0006a547fb5a5b56,0x000566ae79d88868,0x0000f4130033ff0f,0x0000000000000000}}, + {{0x000a6c73f41a4fd1,0x000bf46616431912,0x0009c6ffbd2fe4c8,0x0009e4fd42f313ec,0x000b9f8b100ba286,0x000e18229bbae712,0x000550161a1f1da0,0x0000032c80f2ffe5,0x000f0b1d53bfaa0e,0x00035fb83c760748,0x0000000000000000}, {0x000ed33353cf7a1f,0x00075b8b3c031fa0,0x00053c30e33c1415,0x000945a8fa62217c,0x000aa8b667de4f9f,0x000c4952fb889399,0x000b6e3b711abc77,0x000959e7e12fabed,0x00057ebfe5a1b2cb,0x0005344206e24318,0x0000000000000001}}, + {{0x0005f9c5a301de7d,0x000b800d937115a9,0x0004b1412c82ee0e,0x00039cf3df5a5904,0x00016cd50327ee6f,0x000841dfd19a796b,0x0005d79c493ac5c0,0x00097275eb2319d1,0x0003b6feb4b9d447,0x000542e1eb10df1a,0x0000000000000000}, {0x00051bac56d17f38,0x0007837a907c7875,0x00082e7d67e232dc,0x000c3c225acaf222,0x00056e17100c95eb,0x000198b234622502,0x0006b8a4beb3ba23,0x0005492d30351698,0x000c0dd28973e413,0x00031b2e1155d6fc,0x0000000000000001}}, + {{0x00014d7bc4df6981,0x000c45e951da151f,0x0003964143f3d397,0x00056c9c24be6549,0x0000ae1293e252e5,0x000bfda40e3aed33,0x000e72cdf82159a4,0x000f514f7b173b13,0x0000684bfa5b859f,0x000fce90812f67e2,0x0000000000000001}, {0x000a9abf7e9a0c25,0x000a823b0b3a0fbc,0x00011d2709072194,0x000b7a7f17f5564f,0x0007987f0af999bb,0x0009d6201796c014,0x000d35c45cce25a6,0x0009265843370c43,0x000a55401cbff6e8,0x0000eab503e2ea19,0x0000000000000001}}, + {{0x000caabc4b5cd512,0x00034cdeccde50ae,0x000395d24049ffdf,0x0005918925068e1b,0x0003f93fb9ea4405,0x000a60ba95d141ad,0x0005981c42f76f02,0x0005946bf800414a,0x000435023138c47b,0x000f3cf314147e38,0x0000000000000001}, {0x0008bdcc69ae19e3,0x000ac73cebd917e3,0x000c35337880966b,0x000e6ede2718c3e8,0x000fd10236ae833e,0x0004797bb14f5b88,0x0001296485e76bd4,0x000a68194c12b2b3,0x000b75dc1e45112b,0x000dd88574000b0c,0x0000000000000001}}, + {{0x00037d315eea24bc,0x000160f77a65b38f,0x000de27962237731,0x000bd3346f06ae65,0x0007a25b38b1587e,0x00055c6b9f2a1d2c,0x0002f34b1687394a,0x00054f27c66a0ccf,0x000866c84ecf3de7,0x000b4da4a0f4aaa9,0x0000000000000001}, {0x00066dd8b8dffeb8,0x000121bfeaeff003,0x000a80b5c3bfe941,0x0005b6a4c3fed2fa,0x000c623dfdf4718a,0x000b0791d22ef007,0x000949ccec61c6bd,0x0006e328d9cc6d79,0x00014a1530d03e69,0x000d45fdb36710aa,0x0000000000000000}}, + {{0x0006dcfa61a8d3cd,0x000ff977e2649373,0x000089ee4ac6af49,0x000b86d61720bd71,0x0007848d2c5df2f3,0x00078e07afbcc66a,0x0007ceb1f230a9ed,0x0000d2f61bf5077a,0x000770c3ffbf99e7,0x0001487ae5f08492,0x0000000000000000}, {0x000ee44c464c2996,0x000d8f990f4f03a6,0x000453774274aacb,0x0005c8730ef447b6,0x000e5e02e661f55b,0x0009f13f1011e65a,0x000f4c8fe17d3ed9,0x0000dbeb35dd393c,0x000a7d1cd2327711,0x0003fb444802cd65,0x0000000000000000}}, + {{0x00071a842f621fd7,0x0008594c057a1dea,0x000e1689c80fc8fb,0x00022f52adc9a8e1,0x00099c47b816309c,0x0000c495f00a960c,0x0002e200a0f356d9,0x000a87494b798824,0x000dcd587b7f9ca6,0x0009c4d76d2c396f,0x0000000000000000}, {0x00035970dd6ecf15,0x000449aee47a261e,0x000adfd394c8df13,0x000dfbec67553f2c,0x000aca43c615471c,0x000606556ef09db2,0x000a225f2e040114,0x00099dfb28da12ec,0x000812bc587a4c83,0x0001ab8cba898428,0x0000000000000001}}, +}, +{/* digit=16 [{1,2,3,..,}]*([2^64]*G) */ + {{0x0006146954276a1b,0x0000c171344edf1e,0x000b30130812b4a5,0x000314a14896c770,0x000e796a686592cf,0x00079226d890053f,0x000869a5847ac79d,0x000cf60993a83ada,0x000e4b5fe7d156a5,0x000add7850cdf667,0x0000000000000001}, {0x0005bcbb35fb3dea,0x0000a34e2d6021f3,0x00090be93989877f,0x000303404d6435bb,0x00007e5919257861,0x000c99d1992710c0,0x0001c7987d3586cc,0x0008e8681c58c145,0x00059a487fa896da,0x0009a8b1a9e54366,0x0000000000000000}}, + {{0x00029533273f3ddc,0x0009580b259ba7fa,0x000a4092fea94f8c,0x000efd38be9d56f6,0x000720f2ba425622,0x0007c0adb2a4d25a,0x00018752498a9ea5,0x000d893bbb4d11f1,0x00056b02f195ec41,0x000bca2ad72c4b2f,0x0000000000000000}, {0x000a4013f1ab7060,0x000f17521f983a0f,0x0005292b2f1ebae7,0x00075002debce289,0x00003b6cd203ad6d,0x000c3592c993bfe5,0x0005400a40b351b3,0x000e9b6bafed180f,0x000d6a9f0291283a,0x000563036cf95dd4,0x0000000000000001}}, + {{0x000efa3e474a5b75,0x000e5a6d55141813,0x00008a31bd3435d9,0x0006a68b3c599425,0x000252817b3af9bc,0x0006a695a37f35e7,0x000f06836ac1d6b0,0x000b19a92f525bb2,0x000215d9b8e544be,0x00080723f1554fe6,0x0000000000000000}, {0x000e709e6109d280,0x000e367fd7bcb2f8,0x000be531b1c6fe21,0x0002d1fc5388b64a,0x000c169c0609b0df,0x000cd0fc3d5f664a,0x0003de00815f78ff,0x00049af9ff38956a,0x000e62c250aedb30,0x000af977c662b6e7,0x0000000000000001}}, + {{0x000cfa5a95db7680,0x000cc333878665a8,0x000809b2a4ba5401,0x0009594f6cdc3f0e,0x000e99bbfac6790d,0x000d836074d551d6,0x0009d9ae874e847f,0x000a264b3b0b13f8,0x0003ac51f7a6ec5f,0x00021d6dd250c60a,0x0000000000000000}, {0x000e14abaa7747bd,0x000f127c3196cad1,0x00078a629241495e,0x000ded5d0cbcf8af,0x0005b83d56ec31f4,0x000c6ef029fa54b1,0x000cc516f0a12c6c,0x0000ce830e11ae62,0x000747fe9964fd2d,0x000c6756076a3288,0x0000000000000001}}, + {{0x000b9a69b333564c,0x000cec5ad994fbe9,0x00091ab4f5d4d95d,0x0004dbde503f2b0d,0x0003c8b32fed7498,0x0009fa10339bf799,0x000b1e6b4c7bd908,0x0001c54e64a99794,0x000598ef7979d915,0x000f29a322ae1bc8,0x0000000000000000}, {0x00042520c237fcf4,0x000303a14c0e24d4,0x000f7ed62c67df4c,0x000ee5dfd3635dff,0x00097bcf94654c63,0x000eb529e2ea1f28,0x000969c7cff702bb,0x0003bf0591306903,0x000474be25c3afe3,0x000d8f2570e7350c,0x0000000000000001}}, + {{0x0003d3d928f89c37,0x0004d9c668cfa4b7,0x00097ee2907da69c,0x000fb743bf4c3402,0x000cd4034c59cbf5,0x0009bc4b73d60ae9,0x0007664da82be729,0x0007e3800a84da1f,0x000700f12fb007b6,0x000882b546161eb7,0x0000000000000000}, {0x000e150bbd0f66b9,0x000122fc5d0def4b,0x0001ba0f43d660c9,0x0004e9263a5b4550,0x000ef33c24e5b722,0x000249e1b7ba92b4,0x000aa152b1856c8d,0x00095fe68108b2c9,0x000e766ae6e54017,0x000903a379f58c2f,0x0000000000000000}}, + {{0x000980153e2aed11,0x000ca0053853a90a,0x00003bf5f23c9661,0x0009061283e91bda,0x0006236d967ae1de,0x0004f3d80708ed52,0x000bb014e9c6763d,0x0002f3d82f09bad2,0x000f8d828de34c80,0x000187805c46ac1b,0x0000000000000001}, {0x00086a0c8fd9feb0,0x000d468bed15d861,0x00036573dedab729,0x000d88e4dce7ee93,0x000d9e86caa8e75e,0x000c0d08d6110fe2,0x00082b709f0b0d08,0x000ab3fbb5af39b1,0x000c1b7910befecc,0x00062bf43fafb941,0x0000000000000000}}, + {{0x0005341f232f3278,0x000c66dff5ad0b4d,0x0006270a82ebb141,0x0002897d7912e413,0x0006b6b16ad87fc6,0x000fe7c18f348f2e,0x000a03bae57af6d0,0x000a6d2d6ab02f22,0x00017c3e7efa7a28,0x0009c673423958d7,0x0000000000000001}, {0x0004f0f2ce49ed5e,0x00055b8c6c92190b,0x000aff1be7fa884b,0x0002c375de74b331,0x000f37a676c7d888,0x0001190b6b57c355,0x0009c95180dbbfa7,0x0001d7dc77b1599b,0x0007eba118f76648,0x0004aa840229ee22,0x0000000000000000}}, +}, +{/* digit=17 [{1,2,3,..,}]*([2^68]*G) */ + {{0x000507b4b3de6fe0,0x0009064574b533f1,0x00076eaa707b56c3,0x0005e98aa9532376,0x00034611c9b6716e,0x000eb6a26112d9a9,0x0006e068430b4789,0x000e50fd96103fab,0x000509b62d215cdd,0x0000b7c4da786d1d,0x0000000000000001}, {0x0002a0af86e4be0a,0x0008383ebf635e75,0x0009175f3f7680f5,0x000b999d9f1a0d87,0x0001f04cce1e2861,0x00086e6afd75ef23,0x0004476af7240e6e,0x0005e887f56c0473,0x00094ba352837e09,0x000f469e3dc524c1,0x0000000000000000}}, + {{0x000726e7d4a98265,0x000a397d1e874b3c,0x0008c78c755a513d,0x000fef1392677915,0x000e1e9e24f3ae62,0x00096cf6213d1cf9,0x00004b1f503d4fcd,0x0000f07e39bb0e12,0x000406c607195818,0x00046d3b7b9a617a,0x0000000000000000}, {0x000b43a8c35ac9c8,0x000a92f37f3857cd,0x0008ae49b6bed377,0x0000e3380827d789,0x000e2deff6865bd5,0x000758e466fdb287,0x0001f3ba0c560a0e,0x000db418a2645432,0x000aa26f5d44767f,0x000d36cc7b7f8bfa,0x0000000000000000}}, + {{0x000020f87c5aeb2e,0x00035693c551e0d7,0x0004a7f0938f3f98,0x0008d4d829ae419e,0x000d8e42f19ee358,0x0004d94560e54e60,0x000ddb20bc96b546,0x00014bfcf64c2c92,0x000518fec9517e43,0x000a92773a95b0b8,0x0000000000000001}, {0x000c159b5e61b250,0x00070c6538519060,0x000d0372c4dd1c2d,0x000fd8998866b5fd,0x000ceccabb79115b,0x0000eee1fb689260,0x000b839fc8378f05,0x000677dee33a7134,0x000b9326b080c62a,0x0003ae07e602129a,0x0000000000000001}}, + {{0x0000dde02f16a4fb,0x0001ca814d149b58,0x0003cc8c599eaa3b,0x000d833d43a45440,0x000afdba29de3c98,0x000b2ff2056e31f2,0x000ab3bf81e95cba,0x000649419f19b530,0x000585b648a6e1bb,0x000ff11dfbba1ed0,0x0000000000000001}, {0x0004734696bc60cf,0x000199c0250d4a2b,0x000d4759758b9f4e,0x000f68bc326d4e2a,0x000cea78113abc32,0x000d248f92840b01,0x000d61ebd87644ac,0x000ae9a32d38a8d8,0x0000c706b58a69c2,0x000561f4b942e16a,0x0000000000000001}}, + {{0x0002fe7e20769488,0x000a1c068b87a199,0x00032a3459c496fa,0x000e96c391b8839f,0x0007f4914cec7211,0x00039756176c58a1,0x000aa63441905f9c,0x0008ea16377fb867,0x00042578ce2a2865,0x000cd140c73dd697,0x0000000000000001}, {0x00074ef7ab059a14,0x00031e0f7d0f636a,0x000fea67e10b4430,0x0006ef5711a15a06,0x000f717856862153,0x0002513381cfd3a9,0x000a8b9906086974,0x000a08521bae079c,0x00052ef6a1d4b90c,0x0004f5ad24ad39a3,0x0000000000000001}}, + {{0x000c2d28a3a30e7c,0x000933949c694069,0x000d1f2a5ed527fa,0x000e2b5573d97d5c,0x0006aa892b500798,0x0008ac4418f1419e,0x0007bf2d1e227f12,0x000bc2143f4c8a62,0x000ae6ef99da3403,0x000679827f46143c,0x0000000000000001}, {0x000a8cba49268935,0x00038ccee58ce9de,0x00061df0ac493218,0x00075eb49225f314,0x000073000d3eb206,0x000145d79e65edfa,0x00007b1990aba7cc,0x000c916353a37685,0x000854a40d94fba3,0x000cb2e109e1b50c,0x0000000000000001}}, + {{0x0000c263771f69c1,0x000d1595543d358f,0x000ef2e3d8a56184,0x0005866bc8c3965a,0x00021c100a7748c1,0x00029b46541a4ddf,0x000c5cb49ea744d6,0x0007a3a96b8b1fb0,0x0005aa7e85061bd2,0x000163e83c339e28,0x0000000000000000}, {0x000102ac6923a672,0x0004153c3e1bd59a,0x0003473bcdaf9384,0x000f589da2a6c958,0x0003af0eb1a0311a,0x00041f50baac8d3b,0x000bd2c8728d9afa,0x000fc24b4e16e953,0x0003e6d4570ad37f,0x000aac57efaaa8de,0x0000000000000000}}, + {{0x0008c7fccb0a85bf,0x0003bdff198eec53,0x00029ee8af84ec04,0x000572ea125b846f,0x00025280cfc9ed01,0x000f73f2654ba803,0x000ffbf57e3b7be3,0x0004f83701a26bca,0x000d20a251a2d372,0x0003ec410f80b319,0x0000000000000001}, {0x000961197dc0e519,0x0003d8c136f93b3b,0x000000ba8d6c2646,0x00084f848d99824e,0x0003fcfbb42b20e0,0x00038715f781fef3,0x000bdd048ed10781,0x000042869724ca7d,0x000de2c203c66b90,0x00090489fab2c4cf,0x0000000000000000}}, +}, +{/* digit=18 [{1,2,3,..,}]*([2^72]*G) */ + {{0x00038bb0c7aa46ca,0x000d2cad2e37a158,0x00054d33ad65a28c,0x00004b20d4f1caa9,0x000b180e4c9d244b,0x00070a13f58c65ce,0x0008816ecff016c3,0x00016ad260aeee75,0x000dbb7b595c36fe,0x0002944d06dfe8bb,0x0000000000000000}, {0x000c325d67839921,0x0006a5955c2a22f5,0x0001664092579a37,0x0003aac4f8e9390b,0x000722a8dbf2236b,0x0007b02d94034f2b,0x000fcd8d5de86b97,0x000fa8bc9f80729c,0x000bcbc03be296bc,0x000364ec1469f11d,0x0000000000000001}}, + {{0x00048e2fcaa83f58,0x000495fafcc6105f,0x000d34073fc2c5d9,0x00042510321d1a08,0x000d83427742e304,0x000ec5f97b8068ff,0x000530da7faa8a52,0x00005d010e52ac14,0x000df5701f277a14,0x0000eacdd532283e,0x0000000000000000}, {0x000f566cbb172daf,0x00050c51771845f8,0x00066aafeeea7b0e,0x0005258081cf4ee6,0x000c71bc2c6ec8fd,0x000790d250232a19,0x00011bed4c06ab26,0x0000acd06e0bdc44,0x000734273e0fd2a2,0x0005c1c9fb738a19,0x0000000000000000}}, + {{0x000b6ef22ab96e69,0x00026b8cd18dc5dc,0x000563a07bc4111e,0x000f73913482455e,0x000edcb5ec4ad0e2,0x0000caacaf21483b,0x0000f16a5a48441c,0x0005bffbf280c9e8,0x000f37a7690242b8,0x0002eed954418691,0x0000000000000000}, {0x00066d6e31428479,0x000f47fc4b8794a7,0x000a81360ec38293,0x0004d77d31e9f867,0x00042db92af31be3,0x000d799976882df8,0x00005ddd34a906cc,0x000b961ddfb3abb5,0x000bbb326a3a37b0,0x0005d4f7af85a74f,0x0000000000000001}}, + {{0x0006d183b4b1bfc3,0x000143cdd1f50d0b,0x000721cf9d088770,0x000338fad247118a,0x0004efa498ee55a8,0x0008d9808733ff45,0x000faa72107a954a,0x000a392986064eae,0x0000c503bc385af5,0x00095cfc7e0cec3e,0x0000000000000001}, {0x000f6c1133ea5097,0x000685bf161ebfa2,0x000bb087e9c48b5f,0x000987c958eb481e,0x0001ea465a54c3b7,0x00081943446e92e0,0x000a8941e66d88ba,0x000d40dc6c71b0c7,0x0007f59a3690cafa,0x000130f02679ac05,0x0000000000000000}}, + {{0x000c5b4b93b19e8d,0x0001b316df16919e,0x000549bdce1a1a0a,0x0001cc00d06d528f,0x000cf17722a1d1b7,0x0009c7821f1ae098,0x00007c6fc2825a3d,0x000f796a6b06d5bd,0x0004ca4366a06040,0x00072103b88040d4,0x0000000000000001}, {0x00038b6ea0266cec,0x0003d4ff03eab713,0x0008f81bfb033aae,0x0004b58cdd700e03,0x0002486b62595d6a,0x00027288e5a75ffb,0x000043649c2c3428,0x00056516d9dc3a87,0x000145473b4564bc,0x000f9a0961272b27,0x0000000000000000}}, + {{0x0006ad885aea13aa,0x00022fbb0e237052,0x000115fb0a1aa034,0x000ccb8ce398278d,0x0004178f8f78a3bc,0x000403be5854a510,0x0006bcde0e2e720a,0x000edf5f632b7fc4,0x00038eed71d606d0,0x000d21f92dbc6d75,0x0000000000000000}, {0x000631ff7cd8f78c,0x000698475f524849,0x0005eafa796bd6f5,0x000fa7a4ee42bd53,0x000b8f8540687776,0x000540f9fdea3095,0x000f5bf17004e5b9,0x000834c440599d47,0x00063874f7c8a4ef,0x0006bec9a4f28185,0x0000000000000001}}, + {{0x000c12a0297bd390,0x00020c5821857743,0x000e07f2b528d6e4,0x000318df51deafc4,0x0009c03086d94e26,0x0004d01fde5e064b,0x00071c1d5855c1d9,0x000cdd2d762cca56,0x00022e583500cf8e,0x0003f52279b68b61,0x0000000000000001}, {0x000b572837258eb6,0x000b88af33e25ea0,0x000ba976e7082c63,0x0000d0abef5e06ce,0x0005c8fda6ac69af,0x000f82b26435f42d,0x0005ef9369e629b7,0x00029491bb2be768,0x000ae13ca4da59b7,0x0000f00ce8846bfb,0x0000000000000000}}, + {{0x0000079959682d5c,0x00076719e3233b3a,0x000c78c2194cd545,0x000e51d3c744ff86,0x00053afacd6dd789,0x00098cb1ba7a5cd2,0x0004fb918b560853,0x0009ff1bce38273c,0x000a7efa90ba240c,0x000ba73bb2e372bc,0x0000000000000000}, {0x0003a114b353d398,0x000d2df4adbd1d56,0x000e9ad940e90284,0x000fca7fe3af63ef,0x000de96feaa4e61f,0x000bf94ff4ba0669,0x0005279d7b8471ad,0x00071dda976e696c,0x00066b8800a22911,0x000f5fba44b58815,0x0000000000000001}}, +}, +{/* digit=19 [{1,2,3,..,}]*([2^76]*G) */ + {{0x000b896b5317311c,0x000ccb025efc3b29,0x000bcd9f85c60e34,0x0005f821ae29c1d9,0x000293dcc63561e8,0x000f95824c27219e,0x000843c9e01039f3,0x00048ef0f79fd3a9,0x000ddb5a3bba44b8,0x000011f0a7f5379c,0x0000000000000001}, {0x0000be31597dedd3,0x000aaa0d73669dd4,0x00090c605d60706d,0x0007d62f262a826a,0x0005e90997b0e2e6,0x0004dc73225ac29c,0x0008be39728fe4ca,0x00038656b7a746c2,0x0008bd5a3cf46a3d,0x00080a0c58ac7031,0x0000000000000001}}, + {{0x00070e2b4e96cd1b,0x00023ea39d68ac53,0x000f98a99040b26e,0x000362a9be557ba1,0x0003c202765cccef,0x000726d7b5a7731e,0x000faf8cd815e2ba,0x000ba6579cd91c25,0x000ec8fb3dafb2e8,0x000af4648049fac5,0x0000000000000000}, {0x0004b6251452046e,0x000cb296110b89a0,0x000551f88c9d1ddc,0x000bbb0b0f26d015,0x000c2b5bd39d39b8,0x000dc18ef79d52ff,0x000b527ab6d006e2,0x000a804f61d0142f,0x000be5992391511f,0x000815a3e717ea9d,0x0000000000000001}}, + {{0x00029eaddc553333,0x000212246b16c20f,0x0009da31a639b833,0x0005a63d297cf150,0x0003190a2a3a8499,0x0002ca8af6260545,0x00018490cdf918d2,0x0005a5ed4b646253,0x0004fec387ca9de6,0x00010b72b35acffa,0x0000000000000000}, {0x0006d539b23d78ec,0x000a4b221e3646f9,0x000b6bf83af256f3,0x000d62f0c408a90f,0x000fdaefff14a7ab,0x000e41ce0c4069cd,0x0001cba29824953a,0x0007a382ab7eb47d,0x0007f64599eb440b,0x00074a4c148b6095,0x0000000000000001}}, + {{0x0005a160c85caa71,0x000f0e79edff6be6,0x0007704970c1ae3a,0x000a395f8b957c42,0x000c85f0f181eb8a,0x0007d8f529bdf3d6,0x000d3534e626c58e,0x000c770dabfad83f,0x0003e65d7e5ada98,0x0005676430730bac,0x0000000000000000}, {0x000d0476d73fcec9,0x000d714dcf97c309,0x000a56a3252ab9b5,0x00097fc79648c08c,0x0006b897ba609ff2,0x000c446a06ff8430,0x0002d01ddb643744,0x000f97ee1218bc20,0x00048db33f9b0f80,0x0005e8b5f54bb8f0,0x0000000000000001}}, + {{0x000851bfcfbb80d1,0x00082b51c40077e9,0x00087cd565dbfe09,0x000954bdd372a1cb,0x000f6bbff7b4dccd,0x000237c51d294b36,0x00003d64ce0f8798,0x0000569d6e3c2614,0x000a6224fb79e0e6,0x0004d7c33dd3b5eb,0x0000000000000001}, {0x000f054ad5a9cfa1,0x000bee5f93daceaa,0x0008aa260aa160bb,0x0005025da9f4b722,0x00004817d1e67b1b,0x000e00279781308a,0x000c2084afd2f00f,0x000c154f68e6680c,0x000c6b0f1d4b7ecc,0x000fe2b89761184c,0x0000000000000000}}, + {{0x0007fbeff2b9ea68,0x00035c954c6cefab,0x000062277d67291b,0x000820637553137c,0x000e8b75730d8af4,0x00068d2250710c68,0x0007a2fbae3e7c1b,0x000f6b643e1aff63,0x000dc46991ef002b,0x00096e38ab4582dc,0x0000000000000000}, {0x000bea0d80f3758d,0x0001d6899ee62692,0x000cdd2a79a4d763,0x0002f50f2ee3aea9,0x000fb1476eea0816,0x0007c81475c4d433,0x0009d9fe82142372,0x000756c76934dc96,0x00033eb086d918c8,0x000e3056d2a89175,0x0000000000000001}}, + {{0x000ef0de9496fb55,0x00076806fa762c50,0x000770bcaf6f226f,0x000a0d3e47cff6b0,0x0004eb780ef8cec1,0x00034df87449872a,0x000896382a505c84,0x00046b56a94dfc29,0x00021c7a5c037f2d,0x000659ef98ff9417,0x0000000000000000}, {0x000001c935289072,0x000f4a229e4010ce,0x000b1be023ab7710,0x00073fb44f780b68,0x0002944ddc611373,0x000ba09ab8b61290,0x000be003d4bb157e,0x000f7557730f52d1,0x000506c275d184bf,0x000c57abea8b4979,0x0000000000000001}}, + {{0x000004498fadaa48,0x0009d0bea3c3894e,0x000a8f46458aa39f,0x0000008b0b3654a0,0x000ca4cd7392bf83,0x00012eb97aa46a22,0x000b9cb80e1d7afb,0x000cf74c8adcd888,0x000b51d04bb6e179,0x000b50968eb22473,0x0000000000000000}, {0x000f96d03d831756,0x0007499de9e10051,0x000ecddcd4ffade3,0x0009dbdffc72771e,0x0006b5e1bb9647aa,0x000addb508dc2415,0x000ddf40de78eeab,0x000ab1c488946fac,0x000c823824a964d6,0x0003b16f258c8749,0x0000000000000000}}, +}, +{/* digit=20 [{1,2,3,..,}]*([2^80]*G) */ + {{0x000cc508d53fa611,0x00057fcd40894532,0x000f3eb54e960b10,0x000ea40877d231ff,0x00043e5110313bd6,0x0005209f6eb9ee83,0x000589764924e778,0x000332e258b2e7fa,0x000e2e038618a6eb,0x000aaf96067c3511,0x0000000000000001}, {0x00022156015c8ff4,0x0003a0ef974e440f,0x0008ea1f931a1b7a,0x000e417472932b48,0x00003bb75d745720,0x00096758e51bf9c8,0x000f97c7f0b39099,0x000b39d56a488d83,0x000fe1ded1fac932,0x000399aaf43ccf55,0x0000000000000000}}, + {{0x000fb78344f73774,0x000a49ad3e73876f,0x0008771e37ad3158,0x00003f2cb98ec469,0x000f31bd531106f1,0x000434959ff8325a,0x00064eee47f875ba,0x0001cf224bc0a102,0x0007d33a19ccf2f6,0x000f2186ce603133,0x0000000000000001}, {0x00006e91f2869773,0x0000239179c5ea67,0x0009aa4ed3879ba3,0x0003eb977e239f26,0x00090ef091443aa8,0x00036fc4d282853a,0x000a0bb2b260d343,0x0008119fbd0756b3,0x00053a3a6e0f1619,0x000016a2af080234,0x0000000000000000}}, + {{0x00000185bd1fcc92,0x000a0002ebe1f280,0x00030d3e5f23ebcd,0x0009d40f75cccaba,0x00063108edd488ea,0x000028024e152a65,0x0008296732e422c6,0x0006142e6cc11761,0x0004e44889dea726,0x0004d1b05325e95d,0x0000000000000001}, {0x000270a2e4063870,0x0004d9c29b5dcaf3,0x000d2f759d7bad98,0x000ff7c2ad7bc046,0x0000fa4e4f59d347,0x000a06be29c16d4c,0x000bb31872d14ff0,0x0002b7a5b6ec2390,0x0008ae4cc66be2ce,0x0007006b9b1fe040,0x0000000000000000}}, + {{0x000c4cee52cfafd0,0x000916a99628d59c,0x0004417813a4764e,0x0003f0c49a05da16,0x000992babb644e42,0x000179a66e24dca8,0x000cbef894f6883d,0x0001ed7756c7c157,0x000ff08e144c3013,0x0008ac78b0a3e9cd,0x0000000000000000}, {0x000dabd753963ba6,0x000426be7ba3ec43,0x000d17b8f8b93626,0x000d7ac0bfcd2a78,0x000c2aeda53c9486,0x000c99eeaefc3c49,0x0003d0949fb4a9cb,0x0005db07562812ab,0x0005da4c6c0f863b,0x0009e08ec31fe43d,0x0000000000000001}}, + {{0x0008d00b0f45825d,0x00075f4acb7a9109,0x000fe317cf8f4f81,0x0009a77cf8155d16,0x000d7ac3ddeef2bc,0x000aeae3c417520b,0x000e6ff44ee6fbc0,0x0008d8c238521aaa,0x0003f42c9f47bc82,0x000d2fc09b26d055,0x0000000000000000}, {0x00032ac7c6897ed6,0x000498c1e669bb9a,0x000697322f4c8aca,0x000625a543042d46,0x0003cdf16aa69334,0x000b4b67c267bda0,0x0005d6f205d341fa,0x000005daa2bd83a5,0x000c9573fcdfd94e,0x0004e81cb76afe9a,0x0000000000000001}}, + {{0x00074fcede51930f,0x0001c997863b91f0,0x00092d449a3c4328,0x000c91197a68c2d7,0x0006a3b2de0b3063,0x0003e7d82555e166,0x000f427f70b4227f,0x00066c04e18d6aac,0x0004c82c2c2b9b61,0x00095a376fa210aa,0x0000000000000001}, {0x000de0f4a3a29f55,0x00023263844f1727,0x000fd0bec7770941,0x000e79f43b5f4e85,0x000035cbc9a5768f,0x0005bb2328826a73,0x0008a77da7d22096,0x00096978fe424078,0x000ae1a0514c7cf9,0x00085e77943ce3c2,0x0000000000000001}}, + {{0x0000d1b4594afc85,0x0006a925e9937fdb,0x000d1cf398814c56,0x00000694cd250848,0x0005fbfd82b6ccc3,0x00047db4ae135bc7,0x000c1f18639e63fa,0x000730a5e5b32295,0x00041bb1c61f91b2,0x0009451335383b28,0x0000000000000000}, {0x000df27e3f2dca32,0x000ee40fb695c7e1,0x000c8c313d1721a9,0x0008bc93267e9801,0x0006a9aafbe12b28,0x000e34c2b180d7a3,0x000e6b65e8b79ae5,0x00047da7f03b22ef,0x00094e563552e913,0x000f4aab16538cee,0x0000000000000001}}, + {{0x0006db42e9e50fad,0x00042149f7546b33,0x00057093c06f6900,0x000dbce88e00d7d3,0x0004ad9ede7428d2,0x0001940521d004dc,0x000e4970d3be2ce0,0x00031bb5cf60dc4a,0x0003df5670a6ccb0,0x000b64df04605d80,0x0000000000000001}, {0x0005f0fb0c7d8a77,0x0006951f8ad28aa0,0x000e5b908dd39d0a,0x0005f76fd67e92ff,0x00080f281077f416,0x0009ee2db2c8d529,0x000e9a09ff0b841c,0x000f9a5850f2e792,0x000e9887cd74d1ff,0x0009aa468c4978db,0x0000000000000001}}, +}, +{/* digit=21 [{1,2,3,..,}]*([2^84]*G) */ + {{0x00045bc39568d9b4,0x000deebca99c74cf,0x000ae132b3816aa0,0x0003fb57b713a9d8,0x000e4139b0131f8d,0x00042164bbc38156,0x0005a533293d5ffc,0x000f28a54d0e74da,0x000246758970fcef,0x0005cb3ef8fd2b56,0x0000000000000000}, {0x000e59886b3941bc,0x0009ade1a4b217a0,0x00023117719aa3f6,0x000c88c1b7b4e45e,0x000c3f1294233118,0x0001ed8c9dd7dfa5,0x0003ffafa2104f8f,0x000e89ed71382221,0x000d0f8573ea0a97,0x000a81f09db9f82c,0x0000000000000000}}, + {{0x00077d542b1d8c85,0x0009eb6c76648a7d,0x000ae9936fca1675,0x000c8db3d9556eeb,0x0008f1fc23af7239,0x000956a6643df02c,0x0005264fec894e1d,0x0000aaa92f802a0d,0x000d756f014a90b9,0x000e0753d197ff93,0x0000000000000001}, {0x000dc80b7f62d4df,0x0005de026f897406,0x000bf46ad6add1ea,0x000f1c5d1c416858,0x0004d6b3d82ce8f0,0x000459159df587ce,0x000ca473b92c19aa,0x0006da5bec10b6ba,0x000e0b4be600af3c,0x000cf6c81e2b9b40,0x0000000000000001}}, + {{0x000c67c15f2b99a5,0x0002e78f41b12d02,0x000f7e85b9e3bb96,0x000e6c293dd82110,0x000a0b7296f35eee,0x0008b9f01a446a48,0x000f7d8d89873daa,0x000d90409d0bfd67,0x000b42d1c91c3962,0x00005e6c3afa1fdc,0x0000000000000001}, {0x000607a511880114,0x00046b3caa5f65bf,0x000ed1d7edcb6427,0x000223b76ca3eaf2,0x000c254785aeacd8,0x0003d96bed772614,0x000fe17074c3138f,0x000fc4599803c303,0x00029d4441854c4a,0x00068d2f47c7e6ad,0x0000000000000001}}, + {{0x00027dadf66942d9,0x0002d934d4f0887d,0x00024ab4a3b4ff58,0x0008094d151ee4b7,0x00059df116aee58c,0x000c8ad8141ceee5,0x000c277f6cc0cd16,0x000b9d41dd2c3d13,0x00018d63c75e0cd1,0x000c06bb0767f3f3,0x0000000000000000}, {0x0003f3f369ef6b69,0x0007808f1170f91f,0x0006338ef6344906,0x000cb4597495e6b8,0x000283f524ab766d,0x000d7731127ec634,0x00075be86373e0af,0x0008d2af0e3a5495,0x00014c6819dfc2be,0x000f7affa7af5c63,0x0000000000000001}}, + {{0x000892c7124cd33e,0x000b92182e71043c,0x000db014cae4df90,0x0009ceb0a4ea4119,0x000ef942ed19b706,0x0000628b68098d1b,0x00084c6cb159ff0e,0x0004d4b5184cdc53,0x000190191d032e77,0x0006b2ec68efce97,0x0000000000000001}, {0x0003436236c90d20,0x000e12c74816ecfc,0x0003e074e74849dd,0x0007be255b09c30d,0x000526507824bfa8,0x000ef7cb512aeee2,0x0002afeb20a50c2e,0x0008aa489a2c4123,0x000334adba554c1c,0x000ed1b94b601c93,0x0000000000000000}}, + {{0x0001c752e894f83b,0x0001102db47294b0,0x000efd031528755f,0x0008e7a5233b9d7e,0x000dc69c62c7500f,0x00051f44711ee3b5,0x0009d94a5e280f05,0x000b11042cd8c7dd,0x0008b14370c2167e,0x0007807f4636e4af,0x0000000000000000}, {0x000598691d482817,0x000b8412f599a077,0x0000459d6b4cf61c,0x000405e26f27cc0b,0x000ddbc7fdaf5126,0x000cdbba7c4a3026,0x000b262658e4a3b0,0x0000f2e795bb25d0,0x000766509eec95e9,0x0005a452259c52c8,0x0000000000000000}}, + {{0x0005f6dc2c523bd7,0x000003d082f18f83,0x000ef7ee57fec42f,0x000f2558c3c5a1e3,0x00038f0a9e89f93b,0x000efbe9151ca6a7,0x000ed767c4685458,0x00079d486204e6f8,0x000a36a1a4b728f1,0x00043b885c25b705,0x0000000000000001}, {0x000eabc2b52d4646,0x0006c95bf22f084b,0x000314aed2f78fee,0x000dd61877362fd0,0x0009044697d005db,0x0001d56d41727a4e,0x000627e07ab413dd,0x0000b5903e3cd67c,0x0009c224e9a02779,0x000806739d1d3428,0x0000000000000000}}, + {{0x0004109673b0becb,0x00029788f9eb9435,0x000ae5dfb3d20da6,0x00057d88527623e5,0x00015c844e99c175,0x0006a57ec3b40311,0x0000b594aff5aa0b,0x000cea2e84adbe7d,0x0007fcbaf1e84a37,0x0008e3048c293594,0x0000000000000001}, {0x000f58bede275de5,0x000b21503171b093,0x0007b8e1c737aaa2,0x000dfceb6261f263,0x000d21e8e8701620,0x000e453d37db241d,0x000825774e79c85c,0x00066f92bc717db8,0x000b9d9ff5a2566e,0x000bfd4ae0bd7b6f,0x0000000000000001}}, +}, +{/* digit=22 [{1,2,3,..,}]*([2^88]*G) */ + {{0x000cd0f9ef7a9efa,0x0001757bbdb01042,0x0001996a380a3fbb,0x0009e651f39db731,0x000cdf3f08146bb6,0x000679b0b6ec6679,0x000ae26608474788,0x000d883e5a5990d1,0x00061924fa5e209e,0x0001de3c755c0bba,0x0000000000000000}, {0x0007c1f82ebae92f,0x000ccf8ace9c6a84,0x000857d9026a1434,0x0005b0b7ad864d4c,0x00018f613e2210ee,0x0001165b2c86a357,0x00019fb55984d679,0x000f15401901080d,0x0009a0e8b3389ecc,0x000e608b509b98d9,0x0000000000000001}}, + {{0x000a526cf604c662,0x0003afa9ed0f7a0c,0x0001dd685ac125b3,0x0007ff8f516f5290,0x000ad927c416e17e,0x000fc77cc9720475,0x000071767e1e9190,0x000f6652fcb33aec,0x0000f0d48cb2653d,0x00086c8bf16720d8,0x0000000000000000}, {0x000404c440590fcd,0x0003377f43e4e408,0x000defb22729c42e,0x000241aec3b37e10,0x00092c795c866daf,0x000f4d30790a07c8,0x000075bb2425f5fb,0x0001b7cb5830a5db,0x000c950890875f16,0x000e6591cad66493,0x0000000000000000}}, + {{0x0003de8c91c31e47,0x000a2de46bc7eb7b,0x000f3de301a0b479,0x00094aa7479aa235,0x000be5191502fe1c,0x000266a8b4adef28,0x0002961e0e185a17,0x000f211c2c23ddea,0x0007d3314a5b8f85,0x0007d3d59f2b026d,0x0000000000000001}, {0x00032b1fd4f159bf,0x0003721f923e006e,0x00024a7d38b715f2,0x0000994620d23277,0x000ab6678b2917d4,0x000fa4a8245b0c3a,0x000912038e0b4172,0x000f6fdcf8c4b1b8,0x000ccd3b3eaaf19e,0x000d40f97e7249e4,0x0000000000000001}}, + {{0x0007975597dddacd,0x0009d9266f6975c3,0x000599f544c22dfb,0x000c2be6db081480,0x0002aeb8ec462839,0x0009d49cd3b5cdf1,0x0006a8ba917fb29d,0x0008233b216f9614,0x000abf1a9d936c0b,0x0007878c8a45a2f0,0x0000000000000000}, {0x0005dd64a0356029,0x0008c2d1625aef0f,0x0005ff56fc705652,0x000323cb9b293d67,0x000bd02b295cca5c,0x000c7129104d697c,0x000d83fe4eb4b02b,0x0004a4e9327cc1e4,0x000f46cdc9c23cdd,0x00053f946406995a,0x0000000000000001}}, + {{0x000d1e43a1e16a36,0x000514cd5be788b6,0x000f3827d2f55b7d,0x0000a3f8f594d05c,0x0006accb980d1448,0x0004ad1e4f1f6b59,0x000e7dd3016801fc,0x0007c435098ccf31,0x000c6bc2e61b9059,0x000588b12be241e3,0x0000000000000001}, {0x0000b8e2e65481b3,0x000aef410135fbc4,0x000c7a2e23221960,0x000fd4a513faa141,0x000a0357c6f569e3,0x000f4ac3eadff143,0x000f6174146f64bf,0x000c13f9a5506c59,0x0005ed7b5be845e0,0x000d0deb4721eef8,0x0000000000000001}}, + {{0x000e8dfde28d4a2d,0x0001761a2d5208a2,0x00005b6ff5ebb541,0x000b58d09bebbafe,0x000a9eccec1be769,0x0004b3933ce41d83,0x000e22e31e1a1a39,0x00081430e109c5df,0x0005c36dffc66dc3,0x0006a93ad68549c2,0x0000000000000001}, {0x000135d5a197f614,0x000b4beb4875f558,0x000374737f60f213,0x000b3391150b17e2,0x000fd51cce3f02fd,0x00090f492191217f,0x000bccbadc4f8a8a,0x0005d89eef2479ce,0x000b7145781a1c26,0x000e4d791a7d8953,0x0000000000000001}}, + {{0x0006a3d38b806bee,0x0002ffcbc7fbb018,0x0004904b1192a4cb,0x00039b7f571c01df,0x00040717c8035f6f,0x000f75f4a725c0f7,0x0003575b1f8427ca,0x000cefd7922924c3,0x000c3f91c6705a27,0x000f14804d6b5694,0x0000000000000001}, {0x000d6688c387623f,0x000533d1b2b7249a,0x0006a96c61952611,0x0009fa8adb7cd547,0x0006614cc5f19687,0x0000dc38a5b55739,0x000b43a1185de8e8,0x000489b0890de38d,0x000cfe16e6aaab3a,0x0009a02c0ab01066,0x0000000000000000}}, + {{0x00047414fc9de104,0x000b82aed9435d61,0x00062ff16a9bd16d,0x000caf4a3b07e71a,0x0003ff9456ee752d,0x000d78dd65ea0d3e,0x0005bf864901fef1,0x0007bc9f42253114,0x000cb13ee366fd36,0x000fe6290083f481,0x0000000000000001}, {0x00032088e6e77ebe,0x000f38c5e887c852,0x000f005e149cc7b3,0x00089874e1bede78,0x000c72dfeaf32e8c,0x000cb0a4d9cb4e28,0x000aba5da48c7113,0x000b1fe289a0af7d,0x0005d0dc00d3633a,0x000801c0b05c86bd,0x0000000000000000}}, +}, +{/* digit=23 [{1,2,3,..,}]*([2^92]*G) */ + {{0x000567a7a06de008,0x0007cf045155d807,0x000ce8bb24635169,0x000c2900cba64633,0x000c724297174dd5,0x000c3a3851e7f044,0x000f59548c830bf0,0x0003817a26f0d35f,0x0003d8b027d923f5,0x00062c2b3dd7cad9,0x0000000000000000}, {0x00094cbf6924bf9f,0x000c09986d299bcc,0x000f89ccb5adf6f5,0x00099f825aee26f4,0x00056c1b545bb186,0x0000d22aa56595e6,0x000ba5ea3953faeb,0x0003a9580b4b6abc,0x000465246d4e240d,0x000d8613b6fdf7ef,0x0000000000000000}}, + {{0x0008a64cd725af51,0x000d6dda66823389,0x0001f40d7d1c8516,0x000115e05fb1f564,0x000ad7906c2d8d5a,0x000f4efe00496ac4,0x0002d973643f7076,0x0004414f58386c89,0x0002d83c2e34b14c,0x000437c00d08bc7c,0x0000000000000000}, {0x0004451656bebe71,0x0003f2219e2e5bca,0x000118227eacbf3a,0x0007e2cef1a84019,0x0002d58a5f9de601,0x0001ecfa6e192212,0x0006fb198696eb0f,0x000854826be2d3df,0x000dea0068fefc08,0x00074e77c2979102,0x0000000000000001}}, + {{0x00048eb205a5b22a,0x0007d680f21ad148,0x0008e50bcbdde0fb,0x0006f6074c1119fd,0x00079f9f2e43583d,0x00065361f1d9961c,0x000463e625f26bb7,0x000f2b47c8db008a,0x0008c397787cd134,0x00029b36eea7ef32,0x0000000000000001}, {0x0006647223894ce2,0x00087adfe036fb3d,0x00067daf1eb206e8,0x000b19b372f017c4,0x000a8ad33a99ef7d,0x00055c0da806ea7b,0x000c02414bd637ef,0x000b598649739b12,0x000feed3dd282f3b,0x0003ecce69b37255,0x0000000000000001}}, + {{0x000e5f81ac137092,0x0000518f81e9a3bd,0x0009720014a7f4da,0x0001f457a02bf073,0x0006be074553e9ff,0x0004351eaa3a46ea,0x00022b27e32f0dd6,0x0003b488462fd22b,0x0006ddddacafc2c2,0x00035bfb75908f56,0x0000000000000001}, {0x000987332b5b9a11,0x00044ffe94dfd9e8,0x000f9b91bc64f63b,0x00077f430dbd772b,0x000dfd580392aecb,0x000ddc69fb2fb67d,0x000314d2fdb69c91,0x000e754b9b9f9ea2,0x000f2e9c2e624f23,0x0000a3c6e677e1f3,0x0000000000000001}}, + {{0x000e37a930c3bc57,0x00006f969b4cf1af,0x000df3ff74237c4c,0x000d1ea35c488b6e,0x000e084f47372a77,0x00065b61ccb75b2d,0x000a4e47577fd773,0x000dbd98a086f5f2,0x000533002b74aa9d,0x000bedfb9c8068e6,0x0000000000000001}, {0x000b7aec02da823b,0x0009baf08dcbb7e2,0x000400bf0e5f485f,0x0003a3c7f9dfd4e3,0x0007472ece43050d,0x00091a092b1134f2,0x0004c1a0e70ff8a8,0x00037f3b75a31807,0x0000f51f254e9906,0x000c2183d0367614,0x0000000000000001}}, + {{0x000a39c0dd80f6fe,0x000a13cb07adfd25,0x000ee3ca4d8c1a3a,0x0007a28212986a18,0x000c5167c4c47943,0x000beb7b7306e9bb,0x00038cfc9b26c1f3,0x000b68e5bc206604,0x0003bd9665b63ba8,0x000661b9d7de6c40,0x0000000000000001}, {0x000c9e6e7c3229ea,0x000fb93cb721672d,0x0002429ae120fbe8,0x0007d3de1f64e3d5,0x000492888bee099a,0x0009c2f4894dc1d6,0x00011bda9ef3c7dd,0x000289559ae47758,0x0008fcbb34df4ae3,0x0005c440c5d0ce22,0x0000000000000001}}, + {{0x000fc1fd2377e231,0x000257cc9235e4c6,0x000a8f0482692e0c,0x00027d24355728b1,0x00087856d3663645,0x000dfe4f7c7f19ee,0x000c0f9ccd7eb847,0x0001019511788244,0x000c4838c54b2a98,0x0008b89d2d46944a,0x0000000000000001}, {0x000e02a1a1d2a70d,0x000b7d63b4d27dff,0x000ab61d5fa9a970,0x000d7730420f7a7f,0x00079f4479996041,0x0001189500cc1597,0x000981b342dd1a18,0x000b0c87b154435a,0x000fe5a645fb9438,0x000a1ab34fb7fc43,0x0000000000000001}}, + {{0x000f438f4b100160,0x000f22d13ff0c314,0x000d55796ecb8e45,0x0000ab873dd2e2bb,0x000789eb71d33f83,0x0003167e0b14a364,0x00002c246513aa48,0x000b03e86d3a7935,0x000b2db2bb0fe98c,0x0000d70404a0ec4e,0x0000000000000001}, {0x0004384c5c6b60bc,0x0002570cd19a5c8f,0x0001c33b468c19b3,0x000cbac39210942f,0x0007d36048a2a29c,0x000f69ef5fd4ffa9,0x000ece5cd6b0a674,0x000b1322973982d0,0x0001493bd4bce1b8,0x00083d4d6596bf49,0x0000000000000001}}, +}, +{/* digit=24 [{1,2,3,..,}]*([2^96]*G) */ + {{0x0000db5e813acae0,0x0005831117f6d456,0x0001106059c8e19b,0x000f908ce8232c57,0x00092d0f09782c78,0x000bd0fcb64a24aa,0x00077e3d766becf8,0x000f155f53d2f599,0x000389ae2fa9a727,0x000920ff877e9249,0x0000000000000001}, {0x00085d510d2d4458,0x000dc73b4e520499,0x000aa68342be4788,0x0004f89c8a0ca8e6,0x000e8668748927b1,0x00017375ddf19eb3,0x00041d15e5f8b7ce,0x00052912af54652a,0x000b9a777a86a727,0x00003bbaf706d85a,0x0000000000000001}}, + {{0x00017d39542a7b35,0x000d6953d2cbb145,0x00044a3ef5b5b733,0x00076565472126ff,0x000b2a4a1334dee0,0x0002573d17b26c37,0x0002c7ab5b295171,0x0008d328148c129c,0x000907f5aa2c72b0,0x00018b1d10e10308,0x0000000000000001}, {0x000159666154b57d,0x0005dd9359d8885e,0x0000281b6f14827d,0x0009bc4ba475f3a4,0x000c32eef44696b1,0x00082b50dbdc6dfb,0x000236a9ef4383e7,0x000fd73208450583,0x000d190b07767db3,0x0003153c0278a00d,0x0000000000000000}}, + {{0x000ba905f5456251,0x0008ee5b3d8d39c0,0x000ea2a0a4462a26,0x000032f3094457cb,0x0009bab36ceff80f,0x0001b0fdf3879073,0x0009dc840209bce2,0x000df0c1c8e03824,0x000c51d81213ecb4,0x00063c2b025e0d70,0x0000000000000001}, {0x0003bb32c4b899f8,0x000ccb798bfbf249,0x00028838277f622f,0x000e5b67c2594827,0x0003c2c07c4dd5cb,0x000c19526a2c4c70,0x0000177dcd0df4c1,0x000457a743a1ed39,0x00032bea63a4c527,0x00075d1c302e78ac,0x0000000000000000}}, + {{0x000fc5003828ff0b,0x000436a9841f4323,0x000c6f35f8a62407,0x0009286efc260a1f,0x000fbe74c4b2df5e,0x000cb3568b504bfa,0x000dbcf3548e5047,0x0005d92aaad71af9,0x00018241085e423c,0x0004f894d1d8842d,0x0000000000000001}, {0x00075b2a3f29b75d,0x000ec555f7834899,0x00092b31a410939e,0x000b7bc223255263,0x000db65a25c264a1,0x0008fc1aed283464,0x0005c70ecd1a9b70,0x000d90a7a2a0ea33,0x000d21f2e9f14ffd,0x000fb49566dadd7f,0x0000000000000000}}, + {{0x000fa5757545376f,0x000ae164cbfd55ff,0x0008a8545451f1c3,0x0002e007d0be9705,0x000ed2a8f4c49727,0x00097ed736254138,0x000516215e864c7c,0x000bb624fc1b83df,0x0000313aaf4114fd,0x000a7a8c7f0423cc,0x0000000000000000}, {0x000ed76abc8d276d,0x000bfe3e74f599a4,0x00025d1f92d8b381,0x0005a3599e406956,0x00071869bdf5e06a,0x000ec86f625afaf6,0x000d724bbcc12cda,0x0003da7516890dd1,0x0009b69252163060,0x0002541f15a18b40,0x0000000000000001}}, + {{0x000e1c4e632e5d61,0x0002987ad659b7d5,0x0003f7f338de2f2a,0x000b55a5aaeb06f1,0x000b9a60e84f26d9,0x000d10563130c6f8,0x000e760d017d58e9,0x00039e20b973fa41,0x0000eaafdb2f4acf,0x000501ec9c6ab584,0x0000000000000000}, {0x000f4549ba5a6302,0x000a98b140b89722,0x0003e099225c2510,0x000f31b19117bbe6,0x00046ba7147bd18a,0x0000f540e368bb5c,0x000eacf29d33114f,0x0007e59588a01c9a,0x000ef0e25eb2d0e6,0x00058e4bb1b8d029,0x0000000000000001}}, + {{0x00057915cb5440b5,0x00088e89a3e1eb04,0x000ed12c670e08cc,0x000eab1d89133ab9,0x000f615d9bc0c1fa,0x00081504d63c4250,0x00062cd084c8e8f8,0x000aaf76dbe53ead,0x000bf1dcc49cfac6,0x000fc7007ea0b885,0x0000000000000001}, {0x000472352fc50515,0x000fa2123835c747,0x00067bab29e80692,0x000cca008379c2a8,0x000799065aafbc2e,0x000a605d2e32da97,0x0002283421bbbfbd,0x000dbdc2e1151243,0x0007a9d899c126b9,0x0007467ce3f8d643,0x0000000000000000}}, + {{0x00000c97f4a1ee70,0x0000a55fa6cdb3e0,0x000fd5fcd60595ed,0x000522bda02a23c6,0x0000361844a1d76e,0x000c6c179ebaf8c0,0x000a6ccd0a47af40,0x000071e2a1156aa1,0x000a1b0fc4eb0062,0x000bb4c1c5314a2c,0x0000000000000000}, {0x000c048376702b16,0x0002ef5b4e8123cd,0x000a7d67834242a3,0x0003bc3accb0fead,0x00007e65ed32fc2a,0x000b8b44e6e71194,0x00077e9aeb1712aa,0x000039ce4f895a59,0x0009d43ed708cfeb,0x0004254957cd1ca1,0x0000000000000001}}, +}, +{/* digit=25 [{1,2,3,..,}]*([2^100]*G) */ + {{0x0009a3fb62f03e91,0x000fc3cfe3b9a1c2,0x0008c5a528bca033,0x00096cd7b4bc3e3f,0x0008c4bd134e2233,0x00065224c739c3eb,0x00071ec25548c0a5,0x0007b0fb17f6f014,0x000aee1015fc6579,0x000448c4d69b6d18,0x0000000000000001}, {0x00074be708f600f9,0x000042a14b550a00,0x000f8e6b95a52425,0x0004e9813f438c42,0x0005181004aa1017,0x00010cd9a834ae43,0x0002105b1b67e295,0x000141438bad8cdf,0x0004d11308ec5ba9,0x00034300e8c08dc6,0x0000000000000000}}, + {{0x000625d111480c24,0x00034cdcf3505fb2,0x000c306874b99629,0x00004410981e8fcd,0x000492bd0a650027,0x000a534a84249eb3,0x000e1326b6bb40b6,0x000ebe5d29140c32,0x0009956b2cb2ca52,0x000b8c77c7225102,0x0000000000000001}, {0x0002b4e077c5c4dd,0x0008846314442efe,0x00066618e794431d,0x000a933fcd3eeea2,0x00006644159656a5,0x00022dc52fbda24f,0x000542f82f45dda5,0x000e0e5075c9d412,0x0002aba0fff34a66,0x000d69512c4a1d9a,0x0000000000000001}}, + {{0x000dc5b949f6aa55,0x000cb79872016ba3,0x0001df5e18d2889c,0x000aebf5e0129254,0x0003a4cd20b4cdbc,0x000f30108963d6c3,0x000c0dbc46a1dad1,0x00042c0e39b6755f,0x00007fa126ef9e69,0x000805d500d36fac,0x0000000000000000}, {0x0000b5e7bd19e5fb,0x000b3765e8dbbff9,0x000e491cc2deb8ec,0x000ab995d314c068,0x000b4e810513ad31,0x000b50dc0fcca181,0x00029580c1e05269,0x0006b6453c858930,0x0009a98b2de5a7d2,0x000b386f7a77183c,0x0000000000000000}}, + {{0x000d861fc542368a,0x000a737b3c184cd7,0x00014a6e3b95c425,0x000f514e85d4a651,0x00098b665bb45532,0x00066a39b08b87e5,0x00008dbdbcbbabba,0x000ba64b561fa462,0x0005692509520864,0x000281de8b31e205,0x0000000000000001}, {0x0001bb6a74473c21,0x000932e76a8c5ddd,0x000c6ee633cc0f66,0x000f68d0c546bb80,0x00006828f4e0c911,0x000b2a4276c213a2,0x0008cb204a16b2ce,0x000d38c09aa1cbe9,0x000f3ebeebcc1671,0x00032c7a684ba9a6,0x0000000000000000}}, + {{0x000a3463989cd762,0x00035114b160b22c,0x00057f2d520e3cc4,0x0000ff788707011b,0x00068b1a346a61d3,0x00084618b8d69eda,0x00020c04008115fa,0x000dfeeecaa806f5,0x0007e08436a14e30,0x00005f68bc839ccc,0x0000000000000000}, {0x000ae58e3c998f3f,0x000951d35d5af6b3,0x00038625415f29bb,0x000fd087552cd755,0x0002087ef7e8ab49,0x0006b067b5de9ebd,0x0001e74110309c17,0x0007b224505a1ece,0x000ba962991a5a2d,0x000b8879263dad03,0x0000000000000000}}, + {{0x0001b7e0189fcda6,0x0008775ba885f2a7,0x000b98305b9915b6,0x00019b2753769a90,0x000638d87ac0d10c,0x00083c77c18f7acf,0x000d23964d02af25,0x000de5be92026e04,0x0005a30998f85294,0x0002d2bb22f8803a,0x0000000000000001}, {0x0000daae09876e93,0x0007b9f1b9b10415,0x000e48eb13c50096,0x000cf9ccec3e5c4d,0x000f7b6158629895,0x000aa201ea7d90f3,0x0006e88c0cda29f8,0x000f4c0d70150c9a,0x000ee70bc97d1c62,0x000b8e4fd0f68d56,0x0000000000000000}}, + {{0x0003edbb844d6c8a,0x00076a792ccd3b41,0x00072527a7c1564f,0x00055b682778d6f2,0x0002167ba3cee45b,0x000d96d43a6e138f,0x000806538c932f15,0x000d4892afee6363,0x0002b82f06ed7c45,0x000fee287b4614b8,0x0000000000000000}, {0x000953f4fc1bb9d4,0x000e995150d18cb0,0x00067e23c2e107a5,0x000bfba071a733f6,0x00008ca46066c2e8,0x000cfb49871d6c61,0x0004ece39bb5a648,0x00040cf34f514816,0x0009b9250336996f,0x000c69d6e08146e9,0x0000000000000001}}, + {{0x00008e517921a752,0x000ab87a6c13d140,0x000c4597b07c5d69,0x00074a68c66db12e,0x00019ca40dec9dbd,0x000a617fff4579d7,0x0005876131725395,0x000d09e3b946e383,0x000d20c852478942,0x00087982ecbef742,0x0000000000000001}, {0x000589886da1602b,0x000c3fc9ae2bbd5f,0x0002126ee978ba22,0x00075595e212b5ab,0x000a2389b739eff8,0x00063595af9d6707,0x0000f9787d12dd72,0x0003b014c3309267,0x0002f506a0067880,0x000f67060764da69,0x0000000000000000}}, +}, +{/* digit=26 [{1,2,3,..,}]*([2^104]*G) */ + {{0x000365108b90f9d0,0x000f80765f67fb1e,0x000b1b38d141aae9,0x00024d697a9407e4,0x0003f9693e7ccc84,0x000a50e7d291d93e,0x000cd34385c13c5b,0x00066fcf73c9d94e,0x000598f4a80eb0bb,0x000b721c4d4c290d,0x0000000000000001}, {0x0000fb9a3bbeb3c7,0x0007ba326d546e3b,0x000a848cf094c6d2,0x0000e41609d2dc18,0x000266f0069ca46c,0x000c4aef799231b9,0x000abacbdbead081,0x000b272ba1959d4d,0x00049b7208e216ce,0x0002ba83cc03eccc,0x0000000000000001}}, + {{0x0004c0998b5250d8,0x000c867c43b599d6,0x0004c9f6ac785a2e,0x0004ec8b59f29f0d,0x0004df16ae8c0faa,0x000b8d8f78201760,0x000bc38bb59089da,0x0007384039822772,0x000d1571c6e88e81,0x000a7b7d4e8e0cf3,0x0000000000000001}, {0x0007bc572ea0f919,0x00064539b5eb1047,0x00077d71bc88d22a,0x0004dc62d769223e,0x000adfe2b562c973,0x000173fab241cdb0,0x000603370ddf3ff3,0x000f70dbbbbd997d,0x0008a88a56d59561,0x0004be64aafc3299,0x0000000000000000}}, + {{0x00045d30a00d827f,0x0006bbe20955e997,0x000cead0ee1643eb,0x000fce2db7c5ac6f,0x000ad1d2efb30bc2,0x000d9fbc667affb8,0x000bb6fb3983d3ef,0x000f70a834538091,0x0008384fd172dade,0x0003f32390fa3030,0x0000000000000000}, {0x00022aae53d2000c,0x000caf0273baad84,0x0004a6a878c712a8,0x00002615beecdb4d,0x000b0e868dba85ad,0x000964d03b5c6537,0x000cbcf8b6d0fd98,0x0006a16ca9f74a32,0x000552ffb8c5dbeb,0x000bf0f20d682f2c,0x0000000000000000}}, + {{0x0009a64c8deb9f4b,0x000532674c0fe944,0x00001e88fe681603,0x000b8697595c6e13,0x0008cf6f513d4913,0x0008c1e3203b6d47,0x000b68db28573518,0x000bdfb9fd4390cf,0x0006601496c4bb93,0x0000633f388af7cc,0x0000000000000001}, {0x0005258fb2317523,0x00040dacae0a8b9a,0x000ba0560abb741a,0x0008bc6a795d005e,0x00096caa47999397,0x000ff04fef1c0b24,0x000b0926ddcefe71,0x0008f281ff3947c3,0x000027cc7cc93f3d,0x000e78773c9a3f23,0x0000000000000000}}, + {{0x0001c60cc85e7257,0x00011fa52f42f768,0x0002f380ec8fe41b,0x0004893bc07fb9d3,0x00034451f49fbf7a,0x000388023d1b4705,0x00079c42053de8b9,0x000cf909c04d42cb,0x000eca57f6797b3a,0x000cf34d35027c43,0x0000000000000001}, {0x00086008351763ff,0x000f9e1103300427,0x000991707dcb32e9,0x000efb21a41fbb4f,0x000b5649d55978db,0x00004e4d7b3fa6d3,0x00012196968dee3a,0x0003561c60989b74,0x0009c91c53ee540c,0x000b9823a2ee0db7,0x0000000000000000}}, + {{0x000f6a15601d1f8d,0x000406c4591dc921,0x000b36c8aaaf7c15,0x000834fd3b0d0813,0x000e544ef9e76287,0x0002fb609294a18c,0x00008d9bd0198775,0x000cd4816092b24d,0x0008b097d39d2d32,0x000b3a5b9f00f218,0x0000000000000000}, {0x000da9d6f0979e9d,0x00080741dad104cc,0x0004ee619b7637d2,0x000d71560f5a9cc8,0x000b897bb554b4f3,0x000890a210367054,0x000aff63f1f61c3e,0x000cb92963c20784,0x0009317afc9acc43,0x0005c1dadb0d3e30,0x0000000000000001}}, + {{0x0004debc7af95096,0x0002bf8bc26e7cbc,0x000070bd1c43e146,0x00017181a62e8acb,0x0001c46b5339f3b4,0x000a2c14c9b646ef,0x0005de576b9f4c81,0x0004f9a155e97645,0x0003cfbba219f7e3,0x0009fb1fd4f92031,0x0000000000000001}, {0x000bcf11449769d9,0x000ea5acc7cb996d,0x0006bdb653213f43,0x0003a2cf9c396c5f,0x000777fb3075bc28,0x0009fab46fa97f51,0x000d80cd600dc1e5,0x0008ba6a57a9d490,0x000c9975b8111175,0x00067a2ad8270658,0x0000000000000000}}, + {{0x0005e67b8d52ab83,0x000eb2049665d86d,0x000b56e1ced19993,0x0009c1fc7a62ba87,0x000276fc5cf75dfb,0x00054f5dad4712b6,0x00089fbe0548bd15,0x000d1ee24125ecba,0x000176a53fa18f5a,0x0004e18796b5267e,0x0000000000000000}, {0x000a0f1a17a9eb45,0x000584e4e5f968ad,0x0008e12a3e089107,0x0009c73cd6a2ba69,0x00002e23b2a1f1ee,0x00028e9adc43a76e,0x00062c6e3d7526f4,0x000d0557ab8af09d,0x00058b1d337cd537,0x000000e54434b827,0x0000000000000000}}, +}, +{/* digit=27 [{1,2,3,..,}]*([2^108]*G) */ + {{0x00036c3a6abc7042,0x00004a11953f80fe,0x0000b4cc57cd15f7,0x000df44d3d3a8bb5,0x00015b5099398347,0x00081f3a553789e2,0x000d0115f2bce30d,0x00090b7f91f0853e,0x0007ec29320d53ac,0x00085b63e7bfbe8d,0x0000000000000000}, {0x000cdcd80232c6da,0x000d8fc636cf5e56,0x0006e4c3d9621241,0x000b84a86812f9d5,0x000287741d3de81f,0x000ab3d77eb50a79,0x00048627cc80386b,0x000a1901afee8f37,0x00095591fbf5ceb2,0x00080aed0c8140dd,0x0000000000000000}}, + {{0x0002fcdd07efc38c,0x000c9ad8b1dbd166,0x000af6b6e1566c06,0x0005c4ad28998a9b,0x000b12d2005dbca4,0x00009acb17fcd947,0x000af2e6bf7b35f6,0x0000b8a8aba325eb,0x000302e3f599df52,0x00080d2bf9b973e4,0x0000000000000001}, {0x000aebd112a3c0c1,0x000c408868630c25,0x000af7c4f6ba5529,0x000d49e0f5657b1a,0x000da3fa70b84c0f,0x0009f530044d86ec,0x0003f7ec59dce6d3,0x0004bdaf73433951,0x00022dd61822c292,0x000466acb0786ece,0x0000000000000000}}, + {{0x000bab3d6168fc90,0x0007cbb6b2be9857,0x0001cf2ccb017656,0x0001630304c6ccfa,0x000ab9344dbdd28a,0x00075f0fec3093b4,0x000ff2494e7a4029,0x000ca3bad1c9c568,0x000b17aa4b7bc2e4,0x0001be8844e06c12,0x0000000000000001}, {0x000748f359f51efa,0x0006b016fa4643bf,0x000de784a5c3f1e9,0x0006412db1ab3996,0x0002a042b01eeae2,0x0005a0d3a2a424ea,0x0009b9a7aeefe348,0x000b209614c9d52b,0x000556ff1e6afd00,0x0001e2290dec8fe5,0x0000000000000000}}, + {{0x000dedb27f20e8c4,0x000f535a0fc33855,0x000788ccd8803e8a,0x0001f7d6e10cabd0,0x000355f889d7fa1f,0x000583e3030487ee,0x000f3dd1885d800a,0x000f09ae9a4a2fc9,0x00054fc302887b5b,0x000678d91181d3a5,0x0000000000000001}, {0x000b146d6cdca631,0x00046652f280d553,0x000e0b73d63dfaac,0x000399cd0d77869d,0x00057ba5ffe6aa8a,0x000ffc1da65c61b7,0x000738771cf6c9ea,0x000620ae124834d2,0x0006504def184b95,0x000d761c974cb47f,0x0000000000000000}}, + {{0x000b12824a32cfa1,0x0009c552049f6e5b,0x0002cf47aab27c7f,0x000c49affe61e35a,0x0002247309f28abd,0x000d8f5d3d157087,0x000c6b864fe18c22,0x000addbbfad216e7,0x0006e83c0b6f0f26,0x000fd5df730de2cf,0x0000000000000000}, {0x000d5ed49aa6d720,0x0003b8bf56601c9e,0x000932a05f265897,0x000b6e7997e501b5,0x000a12ea7ac82871,0x0009ee973ffc91a5,0x0008bc4a89b5c6c1,0x0003ff7cc4ea3cde,0x000dd04f7b689969,0x000d1dbe61024f53,0x0000000000000001}}, + {{0x0009c8cbe80fbbcf,0x0001820704f59919,0x000a2837d2432345,0x0001d7c57c052221,0x0004dd7abcf6f7fc,0x000ceb0c9ac384fa,0x000cbcbae19fcf70,0x00044465c2e5cd22,0x000a11ffc8ca98c5,0x0007b72231b3018f,0x0000000000000001}, {0x0005516f6500002e,0x00015b8dc7f05ee2,0x000111229aa6356e,0x0006bd0e54525c30,0x000dcfc5ecb4c0df,0x000355785f844693,0x000ab74792af79c9,0x0007357c34213765,0x00008a7b5867734f,0x00069a6d820a3083,0x0000000000000000}}, + {{0x000b3a34b0a583e8,0x0004109d1c2a05b3,0x00056d185c8227f4,0x000c30eb0f8b309e,0x000e5f0d836e7213,0x0002fdae4ffef15c,0x000257bcdd2d7309,0x000dad1ff9a8d757,0x00098efcc192b734,0x000fe0e5b46f1d9f,0x0000000000000001}, {0x000ec3057c0b1268,0x000e1eea4fea07c8,0x000ac5ab201c4cbc,0x0001b75086654bcd,0x000e612c2aee9474,0x00065fa077070bf8,0x00015dd97afeaab6,0x00088802f9979ef0,0x000c9b3109eb556d,0x000e513d135fae08,0x0000000000000000}}, + {{0x0009b9b53c1efab4,0x000babd37156ff65,0x000a115d2c7f8338,0x0007371c9d1175b5,0x000a353c22d6aa92,0x00079ee37be5b07d,0x00020293585421cb,0x000abe2b0a938ac9,0x0003622f3d489e47,0x000ac9ccd5811b36,0x0000000000000000}, {0x000cb54f0f506ac3,0x000feebf83fb7441,0x0007d9fa2d5527b4,0x000b40376d4a3597,0x00045e4619c87f8a,0x000b913b27d590e9,0x0001da0e8861075a,0x000dd8fb707f389b,0x000140b6fe0beb49,0x000bf7392dd17235,0x0000000000000001}}, +}, +{/* digit=28 [{1,2,3,..,}]*([2^112]*G) */ + {{0x00082a01cdf12457,0x000bf550e3442670,0x00027cfd7b191616,0x0009bf54426bd9ae,0x000375f468d0ec29,0x00095e63540487ca,0x000f558b93aa7dc6,0x00028f48edec9322,0x0007ee742818f059,0x000d23aca5b08895,0x0000000000000001}, {0x00018972085008e4,0x0009e445a0130711,0x0005bf246e5348cb,0x0008ccf1f5c183c6,0x000f2e9a40aeb3fd,0x00087abdef0fbda6,0x00050f5daf09cee0,0x0003e33344ee90c4,0x0004044243abe107,0x000b8f02a065d1a3,0x0000000000000000}}, + {{0x000b66d418145236,0x0007896ea70c3fff,0x000cb17d54fc5491,0x00042a641eaf6e4d,0x00096b15be10c7c6,0x00011efe5f993282,0x000c04930829e9c6,0x000a5f18e8613cde,0x0007985a51a7c38d,0x000b8f3536d908ab,0x0000000000000001}, {0x000ce50b447f989e,0x0006725435f6e48e,0x00060505d7413d04,0x00051fa907efc4e5,0x00091cc601ad28a5,0x000eeaf4b18fed33,0x0002e1a4338a8549,0x000b61868d3372c5,0x0003a511bce70bb6,0x000e1f5c8d75eb9c,0x0000000000000000}}, + {{0x000d68d2a3e7e05d,0x0009bf6f65a63322,0x000368db479d7794,0x000e22f5738f46ed,0x000947212d465e52,0x000bb783e24758d1,0x0009d33d677a59c8,0x0005609046041b23,0x000f6497a9c2f277,0x000e7a9be5339a8d,0x0000000000000001}, {0x000804d780847503,0x000fb6bd5cd190b5,0x000d58769b6bfbeb,0x000a5b2366d25685,0x00084206ac283f9e,0x00045e93a909d14a,0x0007818e03b612f8,0x0005061fa312c680,0x000501efdeb98070,0x00043cfa3670b66b,0x0000000000000000}}, + {{0x0008296f30c9bed5,0x00099a4bb11c1f32,0x00015b4084960501,0x000c50ce53a7ca7c,0x000f50a2c1da281b,0x0002c0e34f682873,0x000f21f441021705,0x000e9f354fbc9c5e,0x000d7990a0bba954,0x000ca402432a326c,0x0000000000000001}, {0x000e6dddd976d76d,0x000a57e55cac7b2b,0x000da37392c8a3b8,0x000fecd4ec1dc93e,0x00009cf4f78c92e3,0x000ff689fefedf3f,0x000abd5033740521,0x0000df4087ca092d,0x00002763eb9e4e11,0x000689f3f329b79d,0x0000000000000000}}, + {{0x000d09236f94e981,0x000f3b87c2492089,0x000090559806590f,0x0006b638cb962e83,0x00043caf0b4cf40f,0x00066f61c7a2faca,0x000edaa5c1d246d3,0x000b19d9f9d7b0bc,0x000d3b97c04cafce,0x000f23bf9f27399e,0x0000000000000000}, {0x000b2a804eb31689,0x0009a2a7437938a7,0x000f3c9cd990c2d5,0x000b68b7f843ec1e,0x0005141dcc07de04,0x000fdd1e7d56eecb,0x000a099220b61db4,0x0004184b463b9f76,0x00076a6a6fd2bcb4,0x00067d87f98ad9c6,0x0000000000000000}}, + {{0x00025736286ff497,0x0002401f1271e328,0x000981dcd7607683,0x000cfb7ab6548084,0x000df4da03d9c990,0x000d822ebf4ae8a1,0x00099db5f23d4a99,0x000f6315e3fb9894,0x000021f262e80d80,0x00027457b91d5ecd,0x0000000000000000}, {0x0007a0ed90d98205,0x000cf6f6b6c517c8,0x00057c5ae11bc6b8,0x00000efaef845be3,0x0005e74813e365be,0x000c8e0106700aae,0x000d776c99ef5ca6,0x0001e3d37bff71e1,0x0003a89779b96572,0x000c15ea2a31bd25,0x0000000000000000}}, + {{0x0000092959940ede,0x000bb4c7bf2ceef4,0x000d604555c30636,0x0003fe5635d8d5c1,0x0008cdef0cb902ce,0x000b6f22235732b0,0x000fddf6f7cfc6cb,0x000369004df1a446,0x0004ba65bcff1dc7,0x00020adfd4756b77,0x0000000000000001}, {0x000afeaa9dfdfa89,0x0000b80d92606c1c,0x0006888e19c65eb3,0x000bf9bb4db51900,0x0008e84280b5d191,0x00077a1cb4c3d007,0x00055fc83156ea4d,0x0009f40279edea0b,0x000aa5223480bee2,0x000f264b4f70afa9,0x0000000000000001}}, + {{0x00054382d016c8d9,0x000ec7826f7b17bd,0x000dce64f2832c36,0x000193ae22a16680,0x0000aaf6a85c2ab2,0x000f20270252cc0a,0x000f317cc1335b32,0x00003743776e2afb,0x000a199000deb474,0x0006bc61591f25f9,0x0000000000000001}, {0x00084eebf2800729,0x000608b06a4eb61d,0x000b23e73968bb72,0x000a3ae82e886104,0x000d27c8605d2992,0x00033bec6e418a91,0x00029d45f2b49e6e,0x000bd1f4a3f4a9d8,0x000bc4ceaeb2f044,0x000c63b1ef09fa28,0x0000000000000001}}, +}, +{/* digit=29 [{1,2,3,..,}]*([2^116]*G) */ + {{0x000826845611f97e,0x00015b6b1ee54a04,0x000608b1dc092400,0x0009050925698b8a,0x00031b5e532ada13,0x0000c41c46df4acb,0x00090c116e05bee3,0x0009642c127307d1,0x000365a48b566eca,0x00046d5c3cffa21b,0x0000000000000000}, {0x000b8836b9754189,0x00079ea005768621,0x0007bf51510520f5,0x000bbc0ca43d38cb,0x000c9fe21891a0a4,0x000242b093687446,0x0006d618feab8811,0x00047a921f31cacb,0x000cb09d3cf611aa,0x0007d8fef9a8efc5,0x0000000000000001}}, + {{0x00042c81af50749b,0x00081540e019366d,0x000d6072e7b7487e,0x0004156c32da913c,0x0003df1e87478e7b,0x000880f5ccb21742,0x0002347ca344dd54,0x000d15da2c269018,0x000993e2887d2337,0x000379604cc23f8d,0x0000000000000001}, {0x000778d40c2ec9c0,0x00027ec9dd1808f9,0x000dcd7b63f43450,0x000cf65f198a63ab,0x000d3a7a4c38803b,0x000476f99f1130c2,0x000d6b9c1ea5019b,0x00034f67377e991a,0x0009047dfa9f5ad1,0x0005dc80641e2fd9,0x0000000000000001}}, + {{0x000de0d9dfa7af75,0x000eba7ea5710283,0x000dd5435234652c,0x0006f821b8a36856,0x000c319e00261b58,0x000ed079e56ce309,0x0001099e0f75ac31,0x000e2442020d51ff,0x0008b83fa0c077ae,0x000e6fc85e1f8724,0x0000000000000001}, {0x000872b798445d10,0x00032b311d3108af,0x0005040c97d2ca2a,0x0005703d4fa4c2f0,0x0006980d5eb27761,0x0005f074a536c8c1,0x000181395daa1e3b,0x000b672dad89bda9,0x0001f3d94395bd4f,0x0004b4c4a2c81ef6,0x0000000000000001}}, + {{0x000924e7856ff846,0x000ee4f61f664ccb,0x0005ac67cb02e404,0x00050da76b002de5,0x000c4537e3c3c875,0x000c36c052b6b43f,0x000b204b2d5ce01c,0x00088e7f6d0e0c5b,0x000c188bbf930fde,0x000168056f87d909,0x0000000000000001}, {0x0001106b668bd3a0,0x0008dce76203aabd,0x00002fff3110182e,0x000f7d1e1307d3fa,0x000347101339296b,0x0004a22e456ed2ca,0x0002d011b668eed2,0x000b79cf95e5e410,0x0006693b0681d10c,0x000355f94e08ac6c,0x0000000000000000}}, + {{0x000323242b9413a5,0x000d8925b57cdbcc,0x0004d31e6960afac,0x000cc1c8075e88b1,0x0003a4d853d5880e,0x000c2d17b4e21339,0x000c35a1d02b3405,0x000f7f4eb22a29f6,0x0001b6570763f945,0x00008a38d9e91699,0x0000000000000000}, {0x0009e262a8faf74b,0x000d89cdb707d091,0x000c28362e27b3cc,0x0000a8d2e31adec3,0x0004f2e5340b0d97,0x00076d44ac11f1ff,0x000ddee42bd388ab,0x00052165e718528c,0x000c2384a7cb055f,0x000e3bd81cae87a8,0x0000000000000000}}, + {{0x000e0ef29067ec5b,0x000b3efb1759268e,0x00093d33d241c82d,0x0005ebc6b912da50,0x00004cea7d557bb1,0x000a95c0c2531329,0x000338d1728bce52,0x00023e934774d703,0x000bdaa179ff6232,0x0009c05a25267ea4,0x0000000000000000}, {0x000b3f1bf490cbf7,0x000ec049cf21d24d,0x0001567c730a18c0,0x0008c3e0f359d391,0x0004ea1bf7eca8f7,0x000252d4d89f9aa6,0x0007a2e5b2ffd6d4,0x000e70d5197d3cf7,0x000ac046e420f1fd,0x000f82fbaabfd6c4,0x0000000000000000}}, + {{0x00023d833bf81b28,0x000064787960d20f,0x0001e23da2c1a82d,0x000fca0df31fd1ab,0x0000d67beaa32632,0x0009e45d2648f548,0x000d563bb162f9bb,0x000110e02089d434,0x0007082d3a10eef0,0x000cb7b7735d1d64,0x0000000000000000}, {0x000d95b89701e6ec,0x0003bbe61d29d940,0x0001c7d5b4e68b4d,0x00012a5ad78df4bc,0x00047d83302cabd6,0x00011140b2809827,0x000211a754f62795,0x00041d43610e16e7,0x00099e665f0dec95,0x000ee6baca9f0f39,0x0000000000000001}}, + {{0x000657257b2d9252,0x000915208861aaee,0x0008adfc02b5d4bf,0x000f78398b2a8792,0x0002cd1929e3951b,0x0001878fc66ac2d8,0x000a1972453f26a5,0x000c0ebd963c67c6,0x0006feb8829e6f9c,0x000c986a8aecc7ab,0x0000000000000000}, {0x00030636d8df74f1,0x00011de6a5beb09f,0x000247b37675f6af,0x0003d122a04301fc,0x0003f577167d7789,0x000a69addd4d974f,0x000f8be983fc60de,0x0003627055a8d35b,0x000c83aaf95c80a8,0x000a9aa21f06b151,0x0000000000000000}}, +}, +{/* digit=30 [{1,2,3,..,}]*([2^120]*G) */ + {{0x000c1e136664d27c,0x000e853cf04eac1d,0x000599f98901c4f5,0x000f0e3ecbc44867,0x000ee5a12a7f834f,0x000066c152851c12,0x0002df97ca61be6f,0x00027153da2c7383,0x0003e882e14acdbe,0x00030b87567338b7,0x0000000000000001}, {0x000fe8148de5b00a,0x0003a405fd56d3d1,0x000e986a7db49ee5,0x000cf7bc11101981,0x000a9750760e2695,0x000815cb90b6aca2,0x0009f299f5ace2a4,0x000d6b06b61bc3dc,0x0002e5c223b28698,0x000c0b5687880a6b,0x0000000000000001}}, + {{0x000f552c0e5d59cd,0x00029aaaadcddf1a,0x000f071e91a160c3,0x000bbaf777f33e93,0x000696e836178f9c,0x00030ecc6d74f3bc,0x0002571349ec6474,0x000dbbec63ff9e68,0x000eff8b43f624e0,0x0008000d19e23a64,0x0000000000000000}, {0x00060d53484cb54f,0x0000d83eff3832ce,0x00012f600dae89d0,0x00089d2df8745dbd,0x0008a48217cd83eb,0x0005ce0f8ae79b86,0x0004021c2c4ae44c,0x000ea980ca2b0fe9,0x0004146745ab9482,0x0001c2cffa33fcf0,0x0000000000000001}}, + {{0x00076fd51fd99bf9,0x0007e3a2b01fa7b1,0x0001a17875cbebf2,0x0008df20ca98073a,0x0001c738732531a0,0x000c360b05cea958,0x0008986bad316bfd,0x00009591db5fb8a6,0x0008ce8516941db2,0x000cd50df495ad83,0x0000000000000001}, {0x000d46b24a5b2933,0x000a4af0d09b27b5,0x000e34ef392f2b04,0x00028d0cc4e0cb50,0x0005bbe1270619c0,0x00002d927660b899,0x000c444a9beaf922,0x0003686effea3a61,0x000321e427cc238c,0x000fe609075147ce,0x0000000000000000}}, + {{0x000e9dd164c62b53,0x0001878a3599a216,0x0000821091d05317,0x0002cda324ef2697,0x000e94950f2f16ed,0x000815b553eaefd2,0x000f8019f00612dc,0x0001930eacc547c1,0x0006fc4a1fd1730a,0x000db88252d52d13,0x0000000000000001}, {0x00077522a6c4bee6,0x0006b12deb38426b,0x000ca869197aea9f,0x000c43193823d16a,0x00028f12c9d38187,0x00031f43dad5cc98,0x000728a436529c3e,0x000863d40c6f0781,0x000da1798bfbb097,0x0001e17a19693394,0x0000000000000001}}, + {{0x000a20633820f8b6,0x0008884ce6057395,0x000b9e9ac4298b05,0x000f80c79f28e7bc,0x00012abb15751770,0x000ce75763d01472,0x000afbe67296f82c,0x0002a2950d9f8034,0x00031ca6f1179141,0x0008bb87c616f997,0x0000000000000001}, {0x000f27dc8004bd5d,0x0004fc5fa5d017c7,0x0009fdb4deb95bcc,0x00051c1e39917e40,0x000cfbefa777d300,0x0006ebd51f3f36df,0x000089ed9696a852,0x000c58a6c0bc16cc,0x00093efb56723f03,0x000f77e4f7a67531,0x0000000000000001}}, + {{0x00082edbf63cd0fa,0x000fb67ff0d41a00,0x00076aa53cf1522f,0x00099dda453dcda7,0x000bf634bcd8a3ac,0x000f09af12ca31a6,0x00045d3da6aee65d,0x0000b2e1c131b960,0x0001888166f3c7e7,0x000d21cb58f8b972,0x0000000000000001}, {0x000f3e0321dcdf91,0x0009a8d4da7b1151,0x000e3a95788cafbe,0x0007071e39c010af,0x0004b05cb3faf8c8,0x0008a702fbafcfc0,0x000618742c775b70,0x00068aab53d65b3b,0x000b27ffbb84f938,0x000a7508491b708b,0x0000000000000001}}, + {{0x00020328d4b15dd1,0x000274b581eaa62f,0x0008fb2a285d269e,0x0006ea89604b1779,0x000933aa53ad75b2,0x000fa62691d5119e,0x00067e03e002a949,0x0000629215018ba1,0x000ae2796195dffb,0x000282dc1f93eae4,0x0000000000000001}, {0x0000977c61f7743f,0x0008f7654950f798,0x0009f0fcf77422ba,0x00070562b7dc1d4c,0x0008f0b2f76176b9,0x00094ad6c12de606,0x000d53dd34579508,0x0001fc63f78fe569,0x000a90b5214981ae,0x0005ab902dadf9f2,0x0000000000000000}}, + {{0x00006fc86d7474a9,0x000491c759885f54,0x0002d4cddc55bd2a,0x00061045c35aa122,0x0007a2154985eb54,0x000f0dcbe4188b45,0x0006a7e235148dff,0x0006a2535a30d70c,0x000e2be337d8e901,0x000cf899a19ee96b,0x0000000000000001}, {0x000dc1860747030f,0x000a1d519771baa1,0x000e6bf7f8dea4c9,0x000b88d5c44825c6,0x0001648270d80fd4,0x000d7c088d619d7b,0x000e67f50ac4887c,0x0009d1ac72f9cc2c,0x000dce091aafa6b8,0x000ac9b9365de8af,0x0000000000000001}}, +}, +{/* digit=31 [{1,2,3,..,}]*([2^124]*G) */ + {{0x000c9884018b1bd5,0x00017e335350476d,0x000f240ea34a105e,0x0007225c0ca7c1ed,0x0002e60ee9bcde0a,0x0001b7a04f8d5abc,0x000d636ed201196d,0x000fee08dcde421f,0x000648f1c3a41da5,0x00014b37a2b18a4d,0x0000000000000001}, {0x000574ca3d13216a,0x00000c8f4aa46ce2,0x0005e6cb8b142b50,0x000aeecc2cc007b3,0x0008b139d4602d18,0x000857b6e6fad62b,0x000703a0b8035154,0x00047dfe5be4aaf5,0x000e255f15b88d9b,0x000ceeb42f23b0c7,0x0000000000000001}}, + {{0x00027bf41035c3be,0x000003d6c228d698,0x000ac8482db53bd6,0x000f6c6cedd6d84e,0x00055554b59c1199,0x000b3dd0d5c80a25,0x00098fd9a255d70b,0x00088ce8ece5b616,0x00010e4ff0160238,0x000e8b21f2b5b409,0x0000000000000001}, {0x0009be6e93956f12,0x00028be014bad7ba,0x0007941a6f1d6c8e,0x000374aa983d3be4,0x0001ab03efe8a93e,0x000ecc1517778750,0x000a07f3863f0102,0x00022339ade08ce1,0x0002e138fb118165,0x00037ded66083914,0x0000000000000000}}, + {{0x000be450955d0f49,0x000350db3612b4e4,0x0008b05c6937d0f5,0x00091d0bc00589e9,0x000f08134602fbb3,0x00072898cb46a43b,0x0000c3f425983612,0x00092a9319d102b7,0x0002adb9889640c9,0x000f69984da61b9f,0x0000000000000000}, {0x000925b413e1ae3e,0x00046d808d55308e,0x0001e8e95f85495a,0x0001aa328926b87f,0x000e6f7c0b2cf48a,0x00097c234327b453,0x000bc1b584c452ac,0x000ac04a1db1b2a7,0x0002c2df96a4716f,0x0008b35f6c998afc,0x0000000000000001}}, + {{0x000c246de542c405,0x00006abed2f33bb7,0x000d46decdec7b50,0x000afeed50c509c6,0x0007109502cf683e,0x000fa7b0916c8d21,0x000971ce284eb826,0x000b5478a9a06ef3,0x000dbb05d7e812b4,0x000fa9bdf3afd0be,0x0000000000000001}, {0x000c0e4a6519aab5,0x000d79de9fb97617,0x0002d46f889510f0,0x00025cb75085caf9,0x000f963379f4c576,0x00002dc4877679ee,0x000748161e8da062,0x0007933c7094d95a,0x000527ab96f198e7,0x0000a23cef9bb67e,0x0000000000000000}}, + {{0x000254c4bef57a31,0x000f8fe05ebbca4c,0x000036d19011dab0,0x00092f09f97449f1,0x000feb784d8ec601,0x000551e5955591a4,0x00088c74b266204b,0x000fefa58c8cd4d4,0x000954c3d29f9c68,0x000262f04bf8acfc,0x0000000000000000}, {0x0005e5c9d99f906b,0x0005ebe8f2ee55b8,0x000f127eee6b4d59,0x0005b10ba97a057a,0x000c7680e692309f,0x0007c47d151a8543,0x000975bcc38cb298,0x00056b40037e7ed1,0x00069095953732ea,0x000e079710f9d766,0x0000000000000000}}, + {{0x0007ef5561843b50,0x000725adb4b17e58,0x000223554b9e6db7,0x00040d6a298840a9,0x00089b9987d3e8ea,0x000c544359088f19,0x000712498c4e6798,0x0009d49555742687,0x000531911aeb4757,0x000c25edd6bd8c42,0x0000000000000000}, {0x000da2be384ee90b,0x000ed1578452ef17,0x00026ec7e64f3506,0x000d93fd400c530b,0x0006442c14bcb0a9,0x000bc44330eec280,0x000b7a321d894abd,0x000383284ca21784,0x000aabf2cbd2fe67,0x000a0b333a314bbd,0x0000000000000001}}, + {{0x000deccb4cc0fcd3,0x0002b5d6f0d87e14,0x00022447f7757c4f,0x000f03b5e4d05426,0x0003dcf9d56f98d5,0x0001c6d571746b68,0x000648164a40c197,0x00086a775d32054f,0x000bbccdf9c7e93f,0x000ab95b370c616c,0x0000000000000000}, {0x000ad22f9be04193,0x00049e68cea75e3b,0x0004683245da929e,0x000316e38a93792f,0x000965e00dbe19e7,0x000e9c59fb61ef64,0x00094c5036fbab0d,0x0009e4c2c8f5993c,0x000c76d4a049b3bd,0x000c781dc2d523d7,0x0000000000000000}}, + {{0x00010cba8003a62b,0x0002963dead37561,0x00024e572ee261b1,0x000924c14f710c53,0x0003a3234879da4d,0x0000242c6b2bb72d,0x00025965319d73bf,0x000c5d438ac356b7,0x000eb1ea69c1467e,0x0006ea40556d55e4,0x0000000000000000}, {0x0003bb0cfbfbdc6b,0x000292f755482f11,0x000b750229b1fdd8,0x0006dd9d36eb56b3,0x0009fd65055f0875,0x00005fbea1ad24bc,0x000b5ba29626eb13,0x0004c9855409fcec,0x000000d0af627326,0x000b249d561b2281,0x0000000000000001}}, +}, +{/* digit=32 [{1,2,3,..,}]*([2^128]*G) */ + {{0x00097f8c2da756c3,0x00065716b51e78af,0x0004d4e4ac9bb4d7,0x000be63f12ece85a,0x0007f2c2556ca2a2,0x0002341b0c191c3b,0x00063796c15ecee1,0x00092e302dd7df66,0x000d162a4ce9cb82,0x0003dfa7f8ba9276,0x0000000000000000}, {0x000403973587aa55,0x000a9956dae839d8,0x000d9da7dcbd9d38,0x000d0fffb69b8acf,0x000544e0adb2ad93,0x000b2ad644f74f04,0x000e7d5b5de013bb,0x000f944ef674d489,0x000e01d0ea2d2bd3,0x0008aedd32d1ec0a,0x0000000000000000}}, + {{0x0003c3743a6b3bf8,0x00083b2cea274d8e,0x000f6accb4a79b20,0x000ac9cff7eff159,0x000c5bd1a458b1a2,0x000af5afd8c30597,0x000e95f67ad0a34d,0x000ffcb5f547ad0c,0x0002c927ef492633,0x000118d70d201bd4,0x0000000000000000}, {0x00025271d14dfd7c,0x000f83511be77473,0x000e33f2540532d9,0x0002d9c50e1e6624,0x000b9f8f4394e620,0x00085289919c8fa1,0x000d6412359d3b9f,0x000a4c00c9ead88e,0x000626daa054c125,0x000853e0db1f33bd,0x0000000000000000}}, + {{0x000ad013227cbee4,0x0005d963a674d5e9,0x0002422839e1c90e,0x0006f11c073f8abd,0x00051cbde443fb90,0x000f94add68e37cb,0x000d9cdf247fb6d7,0x0003b3ec024c71cd,0x000015c9b5032da9,0x0007607e0d83a94a,0x0000000000000000}, {0x000d4287c9083b11,0x000b167379dfa59e,0x00092871f05ea23f,0x0000a530c1fbcc90,0x000c9b670aaadfc5,0x000840b4f012b3f9,0x0006d74574236725,0x0005827f0582ac31,0x00047f13870e7720,0x0006f314a9061f10,0x0000000000000000}}, + {{0x0003be466658f617,0x0009fd565e43add7,0x0004a046e438ce3b,0x0007e9edef2d69e6,0x0006c7f11d4e7b33,0x0009fce23db4d264,0x0007ee69cfe36cf0,0x0009d497797ff857,0x0000fa9f71e1b23f,0x0002d2813fdfceba,0x0000000000000001}, {0x0005801d34f0db76,0x0008b9ba1d6ad8bc,0x00038f8437efa8c8,0x000755dc58d2c493,0x0008ea5d4147adf5,0x000454e0d19f3138,0x0005174d880f0ef2,0x0006f4ab4400ed7c,0x0002f97c02972f59,0x000bb7fd1f05bd42,0x0000000000000001}}, + {{0x000f26521378e02f,0x0002d730a6852468,0x000763f810a221e5,0x000f48636e9cb246,0x0004c1b9e4959290,0x00093bb4ca5e4a52,0x000f2a6103eedfad,0x0009e3ea2bb0a0a9,0x00013663d597d7ac,0x000a98c50bf00061,0x0000000000000001}, {0x00050b19a3ef90bc,0x0000ea69cbaa303c,0x00074f8f2fe993d5,0x000223c8be835b07,0x000f455e7085ef84,0x0002207ec6b0871e,0x000e48bd79a714e6,0x000daa1613ef3b51,0x00098f36229d3dd8,0x000c2d9134c94bcc,0x0000000000000000}}, + {{0x00050a19f5342c54,0x000dc653d78301f8,0x0006bbf10f389ca9,0x0004326e54a5af67,0x000fdd8a13b29709,0x0008aa3fafda8b65,0x0006c2e3f7baf8bf,0x000bca4111083f35,0x000cdf78842a34d7,0x0002a973106a6473,0x0000000000000001}, {0x000155f05f4ec88f,0x000ffd8679f932ba,0x000e00ae69c605da,0x000b7c72ba823e0d,0x0009e0bbb62edf8b,0x000f7bf5bd871069,0x0004e610f36c3cf4,0x000e41a06519118f,0x00014868b2ff5976,0x0000f7b8d8554854,0x0000000000000000}}, + {{0x0003e5d502754293,0x000e9ee9e0f2543e,0x000a3731f423dc07,0x0003b8bc5bf91184,0x000726f4ea8f1d1a,0x0008035844d23059,0x000c7266e29e66af,0x00082574f0c5767c,0x0001a6dd0e57d5d2,0x0004e537115bccdf,0x0000000000000000}, {0x0009f7ce1bfb6728,0x0000ccb130bc8170,0x00039a62c614cf63,0x00068e187476935f,0x00034de841f4a723,0x00011647b88ac5f1,0x000e09f54f07f6bb,0x000f505a8bf2adfb,0x000d75a5f605b64d,0x0000df9bbeb2b499,0x0000000000000000}}, + {{0x000308733efc5f8c,0x000b75cdb37e83e5,0x00060b5bfda48081,0x0009f06138365296,0x0009688a8974b9f6,0x0005444cc05fb9ec,0x0005a67f252002f7,0x0009664675a1899c,0x000b6d7be11db7cc,0x000549e5e85617c6,0x0000000000000001}, {0x0000536e04ec0d89,0x000ceb7897a84665,0x000b8acad3957bde,0x000ba89439f416b8,0x000cfde12e814bb4,0x000a77e0ef45c679,0x000f35bbfcd091bf,0x0009f3ea6cc5ae92,0x000f66583ff4f9db,0x0009a867f0fed315,0x0000000000000000}}, +}, +{/* digit=33 [{1,2,3,..,}]*([2^132]*G) */ + {{0x000487d30649cced,0x000796a5efc98a70,0x00086f3d00556482,0x000c177d81ed5742,0x000f3693c918841a,0x00044078e141f63f,0x000ad9ccb0cceba5,0x0005cd9ca803f396,0x000a3b9f81f2f890,0x000c5b4318691bb9,0x0000000000000001}, {0x00076e3095e41a52,0x0001ffb6fd45a8f8,0x000a8a0715ef8788,0x000192a0b8d73d7d,0x00086ca88981c074,0x0000f41a80dc66d0,0x0002bbb8f279d460,0x000cb55640383488,0x00052b11c10c7a90,0x000053f89b04d855,0x0000000000000001}}, + {{0x0002e1e8d88792dc,0x00022e3ae581427b,0x00090f869db2e712,0x000e06689d393376,0x000702d537bfdb1c,0x0007346bbf1a9bff,0x00090f54aeeb8464,0x000273c9dd468a0e,0x000c871a654e3afa,0x0007465945d8c3b6,0x0000000000000000}, {0x0000e770af4a5960,0x000be4ac70e87a10,0x000797d6d911c87d,0x000533fb961a5c5e,0x000b8548c0001c5b,0x0009d47191b560cf,0x0009eeca65c8463a,0x000ecad37d2137d3,0x0000514ad716bab4,0x000f8789ad5bc27b,0x0000000000000001}}, + {{0x000dbc583693bf30,0x000ef8d24b6766c6,0x00095890706d31b0,0x000c51cce3a35296,0x00080b8ed7618c90,0x000973ebf17cff3a,0x0009c68d473b1c44,0x0001098525e43a12,0x00074031f5036c9f,0x0001d33955ea92c3,0x0000000000000001}, {0x0006f1c314ce3a37,0x000a4064ddf24cf4,0x00070db52569e1fd,0x000405305ea2c55e,0x000d5f14297acf89,0x00046ea96e034f59,0x000622a42888331a,0x000a102ad1347dc4,0x00088a514e007741,0x0000461db8ec7cfe,0x0000000000000000}}, + {{0x00023847d8cf0718,0x00021ccb2c301d0f,0x00024b1883c46a31,0x00063cce64fb5faa,0x0003cc10bc090432,0x000510506a731fce,0x0009a05134986c0e,0x0003aa30a907d289,0x00071f165d859243,0x000eeaa5074a4066,0x0000000000000000}, {0x000b1d8c9f3b369e,0x0008874f03f7bd39,0x0004a870054ed9a2,0x000756adbd121753,0x0001a9a0d0a37510,0x000529605385faa5,0x000c2eddf5c089f3,0x000a130a237e15a5,0x00074ff2cbd316fb,0x0007ee2c9d3ce137,0x0000000000000001}}, + {{0x0009dbdc68f3280d,0x000657e090c25dbe,0x00024c6421981e70,0x00037a5a5d0a99ac,0x00037fc8c760c5df,0x00044d1a53e37845,0x000ff43dddfe06f5,0x000e4ab44983d93f,0x000c0915a82bb80c,0x000646c74033c3a1,0x0000000000000000}, {0x00004b5d5c7ce211,0x0001372b933ba5de,0x000ea64d116a103a,0x0006df2d3d823414,0x000619b7a1bf8333,0x0006eac1655d2034,0x0009aeedd521f9b1,0x0006d98d98725948,0x000610e8ae934636,0x000efd7b215cbea4,0x0000000000000000}}, + {{0x0005d0308e1127f5,0x000cf03c8e2019c4,0x000aa0237cf6545f,0x000764372510cd18,0x00097139d47739de,0x000a85188927140f,0x00037dc9b8b5e192,0x000b2545731c10b5,0x000991ea59d03831,0x00067dea66743632,0x0000000000000001}, {0x000cc6866cf98c39,0x0009016f3fd54524,0x000592bcebafd818,0x0002f06cc34bdbb8,0x000b6cb2e7a209f5,0x0005749dab7ee649,0x00076406958e2abb,0x0009516d049dfaf2,0x000de5f7e8adfc4b,0x0007ce97062d4c5e,0x0000000000000001}}, + {{0x0007da121df0699c,0x000ad3597a95de87,0x000181d213d52a92,0x00002da71c92de4d,0x000dd3803396793e,0x000c8761f332f0e0,0x0005c1e0d83b70eb,0x00048cf70527f5a7,0x000ceb82fe3e31cf,0x00029a56f1047f6a,0x0000000000000001}, {0x000ae721531eb7f3,0x000c70c79169f267,0x000d345b58d29bb0,0x0006c1daec3533a3,0x000115913b369665,0x00099820ca585f3e,0x000623473863b5d7,0x0003e2b952a9549d,0x00011f22279d7812,0x0007e6cad8cfd481,0x0000000000000001}}, + {{0x000c384b610f9960,0x0009e42d4100e245,0x000c5264e577187b,0x0000ec202477a817,0x0009dc146fbb4cb2,0x000c49fc51a5dd07,0x000dd34b66b540f6,0x000418cb3114a207,0x000042a4afc85f36,0x000592a886f4d479,0x0000000000000000}, {0x00062b5959d642be,0x000c107df28ef33c,0x000c98bc18d09a83,0x000908cb61720266,0x00034bfa40c64e8b,0x0005f7d00d3266ed,0x0006699785d5c5ac,0x00070fda50cded6e,0x000a7129a0528d63,0x000df6226a01349f,0x0000000000000001}}, +}, +{/* digit=34 [{1,2,3,..,}]*([2^136]*G) */ + {{0x0004a83b5020b6b5,0x00064ea6b7250085,0x000f5cc5dee82b8a,0x0007e307a44f4310,0x00061a979f99982f,0x0006271c95260383,0x00087bd9d4a6e7e3,0x000183121a682c2e,0x000a0c42c801461a,0x0009efc46dd1bbdd,0x0000000000000001}, {0x000ff9d53c8adce8,0x0004cbac7e6d6ff5,0x0008a2a18c9ba604,0x000457234e0b1c61,0x0001b538c1881476,0x000d20849fff1d07,0x000e3333d9430380,0x0001d1326f05033a,0x0004a49c4e89c642,0x000e640c63716450,0x0000000000000000}}, + {{0x000a7c6657e0c514,0x00010e4ba5060489,0x00003183bcaf92c4,0x00051063325bb838,0x000624a227afade7,0x000d611fad61ce2f,0x0001f27e1c057fe8,0x000426a808156374,0x00051e188cc3f494,0x000e601fb19202dc,0x0000000000000001}, {0x000f5c4ba35ecd6e,0x000c838b90f28423,0x000ecc8f9f7eac00,0x0004ae3bc63ca5b1,0x0000a61f4eb49abd,0x000e5e94c7586825,0x00050828aa62e59d,0x000ca27ce17d2e20,0x000f7dcd24d94b7e,0x0004ed84ff72ff3c,0x0000000000000001}}, + {{0x000dae22413fe9d4,0x000881c93ceb2c9e,0x000376b68f1c40b8,0x0004d107493ec443,0x000de2613f0552fe,0x000264177a2adbc0,0x00044456850f4d4c,0x000c024b1759999b,0x00032c490b5528e8,0x0004e4fe9cb25fa5,0x0000000000000001}, {0x0002401de7dabddf,0x00056529f2c840ea,0x0006004e218ae4f0,0x00026d7d9745c833,0x000bc1aa8e8c745a,0x000254366c2e1e3a,0x0009a65d176c592f,0x000575f2ce2f5dba,0x000390121cb70eda,0x000ad4df3bd7c9ef,0x0000000000000001}}, + {{0x00052f406338228b,0x00000044f05c5be3,0x0003a7061d336069,0x000371f2e7fd3e15,0x000ddb123a32ed82,0x000a15e8eec0c29b,0x00046b80938b2d11,0x000ba2ae38c19bba,0x0000c4e7466a69b9,0x000ba8e7a0607a47,0x0000000000000001}, {0x000e250e34d513c2,0x0009900d3d611604,0x0002850e69a99aa8,0x000ea018e87aacf0,0x0008ea9b70f5d0f5,0x0009dfec50e62995,0x000ef7267ad0ad8c,0x0009fbbc4dd8a19f,0x000ef73af4e91334,0x000d0636506a6e44,0x0000000000000000}}, + {{0x000533f2ba0eb14a,0x000b9fc6b2073504,0x00059889c71f37ca,0x000d3e3b2957243a,0x00033cd4ef031ee6,0x000e1fa792c82e2f,0x000936a9431aaa2b,0x00075897dee2d5df,0x0005c1a2769038db,0x0004c149337ba93c,0x0000000000000000}, {0x000ff077c9595fa2,0x000b4e92632965da,0x00073090129d489d,0x00024c2f940397cb,0x0000f08747c463ab,0x000063f57ea7844d,0x000a687de4ab15b4,0x000d7bdc8db9dfb6,0x0000393c5c4b7272,0x0002b3cc129fac67,0x0000000000000001}}, + {{0x000d85352986f86b,0x000b3bb5e1a9d134,0x00096ed674c04b6f,0x0001eaebc5869197,0x0001ec13b24f0220,0x0005acb88043fe14,0x0006b2f77717702d,0x0001f913c28eb4c3,0x0008bc0cbbd9e8fe,0x000014871dc376bb,0x0000000000000000}, {0x000b182392919d22,0x0005619062a004b3,0x00084b9c0aa0d96f,0x000e6a14d38134a4,0x000b962e9b9dd384,0x000d2a3f87434945,0x000e17c26111d5b0,0x000cca088afb0558,0x0004109b67e83601,0x000ebef3372d865f,0x0000000000000000}}, + {{0x000c5a0356b17ac7,0x000f616fb80668c9,0x000f7001431d3037,0x0006786061783bd7,0x000e2a8db044a7eb,0x000d63e80e687c5b,0x0006dba72619e19b,0x000b3f54433d79bd,0x000179eabd3da5ab,0x000fcebeded88553,0x0000000000000001}, {0x0001156c4d223604,0x0006fa0e48c3398c,0x000d70c895f6a870,0x0000aa32def1e5d8,0x000a3628036e774e,0x0006fa3b42b31a93,0x0003f15e7bb3f2aa,0x000fd667e0a491ab,0x0002f04b61d5276e,0x0001fdac2e330e17,0x0000000000000001}}, + {{0x000760d231c36820,0x0008bd2a50426c8d,0x000d4451d722fd74,0x0001877484a5084a,0x00019395bd1ac7d5,0x000dc03d6541ad77,0x00068a254b40eaa5,0x000dc699a962f42c,0x000f2ecdd481b2b4,0x000345d9badbf178,0x0000000000000000}, {0x00090c94035684fe,0x000e517a9849bb68,0x0005822be91e8615,0x00067ca7e3c3e516,0x0004c5ebee67a9ed,0x000f03236f5438f4,0x000029ef9e45ec0b,0x0003412d001129c5,0x000bad0b64fd4f4e,0x0000e15f591e3c09,0x0000000000000000}}, +}, +{/* digit=35 [{1,2,3,..,}]*([2^140]*G) */ + {{0x00067ca62dd9afd4,0x000678b2e8cc8368,0x000d7d6c96a2abfd,0x00075f62bb6c702c,0x000988eb9ab34b7b,0x0007b382272a8eb6,0x0005e40ee1d17286,0x000b6f600751bff1,0x000ff996b4ec3001,0x00015d7fb8efdf30,0x0000000000000000}, {0x00062d76a29a2746,0x000f091c80dd81fc,0x000c1a9825d4a2f2,0x000a4fb54ae9b61a,0x000db71a812fcb05,0x000bb96eaaa7baf2,0x000dfd9cc434e4e8,0x000b8fce567253c2,0x000b948eeceeb8e7,0x000abac787b7e9d6,0x0000000000000001}}, + {{0x000566d2087a8f7e,0x000f8d816dab3c44,0x00068ad0a5ea555e,0x000ab76093fa3eae,0x000bcad51a41fb45,0x000c784a1114a732,0x0002d99cd96f3573,0x000f7808bc957e91,0x00022a461547dff3,0x000e9dd3f93d98d0,0x0000000000000001}, {0x000f5792b3bed20d,0x0003199e50e443dc,0x000ab35921f1c5d0,0x000cb763ce7e3777,0x0009ec69a2c8061a,0x0004921d8bd5a1f1,0x000d3f186d49b86d,0x0003d287849a3eff,0x00019a1d3969ee2c,0x00097e7987e8d923,0x0000000000000001}}, + {{0x000e6b3554a3f3c4,0x000c8b48d7c64666,0x0004319bb26494ce,0x00023bd53c15f132,0x000a4b25b7340a49,0x0002c82187e36296,0x00076cb62a70b23d,0x00013ce0a44b3a26,0x000e1376215ada95,0x0004bdcdd5bfa093,0x0000000000000000}, {0x0006f0577c34a522,0x0002d6eb1d23f2e1,0x00074b1ae5a563bc,0x00076c1922ce417d,0x0008d8b56e586f06,0x0003d2111864665c,0x0007a1f4a9d1f08d,0x00009ad18a2eb5b5,0x000f16f69121b144,0x00055ad3dba51f31,0x0000000000000001}}, + {{0x000a0990f6c14c34,0x0002ae1571f4bd14,0x000a7e981428a12a,0x0008a57064ea4bd5,0x0005ec2f56d89f54,0x0004fcfb513a99f0,0x00081deb029c28b2,0x000c16eb364a4688,0x000f6df6754a22d8,0x00039a8e7ba7c29d,0x0000000000000001}, {0x000585b840875f9d,0x0006688b87eab66d,0x00061b8a4aef8f2e,0x000968d01b210ab1,0x0007a38c32d9fcd5,0x000170203f9469f2,0x00027ba7e65bf262,0x0003268e8f3ddf53,0x000d5d6a50d743f2,0x0006f76866dcf3bb,0x0000000000000001}}, + {{0x00075ceb39ee406f,0x000fddc2dbf93cfe,0x00005aa3d0f7d044,0x000c04043459ab15,0x000bfbea051fd1e4,0x0005c86723eeca2c,0x000dd90428637a5a,0x0006d3aca9d581d9,0x000277709f646127,0x000d9e5fdc588878,0x0000000000000000}, {0x0005fdeadee7c5a7,0x000b59b799ae3c10,0x0005e3595acc919d,0x000b6f6b2aa1f7f7,0x000cc519dab324e9,0x00070aa0c81054ee,0x0006840dab1fa02d,0x000ce8162c464504,0x0007fc117382d8fa,0x00079cc63a2e343f,0x0000000000000001}}, + {{0x000f45643ca65cbd,0x0007305e42072e40,0x0006980bc47b22b4,0x00091f480c0959ae,0x000df17382117d00,0x000fb6755fe76ce6,0x0008195083b13716,0x00053ce928778e33,0x000eadd235784446,0x0005f288650fd122,0x0000000000000001}, {0x00032d4f966b7e9c,0x0006ec40795011b8,0x00056106a162f5eb,0x00060472439d72fa,0x000ed9a6959807a3,0x000d3315f177c4b5,0x000b196cd83808fb,0x000c21f3f41dc773,0x000518607dcca40d,0x000920d975bf1042,0x0000000000000000}}, + {{0x00043d0a4a0b7f26,0x000c9bca61488d76,0x00078d40864c9a4e,0x0009191208ac32aa,0x00065e2c33dbbd1f,0x0006b041d84ce172,0x00022f6c73e5e84a,0x0000caf07f551302,0x000e0bc76bc20bdd,0x000a43482195b22f,0x0000000000000000}, {0x000f04c8745c6a12,0x0002b2bdd6ee1437,0x000b9431fd260182,0x000e54b7f10879b1,0x000a6b8d5c027ebe,0x0002358509530c61,0x00071ee3b953e075,0x0001d055e247c05d,0x000f78c21fc120f3,0x000c40b71a77f551,0x0000000000000000}}, + {{0x000ca1655dd01fc4,0x00023cfcdcd83fdc,0x0006fe01dad6f137,0x000a92f448fc724e,0x00071e9506b3510e,0x0002c50316bacd31,0x000812e5b9c38263,0x0005b06a2041b525,0x000d1e51d6095bd3,0x00018f8c9f2aff29,0x0000000000000000}, {0x000e8440d9f6b1c5,0x000d8f5815a76ff5,0x0000ba6e7eb4652d,0x0000cfa7a2d772d1,0x000e12c2c10a367d,0x000122408a9134fb,0x0006be74d3fc999e,0x0007f158ed729839,0x000445a86f173644,0x0008103589b3e72e,0x0000000000000000}}, +}, +{/* digit=36 [{1,2,3,..,}]*([2^144]*G) */ + {{0x00002ef5664a50e9,0x000569e6626b5584,0x000bb9dc4c85c34e,0x0006cac4009d6dab,0x00047cf68656c674,0x000e65ab973336b9,0x000ecf3e266a898f,0x000b5830a2ee0371,0x0007109821d57e75,0x0002643e097669c9,0x0000000000000001}, {0x000e2ad77fec8187,0x0001deddfb754e78,0x0004aaa3d5328431,0x000f5938ac9d56ca,0x0000419e9ec29fe5,0x00089e92d324185a,0x00068c4746f628de,0x00086959a461fd09,0x00039e1752cc1b19,0x000f685c4efa867f,0x0000000000000000}}, + {{0x000578941d3daa6e,0x0001e81a86314a15,0x000e2ec49066a742,0x00085f37e975bc97,0x000abd59fd20aa74,0x000b001318e5e712,0x000bdca951133a15,0x0006057f57ee1259,0x000dad04acbd3b2c,0x0009e7ef3153ef33,0x0000000000000001}, {0x000d37d508c6263d,0x000d87a4e81e7b2e,0x0005a01a3eff8f36,0x000726730288c3e4,0x0009b846f52088b3,0x000f560651a99118,0x000aeef71db52e56,0x000e58e36c06431c,0x000d03f83a3f98d5,0x000adc020099b8d8,0x0000000000000000}}, + {{0x000d0b7b6a8a3320,0x0007c1820c541666,0x000008d89959635c,0x0006f6b2d261e2b0,0x000b20857dc9286c,0x0008490da31aca2d,0x00001835ecf8b430,0x000769b376d5c408,0x0006c3593326d702,0x000ae359276d5c27,0x0000000000000000}, {0x000052b955ccca9f,0x000132f8bdb8b0e1,0x0001fc788aac66dc,0x0007aff377f81134,0x0005f3f5c42cf8fe,0x00063a0cd5ec56c8,0x00044e19f92551b9,0x0005a03e4e9df2c1,0x0005981a18a9c38e,0x00038410aaa01483,0x0000000000000001}}, + {{0x000ab1b79d73f8b8,0x0002c67e2040bd52,0x00089ab066095a12,0x00020058f1cb78af,0x00035c77cb75101a,0x000e13361531375e,0x00075eaea159ba65,0x000a7ecbfca3524c,0x00019d039ab8ae0f,0x00099c623ac91c57,0x0000000000000001}, {0x0001430a249d36df,0x000efe8450eb5d6b,0x000afb92b30c47b9,0x00024beea9991147,0x00039e1752c3ff68,0x000fd6a6252b160b,0x00046e76256f4b47,0x0009076f7bff5746,0x0003f350ce5bbdfa,0x000da84642b5dbcc,0x0000000000000001}}, + {{0x000f59f034423df4,0x0000c74c6502a253,0x00051f6b04936dd5,0x000ac1dfa20fd420,0x000a26ee480942df,0x000b8f32c2aa1793,0x0008f9fd5819aba5,0x000593ae9a68c3be,0x000809f95c514dfc,0x0005832a0100c51d,0x0000000000000000}, {0x00084df91d0b9d7b,0x00019af96dda8b28,0x0009e06515fe82b7,0x0004e7882cd87d52,0x000d3f4a8cd84bcd,0x0005a839e5e4c173,0x000eace8b4e2b402,0x00009c378fed4c2c,0x000b8a183c245522,0x00026032daca8b53,0x0000000000000001}}, + {{0x00021d74b7f15174,0x000b1737719209fa,0x00000c8bba28cfe5,0x0000523f1c2878b2,0x000c0170331c9a62,0x000cd83b50a5843a,0x000131d0381135b8,0x0003a643b75eb047,0x000ef1464d2ab54c,0x0007f362ed0e42c5,0x0000000000000000}, {0x000bb20fbad15614,0x00040a78f8613291,0x000895f7e0d7805c,0x0004b54ca2a8624a,0x0000e6579a8713ce,0x000626e2cc1b0cde,0x00093c66377df41d,0x0009cd6454de0451,0x00009db1f1c3ca34,0x000d91b047b0a149,0x0000000000000001}}, + {{0x0006148c3607c309,0x00072dd5c6a1cd15,0x0004ea2e6d51f4ab,0x000bb9012a38398f,0x00025cc1f09df84a,0x0009c3bace064bf4,0x000f3b1a1aeaac49,0x0008ba0e470586b9,0x00026aca2a7b1cc0,0x00037064de7e58e0,0x0000000000000001}, {0x000f70f6864c071b,0x000cebbcde808997,0x00042d9268c9d10f,0x0001f3646bf97f61,0x00099c6124289f61,0x000f65308f5fa877,0x0003b1cbe10f2164,0x000db1cfb717f6c1,0x0002178fd0705446,0x00053784aa2a3cca,0x0000000000000000}}, + {{0x00032b93eb6bf0f8,0x0008d44a6f35d7f4,0x00062f74f5a61124,0x0008d968ff45509d,0x00090f78b11dcef9,0x000e0fdb4e540d2d,0x000178df19486918,0x000b775c9c48f839,0x000a4516e1546952,0x000548b05a9a422d,0x0000000000000001}, {0x0000e6542e705240,0x000ea85c40801a5a,0x0008cf4381fc9bfc,0x00026551ecff5ed1,0x00006e3765708042,0x000f10bb393addaf,0x0004c0be6d6327db,0x000ade98dcbda7a9,0x00045d1d2c9cc265,0x0001d23919800694,0x0000000000000000}}, +}, +{/* digit=37 [{1,2,3,..,}]*([2^148]*G) */ + {{0x000136d3adf7c0d8,0x000a615150ed1e22,0x00048b602d1f12f4,0x000a438f58c86ca8,0x0006c2ad94dbc8f3,0x0001741520fd2861,0x0006926fc8f344fa,0x000aa2867b7697e9,0x00063769f3f74f49,0x0003389eafe4ecc9,0x0000000000000000}, {0x0003271ab880c04c,0x0007eceb904c8b8d,0x000cf0e8b6b36124,0x000b8dfe9dc846a9,0x000c71bd5a3dcf58,0x000bb872ef46766e,0x000ea257028f76aa,0x00037d56cad75976,0x000e6e410a7a4c1e,0x000aa4d9ef6dff50,0x0000000000000000}}, + {{0x0003f21a068b1990,0x00028b83926d837e,0x00046424f5058ff9,0x000540b150a21088,0x0007bd69839e2656,0x000836bb43217215,0x0008f5d34535e3bc,0x000a61ec6b271f81,0x00014bd57f4cd40a,0x0009c8fdb8302a87,0x0000000000000000}, {0x0006b22f2553a3a3,0x0003b58b7033af0a,0x000213a07cddbf4f,0x000434d1d71e271e,0x0003ac069f3affa9,0x0004ccd448d5d23e,0x0005a3de785990cc,0x0009500536e9dd21,0x0005a1f484316890,0x0008d39f92d8e2e4,0x0000000000000000}}, + {{0x000e463b75bb6ea5,0x000ed0a11789a549,0x000dd32eea4152e4,0x000b779f3244c612,0x000a765d467497f7,0x0001f2317d762ed2,0x000fb479fdc0f1f6,0x0005b625c778a26f,0x00086e4279e293c7,0x000778007cc51b6a,0x0000000000000000}, {0x0008d10c9ca3a103,0x000303626747aa01,0x0009d0d1059a4b9c,0x00047b992888178d,0x0005dee73f8df999,0x0003009b79f58ee6,0x00095bbb19efb6cd,0x000ee364f704fdad,0x00075210ba101581,0x0009e1f5ffb0008b,0x0000000000000000}}, + {{0x00061dbbf0d6ad65,0x00037943121ba6f6,0x000176edfca2325e,0x00061e28f0cef68c,0x00084617ac6eda38,0x0007535e8ca77e7f,0x00023d1d31f498d5,0x0000546d78b2f36e,0x000c7d55e2c3f881,0x000891156a1cb9cf,0x0000000000000000}, {0x0004ce76bf8c0b46,0x000f73894a4c0a97,0x000e4d65f8fc178a,0x000cb9405d4f42d7,0x00089f73dac29f71,0x00028141921d35c6,0x00055dee3cb66f43,0x0003af9effca7532,0x0004e3d9ea981425,0x00042022e23b71d3,0x0000000000000000}}, + {{0x000d0b7e99b465d6,0x000dcdbbc6a7f631,0x000fdb4862b8d0eb,0x0002d00e72a3f599,0x0003fff9e95d96f0,0x00098b66f4cdbc62,0x000ac921634cb2b5,0x000ec0deea0b5c81,0x0005b0a27a76f063,0x0001e02d30431834,0x0000000000000000}, {0x000c98aa1c9ac7a1,0x0007c5a7466568e8,0x000fb66755607c49,0x000ef99d83842e7b,0x0003f63d03b04212,0x000eea39bde1d43a,0x000c0a3d0947a656,0x000da642406f1e3f,0x000c396f50ca4ece,0x00002261add500ea,0x0000000000000000}}, + {{0x000faccc08dd24e3,0x0002a75b2039c36b,0x0007b00e10c370cd,0x000decd44a6f8b5d,0x0005623fd6ab9f44,0x00019a1ddd85e65f,0x0007f5f2f27bfaa2,0x000934e1d6c8baa2,0x0002398cf5bc3e40,0x000936e76ebca08a,0x0000000000000001}, {0x000b72cc3defb1bf,0x0007c48e0f8a00ef,0x000b50a281abffca,0x000290112a957074,0x000c6cc0bcd2913c,0x00080828c73f9cbe,0x0002b7332c142bad,0x000b82fd75dcc17d,0x00071d9dc563b115,0x000fa46eb1fc833d,0x0000000000000000}}, + {{0x000293af2196db74,0x000aa8f516dffa78,0x000ab476917c3f2d,0x000a17a44ba1c0ca,0x000771d0d213b4e6,0x000872d37a8c50d3,0x000a9d4f31d594dc,0x00020ae17df8f34d,0x00012ccecab30c4d,0x000fa5e5b7722b87,0x0000000000000000}, {0x000e6e4b1df4a900,0x0005d66c421f23c8,0x000473d466e42335,0x000dad17c61ddae2,0x000ed279fa7168cb,0x000addca2ea2657f,0x0004d479a5bdfda6,0x0009ec1a80eb84e8,0x0001a95ca1a89847,0x000f2ca605fdb8f2,0x0000000000000001}}, + {{0x0000bda5161e9684,0x00055c62c59939aa,0x0001e39fae89d4f2,0x00072aef74c49bbe,0x00060180fc9e6093,0x00063da12ade7248,0x00028defa823f501,0x000a965a30e8a72a,0x0005cf1083c600ec,0x0004af9f8b968790,0x0000000000000001}, {0x000afd7d7d936a7a,0x00003b13810cfd26,0x00037d1ddbf986aa,0x0005d035eede05c2,0x00071b7ae0b88271,0x000812487895ef9e,0x000f170e50423460,0x00003054f1639f87,0x0002e674eebc0936,0x000654593b42f2ce,0x0000000000000000}}, +}, +{/* digit=38 [{1,2,3,..,}]*([2^152]*G) */ + {{0x000a2f3a1cbc282c,0x000e19a09feeb1c6,0x000c54628c6180b5,0x000cefbae8c61be2,0x00031054eeedc773,0x000005e41190648d,0x0004925364893510,0x000a54e9064644b7,0x00026639e573a22c,0x0003b5a6074dacd6,0x0000000000000001}, {0x0002e1f28cb4398c,0x000fba11161ac99e,0x000aaf012b04f328,0x00060a6cb74a91c1,0x000690cf3c48dadb,0x0007c4e07d1b8182,0x000bed19eb0dacbf,0x0001aba090482e6f,0x000b8c6fb9ea1ef8,0x000d6d4b567809aa,0x0000000000000000}}, + {{0x0006de757b8ee9dd,0x0004f4a9eb572b8b,0x000650813f9e5a92,0x000081024cddfbbc,0x0003e750529ae0f8,0x000c407a678dbdc2,0x0005c643db36c6df,0x0009adee9ab1549e,0x000add1f855d46bd,0x000aedf68182d8ac,0x0000000000000001}, {0x000e2fb66eef3f12,0x00004b24a7282866,0x00050b3c877e75f1,0x000590fae38bb301,0x00008b7b5535d2f8,0x0001b50eaef87c62,0x0000c4541ba355de,0x00098bfe96023f0d,0x000dcf2eadc15969,0x000f41ab8c033f3c,0x0000000000000001}}, + {{0x0001bfbe835763b7,0x000da08736745919,0x000d000c52158859,0x000dbafd4373d9cc,0x0003efeee235e560,0x000fe980f98d303a,0x0004f012a6082ad1,0x000e567ed43eb524,0x000eddca68306748,0x000f954e531e38a7,0x0000000000000000}, {0x0000101b465ee778,0x000f8f4e95956310,0x000bcb6c6057ab1d,0x00052b140218cd6f,0x0000a217b7b093a1,0x000924c99c9b3267,0x0000b388550cfd67,0x000eda396f8cf9af,0x00035154327557bf,0x00098cf74a0d9f01,0x0000000000000000}}, + {{0x000cd9335dea0190,0x00015fcbf855836a,0x0007808f96352fb5,0x0005c5cb374fc6d2,0x0001eebbbb50f586,0x0007f3a5b9a4d0c8,0x000bc6329ed702e4,0x000264f0fada97b4,0x0005e3bbcf73be9b,0x000dd442f9f14de6,0x0000000000000000}, {0x00029cdbcd6414f5,0x000db247ce590b47,0x0005be836ddf363c,0x00032e4b6a8da968,0x000c049bdb9815fd,0x000d8f7528076e41,0x000d50b097db4cb8,0x0007f829470b7fc1,0x000b6caef75b1cc6,0x000ed3d55324b1d2,0x0000000000000001}}, + {{0x0002ed4b0886efde,0x0002e69216f70caa,0x000a19fc084c43fa,0x000bf1fcd447360e,0x00088b44bef17255,0x000c941e542e92c6,0x000c9462ab05687b,0x000f4a55ed27b06d,0x00006c4879438508,0x000b3f5a16cdbf59,0x0000000000000001}, {0x00090da308f335bb,0x0002c62f75607156,0x000ac3878c204f5d,0x00019a70e1d9ebee,0x00065ce18d8dd345,0x00053d4a7a6a59b3,0x000ab840d7a66249,0x000e9f8efc3edfda,0x0006a32d022f5317,0x00072a29be1120dd,0x0000000000000001}}, + {{0x000bdbfdee3c7792,0x000b747684a7e37c,0x0006fd64e032b7ae,0x00031287b0371790,0x0003245ed96aa7b2,0x000bcb2d9689dd51,0x0009eceb1d9918cd,0x00053e6e8dbeb445,0x0003cd8435c2226c,0x000a212888b11938,0x0000000000000000}, {0x0001020457ce98be,0x0009fbe80ddb3555,0x000e385ff7a0d009,0x000821584b1de3c2,0x000f36cd64d47bdf,0x000d2c6d2f0d97c5,0x000126f061319f64,0x00050f118e4e9aeb,0x000dd0fd66a1918b,0x0008c9bb60247ff9,0x0000000000000000}}, + {{0x00006452a69b40ef,0x000a58916404d8d3,0x000489351fe54188,0x00083228fc171eb7,0x000410e07508f15f,0x000b2d0bdc317f64,0x00075755181b0c85,0x000c4ca35cc09ad3,0x000bdec4e4688b7b,0x000ec2f6e18ec10c,0x0000000000000000}, {0x0000b91ff7109c43,0x0001353357108dc1,0x0006db397b6c17eb,0x00038d3f418ed53c,0x000a1a7d0d6f5104,0x000558963fe8aa2f,0x000629ed52f70d0f,0x00070987dde60b3f,0x000b95a72bb4ccee,0x000aae64e168b593,0x0000000000000000}}, + {{0x0009ff50fcb47b0c,0x000001fc1e2456f6,0x0001c124be702b84,0x0007f671a6c9b545,0x0000e07337c72285,0x000d3661d0b0a89f,0x0007db2af0223087,0x0001d9b173b261f1,0x000c65404d0457b5,0x00086eefc1cd30f8,0x0000000000000001}, {0x000fafb34fbb3972,0x000db4c5bd6770ff,0x0000de59815fc7a7,0x000b5602342e8ca8,0x000220e1c9e4f843,0x000b0b7c5b3bfe91,0x000b313a1e2826c8,0x000988ce465ce442,0x000217ce5f2ef9e9,0x000682a10ff59077,0x0000000000000001}}, +}, +{/* digit=39 [{1,2,3,..,}]*([2^156]*G) */ + {{0x00041390c3361b6e,0x000b58054f802294,0x0009b74e1597143a,0x000652a48a901ba0,0x0004b9b4f3635116,0x0005e2ee300afb31,0x00079f7d46228864,0x0001b66e61674d2f,0x00005aad2298ff3c,0x000bd327d6400925,0x0000000000000001}, {0x000b20dd543f093d,0x000dac4b51c2ba0e,0x000bf1d364874c9c,0x0002601310d4063e,0x000c6c8c6fbaa6b7,0x000ce6639ff8b94a,0x000066c91f488ec6,0x000524c600b8f454,0x000ff656ef37706e,0x0008a0434286c21c,0x0000000000000001}}, + {{0x00077e284b1fd44f,0x0001dc37f60445d8,0x00069f0b4dfca419,0x000aa285358c7759,0x0003172cf55e112a,0x000ffea4f47f71ae,0x000412afc352eb30,0x000bc7ffc3d95b8d,0x0009ccbacabdbf74,0x00030dd4b6acd123,0x0000000000000001}, {0x000870d6326f819d,0x0001f9d1598751a6,0x000c925f0b6c6b0d,0x000c309ba890fd44,0x000f1cedd20fe106,0x000408588dc46673,0x00053cbebfbcf6f2,0x000da52fed53b541,0x0002bbf3f7b4aace,0x0008d484a22a2167,0x0000000000000000}}, + {{0x000d633fc6b2523f,0x0003c8573eaf11ae,0x0005f254d2bc0511,0x0001c7fd283764aa,0x000f770135776ee7,0x0003df5ba988759f,0x00065da842051883,0x000b809c0705d522,0x0001067f4912507f,0x000fb628d91a9464,0x0000000000000000}, {0x000e6e2ac33e3aac,0x00065c0000ebfac5,0x000ced796bda6c05,0x000a32836c90c0d4,0x000d2ee187fc8100,0x000d7848e982bcb3,0x000be08290e6b628,0x00085ab586db4a59,0x000b07e2fb9a0080,0x000d56210d8de2f4,0x0000000000000001}}, + {{0x0000ac7a2fd05258,0x00034c744e57f4bf,0x0002edb448a88343,0x0004d56a4c1f9523,0x0003b85d4cde6c8e,0x0003063d710bd23e,0x000833d45b52f378,0x000d2012d08a14ca,0x000ccbe55ff85aae,0x000e919fa9b95c02,0x0000000000000001}, {0x000999b76646e255,0x0001f5f355b09a04,0x00000d64b669309a,0x00089605b2bd55ad,0x000656b121bac578,0x000d693b7220d91b,0x00053ea1faab888e,0x000745d07a303444,0x000e7a52e75e36d6,0x000a7986433618f7,0x0000000000000000}}, + {{0x0002484fcef15b60,0x000485fc2dc91c4b,0x000d7e9f8403b5be,0x0005ec8542217cb5,0x0003f3deede9d858,0x0008c56ddda1f005,0x0009028845902ce4,0x000cdbb111feb2e9,0x000537c2b8f6659b,0x00075d89960f5bf1,0x0000000000000000}, {0x000a9e85b9799e89,0x0001de39c6986f88,0x0000fa555ee69af1,0x000f3b270c555b9d,0x0004c62266b30411,0x00084a11940b0e86,0x0005b26112da8247,0x000fc56950bfb7ec,0x00066d81f8a57ba0,0x0003b772e0aa0038,0x0000000000000000}}, + {{0x000710f55ddf9e13,0x00018f67ff0e8dc1,0x000601481b67ef67,0x00009ffab39b462d,0x000b5ad90ba1057d,0x00018d94f2f83bbb,0x0008d2eed4c7a169,0x00019ddb61a12bee,0x00096ab74dadd029,0x0003902e5753e9de,0x0000000000000001}, {0x000926dcd7de034d,0x000eab5af3e375a3,0x000eb250dce827a6,0x00008bd108623cdc,0x000d49a7d0e9c524,0x00066e3019236fda,0x00040ab55ed033af,0x000667077bc755cf,0x0006972e633b49be,0x00095334396ea43d,0x0000000000000001}}, + {{0x000671c0c20009e5,0x000956db94fffef7,0x000bca8fdc30361d,0x000ebfa5860aa7a6,0x000feca2b724bff4,0x000572f34fd506fb,0x00048ff2e88a7d1e,0x000874822e19430c,0x0003c0129eb20b17,0x0003db07cc6f0162,0x0000000000000000}, {0x000244f5da60b490,0x000fbd8954a885e9,0x000f39699542bf3b,0x000c93a6a7e331fd,0x0009816b29c5180d,0x000ad960c8e85d80,0x0003a3a7931b35df,0x00092e570f2974ab,0x000904daaf442234,0x000cf25e1f700754,0x0000000000000000}}, + {{0x00056e1b7ae8ee13,0x0009b65f8128c4ea,0x000e5e0d92d02840,0x00074f688ed0e1c4,0x000f9c55f66d6f3b,0x000eb2ab8035d3f9,0x0006b643bde4296d,0x000f25e29f7ea7cb,0x0007f5f239b9d057,0x0001af17e3fac208,0x0000000000000001}, {0x00063cbb323c7d21,0x0002b6d926fd3ed4,0x000ab9ee679014b1,0x0007e6093a1bcb9a,0x000dc171705931f4,0x000b0a4387f44f73,0x000c7cdd2a12e513,0x000a473ec3b73ce5,0x000ef17967f341e3,0x000a3809a474c86a,0x0000000000000000}}, +}, +{/* digit=40 [{1,2,3,..,}]*([2^160]*G) */ + {{0x000dc98081f0b504,0x0002377f1bc655c2,0x00067de245fb688a,0x000260cdd7a61e34,0x000d89aaf28a330b,0x0004e078039aeae5,0x000a42253d349d8d,0x000438cabcfed7ae,0x000a9960f3728bd2,0x000af658af568325,0x0000000000000001}, {0x0006e52ab17d640b,0x00019d1bc21ee481,0x00026613d4c31a58,0x000c14072a5969b3,0x0005babfa75ee1a8,0x000c563bc4d35701,0x0000425d2086ecb7,0x000c9b8fafb1a4a8,0x0007ef737c2661a2,0x000c20e7afb2d654,0x0000000000000000}}, + {{0x000726f329838a5e,0x000204b7a9942b65,0x0004a26b80fa33e2,0x0006f40abbf82a56,0x00026970dfcc973c,0x0001c38e96f95485,0x00019abd2bbae55f,0x000c1edd71d62ecb,0x00020adf26d97496,0x0000d917e1cf322e,0x0000000000000000}, {0x000aaf44ac399116,0x00067bb67e29ba76,0x0003d1213c21031c,0x0003b345e37fdfde,0x000dab46f2bbeb4f,0x0003442227ef5d5b,0x0005c11bdace9105,0x00060e12dac175a6,0x000cdb1cf99010c3,0x00087106f2502658,0x0000000000000000}}, + {{0x00044188249a4961,0x000eb8deb1c61cee,0x0006080b71cf8ff5,0x000b75b57b2ccc29,0x000b9ffb3c6aa214,0x0000a50e70e80f53,0x000fd2ffeb156be9,0x0005a94620e80211,0x0005db41e15422e5,0x00055d2030526508,0x0000000000000001}, {0x0009e1933e619307,0x00086b5084131313,0x000b6d55898976e9,0x0003f79536a0866b,0x000b8e06bc0b2a44,0x00034e542863ba00,0x00040e4dd7a73a37,0x0008b2efa3822134,0x0005312ecb0905af,0x0008efb084f884e9,0x0000000000000000}}, + {{0x000172c5ec6e6f32,0x000784d8ddaafa3f,0x000785f2ae4eb6e8,0x000db162f77d65ef,0x00085dec5c58d4e5,0x000a30bffa2375c7,0x0000bb7c92e0f7f4,0x0002294b17a00c92,0x0009107e026f93d7,0x000911ce9dc0950a,0x0000000000000000}, {0x000841c6766f1f49,0x000079724523292b,0x0000e7ddb4cb0490,0x000d47f955646515,0x000b44b2a0877c3a,0x0004c3de4bd8708d,0x0009d24b4a9131fa,0x000585e650ae938e,0x000bb2e4980176c4,0x0001820248559a60,0x0000000000000001}}, + {{0x000cef71c9a9281b,0x00078b2e3e260928,0x000b15a4e8453115,0x000c76cc66031c77,0x000e2f2c06ffcc30,0x000a471db8c352a0,0x000184b9a687b94e,0x0009b19798642e1e,0x0001d84cf08e1a1c,0x000462a36c823a7d,0x0000000000000000}, {0x000b775551fedfb4,0x000c921b0d298e47,0x000071e1319e7833,0x000e6f4ce5e5ae43,0x0001348ff7cbdbf6,0x000c042f31447260,0x00061f1861a992ae,0x00020e5f80d48204,0x0008846b75b72853,0x0005ef4edf14c058,0x0000000000000001}}, + {{0x00051608c9277436,0x00036641c6cf4e0f,0x000263e7b7515b1b,0x000a50636eb6d459,0x000df53679a56041,0x000b4abcaa6ef1d0,0x0005077b47a03019,0x0009d2d427efae97,0x0003dfa9162f30c4,0x000f8bc801e5655a,0x0000000000000001}, {0x000202783ac347e0,0x00001a26d59f4868,0x0003895e6664e175,0x000031f4202e3866,0x00069d2af7613aa8,0x00021cc1e58ddf28,0x000159ee13d84ffa,0x000c8f6eb59a5da3,0x0005df9b7b87bbc9,0x000771b8b6006cdc,0x0000000000000001}}, + {{0x0006aa5bb86ea29f,0x0000e29e7a21c03e,0x0009e430844ee3c4,0x000584b091ca8307,0x0006afb05a033420,0x00015d7ef65dc354,0x000acb0dfae44d05,0x000bad35608c8e97,0x000a78e5d1c181a0,0x0007cad8ba90d885,0x0000000000000000}, {0x0001cba5026e7f38,0x0009593d89eff94b,0x000b88834191828d,0x000881379cd1acbe,0x000c4d9c16250e77,0x000f4d66dbf51b1f,0x000703cbf985d682,0x000998e4fae0e78a,0x000e124668125e5c,0x00095c7096d1799f,0x0000000000000001}}, + {{0x000c267dbe90f79a,0x000a682de8af3a96,0x000fc2373c7d7a0d,0x00046c045ae71058,0x00067b05a94e6008,0x0009ec9a78879108,0x000973f0df20f654,0x0003d4a6c168aecf,0x00050f6bc30604ed,0x000f342722d4210b,0x0000000000000001}, {0x00089badc8348ffd,0x0005ea32767a9d3c,0x000dc1a4baa76ac9,0x000219cd3eced60d,0x0007d2d3cddf3114,0x000c14e1ea557cfa,0x000c466d40b6e234,0x000224ae183077a3,0x000e59e159bfca75,0x0004c30d62fa0c8e,0x0000000000000001}}, +}, +{/* digit=41 [{1,2,3,..,}]*([2^164]*G) */ + {{0x000acef3fd9ab352,0x000a04dda16fb097,0x000e90de3351fbdc,0x0001f9baff197a3e,0x0003610909fc0701,0x0003d538e2bf8355,0x000fbd9d3c214c4a,0x00064b2db047d1ad,0x0000b9e3fa7800e1,0x0001033ba0bb0ce3,0x0000000000000001}, {0x000cb2552f015a84,0x000cdab20301de3c,0x000af7c3af2c8c0d,0x000fe99606c79c8a,0x000d638fb52847ee,0x0009bf56737cb586,0x00000b1ec4f260f5,0x000362ff887d36f2,0x000ed3b693913291,0x0006e3c40f0d7ae5,0x0000000000000000}}, + {{0x0008a516e5648dad,0x00047202a3fb8a9d,0x000ecb3edff5fedd,0x000220d17fb9838f,0x000dc8b9ac40f762,0x0009a8311e23ad98,0x00084edfb615d6c5,0x0004e6c85dc486c7,0x000bf81a7ee5f8f8,0x000ecc58d5bb866b,0x0000000000000001}, {0x000d41ef176fcfa3,0x00078f007acce1a1,0x000d8b8126b20e97,0x000a71b3438944de,0x000134e76c73c437,0x0004a56abd9a1b4b,0x000de8db7385f9b1,0x00043115d58229b4,0x00034725891b4078,0x000c55ba8c32f815,0x0000000000000000}}, + {{0x0008e051939a3b7d,0x00010361ba38d482,0x0009c9a686091017,0x00050f6456db6700,0x00031ba66d450ac6,0x000e0bdd4225b759,0x0006e52d315738bc,0x000d21403d74797c,0x000a041eb53ed58e,0x0002f948ee60c564,0x0000000000000001}, {0x00014e408d1cd601,0x000cfebde9152461,0x0004123bd5f4532c,0x00083ab7b9fe9b7a,0x0009704a01b31e49,0x00040d9c79a4402d,0x0009ff5b8e0a168a,0x00048d442aea1790,0x0001410d782cc3c0,0x0002bf2d98ba10cb,0x0000000000000000}}, + {{0x000943818d1af858,0x0005d42684f68399,0x00068a5f9139d27b,0x000d03a1a3ed9c84,0x00024e699de7f9f9,0x000ddd7e41e31174,0x00089cc967769d86,0x000a0e9e00b5f6fd,0x0007b63934a6926e,0x00011c5b068a8b3d,0x0000000000000001}, {0x0007d21cfa86aa41,0x000f529a2aecb429,0x000251f8677cf147,0x000bad3bd2a35774,0x00090bedc57bbf0b,0x000a31f1dbfe5b37,0x0001e75b3cb7422a,0x0006476bcd9901bb,0x000278bd8b31cdbf,0x00082a6fb171258c,0x0000000000000000}}, + {{0x0009d9b88bd44811,0x00086c7bafe985de,0x000eb018e7fc2f20,0x00037a5b53cb3738,0x0000a097f28e364d,0x0000b5541e546ab4,0x000530e972bd2ec0,0x000b65a95e020994,0x000221ddc10db4cf,0x000295000b94fc68,0x0000000000000000}, {0x0006ace5e2ed6000,0x0008ffd606613047,0x0002a24af3b853da,0x000aba583e1b87cd,0x0004618719533717,0x000d61f56ae2be40,0x00025ef5e9069ea8,0x000f94027fe98e78,0x000db6fc7d9c1583,0x00075271696c0d71,0x0000000000000001}}, + {{0x000f12734c3ec92d,0x0002bb3d48fbed19,0x00049bdd26ff69ad,0x000fbf26985b989b,0x000ad451c21eb61b,0x000237a30e35f12c,0x000a3b3680a082df,0x000188ebe4c92751,0x00087a8fbc731694,0x000b03a8bdfe9408,0x0000000000000001}, {0x0006f89f4e0d5883,0x000d80de19c8b935,0x00077afef27eab9c,0x000538f8f941390d,0x0002b8c79f62a16f,0x0004a907ee9a2c1a,0x000951eb7aa5d968,0x0001fe7d75aa9877,0x0007b983b59fbafe,0x0002bab437db42c1,0x0000000000000001}}, + {{0x000bf512d363aeb8,0x000e0db50c6d9411,0x0000e1101f753fc2,0x0006d62a0ded1b7d,0x0004a1ec0fcb3b8e,0x0000a9719f9cb02e,0x000fa60331be1189,0x000525c569643656,0x00081aa5691f4f2d,0x000a963a9ea530b1,0x0000000000000000}, {0x000098b88fd83f01,0x0009e9aed5969329,0x000769c1597dbeae,0x0003f34dc1aadb7c,0x0006d041d0f773cd,0x000dcc7a18555ae3,0x00057b66cab6672b,0x000c3dbd797513fa,0x000ec420f27eb3f3,0x0003c62ce13b7853,0x0000000000000000}}, + {{0x00057f7ec577ceaf,0x0009584ce56b583a,0x000ce15377e1306d,0x0005b26b1e23a49b,0x000f42d98c317bad,0x000c523283ae8b11,0x00081ddf50073f0d,0x0004ab516099e7af,0x00029299e519277c,0x000c0d8cb7cfdd6a,0x0000000000000001}, {0x00029a85f4a1c822,0x000b7e9213cb42ad,0x000364e5e4a37030,0x000a3941f8a54d03,0x00050b7d507ec771,0x000db1def4d6f8ad,0x000eab3bd493bf4d,0x0009716822a9c65e,0x0005d463b7e2f601,0x000728062fa75d1d,0x0000000000000000}}, +}, +{/* digit=42 [{1,2,3,..,}]*([2^168]*G) */ + {{0x0002ee214ad0e3d6,0x000d51de6a66c4a4,0x000c1ce94446c6c7,0x000c0d5dd2eee21b,0x000e88f8f4a8deaa,0x00055296fd5914a3,0x000dd876c3945207,0x00083798ebb4e647,0x000fd6484696a7a6,0x0001d866ec9d8ec9,0x0000000000000001}, {0x000ca34e120495f9,0x00050701e46446f4,0x0004431e6d90fc27,0x0006b7610b310d40,0x00086e5199614976,0x000e7e80f704e266,0x000aa764f7efe74a,0x0003d9535c6d9829,0x0005a23c25702e18,0x0000a0457bd92a75,0x0000000000000001}}, + {{0x0009cf085024c2f1,0x000c0aff178cc9fb,0x000cd1f6670717cd,0x000588548870fa8c,0x0001a99c44c6bc4d,0x0007a4c31ed62743,0x000f88c552f232dd,0x0008940140f085da,0x000a8211a1d88681,0x00041216e4c1b09a,0x0000000000000001}, {0x0006cac59e6c3159,0x0000ba3374279c4b,0x0008991eda2c878a,0x0003b4cf84ea0b3f,0x000025e729a3932e,0x00047222c0cc5f31,0x000ba94b4346c5bd,0x000e2995032ec5c9,0x000db493f41a4bab,0x00024e7b6e042b7d,0x0000000000000000}}, + {{0x000aefc5789d3eda,0x000070117b5af24b,0x000a5c6b9c3050d4,0x000dbfc9621085b7,0x0006f4c0b7973deb,0x00006f6cf4b4e834,0x000082f092a35673,0x0002d877db7b37b4,0x000c2eac8682b506,0x000eac10f86afedc,0x0000000000000000}, {0x0002caf651b8b0a4,0x0008310eef2a1934,0x00026025b8808ec6,0x000dcff1e64f055a,0x000a67192e09ae5e,0x000d785482258125,0x0007daa7d24e92c4,0x000a9c45e5162876,0x000fc7c72fb7aba4,0x000522976bb5f88c,0x0000000000000001}}, + {{0x000921c0f982798b,0x0001a2079475b7fe,0x000ea0fd52e410ea,0x000e44af77d4bbcb,0x0001f260a54b0212,0x000269af2ec66a7d,0x00034794993bda84,0x00050b15e358d04f,0x00067a4d30bdfadc,0x000e912250ea3d1c,0x0000000000000001}, {0x00083de4fe7bebfa,0x0004fdfb63579e27,0x0001abe0cbad5ac8,0x000820014b8a145c,0x000a85d987c51840,0x0009eba9aacfadab,0x000291af5fccfd5d,0x000785e551a982de,0x0002bcee4372c455,0x00049c9d89842d5e,0x0000000000000001}}, + {{0x0000817678d00826,0x0003072fc12b3906,0x000fabe24fd4868f,0x000b2a4f9e0b8813,0x000ccd87b27441cf,0x000e7fd0a48234db,0x000ea747d9a2a9fe,0x000c91ba0d4add56,0x000c9d0dd2d3e7c8,0x00005660e4fd17f4,0x0000000000000000}, {0x000a904f88c1be1b,0x000e5cce4c6964ff,0x0005fb6194a74952,0x0000f033d222444b,0x0008c26fbb11965c,0x00055ed1ac1d1bab,0x00020c09d970630d,0x00075b608324cefd,0x00050cf259835d15,0x000462cd49bc1143,0x0000000000000001}}, + {{0x000b46239d54de36,0x00013af2871bf6df,0x000bb6d9f31a1b7c,0x000528b0f5b2569d,0x0006b9497778a81b,0x000c963043af6788,0x0003bf9954a12672,0x00059feeec8df36a,0x000d60c22b5fdead,0x00060d265f0f8b6f,0x0000000000000001}, {0x00050d0b6534b1ea,0x000703b71e08797c,0x000d2bd35e284a7c,0x000105aa68827a45,0x000902245c12e4ca,0x000f1afdb8eeeba0,0x000caa7693b8db6c,0x000824a39f45018e,0x000945d0c9d12756,0x00055f86289ff82e,0x0000000000000000}}, + {{0x000f246813a5e2f3,0x0004d2dd1bc96870,0x0005743352958a8b,0x00007a9386d4a79c,0x0008b4a29091d043,0x00059ba9e47bd2da,0x000478de8a606e11,0x0008c9f2a8c27ae1,0x0002946224c1c93c,0x0003ef3adcff2ded,0x0000000000000001}, {0x0005100b79546483,0x000ac268f6c48348,0x000e4b2ec17a54ad,0x000f85818d6815bf,0x0004f425318546e1,0x00013cbacecaadf3,0x0006d908fa2a9c92,0x000e8808196d6c46,0x000ad801f4a291fd,0x000b7ecba0623fa6,0x0000000000000000}}, + {{0x0007b0f39088ef39,0x000a435ae74e03d1,0x000fbdcdaf3b17b1,0x00090868e5084910,0x000019102285b63c,0x0005454d88d8e63c,0x0009e2380d185fed,0x000af9e19dfe50f9,0x0008e09d7ce8d3eb,0x0000155127749872,0x0000000000000000}, {0x0007fda1b031ef4b,0x000dfd7188feeb77,0x0006801e0f6f597f,0x000e9d1729652f82,0x0009d58dec034252,0x0003cc68d0c6aa0c,0x000a4e76779b37e4,0x00008d509f569c62,0x0000c41330558ca7,0x00056956b5657bd9,0x0000000000000000}}, +}, +{/* digit=43 [{1,2,3,..,}]*([2^172]*G) */ + {{0x0009008a556937c2,0x000f76241f10378e,0x0004e7ecf0092193,0x00097f48f8905d70,0x000c86b4870ad280,0x000f86eb6f389aca,0x0003ffbc3b9a3132,0x000a9c6b9598fe5a,0x000429f1014fb463,0x000e06408908552f,0x0000000000000000}, {0x0000c94ae41024de,0x000a6dd0399afa53,0x0007da5ef17ac70c,0x00080b49854eb299,0x000104afd62b1e2c,0x0000b1375777d7cf,0x000794db8dbecfef,0x000ff21b1b05dfbd,0x000f1e68e47404db,0x00080928abdaf296,0x0000000000000001}}, + {{0x0001636898bf4d11,0x000fb75ab01bffaa,0x000ba4b1f3e58bf7,0x00059cab50bc67e3,0x0008acea4689ce8b,0x000f1932a30d30cf,0x0006e5cb3d1d8eda,0x0005fa7949e492c0,0x00041db2db16d8b2,0x000b0610d851b96f,0x0000000000000001}, {0x00054a2b36667691,0x000ca196d36fe2b1,0x000766e2a6109d47,0x000f9263f1863dba,0x0003be92b5a5ba8e,0x000aad9918a5da16,0x0009189520c8298c,0x00010a27963af5e7,0x000b8c3b84ab05f9,0x0007bafd0103c420,0x0000000000000000}}, + {{0x000e3e45f8d73683,0x000eb8ebb6d11caa,0x0000f274ee508fde,0x00020c83562c576b,0x0008510e47baeee6,0x00079b810588c571,0x000894a919ff42e2,0x00007edf259b927c,0x0009d16a223100a0,0x000a5b2acb9ccb16,0x0000000000000001}, {0x000e8415a9179d06,0x0004b594d74f07fe,0x0004fb6e0e5cacca,0x000788b708cc549b,0x000c0c62edae508e,0x000b9ef886c2847d,0x000e40664c8eee69,0x0003ed24b57a9dee,0x0002b9d44a547432,0x000a6f16f12261ca,0x0000000000000001}}, + {{0x0005496bc42e6e51,0x000d33d320585033,0x000cf402bd388067,0x0003e6730074be0b,0x00043e8db4e94291,0x000f7beef462a0cb,0x000e58a8c2ead81c,0x000b97eccd5df06d,0x0005954e3501f23b,0x0004b4a8b8e4e11d,0x0000000000000000}, {0x000598c1e025da1b,0x000b09bf9648fc1d,0x000d224f8ad9987c,0x00065a60d88fba1f,0x00054d86a1d9f606,0x0006c4ad1df1e7f7,0x000e1da4acf77f72,0x0000938971a27713,0x0007fc94e0f78da1,0x00083992811d7d3b,0x0000000000000000}}, + {{0x000c9cc6dd4a5914,0x000862d80be96443,0x00008c7a249fd0f7,0x0001ea54a0d2c9f8,0x00048d9e55013f6b,0x0007ab76e8d002ac,0x000cbf4462d73cac,0x000faf5cdb58c492,0x000ca322b819e5c1,0x000b840745406425,0x0000000000000001}, {0x0005f739a14940b4,0x00097d20ee2a886c,0x00035ab04c341a53,0x000f7a9d2904ba7b,0x000d9cae762f47f9,0x00007d2eaeeeb5be,0x00070ab079042e0b,0x00060a339ed63e5f,0x000c7b9658ba8e43,0x0002d18d85499745,0x0000000000000001}}, + {{0x000b401e3813737f,0x000ff61118cf0239,0x000909b65cbad1c8,0x000da081fe099573,0x00028cfc70caa9fc,0x0002bfc31062db69,0x000e85af4aac9c83,0x000ba3d1d4e51a7e,0x000363dd1405fca7,0x0009117097ec2370,0x0000000000000001}, {0x00065c78edf4f376,0x000f36321b45daed,0x0003283085bc71ee,0x00018a85f5b5aa08,0x00055758f2c2f181,0x0001623497212c90,0x0006ecdfee014f92,0x0000a48aadb790d1,0x000881e12f4528d4,0x0004482acaa286a1,0x0000000000000000}}, + {{0x00050bbeb7940a25,0x000eb62b0dec1d2e,0x000f7146dd0fa42c,0x00051beef911c829,0x0009947cccb28e02,0x0009505362c5e903,0x000767ef06d55451,0x0003dcda2ee6b16e,0x000b2373652d7be8,0x000cb5af116c86d2,0x0000000000000001}, {0x000d36b874b6449e,0x0002d7bb963c1def,0x000de8609229e57d,0x000375fd0ce127b2,0x000b5890dc213b70,0x000a1d88db3c82d4,0x0007b583b09ebdc9,0x000cc6b137b10089,0x000cf29a5fc13efd,0x0001907605ca17a3,0x0000000000000001}}, + {{0x000665e2bbfdb04a,0x00017e4232c5cb5c,0x00026232f5f9a245,0x0007a0275981cd79,0x0005ae253d4d80a2,0x0006c00bb7783b1d,0x0004c2589b5ab0bd,0x000f6c48caf740ea,0x00082e177fc5351c,0x0000fd2b0e714ba4,0x0000000000000000}, {0x00052b5ad6ac73dd,0x0007a311881ba785,0x0004ccac10cfb206,0x0003dcbe5d449097,0x0008b8873accd901,0x00080d70e5b2cf2a,0x000440b2c2817333,0x00067c3b711d4631,0x000b996623747bc6,0x000383423c70b2d6,0x0000000000000001}}, +}, +{/* digit=44 [{1,2,3,..,}]*([2^176]*G) */ + {{0x000b11d1789dc869,0x00016c2eed227fa8,0x000916842cb7fd9a,0x0008564ce12a5d02,0x000bee59ed474675,0x000e675f354b48f9,0x0005d69ece126be8,0x00018ce3aca2f7c7,0x000768d6000f88d2,0x00090f26ea6ff29a,0x0000000000000000}, {0x00096ef4ce69e270,0x000f2da0efb2f05d,0x000a99dc276ac3a2,0x000e0342757c443d,0x0003b390d2a5e23c,0x000e7ea78e9b674e,0x00085e132e72b987,0x0006b6c21856dca4,0x0005bed8cda17d0d,0x000237220788bdee,0x0000000000000001}}, + {{0x0002364996ca25a6,0x0007ec8d70cd440e,0x000d8467c5161afc,0x000408724c9aa882,0x000b962a215cbbc1,0x000c4986c1ad6d3c,0x0001332912aaf7a6,0x000d6db2702d8369,0x000a17e017d4a1ec,0x000f2dbf2405e93f,0x0000000000000000}, {0x0000641168090c5e,0x0006fce42ae3e68a,0x00039938713395b0,0x000394a15bb1098f,0x000db97734c1bca3,0x000edfc62ae8c0be,0x000bc2f9b0452cb1,0x000304c79c90c661,0x0000dada4e625332,0x0006fc2e4ae4342a,0x0000000000000000}}, + {{0x0001bd7fc156252b,0x0000ddb25b337fb0,0x000ac5d025ae1e66,0x000c26056a73c379,0x00095dede6af2b69,0x0001ae9121b7e81b,0x000754f6cd030d2a,0x0008b47d1e9a5f7f,0x00025d238c9b7c0f,0x000d32d6fa902ce9,0x0000000000000000}, {0x0005ace423d94184,0x00056f6ab6a655f9,0x0008fa78d47709a3,0x0003f5d39d32f258,0x000beab0a90b8c58,0x000bc517995d68b6,0x000ea4acae9d65d8,0x000fd569fb104a80,0x000b09db02cb3b12,0x000c624b3e1e5f67,0x0000000000000000}}, + {{0x00093cbca6dca0b6,0x0004a2146559221e,0x0006c357ebc20032,0x000e73dafbe29569,0x00073f1c77b70537,0x0002b0a8e959d415,0x00055d9c50a71dc1,0x00037c9b3656d184,0x000283b617fcbc17,0x0000976acf8093d9,0x0000000000000000}, {0x0008b573715b4734,0x000173f0027024fa,0x000386bfccf3b38a,0x00095480bbc99c54,0x000668bfbf241bdb,0x000353dffbcc88d5,0x000216b7968e8858,0x000de22f661faa2a,0x000189437f0cc373,0x000c1f5601679c0c,0x0000000000000001}}, + {{0x000a86967182c501,0x0001f634e40148fc,0x0001c864ffbfa398,0x0009d6d142879632,0x000443e4b6047507,0x000e1e5a879eef57,0x000d2b8fd7f7f136,0x000d19b6378838d5,0x0007815ed1c2726d,0x00042ef17abcb4c1,0x0000000000000000}, {0x000b9a5999895b25,0x000be140e558227b,0x0007f28ae923d146,0x000d00a58852f582,0x0000e60ada16c8cd,0x000158a85a7def11,0x000e5c61d1152d28,0x000d4be61bf1a55a,0x000cf413c0a31606,0x000b3cd625cdfd8f,0x0000000000000001}}, + {{0x0003f2f8ccce2027,0x000fd5cd45c4a564,0x000a6b2411224a0a,0x0007a5ca2258c4c8,0x000678f855fedfa8,0x00055199f43975cb,0x000e9a39edc6298c,0x00007312684e5a48,0x000adaeb9f55daba,0x000b39c9f5f377bf,0x0000000000000001}, {0x0003e0968382a7ce,0x000869c70ffd115a,0x000ba001f2afcccb,0x000107bdfe8068fd,0x0000206868f7c124,0x000821a90928b9fe,0x000afc533728dac3,0x000b3e9edff0ac94,0x000d10c697f67565,0x000bea250773ba0b,0x0000000000000001}}, + {{0x000275a2c8f91400,0x000bb4c241f78224,0x000c4fd93b4ba60f,0x0002941b616268c1,0x00020107f7964087,0x00031b438825e04f,0x00019247786625f8,0x00028de20083c5f6,0x000abde39791c6d5,0x000b3b75c25ecfb0,0x0000000000000001}, {0x0008e09f47b9d8c3,0x0009374c6bc5ceb5,0x00038e27941b3112,0x000e3235cee2666e,0x000ea8dbee896ca0,0x00030660009b498d,0x0005f8f0f2897645,0x0000fb5ee44458ff,0x000fb559aa7b5e14,0x000272ac85e138f9,0x0000000000000000}}, + {{0x0009f0c6193905a5,0x00013e99256667bb,0x00027fdbbfc34892,0x0000d2c71218ca33,0x000915a83f00e563,0x000d628331bdc8df,0x0003e8128ee96b80,0x00016a5f7e06bfe7,0x00016364a2a7cd33,0x000c748cd2a08bdd,0x0000000000000001}, {0x0001d90fa51d3800,0x00020c814ecb8822,0x00000fc79208b5df,0x000f252076343a10,0x0008a14b68032c99,0x00054fe0dc71413b,0x000d97c9a173cb46,0x000c85a386e9a9ac,0x000bf160a14a40bf,0x000032849e997087,0x0000000000000001}}, +}, +{/* digit=45 [{1,2,3,..,}]*([2^180]*G) */ + {{0x000384b0dc2ffbb2,0x000e0c16c289b477,0x0009eabe48cf9601,0x000199d671ddca51,0x0006f3fce7863b3f,0x000e01be3ea3ecba,0x000b70167c58c7d2,0x000f4893679afbf4,0x00019a4362cb78d1,0x00061515a3d7fee0,0x0000000000000000}, {0x000f2840f746e722,0x0002ef160c51fc25,0x00097156a16516e7,0x000e8398d9625db3,0x000d63f5b2c0ebf6,0x000c5b6523651404,0x000476dd10c4d87f,0x0001f40ffa318eef,0x000788025e5d3977,0x0003c298fa2547e3,0x0000000000000000}}, + {{0x000523e81658a625,0x000aef8e050759b2,0x000b0377d5042659,0x000b9ae72b36823c,0x0007eff957169419,0x0009705cebf46fc1,0x000bd18b61ce7ad5,0x0007a7135b602fff,0x000f2e092fe9192a,0x00074d30a3a8e596,0x0000000000000000}, {0x0007c895ead96751,0x0006523da4889766,0x000467afe86eb732,0x000a5ee25b7a7cf8,0x00000f2568e46393,0x00079a3304b15dd0,0x00036bd203f1569b,0x0009a5e938c0d91a,0x0001da1271a34645,0x0004c688c575bf52,0x0000000000000000}}, + {{0x000c62a6b633bf04,0x0003c0eaef0121c8,0x00058d7354098cc5,0x000448cc925273a9,0x0006f73c56bf4c04,0x00042b800bc52be4,0x0008d6b39147d475,0x000444cb5cfe3029,0x000d4247fb2312e0,0x0007054c4d89dd9e,0x0000000000000001}, {0x0000edd6a97a9163,0x000582ed4f4d5b46,0x000b9ca61309206a,0x000fafa93e18c6dd,0x000bda68f9bb8a3e,0x00070a52c8b2d783,0x0007728c0dda564b,0x000c0dc789e7dbe4,0x000119aa3e8a6481,0x000bed27f421a4e4,0x0000000000000001}}, + {{0x00001ee133405081,0x000b94055dadf3f3,0x0008803374bd3d6a,0x0000e431a078817f,0x0000ae1298465c73,0x000a08da98aae817,0x00076bc8b779119b,0x000c1b8f7410f128,0x000bc98dcbe46247,0x0001761805980867,0x0000000000000001}, {0x0009de67dab5cae2,0x0003d2d0125b70f5,0x0008c5ad3a01682d,0x000cf59a9c7c1b26,0x000ada095cf6362f,0x000b79b1ed6482c8,0x0002b3bc253c84e5,0x000756917d1dd695,0x0008f439fdfad9c3,0x000651a63232aa5c,0x0000000000000000}}, + {{0x0003a055275e1f13,0x000bce620ca4b51f,0x000765c9fcc48133,0x000387e5710e23a7,0x00041d9c294797a6,0x000fe4eedda6621b,0x0009f733bf9d9ac9,0x000e4cb8a3045df1,0x000d5c96c4f51d70,0x00041a25c50ad245,0x0000000000000000}, {0x000acd86687a04f6,0x0009bc4b6a5c45b3,0x0003f85a2b09f7d1,0x000f69420758494b,0x0007e554c9337d50,0x000ccb9c2f40c240,0x000e482c5dfc1a60,0x00016ad44e8b11e7,0x000d080e60fea531,0x0001889fd549f4ed,0x0000000000000001}}, + {{0x0007e29c5ef5cdd9,0x00046b2b2e558b7e,0x0004702314f3e6bc,0x00026fae56eeaa30,0x000145ca44a1b067,0x000ea8da792ee6f2,0x0007e4c829cf9680,0x000d723cb279141e,0x000c514c645b326c,0x000b3d5e8e8931da,0x0000000000000001}, {0x000e5ed0862bd48f,0x0009404a34e74e61,0x000e1d4a98483644,0x000f45001f65c56b,0x0008e062ee7183e5,0x000a39ef75aa764b,0x000f4509012ed646,0x000742837f0ebdde,0x0009ab588faa786a,0x0008d7474accf0f8,0x0000000000000001}}, + {{0x000f31aa7add26e0,0x000b5f70683b341e,0x0002190eb5f5ed33,0x000e3b2bf3278604,0x0002cdb29e4008f6,0x00042f0700c911a6,0x000f5e3688f5189d,0x0007c2de5c257eff,0x00089c193e2d4667,0x000cca5de47c9861,0x0000000000000001}, {0x000dddac10383cca,0x000803caddccaca7,0x0000778df17cf555,0x0009278c5faf93e7,0x0000e7cfbb523b02,0x0003ef004ba7546c,0x0007290d52d052d3,0x000f54a34c36c895,0x000e1b89dcc555fa,0x00058777136cbca3,0x0000000000000001}}, + {{0x00060b5ef6c20e82,0x000bf430fe1ead47,0x0003a480e70d1479,0x00097c0aba684ec7,0x000549990971954c,0x000a1c5645d306cb,0x000cc85cc5c264ce,0x000739efac323d9e,0x0003b20c4465cbfa,0x000b4ee9cad749e6,0x0000000000000000}, {0x000242934808827e,0x0008a9860bc18213,0x0007a452bdb41b29,0x0006c3f651ceda44,0x000f153ca2965078,0x000e0cd8cc7845a5,0x000c9cd5913baf87,0x00050312de2e060b,0x000a1444279bfb31,0x000ff8a16f8265f7,0x0000000000000000}}, +}, +{/* digit=46 [{1,2,3,..,}]*([2^184]*G) */ + {{0x00033bcc8924eb55,0x0002e9d518ffb740,0x0001ae0cd732da2c,0x000cfbc19a4290d6,0x000d8784c1f06357,0x000fe209893ca1ae,0x000969a85a8dedb3,0x0009c8eb2e932f87,0x000bcc740550ff52,0x0004f554bf85aafb,0x0000000000000000}, {0x000bc6372d7f8438,0x000dc0557f4f2ed5,0x0009d0c30f3c2efb,0x000ddabb262ac2fc,0x000f7a05b87d4d5d,0x000c91e745769d1c,0x0008c994d0a4907f,0x000889250072dcd3,0x000ffae1ac453a28,0x0002a8e72458000d,0x0000000000000000}}, + {{0x000d9177fbc76a5c,0x0003975e3cbf7406,0x0002f09def039ed5,0x00034da80caf736f,0x0008efcc790febc5,0x0000ad47e746448c,0x0001ca336b92fa7b,0x000b90e92c64c767,0x000dfd8d6637080a,0x00032f5711517b52,0x0000000000000001}, {0x000a0257ff4a1581,0x000bc02441238656,0x000364971ed77234,0x0008b1d09b2d316f,0x000bfdf4ae5e00cc,0x000468fa8d307856,0x000805be3791c041,0x0003fa589236fc69,0x000a337620c1fe73,0x0001e737b5760989,0x0000000000000001}}, + {{0x000e1990ae2c44bd,0x00083ee2175b6a29,0x00005d1fb5989698,0x0001375eaf1c4bd3,0x0009f54aec5725cd,0x00017c1f0f7d222f,0x000f9f96b74e2d73,0x00053fff253f88e6,0x000b31a11aea1b12,0x000aa32748b4a307,0x0000000000000001}, {0x00007b074c7b461c,0x000258e6e4224b52,0x000162fc7af983c9,0x0004966825052f5a,0x000f9138a0346a4a,0x000a7041242b7952,0x0000a366f5699476,0x000460c88c7d5eb7,0x0003f2b3125e31d3,0x0002a892d14ada09,0x0000000000000000}}, + {{0x00001e6a21a7b432,0x000c3971b4886b8a,0x000dae7cb7883120,0x00059d28f3efe6ce,0x00028e1699713fd6,0x000252af65756250,0x0002a3acd7c4a210,0x000f7efc9c5a81fe,0x000e2a5f82fab4ec,0x0005c0924441558a,0x0000000000000001}, {0x000495dd493563c9,0x000ccadf9b0e7295,0x000ab5a8f70bb0fa,0x0005b32501ed29d2,0x00036f439adfe6b6,0x0000a6c7202e9c24,0x0006531bcd403e24,0x00064526a2b69777,0x0001dc2d590cd125,0x000b64170acdcfa6,0x0000000000000000}}, + {{0x000b0bb5e03ac48f,0x0009bb5837030273,0x000a05cded5e6ec1,0x0008034dce79ed12,0x0000fc54532094d5,0x0008fcc6c534a769,0x000bca87d8be0fee,0x00041150d981f9e6,0x000222345254c456,0x000b2aa496b6e112,0x0000000000000001}, {0x000d2c8de5bb7e5b,0x000278f0be2794e5,0x0001b31bbb57b1c7,0x00001958330187ab,0x000f5dd751abf9bc,0x0002de4b57d090a0,0x0009cb74fbe565f2,0x000f83e310b95170,0x000d1301cd0ee2f4,0x00049fd2006501c7,0x0000000000000000}}, + {{0x000e8d883bcef9f8,0x000be3770de7cc8b,0x00007c65e3e95107,0x000ac96f780e3eca,0x000413d615089cf6,0x000585b5b22549b6,0x0008b5facd5da79f,0x000f3c8b5c5a4c0d,0x000d6dfaa970b49f,0x00065cc025c0e7ad,0x0000000000000000}, {0x0003c64dd34154da,0x000343c797b7cd0b,0x0001f367813bc308,0x000fbf3f138ae118,0x0006f1f8c6302e7b,0x000f35ea2ee3cc54,0x0003a0b904ac34ee,0x00052596f106852c,0x0006e533ab1310ec,0x000abf763b19381e,0x0000000000000000}}, + {{0x000d9a73e24c887c,0x000707461d095f01,0x0005d3ad552ce968,0x000f402b6c527f5f,0x000818672d6016b3,0x000279bc4633dd66,0x0000c571c90fed28,0x000cee78b5512020,0x00048d6ae97b4812,0x00055292fa8b91d9,0x0000000000000000}, {0x0004608b8c1577cc,0x00058615049c4716,0x00077fa05c3b187a,0x0000d33110dc1846,0x000554a923122c03,0x00015b3d3cf40b2d,0x0005e05a3843c4de,0x0006438408a6964a,0x0005f646af7c591a,0x000ae89a0f7132ab,0x0000000000000000}}, + {{0x000b5e462dd556b6,0x000f3aab5e9c2f29,0x00040c3ae00c87a1,0x000aade98fdfc7cb,0x000f2f671ec86f72,0x00069dd7b2aa376f,0x000b6f90c4b07483,0x000ae5c39e831a9e,0x000ef6929b8bdb31,0x0007125a4c5224c9,0x0000000000000000}, {0x00080ab908d10b8d,0x000b7c8a32a9943d,0x00051b7d4fd0edbb,0x0008eaa89eb83ad0,0x000fb343de0ebbe0,0x000d3c4d0cc33cc9,0x000b15124b0953fa,0x0007582773fc9c30,0x000ab2c193a021a4,0x000b73ddfb881675,0x0000000000000000}}, +}, +{/* digit=47 [{1,2,3,..,}]*([2^188]*G) */ + {{0x000f014ad29ca649,0x00075a10e7e9c3d6,0x000042dda6a91edf,0x00069276fbe9047f,0x0005a497f91416df,0x000b982ab7fce403,0x000b8b61e6adadfa,0x000d218a9fd9d973,0x0001c8c04e2c23f1,0x000cb12274d47d9e,0x0000000000000000}, {0x0000ec3de397b98a,0x000d9a272cecd709,0x00050e492db6d724,0x00082a50e32d2f19,0x000db6bf40e9c68f,0x000b25727f0678af,0x0007a36e6ae78194,0x000cbb096d1806b7,0x0001afd3feedfa35,0x000e57c17d9b9ff4,0x0000000000000001}}, + {{0x00078e57ab05c549,0x00081a123d2b219f,0x000ecb0183ca3cf9,0x0008ed9f1eddfd07,0x0002f8f90e3c6699,0x000ad41bb20e0515,0x00019c77dab5c5ef,0x0002ca7830069394,0x000ae5cd1de605b3,0x0003933d6039cc98,0x0000000000000001}, {0x0000ae5b05bb2b74,0x00071168c4bf8259,0x00001a66f3efdf4f,0x000e1da4a65b0015,0x000ba0665dbdf241,0x00015f360d4c3387,0x0004e85c88fe301f,0x000c061a8e048acf,0x000bcc0119ca9957,0x0009ea8585dfcf51,0x0000000000000001}}, + {{0x000c2a4869f8ca68,0x0005748af64adfdd,0x00044c661ce61bf4,0x000fa33532bda5e6,0x0004409ebd1c4df9,0x00063b107af0f45d,0x00013ef2dc8804b9,0x000a186c6c3f3d75,0x0005a7ed73f6b67c,0x0000b525bc5b72fc,0x0000000000000001}, {0x00008fa0f93430b8,0x000bf3cdf63616c2,0x000a2ed8b79415ed,0x00008df169f3be1c,0x00006a9b7669442a,0x000be581a2acda6a,0x00093d2dae365779,0x0006ea987608134d,0x00059b31b3052589,0x000d8078ebfe0716,0x0000000000000000}}, + {{0x000a0e4faa59f069,0x000d33d96e52e437,0x00029f2632f3aebd,0x0000c85e4fd15682,0x000de4f3be7892c7,0x0007d9fb180a1634,0x000b175638b44c2c,0x0002e33499b53e6c,0x000cdad290b60dc3,0x0009d1cf1fcbab2e,0x0000000000000001}, {0x0001854cefb1da9f,0x000257b5b7539f5d,0x00096df1240b9d47,0x000f9e9a561ffc72,0x0000d6d9452715e9,0x0005aea9109f0df3,0x0005d521e814b452,0x000c7037d6e74c47,0x00060afbb2239aca,0x000dfe3a178a1e6b,0x0000000000000001}}, + {{0x00085fac7aac5d24,0x000e6efcd816f4de,0x00001aa65b85ec90,0x00042ef288a94272,0x000527aa3e5baff2,0x0002c225dbf973aa,0x00073237d80e262e,0x000ac84d268b446c,0x0000056980d17882,0x000089041c6047c7,0x0000000000000001}, {0x000f5b36106ae92d,0x000b658d82ab92e5,0x00036754a0792779,0x00032fae5cee6721,0x000bd0394c90fe6b,0x0001825f86d43336,0x000fa2e7d57eb880,0x00022294f3c7cd21,0x00070bb4a5b81f76,0x000e5c797152fbf7,0x0000000000000001}}, + {{0x000d4560b8c46ee0,0x000b73ca40a188f7,0x000b593661397fde,0x00026cffd18ba8cc,0x000c47a735170abd,0x0002887e49a94d3e,0x0002a2ec01caeb9b,0x0007426e36269901,0x000522aa9c52b5bf,0x00066af9e1ef518f,0x0000000000000001}, {0x000d742afa309fe0,0x0006f810d9df60b6,0x000084e739c300f5,0x00056da63d1a036a,0x000cd4f33c2df42f,0x0000b456fee1f0f2,0x0008f569e5589fab,0x00062492c9b09b89,0x000079adb7885960,0x000cf412b618c75e,0x0000000000000000}}, + {{0x000e60d6e9faeb2f,0x000298054778495c,0x0000f99d00d5919a,0x00052dad049c1204,0x000e8b4f342e902f,0x000c225bd4b3fa2a,0x0004f48aaba21576,0x0009d33ad4b48b96,0x000a24fa9d5b26df,0x000f2799a4fd763d,0x0000000000000001}, {0x00017650a566263d,0x000e25ee8cfe1893,0x000763623662e408,0x000a4360ad7e6d05,0x0000ba65a8e233ee,0x000338261baaa5ea,0x000af51b8dde194d,0x000003aaa23ac37d,0x000ab55391298a43,0x0008f52d2a4172a8,0x0000000000000000}}, + {{0x000760e887bee904,0x0009f91c8cff613a,0x0003af1d33766225,0x0002f55a798ee44c,0x0002c7171b76304c,0x00051b89de6b4202,0x000754ce995dc454,0x000a5d7e90f40166,0x00037fe2c45f5e9e,0x000d835f81a1be14,0x0000000000000001}, {0x000c04a7dafbcd8e,0x0007d22e0b1aaf34,0x000ad92815662ecd,0x000b88ed3ef4d947,0x0000190778cccdc8,0x000ea32bf7d0a755,0x00094ed615d6df41,0x00066cdce7de3703,0x0005d94b6a5a2d85,0x000869b1500a755b,0x0000000000000001}}, +}, +{/* digit=48 [{1,2,3,..,}]*([2^192]*G) */ + {{0x000556d5fcb0ca3e,0x000bb40eb6f5de7b,0x00092b00751e1fef,0x000d985badf10d77,0x000bda78c0fd8245,0x00097cec621ec6c5,0x0009de36f6534761,0x000929578b20f59e,0x00008129148b6a34,0x000260858df1e4dd,0x0000000000000001}, {0x0002df7b80140bb6,0x000f0872cf54b64a,0x0005ba02c9e702cc,0x0006a4694fa2136f,0x000ae62ca46c9431,0x000a69d6c72a601f,0x000ff0af210ce686,0x000a108647e23ca9,0x00072d54b7301dc8,0x000b4fc0d011e4bc,0x0000000000000001}}, + {{0x000c2c9272139ecb,0x0008248890056b04,0x000319a82e4c5944,0x000bd6a55d37d95b,0x00074d80dfb735a8,0x000b368732a7edec,0x000dbb960fac47dd,0x0009b7d149244f46,0x0005c8153e4ae15b,0x000dd7d6f5637025,0x0000000000000000}, {0x00023077c37f59fa,0x000e01c814ef1183,0x000d2dfe1b52b965,0x000d66c5cad600e8,0x00064cd44f8d02cd,0x000b170f04ad1f49,0x000b95d6b03da74b,0x0009721ac42809f8,0x0003fd08dc3ee705,0x0005bd69cd062aab,0x0000000000000001}}, + {{0x000b883d845a0ae0,0x000afbae353f2a2f,0x000473d0adeb61aa,0x00037ae3890f51bd,0x000480f0c4103d1b,0x000087e22deca493,0x000ae96c669a58f0,0x0000d7ec27b93462,0x0005f63a771fe3af,0x0004035d6f692734,0x0000000000000001}, {0x00047ce82b6063a3,0x000a032d78ca1a20,0x000ffe80d92bf2a2,0x000357128144148d,0x00065f437565141e,0x000044794e70453a,0x000ed9d74d6e72f4,0x0002c9dec888c3b6,0x000d35b1703c9efa,0x0005564a8b5ee101,0x0000000000000001}}, + {{0x000408c0ef420ccb,0x0003751466bdddb5,0x000686318317d090,0x000f5c351d77faab,0x000e6e1c56990fd6,0x00044f54fded7bc7,0x0007a03658746405,0x000acb87ac9e9b1f,0x000e1a951d060b45,0x0002d4d46b22b133,0x0000000000000000}, {0x000b308639e1f9dd,0x00088fb9f340687c,0x000545e0d0da3dfc,0x0007c0127b5897e6,0x000708cdc1322bbf,0x0003ce8bdc5bfb35,0x0009aefe13ad9991,0x0006b6cb7333e158,0x0005d1b9d92265f8,0x00065e9dfffba15f,0x0000000000000001}}, + {{0x000dfc8b990a47ce,0x00010afcf7dc8ef3,0x000e84517ba6c8fc,0x0002906aab79d6ea,0x000563aab2198045,0x00005b6fc1e2c314,0x000e1c75139c8775,0x000f87b03c5339f9,0x00013db5c7dd8d56,0x000be729d4f04194,0x0000000000000001}, {0x0003ca2969a49a45,0x0003d7263734a805,0x000a96220f01659f,0x000a81ea503a05cb,0x0009a4b1e826b6e7,0x000b9124852cc317,0x000a53155592390b,0x000adcbcb9064287,0x000f6cda7ecb3423,0x000745fa7b75f585,0x0000000000000000}}, + {{0x000020f9870c5652,0x0008ed2cf4bf5440,0x00043a293130a211,0x000b5775772ea602,0x00033659b8c321b9,0x000b8b914e8eddd5,0x00016d7acd2ea084,0x000600a84078289a,0x000073293df2fe16,0x000f1b7632be02d1,0x0000000000000000}, {0x000ae60c00fc41df,0x000dee1aec5298cf,0x000dc3a384668cb0,0x00039330262dc7a6,0x0008200480e942b2,0x0007f1ad6908e660,0x0004d8902ca00250,0x0009dfae47555ead,0x000dd7fb7e96dfa4,0x0004d09336664145,0x0000000000000001}}, + {{0x000d1eaa79614564,0x0008f53026152677,0x0004e1ef41c569ec,0x0009c61bebc6f47b,0x0005a764d392b91a,0x00081c91fd03bca4,0x000d12e91e91f33f,0x000ceb9cb7de2868,0x0000e9b7cc6516bd,0x000c8bcc47c28272,0x0000000000000000}, {0x0007539aee683a33,0x000df86171edb94b,0x000cb40fe4798476,0x000b93f35533bd14,0x0009da702f13dc6d,0x000ba7ad4d348188,0x000a392741f6a108,0x000c52b97e5c2cce,0x000568205c383605,0x000d4f58766e7a6c,0x0000000000000000}}, + {{0x000d3d9203521aef,0x0003dde5091b5f60,0x000ec304735ae314,0x000afe69e360b755,0x000f3119298c9f78,0x000c6a7738e3ed2f,0x000298a24d640365,0x00078b486bf006b1,0x000e9050b3448a96,0x000ac46d50f02b81,0x0000000000000001}, {0x000b1ce68a1d8699,0x000b8559ff13a9f1,0x00023011f5efc1bf,0x0004e57b1d2b17a5,0x0008efdcb9ac6bec,0x00009c3a1153d5a5,0x0006b2a4b16461a1,0x0005a5edc709e6c0,0x000d62c80c93e99c,0x000b0218529aa94e,0x0000000000000001}}, +}, +{/* digit=49 [{1,2,3,..,}]*([2^196]*G) */ + {{0x000e7cc655ddb9f3,0x00013549c78f7abc,0x0006489c7f6e90f1,0x000e52627145775f,0x00027c353f1cebe8,0x000a2f29fc36a4b9,0x0000acc3ef5baced,0x000fb8074e6a3d4c,0x0008d3fd643a9c64,0x0004c070fffe4c63,0x0000000000000001}, {0x000fbd2cdf57826f,0x0005bd9bf19e6e5e,0x0007942ab0ce8665,0x0000c790e82b0e8c,0x0001e2f2b552cb1c,0x00090a098c9dab8e,0x0009810a67eba463,0x000c6a4756fc9b4d,0x000e8cb25c97785e,0x000215f5f5b6c18f,0x0000000000000000}}, + {{0x0002bf06c6b067f9,0x000a2b3dcbcaa8f7,0x000559f9fa4cedc2,0x00001ff4707cbdc6,0x0005eb59f1a1d653,0x000b9620b3fc409d,0x00091f76c53a5feb,0x000e766a3eea48b5,0x00013597ec3fc458,0x000a5eb4cf309e19,0x0000000000000000}, {0x000b24162ae5ce89,0x000e57dda1da6f8b,0x00092393366cd895,0x000d0cbc02de8414,0x0007365ce8f0759a,0x000cfa2564893b65,0x000a74873186b40b,0x0004d0156cb04fbe,0x000490f66512a03d,0x00036b328165e70e,0x0000000000000000}}, + {{0x000d63dbde7982b9,0x000c29e4cd39f5d1,0x000aa87372309f98,0x000951ea561fec44,0x0001afd07b42ddc9,0x00045755866c4665,0x000b2e07bf78c6a3,0x000e9284f87ca447,0x00035d4199cdea2e,0x00049a677e175372,0x0000000000000001}, {0x000229ff95010f5f,0x0003cbc8f306c814,0x0001a7861e7a79e9,0x0002d63c05616521,0x000d995f90f64784,0x0000e16cd8cf737e,0x0001408ff0413e3c,0x0009c3a4f30dedcf,0x000a0a6b443a170a,0x0006e0cc49b5c711,0x0000000000000001}}, + {{0x000ac308c74d834c,0x000a8dd825f7406b,0x0000fbb496f34b0d,0x000075e1de870fb5,0x000227841bcf2365,0x000b8e05ffd3c983,0x000e33dc39c86d83,0x0009f0fd6d0e74cb,0x000d62a5a8904ae1,0x0008c32b1e28056e,0x0000000000000000}, {0x000d2a2671b67c79,0x0004494c1cde5597,0x000c0326e9105031,0x000b1ee150606033,0x00061f18317b0423,0x000cc474ed398c9c,0x000df4796a972375,0x000696b52ef07eb4,0x000ed96071372ae4,0x0005ffabf9d1feb1,0x0000000000000000}}, + {{0x00045b8e28f19059,0x000286054435bd90,0x00012ddcca1e377c,0x0009f510d747b1a8,0x000d3775c0ea63eb,0x000865c7834fcce9,0x0007bbad37d19f75,0x0007bbc7cb402eb6,0x000530a0f5327111,0x000908600a1a8bd5,0x0000000000000001}, {0x000abbe5e02132c6,0x000b17b10fe3c6dd,0x00011b655993587d,0x0008aa4f1c163208,0x00092e7539751ad7,0x000229bfb751c187,0x00003ce5719f77dc,0x0009dd5c3eedf84f,0x000b8c257bb9c60b,0x000345e60da1b9c4,0x0000000000000000}}, + {{0x000e7d40935779ea,0x0002b583ea2a70e6,0x000137328e54c9ba,0x000501ecd4654390,0x000e5d733683ab93,0x0009f374dd118e98,0x000b90700d407bd8,0x000c13b0afb65295,0x00048095857db6bf,0x00070895fc47c66b,0x0000000000000000}, {0x00037df304273762,0x00053684543a49aa,0x000af1483095c127,0x000176dbbf08a1c2,0x000fbbab267dccb7,0x000bd6efdfa7bbd2,0x000abfc8aeeb27ea,0x0008c902ad03e86e,0x0009e682a4e44e71,0x000fe0064991f1f0,0x0000000000000000}}, + {{0x000aed77f4d8e151,0x000d64f6fa111299,0x000d4feb2e79c04e,0x0007b4f97a120999,0x000d370550af65d5,0x0001340660d07357,0x000084ce4afb7c64,0x00040826e57205ac,0x000a7fc0bae197ca,0x0008238f07d6803e,0x0000000000000000}, {0x000454a02cb353a2,0x0000deb5cdf6d6af,0x000f3bb89c8b32bd,0x000b355a1bd8c3f6,0x000e63db355ab5de,0x0005c6b3982f043a,0x000910f0e90987dd,0x0001380521adabe8,0x000fa044a4bf6a24,0x000fd8fb752ed23d,0x0000000000000001}}, + {{0x000fdd1be70b4926,0x0009a826f6dba658,0x000cb4e95121bc79,0x000f676af00f6eae,0x000cee75a521d56d,0x000eca7d7729d333,0x0002e5027fb68ac4,0x0004a49aec5f206d,0x0006a988faa272aa,0x00047f341efc691b,0x0000000000000000}, {0x000df0f078415461,0x0006ca3afd9193e6,0x000e7785c7d78268,0x00030f1e3c2a9148,0x000aaa49f1fa54c3,0x000e96229782ded4,0x00093b6b845da08d,0x00020729c99179a9,0x000904b0df8fef02,0x0009cee6016c6aad,0x0000000000000001}}, +}, +{/* digit=50 [{1,2,3,..,}]*([2^200]*G) */ + {{0x0008c22f80cbfbe4,0x000130943a347193,0x000e2773aac837e8,0x000010c64a3c4f46,0x000be2b750229f24,0x00007131ff138446,0x0000ce7731813b90,0x0000e94672d6c2c9,0x000dcb075dd149a0,0x0006d07531381b69,0x0000000000000001}, {0x000b38c7be8e6de0,0x000a9739ced7c6b6,0x000a61fbc4fb63d5,0x000fe4d18f6b6bae,0x000bd6ae1dbab075,0x0002c3dbf8c1ebed,0x000b0516dce109a1,0x000e4a2962c4c087,0x0000db685a1e1733,0x0001ad9f800e79f4,0x0000000000000001}}, + {{0x0003feea78bb9ff0,0x00046e4fde5cbe66,0x0003f440437dd027,0x0002a08933232942,0x000e421f2f6038cb,0x0008d7b95a50b4f0,0x00073ae18c0b0f4a,0x000bc9451cc035a4,0x0005154ba8955b22,0x000d349d1fd0859c,0x0000000000000001}, {0x000f04652c1a8bcf,0x000b73e19db868af,0x000eea574f2961dc,0x00027664f8c3e1f9,0x000839b512b73f43,0x00002a0ec5b683e4,0x000b38a0fb615a0b,0x0001b1e55bb87991,0x0003094173f71955,0x000d3c0ba8f16419,0x0000000000000001}}, + {{0x000ed0fcb63247bd,0x000c61e26950f920,0x0008ac76960a5916,0x00049cfc2ae5b02a,0x0008cda5eb1a5171,0x000118595b5c8f4a,0x000c88e0004518e9,0x00089dcbce699e0c,0x000dca7cdb0b0583,0x0009678f7a0d455e,0x0000000000000001}, {0x0003e3080d452c74,0x00067cd9ebc5ab77,0x00092748ad132f85,0x0000eb812f890896,0x0009083d0c649d6e,0x000d13cd26dc1732,0x000539d6815ffdae,0x0007727168b4775e,0x000ad256509166ff,0x0001747a36c1d3bf,0x0000000000000001}}, + {{0x000e6e936a559258,0x000787f262712646,0x0007ee8f55296d6c,0x000b44326540e78b,0x000ef2fb8850453e,0x000e4739b6073a9c,0x00032c19bbfb39a0,0x0005bec805ba5b65,0x000b74df44331c49,0x0009d3002e8ec8ed,0x0000000000000001}, {0x000d7ba6c48685b3,0x00073d4bae18cecb,0x000a9e818b43a66d,0x000e109d5a439da7,0x00084e2bd60c3422,0x00082785ad715748,0x0009a5bf6bd330b4,0x00066c8383da8c0c,0x0009a00bf0007cc5,0x000256a489783e2a,0x0000000000000001}}, + {{0x0004a8e407a2ccd9,0x00086ad221fba29d,0x000fddda1e9d46ad,0x0008643114fcb5bf,0x0005a60db0e24f96,0x000659be98b0468c,0x0000c785c91bca8a,0x000b1e072204cabd,0x000ebbe04d9453df,0x0007688aef77cf50,0x0000000000000000}, {0x000b62e349b426c5,0x0004467872d194b8,0x000ddbd4e1c43334,0x00024117aad0f260,0x000ea7d8cfb9c423,0x00083e18f4bd6c92,0x0008dd5687682258,0x000359ac483a7289,0x000ec708225923bd,0x00062148de2e57ff,0x0000000000000000}}, + {{0x0004d07f78796d38,0x000a0307a33d42f6,0x0008948a2a44d434,0x000f90db03ccc6f0,0x000696ff7592ebb8,0x0007ff2ae969af49,0x00014fcec7fca3c8,0x0000d6cec6f56874,0x0009ae9c6b325541,0x0003ea961c98239a,0x0000000000000001}, {0x00084bbf91f7e4e9,0x0009ac472d023742,0x000d63ca5a686ea8,0x000614346cd552ac,0x000072fb24ab8e61,0x00080d3677dc07d1,0x000c8c0d1833f7c8,0x000717c50635d225,0x000f8a2192bf84ae,0x00027f2e83c678c1,0x0000000000000001}}, + {{0x0007ea965c0d1be3,0x000dffe0762dd1bf,0x000d60aa7917e003,0x0002fc7262c54da8,0x0004421eaa7edfa9,0x0002c86ea7ff6dc2,0x000473729f82d5e6,0x000d535c6df46821,0x0004bda6bb69bc00,0x000b33f34e260149,0x0000000000000001}, {0x00096ecec3563dba,0x0004fd12da169210,0x0002f945903cc5db,0x00014cb4c4f95586,0x0007af70f6fbb150,0x000a6967e23d80e4,0x0003ebeadb489f20,0x0006a009490665a1,0x000afaa96a28958e,0x000a31da82221f9f,0x0000000000000001}}, + {{0x000b5dfd8f4cc713,0x0009de59f2c453ba,0x0000c9b2cc1e3fa6,0x00033c17b6318b0b,0x000e92d5d399a56b,0x00008f8a6f6dc3c1,0x000b7f39e28633ff,0x000ff1fbcd4351ff,0x000013e8c77388ee,0x00066a953e5ebf9d,0x0000000000000001}, {0x000bfd2f419a3879,0x0001d195e5a481bd,0x0001ef3e1ae017a3,0x0008b706b5b37267,0x0003f748e8ba9898,0x000e9de7d3391698,0x0007cde6e3e3c930,0x0009ee4ca324e7e3,0x000b6a772ae3cdd9,0x0004235fda48d82f,0x0000000000000001}}, +}, +{/* digit=51 [{1,2,3,..,}]*([2^204]*G) */ + {{0x000ce56e11c7765f,0x000fe4cfdef6377d,0x00035b399363df3d,0x000ca630a715e9e9,0x000a21011f820ffa,0x00022d3bc633f64d,0x000dab0c59875522,0x000dc95736a8523a,0x0008fef5b787c715,0x00032a66393c6305,0x0000000000000000}, {0x00034ecf897f6f48,0x000d40891f4ace54,0x00051c5f6bf7708a,0x0007ca62fe89ee25,0x000eae6011a07c37,0x000028c949d24cd2,0x000ab99c094a1a4d,0x00031fed19d9cf84,0x00051c154036f7f0,0x0003d437b50c3205,0x0000000000000001}}, + {{0x00036aa0bf5fef4c,0x000b4bb069d26c89,0x0002e3dd1d0d3718,0x000bf3daeaaab400,0x00076315a34c426e,0x000eb5f38604c676,0x000805197e2eb1b1,0x00037226db1abc31,0x0009ad73df5e17eb,0x000d3797d098f510,0x0000000000000000}, {0x00082cf0882acea0,0x00006b48806f5a59,0x000abd275055f094,0x00049a504db94328,0x000ec38e43c40b3c,0x00082b99e608d386,0x000cf443b07fe475,0x000b7186cac29089,0x000d982cfac474a1,0x0009aa7b0368d422,0x0000000000000000}}, + {{0x0006f1287e600c1a,0x000cffcc5624ecb0,0x0002d1ca07fb78a4,0x000a1f9666cc7bc9,0x0008539fb634b6eb,0x000c73e361397798,0x00014a496c8d68c2,0x000d7ca4181be620,0x00069a299a451732,0x0001c06004061fe8,0x0000000000000000}, {0x00029e4b308242d0,0x000a13eeecee128a,0x000f659548ea451c,0x000490cf14707b99,0x000362f80a26ba79,0x000292eee64971fe,0x000871b89c8fc38f,0x000d6dc0d122e55c,0x000979f11919fbb1,0x000f85c98fe350a4,0x0000000000000000}}, + {{0x0006cb60e51b31a9,0x0009c2f6d82cb3ba,0x000f86e04fd98949,0x00087bb6cb66fb0c,0x000cb7e6257cf354,0x000caa5a38dbe642,0x0007ff70132dd977,0x000a9fe7cec8cf0c,0x000f2a9f98b24a15,0x000c7552eb7ce954,0x0000000000000000}, {0x000aec9e842f8ae5,0x00096d766fc55447,0x00099065768ced0e,0x000adad9493166bd,0x000c328be045e2b9,0x000e70d305222a08,0x000ec1d4f554727a,0x0002fed1873d61d8,0x0001c46d541e23d8,0x00042ef348b3f19f,0x0000000000000001}}, + {{0x000ccf976377d19f,0x000f1cf68ff2fc1a,0x000d98b274ba0faf,0x000788c074f0bd37,0x000a798f5cd04250,0x0003cafaaeab508b,0x0009fd1e881c2856,0x000dd63706ee8361,0x0001b79794f0ad14,0x0006f33d5c0505bc,0x0000000000000000}, {0x000090b50612a9e3,0x0003e3a662f706ea,0x000cdebf25eed8bb,0x000d3a7d888eb094,0x000fdc87a6c4d17f,0x000574531e25001b,0x0007e6d823c00762,0x00014c6a486f9d98,0x0006d2d6573a4d0f,0x00074c191e9a8851,0x0000000000000000}}, + {{0x000d58220482e6f9,0x000c77488e8bdb82,0x000ba1084c7aabd7,0x000ae76567c7272c,0x00079ab82a151ca6,0x000826c75c1ca84a,0x000434806316ad0a,0x00040dd7f329f3c6,0x00033bd2dde3d6a0,0x000c43689c3e453f,0x0000000000000001}, {0x00061be6ee98af37,0x000291b8361cc48c,0x000b8a7b622624c3,0x0000547d4c3e4e24,0x000c937a21d5f3c1,0x00007a153ffa09bf,0x000a63d54fa325a4,0x00048d13bea75c7f,0x000e1e55bfb7c45a,0x00055a1a8b85312f,0x0000000000000000}}, + {{0x000094bb55dffec6,0x000b1ee483a0b1a7,0x000e63531718eed4,0x0009354c0be15b59,0x00029a3b937fed84,0x000295e4506353d0,0x000205f7b6d68fbd,0x000e59b3375fdbda,0x000b4aaddd0cb573,0x000348c879da0f51,0x0000000000000000}, {0x000afd066ea2106e,0x000ee293d8c45af2,0x000bc67374a1f66a,0x00016d7c0fd1fd2b,0x0002493231dcd541,0x00019502a277efda,0x0003c7872e2fab64,0x000a95c3d06567be,0x000fa48d7ff41d2a,0x0001cc69c52d2e6d,0x0000000000000000}}, + {{0x00046d25ca0b12ed,0x0003e7af11f36102,0x000d246f5c00ca56,0x0005f2713dbda22c,0x000571517fe92c1f,0x000e558aff5b82f3,0x000852ec7e4811aa,0x000a0a7d7a0cfe93,0x0009a571e80c69ef,0x000d5ecf0c8dc85a,0x0000000000000000}, {0x000629e0362ea61b,0x00019570463933fa,0x0004d8bdb2997e91,0x0008d16c1670ff63,0x00042ac0e352c375,0x000bf5c218c31489,0x0009269789d4077e,0x000eedd9111468df,0x0009e59bdc949bb1,0x000694a97ced001c,0x0000000000000000}}, +}, +{/* digit=52 [{1,2,3,..,}]*([2^208]*G) */ + {{0x0000e658f63653aa,0x000362a4e263b15e,0x000f1a72f5cb787e,0x0008dd85ade21c8a,0x0009351d6c477346,0x000ea4254fd69f8f,0x000c982ae15e0af4,0x0005dd836935db86,0x00023278398d3a2d,0x00082c5ffb0769dc,0x0000000000000000}, {0x0002ae6ecd27779d,0x000456043db3d94c,0x00073642e7c09230,0x000692df6f9dd795,0x000422cd985762a8,0x0006ac0a49a83e72,0x00059cfa2e9e20f1,0x0000d0093708d3fe,0x000c84d0b10a4692,0x00035fa5bda12a10,0x0000000000000000}}, + {{0x0009924edfb9aeab,0x00028a46d2968a10,0x000fe84ed7a9147a,0x000478aa49744c91,0x00030fd88965188e,0x000dc8d99e65a34f,0x00006f221fd955c8,0x00027ea7cd997402,0x000f83ab9dedce89,0x0006e8a7c26d23d4,0x0000000000000000}, {0x000728e182c8cb8b,0x00078b0fa5f32091,0x000760a4e2a3ad9d,0x0002b50f65aca369,0x0003d46ee027e681,0x0005a7e2b8db993f,0x00003752acac076e,0x0003a179054a6029,0x0007bff0fddbfa0d,0x0008feee0dfeeff8,0x0000000000000001}}, + {{0x000e1e66266af2df,0x0002e36c081c96d8,0x0002c3c896d714ef,0x0002acfe977ac59e,0x0001a95d1b3f90be,0x00003c79555da335,0x000ac68d1f4138b3,0x000d2dc5e8301aff,0x0007f5144ad06837,0x0000d0ff81349a59,0x0000000000000001}, {0x0003ebad0af48caf,0x0005e5d9aadd9976,0x00026bfaeb96c7e8,0x000c9ceb03564b50,0x000254586ab1371e,0x00038a2ddf7b0228,0x000ca35a77209961,0x0003fd9f60e7f5a6,0x000f0eb7073273c4,0x000a1bce76f5e62d,0x0000000000000001}}, + {{0x000a60a63bd16196,0x00077f090aae19bd,0x000e7638c32cb3f5,0x0002a6ddf59abf93,0x0001d3548613634b,0x0001b5e6513c50db,0x000b5bd49476ec89,0x000a83b636d2f4bb,0x000071e3a3dd95f7,0x000ef977c02f69d5,0x0000000000000000}, {0x0008603799531d83,0x000ec49c9ad3cfc6,0x0004e50cb9635f1e,0x0003ca9d26b39588,0x0006f3bd6a0e5d70,0x0008ef03a93fe903,0x000aad2605b0ecc0,0x000b6abd3a9b070f,0x000a81977f3494ea,0x000831164f95f67f,0x0000000000000001}}, + {{0x000d576c129bba19,0x000dc781b1958f67,0x0004c725830444b8,0x000d772989a02957,0x0003160191e1f14a,0x000b3aba62ef31af,0x000a9c026782cae3,0x0005e2df9de1d797,0x0007d5d393897669,0x000e09de2e3f7f60,0x0000000000000001}, {0x0002f590d9230f6d,0x0008e59ce71ff8f2,0x00065bfab0b97ecb,0x0005b773840ae894,0x0005df550efa55da,0x000a03525e361e4f,0x00032a275d9fd4c6,0x000f30d16226c4bb,0x000512eb88ebc1cc,0x000a048dd37790a6,0x0000000000000000}}, + {{0x0005c82e7c82fe0b,0x000b6b153f48688a,0x00079cad28ce32c1,0x0004dd65d7330f19,0x0005fd6ce0841643,0x0004ad25ddac7886,0x0007705833e0df48,0x0008294753bb4502,0x000ad8dbbbf63937,0x00019b436c10a463,0x0000000000000001}, {0x0005b7516a304d40,0x0008185865f25816,0x000cfc1c163a4965,0x000ed7eed35d77f6,0x0002d2246fce8e80,0x00064fa59660ff02,0x000081304ce77dc8,0x000c6fc813dd85ee,0x000db13057fabb84,0x0008d66a509ef64a,0x0000000000000000}}, + {{0x000ae85bb3c1e90e,0x0001923787b4a031,0x0009e575e9892a6e,0x00074c2ebb769eea,0x00063064d9df4bf8,0x000e1bf584f54d2f,0x0007ecc49f863e83,0x0003213fe023848a,0x000c07e3d527042f,0x000c51626ab6da53,0x0000000000000000}, {0x000783a377f5ac26,0x00010cfafa402a74,0x000f762b70abfb39,0x000eac42e208b3a1,0x000497274db0b567,0x000b54ede1f18ddf,0x0001de4808fda6ea,0x0002ceac81237a87,0x0007edd6e7428e9d,0x0000cb9fb5801abb,0x0000000000000001}}, + {{0x000205d81676b493,0x0001cfe8f546e257,0x00087afe8b644287,0x000676bad5e346c5,0x000f4a964afa3748,0x0001422f71ca39ba,0x000328b0e9e0a58e,0x0001d31cca18d62c,0x000787f6507714d7,0x0009ad810168e375,0x0000000000000001}, {0x00030f78aa1440c8,0x000f7e509d6354b7,0x000beae80e0ec14f,0x000f7cc09793053f,0x00025b6b1fd1b019,0x0004558d48e4fca0,0x0002aae7ed4a0374,0x00070e2db1c48699,0x0007f4b02f033375,0x000af43011764955,0x0000000000000001}}, +}, +{/* digit=53 [{1,2,3,..,}]*([2^212]*G) */ + {{0x00015e37a2d438c7,0x0006bb436c808cd7,0x000782325918615d,0x000d68ce58c6e6b2,0x00075a40e8f75ca6,0x00001da381c4c378,0x00055d9be962879c,0x00075dd3d4cf58a1,0x00099fd85847d5de,0x000e158f7f76b4ee,0x0000000000000000}, {0x000c9f66a4cec18d,0x0006e45302a76bd4,0x0001b679cdf64708,0x0002c24293b84a7e,0x000a092243bc4d41,0x0005c3c375519ccb,0x000d06b585371f2d,0x000590f4c0f28ba3,0x00073b4091daa768,0x00073c6342e78bf5,0x0000000000000001}}, + {{0x000dcb7fa9fca3c3,0x00047144679a2a9d,0x0003be369b38069c,0x000a82620de84dbc,0x00098d5e1f28d82c,0x000205de8989e877,0x000abac84051f10a,0x000ac26a5b9c5c22,0x00074ecbb99bba5f,0x000253359fa6c2dd,0x0000000000000001}, {0x0002ac09a82c210a,0x000b422ac08c572e,0x000c5c720e071535,0x00095966a7bd1c3d,0x0003a0a4b0c9e18f,0x000faa6c6449a62e,0x000bdd44f85595c2,0x00047bb9f75f529c,0x000f46a6493955cf,0x000622defa5af650,0x0000000000000001}}, + {{0x0006df335e8acd01,0x0009b6996f91a727,0x0008be8a5f6bba5b,0x0006ef24a13311be,0x000b7fe2a95d51f2,0x00084a38b4ee11ba,0x000caf93233a4f76,0x00073771e4762ff2,0x00024adb050176c1,0x000a9e7e6a96ea5e,0x0000000000000000}, {0x000540fbecee1541,0x0003f39f444fded1,0x0006ce01534c5cef,0x000a886125f5b460,0x000b437e2199b1d8,0x000a994fb5a77157,0x000fcae599e6b65e,0x0002599cf24da91d,0x0009f0d7248964c9,0x0007ac171d21915d,0x0000000000000000}}, + {{0x000ca4b8955dde0b,0x000f850c9b5cc251,0x000b99572d6cb500,0x000f8952b82356c8,0x00088575ebe209ee,0x00089253a4607df2,0x000ff9ba421a6b33,0x000799f0eb4dd513,0x0003ed789c84b56b,0x000293e6a5c4d425,0x0000000000000001}, {0x000ff217c0f18530,0x0008986930ae0198,0x00012fd1e778abc5,0x000f509594fad4d8,0x000656733dccef2f,0x0006afad83f10fab,0x00078f717d75c593,0x00018a1a313cd995,0x000cd73871602741,0x000f5446078ec40b,0x0000000000000001}}, + {{0x00034871165b0407,0x00041aa1bdd6d509,0x000fcc9e894c7b1e,0x000369e1389494fd,0x00087f5116c5472d,0x0000a1b0d04b3663,0x0000b4c49058303b,0x0002bdbdc1d8cd26,0x000ae8f01c121d9d,0x000dc1d8c7270b76,0x0000000000000001}, {0x000d922f0e06b7cb,0x000f07bd16843c5d,0x000c01dd32245ab8,0x000fa9f6081477df,0x000c2db624f56ea9,0x000b2fb3e2bccdb5,0x000cb3b768793cb3,0x000c8d894f04384c,0x00010a3af20b15d0,0x00045ab5af8411db,0x0000000000000000}}, + {{0x0006b9ed8590e4a5,0x0008a58a2974d283,0x0001c7668bc64c35,0x000acc81cd837cfc,0x000fc7b9eb02f729,0x000fee9d6ad1171e,0x000a8d7ca7eeb433,0x000a79ea2e79ed5e,0x0001fb8522e38381,0x000cc8162a76e7a5,0x0000000000000001}, {0x000c67543c98a4ea,0x0001f713941c4de6,0x00081dc75d38a3eb,0x000374ce9c459cdd,0x000bdd7ff2d1d555,0x0000d7b03a560a54,0x000ad6193d7f976c,0x000099ff27e977d0,0x000749edcba5d394,0x0002c6804ed826ed,0x0000000000000000}}, + {{0x0003e3bd3291cd15,0x0006a26271a72635,0x0001ef4581082f50,0x000cb6109637a812,0x00043e376f158b63,0x000e1fb8b4f92150,0x000fd4238523b166,0x000edb09ffe09019,0x000a7d8e27b82fb2,0x0001989a4df94f9f,0x0000000000000000}, {0x000410feb007ca74,0x000ed314198cb45e,0x000e234a35b4b72c,0x000c1779579fdfb5,0x000da54f63558ebf,0x000b68211f279305,0x000b7f7a88ea265f,0x0000914728cae4ca,0x0002ed65205137c3,0x00045ed6df45a086,0x0000000000000000}}, + {{0x0001cfb71cb23bf8,0x00067b25479ac001,0x0002f48bcdb785ff,0x000da267c454fd25,0x000154e460f65f62,0x0006fa21e0ac759a,0x000770bdb56d239e,0x000db91d7ccefa07,0x000f07f46fe275a8,0x000527a152927368,0x0000000000000001}, {0x000969a0d74d64a1,0x00059c36f066b2d7,0x0003a664061d81a7,0x0008c41d929c3e1c,0x000a4033ad63b5ba,0x000387b665fdb5a8,0x0002e2c2b433c84f,0x000e4f2d4d931e74,0x000631141b030d2d,0x00086de3845c248d,0x0000000000000000}}, +}, +{/* digit=54 [{1,2,3,..,}]*([2^216]*G) */ + {{0x0000c039d020c1dd,0x000d44ab7b690765,0x00026b42700b4f64,0x0002c1e20576d051,0x0009e70d2ad712c7,0x00015f46314322ae,0x000dfb189904b573,0x000124905a45ef02,0x00026cce68a7b470,0x0004ac5f0db2cade,0x0000000000000000}, {0x00042cd205524778,0x0000d86fcfeb993a,0x0001cc7f2d2ee992,0x000c209546bdb299,0x000531516a6eab71,0x0005f1492f99e62c,0x0002cf5197ae7709,0x0002013e12e8e95c,0x000b78cb72aad7be,0x0000320e70b96760,0x0000000000000000}}, + {{0x00011fae297851e8,0x0004f69afd113575,0x000ab322fa242843,0x000f361ecbe5e3de,0x000b7c1b08880d89,0x000a50aa80c1fbd2,0x000104d9e40537e6,0x000ed4f51df57fdf,0x0008a6cfbe707164,0x000f36d887e3d0b7,0x0000000000000000}, {0x000365e7f9a5983b,0x0003c6129d87d996,0x0002952186a64aad,0x00009284f8224d3c,0x0007bc689c1d4452,0x0003f44aec2c194d,0x00057e00e7c6b2d0,0x000de28c9eb3f18a,0x0008b7de6fac4981,0x000009552159064b,0x0000000000000001}}, + {{0x000147285c5f1e84,0x00054bbe273f4c79,0x000ec66bd4d71c06,0x000771b4dd0505e9,0x0002d4c891619ed4,0x00021a0542316ff1,0x000ffae0c65ede48,0x000db678c0c5a23c,0x000bbf48abf05e02,0x00032691ff4f9e32,0x0000000000000000}, {0x0004ec586f495f39,0x0007665c351b2d94,0x0003939c0c800e74,0x0000a9b2fe310474,0x000c05e63ba55b78,0x00011119c911cfce,0x0000043322972c9c,0x000235f5ba3b0c8e,0x0002b78daa0946bc,0x000a82622257dc22,0x0000000000000001}}, + {{0x00053147b3b22c3b,0x00004897a83e73cf,0x000d11d0f510fba4,0x000de4abfcf9128e,0x00047095c92d56df,0x000f3cd334bcfa2a,0x0000038a5b6f4d4f,0x00095df73169a249,0x000aba80e663bcbc,0x000c64a47769e841,0x0000000000000001}, {0x00004365a62c4cf6,0x000531a582938f21,0x00043321e6dc439a,0x00061253f7387146,0x0008352424fe3ee8,0x000a676302135902,0x000702fe71ab2ae0,0x0005f460396ca837,0x000882e04f7bce51,0x000f93d936239901,0x0000000000000001}}, + {{0x000dc17de84a8cec,0x0000802c374c6c6c,0x000bd6a5b20e3454,0x00046c40337b0f58,0x0003431d7aff9402,0x0001dba6a849292d,0x000f90b37486a260,0x000c12d61cbe41a0,0x00033f828782067d,0x000486ebc39fcd30,0x0000000000000001}, {0x0006818da93cb68a,0x00039ef42673be02,0x00040e906d3f2e27,0x0000e1f1d8ecf6f0,0x000655c024d3837b,0x000417cf596708c0,0x0000946c57882083,0x00058942a1031702,0x0009752bc7086a40,0x000474bd8d65b027,0x0000000000000001}}, + {{0x00057bccc2137aff,0x0003cdce9d3ef437,0x000021aa7d7b10a8,0x00014071804bf03e,0x0005009822fa479a,0x000c4d2fe91adecf,0x000bad18fc3b061e,0x0002ffe82ea4479f,0x000b0bed7b70c576,0x000d5c8a667da425,0x0000000000000001}, {0x00036c22b574750b,0x0001a79beeade4d8,0x000d7c41634d28e0,0x00030d7e124715f6,0x000b28e33c4f983d,0x000442a0682ee2bd,0x0000a48a18cfc1f5,0x0009a6cb16376839,0x000ecc58844abd78,0x000383bd6682cba8,0x0000000000000001}}, + {{0x000c36944a224339,0x0002221a86a3568f,0x000a933230388a55,0x0002ca0cfc1f186c,0x000051f24d4d5f8c,0x00053ac024a82a69,0x000e4e0b6761f2a7,0x0006b4a03fa98a0f,0x000862d5fd1d2058,0x0006922bcb9949cb,0x0000000000000001}, {0x00085574f50da47c,0x000f8cb1192295a2,0x0002436f1d423eb5,0x000f6ae23f4febd2,0x000640f1f266f842,0x000e94d3498091b2,0x00016f1f4561f25e,0x000c6e303b526f4a,0x000bbc14e80ed7ff,0x0009ac6f957c1908,0x0000000000000001}}, + {{0x00089a398d206da8,0x000e6a92326c61c3,0x0000b658149c80b1,0x000f4f6423067f4a,0x00022c96735aed5c,0x0003cb53c31cf4dd,0x000c7214b478ba14,0x000d3b1bbb1eb353,0x0005aa79b46f2e84,0x0007d44a4fa3d67b,0x0000000000000000}, {0x000d31f07cf711e2,0x00031e5402e45d5a,0x0005565057819f0b,0x00050ebe214cdf81,0x000e63efd8e063fd,0x000c82d63ad808de,0x000d0ee39ccec300,0x00085249be7d39df,0x000e9271ab6c788e,0x000a727ffa3cadeb,0x0000000000000001}}, +}, +{/* digit=55 [{1,2,3,..,}]*([2^220]*G) */ + {{0x00061361be3ee97b,0x0006e1f3125658fd,0x00096b636a1d69b6,0x000b9e470c7ac9e9,0x0005bf9bb3617e69,0x0002050a8c1b1e89,0x0002213fa5a11a51,0x000bc2919affa249,0x000be1b1008d55c3,0x0003926dcf2c08c4,0x0000000000000001}, {0x000fb57fda93efc6,0x0008276ce4aac2b4,0x0009277cab16292f,0x000e677dc90518a9,0x000ab0432d015144,0x0005d9214eea4408,0x000b8a649b20eb23,0x000b48a45d8a2560,0x000cf7d1d37dd269,0x00049d71a47616ce,0x0000000000000001}}, + {{0x00034d86d189072b,0x0000b9475ac257a7,0x0003a9d12f132433,0x000adf08cecaa5dc,0x000dc33641cc3048,0x00040352a6bfcc4f,0x000dac876f01bade,0x00035bd7dfb06e93,0x0001d4494c0e1ca5,0x000219b51965b8b2,0x0000000000000001}, {0x000f90e9be7f6998,0x00040776cb857a46,0x00031907caf1b517,0x0007040038843e17,0x00014b3c14ab5377,0x0008c99d12b47cfe,0x0001590d18daa185,0x000f84db2817d455,0x0008544bbc4d8f7e,0x000ee1752b595c5a,0x0000000000000001}}, + {{0x000718c9c3382876,0x000798d016412ea8,0x000c7059eeb3d459,0x000e910707afd251,0x000366ae603f57f6,0x000f1d424b7c29b3,0x000f4591f08b6d11,0x0004fa0b188406f8,0x000a09ce46dfa46f,0x000f3e1ee6193a22,0x0000000000000001}, {0x000ef37aacfd9628,0x00044abc282a7b59,0x00016eca5fb04908,0x0002a4a0fa414af6,0x00010748c915edd4,0x0003d4af5a6f1ab8,0x000aff331ef86bba,0x0006d8ffc35ce768,0x000278d36b6c4e53,0x0006238f61ddd56e,0x0000000000000000}}, + {{0x0008204ec1fe994f,0x000a401febacf810,0x000a1fbe66e0a0c3,0x000654b72570b727,0x0007b5299d8ae343,0x00058b323a83e064,0x00034d13d86ee0d1,0x000be946224b7585,0x000a38321cb77d0e,0x000ea9b845ec39e9,0x0000000000000000}, {0x00021327ccd07b3e,0x000b9c13edfc5044,0x000efee69b80c4b9,0x000b9736ce07a452,0x000de2779de28110,0x0009cb506b810433,0x00009831468d2371,0x000fb54615eed293,0x000a72f30895d360,0x0000822a939f9fa6,0x0000000000000000}}, + {{0x0003613a0bad2c78,0x000a2841479bab61,0x000ae853dfe64b3b,0x000c5d69d7d5f8f3,0x000913c023c98edd,0x000e51c064c2af1c,0x000caf23e811beb5,0x000a297f73a13e28,0x0002c2db1b2c63f7,0x000869272783c6e9,0x0000000000000000}, {0x0009e33e7c2b0e8b,0x000a6e2930859a4d,0x00021c82319f96d2,0x00062855234b3a37,0x000cfe1e952c7999,0x0009fff526834c3f,0x00082932d66290aa,0x000ed0618b6fc1aa,0x000f51b1765d795b,0x0000fa7ad3a784d8,0x0000000000000001}}, + {{0x000b99ec5345605e,0x000d705b03ab3b6f,0x000d514df02df673,0x000d979497926d51,0x000327f3cad9968b,0x00007b8edfa7bbd7,0x000fc5c1dee65278,0x00036db8f1709072,0x000d430f38a088d0,0x000f9df3373c9bfa,0x0000000000000001}, {0x0002932b26523f7a,0x000a4698ce826d56,0x000d64992b915d43,0x000f137e0e1471fe,0x0006811a1c25612c,0x0007e5e74619907c,0x0000f455dcff6a59,0x0006fe503afcb4d1,0x000af47382daf8e9,0x00085e51a1e9e2bf,0x0000000000000001}}, + {{0x000c899ed52c7ebf,0x000805309ddc9f57,0x000f9ec0561000b8,0x0005599061be65ad,0x000c7c6ac2e8bdb7,0x000546a9f7f8f392,0x000b38f709f90fbc,0x0005b81ee256aa4e,0x0000cd9ffe3bb73d,0x000ad1e54f791392,0x0000000000000000}, {0x000e432eaecfeb4f,0x000efd7d99d4376f,0x000ecbf257042314,0x0006524d0d11bf19,0x00070c070e8811a2,0x0009bb46ac80db71,0x0005deced6976256,0x0001f5d4f1996e7f,0x000c6d1bc35c855b,0x000b7fcfaf131b63,0x0000000000000001}}, + {{0x0007faf89dff5e6d,0x000701e289504c4c,0x000c21c143d7c67c,0x000b8b1051104808,0x0001a8547ea3f429,0x00042d1597643f8b,0x000c20d8e30463a4,0x00019700b9ca1322,0x000c7c74112313e3,0x00021d429e582d53,0x0000000000000001}, {0x000df174dc25e320,0x000421f30a9c65a6,0x000cca7bd70734a8,0x000970e912f441c1,0x000b0da35c85642e,0x000fbc6108990f29,0x00004f9201a5ca87,0x0000b4ba5b8a0067,0x00032f15dd79c420,0x0001dcaa2c572053,0x0000000000000001}}, +}, +{/* digit=56 [{1,2,3,..,}]*([2^224]*G) */ + {{0x0009f88e8ef42daf,0x000fa3828b99d9e1,0x00051fa512ec2bee,0x0004d684d33e3c3c,0x000ae34a6c37abe1,0x0009d4bea55b5936,0x000e6492802583c8,0x000098da605bd938,0x000288cfc1f04542,0x000ed0659c47e455,0x0000000000000001}, {0x000ddaea5046a68e,0x0003e422472a49b9,0x000c2da95690aefe,0x000dcef36e21cee3,0x000eab14f0abf0c7,0x00064941e198c3c9,0x000fcf64819eee0b,0x000dbfe77fa8433c,0x000c3b28a2f7686d,0x0009c3dbfd233403,0x0000000000000001}}, + {{0x000b7107c522fbbd,0x000c42e8887082c3,0x000cd304c29d218c,0x0004d8477a96d44b,0x000ecee7f483ff1c,0x000951d19c530d4b,0x000d68d4d6bf1fdc,0x000fe03d009b71d2,0x0005537694d3bd1d,0x0003c3db4cb1a2c4,0x0000000000000000}, {0x00086f0e3fa15f33,0x000388caf5fb4c5e,0x000b2f14ba7715c3,0x0002610381191db4,0x000e0f68a08e3384,0x000342059cb75d25,0x0002bd292f767fcd,0x000a7a696b414dec,0x000dfb67016057d6,0x00035999c277b18a,0x0000000000000000}}, + {{0x0008ba73e1f475e0,0x000dd3f1b783f77c,0x0007375b491b116c,0x0007d166e4a377a6,0x00011bc5434f7131,0x000a2dbe9d40b215,0x0001220856a6b5e6,0x0007b890e757f7e0,0x000f02578a944660,0x0004883ff77008f5,0x0000000000000001}, {0x000a3e2632a212b7,0x000302b8f038a3d8,0x000a306506181a49,0x000301702b3192c1,0x0002c30f8e6d2d73,0x0002a0c5b0c340d2,0x000f8bfe6e8c90fd,0x000e8d703d0478e5,0x0005135b5979aaf2,0x00034da81fa7add8,0x0000000000000001}}, + {{0x000f16b10733860a,0x00007e088e20bfd6,0x000859fdd39b30d0,0x00096072d4c40b6e,0x0007d0a59d2a4c91,0x000f5b531a3c4f60,0x00005885c546c30a,0x000ddc1d5df2fdc4,0x00026f4df2971b1a,0x000bb67cb15104fb,0x0000000000000001}, {0x000f74646d17eec9,0x000e3de3bee8f39f,0x000560fc63e96143,0x000d7aab2e0395d9,0x0003f099cc808fd9,0x000e3c3dca422f15,0x0002c61efabb0d74,0x0008d736943aed2c,0x000d74e4178f87c1,0x0003d1b76afadf96,0x0000000000000000}}, + {{0x0009583eea912fe1,0x0006a5587f3b9010,0x0002a020829ea8e9,0x000b82ec346b73a8,0x0001f44eda406972,0x00067b4504fbb553,0x000e7a6189bcd268,0x00039bdb40c39e05,0x0005c436a03e5e06,0x0007019eedb498d3,0x0000000000000000}, {0x000662e3f77764f2,0x000cda79a7c8507e,0x00040c75e0594299,0x000bed938a93a1fb,0x000829d1bab3f2df,0x000df99ce48d52d8,0x00032fb8c4092f5f,0x000036b5fa768aef,0x000201a9ee7997be,0x000ca0344038e8d5,0x0000000000000000}}, + {{0x0004ae17c6c4ef83,0x000409ab5b4b2b20,0x000c4c863d0780e0,0x0003a003f73d00d9,0x000f7d17b97edd24,0x000335ec2eca6e6e,0x0007def3f246d97d,0x000f0e2518ced6c0,0x000f728fdb8b0595,0x0006dcc1ccb10bfd,0x0000000000000001}, {0x0009f012dd02f504,0x0009d50a767d8e86,0x0001bbb0510be6fa,0x000cdab7deeebbfe,0x000f08332cdf98f4,0x000468782175d651,0x000610add0fc83f5,0x0004965277afa428,0x000ff5c3438635dc,0x0001fc0961df5b9d,0x0000000000000001}}, + {{0x00069bd3ef1fabd9,0x00064fe7b00b747c,0x000195c6097cd6a8,0x000470709e13f896,0x0002cf79709fe958,0x000b72450de744f3,0x000df3f80fb1524e,0x000527ea0bf24b8d,0x0001b2774e7c3f06,0x00031ad38e7aad24,0x0000000000000001}, {0x000b784c1f4489c7,0x000f2a889f589505,0x000a5a3175ee6df1,0x00047a68c157fe56,0x000497f7b784bd67,0x0005ed8ff0792025,0x0001528aca7e0427,0x000d473d7c41594f,0x000cb017e35b5669,0x000f1ce78bb2a52b,0x0000000000000000}}, + {{0x000290556eb6cb2d,0x0005feea12a072e4,0x000d082bac976238,0x0007c4331be16424,0x00021f06c7a59869,0x000de72b68a812dd,0x000652502f900697,0x000d9acaec02a2e5,0x0009120c4a89c7ef,0x000a1bdf713a324f,0x0000000000000001}, {0x00067220e1027a34,0x000cabc964145333,0x00029a9fac99b048,0x0001c2850e9e757a,0x00057de9cc170cb2,0x000d017c03bef969,0x0008cbf0f3534a37,0x00085a627fd6a9c1,0x000c29da8ed78ac5,0x000ae4ef092acab9,0x0000000000000001}}, +}, +{/* digit=57 [{1,2,3,..,}]*([2^228]*G) */ + {{0x00060edf11bb374f,0x00007883a0baf3f0,0x000eb8cb2a326de7,0x00095554f5a8d631,0x000236ba14fb5cfa,0x00053769e29d950c,0x00043eac3b6e6d4c,0x000400e396b6857d,0x000b3b40a28cc2c6,0x000f0142ca52de79,0x0000000000000000}, {0x000536d43744d95a,0x00080dbb696fec4e,0x00086879c7360fe9,0x0006d4736a564e15,0x0001262e7ce9d679,0x0004cced89ee75c8,0x0005fe2dd4f732cd,0x00004a4d97d2a3a7,0x000e046cc62def6c,0x000c590bfe781afc,0x0000000000000001}}, + {{0x0001e39da3d53729,0x000c68258dcf68b5,0x000b1289a021c50c,0x00028ef654112229,0x000f4c73b83c7489,0x000a0ebdbdf0df33,0x00033245f1663936,0x0006bdfac3bf0988,0x0001bcc9c21bceec,0x000bcb64a15de987,0x0000000000000001}, {0x000fb2f27cba002c,0x0002f0ce8d66f97e,0x000c4f49e106a48e,0x000720d60d05177f,0x0001b9e35427304e,0x00008c355b746f84,0x00061577b7b7cd5e,0x0008586eb9a88122,0x00003b21c38ce383,0x000aec02872c5a8c,0x0000000000000000}}, + {{0x0003ee736c032ac5,0x000efdfdb4c3d4de,0x0006ff01d94dbf41,0x000a0bb5ee1889b9,0x0009e7a8dc456a87,0x000c2bb8ca379bb9,0x0000100b2eae0668,0x000d2063b0184325,0x0006daafabf5c71a,0x000da2051e835c68,0x0000000000000001}, {0x000eaacab83acf78,0x000761a09a5ec30c,0x000838055c361c20,0x0002e34e4fa05860,0x0001b9a7a3c102c4,0x000f91746c97086b,0x000c731ba10529e6,0x000e8a7f6450292d,0x00076f3144af4061,0x00077cacdce041a7,0x0000000000000001}}, + {{0x0004b67237f032bd,0x0003fda08c68528a,0x000c719571479a5b,0x00011c75d8054fed,0x000bf503c35807c6,0x00037b9ea9e88e5d,0x000c9422b4c4521b,0x0002950eb17e75f1,0x0008eca427950847,0x000c0f845c91f923,0x0000000000000001}, {0x000c1e80ea8c1010,0x0002e1978ae396a0,0x0002c0d00f914b24,0x0004e43fb47bf7bf,0x00034b416e50ae94,0x0007c8d114906c36,0x0000ad0347f03a3e,0x0006a6eba25165b0,0x00021e9dc53a14e2,0x00076911eb83f4f5,0x0000000000000000}}, + {{0x000bb95adc5ef776,0x0008da0ab8c5f233,0x000a4ab915fb3034,0x000423a63a5dffb8,0x00063715aa2304a4,0x000018304840df01,0x00037b80028296bc,0x000854f7402fb23a,0x0006ebdeaf3e5fac,0x00005463cd3d8406,0x0000000000000000}, {0x000c9468fa481453,0x0003a0abd61a0348,0x00047c025f468b21,0x00090d7078cd984d,0x0005284e72a10788,0x000156fa2b542f64,0x0003b3c956017512,0x000991f2c57b982f,0x000989b3c05c1c45,0x0001ce6eb96dccdb,0x0000000000000001}}, + {{0x000c0c76c91a3d86,0x00001a3cab8eb5fd,0x000752ec9cff796c,0x0003f81d37bdda1a,0x000e12c746df5d95,0x00027c38d512fd50,0x0004e23ce093d983,0x0004adeb3545f200,0x0008acce0db1764a,0x0004097d3ea0eee0,0x0000000000000000}, {0x00057c8acd0df62f,0x000a235a3f7e354d,0x00043547a0eeee98,0x000831405ee1adbc,0x0006a916c043f6f0,0x000e4844ab1da766,0x0000728cc7b74a71,0x0007192544b4fdc6,0x0009d41decd4aad8,0x000497605d2bc303,0x0000000000000000}}, + {{0x000c16253894ed63,0x00072cd0f6a2350d,0x000412983f7baee8,0x00009f78d054975c,0x000d7bad42e9614f,0x0005f2ac60d67eaa,0x000622d671f099d8,0x00021f07bed1e830,0x00081424aeea7845,0x000f7592014c5f2e,0x0000000000000001}, {0x000619d3fbb4d7a6,0x000a9269d1beee9e,0x0002e4ef27debfd4,0x00069156fc40ac5c,0x000b31c86af27730,0x000f4aa3fc30b8ff,0x000bfd86f500b895,0x000c9dccb88b68a2,0x0003b7832b9f66f8,0x000853497ba1895e,0x0000000000000000}}, + {{0x0000b0ab29d3bde4,0x0009c71ea0d8e5cf,0x0004b6204df30d49,0x000d3f548f748b9e,0x0006eba95c754c4d,0x0007ad6cc9c44387,0x0009e2d08e10896d,0x000ea9428b4a544e,0x000fe71891a2d3e1,0x0006b905c7660eb6,0x0000000000000000}, {0x00066e6414125179,0x000e2877106f8fb3,0x0002d9d82f542b36,0x000b4eca63743c2e,0x000cfc887146aaf2,0x000ed669e0b28d08,0x000c885c73913714,0x000914ee1f56da28,0x000d4479d84424da,0x00015e3364622742,0x0000000000000000}}, +}, +{/* digit=58 [{1,2,3,..,}]*([2^232]*G) */ + {{0x000bd25b5edea690,0x000347c3abc5bd1f,0x00078a11d29a3138,0x000c6d62f2283223,0x0007b4af4ece3ada,0x0001c75e4372b0dc,0x0009560a7308e28f,0x000d0ea7127d9913,0x0000172da9cb3c31,0x000c36c69386a7ee,0x0000000000000000}, {0x0003be2a2cae0b56,0x0000778293313929,0x00041cdee07af8d3,0x0000895f4118b415,0x000b9b3a9502ad6d,0x00004e1d44242767,0x0001a843924f3834,0x000a3c5c40dce7a9,0x000443e9e0f30db5,0x000c5338df60b62d,0x0000000000000000}}, + {{0x000d88264070b39a,0x00060f5a265bdb95,0x000717063ef771ed,0x0004ca000dd813de,0x0005a4d348196d97,0x000def16129d2691,0x000bcbbfcdfb3523,0x00057290d6981eca,0x0000c5d2444e08c7,0x00091c65a063c65c,0x0000000000000000}, {0x0001941d0c9b7cd3,0x000cc0a5845e0e75,0x000c51c09cc90c04,0x000ce8aff11f3a2b,0x000303944ca09536,0x0005cb9d10d614a2,0x00062674cf73a00e,0x000639b5262911f3,0x000379f49e681d43,0x000ea07634bb0927,0x0000000000000000}}, + {{0x000aea3b47740815,0x000ede22e76e20e2,0x0007e03afde6eca1,0x000f70d608d1c44e,0x0000b88d585444ed,0x00074d298cd8ec39,0x0001146c4c3f6754,0x00073274110e9ce2,0x0009cf1f4b12ae7e,0x0004fa67e9f4b4ce,0x0000000000000000}, {0x0002ef9318bf7948,0x0004cb3efdaff610,0x00059bd31047a8b9,0x000fad7c4b8b1235,0x000a02488482725a,0x0003b462badd939d,0x0009f8945cb99ab4,0x000c0f6b65c66a4b,0x000e97dd0bddb4b1,0x0009d75a1f797639,0x0000000000000001}}, + {{0x0000ec0aca9d0f94,0x0002fb13ee984570,0x000e5ad047be55b4,0x00073a0845bc6993,0x000ffee41f2aee8f,0x0003c832041ac680,0x000bb4a740b12fcd,0x000521ebc1645e36,0x000dfcc7735a45c9,0x0008ef92b0fbdab8,0x0000000000000000}, {0x000805ed4e45e0eb,0x000584829d754db5,0x00036cc488a6c810,0x00060f1ec9632468,0x000c0f2ebc30d39f,0x000e960758390502,0x000f4626d1feec9d,0x0004f944bca363ad,0x000375dbfdeea282,0x00050ba887d0959c,0x0000000000000000}}, + {{0x000857ce82a61386,0x000fab480b18fa40,0x000223002b9297b0,0x000f9b41f698e6a5,0x0005eb688e33b253,0x000cc09f30bec2ad,0x000eeb6333a5ffe4,0x000bc81e94524303,0x000091b660b277fb,0x0008fd03ed404dad,0x0000000000000001}, {0x0005dd59faaf5be9,0x000abe017edd5410,0x000aaa4856720e0a,0x000d987f4b1178d8,0x000fe5e556ef405d,0x00081f659d8f4002,0x000f7836cb450ed9,0x000350d35c2f69df,0x00095c15741d4705,0x0004c982d30172da,0x0000000000000001}}, + {{0x000f854cfa0f1b9c,0x00036c27f5709e7d,0x0002fcca4c8efcec,0x000a965fa53691ad,0x000f70705ad9f4c9,0x0006ca89080d5043,0x0005cff60c807ea8,0x000ff2e47ec8fa95,0x000846288bda4094,0x00003dd7b4f68fa6,0x0000000000000000}, {0x000c953f0c60b7aa,0x000060469787d616,0x0009579f0ed52085,0x000faf3c529ad3be,0x0006f18223d47482,0x000c07ac1ad869d6,0x000437b5cb0dcd56,0x0004dc765feb592a,0x00004a294944df05,0x000184cb4ad2de82,0x0000000000000000}}, + {{0x00003d153000960d,0x0008f85dc5ffb696,0x00009b3d9302c4f3,0x000fd9668bf9ae36,0x000a25bdc3c8ecf4,0x0005c1a7bbfdb2a9,0x000393a4a4463083,0x000a4d41df85c97b,0x000d92cc6610ea8e,0x000bee7a14446540,0x0000000000000001}, {0x000fe7092ab307cd,0x00055b75f48236e0,0x0009f206af987311,0x000191b95f1aca72,0x000b01a57a7e9b13,0x000784300caa5513,0x0001b4e8ad08534f,0x000e7a92df980dd1,0x000edd78871ce4ff,0x000f2fbe3e6a8b6a,0x0000000000000000}}, + {{0x000cf8b0ae3366de,0x000c06074154422f,0x00007a17d21e5dad,0x00001e78c237ee85,0x000d108f5fd2183c,0x00098eec2dada49a,0x0001c303e35f7659,0x00091f2075445a12,0x0007426d8f5fdddb,0x0006af1dd7a92c53,0x0000000000000001}, {0x000c31da2f658936,0x000e72228504949a,0x0007646877ed3189,0x000cbac9e04fe426,0x00093f8802494ee0,0x000975ea85d9fdf7,0x00038fb0c9c6045a,0x000802b88cb118cf,0x00057e39ea12c877,0x0002548e571e0697,0x0000000000000001}}, +}, +{/* digit=59 [{1,2,3,..,}]*([2^236]*G) */ + {{0x0001f89eed993384,0x000860d77e7f7ed9,0x000e9a249862cf0f,0x00026963705ade19,0x0005d0f929eafa4b,0x000a12f7eef6f18b,0x0009d590ff1861d1,0x000d6b67d2954ef5,0x000540dd418bcbd0,0x00096e83b51e5170,0x0000000000000000}, {0x0000a67de6f33ef5,0x000d0f090cb0d50b,0x000b8b3eeba89c8f,0x00026dc5289c0d96,0x000e0e6dd7431a8a,0x0000a660c8afde18,0x000cfa02cc76374f,0x000b7654494d397b,0x000476b8fd958a15,0x00097d53a314e324,0x0000000000000000}}, + {{0x0005d2d913938d7d,0x0006d9d471ad5a92,0x00018a88bdce6ad3,0x000b90be599734aa,0x00099aa6c0fb7b20,0x000eda14b233c887,0x00024ee7dad41f1c,0x000cb7c643d13bb9,0x000e7d84ea63db6f,0x0003a846de4666cf,0x0000000000000001}, {0x000e6128e81e8c45,0x0001464a9235f4eb,0x0002db34ea2780d1,0x000ad3e8b3ecdbc0,0x000f8cdbe120731f,0x000f4318097ab5eb,0x00079ebbf1d4990d,0x0007e93c15830f8e,0x000c40a1cfba03ee,0x00068df76de664ef,0x0000000000000001}}, + {{0x000267eeb458a4eb,0x000e492a1279f17f,0x000c76f729af11c0,0x0007339e92b8f2a3,0x00014e00c8ca3543,0x000dbcc01641f96b,0x000755e449dde571,0x000a0a0df11e5fa3,0x00031770742dc646,0x0006f53610d1c65d,0x0000000000000001}, {0x000e5b808553c438,0x000499c99fe83176,0x00019eef1a061f75,0x0007deb8d4c21a60,0x00033192fdd7ecc6,0x0003250ef2ec37ce,0x000aff8f6ed93441,0x00058d1e9777ac4b,0x00083407f0d67d5c,0x00079fd1b5287171,0x0000000000000000}}, + {{0x0009d6152b9c90b3,0x000aff28f438100d,0x0000a2146e64ecd5,0x0009ad5b7e0c0df1,0x0003f6775181b0d1,0x0003c7a1b8151d7c,0x00035c338a3ef9f4,0x000e1fdd81712dd2,0x00034726a1f2597e,0x000cb3083971de47,0x0000000000000001}, {0x000c041200d9bef5,0x00067c85ff590dfa,0x0008b72aaea952b7,0x000265e7843ae9c1,0x00032307d7542004,0x0002dea503395a89,0x000e1eec7f73a8fc,0x0004b86c7eb53399,0x00021cf862926b2c,0x00019347a4f0820d,0x0000000000000001}}, + {{0x0004f7b384cd7297,0x000acc1a4787e2cb,0x000a5590739ea069,0x000be73fa99d3f7a,0x000e7cf6cca23601,0x000960cc53aa83e6,0x000b86b5920b1173,0x000fa7ac9dbe20e0,0x00068bb6f6b3ef99,0x00064060ac6c56f4,0x0000000000000001}, {0x000626c189f97eeb,0x00092eb7dd82a1dc,0x000142f8425008f8,0x0004b241705cbf37,0x000c04c8483d101a,0x0002075cfd275951,0x000f282a7dae45a8,0x000263c98e81a9d9,0x000c331b130bf290,0x000c95c5f55addef,0x0000000000000000}}, + {{0x000ac9a1c36e3c24,0x0008e09fb461cd04,0x0007bca251b2abd0,0x00098ad8fa6e8384,0x000087e953f04f5d,0x000d9f6c57ca3dcc,0x000f179679c5992f,0x000cc9cf93cdfac8,0x0000d1a320e29622,0x000580703c650a05,0x0000000000000000}, {0x000188e58d397fdc,0x00021860a160b5c1,0x000be9a3f426d729,0x000f8747311911d3,0x000b78a79eb1a864,0x0006b881dee2ff5e,0x000fc87ef83107b6,0x000ed139997a8784,0x00080e84fd894e5e,0x0003aa1a2f41979a,0x0000000000000000}}, + {{0x00031707560bd53c,0x000c9501712fc58e,0x00010e30ced00607,0x000465e677b023a2,0x00067620a31c653b,0x0008ea62e5b10af4,0x000d99048d08ca5a,0x000d65af877831dd,0x00029f5ab5be8899,0x00028321d38a082b,0x0000000000000001}, {0x0006bb42378c6e38,0x000fb8ba4bc4d5d5,0x000a4ef2e4b28b7a,0x000b41f7c314822c,0x000a2882d9a51ae9,0x000e6c2280f2b327,0x00019697ce965ab1,0x00075708dae4a49d,0x000f1175eea346d9,0x0000e9572d2fbccf,0x0000000000000000}}, + {{0x0007d4bad9839247,0x0002744af0bab5bd,0x000cd1e04b0ade61,0x00080a7fe21c3a53,0x000864b9871244ec,0x0003f6fd1428f483,0x000889d63b180bf2,0x0006acf748b0e2b4,0x000a061072996dea,0x0006607cdec9d5f3,0x0000000000000000}, {0x000a9858e242fc0a,0x0001fa2d38ce4fc2,0x00087521e69c709b,0x0005bae8f5996fd9,0x0005e131ac99f4d0,0x0002578fe3c3bad7,0x0002279c9d139206,0x0004b6fbac903ada,0x000292885696da36,0x000ddbd02ce1356c,0x0000000000000000}}, +}, +{/* digit=60 [{1,2,3,..,}]*([2^240]*G) */ + {{0x0001074e0ee1b4c2,0x0001a0ff10eeb73d,0x000c77549f2e087c,0x0006808a5e2e0837,0x000e948c7156c74d,0x000c13bf7c11f82c,0x000a51472ee287eb,0x000b28c4e6f906f6,0x0000fef0b165038c,0x0002d8b6f1c9d932,0x0000000000000001}, {0x000d310ce2cf5a19,0x000c08add6b6df57,0x000dcd4cb28cd825,0x000e1cbe48bebf85,0x00051f1d5aa6a644,0x00008ba85cbebbd3,0x0001bc84d8a2aa19,0x000d343d2d77518b,0x000a9098229b988e,0x000a9f940fc8d07e,0x0000000000000001}}, + {{0x00096ebc173a5d8b,0x0000ed54a67c958f,0x000ef67809a984cd,0x000abb728dd8453d,0x0009f4fe5f363de0,0x000e4fc4614b7360,0x000fee4aab1b83c8,0x000606d2158cb989,0x000096597fe56f7d,0x00057270734a0c52,0x0000000000000001}, {0x0004ef503a18dccd,0x0009b732b2f44d09,0x0002c29898debd6d,0x0005ffd4e0ef3ff7,0x0003830b99ae2e5d,0x000dd5fca5e1c94a,0x000e97015b084de2,0x0000e94504be6d08,0x00079eaed90fe0fe,0x00051eafa2897ddb,0x0000000000000001}}, + {{0x000dd470cee4255c,0x0008ea907208e445,0x0001157ba3e551b3,0x000f94c51b72b693,0x000b183c616c9cf9,0x000fe84fcaaf1c59,0x00077c97ed67f1d2,0x000a1d1e1a09f1bd,0x000c2f47751550da,0x00038d58d345e7ba,0x0000000000000001}, {0x000d95b5f854d3f8,0x0004daa404c99ba5,0x0005a1bac7d29f41,0x000da46981c9d673,0x000652c1bc49956f,0x000e505f2a66bdcd,0x00081069783eab5f,0x00036f9996ab9237,0x000238170a7330e6,0x000670fa70e33da6,0x0000000000000000}}, + {{0x000e86a58386cf1e,0x000572f15be04bfe,0x000e35f663e32b87,0x000f5c5294b48632,0x0001362fb4267165,0x00007dadeb7d3b94,0x000012e189d86c34,0x000d63e5f7805689,0x000b7e33561c7907,0x0003c0c0dc085fa2,0x0000000000000000}, {0x000cbe4f2b4f6916,0x0002f30cfb081da9,0x00011fe0a525ac2d,0x000bbd4632662679,0x000223e344c2d8fc,0x0000a08757b1ff41,0x00001c0a48229e9a,0x000456b8c92c6f71,0x0009a086c4af0e80,0x000d74c21360d8bb,0x0000000000000001}}, + {{0x00000ce7164d30ca,0x000fedb34ef3a45a,0x000800620f8ac5c1,0x000c7c79ba6237c0,0x00088ff56f44997d,0x0008ab94745563d5,0x000df21b2b00a9ae,0x00086a286295b8d1,0x000426dbb4cf9b37,0x000ec830b7004300,0x0000000000000001}, {0x000a97f98f792db1,0x000f9d5a4e4b251f,0x00061b4d385d3a62,0x000b4cdaad987701,0x0002341727a73b90,0x000c4abf7e84f908,0x000f0ded814c4a3e,0x0002c453671bed3d,0x000a888690c8945f,0x00019a94087855dd,0x0000000000000000}}, + {{0x000e5f28e7fb0752,0x0005ca705e5e780c,0x00093ca952d3af74,0x000eccdc7351cb28,0x000bfc12e9837fed,0x00014b93567e7bfb,0x000546247999f34d,0x000a3f729887c629,0x0006edd285692b0d,0x000fc9812e383cc9,0x0000000000000000}, {0x0001bc941d72f110,0x0009ba70199860ef,0x0002d090c33493b9,0x000503ff279e0c37,0x0005c3fbe286bbe1,0x0006d81c3e80f646,0x0008d3a0a9257bf0,0x000e402ee72a2a44,0x00092b91c6a5669f,0x000ad95315497ce5,0x0000000000000000}}, + {{0x000b397c9c0b3f5e,0x0008573318572b93,0x0000e667f17a3327,0x0000b9137af5bff7,0x00017fb65a96b2e6,0x000188dfee9e25ef,0x0002c9edd117ab29,0x00015472d03d830a,0x000512ca1063aa4e,0x000adf9593f42bd8,0x0000000000000000}, {0x00043be475ba796e,0x00023dbbccf7b7a1,0x00080d4a4b140a81,0x000247748562ea6a,0x0002f53e144c7e84,0x0009ac8b6fa39d88,0x000b1eab451fe154,0x000ec4538f34ffe4,0x00076cc290527aed,0x000d83bcd8efdc10,0x0000000000000001}}, + {{0x0000c5c5bcdaef57,0x000720046d980167,0x0000601ff33f7b5e,0x000e6b0aa07084cc,0x00007791af83b900,0x000dd856fdd2391e,0x0008d880430654d2,0x000f826068896340,0x000a4b443c176e8b,0x000c4e18c663e62f,0x0000000000000001}, {0x000d8ae0441a598d,0x00065c950e8cec48,0x0001316d53d11b80,0x000cd6863a5ccc8c,0x0008ced41a668fee,0x00076ba771bb4164,0x000b49a9a4bb4b6f,0x000e370974d5a9b2,0x0000485def9f130a,0x0001b84c25f49d20,0x0000000000000000}}, +}, +{/* digit=61 [{1,2,3,..,}]*([2^244]*G) */ + {{0x000f43bc2dfb404c,0x0008b314169ebb20,0x0004d9ce016776ec,0x000b6f3c40cc814f,0x00075de6701fd64d,0x000be16687bc27b3,0x000c8ca41b2641f1,0x000e5c7ebdd8aaf1,0x00021918a667e429,0x000d9384d2d4cd92,0x0000000000000001}, {0x0009667398bc8b35,0x00026c502a6f1f65,0x000612fc65ae7726,0x00047588d1717c45,0x000b0cd0bd27306b,0x0007b68702fd9dbc,0x000a43a5beaed3af,0x0008d9e3bfb560d2,0x000c14b9b618c615,0x0002421dad1537ee,0x0000000000000000}}, + {{0x0009e7889bf8b6ea,0x000292e957ebe27b,0x0002bd715ca3ae1f,0x000ea4756ae08fea,0x000f5cab67aaf3b2,0x00047caa37f247c9,0x0001b8953af0925d,0x000557e7bd1f409f,0x000979eb14ee5b8e,0x0008bb0e2890300b,0x0000000000000001}, {0x00078d6b71ea9467,0x0000c63dfcb1d01d,0x0008e6aaf05595db,0x00006ff49217ec90,0x000b8ab12df36451,0x0004207aff8489ad,0x000b85d257b835bc,0x0003706e08a1abbb,0x0002a5b7d71ad10a,0x00056ae224792f6d,0x0000000000000001}}, + {{0x00028daca765356a,0x0000ee74680323bf,0x00005f51f95b6f29,0x000c6656c3eab37e,0x00038ce239752348,0x000cf8aff4ec8ff2,0x0003adc6745b80af,0x000e56c344a76be9,0x0006011d48a9f827,0x000ed4a3e016c1c3,0x0000000000000001}, {0x00096e22c7a80fb0,0x000891624b09ab2c,0x00049f6d4616a383,0x000b275d1cd7006e,0x000adb9075332618,0x000586e58c4079b5,0x000645e2b19bf36f,0x000f76906dc5820f,0x000f6bdd7a51a11b,0x000b67dd8137d034,0x0000000000000000}}, + {{0x00095b8b650fd71f,0x00014965b39aaa82,0x0001270c9510690f,0x000b3a9f763e6fce,0x000be3f839143baf,0x000866c1890dc990,0x00074c8a0d6a0e73,0x0004b520d476087b,0x00081006ed8910a1,0x00052b16e6fb91bd,0x0000000000000000}, {0x00057756a63f7354,0x000a1ebe4b13d097,0x0003f1884cec54fb,0x000503924519ff97,0x00059fb9e4f42689,0x0002ce5e202d309b,0x00098e0004b85f0f,0x000b05c20b1735d8,0x000add1fbd4f54e0,0x000dcb178e2a7f34,0x0000000000000000}}, + {{0x000b4ec871f1c2e0,0x0003b5de22756058,0x0000837f536100e3,0x0000c16f1ccaf424,0x00056dcf857da01f,0x00038c07ebb5fb6d,0x00096a096d6242a9,0x0003055c2b2c30cf,0x000a782e55f66ac8,0x0004f012cc9c2c3a,0x0000000000000001}, {0x000506352d9bc635,0x000b06aeaaa56b9a,0x000095186c35174d,0x0002167f537385d7,0x000cf421a089f0ea,0x00022c37dc99678c,0x000356bf33a69466,0x0003e97eafc66d63,0x000e3c0197b16669,0x000d912a0fa3fee5,0x0000000000000000}}, + {{0x000a52c892d50f2a,0x000dfd8fc48ea89f,0x0004b09bae86a680,0x000a1e1278d67917,0x000370f37ae3b7ce,0x000f51107a8383d3,0x0005c8e157913dc9,0x0004c347e479bbd0,0x0007dfb63e7f7f02,0x000ac4a32c2410c2,0x0000000000000001}, {0x000c5983d275b47a,0x000f3ebc8a42a9a0,0x000db0533ac31f03,0x000cb9b707b4440f,0x00064522041e91b5,0x000076367218816d,0x00023be78c44489a,0x00060289668fa7d8,0x000b7bda9a033e06,0x000c041bf9880e14,0x0000000000000000}}, + {{0x000df487e3ac46f1,0x00026f6aeda132f0,0x000f9138020c745f,0x000a8841334fb2d5,0x000c66722bd6a712,0x0005a1ea4533d309,0x000636323c57e66d,0x00008eb26f26bbe3,0x000fa479b5dc3fac,0x0008f02e50177e02,0x0000000000000000}, {0x000c92b56635aa25,0x0007f4d610ada23b,0x00010104e5566f0f,0x0004f4b30ea26b57,0x0004efd9675fd322,0x0001af8f50ca2f0b,0x000382d177aefb1d,0x000538151171ef73,0x0003347891a319cf,0x000711dcdab779a2,0x0000000000000001}}, + {{0x0002b024db8d3d78,0x000aa5cc47fd4499,0x0004ca3c2ae301e6,0x000f6635a239d460,0x0001e72a93968b59,0x000f3e7cb493da74,0x00057a0e451c8476,0x00090539a4ae9584,0x000df123a0ccc6f4,0x000f3e4b36ee4a70,0x0000000000000001}, {0x000bf5964aec0226,0x0008d0d54e934ea5,0x000838881f3a4f9d,0x0001904c5759057f,0x00054f74d21e3d23,0x0009110e0965fa28,0x00025473e3fbb9d0,0x00066659568773f8,0x000e05953c3213d4,0x000de0c6c9fbf74e,0x0000000000000001}}, +}, +{/* digit=62 [{1,2,3,..,}]*([2^248]*G) */ + {{0x00023dd4a823eac8,0x000e2984db5faf62,0x000d39d495d0b0f5,0x000841ce0a14e52d,0x000cb365ed8de723,0x000bcb1fe9b4ef0f,0x00047ee3aa7d5d2d,0x00032d33c7e0b190,0x000d3b9dde2978f5,0x0005329dbc97d12a,0x0000000000000001}, {0x000d165556c00ac2,0x00057fb3172c7a02,0x000beeb32c48804b,0x00099dadad774958,0x0007934b2bc9659e,0x0003fd281f90ab3c,0x00013abe477effe3,0x000378b329a3faa7,0x00036cbb9f3df235,0x00014562e824d0ac,0x0000000000000001}}, + {{0x00078de5aeb19662,0x000d1f458b071bd2,0x0000313ed9b5b584,0x000ce5fe5f476cd6,0x00077007678c1c54,0x000258964a6d145a,0x000420c0c62f8fee,0x0008595f7056fa68,0x00089119263621f2,0x00019c5d8cf9f82d,0x0000000000000000}, {0x0000243d2e6cb7a7,0x0008d78d5a87bb13,0x000c4e517b3bcf90,0x0000f65dd0ef67bc,0x000d930e5dd35ba7,0x000eb9fb0741af2f,0x00075eb448314eea,0x00003702fdd3f71f,0x0002c0a91bac4835,0x0008e91ebd11ec1d,0x0000000000000001}}, + {{0x000160f761d5cda1,0x000d737ebbdcd5d3,0x000d7809f98ac1a9,0x000dcdbd555d558c,0x000fa398a682b263,0x0004f2c8dcb516cf,0x0003f1fdc39fa308,0x000c8f35f0892119,0x000b1a2e7b6b0e00,0x00060305cb4b78de,0x0000000000000000}, {0x0005dc3804225ef5,0x000a4dd0142d36ef,0x0006183af50e0ed5,0x0006c670fcbeb371,0x0008629b90cb9a81,0x0002af3261b35739,0x0008aa6b6e9823af,0x000fbd19a0ec458b,0x000311a98270a0a9,0x000299fe388efd5b,0x0000000000000000}}, + {{0x000bf04df8057734,0x000b1031cb2cfcf4,0x00013e25b6e0c3ad,0x000fa4cf3dd2ab40,0x00007758e2edfff8,0x00031ad907feffd3,0x000b4564baf111bf,0x00073c4f6a12eba1,0x000fad75513a8160,0x00074de7fc43bd94,0x0000000000000001}, {0x000d44eee593810b,0x000f1e369ffff5b8,0x00064f19da65334d,0x00021d0f5c2b9ceb,0x00091c5ca3390013,0x0005689acf02b87e,0x000bf7c3a49c8b54,0x000093f7ed7c049d,0x000a0a1d58d27784,0x0003d7726f20ba73,0x0000000000000001}}, + {{0x000ec366cfe8b8dc,0x0005109510bbe0e0,0x000aea6451541d52,0x0007f0a613f8478f,0x0000f85f758544a0,0x000f2b250e479f86,0x000b98246494bdaa,0x000483bd40872db5,0x0001ef43cbf9eee7,0x000bc8abec7ccf7e,0x0000000000000001}, {0x0005c21b7bfcf30c,0x0003ee117f493af1,0x000fdb4b0534832d,0x0001fa1bf55f45c9,0x00078f7e3c1efc43,0x0008a7de78a32d11,0x000dc3559e7ee791,0x0004e7a865c863d0,0x0007e1e671e05898,0x0005634bd3bf85ef,0x0000000000000000}}, + {{0x000d81691c2ab48f,0x0002541bcc0607cd,0x000da791a7f87cac,0x0008347771261610,0x0008e3bac250c120,0x000659e04001e2db,0x00052cf1b9282899,0x0009b392289990cc,0x000164d7d417f75a,0x000945df41fb11a9,0x0000000000000000}, {0x0009c07103c6ae84,0x000480fee72710d4,0x000b8360a2b9d155,0x0000e0d3d7e07da2,0x000d149f0388daf1,0x0002f71dfdfd1523,0x000418130ff86f14,0x0009f7346c3e8b09,0x000b2e35c5e699a7,0x00052b1c1fdc6ba3,0x0000000000000001}}, + {{0x0003345adc3b52d1,0x00068ceb5ac071d1,0x000e11041a7e60d2,0x00074a2a70677323,0x00040734a03353e1,0x00041105a4c9d8da,0x00044e1f19873918,0x00091f314a743285,0x000322e1b26f9179,0x000d3d1613ed69ba,0x0000000000000001}, {0x0004543cf00ed5fb,0x00053fab663cf4cd,0x000be8cc8843f104,0x000f74fe45dc2276,0x0004319b78ce0e03,0x000389520a05dc90,0x000cbb592960d2ca,0x00026ed0dfe3e1c7,0x0009036fca8841aa,0x000549d9ea193025,0x0000000000000000}}, + {{0x000d95d4b57fe6e7,0x000237e45eed99e3,0x000ddc0cb978fe19,0x00073f687eb46e14,0x0006f57bdaf6e4df,0x00047741a18670ac,0x0004925a46fbe2b8,0x000282f9632d0245,0x0002e144fc15a10d,0x000815c55aed10af,0x0000000000000000}, {0x0004dce0659c9bac,0x000b25a506cebbc4,0x00048b6559b03aaa,0x0008048a933863a2,0x00020d37a9de4c34,0x00020f440226cd5e,0x00074e9ee95db69c,0x00082f425e1eff1c,0x00033a6f8d820bb8,0x000ef06f95cad219,0x0000000000000000}}, +}, +{/* digit=63 [{1,2,3,..,}]*([2^252]*G) */ + {{0x000e4e5c561d5097,0x000b584aedde8b8f,0x000aba27830c0d36,0x000c7d59c0f869dc,0x000b714a35cbc32b,0x0004bed4bc22d71a,0x000061f00680d9e0,0x00033bf0836adf25,0x0000d7fedb3d768c,0x00095c616b984e3b,0x0000000000000001}, {0x0008c5d5c16b3659,0x0009b2bd923c283c,0x000fbaf03219a32e,0x0005bdb0f0d8e95a,0x00027d3039b5fed1,0x000c59ce26d89427,0x000676fa1dec9a4c,0x000121992696f0b3,0x000260c98b8c7cfb,0x000541c1c9792927,0x0000000000000000}}, + {{0x0002f821dc52854f,0x00009f01e839c81b,0x000f9520b0acb7e7,0x0005ef6ec4857ab9,0x0007209f9eda63a8,0x0006daca6651f475,0x000d6976e7173379,0x000240b37bfaccec,0x000b4437229f4ce0,0x0001910fe8a0ffcb,0x0000000000000001}, {0x0007819702adce4e,0x000559cf2ed75ea3,0x000524a7d80f948f,0x000a15733e1fceef,0x0008e25fd5510058,0x0004c29ed3586531,0x0004e51cb7d9fa11,0x0006f171b487caa6,0x0006163b82558054,0x000686de74000000,0x0000000000000001}}, + {{0x000b150deff72861,0x0006369a87c08222,0x000222ff210ad76f,0x000b4d66f2177234,0x0006ffb6d673f874,0x00059b847a7a63aa,0x00088181fb601b85,0x00061d5a56e17f52,0x0009cad3b2dceae5,0x000b7064799ea515,0x0000000000000000}, {0x00071777678fbf96,0x000ee3fb840953c6,0x0006c82ee1fe6943,0x0000476345986586,0x00027b2c01a1da88,0x0001dd9ed1d2e620,0x00093b57ac9ecf98,0x000aea3ed52ca86a,0x00083c732ab42d43,0x0005274badd57297,0x0000000000000000}}, + {{0x000860f86c3463c3,0x000fa451b92975d3,0x000e9c89aced751f,0x00082df00fff3b8f,0x0004d44ceed1d22e,0x00022e7d389ef4bd,0x000e91dec43e5b63,0x00003ac6cd725daf,0x0003c7139385f22e,0x000deeecc87ca1c2,0x0000000000000000}, {0x00051580221e0fc8,0x000941fbf97dbfd6,0x000689ac9e872372,0x000740f84611974e,0x00046e04ba0c6ce7,0x000419caa07a8f97,0x000565905b0cdaf7,0x0003cd2570033075,0x0003b2c015af7e40,0x0003287d54b47d8e,0x0000000000000000}}, + {{0x000087653c1971e3,0x00077f0a153e5f28,0x0003ed97c2828991,0x000095935157a12d,0x0003722663e4ec8c,0x0008ec4fe824b403,0x000dd44aff641d97,0x0006da087485323c,0x00044e2cb2c2b861,0x0006eba170078d74,0x0000000000000001}, {0x000b5b83303bc746,0x00013dec62dad85e,0x000eb52890ef39fd,0x0007414a67a192cc,0x000ca32294492ed1,0x000e1460a8a16787,0x00024c190537d3f9,0x0004d8dd49fa5c1c,0x00025dcc6564966f,0x000872ce77f122af,0x0000000000000001}}, + {{0x000f907cb543fab3,0x000c69a83a50077e,0x000cb4bde1d63e29,0x000c33fc44a3bb56,0x000f82a91020da98,0x00021551aca18ce5,0x0000524e8061c783,0x00030bbde84c8ff8,0x00080aaffd7e7a07,0x00025503d2cefb4d,0x0000000000000001}, {0x000d489c09cd7406,0x0003bbf72e670d86,0x000584e3d6cd578d,0x000c5357fb7dc505,0x00078eb6069dbc02,0x0003d4fa59d19b85,0x0000398d29e9ce8a,0x000efb99649fd29d,0x000ce49c616fbd54,0x000860686cd02ccf,0x0000000000000001}}, + {{0x000b5280bf38c012,0x000b16e5d0e927a5,0x000e0fabbb77cb27,0x0009a5fc1a89a6d3,0x000a92cdb34cf1fa,0x000464dc9a9cf793,0x000eb464387be7aa,0x0007fbfde96cd4d4,0x000232f3e162772a,0x000656aee57c0a02,0x0000000000000000}, {0x000a4f88de057838,0x00017126373b6048,0x000e4a0f6263ace7,0x0001d00ce841a9eb,0x0004964f3fadfdda,0x00053a6795df2847,0x0007970db46fe5d6,0x000fc5cc5ee32eac,0x000634cedbf5f3ce,0x000fca10e755fd0e,0x0000000000000001}}, + {{0x000aeb844e0e5a47,0x000278171b725206,0x0003ef95a8f2f67f,0x0001fdfb411d7c3d,0x0002cbc9db5d5134,0x0004cd3d499c831f,0x00090a75fa0db406,0x00072c8d72cfb8bb,0x00070a986050fdef,0x000f2a584d26e897,0x0000000000000001}, {0x000357b6cd8cc5f7,0x0006b375fe114c8a,0x000aa2296d0e1fc2,0x0007cba1e2fe623c,0x000b8ca73315ce03,0x0007e86db236843e,0x0005e04b5b70ddfb,0x000420198f9d4b15,0x000535cfa0692139,0x0005a2aa06d43751,0x0000000000000000}}, +}, +{/* digit=64 [{1,2,3,..,}]*([2^256]*G) */ + {{0x000cc45663bc9f5b,0x000b2868f57feb89,0x0004bd3cbd6a2543,0x000a5e560bf63c0e,0x0000e648f4a5666d,0x000591427cb7d9cc,0x0005977ab848b1a7,0x000af4656829e85c,0x000ae8f7a4025667,0x0003b4ab876527cd,0x0000000000000001}, {0x0004ed81840ffbcd,0x000e4830db96c420,0x00026c352dd1b3e5,0x00003369497308c9,0x000023370174e547,0x000c6d8497a95345,0x000ecbfae86058c7,0x0008a32e4cdcae7a,0x0004e9eb567daf0b,0x00085eaf8dd7df3a,0x0000000000000001}}, + {{0x000deae24e7511a4,0x000762cb23734ed7,0x00066bcd84d23939,0x000c037c989a46bd,0x000c06543988385e,0x0003f08c8acc808e,0x00000e7680dc66ca,0x000e4c3c5332a768,0x00063204acc98ee9,0x000db0a0ef46de86,0x0000000000000000}, {0x000b4a4e27fff289,0x0007eeb14e22b305,0x000e9d3141e930a3,0x0004f15435f5cd09,0x00052f55ccf3f336,0x0006c9377055f313,0x0000f610c74549ef,0x000c8d0207da4bf8,0x000b6ee97b1c2b15,0x0000920992fd2caf,0x0000000000000001}}, + {{0x00043f2525d2ebb4,0x000811edb2cca106,0x000c996f27900e32,0x00092edb6f6af92c,0x0002dbdef8275bf9,0x0004dd3d26338446,0x0004401818a7ff9a,0x000d60e7694d8e21,0x00054e87fa7aec62,0x000f988bdd22449d,0x0000000000000001}, {0x00063c9fbe4e6775,0x00026d7e7ff11afb,0x000c6b3e18f7eec0,0x0005c983e08b80f1,0x000b75d6b5a4c84b,0x0005f99e3a4b0fd4,0x0005a7cfc4904ce8,0x0006c336a99a7afd,0x000e4a736ba1e62f,0x000595be20ba2924,0x0000000000000000}}, + {{0x0008c92f010fc781,0x0002463423f6e871,0x0005f129e35dae8d,0x000d59f4aeff7db0,0x0000c963932f5dba,0x0005e468db3cf82c,0x000e23c6b7d10e1f,0x00006e08595910e6,0x0008880e8c76fb1b,0x000134e8c1259453,0x0000000000000000}, {0x000506649d87b671,0x00014c272ea4f089,0x000aa2740669dd1a,0x000622f8a6cc0d62,0x000e392244f6f191,0x0003dcbd9dd28338,0x0000c61a8dd7166c,0x000e4930a90ca39c,0x000d41296b979b8c,0x000037aa5c88b76c,0x0000000000000000}}, + {{0x00051d16f31cd955,0x0003fd720fff5e54,0x0006c62e42ceacd9,0x000b72856f74fc83,0x0002b8a51db93ff9,0x0006ca983e7b6bb4,0x000e06f8fd893a36,0x0002491c6c8908ee,0x0008e9f64e123094,0x0005764984e58063,0x0000000000000001}, {0x000ad9aba979f347,0x0005557b9d835c0b,0x00089b7877984846,0x000ce8c3c6bb325d,0x0002e0fb571c388f,0x0007185f17237c5f,0x000ac5737bcf4832,0x0005f037df6f53b0,0x000b6f7ae34a972e,0x000821f685c7b273,0x0000000000000001}}, + {{0x000e5575bc0dd4ad,0x00028eabf66053ac,0x00054861cbd6dc53,0x0005b123ea9fdaff,0x000c00ecf823c855,0x0005d8934d09e411,0x000eb090ae97a01a,0x000591dabc9c170c,0x000f751f273c40a7,0x000f0f52861011d8,0x0000000000000000}, {0x0002bc9a33075cd8,0x000bb779de4fde35,0x0002eb1b199f0130,0x000e29003c4457b6,0x0009ff04878d3a95,0x00004ebfeec1a9dc,0x0007d0d097a6545e,0x0008673c7b41f5aa,0x0007894e63c5c4ce,0x00005b385d1700a6,0x0000000000000001}}, + {{0x00046d7efcbdeae0,0x000fbb2f349cfbe6,0x00022f14a9cfd187,0x000ef46f7fb5a2ff,0x000d8084df701781,0x000b2e7da6ada115,0x000273537b36285a,0x000d779e5cbe2143,0x0007b1bb342159b5,0x000321182d17ef98,0x0000000000000000}, {0x000974b9395d5c1b,0x000a203e9046670c,0x000c9fa51be4f31c,0x0000167fed87df23,0x0006d7ab1aee3553,0x0006c8a7b334d671,0x00037b8b3f821601,0x000677ee013df3eb,0x0007a3a1013ff132,0x000a7ed7d1a2e9a5,0x0000000000000001}}, + {{0x000b3fd002806466,0x000fd4605cfac818,0x00085d2f0b811efd,0x000167145bb41efb,0x0006e3c03cac7ee2,0x00085c4b2dade36a,0x000220dcd3725a14,0x00032cf525a550bc,0x00014db66b11c84f,0x000013664e47ace3,0x0000000000000001}, {0x000a48858e7d464c,0x0002276f7bbfd1a7,0x000e24ada567d04c,0x0006a941adced466,0x000c270addbb103a,0x000761ca82f14e02,0x0004d0794b62798c,0x0007a0bec3f90326,0x000caf618966e8d4,0x00021a1f211c02e6,0x0000000000000000}}, +}, +{/* digit=65 [{1,2,3,..,}]*([2^260]*G) */ + {{0x000c24408b5b4bc1,0x000d861e48c2aa26,0x0008746f93b2fb6c,0x0005f018515690c4,0x0008d76a3c1b771e,0x00093035c899fbb2,0x000d18ac338e0049,0x0001b4e7f02fa3d8,0x000fabf9b804c035,0x0002f83e6175e309,0x0000000000000001}, {0x000830680485a054,0x000962c1a8a622f6,0x000f94a3f3450d94,0x0007a83e0a44d62d,0x0000105319e21805,0x000a4a1ebdd2bed2,0x000f4863d6076c13,0x00034672ca137368,0x0006135e4ef4b1a4,0x00020b6692537ff9,0x0000000000000001}}, + {{0x0001b7a5e08fe30e,0x00029eb048815ab8,0x0002e0a161f11e12,0x0001801becb84207,0x000ad5b394a58f8e,0x0007512807890edf,0x000f675b3e4e4773,0x0008c99841055d81,0x0003ed35c1050ce1,0x0006c217bd56acf4,0x0000000000000000}, {0x0000d6c8c5a1e005,0x000ddbaf53db5dcf,0x000c6ab4e4f6ee72,0x000e686032c8481a,0x000415c545af61b4,0x0003595ad60e7c0e,0x000f59b261ffe75c,0x000cf66fa7cfbf47,0x0002e7097fa1aaf6,0x000a8386b7977f21,0x0000000000000000}}, + {{0x0000029a7619e46f,0x000f29f5c333074b,0x0009e45f3bb1eec5,0x00035aadf8396133,0x0000825d3e2a3176,0x00034ef799bdaba5,0x0007f38574f4d09c,0x0009085e808678d4,0x0004a57483db0387,0x00054465ae9f6d1c,0x0000000000000001}, {0x0002fb74fcaebc4b,0x000ef4b901e46eb5,0x00068ec4a861868e,0x000f51a07bab1199,0x000748f19df109f2,0x000d75da4f6a75a0,0x000f255613859652,0x000e60c8067759f7,0x000b663826b7b569,0x000f450533f4d440,0x0000000000000001}}, + {{0x0008e66704c4911e,0x000b5ad20de07d3b,0x000dab27e9b7aafa,0x00052dbe3fb66eb5,0x000cf7ce856345ac,0x000025496cdf84c8,0x00082f095b8e1e29,0x000e8e6db4cd7761,0x000b0faa321aaa54,0x00088ae73fef003b,0x0000000000000000}, {0x000d643fd4fac454,0x000c4870e138d616,0x00069ca59b85612f,0x000a26a00889a9e6,0x000c093e3dad6fb3,0x0006ce66bb43df1b,0x000244dae036271e,0x000c05182a82fcd4,0x0002559d8958ca2a,0x000a7526838c8510,0x0000000000000001}}, + {{0x00084b954cc2f383,0x000a776cd88b3801,0x00070c99422dcf3b,0x000bd45086f66f43,0x0005f7b81c0e8bc4,0x00004cad2493575c,0x00070ce4091825d7,0x0004f1ff4bbf4b9f,0x000d28bd9ac2a0a2,0x000c23e5ebf7a3b5,0x0000000000000000}, {0x000270e7ebdf9c15,0x000050eb783548eb,0x00081562bc5fd0b1,0x0005f68896b8a59a,0x00073bc130375edb,0x0001c5bd938ab1fc,0x000c19c89b28feaa,0x000f8d6e4b1c7f18,0x0009f7384e98a494,0x000a3f55131bf640,0x0000000000000000}}, + {{0x000a27923d9331dc,0x000f04ffb6351b25,0x0007f29f1e1c9bc2,0x00069b7d91e80528,0x000d48a56cb26370,0x000d9a9a20ff75d6,0x0000b393f52dd397,0x0003703dee3c5227,0x000f9c1c267288a6,0x00083849651d4713,0x0000000000000001}, {0x000d56c853c2dd2e,0x000a93bc1a8d521c,0x0008735173646598,0x000967ee4685de4b,0x0004cf35701ef418,0x00080b116b6dbbce,0x0006b03c5acf7cf3,0x000fe839b4243541,0x000841fbd8d1a9cf,0x0003a6e1730d1f15,0x0000000000000000}}, + {{0x000fdbea99958b96,0x000e01f76bf65ac0,0x0000c6778adf573b,0x000d0f51ffd85a6f,0x0004afe98c72d927,0x00087e8ec8173887,0x000d76d032ae57d1,0x0004df95e88800c6,0x000ec4042dee55d1,0x0008e8cd5760c30d,0x0000000000000000}, {0x000eac1084c10f00,0x0001c37bdb463a14,0x00076281603cbf77,0x00034037d48543b5,0x000f3b965ac3efc6,0x0009a7be5b6e5426,0x0003d0f87fba3664,0x0004ddb5ca9f2e20,0x000052649abbc317,0x0004b72eb6083655,0x0000000000000000}}, + {{0x0001bc4f61438294,0x00002964a43b5d5c,0x0005d7c26171c634,0x000967cc93c0fb82,0x0004b96145dca1b7,0x000b5c4ddfd06836,0x0007f0eec5bd3c73,0x00082d7bf8f0d500,0x0005b93c7771d6fd,0x000475f222990f21,0x0000000000000001}, {0x00026e01b4722367,0x000926340c9a0a1d,0x0007edb2bec04b5b,0x0005d17d0417ca25,0x00072b41c7280efd,0x0003c942f670df33,0x0007910fbfcef999,0x000437ef3a577e3d,0x0004d4c9039005c5,0x000edddb0ceb3a8c,0x0000000000000000}}, +}, +{/* digit=66 [{1,2,3,..,}]*([2^264]*G) */ + {{0x000a3a0634a3ce07,0x0008f38d14324956,0x0002232fc3562771,0x000d389e5a0479ef,0x000b882744b806a6,0x0000bd687af4d435,0x000d3172792b960b,0x0001f792e60e4cec,0x0009dcb17063be91,0x000c5902f6ffb4b0,0x0000000000000000}, {0x0004fbf6f215ba3d,0x000918e7c66f8fb0,0x00095b38bbb07b90,0x000022d201c5b207,0x000344fdf3937c67,0x0005a11142ee01b8,0x000cb5fc7b97506e,0x000c2ae4043311b8,0x000e1937f2450b7b,0x00045fa26a70cfe3,0x0000000000000001}}, + {{0x000fdc4396bfa539,0x00092b7edbcb88f2,0x00019d35421db912,0x0000a538d5dee79a,0x0003b035e9ea2a42,0x00021709fe9cf14f,0x000a5b749703f94e,0x0002495b47e8690c,0x0002ef0574deb7af,0x000dd4b09d6324cc,0x0000000000000000}, {0x000f7df3b9fe6e0b,0x000f9e25c764e67f,0x000b9153d851593e,0x000822d7ef6d9489,0x000c9238e5449117,0x000bd3333b1e34e4,0x0006cfb58cc8198b,0x000b7b487650416c,0x000068c07a8085b4,0x000b7b5e20cc8eb3,0x0000000000000000}}, + {{0x000af6a4691425a8,0x000eb7a958bb9efe,0x00085a7002151b0a,0x0006a8775208123e,0x000055575f5446b3,0x000528fdeea0c896,0x000a43b1b4d824bd,0x00096f550c5c7315,0x00050cdbb6610168,0x000b9c3638cd4a93,0x0000000000000001}, {0x00039bfed8d0ec30,0x000230c0cb3e1745,0x0004b3cb7f69dddc,0x000a60a97d0d2a3a,0x000c6d61d74827e9,0x00048c2e237c10e2,0x000c78dae64ccf95,0x00031e036445ee96,0x000bee13f244e4cb,0x00012ec220b81c78,0x0000000000000000}}, + {{0x0008b837d4bef687,0x000919e15922b2f9,0x000c8afde9c62c29,0x0009534c95a1a3c5,0x0008f604b1043ffe,0x0007a01a13fa2f63,0x000393b04cd8a8d2,0x0006e26fd0c22660,0x0000808a072545d9,0x0003871dd10699cf,0x0000000000000000}, {0x0007dfe3a4ea56b7,0x000094c38223ef03,0x000e8b66c87d36e2,0x0002766ee28405ae,0x000f0a065b535e3e,0x00084a317ded4b87,0x000866d3f53ac0b0,0x0007a0ee55860ca5,0x000a7080382b21bd,0x0002f3ff1d58cf1f,0x0000000000000000}}, + {{0x000760a0d077a674,0x000256d1eb02b933,0x000a86e63e76464f,0x000e22b473632441,0x000db481034e4b15,0x000914ef870834a6,0x0004c2dffa8bbf34,0x000d9d0466943feb,0x00031a2dbf893cc1,0x000615fa26911a87,0x0000000000000000}, {0x0002f76d13211f86,0x000b53e5f9871239,0x000bd8e3f6da4b87,0x000367de3555d6f5,0x00095ebc54c2b323,0x000413dbb2bd8e48,0x000ddef4e974c105,0x000de08da1d15f48,0x000eb47f901deb65,0x0004f245e4c32880,0x0000000000000001}}, + {{0x0005b2c93a39a68f,0x00012def13c9b690,0x000a5eaf60ed34ff,0x000eed4546115d13,0x000acc0704820ad4,0x0000c499c60761b0,0x00013af5dd51e45f,0x0007a978e5524abd,0x0009f811db1ec09b,0x0005900dab7d3aa7,0x0000000000000001}, {0x0005f21e6490481e,0x0009c0ea3d3b19a7,0x00068df5edf0364a,0x000c93c95e1d6b4a,0x0008333e2dcc0b44,0x000a8fc7be078322,0x000350437cb9512e,0x0005c965c20fe9e1,0x000d3176a887068c,0x000004e870a54162,0x0000000000000001}}, + {{0x00005c3b211947b8,0x00027fda98024305,0x000832a6aa27a459,0x00057be3feea006c,0x000ba0bb599ef407,0x0009c187eff74059,0x00032aa4ef6180b7,0x00072b398f55afe2,0x000cc5ddc46af4a2,0x000e611d5ea71271,0x0000000000000001}, {0x00085981da5484c4,0x0002661fa0fa34a4,0x00025e8fb282c8f6,0x00009d9542344d00,0x0005e720cc73e773,0x00081ad0388c6b65,0x00097a5b5f85f411,0x0006173a273c5fde,0x0000126c0a036ea1,0x00030085c778a65a,0x0000000000000001}}, + {{0x000dd3a67c2984a5,0x00072c7824703af1,0x000bf0c69c5b39c7,0x0000901a46f942bc,0x000f89e0174be31d,0x000b6326f7d38bdf,0x000eadb91bfcc1ea,0x0002541868bb8787,0x000a48a8ba038566,0x0004606d8787616b,0x0000000000000001}, {0x000e290f5d5f8e88,0x000eb41d33d54569,0x000662b634f1ba52,0x000fc5049dfd1d1b,0x00059be51c909876,0x000b7406ba93e420,0x000475a49355b9dc,0x00048963ea182651,0x000985ceebcf7670,0x00062eaa85c80508,0x0000000000000000}}, +}, +{/* digit=67 [{1,2,3,..,}]*([2^268]*G) */ + {{0x000b5648e1eb2291,0x000ba734c2f6e413,0x0008535398b202b7,0x000a8b23ebd7f177,0x0009fd3b0fc5e7ee,0x0003df55ddc7a0f1,0x000261d577641e2a,0x0008f496646ecb9f,0x000f2be9c112454e,0x000e02c23da6a9da,0x0000000000000000}, {0x0009d35c52fed679,0x0007671efd66d8e5,0x000904f29b91b401,0x000c352b084f27ae,0x0000123e88566129,0x000229faaea32636,0x00087f4c04620cfd,0x0005535f695ec91c,0x000fc2a2127c5854,0x000fa1c94873fa3c,0x0000000000000001}}, + {{0x0004b028b8f44cc5,0x000fa18c73e470d1,0x00046af781c684d5,0x000aadf0eb44feae,0x000604610320e3e3,0x0008fffa44fd9c59,0x000908270d9d9d3d,0x00088a6283f3ebbc,0x00060d649fcec534,0x000e1d44a603fadd,0x0000000000000000}, {0x000740ae33023df3,0x0000135f6a91ebce,0x0003780772b2000b,0x0004747ae7ea71ec,0x0007f6f03b13d0e5,0x0006603e33fc299d,0x0000d28b6e9df68d,0x000a2043747f8604,0x0006089683aeee37,0x00024e9926fb8de4,0x0000000000000000}}, + {{0x0006c9f15018542e,0x00079fb1f6fe5f6f,0x0005889ab7c13ba5,0x000129b7c5358418,0x000a9ac0306d8ad0,0x00008b37c84cfb84,0x000acf7da701dcb0,0x00014bb42427b3eb,0x0006e318d3d02d27,0x000958db234da419,0x0000000000000001}, {0x00031a9f2e6d1d68,0x0000ba7009240e02,0x00063eb8c05422c8,0x00081b3a6ec6a77b,0x000bc58689a5aa19,0x0006306dfb930061,0x00007013549ec203,0x0005410416293e63,0x0001a4d303fabd7a,0x000ac172f1e81c7f,0x0000000000000001}}, + {{0x000b9ad3970b9e18,0x0000bf6af8e430ab,0x000f59d55e652348,0x0002bd614bc56b8b,0x000183df0a6ecc07,0x000ee1786d25c98f,0x0003feabcc84059b,0x0007b20a09a6f26f,0x000d600ce79a2dfb,0x000f39cf2e6f0393,0x0000000000000001}, {0x00072a39f3507cb8,0x0007042b1470af9b,0x0007da313b04804a,0x0000e590e67c9622,0x000cabec90cccc29,0x0005e76e6a796f29,0x0001637cadb620bf,0x000d15b03af38ec0,0x000dcf7634087520,0x0001906c0ca6b7d8,0x0000000000000000}}, + {{0x0007eccab473c8b5,0x000354c80ff211e0,0x00045d2bf3c3aaa3,0x0005bdea352cfb52,0x00016f804d4fc256,0x000e5ad706ca34f0,0x0006619c96cff608,0x000490525e689857,0x0004163c78de4a6f,0x0007e4b87dad9b7c,0x0000000000000000}, {0x00035a48ce31e1e3,0x000c4a6aef85129a,0x0002e4b9afe02466,0x000a8b4fb7e5c1b3,0x000e4ea65ec8abb1,0x0009979db25393ba,0x00064b047eeb2086,0x000d6e5e56906ac0,0x0008d958b203281d,0x000de4155ed9842b,0x0000000000000001}}, + {{0x00019bc174c6d988,0x000cf4d9f702e572,0x000883fd073431fc,0x000f7c52e0996638,0x000331a7f7f18050,0x000e9196a2374342,0x00087a7400e71f3a,0x0003791b5d724c12,0x000499ca89fe518a,0x000e565820d945a9,0x0000000000000001}, {0x00035b7cb9fac00e,0x000a632b3417a9e4,0x000077b9768bc095,0x000afb46dde432fd,0x00032553ea0ffde2,0x0000725529dbe056,0x00071cbdddb8ca1c,0x000edb41a198fcb3,0x000f5060041dd314,0x0009a1fe368a743c,0x0000000000000000}}, + {{0x00059195e6de2d3f,0x00030ea9f4518076,0x000e3f066c30b9f7,0x0003bd981db2f294,0x00046e95c601e580,0x000782f5d49b6ae4,0x000f2a24bea15da5,0x000d57d82c482be0,0x000747b852b55a3b,0x000400c12ad66ee4,0x0000000000000000}, {0x000106fc9ab22be0,0x000f3ff85e4be397,0x00064c04ee049a9a,0x00068c1771b2f4aa,0x0009c56a4f6e25c6,0x0006c0243a7cc3be,0x000082558b6f8b1c,0x000d25ba47737910,0x0009a82551abc22c,0x00044f195c4a902e,0x0000000000000000}}, + {{0x00037e57ac34630d,0x000fc2c030f2d54f,0x000b84aa880649ef,0x000b55a13ad19d77,0x0002091bd296d1ca,0x0008f7f0b28c0816,0x000d7580c469726c,0x00021840b8cfb847,0x000fd3c1cc59a8b1,0x0008b90e2778fdc8,0x0000000000000000}, {0x0006cc1f42f98900,0x00026c3bf2f82644,0x0008d5f8bb74fc86,0x00077c747df08423,0x0005d41c77ea13a8,0x000556d8fe471d93,0x0008287e98cde5b5,0x000c068f4d40279a,0x000305a88e400538,0x000e77c091d74bdb,0x0000000000000001}}, +}, +{/* digit=68 [{1,2,3,..,}]*([2^272]*G) */ + {{0x000190a6f12cf864,0x0009e0eee766f86e,0x0005b775cd536070,0x00057c69d4566a98,0x0005745df1e07e40,0x00047733f9c06722,0x0006e2b1b1c2a5a9,0x000fc80987a24bcd,0x000f4061ae7293fb,0x0007d711f7042b89,0x0000000000000000}, {0x0003c1b0341e791c,0x000537daedd9c1c5,0x000495a12d7c48bf,0x0002d4a32c8c9765,0x0005a662fe9dfe7c,0x0007c6bad9faed52,0x000660c5c4df70a2,0x000bba7fb07624dd,0x00091b1d621abac8,0x000771b618ce5d4a,0x0000000000000001}}, + {{0x000c915030a7a399,0x00068b673999f751,0x00008c22b2afe146,0x000bf6a5fc300d5b,0x000b3e178c0bceca,0x0009d38258020b90,0x000176381c171fe7,0x000e86f32623a2f1,0x000f64b9bf3a66cd,0x0002c55668f5ac6f,0x0000000000000001}, {0x0001812516e735d9,0x000aea3ea58c5dac,0x000356a5f10de279,0x000295b07e9f7153,0x000f586ce9eb4d08,0x000daab1a3f7c783,0x00000a0030b2d7c4,0x000c2198f3166033,0x000184aa9d0c0475,0x000b11a6fe88caba,0x0000000000000000}}, + {{0x000ba5c8d0193b1f,0x0007644c52e9d2ec,0x000084d971b1a2c0,0x0000ba2904071452,0x00016420810c95b0,0x000726c12ee37ace,0x0005cdbfb3b34658,0x0001ddb8f12176e9,0x0002665465a782ea,0x0005989e91fb9ee1,0x0000000000000000}, {0x0000c16d55245f9b,0x000a5ab01d1b1ade,0x000186cc016cdfa5,0x000f20c1907f643d,0x000b819ce2692951,0x0001e463db499758,0x000551bae173a15a,0x000c9960164a1a60,0x0005b509da7db4de,0x000254d9cec8875c,0x0000000000000000}}, + {{0x000b44e11c323d06,0x000fd5d6d986113f,0x000894e506cf902d,0x00052247fd0b1d00,0x000a2782247b185a,0x0003bd1827ee7a96,0x00075cc817a81ab7,0x000d58e21da1b5a6,0x0006b1f8c96f3b0a,0x0005eb0b4feab1ba,0x0000000000000000}, {0x000e1e70f2721b75,0x000160a2caaa6a94,0x000c595ff3de4a5a,0x000a75c84e2aab67,0x000e555f145b7c4c,0x000c6003a47731be,0x0008f07e7fe03b5f,0x0007c95ac06b0bfb,0x0000ec8f9062bb21,0x000d58a73aafef97,0x0000000000000000}}, + {{0x000123c6709431e0,0x00010ae6e4e2793d,0x00084b28d12b01b7,0x0007bca51962e973,0x000f98e6a52dea7f,0x0002d8bb8dd35519,0x00063c8d202bed1f,0x00009ec87640cd7a,0x000e8802cf11aa3f,0x0002789d1734b8f7,0x0000000000000000}, {0x00015a8a3e8d859e,0x000534ae889a29d5,0x000da86637742cb4,0x0005ee22c15c8252,0x000ad4f9397a87f5,0x00093d8a684e2ab5,0x0006af7ba63ff2ec,0x0004f0fe6a52e297,0x000effff8321e195,0x000d2957562f8dbd,0x0000000000000001}}, + {{0x00023e7069e42e0e,0x000d2239c3ca8089,0x00005746ae0d546f,0x0009bcbb8ee8194a,0x0005279f5c84f339,0x000373c06a06d19b,0x000c70d34e701108,0x0008ba438171e418,0x0007c7306769b5b5,0x000064bf193f46da,0x0000000000000000}, {0x0005544b033a42d2,0x00061e556329a54d,0x0006aab67ee7a477,0x000c61bcc63287d0,0x00034397f22f1672,0x00021907d612d307,0x000ccf64c77cc188,0x000f80df2be0b759,0x0001db9d853899e4,0x0007cb0c4b9b73aa,0x0000000000000000}}, + {{0x000ff9f36c3349d4,0x000c3db537864c16,0x000d990835ee2990,0x0003c5fb1d408a12,0x0004b5ff931b6504,0x000452158a1b1e0b,0x000db8a2c29f72a6,0x0009d61be31c5985,0x0007a2320489621d,0x000480174d202617,0x0000000000000000}, {0x000294c195148880,0x0009ed88751bd34b,0x000f46ac8a70ae99,0x000917e5c0560f97,0x0002525c8ba24922,0x00048c3502a88bf3,0x000ee6a458ba7134,0x0003f607a14e42a8,0x000dd29054842573,0x0009c1acd66ae0e2,0x0000000000000000}}, + {{0x000edee700e2b9a1,0x000ca05fd3e47e10,0x000775354363597f,0x000b8ab9d14d9e5f,0x0009809ae6cb63e8,0x0008a4dd84740965,0x000dd249f1a5c96c,0x0004d2f79af0cb6e,0x000166e5361d2b7a,0x000692fe3d22a60e,0x0000000000000000}, {0x0002fa6f8995a329,0x0006e396d7a363f7,0x000d92f57cc488ad,0x0009d1958510a286,0x000c0b888aa8bf0a,0x00042decec317136,0x0008b9bdf9fc71bf,0x000ac0298d41b6cc,0x0009ecbb249e5d99,0x0004db314b57f810,0x0000000000000001}}, +}, +{/* digit=69 [{1,2,3,..,}]*([2^276]*G) */ + {{0x000d708c668b7357,0x000d96ce839038b2,0x00020e1ea62ce28e,0x000713c58763eab5,0x000e2c4523fd6ed2,0x000eae1cb271027f,0x0005e4f9c4b8cf67,0x0009601ad02024a9,0x0007d73ac0716494,0x00064337442ffcdd,0x0000000000000000}, {0x0007851b23ec84bf,0x000beedb8574d7b7,0x000286ebfe9b645b,0x000e45ce0c8710d8,0x00056a79aecb4766,0x000f379f83c2d312,0x000bbc5340ea164b,0x000df851521a164b,0x0009d5d5e1ac3081,0x000081b205779b7e,0x0000000000000001}}, + {{0x0003e4b56c489ffe,0x0000450823173431,0x000479ae5276717f,0x000cb05ce3d436d8,0x000ddf2257834d02,0x0007cf804300a63f,0x00048d555acde6aa,0x000b3233d0de457f,0x000aa55b4de5db66,0x0008a60379d9ac81,0x0000000000000000}, {0x000e90717067058f,0x000132c47bead613,0x00090e8a449f2111,0x0001c278b92dfa6c,0x000a20e5052e4286,0x000d62ef7fbb21a8,0x0005d03da29cea2d,0x0002b105405706ce,0x000dda27b321921a,0x000d13a8070a212b,0x0000000000000000}}, + {{0x000d23cf55c45926,0x0005b98778f5f299,0x0001705a5c8c8b02,0x000b88f43919b71b,0x0001fcb92372e0d5,0x00043296e160fa37,0x000a4970a89cc719,0x000334a3ae695fe1,0x00051e4b95dec2f9,0x0004d6275a594212,0x0000000000000001}, {0x00047e08dd859c11,0x000f2faa12f1b2ff,0x000ffb55ea0ad152,0x0005927a2d49016e,0x00063e898a743956,0x0007e768ee6cdbde,0x0009ce79201bbe74,0x000b64e8832a0a06,0x000cff0774d3af5c,0x00081d58cc25a522,0x0000000000000001}}, + {{0x00027f0e1b46fd00,0x0003f1c1b9a1af5b,0x0003c2c754fc491a,0x0007d3165cbaab1f,0x000360310665c2a7,0x000a6d64bd760e64,0x0004ce1abfa1968d,0x000d2b0e1701fb5f,0x000d3c4d7b466c4e,0x000f912ebf21257e,0x0000000000000000}, {0x000f6e44d2ef47a9,0x000cdeddd0d09650,0x000acd3234c8ca37,0x000cba5fb7244def,0x000f3acca56c2d39,0x0004d3ff0e42e4fe,0x0003498d03959e10,0x000b101ed923e651,0x0000842e240deada,0x00047140cf65c53a,0x0000000000000000}}, + {{0x000420de7a3cca92,0x00078c40b5d961dc,0x0000669c0650c56a,0x0006512b20ea2fcd,0x000c80cfdc1ced7d,0x0002dc4c42793f28,0x00014af2a2c66b61,0x00038712d0f465ef,0x000a3e10f498de28,0x000eefd43378b1ab,0x0000000000000001}, {0x000182339af2de22,0x0000f52743e62529,0x0007cb967f975c8a,0x000a495f3b942e5d,0x000442c93c4c7a6f,0x0005ea4e81af911e,0x0001c3361393032e,0x00046ad975cf453b,0x0008fb85fbd84437,0x000e9f19bef58359,0x0000000000000000}}, + {{0x0001c8a03a758373,0x0002ff5d3c5d987b,0x0008d28755f9b053,0x000696c9a6811aec,0x000671a05c762526,0x000b50917885c6fe,0x000e435b1cd97329,0x0007d6424b129ce1,0x000fb69b5d25c001,0x0005bf8fd3449e7d,0x0000000000000000}, {0x0003b3b4745aa0fc,0x000a304f13caa8db,0x000c42581d38c65d,0x000a26e3f1637449,0x00075086fb17831b,0x00070bccf51f20a3,0x000ec67794f66bbe,0x00008759114e8fb9,0x000e55fefba15c3e,0x0006b06274e840ba,0x0000000000000000}}, + {{0x0001392a85eee65b,0x00037ebdba8a73a9,0x00057c70feb31623,0x000f9841fe6064d0,0x000abb8ea58571cb,0x000f4d78e10f9265,0x00010194ea34ee68,0x000ee932bccc86c6,0x00018b05ba4d88af,0x000c7f1b666c9e95,0x0000000000000001}, {0x000bfc49e59e8fd1,0x000ae405208c75f8,0x000d373c5a99df54,0x000a977279933e71,0x000da64aa0b3f5b8,0x00036d2c0da96b6a,0x0002eed5227ca046,0x000b6e0ffb64c414,0x000baa05111d0f26,0x0006496cce8a3203,0x0000000000000001}}, + {{0x000c440343aef952,0x00019eda15115cbf,0x0004c0ca17c7be65,0x000b899e1e404697,0x0006bd56cb968d39,0x00077f25afafaffc,0x000e8460c48baa89,0x000cb8ddcae733c5,0x0004e7b4d8db3dd6,0x000d4730c42ef0ea,0x0000000000000001}, {0x000ded4336e0e996,0x0007761c36485b3b,0x0003a9ef9b460a85,0x000ea119c163b2e2,0x000f8699c32d39ca,0x0009afc21d6fbf2a,0x0006105b2f30acbc,0x000d2782e1795cd4,0x000bd0296c5de1eb,0x0003f540db331e94,0x0000000000000001}}, +}, +{/* digit=70 [{1,2,3,..,}]*([2^280]*G) */ + {{0x000566c4dd75c1b4,0x000a0570856265b3,0x000cbace31affa63,0x0002b4ed64645336,0x0006de49945b2d79,0x000ffedb2ccdc41c,0x0001239fc3fec1e4,0x00056c094341fb38,0x00028185bb5868f9,0x000adaf680572da8,0x0000000000000000}, {0x000e0585aa2d876a,0x000b95480f8f0fbf,0x0005be334d530bd3,0x0002f278c2d3c86e,0x0006b676d6c82d76,0x00039dec8e1488b5,0x0003e4b756194ec5,0x00094e5ad8a2c0fc,0x0001d4129e01cce4,0x000c459cb7e94c1e,0x0000000000000000}}, + {{0x000e172f71314c57,0x00073776d707122e,0x00007937b43cc86b,0x0005775ac42e1bf4,0x00055e0abab130a1,0x000a4930e58a6f41,0x0005a237af5f75c8,0x0004f7ffdb8ee4d2,0x0008fe7af47745ba,0x000467da1f09c11c,0x0000000000000000}, {0x000ab833ab90a6b8,0x000669bed4019358,0x00007b7fe4d74f45,0x000d834f00a5516c,0x000f035a36d318fe,0x000f8a31bb46f3d5,0x000972946e1df3bc,0x0001ef24a7e886c9,0x00085864c383ab5e,0x000a5b85a50c0d32,0x0000000000000001}}, + {{0x000d65b66d82688c,0x0003872c2925e382,0x00080fde81ac79d0,0x000a2b38a2432027,0x0005fa34422b6bc3,0x000948d55e24a659,0x0004efa7ed8f149b,0x00071011867cfac1,0x000bd1e94e578bbe,0x000803efa0276501,0x0000000000000000}, {0x000f7389f88ed003,0x00099b4f58dfa2c9,0x00027af2024d3bff,0x000a0ca5e0ca7fb1,0x00083782dbf7ba09,0x0009fa5d6101098d,0x0005231abdbaa4b1,0x0009c03b0d70ddb8,0x000cba60cbb85949,0x0003ba8d596ce326,0x0000000000000001}}, + {{0x000ec042211830fd,0x000f78b67d56f78a,0x000d0dd5f0085f0a,0x00077ef6d1879493,0x00009d412929cede,0x000f8f47fc6254c3,0x00038136cfbb6a12,0x00090a561c304af6,0x00064012770283e0,0x0006e8c77dd363b1,0x0000000000000000}, {0x0009730231689c1a,0x00078a2910104065,0x0006adb22ec70366,0x00007eaa5919dc9a,0x0000eff75ea6bb96,0x0001b154022f103f,0x0002e9dc4300dcfa,0x00023ebbf1abfdcf,0x000c7610a1be47bf,0x000ee8e48e43b23b,0x0000000000000000}}, + {{0x000366222079ff20,0x000dd6c255282fb1,0x000e5f65eafcf45b,0x0009b30515c02959,0x000efd5074c2f0d5,0x000460bf3169d34d,0x000d88198c4daf0f,0x00042d35aae9cd4d,0x0006f8fb3a3e2b92,0x000e5b7ee19179cd,0x0000000000000000}, {0x00015c0c8bdd26b4,0x0009fc2499885330,0x000c27ee4ed9b18e,0x000e30d901ee8c44,0x000a438c4d057961,0x0007a847d74e4722,0x0005e9ebd34c3ce2,0x00049ec1f371fc17,0x00062cfa628ad626,0x0000258cb8ba2199,0x0000000000000001}}, + {{0x000414eef61746f8,0x0002f44b16d63548,0x00091c53690466a2,0x000700b4b9b4826e,0x00069ba41d6fddfa,0x000f48b184a9d1c2,0x000af1a9a1ae5623,0x000688445e2f1d66,0x00028a0ef16e7621,0x000ea0a509e87455,0x0000000000000000}, {0x00016c4ebf9d1f30,0x000002405c88d64d,0x00021995ea2cb159,0x000476f59206340d,0x0007fbdb47138c1b,0x0001fc51a673c4a8,0x000c132d81d7d81f,0x00033035e2c568d2,0x0001981e0c2e86c3,0x000c9f25fcaa15dd,0x0000000000000001}}, + {{0x000b2a49ca5dfb81,0x0008d370aadb7c4a,0x0009f4ebf1398343,0x0004d610d25c9ac8,0x000b49c7f0f75d8d,0x000cfed5c3ca14e0,0x000cd7f4d6f7590d,0x0007d93cbfaa36f5,0x000dce79a65cb3d1,0x0004b4bd97f1011d,0x0000000000000000}, {0x00001207087dd6cd,0x00049477a85a518b,0x0007671964d279a4,0x0001d11aff88af2c,0x000a1dbb6c2c5f27,0x0005ba326b82395c,0x0003c2898f431018,0x000463dc513c0cb7,0x00069f2786b20305,0x00061fc5c18db944,0x0000000000000001}}, + {{0x00080adc636061c1,0x0006244e403a263d,0x000ad04e1deab320,0x00059720dcbb6130,0x000d66e8505322a2,0x00060fb3f5231b1a,0x00049e1cd79b6e2a,0x0006179c366e663d,0x000c6ea0d01277eb,0x000473883c4ffde6,0x0000000000000001}, {0x00040167fca5210d,0x000d95aec71f687e,0x0001c63bde59a231,0x00074dfa4af79a44,0x00060fd79e68ddec,0x000b613ae2a19527,0x000036b08e61ca70,0x0003e30d2b549e73,0x0008fb383d922e0f,0x000aa028c146216d,0x0000000000000001}}, +}, +{/* digit=71 [{1,2,3,..,}]*([2^284]*G) */ + {{0x000090e2290855e3,0x000c63856faf70c3,0x000a537fff7cf9c3,0x00002007d0317a7a,0x00010dc853b3261c,0x0006ccdf2c616875,0x000385cd6f188d53,0x000a2955fa1d26fd,0x00087bdaef2d7d6e,0x000b0f3173148e30,0x0000000000000000}, {0x00084c9a254216ed,0x00047cc0770de6d7,0x00035e4c8fa4bf8c,0x0000f637aece660d,0x000adedb7b99e891,0x00082ce72b5ced4f,0x0001d4d0fa07446a,0x000194600c851570,0x0004ffcea4152f30,0x000c96f31c15edf3,0x0000000000000001}}, + {{0x0005b1c7831bdcc8,0x000d027bafc7d23f,0x00050c19efdbe5d1,0x0007a7533cdce225,0x000573af279d2535,0x0006015a5e144120,0x000803b571412097,0x000eb203384ac91c,0x000edd68012ba72d,0x00091d825c3d8d2b,0x0000000000000000}, {0x00023553a39f8385,0x0005b8eaf27fe164,0x000c4539fb7ef933,0x000adc9ffa5830e5,0x000950a5e503466b,0x0003a2a96a1dcbb8,0x0001a89a62dca0dd,0x0001d5f98db48a88,0x000554b9506e0a31,0x000dcd69efeec8e2,0x0000000000000001}}, + {{0x000b8cbe4b97fc6c,0x000e5a71915ba245,0x00044263935f5290,0x000ce189775a8de5,0x000cd0549c8def51,0x00027c89477b9645,0x00047cfbc54728e8,0x0000c7cb6412bf4c,0x00016bf97806ea22,0x000feea61b447d92,0x0000000000000000}, {0x000a3fafee06d0a1,0x00032a2cf4c624fa,0x000baceff4ee9dd4,0x0007411d429aa2e7,0x000d154759f7bffe,0x00091cde409d4d9c,0x000a9a31ac9508e0,0x000deb0f736891db,0x000c17f78c058b52,0x000b43e218c9736e,0x0000000000000001}}, + {{0x0000f4f238352945,0x000f07d445a023b0,0x000551441addd54c,0x00008f85e62fb5bf,0x000ffd275f3aa334,0x00001eb4ed1f4e87,0x0000af1b08c2f8d7,0x00093b5987facbec,0x000542336f0bc311,0x000587219a8c1237,0x0000000000000001}, {0x00052f04dae724bc,0x000e221a68b0a55d,0x0005bdff6f9d5fa5,0x0007e3f2da24e831,0x000628b43c649948,0x000a393f548cc549,0x000acee934d621cc,0x000e0ce12d1b52e8,0x0000b93c4f0e6025,0x00077116663ffdcb,0x0000000000000001}}, + {{0x000b53f717a44956,0x0002c65fee76e1bb,0x000ae70d2da363ec,0x000cd6887a61f9c3,0x0002283a4e23325b,0x000ae6d909c60aab,0x000702a1dcca3dd1,0x000938d6a31c5517,0x0003e01f167bc2d3,0x00067596be65d549,0x0000000000000000}, {0x0007978a4e132cb0,0x0005df10adf1702e,0x000364f7e05324c1,0x000693e3866569f8,0x00082e303b6c9df3,0x000298933d134854,0x0003d7696e64c215,0x0000f465322046cb,0x000e839e686c1c09,0x000b7f8ac2e6f682,0x0000000000000001}}, + {{0x000e71627b708629,0x00012bf54385d85d,0x0007bb4ab478b057,0x000a3b2ccc6b5489,0x000f67fa6f6a05a9,0x00073cd5233bfef8,0x000c4c4f505a80c7,0x0006a3fe8c185c9c,0x00059e110f37ae33,0x00054cb440dc7a62,0x0000000000000000}, {0x00050c98bc7b0e1e,0x0005a04b3793331e,0x0003b8d31034a8aa,0x000c95dac900df01,0x000fd52027ee1c99,0x000b5be3aeb891a1,0x00042f3b68574224,0x000ae00bb37d5ba8,0x000a7f31f3f36375,0x0008c304743a2813,0x0000000000000001}}, + {{0x000c7e03b7aa645a,0x00091f7e39a28977,0x000995d7d443d68e,0x0009f39216c442e1,0x000b376cc21c4025,0x000705644f75967c,0x000592e3b90bca20,0x00056be0ef613507,0x00097f2f7456e836,0x0004b46438409baf,0x0000000000000001}, {0x0005a21df85e1c6c,0x000b0ab454500393,0x0004c5912b476c6d,0x00065a0d1aa49d94,0x00059cdd7e7069d5,0x00090a402b7eeef8,0x000283db40b228bb,0x0005dcdf2c268733,0x0005a42d37d44810,0x000b80e5d1af0360,0x0000000000000000}}, + {{0x000c14cb537e819c,0x0005f2b3bf6b7569,0x000cc275187a0bac,0x000925ef6c2d559b,0x00025a54533380eb,0x00019f0d6cd0382a,0x0001433bbf74a021,0x000bbd994d4caf68,0x00027772c53999a9,0x0003af249226a38d,0x0000000000000001}, {0x0006a46b5b493127,0x000bee887c8f7779,0x00045dd063eabbf7,0x00029199a0a8e117,0x000918d22e28aea4,0x0000ecebe77e69a9,0x0007cecb0ed2deab,0x000637aa98d20edf,0x000a194790e52888,0x00060c73078fbd0c,0x0000000000000001}}, +}, +{/* digit=72 [{1,2,3,..,}]*([2^288]*G) */ + {{0x0009af391a8697ed,0x0007fe5bb7320546,0x00080d05ad03f5f9,0x000f9b7973b1a3ca,0x00033b52add9800a,0x0003cc487ccc82c5,0x000f71a0da2ae069,0x000c060c7047e46c,0x000a21503aeb64ab,0x0004ed0075b1d33a,0x0000000000000001}, {0x000a2bed91298551,0x0000c49a79f6b129,0x00022374a19da463,0x0001305962f001a1,0x000a3a3cc4dcc90a,0x000188b4cc026cef,0x0002ff30fbb1d3fb,0x000c36e3761cad09,0x000dbdbdc6354b93,0x000b2ab73317cff3,0x0000000000000000}}, + {{0x00003f3665635e4e,0x0004e17fd85da26c,0x00052fd006ed5f69,0x00032d252f043a61,0x0005c8bc9cc74510,0x000f5370ca9348d5,0x000540b56333c4c6,0x000bc9a5ca533610,0x000d8071b716d25c,0x0002367337f70a39,0x0000000000000001}, {0x000db6fc5387c11b,0x000cfd3251b14397,0x000d84aa2bfdb755,0x000e38100cc3e62a,0x000046071f1f89e9,0x000e7012d9e47fb1,0x000e6ad97ec5c7c3,0x000698bc4de4f6c7,0x0000c6a07a4e7cef,0x000198a03a3a1224,0x0000000000000000}}, + {{0x0003b8713123ec24,0x0000419a51d2d151,0x000c8ee7e2c9c855,0x00039f110d3d8f17,0x0008ce870e320ac3,0x0000d4e71030e599,0x000ace1a5cbbd4da,0x0007c15bfcfe41d9,0x000408b36096093d,0x0006b26a0c56d39a,0x0000000000000001}, {0x00035ac9181317f5,0x0008d46d26280ae2,0x0003c29370c4bff6,0x000addee48f7eaa2,0x00030f605627b203,0x0003e03ba6674883,0x000cdd81ae186660,0x000d8bc04a9667b4,0x000cbc45ee0c1b5b,0x0003d34e81dc5ff7,0x0000000000000000}}, + {{0x00063b161dddd94b,0x0003d7c93f0cc576,0x00022d6ac11071af,0x00012d84b9149bdc,0x00088e44e4632e63,0x000448cc8ec50d4c,0x000f89a6c85277ac,0x000e128700eabfe4,0x00042928ea38e5f2,0x000e293e261880b7,0x0000000000000000}, {0x00051028cc113b68,0x0001919b6a14e2fa,0x00082dfd5da09549,0x000ca662e13022a3,0x0002996fafc24233,0x00018dea4f505fe4,0x0005a2d96182166a,0x0000199ba55815ea,0x00077232622a4ac8,0x000311b13c3b8133,0x0000000000000001}}, + {{0x0008673adf41d204,0x000e2a0a390575d9,0x000dcb844f35fe0b,0x000b6f4f917550d3,0x000fc6ecf7285d09,0x000925f580342d9d,0x0008746022d9fb4c,0x0001b6b73d4bc619,0x0004aa1c30d7a3bb,0x00003bf32bccb3c5,0x0000000000000000}, {0x000783e73d6e9052,0x00006db6e57f7b8e,0x0005d1afd274f869,0x000318770758e016,0x000290175b314342,0x000e6d25df464413,0x000da9ee1f0d5093,0x000297855e553138,0x0007c7b64937a717,0x00057ccbf262664b,0x0000000000000001}}, + {{0x000d9056eeba30b5,0x000355a8c404c7b9,0x00039ffec5e11fca,0x000c3ac7b4f4c637,0x0001ace1869851e4,0x000e1816afc7de47,0x000d517fb7e6978f,0x00040e256f62bd56,0x0004bb4c53034a6f,0x000aa98eb4b9ad2d,0x0000000000000000}, {0x000fd2909662ab1d,0x000c55f8463596cd,0x000074f62f74b11c,0x0004dfc43f12c5ee,0x000ad69405653ae5,0x00089f1d6e4668a4,0x000187ca431697ab,0x000831cc4eed080d,0x0003a69f7d775edf,0x0004e434435eef6f,0x0000000000000000}}, + {{0x000a1403851eb93a,0x00085c9ae49b53f2,0x000926a512455b57,0x000ca729dbef4a6b,0x000b8440248e399a,0x000191c77a9a0e66,0x000873c8f27a8296,0x0005408861a08ed7,0x0002bbe9c5e3865f,0x0003549235be42ba,0x0000000000000001}, {0x000a46fbe3334020,0x000abf1524bdbc9e,0x000b376fa382ce55,0x00075b35cc1fb214,0x0000910d13b19369,0x000cf5d3212ad3b7,0x000c7e93b691b3bd,0x0001ba22d0312eb4,0x000285b3975e5da3,0x000c999e2ec96eee,0x0000000000000001}}, + {{0x0005bf15f46be575,0x000a1061cb09c821,0x000ae2de789e5912,0x000cecccd84851c6,0x00001b95ccd21d74,0x00032dddf26a2851,0x00049210122d3f6d,0x000f02c5d952db55,0x0004be99796a4aa1,0x0004078cde88aab2,0x0000000000000001}, {0x000d753b80855f9f,0x00078288aff9b92c,0x000f7cdce61cc49d,0x00048cc3dac4e445,0x000cb0ac2a937fad,0x0008c5bdda956fdf,0x000bb3e81841ce29,0x0005170e6c819f12,0x000efd73ecab58ad,0x0000f476a3a48130,0x0000000000000001}}, +}, +{/* digit=73 [{1,2,3,..,}]*([2^292]*G) */ + {{0x0004f332dc67a3c5,0x0003213d82a4383a,0x0000fc0621356af6,0x0004ca32ca05acfd,0x000c1e80103ea886,0x0009dd0ff4c2e9c7,0x0009d764c64b7589,0x000680c128a8c988,0x000fdb7cd881a125,0x000ec5f09f58ad77,0x0000000000000000}, {0x000ecd4f9cda86d6,0x000c4c41a633a8c6,0x000847c2f58ecb1f,0x0007b29592dae51a,0x0006e268f50e3a7d,0x000a27de2a3b5eef,0x0006d540d7a599e3,0x0000d01f9b571491,0x0005e52e5204fbca,0x0003d9eb48615c67,0x0000000000000000}}, + {{0x000d9eaa867420c4,0x0002f7f5caa2ddb0,0x000c31038c0ddc32,0x0001596a08fb4b57,0x000a6d9ca6980a65,0x00095c78a8ab32e2,0x000ba78e5808eeac,0x0004f5f9923d5a32,0x000ad1c8d3bbece3,0x0007098f8b845926,0x0000000000000001}, {0x000843645beef787,0x000fa28875d75316,0x000e13608c5a90e9,0x0005556eaf90c364,0x000dacc40e05857e,0x000c5012b59e332d,0x000230b2b76768f9,0x00032932d53c8a76,0x000999fbd573bdff,0x000840eee9300114,0x0000000000000001}}, + {{0x0002362363f6901a,0x000be66748446167,0x0005f5c47c0db36f,0x0002fbb39fb54024,0x000e344525a7871f,0x00071375bdbedf63,0x0000bd89f085fb8e,0x00001c59b6e647d5,0x0008031fa2f2ea43,0x000d9b58012b6657,0x0000000000000000}, {0x0003015fd48eca8d,0x00082cfde2151d47,0x000e4d908c99616e,0x00004977d4ec3b2f,0x000f513df9ad204a,0x000b66641e3ee923,0x0009df2175bb5d92,0x0004cdb5c3c90fcd,0x000de8809fac5725,0x000985c6981a627f,0x0000000000000001}}, + {{0x0001a0d912586411,0x0001b0d49fbe5702,0x000bb57b277de5b2,0x000a4b2d7291e7e7,0x0008c16da29ce1e6,0x0008f8b71f4426f8,0x000fbf76a6ebaff6,0x000cab510adb9995,0x0004b996a6ec18d2,0x0003bc3ce11f1f8d,0x0000000000000001}, {0x00004c4051321f3c,0x0003af34703d798e,0x000dd55e68b6b0a3,0x0007f09cb14161e8,0x0005357558d9c473,0x000d9a485a90c00b,0x000dac2508e73fc9,0x000ed252e5f5bb09,0x000b1efcb4ba2132,0x000593c58bf23933,0x0000000000000001}}, + {{0x000d1217cba9c6d8,0x00087cb7a562f7c7,0x00004fa0d0ea0e23,0x000c05e3acd8379e,0x0006d159e80cdee0,0x000029ffd63834d6,0x000500f3ee777b61,0x0007b75be130d659,0x0003756de768a261,0x000a541b809584c6,0x0000000000000001}, {0x000af54a67204972,0x00085857b547d484,0x000954a25746036a,0x000bc1881e0295bb,0x000d3d231831b4de,0x00002708fd517190,0x000b1e9571812770,0x000c3e25dfa6c88c,0x000fb3c57a9f5467,0x0001d404949d8103,0x0000000000000001}}, + {{0x000a34fad1eabbbf,0x000de71fa7ad8210,0x000d67acfecf1047,0x0005ec7ec527279f,0x0007906ba2abc3a7,0x00066a25ce54f7d9,0x0001e7d558ddfb44,0x000ba77bccd91efa,0x0005405027796dc3,0x00072f9ae8e511f6,0x0000000000000000}, {0x000feaa4cfcc2a79,0x000d60cf749b854d,0x000ac1632a7218b2,0x000a7ce21f4e0ce3,0x000550325628caf5,0x0006b7d84c8e6b8a,0x00081d8e5bd9a461,0x000bb7affd9d5730,0x000dca775afff520,0x0006a52e629d73fb,0x0000000000000000}}, + {{0x0008f6b8297fa26e,0x000d76f386ac99ac,0x000ce235f3a2acb0,0x000620801a076df3,0x000c62031f455ef8,0x000d6e488b1627c3,0x0008ebc5bf3c9afb,0x000c4512c254ea59,0x000abb87bcef237d,0x000cb4dcacb5a5ed,0x0000000000000001}, {0x0008e7718ef189e7,0x0004207cf70cc015,0x000e9a66810fd747,0x000d69bb7c99d1b5,0x000454409a3962cd,0x00014e1d3b541825,0x0008bec33aca5afe,0x000a9c5e4a92268e,0x000bf2e95060a307,0x00093a289c336375,0x0000000000000001}}, + {{0x000781a84aba36d2,0x000ad7584291d55b,0x000992c0a266ea73,0x000e02af20e9954c,0x000ca4d73d175169,0x0001612ee12718e0,0x000cf1ded50926de,0x0008c1060d91c638,0x0007dc332b5998df,0x0002624eb7dc884b,0x0000000000000000}, {0x0008eae21aadf4bc,0x0005c2a9f4bf2cd7,0x00086c74c6b37272,0x000a4de4b5b5158f,0x00093ba4800d6736,0x000138590e451f46,0x000263ed2239cb95,0x000545bdc4c56f5d,0x0006676d4c0f8acf,0x0004038bbd0743d0,0x0000000000000001}}, +}, +{/* digit=74 [{1,2,3,..,}]*([2^296]*G) */ + {{0x0004fe80d04c169d,0x0001452da244cb71,0x000cda8b72206f04,0x0005887084ee9fa6,0x000920e111da7c3d,0x0003bb35ef1c2673,0x0008e61906e57c45,0x00073eebaa206a85,0x000dd3b5bf458238,0x00015dcf71c4a720,0x0000000000000001}, {0x000605cdd81e2955,0x000ac0e98756fb91,0x000f0286c4ccda7a,0x00017819b4372718,0x00031dae0a5d8a40,0x000720f8cb219351,0x0003217261dafa40,0x00006fb18c8c40e0,0x0003c7d483485194,0x000d46e02770cacd,0x0000000000000001}}, + {{0x000b16ebe5676f0c,0x0006e7f47e5b4652,0x00071c81701e6ae3,0x0007f8d6e3a4cc33,0x00067b29431c31ae,0x000869125faeb29c,0x000fd62e16be2afb,0x0004934ce6d0052c,0x000cd74901063882,0x0002e62b021800e1,0x0000000000000001}, {0x000a39e3523a9de5,0x00076121f34b3253,0x000d9c36b4aaa3b5,0x000b23992e0442ed,0x00023d2725144419,0x000ced8d6e4d3747,0x00011186d58a7080,0x00050cd63ed11b1f,0x000c6faa6d7a4d0b,0x0002559c897561f4,0x0000000000000001}}, + {{0x000657d806465325,0x00095fd29c362706,0x000e70b9e028365a,0x000fa40d084d6b18,0x0009d80bbbcfec68,0x000a06728aae56df,0x000361eb62533738,0x000dc77e1c92bd4a,0x0004fbeefd3c11b1,0x000cfba52dffafa9,0x0000000000000000}, {0x0009acb1f00a47f4,0x000024e668d3f2f1,0x000095d2d5d23cbf,0x00076b697f105b84,0x000da5b550d7489f,0x000c9d3a15e73345,0x00093e0d2b26a8fb,0x0001e4494adfd6a2,0x0008e6417e038745,0x000fb0b051833181,0x0000000000000001}}, + {{0x00013685fee3f167,0x00059193a123f397,0x000f28bae616ef5a,0x000d2c6567b33050,0x0004dae5b6596da7,0x000481adbd59fc9e,0x0002c1618dde4a40,0x0001f2a194682fe9,0x000c05bc15adc043,0x00013edb30fcceea,0x0000000000000001}, {0x00047d7a2900c6ea,0x000e954cf35b8dee,0x00074908606c425e,0x00094a88cdac359e,0x000fbef2c858622c,0x000d0f745810d20b,0x00041d71ca77f658,0x000782f59f1e4267,0x0004af36c640314c,0x000e35ec0709c829,0x0000000000000001}}, + {{0x000c5c739f094c18,0x000b75fcd89a8d55,0x00071a4d047c0dcd,0x000ef9a3d498ccfb,0x0008f669e7edb8ba,0x0008d6b13c20d8ae,0x000a564d16a48c79,0x000250d241703f4c,0x000680ef9509f9dc,0x000970b2c17a388c,0x0000000000000000}, {0x000778808942861e,0x0009abe8d8d33c23,0x0009bd2fb189dd26,0x000112581e8769b1,0x0009d657bacd662b,0x00084fbcaa8521c1,0x00091c546adc05d5,0x000c68fad3f4e938,0x00057bce78617aeb,0x000701e39a422608,0x0000000000000000}}, + {{0x000d79f624f8eb1a,0x0008e9c5d103b9cf,0x00071b65a5829554,0x00034991b8462f3d,0x000b51453c2b2409,0x000cf62fb341c087,0x0006cdb37eea50d1,0x00027ada571b02ab,0x0009e677d1b49b69,0x000004b52dc4d6da,0x0000000000000001}, {0x00042f171b839d5a,0x000765b6cceda32e,0x00009960889dbcc6,0x00040e8e2e336d9f,0x0001f34cefd7de54,0x0008bc6b5ab410a8,0x0000cbb91782d60d,0x000a95e22f3046d8,0x000775cf7c99291e,0x000282ca473be09e,0x0000000000000001}}, + {{0x000b6a03ba3e1ec6,0x0006b7fefa851ec2,0x0003a92f6680a7bf,0x00075deabf4905c0,0x00096ff483a6ae8a,0x000c163b2b784673,0x000be098999b6fbd,0x0009c4a535388968,0x000b8e919419a12c,0x000b0bd87c889640,0x0000000000000000}, {0x000f69e3681b9e47,0x0008fc5971b3c868,0x000bd601db453ea0,0x000aee960ff01a96,0x000a0347158b6a72,0x0008994151ef1dc3,0x0007de4b70d9ea4c,0x0005a29068423993,0x000e4276c73a1788,0x0008a834f8bee71a,0x0000000000000001}}, + {{0x00059366b70a0b8b,0x000d60abadb72457,0x00087caf5b975b84,0x000cc3f1983f793d,0x0002e94d8de541ce,0x00090b687cd8885c,0x0006b6cd952f2acf,0x000016cc05b6d504,0x000f97cdb266e5bb,0x0007c95959c78452,0x0000000000000000}, {0x0001e70e0efd1f63,0x000528c8989bfc93,0x000c244410e36663,0x000645d5f14f2667,0x0008cf8059cc8bca,0x000d2134f71c94e0,0x000ba7b1d1a84e91,0x000e21c35b983489,0x00003bfc5dddbab1,0x00099886156a8fa3,0x0000000000000000}}, +}, +{/* digit=75 [{1,2,3,..,}]*([2^300]*G) */ + {{0x00083fd796b1a72b,0x000c549640b910ec,0x0001f3c076c5166d,0x000b7e4e553ff932,0x000e97e2f7e676d2,0x000a2cdad5097c11,0x0004bc0c5cedff5a,0x000757d09eef397f,0x0007434958d95f66,0x0004eee32eada9e7,0x0000000000000001}, {0x00041f85b3f06836,0x0003aba0e3496ce6,0x0007931548ea6b77,0x000149273aecbf5b,0x0001858c42c5d2ef,0x000ddc70d91528b8,0x0008d341c157c1ef,0x000a690e10df0a32,0x00064f5d30760fd8,0x0000d14ec3b44ea3,0x0000000000000001}}, + {{0x00087f011572bb6c,0x0000d126f1e48db1,0x0000d05c4b7b6201,0x000a2ba1e0fbe5e8,0x0000224d802f13f0,0x000e0b8698189e96,0x0001b4bc0b8af43d,0x000c706b742b2859,0x000d9196336bf7ae,0x00007fec0d2fd363,0x0000000000000001}, {0x0000b0b6ce7179b1,0x0007cde982e3e244,0x000d814a6fad99d6,0x0007d3497edfbb4d,0x0007a46c6afc84ac,0x0004cd907a63bbd7,0x000909a18bcc3d68,0x000d756b5193f098,0x000f37ab5d6e0581,0x000ffb06adf4d102,0x0000000000000001}}, + {{0x0000415e4f4b1ce2,0x0002f16bd8544ce9,0x0003e7600d6c664d,0x000d9aa401912f05,0x000dd3e0c268e1e8,0x000f14013443dcda,0x000f59cbf936e212,0x0004b8aaec39252c,0x000d652c200b8cef,0x0000aac64c11e47e,0x0000000000000001}, {0x000c08e2e7a2dfe0,0x00048459176893a7,0x000c7c5a8f0afe8a,0x000e999b6f043d0d,0x00000b3c3cfd6e33,0x000b4fbc5e2a496c,0x000f5eee2690de5c,0x00073d2db4511dfa,0x000fecdd2743e927,0x0004f4ade6a1948b,0x0000000000000001}}, + {{0x000039609cadf211,0x000339488fdf258d,0x0000e39945a1e037,0x0009a8444d16fa60,0x000c454408c23f53,0x000ef729fbf7f8cc,0x0001fe369b8abc6b,0x00053dbd87a47666,0x0000f6266a890341,0x000de3c7b4bcd279,0x0000000000000000}, {0x00006e205ec9aa47,0x00063cce1af477a4,0x000f7fcf64572e77,0x000c931743d00999,0x0003902adccdc1f7,0x00041be26f8b87a1,0x00006779ffcb96f5,0x0001c0636264bb59,0x000484331c0db1d6,0x00076e5585c8ae19,0x0000000000000001}}, + {{0x0005899b377cc15c,0x000f315fff92e2e8,0x000a8cf599eb4a44,0x0002e4cfca1a3e87,0x0003f205f34e217a,0x0009eb1362e4f28f,0x00061c84aa7c205e,0x0006fa76515b6ace,0x00083aca37e55058,0x00030435e870f8ec,0x0000000000000000}, {0x00002203ed113de7,0x000655318327d42e,0x0005d903904004d9,0x00094adb7cd1d2f1,0x000ebca1242c89d2,0x000af2a1423b5bb4,0x00066f393818e824,0x00079a30444115e3,0x000183b2cd6a53de,0x00047bfd324d8249,0x0000000000000001}}, + {{0x0001ee196db2cf29,0x000718a1d049034d,0x000acaf386ae53d6,0x00006d746605e4e6,0x000c4458e136fcc3,0x000b7a1ac677dc40,0x0006ee4bb331955a,0x0007b95c386ed47f,0x0006640fb4384103,0x00081000a37bb752,0x0000000000000001}, {0x0008cc55b8f1bcdb,0x00025171d84cd78b,0x0001b8eb12ee7e03,0x000bd8c564c45f59,0x00076c2283df12d7,0x0008a36461b03bfc,0x0000c8cafd6fc812,0x000ae72b6275a058,0x000438282511c376,0x0008780ca3213fce,0x0000000000000000}}, + {{0x0002c0f6f353cadb,0x00056c17179cb81e,0x000af140db008101,0x0001bb5e63cd1fa8,0x000b0729533d8217,0x0006676827544c77,0x000fe1b0143af56f,0x00045a759df2599e,0x000962b593accffd,0x0007a3c61321e555,0x0000000000000001}, {0x000e58a6aa5358dc,0x000739bb4d42d566,0x000e6ef3760e0cfd,0x0004fb370620ef46,0x000c2253e0f9e680,0x000b5c8c64f4e9cd,0x00008daec6f6658c,0x000504153037a80d,0x000215b47cc959be,0x000e4e7aaaa8653d,0x0000000000000000}}, + {{0x0002bc233166381e,0x0000199700029af5,0x0003201dddf2a837,0x00085cdad02e2c74,0x0006daea6cd36d88,0x000c3f784e7e3512,0x000e882fa40cea6b,0x0001eec16a3a5130,0x000c43706ace2f12,0x0009ccf3c3a16a13,0x0000000000000000}, {0x00064823f9e7a6c2,0x0007bae8e4729ac9,0x0003fe54edfeed8d,0x000a4e7bcfba42bd,0x00039e917bf88ec6,0x00004163c606be20,0x00009b017d5a63a6,0x000c276869bd6bdf,0x00094144ff021410,0x000a13f038cdd94b,0x0000000000000000}}, +}, +{/* digit=76 [{1,2,3,..,}]*([2^304]*G) */ + {{0x0002f8bd1741a941,0x000d4f3177637189,0x00060a2c8b16c0b1,0x00012ea49a688a19,0x0004556eef3eca0d,0x0000c833814746c3,0x0008d8b842db71e9,0x000b1bf4ae9e3fdb,0x0000ee706bc576b9,0x00054e85a8de649e,0x0000000000000000}, {0x00032799b9f19edf,0x000975259805923c,0x00076f95241c4760,0x000d3b18d6c8d637,0x000fa677dee0ddd3,0x0002ad9334e5bcea,0x000ceb27ca464781,0x000594d56dac3990,0x000338deb39d6e55,0x00096c7a9d83d78e,0x0000000000000001}}, + {{0x000d7b49014b2147,0x000f134215e0ae52,0x00084df8cb33a781,0x000b10bfaf858985,0x00065449e13a9c3a,0x000859368bea1a4d,0x000d78c2c8134dfb,0x000ba7d94439b51f,0x0003f1c754b1e6f5,0x000d464cf5d47b00,0x0000000000000000}, {0x000c286403cdb3ba,0x000c15d6597a8e6d,0x000e5e853fbe3d2a,0x0004c9898bd192da,0x0003c76032d13f80,0x00038d5e8bdb66ee,0x0007ea1349bcb167,0x0001e4001679e16c,0x000f1f584d22ec20,0x0007b7d9f317a982,0x0000000000000001}}, + {{0x000196d862941c7f,0x00015fd4c9380d4d,0x0000a23a9cf4968a,0x00039223ad85ace8,0x000b20fbe77f6374,0x0003ae3be886de7e,0x000a36dd5eea175a,0x00061d0a78e19da4,0x000a64f9638a5733,0x00007bdb82dd1f3a,0x0000000000000000}, {0x0007fe74ebe49201,0x000a7702be0b4d0b,0x0000cad19bf2f6ec,0x000d6c0b5b514f13,0x000c08518708375c,0x00035c8f337b8d0b,0x00055fc06adb3974,0x00083bfc4b137fa3,0x00045b3d91207a07,0x0009d53cd23a6fdf,0x0000000000000000}}, + {{0x0000a67f092fe026,0x000e142efbca10ec,0x000cc433a343767e,0x000f6b9e68131944,0x000e836fcc884370,0x0005328231b4d1b5,0x000895d85b956e68,0x000117afd7ce3e4b,0x00028a48e23cd96a,0x0007756cbf1cc4fc,0x0000000000000001}, {0x000c45bb98d0ddee,0x0002774201b8565c,0x000b4f52020bec2c,0x000bda65cd76ab62,0x000ae2fb221ac3aa,0x0007ba962fd348e9,0x000e7c381d5e875b,0x00068ae119edacdd,0x000b495d57186eb9,0x00056fdf795bd0d5,0x0000000000000001}}, + {{0x0003ca5ab59ddb41,0x000f7f1ebc9c6cc7,0x000f57e85f634cbe,0x0005a16eddf122c7,0x000c8a63efc695ac,0x0003541555a07f9c,0x0002a4d80e98783f,0x0006230998aabf3c,0x0008cb24667129fc,0x0002de1b3b8ba86c,0x0000000000000000}, {0x00062aec0af56692,0x0007719a7045c1fc,0x0005b9718436f251,0x000c9b5d581f4e26,0x00069b5489f5725c,0x00051e7c1aa8e91e,0x00028ec256fe93ea,0x000487a680376ebe,0x00030e63df392cad,0x0009887cd96ae342,0x0000000000000001}}, + {{0x000e42367a9d725f,0x00039383477d80ab,0x0009bf84781ae655,0x000527eaf1389d3f,0x0004102ffac6350d,0x000a3b0583300052,0x0001cf3332af83e5,0x0006d633aac030a5,0x00033508c7e87b5f,0x00094e54cf554461,0x0000000000000000}, {0x000f352ebddfa61b,0x00025e3d5e304a79,0x000673478bec8a85,0x00072acfcd082890,0x000446528a7ef652,0x000120a7a607746b,0x000c8aaaa126f2d6,0x000e0714c411ed8d,0x000219322242ecc1,0x00076f4dd29b9909,0x0000000000000001}}, + {{0x000c19050d8003bd,0x0000966c18887790,0x000653ed598ed239,0x000bbd18179a89ab,0x00050e32e18d6ebb,0x0003a242275a6e33,0x000cf9fda141c240,0x000cba7fbc6f732a,0x0007d9042bdcaed5,0x0004cddd17f09749,0x0000000000000000}, {0x000dc494bed8e1aa,0x00072f9b4efbdcb4,0x000846e40ceacaf0,0x0008e7a08c964b79,0x000b697992c6dba0,0x000a31236d2a3b33,0x000f13c67e503cfa,0x000758371b2297ad,0x000a30fe4d1b0d5b,0x000937f2f3ba13a0,0x0000000000000000}}, + {{0x0001e3e792f54f5e,0x0002d66e1c349e9d,0x000411c782a7cb86,0x0004b067774f6f73,0x000f3d88b7029f91,0x0008ac9342be145f,0x000fba10730a2fc6,0x00017ace014c77dc,0x0006ebecfe34f062,0x000f04b7a85b9087,0x0000000000000001}, {0x000e45d39d99da4a,0x0006122af68cfe2b,0x000beab553aeda14,0x000338ecc8cf47bb,0x0007a9f26575d185,0x000dafc93ebcf570,0x0006b569b4f26152,0x000a1c5170968500,0x00059575a58e4408,0x000d43a451b6b327,0x0000000000000000}}, +}, +{/* digit=77 [{1,2,3,..,}]*([2^308]*G) */ + {{0x00031eea8652aa77,0x00075a9b33eda2b8,0x000ccb42fd2dd1ac,0x0005ab8fe55769fc,0x000ee2ea2c9e4a86,0x0008effb93c60208,0x0001522321dff3f2,0x000e5124fb782d6b,0x000f961e96e29dc3,0x000a6d2f39193c4c,0x0000000000000001}, {0x0005e63b7e317479,0x0009d667da851f03,0x0007183aa797e9f3,0x00059f9fe886c75f,0x000d96d9e68574aa,0x000ee25277706045,0x000aa94b18ceb8f2,0x000d5bc3971ae4bc,0x0007b608157fc8f0,0x000150dd6428487b,0x0000000000000000}}, + {{0x0003f45b89aa7776,0x00091f2af99e2d25,0x00076414d1e112b4,0x000800014a8e2b12,0x000ffa17691a71c3,0x000bd4233ab0f6f9,0x000bd49cf4e764a4,0x0004012735f0ccb8,0x000037d3dc7fc071,0x000ae03da811ddf6,0x0000000000000000}, {0x0004a8fb960d3228,0x00048eb39e1a42f2,0x00007ea05b5a3c40,0x000b7bd9d581bb93,0x0008feeb6ce36fa0,0x0000329bb28b7b8d,0x000e608fcc8cab6a,0x000504c03b085c44,0x000593cce3f2c4c9,0x00093bb2ef5a3d51,0x0000000000000001}}, + {{0x0005eee4a29b9e00,0x000dc20740482d3d,0x000e9ca5a465e210,0x000dbcf9b04be3f8,0x000cf79a10d11c21,0x00098890d16c757a,0x000df4eccacfdb2e,0x000f11b8ece9836e,0x000bc4d7a9a35695,0x0009ee2c9b7d003b,0x0000000000000000}, {0x000e2dd26ff13203,0x000133cc51c6bc72,0x0009ca679c2a2da6,0x000951011dc4426e,0x00093bedba6ea308,0x00088d9217c2a099,0x0000e9979694eb83,0x000d521e97150620,0x000035d356c66be4,0x000161e2cef90abf,0x0000000000000001}}, + {{0x0002a5d7a2fe5424,0x000c5abd77fdcc05,0x0007ae8084591e67,0x0006481030c17511,0x0001f92e55b534fb,0x0006bd1eba6c21b3,0x00096514d056aef9,0x000df597cfd93bd8,0x000c4de2939ce9f5,0x0004ef6737cbaf10,0x0000000000000001}, {0x0006ccf17f5b4aa6,0x00049ba070d7b272,0x00033084cb0ef8a2,0x00018fa7fece71d7,0x000e7f11b328ddbf,0x000b8c5b898e7f8f,0x000699bc5842d33d,0x000944b7141938ef,0x00039a13d619477d,0x000d5228db36f5cd,0x0000000000000001}}, + {{0x000b8747f420dd4b,0x000fb52bb177eca6,0x000f43dcaa9b6760,0x0007a3f15a0ec05a,0x000736800fc5d305,0x000bb74572a151d8,0x00065a96cf22ff7d,0x00019519394cd42f,0x0008227699f90840,0x00068419e24fe2e2,0x0000000000000000}, {0x00025578363927ed,0x000574db66d688d6,0x000d409ddc1459fb,0x0007f82b70d69f87,0x000cb67c0931340b,0x000a17e9ba495398,0x00095f5bb6fc86af,0x000372330a201b0b,0x0002bfb684331f96,0x000759b31f0fa4ba,0x0000000000000001}}, + {{0x0008a9686e50ea9a,0x000fa53caf16bb29,0x0001e6775d854b08,0x00037450aecb3742,0x000a7e3971ac67a8,0x000916720ee3a053,0x0007ef527af5a2ad,0x0008c78907d26a51,0x0000187ff88dddeb,0x000580501085ffc1,0x0000000000000000}, {0x000bd54c631b5ef7,0x0000a533ff81bdf1,0x0000234fa962cf5c,0x000a7f99121d8411,0x0004d769fb90a1f3,0x0005f3f42e621b7c,0x0007e5f02655798a,0x000d995fc9fe9a95,0x000560ed712fc645,0x00059000f20fc194,0x0000000000000001}}, + {{0x000ca8aff309b100,0x0000c85dcacc4c44,0x000282732cee9318,0x000efc73a9f020ed,0x00070ac2ec46ccda,0x0004edd612a22d61,0x000fb7673b4cf6d2,0x0000510828deb00f,0x000a061d4c49e843,0x00016c8aea6c342b,0x0000000000000000}, {0x000ffaa28b87e79b,0x0001bb9617a5663f,0x00097f1ba9e99062,0x0002ee20742a4b1f,0x0000500e34cdf43b,0x0002b6de4ad6c0ab,0x00002eba08ef6f81,0x0004e0bbfd3a6364,0x00094460899a3582,0x0008335d3e07b17d,0x0000000000000001}}, + {{0x0005a71e395d239e,0x00026a160db974ae,0x0004df55ba5b2671,0x00091fe0f2bfd214,0x00027e4215d39cf2,0x000849498b3dc0a6,0x0000c7dfec311ed8,0x00029fbb50995b22,0x0005f9ca4a3d83cc,0x00085332f62dd6c5,0x0000000000000001}, {0x0003278db69ec48f,0x000c56caebf5d4e7,0x000e4ab979b38b01,0x00005e1e7e210f66,0x000e800e35bf7fe3,0x00041625bdbbd247,0x000140764eabbcaf,0x000ae49d3fb6b3c0,0x000bed09dc31a840,0x00044a6c67185e6e,0x0000000000000000}}, +}, +{/* digit=78 [{1,2,3,..,}]*([2^312]*G) */ + {{0x000976185cedb78e,0x00000d5ac29ab973,0x0005376ef5010492,0x000fcfdeb7cabe96,0x000e82ebeaa6eb29,0x000856863e849702,0x0002b7dd5820c1d9,0x00080b85b8b6adb3,0x000cc2bebcb2a1da,0x0001cb911240a24e,0x0000000000000001}, {0x0009339d66471f42,0x00086865738f288e,0x000ec9ab31ac9389,0x0006dfc0b78b477c,0x000d22531d4c9b75,0x000957b1f72bea7b,0x000f90819668750a,0x000023544082b7ac,0x00010dd35fa97aa9,0x0000657c9376d4d3,0x0000000000000001}}, + {{0x000d393a7e12c9a1,0x00087315af1e7695,0x0005dad48c909f6c,0x00065a7e168b010a,0x00026d86fdc5603c,0x0008f52d5373f51c,0x000a497697c8b7d8,0x0006670982debc64,0x0009f942aaf7a067,0x0000beb15cc57a80,0x0000000000000000}, {0x000728397fd456a4,0x000c8ff5563ef971,0x0004e73a2dd305f3,0x000cc516a80ae4a1,0x000352258160b828,0x00008533e6e74db7,0x000028314ad68011,0x0007541598a03b32,0x000c3d815763ab10,0x000089f632644f56,0x0000000000000001}}, + {{0x00096953520ee092,0x000306d200f67521,0x0008a20dfb41aa95,0x000ecbdc450070d5,0x00044a73c2aa2e56,0x000cf15e09936979,0x00039822bf1cc5f8,0x000ba98dee98b81e,0x00049763f39d2614,0x000dce88bf80d042,0x0000000000000001}, {0x00090be494194f3d,0x00009eeb5f7526ea,0x00042892f629d76e,0x000de6b9665e7661,0x000e4db45bef0dd2,0x000f0c29ede66edd,0x000cdcbd947a3fe0,0x0009dc0bb66739bc,0x000aa185f9760092,0x000d98f355b62fee,0x0000000000000000}}, + {{0x00095d178d8fc8db,0x00088ab909e6143a,0x00089b7600c18603,0x0000f4aa942112c1,0x0008b5a1967f7a08,0x0003543a0e5057c0,0x000cafbf9ac78fd1,0x00049408a20a1598,0x000b58bcdfa7974b,0x00036217ad4e198f,0x0000000000000000}, {0x000138c5bd603cb6,0x00072af896026457,0x000ea9d78b2185f1,0x000482318652917b,0x0008b9b757159621,0x000f2c7ae3b7470a,0x000d10f532d77474,0x000d6b40b8bfc96f,0x0004da23277081db,0x0005192cd44f13a5,0x0000000000000001}}, + {{0x000210b018203491,0x00081f7f56d37ef1,0x0003499eb28c2bde,0x000d16d800ddf3bd,0x0001a8ad11d2638e,0x00081f6ac5a19953,0x00073cbdac06d819,0x00089e5df343701d,0x00080e56132b9d88,0x0008f82b847a3187,0x0000000000000001}, {0x000ae2fb5a3b782f,0x000dfec66b9766a4,0x0003802b7f84c2ba,0x0002adc75633f423,0x0009ba300fd1c729,0x000a85bdc16e7491,0x000bea7ecdc049f9,0x0008cba7fd0e8182,0x000a5a9aa8bce3aa,0x00008a0977d90d48,0x0000000000000000}}, + {{0x000ee51072d71e54,0x000e9e2a68ca9bef,0x000a9707bfc11ef4,0x000ae91635e6d8f0,0x000dd87435cc0e00,0x0002d6a89b217b9e,0x000c5bada1452b8b,0x000d602de29b175c,0x0003f43b9bd73fb3,0x0000724373642133,0x0000000000000000}, {0x0002c3dc04031002,0x000c73ba54808162,0x00084fe7d46fa018,0x000e61eb44ff91a6,0x000448f89778fb3a,0x000d0dc5c96379eb,0x000f26ee0588d9eb,0x000aafd623f750bb,0x000b6e0aafb4a855,0x000ab06c0e1eab53,0x0000000000000001}}, + {{0x0001292993c3f0c9,0x0009243ce8e0477f,0x00002cd6d125a6a7,0x000424b8f8d2f4dd,0x0000e036966f98b0,0x000b28d9609c6ed4,0x0006f54dde908ae8,0x0001261395876628,0x0001a89f477ea392,0x000b89a6253dab22,0x0000000000000001}, {0x0000776cda9e2906,0x0004dd51a83a13fd,0x00024cb69c0833d7,0x000afa9f2d0d7515,0x000d5068104d274c,0x000b1536c256cfc3,0x00005c2c5a3bb4ea,0x000e695b252297e0,0x000674ad5032178b,0x0007afaaac5118a0,0x0000000000000000}}, + {{0x000b5f87444140a9,0x00070937761e89c1,0x000052402d8c768e,0x000e0d8fa7063fcc,0x000065032ca28437,0x0000560b81d70497,0x000a63bfcc5af72e,0x0007abb68cfddac1,0x0007b3b85e89f391,0x000e3880d7454a25,0x0000000000000001}, {0x0001d4cbebc9cca5,0x00014fe965181800,0x00064d65a9766b10,0x000646ec511b3639,0x000cc26e7c4e0d56,0x00094ae11adfae8d,0x0001a6886e8d406a,0x000547bbf4ad6e9a,0x000e8901b3004a68,0x000c8d5981c48013,0x0000000000000001}}, +}, +{/* digit=79 [{1,2,3,..,}]*([2^316]*G) */ + {{0x0003223ffb7f0034,0x00056eeed01f7ec6,0x000a95759d3b10c6,0x0008fe9fc8ceacdd,0x00036b6ab8ec35f6,0x0008c0b21550e979,0x0008e1de4ccc3b92,0x0007fc6be17e92be,0x0001bdb6320828c7,0x0009d213352a78b8,0x0000000000000001}, {0x0009b3c1ecfa9e13,0x000b3a0b5daa423d,0x0006bf95aa594567,0x000fbfb5a3d149d4,0x0003d4e9979588d0,0x0001e08ca45e636b,0x0005c358fa3b11bc,0x0003552e11b17364,0x000b67bc8931ab99,0x0002d3ce614c5f0d,0x0000000000000000}}, + {{0x000ed714844f3544,0x000cc4bb9010843a,0x000996ac0026321b,0x0003453553c74ad5,0x00041741982c7cb6,0x000196d8fedc48a0,0x00020d244c9f0921,0x00070a8fce97fabc,0x000a6d44732828d2,0x00060bcab0c77562,0x0000000000000001}, {0x00061da3943969d2,0x000d674b749d7b3a,0x0009df6dae1fb5b6,0x00005c30ec275083,0x000ae7da1a9284fd,0x000c82a28eb7bd6d,0x000a71d66cd19bae,0x0002d599b2c6a08c,0x000faa1546c312c5,0x0007f06795e3065b,0x0000000000000001}}, + {{0x000fff9188aa7d8c,0x000deb80ad064a0a,0x0007114ab9689af5,0x000fad0b4dbde778,0x00055fd29cd3c099,0x000d379d42525d60,0x00016d04df50e85b,0x00053602e006dfc1,0x000e6c63f374d96b,0x0000f26509a7f32e,0x0000000000000000}, {0x000822c176aa9790,0x000f58fc039cb1ee,0x0005872cad56c2fd,0x000e8ae0ea665324,0x0004daf2e64bf3b9,0x0008f96bb4b8314c,0x00090e063c57f41a,0x000d5149d3063df9,0x000ba61281d5f9b0,0x0004ba3d6cc9c608,0x0000000000000000}}, + {{0x0005d4a76da49005,0x000c3b224e91125c,0x00062ab184a74621,0x000b682a17406495,0x00053c3f7c8cdedf,0x0008e38d4416ae20,0x0009c28df044060d,0x000f86143e5739ed,0x00095f9f7f327b97,0x000872538531478b,0x0000000000000000}, {0x000b98e468155010,0x0001fc05661b3943,0x0009ee23198c1bcc,0x000744fc64ff1647,0x000560f20d871115,0x0002c9feeacdf5ac,0x00070b263cba9c39,0x000badbac8fda72f,0x0001aad35365c71d,0x0002f99d51687d17,0x0000000000000000}}, + {{0x000acd4ec9302acf,0x00046876812a6969,0x00062f921abd47ef,0x00030834c8ee3434,0x00087c08c033bb79,0x000e51d0a2369c3e,0x000c1fbd98cac8fe,0x0006a309b704c675,0x000a173a43fcbb3c,0x00009432c4949569,0x0000000000000000}, {0x0005e781f2ef36de,0x000e3479bd3a702b,0x000d74e86eb68837,0x0003849622881aa5,0x000ba91b89a84ecb,0x000caeee87dd2964,0x0000f40b0230b757,0x000e7853cadc83a1,0x0005f5ad1465657a,0x000e75100e5033e4,0x0000000000000000}}, + {{0x000e44f3911651e4,0x00088f7f22b492d2,0x00072f850dbf1662,0x0003ab2a45a14853,0x000e1480b82ee674,0x000ca609c8235a84,0x00085d8422668b9b,0x000a5d6f0bf02f4e,0x000afb880792321d,0x000cb82c095f0261,0x0000000000000000}, {0x000bc2f5725cea9c,0x000e1f43f99381e6,0x000e6089c844a832,0x00000aa951ad7011,0x000282a695207656,0x00007e689c114477,0x0000e537bb9a4c6a,0x0009df06eaf0cd7b,0x0007c474d7895232,0x000f0710d2b00b77,0x0000000000000001}}, + {{0x00093c8e0aebae92,0x0001a4248ef98180,0x0005353d71384b97,0x000564228ab8dd10,0x000ee9e95615cf3f,0x000dbed91f163427,0x000ff8c7cd8d83e1,0x00002a117b26a05b,0x000d651309094b7c,0x0005fa8d73f3a728,0x0000000000000001}, {0x00045cf4f3fc6c29,0x00069a5dd01b3bfb,0x000e3b24278a983b,0x000ba6e8dda15e64,0x0007ceabdafeb0be,0x00028dd1f4ce3cbe,0x00083c003c3a01ee,0x0004286b68f03154,0x0003661bd44cc13c,0x0006d7a5a2b18a65,0x0000000000000001}}, + {{0x000f4f2d2f859a83,0x000584cf466ff03e,0x00078dc82b847044,0x000110dcb52d320e,0x0007adfe140d78b5,0x000b45fd46e07b11,0x000943939af65810,0x000f26b6d5c5bda1,0x00091095a8309f53,0x0000d4aad23c7d80,0x0000000000000000}, {0x000dd82bca9e7ca1,0x0001551ee78b6090,0x000453fc776839c5,0x0001d0262966b875,0x000729e2a29966bf,0x0008c49cfc825d3c,0x00009961345ab1d4,0x00007e6049f3ad60,0x000007da4f1b3985,0x000c382c8f36cb0b,0x0000000000000000}}, +}, +{/* digit=80 [{1,2,3,..,}]*([2^320]*G) */ + {{0x000a284c690646dd,0x0003eedcd5cc91eb,0x000ea471fd6292fa,0x00023a125841cc32,0x0002e35810a749fb,0x00061336484e18eb,0x000b6f65228a2bfd,0x000207542e7452ca,0x000526cd940c7469,0x000d2936b9b32962,0x0000000000000001}, {0x000573e4f063ef2a,0x000418ca996c2a17,0x00033e1f9c1c3d42,0x000b3cbf970fbd47,0x000088c0b1561246,0x0006e93234f08535,0x0009d7f8ff901881,0x0000aa556f8574d9,0x000a5d989d3b1d29,0x000ed0ab78218edd,0x0000000000000000}}, + {{0x000233d6557077d6,0x000f2b32cea9ff87,0x0006963e65db1454,0x000b05b7d5f2627e,0x0005a68cad15bd15,0x0009679cfef2c921,0x000cc1f982da4ecf,0x000673910763dd21,0x000110fdc3925aff,0x000a3cad0858b1ce,0x0000000000000000}, {0x0000c51bfdb7b566,0x000e8b88f58f7516,0x000713a7cf39e39e,0x0004ac365af813ed,0x00040a788e43e8ac,0x00010b5e01e789c0,0x000ddf33d0cb49ec,0x000a3f2d9bd726fd,0x0006e3c30504e525,0x000e51a456acdf77,0x0000000000000000}}, + {{0x000707d7da6499d4,0x000c004b6d85bc23,0x0004b483dd72372e,0x000b15c9838f63c9,0x000dd40b6584e869,0x0005bb5ad6291644,0x00069be693ec1c90,0x0007b5c6018dc109,0x000c9c113b81150c,0x00040bbd460de804,0x0000000000000001}, {0x000d558a1b81757d,0x00046da356589b5a,0x000f093ea9cf88e0,0x000cd54eede9de0f,0x0001f19ec3f8839a,0x0004ec243ffcbf45,0x000d0d1f8b02c0e4,0x000a42d2cc07981f,0x000363b43f4701bc,0x0007f930f2e9e43f,0x0000000000000001}}, + {{0x0006b0772d9f5845,0x000f28a8c7c3d700,0x000c1d96b231ba3f,0x0000f432c17a4f5f,0x00014ca88f653da5,0x0001ac5da9fce5ef,0x000105dd10257bb3,0x000206b910de18d3,0x000d121d64f95008,0x000e83748b9a298e,0x0000000000000001}, {0x000dc94560ad3e4a,0x000b34cda193affe,0x000f39dff5030add,0x000d72c1a3a58a0d,0x0001ad7c02e84586,0x0006dddddd7190f7,0x000431dd7f7815ab,0x0001b059af2893fd,0x0006d66ebdb90a30,0x000ac5b55b254562,0x0000000000000001}}, + {{0x000a7032755a2cc5,0x00087b60173b4c02,0x000853a0c8b700e1,0x000d3fcbebfa5d41,0x000106636a248a74,0x000d439df17f1529,0x0001a48433bf866c,0x000ed52b92a93d36,0x000de5dbf96508fc,0x00064c08fb48dbcd,0x0000000000000001}, {0x000e6d70757d607d,0x00099aa287bad741,0x000aca83d8bc1d01,0x000a6deb3248272c,0x0006281490886dee,0x00003b3e7ed5830e,0x000b8f50b5515018,0x000ee61ae410329b,0x000dd209b1b1ec67,0x0009f79d8f057d2a,0x0000000000000001}}, + {{0x0009240ac4aeeb4f,0x0009de701cba0d15,0x0001e2030d5be49e,0x000f5b9fa8d80ea6,0x0009e389aa0a7891,0x000f08f46a281d5c,0x0003d8942c2a6a9b,0x000dcae5c6263013,0x0006ed6f226d80fb,0x000e7cd744527397,0x0000000000000000}, {0x00064112eebe5a16,0x0005561ba10f054b,0x000076de3983c715,0x000338a0051c721c,0x00017ed93ec2b1b6,0x00040d08e30b18e6,0x00086d02546a805e,0x000b289546bf39d9,0x00029a40d87fe36c,0x00003828ca6d96cb,0x0000000000000000}}, + {{0x000888aaaf08b61b,0x0004baeb89b8a3b8,0x00013c31ce0504b2,0x000084891577d88f,0x0001501541da01d3,0x000be18906c31edb,0x000cf8acb88a0c0f,0x000de0a54814b123,0x000d30b10ce17eb8,0x0003c65435ad1112,0x0000000000000001}, {0x0003c3081d6e0b2e,0x000bd1198cbd6e7a,0x0009feff60218481,0x0009559a8a4e33b7,0x000ae242155d34dc,0x000458dbdb49b265,0x0003688660033750,0x000853753ede19c3,0x000b6969aac09e0c,0x000a5225b275670e,0x0000000000000001}}, + {{0x0004c030e13db910,0x000b8e4bb182d8fd,0x00024d5733c45ba7,0x000e09929bbae322,0x000c5e18395c5857,0x000beb34317a4b7e,0x00065979d2ffacfb,0x00000dac7ff47099,0x0002ac181634b33c,0x000b325a113dabd2,0x0000000000000000}, {0x000f6d0b44a18451,0x000ebd4b60b5e31a,0x0007c6c236a60067,0x000b1be8ccf47b3d,0x000a21dbd1cc7199,0x0004932466e888eb,0x0001dee034c21f8b,0x000ff9da169619ff,0x0007e95c7e040c95,0x000b5a9dbe56efee,0x0000000000000000}}, +}, +{/* digit=81 [{1,2,3,..,}]*([2^324]*G) */ + {{0x0008f08161486b22,0x000be755ab4efe68,0x00048d9609c012b1,0x00098843db068e78,0x0002d958488ad0fb,0x0003f6d23cff5eda,0x000c41d3ec7372a8,0x00035d185ec0b176,0x0006314a5925e490,0x000a3de4ff957947,0x0000000000000000}, {0x00029cc94c1b43aa,0x000d43d2ad417dd5,0x000360ab4fa2dfe7,0x000e9eb552cc454d,0x00035a4732c24fb2,0x0008d1e843cf82a2,0x000ab8e67bb7a406,0x000191877eafbca2,0x000574ab099566cd,0x0009f922f9872f62,0x0000000000000000}}, + {{0x00085f8208bbe4fe,0x000ecc287db7bb2d,0x000e568667e702fb,0x000cbeb7a157f36d,0x000bf484f3352f4e,0x00058da014941bbf,0x000586c3d5fe38d5,0x000a8a8ef1b37b22,0x0005949627a9e7fe,0x0004740c422ebeba,0x0000000000000000}, {0x000c2ec0fe63724c,0x000be0eb88269034,0x00016f607ed8c7ff,0x000b235b0a729f09,0x000b2fb783d219ca,0x000a1f91a0e85a3b,0x000f36eaf1659ef7,0x00023c3d4be9067e,0x0005e5bd92b43e99,0x0001e3e81391aa3b,0x0000000000000000}}, + {{0x000cdab06c0f497a,0x000ba682d62fd58a,0x000624d8816a960e,0x0006c48669b75099,0x0001fb169d6d2670,0x0009d5b924784941,0x000361b9811c3888,0x00005ccc21278392,0x000173173a0f5de5,0x000862da0030f596,0x0000000000000000}, {0x000580503b5bd1a6,0x000460a53f77d734,0x0009ac0fc516703a,0x000cc1b1f0c1292d,0x000599d98d3204b9,0x000365fabf012bdb,0x0001f82c240390bf,0x0007f3c05cb807a9,0x0001aacb4486b03f,0x0002d4bf3cd7e64a,0x0000000000000000}}, + {{0x000cf4fb66902a59,0x0006522d970cf058,0x0008db985646108b,0x0005bd091c524ec7,0x0001c8ded01bac37,0x000a7571d5eaf41a,0x0003ae41513bf75c,0x0000831a58ce8343,0x000b5c1d0b1cbad6,0x000a127e3558b4d4,0x0000000000000001}, {0x000c5bcaa2423577,0x000e95cd90416c5f,0x000f9cd3e851fd11,0x00043cec77429d71,0x00033818263e74b1,0x000b0bed2ab694e3,0x0009d3b078fef207,0x000322c62d90900d,0x00013057fb2dcc39,0x000b0aee2cd8c7f7,0x0000000000000001}}, + {{0x000ca837cf1af373,0x0003ae0bc7638546,0x0005fbd88b9c057e,0x0000f14623993437,0x000d8418b02866d1,0x000938602b2a7398,0x000d172bab8e8fb6,0x00001960cc17eb57,0x0000f437b4d86f8d,0x0000c3266073d8a3,0x0000000000000001}, {0x000b4a64de3a3170,0x000d07d300586858,0x000590b48d2c8237,0x00064608c48f8521,0x00055a4fad372745,0x000ad1664c3cb9d0,0x0000e5148678bfbc,0x000ce1f67995bdb9,0x0000ce6b82ba0137,0x000865e65d5e9060,0x0000000000000001}}, + {{0x000a3697b62e274a,0x0004f036666b6cc6,0x0004615de0a77111,0x000d1a544656bc00,0x0004cfe1b0ffd27e,0x0007b011fe7a367a,0x000028f9395287ba,0x0000e474d2cd1539,0x000df81c967ab663,0x000809d416f7d850,0x0000000000000000}, {0x0004ff0171522411,0x000cb614e9eb99cd,0x000efe5013168ce9,0x00000068878690dc,0x0003ea58b25b4f05,0x000697bfbefe708a,0x00061484cbc18871,0x000fd61b572d109e,0x000507a67ea6e538,0x000c108cf0642bcd,0x0000000000000000}}, + {{0x00073712d28f1037,0x000dcbb85bee07b2,0x0002d427b383b0b0,0x000886a4c921569c,0x000a22bb54a08b63,0x00016efd18019f62,0x000be30e896a901a,0x000b1c79b63790d0,0x0007550d4ad3f339,0x000c014ebb2a7324,0x0000000000000000}, {0x0000eb80b7360646,0x0004292a8064819e,0x0001aa8d66966eba,0x000cb5433c6786ce,0x000c176010116f6a,0x00042b7f40b26c85,0x000bb51bb431c00e,0x00079fd699bb4a71,0x00083711f2c2acd8,0x0008a64b1e3905a7,0x0000000000000000}}, + {{0x0004f281744c4f31,0x000fb974f0427525,0x00022ee390de534c,0x000caed9f368e25e,0x000470d3f5a1ebc4,0x0006bb7427d70104,0x000830891f024042,0x0006993cd5f37f0d,0x000a9d89f4ebe578,0x000c7b2549a02c1b,0x0000000000000000}, {0x0005527d2ef3c160,0x000aa733afd2d857,0x000200435a36240e,0x000dbed50df72a8e,0x00064e9f3dcc7104,0x00084041401f5c34,0x000b281791b398c9,0x000ad77cd49e1581,0x00029530ca203aa2,0x00024a7004073823,0x0000000000000000}}, +}, +{/* digit=82 [{1,2,3,..,}]*([2^328]*G) */ + {{0x0004b896de83f7b2,0x000dd1b2e000727a,0x000d43d965547a68,0x000c1f1e1ce79a50,0x000e5ec87252c6ab,0x00060d6a5ae7160f,0x000985fdcf45caab,0x00091b3180d9d235,0x000646ab8f898256,0x0009f0446d6798dc,0x0000000000000000}, {0x000ec719b8387586,0x0001de5f9db6636e,0x00038f4a187cc943,0x000ac366ef383b03,0x000022fa366743db,0x000760fac1cc0b0d,0x000265067948b1b5,0x000693934495e8d4,0x000c64b8f4f88921,0x0005281cd8ec2ed3,0x0000000000000001}}, + {{0x0008479d81fdc38a,0x00096a893e0d1c8a,0x000972cecee045ea,0x0000ed0b37445b26,0x000bf1c0a16a9b25,0x000509c76808e477,0x00070c4c826b6837,0x0008008a3f9fb748,0x00088d0f74d58040,0x00063d18d474fc4b,0x0000000000000000}, {0x000a143fb2178ecd,0x000ee2726dcabd2f,0x000e017d3cb36d39,0x000d77e2d8d9e011,0x00069332c865043b,0x000231a13fc89650,0x000e078f7a775c43,0x0002e93c91cf1e3d,0x000d48604e0f7c89,0x00031d2709749250,0x0000000000000001}}, + {{0x0006fca2f24da625,0x000adeacc535625c,0x00020bfcd99308f2,0x000732872a2697fb,0x0007578b9abdd39d,0x000d7e3c3e30d29a,0x000dc63c314955de,0x0002de0cf2a14e6d,0x0002e4f4d6b0bbff,0x0008423dc75d4cf8,0x0000000000000001}, {0x000dc6563086abd4,0x0007f2e300951d8c,0x000a989054197751,0x000a88edbb68daee,0x000ee17db6bcc0fa,0x000d996d71fcc36f,0x00038a1dafea4a84,0x000fc8d72ef68c32,0x000c6b07cf974baf,0x000b2a140ef1e1e5,0x0000000000000001}}, + {{0x0006034713a3e42b,0x00008cfb50e4479e,0x000f617634a25363,0x000ba17f9549272f,0x000163e264556302,0x000db056ef0f6ed9,0x0009449c67d2e92d,0x00027bf608bef04b,0x000a41494e0fd2d6,0x000ecafaa0f9ad5b,0x0000000000000001}, {0x0000394a908a0374,0x0002739ed3e6c1ac,0x0002bf950e49017e,0x0006ca69441c9b0d,0x0003c717b7978985,0x0006f1bf123315b5,0x000d5ffc5bce1316,0x0000f7b0dacbdc85,0x0005306a73236570,0x0000fc22ce0f19f8,0x0000000000000000}}, + {{0x000a6e6af09b8eae,0x000343b43c513568,0x0003425270500040,0x000fb4508580e8a3,0x000b307fa3d1f99e,0x000113638a6f65cd,0x000be36db7f5ca63,0x000ede447eb7ec8b,0x00022e6ed48cec2c,0x0007bb1afd7c927f,0x0000000000000000}, {0x000c11f9a334fdc7,0x00026ae8011e9b06,0x000ba4fc7ee152d9,0x0005ebe2d92b45f4,0x000643466d3d6908,0x000a09e637a743ca,0x000aba719664e4ce,0x0003162cc6d5e41c,0x00060cdf202e7dd3,0x000c307d09d0c5c7,0x0000000000000001}}, + {{0x000b621fceac0c99,0x000cd9f0c6398393,0x000246af19db94f8,0x000def03f4511189,0x0007e278282d9eea,0x0005d51872d67451,0x00069251c97cb275,0x000152306520f8d9,0x000f0746e9696ce9,0x0009ec26638c1b12,0x0000000000000000}, {0x0006fbb28d6db318,0x000282f1e3fd8169,0x0001638aa745ba97,0x000741cbd6f6afe6,0x000674be06f9a37c,0x000f0e4d5ef37aa1,0x000c66c847151102,0x00086ff305cd0c0a,0x000dec0bec87dfaa,0x0006e29a33525521,0x0000000000000000}}, + {{0x0009e94294764d17,0x0002085b396e5ed1,0x000a114f1c4a72c2,0x0007528c8545f538,0x000963b0d0aaa085,0x0000ee6c82fe930c,0x0009cc321991f876,0x000bfd79b22e04e6,0x000e404e5103af68,0x0009a8126bf818d5,0x0000000000000000}, {0x0006f18c56b9245c,0x000be8cf2d749759,0x000d0534792607af,0x000737082c9ac67b,0x000b81d246c242f3,0x0004a1675a873285,0x000b8be65c0f0c83,0x00031f4367aaf8a7,0x000eeb7fdd2c4760,0x000b71b7bc940b35,0x0000000000000000}}, + {{0x000e474b29bda866,0x000e2b3a2640ed4a,0x00021f91da93684d,0x0005df67424bab62,0x000b550d60209995,0x00096c99d55193b4,0x0003c748e9f27481,0x000631e6b3fe7e7f,0x000f47f4e257248f,0x000f8f56db9ba373,0x0000000000000000}, {0x000bc357ebc4dd12,0x0003a33556892c48,0x000b72124cffa435,0x000e35056ea40bee,0x000a1b37aa3c71db,0x000ddc96c72e951c,0x000291c71de6fcaa,0x000ff88244eb58ae,0x00089a7bf96fd42a,0x000eb41c5d8ae97e,0x0000000000000001}}, +}, +{/* digit=83 [{1,2,3,..,}]*([2^332]*G) */ + {{0x0008ac7511fc1235,0x0005af51e9a589c7,0x000c09d63e56a5f8,0x00055b518ce24a89,0x000a75441652a9b6,0x0004ffab489b445d,0x00071289523b0e9f,0x000290aaf7cb23e7,0x000c9bc7899234af,0x0001dda65dc198b5,0x0000000000000000}, {0x000eeaccabd5a6f7,0x000d5900e72c44fa,0x00047a63782a2bbb,0x000393bbf531aecb,0x00009c7dda45067c,0x000719fa6f31630b,0x000f3a0c95e46b2e,0x0004deaf5d70f849,0x0007dd5d46829965,0x00096c286bc1f082,0x0000000000000001}}, + {{0x00012d5838ca6af1,0x000964e9241a8a04,0x00024077ab2ac6aa,0x00089da3e1536c1d,0x00001df56af4aad1,0x0006ef9e57fbfb6e,0x000b17f8ca8e6244,0x0009cbcb351a7c9e,0x00085fb54f3eda4f,0x000c296970873909,0x0000000000000000}, {0x00056d8d32e5fdd1,0x0006814d7980be46,0x00065dbc6a68d7ed,0x000cebebe9b6528d,0x000269dfcc27d433,0x00073aec8225c88d,0x000d90643f7caaf2,0x00041c78327e8662,0x000763fdae4a09eb,0x000b0ead9bd2f604,0x0000000000000000}}, + {{0x0008f42a8c29bd6f,0x0001879cf21db610,0x000555cfe2bddcfd,0x0004d452af269c11,0x000fcc4011856e7b,0x000e6e45593cb7c3,0x000b215415957c0f,0x000d3b983b2c8996,0x00012f953be5cf31,0x000f7078abc3a003,0x0000000000000001}, {0x0002c0d6b39b80ab,0x0007b2847e724ed5,0x000cfdfc83942bcb,0x000c1c0a2ddac314,0x0001da690e67e2aa,0x000310507c60736a,0x0008b8515f2f407e,0x000203447dd4a30b,0x000208fe5f3ddc7c,0x000470482e113587,0x0000000000000001}}, + {{0x000d7682eb5538d3,0x00058bd8734ac8b7,0x00075fcdec6d3223,0x000ffc3556d9d86e,0x0003a6c363b61e72,0x000d03c2ead1066f,0x00074fd7095dbd73,0x0002b42f9972cd16,0x00070acc2acc3e68,0x0006fd80c7114993,0x0000000000000001}, {0x00049c09589c235f,0x000294e10709485a,0x000e55bcaedd0d2a,0x00084da3a073e38c,0x0004725346561d28,0x000ed8195e95d347,0x00001ec990b2c19b,0x000817664aecefe7,0x0005f12464c59ce8,0x000f85a23cc1c6cb,0x0000000000000000}}, + {{0x000fb48cd5b9dd85,0x000ea5640b16ec8e,0x0004991733d00d1c,0x000e2230142b823b,0x000d1646ff2bfccb,0x000e87a4efbbe898,0x0007933417bfbbcd,0x0002ac0c744278e4,0x00079c8483d8b4b3,0x000fafae62ecea8b,0x0000000000000001}, {0x00063d47c065003b,0x0004185cb3cef55d,0x000bfcfe8d3872f3,0x0009112c490c5e61,0x0003b1e7d4274313,0x000517366c9c6308,0x000bc9bb53389bc7,0x00025afa81750fe0,0x000afda6e1b71dc2,0x000c6a9aba51c85b,0x0000000000000000}}, + {{0x00000e070af57b1c,0x000441246548fb7a,0x000c563ead66464d,0x0007508f0851b47e,0x000f31999ecfeae6,0x000f85797a3840b6,0x000c74080d60c378,0x000f3eaf1e242bf3,0x00003ca35d886cbe,0x00022754c9a357ba,0x0000000000000000}, {0x000982d0aaae9f75,0x000176d1b4b6ca3d,0x0007980f7d421826,0x000979133a61fcc4,0x000cb63f41cac3bc,0x00079bb4b9516419,0x00033cd18a279569,0x00075e18895b57d5,0x00063599127b0d23,0x00063dbbded01670,0x0000000000000000}}, + {{0x000342ad817f361c,0x000cb1bd9eb81b06,0x000c7b97a0843c43,0x00083f804f029e13,0x000b3486644af0ff,0x0002a64dc632346d,0x0002a39cc0dcbed6,0x00056457b2ff9c5a,0x0000fe8536001cce,0x000def7d14af048a,0x0000000000000000}, {0x0006c4115861eef1,0x0006a14bad6b1ac1,0x0005dee3e9d201f9,0x0002b396cf147ed1,0x000f4643f8a21b3b,0x0004c9915584149b,0x000a893cfd7ba449,0x00027a8a4eb7aef9,0x000cc4fa992b1d31,0x000eec6c99a8fa7f,0x0000000000000001}}, + {{0x000a8d985f7ab9a0,0x000b7d39a70c2f72,0x000f6f8acabea7d5,0x00095d0273f642dc,0x0007d8a54bc56deb,0x000a63f1883f3cc3,0x0004b831ef1bba05,0x0008e112f14c8c07,0x00089737917f937a,0x0004874d335ca229,0x0000000000000000}, {0x0008374f770af11f,0x000cad8ee96d7e99,0x000bff9e11a7d432,0x00056384e6665366,0x0004a9b692423f6d,0x00075f044efc7e34,0x000ee60ee1b3dddf,0x00039cd00df7a827,0x000529eb5b2612c9,0x000cd7c4ffa6a13d,0x0000000000000000}}, +}, +{/* digit=84 [{1,2,3,..,}]*([2^336]*G) */ + {{0x000c16671a6774f1,0x000af03753ce5839,0x00054c5f8c07356f,0x0001afc71165a356,0x000f9d6adf86cf5c,0x000a6b4966903b89,0x0009f4ebff86c3fb,0x0004a87b0151b151,0x000efd27bbe4f95b,0x000b040513d26385,0x0000000000000000}, {0x000622a63fa5d90d,0x0008d92aca99c7d4,0x0001d6acf3aa6efd,0x0001b7387e55d6dd,0x00010db119c2295a,0x00011a67dad9703d,0x000eedb427c0f52d,0x0001e055192fe412,0x0004a5758174c7a3,0x00055cfd4b1dde40,0x0000000000000000}}, + {{0x0000d119f7fe9853,0x000fe49990254b37,0x0008a86cb40764d3,0x000820af39be0e2c,0x00027458321b04b9,0x000c2ba583294752,0x000ae89a07a5c7f2,0x0008fa6d520652e9,0x0005604c9bdc0eee,0x0000c06f2e484243,0x0000000000000000}, {0x00014a30a2bfa81c,0x000cbd6640003017,0x000ab87a938a3f37,0x000e1c91f3132874,0x000e57e9d7ac6ecb,0x000e33fb881734fa,0x000073b600765b07,0x0009428cbfb5edfc,0x00029028585e9a20,0x0001e077ef7692bd,0x0000000000000000}}, + {{0x000cf34a0d6f6ffe,0x000370c22b280291,0x000e87a26b1fd975,0x0008088a662b3666,0x000eea746601046d,0x0008edbdfb0988f2,0x000f2131f7fc1ebd,0x0009266b6d41f4b2,0x0003c1c020089694,0x000be27c849de8c8,0x0000000000000000}, {0x000bcda37f3a594e,0x000726480ec74a90,0x00026216e2ddde9d,0x00064b02bef16495,0x000aafba3c749a5c,0x000a872930c3f630,0x000654a8695df3be,0x000cb5372491b21d,0x00017f3b3a2f3f6f,0x00094613fe01cfe9,0x0000000000000000}}, + {{0x0008937b47b39090,0x000415bb7112fdec,0x00066e9e19d3ed5a,0x0006597801eab0fa,0x0005e740c409bbbe,0x000050b19bba9267,0x000df2c3e8b56daa,0x00012fbcf099e6ee,0x000195262a55e069,0x000ee2f2c7d1e980,0x0000000000000001}, {0x000c1384c013e53c,0x00074951ffea5bee,0x000d0ad477deaca6,0x0004ee3245756473,0x00030808642161fd,0x00050c8b97a30694,0x000340f405f653b8,0x000d5a543cae9de4,0x00031ca24347d550,0x00092a75f4312ea3,0x0000000000000001}}, + {{0x000c8162c746ef6f,0x000f6715dbf9ea58,0x0005523d8217dd87,0x0000b2c52bc5b0b4,0x0005db8903ecd878,0x0004296f75f92e78,0x0003c6e6397e4045,0x000e84bad1cccce3,0x000b82162d3c5f54,0x00009333f935ae95,0x0000000000000001}, {0x0004ff1c33e26a2e,0x000785b4ec10e1f5,0x000a1634274c2886,0x000b5d5ee5822d49,0x0002fbe9122e0bfa,0x00003c2cc2955a06,0x000e08e579ad9b7f,0x0001dd6ee255f2e5,0x0004f08f71b65e70,0x000c9dcd7d23cb93,0x0000000000000000}}, + {{0x000927819d85c389,0x00088dd986cf5001,0x000eb6ebdddab174,0x000a1e388628281f,0x000014511bc0392f,0x000b79c2a5e1691b,0x000d866b842f8440,0x0000343c71a48805,0x0002ac5c5d5d795d,0x0004aaedd8558804,0x0000000000000000}, {0x000896067875f110,0x00080b0d43dab8fa,0x0009a3104ecd6f15,0x00017c31840b3b59,0x0007841201091767,0x0008de871b2243eb,0x0003f7be2323a388,0x000f764799c353ac,0x0009d244edaf476f,0x000513c595b87c99,0x0000000000000001}}, + {{0x000800832e6fdb7c,0x000df2ceb7712003,0x000625cbf7ec3398,0x0007b9eb4c74b442,0x00090424f2515df6,0x000abc10516b9778,0x0005df82462b4902,0x000a6d60d9807c8f,0x000606aee1bb838c,0x00030e1e7a2ff1ff,0x0000000000000000}, {0x0008fce9b8ce853f,0x000b7049a4c20923,0x000ae2b39f773f7b,0x0002f55bcfbf4b1e,0x000b07309ae9653e,0x000cf869d6026775,0x00099fe1b0e83daf,0x000202f8a21d72ed,0x00037619de81bf7a,0x000b7b2bf238b7d7,0x0000000000000000}}, + {{0x0008ca9dd54298a1,0x0000e1308270c47f,0x0002be3ff9743378,0x000471b0b186cdf8,0x000b1747ce696eb6,0x00015fe31005f60a,0x000ac1b5a457da88,0x000f4901af0f6bb9,0x0002f972c925bd14,0x000186acd8b58a65,0x0000000000000001}, {0x0000f355372184c6,0x0005e1f7ba0c693e,0x0005b11db3d6dbda,0x00051b89e46fae1c,0x000c97c0b46b0f1f,0x000cc037caa48d5e,0x0000355bdcc75991,0x000f28784dc0e5f4,0x000837eadd1a3fa4,0x0009eeb5a1926d0b,0x0000000000000000}}, +}, +{/* digit=85 [{1,2,3,..,}]*([2^340]*G) */ + {{0x000847fc397e26dd,0x000de02cc0ff17de,0x00063e6fe388ee8b,0x0001e73a774123d0,0x000daf8cb9f5597b,0x000b938535ee20c8,0x0000d7da8b1bfac8,0x0008e2a819363df1,0x00079861ae7d4273,0x00003eaf0a677999,0x0000000000000000}, {0x000611141de00c8a,0x0001d2aefc5b58bb,0x00033633ce29b3d9,0x0004bfef0d5e3306,0x000d78956c10a254,0x000fa84101beaa2d,0x000f9588ba22402c,0x000e0df8f46296a5,0x000c7018734ace12,0x000ca8e0e00d25c6,0x0000000000000000}}, + {{0x000c26d9a28cda66,0x000ac34788e2808c,0x000a2e3c895ddb52,0x00092cc3305bc55d,0x00086ee8076376e7,0x000da2d9cb5f9d99,0x000f0d8aea185a30,0x00068ef462b956f8,0x0000f61c2096bebd,0x00044e7be11b5930,0x0000000000000000}, {0x000ea58bb15fdd13,0x000d5b2585fb779d,0x000d75c3d978271c,0x000e827a5ac1a4b5,0x0006c6fe4d4804ee,0x000f66c09f0147df,0x000b3203f6a4c217,0x00081eeb950292c3,0x000da3f441776841,0x00071712f688beeb,0x0000000000000000}}, + {{0x0004586de8027c8d,0x0003b7dc25b073ea,0x00049b36fccf2477,0x000032458466e794,0x0006f36f854043cc,0x00051e24c902d71d,0x000ec681a81ea4c2,0x00060e710d119e39,0x000dfa8e50e27e69,0x000bc96885ae0f44,0x0000000000000001}, {0x00072cc7ee3e54e0,0x000e54b8224b0f78,0x000e5d4bd3db5696,0x000d27cc64ead37d,0x0009b5b2f36d2cba,0x00021210e2a45e52,0x0002c8d788fbf745,0x000440c5440be1bd,0x000157b392b99018,0x00045deeecc510bc,0x0000000000000001}}, + {{0x0001f34441941c1e,0x000786af242ed224,0x00082ffb7bd73f71,0x00040040ee8f684b,0x0000b0e0a77660b9,0x00055090773cb918,0x0003b4341ff934d0,0x000efe154397d7e6,0x000566086597b1f2,0x00038e115dad8d73,0x0000000000000001}, {0x000320279e97eed7,0x00092b44ad3c59a0,0x0009546c02d95d0b,0x000317d617644016,0x000b3b1278de80cf,0x000150eec20cc035,0x0004047a454911ca,0x000eb15350f140f3,0x000297dd664a854c,0x000c1545fd389a24,0x0000000000000000}}, + {{0x00054c6d42f4ddeb,0x0001a5d46442b31b,0x0008eb97dd3c9497,0x000f35259e3e1ff6,0x00049058db0b2e2d,0x0006968077e0b694,0x000456e4ea6ec9f4,0x00098457796aba76,0x0005412cc7718336,0x000c5eb4e4306f25,0x0000000000000000}, {0x000f4a91bf2060dc,0x000e9a57dadc33ce,0x00051f56adeb934d,0x000b29d22e8fe341,0x0000f85723b5e49a,0x000ee66b41fabf52,0x0005253bffe67611,0x000c50202f550a60,0x0006d250b9e49468,0x0001a2956ea13fef,0x0000000000000000}}, + {{0x0000dc097ea2b524,0x000c5eb5323240e3,0x000082d33c53dd49,0x000c5d6917c692b6,0x000337e9d695a12b,0x00078372d602c7fd,0x000ef2985e92117b,0x0008ceacebcbefa0,0x00068cc3e9b4e8e1,0x000db3e3e50ee13d,0x0000000000000000}, {0x000980bec0d2f5e3,0x000eaa9b062f585f,0x00083ee3b5103eda,0x000605534f1f8028,0x000add292d29ee4a,0x000f9ba28df95f8d,0x000d134fb5785f57,0x000f0fe162fe54b8,0x00047f0902bd8287,0x000de3769ce1a122,0x0000000000000000}}, + {{0x0006e3d538c32c91,0x000eb774341e7141,0x0009e6b225ba2b4e,0x000824dcff742236,0x0004d5e3b9c67d3c,0x0006e4276dd54722,0x0001d2dcc105d46c,0x000392da4b3a8a00,0x000e2f3953b25248,0x0002fecbb5174b67,0x0000000000000000}, {0x00008d720508826b,0x000b9b4e2e807123,0x0001bf6b1169562d,0x0006d2acb14e6841,0x0008cbfd3257245f,0x000189ac1c8cdf45,0x000ee493f894fd3e,0x000fa59cbf0ab5d6,0x000476d8e672a0c8,0x000ed9fbb78753d8,0x0000000000000000}}, + {{0x00039334c1cd9788,0x0008ab0560e3e74e,0x000fa2ff6e2c62b5,0x000acd7d25b0cfee,0x000c456f469aee3d,0x000a862fad6476d0,0x000688f0d8d2340a,0x000b648a9494468a,0x000f4ed209d4d2fe,0x0008a93e7c5890e9,0x0000000000000001}, {0x000cdd07e3f60721,0x0002466078437612,0x000ea1835868d6d1,0x0005ba6a85400753,0x000f7f252808d5c3,0x0007b45d6857ba4d,0x000d683048ddde70,0x000c759393e38c60,0x0000c630e919b183,0x0007209017172576,0x0000000000000000}}, +}, +{/* digit=86 [{1,2,3,..,}]*([2^344]*G) */ + {{0x00060ead50b81885,0x000bcef73cb6f0c1,0x00053fb7eb3e2cd2,0x00062b319bb7bc05,0x000a38a471706b6b,0x00046c6b42daf298,0x00005e59d404cf98,0x00048fac2e73085e,0x000ff0af6c53893b,0x000de9e3d8eea7a8,0x0000000000000001}, {0x00034470acd5b055,0x0000c4c1af94ede9,0x0004fba6b3889b3f,0x00023ee49af80496,0x000f0d89fd53a3e4,0x00053cc302793fad,0x000b36dcd463b613,0x0003782e102e51fb,0x000c63732d6d1c6c,0x00059dc97f604bbb,0x0000000000000001}}, + {{0x000db31484a7e177,0x0001740a1bc3a05f,0x000c265e95ebda07,0x000546d643b1d3c4,0x000c2611b9709edc,0x00015784fc807b04,0x000e5bd473ceec4e,0x0003c97fb33e58af,0x000d6d5327b94dc5,0x00062a914fc6dc39,0x0000000000000000}, {0x000cbcf73d880ddb,0x000029df80627a67,0x00091ccf95a67d3e,0x000ec7aefd91b52d,0x0003aa855273ca53,0x0007213a95113157,0x000b98a49db550c0,0x000b470643affa5f,0x000f0628a0fa67f5,0x000d2cf906186e6b,0x0000000000000000}}, + {{0x00039cff07a9f5bc,0x000b27814ba6cdbc,0x0008da0e67469efd,0x0004ccf0a198fcd4,0x000cbf71a6e5b71a,0x000500eabe51b9c8,0x0007908463c5cf80,0x0004962539aa0260,0x0001de61db956e33,0x0004f9e2c1ac338f,0x0000000000000000}, {0x0005f37dc080da53,0x000c1c2a74369386,0x000025722593f4e4,0x000a4e09b626a8d5,0x0004b1e7f8c96db6,0x0002c5a75c187079,0x000cb91e644a3045,0x00024e7eb18af641,0x000fb48086d9cefa,0x000b4dd6532fa3f7,0x0000000000000001}}, + {{0x0001be067e3e2f37,0x0007737f152d2c7f,0x000242f8dec8659f,0x000c7d958df47d63,0x0001b91c63b0acba,0x000c2c6ad3f62088,0x00099adda54c0028,0x000a3012f6937019,0x00014f4c499516f6,0x000da068d44cb73c,0x0000000000000001}, {0x000209ec58d1b414,0x0008e876dcc7401d,0x0009323106751dc5,0x000f75f24e14fe98,0x000ac88f5086e5a4,0x000294dbdb4ccd6a,0x000be99edf86543f,0x0003767f48ab30e3,0x0007667c622dcd0b,0x000d7fd6615681d8,0x0000000000000001}}, + {{0x0008c453ac10be58,0x0006d75a48dfcf2c,0x000c4944bc5042f3,0x000e7ad5c9cee1eb,0x000996c45d109bc9,0x000689f02d424fbf,0x00023528e926c326,0x000e84ac793d58e5,0x0008eeaddb5a4ed0,0x0005d31f9ea2560c,0x0000000000000000}, {0x0007ba171a6ebc35,0x000d39e242ba28da,0x0000694f0cc97464,0x000d7fb496b0d2fb,0x000d7e1b5b66b3d8,0x00001db81788dc1a,0x0003854dd5fe4e9e,0x000f9965063e6021,0x0003751c74ab4631,0x00006945420e7941,0x0000000000000001}}, + {{0x0005b110bb6dcc9c,0x000e9d1e6c13e60d,0x00058585159a0842,0x000b46fe443356f4,0x000853b25c086265,0x000ebbff20877291,0x000136ec71c4d1d0,0x00003ebeca5e79c2,0x000a0af3b3a96ed2,0x000f93012e330a32,0x0000000000000001}, {0x00017ed67dacd4b6,0x000ceade583a567d,0x000316840b4e5703,0x000c414303d396ce,0x000ed6970ea480cf,0x00063761b9b1974c,0x000788f3de4383da,0x000ad07d6726e400,0x0003056bda545993,0x000385d3fe822ea3,0x0000000000000000}}, + {{0x000285c58225166f,0x00072d2451ec99f6,0x0005efbddf5101b0,0x000dbc066e055890,0x0002c29985ac7cd6,0x000f1839b6caca94,0x00093cafd8d9c1ce,0x000b177ba7911d6a,0x00098fc762e30d5d,0x00067686b89a78b5,0x0000000000000001}, {0x000dff4557a32b93,0x00037e16c6af191e,0x0007ad7362550f1a,0x000983cb772b5537,0x000b0746f50f2068,0x000dbb42f7ee6b8c,0x000f0cdda882070d,0x000a732384a13e83,0x00055b3be67dc4f2,0x0003cf20d84f4cbc,0x0000000000000001}}, + {{0x00007348004e40c0,0x000448f78fb0602f,0x000c2ac8aebb2604,0x00064b78277ca03f,0x000eb2c6f473d278,0x000cb793a9eb1664,0x000e2b358eee9a37,0x000194f18cbc9c2c,0x000f6078bc87a3dc,0x0006220e93cd112d,0x0000000000000001}, {0x000d1ed5d96d6d2f,0x000b72be10752f3f,0x000d1e476660c38f,0x000b1d6d9b093c35,0x00026d898dff773a,0x0004b445df00e4cf,0x000d1ce422c1136c,0x000db6e821b59ee0,0x000de6252e82511e,0x0002f481c804e41a,0x0000000000000000}}, +}, +{/* digit=87 [{1,2,3,..,}]*([2^348]*G) */ + {{0x000d7989af2b44ed,0x0006f91b1a9086f8,0x0002b4d5672b30f1,0x00072009919c3dae,0x000003b6ec0a964f,0x00012b7f4f64ce56,0x00076bfe0f0d4fbc,0x00043eb40f821444,0x000cbb4480332a8a,0x000f3ff375566080,0x0000000000000001}, {0x00018caf35e5d712,0x0009aba53d6591e5,0x0004f1b1b50e170d,0x000f9eca3e56ed3e,0x000288dfe4cc67c4,0x00059c7726fa0dd0,0x000a0660a01f234a,0x000a704007db6c8f,0x00070b32e8366767,0x00096994810fb845,0x0000000000000000}}, + {{0x00092a1897b4c0b5,0x0009027fe35612df,0x000a5105c7f9f97d,0x00021853ba021326,0x0005dd2cadb219c5,0x0003ab9d259b3ed4,0x000857fddadc1ebb,0x0004addab0607b4a,0x000ff916fbbc92a3,0x000721ee7e6c527d,0x0000000000000001}, {0x000ba1dabc6f5958,0x0000087b0e564aa3,0x000b9c963dd27f6c,0x00028eca3c030970,0x000c23cd7a457768,0x00017b833d0834d3,0x000be25f44c50d54,0x000a153d4a6bf0d8,0x000e9c71da49590e,0x000d43f3a30dc247,0x0000000000000001}}, + {{0x0009bd585a57c01e,0x000ee844d7078c7b,0x0006c16cb258be3e,0x000a3282f9caf55b,0x000e08d004fc2ebd,0x0009db3ca2d054d0,0x000ff89010ddde2b,0x0006615b9b156d6c,0x00000ee87cbb5e96,0x0000965826673f04,0x0000000000000000}, {0x0005c973c58e4e2f,0x000bfed17990af45,0x000982806a235d03,0x0003a7e6203578b6,0x0002679d20d4837e,0x000fa09a67212915,0x0001e993e548aecf,0x000034e7752d33de,0x0008c0133b8d99be,0x0005d644443ed88b,0x0000000000000001}}, + {{0x0005d09795891669,0x00055ce0c6b8f799,0x000cdfa7a67baad2,0x0009a5462a84102f,0x000ca5fff322b2ed,0x00016895f0238b4c,0x000e1fc27a1fc8f3,0x0007399300db4369,0x00016f718708ed71,0x000aa4931503fe5f,0x0000000000000001}, {0x00040da9e1ff0c6b,0x00022af9967269a0,0x0005871908b86944,0x000801c88350fa73,0x000f680cd1b5d61c,0x000f0b415826cc63,0x0008b363474f5f7a,0x00027800e5401e93,0x0002305262f20f7f,0x0005e6d27bb44c56,0x0000000000000000}}, + {{0x000dab714dbca0be,0x000450d634bc786d,0x000d2802c42f3afb,0x000ef6e542d9994d,0x000946669336b0ea,0x0006a8fe65059b68,0x000f702cce1812e5,0x00030e3c70e359c5,0x0009fd5b9f2069d5,0x000093f3f0186d75,0x0000000000000001}, {0x0006d10e2b40858b,0x000a417e22a4fbc2,0x00013ba0de831401,0x000d41a31a86b763,0x000893ad0b78b7c2,0x000aacf20f1564cd,0x000bac0041297947,0x0004ac69dc2da24d,0x000ba071987933c2,0x0007176068dfbd75,0x0000000000000000}}, + {{0x000c8b4eb4601064,0x00099a991f16e053,0x0005bcf58e425bfa,0x000b1e6fefc21bff,0x0005da04d7e97cf6,0x0003f93bddaedacc,0x000336130ff2741f,0x00028f428d033ae2,0x000b98d58f0cc054,0x000281be6796c6ea,0x0000000000000001}, {0x000562d3cd261bc6,0x000c4e652831be2e,0x0007f84fc06f2ac2,0x000b078d9ca13774,0x000d2b248e882f5c,0x0006981dfa0231e8,0x0005f7dcfabfd673,0x000f3658f51759de,0x000d6a68de41452d,0x00038358c049f993,0x0000000000000000}}, + {{0x000dba36a11f468b,0x000ddefecb4596c7,0x00004044ba328343,0x000e3d89658e943c,0x000955f5e1aeb372,0x00008650d658c0f4,0x0004309aed6ff5c1,0x0000ad875f7bb480,0x000c35156e670707,0x00066875cf1c6033,0x0000000000000000}, {0x000bceb289713705,0x000a8a9a03fa8061,0x000cac052e91978c,0x000b61eb6bb99c20,0x000e50fb8f33460f,0x000ec61a887398ae,0x0007437f45e3c633,0x000fcf4c74c22971,0x0004691a2d9b4866,0x0009004647f4a64b,0x0000000000000000}}, + {{0x0004f3cfa7e54e73,0x00076a7075c33047,0x000b446bdd4b3ee5,0x0003371a1b7efe90,0x00013826a3c98a14,0x000412cdba45fcf1,0x00048a44b5601caa,0x000206ebe3f76143,0x00050e4439f111b6,0x0008451f6bb4a9d7,0x0000000000000000}, {0x000c0d59b5f2d48b,0x00029bcb29ae2863,0x0003a9a3c78e216b,0x0007856c2465c5b5,0x0004736d155fd956,0x0001ce6b07dfbfe0,0x0008361c3a4fa43a,0x000d0e9f03c0f19b,0x000e803f9b21f548,0x0002f885460ccb9d,0x0000000000000001}}, +}, +{/* digit=88 [{1,2,3,..,}]*([2^352]*G) */ + {{0x0006b287cbed671f,0x00071c978130d6d2,0x0007aadd881d433a,0x0004f45fb4ad7bb1,0x000d7b1940d6b52e,0x0002d44569722e2b,0x000de70f91dc84e7,0x000ed42546436d3f,0x00047e41abd1bc41,0x00010f544a7be2b8,0x0000000000000000}, {0x000e82545325818a,0x0003cf3d8e5d2be2,0x0005f30317d1c986,0x00015ce098fb8ec2,0x000158947db8581f,0x00055d8793f3e6c3,0x000f50843a7feb50,0x0008ac153d3d8417,0x0004329e7248bbc3,0x000d2ffcfbcb0366,0x0000000000000000}}, + {{0x000b91e88872c802,0x0009859329a6f390,0x000332091e85e0fd,0x000d0a1fc7233994,0x000c07172741e069,0x000870fafc953488,0x000d8073b040fb91,0x00089e841e1bbb2f,0x000f582161687272,0x0007bfa72dc0f548,0x0000000000000001}, {0x000c2f4044695d52,0x000e9fc898f3ae4e,0x000d6d16346893df,0x000cc356cfc2a2d6,0x0008f9780e14adcf,0x00040c34a952a0f5,0x000bf1f1f74017fe,0x000ae85cc7e49637,0x0002400547db8273,0x000eafd4e119d7f7,0x0000000000000000}}, + {{0x000a5355afe08a2e,0x000687a2f29baf29,0x000653058a23e11c,0x000110b2ab5abb63,0x000e4ead1d1b9533,0x0005d1b7b6254324,0x00074059ad5a8616,0x00090712ab62d100,0x000e9d5016f88f2a,0x000afeefd62c6b78,0x0000000000000000}, {0x000d2d42ce0d173b,0x0009198d15289e62,0x0004baf7b535d68b,0x0008566e4a9af773,0x000402c278158bfd,0x000603f6310f0f5b,0x000331a366d639ea,0x0007457655beed79,0x0004b46175b5f4bc,0x00048f6ced012274,0x0000000000000001}}, + {{0x000b20efdb706d0d,0x000c40117c40b081,0x0000a6d9c2aec008,0x0004d3e0693270e3,0x000674266a5ea611,0x0001ebf62144a6af,0x0003d45ee3917e38,0x0008c35ff5d67fca,0x0006e79dba352604,0x00081a7e7bfed40f,0x0000000000000000}, {0x00006eb8dc692380,0x000fe33343c5a20c,0x0003e67d0a53418c,0x0008959e15eb001a,0x000ac0ead5e7c7e4,0x0002e4162f0962e6,0x00017bb3e28513c3,0x0004317568fafb81,0x000912ceb3a2e303,0x000102559381740c,0x0000000000000001}}, + {{0x0005f372c77f1047,0x000b3d958eb7b744,0x00069ac2a8ab1157,0x00057d2ec5015809,0x0007f4db5c158ff6,0x000e3fc15a71737f,0x00048e949735c5de,0x0003845758233ed6,0x000e91137608f198,0x000720cc72b9199c,0x0000000000000000}, {0x000b050528cb006d,0x000ac29d53a71cfd,0x0009a1f2ad6eb262,0x0006c829676f56dc,0x0003dd6ddbfb591a,0x000c61cac801979f,0x00031b13cd6cc83a,0x0003cf1a5e5c85bc,0x0007cff1f95623b3,0x0004f7b2cd595d6e,0x0000000000000001}}, + {{0x000699fc8b3dab7b,0x000229f393f97b3f,0x000012376dbad08a,0x00038a797638cccb,0x0006110a40e7e328,0x000e3d1acf08dea4,0x00003f85adfc07de,0x000fffa8d9d37eb4,0x000db3ca114eff2b,0x000b41dd72c79e23,0x0000000000000001}, {0x000822fbbe3ad3af,0x000ddd71461cfcd7,0x000d3d8f03aedaa8,0x000f069c35282be1,0x0009283f776eb004,0x000746b05b9838ad,0x0002fcc85c205f31,0x000bd9143e61b0eb,0x0004e7f4435b3321,0x000a673088e100ce,0x0000000000000001}}, + {{0x00052e132b686982,0x000419eaf166734f,0x000edb4ffc59ba11,0x0004fda13f9e0e45,0x0000c12226a0efde,0x00070adf716ee2a5,0x000402012f467257,0x000f2ecf2b32d94e,0x000205b1386ade63,0x000b3bc779c31b50,0x0000000000000001}, {0x000cbbdbe0875f47,0x000a208dcda3e65d,0x00048b61b40cb945,0x0002b16480725e0f,0x000aa186079b7359,0x0007e306ab144462,0x0004b09effdc0e10,0x000a76ab4395ae17,0x0008c3ddeeefeec9,0x000216cdb5d669d6,0x0000000000000001}}, + {{0x000c70f6d4b2fef7,0x000c0c92ef06acfe,0x000790f3344c38aa,0x0000fed753c30edf,0x0003dfc8006501b4,0x000df722f2f6da80,0x000c340284a42e2f,0x0002a0f154005cec,0x00082f0dfb36ac65,0x0007bed1506b21ef,0x0000000000000000}, {0x000d76b785906061,0x000edca1c3d7b884,0x000307e9a89c6050,0x000e0ccc1519baa6,0x000663495eff88c7,0x000a17475b22e916,0x000c39e69639f1ce,0x000b1f0e827f8c53,0x000066355ede8121,0x000d5b91249281eb,0x0000000000000000}}, +}, +{/* digit=89 [{1,2,3,..,}]*([2^356]*G) */ + {{0x000ccccf35a37229,0x000465167517203a,0x0001bf938eaa2ac7,0x000d73d683a983dd,0x0007f598a6f1db73,0x000235f9ed630b4f,0x000f332db784cb56,0x0003b330540f52bd,0x0005843b0221e5e8,0x00044a09499b4ec2,0x0000000000000000}, {0x0000fb3cab0a1b02,0x0008968b6e52dc01,0x00022c046dc60a24,0x000695beae1e187a,0x000d3006acf49482,0x000960f10535934b,0x000df011e1d0143b,0x00085de371d84cfd,0x00082841456c439e,0x000585582ff3b564,0x0000000000000001}}, + {{0x000e94ac6fbf17fb,0x00044289803b61bc,0x0002e798f31e8afe,0x0005e43d9a42f37b,0x000f377aef7a7947,0x00003a6947a8f685,0x000c1b4969c3b8c2,0x000b9c542cdbdf0d,0x000501682c76bc8f,0x0006972a768660ff,0x0000000000000000}, {0x000f9daec5f3009b,0x000325c4a46652c8,0x000b09499ac89b1c,0x0003ccd5721c0cae,0x00098da46e3445e6,0x0002db691caca0b9,0x000845a793a1fc73,0x000ad927f614049e,0x000024bf07aea310,0x0007245359be8b80,0x0000000000000001}}, + {{0x0000db1596cc9e80,0x0009a231d0f49f13,0x000f1d499d6bd9c7,0x000757ac7ea9b7cf,0x0005305a4d545367,0x000f2c85480a42a4,0x00029efd461a5b51,0x000fe691c9e6b8ed,0x00090ea1ca549541,0x00058f09c0153e64,0x0000000000000000}, {0x000525f593f9a0ed,0x000edbc140a1f67f,0x000f5bef166a98aa,0x0007a559750be5c2,0x000fb8cba58b2d45,0x00014d93d0c5d96c,0x000723470bfa2f95,0x0004f6b79058b86d,0x000d58f11a8a7858,0x000fe32b2d0ad418,0x0000000000000001}}, + {{0x00016f6d1a0d42ca,0x0000a7c2c062afb4,0x0001630676c3dacb,0x000c297ad74ee6b3,0x000d18f736e4995f,0x00064edc2548a7a2,0x00031596b5d5f53d,0x00040e945b2c8330,0x000587c06dfaa52c,0x00037c462a8f05b0,0x0000000000000000}, {0x000cd636b4f0d870,0x000b2b0835ddc02f,0x000d233347086482,0x000bf92bc7f1c7b2,0x00050d5f30c92b32,0x000ce136c0491539,0x000254d29288cec9,0x000c34eb38494ac8,0x000ba2a1b0b3117a,0x000c473a85376a14,0x0000000000000001}}, + {{0x0002a61629b67537,0x000edcfef26d3aba,0x0004bac42f2af22b,0x0004da8b32bd0514,0x000be474d59e6af5,0x000c190f17846ca3,0x000f3e17e7c79bfa,0x000a13543ecbaea0,0x000e74acd0ff996b,0x000cbde27a5f5aab,0x0000000000000000}, {0x000ccc73ffeccff0,0x00082b1e746179a8,0x000b19b717d62af8,0x0005045a4e0895be,0x0006b8f194a8bb25,0x00089f1cd50b3736,0x000f2a57b3da3e10,0x000691e4f67468b1,0x0008976ca9c4602f,0x0005e53ed98ad969,0x0000000000000000}}, + {{0x000f79bf523ecdd9,0x00097ebf36d74486,0x0000fe48147bf45f,0x000868d46235b3ae,0x00073a9b13d93d40,0x0004e9264c45fa91,0x0008c79b5705f4c3,0x0000fd4b166fd0d5,0x000aca2edaf4ff87,0x000955b68a488f7a,0x0000000000000000}, {0x000f3e0b19951fb2,0x000c26747dd972f5,0x00092d84bc8f6fc0,0x000255f7088102b5,0x000c984970893201,0x000a6791707f6288,0x00072769309b54e6,0x0006389f4da5d532,0x000c1eb23c48b5de,0x000d1bac794b858f,0x0000000000000000}}, + {{0x000e2d8dfd2d5a87,0x000e5a708c918328,0x0009b9fb00f1dbe8,0x000fe9c7a3695cb8,0x000749205b4caaea,0x00056f204bd6ec0b,0x0001a73a9e0254c7,0x0004152441cfd51d,0x0000d2b8b0ca9156,0x0004dee3cdd9e937,0x0000000000000001}, {0x000ab13754dec146,0x000b5d5322d78e9d,0x0000adbfc5578b8a,0x0000ce27a2b97f9c,0x000b49cd573f2d5d,0x0006ee23d2e94e39,0x00061ea213bd15a0,0x000e561b9d34708d,0x000fb576c6271f59,0x0001669ae9450741,0x0000000000000001}}, + {{0x0001a66376c54f37,0x000a06b888102e23,0x000430b0efaaecdf,0x0003b1e3d888793c,0x0004f8beed2dbb12,0x000ea5e72a8887df,0x000d4aa2425e9853,0x0009d98e93f3ccc7,0x000cba97fe918171,0x0006b08ea6eef307,0x0000000000000001}, {0x00019b171f51c344,0x000d5e5d9f40be57,0x000fea96313e16ec,0x0001461efe1e359f,0x00043de9904f3d9f,0x00081bb7a038f6d9,0x000ed552c5787d58,0x000a67fc2cd9a74e,0x000643f377ccb483,0x0009db7070b5762c,0x0000000000000000}}, +}, +{/* digit=90 [{1,2,3,..,}]*([2^360]*G) */ + {{0x000188556b53942b,0x000736bd7c7672ca,0x000a466705820ced,0x00039ba4b83d6897,0x000af174ecbf7e3f,0x00003dc58b34188f,0x000b453db5dba0b2,0x000ef54df32d5206,0x000d08e3c52fcf51,0x0003732f551f3408,0x0000000000000000}, {0x000937ade92603b2,0x000b6a7f7f5dfd6b,0x0003151876b632c9,0x0009040d3ee4a789,0x0007441b009fd7a5,0x0008b427fedfc2d2,0x0007f921c0ceded6,0x0002220fc8f207d5,0x0000f675383c79a4,0x000d6f410a2e837b,0x0000000000000001}}, + {{0x000fb8b792ff9c0f,0x00062d82addc4301,0x000b9cdf1d9fdb00,0x00042255b1cf25ad,0x0009daaea42ebb5a,0x000dffd105199066,0x00001d688f207641,0x0001da7769bd6130,0x0004ea507a275aa1,0x000073ea612e43e0,0x0000000000000000}, {0x000f18b4b24386fb,0x000f72268a5e0821,0x000ad126436a7554,0x000ba02f714fe1c3,0x00019b7c7cde45b5,0x000c576f09f2da35,0x000aef34fb328e0f,0x0001a0386e0f185f,0x0007a6bb32adc73d,0x000733da21be9ac9,0x0000000000000001}}, + {{0x00023d540d542b80,0x0004cc500040b26a,0x000e6a09fa7f8755,0x00027fbb548aea96,0x00065fa1d8c060cf,0x000943cfee1a6187,0x00061bce6a8c7ea1,0x000d99730b0b20bf,0x000eac170744528d,0x000423d049742c42,0x0000000000000000}, {0x0002bba636da345a,0x000a62e601cd801e,0x000c9e240a6cbeef,0x000103af8106469f,0x00007c7109e54da8,0x000b9a3ec3dcc449,0x0007788e44b6df8d,0x000d0e67c93ee34b,0x000e8347b4a58495,0x00037223b5096e63,0x0000000000000001}}, + {{0x000417e035b970b5,0x000ca1b60364c1bf,0x000ea847f52dda37,0x000517fb28527f5c,0x00007a1e399f798d,0x000452c79fff102f,0x000688a87dfab3cc,0x0006490b0295c5aa,0x000d17acd0dbc605,0x0005acb4f6972c3d,0x0000000000000001}, {0x000fa35559042635,0x0004e33a903ffa23,0x000c46f6e3526281,0x0007bc6b1cec4214,0x0004bca2e1dc8726,0x000b50045720b747,0x000c697f394811de,0x000ba5001f3d4304,0x0009ea7fd0f7a5e2,0x00090dead3d0124c,0x0000000000000000}}, + {{0x000e47c23d19de00,0x0009afd475bc3cb7,0x0001acc6490ff459,0x0009f5dc39b1950f,0x00064d14540f1ee0,0x0001b75050e51c95,0x0005647bebd088ff,0x000f240dba4c1789,0x0009b95e8097400c,0x00085b5d4b842055,0x0000000000000001}, {0x000986a76d06fbfb,0x000d7fc2ffb65385,0x00018e264c5a478e,0x0005a2784841d184,0x000fe21d9e8a017a,0x000bf52154297fe2,0x0007dad072d6d911,0x000eae77c8ea8832,0x000786b6a02d1fcb,0x000e682555450013,0x0000000000000000}}, + {{0x000de731f9f48a0a,0x000a357753cff617,0x00073655403972b7,0x0000484d28d73a10,0x000c846d46c140c7,0x00055b7ef1516a9d,0x00014890b5525944,0x0005f418ade1b816,0x000a465f264a9164,0x000ed37693e9a176,0x0000000000000001}, {0x000e5c3bfdcbca2f,0x000121dc135bc4e5,0x0003d39b5c7ca946,0x000be46855877498,0x000879fb5d801318,0x000afd92b1e5cb62,0x00024aecd7f80343,0x0004a3835c8434ed,0x00025764c6aa7d95,0x000a0241780668d3,0x0000000000000000}}, + {{0x000c0928cf2280c1,0x000f2c37933b1734,0x0006bae2a2974a56,0x0001e8bdb1d26ac8,0x0009c84c336cb6bd,0x000ca41014cdaa1b,0x00024b87838c44fa,0x000f525239cae2ce,0x000cb0507515f204,0x000993dbd0e0a58b,0x0000000000000001}, {0x0001411bdc3926ce,0x00087f3e15aa5363,0x000ade47bf68672c,0x000028e493da50d5,0x000120048f8cd148,0x0005ecfaeb03c756,0x000e1347b7867aab,0x000ba0208953afcd,0x000be9b23e2411e3,0x000a2e848d40b424,0x0000000000000001}}, + {{0x000583ec08d4ad28,0x0007687b7ba7d916,0x0002b3f0b4e2bbb4,0x0002caace0e4b3fb,0x000b0e6fb63a6917,0x00000520c822aab4,0x000f41f7930e37aa,0x00046bfa91da4dcb,0x0008bd604f521a69,0x00040fa707c1f0b8,0x0000000000000001}, {0x000520b880d23952,0x000bb822333018d8,0x000aa6a00bca6bc2,0x000f3469011553af,0x000c20ed5fc0a5de,0x000ee0e8c5bcfec7,0x000476e2f464224d,0x0002d844542e8adb,0x00009924fd3c1bdb,0x000f2fac98d161a7,0x0000000000000001}}, +}, +{/* digit=91 [{1,2,3,..,}]*([2^364]*G) */ + {{0x000c9a655367407c,0x00001acf48b04d30,0x00004344830b68ea,0x00058a53174d6fa7,0x000f59044eeb31ac,0x00087d51a60524d6,0x000a344fb882d4df,0x000d1ed41d08aa0b,0x00086b6aea85fb93,0x0007f27fa57f4860,0x0000000000000001}, {0x0006f6fa7b7febd7,0x000ef92aae956259,0x000abc183c404813,0x00011be4ded30d2a,0x000e220b7ae966a0,0x000c3e6cfc88e77b,0x000a92b77d5e0cab,0x0003d7f99c6dac06,0x0000a6be4c76c302,0x00032d1d55150da8,0x0000000000000001}}, + {{0x0000d37f0300cf42,0x000564d1a1ebfaea,0x000bce4cf04b07ea,0x00084f2b4677d784,0x000db14a4f867741,0x0000b95ce93b8741,0x000a31735b5960b0,0x0003d2c80a76fae4,0x00022c4d123107ec,0x0006cd8678a9d705,0x0000000000000001}, {0x0004b58dcaec13d9,0x00067d88c3d5f230,0x000f847248f45f52,0x0003da2628ef4e85,0x000e37945a7b9c0f,0x000a2da387ea2c17,0x0008e98e84de9888,0x00038290c88f211a,0x0004ce3667557434,0x000040ca4612f56b,0x0000000000000000}}, + {{0x0009e872c584974d,0x0006ca75132450c3,0x0006c420ca2dafbb,0x00069da0e632e68f,0x00018e1d45c5a963,0x0000394fa7a8601f,0x00098adcea6c9852,0x0001b23e3c6dad95,0x000f0655b2b99628,0x0001992529d81db4,0x0000000000000000}, {0x00000b6625e14e8c,0x0006611065dd0a46,0x0000a833140f2868,0x000735d82490f09d,0x000326d182482735,0x00074f69a678ac02,0x000e336a3425366d,0x0000093a8f215358,0x000644f6ddf3569b,0x000d19beff776a6e,0x0000000000000000}}, + {{0x0007a73bd7975b73,0x0004ba1e3ef4b56c,0x000835871e0104fc,0x000ed4624759b57a,0x000eed3c95d4d9b4,0x00029d8353648a71,0x0006bedece81ad28,0x0001452c12f2b2a5,0x000ab19b8b67ec3e,0x000ccd3f8f88bf35,0x0000000000000000}, {0x000062e0d5c7f0b6,0x000dd34abff69676,0x0009b89962a6641c,0x0002be1c0add12e1,0x00014a078191a9f4,0x000c488cf972d9da,0x00090e9607f65fc7,0x000d5cdadd7da7e7,0x0001ca37f83b3584,0x000dd43c6df02d38,0x0000000000000000}}, + {{0x0003f8ca58799292,0x000601d1cbef0700,0x000f41308b0cefb1,0x00090de4387a468e,0x00035f90ae8d18bb,0x000b356306c0a768,0x0005e167044866de,0x000114237a5a47dc,0x000143e3b4bbc084,0x000c1c6d0277c186,0x0000000000000000}, {0x000c81887bc12544,0x0006af4b6c2a8e8c,0x000ba5958fe82cbe,0x00067cd479340299,0x000d360f1809e7e6,0x000de77ee94fcc0f,0x0009d25d201341ff,0x000fac2af6e20977,0x000dfea19f8974a2,0x000fba7a5252c712,0x0000000000000001}}, + {{0x0001e6367ded4905,0x000fd6fd8b41173a,0x0000c717ef3cdcf0,0x000608fdb3300d01,0x0000d527d7c8e07e,0x00039cd9ece69c0a,0x000677211bdaf48d,0x000b5d520c7fa557,0x000bf842692f3c61,0x000055814bffe31f,0x0000000000000001}, {0x000c94502a0e0f49,0x0000528193bb953d,0x000d7bdda5ab1a23,0x0007c4a219650b25,0x0000f78abb7ba4c6,0x000eb157bdb9dbe1,0x000ace6b3d0ff943,0x00048180c4dc1a32,0x0000124b69e9b36a,0x000da7fee72796eb,0x0000000000000001}}, + {{0x000530dbdddc111e,0x0004131a0423e417,0x0002a2fcd4c890b5,0x000685a6ffdb8021,0x0001c68bcc7cf314,0x000fb29a153281e3,0x00090775f2e2a729,0x000fe4fc20716627,0x0005fa8915460a5d,0x000b7744dbded762,0x0000000000000001}, {0x00058330ded93a3e,0x0002ad9ca2943c52,0x00031d21ab8030d0,0x000fe30e76f64897,0x0002b30fd418c647,0x000815868a20f82e,0x00098d35bf8cbd5b,0x000facccde412b85,0x000358ff02bb4ec8,0x000fa54fb1673bc1,0x0000000000000001}}, + {{0x000aed08c7bd3b0c,0x0004e546195fa3ed,0x000c31c13efbcb9e,0x000f2eaeb2cc8a6a,0x000a1912ca200483,0x000f0ff27a5ee60f,0x0000a7b9e9c56cff,0x00044977503cdac7,0x00064deabbda5a3e,0x00038efe3a9fcba5,0x0000000000000001}, {0x000821113784eeb7,0x000a12560a5e577e,0x000ae4b9aaf4ec38,0x0005c9a38358d926,0x000497b69c24b1cb,0x00007485410e5464,0x00026fb660a2d50c,0x000987263ee5a4c4,0x000b3ba20c286e0b,0x000ae54bed6c50f7,0x0000000000000000}}, +}, +{/* digit=92 [{1,2,3,..,}]*([2^368]*G) */ + {{0x000e2418ebe8f2ee,0x000f4560bac026d4,0x00008c6a85ee3153,0x000c7d7d8e05a0fb,0x000d35867d053abe,0x000ebaaa06ca6918,0x00022207d8627f01,0x000fdfe74b9c6ea9,0x000478deb27dc332,0x0006b633ddba7b54,0x0000000000000000}, {0x000eb3b84da8ae44,0x000dced254321c2f,0x000ae0be12cbd92c,0x000b5fae91edd7e2,0x000adacef448565a,0x0003f288c1607c22,0x000ea8d01e22b70e,0x0004e3598c73a3ba,0x0009cd9f6c24e3c9,0x0007a5595791d3f8,0x0000000000000000}}, + {{0x000cd049d3a9026e,0x000dbe859af0b3a1,0x000d9aa6b9632b70,0x00029dc483656cba,0x0007d02bc7ba1a52,0x000fc68a06574b48,0x000470a9518ff35f,0x000aaf20c720ad36,0x0003bb49ecf8b908,0x000f66f8b9d88aee,0x0000000000000001}, {0x0004ae92aaca41ff,0x0009e2ff799aa5c0,0x00048de6d0a352ca,0x0008f5f2eb0f3051,0x000fea98f1062e2b,0x000285eca4dfe726,0x00044d322419400c,0x0001441ba1f95272,0x000c0f6113ec0c84,0x000b67f7b093769a,0x0000000000000001}}, + {{0x00010c16516e4d3c,0x000611e277c39f9f,0x00040673dd719aec,0x000e007e471514eb,0x0006cb0a884f04c5,0x0007cdfc977e1e7d,0x00096fd19b101b5e,0x0008b661589b4413,0x0006ee58455ad5da,0x00030384dfd6a666,0x0000000000000001}, {0x000dcee7cc09a195,0x000f049d8452fc3f,0x000453637f0d8b3f,0x0001dc43d12fc712,0x000f4b97aa7b885d,0x00030970db43c87b,0x000015eb8214e6b6,0x0006a5744b5ac5b8,0x0008d3fab8987808,0x0008438a227d82a5,0x0000000000000000}}, + {{0x0002d58e6de7c70e,0x000a184fd2b399ce,0x000d46ffafde56a0,0x0003266443a772e3,0x0009e5e99ec73618,0x00068acc975a652a,0x000b99dda22ced10,0x000f17534159829e,0x000ab0176c94c616,0x000fbda334609df6,0x0000000000000001}, {0x000e586ebac6018e,0x000f2f03144a03f0,0x00070d82d13df49e,0x000fad35f054795a,0x000bfca4e83c93d4,0x000ccd2e817178dc,0x00006d906f96d5dd,0x000999860a4c0599,0x0007c4473b0cc898,0x0002a9c7a2422f0b,0x0000000000000001}}, + {{0x0002b09542f251bc,0x000e9c6a4a88693c,0x000ca160d5142fa0,0x000a8912377d1615,0x000e8a6efbddaae6,0x000a4c058235658a,0x000f27a6a6dd7c56,0x00070769ea7bd61f,0x0006f5dde4e365fe,0x000d56d5c0ff87a3,0x0000000000000000}, {0x000ceae2e0074d51,0x0000c081ba44424f,0x000d9e0eec434166,0x000bbc85793c6ecf,0x0000cfdda19dc769,0x0009ff9b44e244b1,0x00055190f00a8e3e,0x000a30e6f1e94105,0x0002df24f630f9cd,0x000164028859bc1d,0x0000000000000001}}, + {{0x00091d8b229586b3,0x00062d4212a9d0d7,0x000a44dcf82dea37,0x000886a8066a0e31,0x000c8d5c7f9428cd,0x000e7fa7c681f5e4,0x000db54aa6ccbecd,0x000e8442c468c6ab,0x000eb0d35eb4fd66,0x000ecc62bfda1f45,0x0000000000000000}, {0x00043fbb4e2cb5f4,0x000b78f0a96488d5,0x0000f4585ef033e7,0x000ccf9f3f7c9386,0x0002c56f736843a4,0x0003c2a98300d2f8,0x000d0e9651445c74,0x0006ad9774234178,0x000b0d7d2ff89fee,0x0002dc8f018f3a2c,0x0000000000000001}}, + {{0x000a0b1e42fb1d75,0x0005875ba8ec633c,0x0009652adf59ccbb,0x000c9a8c4da98461,0x000058421fa35715,0x000861494cf70c21,0x0001a367a4a6e22b,0x000f1f29e364cf7f,0x0002eb8412063103,0x000046241f101761,0x0000000000000001}, {0x00035295fd113dfe,0x00087ca26d83dc56,0x00009b60485e1379,0x0006992ee53da791,0x0003f63e81d304b9,0x0005e83c82169ed5,0x000cdfc2181cd77e,0x0002864256eb4e4a,0x000dd24e836cab6d,0x0000560017a3ed5b,0x0000000000000000}}, + {{0x0000d40ea7dbd218,0x0000bffd292d5f99,0x0000b3c033efe2aa,0x0003caf5350ffa07,0x00062cba18d05709,0x000de1ef348e77aa,0x00050628dcafce95,0x000654c13b978d30,0x0004b7581a218420,0x000aebc1eed7a302,0x0000000000000000}, {0x000467c3cff7787c,0x000cc65919f6e7d2,0x000e4ef4ee66f3a2,0x0005339dd95dc335,0x00083538624188b1,0x000c9f6ee9c47f71,0x000c2d00164075ad,0x0000fb8c9b9b8fc3,0x000ab425082f15ec,0x0003706da80b242c,0x0000000000000001}}, +}, +{/* digit=93 [{1,2,3,..,}]*([2^372]*G) */ + {{0x000981c54aaf54da,0x000345cebb5e6933,0x000544b1b1632546,0x0001b01f84cafbc6,0x000115cddc181798,0x000378ad86aa1393,0x00075fe68cbb4941,0x0009588ce3ac7e26,0x000708d627f2694e,0x000a9deda381ddab,0x0000000000000001}, {0x000d5f56fe3c020f,0x00064c1a13df6f31,0x00002f2a54ccdb45,0x00018f47586ae362,0x000f6db9ebb1f191,0x000b71b6517fa3e3,0x000f8c282c695b00,0x000758306aa882ec,0x000bb71fac8a72bb,0x0005351671f4f924,0x0000000000000001}}, + {{0x0008318350691a0c,0x000b269d0103c9cb,0x000bca486ca47a18,0x0001af062d151fe9,0x000e7c2bcd6c38a3,0x000a2dd4944c38da,0x000d5e2d0cab5f56,0x0001aaffb6b80e8d,0x000b4ab87f1aa045,0x000baad7ec926bd1,0x0000000000000000}, {0x0001499620b24214,0x0007f0d276179361,0x0005f42f5e14dbe6,0x000a3745bdaa4d19,0x000b02770c1ff9ae,0x00030746fe336a10,0x0005a19965986ef7,0x000598b61ae7500d,0x000c2b7c92c56a5b,0x0007c0f20ece26fb,0x0000000000000000}}, + {{0x00087e69d72ded5b,0x000cbed0493c7849,0x00059557a83ba0a5,0x000cabc475bbdb17,0x0003d65e4a623f2a,0x00071fb7df8bf3c5,0x000f576c8eb2466d,0x0003c6d8140f7545,0x00002bd4e620a012,0x000cb967837a469f,0x0000000000000001}, {0x000871f9216cfc43,0x000a4dc25382488b,0x000ef93af7d4b3a3,0x0003ed77dbf730c3,0x000a6f764526ebfd,0x000b6152b407f935,0x00025711c016476c,0x0003dee3ad5a2bf3,0x00009ed5688aaec7,0x0007cb5a614fc9fb,0x0000000000000001}}, + {{0x000d2983eb57fb01,0x0006f8090fc8d49a,0x00055cd8ed43313e,0x000063128651d168,0x0009d55315e356a9,0x0003731bdb591a91,0x00090f35172884d8,0x000ce2ac6b9b209c,0x000e9deccfbd125c,0x0004c67d19839d92,0x0000000000000000}, {0x0007835fb081749a,0x00038c29318405ed,0x000c88969fe7858c,0x000d9e6b39c13839,0x0001a193b85889e3,0x0000167b73a0e542,0x000e3a6c6ebad78e,0x000fea5061213cbb,0x000f99af7a8cbcf0,0x000934dbdf82d320,0x0000000000000001}}, + {{0x000dd97cee516977,0x000fc49266fa93ba,0x000b3371de037896,0x000f3467c9ec4ba4,0x000bff13f9988671,0x000c0500963fcb88,0x0009447add0b56dd,0x0005dc495e382a32,0x000c531e8e3d8f13,0x0003f8ab53419d64,0x0000000000000001}, {0x0005999b06e01bce,0x0004ac4c86e9d1b3,0x0001acac55379f5c,0x000ed5f60cf793db,0x000deecc778efaf8,0x0005438ff64587a9,0x000da09df4f7f063,0x00095b550b56ea7d,0x000d4a2f2118371b,0x000897f5ee846337,0x0000000000000001}}, + {{0x000e73c876ead4f8,0x000074f77e1ef006,0x00068e9d113a8d10,0x00068c4c79701512,0x000c8af63c9e4975,0x0001a355ba1ddb87,0x000a7f69e6cd38b6,0x0001a9ae068bd113,0x000bc3a633214952,0x000e40b4f3b0cfc9,0x0000000000000001}, {0x000dc3573f4bb5e4,0x000ce2ef087b7068,0x000ffeeffd386368,0x000053bd77e46931,0x0008c8fd63c43732,0x0009915ece7b6a54,0x0009d1980b122996,0x0000807d3a509f62,0x000abb80ee8c76fd,0x000ea6d20af82699,0x0000000000000001}}, + {{0x00023cc70a1b7646,0x000fb2fea8f69076,0x00051b6f61d27313,0x00053b59960e18ea,0x0003b52c79117607,0x0003c4870ae1dead,0x000b0ba34bd1f16b,0x000eb6ec0bc7a291,0x000e48716fbdf694,0x000f3b9ceb0fbb83,0x0000000000000001}, {0x000f4df150c14682,0x0004d73e71ecfad2,0x0001f4eed11e2bf5,0x000e8f7ba0e6130a,0x000d0ee611881177,0x0004b3748fb1ee0f,0x00018bc0f1b2681f,0x000d95b14d7b81f4,0x0005715bb9df9123,0x00010458047bf5c1,0x0000000000000001}}, + {{0x0007c0378bc40ed7,0x000325851811caa9,0x0000425fd5a042ad,0x00083181ae223e37,0x000dcf721e5a193b,0x000e3457f090e949,0x000bab83df6a8520,0x00045f22447cb654,0x0000dfbcc720ad35,0x00030064eddb51ec,0x0000000000000001}, {0x0007e2d48a4ebf78,0x00086d5a22dda9a9,0x0004ad7ed63a6603,0x000bdfd6f4eac86c,0x0006ea0b3cfe90e9,0x0009746e43f3d057,0x000247c38598edeb,0x00066c63f53c5a4f,0x0007de7ce9110d7d,0x000580edc5628f93,0x0000000000000001}}, +}, +{/* digit=94 [{1,2,3,..,}]*([2^376]*G) */ + {{0x000ceb75a39240f6,0x000cd422d17ed307,0x000816d46db1d003,0x0000a452acb6fef8,0x000cb9bbe93acb00,0x000e7044e1a33425,0x000fc32d94105ed8,0x00077b448d72bf04,0x000527b278a8006d,0x0003f3ce1c27af77,0x0000000000000001}, {0x000e21a0404fea41,0x000ba2bea8a562ed,0x0007dcaa390fe905,0x000e3be58bd01814,0x00024ad37906a8c2,0x0004147c934bd6ba,0x0008700ab35993ee,0x0003eb32c19654b1,0x000e9fb6dc4b6283,0x000fffefce4982fd,0x0000000000000000}}, + {{0x0008ddce89d919a0,0x000c8653953842cb,0x000510be22ec94ea,0x0004fc6818af52b2,0x000d36cd384c6520,0x0000918e38b08bd4,0x0008cd3bbca8f664,0x000f9b3d5866c2ac,0x00015b5a22e4fdae,0x00028fcebfa696bd,0x0000000000000001}, {0x00086becff5dbeff,0x0002b0a9f4f0a089,0x0002781857ab5bde,0x000f7d34f623a384,0x000ab2fc32d5df48,0x000357b29a5aed2e,0x0002a4f85d8000c3,0x000a47c091a8d7a0,0x0008c875883e2289,0x00004c78f3991d74,0x0000000000000000}}, + {{0x0006929bd70c6394,0x00085f2018c22473,0x0004be72094183d0,0x00086c43b8504d5a,0x00086b6f18e21a1b,0x000ad297dd4ff469,0x0000f368a9142ac6,0x000e09fe86beaf09,0x0008c552b3b6921a,0x000bce953006e93c,0x0000000000000000}, {0x00072ac6e0006309,0x00015780956c6baf,0x000c4c2c5f7fa741,0x0001eba55e9870c9,0x00060f57cfca17ec,0x0006e490ccc10b4b,0x0008a9db0618cc6a,0x000131fe5f0039e9,0x000970dbcf0f5361,0x000700a442baa6b1,0x0000000000000001}}, + {{0x0003ec842981eb23,0x0008fb16678799e5,0x000db8c26f2eb3f7,0x000307e41091298b,0x000c2265d3b22e09,0x000829161a79682b,0x00094108a62536ff,0x0002002fb6dedea9,0x00079fdbc3d369ec,0x0002baf58b2f20ca,0x0000000000000000}, {0x00056d9fea698757,0x00041a61419646f7,0x000308b017c99346,0x0002b76a5de627fc,0x000cb23ee7d29bea,0x000ab47900ba8603,0x00096bb85c794766,0x000df41684cc004b,0x0007656ed94d547e,0x000e302003142b8b,0x0000000000000000}}, + {{0x0005cdf0222a69bc,0x0005e1e346fabbc8,0x000e07629de2b094,0x0005724f76a1e2b5,0x0006b43bc885c45e,0x000d1f53506f8c50,0x000458ec4a247aeb,0x000ee8c49a8e9759,0x000961f24ad9f81f,0x000c780789ce81ef,0x0000000000000000}, {0x000ac3a5a536c8ac,0x000fe30d120ebbbd,0x00029a29c912f38c,0x000d27e5470f8673,0x000f785f54b6aa93,0x00069bc2c6347ce7,0x00091c1681c6e838,0x00015f8951322402,0x00054c132d778a68,0x0009133565718293,0x0000000000000001}}, + {{0x000f8e71a3d24acd,0x000de7f8ccaa4039,0x0004a565eb389f9f,0x0001445b3a88c7c8,0x000b1b88e20b6224,0x00022d8db00479b1,0x000369096695cdce,0x000d48a7013202fe,0x000e3713baba6a66,0x0009b65be868e0af,0x0000000000000000}, {0x00018718a6375d71,0x0003fe3e38b8c6c5,0x000ee16d3bd00a61,0x0001a73a8bab2dac,0x0001decd0dde75f5,0x0009a19d5d5d598b,0x0002f5dcc2ed8e1f,0x0007b66c768674ed,0x000717b781a03645,0x000cd5eb14d9fd45,0x0000000000000001}}, + {{0x0009a4c40a33f00c,0x000d2d3bef2c63e1,0x000922215ae57c68,0x000763ee03e85348,0x000337a4a0d2cb54,0x00047d23204381fb,0x000be443d9712227,0x0007d96627f6c828,0x000fb6b5f6d1199b,0x000a433d2170b3ab,0x0000000000000001}, {0x0001d3366971bb69,0x0007a38c946a2ebb,0x000b29fb103a111d,0x00047d3675997def,0x000fd82824e10a96,0x00029d6d05a45fde,0x000500fa9b94f37f,0x0002317e08c1e35c,0x0006ed9214c60102,0x000ee0a2afcd4a2a,0x0000000000000001}}, + {{0x0003291f02ce62ff,0x0005b8a731176ae1,0x00053e5b4c8c2f09,0x000d5a2322d8f01b,0x00092f09c90539b8,0x000c4c22646cfad1,0x0008df5016016f3a,0x0002500c56f4c2d4,0x0006618c9bed5731,0x000f52249d380720,0x0000000000000000}, {0x00052bb2164b93c6,0x00009ba854f0dba7,0x0002bd9fbffad821,0x0002cee339d0e928,0x000515cfc63fee61,0x0008541bf33aca9e,0x0001f0f0fd5f8231,0x0008dccc44f51df1,0x000e26d92e5d7f0f,0x000334dc204c43c8,0x0000000000000000}}, +}, +{/* digit=95 [{1,2,3,..,}]*([2^380]*G) */ + {{0x000df775a4cea698,0x00057fa877dfb590,0x000f628f95d0e8c6,0x0004b622c58775b2,0x000e755966c521f9,0x000b826bcad94ba3,0x000585e536e18362,0x00056a8bf64e1429,0x000c0d065ab9cff4,0x000c6b7ad254f1fd,0x0000000000000001}, {0x000be22413e6824f,0x0009f5a869cd6010,0x000399cde94b5cc4,0x000c96f6029dfb84,0x00068c7d0822053b,0x0006cb5f4bf3d33d,0x00090bad70bd72fb,0x000c5f85b78281e7,0x000fbd3aad6b87dd,0x00067e9bddab2fd6,0x0000000000000000}}, + {{0x000f0963f658551e,0x0002ea1215b91acf,0x000276c4b8c7ce3e,0x0002c599ae4d76fd,0x00006cf3a3b9f27c,0x00006667b3985a81,0x00095ab3dd5545f7,0x000e9ae8ea63e5bf,0x0008eb3015a494d9,0x00014b36df8e2aac,0x0000000000000000}, {0x0001e0605f96eb43,0x000d54ec384ae024,0x000bfb3e2ee19fc3,0x000c041bce0a2d7c,0x000b57e0aa0d1a5a,0x000f6adf1022b978,0x000452550508726c,0x000b77d6d81e1380,0x000536c9802fbac9,0x0007d16666d2c234,0x0000000000000000}}, + {{0x000d75964c31ca58,0x0004e22c167fba0f,0x0006865896879fb3,0x00085f113d2ac14b,0x000fcf92650326bb,0x0009815c6ae567c4,0x000703330478f2d0,0x000c2d4e045bb2da,0x0004917027450186,0x0001cb3d6ff5bbeb,0x0000000000000000}, {0x0006aee5ed6230d6,0x0006f57fa7f974a9,0x000d445b19954b86,0x000cc41b7edd540d,0x0002f6672b9eada3,0x0005adb45c3c302e,0x00085355ea3de1bf,0x000a70efd3fa201f,0x0002e28049bc11d2,0x000a9e4d97e0553d,0x0000000000000000}}, + {{0x0005aaa83e0b7193,0x000ef4020dd38cdb,0x0000a2db89cf16c4,0x0008b727a5cd426b,0x000baf5617c8e43e,0x00043d6e5823ddc0,0x000180e259e17f2b,0x000506737413c826,0x0004e741255f63ef,0x00009623e6163c43,0x0000000000000001}, {0x00095e5ae0c64c88,0x0006e547505a1996,0x00074ec16e26e1e3,0x0001814a43d8b0e2,0x0004c037ed439483,0x00075672e85b1a10,0x00064a05bc4b4563,0x0004f4e8604b9fdc,0x000d8d54cdb5b099,0x0008d7035e5850a1,0x0000000000000000}}, + {{0x0006358cac1e1dd9,0x000326ae97ec9d6b,0x00096931bf3f89cd,0x0008a8a20db33ff8,0x000f15df6988b172,0x0009cd5efc8b413b,0x000bb187876052fc,0x0004662d8014980d,0x000d9235f7d44d41,0x000edff0c8921456,0x0000000000000000}, {0x0002553d46a6bdb6,0x000a25d43349dd7d,0x00098c5095dc275f,0x000e81697e9d6a23,0x00021486070955da,0x000de66e5e004d62,0x00061fc887538530,0x000b0cbeddeb407e,0x000968acbb1e576a,0x000ee36df6504685,0x0000000000000000}}, + {{0x0001376b4df2834d,0x0009122c927fc01d,0x000cb3e3200f4b1b,0x00077db8d6a633a1,0x000324c991410544,0x0009a4d4ddbf0c1c,0x0004f89da008df0a,0x000df68e550730c0,0x000e5c51e1a10f51,0x0008da410315475c,0x0000000000000001}, {0x000c76b031581137,0x000f3f4ca12b9bde,0x0004e3a329753a8b,0x000499c86ef86a89,0x000c838a372fcc5a,0x000ec44e4a97d666,0x0000ea241b991236,0x0001650c8dbf9560,0x0007627fd4eb71cc,0x00078654f79c844c,0x0000000000000000}}, + {{0x000ce7225c38fca0,0x000cf6f1a6e96969,0x0009cc6268e77785,0x000308efc7d8303c,0x0008a6b0e52762b8,0x0004bf9968cba9dc,0x000db7c416fd26fa,0x0000e7d932fa8c4c,0x0002063b5de7df0b,0x000a1bd8e36d9447,0x0000000000000000}, {0x0002f11b9d88f945,0x000b6a528d0c6d85,0x0000491c222e34eb,0x0005246f572cf3b4,0x000826a507a97323,0x00051b49545419c4,0x000f246a45d14681,0x0008555cac591e9f,0x00067c66c74cf909,0x000c814d10852a3f,0x0000000000000001}}, + {{0x0004d6495109aae5,0x000c4b86a81e7f4d,0x0004316eb1054df2,0x0006877c19b90005,0x0007963ac12d941b,0x000bdc46a9dbe383,0x000befbe68280f87,0x00071d97d1dc04af,0x000aabbcc64f8fe9,0x0008543ef9d7ecfd,0x0000000000000001}, {0x00056ebb21998e32,0x000ab05f744a3f42,0x000c6587a4d462bb,0x000c14a18de305cc,0x00056d2c0e8c5f7c,0x0007f552fd1c2fa4,0x000165b93096db0d,0x0003eef9e935df5d,0x00013440e0a7d4ef,0x00020c2ecbc3b683,0x0000000000000000}}, +}, +{/* digit=96 [{1,2,3,..,}]*([2^384]*G) */ + {{0x0003ac5243ed86b5,0x00075f9805e79dc4,0x0002bee2dfe95a21,0x000284b06125c31c,0x000a0103195082b6,0x000cedfa4a2264eb,0x000afa325bc143e8,0x000ae3ae24853199,0x000ebe96963067c6,0x00096c54a7cecdeb,0x0000000000000000}, {0x000e3a5229434e36,0x00055f3a1a50446d,0x000644f2db4f7215,0x000ad43f6dc38924,0x0002239beb126b72,0x000840de05e7dd77,0x0002862d6caacd0d,0x000bce6fa639c67a,0x000087602ba53021,0x000679ec9b598271,0x0000000000000000}}, + {{0x0005ed4dab9a7e9f,0x0006797aaab2f5b7,0x000301593051ee37,0x000962a30b02f44a,0x000a1d622cf13021,0x000a7dfa0555a3ee,0x0006aca4fcd685c9,0x000ad2e750777a2a,0x0006aa905bd4c914,0x0008ffeec52d7b1e,0x0000000000000001}, {0x00095204a3f6fa1e,0x000c34539f85e4fa,0x000e8ddc16e36eee,0x00044a9e74599d1c,0x0007ab343c6c5502,0x00007951ae714c01,0x0005c8c4503f92db,0x0006830499e544d1,0x000188a7b94680ff,0x000147d6c4809fe7,0x0000000000000000}}, + {{0x000e6ff25ab97af1,0x000ec754ac7f57b5,0x000607a92403e802,0x0000eca9aece9592,0x00080834b57b8bd9,0x000e2fcd37a127de,0x000ed63155d3ec1e,0x000f99b11a3b9b54,0x000db302a147172f,0x0005d9290b10d36a,0x0000000000000000}, {0x000eee6089f16e1a,0x000772f210e83cea,0x0002496230bbfc3c,0x000c1fbbcea01caa,0x00032cc99a5937d0,0x00074603f91be511,0x0003cfecb00c641e,0x000ef833255bc0f0,0x00085cab2f6311c8,0x000c7fcc61e9c871,0x0000000000000000}}, + {{0x000f147189a2546b,0x00070bf89fcfd41b,0x000a403ed89c0790,0x0003f861a107b324,0x0009b4dade6e318c,0x00032b6327665e9f,0x00016ecb408e3b33,0x000c11ee2181f62f,0x000ba590467bbd1b,0x000a0f4b5440b9d0,0x0000000000000001}, {0x000b1aa7ade86660,0x000d1a8a32a33d9c,0x0009ae722d5edb96,0x0004c7770654bd1b,0x0001d03d0e5a5166,0x000b01ee816a4a63,0x00015843d1d9344f,0x000c1e1821b3c769,0x000c22520be2d285,0x000fe7e0834d6faf,0x0000000000000001}}, + {{0x000a30d8168852db,0x0007853d06621fd0,0x0000e561a31ba03d,0x000e2351f7abbee6,0x000bfa802185d2d6,0x000a8cb11d19a8eb,0x000b4f2b7623f602,0x000f149b3db4a55d,0x000399a66e1d6eee,0x000dded7de613b1a,0x0000000000000000}, {0x000694da4aed0861,0x00013409999d11ea,0x00021cefe8e175cf,0x000ca47b4e5a0420,0x0008274d934ac552,0x000e270720e741b5,0x0003c0a8bd69787c,0x000e75f9cc756f74,0x0003e654e1ea2208,0x00040979701ca5a9,0x0000000000000000}}, + {{0x000d12f4d256a679,0x000fb6f240706408,0x0001c6799f2d7255,0x0002b1c7ad7d86d9,0x000c6259fb2898c3,0x000dc9f2eb172083,0x000f61ec85a5a26c,0x0002303eee79dca9,0x0003cc24582cff2b,0x000018aea38a1c28,0x0000000000000001}, {0x000fee514ac3447b,0x00004db0b385f6f7,0x000785b0f880a482,0x0006256aaff858dd,0x0005224f69e65ae6,0x00099c8d9092f5e3,0x000a4d5b2e1cceee,0x00065201f6ee40cf,0x000abc908fefbb83,0x000f4461f21689bc,0x0000000000000000}}, + {{0x0006ba04a3b21865,0x000b257a4daf424e,0x000a3bec5e4ad4b6,0x0001bf9b577cca5f,0x000e764bad272e02,0x0003f73ae3883e3d,0x00078df5e21be44c,0x00046e6d9107b020,0x00009a9e8ecf5414,0x00062dbf7ce0cbdd,0x0000000000000001}, {0x000ed268a93dd876,0x0001345efa0e6741,0x000a124edec5cb04,0x0002301ceb5830ec,0x000c074b8d138854,0x0003717a195a4b92,0x00078d388cdf3e4c,0x000aa6e0172f6f1a,0x000474fe593756f7,0x00063e14f10ad2f4,0x0000000000000000}}, + {{0x00056d8ef5183d33,0x0006d4aef07505d7,0x000fd1d5c09dd4c2,0x00026645b43bb4e7,0x000876a817eb253c,0x000209c32af92f54,0x00083a63e205a086,0x000d802502f8b9b8,0x00069b5a7a6f9cb8,0x000e8d87089ec121,0x0000000000000000}, {0x000c2981d5c3f78b,0x000fd7c0b6dcff4c,0x000e2051e1d39135,0x00056f990f800ca1,0x000bbae12c766764,0x000fc5fcbf987a86,0x000db02cbf344cfd,0x000965a4f55ea853,0x000d8cf4b24b115f,0x000ddb28cffa2bf4,0x0000000000000000}}, +}, +{/* digit=97 [{1,2,3,..,}]*([2^388]*G) */ + {{0x0004c9f1b01bbcb1,0x0008c564ba5d43cd,0x000b6d345ac26289,0x000156220f11b91d,0x000dcfa53c395f80,0x000e8f0647eb32e3,0x00071de125c49a24,0x00035c7837015374,0x000bc6a877a9741e,0x000a3e1c5d3aa600,0x0000000000000000}, {0x000eae31d5cbf2fb,0x000ff21336f73218,0x00063097ec47555d,0x000e41ac8f16a8d5,0x00010c063790a928,0x000842e2a872cf72,0x000d214bed4668e2,0x000e7b5aab91e7f0,0x0002089cbbd94783,0x000d7755df6f3c47,0x0000000000000000}}, + {{0x000c45be80c5ceff,0x0008b5f3a736f81d,0x000f684853db7b94,0x0007af37a1d8a120,0x000c1e37602997de,0x0001ba866cadb401,0x000496a24e3e6caa,0x0002c3eae9d6cd94,0x00074e14e06f82a8,0x000343069d99834a,0x0000000000000001}, {0x00060baf525e3e5c,0x000adc3855bae42a,0x00020017824fbb6a,0x000d2439950ab7cd,0x0001396b971049df,0x000925d16b65cd1e,0x0000815f2ec4dd14,0x00099059e2e1d82c,0x0007231633ba03a7,0x000c17f35a3c602f,0x0000000000000000}}, + {{0x0005cd392892c72b,0x000067c79588a466,0x0003e61043554996,0x000e4fea3dc40ec6,0x0006245b592ed7ab,0x000d5fddb91366af,0x0005bcef5cf9183d,0x0006f2afe7d93c79,0x00038f789bd81e98,0x000a24ae25969d23,0x0000000000000000}, {0x00015df628bff28b,0x0008b0238f67e086,0x000debae13d569b1,0x000537b081c52561,0x000e3e637e8d5f7e,0x0000b558a9b4dba0,0x00092e9e6c338cb1,0x00054c7742924ded,0x0001b877e357069a,0x00077dfb12df7675,0x0000000000000000}}, + {{0x000eac10daa6fc06,0x000a451d0d93fa05,0x00030dbe3663fb80,0x0003df5569a67246,0x000583a7b7740137,0x000a1bf5763b688d,0x00007976f4ce0d19,0x000972348a80c203,0x000296b9dd874e74,0x000ad726fc872424,0x0000000000000001}, {0x000626a33acca649,0x00019c0b206a34a4,0x000597e1cfe661fc,0x000915e61f98c11b,0x0006400c41adb010,0x000ed21859bdcb2d,0x000247507de82c2f,0x0007ba1295daeffa,0x0000842d673a4f29,0x0001721fc083b457,0x0000000000000000}}, + {{0x00034490e59e8714,0x0004db94a827c594,0x000be5c43f7b8f49,0x0005c1ed99f38332,0x000c89e5a5eae6d0,0x0007a328c3f873b0,0x000b195eb0205cc5,0x000957c8e1b9bcde,0x000a9c05764ec15f,0x0003785ee6082c58,0x0000000000000000}, {0x0006efc5f7d22a96,0x000c221343eb9e1a,0x000752365f225594,0x00006ed92aaffd4a,0x00047b4ac0a51a79,0x00034752a1b444db,0x0000b01d86935b36,0x000ff91905b43171,0x000a16a9f33a34ae,0x00098febd2ea9993,0x0000000000000000}}, + {{0x0009425bd3be24b1,0x000ddf604bb70186,0x00004732993f7d31,0x0003e481243d40e7,0x000cdd085631cbfd,0x00020a779443a447,0x0005a3d17a4bd6e9,0x000c2ba013a6f159,0x000091ba40454351,0x000bd45892d66dc6,0x0000000000000000}, {0x000413e025588ac6,0x0006b5fdc28cccb3,0x0001b7df9aa499d4,0x000dec43b5bc5fae,0x00066d0478edbc5a,0x000c5c7523db595f,0x000f66c3689cc921,0x000369286e3460ae,0x0009f5650aae055f,0x000ecf05c2d82ed3,0x0000000000000001}}, + {{0x0002dc4c335595b5,0x00087d3146806d12,0x000280502c85e0da,0x0006080ca34c63d6,0x00057bee19e4fbfb,0x000b329c24618627,0x00059c63cee44932,0x000729a1018b673d,0x00038096ab196705,0x0008deabcefcd004,0x0000000000000000}, {0x0008d702c45db29c,0x000164a83c7ff2f7,0x00004bdcc8d9c10d,0x0006a51badde1985,0x0007b3d628a8f0d9,0x000238514e884060,0x0003ad2af9123d94,0x0001c02c79997897,0x000547f0e64e4ad3,0x0009546706bd2921,0x0000000000000000}}, + {{0x000474c793f4e626,0x0001b9f27eccc505,0x0004061bf3c9bbd8,0x000776661385eed6,0x000ca2d5cbcfee54,0x000b0a8618a0f415,0x000c45418675f88a,0x000e0d5df7ae47e7,0x000f8808275324e3,0x0005bfe818ccf0e2,0x0000000000000000}, {0x000d8f452acbfbda,0x000031ef224de6c2,0x00070dcb7f79ce12,0x00097b18d958660b,0x000224dfd1c89c45,0x000d69ce10da0fa5,0x0001b65989afa822,0x00094e1b3fa146e8,0x0002ae217dcac503,0x0006327d3eef4183,0x0000000000000000}}, +}, +{/* digit=98 [{1,2,3,..,}]*([2^392]*G) */ + {{0x000f424e1d9c6487,0x0001fab2d1847091,0x000c18abe11482b3,0x00002a204548d244,0x000d2901117bfc9e,0x00086c34a4b1dc1c,0x00025a48ee98a6d2,0x000cb76b3ae604c3,0x0001c8298db518ae,0x00008fb7194b2498,0x0000000000000001}, {0x000055d976a5cca1,0x000a2cc7061a3bfd,0x0005a667bf88a107,0x00028dc3af0a63ac,0x000873e8641fb210,0x00040a558da80d94,0x000fc0963f8d8c7d,0x0009ccc465473acc,0x0005ce362fbd5307,0x00081d15006f15e8,0x0000000000000001}}, + {{0x00022141cc3287fc,0x00038266149a76c1,0x000fdcc4f6ec602d,0x000673e316b23844,0x0002f7ff0591cea0,0x000226449753082d,0x0008c9cafeaf6b58,0x000801a84f6bca52,0x000738672e240363,0x0000fb54c4174c43,0x0000000000000000}, {0x00005f340ab8f558,0x0003c50b6de62a42,0x000c67ea2a5a8485,0x000584f292e0d583,0x000976bb7a441296,0x000e6eb31fa9f0cd,0x0005886d0d33dfac,0x0005ca9b5dbb850b,0x0003bbd95e75a3fb,0x000aa9fc35ee4214,0x0000000000000000}}, + {{0x00048296df946f82,0x000d1ae0f7178948,0x0009fea01e8f3835,0x00032c83af761a04,0x000c81060dc76c7d,0x00037519156c5ff4,0x000bd4aec6a8a45c,0x000cdf6166cd6f1d,0x00042edd4a64bcec,0x00087353c4d352a9,0x0000000000000000}, {0x0007c5d100fac674,0x000db9e757c12a95,0x0005aae530260819,0x00099d6efbc64c70,0x00069b00bbd4972a,0x0000b755c13bf68d,0x000c9bf125fdc71f,0x0002e7e830894502,0x000fe09937e20c9f,0x00002b5fa7bc0dae,0x0000000000000000}}, + {{0x0006f3e4a3b653b6,0x000d1a688f6e0a4c,0x0009e0c6a622a9b6,0x00092759febdd4b3,0x0007de14d66ca7fb,0x000a2edb8e3d698e,0x0001ab808ea4d110,0x00080f85570610b2,0x0004c29e8f8405b0,0x00027dadf7ff6310,0x0000000000000001}, {0x00057c2f52f741db,0x0008e2e671ed8847,0x000d5288875b3801,0x000d96a8629331d9,0x0005a7e602196279,0x0001f2dc09559eca,0x0008fe5f274c1468,0x0006483d04249681,0x000f5286b1c8246f,0x00077944ba105276,0x0000000000000001}}, + {{0x00065afeabf73b11,0x0008c74def9ab376,0x00022e8ebbf2f73b,0x000cbc3cc29566a6,0x000e491e3c34a56e,0x0005e3e30062fb22,0x0008f34bd61c9faa,0x0006ad9002a6b766,0x0001e57f55fbc947,0x00052212e9e41cf0,0x0000000000000001}, {0x0000895f7a4eff76,0x00077650f0c0269a,0x000b638552f2435d,0x000cabbe1734cb6d,0x000ac1912708cc1d,0x00051cb37cd08219,0x0002a65eea8444e1,0x000bbf996f86f52b,0x000fc6bb767f7430,0x000d61b13f4e2c8e,0x0000000000000001}}, + {{0x000d5ccc9d50240d,0x00084d2e5496b2b0,0x000f884405e473c3,0x00057903c51b5ae8,0x000d55afcc0675e0,0x000125af4b250a76,0x000937bc2b1d7e99,0x000a633470b9206a,0x0000cfe28137ad4e,0x0002cd8906210714,0x0000000000000000}, {0x000877153680ec3c,0x00019913626b5dae,0x000bd89f532b2d72,0x000d578470af1c82,0x000b3f975bb94d7b,0x000f53a6076b8092,0x000db22bb5b744d5,0x000f43f924c34cb2,0x0006114e078b5812,0x000fb3e3de49501e,0x0000000000000000}}, + {{0x00028536905d5e34,0x00092fdd83e93bc7,0x0001a64b28a1f75a,0x000a170fe5a66cb3,0x0001e6a59754016b,0x000c5178f6263ade,0x000bc78f0892f5d7,0x0002723285c0f5f3,0x000ce420f8de3807,0x0004c264b36ac9d2,0x0000000000000001}, {0x000334fb73061393,0x000f6bbeabea50b7,0x000a60e0b4b57374,0x000565e15377dc7c,0x000ef0ff94b0e964,0x000e47e49a96f0ea,0x00030ac68d3dc2bc,0x0009347ee5d6453c,0x000eac1f6901d599,0x000086abf91bda19,0x0000000000000000}}, + {{0x000d441660f7e89b,0x0004d20a8f4b5442,0x000f7bf31e592476,0x0002835026e4af60,0x000359fc53bdf7be,0x00012726ea080568,0x00075b1726e147ea,0x0001ca2a0207d5e1,0x000322cae833e591,0x000e936cba51a14d,0x0000000000000001}, {0x000a1d6534e7b937,0x00065ce0948dd2da,0x0002f88e2acca2b5,0x00091e6c94ccb3a4,0x000c7d8fe477585d,0x000807f46db59ceb,0x000b940f42378210,0x000e402ecff0fc9e,0x0007bc598d173f94,0x000f16af975145bf,0x0000000000000001}}, +}, +{/* digit=99 [{1,2,3,..,}]*([2^396]*G) */ + {{0x000942a4c75e9fe7,0x000c673dad29e115,0x000e6be55a2350de,0x00045659ca3c399f,0x0003c87e22652712,0x000bd4c4458f51c6,0x00057db758ae1a11,0x000c547db810319b,0x000c8ba847d5c89d,0x000239959a5bbe62,0x0000000000000001}, {0x0003d490f3876f02,0x000521482b690e8c,0x00083aada0850d48,0x0004a53582c13331,0x0007bae5a342545c,0x0001c50d6f31c146,0x000d97c2d093b815,0x000e82d6fbdb84a7,0x00053468edb41ffb,0x0009e6ae0e9fad49,0x0000000000000000}}, + {{0x00060773f6c223a7,0x000ee1e8255739aa,0x000672547b158459,0x0000639d3f4e65fb,0x000635059b89e003,0x000ffb7b9ac29a4f,0x000cad3267e18233,0x000a7f7d6bbb6854,0x000d91b6e79c3b99,0x00065ebec22c4720,0x0000000000000000}, {0x0002fbb0f6be8071,0x0006dc307d9b73d7,0x000c6fa2f527f8c4,0x000e537cfbf3d06f,0x00018d7888122d62,0x0001ea61a84ebd38,0x0005532479a6f831,0x0004a4bf3325eafa,0x000717809c7a3155,0x0000399520a8095a,0x0000000000000001}}, + {{0x000b807898f2ae67,0x000d7e4f60fc8927,0x000bd2ecac1c97bb,0x00075b365b008578,0x000f53b54d53eaaf,0x000e5a0ab86540b6,0x00009c6ac54c1d9c,0x000267e6b65d52d9,0x000a061126607ea0,0x00011c0a94e0f97e,0x0000000000000000}, {0x000dd52c7077ac64,0x000e6849bb24e97f,0x000e60101c7fc9b8,0x000e52f0dcbe3ce9,0x000023c779930c2a,0x00050df58185f016,0x000a6864a7c2cfea,0x000d52f720dc8c89,0x0006aecf1e2d3b3f,0x000fe51ac6cabd56,0x0000000000000000}}, + {{0x0002f47341f417ba,0x000cd8bb740324e4,0x000e45bfed89909b,0x00089a7fe9550616,0x000b7861f9828778,0x0008946a71446a53,0x0006502fbe6cdbfc,0x000f70373dcf7291,0x000b59a0e59c8ba5,0x0004ed07606ee31a,0x0000000000000001}, {0x0003f237a7cf7e71,0x000f61b5ddd50efa,0x00056f3a63a0f211,0x0004dae68319e664,0x00090e5acc6a3312,0x000d6c5fe3f2c7d0,0x0005f435cc8df625,0x000420f4229fa016,0x0009c92c95adf8ff,0x000c03d951affc2e,0x0000000000000000}}, + {{0x0008e856f1a63237,0x000c38cfb4864e09,0x000a9b6c8ef2a48d,0x000565ee070c9954,0x000b247d25c31c3e,0x000d383653eec5d7,0x00033815ac0ce45d,0x0001d40035f31f5f,0x00016fd77486c728,0x0002145bbe546eb5,0x0000000000000000}, {0x000f8a00f9535f58,0x0006793f432f2f0d,0x000a22b5ed0740ff,0x00046e71ff907935,0x0006ed013a6683d4,0x0006c4dc5feeee1f,0x000e49d371167831,0x000c07fabe848ed6,0x000dbccd8d710493,0x0003939263e99e29,0x0000000000000001}}, + {{0x0009123d0b7caf2a,0x00010c9813e058eb,0x000b14ecd8d170c4,0x000aafb543a445ef,0x00066b13251ba12c,0x000a58ee44692c76,0x0009f4193734f736,0x000a0ff39b6d1ecf,0x00049d16c582e2c3,0x000a54bc874e11dd,0x0000000000000000}, {0x00086c1422f1debe,0x000be6be161bed99,0x00094c11992adb85,0x0007d02a974392b1,0x000dc1355dd0b472,0x00001779ff9ef84f,0x0008dc1dbbcd1c30,0x0007615360f70921,0x000cd90cece2bff5,0x000a085b2b4772a1,0x0000000000000000}}, + {{0x000bac8ffc9b1cbb,0x00041fbeb1c7a5f9,0x00032bb744d70f6e,0x0006377a1f50a7aa,0x00008611c61b42a6,0x000ce936f3324a60,0x000a084c3dc201c1,0x0002b70db700c5bc,0x000f0caf347988c8,0x00025e2dfe3675fe,0x0000000000000001}, {0x000fee427e9f9424,0x00069a165d8d1c76,0x000dc1fbe1e937f2,0x000db180dfee35cc,0x000b55475d4c9745,0x00036d0a2fefc6cd,0x000476726ccd4e8b,0x000f3783e580f7ab,0x0005d079b31ff3c1,0x00009ced18ab19ce,0x0000000000000001}}, + {{0x0000806a0ffa1183,0x000b368c8ad93735,0x00045322006bc207,0x000356cef3feb2a7,0x0000d9772041b29b,0x00015326534db436,0x000c3a4ffd400337,0x0004525bb0cb62b4,0x0003dfc1aef8cba9,0x00023c5f95e7e7b2,0x0000000000000001}, {0x0009f0d5faafe7a7,0x00079f056236883a,0x000e0f2e02fe7ca4,0x00020a75821e669e,0x000fc3208dc12d93,0x000c95bd4d2a90f1,0x000127ab836d5d0a,0x0006a56f1f5b0f15,0x00053533b35a8c80,0x000e1db5d6ee484b,0x0000000000000000}}, +}, +{/* digit=100 [{1,2,3,..,}]*([2^400]*G) */ + {{0x000b291af321cc48,0x0000d263480c29b1,0x000ec9060276afae,0x000489d0d90afac7,0x000ec62d3da37067,0x00088f38b3e31b78,0x0007f35ff5fafe7d,0x0007885361013dfa,0x000270f897a6237d,0x000c2a42a2eac980,0x0000000000000000}, {0x000b6527c69198d0,0x0008c038b2196095,0x00056c8573aab3e2,0x000c299349dbf002,0x0009c016fc238b06,0x000c26c63dc550c5,0x000712822cb43957,0x00044a5765b7d6c8,0x0000be87f42f34ea,0x00061a9436ed5036,0x0000000000000000}}, + {{0x0006c0f3c9200410,0x000c1523b6d0c7cf,0x000b2a524b701323,0x00080e1d445c4f05,0x000cfb721bd24fb2,0x0004d74f5043a450,0x000c3ca459a36903,0x0001f8a997769ed3,0x0004569349bd35cd,0x000fa7a1b94559af,0x0000000000000000}, {0x000479e9fda50d86,0x00039193e5dd5ac7,0x000fcfc382ffdaf1,0x0006e937727251e2,0x000819f976e0e477,0x000e0dea37faf936,0x0002b3218ec38c97,0x00020308bb264566,0x000441534d581e3f,0x000046c275dc071a,0x0000000000000001}}, + {{0x000cb5968e20ed2c,0x000a52702c5bb89b,0x00033f897addfb47,0x000f030ce16c51a7,0x000945e8bc092078,0x0000a224a9b9a4c1,0x00074fb0d2a2dbea,0x000a00b01506244c,0x000403180ef3b3ed,0x0000d544f09c72f4,0x0000000000000001}, {0x0007f0289c261b7e,0x000d803c0211ff4a,0x0001ffe93b188323,0x000b503181a127a4,0x000960bd651117de,0x000f238b1565ffd2,0x000706280cef133f,0x0004ddb33d18643d,0x00057a46393ccc6e,0x00011d1fcc4678c9,0x0000000000000001}}, + {{0x000cb18067eb6c9c,0x000904e0e232321c,0x000c2c362eb5eae4,0x0004f20ada675b34,0x0009513d2fa912c1,0x000c8c7ff960f4ae,0x000ea3278df20646,0x0004b702cc146790,0x000b87bb57608da3,0x000f1fadae0fb9e2,0x0000000000000001}, {0x0006a843b0ca7a84,0x000ffac89f77d5a2,0x000265d14c35a368,0x000486e7957c89a9,0x000f9514b7e05bd1,0x00037cf3d5a9030e,0x0002008b9ea39985,0x00050c45cfba4fb3,0x0008d5a1561ff956,0x000bc01cc6a56407,0x0000000000000001}}, + {{0x0005bb52ea9ac7ec,0x000e28f7ce0ec366,0x0004c059fd569d2f,0x0000e89276b354dc,0x00013674b639e129,0x00051c92206d8283,0x0005fd8d62852509,0x00083a0ba16eee81,0x00023ff408ee3851,0x000e736678fced53,0x0000000000000001}, {0x000a8b28dba67d24,0x000216ba84ecb673,0x00034998afbbd048,0x0004e06cfb264967,0x00064c024c958fcd,0x000c3e07fdd668c7,0x000e902ee5004559,0x000c7be484240ea9,0x00085d1cbdf78504,0x00001fc315ffe9b1,0x0000000000000000}}, + {{0x0003f67e6d554604,0x000a7edbdf25d6db,0x0004a86faf823503,0x000561d458cbd82a,0x000fd50ad1fb2727,0x000d57e2f00994b8,0x0006a571b2b47251,0x00025dcba3cd1573,0x0003351634df5819,0x000f90f716e579ec,0x0000000000000001}, {0x00015c741fd15e62,0x000782e4509eb436,0x000bb1dede8ef754,0x000e32d87a793f6d,0x000cb27d972fea02,0x0000af4ace00b65e,0x000665980ede0c9d,0x000f6c809dcb9681,0x0003f6f1f979a653,0x000fa70638c8e694,0x0000000000000001}}, + {{0x00069573569a82f9,0x00074c79907894c3,0x000923a4f546a942,0x0003c148a698895f,0x000c357afe3d1621,0x0000597be114eca3,0x0008bde57638ac14,0x000ab1f30c4d2325,0x000c1e648b9d09ce,0x000b5b544e3974e2,0x0000000000000000}, {0x000a45f392d296b8,0x0008740111ad4028,0x000c3e262fdbe28f,0x000c3453830a9ee4,0x000ae0fdaa4f4e3c,0x000db8b9c8044def,0x000827b06665f64c,0x0004b0bfa5e94a06,0x000288ab3926f336,0x00095cd9d3c3ecaf,0x0000000000000000}}, + {{0x000b181f2f210670,0x0009ac047db30c5e,0x000f0b9977abb73f,0x000685fec5355db3,0x000cba356655da26,0x00050bb7fd48c1a2,0x000094ac2c2dc459,0x0002437ff8a8e766,0x00026425d80146ca,0x000d215b7aafe50b,0x0000000000000001}, {0x0008a54b77260e44,0x000200683c3c461c,0x000806c758fee244,0x000ee02fb90af5d8,0x0005167ac6f6571d,0x000f1dfed253aeab,0x00051fc762d73380,0x000f059d556559bf,0x0001430491432973,0x00077ebf133e93a9,0x0000000000000001}}, +}, +{/* digit=101 [{1,2,3,..,}]*([2^404]*G) */ + {{0x0002ff226442343f,0x000cd833f176501e,0x00095759918597f6,0x0008de415f1a7fb7,0x000e4316d41dcb51,0x00029d89c6966644,0x000513d7775eaca8,0x000a9d45da4eed7a,0x0002852aadd14ef8,0x00062ec016fff623,0x0000000000000001}, {0x00091f95c7b230dd,0x000a732cc46d0d07,0x00004775f23d4641,0x000d9d15b9dcdfdd,0x0007a727ace99c9b,0x00077fe3e5aec6f6,0x00023beeac1ee2fa,0x00008f21d63275b9,0x00065885355852cf,0x000ce1be6d4550b6,0x0000000000000000}}, + {{0x0006e321251c61dc,0x0002560a1fff242c,0x000a45d55894b5a6,0x000bc8f1bc1ece0d,0x00099655945af852,0x00081d51a6f4152e,0x00048184573b7d9f,0x000f69e42e8015c7,0x000b2c20669dba53,0x000e9f9624512355,0x0000000000000000}, {0x000c96019553b866,0x000abe8a5146d011,0x0006c8e83c7c8d9f,0x000d33f93fede45c,0x0006fb87d2fad3d5,0x0007a48456e1fe30,0x000e6e9f030c2436,0x000ab8f59e2c2aa7,0x000e2ecee8097392,0x000ce7dbed7e8f7a,0x0000000000000001}}, + {{0x0009869c0eb3066b,0x000607798fe984da,0x000507796822415b,0x000b35e4cf4e7bb0,0x00043514e2eac3dd,0x0006f5f2ec73d5e0,0x00073c55dac4fc5a,0x00009ae9d9b66b29,0x000e7849157dbfd1,0x000a5482a9437d4b,0x0000000000000001}, {0x000c4e943ae95512,0x0006cf652253cdb5,0x000d6ec12d70a741,0x000569eacfb8355d,0x000eedc557ace0f2,0x00079ed65a726737,0x000cbaa427929f79,0x000f9f7b12d69985,0x000f61768135719d,0x000ae5a402492b60,0x0000000000000001}}, + {{0x000feaf2d1e3892c,0x000ab68c7ed96e8e,0x0003cb959b9f3e2f,0x000c2e34fe65989c,0x00089b397dfd24ce,0x0005a4f7a72a8210,0x0003d4d194fcfc29,0x0004009eac36cd18,0x0004df54ac2005f3,0x00037ac4355ce338,0x0000000000000000}, {0x00018f15a5288002,0x00084aab158f0c62,0x000919d3c1cc9db6,0x000c654f22157c5c,0x00061a5d7e7c5733,0x000dc89cd41bb67f,0x00046690cac1f786,0x000f2b55f183f6e7,0x000f41e4cb445fa4,0x0005a969c4dd429d,0x0000000000000000}}, + {{0x000f3df64800b6f3,0x000d7480b73133bc,0x000b600425660d57,0x000b837e3aef4df6,0x0005fb5d2c2dd02d,0x00011fe018b03d5a,0x000c71322317085a,0x000d374ad3a452d3,0x000a4d81aa830346,0x0004a3299709ce7b,0x0000000000000001}, {0x00016776ed8701be,0x0009a6af0a211f00,0x000ee5799e2d0b99,0x0004ffaa89dc73ac,0x0005974371ca9e1e,0x000e0cc77bcb98c1,0x000c6b4ceea4b327,0x000a8308bd6b7617,0x000c8a4f2fabfb14,0x0006930491f45b7a,0x0000000000000000}}, + {{0x000d9bdcbcf79471,0x00074d3dd4ca53d8,0x0001af7d8d51306c,0x000b82f03e680d58,0x000a7884ca0be9c6,0x000c62e3720aacdb,0x000ad9ad633f5955,0x0000c84d067a1c4c,0x000e24eee54e3c55,0x00019dbe3f67b54f,0x0000000000000001}, {0x00099b839c026b9e,0x000bc7d75cb7b6b9,0x0005e6b4aa8a5275,0x000156cdfaa9f40a,0x0004a1992d1c2e6b,0x000b18092816e51f,0x00027a900c94afd0,0x00010f9d0fb16b34,0x0008f98b409eefa5,0x000aaed3cae46309,0x0000000000000001}}, + {{0x00060d5e996dc11b,0x000619082845b56f,0x0006ef36f7ab0ecb,0x000d028f81fdfa6c,0x000060da295844af,0x0009596e31cc9ebb,0x000308c32eb79211,0x0001ea951754105b,0x00008750f063690e,0x00083b8a021ee964,0x0000000000000001}, {0x000d813e43ebc1f3,0x000af78ecca7ac06,0x000d8eb52a4c2d59,0x00067d3099d75877,0x0005b00d48668cb3,0x0002755481aaf570,0x0000b8ddd277a751,0x000dd4573c6cfa3c,0x00075e83624bf68a,0x000ee89ee03e9ce2,0x0000000000000001}}, + {{0x0001adc31d17fe65,0x000ba5bb3a93b688,0x000b603dd9e8ce1c,0x0008b0cf4f5b70c1,0x000175f958dd3aed,0x00070f44e15eae25,0x000177aa5b942f5b,0x000302efb949c526,0x0002ba3a2c8dd115,0x000b89f9288a9513,0x0000000000000001}, {0x0005972ebede20db,0x0005b1bc841aedc4,0x000933a99b87853d,0x000597276e1536aa,0x000a0238abf3c852,0x000485ab1105488f,0x0006d521debe07d8,0x0002f1ad18f16d6f,0x000d3c55a54637f9,0x000804a2b58773df,0x0000000000000000}}, +}, +{/* digit=102 [{1,2,3,..,}]*([2^408]*G) */ + {{0x00085142795b0fc9,0x00032b75a8973a1f,0x000125d27c2f65e5,0x000245efb7e6ca7e,0x000a3d37a1c1d680,0x0008ed8871c0ac3f,0x000f92273ed1f61c,0x000f1c0619b125a3,0x000c151e1baaf99b,0x000b5fb9c92ca12f,0x0000000000000001}, {0x000d45f1302c2800,0x00028a46eca65c4c,0x000181d940c2f16b,0x00008156dae561c3,0x000ffdb51b5dbdef,0x0007d0f3f05aff9f,0x000216756731470d,0x000d3e4323b09f64,0x000c256f1c5c736f,0x000c46af757ebac2,0x0000000000000000}}, + {{0x0008b99d3a5cf086,0x000a6f7dad9f189d,0x0009613956296f2e,0x00029ebaf5d410b4,0x000d4c6b1f46d86c,0x0001709ad92dbba6,0x0000cc09de07504b,0x0006c7c9ec95eea8,0x00007648647d01eb,0x000b3919b1d6cd99,0x0000000000000000}, {0x0005f9f34e61ba7e,0x000eff53cc24a00a,0x000672781ea5e367,0x0005266f275cfce0,0x0009992d98139edc,0x0002c0efd50d9e20,0x000de18f3d9cb26c,0x000b647d23fe687b,0x00054a71ad97b9cc,0x00027a258eaff20c,0x0000000000000001}}, + {{0x0008f39dfcb8ed4b,0x000811922f1fcd37,0x000a2846f7edab95,0x000566857a64449b,0x0005d6755b91a9b1,0x00033e748b542e81,0x00055f4eecb18a57,0x0006c4663a3a4f59,0x00052acaf4bb5b32,0x00079a0d2cfa0bdf,0x0000000000000001}, {0x000a0b89e0ca131f,0x000be75df1d52db6,0x000534480e6cb8fa,0x000be7af1d438291,0x000ad1a493673eca,0x000a16c7e34998f0,0x000d3ab56804db13,0x000479fc142bd461,0x000daa988aca4c3a,0x0001acd3bd93e84a,0x0000000000000001}}, + {{0x000cbad8f99ea394,0x000cf3fa23022a30,0x0009f3a186b0f3c6,0x000e922b33420e3c,0x000c1bffbbdb1dc6,0x000aa59cdeeac227,0x0003dd943d5b8787,0x0005513a5be5e367,0x0004c0fefae77a5b,0x0003518e4c10fcbc,0x0000000000000000}, {0x00024505728229a8,0x00014b020fe0ed2a,0x00039e8625b5e8e9,0x000ac893dbd2dbf4,0x0002a5bf5b95c3df,0x0009c6d879c2cfde,0x000a75fca30a3152,0x000f3ac05ce905a9,0x000445553894b84c,0x000dc2eb87696cb5,0x0000000000000000}}, + {{0x000c1e6489e4e3ff,0x000e40fadb280ee1,0x0009bea8383ad81c,0x000e1b3d235f576b,0x000a3198405d1a52,0x00058352c1e2f66a,0x0004288cd9077fa6,0x000dbd50c9d00a90,0x0009bd9a7d893793,0x000b363fd16ecce4,0x0000000000000001}, {0x00020eac52973702,0x0003e0c70e76207c,0x00006b2a996d0f74,0x000f4f6a44bb0744,0x000df28bc6807b4d,0x00017b088a4a4664,0x000ec1355d57eb8f,0x000182cf9ce6cfbc,0x000610a314e418de,0x00010173ffff5e3f,0x0000000000000001}}, + {{0x000165051d386e1e,0x000bd47b826f3a9b,0x000a1c2b86d2360d,0x0009b8835ea5cf34,0x0009783bb9260611,0x0002b789b3e982a5,0x000e987100210011,0x00097fc2b9ab3e6b,0x000ba0e81d76b916,0x0007fbcd1d8d4518,0x0000000000000000}, {0x00091e776c559b38,0x0004c1e4fe7c517d,0x0006b3ca068cf1e7,0x0001a6fbc371e38f,0x0003b1539a3d7c5a,0x000cd0c3d50e8468,0x000c7e40ea4660b7,0x00012b9d873faeb0,0x0007f3ea3f2ff663,0x000c8c52e58fcf13,0x0000000000000001}}, + {{0x000e0d1c0e7b0a6e,0x000182d5a79223c1,0x0002d07160d41c08,0x000de95d46cca38e,0x0004d00dd9df759c,0x0001f3ff3dc94e0c,0x000ad18e4b33fc34,0x000d2cfdbdfdd807,0x000c70a120714770,0x0005cb86265c3704,0x0000000000000001}, {0x0002e095e7588e93,0x000bb0fc7a7538c8,0x00049b1bf8ce184d,0x0006eac46bad884f,0x000da577205521af,0x00073592b9bc40d1,0x000dae6302c8fae5,0x000e9a408dc07f7a,0x000d86c203a973c3,0x000db4c1145f0edb,0x0000000000000000}}, + {{0x000b134fd6bc3c71,0x00030305a92256f9,0x0007ccfce0b23245,0x000ca36a6d8cb621,0x000236d0ef54fd61,0x000a182b1b210c1e,0x000e2c48ae4f2531,0x000aa16671b7b1f2,0x0001cf556d29f38d,0x0001c83fa6c8eaae,0x0000000000000000}, {0x000a18df67396e49,0x000978a098406db9,0x000d15a5ed3d588a,0x0008786d781ea818,0x000c4ad06fce15e6,0x0006d7a550f98680,0x0004140981589bd6,0x000f7ff83976d3ff,0x00088eabc6ffe6df,0x00031647479f189c,0x0000000000000001}}, +}, +{/* digit=103 [{1,2,3,..,}]*([2^412]*G) */ + {{0x0000d7e3378fb507,0x000c9bb0731c42eb,0x0005dc372a3657ba,0x00074ab3d967282a,0x00057f9ac8856a93,0x000b740967bdf210,0x0003024fec8274b3,0x00015596459a5693,0x000d21c1794a1687,0x000a98ef7bcfc7c8,0x0000000000000000}, {0x000af7b9ab0a89f0,0x000a24bfd8b660f9,0x0009cb13ed97b728,0x0000fd15ee5e0227,0x000febd3b7d28a45,0x000367bf5b972ff1,0x00091b608f71ea2f,0x000f496276edaa41,0x000a6a4c152d016b,0x0000bf52e7daddc4,0x0000000000000001}}, + {{0x000ed176dc4c781a,0x000f2149979e6d8c,0x000b3c390a71d8f2,0x0009ec42d1cc9018,0x00013805d407dff4,0x00092c79f656592d,0x000250a69b4fae8b,0x0009ea40b75d7816,0x000e49fbb9c23c18,0x000630012080c698,0x0000000000000001}, {0x000297ec2f3d27ef,0x0009b4394adc76de,0x000084a2dca39d2e,0x0004162fa4a3c98c,0x000de4df52d9fff5,0x0006af6c27847a48,0x00049729a4d9dcca,0x000b96ed3609128a,0x0002001168323c41,0x00051e81fed2290a,0x0000000000000000}}, + {{0x00023ceeaf6fda5e,0x000fb751aba8d27a,0x00058878762241c7,0x0009d28160069d96,0x0001830ade2dd51c,0x0001c3eb509b6317,0x000cb86f909879a9,0x000dee7eb48aca8d,0x00028bf799244bc3,0x0009202a06470538,0x0000000000000000}, {0x00015f7fc2843df9,0x0002f53ba48f85a0,0x00095ae129c2b6a1,0x000ddb2a444e10a6,0x0006aecfba54d8ae,0x0007c39b4f0e8bdf,0x000e6010a72c4d1c,0x000d5cdfd0f373b3,0x00068c9e099b50a0,0x0001c8bed1929d51,0x0000000000000001}}, + {{0x000c2c46683ba5b5,0x00057c57128a2090,0x000dbf610a813452,0x000009fdd16c4a33,0x000c78f1d5b65bff,0x000560ad02b49af8,0x000ea450eb8499df,0x0003a52dc630fb45,0x000da8ac57e35202,0x000fb72fd6cb3d8a,0x0000000000000001}, {0x0006d77839d10202,0x000f1f4a52a42a8f,0x0004175b2fd3f44a,0x000ac14905fe7f14,0x000701757d0c0079,0x0008ae6d1c475fac,0x000d56b7bfd93878,0x0001b7dbf13b33f3,0x000d1df20cacad13,0x0006098aef62c8eb,0x0000000000000000}}, + {{0x000224c02776a096,0x000fdf0428bd2668,0x000824486a9c4320,0x0009f3de8bb8cb98,0x00008a99c85515ad,0x00087cd9c5ea0fc3,0x00030bd0c213cb33,0x00075f7ddf36e0e9,0x0008b09a16a3e9e2,0x000ff92ba0c69ed0,0x0000000000000001}, {0x000dd8b8dacd787a,0x00044526b1b4ed23,0x000c4fcde2872824,0x000a0aecc5884baf,0x0000edf7e8ded34b,0x000a4bc010f4ef7d,0x0007a006485cbfe6,0x000ba70311a2f225,0x00032199cd733758,0x000ee0992d474968,0x0000000000000001}}, + {{0x000d4dc4b29d1e7e,0x000ced039d3b8a01,0x000815590ff2c2bd,0x00021d5655ef3773,0x0006fddb06996f34,0x0006f2915cab4832,0x0005a14903f506e5,0x0004bd51fa3522da,0x0009df3d4ada1113,0x00041c2c8aabe985,0x0000000000000001}, {0x000708b80fba1a02,0x00028b6a0afd8d60,0x00058773df43d0b1,0x000ecec358a258ce,0x0008b0ae7440af59,0x0000e74789fcab99,0x0007429cf7b8fd56,0x000584c72e545e3c,0x0001897264b148aa,0x000a07b9a2d9b131,0x0000000000000000}}, + {{0x000edce9ca1286ed,0x000ff3a46b7fad16,0x000535c9d7d2b840,0x0003708d5ca958c2,0x0005420c1e063332,0x000713a6fa0fa9a8,0x000918405a51ff70,0x000da85f855da776,0x000d1bea73da5e0f,0x00064183a6508fa4,0x0000000000000001}, {0x00083d4f23ca730b,0x000725501529af4b,0x0008000d1b6d21a0,0x000a7b3eee3507cb,0x00022a88a07a33b4,0x000540f9d09c28e9,0x0004983b28faa40e,0x000f54edb432d0fc,0x000ce23569311803,0x000d266759b9bb64,0x0000000000000001}}, + {{0x000fd03e2a0c4500,0x000d5cf3f782f796,0x0003ffeae620bf4a,0x000ca0158514f603,0x000545633e085e39,0x0004884fbea88f4e,0x000882a85fc77f29,0x00012678c646f605,0x0009ba323a505f9b,0x00029223217b4379,0x0000000000000000}, {0x0002e8744f4170bf,0x00081a29194f6c03,0x000a932a7916cab1,0x000fb0f9f60ec063,0x000ff217a0ff0b94,0x00003ea56b8f066a,0x000ed4a56ee8b26d,0x0005efbf8ce2c1ff,0x0002eb114ed13051,0x00043d7447433992,0x0000000000000000}}, +}, +{/* digit=104 [{1,2,3,..,}]*([2^416]*G) */ + {{0x000affb50d183763,0x000c1c5fc7f37a88,0x000a1f73a2f170a0,0x0001192983474ff9,0x0006b4738ed4fea8,0x0004d293dcdd7868,0x0000cd916188a232,0x000bc585fd523667,0x000b67188a2e54db,0x0008ed10b37344d3,0x0000000000000000}, {0x000548b086336003,0x000991fbe0b348b4,0x000ef3cdca5ad120,0x000cfcd2034c9a59,0x0004f56699960d16,0x0006df1f5f10f252,0x00000324733c5f1f,0x000a757f84ed98a5,0x0002f7eec2ce4fa9,0x000ae3d296f3ba03,0x0000000000000001}}, + {{0x0004e9382b32007c,0x000d81cd77eae7c3,0x00013604a2dad7f0,0x00043d13d0e3fde5,0x0004c6cb59871704,0x000a8441d1c5f3e6,0x0002361adfd909c7,0x0004efba78610053,0x0001559070eb9abb,0x000708ee4fe6fb05,0x0000000000000001}, {0x00079200ca4f6cc8,0x000579128ad5ec75,0x000c265973749006,0x000f0a7f8cf2fa39,0x0002ddb548c37c9d,0x000da31069648b65,0x0006a7e075eeef13,0x000504d0e4093491,0x000fd613bea6b782,0x00055bba92eb2c08,0x0000000000000001}}, + {{0x0005ab4da4a4107d,0x000f0dbf85411a8b,0x000a933992a7b991,0x0000accdce47a748,0x000c5662f2eb8c82,0x00064b5fdd12508c,0x000db73adddfe6b9,0x000af97a44a31358,0x00044c5ddfefeaca,0x0003403a084f6ff5,0x0000000000000000}, {0x000ad406fec21428,0x000e8954cddbeba4,0x00092a7fe19ec844,0x00084bffa4c49f5f,0x000ec8eb76b96304,0x00014948f0f75a70,0x000dff2c139503c4,0x00032fdf031b7606,0x000fa11a5ead6208,0x00047ef7a1eba7c5,0x0000000000000001}}, + {{0x00073a3dabdb3534,0x0006dd5d8f611c69,0x000799bf33f3f4e8,0x00026f63749fb625,0x00092667bd3584a3,0x00060fa9fea81613,0x000999ea3a8de550,0x0004dd75d71ac4af,0x00078319418b1e64,0x000c67e995c857c8,0x0000000000000000}, {0x000c7afa552eb541,0x000a7222bb4a0732,0x000c7e0e1a608c59,0x00009057e1132506,0x0009c5c6a19981fa,0x00019205096b7bf3,0x000a7d38ab7490c4,0x00099e016ba7462b,0x00007f474dc3595d,0x0006636a3d8c9f12,0x0000000000000001}}, + {{0x000058d88f35f241,0x000a59b2e8d2532f,0x00015502597aca02,0x000fb1e8bd7d1caf,0x00055680e361da0d,0x000ed31cfd9355f1,0x00047c0b0064d2b2,0x0004f348830f3080,0x000b7440fbffaf7d,0x000a5a553b98e17b,0x0000000000000001}, {0x0000f6eacb637570,0x000d4925881bc39f,0x000d655e7e9a7105,0x0002f09a033db883,0x000e97d5a4975dc8,0x000036e619a17847,0x00079d0e9b209304,0x000434fdadf80484,0x000412216e6c7daa,0x0000eb152f330b19,0x0000000000000001}}, + {{0x000362cca8e2db27,0x000b58fb4260cc2d,0x000e2d527b899614,0x0001f0b467cb8aa3,0x000d1ef71b82f08c,0x000dc68072c4649a,0x00098aa11a9313c4,0x000165002fbe1ac2,0x0000bf5399f23796,0x00024b537dfdd6e9,0x0000000000000001}, {0x000f6c830ade2b53,0x00074af2e76469fc,0x00051f1bc5f4ed41,0x0000406c3a450f7e,0x00002b53708a683c,0x000428a6e3aa7dce,0x000c0df44b377b62,0x000e9c1a58f5f1ab,0x000c5458b0f02c35,0x000b16ea8718da27,0x0000000000000000}}, + {{0x000134d37c05c59c,0x0007233d57586878,0x0006cf5af7409f53,0x0008ae6dfc4fd018,0x000e8b54df4cc39a,0x0005f3046db1d402,0x000312aeece717b2,0x000da13a0c5d98af,0x00073d6305f96c47,0x000587c80a3e3a7f,0x0000000000000001}, {0x00027d5a2516f5d0,0x000f9338bbf8fa7f,0x0002109c7d0c4360,0x0006004be57b26a1,0x000da32aad5aeeea,0x00041aa5daf9dede,0x000b0a16abc83073,0x000088080bdafdd6,0x0004fa8814cecd6e,0x000f5f24b2b7fe1d,0x0000000000000000}}, + {{0x000cc19d6d74494a,0x0009296d31ebaa05,0x0003edd43b02f30e,0x000c79aac72cbbb4,0x000d57829df3e827,0x0008bb62624e4cf8,0x0004f05ffe745fb9,0x000950b350aaad89,0x000ea5e2db566ef1,0x000e1e37f6dcf4f2,0x0000000000000001}, {0x00034202ee7f3c59,0x000ed5d748da48fa,0x000e1cf505b68fd9,0x00031b86c7778cb3,0x000dbdadb45073af,0x0004b6e80bfe717f,0x0001ee413036b30b,0x0003482b138b2c3f,0x000e1ed1e4fc0159,0x000e84d788bd2771,0x0000000000000001}}, +}, +{/* digit=105 [{1,2,3,..,}]*([2^420]*G) */ + {{0x0009afb73836ce2c,0x00035d1c0854627d,0x000acf6649f2eb9d,0x000acf4c38a8a9ec,0x0002178be52c0095,0x0008f6e06df7d7ea,0x0008285115ce7bb4,0x000b7f232680bedc,0x0004103a6e51e8f4,0x000aaa09aa0bc0d2,0x0000000000000001}, {0x0001c4bc539f42b1,0x0009f1f757159ce1,0x0000e9e10c0bc8d0,0x0007345be3621884,0x000d1822e5e0a60d,0x000ae792acddc802,0x0006be0f49763d76,0x000dff0f1717868a,0x0004437867cae1bc,0x00070e8bfe19f269,0x0000000000000000}}, + {{0x0001b8994b02326b,0x00031ce496416ae0,0x000dc0825cea213f,0x00050bdf0281aa93,0x00059236853f9e44,0x00041e294dc8c09e,0x000b03a595c72a58,0x000c2bc6e5381e14,0x00020b03546b3008,0x0008eca57d1874ef,0x0000000000000000}, {0x00051a61ce5948af,0x000925d36a169359,0x0001712f7655b84b,0x0002f3fdc1e05016,0x0009aa758020ba42,0x0006927405b02281,0x0009822fced2aa8b,0x00019ae63d9313a7,0x000d9c07887cbebb,0x00065e13e45febcd,0x0000000000000001}}, + {{0x00034f1f7e499842,0x000a48878049fec4,0x000692a3fe1f0c9c,0x00048261277fbbb0,0x00032263dc0fe7ad,0x000e09052ac6fec0,0x000683804d38aeb7,0x000d6c55fb1237fd,0x000bce4b453925e9,0x00056ac33e2d8263,0x0000000000000000}, {0x000a764c5c6d3730,0x000ed9705b2adb70,0x0003bab4631b9f1b,0x00014535b4850149,0x0009c2385e82937a,0x0007aa5ebdcd9ea7,0x000656517e5b5eb0,0x00078fb6885fd321,0x00087f791c6b2bd8,0x000bcad44bfcfcce,0x0000000000000001}}, + {{0x000a7c1bbda99502,0x000d13ad86ee95d4,0x0001edd6edef741f,0x0001221485b8fada,0x00062b65b3c0a089,0x000583aac0300922,0x000ec2e6716727e3,0x0006d6729d8a817e,0x0002ad34233bfe29,0x000806779921dca9,0x0000000000000000}, {0x000373f0b0bc37c3,0x0006835632a2bfc1,0x000a8cd4f472c2b1,0x0004b6c9a4cedeb1,0x00076180690c132a,0x000eca05f9fa510c,0x0004e0551eb02c41,0x000e63213fc52e0f,0x0001b429bb6165cd,0x0002bc1fdd188cd9,0x0000000000000000}}, + {{0x000629f7658d599b,0x000b1e87d39d0d52,0x000caaa997607dbd,0x000fcf23eb7d6dd2,0x000857e0cd30a02f,0x0003ecd22778d510,0x0005c5e961e1f158,0x00068aa70a145465,0x000c8fb1ff96ec7c,0x0009b464d2f55e62,0x0000000000000001}, {0x000e904d20ac7941,0x000e4f0bec2602b6,0x00080f6effaef59a,0x00060688330793e6,0x000db2442ae08549,0x0005e3d773ff5a5f,0x000cecbc6ac0199c,0x00012fa7a795cacf,0x000d6f9bfc57e52d,0x000ce32e4eaeafdf,0x0000000000000000}}, + {{0x000eeeccbba2a7d7,0x000ad1d77ed0ffef,0x0005e752d769db74,0x00015b240b6200a7,0x000597b48ab8cfc3,0x000f97504538ef48,0x0005a2e980da41b6,0x000c0010c2010969,0x0007fe53f0d2b23a,0x000a1efe8b48887b,0x0000000000000001}, {0x0007b952e8dd021c,0x00026e6b8cb163c9,0x000d62feeed16bdd,0x0007e3db6129cfd5,0x0009dcfa4489e9be,0x000f551707d804ec,0x00012a2adb1fbcf8,0x000c05be22830553,0x000c9f74b7d87937,0x0008d7e5edd3be33,0x0000000000000000}}, + {{0x0004b2f4546a8f7e,0x000d2a223a807885,0x000155358a6954ae,0x00086e4b30349ae5,0x000a5982c7a2aae1,0x0006ca4f6415564a,0x000013abb73fd2e2,0x00092d10cb063fca,0x0006104963b01aea,0x000bbcd0f1f68f33,0x0000000000000000}, {0x00069aedbf43f1a2,0x000ebe1ef2c9b47f,0x0003246ee727e6d3,0x0003f5bd1a5a6120,0x000987e829b05fc2,0x000b70444c023806,0x000f0d6e903fa23f,0x000baba52743b14c,0x0009994d97e9872d,0x000a52cbf74f834a,0x0000000000000000}}, + {{0x0007ea3240feb8cc,0x000792ae77094407,0x0001201418e13db8,0x0003920cbc07e7f8,0x000aed56c93837f4,0x00013ae3d2a87da3,0x00044b8ddc44f1df,0x00069328fcaa2209,0x000b928d8e381964,0x0007313e55f26bee,0x0000000000000001}, {0x000ec4f451bd8008,0x0007bdfe195b7e42,0x000e12b70e68b680,0x000159222b99f5fa,0x0000c4e0659127ea,0x000efe86909b5076,0x0008c58b9b667112,0x00047b7026cf16a3,0x0007373740ea01a9,0x0000153963989580,0x0000000000000001}}, +}, +{/* digit=106 [{1,2,3,..,}]*([2^424]*G) */ + {{0x000546c7f70bcb2f,0x0007212027084052,0x00013f0d6d83c3eb,0x000eaca7142e7b62,0x00029f973a763019,0x000be00c2a025efc,0x0003eca6f6199ab8,0x0002b5618bcc0394,0x000b02749fbfb9ab,0x00013c0ae9ab795f,0x0000000000000001}, {0x000f295b93c3d712,0x000bf1ab6d0e9f50,0x000bf53d7cb083d8,0x000d744e07076abe,0x0005a51a53561f29,0x000d647b915c2fc1,0x0009ead253f84287,0x000e91bd9d6251a2,0x00006dd74006b7bc,0x000c92c770e4efe1,0x0000000000000001}}, + {{0x0002703ae4d5c812,0x000427e95afaa341,0x000271961d1eb61e,0x0000fb71509a4412,0x0007bef04644ecf6,0x00044d4556d64abf,0x0002a7bfd03e651f,0x00094add4bdd9b05,0x000438d74bb2156b,0x000fa36c6c16572a,0x0000000000000001}, {0x000ae16f96be2911,0x000d52afa2d73a0e,0x0005bc81a5a44951,0x0000fc89324d90b4,0x00018cab36337849,0x000cb411d87db838,0x00021f7d6cb710ad,0x0006ad26521480af,0x00094a666f370ca0,0x000375d8bc966e31,0x0000000000000000}}, + {{0x00082baa7d62ed15,0x000a2a0c35293a3e,0x00068a91eba6b63e,0x000715c2cd81a65b,0x000ed39839d20270,0x0001cd1c736c49a2,0x00023b8eae3fee67,0x000006012e0a11a6,0x0006b901f0657746,0x0003944e0864a739,0x0000000000000000}, {0x00089e4689d04d8a,0x00060efa98dca953,0x000708ea3f1bf018,0x000f379e0740bbd3,0x000ab07cf1111586,0x00040f6bae4389cc,0x000c0c62ec14d78f,0x000a8ca2edc050cd,0x000eb9e22353ae3b,0x000221497d609a22,0x0000000000000000}}, + {{0x000dc15d0ed0cc63,0x0001ce35b540e1bb,0x000973de7aa979e9,0x0005e965745fa684,0x0008a999e957c84d,0x00071ead702675f7,0x0002beeec63eb2b6,0x00062262a9349f4e,0x0005b027fa151a2d,0x0005b45a63374388,0x0000000000000000}, {0x000ade9fdcbfe121,0x00019964a13f4063,0x000ea6e576b23957,0x0009fa2021e1e294,0x000f93e8e8d0c1a3,0x000627a8aaeb1db8,0x0001c83a239f73f2,0x0009dc2704f34ba1,0x00053062be1591a1,0x0004c975cd21326d,0x0000000000000001}}, + {{0x0006b8fb25e6414b,0x0003f3c35210ef6c,0x000fcf4305218c66,0x00044c1aba52d092,0x000ad9ddbd46b892,0x000a630015bf7459,0x000b32b246737751,0x000ced081a65d447,0x0002e2e5f643a94b,0x00044927f131866d,0x0000000000000001}, {0x0004645014a53592,0x0009ee4e5661d1ca,0x0009024d9b573293,0x000159f5f55c84df,0x000a0100b8c24f2c,0x0002de6a0aef0178,0x0009e7eedb7e96ff,0x0002067fc1f195df,0x000897eb377d8a42,0x000f1229a3daed81,0x0000000000000000}}, + {{0x0005287cf2f5961b,0x000cef9beb6f8461,0x000a7eaedc75201f,0x000d39dfabd8969f,0x0004252116284485,0x0002945b419a298f,0x0000f03b98d985bc,0x0007be292f990254,0x0007d1f59fb5c829,0x00002d379bfc2592,0x0000000000000000}, {0x0001179af2527211,0x000f2724128932bd,0x000920719e949c7f,0x000f184c1ce441c4,0x00091d4d77a7c7bd,0x00086d6826d00ec0,0x000125b3901be8d2,0x000c6d7da5fd8de5,0x00084f0587b71c09,0x000ecbd13d8a200a,0x0000000000000001}}, + {{0x00034258888b309d,0x000da5700df117ec,0x000eecc1e03d3856,0x000d69afef5df311,0x000ced3bd1e65823,0x0009ca00a3d94409,0x00053de41e7eb569,0x00002f1291722236,0x000dbd2ee0ded7b4,0x000a909f40f6e980,0x0000000000000001}, {0x00082d95c76e2f3a,0x0006d18779f8f974,0x000021eeb4e7e5b3,0x00070089298e7654,0x00003aa90b7f11d4,0x00020fd2c120d0de,0x000ddfdcf3b2e6f9,0x000134293b957e6e,0x000d5361a94456e0,0x0004bc62d97be2a1,0x0000000000000000}}, + {{0x00047691dfe1ac32,0x00095bb25141cf40,0x000cb8b02c59109a,0x000eb57046b0d286,0x000a8b7de816343c,0x000109a4f745b33c,0x000bf34e4de2617d,0x000950869270fe1e,0x000a3a1b01ca6170,0x00012e38634a9ad0,0x0000000000000001}, {0x000f079109fbe1e8,0x00071bce15260c38,0x00059aca1efc1bab,0x0006c7ad5da68584,0x000030bec54d9816,0x0009a9c32e8f4612,0x0001154d88a51704,0x000ebe8180a7efea,0x00008abc6ec5ede1,0x000d7de0d26459ad,0x0000000000000001}}, +}, +{/* digit=107 [{1,2,3,..,}]*([2^428]*G) */ + {{0x000019394140932c,0x0009a362bec3a200,0x0005ab884c7a2c57,0x000cab2ba06cf4c6,0x0003ee7eda004083,0x000107bb12f5f6cd,0x0008a0ae8649e92a,0x000a5a344683492e,0x000cfb24291fc3fb,0x000c3894cfd1718d,0x0000000000000001}, {0x000751b1b1d1f30f,0x00043fce22373903,0x00023da45dac2f10,0x0000105df9307b4a,0x000892b6d1b2c311,0x000f645131b39e6a,0x0006331d8e317be2,0x0000b2f395de9046,0x000d56345f1e436a,0x0001b0547fe7481c,0x0000000000000000}}, + {{0x00078f4b7084325f,0x000727f8bc0fa450,0x0001eb36b7bf5948,0x0004268cea0106cf,0x000e8c80e06d4c70,0x000c216323300186,0x000b3016e94d5075,0x000d882e86c372cc,0x00048a28a053b8cf,0x000343d171001931,0x0000000000000001}, {0x000c0287664ae003,0x0002766d8c63b224,0x000b8220989c0738,0x000b967cd418521b,0x0004152d761e7a59,0x000fa019e4601efc,0x000f11b41cb4d7c0,0x000dcac0131d9d52,0x000c991e4e27a162,0x000789975ffa76aa,0x0000000000000001}}, + {{0x000adcdaf64c0019,0x0002a5ef0c2175ba,0x0001cfec89a01d6c,0x000b25fa94296a25,0x00089c7ef16afeec,0x000c007e24585f16,0x000bbbc143cb4742,0x000531bcdafbbc24,0x000013773a99937c,0x000e19b7d849db6c,0x0000000000000001}, {0x000e83417610eec6,0x000e934e9e9d81b9,0x000af4480e73d7b1,0x00086003aa3fea2a,0x00079c57f5c52d5f,0x000495c2aa9268f2,0x000922e4d082c9f5,0x000448a981d714f5,0x000bdf27622c6821,0x0001ae01e6436230,0x0000000000000001}}, + {{0x0007602120347ad3,0x000dd3e9ad407753,0x0002f89c76c52251,0x00044aaf5957d5e2,0x0006157c28051b58,0x00069eb597630bd3,0x000a387acac6af33,0x0002aad0ef875611,0x000ccba64de2edc3,0x000ff1731d170b0b,0x0000000000000000}, {0x000557f665f21957,0x00064548e1ee52c2,0x0000cf955615f44f,0x0002bf2c64b6036a,0x0008d89213e751e5,0x0005aa570d34bd85,0x0005ae3bf1cb2e71,0x0003f29e703acc20,0x000a34a47fb1911e,0x00089578616fc498,0x0000000000000000}}, + {{0x000dc828f86252e4,0x000d09ec8dfb2555,0x0005077092f2430a,0x00078d5b0c56862b,0x0002f9b2b116f869,0x0007b998e4dcbf23,0x000fbcf91af3491c,0x00056f1110038af4,0x000e0888ba8f38ec,0x00043daf07963a9c,0x0000000000000001}, {0x0008da925a008487,0x0004f369188e0311,0x000ec8e641278107,0x000a270db9969754,0x00072aaa78c429c9,0x0001a344c1da5196,0x0003087bd58c2f7e,0x00088aeffcfb4dad,0x000fbcec5f2532f6,0x000a809230f905b4,0x0000000000000000}}, + {{0x0000b4a0ddb8dfab,0x0009deda0e27481d,0x000949ab745b2f9c,0x000dc2842dbe3df2,0x00059b98be5c35da,0x0002ed3249c0c912,0x000dc475c0841958,0x0005a3cc60128529,0x000078110f67e016,0x00017fe4fa3624ca,0x0000000000000000}, {0x0006abd2bfebf71d,0x000617fcd7e4413e,0x000ac8bf7c39197b,0x00022ac8be897f87,0x000878bd60d51691,0x000ed69bb2587c48,0x000a51aae1defc49,0x0003d1f77705c7ae,0x00021ab01c739c3e,0x000801dacd9f17c4,0x0000000000000000}}, + {{0x0002a912967cbb3e,0x000f46687e4df221,0x00007e4eef350ebe,0x000e38a4de052972,0x0009d5282d27fb20,0x0006e64b3686af97,0x00067350cb71692e,0x0008437b220d3baf,0x00042c826bce6850,0x000642d60e139b00,0x0000000000000000}, {0x0004eb43ead3cd1c,0x0007d1cbedb5fe6f,0x0001b45fb1ca7933,0x000a503cea7b46bd,0x000fef3868c8b110,0x000f9fe2cde99fc9,0x0006dd89f348e20b,0x00022d25b6cd54cf,0x00051adcb2128528,0x000ea79e7afb9c6c,0x0000000000000000}}, + {{0x0001fc12e7ca5055,0x00025e91d6adbf30,0x000587a0d31dc42e,0x000cc8a9cecd10d8,0x0009d8d1b41ccf88,0x0003b3f96490bd95,0x000e028999c5a513,0x0004c3a0882547be,0x000fc3438b39f5f7,0x000fa75af39ed4d4,0x0000000000000000}, {0x00002b54c1f9ad9d,0x000dbd0a29ee5fbf,0x00057045bbc4f0d6,0x0000fce2e6fde293,0x00033040027dfc7d,0x0003be51ffcda45a,0x00054869841c14d2,0x000156ff1e336d04,0x000a634d9fdc8522,0x0003ae619b696b5e,0x0000000000000001}}, +}, +{/* digit=108 [{1,2,3,..,}]*([2^432]*G) */ + {{0x000019f2fc6beb7f,0x000666646b70deb9,0x00090a5717d2e533,0x0006810fd38f5274,0x000233bb33bad791,0x0009b6b88efd9526,0x000d5745dad71c05,0x0004ca300788fe6e,0x000ed4a6a21a98ac,0x000e10eeff58916d,0x0000000000000001}, {0x000da52e803a67cf,0x0007d398c1a5fc48,0x00042b185c01901e,0x0004eda6cb2700b0,0x0004eeec867cdaca,0x000dcb7e930a19ae,0x00010a288471dcc7,0x0003198ed17517e0,0x0004e8756bb36f68,0x00027df915ed36bb,0x0000000000000001}}, + {{0x0009d95ae2bd1168,0x000fb9d6f73873c0,0x00027b3bc9e7cafd,0x00005fa81f14d40f,0x000ed4eebf4e66fc,0x000bb6036a4e6b76,0x0009e3f736cb7387,0x000b2277b08a58e1,0x00055c0d7a4986be,0x0004d18f830adabb,0x0000000000000000}, {0x0005e59f982356bc,0x0000599a3ae563b2,0x000d06e16bf18997,0x00088dac7505891e,0x0004c76b3eb973d6,0x0006498b2d10cb08,0x00074af2d6e8e95c,0x000a503f71b8afa5,0x00066999bc16133c,0x000bd76f3443fa99,0x0000000000000001}}, + {{0x0009cc5a1fdf300d,0x0000ede5aa101a8c,0x00053342a7d32907,0x000bb4fe852398fa,0x00036673ceb9c007,0x000c9247b3a94312,0x0001ee51cb682683,0x00075c07f2ef115b,0x00083b0143f6486b,0x000d101585ec3771,0x0000000000000000}, {0x00080da54d84cedc,0x000e569a2e8c0006,0x000e94c0472e41a7,0x000a41c1b7c713ec,0x000e5742d18ccf2a,0x000df3c3763e3162,0x0006b9977417a84e,0x000f165b44c75791,0x00045d988246f2e2,0x00095c8287828d1c,0x0000000000000001}}, + {{0x000f9c015fdb1c06,0x00099f25767d4418,0x00077c536b749d61,0x00054bfdb54e847b,0x000237979776c1af,0x0001eefa220b8386,0x00018acdf9bb4a75,0x0008ce45beb5028f,0x00030f98a7dd8621,0x0003023f055e3ab9,0x0000000000000001}, {0x000f321b72c7f6a2,0x000659eeb57c141d,0x0000b2255cf53902,0x0009dbecf2a776fd,0x000e6453cf8ab4cc,0x0002d5647863e94e,0x00049fe93a4007af,0x00039cf116d00271,0x000dc8184a637605,0x00061de7465f7317,0x0000000000000000}}, + {{0x000dae8508371ab1,0x0009f5759cf42671,0x0001705cf63d991b,0x000e90ca5edd0265,0x000af5c715a86e95,0x00096331dd94a89f,0x0005808565c73b46,0x0004c9ca8a595723,0x000561170be646eb,0x000a57b5fc879ce5,0x0000000000000000}, {0x00080540adb44437,0x000abb97c57c16dd,0x000be7baf4488307,0x00019e753dbf21d2,0x000ac5be29dc51e1,0x000df21c9ad3cc57,0x000a9aa74fbb9193,0x000a11ec7b8e665f,0x000c01bfce350115,0x0009909f2c73b53a,0x0000000000000000}}, + {{0x000744ab95b71183,0x0005aa6c81566c3d,0x00033310fbbacd74,0x00006a2dd55a7d6d,0x0002db05de914221,0x0001ffd4282301fd,0x0008cf7489b2de85,0x000c6f0a02baf309,0x0002ce9af0a85604,0x000261197b0884c2,0x0000000000000000}, {0x000e2c60da7deb89,0x0008d4b2f2e54489,0x0002e5dee75d5785,0x0005e0e953899934,0x000ecd968f2eb742,0x000229cbd8cbbbd2,0x0008dcc6b871d5e3,0x000490c7582e68a9,0x000cde3999720429,0x0005eaec58ffb639,0x0000000000000001}}, + {{0x00028ab59549216b,0x00005d61d82596b8,0x0009fa679fcfc930,0x000ccf438ed2cad9,0x000a40190b3069f6,0x0007f5af4bb6d49c,0x00002f9e7c50f6a7,0x00019940efdd8671,0x00002add96c554a6,0x000e007d80786684,0x0000000000000001}, {0x000ad0fb54ef0a37,0x00046af5d15fc57d,0x000b825452e8ccd9,0x00044232c710cb6b,0x000cf88855ecdc6f,0x0001d6d2b8f40b3c,0x000436e2bc3e510d,0x00007b1ff42aeddf,0x0000ed88484552b4,0x00075c1505c18430,0x0000000000000000}}, + {{0x000a55f734e8c16e,0x00026ef042fa2f6e,0x000b24c1848ab1d2,0x000acbe86862a1dd,0x000651f4168e7413,0x000d596e07914083,0x00079ca23961d189,0x000e6d53679701b3,0x000cf35fa05ec7b7,0x000d20d7f6b70713,0x0000000000000000}, {0x00018785b4c707b1,0x00038676095f2dbc,0x000e28a0370a0054,0x000af09e50c89610,0x0000f144bba0bfee,0x0004cf6dd7455cf1,0x000e722f509d9783,0x000b05c279e5f94f,0x0001244fe8092deb,0x000153b314f061e7,0x0000000000000001}}, +}, +{/* digit=109 [{1,2,3,..,}]*([2^436]*G) */ + {{0x000803868bfafe94,0x0002ddeb7719717f,0x000911e1ad005b4c,0x00076f1e0df34f87,0x000b29958d5da570,0x0005d1ebf66f49ec,0x000f5712ca7b49e5,0x0005b2ff1b32fcb4,0x000f97d3d42a971c,0x000804838bb32749,0x0000000000000001}, {0x0002e7908789eb65,0x000af786529c3ba5,0x000e71594737ddd9,0x0005400f6dd64d51,0x00039922bf016830,0x000db4bbaa21ca42,0x000935d8c94ee851,0x000c80623440afda,0x000110efc0a576c9,0x000396d79f58dcd9,0x0000000000000000}}, + {{0x000b38ccc9df95f7,0x000d1849ee6062ae,0x000164333dec1411,0x000e81b4a4a4727a,0x0009c74b241dc566,0x0005069fb7290aa5,0x0009d322865cb6d8,0x000ecbd648392838,0x000909864ce3c8f7,0x000ac9fef248d28c,0x0000000000000001}, {0x00061d435126259d,0x0005a6b96bec8542,0x0001f509bbc62a6f,0x000ddbc8b43f9c90,0x0005e94118466c53,0x0002a386779ad388,0x000db58d109dd2e2,0x000aeef4f2af60b6,0x000bc0cd7adf1adc,0x0008220fb47811d5,0x0000000000000001}}, + {{0x000f1b4d9b2b9d58,0x000635d4601e4549,0x0005712ad28d37e9,0x000e19b42c3143dd,0x000fdc6366f04af7,0x0004b34637aa565a,0x0005b2ac12b452bc,0x000277fe7f5bdd13,0x000c6ff31feea8b4,0x0000c45cdaec8e9e,0x0000000000000000}, {0x0007813178366af1,0x000e83664c221b3f,0x000afac4ecd652c3,0x0007c4666da93d03,0x000652ceac0d6cfa,0x000b1a4cf4039d2b,0x0005c58a6eb1946c,0x0007422c9b53a228,0x0007349ed1b4d836,0x000afa4c55a379cc,0x0000000000000000}}, + {{0x0009ee173ac4010e,0x000fb942f467a4b5,0x0005770a30141b66,0x0004198802513df6,0x0002c7e0148f2f1d,0x0009a1b6c6f54abf,0x00056f2bb47b51bd,0x0004f5846505fba9,0x0002a3ddcb02618f,0x000a7b69ec8c6450,0x0000000000000000}, {0x0004b69533aa9231,0x000b0804316d8f19,0x000006107c58f7bc,0x000310f29f43afd0,0x000e7a15ea5dc32f,0x000849a363e2b91a,0x00074452b4966c63,0x00071d63455b6a45,0x00039b535b8835c1,0x0005091ae86f54cd,0x0000000000000001}}, + {{0x000de1ec8614daa5,0x000f0834c4db4a7b,0x00048a29f6ba70a6,0x000be231587d1015,0x000bb998c9a2049d,0x0005eedcf88aceaf,0x0008878e6b738f7e,0x00019b693ecfcab0,0x0008dd1ddb374ede,0x0003be7a6cd94f00,0x0000000000000001}, {0x000bd130c16a2f12,0x000343c20757ed83,0x000228e06b217bff,0x000046a91bef19ad,0x000400e88da5cc51,0x0000a9f961011b58,0x0003f9049ae6f047,0x000e9d079a03c6f8,0x000401435912f072,0x0007490b78fe3f9a,0x0000000000000000}}, + {{0x00081fdfff4245fc,0x00050168c14a6614,0x000764d4d2b84edc,0x000aaa60be356501,0x00030e6771588736,0x000714d50c0c40e3,0x00029157b8fe8875,0x000a9215aac4fbdf,0x000b4091b549e25d,0x0001ad442d2b0058,0x0000000000000001}, {0x000ee6a91c584138,0x000d84934568ba88,0x00010dd1585307ef,0x0009b046cb644b24,0x000dc2bd376e1321,0x000cecc49bef0c68,0x000f76556c2d2d1d,0x000d810f8810c907,0x000a20da5052b3dd,0x00079ec3448a3c1b,0x0000000000000000}}, + {{0x000084366f4dadcf,0x00087f5cc0a55e0e,0x000a139c3fe7ef01,0x0003f2e749d53f7b,0x000ffd809a727542,0x0002e74f9e5a94a4,0x0001929541f08d0e,0x000114dd07932254,0x000f53ad14915984,0x00078a408f5bb7db,0x0000000000000000}, {0x000b74adedde4d64,0x000eeb46e287111d,0x0001ad3605f3b22d,0x000070fc8863541b,0x00093fdd530f51fe,0x000f3d69c04af47e,0x000f551d93cb6477,0x0002bc684cde6d16,0x000154a9f50cd685,0x000e9f16ac0cc2b5,0x0000000000000001}}, + {{0x0006190bccf09089,0x000de2a4a386bf1b,0x00095703cbb17c47,0x00013d223bf84891,0x0007a124742673f0,0x000290f2b86fdb82,0x000f44e50e9b7e10,0x000ecc65826079a8,0x0004d12b589a9228,0x0001b683a119d1ab,0x0000000000000001}, {0x00025950f93cc637,0x000b7a6b02229a2d,0x000fb0617d538c46,0x0001dd7b6bc581dc,0x0005b6b522d590ce,0x000133e3f5d0dcdf,0x0005bdf5cce47e79,0x000e21b8ecd0d71f,0x000ac21b717d9aef,0x000b887b6090257a,0x0000000000000000}}, +}, +{/* digit=110 [{1,2,3,..,}]*([2^440]*G) */ + {{0x0009fcba235d8a1e,0x000ca359a63f98f8,0x000f60025c086d07,0x00018d4e590915cb,0x000c915cc7c3b68c,0x000933480185575e,0x000511ae8d10d820,0x000082704b904789,0x000a4e997db2e76c,0x0008c3f5824d99f6,0x0000000000000001}, {0x00053628d8f32dc9,0x00000ea757555069,0x0008537e14185044,0x0007f7a0609d8295,0x000c55da70118c7b,0x0009ad1223c50379,0x0008629c936f6ea7,0x00054f7f839cbde4,0x000f8def61ba0172,0x000f5c1bef09ebdf,0x0000000000000001}}, + {{0x0002eeedc5fe3f41,0x0004f9330d665ae8,0x0003f5e64a30753a,0x000e92f39e477096,0x000aa07f9d297ef9,0x00048c3ddf388062,0x000e61e60ab0df5c,0x00095a47567e55e6,0x00066d0129872a6f,0x000953425f368c3a,0x0000000000000001}, {0x000b7cc7bf66ffa4,0x000f16b2825eba03,0x00090e67535ba3ce,0x0004aef14aec5704,0x00001511ac67bcc3,0x0001002739d95c0e,0x00029220f4f36575,0x0001465557ab45e9,0x0009abecf1baabf9,0x000fe98337c9760e,0x0000000000000000}}, + {{0x000025751d2b325a,0x000cde6a01039da1,0x0005ba8462228499,0x0003490747232500,0x0001a523417ab4da,0x0003451baf54b07c,0x000516f3fa7e4ffd,0x00042fbff2147ce5,0x0003f1b0afc522cc,0x000cb395c7010ca3,0x0000000000000001}, {0x0005ed5f55af51c6,0x00015b980e568466,0x000a5a1b30bd5964,0x000bb04e8834a37b,0x000becf282494fee,0x00040dc6ceb29d17,0x0004a868d5399a53,0x000276012bcea50f,0x000c769aa83faa31,0x000d8e6550a0654b,0x0000000000000000}}, + {{0x00066fa0ef4dbc14,0x00071d134a53f7ec,0x0001ee39cdaa7b28,0x0009b3f183070c04,0x000c76da77991974,0x000916f1eb867841,0x000b27421e5438ae,0x00051b0e12d8e409,0x0005e08b83842a6e,0x00094374b9e008de,0x0000000000000000}, {0x000a4cba763a6340,0x00013308e07acdfe,0x000db2143a906789,0x000fe6dd815c887a,0x000e2a9d2043c85f,0x0003ceab79a68d05,0x000986393674d33d,0x000d12ee73ca7a8a,0x00003b9bbd54b7af,0x000b702eead11264,0x0000000000000000}}, + {{0x0000b987f57f80d5,0x000d4367c06145fa,0x00034438e79fd55d,0x0000f9f8e8ca9c52,0x00036810f12c2fad,0x0003ec5af1a97a71,0x00085613d0eababb,0x000aecb3da01b7b5,0x0000150798aadf26,0x0008029cce9cadbb,0x0000000000000000}, {0x0005a72e54383960,0x000decd025e95126,0x0005b2c914390e3d,0x000864784955e972,0x0008cdae63ed0053,0x000aa5ded860c28f,0x0001dd80fb99e774,0x00020f07854bb74c,0x00091581c2caae0f,0x000d48069f6ba766,0x0000000000000000}}, + {{0x000f135836126647,0x0004208b738df6f5,0x000786c7341e91f4,0x00084ed91ae2188a,0x0000b08e3293bda3,0x000e09af3119b1a0,0x0003662652676669,0x000fd07b9f37322f,0x000ae129d764ea40,0x000b85c16a911b11,0x0000000000000001}, {0x000021aec95ad18b,0x00080eeb3197c759,0x000dfd4a4334daed,0x000ff78d606234ad,0x0003ff98a1d73ab3,0x000c9cac669f90a4,0x00063cf99bed1762,0x00028f03fcd45e80,0x000b17bf750672a2,0x000879027e080bfd,0x0000000000000001}}, + {{0x0006a647cfcf8e23,0x000d745dafe04723,0x000c3212b4d30081,0x000945780f548f13,0x000a0d885e14f51f,0x0006ed3092941059,0x000651e189c478f0,0x00007a26e8c75042,0x000a14b52f36b6ee,0x0001a032dfdec009,0x0000000000000000}, {0x0008ca67379eb582,0x000b6f527f0a50be,0x000adaba76e4c6be,0x0009b987fc7fd1fd,0x000a047c90091990,0x0006d6f45b992155,0x000e37e2da697e00,0x00011a38bcd1740d,0x0009b93e8ce3867f,0x000b60503be8b250,0x0000000000000001}}, + {{0x000c16ca71ab9de2,0x000d4b4d3bbd16e8,0x00053785c45519f4,0x000fab776454947e,0x00019d9b9416f1aa,0x000337e34e6883b4,0x00041570208ba8ab,0x000cab67774a7e58,0x0008ac5165a84d18,0x0003a977b69d3110,0x0000000000000000}, {0x000e5bcfd652943f,0x000743cd5e892a91,0x000502744c85aa27,0x000bb91ba0414bf5,0x000f8bc4ef773e26,0x000f9e301b8bcd45,0x00028983589038c8,0x00019a5f5e5a30d4,0x0005c6671a609f77,0x000a3aade09eb4af,0x0000000000000001}}, +}, +{/* digit=111 [{1,2,3,..,}]*([2^444]*G) */ + {{0x0008fa162e0dbf60,0x0003f486c08a442e,0x000d94f9cd2b15f8,0x000f2a2350bb6a89,0x0005b606cc5727bb,0x00003f198a74b132,0x0001a6f73b79d3ab,0x0001c95046a9cf73,0x0005ed71e298efd1,0x000274d622bb2409,0x0000000000000001}, {0x000c383b3b59eae1,0x000e81b2f1927508,0x0000d888be6e14de,0x000fb612ce4b12e8,0x0001378248f53213,0x000330dbca43092c,0x0009ef5e40c52b24,0x000e9d869889952b,0x000c05f4131f1126,0x000503fd03ae1dfb,0x0000000000000000}}, + {{0x000868b8bbdfd1e3,0x00069e244e266c6e,0x0007c7bf40b05a43,0x0003b7e1e296776b,0x000e22cfd9c18aed,0x000ea90d63ddbc31,0x000929198abb7bc1,0x0005791f36a5e50b,0x000737c71c2f87e5,0x00024f75c6d8e7fe,0x0000000000000000}, {0x0004c9eb7d596ad7,0x0001e1a0fb486ad5,0x000d820f02c91d1e,0x000160179160a67f,0x00057f1163f25e5d,0x000cbc9a928b61c5,0x000df9b84ed79f2f,0x000854ba69556a33,0x000c5cb8ac8febe1,0x0009a7ec5a344343,0x0000000000000001}}, + {{0x000a1f2d5d90dce2,0x00048086d7fdecb8,0x00005e3d618ccc17,0x00081c1e93d5b7f5,0x000a9cc469c44a02,0x000561bd59d9df3b,0x000788578aa5fba5,0x000f6a295757cdc7,0x000387a16bd50e1c,0x00029d1a6695b773,0x0000000000000001}, {0x000b40770bcc13d9,0x000a0b4cb7c0701f,0x00058a0433c71ad0,0x000e93cd5406bd3d,0x00098c999eda26c3,0x000092f416481951,0x00047a1b9bfd4b10,0x000f450e0ae46263,0x000aa14d201a717d,0x000a1f0cadae7402,0x0000000000000000}}, + {{0x00025a8ecef36f15,0x0000495828c615a7,0x000ac113424f603a,0x00042c1687a77e81,0x00098761c2762346,0x0009b1a4749d0db2,0x00097828ac3391f2,0x0005050c5b69143a,0x00078b0a25ae0092,0x000d6c144730edc5,0x0000000000000001}, {0x000bfa3844e6437c,0x000f55606aeb933b,0x00097e413566e3da,0x0001c1ae4263527e,0x000823a0378934ac,0x0002143f588c3363,0x00027262bb7d997b,0x00082419935b6941,0x0004eeef99cb555c,0x0009b82ef7f7cb72,0x0000000000000000}}, + {{0x000f0062dfac9c48,0x00002d81a18ce131,0x000ddcf23a0713ab,0x000a1fcdc4c43fef,0x0007bce6f244518e,0x000ad87348b34742,0x0006967b2b850180,0x00045248a8e5aac5,0x000029c18af90606,0x000027d44e26ac08,0x0000000000000001}, {0x000cb47b7d6fa01c,0x0000fa06669eb12a,0x000ba3426d87d938,0x000cc362d25b1811,0x0005c938c9a5e9c2,0x000513be5311d61f,0x000c741f99e49306,0x000858e6ea44630f,0x00001034df68ea15,0x0009fda601eea831,0x0000000000000000}}, + {{0x0001335ca463c947,0x000056d6526151f8,0x0003c494f0f999ff,0x00082dc10a7433ca,0x000067fa3bcdc41b,0x000c803bb39af11e,0x00031fc0ac7bb35e,0x0005b5e185aa457b,0x0006ab2cbed55591,0x000c973304481958,0x0000000000000000}, {0x000c07b3b2112108,0x00057bae813666bf,0x0008eee1f424f0a9,0x0002122582cf0958,0x000314daeb7bdc33,0x0004de4e23458ec0,0x000a768d0f97884e,0x000a1655c201fc50,0x000f1a537b424f36,0x000a6c0cdff48114,0x0000000000000001}}, + {{0x0007e2caf6bd29ac,0x0003376357d313a7,0x0007372bec99c462,0x000dae50c449ad64,0x000a9851fff00473,0x00047fec5d20efdd,0x000826de033e859f,0x000f46311785804a,0x000104e3184caf5e,0x0001ecedcc25e31e,0x0000000000000001}, {0x000e47c0beec6da0,0x0001b1920715db74,0x000ae32e6188b124,0x000b5dfc6ee77c87,0x0006b132638e4040,0x000b86b665e1c6f4,0x00028fe14b060006,0x000b71efffb869b7,0x0006eab86370f188,0x0003e80379ec88c1,0x0000000000000001}}, + {{0x000f8cb24f3b637b,0x00076af131c20322,0x000815ccfff0c2b0,0x000fbdbff056364e,0x00060c8028853dbd,0x000af0ee0841ab57,0x000da56ca93ac088,0x000d301350923094,0x000228a255054010,0x000a058e7dde6774,0x0000000000000000}, {0x0005176aa5c512c3,0x00020d3779fd868a,0x0007658fb3dbe164,0x000cf13041f45c5c,0x0005049dea64d110,0x0003f6746e19e0e3,0x000a39087ca45757,0x0008108ab4e2fe7d,0x0009cce4e874c545,0x000b791d64965ce3,0x0000000000000001}}, +}, +{/* digit=112 [{1,2,3,..,}]*([2^448]*G) */ + {{0x0001f7c72b336855,0x000f76e247b483cf,0x000202781dc97b6b,0x0005bb58c0f81747,0x0000c92efba888b6,0x0009612af59611a6,0x000ccbeaf54a57cd,0x000ef8689ba520d7,0x00091cc366d3cbf9,0x000056dc1abfe905,0x0000000000000000}, {0x000a04beccd53894,0x00021b1e600b02d4,0x0006c3ebe8f2a150,0x00007cfe9586be60,0x00062f4028af5b85,0x000d392e8954dda7,0x000dadc519d37584,0x000b58c3813ebde8,0x000557ce681db641,0x00006323fa3b9991,0x0000000000000000}}, + {{0x000327209a17e11d,0x000d294da0ba85ba,0x0002e3b7145fac1e,0x000fef1248cf218d,0x00034de112f175cb,0x00094a8f1676f3e2,0x000f9c2657870861,0x000518958d56de1a,0x0003dbcba495c76a,0x000cbfa5e9c9c9dd,0x0000000000000000}, {0x000fcebaff9f1e95,0x00070930a1b712b1,0x0008296f1f273d82,0x00071eddfa6e1f41,0x000af7dd190815ef,0x000f6fda9bc4a2f8,0x000482585b1234b2,0x000e23556036541a,0x0001ac1cc79e6b22,0x00079988ea71f991,0x0000000000000000}}, + {{0x000a9c0f0a888100,0x0007d24e7957d723,0x000347cf9592bff9,0x000365bb77516430,0x000532639eeab522,0x00075ffdcf80bee0,0x000715b06a25a527,0x000f841be2cacf87,0x000bc301ef1a4c9c,0x000be09159817e68,0x0000000000000000}, {0x000ae077a00f18c6,0x00066d4baba09883,0x00026a600e57c245,0x000d614baadec7ac,0x000df9f1c7a24538,0x0006e61c4241b694,0x000e25d7cba200ff,0x0002e6c811de028a,0x000a7a2ca09bc137,0x0004b214e3d38684,0x0000000000000000}}, + {{0x0002cc5d8d86efa4,0x00086c8ee779a0a2,0x000fd2159545dd5d,0x000c7262fd5e2c81,0x0008275f13cf7ab1,0x000759a0b74f36ad,0x0003c0c8c3ddc91e,0x000d10948a51d222,0x0007160cf9b2c7f7,0x00060f285822b597,0x0000000000000001}, {0x0001e962392851c3,0x000b50d7c127ef1a,0x000d984c5287e5e2,0x0005ce7d3999dfdf,0x0006fd1373907aad,0x0007f8f0825c8472,0x000b5c55ebbc32d9,0x0002bd51b3a068dc,0x000935287a1b4f59,0x0006c8f3eb9dca36,0x0000000000000000}}, + {{0x000e20e45266c880,0x000d20d089ee45a2,0x000924531cdc6e8e,0x000f7ce2e1fa4436,0x0002517f9a18e676,0x000d570398bec610,0x000dea4d0f9eb1c1,0x000c9914ddf30f65,0x000cb21d0d421e9a,0x0002d38fb5fd0304,0x0000000000000000}, {0x000db46cd0b507ab,0x000ba53f3ce83b8d,0x000a8aca81c50a03,0x000245a83198cc38,0x0008bb4ffb6faabe,0x000c93601934e1ba,0x000061b676968674,0x000710e9ebcf5033,0x000185f4475389bc,0x000687b9bd9763f5,0x0000000000000001}}, + {{0x00029514447f1de3,0x000d950771604321,0x0003a8cfdfffb078,0x00022ba412e08811,0x00097ef9f248a108,0x000b82c1196accaa,0x0001f8d1787041bc,0x000ec8430565e6d4,0x000eacea7ff3e1c3,0x000896b9bec2206b,0x0000000000000001}, {0x000508946b504e8a,0x00067cc71474da28,0x0009ad4094d373bd,0x00068f87798411db,0x000b0b90d5d507cc,0x0000617d88846f61,0x000a437ea42aac2f,0x00012a4ea205a926,0x000adcb25d613e80,0x000fb7ef615aab09,0x0000000000000000}}, + {{0x000127afd82f8260,0x000f4ea3f09c69bf,0x000b49f3f0e43a3e,0x00052457b8b9c3f9,0x0004fe6ce00409f0,0x00044efa3e31f441,0x000e3e0240cf1253,0x0001f0b4a170983e,0x000d166b0cc9e8c1,0x000b455417a2207f,0x0000000000000000}, {0x0000fd522cdf167d,0x000b48039daf9a36,0x0007cdbcd7341cc4,0x000c7db3b7805076,0x000641b0db786edf,0x0006ea9b69b8bccc,0x000eded2a4e25e3e,0x000651b8f0911cf0,0x000614b601e95ba9,0x0005000aeb19e877,0x0000000000000001}}, + {{0x000c0c51d56f967b,0x0008dddbb7684495,0x00035dbc45f7bb76,0x000f9e6deda49098,0x000e73639006a39d,0x000878e5a247f77e,0x000cd83d141b2c8d,0x000804a47e332c8c,0x0009dc7a02d4027f,0x000b1b9934bb002c,0x0000000000000001}, {0x000777a838efe004,0x000368d9919c1d8d,0x0009dd721650f685,0x0002b1de892863f1,0x000d78f2b25a32a9,0x0002a4320690ff3d,0x00005a4af7bb8bc1,0x000cd763efcfe035,0x000701c70cf4f256,0x000443ef26775353,0x0000000000000000}}, +}, +{/* digit=113 [{1,2,3,..,}]*([2^452]*G) */ + {{0x000e0800cd422057,0x000077d1b7504d49,0x00004fd80e159ebe,0x000a18a8714afb4f,0x000de28810d8b90e,0x00019cff83f02c3c,0x000eb9e19367a867,0x0009952bac438786,0x0000e0748ecceb4e,0x000faa55aefa6646,0x0000000000000000}, {0x000aa315b1f2623e,0x0002b544f96e097a,0x0005dae237e7a5db,0x000873d0a9362519,0x0005569799223163,0x0001a58ea84d0fbf,0x000d43fd3bb728ac,0x000f100cfe43661e,0x0004f55c6f1cd21a,0x00091825dcbe9fa2,0x0000000000000000}}, + {{0x000fc794a5858d75,0x000a4affcd84d641,0x0005082ece4f5985,0x000b4853cf3bb3f4,0x0000bb1d8af65850,0x000953dc3e670d98,0x000424ef579458a6,0x0008ac2f2e4a7963,0x0000d771e540b685,0x0009be1f5fed2292,0x0000000000000001}, {0x0001be223c4864af,0x000fbb662c4dc573,0x000a57017521419c,0x000c0240d65099ca,0x000b13af88f3bfbc,0x000bc4861e1643ac,0x00067ed67405bcdf,0x000d9351f1c835f0,0x000d0e188cb8018e,0x0001f8d276f971ee,0x0000000000000000}}, + {{0x0006a9e449621f3d,0x00052e91ee418d60,0x0007e5ea0e7d2dd0,0x0006f73c92e787f3,0x000cc4508ea4869d,0x0002d461acfe248e,0x000bd24529ffcc1a,0x0002f90dc5dc22dc,0x0002f8abba336456,0x00056cd254e4f954,0x0000000000000001}, {0x0000aa036c0262fa,0x0002edc4f4234a99,0x0001031cef9b59ee,0x000145d5d5d4b081,0x00087df2c037b984,0x000a2d6af2c7b077,0x0006ff431d568532,0x000e309a7c9b6d5e,0x0006a3ee9bb40c66,0x000c81af9db41cda,0x0000000000000000}}, + {{0x0000f6f067a194c8,0x00066d7ad7abcf3a,0x00041cf832b531ca,0x000f470a4ac4965a,0x000cbe00766a00f9,0x000432af80a92657,0x0008968ac40c892b,0x000cbd44ededa94b,0x0008d46204be8b74,0x0005dd98f760bd2a,0x0000000000000001}, {0x0007f464f2479db1,0x000c0fdb3e7dcb2a,0x0001a96b28900b58,0x0000a2993c1d7ee5,0x00057bf0ce9357a3,0x0001f5d39eb49f5f,0x000fb8b8f0c1970f,0x00060718d4faac9c,0x000ec4ed9e1c25a3,0x00004bb0d7504b66,0x0000000000000000}}, + {{0x000c9e1b2fe9b1f5,0x0008d34221feb754,0x00042a589958e706,0x00008c19119e3c31,0x00021ac738645a87,0x0004d7be72a33383,0x000a72a39907dc71,0x000beb5759bb91f8,0x0009abd4b13e9374,0x000b96398db67d76,0x0000000000000000}, {0x0000945a75d3799d,0x00031dd530ad32ad,0x0000c605c645aa0c,0x000c247141d0f230,0x000876cc8fca8350,0x000943188b899279,0x0001484afb9ffa8c,0x000536d0aa571f07,0x0002b7a0c7fbe8ba,0x000c736f36c17e51,0x0000000000000000}}, + {{0x000618a7bb29e0e4,0x000c09c2bff10aed,0x000781dbe57f3993,0x000965b0e2b9baeb,0x00005cf5dc5bd4b3,0x000ce5975c6e0f92,0x000471ab0fcf2bf8,0x0002a2d43bae9f5d,0x000bcafa18f22cc7,0x0005911e3b357902,0x0000000000000001}, {0x00057df200a9fe3a,0x000b50d956c845fd,0x0007f211bf361887,0x0009b4007c382ac0,0x000ee4f517e7f4f0,0x0008256df5e905bd,0x000fa86a9368d1c3,0x000a7df56d37c426,0x000efab6ec26a5b8,0x0005cdfe9cfac9c1,0x0000000000000001}}, + {{0x00082202f99826cc,0x0004867eca10dd40,0x00071f5c4e281cfc,0x0007b5da880b8221,0x000325e5cd3e67f8,0x000671c0e0906564,0x000df4bd6ef65d7c,0x000c8a693695da96,0x00076e79dcdb1aea,0x0005bd96f080eaaf,0x0000000000000000}, {0x000240c6e69e5a49,0x00080c138884ae0a,0x0002f7e55e87844e,0x000a57d17ee0d45e,0x00033a1990475a82,0x0004fc5139e3efb1,0x000093a8d1fcb820,0x0007e2fdf23f8eaf,0x000aec2eac127538,0x000a0742581e77ed,0x0000000000000001}}, + {{0x000de604b9db6d8d,0x000262da62b65572,0x0009db8d0d3fb7d9,0x00067b7f8e9c2aa3,0x00074f2912d3c9b8,0x00079e6d831a5ad6,0x0006f3cc6935b1c2,0x00009a75e08b8223,0x000aaa28cfcf8f6f,0x0004f75ff407271b,0x0000000000000000}, {0x00024706c120a90c,0x000b4f991d9aa4ab,0x000767e06952ed85,0x00006ffc0793e3c1,0x0003d6115d975f7d,0x000c57472c131644,0x00024430b8651df9,0x000bda1a64f28b97,0x0006db846f1bbd8d,0x0002f562aea1650c,0x0000000000000000}}, +}, +{/* digit=114 [{1,2,3,..,}]*([2^456]*G) */ + {{0x000d810c86eab937,0x0008e6e6eb51bdd5,0x0009bc2e3a0e4219,0x0007e391c4b3dc48,0x000736ed77fe5eda,0x0005e60972cd0e3d,0x0001aaa05f70f41c,0x000db07669f426a5,0x0005914839830a47,0x00029045b98cf434,0x0000000000000001}, {0x0009d9c576932524,0x000409f4c3b8bddc,0x000d467dc9295086,0x000bdef23e6cf0fa,0x000729684c1e0fbe,0x000c3a23011daa3a,0x0008c4ef40ca0da8,0x00034dda12c0850f,0x0002c5e51990ccbe,0x000494c2f0adaf8f,0x0000000000000000}}, + {{0x00087b36a390065e,0x000c93a9bb0064f8,0x0004572329d651f2,0x00010e01d988aed3,0x000be48541e9e6f5,0x0000ac10a8abc023,0x000b700a8621efc9,0x0003eb208400f943,0x000d85b1c768bb3b,0x000f42634af0db64,0x0000000000000000}, {0x000e0a10a250b4be,0x0000634e42e593a7,0x000adef0026acaa8,0x000002da6f2f96cb,0x0008dca66aa2e955,0x0005a69e8157271d,0x000f32666dc76291,0x000c378977ddcf29,0x0000e5eaa07d6619,0x0002a548e47a94e1,0x0000000000000000}}, + {{0x000cd510985c6ead,0x000d1cc399a1876d,0x000fc77243f5c966,0x000c3b2c4abf82c9,0x0001efaf22c713ae,0x000a22e1704988df,0x0002a2d8f28a287a,0x0000f724ea967d19,0x000ed48e76179ade,0x000ff318acdc5b8c,0x0000000000000001}, {0x00063196ac8ad685,0x0002708e70052b0c,0x000d8ff45f4a08be,0x00064862e9bd37fa,0x000efcc39748e461,0x000dcfa2843cc067,0x0006ae8688367317,0x000b8aabfd38c458,0x0005642ea85ecef5,0x000873af78e84b81,0x0000000000000001}}, + {{0x000b763fd4a23b33,0x00004ce905f61100,0x000ad5b3ffa31c96,0x000817bf059b27d0,0x0004f957d997fe85,0x0008adabfcc9cea6,0x0002fa1bf24cb58a,0x000afe218a3174dd,0x000ee69ca08cb0de,0x000fdf310fed00cf,0x0000000000000000}, {0x0006e131eb000160,0x000a4d18c779a4d6,0x00013180dd747a1f,0x0009340c23f27ad3,0x00004df4a2f35316,0x000ec77b35a8c2be,0x0007fb23a1f8aa1a,0x0009e69edc272eed,0x000d58ddb6110abc,0x00042ae7590226a5,0x0000000000000000}}, + {{0x000f3cc8ae818a77,0x000f54ede150dbf6,0x000c2f06bc371295,0x0006ab2d173c2266,0x0007fda0b8b46bd2,0x00070909e3958aad,0x0005242beb035184,0x00006b5aad800c13,0x0003a70dce6b782b,0x0007761ebe42a4f4,0x0000000000000001}, {0x000ee87f40792877,0x00080b5c7acfe8e9,0x0007c5775efcb053,0x0007bf0a2d540e71,0x0008c839d644d450,0x000d1ff451c6b81a,0x0001b7ea45f8834b,0x000306bfc9c3c653,0x000cabe92f1a607c,0x000c13152a3731fa,0x0000000000000001}}, + {{0x000ad1ff805361f9,0x0003b3536327ce4d,0x0005b0b62672d1bf,0x000ddf3847367af0,0x000278798d158f13,0x0004fbc252a948f1,0x0006fc03d0fe92b8,0x000c0138676b978e,0x000ea4ba8ef9334e,0x000f98b13cf22496,0x0000000000000001}, {0x000ac5c627d693fd,0x00015f19d6d21fa8,0x000b4a2fb70b6705,0x0002c39928de441a,0x000db0fd9e912476,0x00096888c13371fc,0x00013f185fb68ee8,0x00092d86c189d0e2,0x0001cf6facea1f84,0x0005b949b94eff15,0x0000000000000000}}, + {{0x0003e85508c930d5,0x00050167c798885c,0x00018ef2850f4f27,0x000547f00d7f6b01,0x000899cb15ed2f40,0x000bc417a989d6f1,0x00008165c9378941,0x000bb0e0c28f2218,0x000839d9572ddfe6,0x000d6ab0b70e2e95,0x0000000000000001}, {0x00092c31b5a755fc,0x0005623261be8d00,0x0002ed776ec81547,0x000ed7b92bc72da3,0x0006d943fbdb47af,0x00096c45167b5a5d,0x0006fef44f208158,0x000d5bd5c28e23f0,0x000da432e8f4f6c6,0x0007b3355da25eae,0x0000000000000001}}, + {{0x00076c8ab8c349a0,0x000850693d15749f,0x000d18a6991254fd,0x000f60f54944e4ef,0x000febc73879cf78,0x0004c63e00125696,0x00042f68e1e2dbe0,0x000b688fd93a88e1,0x000f1e83abbe7321,0x0003aaa7b2fa13d6,0x0000000000000001}, {0x000caa293a9d97c5,0x00059519ff3b97cb,0x000cedb5893b39ad,0x00064ed07e59369f,0x0003315852af3473,0x000d5a40d47c32e9,0x000386582768fe34,0x000453f8e4c60653,0x0004a96dcf43bf2b,0x000e332542e1828b,0x0000000000000000}}, +}, +{/* digit=115 [{1,2,3,..,}]*([2^460]*G) */ + {{0x00011a7aae3fec0f,0x00081883626ec806,0x000aebe504491b56,0x00031dd2ecc113fd,0x0001068171e0f3fc,0x000d2fbb6b5fdadf,0x0000ee0e94a492b4,0x00090723f06e06c2,0x00016b906e2fed2d,0x00033b2f32d5d083,0x0000000000000001}, {0x000731c43f27e685,0x000c912924bed063,0x00058df5bd8f8996,0x00080e3a04d16a64,0x000303ff6f14cb47,0x000f56c8175aed03,0x0003011b62e0f3ec,0x000eee8bd1d8f816,0x000a055be5c28fa8,0x0005635edb8d9c9f,0x0000000000000001}}, + {{0x000eb4c2e9d6a36d,0x000f1c0e66e6ba0d,0x000058a747cb2451,0x0004b10a20962d66,0x000e1da104e82021,0x0004693d32e594ca,0x0003bb7f837609cb,0x000e53eda7c5059d,0x000b602751dd16ca,0x00007ac67ede2a3e,0x0000000000000000}, {0x00038202cc49b145,0x00003535c208aec8,0x00056079145b2fb2,0x000814d455be9713,0x00082fd8f0bb395c,0x000c755426c09f67,0x0000b748edafadbc,0x000deaf4bc3a4ecf,0x000bff049553943e,0x00057ee542c407a2,0x0000000000000000}}, + {{0x00058193376e77cb,0x000e72c0bba4380d,0x000bc54c3a89ddc3,0x000dd63a1bbc6d0f,0x000bf1518f660f2c,0x0007e5bf5faf2f62,0x0004682bfe7727a2,0x000da33defeb7b16,0x00032b5ef40ec257,0x000a0e902b2e8961,0x0000000000000000}, {0x000ab468387524e3,0x00024e69a88271e3,0x0004545479e82998,0x000121a7373761e0,0x00093e3b1f753397,0x000463acb40aaebe,0x000cb721af707dad,0x000c14e152331d6d,0x00065553048e5280,0x000612aaad009d91,0x0000000000000001}}, + {{0x00092b39f964e28f,0x000fa7cfd7976897,0x000f279c07ec556d,0x0001cf40d7670e8c,0x0007b5e04abdedad,0x0006fcc19990b137,0x0008572f5067ad94,0x000a28101c966a08,0x000b5e33c2ad58ac,0x000eb0e333e24c43,0x0000000000000001}, {0x000f0361a475ac89,0x0002dcf79463ef54,0x000dd053538c22ce,0x0001013b4e6817ca,0x000b44b01a6e12bc,0x000d109d85844a6e,0x000b985bfdaef54a,0x000f6830be544481,0x000fd8dd0297f121,0x0007cb6d0b67a68b,0x0000000000000001}}, + {{0x000ba65ede23b338,0x00026845dcbcdc45,0x0000a0b1cdddd83e,0x000092ea63968b4f,0x0004b71e6e72d35e,0x00046c5ed03f1ddc,0x0006efc5a166bbfa,0x00090de0b5f6b7c7,0x000445136c1387b7,0x000ba78923450ab6,0x0000000000000000}, {0x000a850194005e74,0x0009c00cae44ea99,0x000e5e17b631e4af,0x0001c80d51c0dae8,0x000120c3cbe08ee9,0x00023c8041a40936,0x000ada73446b0eda,0x000643d14026f215,0x0006fbac37813fb6,0x000fc70031b68bb2,0x0000000000000000}}, + {{0x00075ab901660b40,0x0001e645ee7e0ee2,0x00037b06a37399f2,0x00080496e9bd12b1,0x00026a5d50d58960,0x000c1e3f3705ba3f,0x000274a1d4a00817,0x000d2d00866a4d39,0x000f146bf0317c40,0x00024e2ec71ea064,0x0000000000000001}, {0x0003c199f5c7f563,0x00062e4f78f16893,0x000a194ad2a3fb13,0x000080225259655c,0x000bc5898f9da5f4,0x000553edffd8a6c1,0x000e20b2793c4797,0x00041ea73083e26d,0x000a2971a533d937,0x0000e54fd22035b2,0x0000000000000000}}, + {{0x000f9d4de9a27585,0x000400debb5987f8,0x000d526ec09e3156,0x0008bd03e0023f66,0x000aed7d715cc557,0x00058cc9c03099f2,0x000506b4417ca0c8,0x000d23b4df572ea5,0x000c5dc147420ffb,0x00032899652bdfe0,0x0000000000000000}, {0x00031987cf8e9148,0x000b259461ad7d0d,0x000c859a4a7e6bba,0x00030e2b3d2a289c,0x000e9be629139087,0x0003904cf1cf6e14,0x000cca9dab045a7c,0x00082a43de3eacb2,0x000d82c3afa439f6,0x000decd187ae70d4,0x0000000000000001}}, + {{0x00087b4497808581,0x00012be293d33455,0x0004ce4906c6f8ad,0x000a66938d521bfa,0x00046a914bdb3b26,0x000ae5f6e99dc3e7,0x00059dbb0881c2e3,0x00009191a1b5ac25,0x000c6a97a72e5343,0x000ddd07226ad4a6,0x0000000000000000}, {0x00039e249234483b,0x000669419af2063a,0x00042122752c72bc,0x000aa7e19a44c7a3,0x00011188ac573c28,0x0009a3360e14ec6b,0x000bc0bc86245880,0x000741e192993af7,0x000d8d75742bd481,0x0006cd8768555ba8,0x0000000000000000}}, +}, +{/* digit=116 [{1,2,3,..,}]*([2^464]*G) */ + {{0x000ebaeb9266fcb3,0x0006c49166afc8c7,0x000b1a4fb9f8df09,0x0003a2759ef63e0b,0x000ded0e62d1d0a6,0x000215cb79a13c16,0x000942482d5b46ef,0x000a5cf390334543,0x00039ce21c9b239a,0x0000a2fcf03ed34a,0x0000000000000000}, {0x000466a8adf517f0,0x000775523be0b6d8,0x00074759167493a7,0x000284c64894bb12,0x000e2864e9ca25e2,0x0008b7f98fd07d26,0x0003fdb6d6620610,0x000ef64b5a668e1e,0x000d31c44a0ba6ca,0x000891dac14a11ed,0x0000000000000001}}, + {{0x0007e005c5b2f805,0x00010b6d99c24dd3,0x000813da1409fa02,0x0008bf13d53cbcd5,0x0005bb6d8655f948,0x00021f224e5b2d05,0x00077dd3ba305b4b,0x00005337f568059a,0x0008b4b1e783aa9f,0x000d6ce8c56442b8,0x0000000000000001}, {0x0007e0acb71f23b1,0x0007f2e0e90fde9b,0x000336f8ff1da186,0x0004751614e3d072,0x0003187e51c7e8e6,0x0007ef17101ca72a,0x0008a9761c42d892,0x000ebb69cc0c641d,0x0002903e96138250,0x0006b8d2873a54c1,0x0000000000000000}}, + {{0x0007b97b6b68155f,0x000799e78b707b09,0x000e1f75bd19e80f,0x00035096285e6628,0x000d9b8661ae0f1a,0x000fe90a8911dc8a,0x00018180a1d7fb12,0x000733dbb76ae258,0x000085cc47bdbf1a,0x0008c417eada4711,0x0000000000000001}, {0x0006498715c0f18b,0x0008cd3093549e44,0x0007888a48d3a384,0x000a0d971c9394dd,0x000f79cc4aa385aa,0x000d44729501e42f,0x0002ea4042d9ad6a,0x000c1e43ab081b30,0x0002a374ef11901b,0x0009de3ad60e42b4,0x0000000000000000}}, + {{0x00023f47f06415e1,0x000ec51fe7219c05,0x000bd8a88f411a49,0x000308976713e8b2,0x0006ee0f84892d3f,0x000957e9fa410c61,0x000903d60b015584,0x000f41fc07f1fce0,0x000fa3ce182117ef,0x000654b039b5693f,0x0000000000000000}, {0x000f700d59c0d688,0x000bcc693fd9aa04,0x000b8b0e7fea0743,0x000182c181c35812,0x0008864896cc8fcb,0x000c77cf499f019f,0x00010bba6594c508,0x000e88406e142c41,0x000b45fd50fcaee7,0x0009894dc1ba3eb8,0x0000000000000000}}, + {{0x0007488ba3fe2367,0x000752106ad62ea4,0x000980bc62d3b8ad,0x000328ef733708df,0x000d47b00f88c069,0x000f6bb0176a2cf3,0x0009f6480ef1ba36,0x000ac0712001822e,0x00032f8aac418a68,0x000ff326047c12c9,0x0000000000000001}, {0x00039bbf3cc8d8c5,0x00091d5e4cf0789b,0x00042078d73c679f,0x000b71a650f7aeca,0x00082a530b74abff,0x000a71711bd8405b,0x000c3bb0ac95d510,0x00094cee94894b3c,0x000906c5e7b66990,0x00030a027cdba56d,0x0000000000000001}}, + {{0x000c446c47eef0c8,0x00044a0878421c07,0x0008722c55cf2755,0x000ec763424a48fb,0x000f4f6b5b3b9028,0x00078d4fe3ca8f7b,0x0003f4277d82e20f,0x000fbc6300a704e2,0x000a908b8f5f71bb,0x000b090bc8e8a53a,0x0000000000000001}, {0x000dcad6549fc8da,0x0004e635d31de3d0,0x0009ac9c9dae5fbc,0x0005d812525deba7,0x00028465a1ffb0b8,0x000039c002085422,0x000d1431962a343c,0x0001729577d460c9,0x000befcdb0fe4b63,0x000f982552806705,0x0000000000000000}}, + {{0x000c16470afeefa0,0x000c0df482969752,0x000988294a6b7345,0x000192c323f56fda,0x000ece866a0fbb9d,0x0003fb58e433eb87,0x00044e600bdf291f,0x000bc9a43f5fab4d,0x00060683951124ce,0x00065a7c641677f4,0x0000000000000001}, {0x000ce490f8bb04ea,0x0002e7ad3382ee72,0x000272231533e6b8,0x000186b13ffe5f10,0x000cc8faf0cc5e1a,0x000c709ed2b173b2,0x00067e514381962f,0x000cd58bc198455b,0x0006402864604346,0x0009d46f62db1463,0x0000000000000000}}, + {{0x0007bb607e896c28,0x000e0a894887a4bc,0x0006eb1e97614230,0x00003e71e2c653f8,0x00096dded494be93,0x0008ac95d09fc0dd,0x000bea563fba0619,0x0008f3c1624d738a,0x0009df64d4a0ea98,0x0001d5c6ae182338,0x0000000000000001}, {0x000eeeb900454516,0x0006377d7a8b0a7f,0x000a9c345a7de36c,0x000100d5611067e9,0x000806bcdcedd0a9,0x0002b5dec6f6c68c,0x00051f38d7d4a349,0x00022d5061b9ad36,0x0005c9ea7f739c0f,0x000ee734e6cedbd1,0x0000000000000001}}, +}, +{/* digit=117 [{1,2,3,..,}]*([2^468]*G) */ + {{0x000eb05013cb5aab,0x0009afca55bd420f,0x0009c3c71fd8695f,0x00000cafcc6e5ed1,0x000728edc89ca3d5,0x00076471854e21b8,0x000201f0ff872ac7,0x0003e23036d8ee45,0x0002c13f7c8bee8b,0x000707a1d51a1e5f,0x0000000000000000}, {0x00022b011bf68532,0x0000cf529ed8e280,0x000b8a477a52b6ff,0x000cd2f63b6e8238,0x0007c291c55c9cf5,0x000f4796ab42ab24,0x000c0989f93937b3,0x000bca7b47aa5dbf,0x0006f0a6b7620e79,0x000f58dd4ea00729,0x0000000000000000}}, + {{0x0004f87aac9cea1d,0x000fabffc3ba2342,0x000a3b2b167162b1,0x00073da5b86c7978,0x000769b5f991d83e,0x000cb3d9088d484c,0x0004542e085b4392,0x0004eba2ea8f2806,0x000c46cb82b91d2b,0x0008c0e83321f8cd,0x0000000000000000}, {0x00097601a9d31426,0x0009ef4a0c50dffd,0x000fc4b8056305e9,0x0000c1c8a29f6e86,0x000d4be1babedf5b,0x0005e98d4d558d2c,0x0007fd7d17d7bc87,0x00009a0a33b745e5,0x0008c2a2bc3cf9b6,0x0006235277b76d2d,0x0000000000000001}}, + {{0x000ad9eb395de05c,0x00050bddffa75224,0x0008d48f88a50e57,0x0005bffe3c2175d5,0x000aded74a44ab8f,0x000f36097483dc36,0x00041e5290fdaaf9,0x00051f0f28ee6aea,0x00006d6a082ff0ae,0x000068e3568c35a6,0x0000000000000000}, {0x00052b137174e56f,0x000a242808c3c521,0x000cee713ccf3a0d,0x0007f4523126e210,0x0009790a26049fad,0x000293cb237c8c65,0x000391754b82c700,0x0005060129b75ceb,0x000ad78c9cbc1be8,0x000452fab6d7a60a,0x0000000000000000}}, + {{0x000614e7a62b479f,0x00091b1806f1503e,0x000c937295d07735,0x000468f3b432690c,0x000027af2bc376d3,0x0001568b1fc765b5,0x00004c84508081cf,0x00093b08d2fa4f2d,0x000aa23530d43841,0x000eddd4118eb0fd,0x0000000000000000}, {0x00095d916a74bf7e,0x00052f879c30fb73,0x00033e906c3732a6,0x0009ecd6d707078b,0x000764fe7914feb0,0x000164429576e244,0x00070601ef70830f,0x00038d1a94c290ff,0x0007e067e8e38b39,0x00004d7b7a7e7934,0x0000000000000000}}, + {{0x00092dd8b3c6e87d,0x0006c49275954a31,0x00016b291ceab935,0x0002ae4fca80283b,0x0002ae3f769e974a,0x00047d8e3a1a96a2,0x000875bc17f080fc,0x000a5eabb84f45c1,0x000e2f2661df31c8,0x000663147f43e69b,0x0000000000000000}, {0x0002716571c12164,0x000b5580b0b11754,0x000bedf056c51a6e,0x00019d3572bd29fd,0x0006ce46ac6d5756,0x000d711d909ced7c,0x00056c0ec115b51f,0x0005ebe8c1ea4d63,0x0005d77cf7a0032e,0x000ddbec9919b1e6,0x0000000000000000}}, + {{0x0002437a972f2c78,0x00015b5e5010e549,0x000fb968e48079c6,0x000ee7041338af95,0x0007cad3cea2147a,0x00063a330c01aad8,0x000ecdd3002cfb58,0x000bf58e568343ac,0x0007567e97251d0a,0x000b568be5a71fbd,0x0000000000000001}, {0x0003982ec0a47e47,0x000862df20207b07,0x0002c656e08dda4d,0x0005ad59afe9aa6b,0x000ee04d8bf8d524,0x000d84cab42cc2b5,0x000ee837ff8c6d4a,0x0008b691af448525,0x0003b100c345fb36,0x0000d061abec1e80,0x0000000000000001}}, + {{0x000c9cc4477a0393,0x0002f0a9ecb165af,0x0008a80444bd4fb6,0x000a9d4dd001619b,0x0000b10c19a21051,0x000801b4ef3f9384,0x00076219bc4d71f3,0x000ea7190314526d,0x000da93e188e8697,0x0000967b67776acf,0x0000000000000000}, {0x0006ab2717f542c5,0x000ae82426abc0aa,0x000641af35023777,0x0005dcf6c7153a96,0x000977e0ee60ca55,0x000c371732ddcc64,0x000063579dd744f3,0x0004f34c496af4bc,0x00000f9df0536028,0x000b1c2a26167cf5,0x0000000000000000}}, + {{0x00061fdc0b1fc134,0x00059fe59da03f7e,0x00051831e7698bd3,0x0009f81df982fb68,0x0001c64253ce4407,0x00084a0c0fffbd0a,0x000fd2724ab08376,0x000aaefd7b900fa3,0x000017be08cd54b9,0x000359398932034f,0x0000000000000000}, {0x0008bfed571b358d,0x0006e5bd73f12c5f,0x0007574722cbd9b4,0x000789de672fc532,0x000cf9d4c5de795b,0x0008e00647313e84,0x00001e2002f19344,0x0007649a15d6dda4,0x000a4f04c96114ef,0x000ac34a9dc08537,0x0000000000000001}}, +}, +{/* digit=118 [{1,2,3,..,}]*([2^472]*G) */ + {{0x0005e5ed4ea72f88,0x000dc27eafbd5d75,0x0008274c8f222817,0x000e4b956ed11c56,0x0002ac506cd96a0b,0x000c56121cfca3c6,0x000e3c5160f64376,0x0001cd969d9794e1,0x0002818ba2a1b9ac,0x000bbd5d12cb946a,0x0000000000000001}, {0x000065269838f81f,0x000a256eaf242367,0x000743908912f4af,0x00048d332342e954,0x000f75565f855ec0,0x000e4ec59a3a8816,0x000505255b015d48,0x0006bf898ef06a71,0x000af90ae385313b,0x0001d5415dc8688b,0x0000000000000001}}, + {{0x000115486df6f2ba,0x000803b08c738eec,0x0002302443593e59,0x000e60eb4f00e934,0x000e57c91438ebdb,0x00080e89c523e859,0x000d75cba0053e05,0x0000d11317c9b329,0x00035e5703d38955,0x0003845aac6426d2,0x0000000000000000}, {0x000349105fa06b94,0x000e9b21c69e6c42,0x000f70eb1519a81d,0x000827ae2990539b,0x00048fe9bf0bbafc,0x0008dbb3c1ee9dd5,0x0002a908c4d4274b,0x000cb7224476f3be,0x00028346b6e6842b,0x0003ad39da0c7384,0x0000000000000001}}, + {{0x000a9a342fa007b1,0x00065415bb7a90ea,0x000dcc6aa0c26771,0x000a1f0bc720264a,0x000b5e93f7bc1986,0x000bf6b3fca182f9,0x0009a7b5c22f84c1,0x00087bdec7ad14eb,0x0008902c942c3b07,0x000a69d7973e7810,0x0000000000000000}, {0x000230ff44676602,0x000d3064b8821220,0x000d730a5228928d,0x000ea087c0f54e1a,0x0000abe8035de528,0x000d9e98d62188b8,0x000d85f12fe3f391,0x00095d1c13b3c4b4,0x000c9a4940436d0a,0x0007d2cc5f243602,0x0000000000000001}}, + {{0x000c484e2708d61a,0x000e14446b62d85c,0x0009c0fff451d84c,0x000a49ca08f1ae70,0x0003d71899e1ae53,0x000709a90d9917f9,0x000f39b12fbb0503,0x000038af72a2045e,0x000817cd90b8cc9a,0x0002f352b4ed83f0,0x0000000000000001}, {0x000a71d48ae023ef,0x000d8744e50d53a7,0x000192ec22667e7a,0x0002e19437867d3b,0x0007124825f0cce3,0x000aa41f07fce902,0x000fc674b8262125,0x000d1f602e03c9ce,0x000f93cc0c071ae6,0x0003fae4c52caef9,0x0000000000000001}}, + {{0x000d102ba58b5201,0x000cb021f49742da,0x00078984de6430a2,0x0005002098f1e552,0x000c9b804a37766b,0x00095479cd052eaa,0x00083f87dcee9cc2,0x000ce159b125b614,0x00001a384dc69331,0x0002e4ec5a2f8699,0x0000000000000001}, {0x00049fffc681e26f,0x00001b52ea90f687,0x00080117445f69ef,0x00093fb5f32e704c,0x000d36110b81abea,0x000a3e64f4ffada3,0x0000ff99dcba475c,0x0000eb1b968c2040,0x000ded29d4be59cb,0x000ba78061ed859d,0x0000000000000000}}, + {{0x0004949755c59603,0x000c796691a84904,0x000a8445d0aabdba,0x0002396cff6a7bd7,0x000562547f935def,0x0009038b91c344b0,0x0001f20274812ef2,0x000e1fe565e5087b,0x00074866fd902829,0x00032844aad6b8f8,0x0000000000000001}, {0x00092e9251d05a0d,0x000328420cd4a328,0x000e00dfce4ccaef,0x000beca3a94d516d,0x0000fce64bd11bb9,0x00054cb1e99e5269,0x000cfc1bfec9624a,0x0004e634fe2023b3,0x000e53481128c3b5,0x000cd2258147276f,0x0000000000000000}}, + {{0x000e9ad8d783865a,0x0008e5a58d54402a,0x000e236d64e3be62,0x000daf46f4aae16f,0x000049ba7000cd7f,0x000deb437c36e86b,0x00025e5a9179d63e,0x000fb30e5ec5adff,0x000fc4d691408156,0x000d8b4dff418608,0x0000000000000000}, {0x000c4eed479219ce,0x0008e4d5ead8e114,0x0005f4f71316535a,0x0001c636b70e9489,0x0008aa2310479eec,0x0004e1dcd4f2eb70,0x000474c99b5134b7,0x000ce4754e99035c,0x000218919b4c0dad,0x000e1d1dd3f97981,0x0000000000000001}}, + {{0x000cc18c358773b9,0x00036bf6385f12eb,0x0008a0c24ba8caf5,0x0003093cd83891ca,0x000f5b8c3762108b,0x00041e3399b26a0e,0x000fadfca8c426db,0x0004173bf6760263,0x000a6677ad40bf58,0x0004eca4760acdd8,0x0000000000000001}, {0x000c42b8b4207fa1,0x0006dc5a60d34efb,0x000a367e08d67868,0x0008cead3e942c85,0x000409d289bdc2dd,0x0004b034c3a6d1bb,0x000889304955940a,0x00080034f3684e43,0x000808a7cddeee0c,0x0001148f3d9aa263,0x0000000000000000}}, +}, +{/* digit=119 [{1,2,3,..,}]*([2^476]*G) */ + {{0x000df4a3fd3e1bab,0x0006287d84daca3d,0x000d7eaf57017e3e,0x000d1a4e4870b354,0x0004c26a3e3ca9fe,0x000ce1ea5e5710e0,0x000709e17a2ff920,0x0000f8a3bc06ee67,0x000788ab8c019a66,0x0006f43d909c0fdb,0x0000000000000001}, {0x000c0c61eebda5c3,0x000395c130704b51,0x000762ffbcb5d086,0x000f660bf6639983,0x0001646d9fb03337,0x00065cf06a8fa37c,0x0005f2e3f14b6d28,0x00088227d360e736,0x000a48fcdc5c3e58,0x00059e8c2eaf07b6,0x0000000000000001}}, + {{0x0001ce2b5f2dc50d,0x0007e39f8c4d01b0,0x00078d34284f417d,0x000d157acbf04214,0x000c0c4238071f59,0x000b0a1e05f8a594,0x000d81bbaf85cdc7,0x000d1d1329e8c9cf,0x00068fc55c9be4f2,0x00029b5c20884e31,0x0000000000000000}, {0x0009fd4109cb4727,0x00021d96d542276e,0x000d61e57db16c16,0x00052da58656adf3,0x00089d546ecce2da,0x00041508ee2098e0,0x00011997499c874b,0x0006f525839d9cf3,0x000de08e59654896,0x000e511cdd85c0a0,0x0000000000000000}}, + {{0x000db13610c4d993,0x000192018344e51f,0x000cb8a7e81016f0,0x000425ff1ca2c27e,0x00047a8df5318c36,0x0004872bcd56d5d2,0x000d142a2e0d2618,0x000a83feb22e4866,0x00013dac70999b14,0x000ed007863be6ab,0x0000000000000000}, {0x000023bbbd62b467,0x000e6ef8f48d21ce,0x000ea9c5f9c35940,0x0009af532bd76e0a,0x0000f8ff97911a1f,0x000efcff41750c50,0x0007cfa3985ad13d,0x000136812ef99c02,0x000319ee534694b3,0x000b9d9722dca85d,0x0000000000000001}}, + {{0x00040d25bac4c923,0x0005026132b9fa82,0x000ddc2ef3e74ec9,0x000151b0a9db4e16,0x000ff5ad95c1429d,0x0008144cde9bb57b,0x0000c02f2a19e480,0x0005655b0b6aef98,0x00038725b1f2df6c,0x000672346457ed21,0x0000000000000000}, {0x0003077ffe12bd18,0x000e682804b9bb8e,0x000b8a3a7328db75,0x0007b6f50cb1bbec,0x0007a823e8549b58,0x000d7be7a7e70575,0x0007103b60b8617d,0x0004131d7bc32367,0x000713f91128ac22,0x0000deadb3b9bf03,0x0000000000000000}}, + {{0x000b46eea58e4f6c,0x000e8500cef9c4a5,0x000e36179b50381a,0x000498cb317e7dbb,0x000df2d824ab9ac6,0x000aa97d96328707,0x00068fd80e79f5f6,0x000ee03799c5c193,0x000688d4b109d20b,0x000fae5dfd91a5a4,0x0000000000000001}, {0x000e4aa8649aaa1b,0x000caf8a4a894ffc,0x000f0a6af855f3c5,0x000fa6ef0a082826,0x000564cf46392869,0x000d9060255a7500,0x0007688b437590a1,0x0005fa2a21425afd,0x000dd69d65b91f19,0x0008045302895146,0x0000000000000000}}, + {{0x0009a872aecfc094,0x000c8cde3af050fb,0x000ebe6b500bec0f,0x0004d4b7e7c4ef2e,0x00094b38a6c4228e,0x0004f9fb0e82362d,0x0000dbf4e229d20c,0x0003a6e45bdfa369,0x000b1c90f730c74e,0x000306f2fc481fa7,0x0000000000000000}, {0x000e496c4b887a36,0x000e6ae46148f8e5,0x0004268188f16f8a,0x000cf1b360936452,0x000ea828f2ec9dce,0x000a581be5eec097,0x0000a093e062b3a8,0x000e4da12b498543,0x000b50541562092d,0x000eae33c27b17cb,0x0000000000000001}}, + {{0x000bed4ffad0684c,0x000bb264e57bfffb,0x0009eb6b035825f2,0x00013466fd8b6643,0x000ab9c3537903c2,0x000b0366be7313de,0x00096ae2121723c9,0x0005953e87c3ac29,0x000b6974bbd38278,0x0003e43a30236cf9,0x0000000000000001}, {0x0003b2707ffdea7f,0x0002da68809f795f,0x000374c5228ca4a1,0x000ef9a132cc5a86,0x0002bae5f8c0d15c,0x00061ce20672616f,0x000abed75c41da6e,0x0006a5fc5af7de33,0x00076a4d15065912,0x000ed44c16e78857,0x0000000000000001}}, + {{0x0003498018e534a0,0x00031b029f064c8f,0x000b893aedc07be9,0x000b0eea14f71f6a,0x000eee179067b242,0x0009f6bf528af895,0x00020985e852a279,0x000b94bc19691d5c,0x00005deba296ab7d,0x0000b231b9475f76,0x0000000000000001}, {0x000c45d63f8d3bc5,0x000aab0d9145a0f8,0x000bc0cd8bbb3a1d,0x0001299d614875d3,0x0008bad650d624f5,0x000b91d8407baf74,0x00054a383b9d385d,0x0006840ae765f5cc,0x0005a54bdbe2653a,0x0002865728a0edab,0x0000000000000000}}, +}, +{/* digit=120 [{1,2,3,..,}]*([2^480]*G) */ + {{0x000e419064732d33,0x0000fb9f1fdd6e2b,0x000b458dd169ab15,0x000b79def3f55fa5,0x0001fd9b88ebfb1b,0x000b8b17a8c1d98e,0x000e6b37f6beb8b7,0x000dbc72340b6c86,0x0007c19d37bb70ed,0x000f6867a99418dc,0x0000000000000001}, {0x000a4a09f22c0fb3,0x000bb19cb6bc1256,0x00077d8b51c8ded9,0x000574809f35ca45,0x000bef1168ba7eb2,0x0002cdae11770b52,0x000ff68ed4f42bd1,0x0003d326b225de9d,0x00037f1445631a8c,0x00012cb14a3c371d,0x0000000000000001}}, + {{0x000b0b95f0603f03,0x000fa7f969adad66,0x0001acf774657813,0x000a2615220707f6,0x0002f71d4cd53712,0x000f82a44a2fd4ef,0x000773fd9e262931,0x0002f763ad200681,0x0009b206fe31fd70,0x000765c3a8767fa9,0x0000000000000000}, {0x000a7f8ce84569e5,0x0009a821c3dd4741,0x000e90e3290cc915,0x000cf99306b623fa,0x000d7531760ae9e8,0x0009e7cf282874af,0x0006e1ae6527ae8c,0x0008f99eef73293d,0x00037109e03d3d87,0x00036ee1efdba892,0x0000000000000000}}, + {{0x00051928ed074ce9,0x000b292af7a58dcc,0x0005ec5d4bdfb374,0x000cdd85d01fb1db,0x0006e626365656d8,0x00074fc478641e47,0x00016a5e28d244d6,0x000adbaa94ddb39d,0x0007fdde95fd5183,0x000b47ea66d8626b,0x0000000000000000}, {0x0004c9d4962ab02a,0x00021388f7fd2b57,0x0006c23d66031232,0x000a1a6ab2ca0c2c,0x00017664a406bce3,0x000f5497442ca199,0x000866b6f2fc1498,0x000a41cbc3b0ab32,0x000557ca37a277ae,0x000af01602653825,0x0000000000000000}}, + {{0x000db75622a8dfc2,0x0005479be9e5c74f,0x000548d39ec29bd5,0x0007942d29c79da4,0x00079c4bc1f5df3f,0x0004a7cecb948e1f,0x0008793b63229ed3,0x000939c1a7d67689,0x00057ad78be3b341,0x00052f2801351b91,0x0000000000000000}, {0x000cbeae6fece889,0x000b3085ddee3b59,0x000eeab1d348140d,0x0006bba941a033c2,0x000b685703aafb67,0x0005046b6423a9d8,0x00075dab832a7c83,0x00015b8c259b9e24,0x00018a6bbb51f863,0x0003a253eb5dc8db,0x0000000000000000}}, + {{0x000cca37a85cafbb,0x000b3657f26e3623,0x000787ec793c4d2a,0x000337f7520b9137,0x0000dbcfb7906436,0x00018cfaf22caa7a,0x00044625a502d754,0x0000066c6a130ba1,0x0001212f51d083e4,0x0004ebb9541e99d2,0x0000000000000000}, {0x0009384f4e2ab22a,0x000ff707cf7953a3,0x000aa5f9b05bdfba,0x000626e81b083e95,0x000defb350599782,0x00092399d206f421,0x0008bd9415729d3e,0x0008cf10387904ad,0x0006e0bc19370ad7,0x000b48f2c002a076,0x0000000000000000}}, + {{0x000b8bb85d8adb3e,0x00067b9a142f9bd6,0x000fc51be0f979dc,0x000cb118f84e32d8,0x000a7f5b6ca36f9c,0x000a900f565e79ab,0x0003143fcfd2df63,0x000122db9b751516,0x00086015e5f85f9f,0x000bf0e7c48af6d8,0x0000000000000001}, {0x000cbc466d0dec7f,0x000fcfc13f4daf5d,0x000613ac2b0043ae,0x0007d2ec60909041,0x000eff4b79cb6956,0x000e04188e57b5e5,0x00045aa9dd05dcf8,0x0007cd8106c6759c,0x0004b84b0c6c633a,0x00041ee796334569,0x0000000000000001}}, + {{0x0008ed21f68b4a3f,0x0009e0f39b982afe,0x000ef033664df945,0x0006109c1245ad2f,0x0004d6578f9c34c2,0x0008e9fc097b7383,0x0007b3121a085c72,0x000365666df584bd,0x0007af58ed558596,0x0007e9fd1e18ec9d,0x0000000000000000}, {0x00017df29af6bc16,0x000dbbf468848de7,0x000d747cd3b7c888,0x000801a051097e9d,0x000f68bb9b824e70,0x00027a8a5f172bbf,0x00074f9f45d5351c,0x00080ba6fcc24020,0x000d4e050d7e5a57,0x000cebcb9d2f1cc1,0x0000000000000001}}, + {{0x000b990fe3b9d773,0x000eb81096bf3df2,0x000eb580e653b2d5,0x000cfd31a2ad7396,0x00065cddd150bca4,0x000cde916b4cdae8,0x00019b56ffe74e35,0x00021e7dc0b21b6f,0x00099d8bf333016e,0x000eb146cec318c7,0x0000000000000000}, {0x00030acdbab36d51,0x00089ddd1e911c98,0x000891db5801a0df,0x000f1a5d646bbddd,0x0000ac4d27510e25,0x00044af2f910d55b,0x00024a75bcea08e1,0x00037ae5f37d50da,0x000d372739ad211e,0x0002a2d9d5c41773,0x0000000000000001}}, +}, +{/* digit=121 [{1,2,3,..,}]*([2^484]*G) */ + {{0x000ce74763660052,0x000da3e409da1731,0x000098b5f715b328,0x0003538d607382a2,0x0007bc3ee7b0651c,0x0006d5eed9abf1dd,0x000eb18e8c0d16d9,0x000e3fe464dc1a4c,0x00030d6fa6b9f8f1,0x000cfa359d987d0c,0x0000000000000001}, {0x00047d09810803ed,0x0007b5b97b578929,0x000cc27fc5005d73,0x00040feb2087e2c1,0x000b7dd0d960662d,0x00025ee555f37345,0x000d7c17f3858a72,0x000a0cf2ae739ae8,0x00000ee77dcf4e1a,0x000c12649e41ecee,0x0000000000000000}}, + {{0x0008de672d619b61,0x0000ea8326922a80,0x0001a0841b015626,0x0000f8a963e3e317,0x00037806aeb44acb,0x000d9d8a14334837,0x00026bd761a3419a,0x000d2e7a343fbffd,0x00086e32c6d361b6,0x00023ef433219c4d,0x0000000000000000}, {0x00025620f22d4f25,0x00067dd5c03d381f,0x00080f734643a87f,0x00006c5ee876505e,0x0002b491baac4e49,0x0003e07deb178a01,0x000ad060f735b869,0x000576ce5dd8d75f,0x000dd4cd9c97cb18,0x000cbc634bbb55f5,0x0000000000000001}}, + {{0x000b9733710e8e01,0x000e73a5711788b8,0x000bf8afcacf73a9,0x000d6725ee57149b,0x000e7fe486c64e2e,0x000322f9087bd5a6,0x00009af08709418b,0x000084990390cb99,0x000a6bb3ab911d03,0x000d2868a69e665b,0x0000000000000001}, {0x0005e749b382f6d2,0x000a9a1034406b89,0x000826ec06265b52,0x000e64e9aec95b07,0x0003982f9a9c5d16,0x0000698b37d7e83c,0x00050d8bbdbeb42b,0x0007c82f4fbc8ae9,0x000adc1e63d423d5,0x000b249310802372,0x0000000000000000}}, + {{0x0007033614a6d5dd,0x000fddc5f2fac137,0x000e014aa4b4dee5,0x0004a9b72218fde8,0x000c10e229612a68,0x000cb5b99f1d9b66,0x000eff01796c1307,0x000ec087152271c7,0x0009f171d27930b1,0x000dd53091f21ad1,0x0000000000000001}, {0x0004c873e4172f54,0x0006ecbd512368a7,0x000d3ea21d4bc31a,0x0003a95f62eff689,0x0007c73a33474bd4,0x00088fa97a141350,0x000b4d3b01846eff,0x0005fbac8f6a8f06,0x0009ddd58dc2a301,0x00001f7b911f1a15,0x0000000000000000}}, + {{0x0002e681058bcd0b,0x000ecb766f6bc98d,0x000866fedccaaef5,0x000b2e2473204d11,0x000f6e18757016ad,0x00011d59effc1a8a,0x0002050629e88cfc,0x00093c7bdc024782,0x000a9b2aeb9bb00e,0x000336991f06d2c0,0x0000000000000000}, {0x0001568955531744,0x0008281170681859,0x00050d7be99cf6e1,0x0008cb9d185c0963,0x000f49cfc22a2afc,0x000f9d20626a2a56,0x000ad87b48f04b95,0x000bd1441cc30d3e,0x0003e9b72d43f56f,0x000e3b3843d17383,0x0000000000000001}}, + {{0x000e873b05f77e97,0x00071d5ebf3c8d2f,0x0005b9ca7cc32fe3,0x0008798cc245b054,0x000e6eaf83f8b265,0x00061d87bdf09afd,0x00048a529e1b9707,0x00001501c97ba4fd,0x000ca96655ab0a10,0x00042f0ec7beee1d,0x0000000000000001}, {0x000296b82c7a9289,0x00070c171dfdb228,0x0001dac3a3a171bb,0x000b7ea6ad9a13af,0x000251fe361dde21,0x000cea9acd2f8b81,0x0008480e8df3c1ec,0x00038a5f495ca4b4,0x000fd225cb8ecc78,0x000454bc6bffc707,0x0000000000000000}}, + {{0x000af33412f12687,0x00015e41163cf0f3,0x000967fdc5a6a476,0x0004235cf9f62e34,0x000b314d06a6a848,0x000820f5665619a2,0x000f11a14ea427a8,0x000ce9a80c44b6a1,0x000f92bed7985fca,0x000dc713540bdff6,0x0000000000000001}, {0x00065826c0cb51e6,0x00030220ec95e76a,0x00064aea77a786d0,0x000cb5ba9e93c602,0x000f020c5b781189,0x000e7b4655282299,0x000e97af8ea95e4d,0x000a8a80a0f194a4,0x0006433581c41d62,0x00066e34a29ca8e3,0x0000000000000001}}, + {{0x000cef36ab807b63,0x000040cdf4c99984,0x000c211953a5f8d7,0x000ab4c0faefc5ed,0x0005ca17066a1563,0x000fb2c0940c339a,0x000b1e8517a5667a,0x000c3d2a94a0b135,0x000185e4d4526e2e,0x0001b53c05d493d9,0x0000000000000001}, {0x000c5ced3676f843,0x000195ff470fab2f,0x000ed29f4a221ddb,0x0000868b2d94f5fe,0x0003caf8fcc5069f,0x000dcfc1418631be,0x000998943a070623,0x0009bafa5f731c9d,0x000c5c56c1cc4a06,0x000a82f502e626e1,0x0000000000000000}}, +}, +{/* digit=122 [{1,2,3,..,}]*([2^488]*G) */ + {{0x0009edf282d019ec,0x00099e8e335e18d2,0x0004ace8ce0e046e,0x00001d0f72c0503a,0x0007e9c6d09e242f,0x000998b6c2fcb456,0x0000be40686ceb13,0x000db8fca6af9143,0x000c77e852236ef5,0x000ba3718e1a2901,0x0000000000000001}, {0x0005ae430ab427d9,0x0003d8a843a1b6ab,0x000c9500fb6025f6,0x000b9cb8d803e788,0x000fcea023d9bfb7,0x00003f3ec5cdad70,0x000188da7e50d4c8,0x000f9eb540fd9c07,0x00014ab57822ee2a,0x000574aff12ba00d,0x0000000000000001}}, + {{0x0003c20dbe0952a3,0x000480b6013f7fd5,0x000447348d109d4a,0x000fe6fecb6a7da1,0x0006564e8c529d8b,0x000034045fa60672,0x0003ee2a8df68fa9,0x00021796dbc7ff3f,0x000a130fededc279,0x000fe24c3f368ae9,0x0000000000000001}, {0x0002961eb9eed66d,0x000919ed55f27279,0x0000068193a9b014,0x000f317444cb0bf8,0x00096e22227ee32e,0x00047c8b854cb4a8,0x000bf6dc6a73b281,0x000d6804296e2ed1,0x0003e6f8a77be001,0x00084c89b143ab22,0x0000000000000000}}, + {{0x0008d791f2d40215,0x00003b05d1fd525c,0x00037b16b3ca30ae,0x00070792a856131f,0x000b7639faf0f678,0x00006b7cf12eff42,0x00098ab7a44f2173,0x0006714e846ec06d,0x000eac350874a266,0x000b56e5920dc3ae,0x0000000000000000}, {0x000d703e34853d1e,0x0002cd53a7cce717,0x000410bff4f394e1,0x00074ecb0bba0cc1,0x000de8fa9da2c436,0x000f5e3f74e2caa0,0x000cc28b148d1eb1,0x0009fc1ac5bad585,0x0001220666fb73af,0x0000c3241a57ee07,0x0000000000000001}}, + {{0x0000e99218ea1f1f,0x000ccf21044500ec,0x0000c873630cba88,0x00064f806fd4e4b8,0x000a7056645dd457,0x0002ed87394551a9,0x0008987025ba6b17,0x0005dd01b45fa9a7,0x000ccea3a1f9f135,0x000592807cbab8d2,0x0000000000000001}, {0x0006c96e8e24e119,0x000d921a51e8134c,0x0002ab9759957065,0x0004035ca89e1baa,0x000df057c2aafabc,0x000c0890aa1a6716,0x0006bd3f802387d9,0x0006a39383e5c778,0x000601e4e1f62705,0x000096f226577900,0x0000000000000001}}, + {{0x0001d3b5076ee66b,0x000068e996c31106,0x00063f8bf5d922ca,0x00008ed44203a2fa,0x0001df0821d991eb,0x00019d54e602f04f,0x000bf35cd4ee7bb8,0x00015f2609e3729a,0x0009e8e65b2fcd60,0x0001df9e9c109298,0x0000000000000001}, {0x00058eae5edc3042,0x000fe31ba09cdc97,0x0006f52853b56fdb,0x0003df0f5ca36adc,0x0000a54940d61878,0x000902cef58665ba,0x0004efbb68e67641,0x00036806e0aaf0f5,0x000194a89e785e8c,0x0008883379ceb241,0x0000000000000000}}, + {{0x0001a19f7b341ee3,0x000076e0e5354bd1,0x0001806a485286dc,0x000d2bf681431840,0x0002c82334b1343e,0x000b6908add258b2,0x000737bf47a5dbe4,0x0007303c531a0b11,0x0003a29d6501615d,0x0009313aaea01e10,0x0000000000000000}, {0x0007b22b906a725c,0x00024ad4ce011033,0x000bf242639d28fc,0x000fa39cd38de6c9,0x000a72beeb0f8fd8,0x0009a0ad9b93380e,0x000c8221fc799b7c,0x000be51116d9d8de,0x000a10526402ca53,0x000777aa2c9ac3b1,0x0000000000000001}}, + {{0x0001a723455b8da2,0x0001d0c7c777796b,0x000fb41cc9b1644a,0x0000ef5ce0972939,0x000637a26ddcdf7a,0x0008cf1a639f0844,0x000023a3ce642ca3,0x00098f7db827cdb3,0x0005279eee7f6f0d,0x000565523e47e762,0x0000000000000001}, {0x000fd1b034828d4a,0x0000721a61eafcaf,0x0001ce95f4ec0ae3,0x0003ba2db4a66fba,0x0006525e6f06fee9,0x000817b26fcb0ef3,0x0000265db68ac06f,0x000f74e811cb24b3,0x0006550f9c2bf885,0x0003940f2d2fcd83,0x0000000000000000}}, + {{0x000e8955d1b109b2,0x000e1de1d0f381d4,0x0006407a6cf45a79,0x000f2393e689a76d,0x000d3d92aed2a407,0x0005547cc6ac261b,0x0005e0b9e62fcac9,0x00081e2910774983,0x0003d6780dde8f90,0x0003c5c4cab77f7b,0x0000000000000000}, {0x000e052d3f3dc82c,0x00039caa1aeecdbd,0x00024153092958c9,0x000c11b7ca5c0f7b,0x00027c92847965c0,0x000732af643698d8,0x0000367351c0ba1d,0x000f1b1bf491a3ee,0x000df3514ec2302c,0x000b4c4436d640af,0x0000000000000001}}, +}, +{/* digit=123 [{1,2,3,..,}]*([2^492]*G) */ + {{0x0004265bd7179d88,0x00032014b97128c5,0x000fd3dafdfe0b08,0x000b1956b3fd6699,0x00091416a87bbb8b,0x0001dd4344038f86,0x000566c88826c840,0x000f07a8a4b77456,0x0007671e1b2fca59,0x000200797dc52a03,0x0000000000000000}, {0x0007843bbe8d7f70,0x0004f9ee9b4c465d,0x000303b1652fa39c,0x000ae7c4c4a55ae2,0x000263ccdcb67c15,0x000a17fd06da8ac5,0x000c10d8d1d1e927,0x000e5bfc6232685a,0x0003162cd048bbb8,0x000b11c2cffebb23,0x0000000000000000}}, + {{0x0002cb202ec3c178,0x000285de81ad92d1,0x000b71b77497dfd0,0x0007a8a10c150a03,0x000bbe99f4ad3f59,0x000f4533b0aef51d,0x0003b27838ed4931,0x0008ffc95a8ebcf1,0x0002cefcf5623ddf,0x00010737c166832b,0x0000000000000000}, {0x000e740d8da2ff7e,0x000624f3d3ab048c,0x000376415ced03ed,0x000fbe5676391c7d,0x00081671ffe7b22f,0x0000390438cc5f49,0x00084a5ae289dd49,0x0008f9a1f5bbef09,0x000b05c4941c6652,0x00083aef77ff073e,0x0000000000000000}}, + {{0x000604a9bc6900b6,0x000878e5f51ce9df,0x000b763f98cad97f,0x000f5a1389d3ab54,0x000ab3d0efdef6fe,0x0009be5cf0df2543,0x0002a0d518696763,0x000134850193d832,0x000860abf9047761,0x0004b4f04b3d8de0,0x0000000000000000}, {0x000e57c44a551894,0x000a3fa66238e065,0x000d140af7878a12,0x000de14d55dcc858,0x00061c7b391cc65f,0x000d7dbf324d8825,0x0005d09aa74e78f2,0x0008ed166a4503f8,0x000da0c2a7ad860c,0x0002048d8fe387e6,0x0000000000000000}}, + {{0x000505ddc7162a8e,0x0001fb5ed0deab66,0x0005e972cc689dc5,0x000e495fb69dc78b,0x000ca3f1826690c6,0x0005f6186896d605,0x000fd32f66789288,0x0008863f96f8edfb,0x000e644e5dece22e,0x0009b7f857a4d564,0x0000000000000000}, {0x000691ddbeebfc5d,0x0005901566a70055,0x000e1f4e6067fa43,0x00086e62796a672c,0x000b4e5ee14cf308,0x000b327c1a40aaca,0x0002fed3294bd689,0x0009103e56992c00,0x000b1323df8494f5,0x000d7b51fec2bb9e,0x0000000000000001}}, + {{0x0009f1b91641749a,0x000415c17bca5ff6,0x000c6d3a845d2248,0x0005f9b6f404856a,0x000cbf400b63630d,0x0003b4ba273b60be,0x0005d238fa843e67,0x00015df9fe916d32,0x000747b338f22dd6,0x000213df0dde0478,0x0000000000000000}, {0x0000235c1fe8e923,0x0003196a6c0855b8,0x0009e7caa33347ee,0x000bc0a45596b47b,0x00011e3fa8377c58,0x000acaedbc6f8b16,0x000fc365c99edd72,0x0002dbbec0f6e0a6,0x0002b70a4b7723c1,0x000a4bf65f3c20c8,0x0000000000000001}}, + {{0x0004bf14c1a4d603,0x00062d773ba946d2,0x000a858973ed2e17,0x000af47e821e4637,0x000c4e2f1b685a3b,0x000a24d7fe743f00,0x00011c916f0b3711,0x00019d3f29631796,0x000070eeb3ea27a8,0x000fcf9d0e9d8d24,0x0000000000000001}, {0x000a9e0cc6d6de0a,0x0007d8c5fca34ad4,0x000c81d46494c7d6,0x0008c618856d1751,0x000ea22fc514e835,0x000e085f741c8235,0x000c321d004a049d,0x000bcb516087e553,0x0002d6363ccbbe68,0x00083e572fe7f6b5,0x0000000000000000}}, + {{0x000e20ba96faf5c5,0x000e8d8c8a0cc4ab,0x0009b5c593a344d4,0x000c6c34af049395,0x0005aa8d456d94a3,0x000ed953bf7c9473,0x000962cd0b8cc1dd,0x0000c01bb3088b5c,0x000c82c42c7d7139,0x0005bd26c576d9ee,0x0000000000000001}, {0x000ee2f364e79144,0x000e681f5a9a561f,0x00014c6812d4021d,0x000e552f50051d32,0x000f6a99f35033a6,0x000e505a2349153d,0x0001622a1be8e97e,0x00067971f625164b,0x000d441d9fc2328e,0x000d2eb0550478b4,0x0000000000000001}}, + {{0x000769c73aea3c08,0x000df9a9593240cb,0x0004e8217f3b057c,0x000ceca2220054ab,0x000c95ba1c2a734d,0x0006500d1322b719,0x000ec571b4360381,0x0006f76e87cb0ba1,0x000c5938559db2c7,0x000397be033b5877,0x0000000000000000}, {0x000b6a77feb075ca,0x000d9cc6a6cde3dd,0x000e49872538f578,0x000e469feaf37819,0x0002ddc9c48cda10,0x0001a6e5450f9883,0x0002d31bf05ea5f5,0x000375cd216a195d,0x0008007e689987c4,0x000cbc358f3e07d1,0x0000000000000000}}, +}, +{/* digit=124 [{1,2,3,..,}]*([2^496]*G) */ + {{0x0006a73f6f2770c4,0x0000f968ca281cd6,0x000827efca6a0867,0x0003a96b180e8f32,0x000809979b757eac,0x000d9223bfbff7df,0x00047dd166015fc2,0x00065475a88730d7,0x000ce16229ea9d12,0x00076d23756de3fb,0x0000000000000000}, {0x000ed537bee27c6e,0x000943e46c7c15a8,0x0004b3f87656d7df,0x000a9213335be530,0x00076cb0ee208db8,0x0004f5fc16b61ee3,0x000c1114ee85495a,0x000253ced62c2d47,0x000641c92453ad35,0x0003e4e1a21d73af,0x0000000000000001}}, + {{0x000483ff2c9de102,0x00017f0cb9492bab,0x0001999673c19107,0x0005c7a75ba40ad7,0x00022ec8c1ec861f,0x00078704457a9540,0x0001194ab6d023c8,0x0000daf5008c607c,0x000d394925361233,0x0005ab5bf20a934a,0x0000000000000001}, {0x0008ed3301b16277,0x000b31045574d7ab,0x000ed11cb44f38e3,0x0001af67ab10bb4e,0x000d033cee10ca51,0x000549874c9fe7c1,0x000392d6999489f1,0x000ffcfe4a15a85b,0x000b006a13684dec,0x0007b8baefda3eb3,0x0000000000000001}}, + {{0x000e82cbfe7306a8,0x00002e52832eb494,0x000b381c8b461b41,0x000a6e877f0afbca,0x0006b8482ae88f1b,0x000709eb28c8cc07,0x0000cd45fc8e5ced,0x000b1363d1cf0c64,0x00093a63d6be8f78,0x0001407a8e7f6a49,0x0000000000000000}, {0x000fd703c088bf64,0x000708b3415df01b,0x000f82eeb2c8b57f,0x0003ed35407aa69d,0x000449767c6b4a72,0x0002e1a8184dbc3f,0x000d25edffc3e965,0x0003e8855e29ad89,0x000c695a889f2e49,0x00025dae2a995ab2,0x0000000000000000}}, + {{0x000c300d63dacf22,0x000f84149cc93249,0x0004f71e87a984e1,0x000ebeada635f884,0x000cac51f48942eb,0x00076c6b878d815d,0x000587460dede95b,0x000883c91cf8c3f2,0x00013e9be5375387,0x00076d4ce987a56d,0x0000000000000000}, {0x000fb20151675f42,0x000bf54bf1c2d622,0x00022da7f9e8bc4a,0x000051b0e7b83f3a,0x00073eda536a6e42,0x000ce8431a9d89d5,0x0002b7a64d23c5a3,0x000007a6be7b3eec,0x000672919b5fb43a,0x000c454a7d18005b,0x0000000000000001}}, + {{0x0009f484a29e97a0,0x0005e90aa411ba92,0x00016fc135a72a99,0x0006cfa3c8dd8a3a,0x00066efeed6df222,0x000e66eb40ebb1a2,0x0000f8ad15ad7b0b,0x000e0c3a19929e39,0x00051e3404d13a05,0x000175cf393bacd1,0x0000000000000000}, {0x0006cdbeb4d89814,0x000b884f6ce10295,0x0003138d1321a20d,0x0006528c4bd065b9,0x0003d082878ee395,0x00029465651ab383,0x000cb4e49e6b0dcc,0x000710248d30e955,0x0006e01a51ae9cc3,0x0002fc5567eab89e,0x0000000000000001}}, + {{0x000c59218072f54a,0x000195bad5f014fe,0x000deabd55429cb3,0x000b2ab5fb9c1406,0x0006cf39524ff8ca,0x000fbb57c01480bd,0x00018cbc932f5376,0x000c9e4e5da034f1,0x000fb16c36eb8d83,0x000048c80fc4eaa6,0x0000000000000000}, {0x000668aa11e1cfe6,0x000d4f614afa98be,0x0004cadab479a412,0x000864b0b94d7822,0x0002651053a74933,0x000dd43fe6424e5f,0x0004f2c600bdaac3,0x000ec0b432ccf8a0,0x0004d82574257110,0x00024e58edc3e12f,0x0000000000000001}}, + {{0x000d5163c2f29845,0x000271b8856ff717,0x000a79d0557c5c08,0x0001484032810d21,0x000a5a6e95174165,0x00093b8782ce8f06,0x000a6f7f14b15d03,0x000ca398eeacdf3d,0x000eb31c7c040c1a,0x0003ace9ed34f4d2,0x0000000000000000}, {0x00086bac2cc4ff50,0x0008d5294f7bd063,0x000ec0b7a55f986d,0x0006868155592285,0x000e215833824965,0x0003366d9a307162,0x0001de9196efa150,0x00076afbb75f7833,0x00046ce65ce11aa8,0x0002f7a207e31942,0x0000000000000001}}, + {{0x000d6129f7d0fa54,0x000150bddf5a7cf8,0x000b4988625b2f43,0x0009bbfb3c2f3809,0x000f5b080f7b3129,0x0000ab0abb84ed45,0x000510d824f7bed2,0x0006d6447243533e,0x000c576b7b64fbbb,0x000e16caa9ee8267,0x0000000000000001}, {0x00053a269ea0b07f,0x000e06f68fe62242,0x000a777b6874572d,0x000d5f86cf599bf5,0x000fe2a811045a16,0x000873264294a33d,0x0007a04ac970a0c0,0x000ceb2b7d05d686,0x00029a28a0e51a57,0x000bacbf79a38ead,0x0000000000000001}}, +}, +{/* digit=125 [{1,2,3,..,}]*([2^500]*G) */ + {{0x0003efe866dc2f62,0x000853cb9e407c10,0x0000e6c71edaaa13,0x00018f751b70a2af,0x000b0cf7e3e825ae,0x0005c36a5a1ec11b,0x000487f56ab4b564,0x000a86df052ea4e5,0x000d750313868ef9,0x000e60ee422740c7,0x0000000000000000}, {0x000ee652bd47edd5,0x000397faa97f40b7,0x000294d2d1ba3dd7,0x000344d3453daecf,0x000d324bd3f56e65,0x00078c6f611c9985,0x000e24f2675985ec,0x00038b4060d38ad7,0x000dfb7496c92821,0x000be627a6ad57ff,0x0000000000000001}}, + {{0x0009be6a7e5a166a,0x000d06313031bd58,0x000704962d984289,0x0002ec5b522512df,0x000386a669eef493,0x000fe747674db075,0x0004dbaabd7aebbc,0x0004d27fb22ce794,0x000e70494458ef81,0x000fadc96805636a,0x0000000000000000}, {0x0005b60c511b3ff8,0x000584915adb1e6c,0x000f8937e8e108c6,0x0008406d64ea3a9f,0x0004461d268f9ab8,0x000e3ff279d6126f,0x000d3b3ed1f3032a,0x00023a1b63af22a5,0x000caf9282fd7a53,0x000d99f7a42a7590,0x0000000000000001}}, + {{0x000dfb005014a6bc,0x000d36179f05f79f,0x0001f0a00c591c70,0x00009f861bdb8aa0,0x000851877e4cc13b,0x00004921bdab098b,0x000265f47ca34718,0x000478a5d59cb874,0x0008aac74eb734d8,0x0002f6a87e5bc7bb,0x0000000000000001}, {0x0005dd559082ec4f,0x000fd3340b409a63,0x000e395e6174cff9,0x00035fdf83237476,0x0006e995df5d90e4,0x000535beb0acf902,0x000ddc2f60fe3f20,0x000821d68a60c3ba,0x0008005435d079f0,0x00084ef7a20c388b,0x0000000000000000}}, + {{0x0001ce9624902c85,0x000dacee54d7fc06,0x0008e982883e676d,0x000cea68fc5997cf,0x000b94f3d06f1e8c,0x0007b80d8242831b,0x000ec2e625c36045,0x00015466a1d87389,0x000c5009313ff25b,0x00045efb5d45d1f9,0x0000000000000001}, {0x0004e32c6f246301,0x00064608ec78b5ab,0x00053a9c0b324014,0x0004ddead5ffb795,0x0008bf933bdcc559,0x0005249289dc110f,0x00047d25d52f652d,0x000d95ab06c0cb41,0x000bbff17968adb2,0x0002664039be6fa1,0x0000000000000000}}, + {{0x0006c159dee64527,0x000d486c1da85118,0x0002ffecac887f87,0x000401c5326e8ac4,0x000fe68a082d270a,0x000b17b4ec0bd703,0x0009b1a9fe82427b,0x0008f75c6b25d502,0x000ac56e28859df5,0x000bc70f97f9bffa,0x0000000000000001}, {0x00073b932e4e3e21,0x0005f8e721dc13af,0x000825bde1498f11,0x00080fd3102f60c7,0x000f292e5a9e9f07,0x0006a4edbc9fcac3,0x0008f2651ae44279,0x0002622ca1bb123f,0x000a4ca103d2d6a7,0x000ca577d0f1994f,0x0000000000000000}}, + {{0x000e577af10e1302,0x000156bf557eaa33,0x000ca7cba0f98005,0x0001e9d6e3d41486,0x000ad3a30b9f973d,0x0009856b55aa3443,0x000724f819219409,0x0008250cf13f0ca4,0x0006f0f69ba1696b,0x00063bba1deb1e9b,0x0000000000000001}, {0x00037cb6672f9435,0x000d7e7437eb08a0,0x000579149c6faf55,0x000e4c6943f7c61c,0x00026eba03e36921,0x000a3ade34c342ab,0x000052386264eb60,0x0000ffc57653cf12,0x00070eec7914e6e2,0x0004912d67845657,0x0000000000000001}}, + {{0x00006fc3e072e1bd,0x00066e9739a62b31,0x0006ea97e6ad1669,0x0002aa2bbc843284,0x000871768bc4f0fb,0x000a51f2d1bd1f5f,0x000245a8890f99b0,0x000b6a0000aa5f53,0x000547c1538e0dc0,0x0001c33dbece2149,0x0000000000000000}, {0x000b56de17b97c51,0x0000fccb15a8e3ea,0x000c00352a78e0ff,0x000e7d480dae3bf3,0x00092273cee30716,0x0000962a4283dde8,0x000e674e18ae53b0,0x0002b8c78835cb2c,0x000faee271217641,0x00046294e0f5e7c6,0x0000000000000000}}, + {{0x0005697612d854d6,0x000590266d871a78,0x000015aa94be7df9,0x000482ac4e8bbf72,0x0007c12880439150,0x0003495f23aa4b2f,0x00074815ef777bb5,0x000838a798c004a6,0x0008a425cc7aadc8,0x000c432ccb5f5730,0x0000000000000001}, {0x00006af85640f288,0x0003a6718ebc6cb5,0x0002c50bba9dd21e,0x000e3c4d56098fde,0x000a4a8a721857b6,0x000218f9c402f4d7,0x000a6f255530e5d9,0x0000bf7b3c63a541,0x000f0181b97421bb,0x00023de7a08f2804,0x0000000000000000}}, +}, +{/* digit=126 [{1,2,3,..,}]*([2^504]*G) */ + {{0x00061ebad9ee2c24,0x000dd46aced3ddaa,0x0000bd3e3fde5fe2,0x00020569fe14f9f4,0x00088d818d1a2095,0x0002f0bdc9b4968b,0x000e3de0b8b77328,0x0007fe9e8edc6520,0x000017cf0272ff76,0x000eda0f65dc99bc,0x0000000000000000}, {0x0009b50b03dc034f,0x000ff04ea634ab0a,0x0007b191db6e6308,0x000a9de7ee04399a,0x000e6da7bdea8dde,0x00054c55ae492d45,0x000f4e939e666b7b,0x00090c925a51f573,0x000f916220292c15,0x0002d380fc7f5071,0x0000000000000001}}, + {{0x000639a92b83d191,0x000b3a1ce7b1b453,0x0000d260e431474f,0x00032954aefab808,0x0006dfaf9e670c4e,0x000e42d0d7b5bae7,0x000bfa89eb4687fd,0x000c7d89b1ca5f45,0x000ecce4fba638bb,0x0008a21de873fcc0,0x0000000000000001}, {0x000c9c2b49165fd5,0x0005fb318f9f9636,0x0006f676d6c2cb81,0x000c633a7560919e,0x00011e2d4752541c,0x000199c5999a79e2,0x000515dfbee081ee,0x00053107dec5265f,0x0002bdc9ed0ea4e2,0x00041c5a539ab36f,0x0000000000000001}}, + {{0x00009de7ecb2ac56,0x0002d837bb7a345c,0x0004863cfd4369c7,0x00077c66a5755a3b,0x000682ccc872cef3,0x000bc1363c743442,0x0008b997f1a0d907,0x000d72224eed734a,0x000d1850457f924f,0x000f3bbd996258f2,0x0000000000000001}, {0x0004953714391350,0x0002a08de1fb04ea,0x0009bb0ca7d3f0e3,0x00020e2fc4a54760,0x000d525812eece55,0x000dc3c7c680f4ea,0x00064f097079b269,0x0001e81a267b891a,0x0002df53061afa20,0x0007339d7335dbc1,0x0000000000000000}}, + {{0x000fa11c22d57017,0x0007061bfc65866f,0x000e25c9a781f882,0x00052d54eb5d1a25,0x00034e5fcb1fe128,0x0008c5dfb74f3317,0x000cca2e48b7e54d,0x0005e9b41639cfad,0x000e1f8c2193402d,0x000348c49e8f71b8,0x0000000000000001}, {0x00033ca43943f50d,0x00016ce98a1f644e,0x000bd595ac8a7cb4,0x000d4eb1e328cd45,0x000e8ec3fd8cbf8f,0x000eb626b0f768cf,0x0001524476b1bbc9,0x000d8d0ffe31069d,0x00025aa89220edd8,0x00063660b3755829,0x0000000000000000}}, + {{0x0003a78191abe914,0x000aed083110e37c,0x00072de9a657b228,0x0003d434d0dfafec,0x0009778c8ba568ce,0x00021f193e09ab8a,0x00032b59891165df,0x00090a0e20c8f7ac,0x0005b8a1f64088c4,0x000b717ed2f2d69f,0x0000000000000001}, {0x000ac4849a39a0ab,0x000e9224368b5d72,0x0009a57abd9c0589,0x000f04bfeb21218d,0x00049bfe57f2080a,0x00082fbca66b72d8,0x000f0c09eaa93852,0x000e04db305e15a6,0x000bb2bcc4365052,0x000d0e18047907b5,0x0000000000000000}}, + {{0x0003c1031f4f778c,0x0009bbc5cc621ce8,0x000ac957c67434bc,0x000a368627bcbc47,0x000adace0a905430,0x0008aa0831ed2cb5,0x000c11d0f4f5d323,0x0001c48e91c91fa3,0x000229765edbfb35,0x00032bdf2591e498,0x0000000000000000}, {0x0008e455572bcfd0,0x000ecba53bea9ae3,0x000196518c997db1,0x0005c33258970b56,0x000a3c6b46d1e689,0x0008e44ad30fe772,0x0005dd6482160561,0x000b86d1933faf1a,0x0005a53d718ae6de,0x000e4e4345a6badf,0x0000000000000001}}, + {{0x0009f15950170034,0x0008a7d9e0681bfb,0x0002dc602830c283,0x000e8af178838dfd,0x000987d3f8bcc134,0x0008904081cb94f3,0x000a460c0da2bca6,0x000f718b8913d63c,0x00022ed274e647de,0x000ff3a58fd52338,0x0000000000000001}, {0x0002950ff4836dad,0x000fe4a51fa111b5,0x000e13de206b61a9,0x000941373ab42508,0x000c7bf8c1651d7a,0x000dbd9276666a02,0x00056dcdd411c37b,0x000f5a9ab3e87536,0x0000f464671775e6,0x000bc59bc827eaae,0x0000000000000000}}, + {{0x000eafce57fbc903,0x00072ba11885f40d,0x000d640aa98f9421,0x000bfcf817ce6b52,0x000389d8e40bcdc1,0x000c804e7d14a7d6,0x0008fa880e955163,0x00034af7c92b6304,0x000fd685115381b0,0x000faf73ec6a9688,0x0000000000000001}, {0x000f85d1af3848b2,0x0008716ba36666f4,0x000fbfc6c17f47de,0x00003b35f9474540,0x0004b72b1ddc670e,0x000f48bdd3ad6387,0x000d7cfd249ea687,0x0009c16141926a15,0x000d8d963e9d101f,0x0000771b9b1c2fac,0x0000000000000000}}, +}, +{/* digit=127 [{1,2,3,..,}]*([2^508]*G) */ + {{0x0006bccc65f30461,0x0006e44b51d15f3b,0x00061b0ed084d989,0x000f84c9df4f3be0,0x000626ed8e29bced,0x000a49395cf45bde,0x0001bfb128499bea,0x000d30e7b9a9812a,0x0000016d9442e44b,0x000577251b4710e5,0x0000000000000000}, {0x000c58ea8faa5f8e,0x0006398972e1afb4,0x0002d603c3a6a3b7,0x000090f464b41953,0x0003e1c1cc6d5ad6,0x0009fc4551644bda,0x0003bf0e003b3c67,0x0004879a2d4dc4fe,0x0002492059f3993d,0x00089490933a0bd0,0x0000000000000000}}, + {{0x0003d39ba21abe88,0x000b8dfe497366b8,0x0003fb2e32d1a08b,0x00040dcc68fadea4,0x0009bc78d9e5b900,0x000999242f620135,0x000e38d4ca94e098,0x000f3e9a937783f1,0x000c271edf5e42de,0x000cdad3d42e09c5,0x0000000000000001}, {0x00014caf1b7e4897,0x00092052b710207f,0x000840b578b42fb2,0x000c859d4e0acb78,0x000efbd1059c69f5,0x000fc0d187f5ce1f,0x000018dd99b98e1d,0x000fd9d695b6b702,0x00087f7037056ff1,0x00080b73121d9b11,0x0000000000000000}}, + {{0x00078d674ce9ba28,0x00098667edd1931d,0x0007a2c798ce814e,0x00054444ef1eef63,0x000bbc4342cdf2cd,0x0008690dad1f9bf2,0x000e17a85c72b72f,0x00027215c615b685,0x000ab98eb37dbb69,0x00072850ebda62ab,0x0000000000000000}, {0x000b543dc482d1d8,0x00066dc72bee2f05,0x000576c719cbc26f,0x000bf9361970fe88,0x00012aede3868858,0x000d0a81339054e9,0x00050f3227db12ff,0x00054f85925da234,0x0007c4995e06e9ee,0x000343899d0c96c8,0x0000000000000001}}, + {{0x000622bfd7a6e63b,0x000ca212bebe9c89,0x000487abee2cafd2,0x000143f4d290457b,0x0007d05d13bf4c04,0x00067f0ae3716aab,0x000a3097740d4130,0x000debe6d02d5925,0x000ef2c2714370b8,0x000fcffae20be9e8,0x0000000000000001}, {0x00009dc727eac0b8,0x000802b463618a8d,0x000e00f824949d83,0x00021e8850666aae,0x000a354be3730d5a,0x0006ce164b258522,0x000386cbf3fd223a,0x0000b7ba5ba4fefa,0x0008ecff5479bc6a,0x000ec6ece410bb37,0x0000000000000001}}, + {{0x000fb0fd2b03ceeb,0x000c34d346742d91,0x0007fb1da808f925,0x00014c3e50e576d5,0x00045dde45aec274,0x0007634b06ff5225,0x000e8f635cfee4ae,0x0006af16b33722c1,0x000b6c9512df9861,0x00087905f5ba1ed3,0x0000000000000001}, {0x0006849e44fd7308,0x000ca8c188190c88,0x000568eece8b82ee,0x000afcede37dce03,0x000956f0105616d6,0x000fcd4f42159b96,0x00016a63204d42e3,0x000f90e66b95446b,0x0003ff7137d7895f,0x00015f08ed8c3b6a,0x0000000000000000}}, + {{0x000080c877cdc551,0x00063fd88170e394,0x000069a5394fa226,0x000fe13aafd4abd3,0x000aac77685a1a56,0x0002270caf504acd,0x000b15ab3b28bcb6,0x000ff30df2535d28,0x000dad1836a25af3,0x000fa6c25940501a,0x0000000000000001}, {0x00072e0f5dfd0431,0x000680aa3b20ecdb,0x000be825e1b5fe43,0x0005a5c2422130db,0x000f0dc02a5e8b8e,0x00053d6d0a6ed0ac,0x0007b39bd52bad22,0x000defc7beec9197,0x000d80e249e46059,0x0009869fd32b5153,0x0000000000000000}}, + {{0x000d5488a9c159ea,0x000c7c99f362daa9,0x000520e4056562a1,0x000f58587c3780d4,0x000cf122d12decdc,0x00066fa3df2e094a,0x0006c99c7f2dc2a5,0x0005a3f3e1f8fe88,0x000ead106c542a2f,0x0001bd548c70f9bd,0x0000000000000000}, {0x00025e9e39905a6d,0x000d9b4651431540,0x0007d3343ac64109,0x00071e3dd6b3d33e,0x000a550593155d88,0x0005e988444776c0,0x00092c204b5ef211,0x000132ec6af46f8f,0x0000975bc42ba82e,0x0000ec605e3e60a7,0x0000000000000000}}, + {{0x000ebdae497fc314,0x000068fcc42342b8,0x0002e8a76fa0addd,0x000d98aa1ba58a99,0x0006f585d2056972,0x0001a667125a290a,0x000e664a47990be5,0x0003e44696beab19,0x0000ab4a22f64d1c,0x00054a0b8ce48449,0x0000000000000000}, {0x000cf9a538895145,0x000e7e3b3fd1990a,0x0001c1f1592daaaf,0x000016843015bdb3,0x00095b2dbb2d0adb,0x000f77ef5d670d12,0x0008bf1b3f98aca3,0x0003b5280fd35140,0x0007a58660b5ee9d,0x000f86e58791223a,0x0000000000000000}}, +}, +{/* digit=128 [{1,2,3,..,}]*([2^512]*G) */ + {{0x00031c51f5b59b03,0x000c7665ab584951,0x0008739b754d5115,0x000b253e840523eb,0x000078a1f77e3b96,0x000742a046765d97,0x000823d7e942e5b8,0x00080b3194bd1539,0x0002679bf560b997,0x00061abda6ff32b5,0x0000000000000001}, {0x000820e93e66dad8,0x000f2c881e08a892,0x0007e5fd839208f9,0x000d25804e86968c,0x000fc76aeb554305,0x0004c686c9b44037,0x0002e51b80d02e02,0x000e5774d5a620e6,0x000000eae653df90,0x00072ac9b31961f0,0x0000000000000000}}, + {{0x000f8d7860917b4f,0x000bb3357359429f,0x000995f4b0a05d76,0x000e0f1c58d0fe01,0x0002921dccd2ee40,0x0000ab0ca33af9c6,0x000637c074069c34,0x0002098a102fa30b,0x00037701844888bc,0x000d6cb2e96e33de,0x0000000000000000}, {0x00070d506a96d190,0x000f2ce57ed3ba0f,0x0002492cf26e59c4,0x000305995879a0eb,0x0000675760ae93d1,0x0009f9d0d04103b2,0x000d618a2b740898,0x000e723e7b444b0a,0x000b80451ab5c813,0x000616305a1f27eb,0x0000000000000000}}, + {{0x000b480b60680f46,0x00097a71a65ccb4a,0x0002360920f061e1,0x000428aeb306dab3,0x0005aee5267509d9,0x00058e47b1cbaf9d,0x0003350d9a6f7e9f,0x00035af36c30696a,0x000ff438c1f66ddb,0x000119d4937e17ea,0x0000000000000000}, {0x0009df61e7821be6,0x000b8322655044ac,0x0001ae7bb1e106e2,0x00039508343bc8e6,0x000a2bc1e06e0991,0x00066bd6b8166453,0x00044e23756d0eb9,0x00003795c5a1b4bb,0x000605deb625fe17,0x000248426f42f1b7,0x0000000000000000}}, + {{0x000ebb49b8b6d8d8,0x000c9576edd0b2c0,0x000089746d87ef78,0x000ee54686ff89a1,0x000cd51992a8e30d,0x000fcb70ff8362ad,0x00008d8883f2631a,0x0002a13e25b5a551,0x000d32baa9313847,0x00049764387fbe1f,0x0000000000000000}, {0x000652373d0f2fcd,0x0007e9e1299928e8,0x00016c54d21ce8fd,0x000e62d7938b0123,0x000ce4d602bacad0,0x00055138cbf9df41,0x000e0e625dfe098c,0x000dbf9a6851bc01,0x0000b0da12bdbc63,0x000aec8b07cebaa7,0x0000000000000000}}, + {{0x0006a52b356ba3e0,0x000fa486ee1abbf6,0x00074bf145f1f28e,0x000ef4af39b6c538,0x00084376c3d3f698,0x00007cbe4b46d363,0x0003db26e35a57f0,0x000ecc45a1933b62,0x0007b9610a9a44f8,0x000057b84f95bccb,0x0000000000000001}, {0x000bdacb48b9bb6b,0x000642a5eb931fe7,0x00097c327f5fbd2e,0x000a3062ea3deca5,0x000a3dad9ef787fc,0x00092f83b35a8216,0x00042427945728d9,0x0007537c6394f3cf,0x000b615698a93762,0x0001f468793ceb99,0x0000000000000000}}, + {{0x000217c6562eea14,0x000a6a042cb3da1e,0x00060f9ac91e9595,0x00013fef00e33063,0x0001954b4b6c8a05,0x00007e6bb37592ca,0x00018fef0e569527,0x000f4449441c7534,0x0003c8e93a050abd,0x00050bb02598e0eb,0x0000000000000000}, {0x00044683d956af18,0x000340076bbbf06c,0x00006b8e5826911d,0x000bd47b4442693c,0x000bdbfb4883b7be,0x000a67f8d229f111,0x000eceec785481be,0x000d18ec8ccc0a58,0x000ed2b1cc9fd671,0x000ee9b77fbdfe40,0x0000000000000000}}, + {{0x00033f2848525772,0x0002def8aeb9f6cd,0x0009c738e373fa1e,0x000ca3f54bf1bb33,0x0000dd3b12e5c621,0x000856fbe6604112,0x0004fa2d98399489,0x0000ba4686140d31,0x000be8a65034e7d6,0x000fc1d2c9034f2b,0x0000000000000000}, {0x0002e0e02cbac5de,0x00036d1bbef4bfe6,0x000db812e9ceffa7,0x000c4f097569d138,0x000c2795f975ff41,0x000cfa1a794c9978,0x000e1e65c9b377a9,0x000e6d66f7a547fd,0x00058785dc7270ac,0x000b34d6e188dec1,0x0000000000000000}}, + {{0x000a511d09de2eac,0x000031698e16faac,0x0002e96a74ccb4d0,0x000b85017b609854,0x0005887d91373679,0x00039fd4a56f39c3,0x000f60f3aea2bb2a,0x00084e8edd3fc7ee,0x000e1d001d481288,0x0003732f4d1fa989,0x0000000000000001}, {0x000629c27855b7b6,0x000652d6fdccbf0a,0x0005f32800f6bc14,0x0007f62ad29c1358,0x000dbf3a9fdce69e,0x000aa9f4b69418d0,0x000e5fef492796fd,0x000332f4a27a525a,0x0001a7293d91d135,0x0006fadc6b1bb1cc,0x0000000000000001}}, +}, +{/* digit=129 [{1,2,3,..,}]*([2^516]*G) */ + {{0x00076fe9e4b410aa,0x0002e2e46e8257e7,0x000285dece1c90ea,0x0004f07e02e70a0d,0x0007d9e22652fd2e,0x000325ca4b7fb066,0x0001305e36daaa95,0x000feebe9d3d04df,0x0000e3be0fcd0755,0x0005f74f74e60357,0x0000000000000001}, {0x000a7ff33f158714,0x00012737b93b28b8,0x000a408dc4f02791,0x0009c78d219fcf52,0x0008fbf0f03e3758,0x000d6cdb0c0bb10f,0x0007428d517b4d78,0x000d37e06b1f9904,0x000f8786ba69f3ae,0x000b508624d39698,0x0000000000000000}}, + {{0x000314e66fead340,0x0009f5c35e0de977,0x00086281b424220d,0x00095c9d69b8f421,0x000075fd90a74e5f,0x000c09fc499a89c7,0x0004b64e12f34808,0x00068c161166a676,0x000f40cf1886f3c3,0x000c4968ad8aaa55,0x0000000000000000}, {0x0001f4bec36790bc,0x000d77489002d4f9,0x0001759ca38e8177,0x000e759ae14e5a1f,0x000289005868dab2,0x000a1dff8ca02b41,0x000a4d82b9cd06ea,0x0004578741ea12d6,0x000343e8d641aef6,0x0002d4c6a85c8b1a,0x0000000000000001}}, + {{0x0007c33639a0b3f1,0x000010f791b828fa,0x00069b98b785836e,0x000f9367ba2d3b6c,0x0005ecea4290d504,0x0003dd0621586083,0x000de6622ac6245f,0x0002d8153e71208b,0x00091cd77d2a1a55,0x0007ebc7df7c52cd,0x0000000000000001}, {0x000fe4e1268ac84e,0x000d05fdf620f30b,0x00078c9c26de9aa6,0x000fef3a7d0e875e,0x0001f45acf57d581,0x0009769d1b4f2f3f,0x000f8a2b516d4fb8,0x000ca50e2afa7161,0x00069ea98a831731,0x000a4069e2a679a5,0x0000000000000001}}, + {{0x000573b5ab69b37b,0x00099e464e098a37,0x0004b6ae9bd2eac1,0x0007e83941109e44,0x000b3638c710996c,0x0005b0374099e6be,0x000e0cd0943a1c3a,0x0008fab6ecb18490,0x0009f0eb14e71bae,0x00011229f06246b6,0x0000000000000001}, {0x0006e2d865b8fa8c,0x000e4ba2a3d8ca53,0x0004c428120f4f50,0x00003e5bcb6eaa4c,0x0006c69871129b42,0x00018bdacf6da13d,0x0006a314f621f852,0x00031ea900e85204,0x000d280193a13fa0,0x000437167b70a8c7,0x0000000000000001}}, + {{0x000e47d41aa029d1,0x00083c1c1e4e8c82,0x000819d110bf6923,0x00080e340caaa47b,0x000e4f6e315e8cb2,0x000865960449f5a7,0x0002a736db3e3bf6,0x0008b38233a8c18d,0x0006ab95cab54c2d,0x0007b4670af6e721,0x0000000000000000}, {0x0009cd8d15df2f21,0x000c9e95f873ccf7,0x000d4ae259b8946e,0x0003026a352c8cec,0x000c84b6773b4638,0x000327edc3574f14,0x000980243f9e1167,0x000c58a2e8d3fc2d,0x000e789b426360b9,0x00084beac487c72a,0x0000000000000001}}, + {{0x000a504d7a128f19,0x000bec706161d077,0x0008d3f449e0d717,0x00027553cd6aa437,0x00023bfa097584c3,0x00031032e9ecfed0,0x000de734abfe6661,0x0005972524c62301,0x00068a20ad67cd7b,0x0001d7d8e4bd981b,0x0000000000000001}, {0x000dd411d75c8560,0x00020a136bd0f9a0,0x000e9c06f6a8521a,0x0001770134117a07,0x000c6625cc2c0b14,0x0006f01c93534fec,0x000bcd3698e9742e,0x000543a9724acea5,0x000820ced54b554d,0x00005a7954cbcc77,0x0000000000000001}}, + {{0x00060a7aed1a8afb,0x00037a6526a7b404,0x000f752335e80ab0,0x000a8f65f14c43bd,0x0004f9b989eabd11,0x000976a80f23fa92,0x0000171f16dad870,0x000d3baaac454e44,0x000f34704b9635c0,0x00051b0e1ca863f0,0x0000000000000000}, {0x000fcc07647c957a,0x0007e8420a4e5f40,0x000cec847e324cf5,0x000db2fe1f189869,0x0009827f88deeb20,0x00023c17a4487124,0x0009568284e63444,0x000010b9bfa3fe02,0x000626ee2d426c18,0x0006c496cbf2d807,0x0000000000000001}}, + {{0x000a48e1016cc257,0x000e7a08472093bf,0x000b7492e4ec8ac9,0x0001bd1c471d73ed,0x000a7243428ce53d,0x0007d49d1740adec,0x000c66dd60077c0b,0x0005737593cb7a95,0x0007c6f6ef237e1e,0x000660a7929ff06a,0x0000000000000001}, {0x0004bcc2e6c3da59,0x0003e87b3c416f89,0x00021231992a493d,0x0006d0824c0a5993,0x000e60d5c6c61f37,0x000a3430f29e1550,0x0005e81f1d1beb92,0x00027b6bc5ab11f9,0x000858c60a78bae0,0x000f95bdd9ea9017,0x0000000000000001}}, +}, +{/* digit=130 [{1,2,3,..,}]*([2^520]*G) */ + {{0x0008b69f6c0a103c,0x00014cb96747cd0b,0x0009e707ed23f30c,0x00092336029690c8,0x0002b884265ced12,0x00087e627555cfec,0x0008f62a097ab7dc,0x000df635118ea555,0x00070b69e23dda1a,0x00011d1b70ffff57,0x0000000000000000}, {0x0002606bce985157,0x000cf38c9c6bbd4f,0x00058d1b308ccb92,0x000fb6fe2ce6913c,0x0005b1967bfefbbd,0x00053132fce5cb51,0x000b2f22527584e4,0x00082a1591dfc389,0x0002d57610460d61,0x000dc57498b0d160,0x0000000000000000}}, + {{0x0008cffee5938175,0x000028a135a61fb3,0x0002c9fe1a18ac17,0x000b968e30418249,0x0000b4c958b813ec,0x00023285e86f834a,0x000b05e54df836f5,0x0008f77b5fb229e4,0x000371505e864d89,0x000fb3cce3c32e48,0x0000000000000000}, {0x000a0e5e3ce8bcd8,0x000dc2fb74fd48c4,0x000875aaa2018996,0x000827185e9f86f5,0x000196642422b5eb,0x0006d6f0dd874310,0x000f5719d7e7982d,0x0005346044e6cb46,0x00002d250a348c67,0x0009ec517508afdc,0x0000000000000001}}, + {{0x000145140a7a865f,0x000095ded9f8c118,0x00040df2b19ba498,0x0007dec03834dace,0x0009e3687e664a0c,0x0009d971f2feee9a,0x0008796a032ee720,0x0003a0b2cc310714,0x000067d48b65bcec,0x0007f5d8c2af6546,0x0000000000000001}, {0x00015c856b094864,0x0004f779c821c5d7,0x00062cf6310bebe0,0x00086c833844c220,0x000322644f899899,0x00080ac439439357,0x0009f2dfd741ec2f,0x000c3022b589299a,0x000663af1bcf4790,0x000a46767587f6f5,0x0000000000000001}}, + {{0x000cdfa0667f8929,0x00055eda7b7826f1,0x000dd93845997fea,0x00073f54c39699cc,0x000d85971fbd417d,0x000631f8049f08e1,0x0003b8e65a870ee3,0x000aa3cdc8e351a7,0x00099901454f69b2,0x0002d659cbe0fe85,0x0000000000000001}, {0x0005bfaeb9c77879,0x00078817f6a7aa08,0x0001e5f6c61e07f9,0x000a5dffb271762a,0x000df5d3e3cd3cc6,0x000eadd52e895eb0,0x0003b342cd665b46,0x000393078c5b9252,0x0006202122ebcff3,0x0005f4777dd50c48,0x0000000000000000}}, + {{0x0003fcdbd57bfa90,0x000d1b118dee98fe,0x0000a524ceccfc4a,0x0007420c6d1ffa5c,0x00053919d859de0d,0x00081fb74544af55,0x0007c3f0981d6def,0x00076680c297a17a,0x000d22135e0cc4fe,0x000d15de36e57af6,0x0000000000000001}, {0x0008ae03fd78b0a1,0x000c06ac0c41950b,0x000c994e2c7c2638,0x00075a944523ebb6,0x0002bdb72a76549b,0x0007a73d0573310e,0x000ae4c8ce6d6b85,0x000e4309eb9e1e23,0x0008aedc0842f06a,0x0006ededf71264f9,0x0000000000000001}}, + {{0x00047c62f7f81bbb,0x000568d4c38f8cbe,0x000085a52c049173,0x000151d87d057b19,0x000462de5c929612,0x0004036de18a4ea9,0x000290b0a8300ddc,0x000e195518bc5a6e,0x000e21097894ed83,0x000a74d8dd898c35,0x0000000000000001}, {0x000eee2e29a47676,0x000325c0b89ddfd0,0x0000c5c995a8a7bf,0x0003e514a5131a24,0x00078e5998b6e957,0x000ba59336939798,0x00032157906f3c87,0x0001093cead5ff51,0x0003bc2e1b76588a,0x000a055abc03c824,0x0000000000000001}}, + {{0x000c9d2081074b7d,0x000b4b234d223107,0x000c6fe7d17524b8,0x000b3e26469a9182,0x0001d35461f18688,0x0008f9ef1eb0f49d,0x00072164bb2130a7,0x00069240661ee72b,0x0004674d4ec1e4e4,0x000df195346b1529,0x0000000000000001}, {0x00094d4cd2beae09,0x00021cf533e3fea5,0x00099051c26db75a,0x0000237aea1a808b,0x00066230cfcd19c1,0x0000aedfcfe63965,0x000965518cdd9349,0x0000446502d88c39,0x0006de0650de6be1,0x000c487b3640aa4a,0x0000000000000000}}, + {{0x000315f2c4f3b202,0x000844f2e07c55f6,0x0009040fc8f62ee6,0x0006bb02af168881,0x000eea1922677aec,0x0003b90132968e4d,0x000b75f92b21f9e5,0x000b4ae4c1d62b9c,0x0000f373265408d1,0x00013d70be8f94d2,0x0000000000000000}, {0x000c00995b2d1ac5,0x000acf4e16ae2b05,0x000d04ee882f20b5,0x000b5e061f8c5834,0x00040c9dbfe4135d,0x000f7b55b63a42f5,0x0003b47500e266e3,0x0004da6911e652ce,0x00043c4db0c53d0a,0x0009946052a6d7ee,0x0000000000000001}}, +} +}; + +#endif /* IFMA_ECPRECOMP4_P521_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecprecomp5_p256.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecprecomp5_p256.h new file mode 100644 index 0000000..f5e41b1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecprecomp5_p256.h @@ -0,0 +1,968 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_ECPRECOMP5_P256_H +#define IFMA_ECPRECOMP5_P256_H + +#include + +#define MUL_BASEPOINT_WIN_SIZE (5) + +#define BP_WIN_SIZE MUL_BASEPOINT_WIN_SIZE +#define BP_N_SLOTS NUMBER_OF_DIGITS(P256_BITSIZE,BP_WIN_SIZE) +#define BP_N_ENTRY (1<<(BP_WIN_SIZE-1)) + +__ALIGN64 static SINGLE_P256_POINT_AFFINE ifma_ec_nistp256_bp_precomp[][BP_N_ENTRY] = { +{/* digit=0 [{1,2,3,..,}]*([2^0]*G) */ + {{0x00030d418a9143c1,0x000c4fedb60179e7,0x00062251075ba95f,0x0005c669fb732b77,0x00008905f76b5375}, {0x0005357ce95560a8,0x00043a19e45cddf2,0x00021f3258b4ab8e,0x000d8552e88688dd,0x0000571ff18a5885}}, + {{0x00046d410ddd64df,0x0000b433827d8500,0x0001490d9aa6ae3c,0x000a3a832205038d,0x00006bb32e52dcf3}, {0x00048d361bee1a57,0x000b7b236ff82f36,0x000042dbe152cd7c,0x000a3aa9a8fb0e92,0x00008c577517a5b8}}, + {{0x0003f904eebc1272,0x0009e87d81fbffac,0x000cbbc98b027f84,0x00047e46ad77dd87,0x00006936a3fd6ff7}, {0x0005c1fc983a7ebd,0x000c3861fe1ab04c,0x0002ee98e583e47a,0x000c06a88208311a,0x00005f06a2ab587c}}, + {{0x000b50d46918dcc5,0x000d7623c17374b0,0x000100af24650a6e,0x00076abcdaacace8,0x000077362f591b01}, {0x000f24ce4cbaba68,0x00017ad6f4472d96,0x000ddd22e1762847,0x000862eb6c36dee5,0x00004b14c39cc5ab}}, + {{0x0008aaec45c61f5c,0x0009d4b9537dbe1b,0x00076c20c90ec649,0x0003c7d41cb5aad0,0x0000907960649052}, {0x0009b4ae7ba4f107,0x000f75eb882beb30,0x0007a1f6873c568e,0x000915c540a9877e,0x00003a076bb9dd1e}}, + {{0x00047373e77664a1,0x000f246cee3e4039,0x00017a3ad55ae744,0x000673c50a961a5b,0x00003074b5964213}, {0x0006220d377e44ba,0x00030dff14b593d3,0x000639f11299c2b5,0x00075f5424d44cef,0x00004c9916dea07f}}, + {{0x000354ea0173b4f1,0x0003c23c00f70746,0x00023bb082bd2021,0x000e03e43eaab50c,0x00003ba5119d3123}, {0x000d0303f5b9d4de,0x00017da67bdd2847,0x000c941956742f2f,0x0008670f933bdc77,0x0000aedd9164e240}}, + {{0x0004cd19499a78fb,0x0004bf9b345527f1,0x0002cfc6b462ab5c,0x00030cdf90f02af0,0x0000763891f62652}, {0x000a3a9532d49775,0x000d7f9eba15f59d,0x00060bbf021e3327,0x000f75c23c7b84be,0x00006ec12f2c706d}}, + {{0x0006e8f264e20e8e,0x000c79a7a84175c9,0x000c8eb00abe6bfe,0x00016a4cc09c0444,0x000005b3081d0c4e}, {0x000777aa45f33140,0x000dce5d45e31eb7,0x000b12f1a56af7be,0x000f9b2b6e019a88,0x000086659cdfd835}}, + {{0x000dbd19dc21ec8c,0x00094fcf81392c18,0x000250b4998f9868,0x00028eb37d2cd648,0x0000c61c947e4b34}, {0x000407880dd9e767,0x0000c83fbe080c2b,0x0009be5d2c43a899,0x000ab4ef7d2d6577,0x00008719a555b3b4}}, + {{0x000260a6245e4043,0x00053e7fdfe0ea7d,0x000ac1ab59de4079,0x000072eff3a4158d,0x0000e7090f1949c9}, {0x00085612b944e886,0x000e857f61c81a76,0x000ad643d250f939,0x00088dac0daa891e,0x000089300244125b}}, + {{0x0001aa7d26977684,0x00058a345a3304b7,0x00037385eabdedef,0x000155e409d29dee,0x0000ee1df780b83e}, {0x00012d91cbb5b437,0x00065a8956370cac,0x000de6d66170ed2f,0x000ac9b8228cfa8a,0x0000ff57c95c3238}}, + {{0x00025634b2ed7097,0x0009156fd30dccc4,0x0009e98110e35676,0x0007594cbcd43f55,0x000038477acc395b}, {0x0002b90c00ee17ff,0x000f842ed2e33575,0x0001f5bc16874838,0x0007968cd06422bd,0x0000bc0876ab9e7b}}, + {{0x000a35bb0cf664af,0x00068f9707e3a242,0x000832660126e48f,0x00072d2717bf54c6,0x0000aae7333ed12c}, {0x0002db7995d586b1,0x000e732237c227b5,0x00065e7dbbe29569,0x000bbbd8e4193e2a,0x000052706dc3eaa1}}, + {{0x000d8b7bc60055be,0x000d76e27e4b72bc,0x00081937003cc23e,0x000a090e337424e4,0x00002aa0e43ead3d}, {0x000524f6383c45d2,0x000422a41b2540b8,0x0008a4797d766355,0x000df444efa6de77,0x0000042170a9079a}}, + {{0x0000b650bc6fb805,0x0004effe2e6b808b,0x00083f5495882e07,0x00072385ef2f7c2c,0x00004d63c80e103b}, {0x0001bd652a23f9b6,0x0008eb0b6587f2f1,0x000580e9e3670c31,0x00021ff5c4623bb1,0x00004edf7b261efe}}, +}, +{/* digit=1 [{1,2,3,..,}]*([2^5]*G) */ + {{0x000fccfc5e3a3d83,0x000c1079dfbfd8c5,0x000ad0197befd904,0x0002a48c6d6a58fe,0x0000922707799553}, {0x0003e6ddbef42f56,0x0003e80a990809e2,0x0009a2e407e449b6,0x0002a41b969c1aad,0x0000231d792f591c}}, + {{0x00096f305968b809,0x0002789f73b9db6d,0x000c61e01380a091,0x0008c6eda70b83c2,0x00005fb8394e69b3}, {0x000651280edfe2f2,0x00096faeaf829a3c,0x000424bf88f726bb,0x0009706010a4a078,0x000096720442e844}}, + {{0x0008d2333e12b701,0x000b09dd329b802b,0x000bc354d6d490a4,0x000a0d04f356cc6a,0x00001eddf7fe0a0d}, {0x0008328d87fd1d83,0x000ccd0258131e20,0x00029bca2fd2f4f8,0x000b70d3b48cc47c,0x0000f2a78b3541a2}}, + {{0x000cb817a2ad62aa,0x00090c62ff5463c5,0x000ad9db57ef2b6b,0x0006169749bba4b3,0x0000d311f2ce6d5a}, {0x0008087c2ff3b6df,0x0002467834ffb77a,0x000d6b138b46feaf,0x00018808aa266d75,0x0000a38d321dc008}}, + {{0x000fa1ae3451a09e,0x0002dc117423c04a,0x000cf56ee7cc6910,0x000a24a76be3aa51,0x00007577d588d844}, {0x000ed8cdb77f3410,0x000c23ae4a2a266f,0x000782760cfa258d,0x000d4a53a7a98cda,0x00004b48868ceaf7}}, + {{0x000affc4e6916c6d,0x0006b11842dac0f2,0x000b4d1576fb9495,0x000b8f9034bcb624,0x0000e2efdc8692ef}, {0x0007eceac793c87f,0x00058dc6fdcdd66f,0x000a1c50102f0262,0x0006efa0d3235c9a,0x00006e494971b466}}, + {{0x00036d19c0859c3e,0x0005d269386a0827,0x000a87b3389ea551,0x000236125071871a,0x00009d82f60504e8}, {0x000612442e855f5b,0x000d895e00d87834,0x000e7e62b209f50f,0x00037b1d9e03aae6,0x00004b4959f0e5be}}, + {{0x0008ffa696946fc7,0x000849cba56d486d,0x000f35a1550fbc6d,0x00062c0e3d423e90,0x0000c3da19630dd9}, {0x000fdb03cfd5d8b3,0x0002589dfca5e673,0x0002305aa0704b7c,0x000e53c6ce581ff5,0x000099d49ebc14d5}}, + {{0x000afbb10e6d9501,0x0004d2bf970a7966,0x00054ca2a37e4a4c,0x000a1013d0c8559d,0x00003d628660ee39}, {0x0003c38d3bd15e94,0x000d44a80bcc15f5,0x000d8f608014b8be,0x000feed0674c77bf,0x0000dfab987093fb}}, + {{0x0007c367b92d4531,0x0002149a26c024c9,0x000d14a39d2c271a,0x00000550eb4b2b89,0x0000198de20532e8}, {0x000ea75799b80d93,0x000b5f826ae59eab,0x0000061ecab6e0c2,0x000fd89a004eedd1,0x00001a9f43a199c4}}, + {{0x0008b75b5498a7c3,0x0009f9f64c2d734c,0x0005b1677aeccad8,0x00096865dd54fa29,0x000083902a0e4c54}, {0x000276feb9d33a9f,0x0000bf55286978cb,0x0005c163000aedca,0x00050cb01d14594c,0x0000eba17076cf7d}}, + {{0x000496d6ec293cde,0x0006ae7051f5380a,0x00049140a733dbda,0x000bf5237e388db8,0x0000e4b32b13946d}, {0x000fda9cae368d14,0x0000bdb0b2f3b1c4,0x0003ac46e5001a7b,0x0006562df593742e,0x0000af675f279b3e}}, + {{0x000ffdcc01b0a469,0x000ef3f843c21a1f,0x00038e81a07ad675,0x000da82cec2d0767,0x000010aec763a8b7}, {0x000d740a4509ba71,0x00033c7b821613b4,0x00091f1e50570107,0x000d902d75c8f715,0x000034c1b6f1dfe1}}, + {{0x0007c44d67826eb7,0x000f3c379637b20e,0x000e03a5d1212d3c,0x000a25f14f67877d,0x0000538a2fcf02ba}, {0x00037a19d252415f,0x000f7eb4b587133c,0x00049d1d97db3904,0x0004b96d40d7574c,0x00001126b9a9801c}}, + {{0x000aeb9c56729fc9,0x000857a8bd8539a7,0x0009d10c621a5944,0x000afb47c6da5d04,0x00003a4c4a955197}, {0x000539c259338612,0x000a07494d9dd185,0x000e7bfa94994bcb,0x000236cf033de14c,0x00002e9b000521f9}}, + {{0x0008110399492969,0x000aa61db1b544e3,0x0006eaff55b63827,0x00028fae5323ed20,0x000042370d3521f4}, {0x000af2ee0d985a17,0x000b0239846df2ca,0x0006312f8192cc64,0x0001080c0b8f47ae,0x0000dc61f9206620}}, +}, +{/* digit=2 [{1,2,3,..,}]*([2^10]*G) */ + {{0x00081e144cc3adda,0x0005e7be82cf4f70,0x000dd6472d5ffa1d,0x000862e9890b6c0e,0x0000da26e1aded17}, {0x000271563483caaf,0x00083f6077fd276f,0x000466e3ce6924cd,0x000d1e15a7fe980a,0x00001c794b1a1902}}, + {{0x00024b9eb7926b83,0x00039dbe55093d2b,0x000dd640bbff88cb,0x000d45a0f399afe4,0x0000c5fe1305f76e}, {0x00062f43764fb3df,0x00074151b62d6f35,0x0009ce5f37b5af31,0x00090ee5bd0bc7d7,0x0000daf6b21dc668}}, + {{0x0007fe891e5d7d3f,0x000683a076783202,0x000dfdd61f14b7d1,0x000f48088497b3c0,0x00007c2eec11a8c4}, {0x00073f43756e621a,0x000f7825b948aa55,0x000878572c013a23,0x0001837c03b34563,0x00000472beb053a4}}, + {{0x000b0e5ab4b35a24,0x0001b5eeaacf6772,0x0005b95801d8b600,0x0001da328f7ce479,0x0000a20ed2a81fb8}, {0x0005cd44fec01e63,0x000c77ff50ad9f68,0x0002d97fd3ed7ddc,0x0004f9160fd2640c,0x0000a2414271b82f}}, + {{0x0002cf983dfedc91,0x00047d87631a29ae,0x00029c8d2f843713,0x0002729f57171174,0x00008d15867246d9}, {0x0003ecf69769bb7a,0x00062479ab828305,0x000b0f4b2c55eb85,0x000524bef7791c21,0x0000a5956badd491}}, + {{0x000bfa1c5c5ea50e,0x000b8796068184cf,0x000d50942d3baf14,0x000662463984030d,0x00004b7839d2716a}, {0x000f794e7de6dc08,0x0003e22aa7ced5f1,0x000acfeec5cd0f4d,0x000606d295f3f159,0x0000d933553153e0}}, + {{0x000533b3cef0d7d8,0x0007abbb4381e652,0x00080f500d94f7b1,0x000bfb038752be0e,0x0000e6e24891e9c9}, {0x000169716caca6a2,0x000818531ad9c975,0x00051ade1866c49d,0x000407a917e23971,0x0000d016ec18037c}}, + {{0x0009862d5d721d5b,0x0002abd3a1828000,0x000a2cda40c3357a,0x00008477f3a83b7a,0x000058ae74fa6f83}, {0x0001a812e6dad6be,0x0001143d6c5b2a91,0x00096c4d8de28605,0x00024d6bdccc41f9,0x00007312ec0eae1e}}, + {{0x0006e3960e913af1,0x000e88bf140d903f,0x0004890b8b2b58be,0x00024e8deff02535,0x000055810069d2e9}, {0x0005db493c95e5ba,0x00003ae20eb8b575,0x000b6d8e03fac42f,0x000efef377c8c109,0x000043e2b474b47c}}, + {{0x000b2fbfacfa4591,0x0008b1b5aa6b5f57,0x000db2092874b149,0x0001daa9e89acac4,0x0000362bf8def438}, {0x0006830b76328a07,0x000028572ae425d7,0x000132f7d38188b7,0x0002c9443e941429,0x0000895a29f92dd4}}, + {{0x00078dd5e22cbb29,0x000ae6bb4391cbde,0x000a4273bf449c85,0x000a18b289f357b6,0x0000fce23fdd8e84}, {0x0002730939eb3b47,0x000ba6c32280cfc3,0x000f1346c8b3d982,0x000fc8eac234bad5,0x000081954b4e0769}}, + {{0x00068feaae6ee702,0x00053602b0c96faf,0x00094052a78f4cc1,0x000d805e3321a86e,0x0000fb3a0d6934d5}, {0x0008f3bb25a43ba1,0x000109ee2951f3b9,0x000b0612a30bf803,0x0001d06ffee43321,0x00002f775e43eb82}}, + {{0x000f805f57209b51,0x0003e952ac8d4fdf,0x000969a6f9bd65ac,0x00075ef2a3abd3c7,0x0000359927f05237}, {0x000463f88d2e8610,0x0009623287c3e09b,0x00070eb7a661d219,0x000684821e64495a,0x0000afbbb1dd67dc}}, + {{0x000e3d3acff89f9f,0x000de852251f7418,0x00084658b227f16a,0x000f7ded5bc6e4eb,0x0000066b9c9e90a9}, {0x0009071800a7f873,0x000d2a72862ac236,0x000776da5383ddc0,0x0003182b48465d8a,0x0000d82f64f8e2d8}}, + {{0x000104b87453b283,0x00088387344d5852,0x0007cfc08073e812,0x000088000e78e481,0x0000a82ed47c9362}, {0x000304c88de46a44,0x000d17fadf4ae222,0x000c8e108666c94f,0x0000fa00b2d08ea0,0x0000b2955b949e05}}, + {{0x00012e76e6485b37,0x000b071c52f8f8d1,0x0004a2f6d4d3e24d,0x000550d8e3ee4168,0x0000161957d91d95}, {0x0001283cdb12a6c3,0x0001fe50e1641963,0x00066cc73bf3fa88,0x000c38c6254b6331,0x0000aefa7aedee8c}}, +}, +{/* digit=3 [{1,2,3,..,}]*([2^15]*G) */ + {{0x000fe623b36f9fd2,0x0003dde19fc079b0,0x0008482ef26543b2,0x000824f36e64a095,0x00003f63771bb095}, {0x000d596b6a1142e5,0x0005e35aac0b14cf,0x000081dd55ea6aac,0x00012a36a0e8bdf3,0x0000fb89d79503dc}}, + {{0x00065fce3779ee34,0x000d7d495d9e0f01,0x000284e7ae00e7f9,0x000218dfa4efa220,0x0000564bade87ac6}, {0x000312ac4708e8e4,0x000b671e9adf90e6,0x000684b9f4f5725f,0x000415a95f55ae3d,0x00007f7ccb15e94b}}, + {{0x00077e5522e6b693,0x00038bcd6c18da3a,0x00024fd5669c908c,0x0003f6ef1b9e48d9,0x00007c64e36da4bb}, {0x000dbdfee478d7d1,0x000bf193f7a05a4f,0x000cd16dfba75c8b,0x00015174bc1e8456,0x0000fb08f0856fad}}, + {{0x000890361a341c13,0x000fdcfd61423617,0x00033316c3604dc5,0x000921d22295eb85,0x0000dbde4ac74af2}, {0x000fc5d1c7eef696,0x0005714f4fa1898a,0x0003c21ca5889680,0x00030aa500216020,0x0000f0d1f30a0ef7}}, + {{0x0001b4b12cfe2978,0x000ba92f74e54820,0x000e874e83eee129,0x000c4161fe114ec9,0x000099b055d12c5f}, {0x0007a643a39c8cf8,0x000df8963cc94e47,0x00033f86382f09ef,0x000c62efd3fd8fd3,0x00005132b2b5c949}}, + {{0x0008b1dbe7a2af37,0x0008dfb74a72bd9b,0x000879697ec51caa,0x0007d549937a4b63,0x0000c9a9d215c268}, {0x000e44f6ef5f0145,0x0002990c69001773,0x00042161e8abcf41,0x000f29e87bd02281,0x000003937564cb6f}}, + {{0x000072232398baaa,0x0001bcfca031766e,0x000029cf2205fee4,0x00090d049f53417a,0x000088c68b8e0238}, {0x00050417337aaa82,0x000ceeb384f4bc27,0x000aba92f9ed364a,0x000a88c0816f8529,0x0000e9e194124e38}}, + {{0x000770977f7195ad,0x0006ddeb838ffabf,0x0004f012d8ec8616,0x000b3f1a1285a8bb,0x000068835046a3ea}, {0x00024f8309004c28,0x000593ffe95eee5d,0x000223ea4a96e4b7,0x000a528cdffe12bd,0x0000f5c2ee636739}}, + {{0x000e1dda1887395c,0x0005d32a65deecac,0x000a9552940960f3,0x000a35d611ff5c3a,0x000058215b13c1e5}, {0x0009b58f0e1a5240,0x000bf590dfb8d48c,0x000d95662b406856,0x000f82c7605e049c,0x0000dd036eea33ec}}, + {{0x000333959145a65f,0x00080a4063373d61,0x0008a52a0cd9bc36,0x00058f92d11be32d,0x00006877b2887a1c}, {0x000819bf5cbdb258,0x00085e090249837a,0x000990e5f2a4fd1d,0x00011ae22a7de774,0x000040fa5a0f9455}}, + {{0x00093277946d3f97,0x000397472273fe28,0x000b6ae86e132bd2,0x0000677eeb510c1e,0x000077708c660595}, {0x000c8cd1297029e8,0x000c3bf9305e18e2,0x00085d6d92c61095,0x0005306466c2586b,0x0000ac06c375a1ea}}, + {{0x00090b36b0cf82eb,0x00057615b5e7e58e,0x0009c145a6438d24,0x0001ca57b1f8fc66,0x00000d8b2dae6f1e}, {0x000dadbd9184c4d2,0x0005d93d997654d5,0x000147d473dbb18d,0x000608ea3e0f56d1,0x0000afa8c8dc0a48}}, + {{0x00039ad653ae3263,0x000a774cbb438712,0x000d4c08314bcf72,0x0004af5737650e20,0x0000df86536410ed}, {0x0006fe7b53ca5557,0x000d3bd5d538d2d8,0x000d38468688cb00,0x0007b65f81bda31a,0x0000ccfe3cd60116}}, + {{0x0008c07e3533d77b,0x00097e341c9926e0,0x0002dc4edd7222e6,0x000cf7ed60ec3d8d,0x0000dfe0d902c476}, {0x0009ab61d056605f,0x000596a8551f1fe5,0x000fb8d8ca9ea9df,0x000b0f9489941e47,0x0000eb874ec3a7f1}}, + {{0x000181060a716760,0x0008f66a8ad161fc,0x00017231ee852d1a,0x00011f172bbd6564,0x0000d6de7bd3babb}, {0x0006f88c8e347f89,0x00070bd99cc36fde,0x0000769501c58754,0x0003b9e8e54ed034,0x00007f0f335096e8}}, + {{0x0006aa9bd7638026,0x000005303da1ed40,0x000e62ec4c21486a,0x00033e01ae291ec7,0x000022a04933f993}, {0x0000c9dbb7a8ee0d,0x000b9c01aedb7fd8,0x000be74ecdc2ed3b,0x00071e65c35a1208,0x0000540cb1b169f6}}, +}, +{/* digit=4 [{1,2,3,..,}]*([2^20]*G) */ + {{0x000746a584c5e205,0x000169dc7035a7a8,0x000548c9b267e4ea,0x0002f3093a15cfb9,0x0000e6e21359bd01}, {0x000cc6a8c8f936e6,0x000455c241dcdf31,0x0005efb868af84d0,0x0002cb03990a6f34,0x0000fef4e6219b96}}, + {{0x0008f09257226088,0x000a931cf5c6f636,0x000b4f7ac131260d,0x000828c0eb353bfa,0x00005c78880b7eee}, {0x00081ffc3bdf24eb,0x000b45c3c5a84c15,0x0004e6f405bff75c,0x0000c985e8c83fa1,0x000081d1c0fb295e}}, + {{0x000e23d442a8ad11,0x000cf6b9c164f2ef,0x0000aa5e5c3816a7,0x000e6599df2d8bdc,0x000091ae46f220a8}, {0x0007f8700611c5bf,0x00070f1099488366,0x00069595283171ed,0x000a1243a2ecf8ca,0x0000a4a73efe48d1}}, + {{0x0007cc8f43a730f8,0x000bb3ab590efcde,0x00003240be89b6f3,0x0005db4823f529ad,0x00002b79aff18bea}, {0x0002856962fe5de3,0x000b30c591f3568f,0x00028a8580c590ad,0x000f4befc74a144a,0x0000b662498e3203}}, + {{0x0004ed082dd1b6ad,0x000797b703af48fc,0x0005d6aaa5783a13,0x000d425463cb9a00,0x000031ec55d406ec}, {0x000d33f8e9a76414,0x000cc98d9e7a9f8e,0x000887493625453e,0x00056663beade4ec,0x000042b80509a795}}, + {{0x000cf0d6c39765a2,0x000d8c3cca0b91e3,0x000953b50a2db3ac,0x000f1a088f2f08cb,0x0000414582cef43c}, {0x0008bbc60eee9a8a,0x0001d29aa0428dec,0x00032f5d554c79f0,0x00015f381cd5ec65,0x0000672303b6f82e}}, + {{0x000582d3bfab839b,0x00037f8adade46df,0x0007a1bc392474e0,0x00097886a7766a14,0x00006940f54bdc0f}, {0x0008ef2f2759f255,0x00095719f4c64473,0x00050c3459dd9578,0x0000d4d859b7f407,0x0000e788bf302218}}, + {{0x000afa8719c05631,0x000cac5fc79f376a,0x000750cd3cd8ad2d,0x00008e203fdb9fcb,0x00004ff052f5418b}, {0x00084cf3e2d65208,0x0007944ed509f750,0x000f25b987ebdf0f,0x000837743bf0f2d3,0x00006ad71d02354d}}, + {{0x000c9fdfd67ca259,0x0007d0f2015ca839,0x0007b2a65023e626,0x000cba9414a7930e,0x00002dbe373413ed}, {0x00081ee64c2200f7,0x000f1446f2f3f649,0x0001367bb94fb9cd,0x00033091411a6a3f,0x0000985c191ca1e8}}, + {{0x000fe9226f435728,0x000add824758b827,0x0009094c1dfd3ab5,0x000d67b15dd23a53,0x00005c0e37ae6623}, {0x00079727be19ae06,0x00067f0d36b5575c,0x000b1ff7e616a339,0x00045341ebb3c826,0x000035b9485740ad}}, + {{0x0003c6037e2efeaf,0x0001134a96f6c812,0x0003e4a958d49b50,0x0000fe566a346b97,0x0000176b5bba7de0}, {0x000fa3b82dfa945b,0x000559e429ae1c58,0x0002b187c2eb27a9,0x000bb9a7c67a67a1,0x0000155ba8392298}}, + {{0x0003cdada430c0b9,0x000daa96dac692bf,0x000ac326a4702850,0x0005e4391cf0a515,0x00005de4f4a3b8c2}, {0x000ad09e265c17ce,0x0003287b3881b01b,0x000fac5ca24e4546,0x0007a5f43e583ce1,0x000017cb3194ead9}}, + {{0x00042073d99bcfab,0x000c38becf6df1a5,0x00045956959db703,0x00090f7e455142d2,0x00000ee51445901b}, {0x000d451e26d994fa,0x00037360caaffc05,0x000a639b17a6062b,0x0008b03f1ded5f4f,0x0000f9303497335b}}, + {{0x000924374dcec468,0x000cd4c2b73f6cc3,0x0006cd99c33cfc02,0x000f8902917844f2,0x0000819dd9651773}, {0x0002aa60871f4274,0x0005b6f01c340957,0x000f1f5af8e0cf36,0x000e503fa52988bf,0x0000eb357eb275e8}}, + {{0x000248a21fd08616,0x00043bd5a4b63d8f,0x000e2a6bfade3bd6,0x000c5f6b56c953c2,0x000099cd2b5887d6}, {0x000e1be47d05e8f9,0x000318f53732debc,0x000852b081a4fbb1,0x00087a07163beaa5,0x00002c49e6d7ec69}}, + {{0x000c8c4868af75df,0x000e55c8c7ead9d0,0x00081ecb0d7325cf,0x0004ecbb471996cc,0x0000f5d55f451182}, {0x00045411977a0ee8,0x0004f22038c6be31,0x0004bb4955085c4c,0x000081ad5335bff9,0x000094ad8a748e2a}}, +}, +{/* digit=5 [{1,2,3,..,}]*([2^25]*G) */ + {{0x00034b22c11bb373,0x000dbd4c74a35402,0x000c5f25d2d0366d,0x000142c9a968daee,0x0000660106897b63}, {0x0006d2c68d7b6d44,0x0008cc84294207cd,0x00068b1eea8f74f0,0x000ee4a275140477,0x0000b5f7e8a3e62a}}, + {{0x0009f0b879fbbed4,0x00069a9d1869f236,0x000766f450ff0ae8,0x0000fc1251d75956,0x0000984d8c06be8d}, {0x00095a6d21008f03,0x00000a1a1c497ecc,0x0006c50f329bd54a,0x0002517b9828c5d2,0x00002c0087c81d0d}}, + {{0x000eb6662b5f3af3,0x0000dabb373447fe,0x000f35cb1cefab56,0x000eb9149de60e19,0x00009f8db14457f0}, {0x000cc5b3c61bfd60,0x000d41216703ffae,0x0004e1cc2a5a4d41,0x0009537f8fabed22,0x0000d5a8186871ad}}, + {{0x0008ea601799a527,0x0001486d2952190d,0x000ff2a7ca20cec4,0x000d36c062ffb27f,0x000041b32e5e9f19}, {0x00081814eb57d471,0x000406aef06bf80d,0x000ecb5887a2d0ed,0x000f5af9735fb01c,0x0000641caaad6061}}, + {{0x000d171656e8c3aa,0x00027acd0705ae2a,0x000b6055cc0e2a46,0x00026d606f6a8aa0,0x0000f4513d7cb65a}, {0x0009e14d616d5bcb,0x0004c1253b1f3f54,0x000ce243a64ee395,0x000e7738b10bc1b8,0x0000cbeace6413a4}}, + {{0x0002256f779d6a77,0x000ca9e7284e68d3,0x000f3b15320423b4,0x0007036e19aa1b38,0x00003d0b6c2f444f}, {0x00061489c64e6a37,0x0006c72562246661,0x00033156399587c3,0x00002dff5277fe61,0x0000174ad4b16565}}, + {{0x000fc920133918d8,0x000e32c9ef97ec1f,0x0006cbb9a15d9bf5,0x000f5cf885b542fc,0x0000abe6453d720c}, {0x0005dced4ec68aba,0x0003879c24a34e71,0x000bb61cf57a6761,0x000002d88ed52a31,0x0000ba824459437a}}, + {{0x000824f20151427a,0x0005f24302067f99,0x000112357206828b,0x0004ec0a9097d7e1,0x0000cf9a2f2a9e41}, {0x000c9da279153564,0x0006c01efee3dbda,0x000b288e27e0734b,0x00009c14fab5bbd2,0x0000c630fc5362dd}}, + {{0x0003e66bd70a5e18,0x0001505de7136d88,0x000d8487d9bd884c,0x000404c8d9d445d4,0x0000440cd8b8eeae}, {0x0000f293d26f83eb,0x000f72a3c5e4a3cd,0x000fa2ca73fd9452,0x000096b3a078663f,0x0000feadd29b0d4a}}, + {{0x000a559d68432ba6,0x000b733915db13ae,0x000e8da08940043f,0x0005bae76c7c94e2,0x0000b1630c4a4306}, {0x000546d19f06c75d,0x00015eb35abdd6d8,0x000157d8e7e1e98c,0x000074a51c9310f0,0x0000ef84328c6fa6}}, + {{0x000993c3a7a4e0e9,0x000f0d79d314765b,0x00019eb24fb8f4ae,0x000c8bcea2c50e2a,0x000025016e64891b}, {0x000afdc3ad1ece76,0x000c908fabe28f70,0x000f6132b0abba84,0x000d76aa9922c815,0x0000c021373bd076}}, + {{0x0002615a44539610,0x000b993d112c00af,0x000cdd6521705494,0x000cb30032e12aa1,0x0000ebfa612046d9}, {0x000a9f31b63728c8,0x00025e022b059f03,0x00077c5c98f3618a,0x00077c55b24d903e,0x000037838a3f2acb}}, + {{0x000cbd98fceb0471,0x00052f8b11a1fe65,0x000ec802fde872ef,0x00081d8aba0d3d8f,0x000039f1d32aa9f3}, {0x000aed9c587958d1,0x000ca9f6a7da0721,0x0005d9d06066a015,0x000a1ef59ec4e3ed,0x000044285717cbac}}, + {{0x0000fffe298cdf56,0x00041b2e51b666fe,0x000d3afa43f61bea,0x000e2f1d372117ba,0x0000521a09d4f656}, {0x000c966e8a58fe76,0x0000fa47ebc7b3b8,0x0004be57325203a1,0x00003c9e81588d5c,0x0000132e2f37f49a}}, + {{0x000b0130f8c2c991,0x000fe17725c2ec1a,0x0000a980b60e8968,0x000cb1d1a4a59394,0x00005ed15b030e9c}, {0x00054cc64a00bec8,0x0002c168738277d7,0x0001fb65190d09c3,0x000e52a94fe02dc3,0x0000372cd1a9c94a}}, + {{0x000107a1ac2703b2,0x00084bc857b58537,0x000daccd1b49258d,0x00052937df14debc,0x00004ab68d7e4ae8}, {0x000b5d4734e59d08,0x00084495cc807ed8,0x0001db9b35f8740c,0x0005be04aedd5a29,0x00000b360f8cfb99}}, +}, +{/* digit=6 [{1,2,3,..,}]*([2^30]*G) */ + {{0x000c68da61a76fa3,0x000d9a1554dc55d5,0x0003b279c598b441,0x000efca39923b977,0x00003331d3c66bf9}, {0x000848e298de399d,0x0006d1a27f562d4c,0x000b8ab70cfdb8e7,0x0009b9c4c855ea57,0x0000cdb9daf3f787}}, + {{0x00007367f636a38c,0x00064e76d5cb4c4f,0x000b68b8b9f943fb,0x000a1ef03510baa8,0x0000246780b5ed07}, {0x00014156d549fc2b,0x0000b07781ca3c05,0x000d95413c2953f3,0x0002e2e55e2c69d8,0x0000300fadd2bd28}}, + {{0x0001163c27901b4b,0x000fd99b8bf3a14b,0x000c6da0afd9236d,0x00029692b091eccb,0x0000b1dac700ad1d}, {0x0001d53a91cf76e1,0x0002c31f1ee780e6,0x000efcf774110a41,0x000d761d87c3ba13,0x0000f374bb4ef450}}, + {{0x00086024147519ad,0x000b56b372f02028,0x00085ebc8d0981ea,0x0008e8d9d4a7caa7,0x0000953c50eabdf5}, {0x00061ccfd590f8f8,0x000ac4e6c9179d63,0x000eb64cf72e9626,0x0008f2ffd9611022,0x000063ebb7f1eb28}}, + {{0x0000a0a097e4403b,0x00056985466515e1,0x0007d4826cb3d0a8,0x000838d8d8e211d6,0x000039af66ebb9d2}, {0x0004588bd475ca8d,0x0005f077b80ba5f9,0x00027c26ce06b796,0x0005e02edb1485da,0x0000290d33bce0fd}}, + {{0x00002f31306583d2,0x0000237c622e6862,0x0006a7bc805b10da,0x000e439f9aaa0f07,0x00005e94efbaf8f4}, {0x000c9b7fa3dc26dc,0x0002d6ff03c58a35,0x000c394cee0e5fb9,0x000e5fe77e3843eb,0x0000ede65964361d}}, + {{0x00075090c22b5403,0x000a87c267d4bf90,0x000b0d6932cde42a,0x0000d98a18f9ed5a,0x0000ba62aa69e466}, {0x000f97bab9ea96a6,0x000283b60e32b24b,0x0004d9bd55d03964,0x0006a3ee6a45067c,0x000066c5b9eded4a}}, + {{0x0007cf5678a31b0f,0x000d4998b620877b,0x0000fb396d50301a,0x0002a5834257c5c0,0x00009fb18a0f4e67}, {0x000d8ebe8758851b,0x0005ad99ba44ff8b,0x000fd93b71e64e4c,0x000b8b9b8eaedf7d,0x0000a2f2a98b4e76}}, + {{0x000a2f09b56ddbdb,0x000bda2f1cf3ae02,0x0009dff0d1339b5a,0x00043d42b569c783,0x00000b9e865aee9a}, {0x000ca4177bb064e9,0x00026d249f634ff8,0x0006f689a145a281,0x000f5daab7beacf8,0x0000bafec2791d35}}, + {{0x0003fddae57c7b7a,0x00017b932522bba2,0x0006d4aa3345342f,0x000b615d9c80fe55,0x000003907bb0525b}, {0x00010e1ff2189336,0x00044a52117b38b0,0x000f2e6eac066b65,0x0002b22e14192094,0x00006a27dca6d32f}}, + {{0x000b121fbabbe926,0x000b81330076b2e6,0x000890015281850f,0x0007f4a93581ec97,0x00009b1ddedd5ff7}, {0x000b18fab1051053,0x0001789ccfef7cf0,0x000914009953ced3,0x0008ad0151f85feb,0x0000c9f1b87b8ed4}}, + {{0x00007e0e90fb21e5,0x00006ba7fca1a18f,0x000cd67b500fd2b8,0x0007f6d0387f2795,0x0000b89a4e823970}, {0x000ad3f894407ce5,0x00041c2261328f83,0x00006c13ba0025b9,0x000025779563c7f9,0x0000f548f319e7bb}}, + {{0x00089494fde0c1fc,0x000e25b6ec20ff75,0x000e1db6cbf8a1ab,0x00058eb02278fb87,0x0000447ad7af5ed6}, {0x000aa3803d0ccf24,0x0008419a7c0348d4,0x00017cecc80acb33,0x000d825bc7c89e6e,0x00006736b8b43be1}}, + {{0x000f742d43b5eaaa,0x00063fa59b852126,0x0006bfd45054a076,0x000a8efd0d5e3612,0x00001f8fbd7d84f8}, {0x00080f5d563fccc1,0x0005e280a9283176,0x000b578cf48ca505,0x000514d00b81b227,0x00000aad9183994a}}, + {{0x000301f358bcdc04,0x0008ca9d47f8e63f,0x00043d43a07689e9,0x000903df689e2f4f,0x0000d542a16d0920}, {0x00093d59ca0a7072,0x00056ac68065aea2,0x00090008cd061fe4,0x000db5f033bf1b00,0x00009749558e08a6}}, + {{0x000d3a7c35d87949,0x00077356bae50ee6,0x0003322fd042e655,0x0009670f59698d64,0x0000379ae15e0a61}, {0x000ae62fcc9981ea,0x0000cd2934c664b9,0x0004e65ebaed3d63,0x0004278454b3025e,0x0000b09f64899950}}, +}, +{/* digit=7 [{1,2,3,..,}]*([2^35]*G) */ + {{0x0003a1222248acc7,0x000ec264e366b208,0x000fdee281f6ec0e,0x000bb4e659b7045a,0x0000a823a4156430}, {0x0002a04e1900a791,0x000ab9ee65762459,0x0005ea54acde09d4,0x0005a742b6463f4b,0x0000efe9ed3e3ca6}}, + {{0x0006dbe305406dd9,0x000f4d5d1957e27a,0x0007d4d8f8eb7dc7,0x000de4654a687638,0x0000c47940a57762}, {0x0005b5d99b307781,0x00065e793682be4d,0x000c740e325380c5,0x0004ae502d37f3da,0x000040deabe2566e}}, + {{0x000d067afd32acfd,0x000a11f71ccf4481,0x00096f2dad8f0fcc,0x0003f90208dd0cb5,0x0000049d7316aad9}, {0x000263d42ab580e7,0x0000b3f707b4c79f,0x0005e0eda09411bb,0x0004021cfde1ff83,0x0000270749100f03}}, + {{0x0006126c49a861ec,0x0005214f0d06eaee,0x0009bfc17024f3b6,0x00038091a3f1e8c6,0x00003c3a8ea67686}, {0x000752cb103d4c8d,0x0002c218b36b3400,0x00051504a02bc461,0x000bf9f67f75eb76,0x00006848b57a02ae}}, + {{0x00002e6c30fa92bf,0x000caa552784bd98,0x00083169b5a70d96,0x000227a085c4ea3f,0x0000a9423bbf6908}, {0x000be12fe97a5b9f,0x000881b991182ffe,0x00017884685da604,0x00018dacbc2f7f63,0x0000d96bc7181532}}, + {{0x00081db1782269b6,0x0008c597e5509583,0x000385153ae34bf7,0x0000485b5c60645f,0x0000f0e96b043088}, {0x000021577884456e,0x000b89310ea7bf6a,0x000fad2deb3b5688,0x000d4c37c9429504,0x0000020f0e5f7896}}, + {{0x000a0ab0976505f7,0x0002995e2ec5730b,0x00031ab71567f681,0x000b9ed706201063,0x00002cfa977b1d22}, {0x0005ead8a2373da8,0x0003fba45a6833e5,0x000029d15a8d0d5f,0x0009f33a1d8f9c03,0x0000f34f1cd7c55b}}, + {{0x000428dbbe5a1a9a,0x000e9126bd67cca4,0x0001058268187fd5,0x00019f6036973a48,0x000039b666458bd6}, {0x000deef2d65a8087,0x000f24636b196d42,0x0005d564c4969044,0x0000778611ee47dd,0x0000b2f3a4a42873}}, + {{0x0007d45300eb294a,0x000d769c14949415,0x000a47aa92b2a656,0x000ea42000dd76d3,0x00002864e5046243}, {0x0006c47db89842e4,0x000721479fb78271,0x000b2f6dc12dfd7d,0x000d66fb9a2c56e0,0x00006be862b17f85}}, + {{0x000d8dd0f82b2148,0x00097103cbc603b0,0x000d79e19460c34f,0x0007f8732e5c0318,0x0000b8888bb28411}, {0x00037dcc07226779,0x00088c1c0f278f3c,0x000f7a0c610d21be,0x0000e0447c8468e0,0x0000bf022143decc}}, + {{0x0007d1242b48b998,0x000cc84240960baa,0x000fb5cfb1bcb665,0x00010d0b847cd6eb,0x00007c2ae571ad4d}, {0x000b1220de367261,0x00082fdfbd21f1cb,0x00079d460e7043c6,0x0002cb3bd0826a4e,0x00001f5e5985bd1a}}, + {{0x0004160b7fe7b6e0,0x000a400a3fb29755,0x00028ca1e7d16189,0x0008ccd73e9beae3,0x0000dd04b97e793d}, {0x0003c9b506db8cc0,0x000ecf38814ca9c8,0x0004b45e65cd47aa,0x000a8426fc430db6,0x000079b5499d818e}}, + {{0x0001102c1c24a3b9,0x00078c161c1aebb0,0x000f00a4aca24e56,0x000c7a803eea6936,0x0000ad76ee906176}, {0x0001fc2538e0ff72,0x00094604b3b09745,0x00049cfd794f8980,0x000f694311436e32,0x00007b4a7bd61224}}, + {{0x000d21ae0ac29416,0x000462d3193703b5,0x000c992d0279b025,0x0001f2d307c052ca,0x0000aa7cb934fa8b}, {0x00025800d37c7a50,0x0007342d54225a18,0x000d2ef9213380c3,0x0003c692ac2d66d5,0x000035a70c9030c6}}, + {{0x00025dd4ce4f152f,0x000109df7c06c160,0x000e4bb141f419a7,0x0004cd7d5b221491,0x0000c43c6cd739fb}, {0x0006591925d6b2de,0x00028218659849f0,0x0001b16294b37d9d,0x000e04ac54a971d0,0x00001a9c2a031d50}}, + {{0x000b78571ba18615,0x000c80c8f93d5109,0x00033bb9348b22d5,0x000d0898fa84a786,0x00003fba6baaaebb}, {0x0007df3e5eea7d82,0x000648ca71587ff2,0x0006f1a05521c879,0x000ee499d5133bce,0x0000d50cd541d0eb}}, +}, +{/* digit=8 [{1,2,3,..,}]*([2^40]*G) */ + {{0x0006d65533ef2177,0x000453ca2e87889f,0x0002b41677158c7e,0x00057f8b670dfbdc,0x00005910a01f44c2}, {0x000bf07cf88577d2,0x0000c45e2acef336,0x000a23d852224525,0x000f580ed92e8d7c,0x00009f8be4c4b812}}, + {{0x000b2452133ffd9d,0x0000b30f1a20fbb9,0x000a1f52a39a8b2f,0x000df7784bc97dd5,0x00006aebf57740ed}, {0x0007acb76ccdac60,0x000c1586ff273225,0x000de7dd1af4d36e,0x000c168eaa8863f8,0x0000045d5cf88647}}, + {{0x000414351facc618,0x000d668a25bcc51e,0x000f872edbaf2647,0x0006590f5271a00f,0x0000f32ef99bd2d9}, {0x000488c7593cbd4c,0x000c42b82fabca12,0x000eb3f16ed266c5,0x000fe24a2f78ad14,0x000034049490d47a}}, + {{0x000d574c005979da,0x0001ca40e350a6f3,0x000e2ecf9c2072b4,0x00044e5ca5c1568d,0x00008c8bf5c45153}, {0x000e555114df14a7,0x000d8dc5ec6b97ae,0x000a85418d4374a4,0x000f78054cc28f2c,0x00001cb9e2843c41}}, + {{0x0006702094704963,0x000dbbd2381509c1,0x000dd4398a489a5e,0x00069694dde4648e,0x0000ca7b94ab0111}, {0x0005d682ad636a41,0x0004f8dc5f1e3c38,0x000a219436702702,0x00069dfc1965deaf,0x00008666e16710be}}, + {{0x000fd350369c8e10,0x0002b9dc843b6792,0x00002e2ab9271aa6,0x0003838711a4b14d,0x00002b2a3e27ee1a}, {0x000e35f0e2b379be,0x000bf652ab25b226,0x0005601063d3de39,0x000876cca6d4c93b,0x0000ced0cf5a95bd}}, + {{0x000b4ca2a604b3b7,0x0003ca61676245be,0x0008b806e56f6518,0x000480852f5a7097,0x0000aa3978781dc4}, {0x000ac2a0e01fabc4,0x0004e37d99f9e13f,0x000211ffe7c6ee8a,0x0003eae51384ee05,0x0000ff6976d5bc9d}}, + {{0x000507903605c397,0x000e3142c96c8910,0x000923684f0843d9,0x0008938374493416,0x000032caa306a0a2}, {0x000c27061160170e,0x000b637fbaa3b2e8,0x000eda3acc32788c,0x000e0659cd818ea6,0x00002e9423a7e2b2}}, + {{0x000492316e043a20,0x00011dd3d209dde0,0x00031ebe898a4526,0x000abdeaf9f61bd4,0x00000919f4dbaf56}, {0x00017db6d8774b1e,0x000b78e0e309e424,0x000df81ea5fc5279,0x000c7e84aa40613a,0x0000f419edb9c627}}, + {{0x000759239ef620fd,0x000fa4fa29c43919,0x000416d839d47283,0x0002687e428fa39d,0x00001a7c251f9f30}, {0x0006e1cd746218f3,0x0007e10d967e4607,0x0000ae61ff3ad6ee,0x000929cbb5f434a0,0x0000cd2c019b0d4c}}, + {{0x000d0537a4af00f8,0x00064a294614fa24,0x00082182e3f93892,0x0009468d700c1839,0x00000133443ccc59}, {0x0007106ec87c9255,0x000bbed6665cf039,0x000cca8b562bd59f,0x000a3098414348c7,0x000074c7620ef9f0}}, + {{0x000d39b0260e52a4,0x000c506533256967,0x000ca7954d42585c,0x0007b2cd9bd60521,0x0000fa20877c1ed5}, {0x000eff8e34a0bbeb,0x000bd4f6ef6460c1,0x000af848356b0040,0x00061378be2b24b1,0x00002278164a5531}}, + {{0x00042e2bb8b6a071,0x000eba23f86a95be,0x0004ce47064be74e,0x00076a973d74fd15,0x0000c2d2857e8dc0}, {0x0001c575a8878682,0x000b1de64818b1fa,0x0004e896738df8e0,0x00076cb88e52f9c3,0x000074b4f01ab4cc}}, + {{0x0001ec2c11c208bc,0x00098e3a723459c8,0x000a089d8240207d,0x0000718957eb80f4,0x000086960bce137a}, {0x00054fd7f1a932c2,0x0004b3f2fa17de11,0x00011c0ebb517fe7,0x000d360d5d945571,0x0000aa33789a0929}}, + {{0x00005b4f8b7559df,0x000c0ae292003f5c,0x000532acc0be4c7a,0x000284ed6d3ef756,0x00006c3ed88dea7a}, {0x000b0a8f46ec59b3,0x0004bcea6c83e463,0x000dc836b531d9b1,0x000f0b0d6bdbafc2,0x0000ee501e95ab27}}, + {{0x00075455922ac1c3,0x000c752b3f638df2,0x000de57c4a7b3ef5,0x00008b5e77b21471,0x00001682c10b34c0}, {0x00024f04bd55d319,0x000587b61c71c768,0x000a5089db6d1c08,0x000d3ea1db0903c2,0x0000c092172a84e5}}, +}, +{/* digit=9 [{1,2,3,..,}]*([2^45]*G) */ + {{0x0005035ea6c39976,0x000a62610bef5ace,0x00080dd3954259aa,0x000a398f18bb3f3c,0x0000910b95bbfc3f}, {0x000f51043e09aee6,0x000f47675665fce2,0x00072db61ced56c9,0x000e68b0e265acd8,0x0000982812f0e9fc}}, + {{0x0003d931341ed7a4,0x000c67b59d49b8fa,0x000b8c4a44223272,0x0002e3fdcb194783,0x0000e413c022d130}, {0x0009127e17e44ceb,0x000483b3adfb6d99,0x000aa96caee86bf7,0x00047d46902fe625,0x000073540e595aae}}, + {{0x000ef6c872b4a606,0x0003e613521bcc50,0x0003e15d1ab2a34a,0x000511d9c5c19098,0x00001dde5dfb9905}, {0x000f6219f2275f33,0x0006151d894be417,0x000b0bdaa0750c8b,0x0009bd45b04ab978,0x0000bfd9fd475858}}, + {{0x000bfabaf95894c5,0x0006d76b2241aafc,0x000dda48b7b9bdc0,0x0004df9af983625b,0x0000977faf2f3fcb}, {0x00042ef052c4b5b7,0x0000967591f0bed0,0x000f24ec79fe87f7,0x000f1b589c73ca22,0x0000d37fa9f564a9}}, + {{0x000148045ee2f40e,0x000e616b60cfbd78,0x00049a8c475e354a,0x000535fe0b58a18d,0x000040e94e3da359}, {0x000a59f62accd765,0x000a3c762837bd4f,0x0008c277b05cf466,0x00074065abda9944,0x0000a9e01bf98b13}}, + {{0x00055e6f2606a828,0x000110f2fb57b51e,0x000a4e37ce25f706,0x0007062cef6c2ab1,0x000064e359dddcf2}, {0x0006b187ce573162,0x00001a96b23d479e,0x000f16df72cab250,0x0008b5cd4898628e,0x0000056538d0f375}}, + {{0x000a0c3d41d3bd3d,0x000837a26bdeef69,0x0002edf9fb533b8c,0x000af012801d97db,0x0000c4a826ab1877}, {0x00058513d590dbeb,0x000b3e4e93576c1c,0x000b3337484632f6,0x0002e6236d36b779,0x000046833e44bbca}}, + {{0x00064880a750c0f3,0x00031e548e83cc7a,0x000110f0539bacfe,0x0005880d418c760c,0x0000e4daa4ce1f11}, {0x000e7b55ffc69ff6,0x000c320531272733,0x00022df9446f147b,0x000b7c285b2434d7,0x0000a444f6646fc6}}, + {{0x000e4bb346ac9cbf,0x000339811bb2008b,0x0001540c12e6cb02,0x000e46b6ccc747f4,0x0000119334febf6d}, {0x000ffe9cc97fe6e0,0x000bf3d828834e6b,0x0003dd1c77f4b578,0x000495459db72275,0x00005843cd82066b}}, + {{0x0008cd5dbf2af333,0x000e7f6b0eab7cb8,0x00040d844ded9566,0x00067fa1caf5488e,0x0000ae59bd6ace1f}, {0x000ad33dd82429e5,0x000b6c4467c7acf4,0x00083d011cc7ebd2,0x000db7a4f6e95de7,0x0000197e39166e92}}, + {{0x000031885e7a4af1,0x0000e7a6878dd4f2,0x000051913291bf9a,0x0000349dd8b6eebd,0x000074f5125036be}, {0x000364527f601893,0x000d5b37ec732ac3,0x000fdb42ed8902ee,0x000421246166a11c,0x00001c4e3b837076}}, + {{0x0009dfdbc6ebdfb0,0x000589de549b7de2,0x000f26a64a1ee450,0x000980fd7181aa3e,0x000003179eedf2f3}, {0x0004d30fd71bc789,0x00042e86a5831bce,0x000b85711eb842b1,0x000405500125dc5e,0x0000f5869848f11f}}, + {{0x000089eb833eccba,0x0009dbc51c795b5a,0x000be0e93c1b3077,0x0001724581157f9f,0x0000b487d69fb29b}, {0x00051082222f24bf,0x000c06dd35dfc725,0x000c24afe338cd22,0x0006eca2010e6f5f,0x00008298f16581d4}}, + {{0x000f723042389edd,0x0004d54b19722ce7,0x000251d75df7de0d,0x0008c7e8ea2e142c,0x000022a4d38c4e8e}, {0x00058566c29c8bcf,0x000d2d1e201ee999,0x000c885c99fefdbf,0x000c6fc97d946eda,0x00006ea9768c38ea}}, + {{0x000a8e974e75a2ec,0x000dedac968f9aac,0x000306befcda6fc1,0x0007f87f6651bf46,0x00007445e4f8b2ed}, {0x00012d8d1571ac12,0x0004baf3a679584a,0x0003fbc43684846c,0x0004f2afc622a986,0x0000f9e101e0a681}}, + {{0x000465ac3f16ea83,0x000d82f1d11c7a1a,0x00068a172115a461,0x0006981767dd956c,0x0000392f2ec013a4}, {0x0009ccde526cdc7f,0x000b32292b81c7a9,0x000d391988e537fd,0x00052c86d8cf69a6,0x0000fc5ff4414468}}, +}, +{/* digit=10 [{1,2,3,..,}]*([2^50]*G) */ + {{0x0004f7ea90567e6d,0x0006e6ae5cb797b1,0x00010903d513257b,0x000723b5454a3c9f,0x00008d2c9ae39bc3}, {0x00093246b29cb44f,0x000c87c8cbac38da,0x000918e42b540a21,0x00014dabbfe43501,0x0000ffa707b46c36}}, + {{0x000e05b1cb76219d,0x0000a1567e7e56c2,0x000c4c9100ec0bf9,0x0004d917076f8661,0x000067b085c8abc0}, {0x00004595e93a96a9,0x000a6bdc249a9fb9,0x000dd0bb77526c1e,0x0006947d44d367ec,0x000053999182dc0d}}, + {{0x000c583a4c506ece,0x0007f1acfe972ccb,0x000f1aea2957ed18,0x00062cabaed83312,0x0000f2a6cb563253}, {0x000de428e195c43b,0x00095e6050c6130d,0x000686a5dc842025,0x0004a77da972a708,0x000052999a29508b}}, + {{0x0009167ceca9754a,0x0005ab7939a083f4,0x0003fd0bf426d2cf,0x0004e18555e35572,0x000096e6d0764f14}, {0x000a8dd87880e616,0x00058508e4d54768,0x000b65e151554381,0x000f9fa9d7e772b1,0x00003439dd70c302}}, + {{0x000f8e1698e04cc2,0x0002f69005c8b11d,0x0003c6179877be20,0x000c0512749e8c4f,0x0000dbc9d0a9853f}, {0x0004f939454d9370,0x000db4800e1b187d,0x0005e68e8e682ce9,0x00085ba9129ad816,0x0000fe29735be7f7}}, + {{0x0009ccf3a4434b4b,0x00006a7954dc3101,0x0004972a7a345811,0x0008dcf9dac80de3,0x0000043d05524f6b}, {0x000319e11137b1a0,0x000eed5cc03f021c,0x000ea5ad400a754c,0x0005b60aa2c794cb,0x000093e67f470c01}}, + {{0x0000351d598d7107,0x00065b3a4da420ac,0x00071de1f272c416,0x000b0f6b82fe1aca,0x000046e79f348f54}, {0x000c7364b573e9b5,0x0006ad4b50406e7f,0x00098d87b75d03f4,0x000da10c1cc36d0b,0x000013ba3f16f472}}, + {{0x0003145983c38b5e,0x000b837abc8b859d,0x000ff7be6b14f176,0x000a594793fb9dca,0x0000be5a56015a66}, {0x0001dcd9f87dc596,0x00039bdbf5607cec,0x000eb32577c595cd,0x0005fcfb543b2226,0x0000908064724c93}}, + {{0x000b70678a6513b9,0x000a0edb1943dca3,0x0006e2dd892ea4a2,0x00089372642216db,0x0000b45d0b52fd57}, {0x00070dbc69d11ae9,0x0008bc57595f114e,0x00077c2721477dd1,0x00059c2c2208b4ec,0x00005c5b4d86b68f}}, + {{0x0001fa47ea67c77a,0x000f43ea810cfe54,0x00001d374952bd2a,0x00036dd91fef568d,0x00003a1c621af113}, {0x000d5a9c7ec6d792,0x000f1225c3425ad0,0x00069601bff7038a,0x00047ce03c6689bc,0x00005059bc765e87}}, + {{0x000c80676cb25661,0x0001a24892d99a75,0x00008fe458f76acb,0x0007d86ae7b9cc1f,0x00009ef73297a490}, {0x000ab715f228bf03,0x0009517032d72db4,0x000abe3c0f3cdea3,0x00025bdb1f482edc,0x0000baf76b4eb863}}, + {{0x000688eadd70482e,0x00099b4a4e8a6aac,0x0008a6eef708de92,0x000c4295b6dd7375,0x0000a4bf353525b3}, {0x0001f2c87912868c,0x00052f09297a1004,0x000f3860ab1b1be9,0x000f4a59ae23c5a9,0x00004f0f83a115dc}}, + {{0x00027bf8538a5c69,0x000cf9abff17c71e,0x00071e3da195c63d,0x000fa06d3152851b,0x0000cbdfda88a680}, {0x00076ca849d7eab8,0x000abc2732719db8,0x00008dceaebe2764,0x000b6fe63357e3f2,0x0000c5bd833d65b1}}, + {{0x0000d7c2fa4f1264,0x00051c99a2327590,0x00025e0c308a3b86,0x000edee478b6bfdb,0x000082cc2c2b1db2}, {0x0007e645f321bb80,0x000b9a8005b437df,0x0008c19588a93821,0x000d0a3fa2f10ccc,0x0000d3322182c269}}, + {{0x00014df3fd165e81,0x0008f61f8811ba55,0x000ef9f00499fd6a,0x000e8a62cd1fe0bf,0x000020a4bb989ad7}, {0x000d0955f4a5ac5d,0x0000c5a7a2f0f2ff,0x00017baf1cfd174f,0x000089042301ba9d,0x00002fa487b47f22}}, + {{0x00052381d531696e,0x000fa8cdde69b934,0x00086afc757201bf,0x000ea7fde922519a,0x000030438969d35c}, {0x000c1e18555970de,0x00084535935e7608,0x0002ea38b8267dfa,0x0008b4f4c60a5732,0x00000bf7978604ef}}, +}, +{/* digit=11 [{1,2,3,..,}]*([2^55]*G) */ + {{0x000ea68c094dbb56,0x000e7968d4106233,0x000b3002db77d062,0x000d57de719bbc58,0x00008e7dd3d9dc49}, {0x0005740013a5e585,0x0006ec9e3c1b8d82,0x00099b6ab2131174,0x0008f1bcb0a2a77c,0x0000c48a3b412f88}}, + {{0x0003e91991724f36,0x000bd9cbd686c791,0x000d4fc1e5eda799,0x000d547db595c763,0x0000b63b80c0c4fe}, {0x000fc697e5fb5166,0x000a70f1c9646ea0,0x000a92ca5737708b,0x00067a3628745f11,0x00001f37958fa869}}, + {{0x000fd610979e6b3b,0x0004e3d407235320,0x00080bacc1d0bd3e,0x0005145ac006fcaa,0x00001d9c60f82002}, {0x0007ed0af780b927,0x000e10a9ce31610e,0x00020b54bf5bf446,0x0006c2d41c011d3c,0x00007f76da189159}}, + {{0x0009b2caa6650728,0x00046fd324ef9af3,0x00027bd3178322fa,0x000aafbd153394c3,0x00001d5f271b129d}, {0x0000c42f48027f5b,0x000bd536e717c72e,0x000369d0faa40cdb,0x0004e6445a657a2d,0x000003bbfc59a7f7}}, + {{0x000d5429b6a42eae,0x00045daf41b97a04,0x0006b0b89fa597e8,0x0003dd6c58ec2772,0x0000d8eb16ae30d4}, {0x000e5e1ec9dcf577,0x0000f97cff8165e1,0x0006e2b22b7a770c,0x0002b6ae6d918f9f,0x0000c277a9b364e8}}, + {{0x000c4180d738ded3,0x0000b0de572946a8,0x000a816756f1a5bb,0x0003d4c10230b98b,0x00002c6f30c412b3}, {0x000129dd8fffb620,0x0007b459bf057559,0x0003b67766a281b4,0x00073a77c1bd3afa,0x0000709b38078299}}, + {{0x000e0c1da36cb47f,0x000a2e0210f00875,0x0003787c8fdf5b7c,0x00041e8e0c7e4d3a,0x0000043f5271b1c7}, {0x0001b006d74d72d5,0x0002e8b45ba976df,0x0006b797a0514df7,0x000c395ecf7c3e0a,0x0000e30b2862deaa}}, + {{0x000b232a3326505c,0x00022e1d41bf8c26,0x000e32afa38d6927,0x000a864459453eff,0x0000e8143ae3cb3e}, {0x000c1fa7e6ab6665,0x000fd2286264932e,0x00036f8ed6cd2d22,0x0005baf59a46fe67,0x00000bf0d00eeca8}}, + {{0x0008f7d605238a52,0x0002248496ab15c7,0x00010acc1e9d8784,0x0004e2afcf75d0d2,0x0000948f2297c8c1}, {0x000d76de8daf0cb3,0x0004703be800e81f,0x00078d30cd02d11e,0x0009cadc4df35187,0x0000482bc96c6513}}, + {{0x0005852877a21ec5,0x0006bf537a940b82,0x000a9a6a2300414a,0x000bffef1cba4021,0x00000824eeec6943}, {0x000fcecf83cba5d5,0x000843b4f3c0a0db,0x000f24dd7f953814,0x0009dd1174416248,0x0000322d64e34fb0}}, + {{0x000cf278a448bbc2,0x000fa85251da56b6,0x000e79b240ca898d,0x0001a77082cad836,0x0000e7b9ed33a8e5}, {0x000318a43e1c8025,0x00037bf8689ddc7d,0x000071b1a4750e52,0x000dc14887a072f0,0x00002090f87d14bf}}, + {{0x00073843d9325f3b,0x00004371cb845744,0x0001e36c5a9bef2d,0x000f71c7d2188ba6,0x0000bd6a7d87602d}, {0x000a9028f61bc0b8,0x000ceed0b6a1ba3a,0x0006e8298f49085e,0x00001d0bc625d6ae,0x000032b0b1e22e9c}}, + {{0x000f2196196bc6e2,0x000bcf097efd5beb,0x00087293a0e66736,0x000cddf128f3b8ea,0x0000998e4058addf}, {0x00031b85d16c9619,0x00009418f05655cc,0x0009fe81987a434e,0x0000c59e94aaccdc,0x00004a486c235649}}, + {{0x000c447f1f0ced18,0x00031492dd2ba337,0x000a08efa800cc79,0x00041dcb93151dbe,0x000020cf3f95e0a7}, {0x00082dc1c0f7d133,0x000054dde6caff19,0x000f96ee3ef92196,0x0000c6ead7d97245,0x000019c8dbe59dea}}, + {{0x000385bde259ec84,0x0009f0f67b0d1ff2,0x000661cfff6b0836,0x000f5bc4cc65b006,0x000067e635914230}, {0x0008c802dbed3d35,0x0002a1920da6ce46,0x00037479f7b98426,0x000de0e4d2574215,0x0000c8aa88cd7bb8}}, + {{0x00038717b82b99b6,0x000ce70eb624d3ea,0x00095d46675922d4,0x0003462f66ec543b,0x00006e673cd1ee1e}, {0x00067c4b5f2b89a4,0x0005e90e5cd36afe,0x0000a2ada3de9c1e,0x00023b4c278bb631,0x000020fa3844bdb3}}, +}, +{/* digit=12 [{1,2,3,..,}]*([2^60]*G) */ + {{0x00096796424c49b9,0x0007d7c241c9646f,0x000f68b49f888dfe,0x000f20512d4b9324,0x0000a6b62d93571d}, {0x000b26d179483cb7,0x00022511fae281b4,0x0003aa51f666f963,0x000d166281b3e4d5,0x0000f96a765ef3db}}, + {{0x000d37c051af62b9,0x000a7bf944968553,0x000d59aa1e9a998e,0x00081350844f9fb0,0x000083fd55976afb}, {0x000c0ca65d698049,0x000ddea5ff2d9670,0x000d8623b732b22d,0x00078247640ba95f,0x0000f61916436351}}, + {{0x000b4e0bdefdd4f1,0x0005e366e401f167,0x0003bbec06995846,0x000c214aa368aba7,0x000021487098b240}, {0x000323318969006d,0x000e11fe53d1378c,0x0000c4361cb4d73c,0x00012a8f50a80e13,0x000067f59524ef52}}, + {{0x00081088cad38c0b,0x000fbbd68ae2332f,0x0008e27a3471b7e8,0x000b0ca6ac3fb20d,0x000054660dbc36b4}, {0x0001e11a6fd8de44,0x000a637799ef123a,0x0006ac17c44dbffe,0x000cef0540b977ce,0x000095173a8ef60a}}, + {{0x00037434573eab0b,0x000b21ac6031eb44,0x000dd9afd11570df,0x00023147d9b45b44,0x00008066addd2067}, {0x0002ad8f8a3f0b44,0x0009a0ace2a215f9,0x000b38b809e0e489,0x0000527dcd0aadfa,0x00006506ae957020}}, + {{0x00069f78fca399da,0x00016207bb63429a,0x00088f582fe9e27d,0x000f6e4c655ed687,0x0000426d7494db75}, {0x0005c02ca81c66da,0x00070531d4251869,0x000ff48ba84fb8d2,0x0002469a3a8956de,0x0000f1d0d57166d2}}, + {{0x0009565352c4b5c2,0x0001390bc3e25a05,0x0006f9f5f4926153,0x000a9b609f7521f6,0x0000baef6bfe70a4}, {0x000fa6509ed3561b,0x0002e84b230ce7e6,0x0004cdc691137023,0x0004157151659bd0,0x0000db83c64a007d}}, + {{0x000284d391c2a82a,0x0002758308e89ebb,0x000f1edcabcdd486,0x0006c7606f16ec83,0x000013e2c38095dc}, {0x00056f04a057a87a,0x000006b48f982ab7,0x000651c44a876550,0x000e01a252face68,0x000052b540c81765}}, + {{0x000a1a85ca37ff0e,0x000bed2f1c8fcb35,0x000a26112e1a04f1,0x00077d438816ce15,0x0000206a111e95b1}, {0x000b6048a4241497,0x000704752cfb3c10,0x000f1dbb8c6a3f56,0x000dfb4f16a37a47,0x0000c372f9b431a3}}, + {{0x00005b5c20b4d2a0,0x00050c662a67122d,0x000ffc9e9ff659cf,0x000683ed57c128e8,0x0000fbb15859e987}, {0x0000df247319a2b3,0x000a074be470add7,0x000a7b3074b98bab,0x000f30c03d747356,0x000042e696e6efeb}}, + {{0x00048f7864ac537c,0x0008e6940d3df84b,0x00074c7ae0471340,0x000033414db22d61,0x000073a1c444c213}, {0x0004ea5ffdd93ec9,0x0006d102783e18ac,0x000c3e83f724fc75,0x000bf50fe13fcc91,0x00002a8c2c9808f0}}, + {{0x00022fd8f67f6de6,0x0003786931777f2e,0x0003eca95ab01883,0x000f2ac66d1db686,0x0000bb7732b91b5e}, {0x00027c6915f80ced,0x00018f90efd84fa9,0x0006b48ad6fa1d6d,0x000428abd75de845,0x00002cb507b545b6}}, + {{0x000f82ae255d7eca,0x00030460e204a72c,0x0005b0a4452025c2,0x000ed970ae542d7d,0x000085143113305a}, {0x00015f5a14bbfe81,0x0006285365fe9583,0x000d950403f36182,0x000da2b2b3a36b66,0x00002c7b3348cf4e}}, + {{0x000b4d1e744119d4,0x0009429a71e7a545,0x00017fcbf4c93b16,0x000d84edbb908c61,0x0000dc97320f35a3}, {0x0004f856bf88105f,0x000d2b51bc1f94c0,0x000013af3452e1bc,0x000e231c41ff50b3,0x000007af446124d7}}, + {{0x000e57ca3d24f6a1,0x0007e345a763bdb9,0x000cfbb5f8a8246d,0x0004db63bd2a6d98,0x0000dd8e85e96ed0}, {0x000da42c01f420b6,0x000304407bc776f2,0x000f548f57ef0547,0x00064a1e98ba7faf,0x0000b7afbef5d30b}}, + {{0x0002fc516a0d2bb2,0x000bfa6234994f92,0x000c62c8b0d5cc16,0x00067f7241cf3a57,0x0000f5e69621d1b6}, {0x000c70bf5a01797f,0x000c709561925c15,0x0001fdb523d20b44,0x000f7a14911b3707,0x0000648f9177d6f0}}, +}, +{/* digit=13 [{1,2,3,..,}]*([2^65]*G) */ + {{0x000c8b8fac61d9a1,0x0002d3c6fe8a027c,0x000bff5037d25e06,0x0002f7d08805bfe5,0x00003271e6c7ff63}, {0x000a6c0232f76a55,0x000d201ef42655dc,0x0000a51788957c32,0x0001739e728bcba1,0x0000ea60412062c5}}, + {{0x000462bb4d8bc50a,0x0006091957709ad5,0x000412a68181c0b1,0x00048c4bd4fe1c78,0x0000e0341bd60dff}, {0x00045cf7003e8666,0x000a2a24a41bb6bc,0x0004c24c2f11a6de,0x000b67f407151ad0,0x00002c9d27e3a5b7}}, + {{0x0005719a8afd30bb,0x0007da826dce3286,0x000a8fbe08679832,0x000ad32f04e891c4,0x0000b6b6e1c9bf56}, {0x0005b11471f1ff0f,0x0008ce15baf00a69,0x00096c43ed76c338,0x000157118edb95be,0x00002beaaf580794}}, + {{0x000b12e523b8bf60,0x000b8f910c1b0a50,0x0001675888009eb5,0x000abdf535af824a,0x0000f835f9cfb2a2}, {0x00029312afceb620,0x000a169d383ff59b,0x000ac02b0c797df2,0x0000caeb3f5fb066,0x000029d4c6fdaa2d}}, + {{0x000ebd1e0a1b12ac,0x000cb70ba95f87a7,0x0002ae9cb1e4ef88,0x000402cc33345cdc,0x0000ecf1276c1cc8}, {0x000012e1b39b80f3,0x000d05c33ba4687c,0x0009661c2fd90d0a,0x00029e73ef5a675c,0x000068fc88f10174}}, + {{0x00026052b7ce5421,0x00096472bde1b822,0x000d2f4dae6d4ce9,0x000b2e43e16ebe09,0x000080ff42e63b92}, {0x000cc022c34a1c65,0x0008f22c46c2c59b,0x00014a8a23803d6f,0x000b27c8aff74f5c,0x00005aebf8060a08}}, + {{0x000970e2fdd23cc0,0x000c5682e971b956,0x000e86ebcb80288b,0x000939e6e6d91e9a,0x0000564c83f8c9f1}, {0x00032a239560368e,0x000a249c28e25519,0x000a158c3e893752,0x0002622b03cee5a6,0x000012d656be4964}}, + {{0x0002010f5b343bcf,0x000a02f142fe58af,0x0005f4bdf0f2e400,0x000aa84483bfdea8,0x00000b1d093f3bfe}, {0x0001b95c70816030,0x00093dba10972ea0,0x00038f3a6e943e4c,0x00063647be92adb4,0x00000bb7742e5bf6}}, + {{0x00014576be5f7de2,0x0008a2263c9e4ed7,0x000cacb36d93006f,0x0008abc073694cca,0x0000ff7a5b45ae11}, {0x00053f1cd871236b,0x000d12aa6d523cce,0x00098d76df156a39,0x000d38ecc5f271b1,0x0000c615b7031383}}, + {{0x0004fb486df2a61b,0x00073cf7b4a2137a,0x000d042ffa1ed9c0,0x0005ec02e460e27b,0x00007f5e2fb0f62f}, {0x000ec6bcc2423b79,0x000f2a63eea77aa6,0x00050a6e175ce0a7,0x000c9ed7a45fb1f2,0x00003bc919d753cd}}, + {{0x0000af231f63950e,0x000034caa2c96793,0x000ac7e62a77797c,0x000aeb726e80ee27,0x00001e6e62738b28}, {0x00078b0b3c9fef02,0x00004d5f90be6361,0x000ce51cfaf7752e,0x000e1f74ecaf18ee,0x0000864d0edea806}}, + {{0x000ccaaddce33450,0x0007012a4350ec2f,0x000598bdc2a6811b,0x00012896760ff1ac,0x000054d652ad1bf4}, {0x00051d492a210056,0x0000d3110fdf0a11,0x00060100fad7f397,0x0003622c95928c19,0x0000c91c825dbf03}}, + {{0x0005b7799eb6df0e,0x00009386b7791778,0x00017a48e26c3cc5,0x000f30545ed98864,0x0000990b4e4e7d6e}, {0x0006b7e2586abba3,0x000529c96e9a0f45,0x000eb4206239ca6a,0x00090ab327459ce2,0x0000a4c3313d002b}}, + {{0x000125dec3f1dec5,0x0000411178da19e6,0x0004a673807b1f04,0x000dcd893ededa90,0x00005187a5a5bebe}, {0x0004722eb329d415,0x000ea170b391f7d0,0x00099f828f449099,0x00076dad317a69ca,0x00000c3db2b84a49}}, + {{0x000b69b92222f1f5,0x000a1cf7ae703806,0x0005217ee5a2459c,0x0005ac1789f69ca8,0x0000f232b5f33dc8}, {0x0003ec548e9e5167,0x0006c197eb31660e,0x000fcca23124b4e4,0x00024ea0a0cb13aa,0x0000bd63ba4f2132}}, + {{0x000481b7bfe71786,0x000e05405868b674,0x0008c867d4e1deba,0x0000e9a61b2821c4,0x00009c15b35b13b3}, {0x0001666368710884,0x000cd220b1ff3b4a,0x0003d9f4de5e29f5,0x0006750b82bb3523,0x0000e07633358cdc}}, +}, +{/* digit=14 [{1,2,3,..,}]*([2^70]*G) */ + {{0x000f5c7a3e6fced0,0x0005f45fbdeb0d53,0x000339a70e8cbbdd,0x000b81f85c01df13,0x0000ff71880142ce}, {0x0008774bd70437a2,0x00019a0bda6a4c4e,0x0008bd26e5fb3289,0x000521fcdbebd2f1,0x0000f9526f123a9d}}, + {{0x000305192c4d6840,0x00057612efcd40ce,0x0009cae208b04d72,0x00056cb9dcda366f,0x0000edc4d24f0588}, {0x000e6bf854279005,0x00058c09dfea64f2,0x0009bf26c3de8129,0x0005a9841b448737,0x00000b62c6dbdf13}}, + {{0x000e3b4c72dfe67a,0x0005f0e19fdfd4f8,0x00013bd35c416b0f,0x000b9d78b9098d4c,0x0000c11118ab5b8c}, {0x000a318f00628413,0x000f59f356f4f598,0x000177a0cbfe0602,0x0005374ae3637e30,0x0000409774791136}}, + {{0x000fb5ed005832ae,0x000ab1042e4f0db2,0x00070f8ca5f5efd3,0x0009cbac4ffdc6ed,0x00004645d0c952da}, {0x000f58bc9001d1f8,0x000bce1172059596,0x00098a08452c8f0b,0x0009de7d4aa0d2e3,0x000015bfe3a904f4}}, + {{0x000443f23885e5fd,0x000918433aab97e5,0x000d4e604f72f8f9,0x0003feed00b154e4,0x00000b35e6bb5e17}, {0x000a0489164722d3,0x000b58761ec857b2,0x000a838323e3c665,0x000e3b3bdd13973d,0x0000c8b1a1ea3daf}}, + {{0x000ace654317cac3,0x0009221771b34497,0x000dfe8b8be600ab,0x00010f842e409eb0,0x000086a67d769423}, {0x0008d8d4431cc288,0x0002185dc5242554,0x000c4be32a7cff14,0x0006e0cd60f5a193,0x00003ebd5c95071c}}, + {{0x00080a7b1fd2b0bc,0x00059bec33e8ba3a,0x000743fb39b3ad39,0x0002f9f3868d6179,0x0000fd169fdbdb46}, {0x00099d79ce0a6af4,0x0001a42d3ff8d3b4,0x000c3e1b255dc1cf,0x000473c4fb9e6cc6,0x00007e6961daf69a}}, + {{0x0003acce548b37b2,0x000264d4054954eb,0x000341b4fb38e754,0x0007fa6c3daa517b,0x0000f6928ec890bf}, {0x000b32386ce6c413,0x0004e0adadcd0496,0x000b5faf901be1c5,0x000985904e67e74b,0x0000cbaf679115c9}}, + {{0x000214550ca42470,0x000ae7dd30aa8cd1,0x0008fee24ba1aa47,0x0000e82f81ddf1e5,0x00003452936eec9b}, {0x0003b81243aea965,0x000ec5c3d0e58bdc,0x0009483619a2919a,0x000ccf4ea640ec10,0x0000ac86d5bbe0bc}}, + {{0x000d918c36cf4406,0x00076939719cf892,0x000218b64aed3e83,0x0000dd507b08d2c0,0x0000f1bcbbb2e979}, {0x000d6ed60919b8eb,0x0000dac1f9eb4a84,0x000d5daefd890079,0x0002c5484941aa0d,0x000022fe40b17fd6}}, + {{0x0005ba2157f2db33,0x000f5e28ca9c97e1,0x000b9f454bda2fc8,0x00072e2d050da437,0x0000d57eb575379d}, {0x000eba2fb5ee9973,0x0002b11538cae9b5,0x00032797401648ca,0x000bb702bb76f6f6,0x000038f14b92f3f4}}, + {{0x000226ad7ab9a2d8,0x000cfdfae958524d,0x00051d8c29c00090,0x00062c8ba5f53987,0x0000afcbcddbab82}, {0x0002729e99d043b6,0x000b4ebc943a5739,0x000862935ef51263,0x00017b3feace9320,0x000039efc04106c8}}, + {{0x00054b366b4be7ad,0x000db4a37a1e1fe0,0x000d75cd93f25a9d,0x0001b5239ef1ad78,0x00007b58f4a2062c}, {0x000f9a9ff563436b,0x000938af51e76f74,0x000e97fecf718ff2,0x000c09a234d31315,0x00006a8e2b1d92f1}}, + {{0x0003aa8327720c15,0x00026a092cc8a7f5,0x000746c4d956ca32,0x0003923f03d64a28,0x00001fe1782b6d0d}, {0x00034db3c832c80a,0x000bcda2e3b4d19b,0x000104ccc60dccc5,0x0001fc845dd62e0a,0x00007ab1de2020b2}}, + {{0x000ae0b3893d123d,0x0002e15ee71cb293,0x000a9468bf7b7578,0x0007438aa3c61442,0x0000686123dab15d}, {0x0006891a7ab4116f,0x0007b4e6a4598c61,0x000e5fad76fcd72c,0x00046abc21911077,0x0000b6a20e8604fa}}, + {{0x000be7d341d81dcf,0x000842148379e839,0x000026eadcddb688,0x000c5dea6211a1f7,0x00003b25760e4d1c}, {0x000c8f6a7a73ae65,0x000e11d5b48340cf,0x000a50ebc83879a5,0x0008fa75acb1ed41,0x00009a60cc88c07d}}, +}, +{/* digit=15 [{1,2,3,..,}]*([2^75]*G) */ + {{0x00030b665c7322db,0x000103c1b3fb4395,0x00072f685cf12cc0,0x00091d170b018601,0x0000915ee22cb583}, {0x000f03ba317db248,0x000897b8ffc49afd,0x000d3d05087dec65,0x0000e6ff46597be4,0x00000a1c1ed80650}}, + {{0x000a7b397acf4ec2,0x00003ea8b6403e22,0x0009692850426c40,0x0006703e3295a64e,0x00002aabc59c6a45}, {0x000714c5f5942bc4,0x000dba3182edb929,0x0004152ba9a6168b,0x000367e216a66510,0x00006908d03f6926}}, + {{0x0008ac6c2babcc13,0x0008ce81ae8d52cb,0x000f1a7114748d44,0x000ac42844f03f80,0x0000db784b328df4}, {0x0008f122df1fe365,0x0009a33cc7c0ad91,0x0005e5555e40f25b,0x0006f8f700d0e73b,0x0000c28fa0900332}}, + {{0x000d8745a1251fb6,0x0008672725c7a9f5,0x000ffe89e967747a,0x00035db95c33e531,0x000009d211049649}, {0x0006ca82fe122271,0x0005f426469dcafd,0x00093183caf9b5b9,0x000fef1e9ee04c56,0x0000084a333d8146}}, + {{0x000b1c8321a518cb,0x0005dbce226f56da,0x000acb9fafd4439a,0x00071b30b30d194f,0x00005052f3125835}, {0x000641012afd4761,0x0001f3fe624a1442,0x0001c92e6d02e417,0x000b52ec394f6553,0x00006d4bf5ae4bc0}}, + {{0x000b88210395755c,0x00039ec1df80ce06,0x000f55e96117ce63,0x000d1e3efae513ef,0x0000f36cba7bd7fe}, {0x000eca9a40ebf884,0x000f73d37e127340,0x000bbf9ffe6ec1bc,0x00005e8a51b64e86,0x0000e0dbb58cb40e}}, + {{0x000d0bf5f8b8a84c,0x0005c66f3ede120a,0x0000ebc2cbfa00f3,0x000e3ae8c6064ec3,0x000039e40b8e3001}, {0x000f7cde7b1cbab5,0x0006b4a562849614,0x000857a19cd4420c,0x0006cb3446a8f316,0x000019b93b236f70}}, + {{0x0009933aed1d1f71,0x0003405630909664,0x0002e39cf566eaff,0x000124245057f0ad,0x000048ff65b2f832}, {0x00089d4cf94cf0dc,0x0003920c58b3042e,0x00061aa0d319bec8,0x0007ac6a26762653,0x000086fa3034fbc8}}, + {{0x000e5b80863b6647,0x000accabdb533805,0x000feff918be7ac6,0x0002062f7d70505d,0x0000dea8bd105896}, {0x0005a3b410e3c4e4,0x00097c603ebf2800,0x000ec15d224a4f0e,0x00041d8c4fd5ae4a,0x0000dc253f84f966}}, + {{0x0007b9c7ea2ee345,0x000bb9cc3a71359d,0x0001ea37a3fd0d94,0x000c876b53c31c3a,0x000033425fb1f818}, {0x00099c3810156e0e,0x000350c164487cd1,0x0006425420ea020e,0x0005ea0557ba094a,0x0000657e7e87465f}}, + {{0x0003ad1fbc824398,0x000b7896e9fdea72,0x0001c33ffc726868,0x000231597c913c51,0x0000a3fa8e362114}, {0x0000e5608445b3e7,0x000488d098d32a6c,0x000ea77d42c4cd88,0x000f710bf51faf2f,0x000009f208aeb1c4}}, + {{0x000d2ab5c8b06d5c,0x00023e4eac46fc83,0x0006f7779b1a785a,0x000504f99315bc84,0x0000f31d817af9ea}, {0x000fe6a15d7dc85d,0x0003e4016b332391,0x0001cb4c72f132b0,0x0005a059547fe318,0x0000b66d8a735015}}, + {{0x0004843f45aac50f,0x000da1eaaad05b9e,0x00054eea0c31e042,0x000a567e8c0b345a,0x00000437b95862c7}, {0x0001ce8fe1d348b4,0x0008a3786432e453,0x000510d38e148937,0x000b0095e19c4a3f,0x0000df3f0170e348}}, + {{0x0000e8b593d070f7,0x0005e255625d59cd,0x0007a03994375751,0x000eeae51fdda75b,0x0000bb6e6b09dec1}, {0x000b662334c0922f,0x000e1cf41b79729b,0x000f324023df631d,0x000c8711abf3c578,0x0000cb4666d8cd33}}, + {{0x000b445735e843cc,0x000cd7379134b805,0x000ab9f872a8e890,0x0002b62bc7c10c52,0x0000bb5e1ed780a9}, {0x000a2d9dc2c4efe8,0x000e2e8cc7bdd6ad,0x000418a74fccf504,0x00080be50115acb2,0x0000dd90e07452bd}}, + {{0x000d7e1adc1696fc,0x00024acd72d06b66,0x0001b743598ebe59,0x0005eba5f24550cc,0x0000e23139474b9a}, {0x00022d4db067df91,0x0005baff9b00234a,0x00000c9c198dda09,0x0006950bbc75a061,0x0000560a9c8a39cf}}, +}, +{/* digit=16 [{1,2,3,..,}]*([2^80]*G) */ + {{0x0000f1cf1c367cab,0x000b190fbc7de405,0x000a110329bc85a9,0x0003a8f373c4a2e1,0x000064232b85d039}, {0x0007eb0167dad297,0x000124b78ab2f557,0x00029348b1604f30,0x0003419baa94afe8,0x00007fbd8ddb1654}}, + {{0x0004802fcf0a7fd8,0x0008d488b01e31f1,0x00052b49842fd078,0x000200f1d78d6d99,0x0000eb572d987ac5}, {0x000a44c4d194a88b,0x00090a017e66e0a2,0x00088aefcd2b63fd,0x000a10c8efc6c8f8,0x000076f6bdafa881}}, + {{0x000932c68af43eec,0x000db03d00bda2f7,0x000b061f55502468,0x0005ad25dc978f2f,0x00009a1904ae8c81}, {0x000538d470c56a4f,0x000e293d8cedd3af,0x000108ef3159abc5,0x0001773a37245f20,0x0000a17081f123f7}}, + {{0x000a9b2b4b4b67c8,0x0000680206041fe2,0x0008058d8c1d10df,0x000fbb1d64abfcbc,0x0000943b9b2f12a0}, {0x000d9143b3def048,0x0001cce775ff90ee,0x000bc904085ab3aa,0x000dfae05fd4ca7b,0x0000b34a56562c75}}, + {{0x000acf88e2f7d902,0x000757be32cd5c18,0x000eb5ee9fdbf33d,0x000114ea085cd7d2,0x0000d702cfbd3201}, {0x000ebdb85c88ce89,0x000b8e01d617b6e0,0x0007333ac23a3ce3,0x000b6aa041618e56,0x0000dd0fd8fa57ed}}, + {{0x000610798fa7aaac,0x00043073aa4eb2b2,0x000d6b19b41209ee,0x000caf31570359f2,0x0000be6868dbc577}, {0x0004bdc32c04dd3e,0x000defeee397186c,0x00086c0cfa6c35fa,0x000fe1d4a1b312f0,0x00000a5ccc7b9461}}, + {{0x000f3a36fa6110cc,0x00013b93561f516f,0x00057522b74fb1eb,0x000dc5ac0c904784,0x0000fd321052bb8b}, {0x00084a2cc80ad571,0x000576a9b6372d68,0x000f4e8cd7c27fc3,0x0002f02461baedad,0x0000d56251a71724}}, + {{0x00011a9431dd80e8,0x0009f3306cd9b840,0x000b3b730eb7c7cc,0x0003d2a0fadd29d1,0x00003858b5c7e37b}, {0x000d193b6251d5c7,0x0002a352d952bf4c,0x000fbc0511cca1fd,0x000636566157a490,0x0000990a638f9b98}}, + {{0x00081a321175ec17,0x00057e018109892c,0x0008be3169159a50,0x0002e5570130532d,0x00006060c21b26fa}, {0x0002dfc6b6f0f228,0x00044a01a671074d,0x00070bd8e9725fc6,0x0003fdbf6679b927,0x0000fe6605057c9b}}, + {{0x0001154b6e00a84b,0x000174890e60ce71,0x00060988fd9fe7e4,0x000004210bc6c345,0x0000dc2ef531859b}, {0x000d868d5c890ee1,0x0006019c47dcdcf0,0x000714567893115e,0x000a53d97966fbee,0x000017813356c85a}}, + {{0x00030cc732043499,0x000d04a0679c71d5,0x00061e031c9df473,0x0005fe3572f00142,0x0000786b71fb2f13}, {0x00005fa6b64e56fd,0x000835219c46ed65,0x000f53d71e2fb48e,0x0004053dbec45bed,0x00007d782f39589f}}, + {{0x000b66b73ed09660,0x00082886032b350f,0x000390493968e4b0,0x000dbe5a16ed6e88,0x0000a83ce84a121e}, {0x000fc10a3d9cda09,0x000505ce67fb7c86,0x00037dbf65a40a6d,0x000282dcb8cda7f9,0x00005b44768f51f6}}, + {{0x0003c8a446cd7f44,0x000b506d52a60651,0x00023866c158c423,0x0008ee31503261c4,0x0000b96f570d3c14}, {0x0009cc7239a85237,0x000625ac4b8b5daf,0x0004bf7f6611b597,0x0004436e3981db72,0x0000e7d0f78d7afc}}, + {{0x000101ed6fc38377,0x000eee1b3a09984f,0x000942626340bf99,0x0006184b96036f06,0x0000bc878ab7c7da}, {0x00016441c6fb0358,0x000e2182fe9fb374,0x000c7a67ac65bd5a,0x000d7c9b9c2fb86c,0x0000d1b19afdce68}}, + {{0x000b80c8ce59954f,0x000388222ac03d1a,0x000f878dd742c5a9,0x000a984ddacbf894,0x0000c085118d7d54}, {0x0001dfa21e38ec27,0x000aa6f4ff7ffb0f,0x000a888fe1c7b59c,0x000889288752397e,0x000005d270d210dc}}, + {{0x000692a87dec0e16,0x000d97b39d00e5aa,0x000cfa0b5010ded8,0x000a281b1b80c854,0x00006beb87700f8e}, {0x000f5313476cd0e8,0x0005308d394950d7,0x000479fc6a63d0e6,0x0007419a09eea953,0x00002ae98927499e}}, +}, +{/* digit=17 [{1,2,3,..,}]*([2^85]*G) */ + {{0x000b9105ca7d8669,0x0001aadb3b34ab58,0x000eac0bc582967e,0x000af4f9ae4447cc,0x000019c667d0bf56}, {0x00017b160f5dcd7b,0x000f2dcaadbc9aec,0x0003467f5ec697b9,0x00032e5b98f34146,0x0000187f1f859671}}, + {{0x000dcb6ec7fae9f1,0x0004dfb66e5aeb5d,0x000445d52995f271,0x000620cee95d8e69,0x0000b6c2d4619e27}, {0x0001c318129d7161,0x0000f958c1aa3262,0x000f4af63b03909f,0x000df67c468ef91a,0x000062c42a00ba5c}}, + {{0x0005eef9c053df72,0x00079300ea6fe8cb,0x00049cffb8de25b3,0x0009bbbb03fa92c8,0x000042e43a808416}, {0x00051f4dd6f958ec,0x000f34445a8de4fa,0x0000d89496925a77,0x00039026e72a50e9,0x000066648e3eb1f6}}, + {{0x00096f31711ebec8,0x000f4e98fdc46c3b,0x000b4411f2da40f1,0x000bb6399774d357,0x00007c8bdf495b65}, {0x00089e3c2eef12d5,0x000aec7471f3da3a,0x00012c594de95bb9,0x00056b100f225bd8,0x00004907c5d7b75a}}, + {{0x0004d2945c1dd539,0x0007931debb5699e,0x0007f00e0cadc589,0x000a0e4f49fcc7a7,0x00003057bc0373e5}, {0x0007ecd027a4cd18,0x00034614011a2f8b,0x000677c68114734b,0x000f4f67a01db767,0x00009d9be5efe273}}, + {{0x000ee5931fba2393,0x000368bd91d1e0de,0x0001a3c1df47424d,0x00033adf8886f407,0x0000f7d41e8d8192}, {0x00023c2cf6eb9980,0x000f609a287f7086,0x000c9076286bb49a,0x00054b942bb24963,0x0000ef6eea555a96}}, + {{0x0004a2e649d4e574,0x000fd917526e4add,0x000b44ac4cd53a2a,0x00031d8526233020,0x0000028746afaa2c}, {0x000839064291d4c7,0x00017e5ad9095131,0x000185681bf48f15,0x0004425ce57f597b,0x0000c3ac1b0b854d}}, + {{0x0001db6f79588c00,0x0009b55768cca80d,0x00054438afa52fc6,0x000a4f0b4df1ae7f,0x0000cadd1a7f9b46}, {0x000a6b31803dd6fc,0x000495eaae35b40e,0x0002e4e16488e4fa,0x000c97df047d5538,0x00009b5b7e0ef6e0}}, + {{0x00072d0b611c24be,0x000480a8f351c199,0x000cf64211d468e6,0x0004910b7580697b,0x0000c9dd0ef68fbc}, {0x000d2bf956c2e32a,0x00043cddf94e5b59,0x000ee766573dc686,0x0006c45d5e2321bc,0x00007b4f8effe9a0}}, + {{0x0008f17cf41c6e88,0x0009837b925c03cc,0x000d2427cf1f03c2,0x0008e4439c19cc66,0x000023d24bafb6c1}, {0x0009013901f0b4f7,0x000188941c2e32ef,0x00028092e684360f,0x00032e9ebaff522c,0x0000891e4e3956c9}}, + {{0x0008f82d8d38a9b9,0x000057de1391174e,0x000c175dd2e97c60,0x00003535709850a1,0x000069041a0c2ae5}, {0x000533b76a2086b3,0x000ba7c2e8fecbfd,0x0009dfb67d6bba71,0x0005d982d58ee609,0x0000a8b342d364a8}}, + {{0x000bf0502f40d9ab,0x00077c318a4df83c,0x0009c26744681c46,0x00092de85756180e,0x0000e79d046c8470}, {0x000480a78bd01e0c,0x0003b2a51db9af1e,0x000afbab66dd359e,0x000198a2ce3821e3,0x00005cee5b6d7733}}, + {{0x000d8fc3b922bf8a,0x000fac29b133671e,0x0006e99c4e4d8c09,0x000eb9e7eb12393b,0x0000ff3974d2793b}, {0x00094052c18df9b0,0x0003910071390374,0x0007a0b95c5c3a29,0x00096b6a77234fe3,0x00002c29a21b661c}}, + {{0x000fe2e34d74e31b,0x000f8bf79ab65a52,0x000feeb8fa352c30,0x000304e7ff6c5aab,0x0000fbe8ff0a5c97}, {0x0001ce6a79046089,0x000a34fca249d608,0x000e5e2001f812f3,0x000ee80b24bc9ab9,0x00001022c67c8012}}, + {{0x000e7d7cc5f43941,0x000c3536e142184d,0x0004aa60ab5551b5,0x0001d51e89b212d3,0x00004a96feb05005}, {0x000ef740d12bb0b3,0x000030b9677e4e21,0x000f7731dc522f02,0x000d315b12e4672d,0x00009f80382ab326}}, + {{0x00038b67c4a658ad,0x000870e72182c127,0x00098e44fb3c4763,0x00085e6b77be4687,0x0000c047df2e7a7f}, {0x000d4c55e59d92d3,0x0005b8e64d8d2439,0x000ca9b16cedca47,0x000dfe7724cd0d87,0x00005e4fd59d5540}}, +}, +{/* digit=18 [{1,2,3,..,}]*([2^90]*G) */ + {{0x000344f3a29467a3,0x0009951eba6d9894,0x000e5c2f2de81e94,0x0007b3aaea066ba5,0x0000fc8a61438c8c}, {0x000f88f06d0de9fa,0x00049b75ce0a7adf,0x000bc87d5bbc11cf,0x000de1ffbb7accfb,0x00001458e271badf}}, + {{0x0003668e039c2560,0x000b7c17fd5d1cb4,0x000aa062b5f26fb8,0x000f04eee426af79,0x000072002d0d78fb}, {0x000a237e84fb7e3e,0x00002c82133d4c9c,0x0007e4181b401d8a,0x000151caa525926d,0x0000943083453dbb}}, + {{0x000ad1e6ee7c983a,0x000c141328f52f5f,0x00078d5f7eb0f9d7,0x00026efba68b441d,0x000006b3b9ed5bc7}, {0x0000255593e1ff1d,0x000d0fbec115a255,0x000a8f046552dd43,0x0001325c48a7abbb,0x00004fc56a47d0bf}}, + {{0x000da31be24319a2,0x000bc095a8e7f92d,0x00078218503f7d28,0x000dbc852fe84098,0x000076ddafe49c24}, {0x00054961d7a64eb7,0x00090f1dbe4280cd,0x00038d2d5e436088,0x000035bf81a87784,0x0000e4d52a8f5169}}, + {{0x0001cefba04b5d1d,0x0006524c735262f2,0x000c6171f9a44270,0x0006afef07966c33,0x0000b7ba891ee081}, {0x000ea59033745a93,0x000090a78f67306f,0x000251bf92aacf7e,0x00008685aa3883ad,0x000045aede956bd7}}, + {{0x000d5b11d59715df,0x0001e788983e19e3,0x000f1f248c7eaa76,0x000d82f5a730b0ab,0x0000bab8085eae3f}, {0x0000d2153765b2f5,0x0003aa127f3d65e5,0x0007b1b10bdd4e08,0x000fd34cf3c07439,0x00009f8090d01b59}}, + {{0x000e17d98119f103,0x000d188c36a6dc1d,0x0008e23df74353c5,0x000592e4aaf33a3d,0x00001e075c0a8baf}, {0x000a03a46d1ca3c2,0x000c27b660c70f7c,0x000fe2e5999c5e3a,0x00000550d0241388,0x0000e9a6be14a7ec}}, + {{0x000fd9d615faa8f2,0x0000768554ed7b15,0x000a448828fa1eb4,0x000f325bb4447e7a,0x0000bb2d0d1229ff}, {0x0002a646caa6d2f2,0x000e02e7351b075e,0x000506c628eb879d,0x0004dc9cd5624e9a,0x000018eaef0c87e2}}, + {{0x0009695241fbd6f3,0x000fd81c1223ac44,0x0006aac6f67c9b16,0x000720e6868f21b5,0x00004bd8fa428bcb}, {0x000bd33b6691c76d,0x0005681a797306b6,0x0004078db6c92476,0x0001cb9a12444ca5,0x000002e91aa3d105}}, + {{0x000684744ddfa350,0x0005dab3f74737e5,0x000e96cf49ccfc5c,0x000b8f9ac1df3f1e,0x0000c0571a13b480}, {0x000b3d54b3a7b3cd,0x00088dcdbb992fbe,0x000415b3a35c0366,0x000d9982a0f5dcb2,0x000057759b51413e}}, + {{0x00079c00b33d3f8b,0x0005c064e4090773,0x00029c8f6421883c,0x0000c77d0873d76c,0x0000fa433a48274c}, {0x000778f23a5891e0,0x000535e2de0456dc,0x000b517ced663bf6,0x0006c2488fdb485d,0x00000bba55e19b22}}, + {{0x00047d83d30a2c5c,0x000e378a81dc1fe6,0x0001a4a9b0857f77,0x0003f451d5a33413,0x00000a94af9e9d39}, {0x0005c0bdaa6ec1a9,0x0002f8d2d7edbc3a,0x000614797ba9fe49,0x0005332b4335b4bb,0x000091c4d6902f83}}, + {{0x000eee78a058fb62,0x000cdb09121aedbb,0x0004dddceb9d19dd,0x000bc4841bb45bd3,0x0000dbc80b920964}, {0x000137d1d6cb654b,0x000a983d01c54ed9,0x00028e22e1b9016d,0x00046aafc501bc65,0x00002d2f8821cad6}}, + {{0x0008c28d2f01cb3a,0x0003375db0b15325,0x0007d0db493d6eaa,0x000492a19a2b0de8,0x00001e48f0478fe8}, {0x000faf6c508b23ac,0x000975d53549f747,0x000f9b838f137571,0x000cf4df5e58e2fc,0x00007186cef67fd3}}, + {{0x000fb2f286bad396,0x00033dcad1e25e76,0x000c7e904bad9efe,0x0005a500e75190ed,0x0000a6f063e6fecb}, {0x000ed85aed8acc3a,0x000bcd20af6c5150,0x00069dbfab56ccfb,0x0000a8de0d1e982c,0x0000bf5628b1c7e1}}, + {{0x00068cee978a1d35,0x00032ab92d0477b8,0x000a5b862e3a68b3,0x00041d0102979487,0x0000f0606c38a61d}, {0x000be276f9326f11,0x0004b6fe3c2e2814,0x000df73512f521c1,0x000e4407464d7dac,0x00000f5f9d3877f7}}, +}, +{/* digit=19 [{1,2,3,..,}]*([2^95]*G) */ + {{0x0001244c5f95cd80,0x0000f4ab95f4b06b,0x000e5836dda8c8af,0x000ffc1bae59c2b9,0x00007d51e7e3acff}, {0x0005e6ac2ccbcda6,0x000f2528c3e001e1,0x0009fead43bc1923,0x000710e3324577a4,0x00001a1b8848aa7a}}, + {{0x000ee31b0e63d348,0x000229e54fab4fe7,0x000e7b5a4f460057,0x00083140493334d5,0x0000589fb9286d54}, {0x000f5cc6583553ae,0x000a025649e5aa70,0x0000446520879094,0x000c4eec90450710,0x0000bb0696de2541}}, + {{0x0005247d95ea1682,0x000a0970764a172a,0x0009781691758fad,0x00001b8c803a511d,0x000099cfe2efe77e}, {0x0001e17b0a98927c,0x000060014495652a,0x00075b56a2e26e1d,0x000f94bae0af9f71,0x00002e22a81964b9}}, + {{0x0001a3ea2dee7a63,0x000c434b2284758c,0x000aba6addcde2f3,0x0000a77ba445d24e,0x00005aaf668a6cee}, {0x00004a9e5aa049a6,0x000d31103e847e0b,0x000afecc3e74083a,0x000f7a5eb183ce40,0x0000b89dea04a043}}, + {{0x000b1eda4f7e665e,0x0009c23df8e7afbf,0x00012c2f4403cce6,0x000c33d49cc83f13,0x00001cc2367571a9}, {0x000a6c0db92faac1,0x000ddc3befe17ab6,0x0003a0f36acd15e0,0x000e096f8ed98858,0x0000821de7715f9b}}, + {{0x0000e0399375235e,0x0006d9917970b99f,0x0004ec0677614c84,0x0005201ec93ce952,0x000040e7bf97b122}, {0x0000631ee4c47744,0x0009fb04914cb567,0x0009dd2266f03847,0x0001f8896e9429dc,0x00003489b6ccc57c}}, + {{0x00022a5c5f264649,0x000b1442809a7b97,0x000644810ca4c3df,0x000f81ed10986723,0x0000e495172cc924}, {0x0006968a2c5bc14a,0x000ec8de6b7aef4a,0x000a2cbb5750eac4,0x000445ae01884d52,0x0000c40830b94a5f}}, + {{0x0009d23fe67ba665,0x000043cf2f340e29,0x000fcf9139145076,0x000ddaa45b5ea997,0x0000be00843dbd7d}, {0x0003e05d53ff04d3,0x00032de91ef7358c,0x0009ec1a0bf7ccdc,0x0009977d684dbfb6,0x000067e7cf2b01fd}}, + {{0x000f1076103adf86,0x000c5e8e98108c54,0x000bd3c9b2d813d6,0x000a7651466fa85f,0x00008c65d22a0e1c}, {0x000aa40255f9164e,0x000df1a864b2b81b,0x0002ca1415b34c3e,0x000dea5602209b12,0x00007d7248f685ba}}, + {{0x000d227cc2338fb4,0x000950e2615346ff,0x0001a007689ff6fa,0x0003af7e57077933,0x00003d241c546e1f}, {0x000b97dde9b62a31,0x000490ae30eafdcd,0x000bddf7d6a06e98,0x0003c4b9bf16804f,0x000070471a2e3616}}, + {{0x000e207a6469d438,0x0000af753a85fd23,0x000b5ca72cb9f5f1,0x00078b5e2625d4fb,0x00002e4e54b37b1c}, {0x000ca5378b9e8143,0x0000e25b817a2cd0,0x000cfd965fcd4405,0x000974868f719f30,0x0000164471a0848a}}, + {{0x000a8ae3113655ef,0x000a67b83180ff5b,0x000e0eabefa2c6e2,0x000a962c48271977,0x00009f3c556237fe}, {0x0007022a42581cb1,0x000208f710e3340f,0x0002e5aa8e1de0bc,0x000940de640adef6,0x00006b2389159428}}, + {{0x000738f25f653f59,0x0007a764f6355045,0x00011ffc3e42b8cb,0x0004aa24f89406dc,0x0000959314546b3e}, {0x00049f3c97400525,0x000105ffc48b81c8,0x000f67a492c9cf4c,0x000c621299e52177,0x0000869f6e4100c6}}, + {{0x00019e455950cc3a,0x000bb6b66bb83616,0x000ac6d84c71d665,0x0007e34a034b34af,0x0000987f83385e4c}, {0x00027727a79a6a7a,0x0007426d6c23a074,0x000167e1056e5d01,0x00084dde50b97638,0x0000a6c81f0888aa}}, + {{0x00086f75fe015763,0x000caea51395b841,0x000f3f8d4446e276,0x000142c3c5ef8105,0x0000d7f7df6a4f7f}, {0x000b565f9ef16565,0x00042c414ef67c69,0x00029206087efa42,0x0001ae9b7e620d3d,0x00000b1de34beec2}}, + {{0x000f3b7b0dc85957,0x00082f1d9f2e0ca1,0x000dd82a727de460,0x000447aaf3bf39ba,0x00009356a79d5862}, {0x0002345f5f9a0529,0x0008839a42f9c060,0x0004d40fc1a8b0f8,0x000368253eee4284,0x00003b0bfe5de5b6}}, +}, +{/* digit=20 [{1,2,3,..,}]*([2^100]*G) */ + {{0x0003a5dc8de610b8,0x000ae7e223ce0f89,0x000ad6dc5e8c515f,0x00028ef774bfa64e,0x00009d20f96125c7}, {0x0000966098583ce0,0x000493f2a7d77a1e,0x000304d4aa2eedb9,0x00082d1b2820974c,0x0000842e3dac0772}}, + {{0x0000e9d74cd06ff8,0x000f2ca3eeaca101,0x00063aa2b9c17c7d,0x0004fef4c86cd380,0x0000595c4b3f3461}, {0x00000ca990f62ccd,0x0002fa0c3be5a3de,0x0008ce9f5d9bed21,0x000443a886078adf,0x0000db27ce42cd44}}, + {{0x00097befc15aa1ee,0x0007d54b07455a30,0x0009a5f1240d1254,0x000ee57bad470651,0x0000d03f7188439d}, {0x000bb6c4a02c4997,0x000d5ffe71d20794,0x0003adcaff725083,0x00030fbcad75190f,0x0000f68ea1cb3729}}, + {{0x000581d26ee83821,0x0005259d638e9c7c,0x00028ae3dcf17dcc,0x000047de8273abb7,0x0000d1129270821f}, {0x000847750491a746,0x00019de0dfb91149,0x000a435ab687fa76,0x000e3ecc2580227e,0x0000b8bdb94f1ce7}}, + {{0x0006d8af7f91d0f2,0x0001882a57289c80,0x000d767543b61b0f,0x0004c62640032d94,0x000073eb5de67d83}, {0x000abf77b4e4d538,0x000f5e4017772988,0x0005071b3b7ce66b,0x000a981fba6b3271,0x00002413c252d3a1}}, + {{0x0007b7e3cc8ac855,0x00078d02753b7553,0x00037df2f8d725f5,0x00031dad05ff64b7,0x00005fe871346d25}, {0x00004a96ab6b01c8,0x0008fcd9372457ce,0x00086699b69a02a8,0x000231cf82ac35cf,0x0000242d3ae1cb4b}}, + {{0x00035269be47be0e,0x0007eb28fea169c4,0x0006c67e5323b7dd,0x000e461a5538ba3a,0x0000f921d70fd378}, {0x00061fc3c4b880ea,0x000df8940a67f929,0x000f0ff393f6f914,0x000f9c0990eb0afe,0x00006c2921090eef}}, + {{0x00003a553fb2b560,0x00074e057f78b23a,0x000e490d96ce141e,0x000e75796525c389,0x0000bc95725a31a7}, {0x00067911220fd06a,0x000ba08b0bd61ec5,0x000ebeba9716e3a3,0x00066f91cd6bf7e8,0x00007326ca75ee6b}}, + {{0x0003d32343bf1a9e,0x000757d1a6b170b6,0x0006865b48fd3bd2,0x000fa12454879c31,0x0000e959ff7a458e}, {0x000dcf89706dc3f5,0x0001c64e4b2e0461,0x0008843c8737db0e,0x0006f5b92626802f,0x00004498bbcc745e}}, + {{0x000352b5acf6e10e,0x000ccfe652c35341,0x000577a7fc50343f,0x00023c6af3792d18,0x00001a4c6188f168}, {0x000d0cd33425d0a3,0x000d6b7bc47f9b26,0x0006bb20b306399e,0x000054fa792f3370,0x00001219614c8111}}, + {{0x000428cd5f30851d,0x00065a4f66304c1f,0x0005d48a494dfed2,0x0000cd7df53772fc,0x0000d2d5a3063326}, {0x00015bdd44cc7a57,0x000d4d12533a5741,0x0003057c94ba6b20,0x00020dc0e93cb824,0x000094c486a84de3}}, + {{0x000355699f241d77,0x0006901a349d6095,0x00089e491ee4adbd,0x0005459b35bf6aaa,0x0000f0076f4836f7}, {0x00018ba9264da3d1,0x000d52a7a28bd19a,0x00061c9716eb2d2c,0x000a5dbdba941f87,0x0000550518bb3be4}}, + {{0x000d97302f1cd1e1,0x000b0dd212a4c232,0x0009802f7ce87eac,0x000dbd5e4c8c73e6,0x00002ef02902fffd}, {0x000c74e1bcea6e23,0x00040cb92cbb941e,0x0008f9d05d0b5402,0x000aae509fb9d47e,0x0000bf1615a22992}}, + {{0x0007007eafbb1e1a,0x0006475b7a93b249,0x000b68d78d75c9ce,0x0003959558352def,0x00002f26699c23f6}, {0x0001ecfe469b17ae,0x0009072d3ec2eb91,0x000cb113f6254577,0x00098caea47de782,0x0000be4b0872e1fa}}, + {{0x000e675b055cb405,0x000b477b5167bdb8,0x0002fb863898f8e7,0x00001f9cc65651b8,0x00006544814bd88f}, {0x0008e95263a75a91,0x000f0a22fcdab092,0x0003bd37ccfb6836,0x000664551d14db3f,0x0000d3837fbc6ad4}}, + {{0x000c982cf7d62d27,0x000cb3ba815020d3,0x000763f9e1f36e29,0x000006d8ae0bf092,0x0000a527e6b8d3a7}, {0x0009097581a85e38,0x000f5c158be5b4a8,0x0007d726e1f1a520,0x000862798db37d16,0x0000802786e9113e}}, +}, +{/* digit=21 [{1,2,3,..,}]*([2^105]*G) */ + {{0x000149e36f09ab05,0x0009fa10bb5befb2,0x000e2099803f163c,0x000bab8029704506,0x00006f0af006b5a3}, {0x000cfec70880e0de,0x000ede3d913f7af4,0x000ceb4bd7332a66,0x000f5452e6c84a7e,0x0000dc4a79b7c228}}, + {{0x0007dd0c55c44969,0x000695bbabd2c37c,0x000d7f363a6a9635,0x0001decb7e63f2ad,0x0000dce3782be73f}, {0x000a16ab2b91f71a,0x0002bba0163ce1e5,0x000e515ade448982,0x000ecf52759c32f6,0x00005e2f1f92615e}}, + {{0x0009be7abded5516,0x000868b744107451,0x00010d9a903d358b,0x0002b6ed00b10b0e,0x0000392b0b188da5}, {0x000a2980b75c904f,0x000db8f7f96c6744,0x0002cf932c305b0a,0x0006c9142e421d18,0x00006fc5d518e463}}, + {{0x00047c9d64cc78c4,0x000b5b6cb27b7958,0x0008022ab6c50621,0x000a1cc7099bf8df,0x00008f862ec004ed}, {0x00032ede1603c166,0x000efc9a9450d127,0x00029b4fc19a80e0,0x00051582257f54b4,0x00006d3b2c6a5460}}, + {{0x000f87e822e37bef,0x0003353bda4e6ca4,0x000190aeb73f237b,0x0002840747f3a241,0x000006fa370704cf}, {0x000bb6efc621c127,0x0003d0b80ec60a6b,0x000a556f35d624b6,0x0000a7db0724257b,0x0000fa0c354ae2d2}}, + {{0x000fa31e3229d41f,0x0001a4531bd4e921,0x000d38209a929c65,0x0007bc94156027a6,0x00003d69f745bdb9}, {0x000d19a168336319,0x000d73d51be38906,0x000511cd868a34c2,0x0002a83b59583b0e,0x00009ce6bfe8dc13}}, + {{0x000daaaffcdb4631,0x000a24a38b083fac,0x000a9078d658bbc1,0x0005de02a801f8f1,0x0000567bcf97ab85}, {0x00098e03572359bb,0x0004d659e68be084,0x00023807ccf0353e,0x0008a20b86e9c87d,0x0000c08728dd198e}}, + {{0x000b7bc453cadd69,0x00072c0bc1f88de2,0x000abd3af203900a,0x000ffb2cd86e47a6,0x000011cac131502e}, {0x0000242ec965469f,0x000139e0017e2d55,0x0009798850e9f769,0x0001ee733f078f65,0x0000b87d44a3cf75}}, + {{0x0000e4bfc25419a6,0x0002ebff3cfde179,0x000b6e83f3646720,0x000fd268db638625,0x0000cc69f23ccad6}, {0x000e45a6bc68bb94,0x0005e97f73340219,0x0005dc97ce43d79b,0x00049a3d44536846,0x0000b9eea326a0b9}}, + {{0x000c6ba6102d0212,0x00080f4461ea1b96,0x0009f19a8eaafac7,0x000875b4b85c41c4,0x000075c28e4ef538}, {0x0001a9ddd2e54e03,0x0004d605618b3545,0x00036cd246991adb,0x0002162b8b4bcd7b,0x000072a4f8c86f37}}, + {{0x000bd73a6a5da60f,0x000fec4c9ff0c890,0x000536e576f083d9,0x00024304e14d94f0,0x00009ee1edb9aec8}, {0x00041ec8bdcf8e7c,0x00004b041e265712,0x000fff040a5db827,0x000201da0b9a99e3,0x0000aaf21de3c271}}, + {{0x000b2e14f0dd2e8f,0x000e1a377ac7b4e2,0x0007a2198e77e7c4,0x000eb779202c3f0d,0x0000759b80018200}, {0x00026eddcfe314e5,0x000403d5cf99c875,0x0005138b6eb84c52,0x0003f461b52ace51,0x0000aa7ff8c73fca}}, + {{0x00013c3b9791a263,0x000a9dd58b16ff0b,0x000aad2de960022d,0x000619abd55c9257,0x0000baaaaa4230fe}, {0x00023460d881efdb,0x000f96325e2a9a4b,0x0005c18d4506416b,0x0007afe1381e7603,0x00003bb68bfa2781}}, + {{0x000b8bf5116f9379,0x00063126894315bf,0x00019a2c87c64a58,0x000462e1e25cc384,0x0000fd6b0c51335f}, {0x000ba3ce8ee0e0e2,0x0000098c21fa4bf0,0x00066bee06f6fba6,0x00054437d57b39ae,0x000092d513042672}}, + {{0x000105dbab093b30,0x000902839986f451,0x00074a89c012f59b,0x000e978a91580234,0x000048c919c2de03}, {0x000a2b591071cd58,0x0009834970a5c476,0x000b7994b791ed89,0x000ffd09bd9042e1,0x0000eaf517a21057}}, + {{0x000e2a2d551ee106,0x0008127e09a66066,0x00001148d87a8f1d,0x0003fda0d08bab2c,0x0000da8e4f1a24f3}, {0x00017f0cf9a4e718,0x000fbbf5cb19466d,0x00062ecc0ff50200,0x000ac45ccf97d8d0,0x00000c0d9b001d80}}, +}, +{/* digit=22 [{1,2,3,..,}]*([2^110]*G) */ + {{0x0007ceb1bf4581ca,0x00060ca7b1669885,0x0009722ace635e18,0x000006878ddd2265,0x00000903c4cbdb68}, {0x000458948f214029,0x0004296abda2366e,0x00040319031b49c1,0x0003fda29c4b09e0,0x00007197ca4629f4}}, + {{0x000de1cf3480d4af,0x0006cc8acf1a03e2,0x000295a9cf0d8edc,0x00097d023e330368,0x0000add5f69b546a}, {0x00097ad96f8acb1a,0x0004c71bdae28955,0x000dd43f4bddd49d,0x00041976fcd52821,0x00005a4541306191}}, + {{0x000c79f439f29cfa,0x000fadd82a3bfa32,0x000a2c1b17bf321b,0x0006185f127f54ea,0x00008365f2e935e9}, {0x00029024d0ef8dd1,0x000149228c4a852d,0x0005afed3395ce2c,0x0008e3a69f44e821,0x00006c1f898c2745}}, + {{0x000b6bfc360e25a8,0x0004875a1a788ce9,0x0001732f4e642519,0x00057a1dc756a848,0x00003c0440fd432b}, {0x000b3f1d720281f6,0x000e7135e051c670,0x000052be72205910,0x000a397ed14b0edb,0x000097b3d282568e}}, + {{0x0009005dc4532331,0x000a1fac4bcfdef2,0x0003d55acc208c47,0x0002075057a3feac,0x0000723725da02c1}, {0x000c62733eb0fec9,0x000b2b3c63bb9c31,0x0003058384913ccd,0x000d96813e542b07,0x0000d48e72ac10c2}}, + {{0x0004ec4d9bbeb3da,0x000337b2921c5442,0x0006d37c534ceafe,0x0006afbe68022e29,0x000028e0a2bd59f1}, {0x0002dd9bc3f9d739,0x00092af3e1fffdd8,0x000736dc46939a8f,0x0008e6c5cff45c31,0x000010f56430892e}}, + {{0x000b349dfe392607,0x000703549093de09,0x000853b02984612f,0x0007030cede28167,0x0000d809bb499d17}, {0x0006bfb2756c4f00,0x000a93f02b80450b,0x000ad9561c59ff9b,0x0002c73e69545720,0x0000c7cf0be3331e}}, + {{0x0009b9afb3ff9ed0,0x000b17f6515c2e59,0x0004da44928c2e0a,0x0004521cbee4fd47,0x000071279a44f364}, {0x000ff6601fbe8556,0x0007ffda51c497ab,0x000597c0b3ee394e,0x00034ab90385f667,0x0000e9fccc7027ee}}, + {{0x000d419362fb228d,0x000136aa0be4ced2,0x00094b197894637c,0x0001e7aa4fc55eb2,0x00009cbac1dcd4ca}, {0x00074800ccdb6ce3,0x000550dfaa49068b,0x000033b78ca4c556,0x0008a7b835382176,0x00005db7525f5ce9}}, + {{0x000927274fd1997f,0x00010fadeb588cf5,0x000db260c987d203,0x0003bd6647c6c3d4,0x000008bb13c88554}, {0x000eb1056fad695f,0x000d78334cc7566e,0x000dd1db239e17dd,0x0002ff9a97b3bb7d,0x000091199d9b1c5a}}, + {{0x000a437bdc47fe45,0x0002ff3751f0402f,0x00057d48535bd252,0x000cb63c2281b7cf,0x00000083ea5df607}, {0x000e480df6d730de,0x000db1bb06a26f41,0x0001fb48a1164e47,0x000da6f9040c2239,0x000012df252ba23d}}, + {{0x000de9314092ebb4,0x000d028e240c0b89,0x000d2f064f17256b,0x000b148f89a7f393,0x0000f57841f21ed3}, {0x0004405e708d8553,0x0001d3f1c3d04ee1,0x000d7eed5856aae7,0x00027098e5424fbd,0x0000333e4efa3ab4}}, + {{0x0002b5148f835b12,0x00030adaaba8f9cf,0x0007beade4a70faf,0x00003280d2e24fed,0x0000b1e4e6735f04}, {0x000c0b91fb3ceeef,0x00054a54b0b4eadb,0x000d4a188b817b08,0x000e5f24a787257f,0x0000f13304713482}}, + {{0x00012ec4efe42dc3,0x00029664b2bc3120,0x0008f9cb97251b2c,0x000465696c2e6b79,0x0000543376a6b8f3}, {0x000e8416fbfb6a97,0x00039cb91a03337a,0x00002d855e0893a8,0x00073ee3744e9d7b,0x0000e673186906d4}}, + {{0x000996f3ef143db6,0x000dd99378bc9509,0x0008ccc899cf1e82,0x0006be559cd09b0b,0x00004b4a2fcb2cda}, {0x000a8b31bcd55df0,0x000d2507cb37d3bd,0x00010e49bfec4e87,0x0004453d48a85eb6,0x00002fbe1bd1f9cc}}, + {{0x0007adedda492f81,0x000a682972053bc7,0x000931b4cc11a3ae,0x0004bb4e89a3e734,0x00007512e2eaf569}, {0x00049f3177bf8b6c,0x000948c7ff3e5dc3,0x00011145d232ea4b,0x0009c2dc4f9d16f5,0x0000cf109a3f3b37}}, +}, +{/* digit=23 [{1,2,3,..,}]*([2^115]*G) */ + {{0x000857080eb24a90,0x000d488e0cfd60e2,0x00059cdb87bedfb4,0x0000a9721ebbd7c2,0x0000b0da855bc639}, {0x00004dbde314c709,0x000bdc32e8462b4d,0x00062fc9ecdbf1fb,0x000ab6a3833eabb1,0x0000939b48c40dd3}}, + {{0x0002c1f711b0eb9e,0x0007980ab9549689,0x0000792dbb905f2c,0x000125cce26309a2,0x0000c8ac9b3e684e}, {0x000d8b6b40a24474,0x00015fe3fb24486a,0x0008e3b3f60121fc,0x0003941626fccf1a,0x0000e568622aad1f}}, + {{0x000bb22236f4a982,0x0008e66800bb5cfb,0x0009a77740b0c59e,0x0009482ac69a8f5a,0x0000b33f804f6bec}, {0x000929532e6c4662,0x000f2e599c73b372,0x0005c31cc68956d0,0x00084e847a249f15,0x00004d80f0e01ce2}}, + {{0x00046c64a8a3d626,0x000fc47743d25a4b,0x000f7e4338469c4c,0x000848cbb3a13d88,0x00002b23a1061be5}, {0x00096b4a63d1a4c4,0x000a3d183f3ee835,0x000afb01c454e7fe,0x000638243fce6117,0x0000e65e5e65c4c3}}, + {{0x00030e3dc09508b4,0x000f34317655b7e8,0x000690355faf6d2c,0x000ed632606cebdf,0x00008bb92b410c3d}, {0x00054845c7cf8929,0x000945d5f01f65b7,0x000401d69f6cd7ac,0x00087832c30a5996,0x000012686517d921}}, + {{0x00077176add85450,0x000672c49b66e5db,0x000421d771b71cb6,0x000fead856073968,0x00003840fe883e3a}, {0x000dad51ec699775,0x0008e07f6726b391,0x000ca160cae243fb,0x0005f4788ac87be8,0x0000174cced9ce35}}, + {{0x00067f8ff81578e1,0x00008045447d7520,0x00005aa6f7862215,0x000c77b0c22fcf05,0x0000030f0a67bed1}, {0x0009f151f0bd7390,0x0007e6debe8531f2,0x000677e982d7989c,0x000fd55c070e728e,0x0000a817bd306e81}}, + {{0x0007eb6cbc613e57,0x0004d97ea61cc1e1,0x0007eded533131d5,0x00011abf69d39eaf,0x00003c2f4354e6af}, {0x0002493a4a375fa7,0x000c4833c5c24ca5,0x0006e71cf5f06787,0x000666114e091f3e,0x00006451f57fb746}}, + {{0x000e082e1539388e,0x000f959a9c6b01c7,0x0006bacfbfd286f2,0x000e52b458104117,0x0000580f07d1bc3d}, {0x0006f75884d772fe,0x000760bb9d4fdcdb,0x000083011d75c3bb,0x00098cfc6c2e4edb,0x000018789a3ad9c2}}, + {{0x000ab8495fe1347f,0x00087f24503c5ee6,0x00086dd6bab0f6c3,0x000ef4907e3ffb44,0x000000b6c757002f}, {0x000f9a6a78629992,0x000c9ed89e2648bf,0x0008419eb85e5a06,0x000f1666d311af3d,0x00004f3ad7854733}}, + {{0x000551311820d440,0x000c86c4473a5e20,0x000d80eef0c651bc,0x000be51d230c2b9b,0x000042c4a1207515}, {0x0007ca0bfe9e2841,0x00081369827a4251,0x00038699de8f6057,0x000d05084f04f016,0x000074618ee02bc6}}, + {{0x00003fc6c68d6876,0x000cbff052c75e3e,0x000d16e7a3e732c3,0x00074692d0efa66e,0x00003d92b27165bb}, {0x00082badd44867cb,0x000e48c081b8ffcd,0x0006c8785a71b4a9,0x0002cfbc1676a773,0x0000e2c06174d893}}, + {{0x0001a4b9eca1c9f0,0x000c8ce5e5357bfa,0x000eec30a960bc1d,0x000d38e267d9f317,0x00006fb89ef6c257}, {0x000999ad364a26d0,0x000b26eaab582328,0x0005ba59669b794c,0x000a94ad28ab1fb8,0x00005dcbff356d0a}}, + {{0x0006d282bcffbc46,0x0006a6eadb7a5337,0x00035ae69708817a,0x000fde0ff50e05cd,0x00003b5fb75d4bc7}, {0x000e953e7fe08c4a,0x00045583ca1871c9,0x000e81c5cb4d8bfd,0x0001383e8d788245,0x00005f5e93d80474}}, + {{0x000160f293c9f316,0x00038864d7a7426c,0x000bbba308eb5633,0x0003025e1164023e,0x00004c2bae38fd5a}, {0x00001e1265aff7bd,0x000f96fd4b148006,0x000075a9c52d8a87,0x0004c58aba20e746,0x0000aa32bf94e123}}, + {{0x000bdef694db7e04,0x000779fcddc680f9,0x000b8dce1edca878,0x000ba111981c3403,0x0000274dcf1b0e10}, {0x00043b86def6d1a2,0x0000cbdb1866f727,0x0000c6f58d25b167,0x0007f5a4491e8c05,0x0000be2b2aba7fbd}}, +}, +{/* digit=24 [{1,2,3,..,}]*([2^120]*G) */ + {{0x0005c9dd111f8ec7,0x000d47c4e7603e0e,0x000392a51bcc33f8,0x00092d002f9a91bd,0x0000da4a7963132e}, {0x0000ae30bb1151be,0x000722e322511a0b,0x0004e9e7854febac,0x0000b80a3a508269,0x000058ffec2c4fe4}}, + {{0x000349d29c4120ba,0x000f20d0d915fbb8,0x00010ba519f94391,0x00091124074fa754,0x000066adbf6b50a5}, {0x000543c34bfca38e,0x000fd9e1ccfcc164,0x00020219ce0f2755,0x000979b9da0f53e8,0x00008234499a6b49}}, + {{0x000c513516e19e45,0x000775c4d5937b23,0x000e71ef656e2e84,0x0004c54f727d735c,0x0000b6304a7479a4}, {0x000a7363ab7e433f,0x00000e742f836638,0x0007fc19f1adea47,0x000697f054b8545b,0x0000935381baa1d0}}, + {{0x0004f9d918e49366,0x000652513982b550,0x0004d9cb965035ef,0x000508a553a0c26f,0x0000cb10d571ea85}, {0x00057b7a242da112,0x000d472b726848d9,0x00002a96b16a4d3d,0x00063b1d7e637c85,0x00007c7032b930d4}}, + {{0x0006b7d5846426f7,0x0008b47d441d5536,0x0006fbf48e7d09e8,0x000d7ce10b404d73,0x0000fa003d15784b}, {0x000614f17fd95965,0x0000e5cb98db25f7,0x00083a76a49e0e0a,0x0000f7dc65957b2e,0x0000d40da8e1ddbe}}, + {{0x0008bb4a595939dd,0x00085874021737f6,0x000ad76120355647,0x00095ebe740e7c84,0x000089bc84460446}, {0x000da5d85a9184d6,0x000b3fc0b074f7f3,0x0008a888e562563b,0x000e7ba6d2e6aaf8,0x000012d8643761fb}}, + {{0x000bba354530bb24,0x00078b0869ea9fb3,0x000431163bde3ef7,0x000a3549bc90460b,0x0000d03d7d324819}, {0x0004f9e43b6a782b,0x0006ec88a68633ae,0x000ffedd9216db30,0x00083fe1dd88e000,0x0000280da9fc2bd4}}, + {{0x00086913e538cd75,0x000c3e08ad53458f,0x0005d15ffa7001f6,0x0005dd02b8c6e6bf,0x000048234a451121}, {0x0009d2d3d5b40458,0x0005ca904190ff5a,0x000607f8bb0ffeeb,0x000729d5a3aca448,0x0000cbd665cb0a06}}, + {{0x0003573f37f59371,0x000dc1e4fca5a37f,0x0008ab0fceb0f6c7,0x0006ac1965a554ac,0x00007fbf56c37467}, {0x0006bd9acf7d720d,0x0007402247662e2f,0x000d53bef41fc8f8,0x0007d0817a14b385,0x0000ae327a64d76a}}, + {{0x0005c891f5f82dce,0x00068361079e515d,0x000a353309a7f67d,0x000e1ac8da81e311,0x000044990c52b18b}, {0x000ed95af103e596,0x00072dac9261c7d5,0x00094b8d3ece8aba,0x000e835e82b09993,0x0000830f09a76adf}}, + {{0x0001ac194d7d9b17,0x0002582e7f1743c4,0x000da0fca5bafdd8,0x0007ad6f0614c15f,0x00004b043a818ae3}, {0x000afa19e71734c5,0x000e4c450f2e3ba6,0x000e242b115d5437,0x0003c1fa5883fe67,0x0000143bdc27c195}}, + {{0x000ce7c6b53f5f95,0x0004cb176d99a2a9,0x0005c081b6424659,0x000ee661298d36b9,0x00003505bb86d9a9}, {0x0009e61f2ba70b0d,0x0008bafad4533f6f,0x000eb4a6ad07e16c,0x000c8dcf1694bbe7,0x0000febceda0cb0b}}, + {{0x000d7f2b1f3390b4,0x000c65b61272c676,0x000e127a99f7a1b8,0x0007bf0ebebfc9c2,0x0000602500c9dd99}, {0x000771c4711230f9,0x000b720f09c17f09,0x000e5e38b058eb37,0x000bc01b693d4bfe,0x0000289eb1fd653c}}, + {{0x0002b391770f5a7b,0x00005e44eb82a44d,0x00069712ae4d4d79,0x00020d92e69d1e3f,0x0000f11c4d75c6a8}, {0x000f3e542c4224c6,0x000be49d941cb5e7,0x00050e878d6b4e81,0x000c53fd72bd1654,0x0000a61e28b4e25a}}, + {{0x0009dc7ab952578a,0x000186e84d0b54da,0x000872042b5423df,0x0006df08b64eeb9b,0x0000c205782f990f}, {0x00096eb21f4c77af,0x0003bab273af4ff6,0x00036b3f11a79c3e,0x00027939bc922e94,0x0000f807ef9c6d9a}}, + {{0x000a8f833f6746cb,0x00054ea990cac7f3,0x000ddb0a921e46f6,0x000554e15fd5c5ca,0x0000d41f01728614}, {0x0004434426ffb589,0x000584dbc204346f,0x000969b7f8055943,0x00039a63dd20fe5a,0x0000d59e9577899a}}, +}, +{/* digit=25 [{1,2,3,..,}]*([2^125]*G) */ + {{0x0000ba9b733aa5fc,0x000b305af2353c2f,0x000ac82a5dece47c,0x00018a38e3f715a2,0x000097ba641e203f}, {0x000550409c110608,0x0004c6af512dc3af,0x000f2814656ea2c0,0x0004947ac28daff3,0x00007fab43b159ef}}, + {{0x0001641d4c5105f3,0x000e3d7fbd650989,0x000e6bdb01ae80f8,0x0008606d67225fbe,0x0000b433b59afc4d}, {0x0006db693e856387,0x000273e9862f44e6,0x0005c32ecf7b5925,0x000f506b78515766,0x000002fefd81e362}}, + {{0x0006a5c97290293f,0x000be388acbffe75,0x000916bdabf04a19,0x000bec0bbbb9cf5e,0x0000489527481f93}, {0x0007ec32a5923d75,0x000bade0c370dee0,0x00019d8fcc7bc949,0x00093f6d51217504,0x00004f5d4768fdcc}}, + {{0x000475d0fefb0c3a,0x000aa6d7c35d3754,0x0003798a4d48fb56,0x0008e60070b63336,0x0000e89f3d32fdb9}, {0x00089c86363d14cb,0x0000b7abd27d970b,0x000d5a0218981752,0x000aedebf7d47444,0x00003083bb07ac72}}, + {{0x000a2ffcc5e62e9f,0x00046edb02a40acd,0x000120c763b8b7d7,0x000063b700741c66,0x000077e84806d974}, {0x0000678a5e3d4647,0x0002d6bf35a30d31,0x0002f9043de68b1f,0x0000cb5ae83028d3,0x000024e4717c17cb}}, + {{0x00041debe949a447,0x0007e46a4fa53897,0x000047bdc638e938,0x0007c9cfe6419ca0,0x0000047f6491aea5}, {0x0008a9041fbab170,0x0008576bdba254e4,0x0002afddcda8e0b2,0x0007bfe807eebcc7,0x00007d3336df4257}}, + {{0x000563b4f3011dbc,0x000476610c03fae8,0x00081af0bda33776,0x0008ebed4f6f2a63,0x0000277984d99537}, {0x000a10c5751d2b2e,0x00089a4bf3f45ab0,0x000e3caf370a18bb,0x00036058d07ad4ea,0x00008ef5530cc430}}, + {{0x000c244bfe209256,0x00032fdce86762a8,0x00038706391c19ac,0x0004f5fa96a5d5dd,0x00001d587d481d32}, {0x00073a2a37173eaf,0x000763778b65e876,0x000bab43e2384800,0x000fbd20f8441e05,0x0000a11fe133621e}}, + {{0x000389a8391d54b2,0x0008a8afe546edd0,0x000e60bbddac74c0,0x000b876525a4ad5b,0x0000419ac96b0ad7}, {0x0003a0277eee51b9,0x0005e239768d078a,0x000d48035e86ecf3,0x000a5b3b0a259d8f,0x00004db43cc2d29c}}, + {{0x000772e81685d7b3,0x00018f34a976047b,0x0005f48ef23f27d8,0x0005c3927608e291,0x0000b0b43fad521d}, {0x000fb2663ca72840,0x00041d4db8377613,0x0003b526b7f5729b,0x0003d187b1489858,0x00000b732a6bbadd}}, + {{0x000a25b90f8550aa,0x00027c8dae9d4fff,0x000fbb232254db3d,0x000f0eb8cef963c1,0x0000d1cb48244bfd}, {0x0006ea1958773c2f,0x0003110114f184d2,0x00051e67e5862266,0x0005c5fadb3a87f0,0x00003185723a69e4}}, + {{0x000f4262048e3968,0x0005b83d9de48e02,0x0001e85ad436b50b,0x0008d6778d348147,0x0000b01ea6b5005c}, {0x000afee97015c07d,0x00087e3ba2aed3c7,0x000d3a1d246cdf1a,0x000ff3aa42e50183,0x000054b52698541d}}, + {{0x000c4f948a48e22e,0x000df0335a96e120,0x000151c7bf24977e,0x0008d7baf3442336,0x00002815a80b3664}, {0x000deba66e0a6b21,0x000f717515e8b4d2,0x0003ff24a783a84c,0x0005d968424c0bff,0x00002dd1bb648e1c}}, + {{0x000cf304e23e9bca,0x0005726e36243f24,0x0000b6d614387f81,0x00077b86a46a033b,0x0000f1bc8462b2d7}, {0x00001ba527de79c6,0x0003e261bbb625c4,0x0007b4bc70e1346d,0x00062eeb96c44b28,0x000058493c7b2545}}, + {{0x000e9d8fdd41d983,0x00018cc7c7fbfcb0,0x0008fa4d98be980c,0x00088d772e86506f,0x000056b14ada48cb}, {0x000ccefea6178f26,0x000d18bb0e35fd9f,0x000f391a9c84a620,0x000c055c75367c62,0x0000a83a5c683100}}, + {{0x00049feb8a24a205,0x0001a52ca53f23f9,0x000fb485317ebfed,0x00005d4b691bbebc,0x0000617ff6bb278a}, {0x00034c5e3c99ebdb,0x000d6784156a241b,0x0005d67dffc64242,0x0000109206482f69,0x0000967ce0f9e27c}}, +}, +{/* digit=26 [{1,2,3,..,}]*([2^130]*G) */ + {{0x0000f0441c23fa36,0x000061989a2eb448,0x000a29ca7b4712eb,0x00028bdccbba0f93,0x0000e205c1536194}, {0x0007957b36416860,0x000d45ac8b4e90db,0x0004e03500432691,0x00051707a759acf6,0x0000514d89c9c972}}, + {{0x0002c6bc4fe3c395,0x00031c7bebdfe3b2,0x000693459ba4a815,0x000b11a23ab6b725,0x00003bc377064922}, {0x000c8ab5afc60db8,0x0004a0b9f2a34645,0x0000fc507aa02235,0x0002e6d2a2954cce,0x0000c2731bbfce1c}}, + {{0x000efb6d9790ed61,0x000c96aa793b5066,0x0003e042ea77a0cb,0x00074b0a915f3c22,0x0000c5def0479c58}, {0x000007873b6c1da8,0x00027cd8557a0e83,0x00060f3b155cf85d,0x0000628f7c7c7604,0x00007052acbc6e58}}, + {{0x00091c2e48fb8898,0x0002fb8a9d066a70,0x00082a0e226882c1,0x00052d224986631b,0x000044ed736b5181}, {0x000476fd86e27c75,0x0009b4afefdc282f,0x00019e34da04edac,0x00078b3b256ebc61,0x00006a413e95787d}}, + {{0x000b6a20d645fd62,0x000dd61d314816ea,0x000079ae9632cbd8,0x000cbbac1bf7cf62,0x000057ee5c8133ec}, {0x00034a81680ac730,0x000872c77aa0bf6b,0x000a0a1d1aa084e8,0x000b167b5a864e05,0x0000641f6db359a1}}, + {{0x000095dc8385050d,0x00050f4b441cf01d,0x00027706a0d54a5d,0x000b7d5a37ccb409,0x0000f008f5515d7e}, {0x00034f35bf716c71,0x0008541bd6ca74eb,0x000e6fa0257a65b5,0x000a09df345e4835,0x000091f913b98342}}, + {{0x000d46da670ff1d2,0x0008ab97a1cc1554,0x000d9749324833d8,0x0004986fa6ab3cde,0x000015e0371a9926}, {0x000d592e56d74ff8,0x0004c3b5e1ec549b,0x000e93cb958a8caf,0x00083bbc6087a323,0x0000b054987d648b}}, + {{0x000061d5a74be506,0x00047ea16ff582ee,0x000bfc8a2e41781c,0x000e2d80b0c81e99,0x000024f4d696b547}, {0x000545dbdcc9ae4f,0x0005509b1e8e3a83,0x000c935392573dbb,0x000797582960c4a6,0x000001059ae4ae18}}, + {{0x000a238013ff83b1,0x0002ead69d08c431,0x0009589ea7c0018b,0x00019f89aeb52a4c,0x000021f41abab1cf}, {0x000bcbaef0f59585,0x000b2be8fbdc0cfb,0x00015aa318deb3ae,0x0006fcc2b954081f,0x0000acc09b39c0c0}}, + {{0x000bfa86d518ffb0,0x0006930f124b775c,0x000e81d0fdecee1f,0x000b2f9a402804f5,0x0000e8225c52a0ee}, {0x0005d39fee9e867d,0x000f2b505454884a,0x0007a70d19540428,0x0002a9e2bf2e2010,0x00009917c3c7010b}}, + {{0x00042fa3d843d538,0x000593ef927aa98f,0x00084ca7433777cc,0x0005dd4440cdbecb,0x0000c22f9639dc7c}, {0x0002b70c8d947081,0x000cb814364f4bc8,0x000f59b7e7e0b43f,0x0004c4186d4e2486,0x0000abc895e5d6bf}}, + {{0x0001e274d559d96d,0x0002e8db6c013815,0x0009921af4f18c0d,0x00002879a3aa836f,0x0000beab27c5c046}, {0x0009eaa7040bf3b5,0x0004c614b091242b,0x0004baf5d39c479e,0x000944e38ede2b0e,0x0000bb192b840a53}}, + {{0x000572337e440d7b,0x0008fde23f68896d,0x000c64918685c5fd,0x00080c05b1a26dc2,0x00009390e318ad65}, {0x0001c4e7dee5b9b3,0x0003aeb04f6e8791,0x000065aa6b90c505,0x00028d07b942a18f,0x00004acdf2a4ca09}}, + {{0x00064d39de40ca36,0x000f72f3857e733b,0x000ed92f71d4b6d6,0x0002485e2be8e9b2,0x00004ca7048177da}, {0x000ae9b8da993153,0x0004dfc698a4c65d,0x000958c279c14517,0x0000975a296b94ff,0x00008684e0873950}}, + {{0x000e34b3390ff239,0x000b4e7d18ef7872,0x0007fe7b1968ce4a,0x000e2a0b4a745e62,0x0000607b0a15aff3}, {0x000818eeb40e3a5b,0x000410fa8d7a1b05,0x000ed48096ac6220,0x0005f205b9058571,0x00002432ef1a7cb6}}, + {{0x0009f973112795f7,0x0007284e6ee1715c,0x000b66bcde824443,0x000bede5cb4858ec,0x0000c1367361baff}, {0x00015955dbec38e2,0x000c188ad1535466,0x000e0952f51c0782,0x000fa87ba4c53ac6,0x00007e6782a3b21d}}, +}, +{/* digit=27 [{1,2,3,..,}]*([2^135]*G) */ + {{0x0009740e2c2bf152,0x000589e99704feb0,0x000fbc565627a220,0x000de8cc8d73d0c2,0x000023eed8fe20c8}, {0x0002583a8363b49a,0x000929c2b0a61ee3,0x000dbc85c1a0b6cb,0x0001aba9f7c3d290,0x00008dfbb97bef4c}}, + {{0x000236e846e364f6,0x000c7ea50ca0c16c,0x00026b86d7f33527,0x00070c6481077509,0x0000c2a36096598e}, {0x0005e52f024e9245,0x00044db4afcaa675,0x000831790e0fa07a,0x0000d5c5c3ce7d66,0x0000b4ef350f6cbb}}, + {{0x0006b2da6dc1d29b,0x00094871e1444280,0x000f49276d303000,0x00004af1feb333aa,0x00005583b9f770bc}, {0x000be7895695f204,0x000149d012b51db0,0x000f61643fc84181,0x0001282409f27205,0x00000d3417515883}}, + {{0x0004c050f15dde91,0x0007fd5f2b820521,0x000e82b62a47a76a,0x0005eeab254d3062,0x00001a05fe04ec95}, {0x000f46e9d529b36f,0x00009f9e3df67eaf,0x00031769855ab130,0x0007acd463e37199,0x0000d251439bcda4}}, + {{0x000669c72ba075b6,0x00055a4690158c3c,0x0009f8ba889f78b5,0x000ed6f706aade3e,0x0000d8bd566132d7}, {0x000e63b805f08d67,0x000d53bcc1b525f4,0x00025d8477f48200,0x0000a7de801968b0,0x00004afac04f7cbe}}, + {{0x0007598a9d82abfd,0x000b16c170f5e2a3,0x00066b0875f188cc,0x000ad9b168220050,0x0000a22c21397155}, {0x0005d3afbddb4799,0x0003dd715b99151e,0x00097cb2e4b606b8,0x000b65ba73b54bf9,0x0000a1bfe43cecd8}}, + {{0x0002522cccc18ad5,0x000da1a6e0277973,0x000c2354daadf3f8,0x000689a7382c9317,0x0000ce1680d2818b}, {0x000bbfcd9ecbee97,0x000bacae62ac359e,0x0001ac38a4330689,0x000ee8455ce5b4c5,0x0000921dfeb6e238}}, + {{0x00022f3dbfb894e2,0x0006ae274b18e131,0x00058aadfbe9b79f,0x00035165a49de5ca,0x0000495775831487}, {0x000ef61bb9390993,0x0009f6d13694111d,0x000fc253b1d6a974,0x00015e1474b4ced3,0x0000a1485e67c5db}}, + {{0x0004ddb0f5f27cac,0x000ae80d59ff6599,0x000601023e85461f,0x000bfb3f05481a66,0x0000665427bbc9eb}, {0x0001a697587fd52b,0x0007dd49efceb057,0x000420688935289f,0x0006aeb1becc60ea,0x000022639d9c3a78}}, + {{0x000dab61430c9ab7,0x0002b238e9975afd,0x0008042ae0bdd41d,0x0004cb8094743041,0x00001f9addb3dddc}, {0x000c016c52dd907b,0x000c79e2047f7090,0x0001011a6d9bdf44,0x000c7836f1fe801b,0x000063accbd89acd}}, + {{0x00076680448087c4,0x000f31432dae264c,0x000f9bf47ac30903,0x000d02f851b26600,0x000000ed311acdd6}, {0x00079fef8fd24247,0x000a8a6da98b045e,0x0001e673afdfd974,0x000167d5c9f6410c,0x00006f2e733cb2c5}}, + {{0x000acab4baef62e3,0x0002785b91e87817,0x000e576109f5a220,0x000e036666ebe66c,0x00002ad31f4273bf}, {0x00030a425bcf4d6c,0x0002915056e66283,0x000332156ea95059,0x0002d699811c89e1,0x000089cf1ff4c11b}}, + {{0x0005acbc46d7ce17,0x00071b085877889e,0x0007a50509a515bb,0x0000358ac1a03d0b,0x0000d3e738b62926}, {0x000c2ce2a6cb0ebd,0x0004bf7adc79861c,0x0000163766f2e295,0x00008f91c4d45133,0x00009fd2c812ad59}}, + {{0x000337ac0b7eff35,0x000e75e48b3c0ad7,0x000f13a5f8552225,0x000cbe96f78b0c73,0x0000e70062ed2349}, {0x0005048e7073969a,0x0009233cb3d26b8d,0x000caa20f392d2a2,0x0007074e4f727c4e,0x0000068c99ecccde}}, + {{0x000d65861a023efa,0x000419e5246e1888,0x000563ec01d72aab,0x000a4309a26348e5,0x0000097196463439}, {0x000d54badb9b5b76,0x0001645a524b567d,0x00038e60873fac1a,0x000f482fe97ef7fe,0x000008748d29f384}}, + {{0x00086ec1ed66f181,0x000bc61fce43ebde,0x000bed74d225d906,0x000ab74cab07d6e8,0x00006e4617f37855}, {0x000aaddb2fbc3dd3,0x000f5aeddf5b6568,0x000cf2fadedb5484,0x000699578f20e86d,0x0000516497c915f5}}, +}, +{/* digit=28 [{1,2,3,..,}]*([2^140]*G) */ + {{0x0003fecfa181e695,0x0000e0d69a98ef0a,0x000eab95d9ea02f8,0x00002162e9cf8e66,0x000020f2beb74720}, {0x000540a1df843618,0x0000f1fa6d5d621c,0x000f5f6ff1203772,0x000ef2ee3c7b510f,0x000017a069c2bb2b}}, + {{0x0002fb6b294cda6a,0x000519039f348357,0x0005cbb216ce9bf7,0x0000d980e012f009,0x00000aecc1c7063f}, {0x0001c3af02909e50,0x000f48ce9cdc57c2,0x000e336f8c7d59ec,0x0005f42732b8448a,0x000056e37233f4f8}}, + {{0x000b53189e800ca4,0x0006d45208fd8a10,0x00014ba3750fe0c1,0x000acc5e43c0d3b7,0x000027d200e74189}, {0x000e24fe616e2c00,0x0008ee1854c105de,0x000342a739c25f4c,0x0009524d3222a58f,0x0000807804fa027c}}, + {{0x000653a4f0d56f34,0x00078a28b805c222,0x00073434b961e404,0x000a18ec03f8b04a,0x0000c966787eb712}, {0x0006c42864fee422,0x000a3b0ece5ccc19,0x00031c159c1be93d,0x000655887d9f22c1,0x0000bb6d593fce45}}, + {{0x0009ec9b809b7ceb,0x000b32c72c2c22c4,0x000a0bf368a41486,0x000c68d13b9420fe,0x00003d36eea566da}, {0x000c08a328cc987f,0x000b4a3264616fdd,0x00010dbba0a3bcd2,0x0004c38103c49dd8,0x00009d81a293b78a}}, + {{0x00065ade4d559419,0x000da03840873de8,0x000f18b9bdedafa5,0x000267df414abb4e,0x0000ee9ea438aee5}, {0x000aa1637a55a4a5,0x0003b15f93b9260f,0x0009c3598eb19a51,0x00078e01d7ebd29e,0x000023fc56d69321}}, + {{0x000070cb98fe684f,0x0009224a1458501d,0x000bc6b3fd60fbe9,0x0007cab45761c892,0x00005384859ee6f2}, {0x00071f7b59e763bd,0x00088b5a8e5e4b02,0x000a482923d4606a,0x0004454eda5d9b05,0x0000a7731d1b6fec}}, + {{0x000369390d458714,0x000fc6166d8da3e3,0x000a90403e976403,0x00063775c3368289,0x0000bd17983b2f1d}, {0x000679ed5d2c53a7,0x00088dcf3b87a616,0x0006a694e5ec4bcd,0x0007e53e6d7613b6,0x0000460fc7753fc2}}, + {{0x0009b8295caabee0,0x0005889501e37046,0x0006ed265de024ca,0x0008b26bdadc0607,0x0000cb1236b5a0ef}, {0x000ddbf0972ebf9d,0x000452aca4324065,0x0004aff76f1dd387,0x000d23d88b97cf74,0x00001359afece8e3}}, + {{0x000ba2b91502cf37,0x0007984db75d52a3,0x00030b1c92c3832a,0x00060b94a12dddde,0x0000802eabd531fd}, {0x0007327a37fddab7,0x000b8aafa9733370,0x000e6f91a65d6f2a,0x00030ac525c5b811,0x00006aeb0c9cf465}}, + {{0x0005ff62f93a6750,0x000405f48679e881,0x0008ae884a6ec968,0x0008736dcbb55635,0x0000af61472e19e3}, {0x0004372a5f696be1,0x000a5f22fb707233,0x0006cea90c65e57e,0x000b2a168da30c94,0x000036a8a8775681}}, + {{0x00081dc0f9f44d43,0x000ffc46585aad5e,0x00047d1b1f09a695,0x0008b1a1649164c4,0x0000b4b36c8b79dc}, {0x000177b3b6b234cb,0x00046730d9d020d4,0x00080531d096a250,0x00095c5611b9b8ef,0x0000a904b3c14bb4}}, + {{0x000d9d493a3147a2,0x000c7a5655451192,0x000f072129f30a5d,0x000c1370b1f9cb6e,0x000099585462d87f}, {0x0003effc17db9ba2,0x000cab1644a8d332,0x00049ffbccb18548,0x000683f8a306d44f,0x00008d658f16c2e8}}, + {{0x00060cda99f8c71a,0x000aabf742ff44ba,0x0004b3f9967b7abd,0x000160c6310f9c91,0x0000e430a339412c}, {0x00076d388ace52f4,0x000412d7067d1e67,0x00007cd1b4bc0fa2,0x0002c0c3c286aa8f,0x0000cb8f38ce985b}}, + {{0x000be808c3bff364,0x00021263e57583cc,0x0009bdcd1005a0bd,0x000b6b060d7dda25,0x0000a1c56433a5ca}, {0x000dbb99fe4fc88b,0x00081c97bbb52b7b,0x0002321ae09418e2,0x00064e28274fb4a1,0x0000137007e0c87b}}, + {{0x0001fe1c63c4962c,0x0008d81fdb258053,0x0004c2b6b50541e8,0x000fca1c1291a1fd,0x00000693a1866df4}, {0x000604e0117f203b,0x00025a99b8d0b2c4,0x000212c44245f196,0x0002a7fedc20aac6,0x00001ed4e57020f5}}, +}, +{/* digit=29 [{1,2,3,..,}]*([2^145]*G) */ + {{0x0000fb6700a1acd0,0x0003fd999681b556,0x000b4e1bae823fd7,0x0000a3da915d1f6c,0x0000d0301186ebe0}, {0x000b0c989fca8cdb,0x000b49da0e0b744f,0x00031d76f970d01d,0x0009695ad8c56479,0x000015737c0a659b}}, + {{0x000384ece53c2d04,0x000d1e4606daa12b,0x000ec12b0779d897,0x0001ad653e47b073,0x000062dbbba9756f}, {0x00009f2cafe37b68,0x000f6cce2e1769fe,0x000f607fd273d1eb,0x000c250ac1d5383c,0x0000035f7ff92e10}}, + {{0x0002d5a2093c22a9,0x000145703aedca44,0x0003287b6ebd0bd3,0x0008b9a08f2afd65,0x0000bb88bac9d1bc}, {0x000853875c1e3b2d,0x000ffa11447cfbaf,0x0005c4c8dbd2ac94,0x000207586d816cea,0x0000c3aa800f8dc3}}, + {{0x00034c77e6c55202,0x000fbcb9ea58854d,0x00086666dc27df9e,0x000a85205f2369d6,0x00009d1febf2417a}, {0x000819e93470afed,0x000912a27f9e9846,0x0001e65043e6a966,0x00080954d008a2e3,0x0000ba7ced06cb76}}, + {{0x0003e41d07fa53db,0x0009c4e35bc526a4,0x000da2f8c3154a78,0x000f99cb768924e0,0x0000a964a2bd3613}, {0x0008d35ba1d16c4f,0x00010b54d0575a54,0x0006402052e1bfed,0x000f290f992136bc,0x000039cb9157156d}}, + {{0x000e64699b444ad9,0x000d340504c5c913,0x0002e53dbddfce99,0x000536c8482a99d4,0x0000aaf2c2661aff}, {0x000f7962664cf67c,0x000931393e2bee90,0x000225bb074ab5c9,0x000d6c3ad0faeae6,0x0000355648ced63d}}, + {{0x0001d271d91cf9d3,0x000d8377b20ae4e3,0x0005e1327cb35d4f,0x000e7544de1e4505,0x0000298e31b58703}, {0x0007237de0133394,0x0000e3d101c65508,0x000aab0dc32cbf30,0x000bb9870dba22e8,0x0000a52623d7d155}}, + {{0x000f541338d6e434,0x00030541d5ccecaf,0x000bc88ca56f7dd7,0x0002c375d426de96,0x00008d94f6bded3a}, {0x000a3bb2ef8279cf,0x000a1b1867f26354,0x000225151d575465,0x0000d7ff99b0ff95,0x00003e19d89e9450}}, + {{0x00035acc85dcf572,0x0008a88f5415dbf4,0x000c558076174565,0x00056f06367e9a17,0x00002d077a5ea90c}, {0x0002258a2e04e763,0x000e3e06e405fbf7,0x000e6f954ba965d3,0x000a742724d06fe3,0x0000e47d475b1251}}, + {{0x0000151e0fb82f76,0x000675668ac2b8ba,0x000d711b00d16072,0x0002f2022ba25814,0x0000addf557df3fe}, {0x0001e1c6b9c94354,0x000a773826bd2b83,0x000240f89ce3a060,0x00003a53fa11c11c,0x0000f9cc8d8d56e3}}, + {{0x00031f6641f82c90,0x0004dffec7564b93,0x0009158abd97c7c5,0x000a7ff5ee6d1f1a,0x000054493a385c3d}, {0x00005f5eb7d96df8,0x000764473a39a57a,0x000e16d55a3afd44,0x0006adc2a4d9c488,0x00003e144f675f87}}, + {{0x0003268e32dd620a,0x000ec27849a292a8,0x000378882913ec99,0x000cfe4dd8fdfa2c,0x0000f96f33f8e6f8}, {0x00037e5dc3fa8a51,0x0001a0b03a1dc067,0x000f037b0236bb53,0x000a5323e59f2989,0x00003f9b5a7e9a12}}, + {{0x000ae9559029aa6e,0x000b7a4db2ed50d8,0x0008f373dd74e292,0x000b372b9c335584,0x0000c018db78c45a}, {0x000690269cc53a8b,0x000c6a8798641f44,0x00013475e8c4b628,0x0009f56c743d284b,0x0000f4a933923de1}}, + {{0x0004f47f0cefa69d,0x0009d45468664a8a,0x000f603c1dc8e4cb,0x000ac4659ba69b23,0x0000ab4d601e87b7}, {0x0004337c1ebc8d9d,0x000382b4074ba6ca,0x0002fb7339fa6585,0x000ea94a4b4f8190,0x00002bb5d7b7525d}}, + {{0x00092d5f81f95671,0x000a3d698470eb2e,0x0004c81eb54cb95e,0x000c644f2acb28e0,0x0000c1ebfc508cee}, {0x0009fac06e074235,0x0008e7fa0c858f79,0x000f4db4472225f9,0x00009160cd861634,0x0000ec36159c52c9}}, + {{0x000f6ce51efb3108,0x0004158df5be0d0d,0x000158e59cb5b2eb,0x00033656459e2936,0x00002aae2b99466e}, {0x0008a39411aa636f,0x0004e4c0a933fb65,0x000f026b77152ecc,0x00011f010c758a49,0x00004837f98bb093}}, +}, +{/* digit=30 [{1,2,3,..,}]*([2^150]*G) */ + {{0x000523a626332d5d,0x00028561bb44994f,0x000845ea27bc3883,0x000089305ed4b03d,0x000039d3ee292a1f}, {0x000fdd3e7676b0dc,0x000f3b7060176561,0x00064f9a8620e35f,0x0001f676ce424ff2,0x00004c341a09a268}}, + {{0x00031769bb816483,0x0005353120d000f8,0x000cabc62d69eb48,0x000cb1a17d75f44c,0x00004a07f82e749f}, {0x000f787bbfb55541,0x000052e283f82c3a,0x0009213a0b06ed4d,0x0007b44722889fa1,0x000062b085eecf3c}}, + {{0x0000ddaeb300f7aa,0x000c4db5e80136d9,0x000d5244c9dcf7df,0x000aa1c45cb26874,0x0000127ee79d48e3}, {0x000cc53575f1dbba,0x0004e0e6161e488a,0x0002650d095037e8,0x000215b7e5928329,0x0000be67d99b4938}}, + {{0x0000d2f7189e71f4,0x00081ecf91e73267,0x000757a21c643874,0x000ce4d5758e57db,0x000027d09f8690a9}, {0x000308f38384a7a9,0x000420732b99846a,0x000845819aac3acb,0x000e030e94100917,0x00005cba11237ce5}}, + {{0x000aa377378058e6,0x0000a4411154eb81,0x000828ac741c746a,0x000b29410c73bcfb,0x0000439be91fd972}, {0x000b4b043a2fbad0,0x000f82b5e8404bf3,0x00097bd4c39e6dad,0x000ccb4f71640863,0x0000f7de5687f1ee}}, + {{0x0002a73f37ec3c39,0x000d4d59eba0db33,0x0004d3257c65259b,0x00038f9291709cdb,0x0000a793b264d389}, {0x000e34be43756f0e,0x000dbafb56c9f39f,0x000208b272f76bdc,0x0002c2bf37867a61,0x0000a1d4307e8997}}, + {{0x00044878a429f4f8,0x000b5b516609d0a7,0x00069b5df0649712,0x000af23826ba57e7,0x00002335df29fc7a}, {0x000f0675c93d9950,0x000a68677be62389,0x000d9951b59ac367,0x000ccea77985ff21,0x000038956fb85011}}, + {{0x000851aaaca5e9b7,0x0000e6713b9797b7,0x0000a61f6518aa52,0x000b68c357e8c715,0x0000842e7e35c2c2}, {0x000af656868a5489,0x00025068fc818dff,0x000917733d963bd8,0x000327d4da5c8b65,0x000027091000b247}}, + {{0x000db942f6d0d97b,0x0000943b9373d6ff,0x000d36db605c3ee4,0x000c3e02e541ebff,0x00007415a977e1dc}, {0x000682e163aa2f64,0x0003ef3af218e383,0x000fdbadb46febdd,0x00021a50a0507eba,0x0000ca8ab4e0e52e}}, + {{0x000832e1b7cfb737,0x0001cec354c9a3f0,0x0008eadca7a8afe5,0x0002bb3e91c97e37,0x0000449c59a1c3b3}, {0x000c3710b1c4655c,0x000a379da87ea619,0x00038d96a692e4c6,0x0000ec3f3f5d86de,0x00005320f42dc08c}}, + {{0x000f2bc8fec5dec0,0x000f84d6786c92a6,0x000ffa084a71383f,0x00071587588c06db,0x0000d85f5ca6857e}, {0x00011b3b6c774d40,0x00084c3521a8e873,0x00029fe0f672357c,0x0008cde5fe74615b,0x00002bc51105b715}}, + {{0x0001e48a105fc8eb,0x0009789ba48c37a0,0x0001c2180769d754,0x000387108c6fe1d5,0x0000032dd3467bd1}, {0x00026db020b0aa69,0x000f7664c73c9538,0x0000cf95d05137e7,0x00028a366302c466,0x00009004e1242cef}}, + {{0x0005b284f9735368,0x000e535b04ea824f,0x000a03089b43e299,0x00042c472c88f74d,0x0000269d57ac5a2e}, {0x00063fc6607b38e8,0x000ea9390b0c7c1e,0x000ee2869e89e2aa,0x00097447c740da1b,0x0000556f6fd2f3fb}}, + {{0x0003995a4b6bed1a,0x0004f7095c541a0a,0x000a73ce92dab579,0x0008a666c6a1ff2a,0x0000dd0a54bede43}, {0x0006b1afca906cd1,0x000619de10dd160b,0x000633da325fc601,0x000d5a248e9c99d3,0x0000fe3f746258a4}}, + {{0x000edc88be85c1d0,0x00083ca09cb6c253,0x000e3055add3d0e4,0x000dbfb997f6879a,0x0000c929ad007431}, {0x000621584d2db42b,0x000e578828cdcef1,0x000bd4b66b50df3e,0x000b039589da9d6d,0x0000c4fd2e4899c2}}, + {{0x000c9a7d298c241d,0x000986807cfd214b,0x00064eadbe3b697b,0x0009c51f1c780245,0x0000de8cdd084814}, {0x000f0a75a4d2604d,0x000e9c1538af946b,0x0005b1fcc27154d7,0x000f81c5cc9230de,0x000088519ea36864}}, +}, +{/* digit=31 [{1,2,3,..,}]*([2^155]*G) */ + {{0x000dd1a7cb1282c7,0x00064e46973ab828,0x00008d6b2a08d762,0x0003f2fbaf8d40e7,0x00002571fa1bdaeb}, {0x000732ff22dfd98c,0x00064087108d85b1,0x00088207a87ab01a,0x000754eaaafea859,0x0000cc832f929f00}}, + {{0x0001185ca8d9d1ae,0x000cf987ded2488f,0x000c46124adf2c77,0x0005f37f3039f060,0x00005d70b7651e09}, {0x00086506260e70ff,0x00070750d10582d5,0x000bac36439d75ea,0x0003289cf3d0b175,0x00003a7564e11d01}}, + {{0x0004700ec3128c2f,0x00019e383f4994ab,0x0003024eb6c76d86,0x000c68ec36b150c0,0x0000b43947843daa}, {0x000764a8dc796230,0x000db440fbb2fc68,0x000c5ee0d5b86995,0x000bd3d66879bfcc,0x0000522894295aa8}}, + {{0x000a43e3fcd3efca,0x000418088e9ab24a,0x0003d46eadd26c03,0x000a6f05ef4dc9bd,0x00002f99d592a4c6}, {0x000d3552f1da46cd,0x000d4afacdd1ddab,0x000d4057872c3f8c,0x000b94090c4eee92,0x000028bb4209a623}}, + {{0x0003becee8314f30,0x000ddbea298f5e7c,0x00080acec1c068ae,0x00095b08d381f17c,0x00003b56be8e3304}, {0x000b8f29222882d6,0x000664af8bf7aeff,0x000c57d8c95ff38f,0x0004eff0e32d351f,0x0000635be5277b44}}, + {{0x00092d2e7417ce17,0x0001270ee7f52427,0x00067a41eff42bc7,0x000a57aff4dc6d5c,0x00007709b7b90882}, {0x000731dbe217f2cb,0x000cabb721773554,0x0001dd0592af2a8c,0x000476a8eee76959,0x0000b2930c9fbba6}}, + {{0x0009126cecdaa7af,0x000d1b13247b174a,0x00084c1c4fc8c7e0,0x000c3a39c110d234,0x00008eb8758731df}, {0x0000212c00674527,0x000e0b9b926c022f,0x00042daf43f6f69e,0x000de399032da0ef,0x00009f00adef3f80}}, + {{0x000dddaf176f2c08,0x0004625726581e6a,0x000342ffb01ca460,0x0008d58a404ded85,0x0000cf60f96c4183}, {0x000691cc9071c4aa,0x0003944428039bbc,0x0009c0d81fd58874,0x000f7ef7101c8580,0x00007fb754d2c456}}, + {{0x0009ff4c1e99d816,0x000643c617c0f855,0x0008c6ba708e1a7d,0x0007945398fd4324,0x0000ffedd9231283}, {0x00059d2d629d2080,0x00053490530e8a6a,0x000505989a9d141d,0x0004ee42f6fc1838,0x00009bf250d479d9}}, + {{0x000a1d5af71013f7,0x00049bedc9466af7,0x0007370a0e68216e,0x0001cc84cba30bd2,0x0000981afbff7042}, {0x0006a679449f0e1f,0x000d1a47edae0249,0x000feca2286cfc4b,0x0008fa4073c936b1,0x00005694612f3f8f}}, + {{0x000f9e12cb7191e8,0x000865b08ea6ec14,0x000332bb978ea1bd,0x000e24bc65aa9b46,0x00004cc22b43f80c}, {0x000e9e9d49d5bf18,0x000599087da40098,0x000f6e357cd4ec1c,0x0004b7bc9d07c5ae,0x000039a02691f8f6}}, + {{0x000d6715bde48f8d,0x00025189bc7dbcad,0x00009ee8ac970387,0x000ff78d45299ec7,0x00001287ee3545aa}, {0x0008874db1dbf1fd,0x000ac90c88d67d1f,0x000368313ea46588,0x0003ad90ba649a84,0x00005fdcbcf30d54}}, + {{0x000643037577dd85,0x0003d9c5fe88f795,0x000bdc13283b82af,0x00039e4c1bea26cd,0x000089fa086ec043}, {0x0009538b13799dff,0x0000e295d034033e,0x0009ddcca85fa8b2,0x000333ef17f73fbd,0x000032bd123cdb66}}, + {{0x000e9959890272db,0x00098e713a10cf3d,0x0008227b875f3432,0x000dc7ae13479fe2,0x00008561eaaaefac}, {0x00097a08332aafd7,0x000503809b62a6a2,0x00063036f9b0d8bb,0x000da862fa1cfd0c,0x0000a16eb562d64b}}, + {{0x0004ec13cf482834,0x00055c8a705e4cc3,0x0007d4f84b09daa2,0x00029d91e9d0d05b,0x0000df6ef651b389}, {0x0000763aa21ba469,0x00081293f8fbe16b,0x00020aabfc6b1d17,0x0009797ff5b602d5,0x00004d671be53393}}, + {{0x0008e8a2ac13e274,0x000f0eb1a9f5f7e4,0x0001f0a624494f6d,0x0008f0adbf84eb98,0x00009badc3293643}, {0x000a541004f7571b,0x00002f1c94ee50be,0x00027bc31bac67d1,0x000e27753d73a1b7,0x00003d01cf2e0686}}, +}, +{/* digit=32 [{1,2,3,..,}]*([2^160]*G) */ + {{0x000e50f6d3549cf8,0x000f7acd665ed433,0x00011fcb46f33696,0x00085fe95bfdacce,0x000010ee2532f7c9}, {0x0000fe17159bb2cd,0x000da58b357b6545,0x0009fea72f7dfbeb,0x0007445b057e74d6,0x0000485717b62731}}, + {{0x00042e8ee36860ce,0x000c6113c22d896c,0x000104213daf04df,0x0004e93adbb7b744,0x00005fd5fa1ffd39}, {0x0005d941a4e0551e,0x000d38d101516823,0x0009845236772cfb,0x000a97476071e309,0x00004e879df3a56b}}, + {{0x0000aa9b898fd522,0x00079e9af1a76c8d,0x0004f03f82fb38a5,0x000c6fc1f2b9a93b,0x0000b1aad44e3f0c}, {0x000332e7cf2c0846,0x000ea367d26d58b5,0x0006e4a8d1c57d96,0x000b69c297eabdfa,0x00005a947eeaa0e2}}, + {{0x000afb0285b94916,0x0007be4c705eaaaf,0x000d9caab01a0be8,0x00033f9f1d4f5d2a,0x0000e349a4b237a2}, {0x00012464a1c6a163,0x0005f9383260cf1c,0x0006d5471d99e6b6,0x00089bba3d43665f,0x00006974d052f8cc}}, + {{0x000b616fdd5b8549,0x0007c728719ff535,0x000921cad592549c,0x000ef85231468606,0x00008c8ce34c11b1}, {0x00037e7e9090b363,0x0008dbf7bbb728b9,0x000d8797467fc3ab,0x0003fde2337097a9,0x0000e5adca22970e}}, + {{0x00049a1cfe89d80f,0x000cea9c8371c26c,0x000d066d2b42c026,0x0003edda6c013ada,0x0000b8f722946a4f}, {0x00079ecd850935b3,0x000ca631e1b308b5,0x00019853434c1a74,0x000f259b5fe596ac,0x00009ff21f711f24}}, + {{0x00068a7b3f85ff0f,0x000c2a888044cdcc,0x000dbe894acd21cd,0x0000d3c6719b2e05,0x0000ae1d3d97b826}, {0x000dece8a1c5d92d,0x00040c52077eedfe,0x000dd13edbca01a9,0x000aacf085549c16,0x0000c5c3baf195eb}}, + {{0x0009e148f9290579,0x000630c853df27f2,0x000e9c5ce7a64ae0,0x0002a4956cd18358,0x0000d9cce836ed09}, {0x00059796e93b7c7b,0x000181bb9e27cc6e,0x0009e29a0e1e4709,0x000644070b3083aa,0x0000f181a75e785e}}, + {{0x000063fbe7b643ab,0x000796085760cc17,0x000214c9e7872e1c,0x000637d6b0fffbb4,0x000018bbc0f22bf3}, {0x000de0c722591c92,0x000928c29e0c8b17,0x000304f201edeab1,0x000fb67fbfd98ef4,0x0000d1dbb6bbc77f}}, + {{0x0002c658ead09f79,0x00050780d14df53f,0x0001b66bc1335e1d,0x000fc7d9cc20e0cd,0x0000b670a384be0b}, {0x000dc8128efbeedb,0x000bd326a6e5ce53,0x0008e9a630c74e77,0x0002478604e0d2b8,0x0000ab38fcac3dc2}}, + {{0x00016d3c7141771f,0x000a3f226b662556,0x000ca63a9a86691a,0x000f2aea19fea4b3,0x0000c05dc439e672}, {0x000e786718ba28fd,0x000acc66b984a9c6,0x0003702f207b7995,0x000efe3f434f551b,0x00006f62130aa84e}}, + {{0x000e8c85c0a3f1ee,0x00019c87c37f8ed6,0x000e3b78dbcad249,0x000a461dfb62bb9e,0x0000ba8e478abceb}, {0x0008cb0eeaede4b8,0x0007f976deb637d3,0x0006147fb0bc498e,0x00060932944c046b,0x0000b123f36771f9}}, + {{0x0007987b5b41d786,0x0006ebf0c4f84b0e,0x000b80ecdea7df90,0x0007e554d03560fa,0x0000cf306f75b1db}, {0x000fb5689fd4773f,0x000f10f9be330d59,0x000352da4ab254f3,0x0003ee28a09a9277,0x000081862f6541ea}}, + {{0x000dcc7de79dc241,0x0002458f69cda155,0x0001850dff1168a3,0x000848aac215950d,0x00005c8295bc204c}, {0x000aa367d8184ffe,0x000d50447bdbf661,0x000e4a59ec396228,0x0005e531cd5143bd,0x00003a26e3c4beab}}, + {{0x00001579f759d014,0x000f3eae4fdeb59b,0x0000ba8c0a2923d2,0x000442d832775769,0x0000bf7e38b84f51}, {0x0002563b413fc268,0x0002f9e53b36b681,0x00089f66dedb7d36,0x000415cfa585c4c3,0x0000e1adc31d4bd3}}, + {{0x000a13f1402b9d0c,0x00026c7bc863d3b3,0x0008c3e6e573441c,0x00057d8b301ec457,0x000026fc9c4cadaf}, {0x0001bfd7493cea35,0x000ecaf8145696e7,0x0008c608fd05d4b3,0x0002768aca2a8a6a,0x00003ef07f65725b}}, +}, +{/* digit=33 [{1,2,3,..,}]*([2^165]*G) */ + {{0x000c9d646ac49d20,0x000b83137aa9a6b5,0x000225a3842c77c0,0x00090724d000fc68,0x0000f63cfc82fe1e}, {0x000b01bc6441f959,0x00095c8e448f22d1,0x0007fb1ba7d38f71,0x0008df0b33fa5f78,0x00004dcfda1a9015}}, + {{0x000de10296c36eff,0x000192c4da77211c,0x0007836da7ee8967,0x00060ac617d270a5,0x00000cd9c328cb75}, {0x000cbf7e455fe908,0x000afe7334f301fd,0x0007de4ec3fb53cb,0x000fcff81e2ea44e,0x0000adab3ad8b384}}, + {{0x0008a2b599ff0f94,0x00074104fc6b0177,0x000694ff368a923d,0x000f121bfa44dfda,0x0000f7199dc37667}, {0x0008ff6e46f2a79c,0x000d29f8131dc06d,0x000b4ce7c08b5dea,0x000c3d42519a59ab,0x00004f710bd742ae}}, + {{0x000368b4ed80940f,0x00058a6fcedd3014,0x00097579f67e6d05,0x0007f58c208c49ca,0x0000e3d7a8292359}, {0x00032027e096ae27,0x0006b4b393665e20,0x000dcdffcb1f3e1e,0x000e82b6da26f32f,0x00009422f1dd097b}}, + {{0x0009c748c878145f,0x000e49c635655054,0x000cf2d5e67f14ed,0x000d0f32ddf78c9b,0x0000faa843058201}, {0x0004a9dd1b2de28e,0x0007254be41447d9,0x00082755bc09c4be,0x0003bccc80e178ca,0x00003251d113697c}}, + {{0x00009d35001417b1,0x000e4962ed5e2bbe,0x0009f46c3795e84e,0x000a2e4b79d1ca27,0x0000f7f8a3b93836}, {0x00000b14a64dc32e,0x00042f84f7396922,0x0009155c0c842433,0x0005071110da07cc,0x0000e8fbe62394b0}}, + {{0x0005391035703ccf,0x000950c7e24d2f6a,0x000fcbb9a9899bf6,0x00075ad3f7f248bb,0x000065027dfc5558}, {0x0006b69ffff3b37b,0x0003645b4431a7a1,0x000afd67967b6eb5,0x000cde69d7e1d249,0x0000d819babc10fc}}, + {{0x0002cfb9db3b3818,0x000e54df0a4b263a,0x00004e61f9c3a2de,0x000324f28d06e97d,0x0000b1adfbcc2449}, {0x000d9397e053a1bd,0x000696daf7076ec1,0x0000ac7abee2be5c,0x000173b0ba1e1481,0x0000d2ae779c530f}}, + {{0x0008e703b9f04267,0x00059c84f17c3e70,0x000b70a0be5b02fb,0x00065cbf4ff35be3,0x000081b3c600b155}, {0x000636a6e124c3a1,0x000a7b496784e76c,0x0003f0370bde81eb,0x00045d9412f8e244,0x00005d42362a99be}}, + {{0x0003ae2a4af76c14,0x000bc8276fe70f50,0x0003a33c6550e66d,0x000f84d1e0c1fcbf,0x00002be231046629}, {0x000743e516ace497,0x0007d36a2262edf7,0x0006ca192ce43666,0x00013e71ac703644,0x00003631cb5b76fe}}, + {{0x00082d4a160bcfa6,0x000558ceb1f2d47f,0x0005a8d25258fb07,0x000c9cef818e8fdf,0x0000685475ecfd31}, {0x000385ae1e9b13f4,0x0004b0508bdbcef6,0x000f1f90de0a4259,0x00081e1ad7ae16ae,0x00005a155efa3f8a}}, + {{0x00007c28034b95e9,0x000587bdc5608ca4,0x0009807e8c93eb97,0x0004eabec24e8d33,0x00001b734d76d64a}, {0x000ce398f668b26c,0x000a541823d5d0fe,0x0009e4e004822cc4,0x0001adf953bc32f0,0x0000a0a7c60c6d86}}, + {{0x000d4470f33e712d,0x000555a87cb04c74,0x000962db43cec1e0,0x0006a98cec0610e5,0x0000971af1641a25}, {0x000c983a2ef4ac95,0x0003874e9d00a044,0x00073d881caa1da6,0x000c8bab972d8346,0x00000747c5a53a26}}, + {{0x0009982a3e25566d,0x000d48e1b8960434,0x000b2d24beef9075,0x000a547c7bead092,0x000099f72fbda21b}, {0x00009315005e541b,0x0007eece3205b93e,0x00062f2f62a7a983,0x0000a63b388ed114,0x0000488b15ade346}}, + {{0x000cd67969d56af1,0x000ccfc58a8b43f6,0x000e703779e0d872,0x00046c301c1509a4,0x000003d1a1318ad6}, {0x000e37e9d0624278,0x000c39bef78c078e,0x000e661424e69c5a,0x000d81d21ec00136,0x0000de1ecb32634c}}, + {{0x000d97a205b9d8b0,0x0004056756d40435,0x000f8210e6eb8f06,0x0009ead5e88a8bb6,0x000070ef12dec9fd}, {0x00095053bcc876ae,0x0007c7404ce34d84,0x000a1db5e12a7533,0x0005acf22b49e1b8,0x0000c1f2051f4bfa}}, +}, +{/* digit=34 [{1,2,3,..,}]*([2^170]*G) */ + {{0x00081b5c4e83d33c,0x00089efd488b43ed,0x000eb4d0fd9f3587,0x000393564a620f9d,0x00006927bdc6c6a7}, {0x0008df79f9e0f036,0x000e9cd7e1a945c2,0x000a348f12868661,0x0008e01cf4e8d0ff,0x0000bd4c28499853}}, + {{0x0006e8c06d75fc1f,0x00064249a89f5603,0x00045e7dd2dcf7bb,0x0002a691dd1d3de2,0x0000578dc4cdbd6e}, {0x0008903df2ce7a06,0x00083c39afac4c02,0x0006404abaee3628,0x0008187c847c3114,0x0000304c0d904e97}}, + {{0x00036c327d02dccc,0x0001ba68bcc2fb68,0x0005e912d5ad0098,0x000cfd5b24b44c00,0x0000c83d210411fd}, {0x0007ec1666fba0ca,0x0006747546353652,0x0006da9c26994819,0x0002b25cdcb1a855,0x0000593426821a73}}, + {{0x0008b33070d3aab6,0x000b3a2cd5e5e4ac,0x000fc91732643672,0x00013ef2eff79b1c,0x000065ca49bf0a7c}, {0x000da59b3efb9983,0x000cd52f13415a8d,0x000f9a5308a5b922,0x0004d77e9ebbab3c,0x00005986e7c256da}}, + {{0x0004cbd8088b454e,0x0006ba9e0c8a63ee,0x0002447cbdb7f32f,0x00019ad377d4186b,0x00003e982abb3702}, {0x000c1e4c2a2a5938,0x00048773f24f06cc,0x00085942372c3686,0x000c8f213b4da795,0x0000bbf1d33f5040}}, + {{0x00087d0ad886aac2,0x000a9771b64503c1,0x0004045ab5c16878,0x000aed907dfc6fc7,0x0000c6360bf9800c}, {0x0005bb5b9c972a3d,0x000dac9a6dba2429,0x000a79aa6c9e6f88,0x000ac1c0ffbf2492,0x0000e29d50b11c26}}, + {{0x000fb68d84d835d2,0x0009661dc1e6b1f0,0x00094f8d7c90caf3,0x000b91f2e5b04675,0x00006897ae285012}, {0x0008a08a4d6755dd,0x000b3991fbdabcf6,0x000bf17e8403ee41,0x000d64a33e343e3b,0x00002c7980e379b3}}, + {{0x000cf4fda79e5acb,0x0009d630215f534a,0x00085756e68b83b3,0x000cb1ac748b2ed0,0x0000031725995d37}, {0x000841ac5ccc2c46,0x000add9d50696735,0x0001754bd7d7dc96,0x000dd54147e410fd,0x00005296e953399d}}, + {{0x000c9a1deb8568b6,0x000020fb3d321e71,0x000f8fb81a35daea,0x00096a88b6f2662c,0x0000d51afe8f4906}, {0x000ac6e51803a198,0x000f0621908081be,0x0006f463ce3d24b7,0x000ee7f27cfd9ddf,0x0000c6865caf2284}}, + {{0x000f169d23233f3a,0x0005d7cb637fe00d,0x000a0cf6c3e32279,0x000bdc7f897c0e1d,0x0000651f5d8d1d6b}, {0x000af191a230c767,0x00025daa5e4add61,0x000abcd7ebd52727,0x0008dc5a753636d0,0x00008bdd37ca70bd}}, + {{0x00027c17078c4322,0x000c977fedb7cddb,0x000290570e1961b9,0x000885fedc2f5cc2,0x0000c3fefca39cbd}, {0x0000a36c2af389a8,0x000d3da71ceacf88,0x000aa846396c610f,0x00090a703977a932,0x0000eb776400586d}}, + {{0x000b3edf0290a8fd,0x0005fb47c387831a,0x0004efb4fcae8196,0x00010ddad7dece18,0x0000cfc53b417491}, {0x000f23c4cb632f9e,0x0005d91f80676698,0x00084180ac42a1ad,0x00026ed116a81d62,0x0000bedf5f9c9013}}, + {{0x0003f0ab2cf89408,0x0007ef948f519163,0x0002653c872b0b17,0x000a04ad28dc3078,0x0000882984a5b903}, {0x0005d0c6a19d2bbd,0x000bb6f782cbb809,0x0009070644b9e7f0,0x00043baaf739882d,0x000012be0ff5b326}}, + {{0x000f28b638a7e819,0x000ec980ddc39561,0x0006f247a54155cd,0x00010022db4a96d2,0x0000d774e4ed787d}, {0x0006e2e078637d27,0x000cee0ae06a1a9e,0x000cfa3541c363e2,0x00098d0493483ee9,0x00006843cb3ef74b}}, + {{0x000f1bc789a283b1,0x000d780836f40491,0x000e5402d72d3ac3,0x00073d9a1c5ea388,0x0000b192421e5cc4}, {0x00099989dc84cacd,0x000ccc6e75b85c0b,0x000191ce2b0a8482,0x00092f939961d03a,0x0000a3bc8663d837}}, + {{0x000c4cdb30cfb3a6,0x00010c9db4c8d7e0,0x000c8d9df6d09b8c,0x00066ce0ba1a4207,0x0000fd495f77c52c}, {0x000169f275264daf,0x0005f57d8362fb0e,0x000ad722280c2b74,0x000c7afdd987f749,0x0000dc229b03398e}}, +}, +{/* digit=35 [{1,2,3,..,}]*([2^175]*G) */ + {{0x000ed8452666a58c,0x000026a9c3c2b0d1,0x0009064084bcb6e0,0x0003ff7c57411c26,0x0000fc20755d3556}, {0x0001c505294dba30,0x00068b7dd31ea08b,0x0001eca74a30ba28,0x0002b9d70ba90e99,0x000094e142ce762c}}, + {{0x000783e979f39254,0x000a6f4c89a7b81d,0x0001bf7fa1efd130,0x000a9e125c2144fd,0x0000b2969045b265}, {0x0009634b9db65b69,0x000173599d8aed8e,0x0003563f335c82e3,0x0008ab4aa7a54f40,0x0000df088ad922c3}}, + {{0x000b066bb3fd30a1,0x000adff0354ee5cf,0x00024e36c429169d,0x000e1d709cf85235,0x000036f4fb31155b}, {0x000af011fbba712e,0x0003706ba1a14826,0x000aea73e6ef0f0b,0x00044df9928b3177,0x00002bf6af33eaa2}}, + {{0x0004f124237b64b8,0x000963ecfd078d08,0x000845dd8688ebe9,0x000324d7b8a70cf6,0x000008fc59cdda4a}, {0x0002b2ba3585862b,0x00053df29386a903,0x0001ec29bb66825d,0x000dc805a5a8db43,0x0000b143a98ea1e8}}, + {{0x00094ce12ae381b9,0x000bf6ccda9035ee,0x00006eaca3a7f176,0x0004df363a657e46,0x0000ae5a380d3cd0}, {0x0008d15ed251b464,0x00008aca5e649bec,0x000f20f071f5d6d3,0x000285f47b3b359f,0x0000d65f03537e4b}}, + {{0x000ba24f111661eb,0x00040105eb049e93,0x00024b578edced48,0x000068e6dc9ba1f4,0x0000f8f66b8983e9}, {0x0004df4d7ed8216f,0x00069e2cbecf872d,0x000e73754bf07f37,0x0007075281d89998,0x0000ec85fbc7aab8}}, + {{0x0000deea5ba5b0b7,0x000dd2d052999a3c,0x000b02d42e6a116c,0x000cb63e9775fee9,0x00002b0520111545}, {0x0006f7d31a3b4ea6,0x00082bbd9b32bc50,0x000b12a97e589307,0x00067168bc5f37e4,0x0000b000c06aa73b}}, + {{0x000bf22765fa7d04,0x000fdd6a537013b5,0x00080db9859805be,0x000ce327a5e29d42,0x0000f53916fb76b1}, {0x000f61f33ddf6269,0x000e1085d103714f,0x000809ee34206238,0x000b1c8c50d4b7e5,0x000099f450e15f8e}}, + {{0x0006051e4c79e9bf,0x0002d66a9fea658a,0x000be7b231394cb7,0x0008fd37f31ed5c6,0x00004c88f374aa6f}, {0x000721f4aaa499e0,0x0005e3fb2a6b0fb0,0x00092851d68b3a7d,0x000913a788097d3a,0x000060e7f8ae96f4}}, + {{0x000be731a3a93bc2,0x0005821adc1a82ee,0x000030efd42bbf46,0x0007bba10b6fa4ef,0x000047aa4c7a7b09}, {0x000c632f60c77da5,0x000a7223523e8b8d,0x0004579cf6ffbc26,0x0000f654f6ff1134,0x0000825653ce8025}}, + {{0x000097ebc1aa2b92,0x000317a0333ab2dd,0x000a0db380788939,0x000612fcf55e7137,0x0000648487f992c1}, {0x00013363fcef2614,0x000cceabf129dad0,0x000276be26239c81,0x000ad34ee761de9d,0x000006a7a345eda6}}, + {{0x00067ba4a493b31c,0x0001dbf7f0264bf3,0x00095914b54f20a5,0x0006abf696e06297,0x0000ddab96e4bf23}, {0x000c70aed25ea138,0x000b01cbbbe74ff2,0x0008544c5fa1d09e,0x00031708fc8c8746,0x000047a670de96b3}}, + {{0x000421e64bcb626e,0x000746dee0b5f133,0x00010346caea638c,0x000ed2f6e7680bb3,0x000006f4098b5d4c}, {0x00014527512a30b9,0x000d5589a59a0996,0x000d0c180f3d867f,0x0004ab9e73254f52,0x0000063d8a3c33c7}}, + {{0x000c595d314e7bc2,0x000b267899ededa6,0x0001ed5d32ee7464,0x000612fcef423c0a,0x000017e76ea89cc7}, {0x000ce1fe7cda917f,0x000a9a893f1627cc,0x000c74f6b12d8016,0x000e60ccd6de849f,0x0000a5817e3e3144}}, + {{0x00041640821ee4c9,0x00037bc619921f35,0x00072879f1583eab,0x0007b1e490caf61d,0x000098ad9f4876ae}, {0x0001950a41157f70,0x0006e8da3a7e1e18,0x00026b95fa9d7e1e,0x000a10963784eb84,0x0000ee4ed6e542e2}}, + {{0x0004cc5ac751e7b7,0x00028d4211bdb79d,0x000de4fc693f9647,0x0000641c72d3d2c8,0x0000b69cbf64f44f}, {0x0000ca2f4bf94e19,0x0008612894e23da9,0x00017d60b1a5325f,0x000b5c7a437f6c79,0x0000be7048726c9c}}, +}, +{/* digit=36 [{1,2,3,..,}]*([2^180]*G) */ + {{0x0009976e1337c262,0x000db73d68e5949c,0x000b768d96faadeb,0x0000697e158614f1,0x00002dfa557bcc4f}, {0x000da17be93c6d61,0x00019504f5b9ccd6,0x000694da124866c6,0x0008c61121353c8d,0x0000c6ca5801140b}}, + {{0x0004a7dd4b79bb87,0x000ae2c878c8f160,0x00047b8e8aee806f,0x000053c4144f118d,0x00002edf52c049f9}, {0x000a84e2127015a3,0x00006cb7cef3ebfc,0x0006deec89051d0c,0x000d7456e8fe5829,0x00003b2818871010}}, + {{0x00060ed9aed9f40e,0x0000732a8c99bd56,0x000c371ea70ca6ad,0x00009ce4978bfb95,0x00005464d0e50031}, {0x0002fdfd9e535ef8,0x000718c9185b1af3,0x000b42488abf57ea,0x0006fa6d7a741712,0x0000e0296a869728}}, + {{0x0009383171b445fe,0x0002a131ad4c0107,0x0003987e89bcf21e,0x000c8eacdfe205c9,0x000063f4153a92e8}, {0x00062a930add43df,0x0002d980f05a7294,0x00006e96862ebb14,0x0006b05f3954e53b,0x0000e1d75ae142cf}}, + {{0x000416e1f017d5eb,0x0005c674e99b8b57,0x000f488a03753339,0x00095dbe6d94c0e8,0x000093a787b8c16f}, {0x00051a2dcc99ccc0,0x00039aa47c1dc3ac,0x000dfd8d5c134b41,0x0001edf28fcdafaf,0x0000d57bd8e10b83}}, + {{0x000fbccf0bcfc46a,0x000e15cffee69276,0x0005d915b3a822ac,0x00076b928ed2fec7,0x0000145c11463594}, {0x00081538be17bcdc,0x0002ea6c3d8ff61a,0x00016c82f01e867c,0x0009af9634e15d65,0x00001437bd32948b}}, + {{0x000d2006c19d4c7d,0x000711b1e976d2fc,0x000f237e8a0f3c43,0x000bb120545ff694,0x0000d10ec4090bf8}, {0x000696cac7cd3e1f,0x000b6f24bfe64f89,0x000af7706ed3714e,0x000c31463eb1d85f,0x0000cbd604eb027c}}, + {{0x000c6c7af8685c8b,0x000d7f8f01aa5f95,0x00074692ad4c1c8c,0x000068144bbe3225,0x0000800347984a4a}, {0x000c6e52eca3cdb7,0x000b7c04d3997c8f,0x0002bc5cfea1db16,0x0003d2405bc82e8f,0x000063d518064479}}, + {{0x0008eddc355363b4,0x000fb8820d6e16ce,0x00061a5084af2f70,0x0004728b7ed4d276,0x00001d3444f1d195}, {0x000a2b438da9649c,0x0005eeb4a20017fe,0x000b19c3d9bf6935,0x0000a5e13b5f916a,0x00000519c159c936}}, + {{0x0001ae92e42e1716,0x00038d41ccf9c257,0x000c8854fcb31ab6,0x0002a0d7f3c576b5,0x00006e5191c26239}, {0x0005a1c6cd5683b8,0x0002e06fe6897156,0x000e2fda6484b028,0x0000a473a25d6767,0x00007ba21df0a65c}}, + {{0x000e49ca70684d10,0x000133e80c3dde74,0x000a5c34d3ae8766,0x000c355984a2a916,0x00009a83eccb8298}, {0x000867caa4ca4c09,0x00002375b8ff9a19,0x0000396dc0208561,0x000e636296328bf7,0x0000c9ddc4d6e6fa}}, + {{0x000c1b808bd98d05,0x00041575f2404451,0x00075d270644b1cd,0x0006bd1907eb3373,0x00006c8bebe4a228}, {0x000d2acc4632b46f,0x0009bfd60242c713,0x0005c754617da427,0x0003dd413065b7c9,0x00008239899af17a}}, + {{0x0003d260b083b6e6,0x000aa6f6a54d9468,0x0002074dd0a3752e,0x000592e8bedc2375,0x000037622fc9e822}, {0x00005136be55d3b0,0x0004324d006dea00,0x000fc02709f5e12f,0x0009e6529486a964,0x00009ba0d0c92339}}, + {{0x00058473a9770805,0x0007bc6a6ab63638,0x00082261e4cf8e1b,0x000806419a5c6c04,0x000017a9ad135ce4}, {0x000d40c056aa7aa0,0x000ae56c61b02792,0x000b19e0c4c7c6ad,0x00067ff19cb178a4,0x000046d5c4fe4ba2}}, + {{0x00026ab121550b37,0x000a5147ce84d3e9,0x000ff722ae4975e4,0x0002a00a8be0f95e,0x00001e4702cdfd4f}, {0x0002acf3cb7b2807,0x000cb8272d7313b9,0x000a9fe5cc588716,0x000e42162c7bf3da,0x00008c008f352a79}}, + {{0x000963f4c8303203,0x0000603203e3f3b7,0x000327afb842c7aa,0x0009b67f22ca0ae7,0x00008e13092c6760}, {0x000fb62757558f1a,0x000157eca8c173b8,0x0003316273cc3e83,0x00023444174474f6,0x000077989cb63c40}}, +}, +{/* digit=37 [{1,2,3,..,}]*([2^185]*G) */ + {{0x00017f4b0166f7a3,0x00079eec74e6ae83,0x000874bfdfbd3e3f,0x0003a3cdb516ace0,0x0000d846019f681f}, {0x000ee5c7c1620b02,0x000d0b63c5010b12,0x00068c51eba68b4d,0x000b5b8c03cd3266,0x0000a6279f76e0bc}}, + {{0x000139f8f5fcda83,0x00048dee5bfdfd8e,0x0003f9f77f3e558c,0x000969a76cbaf4e3,0x0000a4c97a4a1771}, {0x000e84bf6dce6a73,0x0005e3e6c2d1da27,0x00059a6e9ff373d9,0x00062cc115193cd7,0x0000f9b702593d22}}, + {{0x0006fea87baa6279,0x0001672aa6801253,0x0001e5dc958c1fec,0x0001b8dc29b63760,0x0000e3c3c1d6e9e0}, {0x000127b2bcfe0b0f,0x00013a12f50defc8,0x00079b3973510710,0x000f207ccd6cb148,0x0000792f805e8a82}}, + {{0x00084911a335cc88,0x0009ea5913e48c31,0x000b32919563459b,0x0005ac9b920d61c7,0x000005ab8b720242}, {0x00012da8d006086c,0x0009fcf5c0fd2ac5,0x0002138d76ca4846,0x000442efea51d8ac,0x0000b647545f44cd}}, + {{0x000f521e447f2c48,0x0009e04291f0a3f4,0x0005926de81b8da7,0x00002f5680bc467d,0x00004f21fd5b4a12}, {0x00031814e9df3d85,0x0009e9ab8d341d1e,0x00019aa4a1ca4861,0x000366309ddeec5b,0x00009f72f7e9d329}}, + {{0x0002466cdedca850,0x00001a09538c9f1b,0x00011115d140bb71,0x00059eac8ae8515e,0x0000d63ff676f03f}, {0x00055517d234afbb,0x000dce208fc1755e,0x0008a4b5d61c2db4,0x00030efa9859cef2,0x0000dd6d4fce4af0}}, + {{0x000e67f906151e50,0x000f55e106493f39,0x0007cf7b7cea27f5,0x00062ddca1d4e1c1,0x0000c326d122fe23}, {0x000ac337dd35df39,0x00093396dbdf05f7,0x000b7db1c0c3b763,0x0004a87912f5ac03,0x0000dea4b70ec9ed}}, + {{0x00053e453544774f,0x000b4adba2bc5110,0x000e371f5834d0ec,0x0003bb5215d7f7ba,0x0000cfd57c05c866}, {0x000383dd6901b1d3,0x000485587dc3ded2,0x000625f623b49fbb,0x000762cd44a08d07,0x0000ee4d65bcde9b}}, + {{0x000fe5bdf53aad08,0x000564604a673e56,0x000a261a051314de,0x000c90b86ad98607,0x0000b7e021297afc}, {0x00011273b72aec51,0x0005b4f9f509cbf4,0x000484b5713c85d0,0x00095bdda56845b6,0x00003cb1642c3d09}}, + {{0x0004b6e717815de8,0x000027e131d15a99,0x000c023aad995c7a,0x000ced48b46df226,0x0000cfd094df02af}, {0x00043cdced6a8867,0x0003afcabe757bc7,0x000ac9390b7d70ec,0x000a8fba6c9e47dd,0x000020694260310a}}, + {{0x000b3263e8f793c6,0x000e6de3b289a607,0x000915b21a541166,0x000da6944ff2924a,0x00008bea907158ec}, {0x00037b4402928970,0x000e4b768423d85b,0x00013bbbed2a508a,0x0001f4d10bb79da4,0x0000262f48106149}}, + {{0x00080f55464d0ebe,0x000687a613a7d17e,0x000c97dad89d3e1a,0x000be697791260c8,0x0000c2ff220c4f0c}, {0x0004ac2e9e6bc105,0x000827cba305ed98,0x000624fdce3c53de,0x0000ec4bbaf9b283,0x0000e9451cd6485c}}, + {{0x0009f63b7abad118,0x0001e3a46189fdfc,0x0007037b7c7ec3a9,0x00070fc9ebee42f6,0x0000247f5054dac1}, {0x0008f8d397583e59,0x00067c3f8c2efc51,0x0005edcb1e7d24de,0x000efba54832669c,0x00004bba48488c2c}}, + {{0x000756db1761ec84,0x000864b97e551f13,0x00096cc28e53c8b9,0x000a8d72aee3f840,0x00008c361a0d20f1}, {0x000672d8c31190a9,0x0000701855d4a98b,0x0003f4b2a7bc1e7d,0x000bc3942cfb07bf,0x0000bf44a3fc2a28}}, + {{0x000976299da550cc,0x000956baa042eba8,0x0008c2f9db2c1781,0x000a9d7068b08278,0x00000fa414654869}, {0x0009e39d50b693f4,0x000c588e2c5e73bd,0x0008f9f62b79e2a9,0x000c7bb1cb8de40f,0x000015b04ee5ea50}}, + {{0x000137d0d63d1fae,0x000122a9d89f64e5,0x000436309658fc05,0x000a606889487450,0x00009ae30f9b598d}, {0x00010d1818baf918,0x00060b6a0c202ed7,0x0001a6b44e27e9e0,0x0007db9e28dcfb1c,0x000083acb6556ac5}}, +}, +{/* digit=38 [{1,2,3,..,}]*([2^190]*G) */ + {{0x000728dc2c6ff707,0x000f55dc22358735,0x000e277f979d6122,0x000cc6b3f5d00319,0x0000ee84e264ded8}, {0x000afb063cd880a6,0x0005d574af6091a8,0x000de7f423f3ea7c,0x000151acfcdc8402,0x00002d07930131aa}}, + {{0x00083f9dec31a21e,0x00028ad9d573b02c,0x0007be365988c8b2,0x00034d73e983aea5,0x0000968734e446f8}, {0x000ea8f5da6309bd,0x0003f1f1ce169137,0x00044092110f3a62,0x0001b4a82a9ea2ca,0x0000f94739f2b46f}}, + {{0x00065a434a35ea84,0x00070d4412f61df1,0x0008836c33418e0f,0x0009651af1f8af51,0x00002ceef4d530e1}, {0x000ca0b543a1957f,0x0004986cb1235560,0x00098ed30c33761e,0x00097c76624b1ffe,0x0000772f4c000909}}, + {{0x000410ef4f8b16a8,0x000e447b266a56f8,0x0009c87c197241af,0x000b1a8a406b8e6d,0x000003f3e034d42a}, {0x00009a804dbec69c,0x00067bbad05f7f03,0x0008e197fa83b85f,0x000dc106097273ad,0x0000097440f1067a}}, + {{0x000b6dd418e7ddd1,0x0003372f19d6c507,0x00027eb4d39888d9,0x0008846eae26be0c,0x00007b53ed40babb}, {0x000021b2b01ae4fe,0x0006ef488682fc27,0x0005e2d8788462e8,0x00029adee096ec21,0x0000b2fea9bb242e}}, + {{0x00000c756b95bce4,0x0006216da680bbcc,0x0002142525ec0390,0x0002d239162ee672,0x00003132b63c6a89}, {0x0003ff22f3263bf6,0x000c3cd0a1424bdd,0x000415ccbd5b3733,0x0004e9f92eaa8244,0x000063e8924ed547}}, + {{0x000ab35ce7c42d4d,0x000eb2feab100d38,0x000111b459fd493e,0x0005c276056b6d82,0x0000a11dae243efc}, {0x00002785545a7fb2,0x000c20d507e6dc74,0x00066fa58bdb2601,0x000c29f21dfeeb70,0x000014369a859ae8}}, + {{0x0007fa0b311898c7,0x00045d0eac653f74,0x00014d0bce2a272e,0x000ee2dbba5851f9,0x0000a1a966134a43}, {0x00067cea1c8cde9a,0x0008d271abe3e5a3,0x0001615cd9d958ba,0x0000b053ff7eb63d,0x00002280dcf95ae2}}, + {{0x000d566d4312483b,0x000cb43e216f8c0e,0x0000444935179a95,0x000a211c185fec17,0x0000306333a04991}, {0x000ecdb0081a726e,0x00056fa89bbbd801,0x00091b6b90149b0c,0x0003a2cfe9065a43,0x0000dc92787b633f}}, + {{0x000d9e37a6a308bd,0x000f7c2767d3d6d9,0x0008cbeb66237582,0x000d9db74a8bc6f3,0x000094d3f1b9cb6f}, {0x000735bba21f248e,0x00048cd1efb092a9,0x000b03284272ad0e,0x0002249437b69c05,0x00007f047034948c}}, + {{0x00068a9e9adfe1c3,0x000d014e39bb9ae8,0x000fe378f3984403,0x00062885875720f2,0x00003f901e0ea44a}, {0x00025fe3652438c6,0x00063dd1f20bea11,0x000bf7fbdae9ec4e,0x00079bbe740d9ebe,0x0000dbd3ddca2dbe}}, + {{0x00044a43794f8dcb,0x0009983c5c362663,0x0009d10a0dcca923,0x000df27d6b6bbf3f,0x0000320c5cb31d9b}, {0x00028ff47b50a951,0x0001bef03371620e,0x000100153933e3b0,0x0008d6e081bf8599,0x000083be9a0d3a8c}}, + {{0x00043d77613aa81f,0x000f95fe65848a94,0x000b10288801007f,0x000ee780fc4dbc7f,0x000058280d4d86be}, {0x000d82f7c978c385,0x0000bde44d7b14fd,0x00060252fdf1204c,0x0006a5508a1c8441,0x000091554cb11764}}, + {{0x00081ea77b46a081,0x0007b74806997ddc,0x00033f683cf5a647,0x000c6033a8cb3466,0x0000b867e6ba2363}, {0x00011141f60558ee,0x00024f41450e4392,0x000630e8bcdbcdd6,0x000b429fc04601cc,0x0000a7c66d677038}}, + {{0x0008820fbeee3f97,0x000fd9091afd34fc,0x00031f35c93e5348,0x000924064b9be59a,0x00001f37864c7e3d}, {0x00034e0943aa75e5,0x00085b8ff6e402fb,0x000cf0d19a18c9c5,0x0008a6b80f31b133,0x0000c9682db58351}}, + {{0x00085c341dca5663,0x000aa8622aa3b6c1,0x0001b6dfb7de7fed,0x00028869e84d9290,0x00000a02b0eac4ad}, {0x0001daa2fd3cf36d,0x00070f89e59fc7c8,0x000496733d131954,0x00012ae2be8184cd,0x00005f449ec63d34}}, +}, +{/* digit=39 [{1,2,3,..,}]*([2^195]*G) */ + {{0x000ec644cd8f64c3,0x000ff79d7b51c492,0x000c7525658a2d78,0x000016dced1fc51f,0x0000e658aedbf433}, {0x000942e05da59eb6,0x0002addc37220b61,0x0002e7f87ba3d60a,0x000b6e1c311cd174,0x0000473ffef56b01}}, + {{0x000604f692ac542f,0x0000327b91d38303,0x000aaf9bdf079ffe,0x0004fa29f63e6315,0x000099ee566e1f34}, {0x000661fd62191997,0x0006648ce41c8a1d,0x00074d9048c883bc,0x000b1aa065118f3c,0x000013889ee7faf8}}, + {{0x000f697960eb8c77,0x000c72de04d3c035,0x000ad9228f1599f2,0x0001ab192450f8d2,0x0000d48129c2829c}, {0x00085e13a50afc95,0x000713a96ee024d7,0x000fb6d7b2745ba2,0x000a42456534013b,0x000036202676bad2}}, + {{0x0003f8f81a1b3bed,0x0004fe2764a0972b,0x000c4f5f74f3ce14,0x00085b12d0f1cc28,0x0000eee0c0e97f39}, {0x000adc0d39e25c35,0x00007467a0807df4,0x000cf5a584061982,0x0005fff40ebc9361,0x000027729a6922ad}}, + {{0x000398a4a9eb3f0a,0x000e9b99a48fa691,0x0004b5b3256c1dbf,0x0005fdfa87e1b91b,0x0000d639614f378b}, {0x000243ec26b53020,0x000c3ccb4c10437a,0x000e070150275878,0x0003c00e81e4a21d,0x0000c6265c9850df}}, + {{0x0000937b1b76ba6f,0x00045d2026dcca6c,0x000d9ae0a1a2eab8,0x000025c1715e1519,0x00001ad919aaac4a}, {0x000dfb807ea7b0ef,0x000e4ed9eb8935b3,0x0006d08abedf5496,0x0007309932e5ff2d,0x0000314874f15bd2}}, + {{0x0007149a8c25ff63,0x0006482e6569c832,0x00068fc3829bf255,0x000ad56012f5c6cd,0x0000e67e8bd6b982}, {0x00075386ecdca88a,0x000621753a045e3a,0x000db3256f297eaa,0x00005470121e5405,0x0000b9697d590851}}, + {{0x0006a753f73f449b,0x0007dd44fc79efb2,0x000c0dc4d1d1c94f,0x0000cf99f0fbc53b,0x0000747ea0be698a}, {0x000c3fe228d291e6,0x0004e3c129d65218,0x000acc51635b804b,0x0006689ac859b8d1,0x0000c10697df5d6e}}, + {{0x0007171f6bdf1bf4,0x0006bacb0d8fe5d2,0x00096a31b0b77b87,0x0001039a95471d84,0x00006a50dbb7f16b}, {0x0003f977b865bff3,0x00063b1c198c2a4f,0x000702ea6848195e,0x000228191ad08821,0x0000f20b43779035}}, + {{0x000438f0876fd4e6,0x000723d2f383c38e,0x0000934cb45f0c30,0x0006edc03cc2ecb1,0x0000a8f24398c9d4}, {0x000431b65ccde7b6,0x0007c7e76a6ff16b,0x0003484d741e2cd1,0x00044a59c8cf8f4e,0x00004426efde3152}}, + {{0x00098093a69fc01b,0x000b4aa9dfc2e6ec,0x0006f2a557e20fec,0x000fdeacfdbb07f5,0x00001cd6868bbbdb}, {0x0004995986eb9ed8,0x0004bdd0955e247b,0x000c7a20174785bf,0x00080d08f74f61c0,0x0000861a15bdd01a}}, + {{0x0008e44fc94dea3f,0x000eead6a0b01c0a,0x000113cef34c8cdb,0x000ff9a19c384004,0x0000d32fba505490}, {0x00090f6795dcfb75,0x000333588baf58d1,0x0001fc1c0fef01b0,0x000ac94e6d1d63ca,0x00003173f9740a41}}, + {{0x0007e4182997cc14,0x000ca3720c9c5463,0x000de5d4508c5a96,0x00075a38bce01c11,0x00009d623e54dfdd}, {0x000a4680fb2a3acc,0x000e719c25af8c72,0x000a92421cc53bbf,0x000cf2598eba7978,0x0000d61f28c63bde}}, + {{0x000402aba16f73bc,0x0003ccf9b9fc2b1d,0x0006ef7bf2fb3101,0x0007446d51e60e44,0x0000731021c791e1}, {0x00047244fee99d47,0x00068ac5c1ea9d3b,0x000ea9af74bca48b,0x00083a00f5f514bb,0x000051f55a6074c2}}, + {{0x0009fe8662595c2b,0x000661a807732389,0x000d1d43b495d672,0x00065ed6c971d2b0,0x0000518637d43b7a}, {0x00053bad98c99cec,0x000954d39f5b30e4,0x0001ce415ba6e0d4,0x00082337db02a643,0x0000d909c7db6e1d}}, + {{0x000251acb452fdbb,0x0004a0f306506e30,0x0003548d931ee696,0x000f5b00b3e50893,0x00008949a50a4b0e}, {0x00083263c88f3bd1,0x0000cb1d9989208b,0x000d4df03ab147c3,0x0000c5dd6515fd44,0x00007a12f75f72eb}}, +}, +{/* digit=40 [{1,2,3,..,}]*([2^200]*G) */ + {{0x0004f7881fdad909,0x00057d2cf6ab2591,0x000054de5cf638f5,0x000350290bc03fcc,0x000032811a7a8b06}, {0x000b3309bbd11ff0,0x000fb40449742f00,0x00051d26676108a6,0x0000c1801bb9e0a8,0x0000dd099bebf899}}, + {{0x000dd8a58d6cd461,0x00057e6634d214c6,0x0001bc3289cb633b,0x0007e5b1305047f8,0x00002ede0e236a17}, {0x000ca62065a6f4f9,0x000cd7be487b332c,0x00047ed1cc3a47ec,0x000b13e41eb1870f,0x00009e66e5977598}}, + {{0x000a6777b0ac93d1,0x000d68f5e0d7ebd6,0x000f5492ba6e37b0,0x000f3a1516c09676,0x0000e4bf888aac05}, {0x0002ce04df0ba2b4,0x000d1062341bcdb4,0x000acac20935d5cf,0x00000e4a30333382,0x000029438c49198b}}, + {{0x00038be67e573e06,0x0008e084c44bfb28,0x000c1c2c505891db,0x00044b3131137396,0x0000aebfa4039584}, {0x000dce9e56e55c19,0x00029caa46d0ac9c,0x0001fe8eb7148ced,0x000f4c9e10c7efb6,0x0000fd835db8f97c}}, + {{0x0006f56c17706169,0x000d79da9a2d6c62,0x000730e455351909,0x000a79558e6825a3,0x0000d8c8bc093ef0}, {0x00078b6056becfd1,0x00039090b36d543f,0x0004432f933f1325,0x00050272ad499779,0x0000386493c5721f}}, + {{0x000eefa5abea82a6,0x000933fe62d43794,0x0001ef6068dc611b,0x000e6909f1af3728,0x0000af546c899839}, {0x00078c7c977ec238,0x000c3d5c05766255,0x000d1a4c0a8de294,0x0004d462ddaf0f7c,0x0000243fc70cf95f}}, + {{0x000f400b008733a5,0x000d012e1f57e566,0x000509cd0cba0697,0x000d8c4537c2b240,0x0000f989c69a7353}, {0x000c9724c3c2b2fb,0x00084f031fa87dbe,0x000d5d11f90e02fa,0x000dfc44d15c53cf,0x00003404faef8314}}, + {{0x000a109081e9387c,0x0006cc935828a36d,0x00040b015fb9780d,0x0006fa15940332e5,0x00009d7b51ca0f46}, {0x000cd41d6d9f6711,0x0008a1a2ac17faad,0x000201e5fba6c1e2,0x00062af66a7833ed,0x00009d9971a090f4}}, + {{0x000c3a9f327a07f6,0x000ae490937df02c,0x000b3afa5efb27a9,0x000be421451e96b1,0x00007e24de8f1883}, {0x0005d4770869e549,0x0003d4a3856a1ad6,0x00032e880d36291a,0x000fde770a1abf71,0x0000511d0a39e28d}}, + {{0x000d650f8d1cac45,0x000b1d16bda5fdee,0x000abbe46eb99194,0x00076c653b19f71c,0x0000f45af5089b92}, {0x0004c6126ee9d77a,0x000a6c02ca5dd078,0x00032e720f7a1558,0x0003f7161d6c59f0,0x0000e3ffb95e70cf}}, + {{0x0005facc72a4be52,0x00062d8480899b18,0x0007afea9f66de23,0x000c1c9a14d07c71,0x00005bfbfc04d551}, {0x0000ecd4cdf3d880,0x0003647f73c42cef,0x0002d67f78cee2aa,0x000a21c10a7d3d72,0x000090037a294564}}, + {{0x0007987fa9ced27b,0x00037a9e5dbce656,0x0000d8e8c36b2d88,0x0000d044bdeec638,0x00005e6a6b18bc30}, {0x000e9b9592bef367,0x000f7e81b7497ba7,0x000ce596b2b373c4,0x0004cbd84b5e01f0,0x0000cc51c6330bf5}}, + {{0x0007bb84f3815c4e,0x00033aa9017e6ac0,0x00085720addb9f62,0x00083751e30228ca,0x000059d63f65cb75}, {0x0008e777baad2d02,0x0004b42f5d7369e1,0x0007749832cfdb78,0x000d51e25dd53df5,0x0000f80e7cf0042c}}, + {{0x0006bafec695bb00,0x0008f13c78ad4bf5,0x00082abb022da1ca,0x00049e0f9c4b1311,0x00002ea555aae7d2}, {0x00083f25e05d9e30,0x000f70382afa8685,0x000080408e09cdcd,0x000658df072ec787,0x0000a317847cbf75}}, + {{0x0008d7f4d6ee4ab7,0x000e2570c3dc43f1,0x0008c9b2ad3ac8cd,0x0000f0e27e49070b,0x000016709a835a4c}, {0x00052b0916a26b1e,0x000e6e0711779308,0x0006948683cc17fc,0x00055c54f5e3d459,0x0000e0341ac828f6}}, + {{0x000f462060b5f619,0x0003ebd057c2f431,0x000e1bf65a56f46b,0x0001fea48dca6c47,0x0000a38783ed1bcf}, {0x00033a9da710718f,0x00063e0aeaf67a5d,0x00029d1875a77998,0x000732da87314d2d,0x0000a0edc3fb687d}}, +}, +{/* digit=41 [{1,2,3,..,}]*([2^205]*G) */ + {{0x0004849cb198ac73,0x000cdf2646651c89,0x000200678a884a93,0x0004e5fda964ef9b,0x0000c351b8730983}, {0x000ef9fe2c4b44b8,0x0006b326790cafb2,0x00002264a580f6c4,0x0004e2384805210b,0x0000ba6f9e2c2a19}}, + {{0x00034abd3c095f18,0x000e64b76d7139d9,0x0003e698404b261b,0x000b109d2e6970e7,0x000079fb23bde5fc}, {0x0006c72dfd754907,0x0004f1bcf1c11150,0x0005e70073a97d08,0x0002a6d3201d82bf,0x0000f0ac52fe9823}}, + {{0x00001fb319d76820,0x00090a982feeb251,0x00061b344b029312,0x0001fa51c1c9b902,0x0000e008c5bbfd37}, {0x000dd1c0278ca331,0x0006d5aa53b1d866,0x00013a2cf666f76a,0x000836d5cfb77960,0x0000d3a1aadb3521}}, + {{0x000131a567193ec5,0x000a95f6e70b76b4,0x0001eebddaf3c305,0x0008314587bd3903,0x0000709def8c1bbe}, {0x00099830eb2b6692,0x000b675b70295705,0x00064ac164d80ce1,0x0003ab638a7da803,0x0000f431d23de1c8}}, + {{0x000276638235d4e4,0x0007096e3298e781,0x000175bc81c62bd6,0x000d4d4378660c3f,0x0000d04e18957afd}, {0x000160185a8068c5,0x000142b29a8532a8,0x0000d8a3bdb58e4e,0x00003b98a65b86c7,0x0000f0e6f4ee8a04}}, + {{0x00092b73f894ae03,0x0005875f18ce2c24,0x00053cad0f59df3e,0x0002944cb740d28f,0x0000eb585fbf4f01}, {0x0000c8632c7f717e,0x000acf943f4c17da,0x0007c51d2eb8c795,0x0009486ee23fb5f6,0x0000f18757648889}}, + {{0x00011a4e822f0d08,0x0000da8704f83ea0,0x000c6820fbc647ad,0x000bec3b315b3550,0x000063dec3e37e76}, {0x0005d3af017bfc7f,0x0008a76b822901ff,0x000bd0d3b2005443,0x000d0e167fca370b,0x000063dde656f5e3}}, + {{0x000bc15adf7cccff,0x0005efa1e1b075d9,0x0009bc17e81a3e5d,0x000d429c39e44424,0x000037dccb37ea7f}, {0x0004873907fba12d,0x00097a372904da65,0x00083a6c535daa6d,0x0005be3564cfc662,0x000009fa4f71a939}}, + {{0x000b3b56aa39dff0,0x00029f8e4d8cb510,0x0004c4b9f59b43da,0x000c01a8ce31fd9e,0x0000e20be26c1303}, {0x0007182e8ee47c94,0x000b3db981011818,0x000e14ff6d9687cd,0x0005723a520e4da1,0x000029808bac836d}}, + {{0x00046996d16768e9,0x0009d28bf217174d,0x0004e490d9fc4ff6,0x000979e7705a9415,0x0000d96dd291d2d9}, {0x000d9d8ce5d72c41,0x0004b11c714f77e2,0x000e4a03e9d06c5a,0x00028af2aa513679,0x0000386b3c2130ff}}, + {{0x0002d69ea40dc3a4,0x00026ecc018f26a4,0x00070f04adc84ad2,0x0002ece5c36c7b32,0x00006ba6d4790fa7}, {0x000d1c593e58a8e2,0x0000f20c088c6c37,0x0006e86daa239473,0x000038a3be4263cb,0x0000c417d369126d}}, + {{0x00023bb440e22296,0x000213ef4d04cca5,0x00011ec39324673a,0x0008d34f3adf343e,0x0000136d7f23c596}, {0x0002899b053a9270,0x0001ae067ecd7a7b,0x000779cd93eaa266,0x0005ea8549b9c802,0x000061d7940c5338}}, + {{0x00041fcaaa2902bc,0x000924f69ad3e071,0x0003f9ffd539ad79,0x0002f6e6453f9481,0x000058d3c48f75bc}, {0x0006fad5dc64e964,0x00097240e354b332,0x000a1e7a93aafcaa,0x00089fdd1b0903ac,0x0000ceb97675211b}}, + {{0x000de4687032d585,0x00030e2c79e01eb4,0x00004ef23c54f3d8,0x0001b3b7818df45d,0x00005faa9c8b73d4}, {0x0004f6f89b95355d,0x0009e7415c84ced6,0x0000ebad34860d2e,0x0005be8fdb9bd205,0x0000b53e0cd3685a}}, + {{0x000ca5fabfae1ca1,0x000c0a21459b919f,0x00066a4d2937afaa,0x0003318e0ca91c1f,0x000094cc7f333ec1}, {0x000143a8aa116906,0x000ca9b59e08ad25,0x00050860abe40ad8,0x00034bd7d60d9be7,0x0000c53b00926bf4}}, + {{0x0007080eb6b242d6,0x0002db71e246832d,0x00031139dd30bd02,0x000e531027991bbe,0x00008797e91a62e4}, {0x000e20a6b4e185ab,0x000d92d9b707423f,0x000f7811b82f2c67,0x00095c75c817684c,0x0000d53005eb45bb}}, +}, +{/* digit=42 [{1,2,3,..,}]*([2^210]*G) */ + {{0x00049be9d8e68fd9,0x00028b044320e5f6,0x000c33398db0f053,0x000fae66fde9b3e0,0x00002f4209bf6c8c}, {0x000afcc1a739d4b6,0x000f428ab8dee9d1,0x000c6f1d009aea75,0x000aa4b4375fb5ea,0x0000420b560d08f7}}, + {{0x000499c6254dc41f,0x00038a837e7e9eae,0x0000524a77e29392,0x0005f184aec08c09,0x000082b921a7d6f5}, {0x000962e1402cec5e,0x00071a2f30e7493c,0x000b879cb9f17ca1,0x00045edcd783e8e9,0x0000a3d8c153a6f1}}, + {{0x00015e75e0dee6e5,0x00028c628aa2dede,0x00061bb9374f2487,0x0002e083e9c4fe78,0x00006d4822ab187b}, {0x00017cfc59826f90,0x00092408169eb664,0x0009ef885ca26096,0x00038fedf69d06c7,0x00000031f8adc7d1}}, + {{0x00046e60ebcf7262,0x00014231470e103c,0x0007c21094482b83,0x0006ef4f6dfaca48,0x0000e0ace9782e66}, {0x000a9d31f8d1f420,0x0001574944d23246,0x0007f334b1b1e83f,0x000d8113dfa63aa5,0x0000cf8daed9f025}}, + {{0x0008ea800ee11c1c,0x0003f5e3dd7530d7,0x0008c43c5eb053cd,0x000662db65b13ed5,0x00003ad49be7d151}, {0x0008e41b64279905,0x000d207eae1e99fd,0x000abb71e12cf15b,0x000d0dd9ad4f1b1a,0x0000143e74d57545}}, + {{0x0006336c88bdee1c,0x00059876767c3026,0x00073199625f2930,0x000950dc078571c6,0x000088690b3ad552}, {0x0002c2d852705b44,0x00040e09552d274f,0x0006575d1b0bf8d4,0x0006513628beeb98,0x000007be238bf864}}, + {{0x0003049a639fc6b8,0x0009060036250e5e,0x000cc1646e75c35d,0x0007398cf35bd85d,0x0000bcaced2ec262}, {0x000cf1db55367425,0x0006ca9e068be22e,0x0007909c5013dd89,0x000505c7f411cb8a,0x0000757ac98d61dd}}, + {{0x0001f0d1e935abb5,0x0003c54de37a85de,0x0009cebb5defd10b,0x0004be68d9e39236,0x00004d5ef9bc6132}, {0x00041ba74f17e266,0x000818c1dde44d63,0x000d918fdc0a0e3c,0x000a1346d7758187,0x0000687601562ca3}}, + {{0x0003e9cf36658f0c,0x000bb1f8057ec731,0x0006a835ac433ef1,0x00094bc53262461b,0x00008f053993c863}, {0x0008cdfe983c4a11,0x0001f3b7b931ff39,0x000b9045bbf5e816,0x00046b83193c46b7,0x0000e4ebf5db4a6e}}, + {{0x0002a6043a24fe7c,0x000e3fb3492bf994,0x0002fde0529c1191,0x00032cdf66244990,0x0000792a7ad2713c}, {0x0008ad8b737982c6,0x0009421e60e32fd8,0x00083591a7e3a031,0x000455a9b0de4473,0x0000df141eee310a}}, + {{0x000a039e6d6f4714,0x000ed198d12eaec1,0x000eee5ac14b2ba0,0x0004ceabc1a1603a,0x000001f483720b96}, {0x00037964fd03f663,0x0009ad8f3f122ee4,0x000380f183fdb4e4,0x000d163ef267f629,0x0000e8e9670bda64}}, + {{0x000180c207674f17,0x0006c3ae8fdbbc19,0x000aeb71e112e09a,0x0001c7296675546a,0x00009432af25101b}, {0x000558fde2ddec64,0x000f1357753fd5eb,0x000e1158a81392d1,0x00099167a76b973a,0x000016fbbff8a899}}, + {{0x000fdfd0d4a9dcf2,0x0008744ddf129e65,0x0008568667bc29e4,0x000fe29c1a92d93c,0x000073c69058e98d}, {0x000e418cdfaa6b88,0x0002d061c69f69fc,0x000f75e27606bd82,0x000a1ec2d495a06a,0x0000ed3d505ed873}}, + {{0x00028416ab25b6a0,0x00072b1a4523af55,0x000c99e03c6c0ffc,0x00091bab18827b21,0x000060e864890346}, {0x000f90f93c7f3988,0x000b02f8d10b5207,0x000d0f9e39f4a96c,0x0004f55d71cd793a,0x00004f435d37c3a5}}, + {{0x000c55b8e33787f1,0x0005963846734b03,0x00051b9f0ef42f97,0x0007c2ef7304f750,0x00008aca1dc841c8}, {0x00020a72d4bfe80d,0x000c353e732c56f1,0x00037ca16fd823b3,0x00096a41bccfe475,0x0000f6c9c74eb5a9}}, + {{0x00032c7904fc3fa4,0x000587e3636aee73,0x00091d9aa14a23f4,0x000540838659c3f0,0x0000a995e5df12d8}, {0x0003becf3a5598a7,0x000741eaa99520a5,0x00004e03c56534b1,0x0002682ed3dca4bf,0x000016c563b48d56}}, +}, +{/* digit=43 [{1,2,3,..,}]*([2^215]*G) */ + {{0x000dea7e0f222c2d,0x000ba2e651425043,0x00016cd30309d42a,0x000eebc4fe9ddd92,0x00006539c7ddf87f}, {0x000a57c432ac7d72,0x0000127fda1003c5,0x0000698de72692cf,0x0003b1cc28c85f28,0x0000331fb469ec28}}, + {{0x0008eeae457a4773,0x000bbe6ddc05a015,0x000c41671d19857d,0x000d588326522418,0x0000ffdfc7e6c2c0}, {0x000525426ee7cda3,0x0009af02c3a83a3a,0x0003bbfc8341b086,0x0006917023bf4272,0x0000d15002a44452}}, + {{0x000b2f687500b96e,0x00064dcd9425c961,0x000615433795510e,0x0000282308172978,0x00005d0145545445}, {0x0003302bf690cbe3,0x0001431eca675bd1,0x0008038a444e4883,0x000f0243306bc72b,0x000051d151eef57b}}, + {{0x000324c85edfa308,0x000407d4f3da5ef7,0x000b50c862597655,0x00096bb52f5bc0dc,0x0000f6927b0c832a}, {0x000e1ba55f2f94c5,0x0008e44b45fad08e,0x000aa455d6a996f9,0x0001f79133cb8da8,0x0000d0721ecc58dc}}, + {{0x0004e1f9a8764417,0x0003a47a818df3d4,0x00003edde82bc0c1,0x00025fc3ddc11706,0x00007163f2e7a25f}, {0x00002caa3bfaa53f,0x0001f2256982b54c,0x0006f37dcbd1b250,0x0004a4d4e1728c1a,0x0000e36c214714c9}}, + {{0x000e7e9262a35393,0x000d3670d59ef3ca,0x000c5e1b978a49d1,0x000c1c07de0f63c1,0x0000072c30c99cb7}, {0x0008a5277c850e61,0x000f0f6a3de61d27,0x0002ca7ad84f15f8,0x0004b836a8bb4559,0x0000912e3eef4d42}}, + {{0x000e2d490e317342,0x00065fca007cc1ff,0x0003f77e891b1f12,0x000c883459b1d0ae,0x000062b051d66425}, {0x0008f765c51e2741,0x0002204e6146cf5c,0x0004046b5997481e,0x0006bd5fc1198bd8,0x0000cb0a6bbf7f7a}}, + {{0x000a92079e5fb67e,0x000a90aa725e6ba7,0x000f5d837e1331fe,0x000e207080ccf57d,0x00004cae01e5ff72}, {0x0003ee60412a77db,0x000c2f449025d924,0x000ef5a3106ff7ca,0x0007a80e75f7cd23,0x0000c957822bddef}}, + {{0x0007ad11b7f30b08,0x0005b629dcf9c473,0x0006ae160525ab2c,0x0006e0163f4cc118,0x00005076713f3e6b}, {0x0003180f6998bcf4,0x000371c8d8d8d9be,0x0008c005493d91da,0x000768b902ce661b,0x00007e7924db4a8a}}, + {{0x0008086365e668b9,0x000a1abda5fbdc98,0x0005f1fbeada8dcd,0x000fc32c146b4c25,0x0000cfcde2a5f34c}, {0x000453e7e85d1e48,0x0009792358b5acbb,0x0000823ff9ca0967,0x00011d95fc2d9624,0x0000d65adf78c11d}}, + {{0x000fd177e19a46f4,0x000746161156323c,0x0007d33630948a7a,0x00079d10d06b977e,0x00001c47ec1e7025}, {0x000998e59e9260b4,0x000406c242609455,0x000bc3744c865e44,0x000149f93021ec13,0x0000981994f0d92a}}, + {{0x000230cb0ce1c552,0x0004ebbfb6078cf7,0x00016363b5b534d0,0x000e82ce1ef1130e,0x00007e0aa7ad4999}, {0x000ac2d79362c410,0x000091bb6cb0ce1d,0x00023df2467920c9,0x000f281e648d6322,0x0000f7d9eefe32e8}}, + {{0x000f264d66f51c09,0x000acd9ccea59766,0x00021e232644317a,0x000737139b37bcd7,0x00008bdde0c49daf}, {0x000c758165166bd1,0x000592802108c2ec,0x0007aa66f0951a28,0x000a27d39fbf2499,0x0000a2f6862eb62d}}, + {{0x00057f10296f4fd3,0x0003ba51b4367755,0x0009508051dca76a,0x0007f1c3e98f60fb,0x00001ff32eab31cf}, {0x0007bf18d2c714bf,0x00083e9d2aca643e,0x000dc2d2364b5c33,0x000b9ab9fd9ccc6a,0x0000c2397edbc721}}, + {{0x0001fc764465367b,0x00047b098f57ab65,0x00082376fe438701,0x0009285a91d519d3,0x00005afdb0b03cad}, {0x0001875138d55231,0x000e0aecacfb457e,0x000484f49a92beca,0x000c2af762f6e811,0x000014b5c86eda16}}, + {{0x000f39afa8338349,0x0002163285626943,0x00070fc102295172,0x000e6cf1d63dd541,0x0000f5fa5903ecc2}, {0x0008725e77d9a3b0,0x000a6384ebe0b66c,0x00045e24a11235ce,0x0003b106a8c11858,0x0000137b286ebd09}}, +}, +{/* digit=44 [{1,2,3,..,}]*([2^220]*G) */ + {{0x0007d6ac42bd6d29,0x00082b1f96aedb56,0x00043b28e6df8646,0x00023f7efe5b1a48,0x000061bbb05f379b}, {0x000f5f070a6a26b8,0x000cb28e6e39b6ca,0x0005fc8d370686c0,0x000dc900da06cf89,0x000004d88113363f}}, + {{0x0009ce74462007dd,0x00047cb5f5b763b9,0x0005edde7b8ab48a,0x000fd9cec673d2f5,0x00001567f755cfae}, {0x0001b6b0887bcec5,0x000e9178f3c24638,0x0006266cb694497c,0x0004130e6525e31e,0x0000931de26b97d6}}, + {{0x0009da11f17a34c7,0x0008b35a145614e4,0x00050363b5420ab3,0x000b6e476372412f,0x0000b15d62433fab}, {0x00040b1e274e49c4,0x000456b1860aa0ef,0x000afe5a45cf5074,0x000e9a96583fbf66,0x00004240511347e3}}, + {{0x00055021a93507a4,0x00074d3c06cf142b,0x000ec3f40b4cd118,0x0003c29f70e76a91,0x000084e81ad8e755}, {0x00087b5272e9d6ec,0x000506ff514a830f,0x000192a8eea1c93e,0x000359a7cc2adcc4,0x000077e27e302f45}}, + {{0x0008b1e48ac28403,0x0004bcba9477b535,0x000946b431831129,0x000819aa58f990a6,0x0000098baf9cab41}, {0x000c1584198da522,0x000bf46bfd1b66c4,0x00036a908ab4fc17,0x0008380f0a4c3cbf,0x0000ae9e34b78cf7}}, + {{0x00040ec0ccced589,0x000b7da44f9845eb,0x0001812c625cd4b9,0x000650b3e0645887,0x00009f80d55a6cef}, {0x00040c9ce6dc1532,0x0007b86655215713,0x00007014d138d511,0x000b918cdb45bc4e,0x0000f34bb38a4b60}}, + {{0x00099e84a34f239b,0x000c090402d54174,0x0003aa83215fdb83,0x000db1075f46bf43,0x000061e15b013215}, {0x00059d4a127f89a5,0x000bb7e816daaabe,0x00018b6925d541e0,0x0000265aba0659a6,0x0000532773367266}}, + {{0x0009cf2d0c051995,0x0002aae784548cda,0x00072a182502fbc2,0x00037270bda9dff5,0x0000f9b71b8b158b}, {0x0003a592b82dd077,0x00052523032ee0f3,0x000505a327630273,0x00009f0fe1a721c4,0x0000b6e3e8367964}}, + {{0x000155d3f6effc78,0x000f1c90f0c7023c,0x000ec2c5d1fbd69f,0x00027365d7da8abe,0x0000813872c57e86}, {0x000c2c655f5e228d,0x0008e0923b419f3b,0x000a307ca1148286,0x000ee495d75c741a,0x0000a92c2584f24e}}, + {{0x0007732edc665d13,0x00017939ef1b2635,0x0009680899fb5b73,0x000c524db720fb94,0x00006f75f2c63138}, {0x00093ec48d3cb97f,0x00047d261726f8b7,0x0005c4ffb8dff1d4,0x0008700b65791b88,0x00007c79e2ee1a3a}}, + {{0x0001028754c92e1a,0x000a2f0fef3408dd,0x000f55919ca90b57,0x00086a7a9b84ac8a,0x0000a95e0e28d936}, {0x0007315167021a4e,0x000940d5ff984673,0x00092e7066cb6a0d,0x0001a60cc4801a10,0x0000dcab23b1c5e6}}, + {{0x0006356ea37ba9e7,0x00049afff55db4c6,0x0006cc8621adc841,0x000ab680080ef959,0x000056c85b8dd647}, {0x00094aa1db9c2153,0x0000f013b1a5c9db,0x00086c9032dd36db,0x000884ae6ac61c42,0x0000fd32f88f76cf}}, + {{0x00090fca06d107eb,0x000d076611377f12,0x0007b4b38697261f,0x00012f6bb5be4e94,0x000049826b6ebb79}, {0x000dfe85ba8bffb2,0x00005e3fa8e4019d,0x000bcfe7fb1af790,0x000aea52e1bdf201,0x0000ed3ca8ff1169}}, + {{0x000da8cd745fff18,0x0005e9b9924e3bef,0x00051138170b9e9b,0x00053525df48cfd1,0x00004f93fe3506bc}, {0x000c5a9b279a6c36,0x000478f96132aa42,0x000b6aea5651da6c,0x0002a250368c8b01,0x00004e44c480c786}}, + {{0x0009947d9de99a88,0x000d493477bde17a,0x00019e287c2e61b2,0x000134d7f684d41d,0x000043c21237e358}, {0x000e2e904f7e8aba,0x000915f27aeee2d3,0x0001858c4bf93ffe,0x000dbe89830d1d7b,0x00008f449652106a}}, + {{0x0007bc035d0b34a2,0x000b6327c0a7e341,0x0000362d1440b386,0x0009436fb7262dac,0x0000c41114d00cdf}, {0x000cef1ad95a0b12,0x000847d543622ba5,0x000e486c9c09b37a,0x00029706d6cdd201,0x00000477abf62ff9}}, +}, +{/* digit=45 [{1,2,3,..,}]*([2^225]*G) */ + {{0x000dcb3292a92874,0x000637b092c7a004,0x0006c0605ddc15cf,0x0007afc83a846480,0x0000a68df707db99}, {0x0004e4505bf7dd0a,0x0008eccf7f8c9c13,0x000b5f8afa4e63d3,0x0001cc06e6517f41,0x0000a8b93434d7bc}}, + {{0x00064717af715d2e,0x0003f0134a96c417,0x0001ec956e2f7f59,0x0003034c1873efa4,0x00004e7b4f757821}, {0x000ff9788d5374a6,0x000320823d5be5c8,0x000ee8fe22b915e6,0x0006518a6bc755b2,0x0000657624d47112}}, + {{0x00000e07442f1d58,0x0000e6e0e3abd6f8,0x000c64047475607d,0x00083d02807f16b7,0x0000858e1e427498}, {0x000120b8231ee10a,0x000ac38a1ece5859,0x000aa73a41b80e7e,0x0003ac2b72525ac6,0x00007cdea3e24442}}, + {{0x0007b11e51732d26,0x0007c538fc0e5747,0x00039eec5dfd6eb2,0x000c56fc43b0cc3b,0x0000af127792b36c}, {0x000852d06c425aef,0x000b6c221b9b70b0,0x000826d9c6df92f8,0x0009c27c8d4f9ece,0x000059aba7ca4935}}, + {{0x0002cf152bfda056,0x00090197b98cd2eb,0x000a1726fe0e4c4e,0x000e3cbd35076cf8,0x0000c06085b8db11}, {0x000c4d74463ba14f,0x00021030238c15c0,0x00027536d9d292f8,0x000c1d2311ee8b37,0x0000eea86f0aeaed}}, + {{0x0006054afa05dd8e,0x00011caf119ea7f9,0x00064bb5926dfcf2,0x0002b8020ef2e305,0x0000f4dca5141cb0}, {0x000838a65d306723,0x0004cd657e86cda7,0x000d595c88b08d53,0x0008361c5b439546,0x00009b58725725cb}}, + {{0x00053931a62cc262,0x000630c0e052fda8,0x000c633f323c69b9,0x000d488227df15bf,0x0000ac788483bae7}, {0x00078f9187d073d8,0x0009167f807d4878,0x0006e6d8f6c2be91,0x000a42f65861d833,0x00008b8974d4e528}}, + {{0x0002c2e421d3aa42,0x000cc84fa840c37e,0x00054e41cf926407,0x000643f8abc03d14,0x00006605ecd5f7af}, {0x00041a6d6a5eabf5,0x0003d16b668e2423,0x0000101021edb84f,0x000d8c8836edb804,0x0000b337ce7e45e1}}, + {{0x000a2dccc78cf662,0x00044fdbff77666b,0x0008d4668b301817,0x000a2a6d4dd0db16,0x000059455d03dab3}, {0x00064c5cde3acec5,0x0004c3adb276f585,0x000303f657714192,0x000f7b027d725d8a,0x00005deb6ca36f38}}, + {{0x00005d73272d8383,0x0009ca6295c5864d,0x0002fda32e22924f,0x0005445189593f6c,0x000030d7189e184b}, {0x000a62cbde1f7140,0x0004e5cb1a6379ef,0x0001c833235771c9,0x0008542f4826b864,0x00000a894fbc8cee}}, + {{0x000cd0a1058a3184,0x000538053a9adcda,0x000c68de2369cf3f,0x000d9f86c3de5031,0x0000653a5767c4b6}, {0x000dd5aaa4e5c975,0x000167ab3c741688,0x00065c2835be80aa,0x00009120cefe7cbc,0x00007f95f1356867}}, + {{0x000b6de34eaacda9,0x00079ba0f1dec240,0x000438e55d9e116e,0x0002d73be45ec779,0x00001787c9e26f75}, {0x000532bf129ac2fd,0x00078a36e22c897f,0x0009fb8f3d307b7c,0x000b27c194067574,0x000014f95d0e57fd}}, + {{0x000360416df4285e,0x000af0c56ae21625,0x000c5cfc3b0c9bab,0x00005593032b19cf,0x0000497e5c3e9752}, {0x0006bb4164bda960,0x0009a0b74da11209,0x0003826ba1ee4241,0x0006608fc3624340,0x0000c8f0069dc09e}}, + {{0x00074a15b0ec6f5e,0x00037289b2b8c44b,0x000d6fc7347989fe,0x0000aa945f848458,0x0000c362a70d61c7}, {0x00098a7b3a8ad418,0x000ffb63db51070c,0x0004c35f473a20fb,0x000dca6d2c2173f4,0x0000a56149e1acc9}}, + {{0x000c36f35d33ae72,0x000330bb5a94a395,0x000afe84b200ea12,0x00076a00c789bd0b,0x000043ef52d29192}, {0x000c577e23ae233d,0x000ed460d1ec3934,0x000fa76a4b93807a,0x000490e72a53b1f8,0x00008914cb193ca4}}, + {{0x0006b86d23ddc820,0x000c7e0143f04c07,0x0007af2c503fd344,0x000a4fa95362ff31,0x0000add3db7e18b7}, {0x0003e3f8260e01bc,0x000494a1cc919c67,0x000f2e433fbeb49e,0x0001ead1351bf292,0x0000755e7ed45114}}, +}, +{/* digit=46 [{1,2,3,..,}]*([2^230]*G) */ + {{0x000db47f23206d55,0x000fcd2601522bf5,0x0008ff89a2f6d341,0x000457c7b876533f,0x0000157c30c878fa}, {0x000c5c52d4fb936f,0x000bf6518cdc7517,0x000847a64ef22f7a,0x000a88eeb483e6bf,0x0000508455982e0f}}, + {{0x00059d8df7304d44,0x000bbf210e8eab96,0x0003fbd60b71bcf1,0x0004de69a2438bd7,0x0000595cd1f9d11b}, {0x000329a4835859dc,0x000cbdbb6e569c0d,0x000928a4e4a0f0d2,0x00015406038e5edf,0x000094296224f5ad}}, + {{0x0003840a70c6ec41,0x0007d031119508a3,0x0008ab2029e8819f,0x000a27dd209d9670,0x00000d7c4e7de943}, {0x000b317a29b49a1f,0x000256627d1f372f,0x000df39ec57a67fb,0x000f170912561e7c,0x0000a3ce6f35f7c8}}, + {{0x0003462f23f2d925,0x000d10b940789121,0x0006cde206cab71b,0x0004bc1bdd0a6317,0x00004c9b20d3e4d5}, {0x000d8aa9f2ac02f8,0x0006a06eedb03cd2,0x00008643403f8e61,0x000db947f68e1693,0x000031469c612dd3}}, + {{0x0007465653f3c5f5,0x000ea040feb1fe7d,0x000b7edfe283dd45,0x00031141fe599bf3,0x0000ff039add0379}, {0x000995b4e96fa49d,0x0001d3e25094bf76,0x000c83f742640b6b,0x000ac2bb096341c1,0x00002bec885ca560}}, + {{0x000df248f9813540,0x000c3588a2598521,0x000a0992c587e23e,0x000407cbedf281d7,0x00006930a5538961}, {0x0000debbe5bbe21a,0x00048491817f0932,0x000065160a7ffa5b,0x0002a946c8b4d909,0x0000c4f39939ff6d}}, + {{0x000b2a3cc53da664,0x0001b17a9b4e2e4e,0x000e05b2c04708a7,0x0002019bfdc7b20d,0x0000cdc9aee8907a}, {0x0008dfc6c475566e,0x000a67be1691e5cc,0x00005c3fd2b83cbf,0x000eaf895833a74c,0x0000e938a7323b0d}}, + {{0x000a1583ae9c1bde,0x0008037ce2407aa7,0x000ab38b4e0af6d9,0x0008ca054342d928,0x00008b75007ea1c9}, {0x00086afe02358f2b,0x00063a921228efce,0x0001c67fc31b8b85,0x000d58552a19120a,0x00004069ea593aea}}, + {{0x0006205c6ffcfe64,0x00066a39418a3b82,0x0008521fb2eb3125,0x00061584b4a9e1c1,0x00008614dd91b610}, {0x00062bbeea7475fe,0x0008c8ead16204c3,0x00010a876916ab96,0x000ec38a7975bc03,0x0000a4e7d12ccc77}}, + {{0x000d6e27fa03cb3d,0x000a3fdd7d883232,0x000cbfc5ddb938e5,0x00080e34c1d2cd2c,0x00002f45c137f3a5}, {0x00020b57883e6142,0x00089e7c5f265926,0x00067e1e35fd27e6,0x000aaef39e45a915,0x0000cc71d2d64d8a}}, + {{0x00093c706ba8e6ff,0x0007223deffd6ef4,0x0002b5a4f05c07bf,0x000e9bdebf9b4d9c,0x0000958147f1d038}, {0x000cee03e5561c4e,0x0009994a6afc5af5,0x00072168b8d0b5a7,0x0007fcbde22df137,0x00005d249c6b8409}}, + {{0x00090cde36d07576,0x000179a293824a90,0x000b48ddff722d7b,0x000f439b7fb04c04,0x000028ad2a84be16}, {0x000bfb520226040e,0x00007104b6c4cd3f,0x00003c1886c34ecb,0x000aaf50c0754ec9,0x0000c336b090d23c}}, + {{0x0000b4295c69248e,0x000d1ed9030374c7,0x0008cc87a8813259,0x000a94c3e330684b,0x0000689371cf11a7}, {0x000bbf7fbbbc20ba,0x000cc5f9a6e252bf,0x000413e6d73a8543,0x000031c7bb2fdd7e,0x0000066b3f065cf2}}, + {{0x00062a21e206ee56,0x00002c49a633473d,0x000f6b2c3f1e2748,0x0006ea27ab956ce9,0x00001830b48c2b60}, {0x0006846e78e815f7,0x000edc02082a67cd,0x0002ec365fe40139,0x0006aae2bbbfcb95,0x00004c11642db983}}, + {{0x000b89c44f489710,0x00076d660683cf61,0x000d431bde2d700f,0x000f4a72ef285cd1,0x0000593b24e9bdeb}, {0x000bc5b0561f8a1a,0x00000a16f2564084,0x000f6d27784ce74d,0x000dfdfcbc79d309,0x000094fe2fee139b}}, + {{0x000439e558df0194,0x000a6c712b279f51,0x000185a24230da4b,0x000f50118919e355,0x0000dcefcddc4b78}, {0x0000fb2a47d4c5ab,0x000f030e009ea7d9,0x000eed27355ac9ab,0x000faf4d2fc35974,0x000072d824d8bea8}}, +}, +{/* digit=47 [{1,2,3,..,}]*([2^235]*G) */ + {{0x000b2b5ef7d92893,0x0007e97f015a549d,0x0000493b62480d4a,0x00033131d5590bc4,0x0000a55b52e9f780}, {0x0008115309eadb09,0x000a02e5c62540eb,0x0006a3d5adea7de5,0x000d60d4d631f0cc,0x0000d5e9d7d23e8d}}, + {{0x00060411e84e0e5b,0x0002fea34c931968,0x00073a732a5db84d,0x0007c049d5bb1970,0x00008d2fe571bcfd}, {0x0005f36f3eb82fab,0x000c4dff8b584577,0x00074c1108cb20cc,0x0008996659b65f83,0x00008b4a422e30c7}}, + {{0x000ff87aedbae9f1,0x000926880a54c925,0x0004d0e717daf0eb,0x000cf58284ddf59c,0x0000581cf93416f8}, {0x000a8873ac1f4527,0x00098b6aeffe3eec,0x000fb8dc3b417fce,0x00040035918046ee,0x00003d318ac72209}}, + {{0x000cda3af2ebc2fd,0x000cfb4efe24c4f4,0x000cd10b1a0af843,0x000e0383b857c19c,0x0000dc9d1ec614d3}, {0x000c8bb62771deb1,0x000a81c5aa817bde,0x0002391ae829277a,0x0004ca6af18dd683,0x0000740f316d71a8}}, + {{0x00012d8e12b31f8c,0x000b577736e6dd4a,0x0008935e8577e29b,0x00086c6353722ba8,0x0000a1d3729c15f2}, {0x000b6a239a3e0358,0x000f53b03a9f86c7,0x0000d536e6e5250b,0x000831f9d98930fd,0x0000c4cbbac7a0c3}}, + {{0x0000910cab91f1eb,0x00039d1cd2162d50,0x000d02252bedd9e4,0x00017a2634b74fed,0x0000d60f8e1c2586}, {0x000537b9e05614a8,0x000667af5fc5d8c7,0x0002bd926fd26c76,0x000fc78660b58158,0x000070192452cf07}}, + {{0x0006ccfaf64ecb6b,0x00069bd72775afa2,0x0000f695ae6054f9,0x000fde0bcbab5b14,0x0000c71b4a59348e}, {0x00052becc96d9633,0x000f2e5d9018fc2e,0x000568771150abf5,0x000f838d182fa604,0x00005b4c061a0339}}, + {{0x000e99aeeaf8c495,0x000d1e24d7288928,0x0002b156cee7aa73,0x000a1cfc5007c2e7,0x0000fcf57c63d408}, {0x0009e39b6057604e,0x0000e2868bbf9f71,0x000103e2d7d343c0,0x000ea14cca254b7e,0x00006eb38aad131b}}, + {{0x000a04e5f40ff526,0x000f9cd2914ac624,0x000d3bfd63611d7c,0x00063a57e8b42b1f,0x0000cde40fe5a850}, {0x000c449178b7e5bc,0x00006972cc137811,0x000b461354cb6093,0x00056e3579125e33,0x0000a102ef848a4e}}, + {{0x000e3520a981b0d0,0x0003bd1a41a40ba4,0x0009fab9c1c354cb,0x0008d51aabaa3adf,0x0000701a7d153c41}, {0x0007cefdcf2b9218,0x00033cf48061dd1a,0x00025cce66ceef0b,0x000e339083b598de,0x000090a54c8690a5}}, + {{0x000956640ac1807d,0x0005b4da0b1ec535,0x000b82421d4c1e56,0x000dfff4b33f97e0,0x00000bd23184b41b}, {0x000b42e147b72e1a,0x000f5777104c53a4,0x000fb530df8d39f5,0x000fdc96e64e64d3,0x00007a6ba67f6074}}, + {{0x00018db4f6d01b11,0x00027f11e8a04057,0x000591a3be73c6bc,0x0005319c11bb8ca0,0x00002d09a5a1acc4}, {0x00074eee7de13f4c,0x0005444fd682cbf1,0x00048af70177e2be,0x000b7ba5f574cb1c,0x0000e5966935961c}}, + {{0x000e84ebc6fab9f1,0x0001b80f6474989f,0x00002a1bce70ce6b,0x000fd3ae9ff3053b,0x00002ef4866a9c0a}, {0x00057f9e3411a268,0x0000fb485b9822d9,0x000e201190b41d88,0x0008e0020e56d04a,0x00001e3d01fb2852}}, + {{0x000ed6c048752a14,0x000e601341b4c59e,0x000c6b09241f2702,0x000b232e35903b9d,0x0000291aba85f5b5}, {0x000aa70a653d61da,0x0003af2eb51e8173,0x000b93f8fd1b648d,0x0004fd91b7ce065a,0x000055408ef39e2f}}, + {{0x000ed0985e341605,0x00032a03cde21fa4,0x00084df2da26d7dc,0x000107cdac0848fa,0x0000e9a28d66b697}, {0x00004d914ea0ea1e,0x00075ffe5520a880,0x0009b139ca8d1542,0x0008606f422dae63,0x0000edccc0eb4b5a}}, + {{0x000624f8be762b49,0x0000758e3413b33e,0x000d805fa2a9ee4d,0x000fd7068e636967,0x0000848949c0db8b}, {0x000d7e5d23a84178,0x000d73e29da55308,0x000ee471f892f3b1,0x000089495c139e3d,0x0000631594e5757e}}, +}, +{/* digit=48 [{1,2,3,..,}]*([2^240]*G) */ + {{0x000e2ea1f095615b,0x000664e68c331083,0x0008818be0a28ad7,0x0000ccbbfc02523d,0x0000585113ba3585}, {0x0005f0b30df8aa1c,0x000b8ab7e3ac7d93,0x0002f00cbaddda07,0x000f6bd2c3429955,0x000033ed1dee909d}}, + {{0x0000d483e07113c9,0x0008ed8b63ae2dc4,0x000684c2b6e4a5d3,0x00026bc582a94b79,0x000032b33d4f22da}, {0x000f6510dbbf08dd,0x000894c23a52f534,0x0005bdc9b211d07c,0x0005573eeece0fee,0x0000f178169c7015}}, + {{0x000905a83cdd60ed,0x0004d1170184abe7,0x00023642a50602fb,0x000aff989886cdb0,0x0000568d09176e1f}, {0x00022c70259217fd,0x0008f43141e45b19,0x00095f86e93831cd,0x0008280fca35870c,0x0000ec2057b268ae}}, + {{0x0008925913cc16df,0x000cf1a26f5a568f,0x000f499ae18bc5b6,0x000e83efa413bef5,0x00008835dedb3f0a}, {0x0000bd865a40ab05,0x0008c94b377eb6e6,0x000084a696559643,0x000de06cd8562592,0x0000ce433b99f23e}}, + {{0x000523d42e06189e,0x00006e3aff13860d,0x000b20650bf07794,0x00000c2b616dcac1,0x000066dd6d201313}, {0x000fd67ff99abde3,0x000097aac50dd4a0,0x00046b2d7c990355,0x0002aed22ecf8b7c,0x0000333b1e86abf9}}, + {{0x00065e784d6365d8,0x000f0f759fb8c0da,0x000e81930bcb7443,0x0008aab5c712b17a,0x00000428dffcc6e0}, {0x000afefa4faf8433,0x000dcfa9855ff19d,0x0003ac7ceced8538,0x00071df0ac409cbe,0x000058c1fb6b82da}}, + {{0x000def7be42a5821,0x000055046be6efec,0x000e8dba9d3fc608,0x0001ffb9af13c809,0x0000e6c984774149}, {0x0004925d30c31f70,0x000aac2a21223b57,0x0000859e7b7eb72b,0x000942776a0dacef,0x00006fec31421900}}, + {{0x00094b07e50122b3,0x000b1af07ca53247,0x0003fc97bdd744f8,0x000d9d00a12f08d6,0x00009650f1aa6626}, {0x00047f71fa38477c,0x000914dc124f101b,0x0006eb58a3d815f1,0x0008865569ae95b2,0x00003cde18955fb1}}, + {{0x0002fbbf4737f217,0x000af209f5ac7ec6,0x000f9adbed8dba5a,0x000767b4b5d7a9a5,0x000007d28f8161dc}, {0x000460bcaa999eab,0x000c6c92e4cc7711,0x000d4bf2dba7b174,0x0002788c4bab6618,0x00008f0c9819b8bd}}, + {{0x0008932790691bf5,0x0008b6b736ae9d65,0x000d63b6eed61058,0x00088f212c2f04c0,0x0000cf06fd6163d4}, {0x0003facd9588e411,0x00021b93257e9736,0x0007acace1f9bf76,0x000ecf99d1ffc466,0x0000cf4a1aa1a061}}, + {{0x00075b2c0519a238,0x000dcf6952e328d6,0x000294a8a9ebf94f,0x0003f5728bb767a2,0x00005512b4e7e0af}, {0x0008ba899b16a0de,0x000bda7548a71895,0x0006be61595c2430,0x00074bd30d1b10a1,0x00003ebbb9865bfb}}, + {{0x0003f5eb2e636456,0x000724c8423207b5,0x00014d5cfbe57e54,0x000aca7779c21672,0x000017969cd629a3}, {0x00068cd8a7017a0a,0x000a1e9b7ee8d176,0x00016177677b4d19,0x000a71b8fd0e939c,0x00008c4f4f075968}}, + {{0x000b865d2fdca23e,0x0007ec8ef89581ee,0x0009056145a15ee0,0x00019a968fa10a01,0x0000ff5b8f0680ee}, {0x000cabbcb1c8a0ef,0x000cc8c838f9f0c0,0x0004a14c02e1ee9c,0x0008e41587d8b88a,0x00006f278971ff69}}, + {{0x00034b3bf44da801,0x00094ab32e66519d,0x00078a000283834f,0x0002f65e60879762,0x0000e62960e72731}, {0x000b27be6901c550,0x000824fdbc1f9b87,0x000acc27d80e7853,0x000b5abbbc09512f,0x00006394239ac143}}, + {{0x000646e9e2fce99d,0x00004e80857f9c4b,0x00043b52a68a2108,0x00084236d54e4436,0x0000e8d6d63dd8eb}, {0x000156342146a0a4,0x00021eaa36227032,0x0001387878ba826f,0x000d36e27a58bd86,0x00003b6c03c50281}}, + {{0x0007a952f41deff1,0x000ad63b89b702b3,0x0003ff9510e44a59,0x000af4573257dc14,0x00009c02205e752b}, {0x0003069c4b7d692e,0x00031d1502ac46c2,0x0002208462e6392c,0x000b628057b1a21b,0x000051ff946ec1b5}}, +}, +{/* digit=49 [{1,2,3,..,}]*([2^245]*G) */ + {{0x000cb51566c5c43e,0x00085597f0466e85,0x00094d94acff9c91,0x00027cb354e90c49,0x00000a3933301479}, {0x000fac10dc1eb2bf,0x00013ff319fa8427,0x00096527488cfd8c,0x000745f2d4e68401,0x0000a2e067e57aaa}}, + {{0x0002a7f3e5f9f11f,0x0009e6cb3b8eb6d9,0x000f800bd9afe153,0x000e185d1a6dd7dd,0x00006c13cc1baf17}, {0x000c58e325fc3ee3,0x0000731dc3b215f6,0x000a3d3e77109540,0x000e2ce68e7c07af,0x0000f8417a1c4c7a}}, + {{0x0004772813b230d7,0x000ea7344427ec23,0x0007fc56a634d0f5,0x000f76a1548ab1d7,0x0000fab17513e06a}, {0x00010a74f7c4f830,0x0004220a67d9b62c,0x0001209a0a7d2edc,0x0009f01c40417092,0x0000b9815a0face5}}, + {{0x000589b319540c33,0x00097283d6f82842,0x000ae9fcb18490f5,0x000ba072731f84da,0x0000db6d960f3683}, {0x00063bb146110697,0x000e9788bf05c85c,0x0007460d2b19436a,0x000db1205459df34,0x00003f6e095511a7}}, + {{0x000938eb6357f373,0x0008fbd8aa62dc7f,0x000a979fcc5d00f7,0x000a999878dcb92c,0x00007e83eda1b023}, {0x000e2731560bf3d1,0x00090d0fae616b23,0x0009414bd1086e45,0x000ea1682483169a,0x0000b956bc100ea9}}, + {{0x000bb91c31b9c38e,0x000a68ef57b57b85,0x0003bab6f0c5aa90,0x000684fedeb169af,0x0000610ad740d373}, {0x00070df02ba8e15d,0x0005bca7f771f138,0x0002c036c0337edb,0x000e8114acf747b6,0x0000921d57786b94}}, + {{0x00064392c422f7ac,0x00022d348898dbc8,0x0005bfcd1fb63536,0x000e10c3084668c4,0x0000357c9e3eb315}, {0x000b5405b2e5b8ce,0x00010102b9a4b173,0x0000fb1997e94693,0x00062a37c890eb7b,0x0000c225a84b61b6}}, + {{0x000a3c8ee3c76cb3,0x0003a32a1f6ef306,0x00063e9563cf1162,0x000c26b6d5ab6468,0x0000b8a4cbe8c005}, {0x00029a59ce6bb278,0x000184d4b16fdcd5,0x00023798dc4afaa5,0x000fab30624a2679,0x00005e56df6eb307}}, + {{0x000893c2bf296984,0x000497ce76030281,0x0009a558f91fc19a,0x000f7735a5dca3ad,0x00000ceb3fa8d50b}, {0x0006060bc9ba369d,0x0006897888c21baf,0x000a34c07927e103,0x0009800936bf1986,0x00004cf10c2934ae}}, + {{0x0005334859dd614c,0x000a58d0c8ee3a3e,0x000cd51d59c475b5,0x000325a3080d1f07,0x00009c0d0a7788b4}, {0x0008691c234296f5,0x000444887fb61ac9,0x000ea9cf22a0a83a,0x0002f5065114270c,0x0000230a6e8f2480}}, + {{0x000bf0f72e3d5c1e,0x00056f21439ef7a2,0x000e303343771744,0x0002f91edcbf259c,0x00000030a795ce20}, {0x0009ebf1202e9ca6,0x000c15e6e5916f2d,0x000dac4f8e79dde6,0x00004d952072aff1,0x0000c8d087f1b9b4}}, + {{0x000c73dbce913af4,0x000b058a07cbad0f,0x000f00c8a909e587,0x0006abd300da84d4,0x000025cd048f5446}, {0x000b9be90e9d8bf9,0x000aae431b0eb59c,0x0001aecff991616d,0x000c3b43aa117a53,0x00001af92d3e9f4d}}, + {{0x000c292e93fda293,0x0007b97d91bc9b1e,0x000ace1e676bb6c1,0x000ae34509d95faf,0x0000653fe47ee855}, {0x0000b280f680e752,0x000bceb6c26c7318,0x0006d423675eefd1,0x0001d884cdf29fb6,0x0000d70a9978b582}}, + {{0x000e20720445c36d,0x0001898771747a3e,0x0009f73e971d1ac8,0x0000803fc539f794,0x000005cf3d8682e3}, {0x000e20b7b1c7129b,0x000fa69e61f28758,0x000544c2dffadcc1,0x000e53005d3a2f59,0x0000e16f5c24fff5}}, + {{0x00065b8aad581350,0x0009037aa5be73cf,0x0006fd6a0622c211,0x0008cf79373b3f64,0x0000e029db50d397}, {0x000c43794fba0377,0x0006f20797a68bdf,0x00030d38eaefbd68,0x000463cfa5382bbd,0x0000627cfbfc85d7}}, + {{0x0000fef4e4ca4631,0x00072566cc63b233,0x000780900bcef728,0x00027dc161d2cacf,0x000035dc5396b548}, {0x000052e27bf1bc68,0x000f87dfa06c638f,0x0003321da10a224e,0x000c8f6973586d6d,0x0000b0c5738a6152}}, +}, +{/* digit=50 [{1,2,3,..,}]*([2^250]*G) */ + {{0x000ba6b23a5d8961,0x00056fe4364e9910,0x00033c6771fe19e3,0x000fd05e1da8c39a,0x00005b4488b39fd9}, {0x00092541a1f22bff,0x000fbb8163e81f43,0x000e5658e920a8a6,0x00039a4fd1b24907,0x00002c4f79da6ec8}}, + {{0x000619c76497ee80,0x0006c717370e8b5c,0x000cf68e15d2b0ac,0x00079298204cb64f,0x0000bdec21162bc6}, {0x000ccefa63b10110,0x0007e0de1ac56973,0x0000e0c8bf9e3fa9,0x000cb45efb693e3d,0x000037248e9d2d4f}}, + {{0x0004e2ab20364e46,0x00088770b20dd369,0x000d77d1c6269971,0x0000a50183291b6e,0x00009aada7fcd618}, {0x000054bf185509bd,0x0005a870107751f9,0x000b96d8dfd7e845,0x0007ee06a2308a7f,0x000053e48d2f8dc8}}, + {{0x0002dc91ec34f9e7,0x0004c38106038080,0x0000cb4f3d8772d3,0x000128cf06d66c53,0x0000be5ed0e3475c}, {0x0003c1931e82b100,0x0007c9ff6b4ccb9e,0x000a1b45ec63d285,0x000bcab92118c692,0x0000aec44147285b}}, + {{0x0001afbe8316c455,0x000cf6700b8b3707,0x000ff8578982be4f,0x000fa73d5d177c64,0x0000ec4058319a82}, {0x000e37bcfa86678a,0x0003ff0312845518,0x000bbb74c24e809f,0x000de5912f39604b,0x0000d4b4f7060c14}}, + {{0x000f4868eb6e843f,0x000abc4ab3aba3d6,0x0003cf2dc6415834,0x000c3d128a81514f,0x00003b496304f3e6}, {0x000ae90987dd2f21,0x000ad10bce559119,0x000149ed6437a6d8,0x000332b31cdd6b9b,0x0000b77791d16871}}, + {{0x000b650e16e05e91,0x000084e746409c34,0x000a3f029965a774,0x00077c93fd22fbce,0x0000eb6a9689f952}, {0x000a63d7bad84f61,0x0001e58f2fee2520,0x000840d48ad91720,0x000cbdda92c1669b,0x00002109c4abcef5}}, + {{0x0009ae71e29a3efb,0x000f9c93302efc18,0x000aae10ecbe906e,0x0009f820107914ce,0x00007a23f35668e1}, {0x00075c2efd2119d3,0x000eccadc9c8e9d8,0x000a1711303198c6,0x00003835591bf64d,0x0000cf0bbf86d443}}, + {{0x0003027aad991c1a,0x0007e49be4b7ab29,0x00072da90598d0bf,0x00083fec94a21ab9,0x0000da4cfde1ecfa}, {0x000b9c0fbcec63aa,0x000163219a3493d4,0x000652a557ca617a,0x0002dff00424eb6a,0x0000f9346ea8b856}}, + {{0x0004d79023814615,0x0005411bc478c3e0,0x0000b3ef2b1643ab,0x0001b6792becfa39,0x00004476778fcd2f}, {0x0000c4d66bf3aafb,0x000a7c21c65a8daa,0x000a13ac32bc1287,0x00090a3e182910b5,0x0000b04730140b07}}, + {{0x0006947d17d4e0b1,0x000eedc3a47b8376,0x0006fd0ffc5772be,0x00095e665a50db1a,0x00007f904ba55b09}, {0x00043832883487ec,0x00030270aaedcee6,0x000cb1fd9f56db7f,0x0004d4a738d94f46,0x00008fa426ad42fd}}, + {{0x0005bb72b7247593,0x000182d4c63aae48,0x0007d6f2c945353e,0x00010952159d07de,0x000089caef37ec5b}, {0x000bb53db65ef147,0x000e6d99de434a8e,0x000f2405f2dc2cb7,0x0008a3116fa3ed83,0x00003429bba31420}}, + {{0x0008febd56daf065,0x000afa837f6908ed,0x000b6e05a8d98277,0x000abd4947c636a9,0x00008c8a77b10d58}, {0x00096a45f121e4ff,0x00062076d3d3f454,0x000fb0c5d16cd67c,0x000b46fcbd1958e3,0x0000be185ed28e1e}}, + {{0x0009760ff79d2ee9,0x000f639e7832b7ef,0x00005b499dd4d06a,0x000d316d025001b9,0x00008fe1c6285b61}, {0x0003980c5f128050,0x00083009cd9aa9f8,0x0004eb16d376e3b7,0x0002dfa322b09f51,0x00008e213122c383}}, + {{0x0008626dc4ad4f4b,0x00019ddbc0b6aea6,0x0002e90655dc5168,0x00088df76697bd60,0x0000eeb3ea63c378}, {0x000a2f2145691136,0x000c435f44841ec4,0x000e44df2e48b820,0x0009fd3fb560949a,0x0000ca13462f2cc0}}, + {{0x000d590b01e6e274,0x000da180b2dcb618,0x000aea4a9047e2cc,0x0003a491b299b504,0x000012c9e1edfa40}, {0x0008a36794075521,0x0006e332b8e388d2,0x00068de1949c5013,0x000b972a1b6fcce6,0x000078851bc85122}}, +}, +{/* digit=51 [{1,2,3,..,}]*([2^255]*G) */ + {{0x000d46d95a7b1a29,0x0005ac6341fb197d,0x0004c2ece9c4e7ad,0x000f89b26eca2948,0x0000211e48a6e7f4}, {0x0007f6ec78ef1f42,0x000fe65745861499,0x0003eede82b2c090,0x000017f7286a6e1c,0x00005f92e472f60e}}, + {{0x00070af3aeac968f,0x0008d4b63266b4e3,0x000ac5664e4f7fee,0x000cbec4acd4c2e3,0x00008910bd3beb38}, {0x000e50cc9c0726e3,0x0009a97b40bf1c3a,0x0005a5a1b1530956,0x0004cd40884b7ffd,0x0000890896b1f831}}, + {{0x0007d205ffe7b376,0x000ab747d2da090d,0x0004fc5193b7f3ef,0x00071e42cb525fb5,0x0000e220933566a9}, {0x00060dfb486d440d,0x00056fe13465ddc1,0x000e4c1517fcfec4,0x0002b4b3da7e4e76,0x00006fa48a2a8d30}}, + {{0x0003c9f82e4c6346,0x0003da4464f85ced,0x0001dca258efb831,0x00012b8706381b7a,0x0000cd15a3cba2a4}, {0x000a8fdbfcd8fb51,0x000f5e54cd229347,0x000d8932f31db2ee,0x0001afb4aeb11ef8,0x00001e7c1ed44441}}, + {{0x00065ef12045cf9f,0x000eaccce8bdae40,0x000cf65256fcb2ca,0x0003112fa0ba4ef2,0x0000683125ebb72c}, {0x000a4eae312410e0,0x00076cd8e830a01d,0x000fb3f0767e2867,0x0009a5abd9575298,0x00005f11e11eef64}}, + {{0x00084f5903fa2711,0x0002a9da921e9968,0x000b01e54e6da0fd,0x00014e96f2f2695d,0x0000ee3e9bd78762}, {0x000181ce27a94979,0x0003fe215e04a26e,0x0002cabca36d254e,0x000613b2f32a6c25,0x0000948148810b57}}, + {{0x0001058fa28a7e00,0x00050bf5ec74ea96,0x000229666c726cf2,0x000799e74d55c8db,0x0000bd9abbfa57f5}, {0x000ef074dfc47b3a,0x00026c52f91d7479,0x000a8bde2d9c65fc,0x00027fee0283fe36,0x000032a8b5f1d4b7}}, + {{0x000b43a43228d831,0x00003ad63f99ab41,0x000a5122924ae1c3,0x0002b47e525f1a46,0x00004af860fdd26d}, {0x000ef613f714aa18,0x000d6b78795ed6ba,0x000a9d694f51865a,0x00052753e21fcee6,0x00002ceb1de0a37b}}, + {{0x000d27d483076829,0x0003510566da8605,0x000f2d7ab745aaba,0x000823557cae36bf,0x00001332ba1db127}, {0x0003638f3429f438,0x000f6c462929cb5c,0x00018b7bd43a21c4,0x0000c669bc95352b,0x0000c2addf4678cb}}, + {{0x0005bfd2f9fd51a3,0x0002181b97f74a66,0x00036ce507f2f1fe,0x000ded9ad05d69ad,0x000014fc2a4b44f4}, {0x0003d8cb55fc5c6d,0x0007efb1e23dd559,0x000453ccee3510ce,0x00063129b7be6937,0x00003541b7a39fae}}, + {{0x000cbcad7154cedc,0x00067f52651ab013,0x0005e642e949b285,0x0004a343b41f6e9f,0x00003a3462a46ed9}, {0x0006cd6222b24dc4,0x0009c28c9c26f3c8,0x000dad3b67578fe8,0x00040a6ac7bfa12e,0x0000112c5d7c8479}}, + {{0x0006525eca445df4,0x0001ecdfa4c69929,0x000a6d3bcf1af24f,0x000cc7b5b4eb61eb,0x0000560910cd8972}, {0x0001c32093eaa327,0x00090d3c67bb5475,0x0008711100183134,0x000a7dcbd90ce62d,0x00005fc863ac38ba}}, + {{0x000f8b64ae3a2787,0x0005a37cdf658fe1,0x000be0492160c530,0x000d81b2f029e733,0x00009a68042ea75c}, {0x000094a5b3f3ada8,0x000d965c32501d8c,0x000792e225d723bb,0x000ea988b958ffa5,0x000029fa987311c1}}, + {{0x0008a4176a9f05d0,0x000b9011d488711b,0x00048a65e06ca4e4,0x000894543bc62ba2,0x000017535ffc9290}, {0x00084ce406851d75,0x000f40e960b4840b,0x00028fd34afa3acd,0x00092c5c3394af71,0x00004eb4d5b7ac0f}}, + {{0x000a73e4a14f8366,0x000790946328923e,0x00017a4dfc8cc8c5,0x0009296e3f117fe9,0x0000372f934572ce}, {0x0000249c29ba567e,0x0005ac829cca7500,0x00002ec7acbd437e,0x000d168c63aaabdd,0x00009b2f90d5b42b}}, + {{0x000e87355dbd4b3d,0x00079639bbb1db09,0x0006519621f87992,0x000573e83e47e51c,0x00004ef0fb7943fb}, {0x000b9d8f1bfb12a4,0x00082e5e8b7227d3,0x0007b90146ab877e,0x000b644eebdc9d15,0x0000c2110057aa5c}}, +} +}; + +#endif /* IFMA_ECPRECOMP5_P256_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecprecomp7_p256.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecprecomp7_p256.h new file mode 100644 index 0000000..e32a403 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ecnist/ifma_ecprecomp7_p256.h @@ -0,0 +1,2474 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_ECPRECOMP7_P256_H +#define IFMA_ECPRECOMP7_P256_H + +#include + +#define MUL_BASEPOINT_WIN_SIZE (7) + +#define BP_WIN_SIZE MUL_BASEPOINT_WIN_SIZE +#define BP_N_SLOTS NUMBER_OF_DIGITS(P256_BITSIZE,BP_WIN_SIZE) +#define BP_N_ENTRY (1<<(BP_WIN_SIZE-1)) + +__ALIGN64 static SINGLE_P256_POINT_AFFINE ifma_ec_nistp256_bp_precomp[][BP_N_ENTRY] = { +{/* digit=0 [{1,2,3,..,}]*([2^0]*G) */ + {{0x00030d418a9143c1,0x000c4fedb60179e7,0x00062251075ba95f,0x0005c669fb732b77,0x00008905f76b5375}, {0x0005357ce95560a8,0x00043a19e45cddf2,0x00021f3258b4ab8e,0x000d8552e88688dd,0x0000571ff18a5885}}, + {{0x00046d410ddd64df,0x0000b433827d8500,0x0001490d9aa6ae3c,0x000a3a832205038d,0x00006bb32e52dcf3}, {0x00048d361bee1a57,0x000b7b236ff82f36,0x000042dbe152cd7c,0x000a3aa9a8fb0e92,0x00008c577517a5b8}}, + {{0x0003f904eebc1272,0x0009e87d81fbffac,0x000cbbc98b027f84,0x00047e46ad77dd87,0x00006936a3fd6ff7}, {0x0005c1fc983a7ebd,0x000c3861fe1ab04c,0x0002ee98e583e47a,0x000c06a88208311a,0x00005f06a2ab587c}}, + {{0x000b50d46918dcc5,0x000d7623c17374b0,0x000100af24650a6e,0x00076abcdaacace8,0x000077362f591b01}, {0x000f24ce4cbaba68,0x00017ad6f4472d96,0x000ddd22e1762847,0x000862eb6c36dee5,0x00004b14c39cc5ab}}, + {{0x0008aaec45c61f5c,0x0009d4b9537dbe1b,0x00076c20c90ec649,0x0003c7d41cb5aad0,0x0000907960649052}, {0x0009b4ae7ba4f107,0x000f75eb882beb30,0x0007a1f6873c568e,0x000915c540a9877e,0x00003a076bb9dd1e}}, + {{0x00047373e77664a1,0x000f246cee3e4039,0x00017a3ad55ae744,0x000673c50a961a5b,0x00003074b5964213}, {0x0006220d377e44ba,0x00030dff14b593d3,0x000639f11299c2b5,0x00075f5424d44cef,0x00004c9916dea07f}}, + {{0x000354ea0173b4f1,0x0003c23c00f70746,0x00023bb082bd2021,0x000e03e43eaab50c,0x00003ba5119d3123}, {0x000d0303f5b9d4de,0x00017da67bdd2847,0x000c941956742f2f,0x0008670f933bdc77,0x0000aedd9164e240}}, + {{0x0004cd19499a78fb,0x0004bf9b345527f1,0x0002cfc6b462ab5c,0x00030cdf90f02af0,0x0000763891f62652}, {0x000a3a9532d49775,0x000d7f9eba15f59d,0x00060bbf021e3327,0x000f75c23c7b84be,0x00006ec12f2c706d}}, + {{0x0006e8f264e20e8e,0x000c79a7a84175c9,0x000c8eb00abe6bfe,0x00016a4cc09c0444,0x000005b3081d0c4e}, {0x000777aa45f33140,0x000dce5d45e31eb7,0x000b12f1a56af7be,0x000f9b2b6e019a88,0x000086659cdfd835}}, + {{0x000dbd19dc21ec8c,0x00094fcf81392c18,0x000250b4998f9868,0x00028eb37d2cd648,0x0000c61c947e4b34}, {0x000407880dd9e767,0x0000c83fbe080c2b,0x0009be5d2c43a899,0x000ab4ef7d2d6577,0x00008719a555b3b4}}, + {{0x000260a6245e4043,0x00053e7fdfe0ea7d,0x000ac1ab59de4079,0x000072eff3a4158d,0x0000e7090f1949c9}, {0x00085612b944e886,0x000e857f61c81a76,0x000ad643d250f939,0x00088dac0daa891e,0x000089300244125b}}, + {{0x0001aa7d26977684,0x00058a345a3304b7,0x00037385eabdedef,0x000155e409d29dee,0x0000ee1df780b83e}, {0x00012d91cbb5b437,0x00065a8956370cac,0x000de6d66170ed2f,0x000ac9b8228cfa8a,0x0000ff57c95c3238}}, + {{0x00025634b2ed7097,0x0009156fd30dccc4,0x0009e98110e35676,0x0007594cbcd43f55,0x000038477acc395b}, {0x0002b90c00ee17ff,0x000f842ed2e33575,0x0001f5bc16874838,0x0007968cd06422bd,0x0000bc0876ab9e7b}}, + {{0x000a35bb0cf664af,0x00068f9707e3a242,0x000832660126e48f,0x00072d2717bf54c6,0x0000aae7333ed12c}, {0x0002db7995d586b1,0x000e732237c227b5,0x00065e7dbbe29569,0x000bbbd8e4193e2a,0x000052706dc3eaa1}}, + {{0x000d8b7bc60055be,0x000d76e27e4b72bc,0x00081937003cc23e,0x000a090e337424e4,0x00002aa0e43ead3d}, {0x000524f6383c45d2,0x000422a41b2540b8,0x0008a4797d766355,0x000df444efa6de77,0x0000042170a9079a}}, + {{0x0000b650bc6fb805,0x0004effe2e6b808b,0x00083f5495882e07,0x00072385ef2f7c2c,0x00004d63c80e103b}, {0x0001bd652a23f9b6,0x0008eb0b6587f2f1,0x000580e9e3670c31,0x00021ff5c4623bb1,0x00004edf7b261efe}}, + {{0x0001dcbd53c5c9dc,0x0005ec0a177b9709,0x000fe2dfff17624b,0x00074df0f139752c,0x00001a35c0b2c7a5}, {0x000314693e79987e,0x0000089cb80e227d,0x0001883bb0575bf3,0x0003cf4f4e247f0d,0x0000bd512271274c}}, + {{0x00051c856ada97a6,0x000d2f8b403e5f3e,0x0002e29794afc964,0x000bda46f247ab41,0x000075abd1bcf80e}, {0x000bd725e485a1d6,0x000f2f4f0b3c66a2,0x000847bba4b2a5ca,0x00094cc626927f1b,0x0000c6fc7d965023}}, + {{0x00012baa5659ae8f,0x000935e1a16efea9,0x0002c41ac68363ab,0x0003fbc884227775,0x0000e545c291897c}, {0x000e9e7dc4c696bf,0x000a0ba977c52d36,0x0009508c15806244,0x00097a95665e9be3,0x0000720ee265d125}}, + {{0x0009129d2337a31e,0x000e2f862bdc8a97,0x000d283ba5916868,0x000b4d248099d95d,0x00002d1eeb7de5bf}, {0x0001c417884005d5,0x0007afffcbae82ef,0x000a95e66a2d4ec1,0x0000d04161c53f8a,0x0000ee104e215fee}}, + {{0x0004cecc135b208e,0x00046783f47d562e,0x0003f3b3074e1b26,0x0002fb8d2a506c5a,0x0000cead9f5a1676}, {0x000d4b2e286e5b9c,0x000fc3bb3c61f29d,0x000ac29a41b0fadb,0x000fa2ba75023e7f,0x0000086d5f289477}}, + {{0x00011352f6f3076f,0x0002f3912a9a0fc6,0x000f8ba3dc99ffa2,0x0008a37a0b0685d2,0x0000dc777e9d9335}, {0x00087bb35415f047,0x0009dd23fea494a7,0x0003a35b5640c2d6,0x0000742de917da15,0x000093e8d07cd5cd}}, + {{0x00076532de45068a,0x0007fe2e1f6ef4f8,0x00058406937c7a7e,0x000f4230825fa2a3,0x0000f2cea7cb727b}, {0x000a4fb9e4785a98,0x000ba7299f4a0360,0x000ac2f71e5fda49,0x00066ec8068e1371,0x00003d0687c10776}}, + {{0x00083b215d02819f,0x000f50dd9a356d38,0x0002b469f6d0d754,0x00011471d7cbf91d,0x000097b23301efc3}, {0x000d750b24bcbc75,0x000938a1e356a551,0x000cb750111ea494,0x000b8a2669f03193,0x000095dc55f1a737}}, + {{0x00019acd837879f9,0x000e5d6b67b0a4a3,0x000f1f3af6fc1b49,0x000a2e5395993332,0x000066742ebf5432}, {0x000c9feb49662287,0x0001d3f439504b8d,0x000b731ee96cc631,0x000967a2068859c9,0x0000b948dc3c6f79}}, + {{0x000ad32ed1f80089,0x000a48b1753861e4,0x0007ff6fbe6c9267,0x000b0f8ac7c5eb85,0x000094baaa8e5f2f}, {0x00014e11d248018e,0x000a828ac50884cf,0x000a944f55a39898,0x000ac634fde97b5f,0x0000d178031b12e5}}, + {{0x0002af497e2feb49,0x00071ebf7313042c,0x0004ffdd7d36a42d,0x000769b9d2c9eb08,0x0000f8aa54bbef7c}, {0x000b7ba09895e702,0x000fbdb7fb589200,0x000eb4cbb3bd0c66,0x000e310d97d10878,0x0000d431068f84bd}}, + {{0x0003eb7172ccd1fe,0x000750a6a8924b52,0x000e153eb7323cb2,0x000b96b7082ec0cf,0x000097f6b6bd2aad}, {0x000393ed1a83da1a,0x0006e04b2a681d3d,0x0000cb71ea6a7f9c,0x000277aa688b482d,0x00009b4cc5fe0585}}, + {{0x000b46acb66e132e,0x00092d9258805e5d,0x00017b9e2f1be963,0x000d47b44a702703,0x0000266f95a28603}, {0x00066735c2088993,0x000772fb18a398db,0x0007c619f9047244,0x00021b5a96693977,0x0000798142a5a3be}}, + {{0x0001cb13298b343d,0x0008e44f65a1b424,0x000c77acda3a14e4,0x000c3bf5f4d6cd3a,0x00000288cb622b6f}, {0x0008c2f1c040abc4,0x000dc6bf9b4ad5cc,0x0003aa441b675511,0x000f729667da379b,0x000060d45ce91601}}, + {{0x0003c696755ff899,0x0006b73017e6e2f7,0x000f7600ddd3cf7e,0x0007b3fef5689d3c,0x000048dc4f941fc8}, {0x000fe814ea53299f,0x0001a8eb6028d9e9,0x0009803fc2d921ca,0x0007450aecedfd0c,0x000038ae8923d7b4}}, + {{0x000fccfc5e3a3d83,0x000c1079dfbfd8c5,0x000ad0197befd904,0x0002a48c6d6a58fe,0x0000922707799553}, {0x0003e6ddbef42f56,0x0003e80a990809e2,0x0009a2e407e449b6,0x0002a41b969c1aad,0x0000231d792f591c}}, + {{0x00014560f6645343,0x000c1b68f1038715,0x000578ab985ceae7,0x00010c7c09c4ae65,0x00003ec68692044b}, {0x000832b3a8ec1f1b,0x0007a847d5ef6ac4,0x0003f15745509d12,0x0003c44909604f76,0x000016c4304732f6}}, + {{0x00020147ca23cd3c,0x0005e391849db6ab,0x000678d94caa7a5c,0x000e639b0673a375,0x0000982ddd59d303}, {0x000000b5db6f971a,0x000ecf876f92fd7b,0x000569426bba2cb1,0x0004f8277332a33c,0x0000159100cf70d7}}, + {{0x000847fdec67ef5c,0x0003633e76b7fd16,0x000c2b4c8742ee46,0x0005204b8e4134ef,0x0000a640b8702a3e}, {0x00001908ceb6aa92,0x000c347852d5653a,0x000237af7313c300,0x000af804e4ab126b,0x0000ba90162abb47}}, + {{0x00058d6a8219bb73,0x000ceb06c57f3d5e,0x00057576ec691d0b,0x000dc2dae4cb10d2,0x0000569656d054a3}, {0x000aebd94cda03a4,0x0002d62bfe13e5eb,0x00051a0c6934e82d,0x000526050ac0bae2,0x000080b9e121d6da}}, + {{0x0007bc58cce08b52,0x000c5f178d550046,0x00077d806b636458,0x0004eba5748baea6,0x0000763a387ffa39}, {0x000448a7d3cebb65,0x000e1f20d850a12b,0x00058462ce7adda3,0x0008a8a63ebce515,0x00008b36143b2008}}, + {{0x000c3ca4d63c0eea,0x00066fe948ce8a2c,0x0002ef33b5123311,0x000d6bd463fd8522,0x0000df0c7dd1c603}, {0x0002d3bfe7765e5d,0x0008ef3804090ec3,0x00059319cccaab35,0x00034cedaa84d68e,0x00009a4c2816c80c}}, + {{0x0009488a059c1425,0x0004af0b9346a9d8,0x000fb36646f5ae71,0x0006abb68f237d16,0x0000853e4c486318}, {0x0007d2363c52f980,0x000681828876e2d8,0x0004e7b1c2ec4a76,0x00040847b864fae1,0x0000c0bc0e569192}}, + {{0x000681db82e9f3e4,0x000b9f25e13ce4d7,0x000f2728083200f0,0x0002274909984c66,0x000062d7b00b5f73}, {0x000a188f26517980,0x000c36ab1c34d90b,0x000f5435974c6e18,0x00002fab256ea35e,0x00003466612d1aa7}}, + {{0x00060492ed22e919,0x0004df072822624d,0x000ce22716fdfe0b,0x00014f5eca111539,0x00008100a506b016}, {0x000daa2a35c628ff,0x000dd87e9a47b6b0,0x00057d9ceb6f94d2,0x000a7ad67732591d,0x000070bfeecf3884}}, + {{0x0005ccfed2bad016,0x0002bda6a5c75fb3,0x000a92f8fa155cbe,0x000e4362e2594c30,0x000049c89cebbfaf}, {0x000667de9ff257af,0x0001032c50aed158,0x0006014cf9b35961,0x000d3c5b00b20b90,0x00003a8cfe479bc7}}, + {{0x0003ffd248a7d061,0x0004778873fa4ff2,0x00074598180c5bfb,0x000994a7d9ad9005,0x000079c85db4db01}, {0x000b06261a6966c4,0x0002aadce5a8ba41,0x000e6a3184d82d05,0x000da05e91cd3ba5,0x00007795f4fd5b2d}}, + {{0x0007c1fd55a897c5,0x000b629110fbecfd,0x00081d3b0009194a,0x0002910f0e2046e3,0x0000f3425f6f98dd}, {0x0006687730d50dae,0x000b6b083b7fbfa0,0x0009d34170423446,0x000429597a247dd6,0x0000b629f91187ba}}, + {{0x00026ccd5cd79bfe,0x000ab46c6e181ee4,0x000477f580032940,0x0002773b1e8ae057,0x000094f7d354d823}, {0x000cb96782ba21ad,0x0009272b33a5c747,0x000f80c81c525446,0x0006b4a72ef6dec7,0x000073acbfefcd9e}}, + {{0x000b5b149ee90d9c,0x0009e06e9eba4075,0x000f825e0785c339,0x0001dbe1030d5bab,0x0000ec684c464293}, {0x00062c9c1586e630,0x00065ab43f2b42ab,0x000f7835d45431d6,0x00086557c8b2c055,0x000033da338c1b7f}}, + {{0x0007513caa760974,0x0008f6c83906283c,0x0005af2c70a624fa,0x000bfd2b20afec71,0x0000b9699752ba78}, {0x00055ccd921d60e9,0x000febaeca132207,0x000ed93d49b944e0,0x000d2674819d515d,0x0000bbff86efdddf}}, + {{0x000413077adc612c,0x0008fbd803a06b34,0x0008805bda749652,0x0003ac5a1baaa76d,0x0000840390307034}, {0x0009f66175adff18,0x000b37d8c5b739f5,0x0009d75e30b26d7f,0x000cc22875f5ce52,0x00005efc7e9c1325}}, + {{0x0000b421ff6acd3a,0x0003b3dc69092195,0x000766127ffe7048,0x000b2b5f4cd0b228,0x0000bdbe608efb7d}, {0x00092285e1109e8d,0x000724645b5a837c,0x000818ed826147d2,0x000a357d78f592f7,0x0000394077fc247f}}, + {{0x000c2d0488c171a0,0x000a136852780fb9,0x000b1fa6aa78bfba,0x000ba7edfbe268d5,0x0000dceb8db2b7ea}, {0x00080899ae2b710d,0x0005d4449c96bf9e,0x000143a46efde7ae,0x000c1273b7716bcc,0x00007d3419593628}}, + {{0x000ec1c3b3f64c9d,0x00094e5edf3f508c,0x0004318d4e20bc0b,0x0004430a1deb852f,0x000020ebe0e2c3fa}, {0x0004ea773241ea3d,0x000b8e1a5f65370b,0x000681c6261f1511,0x000c2cc9a5e23d82,0x0000731e38472f54}}, + {{0x000f36e834459048,0x00092f45f9c02692,0x0007528b72e0ec46,0x000542105a3201c6,0x00008f77f3550e5e}, {0x0008d295864687c7,0x000db2df3562f67a,0x000bec39e23b92ea,0x000f8cec27014b9b,0x0000ef2f2270c0f0}}, + {{0x0009638546c4d8de,0x0003b2f246799735,0x000c8acd95f9c3fc,0x0004afb12e8beda8,0x0000c3a318e10663}, {0x0007f41c31cb264f,0x000e622113f28016,0x000afe1973db82f6,0x000282c155bcd2dc,0x0000ba1da5a33465}}, + {{0x0005b8eb212cf53b,0x000e48557c5fa042,0x000c4d56c4f2e512,0x00085111286ff925,0x0000b8a0feb9e26c}, {0x00070d2e7d6107e1,0x0004d76265aac28f,0x0001936b17ee0c44,0x0005eb2df277a41d,0x0000a556e3ffa959}}, + {{0x000bbf9e73056835,0x000eb7ef5be6258b,0x000c814c131eea5b,0x0000dcbdeb0e4a46,0x0000cee8449f7b73}, {0x00095c5a0182bde2,0x00077e27a6b4eab4,0x000e518caee759f8,0x0003f4a2cf6a6880,0x00005e80140114cf}}, + {{0x00041407e8d7a14b,0x0009e556f36a8fc4,0x000600044bb1ff3c,0x000e62ba84438514,0x0000a3f0c4b2451a}, {0x000c25b1f9af32a8,0x000631f2214bdfca,0x000b596ac01e0db8,0x000c07ce9a5bc2a4,0x00003927681826c2}}, + {{0x00032e77acaca28c,0x000707385b293ec8,0x0001eaf381bfeea5,0x000ccb468212e3fd,0x000013298312acf8}, {0x000f2db2aac9e59d,0x000ce661782ab909,0x0009b7a015748060,0x000625f5ab2632c7,0x0000a44c6c6d0017}}, + {{0x00000e8a7ea82f03,0x000db4299aaff26c,0x000d78be199cac80,0x0002cda66fe3b67e,0x000005f725f948d0}, {0x0001bc4623fb21b0,0x000e7a6319ad33ed,0x0005ffb3efa70533,0x00074117ab562dbe,0x0000637499456674}}, + {{0x0004ed65c46aa8e4,0x000368d063d169d4,0x000d17c362100d5d,0x0003b78b9727eaa2,0x0000c2bab1bcadd5}, {0x000e90c15426704a,0x00030837ebeaa084,0x000e477f8778afcd,0x000a8ac651f7017c,0x00000624998e6fb7}}, + {{0x0006828ed8a6e19d,0x00057189d9c7dc1e,0x0001c39bc33fc233,0x000914326f8fe267,0x000040c4cce8c6f9}, {0x00035bbf80e75ca0,0x000022adff2cafa1,0x00051ad9612c651a,0x000832c40a04bd4f,0x00004820109bbe4e}}, + {{0x000eb1a7f4c04cc9,0x000119404f843667,0x000ceb50a5955662,0x0005f9e1cdf6537e,0x000094a44a72b833}, {0x000f819dbeb9b690,0x0000eed4350dd7fa,0x00044bba2473c568,0x000bf3b6658466da,0x0000d1bc780872bd}}, + {{0x000f175a1962f910,0x0001ed58f5a7e535,0x00089a2336ed7e06,0x000413177aa4c020,0x0000dbcb03ae539b}, {0x000424ebb32e38e6,0x000f0806701ee3dc,0x0004be9ee6472e5e,0x0000097d47ff9881,0x0000b60cfff95ace}}, + {{0x000d9319ff91fe50,0x0000f0518eedb8d3,0x00082cb26039c480,0x00068d95c3763291,0x0000763a43482fc5}, {0x00004d5383e76ba9,0x000ff24e8197707c,0x000230de0ac98b92,0x000b7002bf7c8f91,0x00000876a01d0959}}, + {{0x00096f305968b809,0x0002789f73b9db6d,0x000c61e01380a091,0x0008c6eda70b83c2,0x00005fb8394e69b3}, {0x000651280edfe2f2,0x00096faeaf829a3c,0x000424bf88f726bb,0x0009706010a4a078,0x000096720442e844}}, +}, +{/* digit=1 [{1,2,3,..,}]*([2^7]*G) */ + {{0x000cb817a2ad62aa,0x00090c62ff5463c5,0x000ad9db57ef2b6b,0x0006169749bba4b3,0x0000d311f2ce6d5a}, {0x0008087c2ff3b6df,0x0002467834ffb77a,0x000d6b138b46feaf,0x00018808aa266d75,0x0000a38d321dc008}}, + {{0x0008ffa696946fc7,0x000849cba56d486d,0x000f35a1550fbc6d,0x00062c0e3d423e90,0x0000c3da19630dd9}, {0x000fdb03cfd5d8b3,0x0002589dfca5e673,0x0002305aa0704b7c,0x000e53c6ce581ff5,0x000099d49ebc14d5}}, + {{0x000496d6ec293cde,0x0006ae7051f5380a,0x00049140a733dbda,0x000bf5237e388db8,0x0000e4b32b13946d}, {0x000fda9cae368d14,0x0000bdb0b2f3b1c4,0x0003ac46e5001a7b,0x0006562df593742e,0x0000af675f279b3e}}, + {{0x0008110399492969,0x000aa61db1b544e3,0x0006eaff55b63827,0x00028fae5323ed20,0x000042370d3521f4}, {0x000af2ee0d985a17,0x000b0239846df2ca,0x0006312f8192cc64,0x0001080c0b8f47ae,0x0000dc61f9206620}}, + {{0x000fb5bc2da7de94,0x000ecff8d3beb830,0x0008a9641d0e643d,0x000501f1ee77ba18,0x0000e8aa3aafcf6d}, {0x00065329a49110f0,0x00062dd6b220f9fb,0x000c3ea5ad18317f,0x000c4a7e3ced4152,0x0000d296a147d579}}, + {{0x000a53eed4c37174,0x000efd0ed2a335d6,0x000543aa59f8240c,0x0004b44c0d4d05e5,0x00005d5bbfc1d33b}, {0x000cc73137fd28e3,0x000f973b3ffdfa04,0x000f51ef2862ac6e,0x0005a2103ff9f531,0x00004d5e0fcec73f}}, + {{0x000682008913f4f8,0x00016ac93d95f252,0x000a6b26cea20ed6,0x0007afd1ed38b46c,0x0000662dcbd6a432}, {0x000295c725d2aaaa,0x000eee52dcda6daf,0x00017daccbad2752,0x0002318210e7210b,0x000037f7913751e8}}, + {{0x00081e144cc3adda,0x0005e7be82cf4f70,0x000dd6472d5ffa1d,0x000862e9890b6c0e,0x0000da26e1aded17}, {0x000271563483caaf,0x00083f6077fd276f,0x000466e3ce6924cd,0x000d1e15a7fe980a,0x00001c794b1a1902}}, + {{0x000368882a8042c0,0x000fcd278298e521,0x00097a740d931cfa,0x00007c069a0ae0f5,0x0000adbb3f3eb591}, {0x000951e5eaa8eb87,0x0004a1b48e78983e,0x00003f2c5e663a8b,0x0001e1a631cc0d8a,0x0000577c11e81e27}}, + {{0x000385c08369a907,0x000aa90eb4f833b2,0x0008eac802990c59,0x000014119a6145c6,0x0000a786d629ec4a}, {0x000adbe20ac3a8dd,0x00008aba2d3033fa,0x000a4f56531a2178,0x000fba509d2742db,0x0000b2ce9e425aa0}}, + {{0x000334b168984df5,0x0006e38796388cef,0x0003720f0e81dce1,0x000beca6e6949c26,0x0000c56feb04593c}, {0x0005601fde58c84a,0x00068eccb3148bff,0x0009a8a7874e2411,0x0008681cf01b614c,0x0000233e35ef44c9}}, + {{0x0006bf38bd7aff18,0x000a9d81b146b315,0x00028a9151b5ee4c,0x00099dfba1ac41d6,0x0000f3a8f9d7d896}, {0x000b9c9a0748be7d,0x0004d92e621f7329,0x00010a8371d391c9,0x000435151e6b214d,0x0000255f53b1947b}}, + {{0x0009e04f1788ee3f,0x000eb86938a20766,0x0003a01c0c14f27a,0x0008079b47a334e9,0x0000f627439c9366}, {0x00085d8ca2a59655,0x000286e9b9b37a09,0x000f972e83d9a554,0x0002fd723eb80b4c,0x0000c1c33bb9fdf7}}, + {{0x00058d474a861082,0x000fce4c5d900c4a,0x0006d4c80f8048a8,0x000e60c3c7c924e8,0x00008c889de256a1}, {0x000662eb214a040f,0x000747e1034757e2,0x000ac748ae8c48e9,0x0006f19774286280,0x00001c24023086b0}}, + {{0x000d4c35f74040ab,0x00014ceac957ac2d,0x000c4ec23409aeb7,0x0006eb9fbad78255,0x0000359ed623a7b7}, {0x0004926ed6f4a607,0x000edb912de31274,0x000705a59e21e8d7,0x000c0e72575a59fc,0x00002f1d4df5d2db}}, + {{0x00024b9eb7926b83,0x00039dbe55093d2b,0x000dd640bbff88cb,0x000d45a0f399afe4,0x0000c5fe1305f76e}, {0x00062f43764fb3df,0x00074151b62d6f35,0x0009ce5f37b5af31,0x00090ee5bd0bc7d7,0x0000daf6b21dc668}}, + {{0x00067ec6063540cf,0x0001f5f9cb8f735c,0x00099c6ab50b259c,0x000c84c8734f9a3f,0x00008cc13d693a7b}, {0x000b305c52176597,0x0003dec12a5480c1,0x0001345fefe5364d,0x00097f4d87045e68,0x0000f8efeb1c82f8}}, + {{0x000f1e5d59233595,0x000d039b9fb0e8cb,0x000859b98db0cea9,0x000cc5bc5b34cf49,0x0000e583c56f4403}, {0x0001a2dd48185b77,0x000dfe52178711fc,0x000105b8bc93fbc7,0x0000c7d7e7a05805,0x0000b4d4d594b826}}, + {{0x00030b046eb842ab,0x0009cbdae56de339,0x000f7fdfc8e844a9,0x00017584ef3a9e13,0x00003768f83136ca}, {0x000f4e04e09e61c1,0x000190c7cddc2821,0x000945fcd414dc3a,0x000ff1c537943754,0x000051b6eefc3555}}, + {{0x000d6136339c0831,0x0005cfb64701b31b,0x0009604ab39ff815,0x0004426c3388d2e2,0x0000e19084bb6b10}, {0x00054c0eccd47ef9,0x0004ba5dfb3017cf,0x000daf9f68969338,0x000958d9d023fb47,0x0000222840c0d91d}}, + {{0x00008f5803bac625,0x000ce79bd45f4391,0x00063c5810b7dd91,0x00004f8651e827ca,0x0000c5d75f6a09c1}, {0x000c7381f2dc308f,0x000ee98454be7d5f,0x00017b03120faa7b,0x0002aba5374beea5,0x0000036b9b254269}}, + {{0x000610939842194e,0x000d69d05295c510,0x000b42ee0b7e2353,0x00011c1c8c1d5cef,0x000004884ebe8ce8}, {0x0005d817419f40e7,0x00023995c241f1f7,0x000c556465b0ac16,0x000f96a20921bbc4,0x000013520c2fd33c}}, + {{0x0005a5ce98c51002,0x000d0ddd0f5ab4a6,0x000a2e78b6cec871,0x000a5f051f0b7f9b,0x000024a8434ee3a2}, {0x0007f6125f5c46fd,0x000b78545ec02682,0x000bb5cdc6a22bed,0x00098e55ae5fa0b1,0x00006936830ccb9b}}, + {{0x0007fe891e5d7d3f,0x000683a076783202,0x000dfdd61f14b7d1,0x000f48088497b3c0,0x00007c2eec11a8c4}, {0x00073f43756e621a,0x000f7825b948aa55,0x000878572c013a23,0x0001837c03b34563,0x00000472beb053a4}}, + {{0x0002e270ac69a804,0x0005b51e54f6f422,0x000ffa59134096d2,0x00027ec0a648cb8f,0x0000e87acdca9b65}, {0x000e037e285ccb47,0x0003e0ddcf520575,0x0000ff719188089e,0x0003693a96c9a887,0x00004a56cd88fc7e}}, + {{0x0004ee21726931ae,0x00075660ecfd41d0,0x000818e180bbbb2c,0x000886c6ef6de524,0x0000421cc52c7d57}, {0x000d208bea87be65,0x000361cdd682f127,0x0009b63f716a475d,0x0003b64db1b68443,0x0000359b3dc40f11}}, + {{0x000f1de8bf06e31d,0x00040d383901dfcc,0x00017e7d21fdf8f4,0x000eee40775cad50,0x0000fc3a59828d11}, {0x000c8a0b1ecff106,0x000bc84005496ec9,0x0004f8d73ee6ed6c,0x000ab955ad7bae1b,0x00001b4f11e400aa}}, + {{0x000d69bd4eff2d71,0x00013288b60f7b32,0x000a1e72388ae677,0x000e8c059461b437,0x0000f3d4789670aa}, {0x00018c07f9871da6,0x00089635e2788691,0x000541dac35fbda7,0x00045f138f3641e1,0x0000794b13b20dae}}, + {{0x00064ac09cc0917d,0x0008f68540fd0650,0x00022767127c5372,0x000a033d2d4c8eef,0x000023a9f8171785}, {0x000952852650359a,0x0000d4a1acad98c5,0x00055bf5cfa09ad0,0x000083682d5a290b,0x000040f1c67e19b8}}, + {{0x000752edcc18770a,0x000ee825c3a53a5c,0x000b153ed4baf1f2,0x0007234bd63f7421,0x00002383e4852f64}, {0x000620a2646d19a8,0x000b83c83ffde7bf,0x0006be9f156cb44e,0x0005e92f7267c94f,0x0000b2dfd7c406bb}}, + {{0x00072f2a672c5c73,0x0007dd53c5e2b870,0x000435932eacb11c,0x00093bf2dac29dff,0x00007bdb99d74086}, {0x0002fb62899c20f7,0x0001d47ece24f6e6,0x000577ce33535d51,0x0005f28bdc6b88ff,0x000026693bd89057}}, + {{0x000b0e5ab4b35a24,0x0001b5eeaacf6772,0x0005b95801d8b600,0x0001da328f7ce479,0x0000a20ed2a81fb8}, {0x0005cd44fec01e63,0x000c77ff50ad9f68,0x0002d97fd3ed7ddc,0x0004f9160fd2640c,0x0000a2414271b82f}}, + {{0x000df2c6a8ea8208,0x000b722cc25417d1,0x000291426b2b50d3,0x000e3883856cbab7,0x00007fd26ae84f5e}, {0x00096cc02bee4ba6,0x0003a6820fd69cb6,0x00012e9855312180,0x000a0945dfc26902,0x000066f7ffa760f9}}, + {{0x000cd33bccd9617a,0x00041a7730a3c503,0x000db0786365dede,0x0003bbd98c63555d,0x00006c3200f9c9cd}, {0x000fb2ce5e35efd0,0x000b5555a1c1060f,0x0000b375199a4e25,0x000bf611d95375f7,0x0000a57354a160e1}}, + {{0x000ae4bf8e4b0651,0x00041e53022becb3,0x00092ed9607a834c,0x0004ec0cd300b386,0x00006a6f79271ee1}, {0x00063c66a8649edc,0x000dc69f3e148f10,0x000a7b3ecfbcdfcf,0x0002f06cfb97c100,0x0000ea49b3d3130c}}, + {{0x000044fe9d964888,0x000e0182a0c1462d,0x00091e9e94b53d52,0x000a0904b6ddd303,0x00000ab7b4931741}, {0x00015d427d3317fc,0x000a5a64671eec0e,0x0009c5b928dfc1dd,0x000330d3cc5d5fd4,0x0000995d53df674a}}, + {{0x00041ec090090ae0,0x000cedb06830302e,0x000c996902278a0c,0x0008da1d025932fb,0x0000c32fbd2b80d6}, {0x00046daf341a6c11,0x00090bef68a0d791,0x000774b3aae0ba13,0x0004d7b6b8a5638d,0x0000cf307bd980ba}}, + {{0x000bdc719803511c,0x000ac888c3bec033,0x000c6d05ea9f97b3,0x0009ea7d68aebc85,0x00003b88a9dd9391}, {0x0000748c48b0ee35,0x000bb7a746c12d30,0x0006d57f37506bc7,0x00091aac48437c6e,0x0000bd715881feaa}}, + {{0x0000408c1bc52257,0x000ab719226da4ed,0x0008d2d43d0b946d,0x00059aa09ecd6275,0x00005c8485a97517}, {0x0005f499ce4177a4,0x000e39c10c3db0b7,0x00067fcd74fa61a1,0x000fa88062d300a1,0x0000df3874cb50f0}}, + {{0x0002cf983dfedc91,0x00047d87631a29ae,0x00029c8d2f843713,0x0002729f57171174,0x00008d15867246d9}, {0x0003ecf69769bb7a,0x00062479ab828305,0x000b0f4b2c55eb85,0x000524bef7791c21,0x0000a5956badd491}}, + {{0x00096c29fe20ebae,0x000b052a5ad3407a,0x0001d9d89f27168b,0x00027963b60ab3bf,0x000045c51f0510e7}, {0x0005276099b42210,0x000c2557a159dfca,0x0000358958dc6407,0x000c320ead833591,0x0000a9db9579c55d}}, + {{0x00036d3df61bc767,0x000fcf778cdbe407,0x0006ea28f13a619b,0x0007b3fdd921a4c5,0x00006a524339fa64}, {0x0001891ac5bdc5d3,0x00028ac7dc012359,0x000df8453ff4a1a7,0x00065f6905e26162,0x0000ac045e0163b2}}, + {{0x000341bad53dba78,0x000c037b625a8a3f,0x000e311898ec269c,0x000120571a27823a,0x0000fb4f9a3d5e96}, {0x000f823ff9875cf9,0x0006cd442a9b804a,0x000c6267923224f5,0x000db08c4d3b9eec,0x00001da22fc30e7d}}, + {{0x000324d6c04a661c,0x00059e376d17a370,0x00044e3579710d3b,0x00001c2d8c98f030,0x0000364ebbf24227}, {0x0005d517733d61c0,0x0009cea826c3347f,0x000a25548d55644b,0x0000a780c6e0ad55,0x0000aa7641d84422}}, + {{0x000ec81318106601,0x0007ce4b40431438,0x0003e02739dfa650,0x000fb200b515d8cc,0x0000b6066dd38d8c}, {0x00045919c9efebdb,0x000ef21c1ff4d3b0,0x0007607d3425d4bd,0x000083afe5af19d5,0x0000bf773f804481}}, + {{0x000bd6994b03ed1e,0x0002834cc5468435,0x000e420cad9ad1de,0x0006dc4cf423fc00,0x0000ed26d8180309}, {0x0000be7a4db09d2e,0x000cb60622f7d7f6,0x00096c729f47f569,0x00071505925fd772,0x0000ff2db2706ca2}}, + {{0x000d014b913e759b,0x0005dff4de93a6fc,0x0002068e153da478,0x00052d64616d79c3,0x0000187d6657cdf3}, {0x000b6501dc90b596,0x00031daa1b26f7af,0x0000c0a848170e94,0x000dfa68e3bdd870,0x0000e8d345fc482b}}, + {{0x000bfa1c5c5ea50e,0x000b8796068184cf,0x000d50942d3baf14,0x000662463984030d,0x00004b7839d2716a}, {0x000f794e7de6dc08,0x0003e22aa7ced5f1,0x000acfeec5cd0f4d,0x000606d295f3f159,0x0000d933553153e0}}, + {{0x0008ec5776c57224,0x0001eb5f290cc7db,0x000f425a9dc467e6,0x000b7294297e704f,0x0000be924c14cf7b}, {0x000c5aea18921310,0x0003a705c9920d5d,0x000305ac58bf8a8e,0x0007a873a0b0647a,0x00000c9ca4e9a8c7}}, + {{0x000e80f83774bdde,0x0001a57344855dfe,0x0004a69a96313160,0x0000d6c1b524ae91,0x0000bc2ffb0b4e30}, {0x0003db77cfa46a5c,0x000e61653b5052c9,0x000bc580a71e6161,0x0002527574fc57a4,0x000009015dea1bc1}}, + {{0x00047b2d174d7aa6,0x000893a15d044b7b,0x000fa07ed4072d8e,0x000fb18eb7d47fd6,0x0000f2b9ffa4dbda}, {0x00016153760fe8aa,0x000f506c6c1318c5,0x000a2d0717a96e6b,0x0005cdad7a04100e,0x00001914e9babe2a}}, + {{0x000e357d8a3c5cf1,0x00031abb2b135726,0x000ae88dd1197ecc,0x000efe5c0d7f7f31,0x00005b20d1b0dbb3}, {0x000aa26705840398,0x000927dc9747cd06,0x00055d8152277c96,0x00032a3ca6958778,0x000099ea238d188b}}, + {{0x000228b760c1c9d4,0x00015b5c18da37d9,0x000f6dbc5c7efbb1,0x00005b3f0d1bc819,0x0000875384b47e69}, {0x0000baa3ba8cd86a,0x000b22905de0c7c5,0x000231952b0ce40f,0x000e25d08406737a,0x0000912a2636f43d}}, + {{0x000ddcceb5b76c16,0x0004c6fc0ab49c38,0x0002c269f746f528,0x000620f2a63a50d6,0x00000049c55f9458}, {0x0008f823c2f7c9eb,0x0002e17d5cf3e7f4,0x00001f4696bd9904,0x000fe03b1317a887,0x0000d3fe2ee4a449}}, + {{0x00079ca12ef3d36a,0x000b9e7ea5de421e,0x000ff36f79ee3c36,0x000228448198b5cd,0x0000ff4f96866b82}, {0x0009dd0c47adb7e5,0x0002b32e7dfa15e1,0x000ae026a45699b2,0x000f4cf0680c8b1f,0x0000a347a48a50db}}, + {{0x000533b3cef0d7d8,0x0007abbb4381e652,0x00080f500d94f7b1,0x000bfb038752be0e,0x0000e6e24891e9c9}, {0x000169716caca6a2,0x000818531ad9c975,0x00051ade1866c49d,0x000407a917e23971,0x0000d016ec18037c}}, + {{0x000ccc900eac3f90,0x0000e2ed4748a407,0x000c98e0d835f628,0x0002ebcc54c3471c,0x0000e969937dcb57}, {0x000c8e88f30c9cbf,0x0004473c46611b16,0x000502caba606ae7,0x00064e57aa689b35,0x000089014af3d9bb}}, + {{0x0006a9c31c71f7be,0x0002496ffe5c202f,0x000cec3a301f95aa,0x000a447fc0601453,0x0000b9912383f498}, {0x000935e5d91ba877,0x00009b564a19ae9a,0x000d44e69c6ac628,0x000d451a8fe81c3b,0x0000c8b46800dd11}}, + {{0x000251fea5b8e69b,0x000d15b75fbcf772,0x0007ff0e5aeecb3b,0x0001306aca333188,0x0000e5d49ffc9f0a}, {0x00013aae5c8646f1,0x000810e19980582c,0x000abbd94dbaa12e,0x0006637f40f31af7,0x0000f13f5a82dfc7}}, + {{0x000f1eeaceb4fc02,0x00023e6f0f425d81,0x0001370c83625600,0x0005892b67d6d775,0x0000608b69823e80}, {0x000d2fc05268301b,0x000890309212cfc0,0x000d0e1c2a6943d3,0x000c75692a90c21f,0x0000209f113e7f1d}}, + {{0x0005e0697bf1298a,0x000f819d639eefcc,0x0001e8c6fcbdb672,0x0002e53009c116b8,0x00003ffdde3ba7ce}, {0x000baaaa914d3ba5,0x000f38df85eec53f,0x000ee0751836d500,0x0006fd898dc71b66,0x0000a3d7005c1451}}, + {{0x000634d39eedbbab,0x00075455a46d21d3,0x000d7eb0c35cd2e6,0x000b3e18cafe65f9,0x0000da3ce9eb0cef}, {0x0007a602c9cf7a47,0x00040bcb8773ddc1,0x0007548df01572ee,0x0000e3392b2b018c,0x000032fd30a18460}}, + {{0x00009c716543a402,0x0006dede3c6ce221,0x00024e6149acafd3,0x000ca0db20685268,0x0000a4544a9fa25d}, {0x000526291d60b063,0x0008f87535452598,0x000f13b27281b7be,0x000eb4bc667b1a90,0x00003a83affc40e2}}, + {{0x0009862d5d721d5b,0x0002abd3a1828000,0x000a2cda40c3357a,0x00008477f3a83b7a,0x000058ae74fa6f83}, {0x0001a812e6dad6be,0x0001143d6c5b2a91,0x00096c4d8de28605,0x00024d6bdccc41f9,0x00007312ec0eae1e}}, +}, +{/* digit=2 [{1,2,3,..,}]*([2^14]*G) */ + {{0x00012e76e6485b37,0x000b071c52f8f8d1,0x0004a2f6d4d3e24d,0x000550d8e3ee4168,0x0000161957d91d95}, {0x0001283cdb12a6c3,0x0001fe50e1641963,0x00066cc73bf3fa88,0x000c38c6254b6331,0x0000aefa7aedee8c}}, + {{0x000fe623b36f9fd2,0x0003dde19fc079b0,0x0008482ef26543b2,0x000824f36e64a095,0x00003f63771bb095}, {0x000d596b6a1142e5,0x0005e35aac0b14cf,0x000081dd55ea6aac,0x00012a36a0e8bdf3,0x0000fb89d79503dc}}, + {{0x000c33af72e34d46,0x000fb10eec35f615,0x000dea34e0bd9ea3,0x000698bc12bc5bc1,0x000086584c9a9ae4}, {0x00095d38c97b942f,0x00095e5c756213ad,0x000737f894609561,0x0008b5ae94a4aef2,0x000057594c7271c7}}, + {{0x00065fce3779ee34,0x000d7d495d9e0f01,0x000284e7ae00e7f9,0x000218dfa4efa220,0x0000564bade87ac6}, {0x000312ac4708e8e4,0x000b671e9adf90e6,0x000684b9f4f5725f,0x000415a95f55ae3d,0x00007f7ccb15e94b}}, + {{0x000851b8d9465810,0x0003bdf4a0127322,0x00084dae0f0d1313,0x000c6da3510f6965,0x00003a7c1713c9f6}, {0x0007f38e475381a1,0x0002758233345be9,0x000e17ddaca1ba42,0x000c0fe83cc5c70b,0x000058b14941b918}}, + {{0x00077e5522e6b693,0x00038bcd6c18da3a,0x00024fd5669c908c,0x0003f6ef1b9e48d9,0x00007c64e36da4bb}, {0x000dbdfee478d7d1,0x000bf193f7a05a4f,0x000cd16dfba75c8b,0x00015174bc1e8456,0x0000fb08f0856fad}}, + {{0x000abf9842e9f302,0x000f3eab83af8a7c,0x0007f2a6aa331d4b,0x0000e3b272cfba01,0x00007560abca3aba}, {0x00033870e3a6b750,0x00026b9f50f594b8,0x000fdf6d025c6aea,0x000514803d691db5,0x00003b77509e6333}}, + {{0x000890361a341c13,0x000fdcfd61423617,0x00033316c3604dc5,0x000921d22295eb85,0x0000dbde4ac74af2}, {0x000fc5d1c7eef696,0x0005714f4fa1898a,0x0003c21ca5889680,0x00030aa500216020,0x0000f0d1f30a0ef7}}, + {{0x00044d4196224f85,0x0004e74d079d8e8c,0x00048f12375a4ab9,0x000ad829085ecc7d,0x00006f04d316bf65}, {0x000bf1cbda602b24,0x0002b9612c69e220,0x0004fd06b73ee174,0x0000d136008fc808,0x0000000efa031138}}, + {{0x0001b4b12cfe2978,0x000ba92f74e54820,0x000e874e83eee129,0x000c4161fe114ec9,0x000099b055d12c5f}, {0x0007a643a39c8cf8,0x000df8963cc94e47,0x00033f86382f09ef,0x000c62efd3fd8fd3,0x00005132b2b5c949}}, + {{0x000a3ab516eb17bb,0x000f22c7372b7e06,0x000896da673bec06,0x00040f34f74f55ba,0x0000b4afef93e9eb}, {0x000bec8e61d66b0f,0x0003ff29300b2d75,0x0006baa5a02bda4b,0x00043f9bbaa8de02,0x0000f54befe907f4}}, + {{0x0008b1dbe7a2af37,0x0008dfb74a72bd9b,0x000879697ec51caa,0x0007d549937a4b63,0x0000c9a9d215c268}, {0x000e44f6ef5f0145,0x0002990c69001773,0x00042161e8abcf41,0x000f29e87bd02281,0x000003937564cb6f}}, + {{0x000fd56ed6def63f,0x00018d53106c9813,0x0001f7ac153cf648,0x000faea91a35bd43,0x00001e274de53e65}, {0x000fa3c44cc78808,0x000afc256981f63f,0x000a420e0411a426,0x0008fe3698b9fd93,0x00009fdddc12e53f}}, + {{0x000072232398baaa,0x0001bcfca031766e,0x000029cf2205fee4,0x00090d049f53417a,0x000088c68b8e0238}, {0x00050417337aaa82,0x000ceeb384f4bc27,0x000aba92f9ed364a,0x000a88c0816f8529,0x0000e9e194124e38}}, + {{0x000f44a3dafd2d50,0x000597ed98d857ee,0x00007f9b135d1fae,0x0005c650628c0923,0x00009d84aaed6cba}, {0x0001bc788aaa6917,0x00093fe6cb036707,0x00078ac012dea57a,0x000a516fe11bb43d,0x0000286418cefd7a}}, + {{0x000770977f7195ad,0x0006ddeb838ffabf,0x0004f012d8ec8616,0x000b3f1a1285a8bb,0x000068835046a3ea}, {0x00024f8309004c28,0x000593ffe95eee5d,0x000223ea4a96e4b7,0x000a528cdffe12bd,0x0000f5c2ee636739}}, + {{0x000aaa5dd968198b,0x0001c2413a6c5cb4,0x00036d903fa131c5,0x000d8da3d46a9095,0x0000270f0d3f8606}, {0x0007564a053a3bc5,0x0006ca86caef518c,0x000b5efd0088254b,0x00045d63ba8cb40a,0x0000c59900e96059}}, + {{0x000e1dda1887395c,0x0005d32a65deecac,0x000a9552940960f3,0x000a35d611ff5c3a,0x000058215b13c1e5}, {0x0009b58f0e1a5240,0x000bf590dfb8d48c,0x000d95662b406856,0x000f82c7605e049c,0x0000dd036eea33ec}}, + {{0x00071acc33156b3e,0x00096a80172ea501,0x000dc8eeff09d24e,0x0004ed6e1f72c676,0x000060caadd3e3d4}, {0x000f8a6979b1d8f6,0x000c37788d26006e,0x0006feec060908a1,0x00094e0e08f95b26,0x000018427c282e8c}}, + {{0x000333959145a65f,0x00080a4063373d61,0x0008a52a0cd9bc36,0x00058f92d11be32d,0x00006877b2887a1c}, {0x000819bf5cbdb258,0x00085e090249837a,0x000990e5f2a4fd1d,0x00011ae22a7de774,0x000040fa5a0f9455}}, + {{0x00074be6558842d7,0x0003a7f3d0a630b9,0x00042e46d70df8c6,0x0008230c80352075,0x0000251fe8054ecc}, {0x00034cb5e9aac9a8,0x000470045d71e591,0x000cb1d4e11bb093,0x000c90d3e5d9b5db,0x0000d97a90612def}}, + {{0x00093277946d3f97,0x000397472273fe28,0x000b6ae86e132bd2,0x0000677eeb510c1e,0x000077708c660595}, {0x000c8cd1297029e8,0x000c3bf9305e18e2,0x00085d6d92c61095,0x0005306466c2586b,0x0000ac06c375a1ea}}, + {{0x000dc39a13046683,0x0004d7f89606a365,0x000c7228de4a9c88,0x00030335a4898fac,0x0000e2347ffb4ca8}, {0x000fb77ea7d23a38,0x000ce72a71cda5f6,0x0006a44d32fac257,0x000d79e908bef87e,0x0000ff87567091d3}}, + {{0x00090b36b0cf82eb,0x00057615b5e7e58e,0x0009c145a6438d24,0x0001ca57b1f8fc66,0x00000d8b2dae6f1e}, {0x000dadbd9184c4d2,0x0005d93d997654d5,0x000147d473dbb18d,0x000608ea3e0f56d1,0x0000afa8c8dc0a48}}, + {{0x00053e8bc36742c8,0x000e6ea0ed902753,0x000477b00898f427,0x0001e2a6f4947e3e,0x0000ad8848ab0874}, {0x0003c38d74a2a464,0x00095ba17ba26c70,0x000b9a9e45e3e05a,0x0006ec81fa6f664a,0x000074a2d9a7841d}}, + {{0x00039ad653ae3263,0x000a774cbb438712,0x000d4c08314bcf72,0x0004af5737650e20,0x0000df86536410ed}, {0x0006fe7b53ca5557,0x000d3bd5d538d2d8,0x000d38468688cb00,0x0007b65f81bda31a,0x0000ccfe3cd60116}}, + {{0x00047e06c4c1fe61,0x000a198bbb79cf4f,0x000d45a14557e1f1,0x0007c4e93b974f30,0x000074a1d2d1baf9}, {0x0003b30c51fbf539,0x00015e68b2257a00,0x0000f4173d894099,0x000152cb0aa7b71c,0x000075797ca320a7}}, + {{0x0008c07e3533d77b,0x00097e341c9926e0,0x0002dc4edd7222e6,0x000cf7ed60ec3d8d,0x0000dfe0d902c476}, {0x0009ab61d056605f,0x000596a8551f1fe5,0x000fb8d8ca9ea9df,0x000b0f9489941e47,0x0000eb874ec3a7f1}}, + {{0x000ea867ee0d98fd,0x000b0bf61864fe5f,0x000c031d4201ad34,0x00082175d8fe4737,0x00005f49faf495f0}, {0x000b291c7f4a40c8,0x0000f30ddd92db0f,0x000d769872e69d9c,0x000b86f54e105449,0x0000a24911df662d}}, + {{0x000181060a716760,0x0008f66a8ad161fc,0x00017231ee852d1a,0x00011f172bbd6564,0x0000d6de7bd3babb}, {0x0006f88c8e347f89,0x00070bd99cc36fde,0x0000769501c58754,0x0003b9e8e54ed034,0x00007f0f335096e8}}, + {{0x000e1ce4924867ad,0x000f90b84917e4db,0x000b09a79bd5f51a,0x0003d7675300403c,0x0000b3fe0f9cf174}, {0x00094d8556fa9db6,0x0008c3412fbfed78,0x0007b9291fa26216,0x000233f63be0dbba,0x0000ca8b8c06c9fb}}, + {{0x0006aa9bd7638026,0x000005303da1ed40,0x000e62ec4c21486a,0x00033e01ae291ec7,0x000022a04933f993}, {0x0000c9dbb7a8ee0d,0x000b9c01aedb7fd8,0x000be74ecdc2ed3b,0x00071e65c35a1208,0x0000540cb1b169f6}}, + {{0x000ed4ecf84f6c7a,0x000b8d090f43d16c,0x000239db48561fb9,0x000d93de693d796f,0x0000736f92917bd0}, {0x000d9292c1950ee1,0x000346dc11b307b4,0x0006a878eda17754,0x00008a95dfbbaa7a,0x0000c70cb295decb}}, + {{0x0008c8b6f0f7c500,0x0008854dcc6dfba2,0x000b78642a8eba2b,0x000adf5ff8e89a36,0x000070c1c8ef6873}, {0x000c3716484d2e49,0x000e7d414129bbd3,0x000d93b0bfb78318,0x0007f69621a39c6a,0x000079d74c339e91}}, + {{0x000564761fb04286,0x000a5ee624d4fc19,0x000ae86fd4d78954,0x000b13594896e0b8,0x0000667ac0d291c8}, {0x000051243bcf832f,0x0006b00101379f18,0x000ba8aa7fbadf8b,0x000e84d69b4089b3,0x0000ac4baced687c}}, + {{0x000088d977eab403,0x0005f760b3909164,0x0000dd55351f4c5b,0x0001c9a238238f34,0x000058566c40b1d3}, {0x000d69e5068f5ff5,0x000c8aff6b063a5a,0x000debff0f31435f,0x0003315e549a5bd6,0x00009e5f0b7c5e01}}, + {{0x0002fb898559acf1,0x000e3db79b505d49,0x0009f66aa96018c2,0x00014f45f4a48f60,0x0000943b3af5900a}, {0x00096df15a40d39b,0x00039c20f7c5c224,0x00098404cb2a4468,0x000b76c6a35afa3b,0x0000ec75726af5d1}}, + {{0x000a163bea064441,0x0002e724b6f2b67a,0x00038c8ab27e95bb,0x000e172c20e3e9d2,0x0000213754eedd6a}, {0x0001020716e0f742,0x000edfc095c28c43,0x000ac29326679c82,0x000a760eb3adf4d0,0x0000cc970d321bb7}}, + {{0x0001f2f740f0e66f,0x000a3b6b23cc70c7,0x0000a8bd7545c616,0x0007215528cfcbb4,0x0000f8396341ab27}, {0x00027d9025ac99a0,0x00002b63e33b0491,0x000d84519d314d4a,0x0004bac8c310e728,0x0000fcb8983b3bc8}}, + {{0x000226138634818c,0x0003f44c2e0b2cc5,0x000dfdba3501814f,0x00018c37e181aa54,0x0000fd58ff1a7597}, {0x000db14d3b507a84,0x000e850bdad8f90c,0x000e5f9aa57bd478,0x000854e9c197e250,0x0000db6eef9240bc}}, + {{0x000f21ad1fc0654d,0x0002b1269d732cc8,0x0007f49f9c71cc96,0x0007931cfbb20407,0x0000de925729a56b}, {0x000d6a3f97ad8f78,0x000f124de3bd9abe,0x00040a800e6c19d3,0x000f070dce92f4a1,0x00005f44d1e9337a}}, + {{0x000c08b09d64c52b,0x000f45df97495953,0x000735f7da1b5e49,0x00076a836a8fb852,0x0000332b6dc4add6}, {0x00088a0b4511aa47,0x00026bd5cc55558b,0x000cd52bd0978875,0x00096aa6b43b9cd8,0x0000f0bc5a132a26}}, + {{0x00012d4c11f61ef0,0x00043a83e79e146e,0x000bfca159ce1075,0x00053f08ec73d96c,0x00009ff29ad5b496}, {0x00072bde7da946e1,0x000bde80a4f2e31b,0x000598ce4ebf9eb3,0x000e80c1aabd0817,0x00008b5fef463f37}}, + {{0x000cdd35958cd79a,0x00047d373114d5d5,0x0009357263580a1b,0x000760036e4c91fa,0x000038c534e8f20d}, {0x000e40a2ff5845be,0x000cdd78177f7088,0x0007f9920e5bb40b,0x00005c6f06a7a885,0x00003cc3e51c968f}}, + {{0x000b7fee5682d265,0x000f5ec7f87c1d68,0x0001951ab5206f76,0x000719f111053004,0x00008ec52c224b5a}, {0x0008f990f75cf9ae,0x000eda82d0d5f348,0x0008895abf411951,0x000b1347ee75be61,0x0000ae060d54d8aa}}, + {{0x000df737fb54dc25,0x000ad59636499ae1,0x0000550811f3e391,0x0009bcd42ec32afe,0x0000bd450efd491c}, {0x000fc67981eb389e,0x00075a0550d5367e,0x0003ce75ced7e192,0x000522562e776bab,0x0000890e308ff24c}}, + {{0x000b682feccef760,0x00058bba6d92b961,0x0002375c48b8e11f,0x000cfa8f2ccc4c2b,0x0000d7f7a52e2f86}, {0x000d30a9efe5633d,0x000a8451f934fd94,0x0004e6a002d8d246,0x000c4f5234c6e324,0x0000e2b5b0eadec8}}, + {{0x0003c5abf776f5b8,0x0000e0357b052ce5,0x000bf3f7a6f72407,0x000a9f3259371771,0x00007d2501cc40c4}, {0x00052e187b053405,0x0007d1624c324405,0x000facddbb7bf7cc,0x0007eef155a6ce22,0x0000a4228cbd8983}}, + {{0x000d6d6fd4fd6717,0x000e52daa10eef87,0x000c0eb96a233687,0x000be60562224403,0x0000632d184fbf19}, {0x000f8e940735ff4f,0x0002d00931f105d0,0x000fe3f183a3e6e1,0x00020641ccde6ada,0x0000381366bbfe51}}, + {{0x00022a960167d923,0x00084529f18c24c2,0x00053b11462f9d6f,0x000043112397c003,0x000034d89dd1f808}, {0x00063ba2a4383ce6,0x0006fcf92ba0d9ec,0x000be74c0cec8e93,0x0003919b8b4288c8,0x00007d6912f705d4}}, + {{0x0006c461b9131492,0x000f1a4e02da7b99,0x0002de59436aae2e,0x000545968aa00397,0x000084ec70d6ec6d}, {0x000b2d061391d54e,0x00061e114e92f3d2,0x000482dff69c5d5d,0x0003c4de0f00b5b4,0x00001596fa6d5bf3}}, + {{0x0005b5696a71cba9,0x00026dcadeb71059,0x000cd8471944938b,0x000fe11282da4cfc,0x00008ec05f39d37b}, {0x000ce1b0698304a7,0x0003b1bdf79be171,0x00021dec12d69144,0x000f7160cd3b741b,0x000012ecd8b86a15}}, + {{0x00000a700fd56e19,0x000269527c188d4c,0x0003e42e102ec969,0x000e0991c449374a,0x0000176fbaba392a}, {0x000f1ba44b7b6187,0x000981de491c8726,0x000b582c0b4d7aae,0x000a3a891df7b907,0x0000e116c315f60a}}, + {{0x0000f81466265d74,0x00020df7adf09927,0x000738f7fb15b6fe,0x000f95be33b2d3f9,0x00008553aba16d70}, {0x0002ac8c21e94db8,0x000d3dc0bbee2cc7,0x00040478f795ac38,0x000e548a1be4492e,0x00001bd3394852bd}}, + {{0x000dbe956b3c4f2a,0x000ef04177cc63c8,0x000010fc1017a99c,0x000b20f47bbddb4d,0x0000cf9b00c5b2c9}, {0x000bc8d47173611a,0x00080c7d756f2970,0x000d541a21a4cbe0,0x000f4366d9f4aa67,0x00003e8b689f9c2c}}, + {{0x00066da4d88f1dda,0x0004dad35deaaad0,0x00078ca67c604f16,0x000e05dedc072044,0x000010dfae15a02c}, {0x0001c76af36f4e47,0x00023f3f8f48eceb,0x000c8a68c994b229,0x000c9d4f9ed77b77,0x00004f544eac1744}}, + {{0x0005bb98113a7572,0x00046a9885e482d0,0x000a7865f4ef2d2b,0x000a51fe332be51a,0x00002b76b18490d1}, {0x0002310443516830,0x0006a3f22840308a,0x0001ed9479d86189,0x0003bf5959ddcd84,0x0000def0c94154b7}}, + {{0x00054174c7c15e09,0x0001aa277c32f010,0x000dccf5f539bfb0,0x0003bd5699268ef9,0x0000f5796a59247a}, {0x0009de84f157269f,0x00048a30196b8b83,0x0008a5a91c825c1e,0x000fe57ef0aabcdc,0x00004a8ce6d398b7}}, + {{0x00035a770cbac78d,0x000b26b239581cce,0x0006cb01183488e9,0x0006573341a070d7,0x0000a6c9d077e1b2}, {0x000fb30dd648c523,0x000c22fb9fd1b701,0x000563086994ca02,0x000baad69331176f,0x0000d2b810047856}}, + {{0x00048c85963a46e3,0x0005799e61c7e89f,0x0008517b4658ab87,0x0006563e296f874b,0x00006c4fcdd2c1bc}, {0x00027a1a3906def5,0x000712418945de52,0x000d96cde9fe95f5,0x0000ddd0c91e81fd,0x0000adbe47f2a448}}, + {{0x000370f396de2b60,0x000bf0ecc7bda009,0x0001d067298583d4,0x000984f44f6b57e5,0x00003d6b078556b1}, {0x000dd93b0b64912e,0x000335687b0927db,0x000ec20a99b3a343,0x00087b2dba646151,0x0000c93db80df281}}, + {{0x0008c2466e48bdd8,0x000891ccd78e00ff,0x0002506032514f2f,0x0001566ba11f4fe1,0x0000a22cd41a43fa}, {0x0008df4b283e4c6c,0x0008cb39783fa4e5,0x00025980978c2985,0x0007dc9235aee2a5,0x000016284b5ce022}}, + {{0x00079161338830dc,0x000b12123fcaa5f5,0x000c546f86d4b8a6,0x000d35636ea68af9,0x00001d36874ba608}, {0x000e4958d436d13c,0x0000cfb080afcd76,0x000ad3fb5d4d9c22,0x0002dfa65c1728e8,0x0000f1ebe4e73d57}}, + {{0x000746a584c5e205,0x000169dc7035a7a8,0x000548c9b267e4ea,0x0002f3093a15cfb9,0x0000e6e21359bd01}, {0x000cc6a8c8f936e6,0x000455c241dcdf31,0x0005efb868af84d0,0x0002cb03990a6f34,0x0000fef4e6219b96}}, +}, +{/* digit=3 [{1,2,3,..,}]*([2^21]*G) */ + {{0x0008f09257226088,0x000a931cf5c6f636,0x000b4f7ac131260d,0x000828c0eb353bfa,0x00005c78880b7eee}, {0x00081ffc3bdf24eb,0x000b45c3c5a84c15,0x0004e6f405bff75c,0x0000c985e8c83fa1,0x000081d1c0fb295e}}, + {{0x0007cc8f43a730f8,0x000bb3ab590efcde,0x00003240be89b6f3,0x0005db4823f529ad,0x00002b79aff18bea}, {0x0002856962fe5de3,0x000b30c591f3568f,0x00028a8580c590ad,0x000f4befc74a144a,0x0000b662498e3203}}, + {{0x000cf0d6c39765a2,0x000d8c3cca0b91e3,0x000953b50a2db3ac,0x000f1a088f2f08cb,0x0000414582cef43c}, {0x0008bbc60eee9a8a,0x0001d29aa0428dec,0x00032f5d554c79f0,0x00015f381cd5ec65,0x0000672303b6f82e}}, + {{0x000afa8719c05631,0x000cac5fc79f376a,0x000750cd3cd8ad2d,0x00008e203fdb9fcb,0x00004ff052f5418b}, {0x00084cf3e2d65208,0x0007944ed509f750,0x000f25b987ebdf0f,0x000837743bf0f2d3,0x00006ad71d02354d}}, + {{0x000fe9226f435728,0x000add824758b827,0x0009094c1dfd3ab5,0x000d67b15dd23a53,0x00005c0e37ae6623}, {0x00079727be19ae06,0x00067f0d36b5575c,0x000b1ff7e616a339,0x00045341ebb3c826,0x000035b9485740ad}}, + {{0x0003cdada430c0b9,0x000daa96dac692bf,0x000ac326a4702850,0x0005e4391cf0a515,0x00005de4f4a3b8c2}, {0x000ad09e265c17ce,0x0003287b3881b01b,0x000fac5ca24e4546,0x0007a5f43e583ce1,0x000017cb3194ead9}}, + {{0x000924374dcec468,0x000cd4c2b73f6cc3,0x0006cd99c33cfc02,0x000f8902917844f2,0x0000819dd9651773}, {0x0002aa60871f4274,0x0005b6f01c340957,0x000f1f5af8e0cf36,0x000e503fa52988bf,0x0000eb357eb275e8}}, + {{0x000c8c4868af75df,0x000e55c8c7ead9d0,0x00081ecb0d7325cf,0x0004ecbb471996cc,0x0000f5d55f451182}, {0x00045411977a0ee8,0x0004f22038c6be31,0x0004bb4955085c4c,0x000081ad5335bff9,0x000094ad8a748e2a}}, + {{0x0002341ada354383,0x0008d49b8c4e5c3e,0x00017cf34f4a9fc8,0x000e108eeb355a9f,0x0000f311e0e9c91f}, {0x000003892ab98919,0x0000ae8ce9a9c2d2,0x000c53bee257bdcc,0x0004398b2d978988,0x000027ce89b5dba1}}, + {{0x0002cca523db2808,0x0009d0d43783b0a3,0x00097d16f5c889f8,0x0002e7d03e04b348,0x0000cdb6e7888f5f}, {0x0001cf0179c8e744,0x000208211d606ab9,0x000851200d8874e5,0x00040ab948d4d5ea,0x0000076d41f26f98}}, + {{0x000263c47b517ea8,0x000cb0685e5ec20e,0x0000631a079a448f,0x000346655f6f78f9,0x00008a790b2279e6}, {0x0000c7d80969fe8f,0x000351491bb96216,0x00095752654f92fd,0x000e7ab6645c235c,0x000044cc5afaea3c}}, + {{0x00083278b1e68b79,0x0009a03f29d3f762,0x000d03ecbc731ad7,0x0007a76e5a9ca957,0x00006c0d50cd1bc9}, {0x0009fe79b4f7f248,0x0007bd9967efc466,0x0002c208dfdd781d,0x000cb2f892c7c35d,0x0000bf64f7d2e545}}, + {{0x000862c467be912a,0x0009273d30ccc01f,0x000b83ec7f4c85ee,0x000cf87fa6f4be6a,0x000007a3c1cee3e3}, {0x000ef450c00beb33,0x0002d00d4c3e87f8,0x00008bf5b30e2c2b,0x000f51eaa00b94fe,0x00002c133aac224e}}, + {{0x00016bb32e5685df,0x000868e6f54438df,0x000c5ebc668a9e06,0x00035e595aaff7cd,0x0000894a646278b1}, {0x000350a09e27ecf4,0x000e18f7179df316,0x0007861baeced201,0x000e2deeec273ce9,0x00007ec2caf1693b}}, + {{0x00097c4f68367ced,0x000aee5a5755fa4c,0x00098a979e4f47d0,0x000c7c47de815db2,0x00007eca65a9177d}, {0x000bb7149ded0a33,0x0004cb34d3c520fd,0x000858a334cb2aad,0x00040efcf31d2860,0x0000b6873efd24aa}}, + {{0x00034b22c11bb373,0x000dbd4c74a35402,0x000c5f25d2d0366d,0x000142c9a968daee,0x0000660106897b63}, {0x0006d2c68d7b6d44,0x0008cc84294207cd,0x00068b1eea8f74f0,0x000ee4a275140477,0x0000b5f7e8a3e62a}}, + {{0x000717789070d267,0x000e6d1c8bc7c6a7,0x0009e1f17a1f28e4,0x0008e07a5f4f0646,0x00008fc242b6bdb7}, {0x000c5928b0588f1b,0x000c6535921ec9c7,0x000e5ae35b6b7a0f,0x0008641c5bdb91bd,0x000042c485ec2ff1}}, + {{0x0003e13dbab98aa3,0x000b717b1024a111,0x000462d3ade9d469,0x00078cf3f48b37c0,0x0000752e537ac5c0}, {0x0006add15544eb9e,0x0006a0fba279e3a8,0x0002001b5f013aea,0x0001aaab5bb76cf2,0x0000617ba15d0289}}, + {{0x00082a6936219d36,0x00044e51cb19d391,0x00007a74c5ce1f19,0x0001bc678f8598bf,0x0000d7158f282cbf}, {0x0006b21e300ce18d,0x000f5d11275d3b84,0x000239b9b35fba62,0x00093f8fe25c36a0,0x00008beb35eaf05d}}, + {{0x0002bb01f7e320d9,0x0003dda320ea4db0,0x0001389a30641c36,0x000e3cdd95fa5d82,0x000026997491fcd8}, {0x000ef17ceb6c1434,0x00019933762b316f,0x0008b17f867fcb84,0x0008217b837e3511,0x0000b92552fdfd24}}, + {{0x000c70e46aca7930,0x0004e579311bae6b,0x00002f7161cf0b0e,0x000e4d8dc631be58,0x000099bdc6fbddbe}, {0x0002bb20caf8b051,0x000a62d63df2cc35,0x000c4f408f74d505,0x000b2da9876d4b91,0x0000ce18473ae229}}, + {{0x000759783abdb4a0,0x0006dee84b184950,0x0009e67dc850fbcb,0x0006d86325236e60,0x00004d831d99336c}, {0x000ae3bfa12d45da,0x000da746e2468dea,0x000f5f31ee425f8c,0x0003b6e004c17524,0x0000ca16d904d62c}}, + {{0x0005a6a9152f9347,0x000d7d0e12c10dc1,0x000477dacf1235e5,0x000006533c06ecda,0x00006be873322ea0}, {0x00078310c0cd3130,0x0003a614260dcf3f,0x000b22d153c52455,0x000a2031a756f8ca,0x00003ee10d177827}}, + {{0x00059b21994ef202,0x0009438ae318d1e0,0x0006990102a653b6,0x00084a50d5eb582f,0x000079739f729f5f}, {0x000663c8b799336a,0x000c803c37eb5da4,0x000dbfb2dfdfdf14,0x000f9a92d8a9dca1,0x0000b40cff117d48}}, + {{0x000b383d20b42d5b,0x000eef78845fc018,0x000ba9df0f9a810e,0x000df890af3753bd,0x000090bdcfcc31df}, {0x0000591f01ab782d,0x00009af12a881872,0x000c14401c823f21,0x000be2d51b80f30d,0x0000e248f78cb2df}}, + {{0x00044e50cafe751d,0x000c04dcd221ef5a,0x00085402473997c9,0x0004ba62fd86d1de,0x00005b53add709b8}, {0x0007a11dcedd8d12,0x000854b32c84008d,0x000dde8b1406bd1c,0x00032f3d4472ff05,0x0000e25f2ce1ce2b}}, + {{0x000dd5e29dfc254e,0x00054b98b267bec0,0x0002df2ad4455fcf,0x0003962b4d43a5c7,0x0000a70e6bf28a75}, {0x00061695820f3bf6,0x000d3e37f68f2aad,0x000e5ac83f410d2d,0x000eec10fb7dba7b,0x000036bb64596ec3}}, + {{0x0004ea39754e21c0,0x00068d63c3732710,0x00009db9abc87a3e,0x000da7483351d741,0x0000fa724e360134}, {0x0004c29b0720b169,0x000276aceead9ff4,0x0006929a62dd0cf1,0x00092ac942758ce2,0x00006c5db934766a}}, + {{0x000d4c05f18395e0,0x00041f80d032cec7,0x00086075bd3f2274,0x0002db7a68b37acb,0x000074764ddafef9}, {0x000e9507bc7f3893,0x000089756460ded1,0x000a48157c580c85,0x0007b37eeec2a47d,0x0000f0b4e7fb2c58}}, + {{0x0006de8a9f19c539,0x0002d974e34e231c,0x000508fa95717bd7,0x00012449e1d216f1,0x0000f1123626adaa}, {0x0005e31823b73482,0x00058c6340698014,0x00097c2584dd8f0d,0x000431c3d82fc722,0x000076fcfeebcee7}}, + {{0x0001b5e2bc0aea9e,0x0004fe3294318eb6,0x000e4b87e4f668fd,0x000f0a23a32ab138,0x0000137451853d0e}, {0x000f7e6853ac983a,0x000dc8e78a571a46,0x000a96dd1c3bdf42,0x0004600cf207852e,0x000010649ba91638}}, + {{0x0009f0b879fbbed4,0x00069a9d1869f236,0x000766f450ff0ae8,0x0000fc1251d75956,0x0000984d8c06be8d}, {0x00095a6d21008f03,0x00000a1a1c497ecc,0x0006c50f329bd54a,0x0002517b9828c5d2,0x00002c0087c81d0d}}, + {{0x0003ce60c1cdb260,0x0007557ca2059bac,0x000b1fdcdcd94d94,0x0001491b1bd5989d,0x0000eda0108a3d8b}, {0x000661056152fcc1,0x0006d7192b339506,0x0002e05a4c2f037e,0x00062eceffb41ac9,0x0000105f6c2d2f6c}}, + {{0x00035008733913cd,0x00026f3adc4068e7,0x000a278e9cce8616,0x0002922407a94238,0x000013c1b9dfab21}, {0x0007ec71c74cf5c3,0x0008c1a4c1b493ed,0x0003a11f18887dc4,0x000cb60830ff304b,0x000058c5a3c88937}}, + {{0x000c404890228299,0x0006ab798f79027d,0x000be6ead40e9397,0x000a5d00ad333738,0x0000c23f6bd834c0}, {0x0001a35fbffd8bb2,0x0008f949d3ddd171,0x00025d93a60fcfb4,0x000967e9c8ef4b78,0x00004233cffc0a8c}}, + {{0x000e46ce6d982af4,0x000ea7544d7c67ad,0x0008bd087ebb6bf3,0x00028096b9ba763d,0x00006fe382d8dc61}, {0x000a7e8b5bdbd756,0x000158f228febd39,0x0001c4300ab38133,0x000eaba709a77cce,0x0000a247e575337c}}, + {{0x000f21b636288bec,0x000408a7c3058f34,0x000919e049dfdca7,0x0001f7adecfd1bea,0x0000df2688e4e199}, {0x000df44d0f8a67ee,0x000a2b58d010e607,0x00024f8f4d985df4,0x0001ad77f834c50c,0x0000976ef5780bf0}}, + {{0x00095aca1c323732,0x000a534c0a135363,0x0006bd5bc351027a,0x000bedb2f1b5d65e,0x0000b539e24423de}, {0x0004cec0eaa1d710,0x000d661dcf65d499,0x00054c7402a83381,0x0005ee5f1aed2f7b,0x0000bea3fa5d6dda}}, + {{0x000b68436cc61347,0x000350a443dd9d4f,0x0003b7d2a8eb9bbf,0x0002578c500e2e38,0x0000aad621ccb775}, {0x0004d740a8f7cc07,0x000d97562d656928,0x0009758eee820c2c,0x000c2d4f9531b949,0x00003e95ca5dee0c}}, + {{0x00090abfbaf50a55,0x000b184e0750f617,0x00076b005df55e76,0x000dc79c516da7f1,0x000075553bbca2dd}, {0x0007ca3553afa736,0x000bed55c25137c8,0x0003e5d35315f3ff,0x000f288846442aaf,0x00001b91149c495f}}, + {{0x00095d3fa326dc3b,0x000e68fc2cea23cc,0x000a37d591df4da1,0x000e1d74bf9adcd0,0x00006710053e20d6}, {0x000667e618344d14,0x0002606445af96f9,0x0008dbc3acc7ce04,0x000a5b602d8514d6,0x0000ea109e4680b5}}, + {{0x000a7acb40961bf5,0x00071aa56bfa5741,0x000b765d14ada593,0x0005822feb914502,0x000061e97bf36ad1}, {0x000a5b6da3982f5d,0x000ce546f468bbc4,0x000612d200c2659e,0x0008dfe8e7e6aa59,0x000083dfe217c19e}}, + {{0x000c45fb835398c3,0x000f838a41c28530,0x000f5dcdb6106a8b,0x0008ecf1e8f9a635,0x00009707137fae49}, {0x0003834d8249f00d,0x000edb2537a070c2,0x00061c0c29f14b58,0x0004a70043c3655f,0x0000c5926d6d9a19}}, + {{0x00003398e77738ae,0x000f1ba46426ddec,0x0007f6e86d07a63e,0x0002404e58e79cee,0x000059b045adf32d}, {0x00084e520fa03389,0x00085aff5acec5ec,0x000a3831397939ac,0x0008847310a4e3b4,0x0000115fba31f9d9}}, + {{0x00010c25fadf8c35,0x00027e19c0e28dd7,0x000fe502266be38a,0x0001b8842a279c4c,0x000097bb5305e24e}, {0x00086b7c153ca7f9,0x0002e07d63bd3cde,0x00060d21ea8d30fb,0x0004ab1c905f92bd,0x00008e7ffb70b9a5}}, + {{0x0007df8e9726a306,0x000f4fce3533d714,0x000f1ec40b5e216f,0x0003fd5550b7992f,0x0000b613b8801e95}, {0x0008dba792d56100,0x000aa190fbe187b8,0x000f581da2ee1270,0x000a9502f4e2dc2e,0x000016530e4eff82}}, + {{0x0003dfd8fd6ee891,0x000636848fffcbb9,0x000a47adf16d3d98,0x000a71500eff241d,0x0000b9754a01ad47}, {0x00066df70c33b988,0x000e5f34186e8f92,0x000d24132aadc87a,0x000eb98d2ce8e14a,0x0000a47cbfc99946}}, + {{0x000eb6662b5f3af3,0x0000dabb373447fe,0x000f35cb1cefab56,0x000eb9149de60e19,0x00009f8db14457f0}, {0x000cc5b3c61bfd60,0x000d41216703ffae,0x0004e1cc2a5a4d41,0x0009537f8fabed22,0x0000d5a8186871ad}}, + {{0x00074f7d22da9a9b,0x00081c8a9b0df107,0x000c32cff45b8a67,0x0002a529c2e722bd,0x0000f71b5f5e3720}, {0x0007f2f69fc4db96,0x000c165d01e195c5,0x000904635b6dad34,0x00088c1e0bd13fcb,0x00001751253d63a5}}, + {{0x000299781af2c2d1,0x000471b9d7dad85c,0x000533e8dc0f7d9c,0x000311738a34ae08,0x00005c4cb08411d8}, {0x00032858e121e14e,0x000da5000a5f97f8,0x000256274eea7dc1,0x0005bf2c6059b65d,0x0000c9beacf99507}}, + {{0x000aad71df978283,0x000578937877173d,0x000646f3cbf851cb,0x0003528083c59401,0x0000bad30cf80c6d}, {0x000b202496bbcea3,0x000ee8a1e8bafeb2,0x0000660293cf9fd4,0x0004f8a26de7ff1c,0x00009c81e9e7e9ed}}, + {{0x0000cb97b390d355,0x000d464aab27d8be,0x000ef64f801df2bb,0x0001dcee8c1a65c3,0x000067291d1c16ed}, {0x0009c6c5f5406d3b,0x0008aba8e23f9549,0x000096ece71fdda3,0x000d161feb320ed5,0x0000e7ba92c7a66d}}, + {{0x000d36bc6fb5a7df,0x00097d2dd0e04608,0x00097a36ae3eea15,0x000e1d85b0a3eb8f,0x000059814cd0c83d}, {0x000c5b01c33c23f3,0x00043faa413656c9,0x000316551a96c1da,0x000c8f16bf2074de,0x0000b866e7b4f756}}, + {{0x00027d81495ed6bc,0x0002f682dce77277,0x0008610f3b239424,0x0002a3cab8454e75,0x0000243ce85457d7}, {0x0000d71dbbf370fb,0x0006c8e0f7ca7b32,0x0007b523fff9afa3,0x000d415119d1e0ea,0x0000997f8cbb58c7}}, + {{0x000cd2a37bbb1840,0x0009a45d1fa6285b,0x0009634cb51dcec4,0x000ef16ade3b64e2,0x000080c94a726b86}, {0x0003db12283fbe37,0x0007ea9315edba58,0x000964bec902bddc,0x00030097c1ccb386,0x00008f4ead026258}}, + {{0x0003a4956f908239,0x000fe41d777b4bdf,0x0008bf760ba0f507,0x000b01791d71c3f3,0x0000633d5102b625}, {0x000b743b8c9de617,0x0003ede7472003ec,0x000ce1cb2b475125,0x0002ef2f9defc974,0x000074a4f6a70bd3}}, + {{0x00085f773848f22f,0x000603f8558eaca3,0x000c471f953dad71,0x000bc6bb7b34b093,0x0000530e06a09644}, {0x000b1ffdd59d31ae,0x000e28daa7953d9f,0x000cc88d74382e0d,0x0008ff365c6f4bd5,0x0000aa392d62a18c}}, + {{0x0003c67648024ee6,0x00022c2fabcd9420,0x000aec835188763f,0x000d54480f87acbb,0x000032c96e1529d8}, {0x000a60e4c00a95ea,0x00004011e9fa29b0,0x000b772232ef17f4,0x000e3256c0e1d115,0x0000aec2c62b4b04}}, + {{0x00088d83d84e58c4,0x000c558571dbd356,0x0000682a62af5094,0x00007c0fff7e1976,0x0000cb27078239a4}, {0x000c5474ff0e321a,0x00057b34c8ff0f59,0x000bc1ba7169f34a,0x0005438bff109652,0x000025423b823583}}, + {{0x000d5d50ac8b7823,0x000bfdb3c8925d55,0x0008bb642ff6622e,0x0003dc18fce7416b,0x00001d6998c99d7e}, {0x0008004cadcaed01,0x0002c81d053cdbaf,0x000630ec6801b014,0x000c8e84b189fc59,0x000020e9934bf762}}, + {{0x0009aa4fdc6a4043,0x000e7190994853a2,0x000e8968119d8e01,0x000d370cfcabf1d7,0x0000321a50d7e132}, {0x0006863e9a861112,0x0000e6a3bc65d049,0x0009f8eef8c0cde6,0x0001418f866c49fc,0x0000066350f1f7f5}}, + {{0x0004689e56ddfbd6,0x00079e32983a4f8a,0x0003cb8cbea1b0c0,0x00029ecb31746287,0x000058deddc8d932}, {0x000af4d0f64ef589,0x000ce30cc7a865ef,0x000047d70fe43287,0x0006c91ebc0c723d,0x00002efa53a692d2}}, + {{0x000845794b565268,0x000e8961002d06e7,0x000dcb10f415cb80,0x0009fe09e5c56576,0x0000bbb69837f925}, {0x000795b9abc26689,0x00038e678fb14fe8,0x0007da2b9b5d4f53,0x000804d601f3be7b,0x00008da59e3313d6}}, + {{0x0008ea601799a527,0x0001486d2952190d,0x000ff2a7ca20cec4,0x000d36c062ffb27f,0x000041b32e5e9f19}, {0x00081814eb57d471,0x000406aef06bf80d,0x000ecb5887a2d0ed,0x000f5af9735fb01c,0x0000641caaad6061}}, +}, +{/* digit=4 [{1,2,3,..,}]*([2^28]*G) */ + {{0x000824f20151427a,0x0005f24302067f99,0x000112357206828b,0x0004ec0a9097d7e1,0x0000cf9a2f2a9e41}, {0x000c9da279153564,0x0006c01efee3dbda,0x000b288e27e0734b,0x00009c14fab5bbd2,0x0000c630fc5362dd}}, + {{0x000107a1ac2703b2,0x00084bc857b58537,0x000daccd1b49258d,0x00052937df14debc,0x00004ab68d7e4ae8}, {0x000b5d4734e59d08,0x00084495cc807ed8,0x0001db9b35f8740c,0x0005be04aedd5a29,0x00000b360f8cfb99}}, + {{0x0005f5d5fa067d1d,0x000ec668960cae91,0x0008edaac4134b57,0x000435ed3656d6a4,0x0000ac1e3e5cc1d7}, {0x000f869d81fbb26c,0x000bf26c33d4674f,0x0004203e8449ed3e,0x000f49c5138705d9,0x0000cde538c7eeb6}}, + {{0x000c68da61a76fa3,0x000d9a1554dc55d5,0x0003b279c598b441,0x000efca39923b977,0x00003331d3c66bf9}, {0x000848e298de399d,0x0006d1a27f562d4c,0x000b8ab70cfdb8e7,0x0009b9c4c855ea57,0x0000cdb9daf3f787}}, + {{0x000f8c2019f2a596,0x00036b4fbc747bdf,0x0009173ddb3ce5bb,0x0004398a907f688a,0x0000cd3d0d3f5a75}, {0x000c4d6efed021c1,0x00005a77339a92ec,0x00088c64a09a9f9b,0x00015877ca6b1571,0x00000c2996854899}}, + {{0x000a229ed6e82ef1,0x000355ebaf4e5859,0x000ad67ae16f338e,0x000bb3fcd313875e,0x0000c73d22864ef0}, {0x000513174a5c8c7d,0x000faf69ad6a4cb5,0x00066f87e01cd296,0x000320d04d00dde9,0x000096fe447db7b0}}, + {{0x000c06e88fbd3813,0x00042c35a493342a,0x000f1bbcd02cd4a8,0x000d4cb8fa89de54,0x000041d63675575e}, {0x00057fbd238202b9,0x000a1984ead9ebe3,0x000436ea0600b4d1,0x00051b335c9f4452,0x00006fe0a3a33707}}, + {{0x00007367f636a38c,0x00064e76d5cb4c4f,0x000b68b8b9f943fb,0x000a1ef03510baa8,0x0000246780b5ed07}, {0x00014156d549fc2b,0x0000b07781ca3c05,0x000d95413c2953f3,0x0002e2e55e2c69d8,0x0000300fadd2bd28}}, + {{0x0007b5087e9189fa,0x000c542dda2781fe,0x0000a5904db17375,0x0008e582f7d896cf,0x00000e57c5b8be34}, {0x00011d3f40e3c80d,0x0000bdb705c5a610,0x000fedec3b118932,0x000c1c7ed9309e50,0x0000cf14a111d6d5}}, + {{0x000265b556913429,0x000401049dc7056c,0x000bae20ae8e0850,0x000db3831329f5c9,0x00006c8b3e969dcc}, {0x000f838fb4ee6b40,0x000b41e8ccf08c5f,0x000e050c6fc5a9ae,0x0000807417b764fa,0x0000953c3d700452}}, + {{0x000268238dfe7e82,0x00050bb79d4b2137,0x000e7cf2dea417e1,0x000fcc39641f1c76,0x000071e305a0a0bc}, {0x0007dfd7253ecbd7,0x0004dfca6186624c,0x000866e9c2f552e2,0x0006105bf84ecd4d,0x0000396770a668d4}}, + {{0x0001163c27901b4b,0x000fd99b8bf3a14b,0x000c6da0afd9236d,0x00029692b091eccb,0x0000b1dac700ad1d}, {0x0001d53a91cf76e1,0x0002c31f1ee780e6,0x000efcf774110a41,0x000d761d87c3ba13,0x0000f374bb4ef450}}, + {{0x000e2f20d188dab3,0x0000c4b885ef5e78,0x00014570fe3968ed,0x00052116c0568e73,0x0000161633831170}, {0x000e7e24f0c8afe3,0x000faeea78da18e1,0x0005d8a514caa75f,0x00037052db67f27c,0x00006a44d869f505}}, + {{0x0005bda0333974fb,0x000d77a70146d72c,0x0000ef9215db516a,0x0008e48470528121,0x0000ff17a8fbc9c3}, {0x000814e12476da14,0x0002f3c1698078f4,0x0004d4bcac1e1661,0x0000de5e5b386f42,0x0000c274e8808574}}, + {{0x000b88d6c2f5226c,0x0003950d7ca8b6a9,0x000c4170914d1b94,0x000518980c85fc1f,0x00001da368c14c6d}, {0x00085ced5113cf71,0x00034a34708f2b07,0x000cc3f880670f63,0x000c8f36e2376715,0x0000b480cfa60c72}}, + {{0x00086024147519ad,0x000b56b372f02028,0x00085ebc8d0981ea,0x0008e8d9d4a7caa7,0x0000953c50eabdf5}, {0x00061ccfd590f8f8,0x000ac4e6c9179d63,0x000eb64cf72e9626,0x0008f2ffd9611022,0x000063ebb7f1eb28}}, + {{0x000b7616aca8ee7c,0x000917b403586e6a,0x0005feb0d97d10b3,0x0002795687d3771e,0x000083e50e54265a}, {0x000a9fec954b3132,0x0007110d1f618f75,0x00057d0e0cc2e8f4,0x000207d5ba81c565,0x00005f9680c5eaf6}}, + {{0x00066094354080b3,0x00054bf2fe1cf95c,0x0007d98fa5225bfa,0x000f6095c004e25c,0x0000561bf1c319aa}, {0x0009f17ba1514741,0x0004a04f6eca5e6f,0x0009acb1edec2f93,0x00049354e368a126,0x0000332d9e41cdda}}, + {{0x000cf69df23de05b,0x0001509339a060d6,0x00069392366d17da,0x000a6cefcac9850a,0x0000cf057fd9d7c6}, {0x000c8c5f0b5662c7,0x00086cba4f24c3c5,0x0002b69ff25318dd,0x000c00f0e8cb7508,0x0000c23b3ee8e728}}, + {{0x0000a0a097e4403b,0x00056985466515e1,0x0007d4826cb3d0a8,0x000838d8d8e211d6,0x000039af66ebb9d2}, {0x0004588bd475ca8d,0x0005f077b80ba5f9,0x00027c26ce06b796,0x0005e02edb1485da,0x0000290d33bce0fd}}, + {{0x000cc47f34fb0fa1,0x00080fb1ab09a40b,0x00073bfe3b4760cc,0x00013c7fca0993a2,0x00003e4fe08070b2}, {0x000b992fdb05163c,0x00004c2b19b63bcd,0x000f2e3e28c484b1,0x0001b35acb815faa,0x00006905936789ff}}, + {{0x0006f9d586e74e18,0x000ce7b80484b2ad,0x0009c3ddb488883a,0x000d30f58aa2c736,0x0000ab74e6a1f9af}, {0x0002d285e21beb19,0x0009a18c42f910fc,0x000cf40c33484518,0x0001d8a77427dc53,0x0000de0781ac91bc}}, + {{0x000e858693807e1b,0x00069e81ccc78fae,0x000835b84a386532,0x0008d352c30ff26f,0x0000604437bbd3d3}, {0x0008a985ca1823d5,0x0008b3be0324b3fc,0x000684a33b82f7ec,0x000f7d9e36d761cf,0x0000a01df0eef29b}}, + {{0x00002f31306583d2,0x0000237c622e6862,0x0006a7bc805b10da,0x000e439f9aaa0f07,0x00005e94efbaf8f4}, {0x000c9b7fa3dc26dc,0x0002d6ff03c58a35,0x000c394cee0e5fb9,0x000e5fe77e3843eb,0x0000ede65964361d}}, + {{0x00022f6a19935459,0x000594d671bad27c,0x00069c28eab01cc3,0x000375d3fa2877a1,0x000025ef904beb08}, {0x000a3cf53aa0b32d,0x000aa1c49d7a3b2f,0x00005e27fb27beb5,0x00070ce60e1834d1,0x000060897891f685}}, + {{0x0004ce0d6fbc2acd,0x0000b15ff5512309,0x000ef119c738037a,0x0006ba0a73b1bb6b,0x0000cf6c431bef50}, {0x000fe7be3ef104a2,0x000bea06562800e4,0x00092043eebdd9a2,0x000108653a81c387,0x00002ad6eced3b59}}, + {{0x00013c039cd297dc,0x000d45bda5d99fb8,0x000104b968ec7e16,0x00050f6834797c0d,0x0000c11a2e83c511}, {0x0005a5396ee63809,0x00053ea3874296ca,0x0004dfa7d054c865,0x00020725946852d5,0x00007c422e7af4ab}}, + {{0x00075090c22b5403,0x000a87c267d4bf90,0x000b0d6932cde42a,0x0000d98a18f9ed5a,0x0000ba62aa69e466}, {0x000f97bab9ea96a6,0x000283b60e32b24b,0x0004d9bd55d03964,0x0006a3ee6a45067c,0x000066c5b9eded4a}}, + {{0x000dcd98edbd7cc8,0x000746ccd753fa3f,0x000e6b64f4660bb8,0x000b3f1ae9082021,0x0000a56a714336bf}, {0x000e0965726d47f7,0x00019b1a9a7fabfc,0x000b74a379eed01b,0x00066cc0e9cad44e,0x0000b2524ccc3e96}}, + {{0x000683b8f4b002f6,0x0009e1f4fc206a29,0x000338accc2200d7,0x0009756f3af47a3a,0x0000539a4fc47128}, {0x0001c14c33c7fcf7,0x000b57be322bcec3,0x00046f6237eb6799,0x00099aa19ef4e966,0x0000b7a26a5c4d72}}, + {{0x000f08d403f46f2f,0x00022a0ec0c7cb37,0x00032142f94b8fc4,0x000a79cb8514e3c3,0x00003ed2c34d80d2}, {0x00080afb639126c8,0x000063553ade8d20,0x0007e2b09f7b6be6,0x00002ab950aa9f1c,0x000047ff958e410f}}, + {{0x0007cf5678a31b0f,0x000d4998b620877b,0x0000fb396d50301a,0x0002a5834257c5c0,0x00009fb18a0f4e67}, {0x000d8ebe8758851b,0x0005ad99ba44ff8b,0x000fd93b71e64e4c,0x000b8b9b8eaedf7d,0x0000a2f2a98b4e76}}, + {{0x0000cbae8053433d,0x000f6d2c95857d79,0x000d8f5edc8e7259,0x000e1c88c5c476cd,0x0000106b953bfa9f}, {0x000775b0eff13a96,0x000a8057b9303c5c,0x000b70cbd242442b,0x00089a89f458d4c9,0x00009b7144903cdb}}, + {{0x00046f60e2ed7422,0x00052006749341ee,0x00054c304573f104,0x00050291e154ff9d,0x0000ad0436aad3a7}, {0x000aa2d431a8121d,0x000ab86f11edee4a,0x0004a0eb7cd38b3a,0x000e5787d49ea603,0x00002b773bec7e85}}, + {{0x0009ac49b5c1f14c,0x00097e54df2b4a55,0x000a41891c444be1,0x00078753aad704ed,0x0000d927bed1eb5c}, {0x0008516e48c8a348,0x0001cb546669eb3c,0x0004df8ec1b7ac81,0x0008649815f89659,0x00007c6a79cf9227}}, + {{0x000a2f09b56ddbdb,0x000bda2f1cf3ae02,0x0009dff0d1339b5a,0x00043d42b569c783,0x00000b9e865aee9a}, {0x000ca4177bb064e9,0x00026d249f634ff8,0x0006f689a145a281,0x000f5daab7beacf8,0x0000bafec2791d35}}, + {{0x0004c654265aa91f,0x0003135efe422805,0x00039dec7a4b1830,0x000ea47887b0e696,0x00004b8f6ae2d52a}, {0x00093cc971a8a130,0x000d4c934d07fb92,0x000acbc293f159e5,0x0001292c50e9b109,0x00008eb65e67154d}}, + {{0x000f58930b75c3ed,0x0001c4491c934fef,0x000af62bb0bb82fe,0x00049f08ac377a89,0x00007b514916685e}, {0x0009a7b04497f19d,0x00094a7ad13fabca,0x000c86ed61b35ed0,0x0006f09b601e213e,0x0000a91fcba9e0c7}}, + {{0x000507bd7ab27e14,0x000523945b7b9e28,0x000fc98277c19a55,0x000b912b43f0a1aa,0x000043b4fbd7aa55}, {0x0002e656962c88f7,0x00045e0db0ca962b,0x0008d6c4f139da8d,0x000824493f05dd1b,0x000079cdff7880b9}}, + {{0x0003fddae57c7b7a,0x00017b932522bba2,0x0006d4aa3345342f,0x000b615d9c80fe55,0x000003907bb0525b}, {0x00010e1ff2189336,0x00044a52117b38b0,0x000f2e6eac066b65,0x0002b22e14192094,0x00006a27dca6d32f}}, + {{0x000f993048b37172,0x000a9178ae1c69c7,0x0004f1d6bbf5a989,0x000e4e29fa905856,0x00007ec6e15f31fd}, {0x00003737276e7fc9,0x0008f9d6bf024cce,0x000cdd97964086d7,0x000630ca72f0464c,0x000009c3566d7775}}, + {{0x000bc6b75dd7125f,0x000697a0428d1c07,0x000eb6b9db4c6bc9,0x000431607ece52fd,0x0000ca56513a2c95}, {0x0007181d0e8bd06b,0x000716bb46ea15d9,0x00052b624384dd31,0x0002faa441ea2039,0x0000cf70deefe7dc}}, + {{0x000016e6628e8c30,0x0007b60a7522372b,0x000344ee207a0d66,0x000eeccf05751b0a,0x0000ec09a48118bd}, {0x0003d4ed83dce467,0x000d29d2fc6e6e4b,0x000cf044c43a6316,0x000fb7399d898956,0x0000c7f44551e3e5}}, + {{0x000b121fbabbe926,0x000b81330076b2e6,0x000890015281850f,0x0007f4a93581ec97,0x00009b1ddedd5ff7}, {0x000b18fab1051053,0x0001789ccfef7cf0,0x000914009953ced3,0x0008ad0151f85feb,0x0000c9f1b87b8ed4}}, + {{0x000a1a14a7eadcb4,0x0001122e71cfc9ab,0x0002e4f83928e750,0x000bd3aaede7273a,0x000067e10d15ce3b}, {0x0002ac3b955dcf05,0x000d83d5e527f344,0x00077f474ba96307,0x000ff0a763a10efd,0x0000d744bd0ba6e1}}, + {{0x000282aa777899e0,0x000fd03f3cded287,0x000b07d31e20eda8,0x000de46a7e75bb50,0x0000b7e2a946f379}, {0x00064ad19f593cf4,0x000ede76ef1d31cb,0x0002d609c7b1a9e4,0x000650a18c9c9db6,0x000039bad6e2779a}}, + {{0x0009066e032f1445,0x000898b2ec6a219d,0x000a12f781db632b,0x000265af0d0fd4fd,0x00006fb4c2d7a25d}, {0x0002ee1255a03f19,0x0002596af1765f4e,0x00068bc9761cd6af,0x00088e50317ba8d0,0x000027d6babb64b9}}, + {{0x00007e0e90fb21e5,0x00006ba7fca1a18f,0x000cd67b500fd2b8,0x0007f6d0387f2795,0x0000b89a4e823970}, {0x000ad3f894407ce5,0x00041c2261328f83,0x00006c13ba0025b9,0x000025779563c7f9,0x0000f548f319e7bb}}, + {{0x0006b8feac6d1135,0x000bbe813c762b4c,0x000e1f4b9a67e3f9,0x00050de982717c3f,0x0000886581976d80}, {0x000640cf7f06f200,0x00062a66ebc299f3,0x0007a1e08dc61021,0x00071b52f2c17576,0x00005660e1a59998}}, + {{0x00017626d3c46934,0x0006f7ed7bea6b0f,0x0005b226df0e7d62,0x00013b851758c7b7,0x00000a886285f916}, {0x000baa7bbb38ce0f,0x0004cddcad81889d,0x000c9671fe0404b6,0x0003750ebccd3a8b,0x0000bf9a358de1f5}}, + {{0x00069b028f33398d,0x000ca2e90f655dc1,0x000ab1eb1b07ec11,0x000389de7f3b4afa,0x0000970195f2f175}, {0x0005cbe0181e6402,0x00020643313d52b0,0x000f31f82f5debd6,0x0003c5561481545d,0x00003e03b335a9e1}}, + {{0x00089494fde0c1fc,0x000e25b6ec20ff75,0x000e1db6cbf8a1ab,0x00058eb02278fb87,0x0000447ad7af5ed6}, {0x000aa3803d0ccf24,0x0008419a7c0348d4,0x00017cecc80acb33,0x000d825bc7c89e6e,0x00006736b8b43be1}}, + {{0x0007b60c0432f963,0x0003aeb5442fd65d,0x000ff69a2ddebe7a,0x00012249a253077d,0x00007a56d9432cf3}, {0x0008aedf2350d0a8,0x0005837b0d9a8bab,0x000c65cae13c3f27,0x000a884664957c44,0x00008b4408a42e71}}, + {{0x000e5a35cb026649,0x0000f686c72edb88,0x00082d53e5d4c0bf,0x0004315a3d9b62a6,0x0000b605ef49b2ad}, {0x000c202c69645d0f,0x00097a1b66e771ba,0x0008f4dc4a115f03,0x00078c0e2c563a15,0x0000715b3a13d12a}}, + {{0x0000a48d413213a4,0x000d804becdb8f7f,0x0008587f52035806,0x000a71acd34a995d,0x0000d8c3079df6d3}, {0x0002a678d95a8f6a,0x000d52110d0d1b2a,0x0008fba3fc58c9d7,0x000f683eee81d5cf,0x000042be3c0ac7cd}}, + {{0x000f742d43b5eaaa,0x00063fa59b852126,0x0006bfd45054a076,0x000a8efd0d5e3612,0x00001f8fbd7d84f8}, {0x00080f5d563fccc1,0x0005e280a9283176,0x000b578cf48ca505,0x000514d00b81b227,0x00000aad9183994a}}, + {{0x0007b62b7bdc9534,0x00021bc086ddd9e0,0x0005eee779f0f6ff,0x000df0c9d1ccff65,0x00005475f799bef7}, {0x00028fa86f702ccb,0x00045f021f073faa,0x0008fa8c692e6090,0x000a2c39e629687f,0x0000d71419ba036e}}, + {{0x000e1cc6028da9ad,0x0009f251f573171e,0x000a997f45352fe1,0x000d5f28ff236e3f,0x0000831b6ca75749}, {0x0002e1de350e2c27,0x0008ae0ce4037c87,0x00074f5cbc56240d,0x00072889deb07769,0x0000d50ba88061c3}}, + {{0x00094265a3a2518a,0x000926303d43d6f8,0x0009e5696cf81779,0x00007ab10a047161,0x0000b049ff6da5e3}, {0x000f9b0feb13ec70,0x00079d8ff90ce4cd,0x000fa96afd5e9711,0x0002a2f6f64d069a,0x00000d0bf5e9d201}}, + {{0x000301f358bcdc04,0x0008ca9d47f8e63f,0x00043d43a07689e9,0x000903df689e2f4f,0x0000d542a16d0920}, {0x00093d59ca0a7072,0x00056ac68065aea2,0x00090008cd061fe4,0x000db5f033bf1b00,0x00009749558e08a6}}, + {{0x000fc59c1d5d0340,0x000667e215e074b5,0x0000200e6f712e9f,0x000588fd520cbd86,0x0000229acb43ea22}, {0x000e14cfff0c82e2,0x000239c69e739cd1,0x000ccb98987684b6,0x000493ba85e61c96,0x0000d5dbb02c3d06}}, + {{0x000d33ae86b173c3,0x0005779ff0e3f22a,0x0000d0c10e8e41ea,0x0008f8d1d2d725dd,0x00001f39088332d2}, {0x00071e17829839ea,0x0003a502ae587b3f,0x000fc61150cf691b,0x0003144f658dbdbe,0x00005cd6ee653ab5}}, + {{0x0008d7b5f1d23474,0x000a0cc2253a206c,0x000389e08794645b,0x0002889517d8ff58,0x0000fa20deedf847}, {0x00072d8d797770a6,0x000d5f429e26eba0,0x000af82797360c91,0x000ce31200a3b380,0x0000a1c9150e2dad}}, + {{0x000d3a7c35d87949,0x00077356bae50ee6,0x0003322fd042e655,0x0009670f59698d64,0x0000379ae15e0a61}, {0x000ae62fcc9981ea,0x0000cd2934c664b9,0x0004e65ebaed3d63,0x0004278454b3025e,0x0000b09f64899950}}, +}, +{/* digit=5 [{1,2,3,..,}]*([2^35]*G) */ + {{0x0003a1222248acc7,0x000ec264e366b208,0x000fdee281f6ec0e,0x000bb4e659b7045a,0x0000a823a4156430}, {0x0002a04e1900a791,0x000ab9ee65762459,0x0005ea54acde09d4,0x0005a742b6463f4b,0x0000efe9ed3e3ca6}}, + {{0x0006dbe305406dd9,0x000f4d5d1957e27a,0x0007d4d8f8eb7dc7,0x000de4654a687638,0x0000c47940a57762}, {0x0005b5d99b307781,0x00065e793682be4d,0x000c740e325380c5,0x0004ae502d37f3da,0x000040deabe2566e}}, + {{0x000d067afd32acfd,0x000a11f71ccf4481,0x00096f2dad8f0fcc,0x0003f90208dd0cb5,0x0000049d7316aad9}, {0x000263d42ab580e7,0x0000b3f707b4c79f,0x0005e0eda09411bb,0x0004021cfde1ff83,0x0000270749100f03}}, + {{0x0006126c49a861ec,0x0005214f0d06eaee,0x0009bfc17024f3b6,0x00038091a3f1e8c6,0x00003c3a8ea67686}, {0x000752cb103d4c8d,0x0002c218b36b3400,0x00051504a02bc461,0x000bf9f67f75eb76,0x00006848b57a02ae}}, + {{0x00002e6c30fa92bf,0x000caa552784bd98,0x00083169b5a70d96,0x000227a085c4ea3f,0x0000a9423bbf6908}, {0x000be12fe97a5b9f,0x000881b991182ffe,0x00017884685da604,0x00018dacbc2f7f63,0x0000d96bc7181532}}, + {{0x00081db1782269b6,0x0008c597e5509583,0x000385153ae34bf7,0x0000485b5c60645f,0x0000f0e96b043088}, {0x000021577884456e,0x000b89310ea7bf6a,0x000fad2deb3b5688,0x000d4c37c9429504,0x0000020f0e5f7896}}, + {{0x000a0ab0976505f7,0x0002995e2ec5730b,0x00031ab71567f681,0x000b9ed706201063,0x00002cfa977b1d22}, {0x0005ead8a2373da8,0x0003fba45a6833e5,0x000029d15a8d0d5f,0x0009f33a1d8f9c03,0x0000f34f1cd7c55b}}, + {{0x000428dbbe5a1a9a,0x000e9126bd67cca4,0x0001058268187fd5,0x00019f6036973a48,0x000039b666458bd6}, {0x000deef2d65a8087,0x000f24636b196d42,0x0005d564c4969044,0x0000778611ee47dd,0x0000b2f3a4a42873}}, + {{0x0007d45300eb294a,0x000d769c14949415,0x000a47aa92b2a656,0x000ea42000dd76d3,0x00002864e5046243}, {0x0006c47db89842e4,0x000721479fb78271,0x000b2f6dc12dfd7d,0x000d66fb9a2c56e0,0x00006be862b17f85}}, + {{0x000d8dd0f82b2148,0x00097103cbc603b0,0x000d79e19460c34f,0x0007f8732e5c0318,0x0000b8888bb28411}, {0x00037dcc07226779,0x00088c1c0f278f3c,0x000f7a0c610d21be,0x0000e0447c8468e0,0x0000bf022143decc}}, + {{0x0007d1242b48b998,0x000cc84240960baa,0x000fb5cfb1bcb665,0x00010d0b847cd6eb,0x00007c2ae571ad4d}, {0x000b1220de367261,0x00082fdfbd21f1cb,0x00079d460e7043c6,0x0002cb3bd0826a4e,0x00001f5e5985bd1a}}, + {{0x0004160b7fe7b6e0,0x000a400a3fb29755,0x00028ca1e7d16189,0x0008ccd73e9beae3,0x0000dd04b97e793d}, {0x0003c9b506db8cc0,0x000ecf38814ca9c8,0x0004b45e65cd47aa,0x000a8426fc430db6,0x000079b5499d818e}}, + {{0x0001102c1c24a3b9,0x00078c161c1aebb0,0x000f00a4aca24e56,0x000c7a803eea6936,0x0000ad76ee906176}, {0x0001fc2538e0ff72,0x00094604b3b09745,0x00049cfd794f8980,0x000f694311436e32,0x00007b4a7bd61224}}, + {{0x000d21ae0ac29416,0x000462d3193703b5,0x000c992d0279b025,0x0001f2d307c052ca,0x0000aa7cb934fa8b}, {0x00025800d37c7a50,0x0007342d54225a18,0x000d2ef9213380c3,0x0003c692ac2d66d5,0x000035a70c9030c6}}, + {{0x00025dd4ce4f152f,0x000109df7c06c160,0x000e4bb141f419a7,0x0004cd7d5b221491,0x0000c43c6cd739fb}, {0x0006591925d6b2de,0x00028218659849f0,0x0001b16294b37d9d,0x000e04ac54a971d0,0x00001a9c2a031d50}}, + {{0x000b78571ba18615,0x000c80c8f93d5109,0x00033bb9348b22d5,0x000d0898fa84a786,0x00003fba6baaaebb}, {0x0007df3e5eea7d82,0x000648ca71587ff2,0x0006f1a05521c879,0x000ee499d5133bce,0x0000d50cd541d0eb}}, + {{0x00015d6c5a3ef169,0x000d2a079221c821,0x0005da81c993eff9,0x000d8554da2c5e4b,0x0000a89dbdc1033f}, {0x0009ebf2b892891c,0x00009d14a4d56081,0x000fda42153902b2,0x000283aac35051d7,0x0000c6ab88621c83}}, + {{0x000a133f74cff170,0x0003ecb813f214eb,0x000665bee240aaa0,0x000d73cfbb65406f,0x000084b1fe4a425a}, {0x0005d16d081f6a60,0x0008eef82c90009d,0x0009eaa2235304fe,0x0001e3f20346d5aa,0x0000ada9f07ac1c9}}, + {{0x0001678968a61446,0x000ba7b31a1ea6e2,0x00081fbe154c1f77,0x000c4a76bb787e57,0x00001bd2ee1431f1}, {0x000a1e9781105fc0,0x000f7b2f8e80f25a,0x000ff919b9cf2971,0x00096e26d15412cd,0x00001db4ebe34bc8}}, + {{0x0003e23b40df1cf8,0x000314e971b47d9b,0x0009cf9215933737,0x0000643f57bd1466,0x000065daedf8c1a1}, {0x0000bd3832791254,0x000ef4ecdaab3eb7,0x000755cafbc3d5b9,0x000f0251e3ed7e5f,0x00009699f55141e6}}, + {{0x00070e1d4a7a15b6,0x000a8aac87e71857,0x0003133ea08f3587,0x0000fbd52018db47,0x000074ce71964fd3}, {0x0009835088b3e0e6,0x0008fd0d47a17b8d,0x00074a3c47a0356a,0x00066c3d9e765964,0x00001ea48a85f669}}, + {{0x00077580f3e48342,0x0001f7a9afcb3041,0x0009a67b3fdbb21c,0x0001a8556fa17f2f,0x0000a6b2421c245c}, {0x00027944af02291c,0x00056a5804fe64be,0x000f08fd7ade465c,0x00003acdffbd39a6,0x00004efa84d6a144}}, + {{0x0001b2a442b0f5c9,0x00073f997736a1b9,0x000e90e16b748e31,0x0008bffd1b62bfce,0x000007ae2719b207}, {0x000534b0c9bcddd6,0x0003d9adce83df31,0x00026846a043fb05,0x00039339031043d8,0x00001a9c0d71144f}}, + {{0x0008046477184276,0x000b0e830f8bdab4,0x0009a1347df17ff9,0x0004ade08d7ee8e4,0x0000ac71e23f1c1d}, {0x000b9fd1defd73c7,0x00074bbbfec5c8cb,0x0007ef8ea1984065,0x000f2fc9db1cb59e,0x00008aa8296d4105}}, + {{0x000b7f0a3738c290,0x000abc3250a3a3d9,0x0005e4caf0a2f235,0x000f7a55e506f644,0x0000974f73d33475}, {0x000bba35ba2f5a80,0x00036af40066d37d,0x000d73e2c542c6e6,0x00033e26d99b53c5,0x00006060d7d6c3ca}}, + {{0x000f1c2065fef4a2,0x000ddd5b92e3cdbe,0x00070835077e60f7,0x0001bfb7c549f026,0x000001b3ad054f12}, {0x000c2a10334fc145,0x0008e44552f65fca,0x0006530828a9a9e0,0x0002892dd8a1d397,0x0000fc0738fc9d4f}}, + {{0x000244d17d2d8c38,0x0004f0830684787d,0x000f73ae5effc634,0x0009a4dddb96dde4,0x0000efb14b197254}, {0x0003eee2245ae7aa,0x000e4a11f13e6eb7,0x000b01f5dbca4061,0x00052c1577421d30,0x0000a688b25182e1}}, + {{0x0008e71bd3502bad,0x0003e4de75a06760,0x0006125e54ef41f2,0x00043ee08dde5efd,0x0000e48482674095}, {0x0008d9865cc22950,0x00016e0edfa21f19,0x000f35fc7428a377,0x0003c74f9697a2ad,0x00001a43c79f7cac}}, + {{0x00070590fd3659a3,0x000c8b7f2d9ab05d,0x000f984d38927f30,0x000a451023d1ac8c,0x00002125ed332897}, {0x0002dad3d4142055,0x000293fa82a9fb57,0x0000a558173000ef,0x000ca4fc0868e9f1,0x0000b61fc67bb0b3}}, + {{0x0008d5b7cae440c3,0x0001102b7531c125,0x00093232121c08b4,0x0008afc61a8955de,0x0000568faf85d140}, {0x0005e999ecf965b6,0x00088917276f71b1,0x0000cf9e2f14ed24,0x000c7e66f4caa182,0x000081b20b278d83}}, + {{0x000738dc6c011203,0x00037e70e0db6cde,0x000afe18c71db081,0x000be765fc064474,0x00004619053429e2}, {0x0005ceadb2a3b15d,0x000e0b4c70738061,0x000d2d3670a49a19,0x0008a93e1b84c88f,0x000074bf462d33fb}}, + {{0x0006d65533ef2177,0x000453ca2e87889f,0x0002b41677158c7e,0x00057f8b670dfbdc,0x00005910a01f44c2}, {0x000bf07cf88577d2,0x0000c45e2acef336,0x000a23d852224525,0x000f580ed92e8d7c,0x00009f8be4c4b812}}, + {{0x000baa7076fe12bc,0x000aee1537f9dd9e,0x000bdfb463f2400c,0x000b405aa9352817,0x00000f9843127883}, {0x000ede10170911db,0x000a84d4b17f5590,0x00026b8d27562f5b,0x000d5931fa1df218,0x000040b796b1bd80}}, + {{0x000f1973467ba92a,0x000b570954b0d65b,0x00078f15d8c9b46d,0x000960f7c8a0f30e,0x00008f3a69b25a4c}, {0x000660f61e4ce9b9,0x0002dea6790c4242,0x000986416bf06aab,0x00022536706f8eec,0x0000e56dec22a9fc}}, + {{0x00046f49a9898d9a,0x000ab633cdef527c,0x0009e4297d799e77,0x00073384eacc167d,0x0000bb61ceb0b1cb}, {0x000e8a7f778443cd,0x000059de2fe6bee2,0x00003bb6f3bb42bf,0x000df5fbed86a130,0x00003918e6da781c}}, + {{0x00032719a5103f11,0x0006e50eac064bee,0x000dcc1195243efc,0x000a08a8e122cb6a,0x0000b7faa84d0b80}, {0x000d1bd6dfcd08ca,0x000d6be427de32c3,0x000263c83129dec4,0x000efef8ab679c1d,0x0000fc83cb86ef64}}, + {{0x00060882fa6be76c,0x000a5328cbfe85eb,0x000618dda892585f,0x00026e0154d3edcf,0x000044f601bfabaf}, {0x0007d0b2be1fdfdc,0x000c61137fee7bf5,0x000b591a8a833bd2,0x000055d353af362d,0x000076f26dd1562a}}, + {{0x000e47d3fdf5a517,0x0002e5c9cab01d87,0x000e0586e7afb5f9,0x00070921bbf58f89,0x0000c72c0187d843}, {0x000aafb99b5c3dc4,0x000cf844aeb0a9a5,0x00067e482a48a0f1,0x00059a3178b7ddb6,0x000053985e9ae23a}}, + {{0x000c86001b25dd8c,0x00083b897c8a4a54,0x000a90cd90dd37f4,0x00030c9f8aa6100e,0x00008892c68d6d58}, {0x000efc0ef514ca5f,0x0008f72c9ee6eb4b,0x000c40d5f478eb67,0x000649abca20dadb,0x0000015de22cde4f}}, + {{0x0004de0eaf4b8a5d,0x00097bc60e32aa6a,0x000d15e7068cfd9c,0x00009c968a4b017f,0x00009f0694bc27dc}, {0x000cad5ba708bcd2,0x00093bb95c2af6c3,0x000c0a58f5cd2ba6,0x0008708a28c1d333,0x00003e274e3cbc77}}, + {{0x000692ddfd20a4a9,0x0002f1a6665344c3,0x000a0757d091c5fd,0x0003e9dc0bb69109,0x0000072e8b9f6734}, {0x0000eb080848beca,0x0009d9fd36cc31d4,0x000ed43f595bd480,0x0000be61a77c6165,0x0000fccd127ce0d4}}, + {{0x000c82d1cc1884bc,0x00009d4753b4eccf,0x0008e099fc85ac20,0x00039007a6caac65,0x0000f46369ec4b27}, {0x000d049506467ea2,0x000217cdeccce2e7,0x00080143a481b63a,0x000b882029abd8ed,0x00008bfe3c7dcb00}}, + {{0x00010090643d84a3,0x00087bd110413bec,0x0003a34d6885f366,0x000ebeab02432cf8,0x00002f7b360a19ce}, {0x0007837dad1fe7ac,0x00089441a0b0f06c,0x000d4755060a157a,0x00001fb04970e9e2,0x0000d2bd553e71b9}}, + {{0x000f82f33e24a0b2,0x000fd2565079ff57,0x0005f58259cbee23,0x000661f6353427eb,0x000076feec50948d}, {0x0002bc6da10032b9,0x000d60e72a53d1b6,0x00020e7ba718351d,0x00018d0345207624,0x00006368fffca001}}, + {{0x0002d26150a49e46,0x0005df04706b00ce,0x000b196d00c28b63,0x000b7c5ad65a4658,0x0000c8455fd4c9f8}, {0x000895f2d71867e2,0x000bcdf9f38ce90c,0x000f6ec045c0be31,0x0002510a37a15ed8,0x000039639e7acd85}}, + {{0x00053159c7c4c6b6,0x000077409af7d897,0x0007132fb603aa3c,0x00023858d53d0c00,0x00008d12af806849}, {0x00007e7bf5d92796,0x00054ada74cebe06,0x0007e8ccb9aa5005,0x000f5e881079cbba,0x000010c71d205f4f}}, + {{0x000e1a75aa07093f,0x000ab75da47c9e2e,0x000e75401ca84004,0x000591174d39513d,0x0000938f757ab311}, {0x000761800a43421b,0x000157bc78c89619,0x00017127639a2536,0x000f07778f710a0a,0x000028446eb3d1a8}}, + {{0x00081bfe3b6a6611,0x000dd6d279f71847,0x0009eb6627751cb1,0x000ea7e8ff95d6c5,0x000086d90b768d3d}, {0x000b6c1dfb4f754b,0x000a7b2801dc0e4b,0x00054564d5c5cf56,0x000f131561e4521f,0x00004fb8c61a0dd7}}, + {{0x000963033ff98c71,0x000abf17769cf884,0x000fdd80a9619fff,0x000e63e8090bf61b,0x00004d9a149522cf}, {0x000c3606f6df9ea2,0x000d018f17eab354,0x000eb3480dbcf770,0x000a26007db7c879,0x000013dbda8759b6}}, + {{0x000200b29fc81b30,0x000f171d87c1ac4c,0x00081aa9eebc3e09,0x0004fa9179953014,0x000051b92e192e11}, {0x00092e9ecb5537f9,0x000b990c7483df8f,0x000deb01644b1b2c,0x000c2c1711455a2a,0x000064b685711a10}}, + {{0x0009d99cec036235,0x00059f3271ea4f15,0x000ee48490553222,0x0001036231bea3c5,0x00007a54f50c094f}, {0x000421d9598b3525,0x000c17412ab43e2d,0x000c3a912e865a49,0x00065d82998a251c,0x0000d0928085c74d}}, + {{0x00059084088567a3,0x000def214a6173f4,0x000f0c13deb6b280,0x000b805c9adc34ca,0x00009d129392561f}, {0x0003a5ebc6edfb49,0x00016e4d210eb2dc,0x00086ae727485b1b,0x000b87762e0400e1,0x00001e32d5cfeeb3}}, + {{0x00074d74be59224c,0x000bb16d55f36df5,0x000d6ed33ebc88cc,0x000b0f66c2e6d0ca,0x00006e21e7dcd3e8}, {0x000840e5bcc36bb3,0x000e4da74f692cc5,0x0005193a89292445,0x0004135be8d3214e,0x0000ec23629bdf06}}, + {{0x000ae85b134defa9,0x000f8bb2d475c7e9,0x00063c00d6073b1c,0x000ac429ad615e28,0x0000e29493de25f4}, {0x0001dea4e9acf4f7,0x0008350db88dc32b,0x000da916c3e1f01c,0x00003e405d70ea04,0x000014b0d0b48658}}, + {{0x00093fc9920cb5e5,0x000742c7a3ac4bd4,0x000ec92355b44b1f,0x00055352a77293bc,0x0000ee06e881d378}, {0x0008173da621607b,0x000be9f5d290ceff,0x000f734ac2bb03e4,0x0007317945106aa6,0x00005056605825c4}}, + {{0x000920ce079afee3,0x00003789831f5945,0x0004a5ae5686e17a,0x000d462966bee8b7,0x00008a673a24e258}, {0x000c1f283141c954,0x000ece96e486bd1c,0x000e5fc783b2ecf4,0x000a7a8d3aa89674,0x000015ec10c6482f}}, + {{0x0004419805033800,0x000ad314b3921523,0x000caecae513d917,0x00080bb0b52f4e63,0x00007bf22ad2dc77}, {0x000e8a1e4306839f,0x00016dd7feaae761,0x000c778f11b3be96,0x0000f55fe728de74,0x00001fa0bdb4e007}}, + {{0x0005a316ec3f510d,0x0004029804758520,0x00030ebfd2c7e4a1,0x0006440e3c19c06f,0x0000b1c1f39a4b7e}, {0x0001a755dce364a2,0x000be58f5be3fe29,0x000fea38cb7b22a3,0x000e170cd2c30237,0x0000930967a4e17b}}, + {{0x00009de0c061c658,0x000c6dc6ed4487f0,0x000afb1ebcb014aa,0x000687c9bd1cb43b,0x00001bd8b5ca82d3}, {0x000b87ef01a17aff,0x0003f710063b1cda,0x000fc819321f37ac,0x00015b6a6c567642,0x00004753e7146a60}}, + {{0x000795ea15b0a44b,0x0006d958a958020f,0x000b675b58f37c8d,0x000ae9b3b7e89ba4,0x00004fb0c0cbfc31}, {0x000e639a7ff1f2e5,0x0003119614fbed95,0x0007151ab9880f5a,0x000cdb8eb6ff0294,0x0000bc5118cf868d}}, + {{0x00020554c20cea5a,0x000d74c4d69ad8da,0x0002d599bcac2776,0x000b502ccb22c162,0x00004ddb65408a9b}, {0x000f1511b4941b44,0x00042efba5882c4f,0x0008345e0e1ff19b,0x000fc3f5034363c4,0x00005542e3d5e29d}}, + {{0x000cb91349f7aed5,0x00003fca8420f197,0x000aaf6d83b2b5a0,0x0002b62c175ee823,0x00004dcf42185af3}, {0x000430727d6561e7,0x00046175b1e20ba1,0x000807db5879d5ee,0x000bcd57c4367399,0x00007a544560cd55}}, + {{0x000ff130105c0720,0x000f8dda7da4e6c2,0x0002d35c118f7a99,0x000c824c3018200e,0x00006a53ca0d9cc6}, {0x000cc1ef1aa1d9e1,0x00043a75b1e8aa21,0x000be9fdc3241433,0x00055a1a6d13280e,0x00006bd173fa8a47}}, + {{0x000b2452133ffd9d,0x0000b30f1a20fbb9,0x000a1f52a39a8b2f,0x000df7784bc97dd5,0x00006aebf57740ed}, {0x0007acb76ccdac60,0x000c1586ff273225,0x000de7dd1af4d36e,0x000c168eaa8863f8,0x0000045d5cf88647}}, +}, +{/* digit=6 [{1,2,3,..,}]*([2^42]*G) */ + {{0x000d574c005979da,0x0001ca40e350a6f3,0x000e2ecf9c2072b4,0x00044e5ca5c1568d,0x00008c8bf5c45153}, {0x000e555114df14a7,0x000d8dc5ec6b97ae,0x000a85418d4374a4,0x000f78054cc28f2c,0x00001cb9e2843c41}}, + {{0x000507903605c397,0x000e3142c96c8910,0x000923684f0843d9,0x0008938374493416,0x000032caa306a0a2}, {0x000c27061160170e,0x000b637fbaa3b2e8,0x000eda3acc32788c,0x000e0659cd818ea6,0x00002e9423a7e2b2}}, + {{0x000d39b0260e52a4,0x000c506533256967,0x000ca7954d42585c,0x0007b2cd9bd60521,0x0000fa20877c1ed5}, {0x000eff8e34a0bbeb,0x000bd4f6ef6460c1,0x000af848356b0040,0x00061378be2b24b1,0x00002278164a5531}}, + {{0x00075455922ac1c3,0x000c752b3f638df2,0x000de57c4a7b3ef5,0x00008b5e77b21471,0x00001682c10b34c0}, {0x00024f04bd55d319,0x000587b61c71c768,0x000a5089db6d1c08,0x000d3ea1db0903c2,0x0000c092172a84e5}}, + {{0x000ed5bc00cc638c,0x000aa1278fc2dd7c,0x00037f8d61a2015e,0x0003ac6e8e52886a,0x00004577870a7993}, {0x0004cce2c51211a6,0x0001c4c20498b3fe,0x0008db5e5ad9b10b,0x000fc330d87a4fd2,0x000098cd1059aca2}}, + {{0x0002d07e91b536de,0x0001beba09d64f11,0x0007c396fceb982f,0x000b235c157b2c19,0x000023c2d425b66e}, {0x00057d93f330d370,0x000179108deb480c,0x000199ce5b3a4c8a,0x0008d4702388decb,0x0000b019211b944a}}, + {{0x000a692840bb3366,0x000c4669fa7b24f2,0x000c9c3007c353bd,0x000f177a20d6fcde,0x000025fbe30013a4}, {0x000b61adbc173281,0x000f99515621a2b1,0x00020ff46008965b,0x00091c39690939c6,0x000082dd27d9717e}}, + {{0x0005035ea6c39976,0x000a62610bef5ace,0x00080dd3954259aa,0x000a398f18bb3f3c,0x0000910b95bbfc3f}, {0x000f51043e09aee6,0x000f47675665fce2,0x00072db61ced56c9,0x000e68b0e265acd8,0x0000982812f0e9fc}}, + {{0x00011c6ce8009982,0x0002990360d929be,0x0004ad59072bb175,0x000bc00c1931975a,0x0000ba2f548bfc1d}, {0x000eebbe490ebe0c,0x000cbfae11c07fe4,0x00003ba3712a0a4c,0x000dd7b197cf81e9,0x0000f7d4aa99e1c6}}, + {{0x0006bf43fd5684c4,0x000f40360aa192af,0x000546a822b26eec,0x0008fe7d960f3000,0x000007b3c44359ad}, {0x000e5fe249c82ba0,0x00072463744c86ca,0x0009162729e0faec,0x00061587f551e894,0x000033f93446ceb0}}, + {{0x000b0d18be82e84d,0x000daa582fef1e5e,0x000e921fa89967f0,0x0009b9ecf687d5a6,0x0000fee4cf4a37a0}, {0x0006965b493c465c,0x000bb635c03094f0,0x000f05e9f638b9a1,0x000724b666786466,0x0000caf6809804da}}, + {{0x000b690768fccfce,0x000cd835b362ca2e,0x000fdfccef402d37,0x00098f2efac0d0e2,0x0000fc9cdf09638d}, {0x0002b72d1669a8bc,0x000b9774ccbd2af1,0x00034870e33c536b,0x000ac970b21909fb,0x000038fa2f83df25}}, + {{0x000f02bbf81f3f5e,0x000dcf7e458174c5,0x0003c54ae0525a5a,0x0006c4a8d2aaba43,0x0000d9775dc606a5}, {0x000738ac0edb37d9,0x000dd6cc1f51d320,0x000600d7625fdb6e,0x000d761c661d1710,0x000031ec1f44dd1e}}, + {{0x0001d6219ee43f1f,0x000d70829d9765c1,0x0004be6e85cd57c3,0x0003bce26c91a398,0x000008d930a7b0c5}, {0x0009e5bc016e4eae,0x000831d43d2b94bc,0x000701155d391683,0x000714a86c5ad773,0x0000037762700b00}}, + {{0x000c9ecaa80ba596,0x000a08538e517f01,0x0008128af3083411,0x00014b370370f1e8,0x000025cc3dbf1dec}, {0x000666c01ac3107f,0x0006e5057ac3fef9,0x000be5df7b2a8d57,0x000923c0f2629992,0x0000579c8e5f0353}}, + {{0x0003d931341ed7a4,0x000c67b59d49b8fa,0x000b8c4a44223272,0x0002e3fdcb194783,0x0000e413c022d130}, {0x0009127e17e44ceb,0x000483b3adfb6d99,0x000aa96caee86bf7,0x00047d46902fe625,0x000073540e595aae}}, + {{0x0001d7b1b4a158cc,0x000d67e2a3693280,0x000d9f197e571c99,0x0000ad80cb76c010,0x0000308c289f167c}, {0x0009dd3eb7958f24,0x000bf00879b1a6ef,0x000df0636a7226df,0x000eed2cd0b3627e,0x0000efbce6cbbc37}}, + {{0x0002a058d6990216,0x0009172566e375f9,0x0001ad23a586d4c7,0x0008abd78ca5f176,0x000050d86fc7465a}, {0x000d457842ba251a,0x0005a22349337a4e,0x000aad6576b65e3e,0x0003690f1543b731,0x00004cefe996bfec}}, + {{0x000da909f47befbc,0x000a81312d13b587,0x000f1cefe6562e9f,0x000cf5e691ea59ef,0x0000c30477ac5fc4}, {0x00024610b0ffd3dd,0x000a8b355956a163,0x00024ec24a1f16f3,0x0001298b148d5342,0x0000c834e7cc9770}}, + {{0x0005e75b2c69dbcc,0x000843c3da6c7bfc,0x0009102713aa77a2,0x000c551e0df03cca,0x0000bd5ca4b3806d}, {0x00058076db476cb9,0x0001cf37a31ee1ca,0x0001af416fde15d6,0x000db5649af520f4,0x00006c5c5b20d342}}, + {{0x00043b7eb4ceb9be,0x000f6e77371a155c,0x0005d43af2e99300,0x000d713d2987da67,0x0000f2bc1c16599f}, {0x0004b7b9342f6b2e,0x00019c8e71f09689,0x0001f3efc201eadf,0x00070413479d9f4a,0x00000f8a743502a9}}, + {{0x00044b6b3eba40cc,0x000901c1e0d0eafd,0x0009d505ef9739f2,0x00063d4091471a61,0x000015f9c975d7c2}, {0x000728583afbe332,0x000ee4f1e0925be4,0x0001a9d11a3b6d6a,0x0004d2c76526b975,0x0000ec5b26dba4ae}}, + {{0x000f4d902f6fb8d2,0x000176912164eb66,0x000ef30004063c56,0x0003f0cb7050c180,0x000088d1c340aa5b}, {0x00068d607806fd8f,0x00045bbbf50fe87c,0x0008d6627b2f7f9d,0x00013a35972f3aac,0x0000854777500e8c}}, + {{0x000ef6c872b4a606,0x0003e613521bcc50,0x0003e15d1ab2a34a,0x000511d9c5c19098,0x00001dde5dfb9905}, {0x000f6219f2275f33,0x0006151d894be417,0x000b0bdaa0750c8b,0x0009bd45b04ab978,0x0000bfd9fd475858}}, + {{0x0003e30ee9120b67,0x0002b3a4743ef101,0x000d14d9e2b51af9,0x000d327a96ffae48,0x00001dc0dbed98a1}, {0x00062d20180cca4c,0x0007035689639149,0x0007bc4441ae6067,0x000c79ccf227b143,0x0000650c83c89962}}, + {{0x000c7ddfe7ccfc4a,0x000c7b929d4823c2,0x000783c33f925c89,0x000759a460f74b06,0x0000c2c8d4a45904}, {0x000b407b807bba06,0x000d09ff8f3afb40,0x000ef64a49d1e362,0x0004b2433e9681cb,0x00007ece5fa932fb}}, + {{0x000a99b739f10e3a,0x00095f5259256900,0x000e2d041c3341ca,0x000ddd4e18a626a9,0x00005a83685c9580}, {0x0000c819d7de3cd3,0x0005f062cf9cf347,0x00010edb0edf0258,0x0001aec43522fac0,0x0000031413543a4b}}, + {{0x000e02adb22b94b8,0x000921eaa45bc792,0x0001e1c63993d8ae,0x00088a0aad6cd3cd,0x00009529ca845ce6}, {0x000e3aae572a2530,0x000802a21efb2cce,0x000430358e02b643,0x000504a7091b6ec9,0x00006d1b1fa9d7db}}, + {{0x0006d32c47447335,0x0000e79f9e345884,0x0000ef6ca40517c7,0x00003edf65655f13,0x000026e448941f35}, {0x000bd177ee4a9765,0x0003421363d18467,0x00069e0411d9dc91,0x000f5188d24c33b0,0x0000eb5da0a7cdf7}}, + {{0x0003cb1197b994f7,0x0004b843eae91c0f,0x000097ea53c95a6c,0x0008670766ffc9a6,0x0000bea40944723b}, {0x0001f734db378f9b,0x0000337b77acb48e,0x00024ad4670025b0,0x000a84e43dc8e7af,0x000098a15acc6d00}}, + {{0x00038ba2743b0043,0x0007034415ee3adc,0x00062d05ab1c7f4f,0x0003b6ba43df8f1e,0x00002618905cd76a}, {0x0000bb5a23a0f46b,0x000aba01918c2fbd,0x000743f945bc971d,0x00022ac801d94ab4,0x000094df65f176ae}}, + {{0x000bfabaf95894c5,0x0006d76b2241aafc,0x000dda48b7b9bdc0,0x0004df9af983625b,0x0000977faf2f3fcb}, {0x00042ef052c4b5b7,0x0000967591f0bed0,0x000f24ec79fe87f7,0x000f1b589c73ca22,0x0000d37fa9f564a9}}, + {{0x000841a15562627c,0x00030243b0342710,0x000c686092c01a61,0x0001f55d135c562b,0x00002ca17164b03f}, {0x0006c2d3eb81d82a,0x0009ef6df13ec996,0x00072b43bc02abf4,0x0002afd7b34bd78f,0x0000ff6218fd60c8}}, + {{0x000726c8d55b9d21,0x000989e9bffb0aa5,0x000b9e72adc0adbe,0x0001118097549cef,0x00006755712adfb3}, {0x000f984f26847f99,0x00074fb30cb7dd8b,0x00071ef9cbcb8e38,0x00063f31fd32a751,0x000077f3fc7c89b3}}, + {{0x000af2bf4babda01,0x000de7113c8e116e,0x000def526feab68b,0x000a02c1e3f064b7,0x0000ac30885f0b3f}, {0x0006e7b40142d9d3,0x000300921c0b1c5a,0x000a116a3839b560,0x000e6d18f301fa36,0x000080e1107ffd9e}}, + {{0x000ead858854be16,0x000e6bd4d49d7945,0x00029c2ef4111c12,0x0006f58ce3b1ec3a,0x0000356d404ed361}, {0x0006a8f594d320e9,0x000d6651ccd29f0d,0x0008fdde40989316,0x000bbbdc32117a0f,0x0000abe5cc6326a9}}, + {{0x00060fb9723f6711,0x000d6f3d593ccff5,0x000ba069621b2a12,0x000384d4cb18da24,0x000086e2220d3543}, {0x00064e088312c29f,0x000827dc7752722f,0x00085ee8994282a9,0x00069f72467bbf5a,0x0000435c651e1007}}, + {{0x000153943b3a50bb,0x000b6a53efbcc9ff,0x000b0c5b77132130,0x0004cb81bfe063f7,0x00000179a7dfea99}, {0x00064b3c85f455b2,0x00086f6e006212d0,0x00075d6d94725932,0x000cc7d64e590bb8,0x00002dd6225cd92b}}, + {{0x000038eb9c3bd6dd,0x00062bba27c8b658,0x00062c45d00cdb0d,0x00007c3c68133710,0x00008515b8cfd334}, {0x000699e8cbb5ecf2,0x0008a608d7d8cb8f,0x0003e00db8c4347f,0x000d190c11850abb,0x00000a8dafe0cb49}}, + {{0x000148045ee2f40e,0x000e616b60cfbd78,0x00049a8c475e354a,0x000535fe0b58a18d,0x000040e94e3da359}, {0x000a59f62accd765,0x000a3c762837bd4f,0x0008c277b05cf466,0x00074065abda9944,0x0000a9e01bf98b13}}, + {{0x0007798326aad8d2,0x0004a396f7e79d45,0x00053e292bdef495,0x0003e74fb274a2c2,0x0000800bf0a3cfe5}, {0x0006d3144438fd49,0x0002ce259f9a2242,0x000f66264ef23392,0x000faab188503c03,0x0000e5e7f140f9fd}}, + {{0x000b76c5fcc1aba9,0x0007c9b5bff8565e,0x000b6d3faea63254,0x000c1ac587c087aa,0x00002b639eafce39}, {0x000e782953b135c8,0x000dc25268ef0706,0x0000e74697308912,0x000e74d99e92c709,0x00003b90f531bc35}}, + {{0x000b3d0244975b3f,0x000721965d724750,0x0008dc751f3a4435,0x000fef279c67749c,0x0000f18cdffc23d9}, {0x00038332028e2472,0x0002d3bfbc79c401,0x000880a8496e280e,0x000151d60417bdd0,0x000063c9f3d4a568}}, + {{0x00015b32d2ce8114,0x0002b8291d2136be,0x0009fcfdb846dc0c,0x000b9a1cfa0ecb78,0x00005a0beee17535}, {0x0009f0796d69af13,0x0008299ab6dcec8e,0x000e2e09f31a7c5b,0x00054ba36d45eff9,0x0000cf49ef20cee9}}, + {{0x0004cf3086cff9bc,0x000079a3360f6be3,0x000bfbd1d88dbd49,0x0005515e96b8cc0d,0x00001e5f7c08b7e2}, {0x000b21428819d98c,0x000bbaea9dcb0547,0x0001d68c8c770dd9,0x000ba7eef0d4c704,0x0000c2b9818d3cb9}}, + {{0x000bc76fe86c6072,0x0007302a9a957fc7,0x0004dab636b7b933,0x0007bdf948dc27d1,0x000049dd198fae04}, {0x0006584a981a2029,0x0007aa893387e835,0x0005c72093531dd1,0x000b598be11f90c8,0x00003d2fe1f72a52}}, + {{0x000bfe2ec6d6b975,0x00046d0aa5de8225,0x000779f5f9cf6d6f,0x0001f3411459cb54,0x0000649cddbd6aeb}, {0x00035793f26ce5a8,0x0001d50f431e3213,0x000b84c6fc289a10,0x0004d6d59dcfda73,0x0000497381a6e3ac}}, + {{0x00055e6f2606a828,0x000110f2fb57b51e,0x000a4e37ce25f706,0x0007062cef6c2ab1,0x000064e359dddcf2}, {0x0006b187ce573162,0x00001a96b23d479e,0x000f16df72cab250,0x0008b5cd4898628e,0x0000056538d0f375}}, + {{0x000865ef15d31016,0x000a01b553d7a7df,0x000d1429480c5533,0x000cd5d66e19974e,0x0000620742013c0b}, {0x000d9c4edc454181,0x000eb1cc4a9d21d1,0x000c462f0005b859,0x0000c7cf01f630a1,0x00005d06cf402682}}, + {{0x00024ee3484be472,0x00062a0c902f9f7f,0x0000bc4532ff33e9,0x0008dae0bdf4575a,0x0000378dfaf3aa23}, {0x00020ec856720f22,0x000b767972912724,0x0008a15582ad9d95,0x000aa8b1242cc676,0x0000e287f8b7cc86}}, + {{0x00073d0990cecaa7,0x0000f75d40807968,0x000f0cd84ade55f8,0x000d01b645eea321,0x0000a1efa1024e17}, {0x0008420037cc0618,0x000055d43e12f685,0x000218710682e05f,0x0002fbd9c3699427,0x00005cbba4dbf7cd}}, + {{0x00097297a3cd22af,0x000b5a628397726f,0x0003165ed9f8cd5d,0x0003d327b93ab9c2,0x0000f5f5dc002282}, {0x000e4b5654a446d2,0x000f477257bac1e4,0x000766a56d1a9496,0x00074a4387ba94de,0x00003608bc8721ec}}, + {{0x00022d76688c4d4f,0x000117373abd16a5,0x0002efaa39d6b428,0x00017fb62f07acb4,0x000073e00f8d3b90}, {0x0005fec49421c3e1,0x000b2dcf26783617,0x00020f09fc4e44f9,0x0006cf66df436b72,0x000072755fb4aa8b}}, + {{0x0009d57446139cc3,0x00022fe0208fbab8,0x000e5d3990a0a6e0,0x000f0b9dbb63e211,0x00003ecaa12d8977}, {0x0008b21f7c426641,0x00029b65d08a5959,0x000502526b3e91b3,0x0002a8f35822eef4,0x0000dcf0176820a8}}, + {{0x000598f3d589e023,0x000f81d63d2c50f8,0x00071cd07df0478f,0x0000cd5b8068bd15,0x00000c3aa5007967}, {0x000fd4b941ade7f2,0x000c1279001125e8,0x0003f9ff03d1debd,0x00069c45b6dcbd3a,0x000082736a4993de}}, + {{0x000a0c3d41d3bd3d,0x000837a26bdeef69,0x0002edf9fb533b8c,0x000af012801d97db,0x0000c4a826ab1877}, {0x00058513d590dbeb,0x000b3e4e93576c1c,0x000b3337484632f6,0x0002e6236d36b779,0x000046833e44bbca}}, + {{0x0003913f7fc0586d,0x000696bf47193789,0x0001855dc385315f,0x0001fda2c56293b3,0x00001416d4f54906}, {0x000ab7851047213e,0x00011040c996beb3,0x0008b1d0c447f6e6,0x00022df06d310d63,0x000028a41409ad15}}, + {{0x00076cb82003f867,0x00069bcdbca3685a,0x000a4c455610d07f,0x000eebfff660219c,0x0000df39b883ea10}, {0x0005f96e22db2188,0x000a88a34c44b925,0x0009f92768cc6d9e,0x0003354d4ffb8685,0x0000fa15eb2d0d07}}, + {{0x0003845cf2c24b5a,0x000eb2f9c3badf55,0x000a7ceb389f66a9,0x000685ef22b5b9e4,0x0000ffef809ae134}, {0x000e1c68eb8fac22,0x000b08aec98e3e53,0x000a43bcb93c1e4e,0x000a5196b91ec532,0x0000dbfa947d2d74}}, + {{0x000d190ca84bad76,0x000f4d58e65ce065,0x000cb6e31fb13919,0x000c3edc41718bf1,0x000088969f066d05}, {0x0004ce721264d452,0x00095367532bd4f9,0x00045a39dfdfb65e,0x000f3b3b1be8b109,0x000029f789c4b8ba}}, + {{0x0001f3e6f49f15d1,0x0008807f0792d8f4,0x000a6e867678ce82,0x000c89b69ace82fc,0x000006451aee01dc}, {0x000f7f019fc32d29,0x000c200c52d21bb4,0x0009ea44564633df,0x000704ff13549aad,0x00009a3bf518b323}}, + {{0x00025a2534d4dbc4,0x000182a2fea30c96,0x00030fc1a45b8f1d,0x00073436ec21a1a5,0x0000bac9c2ade5bd}, {0x000d76a7b4e35876,0x000db182d9e35996,0x00007f13d0045cde,0x000a40baee24b912,0x00006452e97f7345}}, + {{0x000b0549f950cd0d,0x000107fdd07516e5,0x0002496639cc72fb,0x000cca9edd61e766,0x0000e4caa4ec043c}, {0x000f57a55c7ac17f,0x00032a85e24d11b1,0x0006081e7779cbd4,0x00064288030f86e4,0x0000d4a60337e20f}}, + {{0x00064880a750c0f3,0x00031e548e83cc7a,0x000110f0539bacfe,0x0005880d418c760c,0x0000e4daa4ce1f11}, {0x000e7b55ffc69ff6,0x000c320531272733,0x00022df9446f147b,0x000b7c285b2434d7,0x0000a444f6646fc6}}, +}, +{/* digit=7 [{1,2,3,..,}]*([2^49]*G) */ + {{0x000465ac3f16ea83,0x000d82f1d11c7a1a,0x00068a172115a461,0x0006981767dd956c,0x0000392f2ec013a4}, {0x0009ccde526cdc7f,0x000b32292b81c7a9,0x000d391988e537fd,0x00052c86d8cf69a6,0x0000fc5ff4414468}}, + {{0x0004f7ea90567e6d,0x0006e6ae5cb797b1,0x00010903d513257b,0x000723b5454a3c9f,0x00008d2c9ae39bc3}, {0x00093246b29cb44f,0x000c87c8cbac38da,0x000918e42b540a21,0x00014dabbfe43501,0x0000ffa707b46c36}}, + {{0x000e3f1d4e353b7b,0x00043f46b0a00ce4,0x0004b73fd062d8a1,0x000ffcb408d5ab57,0x0000c41d1ca83273}, {0x000e1e76be77800a,0x0007256550313538,0x0009b331a71fe8b3,0x000f727cd916216b,0x0000d825d0c5b388}}, + {{0x000e05b1cb76219d,0x0000a1567e7e56c2,0x000c4c9100ec0bf9,0x0004d917076f8661,0x000067b085c8abc0}, {0x00004595e93a96a9,0x000a6bdc249a9fb9,0x000dd0bb77526c1e,0x0006947d44d367ec,0x000053999182dc0d}}, + {{0x0000ee99e240d181,0x000ca4b944666136,0x000e5325c057cdca,0x000e3bd7667cd12f,0x0000fa297b531974}, {0x00081e7db083d762,0x0006d206bd15fa40,0x000c19f8c31993be,0x0003576949269b14,0x00001468d72c9d92}}, + {{0x000c583a4c506ece,0x0007f1acfe972ccb,0x000f1aea2957ed18,0x00062cabaed83312,0x0000f2a6cb563253}, {0x000de428e195c43b,0x00095e6050c6130d,0x000686a5dc842025,0x0004a77da972a708,0x000052999a29508b}}, + {{0x00090b910a5a8bda,0x0008696864dad9f0,0x00067dbc1ca91d24,0x00064bee6a93be3f,0x0000cae6fbb95f47}, {0x000c6e0d21411a01,0x000fca0a4ad81563,0x0008c803028fa787,0x000f07c524491c90,0x0000257ba0e5c795}}, + {{0x0009167ceca9754a,0x0005ab7939a083f4,0x0003fd0bf426d2cf,0x0004e18555e35572,0x000096e6d0764f14}, {0x000a8dd87880e616,0x00058508e4d54768,0x000b65e151554381,0x000f9fa9d7e772b1,0x00003439dd70c302}}, + {{0x000802fc14e35c23,0x000c1341333cb93f,0x000d4f36271735b7,0x000c8dd3a2510416,0x0000f4d069bef433}, {0x000ae01f78f5a7c7,0x000e0c4eed070d83,0x000e10f8350a8ffb,0x0008e1574f890676,0x0000d0809670ddaf}}, + {{0x000f8e1698e04cc2,0x0002f69005c8b11d,0x0003c6179877be20,0x000c0512749e8c4f,0x0000dbc9d0a9853f}, {0x0004f939454d9370,0x000db4800e1b187d,0x0005e68e8e682ce9,0x00085ba9129ad816,0x0000fe29735be7f7}}, + {{0x000f40c5b9e02b75,0x0001e5ee04e85303,0x000d6632ba37c969,0x00045b0f46cc2034,0x00005ef72b2e6ac5}, {0x0005c1f7b91b0620,0x0007bb33e821abec,0x0009f41170a79e1c,0x00075abb04b4283a,0x0000de1f28ffd2a4}}, + {{0x0009ccf3a4434b4b,0x00006a7954dc3101,0x0004972a7a345811,0x0008dcf9dac80de3,0x0000043d05524f6b}, {0x000319e11137b1a0,0x000eed5cc03f021c,0x000ea5ad400a754c,0x0005b60aa2c794cb,0x000093e67f470c01}}, + {{0x000fee9c97e3f6b8,0x000436da746172cd,0x0009806b9c10bcab,0x000f46bb02d2fcb5,0x00005185e8a21de6}, {0x000931f0eb6c4d46,0x000d74fa5b0439e6,0x000be7eb84d4440b,0x0009bbf418786e34,0x0000380e521fd725}}, + {{0x0000351d598d7107,0x00065b3a4da420ac,0x00071de1f272c416,0x000b0f6b82fe1aca,0x000046e79f348f54}, {0x000c7364b573e9b5,0x0006ad4b50406e7f,0x00098d87b75d03f4,0x000da10c1cc36d0b,0x000013ba3f16f472}}, + {{0x000af26abb177dde,0x00079891d56479d0,0x000232173f82ab56,0x0006184b6768a972,0x0000fbb3bb16c1f6}, {0x00011dba6d183586,0x000750916d3ab29c,0x00088e290519e279,0x000a7f74dc18f091,0x000048e86e3f8b0c}}, + {{0x0003145983c38b5e,0x000b837abc8b859d,0x000ff7be6b14f176,0x000a594793fb9dca,0x0000be5a56015a66}, {0x0001dcd9f87dc596,0x00039bdbf5607cec,0x000eb32577c595cd,0x0005fcfb543b2226,0x0000908064724c93}}, + {{0x000440381e9ede3c,0x00040af6df0a7f2e,0x000073b11243c389,0x000a61bc605bb11c,0x0000d06a5427a6a4}, {0x000894949d4e2e5b,0x00069af668802916,0x000a8503533649d0,0x000f4b0fc0c885e9,0x00004e52114ac410}}, + {{0x000b70678a6513b9,0x000a0edb1943dca3,0x0006e2dd892ea4a2,0x00089372642216db,0x0000b45d0b52fd57}, {0x00070dbc69d11ae9,0x0008bc57595f114e,0x00077c2721477dd1,0x00059c2c2208b4ec,0x00005c5b4d86b68f}}, + {{0x000fc6342e532b72,0x00027ae35290b8c4,0x00001ecbc386ba42,0x000fd6db5dda42d2,0x0000353dc8bc0e38}, {0x00085ea68f7e978d,0x00015ad6d11f9a0b,0x000f6886d96ec568,0x00014c8e279d6ce5,0x00003fe03ce0cb19}}, + {{0x0001fa47ea67c77a,0x000f43ea810cfe54,0x00001d374952bd2a,0x00036dd91fef568d,0x00003a1c621af113}, {0x000d5a9c7ec6d792,0x000f1225c3425ad0,0x00069601bff7038a,0x00047ce03c6689bc,0x00005059bc765e87}}, + {{0x00065b2f2086fbfe,0x0005a6916078fa49,0x000081d6cf6840ea,0x000644f7ac762070,0x0000600da3295328}, {0x0006f63529b8a80f,0x00073d7d6f3e0191,0x00064ca7ce80e485,0x000b0b39eb0fe8d6,0x0000017637cd7b43}}, + {{0x000c80676cb25661,0x0001a24892d99a75,0x00008fe458f76acb,0x0007d86ae7b9cc1f,0x00009ef73297a490}, {0x000ab715f228bf03,0x0009517032d72db4,0x000abe3c0f3cdea3,0x00025bdb1f482edc,0x0000baf76b4eb863}}, + {{0x00065e010089465d,0x0008be77c596d490,0x0003dbd953bab5d2,0x000498a636c3a619,0x0000ef5d2958246e}, {0x00058b9286b24756,0x00096d80862bb22c,0x000992388a0b9393,0x00014bd002c83af0,0x0000de01f9c4acbe}}, + {{0x000688eadd70482e,0x00099b4a4e8a6aac,0x0008a6eef708de92,0x000c4295b6dd7375,0x0000a4bf353525b3}, {0x0001f2c87912868c,0x00052f09297a1004,0x000f3860ab1b1be9,0x000f4a59ae23c5a9,0x00004f0f83a115dc}}, + {{0x000cca397f6306a7,0x000df8a3a4b03c7e,0x000a1d8a2744c44a,0x000577f9cd13a0b3,0x0000cad0a1ec256b}, {0x0003fcd33791d9ed,0x000ca4b2e05fea65,0x0007affa29cc2a05,0x000441a3b391dcfd,0x0000db7091f86b05}}, + {{0x00027bf8538a5c69,0x000cf9abff17c71e,0x00071e3da195c63d,0x000fa06d3152851b,0x0000cbdfda88a680}, {0x00076ca849d7eab8,0x000abc2732719db8,0x00008dceaebe2764,0x000b6fe63357e3f2,0x0000c5bd833d65b1}}, + {{0x000b4f59837fc0d8,0x0008279cf00fccc3,0x000df39909b641ba,0x0006b0f428243ddf,0x00003a594c482078}, {0x000451a526c45028,0x000deadb3f93b712,0x000ff0ccd9d39438,0x0004c37db261e3e9,0x00000344e3d607af}}, + {{0x0000d7c2fa4f1264,0x00051c99a2327590,0x00025e0c308a3b86,0x000edee478b6bfdb,0x000082cc2c2b1db2}, {0x0007e645f321bb80,0x000b9a8005b437df,0x0008c19588a93821,0x000d0a3fa2f10ccc,0x0000d3322182c269}}, + {{0x0008119e246b0e64,0x00049349fd1720ab,0x0001aa100b39781e,0x0001689293231eb3,0x0000b779c97fb032}, {0x00019e1c84705003,0x0008dc4c869d4b3f,0x000a6bbcc45b7efe,0x000bc1ab84f38aa1,0x0000b59cb15e2fdd}}, + {{0x00014df3fd165e81,0x0008f61f8811ba55,0x000ef9f00499fd6a,0x000e8a62cd1fe0bf,0x000020a4bb989ad7}, {0x000d0955f4a5ac5d,0x0000c5a7a2f0f2ff,0x00017baf1cfd174f,0x000089042301ba9d,0x00002fa487b47f22}}, + {{0x0009efeb1dc77e1f,0x000831c996829cb0,0x0006067bbe956693,0x0004559c5469016c,0x0000d37857551c24}, {0x0006cbe81796b334,0x000618e87f8b2b6a,0x0001b01b462d550f,0x000e11db763e1c7f,0x0000b93cfea5b1b5}}, + {{0x00052381d531696e,0x000fa8cdde69b934,0x00086afc757201bf,0x000ea7fde922519a,0x000030438969d35c}, {0x000c1e18555970de,0x00084535935e7608,0x0002ea38b8267dfa,0x0008b4f4c60a5732,0x00000bf7978604ef}}, + {{0x000ab28c06fece4c,0x000dd4e7b49d1a0d,0x0006dab28d405991,0x0008fb0542b6d270,0x0000b228da469161}, {0x0004164107d1cea6,0x000370f5d8f1224e,0x0006e41cdeb9fdab,0x000146602ba3860d,0x000076a72c5fb1f7}}, + {{0x000dd984d6cb00b1,0x000ace2e8d7cffd6,0x0001c7936cef9c5c,0x00072e91bbf5d764,0x0000b95b230fe8f7}, {0x000a92ee8ac25b1c,0x000b7a18b7c6f765,0x000cc8966ceb04cf,0x00000367944cef0a,0x0000bb3c958034c1}}, + {{0x00071a1a43ff93c7,0x000731e358a99c99,0x000d9bc825bc2db1,0x000051d5b4862ea8,0x00000ebfbfb9201e}, {0x00064c792871591c,0x0005f42d0219afdf,0x000d8f03cea5bcae,0x00033c1e536c552a,0x0000d6c3f4e676aa}}, + {{0x000f6230bca6de32,0x000991e706fdbeb5,0x0009059d4dd20dd9,0x000c4e70b3ff9dac,0x0000d7b29029cccc}, {0x0000a59ce98840f7,0x00001410680a8a09,0x0003379a5a5d947e,0x000155d9ae346a92,0x0000dbc84fa228a3}}, + {{0x000d91654a1aff29,0x0009aa78fb9bfd40,0x00029f95eabf318b,0x0007f9c0152ed830,0x0000fc1dd78558ad}, {0x000791513595c172,0x00086f62b3a95fa5,0x0003055b0b950466,0x000125707b5b24ff,0x0000e995e35ba84f}}, + {{0x000cf697e9bbcfb2,0x0006c86d96e387da,0x00095a75c95d0c1d,0x000f2145726e3c2d,0x0000c3c9001ccd27}, {0x000b5616c973f57b,0x000bf52216431dea,0x000ef79d4108b7e2,0x0008c5afee9859c4,0x0000d62b88af0d4b}}, + {{0x00029c4197c75d61,0x0002a7076febb4dd,0x000f2df11266a6df,0x0009ec8512d0ea4b,0x0000320c24f7b0cc}, {0x000e0e101a595966,0x000b8ff9aaac6bb1,0x0005aa6c98317c5b,0x00088f05bb405e38,0x000013439c1ef079}}, + {{0x000049f16a66e918,0x00007a1b0e0dd730,0x0004c28eae97f282,0x000c61c131e00330,0x000020ab732d26ba}, {0x0009ef9287144234,0x000a6db10cb2b2ac,0x00086a4cc54ecfff,0x000d494781476ef8,0x0000b2c87b61b2f8}}, + {{0x000cd200a44295d8,0x0000d8c6b044e857,0x00096757c707d7d2,0x0007142e8521f9f5,0x00007448f03e7b2b}, {0x000bc455ebcd58d0,0x00099122d3c113a9,0x00007664279bcced,0x00067d3c6442479e,0x0000cf227782df47}}, + {{0x000aee471d444b66,0x000f65084a1d5e61,0x000d3eaf6211236b,0x000bf51e15bc9a4f,0x00008df2c350b622}, {0x0004f0f59bf4f363,0x000ba7f34d739e67,0x000497b1df883669,0x00003b948ac1b831,0x000023b925d81067}}, + {{0x0006f4274082008c,0x000a08482bcb2215,0x000173479effc521,0x00048f9c6831bf12,0x0000aa2529084739}, {0x000102a8f1b3c4d1,0x00011d9bec0d84d2,0x000a546efcf64dfc,0x000ef1333febad78,0x0000f621ec38b73c}}, + {{0x000d627373386156,0x000aa1d8edf66aec,0x000e86b66162082a,0x0005db233a811919,0x0000023a2523299b}, {0x0000c3abbf04b89f,0x0008be749a44f5bb,0x00013de3b6735eb6,0x000ccce0e058c547,0x0000df2593f1c3d4}}, + {{0x000414efdd236675,0x000aaa2015ee1b8f,0x0009625ffdd52aac,0x00018bee31b517bd,0x0000ec9322dddb59}, {0x000ac85a96f5294d,0x000291a0666abc73,0x00008ac4282aa5bf,0x000dfb79755810bf,0x000021cdfd6591ce}}, + {{0x000b57b67f8be10c,0x000b96ffa726918c,0x00032de93365d1a7,0x000d016435c50465,0x00000fc5e10e674c}, {0x000fcf89cbbb1423,0x000a7fc506926e51,0x000bcae221d436e5,0x0003b8466bffff3f,0x0000148c2fe2d55d}}, + {{0x000fdc9233222fa2,0x0002c419fb6b52c7,0x00025497789ff109,0x000ca71cd6db9925,0x0000e85a1613cf12}, {0x000547cdc810bc9e,0x000ebd257c22add2,0x000d6b19bea3f458,0x0001a5842c1fbe27,0x0000d07e6b5f4048}}, + {{0x0001d4286d2e0f86,0x0001ae8a9fd56ada,0x0008c1b49e592012,0x000afea2c936af70,0x00000f30fee8b4bf}, {0x000ad06858e6a61a,0x00069fd374d06637,0x00088defbce4c776,0x000b6599d54b2d71,0x00008c9d251956a6}}, + {{0x0000f5eb24fe1dcd,0x000d9b73f24c58fc,0x0006507059eaf9de,0x000728d90d588b33,0x0000e5b62c67f2ec}, {0x000cfaed3c2b36ea,0x0004634435da5c72,0x0007ee145868c19d,0x0005b0e8605f93e1,0x0000a60c4ef17a5d}}, + {{0x000bfd23b60c472f,0x000bfb1d3049bcf5,0x0003895c9af4ef13,0x000821473f44fce1,0x000029b382ffcbc9}, {0x000b85373efaef69,0x000c18c96f401bfc,0x000191e24cf56ac9,0x000dc247adf1097a,0x00008035f454f8a8}}, + {{0x00071b91e750c846,0x000bfdc6c469f40a,0x000bc19c1c57f7b0,0x000db7e9a0e79c6f,0x0000b0f588a048eb}, {0x000d084a07c4e9ff,0x0000bb27de145d3f,0x000e08dccc383011,0x0003a21e4929fe33,0x00004a5ad2530bb7}}, + {{0x000c2bf490f97ca7,0x0005f7a1ce18de86,0x00044478d288f09c,0x00003fc64bb88618,0x0000840fa433eedb}, {0x000fdd25a631b378,0x000e247c8b7d1269,0x0001c626694761f1,0x000fa77c0c2e1748,0x00005e16ea2bdaa5}}, + {{0x0006033924910480,0x0003f4d402d7ccd8,0x0006a865c0c2f696,0x000a876336f7dfdf,0x0000a2a463cb5c02}, {0x0009be7bf2f12ee2,0x000246bad988b0e2,0x00023c1d7f0a2200,0x000c9807f87e0391,0x00001669c55528a8}}, + {{0x000980392f145296,0x000c03954df3186b,0x0005a46f6d3d056c,0x000557cf03fd5817,0x00003e34ebe71558}, {0x000edee5b80cfa5b,0x00002401dbd1e13f,0x000a9d667e872a12,0x000692a2657616e8,0x0000c8da4b7908d6}}, + {{0x000b9bb1b703e75a,0x000634338363370f,0x000ef7bff6773b18,0x0007d978dad378ec,0x0000ac787ee39567}, {0x000ea8b0437164b0,0x000073fe795e4801,0x000e5eb73f430ad2,0x000c0eb164154d8e,0x0000884ecd8108f7}}, + {{0x000c0965f5206989,0x000db4f7b8d90e6e,0x0005a68b9640631f,0x000f4e02fd34fca3,0x0000c5a4b66dd40c}, {0x00054bf80b6783d1,0x000e2a320a109494,0x0000a39b280e701f,0x0007db7d1a564a1a,0x0000436d53d42058}}, + {{0x0006e6d6556c3629,0x000052455d7ef509,0x0007230f9bc23a3c,0x0002fcaa7aee5480,0x0000ba1cfa6b2ae8}, {0x000057a99c5d7062,0x000b642315c9833a,0x000a72f128be85f4,0x0003cdb083179a66,0x0000fc77d5dedcc7}}, + {{0x0008a805616ee306,0x000f87ab108322b8,0x0001270cdfb09548,0x0009ab2ad6ab0d51,0x00001f6c57ac924d}, {0x000bf7290aecb08b,0x00085df784a4a0f7,0x000af1d03849f87c,0x000acd77c79c15cf,0x0000bf9f6767463f}}, + {{0x0002c65765ba5436,0x000be2ea60dd9150,0x00043ecb318ce3ca,0x000eeb85cee6ac6e,0x00003e4e910c8f2a}, {0x0004fa3c85932eeb,0x000a9c90c44d2623,0x0008a50f696883e8,0x0000de79b9e738a1,0x0000fc62b2aef042}}, + {{0x0007d906d3e1fa9c,0x00083e05b8a3d22a,0x0002b9c011711561,0x0006a16a0c9926bb,0x0000739fcc7a07e7}, {0x0009157165e439a0,0x00026a9063d8540e,0x000e927a306353a6,0x000e7f84d9559461,0x000013b9b26e2e0b}}, + {{0x000ec3b973497f10,0x000e093ebc2d4fea,0x000af058315c0f94,0x0003406af5f22733,0x0000c2af206c61f3}, {0x000bdf14457397cb,0x00071abcbae0d25d,0x0008153062e8ed01,0x000d6833010938c2,0x0000aa9933898c6c}}, + {{0x00031823b0ec7de9,0x000ac8df05df0851,0x000c3b6831e1b822,0x0004cdcc14842fa5,0x00008fe977eceba3}, {0x00016c20d5e88732,0x000a9d0d427dfd23,0x0003fc961e48d839,0x000a5e295b221862,0x00004ee56e7d46fb}}, + {{0x000a55b91e4de589,0x00054fdea2889184,0x000dcc943a7488ca,0x0000fc1723862ea8,0x00002d762b3149dc}, {0x0004a12091ff4a95,0x0009bcada2743c44,0x000d8eae2581113f,0x0001ea69de0a4530,0x0000e0fcd862f6b4}}, + {{0x000ea68c094dbb56,0x000e7968d4106233,0x000b3002db77d062,0x000d57de719bbc58,0x00008e7dd3d9dc49}, {0x0005740013a5e585,0x0006ec9e3c1b8d82,0x00099b6ab2131174,0x0008f1bcb0a2a77c,0x0000c48a3b412f88}}, +}, +{/* digit=8 [{1,2,3,..,}]*([2^56]*G) */ + {{0x0003e91991724f36,0x000bd9cbd686c791,0x000d4fc1e5eda799,0x000d547db595c763,0x0000b63b80c0c4fe}, {0x000fc697e5fb5166,0x000a70f1c9646ea0,0x000a92ca5737708b,0x00067a3628745f11,0x00001f37958fa869}}, + {{0x0009b2caa6650728,0x00046fd324ef9af3,0x00027bd3178322fa,0x000aafbd153394c3,0x00001d5f271b129d}, {0x0000c42f48027f5b,0x000bd536e717c72e,0x000369d0faa40cdb,0x0004e6445a657a2d,0x000003bbfc59a7f7}}, + {{0x000c4180d738ded3,0x0000b0de572946a8,0x000a816756f1a5bb,0x0003d4c10230b98b,0x00002c6f30c412b3}, {0x000129dd8fffb620,0x0007b459bf057559,0x0003b67766a281b4,0x00073a77c1bd3afa,0x0000709b38078299}}, + {{0x000b232a3326505c,0x00022e1d41bf8c26,0x000e32afa38d6927,0x000a864459453eff,0x0000e8143ae3cb3e}, {0x000c1fa7e6ab6665,0x000fd2286264932e,0x00036f8ed6cd2d22,0x0005baf59a46fe67,0x00000bf0d00eeca8}}, + {{0x0005852877a21ec5,0x0006bf537a940b82,0x000a9a6a2300414a,0x000bffef1cba4021,0x00000824eeec6943}, {0x000fcecf83cba5d5,0x000843b4f3c0a0db,0x000f24dd7f953814,0x0009dd1174416248,0x0000322d64e34fb0}}, + {{0x00073843d9325f3b,0x00004371cb845744,0x0001e36c5a9bef2d,0x000f71c7d2188ba6,0x0000bd6a7d87602d}, {0x000a9028f61bc0b8,0x000ceed0b6a1ba3a,0x0006e8298f49085e,0x00001d0bc625d6ae,0x000032b0b1e22e9c}}, + {{0x000c447f1f0ced18,0x00031492dd2ba337,0x000a08efa800cc79,0x00041dcb93151dbe,0x000020cf3f95e0a7}, {0x00082dc1c0f7d133,0x000054dde6caff19,0x000f96ee3ef92196,0x0000c6ead7d97245,0x000019c8dbe59dea}}, + {{0x00038717b82b99b6,0x000ce70eb624d3ea,0x00095d46675922d4,0x0003462f66ec543b,0x00006e673cd1ee1e}, {0x00067c4b5f2b89a4,0x0005e90e5cd36afe,0x0000a2ada3de9c1e,0x00023b4c278bb631,0x000020fa3844bdb3}}, + {{0x000d63b0eb919b03,0x000d774b96200ae1,0x00074290cd74ee51,0x000510095458d0a6,0x000024c930f7620a}, {0x0004d19fbac27d45,0x000a4bedeeac2d1f,0x000679ab84086e8c,0x000ec3bcdd211b9b,0x0000970167dc090f}}, + {{0x000f2c9faf1fc639,0x0009a28c8bb43420,0x000f1fe4a616d333,0x000739ed65364c57,0x0000343e877e5e5c}, {0x000176be970e78cb,0x000eb05336275795,0x000cdfc1ba36cceb,0x0003264c7c738009,0x000039a2aff63fec}}, + {{0x0001ba16224408a4,0x0001e47cfc5eb7ff,0x0008bc493cc856e9,0x000726c1f102e7c1,0x0000613ab746091c}, {0x000e89cc420bf2b3,0x000660337ec2aa25,0x000025fc700a5317,0x0003dca2be9f437d,0x0000316fb859e6fe}}, + {{0x0000af59ac50814f,0x0007aa8e42232752,0x000bec5a0fdf95e7,0x000e5cc7e7df2a56,0x00007022f7ecf159}, {0x000eab1cac1fe8ff,0x000b4745116893ee,0x00067dce68040188,0x0002988ee8aa8ad9,0x0000a0e79e82abc9}}, + {{0x0002cfc2064cfd1e,0x000dd06519346733,0x0003bcbea339c31d,0x0005c5919b28d52a,0x0000e74c82c7d6ae}, {0x000d05ebaf28ee6f,0x000bad7190280927,0x000b3028982cecf2,0x000e281b0d353edd,0x0000e4bb978eddb2}}, + {{0x000b990640bfd9e7,0x000712f62108bb5b,0x000ffdd56d226e27,0x0001b4dbf0098502,0x0000756758a9ca1b}, {0x00062a35285fe91b,0x0009dc9cd140c32b,0x0005cb008edbc546,0x0008f16e47a013af,0x0000ca7e720b73ce}}, + {{0x0002ab817a91cae1,0x0004f8e27f63e10b,0x000a3ddf9b89aab6,0x0009726b3074a7db,0x0000c20ce09430c2}, {0x00017b45fcf7e33a,0x0002f45ceb426b99,0x000633d19e679374,0x00047378fc22155c,0x0000d1adb3d67485}}, + {{0x00096796424c49b9,0x0007d7c241c9646f,0x000f68b49f888dfe,0x000f20512d4b9324,0x0000a6b62d93571d}, {0x000b26d179483cb7,0x00022511fae281b4,0x0003aa51f666f963,0x000d166281b3e4d5,0x0000f96a765ef3db}}, + {{0x000b5bf074a30ce2,0x0006e05a32e6a7f8,0x000237ed4d7f5210,0x000a2b4f9e090750,0x0000f21da47a096f}, {0x0009cb4eec863a09,0x000d0527620af3e1,0x0007c1cf8d18f77f,0x0002840505c81c40,0x0000998db4eab6ec}}, + {{0x00089e5c247d44d3,0x00010f4f3d807e33,0x00078a6c71250714,0x0000bea4ba01104a,0x000012874a0a6772}, {0x00059a6759443709,0x000fab2c0bddded0,0x000c108e3d6123d8,0x0003e9156b717b51,0x0000bb7940e97062}}, + {{0x0002d5984ac066c6,0x0002868c69a0794e,0x000d99dccf5954a9,0x000516c8c524584f,0x00000e639fd11012}, {0x00001257de792488,0x0004712fc6d7c2e6,0x0003b5d32e9ef640,0x000b89cc4f28082a,0x000065ad32f4768e}}, + {{0x000331b13fb70b6a,0x0000f5599b27ac02,0x000bd082c037b44c,0x000d007a860fc460,0x00002e257466980c}, {0x00087a81da0263ef,0x00043d10f3d6ee33,0x000f24a32931bfb9,0x000b856b687270a1,0x0000140e65eba494}}, + {{0x000df91b2f1ac7a0,0x000b760fee274f4d,0x000c228e5f99eaab,0x0003bb57f4008a49,0x000090be4401cf71}, {0x000fbe45004f022a,0x0001b69e1af6ac91,0x0001daaa5d838c2c,0x0002c036c7d20b0f,0x0000a063ac1bbbb0}}, + {{0x000a42259558a787,0x00091435da2f0938,0x0004410dc5343c66,0x00080426f67b1803,0x0000cc1e424f4510}, {0x000543f16dfbb7d8,0x0001db5bd59286a1,0x0003dd03c921fa94,0x00051e1dcccb6eb3,0x0000581ddda3843f}}, + {{0x0005fcb81d73c9e1,0x0008fa5e97ab5493,0x0003a6bab6d07e97,0x000e113dc7b30acf,0x000047ab1f3270be}, {0x0008e3d9fafdee4f,0x000a638a8b950aaf,0x000f13871fab3dbc,0x000a48505df4b36e,0x00001f4e9cbf88d5}}, + {{0x00024d366b33f1dd,0x000f8ce445c09bcd,0x00093ff613b97b81,0x00061612926549ba,0x00009c341cf1dafe}, {0x000a76e16efb6f3b,0x0009b05b953cfb30,0x000fffb9fdf24b8c,0x000b95dbd52afec2,0x0000bac5ff8919d0}}, + {{0x0001b87459afccd8,0x00033743265243c0,0x000b5d78e6bd4514,0x000a7d0473453055,0x00001088fdb9554b}, {0x000a52c1e269375c,0x00036dc5ec10ada0,0x000bfbc11f9f037c,0x000d2f0066060794,0x00000a630bc89c40}}, + {{0x000797eab64c31ee,0x000a945071445efc,0x000a6790cffdb1da,0x0001bf161242871c,0x00009609d82c69bf}, {0x000859500d24fc90,0x0003e51fb417db89,0x000f7bbde9c75033,0x00085c51830a91fe,0x0000ce67dc8945f5}}, + {{0x0000ed44763eb501,0x0001b1ab0d669a73,0x0008748f324a0e22,0x000291543b639364,0x0000982daa17d3c6}, {0x000a9f78bbc55499,0x0000ef36384e6f00,0x000977f507a1783e,0x00002a58346323de,0x00001ab688e42455}}, + {{0x000b6b56d0bdd662,0x000e44b71229331a,0x0007c352f0a6ef32,0x0009d2f028150efe,0x00007e04350ee7b3}, {0x0008acdc1070c828,0x000300c9feef2a3c,0x0009f3729fb2034d,0x000548ad72962170,0x0000df290bfe2cb4}}, + {{0x0009f33fc2e43263,0x000d2eddf03202f9,0x000652fb53b30076,0x000f7b8b21f8cf0c,0x000014fb49f1d91c}, {0x000eca52f7007501,0x0003612a4575a013,0x00030fbb02b9e3c2,0x000771d5355557af,0x0000ada35168c77e}}, + {{0x000ecb27b1356705,0x000f2cfc202e45f6,0x000d1be9fe85d19d,0x000343bf1b50c758,0x0000ebf2c0b3ad2e}, {0x000fe4eabc199c90,0x000256bab0ae1531,0x0001fec54c703259,0x00018816ab2e486c,0x0000f87fda804280}}, + {{0x00046fc609e4a74a,0x00031a667f91dc9f,0x000d834362a44a14,0x0009581c3d8b95b4,0x000001e4bd167bd2}, {0x000293273483c908,0x000127c7b5987b18,0x000aac07ea79c6aa,0x0004e63f3983c6ea,0x0000f18181f16e0d}}, + {{0x000d37c051af62b9,0x000a7bf944968553,0x000d59aa1e9a998e,0x00081350844f9fb0,0x000083fd55976afb}, {0x000c0ca65d698049,0x000ddea5ff2d9670,0x000d8623b732b22d,0x00078247640ba95f,0x0000f61916436351}}, + {{0x00027eeacee50437,0x0002beb10f020bfc,0x00043fb05ae419e7,0x000a29a9c028d189,0x00001f01cf86f13a}, {0x000737e8887a132f,0x0008763184107790,0x000db795e6751330,0x0000b1a819e8a37d,0x0000ecb8ef6cad10}}, + {{0x0004a223021926a4,0x00092f9b4c1c59f7,0x0002ad0abb7c28a4,0x00059cad1a733f91,0x00002a910af41a56}, {0x000c6e07bd68cab6,0x000816d70ac83842,0x00053aaeb2b57fa3,0x000b182a6707a83c,0x00002c1c510c5b4d}}, + {{0x000c1fbb2d09dc72,0x0002066bd23b8de2,0x000b27db6c3dfed1,0x0003da727d039bd5,0x0000fb2f0f130324}, {0x000a07b80be73992,0x000dff9f27a8f855,0x0009bdef7ed9327c,0x000d8880bd99c772,0x0000b67125e48250}}, + {{0x00026e88670ced73,0x000f931bd3b4784b,0x000c85cbce3dfe41,0x000a9d6e353a06bc,0x000002e290990178}, {0x000bf11a6eac16e3,0x00007a2b3aac860a,0x0000afdab7644700,0x0004c116ff9d1985,0x00005bdd6a62db2d}}, + {{0x00094b07e5c9ce97,0x0002b0af346ee825,0x00065ad4a0f379e5,0x000825f08b31e3bc,0x000010c6b12967c4}, {0x00066f971954cf1a,0x00026d0aa21551c9,0x000bd23a8b1cec79,0x000e857f15598986,0x0000e2ff99d99452}}, + {{0x000953c340ceaa2d,0x00045e2e9333d8dd,0x00086f06d2635527,0x00054545d4e5f985,0x00006bf94a9c7cab}, {0x0009a0ab76a9af0c,0x0002fa095af733c5,0x000389ca052740ab,0x0000cb0444de8a24,0x0000c6f9864306da}}, + {{0x00041a76b2515cf3,0x00016585c749b5a7,0x00083de9771c4160,0x0005f548350d4fe6,0x00001d6152493d0b}, {0x000c5e1fbce090b8,0x000d7bcb2a5b7a0c,0x000d84c35aac927e,0x000e266920de4920,0x0000c06a0b6a2b4d}}, + {{0x000d58bafe7ddf39,0x000d31e6e55bd34d,0x0000696e755851fe,0x00005f4139561696,0x000040304b2ef227}, {0x000f861b0a2a8600,0x00020e7cc9816f43,0x000b64a96cf12128,0x00083c121862120a,0x00009215b9ab7893}}, + {{0x000b30537387c09e,0x000e103ee760311e,0x000f7ea19c5832fc,0x00055050358f5832,0x000001d3c3571d53}, {0x000ee41da48ea80e,0x000def4fa4c11ca5,0x000f1e1c734e71e8,0x0004a4512abd257a,0x00003afcdec0153f}}, + {{0x00084d700235e9a6,0x00042c4c836f9d5c,0x000332de50308d3f,0x0006ef60a66b0489,0x000010dd399e9e56}, {0x000a460d1ac16352,0x000b00a2c0dff8ee,0x0004a48c584cbb3f,0x00015020afb488e7,0x00009738198f326b}}, + {{0x000747fa6d74081e,0x000475a262142a17,0x00088c5fe60ea4c0,0x00026b73514bb41f,0x0000dd645685e834}, {0x000cbec96460b259,0x000dd8dc115ed5d6,0x0007840eaa12fd0c,0x000e3135bc3ed269,0x000069876a936331}}, + {{0x0006217472ff5805,0x0004fad4139360c3,0x0003b8b92f422970,0x0004f5fbd99ef0a0,0x000001c73181144f}, {0x00009b318464945d,0x000ba4c5c6be1590,0x0001a36606d5e594,0x0000215d58701132,0x00001e184b20898d}}, + {{0x00047524c6a7e046,0x000ae5550b655ba0,0x000c0a9a547fa1e2,0x0002363419daf048,0x00006362953dc243}, {0x00044b15cb12a88b,0x00097b646188cd07,0x000c2c0c0561b6f9,0x000099a99415a566,0x0000e3f0859bf83f}}, + {{0x000c5beb92041b8f,0x000636477d0d9141,0x0002c7a9401ae38c,0x0000dada8b71f3d1,0x0000ab5b320665c7}, {0x0007492487443e97,0x0000290d134976ae,0x000460a378595a31,0x00038f88dbeda87d,0x0000f7ad0828e45a}}, + {{0x0004db61059705ab,0x000a36b9c697ed1d,0x000b38bd5a3dd492,0x000bb69b92ee3a6e,0x0000ab2609e17cc0}, {0x000fe896e70ee822,0x000df3e6b7e37fc4,0x000d26fcaeff2c56,0x000b457b18959e34,0x0000517ab66a89d6}}, + {{0x000b4e0bdefdd4f1,0x0005e366e401f167,0x0003bbec06995846,0x000c214aa368aba7,0x000021487098b240}, {0x000323318969006d,0x000e11fe53d1378c,0x0000c4361cb4d73c,0x00012a8f50a80e13,0x000067f59524ef52}}, + {{0x000e21e9e70c72e5,0x00090566d2fbf145,0x0002397f5b2e52e2,0x0007ddf4eaba4a03,0x0000e56937bce31a}, {0x000f517456c61e10,0x000aa8b0a38868dc,0x000a8b755bc2e954,0x000cdee3552fa760,0x00003442dae73ad0}}, + {{0x000e747ceb26210c,0x0007b87baef937ff,0x000a3de31983545e,0x0006dacb8c853586,0x0000621dbccbacd4}, {0x00042e959266fbbf,0x0006439d471c82e4,0x000cdad96a3514c3,0x000def4a11b77162,0x00000cb3b3ddcf9b}}, + {{0x000dbce478e2135c,0x000efda353423fcb,0x000677af67547b5c,0x000986e97e81f18a,0x00008c2bf83e8817}, {0x000eaaf455809854,0x0005893b45cbdf07,0x0007b4cacc68d1f0,0x000e85d06aa2fec7,0x0000c1d8afc44a7a}}, + {{0x000c3fd9eb45ab26,0x000b74b22e74db41,0x00015958a5b234b5,0x000fa07a253decf2,0x00007e0606f004ed}, {0x000f070ef751b115,0x0005a6f06dceabbb,0x00039f6b4f352f17,0x00048e8fc4b6af68,0x00003ddf9a8e9598}}, + {{0x000c379c21520b0e,0x000ffbd5d1b6da49,0x00049c7f790864fe,0x0002d74f055d235f,0x000051e4e6b8796b}, {0x000a67f5c9dc340a,0x00071ca7c620c361,0x000c756d05ad53c3,0x000e133a1d658832,0x0000d60d9122bb67}}, + {{0x0007bdf0eeec8c68,0x0000878a1821d6c4,0x0000995244a27fec,0x00005f881f7415c3,0x0000effdf0c02cd8}, {0x000ec1c65842df85,0x00088319a901db70,0x00042b5298821b35,0x00028622ee56eede,0x0000bb39592736e4}}, + {{0x0003316fd6f7140f,0x000acd8e81f7d118,0x00002d962f9fadb5,0x000323801d5e0c5a,0x0000dee4dc00b601}, {0x000740735d7620e5,0x0003a48c0012bed1,0x00055449a04e3c2c,0x0006c44ee29da734,0x000062cdef4e1a83}}, + {{0x0002a5f477010973,0x0008cf88d0c28f68,0x000bb86dd617125d,0x000286648fda2457,0x000048abb8f589f7}, {0x000eab599d94bbd6,0x000de684d160eb10,0x000c8f41ad51ba28,0x000f4a4be0e51c30,0x00006588b4573254}}, + {{0x000bf01fad097a5a,0x0007c10e815d147e,0x00011de5649883ea,0x000a6d444d60ba8a,0x0000970de6f227a7}, {0x00014245e17fc196,0x0006a12140572be4,0x0003e723fd833c65,0x000e9ab375813b36,0x0000820bb8946a52}}, + {{0x0006970d875d56ae,0x000b71fbf6bf7e7f,0x000083c12d6a0a9a,0x000b6374ba8790a3,0x0000baeb23e4ae7e}, {0x0005c3ab99a907a9,0x000f726bf40ba868,0x00002cd9ef1e7454,0x000634eb73a027c8,0x0000a8a927cdfef4}}, + {{0x000f60c08191224d,0x000b0e4ec091e1b6,0x000e38d84c4126eb,0x00098511dff4dc4a,0x0000e3f57dc1f2ef}, {0x0004337d446a1dda,0x000fe59e77f63496,0x0001d13f57bf2179,0x000e26eff105278e,0x0000304ef0414eea}}, + {{0x0005e47d19dfa5a2,0x00035fad982bfc6f,0x0003715f5db007de,0x00029e08205ad161,0x000051e672998895}, {0x00051841ae98e78c,0x000c671cac327270,0x000f410f5f818537,0x00039308a15b7eb7,0x0000474357041f62}}, + {{0x000dc5ac242316b4,0x000c9bf4aff592db,0x0009a8ec6abe060a,0x000b942e8c38fe90,0x00003e514e5a116c}, {0x000fa3807d784f90,0x0000f4b5b3572078,0x000adea3d1161a88,0x00010b5283ce7913,0x0000756c3e6cc6a9}}, + {{0x000fe01aaa79697b,0x0008a6391db160bc,0x0009b45a004a73b2,0x0008d92d8dad4718,0x0000fac0dd0f8d5b}, {0x0003af57d3d2ec27,0x000cb07bd3af34ab,0x000550ded6fa2fc2,0x0009132ff4009266,0x000019b3e878fd5b}}, + {{0x000a4966d17fbc7e,0x000993d2b24ea573,0x0006769370cd1a70,0x000f2054e2c5cab2,0x00007050b079f669}, {0x00048b61ede9046d,0x0000c7662659fbe9,0x0000124c5a053005,0x0006c788cbd4edf1,0x0000e2646e5ad6c0}}, + {{0x00081088cad38c0b,0x000fbbd68ae2332f,0x0008e27a3471b7e8,0x000b0ca6ac3fb20d,0x000054660dbc36b4}, {0x0001e11a6fd8de44,0x000a637799ef123a,0x0006ac17c44dbffe,0x000cef0540b977ce,0x000095173a8ef60a}}, +}, +{/* digit=9 [{1,2,3,..,}]*([2^63]*G) */ + {{0x000284d391c2a82a,0x0002758308e89ebb,0x000f1edcabcdd486,0x0006c7606f16ec83,0x000013e2c38095dc}, {0x00056f04a057a87a,0x000006b48f982ab7,0x000651c44a876550,0x000e01a252face68,0x000052b540c81765}}, + {{0x0002fc516a0d2bb2,0x000bfa6234994f92,0x000c62c8b0d5cc16,0x00067f7241cf3a57,0x0000f5e69621d1b6}, {0x000c70bf5a01797f,0x000c709561925c15,0x0001fdb523d20b44,0x000f7a14911b3707,0x0000648f9177d6f0}}, + {{0x000acafe60b7cf78,0x000004a9d8696dc1,0x000ba8ac425860a5,0x00029dd6fc6f09e7,0x000028c5bd0e148d}, {0x000435edc55ae5f2,0x000ca0117411ac6b,0x00024342ca527f56,0x000c0d74d5045efd,0x0000c4c0a3590b67}}, + {{0x000c8b8fac61d9a1,0x0002d3c6fe8a027c,0x000bff5037d25e06,0x0002f7d08805bfe5,0x00003271e6c7ff63}, {0x000a6c0232f76a55,0x000d201ef42655dc,0x0000a51788957c32,0x0001739e728bcba1,0x0000ea60412062c5}}, + {{0x000964ed0b8892be,0x0002b301bb74fc4e,0x000c486269ea1768,0x0001018265c5aefc,0x000060cf82f9b3e9}, {0x000f797d4df55311,0x00017deeefe257ad,0x000306eb1235b59a,0x00092d50adcf583f,0x000005c27534d094}}, + {{0x000914bb5def9961,0x0003133dd1e74090,0x0003d5e761cb69c8,0x000012b1e9c1d39b,0x0000f3338ee0ccf6}, {0x0005d0d2f5378a8d,0x00065f00cd21b1e9,0x0005fe290acf4c2c,0x0008ad9e984240eb,0x000066c038df4808}}, + {{0x000264af94d70cf7,0x000f0314bf7e804d,0x00033ed02bdb802e,0x0005d91fb54de243,0x000040461e098563}, {0x000b2c8365e9383a,0x00029fdef6524113,0x000b956c1ea762c8,0x000fa3aeec6e2e47,0x00003d814bf05620}}, + {{0x000462bb4d8bc50a,0x0006091957709ad5,0x000412a68181c0b1,0x00048c4bd4fe1c78,0x0000e0341bd60dff}, {0x00045cf7003e8666,0x000a2a24a41bb6bc,0x0004c24c2f11a6de,0x000b67f407151ad0,0x00002c9d27e3a5b7}}, + {{0x000423588cceff67,0x000f1b07ed692e96,0x0004d0d0d8594c54,0x000867a578e73cc8,0x0000b4e10566f532}, {0x000c0d5b5ec995a1,0x000504289a54a348,0x000fbd777bf4b9d5,0x00091d8ba155a658,0x000086ed7a82a844}}, + {{0x0002b30614c09004,0x00017d00c24bd499,0x000c4bfa1da98d12,0x0004bc3f534dc87e,0x0000a5ff67477dc3}, {0x00096b81d7ea1d7e,0x0002a0a6d20868c1,0x000cbbd6e38cf289,0x0005b61d56cd09e3,0x0000c72e27f2205a}}, + {{0x00068f5a44f77f7a,0x000d143c52bc15ea,0x000f0e6097aa5f9f,0x00032ae6ff676f94,0x00004cde963ce2d4}, {0x000a0c0eee470afa,0x000dea3f5ec88caf,0x000a3123184137d0,0x000ccf4bb40411fa,0x0000239c1400f7f7}}, + {{0x0005719a8afd30bb,0x0007da826dce3286,0x000a8fbe08679832,0x000ad32f04e891c4,0x0000b6b6e1c9bf56}, {0x0005b11471f1ff0f,0x0008ce15baf00a69,0x00096c43ed76c338,0x000157118edb95be,0x00002beaaf580794}}, + {{0x000b09ec3076a27d,0x000e1416545d152d,0x0006d6f2e5e82908,0x0004e0d2c4127235,0x0000c9c964301fd7}, {0x000b88d519bf6156,0x0005a5a2274e66ce,0x0005e2fa0e29ecd7,0x000e66da0473c4bf,0x0000b6eb671c4284}}, + {{0x0007932b88756ddc,0x0002317e3e61e8b9,0x000e1c4a4ed4e865,0x000c0e02dd14993e,0x00000aaee18197f8}, {0x000edb96c168af3a,0x000f139ae87515c4,0x000adb4366563c7b,0x000ac00dfadb6f20,0x0000d55e8ca3a042}}, + {{0x0001ed8b76da1f56,0x0006458acb94975a,0x00006028210dfa46,0x00051e2dd7f7e3ac,0x0000813e66ab72a0}, {0x000ae1e350cb901c,0x000590cb7822b4cc,0x000ab3b87b653d65,0x000fcf82484710df,0x0000d7ee5385b670}}, + {{0x000b12e523b8bf60,0x000b8f910c1b0a50,0x0001675888009eb5,0x000abdf535af824a,0x0000f835f9cfb2a2}, {0x00029312afceb620,0x000a169d383ff59b,0x000ac02b0c797df2,0x0000caeb3f5fb066,0x000029d4c6fdaa2d}}, + {{0x0009bc1afab4bc58,0x000ed6783247d405,0x0002d3605833f5c6,0x000433353466308d,0x00003387892534d8}, {0x000b30fadd9419a0,0x0009afe3fce8d973,0x00009aac6bcca109,0x000f110817831508,0x00001b7f21a540f0}}, + {{0x0009219909523c8b,0x000ef3a1c74165c2,0x000c9e55aa62f648,0x000479d8598d4f60,0x0000ce9141bbe4f3}, {0x0007d8435f9b9887,0x0001c20475b69af9,0x00091476c0210da6,0x000833cc076e2291,0x0000520dbd9b4fc7}}, + {{0x0002cfec1ab1bbdd,0x000e0c6509386a6b,0x0005d7bc4ef8a65b,0x000dfca285554080,0x0000a389397bd11f}, {0x000bd3674660876b,0x0004045dff35a9d5,0x000f5da9411d67c5,0x000b30baf7d148a4,0x0000b8d4c4070bbe}}, + {{0x000ebd1e0a1b12ac,0x000cb70ba95f87a7,0x0002ae9cb1e4ef88,0x000402cc33345cdc,0x0000ecf1276c1cc8}, {0x000012e1b39b80f3,0x000d05c33ba4687c,0x0009661c2fd90d0a,0x00029e73ef5a675c,0x000068fc88f10174}}, + {{0x0006761196a2fa2a,0x00071d5b312ed30c,0x000f54a31931b981,0x0005411a01000c72,0x0000203d2c906eaa}, {0x000dee098939db32,0x000c1e606c02f2ab,0x0001ff643e37d6c2,0x000ca3d292157452,0x0000781b3c4f7e2f}}, + {{0x00000b07850ec06e,0x00089d3a10cf6643,0x0004ab39dac5a38b,0x000bb8b233188de3,0x000077057e53072c}, {0x000c042b59e78df5,0x0007cd97de52bcf0,0x000e0ca4a4cfc91e,0x0007bbf661a26c3e,0x0000620a4c24b850}}, + {{0x000d4aa049f842cb,0x00046540e82b4b44,0x000c6f156ceabc5d,0x000d71806710fd15,0x0000e5ae52c13db1}, {0x000e7e6334957f10,0x000031144a7006f1,0x00096447b57e388f,0x000a12fb69bb2fdf,0x0000f78ebd373e38}}, + {{0x00026052b7ce5421,0x00096472bde1b822,0x000d2f4dae6d4ce9,0x000b2e43e16ebe09,0x000080ff42e63b92}, {0x000cc022c34a1c65,0x0008f22c46c2c59b,0x00014a8a23803d6f,0x000b27c8aff74f5c,0x00005aebf8060a08}}, + {{0x0007d587135593fc,0x00066be570cd6609,0x0008c860d32e6eff,0x000162984e6a102a,0x0000d18589162eb4}, {0x000e99d6d97e1340,0x0000dd8447ce7cea,0x000c50273d42c6b7,0x000e1e59ddbb4ab8,0x00003c612df3cf34}}, + {{0x000ca1504b6c5a08,0x000898f0e3a384b9,0x000986c0035216f3,0x0008fdbec2d2bcbd,0x0000bf546da51922}, {0x00055a44cd623c3e,0x0007702b8e5ad1c6,0x000a0bfe7366ce71,0x000e8d4cfc84b4ee,0x000001d5cefaf443}}, + {{0x00045d9036520f83,0x000162d40e988ec0,0x000559a04dfb3c3d,0x0006b0dbac4ccecc,0x00005eccae5540ea}, {0x00032dbf8a5a0ac5,0x00059b699700180b,0x00026bca0547972a,0x00025a53765801ca,0x00007e09d0ef647f}}, + {{0x000970e2fdd23cc0,0x000c5682e971b956,0x000e86ebcb80288b,0x000939e6e6d91e9a,0x0000564c83f8c9f1}, {0x00032a239560368e,0x000a249c28e25519,0x000a158c3e893752,0x0002622b03cee5a6,0x000012d656be4964}}, + {{0x000554e63e3bc1da,0x0001a5044ff74b47,0x0008daa07c719b6a,0x000dc2af24d30ae4,0x00003f3755768c1e}, {0x000bf760700d3607,0x0004122ae4e29a47,0x000f1fb4cbb1a182,0x0005f4b2e275a389,0x00002b1aa240968c}}, + {{0x000eacabe063f644,0x00037ce47a09a75f,0x000d07aca9b392f4,0x000d0f942415091a,0x0000b0c591bcd26c}, {0x000ddfd92f1169ae,0x000b6cbf23922d42,0x00091a2af63aeb1a,0x0001d93de9e87706,0x0000be79af8b9802}}, + {{0x0002a4e40e50acf6,0x00074f01d665cfdf,0x00031be1ff0a98ad,0x000da08fb640bf18,0x0000fe8bd2fe0e9a}, {0x00003a16cafbc916,0x00092308e08c94c1,0x00080ff4f170f875,0x0001f1fde2d2ab97,0x000066466bca5b20}}, + {{0x0002010f5b343bcf,0x000a02f142fe58af,0x0005f4bdf0f2e400,0x000aa84483bfdea8,0x00000b1d093f3bfe}, {0x0001b95c70816030,0x00093dba10972ea0,0x00038f3a6e943e4c,0x00063647be92adb4,0x00000bb7742e5bf6}}, + {{0x0007083824297b45,0x00000584455f136b,0x000c7d69e9d0e558,0x000e765b48cedcf1,0x00003a9e4817a256}, {0x000b0e065eb24132,0x00046fc407a70402,0x0007f5492dadbbb8,0x000294865cd5a48d,0x00001d4429394bae}}, + {{0x0007ce63b5f1cc41,0x000abe872e626691,0x00005f24437ae52e,0x00074fab087b7229,0x000020770862e6af}, {0x0004e491058edeaf,0x0002c638ca1d4b64,0x00038591c827510e,0x0000629cf2b70460,0x0000fc8b47bee635}}, + {{0x00020e61b4d5e634,0x00025d961b4b3ae2,0x000d16bedbd86474,0x00047b210c107e9b,0x0000270352a51271}, {0x000ffe664cfc50e9,0x00098e36cb427d17,0x000dc5f9a50dee01,0x00062b768a762235,0x0000a08d5376f53f}}, + {{0x00014576be5f7de2,0x0008a2263c9e4ed7,0x000cacb36d93006f,0x0008abc073694cca,0x0000ff7a5b45ae11}, {0x00053f1cd871236b,0x000d12aa6d523cce,0x00098d76df156a39,0x000d38ecc5f271b1,0x0000c615b7031383}}, + {{0x00038e8de3eee6b3,0x00087b910d91a545,0x000d278bd58c7753,0x000cae01e5bdbc58,0x0000cde4adfe963a}, {0x0001fd25302169cc,0x000fe989ed8bb188,0x00096a0ee8ca60f9,0x00083ce1999458ff,0x00001141f046c6c2}}, + {{0x000408d6dfafed30,0x0003396615887677,0x000726fa033a0165,0x0006da3c9c15ec0b,0x000090cfd936c9b5}, {0x0004baea3c40af51,0x000bc21129f1e34f,0x000207ce83469ead,0x000ef9bc51674a1e,0x0000e293b24d83b1}}, + {{0x0003d131e6c0bb48,0x000510776d351717,0x000e6f9221900469,0x000a2267980e346d,0x000073554cc74dd9}, {0x000c627cbf18a512,0x000b1032c0810316,0x00046834d4d93651,0x000f80007f277139,0x0000c08d7b450cdb}}, + {{0x0004fb486df2a61b,0x00073cf7b4a2137a,0x000d042ffa1ed9c0,0x0005ec02e460e27b,0x00007f5e2fb0f62f}, {0x000ec6bcc2423b79,0x000f2a63eea77aa6,0x00050a6e175ce0a7,0x000c9ed7a45fb1f2,0x00003bc919d753cd}}, + {{0x000f56f871942dfe,0x000e9859ad669271,0x000cb1a782372ff6,0x000a827f4c2b9633,0x00003e291023838a}, {0x0001611e4e8110ce,0x0004530198cea7ed,0x00020efe02a2d70d,0x0001beddf132e867,0x000061a896346a47}}, + {{0x0003a85825808bd3,0x00070fd6e902796d,0x0006219d151dc3cb,0x000d32343c768a91,0x00006cd7685d2ad7}, {0x0009d05b22922a44,0x000e9ba29660e3db,0x000d2ebc76494c87,0x000f8db0ac91dfbc,0x0000deb57a085107}}, + {{0x0001f59c3d12a732,0x000c85c2c51d4227,0x000797bcb5f71687,0x000eb0ab1f50c605,0x00009ed0ed9f6d34}, {0x0005b474683c2eb9,0x000507447c46e5fe,0x0002071674956eeb,0x0005eecb163a4371,0x00003fa2fed9248c}}, + {{0x0000af231f63950e,0x000034caa2c96793,0x000ac7e62a77797c,0x000aeb726e80ee27,0x00001e6e62738b28}, {0x00078b0b3c9fef02,0x00004d5f90be6361,0x000ce51cfaf7752e,0x000e1f74ecaf18ee,0x0000864d0edea806}}, + {{0x000e38397c69134a,0x00064b2912936de2,0x00060bae05a42c31,0x0009d1277792196a,0x000024de3470b759}, {0x00074aab75d49411,0x00061d501ff049d3,0x0007974cf9890058,0x0001158f16d40eeb,0x0000033860bddd8c}}, + {{0x0009ac82094cec3b,0x0007903b770cb6c6,0x00059590d9976fb8,0x0001fc6dea026c48,0x00006acbb473562d}, {0x000c46144569d857,0x000627f0891d7cd6,0x000d5a17dc3190a3,0x000bc856f5319548,0x0000d9199674749a}}, + {{0x0004837dd1c8a206,0x0007cf6834196510,0x00094022e7e5410c,0x000ac2358c3ca8be,0x000005c3197c145d}, {0x000750101683d546,0x0004f95b12343fc0,0x00081277f1d7127c,0x000adab0b8f87c94,0x000077db2a9465a1}}, + {{0x000ccaaddce33450,0x0007012a4350ec2f,0x000598bdc2a6811b,0x00012896760ff1ac,0x000054d652ad1bf4}, {0x00051d492a210056,0x0000d3110fdf0a11,0x00060100fad7f397,0x0003622c95928c19,0x0000c91c825dbf03}}, + {{0x000b2a2ce309f06b,0x00091a27204bc8c8,0x00048e32efdb27b5,0x000f1e2223eaa508,0x000093e4b2f97bfa}, {0x0008ae644aa3dedf,0x0009d015d573c530,0x000979707317a666,0x000957d888ce231a,0x0000141c1e6fd5c4}}, + {{0x0007de5619063736,0x000e8b999595b53b,0x0009e5c36858dbad,0x0008422cbb47b2a5,0x000060318b43cf4e}, {0x0001ccd12ba4b7aa,0x000b58c8282abd16,0x000b2130df399daa,0x000d7c7587633aee,0x0000465311b7a38d}}, + {{0x000eec864d3779b8,0x00062d64c1715f75,0x0009144283c5d047,0x000c29074103712a,0x0000096a89210e2f}, {0x000ae9d23b3ebc2e,0x000ac580cfd6d3d2,0x000b01f6c90bdd6d,0x0002db72dbb7f3c5,0x000068eded5c102a}}, + {{0x0005b7799eb6df0e,0x00009386b7791778,0x00017a48e26c3cc5,0x000f30545ed98864,0x0000990b4e4e7d6e}, {0x0006b7e2586abba3,0x000529c96e9a0f45,0x000eb4206239ca6a,0x00090ab327459ce2,0x0000a4c3313d002b}}, + {{0x0004806f6a3f6fbe,0x000ea5c251dd2a11,0x000a784d3ad5cad2,0x0006d4b2c1f613f5,0x0000c7bfad014976}, {0x000cd333e23cb3ba,0x000425a64b2d04b3,0x0005891063979fe8,0x00027e792e27207e,0x000060c43d2415b5}}, + {{0x0009082be7cf3a64,0x00027c9672742dae,0x000a0a8a9cc86ba9,0x0008b3b28a2ce8ae,0x000004ca6d9aee98}, {0x0009c5d005921b83,0x000114e79bf9fd7e,0x00075ddc2f56297f,0x000e877163b4600d,0x00000b23616d1f2b}}, + {{0x0000d21bfe50e2b2,0x0000c1bfede14b07,0x000ac4ae07ef8cfd,0x0000338dba00112a,0x0000a3e7d01d9ebd}, {0x00077ece38d9d1cc,0x000ddc5d2de39952,0x0003ca8c9b500249,0x0003aec912b820f1,0x0000879811547779}}, + {{0x000125dec3f1dec5,0x0000411178da19e6,0x0004a673807b1f04,0x000dcd893ededa90,0x00005187a5a5bebe}, {0x0004722eb329d415,0x000ea170b391f7d0,0x00099f828f449099,0x00076dad317a69ca,0x00000c3db2b84a49}}, + {{0x00077843757b3927,0x000d3a3ca05ae9ba,0x000e593d4326caef,0x000d1308e5293bf1,0x0000842a9377d98f}, {0x000bf965f96b10d1,0x0005f6a8cd05e694,0x000f0c7fc373a9df,0x00072e897d1e51e8,0x0000d01979073fd9}}, + {{0x000d8585499fb325,0x000927a8aeb70064,0x00008eec57b67bad,0x000e1ccd3eb9772d,0x0000fc047a71baba}, {0x000d159e54a64bb8,0x000b443497e40577,0x000e0608d8862201,0x000aac2d6b4e282c,0x0000b687b7d8b167}}, + {{0x000d3678b2ecfa91,0x000d990c3c386ed4,0x000e5c42b24dfe62,0x000a9f91862e103f,0x0000ca73dcae5732}, {0x00038b776bb87ad6,0x000b9242b81f35f0,0x000fd90cd674976a,0x000091ef2bde7eb0,0x0000efc172f07fdf}}, + {{0x000b69b92222f1f5,0x000a1cf7ae703806,0x0005217ee5a2459c,0x0005ac1789f69ca8,0x0000f232b5f33dc8}, {0x0003ec548e9e5167,0x0006c197eb31660e,0x000fcca23124b4e4,0x00024ea0a0cb13aa,0x0000bd63ba4f2132}}, + {{0x000d7cc290a7f4fc,0x000d4286b461affa,0x000a407af6b409c9,0x00007298ab809fff,0x00003122eee868ac}, {0x0009e504ef24d7e7,0x0003ce2a581117bf,0x000902e015d92979,0x000850e19bc86702,0x00006bba5daa9c8a}}, + {{0x000669cda94951ec,0x000ca6b8d418e9f9,0x000d426a44b6af58,0x0000273a32107417,0x000078e66aa5dde6}, {0x000c0834a53b9649,0x00086f6023300516,0x000c5c897fc659d3,0x0005de7ab55e5c58,0x000085099b3138bc}}, + {{0x0009efcc52fc2384,0x00082ac1da3f061d,0x00083fe08712b272,0x0002f7bb65814992,0x0000954ac94f8aaa}, {0x000ada47fb2e74f1,0x000ea89926b085c0,0x000d1af5bee8ba98,0x00015ed4f9d37d23,0x00004ccdbf9ca9b0}}, + {{0x000481b7bfe71786,0x000e05405868b674,0x0008c867d4e1deba,0x0000e9a61b2821c4,0x00009c15b35b13b3}, {0x0001666368710884,0x000cd220b1ff3b4a,0x0003d9f4de5e29f5,0x0006750b82bb3523,0x0000e07633358cdc}}, +}, +{/* digit=10 [{1,2,3,..,}]*([2^70]*G) */ + {{0x000f5c7a3e6fced0,0x0005f45fbdeb0d53,0x000339a70e8cbbdd,0x000b81f85c01df13,0x0000ff71880142ce}, {0x0008774bd70437a2,0x00019a0bda6a4c4e,0x0008bd26e5fb3289,0x000521fcdbebd2f1,0x0000f9526f123a9d}}, + {{0x000305192c4d6840,0x00057612efcd40ce,0x0009cae208b04d72,0x00056cb9dcda366f,0x0000edc4d24f0588}, {0x000e6bf854279005,0x00058c09dfea64f2,0x0009bf26c3de8129,0x0005a9841b448737,0x00000b62c6dbdf13}}, + {{0x000e3b4c72dfe67a,0x0005f0e19fdfd4f8,0x00013bd35c416b0f,0x000b9d78b9098d4c,0x0000c11118ab5b8c}, {0x000a318f00628413,0x000f59f356f4f598,0x000177a0cbfe0602,0x0005374ae3637e30,0x0000409774791136}}, + {{0x000fb5ed005832ae,0x000ab1042e4f0db2,0x00070f8ca5f5efd3,0x0009cbac4ffdc6ed,0x00004645d0c952da}, {0x000f58bc9001d1f8,0x000bce1172059596,0x00098a08452c8f0b,0x0009de7d4aa0d2e3,0x000015bfe3a904f4}}, + {{0x000443f23885e5fd,0x000918433aab97e5,0x000d4e604f72f8f9,0x0003feed00b154e4,0x00000b35e6bb5e17}, {0x000a0489164722d3,0x000b58761ec857b2,0x000a838323e3c665,0x000e3b3bdd13973d,0x0000c8b1a1ea3daf}}, + {{0x000ace654317cac3,0x0009221771b34497,0x000dfe8b8be600ab,0x00010f842e409eb0,0x000086a67d769423}, {0x0008d8d4431cc288,0x0002185dc5242554,0x000c4be32a7cff14,0x0006e0cd60f5a193,0x00003ebd5c95071c}}, + {{0x00080a7b1fd2b0bc,0x00059bec33e8ba3a,0x000743fb39b3ad39,0x0002f9f3868d6179,0x0000fd169fdbdb46}, {0x00099d79ce0a6af4,0x0001a42d3ff8d3b4,0x000c3e1b255dc1cf,0x000473c4fb9e6cc6,0x00007e6961daf69a}}, + {{0x0003acce548b37b2,0x000264d4054954eb,0x000341b4fb38e754,0x0007fa6c3daa517b,0x0000f6928ec890bf}, {0x000b32386ce6c413,0x0004e0adadcd0496,0x000b5faf901be1c5,0x000985904e67e74b,0x0000cbaf679115c9}}, + {{0x000214550ca42470,0x000ae7dd30aa8cd1,0x0008fee24ba1aa47,0x0000e82f81ddf1e5,0x00003452936eec9b}, {0x0003b81243aea965,0x000ec5c3d0e58bdc,0x0009483619a2919a,0x000ccf4ea640ec10,0x0000ac86d5bbe0bc}}, + {{0x000d918c36cf4406,0x00076939719cf892,0x000218b64aed3e83,0x0000dd507b08d2c0,0x0000f1bcbbb2e979}, {0x000d6ed60919b8eb,0x0000dac1f9eb4a84,0x000d5daefd890079,0x0002c5484941aa0d,0x000022fe40b17fd6}}, + {{0x0005ba2157f2db33,0x000f5e28ca9c97e1,0x000b9f454bda2fc8,0x00072e2d050da437,0x0000d57eb575379d}, {0x000eba2fb5ee9973,0x0002b11538cae9b5,0x00032797401648ca,0x000bb702bb76f6f6,0x000038f14b92f3f4}}, + {{0x000226ad7ab9a2d8,0x000cfdfae958524d,0x00051d8c29c00090,0x00062c8ba5f53987,0x0000afcbcddbab82}, {0x0002729e99d043b6,0x000b4ebc943a5739,0x000862935ef51263,0x00017b3feace9320,0x000039efc04106c8}}, + {{0x00054b366b4be7ad,0x000db4a37a1e1fe0,0x000d75cd93f25a9d,0x0001b5239ef1ad78,0x00007b58f4a2062c}, {0x000f9a9ff563436b,0x000938af51e76f74,0x000e97fecf718ff2,0x000c09a234d31315,0x00006a8e2b1d92f1}}, + {{0x0003aa8327720c15,0x00026a092cc8a7f5,0x000746c4d956ca32,0x0003923f03d64a28,0x00001fe1782b6d0d}, {0x00034db3c832c80a,0x000bcda2e3b4d19b,0x000104ccc60dccc5,0x0001fc845dd62e0a,0x00007ab1de2020b2}}, + {{0x000ae0b3893d123d,0x0002e15ee71cb293,0x000a9468bf7b7578,0x0007438aa3c61442,0x0000686123dab15d}, {0x0006891a7ab4116f,0x0007b4e6a4598c61,0x000e5fad76fcd72c,0x00046abc21911077,0x0000b6a20e8604fa}}, + {{0x000be7d341d81dcf,0x000842148379e839,0x000026eadcddb688,0x000c5dea6211a1f7,0x00003b25760e4d1c}, {0x000c8f6a7a73ae65,0x000e11d5b48340cf,0x000a50ebc83879a5,0x0008fa75acb1ed41,0x00009a60cc88c07d}}, + {{0x000bdceb18762621,0x000002af4ee91b73,0x0006e1d072b0d79f,0x00052f7bcf3b0bd4,0x00007d6af9df45d1}, {0x00004616d7364517,0x0006e6b0bf5a7352,0x000999b9d43cbbd9,0x00039840833a5bd5,0x000002614f15b72e}}, + {{0x000f01a59c3e9f87,0x00075e6b3d160aad,0x000ddafad40200e7,0x0002e16a22bdd3de,0x00006dedaf4a10d7}, {0x000807c4bc2e88f8,0x0000946dd5a549ef,0x0008d59e96ba8129,0x00034921a4077a7d,0x00007b6a2e8002db}}, + {{0x00099971b4e598eb,0x000e56fe4b1dd567,0x000b267c5f499ef1,0x0006cf8978d3aefc,0x0000582b557d3578}, {0x000b2ca1715cb07e,0x0001a480241d32b3,0x000571ecd4c3de6a,0x0009a883b5ffedcb,0x0000af53901cd2fe}}, + {{0x0008d4ac3b819900,0x00029e0cc8fedec9,0x000b427b91cb8372,0x00066cfe0b0491d2,0x0000f2386ace983a}, {0x0004d1eb32912137,0x000de9a62ae4930c,0x0003e89e3a2f82b2,0x000c7f07233853f9,0x0000f8063ac81777}}, + {{0x000b56759ad2877c,0x0001d865c754ff0e,0x0006e9a846f45464,0x000fc326fe701a23,0x0000586ef16c6e40}, {0x000b6e024bafad93,0x000234da906a3f62,0x0003276a0c8b42bd,0x000852998e1eb4da,0x00000d0e5fc36cbf}}, + {{0x0002ae1e8b4dfd4f,0x00069301cbac1b6b,0x0002a39acd754d5c,0x000ab87609762911,0x000086b599a83ba4}, {0x000dea799f9d5812,0x0008a2fafeaa26c9,0x0002505a50473b1a,0x000322f469af553b,0x000027d16d7f6a43}}, + {{0x000f73cad3d97f9a,0x00047f1374553316,0x000954e7c52bf3bb,0x000410f53eafeb09,0x0000721dfee7d732}, {0x0009821141d4579b,0x000bfa3bd435b492,0x000fa60153411321,0x000f0dffb355aa17,0x00004e7ef4ac8e42}}, + {{0x000c97c593710000,0x00007f759c18604a,0x000db6b65e1c48c7,0x0004953f62ecc5a5,0x0000a78b17338a21}, {0x000819dbcc8ad945,0x0006889c34006be1,0x000b4840a70dc04f,0x0001bff62557b4a6,0x000044c6adeb0bd2}}, + {{0x000f24e907a544b9,0x000aa13da2106a00,0x000e4994ba7520dc,0x000d706e939b7511,0x000018b6ba74c275}, {0x000e0fc644be8924,0x0006bdaf6c42d3e5,0x0005c13fe707a981,0x00054a20145567f1,0x0000818ebab2130a}}, + {{0x000d3ad58d2f767a,0x000d37e7c77328aa,0x000afcc98dc5267f,0x000d4aa919cc88c3,0x0000a2e6ab0cdb8c}, {0x000ec04d0c63eaaf,0x000429ffa832d46f,0x0003a631fa1cb92c,0x0008b2778dd178e4,0x0000b5ae1ce2dc78}}, + {{0x000fb906e77de048,0x0000706dbb9768b4,0x00017c01d7992bcf,0x000e01096e6a13c4,0x0000d96332d3956b}, {0x000c93a413aa2b94,0x0005bc98c8a5902f,0x0005f113799a4d91,0x00028112c2940756,0x0000072690f61e4f}}, + {{0x00007cf02ff6072b,0x0008dad98cdc36e6,0x000f56609a47d2ca,0x000da00f471d1ef5,0x0000cf86624a264a}, {0x0000687aa9e5cb6d,0x000147401c6cb70c,0x000a61435c98124f,0x000ea5b189635fd4,0x000028fb8b079d98}}, + {{0x0007c2a40c251f81,0x000792da44beb9a6,0x0009b542388cd5d8,0x000dc1337deb96e0,0x000050467db74287}, {0x000debbcdabb8397,0x000281839a3ee161,0x0002d202ba79e974,0x000d964b8dd3c265,0x0000b3e67f859f97}}, + {{0x000d78fb1cb6ac9d,0x000dfa1d0d455aa5,0x000a5bf95ffa13e8,0x00005d669295dd2b,0x000068bd1f909aff}, {0x00086f926d783f2c,0x00033c3aafc1af0d,0x0007da97c543a59b,0x000e457fcf81d27b,0x0000990a057925de}}, + {{0x00075b8519cce2c4,0x000f6e13d8633e67,0x000c1605cfc9af71,0x0005e8374a4a6f47,0x00006ba42456fd20}, {0x000eea4d3fd524d4,0x00012de1acc2a06f,0x0004e2b421e72464,0x000024b53816f133,0x00009e5918ed22f0}}, + {{0x00030b665c7322db,0x000103c1b3fb4395,0x00072f685cf12cc0,0x00091d170b018601,0x0000915ee22cb583}, {0x000f03ba317db248,0x000897b8ffc49afd,0x000d3d05087dec65,0x0000e6ff46597be4,0x00000a1c1ed80650}}, + {{0x0002a9678bf030ed,0x0009805601488490,0x000362426fb5e9c9,0x000c3f9dae0a9263,0x0000caeecf579e30}, {0x00087bb518d0c6b1,0x0002bb985b9dc0d8,0x0007bc3819918115,0x0002019d186898ef,0x00008168ffbaee46}}, + {{0x000cdaa2502753c1,0x000641407c419a04,0x0003564e5bb279e2,0x000016dacb03aaf2,0x0000833658281e61}, {0x000b8c4eb8098772,0x000dca0e672e8684,0x000ee5867b336e18,0x000fd1cfb601f034,0x0000733edbe3341c}}, + {{0x000809a26025c3c4,0x00065350df88b15e,0x00002fd8ee6e981a,0x000e9b5237623785,0x0000791f2164c12b}, {0x000678925f02425c,0x0003ba974443b725,0x00041cc52ec86319,0x0007f1bc0ce882fb,0x0000266ff7fb25c0}}, + {{0x000a8c3017025f39,0x000c6b9579b43d4d,0x0003716ecefcf628,0x00016dcc4d00161f,0x0000c27ebc4f8011}, {0x0000ea11da1767e7,0x0001d7004c575eba,0x0002373b7fe15145,0x0007abcace6df68c,0x00005c3dffecdbc3}}, + {{0x0002a73ddc925fcd,0x00005f65ee0b3dc3,0x0001cbfebb679c84,0x000a28a15a329545,0x00009889769c76e9}, {0x000ce7fb28ad2470,0x000400894d79ec20,0x0005e3ea7e99146c,0x00003171457d7c9f,0x000097b266238030}}, + {{0x0006ae6cf9f82a81,0x0009338f473adb7f,0x0003856c3319decb,0x00061b963ab38628,0x00003e3172fc06a3}, {0x000f8dc7d5a006c0,0x000675fba7522959,0x000c22c9e2dbc27c,0x0008b2c1227ab287,0x00006f61f7571a26}}, + {{0x000b97104779ce2a,0x00016aadcb1d1b6b,0x000aab2d5aca8381,0x000f12897ae0bcae,0x00005c14ee7fbfb9}, {0x000c583f17a62c70,0x000c173759f6aa00,0x000c9a88f39eb962,0x000c5e1eeba1d486,0x0000ab6c37adf016}}, + {{0x00047dba28a0749b,0x00063e519165a2a1,0x000810715246c20d,0x0000b8a068d1b1d3,0x00001e7018d24816}, {0x000b1faf380ff628,0x000d73cb2c1e03f5,0x00091a7daef7fb1d,0x0005616ab539a8fc,0x00003ddb70873f9b}}, + {{0x000e211fe7df7a4b,0x0001563f6f40c550,0x00076879ca7cd07f,0x0001da00de363529,0x00005f83f8695574}, {0x000d25ef3d8ac3d1,0x000f52819f024ea9,0x000f4a5646fe2066,0x000de33ab2b9c2ce,0x0000e155d966ffa2}}, + {{0x000a19bc3a72d004,0x000b4513c31b0eb0,0x000c646374037665,0x000638efb2b6bf04,0x00005c34d6e48cdc}, {0x000e10ff01fd7967,0x00018e3667b856f1,0x00021d0c04dfb810,0x0006ab70eda25390,0x0000a94e9fffa06c}}, + {{0x000b0d9bb9aa8822,0x0005cc05fd102d3b,0x0001ca64eea20e4e,0x000cbdcd7eeb5f1a,0x0000fa6b43ce6327}, {0x000e3cf3aa91121f,0x00094a34079bb577,0x000e02fc08c6bd5e,0x000bf7e7e5ba3960,0x000016dd2c480141}}, + {{0x00076d980101b980,0x000db82f0f66b572,0x000c3eff3760883f,0x000b4089d7de754b,0x00003b606435dc2a}, {0x00053dfe05beeac9,0x00022c3325cdcd6e,0x0004f03c3f2f1e86,0x000c1b4d0f792177,0x00007ca7221d552c}}, + {{0x0006afe1cd19f723,0x000cc183fbeb5a0d,0x0002c403ca20915d,0x0004426fda4b4083,0x00002738eddee425}, {0x0001df6b5eccf1aa,0x000188bbe1f0469a,0x0000dfc934b5aff4,0x00062791359d7f57,0x000018be23690088}}, + {{0x0000fbab00ed3a95,0x000423cdf8bea5b3,0x000c5679734c6137,0x000ae1dc5c5f46ab,0x0000cecf93e082a8}, {0x000be41a968fbf0c,0x00025a5c7f3d7d3d,0x00087a9c7d23d458,0x0001ca328f69a0c0,0x00002d7547207447}}, + {{0x0009f4a4eb732ec8,0x000d31ca6bed36ec,0x0004578926c943bb,0x000c06564535e1f2,0x0000b84a8eb77e2a}, {0x0006cd32499dd5f3,0x000dded04e57e093,0x000305d9d12053d7,0x0000a21bdd0076e4,0x00004a527b94f67f}}, + {{0x0004af09cec46eaa,0x0000c58b9bc7e79a,0x000af2f75b15347a,0x000434cbd2796f35,0x0000c957990e051c}, {0x000dda3c33a655d3,0x000e58514aa32669,0x00053dd415d503c2,0x000f78afa1133737,0x0000f0546733b754}}, + {{0x0005677496125bd4,0x0007f775006cbf18,0x000037899fb0023c,0x000a57ba0f072f3a,0x0000222b6eb4e4ae}, {0x0005e767866d25a6,0x0007e837aa6f3dde,0x000f1cdb8b6eb04f,0x00083bf315591a2c,0x0000dfb4f418d4e6}}, + {{0x0003ea448ee1f3a5,0x0006b5a2afd57e92,0x000ea49489604d9f,0x000d2f6e1d4a3340,0x0000b45f1f5044cb}, {0x00083764acc757ed,0x000793d68ff75faf,0x0000e404ba7cf9ab,0x000fdebad62f69df,0x000065f33c2e2bda}}, + {{0x000de15a377b14ed,0x000abe39f60cc365,0x000e681486bf5463,0x0003a792030d2d2c,0x000095867f0b6f84}, {0x0000244ef5ab0172,0x00012ab55d12d39a,0x0006391690bd2d8c,0x000c8aa9503db341,0x0000d4e25b117660}}, + {{0x000b3b5e224c5d71,0x000c58616919760c,0x000142552fa3baf8,0x000bf58fbca1138d,0x0000ab18bf18669e}, {0x000f53e9bdf25ddf,0x0002db6cd15455e6,0x000e8908004cc0bf,0x000ac0695bef4995,0x0000e9459a9004a9}}, + {{0x00089cacce9bb324,0x0001b7de8285ad2d,0x00051bd4bddea65e,0x0009a722ed8c35b3,0x0000150ff364c0e1}, {0x000c801345f4e475,0x0000d03a266c86e3,0x0005b1f133bf21f7,0x0005172ae110d485,0x0000d6aaf6a57262}}, + {{0x00012e1813d28f11,0x000d6ad7a5231e0f,0x00044a17b6000e11,0x000a00b7d8deefc7,0x0000e990b4824c05}, {0x000daee93e976d58,0x0000c6610d6368fd,0x0003dda88696241d,0x0009463204e7c389,0x0000bccfa65ea3a6}}, + {{0x00025b4c5cd14112,0x0002df3658b1b594,0x00084cf93701b404,0x000d60c3e56bca47,0x00007de5f15afe68}, {0x000cfcef8d53f19f,0x0000b40a730d4ab9,0x000ee0a8addb1031,0x00019c7fa73cd14e,0x0000d54874942497}}, + {{0x0006316a8123ef0f,0x0003f7f9543849d6,0x0009e785473c32db,0x0005063e2ed2090d,0x000098a932a8d9f0}, {0x0003cf60c6aa20a6,0x000415279bb2c5d3,0x0004a73079a32ba1,0x000dbd1e3202cb77,0x00004ed4bc548c42}}, + {{0x0001a06d4caed0d9,0x0006871d22b3c20f,0x0003268d7b802140,0x0001264426ca04d1,0x00002377007b5f4d}, {0x000cbc371f21a855,0x000aa82369ba4204,0x000c858f918461b7,0x00056970c07d313f,0x0000deb5a5132bab}}, + {{0x0009d46d5eea89e7,0x000398437f4bd595,0x000fe254ffdff842,0x000321821071e43c,0x0000241769705468}, {0x00088b9102cae3e6,0x000d91965dff5d82,0x00078d8472d143e3,0x000730a0c9a376a0,0x0000fc0da3186028}}, + {{0x000eadfe45083a2c,0x00079e5b4bcda2ba,0x0004b8e7f66bc721,0x00086a6c826442d0,0x000019f54522c4b5}, {0x0002c495b7eeed57,0x000d0aa9dfa16018,0x0003884add9954ec,0x0000413403a8ecc7,0x0000fb17de30bb39}}, + {{0x00064c5abb020e8c,0x000359c4eec7694b,0x0004793e53d18c18,0x0002e5dc4673ef1c,0x00007b8aeb5c5609}, {0x000ca43f0f8c16bb,0x000c2679b2f63aa1,0x000a205c9224ed5e,0x00028a5d56eeaf55,0x0000fe115bafb8e0}}, + {{0x00008493927f4fec,0x0003b59aa7c597e6,0x000e90a51f91fbf9,0x000822d85af7696b,0x00001277b7938ccb}, {0x000656ee7a759524,0x0000528da5f5395b,0x000a4454f00df7de,0x0003c0c9c231754c,0x0000ec971f4baa2d}}, + {{0x000c507e75d9ccc5,0x0009edc9030645c3,0x000b44bdc63b7be8,0x0006a1e7e09c665d,0x00000d60da1b841c}, {0x00065ee08df1b120,0x00097ff089df6f9b,0x000e8013d3873487,0x000cc89c331a663f,0x000017f5de95f42f}}, + {{0x0007866e8e575672,0x000ed9fcdb184307,0x00012e174c9f781c,0x00052a18131dda9b,0x00005d84aa3aa037}, {0x0009e094d0c0ce2b,0x000ae2bebba545e0,0x0007284c71564008,0x000baa47e8ad31a8,0x00007c4b46d47e7b}}, + {{0x000a7b397acf4ec2,0x00003ea8b6403e22,0x0009692850426c40,0x0006703e3295a64e,0x00002aabc59c6a45}, {0x000714c5f5942bc4,0x000dba3182edb929,0x0004152ba9a6168b,0x000367e216a66510,0x00006908d03f6926}}, +}, +{/* digit=11 [{1,2,3,..,}]*([2^77]*G) */ + {{0x000d8745a1251fb6,0x0008672725c7a9f5,0x000ffe89e967747a,0x00035db95c33e531,0x000009d211049649}, {0x0006ca82fe122271,0x0005f426469dcafd,0x00093183caf9b5b9,0x000fef1e9ee04c56,0x0000084a333d8146}}, + {{0x0009933aed1d1f71,0x0003405630909664,0x0002e39cf566eaff,0x000124245057f0ad,0x000048ff65b2f832}, {0x00089d4cf94cf0dc,0x0003920c58b3042e,0x00061aa0d319bec8,0x0007ac6a26762653,0x000086fa3034fbc8}}, + {{0x000d2ab5c8b06d5c,0x00023e4eac46fc83,0x0006f7779b1a785a,0x000504f99315bc84,0x0000f31d817af9ea}, {0x000fe6a15d7dc85d,0x0003e4016b332391,0x0001cb4c72f132b0,0x0005a059547fe318,0x0000b66d8a735015}}, + {{0x000d7e1adc1696fc,0x00024acd72d06b66,0x0001b743598ebe59,0x0005eba5f24550cc,0x0000e23139474b9a}, {0x00022d4db067df91,0x0005baff9b00234a,0x00000c9c198dda09,0x0006950bbc75a061,0x0000560a9c8a39cf}}, + {{0x0006d3e99e0925f5,0x00061322375acf00,0x0006af5ba2dd74a9,0x0004f1758b446ab5,0x0000029268430b9b}, {0x0004cb41aeaffa32,0x000f7b9587c1e2c3,0x000d1350c8b17203,0x0006044d559207ea,0x0000b66a2161b7f9}}, + {{0x000325efe51bf748,0x000dde4600940850,0x000da2f259c4f579,0x000f32dc87b92a76,0x000089de4e0efebe}, {0x000ec06646083ce7,0x00054fe127736900,0x000344110be2a033,0x000d203dd1da35c5,0x000057568b82802c}}, + {{0x000977900f7e6c85,0x000ebfacd2f07555,0x000fde37538e8b94,0x000dfcaea1f3af03,0x0000e11a1d8c5881}, {0x000b02ec1e2f2ef2,0x000ba605a6c5b3a6,0x0009a0b2d193d2bb,0x000846125ffeee33,0x00007b6a724be0c8}}, + {{0x0000f1cf1c367cab,0x000b190fbc7de405,0x000a110329bc85a9,0x0003a8f373c4a2e1,0x000064232b85d039}, {0x0007eb0167dad297,0x000124b78ab2f557,0x00029348b1604f30,0x0003419baa94afe8,0x00007fbd8ddb1654}}, + {{0x0000ea5b964e39a7,0x000c60d3c76edab5,0x000d11964d4c29e3,0x000c2f10dae67c56,0x0000307a8c055ffc}, {0x000c1aa91708c3b5,0x000bd8bf0eeb65bb,0x000a34db7a151e62,0x0003a81cb533816f,0x0000139e05cf2940}}, + {{0x00051b494a7cd2e1,0x0000f699336c6ff6,0x0009a896a5671ffd,0x000cef5f5fd2cc97,0x00001e893a8e8148}, {0x00006a165cf7b104,0x0008850d84859889,0x00035b3de81b6717,0x0007993c0deb358a,0x000023ac85601d29}}, + {{0x0000d87dac50b74b,0x000ea869734caf58,0x0004e28fb28b2b89,0x000739e9a3b93687,0x0000b2c9190d5f3f}, {0x000691884a9d5b7c,0x0004be770374199f,0x00038efe27ebe232,0x0002d23442e10707,0x0000f9f3f578f908}}, + {{0x00069e1096187086,0x00046183f9b1719f,0x0006a21afcc9e836,0x00041f8c203a9536,0x0000aec5d6d668b1}, {0x000f78a994f04e95,0x0007d71245b0ee2d,0x000e43f4fb39ccae,0x000a986875a4a997,0x000007dfe122b2ce}}, + {{0x00081cb489b03e9c,0x000aaec414fa4fbf,0x0001b3ae5db86ec5,0x0003fe3ad444f9f5,0x0000a7d33d6d914e}, {0x0002f5c0ae6c4d0e,0x000d93969568a9c3,0x000a7467ea9ca1d1,0x000ac5b8043c311a,0x0000832e75dc21b5}}, + {{0x0007aea5232123db,0x000bb5ae86db314b,0x0004668ed08307c8,0x000c3856e7165caa,0x0000170458c64d3e}, {0x0003ec6c19bb9866,0x00064e0304ed4d2e,0x0009f9722c5f3484,0x000c0a317695a06c,0x0000c7f7317acab1}}, + {{0x000940e9d6d2e8b4,0x0001149f7c976295,0x000713885d318b8c,0x0000fde245320497,0x000068d834be8a44}, {0x000e5b2bfba796e0,0x000b6d71f116d81f,0x000b66e53152364d,0x000192bb8c7c59b5,0x0000b12c61b2641a}}, + {{0x0004802fcf0a7fd8,0x0008d488b01e31f1,0x00052b49842fd078,0x000200f1d78d6d99,0x0000eb572d987ac5}, {0x000a44c4d194a88b,0x00090a017e66e0a2,0x00088aefcd2b63fd,0x000a10c8efc6c8f8,0x000076f6bdafa881}}, + {{0x000314bb46c2397b,0x0005aded2819187f,0x000764d34004cf56,0x000708f9ea570438,0x0000ba4521828084}, {0x00045711171121e1,0x0001d7c9b6710647,0x0000f7507ad7b7eb,0x000bd1cacfbc4073,0x000078cd8c6d7ad7}}, + {{0x000e101b2a672385,0x00075f9c14f2bf0b,0x0006620753556d36,0x000609c04b7831a5,0x00008ca59bbc9d9e}, {0x0005392a569a73b1,0x00084698f6c94bc4,0x000add755517a52e,0x000b8475643da5ae,0x0000aed0cd53a581}}, + {{0x000ff8480af13722,0x0003d1ba5d1fb9b4,0x000f98d31244c311,0x000c2a0a5dacbef5,0x0000c3323e86375b}, {0x000ab4a5594b1dd8,0x000b4eb4797e17a3,0x000886a19a1928bf,0x00074a683af245e4,0x0000979d546f2b5a}}, + {{0x00026bc19f9e9676,0x000288fbbf4ea0f7,0x000707d40d9d0315,0x0006e06fd6f51db7,0x000033084d9c3f6e}, {0x0009cdc55667eaf7,0x000abe44d56fedcd,0x000962b1473b7f92,0x000cbf8b2e39b64e,0x0000d408f6f6671f}}, + {{0x0004ddc164a89bb6,0x0001def3bd05cc63,0x0008decbb74a42bb,0x000595b280dbb242,0x0000103f6bba02c8}, {0x000f581355a5752f,0x000710946674fa2b,0x000a0223b562f96a,0x000a245e4ca16d6d,0x0000e4781a018d3a}}, + {{0x0003075f8dfcf8a2,0x000a756698259eea,0x0007d3fd8a284f0a,0x00091e93fca25086,0x00000757b5f469d6}, {0x000402093b8a5ded,0x00091bc06da6f2c2,0x000739c33d3f9335,0x0006e44178293eb2,0x00002a3e7718cd68}}, + {{0x00049f4cd941534e,0x000b03c71c0ea76f,0x00097f7e30d37406,0x0000dd372d93973b,0x0000c17e23a9d7fd}, {0x00005516f496ba27,0x0001c6ad50e7e329,0x000e7eff56a69317,0x0004cf54e539a283,0x000052737e788e1b}}, + {{0x000932c68af43eec,0x000db03d00bda2f7,0x000b061f55502468,0x0005ad25dc978f2f,0x00009a1904ae8c81}, {0x000538d470c56a4f,0x000e293d8cedd3af,0x000108ef3159abc5,0x0001773a37245f20,0x0000a17081f123f7}}, + {{0x000fb2b10c8c0f59,0x0009b065054727b0,0x000c3bfa72102c3e,0x000d95c94564df8a,0x00008102033e09da}, {0x000643ff1d18a13b,0x000127fc5af06989,0x000eaafd835eebd9,0x000e97578d096afa,0x00007a893428ef3d}}, + {{0x0006e8decf2a73a0,0x00078e5519942a20,0x0008d53a2066a639,0x0004aa3a6a088ab9,0x0000ce7c67c2d112}, {0x000c671759a113ce,0x00026f6f67fa48ce,0x00036727be3b373d,0x000d807455d479fd,0x00005a428ef813c0}}, + {{0x000dbc81c86682bd,0x0006e8d02b2ab853,0x000bc329ab78d272,0x000147daf69bed8e,0x0000b6b40b3f93b2}, {0x000a77db8c4961f6,0x000bc0e5e0abe42e,0x000e8b05eb1a12f7,0x000a8040ec527479,0x000080273925ab60}}, + {{0x000ea5f16b1bd5e5,0x000ffde30ad36bfe,0x000353b9ef957e41,0x000feb7baf664e6a,0x0000c87331276d14}, {0x000f98cb65f57cb1,0x00014e0cdd414e87,0x000881440db60a62,0x0002aa57c16865a6,0x0000093ef1a56ab5}}, + {{0x000afb53f4ece64e,0x000d9604551ac095,0x00026b8cd6a6bb02,0x00068975d44b4e0b,0x00005f9a99ad9712}, {0x000c42511a7de841,0x0005eda469ddc08e,0x0006c90a28356809,0x000831637bfba16c,0x0000cb9c4a0ce229}}, + {{0x000cbbabb2eec644,0x00049a03adbe93bc,0x000e86ac4a0c23b6,0x0001e61f7aa00ae0,0x000070b941f3c140}, {0x000d6799df435742,0x0008ef65d8105ad8,0x0007fbd814ccfb8a,0x000209fbce80e3aa,0x000073291adb508d}}, + {{0x000b46b42a92806b,0x000bf86ab44af5c4,0x0000bc9f8810684e,0x0000539591640bca,0x00005efcdfd0c4b6}, {0x00089076e9edd128,0x000054d792f916fc,0x00003116de29d0b5,0x0005a4245fd01c9b,0x0000503523648176}}, + {{0x000a9b2b4b4b67c8,0x0000680206041fe2,0x0008058d8c1d10df,0x000fbb1d64abfcbc,0x0000943b9b2f12a0}, {0x000d9143b3def048,0x0001cce775ff90ee,0x000bc904085ab3aa,0x000dfae05fd4ca7b,0x0000b34a56562c75}}, + {{0x000c94a10358560e,0x0001be5c28aa41ff,0x000c7eb152d8a507,0x000f5d0915a0fc4c,0x00009efab066f6d0}, {0x00047a9d19e9b915,0x0004b276154cdbab,0x000fede0d8cfed74,0x0004eec54357ae2c,0x000020630df69f5a}}, + {{0x0009f7ce382360f1,0x000978bf58572575,0x00058d46cb6db05c,0x000b7a1917d61d6c,0x00004f8e4920d20c}, {0x000727a11c203407,0x000f3f7ccbb6b68a,0x000e09a200386f86,0x000a34ec8bc6ccfe,0x0000d76ff4b2b7ee}}, + {{0x000ebe7db15be7ac,0x0004189f0302a7bd,0x00019336467a0805,0x000ebd96bf0ea9c1,0x0000824446822837}, {0x0008e8b20d841b88,0x00085bb8a54f32bd,0x000b20236127a054,0x0001fa03dd4ca663,0x00007714718a0349}}, + {{0x000caaaaa8a5288b,0x0009ff23a1c94dab,0x000220e0c91cc0c8,0x00044984c72c6a3f,0x0000cc20bdfc2321}, {0x00042daa20ede1b5,0x000c24a005156e2f,0x0004b8c4bc441f00,0x0009a46f46a5b673,0x00007409503cb56c}}, + {{0x0005261e4585d453,0x000d3734e6429f73,0x00070ee6c9231fae,0x0001bee158a176be,0x00005f1068dac350}, {0x000f900a2d261155,0x00029f0afee36bee,0x0002420a1649406f,0x0004abef43a60abc,0x000009002a825aee}}, + {{0x00036a53ff3571bf,0x0007937927c1b468,0x00033c71624f98b7,0x0001957254256a45,0x000027abb0cc07ee}, {0x00064fc5c6d5bfd7,0x000180cd7a77d7cf,0x00098f5346915c75,0x000b5f69f5901287,0x000072b0da9681d8}}, + {{0x000260c2e03fa69a,0x00099be1a3741244,0x00006b96036cf0e3,0x0000f5ce7c1633ef,0x000071a4c56071f9}, {0x000125133c673db2,0x000053e8c1317a94,0x000f6c734c0bea51,0x00000141a8a699d4,0x00005e78c88541ed}}, + {{0x000acf88e2f7d902,0x000757be32cd5c18,0x000eb5ee9fdbf33d,0x000114ea085cd7d2,0x0000d702cfbd3201}, {0x000ebdb85c88ce89,0x000b8e01d617b6e0,0x0007333ac23a3ce3,0x000b6aa041618e56,0x0000dd0fd8fa57ed}}, + {{0x0004702b57872b88,0x000ee57d5fe127f7,0x000cf3d402ef26b4,0x00067a15426f0a57,0x000047e2ad1e5a60}, {0x000d9a009996a74c,0x000c6a26115cd474,0x0006f4d4316a56ac,0x0005b642a615c3d1,0x0000c3fc9666adb8}}, + {{0x000da73ce07d1b0c,0x000198ad4178386b,0x0002617f4d82910c,0x00076f524f82cfcd,0x0000c2f5e8eaf691}, {0x0002550b8c30ccc0,0x000a1a8e575a8270,0x000ab94597b856ae,0x00038ebb822fefb1,0x000085928bcec24e}}, + {{0x00002ecba8f4b4d6,0x0009a0b4d58b5d04,0x000227e7ac07cd4b,0x00086efd8dffd529,0x00001d44d0c91bf3}, {0x000dc2b135e6f4d6,0x000b879410efe486,0x0000088b5680962e,0x000686461bd343f1,0x0000aa7607742e28}}, + {{0x0003d118fb98871c,0x0002fbc76aff8046,0x000e03614cb26f5c,0x000dee14ab8eddfb,0x00008eb579c80cf2}, {0x0004c15c93bae41a,0x000c9aeca3b2cc00,0x0001e9ab146fbae5,0x0005c0c71235cf0f,0x0000dfba9353ec28}}, + {{0x000d013f216c980b,0x0008479e0bc188de,0x00097a237c8ac4fb,0x0008e6f29b89c6fb,0x0000697b7814922d}, {0x000c639ddb945b56,0x00078094c3a93142,0x000266c90447b06c,0x0000466dcb364272,0x000033aad0909385}}, + {{0x000936bb57c64779,0x0004594dbcc6a36c,0x00091a67b871f8b6,0x0006f498d0fb62a5,0x0000d40e08251d92}, {0x000eaf6f2d84b5af,0x0008b565b6443111,0x00083188b228993f,0x0001961ccbf5922c,0x000087b30ac2df3e}}, + {{0x0008b317642bca8e,0x000e72800f17b865,0x000bf94451a032d7,0x000252251dcae579,0x0000ba6b8ef34a2e}, {0x0009cadd44856928,0x000e0986e9be5c8b,0x0000db44884bda40,0x000188516d16a42f,0x0000ec80051214d4}}, + {{0x000610798fa7aaac,0x00043073aa4eb2b2,0x000d6b19b41209ee,0x000caf31570359f2,0x0000be6868dbc577}, {0x0004bdc32c04dd3e,0x000defeee397186c,0x00086c0cfa6c35fa,0x000fe1d4a1b312f0,0x00000a5ccc7b9461}}, + {{0x00078aa1536189f8,0x000f3a6df571c322,0x00094560e1126c55,0x0006e08f71a602b1,0x0000b2d7405b24bd}, {0x000939e3738be712,0x0009fa4d97a98481,0x0005ba915b5090b1,0x0008a9f16c65a3f0,0x00001863ad3cae44}}, + {{0x0002679a7aae5d37,0x000c9de5c1c4d24e,0x00005b6297076013,0x000fbabd50f8babb,0x00003c1abe2de66e}, {0x000b422f2488af76,0x0002063ba575efd4,0x000a69457e4105d0,0x00073b1eb60a8b53,0x0000221000929459}}, + {{0x000547877a50ec69,0x00067a37a72cfb25,0x000e18e7abf0392f,0x0000af10a7a19c4b,0x00000d8ea16b5b1e}, {0x000a293ef953f575,0x00056dc5465a7582,0x00051071790a64d0,0x00041f7a79c497e2,0x000060dbb7c68cb6}}, + {{0x00032864b66abfb6,0x0004f90309001d8e,0x00084941ad26f52e,0x000957bee3f64355,0x0000d3b3730b69f5}, {0x000a62f4789dba54,0x000532b5c9b79ff2,0x0008f9a0e91fcb81,0x000cb5b446cb7d6c,0x00008f625c179b7e}}, + {{0x000e8011c6219b80,0x000928ac2f23baba,0x000e20588e7a562d,0x000051e1b4873226,0x00006ee1cad775af}, {0x000ae43faff79f72,0x0002452ee9e0da29,0x0005f4bd0c141a41,0x0004f7fe127f6f19,0x00009c6ab4f272f3}}, + {{0x000147730448112c,0x000124a386567b7c,0x00031501082b51af,0x000cd36bf2028a2f,0x00009a4a0202ea88}, {0x00095d8257e5818e,0x000fd4519b16f63e,0x000a910bfdd8efa0,0x0004a90d8973e00d,0x0000d49d0783c0fe}}, + {{0x000ac5eb7caee1e4,0x000d67f4da57ac3a,0x0006669b91033898,0x000aa002145c0e5c,0x00002daa68901aa2}, {0x000c15c1a1d885a8,0x000074b76817629c,0x0008f8f2825572ec,0x0004900312e4359c,0x0000107f8ce01965}}, + {{0x000f3a36fa6110cc,0x00013b93561f516f,0x00057522b74fb1eb,0x000dc5ac0c904784,0x0000fd321052bb8b}, {0x00084a2cc80ad571,0x000576a9b6372d68,0x000f4e8cd7c27fc3,0x0002f02461baedad,0x0000d56251a71724}}, + {{0x000d209c955bef4d,0x000136adb0470b80,0x000c74feedf02cad,0x000a4420d7cb915e,0x00002503375e111b}, {0x000755edf53cb366,0x0001d368551b9671,0x000a025a454dcb61,0x00044506d69aacc8,0x0000be946c7477ef}}, + {{0x00046d1a995e0941,0x0006d51e04d87199,0x0001e311365e848f,0x000503d62f33006a,0x0000541c7c1601de}, {0x000c9faf4acfade6,0x0006e4cd0b714daa,0x00051cd770e58589,0x00016cf44fd8690a,0x00000fc20ed60310}}, + {{0x00004eca42768678,0x000bb4f3499358b4,0x00036e5bd46f6c3c,0x000b46c77ca007c6,0x0000018f5e5fc458}, {0x0002270e47b668f5,0x000d9e14f203a120,0x000ff9b4dcef48cc,0x000ddcd3f98bae62,0x00005acc0361589e}}, + {{0x00012af64db44442,0x00049ecdd4803fe7,0x00030978a19e9d63,0x000733c08bc047a9,0x0000dbf24ecc1280}, {0x000e38c2cd706b27,0x000ac59017b93c0a,0x000e0f5ae5b012a5,0x000fa2c943c38c72,0x000086167eac7176}}, + {{0x000897d594881dc3,0x00089fb820c1e5f9,0x0005018de6b5efad,0x0006ce82179093d5,0x00009ad7d323bac5}, {0x00022e02cfc0e816,0x000196d89daab551,0x00064fa09117c466,0x000ddcd62d01e1cb,0x0000a309b4e9e9c4}}, + {{0x0009fb7abea49b1e,0x000c30e2c6c5fa97,0x000afde7ab4b1d27,0x000357dd61c2c423,0x0000b6614f97786d}, {0x000816b7f6f7459c,0x000e49360e7b4a5d,0x00009914ce431a44,0x0003d7cc27a032c3,0x0000ea5d68b8aede}}, + {{0x000f6653a0a3f957,0x00060ceba27b3668,0x000728fe98936941,0x00056219981fade4,0x0000102c8a0fa093}, {0x000310e235d21c8d,0x0000eefb7f7bbb80,0x000958a67505e55d,0x000feed0a9081112,0x000067e106b1d851}}, + {{0x00011a9431dd80e8,0x0009f3306cd9b840,0x000b3b730eb7c7cc,0x0003d2a0fadd29d1,0x00003858b5c7e37b}, {0x000d193b6251d5c7,0x0002a352d952bf4c,0x000fbc0511cca1fd,0x000636566157a490,0x0000990a638f9b98}}, +}, +{/* digit=12 [{1,2,3,..,}]*([2^84]*G) */ + {{0x000692a87dec0e16,0x000d97b39d00e5aa,0x000cfa0b5010ded8,0x000a281b1b80c854,0x00006beb87700f8e}, {0x000f5313476cd0e8,0x0005308d394950d7,0x000479fc6a63d0e6,0x0007419a09eea953,0x00002ae98927499e}}, + {{0x000b9105ca7d8669,0x0001aadb3b34ab58,0x000eac0bc582967e,0x000af4f9ae4447cc,0x000019c667d0bf56}, {0x00017b160f5dcd7b,0x000f2dcaadbc9aec,0x0003467f5ec697b9,0x00032e5b98f34146,0x0000187f1f859671}}, + {{0x0007a1d214aeb181,0x000c641432f790fe,0x00091a0c41506af3,0x000bc3ab5565f9e5,0x00000d41a77c44f1}, {0x00065e4a84bde96b,0x0008420a6a1ca09d,0x0007f9ce742f060d,0x00039eb52a3bfdf2,0x00006bdb65ceb3d7}}, + {{0x000dcb6ec7fae9f1,0x0004dfb66e5aeb5d,0x000445d52995f271,0x000620cee95d8e69,0x0000b6c2d4619e27}, {0x0001c318129d7161,0x0000f958c1aa3262,0x000f4af63b03909f,0x000df67c468ef91a,0x000062c42a00ba5c}}, + {{0x0002343753b9371c,0x00099f1f9cd72f68,0x00045db9629cab45,0x000998971623abb2,0x0000507db09ffd79}, {0x000f652af036c326,0x0007a5018e5c4e2e,0x0008be35086f0cc7,0x000327610a73d4ab,0x0000519b397de826}}, + {{0x0005eef9c053df72,0x00079300ea6fe8cb,0x00049cffb8de25b3,0x0009bbbb03fa92c8,0x000042e43a808416}, {0x00051f4dd6f958ec,0x000f34445a8de4fa,0x0000d89496925a77,0x00039026e72a50e9,0x000066648e3eb1f6}}, + {{0x0001957173e460c5,0x0004e0704590b2ab,0x0001c71621bbbce7,0x00065d70a90dbddb,0x000005e399e65cdd}, {0x0004dcb57797ab7b,0x0009ba2ca8e86843,0x0003336c160ad35b,0x0000149bfdb1e0de,0x0000bef99ec88b39}}, + {{0x00096f31711ebec8,0x000f4e98fdc46c3b,0x000b4411f2da40f1,0x000bb6399774d357,0x00007c8bdf495b65}, {0x00089e3c2eef12d5,0x000aec7471f3da3a,0x00012c594de95bb9,0x00056b100f225bd8,0x00004907c5d7b75a}}, + {{0x000c5f08db60e357,0x00068a833319a93c,0x0001683c9743e3cd,0x00007e0dad5c41f8,0x00000c1e7da0c341}, {0x0004a39a6be09070,0x000586d0b7d30edc,0x0002bfa6036d4703,0x0004148c76da0327,0x0000b4a07ea0f08a}}, + {{0x0004d2945c1dd539,0x0007931debb5699e,0x0007f00e0cadc589,0x000a0e4f49fcc7a7,0x00003057bc0373e5}, {0x0007ecd027a4cd18,0x00034614011a2f8b,0x000677c68114734b,0x000f4f67a01db767,0x00009d9be5efe273}}, + {{0x000cb2e089808ef0,0x000dd59e4107d225,0x00011b9c9f1f7a27,0x00015953afc76182,0x0000361bc67e6819}, {0x0005d0b7f071426a,0x0000470725672a86,0x0006bcabd6a3c181,0x0001bb9e3bca1e0d,0x00001b02bc1e0859}}, + {{0x000ee5931fba2393,0x000368bd91d1e0de,0x0001a3c1df47424d,0x00033adf8886f407,0x0000f7d41e8d8192}, {0x00023c2cf6eb9980,0x000f609a287f7086,0x000c9076286bb49a,0x00054b942bb24963,0x0000ef6eea555a96}}, + {{0x0002d7236f5defe7,0x000be6f991765f6d,0x0008ce0c7fa9922d,0x00055dfc8c5ecef7,0x0000b44589e2e09b}, {0x0003bca9ea837700,0x000f2ab71547e11b,0x0001ddcc0d7fa2c7,0x0007072a3dd6fa2a,0x00009acb4305a7b7}}, + {{0x0004a2e649d4e574,0x000fd917526e4add,0x000b44ac4cd53a2a,0x00031d8526233020,0x0000028746afaa2c}, {0x000839064291d4c7,0x00017e5ad9095131,0x000185681bf48f15,0x0004425ce57f597b,0x0000c3ac1b0b854d}}, + {{0x0007dc3c093c171c,0x000364f42b656558,0x0005996cbae7acb2,0x00091a9a338adb95,0x00008e656762051f}, {0x0001fba28b8d0b1c,0x0006f6c10a906671,0x000232a8015d7413,0x000d23b0cdd7eb3a,0x00009e2f0802191e}}, + {{0x0001db6f79588c00,0x0009b55768cca80d,0x00054438afa52fc6,0x000a4f0b4df1ae7f,0x0000cadd1a7f9b46}, {0x000a6b31803dd6fc,0x000495eaae35b40e,0x0002e4e16488e4fa,0x000c97df047d5538,0x00009b5b7e0ef6e0}}, + {{0x000d2d3957626499,0x000737aea3f66b1b,0x000c6f896a9604ee,0x000ad0a646ff276d,0x0000bf0e7f5b860b}, {0x000c8217cb44b924,0x00036ea9c1822d92,0x00054a5fda2f5ce6,0x000da690a2afb191,0x000082e474cd5801}}, + {{0x00072d0b611c24be,0x000480a8f351c199,0x000cf64211d468e6,0x0004910b7580697b,0x0000c9dd0ef68fbc}, {0x000d2bf956c2e32a,0x00043cddf94e5b59,0x000ee766573dc686,0x0006c45d5e2321bc,0x00007b4f8effe9a0}}, + {{0x00018dd7280f8553,0x00005baec688fba9,0x000400f42bbaac26,0x0006e473b3f00f33,0x0000d2dba2996f2e}, {0x0001a94985093755,0x000f7ea423ccb6f7,0x00007e6fb8f33031,0x00054bb09b8dd048,0x0000163cfe5acdb9}}, + {{0x0008f17cf41c6e88,0x0009837b925c03cc,0x000d2427cf1f03c2,0x0008e4439c19cc66,0x000023d24bafb6c1}, {0x0009013901f0b4f7,0x000188941c2e32ef,0x00028092e684360f,0x00032e9ebaff522c,0x0000891e4e3956c9}}, + {{0x0004319ac445e3d2,0x00076ea743815126,0x000e9c50a553432e,0x0008c7c6eeaa6967,0x00007ced28482e62}, {0x000d3757a4afa571,0x0003d484c1503f96,0x000bd9923de0a14c,0x000422264a24eb38,0x0000df18da0f5177}}, + {{0x0008f82d8d38a9b9,0x000057de1391174e,0x000c175dd2e97c60,0x00003535709850a1,0x000069041a0c2ae5}, {0x000533b76a2086b3,0x000ba7c2e8fecbfd,0x0009dfb67d6bba71,0x0005d982d58ee609,0x0000a8b342d364a8}}, + {{0x0007649522f9be32,0x000bbf1f49a83bc0,0x00054ec42690c075,0x000dc760e1aee838,0x0000a7dbf4437689}, {0x000fc0e3faf40783,0x000eaf11862cc004,0x000a1b7b3b2f02e9,0x000c80c10a5e0fa0,0x00000aca623b936e}}, + {{0x000bf0502f40d9ab,0x00077c318a4df83c,0x0009c26744681c46,0x00092de85756180e,0x0000e79d046c8470}, {0x000480a78bd01e0c,0x0003b2a51db9af1e,0x000afbab66dd359e,0x000198a2ce3821e3,0x00005cee5b6d7733}}, + {{0x00030d46ffd9fbbc,0x000876c610b7e08b,0x000e262cf6e5bc69,0x0004c13343cff29c,0x0000a2e4e3628b91}, {0x00064c016de36c5e,0x000c62e2b829011d,0x00085aaf8e0b10fd,0x000e969894298166,0x00007511709030ed}}, + {{0x000d8fc3b922bf8a,0x000fac29b133671e,0x0006e99c4e4d8c09,0x000eb9e7eb12393b,0x0000ff3974d2793b}, {0x00094052c18df9b0,0x0003910071390374,0x0007a0b95c5c3a29,0x00096b6a77234fe3,0x00002c29a21b661c}}, + {{0x000f1d6141ecf611,0x000e2bb22f53c3aa,0x000d513579195509,0x000d601959740422,0x0000b083822637be}, {0x0006e35e07289f09,0x000bddd86effcd7d,0x0000f9cfa1f94c48,0x00097d38bb1f82eb,0x0000ee0b7e6ab2eb}}, + {{0x000fe2e34d74e31b,0x000f8bf79ab65a52,0x000feeb8fa352c30,0x000304e7ff6c5aab,0x0000fbe8ff0a5c97}, {0x0001ce6a79046089,0x000a34fca249d608,0x000e5e2001f812f3,0x000ee80b24bc9ab9,0x00001022c67c8012}}, + {{0x0009c5d30a713a1e,0x000fa4ef0f93e83d,0x000fbf9284876e3e,0x0002a3e9777029c1,0x0000f7a6bb49ce7d}, {0x0007228dfa2a659d,0x00080877a48fb806,0x0005d0f3fd5cd339,0x000c2aeea4fd8f02,0x000067d2e35feae7}}, + {{0x000e7d7cc5f43941,0x000c3536e142184d,0x0004aa60ab5551b5,0x0001d51e89b212d3,0x00004a96feb05005}, {0x000ef740d12bb0b3,0x000030b9677e4e21,0x000f7731dc522f02,0x000d315b12e4672d,0x00009f80382ab326}}, + {{0x000630c39024a940,0x000897319452dfb8,0x000a3867caacb96a,0x000fcad68a3961ed,0x0000c58e2b077c4f}, {0x0005d634da919fac,0x000a315e22893d54,0x0008bab10ef79b69,0x000c3694bc3d3d80,0x00008ab300805f82}}, + {{0x00038b67c4a658ad,0x000870e72182c127,0x00098e44fb3c4763,0x00085e6b77be4687,0x0000c047df2e7a7f}, {0x000d4c55e59d92d3,0x0005b8e64d8d2439,0x000ca9b16cedca47,0x000dfe7724cd0d87,0x00005e4fd59d5540}}, + {{0x000ff18e4bcf6b1a,0x0004895018faf8c1,0x00063c949856d628,0x000408a33f665c32,0x00006a76dd741f21}, {0x0002334cc7b4f79f,0x000116720e4a17d3,0x000d9bed5a1d0312,0x0005d0bdb6661d81,0x00000d6fb0301db1}}, + {{0x0001ad51fb747d2c,0x00087033762b7fd1,0x000efaf5aab50f95,0x000bbe6a7e711bfb,0x00007393278ffef2}, {0x000a2440df6f9be3,0x000b41efd215e29f,0x0003d6fd99092757,0x00078bbe60e3114f,0x000038542d43acfb}}, + {{0x0003f0838961a0f4,0x000e586987ca44a2,0x000863cc61426ead,0x000b78f6e6ee2e4a,0x00008059420a28b8}, {0x0003ad87396e1dea,0x000798c5aad13030,0x0008f50665c8bdc4,0x000bbc9e40e11f5c,0x0000bd6e7692d246}}, + {{0x00040bb23330a011,0x0004b34eafa068aa,0x000e02c21d23f5ee,0x000d062bbee3155d,0x00008dd4397e1d8d}, {0x000939a122d7b44f,0x0009b33870d63ba1,0x0004fe3f8e6d3b40,0x000cbe9e620f701c,0x00006bba1a6c3a50}}, + {{0x000bde5cfc0aee0c,0x0006008c50bd4a78,0x00063c9b2847edc4,0x000acafaa2439cad,0x0000eb4a728d0fc2}, {0x000e40e26da033d0,0x000d03e02683a419,0x000ccf7256cc3889,0x0002081cd28559fd,0x0000fd7e0f18d13d}}, + {{0x000733b1f0df9d48,0x000322b5e4f301b9,0x000304fd48cc2c5f,0x000aa6c3053bfa3a,0x0000e87665c8a9f1}, {0x00029ecd73dc965f,0x00044e9023db087f,0x000ce28b415ace45,0x0008493370e3092b,0x00009723443a6b1e}}, + {{0x000662eb72d9f264,0x000eb0e47109beee,0x0003289d0b19396d,0x000e3245b1fa73e1,0x000036cf77e94e58}, {0x00033b3e990ef77c,0x000c5b11fc250ec8,0x000c332ce7373e3e,0x000855fe0eda870f,0x0000ed049714d7ea}}, + {{0x000f7857e977ca05,0x000a8fdd5d2bf85f,0x0005af461b66ee8d,0x00087ca5e3795090,0x000087b9090e66d4}, {0x0008a1b32ba0127e,0x0006341615ac6a19,0x0006ef2f2a7720e0,0x000b3cc23f349999,0x0000f5f64b5270bc}}, + {{0x000a96292b8c5599,0x000fd9740a0fa526,0x000bdc0a50c14aab,0x000ef37d41a9e3a6,0x00007d521072c48a}, {0x000bd303e7c253b2,0x000a27fdedc1cf16,0x0003aab2ecc834b1,0x0000ff5362c6e537,0x000064ed85ee5f59}}, + {{0x000d9c066d41870b,0x00009787ba097a46,0x000d44635a50c20b,0x0008db685e7e51e3,0x00003b3e080e1e2d}, {0x000e558a179e9d94,0x000934a76781bed1,0x00040864f2daa3f7,0x000cb50372baf23a,0x00006900c548fe75}}, + {{0x000171ef76765d05,0x000245c87502b95f,0x0007c99bd4ad726d,0x000fa7dec769da4d,0x0000e2ddd1a136cd}, {0x00017fca93e6dea4,0x000b53771123c221,0x00008a3a2e8a2583,0x0001127e2f6089fa,0x0000809d5edcf0e1}}, + {{0x0004aa3da7a095ed,0x000056f5aadd3b41,0x000e8b84a9049acf,0x0009b2a8d46a4d6b,0x000066b19648732b}, {0x000c2a0de6e95555,0x000865bd87705c2a,0x000d28921cf52d09,0x00036cc5a15fa60f,0x00006ccb81edb275}}, + {{0x0008ab89f4ccbb8c,0x00021b2217290f0d,0x000bed10ced5f44d,0x000b8a8314198800,0x000094348a4dd735}, {0x000e9c429ef8479b,0x0002b14c693f79f3,0x000143a144c13a4e,0x0005c382c9af568e,0x0000c51779a929ac}}, + {{0x00079922774856f7,0x0004fc1bf55f05e1,0x000f19e166e52fb0,0x000b263eda4225e4,0x00000f4728b1f5cc}, {0x00018d1b2947f224,0x0005e81d6fb95d21,0x000f0eabdc827ea1,0x000dcf4412328d8c,0x00005ee9fb243ef9}}, + {{0x0000421bb937d636,0x00056c4b37a68e70,0x000ed7b68df8ff2d,0x000f5944c0d5b25c,0x0000537c1f027308}, {0x0006a263b37f8e84,0x000b9eebc6ce25ce,0x00028d72c170e9a9,0x0004bc9d03795287,0x000045b0e55c5015}}, + {{0x0000e0683a7337bd,0x00042fecf2494b7d,0x000a2b71f1e3416d,0x00026c54840eff66,0x00000d9a50b837cc}, {0x00081506fe28ef7e,0x000543324c7fe219,0x0009b52633cc5ef1,0x000474420f345576,0x00002ade2f2810bf}}, + {{0x00020fa458d36710,0x000c2dc4847b28cd,0x0001941e31549722,0x000ccb6dd01e5559,0x0000e6fbcea27128}, {0x0001e6b3bef0262d,0x000bbf54e103ae1a,0x000c052ecfa8c472,0x0000e8a539c0a872,0x00007b2736a2a349}}, + {{0x000e1f1716843491,0x000e22e19b97143f,0x000980aff36b4722,0x000674cc05922790,0x000075c9c88ae13d}, {0x0005b226e6bfdb1f,0x0001cedb4b46a7de,0x0004a6e445ea5b7b,0x0007e5e5570191d3,0x0000cf60d2f924ff}}, + {{0x000392d677819e1a,0x000e0a5a29e8614a,0x000c85f3f7be74c7,0x0003370b50fece63,0x0000ca2e2a9e6cab}, {0x0000388122a6fe3d,0x0002b82a04a87f70,0x0007aed57db69f70,0x00086eca77935dcf,0x0000f16207d5d91c}}, + {{0x00049ab63ed9998a,0x0004077ddf962fca,0x000344072a3125c4,0x000b5565dd8a8624,0x0000023dda39ec3f}, {0x00041fc0c743032b,0x0000ae438639421b,0x0003c1b074f2120c,0x00071a4b7cae51c8,0x00002370cab7ac21}}, + {{0x000d9626cc820fb6,0x000c585a44bf2eb2,0x0006598f059feee5,0x00005134620fca5b,0x0000b922caede314}, {0x00045ad106bed4e9,0x00054fa1e9abff87,0x000c29487546e71f,0x000530035c1e481e,0x0000509216cdd936}}, + {{0x000306785c9a2dbc,0x0001abe8606fc7ca,0x0004c651dd6ae515,0x000f9549dbcae6e1,0x00009536e245bc32}, {0x00035a934521b032,0x000c678756ffa905,0x000edf03cf39c526,0x000034183172ec8a,0x00000a8075f0fe0c}}, + {{0x0009c62640264225,0x000fd4b9d076f22f,0x000ef29508dd1077,0x00000b444c742a3b,0x00005b9502ed8a2b}, {0x00014b486a098170,0x000c47bb4071a59e,0x000e0592fa39dd3a,0x000f5b55137f663b,0x00007fcafd4c9e63}}, + {{0x00052ee346eb2263,0x0005bc2facb79636,0x0001add267dfab08,0x000c43f73bf2b869,0x00000d7454122b46}, {0x000e73ef2c2d065e,0x000ff42eeac905e8,0x000209d22ff9b89f,0x000a2b4fcbd20597,0x0000b740ffbbe14e}}, + {{0x000f913a8aef5181,0x000beff4cfa2c71f,0x000b360487bfc74b,0x000af10716680cb6,0x000021b2cceaef79}, {0x000c836a01eb3d34,0x000a1f79077bbff3,0x00004bbcf50eb1c6,0x00061d648c32d6a0,0x00007a59316bd64f}}, + {{0x000147f931020167,0x000424d125766068,0x000bc6b9112c5f65,0x000a957fb071a7c9,0x0000c2da0c5de23e}, {0x00045b6d4a1dd5d1,0x00068122b13cf4fd,0x000f57a483e7ad9b,0x00088f242ca118e6,0x0000c2e94a716f82}}, + {{0x0008f075a97d231b,0x00069d83875899e6,0x0008727277c80de9,0x0000160ce0f5d005,0x0000e5d95c2c9c4d}, {0x0005cb19c2492eed,0x0000704d6fb3921d,0x000f988d342192dc,0x000e847c84dcd132,0x0000e26d620717b8}}, + {{0x000dcb6137c7408e,0x000556a266dac466,0x000bebf1b9a38d7b,0x0004e29ef5cb0683,0x00005cdcbbfefd01}, {0x000376df65965a08,0x00026bb3e95e30aa,0x000ee6f2060fe88c,0x00009fb3fd0b6166,0x0000827dcdbbf41f}}, + {{0x0009d240c56c6904,0x000d9db7641dbf8a,0x0006b662b40265da,0x000c9b122b05bf3a,0x000066d1dfef1478}, {0x00069621484469b5,0x0008b2df8f9faa61,0x000b8bf510db6054,0x000ce8737bca023c,0x0000effe34671371}}, + {{0x0005264ff112c32f,0x000c8b971fb2e8f6,0x00075080d8a9c736,0x00059ab4f194707b,0x0000c3f2c5b7839c}, {0x000777e5aeb49c20,0x000dda1addfe1d6c,0x00035affcf3db034,0x0001fdd76fee5a55,0x0000853ac70b9225}}, + {{0x000d5948b2a29d5d,0x00067de00ddb37e3,0x0002c328b28f1f45,0x00073ab083c1b5f4,0x00008ef1d90b493c}, {0x000626041dc61bde,0x000c47ee2f8a96fb,0x000946a5df74e8a9,0x000cfc9c605a802c,0x0000ed48d661839c}}, + {{0x000344f3a29467a3,0x0009951eba6d9894,0x000e5c2f2de81e94,0x0007b3aaea066ba5,0x0000fc8a61438c8c}, {0x000f88f06d0de9fa,0x00049b75ce0a7adf,0x000bc87d5bbc11cf,0x000de1ffbb7accfb,0x00001458e271badf}}, +}, +{/* digit=13 [{1,2,3,..,}]*([2^91]*G) */ + {{0x0003668e039c2560,0x000b7c17fd5d1cb4,0x000aa062b5f26fb8,0x000f04eee426af79,0x000072002d0d78fb}, {0x000a237e84fb7e3e,0x00002c82133d4c9c,0x0007e4181b401d8a,0x000151caa525926d,0x0000943083453dbb}}, + {{0x000da31be24319a2,0x000bc095a8e7f92d,0x00078218503f7d28,0x000dbc852fe84098,0x000076ddafe49c24}, {0x00054961d7a64eb7,0x00090f1dbe4280cd,0x00038d2d5e436088,0x000035bf81a87784,0x0000e4d52a8f5169}}, + {{0x000d5b11d59715df,0x0001e788983e19e3,0x000f1f248c7eaa76,0x000d82f5a730b0ab,0x0000bab8085eae3f}, {0x0000d2153765b2f5,0x0003aa127f3d65e5,0x0007b1b10bdd4e08,0x000fd34cf3c07439,0x00009f8090d01b59}}, + {{0x000fd9d615faa8f2,0x0000768554ed7b15,0x000a448828fa1eb4,0x000f325bb4447e7a,0x0000bb2d0d1229ff}, {0x0002a646caa6d2f2,0x000e02e7351b075e,0x000506c628eb879d,0x0004dc9cd5624e9a,0x000018eaef0c87e2}}, + {{0x000684744ddfa350,0x0005dab3f74737e5,0x000e96cf49ccfc5c,0x000b8f9ac1df3f1e,0x0000c0571a13b480}, {0x000b3d54b3a7b3cd,0x00088dcdbb992fbe,0x000415b3a35c0366,0x000d9982a0f5dcb2,0x000057759b51413e}}, + {{0x00047d83d30a2c5c,0x000e378a81dc1fe6,0x0001a4a9b0857f77,0x0003f451d5a33413,0x00000a94af9e9d39}, {0x0005c0bdaa6ec1a9,0x0002f8d2d7edbc3a,0x000614797ba9fe49,0x0005332b4335b4bb,0x000091c4d6902f83}}, + {{0x0008c28d2f01cb3a,0x0003375db0b15325,0x0007d0db493d6eaa,0x000492a19a2b0de8,0x00001e48f0478fe8}, {0x000faf6c508b23ac,0x000975d53549f747,0x000f9b838f137571,0x000cf4df5e58e2fc,0x00007186cef67fd3}}, + {{0x00068cee978a1d35,0x00032ab92d0477b8,0x000a5b862e3a68b3,0x00041d0102979487,0x0000f0606c38a61d}, {0x000be276f9326f11,0x0004b6fe3c2e2814,0x000df73512f521c1,0x000e4407464d7dac,0x00000f5f9d3877f7}}, + {{0x000616b269fb37d5,0x00042de62de5ce8e,0x000dd4153aaf7380,0x00049b5ba111754f,0x000015759ba8770b}, {0x000ebf8aa423a61a,0x00012d41fb928b09,0x0004c8936592245a,0x00010d7cba8ec19b,0x000087e91e44f367}}, + {{0x0004ce43d34a2e3f,0x000dc43b5d611fd8,0x0009186c7ee3759c,0x000259995bc78c61,0x000019c380abbb97}, {0x00021aade744b1f6,0x000000f8056bc0be,0x0003efe11a7d222b,0x00025314be6157b2,0x0000fab2b4f6cd68}}, + {{0x000ea5f4bf1d7255,0x0001ff6c950fad33,0x00077af069c1d8ee,0x0003e1044ee78aa3,0x00004f489bbe4a11}, {0x000d634992fb7e89,0x000a12a443478f11,0x000020e000169a7a,0x00020a8d49d4af95,0x00005945723708e1}}, + {{0x0003878a4d32282f,0x000c58020ae7b6e3,0x000a9b750e36e029,0x000818f05847fb37,0x0000876812da29e3}, {0x000138ed23a17f08,0x000070b3950e84ad,0x000d67ae06d7b448,0x000af65fa8aef42f,0x0000d3eea24d2333}}, + {{0x0002075b15d5acc9,0x000f2d815bc40d05,0x000a36cf2c6d9c79,0x00006ffdcafd88df,0x000008ccbe2c8aa9}, {0x00022c4ba35afce8,0x00007d6abf0b6387,0x000c335c15a3da8b,0x00099aadce252cc9,0x00004e7f0dee5aa7}}, + {{0x000a522b99a72cb0,0x0007876180162101,0x000f3653e06de6e6,0x00054a5ff8c7cde6,0x0000a821ab5c7a67}, {0x000a52b7cb0b5a2b,0x000c190487907e3f,0x000ce053aa7fb121,0x0009af6a72502006,0x0000490a31fb4e92}}, + {{0x000e47d62dd61ad3,0x000c3be01371e17b,0x000e3cbba781a961,0x0009b9e063bfd3da,0x00005647406af73c}, {0x000957b2736a129c,0x00022d13f256f50e,0x00019fcc5a631370,0x0008b5d436ee653a,0x0000f2bdb2aa7a4c}}, + {{0x0001244c5f95cd80,0x0000f4ab95f4b06b,0x000e5836dda8c8af,0x000ffc1bae59c2b9,0x00007d51e7e3acff}, {0x0005e6ac2ccbcda6,0x000f2528c3e001e1,0x0009fead43bc1923,0x000710e3324577a4,0x00001a1b8848aa7a}}, + {{0x0006e08700230ef6,0x00015d19adf8f9a8,0x0005ad8f20af585a,0x00014c1645f361f5,0x0000e676223a6c36}, {0x000257c4e774d3fe,0x0002cc102d1b23cb,0x000126aa582a3851,0x000ee3bbcddd887b,0x0000716998ccefd3}}, + {{0x000d571fb1675839,0x0008416c8f8a4239,0x000a27519dd011c7,0x000b69971c289569,0x0000ce0a3b862d64}, {0x0007289d5ec67385,0x000a3840ef6b8c97,0x000453419a3b49f9,0x0002d5308c14c99a,0x0000c00295b5cf0a}}, + {{0x00014fb1d4bcc766,0x0001e59a88f15244,0x0000d110fb07691d,0x0009f317f43263f7,0x00004ada5e117abf}, {0x000f94e5b544cf50,0x0005fd2713feafd0,0x0000c74f4b4a13a1,0x000e45b99b7d6e25,0x000097f2f7320324}}, + {{0x00037d8affa8208d,0x000b0c29aafc994b,0x0003a607fc3c31b0,0x0005d56da746517a,0x00008e1b8c2ce695}, {0x0001815c8418682f,0x000e8dc91d97716e,0x000996982541d487,0x00002d58a04669c6,0x000039cab1673a65}}, + {{0x00001a0e68db0554,0x00087a3338d50258,0x0002afa84f356975,0x0002d170c8c0aaee,0x0000f6985d43b656}, {0x0001f15132ed17ab,0x0004104365fe351f,0x000b1f066510ed0b,0x0003dbf3f98138e5,0x0000c9d95d6e2df0}}, + {{0x000cf6e19abd09e2,0x00012ff17edba83c,0x0004a06ce0b4097c,0x0008fd38a5c478d6,0x0000ddcc3fd744a5}, {0x000503d9e8153b8e,0x00019774179bd449,0x000d9120c3324fd0,0x0004dacf5d47c8db,0x0000b86016314fa9}}, + {{0x000bdd1972f07f4b,0x000e227bbceb5817,0x00011e5a6e5579e2,0x000047d6847a1f5f,0x000039ed2562c3cf}, {0x0006417a2f62e55f,0x000e2bcf82a2e107,0x000eb29f96b9ab38,0x000a455bb7c3197a,0x00006d17da407227}}, + {{0x000ddbd0f968c006,0x000ba00c880bab53,0x0009ad24da03da7e,0x0000d01b2396246a,0x000012c040161ec6}, {0x0000493109f5df10,0x000080af755070d1,0x000b9a9b3fbda403,0x00041830b93f95c6,0x0000c74ec71007d9}}, + {{0x00055646edb951fb,0x0007cf22c2829417,0x0008d11965f4a9d7,0x000e7c07870895b3,0x0000c593df45228c}, {0x0005bd46af3641a8,0x000abd9b3dccc78c,0x000e333047802200,0x0007998dc73f328b,0x000047ed87de1ffb}}, + {{0x000974e6d6711921,0x000ace16f60ff85c,0x000c387971e14100,0x000da435cb0d5a95,0x00008923bbaab022}, {0x000e899bbe7e86e1,0x000e116067bfef2b,0x000d5ce3e4a1510e,0x000b90c98c815484,0x0000af777f1092a2}}, + {{0x000b4004ef657241,0x00092c0ca6fe9fbc,0x0000029943e04a4c,0x000cabeb3e2cb555,0x0000f3a93c56363e}, {0x0000efe3923555b8,0x00089e1751ea1fe0,0x000b69357744bedd,0x00018abfb2db596a,0x0000dbd736675e66}}, + {{0x0003099df1ea40e2,0x000b37d61e6499d5,0x0006eb812b3f24a0,0x00040bb088a19859,0x00002c8361b77629}, {0x0001f97f9c0d95c1,0x00027e43cdae66f0,0x0002b15c38846117,0x0005cc01599a7fb7,0x000035a7536520d9}}, + {{0x000f0f75f7ae2f68,0x000d57fa6da22dcd,0x000d441b615fc6e1,0x0006b601ca829ad1,0x00004c10cf884a10}, {0x0006c95a73fbbd0f,0x000a5d8f6ee8a9b2,0x00025a0437f24e0c,0x000dfd58b459371e,0x00008a74fcaf36f3}}, + {{0x0006585c9f842969,0x000aabc278b01ed4,0x0004fcbd07fbaa8f,0x000a5f18e96cd46c,0x000040a120303b60}, {0x000e12055a4aec84,0x00049bd742f034aa,0x0008c68ab550e9a7,0x000ec6394456d722,0x000092f8868e4e25}}, + {{0x00015adb2d8f3984,0x000c1b84c9536829,0x000b917d6f13b51c,0x000ee18da90ab85b,0x0000b6155608ea3d}, {0x0004e850a52c1c82,0x000500b75fc4578b,0x0000bb3c6eab1a69,0x00009440c14f3caa,0x000020f448ad8216}}, + {{0x000ee31b0e63d348,0x000229e54fab4fe7,0x000e7b5a4f460057,0x00083140493334d5,0x0000589fb9286d54}, {0x000f5cc6583553ae,0x000a025649e5aa70,0x0000446520879094,0x000c4eec90450710,0x0000bb0696de2541}}, + {{0x0001fdeb9718710f,0x00080374a9f55a17,0x00039bdc138f1bed,0x0000cdd8c582e1ba,0x0000c457b0b808cc}, {0x0007fd4883841e20,0x0009387253819a18,0x000f843958ec25b3,0x0008972553ed0596,0x000095c76616f6c6}}, + {{0x000c85c4bdc56104,0x0003d79eb301917a,0x00078bdccb2885fe,0x0006991fc655478b,0x0000a9fc894259e4}, {0x000f0cd3ce299af5,0x00035df38b20bb7f,0x0008ddb8f195be9b,0x0001b91a929c87d3,0x00005fcc99d021a5}}, + {{0x0005b4c721a4593f,0x0004868eaac22b69,0x00089f914ed1e9a1,0x0009100b63d71c74,0x000098ba31d68118}, {0x00013739b128eb4e,0x000df448af4a8029,0x000418dd37801214,0x000241fbd2e22b55,0x0000ffb3c0eb3998}}, + {{0x000077cc7bf3827e,0x000a67f8238fdfa6,0x00064d554f2165bc,0x000b981e37cf6885,0x00005f825c4ea81f}, {0x0004f67ffed4d6f5,0x0008650a34b043cc,0x00041faf1bc60957,0x0003b63aa8fcf950,0x0000659f053b5177}}, + {{0x00082c36044d63bf,0x00081cdb0ca0e875,0x000b2bcf6a608940,0x000cfb9c993e0fbf,0x0000c64a71a35985}, {0x000da8083dbedba1,0x00021be67df715c4,0x0003defde804ae11,0x0000d3ca4c9658a2,0x00002002ddd6156e}}, + {{0x000ae895dd21b96d,0x000aff44624de68e,0x000c8897a8b99f28,0x00076d3ae008081e,0x0000d0a93043712f}, {0x00075224e233de42,0x00010b36a8a59623,0x0003993d9192445b,0x000a8f8bf9ff7402,0x00001f37bf44aad4}}, + {{0x0004349f8bd2bbd7,0x0008d868195d340a,0x000fdb6f11d902cd,0x000f1bcd27bbf1e5,0x0000a5ab088824f9}, {0x000ab06f7a09e03c,0x000671f2c123c466,0x0001b66572f8a197,0x000a7c1a355dc704,0x0000b840d134ece2}}, + {{0x000ad9f7db326750,0x000307a06f1bb600,0x0001f609478fea13,0x0007aa5d032269b3,0x00007753ef583ec3}, {0x0005aed9c0bea786,0x00095c3f45240348,0x0007f726d41bb398,0x00081fa940376169,0x0000109beb43f394}}, + {{0x00011ea3b6d1145a,0x0009085826548041,0x000e66562b6271ea,0x000d9bc19615e624,0x0000255494677b6a}, {0x000985e99bfe35f8,0x000ffb51cdf6d9c4,0x0008818329770ccb,0x0006d0fc32701392,0x0000777d45fa86b2}}, + {{0x000da22d847999db,0x000613525d329bbe,0x000a959a103aa33b,0x0002339b7b96d428,0x0000b3786e5e1e5d}, {0x000d3ce6961f247b,0x000e52f93d3faeb5,0x000a7ae4f20aa85a,0x000aa7ecd1ad3dd7,0x0000f6688f1281ad}}, + {{0x0000e867469ceadb,0x0003809fca48b1b4,0x00054bbc71904c52,0x000fa1eb7312af4b,0x0000e24bf90093af}, {0x0000790bd98764be,0x0006c26e299ebe5e,0x0008fe4c7a0f45f1,0x0003e56af0d2c26b,0x0000f170db26ae8a}}, + {{0x00061a029e0ccc17,0x000df0ad36ca0e8d,0x000173822cd53e87,0x000e54c28c6623c8,0x0000ee1767e1496b}, {0x0003259648945af4,0x000ce5c8009c89f1,0x00061ab8c9e45a5f,0x0003856f2febd91f,0x00003f6bc86ca275}}, + {{0x0002348f2142e791,0x0009b6e6238a8779,0x000839d9b17d8925,0x000bdc6536d2f64a,0x0000f428fce86a1f}, {0x00096010db06dfe5,0x000100a3a3cc1c10,0x00030f41bbfc16bc,0x000ccea9cbd9ec9b,0x0000b5da0d650138}}, + {{0x0000a4856ef96a74,0x0008582bf842ec1d,0x0003f700db47eb84,0x0001e026deae32ec,0x0000e43c42cea118}, {0x0002a31d1a4aa2a8,0x00084004f3cea1d7,0x000fe8a7a440d466,0x0003648d6a2d3b45,0x000020e52e37b128}}, + {{0x0005fcf25e51b09f,0x000e3023d15929ac,0x000ebf90e180cd2b,0x000180b9892171a1,0x000097c4c886c132}, {0x000c724c03dbb7ea,0x0004618cbbe49f1d,0x00067d153ae04376,0x000aeb5b0b2a3607,0x00008e2f4d6c49cb}}, + {{0x0005247d95ea1682,0x000a0970764a172a,0x0009781691758fad,0x00001b8c803a511d,0x000099cfe2efe77e}, {0x0001e17b0a98927c,0x000060014495652a,0x00075b56a2e26e1d,0x000f94bae0af9f71,0x00002e22a81964b9}}, + {{0x000f9fbd90a060ab,0x000b0af380854d0f,0x000776bcf496a27d,0x00009d82305401da,0x00008cdcef7225f2}, {0x0000f37436a0bbaa,0x0007d686004961ba,0x0003542cf263fa10,0x000537f2beb98eda,0x00002d4d14b75849}}, + {{0x0009d6812e9a1bcd,0x000b8f6e3268989b,0x000ace63861d9075,0x0003fe652c6aa999,0x0000e4e4a56620f4}, {0x000144ad673c0175,0x000e1f6e05eae5e4,0x000d1bd56667417a,0x000711113416aedc,0x0000eb36201d6693}}, + {{0x000c5043a1aa9145,0x000926dc59752d7b,0x000c8125c175a129,0x0008759900e0f23f,0x000069ef68c61198}, {0x000db6363a113b4a,0x0005f88357669012,0x000412deae3bd3f5,0x000e5c05c94a5276,0x0000d9e2a0a4a735}}, + {{0x000984c508b65e9e,0x00008df1a0d1405a,0x000ba80dabde4a1d,0x000d2d3a9433a1df,0x00009192ffa7440a}, {0x00096965099fe925,0x000bbb27a54a9f64,0x00090da6125ddb65,0x000819c78279ddc5,0x0000479a99a4bde6}}, + {{0x0004e05013fe1620,0x0002632d471bd0e8,0x0000e089fbe11dc9,0x000025df0b0c45fc,0x00004fb15b04c144}, {0x0005fc213c999279,0x000fade2eb35a61d,0x000dacbb4a033e9d,0x00049d68185d5cb8,0x0000a88e26616445}}, + {{0x000af6254671ff6c,0x000a9fa58603f717,0x0007773c04bd4241,0x0007e846fba40be6,0x00001d933d32a284}, {0x000acf3689e2c70c,0x000686bafd31f4f5,0x00073f6e592aab0e,0x000933b98d76aa34,0x0000c6641dc53141}}, + {{0x0007757d31e535ec,0x0005c7c2ee11cae2,0x000029ffa04cc43b,0x000a2bcd1f96752e,0x00002150673a4cc7}, {0x000c1e08d68b0139,0x000f5df298f33b03,0x000804464a9d6816,0x0001248bfbb529a2,0x00005a52faeedb22}}, + {{0x00021600e1cb64eb,0x0005ce7fc9fe55b3,0x000b0fb93004828f,0x00091f63394b821b,0x00006293a2de5f1a}, {0x000ef21d145d2d9a,0x00031b8fa603de35,0x000cf252dbe6225b,0x0008c160fc8f6b32,0x000028e52e6b17cf}}, + {{0x000c89b4c371e6da,0x000496ef0f289d1d,0x000292f81cebe067,0x000082bde05d09a4,0x00008303593d53e3}, {0x0005b0a7e37a9bb0,0x000e2b8faec3a171,0x000c9b1028c56f61,0x0001f05250743133,0x0000130cefca4443}}, + {{0x0009fa0bd865cfbc,0x0007fc5f1dd75603,0x000be72244b03e57,0x0008f580edf2e4ba,0x0000752496dfa198}, {0x0002d3b564beb6b0,0x000039a1c608d157,0x000f601260db1d11,0x000f33568d193416,0x00005ae9668f354a}}, + {{0x0006d37c92544f2b,0x0002f35837d519de,0x000514ececc08435,0x0000661bb6869c1a,0x0000633e728de1d1}, {0x000d69f936c581cb,0x000dc439c4f9f15d,0x000448a5b96e7b8c,0x000bbaae676f482e,0x00002ca7d5cad916}}, + {{0x0002541f50240258,0x000964c2d937d55a,0x00062189f47bc576,0x0006f8fd31b92a03,0x00003f3086f6f781}, {0x0006d94b587579ab,0x000780e76c5ff9f4,0x00000ffcfec2d22d,0x000c2b77d57461b0,0x0000b7e65f9e64ff}}, + {{0x00094776652a220f,0x0008e696c9817c7c,0x000effff361618f8,0x0001626021701d89,0x00002c8ff8f6c314}, {0x00013ad8efb4d3e2,0x000fae176d952da4,0x00067d51c937b5ad,0x000ac902867d342a,0x000062b9b1038eb3}}, + {{0x0004fe4c43ff28b8,0x0006ea664e7a4e31,0x000a565c27647662,0x000830be90e40bb7,0x0000588993b41acf}, {0x00001d68f938829a,0x000d9edd7d4cd7b5,0x000cd34c7996627e,0x000e8c97d44a6290,0x0000832749a93833}}, + {{0x000917d4bf503533,0x000b256765fb2e18,0x000d5ab6685dd726,0x0005fe24fe65d693,0x0000ddbacedc15c2}, {0x000d9a412f22e85f,0x00067d06f6bca799,0x000ca1637e2a2486,0x00030a04f1ee5643,0x0000da2828c51ece}}, + {{0x0001a3ea2dee7a63,0x000c434b2284758c,0x000aba6addcde2f3,0x0000a77ba445d24e,0x00005aaf668a6cee}, {0x00004a9e5aa049a6,0x000d31103e847e0b,0x000afecc3e74083a,0x000f7a5eb183ce40,0x0000b89dea04a043}}, +}, +{/* digit=14 [{1,2,3,..,}]*([2^98]*G) */ + {{0x0009d23fe67ba665,0x000043cf2f340e29,0x000fcf9139145076,0x000ddaa45b5ea997,0x0000be00843dbd7d}, {0x0003e05d53ff04d3,0x00032de91ef7358c,0x0009ec1a0bf7ccdc,0x0009977d684dbfb6,0x000067e7cf2b01fd}}, + {{0x000f3b7b0dc85957,0x00082f1d9f2e0ca1,0x000dd82a727de460,0x000447aaf3bf39ba,0x00009356a79d5862}, {0x0002345f5f9a0529,0x0008839a42f9c060,0x0004d40fc1a8b0f8,0x000368253eee4284,0x00003b0bfe5de5b6}}, + {{0x000dd02c024789cf,0x000951b57bfc5434,0x0003398df90dca9e,0x0004ba9aa898e224,0x0000607c835794a9}, {0x000be97c2c99b76f,0x000628c29302bb07,0x00003a88c6576ba6,0x0001054d79efcce7,0x0000259ced8a6a0d}}, + {{0x0003a5dc8de610b8,0x000ae7e223ce0f89,0x000ad6dc5e8c515f,0x00028ef774bfa64e,0x00009d20f96125c7}, {0x0000966098583ce0,0x000493f2a7d77a1e,0x000304d4aa2eedb9,0x00082d1b2820974c,0x0000842e3dac0772}}, + {{0x00072a33b9e2d7b5,0x0007748218ffe4d9,0x000149d917cc60b2,0x000ecc3fc7083884,0x0000c04346f7f461}, {0x000fdf2614650a98,0x000741f666acebe9,0x000babc835e35b53,0x00093de45613d188,0x00008cace3b45e1c}}, + {{0x000a3753de92e23f,0x00076fbbb6e3209c,0x000b1487eccb03cc,0x00041edcb90f03d7,0x0000a9c2a39b7109}, {0x00038236724ceedd,0x0007492d0323756c,0x0005e038e3a90225,0x000590e150e519ea,0x0000cba286697427}}, + {{0x000237f788907328,0x0008d3fcb4d9e549,0x0003480d6c443bef,0x0001861884d8a6eb,0x0000a35b6a1b048b}, {0x000471665e9a90a2,0x000d453006c0b4e4,0x000e9ae3b45bf380,0x000b716f3f820d4f,0x000044a35a0b79a3}}, + {{0x0000e9d74cd06ff8,0x000f2ca3eeaca101,0x00063aa2b9c17c7d,0x0004fef4c86cd380,0x0000595c4b3f3461}, {0x00000ca990f62ccd,0x0002fa0c3be5a3de,0x0008ce9f5d9bed21,0x000443a886078adf,0x0000db27ce42cd44}}, + {{0x0004a6658926ddd8,0x0009108015b8ed37,0x0001f7ab8138b2d4,0x000b7a086c6579de,0x000088b9aa143020}, {0x000034e3a96e3551,0x0008e30fbe9ad3ec,0x00021367aba65b0b,0x00046df64c8e50ff,0x0000f508ea41b04b}}, + {{0x0001a49747c866c7,0x000d9518a0629856,0x000dc3608bbb1e5f,0x000026b0ff4e8bec,0x0000f55cded90184}, {0x000ec95f38c85f0f,0x000e9bc3b8c38d73,0x00012b66f5b589fd,0x000e00ce95dd980f,0x00005bd1a09fe338}}, + {{0x0003ae55e915918c,0x0008c6f8a46b6516,0x000ebf99c6158d6d,0x0007eec466b538ee,0x0000a8761f77ca47}, {0x00049c29ebbc601e,0x000100c3ae2faf34,0x000e63752ef3b0f4,0x000a50ca6c577d5d,0x0000916660244682}}, + {{0x00097befc15aa1ee,0x0007d54b07455a30,0x0009a5f1240d1254,0x000ee57bad470651,0x0000d03f7188439d}, {0x000bb6c4a02c4997,0x000d5ffe71d20794,0x0003adcaff725083,0x00030fbcad75190f,0x0000f68ea1cb3729}}, + {{0x000c8c7b7ffd977f,0x000490761a22e747,0x0003ffb83ec104c3,0x000db69395ebaf5a,0x0000b3261f5d4b63}, {0x0004960d883e5448,0x00000cc2eeb85354,0x000d65f9913520d7,0x00095a88f6337bd3,0x00003997db2f81cf}}, + {{0x000f1060dbd2c016,0x000abf9ce934ce6f,0x0009939214f8eea6,0x000fc6f46f7c4b0e,0x0000236a324be753}, {0x0001f84a16022e9a,0x0007a3d1dbb265a4,0x0004cef9c0c18d87,0x000c73d3c556402d,0x0000042810910444}}, + {{0x000f15e9afdfb3cb,0x0002abdfb6df68e4,0x000823d9749a5614,0x000c29f9bc1bd45f,0x0000ceb59719a111}, {0x000455fb269bbc4a,0x000e49bc5d62366b,0x00018b0867cd85e1,0x000fb92743c41c4f,0x00004b4099135294}}, + {{0x000581d26ee83821,0x0005259d638e9c7c,0x00028ae3dcf17dcc,0x000047de8273abb7,0x0000d1129270821f}, {0x000847750491a746,0x00019de0dfb91149,0x000a435ab687fa76,0x000e3ecc2580227e,0x0000b8bdb94f1ce7}}, + {{0x0005dc93bf834aa0,0x00094f6c7e4b4c5b,0x00036bcad0437181,0x000f8dc284e00a37,0x0000d88111821ae8}, {0x0000f82f48c8e33c,0x0004e1bf40dbf9cf,0x0002733e5a11fd07,0x000bd71ceab0dedc,0x0000560a8b64e986}}, + {{0x0001fe23929d0979,0x000002f188f148dd,0x0006fcdac3885b29,0x00046b7f2ae613da,0x0000054303f4662a}, {0x0001e440738042ae,0x0006ddaf6449b687,0x000c9df1b98e6a97,0x0008f8f8bc0650d1,0x0000f3d645216e09}}, + {{0x000ae82b6d72d285,0x0001a5d8408003fb,0x0008efc1c77ca9db,0x000b4a3a112cffa5,0x000018d761d1564c}, {0x000740ef0d1b5ce8,0x000c69eb178569b5,0x000f53382717039c,0x00095bbfe29f9022,0x0000e54ba56ebc7c}}, + {{0x0006d8af7f91d0f2,0x0001882a57289c80,0x000d767543b61b0f,0x0004c62640032d94,0x000073eb5de67d83}, {0x000abf77b4e4d538,0x000f5e4017772988,0x0005071b3b7ce66b,0x000a981fba6b3271,0x00002413c252d3a1}}, + {{0x000c8c4e0e8ad934,0x000e1fab868d5b7f,0x0003946f3b5679ae,0x00050a71f9d2fa2b,0x000058897dc9685b}, {0x000c93089d0caf3f,0x000e88642e921e98,0x000bdaf1839564c5,0x0002e52b77729a0d,0x00009170723479e8}}, + {{0x0000317e4515fa57,0x00048b0c790f680c,0x0002e0765f85cff8,0x000b3257a82aab6d,0x0000446bca9a5c82}, {0x00007aa6d63184f9,0x0007962803a65de6,0x000be80357c1a46a,0x0001f84218313dae,0x00002113ffe573c5}}, + {{0x000e08312e7e46ca,0x0009b6126bd54b38,0x000c07e0469d0a37,0x0002675b3f324b73,0x00000c22f682fda7}, {0x00000514d2c7d8fb,0x00031be2cae58f2c,0x000f0f277bc45ced,0x000a9831c6cf07a8,0x0000c392312ceb99}}, + {{0x0007b7e3cc8ac855,0x00078d02753b7553,0x00037df2f8d725f5,0x00031dad05ff64b7,0x00005fe871346d25}, {0x00004a96ab6b01c8,0x0008fcd9372457ce,0x00086699b69a02a8,0x000231cf82ac35cf,0x0000242d3ae1cb4b}}, + {{0x0000f65d62105e5f,0x00093d29be61713d,0x000fbef09bb222bf,0x00082f02f9a79e6c,0x0000c24d8d4c5d67}, {0x0007085d41299673,0x000cac3c2a435db7,0x000d8d9a3db81c3c,0x0000266d655fc005,0x0000f5d057a84298}}, + {{0x000f56d88c546946,0x00053b09573e1157,0x000adffd1b26baba,0x000382ccab03b022,0x00000a412c93d69f}, {0x000e98b54b25039c,0x0002a87e714ded76,0x00000b594d4ee67d,0x00009ac77396487b,0x0000e41977689ef7}}, + {{0x0006f851c203a40e,0x00060afd8f9140f7,0x000578dd230d352d,0x000f3ccf196d3d95,0x0000a4bb3d857cc3}, {0x000bd03b98e782b8,0x000f8624920d42a5,0x00056fcc8ac958c3,0x000e5e3838134cfc,0x00006ec4cd009572}}, + {{0x00035269be47be0e,0x0007eb28fea169c4,0x0006c67e5323b7dd,0x000e461a5538ba3a,0x0000f921d70fd378}, {0x00061fc3c4b880ea,0x000df8940a67f929,0x000f0ff393f6f914,0x000f9c0990eb0afe,0x00006c2921090eef}}, + {{0x000416651b8d9a36,0x0008affb0db1ca80,0x00082e7ce42531bc,0x00074112ce4718aa,0x0000e1999143f574}, {0x000b13dd5d369463,0x0005c68f0194d5f1,0x00010d2308255dc6,0x000988ac9df4cd87,0x0000453c20f438c1}}, + {{0x0008dc089a6ef014,0x00005857df859af9,0x0001ad9244dbcc3f,0x000045f48056015c,0x00000448da610493}, {0x000926d4ee343e2f,0x000ca0e8a301f629,0x000815b3f6343f1b,0x0006faffc9349140,0x0000882a424ce8f6}}, + {{0x000d5f4e7db9f57e,0x00095c384c273a12,0x000c660b17dfba38,0x00021b9a904bfd6f,0x0000b6c5db40773b}, {0x000ee661cdfe0491,0x000e34540f29c350,0x000ec6aad9baac0c,0x000baaac57b6aba5,0x000067ce8c31a7c1}}, + {{0x00003a553fb2b560,0x00074e057f78b23a,0x000e490d96ce141e,0x000e75796525c389,0x0000bc95725a31a7}, {0x00067911220fd06a,0x000ba08b0bd61ec5,0x000ebeba9716e3a3,0x00066f91cd6bf7e8,0x00007326ca75ee6b}}, + {{0x000851ccd090c436,0x0003912c39883d9f,0x0004b7be4561e8f1,0x00037af0490b6a90,0x00001690ce164107}, {0x0009a370f009052d,0x00002026092e299e,0x000fcdc0f258758f,0x000cd1cfa255f3fd,0x0000bc9fb2090e1b}}, + {{0x000dd6e246518404,0x000465c59abc35f9,0x000ca4938dca45a8,0x000f28d03d396fec,0x0000532da0af97b3}, {0x0005ea51999a6bf9,0x0009ce6bf2eec413,0x0005be0933aa9505,0x00052e677cef063f,0x00007d1a0f939431}}, + {{0x000ebba2e1c21dd4,0x000bec6797c42cb0,0x00000101ff41b29f,0x000a8986e17321b3,0x0000422b0ea10d79}, {0x000901c92f1bfc4c,0x000f21e10ed949e4,0x0002926b806ab1f8,0x000ec1c4d35577db,0x0000a349d39f56e8}}, + {{0x0003d32343bf1a9e,0x000757d1a6b170b6,0x0006865b48fd3bd2,0x000fa12454879c31,0x0000e959ff7a458e}, {0x000dcf89706dc3f5,0x0001c64e4b2e0461,0x0008843c8737db0e,0x0006f5b92626802f,0x00004498bbcc745e}}, + {{0x00073faa29e24afb,0x0003c0aa87a13594,0x000573acefcc3c45,0x0009654d2c4bf500,0x000065b514ed8dd1}, {0x000e7cf2193e3934,0x0001b5444d97e46a,0x000ff38ed60e9a4e,0x000f02a7594e9600,0x00003d84d2f4a0e0}}, + {{0x000b141ee398a212,0x000ec3bcc5be8b6d,0x0003460eab88a56a,0x00019aea1aa52f37,0x00000da1a56360bb}, {0x000999d65bf0384d,0x00038d5a180efb54,0x000737b0471a14d2,0x000e91ec44db7b21,0x000084fcb18d1dd8}}, + {{0x000937bfa44b4794,0x00091c98fd4f80de,0x000f087275350549,0x000f52dedb12ab28,0x0000c58b582e5f3e}, {0x00036d88327f246f,0x00095d7df320bfb2,0x0006024f2c3a3bfa,0x000432fcd96c59b9,0x0000c293a546f4e0}}, + {{0x000352b5acf6e10e,0x000ccfe652c35341,0x000577a7fc50343f,0x00023c6af3792d18,0x00001a4c6188f168}, {0x000d0cd33425d0a3,0x000d6b7bc47f9b26,0x0006bb20b306399e,0x000054fa792f3370,0x00001219614c8111}}, + {{0x000c06487f5d28b0,0x0001962277fd864e,0x0006aed5f11392d9,0x0009d9b5aa7942bb,0x000080094dc47e79}, {0x000588c208ba19bd,0x000eb512f2844afa,0x000f5799ad3e7570,0x000491fbae64e602,0x0000eebe7f0214b9}}, + {{0x0000f98e5c298ff6,0x000dd678361f3030,0x000cb9a1617f561b,0x000490952ff31298,0x0000233c3bcb562d}, {0x00015a192e3a2cb7,0x0001763651197bfa,0x0008c53b1961bcfd,0x0004b9cbdd29bf2c,0x000039704dff2284}}, + {{0x000fb587e7b754b2,0x00018806c9b97dac,0x0005044522336079,0x000783c7eb88c923,0x0000983e996a52c1}, {0x000e529958d881dd,0x0002562c7b3cdd4a,0x0000b52d1026bae0,0x000cfa6a6f919396,0x00000980f9162696}}, + {{0x000428cd5f30851d,0x00065a4f66304c1f,0x0005d48a494dfed2,0x0000cd7df53772fc,0x0000d2d5a3063326}, {0x00015bdd44cc7a57,0x000d4d12533a5741,0x0003057c94ba6b20,0x00020dc0e93cb824,0x000094c486a84de3}}, + {{0x000d4cef21496e4d,0x00081c696331e925,0x0008d812ff951d19,0x0004aac810e2de3e,0x00000a4725a08929}, {0x000a2b50e3bab661,0x00059bad306f513b,0x00004c49e462caff,0x000b0bd2dc6d59af,0x0000aeb8750f0b84}}, + {{0x000f12f2f7d0ca2a,0x0008406acf2fc034,0x000facc2f6d2e812,0x000606e01f4f8321,0x00001170c04940ef}, {0x0001d4f7805a99ca,0x00062c26aba5fe0a,0x000531f40bde56a3,0x000107bb1629d035,0x0000c212c2c3afa6}}, + {{0x0006bf315697be5d,0x000b5c63c7c130a0,0x000cdadaf6f0545d,0x0005ba8d8cb8427c,0x000052e379c7c701}, {0x0006147f462c23e6,0x0007e6bc24b0c4f5,0x000856d4fd44a429,0x000cdf5c73d23ae2,0x00001cedd8c6832b}}, + {{0x000355699f241d77,0x0006901a349d6095,0x00089e491ee4adbd,0x0005459b35bf6aaa,0x0000f0076f4836f7}, {0x00018ba9264da3d1,0x000d52a7a28bd19a,0x00061c9716eb2d2c,0x000a5dbdba941f87,0x0000550518bb3be4}}, + {{0x000e2f057d0b70c4,0x000e8d133ba3d0e8,0x000416aeceea8612,0x000061414670f044,0x000024db6c370775}, {0x00039d116213fd1b,0x000468a3478fd960,0x0000c5021c61e7fa,0x0006dcf805bdcccb,0x0000dd6f3a8bcc61}}, + {{0x00096675d97f7e2b,0x0000ff0bf4b60600,0x00091627a31db0fc,0x000fb073680ed454,0x000099a3c672d741}, {0x0005f5536b1ff923,0x0007212b388de9bb,0x000fcf2632973857,0x000b47ab8a2ce750,0x000085346d49c4f7}}, + {{0x000c5ef31631f9e0,0x000103a57a29be86,0x00023f821bf91da2,0x000354c3b1f7967b,0x0000f7d00d2770db}, {0x0006c3bd8fe79da6,0x00007525c9968ffc,0x000ff632acc5e8c4,0x000527e640991dcf,0x00004d97e8cc7112}}, + {{0x000d97302f1cd1e1,0x000b0dd212a4c232,0x0009802f7ce87eac,0x000dbd5e4c8c73e6,0x00002ef02902fffd}, {0x000c74e1bcea6e23,0x00040cb92cbb941e,0x0008f9d05d0b5402,0x000aae509fb9d47e,0x0000bf1615a22992}}, + {{0x000f279f8a7a838e,0x000025615660ad40,0x0001f6fa111aea63,0x000ec8df52e6f1a0,0x0000f0469961dc2a}, {0x000bec9d8080711c,0x0009dfdedf76785d,0x00021c126e1aec60,0x0007322ce797b5fa,0x000066e898fc5e52}}, + {{0x00069c408811fdbf,0x00073fc7f08239bb,0x0004f41388bfe1ef,0x000f97d8e7a39317,0x0000ba8ad1ec58d1}, {0x000d0cebfd2fd5bb,0x0001bee60d61bc21,0x000d222530b839a8,0x0006b2facf7658af,0x0000526bed95ae39}}, + {{0x000bbc2385644641,0x00077c45bc73ccc1,0x000188a789e3ff94,0x0008f7bde9bca358,0x000038b8ee0e73bf}, {0x000234c4123c4899,0x00086a6432975c7e,0x000a15fa366e6936,0x0009267629eeee39,0x00005fab88239e2a}}, + {{0x0007007eafbb1e1a,0x0006475b7a93b249,0x000b68d78d75c9ce,0x0003959558352def,0x00002f26699c23f6}, {0x0001ecfe469b17ae,0x0009072d3ec2eb91,0x000cb113f6254577,0x00098caea47de782,0x0000be4b0872e1fa}}, + {{0x0005ed78cdfedb1f,0x00070e211a74ec2d,0x000d244c5a535c07,0x000a75a678109b11,0x000017c8bfcae299}, {0x000412efb11fbc42,0x000274ab3f65b651,0x000f78243ea0b548,0x0001d4b8dffd950c,0x0000e719e57ee036}}, + {{0x000f085304ddc5b8,0x000ccdaba2ea9007,0x0009d28a9095e8c6,0x000002da33cdb43f,0x00005b95cd962283}, {0x000c819b97447336,0x00089c7f5783bcd6,0x0009038e429c5f53,0x0000180c49b2fad5,0x00008349cc19bbe1}}, + {{0x0000c1d21830ee5a,0x000e49bfa297cc49,0x000de1a9436f9c4e,0x000cdbb8fd729448,0x0000adb13a8ee8f2}, {0x000aaa081313dbae,0x0007e2152dd8515e,0x00053dbf8c76bb46,0x000142557f8d75a6,0x00004d8c4d2914ac}}, + {{0x000e675b055cb405,0x000b477b5167bdb8,0x0002fb863898f8e7,0x00001f9cc65651b8,0x00006544814bd88f}, {0x0008e95263a75a91,0x000f0a22fcdab092,0x0003bd37ccfb6836,0x000664551d14db3f,0x0000d3837fbc6ad4}}, + {{0x000b538ff4f94ab1,0x00025d7fb8f27c5f,0x0005c52877243c71,0x000d1bdf13d60ca8,0x00008cfb7c75bb8d}, {0x000bfe6729082196,0x000b3d5144ab82f9,0x000f4b42f35c4592,0x000dc3f2734f379c,0x0000bac55e7ec60d}}, + {{0x000811e94dea0f66,0x0004818cc1a3b5cd,0x000e660f8259ecae,0x000ff20a0e836e15,0x0000c639ea66e02b}, {0x000b8cb7e1026fd1,0x000b532619428721,0x000f01da39e73b50,0x00057fa8c7097477,0x0000839e6a69268f}}, + {{0x00094155150b8057,0x000e892c7097571b,0x000084b951892389,0x00095c1d69c18e4a,0x0000014c5132e5b4}, {0x000db361b07523ca,0x000d8c1c64fa4780,0x0002c105a2f6219c,0x000360238b81b060,0x0000b4f4f20fdc8e}}, + {{0x000c982cf7d62d27,0x000cb3ba815020d3,0x000763f9e1f36e29,0x000006d8ae0bf092,0x0000a527e6b8d3a7}, {0x0009097581a85e38,0x000f5c158be5b4a8,0x0007d726e1f1a520,0x000862798db37d16,0x0000802786e9113e}}, +}, +{/* digit=15 [{1,2,3,..,}]*([2^105]*G) */ + {{0x000149e36f09ab05,0x0009fa10bb5befb2,0x000e2099803f163c,0x000bab8029704506,0x00006f0af006b5a3}, {0x000cfec70880e0de,0x000ede3d913f7af4,0x000ceb4bd7332a66,0x000f5452e6c84a7e,0x0000dc4a79b7c228}}, + {{0x0007dd0c55c44969,0x000695bbabd2c37c,0x000d7f363a6a9635,0x0001decb7e63f2ad,0x0000dce3782be73f}, {0x000a16ab2b91f71a,0x0002bba0163ce1e5,0x000e515ade448982,0x000ecf52759c32f6,0x00005e2f1f92615e}}, + {{0x0009be7abded5516,0x000868b744107451,0x00010d9a903d358b,0x0002b6ed00b10b0e,0x0000392b0b188da5}, {0x000a2980b75c904f,0x000db8f7f96c6744,0x0002cf932c305b0a,0x0006c9142e421d18,0x00006fc5d518e463}}, + {{0x00047c9d64cc78c4,0x000b5b6cb27b7958,0x0008022ab6c50621,0x000a1cc7099bf8df,0x00008f862ec004ed}, {0x00032ede1603c166,0x000efc9a9450d127,0x00029b4fc19a80e0,0x00051582257f54b4,0x00006d3b2c6a5460}}, + {{0x000f87e822e37bef,0x0003353bda4e6ca4,0x000190aeb73f237b,0x0002840747f3a241,0x000006fa370704cf}, {0x000bb6efc621c127,0x0003d0b80ec60a6b,0x000a556f35d624b6,0x0000a7db0724257b,0x0000fa0c354ae2d2}}, + {{0x000fa31e3229d41f,0x0001a4531bd4e921,0x000d38209a929c65,0x0007bc94156027a6,0x00003d69f745bdb9}, {0x000d19a168336319,0x000d73d51be38906,0x000511cd868a34c2,0x0002a83b59583b0e,0x00009ce6bfe8dc13}}, + {{0x000daaaffcdb4631,0x000a24a38b083fac,0x000a9078d658bbc1,0x0005de02a801f8f1,0x0000567bcf97ab85}, {0x00098e03572359bb,0x0004d659e68be084,0x00023807ccf0353e,0x0008a20b86e9c87d,0x0000c08728dd198e}}, + {{0x000b7bc453cadd69,0x00072c0bc1f88de2,0x000abd3af203900a,0x000ffb2cd86e47a6,0x000011cac131502e}, {0x0000242ec965469f,0x000139e0017e2d55,0x0009798850e9f769,0x0001ee733f078f65,0x0000b87d44a3cf75}}, + {{0x0000e4bfc25419a6,0x0002ebff3cfde179,0x000b6e83f3646720,0x000fd268db638625,0x0000cc69f23ccad6}, {0x000e45a6bc68bb94,0x0005e97f73340219,0x0005dc97ce43d79b,0x00049a3d44536846,0x0000b9eea326a0b9}}, + {{0x000c6ba6102d0212,0x00080f4461ea1b96,0x0009f19a8eaafac7,0x000875b4b85c41c4,0x000075c28e4ef538}, {0x0001a9ddd2e54e03,0x0004d605618b3545,0x00036cd246991adb,0x0002162b8b4bcd7b,0x000072a4f8c86f37}}, + {{0x000bd73a6a5da60f,0x000fec4c9ff0c890,0x000536e576f083d9,0x00024304e14d94f0,0x00009ee1edb9aec8}, {0x00041ec8bdcf8e7c,0x00004b041e265712,0x000fff040a5db827,0x000201da0b9a99e3,0x0000aaf21de3c271}}, + {{0x000b2e14f0dd2e8f,0x000e1a377ac7b4e2,0x0007a2198e77e7c4,0x000eb779202c3f0d,0x0000759b80018200}, {0x00026eddcfe314e5,0x000403d5cf99c875,0x0005138b6eb84c52,0x0003f461b52ace51,0x0000aa7ff8c73fca}}, + {{0x00013c3b9791a263,0x000a9dd58b16ff0b,0x000aad2de960022d,0x000619abd55c9257,0x0000baaaaa4230fe}, {0x00023460d881efdb,0x000f96325e2a9a4b,0x0005c18d4506416b,0x0007afe1381e7603,0x00003bb68bfa2781}}, + {{0x000b8bf5116f9379,0x00063126894315bf,0x00019a2c87c64a58,0x000462e1e25cc384,0x0000fd6b0c51335f}, {0x000ba3ce8ee0e0e2,0x0000098c21fa4bf0,0x00066bee06f6fba6,0x00054437d57b39ae,0x000092d513042672}}, + {{0x000105dbab093b30,0x000902839986f451,0x00074a89c012f59b,0x000e978a91580234,0x000048c919c2de03}, {0x000a2b591071cd58,0x0009834970a5c476,0x000b7994b791ed89,0x000ffd09bd9042e1,0x0000eaf517a21057}}, + {{0x000e2a2d551ee106,0x0008127e09a66066,0x00001148d87a8f1d,0x0003fda0d08bab2c,0x0000da8e4f1a24f3}, {0x00017f0cf9a4e718,0x000fbbf5cb19466d,0x00062ecc0ff50200,0x000ac45ccf97d8d0,0x00000c0d9b001d80}}, + {{0x00071d8033f28764,0x00063d5cc3dbe877,0x000c9bc1db0186ec,0x00060e18e8bb803b,0x0000d1395ccaf6ef}, {0x00062d6186244a03,0x0002e10a5b53a73c,0x0001b7eab918e5f2,0x000e51bd4878ca74,0x0000038d71b0be03}}, + {{0x00004b7a93c32461,0x000990b9b4cd8402,0x000d6421821ab606,0x000191e5fa6e2bb1,0x0000de6ad0f03d56}, {0x000aa88ff1929c78,0x000ae40e87b5570a,0x0005f0cccc6df4c6,0x000c015e8a74f2c6,0x0000b972fd666f6c}}, + {{0x00036b60b846531f,0x000520a5e4753fff,0x00045b6c5ba7e45e,0x000d9c94a1d10e41,0x00001f7f91b4e046}, {0x000a69244de90d77,0x000a8199c15e0317,0x000d73deb951a1d4,0x00024f21f78046c9,0x00004c828296ab82}}, + {{0x00078fce7560b907,0x000137e824ceaa67,0x00042eba8b4073e6,0x000f388f0d693cd6,0x0000ce2e57acdcce}, {0x000c7891df1ad467,0x0001998346fd89c2,0x0002fc17783a0692,0x000f1cbd715d72da,0x0000b6dd71df5b6c}}, + {{0x0006d0a73fa9cb0d,0x000d628bf5a9c60a,0x0002c8c82edd3992,0x000f4ff380ddd083,0x0000182d41172a0b}, {0x0007438d9a528dba,0x00092af539947d9d,0x00019987ce8b1a0e,0x00059d3dd6e5fe0e,0x0000cb8df03b90b0}}, + {{0x0003a328300129f8,0x0001e8c43bfd5370,0x000e540511f63766,0x0008c53cbd191300,0x000012fcc62fbf5a}, {0x0009d5f29fb85da6,0x000a094759e83f96,0x0000726b772f4e00,0x000208e26b6e5279,0x000017bbc879bdbb}}, + {{0x0008bb997aee3170,0x0006e81536a8511f,0x000c09b9b812a409,0x000a7a137dfe593a,0x0000682238fba8c9}, {0x000ead6aeccb4bdf,0x0009792ba6337072,0x000ff9d336a34e9a,0x000b61d82eaec26f,0x0000b7535130d4d2}}, + {{0x00045ff1d7aadabe,0x000ff5f6a67c1a04,0x000cfb26f65d3825,0x0001d58e62fb0891,0x0000f1e0fa63c7d9}, {0x000c7ba33db72cdc,0x00093a7c74b247e7,0x0000a503c017cbc0,0x000a417c931590f5,0x0000ac54f61216ba}}, + {{0x000d380b2369f0f0,0x000d23c761519b6c,0x00062a9c697d3a70,0x000f515f9dd6fc98,0x000044c4ab212312}, {0x000a0fd834a2ddc6,0x00026c7b826d035e,0x000fce49049e6b86,0x0006e9503d688362,0x00002f2497b137e3}}, + {{0x00005b6c64582934,0x0006a8d10af704b0,0x000e617b836bb527,0x0003d46cf2dc138e,0x000070d2d35f004b}, {0x0000832feeb1b77f,0x000895657f9c0679,0x000f600042bb75c3,0x00018ae70bd4edc0,0x0000e797ecd119b0}}, + {{0x000ec2a753aebcc9,0x000c3939eca59b5b,0x00095ad09daf9f3d,0x0002fc46bc6833d0,0x00008abdd526aa4d}, {0x0000a318d168be52,0x00000325a23cd984,0x0006ecfafcf7c10e,0x000f1885c02aa07e,0x0000462e7e6d5bfd}}, + {{0x0008a8ba0cc3f125,0x000d6c672a29ab2d,0x0006f2cd368dd485,0x000d8d2203975259,0x0000d3eea67f0cf3}, {0x0001a81e6602671c,0x000f54026c0c810a,0x000b50f858f144a3,0x0004a3fc753a6d76,0x00004dc21e9245cd}}, + {{0x0002dea521d03782,0x000de5011c6fc526,0x0004c19ea802b8e0,0x000aebfba19cbb0b,0x00001db64b60bf0a}, {0x0004ee970342f9d6,0x000dbbc44a141f39,0x000fd0baa93a10ae,0x000e6547eed31b3e,0x0000e7c824e7d154}}, + {{0x000fa819966e7eef,0x000715b7920dee23,0x00090aad464ec4aa,0x000ad43d44462d2d,0x000044dd196cf277}, {0x00071f1bb46b6a1a,0x00035d8850908d64,0x000a977b41e65d31,0x0001ee93a800f513,0x0000ca9d721a797e}}, + {{0x00085a0fcff6a17e,0x00023eca7cee9a5a,0x000504be39970a3f,0x000ee1db9f0d6bc9,0x00000c504bf8dd24}, {0x000d95677fcc2f46,0x000705bb5fc47e09,0x0009286aaef1a522,0x00028ab45d4fb18b,0x00006fd0c5dc6490}}, + {{0x0007ceb1bf4581ca,0x00060ca7b1669885,0x0009722ace635e18,0x000006878ddd2265,0x00000903c4cbdb68}, {0x000458948f214029,0x0004296abda2366e,0x00040319031b49c1,0x0003fda29c4b09e0,0x00007197ca4629f4}}, + {{0x000dd1e274983d81,0x000e45717c8f8073,0x00061f9d1da1a3bd,0x0001ceed3d4da203,0x0000332d0815c7de}, {0x000f7a3aa6d0e10a,0x0003554f1c4a9b7e,0x000d3556717db2e7,0x000e710f3dffae4c,0x0000aa2f407856f4}}, + {{0x000759e7ace3fc7a,0x000ea5a8d8c68966,0x000834e0e9594eac,0x000420ede3bd8b91,0x0000fe4ca53f48c0}, {0x000e856e6ee81c69,0x000adb891a3afdd7,0x000e638298f671be,0x000c9f67a58f2bfa,0x0000ab186fc2c11a}}, + {{0x000b36910b5be761,0x0009eb040bcd8d6e,0x00073de88046b773,0x000e03bcb4529fcb,0x0000df0fefcdf26b}, {0x00057a6bcfcd0279,0x00052b3165caad77,0x00099a4d9a8786c7,0x00004b59db1e347e,0x00009ee86e0406c5}}, + {{0x0002dddc15c9f0ab,0x00039295989e5b7c,0x000d08fdadf87a73,0x000701a9ece47c03,0x0000074d3de5d5fc}, {0x000790351a03776e,0x000a4a6080072040,0x0001531852bb1f77,0x00044645c58f4fe1,0x00006df62f7566e6}}, + {{0x000d1beed51275ad,0x00065f0f483fefb3,0x000c2bedf5de47dc,0x0008a0a932d98e97,0x00005c11927d219f}, {0x0001200a73a294ea,0x0009fdc201729d75,0x0006f506a5f88434,0x000d48328d9fd3a2,0x0000890cd323d1dc}}, + {{0x000aec170f4d3b44,0x0008cffc8d000aeb,0x000d57838fd1a136,0x00036179d9c24057,0x00005929d26a8bac}, {0x000d06025b15ca6a,0x0000ce4744465a2c,0x0001e51344b3c83e,0x000f407aac7578ee,0x0000418f5d7691e2}}, + {{0x000fc8a213ed68b6,0x000cf10a52246936,0x000f09b53860ae7e,0x00098d03660335de,0x000041b28982d79c}, {0x00038e101110f35d,0x0001948b193729bd,0x0009164f479c26f4,0x00027294dae5199d,0x000085a2310d265c}}, + {{0x000dd5d4b07e2b19,0x000afd9ea2217173,0x00005ab14d144c4c,0x0008f158b04ea411,0x00002dda5438e80d}, {0x0002fa8cf03dce66,0x0004ba22cffce998,0x000ad88c48b5ea96,0x00095c97f4ea7f3f,0x00002db773eca5ba}}, + {{0x00002fb93f245674,0x0009f15257cad20f,0x000cab987fd46c69,0x0002f4cac74cc78b,0x00006f31c019ceca}, {0x000db59888b219e5,0x000791fccd0240ae,0x0001f816ce50ecc3,0x00000bcbcd9dad91,0x000083cc1ecddb9b}}, + {{0x0002e66a483bf119,0x000521b2c169f3cd,0x000e9fa28fa08a6f,0x00011f6375e2454b,0x00009a7ffeceb6d0}, {0x000bddbc4ae62da7,0x000dc74aef5d6a3e,0x0004d05bc6cea00a,0x0002523b5fb98d9d,0x0000cba14244560f}}, + {{0x000cc21208490def,0x0002ccfb287949b2,0x0006fb16f1ca66ec,0x000db28f1166b71b,0x0000ff63e0955fe5}, {0x0005abe8b2610bed,0x000f69de3df4b834,0x0001c32b4b732ed7,0x00027c3e24ed5021,0x000010d8a6a548ff}}, + {{0x0004398ed4de248b,0x000d60488927c107,0x000673e13d7cedac,0x0000aef4aa6bf885,0x000046bae921daf3}, {0x0008472fcef7ad8e,0x0007f4b35e970708,0x000e299866115160,0x000c78dcfe8f26dd,0x0000b84c4c8b5a34}}, + {{0x000c55c164e1214b,0x000cf147bb03c1ee,0x000a96835891be86,0x000e9eefab4d100b,0x0000f01e9b955c1a}, {0x000e139b186ebc0e,0x0005a5b91bca6b4d,0x000d93854d5c74c2,0x000fbb7086a99cc2,0x0000ed62a7c87a9d}}, + {{0x000ed6f76b7618a3,0x0004d3b660628778,0x0005186dbbff750a,0x000d131cb7be22b6,0x000069dfbf0fc3a6}, {0x000b26c7191a321c,0x000880ed718ec7da,0x000cfd1839edac3f,0x000692fc142b36d0,0x00008af82f73c991}}, + {{0x000e4d897ce0b2a0,0x000fc3a55cdfb3d1,0x000b81afee6d7c87,0x0009d835846b9568,0x000018d12afd3c23}, {0x000620801206e150,0x0003a3b882c62b2c,0x0000162d5e0e4245,0x00062a854470a3a5,0x000081574787017a}}, + {{0x0003fb4820357c72,0x000e4f1458ad18bd,0x000b44aa1992039a,0x0002817a1df3c525,0x0000d7803580d3d5}, {0x0007e4dc77ad4d4a,0x000859df4fc458cf,0x00071205ed49a799,0x0004a9a465a8b51d,0x00000ee0ea704925}}, + {{0x000eecfab7bd7712,0x000315c262b94b5e,0x0009d61e76c87307,0x0000d2bc5bd6483c,0x000033d6d5452146}, {0x0005626fc195bcc5,0x0007b4d78b63d20c,0x000ec8ef32544595,0x000781903fcb3d17,0x00004b690d196b8f}}, + {{0x0002c8a212306466,0x0008a84f418c82fa,0x00030ba43f51aabb,0x000df79f4fbec11a,0x0000a5acf73d43c9}, {0x000b357d635b4d5d,0x000d1cd5c1da1da2,0x0001af0ddc3de68d,0x000f98d689080bd6,0x0000ea5938ba665b}}, + {{0x000d71afe637294c,0x0005e5a81cd80231,0x0008e63b501968aa,0x0007e851252d5004,0x0000446bc532ca00}, {0x00050a696d6134bb,0x0004ee09a05cef8c,0x000a3291a9361fbf,0x000a21417f85a6dc,0x0000178d549af251}}, + {{0x000374ba4df39156,0x000ecfd5d60887f6,0x000e35102566ce1b,0x0005e1e25cba4d7d,0x0000b745f8fb8c5d}, {0x0002af663122edfe,0x000c5b989a898840,0x000ba31563190f9e,0x0009a46ad3d387eb,0x0000f385adaa7c46}}, + {{0x00081de3f642c293,0x0008610ffb88b082,0x00029254620be088,0x000262c353dd4ad5,0x0000f1627deb377a}, {0x000a013eefcd6385,0x000624cc77c3a5fa,0x00048f55e8f3bf62,0x000b9de2618f65a3,0x0000787c0dcefefe}}, + {{0x0003aa2d9a23e449,0x0002be10690df167,0x000f9110888dfa99,0x0006488ced1b362b,0x0000193cecacaf48}, {0x000327d2d738fc52,0x0007775fee6cfb34,0x0004079a56697b03,0x0001ac0f485da0c0,0x0000cdf57354feaa}}, + {{0x0004420bd55659e1,0x000b3376090c7694,0x0003b591a7973e32,0x0000ca76bb4fe116,0x00000441aedd196f}, {0x0001f4a045ad915b,0x0006f4afacb13b43,0x000fdbbd86c11b43,0x000acc80b0c7db71,0x000064293209da65}}, + {{0x000e6e89c92b2350,0x000e6b3993a14baa,0x0003dd031a73bbd0,0x00081cd06d60ec69,0x00003cab91b71568}, {0x000862f1db3574b2,0x000544bb061ad615,0x000181e06485b018,0x00075707434988a0,0x0000cd61ad4e1c0c}}, + {{0x000ed5a2ff9f403a,0x000ac22390293eff,0x00017b70d8dc98d8,0x0000148206021e1f,0x0000fbec0cb5f510}, {0x000716480130dfa7,0x00051a02dcf59fed,0x000b10fc0306dc2b,0x000f50d8f06620fe,0x00008d1e1d5ca57c}}, + {{0x0008c5a192ef7106,0x000adb7431f9adef,0x000250c9e88afbd4,0x000c071e1f740764,0x0000e31318e158be}, {0x000c4b824f89b4e2,0x000828c36a2afd4f,0x00024baa765a5dd8,0x0006502f1eccfff0,0x00002a21cf2eba94}}, + {{0x0009dee42a554f77,0x0004902ec4ba95d2,0x000adb73d828983a,0x0008391112a1f78b,0x00009ea8898127c1}, {0x000a5a7d065fd831,0x0001a262a0bc8969,0x0002b5127f49af79,0x000dbcecdea8b6af,0x00000e913e1664c2}}, + {{0x0009d14bc21ef518,0x000acce572925123,0x000bbcc3be51c3ce,0x0001e5f95ff06847,0x00006b46e1f3d7e1}, {0x000ba2380041ef4b,0x0004b262342e0ea6,0x000d294d4d72fe50,0x0002c8dabc6dfd31,0x0000be017a2c278c}}, + {{0x000fa09b389328ad,0x0002001771b5b1fc,0x000b045bf322fbc6,0x000d0034c0d06360,0x0000b652edce0e52}, {0x000932c03ec6627f,0x000cd1ee50e350ef,0x00037a90dde1b3b2,0x000a956ab7bdc5dc,0x0000ea6721421e33}}, + {{0x000b5cb4f2999aa1,0x0006a8cbf0dd6482,0x0003405bb38476cc,0x0009ec83ebfacb17,0x00005cdafe7f5236}, {0x0005ba4d935b7db4,0x0003dc99a4cdd42d,0x000b5545b648b600,0x000faf385101bda3,0x0000bf2c38addd67}}, + {{0x000dc634442449cd,0x000963ad4fb8b1aa,0x000686d82e0e9921,0x00066b8c552313aa,0x0000ee635fb165d8}, {0x000224a18ee6e8a7,0x00067d42e02fbc3c,0x00074cd08eed748a,0x000adf770f930ad4,0x000074ea6ed6ff24}}, + {{0x000de1cf3480d4af,0x0006cc8acf1a03e2,0x000295a9cf0d8edc,0x00097d023e330368,0x0000add5f69b546a}, {0x00097ad96f8acb1a,0x0004c71bdae28955,0x000dd43f4bddd49d,0x00041976fcd52821,0x00005a4541306191}}, +}, +{/* digit=16 [{1,2,3,..,}]*([2^112]*G) */ + {{0x000b6bfc360e25a8,0x0004875a1a788ce9,0x0001732f4e642519,0x00057a1dc756a848,0x00003c0440fd432b}, {0x000b3f1d720281f6,0x000e7135e051c670,0x000052be72205910,0x000a397ed14b0edb,0x000097b3d282568e}}, + {{0x0009b9afb3ff9ed0,0x000b17f6515c2e59,0x0004da44928c2e0a,0x0004521cbee4fd47,0x000071279a44f364}, {0x000ff6601fbe8556,0x0007ffda51c497ab,0x000597c0b3ee394e,0x00034ab90385f667,0x0000e9fccc7027ee}}, + {{0x000de9314092ebb4,0x000d028e240c0b89,0x000d2f064f17256b,0x000b148f89a7f393,0x0000f57841f21ed3}, {0x0004405e708d8553,0x0001d3f1c3d04ee1,0x000d7eed5856aae7,0x00027098e5424fbd,0x0000333e4efa3ab4}}, + {{0x0007adedda492f81,0x000a682972053bc7,0x000931b4cc11a3ae,0x0004bb4e89a3e734,0x00007512e2eaf569}, {0x00049f3177bf8b6c,0x000948c7ff3e5dc3,0x00011145d232ea4b,0x0009c2dc4f9d16f5,0x0000cf109a3f3b37}}, + {{0x0007a88a1f258972,0x000f81b5d4d8e75e,0x000f3ed5c7ac6961,0x000dfbc3e1077308,0x000008a54ec2a892}, {0x0006e1978660710a,0x0006837df2c8be82,0x000704da50cf70a9,0x0003fca18a7340ed,0x00003eeb9a9a8ca3}}, + {{0x0006233169bca968,0x0003ada6aafb49d9,0x000c2fa9404d286d,0x000fb3409606eca0,0x0000869d0d5a3ff0}, {0x00037e5d0150d651,0x0003140c14c9a999,0x0008e2d49a92e250,0x000e2b556bf94510,0x000052a733ab2f59}}, + {{0x0003d588434a920a,0x0002c22103c5b432,0x0008dbf9ac0af8e9,0x000df1c67518ef93,0x0000184307423a9c}, {0x00094aa5447ab801,0x0005b75a3d61350a,0x000411a9ee5e5a32,0x0000c564ba507f68,0x00000581fc1694f7}}, + {{0x000857080eb24a90,0x000d488e0cfd60e2,0x00059cdb87bedfb4,0x0000a9721ebbd7c2,0x0000b0da855bc639}, {0x00004dbde314c709,0x000bdc32e8462b4d,0x00062fc9ecdbf1fb,0x000ab6a3833eabb1,0x0000939b48c40dd3}}, + {{0x00098a7cb0c9c8cb,0x000fd1c4375c5aaa,0x000f1c90f75105f2,0x0007bf1eee50575e,0x000031e0660723a1}, {0x000d275d4b6d45a8,0x000ce2ec89965364,0x00091c65bd363f3a,0x000b4735d2123943,0x000045647666bb41}}, + {{0x0008ecc37107c78e,0x000a770c2a6620d1,0x000d0d845acff3b6,0x000f9f42f975d99b,0x0000f0a0c479a178}, {0x000965176b6028e0,0x0004248612d41a41,0x00038af55c49ec67,0x000a365b6ac4f273,0x00006145e627bee5}}, + {{0x0005d07e75746b54,0x000d840c78be33e9,0x0002ff8e21c1e1f6,0x0000ad567833ef22,0x0000bedcf6af4918}, {0x000e9c13d7a4c8a9,0x000bdddfe7606b37,0x0003a5bbc2748887,0x000e7467055123aa,0x000054ff2260bbb8}}, + {{0x0008ab197c3dfb92,0x0000af168154c42b,0x000b5069255a549b,0x000bcb8d6748e7c1,0x0000775780f8fc5c}, {0x00080b8e1c9d7c80,0x00013fdbcd564eab,0x00069eace8c69dae,0x000b5a47e6b4fb99,0x000002f1085a705c}}, + {{0x000ca446d3fea556,0x0006948105684e23,0x00062f27db4ae9c8,0x00028be7bfb91b2a,0x00000deb4ca39bac}, {0x000d8947de6c34cb,0x00089494587da892,0x0003f8a5b4ee6825,0x000384e14ee14e1a,0x0000b113eaad8700}}, + {{0x00003b92115b4c9e,0x0007a908cad181ca,0x00018179a7c163d3,0x000080e912a118aa,0x000009ed751686e3}, {0x000e3fa26f516cac,0x0006ce732f91a676,0x0003da8b4753cacf,0x0008a991592aea83,0x0000626f4300cbea}}, + {{0x000c899a7b56eaf9,0x000ba4ef7316ef9d,0x000818a8600c0e52,0x000e46cb1e4e24fe,0x0000d31e20e5538b}, {0x000932d3ed689748,0x0007fc4e87c422eb,0x000de9aefe44bbc0,0x000344c121086e0d,0x0000e6b9cff934f4}}, + {{0x0002c1f711b0eb9e,0x0007980ab9549689,0x0000792dbb905f2c,0x000125cce26309a2,0x0000c8ac9b3e684e}, {0x000d8b6b40a24474,0x00015fe3fb24486a,0x0008e3b3f60121fc,0x0003941626fccf1a,0x0000e568622aad1f}}, + {{0x000ae0d196aa5a1c,0x00065041b5fbda7a,0x000b318b7e0df8c7,0x0006e8851465d926,0x000029b6e563ab13}, {0x000b48b711484633,0x000334454a762c2a,0x00003abe4b5738de,0x00058e24ccf9a05a,0x000077c02963427d}}, + {{0x000f0b92bb39c1fd,0x000c1608d8c573f5,0x000fbb80514373f2,0x000cfb0cbfd31400,0x0000f18fb2153afd}, {0x0007f4242b3523f1,0x000d77f650fb81a5,0x0000a7d7ce958532,0x0006be9a8dc8b68b,0x0000b75dfb725016}}, + {{0x000f7c92d7d1413a,0x0004f834f59790e4,0x00008c3e867e2d6b,0x000ec0afd4f4f9a8,0x0000f8237e175281}, {0x0005fdc84687cee2,0x000185b26c0925ab,0x000ea7650c5ded6b,0x00017f6e4a5aecc8,0x00003b73e5c34cc4}}, + {{0x00043183037bf522,0x000558c725d72bfb,0x000b3e5d7b61e6db,0x00088e6efd4060bb,0x0000e014701fbac4}, {0x000cf9a360aa449b,0x0004c9634d08ac75,0x000fb15efb70cfd0,0x00006bf591536dff,0x00002c37583807c1}}, + {{0x0003fdcf50225f97,0x000c40e12b03b429,0x000a8bf64c52e175,0x0003c68649c3bad0,0x000045a8ff05b8ae}, {0x000e5a358321bc3f,0x00061bc4df4830d7,0x000ea5058b1732be,0x0007442f217993e9,0x00007a71cdf2e4fd}}, + {{0x000533e894c5bbba,0x0008c9d8308286cc,0x00015c2446915c7d,0x000ce506aa2d0558,0x0000eeee592e9b22}, {0x0009d13781354864,0x000ed6b76f2f89e3,0x00036e8f53a275c1,0x0009f59b6bcc1be0,0x0000df69b219e470}}, + {{0x000b2502d0f39aaa,0x000a75a85947a188,0x000e0f4fa622118b,0x0005388ebf520ffd,0x000040e9f29e860e}, {0x00051eb22b57f0fd,0x0008ae80644a7b6a,0x000f095fe849a33b,0x00000180e5d16f1c,0x0000754b54fbc55f}}, + {{0x000bb22236f4a982,0x0008e66800bb5cfb,0x0009a77740b0c59e,0x0009482ac69a8f5a,0x0000b33f804f6bec}, {0x000929532e6c4662,0x000f2e599c73b372,0x0005c31cc68956d0,0x00084e847a249f15,0x00004d80f0e01ce2}}, + {{0x0001dfb988baf015,0x000d8bb16647cd82,0x0004cb960e6331a7,0x000ca4ceb8ad3309,0x000093cca39191bb}, {0x000ac8d265674566,0x0009604b6490384a,0x000b6c8f640fa030,0x00055f37834cd6da,0x00008a7318d9f91e}}, + {{0x000d04efc4d31579,0x00019bf3bdeaa00f,0x000a57172b56f8ab,0x000db2714f56484f,0x000048c5860d50ab}, {0x0005df00ebd4f086,0x000cae82938e342b,0x000df5dd03e5168c,0x0005161aedc1ceb0,0x0000bbbc6da45732}}, + {{0x000d486605daaa67,0x00074b9a6c9ec7bf,0x00024fb8946fd72b,0x000fbc74847fb1a1,0x00005959cbe12d8f}, {0x0009f65c8a588eea,0x0006180b499d4257,0x0009a5df1368c92e,0x0006044a4ef6cd99,0x0000a73bb80336fe}}, + {{0x000a70d6457d188b,0x000adb7a388bf347,0x000cd601386eda86,0x000fb207cdff060c,0x0000eb1b6c880053}, {0x000238799240a9f2,0x000f576189b20b02,0x00066193a1bbb384,0x000c7e6695e71e90,0x0000eb5009726ffa}}, + {{0x000a9c04a7d2caa5,0x000155aaa2900654,0x000476e8f6f3fb3d,0x0005e4335db041ff,0x000040b8b0c14229}, {0x0003ac905e214f55,0x000a06a0b638a5c7,0x0009e680b9a74075,0x000af9de4b1090ce,0x00007a5b479bb8d9}}, + {{0x00048e726bfe65cf,0x000b8290c3070dca,0x00069e72e097e391,0x0009ab783c462e66,0x0000505be1ef6255}, {0x0003ea1e3a3035a8,0x00061cd50da85fbe,0x0006407f26431ebf,0x0006b87d169d5c1f,0x0000d838a95e0fce}}, + {{0x000fa7f650006f01,0x000340c0fbb22a2b,0x000f9ad96dfd7dad,0x0005f982452495cc,0x000083bf494e9563}, {0x000df434a7bd989d,0x000bd543109502d5,0x00043f53e505385c,0x0004a90d98e67dfd,0x000061e1a6d200c3}}, + {{0x00046c64a8a3d626,0x000fc47743d25a4b,0x000f7e4338469c4c,0x000848cbb3a13d88,0x00002b23a1061be5}, {0x00096b4a63d1a4c4,0x000a3d183f3ee835,0x000afb01c454e7fe,0x000638243fce6117,0x0000e65e5e65c4c3}}, + {{0x0005ea1ef74c45bd,0x0005de32850641d8,0x000da7da92cfbfa6,0x000fbac8b078f53a,0x0000985fe38bc752}, {0x00068fe5a0148b4a,0x00068d78136deece,0x000b729ce6f9a55c,0x000bc3832dccc4d2,0x000027e0dfe30aaf}}, + {{0x000445212b4603ee,0x00008b706d149647,0x000a9d412a876c55,0x000c33ff145fcf69,0x00002ab75b80d479}, {0x0009a761a23ff976,0x0001fd359d1012df,0x000835f22c613899,0x0004d90e51c7aefa,0x00009a79cb220fcc}}, + {{0x000350d594cc7e13,0x00030350ab79f57f,0x000ff594a3079ca6,0x00062af26fb6149a,0x00005afec029d59a}, {0x00046f406ed2c6ed,0x0004ad939a579bee,0x000d1797e58da173,0x0006c974c504028f,0x00008853e7d2ccea}}, + {{0x000508da35fcd5fe,0x000b695ccaeb4065,0x000e1a9628965df8,0x000cc32f2da85012,0x0000e471b95a1cf1}, {0x0009bc80a08fb758,0x000501de3591cef1,0x000ef4f88704958f,0x000a5ea867f8b23a,0x0000d7493856a9f9}}, + {{0x00055378c9049f43,0x00034b92d8b61b38,0x000e2bd6b5be948f,0x00054da96f725db6,0x00007a222bcc58c4}, {0x0001abb8809bf618,0x000b9346f18de7c6,0x0007c0d1c46f07fb,0x00007a7b567a7ae8,0x00004a461c8fef3d}}, + {{0x000dce6d9278d982,0x00037dfc73e10a5a,0x0004321c324d9481,0x0007062f3528b605,0x0000e03fdde892ea}, {0x000061947b533c0b,0x000e7ca3c05510e6,0x000b62b8f1a8bc73,0x0004e2fe58d4b21b,0x00002045a74084a2}}, + {{0x000d5afbd76e195d,0x000c9938a8103ab3,0x000e3d5cb478dd1a,0x0001e39ffab3936e,0x0000fb693dbf2b36}, {0x000449651dbf1a78,0x000e88a2e762f969,0x0009bba9acab4b4e,0x000d9668c92f25d3,0x000050e61bd71464}}, + {{0x00030e3dc09508b4,0x000f34317655b7e8,0x000690355faf6d2c,0x000ed632606cebdf,0x00008bb92b410c3d}, {0x00054845c7cf8929,0x000945d5f01f65b7,0x000401d69f6cd7ac,0x00087832c30a5996,0x000012686517d921}}, + {{0x000f913b78c558ff,0x000ad8afdaa9380b,0x000f169d343c0bae,0x00020a477f61d554,0x00008da07e49e5ff}, {0x000c49da8a90ea83,0x000b53a29b21b676,0x000d8d27681c1ff2,0x000982083297ac2a,0x0000001122fea89f}}, + {{0x00094be6718e4488,0x0001fc3e6e13e1d7,0x00026b5ef246c148,0x000cdcd6646ef85d,0x00000f5091f08069}, {0x0002e2f724bdd38b,0x0003d471e8c7c599,0x0000ff2a902e915b,0x00019fe6ff320a0d,0x0000f886487f384d}}, + {{0x000e6a6c93f72d63,0x00029ad800eabbe1,0x000acf117d5f75d1,0x000355ca40a09fe7,0x00002c8cdd5a581a}, {0x00019927023c4994,0x0006f8ec39017422,0x0000e83f0a8afe5d,0x000eac1691afcba9,0x00001bcaa034b8f8}}, + {{0x0005ff98d2668d5c,0x0009bad81965e38b,0x000c6ce110715281,0x0004355bc8fc7c03,0x0000bbee6e34b650}, {0x0000fe80cdb9808a,0x00065e3ed31506b0,0x0000b501817d6e06,0x000aee8e9d38c64d,0x0000b8bfd57244dc}}, + {{0x0004a59513aed8bf,0x000c414bd07a4289,0x00042b582f77f3b6,0x000fe5cbdecb8f8e,0x000010e2fa9c2390}, {0x000502262a2f201c,0x000f90ee32b0efb9,0x000a789a84d59ea4,0x0002c4187f77286d,0x0000f98a2d0b7949}}, + {{0x0007239720943c28,0x0004b990b9d0f957,0x000f2884aba044cf,0x0000aedaa8e82395,0x000034de6ed8278a}, {0x000ee9a5f25bd125,0x000a1f7ab271c8e1,0x000d00b769259cea,0x00032a2e6d97a277,0x0000c0c6eeaf4378}}, + {{0x000c20f5606b81d7,0x00049d991ee55232,0x00032d951abd7b37,0x000363dd2bfe3586,0x00008f8514708ed9}, {0x00073f0f30c3282a,0x00000789230b9518,0x00098967f0da8ac8,0x000fb49ac7789c53,0x000069b8f805dda0}}, + {{0x00077176add85450,0x000672c49b66e5db,0x000421d771b71cb6,0x000fead856073968,0x00003840fe883e3a}, {0x000dad51ec699775,0x0008e07f6726b391,0x000ca160cae243fb,0x0005f4788ac87be8,0x0000174cced9ce35}}, + {{0x0005966e58ba37d6,0x00021817335d98a3,0x000fbc7bffdcc8da,0x000983fb75283083,0x00008e419d539c96}, {0x00039f402a403801,0x000f0fe977bc409a,0x0008edea688940fa,0x00047db640a94b8f,0x0000e22cd17fd115}}, + {{0x00068ce59ffc3e20,0x0005c1dee4e7e285,0x0007cb36360aa1b5,0x000bf2c67497c883,0x00006fb438a105a2}, {0x0007ec4500d8e20f,0x000c1670db103035,0x0003b7cfd1ad9095,0x000d278f589a05c7,0x0000544607e780d6}}, + {{0x00093b1a20ef103d,0x000f9ba6577b17ba,0x000a214a0ad85912,0x000da495c91cf66f,0x00007d49c6cf7990}, {0x000ec8d20bb569d7,0x00027effbc33ecd9,0x000ed0467bd4b250,0x00028bb056ca5a6b,0x0000916a1f7cb637}}, + {{0x000497d53a4f5668,0x000417b56810d4f9,0x00094a6218973466,0x000c6878e1da7404,0x00002546a940d011}, {0x000cb19c61ac1625,0x000c5bad0d3e1f3a,0x000b7ea4352f8fa9,0x000124c5356523b4,0x0000a16ad61fe608}}, + {{0x000b87f4faed184c,0x000c9029f45fb0bc,0x000c6b1fc5f236b1,0x000ce3142c76070b,0x0000644324f28aef}, {0x0001d595c5d84462,0x0006f3ae19798e19,0x000a59cc7c020807,0x0001ba8dcaee553b,0x00000ed6d6bc2cb8}}, + {{0x000ba19b6efcffc7,0x000827c0b87c0952,0x000aa30bc60f12d6,0x000f4ddee2c7c49c,0x000067238b807fbf}, {0x0003921501b5d92f,0x000ed2a37737ebc7,0x0001975433279e3d,0x000b4dafc12bc86d,0x0000a94dc6ffa40d}}, + {{0x000b41a530ccbbd2,0x0006ca8235257392,0x000d98d0c87c8214,0x00074c852f984c05,0x0000ae57d737ef69}, {0x000f7bf3042a6dd3,0x000fe9647a649377,0x000ca9767b1a007b,0x0002d5caa9079a0c,0x0000d81a25c268f7}}, + {{0x00067f8ff81578e1,0x00008045447d7520,0x00005aa6f7862215,0x000c77b0c22fcf05,0x0000030f0a67bed1}, {0x0009f151f0bd7390,0x0007e6debe8531f2,0x000677e982d7989c,0x000fd55c070e728e,0x0000a817bd306e81}}, + {{0x000d830b0f2ac952,0x000a8b20e64ec110,0x00029cd9a48d0995,0x000945ef3e00e177,0x0000a570c20fd556}, {0x000bcfd4e86214dc,0x00020f615498912d,0x00030d76e2d014ee,0x0006d095e2b1e635,0x00005135ae5bd0fd}}, + {{0x000273ad4f3049fd,0x0003170874770066,0x000c6e5fdbb8e989,0x000e6b5dba1ddb14,0x0000ba3788721f57}, {0x000e0a65a72f2cfd,0x000eabea56425aae,0x000872c371208bfb,0x00022425c6aa3b67,0x0000726e08413f93}}, + {{0x000daa5061f16587,0x00016f0cd2b31854,0x0003d50dec0016df,0x00001752a3f23e83,0x00003b681d32bbd3}, {0x0006dc43ac343c03,0x000d557164212f04,0x00017eed49c847e7,0x0009c6b1e13c9109,0x0000fc9eebd93a1b}}, + {{0x0006a727fe02299e,0x0001494f33190f81,0x00045c5be6335ccc,0x00066d5820179f47,0x0000647b783722f0}, {0x00049de02cafb8aa,0x000f5cc2ecccc22e,0x0000e8282299bc2f,0x000204fa8feea26e,0x0000627278c9e893}}, + {{0x00097337933e47b6,0x0002ce766402a7e1,0x000440d9ff4ff6b1,0x00080844d8be0a98,0x000058f5c2f98938}, {0x0005677c95b3b3ec,0x00087137b6ff90b7,0x000c47c29fa04426,0x0005b1477b039b43,0x0000ca95dd44a644}}, + {{0x0008ba42333fc4c2,0x0008d736a1b10b49,0x0001d4b2e274f8e6,0x0001994ca348fd5f,0x00004d3be78c8f10}, {0x000f858ca14f530f,0x00026b982e518535,0x000e1bf62a6e7f16,0x000417947c851236,0x00006a7c58ef3448}}, + {{0x0003703f9374ab69,0x0004de564145583f,0x000526d50864f919,0x000495a3bc3f4822,0x0000f323c80a262a}, {0x000a7ae3f046a9a9,0x000e4f8a039aaa97,0x000aa0ba670da183,0x000c2ccb68f71c52,0x0000be0fe51b1459}}, + {{0x0007eb6cbc613e57,0x0004d97ea61cc1e1,0x0007eded533131d5,0x00011abf69d39eaf,0x00003c2f4354e6af}, {0x0002493a4a375fa7,0x000c4833c5c24ca5,0x0006e71cf5f06787,0x000666114e091f3e,0x00006451f57fb746}}, +}, +{/* digit=17 [{1,2,3,..,}]*([2^119]*G) */ + {{0x000bdef694db7e04,0x000779fcddc680f9,0x000b8dce1edca878,0x000ba111981c3403,0x0000274dcf1b0e10}, {0x00043b86def6d1a2,0x0000cbdb1866f727,0x0000c6f58d25b167,0x0007f5a4491e8c05,0x0000be2b2aba7fbd}}, + {{0x0005c9dd111f8ec7,0x000d47c4e7603e0e,0x000392a51bcc33f8,0x00092d002f9a91bd,0x0000da4a7963132e}, {0x0000ae30bb1151be,0x000722e322511a0b,0x0004e9e7854febac,0x0000b80a3a508269,0x000058ffec2c4fe4}}, + {{0x00092fcd1e0cf9e1,0x000db0e7b2e8f855,0x00035584edea75f0,0x00092ab04215cfc1,0x000074fc7273f570}, {0x0007877eb930beae,0x000a7eb02a5ae727,0x000241b9b504cacc,0x00095419fe08f7f5,0x00007fb62f56d5ca}}, + {{0x000349d29c4120ba,0x000f20d0d915fbb8,0x00010ba519f94391,0x00091124074fa754,0x000066adbf6b50a5}, {0x000543c34bfca38e,0x000fd9e1ccfcc164,0x00020219ce0f2755,0x000979b9da0f53e8,0x00008234499a6b49}}, + {{0x0008b769d4c54230,0x0000b0521c49cfb8,0x0008700a19e56eb1,0x00058a418e0b5ebe,0x00000cbaad6f93cb}, {0x000fbded92a5e67d,0x000b4f347f11e923,0x000c0585bca4979a,0x0000e2b9162d856b,0x0000d6254b07c3c7}}, + {{0x000c513516e19e45,0x000775c4d5937b23,0x000e71ef656e2e84,0x0004c54f727d735c,0x0000b6304a7479a4}, {0x000a7363ab7e433f,0x00000e742f836638,0x0007fc19f1adea47,0x000697f054b8545b,0x0000935381baa1d0}}, + {{0x000ab2d799e9a748,0x000e2949f729546e,0x00090055a96239e0,0x0009b04a274c6b70,0x000035142c41020c}, {0x000667aa2e8807fe,0x00043aa3d39ea405,0x000fc72f529f2c08,0x000b3bec555d6442,0x0000856e0e8dbeac}}, + {{0x0004f9d918e49366,0x000652513982b550,0x0004d9cb965035ef,0x000508a553a0c26f,0x0000cb10d571ea85}, {0x00057b7a242da112,0x000d472b726848d9,0x00002a96b16a4d3d,0x00063b1d7e637c85,0x00007c7032b930d4}}, + {{0x0002b18e4136a147,0x000cf78e32bfbdc0,0x0009c3c03bacf969,0x000c4f598d89a3dd,0x0000b92420a93bec}, {0x0001f78c64d565c0,0x000010f28295d4b4,0x0003d051a9f969d0,0x000585ec7f7f76b1,0x00008945e1ea92da}}, + {{0x0006b7d5846426f7,0x0008b47d441d5536,0x0006fbf48e7d09e8,0x000d7ce10b404d73,0x0000fa003d15784b}, {0x000614f17fd95965,0x0000e5cb98db25f7,0x00083a76a49e0e0a,0x0000f7dc65957b2e,0x0000d40da8e1ddbe}}, + {{0x000c405050bad247,0x000d52aa4823f2b8,0x0008365a78918426,0x00068fbaeab3dda3,0x00002031717ec91b}, {0x000d69960a94120d,0x000d199eaeec8b00,0x00060aafd478a255,0x0007b2ef656a5f6f,0x0000fd7cb762dee7}}, + {{0x0008bb4a595939dd,0x00085874021737f6,0x000ad76120355647,0x00095ebe740e7c84,0x000089bc84460446}, {0x000da5d85a9184d6,0x000b3fc0b074f7f3,0x0008a888e562563b,0x000e7ba6d2e6aaf8,0x000012d8643761fb}}, + {{0x000dba7f64085e79,0x000399aa8511465e,0x000a2d188b230f30,0x000648c3388426cd,0x00000885735db666}, {0x000ff9a652f54f6d,0x00038fae2bf06f02,0x000f5eee365c8229,0x000d6fa816ade062,0x0000cdbdf44ccc56}}, + {{0x000bba354530bb24,0x00078b0869ea9fb3,0x000431163bde3ef7,0x000a3549bc90460b,0x0000d03d7d324819}, {0x0004f9e43b6a782b,0x0006ec88a68633ae,0x000ffedd9216db30,0x00083fe1dd88e000,0x0000280da9fc2bd4}}, + {{0x000cb8a1635e7417,0x000a08be02a732a7,0x0007ae030fe14008,0x000ce8cfafb3341b,0x0000fd508e7cadd0}, {0x0003219d607ad51e,0x0009ad40964a72c8,0x000878da20f229c0,0x000b853be2c3361c,0x00000c96743cab2a}}, + {{0x00086913e538cd75,0x000c3e08ad53458f,0x0005d15ffa7001f6,0x0005dd02b8c6e6bf,0x000048234a451121}, {0x0009d2d3d5b40458,0x0005ca904190ff5a,0x000607f8bb0ffeeb,0x000729d5a3aca448,0x0000cbd665cb0a06}}, + {{0x00034e0425830687,0x000b83f6e68387f8,0x000c1224802da2ae,0x000efbfb763e5d05,0x0000230378fd5a8a}, {0x00080b571e8e5cae,0x000bd3b6252493bd,0x0009c552e53ab041,0x000e653b8605136c,0x000084d402db5524}}, + {{0x0003573f37f59371,0x000dc1e4fca5a37f,0x0008ab0fceb0f6c7,0x0006ac1965a554ac,0x00007fbf56c37467}, {0x0006bd9acf7d720d,0x0007402247662e2f,0x000d53bef41fc8f8,0x0007d0817a14b385,0x0000ae327a64d76a}}, + {{0x000a065c48182672,0x000b17c1bbc16ad0,0x000392a9233aa189,0x0005ea44970b5227,0x00001699a1c4d153}, {0x000779cc2d7a7fda,0x0008f9c83cf2cd20,0x000c0b8c7e318605,0x000e4cfb69440b72,0x000081497d81b9e0}}, + {{0x0005c891f5f82dce,0x00068361079e515d,0x000a353309a7f67d,0x000e1ac8da81e311,0x000044990c52b18b}, {0x000ed95af103e596,0x00072dac9261c7d5,0x00094b8d3ece8aba,0x000e835e82b09993,0x0000830f09a76adf}}, + {{0x00029b488172d014,0x00058aff9e02250a,0x000a6329a8b20bd6,0x00092078a7661ee8,0x0000520304e13fce}, {0x000da1f2b47f7ef2,0x00083bffc540ae45,0x00064f874e07f528,0x000f38d799700934,0x0000244c2cdc6fa1}}, + {{0x0001ac194d7d9b17,0x0002582e7f1743c4,0x000da0fca5bafdd8,0x0007ad6f0614c15f,0x00004b043a818ae3}, {0x000afa19e71734c5,0x000e4c450f2e3ba6,0x000e242b115d5437,0x0003c1fa5883fe67,0x0000143bdc27c195}}, + {{0x0008b53fc5e89202,0x00087a9cee08542b,0x000486e08363bf9a,0x0000d1e2375f10c3,0x0000037543bac5e7}, {0x000bccc625640b46,0x000e2bc62c3b7109,0x0003f26eacbc1051,0x00042498455fed80,0x0000badceac4b372}}, + {{0x000ce7c6b53f5f95,0x0004cb176d99a2a9,0x0005c081b6424659,0x000ee661298d36b9,0x00003505bb86d9a9}, {0x0009e61f2ba70b0d,0x0008bafad4533f6f,0x000eb4a6ad07e16c,0x000c8dcf1694bbe7,0x0000febceda0cb0b}}, + {{0x000dcdc53868c8b3,0x0002086107a692d3,0x0009b4e64174311a,0x000cb61109e07c68,0x00000e4587f5df3d}, {0x000ea310811b3b2d,0x000c3cce43ea841a,0x0009a78036144d41,0x00092f764c45812a,0x000003d37200e158}}, + {{0x000d7f2b1f3390b4,0x000c65b61272c676,0x000e127a99f7a1b8,0x0007bf0ebebfc9c2,0x0000602500c9dd99}, {0x000771c4711230f9,0x000b720f09c17f09,0x000e5e38b058eb37,0x000bc01b693d4bfe,0x0000289eb1fd653c}}, + {{0x00046abd51b9cf59,0x00020f0121afbecf,0x0000dc274d2aa9c0,0x000a3ba6aaf7d2e9,0x000009e4ea0d8b95}, {0x00004966f32dbdb8,0x00000b030b3ee6b7,0x000b617e2672188a,0x00009e6effe5b3cf,0x00007e947defc827}}, + {{0x0002b391770f5a7b,0x00005e44eb82a44d,0x00069712ae4d4d79,0x00020d92e69d1e3f,0x0000f11c4d75c6a8}, {0x000f3e542c4224c6,0x000be49d941cb5e7,0x00050e878d6b4e81,0x000c53fd72bd1654,0x0000a61e28b4e25a}}, + {{0x0002094e6f1cd952,0x000ced18673f3327,0x000fc14647512f30,0x00077b12f7a4ca5a,0x0000f0956568bbb9}, {0x00047caa82262004,0x000cdac07369586f,0x00013acbe02c868a,0x00054c0ef2b845c6,0x00003d7563e43860}}, + {{0x0009dc7ab952578a,0x000186e84d0b54da,0x000872042b5423df,0x0006df08b64eeb9b,0x0000c205782f990f}, {0x00096eb21f4c77af,0x0003bab273af4ff6,0x00036b3f11a79c3e,0x00027939bc922e94,0x0000f807ef9c6d9a}}, + {{0x000ea3d778f22a00,0x00085b5e746982ac,0x00018ee7dfb10b2e,0x000a2fc0b1698028,0x000011afff4c91c1}, {0x000d126ad124418d,0x0005172e295f95a6,0x000f4db7531c081a,0x00046166bb283af2,0x000011554104acef}}, + {{0x000a8f833f6746cb,0x00054ea990cac7f3,0x000ddb0a921e46f6,0x000554e15fd5c5ca,0x0000d41f01728614}, {0x0004434426ffb589,0x000584dbc204346f,0x000969b7f8055943,0x00039a63dd20fe5a,0x0000d59e9577899a}}, + {{0x000971c8ad4cf4b4,0x000feffb8fb8f1b0,0x000340ba40344885,0x000758b071ac3c65,0x000008d0596f27fd}, {0x0008ea498c364b09,0x0004751e8ab5e7c7,0x0005d9002a4aac4a,0x00045529e1d56048,0x0000acd518b18844}}, + {{0x000688fd06f56c0a,0x000d3f027972e4ca,0x0009a609da48af70,0x00070dc91f0f045e,0x00009dd82ce8e612}, {0x000ca63a0ef18d34,0x0004fd6ca3bd8903,0x000f47d039fb7ee3,0x000e8e67b4a09cab,0x0000cdada015c67d}}, + {{0x00037499355a244c,0x00058f2151a95200,0x000b4efcbe77fd2b,0x000e24a95d6cf666,0x00005a0cad09a2cf}, {0x000fe5cef8118650,0x00089ea5cc3d104e,0x000b58dbcf52813e,0x000b11855683dc40,0x0000338ecde175fc}}, + {{0x000563774921592b,0x000d09bb9d31f9a0,0x0009c5459b4f1261,0x000f52a51429b74e,0x0000e182e701ea71}, {0x000b07cdfc50573d,0x000992be8d44d3a3,0x000ab65d39ba1afd,0x000801cbcfd2cb52,0x0000f11d54879571}}, + {{0x00003ee02a2404ab,0x000371088a710994,0x00004ae71497406f,0x000361e947940950,0x0000db420795812c}, {0x000a30fd88284423,0x0007ccb5ed1c2b72,0x000a40015283add2,0x0005efc7c0e20066,0x0000e3be64138b29}}, + {{0x0007dc1e038a6759,0x0002fc5c6320ac12,0x0000d2c53729deff,0x000cd327df8fd4a9,0x0000b74b0ecf81e7}, {0x000a623dab407e5c,0x0004b6b340c65cb5,0x00028392ccdbd361,0x00082fe184415a7d,0x0000184c1d9a96f7}}, + {{0x0004f1981d3a80f5,0x000178e02432c320,0x00049e0c1fde0c84,0x000a7328203b3e81,0x0000904bdbb58053}, {0x0001dd1101b68053,0x000c19aa6d4930fc,0x00017408743c223b,0x0000086ed671417a,0x000011469a105997}}, + {{0x000b6845e43fc616,0x000580d3ab57b189,0x000181da8f328237,0x0002b7efa34b67b1,0x000021ed0b2f9ee5}, {0x0008de1ad9906763,0x000b26d540659b17,0x00038c201d51de67,0x000f5bfa2c27c475,0x00003856ec868a40}}, + {{0x000fc15be6cdcde0,0x00039f0c6f892522,0x0003e30a61e603f3,0x00053e7994edc310,0x000033a00db220c8}, {0x000a409f7bb7fd76,0x000de62d18f6d3cf,0x0007fe29570f8781,0x0009f35bd8298068,0x0000eef4c32b9566}}, + {{0x000303b2f7e85c33,0x000141988f9b86a9,0x00038acb55fce462,0x0002122b935bf6c1,0x00000ea7d6755661}, {0x000b5f4e51ab9a22,0x000a8e067c78ef1e,0x000ca9ca60587c98,0x00005793ce1b3c77,0x0000a553d4d74b5f}}, + {{0x00082364da29ec28,0x000339c57316c789,0x000d80d47dbdd5d1,0x000391457d6e6b2c,0x00000b460d07e9e7}, {0x0008cabf963c31e5,0x00037c4d32fd9864,0x000f7c68767f9f63,0x000ea6baf42a9dfd,0x00005f292a35b015}}, + {{0x00068b2cd21ab3de,0x0001e393d39289e4,0x000013af9e504f02,0x000acacb21e1d4a5,0x00003283f79a2c28}, {0x00035f6226bf99fc,0x00034e291e69f38b,0x0000c162de835427,0x000dbda1673a15b2,0x0000101dc76704fb}}, + {{0x000b4c2255bd6178,0x0002ec2a91548323,0x0006793876c96969,0x000e2346e6586062,0x0000e01db0d38c88}, {0x0002873893a55595,0x000af7a3e14933c4,0x000cf35f87630f04,0x00073265d80805dd,0x000082ca080c7dfe}}, + {{0x00056e10b1894a02,0x0001b81c68c02c71,0x000b115b59203400,0x0007f2cd225d00c8,0x000037f9c22a3b90}, {0x000f32f4470e2c05,0x000108be4e950ea2,0x000ae5463b725f7c,0x000c03bf1dcafab1,0x00009ed51876ba2f}}, + {{0x000f316d0115d4d7,0x000f63691599f6e0,0x0007f0a415180b12,0x000cbfa57e32c952,0x0000b0b081e18e0e}, {0x000aa8abf4f0dd06,0x0006ed2526966dba,0x000f864fe99b289c,0x0003ab19b7755edb,0x0000974e2b1d6cad}}, + {{0x000bee206ddd6572,0x00010ff3a96d35db,0x0006be758e7cbdd1,0x000f5d6838196807,0x0000d737e7228c91}, {0x000ab6286ec37766,0x000d345fa7a15f83,0x000ef093398aa649,0x0007b19477ec3772,0x00006f52b1e698c1}}, + {{0x00058fbd803738b6,0x000784e86aa49eec,0x000b5149291aaade,0x0009740b1ae617a5,0x000032721221bc45}, {0x00028f0862c51290,0x00093321a4a07e0e,0x00041c88f0a8f79a,0x000e3ae26d166450,0x0000571b80553233}}, + {{0x000ccdec95207114,0x000cfc8b84bfd1b0,0x000fef31455a9e4e,0x000f2b5426bd39a1,0x0000f5f638eaeb93}, {0x0001ed32bf9341bd,0x00007d42d5a9ba2a,0x0006dc7c5d63c132,0x00085102964a8931,0x000017596079a511}}, + {{0x000201ff9e6ed353,0x00053736925ad8a9,0x000581af7b7b5ee4,0x000050da83fbbc99,0x0000076bc4094eeb}, {0x000c98c02dec312a,0x0007838dcb785511,0x0009c08c9270de89,0x00006d8cf4cf9c53,0x000070cb65ed8d3b}}, + {{0x000c10ecfe57bbde,0x000555a0c2b5b12e,0x0001c67bd82c7b65,0x0002cbfdc7d5cd16,0x000032e89868e3a3}, {0x0009444d11a5529a,0x00018427fa1a7aba,0x000a1770ae964ed0,0x000fcc7528392d24,0x0000152ce2cb2c72}}, + {{0x00053a48ec07649b,0x000f959dd4537145,0x00064b11018b4c28,0x0007a23a32b7147b,0x0000871bfa5de6f0}, {0x00012e59e2e3c9b3,0x000514aa90f6b671,0x000539006fbf250e,0x00066fc77aedb8bd,0x0000b0cdf9b0172a}}, + {{0x0009feaf8c511875,0x000c241e4da7edf6,0x00011434505bb67e,0x0002b0f7df0f3208,0x00006facb080b979}, {0x00007e98f6229e49,0x0003c26fba0ff3e0,0x000339d7962d103f,0x000ec0bf33bef7b0,0x0000841357c459bf}}, + {{0x000bb59c34e67058,0x000affdaa84cfa8d,0x000108537c3c7180,0x000e5a795872fca4,0x0000750cc3c132a3}, {0x000c69db7275d7d1,0x000b1e59b2e9b61c,0x000cbb493ffa0168,0x0002d8ba032abc6e,0x0000d86dbd33c908}}, + {{0x0000b67e28ef5ba5,0x00097b18e169ae1e,0x0006bbd202c9a469,0x0001d1becd0e331e,0x000071b360eff5e8}, {0x000ea58101c1d454,0x000dd8880452cd9f,0x0008dd4466651788,0x0001d0699726351f,0x00004bed02323728}}, + {{0x0002b2d33da525d4,0x000dd3144fd8094b,0x000c1061df193678,0x0000f478ab5ba4f4,0x0000343b5fb1ccbe}, {0x0002371638127137,0x000d87611d93a870,0x00021e1d747bf6d2,0x00077cd6729b8cbd,0x0000484d4e14629e}}, + {{0x0006eea60dbac1fd,0x00040a06a2f7830e,0x000ca535b23d8c48,0x000a9ab96714b050,0x0000c8d3645bbd97}, {0x000f9fab12177b4c,0x0003934d5d9c106e,0x000ab360bf79bf46,0x00044e6537a349a6,0x00007c54254600c7}}, + {{0x000a047e5911a766,0x0008047f1ee7b3c7,0x00056ab4261ffa5c,0x000ac8b5aed36f8f,0x0000a0d41b103ff9}, {0x00069f5cc30d3577,0x0000fb72be9668f4,0x0003ad461be9adf8,0x00041aacd926fe90,0x0000e89e3903aca4}}, + {{0x0002de5facf69d43,0x00061775344cf0f8,0x000e36d04363b7e7,0x0009a53894f312b2,0x0000c6cb4fe41d1c}, {0x000c3394008e1f2c,0x0009649f326c85d9,0x0008c5e065e9a85e,0x000ba91c35c60a67,0x000008b94505f86f}}, + {{0x000c02c89f71f0f1,0x0001ef3da3c0de40,0x000125dedad8f3e3,0x0001832ea5096b42,0x00003879cbfb7379}, {0x00014a56b306a0b0,0x000667646c5e6f47,0x000726368359c2ea,0x00031efacf894307,0x00007a5893565ff4}}, + {{0x00061d168754ab0e,0x000c8f429a7624d6,0x0008ce769801fce1,0x000a2ae068a85fa5,0x0000dc35c553d5ec}, {0x000276fa3f660d1d,0x000de8fc7167ea31,0x0008db0aea0184eb,0x000e113f20f21a1d,0x000096d096026c35}}, + {{0x00002b5f8c2a25b9,0x0008759204b6edf4,0x000b4e34c1bb772b,0x000459c0cbeae219,0x00003109d80cfa08}, {0x000ccf78ef59fb59,0x0001f807096354f7,0x000f3ba9b3b438fe,0x000a920e28c65931,0x0000cc31b477ad9d}}, + {{0x0000ba9b733aa5fc,0x000b305af2353c2f,0x000ac82a5dece47c,0x00018a38e3f715a2,0x000097ba641e203f}, {0x000550409c110608,0x0004c6af512dc3af,0x000f2814656ea2c0,0x0004947ac28daff3,0x00007fab43b159ef}}, +}, +{/* digit=18 [{1,2,3,..,}]*([2^126]*G) */ + {{0x0001641d4c5105f3,0x000e3d7fbd650989,0x000e6bdb01ae80f8,0x0008606d67225fbe,0x0000b433b59afc4d}, {0x0006db693e856387,0x000273e9862f44e6,0x0005c32ecf7b5925,0x000f506b78515766,0x000002fefd81e362}}, + {{0x000475d0fefb0c3a,0x000aa6d7c35d3754,0x0003798a4d48fb56,0x0008e60070b63336,0x0000e89f3d32fdb9}, {0x00089c86363d14cb,0x0000b7abd27d970b,0x000d5a0218981752,0x000aedebf7d47444,0x00003083bb07ac72}}, + {{0x00041debe949a447,0x0007e46a4fa53897,0x000047bdc638e938,0x0007c9cfe6419ca0,0x0000047f6491aea5}, {0x0008a9041fbab170,0x0008576bdba254e4,0x0002afddcda8e0b2,0x0007bfe807eebcc7,0x00007d3336df4257}}, + {{0x000c244bfe209256,0x00032fdce86762a8,0x00038706391c19ac,0x0004f5fa96a5d5dd,0x00001d587d481d32}, {0x00073a2a37173eaf,0x000763778b65e876,0x000bab43e2384800,0x000fbd20f8441e05,0x0000a11fe133621e}}, + {{0x000772e81685d7b3,0x00018f34a976047b,0x0005f48ef23f27d8,0x0005c3927608e291,0x0000b0b43fad521d}, {0x000fb2663ca72840,0x00041d4db8377613,0x0003b526b7f5729b,0x0003d187b1489858,0x00000b732a6bbadd}}, + {{0x000f4262048e3968,0x0005b83d9de48e02,0x0001e85ad436b50b,0x0008d6778d348147,0x0000b01ea6b5005c}, {0x000afee97015c07d,0x00087e3ba2aed3c7,0x000d3a1d246cdf1a,0x000ff3aa42e50183,0x000054b52698541d}}, + {{0x000cf304e23e9bca,0x0005726e36243f24,0x0000b6d614387f81,0x00077b86a46a033b,0x0000f1bc8462b2d7}, {0x00001ba527de79c6,0x0003e261bbb625c4,0x0007b4bc70e1346d,0x00062eeb96c44b28,0x000058493c7b2545}}, + {{0x00049feb8a24a205,0x0001a52ca53f23f9,0x000fb485317ebfed,0x00005d4b691bbebc,0x0000617ff6bb278a}, {0x00034c5e3c99ebdb,0x000d6784156a241b,0x0005d67dffc64242,0x0000109206482f69,0x0000967ce0f9e27c}}, + {{0x000375121c80b5d3,0x000c731ecca065db,0x00038a07e2e7a563,0x000854b56ffc4e52,0x0000d6c296662ced}, {0x0007d1aaf70b8855,0x0008dd686459e99d,0x000c8ba5bafc3bad,0x000aa34c78bf460c,0x0000a43951968955}}, + {{0x00017a85fe4e3142,0x0000dcb8906ff8b5,0x000061b23e60234d,0x00059cdfe542acf2,0x000087e191f8b4cb}, {0x0007ddc09d877d88,0x000b946789412185,0x000e05ea41c23478,0x0004fe3bf0c056b6,0x00002da4b5430159}}, + {{0x0006791fadb86087,0x000d0b74cdf6f752,0x000b90a34049e832,0x00010c343581ccc2,0x00003639eb90360b}, {0x000331fe1e4a71b6,0x00032072f9194fba,0x0006790326ffd6b9,0x0002ce0e53271c65,0x0000720644551427}}, + {{0x00034a3b23358342,0x000a70ef6860c0f7,0x000e2bb0d9526205,0x0003faab8be71704,0x0000418871e22f38}, {0x00076814082c1576,0x000fc9c20073d717,0x00087e728cc914ac,0x0005fd9186c1ebe5,0x0000fdb3c22c1bcd}}, + {{0x00014a6f2f9f8e93,0x00031fec49d230d0,0x00005a8d9963ece2,0x00029a562025c596,0x0000987444549f89}, {0x000ff6512bf476a1,0x0007f9cf7d9101b6,0x000be56ca598a64d,0x000615c7ec774993,0x00000899785dbb33}}, + {{0x00092fd02eee3ad4,0x0004f0145270b8a0,0x00012b675a86b3d3,0x00040ef23d98c685,0x0000b8bc785a2ebb}, {0x0001f54413f9cdef,0x000e3bab56647d30,0x000bfec23a5e4fb4,0x00020c2d2b252d1c,0x0000cd576bcd1771}}, + {{0x0007d3e83731a34b,0x000e3d836e8e0442,0x00012ca7c2bb9028,0x00073a036acff8b6,0x000088fe5f083d9c}, {0x0006bc6edea4eb34,0x0003088eec77be2a,0x0007106e143b9313,0x000a32b41ff566b1,0x000069e9172a54ef}}, + {{0x0000f0441c23fa36,0x000061989a2eb448,0x000a29ca7b4712eb,0x00028bdccbba0f93,0x0000e205c1536194}, {0x0007957b36416860,0x000d45ac8b4e90db,0x0004e03500432691,0x00051707a759acf6,0x0000514d89c9c972}}, + {{0x000147fa8e67fc38,0x000b2b2085be1701,0x000284e579e2e0b8,0x00066455651824ac,0x000090d4325f4893}, {0x0005e6ec55e68a39,0x000ab339c85a8a7c,0x00022b655bf12e90,0x0006ffa1846b85f9,0x0000a54ce4d9bf4d}}, + {{0x000e83af1a142952,0x000c9285d4f9d7f4,0x000ffdaba916f955,0x000152c57bb0e099,0x00008a430350ab0d}, {0x000ffa2b8a9cef88,0x000e39ec051a0a36,0x00068e6725517407,0x0007fb1c796096ea,0x000053db5fc7b3c7}}, + {{0x0004ba9e864a51ae,0x00088e8a1b8b2147,0x000120a286c26769,0x0005da9c82362694,0x000061e9a496383a}, {0x00050039f84216d6,0x00074d43cd857dd7,0x00012c659ab020d0,0x0002ad3437ae48da,0x0000449c2ec46545}}, + {{0x0004c1c2cf9d7c1c,0x000a2e95e5abcc7c,0x000ae170c1320886,0x000661fb7b9056be,0x00008a5b2519bc0d}, {0x0001432c11d23031,0x00020f03769f4ed8,0x0005398287da6691,0x000d022ac7a5fd84,0x00004dada944bccd}}, + {{0x000c3217ef6b0d15,0x000a2c933f228b84,0x000440b8252a9477,0x000d5e0ef6728afd,0x0000c3bd859bce4b}, {0x00080f5f22c2d3e6,0x000057bb6cc5918b,0x00095a11c368d504,0x000a70566142a126,0x00000ac583b4b19e}}, + {{0x000bb980eab2437e,0x00047e2654c8317c,0x000d8307f8cc08c5,0x0009931e2d6520e6,0x00009f147f437428}, {0x0007d14d2fd6cf16,0x0003cd4fcbb05f9c,0x0007341f7a3ecd06,0x00015c4d83fef08e,0x000043f23a09a631}}, + {{0x00078abe65ab743f,0x000045edc89cd38a,0x0000df568bf7c75b,0x0006814dd8752e53,0x000085c4a77d308c}, {0x00055b2e68acf374,0x000d6b32af854c99,0x0005cf493a544df3,0x000feb0b8ec3f5a2,0x0000d8f27645a622}}, + {{0x000f7aaf0dcbc49e,0x000890bbb45b7bb4,0x0002ca2e57de551f,0x0006eeefd0f3e49f,0x0000ce58709ff5c7}, {0x0000edd167d79ae1,0x0002ea7d7ec13292,0x00030af91039df8a,0x000b59e46206c0bb,0x0000ff5e2f532676}}, + {{0x000a0396ea51d668,0x0005007d7a2611f4,0x0005a9b24506c144,0x00019de0da570575,0x0000fc8cc329f1a3}, {0x0002d4d9433d67d1,0x000f5a7dd2968364,0x0007bde077fa5cb8,0x0006fb476591db9b,0x00003173d2551971}}, + {{0x000599dd5b340fff,0x0006c0fe76c5ea30,0x0008f5adcfc6b529,0x00028c2c6968c8ab,0x0000723c7f6801c9}, {0x000c3219773d4025,0x0002cb51dd474203,0x0002be23cdf7c6aa,0x000e86ed49e37a55,0x00007febee85b5a6}}, + {{0x000bee47bd8e7397,0x00043e63bf75c5ec,0x000fb892379d4499,0x00012fa68bd00f38,0x00005d48ee540533}, {0x00077aadb5cdf33b,0x00058c696769554f,0x000fd674e3396e89,0x0003e47fdddbf2d3,0x0000bb8f6ef49d0e}}, + {{0x0000651cbae2f701,0x000583aaa8eb51b9,0x0001df499efc4bc0,0x0007a57ecd8689dd,0x0000aee99a832f36}, {0x00085b9ae8274c57,0x00050d30b39c95d4,0x000c1ef816c14d44,0x0002ed4afea90bbc,0x0000c5f317b1459a}}, + {{0x00010754ef442271,0x000ecc20f4960121,0x0009853cda17bed6,0x000ce6fcdfe42481,0x00003793299071e2}, {0x0003078dbbe307b4,0x000e36ee99363c1f,0x0003caa206dd1c20,0x00040de3ee4b5742,0x0000ac3793bcefb8}}, + {{0x00038ebed1f8ca0c,0x00078ebb25a29344,0x00069896f3e54665,0x00043d0415af0ec0,0x000013eddb15a5aa}, {0x000204fd49eb8f61,0x000cc74f16707a04,0x000fc0558d0d5bdf,0x000ade2697e28656,0x0000020737111ceb}}, + {{0x000e6900647a82b8,0x00040f40054f5f87,0x000853803908e0ed,0x00025229f633d479,0x0000ed13c9aca28b}, {0x000f6761f460f648,0x000ab6d063363e2e,0x000c4979b53930b9,0x000596b47073ac8f,0x00004380e0edecd5}}, + {{0x0002c6bc4fe3c395,0x00031c7bebdfe3b2,0x000693459ba4a815,0x000b11a23ab6b725,0x00003bc377064922}, {0x000c8ab5afc60db8,0x0004a0b9f2a34645,0x0000fc507aa02235,0x0002e6d2a2954cce,0x0000c2731bbfce1c}}, + {{0x00008ab18a0339d4,0x000cf735436cf396,0x000992b4fac7a658,0x000fd4722c2b07cd,0x0000e83daed340dc}, {0x000c7be2f39ea3e5,0x000f60a56d2e8a34,0x000dd8038ef0c005,0x00007512731f6a6e,0x0000721d7409e3cb}}, + {{0x0001511fbeeee1be,0x00030f1d0c051ea4,0x000c07d35d1ef5e7,0x00049262feefd173,0x0000530a00b6a329}, {0x000b7fef15ebfb03,0x000ca322491a5d55,0x0005b3237549de03,0x0002b6c7b5f60274,0x0000632a3a24ab6e}}, + {{0x000ba890ef59f78e,0x0002e9e52b9a0d3b,0x0006314470dfc644,0x0000b03dc7969972,0x0000f03391893be2}, {0x000735db13839481,0x0002b0dd7d7d0c92,0x0003ed068c1fc29a,0x000bdc5485b69740,0x00003bfaab3bac93}}, + {{0x000c6a90deeaf523,0x00021c641c15410d,0x000c504c4b003fb0,0x000a76e384978c5b,0x00007640487b64a6}, {0x0001bc6222a77da2,0x00073e47eb110599,0x0001b432c62260a5,0x0003e9a7af6613f2,0x00002f3acc9cb495}}, + {{0x00049228e41d155c,0x00077ac059ef5293,0x0008844114d02456,0x00078ef02017554d,0x0000e8055d0659a1}, {0x000d1aff62045490,0x000ec7066759cd77,0x00072c229a0a00a3,0x0006b0471071ef02,0x000009bcf6bd3c4b}}, + {{0x00038a822305177a,0x000ea1645bbf2a26,0x000a7a3c0d51d59d,0x0003ee081142fdc0,0x000017eca6dec706}, {0x00087ed60d9dcec0,0x000120ad24550bb8,0x0007102bad6d28e5,0x000408ebed6308a6,0x000042c31148bffa}}, + {{0x0009ac58aa68e305,0x000bc483513efd09,0x0002d8f0c7a6a3d7,0x000954afcc6b75ba,0x00004dacf966e78b}, {0x000696fa4a9af892,0x000fe6ac98ecf645,0x000a67a203a41193,0x00062621b8b3f622,0x0000d0b1e0fb9dec}}, + {{0x000919240be34e8c,0x0006d1907f3527c8,0x00056702bc7162b3,0x00069bd0188ec1a9,0x0000a132f7e9f937}, {0x00044f90e2025b4c,0x00084c62f14c3ece,0x000e3cc1167aaec6,0x00050ded74141822,0x0000f9b75c43ff9a}}, + {{0x0002b164d348272c,0x0009d959d56d02fa,0x000762916bd99d61,0x000c7ffc4f19db18,0x0000c7cce5109c1a}, {0x000ebaad846bd832,0x000c892028494d59,0x0001f4ca98775a9d,0x000f10e7ec4ae16e,0x00007eb5875da893}}, + {{0x0004d51662cc565f,0x000a1db4138d0028,0x00032a59482353a6,0x000c46e9c7aaaaaa,0x00005528b5f95669}, {0x00002312f23c5ff3,0x000a3affa3a1f322,0x0002ddda0e3e8147,0x000bd4cb423d5c20,0x0000d6414ac9b871}}, + {{0x00082e1a51a168a0,0x000148ae5448586f,0x000233eb8b712c67,0x000ca99a2e4bd176,0x0000188223a78811}, {0x0005e21f7c18de1f,0x0000c27bb286553c,0x00051e9297682e45,0x00034e4ed036b30e,0x0000487211cdc9cb}}, + {{0x00042770c24efc82,0x00049ef737a40d09,0x0004cdd280349fd0,0x0005214d1c9dd251,0x00009c135ff50da9}, {0x0004508f78b0b6f2,0x0002478c143cea6e,0x000e21e65176f5dd,0x0008c3e81484184b,0x00007f7525d07df3}}, + {{0x0000e09748ab1a42,0x00003efe44331fb7,0x000f75af29cba50a,0x000ea85846c7a615,0x0000a7c2c577ee73}, {0x00066a43f0a449a3,0x000b7d90fc3d42e5,0x00061d05745474c3,0x0000924447be3d8b,0x0000e9d1cf16a4ec}}, + {{0x000e453f380a6e68,0x00011b1437c21603,0x00029610a0b86e43,0x0007f6fa4173f2ef,0x0000fa729a8703d5}, {0x0006f6e6c9c217e3,0x0009619195243e18,0x0003d4fb1be1d307,0x000f7162a62a7015,0x00002ed3e35068c2}}, + {{0x000027f9eb1a8b70,0x0007c5b22fe8d785,0x000d6a191bc37eb7,0x000816466b34f0b9,0x000008a89af9a05f}, {0x00028fb7d42c10ab,0x000e99b3f6b819b0,0x000a0ade37fe8c92,0x000a7ba8907cc0a5,0x00003154f52059d1}}, + {{0x000efb6d9790ed61,0x000c96aa793b5066,0x0003e042ea77a0cb,0x00074b0a915f3c22,0x0000c5def0479c58}, {0x000007873b6c1da8,0x00027cd8557a0e83,0x00060f3b155cf85d,0x0000628f7c7c7604,0x00007052acbc6e58}}, + {{0x0002b80907eae66d,0x000f7d721c890921,0x00045ac1c3cb068d,0x000dbad87941aedd,0x0000e8d5c0dddaa0}, {0x0001fdce3502e6ed,0x00007d89a084da42,0x000c24bfbc894420,0x0000eea307ba5ef0,0x0000a212bebf0bde}}, + {{0x000a24bf82ce682c,0x000547f71fe4ea2d,0x000fad8de058d381,0x000faa75a024625f,0x0000d7b05dd6adce}, {0x000f8ed1d9f54ec8,0x000832d3b5cad442,0x0006b2ce28be3d61,0x0004c062220ed0e0,0x00002699a5f9b0da}}, + {{0x00006f571c0c3a75,0x0009bd34180c3ff1,0x000d7d3758f580f5,0x000674febb120e22,0x0000e5782cd39513}, {0x000580c99c82a70a,0x000e75ea8c4c2275,0x000415e70e8359fb,0x000013b3b48db87b,0x0000acf2240b00c6}}, + {{0x000ccf5e4652f1d2,0x000ab56157b29faa,0x00061ec50bd6fdd2,0x000d5284f4fb1f62,0x000044e55ad676bc}, {0x0009305047d320ba,0x0004c181263f881c,0x0008fb8ee1ca983d,0x000963954e9a4427,0x0000d2dbc0fd96e4}}, + {{0x0003aa29268b3de0,0x000ae6e0609a723f,0x000f442520d1ca29,0x0007ed794866aa6c,0x0000b59f3e301af8}, {0x000e5ff7f4a6c51d,0x000191dc2f7ee234,0x00094d81fa8768fd,0x000ce10afc73320a,0x00007f84282d6938}}, + {{0x0003c0e0546063ea,0x0001bd61abc6ae0b,0x0009ac4007fbadcb,0x00010c35d7a2c936,0x00005978d0a4e67d}, {0x000211e4f85eaaca,0x00015acac681290f,0x0008384cde61e2ad,0x00030f0e12522538,0x00007fb68ea6cfde}}, + {{0x000b9363daed4c29,0x000f9606f7897a59,0x000a6d90a80a9aa3,0x000885240c1ea5f6,0x000048364d3e14d5}, {0x000bc6070985182c,0x000d73310895062e,0x00029c2f5a6db5b0,0x00037da4a12175e3,0x00005f25bd350ea2}}, + {{0x000c5242d0a4c230,0x00046bb3cc527915,0x0009e2c92eb5d26e,0x000cf8369a9116c0,0x0000c527f92cf182}, {0x00019382aede0ac3,0x00083cc349399e59,0x000a34361b292220,0x000fe60c9d896299,0x0000c81836df1905}}, + {{0x000b57fa001ec5a8,0x000b20dc5dba4bfe,0x0004a1380e993f5b,0x000a03c788410972,0x0000a0369abb2fe9}, {0x0008d608c927db84,0x000f54655741ea06,0x000b6c7eabf5f37c,0x0009cb07d402a204,0x0000551c295aaf25}}, + {{0x00071e7ed77ee8ba,0x00005309d5c7698b,0x000e780cabddf7bd,0x000ef3c201c22c34,0x0000b04f7d8ec295}, {0x00072944313a8cee,0x000bb2ca4cfe1c94,0x000a7a97ae532e4a,0x000d5aa9738f80d0,0x0000c088c898580f}}, + {{0x0001ecc42ce9e515,0x000d625fdd2a612b,0x000e7f8398f9840f,0x00047fecda78c001,0x000046b3d3b3ce05}, {0x00019a980d309162,0x0007384c20c42717,0x000c786084549710,0x000a4c8f8f94785b,0x00008c7d484477e2}}, + {{0x000176788a2ffe41,0x000518e169a5fce0,0x000f9c93adc506a3,0x000e07fea108617a,0x0000ed2436113fa0}, {0x000aa92a3d694e78,0x000d6f50bc7496ea,0x000114db4c0f43b4,0x000fd44e6aa58c64,0x0000218e8eafc000}}, + {{0x0005dfb185f88448,0x000a9557abfbac81,0x000bfecdfcd7e90c,0x000c49a3d16655af,0x00000f3271f885ca}, {0x0009aa7d0e62f477,0x0000d60a48e57fc3,0x0008f101e88d519d,0x000815e9559ac4d2,0x0000981d9ea3a9ae}}, + {{0x000652c9ac382032,0x000f37657fe55c38,0x0001f541686eaf87,0x0007b5368fc472e2,0x0000afff39d07e59}, {0x000bb07256d4eab9,0x0001f285ab893adb,0x0001caefe2259869,0x000c8aa5f8112a04,0x00005df02e435064}}, + {{0x000356ec7004bf30,0x0004db83c7de4d63,0x00009a7b7230a08f,0x000d2dca27b27087,0x0000d1c4cc4cb9ab}, {0x000c66e7550fee88,0x00071cf7247e8a0b,0x000b5b7e7369cd4c,0x000f7af5562e8492,0x0000fed0da0d802a}}, + {{0x00091c2e48fb8898,0x0002fb8a9d066a70,0x00082a0e226882c1,0x00052d224986631b,0x000044ed736b5181}, {0x000476fd86e27c75,0x0009b4afefdc282f,0x00019e34da04edac,0x00078b3b256ebc61,0x00006a413e95787d}}, +}, +{/* digit=19 [{1,2,3,..,}]*([2^133]*G) */ + {{0x000061d5a74be506,0x00047ea16ff582ee,0x000bfc8a2e41781c,0x000e2d80b0c81e99,0x000024f4d696b547}, {0x000545dbdcc9ae4f,0x0005509b1e8e3a83,0x000c935392573dbb,0x000797582960c4a6,0x000001059ae4ae18}}, + {{0x0009f973112795f7,0x0007284e6ee1715c,0x000b66bcde824443,0x000bede5cb4858ec,0x0000c1367361baff}, {0x00015955dbec38e2,0x000c188ad1535466,0x000e0952f51c0782,0x000fa87ba4c53ac6,0x00007e6782a3b21d}}, + {{0x000903d4ed2dbc25,0x00082c3b2d83682f,0x0007e93350eba59c,0x0006d73e9dc84d9c,0x0000f9b21b05eb22}, {0x000d394af267bae5,0x00056e2e15aee33b,0x0008ec500aa86cc2,0x000657ff0bf67d6a,0x0000846aa4549630}}, + {{0x0009740e2c2bf152,0x000589e99704feb0,0x000fbc565627a220,0x000de8cc8d73d0c2,0x000023eed8fe20c8}, {0x0002583a8363b49a,0x000929c2b0a61ee3,0x000dbc85c1a0b6cb,0x0001aba9f7c3d290,0x00008dfbb97bef4c}}, + {{0x0004d4c65c7c2aba,0x000742c5ea84afb3,0x0003c4ab51d4610e,0x0005c3e93f6d1b97,0x00003cdd7ea345ba}, {0x0004983064417ee3,0x000c7d6bdf2b6051,0x000f726c31459b23,0x000549f3b2c3415c,0x0000a82963562d63}}, + {{0x000901fab192c181,0x000e6030164f294f,0x000246ba6ec5fcbf,0x000a0cd2e2fcb7e2,0x0000e7c88b3321a1}, {0x000dd93c92d88c5e,0x000d3106fb5972c7,0x000f60f1441c2148,0x000f30747dd4f5a0,0x0000d9b52b343960}}, + {{0x00049ebb0a5b358f,0x0001ae7e2ed66c83,0x000a462dbb154c5c,0x000b68dad5eccfed,0x00002d6dbe51de66}, {0x000edf38665e5b22,0x00035b7f5723426a,0x000cbb386488a851,0x000878f5cc43b38b,0x00007ad0af3f791d}}, + {{0x000236e846e364f6,0x000c7ea50ca0c16c,0x00026b86d7f33527,0x00070c6481077509,0x0000c2a36096598e}, {0x0005e52f024e9245,0x00044db4afcaa675,0x000831790e0fa07a,0x0000d5c5c3ce7d66,0x0000b4ef350f6cbb}}, + {{0x000afc4b6205969f,0x000206c7854f2c4a,0x000983b4842563f0,0x000754116aced51d,0x0000eb356d989949}, {0x0002c81d1a39bd77,0x000f76934ae98c2a,0x0007904da8f44340,0x000925a48cf91c44,0x0000340185f7f51a}}, + {{0x00000fb7409ab463,0x000650e289b22f8f,0x00088e5d1057e78e,0x0004e1d3e5022ca8,0x0000c87111acdede}, {0x0000e1c7809460b4,0x000231c9abc75b9b,0x000cc1dc9e751c85,0x000a084b944e28c7,0x0000f201ffa5d3cf}}, + {{0x000905c3e6721cea,0x000a30b3674c02fc,0x000810da4d52d70d,0x000d98bdc2e5ca18,0x0000984b273fc69d}, {0x000252784de5ca41,0x0002b852dec463b9,0x000e3de092f1c987,0x000c2f08b03593c2,0x00009d70b01a813d}}, + {{0x0006b2da6dc1d29b,0x00094871e1444280,0x000f49276d303000,0x00004af1feb333aa,0x00005583b9f770bc}, {0x000be7895695f204,0x000149d012b51db0,0x000f61643fc84181,0x0001282409f27205,0x00000d3417515883}}, + {{0x00096f5674198335,0x0002363b7b08d791,0x00056700c6059e25,0x000ec434da18171c,0x0000758ee57028d3}, {0x000771d013b0ea65,0x000b04c5e9b97da2,0x000305d80fddf524,0x00063f2df4faf824,0x00008f5c1bf8a977}}, + {{0x00037f17c696042c,0x000b8a2538dea5af,0x000a42600d4cba22,0x000888611cb9959e,0x0000d105f423b069}, {0x000cf19ddb81e743,0x00092157b8cab1e1,0x0009db885472f2d8,0x000130d86fb008ee,0x000065cd5703f26d}}, + {{0x00002bba2be70539,0x0005eab9a6d6284b,0x000f7a530dcbbf7c,0x0007c7b425559c20,0x000061f2dfaa8876}, {0x000943570dc80c4f,0x000300784120e2fd,0x000567122104d6b6,0x000d768f592bc153,0x00006bc1247e688a}}, + {{0x0004c050f15dde91,0x0007fd5f2b820521,0x000e82b62a47a76a,0x0005eeab254d3062,0x00001a05fe04ec95}, {0x000f46e9d529b36f,0x00009f9e3df67eaf,0x00031769855ab130,0x0007acd463e37199,0x0000d251439bcda4}}, + {{0x000354723d695ead,0x000d46e589b5ca9c,0x00087d08648ce626,0x0009479b5b64c7b1,0x000002e179582207}, {0x000e98f7198111d6,0x00057cf9c3cc8b58,0x0004089b090ca630,0x0000fef691fe72f3,0x00000941af25c7c8}}, + {{0x000c0a222eb51e58,0x00042a9cf09aa09b,0x000159f06c0bb724,0x00060db6a8077f80,0x0000b5c989f5ddc5}, {0x000f316512e1f433,0x00047d08ff6219d2,0x000d20b4e02eac55,0x0004e0d12ab84c07,0x00007d1e11606d4e}}, + {{0x0003e1aab7b19a8c,0x000e1ef8cd45b644,0x0005e03daf08d067,0x000915a3adf3e968,0x0000f15a10f0792b}, {0x000cce5b738a4250,0x00059636b2fdf44b,0x00050d605ebe131d,0x00049d9406884178,0x00009684eaab40d7}}, + {{0x000669c72ba075b6,0x00055a4690158c3c,0x0009f8ba889f78b5,0x000ed6f706aade3e,0x0000d8bd566132d7}, {0x000e63b805f08d67,0x000d53bcc1b525f4,0x00025d8477f48200,0x0000a7de801968b0,0x00004afac04f7cbe}}, + {{0x0002c2b7e63d6900,0x00000223cdb843ed,0x00084d3feefb6bbf,0x0005a44fec3cae28,0x000065ecce6d75e2}, {0x00094ce69f790713,0x000ed44b86666c22,0x000b69d8f0d9a8e5,0x000af72009f23817,0x0000c29f8fef5dfd}}, + {{0x000528febae68c4c,0x000170c5ba219067,0x000dd1aec5b38563,0x000c77940df1191f,0x0000f37825c8fba4}, {0x000f980beb11454b,0x0000b0c1b06677ef,0x00089a1c740a1a99,0x000be038018980f8,0x00009c52aea26c24}}, + {{0x000bcce45650ef47,0x00001aa29ac705fb,0x00004c470ae000f1,0x000c25184b71724f,0x0000cd4fde289bb5}, {0x000b22ae88408697,0x0008efbd06866477,0x00016dfbaa886885,0x000776823cc02e11,0x00006cd5640487d7}}, + {{0x0007598a9d82abfd,0x000b16c170f5e2a3,0x00066b0875f188cc,0x000ad9b168220050,0x0000a22c21397155}, {0x0005d3afbddb4799,0x0003dd715b99151e,0x00097cb2e4b606b8,0x000b65ba73b54bf9,0x0000a1bfe43cecd8}}, + {{0x00028092a67d48a1,0x000df31fa9e21c31,0x00043a34acd6a671,0x0007d3aec3312a0e,0x0000d93563965ef4}, {0x00024898fea73ea3,0x00047035afb25ea0,0x00065b54c8247b36,0x000148858300a652,0x0000286662fa22c7}}, + {{0x000d76bb4ec4c20e,0x00062f3fe3fdb77f,0x000d8c7e8f0a12fa,0x000aa81845bbf541,0x00004d969cb3ec10}, {0x00053b743e232a3f,0x000b47f8a45a4c00,0x000d81c8fdc7a3fa,0x000aff4c4261c520,0x0000d4b3454a00ea}}, + {{0x0008f86d36e30622,0x00078143ff0276d4,0x00076f42e626c527,0x000eac338174deaf,0x0000267aa868407c}, {0x000635172e572d59,0x00072a7330ebfad7,0x0008d8657ab861af,0x000a5210a1c8c741,0x000088821cbb0289}}, + {{0x0002522cccc18ad5,0x000da1a6e0277973,0x000c2354daadf3f8,0x000689a7382c9317,0x0000ce1680d2818b}, {0x000bbfcd9ecbee97,0x000bacae62ac359e,0x0001ac38a4330689,0x000ee8455ce5b4c5,0x0000921dfeb6e238}}, + {{0x000bef8271d1ca55,0x000798aabd183972,0x000a3e5e33e423bc,0x000d6607b09f3f44,0x0000da886aecb444}, {0x0006634a99643755,0x0003199cd0ff6820,0x000a515e9356a2fa,0x00079a5f0faa24db,0x000036e1f5d0321d}}, + {{0x000913a5c04e4ea9,0x000e46f11513d3b9,0x000fd1d94d549dcf,0x000c675e227bf579,0x0000f35afef443f2}, {0x0008d24f1314f53b,0x00081abcd822d263,0x000f48db062baf94,0x000bb1a542de294e,0x00003eb6a05ac5f6}}, + {{0x00010ae1208e16aa,0x000558363e2423c1,0x0004be00b1a4d15b,0x00090c9071684416,0x00008e2482596f46}, {0x00073a290b170cfe,0x000062f191f45487,0x00047aa97a1bef33,0x00014690f418d092,0x0000a06028f28be9}}, + {{0x00022f3dbfb894e2,0x0006ae274b18e131,0x00058aadfbe9b79f,0x00035165a49de5ca,0x0000495775831487}, {0x000ef61bb9390993,0x0009f6d13694111d,0x000fc253b1d6a974,0x00015e1474b4ced3,0x0000a1485e67c5db}}, + {{0x00067b4147c15b45,0x000b2bc61301e796,0x000094381e34f553,0x000a20b32b80f817,0x00005d8bafdc23ea}, {0x0007995f1c0e74e9,0x000e5bba289c5a98,0x0004c82515a9b292,0x000b09b13cd4b2eb,0x00008b5d2446162d}}, + {{0x000bf66683425204,0x00097aa862d1bb47,0x00006abcd08d6894,0x00000dd1f349c7e9,0x000054ce9862d7bf}, {0x0005c9eb55b803b2,0x000a11e3c16dacab,0x00073bf12b03468e,0x0008873c24213dd2,0x000011538eb91587}}, + {{0x0004a2f731dea2d9,0x0001e4ed7b2a198e,0x000a664fed5856cf,0x000290f6a632eb13,0x000032cd90a4da41}, {0x00095d4c0c4ddc0f,0x0007447fc2c9850e,0x000076bcbc0f422f,0x000285f68cbec486,0x00009e7c0c1bd6cd}}, + {{0x0004ddb0f5f27cac,0x000ae80d59ff6599,0x000601023e85461f,0x000bfb3f05481a66,0x0000665427bbc9eb}, {0x0001a697587fd52b,0x0007dd49efceb057,0x000420688935289f,0x0006aeb1becc60ea,0x000022639d9c3a78}}, + {{0x0006220361ecf90e,0x000f455064631a8e,0x0005c2b79001f23d,0x000db504ae9b5d0a,0x0000bc9cdaeb8149}, {0x00064a1934aa7289,0x000e1e9b60f3b331,0x0009cfbfd750eb00,0x0007f5ca91615b9b,0x00007015cc07f45f}}, + {{0x000c4a5bf5151dff,0x0000c07118f2b462,0x0003fa42c21adcc4,0x0001aae60c545b04,0x0000c21aa55d96be}, {0x000c32f4e51ea80e,0x000f459b5d8de84b,0x0008f1b5e3dae45e,0x00017cdb73c7ebc3,0x0000405a74bc8ae6}}, + {{0x000e9c69f1c56bdb,0x000799f196a4bb1a,0x00075092b8c176b9,0x0000331448f31168,0x00005afe3df4f976}, {0x000fd49145813e50,0x0009e2b34226a8da,0x0007ff57f687fc4d,0x000b46f2dfc92d4c,0x000004e3fc1401f1}}, + {{0x000dab61430c9ab7,0x0002b238e9975afd,0x0008042ae0bdd41d,0x0004cb8094743041,0x00001f9addb3dddc}, {0x000c016c52dd907b,0x000c79e2047f7090,0x0001011a6d9bdf44,0x000c7836f1fe801b,0x000063accbd89acd}}, + {{0x000e2351272a95b5,0x000756276ac8cfc7,0x000d7eef70c66771,0x000b3dec0d3709e2,0x0000add2b06ea685}, {0x000d32d14ea5d65e,0x0005ad7dd506363a,0x000b4aac6f8e01f0,0x000465e9ea221375,0x0000d2a2bf9ed353}}, + {{0x00079b5e9d3a7c31,0x000671b7f34b439d,0x000c4ba758e0ee5a,0x0000c7bf3dacf51d,0x0000d3d1773fb331}, {0x00071127747ae836,0x000fb97d6b40a8e6,0x00096140031f4315,0x000767a521cceecd,0x00007246f1256535}}, + {{0x000cc5aef0c31335,0x000d2e16693b702f,0x00029b749247cc45,0x00020fad484e49c7,0x000022cef7e02183}, {0x000f40559ab93b3c,0x0000df181071e56e,0x000330ed0225fba1,0x0002f673bd659515,0x00004be69d5dddb3}}, + {{0x00076680448087c4,0x000f31432dae264c,0x000f9bf47ac30903,0x000d02f851b26600,0x000000ed311acdd6}, {0x00079fef8fd24247,0x000a8a6da98b045e,0x0001e673afdfd974,0x000167d5c9f6410c,0x00006f2e733cb2c5}}, + {{0x000ebb52a6017539,0x000c357c2d491ada,0x0000bfd24b286514,0x000922487696701e,0x000050c547e94478}, {0x0001969e5d32bfe8,0x0001f50d6c3ed1d4,0x000e27f3a30bc147,0x000e0c0f3679fee0,0x0000f64a7dd24a6e}}, + {{0x0009937633dfb1fc,0x0004d77f25472fe5,0x0001ea646ea82c39,0x0004510bdfdf1a66,0x00007ccc59279085}, {0x000796281761e131,0x000f8196885c8217,0x000ffbd70da57596,0x00036fac17e84928,0x0000e6e0a413671d}}, + {{0x000872c4152fcf57,0x00002e77e75461ae,0x0004dff09441c87b,0x00017160799dd5a3,0x000066b4e44f8a6b}, {0x000a51211f1c7924,0x00030be35c3edc06,0x0000c469eea02ae9,0x000f5ca5ca4d6de9,0x0000df4368e96e4f}}, + {{0x000acab4baef62e3,0x0002785b91e87817,0x000e576109f5a220,0x000e036666ebe66c,0x00002ad31f4273bf}, {0x00030a425bcf4d6c,0x0002915056e66283,0x000332156ea95059,0x0002d699811c89e1,0x000089cf1ff4c11b}}, + {{0x000391304e60cc0f,0x000c5625d37575b6,0x00026e562ce811e8,0x00069f130e43fc2d,0x0000bb30b4c508d3}, {0x000f82c485281182,0x0005ad285911634f,0x000358f287c6fe08,0x000c095f2830c099,0x0000e60a95e865e6}}, + {{0x0007d3d9b785dbf2,0x000b8759bce70840,0x000f61239530889a,0x000e3cd228e0e652,0x0000b6d14618879b}, {0x0002c0451a7bbf7f,0x000f86f24a64e690,0x000bc6da030ad99e,0x000abf76d9317a98,0x00004f877f4bb596}}, + {{0x000f62d4c44f1190,0x0006e9b77416b05f,0x000aed63b4555f53,0x0002a9c7c0d0598c,0x0000cd2b7cec358b}, {0x000287b46945fa35,0x0000867c87913f33,0x00037bd08f8785b2,0x0009d7754a7a6196,0x00004d4598c68be7}}, + {{0x0005acbc46d7ce17,0x00071b085877889e,0x0007a50509a515bb,0x0000358ac1a03d0b,0x0000d3e738b62926}, {0x000c2ce2a6cb0ebd,0x0004bf7adc79861c,0x0000163766f2e295,0x00008f91c4d45133,0x00009fd2c812ad59}}, + {{0x0003738b2b836a16,0x00001c0d6622e5a8,0x000c19af1855b41a,0x000acab86fe3177c,0x0000465c2005dd99}, {0x000c23f6974b99e0,0x000ba2717cbe46e5,0x0002be65875a7cf8,0x000c984d2ebc3f06,0x000094b44475f209}}, + {{0x00085edb940cb5af,0x00018cc82f104af2,0x0000526fa6706d79,0x00013fd8c8776c03,0x0000a8e6f7790da9}, {0x0009d34591ee4f0d,0x00067027416677ea,0x0006714575f46e33,0x000fe14bdf98bbea,0x00007c08b47562a1}}, + {{0x000303c1c08ad635,0x000ffc845e7b46cc,0x000f36bf79954343,0x0003a6cb8fbdb548,0x0000b82c392dc827}, {0x00012c4928435d5c,0x0000b933038008f7,0x000da054a071cf0f,0x000b5c074c2d24a8,0x0000b0e720203c46}}, + {{0x000337ac0b7eff35,0x000e75e48b3c0ad7,0x000f13a5f8552225,0x000cbe96f78b0c73,0x0000e70062ed2349}, {0x0005048e7073969a,0x0009233cb3d26b8d,0x000caa20f392d2a2,0x0007074e4f727c4e,0x0000068c99ecccde}}, + {{0x000651fb87a29137,0x0004ccc252f0fcd5,0x0006cd3e4ea3e3c1,0x0002e7077d92df3b,0x0000a41414435a73}, {0x000951aa71ff4939,0x00022bd37cf6a895,0x000cfeefffe980c9,0x0003e8b5bd5e64de,0x000010dc2aa344c4}}, + {{0x0003f26cca9f54d3,0x000b6303f6dbcb40,0x000eee67c928bbdf,0x0001c30c37951ea9,0x0000bd61a5327996}, {0x00038e6395c9a791,0x000d51eb352d09a2,0x0008756316940ca2,0x0000d16d1e5c5ec1,0x0000e19742c2e1b2}}, + {{0x000d90823fc2e6e0,0x0009089591494633,0x000ed7da5a76e29a,0x000d5161069d9c84,0x0000baa11cf5dbca}, {0x000ec64961849da5,0x000f5f3d8c28d01e,0x000a2ee4493b75f1,0x00055807bc4f9f1c,0x0000a26322d50e00}}, + {{0x000d65861a023efa,0x000419e5246e1888,0x000563ec01d72aab,0x000a4309a26348e5,0x0000097196463439}, {0x000d54badb9b5b76,0x0001645a524b567d,0x00038e60873fac1a,0x000f482fe97ef7fe,0x000008748d29f384}}, + {{0x0001794c486094f9,0x0002fbf3a8d6b057,0x0000b0e25869254a,0x0007d7848a8dd131,0x00009ab9f402aa3f}, {0x000c68a6706c02e5,0x000c19790e6c0927,0x00071376c22b5e76,0x0006571c3252606c,0x00003a5769059ef6}}, + {{0x000f852edffcf3a5,0x0003ec0a6f558d63,0x000519b9eb4d2ed0,0x00069a8b3aa8de12,0x0000d38e9c46e0a5}, {0x00028bf303747e2b,0x000c45b5c18d8715,0x0006bf923a208e77,0x00039eed129c88ca,0x0000cbf19806f028}}, + {{0x000f030273231941,0x000b239ca59d9b9b,0x0006695203b055a8,0x000f24a46b23120f,0x00009789f1f597e5}, {0x0009468aaf018013,0x00005b69d59c9c49,0x000f4c07972ee119,0x0000485bd39595ac,0x0000ee11ecebe0cd}}, + {{0x00086ec1ed66f181,0x000bc61fce43ebde,0x000bed74d225d906,0x000ab74cab07d6e8,0x00006e4617f37855}, {0x000aaddb2fbc3dd3,0x000f5aeddf5b6568,0x000cf2fadedb5484,0x000699578f20e86d,0x0000516497c915f5}}, +}, +{/* digit=20 [{1,2,3,..,}]*([2^140]*G) */ + {{0x0003fecfa181e695,0x0000e0d69a98ef0a,0x000eab95d9ea02f8,0x00002162e9cf8e66,0x000020f2beb74720}, {0x000540a1df843618,0x0000f1fa6d5d621c,0x000f5f6ff1203772,0x000ef2ee3c7b510f,0x000017a069c2bb2b}}, + {{0x0002fb6b294cda6a,0x000519039f348357,0x0005cbb216ce9bf7,0x0000d980e012f009,0x00000aecc1c7063f}, {0x0001c3af02909e50,0x000f48ce9cdc57c2,0x000e336f8c7d59ec,0x0005f42732b8448a,0x000056e37233f4f8}}, + {{0x000b53189e800ca4,0x0006d45208fd8a10,0x00014ba3750fe0c1,0x000acc5e43c0d3b7,0x000027d200e74189}, {0x000e24fe616e2c00,0x0008ee1854c105de,0x000342a739c25f4c,0x0009524d3222a58f,0x0000807804fa027c}}, + {{0x000653a4f0d56f34,0x00078a28b805c222,0x00073434b961e404,0x000a18ec03f8b04a,0x0000c966787eb712}, {0x0006c42864fee422,0x000a3b0ece5ccc19,0x00031c159c1be93d,0x000655887d9f22c1,0x0000bb6d593fce45}}, + {{0x0009ec9b809b7ceb,0x000b32c72c2c22c4,0x000a0bf368a41486,0x000c68d13b9420fe,0x00003d36eea566da}, {0x000c08a328cc987f,0x000b4a3264616fdd,0x00010dbba0a3bcd2,0x0004c38103c49dd8,0x00009d81a293b78a}}, + {{0x00065ade4d559419,0x000da03840873de8,0x000f18b9bdedafa5,0x000267df414abb4e,0x0000ee9ea438aee5}, {0x000aa1637a55a4a5,0x0003b15f93b9260f,0x0009c3598eb19a51,0x00078e01d7ebd29e,0x000023fc56d69321}}, + {{0x000070cb98fe684f,0x0009224a1458501d,0x000bc6b3fd60fbe9,0x0007cab45761c892,0x00005384859ee6f2}, {0x00071f7b59e763bd,0x00088b5a8e5e4b02,0x000a482923d4606a,0x0004454eda5d9b05,0x0000a7731d1b6fec}}, + {{0x000369390d458714,0x000fc6166d8da3e3,0x000a90403e976403,0x00063775c3368289,0x0000bd17983b2f1d}, {0x000679ed5d2c53a7,0x00088dcf3b87a616,0x0006a694e5ec4bcd,0x0007e53e6d7613b6,0x0000460fc7753fc2}}, + {{0x0009b8295caabee0,0x0005889501e37046,0x0006ed265de024ca,0x0008b26bdadc0607,0x0000cb1236b5a0ef}, {0x000ddbf0972ebf9d,0x000452aca4324065,0x0004aff76f1dd387,0x000d23d88b97cf74,0x00001359afece8e3}}, + {{0x000ba2b91502cf37,0x0007984db75d52a3,0x00030b1c92c3832a,0x00060b94a12dddde,0x0000802eabd531fd}, {0x0007327a37fddab7,0x000b8aafa9733370,0x000e6f91a65d6f2a,0x00030ac525c5b811,0x00006aeb0c9cf465}}, + {{0x0005ff62f93a6750,0x000405f48679e881,0x0008ae884a6ec968,0x0008736dcbb55635,0x0000af61472e19e3}, {0x0004372a5f696be1,0x000a5f22fb707233,0x0006cea90c65e57e,0x000b2a168da30c94,0x000036a8a8775681}}, + {{0x00081dc0f9f44d43,0x000ffc46585aad5e,0x00047d1b1f09a695,0x0008b1a1649164c4,0x0000b4b36c8b79dc}, {0x000177b3b6b234cb,0x00046730d9d020d4,0x00080531d096a250,0x00095c5611b9b8ef,0x0000a904b3c14bb4}}, + {{0x000d9d493a3147a2,0x000c7a5655451192,0x000f072129f30a5d,0x000c1370b1f9cb6e,0x000099585462d87f}, {0x0003effc17db9ba2,0x000cab1644a8d332,0x00049ffbccb18548,0x000683f8a306d44f,0x00008d658f16c2e8}}, + {{0x00060cda99f8c71a,0x000aabf742ff44ba,0x0004b3f9967b7abd,0x000160c6310f9c91,0x0000e430a339412c}, {0x00076d388ace52f4,0x000412d7067d1e67,0x00007cd1b4bc0fa2,0x0002c0c3c286aa8f,0x0000cb8f38ce985b}}, + {{0x000be808c3bff364,0x00021263e57583cc,0x0009bdcd1005a0bd,0x000b6b060d7dda25,0x0000a1c56433a5ca}, {0x000dbb99fe4fc88b,0x00081c97bbb52b7b,0x0002321ae09418e2,0x00064e28274fb4a1,0x0000137007e0c87b}}, + {{0x0001fe1c63c4962c,0x0008d81fdb258053,0x0004c2b6b50541e8,0x000fca1c1291a1fd,0x00000693a1866df4}, {0x000604e0117f203b,0x00025a99b8d0b2c4,0x000212c44245f196,0x0002a7fedc20aac6,0x00001ed4e57020f5}}, + {{0x000f575f8547be3b,0x000cf9e45f98fe48,0x000c501000a7033c,0x0001d99b45d3a918,0x00002a6cd6b561d4}, {0x000b4f557933c6b8,0x000cab0d7ffc60bb,0x000d626b6a7538eb,0x00025a1ea3ab8d8c,0x0000273a484b6016}}, + {{0x00098450168e508a,0x0001f9a94abd8885,0x000b0a6718cbc9bb,0x000ffbd13ac792fa,0x00003995b1a0c9eb}, {0x000668e1239e1525,0x00086bb8dff4e711,0x000f179635689255,0x0002533bfc7dabdb,0x0000b59fe5b03de1}}, + {{0x00020eb34a9f7aec,0x00021751efe47e33,0x000be2f37e5e8cf7,0x000ef6bea003bcd9,0x00000f551a176c08}, {0x0006268038f67254,0x00052d92d3b65660,0x000cbd6861dd38e3,0x000da7c7dfce7cc3,0x0000e549e04a51c5}}, + {{0x000f93b08b193400,0x0004cac6d89d4058,0x000159cc7c2fae6f,0x00001c4bad8a8c8f,0x0000ddba4b3cb0b6}, {0x000c7b51dd95f8c7,0x00075ea5c255da4f,0x0004a8c4c1d163cd,0x0009cdc0707d0627,0x00009d9e0089802e}}, + {{0x0009ebfe6ddd5053,0x0004850bed1a02a2,0x000327d5737064e7,0x0000bc0f6bae65a7,0x0000846f5f228392}, {0x000749160df1b9b3,0x0004fd1da29f87c3,0x000d1743c4cfb289,0x0007c5e0a478ca4e,0x000090c60306edd4}}, + {{0x00053128c0a78de0,0x000a1e85df708f3e,0x0001b6582ccd02bd,0x000bd1d6c75c03a6,0x0000762921cfc0ee}, {0x0000823d85010c02,0x000ae044cf1fd34d,0x000b5e78ad73aaac,0x000f3fdb4159bba3,0x0000287c7f805826}}, + {{0x000f742580b1a014,0x000d20423b794aea,0x000dea144f080415,0x000472a12622cda7,0x00009ea499699d62}, {0x00091ef571f3913f,0x000405b25a8ab429,0x000b79e8f0610f21,0x0005a157adc58530,0x000090e3df6f7a06}}, + {{0x0005deb43e2e0349,0x0003b44024aa5d0a,0x0000c9f7f53fb5a3,0x00065618628c686b,0x0000c69c29d3c563}, {0x0001febbace47b69,0x00090ea5a2ec5a23,0x00063853ebdce028,0x0008a975da1fac94,0x00006812c52e09e7}}, + {{0x000577157151692d,0x0008098e1c44d3fb,0x000399be1eb2721f,0x0008b7f050608732,0x0000a5a5512a979d}, {0x000d55dc6f567809,0x00037dc7a7f4737e,0x000941a03e20d300,0x00083972ce7301f5,0x00001ef52167d30f}}, + {{0x0007fc14092d85f7,0x0005ec49e41a2872,0x0006a4d8172d223c,0x00087d37cf30a2ba,0x0000c08620a2030d}, {0x0004c7dfc588b090,0x00095874bbb00484,0x0004c0495728cd49,0x00058fcc1281eee8,0x0000769b5baec319}}, + {{0x000228bf99c24714,0x000ad91eb110665c,0x0006d7024f2d8a11,0x0005a10594f494d3,0x000082ded8c0dcb2}, {0x000a9d8dadd48854,0x000eb1d2b547c958,0x0000af5507004477,0x000350ca45f6ef2a,0x0000fc739d66f8d6}}, + {{0x000af27786f08a98,0x0005c2c2737f75cd,0x0004e26708700bb2,0x000fef055a71411c,0x000010188c195076}, {0x000d0c9abcd3297f,0x0007048108ebc251,0x000ceed30ae4c896,0x000ceccd146de718,0x00009d4f07bb986b}}, + {{0x0008ed583fa1e08b,0x000e0eabd1fb5ad9,0x0003b11967780d33,0x0008c43330513c90,0x0000a11de9f547bc}, {0x00034da02c2d064f,0x000cb48de23b6843,0x0009089d87ecf360,0x00034b67a1b4740a,0x000028fa43aef367}}, + {{0x00082cbea4570b35,0x000b55ebcee9f2a4,0x000694cd5ee65d68,0x000d32488d0036b9,0x00003edd0e987885}, {0x0003307beb9bc6d2,0x00077f5c6768e37e,0x000f2160fe9abb90,0x000da62396ccd551,0x0000500888c67336}}, + {{0x0009ed9926fce430,0x000704da2930383f,0x0004cb227809dd1c,0x000b3830f6f5968a,0x0000d700c7f73a56}, {0x000ea33ab64a0652,0x0004f338df801825,0x00063f57faab9b73,0x000633f516100d9b,0x0000574395a47a6a}}, + {{0x0000fb6700a1acd0,0x0003fd999681b556,0x000b4e1bae823fd7,0x0000a3da915d1f6c,0x0000d0301186ebe0}, {0x000b0c989fca8cdb,0x000b49da0e0b744f,0x00031d76f970d01d,0x0009695ad8c56479,0x000015737c0a659b}}, + {{0x00033e8a8b484e77,0x00090a26dec7dc99,0x0001f0136b2fdbdf,0x000ddab349e9a49f,0x0000860368ee0fdd}, {0x0002c1cf9ad3e183,0x0007389f4e79d93d,0x0004ff1b66d6c5f1,0x000d8c4a544d91b2,0x0000e12a5ec2e16c}}, + {{0x00074e9a56b872f8,0x000c7cf68ea25435,0x000560ef7a1ad550,0x000a8ae89e37d23f,0x0000c54b9cb49d47}, {0x0000a4a088ac342d,0x000b4576c6d046d4,0x0009689e9ec450c7,0x000717e589e31c1f,0x0000acf260388781}}, + {{0x00037c6c8cb6b42a,0x0003196ef381a892,0x000f078251326fc9,0x00022cb5d56c6db5,0x0000cba2eeb1449e}, {0x000887a633c30000,0x0002d7cbcf7174e0,0x0006cf1becb6cd17,0x00099b309e81dec3,0x00007a18a6d60ae3}}, + {{0x00026799edce57e9,0x00044f001d41b36c,0x000a1f2c652b892f,0x00070a4884ae5d16,0x0000b3294257fcc3}, {0x000daf2bd2e21df5,0x000cb2470a993120,0x0005db32e55298d2,0x000635bb78af6ca0,0x0000c76a331b01f5}}, + {{0x00061fff8a4f29c3,0x0000a68f8d49aae8,0x000b1321c70dc924,0x0004ce660e649f81,0x0000d2c801bb792e}, {0x000f772425218760,0x000c416c79b1f479,0x0003e5bc90bed93b,0x000049a67fbc0526,0x00001e8e630521db}}, + {{0x0006738c6f3431e1,0x0002d326754176f2,0x0008c877ce609cb0,0x0003cba10cff2d81,0x0000f0e75ce886a1}, {0x000ca641158544d7,0x0008fcb71ed0f4fd,0x000aa47555d777e8,0x000b3fcc233737a9,0x0000b4531935527a}}, + {{0x000f68839f05ffea,0x000fcd82574edb59,0x000292d1b8f4f4bd,0x0000862ce3450cee,0x0000a448a1301ccd}, {0x00091b3f7914967f,0x000a2908a5edabce,0x0001042e74537f09,0x000a33b812421ef5,0x0000af5cebddc0b3}}, + {{0x000fd874ca6b39ac,0x000c42efd342730f,0x0005c8edb70fb72e,0x000a512b4735f9d7,0x000011f21588278a}, {0x000f635bf3bfebf3,0x00043bd9601fc459,0x00020cb733a1ff0b,0x0005a399d12823c4,0x0000e9af3e26c291}}, + {{0x0002c72b41c34403,0x0005b3039a5fe0c8,0x0008795a3175239e,0x000c60b1084b8a55,0x000028d0a1e001e5}, {0x0005f2ed3788a049,0x0005d6c11a9f0a49,0x000d692d625d8ff1,0x000fe465155f059e,0x000054fa107df425}}, + {{0x000abf2e98aaa995,0x000046b0f88ad16a,0x00054026a90cd8ba,0x0006d2457f4782c1,0x00004ee0734a2af5}, {0x0009e5445b4147a9,0x00010a52816cbcf8,0x000b62e773d102f2,0x000ad7d808517e39,0x00002e25421f9169}}, + {{0x000d871bb608558c,0x000e36d4ff9bd721,0x000f2763e60e4eba,0x0002464ba1081941,0x0000a2e45bf11ee3}, {0x00072ec2bfd7a5f1,0x000f64d0b12d66d1,0x000be70dc528a8f2,0x00083cd17f1e38da,0x0000d5d7316af939}}, + {{0x000184adf423e315,0x00015edb1a1051b2,0x0005bcab9cb41729,0x0008efd054ca9362,0x00004396860f9899}, {0x000f6c4a54ae57ef,0x000ffe648e9d4e53,0x000faf6bc0ffeb58,0x000b8a0bbdaadc6a,0x000088ae7979a3bf}}, + {{0x0001d44d2359ed9e,0x000313544ce2209f,0x00051e569ac68dd0,0x0000971378da47fd,0x00001abd8610cc80}, {0x00018d9343b6e3a4,0x0008740a1bae23ca,0x0003f3e67480797e,0x000dfc91f0c71753,0x0000489697046e6c}}, + {{0x000105552a82e8de,0x000498460cdc8ca2,0x000037178b2caf78,0x000b576c1b7b62e9,0x0000fc09d2dbb514}, {0x000f9ee9113be5c0,0x000fb3f9271c5f2d,0x00083fc542fbda78,0x000141e09a81af8f,0x00006b138668afb5}}, + {{0x000480f43e3865d4,0x0007dddf47d938f6,0x000205ff772dd77a,0x000ad8b2a8e9714c,0x00006d449d8dd088}, {0x00019ea185d706fc,0x000b07dd7f629266,0x000bc2031e47e02e,0x000ac927f120a78c,0x000018bef01598d4}}, + {{0x0007a9c6bdf22da8,0x000f10dc82df18f3,0x000703651efbc432,0x0001a5452cef8e5d,0x00002887ba159988}, {0x0009ddab920ec1de,0x00030c3e8d3b7cec,0x000a88747d0d7e8c,0x000534645bc3954c,0x0000deaa2e17fd53}}, + {{0x0001d936cc874756,0x00020d2383bd461b,0x000903546d92a52e,0x0001129abccb59d7,0x0000111a7619d14b}, {0x00084feb3d5f6126,0x000d00e828ec0ae5,0x0000870305ea69b8,0x0001fe0c07898554,0x000049cab050c482}}, + {{0x000edcf8bdce214d,0x0001b6af736125ec,0x00038b9e2b5622f7,0x000a763e1227aa70,0x00000efb2747c20f}, {0x000f88b79df975b7,0x0005a999503e817f,0x00038ec46856bf28,0x000f6e44d5351f50,0x000040a52c66c42a}}, + {{0x000bb152cbb1a3f8,0x000d97a834292e38,0x00066bb74c3eb99f,0x0008fc4a4fcbf1dd,0x000080784d74de5e}, {0x0004c1cb4e7a0be1,0x000dad15a72fddc8,0x000ec30e18780510,0x00073e34bcf1af81,0x000041e50a81a610}}, + {{0x000571847be87ae0,0x0007f76a43720d95,0x00007c3d368a6141,0x000332f57e7e87c6,0x000043afaf85252f}, {0x000e1211552a4d28,0x00023b4d4ab4cc14,0x0003816a4b6dee69,0x000a2936ab74c8a0,0x00004001ae4ef394}}, + {{0x0008344d795fb455,0x000d679f55a55bed,0x000ccdffc57326e7,0x000a0479533ce04a,0x00003473caf8993f}, {0x000eb93a13df4c82,0x000677cbe46f7906,0x000e4ccf8a73e51f,0x000dbcb1ab3ae10a,0x00005614508aa5b3}}, + {{0x000f96211a71b27b,0x000abbb7fa3961ef,0x000d7f3efdf71412,0x000d29031ba6b82b,0x00000b9c41619180}, {0x0004552014cdde5e,0x000a427b4bbbeec1,0x000e988f3702c624,0x000d034b15e8c2d3,0x0000e3bcc6e84f7f}}, + {{0x000822a42ac6c853,0x0005edf9f2b79d00,0x000de1e582db0cea,0x000b61a7cad2ab42,0x000046ed5265d6fb}, {0x00029951a2faf097,0x00000c25612eb396,0x000f564902fa8a58,0x000960c0ae04da7c,0x000056629087eea3}}, + {{0x000f5c53d080847c,0x00089241d4f63609,0x000961a63cb081d3,0x0006fbf4fb381077,0x000020c5984eabb6}, {0x000aa7cf902f245a,0x0005ae536b1e3d40,0x000b3134f9cb1273,0x0001aebeda24da99,0x0000fbd9c69fcd01}}, + {{0x000e30ac7088c7d2,0x00001207389f9a16,0x000407a535ab6571,0x000eaafb09547fe7,0x0000322f9d76fdc6}, {0x000f22d7430de4de,0x0006068ca9a9c0f2,0x0008e58681938269,0x00020337f1eff191,0x00003b5b636386f4}}, + {{0x000f9803fbc43419,0x000fcb5eed4e146e,0x00082e41d359f2c7,0x0004c20f35744e74,0x0000a9ac3ed83b22}, {0x000a6fe91fc50ae7,0x000b5613fa7c9161,0x00032f15a89ccc66,0x000d0319268b14c7,0x0000cd6f4e32467e}}, + {{0x0009869ce56b40e9,0x000c302dde98fbf7,0x000ee2cd7f93e094,0x000d425fe0c3a8ed,0x00000f3ffc14268f}, {0x000fd5608241aedd,0x000730b1afe881a7,0x000310d5295ab7ad,0x000c42701270563e,0x00003ffdeb1d9d9f}}, + {{0x0005c91d11a8594e,0x000751cf6db8c8f8,0x000b5dfd02e74d25,0x000c85a29c7ca302,0x0000389cfbf49143}, {0x0006405941768d8b,0x000453bf825dd01b,0x000cd17e24510399,0x000e791c4ee16656,0x0000ea3c2846a037}}, + {{0x000c06ed9a475207,0x000a3f8524044e1a,0x00087648afbfe18a,0x0000d8e615f8e280,0x0000301e47f29d15}, {0x000f9ddb299b9777,0x000b35b7831479f9,0x0007c90e776697a7,0x0000b4a0d674687d,0x0000afffe0403721}}, + {{0x0003e4b28c22cee9,0x000779fd55ae5aef,0x0002a5d6aefb0ecd,0x000356bcea71320d,0x0000cfb5fa191db6}, {0x0000b57f36e1ac55,0x000ce6cafb7d395e,0x00008c4db008fa9a,0x0004773f6cdf7053,0x00001527a37e5ed2}}, + {{0x000ee305bd213110,0x0002909c90d7ba0d,0x0008696d36ed41b2,0x000a80c5f6b7587c,0x0000db8eaa83ce83}, {0x000fe37b24b4b6f6,0x0007f22d1f0dd297,0x00098dbd9fe58afe,0x000527373587368c,0x0000bc226caf454a}}, + {{0x000384ece53c2d04,0x000d1e4606daa12b,0x000ec12b0779d897,0x0001ad653e47b073,0x000062dbbba9756f}, {0x00009f2cafe37b68,0x000f6cce2e1769fe,0x000f607fd273d1eb,0x000c250ac1d5383c,0x0000035f7ff92e10}}, +}, +{/* digit=21 [{1,2,3,..,}]*([2^147]*G) */ + {{0x00034c77e6c55202,0x000fbcb9ea58854d,0x00086666dc27df9e,0x000a85205f2369d6,0x00009d1febf2417a}, {0x000819e93470afed,0x000912a27f9e9846,0x0001e65043e6a966,0x00080954d008a2e3,0x0000ba7ced06cb76}}, + {{0x000f541338d6e434,0x00030541d5ccecaf,0x000bc88ca56f7dd7,0x0002c375d426de96,0x00008d94f6bded3a}, {0x000a3bb2ef8279cf,0x000a1b1867f26354,0x000225151d575465,0x0000d7ff99b0ff95,0x00003e19d89e9450}}, + {{0x0003268e32dd620a,0x000ec27849a292a8,0x000378882913ec99,0x000cfe4dd8fdfa2c,0x0000f96f33f8e6f8}, {0x00037e5dc3fa8a51,0x0001a0b03a1dc067,0x000f037b0236bb53,0x000a5323e59f2989,0x00003f9b5a7e9a12}}, + {{0x000f6ce51efb3108,0x0004158df5be0d0d,0x000158e59cb5b2eb,0x00033656459e2936,0x00002aae2b99466e}, {0x0008a39411aa636f,0x0004e4c0a933fb65,0x000f026b77152ecc,0x00011f010c758a49,0x00004837f98bb093}}, + {{0x00002c4c753c45fb,0x000649c840feddfb,0x000f8a3e618ca81b,0x000dbbd46fd09ab0,0x00001162ade97733}, {0x000ad20236e3ab64,0x000572a563267070,0x0007cbc7af88cdaf,0x000271c5fc871999,0x000042cd4528b665}}, + {{0x000f364b71698f58,0x00021f7b605e7807,0x0003b2cbb6ba418d,0x00086f7d20b00fa0,0x000083eca385a543}, {0x000e43ff3437f24a,0x0002048bb33cff0b,0x0009df765e910b43,0x0006f6a963a12832,0x0000c1dd5575e2fe}}, + {{0x00010f924a0a3fcd,0x00041881c3f95576,0x0000dac9938e17bf,0x0009179ba84fafed,0x00004a222c429eeb}, {0x0001dbe13f542b66,0x000d8425d457c79c,0x000ebb7791fc65e0,0x000f608ffb754f1d,0x000038d8fd0fe08a}}, + {{0x000523a626332d5d,0x00028561bb44994f,0x000845ea27bc3883,0x000089305ed4b03d,0x000039d3ee292a1f}, {0x000fdd3e7676b0dc,0x000f3b7060176561,0x00064f9a8620e35f,0x0001f676ce424ff2,0x00004c341a09a268}}, + {{0x000fd2f69beb6e85,0x00031d700d03fb6a,0x00083a14f3a50b99,0x000bef7840b2ad0c,0x000073207bea4085}, {0x00082e309fe7e5b9,0x0003ab40a7e15af8,0x0003056e2957678a,0x000c09872d4bdd54,0x0000c1b26b49df13}}, + {{0x000861cf405ff065,0x000cf86e828b1c30,0x0006933fcebac86b,0x0009479791a97163,0x00000e7c2becaeee}, {0x000a095fa90d7679,0x000b5670ab7bc3d4,0x0007b056dae60eb7,0x0002a987633a6439,0x00003a21f33a0501}}, + {{0x000370babb886433,0x0006f2e21599663c,0x00076167191df36d,0x000bf0e83ba8358b,0x000081eea1da28f3}, {0x000f1ba39966e6ca,0x000847295492b9b2,0x000b26b7f7c464a2,0x0009de6fd5f70a09,0x00009aba1fa9be00}}, + {{0x0001f22369b87ada,0x000892fca556857c,0x000b064663c00e5d,0x000af17ad74cab90,0x00007112386f50fa}, {0x000e1986d9bd5f51,0x000849c3463f7435,0x0007bd4b22dcc7e3,0x000f31cc7df748ca,0x00003cd4c08adec2}}, + {{0x0005df8e32377106,0x000e6bd2f7b00d3b,0x000aa082b0dadb26,0x00066e3f5966abe4,0x000066ec8de950e9}, {0x0001ed5ee5242161,0x000b31dab0b61bfd,0x00086d6bacd93c59,0x000195558a8435d1,0x0000b7d34d2259d1}}, + {{0x000e46022caf46b4,0x000f5a96fe4f5936,0x0008f474e6a45dd8,0x000f15b7925434b9,0x000014104124053e}, {0x0008d1241de97bff,0x0000cd80bef471cf,0x000db0037b8547b6,0x000dfec47d3970c4,0x00001bcd329eef20}}, + {{0x0002e0910caad676,0x000ff531a1e131a9,0x0004fc8401f59195,0x000c6bdbb852e05f,0x00003e297caf3a72}, {0x0000b2e49abad67c,0x000c2d3db0d93c2b,0x000ef1d406ec405f,0x0006fc1c14a5307f,0x0000cd19846e8089}}, + {{0x00031769bb816483,0x0005353120d000f8,0x000cabc62d69eb48,0x000cb1a17d75f44c,0x00004a07f82e749f}, {0x000f787bbfb55541,0x000052e283f82c3a,0x0009213a0b06ed4d,0x0007b44722889fa1,0x000062b085eecf3c}}, + {{0x000cb31e0dd3eca2,0x000cc52f13a5bcae,0x000bac297c6237fb,0x00054aac2b6b0327,0x0000ae1cac5d917f}, {0x00007d47845ae4f6,0x00026e5972e04748,0x0007915bbfec7dd9,0x0007ca63bd25411d,0x00006f85dc539490}}, + {{0x000b888bdbcf0ca3,0x0006af279e9fd981,0x00054e934d75f5da,0x00034ae28bbf2470,0x0000c6ff6e5b1db1}, {0x0007cf4047d26e4d,0x00078049ec37795b,0x000d945aff370f7b,0x0002bbf6712d4dce,0x0000f30b5ecd9564}}, + {{0x0004c624896246e5,0x00069e90bbd19b03,0x000fedb735652c01,0x0006139b38636f87,0x0000e32f8475135a}, {0x000b312cf933c835,0x000dca7f47e60703,0x0009c2415d05bb76,0x0006f8325e4f0c94,0x000069e5622c250d}}, + {{0x000eb3a6568013e8,0x000ea2f243fcbbe9,0x00042734a8dbd203,0x0009841dbd7694b3,0x0000f6d12f8c6afa}, {0x00010a2c9eade291,0x000337dd0f18b986,0x0001c0d46bab4f32,0x00042a4779737b67,0x00000b6a7c6e3e0a}}, + {{0x000ddf33035b41cc,0x000ed9c45895fb19,0x000c857e5d336343,0x0007d4a1fe493854,0x00004d506bf6e4e5}, {0x000c8cbbbc33f758,0x000a1262c77d3cd8,0x0001a28237281f08,0x000e32883f4ea6f1,0x0000895041f1fba2}}, + {{0x000ea499c438edfc,0x0002d1edba44fcdf,0x000ba50f07678dcc,0x000c1b307b3b87e2,0x000013888f003948}, {0x0005ad41140af420,0x0003926ed1a7c213,0x000f6695f8e5104f,0x000120f24430cb88,0x0000ce0637b6d73c}}, + {{0x00001e6fe631e8f0,0x0007d7bdd24bb2db,0x0009ad44f1c5563d,0x0009f98daea3ba36,0x000000c81b68187a}, {0x000a951aae1fd9a7,0x00071d5aed8a5f48,0x00098c622e35626c,0x000503b095276304,0x00006d17634e73aa}}, + {{0x0000ddaeb300f7aa,0x000c4db5e80136d9,0x000d5244c9dcf7df,0x000aa1c45cb26874,0x0000127ee79d48e3}, {0x000cc53575f1dbba,0x0004e0e6161e488a,0x0002650d095037e8,0x000215b7e5928329,0x0000be67d99b4938}}, + {{0x000944b3f8e10652,0x0006130e89243c7f,0x000530136ed908cb,0x000168e8ee8fd56f,0x0000227b7d5f7ffc}, {0x000c893b5cd6dd54,0x0001662796e84f55,0x00018e12c82225e1,0x00051a1c6cead1cb,0x0000381ae0cc4f5a}}, + {{0x00013d37fafa4c8b,0x00015491aac03459,0x00069264c3d91808,0x0000cce347871f3e,0x0000ea9dd3d64f4f}, {0x000d0673eadd3e7c,0x00074573bcd8bda5,0x000a2486c0033c1b,0x000ee6655893795d,0x0000b89ee5c46abb}}, + {{0x000a8f322532e5d5,0x0000227dfc4c8fe0,0x0006726dbb6410ff,0x000dc7119b9d5822,0x0000ec25669ca2b2}, {0x0002e064c3beb017,0x00000acea556af4d,0x00083487a852123d,0x0003ea9e9470faf7,0x00005a7ea04c664b}}, + {{0x0008f356798e4ba6,0x000567d0e0914ad7,0x0002904039214e6e,0x0007496420b488b1,0x00004049e0b5c295}, {0x0005af13ae9841ff,0x0008c0b662a603ef,0x000453458dbe4ca1,0x00072156845c5ffa,0x00008dabf1a00b66}}, + {{0x000f0aacce2793bb,0x000e15ec47c1b650,0x000234fa971db851,0x0006cd8eb78f3e3b,0x00000c60f36ac010}, {0x0007121774eadbd4,0x000f8e3238630542,0x00008697625367fa,0x000ad13541b5c9cd,0x0000ff069e31c507}}, + {{0x00052568776e6676,0x000c523c6bb57414,0x0003a8a876e76142,0x0008367bf307121b,0x00000e7363ef8450}, {0x000450eb7366d800,0x000a4837dbdf5741,0x000d4316fe4ee14c,0x000825a765eb9b69,0x00004548dca8ef43}}, + {{0x0004e4c5ae888eb9,0x0000c6e9ac999c9f,0x0006ac029733abb5,0x00038e4aad3c20ba,0x0000b8dd3d3bba3e}, {0x0004c920bc5d11a2,0x00077c5f88a3a9bb,0x0001d3cb8f20127a,0x0000a62f52b06e16,0x00006c1ff098afaf}}, + {{0x0000d2f7189e71f4,0x00081ecf91e73267,0x000757a21c643874,0x000ce4d5758e57db,0x000027d09f8690a9}, {0x000308f38384a7a9,0x000420732b99846a,0x000845819aac3acb,0x000e030e94100917,0x00005cba11237ce5}}, + {{0x0004f7fb00009c49,0x0006fff28b5f6f3d,0x00097975db8396c2,0x000ed521a9ae431c,0x0000d7ba8b075d9f}, {0x000f09f34f485b64,0x000c24122516338c,0x000d471febc0ddac,0x0008c96450da1205,0x0000c3a6250a28dd}}, + {{0x000d103d1295837c,0x000f7807eb2f69c7,0x000b41491a2893e4,0x00023516e1e1debd,0x0000630745c1e138}, {0x000109e48661ae16,0x000b8a2b2674c892,0x00028d6b58d17e7e,0x000f9da0ec0f87c3,0x0000d8586465079f}}, + {{0x000243e19115ead5,0x000dfbac4fcf6cdf,0x00029f25b1ce1393,0x000a04dc960ed09c,0x00009be4d8eed388}, {0x000e06cd0def72b5,0x000d903427480d46,0x0006d4a3db923db5,0x00099ea7d3aacd93,0x000058519cc5b0b0}}, + {{0x000ebf8827097ef5,0x000b8054f55d3ea8,0x0002ed089259353d,0x000a7c34c89abc6d,0x0000c548b69de096}, {0x000f616994b995dc,0x0005e5845601d587,0x0001fd9f04d1531f,0x0006c9b92ab31e45,0x00008b57bb325adf}}, + {{0x0000fcb1cd5ad73c,0x0005a144da4f6844,0x00062beb8b9c860e,0x00097e6ab286aa84,0x0000c6b9000af467}, {0x0000da420c8a471b,0x0000c7ff7fafac82,0x000b5da7769ae05a,0x000b7a09163f39bf,0x0000d03e590dc73a}}, + {{0x0002b5eb2940d9e2,0x00062b9af5647e86,0x000e3033d3c663d8,0x000bc6b8309031bd,0x000098231b2f42c5}, {0x0000d2c552ad093f,0x000c0f8546954209,0x0001f0d00a4799d1,0x000b451a88b5d6d3,0x00008b4082692f26}}, + {{0x000b1edf1bd72186,0x000aeb24c86eec29,0x00095ea65d491c53,0x000f1572fe588f33,0x0000f3764f7a456e}, {0x000116dcdc348009,0x000631e33955db43,0x000ab286bcdbcd45,0x000d7c5fdb554074,0x000048c7a52618c5}}, + {{0x000aa377378058e6,0x0000a4411154eb81,0x000828ac741c746a,0x000b29410c73bcfb,0x0000439be91fd972}, {0x000b4b043a2fbad0,0x000f82b5e8404bf3,0x00097bd4c39e6dad,0x000ccb4f71640863,0x0000f7de5687f1ee}}, + {{0x000c5a1d2ffbfc15,0x0009fccb64515865,0x000b32558f74211f,0x00012e16368a88c0,0x0000b539dc2ead78}, {0x00083d02f3af6f6c,0x0007d9934ece5794,0x000c9e9835213207,0x000b8990b9650fdc,0x0000a989eca6ee42}}, + {{0x000c829d6f62f995,0x0008fc2a7c0c6a44,0x000a0cb0a8f06a30,0x000363fea2b3a098,0x0000c547b710eee8}, {0x00040e1682afe112,0x000a5b41c0a8461d,0x0000d5d369e0fc77,0x000f6359e4aefde2,0x0000916e52052dd9}}, + {{0x00052e83f883faf0,0x00092b868d35f59e,0x000a19881396f963,0x0001a6c902a9df4c,0x0000fc96822db240}, {0x000758766f1c68d2,0x0003db476c0d4123,0x0001f5d9010fc6de,0x00044ad8b6b57984,0x0000ba8446d1a24f}}, + {{0x000b920ef4a9975b,0x00037330435fa237,0x000b7e7b560bb600,0x00039126f4ab5acf,0x00002ac509833435}, {0x000ee2fb0d1ea67c,0x0009b4c56230f036,0x000838ae6ae779a6,0x0006ef99bff8c8ab,0x0000d83ca9a5b38e}}, + {{0x000bef5e33deed3a,0x000e601892a8bb27,0x000dfbd3ee6356f6,0x000c9d1f3be6cc7a,0x0000ecbc81cd3d1a}, {0x000b909e6e861dcc,0x000393f5f801e4fe,0x000346e5790a247a,0x000c1a41c50acb27,0x0000e29242f061ac}}, + {{0x000214a2f998a912,0x0001b4baf27b04dd,0x000c26722271ee9b,0x000ce55e3027d1e8,0x00001d1645c3820d}, {0x000242c7501779c0,0x0007fa0e8009086f,0x000187129f006140,0x000bd0f23ce47760,0x00005bbdedb0fde9}}, + {{0x000483225d984730,0x00055c658427682f,0x00066ffa1f207fe8,0x00099db6fdd7ba41,0x0000c3140569eed7}, {0x000048f4107e28f1,0x0001312168400db8,0x000a3c06e74ed387,0x00013464489f8f56,0x0000e1c005b22777}}, + {{0x0002a73f37ec3c39,0x000d4d59eba0db33,0x0004d3257c65259b,0x00038f9291709cdb,0x0000a793b264d389}, {0x000e34be43756f0e,0x000dbafb56c9f39f,0x000208b272f76bdc,0x0002c2bf37867a61,0x0000a1d4307e8997}}, + {{0x00053308bdf623a6,0x000a2441fb7d8c59,0x000ddfd955f5accd,0x000be79afa941832,0x0000ad40c5a6fde9}, {0x000ba89aeca8709a,0x00008c248a9d43fa,0x000637a76c64a7cf,0x0001ba7662025272,0x0000ee1c791c2b8d}}, + {{0x00098fd21a843b21,0x000d7d005cb1f0f7,0x0000d8abe56e4ed4,0x000326255f77801f,0x000097b04cf44522}, {0x000b31ffd42c13f0,0x0002b40f933d41f9,0x00060bad45ef7feb,0x000f8927326f425d,0x000027ecdb28c92c}}, + {{0x000e4d14e3352feb,0x000ec3591b9004aa,0x000da7d6008414d2,0x0004ebaed6124eb7,0x0000985b931fd13d}, {0x000d3ab96bf36f90,0x0005bbdf51dfa592,0x0006c177d012dbed,0x0009cfa57963c0df,0x000010ec86987ca2}}, + {{0x00000f6bf926dffa,0x000154bf6bc2ba17,0x000da11f57c9fdbd,0x000e753c18dc8f64,0x00006074b7b7938a}, {0x0000066e84f44a48,0x0008527b954e1427,0x000f38e9a99998d3,0x00001641be8ab2b4,0x0000bb55bbf95c01}}, + {{0x00072b40ea2ab303,0x0000c73d68ddf734,0x000c2e1ebd365a34,0x000719901a716819,0x00002f49e3764061}, {0x00057f101d8b4d66,0x000bc6b47700b73c,0x000d8826a03c8423,0x000637d21d0bc8a4,0x0000004213cabc0e}}, + {{0x00064a1c1c06681f,0x000eff018e50f78c,0x00042b2b316e0a16,0x000f5741cbdf91db,0x00008f4ffcfd0d36}, {0x00071cd4cc5e3e06,0x000a4129e3e0cdcc,0x000b2cbf1d55c7cf,0x0003cb6cdb6ba00f,0x0000aba000624bce}}, + {{0x000db30d232cfc4d,0x000e058a3cef501c,0x000e091499ddcf12,0x00025632d2cf9c87,0x0000c5d7ec7fc976}, {0x000986e0b50d7dd6,0x0007207f112a6447,0x0000ae9f688fdbaf,0x0007dff8c9822ab0,0x0000abfb950cd3d2}}, + {{0x00044878a429f4f8,0x000b5b516609d0a7,0x00069b5df0649712,0x000af23826ba57e7,0x00002335df29fc7a}, {0x000f0675c93d9950,0x000a68677be62389,0x000d9951b59ac367,0x000ccea77985ff21,0x000038956fb85011}}, + {{0x00048cbbb734e37e,0x00014be5b26f608e,0x000b1a0d9c08c0bf,0x00031837bbdd3bf9,0x0000ac7d898f0483}, {0x0004bafbc1a6dea1,0x000f072aafdbc95c,0x00035c41afdd0e2b,0x0001d530373cbc82,0x00004303f220b6f4}}, + {{0x00036210408f237b,0x000a3cd2d1edba06,0x000abb6a2cad3b09,0x00017a9667855a52,0x0000a9157dd5a8b4}, {0x00035074f013efbe,0x000aca38c4a2fe7f,0x000a643451b112c4,0x00080ac1406a609b,0x000053cba344993c}}, + {{0x0006063ded40d232,0x000d34908e254546,0x0003c3c313d5f1f4,0x0006247ebefe6240,0x000074ea0b52672a}, {0x0008d99451d1b71a,0x0002ef79cf79ff81,0x000ce37f580e8264,0x0001fd0165df1373,0x0000744ef509e3a2}}, + {{0x000e7f5cf551396c,0x000dc68c676b73f1,0x000442c36c616898,0x00017ca71c28c78c,0x0000fe5e5591e0a3}, {0x000d8187051f476d,0x000544f034421242,0x00044d0f656fad2a,0x000f4d562068bc0a,0x0000fa2cd6f9e6ed}}, + {{0x000813ad15d15174,0x0001f77d44f50f43,0x00039b35f61214cb,0x000c199399aa29c6,0x00002136d7194c51}, {0x000711b084172219,0x0002c2545a579774,0x00050582d0a5546b,0x0005bbf0624c4111,0x0000ec5c4198bc55}}, + {{0x000dcad771849f1d,0x000431d7bf6f2c87,0x000116eb2b0c932c,0x000bd29aa5cd3e89,0x0000378c25b21ca7}, {0x000a0da9e6e3e319,0x000d268ad5d0c612,0x000c6edb80417a54,0x0007cd70451e4a22,0x0000fbfe01a44282}}, + {{0x0002505ba9384a2d,0x000d94ad69c12fa9,0x0003b35a621b8596,0x000671bf4fcc4998,0x0000e09376142754}, {0x000ccc8f7bffe6d9,0x000ecd94263d2f14,0x000f3ec3027566bf,0x0006ba25b4e9c62d,0x00004f1d7d5ce6ea}}, + {{0x000851aaaca5e9b7,0x0000e6713b9797b7,0x0000a61f6518aa52,0x000b68c357e8c715,0x0000842e7e35c2c2}, {0x000af656868a5489,0x00025068fc818dff,0x000917733d963bd8,0x000327d4da5c8b65,0x000027091000b247}}, +}, +{/* digit=22 [{1,2,3,..,}]*([2^154]*G) */ + {{0x000c9a7d298c241d,0x000986807cfd214b,0x00064eadbe3b697b,0x0009c51f1c780245,0x0000de8cdd084814}, {0x000f0a75a4d2604d,0x000e9c1538af946b,0x0005b1fcc27154d7,0x000f81c5cc9230de,0x000088519ea36864}}, + {{0x000dd1a7cb1282c7,0x00064e46973ab828,0x00008d6b2a08d762,0x0003f2fbaf8d40e7,0x00002571fa1bdaeb}, {0x000732ff22dfd98c,0x00064087108d85b1,0x00088207a87ab01a,0x000754eaaafea859,0x0000cc832f929f00}}, + {{0x000950e36ff3bf0c,0x000f30b34638964d,0x000d7585f8ad20f6,0x00019e8d9177b3b5,0x0000f839761af3f0}, {0x000c5b38288c545c,0x000a53116bd1582f,0x0002120ef2f8e4e9,0x000d23391e1b2f33,0x0000f568724ea17d}}, + {{0x0001185ca8d9d1ae,0x000cf987ded2488f,0x000c46124adf2c77,0x0005f37f3039f060,0x00005d70b7651e09}, {0x00086506260e70ff,0x00070750d10582d5,0x000bac36439d75ea,0x0003289cf3d0b175,0x00003a7564e11d01}}, + {{0x00004cd2f52d2a7a,0x000a42df565a182f,0x0009fb2f74fde149,0x000897180c5eeca7,0x0000b491d7bc2ddc}, {0x0006c18c6312c7fe,0x000c8aa41a5799d7,0x0005363a0ca0d5f3,0x0002c191207325d1,0x000082aa2669eb25}}, + {{0x0004700ec3128c2f,0x00019e383f4994ab,0x0003024eb6c76d86,0x000c68ec36b150c0,0x0000b43947843daa}, {0x000764a8dc796230,0x000db440fbb2fc68,0x000c5ee0d5b86995,0x000bd3d66879bfcc,0x0000522894295aa8}}, + {{0x00040a51e6a75c1d,0x00053ea7d817b51a,0x00077459724327c7,0x0001633663018207,0x00006fdbec467fa7}, {0x0009dfb13c90f48c,0x0002a86ef26320c9,0x000f64eebd6ac527,0x000c3206a50bdcfe,0x0000d87b28246fdf}}, + {{0x000a43e3fcd3efca,0x000418088e9ab24a,0x0003d46eadd26c03,0x000a6f05ef4dc9bd,0x00002f99d592a4c6}, {0x000d3552f1da46cd,0x000d4afacdd1ddab,0x000d4057872c3f8c,0x000b94090c4eee92,0x000028bb4209a623}}, + {{0x0000711745edc116,0x000cddc8755850fc,0x0009d1e649dd9ad7,0x0000f96e6931fbb4,0x0000c77a0a3298bd}, {0x000a6296baf7cb10,0x0001ccf72d2262b9,0x000639071cf065f9,0x00032f7203cce979,0x00009ae4885f9cb7}}, + {{0x0003becee8314f30,0x000ddbea298f5e7c,0x00080acec1c068ae,0x00095b08d381f17c,0x00003b56be8e3304}, {0x000b8f29222882d6,0x000664af8bf7aeff,0x000c57d8c95ff38f,0x0004eff0e32d351f,0x0000635be5277b44}}, + {{0x0005276a5177900a,0x00075685875204d1,0x00015796c4e1dbb4,0x0007bebb475622c6,0x00006fa038809186}, {0x0005d562844c6d0a,0x000a63a2477ded7f,0x0003721d6c633cf9,0x0008e656be5c402d,0x0000f312eb889fd6}}, + {{0x00092d2e7417ce17,0x0001270ee7f52427,0x00067a41eff42bc7,0x000a57aff4dc6d5c,0x00007709b7b90882}, {0x000731dbe217f2cb,0x000cabb721773554,0x0001dd0592af2a8c,0x000476a8eee76959,0x0000b2930c9fbba6}}, + {{0x000e0477d930cfc2,0x0001196fd1f4863e,0x0009af7e14c262ad,0x0004f6d4765bc803,0x0000519834b7ba10}, {0x0001b4cd105f961c,0x0005163bca547cd6,0x000a1f17ca5415da,0x00012bb78280a088,0x00004968949e3295}}, + {{0x0009126cecdaa7af,0x000d1b13247b174a,0x00084c1c4fc8c7e0,0x000c3a39c110d234,0x00008eb8758731df}, {0x0000212c00674527,0x000e0b9b926c022f,0x00042daf43f6f69e,0x000de399032da0ef,0x00009f00adef3f80}}, + {{0x000db7181236c972,0x000b1ee0781f6210,0x000e4137274f7685,0x00053e2df7da7ba3,0x0000aae38b1d1a15}, {0x000e222f6dd9d1bb,0x0007ab8b64871688,0x0002edeaa5769544,0x000569978d21274b,0x00002818fa5ce859}}, + {{0x000dddaf176f2c08,0x0004625726581e6a,0x000342ffb01ca460,0x0008d58a404ded85,0x0000cf60f96c4183}, {0x000691cc9071c4aa,0x0003944428039bbc,0x0009c0d81fd58874,0x000f7ef7101c8580,0x00007fb754d2c456}}, + {{0x0003c5cd51805e1f,0x0008c299dca8c95f,0x000eaf500ab4ccd3,0x0008924e03d20b47,0x0000a3165c2c7b80}, {0x0008b54e160e552a,0x0009f019d11f005e,0x0009a4a7adc4972b,0x000fd681a6972e0c,0x000052c258fd7840}}, + {{0x0009ff4c1e99d816,0x000643c617c0f855,0x0008c6ba708e1a7d,0x0007945398fd4324,0x0000ffedd9231283}, {0x00059d2d629d2080,0x00053490530e8a6a,0x000505989a9d141d,0x0004ee42f6fc1838,0x00009bf250d479d9}}, + {{0x000d3b1b38227904,0x000053b8971c223a,0x000f7fa626c5926c,0x000989209efc7e75,0x00005d66a6d5ec2d}, {0x000d663987d2792d,0x000c6eb31d2b4422,0x0002cb9e64a73caa,0x000a84206c2ac1a3,0x00009445c6061aeb}}, + {{0x000a1d5af71013f7,0x00049bedc9466af7,0x0007370a0e68216e,0x0001cc84cba30bd2,0x0000981afbff7042}, {0x0006a679449f0e1f,0x000d1a47edae0249,0x000feca2286cfc4b,0x0008fa4073c936b1,0x00005694612f3f8f}}, + {{0x000b723901515ea5,0x0005249cf038d063,0x0009e50594c6c77a,0x0007c01361e360ab,0x000096cf171f76a3}, {0x00053fa6530ae7aa,0x000d6792a7a6800f,0x000db81c90f5e631,0x0000b9bcc29c24ef,0x0000269e868df9c4}}, + {{0x000f9e12cb7191e8,0x000865b08ea6ec14,0x000332bb978ea1bd,0x000e24bc65aa9b46,0x00004cc22b43f80c}, {0x000e9e9d49d5bf18,0x000599087da40098,0x000f6e357cd4ec1c,0x0004b7bc9d07c5ae,0x000039a02691f8f6}}, + {{0x000eb62c6d8607f9,0x0004daa995e4c5e9,0x000b48317759689f,0x00017ce0464669bb,0x000021474c074024}, {0x000135b2a354c8c4,0x0002412fa4b5cabe,0x000311fe8d51e52d,0x00014bac74109653,0x0000f774535f8645}}, + {{0x000d6715bde48f8d,0x00025189bc7dbcad,0x00009ee8ac970387,0x000ff78d45299ec7,0x00001287ee3545aa}, {0x0008874db1dbf1fd,0x000ac90c88d67d1f,0x000368313ea46588,0x0003ad90ba649a84,0x00005fdcbcf30d54}}, + {{0x0006d43810d5ab00,0x000904d7e5cc90b4,0x000337c336739d8f,0x000c40021c1a580d,0x00000a6116268e67}, {0x000413b379f0a1ff,0x0004f9e2ab9595ef,0x0005f199cfe12660,0x00091277578b852f,0x00005c0032a1cb84}}, + {{0x000643037577dd85,0x0003d9c5fe88f795,0x000bdc13283b82af,0x00039e4c1bea26cd,0x000089fa086ec043}, {0x0009538b13799dff,0x0000e295d034033e,0x0009ddcca85fa8b2,0x000333ef17f73fbd,0x000032bd123cdb66}}, + {{0x00088a7858b044cc,0x00019aa9e39755ef,0x000d855591f0d69c,0x000db195fd9cc340,0x0000774df733785d}, {0x000e9f6d3bd2e1c7,0x0000385dfed05dcc,0x000ed09c4eb30da2,0x0001bceed7f5bbd3,0x0000d42a35cf2a9c}}, + {{0x000e9959890272db,0x00098e713a10cf3d,0x0008227b875f3432,0x000dc7ae13479fe2,0x00008561eaaaefac}, {0x00097a08332aafd7,0x000503809b62a6a2,0x00063036f9b0d8bb,0x000da862fa1cfd0c,0x0000a16eb562d64b}}, + {{0x000f5f678e62ddc9,0x000377fd752b3f5c,0x000437bbe2267c45,0x000074ce361b6b5e,0x00005c595021354e}, {0x0005f85f2b254d9a,0x000c8cb52b4eec72,0x000425fb5844b617,0x0003124d8554f5cf,0x0000b67703ecaf9f}}, + {{0x0004ec13cf482834,0x00055c8a705e4cc3,0x0007d4f84b09daa2,0x00029d91e9d0d05b,0x0000df6ef651b389}, {0x0000763aa21ba469,0x00081293f8fbe16b,0x00020aabfc6b1d17,0x0009797ff5b602d5,0x00004d671be53393}}, + {{0x00098cf4f5792fac,0x0006512152617c7d,0x000c5a6d47c5e0d6,0x00074cdb19a631a7,0x00008511a633a452}, {0x000621ca5a60d998,0x00084f5e48cb0c16,0x000ddee08f7fbab8,0x000f3c2b1e6ca2f7,0x00003bd08cf67867}}, + {{0x0008e8a2ac13e274,0x000f0eb1a9f5f7e4,0x0001f0a624494f6d,0x0008f0adbf84eb98,0x00009badc3293643}, {0x000a541004f7571b,0x00002f1c94ee50be,0x00027bc31bac67d1,0x000e27753d73a1b7,0x00003d01cf2e0686}}, + {{0x0007b1b55fd0b8ba,0x00035eec317351b7,0x0000e72b7a099d18,0x00035e802b1fb767,0x0000dc88b3448e16}, {0x000216af989d9051,0x000019b58d0134e8,0x000e55a93c2e68d2,0x000f4001f81c926f,0x00005f1462a9f296}}, + {{0x000d375ea3d62f2e,0x000221c8977d1915,0x0007b26f6a17765a,0x0007a49559710ae4,0x00000bd29c933507}, {0x000976d08d84858f,0x000479ced5c1615f,0x00034fa56370dfe8,0x000573cbc7503ca7,0x0000bb9f1ed81ac4}}, + {{0x000ec53060dd7ef1,0x000d5e65797995d7,0x000a08235eef2dac,0x000a3d44511af3e2,0x0000e324aa42f4ae}, {0x0007e71e6e676710,0x0000bf52faf7550e,0x0003cc62abccd519,0x000b5df880d31622,0x0000d402c7e2b32e}}, + {{0x000c039306a5a3b6,0x000d36783a1ba40b,0x00053cdd44e0a41f,0x00063841e8d39a02,0x0000480be2727388}, {0x0005e1d2285f3821,0x000fdc0b5c36ee36,0x0000f4d82188d8d8,0x00029a24ef1a481f,0x0000a8f43e1b487d}}, + {{0x000226d77aefb3a4,0x000dde72c2538168,0x000594df1f69a751,0x0004674e04359ae9,0x000075ffd7e114c0}, {0x000c2b13844e95c1,0x0007cd12ef94b5a2,0x000063d0085caf64,0x0003110ecd2a9ff1,0x0000dd2e22933843}}, + {{0x000e09d73d172440,0x00068fc653f138f0,0x00020e21c3ede774,0x0009eaae4459f5dc,0x00000db2ffa6a859}, {0x0002c3930cfd9057,0x000435c112a61168,0x0008bfe954934d07,0x00041a4df063c556,0x000079a440a716c4}}, + {{0x000f21897d6fbdcf,0x0006f0776aac0c23,0x00012e8dbd3a5cd8,0x000e8cdee37f72d7,0x0000b28c70e16f74}, {0x000c728b61301a0c,0x000813724354ffe0,0x0008ffedca628216,0x000de8bff4cb0076,0x000051b3088c3b02}}, + {{0x000147c3902dda59,0x00066e6973b4a5a8,0x00057457e35d2f70,0x00011acac2efcfc2,0x000033f48d517006}, {0x000af884912beb26,0x0005b62edf94c365,0x00032f34b7f5a4de,0x0000746646ba7c0c,0x000032c6af412091}}, + {{0x000f2e3753e43a97,0x0006b4d4e23f58d4,0x000ede6a670e1d21,0x000b60424bf729af,0x0000f4a94d8e10c8}, {0x0000a968d4faa6a0,0x0002b066b690aad9,0x000b6dbfdd9ed0b3,0x00043152fcd37b78,0x0000b64615e8bd2b}}, + {{0x0002048cfb9fad53,0x000cf40b76bd228e,0x000dad7bcbeaa386,0x000f5dfd6681c890,0x0000e553fc336d38}, {0x000db9b9d5f97505,0x000a828c5b0ef27c,0x00047c39b3e85c52,0x000827c90795af52,0x000047831ec0ddd6}}, + {{0x000a2274a82f424f,0x00078e47f89df327,0x000c7392c36919c7,0x000efdf478391943,0x0000101b9ab1316f}, {0x0009e9c1c5009d2d,0x0002ccd18345bcdc,0x000ce77c7fb55ea1,0x000b3d25b5e231a3,0x0000e6b4528a2f2c}}, + {{0x000a3339bb26f5f6,0x000da44d85b610f6,0x000197e541e85db8,0x000ea863697a0894,0x00005e18cc107cb4}, {0x0004f50a471fe6ea,0x00098f13439ca38c,0x00007318bf031747,0x000b3cb3c4a6bac0,0x00008da3ee5bdecc}}, + {{0x000b31c558216b17,0x000bbf79e6c20555,0x0008eed3c90c7810,0x0001262b669f4dfe,0x00000398ec950fac}, {0x000449ef701b2350,0x0003eb94f395a96a,0x000cb74310ceecdb,0x000c64285fc368d0,0x0000d37bb5216a18}}, + {{0x0000d38b880d2dd4,0x000b25930d570511,0x0006235f5a60f177,0x0006b93da34a67f3,0x00007f5e17c58381}, {0x0004b57db394af4c,0x000cb036f789c766,0x00027b47239ba215,0x0004b686d2ca0e2f,0x000042647efb73a8}}, + {{0x000754564488f1d6,0x000894cf85d544bc,0x000e4df63aa92270,0x000ced121a01d553,0x000049c0c51bdb46}, {0x000d64e3cffcb6c1,0x000e40f71d966bf0,0x000c194a0e3bf93f,0x00055465044558bc,0x00006ae33727afdc}}, + {{0x0001adf5ca48f3fb,0x0006322a9b84bfc0,0x000099e4a64352f0,0x0009c01ee54da1c1,0x0000bda54e9aa1b8}, {0x0003df56f6e55fbe,0x000340176f88166a,0x000b7b5ff1ca44a2,0x00049fb36afd88df,0x000034c24386611d}}, + {{0x000bb75861421033,0x000aef34fc4d7eff,0x000c1b1226704ba1,0x000ce94c2a468f10,0x00006b3a610bc6aa}, {0x000c0a775a0d0508,0x0006bce33e32abfc,0x000fe09be066f919,0x0003514e905ef429,0x00009ee25bb28376}}, + {{0x000de22fd29dc766,0x0008d6f172602a3e,0x0004b41267fd32ed,0x000fc7acadcf6828,0x00003422f0907951}, {0x00024f40807e199e,0x000042ad4490562b,0x000b2b1b4fe9ce5d,0x000d0ce2f51b100d,0x0000b361400c4541}}, + {{0x0004a052680813be,0x000c761b08d6bd2c,0x000205558527aa55,0x000bebc9f8a40ea7,0x00003eea570043d0}, {0x0003817a0ff58b3a,0x00047a69e6277b85,0x00069b5d6b67d3f6,0x0003ec6b76bbb9a8,0x00003afeb82f4672}}, + {{0x000416d3e5548920,0x000d430e2a455f24,0x00032a2a08413b53,0x0007b199c56aee90,0x00009432bf6eec36}, {0x00050c6daf0ecc11,0x00054bc920485528,0x00081130749ebce5,0x000597cfb66ba654,0x0000b84f7977f298}}, + {{0x00004818d1d7a0dc,0x00027a6fa5567959,0x0009e5d35d9fabe0,0x000576e40f9c59ba,0x0000b1771c2b6247}, {0x00047cae9a6312bd,0x000f852dd8c5542a,0x000794716a34b355,0x0000942df94de00d,0x000046124aa6c623}}, + {{0x000435d68afe8b4d,0x000f9c0d8ea156b7,0x00018689827f2053,0x00090e42b77e1473,0x0000bc3dd4744794}, {0x0009842c03b0c057,0x00030921bc96951a,0x000202e0a8b1b3bb,0x000d563573b3462b,0x00007e4665db7254}}, + {{0x0000dfcd23e39845,0x000c9bd1423608b7,0x000114ba7ab86e8b,0x0004f25a3e07f857,0x0000ac71689fb0ef}, {0x000a3840139d9af1,0x000866644af088fc,0x000d74f4a72733f8,0x000c7ae122f72a65,0x00003931577b5626}}, + {{0x000d9eb70f8d5a4d,0x000707bbb228d5b5,0x0001c0b32375adde,0x000ba961e88b860c,0x00001f568c4e73ed}, {0x000fc835459df02e,0x000a2fcd9a7e1592,0x000473b0a2beac0f,0x000c47d0a6fdb81b,0x00003224c6fefe8f}}, + {{0x000d00ee87edf5b7,0x0001b0e77cf5680b,0x00042d1b230385f0,0x000d7779ab98c04d,0x00002d191d343816}, {0x000daca0917d9e55,0x0008cf8fed7f1564,0x000bb3896394eab5,0x00098e5209aa8d7f,0x0000564f3ba0e6ac}}, + {{0x0001d05d73654ef8,0x000393d78d74ead2,0x0004973a068d1a9c,0x000329e1e017086d,0x00003da3500c6e6d}, {0x000fca468ae01189,0x0009402da0696a3d,0x000ab8302a1b9a4c,0x0004357b2ff9c7eb,0x00008af07c4244ba}}, + {{0x0007326995f0f9ff,0x000f81b58bc68599,0x000625a2b467fadd,0x0008cc57e4495abd,0x0000dd2d01e23c3b}, {0x000ae28c693f9fa1,0x0009248f79992c38,0x00061f5834862232,0x000cc987bf738e21,0x00005ee2fa7665e8}}, + {{0x000c8455777e1891,0x0000356f2829a1a5,0x000762bd5cc10bee,0x000da87ad95c56da,0x000052e2214f9d91}, {0x0000e727cb23c74a,0x0000090c66df975b,0x0005ffc53fd5d767,0x0002ae15b5b8ad22,0x0000b6dff749aded}}, + {{0x00067816f4cbe9d9,0x0008da574bd7ebd5,0x000a881fa0ed8b24,0x000c6fb1c246fe81,0x0000156480653db9}, {0x0002b085b8628093,0x000125858d7bd7c1,0x00009e92a1facd1f,0x00025f4693747caf,0x0000b69dcba489a4}}, + {{0x0008e9f967365efa,0x00024801f5c90be2,0x00083352f57300eb,0x0002b6f3b8ac6ad5,0x00002cf1f8a6d05b}, {0x0009b744dcc40cc3,0x00057da523fb7c0c,0x00099cc4dfee38c4,0x0009c6849a4dec10,0x000025c377f99f06}}, + {{0x00058ce476cc9ff0,0x000cc6d4cb63e124,0x00072289b580e0b6,0x0006dad561c8b790,0x0000377f264a619e}, {0x000536288e591a5c,0x000cb523ca2b2668,0x000df4533a453a7b,0x000f78ca9536d2c1,0x00008e50f307e972}}, + {{0x000e50f6d3549cf8,0x000f7acd665ed433,0x00011fcb46f33696,0x00085fe95bfdacce,0x000010ee2532f7c9}, {0x0000fe17159bb2cd,0x000da58b357b6545,0x0009fea72f7dfbeb,0x0007445b057e74d6,0x0000485717b62731}}, +}, +{/* digit=23 [{1,2,3,..,}]*([2^161]*G) */ + {{0x00042e8ee36860ce,0x000c6113c22d896c,0x000104213daf04df,0x0004e93adbb7b744,0x00005fd5fa1ffd39}, {0x0005d941a4e0551e,0x000d38d101516823,0x0009845236772cfb,0x000a97476071e309,0x00004e879df3a56b}}, + {{0x000afb0285b94916,0x0007be4c705eaaaf,0x000d9caab01a0be8,0x00033f9f1d4f5d2a,0x0000e349a4b237a2}, {0x00012464a1c6a163,0x0005f9383260cf1c,0x0006d5471d99e6b6,0x00089bba3d43665f,0x00006974d052f8cc}}, + {{0x00049a1cfe89d80f,0x000cea9c8371c26c,0x000d066d2b42c026,0x0003edda6c013ada,0x0000b8f722946a4f}, {0x00079ecd850935b3,0x000ca631e1b308b5,0x00019853434c1a74,0x000f259b5fe596ac,0x00009ff21f711f24}}, + {{0x0009e148f9290579,0x000630c853df27f2,0x000e9c5ce7a64ae0,0x0002a4956cd18358,0x0000d9cce836ed09}, {0x00059796e93b7c7b,0x000181bb9e27cc6e,0x0009e29a0e1e4709,0x000644070b3083aa,0x0000f181a75e785e}}, + {{0x0002c658ead09f79,0x00050780d14df53f,0x0001b66bc1335e1d,0x000fc7d9cc20e0cd,0x0000b670a384be0b}, {0x000dc8128efbeedb,0x000bd326a6e5ce53,0x0008e9a630c74e77,0x0002478604e0d2b8,0x0000ab38fcac3dc2}}, + {{0x000e8c85c0a3f1ee,0x00019c87c37f8ed6,0x000e3b78dbcad249,0x000a461dfb62bb9e,0x0000ba8e478abceb}, {0x0008cb0eeaede4b8,0x0007f976deb637d3,0x0006147fb0bc498e,0x00060932944c046b,0x0000b123f36771f9}}, + {{0x000dcc7de79dc241,0x0002458f69cda155,0x0001850dff1168a3,0x000848aac215950d,0x00005c8295bc204c}, {0x000aa367d8184ffe,0x000d50447bdbf661,0x000e4a59ec396228,0x0005e531cd5143bd,0x00003a26e3c4beab}}, + {{0x000a13f1402b9d0c,0x00026c7bc863d3b3,0x0008c3e6e573441c,0x00057d8b301ec457,0x000026fc9c4cadaf}, {0x0001bfd7493cea35,0x000ecaf8145696e7,0x0008c608fd05d4b3,0x0002768aca2a8a6a,0x00003ef07f65725b}}, + {{0x000fbd27824fc56a,0x00077328907707a5,0x000c483493467521,0x000874bbf69fd5e0,0x0000613ddd456aa7}, {0x000c19c5450d8660,0x000c8f84a4817f78,0x000fce23946f4409,0x0004b99f1d192890,0x000016c4168b2ce4}}, + {{0x00023f0c74359787,0x0007b0e30e19bae0,0x000fa6fafb152c88,0x000e602c241645e3,0x000035d95c1f4823}, {0x0007573039553173,0x0009c03b49950319,0x0002746000b4b02a,0x000507d76bf55970,0x00002c5cc53daf57}}, + {{0x0006d1f606241297,0x00042a5e2b5ee8af,0x000082d72b7bc5d6,0x000779c814b0485f,0x00006f267f33e196}, {0x000630fb36eed930,0x00073bf56803626c,0x0002736a055230cd,0x0005f178837949ce,0x0000d792d60aa6c5}}, + {{0x000dbfdd5c7c5d2f,0x0006172b342d0318,0x0008de38ab38f8da,0x0008414569bddc7b,0x000025b588891c94}, {0x000b2842946ad608,0x000d69d1707eb2d5,0x0006a4509854f29a,0x0008372a5159dc2c,0x000099f94c0d7189}}, + {{0x000dc51f4a55b03c,0x000d75e3b2d5cf6a,0x000827b51261762d,0x0004418cc4301204,0x0000d22a11486021}, {0x000d61a247c9569c,0x00031152becace2f,0x000a716d459a5097,0x000dceac835a1163,0x000026455edd87de}}, + {{0x00036e049ce89e7e,0x0008ec890cb527f5,0x0003c2aa11890853,0x000bd2508909abd8,0x0000cd3142bfab73}, {0x000bf59b3f5ab84e,0x000812bea4c66a85,0x000a4541f3c320a6,0x000185cd8dc5386d,0x0000af34eb197c41}}, + {{0x0000129977c97c4f,0x000ad57eb9fa1c78,0x00022c4785ff9bee,0x000414b24d0524c8,0x0000d8eec2b361cd}, {0x000194ef027458c3,0x0008ed1be115fbde,0x00066d6f4b4ff531,0x0000c933f874d948,0x00005c75015e21ad}}, + {{0x000c9d646ac49d20,0x000b83137aa9a6b5,0x000225a3842c77c0,0x00090724d000fc68,0x0000f63cfc82fe1e}, {0x000b01bc6441f959,0x00095c8e448f22d1,0x0007fb1ba7d38f71,0x0008df0b33fa5f78,0x00004dcfda1a9015}}, + {{0x000b3395f6d4a09f,0x000bfe52b826c47c,0x0001b930a6b4f355,0x000f684d100f5df5,0x00004512fad8f668}, {0x00081d5206c4c74f,0x0003db4d2e485467,0x000085c2dd021d4d,0x0000a7594a54c2ca,0x00001dbaca542085}}, + {{0x0009326490a1aca3,0x000c11526b0263c7,0x000979258cb64dd9,0x0008468b772591a2,0x0000f58297078d97}, {0x00070d17c213ba70,0x0005e8a0ced4d66b,0x0000338c1c28febb,0x0006f36b911831c1,0x0000d54e389bf012}}, + {{0x000d4604af206ee4,0x000637e97cb97048,0x00064802e786c88f,0x00011c94375ae1ac,0x000069bcfe2153ec}, {0x000340d470622302,0x0007a5b4a3acfc9b,0x000ef45ace743bb5,0x000188de00b4aa59,0x00009a4ef2379edf}}, + {{0x0002efeb483689bc,0x0005913ac2624024,0x000a6db722575d3f,0x000be2330037c80c,0x00009fcce8358864}, {0x00012ff0149362d4,0x0001dc4ae97184a1,0x0005cf86c95e5758,0x000a2edfa4b1a894,0x0000525a7344b024}}, + {{0x0008b628f3383609,0x000298edf32be76c,0x0008b1aec483ff59,0x000a20d7e8e90a29,0x0000caab339036d9}, {0x000d2fd668927090,0x000cb55a1d415c09,0x0004a43942496b4d,0x0006c193f5fb1ae2,0x00008c750496fa8f}}, + {{0x000d1c2c905d85f7,0x000f9733ae57caea,0x0007cdd94e9d7f78,0x000930b4c9a65cf0,0x0000389359d14b55}, {0x00009b7367e45f70,0x0007cb7e7adcf587,0x000b728181f20306,0x00003382444bffc7,0x00007303b35baac8}}, + {{0x000e4e4d13b7ea1a,0x000440e741801e1e,0x00070ef70e6489b2,0x00089405f2c6107e,0x000016554135dd10}, {0x000befb7af4194e1,0x000c7e89bd9c555e,0x000895856533c1c3,0x000c15635b9b5789,0x00005fb3cd2667f5}}, + {{0x000ed45526f09fd2,0x000c6128240a057f,0x0002bfd8de8a4f10,0x000a317332efc4ff,0x000014e77a0dd35a}, {0x0006d7314faa40ec,0x000b41e5f1863289,0x000a1813e767867e,0x00079509adf8f117,0x0000b6cda7914741}}, + {{0x0001b6d349d51aa0,0x000ee3c7b8e9b752,0x000a096dff56b5a9,0x000024c6f1e5c932,0x000083667c4a3635}, {0x000a13518087f2fd,0x000c0136e45d365e,0x000aec989f1b8eaa,0x000258f8a0e48473,0x000075a324be42c9}}, + {{0x000d00101dae185d,0x000acb7a94bcb7b4,0x000d8cb0b45434e0,0x00049e254339affb,0x0000cc4569fb98ef}, {0x000318a09a512993,0x000682b025d87789,0x000e8579281b4d20,0x000af7c64aa418fa,0x0000e50258fdcd7b}}, + {{0x0004cdb2996864b3,0x0008ef485fa4dce8,0x0004c6a5aa2e6708,0x000d39828b2bb653,0x00001a7ec6bf94b9}, {0x0007766d6bc20da4,0x000c467611901d21,0x0007010634acdb5e,0x00029b2872632873,0x0000d24ee7c6128c}}, + {{0x000ebd3a19fd8680,0x000cdb8ddd3bc072,0x00064d852612e481,0x0004abb4e1d7541a,0x00000ef95acc4c6c}, {0x000d2edaa0a6c469,0x0005b37747901536,0x0003fda106129408,0x0006f1c4af25e834,0x0000ff9d98e8d25d}}, + {{0x000af7c468b8835a,0x000ad30ecea70746,0x000cf4a81977a31c,0x00037a05096b80c2,0x0000a9868340458c}, {0x0009bf3a6bd9d340,0x000b33c5d8546af2,0x000133b5e6a62fe9,0x00084850e6c304b7,0x00004b601597d6e6}}, + {{0x00096df5579bea49,0x0007cceedaf14cd2,0x000bcc5b110e35ac,0x000cf874c4c5fde3,0x00005f9ee8b19412}, {0x00059ee82b6eb0fd,0x0004c5c2aadd2c94,0x00027fcfe2e84576,0x000475a74a84aed3,0x00008c93722d368d}}, + {{0x0005748f83e8a3b0,0x00068d2495f30dbd,0x000496e9ba579aa9,0x000cc2535996a0ae,0x00007afbfe9b7f9b}, {0x000dc6d5b7bd2931,0x000f6022323d3ac1,0x0000a3e763b592cf,0x000acbaa0deb989c,0x00008e78e9f5b197}}, + {{0x000de10296c36eff,0x000192c4da77211c,0x0007836da7ee8967,0x00060ac617d270a5,0x00000cd9c328cb75}, {0x000cbf7e455fe908,0x000afe7334f301fd,0x0007de4ec3fb53cb,0x000fcff81e2ea44e,0x0000adab3ad8b384}}, + {{0x000ee2f53d648293,0x00077261492b129e,0x000cb4a2c7a471e1,0x000c2db4f9adb9e4,0x0000d359f6fc7ba2}, {0x00067860aacd6975,0x000325c2f8a8346c,0x0005df44e92b444c,0x000f31779fa117d8,0x00006782372898dd}}, + {{0x00090f2bbbab3b85,0x000e3b04816b60e6,0x00092e4d24851f8a,0x00036b772046ab9c,0x000018c74a1ccf31}, {0x000b50af9877d4ca,0x00000919cabbff4e,0x0005eb2b614578d9,0x0006e3e218f8c4ac,0x00003ccc547f4201}}, + {{0x000f48e327f83495,0x0006a43cb641025b,0x0000f1085f3e9734,0x0000558c2bafdf50,0x000071678767f063}, {0x00014b9411925a6b,0x0007f1123de55bd9,0x0002b165d7c078d4,0x0007273e6bf83518,0x000011b5e5c6a519}}, + {{0x000a76c1eea7b85f,0x0000a2d4f85ee33e,0x000e115bb2352b46,0x0005a30101d334af,0x0000abc129578917}, {0x000cdc05233f9251,0x000bd77fec557f6b,0x00069b659e0a802d,0x000d74adb47b7580,0x0000c5e12df098fb}}, + {{0x00058c64b8457ee6,0x0008ef7ea9f7869c,0x00060b38fa5360f6,0x000b368576c09ff4,0x0000b70d54882b7f}, {0x00037f13bfae3154,0x00028bdff3693fd2,0x000b516f93379785,0x000d2d57df25f525,0x00006f388f2fa38a}}, + {{0x000465889d8ddbbc,0x000db0f38ee8656c,0x0001212b08830b26,0x000db18320fd5cde,0x000034f30d0a4a2e}, {0x00031a356ab64b81,0x000cc99c5d26abb1,0x000981d947f77f0c,0x00076e56856a37bf,0x00009e76d09838bd}}, + {{0x0008ac396238f398,0x000e2830b366e76c,0x0004eb499c0a482b,0x00086537b8eaff0b,0x0000ecd83bccbfb4}, {0x0002cb7a2f3776f8,0x000474b88adf971b,0x0001fa446b42176a,0x000bd239617df5be,0x0000b32d5094d031}}, + {{0x000d47d53b618c0e,0x000b8a2279231c6b,0x00092d964c424f46,0x000bf19303ffdedd,0x0000971287951b5a}, {0x000a632f815561dd,0x000503c055d18f48,0x00025684f85f48ff,0x000cc2522a142775,0x00000d841a137360}}, + {{0x000a9260b9267c6a,0x00012f07f8634245,0x0000d9e24c78913f,0x0000170a844c8e4d,0x000042ad522dd5f9}, {0x0001749a2c989d55,0x000f91f5e78ebd37,0x0001ea6da928292d,0x000528f93b383e0a,0x0000136fd8d63aee}}, + {{0x00044b1f2c34a996,0x00045f5855ac860c,0x000af37be3b00aca,0x000c084bf6aaa0fa,0x00005f436828a53e}, {0x0005801a11b12e13,0x000cb20ed4751d9a,0x00041e0d578a7ab2,0x0003e9dde1067e9a,0x00000473f5f60502}}, + {{0x000e09d169c7d97d,0x0003ffaef9cddd3a,0x000a448035cd5baa,0x0004dd8cd7440b65,0x0000c13966b17f36}, {0x0002be82b8357c1f,0x0004f9d57c2a077b,0x000ff363e0cb1b4c,0x0009ee8a4ceb3205,0x0000310fa4eba35a}}, + {{0x000b352f97f68c6f,0x000f1b02cf58dbb7,0x0001f96d90c773b4,0x000814fa2e48213c,0x0000fb357b1dee01}, {0x00024cde0f28039c,0x000986a3fbe4b9c9,0x00046db6c0b36c95,0x000afe5faaaea45e,0x0000ae575c3d928a}}, + {{0x0001302a70dab865,0x000921c58cfc7f67,0x000e0cb92fcbd12a,0x0005837bef9acfbe,0x000073da0ba48c1b}, {0x000fcfe0d41d5505,0x0002d155cffe4752,0x0005ae248e7eec0e,0x00044dbfc39fcb54,0x000022cb8d1d065f}}, + {{0x000962a70cbb96cb,0x000a0cd124a9263c,0x0002ae58de034362,0x0005074120db283c,0x00009a38d4aaef6d}, {0x0002a821ff140fd2,0x000000aee7e0b1fd,0x000251949bd162f3,0x000c3d2e17a5d4cb,0x0000aebcb836f7e1}}, + {{0x000b25f937b05271,0x0007db7d9997608e,0x000a53a29f42e1e4,0x000536dba699c4b8,0x0000f921c71f091b}, {0x0009e7b5b26bbd51,0x000d2b61a680cce2,0x0001f1c7e7a8ef5e,0x000ddad5ef8043ba,0x00006ea821728158}}, + {{0x0008a2b599ff0f94,0x00074104fc6b0177,0x000694ff368a923d,0x000f121bfa44dfda,0x0000f7199dc37667}, {0x0008ff6e46f2a79c,0x000d29f8131dc06d,0x000b4ce7c08b5dea,0x000c3d42519a59ab,0x00004f710bd742ae}}, + {{0x000b05778bde41ac,0x000ff4186b5a3d77,0x000c657416474bf7,0x000153448b3f6788,0x000064519dec3c7c}, {0x00038460edfcc4f9,0x0006b8f1aa6bdf07,0x000909f77319aa73,0x000feefb9f8a02ca,0x0000025813a0580b}}, + {{0x000d3cac0c227196,0x000469ca151ed8bf,0x000a1a69cc60209e,0x000f8f1a744ab5d9,0x0000de5048b74937}, {0x00038d8e115ac04c,0x000f5c6b16d21719,0x0008e94e77df7093,0x000093e6aeb6637f,0x0000130388eea2cf}}, + {{0x000be8477f54e6e0,0x000265d60fe51850,0x0009146d69f258a7,0x00030bff7ff0c06c,0x000039aaf90e63a8}, {0x0007a739460342f0,0x000c3f795f8a38f2,0x00081a97e4703148,0x0005941bb5467b96,0x00000931ba5ecaeb}}, + {{0x000719d786f337c0,0x0002e704397dcdb6,0x0005c2fefd9c01cd,0x0002230f4a3f2055,0x000004525097c0af}, {0x000804784db8e765,0x000a43c8aa0654a5,0x0009194223bacf1a,0x000a3fc1ca957cf7,0x00000641053c8cda}}, + {{0x00038749f7144aea,0x000ea3d4acfd7a30,0x000ddd3ef170c963,0x000ba7be14814958,0x00007bde5833e72d}, {0x000da8b6fa687508,0x0001d72e02490769,0x00019ad31fa64e53,0x0009cd7caadf9d26,0x00007882dab27b34}}, + {{0x000b7316c67a7751,0x000adfc5d0b19f6e,0x000b806b2cb10471,0x000e7ea433750ce1,0x00009c5714d67b1a}, {0x0008b7bed03fd3f8,0x000eb1bc194ec0dc,0x0006320b5dd03344,0x000d93266c52a78c,0x0000bc82ce450b6f}}, + {{0x0003501b35f1341b,0x000c75a43e42f8e1,0x000aeb85ce53156d,0x000eb523adf27e4d,0x000081d837a6bedd}, {0x000546e2e4358674,0x0004aba5dd601b0b,0x00010cb9d9020eb9,0x0001cef7d9116182,0x0000c596b319c91f}}, + {{0x000a90f0e0b040d2,0x000225ff897fb228,0x000fa6122baf02d8,0x0005570aac79e600,0x00004828817ae36f}, {0x0001d31113ec3567,0x000da5eff1f8b952,0x000d417159e48861,0x0001b7baa1d412e0,0x00001f86203c3f13}}, + {{0x000a8da3fd19408b,0x000b778d9d99f60d,0x000c51c904aa716d,0x00051b894531f7a8,0x0000560b0e9a59db}, {0x000c992fa34bdad6,0x00043cd4f8bda28f,0x000a9d0d3f024fa1,0x000b55fcf530f723,0x000015ca194428c9}}, + {{0x000483d6f73c51e3,0x0002ba0dc2dd6d2a,0x000b917ffa4cb241,0x00099e20663c411e,0x0000d3a74d01ade2}, {0x000990f4a7a92024,0x000967b15c3d29b3,0x000df9208a9bccf5,0x00092926a3ccdca5,0x00008027c1483f2f}}, + {{0x000377c40b557f08,0x00064d684660d385,0x000183a27e001c36,0x0003289b18ed6be2,0x000079738d8e3210}, {0x000c74bbda948826,0x00084684b299a687,0x0003b3724d1bbcc4,0x0009f84f6f111286,0x0000943d1b48c8ce}}, + {{0x000a3bb098cafb4c,0x000fa0d48cafe044,0x00031b84d27ed230,0x000ed6942b56753a,0x0000bf3dd51bcddb}, {0x0001f1641b1d830b,0x000d1b0c1e272503,0x000ae75dba7ec851,0x00011ffc1c8fe0b5,0x000024c7557b8c52}}, + {{0x00011dc1d4636c38,0x0005e81a993957f8,0x00081adb3f843652,0x0000d39f6bc6d99c,0x000040f8ac3db7d8}, {0x0009811f4387f1af,0x0002c5156880731a,0x000e688677c501cd,0x00011fb5ca4a07df,0x0000123d8f14fcea}}, + {{0x0000e71d607039eb,0x00051d3a45461fbb,0x0003240912b70e21,0x00019a82d2f01d53,0x0000796ff08c80ab}, {0x0007a863c57c4aa3,0x000f87c49a2732d8,0x000630d982aed9ca,0x0000a36fb35eac31,0x000038e8cdf8c3e2}}, + {{0x000618266cde8dbf,0x000f3d72fd3680f1,0x0006e50724e15997,0x000dc0e7b8f13b9b,0x000052139082b7b5}, {0x0001f1d8ce4396ea,0x000007ed21424d43,0x0001aaf6b37a1a68,0x000b661f375696d0,0x0000a1c0c55863aa}}, + {{0x000368b4ed80940f,0x00058a6fcedd3014,0x00097579f67e6d05,0x0007f58c208c49ca,0x0000e3d7a8292359}, {0x00032027e096ae27,0x0006b4b393665e20,0x000dcdffcb1f3e1e,0x000e82b6da26f32f,0x00009422f1dd097b}}, +}, +{/* digit=24 [{1,2,3,..,}]*([2^168]*G) */ + {{0x0002cfb9db3b3818,0x000e54df0a4b263a,0x00004e61f9c3a2de,0x000324f28d06e97d,0x0000b1adfbcc2449}, {0x000d9397e053a1bd,0x000696daf7076ec1,0x0000ac7abee2be5c,0x000173b0ba1e1481,0x0000d2ae779c530f}}, + {{0x000d97a205b9d8b0,0x0004056756d40435,0x000f8210e6eb8f06,0x0009ead5e88a8bb6,0x000070ef12dec9fd}, {0x00095053bcc876ae,0x0007c7404ce34d84,0x000a1db5e12a7533,0x0005acf22b49e1b8,0x0000c1f2051f4bfa}}, + {{0x000eb79b6828f364,0x0007c1bd5b9eadba,0x000844b0c9d7a025,0x000fc9ada01e0d1e,0x0000b625175c87ed}, {0x0009fdd9669b6210,0x0006f6f87b981410,0x0000df6bc88a2ca5,0x0003f9fe2eb78817,0x0000cea06f4ffa47}}, + {{0x00081b5c4e83d33c,0x00089efd488b43ed,0x000eb4d0fd9f3587,0x000393564a620f9d,0x00006927bdc6c6a7}, {0x0008df79f9e0f036,0x000e9cd7e1a945c2,0x000a348f12868661,0x0008e01cf4e8d0ff,0x0000bd4c28499853}}, + {{0x000a091289a8619a,0x000fc671b1732618,0x00090c632ef796e5,0x0008fac64e46e590,0x000038062d4be66f}, {0x0004a200573274eb,0x0003f92713946c74,0x000dc0e20d07b67e,0x0005a6891223b26b,0x0000e2d93f29b0a0}}, + {{0x0002e533f36d1411,0x00043dfca442f23e,0x0007c023ae84bb3d,0x000c3ba804a48d6b,0x0000e16a8fa86431}, {0x00052adddd472e03,0x0006dd1ee1271b54,0x000a275997d405ee,0x000b3520fc6f1dff,0x000051ac53cef391}}, + {{0x00014b84444896b8,0x000f794027fb7efa,0x00084487d64974d2,0x00089b6fdcd0e8de,0x0000c45b260ab489}, {0x000bbc2d84634875,0x0006efbc476ca8fc,0x000f443c0d1b2b3f,0x00039bd1d005b7c8,0x000018f2e6790c01}}, + {{0x0006e8c06d75fc1f,0x00064249a89f5603,0x00045e7dd2dcf7bb,0x0002a691dd1d3de2,0x0000578dc4cdbd6e}, {0x0008903df2ce7a06,0x00083c39afac4c02,0x0006404abaee3628,0x0008187c847c3114,0x0000304c0d904e97}}, + {{0x000dca2a91f6791e,0x000fbbaa9efcae51,0x0009c7ac12abe418,0x000739f9d2e2f455,0x000082f4b52dc9f7}, {0x00030274073e81c2,0x000cdbb596fca771,0x00084f70cc0276fa,0x0001dffd819fc9a6,0x00009b47fdde9f7b}}, + {{0x000e103459b19402,0x00093b013e93358d,0x000532ad3ec881c5,0x0006de31574c9349,0x0000db1d445d37b4}, {0x0005b87df239fd84,0x0004d51d24eec644,0x0003c6259c718af7,0x0002f76ea1c4a4f4,0x00000c0e5d7b0be0}}, + {{0x00090f4721b33f2e,0x000b1edf04ea6a45,0x00045efe72124f1f,0x0006d918e53cde97,0x00007e1043345f04}, {0x000a28ee4d0c7e6c,0x0009c7253b1bc3fc,0x00043e643847e339,0x000fc4db59534837,0x0000b6a0a0c0fd12}}, + {{0x00036c327d02dccc,0x0001ba68bcc2fb68,0x0005e912d5ad0098,0x000cfd5b24b44c00,0x0000c83d210411fd}, {0x0007ec1666fba0ca,0x0006747546353652,0x0006da9c26994819,0x0002b25cdcb1a855,0x0000593426821a73}}, + {{0x00014eda714181da,0x000ac067b341ec12,0x00045df1f609ac13,0x00076b5f4b4c97a5,0x00001240501d4d20}, {0x0000c231409ca976,0x0007c0638c436efa,0x000fb46cd254cc1a,0x000a2774e363afdc,0x00002c2adc363942}}, + {{0x0009df056e464833,0x000033736356c67b,0x00051bc52a55abb2,0x00064b7b93c098c5,0x000082b49f9e15fe}, {0x00021ad4dff8d47e,0x0004637df4d69ec2,0x00045650979caf61,0x0007137f13dc64bb,0x00004c589d9f91f0}}, + {{0x000a8ab3fd40e095,0x000e27313ea927b6,0x00055988be455842,0x000bfc3b51d1e21f,0x0000716dd73562bb}, {0x00011e54e8bf3de2,0x0005fb85be3b633c,0x00011cca69a0e77b,0x00090f3651072909,0x00007e764960fa65}}, + {{0x0008b33070d3aab6,0x000b3a2cd5e5e4ac,0x000fc91732643672,0x00013ef2eff79b1c,0x000065ca49bf0a7c}, {0x000da59b3efb9983,0x000cd52f13415a8d,0x000f9a5308a5b922,0x0004d77e9ebbab3c,0x00005986e7c256da}}, + {{0x0006b5cff3513cc7,0x0009c198f7dd3a63,0x000f16f86bb0cf8b,0x0007bf48d4052241,0x000060575d94e13a}, {0x0004e169f7aa1811,0x000fe509ed1c36f7,0x00040a491163a3ec,0x000caa5aead61f3c,0x000058c95fcefe8f}}, + {{0x0001b6e13cda46f1,0x0005242faed0a399,0x0006b59707948241,0x0006dde3ba5bde66,0x0000d52e6bcc26ab}, {0x000a1e78608dd3d1,0x000ada076586768b,0x000dc1afa4930db2,0x000817c9575714e7,0x0000fc7bf7e07c58}}, + {{0x000accdd9eee96cd,0x000b158cec376b47,0x00002c42a0ca277f,0x000e50413fe413e7,0x0000d1764ef947cb}, {0x0007cde7b3ed7397,0x0008ece9e1c0041e,0x00025b21250cb745,0x00081bc556851329,0x0000cff95c4701b0}}, + {{0x0004cbd8088b454e,0x0006ba9e0c8a63ee,0x0002447cbdb7f32f,0x00019ad377d4186b,0x00003e982abb3702}, {0x000c1e4c2a2a5938,0x00048773f24f06cc,0x00085942372c3686,0x000c8f213b4da795,0x0000bbf1d33f5040}}, + {{0x0000973da50c991b,0x000ad22d6ee2726f,0x000fd777148afcd5,0x0007a135fc718b20,0x00009e8e77ead080}, {0x0000f4499a7703de,0x000d818e36f37f5e,0x000807bbe6972930,0x000f4f4b7c77b823,0x00005b82406ab27f}}, + {{0x0008be3bd3790620,0x000d2dce4a92ba8b,0x000952e37d64b7a1,0x000eca040a73c5b2,0x0000a9e252ed438a}, {0x000956bc39d3bcbd,0x000fe32b2d63dd43,0x000417a181a31c9f,0x0002c8067133b85c,0x000008e4790fef44}}, + {{0x0001ae9255c09801,0x00011b4a739f98cb,0x0004a45a14bd8638,0x000b2f4a5c31e11e,0x0000e5d55feacb0d}, {0x0001b068ff5cc292,0x000eeeb8a4f47466,0x000848c24026b389,0x0002b0336b21a458,0x0000e5bf8eca1dc7}}, + {{0x00087d0ad886aac2,0x000a9771b64503c1,0x0004045ab5c16878,0x000aed907dfc6fc7,0x0000c6360bf9800c}, {0x0005bb5b9c972a3d,0x000dac9a6dba2429,0x000a79aa6c9e6f88,0x000ac1c0ffbf2492,0x0000e29d50b11c26}}, + {{0x000f483d309cbe6a,0x000a40bced4f9f0a,0x0008023e35b020d8,0x000932c06e986db3,0x0000d8f2c9dbabc6}, {0x0002e1de7400e938,0x0008d2be5e4d1929,0x0000680bffe3e18a,0x00006368e9771d2e,0x0000c5bec99454db}}, + {{0x000662a74a55d1f0,0x000f046f66d82af9,0x000dc4794e3fbf28,0x000dd8a3a72ab4d4,0x00009779f455c7c2}, {0x000bdafc3d19d8d3,0x000427d6a6dfd893,0x0002e6255d5a7509,0x000aff5cf8fef995,0x0000da67cfc0a9a8}}, + {{0x000f62a2c160dcd0,0x00038f90eaef4c23,0x000a65d5a34e6c5e,0x000a3d35865519a9,0x00007c48aae8fd38}, {0x000aeda50068527a,0x00027c90936ab7e7,0x00079324c2c09ef2,0x00093791ecfeb6e8,0x00000871f6c9b0ec}}, + {{0x000fb68d84d835d2,0x0009661dc1e6b1f0,0x00094f8d7c90caf3,0x000b91f2e5b04675,0x00006897ae285012}, {0x0008a08a4d6755dd,0x000b3991fbdabcf6,0x000bf17e8403ee41,0x000d64a33e343e3b,0x00002c7980e379b3}}, + {{0x0006232d2e11305f,0x000203c07a6f3305,0x00015509d966be49,0x0009a37a8878ffbb,0x0000f221101fa9b5}, {0x000564aabe30129b,0x000f836e64cf6c9f,0x0000c8022c6f2c93,0x000b865fe752628b,0x0000e0267ea1ae8d}}, + {{0x00092f193bc042b7,0x00044237c45822e1,0x0002c4168f085b53,0x00071d30d192bd83,0x0000a76e9e42df62}, {0x00082fab88911b55,0x000464db0eb552a8,0x000a7c3ffc85345e,0x00046953be02a681,0x00001889c8d40ec0}}, + {{0x0001369a5e829e5c,0x000b5607aa419d03,0x0001d84c1cbb4c6f,0x0000edb5ac59a624,0x0000043f2c04829e}, {0x0008f758ea5e1853,0x0009a87cbd9f82a3,0x0008fc6018bda40b,0x0000b36e65e75e2d,0x0000d515f74d3569}}, + {{0x000cf4fda79e5acb,0x0009d630215f534a,0x00085756e68b83b3,0x000cb1ac748b2ed0,0x0000031725995d37}, {0x000841ac5ccc2c46,0x000add9d50696735,0x0001754bd7d7dc96,0x000dd54147e410fd,0x00005296e953399d}}, + {{0x000b2d0bc8fa5bc5,0x0007000c277bf6b5,0x000a08a5d8a5ead6,0x000046d14625e6df,0x00001fdfedce59cf}, {0x000430b289fca32e,0x000ebd9bdc3f6bc9,0x000ea0edee36ff0c,0x000b3e4fe187cb58,0x0000d66af213a900}}, + {{0x000968b5fa9f4d67,0x000dc7a362e700e0,0x00007e7722d4066c,0x0001d0399a9748bd,0x000010989c076a4f}, {0x000df35ce40cbd89,0x000f8743293dd5de,0x000a24e2cab55c5e,0x000cb3e66f11448a,0x00004d874f8f05fb}}, + {{0x000f0e8a518001b4,0x00065d04ef0fa365,0x0008d4d25ee605eb,0x0004721a3915cdba,0x00004c0e1b8f5113}, {0x00024e88b6740dc8,0x00036e1d4f0ccbb0,0x000c4e37289087a5,0x000af2288fa05c1f,0x0000bf395cc2f8b3}}, + {{0x000c9a1deb8568b6,0x000020fb3d321e71,0x000f8fb81a35daea,0x00096a88b6f2662c,0x0000d51afe8f4906}, {0x000ac6e51803a198,0x000f0621908081be,0x0006f463ce3d24b7,0x000ee7f27cfd9ddf,0x0000c6865caf2284}}, + {{0x0008b7db743f4ef2,0x000bc7d11dce32c8,0x000f2ebe83793909,0x000796b398f9222f,0x0000c70ca4505e49}, {0x0009929cb1131b10,0x000825888e79df4d,0x000d8740a7826f29,0x000a8b4d3a112cf1,0x00000384cb6270af}}, + {{0x000125b3ab480953,0x000632d05106cb64,0x0009558453451c25,0x000433a73d577da4,0x00009570c16ef9f4}, {0x000aad3adecf263d,0x00010c76e102d7df,0x000c6a836f1c3d8d,0x00047a8e774a5854,0x0000ad4b6730e92d}}, + {{0x000990ff0d796a03,0x0008af0e8b02be7e,0x0000c00ad5fc6247,0x000a0f5aae8bf403,0x0000d2db93bc004b}, {0x0008a79d85d5ddc0,0x00076bb07f34e48c,0x0009eaed5e907caa,0x00072458db343aa3,0x0000ea6e007adaf5}}, + {{0x000f169d23233f3a,0x0005d7cb637fe00d,0x000a0cf6c3e32279,0x000bdc7f897c0e1d,0x0000651f5d8d1d6b}, {0x000af191a230c767,0x00025daa5e4add61,0x000abcd7ebd52727,0x0008dc5a753636d0,0x00008bdd37ca70bd}}, + {{0x00016c217cd93fe1,0x000dcadce6e2c239,0x0004e42f865b97a4,0x00080ad04ed4eb17,0x0000491ccaacb214}, {0x0008280231963324,0x0007187b479a145a,0x000dcd0ed3c3862d,0x000f1f5f4a88a301,0x0000da2b7ef7ea12}}, + {{0x000ae33b126e48e4,0x0002b494e237f8e7,0x0005acadb404a0b3,0x000fd95beac474c5,0x0000ee5cf3c0bec9}, {0x00033b97df3c8c32,0x0003976808fd336b,0x00045c16abd905fe,0x000b626f436981aa,0x000055c5bfa5dd27}}, + {{0x0005cbfc3dd9b4d7,0x000f8c068a877196,0x0005b029bce23edb,0x0009bd478d472574,0x000046107143efdd}, {0x000f75f1266bf52e,0x000138e49bb67116,0x0006f19e30204672,0x000b2eff43df9f3d,0x0000f1bc7d1c685c}}, + {{0x00027c17078c4322,0x000c977fedb7cddb,0x000290570e1961b9,0x000885fedc2f5cc2,0x0000c3fefca39cbd}, {0x0000a36c2af389a8,0x000d3da71ceacf88,0x000aa846396c610f,0x00090a703977a932,0x0000eb776400586d}}, + {{0x00024542a296e775,0x0002f2837a353f34,0x00009c731c871868,0x0006db2dc710906a,0x00004778ffba1b81}, {0x000bfecaf06defd3,0x000f5592b70b6b33,0x000da6114fe3c105,0x000ad7c937fda461,0x0000c13e6517c266}}, + {{0x000a829855938e83,0x000e6de54b72e363,0x000ccfab92eeb5d9,0x000a258eb93b0e20,0x0000dffbb5f55e61}, {0x0005e431acc093d5,0x000ce964ce617f65,0x000e9b4600cb6cc3,0x00072d1ab283a1e5,0x00005d787c5f1c7e}}, + {{0x000fd47deadbf02f,0x0008bc4590684d2e,0x000f311f011e8021,0x000f52910c762671,0x0000a17ef8e3ab6e}, {0x000fd2593e43bff7,0x000e9be40632af47,0x000e61da35cb5ff3,0x000d0ee46871068e,0x0000764196f208af}}, + {{0x000b3edf0290a8fd,0x0005fb47c387831a,0x0004efb4fcae8196,0x00010ddad7dece18,0x0000cfc53b417491}, {0x000f23c4cb632f9e,0x0005d91f80676698,0x00084180ac42a1ad,0x00026ed116a81d62,0x0000bedf5f9c9013}}, + {{0x0004c9f97e3e0442,0x000ff1d09fc9f227,0x0008e6e234201851,0x00083c36a65f17d1,0x0000ea61e2a552b6}, {0x00091bc575eaa945,0x000168ff522d27d2,0x000f04d6f9e7bc72,0x0007480f7268bfa7,0x0000868c73ffba41}}, + {{0x000c2db7be0eead0,0x0002ff7191359f85,0x000ea90d7511e784,0x0006315a06b1e9c5,0x0000c19e28326fab}, {0x000f0cfe9206c550,0x00043553c06a8af8,0x0005f800489389cb,0x00091d39dbed97f6,0x0000621b037c5089}}, + {{0x000e63596e78cc4c,0x00014c06b4a81c52,0x000e87d035385c8b,0x000fad184ddfdbb0,0x000049dfb67534ba}, {0x000e17059f707727,0x0003ca1db56b7071,0x0008af1903a073a8,0x0000ef934949033b,0x0000d882de443292}}, + {{0x0003f0ab2cf89408,0x0007ef948f519163,0x0002653c872b0b17,0x000a04ad28dc3078,0x0000882984a5b903}, {0x0005d0c6a19d2bbd,0x000bb6f782cbb809,0x0009070644b9e7f0,0x00043baaf739882d,0x000012be0ff5b326}}, + {{0x000d23d0e165dc33,0x00021e2378ce358e,0x000b8a0873d47ce6,0x0000b94e2bb0b9fe,0x0000246e8af129e1}, {0x0004ec703ce2b4df,0x000acbc077cf459f,0x0009940c1e9b4ca1,0x000eb03613b4f20e,0x0000c598bb9f47d1}}, + {{0x000c62b45036099d,0x0001467c65d89744,0x000be1943a9dee74,0x0006233c511525da,0x0000a11055563c6c}, {0x000a52c651a3be27,0x000d184449a6ae00,0x00033bed1cda5111,0x0006b3963c06f4ff,0x00003baaf9a7d3d7}}, + {{0x0000c9d7fc63668b,0x000c5c039cde52fb,0x000b223516886c9d,0x000c12b02bd59955,0x000000cab02e60c7}, {0x00016bc81b69442a,0x0000155c3cee8cb6,0x0009ba2784148670,0x00070fd1093281f4,0x0000d956d9d04a50}}, + {{0x000f28b638a7e819,0x000ec980ddc39561,0x0006f247a54155cd,0x00010022db4a96d2,0x0000d774e4ed787d}, {0x0006e2e078637d27,0x000cee0ae06a1a9e,0x000cfa3541c363e2,0x00098d0493483ee9,0x00006843cb3ef74b}}, + {{0x0006591d4b66947f,0x000714460a8cbaca,0x000768f55b452ce9,0x0002de7830d24643,0x00004197ed96dff1}, {0x000b472400dd0f70,0x000f4b1e70936521,0x0000338ae59f5ca8,0x0003c66feff11b08,0x0000ada31f6a29ca}}, + {{0x0004eb694a2c2152,0x000ae5a57ab42479,0x0006f89fed83a43a,0x0007c2064a543a2a,0x0000c2a3868fd5ec}, {0x00039408439d9b28,0x00018acd1f11d337,0x000e6cc19715ea67,0x00085bc2c1d235e7,0x00001ce6e9739905}}, + {{0x000dfe0d809c7bd0,0x000c8f1050ab04e5,0x000a4176fd7b2580,0x00097c6d91ad78d8,0x0000af556ee4e2e8}, {0x0008b73921de0acb,0x0001cea78400162a,0x000ce217452ac9c2,0x000f793e2a4eeaef,0x0000e61844f1d637}}, + {{0x000f1bc789a283b1,0x000d780836f40491,0x000e5402d72d3ac3,0x00073d9a1c5ea388,0x0000b192421e5cc4}, {0x00099989dc84cacd,0x000ccc6e75b85c0b,0x000191ce2b0a8482,0x00092f939961d03a,0x0000a3bc8663d837}}, + {{0x0000653056e6f8fb,0x0000b4d133a7ca99,0x0006abe4084861c4,0x000302db40327674,0x00007b4d51b9bf8e}, {0x0003211220a255d5,0x000bb2419e6e05b4,0x0000c2feac997152,0x000ade26ff47b663,0x00000518677781fd}}, + {{0x000b8bacf902b0bb,0x000487db303b3283,0x0005011bc8d4b4eb,0x00019b1c89f42d75,0x000043d74bc8d09d}, {0x0006bc98adba3503,0x000c851c19276574,0x000ad72ec364eaf8,0x000c1fe3c7659610,0x0000004512228d40}}, + {{0x00099b7ea7b979b7,0x000476fb3bcd6d2d,0x000cffbfecd78cd7,0x0004f5a1e45a9e86,0x00008a61cf4d3702}, {0x000c8723d502295f,0x0003558cb288d06b,0x0002f8586f137685,0x00009dc9db26a134,0x000033effd03beee}}, + {{0x000c4cdb30cfb3a6,0x00010c9db4c8d7e0,0x000c8d9df6d09b8c,0x00066ce0ba1a4207,0x0000fd495f77c52c}, {0x000169f275264daf,0x0005f57d8362fb0e,0x000ad722280c2b74,0x000c7afdd987f749,0x0000dc229b03398e}}, +}, +{/* digit=25 [{1,2,3,..,}]*([2^175]*G) */ + {{0x000ed8452666a58c,0x000026a9c3c2b0d1,0x0009064084bcb6e0,0x0003ff7c57411c26,0x0000fc20755d3556}, {0x0001c505294dba30,0x00068b7dd31ea08b,0x0001eca74a30ba28,0x0002b9d70ba90e99,0x000094e142ce762c}}, + {{0x000783e979f39254,0x000a6f4c89a7b81d,0x0001bf7fa1efd130,0x000a9e125c2144fd,0x0000b2969045b265}, {0x0009634b9db65b69,0x000173599d8aed8e,0x0003563f335c82e3,0x0008ab4aa7a54f40,0x0000df088ad922c3}}, + {{0x000b066bb3fd30a1,0x000adff0354ee5cf,0x00024e36c429169d,0x000e1d709cf85235,0x000036f4fb31155b}, {0x000af011fbba712e,0x0003706ba1a14826,0x000aea73e6ef0f0b,0x00044df9928b3177,0x00002bf6af33eaa2}}, + {{0x0004f124237b64b8,0x000963ecfd078d08,0x000845dd8688ebe9,0x000324d7b8a70cf6,0x000008fc59cdda4a}, {0x0002b2ba3585862b,0x00053df29386a903,0x0001ec29bb66825d,0x000dc805a5a8db43,0x0000b143a98ea1e8}}, + {{0x00094ce12ae381b9,0x000bf6ccda9035ee,0x00006eaca3a7f176,0x0004df363a657e46,0x0000ae5a380d3cd0}, {0x0008d15ed251b464,0x00008aca5e649bec,0x000f20f071f5d6d3,0x000285f47b3b359f,0x0000d65f03537e4b}}, + {{0x000ba24f111661eb,0x00040105eb049e93,0x00024b578edced48,0x000068e6dc9ba1f4,0x0000f8f66b8983e9}, {0x0004df4d7ed8216f,0x00069e2cbecf872d,0x000e73754bf07f37,0x0007075281d89998,0x0000ec85fbc7aab8}}, + {{0x0000deea5ba5b0b7,0x000dd2d052999a3c,0x000b02d42e6a116c,0x000cb63e9775fee9,0x00002b0520111545}, {0x0006f7d31a3b4ea6,0x00082bbd9b32bc50,0x000b12a97e589307,0x00067168bc5f37e4,0x0000b000c06aa73b}}, + {{0x000bf22765fa7d04,0x000fdd6a537013b5,0x00080db9859805be,0x000ce327a5e29d42,0x0000f53916fb76b1}, {0x000f61f33ddf6269,0x000e1085d103714f,0x000809ee34206238,0x000b1c8c50d4b7e5,0x000099f450e15f8e}}, + {{0x0006051e4c79e9bf,0x0002d66a9fea658a,0x000be7b231394cb7,0x0008fd37f31ed5c6,0x00004c88f374aa6f}, {0x000721f4aaa499e0,0x0005e3fb2a6b0fb0,0x00092851d68b3a7d,0x000913a788097d3a,0x000060e7f8ae96f4}}, + {{0x000be731a3a93bc2,0x0005821adc1a82ee,0x000030efd42bbf46,0x0007bba10b6fa4ef,0x000047aa4c7a7b09}, {0x000c632f60c77da5,0x000a7223523e8b8d,0x0004579cf6ffbc26,0x0000f654f6ff1134,0x0000825653ce8025}}, + {{0x000097ebc1aa2b92,0x000317a0333ab2dd,0x000a0db380788939,0x000612fcf55e7137,0x0000648487f992c1}, {0x00013363fcef2614,0x000cceabf129dad0,0x000276be26239c81,0x000ad34ee761de9d,0x000006a7a345eda6}}, + {{0x00067ba4a493b31c,0x0001dbf7f0264bf3,0x00095914b54f20a5,0x0006abf696e06297,0x0000ddab96e4bf23}, {0x000c70aed25ea138,0x000b01cbbbe74ff2,0x0008544c5fa1d09e,0x00031708fc8c8746,0x000047a670de96b3}}, + {{0x000421e64bcb626e,0x000746dee0b5f133,0x00010346caea638c,0x000ed2f6e7680bb3,0x000006f4098b5d4c}, {0x00014527512a30b9,0x000d5589a59a0996,0x000d0c180f3d867f,0x0004ab9e73254f52,0x0000063d8a3c33c7}}, + {{0x000c595d314e7bc2,0x000b267899ededa6,0x0001ed5d32ee7464,0x000612fcef423c0a,0x000017e76ea89cc7}, {0x000ce1fe7cda917f,0x000a9a893f1627cc,0x000c74f6b12d8016,0x000e60ccd6de849f,0x0000a5817e3e3144}}, + {{0x00041640821ee4c9,0x00037bc619921f35,0x00072879f1583eab,0x0007b1e490caf61d,0x000098ad9f4876ae}, {0x0001950a41157f70,0x0006e8da3a7e1e18,0x00026b95fa9d7e1e,0x000a10963784eb84,0x0000ee4ed6e542e2}}, + {{0x0004cc5ac751e7b7,0x00028d4211bdb79d,0x000de4fc693f9647,0x0000641c72d3d2c8,0x0000b69cbf64f44f}, {0x0000ca2f4bf94e19,0x0008612894e23da9,0x00017d60b1a5325f,0x000b5c7a437f6c79,0x0000be7048726c9c}}, + {{0x00080bfe1dc5c055,0x000a9ebeeb57b4d8,0x0000fe6a3d738add,0x000f5a1f0119d3df,0x0000c686e55b6eaa}, {0x0000b50dfd0b7ec5,0x000b1a497c219cb1,0x000546c96bdd0264,0x00042aac0935148c,0x00008a947fac9dbf}}, + {{0x0008d4e49ccd6d74,0x000c48bd5580c0b4,0x000d473b2ff8fb02,0x000af3875235e907,0x0000fab1ac5e2188}, {0x000a3bc97576ec06,0x0007ab7e7d2f030f,0x000305600e8c946e,0x0003b3e0a5c9cc70,0x0000d8260aa28b01}}, + {{0x000304f70bba85cb,0x0000f4a0d3110368,0x00015eec1ad090da,0x000a46c170e87024,0x0000fba35ff3461e}, {0x000019ac1e919380,0x00031afc415f6279,0x000ba0e0fa47638f,0x000c4836c65cbbbc,0x00002160efb034e2}}, + {{0x0001073615cd9e45,0x0007a1243c06e6c5,0x0007b3d8c498ec04,0x0005f0ee5a8809b1,0x0000cd99e615cc56}, {0x00012df7851dafeb,0x0009f79061e281e3,0x0000c590ef156f5b,0x000aa0d0d62b7188,0x0000ec9746fba39f}}, + {{0x000a9c1c8ed1f7a4,0x0005681d5ff21d98,0x000a0794a09e43bb,0x00083695f00f680d,0x000012050d9a61aa}, {0x0007c4e90747e405,0x000b662a3686a89f,0x0008e33536dc05eb,0x000bb98f4de84730,0x00003868fbbefb53}}, + {{0x000d2c3cfdcf7dd0,0x0003723fcab42b09,0x000f57ca341a9fce,0x00055573d905f707,0x000080f9fb1ac8e1}, {0x0008e849ba7a5317,0x00067d9a147f7c08,0x00048c33607d3558,0x000e78f02846abaf,0x0000320fd327ccf0}}, + {{0x0000798b18bd1ff8,0x00002fdd2905aa78,0x0004267cd52c2e30,0x000b5f727ea3d643,0x0000b96d16d95605}, {0x00010494b45706bd,0x000da43d25f87bb3,0x000f30076e7f58b8,0x000d5a19b5e45b87,0x000019448d72d053}}, + {{0x0008cb9d3210a041,0x0003cafb52691ecc,0x000c3489f6bc7d46,0x000e1b2e59b10a67,0x0000769788c75641}, {0x000b82dbd6cb838c,0x0005636d5f228a53,0x00008536e7066d6e,0x00080843aa1c6169,0x0000971da0e26ae9}}, + {{0x000a86bc49a2fac6,0x000fd092e77a01b3,0x0006fb5563b8420b,0x000a86a20573007d,0x0000941b2a21ff40}, {0x00063080658ff2ac,0x00027424ab36140b,0x00051e2998780436,0x000e394253bd5157,0x000075bcd77049c3}}, + {{0x00040907f8f875d6,0x000df6c26bbf92eb,0x00010bbe79c9d754,0x0001e9b58cea6181,0x00002a6b802d45f9}, {0x00041aac6e7394b3,0x00037d57ef10a79c,0x000a6f40c445b6a8,0x000364dc5277eb6e,0x000019fe96bb8633}}, + {{0x000c61f385f63cb6,0x0003c2bdd1270b0f,0x000e942c241250c8,0x000d5d07d153f109,0x00000920d092021a}, {0x0005746724d81a52,0x00023bba3299229f,0x000413032b7ffb89,0x00094c318c51a1de,0x0000a9bfe775c2fd}}, + {{0x000e2393191f4fd2,0x0006b3d6ada1cbcd,0x00076960643093e1,0x00025bf84579f358,0x0000c94a8b3f2366}, {0x000b9c05c437d8e4,0x000398d9f3c86922,0x0007090a23d4ae42,0x0005bdb72c31c12e,0x0000ac3f5f4176a5}}, + {{0x00008fc6b6af9917,0x000fab5cebbd3425,0x000440dd70d5270f,0x0002fd484740d0dd,0x000048ef841e8016}, {0x000fe0edfc6fafb1,0x0005e7300f27a8db,0x000ba4ec9eadfdf0,0x000a976d06555ffe,0x00002c56f83ae25f}}, + {{0x0004203d39b8c34b,0x00058125eddb77f8,0x000e39dc5ed8b1be,0x000789abbf2441f6,0x000000f6ee71a5d6}, {0x0006ecf57d0ea992,0x0007f7e06c43ba45,0x0005b4baadcae0f5,0x0009bde1643de40f,0x0000c324341f161b}}, + {{0x0007f55e126d4682,0x000f56748e098017,0x000a9bdc2ed325f1,0x0004684116004acf,0x0000d8607e65a9fb}, {0x0003e276009d660b,0x000ddd10c5a10e57,0x0009009a03a525d2,0x000448226cb45c3b,0x00006b0cdc18e9d7}}, + {{0x0009976e1337c262,0x000db73d68e5949c,0x000b768d96faadeb,0x0000697e158614f1,0x00002dfa557bcc4f}, {0x000da17be93c6d61,0x00019504f5b9ccd6,0x000694da124866c6,0x0008c61121353c8d,0x0000c6ca5801140b}}, + {{0x000ad8ce964021e7,0x0009932b82b3c245,0x000ef9898b83bffb,0x00048a8aa220c647,0x0000e8d3ac7082c9}, {0x0002091bc2d124af,0x000c15b15ff41faa,0x0007c6fb7bd54c3d,0x000f65486bf3abc8,0x0000b2b0564edeb6}}, + {{0x000c5575b45afb42,0x0009cfb8912d4e77,0x000f6e557e9ded64,0x000f005ec9bbf542,0x0000570dfff82671}, {0x000fb7888e084bd4,0x0008b37fe5b42b3b,0x000649aeea024b23,0x0001d804e7dc0495,0x000098ca2559e7ec}}, + {{0x00066eaaaa07e869,0x000b636085863bc7,0x000c259c80db6fac,0x00049f2add2549bd,0x00005af3c6e941c6}, {0x000928c02e30afbc,0x000c408a88b8b36a,0x0001d9e9d9b5356a,0x0008cd8b67a5f1cf,0x00006542e4865d8d}}, + {{0x0001fe87adfb6cc6,0x0003386781417306,0x00080515acc826fd,0x000082a0e758b13c,0x0000afe3247a1485}, {0x00008b9b6ae8a751,0x000d3acf51e10fcb,0x00061b9d6b8cf388,0x000d0c244a556069,0x0000a6778b87a97f}}, + {{0x000fdc1ecc4c7e38,0x000c96db68ccd840,0x000e216aade9fe47,0x0008be695f89dea3,0x00004f1a6a51594a}, {0x0007d725a7b162b9,0x00091dc817a37ddc,0x000b58d46c5cfda1,0x000f18f0a5d35078,0x00003365b1412978}}, + {{0x000d22526a1fc900,0x00024d70705d2e44,0x0000c45f40d6d10d,0x000079d94b6b10d7,0x0000f201022b216c}, {0x00066c5658fde413,0x000d4e27601dcec9,0x000230be7a8d2bc7,0x000fb58fcce3e1ff,0x0000394ff6b3033f}}, + {{0x000c5098132c9af4,0x000af61e7868d890,0x0002d15aaaac4b0e,0x000b7d1194ded3e8,0x0000550bd2e63ae6}, {0x000318eea5399d41,0x000a81638b803fda,0x0004aa12dd989bff,0x0009444ea124d0a1,0x0000fb1b8994667b}}, + {{0x000796944c44d6a8,0x0009d7e8613795ec,0x000adac4491df144,0x000801115fd62073,0x0000f01732dd9a83}, {0x0009d253aa0a6332,0x000ca9d6d59cec57,0x000ef801006de5e7,0x000a02a132f958b1,0x00009476f97065c1}}, + {{0x00077c0d34c35659,0x00018b9f1e9e336a,0x000e08002ef1105b,0x00009dd3e6d08bf9,0x0000aff2f2256138}, {0x0004f853a80e75d8,0x000debbda681b575,0x00097fd7ade71853,0x0007fa06f041df81,0x0000b332e0892781}}, + {{0x0009be8b9c20cda9,0x000545cd0c9805d9,0x000b9418389f7aad,0x0007f1eef936fe5b,0x00002ca0754405cd}, {0x000db1174a1e0359,0x000783eaea929d65,0x000e4fbf202628cc,0x0008b762d9e24249,0x00004fdfd9c7384f}}, + {{0x000605463428c6ba,0x0001f0b409a565f5,0x00045ae112f7205b,0x0003b25778bb78ff,0x000013045bf65ee5}, {0x00014ff03ef77fee,0x000f1fef8befe00a,0x0009ade22689cd59,0x0006a75578f0ed1e,0x000099f3ec14268b}}, + {{0x0007d91ea1b3c3ec,0x0002f8823a4aa205,0x000ca451e2d1a705,0x000b5cebbb336a2c,0x0000d2466e3e218b}, {0x000f42fc8cb762d2,0x000e5690211f3ac1,0x000d074507e312aa,0x00013fcbb9bd7345,0x000007c4b8266c22}}, + {{0x00025c1375913eca,0x0005c790822099d4,0x00067dbf694e45e9,0x000056208f3087cd,0x00005670fbf60887}, {0x000b64a66f5b8fca,0x0009d86fec286717,0x000ff4952d5a56ae,0x00049b08c3f55fc0,0x000077fefaee57ac}}, + {{0x0002d7c98379d445,0x000b009edc8a2988,0x0006fe464d000bdf,0x000de076f95979e6,0x000004a61164a61b}, {0x000b871effea31ae,0x000d10c21a5456b3,0x0004753bf2d3de26,0x000d8541dbff3183,0x000067ecf4a49269}}, + {{0x0006952151fe690f,0x00038f2adb5f7a17,0x000b62a8d0351580,0x0004e5fe794b15d1,0x0000004ceed9ae45}, {0x000ea7cf0386fac2,0x0002b1fca7510897,0x0007a04ec3b62ff1,0x0007ebf54181df1b,0x0000008e04b1b584}}, + {{0x000148e41dbd7725,0x0002d2942654d147,0x000c544f72b419f7,0x000149169f30d3e9,0x00002a2c22418540}, {0x000ee14634dfb02e,0x0000147869f35da9,0x000933acc5f074ff,0x000ed094ee878da3,0x000065106522fe35}}, + {{0x0009482f1012e7a7,0x000038a566aeb3eb,0x000c00d3b51013cc,0x0000e56d5e924347,0x0000fde089e046bb}, {0x00054fec731b4b3b,0x0003e9fda0620307,0x0001a35bc12a136a,0x0004eecc1064b85a,0x0000f1f5763f46c8}}, + {{0x000a56da16d4b346,0x00097ca21c4fed29,0x0008de4867fba9d0,0x0005e106d7ac006d,0x00000061987d3a2a}, {0x0000f869da28ff0c,0x000783c4599c8b40,0x00028cb0d3133f70,0x00061cd911c9b8ee,0x0000d7e28754e0af}}, + {{0x000f0f272ed91fcd,0x0000ccd4a3735a85,0x00025253c85214f3,0x000c75b81fe5be19,0x00008dc98e161e8b}, {0x000affe585cc3a25,0x000d235bf97a7120,0x000b34581724952e,0x00057d0581e7dc3e,0x0000cbff4f3352ee}}, + {{0x0000a0e87d8cc7b9,0x000361d280d08d32,0x000eec7049beaa7f,0x000056ea0b95719b,0x0000126332eeb7f0}, {0x000c1b48ed3bd6db,0x00022945eb2401fb,0x0008ae25535bb2c1,0x000fb2b404694e9a,0x00006092eed3d6ab}}, + {{0x000143fcc058865e,0x00018e2499224d76,0x00050d3537b0a5af,0x000079aaef94406a,0x000011e4bcd44f0e}, {0x000993aa14a90fa5,0x000c6a0c51d44472,0x00032672d7706e20,0x000382a403292f15,0x00002573bfa71829}}, + {{0x000b6a93b5bdb831,0x000094a723186a7b,0x000eb065f08da65c,0x000685b58d22aa63,0x0000717596c2b15d}, {0x000f0d0b266d88b4,0x00071941945a112d,0x000292cacf688ae9,0x00085c087386e37c,0x00002f3b50d97d69}}, + {{0x000f9986a90fc34e,0x000285ca8a8d6da4,0x00051f762c8f257d,0x0003abe2feabca69,0x00001bc81d154c32}, {0x0008f67251a2a12f,0x0006ce8a70dc1bc6,0x000f84d2e10d8658,0x000c91e648af7ff0,0x00000aa9ebd5a43a}}, + {{0x000be04275968936,0x0005e5bf452b69e3,0x000c698c8b6bb02a,0x000793a875c11af4,0x0000652b5c81ece3}, {0x00055fd4f5c04999,0x000825532b387b37,0x000e96ef76ea1655,0x000f4841c69889a2,0x0000c773c3af1ed8}}, + {{0x0003a409b323abc8,0x000170e1d7912b65,0x00087157ae26605e,0x000615c5d410644a,0x0000f9a78b84bbce}, {0x00044aac407eddd6,0x0008435b964fcf1e,0x00008399981ddd1d,0x000801e73e339efd,0x0000c94bddeee796}}, + {{0x0004ada8545d185d,0x0009a38bb8cb5a30,0x00087e10e82ae44e,0x000fe2928a35e3df,0x00003624f3e715b9}, {0x000209b14be42542,0x000c9dbc2ea5cc44,0x000c37bbe7d0efcb,0x00052bff60336204,0x00001f363f576a58}}, + {{0x0003d1ca85015501,0x0001c8ab10bba150,0x00061c51c2251e0e,0x000f68ce129c9669,0x0000f7246a491910}, {0x00044ee5f2591f26,0x000efe6271572eb7,0x000f3bd683c47d33,0x000855ed6d62c922,0x0000120a64c2b8df}}, + {{0x000c6c07b5d07df4,0x00083ef397833a9a,0x0003a9b4fa92b955,0x000f05a128a134ab,0x00001c18807f1252}, {0x000d08980ba9b1c1,0x000eb532a9ddfc7e,0x000246809ac8dc6d,0x00080faf829cef55,0x000001b784f6b4ee}}, + {{0x00045bbb6f116037,0x000dd1d2801ec099,0x0007534a857b09db,0x0006148ba5202fa9,0x0000fd8ae60317b9}, {0x000a66678308435e,0x000bf3868c4da50b,0x000d7aab09572f77,0x0009fe2cef7bfd2d,0x00007958e090c7c7}}, + {{0x0002e42253466896,0x0000507c70048126,0x000950ee3716da29,0x0001b4d5f911eab7,0x0000fd72969861d2}, {0x000980308b640d30,0x000e887f12a15238,0x0002e93115b0026e,0x000ff720e2166074,0x0000ef6d5415ff77}}, + {{0x00027f0f9c41135f,0x000878a649939691,0x00041875cf21d60c,0x000232756e5d0ce5,0x00001e0f84f91d3c}, {0x000a35906002d609,0x000b761915529bcc,0x000181ec3be2da60,0x000f18cda8bbae61,0x0000f04b823f5806}}, + {{0x0004a7dd4b79bb87,0x000ae2c878c8f160,0x00047b8e8aee806f,0x000053c4144f118d,0x00002edf52c049f9}, {0x000a84e2127015a3,0x00006cb7cef3ebfc,0x0006deec89051d0c,0x000d7456e8fe5829,0x00003b2818871010}}, +}, +{/* digit=26 [{1,2,3,..,}]*([2^182]*G) */ + {{0x0009383171b445fe,0x0002a131ad4c0107,0x0003987e89bcf21e,0x000c8eacdfe205c9,0x000063f4153a92e8}, {0x00062a930add43df,0x0002d980f05a7294,0x00006e96862ebb14,0x0006b05f3954e53b,0x0000e1d75ae142cf}}, + {{0x000c6c7af8685c8b,0x000d7f8f01aa5f95,0x00074692ad4c1c8c,0x000068144bbe3225,0x0000800347984a4a}, {0x000c6e52eca3cdb7,0x000b7c04d3997c8f,0x0002bc5cfea1db16,0x0003d2405bc82e8f,0x000063d518064479}}, + {{0x000c1b808bd98d05,0x00041575f2404451,0x00075d270644b1cd,0x0006bd1907eb3373,0x00006c8bebe4a228}, {0x000d2acc4632b46f,0x0009bfd60242c713,0x0005c754617da427,0x0003dd413065b7c9,0x00008239899af17a}}, + {{0x000963f4c8303203,0x0000603203e3f3b7,0x000327afb842c7aa,0x0009b67f22ca0ae7,0x00008e13092c6760}, {0x000fb62757558f1a,0x000157eca8c173b8,0x0003316273cc3e83,0x00023444174474f6,0x000077989cb63c40}}, + {{0x00017a144a081e00,0x000db70e296ae5fd,0x0001f719cd797fb7,0x000c522b472b3048,0x0000e632a98fe6f8}, {0x000d116c5f0c2848,0x000ead987c6289cc,0x0002de6cff51088a,0x0000f8aa2bccda4c,0x000010f9eff7679f}}, + {{0x00094b97ffe4b3ed,0x000115fa5d21b0f3,0x000fbbc750b691d2,0x000affe0bd77479d,0x00002830fdbcaf78}, {0x000249c52434f573,0x000568096dabf78c,0x000f8c0b34b1f754,0x00034c43bf6f948f,0x00004aef03d754e1}}, + {{0x00051f4b7ac7ec57,0x000a750da7d5f8d1,0x0003a0eb8d6ceb95,0x00086331b492b0dc,0x00005157b6a23dd2}, {0x000c74ec5413d62f,0x0006cc5fc4c7e2c4,0x000fa9ddabe329ff,0x000b86935a2aea60,0x0000117f5ae6445c}}, + {{0x00017f4b0166f7a3,0x00079eec74e6ae83,0x000874bfdfbd3e3f,0x0003a3cdb516ace0,0x0000d846019f681f}, {0x000ee5c7c1620b02,0x000d0b63c5010b12,0x00068c51eba68b4d,0x000b5b8c03cd3266,0x0000a6279f76e0bc}}, + {{0x00069b06ae85c104,0x0008ddfdd3a617bd,0x000078bec7294697,0x000a5299a032682c,0x00001c6a658ffd68}, {0x00010240e0239001,0x0001a10d144dcdea,0x0008ab8dcbaeec12,0x000cdd4a600e7405,0x0000333af21cb89c}}, + {{0x000eae03aaba1f19,0x000dab7144cfdf25,0x000ab98bc2cada16,0x00096dd57ee27d71,0x00009088b4d0a6fc}, {0x000c0a03549dbd42,0x000fd158c3ac05d5,0x000edd68542cbdf8,0x0004d01fb6b3b087,0x00002071cf6a6f06}}, + {{0x000721fff2811e50,0x0003fe7fae8cd2d6,0x000f1f7bbdb81b70,0x000b5d3cfb74efd3,0x0000cdbcd7616cde}, {0x000642a566a808ce,0x0003540064d64f39,0x00028fa6f02b7445,0x000bb61abbadca05,0x00004c3074db3fc0}}, + {{0x000b8b0b796d2197,0x000ec4741dd9b32c,0x000edf6f5c3e95f4,0x000b8e1721212568,0x0000a03aee512b9c}, {0x000c376f53a89aa8,0x0001148a28dc0cd3,0x0002ab04f0d8af9b,0x00002d4f86a3f490,0x0000aacb62aff420}}, + {{0x00085ebf62ffd521,0x000e4797bf101069,0x000e30aefe670b54,0x0005e93b405209c5,0x00002c97a205365b}, {0x00046ce1fe32093e,0x00055907a8c91046,0x0006e726b13cb4ff,0x000498ab9f30d1d4,0x00001985e228ba0f}}, + {{0x000dea910a230cd3,0x00039d30f947c573,0x000e2010a24f46a9,0x000e4fc2623fcfab,0x0000f278cb2a3f00}, {0x000c67d50b920eb4,0x000d4e760571ed55,0x00095b709f1cb9a2,0x0003693c50d10908,0x0000207cf07590d4}}, + {{0x0007e81c4127fe1a,0x000c9ae9c5663b02,0x000bfbba5a9f8b9a,0x0006f4bab10851ac,0x0000747d648f6955}, {0x0002b5c2ba97bf75,0x000d6cfa3324cc17,0x00086279d15e0f77,0x0003d35345b79776,0x0000a72348133800}}, + {{0x000139f8f5fcda83,0x00048dee5bfdfd8e,0x0003f9f77f3e558c,0x000969a76cbaf4e3,0x0000a4c97a4a1771}, {0x000e84bf6dce6a73,0x0005e3e6c2d1da27,0x00059a6e9ff373d9,0x00062cc115193cd7,0x0000f9b702593d22}}, + {{0x0004a31317cd0627,0x000da99f8332d976,0x000b11b0b30779d8,0x000ed86807410616,0x0000917ab9fe8aea}, {0x0009cbe28fb1d8e4,0x0002d36eda33b67a,0x00071a86c2e31356,0x0006b6c10b7069a3,0x00004d90fa2a744e}}, + {{0x0000867d6b3e243a,0x000cb9048c486819,0x0007315389fe6cd9,0x00004f1900b02895,0x0000012062fd2cae}, {0x000c8bc9399d0822,0x000a21df12e28107,0x000ef3f7347e8c54,0x000f0af4ba5117b6,0x00002260beaa1362}}, + {{0x000261e1a18cc20d,0x000e5321d63690ea,0x00011b6a02192999,0x0001f51f64d314e3,0x00007401e4d0b54a}, {0x00099836fbca2bae,0x0002afbffc4b1901,0x00086bf4046ad329,0x0009fbc142d3f637,0x0000b5cbc2796703}}, + {{0x000ae6c252bd4796,0x0009b2b5848f9cb0,0x000c9766305e0f88,0x00025c18f6d2b2a5,0x0000f6e149c21622}, {0x000235cde601a897,0x00088373be1fe602,0x000471827d17bbe9,0x0001165af49a5ba8,0x0000e1a0a8588aaa}}, + {{0x0003196270580c34,0x0009b1c98a146c83,0x00034e0a51e23383,0x00028927b2f7b4ae,0x00007ac874618ce7}, {0x000779a100dd467c,0x00068ee50d092b74,0x000608bc9274a433,0x000387a03dcf1383,0x0000d9da6c4889e8}}, + {{0x000199f355116ac9,0x000926d18eed2660,0x0004bc071cc38bb5,0x00057da075f31f2f,0x0000774457fb65dc}, {0x000a9c8c6db88bb2,0x000f2ec98e0406a6,0x000ecaa8b6429d07,0x000a7b6d05e57b05,0x00000f140b19872e}}, + {{0x0000f09ca4946937,0x00008252e909df8c,0x000b14b1248d3a02,0x000d1bdc5c29af57,0x0000e6fa37e2f47a}, {0x000b50649a0c938c,0x0007abe5f41f66e7,0x000359412b72c0d4,0x00047faa6242b8b2,0x0000d35c7754e859}}, + {{0x0006fea87baa6279,0x0001672aa6801253,0x0001e5dc958c1fec,0x0001b8dc29b63760,0x0000e3c3c1d6e9e0}, {0x000127b2bcfe0b0f,0x00013a12f50defc8,0x00079b3973510710,0x000f207ccd6cb148,0x0000792f805e8a82}}, + {{0x0004804a9b46402f,0x000cd10f0850509d,0x0006208aaedddf85,0x0002dba28410dc4b,0x00006229c4729101}, {0x000c41e7727b9b64,0x000b6a444842c5a7,0x000a947ea289e4e4,0x000ebbc49ba1d9e9,0x00004f9e47fc3c8d}}, + {{0x000a1fe611f8b8ef,0x000a0518f427fa77,0x0004ebac3fd2e416,0x00097ad5fffa7011,0x0000e57c4ea4d896}, {0x00053acb1aaf6131,0x000fda585a45fdd0,0x00098503431df210,0x000130218cc10e24,0x0000a38efd16f1d6}}, + {{0x000f2370b1e9e212,0x000cfdbe88aabf86,0x000c1baf9b258514,0x000691fe38a58890,0x0000936a01eddb9b}, {0x000de986dd5b20c8,0x0000f0f98ecfd576,0x0002d2fd7b586bf7,0x000d7b4ccf0f12c4,0x0000717e61d7b35b}}, + {{0x000572235e6fc06c,0x000e4b3e13d58b1e,0x0008a73723477728,0x000289550c294daa,0x00000291d43fbfa5}, {0x000bc67cec5a196b,0x0003ac2e8a7cc6c8,0x0006e1c51deeb31e,0x0001560a93e244fb,0x00009f8b71bde28e}}, + {{0x000a287968a2ab91,0x000936bbcb1fce65,0x000ae3f30e3c5ce6,0x00082be8c835b9e7,0x00006bbee270f72b}, {0x0002017fd42cd227,0x000088b1d2a0665e,0x0002049321e13997,0x000ee4a25cda2979,0x0000aee94a5b9c3b}}, + {{0x000016089821a663,0x00085f98166968c7,0x000cc3645f7c3767,0x000dfca90829fc48,0x000046af04a070ad}, {0x000b232370bf29c0,0x000e42e650ee2057,0x00026ab90f90c73c,0x00087be03386eaa1,0x0000e266e7e975a0}}, + {{0x0008eb90fca65d93,0x0009e6af45b88057,0x000c75a4e7e2989e,0x0006b84438212dca,0x00008c7ca397fef3}, {0x000c494d402676af,0x0006072c7c488650,0x0003a464e26ab5a6,0x0008405e6cb426ce,0x00008f998971b72f}}, + {{0x00084911a335cc88,0x0009ea5913e48c31,0x000b32919563459b,0x0005ac9b920d61c7,0x000005ab8b720242}, {0x00012da8d006086c,0x0009fcf5c0fd2ac5,0x0002138d76ca4846,0x000442efea51d8ac,0x0000b647545f44cd}}, + {{0x000ee8fbd7d90402,0x000e619b9c960429,0x000a7d744ee66a2d,0x0001bb34f9ec25de,0x0000ffea64287172}, {0x000dbd1114344eab,0x00064d0dbc8b4f19,0x000ec7f910430453,0x00014c514b50aa29,0x00005fc22ff6b060}}, + {{0x00063a91ee682e0d,0x00002e85c72760d9,0x000707c2ddf48abc,0x000efe3cadba132e,0x0000e608d3b7645a}, {0x000c28bedafd8830,0x000ebd94de1f05f1,0x000593e413c362ed,0x000eaf8dd0629d13,0x0000a5e736f766d6}}, + {{0x0002311f68cf9d16,0x000761797556bfa9,0x00001c209a4f9ef8,0x000360b0d75a1f56,0x000051c374c69b07}, {0x0000b5888b5cead1,0x00085fa9dbaa4995,0x00015f33a0ef0005,0x000140e51ddc264e,0x0000f8b5ca63ef46}}, + {{0x000c0a3ee9523f07,0x0002275ea978343a,0x0007387f4bb75eab,0x00062dabccf33210,0x000090f925a0ab00}, {0x00063ad1e4f6a5f6,0x000402519a50f1a3,0x00065f1ee06e08b8,0x00085e0091518772,0x0000a80ca34f3ae9}}, + {{0x0009768aaba4864c,0x0001cd52a7d681b2,0x000ad03f1b13cabf,0x0001bff5c363488e,0x0000932ad9641c7c}, {0x000708ecae1e27be,0x00083b0df6485452,0x000cdb8bc9dac426,0x000173433e3f0cdf,0x00006cecce0cc540}}, + {{0x000845e95081181f,0x000f799355d5bd0d,0x000b375a8cc8a791,0x0000db211c0f6dc3,0x0000d95bc6ced51e}, {0x0006a266888523a0,0x0006cb01a06d4a10,0x000b9b3974d142bd,0x00091479bfd289ad,0x0000bdbfb94e9863}}, + {{0x000a2291660f6a6a,0x0005b51c042d29d8,0x000c3ffe87f6abcd,0x0003fa73039deb0a,0x000001be6298c852}, {0x00041030ca1c3287,0x000d4903928e6ea3,0x0009144b0c74114b,0x000b171aa4ff4e9e,0x0000064091fef9a4}}, + {{0x000f521e447f2c48,0x0009e04291f0a3f4,0x0005926de81b8da7,0x00002f5680bc467d,0x00004f21fd5b4a12}, {0x00031814e9df3d85,0x0009e9ab8d341d1e,0x00019aa4a1ca4861,0x000366309ddeec5b,0x00009f72f7e9d329}}, + {{0x0003f41386d5087c,0x000c1d67d64fa2f9,0x00070215840bf739,0x000177f449420566,0x000033c65bf33b1e}, {0x000657c38ca61533,0x000aac791976cdcd,0x0006e1f3997f4519,0x0009329c7c7f29cd,0x00008de9cfbae3c3}}, + {{0x000eba37b793f853,0x0009c067e914e448,0x00014ae87e9f8dbf,0x000e2a90390266f1,0x00009ed75a7fd6a8}, {0x00048487ffba390a,0x000acaf9bc09adb1,0x0007476db67f8cb8,0x0008d5922c38489c,0x0000320fecff2a53}}, + {{0x0003002b2aced2bb,0x0008b16bd430e049,0x00031be70dfba180,0x00044fa31c4644c3,0x0000c04d32f40d2e}, {0x000a0d10f9f142d6,0x0004e7ee5a231805,0x00089b4e32c44a0c,0x000480d1875a4339,0x0000b1949fd6c063}}, + {{0x0009e08be0f44920,0x0003e9d5e5172dfb,0x0009466a83ff0da0,0x00093203dbe9a1f7,0x0000b87bcd015ea9}, {0x000fc83ab1f58ab6,0x000a217edc8aeb64,0x0003b67e56d9598d,0x000853099cff661d,0x000045c0f29f2635}}, + {{0x000dd82eabaf21cb,0x00097241659e253c,0x0009f709182b9602,0x000d9a7cae07ec2d,0x0000e4c720d3b48c}, {0x000bc036f08d6c97,0x00073f10bf406ce5,0x00010ff1236e8a99,0x00049413422d213e,0x0000b26d3ec2cc12}}, + {{0x000d2d0c9469ad61,0x000d20afa05bb240,0x0006ba286c4a11b4,0x00064c3b604acedd,0x000084866004ee28}, {0x000d6ba8d9ce5be6,0x00059f4bfb0d5869,0x00000cf730d8f68c,0x000135569f210b57,0x00001f6653acd37c}}, + {{0x000432b5aff5a48b,0x0008c2ba3a69ff3d,0x0001899ef0d81c4b,0x000afd3e879ae9fa,0x0000ac7e2a0dd6ac}, {0x0003f6c1c664399f,0x00006bcb135dd6d9,0x000ab7cbf4c288de,0x0005ef93031dab9d,0x0000e23feb12abbf}}, + {{0x0002466cdedca850,0x00001a09538c9f1b,0x00011115d140bb71,0x00059eac8ae8515e,0x0000d63ff676f03f}, {0x00055517d234afbb,0x000dce208fc1755e,0x0008a4b5d61c2db4,0x00030efa9859cef2,0x0000dd6d4fce4af0}}, + {{0x000a26d3be01cb1e,0x000b443aa07cd1c4,0x0005035029ba14ff,0x0003ab195cd3a9b2,0x0000379bc075d2a9}, {0x00018e9d4ca8d68a,0x000be0bb412a3efc,0x00045a968083558e,0x00054f3903b94096,0x0000499f0b73ba60}}, + {{0x000573cb8349abe3,0x000500b4fc1c208b,0x0005249903baab3e,0x000e8057e978bacb,0x0000524194efcdf0}, {0x00017257d4bcc42b,0x000b090109ba6271,0x00023e1e0e90a3d9,0x0005988b1bdd5713,0x000078e9bd60eae1}}, + {{0x000b7469e03d2781,0x0005c70e62970794,0x000c978558017860,0x000b5c071792f899,0x00001b393ef05a86}, {0x0006582d8884f278,0x000a3f19ba5f48ef,0x0002062c6bd44737,0x000c540698de4ca4,0x0000975eb80e1ce9}}, + {{0x00057c7d7fe71f3b,0x00000c97ce38d50e,0x000f07b631534219,0x00087ca1bda2de4d,0x0000a12aeaed00eb}, {0x00035d2a9b4f8f67,0x00058ad6d99cabe1,0x00094937c04619d6,0x000099da6683a779,0x0000a778c8bdf94f}}, + {{0x000862320a71b899,0x000c8c2291658c50,0x000f83a99241a2ae,0x000ac7a52be595aa,0x0000fbfee7fa562b}, {0x00058b95c4017e32,0x000ff5120b86eaf6,0x000034d6f1dc7f9d,0x000038b84f13dd4c,0x000083dd7380aea3}}, + {{0x0002609cd85d6a2d,0x00052ae60177197f,0x00012fede6ebbc34,0x00086ae80f031b4e,0x0000e55d0c2d7a21}, {0x000e37f24dcdd5a7,0x00050ed191fb1fb3,0x000023e0d8d602da,0x0000bfa08fb05676,0x00000178c71b59c2}}, + {{0x000a3863fe54cf05,0x000eb2bbb475fad5,0x0009d94d7a4a3ec4,0x000ab2caa5ec2091,0x0000d3b63b5f81e4}, {0x00033d85ad3d2aff,0x000ce1ac7a377fa7,0x000779614fbc586d,0x000429382925de40,0x0000e0ffffcd74a2}}, + {{0x000e67f906151e50,0x000f55e106493f39,0x0007cf7b7cea27f5,0x00062ddca1d4e1c1,0x0000c326d122fe23}, {0x000ac337dd35df39,0x00093396dbdf05f7,0x000b7db1c0c3b763,0x0004a87912f5ac03,0x0000dea4b70ec9ed}}, + {{0x0006e53aae3f639a,0x000c5c278bac475e,0x00090375ffaba0e7,0x000d0976f9e22194,0x0000ebf974745a7e}, {0x000af3ff41ad5d66,0x000c52e9922445f9,0x000cf56aa03c4623,0x000ed322c5bb5cb3,0x0000431181994567}}, + {{0x000f2118be489ac8,0x000d39a1104bec57,0x00064e0072821895,0x0000fde10dc87560,0x0000e526f3fdb20d}, {0x000ca775b645aee2,0x000f600e10ff6e71,0x0009cf6de3d1dcb9,0x00035316b5116218,0x0000c5a3e308bb17}}, + {{0x000cd3e2a6c6fbf0,0x000a4bf97906c186,0x0009d6901a74516f,0x0007435b4b8f4b27,0x0000c4e57b42b573}, {0x000b229b6e386b6e,0x000cb9deac2775fd,0x000712629b46793f,0x0002dd0eec47eacf,0x0000965f3c5abc3b}}, + {{0x000fb83425c65590,0x00060af06fda8dd1,0x000d956df7fc00ee,0x0008a2e98c922533,0x0000f1ef3354fbdc}, {0x0005145b79b8ea2c,0x0004fdbff2882abb,0x000185db740fd294,0x000099aa814ac4d7,0x00004329d7080846}}, + {{0x0007b52ed1be45d6,0x000d84cd2c74c9ba,0x0004139b1891dd20,0x00070ffa4d4a7f82,0x00006c177171873c}, {0x000c1412843c4e0c,0x0006f97eb5bf5e5b,0x0000c95c7d5ac481,0x0006c500f8af5445,0x000091b3fa0f1840}}, + {{0x000340aab9d97f89,0x000700a2d611360c,0x000a6f7e5fb57bd0,0x000a0fb339ae3ca6,0x0000c1fcd2abfeb8}, {0x000cca9c7ea74322,0x000c108076f6972b,0x0005b4ca51b0b924,0x000b2960b2814a2a,0x0000f78f55b81ef3}}, + {{0x000744ac18a414fe,0x000db03d0a86f838,0x000453f55c611eaa,0x000278b4dabc162a,0x00006f2e3daf4efb}, {0x00060179320dc3cc,0x000ecdf6b5a45b7a,0x00040fa90692e382,0x0003177f5e15e02d,0x000087883af243dd}}, + {{0x00053e453544774f,0x000b4adba2bc5110,0x000e371f5834d0ec,0x0003bb5215d7f7ba,0x0000cfd57c05c866}, {0x000383dd6901b1d3,0x000485587dc3ded2,0x000625f623b49fbb,0x000762cd44a08d07,0x0000ee4d65bcde9b}}, +}, +{/* digit=27 [{1,2,3,..,}]*([2^189]*G) */ + {{0x000137d0d63d1fae,0x000122a9d89f64e5,0x000436309658fc05,0x000a606889487450,0x00009ae30f9b598d}, {0x00010d1818baf918,0x00060b6a0c202ed7,0x0001a6b44e27e9e0,0x0007db9e28dcfb1c,0x000083acb6556ac5}}, + {{0x000728dc2c6ff707,0x000f55dc22358735,0x000e277f979d6122,0x000cc6b3f5d00319,0x0000ee84e264ded8}, {0x000afb063cd880a6,0x0005d574af6091a8,0x000de7f423f3ea7c,0x000151acfcdc8402,0x00002d07930131aa}}, + {{0x0004e438a5807cef,0x0002f4109a7e8e1b,0x000d59ddaad28389,0x000092d30cc9cbaf,0x000065f36c72d8d8}, {0x000469ea60d32b2d,0x000a6e8191c8df31,0x0005bdeb5ee93df4,0x000a27cc1017c535,0x000026231865616a}}, + {{0x00083f9dec31a21e,0x00028ad9d573b02c,0x0007be365988c8b2,0x00034d73e983aea5,0x0000968734e446f8}, {0x000ea8f5da6309bd,0x0003f1f1ce169137,0x00044092110f3a62,0x0001b4a82a9ea2ca,0x0000f94739f2b46f}}, + {{0x000e006cce85c9b6,0x000644c7c2d39f9b,0x000fa1e60360e70d,0x000b6cccd5beeaae,0x00004cf63c0ec3d2}, {0x0007fa3e1cf6f904,0x000695e044e6fb10,0x00034db9fb7e937c,0x000bd034e8ca78ce,0x0000f8b36c17e210}}, + {{0x00065a434a35ea84,0x00070d4412f61df1,0x0008836c33418e0f,0x0009651af1f8af51,0x00002ceef4d530e1}, {0x000ca0b543a1957f,0x0004986cb1235560,0x00098ed30c33761e,0x00097c76624b1ffe,0x0000772f4c000909}}, + {{0x00040bb4885d4104,0x00005ba5f8d7f4e5,0x00098dfb17287f81,0x0008a2e2d0d865de,0x00009ff51a1fcfbb}, {0x000fa536bc3012e7,0x0001a70d541db6b6,0x0000f49663d31fd7,0x000e071018724f4b,0x00009e7399ff7dbd}}, + {{0x000410ef4f8b16a8,0x000e447b266a56f8,0x0009c87c197241af,0x000b1a8a406b8e6d,0x000003f3e034d42a}, {0x00009a804dbec69c,0x00067bbad05f7f03,0x0008e197fa83b85f,0x000dc106097273ad,0x0000097440f1067a}}, + {{0x000afb63524ff164,0x000e423fc6ce730e,0x0003e4ac0d7f9b51,0x000216e7bd0d3244,0x00000c59ad98d66f}, {0x000136f17c387a48,0x00056b86804d6c33,0x0005a73c95043b8d,0x0009b5f497031267,0x000038fdb3271666}}, + {{0x000b6dd418e7ddd1,0x0003372f19d6c507,0x00027eb4d39888d9,0x0008846eae26be0c,0x00007b53ed40babb}, {0x000021b2b01ae4fe,0x0006ef488682fc27,0x0005e2d8788462e8,0x00029adee096ec21,0x0000b2fea9bb242e}}, + {{0x0005b5fb821fc28e,0x0006fc1e2ad25d98,0x00030ba6289d2e19,0x0001c575b566b890,0x00003fd41b62f41b}, {0x000ac2eb9a96d612,0x000a169443f4b738,0x00003a4407f8567c,0x0004dc6698622df8,0x0000b586236afe2f}}, + {{0x00000c756b95bce4,0x0006216da680bbcc,0x0002142525ec0390,0x0002d239162ee672,0x00003132b63c6a89}, {0x0003ff22f3263bf6,0x000c3cd0a1424bdd,0x000415ccbd5b3733,0x0004e9f92eaa8244,0x000063e8924ed547}}, + {{0x000a25e5236344e0,0x000dbda76ee68058,0x000cc3d2282e8df9,0x000529dcf6efd811,0x00000089cda3b4ab}, {0x000a071bd38a3db5,0x00009f72b92591d3,0x0003edf754ea97fc,0x000ed2bc9fc15bea,0x0000a6297cdf4348}}, + {{0x000ab35ce7c42d4d,0x000eb2feab100d38,0x000111b459fd493e,0x0005c276056b6d82,0x0000a11dae243efc}, {0x00002785545a7fb2,0x000c20d507e6dc74,0x00066fa58bdb2601,0x000c29f21dfeeb70,0x000014369a859ae8}}, + {{0x00009cb06e0956ce,0x000e210cd34b1957,0x00071a5324c9d254,0x00054d151e13f704,0x000019d6791fe730}, {0x000a628db5c7be33,0x0008824dde05f702,0x0009b2e2ec714121,0x000dbaac18233cf2,0x0000a6bd1e8b5342}}, + {{0x0007fa0b311898c7,0x00045d0eac653f74,0x00014d0bce2a272e,0x000ee2dbba5851f9,0x0000a1a966134a43}, {0x00067cea1c8cde9a,0x0008d271abe3e5a3,0x0001615cd9d958ba,0x0000b053ff7eb63d,0x00002280dcf95ae2}}, + {{0x000a5c1cf6401471,0x000d4e83d11856db,0x00024c511ea5a2e3,0x000213f4cd6b6dda,0x0000c0f4671f854d}, {0x000b7a9695653812,0x0000becf1f5b91a6,0x000f5d009dc96624,0x000bd4fb22d21cfc,0x0000a05f641b021d}}, + {{0x000d566d4312483b,0x000cb43e216f8c0e,0x0000444935179a95,0x000a211c185fec17,0x0000306333a04991}, {0x000ecdb0081a726e,0x00056fa89bbbd801,0x00091b6b90149b0c,0x0003a2cfe9065a43,0x0000dc92787b633f}}, + {{0x000c24aae6a8e13d,0x000dcf3897abe408,0x0001a071585833fd,0x000c5e73800e7ed8,0x0000e08e347844ff}, {0x000184ccdeff2e08,0x000a965eaed17094,0x0007c468a49f9387,0x000dd7e35d612977,0x0000c0dcfd1d38c2}}, + {{0x000d9e37a6a308bd,0x000f7c2767d3d6d9,0x0008cbeb66237582,0x000d9db74a8bc6f3,0x000094d3f1b9cb6f}, {0x000735bba21f248e,0x00048cd1efb092a9,0x000b03284272ad0e,0x0002249437b69c05,0x00007f047034948c}}, + {{0x000c04acba2ececf,0x000ff3a73e418a56,0x000e937250c18126,0x0001a87cb34e9d03,0x000077c871439652}, {0x0009183fa7f9f909,0x000bfc9707ad9456,0x000c1c9a3f2e7aa4,0x0005073ed2c9ba26,0x0000109fe96d0197}}, + {{0x00068a9e9adfe1c3,0x000d014e39bb9ae8,0x000fe378f3984403,0x00062885875720f2,0x00003f901e0ea44a}, {0x00025fe3652438c6,0x00063dd1f20bea11,0x000bf7fbdae9ec4e,0x00079bbe740d9ebe,0x0000dbd3ddca2dbe}}, + {{0x0002aecedd36776e,0x0008098590396208,0x0002f7065f612c47,0x000210c493b20103,0x0000bd4d8f32ff9b}, {0x000a0aaaac4cb325,0x0006c5ed40053f23,0x000a27e63ea3aadb,0x00066c5cf17ea4af,0x00006125c1b111fd}}, + {{0x00044a43794f8dcb,0x0009983c5c362663,0x0009d10a0dcca923,0x000df27d6b6bbf3f,0x0000320c5cb31d9b}, {0x00028ff47b50a951,0x0001bef03371620e,0x000100153933e3b0,0x0008d6e081bf8599,0x000083be9a0d3a8c}}, + {{0x000dc5ad6bbe24db,0x000fa38437954e3d,0x000ec2d4cc6c7462,0x0009b1c8193dd765,0x00008df26cd7d3c8}, {0x000e3995a483f8df,0x00068dd3313a98db,0x0000bd37572d8a95,0x000d1575087294ab,0x0000cd892496c259}}, + {{0x00043d77613aa81f,0x000f95fe65848a94,0x000b10288801007f,0x000ee780fc4dbc7f,0x000058280d4d86be}, {0x000d82f7c978c385,0x0000bde44d7b14fd,0x00060252fdf1204c,0x0006a5508a1c8441,0x000091554cb11764}}, + {{0x00037d6a05bd5254,0x000ac7957b3c214a,0x000109bc948d5f09,0x000ce6c247cdcbd7,0x00000f9e4bb70599}, {0x000fa03f46ad2ec6,0x000f63e3f9eec325,0x0003a457700f766c,0x000b934b556668d4,0x00008d30a619ee03}}, + {{0x00081ea77b46a081,0x0007b74806997ddc,0x00033f683cf5a647,0x000c6033a8cb3466,0x0000b867e6ba2363}, {0x00011141f60558ee,0x00024f41450e4392,0x000630e8bcdbcdd6,0x000b429fc04601cc,0x0000a7c66d677038}}, + {{0x000b8a504e99fd8c,0x00018785549a7259,0x0000552e198a8dd1,0x00009d4e459a7c84,0x0000dfcf4d10bb09}, {0x0006db253758da79,0x00035ac997e134a8,0x0000c5b7ee643bb8,0x000b5206400bd753,0x0000f97af88441c8}}, + {{0x0008820fbeee3f97,0x000fd9091afd34fc,0x00031f35c93e5348,0x000924064b9be59a,0x00001f37864c7e3d}, {0x00034e0943aa75e5,0x00085b8ff6e402fb,0x000cf0d19a18c9c5,0x0008a6b80f31b133,0x0000c9682db58351}}, + {{0x0004ca6b709c3de9,0x0001a575b8f0873d,0x0000154bb64a8426,0x0001aad275da1f02,0x00007678cab617cf}, {0x000795f951a95c33,0x000320fccc088779,0x000d8f031dd35b16,0x00085c0270962733,0x0000c5ab10a798dd}}, + {{0x00085c341dca5663,0x000aa8622aa3b6c1,0x0001b6dfb7de7fed,0x00028869e84d9290,0x00000a02b0eac4ad}, {0x0001daa2fd3cf36d,0x00070f89e59fc7c8,0x000496733d131954,0x00012ae2be8184cd,0x00005f449ec63d34}}, + {{0x0001b1b25fe531df,0x00017a1d56467ea4,0x000de501af979743,0x00089b96067f722b,0x000091481c0fc85e}, {0x000e465f8b05bc60,0x000f02e83cdaca8e,0x000dbe33b1844e1c,0x000de2ca82114ab4,0x0000f9f87694eabf}}, + {{0x000b1c038b27fe29,0x000b1ba402df4936,0x0006bdbab63b6359,0x00039bb0c0ea2f65,0x0000c992a89f580c}, {0x0008f152a60aed16,0x000480bf49df600e,0x00042d99aeb089ca,0x0002fa3c233d7d2d,0x000048d3f95ac6bc}}, + {{0x00083a8e1add3f34,0x000a0f64a348dcc3,0x00030dbdbf42c0c6,0x00015deabd176f00,0x0000de501a3bd6c2}, {0x0007c1f4b9a64bc7,0x0002b496cd594a10,0x00088dffba77f0ad,0x000d8e8b78ac6276,0x0000025a2cad7937}}, + {{0x000b2d1d1a8f4e7a,0x0006d354927cfde8,0x000205735f5b3da4,0x000917448606a3d9,0x0000c477cc78177b}, {0x00073d2a883239a8,0x00064c8b8357fb1f,0x0001f4f86e12572f,0x000c6e1d355e9cfb,0x00009b795f959f3e}}, + {{0x00056f1b54398dc7,0x0006cfedeed527be,0x0006d01401890efd,0x000ee3f2f77f1f9c,0x0000ef0e314c96f0}, {0x0006631cc61dab32,0x000dd4866e4f50ca,0x000363b394a39801,0x0006aa46c8d032ae,0x00002c591e54ead6}}, + {{0x000a308de02a53e8,0x000f5389f357954b,0x000f40b662a6c060,0x000ce166cfcde8fb,0x0000e02fc5746340}, {0x000779573adb4ba8,0x000c27b03805e495,0x0008e6fa67b86122,0x000803e3f835120c,0x00003660ea0857d7}}, + {{0x000910521ba473cd,0x000e0ed5389dbad7,0x0007c9bc0b6c50be,0x0008a71e2caf4daa,0x000097b8de55c4e9}, {0x0003e70ab3bbddbe,0x000e4597815aa9f6,0x00015b3d93898aab,0x0007839659af89ac,0x0000df7725c503ce}}, + {{0x0000fabe085116b3,0x0005572853102547,0x000bfd52f04a4337,0x000c741e39187ee2,0x00006166b44ad9eb}, {0x000433cfd4b322cf,0x0006ca79ab5192ad,0x000db15eb726aa81,0x000e63096eacd8c1,0x0000af71e91f476b}}, + {{0x000a640641fad989,0x000799622559dd69,0x0004199dcb799591,0x000eb373c6daa5de,0x00002cadc983d545}, {0x000238b256534e4c,0x0005c595409a1028,0x0005dc59b73e80ce,0x000e7fa90d4c66d0,0x000095f7b90581de}}, + {{0x0007014d856ac253,0x000d7c524dcaf433,0x0000499f5441bd9d,0x000182340b3d855f,0x00009cf84aa05fda}, {0x000b055b2aa95a02,0x0009eddf186004e7,0x0003f6b4329e33f0,0x000b0ee82e74b542,0x000017edeb92aaa2}}, + {{0x0003f3583cbea55b,0x000d0c185d7058b8,0x0005f6992c485ee4,0x000dd4d33ff03b1e,0x00005b9b9cd7f0c0}, {0x000ee8e4e9e8a506,0x000b0269dafd7caa,0x000e791c6462e907,0x0007900ed5cee9fb,0x00008ca325a4d430}}, + {{0x000bdf213b5ba885,0x0009e5ef0ac42b72,0x000b99b0860294c8,0x0009aa4c3230ed19,0x000060fff17bc258}, {0x0008487d67703743,0x000d6a56f685552b,0x000f175d9a373202,0x000810a3e7f90745,0x0000c2f31600080d}}, + {{0x000e9dd7b9520e80,0x00020af037b51130,0x0009c104cc078f9e,0x000e9238cd2ec71e,0x0000f684368c472f}, {0x000b5ed6247e7ef6,0x0008d96dfe21d3f1,0x0009aa2c2b32d33a,0x000e40e6f59cf44a,0x00009cd51695f0f7}}, + {{0x000da0f4b3234da4,0x000574579ebe3f59,0x0002476c7cf0b023,0x000f082d1cbb256d,0x0000f0837e6ddc30}, {0x00075bb906f6e98b,0x00041761e7d19a40,0x00073af10253bb43,0x00031c2e2e645f6e,0x000089a4060bc5f1}}, + {{0x00040c5b8cc037fb,0x000ac405bb47d128,0x0006348b83d093a5,0x000ca6b202c25320,0x0000f5d57fd755a3}, {0x000c90c8c3bef483,0x00032a0a960a89f6,0x0002b42ab23ac762,0x0001f6afbd3d6b55,0x0000ef2245843206}}, + {{0x0009bdac97e65165,0x00007230f49ed74e,0x00074ea498877936,0x0005a256ec1de31e,0x000081dcee58fb64}, {0x00023918f483f148,0x000c5137d13bbaef,0x000743a426d2dddf,0x000e66d4cde50ed2,0x00009a34fc664d97}}, + {{0x000f5b312e08ce5b,0x0007f7f0b2ca13f1,0x000982805a80540b,0x000a03d54bcf7701,0x00008653ffdd33be}, {0x000878702b0b4c9b,0x000eeacb170a8e7b,0x0000c14e52675261,0x000be9561a9d9093,0x000059b30e18ef0a}}, + {{0x0009ea60200ec7db,0x00085bce132b1dc1,0x0003e27e0b6f4a3f,0x00016f08d5de90f1,0x0000aee5ef0cfade}, {0x0006aaae4c6cf389,0x0006413698156f40,0x000d550c6ab4cfe0,0x000d387dcffe87ef,0x0000d4f59c805ff7}}, + {{0x00053b151deb6adc,0x0003f1877749b025,0x0006006e1812399a,0x000e770e90f71fca,0x000032363a7702b6}, {0x0004fbedc36c64d8,0x000127e1ae610228,0x00009d94a86c81e3,0x000bafa576c7e5b9,0x0000b6f7d03018b2}}, + {{0x000ed0756faa38ab,0x000be305bb54eca3,0x000c73061a3790e6,0x0006142784eeda7b,0x0000d56d36a1dd50}, {0x0005949229a8aa9c,0x00068595ec28d657,0x000ab4fe6dcca8f4,0x000f15c14305c106,0x00008c39768e4f43}}, + {{0x0005f36523f2b367,0x0003220d93bbe2a4,0x000f1632b995c649,0x00095488afdab790,0x00009ebbecd8c295}, {0x0003ddb79592f48f,0x000a6f88e998c7bb,0x00001193e67216a7,0x0003fbe91f098bbc,0x00007d928a6a1db8}}, + {{0x0008417e991f600b,0x000d7981a93455e3,0x000b13bde2a91113,0x000f441bc9d64806,0x0000011b6acb755f}, {0x000b518045ec6135,0x000172f5930a6f4c,0x0002e65de522d2d3,0x00066f0acae1af38,0x0000764306777bc9}}, + {{0x000705d1c7193f0d,0x00066be8858e5e12,0x000c6dfc7f0f32f4,0x00095ca85c3d7d96,0x000075b4a218f317}, {0x000f17b342659d41,0x000434f0378f91ac,0x00052129de596ea3,0x0005853515708fce,0x00007387e1e89f2f}}, + {{0x000d2e949dee1686,0x0002de2af23972cf,0x00094066a1ae0522,0x000412a09e75be1d,0x0000cca31c798abf}, {0x000d61d9bc499082,0x000cd5e2bc1eb50b,0x0006f83ac4a9b4a8,0x000b28cb6cc5f794,0x00007da93fd0bffa}}, + {{0x0004c964821c8c54,0x000d683c15f4ea31,0x000f330048de49de,0x000e103a64cf207a,0x00005f1bfec09627}, {0x000062654b9df609,0x000c195c0b33878b,0x000035d8e5e4fdc3,0x000b8c554a37cac2,0x0000087cdaa10f20}}, + {{0x0001c238319ade44,0x000a9e8cfdf836f6,0x0006f3705766f287,0x0004a20882194834,0x00009a7b85356e4f}, {0x000f8a75cedadfd1,0x00057db2a815b9b3,0x000f68f958f56281,0x00008eb0b7d55401,0x00002971e27788a2}}, + {{0x000b696d0ff34fce,0x00013222718cc9f8,0x00095284d20824de,0x00023f9213cf9f0c,0x00002ad741cbc158}, {0x000a6df54043ccfa,0x000b384412b30ee3,0x000c98af016ff479,0x0002fb56c74ee0df,0x000078a169ff2fcd}}, + {{0x000874699c930e9d,0x000779e117a5d8ae,0x00024759f1d33e85,0x00001ca581fcb466,0x0000e5064502bedc}, {0x0005d00caf3155ed,0x000bec73e75fbeec,0x0000b01db672d66a,0x000b78b6b9d8c627,0x0000249ef8420f55}}, + {{0x000d6d473978fe39,0x0001e54b00a16131,0x000dfcfe9cc4e454,0x000befbe05df0557,0x00004b29cdde1ef6}, {0x0000cff9bc7edf24,0x000d93da65f3e453,0x000eb0b488ac236f,0x000038cfaf7d5fc8,0x0000d2de14ca60eb}}, + {{0x000bba760430e540,0x0006da3289abc006,0x000979c5910a2d0d,0x0009449c037a5dd7,0x00004d1f3d3a116d}, {0x00024738a0983cd2,0x0008a883cabb9ff2,0x000a5899528e25b3,0x000bdfc968dba547,0x0000c80b505974ee}}, + {{0x0003b714a953beb9,0x000e8642e7f6ee76,0x000d5e722502e223,0x000315dfe4b64161,0x0000d37c5b16bef5}, {0x000ed70f8330bc73,0x000645a727890115,0x000ceccc2139850e,0x0007f5f7d7faecff,0x0000016a8607fd9f}}, + {{0x000ec644cd8f64c3,0x000ff79d7b51c492,0x000c7525658a2d78,0x000016dced1fc51f,0x0000e658aedbf433}, {0x000942e05da59eb6,0x0002addc37220b61,0x0002e7f87ba3d60a,0x000b6e1c311cd174,0x0000473ffef56b01}}, +}, +{/* digit=28 [{1,2,3,..,}]*([2^196]*G) */ + {{0x000604f692ac542f,0x0000327b91d38303,0x000aaf9bdf079ffe,0x0004fa29f63e6315,0x000099ee566e1f34}, {0x000661fd62191997,0x0006648ce41c8a1d,0x00074d9048c883bc,0x000b1aa065118f3c,0x000013889ee7faf8}}, + {{0x0003f8f81a1b3bed,0x0004fe2764a0972b,0x000c4f5f74f3ce14,0x00085b12d0f1cc28,0x0000eee0c0e97f39}, {0x000adc0d39e25c35,0x00007467a0807df4,0x000cf5a584061982,0x0005fff40ebc9361,0x000027729a6922ad}}, + {{0x0000937b1b76ba6f,0x00045d2026dcca6c,0x000d9ae0a1a2eab8,0x000025c1715e1519,0x00001ad919aaac4a}, {0x000dfb807ea7b0ef,0x000e4ed9eb8935b3,0x0006d08abedf5496,0x0007309932e5ff2d,0x0000314874f15bd2}}, + {{0x0006a753f73f449b,0x0007dd44fc79efb2,0x000c0dc4d1d1c94f,0x0000cf99f0fbc53b,0x0000747ea0be698a}, {0x000c3fe228d291e6,0x0004e3c129d65218,0x000acc51635b804b,0x0006689ac859b8d1,0x0000c10697df5d6e}}, + {{0x000438f0876fd4e6,0x000723d2f383c38e,0x0000934cb45f0c30,0x0006edc03cc2ecb1,0x0000a8f24398c9d4}, {0x000431b65ccde7b6,0x0007c7e76a6ff16b,0x0003484d741e2cd1,0x00044a59c8cf8f4e,0x00004426efde3152}}, + {{0x0008e44fc94dea3f,0x000eead6a0b01c0a,0x000113cef34c8cdb,0x000ff9a19c384004,0x0000d32fba505490}, {0x00090f6795dcfb75,0x000333588baf58d1,0x0001fc1c0fef01b0,0x000ac94e6d1d63ca,0x00003173f9740a41}}, + {{0x000402aba16f73bc,0x0003ccf9b9fc2b1d,0x0006ef7bf2fb3101,0x0007446d51e60e44,0x0000731021c791e1}, {0x00047244fee99d47,0x00068ac5c1ea9d3b,0x000ea9af74bca48b,0x00083a00f5f514bb,0x000051f55a6074c2}}, + {{0x000251acb452fdbb,0x0004a0f306506e30,0x0003548d931ee696,0x000f5b00b3e50893,0x00008949a50a4b0e}, {0x00083263c88f3bd1,0x0000cb1d9989208b,0x000d4df03ab147c3,0x0000c5dd6515fd44,0x00007a12f75f72eb}}, + {{0x000796d36cf69db9,0x0008c6670c183b59,0x000070d8e1219eee,0x00090c6e3341f77a,0x0000b70130c3327f}, {0x00024620ae18e0e8,0x0002c6c0a63836a3,0x0002eb0d42021a62,0x000292a51b5817c6,0x00007bfbcdfcc762}}, + {{0x000b505cdd61d649,0x000c38c18857f78a,0x0001475158c7a53f,0x0002d51653ce6f16,0x0000c923aa67a7d5}, {0x00009cb5c18871f5,0x000823b3cc74c247,0x000d1d4c47d53bec,0x00058209264afffd,0x0000555917e740da}}, + {{0x000bbda548f5a0e4,0x0009fbbfbbe1cae8,0x00077afc31910eab,0x000b5c6e57968576,0x00009ea61f1b3ff0}, {0x00054784f7c39221,0x000d10c68eef7865,0x000779ab995d337c,0x0009a858f1e1e5df,0x00004b491b0c5cf6}}, + {{0x000bbe028e3fe894,0x000485aac0eb7a6c,0x0007e5140e7e1fee,0x00021f3f47eda569,0x0000f450137f4549}, {0x0005f8495cd81854,0x00018db2e583db62,0x0005e6de474be0ba,0x0007396ee4fd7cdd,0x0000251437e28101}}, + {{0x00072a0ac6203668,0x000c36d59344686d,0x000eb75b94be3fb9,0x00010bee8b44e7a1,0x00004e39da411a5c}, {0x0001490b38f04092,0x00030c2ade8237cc,0x00090a2d80295194,0x0002ba7b68878311,0x00005627d1443118}}, + {{0x00050aa658a6d877,0x00075f9c73256eb5,0x0008748c91405aaa,0x0000e06147142e5c,0x0000f637e4fc3ede}, {0x000277614ffad2c7,0x000d4afb6791f8ca,0x0008f93fce58fb1b,0x000654a7158c23bf,0x0000f15b3737a4a4}}, + {{0x000add2d842ca726,0x0000ded9630539d4,0x00000be14a71e439,0x000cf5fbb09cbe67,0x00008d69d5538bef}, {0x000536737183bcfe,0x000a5370dff7a45f,0x00012525b7152b7b,0x0003ccef887baabf,0x00007ac7bdeb6d1e}}, + {{0x0004f7881fdad909,0x00057d2cf6ab2591,0x000054de5cf638f5,0x000350290bc03fcc,0x000032811a7a8b06}, {0x000b3309bbd11ff0,0x000fb40449742f00,0x00051d26676108a6,0x0000c1801bb9e0a8,0x0000dd099bebf899}}, + {{0x000aaaaabe329866,0x0009f0d59c2758c5,0x0003073050fe9dd2,0x000b7824951ff48d,0x0000c23f829e6529}, {0x00022180b136a798,0x000df7a2099650bb,0x000bb4da67e2174d,0x0008d9ef00a4b9c0,0x00009a25a186fdde}}, + {{0x000a27ec11ee01d3,0x000ab5f10dfbf728,0x000ec893cf900553,0x000d76e89a83c802,0x0000ca5bdc153f66}, {0x000153797eada9f1,0x0003002562309878,0x0003c69b359c50ab,0x000449246042d932,0x0000b715a6d3c460}}, + {{0x000d4766ae06e0be,0x000dbd42e25fa41d,0x000b25a20cdd7888,0x00027d2f395f7456,0x0000adfe0af6700e}, {0x00052a9699500937,0x000ac27f8d40b09d,0x000df886a3525d9c,0x000ec248235a9467,0x00007e4b0dd735fa}}, + {{0x000b20a517d7061b,0x0002bc2df683115e,0x000c6fc6777fe343,0x000e82b870ddc7cd,0x00001610588bb87d}, {0x00084cad9c4ddbe2,0x000c1d754be23435,0x000e6c894b3164f1,0x00004be731ed3ac1,0x00006327dec6f6b9}}, + {{0x000c6de97b5cd328,0x000e35eceecd9d49,0x000ded7fe40835da,0x000804466350edd9,0x0000aeebb5cfa678}, {0x0002fb75b8ee9ecc,0x000cce3ca11851d4,0x000f4400ed7a17bd,0x0006f380d7511a2e,0x000048990ad475a6}}, + {{0x0007d2a2199e347e,0x00054a39e0518de0,0x0006e51dcbee7555,0x0009eb7691878691,0x0000b1913142a2d8}, {0x000610d37d341eda,0x0000b6d51c2b6679,0x000492dba434fbb4,0x000493454b7ee7d7,0x0000a33a79af9021}}, + {{0x0005054e4bd6d3d2,0x00043ab551d049fc,0x00042d3a609540f0,0x00023b6acc908549,0x000031af02f4d283}, {0x0008cac0992c163c,0x0000c88e3bb49345,0x0008c268c1fef8e7,0x000bff67578da5be,0x0000c8be793a805e}}, + {{0x0007baec61c3855c,0x000c98c1fd3b2926,0x0000b93b8ebff429,0x00095262d886c08c,0x0000a5e00b2eddb8}, {0x0000117c3fed8b79,0x0009f19c01f6cf33,0x0000fbd54d49ac6f,0x0002cedddaa6bd3c,0x000017430691049a}}, + {{0x000981eaff2ef812,0x00050818ae80d67f,0x0002aa892c3654d3,0x00032861d050441b,0x0000db067bf5d099}, {0x0009e86703dcc974,0x0007a133e215e7c7,0x0009a7a5ce66f9b3,0x000b618df119a6e3,0x00007c60de3c76f1}}, + {{0x0005939d860f1b25,0x000ca5ed4d4a6e40,0x000b6bcbd3e9a1db,0x000496ef23619ec9,0x0000ee790cfc34e4}, {0x00034b15bdaf9bbe,0x00066ca295f0f0a8,0x0008e378c02cedda,0x000ea36619aa2bcb,0x00005613245ac987}}, + {{0x00022cc76b23a502,0x000cea6c21ce0bc0,0x000cac3f54a2793a,0x000d561832878089,0x00009176f1beba26}, {0x00061874f6f59eb5,0x00093bdc658e0629,0x000e3040286e9bca,0x0009badca9c4d357,0x0000438b216a16a0}}, + {{0x000063c7672765ab,0x00035547b9bf0a6a,0x000b1a63337a3ce6,0x00096092c099c898,0x00005ab800db5ee6}, {0x0003f5911a5acd6a,0x0006a6201063f196,0x00096210abaee615,0x00038b96d9a649a5,0x0000ed04363bba71}}, + {{0x0007d1ca4a82b765,0x000ea3806be9cf81,0x000dc6bb55586960,0x0007eb2ab67c8909,0x00002ace7a0614fe}, {0x0007618cbbc9b701,0x000a504ca5e1cd98,0x000bde1334f06fd5,0x00026480af14ca6d,0x0000afe4322a48a3}}, + {{0x0002ca6c44b2c6c4,0x0008cef87dfea70d,0x000696377ab72679,0x00069ff10f64dc2e,0x00009b42e688c812}, {0x00044c3cea0b1760,0x0007cb2691820ea4,0x000ba9dcb53a8ddf,0x000d33f3e674ebbb,0x0000d2878a8d8669}}, + {{0x00035d5d019b6a39,0x000db06f1e4604b9,0x00057c111bb5cf88,0x000d7811912d165b,0x0000803fc21a9ebf}, {0x0001c9ec07764a90,0x000eb75bd0554f23,0x000e6c9ded93286e,0x000c9083a9457d8e,0x000046959156087e}}, + {{0x000dd8a58d6cd461,0x00057e6634d214c6,0x0001bc3289cb633b,0x0007e5b1305047f8,0x00002ede0e236a17}, {0x000ca62065a6f4f9,0x000cd7be487b332c,0x00047ed1cc3a47ec,0x000b13e41eb1870f,0x00009e66e5977598}}, + {{0x00044ca63d0ff123,0x00048610a05f6f05,0x000ad7b47e5efc78,0x0000c0c72917b17c,0x0000ff6ea2122cac}, {0x000791bf21db8b7e,0x0000f7d93565cc23,0x0004bdaad7dac70b,0x00016c882cda1d69,0x0000b88bb8cf0235}}, + {{0x00034b4dfdbeb1be,0x0001d4ee4deac4c6,0x00052482122f5ca7,0x0008b13045a368e6,0x0000d9e8a3fe52b1}, {0x0002cb1b961f49a5,0x00012b0096709b7f,0x000507a6d7fee2ec,0x000f1ce50d875422,0x000061bd7119db55}}, + {{0x0009ccc320bbcaff,0x000eef1de48c4c18,0x000a8f128568434c,0x00083b7af1b00e0f,0x00000ba9d0379075}, {0x000400432ff9f60d,0x0005f25dcf33735a,0x000c74cef3dd8e4b,0x0008ad22230f1642,0x00008117623d13fa}}, + {{0x0002876f51fe76eb,0x000b61d625893682,0x0002257188a6811c,0x000bcd13fc7e6546,0x00007df2ca0782fd}, {0x0004e52dd7b205bb,0x000797a2e4143b1d,0x000a91148b695947,0x00067455e4d793ef,0x000047ed447ad2e9}}, + {{0x00098b904c9d9bfb,0x00006b7930481a70,0x0001ee461661e288,0x00046f01a16966b0,0x0000c521308d9547}, {0x000a0fc2477de506,0x000c1dbd51efc909,0x000294905d80bb41,0x000f97485be7ec53,0x0000d465b18e3958}}, + {{0x000f330fb6840fd2,0x00041401e6c816f6,0x000b5b4f8faaeb21,0x000c4b8f83d30fcc,0x00002885739466de}, {0x000367c7bc467dfe,0x0002f842d27a51b4,0x000ea14a6926562e,0x000cd8ffcb66140f,0x0000b394dafd2734}}, + {{0x000e5d211c0be981,0x0001714e81653eea,0x0002bce1cb1e6ed1,0x0004da091086bce5,0x00004b74cc6b75a0}, {0x00011868c060985e,0x000d4dbd7f7c63cf,0x0000942ca071047d,0x00061c6e433b8bce,0x0000cbac448b8fec}}, + {{0x000d0e2ebf3232f7,0x000e552a2edd8f0e,0x000b55fdbfff80f9,0x000c113d9ab43375,0x00003ca7821542e0}, {0x000e0a0e6251b462,0x000c2c0d932d6dac,0x0005da19a89bc6b5,0x000dfb1438cd7709,0x0000f24a939ad48b}}, + {{0x0007e46766561b79,0x00057ed0322a99b4,0x0008e1865736600e,0x000fff76a47cb163,0x000027c1c2e5b135}, {0x00023370cc5df696,0x0001a9d649a92954,0x000efdb2799b37c0,0x000c2765f0043c6a,0x0000cdd99877be95}}, + {{0x0000931390420d28,0x000b8983efa46985,0x00039aead299c40a,0x000192ba05e778af,0x00004274408c3a45}, {0x0000fb991a711a0c,0x0007df52ab176bcd,0x0003c6ed6461592c,0x000066f49302b4da,0x000051fddc7f30d7}}, + {{0x000beb6da50d5311,0x000d96a7b9da94ba,0x0004bdc89521b840,0x0004494305151e40,0x0000bcde201e0d07}, {0x000a78b3b76a59a8,0x000d87791a1bf427,0x00091ed1cf84841c,0x0009436bd314bebf,0x0000e61d34d3f172}}, + {{0x000c4515541b8923,0x0001cc9d9e541d5d,0x000bf610db186ee4,0x000a9f6d9f345ed5,0x0000e7ba65e26acc}, {0x000787aa83694867,0x00071eb5ba539dda,0x000481bc309f9dab,0x000103eafb2033d6,0x00006f4ce311fa62}}, + {{0x00000cff4f066b52,0x0003261dafc2a8fa,0x00038999889ab514,0x00090324339ed7a3,0x0000ff862f1dc214}, {0x000f985b05556e3e,0x000d5467081e2c88,0x000c637eacd96058,0x000cdb9d6a4176ed,0x00001743d0a16a5a}}, + {{0x00072e27eb37726e,0x000d3481a03766fd,0x000f4aa79f7fa264,0x000e21bfbd3bde45,0x0000d1e0148567c3}, {0x000f97982e7abe2a,0x0001a5f633f87621,0x00037bf3a19eedc7,0x00094e469b155e61,0x00000ad13cee14ee}}, + {{0x000d5241c0e651a2,0x0009e2ce227e93e3,0x000b27ecaab1a6e2,0x000f39c7af17974a,0x000045446dedd444}, {0x0002a2156c07613a,0x000e5427549859e2,0x000fd094643deafc,0x000ede70834ccb67,0x000075841e5e7406}}, + {{0x000a6777b0ac93d1,0x000d68f5e0d7ebd6,0x000f5492ba6e37b0,0x000f3a1516c09676,0x0000e4bf888aac05}, {0x0002ce04df0ba2b4,0x000d1062341bcdb4,0x000acac20935d5cf,0x00000e4a30333382,0x000029438c49198b}}, + {{0x0003bc9049d33fa6,0x000a346f67ff1d08,0x000a1d6a358b82dd,0x000ac84c3e2db867,0x00002e6bead7798a}, {0x000980fde46c58c0,0x000969c8d7befc85,0x0007b35eca7f6937,0x000c0c2355792783,0x00006a933d8e0790}}, + {{0x0000e9b077ff55dc,0x0007fb26e680827c,0x0009cb54f5397779,0x0003ee995308741d,0x0000ca3f44a0aac5}, {0x0005c87a07eda0fc,0x000f3d6400c811dc,0x000e5da72c138bcc,0x00017d949680d313,0x000093eed8305406}}, + {{0x000b1574d0b75c00,0x00026386075bfd3d,0x0007b2c169716eb4,0x0002010639605c81,0x00009915109f1e4f}, {0x000a9285cca6c3bc,0x00097505c90035c9,0x0000480c4b25f7d1,0x00001c2b9f7d2063,0x00003c7b8c6ea1a5}}, + {{0x000183c5a1f8e24e,0x0009bdd255f03f99,0x0007f62a6fdb118f,0x000190d9b18b90c2,0x00008f732f8196ec}, {0x0002d910be786ab0,0x000f0ac5a0f5524a,0x00025f6945d32ade,0x000a899b53d4d697,0x000032a76c60510b}}, + {{0x00091a3ebeb15447,0x000bced73ac38403,0x0006cb8b344b7b88,0x000b12624bae7a25,0x0000ceb151b5394c}, {0x00066d05bc1e6a8e,0x000a290f07bfbd6b,0x000937589ec70cec,0x000cfd470644ed7d,0x0000e9e1a3e3f1dc}}, + {{0x0000a84745b98d25,0x0000d556ed40b0d4,0x000148cb9da429a2,0x000936a676eced85,0x0000a22d40d2ed18}, {0x000b9e570e8a4cea,0x000afeae03793bc4,0x0000bd47ebfd1445,0x000531523f2c0c1a,0x00009c0bb3281845}}, + {{0x0004d600a4c3f6b0,0x00092c15ef449ddc,0x000484accbdfaad7,0x000f15ce55a2367f,0x00008653ca7055b1}, {0x0008724538873a39,0x000d1ce1c7e72efa,0x000e332ba09299e5,0x0001b677afab66ad,0x0000be1fdf722dd7}}, + {{0x0005d595758b11cd,0x0002f8654f40a49b,0x0003794470b85289,0x00090be63ef6f452,0x00004957d29e05e6}, {0x0004363646559b01,0x000c39788a8e7d48,0x000ce54a9f4a8273,0x000a9bde406cb834,0x0000e1c2610086fd}}, + {{0x000e228cf6a4a81c,0x00025b488772e150,0x000a9c15b1fa3b6a,0x000a465e6ff110c5,0x00006133b924ad6a}, {0x000d55c9dffa978f,0x000c6f3965f28ac5,0x00032b52fba1d1c1,0x000a070969f4e077,0x0000ceecdb695172}}, + {{0x0000a5f10f2b8f5d,0x000e8c4c2f63b012,0x000f9c213c83a6cd,0x000bd47d47a491f8,0x00009e1cce6a3f1b}, {0x000bc7caba7e3721,0x0008cfd1a2db0d91,0x0004618e5fcdc74c,0x00025df5efa80037,0x0000121696925a79}}, + {{0x0009823f6021c5dd,0x00041ff14423d4c8,0x000cd1396880d5e8,0x00078a9523bc5a6d,0x00001acfdfce13c9}, {0x00064e8bbb66840f,0x000d82b58459b0c1,0x00038e8ecf7f4301,0x000698d29ad4a6a6,0x00005ab896236b78}}, + {{0x00079740e9547505,0x000814f9d2c69dbd,0x00085232e0121de8,0x000776de597b42d9,0x00005b6c3c5a3451}, {0x000e547519cb9fb2,0x000f6428600dbb53,0x00081791af134019,0x000c083a473176e0,0x0000f3e226355fb0}}, + {{0x000301773d273b06,0x00061721ef9ab28c,0x00050dc39ccd2107,0x00045da54cc292b6,0x000062246dec1880}, {0x00052fa6b83c0d12,0x000777e9cd46904b,0x0009725e4a72df26,0x000f22686b43cd89,0x0000b651688f849f}}, + {{0x0009b7902f345331,0x0003fc77c1486047,0x000537c785e354c1,0x00095fa4bb7581a8,0x000088043d7ffe14}, {0x0002f428c1d50261,0x000683d4aaab9ba1,0x00057c4502e0c8a2,0x000fefadba7b8baa,0x000040c9ad6abbda}}, + {{0x000aa4225ac0f182,0x000ae4d1fbf32067,0x000b04824f7b1295,0x000e90f4829111a4,0x0000ce3f19253bd5}, {0x0001d558f2e1b72b,0x0005802aa2439c7a,0x000be9554fe93228,0x000a6d997ca7b4d4,0x00008e821b990547}}, + {{0x00038be67e573e06,0x0008e084c44bfb28,0x000c1c2c505891db,0x00044b3131137396,0x0000aebfa4039584}, {0x000dce9e56e55c19,0x00029caa46d0ac9c,0x0001fe8eb7148ced,0x000f4c9e10c7efb6,0x0000fd835db8f97c}}, +}, +{/* digit=29 [{1,2,3,..,}]*([2^203]*G) */ + {{0x000a109081e9387c,0x0006cc935828a36d,0x00040b015fb9780d,0x0006fa15940332e5,0x00009d7b51ca0f46}, {0x000cd41d6d9f6711,0x0008a1a2ac17faad,0x000201e5fba6c1e2,0x00062af66a7833ed,0x00009d9971a090f4}}, + {{0x000f462060b5f619,0x0003ebd057c2f431,0x000e1bf65a56f46b,0x0001fea48dca6c47,0x0000a38783ed1bcf}, {0x00033a9da710718f,0x00063e0aeaf67a5d,0x00029d1875a77998,0x000732da87314d2d,0x0000a0edc3fb687d}}, + {{0x00036216a31e09bd,0x000cf1350e359df3,0x000a0cf52de89e44,0x000537592148714c,0x0000f379672db88a}, {0x000510a2591d61b1,0x0007485b447bc92a,0x000287f7779aa87d,0x000a80e67db604e5,0x0000697c8bf6efe7}}, + {{0x0004849cb198ac73,0x000cdf2646651c89,0x000200678a884a93,0x0004e5fda964ef9b,0x0000c351b8730983}, {0x000ef9fe2c4b44b8,0x0006b326790cafb2,0x00002264a580f6c4,0x0004e2384805210b,0x0000ba6f9e2c2a19}}, + {{0x000975f8fb547385,0x0007d7c3ead3fc87,0x0004a085a3516078,0x000996334116d2b7,0x00003c99a73f62fe}, {0x0005be05b81c51b3,0x00088e0852b78758,0x0004d19a7925bafa,0x0006d446a4fafda8,0x00009a4598288520}}, + {{0x0006ab65eb03c0ee,0x000392bc3fde499b,0x0003a80d2f19b795,0x00019ec86b5b9c6e,0x000043775094d428}, {0x0003650bb3ee8a3e,0x000bd132075fc166,0x000d834f675eb14f,0x000ffcc8ccc9067a,0x0000a6a2475c6e92}}, + {{0x000fd950f8d67583,0x0001108c07dd9d72,0x000e23221cb84e10,0x00042c89114bfda5,0x000058b5fe3194e7}, {0x00077ec95f40e756,0x0000dd73f3d61c05,0x0001b9b66f015545,0x0003c73d55cd67bd,0x00003e86e790f8d6}}, + {{0x00034abd3c095f18,0x000e64b76d7139d9,0x0003e698404b261b,0x000b109d2e6970e7,0x000079fb23bde5fc}, {0x0006c72dfd754907,0x0004f1bcf1c11150,0x0005e70073a97d08,0x0002a6d3201d82bf,0x0000f0ac52fe9823}}, + {{0x000cbc46eb564d4c,0x000bde570e292715,0x000f5fd5d8d6c752,0x000514380247c89e,0x00003c66b47953eb}, {0x000b4010f87de563,0x000f96c603b59666,0x0004fc942ce62c06,0x000c197e7b4c607e,0x00008ac0b77963a9}}, + {{0x00020ee4b049136e,0x0001556a4613cb4d,0x000e081288b63bf1,0x000b153221aef670,0x000062d8c522acb6}, {0x0004a67379e7896c,0x00020afd7fa571f6,0x00041ba6ab25237a,0x000e7e3077bd9838,0x00004ac0244fcd16}}, + {{0x000a86921fea4ca8,0x000773dfdac1548b,0x000685fafd36d081,0x00059989d8d71ff4,0x0000eff66bf452c4}, {0x000aee70b57235e6,0x0000a106712b182f,0x000fcdcb0ee3c39b,0x0004b9f107331fc0,0x000069fb9dd05105}}, + {{0x00001fb319d76820,0x00090a982feeb251,0x00061b344b029312,0x0001fa51c1c9b902,0x0000e008c5bbfd37}, {0x000dd1c0278ca331,0x0006d5aa53b1d866,0x00013a2cf666f76a,0x000836d5cfb77960,0x0000d3a1aadb3521}}, + {{0x000253173faa4851,0x000fb0a76878cedd,0x00011667dc8ee6c4,0x00095acdbccfc92a,0x0000a418ea92c2f6}, {0x000bd9251f73971d,0x00020a2ed89fdb11,0x0003e03193e4b3c8,0x0001aeca44f3f4e7,0x00001e3de1000343}}, + {{0x00004ff50f75f9cd,0x0002ae752b223c56,0x0009a11181d8eddf,0x000d7a3ef074dd3c,0x00000ffc1739cb86}, {0x000ece3037d90f29,0x0005d055856cabd1,0x0004c6dafe3f307d,0x00099fb22f93287e,0x000002aac66c3487}}, + {{0x000e7bf94cdfadef,0x000c8fc6d634b6a1,0x000fb63f86c97e1e,0x000404762ad24da2,0x000081be1ba95928}, {0x00065e4d14b4206e,0x000dafa0db6586d7,0x0007fc76cbecc2e0,0x00024dc28838e0b1,0x000049a602bc37cf}}, + {{0x000131a567193ec5,0x000a95f6e70b76b4,0x0001eebddaf3c305,0x0008314587bd3903,0x0000709def8c1bbe}, {0x00099830eb2b6692,0x000b675b70295705,0x00064ac164d80ce1,0x0003ab638a7da803,0x0000f431d23de1c8}}, + {{0x00012a6f9294dd32,0x000f7b4b0d77e568,0x000e8305cb448d01,0x00063ed3ae606104,0x0000bead645b4d8c}, {0x000434d84fd8b072,0x000fd7a9dee50a85,0x00055bd85537b983,0x000f8bcdcc5f18ef,0x0000041af6241c6c}}, + {{0x000874cb940c71e3,0x0009ab5f4b3a8e52,0x0001b1dc3211935a,0x0006206435049230,0x00003d2646d59958}, {0x000d64bef9114044,0x000a5a3c5ef416b0,0x000352c789d1f25e,0x000427e0f200eb4a,0x00003929f2c8bd0b}}, + {{0x0006667c7196e295,0x00004391be48a565,0x000e0cd6e7992c2f,0x0009bf5aa97cbd9e,0x00001b0310c8dc8c}, {0x0008acfdd9f22cb0,0x0001b585d584237f,0x000416388bb1d81a,0x00074f8d5d85f58c,0x0000d6e5a5a42fe4}}, + {{0x000276638235d4e4,0x0007096e3298e781,0x000175bc81c62bd6,0x000d4d4378660c3f,0x0000d04e18957afd}, {0x000160185a8068c5,0x000142b29a8532a8,0x0000d8a3bdb58e4e,0x00003b98a65b86c7,0x0000f0e6f4ee8a04}}, + {{0x000968469ed23707,0x000c9871ee260812,0x0009c5b0534dc30b,0x000c86ca5ce9487c,0x0000d487b80b3a90}, {0x000ba37dd0e71799,0x0001240418114089,0x000747ba545f8019,0x0005918c3e105898,0x00008c4e13afe1ae}}, + {{0x00036e6e82c9f9e4,0x000c833a1043d446,0x0008aec05711db87,0x0004aa2f431263aa,0x00003ff120d6744a}, {0x000892fae77779bc,0x0008ccdc9f82d3bd,0x000c5b1bcf0fe0cc,0x000a720a5f7fe6f1,0x0000c63a68304929}}, + {{0x000ba0c09dbe19a1,0x000d5b5c73c2c7ea,0x000e50c302f3585a,0x0000ba7ab8924b0a,0x00007fcd27a738b3}, {0x0004d3410b3d5a5a,0x00018a9accf1af41,0x0006a624209c107d,0x00064175dac49f94,0x0000ec3df2b7707d}}, + {{0x00092b73f894ae03,0x0005875f18ce2c24,0x00053cad0f59df3e,0x0002944cb740d28f,0x0000eb585fbf4f01}, {0x0000c8632c7f717e,0x000acf943f4c17da,0x0007c51d2eb8c795,0x0009486ee23fb5f6,0x0000f18757648889}}, + {{0x000bdb20389168b6,0x00088a577d03a6b4,0x000743082c4ecd25,0x0008ccda63782b55,0x0000f678f4d272f0}, {0x00011cf65e58dd88,0x000e5402c0cd5535,0x00037c14cd53b4e3,0x0002a9b7de3e29a0,0x00006b6c51740571}}, + {{0x000da3eb38dff6f0,0x0002ea636be82834,0x000dd37f8be012c5,0x0002db292d238c61,0x0000e54523f8f814}, {0x000b436036a05d8c,0x000e5e93c0ffe31e,0x000821ddf83e3cdf,0x00033a7fd2fe0f50,0x00008e19b0ebf9eb}}, + {{0x000943fb569a5fec,0x000414342d75c8cc,0x000eca000ad0090d,0x0000eac2090b4bca,0x0000a39687fdbd41}, {0x0000df765959d77e,0x0007bc964999e7bb,0x00041545139d7821,0x000107f87f62e8b2,0x00005efb7759ed76}}, + {{0x00011a4e822f0d08,0x0000da8704f83ea0,0x000c6820fbc647ad,0x000bec3b315b3550,0x000063dec3e37e76}, {0x0005d3af017bfc7f,0x0008a76b822901ff,0x000bd0d3b2005443,0x000d0e167fca370b,0x000063dde656f5e3}}, + {{0x000efb32a4c94e9b,0x000de6f8278a22db,0x00003793dafbff0f,0x000d28d0aea0b135,0x0000223802a0f06c}, {0x0003e578ec3fecad,0x0003693e70536570,0x0006734c406c3831,0x000f1dd0b751eb7c,0x00002e8a436959f0}}, + {{0x00090525e9ca895c,0x000dd72072df147d,0x000c6755c2f4dd31,0x000557e16fda8ee6,0x000066827008f196}, {0x00076a30cf43895a,0x000fe3c3097b1f1a,0x000390e0ea9d604d,0x000eff4190830966,0x000050bf75453c85}}, + {{0x000bddef6a702510,0x000b3c6ab16a0696,0x000d08762548b801,0x000c4e37fcf704a4,0x000090b3defdff76}, {0x000cb8969cb91584,0x0004595ece4387e8,0x000d9fbf544a9074,0x00082db85395f40a,0x00009b0f6c58fb0c}}, + {{0x000bc15adf7cccff,0x0005efa1e1b075d9,0x0009bc17e81a3e5d,0x000d429c39e44424,0x000037dccb37ea7f}, {0x0004873907fba12d,0x00097a372904da65,0x00083a6c535daa6d,0x0005be3564cfc662,0x000009fa4f71a939}}, + {{0x0009ec9aeb19a362,0x000ea7bfbfb4688e,0x000c2faa6d913f1c,0x000c12597b9a3c61,0x0000f979bec8a0a9}, {0x0009d0f359679ec3,0x000cd79b0460b596,0x000fab870ebcf523,0x00039ccd6b000810,0x0000f2edcdac373a}}, + {{0x000f9a76f568431c,0x000b42f8898c0d64,0x0000b5bd5f848c27,0x000ee83418ade126,0x00001f3e3242973d}, {0x000319c26c185ddb,0x0007a46f0ac446e9,0x0007f9d576d85b7d,0x000f47927965f224,0x0000519b63760035}}, + {{0x00063a9ab87d59c5,0x0002e9caaa116b61,0x00077387bff9f58c,0x0007f8fac39cde31,0x0000f6557c2d73e7}, {0x000400636a830417,0x000a05ef196c6750,0x0008c79409b1c96c,0x000316834283deb0,0x0000ea096448128c}}, + {{0x000b3b56aa39dff0,0x00029f8e4d8cb510,0x0004c4b9f59b43da,0x000c01a8ce31fd9e,0x0000e20be26c1303}, {0x0007182e8ee47c94,0x000b3db981011818,0x000e14ff6d9687cd,0x0005723a520e4da1,0x000029808bac836d}}, + {{0x000a60d4944b6639,0x000913f91ae5a37c,0x00036e3b1f901f7a,0x00025054e3e76e9e,0x0000aa219cfb9d93}, {0x000e275056a2512b,0x00092e65d95c347f,0x0009fc3eda4d643d,0x000bbde669d39669,0x0000598dee37f8c6}}, + {{0x000c1e5dda9e5c65,0x00027aa9fc95682a,0x0002bea444e0d3c7,0x0009c7c7faaade77,0x0000ef8428cfb000}, {0x000e47a460ff0166,0x000f125281cbcc4c,0x00023aad2da6d12b,0x000e27e4c6784802,0x0000e342afa96256}}, + {{0x000bb0b93a37c047,0x000b6d10bd961400,0x000ac46b762b1bc9,0x000f510251adeb0d,0x0000d33b92eebe4e}, {0x000a94be61fa29a5,0x000eb642223328b2,0x0000d8d374b2be13,0x000004e6d6d06233,0x0000ef80e1f028ca}}, + {{0x00046996d16768e9,0x0009d28bf217174d,0x0004e490d9fc4ff6,0x000979e7705a9415,0x0000d96dd291d2d9}, {0x000d9d8ce5d72c41,0x0004b11c714f77e2,0x000e4a03e9d06c5a,0x00028af2aa513679,0x0000386b3c2130ff}}, + {{0x000e8a6fb283f61f,0x000503abc3fbfe82,0x0004d36227df203e,0x000760fec7c3513a,0x00007d17dc0cf762}, {0x0006e44522055f0a,0x000aefa748dbc395,0x0001dcc14de3012d,0x000f3a2a9fcb63bf,0x000056d9dd05e4e2}}, + {{0x00086b68bcec9c2f,0x0008780b9f06b861,0x000d292817cf24df,0x000e11d46b45eac0,0x0000ff42bc5f7b10}, {0x0003c404d289427b,0x000904848ec41226,0x00040800c3d5f189,0x000b1f61f97010d0,0x00004c5f529e00fe}}, + {{0x0003f8fde94fdcb0,0x0009c7c2f05ecc54,0x0002692e1e96af73,0x000ae9aa5e003688,0x00009c75b68950d4}, {0x0003df2b5932a7a7,0x000e6e0979ad62f6,0x000e696312658252,0x00066aba19343fb5,0x000018c7501c25b6}}, + {{0x0002d69ea40dc3a4,0x00026ecc018f26a4,0x00070f04adc84ad2,0x0002ece5c36c7b32,0x00006ba6d4790fa7}, {0x000d1c593e58a8e2,0x0000f20c088c6c37,0x0006e86daa239473,0x000038a3be4263cb,0x0000c417d369126d}}, + {{0x000f9c58b6f8efae,0x0009577185365b70,0x00039c92b671a2fa,0x0003c1f3ced3c6b5,0x000056f1bda83120}, {0x0006ec49ff3c8eb5,0x0001f3491cea8b09,0x0009437942deae43,0x000842d465c6eb17,0x0000d267e6670586}}, + {{0x000116db07159d03,0x000f918962109d3d,0x000961579ae07a67,0x000dd65fc84d87bb,0x00000009e494c1f8}, {0x000af22e31328195,0x000ca23ab4ff8a8c,0x0005dd687cffa197,0x0007a208103a4420,0x00007b796c35ded6}}, + {{0x0003a6ca1779ad79,0x000da57c09c50b9c,0x000b4a57ea33cfe2,0x00052d9ea293153d,0x000019596961ebeb}, {0x000b9a6e546c8792,0x00044295c8d6118d,0x000ec806b8e996df,0x000035bd99048455,0x00004f291ca365c1}}, + {{0x00023bb440e22296,0x000213ef4d04cca5,0x00011ec39324673a,0x0008d34f3adf343e,0x0000136d7f23c596}, {0x0002899b053a9270,0x0001ae067ecd7a7b,0x000779cd93eaa266,0x0005ea8549b9c802,0x000061d7940c5338}}, + {{0x000a883f06d18bd9,0x0003227008433e0b,0x0001a9e4d4ba6de5,0x0000ed2966b66859,0x00003f675680f4fa}, {0x000711b4347237bd,0x000f1794608e5a02,0x000f73d8cbc041e2,0x0004f685af10f570,0x00002d4d4f88b756}}, + {{0x0007a89b3e93ce7b,0x0004ad3a2c1bd7d2,0x0005b218af7b5a87,0x000754029e68a025,0x0000533837f3af76}, {0x0005a73579fab2e2,0x0001ccd74385d1b0,0x0005e9115b41055a,0x00074e9236927444,0x0000972a7c515202}}, + {{0x000334ef678e68a2,0x000079b057ed6c08,0x000ccb69a4e4160f,0x0007721cfe11b852,0x0000fd1823a41c8f}, {0x000072f3298f0557,0x00098ec74a6edf7f,0x000b4d0418c0566f,0x0008507549e0195b,0x0000c3930bb0208d}}, + {{0x00041fcaaa2902bc,0x000924f69ad3e071,0x0003f9ffd539ad79,0x0002f6e6453f9481,0x000058d3c48f75bc}, {0x0006fad5dc64e964,0x00097240e354b332,0x000a1e7a93aafcaa,0x00089fdd1b0903ac,0x0000ceb97675211b}}, + {{0x0003e49e32a858e7,0x000e3e907badeca8,0x000b9b4944c32892,0x000e1b65b42ab62e,0x0000fde3ee28eaba}, {0x000ab09caf549579,0x000e55f5d5d513b5,0x00003e2c0bfb028b,0x000843028a065020,0x00000793aacf7476}}, + {{0x0002e79c81710a00,0x000627ccadd45e94,0x000cf6d0c557e4a3,0x00080c72a2bc564b,0x00009ee5f4326d7b}, {0x000dbe9d4292f19b,0x0005b3f16b186b70,0x000fbb42a56f74c2,0x000040123db0f735,0x0000606bdf71ae10}}, + {{0x0005d4d044573ac0,0x0006556b0ba41eb1,0x0000df6f77dc3cf8,0x000e8c97af9a33c6,0x0000b1ef85ca716c}, {0x000f884c96958bee,0x0003556909632922,0x000a000617c32fa9,0x00065b4d7f667cea,0x0000aaf7c1815473}}, + {{0x000de4687032d585,0x00030e2c79e01eb4,0x00004ef23c54f3d8,0x0001b3b7818df45d,0x00005faa9c8b73d4}, {0x0004f6f89b95355d,0x0009e7415c84ced6,0x0000ebad34860d2e,0x0005be8fdb9bd205,0x0000b53e0cd3685a}}, + {{0x000c0319feb65939,0x000fdaccff17b830,0x000555c10dd87f30,0x0000649303ebab9f,0x00004603695b87e7}, {0x00011c32e83358c6,0x00048efb0178f883,0x000ba8652508dd9b,0x000be51ca237062d,0x00002aac5a36047a}}, + {{0x000d2a08b1ea7b31,0x00039e8b14859a61,0x000052f99d495ab6,0x000eea28740f8487,0x000078ebe5bc2974}, {0x000bcca5b36d17fa,0x00030af86eea030b,0x000f8e9e0b5e4cce,0x000e75151a022068,0x00004348796a9eb3}}, + {{0x0002309eef1a7522,0x00074f2aa1edbe59,0x000007dd25d7162d,0x000d228ebfb5ed0f,0x000055e14b2e89ed}, {0x000e0720303b6972,0x0005d05720ffba85,0x00028ebb6c5d17e2,0x000112e2b58d6e51,0x0000c80242df754e}}, + {{0x000ca5fabfae1ca1,0x000c0a21459b919f,0x00066a4d2937afaa,0x0003318e0ca91c1f,0x000094cc7f333ec1}, {0x000143a8aa116906,0x000ca9b59e08ad25,0x00050860abe40ad8,0x00034bd7d60d9be7,0x0000c53b00926bf4}}, + {{0x000415d1356eb809,0x00030578ded8b572,0x0008fb38bb8bf9da,0x000b2192658e365e,0x0000b70ce22eaf8c}, {0x000018a829a81803,0x0003881ed2957c00,0x0003cea8384329f9,0x0005364c343ea25f,0x00008f8655f97586}}, + {{0x000a0d01d3ec517a,0x0001b12321aea661,0x000a925989874465,0x000fea684ca591ec,0x00009bb9dc9bdcb3}, {0x000435578b4c2405,0x000b110cafdc14c5,0x00038846b5ed62a3,0x000160b7512f371b,0x000071bb70b00e38}}, + {{0x000b95b2da705d20,0x0006b1a08f98b556,0x000ecfbe53ef8ada,0x0005cd85302ca7dd,0x0000e53057394310}, {0x0004d5521a9255d9,0x000162f3802a6055,0x00047787563a32fa,0x000da0a5c8c5b0cd,0x00007f458eafad42}}, + {{0x0007080eb6b242d6,0x0002db71e246832d,0x00031139dd30bd02,0x000e531027991bbe,0x00008797e91a62e4}, {0x000e20a6b4e185ab,0x000d92d9b707423f,0x000f7811b82f2c67,0x00095c75c817684c,0x0000d53005eb45bb}}, +}, +{/* digit=30 [{1,2,3,..,}]*([2^210]*G) */ + {{0x00049be9d8e68fd9,0x00028b044320e5f6,0x000c33398db0f053,0x000fae66fde9b3e0,0x00002f4209bf6c8c}, {0x000afcc1a739d4b6,0x000f428ab8dee9d1,0x000c6f1d009aea75,0x000aa4b4375fb5ea,0x0000420b560d08f7}}, + {{0x000499c6254dc41f,0x00038a837e7e9eae,0x0000524a77e29392,0x0005f184aec08c09,0x000082b921a7d6f5}, {0x000962e1402cec5e,0x00071a2f30e7493c,0x000b879cb9f17ca1,0x00045edcd783e8e9,0x0000a3d8c153a6f1}}, + {{0x00015e75e0dee6e5,0x00028c628aa2dede,0x00061bb9374f2487,0x0002e083e9c4fe78,0x00006d4822ab187b}, {0x00017cfc59826f90,0x00092408169eb664,0x0009ef885ca26096,0x00038fedf69d06c7,0x00000031f8adc7d1}}, + {{0x00046e60ebcf7262,0x00014231470e103c,0x0007c21094482b83,0x0006ef4f6dfaca48,0x0000e0ace9782e66}, {0x000a9d31f8d1f420,0x0001574944d23246,0x0007f334b1b1e83f,0x000d8113dfa63aa5,0x0000cf8daed9f025}}, + {{0x0008ea800ee11c1c,0x0003f5e3dd7530d7,0x0008c43c5eb053cd,0x000662db65b13ed5,0x00003ad49be7d151}, {0x0008e41b64279905,0x000d207eae1e99fd,0x000abb71e12cf15b,0x000d0dd9ad4f1b1a,0x0000143e74d57545}}, + {{0x0006336c88bdee1c,0x00059876767c3026,0x00073199625f2930,0x000950dc078571c6,0x000088690b3ad552}, {0x0002c2d852705b44,0x00040e09552d274f,0x0006575d1b0bf8d4,0x0006513628beeb98,0x000007be238bf864}}, + {{0x0003049a639fc6b8,0x0009060036250e5e,0x000cc1646e75c35d,0x0007398cf35bd85d,0x0000bcaced2ec262}, {0x000cf1db55367425,0x0006ca9e068be22e,0x0007909c5013dd89,0x000505c7f411cb8a,0x0000757ac98d61dd}}, + {{0x0001f0d1e935abb5,0x0003c54de37a85de,0x0009cebb5defd10b,0x0004be68d9e39236,0x00004d5ef9bc6132}, {0x00041ba74f17e266,0x000818c1dde44d63,0x000d918fdc0a0e3c,0x000a1346d7758187,0x0000687601562ca3}}, + {{0x0003e9cf36658f0c,0x000bb1f8057ec731,0x0006a835ac433ef1,0x00094bc53262461b,0x00008f053993c863}, {0x0008cdfe983c4a11,0x0001f3b7b931ff39,0x000b9045bbf5e816,0x00046b83193c46b7,0x0000e4ebf5db4a6e}}, + {{0x0002a6043a24fe7c,0x000e3fb3492bf994,0x0002fde0529c1191,0x00032cdf66244990,0x0000792a7ad2713c}, {0x0008ad8b737982c6,0x0009421e60e32fd8,0x00083591a7e3a031,0x000455a9b0de4473,0x0000df141eee310a}}, + {{0x000a039e6d6f4714,0x000ed198d12eaec1,0x000eee5ac14b2ba0,0x0004ceabc1a1603a,0x000001f483720b96}, {0x00037964fd03f663,0x0009ad8f3f122ee4,0x000380f183fdb4e4,0x000d163ef267f629,0x0000e8e9670bda64}}, + {{0x000180c207674f17,0x0006c3ae8fdbbc19,0x000aeb71e112e09a,0x0001c7296675546a,0x00009432af25101b}, {0x000558fde2ddec64,0x000f1357753fd5eb,0x000e1158a81392d1,0x00099167a76b973a,0x000016fbbff8a899}}, + {{0x000fdfd0d4a9dcf2,0x0008744ddf129e65,0x0008568667bc29e4,0x000fe29c1a92d93c,0x000073c69058e98d}, {0x000e418cdfaa6b88,0x0002d061c69f69fc,0x000f75e27606bd82,0x000a1ec2d495a06a,0x0000ed3d505ed873}}, + {{0x00028416ab25b6a0,0x00072b1a4523af55,0x000c99e03c6c0ffc,0x00091bab18827b21,0x000060e864890346}, {0x000f90f93c7f3988,0x000b02f8d10b5207,0x000d0f9e39f4a96c,0x0004f55d71cd793a,0x00004f435d37c3a5}}, + {{0x000c55b8e33787f1,0x0005963846734b03,0x00051b9f0ef42f97,0x0007c2ef7304f750,0x00008aca1dc841c8}, {0x00020a72d4bfe80d,0x000c353e732c56f1,0x00037ca16fd823b3,0x00096a41bccfe475,0x0000f6c9c74eb5a9}}, + {{0x00032c7904fc3fa4,0x000587e3636aee73,0x00091d9aa14a23f4,0x000540838659c3f0,0x0000a995e5df12d8}, {0x0003becf3a5598a7,0x000741eaa99520a5,0x00004e03c56534b1,0x0002682ed3dca4bf,0x000016c563b48d56}}, + {{0x00077a41d6178e77,0x000ff8a1ff8e27ba,0x00013f63de4c80c3,0x0006f3050110990a,0x0000bf33522161d4}, {0x000218e10b365bbb,0x00035fd7ea750aff,0x000b3a9258102180,0x0004e555a3fd8aa4,0x0000829e7604b3db}}, + {{0x00075a54d53e5fb8,0x0002552717e36bdc,0x000a42ec204a5dc0,0x00038206af502fe9,0x0000867e8fba630e}, {0x0005c6ebec9889bd,0x0001fb47c98dbf84,0x0000c2a1254f491f,0x00008ad3091fba79,0x00007f6fd79920f7}}, + {{0x000ac30acde5e173,0x00003852b4d7a569,0x00009ae54d0f996d,0x000061b51d4bb546,0x0000fa37d173daed}, {0x000868434b8fb41f,0x000caefb64f162a8,0x00048e1f299a2acb,0x000068c75c1a5e64,0x0000a99951b32b5a}}, + {{0x0006e892f3b26e7d,0x00000a8752476d95,0x00082dda3f470986,0x0002ef6ad1517924,0x000064110e3d17d8}, {0x0008d2cfad414e41,0x00081ed02b241492,0x000821bf12b155f5,0x0005da381a141bcb,0x00002e3c7705f81f}}, + {{0x0005de59fff83814,0x00021bbec894e49c,0x0004d88c41105323,0x00031b60d051cc45,0x0000f6db89c5f8e5}, {0x0003fd6ca563a441,0x000548da8ab934fe,0x00074f0a17f5c221,0x000a0a7445016d94,0x00007d34d61db7d8}}, + {{0x00039101c474019d,0x0009052ceefb8e9d,0x000622c2bcaff262,0x000a0529cf3e32c1,0x00004b95e3db9071}, {0x000a61f1594438c2,0x0005e4aadedffbbc,0x000e149401eb6e6a,0x000a9c653027f468,0x000021d322affabd}}, + {{0x000a9f6b7cb179a6,0x000d57934dcced8e,0x00009180ddc7b764,0x0002dd9cb139405e,0x0000629a6c0147dc}, {0x0005e4e9f5a915e1,0x00075204441ebfc5,0x0000c5f53b1db9d3,0x0005b1e82d68cf93,0x00007d3a142dbb60}}, + {{0x00044ea308780f21,0x00012845f5e4dd59,0x00024d7a3dc8de76,0x00011e5beaba7d76,0x0000e709afd404df}, {0x0004376021704560,0x000ac8f94b649536,0x00080ca68bf204b3,0x0005744e53af7c56,0x0000526074ae0c67}}, + {{0x000cef8ecd92af61,0x000a5cd1745a95d8,0x00025c3e4e6b9fa7,0x000aae2d546d3da3,0x0000f57691daae93}, {0x000f3fe9d2e1a33d,0x000edc063d35e891,0x00013a327d430093,0x00018f1da59b1255,0x0000c2134f42536f}}, + {{0x000fe2c5c2102869,0x000d8cab658caa51,0x0003572923f68aae,0x000becca23a00bf9,0x0000a626f3a0efda}, {0x0003bf3199d78e38,0x0006f1bbc345fe2b,0x00059802cb7a2af7,0x00051bbd19827a1e,0x000023bbc163487a}}, + {{0x00039f299d0a4221,0x0005e456c6fb8561,0x0001f8bd69ac3df6,0x000f879ddf65c670,0x000049f321e4758d}, {0x000f714721b7ebaf,0x000741a3312ab1ec,0x000c4d581e17df09,0x0001b2eb2fd6ecd5,0x0000d0299707fcea}}, + {{0x000a63e7882f14f1,0x0004f7c6cadce29f,0x00082bed0c9f6dc3,0x00052c36f22d6fb8,0x0000a45755be118e}, {0x0007c277c4608cf7,0x0001e68012c29f2c,0x000729b0e7ccbdf3,0x000dbf8cb0aedd61,0x0000ca2ca9f67d75}}, + {{0x000ecb16f640f620,0x000b39f51946f58f,0x00088af44e274b92,0x0009e57f4dfc0462,0x0000a91f32aeac32}, {0x000274bd6aaba315,0x000fbf6884f943ad,0x000f91e20719a163,0x000d52185d29f6da,0x0000ec1cc3377e49}}, + {{0x000de963b54a0599,0x00055fbcfdb338f4,0x000bb8da60e0015e,0x0007ac877d23d94d,0x00008724aa327a61}, {0x000885bfdb6558ef,0x0009d7899a9630f0,0x0002dc112f9f7a28,0x000458e2ae8ac887,0x0000a0642cb63c3c}}, + {{0x0006981e7dfc8d6c,0x000f5fb5b94a1529,0x000ddfd3767cd444,0x000dc64ec71cf10e,0x00007e5eeb45a8ed}, {0x0008e3d81d950286,0x000210b0e35d02ac,0x000881fe30088f17,0x000faa8c041fabe1,0x00002cf71b9399e7}}, + {{0x000dea7e0f222c2d,0x000ba2e651425043,0x00016cd30309d42a,0x000eebc4fe9ddd92,0x00006539c7ddf87f}, {0x000a57c432ac7d72,0x0000127fda1003c5,0x0000698de72692cf,0x0003b1cc28c85f28,0x0000331fb469ec28}}, + {{0x000fa322867e6332,0x0001ea9cc815d34b,0x0005e2fa578709a8,0x000fb597fe696487,0x00005cc064fbe98b}, {0x000151c493a65c5a,0x0000b31824649eb0,0x0004618e25fb5d94,0x000ab5c9e6f130f0,0x00008ecec23989c8}}, + {{0x000c88bb96209bd8,0x000b33e1c9e0cd6a,0x0008d8eac65fa8cd,0x000963247d22f54a,0x00003895ce00d33f}, {0x000ca59b56cd3d1b,0x000b2af38232a8ad,0x000080a9f10c8350,0x000b397b161fb3a5,0x0000e7f5c64eaf65}}, + {{0x000403997403a113,0x0006e21b96af2c75,0x000983ec294626cf,0x000df7131de7c46a,0x0000780dd3a82cc3}, {0x0000e462baf8e3b1,0x000d41d299aee28a,0x0007a2408abe68aa,0x000981503eb8f964,0x00004c61ed66c750}}, + {{0x0004414c53352e78,0x000b9337d46e88b3,0x0005f2bc85a34889,0x000a39e12c1560f9,0x0000a3f844254807}, {0x0009e975224da68d,0x0007f3eb00e9680d,0x0006bc37560cd6e8,0x0004c16875a98e9a,0x0000c80f9251fd55}}, + {{0x00034156ac774075,0x000ed54206816c4b,0x00007a458a1e5ea8,0x000bf9041bfa1446,0x0000dbc7e7ae6d7f}, {0x000851b31590a478,0x000995ee6df8646a,0x000b43fc0039e85b,0x000e04519fa231d7,0x00004bc8be8a99a0}}, + {{0x0002936f20df03ab,0x0001d608d4722b9d,0x00049202a2405438,0x0007b6c6b6ba0491,0x000021c3831e670e}, {0x0003059d6fdee104,0x000338488e71ddd9,0x000fcfb259da47ad,0x00095459cc1dfda0,0x00002abde10a4696}}, + {{0x00015fc17eab9fe8,0x000453e7097214cc,0x00032112cd6e863e,0x0004d7a9a7765c64,0x00008660001db077}, {0x000175a2c088eaef,0x000d9230b8d43729,0x0005f437913afbca,0x0008115476815191,0x0000086431bc8d22}}, + {{0x0001955c298b9743,0x0000c8711e043746,0x000969d18905fb5f,0x00094e487abf3afe,0x000092167c29f6a4}, {0x0000d2d28c511da9,0x000c266a262dfc7a,0x00063fdf0f127c7d,0x000f4669c4bb95fd,0x00000016589c913e}}, + {{0x000a73c11aa600d6,0x000d3fb5ab5274d2,0x000b700682f5379b,0x0009a7849e53a47f,0x00008dd39e5a04aa}, {0x000cf572ecaa9c3c,0x000b2824826bb9b0,0x00031a3c4ba0e103,0x0006a1a0c2198b46,0x00005ff84acba896}}, + {{0x000be22ac95aff87,0x000b45a46d092d6e,0x000ee4f8d1c9bb6d,0x000feed19062da53,0x0000b9042d12b97e}, {0x000f080830cf6bdb,0x0009bec8a6c60f87,0x0002f01aa4861d19,0x000bd523a0daa120,0x00000111675a25af}}, + {{0x000d6cf1afb20d91,0x000030671bc56d00,0x00085ea9b1369500,0x0001ac813ab0dc24,0x0000f2bed06aeef6}, {0x00082176d799e206,0x0006d271c2de850c,0x0004f591093415f3,0x000420fafb06e96c,0x000088a52e024e9e}}, + {{0x0005ba3e2a9a6db4,0x000d18f9268b3049,0x000b0f04f4601303,0x00036d7e3b0dad7e,0x0000ea4725084569}, {0x0008798d33fd3e7b,0x00093b4337088caf,0x000fd50ad1ccd8a8,0x0004deeeffe3e887,0x0000e240a571b29c}}, + {{0x000fd98ca0e7ebd0,0x000ae748616eec4f,0x0007baa99f586783,0x000c9ca5b00d8fc7,0x0000acada29b4f34}, {0x000d67d0fe723ac6,0x0004d9c36c1e36da,0x0004bea411d8e53a,0x0004e084dd342d1f,0x00004fd5e364bc9e}}, + {{0x0001f9057908805e,0x000c7ed480dd96f0,0x000fd2dd0b5b9ea3,0x000a26566c5dc23e,0x0000d2fe3064e9df}, {0x000e8926e9197e27,0x00093b502a5d4575,0x0001f213f11719c0,0x000456b64c7bece8,0x000041b9241c5f5c}}, + {{0x0007b6849a5f4f41,0x00028fc45b7d78ac,0x000f5f355f91d70a,0x0000d929b05544b0,0x00001f06bcefef93}, {0x000d25d038d05e1c,0x0004facc1d51db84,0x0008ee00b04838ee,0x000a1edda3ce869e,0x00003412058836ed}}, + {{0x000b91364d9c2f41,0x00039010a8ffae80,0x000359d417468bac,0x000acccfd2003737,0x0000a0f5ab825efe}, {0x000ad2f659d0ce0a,0x000ac785cff17c25,0x0002192c74011bcb,0x0000e7728b99127e,0x0000549d8e1b3ccb}}, + {{0x00088d8c85438b17,0x000d4c25cb278055,0x0004bfdf45680332,0x0005666cd1bc961a,0x000079ff428e06f6}, {0x000e998f059987ad,0x0001fc686de78bbe,0x0003cfdb2f6ce8cf,0x000a3628ad3c4a95,0x00001d426d9f205d}}, + {{0x000f13fc781a2419,0x000e475362a8b3c0,0x000a911843e89360,0x0007f43cd05863c8,0x0000bd0c9b87fa8a}, {0x0004d538a912a4b1,0x0008acf518fd97ee,0x00067e1e0de5e15f,0x0002565a055bf8c4,0x00000be4b4b2587e}}, + {{0x00014f2668621c9c,0x0000eb9c92c1d90c,0x000d47b3cd5518f5,0x000174ce6a0100d6,0x0000be980de26716}, {0x0003f10ddd83683c,0x000cd9cac73c500d,0x00083d5503b6cb35,0x00098693730c8b60,0x0000f1597689f0a1}}, + {{0x000cf5343ad73b3b,0x000f9f035a9484bf,0x000eeac691b528c1,0x00023f9294edf733,0x00006283e84317f3}, {0x000c9590a5f25b1e,0x00047844ee22c3fd,0x000dde4deefaf8aa,0x0003befe269ba5db,0x00003347161a5613}}, + {{0x00042198d9ea9f87,0x000b83fc1ab5c118,0x000f22cda090de5d,0x000893d04c37b10b,0x0000de20ec965618}, {0x000588eecdaecabd,0x000cb8342743754c,0x000a938ec6ca4b0e,0x000ccaa6f08bddf4,0x0000182de8a61493}}, + {{0x000c53ec8a4186a5,0x000b446d8e33d652,0x00037663cb3e878d,0x00048ab88453c05f,0x0000cd9daab04077}, {0x000197f586d5e720,0x0008c443ca59a1f5,0x00065242447500be,0x00067d78ef35b2e2,0x00009c5d26f6dd77}}, + {{0x000a79aa74d3f7b3,0x000d9f5ea4597175,0x000d1746d0428fd8,0x000278211cb97ca5,0x0000636393a171d1}, {0x000f95510350bf49,0x0008d0aae782cf2d,0x000688809b381743,0x0000061748c0e43e,0x00008021fc067a5a}}, + {{0x000a70c0e367a98a,0x00046f62b7c29076,0x000fe0343bea1bc1,0x00014e8645a68c30,0x0000caffa79099dc}, {0x0009964457bf9c43,0x000add2ead83f446,0x000c6f3eb0db6407,0x00056c38d56cadb2,0x0000b512e7423763}}, + {{0x0000e1ffce104080,0x00035a5e257de43b,0x00062e5b389ddc00,0x000161b0ae0d1203,0x00007f983c7b0519}, {0x0004d155d5231e7b,0x000c5b4f9513c2e9,0x000d0b0b5cff22ae,0x000cd5002588dd6a,0x0000967d1acc1d0d}}, + {{0x0006bc6cf777b6c0,0x000d4c6d19598dac,0x000f5cc850062bdb,0x00014f53da71b50e,0x00007012c7d4006f}, {0x000f962ac47800d1,0x000bb102ed754617,0x000b8c9d353365f2,0x0001c9a422efcb4a,0x000095cb26b44af3}}, + {{0x0006e2905f2c4ce9,0x000b0856966c3a92,0x000527015bd2bdec,0x000230cd16ab3a85,0x0000f81609ed486c}, {0x0006b2cda350002f,0x00000a1b7d36d8b9,0x0001d79bcbd05469,0x000e4dec90ebf5e7,0x0000241b6f9f8964}}, + {{0x00086432fe3cd4c0,0x000bb4bc633c7c83,0x000139f1fe0f33ac,0x000f49b4a9ecec3d,0x00005ce69cddc4a1}, {0x0001b16f5f98aaf9,0x0005df23e0efa19d,0x000cdfdd345bb71d,0x0000c9a3789fcd46,0x0000b8e29795ee04}}, + {{0x000b246ae0a6828c,0x0003b078d5aa9c69,0x000b4fbdbba533d2,0x00085bba2e42c07b,0x0000fb4879b30353}, {0x000d30b3281705b5,0x000bf04fe0818c3d,0x000604edf7e361c6,0x000e472b21649c3f,0x0000dbf6a40352ff}}, + {{0x0007c234b54d9bf9,0x0000a511c3d9c41b,0x000b2b7581374e68,0x000a958863bf16c1,0x00000e78507ae9e6}, {0x000f98d5d86f1749,0x0002f5e96fe4ab4b,0x000c5d344d74e0bd,0x000846fafde39fca,0x00000946dbd4d91b}}, + {{0x0002358fe1a838c8,0x0004e20ac9d8f5b4,0x000ce5a0b05aae6c,0x000d720e193bd8a1,0x0000f710571cdabf}, {0x000dd48182caaac9,0x0009740745cf8d8f,0x000b93e6d8c4aeef,0x000010e3c6c30af3,0x000091241f3a6f42}}, + {{0x0008eeae457a4773,0x000bbe6ddc05a015,0x000c41671d19857d,0x000d588326522418,0x0000ffdfc7e6c2c0}, {0x000525426ee7cda3,0x0009af02c3a83a3a,0x0003bbfc8341b086,0x0006917023bf4272,0x0000d15002a44452}}, +}, +{/* digit=31 [{1,2,3,..,}]*([2^217]*G) */ + {{0x000324c85edfa308,0x000407d4f3da5ef7,0x000b50c862597655,0x00096bb52f5bc0dc,0x0000f6927b0c832a}, {0x000e1ba55f2f94c5,0x0008e44b45fad08e,0x000aa455d6a996f9,0x0001f79133cb8da8,0x0000d0721ecc58dc}}, + {{0x000a92079e5fb67e,0x000a90aa725e6ba7,0x000f5d837e1331fe,0x000e207080ccf57d,0x00004cae01e5ff72}, {0x0003ee60412a77db,0x000c2f449025d924,0x000ef5a3106ff7ca,0x0007a80e75f7cd23,0x0000c957822bddef}}, + {{0x000230cb0ce1c552,0x0004ebbfb6078cf7,0x00016363b5b534d0,0x000e82ce1ef1130e,0x00007e0aa7ad4999}, {0x000ac2d79362c410,0x000091bb6cb0ce1d,0x00023df2467920c9,0x000f281e648d6322,0x0000f7d9eefe32e8}}, + {{0x000f39afa8338349,0x0002163285626943,0x00070fc102295172,0x000e6cf1d63dd541,0x0000f5fa5903ecc2}, {0x0008725e77d9a3b0,0x000a6384ebe0b66c,0x00045e24a11235ce,0x0003b106a8c11858,0x0000137b286ebd09}}, + {{0x000e1ce44ace1509,0x0008b381e97cc589,0x000c5a4b8e0f8d3d,0x0009f8c9e99b1162,0x00000d262f88d0ec}, {0x00054c9283e13c98,0x00072edc7085fbc8,0x000dcbecb2d04fde,0x000a5e857d776547,0x0000dbdf5921a76f}}, + {{0x00006950de1e5786,0x000789f72bc6d015,0x00039eca52e1463e,0x000f2f9fa684411b,0x000073c8530dc037}, {0x000a600747f91da5,0x000179cb78e9d0d6,0x000b5cef5b08d43e,0x000fd5bfc0c64427,0x0000c1d160af60a2}}, + {{0x000ae5328c8e13bb,0x000402eddcd1f98c,0x000ce06ad375f10c,0x0001ef24eb8b7f5c,0x00004669f4630a2e}, {0x000f9d05bbd8699b,0x000937976d13d593,0x0007e28d35528a4c,0x0005768923e0951c,0x00009293790ef6bb}}, + {{0x0007d6ac42bd6d29,0x00082b1f96aedb56,0x00043b28e6df8646,0x00023f7efe5b1a48,0x000061bbb05f379b}, {0x000f5f070a6a26b8,0x000cb28e6e39b6ca,0x0005fc8d370686c0,0x000dc900da06cf89,0x000004d88113363f}}, + {{0x000877b207f16700,0x00084e615291be22,0x000a3c2bf9b0dd18,0x0006e8625ae8dc97,0x00008584ef7439b8}, {0x00090a5dcd898ffc,0x000f6058ee3dde71,0x00087b1c126286c3,0x000db47db0b2175f,0x0000c334771d02a6}}, + {{0x000e9542f770fb1d,0x000f7cd7535ed99d,0x00009cefc97c1c61,0x0004f803b6c4483f,0x0000725af162a63b}, {0x000d24fc01e20ecf,0x0003aae7121f0c95,0x00077b7ecdfd3749,0x0004ad8d6ddb72ec,0x0000e079d3bf353a}}, + {{0x000e70a2e6ac8d23,0x0002e06e5c053066,0x000e59b8c9c6b5a4,0x0009ae22d3c6f5ed,0x00000d6a5c42ccec}, {0x0007c224fc0a9ef0,0x000395c16cededec,0x000de0fde190ff08,0x000433be12ec8f94,0x0000d131ab8852d3}}, + {{0x000e07e857012919,0x0008894061a842ac,0x000f4a48594793ed,0x000f4ca0e83ed6d7,0x0000eec726a89eef}, {0x000ba590c9d8005a,0x00077e79b9d190ac,0x000506a1e5feca45,0x000fa6efbe54271d,0x000032b2c8ec439c}}, + {{0x000c17373dd0b4ea,0x0003a4a054c61671,0x0008b53f137a2821,0x000b9de1760a1b4e,0x00006c0422599f93}, {0x0004b34cf671e3ce,0x0001eda9b9941878,0x000ab384881bbecd,0x000c2c58831979b2,0x0000f54feb8d2e03}}, + {{0x0007ca7fb8088fa5,0x0006fddc96c5cf19,0x0007771760142724,0x00071d52d2550a30,0x000034698989d0cf}, {0x00037b83a2aaac68,0x0002daf38d9b6ce9,0x000bf2899e9f91dc,0x000c15a598ad83c8,0x0000e706aca35536}}, + {{0x0007495f688dc983,0x0006e24c4afc40dc,0x00018775c26490cd,0x000f4ab651ec841f,0x000093ea6c3e4fda}, {0x00033437f338e0d8,0x000ae053e7b51e1f,0x0009e14d539fb832,0x0006dfc6e702da61,0x000059cacd24deef}}, + {{0x0009ce74462007dd,0x00047cb5f5b763b9,0x0005edde7b8ab48a,0x000fd9cec673d2f5,0x00001567f755cfae}, {0x0001b6b0887bcec5,0x000e9178f3c24638,0x0006266cb694497c,0x0004130e6525e31e,0x0000931de26b97d6}}, + {{0x000df7c0e58d4939,0x000fc8b73f1287f8,0x000a0c34db1ae5ec,0x0001a03368f784de,0x0000bd0a121159a9}, {0x00088b7cc863c683,0x000e0d1f4d65b00d,0x000a855933a1cc11,0x000ee8ba38e0e70a,0x00007f13e98adc4a}}, + {{0x0008667bc947badd,0x000d5a36ee2e10d3,0x00077fcac738e07c,0x00070cf93470cdc5,0x0000ee1b616f7824}, {0x0005e672e793d12f,0x000df0f186da36a2,0x000e07af7d6aa6ca,0x000cd3574d0fd980,0x00007cdc47eaa8a5}}, + {{0x0006d9dab15247f0,0x0000493a537f28af,0x000a334e77c789c1,0x0002777ac9b11023,0x0000236ac0912c9c}, {0x000bd251d7a51446,0x000a913ec4eca7e5,0x000f0abca098b9c2,0x0000f8d639dacad3,0x000042da81b02396}}, + {{0x0005c054f7269b13,0x0007b287c3857d2e,0x000a46f21fcf3077,0x000a35e0edc84ff2,0x000054417577f43f}, {0x0007899fd703431a,0x000576dd587af132,0x000c8352da438d7a,0x00024dc5c34c57e9,0x0000728edabfcc5a}}, + {{0x0008abc42531689f,0x0007110963efaed7,0x00017d9b30a51a0e,0x00028a6776fa0ad7,0x0000356c23a6dd34}, {0x0003fff8d3a3dace,0x00095d94491f2990,0x0004a56a4409597f,0x0004616cd7a5ffbf,0x000050964756adab}}, + {{0x00051265c3427b0b,0x000c2282c9bda97b,0x0002c5c456401405,0x000aec8629f8d722,0x00001c02c1798d50}, {0x000ed75d9635bc92,0x00074e24552fbea2,0x000f1d066226790c,0x000c2e1c33f2a365,0x0000a43463e8dfcc}}, + {{0x000453adb4837611,0x000555d5672b8cc3,0x0003efc87e7cc608,0x000eaf177ed6cbde,0x00009f2f36879234}, {0x00043175c0b800bd,0x0008bb6da6e29aaf,0x0004ec75e1f1e7c8,0x0008c19cfb4715b9,0x0000590dd6015311}}, + {{0x0009da11f17a34c7,0x0008b35a145614e4,0x00050363b5420ab3,0x000b6e476372412f,0x0000b15d62433fab}, {0x00040b1e274e49c4,0x000456b1860aa0ef,0x000afe5a45cf5074,0x000e9a96583fbf66,0x00004240511347e3}}, + {{0x000434311b2d595d,0x00091ec8df579925,0x00073dd05f136749,0x0000296cb12c613e,0x0000248c0344dac1}, {0x0004f13a77739f5e,0x000a43d2af42cf15,0x000e4a1cfbf4288c,0x0008f2ca64c9b632,0x0000e8c07a9a8a20}}, + {{0x00049996fe8393f2,0x000fc91f3a32e10d,0x0002f63c80f809a3,0x000d3d41096d1c80,0x000089e146277750}, {0x000167e9889feea9,0x000250993909ed06,0x000508ac6d5c9c0e,0x000e82b6fca0d856,0x00001826047df1b8}}, + {{0x000877a9a4a27515,0x00027ae6fead4f2c,0x000aa194171bd007,0x000aa7e8df8dcc06,0x0000a074b4cb3bee}, {0x0005934c1cec8edc,0x000deabc03bdd6d6,0x0008a8415a6ecb49,0x0006dfeade91c2de,0x0000fb0efe029113}}, + {{0x00045ee23ab3495c,0x00074b77463d11af,0x0005d06f4a132df8,0x000435c923c15c81,0x00003ceb3f5cd61a}, {0x000291de88fb1da6,0x0003bda12179af52,0x000fef720ea05797,0x00084550d7218cd2,0x0000c0899c9ee1d8}}, + {{0x0007504752ddad71,0x000f91a68a979815,0x00058fb99d60bd74,0x0001e46047a3a9f6,0x0000f5d86d66f851}, {0x000bc424b5a6d88f,0x00022abefa7db8a4,0x000c9c51069eb2c3,0x000b42a5bf39e813,0x0000571960bc48aa}}, + {{0x000fbcf704e23c66,0x0001c8aaa65b7e8c,0x0005e3c83c71b7d2,0x0004ff4041b2bd24,0x00009b9883532185}, {0x00027a3963bfeec0,0x000ade7da7cb89d2,0x00068a9b199947aa,0x0003681d9ee9dbee,0x0000a08f003698ec}}, + {{0x000409478ef24870,0x000502cfec26e9ea,0x000dcf328c8d2d41,0x000937c52f9a6eb7,0x0000ed489e385b6a}, {0x000986bbef3366e9,0x00005dddddb89b94,0x000dddbe20de59c7,0x000a406fdb748cea,0x0000b9784bc1266e}}, + {{0x00055021a93507a4,0x00074d3c06cf142b,0x000ec3f40b4cd118,0x0003c29f70e76a91,0x000084e81ad8e755}, {0x00087b5272e9d6ec,0x000506ff514a830f,0x000192a8eea1c93e,0x000359a7cc2adcc4,0x000077e27e302f45}}, + {{0x000ab36d2b713c5b,0x00001f7b0cd39cdb,0x000af826b86274ea,0x000ea2c84680f309,0x0000fcc837abc72d}, {0x000fe9dd6529b73d,0x000793a88002a8bd,0x0001d45b9708aa22,0x000f559c7a9a54c9,0x0000f1a38bccd004}}, + {{0x0009a26b8bad853a,0x00029723eae72e8c,0x000ca28302d52cea,0x000410654d6d8156,0x00003317d153a8dc}, {0x00062fefd4ddedab,0x000a055d792ba086,0x000c6e944ed2a153,0x000cf2c035c16abf,0x00006bc5834b0171}}, + {{0x00052b383d102b6b,0x00065646b848e271,0x0006e6d37fe695a4,0x00015df5bb09d891,0x00004269d64bd170}, {0x00056a10a1d22850,0x000146d26d728d81,0x0005434a7feef6c5,0x000e319dac57c84c,0x0000282e5be59d39}}, + {{0x000f181721c486d5,0x00006c58824eedff,0x000570031301baf1,0x000e683136a6aa00,0x00005aaf78c6cddd}, {0x000937159c639522,0x00046bc25baf2682,0x000e52dc33a3bd27,0x0006c8ccdf8657b7,0x0000dd8c0881d78e}}, + {{0x0003274f5531461d,0x00008d95499b2055,0x00080f9d28b4a128,0x00075812c8763a1a,0x00001dbe32c1ddec}, {0x000210d30c34169a,0x0002d8baa533af12,0x00038f254ba74a95,0x000f5a9d133c6ea4,0x0000431531ac01be}}, + {{0x0005e22f669d7ec1,0x0004257fb5151529,0x000a3fdb3ca374f6,0x000da87a8406ffea,0x000006ae448ef3f2}, {0x0000a9033c8e9a1f,0x000181ad58858f9b,0x0000aed14234645e,0x000d454d0832241c,0x000010a7d3f6a942}}, + {{0x000deee40d5c9be3,0x000f8a84ed987c11,0x000d58dddb2bae7f,0x000fa363e97139aa,0x0000d8727966f6d1}, {0x000ca818569ff132,0x000b7a600f72483a,0x0006f2b868b89a5f,0x000c0b2cbc27c3c0,0x0000213071383ad9}}, + {{0x0008b1e48ac28403,0x0004bcba9477b535,0x000946b431831129,0x000819aa58f990a6,0x0000098baf9cab41}, {0x000c1584198da522,0x000bf46bfd1b66c4,0x00036a908ab4fc17,0x0008380f0a4c3cbf,0x0000ae9e34b78cf7}}, + {{0x000529e3fa11b1f7,0x0007274af2b4f411,0x00030793b21e4367,0x0000f30c20958ec2,0x000010ea88586e84}, {0x00021fcc5dc67cf9,0x0006f8405718fc0b,0x000e49eb708d5164,0x000a1f4955c21fcf,0x0000722a5d5e6dd4}}, + {{0x00050e2c861baa5c,0x000cd505ac3ec9ef,0x0007c063fc0c21a5,0x0009c0ef6b9a338b,0x00006370339ef477}, {0x00099c7638167c3f,0x0005895db30c22df,0x000854989fe6ffe7,0x000aa43b822d33a4,0x0000ef031de20563}}, + {{0x0009f82d57c667f8,0x000e4c0b76f116b0,0x000118aecc70312c,0x0001333f04a9e6c9,0x00002fcb419b409d}, {0x000b385ab45d44d1,0x0002517b83a31a8a,0x000e81b52fba0722,0x000affa05f50dd58,0x0000d8db55331ce5}}, + {{0x000b8d4e344a8732,0x000dde36d53e3097,0x00075e7507d8d116,0x0004ea4db22f5878,0x0000dc5e37363e14}, {0x00032e6e799eb95b,0x000eb899e6ecc05f,0x000ab23d5e9e5f4d,0x0000e60dc3bd681f,0x000072b8ab823af6}}, + {{0x0007ae02cecc84ab,0x0007cbdb871c8db2,0x000c46f58600016d,0x000d3892a44b13d7,0x0000891972873a77}, {0x000bbbddafd60884,0x00062bd20d39cfc6,0x000c410721a74014,0x000ea14c747abd98,0x0000c91e765fdf68}}, + {{0x000e5ca08819a786,0x0009695879217c95,0x000bbcc7dcf48b72,0x000148a91c7c5fde,0x0000f28740550e05}, {0x0005ac226cd44ecc,0x00060fea250ef83b,0x00006ebc588ae32a,0x000780aac5047a1d,0x00007e550b59434f}}, + {{0x0001cf25c727bd2e,0x00003cf915b061ab,0x0009d39202e4badb,0x000dfd3b4dadecf6,0x000061b1ca7d14c1}, {0x00079ccbd6bd51f4,0x00014045ec3090b4,0x000ef0e628024e40,0x000bc08ab29ca325,0x0000f2e941689e4e}}, + {{0x00040ec0ccced589,0x000b7da44f9845eb,0x0001812c625cd4b9,0x000650b3e0645887,0x00009f80d55a6cef}, {0x00040c9ce6dc1532,0x0007b86655215713,0x00007014d138d511,0x000b918cdb45bc4e,0x0000f34bb38a4b60}}, + {{0x0004fd22ae8921e6,0x000e292ba1e2f44a,0x000c180b2b039288,0x000c873da50174b1,0x0000b70ab667693d}, {0x000abc9e70574810,0x000f9c80dc417e9b,0x0002946824581dde,0x0006e50c890da951,0x0000b5629d33f473}}, + {{0x000c79eb06f5b41e,0x000d6e2434692340,0x0005a71a9a42e84c,0x000fb619a2013504,0x0000fbfb416b27b6}, {0x000ea239d33cd6f3,0x00087a6c0af825eb,0x000ce6f969caedb8,0x00015a23dc7e9ad9,0x0000897f9fd81e0b}}, + {{0x000b1f88e5d788e9,0x000851d490eef51c,0x00058cb3c1aec7ba,0x000d30965991e0cc,0x0000f306e8d2fc3a}, {0x000006e5040a0acd,0x00032b476f2e5fed,0x000ea7a23ca9d504,0x000b62d19c06e8be,0x00002865801dedab}}, + {{0x000293f6967469a8,0x00090d8a8ed8db92,0x000c771222894d83,0x00026a07c9e406bb,0x0000671c6f1aea3a}, {0x000f8d6d7de9853b,0x000601f2bcc7e42d,0x0009d50cf2e3ce34,0x00098f2a601dfc89,0x0000fc913dfab1b5}}, + {{0x000909fe61f7908e,0x000ebbbc7b2981c4,0x00004b338192e304,0x000d60e3ed8738c1,0x0000dbe9e48583f5}, {0x000e9be2db30660f,0x000ed0eb7d8e0c06,0x0002e096eda3e613,0x000246e8fa3e9732,0x0000ebd91e9c336e}}, + {{0x000ccc4df655a499,0x000bceb202108f13,0x00056b6eaa9e00df,0x000946f4631d0fc6,0x00003a058ce68c0d}, {0x000904a67bd3448e,0x000a1394fd5c6846,0x000225f524a3d4e1,0x000e99e102c1a5db,0x00003455bbbdc4f5}}, + {{0x000985b4b9ad1ce1,0x00064bb7f7936b36,0x000b1a416a981853,0x000ee75c25e1d048,0x0000381dd534c81b}, {0x0000d617a4a76209,0x0005a9b8944cd2a3,0x00097c33ac841292,0x0004e6ac1c6fbe7a,0x000041e541e23866}}, + {{0x00099e84a34f239b,0x000c090402d54174,0x0003aa83215fdb83,0x000db1075f46bf43,0x000061e15b013215}, {0x00059d4a127f89a5,0x000bb7e816daaabe,0x00018b6925d541e0,0x0000265aba0659a6,0x0000532773367266}}, + {{0x000a0fc95f57552b,0x000fbcacb0c9af53,0x00021be013294764,0x000145753ff58dc8,0x00000309532506f1}, {0x000bdf505c2e54da,0x000c86e8dd2259bb,0x0007e1e53158f27a,0x00050d2c5b7ffb39,0x0000e03f65c1fc1e}}, + {{0x0004ebd9c95f0f98,0x0001a4640771a978,0x0005561c45ed9deb,0x0007ddb1244af703,0x00007332f3afee85}, {0x0006e9e2b9e0d888,0x0003d6a0604909e1,0x000592f4852d910f,0x000677d07ed477a9,0x00005cb917ba365d}}, + {{0x0001c934c8998d19,0x0000e30ea58ff851,0x000029db02186a3f,0x000759c0189626b2,0x0000137a6d992ceb}, {0x0007f37748bc82c0,0x000180469f8c2fe1,0x000891aa287c2e93,0x000d8d850f71cdbf,0x0000ca1b89b75ec3}}, + {{0x00043aa5e1cd3cd7,0x00082a887c28516c,0x000ea1f9f8939780,0x0008f69059c699dd,0x0000737d6fafe686}, {0x000746a60f1524b3,0x00058a052aa76d93,0x000923ea536985e5,0x000a1111b1d322ed,0x0000429759f55852}}, + {{0x0006ec3092e9f414,0x000622256bbdbeca,0x000ad487d3a238c6,0x000d93982958ea70,0x0000ac8aaf9a5610}, {0x00001b15e4ccab05,0x00024de14bfb3fa1,0x00031899d9bf430f,0x00017d510f5cc665,0x000090005fc3a8ce}}, + {{0x000912f24544cb60,0x000ad79ac2e3c437,0x00058a2129987b71,0x00060613e3d9ddc0,0x00000075aacd2de9}, {0x000508b6cac83696,0x0007954f6c8980ab,0x000c532a487842be,0x000bc847ad663d6b,0x00007813de7d8a91}}, + {{0x00061cec3427239e,0x000fe56934d95dcb,0x0001915915f3c7ce,0x000da6e079e0fbe3,0x000040896be901aa}, {0x00067910492d25f2,0x0009c74082768d46,0x00087aacc8aeb30c,0x0003d4c943749592,0x00003d4708d99fe0}}, + {{0x0009cf2d0c051995,0x0002aae784548cda,0x00072a182502fbc2,0x00037270bda9dff5,0x0000f9b71b8b158b}, {0x0003a592b82dd077,0x00052523032ee0f3,0x000505a327630273,0x00009f0fe1a721c4,0x0000b6e3e8367964}}, +}, +{/* digit=32 [{1,2,3,..,}]*([2^224]*G) */ + {{0x0007bc035d0b34a2,0x000b6327c0a7e341,0x0000362d1440b386,0x0009436fb7262dac,0x0000c41114d00cdf}, {0x000cef1ad95a0b12,0x000847d543622ba5,0x000e486c9c09b37a,0x00029706d6cdd201,0x00000477abf62ff9}}, + {{0x000dcb3292a92874,0x000637b092c7a004,0x0006c0605ddc15cf,0x0007afc83a846480,0x0000a68df707db99}, {0x0004e4505bf7dd0a,0x0008eccf7f8c9c13,0x000b5f8afa4e63d3,0x0001cc06e6517f41,0x0000a8b93434d7bc}}, + {{0x00035b51e706ad97,0x000453a9ebdf126f,0x000608d90b99cebb,0x000858375389afbf,0x00006113c5036c89}, {0x0008eb097e2b5aa3,0x000c33b9130480de,0x000cc066c7e1022c,0x0009000bdab6056c,0x00003cbb144e2edf}}, + {{0x00064717af715d2e,0x0003f0134a96c417,0x0001ec956e2f7f59,0x0003034c1873efa4,0x00004e7b4f757821}, {0x000ff9788d5374a6,0x000320823d5be5c8,0x000ee8fe22b915e6,0x0006518a6bc755b2,0x0000657624d47112}}, + {{0x000f101dace5aca9,0x000181a6a267157a,0x0009c8609c4fdbcf,0x000a654addf340c4,0x00007e49f5379604}, {0x000e790937e2ad5c,0x0007726e17f19be8,0x000bbc0dc846e250,0x0006d57f38007a0b,0x0000f036040711e1}}, + {{0x00000e07442f1d58,0x0000e6e0e3abd6f8,0x000c64047475607d,0x00083d02807f16b7,0x0000858e1e427498}, {0x000120b8231ee10a,0x000ac38a1ece5859,0x000aa73a41b80e7e,0x0003ac2b72525ac6,0x00007cdea3e24442}}, + {{0x000c007f8ae7c38d,0x000b6d7401925ed0,0x000e36db36db07a5,0x000045ee5e9c2a5f,0x00005b9d57b46e95}, {0x00032e78eba20f2f,0x000e81b9a35254ac,0x00098a658ef11ca8,0x000666405e373eff,0x0000fe5a101723eb}}, + {{0x0007b11e51732d26,0x0007c538fc0e5747,0x00039eec5dfd6eb2,0x000c56fc43b0cc3b,0x0000af127792b36c}, {0x000852d06c425aef,0x000b6c221b9b70b0,0x000826d9c6df92f8,0x0009c27c8d4f9ece,0x000059aba7ca4935}}, + {{0x000d8d5da64309d0,0x000691b307045c8e,0x0009b580861a6de5,0x0008a7d6b52f6a2f,0x0000eee419498c95}, {0x0009aab771e4caa3,0x000d48bc21becddd,0x000b504f583965df,0x000290d2affce3b3,0x00000847a21861c8}}, + {{0x0002cf152bfda056,0x00090197b98cd2eb,0x000a1726fe0e4c4e,0x000e3cbd35076cf8,0x0000c06085b8db11}, {0x000c4d74463ba14f,0x00021030238c15c0,0x00027536d9d292f8,0x000c1d2311ee8b37,0x0000eea86f0aeaed}}, + {{0x0008cd366131e2e7,0x000f10fe2682b9d1,0x000160289f31d974,0x00079946e49e0fe4,0x0000c48ec0b78e92}, {0x00011d8d1989aa7f,0x0009fbf926f98181,0x00045474ab34fa0a,0x000755eb5fe2f5a2,0x000080a6ebc2c7ca}}, + {{0x0006054afa05dd8e,0x00011caf119ea7f9,0x00064bb5926dfcf2,0x0002b8020ef2e305,0x0000f4dca5141cb0}, {0x000838a65d306723,0x0004cd657e86cda7,0x000d595c88b08d53,0x0008361c5b439546,0x00009b58725725cb}}, + {{0x00010593de9abe37,0x00012cdc03be8ea6,0x000edce8c4043488,0x0004a12b261245cf,0x00008c318b53f523}, {0x000cf16fde24c99d,0x0004d2c2ff5d510b,0x000960fb42a77cb7,0x00042acc895c2b27,0x000030ce97680eda}}, + {{0x00053931a62cc262,0x000630c0e052fda8,0x000c633f323c69b9,0x000d488227df15bf,0x0000ac788483bae7}, {0x00078f9187d073d8,0x0009167f807d4878,0x0006e6d8f6c2be91,0x000a42f65861d833,0x00008b8974d4e528}}, + {{0x0001177ff57d0517,0x00078b6a19610952,0x000d76ad42ff3803,0x000c178c0aba74a3,0x0000c76480395a7e}, {0x000d75f48879bc89,0x000fc8ce6bc17532,0x000896c16ea7eacb,0x000fed382176b48e,0x0000a30e0b2bc750}}, + {{0x0002c2e421d3aa42,0x000cc84fa840c37e,0x00054e41cf926407,0x000643f8abc03d14,0x00006605ecd5f7af}, {0x00041a6d6a5eabf5,0x0003d16b668e2423,0x0000101021edb84f,0x000d8c8836edb804,0x0000b337ce7e45e1}}, + {{0x0005c77c055dc14a,0x0004e1d89cdfd207,0x000fdcbaf2a0ffa2,0x000866ece815ea6f,0x000034288799b648}, {0x00099cf884655fb3,0x0006064d3e412776,0x0001e1cb7fa5b5bd,0x000d66d1f680c644,0x0000fd61e66e70a7}}, + {{0x000a2dccc78cf662,0x00044fdbff77666b,0x0008d4668b301817,0x000a2a6d4dd0db16,0x000059455d03dab3}, {0x00064c5cde3acec5,0x0004c3adb276f585,0x000303f657714192,0x000f7b027d725d8a,0x00005deb6ca36f38}}, + {{0x000b657b1fa70fbf,0x000ee8073a00fd5b,0x000a02500fa07f50,0x00040d072e3aa7bc,0x000068f895e89757}, {0x00020605cae2a6aa,0x0007628748423011,0x000e47bd301bd721,0x0004f59d4238917c,0x000066663c218954}}, + {{0x00005d73272d8383,0x0009ca6295c5864d,0x0002fda32e22924f,0x0005445189593f6c,0x000030d7189e184b}, {0x000a62cbde1f7140,0x0004e5cb1a6379ef,0x0001c833235771c9,0x0008542f4826b864,0x00000a894fbc8cee}}, + {{0x000a39b36194d408,0x0004f7612601b4b9,0x000cf2f58e857a7c,0x00048774209dd24e,0x00002b9e66dda033}, {0x0006934e4e8b9dd4,0x000d642377d7c1e3,0x0003ae43bd2372c9,0x000f6f11dc94c70e,0x0000c57761e44474}}, + {{0x000cd0a1058a3184,0x000538053a9adcda,0x000c68de2369cf3f,0x000d9f86c3de5031,0x0000653a5767c4b6}, {0x000dd5aaa4e5c975,0x000167ab3c741688,0x00065c2835be80aa,0x00009120cefe7cbc,0x00007f95f1356867}}, + {{0x00014e24415503b6,0x0005ecbb17e9a391,0x000dec966c08ff7c,0x000f62beff674dd7,0x0000d4690afb3376}, {0x000e32eea74237b0,0x000ecd57508eff6f,0x000cc40fec436d17,0x000b4415aa28e1ed,0x0000d769c04581bb}}, + {{0x000b6de34eaacda9,0x00079ba0f1dec240,0x000438e55d9e116e,0x0002d73be45ec779,0x00001787c9e26f75}, {0x000532bf129ac2fd,0x00078a36e22c897f,0x0009fb8f3d307b7c,0x000b27c194067574,0x000014f95d0e57fd}}, + {{0x000d0296ae550430,0x000f44a87de1fe51,0x000e4fee28931e98,0x000d92e57f1cc609,0x0000d063b674e072}, {0x00098b9ed0e4316a,0x000a906aca4670a9,0x000da97c7e74a736,0x000d934cf0fbf24f,0x000040f65cbde178}}, + {{0x000360416df4285e,0x000af0c56ae21625,0x000c5cfc3b0c9bab,0x00005593032b19cf,0x0000497e5c3e9752}, {0x0006bb4164bda960,0x0009a0b74da11209,0x0003826ba1ee4241,0x0006608fc3624340,0x0000c8f0069dc09e}}, + {{0x000e981c27253c98,0x000b12b36a458667,0x000b7bb4605a6aef,0x00027b262c4b369c,0x0000394f37591f70}, {0x000c79c5f109d0fe,0x00057b8cc60a747b,0x000f09e68cad88a7,0x000eaba0c5a66b58,0x0000753d452d6127}}, + {{0x00074a15b0ec6f5e,0x00037289b2b8c44b,0x000d6fc7347989fe,0x0000aa945f848458,0x0000c362a70d61c7}, {0x00098a7b3a8ad418,0x000ffb63db51070c,0x0004c35f473a20fb,0x000dca6d2c2173f4,0x0000a56149e1acc9}}, + {{0x00078819ac6e0f4f,0x000eb413b5ed98f1,0x00000b0fd360fdea,0x0002d21625b8f4a3,0x00001f4d76b4b322}, {0x0005109587f76b89,0x000c9317fdb59d6f,0x00068b0958b4ee08,0x0009b8f8089bb78c,0x00005570e9ae808d}}, + {{0x000c36f35d33ae72,0x000330bb5a94a395,0x000afe84b200ea12,0x00076a00c789bd0b,0x000043ef52d29192}, {0x000c577e23ae233d,0x000ed460d1ec3934,0x000fa76a4b93807a,0x000490e72a53b1f8,0x00008914cb193ca4}}, + {{0x00084943fb42622f,0x000b600907d52e12,0x00095ec633b2700a,0x000fbd0370fb091a,0x00008f30be321b6d}, {0x000f8d269e55f156,0x00016c1323e9f2b2,0x000e5eef61fead85,0x00007e9a366010d9,0x00004d487b143161}}, + {{0x0006b86d23ddc820,0x000c7e0143f04c07,0x0007af2c503fd344,0x000a4fa95362ff31,0x0000add3db7e18b7}, {0x0003e3f8260e01bc,0x000494a1cc919c67,0x000f2e433fbeb49e,0x0001ead1351bf292,0x0000755e7ed45114}}, + {{0x0005139296077453,0x0000726f2b28c9a9,0x000c6f9dd0ca0742,0x000fc09b2790e74b,0x000045bbb58ddcaf}, {0x000a38cbe0f27a29,0x000bd41fcb56c65e,0x000e2c75767c24d7,0x000c489c25f0a7a9,0x00003f5cdb0a6f16}}, + {{0x000a9d7c5ee30a1e,0x0004d909b7292ca5,0x000deff48d159363,0x0000c2a04ce9f3da,0x0000c464752907c3}, {0x0005ff39e49af6a5,0x000a1f3d01bc89d6,0x000ced843f2d6238,0x000fd7c095561e0b,0x00001789e1318a13}}, + {{0x000f929763231dff,0x000cf7cbddefd633,0x000265da846df9f7,0x0006d111c889c0cb,0x0000ce1ad119f433}, {0x0000df6fc6a0a7e8,0x0007eda425dc8d11,0x00034aabedd431b9,0x0007fc4dc4aeab18,0x00004deb1250439b}}, + {{0x000f1693c2a59987,0x00040947190d8796,0x0005970149b9247b,0x0006ede5b9d9a511,0x0000e9dd70deb156}, {0x00078f7cbcd5e64a,0x0006fbd4c03294ad,0x000c222ae0359ac1,0x0008119b11baaf7c,0x00006a6e2855a78e}}, + {{0x000053f24cea1a04,0x0009f36214918392,0x000399ee9c97bce4,0x000ad13eb1db3435,0x000073f78f02ce81}, {0x0002fe0f63d3d0d4,0x00006fab62fc41d7,0x000158383e620b88,0x000f6c52096bc993,0x00001a21357cf896}}, + {{0x000e2fac7dcfcabd,0x000dc546e0071b5e,0x000b02e07650acfd,0x00003cf081b749b1,0x0000a9e41a1c9eca}, {0x000a727175a54ab2,0x0000ca5d8d10013b,0x000fd96a9ca0cd19,0x00094065ea52c095,0x0000c591b9fdc5c3}}, + {{0x000d4e42bad4d5f9,0x00006ef0059b6fb4,0x000122294fa4c359,0x0001d0da10218af5,0x0000a78a81b38575}, {0x0000579a98e84e7f,0x000f5997e5b504f2,0x00021e1e4fe1242b,0x00039cf77a273bca,0x0000cc8b1f084119}}, + {{0x000a30292d0487a1,0x000c194b91fee20e,0x0006b0e8f1442dbe,0x0003180f7a4afebb,0x0000700ef747889c}, {0x000ffc370f1fc62b,0x0005b9c79ccaf5bb,0x000f6340d3b31d4b,0x00010a38bc2aaba7,0x00000b08ab55725e}}, + {{0x0005701ae3400501,0x00060cf0c56944f0,0x000e19a51ba4b301,0x00052e4aa29f83fb,0x0000b9ed428c71d7}, {0x000e54eeb4819f5f,0x000cae18b75b1666,0x000e27b0b616cdfe,0x000e4c212ed5be3e,0x0000bf2831a34c7d}}, + {{0x000ec85e0e60d843,0x000fedb7ee78d685,0x0003c4d6e68037e2,0x000a6a2b65bdcd00,0x00003e7363ac3e29}, {0x0003a6108d0756c9,0x000b9faf134b995b,0x000337823d727f85,0x0008b46ac6edf71d,0x00009b9aa509439b}}, + {{0x000b104e2b4e075d,0x0004737c4926722e,0x000a9b82d4998729,0x0006f4e1e4c0e446,0x00000cb319827a00}, {0x0000f7dd7808c569,0x000ec1f89772f3de,0x000bd31aab5c54d8,0x000caac00a114aad,0x0000afaaaa6b95f6}}, + {{0x0005e2104cf667af,0x000aad3935d79470,0x00009267cfc2a811,0x00053a660b02806d,0x000019ed11ae780e}, {0x0007c09067b6269c,0x00029caef599f022,0x000efeebc967b853,0x000ae5555b924368,0x0000d6d34f68497b}}, + {{0x000d5d36cceb3707,0x0009378d7bf91dd8,0x000b67a622aeac57,0x000f66ed65017d70,0x00000c8e44f87c53}, {0x000095086a34d092,0x0006c7134907d1fc,0x000fdd315e0fca25,0x0009adc24fa29c80,0x0000c4acd03f8749}}, + {{0x00075173b5a9ba6e,0x000532e51a51baaf,0x000154897b9cbe1f,0x000c9ff88edae35e,0x00004309c3d57b66}, {0x0005805f67f37466,0x000a436401fff555,0x000499a5385fc37b,0x00055b7f86e2cad9,0x0000270b2a44cbc9}}, + {{0x00064f5974ad33b0,0x0007fe7b2df1afae,0x000b03f7304d8597,0x00040a2a3db3ff4a,0x0000b87878a87027}, {0x0003f015a0617323,0x000e732a19016d26,0x000155018c25430c,0x00078ec7ebab3ddb,0x0000a86f69393a9b}}, + {{0x000e368da9f3804c,0x000de164349c349a,0x00062baa5470f07f,0x000df3152f4cc985,0x000074a9e86eb290}, {0x000aa3543471a24c,0x000df8194511d3a1,0x000dcd44d239446b,0x00082cfec2dd0081,0x0000a3d7f10842ac}}, + {{0x000b085fdaf45207,0x000fd549daf21f3d,0x000ad5c42bb6d3e7,0x00051185969d8a19,0x0000052b13e4bfd1}, {0x0000d1b682b90604,0x00036c34452c1189,0x0003805b4a71d388,0x00023e6438055b78,0x000032412778725b}}, + {{0x000f96e4901bbed5,0x0000a432a2bbf20c,0x000a9cd7d6419c71,0x00024907a0fbb9df,0x000089111e450daa}, {0x0009a337b60554e1,0x0007dde283a41980,0x0003bfd35ea5f888,0x000a531d71380250,0x000051bb0af685d2}}, + {{0x0008f7443b30ca81,0x000ac993458340b0,0x0001110ade10b5bb,0x0006c5d8a546d6b5,0x0000dd50e6638e0b}, {0x0009d54cff2b821f,0x000c57281760292e,0x00024d6e33882555,0x000da0234838f837,0x00002c679e112ddc}}, + {{0x00088156d2a57687,0x0001ac1e7e2d40ee,0x0004ff4437f227bd,0x00054ad87ba134d0,0x00006e2ff3e3614e}, {0x0008d6fa3177ec75,0x0000d328fff536b8,0x000ba158ebf731d5,0x000188258caea249,0x0000ab8ff4c52938}}, + {{0x000605635edc56d2,0x00095e940d7933e1,0x000866dcb5a69d34,0x000def4c4fd00103,0x00000a38f576893c}, {0x000e790fac3a15b2,0x000e5a4f8e6bfbf3,0x0003aca866ed7ea2,0x0003f78663eb4fbc,0x00002061ea5280d5}}, + {{0x000dfe6f546783f8,0x0009da0a641e2480,0x000de8965d38bc6d,0x0005cf7b093cd12e,0x00009654db52cb45}, {0x000bf9a26e1adee2,0x0004173294d4413c,0x0008083fe291f376,0x000340e079725764,0x00005f504d3408cc}}, + {{0x0008e5ec3a0ee43c,0x0009a79898ff635a,0x000c63d5670aaebc,0x000cffdee9f5475d,0x0000e987967bfb34}, {0x0006b195e26310aa,0x0003982a8ca8f9f8,0x000352fe49e43548,0x000570853bcb81c2,0x00004eac8b0e474b}}, + {{0x0007512c1ad8cf84,0x000e59e0b697c1b9,0x000e85df0193b4e9,0x000afd539d271601,0x0000fb265b40d44e}, {0x0007dcde51e1ae29,0x000653d8b096321e,0x0006049988e3a8ca,0x000aa6fde46cb052,0x00001099ad8c9072}}, + {{0x000f91c93aa96b8d,0x000aafca2e132617,0x0003287230fc8716,0x000521d7106f5e95,0x00001c9c40bf62e6}, {0x000fe8642b7c094a,0x000c7543c021b9ba,0x000befd5d1873439,0x000aff41baa5de5c,0x0000363fc5ef21e8}}, + {{0x000320df862eaacc,0x000262c647dcefe6,0x00046d42814419c6,0x000f8e4e06707c4e,0x0000b6c83500a178}, {0x0003a45d30f917c7,0x00092879afee0f99,0x000500063d4c4b04,0x00060546142a1e70,0x0000c9b41c415d9d}}, + {{0x000fc2f2f8ba2c73,0x000f4c67aa28bc00,0x0007869720966eb2,0x000b9fe3f7b5165a,0x0000bfb7557ba2fb}, {0x0004f235a2b96204,0x00072faf46be131c,0x000172323bff3ed2,0x0002465b4473d17e,0x000021e8878739f6}}, + {{0x000587a25a416322,0x0004835b6c930fa8,0x000ebb8dbc081412,0x00029c0b18a9f559,0x000064e335796edb}, {0x0005ccdc87c51e26,0x000af01e6214af24,0x0003882ce16b3015,0x000e045b31c5600a,0x0000961bb955ec11}}, + {{0x0005b8deff7a3a0d,0x0007e1df73263b82,0x000604a1fbec3373,0x00049898ad747c99,0x0000154c9356a3bd}, {0x000506f1cc7a906c,0x0001ac560e8fac33,0x0003e394473bb539,0x000433a428fcbe26,0x000011828d5dc387}}, + {{0x0004be13e4b12ff9,0x00089d88667c3cd0,0x0008120cfc3aad9f,0x000532352ddcf824,0x000085a892eba389}, {0x000b21b3bb85fa06,0x00002dfc6269fbb4,0x000e2aceaf95375e,0x000d1f9b4fb06c7e,0x0000785426e909c4}}, + {{0x00017c8d8ceb147d,0x000de70a5554659b,0x0006bc6349b649ee,0x00032e9b7fa0b5ac,0x000099fe2c7ed6e7}, {0x000e7628d3abba2e,0x0006c797b79930e6,0x00096464d18fee6e,0x000e117c9d360dc6,0x00003baeb4907bfd}}, + {{0x000db47f23206d55,0x000fcd2601522bf5,0x0008ff89a2f6d341,0x000457c7b876533f,0x0000157c30c878fa}, {0x000c5c52d4fb936f,0x000bf6518cdc7517,0x000847a64ef22f7a,0x000a88eeb483e6bf,0x0000508455982e0f}}, +}, +{/* digit=33 [{1,2,3,..,}]*([2^231]*G) */ + {{0x00059d8df7304d44,0x000bbf210e8eab96,0x0003fbd60b71bcf1,0x0004de69a2438bd7,0x0000595cd1f9d11b}, {0x000329a4835859dc,0x000cbdbb6e569c0d,0x000928a4e4a0f0d2,0x00015406038e5edf,0x000094296224f5ad}}, + {{0x0003462f23f2d925,0x000d10b940789121,0x0006cde206cab71b,0x0004bc1bdd0a6317,0x00004c9b20d3e4d5}, {0x000d8aa9f2ac02f8,0x0006a06eedb03cd2,0x00008643403f8e61,0x000db947f68e1693,0x000031469c612dd3}}, + {{0x000df248f9813540,0x000c3588a2598521,0x000a0992c587e23e,0x000407cbedf281d7,0x00006930a5538961}, {0x0000debbe5bbe21a,0x00048491817f0932,0x000065160a7ffa5b,0x0002a946c8b4d909,0x0000c4f39939ff6d}}, + {{0x000a1583ae9c1bde,0x0008037ce2407aa7,0x000ab38b4e0af6d9,0x0008ca054342d928,0x00008b75007ea1c9}, {0x00086afe02358f2b,0x00063a921228efce,0x0001c67fc31b8b85,0x000d58552a19120a,0x00004069ea593aea}}, + {{0x000d6e27fa03cb3d,0x000a3fdd7d883232,0x000cbfc5ddb938e5,0x00080e34c1d2cd2c,0x00002f45c137f3a5}, {0x00020b57883e6142,0x00089e7c5f265926,0x00067e1e35fd27e6,0x000aaef39e45a915,0x0000cc71d2d64d8a}}, + {{0x00090cde36d07576,0x000179a293824a90,0x000b48ddff722d7b,0x000f439b7fb04c04,0x000028ad2a84be16}, {0x000bfb520226040e,0x00007104b6c4cd3f,0x00003c1886c34ecb,0x000aaf50c0754ec9,0x0000c336b090d23c}}, + {{0x00062a21e206ee56,0x00002c49a633473d,0x000f6b2c3f1e2748,0x0006ea27ab956ce9,0x00001830b48c2b60}, {0x0006846e78e815f7,0x000edc02082a67cd,0x0002ec365fe40139,0x0006aae2bbbfcb95,0x00004c11642db983}}, + {{0x000439e558df0194,0x000a6c712b279f51,0x000185a24230da4b,0x000f50118919e355,0x0000dcefcddc4b78}, {0x0000fb2a47d4c5ab,0x000f030e009ea7d9,0x000eed27355ac9ab,0x000faf4d2fc35974,0x000072d824d8bea8}}, + {{0x0001a744513e2cae,0x000158240b2cce72,0x000baa4500b41861,0x000c2425199968d5,0x0000b1757ee0b0e8}, {0x0003e283dfac6d55,0x000df8a237f56ebc,0x000f61499b2431e2,0x00036adacb5e2352,0x0000558a2a8306c9}}, + {{0x000f923cbb13d1b0,0x00025bfb9bfed213,0x0001144a998799f4,0x0005ee1ae8ddc970,0x0000b8b3bb64c559}, {0x000ef2e3ecebb211,0x000b2671f9a70ea9,0x0006f1d1f17cb6c4,0x00027637ef464f72,0x000071b94847943a}}, + {{0x000ae2d7ef0329c3,0x000261c4402a51a4,0x000d45bbc0850922,0x00085134a61d35af,0x00008f096fe6035a}, {0x0008b74a1dec0270,0x00074fc7dcbac746,0x000a06353e8cf10e,0x000d66ea35ff40f4,0x0000b4c0dfa8b77d}}, + {{0x0008552de7e5c194,0x000981c0256c779b,0x000d4743dfab2860,0x00093b24f58eeeab,0x0000e8ef838bb6cc}, {0x0000d264cb1bf3d1,0x000963dedf61ee65,0x000b70ced4c1f9d0,0x000e1e9ef7c9d7bf,0x0000ec0507e2641d}}, + {{0x0005cc7cde450794,0x000a116ac9e4cd7e,0x00070315cde173c9,0x0008fb117a8494c1,0x000038fd905d1d8e}, {0x000c506c7d9630b4,0x000bb47d4d755145,0x0009a80e86457a87,0x000abe931646bf0d,0x000053add2c0ef3a}}, + {{0x0001109a607419d5,0x00026b6bca80c994,0x000c431f3faa71e6,0x000479e4158c1307,0x000094abebce92bc}, {0x000a691eb78399f1,0x00052f42cba46dfe,0x0007c04f048aafb3,0x00091addcd65af07,0x0000a29a366f8844}}, + {{0x00040e51c21f2bf8,0x000c25057aee023a,0x000ab072ef99a513,0x000bcf23fe7e25bc,0x0000568d2e1c0e32}, {0x00094ebd3f69d9f8,0x000287affab19045,0x000e330f4181a973,0x000fc164d68d76b6,0x00007a6dafc475a7}}, + {{0x000b2b5ef7d92893,0x0007e97f015a549d,0x0000493b62480d4a,0x00033131d5590bc4,0x0000a55b52e9f780}, {0x0008115309eadb09,0x000a02e5c62540eb,0x0006a3d5adea7de5,0x000d60d4d631f0cc,0x0000d5e9d7d23e8d}}, + {{0x000bef5206d3ffce,0x00029d808bd4f297,0x0004cf5ba23d5e03,0x000a896a4f6912d2,0x00004d8163be9cda}, {0x000e9efd3082e8e5,0x000bb192f3600e0d,0x0008eee0a4fe1246,0x00091acf9001504b,0x0000219da8241da3}}, + {{0x000a5c1f7ea25aaa,0x000f5bb07d5f7bf6,0x000e78671d165e6b,0x0002194353936189,0x00003fcac89cbac4}, {0x0006fd4f0baa8abc,0x000122c1c2e5dfab,0x000d858495a4adac,0x000180acd75e3140,0x0000e263fead9b39}}, + {{0x00003d307032c722,0x000e590968c8cb68,0x000e978f07f40d5c,0x00051c86de86bddc,0x00005547c4f568f7}, {0x00085fd65fb2a9eb,0x000e6eb9179cb1e6,0x000504442ce69336,0x00006b015d1c2712,0x00007df465d6911a}}, + {{0x00004a3315980cd1,0x0002ea3bebf7b8d8,0x00053c504693bc49,0x0004a22578aeee22,0x000058de498dd247}, {0x000f5c7cfda83685,0x000328d7177e1331,0x000c1e46ed2d7bbb,0x000be88f61133af3,0x0000836ce7e230e7}}, + {{0x0004f1994f834cbc,0x0003829ed7828308,0x000e58243d35653d,0x00022ce542f16f59,0x00002b52f65c470a}, {0x000221b18f23d962,0x000c1f5252b4e3b6,0x000d61402cb05aba,0x0003e4aa00938b87,0x0000f186cdd61193}}, + {{0x000ece59a29a5c5b,0x00068b6c8402e042,0x000d92684b19b3c0,0x000372197667c719,0x000056246239bc66}, {0x0006e653c04fa024,0x000c4eaa39aa0cb9,0x000a1633f83a7176,0x000f72e033561dea,0x00005a9d0868533d}}, + {{0x0002c1d3dc090bc7,0x000f3a59c167e054,0x000e7fc4d82c996e,0x000b7973f735e80e,0x0000b179393ec35d}, {0x0009e25f8c5dbfdc,0x000d5f327b04b641,0x0008dfca84d9d7a1,0x000669d79f6f9b29,0x00007c5dff24de93}}, + {{0x000588d04c82bdd5,0x0004a8319dfd1b7a,0x000eb95806800553,0x000a818e8a55b5d8,0x0000ea886dadd5bc}, {0x0000a01252a0b4d7,0x000dc5eaa0a1e853,0x000e995631bffb4f,0x000d86bad828b1d8,0x0000de96ef605f9c}}, + {{0x0002d0cd77d970c2,0x0003b33ef9cb4abb,0x000211fe903cfb93,0x0001c690547c018b,0x0000fe64809c56ed}, {0x0005624c2ac98ccf,0x000f2a393e33cb7d,0x0006605212a1372b,0x0003e8d8d1ec1c29,0x00003d31b05a37ac}}, + {{0x000e9df5ece6e7ce,0x000e2facfb55a29a,0x000a233a50603ac8,0x0000b7efe85b7add,0x000061891a09d75f}, {0x000a3d299bf1603d,0x00092184255af555,0x0009a3e021f43afc,0x000a390cdaf34131,0x00003b117efd3903}}, + {{0x000da1365d1d131c,0x0007037ad03ee095,0x0002cd8dd86f1636,0x0000e59f37389e46,0x0000103fa05967a6}, {0x0004344f4b478f01,0x0008d117c98d57c3,0x0001fc12ece91edd,0x000ccaf01777b023,0x00001ae47f2c207b}}, + {{0x000cf8d20f8a2425,0x000da22e1ad8d983,0x000c4feb37aff5b1,0x0003e118fd11d07f,0x0000d53ae9100f1c}, {0x0007905ec0418032,0x0006f440488850fb,0x000628d8f85e3c97,0x00032bee67faedac,0x0000e86515086685}}, + {{0x000aaa46a67a6b02,0x0005925cec4115ac,0x000c6701ef4cdee2,0x0003d829ee565ae4,0x0000a04ca671c7d6}, {0x0005018ef0543fbf,0x0004e1b0d81deb10,0x00015d333f709a4f,0x0000aa6b906ee629,0x00004a8741386f1f}}, + {{0x0002fa74d82f4c2b,0x000fb804efb3b6b8,0x000c3425e90725a5,0x00043e0c82ec46ad,0x00007b80581d7878}, {0x000d91cdd1fc74cd,0x000b1783a6c4df46,0x00004cbbadc1c62c,0x00076389d1b9f31a,0x000087f6f7365e40}}, + {{0x000cfc1317f4a765,0x000b41036bce02b4,0x000e72a568d2703e,0x000b0f48206cc6a5,0x00007be9ed21f53f}, {0x0004571ef0b17acc,0x000e19181b380937,0x000935d0e74b2655,0x00093608f80ea889,0x00000d9e94351529}}, + {{0x00060411e84e0e5b,0x0002fea34c931968,0x00073a732a5db84d,0x0007c049d5bb1970,0x00008d2fe571bcfd}, {0x0005f36f3eb82fab,0x000c4dff8b584577,0x00074c1108cb20cc,0x0008996659b65f83,0x00008b4a422e30c7}}, + {{0x000c3ea6fe8208b7,0x0003b86e78fe75e3,0x000d93a1abd74b9e,0x000aad9be2e81bd7,0x0000ed06e284d0a5}, {0x0005a586be8b800c,0x00011846db28721f,0x000e88ed3428299d,0x000e10d5cb8e6b5b,0x00003186b23dc034}}, + {{0x0002c9e8977d99b6,0x000123f531e7a631,0x000d3b1d4be94433,0x000b732232c0c218,0x000017aae8c41247}, {0x0003fc4282aec3b3,0x000fc7b8f8234015,0x00004f94cc6063d2,0x00034638f10e5833,0x00001efae751e676}}, + {{0x0006c6d40a9b97c5,0x0002ff666256badb,0x00084b2e314702c6,0x000a408eb954f151,0x0000184a526e4b6c}, {0x0005337003c32ea4,0x000ce05974c7fff0,0x0000dd71a5aa374d,0x0007ec5a7638544b,0x000059cd2801eb94}}, + {{0x0008161459c2b92c,0x0007b5ee8ef5a6e2,0x000b063102f020fa,0x00052ff132ec2d30,0x00003e1589a7c6a4}, {0x00053feaa3f451a6,0x000362d9acacdc5f,0x00027e58b3a3c7f2,0x00099ecec2f8926b,0x00008466ee837427}}, + {{0x0004dd41fa266138,0x000b3dc29d639832,0x00012d657a2dc6da,0x000d1579675faad7,0x000013994bea1fd8}, {0x000b722fd4f75534,0x000bb3a36b205ccb,0x000559df55135ff8,0x000f3004be28af69,0x00000b65beddd41b}}, + {{0x000f2a43734e5205,0x0002d09bdcbad98b,0x000945b355e3abbe,0x0004aa27c76553bc,0x0000331c09416ef1}, {0x000fe2976b60c80e,0x000a9ace16f8518f,0x0002b97842285593,0x0001b5cb1f64ccbe,0x00008f2c0da8b242}}, + {{0x0007174c1df065c4,0x000b1f6578fa617d,0x0003b54a8afeeb5a,0x000ce2d6ff132926,0x00005c558090990d}, {0x000b6c0ecc8c1778,0x0004d982ecaa42ea,0x00007ef8e799ea9b,0x000c2c765da244b6,0x0000ab226ceb2a3f}}, + {{0x00041e57ea973dc8,0x000fa0888f2e7457,0x000fd9cf15c00ca6,0x000871fcdce3cf45,0x0000a741ef1d507f}, {0x0001c2f196b4cec3,0x0003997ea61847c5,0x000b18a2b70d08e4,0x000514630da15c15,0x00003b6c6785f610}}, + {{0x000e4f807ac97946,0x00005a06cb79c662,0x000d954e51eccf05,0x0001c2bff08623e7,0x0000ef2c5fb84cf7}, {0x00063d2679784532,0x0005fd654af8b2c0,0x000bdaa37a0cf379,0x0007e05cb242ea7e,0x000006e0b10d8674}}, + {{0x000ae5fd5ecfefc4,0x000882bff8fc481d,0x00032459607084fd,0x0000364040a01aea,0x0000c64698114de4}, {0x000ab4ed65abfc39,0x000e83541ec79eb8,0x000695012e01cb91,0x0009ebff029adbfd,0x0000ae28483cc756}}, + {{0x0004c9ea66d80a18,0x0003f5f5f911a561,0x000ba4fc1680a3e4,0x0001c08c07b14dce,0x000091c285c21307}, {0x0007ceb799ece3c0,0x000941e07e27cac6,0x000e4312329b910a,0x000cbe66bdb409f2,0x00006f8b1377ac9e}}, + {{0x000fafd385470908,0x000f05e3415d5981,0x000e31b2719ab8b9,0x000b427c28c194c7,0x000043be0aaefbcb}, {0x000ed43a6db836ca,0x000361a45c05f3b1,0x000c1a3772a1330e,0x000e32af19f3c595,0x000085f39d0e4b5e}}, + {{0x0008e6d4ae528342,0x00095423dcb03da1,0x000374aef5a403b3,0x0001119b555e0af2,0x0000ad599c43e8ca}, {0x0002fb9014b3bf81,0x0004e66d50071b3a,0x0003401027309268,0x0004ddf79f1426c4,0x0000827cf819fddf}}, + {{0x00005f6f10ff9276,0x0000c3739fc6c836,0x000c1c2ccd387145,0x000ac50d163450ca,0x0000b52129702ec1}, {0x000c4f96e3cb4a55,0x0001278abff70606,0x0008e3a45e47d3f4,0x00015ff25a8d5ebe,0x00003ea9e97f6102}}, + {{0x000106e39cbb688c,0x000233386d32477a,0x000b9b421532401d,0x00033ece564f64b1,0x0000a9b838941dad}, {0x0002b4e2093913eb,0x0001b9bc8112b142,0x000e7b2c7533d2f9,0x0007c58fa017beeb,0x00002767c4b7af19}}, + {{0x000ff87aedbae9f1,0x000926880a54c925,0x0004d0e717daf0eb,0x000cf58284ddf59c,0x0000581cf93416f8}, {0x000a8873ac1f4527,0x00098b6aeffe3eec,0x000fb8dc3b417fce,0x00040035918046ee,0x00003d318ac72209}}, + {{0x000400f728693e5f,0x000a439927ede800,0x000ea9910e87d814,0x000b68a3e94d3b57,0x0000f8a35b71245f}, {0x00053d77f200d34a,0x00076f653ce10438,0x000a06379470f1e6,0x000c28e1ac05bd59,0x000014052c2a3930}}, + {{0x000fab526bc27979,0x000609f167716b72,0x0003e48d113670d1,0x000677701700521e,0x000078fe40247adf}, {0x000fb92d41c5dd45,0x000777b27da555ec,0x0003fb6065ff8e24,0x000a3c9751827201,0x0000768d7e57f547}}, + {{0x000eaa360017a5ff,0x0003ac64ce9bbb24,0x0003dde076b18e6e,0x0007e9d225c65510,0x0000c3672af6592f}, {0x000ad77d06283a1a,0x000044d59d999606,0x00040e7c2542fc65,0x000f550bb57c492a,0x0000c948f1448db9}}, + {{0x0009682b04465c31,0x000a5468bd156d4c,0x000318d7ee3d062f,0x000c95951729ac5f,0x0000fc87df6aeb6f}, {0x00046a80591f6528,0x0006d89621aa63d1,0x00031348ca861b8f,0x0006d9d9f5f15ace,0x0000f663391c40da}}, + {{0x00078acb591ffa38,0x0004ccdfebcecfa7,0x0004ea6b3027ca9c,0x0002543e8e05a544,0x0000aab4e6a278d8}, {0x000f04fb474d6b85,0x0003b45b38552437,0x00047ecaa6597ffd,0x000bfc6b0aea4eca,0x000068aae83d5c7e}}, + {{0x0006e64c73b23834,0x0007917d87620e96,0x000a05dab49eb344,0x0002369e1078218d,0x000043d8baa416b7}, {0x00063a5ea7610d6e,0x0004ee1ca979163b,0x000baa132e47e418,0x000b63ce648b6580,0x0000bf53de2ee0d5}}, + {{0x000fcb4d3c8c1cab,0x0002ad04b3098d3b,0x000e7d3950d914ef,0x0000e7a5ef64153d,0x0000de1666fd6b85}, {0x000ca6ed449ab198,0x0002689a2672dbe1,0x000cb7a538902b32,0x0003ff31674b7eda,0x0000e9faf6f75252}}, + {{0x00035da9a85788bc,0x000dfd0626d46ba5,0x00073dc64d21f03a,0x00097d499f8c47e8,0x0000da8564dc18ec}, {0x0007a5cde92c68ca,0x0000d3323cc43e8d,0x00080ff7c78e035a,0x000da99ef26275f8,0x00004ee3dffc73ee}}, + {{0x0003507af4e18f85,0x0004b672f3285882,0x0009d3186967ec9b,0x00039c4ded19d955,0x0000e2ab3debcdce}, {0x0006e4d11c226dff,0x000297723014abad,0x000885719f9783f4,0x000dbeaa49a0cf1a,0x0000c0c1a5b80da9}}, + {{0x000ec49571d92ac3,0x000e1692517f8bba,0x0004ea4af569e85f,0x0005ad5333b014a1,0x00002f2a62f42e5c}, {0x000ce3a06d89b85d,0x00095ff77a0898c2,0x000f795a2b90741a,0x0001985530defc01,0x00006e5ba0c54b3c}}, + {{0x000845112e4c9366,0x000d5d0be17b7d8e,0x000262bc9ae419f7,0x0002bd4583fc8c22,0x0000b842ac7f1bfe}, {0x000f4e9440d68277,0x000e7f81fb1433ce,0x0004fbb925f69f4d,0x000158816cf6f623,0x00006ae3fc449e7e}}, + {{0x000f6c2e9740b335,0x000cf962d6a14e89,0x000d10d15677bc85,0x0001cd1c6d8a7f68,0x0000f9a72245257b}, {0x000b9164ad859617,0x00077657ab4a7096,0x000461d7e5f8c47f,0x000ee26e57d7d0f7,0x0000eb6094df0ce5}}, + {{0x0001dfd341905477,0x0003805dd1500b1e,0x000df44e68a394f4,0x0007189a9eb24d97,0x00008ca06bff7675}, {0x00034626ffeec228,0x0009b6cdd8fb6f0b,0x00005be479d91bce,0x0000e32c83363ca1,0x00001ba76c186971}}, + {{0x00024cb28c682c67,0x00081612575b3d1b,0x000e66e9827f2522,0x0001fe4587c779e8,0x0000b0c03e9b05eb}, {0x000d03015b548e76,0x000fd8b36af7fdf0,0x000310c40a8be76d,0x000aebecdab04a4f,0x0000287223f547ec}}, + {{0x00060558b399320d,0x0005f01e4646678e,0x000261a5e61fe3fa,0x000449f482866b03,0x0000fcf45b92c2f2}, {0x0009a512f684b432,0x0004a7220a668fab,0x000afa58ff796c65,0x0000ddfd90707ef5,0x0000c421d976fdbe}}, + {{0x000cda3af2ebc2fd,0x000cfb4efe24c4f4,0x000cd10b1a0af843,0x000e0383b857c19c,0x0000dc9d1ec614d3}, {0x000c8bb62771deb1,0x000a81c5aa817bde,0x0002391ae829277a,0x0004ca6af18dd683,0x0000740f316d71a8}}, +}, +{/* digit=34 [{1,2,3,..,}]*([2^238]*G) */ + {{0x000e99aeeaf8c495,0x000d1e24d7288928,0x0002b156cee7aa73,0x000a1cfc5007c2e7,0x0000fcf57c63d408}, {0x0009e39b6057604e,0x0000e2868bbf9f71,0x000103e2d7d343c0,0x000ea14cca254b7e,0x00006eb38aad131b}}, + {{0x000624f8be762b49,0x0000758e3413b33e,0x000d805fa2a9ee4d,0x000fd7068e636967,0x0000848949c0db8b}, {0x000d7e5d23a84178,0x000d73e29da55308,0x000ee471f892f3b1,0x000089495c139e3d,0x0000631594e5757e}}, + {{0x0002a3cde918dcce,0x000346fdcf4be0c8,0x000cb1b2d2e7b599,0x000e06a2c5024932,0x0000a613a9e5657a}, {0x0005f6cf1fdc9f70,0x0008879fe682c2eb,0x0001cbc7fb6eae8b,0x0000e6253dfee059,0x000000da7133e129}}, + {{0x000e2ea1f095615b,0x000664e68c331083,0x0008818be0a28ad7,0x0000ccbbfc02523d,0x0000585113ba3585}, {0x0005f0b30df8aa1c,0x000b8ab7e3ac7d93,0x0002f00cbaddda07,0x000f6bd2c3429955,0x000033ed1dee909d}}, + {{0x000195d80e877662,0x00087ddf4ac022c2,0x000e749349e99e6d,0x000240e9642e4e65,0x0000610ffa31f1ff}, {0x00047d4751c8159a,0x00050f3a93634d1d,0x000477c33697b498,0x000ff26318ca4687,0x000090cb5663441e}}, + {{0x000384836f024cb1,0x00072601616858bb,0x0007e07f185be1f7,0x000f025c59587cdc,0x000091be071bf1d8}, {0x0009fa5cca5e55cb,0x000c47d04eacbf16,0x0007d05db3864ba3,0x000e5ce15e367f8d,0x000048a876e56549}}, + {{0x000c656580e40a2f,0x000b828068bcef89,0x0007990c9f194ed8,0x00064884528045a4,0x000053fc7d84e1a4}, {0x000ae9b78593e7d5,0x0002f1db65d7bec5,0x000a3d39b2cac4ee,0x0003ef58c1eb2404,0x00003b7d63453f8f}}, + {{0x0000d483e07113c9,0x0008ed8b63ae2dc4,0x000684c2b6e4a5d3,0x00026bc582a94b79,0x000032b33d4f22da}, {0x000f6510dbbf08dd,0x000894c23a52f534,0x0005bdc9b211d07c,0x0005573eeece0fee,0x0000f178169c7015}}, + {{0x00046350a712229b,0x000759273f8cd429,0x00013bc8393cbe44,0x00078b50b095ef8f,0x0000b74197337989}, {0x00009a256dbe6e72,0x00063a5d39ec9d73,0x0001f9a31e578ec5,0x000eb41961151b85,0x0000da7715e05709}}, + {{0x000301753dfabf0f,0x0007c8e39259867f,0x0005d9958728d207,0x000be06c75a0cd81,0x000084867a706603}, {0x000b13d70e35b1c7,0x0007a9b03e2cc865,0x0001f31210241446,0x000a7c746041daac,0x0000c9017addf028}}, + {{0x0006de90a482873d,0x0000e77e54d4abc9,0x0007d88e74265d6b,0x000de298c38e79a5,0x0000461d7676ce82}, {0x0009ec564a7e489c,0x000ce0def5f2817a,0x0005d494ecc5675c,0x000149da00e78598,0x0000626833fdb035}}, + {{0x000905a83cdd60ed,0x0004d1170184abe7,0x00023642a50602fb,0x000aff989886cdb0,0x0000568d09176e1f}, {0x00022c70259217fd,0x0008f43141e45b19,0x00095f86e93831cd,0x0008280fca35870c,0x0000ec2057b268ae}}, + {{0x000a599f98a759a3,0x0002c7c23c1dc44e,0x000c4f68755a0a7a,0x000478a5ffb6e694,0x0000563cce242848}, {0x0003517e7b1fbe12,0x00092f7338e0812b,0x000d048db8a7dc97,0x0003b8011ecee952,0x0000eea4056e86ea}}, + {{0x00068a7ba772b34e,0x00007f4e2541d8cb,0x000ec14dbe16ed34,0x0008bdbb32f6a60f,0x0000ee376f819169}, {0x000aa1783674c027,0x0006e843022ae9a7,0x000a4990f65832f9,0x000215b9f3a8da5b,0x00009a59c3b6b8e3}}, + {{0x0004d2ebd19bb161,0x0000a3262d869cdc,0x0009c0b47c6c7cfd,0x000128c4ce14d096,0x0000fa352b723e56}, {0x00055b8973db6d32,0x0000c8e5b7bf383d,0x000bb571f7183685,0x000dd2a7714596e6,0x000059df31f4d5b2}}, + {{0x0008925913cc16df,0x000cf1a26f5a568f,0x000f499ae18bc5b6,0x000e83efa413bef5,0x00008835dedb3f0a}, {0x0000bd865a40ab05,0x0008c94b377eb6e6,0x000084a696559643,0x000de06cd8562592,0x0000ce433b99f23e}}, + {{0x000f04f6ad651437,0x000766e14af6e8e8,0x00095c0c71151182,0x000a15cd390a1082,0x00001e29ee4d21eb}, {0x000fc0963717b46b,0x000e306ad4a2a588,0x000c22b2202be02f,0x000848e31558c604,0x0000b4d4bd6c2f3c}}, + {{0x000f49620efd6628,0x000045952d1454a4,0x0009784c292ba6d2,0x000643adb8ea1ecc,0x00001cc10cacb353}, {0x00070ad4b4d7f6ce,0x0005a4a1dcd240b5,0x00047e7975c9f1d9,0x0009f421379f8131,0x00005c6097c0bd49}}, + {{0x000afa6328e5e200,0x000a5481555040dc,0x000bfc978f7b5244,0x0005b1b9a4f11847,0x0000ea0e79fd2582}, {0x00096eb646c7ecfb,0x0002946dea9da50f,0x000abcf69eb81149,0x0006e77af04677df,0x0000e3a06907713f}}, + {{0x000523d42e06189e,0x00006e3aff13860d,0x000b20650bf07794,0x00000c2b616dcac1,0x000066dd6d201313}, {0x000fd67ff99abde3,0x000097aac50dd4a0,0x00046b2d7c990355,0x0002aed22ecf8b7c,0x0000333b1e86abf9}}, + {{0x000113c6c491c14e,0x0007a0dd3f8811cc,0x000d932ed0597668,0x000b6d15b4d9e729,0x0000982aad982c38}, {0x00053478be0dcf0e,0x000d85ca53f26f92,0x0003ca77f700080a,0x000983f813115644,0x000092d6943cc51f}}, + {{0x0008af885dfe9aef,0x00045d2a86cad2a0,0x000dff020d825d9a,0x000c3f3c53988d39,0x000038b135b330cd}, {0x0008ae062a7150b5,0x000dbc340e9b0c91,0x000bbf02ef31fd8d,0x0002395fa0e7ae4d,0x0000847fb2aaeba6}}, + {{0x00047dcdccbac8b2,0x0007e6f485c86b16,0x00038ecdfb642aa7,0x0003fe673f376570,0x0000ce5e8661a49d}, {0x0003788c98c44004,0x000db1fa5279ea22,0x000becfd78104a8c,0x0004ae7cf7cc7a06,0x0000942431708f97}}, + {{0x00065e784d6365d8,0x000f0f759fb8c0da,0x000e81930bcb7443,0x0008aab5c712b17a,0x00000428dffcc6e0}, {0x000afefa4faf8433,0x000dcfa9855ff19d,0x0003ac7ceced8538,0x00071df0ac409cbe,0x000058c1fb6b82da}}, + {{0x000c0e5fd349961a,0x0000e421c2fcafa9,0x000a28d382b2cfa5,0x000e7d8a80db17f3,0x00008aba539fd138}, {0x0002d1d6e96eb8db,0x00001baf96225201,0x00064f56c65d8dea,0x000da1a7735447b2,0x0000eebef3fcb6c8}}, + {{0x0006d98ce7852541,0x0002ab64a161fc34,0x000794addd50e8d7,0x000ef6b03567c749,0x00005a76065852c7}, {0x000a222961f23d6a,0x0007d3ecc0b059f3,0x00082fde4378e443,0x000f34274be4345a,0x0000e509af378b9c}}, + {{0x000ee46577f44a13,0x000c8611deeb4a61,0x000f7b884e09b748,0x000a6b90481b2cf5,0x0000562667891acf}, {0x000c518bf8d21e6b,0x00010205a76d37f4,0x0004073c022d9653,0x00056687fb85e195,0x0000ceafe5015b3a}}, + {{0x000def7be42a5821,0x000055046be6efec,0x000e8dba9d3fc608,0x0001ffb9af13c809,0x0000e6c984774149}, {0x0004925d30c31f70,0x000aac2a21223b57,0x0000859e7b7eb72b,0x000942776a0dacef,0x00006fec31421900}}, + {{0x000bc10f8c22049d,0x0006b75ebf692464,0x00036326b9bfbcce,0x000cfa07a88e2a43,0x0000a05261d2bc2a}, {0x0005bdceba7efc8d,0x000955dbbf2ec29f,0x00075f127471237c,0x000325d72773f229,0x0000c744e8ed4d0b}}, + {{0x000ed16a56edb730,0x00072c007e7038a7,0x00080b40064357e3,0x000be1a167d15b50,0x00007b4116423de4}, {0x0001e3274c898834,0x0000e882e7edb2d9,0x00003e4823c16282,0x0003316d6b36ba75,0x00008434e8e4ea34}}, + {{0x000f24f2c7ae0b94,0x0000d939b44a79f4,0x000595eb1c46fbf8,0x0009c736fefae856,0x000017b66ac0d5f2}, {0x00032b2c5ceec209,0x000f51a1cae25f23,0x0000286e6d69661f,0x000992cede7e529b,0x0000d06252a7276b}}, + {{0x00094b07e50122b3,0x000b1af07ca53247,0x0003fc97bdd744f8,0x000d9d00a12f08d6,0x00009650f1aa6626}, {0x00047f71fa38477c,0x000914dc124f101b,0x0006eb58a3d815f1,0x0008865569ae95b2,0x00003cde18955fb1}}, + {{0x000f37bf9539a488,0x0005f408c1a554e9,0x000580cbbb0100e0,0x000c56021d9811ea,0x0000af52d3606e50}, {0x0009d47dbbf698b0,0x000a03dc1c73dfbd,0x0006a5df82961a1e,0x00007a203d38f8e7,0x00008a53a686def7}}, + {{0x000fb481bee45d41,0x00062c68803626ee,0x0002f2469b3cee34,0x000162363c5315c4,0x00009d84d2e91378}, {0x000c3c51c4d349f6,0x0003b63d59c522d7,0x000abceae6596584,0x0009d56f198c56b8,0x0000fb1fb1bc2855}}, + {{0x000fd0607bf8fe31,0x00082467734b8bbf,0x000f7f0d346259c5,0x000ff1c8953cea35,0x0000f0bece2e65b0}, {0x000b4b3f3c72914e,0x00045cb53389f7d5,0x0006b6d4629e8ea9,0x000fdd6a36562683,0x0000849f911ca174}}, + {{0x0002fbbf4737f217,0x000af209f5ac7ec6,0x000f9adbed8dba5a,0x000767b4b5d7a9a5,0x000007d28f8161dc}, {0x000460bcaa999eab,0x000c6c92e4cc7711,0x000d4bf2dba7b174,0x0002788c4bab6618,0x00008f0c9819b8bd}}, + {{0x000ea9a324b47379,0x0002a2a83bca024b,0x00032dcedfba9e42,0x0008b9de635643a2,0x00009619367b571c}, {0x000f35754b7032b4,0x0009e442d54ae8c9,0x00090c65af936b3b,0x000fdae263f0f082,0x000089897812e2c7}}, + {{0x000d55a13d4f95eb,0x0004fd9b8500adc5,0x00073f43d737cff8,0x000475771c557b8a,0x0000ed617a5918bc}, {0x00054017dfd8ab26,0x000dda2870aa6624,0x000a7e545ae7b89a,0x000e4bbb555f5323,0x0000791e2481e057}}, + {{0x00036ad324fa34d3,0x00071cbeae288601,0x000dd3299ea11144,0x000c33d23a4270be,0x0000d5c3a7ff1c35}, {0x000db678d0412d2e,0x00021cdc6b9ab0f6,0x00028a982d92625e,0x000e7db2ae5ccc4e,0x0000a251c3727a3c}}, + {{0x0008932790691bf5,0x0008b6b736ae9d65,0x000d63b6eed61058,0x00088f212c2f04c0,0x0000cf06fd6163d4}, {0x0003facd9588e411,0x00021b93257e9736,0x0007acace1f9bf76,0x000ecf99d1ffc466,0x0000cf4a1aa1a061}}, + {{0x0008a49dc1818d01,0x000993621ab040e4,0x0009ef6390643ff3,0x0008544768640ce3,0x0000fc099ea14d86}, {0x000b9c3eccd28fd0,0x00027eec54ab9130,0x000b475b6d743cbd,0x000d1f052b146fe5,0x000058d9a82900a7}}, + {{0x000229291262b72f,0x0008cb0edf0365e0,0x00020684296f924f,0x00071f6cfa59c8fe,0x000060370054eafa}, {0x000699e18d7dd96b,0x00021bab24955f30,0x0008be949381e878,0x0008eee1669b46dd,0x000040606f5d6aae}}, + {{0x000b839fc6751a47,0x00048ba800ef2812,0x0001a28751619621,0x000348d398d5ca4c,0x000020c00eed53d8}, {0x0009eb0d820007cf,0x000fb39b5825c269,0x0001f6984880ee65,0x0009998069469447,0x00007d16ea9d3dda}}, + {{0x00075b2c0519a238,0x000dcf6952e328d6,0x000294a8a9ebf94f,0x0003f5728bb767a2,0x00005512b4e7e0af}, {0x0008ba899b16a0de,0x000bda7548a71895,0x0006be61595c2430,0x00074bd30d1b10a1,0x00003ebbb9865bfb}}, + {{0x0003cfe18549fdb7,0x000edfcdb792a327,0x000aba56cf6e200b,0x000aa1e4a76e1883,0x00003ec66f6f9ef6}, {0x000add7d1b9a305c,0x0008f7ae1b9d8d17,0x000cc094aa959c5b,0x0009b8c86435226b,0x0000c5616c597d42}}, + {{0x000da01e6a33f7ce,0x0006bd4e70ada6da,0x000c15b7cc6217a0,0x000853f619a81809,0x0000a06b329ee80c}, {0x00011cea5f5e7b9c,0x000fb87c65f41748,0x00016ab5466dfc30,0x000f6f8ea7bd6933,0x000012c4acbddcc0}}, + {{0x0008d1a1e407dd9b,0x000be1afa997e430,0x00077b7a5e8a3587,0x000d523a296c12ab,0x00005ad49e5173c0}, {0x000b2b27006085ab,0x000fd7bf6ec240f9,0x0003066a6a88ff33,0x0006e1e78603b14e,0x00003f99fc365e48}}, + {{0x0003f5eb2e636456,0x000724c8423207b5,0x00014d5cfbe57e54,0x000aca7779c21672,0x000017969cd629a3}, {0x00068cd8a7017a0a,0x000a1e9b7ee8d176,0x00016177677b4d19,0x000a71b8fd0e939c,0x00008c4f4f075968}}, + {{0x00071cc67b3de77b,0x000db34f79052960,0x0007100c9ae3c0b8,0x000b41b7e440c28a,0x0000b8c3c1c9b4b9}, {0x000e8eac51b35836,0x000f3525e6426d71,0x000f509f37591f5a,0x000c9b973a2f7b13,0x000018487aab619a}}, + {{0x000e5f79d61718a7,0x000c0592d28c3a72,0x0003c35cf00413bc,0x0006ed0d9b11d396,0x00007623bd0290a4}, {0x000273bdcdd2a50c,0x000a4601846edeef,0x000c6e9294a741f9,0x0002cc73b89e510e,0x0000b0231a04b7f2}}, + {{0x000500d084bae24a,0x0006943d2693bbe1,0x000def8112f0ae8d,0x00094f0cffb5f27c,0x0000a0c030ac63fb}, {0x0000d61a0f442dea,0x000687b139d36eef,0x000d8bc28f92e181,0x00012f7ae6deb70a,0x000089e38dd60514}}, + {{0x000b865d2fdca23e,0x0007ec8ef89581ee,0x0009056145a15ee0,0x00019a968fa10a01,0x0000ff5b8f0680ee}, {0x000cabbcb1c8a0ef,0x000cc8c838f9f0c0,0x0004a14c02e1ee9c,0x0008e41587d8b88a,0x00006f278971ff69}}, + {{0x000ef1c89ee6256c,0x000dab353b45ed38,0x000e903b3f44ee1f,0x0001ded115c0c770,0x000078ec0a2418f3}, {0x0003324b7dccbc61,0x0003063bbc256c00,0x000edd805d96dd1f,0x0002f123aa82dd5c,0x000023aae4f8f7eb}}, + {{0x000fcf5a26262cd2,0x000ce060ebd51723,0x000eaa3af1f7f4d5,0x000ccfd19c5c01b2,0x0000ccb9b14b790a}, {0x0001cad52324aa65,0x00062247df541f9c,0x000c96f826320052,0x0003840732fe42ba,0x00002fe771f51a1c}}, + {{0x000a13db1001684e,0x000dc1709f75546c,0x000db8672b56b4ee,0x000cfa466545a9d5,0x0000d971c90fe8f3}, {0x0008691e3a07b296,0x000c84b696b94e7d,0x000c7e9ae7570d9e,0x0008447c5fa0677b,0x00008b44cb0282c4}}, + {{0x00034b3bf44da801,0x00094ab32e66519d,0x00078a000283834f,0x0002f65e60879762,0x0000e62960e72731}, {0x000b27be6901c550,0x000824fdbc1f9b87,0x000acc27d80e7853,0x000b5abbbc09512f,0x00006394239ac143}}, + {{0x0004a40376c1944a,0x0003c3da151135bb,0x000148a3b7cb6269,0x0002ee0fd29161b7,0x00006f9d9edee2ea}, {0x0002ca2880dd2123,0x0003761139a915dc,0x0000f8785903c381,0x0000fefaa7b46d6c,0x00006ce2871c01c6}}, + {{0x000b028e10d9c12f,0x0001132f33d3c683,0x000a31b587573baa,0x000e1197a9b1f667,0x0000d3ed11be4ffa}, {0x000aa9a0cb2748e2,0x00083d6fdf1683dc,0x0007539418239f01,0x000b369a67b49c72,0x0000beec455e321c}}, + {{0x00056063f8b84cec,0x0002cd38c86f8801,0x0008953dd7641708,0x000428454f1ca759,0x0000939e1110e8e7}, {0x000c2b35a914f2f1,0x0003d74b8f9c9b1a,0x0001b2fb039e35ed,0x000ba2c0debdb278,0x0000585638f3d997}}, + {{0x000646e9e2fce99d,0x00004e80857f9c4b,0x00043b52a68a2108,0x00084236d54e4436,0x0000e8d6d63dd8eb}, {0x000156342146a0a4,0x00021eaa36227032,0x0001387878ba826f,0x000d36e27a58bd86,0x00003b6c03c50281}}, + {{0x000afbbb54dde39a,0x000a3b6f2d5f6326,0x000f158e1744e5e8,0x00018ea8b2a99acf,0x000093c8fa18f879}, {0x000f956de058c5cc,0x0001d36f9e7a2182,0x000e31e67216235d,0x0003e6ece0c0dbd2,0x000096449c0b23ac}}, + {{0x0002874170693bd8,0x000d245e63357e9a,0x000427344a28e14f,0x000df8d757f6b356,0x000022e45572cf8e}, {0x0006ee2e6a285cd9,0x000119df3af02b7a,0x00045b8445866f21,0x000e48b0dde2ddf8,0x000086c3726a10e5}}, + {{0x0000c2af7172277b,0x000e5cccb2447368,0x000438ca757b94f0,0x0003fd0dff72672d,0x0000ad1ce127f466}, {0x000ed9dd8f71caef,0x0005a61fdaa69813,0x0006d1637f43272a,0x000977ceff0119bd,0x0000ebc4f9220361}}, + {{0x0007a952f41deff1,0x000ad63b89b702b3,0x0003ff9510e44a59,0x000af4573257dc14,0x00009c02205e752b}, {0x0003069c4b7d692e,0x00031d1502ac46c2,0x0002208462e6392c,0x000b628057b1a21b,0x000051ff946ec1b5}}, +}, +{/* digit=35 [{1,2,3,..,}]*([2^245]*G) */ + {{0x000cb51566c5c43e,0x00085597f0466e85,0x00094d94acff9c91,0x00027cb354e90c49,0x00000a3933301479}, {0x000fac10dc1eb2bf,0x00013ff319fa8427,0x00096527488cfd8c,0x000745f2d4e68401,0x0000a2e067e57aaa}}, + {{0x0002a7f3e5f9f11f,0x0009e6cb3b8eb6d9,0x000f800bd9afe153,0x000e185d1a6dd7dd,0x00006c13cc1baf17}, {0x000c58e325fc3ee3,0x0000731dc3b215f6,0x000a3d3e77109540,0x000e2ce68e7c07af,0x0000f8417a1c4c7a}}, + {{0x0004772813b230d7,0x000ea7344427ec23,0x0007fc56a634d0f5,0x000f76a1548ab1d7,0x0000fab17513e06a}, {0x00010a74f7c4f830,0x0004220a67d9b62c,0x0001209a0a7d2edc,0x0009f01c40417092,0x0000b9815a0face5}}, + {{0x000589b319540c33,0x00097283d6f82842,0x000ae9fcb18490f5,0x000ba072731f84da,0x0000db6d960f3683}, {0x00063bb146110697,0x000e9788bf05c85c,0x0007460d2b19436a,0x000db1205459df34,0x00003f6e095511a7}}, + {{0x000938eb6357f373,0x0008fbd8aa62dc7f,0x000a979fcc5d00f7,0x000a999878dcb92c,0x00007e83eda1b023}, {0x000e2731560bf3d1,0x00090d0fae616b23,0x0009414bd1086e45,0x000ea1682483169a,0x0000b956bc100ea9}}, + {{0x000bb91c31b9c38e,0x000a68ef57b57b85,0x0003bab6f0c5aa90,0x000684fedeb169af,0x0000610ad740d373}, {0x00070df02ba8e15d,0x0005bca7f771f138,0x0002c036c0337edb,0x000e8114acf747b6,0x0000921d57786b94}}, + {{0x00064392c422f7ac,0x00022d348898dbc8,0x0005bfcd1fb63536,0x000e10c3084668c4,0x0000357c9e3eb315}, {0x000b5405b2e5b8ce,0x00010102b9a4b173,0x0000fb1997e94693,0x00062a37c890eb7b,0x0000c225a84b61b6}}, + {{0x000a3c8ee3c76cb3,0x0003a32a1f6ef306,0x00063e9563cf1162,0x000c26b6d5ab6468,0x0000b8a4cbe8c005}, {0x00029a59ce6bb278,0x000184d4b16fdcd5,0x00023798dc4afaa5,0x000fab30624a2679,0x00005e56df6eb307}}, + {{0x000893c2bf296984,0x000497ce76030281,0x0009a558f91fc19a,0x000f7735a5dca3ad,0x00000ceb3fa8d50b}, {0x0006060bc9ba369d,0x0006897888c21baf,0x000a34c07927e103,0x0009800936bf1986,0x00004cf10c2934ae}}, + {{0x0005334859dd614c,0x000a58d0c8ee3a3e,0x000cd51d59c475b5,0x000325a3080d1f07,0x00009c0d0a7788b4}, {0x0008691c234296f5,0x000444887fb61ac9,0x000ea9cf22a0a83a,0x0002f5065114270c,0x0000230a6e8f2480}}, + {{0x000bf0f72e3d5c1e,0x00056f21439ef7a2,0x000e303343771744,0x0002f91edcbf259c,0x00000030a795ce20}, {0x0009ebf1202e9ca6,0x000c15e6e5916f2d,0x000dac4f8e79dde6,0x00004d952072aff1,0x0000c8d087f1b9b4}}, + {{0x000c73dbce913af4,0x000b058a07cbad0f,0x000f00c8a909e587,0x0006abd300da84d4,0x000025cd048f5446}, {0x000b9be90e9d8bf9,0x000aae431b0eb59c,0x0001aecff991616d,0x000c3b43aa117a53,0x00001af92d3e9f4d}}, + {{0x000c292e93fda293,0x0007b97d91bc9b1e,0x000ace1e676bb6c1,0x000ae34509d95faf,0x0000653fe47ee855}, {0x0000b280f680e752,0x000bceb6c26c7318,0x0006d423675eefd1,0x0001d884cdf29fb6,0x0000d70a9978b582}}, + {{0x000e20720445c36d,0x0001898771747a3e,0x0009f73e971d1ac8,0x0000803fc539f794,0x000005cf3d8682e3}, {0x000e20b7b1c7129b,0x000fa69e61f28758,0x000544c2dffadcc1,0x000e53005d3a2f59,0x0000e16f5c24fff5}}, + {{0x00065b8aad581350,0x0009037aa5be73cf,0x0006fd6a0622c211,0x0008cf79373b3f64,0x0000e029db50d397}, {0x000c43794fba0377,0x0006f20797a68bdf,0x00030d38eaefbd68,0x000463cfa5382bbd,0x0000627cfbfc85d7}}, + {{0x0000fef4e4ca4631,0x00072566cc63b233,0x000780900bcef728,0x00027dc161d2cacf,0x000035dc5396b548}, {0x000052e27bf1bc68,0x000f87dfa06c638f,0x0003321da10a224e,0x000c8f6973586d6d,0x0000b0c5738a6152}}, + {{0x0004f2a346060743,0x000870f7047a07ef,0x000a0e30680fe7fe,0x0002220d1a8152e1,0x00002cf43d8b8da5}, {0x000a95f5f02ffe6f,0x0003906ad3eabf89,0x000c8e55e3d9eb9a,0x000dab112c17bb79,0x0000dcd1a7589c81}}, + {{0x000178bb95560983,0x000f501c38867043,0x00067b9124090a1d,0x0000c8459800ff9b,0x0000e5c030453262}, {0x0003c4b70dceecab,0x0004681f648e4b9d,0x00033345cbb2d3c1,0x00097a4981d8376e,0x0000626289bbcf22}}, + {{0x000c6598baebdcf4,0x000935df01e5766a,0x0005876d81a28ae0,0x000800771283da37,0x0000865a96da07b9}, {0x0001bcd237936b24,0x000b2041749425dd,0x0000a2147332f4f4,0x00020390923d6837,0x000097f5dfc1c842}}, + {{0x0004cbd32be5e0fc,0x000657a013759dc7,0x000d872b17475bcb,0x00063c838477c950,0x0000ec6787abfe1d}, {0x0006014d8578c70c,0x0007b8bb6b8b9b00,0x000fb3806c9ad99a,0x000ab2a799008e11,0x0000fe814368d44c}}, + {{0x00015822f4fb344d,0x000f783fa6eba2ee,0x0002c7749b882344,0x00015a922d323d65,0x00008474a998eb0a}, {0x000154d5d1c00d06,0x0008ae3e7aace43c,0x00025ddf87fd581d,0x000738cb44c61925,0x00007a033ec18ae9}}, + {{0x000fec19ef2d2e4a,0x000e35a0ea7f113f,0x000714c0a1bf6767,0x0009edb7fff75e03,0x000023c422eaa23e}, {0x0006b2d540f83afd,0x000d85ea46a7dd5f,0x0002a1208c2c2c27,0x000f7a1b6b424667,0x000013599f87e634}}, + {{0x0004b5cd7b32c6ee,0x00000af61814cf91,0x0008a1bbb61a5a64,0x000aa4adc3df8b20,0x0000f627fd796d79}, {0x0002ffcc4c86bc81,0x000af61539fe4423,0x0008b9533e6f9231,0x0008850d04f25a95,0x000080cf93459e8c}}, + {{0x00095959884aaf7a,0x000267b348a68968,0x000147c87b1959be,0x0001f7f6250e573c,0x0000e0efb3b7d0c6}, {0x000745eca8c325e7,0x00067cff3f70ed00,0x0009ad41d3c91169,0x0007ef03acbc6531,0x0000b01a02160b1c}}, + {{0x000b29363a1483f9,0x0000ea248f96ea32,0x0003157e589eabe7,0x00046d0c6231d334,0x00003a375e66f3c5}, {0x00093436a2afe69a,0x00004166c88ee76e,0x000872093c4f8910,0x000a60848efd0d4f,0x0000e0eb3eb2fe0e}}, + {{0x000790d9d79046e6,0x000d0cee0976af89,0x000071eca4d650f2,0x000bfe43935d9a43,0x00006fcd2c9883b0}, {0x0005eb5696605f18,0x0007254cd38d0e66,0x000d950cfe77e5d0,0x0009b510ee050a43,0x00006ddebdb532e6}}, + {{0x0004a3dfddf74157,0x0008cf6e8d5a6ad9,0x000957f75f7fa130,0x00044754831d1de9,0x0000de2850245817}, {0x00070789e2aeb6b4,0x0004b67a53c26f1d,0x0009defc3ba2b9ff,0x00022bf6963767df,0x000079deed34d380}}, + {{0x000b89b3a8631e8d,0x000daa213746d2ed,0x0000c5f118de855d,0x0005e3e2056cb7b0,0x0000eaefbd0fc9b8}, {0x0009a8dd150892d4,0x0005e18b798503f3,0x000375f1a37b8468,0x00098df6296dd8b7,0x000072cd4b1f78e8}}, + {{0x000f651e9f05de9b,0x00087ce98ba915df,0x00038024cd404506,0x0005eed466a7ae9b,0x0000910e70195a6b}, {0x00056eab3aa8f0df,0x00068eee74a6ae1c,0x0004c4620bab2a50,0x0001f31dca11e24b,0x0000d896e2f3c47d}}, + {{0x000ae53308fbd93d,0x000d32c36fdaeb45,0x000a4838546cd5a2,0x00095f9a3d4e90ba,0x0000d55e62f6dbe9}, {0x0006aa02a81ede7f,0x000409274ea7a140,0x000414f866860dd1,0x000326dfdcb0c280,0x0000f410b1112f94}}, + {{0x000cc3849ad467b6,0x000baa7335f15a33,0x00053a360efb48b6,0x0009cbb4fb54a4b1,0x000004aa9d315246}, {0x000c486754e48e96,0x0004d7471e8e5e9d,0x0003b37b6693cb45,0x000f079b2fd7cd8d,0x00003345e172f09f}}, + {{0x000ba6b23a5d8961,0x00056fe4364e9910,0x00033c6771fe19e3,0x000fd05e1da8c39a,0x00005b4488b39fd9}, {0x00092541a1f22bff,0x000fbb8163e81f43,0x000e5658e920a8a6,0x00039a4fd1b24907,0x00002c4f79da6ec8}}, + {{0x000c3d04aa38d1b5,0x000c65d9510e1abb,0x00060dec03b0db35,0x000b32c754ac783e,0x00003272fd83a099}, {0x000494f07a8e1078,0x0006ea8191fa5fb0,0x0004ad5444a89e13,0x00097b2113b7f63c,0x00008a2e909ecb98}}, + {{0x0005de3b44a3f84f,0x000327c6c69017d5,0x000232390acb2f34,0x000bf64208816810,0x00002e8a6205c733}, {0x000aab69c2d76521,0x0003dd95c5bca774,0x00081f110fb5307e,0x0008c9905c73c249,0x0000baae31cb3945}}, + {{0x000185bcbea62e7d,0x000e1af630591def,0x00021851ce8ac9ea,0x000f5a398a8cfd99,0x0000959c3f20abe2}, {0x000952520e40ae55,0x0002b7a24aa1a4f1,0x00092b2bc320789e,0x00068ad59e692773,0x00008f6c66769186}}, + {{0x000b2bbc55d2d8b9,0x000764f6ca56ce1d,0x00087761441d58bb,0x0009ece650b6808f,0x000005e16bb84c34}, {0x0005140f661acacd,0x000ffb2270afed41,0x000402cba3b8784e,0x0002196bc280ac8a,0x000053f7146d9379}}, + {{0x0008ee5e5681e833,0x0005c6ac9e4ac03c,0x0006b1a386212610,0x000cbd6503a53f93,0x0000d45e2d4a82fe}, {0x000c43976e8ae983,0x000b8fb4b00e69a5,0x000386c89b53b2ee,0x000ce4c167471272,0x00000ca34a27268b}}, + {{0x000d86c783417303,0x00088525e2487f1e,0x0004fbf388ef5beb,0x0003828bc489fdb7,0x00008a92a0ec1a0b}, {0x000ba3f22433ccf6,0x0006429f05a97a77,0x000189afcde8362d,0x00014f1f6a30ea61,0x000093b5505b9ef1}}, + {{0x0006bc0cd1797a18,0x000e74b7af2d5026,0x000f9483eea17b47,0x0008c956c4025c3d,0x0000cbb9da0237b1}, {0x000fd9c4d8424cff,0x0000bb1c350691cb,0x0008206a3db7048f,0x0006cdaeaf641f02,0x0000986f3fa15bdf}}, + {{0x00043b5224c08dca,0x0003e1b50c912621,0x000a8c84f2bbb09b,0x000ca8216ed709ac,0x00006210d9e52850}, {0x000f67a09cb54d69,0x000fc00919a46d8d,0x00013285791eef6d,0x00028b00f613810f,0x0000acede4888d50}}, + {{0x0001b7190b771c33,0x00048a6426be844d,0x000b802ff563b71e,0x000a40fefa2e83bd,0x0000410cbabdb5b4}, {0x0002d2630da84dd0,0x0009ee1cc29a555b,0x000547792d0711ae,0x000b35cf3e8c602f,0x00003d7d5dedc678}}, + {{0x0002fa8ced806b84,0x0004297f1478071a,0x000fcdbbf222e613,0x0003b89c16fd5dab,0x00004912ebf521b5}, {0x00036742496c27c5,0x000bcffc26b0ac94,0x000debf2c8ea3176,0x00083266e224ac13,0x000024cc2364372a}}, + {{0x000e1d89f6f1b18b,0x000494cce35bd706,0x0008e31fc2552f00,0x000046dc8326c2a8,0x00005468b2da9552}, {0x0003e883ff90f2ba,0x000e8f0a5423ce68,0x00056e32877947bd,0x0004ac30a1b28bed,0x0000ee3525462013}}, + {{0x000367d3567962f6,0x000f2188bffb7e98,0x000f130a1379ed61,0x00073413bba348fa,0x0000c1f75e1f04ed}, {0x00066423b4a79fce,0x000c74ef44931895,0x00011eca1f20bc83,0x000dcea36d425d91,0x00005b5c318e09a8}}, + {{0x000b25d13221bc57,0x0001fb3eeaf73360,0x0003a95a1707baad,0x00009f67279ed874,0x0000450a876069e8}, {0x000bd53e5d0338fe,0x000e4b883bbc32b6,0x00063ecd01e77f7a,0x000e46b0da12cc10,0x00002697b59a315b}}, + {{0x000a5bdda85d5347,0x000f8f980eea2771,0x0000385e753e78c1,0x000b623df1cf8490,0x0000d3b14f739387}, {0x00074b0cb8f2bd25,0x0006327fa993170e,0x000f32bab2d50b48,0x000ab87dbe8c9af6,0x00005e906b113b93}}, + {{0x00022fc8fe280d19,0x000522e114ab747f,0x0000b68b0cd8e0de,0x0004b1cab7dbebe1,0x0000dc63a9d3480d}, {0x000bc3b4be1495fb,0x0007e359122d78d4,0x00009cbdc25eb3db,0x00002e8f8ac05b08,0x0000f4187bc837c7}}, + {{0x000a0691416a6a5e,0x000863ef881c84ce,0x000038a5d8f860c7,0x00006661311f8a38,0x000078c2ec1dc612}, {0x0002e815ad735813,0x000029604097494d,0x000612cbab4cc9e0,0x00039ecf558aecf3,0x00005beef7ace36c}}, + {{0x000c7cfdbcf41b9d,0x0009dea997c01845,0x00002f6d85703662,0x000161bb925afee4,0x00000a1b1af1dd72}, {0x000b37503c41c4b1,0x000af391d0429f47,0x0003b8b0aa023829,0x0000e54f5045c350,0x000023c2688a8c01}}, + {{0x000c0cc36ba06eee,0x00045dd2cc0c324e,0x00033e91fface311,0x0002afd364f3bef3,0x0000f8aff7408e83}, {0x000ad042d05841bb,0x000e856a21e21e9b,0x000dd627e42f0e3d,0x000710f3270bcb4a,0x000009a81598322e}}, + {{0x00026a10fee104a6,0x0007d703f65d86e3,0x000bc4833ad7788f,0x00093a1e76543047,0x0000cee582b8b9b8}, {0x000a167e8f55a7b8,0x000659e4190d9cd2,0x0000c2e9defbee3c,0x000547b3ee7185d4,0x000044cc9c62380b}}, + {{0x0008ecd66926e04d,0x000eb110c1ba323f,0x0006a7f070001e38,0x000826bdbcac12fc,0x000065e1d58dcec0}, {0x0004141be76ca2da,0x000c4892f33ad2cd,0x0007139d27895cf5,0x0004c0f56d230d36,0x000091abd3f7012c}}, + {{0x000488387eb36bff,0x0001a14b8fb434fa,0x000b9c95fc5f0710,0x000194a0f0e579ad,0x0000e6ea8cc18888}, {0x0005065edfa9284e,0x00014b8c8d657b9b,0x000e8aafd6c510bd,0x000a06f7b8ebefcb,0x0000db3af9976b1d}}, + {{0x000779d6295d4269,0x0006afa3ad7b28ff,0x0008e26040c4f6ac,0x0000e15c44d0548b,0x0000b32a66e1b005}, {0x0003366f0476ce2b,0x0002f602c7b41f94,0x0004f28097554d95,0x000bea0e35aca652,0x0000688122aad4ed}}, + {{0x0000c8f508efb637,0x0007fabcefc7e8cd,0x00041ab4f9eb5b5c,0x000b2285621f5fb4,0x00009e6c047276a2}, {0x000792ce37a1f698,0x000a83542b6074a4,0x000c20bd3cbd252c,0x00060bf85f65d5b3,0x0000dea6143cfabc}}, + {{0x0001446de6736294,0x000e303c2d2145e2,0x000c868c757f7aa1,0x00067660e99b7f98,0x0000e42f66dcb641}, {0x00084dc910778965,0x000f72c9885b6028,0x0009a5187a0d690c,0x0007eeaeb4da333b,0x0000f789598653c8}}, + {{0x000dd4752b16dba7,0x0005c524c1b12192,0x000383693deefc0e,0x000d97d65ea76ee4,0x00009401711a61b8}, {0x000ace9f21a15cb6,0x00038fee9aeba5f9,0x00077016c73d2616,0x000e066ca844b3e6,0x0000c122b07b7eae}}, + {{0x000dce715f096905,0x0001ddfc0fc9b782,0x000d89fc6508b9b1,0x000b0f4015ab4b65,0x0000e79dab826d5b}, {0x00021f06c775aa2e,0x000b57c7eca164f0,0x0002fa506df09d8c,0x000ec5ba761367ef,0x0000d4ca4773b81e}}, + {{0x000de3610bbb8b54,0x0002c641ada3262e,0x000831ccc0737ce8,0x0006350c94288ae9,0x000087fc1cec065e}, {0x0007ab3b8bb36591,0x000e755e4120b13d,0x000eb0244dea5df3,0x0000a3a9a1857385,0x0000a1b8ea3b7cfe}}, + {{0x000711967b0867ce,0x0007bd3645203b83,0x00030f0e38d5e0d0,0x000bae72dccc1ed9,0x0000fbbcec89f20b}, {0x000fcab0263ad100,0x000dfcd18f8a99cf,0x000f10617d8199e6,0x00084864e2773fe9,0x0000079e8e108704}}, + {{0x000989f8a3422839,0x000c183012e61169,0x0006a90018097799,0x0007fc5ce966cb8a,0x00003b3afef972ac}, {0x0003a2a2db3d5ba7,0x000219bf4fdce689,0x000396673263dc46,0x0002b61852dfc9e0,0x0000ac70895aaf36}}, + {{0x000ce4d5c2f342bb,0x000a052d7aaebb9c,0x00061bcd0bf80907,0x00044ce7f3d3cd21,0x000025b0834b9627}, {0x0008ea56c3a1ddad,0x000a36c92317c5b1,0x0001c4afefe4ec7e,0x000019e787b890ad,0x0000ccd9a92dede8}}, + {{0x000dddadb58da1f8,0x000f38cae6ee9ac6,0x0005c4a4322bbc12,0x0000c746f8bced81,0x0000105a92d79648}, {0x000dbf37a859d51f,0x00054041196b0dc3,0x0001067c9e3ec7ce,0x000dd7e9f64b250d,0x000023213222d1f8}}, + {{0x000619c76497ee80,0x0006c717370e8b5c,0x000cf68e15d2b0ac,0x00079298204cb64f,0x0000bdec21162bc6}, {0x000ccefa63b10110,0x0007e0de1ac56973,0x0000e0c8bf9e3fa9,0x000cb45efb693e3d,0x000037248e9d2d4f}}, +}, +{/* digit=36 [{1,2,3,..,}]*([2^252]*G) */ + {{0x0002dc91ec34f9e7,0x0004c38106038080,0x0000cb4f3d8772d3,0x000128cf06d66c53,0x0000be5ed0e3475c}, {0x0003c1931e82b100,0x0007c9ff6b4ccb9e,0x000a1b45ec63d285,0x000bcab92118c692,0x0000aec44147285b}}, + {{0x0009ae71e29a3efb,0x000f9c93302efc18,0x000aae10ecbe906e,0x0009f820107914ce,0x00007a23f35668e1}, {0x00075c2efd2119d3,0x000eccadc9c8e9d8,0x000a1711303198c6,0x00003835591bf64d,0x0000cf0bbf86d443}}, + {{0x0005bb72b7247593,0x000182d4c63aae48,0x0007d6f2c945353e,0x00010952159d07de,0x000089caef37ec5b}, {0x000bb53db65ef147,0x000e6d99de434a8e,0x000f2405f2dc2cb7,0x0008a3116fa3ed83,0x00003429bba31420}}, + {{0x000d590b01e6e274,0x000da180b2dcb618,0x000aea4a9047e2cc,0x0003a491b299b504,0x000012c9e1edfa40}, {0x0008a36794075521,0x0006e332b8e388d2,0x00068de1949c5013,0x000b972a1b6fcce6,0x000078851bc85122}}, + {{0x0003752fb85fa4ca,0x000d983c8ce9b1e1,0x000f74daed61257c,0x000bbb343da670d2,0x000035aa2405f846}, {0x000235d4421fc835,0x00007363473b5e74,0x0004aa158f6df8ee,0x00022de4d7f52a3c,0x00000d05aabebc6d}}, + {{0x000e735a64785f45,0x000b0f29cd078c56,0x000e35067bc56637,0x00027003b2bb803e,0x00000235a102c919}, {0x000b6d8f2c4aa658,0x00010396023b191a,0x000f805bac347583,0x00080f00400ba5f0,0x0000881065bdec0f}}, + {{0x000e522cc1b5e838,0x0001060b8bfbc370,0x000b256dfde2d4ad,0x0009972d364df067,0x0000f12502f60138}, {0x000a0dc7783920a0,0x000dc0bc866a503f,0x000064ba6e80014a,0x000ba53f89b744d3,0x00003511dcdcba5d}}, + {{0x000d46d95a7b1a29,0x0005ac6341fb197d,0x0004c2ece9c4e7ad,0x000f89b26eca2948,0x0000211e48a6e7f4}, {0x0007f6ec78ef1f42,0x000fe65745861499,0x0003eede82b2c090,0x000017f7286a6e1c,0x00005f92e472f60e}}, + {{0x000564631890a36e,0x000f77feea5b805c,0x0003c3030703ef5f,0x000738589f747caf,0x00000e5daec34dd3}, {0x000a4c3c9c9f1554,0x000675393962fe24,0x00020bf297e4bf17,0x000c3af7183de2af,0x0000a1bd7b6395a8}}, + {{0x000969946191d3d6,0x000d1b87f257a83b,0x000107588281fc8d,0x000fe8518e2c1354,0x0000372def7fb2ba}, {0x000bb480d8972cae,0x000676167a3fdaf4,0x000310cf43f2dd4b,0x0000a93eace32d84,0x00003bcefb0c4270}}, + {{0x000691ed785e73d3,0x0005fea604675fe5,0x000c6514aa5db5ab,0x000664d2e23d41df,0x00005e8048f103c3}, {0x000118f1adaa0f81,0x000574ce1a5a3f8b,0x0006646b828ec3b4,0x00040fd8cacc6e2c,0x0000343d185ebd0e}}, + {{0x000f844caaa358cb,0x0003e924182ae5d7,0x000875d9a1a1db7e,0x000ec8264cd42d9c,0x000037b515fb42ee}, {0x000d4097b165fbe5,0x00099206eff34d4d,0x000b7e17efc322ed,0x000a002dee410259,0x00005a481c0d236c}}, + {{0x0005312c23fc9759,0x000575d6297b8c88,0x0008edd391571580,0x000e521078868ef7,0x000056b31e093c45}, {0x00075d5ff7b33a68,0x00098c7e673f4702,0x0002f2598c8d5dc3,0x000974c19227b47e,0x0000b37b634cc14a}}, + {{0x0007ed68b11888c9,0x000df03e25dcd066,0x00087a24a5e0e8c3,0x000322a4e5d0dcb9,0x0000f40ac3c3e920}, {0x000de9534e0f63a7,0x0001fb6328f95463,0x00064f1b7a128bf9,0x000e34d91ccd7cda,0x0000ef1ec28347bd}}, + {{0x000240fa36a27378,0x0005e3621bc1a857,0x000fb689735dc136,0x00019cfa3a6453d4,0x00000f1a43a49293}, {0x000274bf8cb0ba05,0x00063078c5ebfc18,0x00001d0efb0b5376,0x00009cab0d49241e,0x00000d7c67d872ab}}, + {{0x00070af3aeac968f,0x0008d4b63266b4e3,0x000ac5664e4f7fee,0x000cbec4acd4c2e3,0x00008910bd3beb38}, {0x000e50cc9c0726e3,0x0009a97b40bf1c3a,0x0005a5a1b1530956,0x0004cd40884b7ffd,0x0000890896b1f831}}, + {{0x000515ca5618c937,0x000b07d942d158e1,0x000f767a8e665432,0x000110432181bfb6,0x000053794e8aa604}, {0x000eb7ce8c0dbccf,0x0002698673a309af,0x00046db0031e0261,0x000b276d98e5577d,0x0000c21fb8d8d985}}, + {{0x0000116b0843e0bb,0x0007b9b04531c904,0x000d7d83053b1b3a,0x0007e82d1649f085,0x0000b3bcc887b742}, {0x0001100c93dce831,0x000191922a2a7726,0x000149ce87e79da6,0x000c83487a2b02f3,0x000047e1384ee92e}}, + {{0x00083d3af077f30f,0x00031658b53a484c,0x0007aec53ea78f84,0x00077ca12076c202,0x000034714e483c81}, {0x0005d15c2376c840,0x00093d1aa78337ef,0x000852a908315b65,0x000bd43a75c484ef,0x0000ba0c58a16086}}, + {{0x0008d7a529a6d482,0x000da2f192032968,0x0002e2df99c7f250,0x0001bbf23042fb68,0x0000b7587e7cd812}, {0x0000233e0182a650,0x0007e3e1128a30fc,0x000fb098fb82ecf8,0x0006a77168286193,0x000043e21ae85e9e}}, + {{0x00049d666c834ea0,0x000847414287ab5b,0x0009a2a473be43e1,0x000f3cf40fb85921,0x0000e6559e9cc58d}, {0x000fe8e0c6615b48,0x000cd6459d70fe1d,0x000de038614abc8f,0x000c7bfbe0fa8e05,0x0000e63ef6969035}}, + {{0x00001b453b31e91a,0x0003a436b4d81164,0x0007afd660cba7ad,0x0004c3f151f9a010,0x0000faca8d0bf0ee}, {0x0005c1d9ee9761ca,0x000b50c0588f75fe,0x00004804c3497a16,0x0000b89ee2bebd03,0x00008fb9a6162c99}}, + {{0x00032fe392511143,0x000c9ac73366d14d,0x000a7495c36bf25b,0x00048b99562c66db,0x000024d301b76ad3}, {0x000620cd670407e5,0x000193733a019f46,0x000c324e00ea8d4f,0x0007cd8396d532b0,0x0000b211a0e53c31}}, + {{0x0007d205ffe7b376,0x000ab747d2da090d,0x0004fc5193b7f3ef,0x00071e42cb525fb5,0x0000e220933566a9}, {0x00060dfb486d440d,0x00056fe13465ddc1,0x000e4c1517fcfec4,0x0002b4b3da7e4e76,0x00006fa48a2a8d30}}, + {{0x0004f265872cd88f,0x000b378b90a1c630,0x000f0bc1c806c1d3,0x000d5b4553e725ca,0x0000f59e604ab9d8}, {0x0000f327a0b85dd7,0x000a23ecc217a455,0x000d62213dec5720,0x0009549b88b74169,0x0000212f245cb365}}, + {{0x0004111b5cae787e,0x00073dfd31242076,0x00075aefb13cb7f5,0x0007744dca77da11,0x0000b75466cdfaae}, {0x0006f3bdb6cff322,0x000a41fcda9a74d7,0x00025028b7440f37,0x000f28fbb3ac92b5,0x00000fbf8f7c1975}}, + {{0x00092e1df83097f7,0x000be54b08009826,0x000ce2f2f28738f6,0x0001946c703717a2,0x0000913b93cb0814}, {0x00045931fe896366,0x000f978834a60492,0x00014a5a17b98443,0x000f4bb1c6ab0151,0x00000deb3845fba5}}, + {{0x00054c601a982e6c,0x00007491cd264caa,0x000bd6b051dd35e1,0x000723d73c315f7c,0x0000ab0077612494}, {0x0009b1f6565e15ae,0x0008ac8fb0260465,0x000a0de37bf30f52,0x0001141c21641ba8,0x00009c7a367da5e5}}, + {{0x0009ca552f03ad8b,0x0008524e35c0db84,0x000c3c789c7e8dbe,0x000261f1a2bbaccf,0x0000f733e7e4c26f}, {0x000fbf5b8444823a,0x0007cbf8483b882f,0x000bef640b7224e8,0x0008ccb3023b8b65,0x0000abfec9274d5f}}, + {{0x0001510079ea1bd1,0x000cc05d5d26a40e,0x000e68d4f1ad9add,0x00003fcb3f2eab13,0x0000cff1ae2740f8}, {0x000b749d4cee1172,0x000b2036d909e0e7,0x0004d4c388e9f275,0x0000fcae34e31d8f,0x00002b37f69f7513}}, + {{0x000f1fdb40146044,0x00094941507883e0,0x000792efea8ce991,0x00015b42375b7541,0x0000f59bf5cd7d45}, {0x000324f923a277d8,0x000ce50f3406ac4f,0x00039bc51d9bc9b7,0x00008f46fa87d18a,0x00002588530dccc1}}, + {{0x0003c9f82e4c6346,0x0003da4464f85ced,0x0001dca258efb831,0x00012b8706381b7a,0x0000cd15a3cba2a4}, {0x000a8fdbfcd8fb51,0x000f5e54cd229347,0x000d8932f31db2ee,0x0001afb4aeb11ef8,0x00001e7c1ed44441}}, + {{0x000050cdc9a151e9,0x0007abb0a8592653,0x0005691e79edbfc0,0x00019a026c81c7fd,0x0000c1b2342ff390}, {0x0001c8b7f8474b9c,0x00064176181964a8,0x000e0375a90657c0,0x00047c790b333155,0x0000676c62776ebc}}, + {{0x0003247b7d6dee80,0x0007796593135162,0x000ab35ed0948d92,0x00040899700161e9,0x00006cc32b48ddde}, {0x000d664061ef338c,0x00020202e9ed6f2f,0x0009ba99b1606fa0,0x000f6895388bc192,0x00004428c5ede81d}}, + {{0x00028aef91b0b2ab,0x0003403dfd3fce20,0x000ffe8edce870a2,0x000bffb6ec2c870a,0x0000205fb46d84d0}, {0x000ffe744cefa48d,0x0007d19876d7bf5d,0x000b72863b6fc37a,0x00074eeecfa84c08,0x00007205ff5f5763}}, + {{0x0000d328887de41e,0x000ba69ea5348033,0x00056ea175de0df0,0x000a7733f427533c,0x0000b1f606a252b1}, {0x0004396e30ea15cb,0x0001114941255047,0x0006bb38f575816a,0x0000f70e1ce55bfe,0x0000901a94946ae3}}, + {{0x0000f08d8fc35489,0x0000473bfd08e5af,0x000fe655a5010b5d,0x0009fd093d288053,0x00009f2630bac130}, {0x0007bafb4e3b76fc,0x000bf840784bd867,0x0000092ce14e51dd,0x00020e726c750cbf,0x000083d306cb5283}}, + {{0x000671577d4715c5,0x00091b703235c445,0x00069e986d30019f,0x000c27d07ccb2ed6,0x00007c824b046dbf}, {0x000532fd8f92a230,0x00049bb98fd2f0eb,0x000e6199a4a557fd,0x0001eda57acea7c1,0x0000c6638208b94b}}, + {{0x000be8ff83a92661,0x0006f101bd459b42,0x000bd9cebc7741c9,0x00074485770c1107,0x0000f50250a9b2e0}, {0x000eec81477b6547,0x000da5efe59af762,0x00046a897c65b900,0x000d7c18c9614895,0x0000e8025b4330b4}}, + {{0x00065ef12045cf9f,0x000eaccce8bdae40,0x000cf65256fcb2ca,0x0003112fa0ba4ef2,0x0000683125ebb72c}, {0x000a4eae312410e0,0x00076cd8e830a01d,0x000fb3f0767e2867,0x0009a5abd9575298,0x00005f11e11eef64}}, + {{0x000faef9d3472c26,0x0007677d1345ba47,0x00015afee3adff69,0x000461e761fa04dd,0x00004f1f61b19e69}, {0x0001fab9bfb9093f,0x000eb1133dfefa69,0x000cc710d3df8ae8,0x000e78dd5f896758,0x0000bb88d5106c7f}}, + {{0x0001b4ce88c50d10,0x0007a8771c4f8e01,0x000278ee47532e80,0x00072a64c78a48e2,0x0000b283e8338450}, {0x000f29149e692749,0x00078868b21c98a6,0x000a8908eb96e966,0x00029ca8f0adc2b1,0x00000afcff7afeb8}}, + {{0x000a383210b08568,0x00025ef048899915,0x00064d509a5a8060,0x000f6f000e9af97c,0x00001382d0c38996}, {0x000ba5381927e27e,0x00016af50182490e,0x000ad62ce46c63b3,0x000735984c5fd9d3,0x00004fa1871d8ae8}}, + {{0x000d0bcd7466b255,0x000a8b235c654ec9,0x0003c168884ddbe1,0x0007470e2645ee16,0x000070bd00e50eba}, {0x000b629128bfa0f9,0x000cdc1d3b68fa51,0x0006778b192fce1b,0x0002baae7361dcb6,0x0000c7d249de561d}}, + {{0x00028bf0bbc6229b,0x000e2fd91497a40b,0x00083df051c83c05,0x000c9caf9f5154f0,0x0000ac38b3d9ee66}, {0x000b7e3ec0dfcfd2,0x000e6b0a8416f71d,0x00012aa66f2ecda8,0x00027232fddd8678,0x0000896ef106e6f4}}, + {{0x000186a0fe9a745f,0x000c59ca70dbff27,0x0001cac4908249fc,0x000f568425a2e644,0x00004a0885bdce5f}, {0x000b7317d7ead588,0x00059898d1046e2c,0x0002c9a89f96cf7d,0x0005bc2fe67c9d4f,0x00009895a509c7bf}}, + {{0x000b8e5573cecfa3,0x000ea15f03e6dc7c,0x00008442066497ea,0x0002b03bc0de693f,0x000023b9b36dcd53}, {0x000390a0115a3c12,0x000b0d65ca0ecfed,0x000530c789414c40,0x0008f2441406bd2f,0x00009369a44a3343}}, + {{0x00084f5903fa2711,0x0002a9da921e9968,0x000b01e54e6da0fd,0x00014e96f2f2695d,0x0000ee3e9bd78762}, {0x000181ce27a94979,0x0003fe215e04a26e,0x0002cabca36d254e,0x000613b2f32a6c25,0x0000948148810b57}}, + {{0x000fe6940d9cae17,0x000fa1a10f094c4d,0x00091b64b0586957,0x00053b5a287b5734,0x00007862d5dafd4a}, {0x000856e503491266,0x000111c5268fbf94,0x000b650a62be30bd,0x0009fcb0393f19cb,0x000039531fed78cf}}, + {{0x0006a11b29353593,0x00067f8c126e0255,0x00060167fda38aa9,0x0001cd17dbe6c209,0x00007bbabb680190}, {0x00079e02c9477784,0x00053a1a1dc6b6e9,0x0009faf0cd69a517,0x0005f083ed50959d,0x0000dd9c0965d5fa}}, + {{0x000304d64f16ea85,0x00062e718623a0c4,0x00067f03e8b1cac1,0x000c00bb5765467c,0x000059cf5ae1bd88}, {0x00077bb0e2af19ae,0x0000c1228c920748,0x0006e89201f717ec,0x00080390bcb80032,0x0000c6e2c5d2f312}}, + {{0x000ea7d3fca4752b,0x00087211f62a426a,0x000e7b6b5f12c094,0x00027c74beecd87b,0x000077eaf4ce6d7a}, {0x0002781fda78fd37,0x000fb64eeabe154c,0x000fe2bc4848a83a,0x000c2a11287ef04f,0x0000b6d88c726b6f}}, + {{0x000b947ce417d99b,0x0002db916cc4805f,0x0002733234b93dcc,0x00086dc2e65bb321,0x0000cc1bade1ea98}, {0x00030114bc5ee855,0x0004718ee1e40e22,0x000bcf1f1a561be7,0x000489262fd2d4a6,0x00000e6a5a4e5231}}, + {{0x000001fa00b500bb,0x000bad7dcdf5ca96,0x000446a855c098cf,0x000c61f64e2d2e8c,0x0000ae9bcf2471f3}, {0x00026838435a2c5b,0x000b9bad46434ec2,0x000ccf4e38ceaed6,0x00021e39f8fb47cc,0x0000d4f3fa4cce3b}}, + {{0x000b110a3db32927,0x000a4536c66ad79f,0x00049e6a9e28a37d,0x0008e2b79ce87b8e,0x00000ccfe8e6dcec}, {0x000e4e03ba464b2d,0x000ddca9a3982193,0x0002c12ab0f39d60,0x0000f6ad7932aff8,0x00008ff50ee61e7e}}, + {{0x0001058fa28a7e00,0x00050bf5ec74ea96,0x000229666c726cf2,0x000799e74d55c8db,0x0000bd9abbfa57f5}, {0x000ef074dfc47b3a,0x00026c52f91d7479,0x000a8bde2d9c65fc,0x00027fee0283fe36,0x000032a8b5f1d4b7}}, + {{0x0007c6112e832330,0x0002dcc9bf286a67,0x000780f610fbb351,0x00089c562e8ea50d,0x0000db8b22b1dc4e}, {0x000d1fb89be01445,0x00067a57113b0a6f,0x00009c91c8c77d24,0x00024bf639075dff,0x0000b47b17fa0608}}, + {{0x000a2b016287b52d,0x000000cd8eb058ae,0x000d58573a134352,0x0008ad9148b4d0c5,0x0000d2b6170f91c6}, {0x00039291da3b3b75,0x0008b8c4ac10a61b,0x00017d5835f946d7,0x000e5df105d4a572,0x0000061da3d75e6d}}, + {{0x000940dec1b49919,0x0000a6f485ae3113,0x0001a2ee0f12195e,0x00096e17507fb273,0x00005057a8efe9e1}, {0x000c9112e1301367,0x0005c3c60d15a3c2,0x00000ee2b97dfbb3,0x0008b85af3c581b3,0x00007f25d9164bac}}, + {{0x0004f986d840cd64,0x0000a634288cdb1c,0x000c8a161471d62c,0x000ae24ec2f85ece,0x00001f37cbd3a6f4}, {0x000a20f4b7099851,0x000bdfa8985b6793,0x0008e64467a7bd33,0x0007c11c6a3fbd93,0x000090426193a8d4}}, + {{0x0008667cc36975fb,0x0007ed5f1dfb1684,0x0003baa9402acf16,0x00066fb2d41ad461,0x000056fbb934f684}, {0x0000d0de9e405698,0x000eb5489fefce61,0x000f18b977b99c65,0x000eda8c88ad1b3d,0x00001b7d9bedd0e9}}, + {{0x00018c0c716cc0a1,0x000075691c49d852,0x000356ac6f4b5ff9,0x000a7a94fd666bce,0x00007c728955b327}, {0x0005085da6be7de9,0x000da301d34ef93d,0x000f448e8ff71530,0x000ff9bcd96442d8,0x0000283d331bed18}}, + {{0x000dd992a8498705,0x000af15763354d33,0x0009be0e5a716964,0x000632af5e3a9b17,0x0000b9d6b1bd3b13}, {0x000d7d4a52f313bc,0x000fa37a46603b8b,0x0003e218fc9dd959,0x000a3b700359620b,0x0000e1481a487b28}}, + {{0x000b43a43228d831,0x00003ad63f99ab41,0x000a5122924ae1c3,0x0002b47e525f1a46,0x00004af860fdd26d}, {0x000ef613f714aa18,0x000d6b78795ed6ba,0x000a9d694f51865a,0x00052753e21fcee6,0x00002ceb1de0a37b}}, +} +}; + +#endif /* IFMA_ECPRECOMP7_P256_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ed25519/ifma_arith_ed25519.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ed25519/ifma_arith_ed25519.h new file mode 100644 index 0000000..9a96a73 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ed25519/ifma_arith_ed25519.h @@ -0,0 +1,159 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_ED25519_H +#define IFMA_ED25519_H + +#include + +/* homogeneous: (X:Y:Z) satisfying x=X/Z, y=Y/Z */ +typedef struct ge52_homo_mb_t { + fe52_mb X; + fe52_mb Y; + fe52_mb Z; +} ge52_homo_mb; + +/* extended homogeneous: (X:Y:Z:T) satisfying x=X/Z, y=Y/Z, XY=ZT */ +typedef struct ge52_mb_t { + fe52_mb X; + fe52_mb Y; + fe52_mb T; + fe52_mb Z; +} ge52_ext_mb; + +/* copmpleted: (X:Y:Z:T) satisfying x=X/Z, y=Y/T */ +typedef struct ge52_p1p1_mb_t { + fe52_mb X; + fe52_mb Y; + fe52_mb T; + fe52_mb Z; +} ge52_p1p1_mb; + +/* scalar precomputed group element: (y-x:y+x:2*t*d), t=x*y, ed25519 parameter d = -(121665/121666)*/ +typedef struct ge52_precomp_t { + fe52 ysubx; + fe52 yaddx; + fe52 t2d; +} ge52_precomp; + +/* mb precomputed group element: (y-x:y+x:2*t*d), t=x*y, ed25519 parameter d = -(121665/121666)*/ +typedef struct ge52_precomp_mb_t { + fe52_mb ysubx; + fe52_mb yaddx; + fe52_mb t2d; +} ge52_precomp_mb; + +/* projective falvor of the ge52_precomp_mb */ +typedef struct ge52_cached_mb_t { + fe52_mb YsubX; + fe52_mb YaddX; + fe52_mb T2d; + fe52_mb Z; +} ge52_cached_mb; + +/* bitsize of compression point */ +#define GE25519_COMP_BITSIZE (P25519_BITSIZE+1) + +/* +// conversion +*/ + +/* ext => homo */ +__INLINE void ge52_ext_to_homo_mb(ge52_homo_mb*r, const ge52_ext_mb* p) +{ + fe52_copy_mb(r->X, p->X); + fe52_copy_mb(r->Y, p->Y); + fe52_copy_mb(r->Z, p->Z); +} + +/* p1p1 => homo */ +__INLINE void ge52_p1p1_to_homo_mb(ge52_homo_mb *r, const ge52_p1p1_mb *p) +{ + fe52_mul(r->X, p->X, p->T); + fe52_mul(r->Y, p->Y, p->Z); + fe52_mul(r->Z, p->Z, p->T); +} + +/* p1p1 => ext */ +__INLINE void ge52_p1p1_to_ext_mb(ge52_ext_mb *r, const ge52_p1p1_mb *p) +{ + fe52_mul(r->X, p->X, p->T); + fe52_mul(r->Y, p->Y, p->Z); + fe52_mul(r->T, p->X, p->Y); + fe52_mul(r->Z, p->Z, p->T); +} + + +/* set GE to neutral */ +__INLINE void neutral_ge52_homo_mb(ge52_homo_mb* ge) +{ + fe52_0_mb(ge->X); + fe52_1_mb(ge->Y); + fe52_1_mb(ge->Z); +} +__INLINE void neutral_ge52_ext_mb(ge52_ext_mb* ge) +{ + fe52_0_mb(ge->X); + fe52_1_mb(ge->Y); + fe52_0_mb(ge->T); + fe52_1_mb(ge->Z); +} +__INLINE void neutral_ge52_precomp_mb(ge52_precomp_mb *ge) +{ + fe52_1_mb(ge->ysubx); + fe52_1_mb(ge->yaddx); + fe52_0_mb(ge->t2d); +} +__INLINE void neutral_ge52_cached_mb(ge52_cached_mb* ge) +{ + fe52_1_mb(ge->YsubX); + fe52_1_mb(ge->YaddX); + fe52_0_mb(ge->T2d); + fe52_1_mb(ge->Z); +} + +/* move GE under mask (conditionally): r = k? a : b */ +__INLINE void ge52_cmov1_precomp_mb(ge52_precomp_mb* r, const ge52_precomp_mb* b, __mb_mask k, const ge52_precomp* a) +{ + fe52_cmov1_mb(r->ysubx, b->ysubx, k, a->ysubx); + fe52_cmov1_mb(r->yaddx, b->yaddx, k, a->yaddx); + fe52_cmov1_mb(r->t2d, b->t2d, k, a->t2d); +} +__INLINE void cmov_ge52_precomp_mb(ge52_precomp_mb* r, const ge52_precomp_mb* b, __mb_mask k, const ge52_precomp_mb* a) +{ + fe52_cmov_mb(r->ysubx, b->ysubx, k, a->ysubx); + fe52_cmov_mb(r->yaddx, b->yaddx, k, a->yaddx); + fe52_cmov_mb(r->t2d, b->t2d, k, a->t2d); +} +__INLINE void cmov_ge52_cached_mb(ge52_cached_mb* r, const ge52_cached_mb* b, __mb_mask k, const ge52_cached_mb* a) +{ + fe52_cmov_mb(r->YsubX, b->YsubX, k, a->YsubX); + fe52_cmov_mb(r->YaddX, b->YaddX, k, a->YaddX); + fe52_cmov_mb(r->T2d, b->T2d, k, a->T2d); + fe52_cmov_mb(r->Z, b->Z, k, a->Z); +} + + +/* private functions */ +void ifma_ed25519_mul_basepoint(ge52_ext_mb* r, const U64 scalar[]); +void ifma_ed25519_mul_point(ge52_ext_mb* r, const ge52_ext_mb* p, const U64 scalar[]); +void ifma_ed25519_prod_point(ge52_ext_mb* r, const ge52_ext_mb* p, const U64 scalarP[], const U64 scalarG[]); + +void ge52_ext_compress(fe52_mb fe, const ge52_ext_mb* p); +__mb_mask ge52_ext_decompress(ge52_ext_mb* p, const fe52_mb fe); + +#endif /* IFMA_ED25519_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ed25519/ifma_arith_n25519.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ed25519/ifma_arith_n25519.h new file mode 100644 index 0000000..f284d69 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ed25519/ifma_arith_n25519.h @@ -0,0 +1,32 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_ARITH_N25519_H +#define IFMA_ARITH_N25519_H + +#include +#include + +/* bitsize of base point order */ +#define N25519_BITSIZE (253) +#define NE_LEN52 NUMBER_OF_DIGITS(N25519_BITSIZE, DIGIT_SIZE) +#define NE_LEN64 NUMBER_OF_DIGITS(N25519_BITSIZE, 64) + +void ifma52_ed25519n_madd(U64 r[NE_LEN52], const U64 a[NE_LEN52], const U64 b[NE_LEN52], const U64 c[NE_LEN52]); +void ifma52_ed25519n_reduce(U64 r[NE_LEN52], const U64 x[NE_LEN52 * 2]); + +#endif /* IFMA_ARITH_N25519_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ed25519/ifma_arith_p25519.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ed25519/ifma_arith_p25519.h new file mode 100644 index 0000000..dab1372 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ed25519/ifma_arith_p25519.h @@ -0,0 +1,154 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_ARITH_P25519_H +#define IFMA_ARITH_P25519_H + +#include +#include + +/* +// prime25519 = 2^255-19 +// in the radix 2^52 +// int64u prime25519[5] = {PRIME25519_LO, PRIME25519_MID, PRIME25519_MID, PRIME25519_MID, PRIME25519_HI} +*/ +#define PRIME25519_LO 0x000FFFFFFFFFFFED +#define PRIME25519_MID 0x000FFFFFFFFFFFFF +#define PRIME25519_HI 0x00007FFFFFFFFFFF + + +/* underlying prime bitsize */ +#define P25519_BITSIZE (255) + +/* lengths of FF elements */ +#define FE_LEN52 NUMBER_OF_DIGITS(P25519_BITSIZE, DIGIT_SIZE) +#define FE_LEN64 NUMBER_OF_DIGITS(P25519_BITSIZE, 64) + +/* scalar field element (FE)*/ +typedef int64u fe64[FE_LEN64]; +typedef int64u fe52[FE_LEN52]; + +/* mb field element */ +typedef U64 fe64_mb[FE_LEN64]; +typedef U64 fe52_mb[FE_LEN52]; + + +/* set FE to zero */ +__INLINE void fe52_0_mb(fe52_mb fe) +{ + fe[0] = fe[1] = fe[2] = fe[3] = fe[4] = get_zero64(); +} +/* set FE to 1 */ +__INLINE void fe52_1_mb(fe52_mb fe) +{ + fe[0] = set1(1LL); + fe[1] = fe[2] = fe[3] = fe[4] = get_zero64(); +} + +/* copy FE */ +__INLINE void fe52_copy_mb(fe52_mb r, const fe52_mb a) +{ + r[0] = a[0]; + r[1] = a[1]; + r[2] = a[2]; + r[3] = a[3]; + r[4] = a[4]; +} + +/* convert fe52_mb => fe64_mb */ +__INLINE void fe52_to_fe64_mb(fe64_mb r, const fe52_mb a) +{ + r[0] = xor64(slli64(a[1],52), a[0]); + r[1] = xor64(slli64(a[2],40), srli64(a[1],12)); + r[2] = xor64(slli64(a[3],28), srli64(a[2],24)); + r[3] = xor64(slli64(a[4],16), srli64(a[3],36)); +} + +/* check if FE is zero */ +__INLINE __mb_mask fe52_mb_is_zero(const fe52_mb a) +{ + U64 t = or64(or64(a[0], a[1]), or64(or64(a[2], a[3]), a[4])); + return cmpeq64_mask(t, get_zero64()); +} + +/* check if a==b */ +__INLINE __mb_mask fe52_mb_is_equ(const fe52_mb a, const fe52_mb b) +{ + __ALIGN64 fe52_mb t; + t[0] = xor64(a[0], b[0]); + t[1] = xor64(a[1], b[1]); + t[2] = xor64(a[2], b[2]); + t[3] = xor64(a[3], b[3]); + t[4] = xor64(a[4], b[4]); + return fe52_mb_is_zero(t); +} + +/* move FE under mask (conditionally): r = k? a : b */ +__INLINE void fe52_cmov1_mb(fe52_mb r, const fe52_mb b, __mb_mask k, const fe52 a) +{ + r[0] = mask_mov64(b[0], k, set1(a[0])); + r[1] = mask_mov64(b[1], k, set1(a[1])); + r[2] = mask_mov64(b[2], k, set1(a[2])); + r[3] = mask_mov64(b[3], k, set1(a[3])); + r[4] = mask_mov64(b[4], k, set1(a[4])); +} +__INLINE void fe52_cmov_mb(fe52_mb r, const fe52_mb b, __mb_mask k, const fe52_mb a) +{ + r[0] = mask_mov64(b[0], k, a[0]); + r[1] = mask_mov64(b[1], k, a[1]); + r[2] = mask_mov64(b[2], k, a[2]); + r[3] = mask_mov64(b[3], k, a[3]); + r[4] = mask_mov64(b[4], k, a[4]); +} + +/* swap FE under mask (conditionally): r = k? a : b */ +__INLINE void cswap_U64(U64* x, __mb_mask k, U64* y) +{ + *x = _mm512_mask_xor_epi64(*x, k, *x, *y); + *y = _mm512_mask_xor_epi64(*y, k, *y, *x); + *x = _mm512_mask_xor_epi64(*x, k, *x, *y); +} +__INLINE void fe52_cswap_mb(fe52_mb a, __mb_mask k, fe52_mb b) +{ + cswap_U64(&a[0], k, &b[0]); + cswap_U64(&a[1], k, &b[1]); + cswap_U64(&a[2], k, &b[2]); + cswap_U64(&a[3], k, &b[3]); + cswap_U64(&a[4], k, &b[4]); +} + + +void fe52_mb_add_mod25519(fe52_mb vr, const fe52_mb va, const fe52_mb vb); +void fe52_mb_sub_mod25519(fe52_mb vr, const fe52_mb va, const fe52_mb vb); +void fe52_mb_neg_mod25519(fe52_mb vr, const fe52_mb va); +void fe52_mb_mul_mod25519(fe52_mb vr, const fe52_mb va, const fe52_mb vb); +void fe52_mb_sqr_mod25519(fe52_mb vr, const fe52_mb va); +void fe52_mb_inv_mod25519(fe52_mb vr, const fe52_mb va); +void fe52_pow_2_252m3_mod25519(fe52_mb vr, const fe52_mb va); +void fe52mb8_red_p25519(fe52_mb r, const fe52_mb a); + +#define fe52_add fe52_mb_add_mod25519 +#define fe52_sub fe52_mb_sub_mod25519 +#define fe52_neg fe52_mb_neg_mod25519 +#define fe52_mul fe52_mb_mul_mod25519 +#define fe52_sqr fe52_mb_sqr_mod25519 +#define fe52_inv fe52_mb_inv_mod25519 +#define fe52_p2n fe52_mb_sqr_mod25519_times +#define fe52_p2_252m3 fe52_pow_2_252m3_mod25519 +#define fe52_red fe52mb8_red_p25519 + +#endif /* IFMA_ARITH_P25519_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ed25519/ifma_ed25519_precomp4.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ed25519/ifma_ed25519_precomp4.h new file mode 100644 index 0000000..e85315e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ed25519/ifma_ed25519_precomp4.h @@ -0,0 +1,351 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_ED25519_PRECOMP_H +#define IFMA_ED25519_PRECOMP_H + +#include + +#define MUL_BASEPOINT_WIN_SIZE (4) + +#define BP_WIN_SIZE MUL_BASEPOINT_WIN_SIZE +#define BP_N_ENTRY (1<<(BP_WIN_SIZE-1)) + +__ALIGN64 static ge52_precomp ifma_ed25519_bp_precomp[][BP_N_ENTRY] = { +{ /* slot=0 [{1,2,3,..,8} * 16^(2*0)] * G) */ +{{0x00003905d740913e,0x00005d140beb39d1,0x00088f8a09fd399f,0x0001267a5c184346,0x000044fd2f9298f8}, {0x000c93c6f58c3b85,0x000c6fb8c0e192fb,0x00043d42c2cf932d,0x000ba65270b48986,0x000007cf9d3a33d4}, {0x00091205877aaa68,0x00023ccaac49eabc,0x000d43598c26d9e8,0x00065a85a1b7dcbd,0x00006f117b689f0c}}, +{{0x0009a56042b4d5a8,0x0000c4e60acf68a9,0x00016e37aa8f2b81,0x0002555e09e236bb,0x00006bb595a669c9}, {0x0004e7fc933c71d7,0x000967a0ff5b5922,0x0001d607029f469d,0x0002e2e5aa69a65e,0x0000590c063fa87d}, {0x000aa8b3a59b7a5f,0x000dd5d9acf7843f,0x000b3d6a3136c16b,0x0000b73500fa0840,0x0000701af5b13ea5}}, +{{0x00011fe8a4fcd265,0x000fde5c1ba7d566,0x00014bd6bd3bd353,0x000da628131f31a2,0x00002ab91587555b}, {0x0005b0a84cee9730,0x00030e8864b8aaf2,0x000f016732025a84,0x000f8f4c11b50029,0x00007a164e1b9a80}, {0x000e933f0dd0d889,0x000221c35da6214a,0x000cf2db4c589423,0x000b4c6d170e5458,0x00005a2826af12b9}}, +{{0x000e050a056818bf,0x000715660faa995f,0x0006a05073327e89,0x000a49ac3e8e3cd0,0x000027933f4c7445}, {0x000351b98efc099f,0x000f47dfd2538287,0x000b0a92656765c6,0x0008727ca348d3df,0x0000680e910321e5}, {0x0003fbe9c476ff09,0x000457b5cc1725a1,0x00002b44946e9e39,0x0003e2b5ddbdcf91,0x00007f9d0cbf6355}}, +{{0x000182c3a447d6ba,0x000d14b2729b77f9,0x000864a087d50014,0x00055f3e33cf11cb,0x0000154a7e73eb1b}, {0x0002bc4408a5bb33,0x000c3c75eed02a21,0x000abfec448d5048,0x00006ebdd1beb0c5,0x00002945ccf146e2}, {0x000bdbf1812a8285,0x00007d0bdd1fcbcb,0x000bbda72d270e08,0x000b69ab41b670b1,0x000043aabe696b3b}}, +{{0x000806b67b7d8ca4,0x0008427d22739499,0x00004553b9575be2,0x0007884bb085ce72,0x000038b64c41ae41}, {0x000ceeeb77157131,0x0008900c8af883a0,0x000a59a7369b2715,0x00038bd8065b668d,0x000051e57bb6a2cc}, {0x000c326702ea4b71,0x0000341a1bb0185a,0x00083bc144be70e0,0x00061e353e4a24b0,0x000010b8e91a9f0d}}, +{{0x000f2c9aaa3221b1,0x000533bba23a7ba6,0x0002192c3a6ca021,0x00017e09dea764f9,0x00001d6edd5d2e53}, {0x000a5cd0944ea3bf,0x0003ab39dc0d26b1,0x0008542e49747035,0x000927e71b252822,0x0000461bea69283c}, {0x00036dc801b8b3a2,0x00047053ea49af18,0x000877adf3b3035f,0x00090a7529c41ba5,0x00007a9fbb1c6a0f}}, +{{0x00075dedf39234d9,0x00080e1b558f9e2a,0x000e3c23fb963d76,0x00001c32c2741ac6,0x00003a9024a1320e}, {0x0007596604dd3e8f,0x00077e288702c59b,0x000ed9c3236cb303,0x000e52fb1339c665,0x00000915e76061bc}, {0x0001f5d9c9a2911a,0x000788bcca7d7e7c,0x000eb62a32b8a371,0x0004e95636412190,0x000026907c5c2ecc}}, +}, +{ /* slot=1 [{1,2,3,..,8} * 16^(2*1)] * G) */ +{{0x000b635449aa515e,0x0009f0bc6823aed5,0x000b42d1c4a865c4,0x00015b9850c1fe95,0x000030d76d6f03d3}, {0x000cdd0e632f9c1d,0x00096768931152ec,0x0008637a5851d0b6,0x000ef3952dfb76ba,0x00006dd37d49a00e}, {0x000444172106e4c7,0x00080928d7f696c4,0x00094d3f26fb53d6,0x0004bb0b4739ea46,0x000010c697112e86}}, +{{0x0003c4277dbe5fde,0x000ad19ad7ea2649,0x0006304590265d4f,0x000fe090e00dfc84,0x000025e61cabed66}, {0x00062aa08358c805,0x000e37a2042470ca,0x000b11eddc6a3d4a,0x00006ef7464d3a63,0x000003bf9baf5508}, {0x0003e128cc586604,0x000ecb459747e3f1,0x000c1268f56f5873,0x000e22ca0b63dedc,0x0000566d78634586}}, +{{0x0007a49f9cc10834,0x000d5a89bc451163,0x000f7fd2dbbc8e56,0x00035d91cb5ec0f7,0x000033975bca5ecc}, {0x00054285c65a2fd0,0x0002af31667c3a10,0x00031aee586c6411,0x000b22a680ae2407,0x000014fba5f34793}, {0x000746166985f7d4,0x00084c9c800573cd,0x000b61131e593e5e,0x000526c2fc3f2b67,0x000014829cea83fc}}, +{{0x00037b8497dd95c2,0x00030aa4eb5a7ff4,0x000c85e88b6c744e,0x00081739e0c5d613,0x00002fd9c71e5f75}, {0x00070b2f4e71ecb8,0x000b940a477e321e,0x000e1d4f80e656dd,0x0007b7ebf6556cec,0x000005fc3bc4535d}, {0x0008b3ae52afdedd,0x0008ced3b30cf24b,0x0009be8195349563,0x0001f0433a4bc83a,0x0000373767475c65}}, +{{0x000a99fd40d1add9,0x0006f96f4d0272fb,0x0005f03baeb30716,0x000f9994363f0521,0x00001fbea56c3b18}, {0x000095cb14246590,0x0004016c15535634,0x000910bc60ef1214,0x0007c8c9e38140c8,0x00006bf590573090}, {0x000778f1e1415b8a,0x000f7bac3a77e0fa,0x000aa29a5006409f,0x0005a566f52d7b89,0x000002521cf67a63}}, +{{0x000fee0b0a9d5294,0x0005c0fdf5a66513,0x000fe107ce8f98e7,0x0002cedd4618688b,0x00003fa00a7e7138}, {0x00046720772f5ee4,0x000b196079aceb11,0x0000ac824ae8f894,0x0006cc44af8224d0,0x0000001753d9f7cd}, {0x0009232d963ddb34,0x000dab49738583c6,0x000091f2851dde87,0x000edb6aad7d1f9a,0x000012b5fe2fa048}}, +{{0x0000fbc496fce34d,0x0006badf35bed71f,0x000f28c56173b982,0x000206fd2047261f,0x0000749b76f96fb1}, {0x000b7c26ad6f1e92,0x00023504b8913df2,0x00051c8bc34b66d3,0x000c7b88c409dc07,0x00006f7e93c20796}, {0x000af604aea6ae05,0x000f1bee49c991f5,0x000eff6b66c12351,0x000215161a808b5e,0x00000fcec10f01e0}}, +{{0x000d58a649fe1e44,0x000a231ad777e644,0x00087fd0d221fcae,0x00011f302441c5a8,0x00004901aa7183c5}, {0x0002d29dc4244e45,0x0007493d8de0a3df,0x00020c214d2b020e,0x000b90a6cc8067e8,0x0000413779166fea}, {0x0001b7548c1af8f0,0x0007c246299b408b,0x000e06d939ce0f7a,0x0001213f760b0f91,0x000041bb887b726d}}, +}, +{ /* slot=2 [{1,2,3,..,8} * 16^(2*2)] * G) */ +{{0x00087d44744346be,0x000d415b52b2540e,0x00013b603e1d48da,0x000bdf77c3a8a18a,0x00004eb728c12fcd}, {0x00034c597c6691ae,0x0003d0a85b4c87e2,0x00054afae764889d,0x0009e1ddae2c90c3,0x00000a871e070c6a}, {0x0001b5994bbc8989,0x0003a5bdd4260330,0x0009d59e3c736bae,0x000d4640d61ade21,0x00003ee7300f2685}}, +{{0x000255e49e7dd6b7,0x0005c610b1eacf5d,0x0002e187ca801611,0x00025c23c99975d9,0x0000138157629791}, {0x000a7947841e7518,0x00059639c46d743f,0x0003052b74e5c6fa,0x0009030a1065e1de,0x00007d47c6a2cfb8}, {0x000ad0148ef0d6e0,0x0009a91546f3c3fd,0x0006bb81579d3e74,0x000ec8071ec62102,0x0000148cf58d34c9}}, +{{0x000492f67934f027,0x000bef6840aa946a,0x0009611854469984,0x000bbd45ca1bc2a8,0x00003ff2fa1ebd5d}, {0x00072f7d9ae4756d,0x000bb88f3487fe25,0x000960a88d56c345,0x000a1b99fd10b6d6,0x0000278febad4eae}, {0x000a681f8c933966,0x0009c20290c98b1a,0x00019d3c528c2194,0x000677b391152912,0x00004104dd02fe9c}}, +{{0x0002bf5e1124422a,0x0003398a33ab572b,0x000a52b666a1fa0c,0x00053d594cb6101f,0x00002c863b00afaf}, {0x00014e06db096ab8,0x000c90ce44f35812,0x00009e2af521a8b6,0x000a4816524c12a4,0x00000165b5a48efc}, {0x0000a474a0846a76,0x00084cd2f7cc0f19,0x0008aa2b8f12eff9,0x000c8b8695e29065,0x0000591b67d9bffe}}, +{{0x000f0d1c80b49bfa,0x0005eabf3ec8a312,0x000ef01c88597951,0x0007bcb727033c09,0x00003de02ec7ca8f}, {0x0009b3719f18b55d,0x000faa18c641e99b,0x00029f05ede465e5,0x000128b61081136c,0x0000489b4f867030}, {0x0002102d3aeb92ef,0x000b46116a861d23,0x00090baa24e16253,0x000bebf3d7eabe71,0x000049f5fbba496c}}, +{{0x00049a108a5bcfd4,0x00070bc6473eb309,0x00007c0d1cdc40dd,0x0006e7492c294c13,0x00005604a86dcbfa}, {0x000d628c1e9c572e,0x000acc5884741155,0x00015763eb8a4d86,0x000515b91a352f65,0x000006a1a6c28867}, {0x0008d1d47c1764b6,0x00040e0418b51728,0x0008acf6d1725411,0x00042c69f031a601,0x000020989e89fe27}}, +{{0x000777fd3a2dcc7f,0x0002ca54fd892499,0x000207e3a032857c,0x0007e29a279d864d,0x00000403ed1d0ca6}, {0x0004278b85eaec2e,0x000077acb2bdf167,0x0001cbf45a5621dc,0x00095d3640a4c166,0x0000730b9950f705}, {0x000b2d35874ec552,0x000cf98246f8dc94,0x0006c035cec5e6c8,0x0003dccf7cb46fa1,0x00005bd745430830}}, +{{0x000ad19528b24cc2,0x000656335c1817f9,0x000fc072367f6b54,0x000ad8366b8b66e4,0x0000133a78007380}, {0x0004932115e7792a,0x000a2bdcdddc985c,0x000da3d762c64c89,0x000f82c9d1e3da8a,0x00005bb7db123067}, {0x0001f467c6ca62be,0x000d6211952ee096,0x000bd5477004ec21,0x000e0d2182360779,0x0000740dca6d58f0}}, +}, +{ /* slot=3 [{1,2,3,..,8} * 16^(2*3)] * G) */ +{{0x0008ee0752cfce4e,0x000f306ec08b7df4,0x00095459c4c3fffa,0x000a38d05710b2ab,0x0000161d25fa963e}, {0x000a8c570478433c,0x0000ec281439d231,0x0003d9079fb7b527,0x00003d9dbaa99eae,0x00002c03f5256c2b}, {0x000f18757b53a47d,0x00030cf0c5879790,0x00057ef7f9307b01,0x000bbaf31903d772,0x0000699468bdbd96}}, +{{0x000f2f46f4dafecf,0x00014a47fd6f7bd1,0x000a47b37f7cef01,0x0005785d31ffdda4,0x0000525219a47390}, {0x000d3de66aa91948,0x000c22fc0d2ccd8d,0x0004fdea2f485064,0x0002e3a9b4824663,0x0000293e1c4e6c4a}, {0x000e134b925112e1,0x000b5dca15da0376,0x00061c3111703778,0x0002823b04589af4,0x00005b605c447f03}}, +{{0x0005805920c47c89,0x0000c923b8fccb96,0x0002e2ef77e7f010,0x000b3ee000125650,0x000024a76dcea8ae}, {0x0009fec6f0e7f04c,0x0009e75e349623be,0x000e1de61a866a57,0x000bdd55542ef161,0x00002f12fef4cc5a}, {0x000522b2dfc0c740,0x0007f40c9a4070a4,0x0008cff66810d06e,0x0003790c6cf14417,0x00005e607b2518a4}}, +{{0x00031d8f6cdf1818,0x0004fc36258a258b,0x0006e61d6e35cfa7,0x000d5f7e1b3ff4f6,0x00005067acab6ccd}, {0x000c431ca596cf14,0x00040aed3e400a02,0x000e0f26dbe3c42d,0x0007068d24526802,0x0000201f33139e45}, {0x00027f6b08039d51,0x00064017c0006fd5,0x000e25a4a818b149,0x0000375d5220eb02,0x0000397cba886246}}, +{{0x00013093f05959b2,0x0008de9a9797630c,0x00021d5e26e23aa1,0x0006c3a222fd4917,0x00002339d320766e}, {0x0005c3fbc81379e7,0x00020dde12af1781,0x0005a8fdd5a66194,0x000c252ffa9c0f88,0x0000771b4022c1e1}, {0x000dd986513a2fa7,0x00071f9d4cf08d87,0x000ea283b3f5ac9b,0x0001a76d06bc31b1,0x0000331a18921997}}, +{{0x00066f45fb4f80c6,0x000de61c775cff51,0x000041d91c9c36c7,0x000fe21e3d4e81b9,0x000031167c6b83bd}, {0x00012f3a9d7572af,0x0008868074a9e265,0x000180f7c45bcbe2,0x000a67b84edc1c11,0x00001ac9619ff649}, {0x000b3842524b1068,0x0003bee9ce987f22,0x000a6250c8506834,0x000b111fc9d71844,0x0000612436341f08}}, +{{0x000d41db874e898d,0x000f16c07dc20d99,0x00000f9bbc09fea5,0x000ff40793d2c67d,0x000046ebe2309e5e}, {0x000349e31a2d2638,0x000009bd3fd358b6,0x0003a06ba49ddfb7,0x00004457f8bf1b8a,0x00001522aa3178d9}, {0x00082f5369614938,0x0009ab72d6d102c3,0x000646f227dafe40,0x000306ce8c83391b,0x000045fe70f50524}}, +{{0x000875a6960c0b8c,0x00076ef0e2f20da4,0x000d0b8fd45b68d0,0x00092d407fb51cf3,0x0000428d1623a0e3}, {0x00024920c8951491,0x000c83f630ca262f,0x0005c9d4b805f007,0x00022456fbb45d2f,0x000016619f6db57a}, {0x000f4a4401a308fd,0x000c376a5caac084,0x0003d1bc7da82219,0x00038c6deb8de464,0x00001d81592d60bd}}, +}, +{ /* slot=4 [{1,2,3,..,8} * 16^(2*4)] * G) */ +{{0x00068756a60dac5f,0x0006aebabdc57613,0x000cce0f7d17e02f,0x000dcf07f193f2d4,0x000020234a7789ec}, {0x0005b69f7b85c5e8,0x0008bd168bab2876,0x000d330f9b6ff067,0x0008e7c3a70e77c1,0x00003a5f6d51b0af}, {0x00020db67178b252,0x000f9d51ed16076d,0x0003e41170071c34,0x000e366f62a4a20b,0x00007cd682353cff}}, +{{0x0001a45bd887fab6,0x00032ba403b6e0be,0x00096e60002a846a,0x0000943d9921012e,0x00002838c8863bdc}, {0x0005cd6068acf4f3,0x000183cd7e3d3a66,0x00036025d942d92d,0x000d8ff5759389d3,0x00003ef0253b2b2c}, {0x000bb0cf4a465030,0x0004115c577abd16,0x0004ab419dfa496b,0x000281282cfae8af,0x000021dcb8a606a8}}, +{{0x000004468c9d9fc8,0x0006ed42aa3cb5c6,0x0002ee2f9c254009,0x0001dab125b4d4c1,0x00000bc3d08194a3}, {0x000d00fabe7731ba,0x0007e629e18899a8,0x0003f3d97f820360,0x000678bb2cc02374,0x00005d840dbf6c6f}, {0x000e380d309fe18b,0x000a6b9e165c7706,0x000dae20ab6eb02d,0x00096dd57bbba997,0x00003a4276232ac1}}, +{{0x0002432c8a7084fa,0x000e3dfb9e5454b4,0x000c58e45d898a19,0x000ebd1be9f00219,0x00001ff177cea16d}, {0x0008c172db447ecb,0x0001fc6282dbd3bf,0x0005aa15fe5fcfc4,0x000a9f980acffc07,0x00000770c9e824e1}, {0x0001d99a45b5b5fd,0x000e91b3a7924cf6,0x00003e3e89860984,0x0000b1ee73009193,0x000039f264fd4150}}, +{{0x000d3417dbe7e29c,0x0006a2b9c139ca7a,0x0003597ba9bd9437,0x0009840a0e91b8e9,0x00001712d7346888}, {0x000b4aabfe097be1,0x000e1dfe01929d19,0x000ca6f1ffa46dfc,0x000f14ec3c908942,0x000065c621272c35}, {0x000b89f8ce3193dd,0x00056a125c0bbe72,0x000e1cfe834d1033,0x000e2720419a93d2,0x000022f9800ab19c}}, +{{0x000a368a3e9ef8cb,0x00022a5504715605,0x000f24248fe3e9c0,0x00026e5553d48b05,0x000013f416cd6476}, {0x00029fdd9a6efdac,0x000be34a54941420,0x0007bdf37bb912ce,0x000cab4640f64b98,0x00004171a4d38598}, {0x000758aa99c94c8c,0x0006fb000b807fa2,0x000dda539223006f,0x000d1abfbd291dda,0x0000508214fa574b}}, +{{0x000269153ed6fe4b,0x00039511d77c4c20,0x000c14af94a65a67,0x000a74bcbde26462,0x000022f960ec6fab}, {0x000a15bb53d003d6,0x00088bcf3c965461,0x000c683a5ab21028,0x000b44727c576756,0x00003a7758a4c86c}, {0x000111f693ae5076,0x000df1dfd54a6548,0x0003115e651dae21,0x000f49412248c90f,0x00005d9fd15f8de7}}, +{{0x000408d36d63727f,0x000efd7c7b533031,0x000caee24b6a379a,0x000bed3a9e18fc5c,0x0000332f35914f8f}, {0x00044d2aeed7521e,0x00028432e96153f2,0x000e9c16d48e3a90,0x00098d8e164ba772,0x00003bc187fa47eb}, {0x00070115ea86c20c,0x000cb6c46d1256d4,0x000a660188998ab7,0x000ba03d77832b53,0x0000450d81ce906f}}, +}, +{ /* slot=5 [{1,2,3,..,8} * 16^(2*5)] * G) */ +{{0x000bb6a1a6205275,0x000d7413c8e836e7,0x00088f5cb2aa4f21,0x0005be16f56d155e,0x00002de25d4ba634}, {0x0004d8961cae743f,0x000f5ee1c63edd07,0x0007f4ed29f86d18,0x000b10897bdc55be,0x00004cbad279663a}, {0x00019024a0d71fcd,0x0000afb288af880d,0x000f3a6419c525c2,0x0007233b1a3974b5,0x00007d7fbcefe200}}, +{{0x000f1e6a266b2801,0x000c4d5739f16fae,0x000b03762c866c68,0x0005a8df68a2fbc1,0x00005975435e87b7}, {0x000c5dc5f3c29094,0x0009a2a9105abcd7,0x00021c3058c781a2,0x000d4d780c61d364,0x00004f9cd196dcd8}, {0x000297d86a7b3768,0x000241ad17a63199,0x000c1c0c17d0d058,0x0000307ba029cad5,0x00007ccdd084387a}}, +{{0x0006422c6d260417,0x00050948240bddca,0x000b68c677ae153d,0x000cf53a9c0c1b4f,0x0000428bd0ed61d0}, {0x000c84186760cc93,0x0007a1ab32a999b0,0x00020bda18cdae00,0x000ca44a88dec866,0x00003593ca848190}, {0x0003189a5e849aa7,0x0003565d8facd921,0x0003fdbbd1d4d8c3,0x00063e68c52545b5,0x000027398308da2d}}, +{{0x00038d28435ed413,0x000603278ccc942c,0x0009da03efbd50f3,0x0003355bb07ab1a7,0x0000269597aebe8c}, {0x00010e4c0a702453,0x00066d57d1bdeb9a,0x000d27daf70fa258,0x00033fdffb9d9b5c,0x0000572c2945492c}, {0x000fc745d6cd30be,0x000d3e3baaefbc77,0x000a5dda0ce4dfe8,0x000ca80a22c8830a,0x00007f985498c05b}}, +{{0x0009ce889f0be117,0x0001b7b54a288384,0x0003fc921c8005ad,0x000f3043da3c39f2,0x000076c2ec470a31}, {0x000615520fbf6363,0x00045cf4dfba6d35,0x00073fa0c208045a,0x00012e7eec24fbc8,0x000030f2653cd69b}, {0x0008c938aac10c85,0x00060db276bcb8a0,0x000e6fac7046179b,0x00073daa920c01e0,0x00002f1273f15964}}, +{{0x0009fc7c8ae01e11,0x000904a6aab9f473,0x0007728f2efd5274,0x00069f241d98a828,0x00005d9e572ad85b}, {0x00088bd755a70bc0,0x000a4f1d442e7304,0x000c59616206d6b5,0x000f784ead1a69eb,0x000038ac1997edc5}, {0x0006b517a751b13b,0x000867e9b858c066,0x00054dde49747d06,0x000e69cacacc0114,0x000022dfcd9cbfe9}}, +{{0x000bd2e0c30d0cd9,0x0005facbb43338dd,0x00022a961fad8e66,0x000c1c78f6b258c3,0x00006b2916c05448}, {0x000c59b4103be0a1,0x000ecd259f96956e,0x0003f5cd322ee3ba,0x000e472797cb2941,0x00000fe9877824cd}, {0x000b34d10aba913b,0x000822e6dac0e7ed,0x000578f8154ea3cd,0x0000a1766083dff6,0x00004c303f307ff0}}, +{{0x000a3bd617b28c85,0x000b739773bead30,0x000e6a5cbfc5d377,0x000b7c4c6c6e78c1,0x00000d61b8f78b2a}, {0x000c03580dd94500,0x000a46fbbec9329f,0x0002e2a7f8ecd27a,0x000a1d5130a155fc,0x0000416b151ab706}, {0x0008d7efe9c136b0,0x000cd58e44b2056a,0x000b57e0abbd07e5,0x000e8d2afe62fda1,0x0000191a2af74277}}, +}, +{ /* slot=6 [{1,2,3,..,8} * 16^(2*6)] * G) */ +{{0x0006f74bc53c1431,0x000ce2072eddece1,0x0005b23ee72b9725,0x000c908b8b9c36fb,0x00007e2e0e450b5c}, {0x00062b434f460efb,0x000d4a63607d69fe,0x0007a0da24ded303,0x0005b93f052210eb,0x0000237e7dbe0054}, {0x000575ed6701b430,0x000e69f0bfd10013,0x0003e47f22231094,0x00055e375320f158,0x000071afa699b111}}, +{{0x000e6f9b3953b61d,0x000eaafa141e665c,0x0009f759fec65839,0x000c28e0f435ffda,0x0000021142e9c2b1}, {0x00023c1c473b50d6,0x0001f3b38ef10ea4,0x0002c9be9551e87a,0x0009a1c9b84bf5fb,0x000000731fbc78f8}, {0x0000c71848f81880,0x000225ecec119e43,0x000bba15e3bf960c,0x0005808b6dae0836,0x00004c4d6f3347e1}}, +{{0x0007eccfc17d1fc9,0x000a651403c1418f,0x0007ee0cdf6c75f5,0x0007a22dbde712bf,0x0000193fddaaa7e4}, {0x000cddfc988f1970,0x00027b0b9f51b2f0,0x00079176be6b9162,0x0009fa86ec7b6c47,0x000038bf9500a88f}, {0x0002c93c37e8876f,0x0005a18d1462c1fd,0x0009241276a2f61e,0x00049695080f5823,0x00006a6fb99ebf0d}}, +{{0x0006c1bb560855eb,0x00038f893f09d6a4,0x000f71acc12416bb,0x00096ead71d11378,0x000075f76914a318}, {0x000122b5b6e423c6,0x00010f286ff8eeeb,0x000dcf5d8c939d70,0x000eb1090a92a831,0x0000136fda9f42c5}, {0x000cdfb1a305bdd1,0x0009d9ff82c08f94,0x0003bb588a0f364b,0x000dcba2a87d8a5c,0x0000022183510be8}}, +{{0x000766385ead2d14,0x00080ca7c58304af,0x0000211e3da08ed8,0x0006c030d13a6e61,0x00006a071ce17b80}, {0x000a710143307a7f,0x0009ec47da45f9d5,0x000e927ad3b063de,0x000426c22bbfe52b,0x00001387c441fd40}, {0x0003c3d187978af8,0x0003d7f0e4413b5d,0x000b477ca0722b5a,0x000dc920d7b4848b,0x00003171b26aaf1e}}, +{{0x000f319097564ca8,0x0004c2275e119a92,0x0004875150ff7bb8,0x000835a4f55fe37a,0x0000221fd4873cf0}, {0x000db7d8b28a47d1,0x000d61770a4f1a60,0x0003ddbd58a6bf14,0x00043e9d4a1f8935,0x00006c514a633442}, {0x0002204f3a156341,0x000e9ba0a032d232,0x00010f030efb73e0,0x0004aaafce0dd4c4,0x000048daa596fb92}}, +{{0x000a8e665ca59cc7,0x0004b2e38aca06ec,0x00021e17cea84725,0x0004af731afc708d,0x0000676dd6fccad8}, {0x00061d5dc84c9793,0x000e3ef41820614f,0x00046277ac9941f9,0x00079a9cdf5b88f3,0x000058c837fa0e8a}, {0x0009688596fc9058,0x000f37b56a01b0cf,0x000935d66a1ddcbb,0x0007f0adcc2e77d4,0x00001c4f73f2c6a5}}, +{{0x000a4fbd305fa0bb,0x000e054c663ad0e7,0x000fe33848829d4c,0x0004c42f421c3832,0x0000795ac80d1bf6}, {0x000e706efc7c3484,0x000b4c3c1cf61b36,0x00081cc7e573dfc9,0x000675ceb1d79c97,0x000070459adb7daf}, {0x0001db4991b42bb3,0x000234b02dcca1b9,0x000f8c78dc572696,0x0001fd39fdf9ee51,0x00005fe162848ce2}}, +}, +{ /* slot=7 [{1,2,3,..,8} * 16^(2*7)] * G) */ +{{0x0009214fe194961a,0x000c70d71cd4f4e5,0x000b50f22d49be7d,0x00072329300cfd23,0x00004789d446fc91}, {0x0009852d5d7cb208,0x00070687df2e7287,0x0001687891b8dedd,0x000aa35dc0bffab2,0x00002b44c043677d}, {0x000c87ab074eb78e,0x0008e99daf4671a1,0x00084f9067fac6d1,0x000a4e43eacbbcd4,0x000060c52eef2bb9}}, +{{0x000d89bc3bfd8bf1,0x00037c9f3551a0b5,0x00053028f5b06b92,0x000caab0e4c16b0d,0x000010bc9c312ccf}, {0x000bc5c27cae6d11,0x0009b54a48cab702,0x000a492eb244c769,0x000676defbc4056b,0x000070d77248d9b6}, {0x000ae84b3ec2a05b,0x000f4ed1781e0aa8,0x00008e85d198699e,0x000f413794513e47,0x000063755bd3a976}}, +{{0x000fa03e2ad10853,0x000909ee63569b55,0x000e69b890356f75,0x0006f849ff9f1fdb,0x00000d8cc1c48bc1}, {0x0007101897f1acb7,0x0005ec165bbd83dc,0x000fa1020f5dda7d,0x0002a56508e5b9c0,0x00002763751737c5}, {0x000402d36eb419a9,0x0007e77b460a5029,0x00043c4956f0b44e,0x00066e7cfa86230d,0x000070c2dd8a7ad1}}, +{{0x000194509f6fec0e,0x000a946c6518d656,0x0007e09b5cee2e7e,0x00084959733c1f36,0x00002e0fac636394}, {0x0004967db8ed7e13,0x0000ad776817a91d,0x000d85256474252f,0x0003ce5e40982e00,0x000032b8613816a5}, {0x0007f7bee448cd64,0x00067087886d079e,0x0000e4db2e6ac83a,0x0004f41f89fd4d9a,0x00004179215c735a}}, +{{0x000094e7d7dced2a,0x000c347d39c708c7,0x000906d90297fb8a,0x0009d76e13be033a,0x0000700344a30cd9}, {0x000e33b9286bcd34,0x000b6559dd6dce4a,0x0003d38e1fb7ef7e,0x000c286278b141fb,0x000031fa85662241}, {0x00026c422e3622f4,0x000879833502daf8,0x000b389123c12029,0x00024899bc1b7e12,0x000024bb2312a995}}, +{{0x0008ed1732de67c3,0x00018461b4948b1a,0x0006cfbcd23cb494,0x00080088ebd43437,0x00000fee3e871e18}, {0x00080c2af5f85c6b,0x000c304fa679441f,0x0003ba1bad687284,0x0005d168945df99a,0x00000d1d2af9ffeb}, {0x000a8aa132621edf,0x000a159226579a9d,0x00079ac19330b822,0x0001d764004197ba,0x000016acd7971853}}, +{{0x000f72af2d9b1d3d,0x00036a432245a72d,0x0006b3963763462a,0x00023093ecea0791,0x0000123e0ef6b930}, {0x0009c6c57887b6ad,0x000ad5f90febac95,0x000342f50494e19e,0x000170016e24e62a,0x0000164ed34b1816}, {0x000ed94c192fe69a,0x000ea3a911513487,0x0009a4de2761ae2c,0x000f3eb877bf6d3b,0x000078da0fc61073}}, +{{0x00015d28e52bc66a,0x0001870f01a8e5bf,0x0006c28bdd2c47e3,0x000173a2419afbc0,0x00002d25deeb256b}, {0x000f80f1680c3a94,0x000151ae9e7e6a29,0x000801797371f77e,0x0008ddd1100f1584,0x0000054aa4b316b3}, {0x0008468d19267cb8,0x0009c66e54dafdfc,0x00066eec170b2878,0x000a7602aeb1d2a6,0x0000134610a6ab7d}}, +}, +{ /* slot=8 [{1,2,3,..,8} * 16^(2*8)] * G) */ +{{0x00038ec78df6b0fe,0x00089e575f51b511,0x00017af1b95397da,0x000d65009207a1d7,0x00002102fdba2b20}, {0x000a65e777d1f515,0x000878faa60f1cd2,0x000abc06e5548991,0x000c9fbb1b73bbcd,0x0000654878cba97c}, {0x000ee405055ce6a1,0x000681251ad29969,0x000a7da41536bca7,0x000b2ba3a1af517a,0x00000ad725db29ec}}, +{{0x000267b1834e2457,0x000b570ce1bc5dc4,0x0007d15ed7b67544,0x00036501af07a0bf,0x00004aefcffb71a0}, {0x0007bc0c9b056f85,0x00068e7f5ffd7fec,0x000312aefa537d52,0x0009fd977afc6624,0x00004f675f530239}, {0x000d36360415171e,0x000118998483bc32,0x0000945110cd2bef,0x0006561870a6eadd,0x00000bccbb72a2a8}}, +{{0x000e962feab1a9c8,0x0003565147dcd185,0x000b5b6df286e7e6,0x000b73eb092e031b,0x00004024f0ab59d6}, {0x000d5e4c50fe1296,0x00082fee89f7e186,0x00007031b0e0397b,0x00037c23bc7f6c55,0x00006678fd69108f}, {0x0006fa31636863c2,0x00048572d33f2158,0x00089eaefc07f68c,0x00047014f73cc9f7,0x00002d42e2108ead}}, +{{0x0005131594dfd29b,0x0005d313f4c6a97f,0x0008455010615598,0x000d322eba13f070,0x0000676b2608b8d2}, {0x00017b0d0f537593,0x0000b131e064c217,0x00052ae09f914e69,0x0003c6e1bb687ae7,0x0000420bf3a79b42}, {0x0008ba651c5b2b47,0x000ec311b1b80813,0x000c3135b08671b6,0x000f1e07bff0cb1b,0x0000745d2ffa9c0c}}, +{{0x00025a1e2bc9c8bd,0x0000826479d81bf5,0x000f0155dbea5b26,0x000f5d0d511c70ed,0x00001ae23ceb960c}, {0x0006df5721d34e6a,0x00027997bb3d0603,0x0008756afab1db88,0x000c839d3c209c3c,0x000006e15be54c1d}, {0x00025d871932994a,0x000b5ceb1dab05b7,0x000ab7ca0532351c,0x000c1f77dc41549d,0x000058ded861278e}}, +{{0x00073793f266c55c,0x000c5cc454e49d81,0x000c26c3a8c8c976,0x000f6f95ce382f8b,0x00002ff39de85485}, {0x000b5ba8b6c2c9a8,0x0008ef52c598c2df,0x00012d157348eeef,0x0005bd833809107f,0x000008ba696b531d}, {0x000d3eeec3efc57a,0x00017d4ff481177e,0x0001a671cb04e055,0x000fe54ea3d7a3ff,0x0000120633b4947c}}, +{{0x0004987891610042,0x0003cecebfae80b9,0x0004f0a4c04ee7b1,0x000918570be73959,0x000035d30a99b4d5}, {0x000d31474912100a,0x0006d7e6fbe0682b,0x0001ea79c6de237b,0x0003bdee11e76191,0x000007433be3cb39}, {0x000944c05ce997f4,0x000e4b05c51a3ff7,0x000a76847c575d3d,0x000da9f583381fd5,0x00002d873ede7af6}}, +{{0x000a316443373409,0x000eef4aa81d9157,0x0005a64806fab8b7,0x000a7b6b093fee6f,0x00002e773654707f}, {0x000202e14e5df981,0x000175015e1f5aa6,0x000ae21d6ca20d59,0x000025318a275d3b,0x00000543618a0160}, {0x000abdf4974c23c1,0x000259dce46930de,0x00029aba2caa6f0a,0x000960d04202cb8a,0x00004b1443362d07}}, +}, +{ /* slot=9 [{1,2,3,..,8} * 16^(2*9)] * G) */ +{{0x0004b7c7b66e1f7a,0x00025f50c2f7eccc,0x00013eaf1c44157e,0x00063f73ef06dfc7,0x0000582f446752da}, {0x000c54e91c529ccb,0x0009264c635fb967,0x000812196530f626,0x0006f5c2747aff47,0x000017038418eaf6}, {0x00017bd320324ce4,0x000e8a4488bc4c63,0x000e5a1364a81042,0x0008dc9b21ef18b4,0x00000c2a1c4bcda2}}, +{{0x000dc7d06f1f0447,0x0003edb87c059d24,0x000bb2d28fb2269e,0x0004877d15b0272f,0x00007c558bd1c6f6}, {0x0004814869bd6945,0x0007dbe1c8d22edc,0x00055cc5ab0d6d90,0x000dc83c63bd212d,0x00005a6a9b30a314}, {0x000c1524d396463d,0x0008ac35a24f0d0e,0x000cbc5fa412bb62,0x000afc3a50c3a791,0x00000404a5ca0afb}}, +{{0x000f40070aa743d6,0x000cb5b265ee88c1,0x00068fd2deccbad0,0x0009633574b046b6,0x000046395bfdcadd}, {0x000c9e1b2a416fd1,0x00028e350598b62b,0x000d5d6967b5c6f7,0x000ee9804343fd83,0x000039527516e7f8}, {0x000fdb2d1a5d9a9c,0x000bcd1005c2a117,0x0004d56fea9c7745,0x000d016efd4bef15,0x000076579a29e822}}, +{{0x00068e7e49c02a17,0x000a2bca9a37f45b,0x000c224c1b23cd51,0x000bdb13ed65f11e,0x000043a384dc9e05}, {0x000cb51352b434f2,0x0004993de80e1333,0x00050d35ced83228,0x00077c1b55128877,0x000002c514bb2a27}, {0x000bd5da8bf1b645,0x0007ef6b54b53684,0x0009b0d253fb8bd3,0x0008059313916d7a,0x0000116092096154}}, +{{0x000d166929dacfaa,0x0004c8413598fb44,0x00053d5559da529f,0x0008e0be9ef63ca4,0x0000351e125bc569}, {0x00085616369b4dcd,0x000a7655c35637a3,0x0004f1802175c02c,0x000e0427dc21bf9d,0x00002f637d7491e6}, {0x00049b461af67bbe,0x0007ac8ab8961d4b,0x0009a699fbd60303,0x0002a9a71dee19ff,0x00007f182d06e7ce}}, +{{0x000c8e64ab0168ec,0x0005515edc5437a7,0x0007cd0edacb5a4a,0x00093b0095519d34,0x000067d4ac8c343e}, {0x00054b728e217522,0x000f4d484b8d8094,0x000f46903caa58e8,0x0005217d358254d7,0x000044acc043241c}, {0x000d6bbb4f7a5777,0x000d4918313e11c7,0x00096b46848b35fe,0x00071bd4adca1c6c,0x0000556d1c8312ad}}, +{{0x000f40e30c8d3982,0x0003e15a3fa3417e,0x000773646e31f707,0x0004eff4f21f3cb0,0x0000746c6c6d1d82}, {0x00006756b11be821,0x0002310a3f3dd81f,0x000a99465d0faff8,0x0007f05f8b2d0556,0x0000097abe38cc8c}, {0x0009c9877ea52da4,0x000559bdc1d430c4,0x0007ccebd24c4369,0x00084bd022c3809f,0x0000577e14a34bee}}, +{{0x000268ac61a73b0a,0x000103791a5f5f0e,0x000b6d00e9f2fafa,0x0008f42c1e13e826,0x000060fa7ee96fd7}, {0x000ecebebd4dd72b,0x000da060f221194f,0x0000c8d1fff46a4f,0x0009295124a5977c,0x0000705304b8fb00}, {0x000d1d354d296ec6,0x0003e5fad31d8b63,0x0004bd42ecf3c305,0x00053fd670b958cb,0x000021398e0ca163}}, +}, +{ /* slot=10 [{1,2,3,..,8} * 16^(2*10)] * G) */ +{{0x0005058a382b33f3,0x0000bad48c0b489f,0x00053db36e5ae2ba,0x00032e68f93b503a,0x00005aa3ed9d95a2}, {0x0008aaf9b4b75601,0x000135c8dad72279,0x0001b7a0235eac72,0x0007d4ed2ceaa616,0x00001bbfb284e98f}, {0x000777e9c7d96561,0x0005472c78036656,0x0009506eeecb2b12,0x00057cc65053299d,0x00004a07e14e5e89}}, +{{0x000412cb980df999,0x0006f3c6ec7714ee,0x00025c77fda315d7,0x0003402bba5edde9,0x00003f0bac391d31}, {0x000b58cdc477a49b,0x000de6447f017240,0x0007c86aadfd38da,0x000a08119928d32a,0x000050af7aed84af}, {0x000fde0115f65be5,0x00021216109b26e4,0x000badd6d9299826,0x000d006780205810,0x00001921a316baeb}}, +{{0x00022f7edfb870fc,0x000eb4f76b3bd894,0x0006c24df72c296b,0x000aeb00738f1d43,0x00006458df41e273}, {0x000aad9ad9f3c18b,0x000ef60b1c19cd75,0x00055c0ed9566a0e,0x000c7f53e9a0bac2,0x00007b049deca062}, {0x000be37a35444483,0x000330fedbe93dcc,0x0002c5dd87758879,0x0000e64786004c31,0x00006093dccbc295}}, +{{0x00039a8585e0706d,0x000d8b3e739331ff,0x00018f453b36d0a5,0x000a97c43b9f2e17,0x000057d1ea084827}, {0x000eeebe6084034b,0x000b6780fb8546bd,0x00062d06953199c2,0x0007d90973376abb,0x00006e3180c98b64}, {0x000ab6e7a128b071,0x0006d93a88baaee7,0x0002216130a4c159,0x000bd18f7b4de82b,0x0000363e999ddd97}}, +{{0x000843c135ee1fc4,0x0005508e4c8cf96a,0x00058cd330976eb3,0x000052bb42f6801b,0x000048ee9b78693a}, {0x000848dce24baec6,0x00055babcaf602f1,0x000cefe931769b72,0x000b35590cb3c6e3,0x0000231f979bc6f9}, {0x0001de4bcc2af3c6,0x00030fe208d1f5c3,0x00014fb466b04bb0,0x0002413b78d7009c,0x0000079bfa9b0879}}, +{{0x00003a51da300df4,0x000233da95ab0e39,0x000b356480843964,0x0007194ed3cf12d0,0x0000038c77f68481}, {0x0009ed80a2d54245,0x0007877f63952f3c,0x00010854750aa08b,0x000636bd76dac63d,0x00001ef4fb159470}, {0x000e5ee65b167bec,0x0004296d0cdc2854,0x000810219959590a,0x000ff5672b2df349,0x0000575ee92a4a0b}}, +{{0x000080908a182fcf,0x000c299489dbdd4c,0x0002f733de30e170,0x000fd0005babd575,0x000043d4e7112cd3}, {0x0006bc450aa4d801,0x00027a533b9d85d4,0x000b8906c2c3af12,0x000581b389e3b262,0x0000200a1e7e382f}, {0x000db967eaf93ac5,0x0009b056652c0518,0x00067197f571bc98,0x0004e38fe2b85d95,0x0000050eca52651e}}, +{{0x00031ade453f0c9c,0x0005eff703b9bc34,0x000d847b3de9f504,0x000f4c6fcd97ac9e,0x00004b0ee6c21c58}, {0x000c397660e668ea,0x000fe153ab49797a,0x0004eca79f9b19bb,0x000ae574cb179b53,0x00006151c09fa131}, {0x00055c0dfdf05d96,0x000e02ab4ee7a3af,0x0002171709dd262e,0x000030b11b2bb871,0x00001fef24fa800f}}, +}, +{ /* slot=11 [{1,2,3,..,8} * 16^(2*11)] * G) */ +{{0x000653fb1aa73196,0x000303fd7641837d,0x000b3a17b20f9495,0x000613ead200b09f,0x0000544d49292fc8}, {0x0002aff530976b86,0x00006c2d2460422d,0x000de5bae58d90b8,0x0000c17dca1896c4,0x000028005fe6c834}, {0x000fba9f34528688,0x0009425107da16ae,0x0006d94b365c1bff,0x0006dfaf75bbbcd6,0x000072e472930f31}}, +{{0x0005208c9781084f,0x0000b23450ee1269,0x0003efde02b1502a,0x000a34cfd9daea60,0x00005a9d2e8c2733}, {0x0003f635d32a7627,0x000865f6566f007f,0x0008d044507aaa4d,0x00064383c85e7972,0x00001fee7f000fe0}, {0x000305da03dbf7e5,0x000491434cdbd765,0x00024a88eca4daf2,0x00005437b4ad5cdd,0x000000f94051ee04}}, +{{0x00056b23c3d330b2,0x0009bb0471b068d3,0x000e42b83cf21c8b,0x000b10db36c316c6,0x000007d79c7e8bea}, {0x000f93bb07af9753,0x000cf3db766a7d7e,0x000e0b1ec5583ed0,0x0000452ce6998bf6,0x000047b7ffd25dd4}, {0x000bfb9cbc08dd12,0x0003ae1eec29b87f,0x000b1fc1bf8a066b,0x0004bb60d57242bd,0x00001c3520a35ea6}}, +{{0x000253a6bccba34a,0x000a13838219b80d,0x000882e3963e61c3,0x000e66f90c3b6019,0x00001c3d05775d0e}, {0x00086f40216bc059,0x0001d12bcd87ecda,0x0007c709901fbb23,0x0002e55b4956a9e1,0x000038750c3b66d1}, {0x000ef1409422e51a,0x0003c2b5df671692,0x00044ce029cbc0c7,0x000487c21014fe77,0x00000621e2c7d330}}, +{{0x000860cc8259838d,0x000c1c69f9adcaf9,0x0005581e3090ea48,0x000a5bc652648376,0x00000007d6097bd3}, {0x000e1796b0dbf0f3,0x000b9e17ce196b7a,0x0009aaa3b454dfaf,0x0002e9d25923071e,0x00005d8e589ca100}, {0x000f1d950842a94b,0x00063588f2e3ec0b,0x000b51e2efb2d3c3,0x000bf860a961438b,0x00001583d7783c1c}}, +{{0x000ea2ef5da27ae1,0x0001455670174ece,0x000609167a597c3a,0x0008f70c9a62a126,0x0000252a5f2e81ed}, {0x00034704cc9d28c7,0x0009ef72cc58f900,0x000e5b87261d1b67,0x000580a16e12b5fb,0x00004958064e83c5}, {0x000894265066e80d,0x00085307c8c6b0d2,0x000c1112fdfcc3f7,0x000b3881b53da780,0x0000079c170bd843}}, +{{0x0006ece464fa6fff,0x0001e6205e523050,0x0001b8ea42bee343,0x000fb00357942245,0x00006dec05e34ac9}, {0x0006cd50c0d5d056,0x0006dbb03573bcdd,0x0003c3ef489af768,0x0008acc3ca6723ff,0x00006768c0d7317b}, {0x000625e5f155c1b3,0x000a7997b7b9194b,0x000d6b2600417bf3,0x00052f4c22cbddc6,0x000051445e14ddcd}}, +{{0x00002b4b3b144951,0x0006b444bbcb3575,0x00066385db8e67ff,0x00095c8b8bd69271,0x000013186f31e392}, {0x000147ab2bbea455,0x0004f92079129893,0x000e30f7a78c53a2,0x000d43d4b49f948b,0x000012e990086e4f}, {0x000c96b37fdfbb2e,0x0005e121ceaf9f10,0x000a5b983f9f9a93,0x00099afdf1136c43,0x000077b2e3f05d3e}}, +}, +{ /* slot=12 [{1,2,3,..,8} * 16^(2*12)] * G) */ +{{0x000fa9c59c2ec4de,0x000bf4f84f3cb296,0x0007a8f908bc8b61,0x000255d1c7706d91,0x000063b795fc7ad3}, {0x0008639c12ddb0a4,0x00030c024866bd59,0x0008fce460a5d19f,0x0005e8ad17c2f035,0x000007a195152e09}, {0x00068f02389e5fc8,0x00002cf8de43ba83,0x000541264390433b,0x0000137afa1fd5dc,0x00003e8fe83d032f}}, +{{0x000b15b90570a294,0x00070670845492f8,0x0001bbfd8494f242,0x0004007de1c5ae16,0x000075ba3b797fac}, {0x00004c8de8efd13c,0x0008e33e03731087,0x000260cde3dfc51a,0x0008c86a59d5da51,0x000022d60899a625}, {0x0009dbc070cdd196,0x0008b6c7d8a9a623,0x000b40126060fe8a,0x0009e5eb38847bce,0x00000904d07b8777}}, +{{0x000e1fd4ddba919c,0x0003ec74c8daab4c,0x000d86cc51cf31db,0x000de072c63cc63a,0x000043e2143fbc1d}, {0x00022d6648f940b9,0x0000cbd2d0c39f43,0x000081f93106952f,0x0002a6c167697ada,0x00006240aacebaf7}, {0x0004749c5ba295a0,0x0005bca37d25af83,0x0007c9316ad6947c,0x0000cac66f13ba7e,0x000056bdaf238db4}}, +{{0x000ab9e3f53533eb,0x000d56eb93d40362,0x000d5a5572338568,0x00013189e0e14521,0x00001d24a86d8374}, {0x0000d36cc19d3bb2,0x000b7622386b9131,0x0007a14f5c062a6b,0x00057547c9b8591d,0x000003aa31507e1e}, {0x000c7648ffd4ce1f,0x000f054ac8c1cf4e,0x000d09357ce045ea,0x000485988d225821,0x000043b261dc9aeb}}, +{{0x000b1e1988bb79bb,0x0007dc17a359de55,0x00003dea33a09ed0,0x0006bc2b02c2ee26,0x0000326055cf5b27}, {0x00013d8b6c951364,0x00026000bf47b195,0x00054f956794fe71,0x0000964028d10ddd,0x000002b4d5e24294}, {0x000155cb28d18df2,0x00046186ce508b4a,0x000c824389eacc46,0x0003410c49cf4936,0x000027a6c809ae5d}}, +{{0x0006ebcd1f0db188,0x0003a675a5be88ba,0x0005f5585a37d3d7,0x000a17ef22edfa31,0x00002cb67174ff60}, {0x000c270ac43d6954,0x000576a66cab2cd2,0x0009d7036cdd4a3e,0x000259979fa59246,0x0000221503603d8c}, {0x000ecdf9390be1d0,0x00044728ce3f159e,0x000a94f0f4a94220,0x000f43682891c667,0x00007b1df4b73890}}, +{{0x0002f2e0b3b2a224,0x000062b551160e49,0x000d7f7b0e7c6c9e,0x000599215eb8fe20,0x000061fcef2658fc}, {0x000e221807f8f58c,0x0009fd49409d45f2,0x000fb6a630e3555c,0x000e03db2aaa88d1,0x000068698245d352}, {0x00015d852a18187a,0x000d386ddacd7dbb,0x000ff6c482f3e4aa,0x00001cf44bae2810,0x000046cf4c473daf}}, +{{0x000525ed9ec4e5f9,0x0000116903303426,0x000be5cadc0e5eda,0x0005f4072b1a7f2c,0x000029387bcd14eb}, {0x000c6ea7f1498140,0x000f8392b4854213,0x000629ceba7c1e7e,0x000c5bb2488c38c5,0x00001065aae50d8c}, {0x000c4525df200d57,0x000d6bfca674a1c2,0x00018340305c3b2d,0x000e7160a07e7b1e,0x000069a198e64f1c}}, +}, +{ /* slot=13 [{1,2,3,..,8} * 16^(2*13)] * G) */ +{{0x0002b2e0d91a78bc,0x0009cc8509667906,0x00005070b847c988,0x000a1bf9df54a664,0x00007369e6a92493}, {0x00014434dcc5caed,0x000963c84fb33e10,0x000d86a0e747ed5d,0x000f9e470019576e,0x000025b2697bd267}, {0x00073ffb13986864,0x000d9415dc7b89d6,0x000f273b5e3ca5fb,0x0004cd2e04ecc3bd,0x00001420683db54e}}, +{{0x0008bd1e249dd197,0x000005e58c102b47,0x000cbaac5c620c35,0x000a72dfb02d32fc,0x000060b63bebf508}, {0x000ebb6fc1cc5ad0,0x000e99646ac8b34e,0x00066bde536a1b0c,0x00081c1d3b0da49a,0x000031e83b4161d0}, {0x0008c7129e062b4f,0x0004f29320ad897e,0x000f18683f49e48f,0x00003175bece14b6,0x000055cf1eb62d55}}, +{{0x0009101065c23d58,0x0006d5094819c587,0x0002c55fa78b9d08,0x00091d4e2402fa91,0x0000669a65645708}, {0x0006b5e37df58c52,0x000dde799cc36307,0x000913ee20d73ab9,0x0000133bd831ce34,0x00001a56fbaa62ba}, {0x000e6b505c9dc9ec,0x000bba77c371a943,0x0001347651302557,0x0008a5c9873ae564,0x000013c4836799c5}}, +{{0x000a5d465ab3e1b9,0x00087c7f13f61423,0x000cb5b9b6fc13c1,0x000b60719f83664e,0x000066f80c93a637}, {0x000cfb6a5d8bd080,0x000ec571a4842c4d,0x0008e55365deebc4,0x000b827d4b2e883b,0x000050bdc87dc8e5}, {0x000d37836edfe111,0x00015f011abd9606,0x0005b73b9632353e,0x000d5ae64b03ac32,0x00001dd56444725f}}, +{{0x00047ff83362127d,0x000c471cd7c158fa,0x0009220c8bbc9f6a,0x000732e6e7145434,0x00000e645912219f}, {0x0007e60008bac89a,0x00011eae1c3e0c29,0x000fe7977c7d4cea,0x00005cdf3e38be19,0x00003a3a450f63a3}, {0x000f2f31d8394627,0x00083de94a510078,0x0007996f80389d31,0x000a87bd1e36c6d1,0x0000318c8d9393a9}}, +{{0x00045d032afffe19,0x000497f24db66f27,0x000a8598ef0c9f3c,0x0005314bc98d3e3b,0x0000224c7c679a1d}, {0x00069e29ab1dd398,0x00058342d9e3b5d6,0x00035973cdfc9216,0x0000af655851dfdf,0x0000509a41c32595}, {0x00006edca6f925e9,0x000f4641b1f33bdc,0x000d833e89793ef3,0x000138982ec12809,0x000005bff02328a1}}, +{{0x0002137023cae00b,0x0000ad1accf59363,0x00021a1c88544acf,0x00044a796741049d,0x0000780b8cc3fa2a}, {0x0001a0dd0dc512e4,0x000c844a5fafe688,0x000f4a52404fe70d,0x000a3ea1f748e6b8,0x0000576277cdee01}, {0x00038abc234f305f,0x000bd1405de081ef,0x0004e62a0d9a577f,0x000b7a15e82a5143,0x00005ff418726271}}, +{{0x000e080c1789db9d,0x00025f3e778f5398,0x0006bd035da76020,0x00066befa98894c0,0x0000106a03dc25a9}, {0x000b47e813b69540,0x0003b432610e1e5d,0x0008781276f35d2a,0x000cb69ac1f26e93,0x000029d4db8ca0a0}, {0x000d0aaf333353d0,0x000a5acd309e5d9a,0x000888f7f038669d,0x000befa3c57658ac,0x00004ab38a51052c}}, +}, +{ /* slot=14 [{1,2,3,..,8} * 16^(2*14)] * G) */ +{{0x000c2b256768d593,0x000574422ca13da7,0x000a0ace1d98c1c0,0x000a690f1a80bd5c,0x000029cdd1adc088}, {0x000fd1ef5fddc09c,0x000fdf7575dced6c,0x00001634c2e82b3e,0x0002b9b25d56b5d2,0x00003041c6bb04ed}, {0x0002f2f9d956e148,0x000759f356b2e0ff,0x000f6c025cade797,0x0009a7b1a4698bb5,0x0000104bbd681404}}, +{{0x0000fd3168f1ed67,0x000cdd86f3bc251f,0x0004d2f2de2c811d,0x000714944dc5c430,0x00005be8cc57092a}, {0x000d9a5fd67ff163,0x0009d4cc75681a95,0x000e20f257e92be6,0x0002df5b7f8024cd,0x0000204f2a20fb07}, {0x00043b3d30ebb079,0x0005abd652e30c81,0x000f6d5c31758915,0x000161f653c3c318,0x00002570fb17c279}}, +{{0x000a367f2cb61575,0x000761cd6026c3ef,0x0005b52562f5f96f,0x0000acde8c7142a6,0x00003dcb65ea5303}, {0x000ea9550bb8245a,0x000a88f9050d1192,0x0008a4c935c8e6fb,0x00086687986ea2d8,0x0000241c5f91de01}, {0x0008172940de6caa,0x000f022d9733a28d,0x00035b01d18fbf2c,0x000f0e516d7fcdd2,0x000008420edd5fcd}}, +{{0x000f20ab8362fa4a,0x000d4e21a3e6ecdf,0x000c39e62b57e118,0x00069fde3179617f,0x00000d9a53efbc17}, {0x0008c34e04f410ce,0x0005a276e0685035,0x000bb91521b6135b,0x000889c5d9670c7e,0x000004d654f321db}, {0x000dc116ddbdb5d5,0x000b68da5dd2d5e7,0x000334a2922954de,0x0001ad71cb608173,0x00004a7a4f261899}}, +{{0x000718025fb15f95,0x000346b5c1b8ff4a,0x0000e011123df65f,0x0001848cdfcf0850,0x000011b50c4cddd3}, {0x0003b291af372a4b,0x00070718147f224c,0x0006899ef293da82,0x000ee33dd8485648,0x00004a96314223e0}, {0x0008274408a4ffd6,0x0007e9c1576d9a6e,0x000d02b3f2738e17,0x000cc51773348b63,0x00004f4bce4dce6b}}, +{{0x000fce5ae2242584,0x0005692f58a9ea71,0x000cea3cf426ea72,0x00001e6d21a09d71,0x000073fcdd14b71c}, {0x0002616ec49d0b6f,0x0008fcaec231730e,0x00026b4fa6e45671,0x0005f3748eb409bf,0x00003042cee56159}, {0x000e7079449bac41,0x0006dbce2310a427,0x000f841a7c855ae3,0x000e1d64cae76215,0x0000389e740c9a9c}}, +{{0x000cb3ae34dcb9ce,0x00023e348d0ad64f,0x0002c6381b975003,0x000678845b3f07d6,0x000061545379465a}, {0x000d78f6570eac28,0x0003227919ce1c9b,0x00019b91ede55b0b,0x000369065fc3eaba,0x000025c425e5d626}, {0x000e06a6f1d7de6e,0x000278e0623083f3,0x000e8a6c773ef976,0x00047598c14f6264,0x00006539a0891548}}, +{{0x00021f74c3d2f773,0x0004125c46845e9d,0x0009b99e33c15054,0x000186c624e5ce8f,0x000011c5e4aac5cd}, {0x0004dbd414bb4a19,0x0003c98424f8eddc,0x0006ca716919b2bc,0x000bd9048a89fd73,0x00000f65320ef019}, {0x0006d1b1cafde0c6,0x000e3163b5181d48,0x000af2939a4f3fe6,0x000072a59a8af0df,0x00004cabc7bdec33}}, +}, +{ /* slot=15 [{1,2,3,..,8} * 16^(2*15)] * G) */ +{{0x000e9624089c0a2e,0x000c03afe4738239,0x00064fa12ac748c4,0x000858217dbed2a7,0x0000639b93f0321c}, {0x000f788f3f78d289,0x0002ca1404d9fc08,0x000f65cc9dfe30a7,0x0002021f2778bfcc,0x00007ee498165acb}, {0x000508e39111a1c3,0x000d4809074897bd,0x000e72fd192b2b90,0x00002a6e7d2aec2a,0x00000edf493c85b6}}, +{{0x000c8158599b5a68,0x0000febade20eaec,0x0002b67f07ea574f,0x0004fb44fe41d742,0x0000403b92e3019d}, {0x0007c4d284764113,0x0003ff7f5f835676,0x000ae6bedea09040,0x000a3691c8fcffac,0x000004c00c54d1df}, {0x00022f818b465cf8,0x0005a1480eff84dc,0x0004c7d65771a0f3,0x00076f4aee8bfad0,0x0000355bb12ab261}}, +{{0x000e64cc7493bbf4,0x000d9eca3b0c3a71,0x000a05e785e5bd84,0x000c3120a6bc50cf,0x00000f9b8132182e}, {0x0001dac75a8c7318,0x0009db3ceaa11a30,0x000bae3f2ded9003,0x000ad8e6f077cbf3,0x00007518eaf8e052}, {0x000859c41b7f6c32,0x000bcf4383298a48,0x0009b1d1d90f2d60,0x00055c41815a929c,0x000047c3871bbb17}}, +{{0x0004539771ec4f48,0x0007dc98c5d6e514,0x0007c3c66bf805b1,0x00099dcf762c11a4,0x000000b89b857646}, {0x00065d50c85066b0,0x000b0b3a299b0fbe,0x00041ae8e062ecc4,0x0008d5fe53754ea4,0x000008fea02ce8d4}, {0x000ddd7668deead0,0x000204b685d23824,0x000d89d665c86445,0x000d537b514cfcd5,0x0000473829a74f75}}, +{{0x0002da754679c418,0x000d8b2618df082d,0x000c47eb0ae63bd7,0x000c6b4355eef24a,0x00002078684c4833}, {0x0009533aad3902c9,0x000ceef03588f23d,0x000fe12fb464c2dd,0x000d39015257390c,0x00006c668b4d44e4}, {0x0008cf217a78820c,0x000b281273e973b4,0x000c8eed7bf76a0a,0x000433fa96c65a78,0x00007411a6054f8a}}, +{{0x00059d32b99dc86d,0x00075603af1154d6,0x000cc2e488044cdc,0x00034ffb34c712cd,0x00007c136574fb81}, {0x000ae53d18b175b4,0x00059f392a102579,0x000eef35f5687131,0x000398f8455ecba1,0x00001ec9a872458c}, {0x0006a4d400a2509b,0x000020bc882b4b8e,0x00019575619b81d7,0x000646057e7cc9bf,0x00003add88a5c7cd}}, +{{0x00095770b635dcf2,0x0006cf66c1fbcab8,0x000eb6d18702dfef,0x0009e7485530268b,0x0000249929fccc87}, {0x000298d459393046,0x000985ff659ec85c,0x0002f66e3a8f7e35,0x000a7201d2ca22af,0x000061ba1131a406}, {0x0000a0f116959029,0x0006cba7ebd89a3d,0x0006783307023b6b,0x000ece77bf15a3e2,0x00005620310cbbd8}}, +{{0x000993434934d643,0x00006a51222f5528,0x0003f41c22b9dbf8,0x00097308f6d878fc,0x000037676a2a4d9d}, {0x0006b5f477e285d6,0x000676c8f6193664,0x000bb594dd40e8ff,0x000ec4da6ec7311a,0x00007ec846f3658c}, {0x000e8f3f1da22ec7,0x000776c01cd139b5,0x0002989fb8130f1d,0x0009dd5214c8fcfa,0x00006daaf723399b}}, +}, +{ /* slot=16 [{1,2,3,..,8} * 16^(2*16)] * G) */ +{{0x000a7562eb3dbe47,0x000548ebda0b85f3,0x0005747299f7ea38,0x000d55100c3e5314,0x00001304e9e71627}, {0x000b04bfacad8ea2,0x000e8148be884583,0x000810c5db29b743,0x000bbaa2b1e583b0,0x00002b5449e58eb3}, {0x000814d26adc9cfe,0x0003f8b48dd0b789,0x000979c60a3c1bab,0x000d693da0fe1fff,0x00004468de2d7c2d}}, +{{0x000b355e9419469e,0x0004c23ddc75451b,0x00047f996233e6dc,0x000bd6393a5b6d64,0x00006cce7c6ffb44}, {0x000ad8c6f86307ce,0x00031435d0c284b9,0x00057a772c211135,0x0007352d4a866c56,0x00005da6427e6324}, {0x0004c688deac22ca,0x000f7bbae1ff81a9,0x000d59580fb9066e,0x0002ca888ad8c388,0x000058f29abfe79f}}, +{{0x000ecfab8de73e68,0x0009f377e76a5e90,0x000e01598254036f,0x0001e36f0495b0bb,0x0000577629c4a7f4}, {0x000a64bf710ecdf6,0x00038462c293c4b5,0x00050b3ab9b14ce5,0x00048703643d056d,0x00006af93724185b}, {0x0000024509c6a888,0x000134b558973322,0x000c33289fd2e036,0x000c18f83e236233,0x0000701f25bb0cae}}, +{{0x0008b0f8e4616ced,0x0000e9e25a87dc3a,0x0004bca59cf70066,0x0000be961e3061ff,0x00002e0c92bfbdc4}, {0x0008f6d97cbec113,0x000e674bfdbe49d1,0x000c4e60d6844a06,0x0005e5120f5b522a,0x0000720a5bc05095}, {0x000f09439b805a35,0x000376242abfc0c3,0x000c229346e84e8b,0x000f0ec691417f35,0x00000e9b9cbb144e}}, +{{0x000ad48ffb5720ad,0x0006bdbf90d0efbb,0x00035543bfee8191,0x0007bd8d48131526,0x0000221104eb3f33}, {0x000e9bd55db1beee,0x000370a723fb98de,0x000c68d791c9c3ab,0x0003cde44a8f1bf1,0x0000366d44191cfd}, {0x000c1743f2bc8c14,0x000fcb5856c3b9e3,0x0008a7fb972eda26,0x0003244ccb82f0e6,0x00004167a4e6bc59}}, +{{0x000b9d2876f62700,0x000400e7668eb643,0x0001fc06845d1d9d,0x000246a1b4b43032,0x00007938bb7e2255}, {0x000e2665f8ce8fee,0x00014e880d62cc2b,0x000f364eeee967ff,0x000d2f6f12e6e7e2,0x000034b33370cb7e}, {0x000591ee8681d6cc,0x0009ced85a753cdc,0x0008808883ce0210,0x00065e4ed7485c15,0x00001176fc6e2dfe}}, +{{0x000f6cd05b9c619b,0x000f4b2a58480b4a,0x000be94dc42ddfc9,0x0005f343d4fa502e,0x000008fc3a4c677d}, {0x0000e28949770eb8,0x0002aacf440a3db9,0x000ed7879b98fbcc,0x00006b621354ffed,0x00001f6a3e54f269}, {0x0004c199d30734ea,0x000b631165cd660a,0x000759829540c085,0x00000d1e2333e23f,0x00004f2fad0116b9}}, +{{0x000eb24194ae4e54,0x000511857ef6c44b,0x00068d04985f541c,0x000f7aba61e6b2d3,0x0000445484a4972e}, {0x000cd91db73bb638,0x000aafc129c08962,0x0003b61689e60577,0x000ee816f619b39f,0x00003451995f2944}, {0x0002fcd09fea7d7c,0x00094b0935cf6915,0x0007285c404a816c,0x00093b7258e9aaa4,0x000010b89ca60428}}, +}, +{ /* slot=17 [{1,2,3,..,8} * 16^(2*17)] * G) */ +{{0x000947499718289c,0x000c524533f263d5,0x0004c3ef1512ebf8,0x000518e0262bfcb1,0x000020b878d577b7}, {0x000941be5a45f06e,0x000ed6d9c5f65753,0x0002ff51b6d07cae,0x0004da911776b9c7,0x000017d2d1d9ef0d}, {0x0002af18073f3e6a,0x00019d752106927f,0x000ca60022fd3fe5,0x000c6a722e3b72c3,0x000072214f63cc65}}, +{{0x00037f405307a693,0x000d72f336795b4e,0x0003761099aba714,0x000cbc9d6fbd0a77,0x00005fdf48c58171}, {0x000db7b9f43b29c9,0x0004a4f518f751d9,0x00012f9dc4d60582,0x00045b0f2c072bd3,0x00001f24ac855a15}, {0x000608328e9505aa,0x000d10c1420ee24d,0x0006fb25a24748c1,0x00095e6c7ffe45c0,0x000000ba739e2ae3}}, +{{0x000e98de5c8790d6,0x000d345c2a2df592,0x0009b49922e5bfb7,0x00078f3115a3b60f,0x000003283a3e67ad}, {0x000426f5ea88bb26,0x000d984973bfbae4,0x0006694e50360679,0x000d2265c9f030c2,0x000072297de7d518}, {0x00041dc7be0cb939,0x0004d8b633080482,0x000228930832f19b,0x0001945d3dfc90d0,0x000005e129684627}}, +{{0x0002eeb32d9c495a,0x000fcf12bb97cba8,0x0003b5d1e0ceefc8,0x0008d9bb02dabae9,0x000039c00c9c1369}, {0x000fbbc8242c4550,0x000ecd03081d9adb,0x0005c8df92bcc80c,0x000ce4c843566a6f,0x000078cf25d38258}, {0x000e6b8e31489d68,0x000ab9c2bf08715a,0x00004efa05aa851c,0x000f832c9a75a97f,0x0000006b52076b3f}}, +{{0x0000cfe19d95781c,0x00018966310e229e,0x0000516b39b681df,0x000612257df39d37,0x00004d57e3443bc7}, {0x000b7e16b9ce082d,0x0004c417abc29f5c,0x000bf4a7ab3407f1,0x00075ced4b36bce2,0x00007de2e9561a9f}, {0x0000d4f4b6a55ecb,0x0007f5d85db99de7,0x0003ee9a81480152,0x00029eddbc9c440d,0x00006b2a90af1a60}}, +{{0x0003f4fc9ae61e97,0x0001de03f5fd1692,0x0006edd12d573528,0x0003e4aa764ae43e,0x00005fd8f4e9d12d}, {0x000bf3245bb2d80a,0x000472fb9079b77e,0x000cee7333d8301b,0x0002109c647e6f24,0x0000465812c8276c}, {0x0003beb22a1062d9,0x000753831dc164d4,0x000e2968d77065fb,0x0006790180d4a7bd,0x000005b32c2b1cb1}}, +{{0x00005eccd24da8fd,0x000ac05dfef83c8c,0x000df9cd61a1cf1a,0x0001e99dbbeeff27,0x00003b5556a37b47}, {0x000ca42c7ad58195,0x0006e4333f3ccf7f,0x00040b979d321428,0x00007e1b6c29d0d3,0x000031771a485673}, {0x0000c524e14dd482,0x000541a2ba4b632b,0x00082b5af3edb351,0x00036eba3d160482,0x00004fc079d27a73}}, +{{0x000938b089bf2f7f,0x0006502dfe9a751c,0x000880e4532497bd,0x0008e92ffffc09c7,0x0000124567cecaf9}, {0x00048b440c86c50d,0x000c9cc94e651dc3,0x00043e3cb91337cb,0x000cd086422f74d6,0x0000241170c2bae3}, {0x0009ab860ac473b4,0x000ee0113e4353ff,0x000bc6c4aff0911d,0x000000d4ae75060e,0x00003f8612966c87}}, +}, +{ /* slot=18 [{1,2,3,..,8} * 16^(2*18)] * G) */ +{{0x000a0cc9782a0dde,0x000b2ea718385559,0x0001ef238c551dcd,0x000613d7f62865b3,0x0000504aa7767973}, {0x0008fcfa36048d13,0x000b373899ddd9c1,0x000f92d0aa29159d,0x00019d4dc9f350b9,0x000026f57eee878a}, {0x000b2cd55687efb1,0x00062247af17b0ca,0x000f5a24675180d1,0x000306985c15a344,0x00004041943d9dba}}, +{{0x0000eeba43ebcc96,0x0009c26ea9cafc3c,0x000c77ccc68d749c,0x000340fd9fa95ee1,0x00001420a1d97684}, {0x00017743a26caadd,0x00024648ab7ce4b2,0x0003fbc9e347a6b4,0x000d019cb1d4f7a0,0x000012d931429800}, {0x00067799d337594f,0x00040b23aa47b00c,0x00035ff3955e3c51,0x000a01244182854e,0x00001b4f92314359}}, +{{0x000c109d89150951,0x000912de9696a3e5,0x00075f302039cefa,0x0002dae20eae43f9,0x0000239b572a7f13}, {0x000f3030a49866b1,0x000d2215f485933c,0x0001def4f6251f73,0x00023f6ab82aa405,0x00005ff191d56f9a}, {0x000ed433ac2d9068,0x000795fc98523819,0x000593eb3d2883ab,0x00036cbef4572805,0x0000020c526a758f}}, +{{0x000834f89ed8dbbc,0x000f9dc7ca46c779,0x0003e1b074c8f2aa,0x0003877a9524cdca,0x000002aacc461531}, {0x0001ef59f042cc89,0x0009d8e124bb6e93,0x000ec759972c589c,0x000c50cadc8e18aa,0x0000452cfe0a5602}, {0x0000f7a0647877df,0x000270e607c9f86a,0x0001fb11c9bbc464,0x000877bab17ea25f,0x00004cfb7d7b304b}}, +{{0x00043d6cb89b75fe,0x000d99c6adc8072b,0x000ee34c9f54c694,0x0005364b8c3aa373,0x000014b4622b3907}, {0x000699c29789ef12,0x00071df57190de28,0x000cc970d02b6ecd,0x0003ac5c343c857e,0x00005b1d4cbc434d}, {0x000b2615cc0a9f26,0x0002bb88dcce5b6f,0x000369a7053a4f0e,0x0002dd11301498b3,0x00002f98f7125859}}, +{{0x0004a74cb50f9e56,0x000a98e8e13200c9,0x0002300f675b1ff4,0x000aaf99a2acc218,0x00003a6ae249d806}, {0x0002ae444f54a701,0x000f0a9cbd7de2e1,0x0005835de0fcfe3e,0x0004554cebf890d7,0x00001d8062e9e761}, {0x000ada85a9907c5a,0x000b591b90f62657,0x000f34b4e91a0ea8,0x0005ff38d0e1dfbd,0x0000298b8ce8aef2}}, +{{0x00027953eff70cb2,0x0002a791570762a9,0x0000a7cf6a4b89c9,0x000e4859418457a3,0x000034b8a8404d5c}, {0x000a72ea0a2165de,0x000b40bcf79f6837,0x000738ae703fab07,0x000d7dc521636c77,0x00006ba6271803a7}, {0x000eecb583693335,0x000df63b5fefdc26,0x0004b22573d5a813,0x0001c6aa293aa9aa,0x000071d62bdd465e}}, +{{0x0003cc28d378df80,0x000790a0fa4b4653,0x000701da5af6db43,0x0002ba4e3645ff9f,0x000074d5f317f317}, {0x000db5dab1f75ef5,0x000cf16b065f5cd2,0x000f49f085d77f95,0x0002b3d14571fea3,0x00001c333621262b}, {0x000fe55467d9ca81,0x000752b298c37a86,0x0003ac623b398b7c,0x000d98cda6d0892e,0x00004aebcc4547e9}}, +}, +{ /* slot=19 [{1,2,3,..,8} * 16^(2*19)] * G) */ +{{0x0000071b276d01c9,0x000c586c48c7012f,0x0001d6fba9e7b8ba,0x000b7925308129b7,0x00005d88fbf95a3d}, {0x00008d9e7354b610,0x000535ba85b6e0b4,0x000a58a207806b32,0x000df2cdbe63a034,0x0000173bd9ddc9a1}, {0x00000f1efe5872df,0x0002ed43918c12b5,0x0009673ae058d658,0x000a319e6ed278ec,0x000006e1cd13b19e}}, +{{0x0000ad516f166f23,0x000931fab6abe40d,0x00004d088e118e32,0x00062663fe35e14a,0x00003080603526e1}, {0x000baf629e5b0353,0x00090278d0447472,0x000643bf273baa0b,0x0007b130c785f469,0x00007f3a6a1a8d83}, {0x000644395d3d800b,0x00055c901edf6f7e,0x00092c633995a8d5,0x000307e68cd78305,0x000030d0fded2e51}}, +{{0x00094d1af21233b3,0x0008ef0cc4d9ce05,0x000f499a771bdbe7,0x00098686965187f8,0x00000a9214202c09}, {0x0004971e68b84750,0x000296664bbcf9cb,0x0002fa412ba09572,0x00089d95c8de7267,0x00004615084351c5}, {0x000019c0aeb9a02e,0x0000d16034caebc9,0x00059932ec55c711,0x0005dfe0e6df5016,0x00003bca0d2895ca}}, +{{0x000031bc3c5d62a4,0x0003ecff07a6040f,0x00030fb54519fc8b,0x00013cd98183da21,0x00005631deddae8f}, {0x00088eb69ecc01bf,0x000ada644896f9c6,0x000f7a9fe2f0bc83,0x0008241ca2d955f5,0x00004ea8b4038df2}, {0x000d460af1cad202,0x00005a48cee832ae,0x0009f11a5f463053,0x000a463912177454,0x000024ce0930542c}}, +{{0x000890f5fd06c106,0x000355d8810f21fe,0x000e8caf3eb5c468,0x000d74b827808fe6,0x000041d4e3c28a06}, {0x000fa155fdf30b85,0x0008e36372ea43fc,0x000492f844d2f716,0x0004280b2e064de6,0x0000549928a7324f}, {0x000e32a763ee1a2e,0x000b7d25ffdeaf26,0x00017f4d69ae91e4,0x000ff6abc3bd33bd,0x0000491b66dec0dc}}, +{{0x0005b13dc7ea32a7,0x000cc7e16db9898f,0x000bf8d947e3d5f8,0x000e4acac0abf52c,0x000008f338d0c85e}, {0x00004a8ed0da64a1,0x000af67e2284b75f,0x000f7b7ba4ed222c,0x0008b678234a3791,0x00004cf6b8b0b701}, {0x0003a821991a73bd,0x00001df320c7ac38,0x0004777063ab27bc,0x0008a99c13d331b8,0x0000530d4a82eb07}}, +{{0x000c3630e1f94825,0x000268cab535a004,0x000c84ff8b7e2d78,0x00070b9c7482323c,0x000065ea753f1017}, {0x000973456c9abf9e,0x000fc4900a8806d6,0x0008cfb850257fb2,0x000bd5b2bacf412c,0x00000db3e7e00cbf}, {0x0006fc3ee2096363,0x0007f61b5cb6b3d6,0x0003443b1a81d62c,0x000a1db0fbe04421,0x000002a4ec1921e1}}, +{{0x0006259a3b24b8a2,0x000cc45afa0b85ce,0x000ba07037b8577a,0x00009bfcccbe6e88,0x00003d143c511278}, {0x00086162f1cf795f,0x0001926ee57f2f5c,0x000c063578118c86,0x0007fcf172124851,0x000036d12b5dec06}, {0x000d279179154557,0x0005cfc783a0a126,0x000f179bacd5e48f,0x000285936bdb6e8d,0x00002ef517885ba8}}, +}, +{ /* slot=20 [{1,2,3,..,8} * 16^(2*20)] * G) */ +{{0x0007974e8c58aedc,0x000fbabf041a4463,0x000980718ab9ef22,0x000a8a6e185d956e,0x00002f1b78fab143}, {0x000ebffb305b2f51,0x000ad889596b896e,0x0006d5dd25d3f938,0x0000095f0f52dc74,0x000057968290bb3a}, {0x000ab8430a20e101,0x0008d24f0ec47f71,0x000ee2eed1f39365,0x000a3e1cf7509a86,0x00007dc43e35dc2a}}, +{{0x00066665887dd9c3,0x000314bb05355859,0x000f2079b1c90f9b,0x000c12fc6e08df8e,0x00007ef72016758c}, {0x00082a5c273e9718,0x000995e4efd945a7,0x000f237d3e3576c6,0x0000a990f2ed8051,0x0000044fb81d82d5}, {0x000f18c5a907e3d9,0x0001dce4c6359c1d,0x000201bb4957b337,0x000dd2eca704534b,0x00007f79823f9c30}}, +{{0x0004d239a3b513e8,0x000d4b91fa8d8833,0x000590bd33c13670,0x000d9b412b54136f,0x00000a4e0373d784}, {0x000c1ff068f587ba,0x0004e0050c8de6a9,0x000ded5be7082789,0x000d6f03cbf99557,0x000064a9b0431c06}, {0x0003d6a15b7d2919,0x000a0d53a82352eb,0x0009a45d47b0b4f6,0x000346c7156ce438,0x0000071a7d0ace18}}, +{{0x00072daac887ba0b,0x00005bfa562eed30,0x0000ef768b012629,0x0007e9ccf543002c,0x00002c3bcc7146ea}, {0x000c355220e14431,0x0000709b15141cc0,0x00009d5f360d6595,0x00055d39af5621b2,0x00007c69bcf76177}, {0x0000d7eb04e8295f,0x000252f50f37d07f,0x00071798d710db18,0x000a51de951a9a31,0x00006f5a9a7322ac}}, +{{0x0001000c2f41c6c5,0x000c10cfefb9b8ba,0x000cc51c9fc49f79,0x000afca4efa47703,0x0000494e21a2e147}, {0x0009d4eba3d944be,0x000408078af9ee72,0x0007869c038d9e09,0x0003b244525567a4,0x000002ab9680ee8d}, {0x00048a85dde50d9a,0x0004e0fb9a249efa,0x00091ef6d9219a22,0x000bb34fa091f1dd,0x00006b5d76cbea46}}, +{{0x0007556cec0cd994,0x0006f5cd01dba885,0x000f42b4776472dc,0x0007354af0169148,0x00000ae333f68527}, {0x000941171e782522,0x00074036936d3e0f,0x0000fcc746f1e6ae,0x000313e408b3ea2d,0x000016fb869c03dd}, {0x000e199733b60962,0x000b4d8abe133288,0x000991d03e24fc72,0x000d0754811f7ed0,0x00003f81e38b8f70}}, +{{0x00010fcc7ed9affe,0x000a12465874b7f9,0x000b0c4704545cb8,0x0000993a8397ed24,0x000050510fc104f5}, {0x000b7f355f17c824,0x000c3d74299a40ad,0x000bf8eaf774b923,0x000dc3dd57c3e8bc,0x00000ad3e2d34cde}, {0x000c0fc5336e249d,0x00019c331cfd96f0,0x0009eefe1c745ede,0x0001ebef2d6fd000,0x0000127c158bf0fa}}, +{{0x00097c422e9879a2,0x000d452ca3647f61,0x000b4eaccba44add,0x0004f689b413fc14,0x0000354ef87d07ef}, {0x00028fc4ae51b974,0x000d3744dfe96dea,0x00073848a81d9973,0x000df956240680b8,0x00004ed82479d167}, {0x0003b52260c5d975,0x000fceb41b0b8fee,0x0009f6653c50352e,0x000236d8808ac30a,0x0000302d92d20539}}, +}, +{ /* slot=21 [{1,2,3,..,8} * 16^(2*21)] * G) */ +{{0x0003c1a2bca4283d,0x00091a1863dd9781,0x000268fa86ed62f0,0x000ae4caec7bcb8c,0x000010e5d3b76f1c}, {0x000c6fb6e4e0f177,0x00029a4bd6a932db,0x00087af6e804e1bf,0x000d0605e1966d47,0x00000edc5f5eb426}, {0x0003bfd653da8e67,0x000ec24a9f641545,0x0003578a23e9dc1e,0x000ba72bf87263b0,0x000045b46c51361c}}, +{{0x00002abf314f7fa1,0x000dc8e8cf450a94,0x0003a8be84e257f1,0x000713b1dbbd54b2,0x00002177bfa36dcb}, {0x000d4ddd8a7fe3e4,0x0005676620e30ce9,0x00030e9958ab1364,0x00029df4b594f7bb,0x00005c1c0aef3212}, {0x00081bbcfa79db8f,0x0001ec25f59b3370,0x000c832487604881,0x000b5bb087a76659,0x00004ae619387d8a}}, +{{0x000bf6aa5344a32e,0x000b4b41b40788dd,0x000a130d607d88ea,0x0003e035eb0eb974,0x00001a00d91b17bf}, {0x00017e44985bfb83,0x0002a71963136611,0x000425904bfce046,0x0003d6483ac3448d,0x000075685abe5ba4}, {0x00060933eb61f2b2,0x000a8c9ff49526e9,0x000af66569543d0f,0x000e6aadf7275107,0x0000135529b623b0}}, +{{0x0000dbd7add1d518,0x00088cfc11f1118f,0x000114759b979f78,0x0003a018732e1f07,0x000079b5b81a65ca}, {0x000716bce22e83fe,0x00019e80985c1f5c,0x0004254aaeb42beb,0x000a613ec9da6371,0x00005972ea051590}, {0x0004ac20dc8f7811,0x00094ac4d4fa80fd,0x00033604349a9ad2,0x0003bdbc01b2d64b,0x00004f7e9c95905f}}, +{{0x00074bbc5781302e,0x0003989addc0f626,0x0003fbd9c6d8520f,0x0008e4c8c2999ae5,0x000031993ad92e63}, {0x0008443d355299fe,0x0001cdbebead771c,0x0001a494668bcd3b,0x000adc88092499ef,0x00001942eec4a144}, {0x000c5319ae234992,0x000910cea3e927da,0x00053c11222c1b3d,0x000ca75553ce4942,0x00002a0a65314ef9}}, +{{0x0007937ff7f927c2,0x0000617d0a6352db,0x000155af76db741f,0x0002ded5982f3a21,0x00004cf6e218647c}, {0x00061acd3c1c793a,0x000ac5a35bc3bcf3,0x0008cda6ab2f9ebc,0x0001a1360e860e9a,0x0000055dc39b6dea}, {0x0009227cc28d5bb6,0x000bc774dffabb11,0x0004a32c8907e24e,0x00024b6a83c78cee,0x0000121a307710aa}}, +{{0x000b5d5e9f034a97,0x000093034bc2de4d,0x000551d3b1e153fc,0x000e52d460546919,0x0000333fc76c7a40}, {0x0009713ec77483c9,0x00077b82b96afd65,0x000097bcd388bfe0,0x0003a9b289e28231,0x0000527bb94a6ced}, {0x000d992a995b482e,0x0007c6e383801563,0x000f64d8e53405d0,0x000a9f7485035de2,0x00006b89069b20a7}}, +{{0x000aa0416270220d,0x000faf9245b4e812,0x000072ef05995a89,0x000eb73ffadc4ce5,0x000023bc2103aa73}, {0x0002fa8cb5c7db77,0x000f8c734c155408,0x0006e7a57e068686,0x0009bcf29e6c8d9f,0x00000473d308a763}, {0x000e792603589e05,0x0001246dcc492cae,0x000601a94f2b4b42,0x000341a02a1ef74e,0x0000102f73bfde04}}, +}, +{ /* slot=22 [{1,2,3,..,8} * 16^(2*22)] * G) */ +{{0x0008b9ab7f5745c6,0x000ee5787c690eb1,0x000df7afa9023a8a,0x000013db72712da2,0x000036597d25ea5c}, {0x0004dae0b5511c9a,0x000292bffff06a2b,0x00055042347ac860,0x000a12d981f375df,0x00003f6bd725da4e}, {0x000d8d7b106058ac,0x0009e6fc6905f734,0x000202932dd94057,0x000d6d06466f8f99,0x00007b7ecc19da60}}, +{{0x0002373c695c690d,0x000660642906e78c,0x000ae12bd2dd252e,0x0003956951d44444,0x00004235ad760174}, {0x000e4a51a77cfa9b,0x00054e7a386506da,0x000f2d82db822636,0x000caba09bbffcd8,0x000003bedc661bf5}, {0x0008cb0d078975f5,0x000549189f298625,0x0002e36ee4492942,0x00066a1a0cab423e,0x00000e7ce2b0cdf0}}, +{{0x0004643ac48c85a3,0x000f43c6139adc49,0x000ae94d48fd361d,0x000674a09db17dd3,0x0000666e0a5d8fb4}, {0x0006fedfd94b70f9,0x00051c1fcba2dfea,0x000f2fab89f130c0,0x000eeb54882d47e7,0x0000615256138aec}, {0x000bf64e4870cb0d,0x000f0aa458b6b2ab,0x0005e8985dcd65bc,0x000dee49abe4eba7,0x00007f0bc810d514}}, +{{0x00006ba426f4136f,0x0009e57e03035b90,0x000f463c288d6736,0x000dbf5cbc8dfd94,0x00000d1f8dbcf8ee}, {0x000c9dad737213a0,0x000ba2ef72e9883a,0x0003ec69579ff6f8,0x000ab75311e2edd4,0x00001d3a907ddec5}, {0x000693313ed081dc,0x000ad851b3480ba1,0x00030321cb29329f,0x000fde30128013c0,0x000000011b44a31b}}, +{{0x000fa06c3fc66c0c,0x0008e4dd60dd23fd,0x00068e4d715d40e3,0x00057e17ae38b382,0x00003ac48d916e83}, {0x00061f696a0aa75c,0x0005c5852bd6a165,0x000a7966adc1bf72,0x000102611a8dd7f9,0x000063d988a2d285}, {0x00020753afbd232e,0x000b8fdd8f683001,0x0004e72b91e92bce,0x000a066f81669b38,0x000033fad52b2368}}, +{{0x000649c6c5e41e16,0x00030333f7735540,0x000305e7460af864,0x000dca7b2acfcd2f,0x000016c0f429a256}, {0x000cc8d0c422cfe8,0x0007b05a13acb8d2,0x000cf6a56f072b4f,0x00071e2a3feb6e6e,0x00003cc355ccb90a}, {0x00069443903e9131,0x000cb7a5637cee9b,0x000aba9244b8a494,0x0007568c87cd1a4b,0x0000631eaf426bae}}, +{{0x00090410da66fe9f,0x000526c16e5a6b3e,0x000ef9bf8385dd4b,0x00019b5bc3d97611,0x00005599648b1ea9}, {0x000975b9a3700de8,0x000fbe2f8055247d,0x0002e45de17280c5,0x00080b553658f273,0x0000431f2c7f665f}, {0x00026344858f7b19,0x0002fa1ea514ad60,0x000090a9d714ab35,0x0003b268900441a2,0x00007b04715f9125}}, +{{0x000dbd28acf6ae43,0x0008b7d5c7ab483e,0x0007eb2c4486357c,0x000583fc0404769b,0x000059b37bf5c2f6}, {0x0006c280c4e6bac6,0x000dd6d1d9b0bb37,0x00050bf944970ed3,0x000e223b09a95584,0x000048d0acfa57cd}, {0x000f26e47dabe671,0x00097622f3a37b60,0x0009960394f1d1a1,0x0003bdb4208ce7ee,0x000016234191336d}}, +}, +{ /* slot=23 [{1,2,3,..,8} * 16^(2*23)] * G) */ +{{0x000499def6267ff6,0x0007b742c0843b9e,0x0009a4f2b17772ca,0x000500623a0153fe,0x00002cdfdfecd5d0}, {0x00099cd61ff38640,0x000c3063625a0dd4,0x000dd73dc329cd9b,0x000923151e2d8023,0x00004a25707a203b}, {0x0007668a53f6ed6a,0x000581dd170a12ab,0x000ae20161304242,0x00049fc4000144c3,0x00005721896d248e}}, +{{0x000e5517fd181bae,0x0009f2bb963b40b6,0x0002064625902262,0x00013da5509bce93,0x0000578edd74f63c}, {0x000d5091a1d0da4e,0x000a7b5fe3e08285,0x00019393b34baa6f,0x00030fd63e5177ce,0x000003c935afc4b0}, {0x000276c6492b0c3d,0x000c4dfe205fc997,0x000d623a3c47ccc2,0x000c7a2dcd29b84d,0x00003ec2ab590288}}, +{{0x0000d27be4d87bb9,0x000eb61391aeda1a,0x0003cb9b83a98b4d,0x000cace99a0ddd07,0x00002dd5c25a200f}, {0x00013a09ae32d1cb,0x000df40f5c2d5a72,0x00081eab290f2b87,0x000ac5e0baea4c6e,0x00000e1bf66c6adb}, {0x000bd5e9792c887e,0x00018cb926d5de2a,0x000aae5f1e1a0200,0x0008f5fbfba69cdb,0x0000730548b35ae8}}, +{{0x000551a3cba8b8ee,0x0001db2115f16c43,0x000b8c385065a26f,0x000b8ca760f4f52a,0x00003043443b411d}, {0x000b094ba1d6e334,0x0007709353f19805,0x000622702bbf3ef1,0x00045dd423f06cb0,0x0000585a2277d878}, {0x000a5f8233d48962,0x000b5ec78257fa18,0x00073e41ff6698c4,0x000981fa78e6fa53,0x00007656278950ef}}, +{{0x0003cf59d51fc8c0,0x000fd0506b6f238c,0x000b570e8f9bedd2,0x00046a626bf109fa,0x00003f4160a8c1b8}, {0x000073a3ea86cf9d,0x000b707155fdce17,0x0001838a8e3a8cfb,0x000f6164853e7fc3,0x000028bbf484b613}, {0x00012f5c6f136c7c,0x00007f6dd11bef26,0x0003de6f33afead1,0x000f75d527e9ad21,0x00001e79cb358188}}, +{{0x000436c3eef7e3f1,0x0007ffe9e10f8013,0x000cf9defc828b6a,0x00038317ff908e5b,0x000065d7951b3a3b}, {0x000953d8f5e08181,0x00044299dded977e,0x00064525e584a50c,0x000f2f4dc6c2d0c8,0x0000478ab52d39d1}, {0x0006a4d39252d159,0x000bc871ac80766a,0x0006c1c96fe5dde1,0x0002214b82c6b40a,0x000016d87a411a21}}, +{{0x000d7e5a42066215,0x000cd0c5a24c1b3b,0x0006f994b7879be3,0x0008ca657c05db1d,0x000028f87c8165f3}, {0x0004d5e2d54e0583,0x000d72ebd99fafba,0x000ee9778fe21faf,0x0006dde497ac2736,0x00001f990b577a5a}, {0x00044ead1be8f7d6,0x000ebacea798fa33,0x00020de0527d1e50,0x0006d3e77c6569e5,0x000045882fe1534d}}, +{{0x0009345d757983d6,0x0001117aa11a6666,0x00085e128f62b6ed,0x000f6dd7ddd18579,0x0000688fe5b8f626}, {0x000c9929943c6fe4,0x00061a38392a2d8a,0x000ec89af3b5f9f1,0x000f0742699db13b,0x00007dcf843ce405}, {0x0000d6484a4732c0,0x000fdca5632996c9,0x00015dc6e1d52143,0x000191bb3be28c39,0x00006739687e7327}}, +}, +{ /* slot=24 [{1,2,3,..,8} * 16^(2*24)] * G) */ +{{0x00082014385675a6,0x00030aafda9e8ef7,0x000cdfa8cba2649f,0x000c0b34cd1eb505,0x000046115aba1d4d}, {0x000dcc9dc80c1ac0,0x000f41b38a436a66,0x0005dbd7c697a05c,0x0007daba7ebf3be9,0x00007da0b8f68d7e}, {0x000f1953c3b5da76,0x0007321119e9bd40,0x000eb259601dac6f,0x0004b4b03cc6021f,0x00005a5f887e8367}}, +{{0x000301cf70a13d11,0x00015350dd0c48f6,0x0004bca47ecfceb8,0x0001434f70297d4a,0x00003669b656e44d}, {0x000628d3a0a643b9,0x00000e6c320649e9,0x000c2dec32b5c3cb,0x000c70c9b5302897,0x000043e37ae2d5d1}, {0x000e3f06eda6e133,0x0005199a13ac0387,0x000626381167301d,0x000e9bebd5ad8f83,0x00006a21e6cd4fd5}}, +{{0x0006170a3046e65f,0x0002a00d23524f1c,0x000c82b75558712a,0x000ff5769dbbd3c8,0x0000586bf9f1a195}, {0x000129126699b2e3,0x00047708d1301ef4,0x000182b0bd71d308,0x0008b36325432d01,0x000045371b07001e}, {0x000b088d5ef8790b,0x000dc610937e5a6d,0x0001a16eb85278f0,0x0002179ac0349d26,0x00000eafb03790e5}}, +{{0x000555c13748042f,0x000e6820baa11960,0x0003486d0c219a41,0x000c6611c81f7387,0x0000309acc675a02}, {0x0000805e0f75ae1d,0x000e32662cc30514,0x000a92396dec02fb,0x0005bb32cebdf1ee,0x000044ae3344c543}, {0x000289b9bba543ee,0x0009d5ac971429cf,0x000f9360aaf3760e,0x000678f1d82e5c64,0x000062d5221b7f94}}, +{{0x000c299c18d0936d,0x0006c8a0c1a0c524,0x000b4a8631c86bb5,0x0004562a375052ed,0x00005c0efde4bc75}, {0x0005d4263af77a3c,0x00011fee9144d758,0x0009f7193ddfae7b,0x0002037a50670805,0x000014f29a538392}, {0x00017edc25b2d7f5,0x000db99b53040df7,0x0003ed4c6221f970,0x000093eda9234b7c,0x00005e72365c7bee}}, +{{0x000bfc074571217f,0x0005d0694d95b575,0x0004191e33377967,0x000eabc9a0a37bbf,0x000077f1104c47b4}, {0x000339062f08b33e,0x000e5df9f32be7d9,0x000f9ebdfd5b9659,0x00049b7acff3dad1,0x000070b20555cb73}, {0x000113c555112c4c,0x0003a9a881fcdbe5,0x000e503b47668842,0x000404a446677855,0x00000e34398f4a06}}, +{{0x000d22d93ecebde8,0x0004127822f07b67,0x00005b6d8d09b3e8,0x0002372743fa61fb,0x00005e5405368a36}, {0x00030b093e4b1928,0x0000e73f3f640189,0x0003395d6f7de3e1,0x0009c3ef43217da7,0x00006f8aded6ca37}, {0x0000123dfdb7b29a,0x000e1a21ab291e34,0x000de6949e487b97,0x000de97f9967d02f,0x0000780de72ec8d3}}, +{{0x00028545089ae7bc,0x000cf1c7f4d060ae,0x000a4811b8388dde,0x0008ce438ac15510,0x00000eb28bf67192}, {0x000feaf300f42772,0x0002a2a8c41aa671,0x00073732928f72eb,0x00087a629a17fd79,0x00001defc6ad32b5}, {0x000bbe1aef5195a7,0x00077917b15edaf5,0x000ae5da2e148c12,0x00028672991f7fb7,0x0000467d201bf8dd}}, +}, +{ /* slot=25 [{1,2,3,..,8} * 16^(2*25)] * G) */ +{{0x000e919a74ef4fad,0x000ecf6a308a295f,0x0009a47b013a827b,0x000c797964e01d30,0x000071c43c4f5ba3}, {0x000ef4bd567ae7a9,0x000b2d64498bdbc1,0x000c1f4ec83f624c,0x0004001e41064d22,0x00002ef9c5a5ba38}, {0x000d6df6fa9e74cd,0x000bce4af267ab6f,0x0001ef990ef18278,0x000f2938255b3d0f,0x00005a758ca390c5}}, +{{0x00072710d9462495,0x000d2d57d5003a2b,0x0000b487ca3aa8c6,0x00072ece3d400bfa,0x00002dbae244b3eb}, {0x0000918b1d61dc94,0x000469a8130668ce,0x000fe8aad38ded36,0x000d43fd4e6a829a,0x00000a738027f639}, {0x000f4a2f57ffe1cc,0x0000de1839843980,0x0009fb15fd00670d,0x000a69c105c3f4a4,0x00002698ca635126}}, +{{0x0005318832b0ba78,0x000f7925cff8be76,0x0000291fcc381831,0x000eb0708a81b91a,0x00001fb43dcc49ca}, {0x000d702f5e3dd90e,0x00018e4d253862e3,0x00024da96a9e3f09,0x00033325e773ef60,0x00003c004b0c4afa}, {0x000946ac06f4b82b,0x000a5a806c4f39aa,0x0006cd47871ca284,0x000d2173ed3265fc,0x00006b43fd01cd1f}}, +{{0x00075d4b4697c544,0x00048df0fffbfc7a,0x000a46785a15fdf8,0x000f7142868b9eba,0x00005a68d7105b52}, {0x000742583e760ef3,0x000b9ee0ab990b5c,0x00072b923f75dc52,0x000d9f0bf1427c20,0x000073420b2d6ff0}, {0x000cf6cb9e851e06,0x00013c62238c4af2,0x0009fbf3738f5939,0x000bc9eda8ab8969,0x00003db5632fea34}}, +{{0x000eee2bf75dd9d8,0x000f6396759a5f46,0x00099e72730d17b1,0x0005f131bf2d1314,0x000004321adf49d7}, {0x000990b1829825d5,0x000873e9a89912e4,0x000c704af8edeaeb,0x0002b0eeef03d394,0x000059197ea495df}, {0x00016019e4e55aae,0x0007a7e2f92e904e,0x000f159aa4e77b43,0x0000cc0c7ce2dc16,0x000045eafdc1f4d7}}, +{{0x000401858045d72b,0x000a2cf2f0651698,0x000b222dc64c22fa,0x000dade941a36656,0x00005a5eebc80362}, {0x000e4624cfccb1ed,0x00092bd5c0395b60,0x000c0481c959dbc2,0x000d94031a09d1dd,0x00003f73ceea5d56}, {0x0007bfd10a4e8dc6,0x0007e44c9b339b7a,0x000557aefabe5700,0x00018db60c1207f1,0x0000260588912662}}, +{{0x000704a68360ff04,0x000de7661e6f459f,0x0002873551c3d93f,0x0005d57831b2a731,0x000054ad0c2e4e61}, {0x00018e3cc676e542,0x0009303ceccad4c8,0x0004129f085e422c,0x00043b8ec07cccab,0x00000dedfa10b244}, {0x000b67d5b82b522a,0x000469fa5c1ebee3,0x000ec19fd336f163,0x0009408a5b4d2f26,0x000062ecb2baa77a}}, +{{0x000d795261152b3d,0x0007d0eddd7d1e5e,0x00096b4c71496235,0x000d8be7482c8d0b,0x00002e59f919a966}, {0x00072836afb62874,0x0008579e104a5920,0x000630a14a5fcd5e,0x0003f985aad01adc,0x000061913d507566}, {0x00062d361a3231da,0x00032942002700dc,0x000f9594cefa4758,0x0005d5c02d801513,0x00003ddbc2a131c0}}, +}, +{ /* slot=26 [{1,2,3,..,8} * 16^(2*26)] * G) */ +{{0x000c0ff9ce5ec54b,0x0006b8c2f130d9ad,0x0000f89515039c2a,0x000b36b028007c7f,0x000078968314ac04}, {0x000a57a22796bb14,0x000b79b07da21f3a,0x0001a0391c883aba,0x00005f9e54be2183,0x00005ee7fb38d832}, {0x000dfdcb41446a8e,0x000a9434937f9538,0x00063c8c78a5acfd,0x0000d0946af908d2,0x000061d0633c9bca}}, +{{0x00044935ffdb2566,0x00089780b68bb637,0x00053eec03c5bd6b,0x000d7f56f1b32805,0x00006e965fd847ae}, {0x000328bcf8fc73df,0x0005da6f037fcada,0x0008c2a909ee8469,0x0007bdc637fb4db3,0x00005b23ac2df806}, {0x0002b953ee80527b,0x000aafade6d8d9ad,0x00050e82cfe88f19,0x000dedc0e7117041,0x000079b9bbb9dd95}}, +{{0x000355406a3126c2,0x000a868c8c393ebb,0x0005b97a82d26383,0x00021476c0c6429e,0x00005065f158c9fd}, {0x00097dae8e9f7374,0x000f8cfbb0816d19,0x000d445f0aa032a2,0x000b834cd6cba126,0x00001ba811460acc}, {0x000169fb0c429954,0x000acd76ecf67708,0x0000e645bae14600,0x000faf22eaab98a7,0x00003981f39e58a4}}, +{{0x000b8a7559230a93,0x0006960e6f45d18f,0x0004a93cb51d168f,0x000d0fd3a85a9451,0x000038dc083705ac}, {0x0005dfa56de66fde,0x000002c40483ac84,0x0007b4f632e152a5,0x0001b65e9d2e163c,0x000030f4452edcbc}, {0x000d2782c5759740,0x00069f99cbecc856,0x0000ea4e71fa1345,0x00024698844fc73c,0x0000632d9a1a593f}}, +{{0x000b6b15b807cba6,0x000dfbc54f0d7f6b,0x000e29670b1823c7,0x0004a57bb1d97036,0x00000b24f48847ed}, {0x0009fd11ed0c84a7,0x000810d9f693abf0,0x0007cf877963f071,0x0004ba221908c2d5,0x00003a5a7df28af6}, {0x000ad4be511beac7,0x00075ed26ccf2dcd,0x00005f9a65a45380,0x0001f63e19cff9f0,0x000034fcf7447548}}, +{{0x0007e04c789767ca,0x000cb38d9467dc19,0x0003f95fa8b8714d,0x00063f755de88828,0x00003d3bdc164dfa}, {0x000b1dab78cfaa98,0x00067190b72f2a5b,0x000a92608e5ceda2,0x00074b09309c9110,0x00000119a3042fb3}, {0x0002d89ce8c2177d,0x000f66895d0c167a,0x000282a2b0669da5,0x0000a73f56598e5b,0x000056c088f1ede2}}, +{{0x000d3d1110a86e17,0x000320b75b2fa336,0x0005072988d7f388,0x0008b87f91533762,0x000009674c6b9910}, {0x000b5fac24f38f02,0x000febae30cbd581,0x000acf92f0a90be9,0x000038f9a2169028,0x0000038b7ea48359}, {0x000ef82199316ff8,0x00082eaa78d4f9f4,0x000aef31742f49d2,0x000eb650971a5ab5,0x00006e5e31025969}}, +{{0x000c62f587e593fb,0x000deca5d3e71b16,0x0004cc3e6d4999ed,0x000dba8b491c1e01,0x000008f5114789a8}, {0x0004fb0e63066222,0x0008987acba3f330,0x000c1061a3fb3506,0x0008620bd1924778,0x00003058ad43d183}, {0x000c0ffde57663d0,0x00038a22ea610323,0x000c994f9a05c3df,0x000dc99bdc78abda,0x000026549fa4efe3}}, +}, +{ /* slot=27 [{1,2,3,..,8} * 16^(2*27)] * G) */ +{{0x000d5a461e6bf9d6,0x000fc7777a581741,0x000474d3d92305b3,0x000e0ffd45574a26,0x00001926e1dc6401}, {0x00068549af3f666e,0x00004f14a0ea5db4,0x0004ba0c47d77fcf,0x0003c853df23ff7a,0x00003a10dfe132ce}, {0x000f4e8aea17cea0,0x000463a1fc1fde07,0x0001f2c0f12fd515,0x0005d15175322fd3,0x00001fa1d01d861e}}, +{{0x000055947d599832,0x000da37f15520cc8,0x000e0593201e4656,0x000cf3399f6f7744,0x0000773563bc6a75}, {0x000cac00d1df94ab,0x000ddd1080de938d,0x000dd5e2622e712b,0x00001e57f13e93ef,0x000073fced18ee9a}, {0x0001e90863139cb3,0x00067c5a03ecd06b,0x000d638932a493da,0x0004f448d77cec8a,0x00001f426b701b86}}, +{{0x0009264c41911c01,0x000b817a22c25efc,0x00030f1447f1a3b7,0x000b0905875da6bf,0x00004e1af5271d31}, {0x000e35c891a12552,0x00053575e9c76f17,0x000d9b723eb76b81,0x000e438fa83406f0,0x00000b76bb1b3fa7}, {0x0008c1f97f92939b,0x000cbd444ab6e08b,0x0009bb8017be6771,0x000a95522e564639,0x00007b6dd61eb772}}, +{{0x000dc1e850f33d92,0x0004f608cd5cfb7a,0x000dfc5bdb7998fa,0x0002f4fad962dbd8,0x0000703e9bceaf1d}, {0x0000abf9ab01d2c7,0x000dc40143b18573,0x0000cbb28116fb76,0x0006afe866cbe65a,0x000053fa9b659bff}, {0x0004c8e994885455,0x0006665aed4e56c1,0x000cd65af1843a5d,0x0001f50181bb73eb,0x0000398d93e5c4c6}}, +{{0x000bd16733e248f3,0x0008715bf0a5f1c4,0x00010b0376bd9e12,0x0001b13d43f8cf0a,0x000053b09b5ddf19}, {0x00077c60d2e7e3f2,0x000a030828bb1c38,0x00039ef1383b34aa,0x0000577283e26e77,0x0000699c9c9002c3}, {0x0006a7235946f1cc,0x000b5cce5d97df30,0x0001b4e975921718,0x000d90728cdd2478,0x000051caf30c6fcd}}, +{{0x000ba7427674e00a,0x00070a17a7bf3a60,0x000f3324cc630e85,0x000fdaa3758563dc,0x00005504aa292383}, {0x000af99a18ac54c7,0x000dcc51cb30f737,0x000ce10cc7903378,0x000e99a2b89bc334,0x000012ae29c189f8}, {0x000ec0cb1f0d01cf,0x000cc3a34f7aea99,0x00009c4e220dd1ef,0x000a5ea55ca7521d,0x00005fd14fe958eb}}, +{{0x000c2ddf2845ab2c,0x000b10a7fe993b5d,0x000002e346069491,0x00074d14daaf3d64,0x0000093ff26e5864}, {0x0002fe5ebf93cb8e,0x0005136d4565f3c4,0x00084220e8bedfa8,0x000d128e0f0859e8,0x00007dd73f960725}, {0x000d24fe68059829,0x00072dbaf23e5b10,0x000457ac29757306,0x00070a41367253ab,0x00002f59bcbc86b4}}, +{{0x00047d429917135f,0x0001f567d03d7838,0x000e77aad1ad1b91,0x000af4a7e7748d9b,0x00005458b42e2e51}, {0x0001d560b691c301,0x0003fadd7e71e704,0x000133558585201b,0x00028b116c2e1631,0x00002aa55e3d0108}, {0x000192e60c07444f,0x0002d74421d10ed5,0x000db5c86442c54e,0x0008664352b4c82f,0x000013e9004a8a76}}, +}, +{ /* slot=28 [{1,2,3,..,8} * 16^(2*28)] * G) */ +{{0x000d8845832fcedb,0x000c9ae6bf863739,0x00074ffef7fa38d6,0x000e45e32bc0dcab,0x000073937e8814bc}, {0x000e00c9193b877f,0x00090e0dc506bbb2,0x0006de649fece3a8,0x0009e1aecf3b7c03,0x00005f46040898de}, {0x00037116297bf48d,0x00022d4f06834b90,0x000696bdc6a9d13b,0x000e835e19715574,0x00002cf8a4e891d5}}, +{{0x0003fd8707110f67,0x000d37c38b5496d9,0x0002736a86dd4c09,0x0002a097cb16a4cc,0x00002049bd6e5825}, {0x0005487e17d06ba2,0x0001c3950196b2cb,0x0005978a3024d238,0x000a4f6d7659c818,0x00007a6f7f2891d6}, {0x0009fd8d6a9aef49,0x000be5b3db90b7d0,0x00019ebfd4f0ee60,0x000941d4c21b52c5,0x00006011aadfc545}}, +{{0x0007926dcf95f83c,0x00061712890715f6,0x00098f7a5b7c7e85,0x000f9e0d6a1e7f39,0x00006fc5cc1b0b62}, {0x000ed0c802cbf890,0x000ca0dff6aaa63d,0x0009b6ed99fbd098,0x0000b1e624d0afdb,0x000069ce18b77934}, {0x000f5528b29879cb,0x0003cd47e9092d1e,0x00089f2352dd1aae,0x00001f1127e04421,0x000015596b3ae571}}, +{{0x000739d23f9179a2,0x0003197d6ddcf462,0x0003f2148aff8312,0x0004dda1307deb55,0x00000d2237687b5f}, {0x000f31167e5124ca,0x0008bd9c745df09f,0x000ef556e50be415,0x000d138292b7d227,0x00003aa4e241afb6}, {0x000138bf2a3305f5,0x0008fa2e926c32cc,0x000549d2eb48583f,0x000a36c083ab1a25,0x000032fcaa6e4687}}, +{{0x00056e8dc57d9af5,0x000ed9df0bdf27bc,0x0002efe4a33e0bd2,0x0006a5caac014de2,0x00004627e9cefebd}, {0x0007a4732787ccdf,0x00008f213e3f8320,0x00060d964e17e319,0x0000be9d5b2ecd7f,0x0000746f6336c260}, {0x000af345ab6c971c,0x000729943731f3f4,0x000344186de288eb,0x000629333596a8a0,0x00007b4917007ed6}}, +{{0x0005fb5cab84b064,0x000d289f3bc142d8,0x000b15ce0c497810,0x000fd7b476adc447,0x0000122ba376f844}, {0x00041b28dd53a2dd,0x0005bdf42fc3f543,0x000dd2f8f4aa1790,0x000d37d0ff592d94,0x00001d03620fe08c}, {0x000232cda2b4e554,0x00042115d187fc20,0x000dd479d99ed0fd,0x000ec4c2eabb4be7,0x000002c70bf52b68}}, +{{0x0007ec4b5d0b2fbb,0x00090074882caa28,0x0001d0815c415c57,0x000f5e0e044a61ec,0x000026334f0a409e}, {0x000532bf458d72e1,0x000e07cb73cb5ace,0x000e8bbde75be768,0x0003a0356cf7d94e,0x00006b0697e3feb4}, {0x0008f04adf62a3c0,0x000ef076da45db6c,0x0009f0d2a93ef000,0x0002fae9c9cb9584,0x00001cc37f43441b}}, +{{0x000f565a5cc7324f,0x000c0e506a922508,0x000c45ac19d061c4,0x000314afb18abdb5,0x00006c6809c10380}, {0x000656f1c9ceaeb9,0x000f818e5656ad76,0x00044c23341c5b15,0x000683826e728328,0x00003a346f772f19}, {0x00055112e2da6ac8,0x00031b1e851edd2d,0x000ec67262e9bd03,0x000c5d0960746dd8,0x000005911b9f6ef7}}, +}, +{ /* slot=29 [{1,2,3,..,8} * 16^(2*29)] * G) */ +{{0x00039983f5df0ebb,0x0008f512c4cacc13,0x000bb398e1c0f375,0x0000c622cf1130a0,0x00006b3cecf9aa27}, {0x0009acf3512eeaef,0x000d31cc1cb49534,0x00099a688d20c141,0x0002d1724180c07a,0x0000555ef9d1c64b}, {0x000770ba3b73bd08,0x00008a3afbf0c36a,0x00040946f2624aef,0x000749d5737ff98b,0x0000675f4de13381}}, +{{0x000c52036b1782fc,0x000816cad83b40e2,0x000964073e64816c,0x000c520d0dcbdd96,0x000013d99df70164}, {0x000ff6d93bdab31d,0x0000f9d652dfea12,0x000abe94870725d8,0x0003c43019c4ff39,0x000060f450b882cd}, {0x000b5ec321e5c0ca,0x000c9d719bfa2014,0x00050023a04fcb69,0x000ac804e5f1c187,0x00001c06de9e55ed}}, +{{0x000f7ad6a33ec4e2,0x00038be2ee08e990,0x00032845156608f9,0x000b60d9ca143c56,0x00004cf38a1fec2d}, {0x00052b40ff6d69aa,0x00018dc4049bbffd,0x00034d989734530b,0x000ba2d5e4a5c2fa,0x000078096f8e7d32}, {0x000aaa650dfa5ce7,0x0002a48b5478ca0a,0x000003725bf9c49e,0x0001abe4f09cc7d7,0x0000373cad3a2609}}, +{{0x0004634d82c9f57c,0x000e124934536b29,0x00018cdb5a1fcbfd,0x00019fc9e9c4db34,0x00000040f3d94544}, {0x000ea8fb89ddbbad,0x000bc61aeaecbf1b,0x000f9b8d9d3bcb2c,0x000a6868f58a7bb1,0x000021547eda5112}, {0x000de939fd5986d3,0x00089510a380cdef,0x000b3119b9f4272c,0x0004df4b72ba407b,0x000063550a334a25}}, +{{0x0007d6edb569cf37,0x000b00ca52ee1650,0x000b6bd65d178429,0x0008f51ea7c0090e,0x00003eea62c7daf7}, {0x000a584572547b49,0x000fae2c408e09bb,0x000734f18df305c6,0x000767a60e8fa69c,0x000039a92bafaa7d}, {0x0004c713e693274e,0x0007768dbd3759d2,0x000b8ab39a5f6385,0x000c4cd70525560e,0x000068436a0665c9}}, +{{0x000235e8202f3f27,0x000e264f975b0bc0,0x00038c2416c75c00,0x00089f991a4e9d5a,0x000017b6e7f68ab7}, {0x0006d317e820107c,0x00044840ae9651e5,0x00020ffc7ac52668,0x0001472c1e0a1c63,0x00005373669c9161}, {0x000814ab9a0e5257,0x00084c9cab3fc5d2,0x000b2d1eca908f20,0x0007d11afcaf5885,0x00001cb4b5a678f8}}, +{{0x0004c06b394afc6c,0x0002498da5fb1b66,0x000bcad8340c88de,0x00034a24f8d03164,0x0000330bca78de74}, {0x0004aa62a2a007e7,0x000b0f071c7b16b7,0x00000be223f311e0,0x0006eac5707e4380,0x00002dc0fd2d82ef}, {0x000eff841119744e,0x000962b074724982,0x000fc953fbf9695e,0x0001cf5c58ac14fb,0x00003c31be1b369f}}, +{{0x0004864d08948aee,0x000ee91ba1c6fb0f,0x0006aca15807dc19,0x000d4bb7975cdaea,0x0000330b61134262}, {0x0008bc93f9cb4272,0x0001fc7cedb98c16,0x0004ac8d7aaeb871,0x00055bb7f0e52aa3,0x000041cec1097e7d}, {0x000619d7a26d808a,0x0009e1d9e156df79,0x000ba1df27bb1fd4,0x000777d73d7c36cd,0x000026b44cd91f28}}, +}, +{ /* slot=30 [{1,2,3,..,8} * 16^(2*30)] * G) */ +{{0x000048478f387475,0x000f49cbecb3c51f,0x00099f2055b25dbc,0x000a5d69aab1244d,0x00002c709e6c1c10}, {0x0007f29362730383,0x000ffebca8a2ce1b,0x000fd413144b5279,0x000610fdafc778ab,0x00007deb10149c72}, {0x0002af6a8766ee7a,0x000045553cd0ecb6,0x000f0be4b566cbec,0x000e2ea588001380,0x000008e68e9ff62c}}, +{{0x000d500a4bc130ad,0x000493d0bd49c34a,0x00000a89be8d38db,0x0003b09a25c3d985,0x00002f1f3f87eeba}, {0x000d09d50ab8f2f9,0x0008dc55923df2f2,0x0003766cb9acb921,0x00019f54a8f34267,0x00004cb13bd738f7}, {0x00048c75e515b64a,0x000badb4a9038f78,0x000f751b50a59501,0x0002ee8c20d313f3,0x000019a1e353c0ae}}, +{{0x000c7560bafa05c3,0x000a0c6e55e617d1,0x0000d66473b3e1a0,0x0003486e3529718c,0x000041546b11c20c}, {0x000172cdd596bdbd,0x0004398eefc40b42,0x00044109b593e045,0x000ae349fb15347b,0x0000736bd3990266}, {0x00032d509334b3b4,0x0004b60816573855,0x00025c837546fd11,0x000ab5ccc5f5f304,0x0000412295a2b87f}}, +{{0x00099b88f57ed6e9,0x000266df8c82519c,0x00030ad2735393cb,0x0002e345cee3213b,0x000014e153ebb52d}, {0x00055261e293eac6,0x000032133acdb2e6,0x000900996b845a92,0x000dd80460975cb7,0x00000760bb8d195a}, {0x000e1a17cde6818a,0x000a9ed69a084413,0x0006caccb157156d,0x000c5f22cbf268f4,0x00006b34be9bc33a}}, +{{0x000f2f643a78c0b2,0x0001ef22e027cf3d,0x0009c1b5a34c3e97,0x000dd2dec7d1c5e4,0x00002012c18f0922}, {0x000c69656571f2d3,0x00045530e737a11f,0x0004fe5035c6c9e8,0x000d30be33ae7a2d,0x000001b9c7b62e6d}, {0x000b55e55ac89d29,0x0001f45a0a763880,0x0002e76c1f148324,0x000ade83d36efdfc,0x000008af5b784e4b}}, +{{0x000499dc881f2533,0x000da779323b6283,0x00073441f49d0525,0x000168d897addfb6,0x000032b79d71163a}, {0x000314d289cc2c4b,0x00011a287178de27,0x000a3364ce4be4bd,0x000826e18d528d6f,0x00006423c1d5afd9}, {0x0005f8d9edfcb36a,0x0008f3746e5f9cc8,0x0009e5d3cd22bcc2,0x0002dcce49de338f,0x0000480a5efbc13e}}, +{{0x0001e70b01622071,0x000cf8b1dafc50b5,0x000f5aabcd06b505,0x000bf312c6bb061e,0x000047aa27600cb7}, {0x00014ce442ce221f,0x000cc4c053928b66,0x000c1cbe036e199d,0x0008e06663fb4a4d,0x000024b31d47691c}, {0x00041eedc015f8c3,0x0007e7c693f7c2a5,0x000ea278d611a4fe,0x000a094f0af66134,0x0000545b585d14dd}}, +{{0x000f275ea0d43a0f,0x00034089beebe67b,0x000479e72eade68e,0x00054544289134cd,0x00000f62f9c332ba}, {0x0004e4d0e3b321e1,0x0007a28ff1e95620,0x000b99bd9e3baa63,0x000d0710b0ccffd5,0x00004d22dc3e64c8}, {0x00046589d63b5f39,0x0003f57cbcf61fcb,0x00053afa055cae6a,0x0001436febac2d29,0x00001c0fa01a3637}}, +}, +{ /* slot=31 [{1,2,3,..,8} * 16^(2*31)] * G) */ +{{0x000604b622943dff,0x000ce44cfb3a0d2c,0x0007808678bc8cbe,0x000a6bf5d254ff39,0x00000fa3614f3b1c}, {0x00082b0e8c936a50,0x00035c1dac5b6690,0x0004dfb634f9c9a0,0x000c1406fb73e54c,0x00004005419b1d2b}, {0x0003febdb9be82f0,0x000af3a44ac90a00,0x000954fa8e2089c1,0x000ab42f8499f911,0x00001fba218aef40}}, +{{0x00049448fac8f53e,0x0009a7ba63741ab5,0x000c2b5e0181f6e8,0x0006e4274fd6c7d6,0x0000392e3acaa8c8}, {0x000e57043e7b0194,0x000ee08daaf7f4f3,0x0009dcdef1a81d3e,0x00061d5c839c6ab9,0x00006c535d13ff77}, {0x000d34e93e8a35af,0x000445887e8164cb,0x00029ab0ab2e0781,0x000c13b19319c76f,0x000025e17fe4d50a}}, +{{0x00089bd71e04f676,0x00052d6420f950a2,0x0004691fab208e1c,0x000b3515186d8b03,0x0000255751442a9f}, {0x000f7ff576f121a7,0x000272fcd87e3915,0x000d1be526c34a32,0x000899bccba2fde4,0x00006bba828f8969}, {0x0001bc6690fe3901,0x00018a0997ad5e2d,0x000f8460d44cb54a,0x0007be4971d6914a,0x0000559d504f7f6b}}, +{{0x00038378b3eb54d5,0x00066a5553c7ca77,0x00092800ba1d69d3,0x00032170a26cf62f,0x000001ab12d5807e}, {0x000891e7f6d266fd,0x0009b0307781b9c4,0x000061e23b0744a1,0x000d50e88388f1d6,0x0000123ea6a3354b}, {0x000d189041e32d96,0x000c2d8315848118,0x00083245d9b9ede3,0x000a1541eab4271d,0x00004a3961e2c918}}, +{{0x000c3be0f8e6bba0,0x000347effe30a71d,0x00013a476ad6cef8,0x000b763a992425fe,0x00002cd6bce3fb1d}, {0x0007d644f3233f1e,0x0000e34fcf016032,0x0002dab979499a26,0x000111f83b5a716f,0x000068aceead9bd4}, {0x0004c90ef3d7c210,0x00024b7ad040c38b,0x0007e73e23308e6e,0x000f5973860d9f1b,0x0000595760d5b508}}, +{{0x0009bfe104aa6397,0x00008a4a7fccb612,0x000d9094588f9600,0x00091a93f8bc0897,0x0000709fa43edcb2}, {0x000acbebfd022790,0x00005c4115760882,0x000d3473f489af33,0x0005a2b65f492e37,0x00002cb2c5df5451}, {0x000a5d8c63fd2aca,0x000662e694effeb0,0x0008cbb03ad22bc1,0x000131f2723f36ef,0x000070f029ecf0c8}}, +{{0x000307b32eed3e33,0x00033a45581e7461,0x00095f0366ae042f,0x0004858c94449d31,0x00000b7d5d8a6c31}, {0x000aafaa5e10b0b9,0x00070ef041aa92a6,0x000a3ad61f78f0a3,0x000d9e1773efb77a,0x000044eca5a2a74b}, {0x000448327b95d543,0x00000a3340f1d25d,0x0000e1c52b70d383,0x000e9e4de1c531c6,0x0000272224512c7d}}, +{{0x000c92af49c5342e,0x00011b2e6fad01ab,0x000cc84e29ffeed8,0x000c543efa28c8df,0x000011b5df18a44c}, {0x000bbb8a42a975fc,0x0007796ada358bf7,0x000dedaa488c5c39,0x00020a6e27fc76fc,0x000019735fd7f6bc}, {0x000b90d042c84266,0x0000f7f19547ee3a,0x0005a497b9eb848e,0x000895f2503a1d06,0x00000fef911191df}}, +}, +}; + +#endif /* #define IFMA_ED25519_PRECOMP_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ed25519/sha512.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ed25519/sha512.h new file mode 100644 index 0000000..4f65126 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/ed25519/sha512.h @@ -0,0 +1,171 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Macros and definitions that is necessary for SHA512 computation +*/ + +#include + +/* define 64-bit constant */ +#if !defined(__GNUC__) +#define CONST_64(x) (x) /*(x##i64)*/ +#else +#define CONST_64(x) (x##LL) +#endif + +static const int64u sha512_iv[] = { + CONST_64(0x6A09E667F3BCC908), CONST_64(0xBB67AE8584CAA73B), + CONST_64(0x3C6EF372FE94F82B), CONST_64(0xA54FF53A5F1D36F1), + CONST_64(0x510E527FADE682D1), CONST_64(0x9B05688C2B3E6C1F), + CONST_64(0x1F83D9ABFB41BD6B), CONST_64(0x5BE0CD19137E2179) }; + +static __ALIGN64 const int64u sha512_cnt[] = { + CONST_64(0x428A2F98D728AE22), CONST_64(0x7137449123EF65CD), CONST_64(0xB5C0FBCFEC4D3B2F), CONST_64(0xE9B5DBA58189DBBC), + CONST_64(0x3956C25BF348B538), CONST_64(0x59F111F1B605D019), CONST_64(0x923F82A4AF194F9B), CONST_64(0xAB1C5ED5DA6D8118), + CONST_64(0xD807AA98A3030242), CONST_64(0x12835B0145706FBE), CONST_64(0x243185BE4EE4B28C), CONST_64(0x550C7DC3D5FFB4E2), + CONST_64(0x72BE5D74F27B896F), CONST_64(0x80DEB1FE3B1696B1), CONST_64(0x9BDC06A725C71235), CONST_64(0xC19BF174CF692694), + CONST_64(0xE49B69C19EF14AD2), CONST_64(0xEFBE4786384F25E3), CONST_64(0x0FC19DC68B8CD5B5), CONST_64(0x240CA1CC77AC9C65), + CONST_64(0x2DE92C6F592B0275), CONST_64(0x4A7484AA6EA6E483), CONST_64(0x5CB0A9DCBD41FBD4), CONST_64(0x76F988DA831153B5), + CONST_64(0x983E5152EE66DFAB), CONST_64(0xA831C66D2DB43210), CONST_64(0xB00327C898FB213F), CONST_64(0xBF597FC7BEEF0EE4), + CONST_64(0xC6E00BF33DA88FC2), CONST_64(0xD5A79147930AA725), CONST_64(0x06CA6351E003826F), CONST_64(0x142929670A0E6E70), + CONST_64(0x27B70A8546D22FFC), CONST_64(0x2E1B21385C26C926), CONST_64(0x4D2C6DFC5AC42AED), CONST_64(0x53380D139D95B3DF), + CONST_64(0x650A73548BAF63DE), CONST_64(0x766A0ABB3C77B2A8), CONST_64(0x81C2C92E47EDAEE6), CONST_64(0x92722C851482353B), + CONST_64(0xA2BFE8A14CF10364), CONST_64(0xA81A664BBC423001), CONST_64(0xC24B8B70D0F89791), CONST_64(0xC76C51A30654BE30), + CONST_64(0xD192E819D6EF5218), CONST_64(0xD69906245565A910), CONST_64(0xF40E35855771202A), CONST_64(0x106AA07032BBD1B8), + CONST_64(0x19A4C116B8D2D0C8), CONST_64(0x1E376C085141AB53), CONST_64(0x2748774CDF8EEB99), CONST_64(0x34B0BCB5E19B48A8), + CONST_64(0x391C0CB3C5C95A63), CONST_64(0x4ED8AA4AE3418ACB), CONST_64(0x5B9CCA4F7763E373), CONST_64(0x682E6FF3D6B2B8A3), + CONST_64(0x748F82EE5DEFB2FC), CONST_64(0x78A5636F43172F60), CONST_64(0x84C87814A1F0AB72), CONST_64(0x8CC702081A6439EC), + CONST_64(0x90BEFFFA23631E28), CONST_64(0xA4506CEBDE82BDE9), CONST_64(0xBEF9A3F7B2C67915), CONST_64(0xC67178F2E372532B), + CONST_64(0xCA273ECEEA26619C), CONST_64(0xD186B8C721C0C207), CONST_64(0xEADA7DD6CDE0EB1E), CONST_64(0xF57D4F7FEE6ED178), + CONST_64(0x06F067AA72176FBA), CONST_64(0x0A637DC5A2C898A6), CONST_64(0x113F9804BEF90DAE), CONST_64(0x1B710B35131C471B), + CONST_64(0x28DB77F523047D84), CONST_64(0x32CAAB7B40C72493), CONST_64(0x3C9EBE0A15C9BEBC), CONST_64(0x431D67C49C100D4C), + CONST_64(0x4CC5D4BECB3E42B6), CONST_64(0x597F299CFC657E2A), CONST_64(0x5FCB6FAB3AD6FAEC), CONST_64(0x6C44198C4A475817) +}; + +/* SHA512 constants */ +#define SHA512_DIGEST_BITSIZE 512 +#define BYTESIZE (8) + +#define MBS_SHA512 (128) +#define MLR_SHA512 (sizeof(int64u)*2) + +/* Logical Shifts (right and left) of WORD */ +#define LSR32(x,nBits) ((x)>>(nBits)) +#define LSL32(x,nBits) ((x)<<(nBits)) + +/* Rorate (right and left) of WORD */ +#if defined(_MSC_VER) && !defined( __ICL ) +# include +# define ROR32(x, nBits) _lrotr((x),(nBits)) +# define ROL32(x, nBits) _lrotl((x),(nBits)) +#else +# define ROR32(x, nBits) (LSR32((x),(nBits)) | LSL32((x),32-(nBits))) +# define ROL32(x, nBits) ROR32((x),(32-(nBits))) +#endif + +/* Logical Shifts (right and left) of DWORD */ +#define LSR64(x,nBits) ((x)>>(nBits)) +#define LSL64(x,nBits) ((x)<<(nBits)) + +/* Rorate (right and left) of DWORD */ +#define ROR64(x, nBits) (LSR64((x),(nBits)) | LSL64((x),64-(nBits))) +#define ROL64(x, nBits) ROR64((x),(64-(nBits))) + +/* WORD and DWORD manipulators */ +#define LODWORD(x) ((int32u)(x)) +#define HIDWORD(x) ((int32u)(((int64u)(x) >>32) & 0xFFFFFFFF)) +#define MAKEDWORD(wLo,wHi) ((int64u)(((int32u)(wLo)) | ((int64u)((int32u)(wHi))) << 32)) + +/* Change endian */ +#if defined(_MSC_VER) +# define ENDIANNESS(x) _byteswap_ulong((x)) +# define ENDIANNESS32(x) ENDIANNESS((x)) +# define ENDIANNESS64(x) _byteswap_uint64((x)) +#elif defined(__ICL) +# define ENDIANNESS(x) _bswap((x)) +# define ENDIANNESS32(x) ENDIANNESS((x)) +# define ENDIANNESS64(x) _bswap64((x)) +#else +# define ENDIANNESS(x) ((ROR32((x), 24) & 0x00ff00ff) | (ROR32((x), 8) & 0xff00ff00)) +# define ENDIANNESS32(x) ENDIANNESS((x)) +# define ENDIANNESS64(x) MAKEDWORD(ENDIANNESS(HIDWORD((x))), ENDIANNESS(LODWORD((x)))) +#endif + +#ifndef MIN +#define MIN( a, b ) ( ((a) < (b)) ? (a) : (b) ) +#endif + +/* +// SHA512 Specific Macros +// +// Note: All operations act on DWORDs (64-bits) +*/ + +#define CH(x,y,z) (((x) & (y)) ^ (~(x) & (z))) +#define MAJ(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) + +#define SUM0(x) (ROR64((x),28) ^ ROR64((x),34) ^ ROR64((x),39)) +#define SUM1(x) (ROR64((x),14) ^ ROR64((x),18) ^ ROR64((x),41)) + +#define SIG0(x) (ROR64((x), 1) ^ ROR64((x), 8) ^ LSR64((x), 7)) +#define SIG1(x) (ROR64((x),19) ^ ROR64((x),61) ^ LSR64((x), 6)) + +#define SHA512_UPDATE(i) \ + wdat[i&15] += SIG1(wdat[(i+14)&15]) + wdat[(i+9)&15] + SIG0(wdat[(i+1)&15]) + +#define SHA512_STEP(i,j) \ + v[(7-i)&7] += (j ? SHA512_UPDATE(i) : wdat[i&15]) \ + + SHA512_cnt_loc[i+j] \ + + SUM1(v[(4-i)&7]) \ + + CH(v[(4-i)&7], v[(5-i)&7], v[(6-i)&7]); \ + v[(3-i)&7] += v[(7-i)&7]; \ + v[(7-i)&7] += SUM0(v[(0-i)&7]) + MAJ(v[(0-i)&7], v[(1-i)&7], v[(2-i)&7]) + + +/* +// SHA512 structures and data types +*/ + +/* SHA512 digest */ +typedef int64u DigestSHA512[8]; + +/* SHA512 context */ +struct _cpSHA512 { + int msgBuffIdx; /* buffer entry */ + int64u msgLenLo; /* message length */ + int64u msgLenHi; /* message length */ + int8u msgBuffer[MBS_SHA512]; /* buffer */ + DigestSHA512 msgHash; /* intermediate hash */ +}; +typedef struct _cpSHA512 SHA512State; + +#define HASH_LENLO(stt) ((stt)->msgLenLo) +#define HASH_LENHI(stt) ((stt)->msgLenHi) +#define HASH_VALUE(stt) ((stt)->msgHash) +#define HASH_BUFFIDX(stt) ((stt)->msgBuffIdx) +#define HASH_BUFF(stt) ((stt)->msgBuffer) + +/* +// SHA512 internal functions +*/ + +void SHA512MsgDigest(const int8u* pMsg, int len, int8u* pMD); +void SHA512Init(SHA512State* pState); +void SHA512Update(const int8u* pSrc, int len, SHA512State* pState); +void SHA512Final(int8u* pMD, SHA512State* pState); + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/exp/ifma_exp_method.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/exp/ifma_exp_method.h new file mode 100644 index 0000000..59f5511 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/exp/ifma_exp_method.h @@ -0,0 +1,79 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_EXP_METHOD_H +#define IFMA_EXP_METHOD_H + +#include + +/* exponetiation processing window */ +#define EXP_WIN_SIZE (5) +#define EXP_WIN_MASK ((1< +#include + +#define ALIGNSPEC __ALIGN64 + +#define DUP2_DECL(a) a, a +#define DUP4_DECL(a) DUP2_DECL(a), DUP2_DECL(a) +#define DUP8_DECL(a) DUP4_DECL(a), DUP4_DECL(a) + + +//gres: typedef unsigned __int64 UINT64; +typedef int64u UINT64; + + +#define VUINT64 __m512i +#define VINT64 VUINT64 +#define VDOUBLE __m512d +#define VMASK_L __mmask8 + +#define L2D(x) _mm512_castsi512_pd(x) +#define D2L(x) _mm512_castpd_si512(x) + +// Load int constant +#define VLOAD_L(addr) _mm512_load_epi64((__m512i const*)&(addr)[0]); +// Load DP constant +#define VLOAD_D(addr) _mm512_load_pd((const double*)&(addr)[0]); +#define VSTORE_L(addr, x) _mm512_store_epi64(((__m512i*)&(addr)[0]), (x)); + + +#define VSHL_L(x, n) _mm512_slli_epi64((x), (n)); +#define VSAR_L(x, n) _mm512_srai_epi64((x), (n)); +#define VAND_L(a, b) _mm512_and_epi64((a), (b)); +#define VADD_L(a, b) _mm512_add_epi64((a), (b)); +#define VSUB_L(a, b) _mm512_sub_epi64((a), (b)); +#define VMSUB_L(mask, a, b) _mm512_mask_sub_epi64((a), mask, (a), (b)); +#define VMADD_L(mask, a, b) _mm512_mask_add_epi64((a), mask, (a), (b)); +#define VCMPU_GT_L(a, b) _mm512_cmpgt_epu64_mask((a), (b)); +#define VCMP_GE_L(a, b) _mm512_cmpge_epi64_mask((a), (b)); + + +// conversion unsigned 64-bit -> double +#define VCVT_L2D(x) _mm512_cvtepu64_pd(x) +// conversion double -> signed 64-bit +#define VCVT_D2L(x) _mm512_cvtpd_epi64(x) + +#define VADD_D(a, b) _mm512_add_pd(a, b) +#define VMUL_D(a, b) _mm512_mul_pd(a, b) +#define VMUL_RZ_D(a, b) _mm512_mul_round_pd(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC) +#define VMUL_RU_D(a, b) _mm512_mul_round_pd(a, b, _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC) +#define VDIV_RZ_D(a, b) _mm512_div_round_pd(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC) +#define VDIV_RU_D(a, b) _mm512_div_round_pd(a, b, _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC) +#define VROUND_RZ_D(a) _mm512_roundscale_pd(a, 3) +#define VQFMR_D(a, b, c) _mm512_fnmadd_pd(a, b, c) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/rsa/ifma_rsa_arith.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/rsa/ifma_rsa_arith.h new file mode 100644 index 0000000..6793465 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/rsa/ifma_rsa_arith.h @@ -0,0 +1,182 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_RSA_ARITH_H +#define IFMA_RSA_ARITH_H + +#ifndef BN_OPENSSL_DISABLE + #include +#endif + +#include +#include + +typedef int64u int64u_x8[8]; // alias of 8-term vector of int64u each +typedef int64u (*pint64u_x8) [8]; // pointer to 8-term vector of int64u each + +/* fixed size of RSA */ +#define RSA_1K (1024) +#define RSA_2K (2*RSA_1K) +#define RSA_3K (3*RSA_1K) +#define RSA_4K (4*RSA_1K) + +#define NUMBER_OF_DIGITS(bitsize, digsize) (((bitsize) + (digsize)-1)/(digsize)) +#define MULTIPLE_OF(x, factor) ((x) + (((factor) -((x)%(factor))) %(factor))) + +// ============ Multi-Buffer required functions ============ +#define redLen ((RSA_1K+(DIGIT_SIZE-1))/DIGIT_SIZE) // 20 +EXTERN_C void ifma_extract_amm52x20_mb8(int64u* out_mb8, const int64u* inpA_mb8, int64u MulTbl[][redLen][8], const int64u Idx[8], const int64u* inpM_mb8, const int64u* k0_mb8); + +// Multiplication +EXTERN_C void ifma_amm52x10_mb8(int64u* out_mb8, const int64u* inpA_mb8, const int64u* inpB_mb8, const int64u* inpM_mb8, const int64u* k0_mb8); +EXTERN_C void ifma_amm52x20_mb8(int64u* out_mb8, const int64u* inpA_mb8, const int64u* inpB_mb8, const int64u* inpM_mb8, const int64u* k0_mb8); +EXTERN_C void ifma_amm52x60_mb8(int64u* out_mb8, const int64u* inpA_mb8, const int64u* inpB_mb8, const int64u* inpM_mb8, const int64u* k0_mb8); +EXTERN_C void ifma_amm52x40_mb8(int64u* out_mb8, const int64u* inpA_mb8, const int64u* inpB_mb8, const int64u* inpM_mb8, const int64u* k0_mb8); +EXTERN_C void ifma_amm52x30_mb8(int64u* out_mb8, const int64u* inpA_mb8, const int64u* inpB_mb8, const int64u* inpM_mb8, const int64u* k0_mb8); +EXTERN_C void ifma_amm52x79_mb8(int64u* out_mb8, const int64u* inpA_mb8, const int64u* inpB_mb8, const int64u* inpM_mb8, const int64u* k0_mb8); + +// New functions for almost half montgomery +EXTERN_C void ifma_ahmm52x20_mb8(int64u* out_mb8, const int64u* inpA_mb8, const int64u* inpB_mb8, const int64u* inpBx_mb8, const int64u* inpM_mb8, const int64u* k0_mb8); +EXTERN_C void ifma_ahmr52x20_mb8(int64u* out_mb, const int64u* inpA_mb, int64u* inpM_mb, const int64u* k0_mb); + +// 4x Mont Mul +EXTERN_C void ifma_amm52x20_mb4(int64u* out_mb8, const int64u* inpA_mb8, const int64u* inpB_mb8, const int64u* inpM_mb8, const int64u* k0_mb8); + +// Diagonal sqr +EXTERN_C void AMS52x10_diagonal_mb8(int64u* out_mb8, const int64u* inpA_mb8, const int64u* inpM_mb8, const int64u* k0_mb8); +EXTERN_C void AMS5x52x10_diagonal_mb8(int64u* out_mb8, const int64u* inpA_mb8, const int64u* inpM_mb8, const int64u* k0_mb8); + +EXTERN_C void AMS52x20_diagonal_mb8(int64u* out_mb8, const int64u* inpA_mb8, const int64u* inpM_mb8, const int64u* k0_mb8); +EXTERN_C void AMS4x52x20_diagonal_stitched_with_extract_mb8(int64u* out_mb8, U64* mulb, U64* mulbx, const int64u* inpA_mb8, const int64u* inpM_mb8, const int64u* k0_mb8, int64u MulTbl[][redLen][8], int64u MulTblx[][redLen][8], const int64u Idx[8]); +EXTERN_C void AMS5x52x20_diagonal_mb8(int64u* out_mb8, const int64u* inpA_mb8, const int64u* inpM_mb8, const int64u* k0_mb8); + +EXTERN_C void AMS52x40_diagonal_mb8(int64u* out_mb8, const int64u* inpA_mb8, const int64u* inpM_mb8, const int64u* k0_mb8); +EXTERN_C void AMS5x52x40_diagonal_mb8(int64u* out_mb8, const int64u* inpA_mb8, const int64u* inpM_mb8, const int64u* k0_mb8); + +EXTERN_C void AMS52x30_diagonal_mb8(int64u* out_mb8, const int64u* inpA_mb8, const int64u* inpM_mb8, const int64u* k0_mb8); +EXTERN_C void AMS52x60_diagonal_mb8(int64u* out_mb8, const int64u* inpA_mb8, const int64u* inpM_mb8, const int64u* k0_mb8); +EXTERN_C void AMS52x79_diagonal_mb8(int64u* out_mb8, const int64u* inpA_mb8, const int64u* inpM_mb8, const int64u* k0_mb8); + +// 4x Diagonal sqr +EXTERN_C void AMS52x20_diagonal_mb4(int64u* out_mb8, const int64u* inpA_mb8, const int64u* inpM_mb8, const int64u* k0_mb8); +EXTERN_C void AMS5x52x20_diagonal_mb4(int64u* out_mb8, const int64u* inpA_mb8, const int64u* inpM_mb8, const int64u* k0_mb8); + +// clear/copy mb8 buffer +EXTERN_C void zero_mb8(int64u(*redOut)[8], int len); +EXTERN_C void copy_mb8(int64u out[][8], const int64u inp[][8], int len); + +// other 2^52 radix arith functions +EXTERN_C void ifma_montFactor52_mb8(int64u k0_mb8[8], const int64u m0_mb8[8]); + +EXTERN_C void ifma_modsub52x10_mb8(int64u res[][8], const int64u inpA[][8], const int64u inpB[][8], const int64u inpM[][8]); +EXTERN_C void ifma_modsub52x20_mb8(int64u res[][8], const int64u inpA[][8], const int64u inpB[][8], const int64u inpM[][8]); +EXTERN_C void ifma_modsub52x30_mb8(int64u res[][8], const int64u inpA[][8], const int64u inpB[][8], const int64u inpM[][8]); +EXTERN_C void ifma_modsub52x40_mb8(int64u res[][8], const int64u inpA[][8], const int64u inpB[][8], const int64u inpM[][8]); + +EXTERN_C void ifma_addmul52x10_mb8(int64u res[][8], const int64u inpA[][8], const int64u inpB[][8]); +EXTERN_C void ifma_addmul52x20_mb8(int64u res[][8], const int64u inpA[][8], const int64u inpB[][8]); +EXTERN_C void ifma_addmul52x30_mb8(int64u res[][8], const int64u inpA[][8], const int64u inpB[][8]); +EXTERN_C void ifma_addmul52x40_mb8(int64u res[][8], const int64u inpA[][8], const int64u inpB[][8]); + +EXTERN_C void ifma_amred52x10_mb8(int64u res[][8], const int64u inpA[][8], const int64u inpM[][8], const int64u k0[8]); +EXTERN_C void ifma_amred52x20_mb8(int64u res[][8], const int64u inpA[][8], const int64u inpM[][8], const int64u k0[8]); +EXTERN_C void ifma_amred52x30_mb8(int64u res[][8], const int64u inpA[][8], const int64u inpM[][8], const int64u k0[8]); +EXTERN_C void ifma_amred52x40_mb8(int64u res[][8], const int64u inpA[][8], const int64u inpM[][8], const int64u k0[8]); + +EXTERN_C void ifma_mreduce52x_mb8(int64u pX[][8], int nsX, int64u pM[][8], int nsM); +EXTERN_C void ifma_montRR52x_mb8(int64u pRR[][8], int64u pM[][8], int convBitLen); + +// exponentiations +EXTERN_C void EXP52x10_mb8(int64u out[][8], + const int64u base[][8], + const int64u exponent[][8], + const int64u modulus[][8], + const int64u toMont[][8], + const int64u k0_mb8[8], + int64u work_buffer[][8]); + +EXTERN_C void EXP52x20_mb8(int64u out[][8], + const int64u base[][8], + const int64u exponent[][8], + const int64u modulus[][8], + const int64u toMont[][8], + const int64u k0_mb8[8], + int64u work_buffer[][8]); + +EXTERN_C void EXP52x40_mb8(int64u out[][8], + const int64u base[][8], + const int64u exponent[][8], + const int64u modulus[][8], + const int64u toMont[][8], + const int64u k0_mb8[8], + int64u work_buffer[][8]); + +EXTERN_C void EXP52x60_mb8(int64u out[][8], + const int64u base[][8], + const int64u exponent[][8], + const int64u modulus[][8], + const int64u toMont[][8], + const int64u k0_mb8[8], + int64u work_buffer[][8]); + +EXTERN_C void EXP52x30_mb8(int64u out[][8], + const int64u base[][8], + const int64u exponent[][8], + const int64u modulus[][8], + const int64u toMont[][8], + const int64u k0_mb8[8], + int64u work_buffer[][8]); + +EXTERN_C void EXP52x79_mb8(int64u out[][8], + const int64u base[][8], + const int64u exponent[][8], + const int64u modulus[][8], + const int64u toMont[][8], + const int64u k0_mb8[8], + int64u work_buffer[][8]); + +// exponentiations (fixed short exponent ==65537) +EXTERN_C void EXP52x20_pub65537_mb8(int64u out[][8], + const int64u base[][8], + const int64u modulus[][8], + const int64u toMont[][8], + const int64u k0[8], + int64u work_buffer[][8]); + + +EXTERN_C void EXP52x40_pub65537_mb8(int64u out[][8], + const int64u base[][8], + const int64u modulus[][8], + const int64u toMont[][8], + const int64u k0[8], + int64u work_buffer[][8]); + +EXTERN_C void EXP52x60_pub65537_mb8(int64u out[][8], + const int64u base[][8], + const int64u modulus[][8], + const int64u toMont[][8], + const int64u k0[8], + int64u work_buffer[][8]); + +EXTERN_C void EXP52x79_pub65537_mb8(int64u out[][8], + const int64u base[][8], + const int64u modulus[][8], + const int64u toMont[][8], + const int64u k0[8], + int64u work_buffer[][8]); + +#endif /* _IFMA_INTERNAL_H_ */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/rsa/ifma_rsa_layer_cp.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/rsa/ifma_rsa_layer_cp.h new file mode 100644 index 0000000..8b43484 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/rsa/ifma_rsa_layer_cp.h @@ -0,0 +1,48 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_CP_LAYER_H +#define IFMA_CP_LAYER_H + +#include +#include + +EXTERN_C void ifma_cp_rsa_pub_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const int64u* const n_pa[8], + int rsaBitlen, + const mbx_RSA_Method* m, + int8u* pBuffer); +EXTERN_C void ifma_cp_rsa_prv2_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const int64u* const d_pa[8], + const int64u* const n_pa[8], + int rsaBitlen, + const mbx_RSA_Method* m, + int8u* pBuffer); +EXTERN_C void ifma_cp_rsa_prv5_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const int64u* const p_pa[8], + const int64u* const q_pa[8], + const int64u* const dp_pa[8], + const int64u* const dq_pa[8], + const int64u* const iq_pa[8], + int rsaBitlen, + const mbx_RSA_Method* m, + int8u* pBuffer); + +#endif /* IFMA_CP_LAYER_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/rsa/ifma_rsa_layer_ssl.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/rsa/ifma_rsa_layer_ssl.h new file mode 100644 index 0000000..e814590 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/rsa/ifma_rsa_layer_ssl.h @@ -0,0 +1,86 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef BN_OPENSSL_DISABLE + +#if !defined(_IFMA_INTERNAL_SSL_LAYER_H_) +#define _IFMA_INTERNAL_SSL_LAYER_H_ + +#include +#include + +EXTERN_C void ifma_ssl_rsa1K_pub_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const n_pa[8]); +EXTERN_C void ifma_ssl_rsa2K_pub_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const n_pa[8]); +EXTERN_C void ifma_ssl_rsa3K_pub_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const n_pa[8]); +EXTERN_C void ifma_ssl_rsa4K_pub_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const n_pa[8]); + +EXTERN_C void ifma_ssl_rsa1K_prv2_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const d_pa[8], + const BIGNUM* const n_pa[8]); +EXTERN_C void ifma_ssl_rsa2K_prv2_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const d_pa[8], + const BIGNUM* const n_pa[8]); +EXTERN_C void ifma_ssl_rsa3K_prv2_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const d_pa[8], + const BIGNUM* const n_pa[8]); +EXTERN_C void ifma_ssl_rsa4K_prv2_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const d_pa[8], + const BIGNUM* const n_pa[8]); + +EXTERN_C void ifma_ssl_rsa1K_prv5_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const p_pa[8], + const BIGNUM* const q_pa[8], + const BIGNUM* const dp_pa[8], + const BIGNUM* const dq_pa[8], + const BIGNUM* const iq_pa[8]); +EXTERN_C void ifma_ssl_rsa2K_prv5_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const p_pa[8], + const BIGNUM* const q_pa[8], + const BIGNUM* const dp_pa[8], + const BIGNUM* const dq_pa[8], + const BIGNUM* const iq_pa[8]); +EXTERN_C void ifma_ssl_rsa3K_prv5_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const p_pa[8], + const BIGNUM* const q_pa[8], + const BIGNUM* const dp_pa[8], + const BIGNUM* const dq_pa[8], + const BIGNUM* const iq_pa[8]); +EXTERN_C void ifma_ssl_rsa4K_prv5_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const p_pa[8], + const BIGNUM* const q_pa[8], + const BIGNUM* const dp_pa[8], + const BIGNUM* const dq_pa[8], + const BIGNUM* const iq_pa[8]); +#endif /* _IFMA_INTERNAL_SSL_LAYER_H_ */ + +#endif /* BN_OPENSSL_DISABLE */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/rsa/ifma_rsa_method.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/rsa/ifma_rsa_method.h new file mode 100644 index 0000000..63874f0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/rsa/ifma_rsa_method.h @@ -0,0 +1,86 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_RSA_METHOD_H +#define IFMA_RSA_METHOD_H + +#include + + +/* exponentiations */ +typedef void(*EXP52x_65537_mb8)(int64u out[][8], + const int64u base[][8], + const int64u modulus[][8], + const int64u toMont[][8], + const int64u k0[8], + int64u work_buffer[][8]); +typedef void(*EXP52x_mb8)(int64u out[][8], + const int64u base[][8], + const int64u exponent[][8], + const int64u modulus[][8], + const int64u toMont[][8], + const int64u k0_mb8[8], + int64u work_buffer[][8]); + + +/* +// auxiliary mb8 arithmethic +*/ +/* Mont reduction */ +typedef void (*amred52x_mb8)(int64u res[][8], const int64u inpA[][8], const int64u inpM[][8], const int64u k0[8]); +/* Mont multiplication */ +typedef void (*ammul52x_mb8)(int64u* out_mb8, const int64u* inpA_mb8, const int64u* inpB_mb8, const int64u* inpM_mb8, const int64u* k0_mb8); +/* modular subtraction */ +typedef void (*modsub52x_mb8)(int64u res[][8], const int64u inpA[][8], const int64u inpB[][8], const int64u inpM[][8]); +/* multiply and add */ +typedef void (*addmul52x_mb8)(int64u res[][8], const int64u inpA[][8], const int64u inpB[][8]); + +/* RSA operation */ +typedef enum { + RSA_PUB_KEY = (0x01<<16), + RSA_PRV2_KEY = (0x20<<16), + RSA_PRV5_KEY = (0x50<<16), +} RSA_OP_ID; + +/* RSA size */ +typedef enum { + RSA1024 = 1024, + RSA2048 = 2048, + RSA3072 = 3072, + RSA4096 = 4096, +} RSA_BITSIZE_ID; + +/* RSA ID */ +#define RSA_ID(OP,BITSIZE) ((OP) | (BITSIZE)) +#define OP_RSA_ID(ID) ((ID) & (0xFF<<16)) +#define BISIZE_RSA_ID(ID) ((ID) & 0xFFFF) + +struct _ifma_rsa_method { + int id; /* exponentiation's id (=1/2/5 -- public(fixed)/private/private_crt */ + int rsaBitsize; /* size of rsa modulus (bits) */ + int buffSize; /* size of scratch buffer */ + //cvt52BN_to_mb8 cvt52; /* convert non-contiguos BN to radix 2^52 and strore in mb8 forman */ + //tcopyBN_to_mb8 tcopy; /* copy non-contiguos BN into mb8 format */ + EXP52x_65537_mb8 expfunc65537; /* "exp52x_fix_mb8" fixed exponentiation */ + EXP52x_mb8 expfun; /* "exp52x_arb_mb8" exponentiation */ + amred52x_mb8 amred52x; /* reduction */ + ammul52x_mb8 ammul52x; /* multiplication */ + modsub52x_mb8 modsub52x; /* subtration */ + addmul52x_mb8 mla52x; /* multiply & add */ +}; + +#endif /* IFMA_RSA_METHOD_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm2/ifma_arith_sm2.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm2/ifma_arith_sm2.h new file mode 100644 index 0000000..c58ca84 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm2/ifma_arith_sm2.h @@ -0,0 +1,136 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_ARITH_SM2_H +#define IFMA_ARITH_SM2_H + +#include +#include +#include + +/* underlying prime's size */ +#define PSM2_BITSIZE (256) + +#define PSM2_LEN52 NUMBER_OF_DIGITS(PSM2_BITSIZE,DIGIT_SIZE) +#define PSM2_LEN64 NUMBER_OF_DIGITS(PSM2_BITSIZE,64) +#define PSM2_LEN8 NUMBER_OF_DIGITS(PSM2_BITSIZE,8) + +/* set FE to zero */ +__INLINE void MB_FUNC_NAME(zero_FESM2_)(U64 T[]) +{ + T[0] = T[1] = T[2] = T[3] = T[4] = get_zero64(); +} + +/* check if FE is zero */ +__INLINE __mb_mask MB_FUNC_NAME(is_zero_FESM2_)(const U64 T[]) +{ + U64 Z = or64(or64(T[0], T[1]), or64(or64(T[2], T[3]), T[4])); + return cmpeq64_mask(Z, get_zero64()); +} + +/* move field element */ +__INLINE void MB_FUNC_NAME(mov_FESM2_)(U64 r[], const U64 a[]) +{ + r[0] = a[0]; + r[1] = a[1]; + r[2] = a[2]; + r[3] = a[3]; + r[4] = a[4]; +} + +/* move coodinate using mask: R = k? A : B */ +__INLINE void MB_FUNC_NAME(mask_mov_FESM2_)(U64 R[], const U64 B[], __mb_mask k, const U64 A[]) +{ + R[0] = mask_mov64(B[0], k, A[0]); + R[1] = mask_mov64(B[1], k, A[1]); + R[2] = mask_mov64(B[2], k, A[2]); + R[3] = mask_mov64(B[3], k, A[3]); + R[4] = mask_mov64(B[4], k, A[4]); +} + +__INLINE void MB_FUNC_NAME(secure_mask_mov_FESM2_)(U64 R[], U64 B[], __mb_mask k, const U64 A[]) +{ + R[0] = select64(k, B[0], (U64*)(&A[0])); + R[1] = select64(k, B[1], (U64*)(&A[1])); + R[2] = select64(k, B[2], (U64*)(&A[2])); + R[3] = select64(k, B[3], (U64*)(&A[3])); + R[4] = select64(k, B[4], (U64*)(&A[4])); +} + +/* compare two FE */ +__INLINE __mb_mask MB_FUNC_NAME(cmp_lt_FESM2_)(const U64 A[], const U64 B[]) +{ + /* r = a - b */ + U64 r0 = sub64(A[0], B[0]); + U64 r1 = sub64(A[1], B[1]); + U64 r2 = sub64(A[2], B[2]); + U64 r3 = sub64(A[3], B[3]); + U64 r4 = sub64(A[4], B[4]); + + /* normalize r0 – r4 */ + NORM_ASHIFTR(r, 0, 1) + NORM_ASHIFTR(r, 1, 2) + NORM_ASHIFTR(r, 2, 3) + NORM_ASHIFTR(r, 3, 4) + + /* return mask LT */ + return cmp64_mask(r4, get_zero64(), _MM_CMPINT_LT); +} + +__INLINE __mb_mask MB_FUNC_NAME(cmp_eq_FESM2_)(const U64 A[], const U64 B[]) +{ + __ALIGN64 U64 msg[PSM2_LEN52]; + + msg[0] = xor64(A[0], B[0]); + msg[1] = xor64(A[1], B[1]); + msg[2] = xor64(A[2], B[2]); + msg[3] = xor64(A[3], B[3]); + msg[4] = xor64(A[4], B[4]); + + return MB_FUNC_NAME(is_zero_FESM2_)(msg); +} + +/* Specialized operations over EC SM2 prime */ +EXTERN_C void MB_FUNC_NAME(ifma_tomont52_psm2_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_frommont52_psm2_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_amm52_psm2_)(U64 r[], const U64 va[], const U64 vb[]); +EXTERN_C void MB_FUNC_NAME(ifma_ams52_psm2_)(U64 r[], const U64 va[]); +EXTERN_C void MB_FUNC_NAME(ifma_aminv52_psm2_)(U64 r[], const U64 z[]); +EXTERN_C void MB_FUNC_NAME(ifma_add52_psm2_)(U64 r[], const U64 a[], const U64 b[]); +EXTERN_C void MB_FUNC_NAME(ifma_sub52_psm2_)(U64 r[], const U64 a[], const U64 b[]); +EXTERN_C void MB_FUNC_NAME(ifma_neg52_psm2_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_double52_psm2_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_tripple52_psm2_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_half52_psm2_)(U64 r[], const U64 a[]); +EXTERN_C __mb_mask MB_FUNC_NAME(ifma_cmp_lt_psm2_)(const U64 a[]); +EXTERN_C __mb_mask MB_FUNC_NAME(ifma_check_range_psm2_)(const U64 a[]); + +/* Specialized operations over EC SM2 order */ +EXTERN_C U64* MB_FUNC_NAME(ifma_nsm2_)(void); +EXTERN_C void MB_FUNC_NAME(ifma_tomont52_nsm2_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_frommont52_nsm2_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_ams52_nsm2_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_amm52_nsm2_)(U64 r[], const U64 a[], const U64 b[]); +EXTERN_C void MB_FUNC_NAME(ifma_aminv52_nsm2_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_add52_nsm2_)(U64 r[], const U64 a[], const U64 b[]); +EXTERN_C void MB_FUNC_NAME(ifma_sub52_nsm2_)(U64 r[], const U64 a[], const U64 b[]); +EXTERN_C void MB_FUNC_NAME(ifma_neg52_nsm2_)(U64 r[], const U64 a[]); +EXTERN_C void MB_FUNC_NAME(ifma_fastred52_pnsm2_)(U64 r[], const U64 a[]); +EXTERN_C __mb_mask MB_FUNC_NAME(ifma_cmp_lt_nsm2_)(const U64 a[]); +EXTERN_C __mb_mask MB_FUNC_NAME(ifma_check_range_nsm2_)(const U64 a[]); + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm2/ifma_ecpoint_sm2.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm2/ifma_ecpoint_sm2.h new file mode 100644 index 0000000..c19717d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm2/ifma_ecpoint_sm2.h @@ -0,0 +1,93 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_ECPOINT_SM2_H +#define IFMA_ECPOINT_SM2_H + +#include + +typedef struct { + U64 X[PSM2_LEN52]; + U64 Y[PSM2_LEN52]; + U64 Z[PSM2_LEN52]; +} SM2_POINT; + +typedef struct { + U64 x[PSM2_LEN52]; + U64 y[PSM2_LEN52]; +} SM2_POINT_AFFINE; + +typedef struct { + int64u x[PSM2_LEN52]; + int64u y[PSM2_LEN52]; +} SINGLE_SM2_POINT_AFFINE; + +/* check if coodinate is zero */ +__INLINE __mb_mask MB_FUNC_NAME(is_zero_point_cordinate_)(const U64 T[]) +{ + return MB_FUNC_NAME(is_zero_FESM2_)(T); +} + +/* set point to infinity */ +__INLINE void MB_FUNC_NAME(set_point_to_infinity_)(SM2_POINT* r) +{ + r->X[0] = r->X[1] = r->X[2] = r->X[3] = r->X[4] = get_zero64(); + r->Y[0] = r->Y[1] = r->Y[2] = r->Y[3] = r->Y[4] = get_zero64(); + r->Z[0] = r->Z[1] = r->Z[2] = r->Z[3] = r->Z[4] = get_zero64(); +} + +/* set point to infinity by mask */ +__INLINE void MB_FUNC_NAME(mask_set_point_to_infinity_)(SM2_POINT* r, __mb_mask mask) +{ + U64 zeros = get_zero64(); + + r->X[0] = mask_mov64(r->X[0], mask, zeros); + r->X[1] = mask_mov64(r->X[1], mask, zeros); + r->X[2] = mask_mov64(r->X[2], mask, zeros); + r->X[3] = mask_mov64(r->X[3], mask, zeros); + r->X[4] = mask_mov64(r->X[4], mask, zeros); + + r->Y[0] = mask_mov64(r->Y[0], mask, zeros); + r->Y[1] = mask_mov64(r->Y[1], mask, zeros); + r->Y[2] = mask_mov64(r->Y[2], mask, zeros); + r->Y[3] = mask_mov64(r->Y[3], mask, zeros); + r->Y[4] = mask_mov64(r->Y[4], mask, zeros); + + r->Z[0] = mask_mov64(r->Z[0], mask, zeros); + r->Z[1] = mask_mov64(r->Z[1], mask, zeros); + r->Z[2] = mask_mov64(r->Z[2], mask, zeros); + r->Z[3] = mask_mov64(r->Z[3], mask, zeros); + r->Z[4] = mask_mov64(r->Z[4], mask, zeros); +} + +/* set affine point to infinity */ +__INLINE void MB_FUNC_NAME(set_point_affine_to_infinity_)(SM2_POINT_AFFINE* r) +{ + r->x[0] = r->x[1] = r->x[2] = r->x[3] = r->x[4] = get_zero64(); + r->y[0] = r->y[1] = r->y[2] = r->y[3] = r->y[4] = get_zero64(); +} + +EXTERN_C void MB_FUNC_NAME(ifma_ec_sm2_dbl_point_)(SM2_POINT* r, const SM2_POINT* p); +EXTERN_C void MB_FUNC_NAME(ifma_ec_sm2_add_point_)(SM2_POINT* r, const SM2_POINT* p, const SM2_POINT* q); +EXTERN_C void MB_FUNC_NAME(ifma_ec_sm2_add_point_affine_)(SM2_POINT* r, const SM2_POINT* p, const SM2_POINT_AFFINE* q); +EXTERN_C void MB_FUNC_NAME(ifma_ec_sm2_mul_point_)(SM2_POINT* r, const SM2_POINT* p, const U64* scalar); +EXTERN_C void MB_FUNC_NAME(ifma_ec_sm2_mul_pointbase_)(SM2_POINT* r, const U64* scalar); +EXTERN_C void MB_FUNC_NAME(get_sm2_ec_affine_coords_)(U64 x[], U64 y[], const SM2_POINT* P); +EXTERN_C const U64* MB_FUNC_NAME(ifma_ec_sm2_coord_one_)(void); +EXTERN_C __mb_mask MB_FUNC_NAME(ifma_is_on_curve_psm2_)(const SM2_POINT* p, int use_jproj_coords); + +#endif /* IFMA_ECPOINT_PSM2_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm2/ifma_ecprecomp4_psm2.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm2/ifma_ecprecomp4_psm2.h new file mode 100644 index 0000000..9896297 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm2/ifma_ecprecomp4_psm2.h @@ -0,0 +1,682 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_ECPRECOMP4_PSM2_H +#define IFMA_ECPRECOMP4_PSM2_H + +#include + +#define MUL_BASEPOINT_WIN_SIZE (4) + +#define BP_WIN_SIZE MUL_BASEPOINT_WIN_SIZE +#define BP_N_SLOTS NUMBER_OF_DIGITS(PSM2_BITSIZE+1,BP_WIN_SIZE) +#define BP_N_ENTRY (1<<(BP_WIN_SIZE-1)) + +__ALIGN64 static SINGLE_SM2_POINT_AFFINE ifma_ec_sm2_bp_precomp[][BP_N_ENTRY] = { +{/* digit=0 [{1,2,3,..,}]*([2^0]*G) */ + {{0x0008990f418029e9,0x000e6ca6c04fd132,0x00024c3c33e7981e,0x000b05d6a1ed99ac,0x00001167a5f71c13}, {0x0004e593c2d0ddd6,0x0008ed3295fa6135,0x0002a48f8c1f5e57,0x0005bd8d4cfb066e,0x00003cd65d4e1d73}}, + {{0x00037bfbc3be46a0,0x000a2d8fa9380af0,0x00088cd2483bdc9b,0x00036a5349d94b57,0x0000d7e9c18caa57}, {0x0001a1d69db9ac19,0x000854a8e82ada7e,0x0007157acccbd8d3,0x000df5c7b145169b,0x000047e7465fc21b}}, + {{0x00054fdab589e4a3,0x000a0b4f0a0cecda,0x000eb4a0a2676528,0x000c640a265a308c,0x0000019fd6c2e887}, {0x000fbe94b2fc1902,0x000ba7cbce5fea10,0x000c13c97f40aa52,0x000bb4cc496bfa6d,0x00008ad34787bb3f}}, + {{0x0007c5a986150605,0x000039016208e93f,0x0009f9020487ea28,0x000be18a86bcb4a0,0x00000dc8e3b1899d}, {0x0009043fd6199986,0x000adc7383bd9c09,0x000cf70ed1de135e,0x0009e34d0bd55632,0x0000ffc31c585bce}}, + {{0x00056336a9c81620,0x000221dfcc539a57,0x000f5f4c515aa58f,0x0005b97ad354bf1e,0x0000f443ef363f87}, {0x000d68fd34501337,0x000e53607d17be81,0x0002258efb30f4bb,0x000768b1826a4c36,0x0000b415276842a6}}, + {{0x0009c4c0acd72ba8,0x0005de7ec73b936a,0x000e34db6b1274a2,0x000047f15a876e5d,0x00005e74ca10cba8}, {0x0004cddb469eb37b,0x0000799754f75845,0x00030e9848fbf6d2,0x00098a1060e7f8ec,0x0000568bc97fb8c5}}, + {{0x00031c781f06784b,0x0003b713251ffa35,0x000fe18c50b89419,0x000c2884ee5b69ac,0x0000bf492e189fbe}, {0x00035c1e5f6186d2,0x000e957a01b8113a,0x000d99baf0e449a2,0x00077796c9b9922b,0x0000ba05a8f5b84d}}, + {{0x0003a1c091226701,0x000a32cc810cce52,0x00087df9e90be6f2,0x000134086e634143,0x000015c2fc0e9c44}, {0x000430d8799302ae,0x0001c27b7ea3b334,0x0008f3382693b350,0x000247cbe1136f9a,0x000077fd5f395778}}, +}, +{/* digit=1 [{1,2,3,..,}]*([2^4]*G) */ + {{0x00060c939e8a120c,0x00093273b59a3715,0x000f4639d7121d6b,0x000d58649535ce8e,0x0000d01076ed4cc6}, {0x000729a96e74f8a6,0x0000b533037dc705,0x0003c5c62b07e323,0x000ac10846dcc166,0x0000a4759c170fc3}}, + {{0x0001369adbb3c8b5,0x000b9c8ec37856c1,0x000f0a982d78af40,0x00050affb3a80d03,0x000050e3e71f83be}, {0x0000fb2418ee45bb,0x0003b791b963d45c,0x00047e33b5297cf4,0x0005b2676b638ccc,0x00001c52fadaecf8}}, + {{0x000f511b36362ecf,0x0007240859590a8f,0x00070c62b11c5a9f,0x000827272b86f297,0x000006262266c7e8}, {0x00068bfea1e13ebf,0x0002be59b0f4a291,0x000826f34db89297,0x000cec6769e31d4f,0x0000a1dd934fa955}}, + {{0x000a5ae0d038ad4f,0x000426256c8e90ff,0x000978dcef44d3df,0x0003170a3077c8bc,0x00003d9b4b1645b8}, {0x0004484b6b1852c5,0x00045e78ff073bbf,0x000c242380cd02ed,0x000b5591cb827e49,0x00008adaee62aa3c}}, + {{0x0001687af48d3524,0x0000068c04010e5a,0x0003b2e10fa700b7,0x0001f5c06c84139f,0x00007cd5ff4bcf2f}, {0x000c6996a3ebd39b,0x00085f9371d6b52d,0x000c57dadb8d9103,0x000b8bee3f1d85f2,0x0000c987637ef0d8}}, + {{0x00091da5f6fbf645,0x000364de051b9b7a,0x0005a1ac8ad5bf46,0x0002fa13e4455d48,0x0000d09b6e064f3a}, {0x000ffc804bf14165,0x000c1e88e44737c3,0x0003777ca4b5beda,0x0001181fc4f3fd98,0x00005e4509f06d4b}}, + {{0x0007754a47073753,0x00053923a9dc5b93,0x000ab2dd726607db,0x0000a7690f12ae67,0x0000a39fc8c30a65}, {0x00095fd0ae53d66e,0x000319b1c44980fe,0x000eb8e6eefebcba,0x000c8cf8b3ca72f1,0x0000586e40734f5c}}, + {{0x000ce4d033fc12ae,0x00022886f31527e6,0x0006f3f11ba4f98a,0x000de3b24b38f3e6,0x00003f6205b3ea4b}, {0x0005387a77b998f4,0x0007d549f3b0c070,0x0003a61d62c9b445,0x00081adef6625b53,0x0000eda7e2abe4f7}}, +}, +{/* digit=2 [{1,2,3,..,}]*([2^8]*G) */ + {{0x0000aaafd1201340,0x0009c057309ce973,0x000726ce7b22b908,0x00058498e7956584,0x0000e1431a0d635a}, {0x0007023e834ffa6a,0x000fa36ab1ae1d38,0x00024b68464198dd,0x000c6d46e5ebb191,0x0000316fa44c33b3}}, + {{0x0000210a1371aa41,0x000064b5424eac97,0x00064269baff481a,0x0007cfbcdf91fd0e,0x00008bb37bbc02fc}, {0x000dd796f69d4390,0x000f169514b2d99e,0x000e19ae44e27a58,0x00096d80eca1ca66,0x0000470e96507886}}, + {{0x0008bfc3dfdc228b,0x000e69648a35b377,0x0005a99b5c0bae0a,0x00058ada8cb8ab01,0x0000045cccc3366b}, {0x0008ef44164cebd7,0x000935e00e5f04ef,0x00079468e41e71fc,0x00072d753cf90644,0x00008b5223fa32ea}}, + {{0x0009e2f5876d6e8b,0x000682b622d6a28a,0x0005411d793c48f8,0x000e9188d9eac83e,0x00007e4a6bab0a70}, {0x000e5baf1c43b2ea,0x000f42f347de0f18,0x00009638f46578c7,0x000d2919ca736df0,0x00006563f1f5d1ac}}, + {{0x00054b5577dae353,0x0006dac3a655cefc,0x000c31d009d2f054,0x00063db96bd298fa,0x0000328a51d1e695}, {0x000098e43195f4e8,0x00062998010ade19,0x00047ccb6657f4ba,0x000da645f29a8440,0x000033a5443ee423}}, + {{0x000f29899229a90d,0x0008ed71d5319e6c,0x000e3a8b7a6840bc,0x000a0e803e540771,0x00005611ee53afd9}, {0x000ca0ebbbefa73b,0x000c75ec48b72739,0x000dea0ec6082dba,0x000e03a0ab10dfbb,0x00001b7ebe5a1633}}, + {{0x0004475ad1cac223,0x000ef7e1139532e6,0x000479c4a2008653,0x0000d1a875ad01d9,0x0000e6cf633b04b3}, {0x000ef6eb6b06e46b,0x000fa7b8410aa8b3,0x00078458d74c45db,0x000bd102675759c2,0x00002ef49575cd30}}, + {{0x00012dd12ee579d7,0x000b76dd62d6170b,0x0001d75822a9a12a,0x000457bcd5259907,0x0000a36193b1869c}, {0x0006592e976ae5bf,0x00039dfecd57e29e,0x00014686de82c871,0x0003babc83a440f7,0x0000e19344afc21e}}, +}, +{/* digit=3 [{1,2,3,..,}]*([2^12]*G) */ + {{0x000010b2c4391719,0x0007781393878f41,0x0000776333ff85ee,0x0000234ada4c7d8f,0x0000976011b124e6}, {0x0001197eaf49f638,0x0000560b0c4c2250,0x000b3df90dff2123,0x000d5f1a6a3abbba,0x000054bbcc7366ff}}, + {{0x000c28718d13bd32,0x000443c6dd1a9b45,0x000b9cf87bbf3a89,0x00048dc8171c5e13,0x0000dfc779254f53}, {0x000662d985cabd47,0x000cbd971de02b9a,0x0004cba64588a6eb,0x000e67da9fd89457,0x0000e0f0ccad51e6}}, + {{0x000572f9f177d653,0x00055c2394c3f74c,0x000307ad6c39e9e1,0x0009c8c9e7b18c1e,0x00003da596c7c22d}, {0x0000e12b74b9c00c,0x000e19f5d1339c30,0x0009e84425f434a9,0x0001c35b87294371,0x00000164ba54acd1}}, + {{0x000276c4b8bceb84,0x0001b52d110648ca,0x00034dcbf6d4ec10,0x0009d02ad98063f8,0x0000da81d1a1fff1}, {0x0007cd23a9828ffc,0x0000ee64b3318ccc,0x00008b81cf1e389b,0x000ac6e2fb6a6c73,0x00006df66b31bcc0}}, + {{0x00029d74220fbbff,0x0004f09ba10d52f0,0x000858df9405e1d8,0x000b23e53bd89048,0x0000d7bc7aa35dab}, {0x000f3807f06c21c7,0x0000b324828efe76,0x00017d5a3616f89c,0x00060fe30abf04a5,0x0000714fd2e0d0db}}, + {{0x00013306dccac081,0x00050610b07430a3,0x000edd616b67ab62,0x0006666fd86597e7,0x000029abb54bf0c4}, {0x000f5f3c3ca87bd2,0x0002fde60e830fef,0x00077b8c7bee8fe9,0x00030251c192c984,0x0000b0bc40c43995}}, + {{0x000e53d299f5d074,0x00060132e675167f,0x0007ce589b1afc7c,0x0006d18ac9a60464,0x000054e627504184}, {0x000160130ed07dbb,0x000342bd885fb63b,0x00069a710648bea2,0x000c83def2b055c5,0x0000a36e625f096b}}, + {{0x0008c75e1c58c801,0x000493fcc95a4ccb,0x000ccbcf92ba9de0,0x0003adccdeb0eedf,0x0000d667d4f80f3d}, {0x00014a536269820c,0x000bcfe8794006aa,0x000e5cfb2329a308,0x000b2c39869970ed,0x000033c3069b601b}}, +}, +{/* digit=4 [{1,2,3,..,}]*([2^16]*G) */ + {{0x000045511c092897,0x0007864079c97c25,0x0003879a283042ba,0x0002ee4881640c6e,0x00007c5babcf0245}, {0x00059a67088f360a,0x000d574be7e8da77,0x00033828902da352,0x0008c615800cdbe0,0x0000d69f7c9f0168}}, + {{0x000c67faee039995,0x0003c9ff7f90c64a,0x0008da6ea4de174d,0x000df7063e494354,0x0000264880c2b7cc}, {0x000f34b49b992ccc,0x000dd406586f8a18,0x000db0e21e16b6f4,0x0009f8d32479ac4c,0x0000e8151f6d62bc}}, + {{0x0002a1cc216cf37f,0x0006673828be9ae8,0x00051a85bac437f4,0x0005788c12ff189d,0x0000eb563bf24c16}, {0x00053b6c67069662,0x000a2cda87337d93,0x0004953dbcdc6eb5,0x000f7c033c186e3e,0x0000ba46a66d2e37}}, + {{0x000f516da0abf3eb,0x000511b3381eceb7,0x0004188703c92ac9,0x000066bad7320ed3,0x0000ab7a126b7dbe}, {0x00059be2def303f7,0x00079d1e0c9f77ce,0x000f418dc0bf1f23,0x00067612c18d1e38,0x0000fcc5e3f285bb}}, + {{0x000e7047e1754b8b,0x000261c0027aeefe,0x000af4e6d5408547,0x000569c5e7a6fa45,0x00004d3cd58e0048}, {0x0000014f3ae8e79e,0x0004649f3f22f2c2,0x000982a8cd0b6af1,0x000f903a1db91517,0x0000a3c809ad9ffc}}, + {{0x000c9530fa12b5b4,0x0003b7d159b4d8ea,0x000844a0d45ccf07,0x000c18a74804446e,0x0000404e6c7177d1}, {0x00043a6ce1af18fa,0x0001b7fdffcb603e,0x000bf3d178a82808,0x00028691b63c11ca,0x00004dedc224d26f}}, + {{0x000853b60aa5c141,0x00044850cc05c202,0x000abccfd1dc35d3,0x00025a8014357e0c,0x0000aa44ce9d5a52}, {0x000f9203a8444b4b,0x0004795384b0f3ce,0x000e5da54cf3f91b,0x00046a1d625ba1c9,0x0000f1fba3861d0f}}, + {{0x0005caee30ae90b2,0x0000eaabea0d0cda,0x000e678562cc3429,0x0008b7564afcd941,0x000010c7a09ef6ef}, {0x000d352f82a591d2,0x000dcb43d2a1a316,0x0004e94705fe8cc4,0x000a07d8ebce978b,0x00006c78f44da321}}, +}, +{/* digit=5 [{1,2,3,..,}]*([2^20]*G) */ + {{0x00038c7656047260,0x0003247a421e681a,0x00094956e4f8c6ae,0x0003241a51eaa012,0x0000984b1ef47c9b}, {0x000bd0d597b76968,0x000888e57ee6b749,0x000a112d29d432b7,0x0007f53092afe12b,0x00009ccee4996c5a}}, + {{0x0009d7b54089685c,0x000000818348755e,0x00061b80f9f0ec69,0x000a144cf4d8cd38,0x0000ce669fe81f5f}, {0x00025091788f9dae,0x0004d3ccf2390a21,0x000027f3b3295361,0x000bcb1048d09250,0x0000807b39e2270f}}, + {{0x000c73cdf18eddfb,0x000a92465ee8a660,0x000f950c877edb4b,0x00068baf24eff83a,0x0000822c6b31f6cb}, {0x00052f6149d2bb2d,0x000bc4a66ae65503,0x0007e1c68926809d,0x00029b9964c2803f,0x0000253298b86681}}, + {{0x000dc5595e388c34,0x00062a44e3ea1099,0x000b41f7bd0670ff,0x000305d212c99361,0x0000f594afa3af13}, {0x0008bf205c01232b,0x0006e9ff08a50c50,0x00037741a7683353,0x000d2aa1cf70bdb8,0x0000a8e66175af7b}}, + {{0x00068da909bf8c66,0x000bef4bad4a4b7b,0x000e39585b5bd7c8,0x0002d9a0ebb027fa,0x00008932a372676d}, {0x000954b730901cb5,0x000774fa3192062f,0x00047b351108194a,0x000765e32829e6a5,0x0000ca08b2eab343}}, + {{0x000c343def27938c,0x000208cee32a1e04,0x000d142da3f15ca9,0x0009bccb61573b9d,0x0000094eefdd26dd}, {0x000f1a5136bb4da9,0x00036b2f3448cd42,0x00016795e7569395,0x0007db98017cd65c,0x0000e401530bafb6}}, + {{0x000608550643d18f,0x00046902eb90a0cd,0x0007964ba821f27f,0x000ad5766f7c645d,0x000056feaa57c7bd}, {0x00029281bfda5eca,0x000e6624a7de56aa,0x000f7f3b15c09581,0x000e4b33d1a64a6f,0x000019a86e60c36d}}, + {{0x000749f9b7c6c75e,0x000d8cbca35c8376,0x000145b32680eacd,0x0006afe87fd5b55e,0x0000b20d1bb16b88}, {0x0009055779b12bbf,0x0002fbe39fb6da49,0x000a128ce6f290ff,0x000a8133ad6fe0f4,0x000009e2a418b31d}}, +}, +{/* digit=6 [{1,2,3,..,}]*([2^24]*G) */ + {{0x0004783f357999bb,0x000cd6bfacb340c2,0x000ddb9452c21474,0x0005eae3abed6ad3,0x0000b21b7651031a}, {0x000b68b8afc2a09a,0x000921306b70cdb3,0x0002eb6f51aac2f0,0x000b6f882c337185,0x0000dfe0c1b798e9}}, + {{0x000f0633fd5a1de4,0x0001e75b5b8bde0b,0x0003c617a5d05e90,0x000706bbbdb1abcb,0x00004954a8c5aef4}, {0x000eea3ff6a6e47c,0x0001cded1274fc3c,0x000abe95f6140f42,0x000813bb4b4c044d,0x000055e87db3135e}}, + {{0x000fda84e1d5d9a1,0x0005445ccdef5a65,0x000d72c0a0c0fe38,0x00013119ff360fd6,0x0000be1e8d80b016}, {0x0007e91025b45e13,0x000635259bf1b2f2,0x0001cc67e25bec26,0x00020ed7b8b4e7e5,0x0000a839aa5db80a}}, + {{0x000de9738f4194d9,0x000acb996b636fa1,0x0002053a733d2726,0x000c0c787c0ec30d,0x0000447e9cc7ecd5}, {0x000121c284773c09,0x0005115829a0777f,0x000def08b496427e,0x0007784b11978694,0x0000e7b29e72c15a}}, + {{0x0009c5550b66feca,0x0001ac065cfa1f7e,0x000d459ea5db7dd7,0x00041a3525e31050,0x0000d7abe5b21229}, {0x000ba80122d92fa4,0x00062fcc1c2487ae,0x000fd71b1066c376,0x000dbca6d767ca8f,0x0000a75fab5dcc16}}, + {{0x00018af2988abcdf,0x00007e7da51801b3,0x000836b57f887559,0x000cf3b8b9939c97,0x0000793e3b6b0a74}, {0x000008a376841700,0x000b05cb453ce191,0x0001beb2c7708823,0x0008f4ec221d4036,0x0000e5a6cceeb1b6}}, + {{0x000a0068ae66f002,0x000e98d648f0a6f0,0x000d2880f2620157,0x000017fc71776240,0x0000e0e293cde105}, {0x000f214854116f41,0x00003d5cd4eca320,0x000fc1c555e4fa70,0x000a1583fa0a23df,0x00008fcb8d2d9a9c}}, + {{0x000995e2a8ed3d7c,0x0008ef319e47285f,0x0006d98a29dc712e,0x000e35c4402eff53,0x0000a61e310f7521}, {0x0009621c3196672f,0x0000ef17e8a70ed3,0x0002a7c4929e7744,0x0006da47eca48841,0x0000011451423a2a}}, +}, +{/* digit=7 [{1,2,3,..,}]*([2^28]*G) */ + {{0x000a12ae6880b5f3,0x000d12606d252675,0x0003b21259ba1e92,0x000099b012facbeb,0x0000c50fdfbf37b0}, {0x000461c9ce223e96,0x00074efbd8ac69ce,0x00031ea8ecb90bdd,0x000ff6f657e5a4c6,0x0000584520b98a83}}, + {{0x000f317635abcf02,0x000fb9e17618b959,0x0003bd99ba516a43,0x000290ed90ccf2ce,0x0000fc6d460c9fb3}, {0x000be090cde43021,0x00000908003ba61e,0x0000f57875a3b062,0x000c2ff51bb736f6,0x0000717f6e9157ef}}, + {{0x000cf931ca260efd,0x000ec1811d22865a,0x0003e42c87d6e797,0x000158e63c692078,0x0000c9dbce95dcb5}, {0x000dc7ac8e39022b,0x0004e0ebfe466426,0x0007d6395f3037f3,0x00053975aa6845f8,0x0000f792fd611f53}}, + {{0x00021726ddc3d836,0x000f388207bb48bf,0x00003bd7ef68deb6,0x0009a4a8eae2ebcd,0x00004c7f57ef51f5}, {0x000b223a1786d576,0x0000d11827902bad,0x0005a94572e7fda6,0x000e079dc90e369a,0x0000eca838c54b07}}, + {{0x000235b032648719,0x000448b933de1d2e,0x000354c8eb4c5624,0x000c76d9c2bdda91,0x00007d743fffa73f}, {0x000109dce88013e3,0x000623b3bf4f8ed4,0x0001e761ca242827,0x0006c2900d356001,0x00004925d7e124fd}}, + {{0x00066caa81982353,0x000d1d2d7649d8b9,0x000d63f3a355d098,0x000140fac27f7ca3,0x0000e553f6d03edc}, {0x0002c7f11ff4334c,0x000c7c62cb56a4d7,0x000a2108248735aa,0x0006fdcf064294eb,0x00001f9e4577b8d9}}, + {{0x000bdbc293cd9450,0x000bea254e361d24,0x0006fb48576985bc,0x00073d3df2cb6a87,0x0000176969fcd1f6}, {0x000cacb642133a78,0x0008b73880e20b41,0x000b1463f31ea88f,0x000a27ccf1ff85b3,0x00008fffa162ca74}}, + {{0x0009b92167cdd1ff,0x0001e879b893aa4b,0x0005479f5a9118fc,0x000292f6e73387c5,0x0000adf82eeb626d}, {0x000b76156e80e6a2,0x000a127555d1803b,0x000087e4359a783f,0x000ded027d63b63d,0x00009f9ff3222fde}}, +}, +{/* digit=8 [{1,2,3,..,}]*([2^32]*G) */ + {{0x000f92d0cf4efe5d,0x0005660e2d221cb8,0x00059f07988c4721,0x000a7cca9549ef60,0x00000a3774b4016d}, {0x00095f61d001cabb,0x000053feeec1251c,0x000fedf2b2d744df,0x0004a5b7c20cc20a,0x0000f16c5f221d14}}, + {{0x0005450677b7a8f8,0x000f669273d20112,0x0002c5990ba889fc,0x0004c34a40a85958,0x000036638b474893}, {0x000e189f3596ba60,0x000ede8b0754e964,0x000f93f1b2f417c0,0x000389d883169fd5,0x0000318fe4ed45bb}}, + {{0x000e34d674d10cf3,0x0009cf962ec58559,0x0002af42d6955bb6,0x0002568bf1ab6c54,0x0000f2f33fb0fa61}, {0x000019573d1049ed,0x0009afd7f39a6214,0x000237be0f508927,0x000e57b42eb51cb4,0x0000f747f45574d0}}, + {{0x000537af0c1bb4f5,0x0000206b90f4d6bc,0x0007c89cf37f3770,0x0000034d48d994be,0x000011c2158dd572}, {0x000bac1088dda1ee,0x000a5ad4934be26f,0x000dcaf7cc3d5518,0x00073d5233c17685,0x0000c3a8a2a888b4}}, + {{0x0000fbb573c9aaf3,0x000a13cc6c7870a3,0x000cc5c2f0e71124,0x00003e3b57c921a1,0x00005b1badbfd70c}, {0x000286a9384dc1ec,0x000903a5febac82b,0x00046e3c694e98c4,0x000b10f2872b3abb,0x00008258ff96fa45}}, + {{0x000a3a880b5f763f,0x000a54ed2fd06e09,0x00069d361c365267,0x000a6292b39fb7bd,0x0000801859aa1386}, {0x0002d562056479b9,0x0004e837087f6f64,0x000a2e3c3ace29a1,0x00010ad5dac94ce8,0x0000f0d3dce1bbf8}}, + {{0x00001c95b2614292,0x0003bf1c4c90e560,0x000708120f1566d6,0x000a442e30662c3c,0x0000130cfa81a306}, {0x0001a8e4f57f7720,0x00069cca3398686c,0x000fde0e6d53c9e6,0x000b53a355d4890b,0x0000daf31990e2ee}}, + {{0x000667ab1f0c1753,0x00015ab99e7dcd96,0x000e43f9da256a61,0x000cd6ff07c1ea05,0x000005700bcae1c9}, {0x00025462b2887a31,0x0009d772fd142f1e,0x000584057dd782f4,0x000cf99125f99638,0x00009fd039626a02}}, +}, +{/* digit=9 [{1,2,3,..,}]*([2^36]*G) */ + {{0x0002bd34c58174d5,0x0003f900551f58d6,0x0002802c3872251d,0x00055506f5862df1,0x0000d93c48b2d925}, {0x00067d5bd6006f82,0x000c196ccc67a39b,0x0003014dbea6f756,0x0002c0140e853e54,0x0000bdc567509de4}}, + {{0x00058faeda49f746,0x00076e4545a39a1a,0x0004c07e4ddb8995,0x000008d3576489c7,0x00004e4b39f159b1}, {0x0000340f66b546c5,0x00041deb912eeb09,0x000e55cca0e0f401,0x000768bb00b46c01,0x00005b61b34e9ad0}}, + {{0x0002337e989963ee,0x000fb3778d59e0d2,0x0008c41782c4831c,0x000894d775c6a5ee,0x000023916557d0c2}, {0x000fe865d0eb314c,0x00004b2290d337d4,0x000cefa7842801b9,0x000fea73e9b332cd,0x00000d169d9fe877}}, + {{0x0002370734a13273,0x000c4951afa89ab0,0x000638b8aa1df7af,0x000aa6581cfbaf42,0x00009db6d2b5130e}, {0x000805bda2f91a5a,0x000f8569add7abbc,0x00021fa7d3dcb0a7,0x000008724ab65ad7,0x00005152b96988f8}}, + {{0x0008dc441177a460,0x000b7f039a7bedf3,0x0008ae7c2d9a955b,0x0008194f1525814d,0x000063c9f834f848}, {0x0004526841e87839,0x000a3a5f2b31c4ea,0x000846dcfe86a411,0x000d70b7529a3b19,0x00001356a07c5689}}, + {{0x0004dea8910763ed,0x0008d76a0f7f958f,0x00065f9b96e5fcb4,0x0000c56a5447a4ab,0x00007d863d5775bb}, {0x00034a7e87e79168,0x000a4d961e88006c,0x000fe6aeb0539155,0x000ca15def2d8874,0x0000ac350b379226}}, + {{0x00035d081443d166,0x000b06cc364b9f96,0x000bf84382342cbf,0x00029863b0a03225,0x0000ccd3ce60078d}, {0x000d10891292fd30,0x000b14073286f93b,0x00062cd16c887a31,0x000b46eb1275bf9f,0x0000335bae361578}}, + {{0x000aca7614d9ff3c,0x000cf58f2459fee2,0x0006f62abd1f13b2,0x0003749e92d83fc4,0x00001dd32de427d7}, {0x000d593c3e566e7a,0x0007a4ccb02a7636,0x00035b65281c2f4e,0x000d42b57782c6cd,0x0000d88787f28210}}, +}, +{/* digit=10 [{1,2,3,..,}]*([2^40]*G) */ + {{0x00083c6150b0fcd3,0x00011d9b0f4c85b3,0x000d05413f948da8,0x000e64cf075225cc,0x0000f31b12c7f62b}, {0x0001fc8368c17f6e,0x000a200bc68f43b2,0x000ac140e423d536,0x000d615335dd1ede,0x0000631c24a7493a}}, + {{0x000262021afa86f1,0x000396b11457a23d,0x000148d30ea757f0,0x0001700bc4d2d1b0,0x000019b553598ce4}, {0x000bb670aa9c8f62,0x000ca8e05de28ab5,0x000f1e9c3dfc9693,0x000b9c6cae7e57e3,0x0000b1ceb548f6c3}}, + {{0x0009fcb03cdcc63a,0x00026fc3afa8c753,0x00099aa9772fdb55,0x000651573534e102,0x0000ffe7179c664d}, {0x000fa10f5ecccf0c,0x0000003d6c622660,0x0007f57868df4395,0x000d741bec4f0f31,0x0000019916d7a461}}, + {{0x00035ca876361830,0x000a719d1ca312e3,0x0004161d81461a65,0x0001128150080ab1,0x00008da4ebfc612e}, {0x000fb6ba8498a9af,0x000eaa0f8db9d95d,0x000b4f497099cf91,0x00030612d2ae144f,0x0000a3a28b123cb7}}, + {{0x000577b9ec7f1ff9,0x0007936c299fce38,0x000f0939b9deaab6,0x0005c0624127222c,0x00008c2a3492b510}, {0x000a08656706b8d6,0x000bd14add6ad069,0x000d85a98a2296ff,0x000bd6537a875bde,0x0000705fef2603de}}, + {{0x00079bdc3141d130,0x0002918736859cd7,0x0004305b9e4acdef,0x000d6de804c2cc4e,0x00003c2e8392d86b}, {0x0003c44aee7c9a30,0x000a9f8d49b095d5,0x000c57bc9b303ea1,0x0006a65c3579b5c2,0x00009ac7359ad03f}}, + {{0x000cfc3afb8cec58,0x000eb630ab5869b3,0x000322634533076a,0x000a3dd50dcfb8f0,0x000053af6f4fb737}, {0x00062cefa4582dca,0x000b14554a478703,0x0004c8a663019bfd,0x00010e61487a210f,0x00005b5f361c1306}}, + {{0x000c5d00f01c7cee,0x000e5283bdefe89f,0x0001519236fc45ff,0x000cc971dece8181,0x0000d1cb14da433f}, {0x000612bd3959bcfe,0x000c15b5732e6279,0x000d0a1cae163880,0x0007f335414ca771,0x00008b9e6520c1e4}}, +}, +{/* digit=11 [{1,2,3,..,}]*([2^44]*G) */ + {{0x000a78a08b213629,0x000069fe0b35cce8,0x00087837cf37f5e8,0x000993dca66c7f2c,0x00002524b949bf2e}, {0x000020c71745788b,0x0006ecbfbf4c4c0f,0x0003de1c86018de4,0x000e41a8446691ac,0x0000194d41a0de5a}}, + {{0x000a4a89ce76a94e,0x0002a0725c9649aa,0x000e16665847cd08,0x000fc40c099e9097,0x0000409ffca6f7b1}, {0x0005b80690941ed6,0x000af2c0ee9d6057,0x00075837d8e25100,0x00022b71662d279b,0x0000eeb9e98456bb}}, + {{0x0000d52390e35da2,0x000347a9947be434,0x000cc7a3fc098e3d,0x000df6e6d781989e,0x0000c39102e43aa6}, {0x000ed0d300f3cb15,0x00072cfbe054683f,0x000cf45a8c0b1e35,0x000eda3da2224c20,0x0000be55df77f30d}}, + {{0x0006e66528960b13,0x0004084aecb383e7,0x00012ad64445dc39,0x000e371361843616,0x0000ccbecccbc831}, {0x000bd416121383c2,0x0003a0d895a2efb0,0x0003f2f1e316164a,0x000225c3d3415323,0x0000905907000d92}}, + {{0x000f0d3a3d89e152,0x00092d07f5eb1d0e,0x000e4887a4d5a30c,0x000349c359e31073,0x0000f4c6b7efbdec}, {0x000d3e9ba1426435,0x000e61c794b675a1,0x00010c63d8f4fd58,0x00076ccad091d118,0x0000b61623a40bfa}}, + {{0x000afb7b790881c4,0x00047663e76c567d,0x000e824902002b8e,0x00047a3bd28edef8,0x00004dd2e818bb2a}, {0x0000dfedbc3f2f8d,0x00033e2eec700e75,0x0004c4d2fd9b6e91,0x000be2e8400e2f1e,0x00003325697f3217}}, + {{0x0008e40347dfaefc,0x000e24464bf2717c,0x00064ec20795b0f7,0x000d3f15dc99d613,0x000007fce2951600}, {0x000338bc8bebbdac,0x000768547c02d825,0x000040b845e5e89f,0x00033d3c50032f1a,0x0000ea7544feb3a5}}, + {{0x00087934d217a2e3,0x00064fa4702d663d,0x000431f1b099e8c5,0x0003996d91bc47e6,0x0000fd21287d5f61}, {0x0000bae682fa08b0,0x0005c1ca371c2fc9,0x0001c428f51699c8,0x00050416f29d7483,0x0000ecefb669fa2b}}, +}, +{/* digit=12 [{1,2,3,..,}]*([2^48]*G) */ + {{0x000eb9068755cf37,0x000eefe125411487,0x000af8ca81887394,0x0009dc2e4c65d446,0x00002aae64629e11}, {0x00000941ec6ad734,0x0004cce4573e558e,0x000254b9684a7eec,0x0007323d6d00d4f9,0x0000ef44f58ce421}}, + {{0x000e0f87d3ad2ac1,0x0001d0f92c5ce59d,0x00000b6a5d2670cb,0x0008eb8f05944ac9,0x00001aeed239d966}, {0x00038e47c488ea6d,0x000848083d7451b0,0x000d31fe2406ea3f,0x000f9ad22197b43b,0x0000c8f8ccbf8a6e}}, + {{0x000cfbffc9188216,0x000c7e0fbf5fd2b3,0x00062452f51cb7e5,0x0009f0a73f6716bf,0x00002b74252c05f2}, {0x0009d259e88b0f1b,0x000175841f6142f1,0x000378ac867525d1,0x0002d6e54c3b5661,0x00009fade95d9a45}}, + {{0x000648302887281a,0x000c1f9de66ec79a,0x000a7dee213f3d39,0x0008b8289c3c50f1,0x0000510a53ce0b69}, {0x000c3fb06f799adb,0x000a653746501566,0x000c947b2cc95a87,0x00051bbd7343c061,0x0000bbff69e4543e}}, + {{0x00041aa7a0a745a0,0x000243371bd1327a,0x000b5f82492d43e3,0x000478cde41b9031,0x0000f5e1e18cda04}, {0x000c186247853141,0x00025ef5518b5db2,0x000b7c8ee0c4cfdb,0x0009d7acc9f6ec1f,0x0000efb06595d6c0}}, + {{0x00038dcba75aba90,0x0006d150f881b80d,0x0007b47c5e9b61ac,0x0001289f56af52ca,0x000040300d977fb3}, {0x0007184c01fd0c1e,0x00056b6e134a5687,0x000d7165340112a0,0x00062e56daed90cc,0x0000c553aa7974bd}}, + {{0x000472aab109ef7c,0x000d759b268a33d8,0x000314e79dbdfe41,0x00077c147dd06eeb,0x0000ae5583823c5b}, {0x000d83c8b1c89525,0x0008f1ba0321bc8b,0x00098beeecf4b71c,0x00034268f32934ea,0x0000580dbb671a4b}}, + {{0x000381d11476ddee,0x000dfea0923e2398,0x0007745bd4959204,0x000467d67427ad01,0x0000f022a754935e}, {0x00099f524e0380a0,0x000fb5f1a73057e7,0x0001771d86ee2b64,0x00013c9aeaac4852,0x00002c8521c0992a}}, +}, +{/* digit=13 [{1,2,3,..,}]*([2^52]*G) */ + {{0x000c43b153adbf74,0x000fc0351fec7fe5,0x000109e6007a66ed,0x0001623d8042353b,0x0000dc97176e832c}, {0x000c75fb1db1e5c9,0x0007315b98ff73fe,0x000aa1cff6aa02da,0x000a265f9e808f4f,0x00000aa28be36412}}, + {{0x000f1c99ce0fcd68,0x000eb232a8468dba,0x00075d822732ea65,0x0003cadb2ce2186a,0x00008ffd479bd227}, {0x0002ad2f2f26b61a,0x000e086706aae909,0x00023597cfb7041b,0x000c84e3d5fa755e,0x00002035bf8f995f}}, + {{0x0006c1ab9ac4754c,0x000a0d3f305a1b45,0x000a62dc0f497e6e,0x00094f84d27e3a3f,0x000018c3569e524b}, {0x0008954e380f5cb8,0x000ca72ea4581219,0x00032849181d8221,0x000ccf6fa082f65f,0x000010ca5b00e304}}, + {{0x000b892313de667f,0x000856a478a7c5de,0x00099242875c872d,0x000e09b67b5513c4,0x000097e010fe70fd}, {0x000f05360ee268c1,0x0002067cd32139b0,0x000bc187c981b514,0x000da8b5a1ac8d4f,0x000062417e2d12e6}}, + {{0x000e262502e26d19,0x000cc294e23f905e,0x000a7733db961ef9,0x000f737178f1fb6d,0x000089b69fbb32ec}, {0x0008a359a9bccae8,0x000bc00a01f33727,0x000b213bab1c81a0,0x00017e0781855aa6,0x0000acc1b78b4298}}, + {{0x000d3a83016e5135,0x00004b0f7bfadbde,0x0002be716a3c508b,0x000b9fa6a490deaa,0x0000a04d9e644485}, {0x00099d16ad25b5d0,0x000965a72cb4d07b,0x000c45a95a184010,0x000cffc8e2b32d14,0x0000fae6e86e4f2e}}, + {{0x000115e55a0c9314,0x0009e5c920a3d764,0x00090371034ea18b,0x000c7a6a099ddcaf,0x0000b937dc2249f2}, {0x0004a1a430f0a7e5,0x0008921dbe965cfc,0x00011d3fe8f106a5,0x0003c548ac702618,0x0000484226b080f1}}, + {{0x00050f8849e6d32e,0x0005de16485df445,0x000343924e7bc29d,0x0005ce29bbfec62f,0x0000eb802f320f2b}, {0x0007542bbb64f333,0x0006cf9bdb3bfb33,0x000a1cb884c1d3a3,0x000b6e1067cf3bc7,0x0000f12a31d7601f}}, +}, +{/* digit=14 [{1,2,3,..,}]*([2^56]*G) */ + {{0x000a78f1f8a4a913,0x0001a53dbe738720,0x000d0991a59e2221,0x00026f9f5ad99cad,0x0000a0db802a3807}, {0x000761c7dfb4f1c5,0x000aaac819cce7f0,0x000ffe34868e7098,0x0001409683d61037,0x0000bf205e57b7b5}}, + {{0x000b5f661a972624,0x000bd74a82f75846,0x000439654edf2cac,0x000e093dfab85faf,0x00003fb0efa0724e}, {0x000016f53b0119a0,0x00035bc8fc81d0d5,0x00065d2986844536,0x00064f6d10b6491f,0x0000f3c88c621a4e}}, + {{0x00072a17f34c5177,0x00069378bc26c203,0x000aae0245602bd1,0x000daa666a592d91,0x000016886aba17bb}, {0x000fe68e30451039,0x00030de1d7013e55,0x000724cb6f2c4b0b,0x0003148da358857d,0x0000aac623d2ec47}}, + {{0x000ff46b8529a018,0x0002c856b95c024c,0x0005af7f76e4d82a,0x000e5358c6b833c6,0x0000a6c41262e110}, {0x0007c304f0833403,0x000b476cdb310820,0x000882de171aa384,0x000e4a1ada294142,0x00008b1ad2ef16a2}}, + {{0x0002720142bcb30d,0x0004caf604d0edda,0x0007963145617526,0x000d19086189c1e6,0x0000ab01c692b04d}, {0x000e4b0ba8ed3c18,0x000c6281acfb4e54,0x0000a6319f616513,0x00087baf1796295e,0x00005e79ac9b28b5}}, + {{0x0004588c9fd7da08,0x00009238d0c391d8,0x0009d6475a78682d,0x000e18333ddde082,0x00000c88440e9de9}, {0x000f21ac6d8176ff,0x00059509d46f6d15,0x000bbfcd5daff943,0x0006880191bb0a8b,0x0000f7732b8f8fc2}}, + {{0x000772d5ab3d89e9,0x000a2a786c9072fe,0x000323866f1580ec,0x00096f8fd834175a,0x00003711d4a4adec}, {0x000c34a6b9b4a308,0x000138b0de23a020,0x000de3ce5bf10e00,0x000c632a5f298d28,0x000007a398f7e1a1}}, + {{0x00040cd73f7c45c6,0x000d6afe059c3fb6,0x000b168d4eb1f87a,0x00003fa3c3979a52,0x0000eef460cdb1e4}, {0x0003e502724bb3f3,0x000c29d922d13d94,0x000538b4a53f3f1b,0x00045f547e7a03cd,0x00007631e2102c41}}, +}, +{/* digit=15 [{1,2,3,..,}]*([2^60]*G) */ + {{0x000bf06970e164d1,0x0008cd3d3087a7c9,0x000e7c899c27a88d,0x0003718a37c9cdf4,0x00008494d5abb411}, {0x0002375d9d8b29c0,0x000c915a2f740653,0x0005acb02b92dd45,0x000a898a23f6bf51,0x0000e69248c435bf}}, + {{0x00054fd6113a9f82,0x0004070e67e63614,0x0003f2feb78ebe73,0x000f368764360b90,0x0000ba3b3d8b7902}, {0x0008cef87490b8a1,0x0005005f31b31880,0x0009b4f4db117595,0x000e60bd5d60056c,0x00002b13fcaed254}}, + {{0x00074449a7a4edea,0x000695cfd20f1f84,0x000b33b64b26b1f1,0x0008c9f380ed7d83,0x000021f9564dd199}, {0x000c7d3a720e347f,0x00057bdf09d4e985,0x0009476929807897,0x000385a1f34ce2ce,0x000069e6145e419c}}, + {{0x00071ef16f55d400,0x00089bb45ea57fc0,0x0005543ca7019543,0x000c1983cf09f2a3,0x00007e91a8420554}, {0x000d70162a9d06ef,0x00031044a66261ec,0x000423dd900e14c6,0x000b4cb1317c1300,0x000049431bd946ea}}, + {{0x000548a60bffa601,0x000b7e0c4e341ed5,0x000db5bad903708b,0x0008a5cbc42d78a1,0x000052ad7377be14}, {0x000cda3ef8265d6d,0x0004490c675cbb3f,0x00033d235cfcb34f,0x0003cb0c2954b5d3,0x00004a7d16a1a320}}, + {{0x000dffbcdc0e5cfb,0x00040f611432a75b,0x000c16d43eeb8cc1,0x000997e3b489f974,0x00005c174e0722ed}, {0x000a9b6afb9c1d83,0x000295cb574a38ce,0x00015c4a8da546da,0x000307688d2edc0e,0x0000ffc898fef45b}}, + {{0x0005f1be45b69cf1,0x00077a5a8c531e35,0x0007d0264570bb79,0x000a61a0a43c059f,0x0000cd74a04320e4}, {0x0002e876cf3ac0a3,0x0002b212cb77fc35,0x0006c032db7672e6,0x000bc2045a0ba3fc,0x00003e7bc4433ad9}}, + {{0x0008434d0614aa12,0x0009f14184341511,0x000c15b898bae977,0x0002ebf5641d82b8,0x0000383af5661643}, {0x000d3f02c73f990c,0x000f66bbdc7ce552,0x0005ec6348df82e9,0x00053f0f336aa8d7,0x000042e3b2dd603e}}, +}, +{/* digit=16 [{1,2,3,..,}]*([2^64]*G) */ + {{0x000e020bad830d23,0x000e890dffb31b33,0x00080ecb05c101f9,0x00093ecd0e0498bc,0x000002787f882aa2}, {0x0004ced220f8fc84,0x0000fe0ee3777fd6,0x00013b128cf5cebe,0x000279dc03a03889,0x0000b0969723de23}}, + {{0x00089aff727ef3ad,0x000603db88fa9c53,0x000ae077795ffeb9,0x000f0227cb70429d,0x000020afe82316db}, {0x0008e18914bf7060,0x00030517cd090fab,0x000e406443b1e66f,0x0008fa24b46dce12,0x0000ff1016808f2d}}, + {{0x00034c883dc54470,0x000b04e4e9a0e55f,0x0009166a2be76243,0x00080a78fb4cbc81,0x0000bdfb703ae37f}, {0x000288ec217cda8e,0x000242af41561869,0x000ae9d302662bb7,0x0003c9ce64f29150,0x0000e0d4441bc035}}, + {{0x000aaddca9a95262,0x000a89a303e9913c,0x000a50f3c701c63f,0x00057fb97d667ab0,0x00007c03d7c88e65}, {0x000e712eb1056070,0x000d8dd86ccbab24,0x000986d684936aed,0x000a6532196f8a0a,0x0000307b826248f5}}, + {{0x00058b5ebbe949f8,0x0006e982215ad88f,0x000f863c0b776229,0x000f513cc83dd6cf,0x00001ec094d81098}, {0x00069aabe0432d02,0x000c8455957d014d,0x000dc1eabe4e52a9,0x00016f94743ba8fa,0x0000c395d97a1763}}, + {{0x000ab7799458b24a,0x0005ceba3064364b,0x000068f036fe19e2,0x0006559aabd83d74,0x0000ef812190fdf8}, {0x000d27b65593fefe,0x000edaa457b21506,0x00003dff40a1ad85,0x0006e3266d0f06a3,0x00008114f4f8bb41}}, + {{0x000e2d38fc2ed938,0x0005f8f0f8582acf,0x000a35ec48344664,0x0009b158de6f09dd,0x000091e5ece778c6}, {0x0004ba991c13d675,0x000d987d5575af4a,0x000303a7e6e78063,0x0007ed226b621e8d,0x0000c9bc10419598}}, + {{0x000581a88a45b65d,0x000138e58c1b905f,0x0005bfe1ccb78920,0x0001d714cbed65bc,0x00001af7dc8c02b1}, {0x0001bc0cd3a7cc82,0x0008c01a77b79834,0x000e2a4028e9aefe,0x0008598eeafe875a,0x00007a0698fe11f3}}, +}, +{/* digit=17 [{1,2,3,..,}]*([2^68]*G) */ + {{0x0005759c85448853,0x000aabfcad786b80,0x0008fe78ebe9b99c,0x000bcfd1db36e12b,0x00007255a2e05387}, {0x0003a3ea150ad323,0x00039671ae58d44b,0x00052384bc65bc2a,0x00030041ce078e1d,0x0000115f1b1ce72c}}, + {{0x000517480eed542e,0x00021362ef7e34a4,0x000dd0396add6456,0x00038b39228bfcc5,0x00009fdf904ca0c5}, {0x00091ec74d235de0,0x000824aa0a476bfd,0x0008d616896ec237,0x00060bf5699241af,0x0000a7b9be3c548a}}, + {{0x0002854f2b4c5818,0x0009ca26223a3e67,0x000aa188c6df8d84,0x0004ebd1debdf2e3,0x00000bfdc0a33d8c}, {0x00049164502ad485,0x00013c4eec044436,0x000526b7ad1ce82b,0x000ec1b7c386c272,0x0000a6cc4f4bb8de}}, + {{0x0005c302ade9556b,0x000c25ba2e9b3525,0x000ce9e47e328af1,0x000cc99d3391ef41,0x000074cd669ab0ff}, {0x00003e4e3226acf3,0x000d02959e429671,0x000d490fca65ad22,0x0003563aaa840699,0x0000e26a1c2cecc6}}, + {{0x000c3180561c002a,0x000500356bad1178,0x0006c78568e636d7,0x00001af8d07145e3,0x00007730567cae79}, {0x0005ddcb77d9db90,0x000989d16e89691f,0x0005142a2f824b7f,0x000bfaafb5c9b7eb,0x00009b34446f6015}}, + {{0x0000be45824c88aa,0x0001e847b35a949b,0x000027aa0e3a35e8,0x00082301594fd284,0x0000aff184b8e873}, {0x000e135810fa4740,0x000a8f7398d7d5fc,0x00096a1f3aba6165,0x000924e3aaa99a8a,0x00006ff0fd3b6295}}, + {{0x0008b7dfddda1bc2,0x0009d5f713ecc26b,0x00012edcbc93512b,0x000095029ffeb33b,0x0000ea0649923564}, {0x000b5a0fcd32167d,0x000a0c2eb30bde37,0x0006cd0182008416,0x000502b2663d607c,0x0000810c96305b2b}}, + {{0x00075de4e92defc4,0x00041008988331c9,0x000c0919081aeb17,0x000d648a30ce4a2e,0x000026e7838a9a36}, {0x000a0b6309bd2d7d,0x00000cc1a4ae8899,0x0005163b33b1c24b,0x00086db2aa814234,0x00002ad9a69fc78c}}, +}, +{/* digit=18 [{1,2,3,..,}]*([2^72]*G) */ + {{0x00033c6ba4e4651b,0x0000b714d4ac4672,0x000f1f8da8cf9566,0x00070ed70decc371,0x0000674732b20783}, {0x000416d4ccc773bd,0x0006ae87951d7270,0x0003dfb56c848ff3,0x000110a61506a849,0x00008371eaa467eb}}, + {{0x000ee2fda3adfe0b,0x000cbc964e7d1066,0x000859476ce6a9bd,0x00099c04a0115b0c,0x00005e02dca4c956}, {0x0001f6211377eb91,0x0002272bca2ee830,0x00047cf1f57b245a,0x0001c1a7d9b4707b,0x0000b469bab2774b}}, + {{0x000fa67dfd6586eb,0x0005cba23faa1dfc,0x000ec5d6f358953e,0x000b2a0f467275ae,0x0000815967b0b0e6}, {0x000f133012b89b41,0x000c7839cc04a01b,0x0000dfd73dd924bb,0x000098a5cd218012,0x0000abb11ef29bf8}}, + {{0x0007889f1ef22320,0x00019c7409a50397,0x000004e03be2c82f,0x0003a335ac44f932,0x000048bb35985682}, {0x0008d6cec1cf253f,0x0008603eb1d13e10,0x0000ac34de98e74d,0x0007facaf64f6057,0x0000f814e7e3d779}}, + {{0x000b2eb0d1b96aa9,0x0003ce52ccf0cdee,0x000d534ef25783ce,0x000dade4e251f429,0x0000fe9693de5797}, {0x0003d69c6ab9935e,0x00095721ca8d8a17,0x00093eb262399131,0x000a9e38cbcd30c3,0x00003627aba51d95}}, + {{0x0000e575ef764b4a,0x000d5e73c69849b6,0x0002be1ee61ad29f,0x000a7118489017dd,0x0000c95e3b71febd}, {0x00053465ac960924,0x0003ffc835666698,0x00072fa4a906265c,0x0005401692c81259,0x0000e76f87940773}}, + {{0x0005c3ea7b82f5c5,0x000f3a98adfad6a4,0x0004b182c494694d,0x000b4144a06ec313,0x0000570befac5b82}, {0x0006808129f952e9,0x0000214f87c3f19f,0x0007be905a6fad26,0x0005c2831e56687c,0x00003107f397623a}}, + {{0x0001bcc89e35108a,0x000433f1cbaf57cd,0x0005b13ac924efa4,0x000a1de3716559f3,0x00000a88e8800370}, {0x000be0a8c286ea33,0x00069ebd50c6e203,0x000284d0897fc5ab,0x0000e02b5b360274,0x0000055716f994a2}}, +}, +{/* digit=19 [{1,2,3,..,}]*([2^76]*G) */ + {{0x000a63d5b99708d3,0x00077b80189a4d67,0x0001c402efb29bef,0x0007673cb7eeaa24,0x000028cb6de5c5c2}, {0x000a7b49cec231d2,0x000c2e2e6a7eed24,0x000f17b13725955f,0x0001cfa2040cfab7,0x000015eff8dc25c7}}, + {{0x000cab1c0d41a942,0x000abe60f7d4c4d9,0x00036116cc38b202,0x000f132bbf6b1793,0x0000f9aa8774e068}, {0x0000627a4bac9fd4,0x000eca593cb4b882,0x0004179312209cb9,0x000fbfaa78ec63c7,0x00002d21251bcfcc}}, + {{0x000fb305334d7b56,0x0005a11f7073b233,0x00097794e41b5755,0x0000dd099a53713c,0x000056ead5a9a5f0}, {0x000696f1e3511dc7,0x00070e5e30fd2ece,0x0004ace651828a64,0x00061ce007b8ff60,0x00003da66390b892}}, + {{0x000e5ae3e6119407,0x00097aa090ebd0ce,0x000926e424e2d9ea,0x0002a373d167ef1b,0x0000fff36dfcd511}, {0x0007745caffa3fbe,0x000fa5a835034558,0x00047bf2a224f7f4,0x0005c65ceff0183b,0x0000d9bfa74ccfab}}, + {{0x000c13c73f076b32,0x000eaa3baf1a5581,0x00086b1ccf225af7,0x0005b36fa0d82d4e,0x000009f416823464}, {0x000cfcb2fbe643e7,0x000163c46a73f613,0x00019477d806abcd,0x000c02d04bc1028a,0x000068888cb668be}}, + {{0x00070b8d9b429c9a,0x000765ad81cb53d5,0x0006825bd6946011,0x00036a30833a082f,0x0000297122b1c99f}, {0x000b84805c3abdf7,0x0008cf2e24b1ffc9,0x00015d922efe9529,0x000726f045275a89,0x00009146aab998a9}}, + {{0x00051d89cdbd4b33,0x0000e1195d208087,0x000c6618eae8de7a,0x000cfbdb08e2fc12,0x000097fc24f3fc4a}, {0x0002071844eeaa70,0x000b2c004c259869,0x00035fcca2b3e1e6,0x000be4a1211da1e8,0x0000c881208c73ff}}, + {{0x00059054f831d0b7,0x000c9d47d4fd9c7d,0x000042e12faaaa26,0x00077d5ac2859985,0x0000eda370be7969}, {0x000d71d95c0be637,0x000cfe8210052f0b,0x0002ffae97c4601b,0x000173f1ecbc604c,0x0000e3efc580b688}}, +}, +{/* digit=20 [{1,2,3,..,}]*([2^80]*G) */ + {{0x000ca15587feffa0,0x00007d69e4ad8348,0x0005a0745585d074,0x0004dd6fbe561988,0x00004ee9ebab10b2}, {0x000075c0f4c12d76,0x000c9c51c604fc27,0x000e336d0acf4acd,0x0001d2782fa52bfc,0x0000e1d078fa8362}}, + {{0x0007a1bb2b806a85,0x00010ad5d0f42438,0x000a0e9e414efa30,0x000decee7e442123,0x000004ae4288b6ab}, {0x0006fcb927b1aac6,0x000c923b71d358c0,0x000d6dae155e1c85,0x0000b8f47e180f48,0x0000d80dd64084cb}}, + {{0x000c3819ec64698b,0x0001cef36dd1f0db,0x00056bdfd770e4d1,0x00094701d179158d,0x000048153ce25eb9}, {0x0007a54fde983909,0x00079fe2d6fc31d1,0x000a704f10817eaf,0x00018244a63591a6,0x000042346600f554}}, + {{0x00086430d7fff59a,0x000407bfe742e8a6,0x000ce06f982b9219,0x0001341a8b86cfc2,0x000038414a199ad6}, {0x000261028e2c39f1,0x0000580856a06f10,0x000c3034d34805c2,0x000c9d1b3f930218,0x0000713f457674c0}}, + {{0x000c8821b96672b1,0x000b479d1f5709cf,0x00069a5142759c76,0x000413a6cc7a9822,0x0000bc2015b783b1}, {0x00011781bf4be622,0x00059bf2b0bea43b,0x000591cfdd294197,0x0004951eac3587c4,0x000083169e62e66d}}, + {{0x0004800da0723bca,0x000093c8381d3859,0x0008ca980524452e,0x000e56846dfa0213,0x0000a77a80d82d32}, {0x0003fbc419c86b5a,0x0008570216c28757,0x0002036e6e748680,0x0003d88b7a685ac7,0x00001764627b5fae}}, + {{0x000d0556b1fb1041,0x000441fe146f9ea8,0x00063c4132ff339a,0x00097b3d7ef85bcf,0x0000f0f57c5389a0}, {0x000f83b5bda1160f,0x000e5fea66e792f2,0x000e11651433eea4,0x00043b1f3fff4fca,0x0000a71c3fe5b1c2}}, + {{0x000edc2023cb0df4,0x000e15a245918b52,0x0002a907208773a4,0x00086e0d9a6aaae1,0x0000261f56f9bf55}, {0x000040260a081069,0x0004409cfa60884b,0x000dae4831b39805,0x000cdedf7f55b1d5,0x0000554210f16ef2}}, +}, +{/* digit=21 [{1,2,3,..,}]*([2^84]*G) */ + {{0x0006e859204db305,0x00087aa84cdf064d,0x000476456139bb92,0x000a669413d7ea88,0x0000c554483aa1ff}, {0x00030892ed180808,0x0001514e5daefb86,0x0005f81ca589aaf2,0x000415eee4f96f7b,0x00008d470079bb0b}}, + {{0x00000d355c9bd11b,0x000cb6fc28506bb4,0x000063b3e8402465,0x0000c6a81ba22d65,0x0000ab2dcbd1e1aa}, {0x0003f1abe645e253,0x00037df84be1b5f4,0x000a2eaf46232053,0x00026f14ac708021,0x0000f94646488beb}}, + {{0x0003e9a7a82d20f0,0x000c191011f25f2a,0x0006ac8e6399e015,0x000f96fbec312a88,0x0000dd5140aeda47}, {0x000f31326b47318d,0x0009b6685ec73d4d,0x0008442cde2c9ec7,0x0001cf4df119aecd,0x0000b1ca9564b32a}}, + {{0x0005852126506ccb,0x0007b8b3567cce2c,0x0005a3f24ba94aac,0x0008b36905cdf4c0,0x0000f5f559be547f}, {0x0004e62aade7a1df,0x00070fda3087ae4b,0x00043d89f56b8b9d,0x000047ea3eb4c64c,0x0000b7e537d8c69e}}, + {{0x0009491dfe5f6ab4,0x000e01a9c0af823d,0x0007d2b3542fc362,0x000aeb04170b0112,0x0000f0f17bc44116}, {0x00001dfc9184cf6f,0x0008795ceae6816c,0x0000bff2e914dc87,0x0005db696b2ae839,0x00006ccd629e88af}}, + {{0x0009bb90f88095a6,0x000ff19ce3057ada,0x00078b2667155c28,0x0004f832a01e476d,0x0000da9445a1652c}, {0x00083a6827ea8efa,0x00075db1af2b0317,0x00031dab94d69b7c,0x000ace2874eb38af,0x00000ed99114fd9b}}, + {{0x0002e3a4037f17e3,0x00018f91a4fa4d89,0x0001cf02f81fa984,0x000c7517c7292d96,0x00005af0c0e688bc}, {0x000ec90127a29b07,0x0004ad087444c40b,0x00087c273955714a,0x000da2fd430880a5,0x000015ecd51424df}}, + {{0x000066daafd6cefe,0x000fd8c1decb6ade,0x000b96ecece59c8d,0x00010c3e12a24a77,0x0000e7c32fd24cc7}, {0x0000e4f240e9bb7c,0x00052a63b06db070,0x0009644ee837ada5,0x00051ca58ce980d1,0x0000aa5d14de7e74}}, +}, +{/* digit=22 [{1,2,3,..,}]*([2^88]*G) */ + {{0x000d50cf1edd62f5,0x0000229f48d97a0e,0x00072ae88c3c7ae7,0x000666ff47bf288a,0x000084ddfe5848c6}, {0x00037e936731fdf9,0x000c18d98bc79711,0x000a6be30714bc7d,0x0006a5cea912c10d,0x00001cb844e4e62d}}, + {{0x00093634626e343e,0x0009d4e4c8f9696c,0x000648a06f699957,0x0006e5ce7306f6b8,0x00002775c8d8e799}, {0x000e678bf09d221c,0x000f115c2acdbb47,0x000b48b41f5251e1,0x00074b087f912177,0x000040e7726ab38d}}, + {{0x0008aaae4fdd396c,0x000f9371c3f3cdc1,0x000dfefdef6e8a9e,0x000577c6b58042a5,0x0000cc3bbb7bc4f3}, {0x0003e4adedfdd556,0x0004148c5fb23f58,0x0002d61e09ea4513,0x000b38ca2b322923,0x000042101a910b5c}}, + {{0x000ab3cc064f530b,0x000f8e1758919b7f,0x0004c4e60731153a,0x00013a335e65033d,0x00000876a8b276ce}, {0x00066ee22241ecd1,0x000e111e861c98a5,0x0007dd490b7456b3,0x000c40a9aff4eb17,0x000089b1ed9d8f77}}, + {{0x0006f5a394d7786d,0x000dd7ff0605880d,0x000dbaf4d3e66b3f,0x000f09abeb798da3,0x000055a81a432168}, {0x000e4853cae87841,0x00051aa0e48c3e00,0x000e17bcbf4bf671,0x0005a084ce2b4df4,0x000010852e0a41f1}}, + {{0x000666f50956c896,0x000bf47304e9abf6,0x0001770f44938548,0x000bb75c17622116,0x00005f7c0b74c493}, {0x000e3303459a009a,0x0002d40ab80bc6fb,0x000f5d3096dbed19,0x000e7a5fc682575b,0x0000ba698d48d53c}}, + {{0x000db6a830ae3327,0x0006ec5c4f57bc5b,0x000a40ac4ef38847,0x000d07b5329ba2d7,0x00004b55633b22e2}, {0x000719d1f38f3c87,0x0006dba25c7f63e8,0x000c8bb16e712c08,0x000bbc9fb85a3b64,0x00004ee16f039ef0}}, + {{0x000e6872857a1fc7,0x000c9ff8f504f24d,0x00081bc9abd0a0d9,0x00041beecb4fadc3,0x000023862936a94e}, {0x0003f83e75fc753c,0x000467a5a6be754d,0x000f568dc06afc75,0x0006471ce792eeb2,0x00005faaee47d2f9}}, +}, +{/* digit=23 [{1,2,3,..,}]*([2^92]*G) */ + {{0x000beb0f912b74fe,0x00024e0ceedc375f,0x000233ee745fbe8e,0x00026ef0e1aa68d9,0x000055fc1cf206a6}, {0x000a1b9e08712e7a,0x00065cfd635f80ef,0x000c1edac5fd108b,0x0003f1ea431df6ee,0x0000e1c052234080}}, + {{0x0007fb664bc0f09b,0x00027953cd07f8e9,0x000c3bf000b91399,0x000d8f8385a1b37f,0x00006e74ded609cc}, {0x000f026ec473ea77,0x000ec30766bcfe1d,0x00092052bf2f7fbb,0x00066af18cb47a32,0x0000f8d459301148}}, + {{0x000d957b3086691a,0x000bc6640915a8d6,0x000db59a99946a29,0x0001ba932ca93c43,0x0000a61a0c684fe9}, {0x000e112815bf0033,0x0008f86ba8d36e22,0x00069f434a9ed1b1,0x0007541b5d3c1410,0x0000cd2ebd04cc01}}, + {{0x00007cb8e419e083,0x0007df855eebfb75,0x0008683a5785328d,0x0007e9875db0c77b,0x0000d1bc968c0a59}, {0x000fa1047eeeab4d,0x0004a680ca70ad4a,0x0003650232668dd4,0x0001ccc3210d1f17,0x00005bb2ee4e7fb3}}, + {{0x000156fb22c5f89f,0x000baf786759c0c3,0x000b8ad9fd10ece4,0x000420e270e31780,0x0000e7a6211a0c2b}, {0x000d738125ef6358,0x000741a6f2027091,0x00087d9bbf1f277d,0x00084be2727e7b35,0x00003b209aa33e2b}}, + {{0x000bab750f2b65b6,0x000bbe5d53e3b5b9,0x000529212a7403d4,0x0007672e23e37628,0x0000fe903a2ce050}, {0x00091a16cf570fb7,0x0007ea30b325dc52,0x000c572a94bfb860,0x000c31ec4905f827,0x00002eeb8c97f381}}, + {{0x000502e9c14b6997,0x00067948a290b6b8,0x000efd415f1bcdca,0x000fb573e43a322a,0x0000f52335841e2c}, {0x00051c097d3fa948,0x0008f21296302601,0x00054acf5820c0d5,0x000ac3b8f2e1ed58,0x00006d6646cbc656}}, + {{0x000f6acf96ec2b85,0x0003f961cc16524c,0x00017f0a2d06747c,0x0002d657c7001cbd,0x0000f298db084afe}, {0x0001ef2df12f671a,0x0006fce712fdb1b0,0x000a74776c01c506,0x0006bdac0f403492,0x00003e9934fa8d69}}, +}, +{/* digit=24 [{1,2,3,..,}]*([2^96]*G) */ + {{0x000b8941abd31f02,0x000dba1da7d32599,0x000f0217ddb34198,0x00084ea8b89523a0,0x0000014cc44056b8}, {0x0004f8849efd4ee8,0x0000a87f4adfefb9,0x000fd2debf1b8171,0x000a5389d38a9a99,0x0000179277af2b67}}, + {{0x000ee9e2d3a6e28e,0x0005486d8ca2fd5f,0x0000834918eed56d,0x00045736889a2778,0x0000a1a65569ef20}, {0x000b609a501e2a36,0x000a5b23de2f87e6,0x0002c9a6b1ea0ae2,0x0009615f537d0763,0x00001770d1ffa3db}}, + {{0x000d1fa9047d3c68,0x00041e2d51ce4e21,0x000b10c9c847d157,0x00062380167a634e,0x00008966868f8c07}, {0x0008f2cdc401ce66,0x000ac97d1c838e87,0x00062b7c27f46062,0x00036e04a047da68,0x00008a2cacd5afd6}}, + {{0x00060db7497e8a5a,0x00092c6949a8925c,0x000bd589805d8eab,0x000e3f3169e466c7,0x0000dc06264b92d8}, {0x00068f4d55959fe4,0x0000602dba18dff4,0x000cf84bb97b33ee,0x000176aa0c3fe221,0x00008cdc0af44a8d}}, + {{0x000730ec599b822b,0x000e71db1fe734f3,0x0008091cd821b583,0x000cffa5b97c09f6,0x0000eb36d9557afb}, {0x000e9729ef451b2e,0x000cc72474ce4e71,0x00075c396c20a694,0x000fa4d30f6220dd,0x00000b129e07fa41}}, + {{0x0007a8b81ba9b34a,0x0005d196b5125a9f,0x00076f535e32ba94,0x00063f93a571c824,0x0000fdf2923b8108}, {0x0009a3c16169f605,0x0002eb5c7c3503bc,0x000d93e032a0397f,0x000ef4f1059d5016,0x0000e6e9109888f5}}, + {{0x000a9ddaaa05d08c,0x000b651b5a2480ce,0x0005d54b4d0caffa,0x000009d63010c858,0x0000852d0347e35b}, {0x000e50ef3df425c4,0x00023f86f10c9508,0x0001059ff9e45970,0x000ff4e36cccc208,0x00008d3f15023989}}, + {{0x000c4489c0d40099,0x0007be24dba9c3d8,0x00062e5f1d371461,0x000c8add92e7309b,0x00007b344d82922c}, {0x000009b0bfe3e8fb,0x0005a3c82268916b,0x0006235555687383,0x000e54f82a980fe6,0x0000027ecab0ce68}}, +}, +{/* digit=25 [{1,2,3,..,}]*([2^100]*G) */ + {{0x0002c86fbd613c3f,0x0006b366f179f698,0x0005c2343f4af69d,0x0001168683eed6b8,0x00005bb244b14bc2}, {0x00074bcc9fc77d46,0x0001cf44b54b397a,0x000997e76e202eb9,0x00037e77886412a6,0x0000996c8fb62c88}}, + {{0x000371c902c343de,0x0009ba4fdba0d2e9,0x0007703a1f57b2de,0x00020343f9afa4b6,0x0000afafb42d79fe}, {0x0009dc9f649a4940,0x0009fe378232fec9,0x00084e31e14799ef,0x00087fba3f811471,0x0000abbb815dc0e9}}, + {{0x0009ad6e069de1c1,0x000a8ee09300c258,0x0008ae49b3fe8e67,0x000a905b8818602f,0x000086a1482363a6}, {0x0009b399b35b33a4,0x000993d3411d8607,0x0000b2f996c38d78,0x000d08743c446272,0x0000d903dd769e14}}, + {{0x0005efcb4e31aa48,0x000098b26adf5408,0x000a7400dc760aec,0x00012814c1f78e76,0x00007b8acedb17fe}, {0x000582ecbd85bb42,0x0006a6adc0412433,0x0000d578158f0142,0x00082f3596dd508f,0x0000e7f3b803a31a}}, + {{0x000bd80d6d7878c9,0x000ec4ff7c7505c9,0x000ccd599e1ef20e,0x0004f93fab197ad2,0x0000e480594252ef}, {0x0007fd206ea34109,0x000504fa7dd4d977,0x0003bb5fcb028045,0x0008a8641b6860c4,0x0000f359d5c7cd7b}}, + {{0x0002c379ce084523,0x0009d3a870244ff1,0x00049dcb63dd8dd0,0x0006de61ff0d39b8,0x0000f5eab872efad}, {0x0006886251523f97,0x0002cbe2135b4614,0x000d2c5d445ac1d5,0x000e726350799541,0x0000f19f79a23064}}, + {{0x000044ae1118539e,0x000d3192a0069335,0x0003a2bd4ed6d43f,0x0003a51a8bf622a4,0x0000fec3fb7cfa9f}, {0x0004bbc6d834bde9,0x000940fbc743dd22,0x000ea1652aaebfcb,0x000816383b2bfcc4,0x0000cd26d91051ae}}, + {{0x0002a29161f6475b,0x0009ac8797ea7d83,0x0001b609c435b8d7,0x00000466bc156dd7,0x00003dca799ae0c2}, {0x00047bf02fd92d7b,0x000978b03082945d,0x0009a7c511d1c979,0x00089aca46d98d07,0x000093f286ccafeb}}, +}, +{/* digit=26 [{1,2,3,..,}]*([2^104]*G) */ + {{0x0002d2d4919affec,0x00046009bd158037,0x000a62eb12d4a2ea,0x00012448f1e7f8c1,0x0000e083726187a1}, {0x000261be7ebadd32,0x0006257b93e9c874,0x00039e6ff7ca5c15,0x0005c3e8b381e5fe,0x0000d16b32ff3081}}, + {{0x0000b28ad2e996a9,0x000d2df0fc374fd3,0x000c3976364d4eee,0x000e8e8e049e3ddd,0x0000f55e0cec7ea2}, {0x000ba4e4bf019842,0x000dbd55bc4bd3fc,0x000da895f764d5c3,0x000a6998cb92a1cf,0x00007dfe7957c7bc}}, + {{0x000d127df32d462e,0x00069353d1362a82,0x0001a23472b2d2a5,0x000b450ec57e5916,0x0000f4ff6eae5ff8}, {0x000400ac40a9d454,0x00004596718f82b0,0x0007a0dba9ecfec5,0x000cc066c183470f,0x0000f1aef698d4ca}}, + {{0x000decf23a86e2cd,0x000c0387f711b6df,0x0003509d102ac466,0x000bd1c30a1ac2d6,0x0000391342333aeb}, {0x000789f068ae34fd,0x0002187dcd100c7b,0x000e8a4b4499f2d0,0x00026c68e2a3bca3,0x0000f87ba71e4d2a}}, + {{0x00026848490bfdd6,0x000e35430226bf58,0x0007ae001b81059a,0x000449d5557551b1,0x00009a0e4f06a9c0}, {0x000ffb5aa7f70c1b,0x0000de46ce99fb72,0x000a020f9ac4e4ba,0x000477e56ecf595b,0x000074c5d6609374}}, + {{0x000b99934479e7b9,0x00044026e7800e3a,0x0001bd4299d5f2dc,0x00079a131374fd4f,0x00002e2e7392be13}, {0x0002f80d13bc1116,0x0000e6f81c920cc3,0x00063913a6fbfc35,0x00075c12ca1b3092,0x0000f7da200406ab}}, + {{0x000ea765fe235a8e,0x00004de31523f94b,0x0000ebfbb64be92a,0x0007ec6ed505fa08,0x000003434e32a6db}, {0x0006ebad02efcc64,0x0000d651266ce100,0x00052a6a6f23ba6a,0x0009c596fc0cc05d,0x00008d95322f6ce0}}, + {{0x000f12a19301b163,0x0001ec368a201780,0x000adb344233bc23,0x0005a1d9650892cb,0x00008a0d964dd942}, {0x000bcf24a8d4d7e5,0x00020806ce9dd77a,0x00070d4734ccd16b,0x00078082ff40f075,0x000074914152f130}}, +}, +{/* digit=27 [{1,2,3,..,}]*([2^108]*G) */ + {{0x00041205d8f6fabf,0x000a9bed09c79d8d,0x000acbb1e5e42083,0x0009965120e0794a,0x00007e840f9dc1bc}, {0x000d63b9707c3bb7,0x00091e891a716921,0x000bb22783ab4b8b,0x00091cbe46575673,0x000076ce040c53ee}}, + {{0x000b010fdc7945f2,0x000023750a4bcbad,0x000c85b9d62c9cff,0x000bf975192383a0,0x0000aba7b5336eb0}, {0x000adaaa4c06f9ad,0x00032f8e482bc3d4,0x00091a79f573a86a,0x000ec2109c6fdce7,0x00009ed02cf837ee}}, + {{0x00019d38fc7ea72c,0x00063211ef447211,0x000eda1a2390b4ef,0x000596289f39451f,0x0000ee9cb34205c8}, {0x0009e96971b6897f,0x000409701ea6a110,0x0002c6b2bdf70c17,0x000766a07d0ecda9,0x00008eb97ff99eab}}, + {{0x0008c448b63b669b,0x00071972d2d0d6f1,0x000ff59bc53c6eba,0x000ce52a522d8c8c,0x0000c181d161ed25}, {0x000173d5feb0eca8,0x000e2207bd71113f,0x0001715ae34273f8,0x0004ae41b7572efa,0x0000a8ffea2ff16f}}, + {{0x0005e681e13eaee1,0x000419dafdb94ed7,0x000941fc4bebd474,0x000bc08b46621f69,0x0000fd3c13fa1129}, {0x00048d0b7b9da22f,0x000bc87a47414714,0x0001c38b9d452cce,0x000bfee2f04778f9,0x0000b443a5ead516}}, + {{0x000b4797e8215d87,0x0007023b4b27721a,0x0003d21ef8438468,0x000e1ca93c08fe6a,0x0000bd962489fa6d}, {0x000ecd7ca6de3e08,0x0008c7c9fced3858,0x00008239e466a48c,0x00083323ca9b75c7,0x000060d553e35bbe}}, + {{0x000e02a5022112c4,0x00015461b1c6a66c,0x000247c388415df8,0x0004d6c2546e6aad,0x0000b9788e6da9c7}, {0x000b2e0a22be3e86,0x000c895f76ad3d0c,0x00033767015db086,0x000f549758f99ba2,0x00001ae09bbfab57}}, + {{0x000917a2f8ebcf55,0x000be53f056ca356,0x00084bb486f2c400,0x0008e309d9ac41dd,0x00000dc7a8ef61e9}, {0x000a9d33d3a67762,0x000736de8b3df179,0x0006e2beadda312e,0x000ca062a8b7c3ce,0x0000b00036c845e4}}, +}, +{/* digit=28 [{1,2,3,..,}]*([2^112]*G) */ + {{0x000cbd7ab6cf0b48,0x0005713d1ddf1ad2,0x0006003ba7a1e67f,0x000fcca58f0c7374,0x0000263e889264a8}, {0x000be37be2452f79,0x0006fae81a75c35c,0x0005537019312576,0x000d787d2ed0ab3a,0x00003d7e7e040717}}, + {{0x000013cf9cf03baa,0x00099eee3a77c1bc,0x0007e4c9536223b8,0x0005e486efc9233d,0x0000f3801bf36562}, {0x000a1413f32fd9d0,0x000c4e564acba7bb,0x000edcac170724de,0x00098552587f32b7,0x0000b1cd94cb3969}}, + {{0x000407a661fbdabc,0x000718e52150df08,0x000dfe954c4d7c53,0x000dd1a3765bce63,0x00006829bfc6c2dd}, {0x0003f65dc6e44874,0x0002ff04c9305739,0x000838c0a9ba2942,0x0005cd493c691418,0x00001b137ff8b2f3}}, + {{0x000aa244e1c5e60d,0x0009e3253d50f9e4,0x000babe5354bb528,0x000a64f4a86ab39b,0x0000561feaf3ac0a}, {0x00096ab1911bad71,0x00087433730317a8,0x0001f69289cb22b9,0x000c9ff14262fb16,0x0000661885c69ba6}}, + {{0x000ebd3b82574dbc,0x00034d8af3f58666,0x0009319bfc5e8667,0x0001a8cc64520520,0x0000183c12ef834d}, {0x00073da49eb0f402,0x000c8aca649e333d,0x0009e83613bcab0b,0x000c85a02f4c41e3,0x0000391e7aec89bd}}, + {{0x0007c5e608cbe2f2,0x0007116c22f26806,0x000faf9dccdec82f,0x00085c80aa719af1,0x000061fe95143401}, {0x00013669713e72e6,0x000ecd8a2a466b41,0x000106ce0db1e405,0x000e69ed17475711,0x0000d70cf6f571c0}}, + {{0x0000322cf707c76a,0x000fe0b4b7d71531,0x000a2d26d3a1eb2f,0x0000c0b83259fbb1,0x000083ffb1019972}, {0x00019280bedb326e,0x0001c92717150ecb,0x0002d71a94473e82,0x00056d506e6d202f,0x00007b253b223197}}, + {{0x0009335f576cb3cb,0x00048fb3b78e77e1,0x000da0001e16e457,0x000c5eaf96d78563,0x000038deafea7444}, {0x000b38cc0eb0e287,0x0001efe41b983ca6,0x000965b31a6ca354,0x000bc6feb37b4718,0x000039cc322c97d5}}, +}, +{/* digit=29 [{1,2,3,..,}]*([2^116]*G) */ + {{0x000ac99a211ffd3e,0x00051f1f6bca019f,0x000648244408f942,0x000d671f5b76d1a5,0x0000f3942e975b2b}, {0x000ee7fb538f1d7c,0x0001044b8f845b9d,0x0008ea6a31cb7862,0x00071f9f8ecd63cb,0x0000a111b2f79d3b}}, + {{0x0000e21091e1c4cc,0x0005467ccc747cc0,0x000ebbbe4d500db4,0x000ac0a8e2e84bf5,0x0000326688284279}, {0x000b4de7a1706589,0x0001e1da4a2d9e8f,0x000acee19219c5ec,0x000243da69a3fdee,0x0000d4c6fbdc0462}}, + {{0x0001d166087c5e5f,0x000fb53a8bc4e962,0x000d314cdae49c2c,0x000f18d7868882ca,0x0000de10dc89a57a}, {0x000a60d3800f3973,0x000b688b333c0fa8,0x000fa8129cec8ae7,0x000c32efd8d69285,0x00003d5685bc0776}}, + {{0x000f768680dcbf99,0x0005df0a51df60ac,0x0008df9c55072b82,0x0003323a74751cd8,0x0000d20f989acc1a}, {0x000042b6926c34ad,0x000ed076687f7e90,0x0006e9dcb5c728b1,0x0005aef2e3bfe8f7,0x00009822f0ae5a12}}, + {{0x000eafec58b6d022,0x000373443629441a,0x000823652750cd60,0x000d5ea68af789e4,0x000084d1e24e8d6a}, {0x0003ee3c4d56ddf6,0x00000c18e36243f9,0x000a2fe798365005,0x000df90e409c5b85,0x0000127bfe085cfd}}, + {{0x000ff1d35689ad9f,0x0003932addf1ec56,0x000fdde42e0f2114,0x000f5f4d52297877,0x0000402832dcd838}, {0x000267d915ac8813,0x00003c9963a5fa13,0x0002a42bdcf224e2,0x000c500d44419ac5,0x0000ed2898d4cc0a}}, + {{0x00005799f67b96be,0x000b3f509e955e73,0x00016dc34c6d265e,0x000493e985dceed4,0x0000308208028803}, {0x0004271dc7a39d4f,0x00032b9f5dc3afcb,0x000d33ea6839afda,0x00065a1c08ba2fc7,0x00003ae1c287c591}}, + {{0x000db58f51b14b08,0x0006239a79f03f84,0x0005a1f11df73ccf,0x0004000ce1e5842b,0x000041fa6a3985fc}, {0x0009c682455c32a3,0x000eefa71cc364b0,0x000797929383c9bd,0x000a5db63814861e,0x00003036faf923d0}}, +}, +{/* digit=30 [{1,2,3,..,}]*([2^120]*G) */ + {{0x000cf7c90f17cbaf,0x000520c7c5f351b6,0x000cc7f385d655ff,0x00079ec64f29d54c,0x000028e85325124a}, {0x0008d5167bf1e98c,0x00001d7a33af5efa,0x0009a40a48610028,0x000a0b35fe2bb2cb,0x00005cc1bf203d50}}, + {{0x0002bbfa32106c2a,0x000ade91774fc0ae,0x000853a30ef155b2,0x00005b5567c3c713,0x00006be82918ddb3}, {0x0008c21ade26eecb,0x000cb03c17ec7db5,0x00093f8a2fa3c895,0x0000c696ab0de162,0x0000d2365ed5c371}}, + {{0x0003e73e194c6423,0x0006a89e3c856ab0,0x000dacd55607b710,0x000084952aab024b,0x00001ca3ee251cc6}, {0x0009b2b1c6b93f9e,0x000ccad930f7f314,0x000872630cbc5ef3,0x000fe2ed04984f22,0x0000f5d052e4c4b6}}, + {{0x0008d05570286777,0x000c3e9aabdf6202,0x000d4e5d090eebee,0x000873ab977aee06,0x0000a98c52869361}, {0x0001251b7c2474df,0x00074f3e7b01f49b,0x000e54af1cdaf2a3,0x0006b7638bcaf46f,0x0000ec426250dac0}}, + {{0x0008cf51eafe37b0,0x000430dd7c2bda73,0x000dd77af503eac2,0x000e7cf9b7b7a511,0x0000ade03afe9dcf}, {0x000d34af479e3b5a,0x0003a30a33f2a89b,0x000b64068993ab40,0x000110aef322bf9f,0x000047cc71bae27f}}, + {{0x000f8f8c63b9e759,0x000c9d7d130a519f,0x0008490b743f6b4f,0x00014adba3385d7c,0x00007889df79252c}, {0x000ca86b2f18b9f0,0x000ec3a87422fccf,0x000474838f092ff9,0x000e9ff96dd67567,0x000039e82875bad2}}, + {{0x00009112da32c0af,0x0003a2676ca256a6,0x000614dc6b2e1ac3,0x000965fb74093f17,0x000044939e52f27f}, {0x000a402c922422bc,0x000afff5c56e8656,0x0009aa62ed60a55b,0x000cd20d061b41ab,0x00009ceacff6ca3a}}, + {{0x0001d47515f954cf,0x000c6f470a3f5f58,0x0006feaaf145f925,0x00043bfee6b6b073,0x000090744b01ea57}, {0x0008ceaa2f36f565,0x000f33ed4006fd8e,0x00015e6db4239a6c,0x0003d10906b5bdd5,0x00003622990dac97}}, +}, +{/* digit=31 [{1,2,3,..,}]*([2^124]*G) */ + {{0x000667432efbbce1,0x0003f9639719548d,0x000662e7d64a6c2b,0x00018438c0465730,0x00005d1d7ca452c9}, {0x000630ccc3020cc4,0x000cf09f038f30e8,0x00076a744e4b56c9,0x000988db9cb5edfe,0x0000c85f020a947b}}, + {{0x0004873ec83c9534,0x00037a21ef09f49e,0x000f7d93eb4f59fb,0x000c823872d31440,0x000079e1d0302568}, {0x000dc9a65d43d228,0x00026775efa857e4,0x0006defa6cc068e8,0x000956b78ccae932,0x0000f92b296ada64}}, + {{0x000aaa9c8a3e3ac8,0x000f43979374b572,0x000f271ebf9bc4e1,0x0008468117501ecc,0x0000c636f82980b7}, {0x000ef401522bb338,0x000b1a451df84842,0x000adce528726480,0x000bb68955aa02ed,0x0000e83cf34e78d0}}, + {{0x000f8d5dea227eee,0x000d1dda8b9fd721,0x0003e3520f48c766,0x0008b60583d94be4,0x0000bda36cac1d89}, {0x000286a6627adaac,0x00015938368d5808,0x00050949f19c4c62,0x0006d9e0dbd707f7,0x0000adf4beaccf35}}, + {{0x000cce23329e671b,0x000133ba85f9fa3d,0x000d2f0acbfe89cd,0x000dd5a661b1f182,0x0000ec370f3d49b0}, {0x0004271be20fb25c,0x0003f8f79665eacb,0x0008e416948ffff8,0x000ba1df8d3d2cbf,0x00007a6d9edd03f5}}, + {{0x00053eeb5763ed71,0x00076458542565c0,0x000cd718c4a553fb,0x0004bb3b2736a46c,0x0000c4f9cded6489}, {0x00001a0f71211b78,0x0003f2974258adf0,0x000ff25912c5325f,0x00020be8e172f449,0x00002e960cb21a37}}, + {{0x000c4a93631bfbda,0x000ad77461c62943,0x000958036ce38afd,0x000c48f0ad59d5c1,0x0000e284bfe02556}, {0x0003006d8b01967d,0x000c55a31d662fba,0x000c592b66b2f4ad,0x000018fe6af925e2,0x000030c029c1eb3e}}, + {{0x00021262dc890a77,0x000605aa75a385de,0x000070b3276b7b67,0x00068f475fc1432a,0x0000429a646fe31d}, {0x0009aaa09be3dcab,0x000a5f780ed73c3a,0x000fd96c407e119a,0x0005776212562564,0x0000571495098e80}}, +}, +{/* digit=32 [{1,2,3,..,}]*([2^128]*G) */ + {{0x000561a8a914b50c,0x000f5154d376bb9f,0x0009b4c352bf7130,0x000c566800f69651,0x00009e65041168b4}, {0x0006e006d98a3316,0x00074211ce1dd070,0x000562e5f781a12f,0x0007471fff9e3d40,0x0000356cf46ec166}}, + {{0x0008d63003b40dd2,0x00025951b7ae5a74,0x0004a91b08a68240,0x0003ea41e92dd970,0x0000dfb3eb9a58cd}, {0x000af35f5094667d,0x000d4435aa2cf3c2,0x00062714fffa287d,0x000b0ed03f397974,0x0000b550f67f03e5}}, + {{0x000549dd3341d809,0x0007c1a2a0a44759,0x000164f75079e9fa,0x00018275da56c72a,0x0000313ef5b4eefc}, {0x00065b6bde130ad1,0x0007a841117ffaa3,0x00026466a4426597,0x000404a65373f7aa,0x0000a43bee63e2cf}}, + {{0x000e1b8207eecd99,0x00047c07b47c29da,0x0004292dad3f50d6,0x00075b02b4d90936,0x000019a6df48c359}, {0x000d4aab616452e4,0x000d9cfc6abb741e,0x00089b025e58689c,0x000de5eac325d9f3,0x00005ceb1e6cf255}}, + {{0x0005a9858bf21d97,0x00074beab706075d,0x00032d47700eb684,0x0001c29d714c9f82,0x0000cde2c3f270f9}, {0x000a8cee9871f0c4,0x000b59e8444aa6d0,0x0000cd43a902bc60,0x0002228651ed57ff,0x0000418cc081480d}}, + {{0x000188534a70bfed,0x0008736345c44b2b,0x000322ae9cfa421f,0x0007a02f33f4cc6f,0x0000ac0bb761dabb}, {0x0005536923cea0a8,0x0005ed9cb50c7ba3,0x000812c96c16f73e,0x00042423216dc625,0x00002945e67bd7ab}}, + {{0x000bb350b6da2b61,0x00068b65ee5504ae,0x0000a004d91fef33,0x0001f7db77b57b1a,0x0000c59b62833aef}, {0x0008c893ec88d182,0x000a6fde31f1879c,0x0004e30b652cca38,0x000cbbe2f64a94cf,0x0000b4cdbd757ff1}}, + {{0x000b7fc506f7dcd8,0x0006dd037d68f4e7,0x000c8d374985e854,0x000180ff00a4da1e,0x0000c339ae3d05b4}, {0x000d4f23a5f71c42,0x000f87ac3e9f58bc,0x00065dad12fb4d99,0x0004dc7dd25aa6ee,0x0000fd63fc2d62c3}}, +}, +{/* digit=33 [{1,2,3,..,}]*([2^132]*G) */ + {{0x000671fe43332ef5,0x000b941c2217f3e0,0x00022ba1871c5dd5,0x000b674c1d2c1fe9,0x000058e9c302619c}, {0x000cde01ec51255b,0x00015f824507204a,0x0004c6afe824b374,0x0002e362d1b9de74,0x000099616dc5b0d5}}, + {{0x000c0847f6a1cda6,0x0004e23d634ffec9,0x000bad00768839c1,0x00054ecbbb678b03,0x0000a727255a7888}, {0x000fea2ef5c72941,0x00085875e775b747,0x00061a8937485277,0x0008b1ad7b8e8baa,0x00008ff333550da9}}, + {{0x0006ca48d15f612f,0x000661a8a9100439,0x000e4801edc27609,0x000765b388a29a0d,0x00001786469e2330}, {0x000153bbefdde3b2,0x000c66b8ce1ac63e,0x000313e17ecffc16,0x000fc5ab70c46999,0x000063b28a2fa50f}}, + {{0x0009f4f5529ec802,0x000a8fd146d1851e,0x000300c2c0420274,0x000feabbf1ab668e,0x0000d0b3a9d61653}, {0x000180f23a495b9a,0x0004f415d73a8be2,0x0007ae4fc6ef3c37,0x000f5f1d3e1ec8c6,0x00005839e9d38d31}}, + {{0x000c8b9c3fb67790,0x000ca57ee680c4c8,0x000673efc54fc9c6,0x00079e99003261b7,0x000067670079b117}, {0x0001da6250891c34,0x000b9d785ab3fb15,0x0006759c64720dec,0x00008e872b351ffa,0x0000c7ad29ae4f10}}, + {{0x00014d637d77c019,0x000bd1023c876541,0x0006e1221c2e18a4,0x0008f96fa6c3d39e,0x0000a6cf4e2d10e4}, {0x0001140b181828fb,0x000a28cb78333088,0x000cd236717c6df2,0x000922c1eb8df1a7,0x000078f1c8e7a89f}}, + {{0x000bcacbd862cefb,0x000dbf8f23c2598b,0x00045228f58f4234,0x0003688bce359f81,0x0000605d0f84812c}, {0x000dddfe4018fcda,0x000f9f797960b8b4,0x0005d8257b8139dd,0x0009bf631ee83dd5,0x0000c8b3932c4571}}, + {{0x0004777d0d428874,0x00086b7a2707b25d,0x0003966fe4b48921,0x000bec1b4dbf9b2d,0x0000bac5f4881ae2}, {0x00027331733964e6,0x000fca814a6908db,0x000898348a10c5df,0x0003f484ebdaf0a9,0x00000e46824074da}}, +}, +{/* digit=34 [{1,2,3,..,}]*([2^136]*G) */ + {{0x0002d18f9066d734,0x0003e098b3bfa064,0x000b4954cbe1d2ec,0x0009bbefee860c21,0x0000d7c4e6d67b62}, {0x0001e038e8b81b09,0x000f0fe77eb03d8f,0x000247c734a80168,0x0003564d977591ce,0x0000b30c9f3157e0}}, + {{0x000ead488c06b1c4,0x000089d9d4e8273a,0x0009e86bfea8af42,0x000a56a7b4409acb,0x00009f76f719414a}, {0x000c8018c13857a2,0x0002ee7423844603,0x0006657197c26f1c,0x0000c6042fb2242a,0x0000619f2550175b}}, + {{0x000699444bb5a795,0x000cebd1fdbfb12c,0x000e49191f5928e0,0x000dd3d30b8a973c,0x00002792b8633a05}, {0x00089161d3d69c3b,0x00099d59a28c5da0,0x000a05485931759e,0x000964412148d96c,0x00001517aa0ed6e9}}, + {{0x000bcc7d9b45206a,0x000e8abc463fed6f,0x0000e37d100ab735,0x000718428c701781,0x00004365872c7af5}, {0x00058230a910146d,0x0007dff76703bf19,0x000d6f1c8c13ccdd,0x0001b459d34ad644,0x00003dfa6b3495b6}}, + {{0x0005d0a1685e4603,0x0007455d00b5d0e1,0x0005614566fae8b3,0x000c63277126d8dc,0x000031c02e59bf70}, {0x000da4e515f39b7c,0x0006566c206009b7,0x00001f926b7e0d13,0x00037e9a801457c4,0x0000c560826bfb01}}, + {{0x000ffb56487a5d42,0x00054f53e25f6874,0x0006ba8fcc02a12b,0x0005d638654a5741,0x000026356f22c0b2}, {0x000eaa66030f2ac9,0x00022cea9175a4f2,0x000912104b788baa,0x000e3d66fbe9f74e,0x000082ef71dc9a69}}, + {{0x00090f62369ce91c,0x000a1754d218b653,0x000018afb2dc3763,0x000b66bc5523697c,0x0000a835077f5bf6}, {0x0003e4361d4b0a62,0x000747cf66c541b8,0x00057cbf28f87f59,0x000fb7ace5784093,0x00004834481cbe47}}, + {{0x000c2c96fd8b3094,0x000ae055809f298e,0x00022943c512ea17,0x0003e628cb44e788,0x000034dc70a1eb7d}, {0x00024d11be76434d,0x0004082ff0b0e8b3,0x00005e7267cf24ed,0x000abcda8265fe29,0x000057b3916be6c3}}, +}, +{/* digit=35 [{1,2,3,..,}]*([2^140]*G) */ + {{0x0000095d2a819b58,0x0009c8f2f6537901,0x000afe3665291aaf,0x000e27fa533907f0,0x00008a58ed00e279}, {0x0007127fae130bc9,0x000b88a54c747f07,0x000d82b6aee9ccf1,0x00052438a6783ebe,0x0000a1acb3e7d414}}, + {{0x0003ceed9c12e2aa,0x00021fc1308f44e5,0x0002c2d0f11983fc,0x0007233eb4d84d89,0x00000bfc1cb14499}, {0x00044d90145176bf,0x00015f12e75a8083,0x000a67545bb2988e,0x000989df73ceadad,0x000037069d21bb8f}}, + {{0x00035e69cc17f65b,0x000b39d9abdff24a,0x000c09ae3c49b3e9,0x00058782f403032f,0x0000ffe7d4db02cc}, {0x000f4e6424ef7136,0x00050658f65ff511,0x0008baea2b86bf65,0x000759623388d91c,0x00000664a7193656}}, + {{0x0006198d8447e168,0x0006e62171dc9899,0x00048b8e617195d7,0x00092328cfe6a1f4,0x0000a3c28238658c}, {0x0008d899c35e852f,0x00001b3781561c54,0x0000113b6adf1cd3,0x00096f999e41aff3,0x000087515a68cf46}}, + {{0x0002c559778aa8ee,0x000bb1b8d8b18c33,0x000533883290ae3e,0x000ee23e2bfa0bbf,0x000048e3747fa523}, {0x000f1d550fde3eda,0x000be8710432ad40,0x000241f3fb57e695,0x000abca41012581b,0x00000cabf7bd042c}}, + {{0x00019c7edfea5220,0x0008a400db7b68eb,0x000f03a0868e028b,0x000e366cd97bf7a8,0x00009f4d266f442f}, {0x0007f915d713a1ff,0x00061f58ccb42ac7,0x000ddc47d356e3a3,0x00011531d657f1f8,0x0000ea7aedf80092}}, + {{0x000a1d24ad49f66a,0x000e3d40861a8aeb,0x000a225f516ff2ba,0x000f0b25464f377d,0x00002fe66ccbffc3}, {0x0004d7fac757f415,0x00021d0a2c202407,0x00074b2665c85d3d,0x00004fccda2a05c9,0x0000c2e25121c10a}}, + {{0x0002535510d515f7,0x000cb89e785591ea,0x0000d17bcc861c54,0x000c869bc8485b68,0x00001472c11f19cc}, {0x00094850e9b5d8b0,0x0008d029720da7ef,0x000f50161698c9fe,0x000bf66ce987d161,0x000035f6f329d240}}, +}, +{/* digit=36 [{1,2,3,..,}]*([2^144]*G) */ + {{0x00015879c34971bf,0x0008d76545cefe03,0x000a81bb95829eb0,0x000710b7a3a6ae33,0x0000f42db0039c9f}, {0x000ae85bffb951b7,0x00033e70f324194e,0x0008b1f12815fe3e,0x0000bd636564cb42,0x000022e00511029b}}, + {{0x000578b41aeb39f1,0x000e8dd55c1fcfc9,0x000814075eeda86f,0x00017cd4b8fc54fb,0x00002e32a7843a13}, {0x000cd2b2fd217d16,0x00013bd076388b79,0x000dc7d8b5f5f20c,0x0000e2fc57643a53,0x0000512601550835}}, + {{0x0007479d0058ffac,0x000fd80b0de5705b,0x000a0585fe459ecf,0x000fa0fbe00c088c,0x0000169e23b83dd4}, {0x000009a44026f6ee,0x0003fd96fe5785d9,0x0008ed1e0bbc258c,0x000884d7ed379c32,0x00008b4574503970}}, + {{0x00091e139bb481ab,0x000b49984155263e,0x000d8e622dcdc072,0x0000349fce6e38d2,0x0000f6978b73e8c8}, {0x000e8748c37990a9,0x00089e749b861a1a,0x000dc7c12d1f0e06,0x0002115aa303b1cb,0x0000a78bab059130}}, + {{0x0000e8c48790aacf,0x0009154ad24e5601,0x0002090b8ac7a52d,0x0007dfbca462bd2d,0x0000cd84ef5a2f8d}, {0x00055b07ec8e0744,0x0003ba9b88dbf017,0x0007e09a00e4205e,0x0009130170f7d14d,0x00007f7d48d0e9d8}}, + {{0x0000b8acc130e8b6,0x00003c6f0871db27,0x0000d3f3417f5f79,0x0000319683bd22e5,0x0000d5852afc2699}, {0x00054cd8f15058ac,0x000b74cfe9dcc3d5,0x0009479d624038de,0x00059bb19336ecc8,0x0000aa9ed55b03eb}}, + {{0x000c5c0e0b77ebf3,0x0006d45a471d41ab,0x000e535e9dc41d92,0x000a8528f162333c,0x00005db2f52ebbd0}, {0x0005bf4eaec8559f,0x000702b31f5e4c0e,0x000d337b87be9434,0x000280c2b24a8e89,0x0000564495937b34}}, + {{0x000a910b3e2087b0,0x000344a61a335eeb,0x000cd5c0fbd016dd,0x000e94ffd1f08148,0x000041c6aa02e6a8}, {0x00035c5c4ac3d913,0x0005934767a4b09c,0x00040c2b55829810,0x00089f6120e7cb10,0x00002a661efd7135}}, +}, +{/* digit=37 [{1,2,3,..,}]*([2^148]*G) */ + {{0x0006a90bd74c70ea,0x000e2af672f2ba05,0x0004e9a4844f7d00,0x000069dc25ab68ef,0x0000dd15cc49dfb1}, {0x000fd289f3033bf4,0x000c2bb8b8a771f4,0x000d2861c088a49b,0x000fb5ea485869a8,0x00006dbfdafab977}}, + {{0x000ef4002d0aaff9,0x000dfe679fe21264,0x00088bdcea678c07,0x000db8cff13be7fd,0x0000a8efe8df17ba}, {0x000a815ad5a22f46,0x00015ec398b2b388,0x0004fc2da8f82140,0x0004f385a6a565ff,0x000081f0181e58dd}}, + {{0x000b637f8ec1151c,0x000c6adc8544ee61,0x000d0985a415dd36,0x000f50ed21d17669,0x00004d062b057893}, {0x000ba1a337b81f9d,0x000f87c163a17d93,0x00016e4edb995fe9,0x0008e7447eff3b54,0x00007660300dbf4a}}, + {{0x0008fbc4e933b192,0x000d054ea9692416,0x000acbb56af2db74,0x000d1d36fade13ae,0x00009708665a4e6c}, {0x000f1e1b692df97b,0x00031ae66306bf7f,0x000685f205a68c1a,0x000eec85bc544ce7,0x00003f42e6dbf65e}}, + {{0x000bbad2a08a1dcc,0x000b3f2dba7953f6,0x00064f5a3a7131e4,0x000109751dce4878,0x00005af1b45c8135}, {0x00075f74f08636e0,0x00052e251e483c4d,0x000d988539949b2f,0x000e84d04979779b,0x00009d8e627909b0}}, + {{0x000078177ef7f672,0x00074b6340c9166d,0x0007acf63fefec94,0x000dec7ce3a05609,0x00006538c7ccf306}, {0x000b55e287dc8d19,0x000bd48823adfa8b,0x00070fc519431095,0x0009ac8358087a79,0x00005299959d1350}}, + {{0x000ef0bfb48bd3a8,0x000f4508309e0814,0x000709c101bbbe13,0x0006557ddaf06167,0x00002b67847e436f}, {0x00001c603712e641,0x00024e3f9b2e1a56,0x000184b5dac1f036,0x000149cc7e6a0909,0x0000258b265c4625}}, + {{0x000033a675b47a06,0x0004c542378f9bf7,0x0005ec248cb3669c,0x00034696abb0f712,0x0000d5d2047d95fc}, {0x000c9e88f5cffb67,0x00082ea5ee0936c6,0x0004ffd6fb968f2c,0x000ac82f2ce73584,0x0000931b87797e40}}, +}, +{/* digit=38 [{1,2,3,..,}]*([2^152]*G) */ + {{0x00015ecfe73e6993,0x000c7752648968bb,0x0001dcc6d7533abd,0x0003634b26960749,0x000025ec2a0a7187}, {0x000aa197e9ab8653,0x000af25a9448466f,0x00031b5ba1c105b4,0x0005790b6b896305,0x00002691f115db1a}}, + {{0x0005f7341690d1cd,0x0009c0edac8bdbdf,0x00038cfb00e857a7,0x0001d259f40fcf82,0x0000cb54f68211d4}, {0x0006ac80e6451179,0x000f0af5fdcbaf03,0x0007bab2cdc4e833,0x0002bd67d859b23d,0x00002489b23ef8b3}}, + {{0x00034b4c22eb7ffd,0x000bad66596a89ef,0x000b3f3e1fc26171,0x000de06ce57d2c0e,0x0000e4177dcd587b}, {0x000f7eb6ebf95ffe,0x0004ff55b87342f0,0x0000cbc46efde869,0x000190b699bdb308,0x000032804f430cd4}}, + {{0x000aaf7b9e2f9f82,0x000a2ff419acc412,0x0007bcb900484c1a,0x0008ca9d94498941,0x0000b73dbe41e731}, {0x00071e552dd7b0ad,0x000b0954afeae91b,0x0007e3958d61f8ee,0x000f49aaeab13ca0,0x0000e442032d2a1f}}, + {{0x000f06ac55c0828f,0x0001667e2247abc6,0x000c49d4d62a7430,0x000b34e42b4c8984,0x00007a8dd4742091}, {0x000b96540a8295bd,0x000c39e900eb5ffb,0x00042ae0a36db01c,0x0009ebd9b90987de,0x000051f12e4efc53}}, + {{0x000fbd4a3b4ab0fb,0x0004907bdb60d581,0x000ae79b355d3bb1,0x00024ee65507f786,0x000085fa584e36d8}, {0x0004492bc0d8c261,0x0003a9b3e8d4611d,0x0004029cfbbdbeb6,0x000bf2f201f709e4,0x00008a99397c6748}}, + {{0x00020b3838c4ab94,0x000cb736cae32a95,0x00039f308526b695,0x00017f0f8eab84d2,0x000009b53e4d08c5}, {0x0001bcad169e4b4c,0x000767a467b7d378,0x000cc9436a91f107,0x000ec70e885579f3,0x00000d1f792fe123}}, + {{0x000d96d0b054a0fb,0x000924b90779d292,0x000f1d49fa978af8,0x000ab170bd185bff,0x0000e6d0844b279e}, {0x000fe45b8ed07e9d,0x00029b920e54d8ff,0x0001bb143714824a,0x000bb7cd5c628aaf,0x0000151afce4637d}}, +}, +{/* digit=39 [{1,2,3,..,}]*([2^156]*G) */ + {{0x000bd2e5a26a1d4d,0x000ea1c2d78bd433,0x0001025151148bb1,0x0003844aae419e64,0x000003b993a36489}, {0x00058b1d61a99194,0x00068a0ef3d4a21d,0x000e8c0dd17618c3,0x000a752519020d6f,0x00008d837d640b87}}, + {{0x0003a693f11fc001,0x00019bc8eade4cfd,0x000b9f7ae1f2b5d0,0x0004635160359ba6,0x0000e2601dd0a696}, {0x000c6d2915f60844,0x0009a79176d53f5a,0x00026abee6e38778,0x000d227fb99f4bae,0x0000798a2fcea409}}, + {{0x000342d86b383b5e,0x0000506b01d4e192,0x000cdcb89e85f304,0x000e2a19e1921388,0x000088f19440ce0d}, {0x000f8cfe453aecce,0x00097a67b49f16fc,0x000ece9610dcd10b,0x000b53b93d5b4daf,0x0000232f34ba39d0}}, + {{0x000cd7c64ca2efc0,0x000b4e0bcb6c799a,0x00058a26c3397a15,0x000faab9b10ced03,0x0000a30dbbe4b8dd}, {0x0009712e20f6facb,0x0003811451aff70e,0x000eece8f87c7f73,0x000e2df0c967b1d5,0x0000c62882b5b370}}, + {{0x000ba8516a010bce,0x000bca4f3e8e09c0,0x00070a1e52fd90fb,0x0000048af1837145,0x0000869e8f85cca9}, {0x000afb72dd830199,0x000e8d99b38652c8,0x000adab87b877995,0x00091a1e3efc16f5,0x00003105fe53a3b1}}, + {{0x00065d10a020be77,0x0002666eaf8ac051,0x000d2380e4856041,0x000b115ff898ddff,0x0000da35f08e84b4}, {0x0003e2c38fd05c77,0x0002b7ada3a4e0f5,0x0008995de64b3ee8,0x000605672ae31667,0x000047074614fe96}}, + {{0x000ce8b62a0b0ed7,0x00037abbc9be00cb,0x00069c2d692f5dc3,0x0003e1bb92b7d3f3,0x00000dd90c8e9ef8}, {0x00031537937ab45e,0x0005a054af6d00b3,0x0005ebfc43a8d1f7,0x000d0c35cf7380b0,0x0000fb8dac338c2c}}, + {{0x000cb400c11e65a4,0x0002a8ff2be53722,0x0009f9352c94a7f5,0x000544420085cc8d,0x0000addb9870a4b2}, {0x000eac006264a479,0x000472b48ccbac6c,0x00094fef2ebc01a0,0x000a8ac430e7abea,0x000073bb6f14d94a}}, +}, +{/* digit=40 [{1,2,3,..,}]*([2^160]*G) */ + {{0x0004c2e24424a48f,0x000f27d4471b21cf,0x0007a488b843c73e,0x00061cb3047fc561,0x00002a9170ad3cf8}, {0x00044211c3a60f77,0x0006966791481444,0x000d9404b74787a3,0x000ef0115fbd0653,0x00000fd3365d244c}}, + {{0x0005c9b2b574b7f9,0x00065369b6bde669,0x000108dedcca8040,0x000fce1f4bae99e3,0x0000e715ce37a133}, {0x000205554c2ee1ce,0x0001f680742d80d5,0x000e438b956bab30,0x0007cea409b5f63f,0x00003a8e4d16036f}}, + {{0x000fbfea62e08420,0x000360d426c06d25,0x0003a816cf07a613,0x00059db28fe4774c,0x00007987198b8d12}, {0x00021b150044888d,0x000211a2ad57348c,0x000ba884ad0b1f74,0x000a0385be8dae4d,0x000096bc030d9802}}, + {{0x000ec0f247fdfdf5,0x00079a23d1dc91d7,0x0000fdc41fb9d90e,0x00048c7012eb2c19,0x0000c2bbff72dced}, {0x000426a68cd7febc,0x00032b4854e0ca93,0x00072bbd8b596396,0x00040a8ac72b8ee7,0x000010d24d366b30}}, + {{0x0004b14e735cafe0,0x00044e80373d66ab,0x00095617fb275399,0x000da6bc5bba23f5,0x0000802dcbf70cce}, {0x000ba1cc60401601,0x0004eb5524732bf5,0x000b1a5907e18137,0x0003b1abdeb3cad1,0x00004f0c02531750}}, + {{0x000ef1694d5f3478,0x0000ab04af0a0fdf,0x0002ca633f318949,0x00014a30e3da7a6d,0x0000d002aeac8038}, {0x000311f95a0bfe97,0x000ebb4cc50c515e,0x00034df252891ec7,0x0002890936fed888,0x0000e5d7dbfe8e00}}, + {{0x000b4ae1863c1b58,0x000c296f5659ba49,0x000c2f3cb84af9fb,0x000bbdb45c071ded,0x0000db1c01e276f7}, {0x0009a9ea84ee50fc,0x000b09145b546f3e,0x000692134508e794,0x000adc6acff4f7e9,0x00007c5a5c9155cd}}, + {{0x0002d78fbfcf1b56,0x000c48427d7459a9,0x00066e74e17ce4fa,0x0005e5bae98ffdac,0x0000d548304745bb}, {0x000c6030992abe13,0x000aeefdc5c58f3d,0x000f8efb8318cfbd,0x000ef8bb5fa37d59,0x000047874a07ef5b}}, +}, +{/* digit=41 [{1,2,3,..,}]*([2^164]*G) */ + {{0x000cb614b4558fbf,0x000c0f2545a97f5f,0x000e5e4699602597,0x0006bcfd89ab3fab,0x00001daeea3eb2e1}, {0x000acd73a12940f8,0x000ccd7c73116699,0x0006c8ec624980f6,0x0003dc4a5cf97533,0x0000e180e330c27d}}, + {{0x000ece400e25a956,0x0002dac1732def02,0x000149260ef94740,0x000d51ecdb65ac51,0x0000043aa2a09180}, {0x00092bd852deca02,0x000735237c8ce7fc,0x0004b3f38f333829,0x000f17ecfb0e76e8,0x00001f2f5c58b89a}}, + {{0x0007134ca9bf60f5,0x00000c7da3a4dc68,0x0005fb57e2473ea9,0x000adbf54ef685b4,0x000094e84458383c}, {0x000d3fb4a7df4bbd,0x0008917c2c9211ed,0x0008fcba8a783d98,0x000e4c0d498637cf,0x0000ebd801e0acd6}}, + {{0x000aa5c7fd181e78,0x00004a3ad1eb08ab,0x00094aab5136a0ca,0x0004a5e6e5e6c2f3,0x00004d697d51349e}, {0x000578bf76f4b3be,0x0006f2feeb5ea215,0x000876bc381a1cec,0x0002ca5d336eb73e,0x00008afdcb5e7189}}, + {{0x000bc8cc051e9f61,0x000f5be2d9ca608a,0x000ab534875194e9,0x000490031d69c1d6,0x0000785990e88b0e}, {0x000f125f6c41f8ea,0x000aafbf2fe63825,0x00087161e429924e,0x00015353c044befb,0x00003bbdf1ba651d}}, + {{0x0002085894f7f160,0x0002b5c353fb791b,0x0008db0d442eb80f,0x000ef2377777f7df,0x000023c096334c42}, {0x000eb5ea34cb6d01,0x000e65cd1242aa05,0x000cd9f24ffb8b01,0x0009fceab6ff7d87,0x000075e94c9bb3c0}}, + {{0x00067251454f76ed,0x0005125c87a8a389,0x000efb7d652772de,0x000c11e59e4516c6,0x0000ddb8bf4a76bb}, {0x000ebd9c6fd2066d,0x0005ed7082e94acb,0x00069cea388c3b52,0x0004056a3b3d626d,0x0000bf73dfb5d065}}, + {{0x0002f06978f92cc2,0x0003431967527579,0x000cfcac1b11574d,0x000f67c3249711b8,0x0000061c767ef93a}, {0x0009a1b2f63dbe7c,0x000708091edd2ff0,0x000bba5a9527776b,0x000221f0fa985e19,0x000054f89f426ae3}}, +}, +{/* digit=42 [{1,2,3,..,}]*([2^168]*G) */ + {{0x00062846a436476c,0x0003f5dbb9cafc5a,0x00012ffbf6fcc231,0x000d14a77d2d9f50,0x0000c25e9f50ae4b}, {0x000cfc41a5e40c6e,0x0000df085321f17f,0x0003077c47d716a6,0x000e619dcbc50bee,0x0000bfe953dbb4a6}}, + {{0x0006f2fd3d777d71,0x0004df1a6b09d7e6,0x000f88dcf3519dc6,0x0001050df07bebdb,0x00007b09654bcd4e}, {0x000acd04e70c7833,0x0007c6b9d5779bd7,0x000e52f8ada66e74,0x000c1b6d0488a1e3,0x0000ec0fd119ff71}}, + {{0x0004cb6be4f27824,0x000b81c2c0cd3547,0x00065e29c10ef5e6,0x000608592c6b066a,0x0000d424663112d0}, {0x0000949b1a714fe2,0x000c3199f802d528,0x000a4ff3a52697bc,0x0009cec68ba4f8e6,0x00005a5380f55184}}, + {{0x0007f69573ec6f50,0x000e67bd2e8b3320,0x000fe24207ecc4bb,0x000cdda07acd348f,0x0000a957eb8a13f9}, {0x000f95b9ec9c0c55,0x000cba8578ccbbc7,0x00061923cd82147c,0x000f2507a2e7c59e,0x000091eb06682e83}}, + {{0x0001588957c94fa4,0x0007764911fb6aa6,0x000907b196a2bc70,0x000cc409771450c4,0x0000cc48773d694c}, {0x000216e50c878ace,0x000d4f3031f0bdb6,0x000d0d41e6e89210,0x000751b711dcbfce,0x000039bfe3eefbf9}}, + {{0x0007a45764636b5e,0x0007975d48f238fd,0x000a80177e437ee8,0x0000eae323bb1860,0x0000dc3c8f49c94c}, {0x0001164ec8cb0cf4,0x00096472936d9835,0x000059756ccdd882,0x00084aa8db1b8558,0x0000eda8cf9155c1}}, + {{0x000727d2923b8cb6,0x000d46773d5e7fb5,0x0006411656e793e5,0x0000958ecc901ba0,0x0000077ab2736da5}, {0x0009b0c6b127d9d4,0x0001163e2e1ec066,0x00041b6a28140e4e,0x0007b01ad5b03c96,0x00004299f88dbaed}}, + {{0x000296d1ea4a0569,0x000d677811b98736,0x0002dd74b6f74702,0x0004ab5c92754843,0x0000cc7327277a19}, {0x0005eded6328dca6,0x000988db755daf03,0x000192a4a5292aa3,0x00095cb5488385a0,0x0000e7d2fa93fc68}}, +}, +{/* digit=43 [{1,2,3,..,}]*([2^172]*G) */ + {{0x0002e2a0133903ba,0x00038495ee31871b,0x000991f285b3686f,0x000b8a89bcc9740c,0x0000dd20cced4f93}, {0x0001768680b65b66,0x00090ad41c3fff5a,0x000fb42690c453ab,0x000929d479630fa7,0x00000039d01b2c4e}}, + {{0x0008134bce59c0f4,0x000d159f7f86ae6e,0x0002834573d068b4,0x000bf0afa2753c96,0x000053fe33c5aedc}, {0x00094c8483c0b1ac,0x00041c2ad1edb812,0x000383e0b9e6f513,0x0001422a77b6ce69,0x0000b5a83acba9f0}}, + {{0x0003a2c72aa1c89b,0x000eda4d513df328,0x000d5ea181969613,0x0005fe0d4c0347dd,0x0000bad9ce4f3cee}, {0x00050a857313b49f,0x000def09bf30f8c0,0x0000395cb3b91c3c,0x000befe6e5ab6d61,0x0000c36cde1ceb31}}, + {{0x0005b26c3d8ed981,0x000aaa66a6e50991,0x000dab6a3f451e57,0x000ebb2984360730,0x0000710267c4d1a1}, {0x000cfd4e11d88c05,0x000761ce026a7e4e,0x0001227de12fc278,0x000dae9801cecd69,0x000017a92f3c6ce6}}, + {{0x000cdebb47b16903,0x00003a3866812439,0x00031334f0593b0e,0x000b1e46074e6e4b,0x0000e26a9771cf7c}, {0x000122b7e2ad0ee3,0x000b8e1b1a8ab475,0x0004832eadccc365,0x0004afe9dfaa7c6c,0x000071d9e4ff6790}}, + {{0x000c219e2b4f275e,0x000726d34c98a0c0,0x000d9305c8f7b63c,0x000ad6bdc5d00f11,0x0000fd56e73fd1d4}, {0x0003f94904a1f2ca,0x000a28430c3b8940,0x0000badcc559b8f5,0x000d737be89d20c3,0x0000c788b568b970}}, + {{0x000b800d37be97f4,0x000ccd54501eddee,0x0001ba27f575aed8,0x0006ea907f9efe29,0x0000430900142b1e}, {0x000e3526fcec9dcf,0x0004f01459aea95f,0x00067f611a37d324,0x00039d4ec59125d7,0x0000df685f85fe02}}, + {{0x0000fdf648c48e5b,0x00074f45a432d21b,0x000572855689e6d5,0x000547a5a9dca82e,0x00000f07eb83adfb}, {0x000b166552c8d550,0x0008ce85417148ec,0x000ee9bc0fe3fc26,0x000c80323af5ebee,0x0000666a2a341ae1}}, +}, +{/* digit=44 [{1,2,3,..,}]*([2^176]*G) */ + {{0x00020bc9ff262fb8,0x000e5075868b206d,0x0003fd973cba032f,0x000e027037602694,0x00001c57cbb635c5}, {0x000e700ba871f1bd,0x00053b265f564964,0x000950259f03a8c0,0x000a8bc8ebc9120b,0x00002b0ee317d32c}}, + {{0x000f1ffdeec441ea,0x000c701b21568c88,0x00045cf5cab52096,0x000caa37eee2756c,0x0000070d24ea520e}, {0x0005bd1546b9fd3e,0x00050c96db1b81d1,0x0005b29b73276fb7,0x000254c5c1b041b9,0x000018008dc9d7d3}}, + {{0x000a71dd360d017e,0x000609b0ed7245f5,0x00025a0184c0779b,0x000dd0c662fedc98,0x0000ee8912641d4a}, {0x000814d92163d14f,0x000017370d3b2543,0x000e1af7a79f2377,0x0001f7f80c6963cb,0x00002d521bddb9e4}}, + {{0x0000bf0ae309a0a5,0x0008875b8fbc68eb,0x0004993b7a9f52f5,0x000292b8e4f9481a,0x00000ce578edf73c}, {0x000a4a602e503d65,0x000134c68ea2d437,0x00016f34820cdfc5,0x0000dffec5993b36,0x0000d96b4c610c42}}, + {{0x000e136d3399e1e7,0x00016e2f8b74ee5d,0x00077db29e38bab0,0x00021e736126de3a,0x0000b0d186662aa1}, {0x000e45edecf9cde0,0x000e2318be705545,0x00055b0e59608ebc,0x000ca0e6596006fa,0x0000c8c2f41bc4b6}}, + {{0x000a7d03b64dcd73,0x000c858c9d559fb0,0x000fbc656f4c2f1f,0x000bea110c2db2a0,0x0000cad85cb1f5b6}, {0x000dd0e4e0b12301,0x0007198a2fcd6099,0x0001e7407c769b93,0x000b479209f5501e,0x0000b47255d2ba7c}}, + {{0x0005aa5d562b5f1e,0x000b15dac657dc80,0x000c28e5a7101b9d,0x000d0a5b7f211ddd,0x0000a89f24db3d1c}, {0x0009ef57567c80db,0x000e4a60c5ee0aaa,0x00036cd64e0d1f26,0x000de6c88a044cab,0x00008e03d02cb125}}, + {{0x000699827c7e6ebc,0x00032625bc7e8926,0x000c9cb1584ffa37,0x00084eedec924705,0x0000fad0b914075b}, {0x000d316bc0898d3a,0x000cb1f92524f4be,0x0007e59d702481ee,0x00012919896e1b0d,0x000006adb6d92bb3}}, +}, +{/* digit=45 [{1,2,3,..,}]*([2^180]*G) */ + {{0x0006950fe94629d0,0x00059f860b15c35f,0x0004f8f151cbaa93,0x00045829b4bcd3f2,0x0000ae5b06ad29c8}, {0x000c31d1b6c2df19,0x00016804facc1645,0x0002b33e6640b099,0x0008287a4a7f5912,0x00004bb0b2c0479b}}, + {{0x000aed34397ee7ca,0x000cd6fbc7fe5460,0x000b03dfef249e4c,0x0009f3c42d9da8aa,0x00003d73ce3e35ab}, {0x000dbc33813a3f3c,0x000b779c32a2c6dd,0x0008a2c3f86d5779,0x0002bf7c3d9aff02,0x0000687e71c71add}}, + {{0x000b3d6ce2a7da5c,0x000c86d365094a23,0x0009d79cd7efbfc9,0x000529dddc5f43e4,0x0000642b12101334}, {0x00052b60eb07da3b,0x00084623cceb073d,0x000ccb9c3e0c9a6e,0x00040e65e4f274a8,0x0000c6dacba8214d}}, + {{0x000a56209b3fed36,0x000a95799669e612,0x000960971f296c17,0x000a05a124a36f07,0x0000d03b214980c4}, {0x00046e270f1f268a,0x00003341aea42b07,0x0002f59cccc9b480,0x00015d1b3662d56d,0x00006c65b2e74b1a}}, + {{0x0006d5faba617ca5,0x000b0788f1e74ce6,0x000bd2024f5bb17d,0x000aae03ea09fdff,0x000053dd7097abd4}, {0x000458bc19c4561c,0x000dcb2676dc42c1,0x00074f7de4ecc53b,0x00051b1614610252,0x0000f5fa738c9ebc}}, + {{0x0000f5608a104d47,0x000f2317079a22df,0x00031eaa8fa16cf8,0x000c540425bed530,0x0000f7c45cba44f3}, {0x0004e5f9db182a9f,0x000f49f23a6d6157,0x000b6efa6cae923f,0x0005174b19b78d03,0x0000fa74d30af8c3}}, + {{0x0000568ac0e00ce2,0x000821c05e349afb,0x000a524f0ff4bc59,0x000539a94e0be1f6,0x0000995e87848303}, {0x000f99cde02dc40a,0x00037e00fcad06cb,0x00030a510be18069,0x0007327a71994d91,0x000072b5007a9e07}}, + {{0x000113acccb0a4b8,0x000ab615f016796c,0x000ad2f5b24c26bb,0x000abb52fe115aee,0x00005623d268d7aa}, {0x0001fd031a2564fc,0x000e8d0d59a39079,0x0008b74663659974,0x0007b02cffdb747a,0x0000f6b36e611478}}, +}, +{/* digit=46 [{1,2,3,..,}]*([2^184]*G) */ + {{0x000aa41b6223f04a,0x000f241c3faaed9d,0x0000b42cf803c9c0,0x00019360eee3f9c5,0x0000f4a7a5b17298}, {0x000c2e1bf809ad69,0x0002867c0ff24379,0x000f8e637903ab4b,0x000363c779d7ed90,0x000068b0cc0bcf3d}}, + {{0x000871cb79f82af7,0x00029dab52f5f4ca,0x0008239a731304b0,0x0007361825ab5492,0x000040413b2fe4ad}, {0x000d3fa44071d191,0x00014f0b2da8b5c5,0x0000a198183e438f,0x000ae3fd759448c7,0x00003e0c7ee665eb}}, + {{0x000eaba5568f2420,0x000b029afc1f58f4,0x0004d7dfc83b0c37,0x00053293de2d2799,0x0000d9a6edbb1679}, {0x000427995f085b0f,0x00099299355e4b83,0x0004427f846ebac9,0x000d0e0212e48904,0x00009e4ce3592f37}}, + {{0x000d314ef7750f08,0x0005a2da84d6834d,0x0000d647b2ceaa70,0x0001574561a254fc,0x00001cf09163be01}, {0x0003d4e34b798ebd,0x00065b1c7543847a,0x000194168bb5dd62,0x00010b94fee01bc8,0x0000b4c2600caeb0}}, + {{0x0006baf530c98b7b,0x000c0acf023af1b3,0x000de150304503d7,0x00089f96bc444921,0x0000b8a123082a9c}, {0x00099dfd5d4b133f,0x000cb97d3e5176df,0x000b318d61bb4a13,0x00052edab370f379,0x0000a6c823f8f185}}, + {{0x000b22514fb96241,0x0008da4229d70f09,0x000dc8c192eba4ff,0x00053e5b159dd121,0x0000e1f968fc1aa5}, {0x0005976c7674d527,0x000ceb283500fea2,0x0009468c298e73ad,0x000a627cfce0e1d3,0x0000aad0af978438}}, + {{0x00009f62eb1db212,0x0005b2c3e3e183cd,0x000a43da5be37048,0x00025eeb51fa296a,0x0000c7fa809f7266}, {0x000786ff0ec0e998,0x000388135cbf10c6,0x00004751bd315af3,0x00081ac1b6017215,0x00008674e2d8e287}}, + {{0x0000f771e33b2a39,0x00002f78986f8fee,0x000f05b7ed4bed82,0x00062a6396feea6e,0x0000c5d6a01b640b}, {0x000fec96834bea4e,0x0009c131fec9370c,0x000affb4d68d1672,0x0002564be9c5d600,0x000034a423d79a6f}}, +}, +{/* digit=47 [{1,2,3,..,}]*([2^188]*G) */ + {{0x0002874d8247f131,0x0001f3c11f564dfc,0x000503b97c211a7a,0x000c827512563fa2,0x000024cd9846c007}, {0x000b682491cd2498,0x00042683359ccf6e,0x000302b62af4f70a,0x000f562f1dfe71cc,0x00003c474bc657fb}}, + {{0x0007ab7916a8016d,0x0000232bfb9b73af,0x000174971f93d488,0x0004d4a5f9af3ce2,0x00001b9cf1ffd59b}, {0x000941844f4eb91b,0x000d7226edc49a77,0x000d4bb336a131fa,0x0001fe472ab89780,0x000069687a5df6ca}}, + {{0x000cc72520cd267f,0x000403d37ceb7c99,0x000c45cd39b83c2d,0x0001efc2b56ed68c,0x000061a49ae9d3ff}, {0x0006205a05677aaa,0x0009311b22bc4589,0x000beaef3c87845d,0x00092199ef1336bf,0x0000542906a3b1a9}}, + {{0x0003ca2fabd066a0,0x0008f9c78bfdffa7,0x00055cfef494e03a,0x000784e585a878ff,0x00000770b1fd7053}, {0x000da4a056fe70b0,0x000857bd444fdec4,0x0005df668e37395d,0x000583666250d468,0x0000549569ebe6cc}}, + {{0x000aac0ba9b5ca4d,0x000c4512a151524a,0x0000680f2f556fea,0x00041a6cff93f359,0x000033c25c629212}, {0x0007a1300ee2808d,0x0001896afc72b53f,0x000f030ead83e2c6,0x000f23a0abf470ea,0x00006c675098a456}}, + {{0x0009830ab11639e9,0x000b34488d52f762,0x000f06eb6869dd3b,0x000c3710fe1c0bba,0x00009034839a687a}, {0x00083777f1ffe7b3,0x000c55bd7c570841,0x00008ba673334a74,0x00012dc57cb7ed70,0x000084c12d0f1e4e}}, + {{0x0008690fb57459b5,0x00067b878006d66e,0x0007ea63d14bee76,0x0002bc9b5267dd2c,0x0000e8f2a4c53e01}, {0x000c64aca5e07275,0x00053875bdcccc3f,0x0006194383dd2f10,0x000d012a09337b76,0x00005415071fe1e9}}, + {{0x000dca6db4bdb35e,0x000d54d913a7148c,0x000ed94d56bc23ae,0x0009e78f0ccd9d12,0x00004aabd1366db0}, {0x000f31a1e9483261,0x000c27a479a1fcbf,0x0007686f1cf68c47,0x0006693cced8e2ca,0x00005ed1e995eb62}}, +}, +{/* digit=48 [{1,2,3,..,}]*([2^192]*G) */ + {{0x00092a4202bde39d,0x00050d6bab982b39,0x0007125122549f56,0x000e500b56464287,0x000052442b54fde7}, {0x000fd08a3d3e16e7,0x000b383b29bd36ce,0x0006dec8c5b194f0,0x000c1e6db0edd890,0x0000a09095972570}}, + {{0x00095d0eb10175e2,0x000aed236aa11106,0x000dfff4079aaa6e,0x000063f78539ff3e,0x0000369c51722cd6}, {0x0003ae55c8631ff6,0x0002816a60bd21e4,0x00061a5f9065e821,0x0006fa225cb473e7,0x000095ef8610b6de}}, + {{0x000e6c9f60ee3e5a,0x00004236377f06b9,0x000a4af71b397af8,0x0009fbb7a318ac7d,0x0000e64b613aa9d3}, {0x0006c74902b34119,0x0000ea199e52d6ce,0x0000fb76fea256a7,0x000f9c8dcddd8955,0x0000443b47793e70}}, + {{0x000fe2d952980037,0x00011d8412fbc96c,0x000997dd3caa6655,0x000587f41d021583,0x0000066e356b5534}, {0x0005c3e5b6de0d7f,0x0001becd5f251d5b,0x00052ae508ead45d,0x0000bfe2f24e2cd2,0x000071e5d5071515}}, + {{0x0004735e1d88c25d,0x00026bd66b66d27a,0x000af9942339905e,0x00083ca9bfa0ed62,0x000094dd9e0fe2cb}, {0x0008fdaab28e268b,0x000c01a97517779d,0x00091f33ef28ab69,0x000d64ce9bd2ea96,0x00009e8b2ff24be3}}, + {{0x000c0908b9f8e8c0,0x000024e94ce19274,0x0000f3ec1a375761,0x00026f8f2b1f2c4f,0x00002e53bb23938d}, {0x000da2c701e5ae8c,0x000b6271dcc9d0a3,0x000fb237260c2eac,0x00008ac9c08e3931,0x0000aa3245f78389}}, + {{0x000f3382b02578ba,0x000f04b2c6071100,0x0001611794716339,0x0002a092c923ae5b,0x0000da2e4daadf44}, {0x00090c547f854740,0x0009824e31954d4b,0x00078a0d4a378bf7,0x0009e24469339d24,0x0000972e7880c1e6}}, + {{0x000585b36a1f3a8d,0x0000613ba5f1a9d0,0x0005acf11a3d8f18,0x000e791eaee5d622,0x00004dfd0a2dd32d}, {0x000324b6b3ceff30,0x000cab4478700cec,0x000c759ac3acc6de,0x000938abbf7e6db9,0x0000d5c1f47a5196}}, +}, +{/* digit=49 [{1,2,3,..,}]*([2^196]*G) */ + {{0x000215745e7ea912,0x000dccadfc40581a,0x0002cd3934da3f86,0x0000fcc81d6c7d16,0x0000c38a2a1cd6e6}, {0x00025d6f753479d6,0x000f1dec6024f75b,0x0002d5547c914e08,0x000969f81cea3449,0x0000bbb8bb25b1b6}}, + {{0x000082c1279504c1,0x00023c92ffb70ee8,0x00081c7e2a466abb,0x00033a4118b26a3e,0x0000a76cc510c60e}, {0x0008bc25736d7aeb,0x00096d1ef9928499,0x000bf252520b3955,0x000626d669e2ae5f,0x0000f956ec6b1cc7}}, + {{0x0007029b0ccbaa5e,0x0003079b78a5ee81,0x000f45d8957ef5bd,0x000fd3c92837474d,0x000086b91a90ec4b}, {0x0004c6dfe5659152,0x00069c58a042c5ab,0x00001c4bce657471,0x00013ae141deda63,0x0000f95d561a0845}}, + {{0x0004508eecede3d7,0x0005a86440d06c42,0x000de0d7711889b3,0x0004477b229f9398,0x00003fced8aa00a7}, {0x0001c79e31c8f881,0x000dcb277e4fe75e,0x00087c02c8db20bd,0x000c1a8ded0a702b,0x000066281b55d164}}, + {{0x00056cfeedd8e0c9,0x0000744c012af873,0x000aa3eb68afab38,0x000a570795935fe4,0x0000b9efc0d6a6df}, {0x000f8aaa8ab0840e,0x0004a85616042ff0,0x000db93150f3a4b6,0x0009f82ca911efd5,0x0000f70e5bba8ded}}, + {{0x0009209a6aae58bb,0x0008d74edda2943d,0x000be3c9a3d0798e,0x0002dc5c2d462cc2,0x00005488239f3988}, {0x000bb41977d4de44,0x00005e8245c42391,0x000093dc27fd9104,0x000e121a6d3c713b,0x000023a4e3abb22f}}, + {{0x0006f403a9a04a3f,0x0006197d9afef315,0x000a898509b32c4e,0x0000827e0b401e62,0x0000fbf542f984ef}, {0x000c0e0f990caf53,0x00041c88ea9b077c,0x000f96a510270434,0x000c47846fd46c63,0x00007f5cec019855}}, + {{0x0000bd8e6ad29dc4,0x0000aa04da287d14,0x000e05b337dca4b1,0x000acfa84feafcad,0x00004d031f8b630a}, {0x0002fa6cdee269c2,0x000ba697a40af8af,0x000e5f8261e40571,0x000fe6f71d44adf0,0x0000a47ddf9c434c}}, +}, +{/* digit=50 [{1,2,3,..,}]*([2^200]*G) */ + {{0x00092d346410cb1f,0x0002f3a5a1354b38,0x000de068d72097f2,0x000890db3a1b8098,0x0000b7438e53b1a3}, {0x000d5ea3839d3d99,0x000e84bd8125fa10,0x000800261d9ad034,0x000c5207d108efd4,0x000078d98bb2c5d6}}, + {{0x0002d017375f2deb,0x00093aa152399f94,0x0007e5c4c968a760,0x00040e55dc7da637,0x000075fff53d82b5}, {0x0005c15fd4b69512,0x0007104ddfaedee3,0x0007c87146d1d64f,0x0006fd320f1769af,0x0000b5f86a4682ba}}, + {{0x0001f3bdd857b31a,0x000b69513734201e,0x0009327b892c2263,0x000164562652d558,0x00008edd06636a1c}, {0x0008f9879f8df8df,0x000230847ddd3cbf,0x0007c06223d5cf77,0x000f2a69b08ee459,0x0000ff18c4e7a868}}, + {{0x000d7a99107901be,0x000f8211d1153c6f,0x0001b78a85dc4a41,0x000b6a597e94e7af,0x000072da34e33afc}, {0x0004db774512c248,0x000a32811e913c36,0x0000469b1c26a8fb,0x000535fdd39d7f29,0x00004515392a0612}}, + {{0x000606a4a98f93dc,0x000eb3475d7197ef,0x000e3f8cf41b9190,0x000a4f7ec792db91,0x0000a37ec10e2396}, {0x000c219d0879e176,0x00044ac57d3f90b9,0x000037ea761303af,0x0001ec13293c3be1,0x00001557e050058f}}, + {{0x0007cd15c5d384fe,0x0000d74de832cb99,0x000fb7ce336884af,0x000ef76be21274d8,0x00000bb298f86608}, {0x0001a0cd53bd4269,0x0000feb3293b527c,0x0006fc1ac78035a2,0x000fed0d7689eb2a,0x0000a38274adf2c4}}, + {{0x0001c4eccf60d0bb,0x0004bcab95d81e7f,0x00025a060c7918e1,0x000f357af4761e14,0x0000fa0bf6fa8e80}, {0x000629801e11b3e0,0x0008d92d8a4f88e7,0x0004a9e6f4b7ca48,0x000b1f247db98bcf,0x00008745cd1c9705}}, + {{0x00096a01539cf31c,0x000acded7c6dbfe9,0x00016f6144a3f729,0x00016386f1f29930,0x00003d44e1946cb9}, {0x0004531558fa36c2,0x0005569c89d67698,0x000ee923f58e8bf0,0x000fbc287da114f9,0x0000032e9850c271}}, +}, +{/* digit=51 [{1,2,3,..,}]*([2^204]*G) */ + {{0x000579dd39207ad8,0x00058b1fe91611b8,0x0009e01bf6f62c72,0x0008430f1599acd8,0x0000d9bb86dcd1e5}, {0x00090d4726e38d1e,0x000b32a8c6b8548b,0x000d2f6f4b824a1c,0x00012e984d9309b1,0x0000fa485b8231ec}}, + {{0x000560776ac19252,0x000b5442fc587972,0x000311e74086449d,0x0005618dbab9202e,0x00009dee69b9ea25}, {0x000b6ee19a7cd6c0,0x000c0d0dd5a05a62,0x0006d0ff1ba38cc4,0x000dae779279e516,0x0000eef53ccf48b3}}, + {{0x00078408c7c6d897,0x0008b2c2555f255b,0x0009c3165bfa78bc,0x0001bc8c56ae3da6,0x00002b20e73533bb}, {0x00070b4d1aa6416e,0x000c5db03cdc8868,0x000a0fd40f000040,0x000f1fd25b16a9b0,0x0000b89e93316815}}, + {{0x000393579f34a8a7,0x000fa0046434c9b4,0x00005fdb161acf55,0x0007fff0a23fe63f,0x0000d6baee17c4fa}, {0x00062b6e2daf735f,0x00091948637a353f,0x0004e6206e370ead,0x000ad3da57c16ad8,0x000019ffe09fdd22}}, + {{0x000ed1e4263a5660,0x00041d5e8f55fe64,0x00094342c735bbee,0x0009359ac6191582,0x0000f522e5ad4904}, {0x00083b57c1e11f47,0x000060c237f3bee8,0x0009550860a2ce31,0x0002dbf4a7157b8d,0x0000ec0462e722dc}}, + {{0x000a5c739620daa9,0x000fff7c078f1e65,0x000177db88ef6f93,0x000535b06d52bcb0,0x000015cdd08f8fdf}, {0x000d150c51832222,0x000d9817a2ade070,0x000194f0b2b6495c,0x000dfb3ce476140b,0x0000eec6acf713bf}}, + {{0x000814fd7daf836b,0x0007bd6169217cd2,0x000066012877b72b,0x00059dea73ca1bdb,0x0000e336c7c60d41}, {0x000b07f0f8fcd765,0x000ada5935626993,0x00091ec195fdceab,0x0003c0716595fbf6,0x00001a77f61de68e}}, + {{0x00045a51d63aa7f7,0x000623b32da2680f,0x0007088588d5eba7,0x000c610ef233dfa4,0x00004c3f4f7c2161}, {0x000fe6be6420de40,0x000197dd86d5fa9b,0x000ce233b96c0c50,0x000d6328e6827bcf,0x000035cc9a958e74}}, +}, +{/* digit=52 [{1,2,3,..,}]*([2^208]*G) */ + {{0x000997917a86e187,0x00022e029110b2de,0x000cc5a17e2ac232,0x0003cfbfd3439735,0x0000a93461fc25e1}, {0x000542c5122d6f14,0x000ec33982c79433,0x000c24d2741d2d9d,0x000d58e9f1f29a8e,0x0000ae251f433b99}}, + {{0x000a46047920742a,0x0000545f94cba02d,0x000608dd4b142d6f,0x0004b5c2d613e876,0x00002c06cddfd75d}, {0x0001bc53c564ff4a,0x000c4d1d5ecd0195,0x0002135ade60f126,0x000695a634e76570,0x00005a56a6f2df44}}, + {{0x000482697e839b67,0x000b3f49adaf9986,0x000795531cfdae67,0x000aa4f713da0fb3,0x0000b2f6f6d8d391}, {0x000b8c351b5e7aa9,0x000244f29191734a,0x000692f40fc4a912,0x0008ba8f8e2cdc12,0x0000b0bee740f405}}, + {{0x0008c7a9e2207b45,0x000147ee9f61d34b,0x0008e21b61f724f3,0x0006aefa908ca2c5,0x00005587744f7429}, {0x000913002911ae14,0x0003dd3af02e3dbe,0x0003955a1c207543,0x000d66505b724b0f,0x000080e1a930aece}}, + {{0x000246ba01d1d691,0x00061c28f15519cf,0x000dcd8cb66c23cf,0x0000f0441e71089f,0x0000c22323a9398c}, {0x000ea418a3f400f7,0x00008b3a0959e3ad,0x0003b53b4787d2e1,0x000436e7464f0fbf,0x0000e7cda2bec46e}}, + {{0x000f0871be603736,0x00064f9180e995fe,0x00060416e4dd25ba,0x0005530d3996f9ef,0x00002be886c4b749}, {0x000c5fbb48089d39,0x000e6a2b6d24c533,0x000c20f7f06c8690,0x000dbfdf149e4e00,0x00000f3d7e0794a9}}, + {{0x00015dcfc8176c4d,0x000d68bad0d95505,0x000ec582805f29f5,0x00027312d25fc5ef,0x00002f5fa1bfdd52}, {0x00071e3717366b82,0x0002d2d99b2a09d8,0x0005ed437bda0f49,0x0001592abda54f23,0x0000449feb33935c}}, + {{0x0006128446d9f664,0x00050354b5a1720f,0x000558aacd6e06b1,0x0008fda72d287d63,0x0000819be29ee68a}, {0x000c324205fbdf26,0x0007810927f95024,0x000658f802fca94e,0x00004674798be7be,0x000018e07f24f07c}}, +}, +{/* digit=53 [{1,2,3,..,}]*([2^212]*G) */ + {{0x000a94654b01bbca,0x000da55d4820b47b,0x000575f207c56c36,0x000cf48e93362005,0x0000ec65be9da621}, {0x00096df462879372,0x0003933c67e7620b,0x000cf3e0535cea88,0x00077330366a3a58,0x0000580d5654da39}}, + {{0x00061815832cd6cf,0x0008885e90b9553b,0x00014f686c002e33,0x000d454b33afde64,0x00009ab29ea7511f}, {0x00009726fb9a688c,0x000d3202a1b2467f,0x000a881ab7db6e14,0x0006600c15b6e973,0x00008c324e1bad10}}, + {{0x000d412c9489dcb4,0x0005512d084ba43e,0x0004c7cd2cec053a,0x0007f70fd4fe4266,0x00008ee06f4c2a91}, {0x000197083cd65f5f,0x00094569c42ed5de,0x0008761103a17657,0x000bc4f24508f34f,0x0000350374bfa415}}, + {{0x000108255ade88b5,0x00019b212e84bc89,0x0001362fa0525b50,0x000cac9ede010bb6,0x00002f3d088d81ee}, {0x0007b6ebc6f0ae45,0x00036659701cf995,0x000b76f4525fe726,0x000d4241e9b7f507,0x0000f2ad6650da77}}, + {{0x000518c9bb777fc6,0x0001e24151875571,0x00020284bf369c39,0x0001cc2d7c5dd927,0x0000feab634aeec4}, {0x000d5db24ecd0bef,0x0007f03387353522,0x0001452791fca0d9,0x0009257441610520,0x00004492e1ef496e}}, + {{0x000dac0d193a5260,0x0009e2b2d54f3c44,0x000c67d11d071709,0x000ad865624fb4c4,0x00004aa7033bcced}, {0x0002fac31470c527,0x000aa3b05d539c52,0x000565e5708eb33c,0x000370940e0693b4,0x0000e2fd553ebe56}}, + {{0x000f5d0be4d36ec2,0x000348e952fe8568,0x000ad6b241e9e539,0x000de9768113f9ea,0x0000bc798fcc8fbe}, {0x0007f9baaa9010de,0x000b561834c94c78,0x0002c25d832cbc77,0x0006dd449e55f703,0x0000824a5c0aa621}}, + {{0x000065a848bdc535,0x0004578554dcb366,0x000ff3b4dba2af07,0x000cbb3c755fba19,0x0000ea9337285a22}, {0x000fe021eb3e23bb,0x000b265dede35e55,0x000f445da2626ecc,0x000578187bf09481,0x0000a0110184df30}}, +}, +{/* digit=54 [{1,2,3,..,}]*([2^216]*G) */ + {{0x000ee9a1baea5745,0x000853ae5167cae4,0x0006911240d922f2,0x0007c407df28fdca,0x0000aad2f31ddd45}, {0x000f443137384cac,0x000b2620ea8c20d4,0x00067797ad93d424,0x0005bc21d544d350,0x00008a8cc9a5d8a1}}, + {{0x0000e26941d80a3b,0x000a5d36514db10f,0x000f226013092787,0x0000f6aa2dfd4898,0x0000c5b31b838c8b}, {0x0001f376c841cc81,0x00096a412b84a128,0x0008f210ddae1671,0x0009019ec1f6c882,0x0000935d576f8d92}}, + {{0x0009be8f731e66e3,0x000ff7c9ac6742c8,0x0007788475c65a3b,0x0005a7621746f622,0x0000e38982159449}, {0x0004aab948c94967,0x0008741e79476b0c,0x0005283abdfdec51,0x000bc091420e3a60,0x000099ec46fa4c7d}}, + {{0x0007921af3840f27,0x00034f3fcdfbd724,0x0003961bd348325d,0x000c29ef578508c4,0x0000d5e8ccd8bd98}, {0x000ba10f8a30164d,0x00004cb8c65289cd,0x000056ef40757409,0x000d1bcd7ed73fc3,0x000028e7cc2cb99c}}, + {{0x000c0f95182958d0,0x00075749c24bf1b7,0x000638c77ee0c407,0x000cee32f35fcff3,0x0000a46c52952773}, {0x000d4f7e2fb9042e,0x000a32ce916d9ae3,0x000bf7ae89855249,0x0002b7e48a640b04,0x00003eb60417c9cb}}, + {{0x00074946bb62286f,0x0007d7d87466b8a6,0x00073b6d58d6ef81,0x00001b3529f938f6,0x00005c0ee777bbf1}, {0x000ed6149fdc9493,0x000eff405ee00768,0x00018e51a53b6999,0x000bd7bf0108a017,0x00008e609ec8181e}}, + {{0x00079083a6a2a006,0x0007ee549a6dc288,0x0005facfa82df7fb,0x000de38f1d62b44e,0x00005566a01600ca}, {0x000bcf5b76201357,0x000ddb75df8e1672,0x0002fd615cf00849,0x000a8a8f5d3e0fe1,0x0000fb3f7894f727}}, + {{0x0001640d85951597,0x0006c79a38761205,0x00098a67c30e1c70,0x000d27091154c602,0x00006d9a9c9c1132}, {0x000e8b75c661a4d6,0x00068a6dcbaf841d,0x00014e045f24b5e9,0x000c701ed4cb0cd7,0x000005926a46c187}}, +}, +{/* digit=55 [{1,2,3,..,}]*([2^220]*G) */ + {{0x000e0f2aa1ff4af5,0x0007890503a3c558,0x0004f76f161f4348,0x00032e661647323c,0x00000d0706eee134}, {0x00087caf5eba0fb7,0x000ff0bda2c4ef1a,0x000ae646228a95c0,0x000ba610d693e012,0x00009871341645e6}}, + {{0x000b01264b496679,0x0007ae05f7601dc4,0x00085d27aa4bdafa,0x000747171b28b3f1,0x000087e5163c3425}, {0x000ac4ec3864a656,0x00091f449c121c42,0x0009658322dae1bb,0x0009f4680d974306,0x0000ac1ef01de31d}}, + {{0x00068563958e0555,0x0000adf44aeea780,0x000c18b8dac35ee4,0x0006cd2b47891397,0x0000396824f4a258}, {0x0007b251b23f8c4a,0x000f6decdef982b3,0x000fc39c0f9ced36,0x00032b28c3bee5c2,0x00001731faf0d9db}}, + {{0x0004f74ea3d44467,0x0008b0ad1c7f045d,0x000374e92a439f17,0x000008c95d951051,0x00005870ea000229}, {0x00098c2c54e7e812,0x0009b4b3860b1fec,0x000bfc8f6ef537ee,0x000403139dd83440,0x00000b513642f114}}, + {{0x0006ab6dd13b3c94,0x00072371f872f32f,0x0005712ab70e052f,0x0001000574074212,0x0000239152df3512}, {0x0005eaa80b229154,0x000cf896f6fa5835,0x0002b4c8fd0e263e,0x0005bb9378a8a644,0x00000c2b54736579}}, + {{0x000fe8c284745301,0x0003eb7fec008f80,0x000cb05ca5649a17,0x000c41deed5bf4d9,0x00004b1a3a9e7077}, {0x000d239096883ec3,0x000e74ae671d1c2e,0x000b7362ad550edf,0x0004f2b233e5dcf7,0x00002c158207fd46}}, + {{0x0006a81c61ae9676,0x000cd2a2e1e5c52e,0x000211ef8af11042,0x0005fcb353902a1a,0x000044d16e1299a2}, {0x000d6065b67e48ad,0x000421a0b664937f,0x000e072b8fa57096,0x000727aa661c737e,0x0000e1eb4ffc2e0a}}, + {{0x000bc2afda6a4a95,0x000a5106ae0dbdd9,0x00069a4bbdba0178,0x0007153820c9f549,0x0000031e9fde9fbc}, {0x000030ac193d9428,0x0007c54cbb38e42a,0x0008fa77cdc3d6ab,0x000a95507c17b91c,0x0000465bcc963642}}, +}, +{/* digit=56 [{1,2,3,..,}]*([2^224]*G) */ + {{0x000265bc25dfad35,0x0009993f44b6974e,0x000d6d473d03630b,0x000992b3270892bf,0x0000b2d95436c5ee}, {0x0004537a36f7c5f4,0x000dcab0b81daeb9,0x0008b45e59befc01,0x00048b483cdb0818,0x00004c753b741e46}}, + {{0x000bc87b24116186,0x0003507924c48e43,0x000c9255708754bd,0x0007f4ef2050334a,0x0000e7e4fe74e038}, {0x000e2e276961d0e3,0x000767eac10f21f3,0x000757a882b69d41,0x000d3536d0f45f73,0x00008b967e55b0c7}}, + {{0x0008fc4b31fa7794,0x00054f13036e54ba,0x000d754b78024dc8,0x0009aefda2af6382,0x0000a784242ee9ea}, {0x00014abf9887947d,0x00059d555a0997dd,0x0003a46aa7f2ecfc,0x0004b5b37c4244f6,0x0000032cfc2cf71b}}, + {{0x00084c16b8a6a975,0x000e5b2bca35baef,0x000174d43d2e7f3d,0x000c5b721c6c095b,0x0000719cf32252cc}, {0x00061f03adf95177,0x000a1be20ff4fc73,0x000eacc0e1e26416,0x0005d201f9d99769,0x000021eba6432e63}}, + {{0x000e9c825df8bb5c,0x000f75752d7d8225,0x0003b281d931f721,0x0008be3c4ed4750a,0x0000f9276830a466}, {0x000358e75b7e90c9,0x000e47a29b998b7f,0x00067f2c806e5c24,0x0002d30058967aa1,0x0000f1a6fba34ee6}}, + {{0x0009c4f278291f13,0x000524e64c1d9a89,0x00062916e69a9032,0x0007cc46cc5d428d,0x0000c802e661c100}, {0x000f2aa6219cfbbf,0x000dc10258b1eadc,0x000e142af942870d,0x0007a377264e68a5,0x000025675e2f89cc}}, + {{0x0008a3b7336aa166,0x000e1c5c622bb77e,0x00089e0295a92cc2,0x00057333a35a2c17,0x0000f91306eae4d5}, {0x000a581da0a46f52,0x000d62640bbac5a2,0x0007b3ae4fb532be,0x000a7188ff0f114a,0x0000223e7b6d8ff7}}, + {{0x00031335d21d261c,0x000b8abb1fbcb593,0x0001b3b61a336289,0x000511797db2f363,0x00002cedb2697e6a}, {0x0006f34103553328,0x000add0ae37f3880,0x00017c5c7e5f1fb4,0x000ec357cf26a55d,0x00002e8df47e8c43}}, +}, +{/* digit=57 [{1,2,3,..,}]*([2^228]*G) */ + {{0x0001128ffb5e7ce8,0x00032ecb96c1e0d4,0x0005ca788dbd8b54,0x00038c029ab3dd0b,0x0000b1148a2990eb}, {0x0008db869fb19245,0x0001018391a80904,0x000d311b9cd2149f,0x0009b96bece5b6fe,0x0000edbe9b9effd2}}, + {{0x00093e7532e8d6a4,0x0005302a1ee4ab55,0x0000496c09409290,0x0007c8379b32e820,0x00006fb6e9e760a2}, {0x00077ba620051584,0x00014de3f1914a33,0x0004d21271a3266c,0x0002f3e60fad96c9,0x00001553dd1a4630}}, + {{0x00031a2a6ec0fe46,0x0002959645720704,0x000da5c3949da0b4,0x000e51bb12d1b09d,0x00004170fe1d2d3d}, {0x0002b16a4a2f5d9b,0x00059590be923a8b,0x000b9b0b5de1bad6,0x00067eb7f93581f0,0x0000007f4de9115d}}, + {{0x000d36fe12a94abc,0x000a75ad7c477b0c,0x000768469f364b3b,0x000cc596a7a2a78e,0x0000cc31c7edbbc7}, {0x000d5d0080dbb92c,0x000316fb0f1f270a,0x000d99f57fb201e9,0x000da5dfce7a1e29,0x000012a02b0c6457}}, + {{0x0005482a048530b6,0x00040467f15d9468,0x0001b7d84b4a3120,0x00095db5f8a2605b,0x00001ceef97bf452}, {0x000832a07e7578a7,0x0003d915eb7cc48b,0x000cc58340724963,0x00028995bbad2d0f,0x0000fdeed650da95}}, + {{0x000b56d84457fa16,0x00061e11055dfd9f,0x000d8d3cf1cddb6c,0x000b08d4173a6b13,0x0000e1b2d83761d2}, {0x000efe79a7aed4b6,0x0009fc67b13ce887,0x000a8d47351b67cc,0x000cf5ffd958d4f0,0x00002c566abb452e}}, + {{0x0002367962f3f098,0x000b1784e6f456a0,0x000fc496b5d1b73a,0x000d88b82f0e3c6e,0x00005b2fe37d1f96}, {0x000de1a58c6ee495,0x0003ae7abc0c0aeb,0x000075de7d3903b0,0x00051633f3381ecc,0x00009c9d48206fa7}}, + {{0x0002ba62a80f39c6,0x0001c8cbe0887ea7,0x000bd1cc1cb2174b,0x000ce69c73ec69d6,0x0000a2dbe21520dc}, {0x000150beeaae9da5,0x000dedf630d9d0a5,0x000634cd8c935e85,0x000f022147144fa1,0x0000eccb56c94f3a}}, +}, +{/* digit=58 [{1,2,3,..,}]*([2^232]*G) */ + {{0x00079cfc0e2b70a5,0x000d3e8cbae7a77a,0x000db18fc2569c8b,0x0000ff392a5dbefa,0x00009bc96b48ce6a}, {0x00004f48b5510059,0x0006344b2bd7987f,0x000c447d77efe3aa,0x000dd00e9cb8ed6a,0x0000b4eb10b0783b}}, + {{0x0002acc711923485,0x000f8f17ef5fecc3,0x000e25b0e545401b,0x000ab9e209a493cd,0x0000c11886bb63ab}, {0x00081b128ec7c90b,0x000725b57f5c361a,0x000d9169618b125a,0x0007ff86d1b45afa,0x000031a786e54c3f}}, + {{0x000189659471f1f2,0x000d1630357339d2,0x000bf5a4b2e613de,0x000be5a799579478,0x00000adf6b5df19f}, {0x000c95f1574d34dd,0x0009d9323cea6a48,0x0002df9cf95488f0,0x000500450aee7f55,0x0000f016f7b23557}}, + {{0x0004c6f8d9fceb6f,0x00099b528f3748cf,0x000303b2b11e85f7,0x000605e2896d2552,0x0000929675bdd68c}, {0x000237410c708a9a,0x0007e0d7e5a75bd2,0x00041047f4682ca1,0x0008404242b5c590,0x0000f571053af9c6}}, + {{0x00066978e93edd55,0x000207ea4f453969,0x000d33d8735cdb8e,0x0002d636f8305dfe,0x00007623440b2564}, {0x000b580945dd7d69,0x0006031739bc4fd9,0x0007bf127965ffcb,0x0001c734588e1f63,0x000036c0ba0e39d2}}, + {{0x0006a599a93a5c94,0x000ce8d12f613d87,0x000b2aa43026be75,0x000dade49900ede9,0x00004d6dc80f3a68}, {0x000116b7d23e11bc,0x000376814209396b,0x000c659561279121,0x000fae3e2807cf6c,0x0000c606ca8b405f}}, + {{0x000017dc145a6c70,0x0002968798dab36c,0x0000eff5dcca6438,0x000681d13b63768d,0x00006e39e2d2206e}, {0x000ffa43add517af,0x000900d95fee2d9d,0x000c7c7a8e670e6c,0x0003050ecb51abf1,0x000017dff5264945}}, + {{0x00068f9e5137d20c,0x000acf7e70ac618e,0x000b86b0a79588cb,0x0001feb6d37f5258,0x0000b281c993cc59}, {0x0003fed8fe40e06c,0x0006ad9ca79270e0,0x000d3a3da394ded9,0x000130f1d22cddbc,0x0000b88cb27cc591}}, +}, +{/* digit=59 [{1,2,3,..,}]*([2^236]*G) */ + {{0x000eb34649966a56,0x000f86639e18c8c7,0x000ce0f3897587f4,0x00080a0724cc000b,0x00003578addaae62}, {0x000aa57c7fd6a1af,0x00046b017e3501be,0x000df2ed183b1a53,0x00085201c027e3ef,0x0000373d4ebc2d31}}, + {{0x000546827502203e,0x000538ade3161e8d,0x000687415985039d,0x000774efd373f10a,0x0000fccb79223526}, {0x000d46e0f4497d9a,0x0001b601ab9a4ef8,0x0007b2ad14152df7,0x0001804250cd2fe4,0x00002b63fa69b048}}, + {{0x0004ab81e6799c95,0x00006d148443a72e,0x000a73a1b4c0452b,0x0003dc4f3d6068b1,0x00003d967d92a918}, {0x0006305308773b32,0x000dd107d1c4a6ba,0x0002936f7c5535c5,0x000e7282492b6d5d,0x0000207d5e4da0f0}}, + {{0x000cb6f787d1f1c8,0x0004bd219a6658a6,0x0003146b0427bac9,0x00078151d7d49f38,0x0000164b77ff863d}, {0x000b0842f9631b8b,0x0009349388de6646,0x0008cd383ef5b3aa,0x0003a060536422e5,0x0000079d912a43ea}}, + {{0x000234ea69b16e89,0x000bc1d5e5a6f2c1,0x000b57835cef457b,0x000ddf906b1d3021,0x00008fccf893f260}, {0x000af0d389fa01e6,0x0009e0e60cc9a47b,0x000e5dd23920c8f4,0x0001cdf9674c7ce3,0x00003ba7e5f5ab84}}, + {{0x0003293983884693,0x0006f55208130d55,0x000c8b1b292a9779,0x0008e2c2bb329ebf,0x0000c6f15324e83e}, {0x000d963e4290a26e,0x00017cf40c04de61,0x000e829f7d35af4d,0x000a8efac154d763,0x00001d9debc16dc9}}, + {{0x00063e677bca7881,0x000e37d394580513,0x00024e144be8d0ae,0x00028e4dbde8185b,0x000023188f4e65e5}, {0x0008b9cefe44eb7f,0x00022e76be281a4f,0x00062f74431f9f1a,0x00073701910bc8e4,0x00002e6f9dbd037a}}, + {{0x000c041cb73887ee,0x0009a3ce3a32704a,0x000393e75f878b61,0x00008c57ef73d556,0x00004372d2fb276c}, {0x000c8940924cf58a,0x000c4aa317e25d9b,0x000608da5fa2a4de,0x000960e51edccc79,0x0000dcc68fb2cd4b}}, +}, +{/* digit=60 [{1,2,3,..,}]*([2^240]*G) */ + {{0x0004f09a9e0eeaed,0x000f7b05a6569f15,0x00045b85d2246e6f,0x0004324d7c1c8110,0x0000e99ea38a3bb7}, {0x000818763184ff43,0x00020134bfc2d58f,0x0001203202a22342,0x000e9c1560dbed23,0x00007243c95a6a3d}}, + {{0x0006c8fe256b02b0,0x0003fa5946e09f11,0x00035452a7149569,0x00062eeb9696ffc3,0x00001ca592949711}, {0x0001f50c0f28e720,0x000c70d8df1aee0a,0x0009110f82baac62,0x00016acf65d297f4,0x000041dbb019a45e}}, + {{0x0009223d39e1ee45,0x000c9e3e6c2bd0b1,0x000a45c34c8832a2,0x00091b64a8f43da3,0x00002a05eef71fb2}, {0x000e24ae4b68e385,0x000b3e2d8a3fc0d3,0x0005b7da85289120,0x0004e133836b9842,0x0000bd418f4000c6}}, + {{0x0002241cedc2c975,0x000b7b632640e1a0,0x000fb588ff0f5489,0x0008e7b09091ddcb,0x0000ffd0f38ad947}, {0x00041a1dae35eda9,0x0006f2f0b26b83f1,0x0009dde18d25563a,0x000b3680014b171b,0x0000fcf8f820da49}}, + {{0x000380540096f25a,0x000e8c8201311e99,0x000993f9c900d4bd,0x0000072170cfd32a,0x00000e3d894cdfe1}, {0x0000b5a0e7df1098,0x000ac7fde3dce00d,0x00089816ac904985,0x000b9e15597a84cb,0x0000ac8b02839dfe}}, + {{0x000a4fdf7a2df835,0x000bb4524d444e3e,0x0006aee218b68b26,0x000e1c74caeeab12,0x000090a00a5e15d9}, {0x000a6ab49b90eff5,0x000e7df4fe510ae2,0x000fcb6ed74b4cb1,0x000b300306ed1107,0x000064ebe2ea02f5}}, + {{0x000a0c3c0a9d128d,0x00047ed0d3bce0f7,0x000e778e62adc34d,0x0006ba4ebf577813,0x00003b89bd18b847}, {0x000b52837413953c,0x000d98ba3471209e,0x000a79c09952b705,0x000b2acaa81ade86,0x000008eed3e3e0e7}}, + {{0x000c2bc9a8f42ff5,0x00009a1c0299dea0,0x0004dd41b6297424,0x0002495ee5f5f69e,0x0000b1bba807e110}, {0x000eadb78da80167,0x00008809f79eeac4,0x000f8c0dec297878,0x000ba939d2dbcee3,0x0000fb4b5fc764d3}}, +}, +{/* digit=61 [{1,2,3,..,}]*([2^244]*G) */ + {{0x000179c7f034ff4c,0x0009f781eac1a607,0x000022138efb8fcd,0x000039fa57f8a97b,0x00005bb9f1e16ab6}, {0x0007e24e4d2ab7f7,0x0001e7a9e3648902,0x0007f487d3d67ad7,0x000bd0c579e70c1f,0x0000fefc8949a7e6}}, + {{0x0001662a45cfd31b,0x0003c6f65cfd7cb9,0x000f3de5109dd56d,0x0004fc983e005d14,0x00009dc05b0d10f6}, {0x0000afd885eafe56,0x0003d444bdebc279,0x00087300abd5213d,0x000b76289dca9289,0x00009fb2ac313960}}, + {{0x0008bb43e4e99c48,0x0006f1f5ac02913a,0x00083f958b0c5fed,0x000eb65c5ccabdd7,0x0000d10225585a25}, {0x00075bc2701b5ef5,0x000f34ead28ab53b,0x000d37ba17cdfbb9,0x0007b8a111be6984,0x0000324f41a75991}}, + {{0x00048869ae7540fb,0x00021131e9213327,0x000a360c8d738663,0x0001e02e3d4fd8f2,0x000020a59b6ed9d4}, {0x0007eae99082a34c,0x000e1181946fb2b6,0x000d1d007fad6aa7,0x000ed07c299b9aa2,0x00001f841e14100b}}, + {{0x0008bf21619db767,0x000be5dbcdc6ba04,0x00053cfdca928899,0x00089daaf1e8f8a0,0x0000e49dcc581266}, {0x000001ccbf14e0ef,0x0002592029f13992,0x0002972a8aefbe02,0x000c66a3c0cc8691,0x00008495b3926f9a}}, + {{0x0007d4c43e90ebdf,0x00080fc72b063c1f,0x00094f40258b7810,0x0004b0da8503e1af,0x0000bb724b849f24}, {0x0002186fcd8755a4,0x000ec68482b6efcd,0x0009be3d57407cde,0x00002c4d62f57834,0x0000a0125451cc62}}, + {{0x00018c779f099ea8,0x0009290f6a520a5a,0x00025d678dfefc64,0x000de97c3b4cc277,0x0000d87ef5ce132b}, {0x000ee07a7023e899,0x000f309c8784c042,0x000c1268d9681a7d,0x0002c2fa00007a86,0x0000f76f114b2bf8}}, + {{0x00014d2151ffc085,0x0000c40d6b5568a4,0x000f472abaa79acf,0x000f1deeab0104cd,0x0000014a8c1f3aa5}, {0x000340533f134256,0x0009b7eb54d42c74,0x0008a723b2b776b4,0x00063a3a0cc4ac54,0x00005aae6f4279fe}}, +}, +{/* digit=62 [{1,2,3,..,}]*([2^248]*G) */ + {{0x0007ef9dfa782a41,0x000184cd95006944,0x00038f7626dadc8e,0x000a140574020edc,0x00007596d7ef2ee7}, {0x0005af79e1f8adc5,0x0006f4791da04ef7,0x0007b5d805ac5f21,0x0006cc1583226b7b,0x00009f3f053f21c2}}, + {{0x000abd049c879a59,0x0008347ac42e5e16,0x0009c2326844874a,0x00033bee3f8a203c,0x00009a120556eaed}, {0x000a15b63b333ae3,0x0005cd923fa01814,0x000b1b1efee9f28a,0x0009285b0cd25033,0x0000ccc39b9b346d}}, + {{0x0005e1d161e13e79,0x0001207e7068bb79,0x000522fe70f9abc2,0x00034efb3be61854,0x0000e2d0f37a9381}, {0x0007c36d292c6b0e,0x000da1e77894d8dc,0x0006c55f3bafbf59,0x0000877d0132cd1b,0x0000fa02edad609f}}, + {{0x0009359d93f6e1a6,0x00049d41c11898ea,0x000b641af729005d,0x000e784c2934107c,0x0000b2b4671e95e8}, {0x00051d05958fad35,0x000ec8619fe3da12,0x00011d318b69bc2b,0x000704d74df34cd9,0x0000def837865102}}, + {{0x00058678c673c89c,0x0006c99c7bae39ff,0x000b1e1b64925a04,0x00084f0b3bf59adb,0x00007e9d9f2f0ae0}, {0x00023960fa1776f5,0x000fb24c897b2098,0x0003534d5b2e1b98,0x000f63a9a6407d6c,0x0000e22319bf4dc3}}, + {{0x0006a2e916ef5271,0x0000393dc0853d90,0x00091da3eadeb94d,0x0002eb03a0780114,0x000077dceae1b84d}, {0x000a3c17b691e0c9,0x00086172cea2d1e5,0x000a76bce47d40bd,0x00035f7d0646ad8c,0x00000b030aa564d6}}, + {{0x00037b31f6e396ce,0x000f29a782f2d083,0x000c9b9746166b21,0x000d3d1ac6198b8e,0x0000e2e3446ebb44}, {0x000374035039dd98,0x0005a9f5c6925b28,0x000ddb4667c708f9,0x0003cf8914cce098,0x0000bb1b9f25446f}}, + {{0x00075c2a2c539e41,0x000db756daf9d315,0x000f55a120bac5dc,0x000299e917cecf91,0x0000e96433bf96f6}, {0x000c71c3700d8fb3,0x00060c9b4444bec7,0x00074f19c9a1d296,0x000d6b3d2c6970cf,0x0000b444c48dc5e0}}, +}, +{/* digit=63 [{1,2,3,..,}]*([2^252]*G) */ + {{0x000cefd8ccb854c3,0x000b95b0c45ab563,0x00037f743f5452cd,0x0004abb3c787699c,0x00004e9d192c5d44}, {0x000794652ff26b7c,0x000b5b94d6416934,0x000affb8f70d6ecf,0x000c717d201858fd,0x0000288719e05dcd}}, + {{0x000469d0728a2eb7,0x000f3433d11c5695,0x00006c08e7b46244,0x0000834a8b99baf1,0x0000989794fd3422}, {0x0008867d4fc5696b,0x0008ec79cdb7d221,0x0006d50496b021f2,0x0006285ff7bbeab2,0x000078611cb57261}}, + {{0x000f961531313d77,0x000bd6dcdc9dea75,0x00060e99185a1f4d,0x0007ccae3026b964,0x0000d467bef87ecf}, {0x000bf6705118708a,0x000993b2f1c8ea0d,0x000c0e4e054bfa36,0x00011ba9fc9d5cf2,0x00008c2ad11fe936}}, + {{0x000faf0aa1256bd2,0x0005be4631de1ef1,0x0009c26760f22454,0x000b2469cb9800de,0x00006019816b5782}, {0x000172ca66c0ccdc,0x000674407199d45c,0x0009f681a6c25f63,0x000df4917d5dbaa4,0x00000cad04872dc5}}, + {{0x000cf4c5960ef1c9,0x000e1c6979d5445b,0x0009090d6babcb16,0x0007288e3be750ae,0x0000481d2623c0eb}, {0x00036cd0d6a7d46e,0x0003ff97650066b4,0x000e1064f6eb1a6a,0x000c205984ffa2db,0x00006575fb2d809d}}, + {{0x0006d804d974a81a,0x000037a74be65042,0x00048653161304f1,0x00081a2346ff98a9,0x00001242ccb953d7}, {0x00003df97355f15a,0x00045d6058cea82f,0x000aefe28c607ed3,0x0007e403bc8cd468,0x00006e7de5b25130}}, + {{0x000df0fc6af7d445,0x0000015a9187dc07,0x000e15c2f310b252,0x000313d42661ced3,0x0000198fd906b4d8}, {0x00062cdda8368a13,0x00018e9c25424a60,0x000e37cee1a905d1,0x0004ca1d752b70da,0x0000ed8c1a546bf8}}, + {{0x000fb0feecc2f22b,0x0004198d8e5fa190,0x000f3ce723df210f,0x000fc6cce57d3af5,0x00002fb6223e12b8}, {0x000700571867c84a,0x000d5e139ebdd994,0x000e7944e141cd92,0x000058415efc9e5d,0x0000e9ee919e5821}}, +}, +{/* digit=64 [{1,2,3,..,}]*([2^256]*G) */ + {{0x0001a6a32f9eaf5b,0x00055cfc13dbbf7b,0x000b18adf2c440f9,0x00033f2cf39bc566,0x00009939fe94f015}, {0x0004553383a64506,0x000450102086a31c,0x0001f136016d96ad,0x000c08cbd6fa9550,0x000067d3ea0c5f96}}, + {{0x000c13ba12fdf41d,0x000c5224f5d1fa1f,0x000d5872bd798c04,0x000da822f4594e1d,0x0000ee12df5ebddf}, {0x0003ff0ed83420ad,0x00085daa27f3c647,0x000cefd8af41cf1c,0x00047f2772cd56ae,0x00005ddaf191902b}}, + {{0x0005abf53e86ae11,0x00004850830d88e0,0x0006bc33cc1dc4d9,0x00086fbd7fd806e0,0x0000ac330d6bcf12}, {0x0002303e1588c1e5,0x000b6b7e9c18d8ce,0x0007149f2dc25e54,0x000a2511e51e494b,0x000051b839205c7f}}, + {{0x0005ddf1510a406a,0x000569f0e98c83fc,0x0002843a0475a78f,0x0005a6b6d681c4e7,0x000090af2a4ea00c}, {0x0006d45a34f44127,0x000357f5d70325fc,0x00011092560f9c0e,0x0004d52bc0642bad,0x00009abfc112e24a}}, + {{0x000ee0691cf6cf09,0x0001b6b6bb14f844,0x000f77954e05b5ed,0x000619049c4bbcb7,0x0000d0f82198d94a}, {0x000e53617b8a0fde,0x000a78a163c77369,0x0004efcc52f1c014,0x000f9ea95975cd72,0x0000be1ab00c9aa9}}, + {{0x0000621245e20bef,0x0009134bbfc574fd,0x0007b3b6835a502d,0x0005cd40b56e370b,0x0000ca24ac75641f}, {0x00049d7102ae1815,0x00001fe146bde29a,0x0005146a8d2205d3,0x0009632a432de4b8,0x0000f90d9d766c61}}, + {{0x0004e18f143aa0fa,0x000641cf03d329bd,0x000345f4cf3894c5,0x00089dd25d055813,0x0000da5c0a11d580}, {0x000b9b65ddf3b293,0x0005e8afdfa60988,0x0006ebe1439c3438,0x000d381e3c991790,0x0000c9efd259ca95}}, + {{0x0009ce39b3d6844b,0x00093320f9bee042,0x0007dc2d2af562e9,0x0005ea5fbc2c5d71,0x00006fa79ff0722f}, {0x00070a079eea8d96,0x00019377e5ee65aa,0x000ad14cb624a2b9,0x000ac930ec06eaf7,0x00008d07bdd35640}}, +}, +}; + +#endif /* IFMA_ECPRECOMP4_PSM2_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm3/sm3_common.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm3/sm3_common.h new file mode 100644 index 0000000..e90fdf5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm3/sm3_common.h @@ -0,0 +1,207 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#if !defined(_SM3_COMMON_H) +#define _SM3_COMMON_H + +#include +#include + +#include + +#define SM3_MSG_LEN_REPR (sizeof(int64u)) /* size of processed message length representation (bytes) */ + +#ifndef M256 + #define M256(mem) (*((__m256i*)(mem))) +#endif + +#ifndef M512 + #define M512(mem) (*((__m512i*)(mem))) +#endif + +#ifndef MIN + #define MIN(a, b) ( ((a) < (b)) ? a : b ) +#endif + +/* +// accessors to context's fields +*/ + +#define MSG_LEN(ctx) ((ctx)->msg_len) +#define HASH_VALUE(ctx) ((ctx)->msg_hash) +#define HASH_BUFFIDX(ctx) ((ctx)->msg_buff_idx) +#define HASH_BUFF(ctx) ((ctx)->msg_buffer) + +/* +// constants +*/ + +static const int32u sm3_iv[] = { 0x7380166F, 0x4914B2B9, 0x172442D7, 0xDA8A0600, + 0xA96F30BC, 0x163138AA, 0xE38DEE4D, 0xB0FB0E4E }; + +__ALIGN64 static const int8u swapBytes[] = { 7,6,5,4, 3,2,1,0, 15,14,13,12, 11,10,9,8, + 7,6,5,4, 3,2,1,0, 15,14,13,12, 11,10,9,8, + 7,6,5,4, 3,2,1,0, 15,14,13,12, 11,10,9,8, + 7,6,5,4, 3,2,1,0, 15,14,13,12, 11,10,9,8 }; + +__ALIGN64 static const int32u tj_calculated[] = { 0x79CC4519,0xF3988A32,0xE7311465,0xCE6228CB,0x9CC45197,0x3988A32F,0x7311465E,0xE6228CBC, + 0xCC451979,0x988A32F3,0x311465E7,0x6228CBCE,0xC451979C,0x88A32F39,0x11465E73,0x228CBCE6, + 0x9D8A7A87,0x3B14F50F,0x7629EA1E,0xEC53D43C,0xD8A7A879,0xB14F50F3,0x629EA1E7,0xC53D43CE, + 0x8A7A879D,0x14F50F3B,0x29EA1E76,0x53D43CEC,0xA7A879D8,0x4F50F3B1,0x9EA1E762,0x3D43CEC5, + 0x7A879D8A,0xF50F3B14,0xEA1E7629,0xD43CEC53,0xA879D8A7,0x50F3B14F,0xA1E7629E,0x43CEC53D, + 0x879D8A7A,0x0F3B14F5,0x1E7629EA,0x3CEC53D4,0x79D8A7A8,0xF3B14F50,0xE7629EA1,0xCEC53D43 }; + +/* +// internal functions +*/ + + +__INLINE void pad_block(int8u padding_byte, void* dst_p, int num_bytes) +{ + int8u* d = (int8u*)dst_p; + int k; + for(k = 0; k < num_bytes; k++ ) + d[k] = padding_byte; +} + +__INLINE void TRANSPOSE_8X8_I32(__m256i *v0, __m256i *v1, __m256i *v2, __m256i *v3, + __m256i *v4, __m256i *v5, __m256i *v6, __m256i *v7) +{ + __m256i w0, w1, w2, w3, w4, w5, w6, w7; + __m256i x0, x1, x2, x3, x4, x5, x6, x7; + __m256i t1, t2; + + x0 = _mm256_permute4x64_epi64(*v0, 0b11011000); + x1 = _mm256_permute4x64_epi64(*v1, 0b11011000); + w0 = _mm256_unpacklo_epi32(x0, x1); + w1 = _mm256_unpackhi_epi32(x0, x1); + + x2 = _mm256_permute4x64_epi64(*v2, 0b11011000); + x3 = _mm256_permute4x64_epi64(*v3, 0b11011000); + w2 = _mm256_unpacklo_epi32(x2, x3); + w3 = _mm256_unpackhi_epi32(x2, x3); + + x4 = _mm256_permute4x64_epi64(*v4, 0b11011000); + x5 = _mm256_permute4x64_epi64(*v5, 0b11011000); + w4 = _mm256_unpacklo_epi32(x4, x5); + w5 = _mm256_unpackhi_epi32(x4, x5); + + x6 = _mm256_permute4x64_epi64(*v6, 0b11011000); + x7 = _mm256_permute4x64_epi64(*v7, 0b11011000); + w6 = _mm256_unpacklo_epi32(x6, x7); + w7 = _mm256_unpackhi_epi32(x6, x7); + + t1 = _mm256_permute4x64_epi64(w0, 0b11011000); + t2 = _mm256_permute4x64_epi64(w2, 0b11011000); + x0 = _mm256_unpacklo_epi64(t1, t2); + x1 = _mm256_unpackhi_epi64(t1, t2); + + t1 = _mm256_permute4x64_epi64(w1, 0b11011000); + t2 = _mm256_permute4x64_epi64(w3, 0b11011000); + x2 = _mm256_unpacklo_epi64(t1, t2); + x3 = _mm256_unpackhi_epi64(t1, t2); + + t1 = _mm256_permute4x64_epi64(w4, 0b11011000); + t2 = _mm256_permute4x64_epi64(w6, 0b11011000); + x4 = _mm256_unpacklo_epi64(t1, t2); + x5 = _mm256_unpackhi_epi64(t1, t2); + + t1 = _mm256_permute4x64_epi64(w5, 0b11011000); + t2 = _mm256_permute4x64_epi64(w7, 0b11011000); + x6 = _mm256_unpacklo_epi64(t1, t2); + x7 = _mm256_unpackhi_epi64(t1, t2); + + *v0 = _mm256_permute2x128_si256(x0, x4, 0b100000); + *v1 = _mm256_permute2x128_si256(x0, x4, 0b110001); + *v2 = _mm256_permute2x128_si256(x1, x5, 0b100000); + *v3 = _mm256_permute2x128_si256(x1, x5, 0b110001); + *v4 = _mm256_permute2x128_si256(x2, x6, 0b100000); + *v5 = _mm256_permute2x128_si256(x2, x6, 0b110001); + *v6 = _mm256_permute2x128_si256(x3, x7, 0b100000); + *v7 = _mm256_permute2x128_si256(x3, x7, 0b110001); +} + +__INLINE void MASK_TRANSPOSE_8X8_I32(int32u* out[8], const int32u* const inp[8], __mmask16 mb_mask) { + __m256i v0 = _mm256_loadu_si256((__m256i*)inp[0]); + __m256i v1 = _mm256_loadu_si256((__m256i*)inp[1]); + __m256i v2 = _mm256_loadu_si256((__m256i*)inp[2]); + __m256i v3 = _mm256_loadu_si256((__m256i*)inp[3]); + __m256i v4 = _mm256_loadu_si256((__m256i*)inp[4]); + __m256i v5 = _mm256_loadu_si256((__m256i*)inp[5]); + __m256i v6 = _mm256_loadu_si256((__m256i*)inp[6]); + __m256i v7 = _mm256_loadu_si256((__m256i*)inp[7]); + + TRANSPOSE_8X8_I32(&v0, &v1, &v2, &v3, &v4, &v5, &v6, &v7); + + /* mask store hashes to the first 8 buffers */ + _mm256_mask_storeu_epi32((void*)out[0], (__mmask8)(((mb_mask >> 0) & 1)) * 0xFF, v0); + _mm256_mask_storeu_epi32((void*)out[1], (__mmask8)(((mb_mask >> 1) & 1)) * 0xFF, v1); + _mm256_mask_storeu_epi32((void*)out[2], (__mmask8)(((mb_mask >> 2) & 1)) * 0xFF, v2); + _mm256_mask_storeu_epi32((void*)out[3], (__mmask8)(((mb_mask >> 3) & 1)) * 0xFF, v3); + _mm256_mask_storeu_epi32((void*)out[4], (__mmask8)(((mb_mask >> 4) & 1)) * 0xFF, v4); + _mm256_mask_storeu_epi32((void*)out[5], (__mmask8)(((mb_mask >> 5) & 1)) * 0xFF, v5); + _mm256_mask_storeu_epi32((void*)out[6], (__mmask8)(((mb_mask >> 6) & 1)) * 0xFF, v6); + _mm256_mask_storeu_epi32((void*)out[7], (__mmask8)(((mb_mask >> 7) & 1)) * 0xFF, v7); + +} + +__INLINE void TRANSPOSE_8X16_I32(int32u* out[16], const int32u* const inp[8], __mmask16 mb_mask) { + __m256i v0 = _mm256_loadu_si256((__m256i*)inp[0]); + __m256i v1 = _mm256_loadu_si256((__m256i*)inp[1]); + __m256i v2 = _mm256_loadu_si256((__m256i*)inp[2]); + __m256i v3 = _mm256_loadu_si256((__m256i*)inp[3]); + __m256i v4 = _mm256_loadu_si256((__m256i*)inp[4]); + __m256i v5 = _mm256_loadu_si256((__m256i*)inp[5]); + __m256i v6 = _mm256_loadu_si256((__m256i*)inp[6]); + __m256i v7 = _mm256_loadu_si256((__m256i*)inp[7]); + + TRANSPOSE_8X8_I32(&v0, &v1, &v2, &v3, &v4, &v5, &v6, &v7); + + /* mask store hashes to the first 8 buffers */ + _mm256_mask_storeu_epi32((void*)out[0], (__mmask8)(((mb_mask >> 0) & 1)) * 0xFF, v0); + _mm256_mask_storeu_epi32((void*)out[1], (__mmask8)(((mb_mask >> 1) & 1)) * 0xFF, v1); + _mm256_mask_storeu_epi32((void*)out[2], (__mmask8)(((mb_mask >> 2) & 1)) * 0xFF, v2); + _mm256_mask_storeu_epi32((void*)out[3], (__mmask8)(((mb_mask >> 3) & 1)) * 0xFF, v3); + _mm256_mask_storeu_epi32((void*)out[4], (__mmask8)(((mb_mask >> 4) & 1)) * 0xFF, v4); + _mm256_mask_storeu_epi32((void*)out[5], (__mmask8)(((mb_mask >> 5) & 1)) * 0xFF, v5); + _mm256_mask_storeu_epi32((void*)out[6], (__mmask8)(((mb_mask >> 6) & 1)) * 0xFF, v6); + _mm256_mask_storeu_epi32((void*)out[7], (__mmask8)(((mb_mask >> 7) & 1)) * 0xFF, v7); + + v0 = _mm256_loadu_si256((__m256i*)inp[0] + 1); + v1 = _mm256_loadu_si256((__m256i*)inp[1] + 1); + v2 = _mm256_loadu_si256((__m256i*)inp[2] + 1); + v3 = _mm256_loadu_si256((__m256i*)inp[3] + 1); + v4 = _mm256_loadu_si256((__m256i*)inp[4] + 1); + v5 = _mm256_loadu_si256((__m256i*)inp[5] + 1); + v6 = _mm256_loadu_si256((__m256i*)inp[6] + 1); + v7 = _mm256_loadu_si256((__m256i*)inp[7] + 1); + + + TRANSPOSE_8X8_I32(&v0, &v1, &v2, &v3, &v4, &v5, &v6, &v7); + + /* mask store hashes to the last 8 buffers */ + _mm256_mask_storeu_epi32((void*)out[8], (__mmask8)(((mb_mask >> 8) & 1)) * 0xFF, v0); + _mm256_mask_storeu_epi32((void*)out[9], (__mmask8)(((mb_mask >> 9) & 1)) * 0xFF, v1); + _mm256_mask_storeu_epi32((void*)out[10], (__mmask8)(((mb_mask >> 10) & 1)) * 0xFF, v2); + _mm256_mask_storeu_epi32((void*)out[11], (__mmask8)(((mb_mask >> 11) & 1)) * 0xFF, v3); + _mm256_mask_storeu_epi32((void*)out[12], (__mmask8)(((mb_mask >> 12) & 1)) * 0xFF, v4); + _mm256_mask_storeu_epi32((void*)out[13], (__mmask8)(((mb_mask >> 13) & 1)) * 0xFF, v5); + _mm256_mask_storeu_epi32((void*)out[14], (__mmask8)(((mb_mask >> 14) & 1)) * 0xFF, v6); + _mm256_mask_storeu_epi32((void*)out[15], (__mmask8)(((mb_mask >> 15) & 1)) * 0xFF, v7); +} + +#endif /* _SM3_COMMON_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm3/sm3_mb16.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm3/sm3_mb16.h new file mode 100644 index 0000000..5412f7c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm3/sm3_mb16.h @@ -0,0 +1,43 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#if !defined(_SM3_MB16_H) +#define _SM3_MB16_H + +#include +#include + +#include + +/* +// change endian +*/ +static __ALIGN64 const int8u swapBytesCtx[] = { 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12, + 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12, + 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12, + 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12 }; + +#define SIMD_ENDIANNESS32(x) _mm512_shuffle_epi8((x), M512(swapBytesCtx)); + +/* +// internal functions +*/ + +EXTERN_C void sm3_avx512_mb16(int32u hash_pa[][16], const int8u* const msg_pa[16], int len[16]); +EXTERN_C void sm3_mask_init_mb16(SM3_CTX_mb16* p_state, unsigned short mb_mask); + +#endif /* _SM3_MB16_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm3/sm3_mb8.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm3/sm3_mb8.h new file mode 100644 index 0000000..c61231f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm3/sm3_mb8.h @@ -0,0 +1,59 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#if !defined(_SM3_MB8_H) +#define _SM3_MB8_H + +#include +#include + +#include + +/* +// change endian +*/ + +static __ALIGN64 const int8u swapBytesCtx[] = { 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12, + 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12 }; + +#define SIMD_ENDIANNESS32(x) _mm256_shuffle_epi8((x), M256(swapBytesCtx)); +#define SM3_NUM_BUFFERS8 (8) /* max number of buffers in sm3 multi-buffer 8 */ + +typedef int32u sm3_hash_mb8[SM3_SIZE_IN_WORDS][SM3_NUM_BUFFERS8]; /* sm3 hash value in multi-buffer 8 format */ +struct _sm3_context_mb8 { + int msg_buff_idx[SM3_NUM_BUFFERS8]; /* buffer entry */ + int64u msg_len[SM3_NUM_BUFFERS8]; /* message length */ + int8u msg_buffer[SM3_NUM_BUFFERS8][SM3_MSG_BLOCK_SIZE]; /* buffer */ + __ALIGN64 + sm3_hash_mb8 msg_hash; /* intermediate hash */ +}; + +typedef struct _sm3_context_mb8 SM3_CTX_mb8; + +/* +// internal functions +*/ + +EXTERN_C mbx_status sm3_init_mb8(SM3_CTX_mb8* p_state); +EXTERN_C mbx_status sm3_update_mb8(const int8u* const msg_pa[8], int len[8], SM3_CTX_mb8* p_state); +EXTERN_C mbx_status sm3_final_mb8(int8u* hash_pa[8], SM3_CTX_mb8* p_state); +EXTERN_C mbx_status sm3_msg_digest_mb8(const int8u* const msg_pa[8], int len[8], int8u* hash_pa[8]); + +EXTERN_C void sm3_avx512_mb8(int32u hash_pa[][8], const int8u* const msg_pa[8], int len[8]); +EXTERN_C void sm3_mask_init_mb8(SM3_CTX_mb8 * p_state, __mmask8 mb_mask); + +#endif /* _SM3_MB8_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm4/sm4_ccm_mb.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm4/sm4_ccm_mb.h new file mode 100644 index 0000000..7d7e15d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm4/sm4_ccm_mb.h @@ -0,0 +1,130 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include + +#include + +#ifndef SM4_CCM_MB_H +#define SM4_CCM_MB_H + +///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/* +// Internal functions +*/ + +/* + * Set IV for 16 buffers in SM4-CCM context + * + * @param[in] pa_iv Array of IVs + * @param[in] iv_len Array of IV lengths + * @param[in] mb_mask Bitmask selecting which lines to update + * @param[in/out] p_context SM4-CCM context + * + */ +EXTERN_C void sm4_ccm_update_iv_mb16(const int8u *const pa_iv[SM4_LINES], + const int iv_len[SM4_LINES], + __mmask16 mb_mask, + SM4_CCM_CTX_mb16 *p_context); +/* + * Set message lengths for 16 buffers in SM4-CCM context + * + * @param[in] msg_len Array of total message lengths + * @param[in] mb_mask Bitmask selecting which lines to update + * @param[in/out] p_context SM4-CCM context + * + */ +EXTERN_C void sm4_ccm_set_msg_len_mb16(const int64u msg_len[SM4_LINES], + __mmask16 mb_mask, + SM4_CCM_CTX_mb16 *p_context); + +/* + * Set authentication tag lengths for 16 buffers in SM4-CCM context + * + * @param[in] tag_len Array of authentication tag lengths + * @param[in] mb_mask Bitmask selecting which lines to update + * @param[in/out] p_context SM4-CCM context + * + */ +EXTERN_C void sm4_ccm_set_tag_len_mb16(const int tag_len[SM4_LINES], + __mmask16 mb_mask, + SM4_CCM_CTX_mb16 *p_context); + +EXTERN_C void sm4_ccm_update_aad_mb16(const int8u *const pa_aad[SM4_LINES], + const int aad_len[SM4_LINES], + __mmask16 mb_mask, + SM4_CCM_CTX_mb16 *p_context); + +EXTERN_C void sm4_ccm_encrypt_mb16(int8u *pa_out[SM4_LINES], + const int8u *const pa_in[SM4_LINES], + const int in_len[SM4_LINES], + __mmask16 mb_mask, + SM4_CCM_CTX_mb16 *p_context); + +EXTERN_C void sm4_ccm_decrypt_mb16(int8u *pa_out[SM4_LINES], + const int8u *const pa_in[SM4_LINES], + const int in_len[SM4_LINES], + __mmask16 mb_mask, + SM4_CCM_CTX_mb16 *p_context); + +EXTERN_C void sm4_ccm_get_tag_mb16(int8u *pa_out[SM4_LINES], const int tag_len[SM4_LINES], __mmask16 mb_mask, SM4_CCM_CTX_mb16 *p_context); + +///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/* Context accessors */ + +#define SM4_CCM_CONTEXT_MSG_LEN(context) ((context)->msg_len) +#define SM4_CCM_CONTEXT_PROCESSED_LEN(context) ((context)->total_processed_len) +#define SM4_CCM_CONTEXT_TAG_LEN(context) ((context)->tag_len) +#define SM4_CCM_CONTEXT_IV_LEN(context) ((context)->iv_len) +#define SM4_CCM_CONTEXT_CTR0(context) ((context)->ctr0) +#define SM4_CCM_CONTEXT_CTR(context) ((context)->ctr) +#define SM4_CCM_CONTEXT_HASH(context) ((context)->hash) + +#define SM4_CCM_CONTEXT_KEY(context) (&((context)->key_sched)) +#define SM4_CCM_CONTEXT_STATE(context) ((context)->state) + +/* Calculate offsets for acessing blocks in buffers */ + +#define REG_SIZE_BITS (512) +#define REG_SIZE_BYTES (REG_SIZE_BITS / 8) /* Register size in bytes */ + +#define SLOTS_PER_BLOCK (SM4_BLOCK_SIZE / SM4_CCM_CONTEXT_BUFFER_SLOT_SIZE_BYTES) +#define BLOCKS_PER_REG (REG_SIZE_BYTES / SM4_BLOCK_SIZE) + +#define BUFFER_BLOCK_NUM(buffer, n) (buffer + SLOTS_PER_BLOCK * n) +#define BUFFER_REG_NUM(buffer, n) (buffer + SLOTS_PER_BLOCK * BLOCKS_PER_REG * n) + +/* Internal macroses */ + +#define sm4_ccm_clear_buffer(p_buffer) storeu((void *)(p_buffer), setzero()); + +#define SM4_CCM_CLEAR_BUFFER(p_buffer) \ + { \ + sm4_ccm_clear_buffer(BUFFER_REG_NUM(p_buffer, 0)); \ + sm4_ccm_clear_buffer(BUFFER_REG_NUM(p_buffer, 1)); \ + sm4_ccm_clear_buffer(BUFFER_REG_NUM(p_buffer, 2)); \ + sm4_ccm_clear_buffer(BUFFER_REG_NUM(p_buffer, 3)); \ + } + +#define SM4_CCM_CLEAR_LEN(p_len) \ + { \ + sm4_ccm_clear_buffer(p_len); \ + } + +#endif // SM4_CCM_MB_H diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm4/sm4_gcm_mb.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm4/sm4_gcm_mb.h new file mode 100644 index 0000000..c1659c7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm4/sm4_gcm_mb.h @@ -0,0 +1,176 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include + +#include + +#ifndef SM4_GCM_MB_H +#define SM4_GCM_MB_H + +///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +/* +// Constant from NIST Special Publication 800-38D +// (Recommendation for GCMmode, p.5.2.1.1 Input Data) +// len(P) <= 2^39-256 bits +*/ +static const int64u MAX_TXT_LEN = ((int64u)1 << 36) - 32; // length in bytes + +/* +// Internal functions +*/ + +EXTERN_C void sm4_gcm_ghash_mul_single_block_mb16(__m512i *data_blocks[], __m512i *hashkeys[]); + +EXTERN_C void sm4_gcm_update_ghash_full_blocks_mb16(__m128i ghash[SM4_LINES], + const int8u *const pa_input[SM4_LINES], + __m512i *input_len, + __m128i hashkey[SM4_GCM_HASHKEY_PWR_NUM][SM4_LINES], + __mmask16 mb_mask); + +EXTERN_C void sm4_gcm_update_ghash_partial_blocks_mb16(__m128i ghash[SM4_LINES], + const int8u *const pa_input[SM4_LINES], + __m512i *input_len, + __m128i hashkey[SM4_LINES], + __mmask16 mb_mask); + +EXTERN_C void sm4_gcm_precompute_hashkey_mb16(const mbx_sm4_key_schedule *key_sched, SM4_GCM_CTX_mb16 *p_context); + +EXTERN_C __mmask16 sm4_gcm_update_iv_mb16(const int8u *const pa_iv[SM4_LINES], + const int iv_len[SM4_LINES], + __mmask16 mb_mask, + SM4_GCM_CTX_mb16 *p_context); + +EXTERN_C void sm4_gcm_finalize_iv_mb16(const int8u *const pa_iv[SM4_LINES], __mmask16 mb_mask, SM4_GCM_CTX_mb16 *p_context); + +EXTERN_C __mmask16 sm4_gcm_update_aad_mb16(const int8u *const pa_aad[SM4_LINES], + const int aad_len[SM4_LINES], + __mmask16 mb_mask, + SM4_GCM_CTX_mb16 *p_context); + +EXTERN_C void sm4_encrypt_j0_mb16(SM4_GCM_CTX_mb16 *p_context); + +EXTERN_C void sm4_gctr_kernel_mb16(int8u *pa_out[SM4_LINES], + const int8u *const pa_inp[SM4_LINES], + const int len[SM4_LINES], + const int32u *key_sched[SM4_ROUNDS], + __mmask16 mb_mask, + SM4_GCM_CTX_mb16 *p_context); + +EXTERN_C __mmask16 sm4_gcm_encrypt_mb16(int8u *pa_out[SM4_LINES], + const int8u *const pa_in[SM4_LINES], + const int in_len[SM4_LINES], + __mmask16 mb_mask, + SM4_GCM_CTX_mb16 *p_context); + +EXTERN_C __mmask16 sm4_gcm_decrypt_mb16(int8u *pa_out[SM4_LINES], + const int8u *const pa_in[SM4_LINES], + const int in_len[SM4_LINES], + __mmask16 mb_mask, + SM4_GCM_CTX_mb16 *p_context); + +EXTERN_C void sm4_gcm_get_tag_mb16(int8u *pa_out[SM4_LINES], const int tag_len[SM4_LINES], __mmask16 mb_mask, SM4_GCM_CTX_mb16 *p_context); + +///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/* Context acessors */ + +#define SM4_GCM_CONTEXT_HASHKEY(context) ((p_context)->hashkey) +#define SM4_GCM_CONTEXT_J0(context) ((p_context)->j0) +#define SM4_GCM_CONTEXT_GHASH(context) ((p_context)->ghash) +#define SM4_GCM_CONTEXT_CTR(context) ((p_context)->ctr) + +#define SM4_GCM_CONTEXT_LEN(context) ((p_context)->len) + +#define SM4_GCM_CONTEXT_KEY(context) (&((p_context)->key_sched)) + +#define SM4_GCM_CONTEXT_STATE(context) ((p_context)->state) + +/* Calculate offsets for acessing blocks in buffers */ + +#define REG_SIZE_BITS (512) +#define REG_SIZE_BYTES (REG_SIZE_BITS / 8) /* Register size in bytes */ + +#define SLOTS_PER_BLOCK (SM4_BLOCK_SIZE / SM4_GCM_CONTEXT_BUFFER_SLOT_SIZE_BYTES) +#define BLOCKS_PER_REG (REG_SIZE_BYTES / SM4_BLOCK_SIZE) + +#define BUFFER_BLOCK_NUM(buffer, n) (buffer + SLOTS_PER_BLOCK * n) +#define BUFFER_REG_NUM(buffer, n) (buffer + SLOTS_PER_BLOCK * BLOCKS_PER_REG * n) + +/* Internal macroses */ + +#define sm4_gcm_clear_buffer(p_buffer) storeu((void *)(p_buffer), setzero()); + +#define SM4_GCM_CLEAR_BUFFER(p_buffer) \ + { \ + sm4_gcm_clear_buffer(BUFFER_REG_NUM(p_buffer, 0)); \ + sm4_gcm_clear_buffer(BUFFER_REG_NUM(p_buffer, 1)); \ + sm4_gcm_clear_buffer(BUFFER_REG_NUM(p_buffer, 2)); \ + sm4_gcm_clear_buffer(BUFFER_REG_NUM(p_buffer, 3)); \ + } + +#define SM4_GCM_CLEAR_LEN(p_len) \ + { \ + sm4_gcm_clear_buffer(p_len); \ + } + +/* Constants */ + +/* GCM polynomials for reduction */ +static __ALIGN64 const int64u gcm_poly[] = { 0x0000000000000001, 0xC200000000000000, 0x0000000000000001, 0xC200000000000000, + 0x0000000000000001, 0xC200000000000000, 0x0000000000000001, 0xC200000000000000 }; + +static __ALIGN64 const int64u gcm_poly2[] = { 0x00000001C2000000, 0xC200000000000000, 0x00000001C2000000, 0xC200000000000000, + 0x00000001C2000000, 0xC200000000000000, 0x00000001C2000000, 0xC200000000000000 }; + +/* */ +static __ALIGN64 const int64u two_one[] = { 0x0000000000000001, 0x0000000100000000, 0x0000000000000001, 0x0000000100000000, + 0x0000000000000001, 0x0000000100000000, 0x0000000000000001, 0x0000000100000000 }; + +/* Constant for IV of 12 bytes size finalization */ +static __ALIGN64 const int64u one_f[] = { 0x0000000000000000, 0x0100000000000000, 0x0000000000000000, 0x0100000000000000, + 0x0000000000000000, 0x0100000000000000, 0x0000000000000000, 0x0100000000000000 }; + +static __ALIGN64 const int64u bytes_to_bits_shift[] = { 0x0000000000000003, 0x0000000000000000 }; + +static const int rearrangeOrder[] = { 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15 }; + +/* Need to rearrange input pointers and lengths to keep it in the same layout with hashkeys */ +#define rearrange(to, from) \ + to[0] = from[0]; \ + to[1] = from[4]; \ + to[2] = from[8]; \ + to[3] = from[12]; \ + to[4] = from[1]; \ + to[5] = from[5]; \ + to[6] = from[9]; \ + to[7] = from[13]; \ + to[8] = from[2]; \ + to[9] = from[6]; \ + to[10] = from[10]; \ + to[11] = from[14]; \ + to[12] = from[3]; \ + to[13] = from[7]; \ + to[14] = from[11]; \ + to[15] = from[15]; + +__INLINE __m512i inc_block32(__m512i x, const int8u *increment) { return mask_add_epi32(x, 0x1111, x, M512(increment)); } + +static __ALIGN64 const int8u initialInc[] = { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + +#endif // SM4_GCM_MB_H diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm4/sm4_mb.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm4/sm4_mb.h new file mode 100644 index 0000000..7d5ca82 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/include/internal/sm4/sm4_mb.h @@ -0,0 +1,858 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#if !defined(_SM4_GFNI_MB_H) +#define _SM4_GFNI_MB_H + +#include +#include + +#include + +#ifndef M128 +#define M128(mem) (*((__m128i*)(mem))) +#endif +#ifndef M256 +#define M256(mem) (*((__m256i*)(mem))) +#endif +#ifndef M512 +#define M512(mem) (*((__m512i*)(mem))) +#endif + +#define loadu _mm512_loadu_si512 +#define storeu _mm512_storeu_si512 + +#define mask_storeu_epi64 _mm512_mask_storeu_epi64 +#define maskz_expandloadu_epi32 _mm512_maskz_expandloadu_epi32 + +#define mask_storeu_epi8 _mm512_mask_storeu_epi8 +#define maskz_loadu_epi8 _mm512_maskz_loadu_epi8 + +#define srli_epi64 _mm512_srli_epi64 +#define slli_epi64 _mm512_slli_epi64 +#define bsrli_epi128 _mm512_bsrli_epi128 +#define bslli_epi128 _mm512_bslli_epi128 + +#define shuffle_epi8 _mm512_shuffle_epi8 +#define shuffle_epi32 _mm512_shuffle_epi32 + +#define set1_epi32 _mm512_set1_epi32 +#define set1_epi64 _mm512_set1_epi64 +#define setzero _mm512_setzero_si512 + +#define cmpeq_epi32_mask _mm512_cmpeq_epi32_mask +#define cmp_epi32_mask _mm512_cmp_epi32_mask +#define cmp_epi64_mask _mm512_cmp_epi64_mask + +#define mask_set1_epi32 _mm512_mask_set1_epi32 + +#define mask_sub_epi32 _mm512_mask_sub_epi32 +#define mask_add_epi32 _mm512_mask_add_epi32 +#define mask_add_epi64 _mm512_mask_add_epi64 +#define add_epi32 _mm512_add_epi32 +#define sub_epi32 _mm512_sub_epi32 +#define add_epi64 _mm512_add_epi64 + +#define or _mm512_or_si512 +#define and _mm512_and_si512 +#define xor _mm512_xor_si512 + +#define clmul _mm512_clmulepi64_epi128 + +#define unpacklo_epi32 _mm512_unpacklo_epi32 +#define unpackhi_epi32 _mm512_unpackhi_epi32 +#define unpacklo_epi64 _mm512_unpacklo_epi64 +#define unpackhi_epi64 _mm512_unpackhi_epi64 + +#define insert32x4 _mm512_inserti32x4 +#define sll_epi32 _mm512_sll_epi32 +#define srli_epi32 _mm512_srli_epi32 +#define mask_cmp_epi32_mask _mm512_mask_cmp_epi32_mask +#define broadcast_i64x2 _mm512_broadcast_i64x2 + +/* +// Constants +*/ + +static __ALIGN64 const int8u permMask_in[] = { + 0,0x00,0x00,0x00, 4,0x00,0x00,0x00, 8,0x00,0x00,0x00, 12,0x00,0x00,0x00, + 1,0x00,0x00,0x00, 5,0x00,0x00,0x00, 9,0x00,0x00,0x00, 13,0x00,0x00,0x00, + 2,0x00,0x00,0x00, 6,0x00,0x00,0x00, 10,0x00,0x00,0x00, 14,0x00,0x00,0x00, + 3,0x00,0x00,0x00, 7,0x00,0x00,0x00, 11,0x00,0x00,0x00, 15,0x00,0x00,0x00 +}; + +static __ALIGN64 const int8u permMask_out[] = { + 12,0x00,0x00,0x00, 8,0x00,0x00,0x00, 4,0x00,0x00,0x00, 0,0x00,0x00,0x00, + 13,0x00,0x00,0x00, 9,0x00,0x00,0x00, 5,0x00,0x00,0x00, 1,0x00,0x00,0x00, + 14,0x00,0x00,0x00, 10,0x00,0x00,0x00, 6,0x00,0x00,0x00, 2,0x00,0x00,0x00, + 15,0x00,0x00,0x00, 11,0x00,0x00,0x00, 7,0x00,0x00,0x00, 3,0x00,0x00,0x00 +}; + +static __ALIGN64 const int8u affineIn[] = { + 0x52,0xBC,0x2D,0x02,0x9E,0x25,0xAC,0x34, 0x52,0xBC,0x2D,0x02,0x9E,0x25,0xAC,0x34, + 0x52,0xBC,0x2D,0x02,0x9E,0x25,0xAC,0x34, 0x52,0xBC,0x2D,0x02,0x9E,0x25,0xAC,0x34, + 0x52,0xBC,0x2D,0x02,0x9E,0x25,0xAC,0x34, 0x52,0xBC,0x2D,0x02,0x9E,0x25,0xAC,0x34, + 0x52,0xBC,0x2D,0x02,0x9E,0x25,0xAC,0x34, 0x52,0xBC,0x2D,0x02,0x9E,0x25,0xAC,0x34 +}; + +static __ALIGN64 const int8u affineOut[] = { + 0x19,0x8b,0x6c,0x1e,0x51,0x8e,0x2d,0xd7, 0x19,0x8b,0x6c,0x1e,0x51,0x8e,0x2d,0xd7, + 0x19,0x8b,0x6c,0x1e,0x51,0x8e,0x2d,0xd7, 0x19,0x8b,0x6c,0x1e,0x51,0x8e,0x2d,0xd7, + 0x19,0x8b,0x6c,0x1e,0x51,0x8e,0x2d,0xd7, 0x19,0x8b,0x6c,0x1e,0x51,0x8e,0x2d,0xd7, + 0x19,0x8b,0x6c,0x1e,0x51,0x8e,0x2d,0xd7, 0x19,0x8b,0x6c,0x1e,0x51,0x8e,0x2d,0xd7 +}; +// Constant for swapping the bytes inside the words +static __ALIGN64 const int8u swapBytes[] = { + 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12, + 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12, + 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12, + 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12 +}; +// Constant for swapping the endianness +static __ALIGN64 const int8u swapEndianness[] = { + 15,14,13,12, 11,10,9,8, 7,6,5,4, 3,2,1,0, + 15,14,13,12, 11,10,9,8, 7,6,5,4, 3,2,1,0, + 15,14,13,12, 11,10,9,8, 7,6,5,4, 3,2,1,0, + 15,14,13,12, 11,10,9,8, 7,6,5,4, 3,2,1,0 +}; +// Constant for swapping the order of words +static __ALIGN64 const int8u swapWordsOrder[] = { + 12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3, + 12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3, + 12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3, + 12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3 +}; + +static __ALIGN64 const int64u idx_0_3[] = { + 0x0000000000000000, 0x0000000000000000, 0x0000000100000001, 0x0000000100000001, + 0x0000000200000002, 0x0000000200000002, 0x0000000300000003, 0x0000000300000003 +}; +static __ALIGN64 const int64u idx_4_7[] = { + 0x0000000400000004, 0x0000000400000004, 0x0000000500000005, 0x0000000500000005, + 0x0000000600000006, 0x0000000600000006, 0x0000000700000007, 0x0000000700000007 +}; +static __ALIGN64 const int64u idx_8_b[] = { + 0x0000000800000008, 0x0000000800000008, 0x0000000900000009, 0x0000000900000009, + 0x0000000a0000000a, 0x0000000a0000000a, 0x0000000b0000000b, 0x0000000b0000000b +}; +static __ALIGN64 const int64u idx_c_f[] = { + 0x0000000c0000000c, 0x0000000c0000000c, 0x0000000d0000000d, 0x0000000d0000000d, + 0x0000000e0000000e, 0x0000000e0000000e, 0x0000000f0000000f, 0x0000000f0000000f +}; + + +static __ALIGN64 const int8u firstInc[] = { + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +}; + +static __ALIGN64 const int8u nextInc[] = { + 4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +}; + +static __ALIGN64 const int8u shuf8[] = { + 0x01, 0x02, 0x03, 0x00, 0x05, 0x06, 0x07, 0x04, + 0x09, 0x0A, 0x0B, 0x08, 0x0D, 0x0E, 0x0F, 0x0C, + 0x01, 0x02, 0x03, 0x00, 0x05, 0x06, 0x07, 0x04, + 0x09, 0x0A, 0x0B, 0x08, 0x0D, 0x0E, 0x0F, 0x0C, + 0x01, 0x02, 0x03, 0x00, 0x05, 0x06, 0x07, 0x04, + 0x09, 0x0A, 0x0B, 0x08, 0x0D, 0x0E, 0x0F, 0x0C, + 0x01, 0x02, 0x03, 0x00, 0x05, 0x06, 0x07, 0x04, + 0x09, 0x0A, 0x0B, 0x08, 0x0D, 0x0E, 0x0F, 0x0C, +}; + +/* For SM4-XTS */ +static __ALIGN64 const int64u xts_poly[] = { + 0x87, 0x87, 0x87, 0x87, 0x87, 0x87, 0x87, 0x87 +}; + +static __ALIGN64 const int8u xts_shuf_mask[] = { + 15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; + +static __ALIGN64 const int64u xts_const_dq3210[] = { + 0, 0, 1, 1, 2, 2, 3, 3 +}; + +static __ALIGN64 const int64u xts_const_dq5678[] = { + 8, 8, 7, 7, 6, 6, 5, 5 +}; + +static __ALIGN64 const int32u xts_full_block_mask[] = { + 0xfffffff0, 0xfffffff0, 0xfffffff0, 0xfffffff0, 0xfffffff0, 0xfffffff0, 0xfffffff0, 0xfffffff0, + 0xfffffff0, 0xfffffff0, 0xfffffff0, 0xfffffff0, 0xfffffff0, 0xfffffff0, 0xfffffff0, 0xfffffff0 +}; + +static __ALIGN64 const int32u xts_partial_block_mask[] = { + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f +}; + +static __ALIGN64 const int32u xts_dw0_7_to_qw_idx[] = { + 0, 0xFF, 1, 0xFF, 2, 0xFF, 3, 0xFF, + 4, 0xFF, 5, 0xFF, 6, 0xFF, 7, 0xFF +}; + +static __ALIGN64 const int32u xts_dw8_15_to_qw_idx[] = { + 8, 0xFF, 9, 0xFF, 10, 0xFF, 11, 0xFF, + 12, 0xFF, 13, 0xFF, 14, 0xFF, 15, 0xFF +}; + +static __ALIGN64 const int64u xts_tweak_permq[] = { + 2, 3, 0, 1, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 4, 5, 2, 3, 0xFF, 0xFF, + 0, 1, 2, 3, 6, 7, 4, 5, + 0, 1, 2, 3, 4, 5, 10, 11 /* for vpermi2q */ +}; + +static __ALIGN64 const int64u xts_next_tweak_permq[] = { + 0, 1, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 2, 3, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 4, 5, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 14, 15, 0, 1, 0xFF, 0xFF, 0xFF, 0xFF /* for vpermi2q */ +}; + +static __ALIGN64 const int64u xts_next_tweak_permq_enc[] = { + 2, 3, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 4, 5, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 6, 7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, +}; + +#define SM4_ONE_ROUND(X0, X1, X2, X3, TMP, RK) { \ + /* (Xi+1 ^ Xi+2 ^ Xi+3 ^ rki) */ \ + TMP = _mm512_ternarylogic_epi32 (X1, X2, X3, 0x96); \ + TMP = _mm512_xor_epi32(TMP, _mm512_loadu_si512(RK)); \ + /* T(Xi+1 ^ Xi+2 ^ Xi+3 ^ rki) */ \ + TMP = sBox512(TMP); \ + X0 = _mm512_ternarylogic_epi64 (X0, TMP, Lblock512(TMP), 0x96); \ +} + +#define SM4_FOUR_ROUNDS(X0, X1, X2, X3, TMP, RK, sign) { \ + SM4_ONE_ROUND(X0, X1, X2, X3, TMP, RK); \ + SM4_ONE_ROUND(X1, X2, X3, X0, TMP, (RK + sign * 1)); \ + SM4_ONE_ROUND(X2, X3, X0, X1, TMP, (RK + sign * 2)); \ + SM4_ONE_ROUND(X3, X0, X1, X2, TMP, (RK + sign * 3)); \ +} + +#define SM4_ONE_ROUND_MASKED(X0, X1, X2, X3, TMP, MASK, RK) { \ + /* (Xi+1 ^ Xi+2 ^ Xi+3 ^ rki) */ \ + TMP = _mm512_xor_epi32(_mm512_xor_epi32(_mm512_xor_epi32(X1, X2), X3), _mm512_loadu_si512(RK)); \ + /* T(Xi+1 ^ Xi+2 ^ Xi+3 ^ rki) */ \ + TMP = sBox512(TMP); \ + TMP = _mm512_xor_epi32(TMP, Lblock512(TMP)); \ + /* Xi+4 = Xi ^ T(Xi+1 ^ Xi+2 ^ Xi+3 ^ rki) */ \ + X0 = _mm512_mask_xor_epi32(X0, MASK, X0, TMP); \ +} + +#define SM4_FOUR_ROUNDS_MASKED(X0, X1, X2, X3, TMP, MASK, RK, sign) { \ + SM4_ONE_ROUND_MASKED(X0, X1, X2, X3, TMP, MASK, RK); \ + SM4_ONE_ROUND_MASKED(X1, X2, X3, X0, TMP, MASK, (RK + sign * 1)); \ + SM4_ONE_ROUND_MASKED(X2, X3, X0, X1, TMP, MASK, (RK + sign * 2)); \ + SM4_ONE_ROUND_MASKED(X3, X0, X1, X2, TMP, MASK, (RK + sign * 3)); \ +} + +#define EXPAND_ONE_RKEY(X, p_rk) { \ + X[0] = _mm512_permutexvar_epi32(M512(idx_0_3), _mm512_loadu_si512(p_rk)); \ + X[1] = _mm512_permutexvar_epi32(M512(idx_4_7), _mm512_loadu_si512(p_rk)); \ + X[2] = _mm512_permutexvar_epi32(M512(idx_8_b), _mm512_loadu_si512(p_rk)); \ + X[3] = _mm512_permutexvar_epi32(M512(idx_c_f), _mm512_loadu_si512(p_rk)); \ +} + +#define ENDIANNESS_16x32(x) _mm512_shuffle_epi8((x), M512(swapBytes)); +#define CHANGE_ORDER_BLOCKS(x) _mm512_shuffle_epi8((x), M512(swapEndianness)); + +/* Workaround for gcc91, got the error: implicit declaration of function ‘_mm512_div_epi32’ */ +#if defined(__GNUC__) && !defined(__INTEL_COMPILER) +#define GET_NUM_BLOCKS(OUT, LEN, BLOCK_SIZE) \ + { \ + int32u blocks[SM4_LINES]; \ + for (int i = 0; i < SM4_LINES; i++) \ + blocks[i] = (LEN)[i] / (BLOCK_SIZE); \ + (OUT) = _mm512_loadu_si512(blocks); \ + } +#else +#define GET_NUM_BLOCKS(OUT, LEN, BLOCK_SIZE) (OUT) = _mm512_div_epi32(_mm512_loadu_si512(LEN), _mm512_set1_epi32(BLOCK_SIZE)) +#endif + +#define UPDATE_STREAM_MASK_16(MASK, p_len) \ + MASK = *p_len < (16) ? (*p_len <= 0 ? 0 : ((int64u)1 << *p_len) - 1) : (__mmask64)(0xFFFF); p_len++; +#define UPDATE_STREAM_MASK_64(MASK, p_len) MASK = *p_len < (4 * 16) ? (*p_len <= 0 ? 0 : ((int64u)1 << *p_len) - 1) : (__mmask64)(-1); p_len++; + +#define SM4_ENC (1) +#define SM4_DEC (-1) + +/* +// Internal functions +*/ + +EXTERN_C void sm4_ecb_kernel_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const int32u* key_sched[SM4_ROUNDS], __mmask16 mb_mask, int operation); +EXTERN_C void sm4_cbc_enc_kernel_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const int32u* key_sched[SM4_ROUNDS], __mmask16 mb_mask, const int8u* pa_iv[SM4_LINES]); +EXTERN_C void sm4_cbc_dec_kernel_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const int32u* key_sched[SM4_ROUNDS], __mmask16 mb_mask, const int8u* pa_iv[SM4_LINES]); +EXTERN_C void sm4_cbc_mac_kernel_mb16(__m128i pa_out[SM4_LINES], const int8u *const pa_inp[SM4_LINES], const int len[SM4_LINES], const int32u* key_sched[SM4_ROUNDS], __mmask16 mb_mask, const int8u* pa_iv[SM4_LINES]); +EXTERN_C void sm4_ctr128_kernel_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const int32u* key_sched[SM4_ROUNDS], __mmask16 mb_mask, int8u* pa_ctr[SM4_LINES]); +EXTERN_C void sm4_ofb_kernel_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const int32u* key_sched[SM4_ROUNDS], __mmask16 mb_mask, int8u* pa_iv[SM4_LINES]); +EXTERN_C void sm4_cfb128_enc_kernel_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const int32u* key_sched[SM4_ROUNDS], const int8u* pa_iv[SM4_LINES], __mmask16 mb_mask); +EXTERN_C void sm4_cfb128_dec_kernel_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const int32u* key_sched[SM4_ROUNDS], const int8u* pa_iv[SM4_LINES], __mmask16 mb_mask); +EXTERN_C void sm4_set_round_keys_mb16(int32u* key_sched[SM4_ROUNDS], const int8u* pa_inp_key[SM4_LINES], __mmask16 mb_mask); +EXTERN_C void sm4_xts_kernel_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], + const int32u* key_sched1[SM4_ROUNDS], const int32u* key_sched2[SM4_ROUNDS], + const int8u* pa_tweak[SM4_LINES], __mmask16 mb_mask, const int dir); + +// The transformation based on SM4 sbox algebraic structure, parameters were computed manually +__INLINE __m512i sBox512(__m512i block) +{ + block = _mm512_gf2p8affine_epi64_epi8(block, M512(affineIn), 0x65); + block = _mm512_gf2p8affineinv_epi64_epi8(block, M512(affineOut), 0xd3); + return block; +} + +__INLINE __m512i Lblock512(__m512i x) +{ + return _mm512_ternarylogic_epi32(_mm512_xor_si512(_mm512_rol_epi32(x, 2), _mm512_rol_epi32(x, 10)), _mm512_rol_epi32(x, 18), + _mm512_shuffle_epi8 (x, _mm512_loadu_si512(shuf8)), 0x96); +} + +__INLINE __m512i Lkey512(__m512i x) +{ + return _mm512_xor_epi32(_mm512_rol_epi32(x, 13), _mm512_rol_epi32(x, 23)); +} + +__INLINE __m512i IncBlock512(__m512i x, const int8u* increment) +{ + __m512i t = _mm512_add_epi64(x, M512(increment)); + __mmask8 carryMask = _mm512_cmplt_epu64_mask(t, x); + carryMask = (__mmask8)(carryMask << 1); + t = _mm512_add_epi64(t, _mm512_mask_set1_epi64(_mm512_setzero_si512(), carryMask, 1)); + + return t; +} + +#define SM4_KERNEL(TMP, p_rk, iterator) \ + for (int itr = 0, j = 0; itr < 8; itr++, j++) { \ + /* initial xors */ \ + EXPAND_ONE_RKEY(TMP, p_rk); p_rk+=iterator; \ + TMP[0] = _mm512_ternarylogic_epi32 (TMP[0], TMP[5], TMP[6], 0x96); \ + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); \ + TMP[1] = _mm512_ternarylogic_epi32 (TMP[1], TMP[9], TMP[10], 0x96); \ + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); \ + TMP[2] = _mm512_ternarylogic_epi32 (TMP[2], TMP[13], TMP[14], 0x96); \ + TMP[2] = _mm512_xor_si512(TMP[2], TMP[15]); \ + TMP[3] = _mm512_ternarylogic_epi32 (TMP[3], TMP[17], TMP[18], 0x96); \ + TMP[3] = _mm512_xor_si512(TMP[3], TMP[19]); \ + /* Sbox */ \ + TMP[0] = sBox512(TMP[0]); \ + TMP[1] = sBox512(TMP[1]); \ + TMP[2] = sBox512(TMP[2]); \ + TMP[3] = sBox512(TMP[3]); \ + /* Sbox done, now L */ \ + TMP[4] = _mm512_ternarylogic_epi32(TMP[4], TMP[0], Lblock512(TMP[0]), 0x96); \ + TMP[8] = _mm512_ternarylogic_epi32(TMP[8], TMP[1], Lblock512(TMP[1]), 0x96); \ + TMP[12] = _mm512_ternarylogic_epi32(TMP[12], TMP[2], Lblock512(TMP[2]), 0x96); \ + TMP[16] = _mm512_ternarylogic_epi32(TMP[16], TMP[3], Lblock512(TMP[3]), 0x96); \ + /* initial xors */ \ + EXPAND_ONE_RKEY(TMP, p_rk); p_rk+=iterator; \ + TMP[0] = _mm512_ternarylogic_epi32 (TMP[0], TMP[6], TMP[7], 0x96); \ + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); \ + TMP[1] = _mm512_ternarylogic_epi32 (TMP[1], TMP[10], TMP[11], 0x96); \ + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); \ + TMP[2] = _mm512_ternarylogic_epi32 (TMP[2], TMP[14], TMP[15], 0x96); \ + TMP[2] = _mm512_xor_si512(TMP[2], TMP[12]); \ + TMP[3] = _mm512_ternarylogic_epi32 (TMP[3], TMP[18], TMP[19], 0x96); \ + TMP[3] = _mm512_xor_si512(TMP[3], TMP[16]); \ + /* Sbox */ \ + TMP[0] = sBox512(TMP[0]); \ + TMP[1] = sBox512(TMP[1]); \ + TMP[2] = sBox512(TMP[2]); \ + TMP[3] = sBox512(TMP[3]); \ + /* Sbox done, now L */ \ + TMP[5] = _mm512_ternarylogic_epi32(TMP[5], TMP[0], Lblock512(TMP[0]), 0x96); \ + TMP[9] = _mm512_ternarylogic_epi32(TMP[9], TMP[1], Lblock512(TMP[1]), 0x96); \ + TMP[13] = _mm512_ternarylogic_epi32(TMP[13], TMP[2], Lblock512(TMP[2]), 0x96); \ + TMP[17] = _mm512_ternarylogic_epi32(TMP[17], TMP[3], Lblock512(TMP[3]), 0x96); \ + \ + /* initial xors */ \ + EXPAND_ONE_RKEY(TMP, p_rk); p_rk+=iterator; \ + TMP[0] = _mm512_ternarylogic_epi32 (TMP[0], TMP[7], TMP[4], 0x96); \ + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); \ + TMP[1] = _mm512_ternarylogic_epi32 (TMP[1], TMP[11], TMP[8], 0x96); \ + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); \ + TMP[2] = _mm512_ternarylogic_epi32 (TMP[2], TMP[15], TMP[12], 0x96); \ + TMP[2] = _mm512_xor_si512(TMP[2], TMP[13]); \ + TMP[3] = _mm512_ternarylogic_epi32 (TMP[3], TMP[19], TMP[16], 0x96); \ + TMP[3] = _mm512_xor_si512(TMP[3], TMP[17]); \ + /* Sbox */ \ + TMP[0] = sBox512(TMP[0]); \ + TMP[1] = sBox512(TMP[1]); \ + TMP[2] = sBox512(TMP[2]); \ + TMP[3] = sBox512(TMP[3]); \ + /* Sbox done, now L */ \ + TMP[6] = _mm512_ternarylogic_epi32(TMP[6], TMP[0], Lblock512(TMP[0]), 0x96); \ + TMP[10] = _mm512_ternarylogic_epi32(TMP[10], TMP[1], Lblock512(TMP[1]), 0x96); \ + TMP[14] = _mm512_ternarylogic_epi32(TMP[14], TMP[2], Lblock512(TMP[2]), 0x96); \ + TMP[18] = _mm512_ternarylogic_epi32(TMP[18], TMP[3], Lblock512(TMP[3]), 0x96); \ + \ + /* initial xors */ \ + EXPAND_ONE_RKEY(TMP, p_rk); p_rk+=iterator; \ + TMP[0] = _mm512_ternarylogic_epi32 (TMP[0], TMP[4], TMP[5], 0x96); \ + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); \ + TMP[1] = _mm512_ternarylogic_epi32 (TMP[1], TMP[8], TMP[9], 0x96); \ + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); \ + TMP[2] = _mm512_ternarylogic_epi32 (TMP[2], TMP[12], TMP[13], 0x96); \ + TMP[2] = _mm512_xor_si512(TMP[2], TMP[14]); \ + TMP[3] = _mm512_ternarylogic_epi32 (TMP[3], TMP[16], TMP[17], 0x96); \ + TMP[3] = _mm512_xor_si512(TMP[3], TMP[18]); \ + /* Sbox */ \ + TMP[0] = sBox512(TMP[0]); \ + TMP[1] = sBox512(TMP[1]); \ + TMP[2] = sBox512(TMP[2]); \ + TMP[3] = sBox512(TMP[3]); \ + /* Sbox done, now L */ \ + TMP[7] = _mm512_ternarylogic_epi32(TMP[7], TMP[0], Lblock512(TMP[0]), 0x96); \ + TMP[11] = _mm512_ternarylogic_epi32(TMP[11], TMP[1], Lblock512(TMP[1]), 0x96); \ + TMP[15] = _mm512_ternarylogic_epi32(TMP[15], TMP[2], Lblock512(TMP[2]), 0x96); \ + TMP[19] = _mm512_ternarylogic_epi32(TMP[19], TMP[3], Lblock512(TMP[3]), 0x96); \ + } + +/* +// Transpose functions +*/ + +#define TRANSPOSE_INP_512(K0,K1,K2,K3, T0,T1,T2,T3) \ + K0 = _mm512_unpacklo_epi32(T0, T1); \ + K1 = _mm512_unpacklo_epi32(T2, T3); \ + K2 = _mm512_unpackhi_epi32(T0, T1); \ + K3 = _mm512_unpackhi_epi32(T2, T3); \ + \ + T0 = _mm512_unpacklo_epi64(K0, K1); \ + T1 = _mm512_unpacklo_epi64(K2, K3); \ + T2 = _mm512_unpackhi_epi64(K0, K1); \ + T3 = _mm512_unpackhi_epi64(K2, K3); \ + \ + K2 = _mm512_permutexvar_epi32(M512(permMask_in), T1); \ + K1 = _mm512_permutexvar_epi32(M512(permMask_in), T2); \ + K3 = _mm512_permutexvar_epi32(M512(permMask_in), T3); \ + K0 = _mm512_permutexvar_epi32(M512(permMask_in), T0) + +#define TRANSPOSE_OUT_512(T0,T1,T2,T3, K0,K1,K2,K3) \ + T0 = _mm512_shuffle_i32x4(K0, K1, 0x44); \ + T1 = _mm512_shuffle_i32x4(K0, K1, 0xee); \ + T2 = _mm512_shuffle_i32x4(K2, K3, 0x44); \ + T3 = _mm512_shuffle_i32x4(K2, K3, 0xee); \ + \ + K0 = _mm512_shuffle_i32x4(T0, T2, 0x88); \ + K1 = _mm512_shuffle_i32x4(T0, T2, 0xdd); \ + K2 = _mm512_shuffle_i32x4(T1, T3, 0x88); \ + K3 = _mm512_shuffle_i32x4(T1, T3, 0xdd); \ + \ + K0 = _mm512_permutexvar_epi32(M512(permMask_out), K0);\ + K1 = _mm512_permutexvar_epi32(M512(permMask_out), K1);\ + K2 = _mm512_permutexvar_epi32(M512(permMask_out), K2);\ + K3 = _mm512_permutexvar_epi32(M512(permMask_out), K3);\ + \ + T0=K0,T1=K1,T2=K2,T3=K3 + +__INLINE void TRANSPOSE_16x4_I32_EPI32(__m512i* t0, __m512i* t1, __m512i* t2, __m512i* t3, const int8u* p_inp[16], __mmask16 mb_mask) { + __mmask16 loc_mb_mask = mb_mask; + + // L0 - L3 + __m512i z0 = _mm512_maskz_loadu_epi32(0x000F * (0x1&loc_mb_mask), p_inp[0]); loc_mb_mask >>= 1; + __m512i z1 = _mm512_maskz_loadu_epi32(0x000F * (0x1&loc_mb_mask), p_inp[1]); loc_mb_mask >>= 1; + __m512i z2 = _mm512_maskz_loadu_epi32(0x000F * (0x1&loc_mb_mask), p_inp[2]); loc_mb_mask >>= 1; + __m512i z3 = _mm512_maskz_loadu_epi32(0x000F * (0x1&loc_mb_mask), p_inp[3]); loc_mb_mask >>= 1; + + // L4 - L7 + z0 = _mm512_mask_loadu_epi32(z0, 0x00F0 * (0x1&loc_mb_mask), (__m128i*)p_inp[4] - 1); loc_mb_mask >>= 1; + z1 = _mm512_mask_loadu_epi32(z1, 0x00F0 * (0x1&loc_mb_mask), (__m128i*)p_inp[5] - 1); loc_mb_mask >>= 1; + z2 = _mm512_mask_loadu_epi32(z2, 0x00F0 * (0x1&loc_mb_mask), (__m128i*)p_inp[6] - 1); loc_mb_mask >>= 1; + z3 = _mm512_mask_loadu_epi32(z3, 0x00F0 * (0x1&loc_mb_mask), (__m128i*)p_inp[7] - 1); loc_mb_mask >>= 1; + + // L8 - Lb + z0 = _mm512_mask_loadu_epi32(z0, 0x0F00 * (0x1&loc_mb_mask), (__m128i*)p_inp[8] - 2); loc_mb_mask >>= 1; + z1 = _mm512_mask_loadu_epi32(z1, 0x0F00 * (0x1&loc_mb_mask), (__m128i*)p_inp[9] - 2); loc_mb_mask >>= 1; + z2 = _mm512_mask_loadu_epi32(z2, 0x0F00 * (0x1&loc_mb_mask), (__m128i*)p_inp[10] - 2); loc_mb_mask >>= 1; + z3 = _mm512_mask_loadu_epi32(z3, 0x0F00 * (0x1&loc_mb_mask), (__m128i*)p_inp[11] - 2); loc_mb_mask >>= 1; + + // Lc - Lf + *t0 = ENDIANNESS_16x32(_mm512_mask_loadu_epi32(z0, 0xF000 * (0x1&loc_mb_mask), (__m128i*)p_inp[12] - 3)); loc_mb_mask >>= 1; + *t1 = ENDIANNESS_16x32(_mm512_mask_loadu_epi32(z1, 0xF000 * (0x1&loc_mb_mask), (__m128i*)p_inp[13] - 3)); loc_mb_mask >>= 1; + *t2 = ENDIANNESS_16x32(_mm512_mask_loadu_epi32(z2, 0xF000 * (0x1&loc_mb_mask), (__m128i*)p_inp[14] - 3)); loc_mb_mask >>= 1; + *t3 = ENDIANNESS_16x32(_mm512_mask_loadu_epi32(z3, 0xF000 * (0x1&loc_mb_mask), (__m128i*)p_inp[15] - 3)); loc_mb_mask >>= 1; + + z0 = _mm512_unpacklo_epi32(*t0, *t1); + z1 = _mm512_unpackhi_epi32(*t0, *t1); + z2 = _mm512_unpacklo_epi32(*t2, *t3); + z3 = _mm512_unpackhi_epi32(*t2, *t3); + + *t0 = _mm512_unpacklo_epi64(z0, z2); + *t1 = _mm512_unpackhi_epi64(z0, z2); + *t2 = _mm512_unpacklo_epi64(z1, z3); + *t3 = _mm512_unpackhi_epi64(z1, z3); +} + +__INLINE void TRANSPOSE_16x4_I32_XMM_EPI32(__m512i* t0, __m512i* t1, __m512i* t2, __m512i* t3, const __m128i in[16]) { + // L0 - L3 + __m512i z0 = _mm512_castsi128_si512(in[0]); + __m512i z1 = _mm512_castsi128_si512(in[1]); + __m512i z2 = _mm512_castsi128_si512(in[2]); + __m512i z3 = _mm512_castsi128_si512(in[3]); + + // L4 - L7 + z0 = _mm512_inserti64x2(z0, in[4], 1); + z1 = _mm512_inserti64x2(z1, in[5], 1); + z2 = _mm512_inserti64x2(z2, in[6], 1); + z3 = _mm512_inserti64x2(z3, in[7], 1); + + // L8 - Lb + z0 = _mm512_inserti64x2(z0, in[8], 2); + z1 = _mm512_inserti64x2(z1, in[9], 2); + z2 = _mm512_inserti64x2(z2, in[10], 2); + z3 = _mm512_inserti64x2(z3, in[11], 2); + + // Lc - Lf + *t0 = ENDIANNESS_16x32(_mm512_inserti64x2(z0, in[12], 3)); + *t1 = ENDIANNESS_16x32(_mm512_inserti64x2(z1, in[13], 3)); + *t2 = ENDIANNESS_16x32(_mm512_inserti64x2(z2, in[14], 3)); + *t3 = ENDIANNESS_16x32(_mm512_inserti64x2(z3, in[15], 3)); + + z0 = _mm512_unpacklo_epi32(*t0, *t1); + z1 = _mm512_unpackhi_epi32(*t0, *t1); + z2 = _mm512_unpacklo_epi32(*t2, *t3); + z3 = _mm512_unpackhi_epi32(*t2, *t3); + + *t0 = _mm512_unpacklo_epi64(z0, z2); + *t1 = _mm512_unpackhi_epi64(z0, z2); + *t2 = _mm512_unpacklo_epi64(z1, z3); + *t3 = _mm512_unpackhi_epi64(z1, z3); +} + +__INLINE void TRANSPOSE_4x16_I32_EPI32(__m512i* t0, __m512i* t1, __m512i* t2, __m512i* t3, int8u* p_out[16], __mmask16 mb_mask) { + + #define STORE_RESULT(OUT, store_mask, loc_mb_mask, Ti) \ + _mm512_mask_storeu_epi32(OUT, store_mask * (0x1&loc_mb_mask), Ti); \ + loc_mb_mask >>= 1; + + __mmask16 loc_mb_mask = mb_mask; + + __m512i z0 = _mm512_unpacklo_epi32(*t0, *t1); + __m512i z1 = _mm512_unpackhi_epi32(*t0, *t1); + __m512i z2 = _mm512_unpacklo_epi32(*t2, *t3); + __m512i z3 = _mm512_unpackhi_epi32(*t2, *t3); + + /* Get the right endianness and do (Y0, Y1, Y2, Y3) = R(X32, X33, X34, X35) = (X35, X34, X33, X32) */ + *t0 = CHANGE_ORDER_BLOCKS(_mm512_unpacklo_epi64(z0, z2)); + *t1 = CHANGE_ORDER_BLOCKS(_mm512_unpackhi_epi64(z0, z2)); + *t2 = CHANGE_ORDER_BLOCKS(_mm512_unpacklo_epi64(z1, z3)); + *t3 = CHANGE_ORDER_BLOCKS(_mm512_unpackhi_epi64(z1, z3)); + + // L0 - L3 + STORE_RESULT(p_out[0], 0x000F, loc_mb_mask, *t0); + STORE_RESULT(p_out[1], 0x000F, loc_mb_mask, *t1); + STORE_RESULT(p_out[2], 0x000F, loc_mb_mask, *t2); + STORE_RESULT(p_out[3], 0x000F, loc_mb_mask, *t3); + + // L4 - L7 + STORE_RESULT((__m128i*)p_out[4] - 1, 0x00F0, loc_mb_mask, *t0); + STORE_RESULT((__m128i*)p_out[5] - 1, 0x00F0, loc_mb_mask, *t1); + STORE_RESULT((__m128i*)p_out[6] - 1, 0x00F0, loc_mb_mask, *t2); + STORE_RESULT((__m128i*)p_out[7] - 1, 0x00F0, loc_mb_mask, *t3); + + // L8 - Lb + STORE_RESULT((__m128i*)p_out[8] - 2, 0x0F00, loc_mb_mask, *t0); + STORE_RESULT((__m128i*)p_out[9] - 2, 0x0F00, loc_mb_mask, *t1); + STORE_RESULT((__m128i*)p_out[10] - 2, 0x0F00, loc_mb_mask, *t2); + STORE_RESULT((__m128i*)p_out[11] - 2, 0x0F00, loc_mb_mask, *t3); + + // Lc - Lf + STORE_RESULT((__m128i*)p_out[12] - 3, 0xF000, loc_mb_mask, *t0); + STORE_RESULT((__m128i*)p_out[13] - 3, 0xF000, loc_mb_mask, *t1); + STORE_RESULT((__m128i*)p_out[14] - 3, 0xF000, loc_mb_mask, *t2); + STORE_RESULT((__m128i*)p_out[15] - 3, 0xF000, loc_mb_mask, *t3); + +} + +__INLINE void TRANSPOSE_4x16_I32_XMM_EPI32(__m512i* t0, __m512i* t1, __m512i* t2, __m512i* t3, __m128i out[16]) { + + __m512i z0 = _mm512_unpacklo_epi32(*t0, *t1); + __m512i z1 = _mm512_unpackhi_epi32(*t0, *t1); + __m512i z2 = _mm512_unpacklo_epi32(*t2, *t3); + __m512i z3 = _mm512_unpackhi_epi32(*t2, *t3); + + /* Get the right endianness and do (Y0, Y1, Y2, Y3) = R(X32, X33, X34, X35) = (X35, X34, X33, X32) */ + *t0 = CHANGE_ORDER_BLOCKS(_mm512_unpacklo_epi64(z0, z2)); + *t1 = CHANGE_ORDER_BLOCKS(_mm512_unpackhi_epi64(z0, z2)); + *t2 = CHANGE_ORDER_BLOCKS(_mm512_unpacklo_epi64(z1, z3)); + *t3 = CHANGE_ORDER_BLOCKS(_mm512_unpackhi_epi64(z1, z3)); + + // L0 - L3 + out[0] = _mm512_extracti64x2_epi64(*t0, 0); + out[1] = _mm512_extracti64x2_epi64(*t1, 0); + out[2] = _mm512_extracti64x2_epi64(*t2, 0); + out[3] = _mm512_extracti64x2_epi64(*t3, 0); + + // L4 - L7 + out[4] = _mm512_extracti64x2_epi64(*t0, 1); + out[5] = _mm512_extracti64x2_epi64(*t1, 1); + out[6] = _mm512_extracti64x2_epi64(*t2, 1); + out[7] = _mm512_extracti64x2_epi64(*t3, 1); + + // L8 - Lb + out[8] = _mm512_extracti64x2_epi64(*t0, 2); + out[9] = _mm512_extracti64x2_epi64(*t1, 2); + out[10] = _mm512_extracti64x2_epi64(*t2, 2); + out[11] = _mm512_extracti64x2_epi64(*t3, 2); + + // Lc - Lf + out[12] = _mm512_extracti64x2_epi64(*t0, 3); + out[13] = _mm512_extracti64x2_epi64(*t1, 3); + out[14] = _mm512_extracti64x2_epi64(*t2, 3); + out[15] = _mm512_extracti64x2_epi64(*t3, 3); + +} + +__INLINE void TRANSPOSE_4x16_I32_O128_EPI32(__m512i* t0, __m512i* t1, __m512i* t2, __m512i* t3, __m128i p_out[16], __mmask16 mb_mask) { + + #define STORE_RESULT(OUT, store_mask, loc_mb_mask, Ti) \ + _mm512_mask_storeu_epi32(OUT, store_mask * (0x1&loc_mb_mask), Ti); \ + loc_mb_mask >>= 1; + + __mmask16 loc_mb_mask = mb_mask; + + __m512i z0 = _mm512_unpacklo_epi32(*t0, *t1); + __m512i z1 = _mm512_unpackhi_epi32(*t0, *t1); + __m512i z2 = _mm512_unpacklo_epi32(*t2, *t3); + __m512i z3 = _mm512_unpackhi_epi32(*t2, *t3); + + /* Get the right endianness and do (Y0, Y1, Y2, Y3) = R(X32, X33, X34, X35) = (X35, X34, X33, X32) */ + *t0 = CHANGE_ORDER_BLOCKS(_mm512_unpacklo_epi64(z0, z2)); + *t1 = CHANGE_ORDER_BLOCKS(_mm512_unpackhi_epi64(z0, z2)); + *t2 = CHANGE_ORDER_BLOCKS(_mm512_unpacklo_epi64(z1, z3)); + *t3 = CHANGE_ORDER_BLOCKS(_mm512_unpackhi_epi64(z1, z3)); + + // L0 - L3 + STORE_RESULT(&p_out[0], 0x000F, loc_mb_mask, *t0); + STORE_RESULT(&p_out[1], 0x000F, loc_mb_mask, *t1); + STORE_RESULT(&p_out[2], 0x000F, loc_mb_mask, *t2); + STORE_RESULT(&p_out[3], 0x000F, loc_mb_mask, *t3); + + // L4 - L7 + STORE_RESULT(&p_out[4] - 1, 0x00F0, loc_mb_mask, *t0); + STORE_RESULT(&p_out[5] - 1, 0x00F0, loc_mb_mask, *t1); + STORE_RESULT(&p_out[6] - 1, 0x00F0, loc_mb_mask, *t2); + STORE_RESULT(&p_out[7] - 1, 0x00F0, loc_mb_mask, *t3); + + // L8 - Lb + STORE_RESULT(&p_out[8] - 2, 0x0F00, loc_mb_mask, *t0); + STORE_RESULT(&p_out[9] - 2, 0x0F00, loc_mb_mask, *t1); + STORE_RESULT(&p_out[10] - 2, 0x0F00, loc_mb_mask, *t2); + STORE_RESULT(&p_out[11] - 2, 0x0F00, loc_mb_mask, *t3); + + // Lc - Lf + STORE_RESULT(&p_out[12] - 3, 0xF000, loc_mb_mask, *t0); + STORE_RESULT(&p_out[13] - 3, 0xF000, loc_mb_mask, *t1); + STORE_RESULT(&p_out[14] - 3, 0xF000, loc_mb_mask, *t2); + STORE_RESULT(&p_out[15] - 3, 0xF000, loc_mb_mask, *t3); + +} + +__INLINE void TRANSPOSE_4x16_I32_EPI8(__m512i t0, __m512i t1, __m512i t2, __m512i t3, int8u* p_out[16], int* p_loc_len, __mmask16 mb_mask) { + + #define STORE_RESULT_EPI8(OUT, store_mask, loc_mb_mask, Ti) \ + _mm512_mask_storeu_epi8(OUT, store_mask * (0x1&loc_mb_mask), Ti); \ + loc_mb_mask >>= 1; + + __mmask16 loc_mb_mask = mb_mask; + /* Mask for data loading */ + __mmask64 stream_mask; + + __m512i z0 = _mm512_unpacklo_epi32(t0, t1); + __m512i z1 = _mm512_unpackhi_epi32(t0, t1); + __m512i z2 = _mm512_unpacklo_epi32(t2, t3); + __m512i z3 = _mm512_unpackhi_epi32(t2, t3); + + /* Get the right endianness */ + t0 = ENDIANNESS_16x32(_mm512_unpacklo_epi64(z0, z2)); + t1 = ENDIANNESS_16x32(_mm512_unpackhi_epi64(z0, z2)); + t2 = ENDIANNESS_16x32(_mm512_unpacklo_epi64(z1, z3)); + t3 = ENDIANNESS_16x32(_mm512_unpackhi_epi64(z1, z3)); + + // L0 - L3 + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + STORE_RESULT_EPI8(p_out[0], stream_mask, loc_mb_mask, t0); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + STORE_RESULT_EPI8(p_out[1], stream_mask, loc_mb_mask, t1); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + STORE_RESULT_EPI8(p_out[2], stream_mask, loc_mb_mask, t2); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + STORE_RESULT_EPI8(p_out[3], stream_mask, loc_mb_mask, t3); + + // L4 - L7 + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + STORE_RESULT_EPI8((__m128i*)p_out[4] - 1, stream_mask << 16, loc_mb_mask, t0); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + STORE_RESULT_EPI8((__m128i*)p_out[5] - 1, stream_mask << 16, loc_mb_mask, t1); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + STORE_RESULT_EPI8((__m128i*)p_out[6] - 1, stream_mask << 16, loc_mb_mask, t2); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + STORE_RESULT_EPI8((__m128i*)p_out[7] - 1, stream_mask << 16, loc_mb_mask, t3); + + // L8 - Lb + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + STORE_RESULT_EPI8((__m128i*)p_out[8] - 2, stream_mask << 32, loc_mb_mask, t0); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + STORE_RESULT_EPI8((__m128i*)p_out[9] - 2, stream_mask << 32, loc_mb_mask, t1); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + STORE_RESULT_EPI8((__m128i*)p_out[10] - 2, stream_mask << 32, loc_mb_mask, t2); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + STORE_RESULT_EPI8((__m128i*)p_out[11] - 2, stream_mask << 32, loc_mb_mask, t3); + + // Lc - Lf + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + STORE_RESULT_EPI8((__m128i*)p_out[12] - 3, stream_mask << 48, loc_mb_mask, t0); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + STORE_RESULT_EPI8((__m128i*)p_out[13] - 3, stream_mask << 48, loc_mb_mask, t1); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + STORE_RESULT_EPI8((__m128i*)p_out[14] - 3, stream_mask << 48, loc_mb_mask, t2); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + STORE_RESULT_EPI8((__m128i*)p_out[15] - 3, stream_mask << 48, loc_mb_mask, t3); +} + +__INLINE void TRANSPOSE_AND_XOR_4x16_I32_EPI32(__m512i* t0, __m512i* t1, __m512i* t2, __m512i* t3, int8u* p_out[16], const int8u* p_iv[16], __mmask16 mb_mask) { + + #define XOR_AND_STORE_RESULT(OUT, store_mask, loc_mb_mask, Ti, IV, TMP) \ + TMP = _mm512_maskz_loadu_epi32(store_mask * (0x1&loc_mb_mask), IV); \ + _mm512_mask_storeu_epi32(OUT, store_mask * (0x1&loc_mb_mask), _mm512_xor_epi32(Ti, TMP)); \ + loc_mb_mask >>= 1; + + __m512i z0 = _mm512_setzero_si512(); + __m512i z1 = _mm512_setzero_si512(); + __m512i z2 = _mm512_setzero_si512(); + __m512i z3 = _mm512_setzero_si512(); + + __mmask16 loc_mb_mask = mb_mask; + + z0 = _mm512_unpacklo_epi32(*t0, *t1); + z1 = _mm512_unpackhi_epi32(*t0, *t1); + z2 = _mm512_unpacklo_epi32(*t2, *t3); + z3 = _mm512_unpackhi_epi32(*t2, *t3); + + /* Get the right endianness and do (Y0, Y1, Y2, Y3) = R(X32, X33, X34, X35) = (X35, X34, X33, X32) */ + *t0 = CHANGE_ORDER_BLOCKS(_mm512_unpacklo_epi64(z0, z2)); + *t1 = CHANGE_ORDER_BLOCKS(_mm512_unpackhi_epi64(z0, z2)); + *t2 = CHANGE_ORDER_BLOCKS(_mm512_unpacklo_epi64(z1, z3)); + *t3 = CHANGE_ORDER_BLOCKS(_mm512_unpackhi_epi64(z1, z3)); + + // L0 - L3 + XOR_AND_STORE_RESULT(p_out[0], 0x000F, loc_mb_mask, *t0, p_iv[0], z0); + XOR_AND_STORE_RESULT(p_out[1], 0x000F, loc_mb_mask, *t1, p_iv[1], z1); + XOR_AND_STORE_RESULT(p_out[2], 0x000F, loc_mb_mask, *t2, p_iv[2], z2); + XOR_AND_STORE_RESULT(p_out[3], 0x000F, loc_mb_mask, *t3, p_iv[3], z3); + + // L4 - L7 + XOR_AND_STORE_RESULT((__m128i*)p_out[4] - 1, 0x00F0, loc_mb_mask, *t0, (__m128i*)p_iv[4] - 1, z0); + XOR_AND_STORE_RESULT((__m128i*)p_out[5] - 1, 0x00F0, loc_mb_mask, *t1, (__m128i*)p_iv[5] - 1, z1); + XOR_AND_STORE_RESULT((__m128i*)p_out[6] - 1, 0x00F0, loc_mb_mask, *t2, (__m128i*)p_iv[6] - 1, z2); + XOR_AND_STORE_RESULT((__m128i*)p_out[7] - 1, 0x00F0, loc_mb_mask, *t3, (__m128i*)p_iv[7] - 1, z3); + + // L8 - Lb + XOR_AND_STORE_RESULT((__m128i*)p_out[8] - 2, 0x0F00, loc_mb_mask, *t0, (__m128i*)p_iv[8] - 2, z0); + XOR_AND_STORE_RESULT((__m128i*)p_out[9] - 2, 0x0F00, loc_mb_mask, *t1, (__m128i*)p_iv[9] - 2, z1); + XOR_AND_STORE_RESULT((__m128i*)p_out[10] - 2, 0x0F00, loc_mb_mask, *t2, (__m128i*)p_iv[10] - 2, z2); + XOR_AND_STORE_RESULT((__m128i*)p_out[11] - 2, 0x0F00, loc_mb_mask, *t3, (__m128i*)p_iv[11] - 2, z3); + + // Lc - Lf + XOR_AND_STORE_RESULT((__m128i*)p_out[12] - 3, 0xF000, loc_mb_mask, *t0, (__m128i*)p_iv[12] - 3, z0); + XOR_AND_STORE_RESULT((__m128i*)p_out[13] - 3, 0xF000, loc_mb_mask, *t1, (__m128i*)p_iv[13] - 3, z1); + XOR_AND_STORE_RESULT((__m128i*)p_out[14] - 3, 0xF000, loc_mb_mask, *t2, (__m128i*)p_iv[14] - 3, z2); + XOR_AND_STORE_RESULT((__m128i*)p_out[15] - 3, 0xF000, loc_mb_mask, *t3, (__m128i*)p_iv[15] - 3, z3); +} + +__INLINE void TRANSPOSE_AND_XOR_4x16_I32_EPI8(__m512i t0, __m512i t1, __m512i t2, __m512i t3, int8u* p_out[16], const int8u* p_iv[16], int* p_loc_len, __mmask16 mb_mask) { + + #define XOR_AND_STORE_RESULT_EPI8(OUT, store_mask, loc_mb_mask, Ti, IV, TMP) \ + TMP = _mm512_maskz_loadu_epi8(store_mask * (0x1&loc_mb_mask), IV); \ + _mm512_mask_storeu_epi8(OUT, store_mask * (0x1&loc_mb_mask), _mm512_xor_epi32(Ti, TMP)); \ + loc_mb_mask >>= 1; + + __m512i z0 = _mm512_setzero_si512(); + __m512i z1 = _mm512_setzero_si512(); + __m512i z2 = _mm512_setzero_si512(); + __m512i z3 = _mm512_setzero_si512(); + + __mmask16 loc_mb_mask = mb_mask; + /* Mask for data loading */ + __mmask64 stream_mask; + + z0 = _mm512_unpacklo_epi32(t0, t1); + z1 = _mm512_unpackhi_epi32(t0, t1); + z2 = _mm512_unpacklo_epi32(t2, t3); + z3 = _mm512_unpackhi_epi32(t2, t3); + + /* Get the right endianness */ + t0 = ENDIANNESS_16x32(_mm512_unpacklo_epi64(z0, z2)); + t1 = ENDIANNESS_16x32(_mm512_unpackhi_epi64(z0, z2)); + t2 = ENDIANNESS_16x32(_mm512_unpacklo_epi64(z1, z3)); + t3 = ENDIANNESS_16x32(_mm512_unpackhi_epi64(z1, z3)); + + // L0 - L3 + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + XOR_AND_STORE_RESULT_EPI8(p_out[0], stream_mask, loc_mb_mask, t0, p_iv[0], z0); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + XOR_AND_STORE_RESULT_EPI8(p_out[1], stream_mask, loc_mb_mask, t1, p_iv[1], z1); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + XOR_AND_STORE_RESULT_EPI8(p_out[2], stream_mask, loc_mb_mask, t2, p_iv[2], z2); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + XOR_AND_STORE_RESULT_EPI8(p_out[3], stream_mask, loc_mb_mask, t3, p_iv[3], z3); + + // L4 - L7 + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + XOR_AND_STORE_RESULT_EPI8((__m128i*)p_out[4] - 1, stream_mask << 16, loc_mb_mask, t0, (__m128i*)p_iv[4] - 1, z0); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + XOR_AND_STORE_RESULT_EPI8((__m128i*)p_out[5] - 1, stream_mask << 16, loc_mb_mask, t1, (__m128i*)p_iv[5] - 1, z1); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + XOR_AND_STORE_RESULT_EPI8((__m128i*)p_out[6] - 1, stream_mask << 16, loc_mb_mask, t2, (__m128i*)p_iv[6] - 1, z2); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + XOR_AND_STORE_RESULT_EPI8((__m128i*)p_out[7] - 1, stream_mask << 16, loc_mb_mask, t3, (__m128i*)p_iv[7] - 1, z3); + + // L8 - Lb + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + XOR_AND_STORE_RESULT_EPI8((__m128i*)p_out[8] - 2, stream_mask << 32, loc_mb_mask, t0, (__m128i*)p_iv[8] - 2, z0); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + XOR_AND_STORE_RESULT_EPI8((__m128i*)p_out[9] - 2, stream_mask << 32, loc_mb_mask, t1, (__m128i*)p_iv[9] - 2, z1); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + XOR_AND_STORE_RESULT_EPI8((__m128i*)p_out[10] - 2, stream_mask << 32, loc_mb_mask, t2, (__m128i*)p_iv[10] - 2, z2); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + XOR_AND_STORE_RESULT_EPI8((__m128i*)p_out[11] - 2, stream_mask << 32, loc_mb_mask, t3, (__m128i*)p_iv[11] - 2, z3); + + // Lc - Lf + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + XOR_AND_STORE_RESULT_EPI8((__m128i*)p_out[12] - 3, stream_mask << 48, loc_mb_mask, t0, (__m128i*)p_iv[12] - 3, z0); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + XOR_AND_STORE_RESULT_EPI8((__m128i*)p_out[13] - 3, stream_mask << 48, loc_mb_mask, t1, (__m128i*)p_iv[13] - 3, z1); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + XOR_AND_STORE_RESULT_EPI8((__m128i*)p_out[14] - 3, stream_mask << 48, loc_mb_mask, t2, (__m128i*)p_iv[14] - 3, z2); + UPDATE_STREAM_MASK_16(stream_mask, p_loc_len) + XOR_AND_STORE_RESULT_EPI8((__m128i*)p_out[15] - 3, stream_mask << 48, loc_mb_mask, t3, (__m128i*)p_iv[15] - 3, z3); +} +#endif /* _SM4_GFNI_MB_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/CMakeLists.txt b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/CMakeLists.txt new file mode 100644 index 0000000..c9cad06 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/CMakeLists.txt @@ -0,0 +1,163 @@ +#=============================================================================== +# Copyright (C) 2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. +# +#=============================================================================== + +# Define defaults for every supported compiler +set(DEFAULT_GNU_COMPILER_VER 8.2.0) +set(DEFAULT_CLANG_COMPILER_VER 9.0.0) +set(DEFAULT_Intel_COMPILER_VER 19.0.0) + +# Check compiler version +if(("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") AND (CMAKE_C_COMPILER_VERSION VERSION_LESS DEFAULT_GNU_COMPILER_VER)) + message(FATAL_ERROR "GNU C Compiler version must be 8.2 or higher") +endif() +if(("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang") AND (CMAKE_C_COMPILER_VERSION VERSION_LESS DEFAULT_CLANG_COMPILER_VER)) + message(FATAL_ERROR "Clang C Compiler version must be 9.0 or higher") +endif() +if(("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel") AND (CMAKE_C_COMPILER_VERSION VERSION_LESS DEFAULT_Intel_COMPILER_VER)) + message(FATAL_ERROR "Compiler version must be 19.0 or higher") +endif() + +include("${CRYPTO_MB_SOURCES_DIR}/cmake/common.cmake") +include(${COMPILER_OPTIONS_FILE}) # Get ${CMAKE_C_FLAGS}, ${CMAKE_CXX_FLAGS} and ${AVX512_CFLAGS} + +set(AVX512_LIBRARY_DEFINES "${AVX512_LIBRARY_DEFINES}" "${MB_LIBRARIES_DEFINES}" "USE_AMS_5x" "SIMD_LEN=512") + +# Sources +file(GLOB RSA_AVX512_SOURCES "${CRYPTO_MB_SOURCES_DIR}/rsa/*.c" + "${CRYPTO_MB_SOURCES_DIR}/rsa/avx512_primitives/*.c") +file(GLOB COMMON_SOURCES "${CRYPTO_MB_SOURCES_DIR}/common/*.c") +file(GLOB X25519_SOURCES "${CRYPTO_MB_SOURCES_DIR}/x25519/*.c") +file(GLOB ECNIST_SOURCES "${CRYPTO_MB_SOURCES_DIR}/ecnist/*.c") +file(GLOB SM2_SOURCES "${CRYPTO_MB_SOURCES_DIR}/sm2/*.c") +file(GLOB SM3_SOURCES "${CRYPTO_MB_SOURCES_DIR}/sm3/*.c") + +# SM4 Sources +file(GLOB SM4_SOURCES "${CRYPTO_MB_SOURCES_DIR}/sm4/*.c") +file(GLOB SM4_SOURCES ${SM4_SOURCES} "${CRYPTO_MB_SOURCES_DIR}/sm4/gcm/*.c") +file(GLOB SM4_SOURCES ${SM4_SOURCES} "${CRYPTO_MB_SOURCES_DIR}/sm4/gcm/internal/*.c") +file(GLOB SM4_SOURCES ${SM4_SOURCES} "${CRYPTO_MB_SOURCES_DIR}/sm4/ccm/*.c") +file(GLOB SM4_SOURCES ${SM4_SOURCES} "${CRYPTO_MB_SOURCES_DIR}/sm4/ccm/internal/*.c") + +file(GLOB ED25519_SOURCES "${CRYPTO_MB_SOURCES_DIR}/ed25519/*.c") +file(GLOB EXP_SOURCES "${CRYPTO_MB_SOURCES_DIR}/exp/*.c") + +# Headers +file(GLOB PUBLIC_HEADERS "${CRYPTO_MB_INCLUDE_DIR}/crypto_mb/*.h") +file(GLOB PRIVATE_HEADERS "${CRYPTO_MB_INCLUDE_DIR}/internal/common/*.h" + "${CRYPTO_MB_INCLUDE_DIR}/internal/ecnist/*.h" + "${CRYPTO_MB_INCLUDE_DIR}/internal/rsa/*.h" + "${CRYPTO_MB_INCLUDE_DIR}/internal/sm2/*.h" + "${CRYPTO_MB_INCLUDE_DIR}/internal/sm3/*.h" + "${CRYPTO_MB_INCLUDE_DIR}/internal/sm4/*.h" + "${CRYPTO_MB_INCLUDE_DIR}/internal/ed25519/*.h" + "${CRYPTO_MB_INCLUDE_DIR}/internal/exp/*.h") +file(GLOB OPENSSL_HEADERS "${OPENSSL_INCLUDE_DIR}/openssl/*.h") + +set(CRYPTO_MB_SOURCES ${RSA_AVX512_SOURCES} ${COMMON_SOURCES} ${X25519_SOURCES} ${ECNIST_SOURCES} ${SM2_SOURCES} ${SM3_SOURCES} ${SM4_SOURCES} ${ED25519_SOURCES} ${EXP_SOURCES}) +set(CRYPTO_MB_HEADERS ${PUBLIC_HEADERS} ${PRIVATE_HEADERS} ${OPENSSL_HEADERS}) + +set(WIN_RESOURCE_FILE ${CRYPTO_MB_SOURCES_DIR}/common/crypto_mb_ver.rc) +set(CPU_FEATURES_FILE ${CRYPTO_MB_SOURCES_DIR}/common/cpu_features.c) + +# Disable compiler optimizations for this file, as compiler adds some ISA specific code +# which is unwanted for functions that are aimed to work on any CPU +list(REMOVE_ITEM CRYPTO_MB_SOURCES ${CPU_FEATURES_FILE}) +if("${OS_STRING}" STREQUAL "windows") + set_source_files_properties(${CPU_FEATURES_FILE} PROPERTIES COMPILE_FLAGS "${CMAKE_C_FLAGS_SECURITY} /Od") +else() + set_source_files_properties(${CPU_FEATURES_FILE} PROPERTIES COMPILE_FLAGS "${CMAKE_C_FLAGS_SECURITY} -O0") +endif() + +if(BN_OPENSSL_PATCH) # Off by default + list(APPEND AVX512_LIBRARY_DEFINES "BN_OPENSSL_PATCH") +endif() + +set(MB_LIB_TARGET ${MB_DYN_LIB_TARGET}) + +set_source_files_properties(${CRYPTO_MB_SOURCES} PROPERTIES COMPILE_DEFINITIONS "${AVX512_LIBRARY_DEFINES}" + COMPILE_FLAGS "${AVX512_CFLAGS} ${CMAKE_ASM_FLAGS} ${CMAKE_C_FLAGS_SECURITY}") + +# Don't specify architectural flags for the assembler for this sources, because of the bug in Intel® C Compiler under MacOS: error: invalid instruction mnemonic 'vkmovb' +# The bug has been fixed since version 2021.3. This is a workaround to support older versions of Intel® C Compiler. +if(CMAKE_C_COMPILER_VERSION VERSION_LESS 20.2.3) + set_source_files_properties(${X25519_SOURCES} PROPERTIES COMPILE_DEFINITIONS "${AVX512_LIBRARY_DEFINES}" + COMPILE_FLAGS "${AVX512_CFLAGS} ${CMAKE_C_FLAGS_SECURITY}") +endif() + +# Create shared library +if(DYNAMIC_LIB OR MB_STANDALONE) + if(WIN32) + add_library(${MB_DYN_LIB_TARGET} SHARED ${CRYPTO_MB_HEADERS} ${CRYPTO_MB_SOURCES} ${CPU_FEATURES_FILE} ${WIN_RESOURCE_FILE}) + else() + add_library(${MB_DYN_LIB_TARGET} SHARED ${CRYPTO_MB_HEADERS} ${CRYPTO_MB_SOURCES} ${CPU_FEATURES_FILE}) + endif() + + set_target_properties(${MB_DYN_LIB_TARGET} PROPERTIES C_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON + LINK_FLAGS "${LINK_FLAGS_DYNAMIC} ${LINK_FLAG_SECURITY}" + PUBLIC_HEADER "${PUBLIC_HEADERS}" + ) + + if(UNIX) + set_target_properties(${MB_DYN_LIB_TARGET} PROPERTIES VERSION ${MBX_INTERFACE_VERSION} + SOVERSION ${MBX_INTERFACE_VERSION_MAJOR}) + endif() + + target_link_libraries(${MB_DYN_LIB_TARGET} OpenSSL::Crypto) +endif(DYNAMIC_LIB OR MB_STANDALONE) + +# Installation of the shared library +if (MB_STANDALONE) # standalone crypto_mb's cmake run + install(TARGETS ${MB_DYN_LIB_TARGET} + LIBRARY DESTINATION "lib" + RUNTIME DESTINATION "lib" + PUBLIC_HEADER DESTINATION "include/crypto_mb") +elseif (DYNAMIC_LIB) # build from ippcp's cmake + install(TARGETS ${MB_DYN_LIB_TARGET} + LIBRARY DESTINATION "lib/intel64" + RUNTIME DESTINATION "lib/intel64" + PUBLIC_HEADER DESTINATION "include/crypto_mb") +endif() + +# Static library +if(WIN32) + add_library(${MB_STATIC_LIB_TARGET} STATIC ${CRYPTO_MB_HEADERS} ${CRYPTO_MB_SOURCES} ${CPU_FEATURES_FILE} ${WIN_RESOURCE_FILE}) +else() + add_library(${MB_STATIC_LIB_TARGET} STATIC ${CRYPTO_MB_HEADERS} ${CRYPTO_MB_SOURCES} ${CPU_FEATURES_FILE}) +endif() + +set_target_properties(${MB_STATIC_LIB_TARGET} PROPERTIES C_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON + PUBLIC_HEADER "${PUBLIC_HEADERS}") + +target_link_libraries(${MB_STATIC_LIB_TARGET} OpenSSL::Crypto) +if(WIN32) + set_target_properties(${MB_STATIC_LIB_TARGET} PROPERTIES OUTPUT_NAME "${MB_LIB_TARGET}mt") +else() + set_target_properties(${MB_STATIC_LIB_TARGET} PROPERTIES OUTPUT_NAME "${MB_LIB_TARGET}") +endif() + +# Static lib installation +if(MB_STANDALONE) + install(TARGETS ${MB_STATIC_LIB_TARGET} + ARCHIVE DESTINATION "lib" + PUBLIC_HEADER DESTINATION "include/crypto_mb") +else() + install(TARGETS ${MB_STATIC_LIB_TARGET} + ARCHIVE DESTINATION "lib/intel64" + PUBLIC_HEADER DESTINATION "include/crypto_mb") +endif() diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/cmake/dll_export/crypto_mb.defs b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/cmake/dll_export/crypto_mb.defs new file mode 100644 index 0000000..1faf7a1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/cmake/dll_export/crypto_mb.defs @@ -0,0 +1,131 @@ +EXPORTS + +mbx_getversion + +mbx_get_cpu_features +mbx_is_crypto_mb_applicable +mbx_get_algo_info + +mbx_rsa_public_mb8 +mbx_rsa_private_mb8 +mbx_rsa_private_crt_mb8 + +mbx_rsa_public_ssl_mb8 +mbx_rsa_private_ssl_mb8 +mbx_rsa_private_crt_ssl_mb8 + +mbx_x25519_public_key_mb8 +mbx_x25519_mb8 +mbx_ed25519_public_key_mb8 +mbx_ed25519_sign_mb8 +mbx_ed25519_verify_mb8 + +mbx_exp_BufferSize +mbx_exp1024_mb8 +mbx_exp2048_mb8 +mbx_exp3072_mb8 +mbx_exp4096_mb8 +mbx_exp_mb8 + +mbx_nistp256_ecdh_mb8 +mbx_nistp256_ecdsa_sign_setup_mb8 +mbx_nistp256_ecdsa_sign_complete_mb8 +mbx_nistp256_ecdsa_sign_mb8 +mbx_nistp256_ecdsa_verify_mb8 +mbx_nistp256_ecpublic_key_mb8 + +mbx_nistp256_ecdh_ssl_mb8 +mbx_nistp256_ecdsa_sign_setup_ssl_mb8 +mbx_nistp256_ecdsa_sign_complete_ssl_mb8 +mbx_nistp256_ecdsa_sign_ssl_mb8 +mbx_nistp256_ecdsa_verify_ssl_mb8 +mbx_nistp256_ecpublic_key_ssl_mb8 + +mbx_nistp384_ecdh_mb8 +mbx_nistp384_ecdsa_sign_setup_mb8 +mbx_nistp384_ecdsa_sign_complete_mb8 +mbx_nistp384_ecdsa_sign_mb8 +mbx_nistp384_ecdsa_verify_mb8 +mbx_nistp384_ecpublic_key_mb8 + +mbx_nistp384_ecdh_ssl_mb8 +mbx_nistp384_ecdsa_sign_setup_ssl_mb8 +mbx_nistp384_ecdsa_sign_complete_ssl_mb8 +mbx_nistp384_ecdsa_sign_ssl_mb8 +mbx_nistp384_ecdsa_verify_ssl_mb8 +mbx_nistp384_ecpublic_key_ssl_mb8 + +mbx_nistp521_ecdh_mb8 +mbx_nistp521_ecdsa_sign_setup_mb8 +mbx_nistp521_ecdsa_sign_complete_mb8 +mbx_nistp521_ecdsa_sign_mb8 +mbx_nistp521_ecdsa_verify_mb8 +mbx_nistp521_ecpublic_key_mb8 + +mbx_nistp521_ecdh_ssl_mb8 +mbx_nistp521_ecdsa_sign_setup_ssl_mb8 +mbx_nistp521_ecdsa_sign_complete_ssl_mb8 +mbx_nistp521_ecdsa_sign_ssl_mb8 +mbx_nistp521_ecdsa_verify_ssl_mb8 +mbx_nistp521_ecpublic_key_ssl_mb8 + +mbx_sm2_ecdh_mb8 +mbx_sm2_ecdsa_sign_mb8 +mbx_sm2_ecdsa_verify_mb8 +mbx_sm2_ecpublic_key_mb8 + +mbx_sm2_ecdh_ssl_mb8 +mbx_sm2_ecdsa_sign_ssl_mb8 +mbx_sm2_ecdsa_verify_ssl_mb8 +mbx_sm2_ecpublic_key_ssl_mb8 + +mbx_RSA1K_pub65537_Method +mbx_RSA2K_pub65537_Method +mbx_RSA3K_pub65537_Method +mbx_RSA4K_pub65537_Method +mbx_RSA_pub65537_Method +mbx_RSA1K_private_Method +mbx_RSA2K_private_Method +mbx_RSA3K_private_Method +mbx_RSA4K_private_Method +mbx_RSA_private_Method +mbx_RSA1K_private_crt_Method +mbx_RSA2K_private_crt_Method +mbx_RSA3K_private_crt_Method +mbx_RSA4K_private_crt_Method +mbx_RSA_private_crt_Method +mbx_RSA_Method_BufSize + +mbx_sm3_init_mb16 +mbx_sm3_update_mb16 +mbx_sm3_final_mb16 +mbx_sm3_msg_digest_mb16 + +mbx_sm4_set_key_mb16 +mbx_sm4_encrypt_ecb_mb16 +mbx_sm4_decrypt_ecb_mb16 +mbx_sm4_encrypt_cbc_mb16 +mbx_sm4_decrypt_cbc_mb16 +mbx_sm4_encrypt_ctr128_mb16 +mbx_sm4_decrypt_ctr128_mb16 +mbx_sm4_encrypt_ofb_mb16 +mbx_sm4_decrypt_ofb_mb16 +mbx_sm4_encrypt_cfb128_mb16 +mbx_sm4_decrypt_cfb128_mb16 + +mbx_sm4_gcm_init_mb16 +mbx_sm4_gcm_update_iv_mb16 +mbx_sm4_gcm_update_aad_mb16 +mbx_sm4_gcm_encrypt_mb16 +mbx_sm4_gcm_decrypt_mb16 +mbx_sm4_gcm_get_tag_mb16 + +mbx_sm4_ccm_init_mb16 +mbx_sm4_ccm_update_aad_mb16 +mbx_sm4_ccm_encrypt_mb16 +mbx_sm4_ccm_decrypt_mb16 +mbx_sm4_ccm_get_tag_mb16 + +mbx_sm4_xts_set_keys_mb16 +mbx_sm4_xts_encrypt_mb16 +mbx_sm4_xts_decrypt_mb16 diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/cmake/dll_export/crypto_mb.linux.lib-export b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/cmake/dll_export/crypto_mb.linux.lib-export new file mode 100644 index 0000000..5bbea89 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/cmake/dll_export/crypto_mb.linux.lib-export @@ -0,0 +1,129 @@ +EXTERN (mbx_getversion) + +EXTERN (mbx_get_cpu_features) +EXTERN (mbx_is_crypto_mb_applicable) +EXTERN (mbx_get_algo_info) + +EXTERN (mbx_rsa_public_mb8) +EXTERN (mbx_rsa_private_mb8) +EXTERN (mbx_rsa_private_crt_mb8) + +EXTERN (mbx_rsa_public_ssl_mb8) +EXTERN (mbx_rsa_private_ssl_mb8) +EXTERN (mbx_rsa_private_crt_ssl_mb8) + +EXTERN (mbx_x25519_public_key_mb8) +EXTERN (mbx_x25519_mb8) +EXTERN (mbx_ed25519_public_key_mb8) +EXTERN (mbx_ed25519_sign_mb8) +EXTERN (mbx_ed25519_verify_mb8) + +EXTERN (mbx_exp_BufferSize) +EXTERN (mbx_exp1024_mb8) +EXTERN (mbx_exp2048_mb8) +EXTERN (mbx_exp3072_mb8) +EXTERN (mbx_exp4096_mb8) +EXTERN (mbx_exp_mb8) + +EXTERN (mbx_nistp256_ecdh_mb8) +EXTERN (mbx_nistp256_ecdsa_sign_setup_mb8) +EXTERN (mbx_nistp256_ecdsa_sign_complete_mb8) +EXTERN (mbx_nistp256_ecdsa_sign_mb8) +EXTERN (mbx_nistp256_ecdsa_verify_mb8) +EXTERN (mbx_nistp256_ecpublic_key_mb8) + +EXTERN (mbx_nistp256_ecdh_ssl_mb8) +EXTERN (mbx_nistp256_ecdsa_sign_setup_ssl_mb8) +EXTERN (mbx_nistp256_ecdsa_sign_complete_ssl_mb8) +EXTERN (mbx_nistp256_ecdsa_sign_ssl_mb8) +EXTERN (mbx_nistp256_ecdsa_verify_ssl_mb8) +EXTERN (mbx_nistp256_ecpublic_key_ssl_mb8) + +EXTERN (mbx_nistp384_ecdh_mb8) +EXTERN (mbx_nistp384_ecdsa_sign_setup_mb8) +EXTERN (mbx_nistp384_ecdsa_sign_complete_mb8) +EXTERN (mbx_nistp384_ecdsa_sign_mb8) +EXTERN (mbx_nistp384_ecdsa_verify_mb8) +EXTERN (mbx_nistp384_ecpublic_key_mb8) + +EXTERN (mbx_nistp384_ecdh_ssl_mb8) +EXTERN (mbx_nistp384_ecdsa_sign_setup_ssl_mb8) +EXTERN (mbx_nistp384_ecdsa_sign_complete_ssl_mb8) +EXTERN (mbx_nistp384_ecdsa_sign_ssl_mb8) +EXTERN (mbx_nistp384_ecdsa_verify_ssl_mb8) +EXTERN (mbx_nistp384_ecpublic_key_ssl_mb8) + +EXTERN (mbx_nistp521_ecdh_mb8) +EXTERN (mbx_nistp521_ecdsa_sign_setup_mb8) +EXTERN (mbx_nistp521_ecdsa_sign_complete_mb8) +EXTERN (mbx_nistp521_ecdsa_sign_mb8) +EXTERN (mbx_nistp521_ecdsa_verify_mb8) +EXTERN (mbx_nistp521_ecpublic_key_mb8) + +EXTERN (mbx_nistp521_ecdh_ssl_mb8) +EXTERN (mbx_nistp521_ecdsa_sign_setup_ssl_mb8) +EXTERN (mbx_nistp521_ecdsa_sign_complete_ssl_mb8) +EXTERN (mbx_nistp521_ecdsa_sign_ssl_mb8) +EXTERN (mbx_nistp521_ecdsa_verify_ssl_mb8) +EXTERN (mbx_nistp521_ecpublic_key_ssl_mb8) + +EXTERN (mbx_sm2_ecdh_mb8) +EXTERN (mbx_sm2_ecdsa_sign_mb8) +EXTERN (mbx_sm2_ecdsa_verify_mb8) +EXTERN (mbx_sm2_ecpublic_key_mb8) + +EXTERN (mbx_sm2_ecdh_ssl_mb8) +EXTERN (mbx_sm2_ecdsa_sign_ssl_mb8) +EXTERN (mbx_sm2_ecdsa_verify_ssl_mb8) +EXTERN (mbx_sm2_ecpublic_key_ssl_mb8) + +EXTERN (mbx_RSA1K_pub65537_Method) +EXTERN (mbx_RSA2K_pub65537_Method) +EXTERN (mbx_RSA3K_pub65537_Method) +EXTERN (mbx_RSA4K_pub65537_Method) +EXTERN (mbx_RSA_pub65537_Method) +EXTERN (mbx_RSA1K_private_Method) +EXTERN (mbx_RSA2K_private_Method) +EXTERN (mbx_RSA3K_private_Method) +EXTERN (mbx_RSA4K_private_Method) +EXTERN (mbx_RSA_private_Method) +EXTERN (mbx_RSA1K_private_crt_Method) +EXTERN (mbx_RSA2K_private_crt_Method) +EXTERN (mbx_RSA3K_private_crt_Method) +EXTERN (mbx_RSA4K_private_crt_Method) +EXTERN (mbx_RSA_private_crt_Method) +EXTERN (mbx_RSA_Method_BufSize) + +EXTERN (mbx_sm3_init_mb16) +EXTERN (mbx_sm3_update_mb16) +EXTERN (mbx_sm3_final_mb16) +EXTERN (mbx_sm3_msg_digest_mb16) + +EXTERN (mbx_sm4_set_key_mb16) +EXTERN (mbx_sm4_encrypt_ecb_mb16) +EXTERN (mbx_sm4_decrypt_ecb_mb16) +EXTERN (mbx_sm4_encrypt_cbc_mb16) +EXTERN (mbx_sm4_decrypt_cbc_mb16) +EXTERN (mbx_sm4_encrypt_ctr128_mb16) +EXTERN (mbx_sm4_decrypt_ctr128_mb16) +EXTERN (mbx_sm4_encrypt_ofb_mb16) +EXTERN (mbx_sm4_decrypt_ofb_mb16) +EXTERN (mbx_sm4_encrypt_cfb128_mb16) +EXTERN (mbx_sm4_decrypt_cfb128_mb16) + +EXTERN (mbx_sm4_gcm_init_mb16) +EXTERN (mbx_sm4_gcm_update_iv_mb16) +EXTERN (mbx_sm4_gcm_update_aad_mb16) +EXTERN (mbx_sm4_gcm_encrypt_mb16) +EXTERN (mbx_sm4_gcm_decrypt_mb16) +EXTERN (mbx_sm4_gcm_get_tag_mb16) + +EXTERN (mbx_sm4_ccm_init_mb16) +EXTERN (mbx_sm4_ccm_update_aad_mb16) +EXTERN (mbx_sm4_ccm_encrypt_mb16) +EXTERN (mbx_sm4_ccm_decrypt_mb16) +EXTERN (mbx_sm4_ccm_get_tag_mb16) + +EXTERN (mbx_sm4_xts_set_keys_mb16) +EXTERN (mbx_sm4_xts_encrypt_mb16) +EXTERN (mbx_sm4_xts_decrypt_mb16) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/cmake/dll_export/crypto_mb.macosx.lib-export b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/cmake/dll_export/crypto_mb.macosx.lib-export new file mode 100644 index 0000000..b090a28 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/cmake/dll_export/crypto_mb.macosx.lib-export @@ -0,0 +1,129 @@ +_mbx_getversion + +_mbx_get_cpu_features +_mbx_is_crypto_mb_applicable +_mbx_get_algo_info + +_mbx_rsa_public_mb8 +_mbx_rsa_private_mb8 +_mbx_rsa_private_crt_mb8 + +_mbx_rsa_public_ssl_mb8 +_mbx_rsa_private_ssl_mb8 +_mbx_rsa_private_crt_ssl_mb8 + +_mbx_x25519_public_key_mb8 +_mbx_x25519_mb8 +_mbx_ed25519_public_key_mb8 +_mbx_ed25519_sign_mb8 +_mbx_ed25519_verify_mb8 + +_mbx_exp_BufferSize +_mbx_exp1024_mb8 +_mbx_exp2048_mb8 +_mbx_exp3072_mb8 +_mbx_exp4096_mb8 +_mbx_exp_mb8 + +_mbx_nistp256_ecdh_mb8 +_mbx_nistp256_ecdsa_sign_setup_mb8 +_mbx_nistp256_ecdsa_sign_complete_mb8 +_mbx_nistp256_ecdsa_sign_mb8 +_mbx_nistp256_ecdsa_verify_mb8 +_mbx_nistp256_ecpublic_key_mb8 + +_mbx_nistp256_ecdh_ssl_mb8 +_mbx_nistp256_ecdsa_sign_setup_ssl_mb8 +_mbx_nistp256_ecdsa_sign_complete_ssl_mb8 +_mbx_nistp256_ecdsa_sign_ssl_mb8 +_mbx_nistp256_ecdsa_verify_ssl_mb8 +_mbx_nistp256_ecpublic_key_ssl_mb8 + +_mbx_nistp384_ecdh_mb8 +_mbx_nistp384_ecdsa_sign_setup_mb8 +_mbx_nistp384_ecdsa_sign_complete_mb8 +_mbx_nistp384_ecdsa_sign_mb8 +_mbx_nistp384_ecdsa_verify_mb8 +_mbx_nistp384_ecpublic_key_mb8 + +_mbx_nistp384_ecdh_ssl_mb8 +_mbx_nistp384_ecdsa_sign_setup_ssl_mb8 +_mbx_nistp384_ecdsa_sign_complete_ssl_mb8 +_mbx_nistp384_ecdsa_sign_ssl_mb8 +_mbx_nistp384_ecdsa_verify_ssl_mb8 +_mbx_nistp384_ecpublic_key_ssl_mb8 + +_mbx_nistp521_ecdh_mb8 +_mbx_nistp521_ecdsa_sign_setup_mb8 +_mbx_nistp521_ecdsa_sign_complete_mb8 +_mbx_nistp521_ecdsa_sign_mb8 +_mbx_nistp521_ecdsa_verify_mb8 +_mbx_nistp521_ecpublic_key_mb8 + +_mbx_nistp521_ecdh_ssl_mb8 +_mbx_nistp521_ecdsa_sign_setup_ssl_mb8 +_mbx_nistp521_ecdsa_sign_complete_ssl_mb8 +_mbx_nistp521_ecdsa_sign_ssl_mb8 +_mbx_nistp521_ecdsa_verify_ssl_mb8 +_mbx_nistp521_ecpublic_key_ssl_mb8 + +_mbx_sm2_ecdh_mb8 +_mbx_sm2_ecdsa_sign_mb8 +_mbx_sm2_ecdsa_verify_mb8 +_mbx_sm2_ecpublic_key_mb8 + +_mbx_sm2_ecdh_ssl_mb8 +_mbx_sm2_ecdsa_sign_ssl_mb8 +_mbx_sm2_ecdsa_verify_ssl_mb8 +_mbx_sm2_ecpublic_key_ssl_mb8 + +_mbx_RSA1K_pub65537_Method +_mbx_RSA2K_pub65537_Method +_mbx_RSA3K_pub65537_Method +_mbx_RSA4K_pub65537_Method +_mbx_RSA_pub65537_Method +_mbx_RSA1K_private_Method +_mbx_RSA2K_private_Method +_mbx_RSA3K_private_Method +_mbx_RSA4K_private_Method +_mbx_RSA_private_Method +_mbx_RSA1K_private_crt_Method +_mbx_RSA2K_private_crt_Method +_mbx_RSA3K_private_crt_Method +_mbx_RSA4K_private_crt_Method +_mbx_RSA_private_crt_Method +_mbx_RSA_Method_BufSize + +_mbx_sm3_init_mb16 +_mbx_sm3_update_mb16 +_mbx_sm3_final_mb16 +_mbx_sm3_msg_digest_mb16 + +_mbx_sm4_set_key_mb16 +_mbx_sm4_encrypt_ecb_mb16 +_mbx_sm4_decrypt_ecb_mb16 +_mbx_sm4_encrypt_cbc_mb16 +_mbx_sm4_decrypt_cbc_mb16 +_mbx_sm4_encrypt_ctr128_mb16 +_mbx_sm4_decrypt_ctr128_mb16 +_mbx_sm4_encrypt_ofb_mb16 +_mbx_sm4_decrypt_ofb_mb16 +_mbx_sm4_encrypt_cfb128_mb16 +_mbx_sm4_decrypt_cfb128_mb16 + +_mbx_sm4_gcm_init_mb16 +_mbx_sm4_gcm_update_iv_mb16 +_mbx_sm4_gcm_update_aad_mb16 +_mbx_sm4_gcm_encrypt_mb16 +_mbx_sm4_gcm_decrypt_mb16 +_mbx_sm4_gcm_get_tag_mb16 + +_mbx_sm4_ccm_init_mb16 +_mbx_sm4_ccm_update_aad_mb16 +_mbx_sm4_ccm_encrypt_mb16 +_mbx_sm4_ccm_decrypt_mb16 +_mbx_sm4_ccm_get_tag_mb16 + +_mbx_sm4_xts_set_keys_mb16 +_mbx_sm4_xts_encrypt_mb16 +_mbx_sm4_xts_decrypt_mb16 diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/common/cpu_features.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/common/cpu_features.c new file mode 100644 index 0000000..c4265a8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/common/cpu_features.c @@ -0,0 +1,290 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +/* masks of bits */ +#define BIT00 0x00000001 +#define BIT01 0x00000002 +#define BIT02 0x00000004 +#define BIT03 0x00000008 +#define BIT04 0x00000010 +#define BIT05 0x00000020 +#define BIT06 0x00000040 +#define BIT07 0x00000080 +#define BIT08 0x00000100 +#define BIT09 0x00000200 +#define BIT10 0x00000400 +#define BIT11 0x00000800 +#define BIT12 0x00001000 +#define BIT13 0x00002000 +#define BIT14 0x00004000 +#define BIT15 0x00008000 +#define BIT16 0x00010000 +#define BIT17 0x00020000 +#define BIT18 0x00040000 +#define BIT19 0x00080000 +#define BIT20 0x00100000 +#define BIT21 0x00200000 +#define BIT22 0x00400000 +#define BIT23 0x00800000 +#define BIT24 0x01000000 +#define BIT25 0x02000000 +#define BIT26 0x04000000 +#define BIT27 0x08000000 +#define BIT28 0x10000000 +#define BIT29 0x20000000 +#define BIT30 0x40000000 +#define BIT31 0x80000000 + +/* index inside cpu_info[4] */ +#define eax_ (0) +#define ebx_ (1) +#define ecx_ (2) +#define edx_ (3) + + +__INLINE void _mbcp_cpuid(int32u buf[4], int32u leaf, int32u subleaf) +{ + #ifdef __GNUC__ + __asm__ ("cpuid" : "=a" (buf[0]), "=b" (buf[1]), "=c" (buf[2]), "=d" (buf[3]) : "a" (leaf), "c" (subleaf)); + #else + __cpuidex(buf,leaf, subleaf); + #endif +} + +static int32u _mbcp_max_cpuid_main_leaf_number(void) +{ + int32u buf[4]; + _mbcp_cpuid(buf, 0, 0); + return buf[0]; +} + +#if 0 +static int32u _mbcp_max_cpuid_extd_leaf_number(void) +{ + int32u buf[4]; + _mbcp_cpuid(buf, 0x80000000, 0); + return buf[0]; +} +#endif + +#define XSAVE_OSXSAVE (BIT26|BIT27) +static int32u _mbcp_xsave_support(int32u bits) +{ + int32u buf[4]; + _mbcp_cpuid(buf, 1, 0); + + if(XSAVE_OSXSAVE != (buf[ecx_] & XSAVE_OSXSAVE)) + return 0; + + int32u xcr0; + + #ifdef __GNUC__ + //asm volatile("xgetbv" : "=a" (xcr0) : "c" (0) : "%edx"); + __asm__ ("xgetbv" : "=a" (xcr0) : "c" (0) : "%edx"); + #else + xcr0 = (int32u)_xgetbv(0); + #endif + + return (xcr0 & bits) == bits; +} + +typedef struct { + int32u info_idx; + int32u info_idx_bit; + int64u cpu_feature; +} cpu_feature_map; + +/* +// Intel(r) Architecture Instruction Set Extension and Future Features. +// Programming Reference. 319433-038, March 2020 +// see Table 1-5, Table 1-6 +*/ +static const cpu_feature_map cpu_feature_detector_1_0[] = { + {ecx_, BIT00, mbcpCPUID_SSE3}, + {ecx_, BIT01, mbcpCPUID_CLMUL}, + {ecx_, BIT09, mbcpCPUID_SSSE3}, + //{ecx_, BIT12, mbcpCPUID_FMA}, /* supported FMA extensions using YMM*/ + //{ecx_, BIT13, mbcpCPUID_CMPXCHG16B}, + {ecx_, BIT19, mbcpCPUID_SSE41}, + {ecx_, BIT20, mbcpCPUID_SSE42}, + {ecx_, BIT22, mbcpCPUID_MOVBE}, + //{ecx_, BIT23, mbcpCPUID_POPCNT}, + {ecx_, BIT25, mbcpCPUID_AES}, + {ecx_, BIT28, mbcpCPUID_AVX}, + {ecx_, BIT29, mbcpCPUID_F16C}, + {ecx_, BIT30, mbcpCPUID_RDRAND}, + + {edx_, BIT23, mbcpCPUID_MMX}, + {edx_, BIT25, mbcpCPUID_SSE}, + {edx_, BIT26, mbcpCPUID_SSE2}, +}; + +static const cpu_feature_map cpu_feature_detector_7_0[] = { + {ebx_, BIT03, mbcpCPUID_BMI1}, + {ebx_, BIT05, mbcpCPUID_AVX2}, + {ebx_, BIT08, mbcpCPUID_BMI2}, + {ebx_, BIT14, mbcpCPUID_MPX}, + {ebx_, BIT16, mbcpCPUID_AVX512F}, + {ebx_, BIT17, mbcpCPUID_AVX512DQ}, + {ebx_, BIT18, mbcpCPUID_RDSEED}, + {ebx_, BIT19, mbcpCPUID_ADX}, + {ebx_, BIT21, mbcpCPUID_AVX512IFMA}, + {ebx_, BIT26, mbcpCPUID_AVX512PF}, + {ebx_, BIT27, mbcpCPUID_AVX512ER}, + {ebx_, BIT28, mbcpCPUID_AVX512CD}, + {ebx_, BIT29, mbcpCPUID_SHA}, + {ebx_, BIT30, mbcpCPUID_AVX512BW}, + {ebx_, BIT31, mbcpCPUID_AVX512VL}, + + {ecx_, BIT01, mbcpCPUID_AVX512VBMI}, + {ecx_, BIT06, mbcpCPUID_AVX512VBMI2}, + {ecx_, BIT08, mbcpCPUID_AVX512GFNI}, + {ecx_, BIT09, mbcpCPUID_AVX512VAES}, + {ecx_, BIT10, mbcpCPUID_AVX512VCLMUL}, + //{ecx_, BIT11, mbcpCPUID_AVX512VNNI}, + //{ecx_, BIT12, mbcpCPUID_AVX512BITALG}, + + {edx_, BIT02, mbcpCPUID_AVX512_4VNNIW}, + {edx_, BIT03, mbcpCPUID_AVX512_4FMADDPS}, +}; + +#undef eax_ +#undef ebx_ +#undef ecx_ +#undef edx_ + + +static int64u _mbcp_cpu_feature_detector(const int32u cpuinfo[4], const cpu_feature_map* tbl, int32u tbl_len) +{ + int64u features = 0; + int32u n; + for(n=0; n=7) { + /* cpuid info: cpuid_7_0 */ + _mbcp_cpuid(cpu_info, 7, 0); + features |= _mbcp_cpu_feature_detector(cpu_info, + cpu_feature_detector_7_0, + sizeof(cpu_feature_detector_7_0)/sizeof(cpu_feature_map)); + if((features & mbcpCPUID_AVX512F) && _mbcp_xsave_support(XSAVE_AVX512_SUPPORT)) + features |= mbcpAVX512_ENABLEDBYOS; + } + } + + return features; +} + + +// based on c-flags: -mavx512dq -mavx512ifma -mavx512f -mavx512vbmi2 -mavx512cd -mavx512bw -mbmi2 +#define CRYPTO_MB_REQUIRED_CPU_FEATURES ( \ + mbcpCPUID_BMI2 \ + | mbcpCPUID_AVX512F \ + | mbcpCPUID_AVX512DQ \ + | mbcpCPUID_AVX512BW \ + | mbcpCPUID_AVX512IFMA \ + | mbcpCPUID_AVX512VBMI2 \ + | mbcpAVX512_ENABLEDBYOS) + +DLL_PUBLIC +int mbx_is_crypto_mb_applicable(int64u cpu_features) +{ + int64u features = cpu_features; + if(0 == features) + features = mbx_get_cpu_features(); + return (CRYPTO_MB_REQUIRED_CPU_FEATURES == (features & CRYPTO_MB_REQUIRED_CPU_FEATURES)); +} + +/* structure for determining the number of buffers(WIDTH) for the algorithm */ +typedef struct { + enum MBX_ALGO algo; + enum MBX_WIDTH width; +} algo_width_map; + +/* clang-config off */ +static const algo_width_map arr_algo_width[] = { + { MBX_ALGO_RSA_1K, MBX_WIDTH_MB8 }, + { MBX_ALGO_RSA_2K, MBX_WIDTH_MB8 }, + { MBX_ALGO_RSA_3K, MBX_WIDTH_MB8 }, + { MBX_ALGO_RSA_4K, MBX_WIDTH_MB8 }, + { MBX_ALGO_X25519, MBX_WIDTH_MB8 }, + { MBX_ALGO_EC_NIST_P256, MBX_WIDTH_MB8 }, + { MBX_ALGO_EC_NIST_P384, MBX_WIDTH_MB8 }, + { MBX_ALGO_EC_NIST_P521, MBX_WIDTH_MB8 }, + { MBX_ALGO_EC_SM2, MBX_WIDTH_MB8 }, + { MBX_ALGO_SM3, MBX_WIDTH_MB16 }, + { MBX_ALGO_SM4, MBX_WIDTH_MB16 } +}; +/* clang-config on */ + +DLL_PUBLIC +MBX_ALGO_INFO mbx_get_algo_info(enum MBX_ALGO algo) +{ + const int mbx_mb_applicable = mbx_is_crypto_mb_applicable(0); + + int num_width = 0; + /* check CPU feature */ + if (0 == mbx_mb_applicable) { + return num_width; + } + const int num_tbl = sizeof(arr_algo_width) / sizeof(algo_width_map); + + const algo_width_map *tbl = arr_algo_width; + /* loop determining the number of buffers to process */ + for (int i = 0; i < num_tbl; ++i, ++tbl) { + if (algo == (*tbl).algo) { + num_width = (*tbl).width; + break; + } + } + + return num_width; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/common/crypto_mb_res.gen b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/common/crypto_mb_res.gen new file mode 100644 index 0000000..e8bb874 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/common/crypto_mb_res.gen @@ -0,0 +1,57 @@ +/******************************************************************************* +* Copyright (C) 1999 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + + +#include "winres.h" + +#define STR2(x) #x +#define STR(x) STR2(x) + +VS_VERSION_INFO VERSIONINFO + FILEVERSION MBX_VERSION() + PRODUCTVERSION MBX_VERSION() + FILEFLAGSMASK 0x3fL +#ifdef _DEBUG + FILEFLAGS 0x1L +#else + FILEFLAGS 0x0L +#endif + FILEOS VOS_NT_WINDOWS32 + FILETYPE VFT_DLL + FILESUBTYPE 0x0L +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904b0" + BEGIN + VALUE "CompanyName", "Intel Corporation.\0" + VALUE "FileVersion", STR( MBX_VERSION() ) "\0" + VALUE "ProductName", MBX_LIB_SHORTNAME() ". Intel(R) Integrated Performance Primitives. " MBX_LIB_LONGNAME() ".\0" + VALUE "ProductVersion", CRYPTO_MB_STR_VERSION() "\0" + VALUE "LegalCopyright", "Copyright (C) 1999-2021, Intel Corporation. All rights reserved.\0" + + VALUE "Comments", "Intel(R) Integrated Performance Primitives. " MBX_LIB_LONGNAME() ".\0" + VALUE "FileDescription", MBX_LIB_SHORTNAME() ".dll is the intel64 " MBX_LIB_SHORTNAME() " dynamic library\0" + VALUE "InternalName", MBX_LIB_SHORTNAME() ".dll\0" + VALUE "OriginalFilename", MBX_LIB_SHORTNAME() ".dll\0" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1200 + END +END diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/common/crypto_mb_ver.rc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/common/crypto_mb_ver.rc new file mode 100644 index 0000000..df21efc --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/common/crypto_mb_ver.rc @@ -0,0 +1,22 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include "crypto_mb_res.gen" + +/* ////////////////////////////// End of file /////////////////////////////// */ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/common/ifma52_mb8_template.cxx b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/common/ifma52_mb8_template.cxx new file mode 100644 index 0000000..d09a5dc --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/common/ifma52_mb8_template.cxx @@ -0,0 +1,29 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "ifma_math.h" +#include "ifma_internal.h" + +{typedef} +{functions} + +void {function_name} ({parameters}) +{ + // Main code goes here + +{code} +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/common/ifma_cvt52.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/common/ifma_cvt52.c new file mode 100644 index 0000000..1099518 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/common/ifma_cvt52.c @@ -0,0 +1,525 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include + +#if defined(_MSC_VER) && (_MSC_VER < 1920) + // Disable optimization for VS2017 due to AVX512 masking bug + #define DISABLE_OPTIMIZATION __pragma(optimize( "", off )) +#else + #define DISABLE_OPTIMIZATION +#endif + +#define PROC_LEN (52) + +#define BYTES_REV (1) +#define RADIX_CVT (2) + +#define MIN(a, b) ( ((a) < (b)) ? a : b ) + +__INLINE __mmask8 MB_MASK(int L) { + return (L > 0) ? (__mmask8)0xFF : (__mmask8)0; +} + +__INLINE __mmask64 SB_MASK1(int L, int REV) +{ + if (L <= 0) + return (__mmask64)0x0; + if (L > PROC_LEN) + L = PROC_LEN; + if (REV) + return (__mmask64)(0xFFFFFFFFFFFFFFFFULL << ((int)sizeof(__m512i) - L)); + return (__mmask64)(0xFFFFFFFFFFFFFFFFULL >> ((int)sizeof(__m512i) - L)); +} + + + #ifndef BN_OPENSSL_DISABLE + #include + #if BN_OPENSSL_PATCH + extern BN_ULONG* bn_get_words(const BIGNUM* bn); + #endif + #endif /* BN_OPENSSL_DISABLE */ + +/* +// transpose 8 SB into MB including (reverse bytes and) radix 2^64 => 2^52 conversion +// +// covers: +// - 8 BIGNUM -> mb8 +// - 8 BNU -> mb8 +// - 8 hex strings -> mb8 +*/ +DISABLE_OPTIMIZATION +__INLINE void transform_8sb_to_mb8(U64 out_mb8[], int bitLen, int8u *inp[8], int inpLen[8], int flag) { + // inverse bytes (reverse=1) + const __m512i bswap_mask = _mm512_set_epi64( + 0x0001020304050607, 0x08090a0b0c0d0e0f, + 0x1011121314151617, 0x18191a1b1c1d1e1f, + 0x2021222324252627, 0x28292a2b2c2d2e2f, + 0x3031323334353637, 0x38393a3b3c3d3e3f); + // repeat words + const __m512i idx16 = _mm512_set_epi64( + 0x0019001800170016, + 0x0016001500140013, + 0x0013001200110010, + 0x0010000f000e000d, + 0x000c000b000a0009, + 0x0009000800070006, + 0x0006000500040003, + 0x0003000200010000); + // shift right + const __m512i shiftR = _mm512_set_epi64( + 12, 8, 4, 0, 12, 8, 4, 0); + // radix 2^52 mask of digits + __m512i digMask = _mm512_set1_epi64(DIGIT_MASK); + + int bytesRev = flag & BYTES_REV; /* reverse flag */ + int radixCvt = flag & RADIX_CVT; /* radix (64->52) conversion assumed*/ + + int inpBytes = NUMBER_OF_DIGITS(bitLen, 8); /* bytes */ + int outDigits = NUMBER_OF_DIGITS(bitLen, DIGIT_SIZE); /* digits */ + + int i; + for (i = 0; inpBytes > 0; i += PROC_LEN, inpBytes -= PROC_LEN, out_mb8 += 8) { + int sbidx = bytesRev ? inpBytes - (int)sizeof(__m512i) : i; + + __m512i X0 = _mm512_maskz_loadu_epi8(SB_MASK1(inpLen[0] - i, bytesRev), (__m512i*)&inp[0][sbidx]); + __m512i X1 = _mm512_maskz_loadu_epi8(SB_MASK1(inpLen[1] - i, bytesRev), (__m512i*)&inp[1][sbidx]); + __m512i X2 = _mm512_maskz_loadu_epi8(SB_MASK1(inpLen[2] - i, bytesRev), (__m512i*)&inp[2][sbidx]); + __m512i X3 = _mm512_maskz_loadu_epi8(SB_MASK1(inpLen[3] - i, bytesRev), (__m512i*)&inp[3][sbidx]); + __m512i X4 = _mm512_maskz_loadu_epi8(SB_MASK1(inpLen[4] - i, bytesRev), (__m512i*)&inp[4][sbidx]); + __m512i X5 = _mm512_maskz_loadu_epi8(SB_MASK1(inpLen[5] - i, bytesRev), (__m512i*)&inp[5][sbidx]); + __m512i X6 = _mm512_maskz_loadu_epi8(SB_MASK1(inpLen[6] - i, bytesRev), (__m512i*)&inp[6][sbidx]); + __m512i X7 = _mm512_maskz_loadu_epi8(SB_MASK1(inpLen[7] - i, bytesRev), (__m512i*)&inp[7][sbidx]); + + if (bytesRev) { + X0 = _mm512_permutexvar_epi8(bswap_mask, X0); + X1 = _mm512_permutexvar_epi8(bswap_mask, X1); + X2 = _mm512_permutexvar_epi8(bswap_mask, X2); + X3 = _mm512_permutexvar_epi8(bswap_mask, X3); + X4 = _mm512_permutexvar_epi8(bswap_mask, X4); + X5 = _mm512_permutexvar_epi8(bswap_mask, X5); + X6 = _mm512_permutexvar_epi8(bswap_mask, X6); + X7 = _mm512_permutexvar_epi8(bswap_mask, X7); + } + + if (radixCvt) { + X0 = _mm512_permutexvar_epi16(idx16, X0); + X0 = _mm512_srlv_epi64(X0, shiftR); + X0 = _mm512_and_si512(X0, digMask); /* probably exceeded instruction */ + + X1 = _mm512_permutexvar_epi16(idx16, X1); + X1 = _mm512_srlv_epi64(X1, shiftR); + X1 = _mm512_and_si512(X1, digMask); + + X2 = _mm512_permutexvar_epi16(idx16, X2); + X2 = _mm512_srlv_epi64(X2, shiftR); + X2 = _mm512_and_si512(X2, digMask); + + X3 = _mm512_permutexvar_epi16(idx16, X3); + X3 = _mm512_srlv_epi64(X3, shiftR); + X3 = _mm512_and_si512(X3, digMask); + + X4 = _mm512_permutexvar_epi16(idx16, X4); + X4 = _mm512_srlv_epi64(X4, shiftR); + X4 = _mm512_and_si512(X4, digMask); + + X5 = _mm512_permutexvar_epi16(idx16, X5); + X5 = _mm512_srlv_epi64(X5, shiftR); + X5 = _mm512_and_si512(X5, digMask); + + X6 = _mm512_permutexvar_epi16(idx16, X6); + X6 = _mm512_srlv_epi64(X6, shiftR); + X6 = _mm512_and_si512(X6, digMask); + + X7 = _mm512_permutexvar_epi16(idx16, X7); + X7 = _mm512_srlv_epi64(X7, shiftR); + X7 = _mm512_and_si512(X7, digMask); + } + + // transpose 8 digits at a time + TRANSPOSE_8xI64x8(X0, X1, X2, X3, X4, X5, X6, X7); + + // store transposed digits + _mm512_mask_storeu_epi64(&out_mb8[0], MB_MASK(outDigits--), X0); + _mm512_mask_storeu_epi64(&out_mb8[1], MB_MASK(outDigits--), X1); + _mm512_mask_storeu_epi64(&out_mb8[2], MB_MASK(outDigits--), X2); + _mm512_mask_storeu_epi64(&out_mb8[3], MB_MASK(outDigits--), X3); + _mm512_mask_storeu_epi64(&out_mb8[4], MB_MASK(outDigits--), X4); + _mm512_mask_storeu_epi64(&out_mb8[5], MB_MASK(outDigits--), X5); + _mm512_mask_storeu_epi64(&out_mb8[6], MB_MASK(outDigits--), X6); + _mm512_mask_storeu_epi64(&out_mb8[7], MB_MASK(outDigits--), X7); + } +} + +#ifdef OPENSSL_IS_BORINGSSL +static int BN_bn2lebinpad(const BIGNUM *a, unsigned char *to, int tolen) { + return BN_bn2le_padded(to, tolen, a); +} +#endif + +#ifndef BN_OPENSSL_DISABLE +// Convert BIGNUM into MB8(Radix=2^52) format +// Returns bitmask of succesfully converted values +// Accepts NULLs as BIGNUM inputs +// Null or wrong length +int8u ifma_BN_to_mb8(int64u out_mb8[][8], const BIGNUM* const bn[8], int bitLen) +{ + // check input input length + assert((0 0); + + int byteLens[8]; + int byteLen = NUMBER_OF_DIGITS(bitLen, 8); + int i; + for (i = 0; i < 8; ++i) + byteLens[i] = (NULL != bn[i]) ? byteLen : 0; + + transform_8sb_to_mb8((U64*)out_mb8, bitLen, (int8u**)bn, byteLens, RADIX_CVT); + + return _mm512_cmpneq_epi64_mask(_mm512_loadu_si512((__m512i*)bn), _mm512_setzero_si512()); +} + +int8u ifma_HexStr8_to_mb8(int64u out_mb8[][8], const int8u* const pStr[8], int bitLen) +{ + // check input parameters + assert(bitLen > 0); + + int byteLens[8]; + int byteLen = NUMBER_OF_DIGITS(bitLen, 8); + int i; + for (i = 0; i < 8; i++) + byteLens[i] = (NULL != pStr[i]) ? byteLen : 0; + + transform_8sb_to_mb8((U64*)out_mb8, bitLen, (int8u**)pStr, byteLens, RADIX_CVT | BYTES_REV); + + return _mm512_cmpneq_epi64_mask(_mm512_loadu_si512((__m512i*)pStr), _mm512_setzero_si512()); +} +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/* +// transpose MB into 8 SB including (reverse bytes and) radix 2^52 => 2^64 conversion +// +// covers: +// - mb8 -> 8 BNU +// - mb8 -> 8 hex strings +*/ +DISABLE_OPTIMIZATION +__INLINE void transform_mb8_to_8sb(int8u* out[8], int outLen[8], const U64 inp_mb8[], int bitLen, int flag) +{ + // inverse bytes (reverse=1) + const __m512i bswap_mask = _mm512_set_epi64( + 0x0001020304050607, 0x08090a0b0c0d0e0f, + 0x1011121314151617, 0x18191a1b1c1d1e1f, + 0x2021222324252627, 0x28292a2b2c2d2e2f, + 0x3031323334353637, 0x38393a3b3c3d3e3f); + + const __m512i shiftL = _mm512_set_epi64(4, 0, 4, 0, 4, 0, 4, 0); + + const __m512i permutation1 = _mm512_set_epi64(0x3f3f3f3f3f3f3f3f, // {63,63,63,63,63,63,63,63} + 0x3f3f3f3f3e3d3c3b, // {63,63,63,63,62,61,60,59} + 0x3737363534333231, // {55,55,54,53,52,51,50,49} + 0x302e2d2c2b2a2928, // {48,46,45,44,43,42,41,40} + 0x1f1f1f1f1f1f1e1d, // {31,31,31,31,31,31,30,29} + 0x1717171716151413, // {23,23,23,23,22,21,20,19} + 0x0f0f0f0e0d0c0b0a, // {15,15,15,14,13,12,11,10} + 0x0706050403020100); // { 7, 6, 5, 4, 3, 2, 1, 0} + + const __m512i permutation2 = _mm512_set_epi64(0x3f3f3f3f3f3f3f3f, // {63,63,63,63,63,63,63,63} + 0x3f3f3f3f3f3f3f3f, // {63,63,63,63,63,63,63,63} + 0x3a39383737373737, // {58,57,56,55,55,55,55,55} + 0x2727272727272726, // {39,39,39,39,39,39,39,38} + 0x2524232221201f1f, // {37,36,35,34,33,32,31,31} + 0x1c1b1a1918171717, // {28,27,26,25,24,23,23,23} + 0x1211100f0f0f0f0f, // {18,17,16,15,15,15,15,15} + 0x0908070707070707); // { 9, 8, 7, 7, 7, 7, 7, 7} + int bytesRev = flag & BYTES_REV; /* reverse flag */ + int radixCvt = flag & RADIX_CVT; /* radix (52->64) conversion assumed */ + + int inpDigits = NUMBER_OF_DIGITS(bitLen, DIGIT_SIZE); /* digits */ + int outBytes = NUMBER_OF_DIGITS(bitLen, 8); /* bytes */ + + int i; + for (i = 0; outBytes > 0; i += PROC_LEN, outBytes -= PROC_LEN, inp_mb8 += 8) { + int sbidx = bytesRev ? outBytes - (int)sizeof(__m512i) : i; + + __m512i X0 = _mm512_maskz_loadu_epi64(MB_MASK(inpDigits--), &inp_mb8[0]); + __m512i X1 = _mm512_maskz_loadu_epi64(MB_MASK(inpDigits--), &inp_mb8[1]); + __m512i X2 = _mm512_maskz_loadu_epi64(MB_MASK(inpDigits--), &inp_mb8[2]); + __m512i X3 = _mm512_maskz_loadu_epi64(MB_MASK(inpDigits--), &inp_mb8[3]); + __m512i X4 = _mm512_maskz_loadu_epi64(MB_MASK(inpDigits--), &inp_mb8[4]); + __m512i X5 = _mm512_maskz_loadu_epi64(MB_MASK(inpDigits--), &inp_mb8[5]); + __m512i X6 = _mm512_maskz_loadu_epi64(MB_MASK(inpDigits--), &inp_mb8[6]); + __m512i X7 = _mm512_maskz_loadu_epi64(MB_MASK(inpDigits--), &inp_mb8[7]); + + // transpose 8 digits at a time + TRANSPOSE_8xI64x8(X0, X1, X2, X3, X4, X5, X6, X7); + + if (radixCvt) { + __m512i T; + X0 = _mm512_sllv_epi64(X0, shiftL); + T = _mm512_permutexvar_epi8(permutation1, X0); + X0 = _mm512_permutexvar_epi8(permutation2, X0); + X0 = _mm512_or_si512(X0, T); + + X1 = _mm512_sllv_epi64(X1, shiftL); + T = _mm512_permutexvar_epi8(permutation1, X1); + X1 = _mm512_permutexvar_epi8(permutation2, X1); + X1 = _mm512_or_si512(X1, T); + + X2 = _mm512_sllv_epi64(X2, shiftL); + T = _mm512_permutexvar_epi8(permutation1, X2); + X2 = _mm512_permutexvar_epi8(permutation2, X2); + X2 = _mm512_or_si512(X2, T); + + X3 = _mm512_sllv_epi64(X3, shiftL); + T = _mm512_permutexvar_epi8(permutation1, X3); + X3 = _mm512_permutexvar_epi8(permutation2, X3); + X3 = _mm512_or_si512(X3, T); + + X4 = _mm512_sllv_epi64(X4, shiftL); + T = _mm512_permutexvar_epi8(permutation1, X4); + X4 = _mm512_permutexvar_epi8(permutation2, X4); + X4 = _mm512_or_si512(X4, T); + + X5 = _mm512_sllv_epi64(X5, shiftL); + T = _mm512_permutexvar_epi8(permutation1, X5); + X5 = _mm512_permutexvar_epi8(permutation2, X5); + X5 = _mm512_or_si512(X5, T); + + X6 = _mm512_sllv_epi64(X6, shiftL); + T = _mm512_permutexvar_epi8(permutation1, X6); + X6 = _mm512_permutexvar_epi8(permutation2, X6); + X6 = _mm512_or_si512(X6, T); + + X7 = _mm512_sllv_epi64(X7, shiftL); + T = _mm512_permutexvar_epi8(permutation1, X7); + X7 = _mm512_permutexvar_epi8(permutation2, X7); + X7 = _mm512_or_si512(X7, T); + } + + if (bytesRev) { + X0 = _mm512_permutexvar_epi8(bswap_mask, X0); + X1 = _mm512_permutexvar_epi8(bswap_mask, X1); + X2 = _mm512_permutexvar_epi8(bswap_mask, X2); + X3 = _mm512_permutexvar_epi8(bswap_mask, X3); + X4 = _mm512_permutexvar_epi8(bswap_mask, X4); + X5 = _mm512_permutexvar_epi8(bswap_mask, X5); + X6 = _mm512_permutexvar_epi8(bswap_mask, X6); + X7 = _mm512_permutexvar_epi8(bswap_mask, X7); + } + + // store transposed digits + _mm512_mask_storeu_epi8(out[0] + sbidx, SB_MASK1(outLen[0] - i, bytesRev), X0); + _mm512_mask_storeu_epi8(out[1] + sbidx, SB_MASK1(outLen[1] - i, bytesRev), X1); + _mm512_mask_storeu_epi8(out[2] + sbidx, SB_MASK1(outLen[2] - i, bytesRev), X2); + _mm512_mask_storeu_epi8(out[3] + sbidx, SB_MASK1(outLen[3] - i, bytesRev), X3); + _mm512_mask_storeu_epi8(out[4] + sbidx, SB_MASK1(outLen[4] - i, bytesRev), X4); + _mm512_mask_storeu_epi8(out[5] + sbidx, SB_MASK1(outLen[5] - i, bytesRev), X5); + _mm512_mask_storeu_epi8(out[6] + sbidx, SB_MASK1(outLen[6] - i, bytesRev), X6); + _mm512_mask_storeu_epi8(out[7] + sbidx, SB_MASK1(outLen[7] - i, bytesRev), X7); + } + +} + +int8u ifma_mb8_to_BNU(int64u* const out_bn[8], const int64u inp_mb8[][8], const int bitLen) +{ + // Check input parameters + assert(bitLen > 0); + + int bnu_bitlen = NUMBER_OF_DIGITS(bitLen, 64) * 64; // gres: output length is multiple 64 + int byteLens[8]; + int i; + for (i = 0; i < 8; ++i) + //gres: byteLens[i] = (NULL != out_bn[i]) ? NUMBER_OF_DIGITS(bitLen, 8) : 0; + byteLens[i] = (NULL != out_bn[i]) ? NUMBER_OF_DIGITS(bnu_bitlen, 8) : 0; + + transform_mb8_to_8sb((int8u**)out_bn, byteLens, (U64*)inp_mb8, bitLen, RADIX_CVT); + + return _mm512_cmpneq_epi64_mask(_mm512_loadu_si512((__m512i*)out_bn), _mm512_setzero_si512()); +} + +int8u ifma_mb8_to_HexStr8(int8u* const pStr[8], const int64u inp_mb8[][8], int bitLen) +{ + // check input parameters + assert(bitLen > 0); + + int byteLens[8]; + int byteLen = NUMBER_OF_DIGITS(bitLen, 8); + int i; + for (i = 0; i < 8; i++) + byteLens[i] = (NULL != pStr[i]) ? byteLen : 0; + + transform_mb8_to_8sb((int8u**)pStr, byteLens, (U64*)inp_mb8, bitLen, RADIX_CVT | BYTES_REV); + + return _mm512_cmpneq_epi64_mask(_mm512_loadu_si512((__m512i*)pStr), _mm512_setzero_si512()); +} +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/* +// transpose 8 SB into MB without radix conversion +// +// covers: +// - mb8 -> 8 BNU +// - mb8 -> 8 hex strings +*/ +DISABLE_OPTIMIZATION +int8u ifma_BNU_transpose_copy(int64u out_mb8[][8], const int64u* const bn[8], int bitLen) +{ + // Check input parameters + assert(bitLen > 0); + + __mmask8 kbn[8]; + int i; + for (i = 0; i < 8; ++i) + kbn[i] = (NULL == bn[i]) ? (__mmask8)0 : (__mmask8)0xFF; + + int len = NUMBER_OF_DIGITS(bitLen, 64); + int n; + for (n = 0; len > 0; n += 8, out_mb8 += 8) { + __mmask8 kread = (len >= 8) ? 0xFF : (__mmask8)((1 << len) - 1); + + __m512i X0 = _mm512_maskz_loadu_epi64(kread & kbn[0], bn[0] + n); + __m512i X1 = _mm512_maskz_loadu_epi64(kread & kbn[1], bn[1] + n); + __m512i X2 = _mm512_maskz_loadu_epi64(kread & kbn[2], bn[2] + n); + __m512i X3 = _mm512_maskz_loadu_epi64(kread & kbn[3], bn[3] + n); + __m512i X4 = _mm512_maskz_loadu_epi64(kread & kbn[4], bn[4] + n); + __m512i X5 = _mm512_maskz_loadu_epi64(kread & kbn[5], bn[5] + n); + __m512i X6 = _mm512_maskz_loadu_epi64(kread & kbn[6], bn[6] + n); + __m512i X7 = _mm512_maskz_loadu_epi64(kread & kbn[7], bn[7] + n); + + TRANSPOSE_8xI64x8(X0, X1, X2, X3, X4, X5, X6, X7); + + _mm512_mask_storeu_epi64(&out_mb8[0], MB_MASK(len--), X0); + _mm512_mask_storeu_epi64(&out_mb8[1], MB_MASK(len--), X1); + _mm512_mask_storeu_epi64(&out_mb8[2], MB_MASK(len--), X2); + _mm512_mask_storeu_epi64(&out_mb8[3], MB_MASK(len--), X3); + _mm512_mask_storeu_epi64(&out_mb8[4], MB_MASK(len--), X4); + _mm512_mask_storeu_epi64(&out_mb8[5], MB_MASK(len--), X5); + _mm512_mask_storeu_epi64(&out_mb8[6], MB_MASK(len--), X6); + _mm512_mask_storeu_epi64(&out_mb8[7], MB_MASK(len--), X7); + } + + return _mm512_cmpneq_epi64_mask(_mm512_loadu_si512((__m512i*)bn), _mm512_setzero_si512()); +} + +#ifndef BN_OPENSSL_DISABLE + +DISABLE_OPTIMIZATION +int8u ifma_BN_transpose_copy(int64u out_mb8[][8], const BIGNUM* const bn[8], int bitLen) +{ + // check input length + assert((0 0; n += 8, out_mb8 += 8) { + __mmask8 k = (len >= 8) ? 0xFF : (1 << len) - 1; + + __m512i X0 = _mm512_maskz_loadu_epi64(k & kbn[0], inp[0]+n); + __m512i X1 = _mm512_maskz_loadu_epi64(k & kbn[1], inp[1]+n); + __m512i X2 = _mm512_maskz_loadu_epi64(k & kbn[2], inp[2]+n); + __m512i X3 = _mm512_maskz_loadu_epi64(k & kbn[3], inp[3]+n); + __m512i X4 = _mm512_maskz_loadu_epi64(k & kbn[4], inp[4]+n); + __m512i X5 = _mm512_maskz_loadu_epi64(k & kbn[5], inp[5]+n); + __m512i X6 = _mm512_maskz_loadu_epi64(k & kbn[6], inp[6]+n); + __m512i X7 = _mm512_maskz_loadu_epi64(k & kbn[7], inp[7]+n); + + TRANSPOSE_8xI64x8(X0, X1, X2, X3, X4, X5, X6, X7); + + _mm512_mask_storeu_epi64(&out_mb8[0], MB_MASK(len--), X0); + _mm512_mask_storeu_epi64(&out_mb8[1], MB_MASK(len--), X1); + _mm512_mask_storeu_epi64(&out_mb8[2], MB_MASK(len--), X2); + _mm512_mask_storeu_epi64(&out_mb8[3], MB_MASK(len--), X3); + _mm512_mask_storeu_epi64(&out_mb8[4], MB_MASK(len--), X4); + _mm512_mask_storeu_epi64(&out_mb8[5], MB_MASK(len--), X5); + _mm512_mask_storeu_epi64(&out_mb8[6], MB_MASK(len--), X6); + _mm512_mask_storeu_epi64(&out_mb8[7], MB_MASK(len--), X7); + } + + return _mm512_cmpneq_epi64_mask(_mm512_loadu_si512((__m512i*)bn), _mm512_setzero_si512()); +} +#endif /* BN_OPENSSL_DISABLE */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/common/ifma_version.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/common/ifma_version.c new file mode 100644 index 0000000..86cbb1a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/common/ifma_version.c @@ -0,0 +1,42 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#define MBX_LIB_VERSION() MBX_VER_MAJOR,MBX_VER_MINOR,MBX_VER_REV +#define MBX_LIB_BUILD() __DATE__ + +#define STR2(x) #x +#define STR(x) STR2(x) +#define MBX_STR_VERSION() MBX_LIB_NAME() \ + " (ver: " STR(MBX_VER_MAJOR) "." STR(MBX_VER_MINOR) "." STR(MBX_VER_REV) " (" STR(MBX_INTERFACE_VERSION_MAJOR)"."STR(MBX_INTERFACE_VERSION_MINOR)")" \ + " build: " MBX_LIB_BUILD()")" + +/* version info */ +static const mbxVersion mbxLibVer = { + MBX_LIB_VERSION(), /* major, minor, revision */ + MBX_LIB_NAME(), /* lib name */ + MBX_LIB_BUILD(), /* build date */ + MBX_STR_VERSION() /* version str */ +}; + +DLL_PUBLIC +const mbxVersion* mbx_getversion(void) +{ + return &mbxLibVer; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_arith_m256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_arith_m256.c new file mode 100644 index 0000000..8c7410b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_arith_m256.c @@ -0,0 +1,368 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + + +/*===================================================================== + + General 256-bit operations - sqr & mul + +=====================================================================*/ + +void MB_FUNC_NAME(ifma_amm52x5_)(U64 R[], const U64 inpA[], const U64 inpB[], const U64 inpM[], const int64u* k0_mb) +{ + U64 res00, res01, res02, res03, res04; + U64 K = loadu64(k0_mb); /* k0[] */ + int itr; + + res00 = res01 = res02 = res03 = res04 = get_zero64(); + + for (itr = 0; itr < P256_LEN52; itr++) { + U64 Yi; + U64 Bi = loadu64(inpB); + inpB++; + + res00 = fma52lo(res00, Bi, inpA[0]); + res01 = fma52lo(res01, Bi, inpA[1]); + res02 = fma52lo(res02, Bi, inpA[2]); + res03 = fma52lo(res03, Bi, inpA[3]); + res04 = fma52lo(res04, Bi, inpA[4]); + + Yi = fma52lo(get_zero64(), res00, K); + + res00 = fma52lo(res00, Yi, inpM[0]); + res01 = fma52lo(res01, Yi, inpM[1]); + res02 = fma52lo(res02, Yi, inpM[2]); + res03 = fma52lo(res03, Yi, inpM[3]); + res04 = fma52lo(res04, Yi, inpM[4]); + + res00 = srli64(res00, DIGIT_SIZE); + res01 = add64 (res01, res00); + + res00 = fma52hi(res01, Bi, inpA[0]); + res01 = fma52hi(res02, Bi, inpA[1]); + res02 = fma52hi(res03, Bi, inpA[2]); + res03 = fma52hi(res04, Bi, inpA[3]); + res04 = fma52hi(get_zero64(), Bi, inpA[4]); + + res00 = fma52hi(res00, Yi, inpM[0]); + res01 = fma52hi(res01, Yi, inpM[1]); + res02 = fma52hi(res02, Yi, inpM[2]); + res03 = fma52hi(res03, Yi, inpM[3]); + res04 = fma52hi(res04, Yi, inpM[4]); + } + + // normalization + NORM_LSHIFTR(res0, 0, 1) + NORM_LSHIFTR(res0, 1, 2) + NORM_LSHIFTR(res0, 2, 3) + NORM_LSHIFTR(res0, 3, 4) + +#if 0 + // t = res -modulus and normalize + U64 t0 = sub64(res00, inpM[0]); + U64 t1 = sub64(res01, inpM[1]); + U64 t2 = sub64(res02, inpM[2]); + U64 t3 = sub64(res03, inpM[3]); + U64 t4 = sub64(res04, inpM[4]); + NORM_ASHIFTR(t, 0,1) + NORM_ASHIFTR(t, 1,2) + NORM_ASHIFTR(t, 2,3) + NORM_ASHIFTR(t, 3,4) + + /* condition mov R[] = (t4>=0)? t : res */ + __mb_mask cmask = cmp64_mask(t4, get_zero64(), _MM_CMPINT_GE); + R[0] = cmov_U64(res00, t0, cmask); + R[1] = cmov_U64(res01, t1, cmask); + R[2] = cmov_U64(res02, t2, cmask); + R[3] = cmov_U64(res03, t3, cmask); + R[4] = cmov_U64(res04, t4, cmask); +#endif + R[0] = res00; + R[1] = res01; + R[2] = res02; + R[3] = res03; + R[4] = res04; +} + +void MB_FUNC_NAME(ifma_ams52x5_)(U64 r[], const U64 a[], const U64 m[], const int64u* k0_mb) +{ + U64 k = loadu64((U64*)k0_mb); + U64 res0, res1, res2, res3, res4, res5, res6, res7, res8, res9; + res0 = res1 = res2 = res3 = res4 = res5 = res6 = res7 = res8 = res9 = get_zero64(); + + // Calculate full square + res1 = fma52lo(res1, a[0], a[1]); // Sum(1) + res2 = fma52hi(res2, a[0], a[1]); // Sum(1) + res2 = fma52lo(res2, a[0], a[2]); // Sum(2) + res3 = fma52hi(res3, a[0], a[2]); // Sum(2) + res3 = fma52lo(res3, a[1], a[2]); // Sum(3) + res4 = fma52hi(res4, a[1], a[2]); // Sum(3) + res3 = fma52lo(res3, a[0], a[3]); // Sum(3) + res4 = fma52hi(res4, a[0], a[3]); // Sum(3) + res4 = fma52lo(res4, a[1], a[3]); // Sum(4) + res5 = fma52hi(res5, a[1], a[3]); // Sum(4) + res5 = fma52lo(res5, a[2], a[3]); // Sum(5) + res6 = fma52hi(res6, a[2], a[3]); // Sum(5) + res4 = fma52lo(res4, a[0], a[4]); // Sum(4) + res5 = fma52hi(res5, a[0], a[4]); // Sum(4) + res5 = fma52lo(res5, a[1], a[4]); // Sum(5) + res6 = fma52hi(res6, a[1], a[4]); // Sum(5) + res6 = fma52lo(res6, a[2], a[4]); // Sum(6) + res7 = fma52hi(res7, a[2], a[4]); // Sum(6) + res7 = fma52lo(res7, a[3], a[4]); // Sum(7) + res8 = fma52hi(res8, a[3], a[4]); // Sum(7) + { + res0 = add64(res0, res0); // Double(0) + res1 = add64(res1, res1); // Double(1) + res2 = add64(res2, res2); // Double(2) + res3 = add64(res3, res3); // Double(3) + res4 = add64(res4, res4); // Double(4) + res5 = add64(res5, res5); // Double(5) + res6 = add64(res6, res6); // Double(6) + res7 = add64(res7, res7); // Double(7) + res8 = add64(res8, res8); // Double(8) + { + res0 = fma52lo(res0, a[0], a[0]); // Add sqr(0) + res1 = fma52hi(res1, a[0], a[0]); // Add sqr(0) + res2 = fma52lo(res2, a[1], a[1]); // Add sqr(2) + res3 = fma52hi(res3, a[1], a[1]); // Add sqr(2) + res4 = fma52lo(res4, a[2], a[2]); // Add sqr(4) + res5 = fma52hi(res5, a[2], a[2]); // Add sqr(4) + res6 = fma52lo(res6, a[3], a[3]); // Add sqr(6) + res7 = fma52hi(res7, a[3], a[3]); // Add sqr(6) + res8 = fma52lo(res8, a[4], a[4]); // Add sqr(8) + res9 = fma52hi(res9, a[4], a[4]); // Add sqr(8) + } + } + + // Reduction + U64 u0 = mul52lo(res0,k); + + // Create u0 + res0 = fma52lo(res0, u0, m[0]); + res1 = fma52hi(res1, u0, m[0]); + res1 = fma52lo(res1, u0, m[1]); + res2 = fma52hi(res2, u0, m[1]); + res1 = add64(res1, srli64(res0, DIGIT_SIZE)); + U64 u1 = mul52lo(res1,k); + res2 = fma52lo(res2, u0, m[2]); + res3 = fma52hi(res3, u0, m[2]); + res3 = fma52lo(res3, u0, m[3]); + res4 = fma52hi(res4, u0, m[3]); + res4 = fma52lo(res4, u0, m[4]); + res5 = fma52hi(res5, u0, m[4]); + + // Create u1 + res1 = fma52lo(res1, u1, m[0]); + res2 = fma52hi(res2, u1, m[0]); + res2 = fma52lo(res2, u1, m[1]); + res3 = fma52hi(res3, u1, m[1]); + res2 = add64(res2, srli64(res1, DIGIT_SIZE)); + U64 u2 = mul52lo(res2,k); + res3 = fma52lo(res3, u1, m[2]); + res4 = fma52hi(res4, u1, m[2]); + res4 = fma52lo(res4, u1, m[3]); + res5 = fma52hi(res5, u1, m[3]); + res5 = fma52lo(res5, u1, m[4]); + res6 = fma52hi(res6, u1, m[4]); + + // Create u2 + res2 = fma52lo(res2, u2, m[0]); + res3 = fma52hi(res3, u2, m[0]); + res3 = fma52lo(res3, u2, m[1]); + res4 = fma52hi(res4, u2, m[1]); + res3 = add64(res3, srli64(res2, DIGIT_SIZE)); + U64 u3 = mul52lo(res3,k); + res4 = fma52lo(res4, u2, m[2]); + res5 = fma52hi(res5, u2, m[2]); + res5 = fma52lo(res5, u2, m[3]); + res6 = fma52hi(res6, u2, m[3]); + res6 = fma52lo(res6, u2, m[4]); + res7 = fma52hi(res7, u2, m[4]); + + // Create u3 + res3 = fma52lo(res3, u3, m[0]); + res4 = fma52hi(res4, u3, m[0]); + res4 = fma52lo(res4, u3, m[1]); + res5 = fma52hi(res5, u3, m[1]); + res4 = add64(res4, srli64(res3, DIGIT_SIZE)); + U64 u4 = mul52lo(res4,k); + res5 = fma52lo(res5, u3, m[2]); + res6 = fma52hi(res6, u3, m[2]); + res6 = fma52lo(res6, u3, m[3]); + res7 = fma52hi(res7, u3, m[3]); + res7 = fma52lo(res7, u3, m[4]); + res8 = fma52hi(res8, u3, m[4]); + + // Create u4 + res4 = fma52lo(res4, u4, m[0]); + res5 = fma52hi(res5, u4, m[0]); + res5 = fma52lo(res5, u4, m[1]); + res6 = fma52hi(res6, u4, m[1]); + res5 = add64(res5, srli64(res4, DIGIT_SIZE)); + res6 = fma52lo(res6, u4, m[2]); + res7 = fma52hi(res7, u4, m[2]); + res7 = fma52lo(res7, u4, m[3]); + res8 = fma52hi(res8, u4, m[3]); + res8 = fma52lo(res8, u4, m[4]); + res9 = fma52hi(res9, u4, m[4]); + + // normalization + NORM_LSHIFTR(res, 5, 6) + NORM_LSHIFTR(res, 6, 7) + NORM_LSHIFTR(res, 7, 8) + NORM_LSHIFTR(res, 8, 9) +#if 0 + // {res0-4) = (res5-9) -modulus and normalize + res0 = sub64(res5, m[0]); + res1 = sub64(res6, m[1]); + res2 = sub64(res7, m[2]); + res3 = sub64(res8, m[3]); + res4 = sub64(res9, m[4]); + NORM_ASHIFTR(res, 0,1) + NORM_ASHIFTR(res, 1,2) + NORM_ASHIFTR(res, 2,3) + NORM_ASHIFTR(res, 3,4) + + /* condition mov r[] = (res4>=0)? res(5 - 9) : res(0 - 4) */ + __mb_mask cmask = cmp64_mask(res4, get_zero64(), _MM_CMPINT_GE); + r[0] = cmov_U64(res5, res0, cmask); + r[1] = cmov_U64(res6, res1, cmask); + r[2] = cmov_U64(res7, res2, cmask); + r[3] = cmov_U64(res8, res3, cmask); + r[4] = cmov_U64(res9, res4, cmask); +#endif + + r[0] = res5; + r[1] = res6; + r[2] = res7; + r[3] = res8; + r[4] = res9; +} +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + +/* R = (A+B) mod M */ +void MB_FUNC_NAME(ifma_add52x5_)(U64 R[], const U64 A[], const U64 B[], const U64 M[]) +{ + /* r = a + b */ + U64 r0 = add64(A[0], B[0]); + U64 r1 = add64(A[1], B[1]); + U64 r2 = add64(A[2], B[2]); + U64 r3 = add64(A[3], B[3]); + U64 r4 = add64(A[4], B[4]); + + /* t = r - M */ + U64 t0 = sub64(r0, M[0]); + U64 t1 = sub64(r1, M[1]); + U64 t2 = sub64(r2, M[2]); + U64 t3 = sub64(r3, M[3]); + U64 t4 = sub64(r4, M[4]); + + /* normalize r0, r1, r2, r3, r4 */ + NORM_LSHIFTR(r, 0,1) + NORM_LSHIFTR(r, 1,2) + NORM_LSHIFTR(r, 2,3) + NORM_LSHIFTR(r, 3,4) + + /* normalize t0, t1, t2, t3, t4 */ + NORM_ASHIFTR(t, 0,1) + NORM_ASHIFTR(t, 1,2) + NORM_ASHIFTR(t, 2,3) + NORM_ASHIFTR(t, 3,4) + + /* condition mask t4<0? (-1) : 0 */ + __mb_mask cmask = cmp64_mask(t4, get_zero64(), _MM_CMPINT_LT); + + R[0] = cmov_U64(t0, r0, cmask); + R[1] = cmov_U64(t1, r1, cmask); + R[2] = cmov_U64(t2, r2, cmask); + R[3] = cmov_U64(t3, r3, cmask); + R[4] = cmov_U64(t4, r4, cmask); +} + + +/* R = (A-B) mod M */ +void MB_FUNC_NAME(ifma_sub52x5_)(U64 R[], const U64 A[], const U64 B[], const U64 M[]) +{ + /* r = a-b */ + U64 r0 = sub64(A[0], B[0]); + U64 r1 = sub64(A[1], B[1]); + U64 r2 = sub64(A[2], B[2]); + U64 r3 = sub64(A[3], B[3]); + U64 r4 = sub64(A[4], B[4]); + + /* t = r + M */ + U64 t0 = add64(r0, M[0]); + U64 t1 = add64(r1, M[1]); + U64 t2 = add64(r2, M[2]); + U64 t3 = add64(r3, M[3]); + U64 t4 = add64(r4, M[4]); + + /* normalize r0, r1, r2, r3, r4 */ + NORM_ASHIFTR(r, 0,1) + NORM_ASHIFTR(r, 1,2) + NORM_ASHIFTR(r, 2,3) + NORM_ASHIFTR(r, 3,4) + + /* normalize t0, t1, t2, t3, t4 */ + NORM_ASHIFTR(t, 0,1) + NORM_ASHIFTR(t, 1,2) + NORM_ASHIFTR(t, 2,3) + NORM_ASHIFTR(t, 3,4) + + /* condition mask t4<0? (-1) : 0 */ + __mb_mask cmask = cmp64_mask(r4, get_zero64(), _MM_CMPINT_LT); + + R[0] = cmov_U64(r0, t0, cmask); + R[1] = cmov_U64(r1, t1, cmask); + R[2] = cmov_U64(r2, t2, cmask); + R[3] = cmov_U64(r3, t3, cmask); + R[4] = cmov_U64(r4, t4, cmask); +} + +/* R = (-A) mod M */ +void MB_FUNC_NAME(ifma_neg52x5_)(U64 R[], const U64 A[], const U64 M[]) +{ + /* mask = a[]!=0? 1 : 0 */ + U64 t = _mm512_or_epi64(A[0], A[1]); + t = _mm512_or_epi64(t, A[2]); + t = _mm512_or_epi64(t, A[3]); + t = _mm512_or_epi64(t, A[4]); + __mb_mask mask = cmp64_mask(t, get_zero64(), _MM_CMPINT_NE); + + /* r = M - A */ + U64 r0 = _mm512_maskz_sub_epi64(mask, M[0], A[0]); + U64 r1 = _mm512_maskz_sub_epi64(mask, M[1], A[1]); + U64 r2 = _mm512_maskz_sub_epi64(mask, M[2], A[2]); + U64 r3 = _mm512_maskz_sub_epi64(mask, M[3], A[3]); + U64 r4 = _mm512_maskz_sub_epi64(mask, M[4], A[4]); + + /* normalize r0, r1, r2, r3, r4 */ + NORM_ASHIFTR(r, 0,1) + NORM_ASHIFTR(r, 1,2) + NORM_ASHIFTR(r, 2,3) + NORM_ASHIFTR(r, 3,4) + + R[0] = r0; + R[1] = r1; + R[2] = r2; + R[3] = r3; + R[4] = r4; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_arith_n256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_arith_n256.c new file mode 100644 index 0000000..f37f4a8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_arith_n256.c @@ -0,0 +1,219 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +/* Constants */ +#define LEN52 NUMBER_OF_DIGITS(256,DIGIT_SIZE) + +/* +// EC NIST-P256 prime base point order +// in 2^52 radix +*/ +__ALIGN64 static const int64u n256_mb[LEN52][8] = { + {0x9cac2fc632551, 0x9cac2fc632551, 0x9cac2fc632551, 0x9cac2fc632551, 0x9cac2fc632551, 0x9cac2fc632551, 0x9cac2fc632551, 0x9cac2fc632551}, + {0xada7179e84f3b, 0xada7179e84f3b, 0xada7179e84f3b, 0xada7179e84f3b, 0xada7179e84f3b, 0xada7179e84f3b, 0xada7179e84f3b, 0xada7179e84f3b}, + {0xfffffffbce6fa, 0xfffffffbce6fa, 0xfffffffbce6fa, 0xfffffffbce6fa, 0xfffffffbce6fa, 0xfffffffbce6fa, 0xfffffffbce6fa, 0xfffffffbce6fa}, + {0x0000fffffffff, 0x0000fffffffff, 0x0000fffffffff, 0x0000fffffffff, 0x0000fffffffff, 0x0000fffffffff, 0x0000fffffffff, 0x0000fffffffff}, + {0x0ffffffff0000, 0x0ffffffff0000, 0x0ffffffff0000, 0x0ffffffff0000, 0x0ffffffff0000, 0x0ffffffff0000, 0x0ffffffff0000, 0x0ffffffff0000} +}; + +__ALIGN64 static const int64u n256x2_mb[LEN52][8] = { + {0x39585f8c64aa2, 0x39585f8c64aa2, 0x39585f8c64aa2, 0x39585f8c64aa2, 0x39585f8c64aa2, 0x39585f8c64aa2, 0x39585f8c64aa2, 0x39585f8c64aa2}, + {0x5b4e2f3d09e77, 0x5b4e2f3d09e77, 0x5b4e2f3d09e77, 0x5b4e2f3d09e77, 0x5b4e2f3d09e77, 0x5b4e2f3d09e77, 0x5b4e2f3d09e77, 0x5b4e2f3d09e77}, + {0xfffffff79cdf5, 0xfffffff79cdf5, 0xfffffff79cdf5, 0xfffffff79cdf5, 0xfffffff79cdf5, 0xfffffff79cdf5, 0xfffffff79cdf5, 0xfffffff79cdf5}, + {0x0001fffffffff, 0x0001fffffffff, 0x0001fffffffff, 0x0001fffffffff, 0x0001fffffffff, 0x0001fffffffff, 0x0001fffffffff, 0x0001fffffffff}, + {0x1fffffffe0000, 0x1fffffffe0000, 0x1fffffffe0000, 0x1fffffffe0000, 0x1fffffffe0000, 0x1fffffffe0000, 0x1fffffffe0000, 0x1fffffffe0000} +}; + +/* k0 = -( (1/n256 mod 2^DIGIT_SIZE) ) mod 2^DIGIT_SIZE */ +__ALIGN64 static const int64u n256_k0_mb[8] = { + 0x1c8aaee00bc4f, 0x1c8aaee00bc4f, 0x1c8aaee00bc4f, 0x1c8aaee00bc4f, 0x1c8aaee00bc4f, 0x1c8aaee00bc4f, 0x1c8aaee00bc4f, 0x1c8aaee00bc4f +}; + +/* to Montgomery conversion constant +// rr = 2^((LEN52*DIGIT_SIZE)*2) mod n256 +*/ +__ALIGN64 static const int64u n256_rr_mb[LEN52][8] = { + {0x0005cc0dea6dc3ba,0x0005cc0dea6dc3ba,0x0005cc0dea6dc3ba,0x0005cc0dea6dc3ba,0x0005cc0dea6dc3ba,0x0005cc0dea6dc3ba,0x0005cc0dea6dc3ba,0x0005cc0dea6dc3ba}, + {0x000192a067d8a084,0x000192a067d8a084,0x000192a067d8a084,0x000192a067d8a084,0x000192a067d8a084,0x000192a067d8a084,0x000192a067d8a084,0x000192a067d8a084}, + {0x000bec59615571bb,0x000bec59615571bb,0x000bec59615571bb,0x000bec59615571bb,0x000bec59615571bb,0x000bec59615571bb,0x000bec59615571bb,0x000bec59615571bb}, + {0x0001fc245b2392b6,0x0001fc245b2392b6,0x0001fc245b2392b6,0x0001fc245b2392b6,0x0001fc245b2392b6,0x0001fc245b2392b6,0x0001fc245b2392b6,0x0001fc245b2392b6}, + {0x0000e12d9559d956,0x0000e12d9559d956,0x0000e12d9559d956,0x0000e12d9559d956,0x0000e12d9559d956,0x0000e12d9559d956,0x0000e12d9559d956,0x0000e12d9559d956} +}; + + +/*===================================================================== + + Specialized single operations in n256 - sqr & mul + +=====================================================================*/ +EXTERN_C U64* MB_FUNC_NAME(ifma_n256_)(void) +{ + return (U64*)n256_mb; +} + +void MB_FUNC_NAME(ifma_ams52_n256_)(U64 r[], const U64 a[]) +{ + MB_FUNC_NAME(ifma_ams52x5_)(r, a, (U64*)n256_mb, n256_k0_mb); +} + +void MB_FUNC_NAME(ifma_amm52_n256_)(U64 r[], const U64 a[], const U64 b[]) +{ + MB_FUNC_NAME(ifma_amm52x5_)(r, a, b, (U64*)n256_mb, n256_k0_mb); +} + +void MB_FUNC_NAME(ifma_tomont52_n256_)(U64 r[], const U64 a[]) +{ + MB_FUNC_NAME(ifma_amm52x5_)(r, a, (U64*)n256_rr_mb, (U64*)n256_mb, n256_k0_mb); +} + +void MB_FUNC_NAME(ifma_frommont52_n256_)(U64 r[], const U64 a[]) +{ + MB_FUNC_NAME(ifma_amm52_n256_)(r, a, (U64*)ones); +} + +/* +// computes r = 1/z = z^(n256-2) mod n256 +// +// note: z in in Montgomery domain (as soon mul() and sqr() below are amm-functions +// r in Montgomery domain too +*/ +#define sqr_n256 MB_FUNC_NAME(ifma_ams52_n256_) +#define mul_n256 MB_FUNC_NAME(ifma_amm52_n256_) + +void MB_FUNC_NAME(ifma_aminv52_n256_)(U64 r[], const U64 z[]) +{ + int i; + + // pwr_z_Tbl[i][] = z^i, i=0,..,15 + __ALIGN64 U64 pwr_z_Tbl[16][LEN52]; + + MB_FUNC_NAME(ifma_tomont52_n256_)(pwr_z_Tbl[0], (U64*)ones); + MB_FUNC_NAME(mov_FE256_)(pwr_z_Tbl[1], z); + + for(i=2; i<16; i+=2) { + sqr_n256(pwr_z_Tbl[i], pwr_z_Tbl[i/2]); + mul_n256(pwr_z_Tbl[i+1], pwr_z_Tbl[i], z); + } + + // pwr = (n256-2) in big endian + int8u pwr[] = "\xFF\xFF\xFF\xFF\x00\x00\x00\x00" + "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" + "\xBC\xE6\xFA\xAD\xA7\x17\x9E\x84" + "\xF3\xB9\xCA\xC2\xFC\x63\x25\x4F"; + // init r = 1 + MB_FUNC_NAME(mov_FE256_)(r, pwr_z_Tbl[0]); + + for(i=0; i<32; i++) { + int v = pwr[i]; + int hi = (v>>4) &0xF; + int lo = v & 0xF; + + sqr_n256(r, r); + sqr_n256(r, r); + sqr_n256(r, r); + sqr_n256(r, r); + if(hi) + mul_n256(r, r, pwr_z_Tbl[hi]); + sqr_n256(r, r); + sqr_n256(r, r); + sqr_n256(r, r); + sqr_n256(r, r); + if(lo) + mul_n256(r, r, pwr_z_Tbl[lo]); + } +} + +/*===================================================================== + + Specialized single operations in n256 - add, sub & neg + +=====================================================================*/ + +void MB_FUNC_NAME(ifma_add52_n256_)(U64 r[], const U64 a[], const U64 b[]) +{ + MB_FUNC_NAME(ifma_add52x5_)(r, a, b, (U64*)n256x2_mb); +} + +void MB_FUNC_NAME(ifma_sub52_n256_)(U64 r[], const U64 a[], const U64 b[]) +{ + MB_FUNC_NAME(ifma_sub52x5_)(r, a, b, (U64*)n256x2_mb); +} + +void MB_FUNC_NAME(ifma_neg52_n256_)(U64 r[], const U64 a[]) +{ + MB_FUNC_NAME(ifma_neg52x5_)(r, a, (U64*)n256x2_mb); +} + +__mb_mask MB_FUNC_NAME(lt_mbx_digit_)(const U64 a, const U64 b, const __mb_mask lt_mask) +{ + U64 d = mask_sub64(sub64(a, b), lt_mask, sub64(a, b), set1(1)); + return cmp64_mask(d, get_zero64(), _MM_CMPINT_LT); +} + +/* r = (a>=n256)? a-n256 : a */ +void MB_FUNC_NAME(ifma_fastred52_pn256_)(U64 R[], const U64 A[]) +{ + /* r = a - b */ + U64 r0 = sub64(A[0], ((U64*)(n256_mb))[0]); + U64 r1 = sub64(A[1], ((U64*)(n256_mb))[1]); + U64 r2 = sub64(A[2], ((U64*)(n256_mb))[2]); + U64 r3 = sub64(A[3], ((U64*)(n256_mb))[3]); + U64 r4 = sub64(A[4], ((U64*)(n256_mb))[4]); + + /* lt = {r0 - r4} < 0 */ + __mb_mask + lt = MB_FUNC_NAME(lt_mbx_digit_)(r0, get_zero64(), 0); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r1, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r2, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r3, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r4, get_zero64(), lt); + + r0 = mask_mov64(A[0], ~lt, r0); + r1 = mask_mov64(A[1], ~lt, r1); + r2 = mask_mov64(A[2], ~lt, r2); + r3 = mask_mov64(A[3], ~lt, r3); + r4 = mask_mov64(A[4], ~lt, r4); + + /* normalize r0 - r4 */ + NORM_ASHIFTR(r, 0,1) + NORM_ASHIFTR(r, 1,2) + NORM_ASHIFTR(r, 2,3) + NORM_ASHIFTR(r, 3,4) + + R[0] = r0; + R[1] = r1; + R[2] = r2; + R[3] = r3; + R[4] = r4; +} + +__mb_mask MB_FUNC_NAME(ifma_cmp_lt_n256_)(const U64 a[]) +{ + return MB_FUNC_NAME(cmp_lt_FE256_)(a,(const U64 (*))n256_mb); +} + +__mb_mask MB_FUNC_NAME(ifma_check_range_n256_)(const U64 A[]) +{ + __mb_mask + mask = MB_FUNC_NAME(is_zero_FE256_)(A); + mask |= ~MB_FUNC_NAME(ifma_cmp_lt_n256_)(A); + + return mask; +} + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_arith_n384.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_arith_n384.c new file mode 100644 index 0000000..4dfe28c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_arith_n384.c @@ -0,0 +1,620 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +/* Constants */ + +/* +// EC NIST-P384 prime base point order +// in 2^52 radix +*/ +__ALIGN64 static const int64u n384_mb[P384_LEN52][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(0x000c196accc52973) }, + { REP8_DECL(0x000b248b0a77aece) }, + { REP8_DECL(0x0004372ddf581a0d) }, + { REP8_DECL(0x000ffffc7634d81f) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x00000000000fffff) } +}; + +/* 2*n384 */ +__ALIGN64 static const int64u n384_x2[P384_LEN52][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(0x000832d5998a52e6) }, + { REP8_DECL(0x0006491614ef5d9d) }, + { REP8_DECL(0x00086e5bbeb0341b) }, + { REP8_DECL(0x000ffff8ec69b03e) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x00000000001fffff) } +}; + +/* k0 = -( (1/n384 mod 2^DIGIT_SIZE) ) mod 2^DIGIT_SIZE */ +__ALIGN64 static const int64u n384_k0_mb[sizeof(U64)/sizeof(int64u)] = { + REP8_DECL(0x00046089e88fdc45) +}; + +/* to Montgomery conversion constant +// rr = 2^((P384_LEN52*DIGIT_SIZE)*2) mod n384 +*/ +__ALIGN64 static const int64u n384_rr_mb[P384_LEN52][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(0x00034124f50ddb2d) }, + { REP8_DECL(0x000c974971bd0d8d) }, + { REP8_DECL(0x0002118942bfd3cc) }, + { REP8_DECL(0x0009f43be8072178) }, + { REP8_DECL(0x0005bf030606de60) }, + { REP8_DECL(0x0000d49174aab1cc) }, + { REP8_DECL(0x000b7a28266895d4) }, + { REP8_DECL(0x000000000003fb05) } +}; + +/* ifma_tomont52_n384_(1) */ +__ALIGN64 static const int64u n384_r_mb[P384_LEN52][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(0x000ad68d00000000) }, + { REP8_DECL(0x000851313e695333) }, + { REP8_DECL(0x0007e5f24db74f58) }, + { REP8_DECL(0x000b27e0bc8d220a) }, + { REP8_DECL(0x000000000000389c) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) } +}; + +/*===================================================================== + + Specialized single operations over n384: sqr & mul + +=====================================================================*/ +void MB_FUNC_NAME(ifma_amm52_n384_)(U64 r[], const U64 va[], const U64 vb[]) +{ + U64 K = loadu64(n384_k0_mb); + + U64 r0, r1, r2, r3, r4, r5, r6, r7; + int itr; + + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = get_zero64(); + + for(itr=0; itr < P384_LEN52; itr++) { + U64 Yi, T, LO, HI; + U64 Bi = loadu64(vb); + vb++; + + fma52lo_mem(r0, r0, Bi, va, SIMD_BYTES * 0); + Yi = fma52lo(get_zero64(), r0, K); + T = sub64(get_zero64(), Yi); + LO = and64(T, loadu64(VMASK52)); + HI = add64(Yi, srai64(T,63)); + fma52lo_mem(r1, r1, Bi, va, SIMD_BYTES * 1); + fma52lo_mem(r2, r2, Bi, va, SIMD_BYTES * 2); + fma52lo_mem(r3, r3, Bi, va, SIMD_BYTES * 3); + fma52lo_mem(r4, r4, Bi, va, SIMD_BYTES * 4); + fma52lo_mem(r5, r5, Bi, va, SIMD_BYTES * 5); + fma52lo_mem(r6, r6, Bi, va, SIMD_BYTES * 6); + fma52lo_mem(r7, r7, Bi, va, SIMD_BYTES * 7); + + fma52lo_mem(r0, r0, Yi, n384_mb, SIMD_BYTES * 0); + fma52lo_mem(r1, r1, Yi, n384_mb, SIMD_BYTES * 1); + fma52lo_mem(r2, r2, Yi, n384_mb, SIMD_BYTES * 2); + fma52lo_mem(r3, r3, Yi, n384_mb, SIMD_BYTES * 3); + r1 = add64(r1, srli64(r0, DIGIT_SIZE)); /* carry propagation */ + r4 = add64(r4, LO); /* fma52lo_mem(r4, r4, Yi, n384_mb, SIMD_BYTES * 4); */ + r5 = add64(r5, LO); /* fma52lo_mem(r5, r5, Yi, n384_mb, SIMD_BYTES * 5); */ + r6 = add64(r6, LO); /* fma52lo_mem(r6, r6, Yi, n384_mb, SIMD_BYTES * 6); */ + fma52lo_mem(r7, r7, Yi, n384_mb, SIMD_BYTES * 7); + + fma52hi_mem(r0, r1, Bi, va, SIMD_BYTES * 0); + fma52hi_mem(r1, r2, Bi, va, SIMD_BYTES * 1); + fma52hi_mem(r2, r3, Bi, va, SIMD_BYTES * 2); + fma52hi_mem(r3, r4, Bi, va, SIMD_BYTES * 3); + fma52hi_mem(r4, r5, Bi, va, SIMD_BYTES * 4); + fma52hi_mem(r5, r6, Bi, va, SIMD_BYTES * 5); + fma52hi_mem(r6, r7, Bi, va, SIMD_BYTES * 6); + fma52hi_mem(r7, get_zero64(), Bi, va, SIMD_BYTES * 7); + + fma52hi_mem(r0, r0, Yi, n384_mb, SIMD_BYTES * 0); + fma52hi_mem(r1, r1, Yi, n384_mb, SIMD_BYTES * 1); + fma52hi_mem(r2, r2, Yi, n384_mb, SIMD_BYTES * 2); + fma52hi_mem(r3, r3, Yi, n384_mb, SIMD_BYTES * 3); + r4 = add64(r4, HI); /* fma52hi_mem(r4, r4, Yi, n384_mb, SIMD_BYTES * 4); */ + r5 = add64(r5, HI); /* fma52hi_mem(r5, r5, Yi, n384_mb, SIMD_BYTES * 5); */ + r6 = add64(r6, HI); /* fma52hi_mem(r6, r6, Yi, n384_mb, SIMD_BYTES * 6); */ + fma52hi_mem(r7, r7, Yi, n384_mb, SIMD_BYTES * 7); + } + + // normalization + NORM_LSHIFTR(r, 0, 1) + NORM_LSHIFTR(r, 1, 2) + NORM_LSHIFTR(r, 2, 3) + NORM_LSHIFTR(r, 3, 4) + NORM_LSHIFTR(r, 4, 5) + NORM_LSHIFTR(r, 5, 6) + NORM_LSHIFTR(r, 6, 7) + + r[0] = r0; + r[1] = r1; + r[2] = r2; + r[3] = r3; + r[4] = r4; + r[5] = r5; + r[6] = r6; + r[7] = r7; +} + +#define ROUND_MUL(I, J, R_LO, R_HI) \ + R_LO = fma52lo(R_LO, va[I], vb[J]); \ + R_HI = fma52hi(R_HI, va[I], vb[J]); + +#define N384_REDUCTION_ROUND(u, r0, r1, r2, r3, r4, r5, r6, r7, r8, modulus, k) \ +{ \ + U64 t = sub64(get_zero64(), u); \ + U64 lo = and64(t, loadu64(VMASK52)); \ + U64 hi = add64(u, srai64(t,63)); \ + \ + r0 = fma52lo(r0, u, modulus[0]); \ + r1 = fma52lo(r1, u, modulus[1]); \ + r2 = fma52lo(r2, u, modulus[2]); \ + r3 = fma52lo(r3, u, modulus[3]); \ + r7 = fma52lo(r7, u, modulus[7]); \ + r1 = add64(r1, srli64(r0, DIGIT_SIZE)); /* carry propagation */ \ + r4 = add64(r4, lo); /*r4 = fma52lo(r4, u, modulus[4]); */ \ + r5 = add64(r5, lo); /*r5 = fma52lo(r5, u, modulus[5]); */ \ + r6 = add64(r6, lo); /*r6 = fma52lo(r6, u, modulus[6]); */ \ + \ + r1 = fma52hi(r1, u, modulus[0]); \ + r2 = fma52hi(r2, u, modulus[1]); \ + r3 = fma52hi(r3, u, modulus[2]); \ + r4 = fma52hi(r4, u, modulus[3]); \ + r8 = fma52hi(r8, u, modulus[7]); \ + u = fma52lo(get_zero64(), r1, k); /* update u = r1*k */ \ + r5 = add64(r5, hi); /*r5 = fma52hi(r5, u, modulus[4]); */ \ + r6 = add64(r6, hi); /*r6 = fma52hi(r6, u, modulus[5]); */ \ + r7 = add64(r7, hi); /*r7 = fma52hi(r7, u, modulus[6]); */ \ +} + +void MB_FUNC_NAME(ifma_ams52_n384_)(U64 r[], const U64 va[]) +{ + U64 K = loadu64(n384_k0_mb); + + const U64* vb = va; + U64* modulus = (U64*)n384_mb; + + U64 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15; + U64 u; + + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = + r10 = r11 = r12 = r13 = r14 = r15 = get_zero64(); + + // full square + ROUND_MUL(0, 1, r1, r2) + ROUND_MUL(0, 2, r2, r3) + ROUND_MUL(0, 3, r3, r4) + ROUND_MUL(0, 4, r4, r5) + ROUND_MUL(0, 5, r5, r6) + ROUND_MUL(0, 6, r6, r7) + ROUND_MUL(0, 7, r7, r8) + + ROUND_MUL(1, 2, r3, r4) + ROUND_MUL(1, 3, r4, r5) + ROUND_MUL(1, 4, r5, r6) + ROUND_MUL(1, 5, r6, r7) + ROUND_MUL(1, 6, r7, r8) + ROUND_MUL(1, 7, r8, r9) + + ROUND_MUL(2, 3, r5, r6) + ROUND_MUL(2, 4, r6, r7) + ROUND_MUL(2, 5, r7, r8) + ROUND_MUL(2, 6, r8, r9) + ROUND_MUL(2, 7, r9, r10) + + ROUND_MUL(3, 4, r7, r8) + ROUND_MUL(3, 5, r8, r9) + ROUND_MUL(3, 6, r9, r10) + ROUND_MUL(3, 7, r10,r11) + + ROUND_MUL(4, 5, r9, r10) + ROUND_MUL(4, 6, r10,r11) + ROUND_MUL(4, 7, r11,r12) + + ROUND_MUL(5, 6, r11,r12) + ROUND_MUL(5, 7, r12,r13) + + ROUND_MUL(6, 7, r13,r14) + + r1 = add64(r1, r1); + r2 = add64(r2, r2); + r3 = add64(r3, r3); + r4 = add64(r4, r4); + r5 = add64(r5, r5); + r6 = add64(r6, r6); + r7 = add64(r7, r7); + r8 = add64(r8, r8); + r9 = add64(r9, r9); + r10 = add64(r10, r10); + r11 = add64(r11, r11); + r12 = add64(r12, r12); + r13 = add64(r13, r13); + r14 = add64(r14, r14); + + ROUND_MUL(0, 0, r0, r1) + u = fma52lo(get_zero64(), r0, K); + ROUND_MUL(1, 1, r2, r3) + ROUND_MUL(2, 2, r4, r5) + ROUND_MUL(3, 3, r6, r7) + ROUND_MUL(4, 4, r8, r9) + ROUND_MUL(5, 5, r10, r11) + ROUND_MUL(6, 6, r12, r13) + ROUND_MUL(7, 7, r14, r15) + + // reduction + N384_REDUCTION_ROUND(u, r0, r1, r2, r3, r4, r5, r6, r7, r8, modulus, K); + N384_REDUCTION_ROUND(u, r1, r2, r3, r4, r5, r6, r7, r8, r9, modulus, K); + N384_REDUCTION_ROUND(u, r2, r3, r4, r5, r6, r7, r8, r9, r10, modulus, K); + N384_REDUCTION_ROUND(u, r3, r4, r5, r6, r7, r8, r9, r10,r11, modulus, K); + N384_REDUCTION_ROUND(u, r4, r5, r6, r7, r8, r9, r10,r11,r12, modulus, K); + N384_REDUCTION_ROUND(u, r5, r6, r7, r8, r9, r10,r11,r12,r13, modulus, K); + N384_REDUCTION_ROUND(u, r6, r7, r8, r9, r10,r11,r12,r13,r14, modulus, K); + N384_REDUCTION_ROUND(u, r7, r8, r9, r10,r11,r12,r13,r14,r15, modulus, K); + + // normalization + NORM_LSHIFTR(r, 8, 9) + NORM_LSHIFTR(r, 9, 10) + NORM_LSHIFTR(r, 10, 11) + NORM_LSHIFTR(r, 11, 12) + NORM_LSHIFTR(r, 12, 13) + NORM_LSHIFTR(r, 13, 14) + NORM_LSHIFTR(r, 14, 15) + + r[0] = r8; + r[1] = r9; + r[2] = r10; + r[3] = r11; + r[4] = r12; + r[5] = r13; + r[6] = r14; + r[7] = r15; +} + +void MB_FUNC_NAME(ifma_tomont52_n384_)(U64 r[], const U64 a[]) +{ + MB_FUNC_NAME(ifma_amm52_n384_)(r, a, (U64*)n384_rr_mb); +} + +void MB_FUNC_NAME(ifma_frommont52_n384_)(U64 r[], const U64 a[]) +{ + MB_FUNC_NAME(ifma_amm52_n384_)(r, a, (U64*)ones); +} + +/* +// computes r = 1/z = z^(n384-2) mod n384 +// +// note: z in in Montgomery domain (as soon mul() and sqr() below are amm-functions +// r in Montgomery domain too +*/ +#define fe52_sqr MB_FUNC_NAME(ifma_ams52_n384_) +#define fe52_mul MB_FUNC_NAME(ifma_amm52_n384_) + +/* r = base^(2^n) */ +__INLINE void fe52_sqr_pwr(U64 r[], const U64 base[], int n) +{ + if(r!=base) { + fe52_sqr(r,base); + n--; + } + for(; n>0; n--) + fe52_sqr(r,r); +} + +void MB_FUNC_NAME(ifma_aminv52_n384_)(U64 r[], const U64 z[]) +{ + int i; + + // pwr_z_Tbl[i][] = z^i, i=0,..,15 + __ALIGN64 U64 pwr_z_Tbl[16][P384_LEN52]; + __ALIGN64 U64 lexp[P384_LEN52]; + + MB_FUNC_NAME(mov_FE384_)(pwr_z_Tbl[0], (U64*)n384_r_mb); + MB_FUNC_NAME(mov_FE384_)(pwr_z_Tbl[1], z); + + for(i=2; i<16; i+=2) { + fe52_sqr(pwr_z_Tbl[i], pwr_z_Tbl[i/2]); + fe52_mul(pwr_z_Tbl[i+1], pwr_z_Tbl[i], z); + } + + // pwr = (n384-2) in big endian + int8u pwr[] = "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" + "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" + "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" + "\xC7\x63\x4D\x81\xF4\x37\x2D\xDF" + "\x58\x1A\x0D\xB2\x48\xB0\xA7\x7A" + "\xEC\xEC\x19\x6A\xCC\xC5\x29\x71"; + + /* + // process low part of the exponent: "0xc7634d81f4372ddf 0x581a0db248b0a77a 0xecec196accc52973" + */ + /* init result */ + MB_FUNC_NAME(mov_FE384_)(lexp, (U64*)n384_r_mb); + + for(i=24; i>4) &0xF; + int lo = v & 0xF; + + fe52_sqr(lexp, lexp); + fe52_sqr(lexp, lexp); + fe52_sqr(lexp, lexp); + fe52_sqr(lexp, lexp); + if(hi) + fe52_mul(lexp, lexp, pwr_z_Tbl[hi]); + fe52_sqr(lexp, lexp); + fe52_sqr(lexp, lexp); + fe52_sqr(lexp, lexp); + fe52_sqr(lexp, lexp); + if(lo) + fe52_mul(lexp, lexp, pwr_z_Tbl[lo]); + } + + /* + // process high part of the exponent: "0xffffffffffffffff 0xffffffffffffffff 0xffffffffffffffff" + */ + __ALIGN64 U64 u[P384_LEN52]; + __ALIGN64 U64 v[P384_LEN52]; + + fe52_sqr(v, z); /* v = z^2 */ + fe52_mul(u, v, z); /* u = z^2 * z = z^3 */ + + fe52_sqr_pwr(v, u, 2); /* v= (z^3)^(2^2) = z^12 */ + fe52_mul(u, v, u); /* u = z^12 * z^3 = z^15 = z^(0xF) */ + + fe52_sqr_pwr(v, u, 4); /* v= (z^0xF)^(2^4) = z^(0xF0) */ + fe52_mul(u, v, u); /* u = z^0xF0 * z^(0xF) = z^(0xFF) */ + + fe52_sqr_pwr(v, u, 8); /* v= (z^0xFF)^(2^8) = z^(0xFF00) */ + fe52_mul(u, v, u); /* u = z^0xFF00 * z^(0xFF) = z^(0xFFFF) */ + + fe52_sqr_pwr(v, u, 16); /* v= (z^0xFFFF)^(2^16) = z^(0xFFFF0000) */ + fe52_mul(u, v, u); /* u = z^0xFFFF0000 * z^(0xFFFF) = z^(0xFFFFFFFF) */ + + fe52_sqr_pwr(v, u, 32); /* v= (z^0xFFFFFFFF)^(2^32) = z^(0xFFFFFFFF00000000) */ + fe52_mul(u, v, u); /* u = z^0xFFFFFFFF00000000 * z^(0xFFFFFFFF) = z^(0xFFFFFFFFFFFFFFFF) */ + + fe52_sqr_pwr(v, u, 64); + fe52_mul(v, v, u); /* v = z^(0xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF) */ + + fe52_sqr_pwr(v, v, 64); + fe52_mul(v, v, u); /* v = z^(0xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF) */ + + /* combine low and high results */ + fe52_sqr_pwr(v, v, 64*3); /* u = z^(0xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.0000000000000000.0000000000000000.0000000000000000) */ + fe52_mul(r, v, lexp); /* r = z^(0xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.c7634d81f4372ddf.581a0db248b0a77a.ecec196accc52973) */ +} + +/*===================================================================== + + Specialized single operations over n384 add, sub, neg + +=====================================================================*/ +static __mb_mask MB_FUNC_NAME(lt_mbx_digit_)(const U64 a, const U64 b, const __mb_mask lt_mask) +{ + U64 d = mask_sub64(sub64(a, b), lt_mask, sub64(a, b), set1(1)); + return cmp64_mask(d, get_zero64(), _MM_CMPINT_LT); +} + +void MB_FUNC_NAME(ifma_add52_n384_)(U64 R[], const U64 A[], const U64 B[]) +{ + /* r = a + b */ + U64 r0 = add64(A[0], B[0]); + U64 r1 = add64(A[1], B[1]); + U64 r2 = add64(A[2], B[2]); + U64 r3 = add64(A[3], B[3]); + U64 r4 = add64(A[4], B[4]); + U64 r5 = add64(A[5], B[5]); + U64 r6 = add64(A[6], B[6]); + U64 r7 = add64(A[7], B[7]); + + /* lt = {r0 - r7} < 2*n */ + __mb_mask + lt = MB_FUNC_NAME(lt_mbx_digit_)( r0, ((U64*)(n384_x2))[0], 0); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r1, ((U64*)(n384_x2))[1], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r2, ((U64*)(n384_x2))[2], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r3, ((U64*)(n384_x2))[3], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r4, ((U64*)(n384_x2))[4], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r5, ((U64*)(n384_x2))[5], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r6, ((U64*)(n384_x2))[6], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r7, ((U64*)(n384_x2))[7], lt); + + /* {r0 - r7} -= 2*n */ + r0 = mask_sub64(r0, ~lt, r0, ((U64*)(n384_x2))[0]); + r1 = mask_sub64(r1, ~lt, r1, ((U64*)(n384_x2))[1]); + r2 = mask_sub64(r2, ~lt, r2, ((U64*)(n384_x2))[2]); + r3 = mask_sub64(r3, ~lt, r3, ((U64*)(n384_x2))[3]); + r4 = mask_sub64(r4, ~lt, r4, ((U64*)(n384_x2))[4]); + r5 = mask_sub64(r5, ~lt, r5, ((U64*)(n384_x2))[5]); + r6 = mask_sub64(r6, ~lt, r6, ((U64*)(n384_x2))[6]); + r7 = mask_sub64(r7, ~lt, r7, ((U64*)(n384_x2))[7]); + + /* normalize r0 - r7 */ + NORM_ASHIFTR(r, 0,1) + NORM_ASHIFTR(r, 1,2) + NORM_ASHIFTR(r, 2,3) + NORM_ASHIFTR(r, 3,4) + NORM_ASHIFTR(r, 4,5) + NORM_ASHIFTR(r, 5,6) + NORM_ASHIFTR(r, 6,7) + + R[0] = r0; + R[1] = r1; + R[2] = r2; + R[3] = r3; + R[4] = r4; + R[5] = r5; + R[6] = r6; + R[7] = r7; +} + +void MB_FUNC_NAME(ifma_sub52_n384_)(U64 R[], const U64 A[], const U64 B[]) +{ + /* r = a - b */ + U64 r0 = sub64(A[0], B[0]); + U64 r1 = sub64(A[1], B[1]); + U64 r2 = sub64(A[2], B[2]); + U64 r3 = sub64(A[3], B[3]); + U64 r4 = sub64(A[4], B[4]); + U64 r5 = sub64(A[5], B[5]); + U64 r6 = sub64(A[6], B[6]); + U64 r7 = sub64(A[7], B[7]); + + /* lt = {r0 - r7} < 0 */ + __mb_mask + lt = MB_FUNC_NAME(lt_mbx_digit_)(r0, get_zero64(), 0); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r1, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r2, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r3, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r4, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r5, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r6, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r7, get_zero64(), lt); + + r0 = mask_add64(r0, lt, r0, ((U64*)(n384_x2))[0]); + r1 = mask_add64(r1, lt, r1, ((U64*)(n384_x2))[1]); + r2 = mask_add64(r2, lt, r2, ((U64*)(n384_x2))[2]); + r3 = mask_add64(r3, lt, r3, ((U64*)(n384_x2))[3]); + r4 = mask_add64(r4, lt, r4, ((U64*)(n384_x2))[4]); + r5 = mask_add64(r5, lt, r5, ((U64*)(n384_x2))[5]); + r6 = mask_add64(r6, lt, r6, ((U64*)(n384_x2))[6]); + r7 = mask_add64(r7, lt, r7, ((U64*)(n384_x2))[7]); + + /* normalize r0 - r7 */ + NORM_ASHIFTR(r, 0,1) + NORM_ASHIFTR(r, 1,2) + NORM_ASHIFTR(r, 2,3) + NORM_ASHIFTR(r, 3,4) + NORM_ASHIFTR(r, 4,5) + NORM_ASHIFTR(r, 5,6) + NORM_ASHIFTR(r, 6,7) + + R[0] = r0; + R[1] = r1; + R[2] = r2; + R[3] = r3; + R[4] = r4; + R[5] = r5; + R[6] = r6; + R[7] = r7; +} + +void MB_FUNC_NAME(ifma_neg52_n384_)(U64 R[], const U64 A[]) +{ + __mb_mask nz_mask = ~MB_FUNC_NAME(is_zero_FE384_)(A); + + /* {r0 - r7} = 2*n - A */ + U64 r0 = mask_sub64( A[0], nz_mask, ((U64*)(n384_x2))[0], A[0] ); + U64 r1 = mask_sub64( A[0], nz_mask, ((U64*)(n384_x2))[1], A[1] ); + U64 r2 = mask_sub64( A[0], nz_mask, ((U64*)(n384_x2))[2], A[2] ); + U64 r3 = mask_sub64( A[0], nz_mask, ((U64*)(n384_x2))[3], A[3] ); + U64 r4 = mask_sub64( A[0], nz_mask, ((U64*)(n384_x2))[4], A[4] ); + U64 r5 = mask_sub64( A[0], nz_mask, ((U64*)(n384_x2))[5], A[5] ); + U64 r6 = mask_sub64( A[0], nz_mask, ((U64*)(n384_x2))[6], A[6] ); + U64 r7 = mask_sub64( A[0], nz_mask, ((U64*)(n384_x2))[7], A[7] ); + + /* normalize r0 - r7 */ + NORM_ASHIFTR(r, 0,1) + NORM_ASHIFTR(r, 1,2) + NORM_ASHIFTR(r, 2,3) + NORM_ASHIFTR(r, 3,4) + NORM_ASHIFTR(r, 4,5) + NORM_ASHIFTR(r, 5,6) + NORM_ASHIFTR(r, 6,7) + + R[0] = r0; + R[1] = r1; + R[2] = r2; + R[3] = r3; + R[4] = r4; + R[5] = r5; + R[6] = r6; + R[7] = r7; +} + + +/* r = (a>=n384)? a-n384 : a */ +void MB_FUNC_NAME(ifma_fastred52_pn384_)(U64 R[], const U64 A[]) +{ + /* r = a - b */ + U64 r0 = sub64(A[0], ((U64*)(n384_mb))[0]); + U64 r1 = sub64(A[1], ((U64*)(n384_mb))[1]); + U64 r2 = sub64(A[2], ((U64*)(n384_mb))[2]); + U64 r3 = sub64(A[3], ((U64*)(n384_mb))[3]); + U64 r4 = sub64(A[4], ((U64*)(n384_mb))[4]); + U64 r5 = sub64(A[5], ((U64*)(n384_mb))[5]); + U64 r6 = sub64(A[6], ((U64*)(n384_mb))[6]); + U64 r7 = sub64(A[7], ((U64*)(n384_mb))[7]); + + /* lt = {r0 - r7} < 0 */ + __mb_mask + lt = MB_FUNC_NAME(lt_mbx_digit_)(r0, get_zero64(), 0); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r1, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r2, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r3, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r4, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r5, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r6, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r7, get_zero64(), lt); + + r0 = mask_mov64(A[0], ~lt, r0); + r1 = mask_mov64(A[1], ~lt, r1); + r2 = mask_mov64(A[2], ~lt, r2); + r3 = mask_mov64(A[3], ~lt, r3); + r4 = mask_mov64(A[4], ~lt, r4); + r5 = mask_mov64(A[5], ~lt, r5); + r6 = mask_mov64(A[6], ~lt, r6); + r7 = mask_mov64(A[7], ~lt, r7); + + /* normalize r0 - r7 */ + NORM_ASHIFTR(r, 0,1) + NORM_ASHIFTR(r, 1,2) + NORM_ASHIFTR(r, 2,3) + NORM_ASHIFTR(r, 3,4) + NORM_ASHIFTR(r, 4,5) + NORM_ASHIFTR(r, 5,6) + NORM_ASHIFTR(r, 6,7) + + R[0] = r0; + R[1] = r1; + R[2] = r2; + R[3] = r3; + R[4] = r4; + R[5] = r5; + R[6] = r6; + R[7] = r7; +} + +__mb_mask MB_FUNC_NAME(ifma_cmp_lt_n384_)(const U64 a[]) +{ + return MB_FUNC_NAME(cmp_lt_FE384_)(a,(const U64 (*))n384_mb); +} + +__mb_mask MB_FUNC_NAME(ifma_check_range_n384_)(const U64 A[]) +{ + __mb_mask + mask = MB_FUNC_NAME(is_zero_FE384_)(A); + mask |= ~MB_FUNC_NAME(ifma_cmp_lt_n384_)(A); + + return mask; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_arith_n521.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_arith_n521.c new file mode 100644 index 0000000..90f0637 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_arith_n521.c @@ -0,0 +1,777 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +/* Constants */ + +/* +// EC NIST-P521 prime base point order +// in 2^52 radix +*/ +__ALIGN64 static const int64u n521_mb[P521_LEN52][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(0x000fb71e91386409) }, + { REP8_DECL(0x000b8899c47aebb6) }, + { REP8_DECL(0x000709a5d03bb5c9) }, + { REP8_DECL(0x000966b7fcc0148f) }, + { REP8_DECL(0x000a51868783bf2f) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x0000000000000001) } +}; + +/* 2*n521 */ +__ALIGN64 static const int64u n521_x2[P521_LEN52][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(0x000f6e3d2270c812) }, + { REP8_DECL(0x0007113388f5d76d) }, + { REP8_DECL(0x000e134ba0776b93) }, + { REP8_DECL(0x0002cd6ff980291e) }, + { REP8_DECL(0x0004a30d0f077e5f) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x0000000000000003) } +}; +/* k0 = -( (1/n521 mod 2^DIGIT_SIZE) ) mod 2^DIGIT_SIZE */ +__ALIGN64 static const int64u n521_k0_mb[sizeof(U64)/sizeof(int64u)] = { + REP8_DECL(0x000f5ccd79a995c7) +}; + +/* to Montgomery conversion constant +// rr = 2^((P521_LEN52*DIGIT_SIZE)*2) mod n521 +*/ +__ALIGN64 static const int64u n521_rr_mb[P521_LEN52][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(0x0003b4d7a5b140ce) }, + { REP8_DECL(0x000cb0bf26c55bf9) }, + { REP8_DECL(0x00037e5396c67ee9) }, + { REP8_DECL(0x0002bd1c80cf7b13) }, + { REP8_DECL(0x00073cbe28f15e41) }, + { REP8_DECL(0x000dd6e23d82e49c) }, + { REP8_DECL(0x0003d142b7756e3e) }, + { REP8_DECL(0x00061a8e567bccff) }, + { REP8_DECL(0x00092d0d455bcc6d) }, + { REP8_DECL(0x000383d2d8e03d14) }, + { REP8_DECL(0x0000000000000000) } +}; + +/* ifma_tomont52_n521_(1) */ +__ALIGN64 static const int64u n521_r_mb[P521_LEN52][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(0x0008000000000000) }, + { REP8_DECL(0x00082470b763cdfb) }, + { REP8_DECL(0x00023bb31dc28a24) }, + { REP8_DECL(0x00047b2d17e2251b) }, + { REP8_DECL(0x00034ca4019ff5b8) }, + { REP8_DECL(0x0002d73cbc3e2068) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) } +}; + + +/*===================================================================== + + Specialized single operations over n384: sqr & mul + +=====================================================================*/ + +void MB_FUNC_NAME(ifma_amm52_n521_)(U64 r[], const U64 va[], const U64 vb[]) +{ + U64 K = loadu64(n521_k0_mb); + + U64 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10; + int itr; + + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = r10 = get_zero64(); + + for(itr=0; itr < P521_LEN52; itr++) { + U64 Yi, T, LO, HI; + U64 Bi = loadu64(vb); + vb++; + + fma52lo_mem(r0, r0, Bi, va, SIMD_BYTES * 0); + fma52lo_mem(r1, r1, Bi, va, SIMD_BYTES * 1); + fma52lo_mem(r2, r2, Bi, va, SIMD_BYTES * 2); + fma52lo_mem(r3, r3, Bi, va, SIMD_BYTES * 3); + fma52lo_mem(r4, r4, Bi, va, SIMD_BYTES * 4); + fma52lo_mem(r5, r5, Bi, va, SIMD_BYTES * 5); + fma52lo_mem(r6, r6, Bi, va, SIMD_BYTES * 6); + fma52lo_mem(r7, r7, Bi, va, SIMD_BYTES * 7); + fma52lo_mem(r8, r8, Bi, va, SIMD_BYTES * 8); + fma52lo_mem(r9, r9, Bi, va, SIMD_BYTES * 9); + fma52lo_mem(r10,r10,Bi, va, SIMD_BYTES * 10); + + Yi = fma52lo(get_zero64(), r0, K); + T = sub64(get_zero64(), Yi); + LO = and64(T, loadu64(VMASK52)); + HI = add64(Yi, srai64(T,63)); + + fma52lo_mem(r0, r0, Yi, n521_mb, SIMD_BYTES * 0); + fma52lo_mem(r1, r1, Yi, n521_mb, SIMD_BYTES * 1); + fma52lo_mem(r2, r2, Yi, n521_mb, SIMD_BYTES * 2); + fma52lo_mem(r3, r3, Yi, n521_mb, SIMD_BYTES * 3); + fma52lo_mem(r4, r4, Yi, n521_mb, SIMD_BYTES * 4); + r1 = add64(r1, srli64(r0, DIGIT_SIZE)); /* carry propagation */ + r5 = add64(r5, LO); /* fma52lo_mem(r5, r5, Yi, n521_mb, SIMD_BYTES * 5); */ + r6 = add64(r6, LO); /* fma52lo_mem(r6, r6, Yi, n521_mb, SIMD_BYTES * 6); */ + r7 = add64(r7, LO); /* fma52lo_mem(r7, r7, Yi, n521_mb, SIMD_BYTES * 7); */ + r8 = add64(r8, LO); /* fma52lo_mem(r8, r8, Yi, n521_mb, SIMD_BYTES * 8); */ + r9 = add64(r9, LO); /* fma52lo_mem(r9, r9, Yi, n521_mb, SIMD_BYTES * 9); */ + r10= add64(r10,Yi); /* fma52lo_mem(r10,r10,Yi, n521_mb, SIMD_BYTES * 10); */ + + fma52hi_mem(r0, r1, Bi, va, SIMD_BYTES * 0); + fma52hi_mem(r1, r2, Bi, va, SIMD_BYTES * 1); + fma52hi_mem(r2, r3, Bi, va, SIMD_BYTES * 2); + fma52hi_mem(r3, r4, Bi, va, SIMD_BYTES * 3); + fma52hi_mem(r4, r5, Bi, va, SIMD_BYTES * 4); + fma52hi_mem(r5, r6, Bi, va, SIMD_BYTES * 5); + fma52hi_mem(r6, r7, Bi, va, SIMD_BYTES * 6); + fma52hi_mem(r7, r8, Bi, va, SIMD_BYTES * 7); + fma52hi_mem(r8, r9, Bi, va, SIMD_BYTES * 8); + fma52hi_mem(r9, r10,Bi, va, SIMD_BYTES * 9); + fma52hi_mem(r10,get_zero64(), Bi, va, SIMD_BYTES * 10); + + fma52hi_mem(r0, r0, Yi, n521_mb, SIMD_BYTES * 0); + fma52hi_mem(r1, r1, Yi, n521_mb, SIMD_BYTES * 1); + fma52hi_mem(r2, r2, Yi, n521_mb, SIMD_BYTES * 2); + fma52hi_mem(r3, r3, Yi, n521_mb, SIMD_BYTES * 3); + fma52hi_mem(r4, r4, Yi, n521_mb, SIMD_BYTES * 4); + r5 = add64(r5, HI); /* fma52hi_mem(r5, r5, Yi, n521_mb, SIMD_BYTES * 5); */ + r6 = add64(r6, HI); /* fma52hi_mem(r6, r6, Yi, n521_mb, SIMD_BYTES * 6); */ + r7 = add64(r7, HI); /* fma52hi_mem(r7, r7, Yi, n521_mb, SIMD_BYTES * 7); */ + r8 = add64(r8, HI); /* fma52hi_mem(r8, r8, Yi, n521_mb, SIMD_BYTES * 8); */ + r9 = add64(r9, HI); /* fma52hi_mem(r9, r9, Yi, n521_mb, SIMD_BYTES * 9); */ + /* 0 == hi(Yi, n521_mb[10] */; + } + + // normalization + NORM_LSHIFTR(r, 0, 1) + NORM_LSHIFTR(r, 1, 2) + NORM_LSHIFTR(r, 2, 3) + NORM_LSHIFTR(r, 3, 4) + NORM_LSHIFTR(r, 4, 5) + NORM_LSHIFTR(r, 5, 6) + NORM_LSHIFTR(r, 6, 7) + NORM_LSHIFTR(r, 7, 8) + NORM_LSHIFTR(r, 8, 9) + NORM_LSHIFTR(r, 9, 10) + + r[0] = r0; + r[1] = r1; + r[2] = r2; + r[3] = r3; + r[4] = r4; + r[5] = r5; + r[6] = r6; + r[7] = r7; + r[8] = r8; + r[9] = r9; + r[10]= r10; +} + +#define ROUND_MUL(I, J, R_LO, R_HI) \ + R_LO = fma52lo(R_LO, va[I], vb[J]); \ + R_HI = fma52hi(R_HI, va[I], vb[J]); + +#define N521_REDUCTION_ROUND(u, r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, modulus, k) \ +{ \ + U64 t = sub64(get_zero64(), u); \ + U64 lo = and64(t, loadu64(VMASK52)); \ + U64 hi = add64(u, srai64(t,63)); \ + \ + r0 = fma52lo(r0, u, modulus[0]); \ + r1 = fma52lo(r1, u, modulus[1]); \ + r2 = fma52lo(r2, u, modulus[2]); \ + r3 = fma52lo(r3, u, modulus[3]); \ + r4 = fma52lo(r4, u, modulus[4]); \ + r1 = add64(r1, srli64(r0, DIGIT_SIZE)); /* carry propagation */ \ + r5 = add64(r5, lo); \ + r6 = add64(r6, lo); \ + r7 = add64(r7, lo); \ + r8 = add64(r8, lo); \ + r9 = add64(r9, lo); \ + r10 = add64(r10, u); \ + \ + \ + r1 = fma52hi(r1, u, modulus[0]); \ + r2 = fma52hi(r2, u, modulus[1]); \ + r3 = fma52hi(r3, u, modulus[2]); \ + r4 = fma52hi(r4, u, modulus[3]); \ + r5 = fma52hi(r5, u, modulus[4]); \ + u = fma52lo(get_zero64(), r1, k); /* update u = r1*k */ \ + r6 = add64(r6, hi); \ + r7 = add64(r7, hi); \ + r8 = add64(r8, hi); \ + r9 = add64(r9, hi); \ + r10= add64(r10,hi); \ +} + +void MB_FUNC_NAME(ifma_ams52_n521_)(U64 r[], const U64 va[]) +{ + U64 K = loadu64(n521_k0_mb); + + const U64* vb = va; + U64* modulus = (U64*)n521_mb; + + U64 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9; + U64 r10, r11, r12, r13, r14, r15, r16, r17, r18, r19; + U64 r20, r21; + U64 u; + + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = r10 = + r11 = r12 = r13 = r14 = r15 = r16 = r17 = r18 = r19 = r20 = r21 = get_zero64(); + + // full square + ROUND_MUL(0, 1, r1, r2); + ROUND_MUL(0, 2, r2, r3); + ROUND_MUL(0, 3, r3, r4); + ROUND_MUL(0, 4, r4, r5); + ROUND_MUL(0, 5, r5, r6); + ROUND_MUL(0, 6, r6, r7); + ROUND_MUL(0, 7, r7, r8); + ROUND_MUL(0, 8, r8, r9); + ROUND_MUL(0, 9, r9, r10); + ROUND_MUL(0, 10,r10,r11); + + ROUND_MUL(1, 2, r3, r4); + ROUND_MUL(1, 3, r4, r5); + ROUND_MUL(1, 4, r5, r6); + ROUND_MUL(1, 5, r6, r7); + ROUND_MUL(1, 6, r7, r8); + ROUND_MUL(1, 7, r8, r9); + ROUND_MUL(1, 8, r9, r10); + ROUND_MUL(1, 9, r10,r11); + ROUND_MUL(1, 10,r11,r12); + + ROUND_MUL(2, 3, r5, r6); + ROUND_MUL(2, 4, r6, r7); + ROUND_MUL(2, 5, r7, r8); + ROUND_MUL(2, 6, r8, r9); + ROUND_MUL(2, 7, r9, r10); + ROUND_MUL(2, 8, r10,r11); + ROUND_MUL(2, 9, r11,r12); + ROUND_MUL(2, 10,r12,r13); + + ROUND_MUL(3, 4, r7, r8); + ROUND_MUL(3, 5, r8, r9); + ROUND_MUL(3, 6, r9, r10); + ROUND_MUL(3, 7, r10,r11); + ROUND_MUL(3, 8, r11,r12); + ROUND_MUL(3, 9, r12,r13); + ROUND_MUL(3, 10,r13,r14); + + ROUND_MUL(4, 5, r9, r10); + ROUND_MUL(4, 6, r10,r11); + ROUND_MUL(4, 7, r11,r12); + ROUND_MUL(4, 8, r12,r13); + ROUND_MUL(4, 9, r13,r14); + ROUND_MUL(4, 10,r14,r15); + + ROUND_MUL(5, 6, r11,r12); + ROUND_MUL(5, 7, r12,r13); + ROUND_MUL(5, 8, r13,r14); + ROUND_MUL(5, 9, r14,r15); + ROUND_MUL(5, 10,r15,r16); + + ROUND_MUL(6, 7, r13,r14); + ROUND_MUL(6, 8, r14,r15); + ROUND_MUL(6, 9, r15,r16); + ROUND_MUL(6, 10,r16,r17); + + ROUND_MUL(7, 8, r15,r16); + ROUND_MUL(7, 9, r16,r17); + ROUND_MUL(7, 10,r17,r18); + + ROUND_MUL(8, 9, r17,r18); + ROUND_MUL(8, 10,r18,r19); + + ROUND_MUL(9, 10,r19,r20); + + r1 = add64(r1, r1); + r2 = add64(r2, r2); + r3 = add64(r3, r3); + r4 = add64(r4, r4); + r5 = add64(r5, r5); + r6 = add64(r6, r6); + r7 = add64(r7, r7); + r8 = add64(r8, r8); + r9 = add64(r9, r9); + r10 = add64(r10, r10); + r11 = add64(r11, r11); + r12 = add64(r12, r12); + r13 = add64(r13, r13); + r14 = add64(r14, r14); + r15 = add64(r15, r15); + r16 = add64(r16, r16); + r17 = add64(r17, r17); + r18 = add64(r18, r18); + r19 = add64(r19, r19); + r20 = add64(r20, r20); + + ROUND_MUL(0, 0, r0, r1); + u = fma52lo(get_zero64(), r0, K); + ROUND_MUL(1, 1, r2, r3); + ROUND_MUL(2, 2, r4, r5); + ROUND_MUL(3, 3, r6, r7); + ROUND_MUL(4, 4, r8, r9); + ROUND_MUL(5, 5, r10, r11); + ROUND_MUL(6, 6, r12, r13); + ROUND_MUL(7, 7, r14, r15); + ROUND_MUL(8, 8, r16, r17); + ROUND_MUL(9, 9, r18, r19); + ROUND_MUL(10,10,r20, r21); + + // reduction n521 + N521_REDUCTION_ROUND(u, r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, modulus, K); + N521_REDUCTION_ROUND(u, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, modulus, K); + N521_REDUCTION_ROUND(u, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, modulus, K); + N521_REDUCTION_ROUND(u, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, modulus, K); + N521_REDUCTION_ROUND(u, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, modulus, K); + N521_REDUCTION_ROUND(u, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15, modulus, K); + N521_REDUCTION_ROUND(u, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15, r16, modulus, K); + N521_REDUCTION_ROUND(u, r7, r8, r9, r10, r11, r12, r13, r14, r15, r16, r17, modulus, K); + N521_REDUCTION_ROUND(u, r8, r9, r10, r11, r12, r13, r14, r15, r16, r17, r18, modulus, K); + N521_REDUCTION_ROUND(u, r9, r10, r11, r12, r13, r14, r15, r16, r17, r18, r19, modulus, K); + N521_REDUCTION_ROUND(u, r10, r11, r12, r13, r14, r15, r16, r17, r18, r19, r20, modulus, K); + + + // normalization + NORM_LSHIFTR(r, 11, 12) + NORM_LSHIFTR(r, 12, 13) + NORM_LSHIFTR(r, 13, 14) + NORM_LSHIFTR(r, 14, 15) + NORM_LSHIFTR(r, 15, 16) + NORM_LSHIFTR(r, 16, 17) + NORM_LSHIFTR(r, 17, 18) + NORM_LSHIFTR(r, 18, 19) + NORM_LSHIFTR(r, 19, 20) + NORM_LSHIFTR(r, 20, 21) + + r[0] = r11; + r[1] = r12; + r[2] = r13; + r[3] = r14; + r[4] = r15; + r[5] = r16; + r[6] = r17; + r[7] = r18; + r[8] = r19; + r[9] = r20; + r[10]= r21; +} + +void MB_FUNC_NAME(ifma_tomont52_n521_)(U64 r[], const U64 a[]) +{ + MB_FUNC_NAME(ifma_amm52_n521_)(r, a, (U64*)n521_rr_mb); +} + +void MB_FUNC_NAME(ifma_frommont52_n521_)(U64 r[], const U64 a[]) +{ + MB_FUNC_NAME(ifma_amm52_n521_)(r, a, (U64*)ones); +} + +/* +// computes r = 1/z = z^(n384-2) mod n384 +// +// note: z in in Montgomery domain (as soon mul() and sqr() below are amm-functions +// r in Montgomery domain too +*/ +#define fe52_sqr MB_FUNC_NAME(ifma_ams52_n521_) +#define fe52_mul MB_FUNC_NAME(ifma_amm52_n521_) + +/* r = base^(2^n) */ +__INLINE void fe52_sqr_pwr(U64 r[], const U64 base[], int n) +{ + if(r!=base) { + fe52_sqr(r,base); + n--; + } + for(; n>0; n--) + fe52_sqr(r,r); +} + +void MB_FUNC_NAME(ifma_aminv52_n521_)(U64 r[], const U64 z[]) +{ + int i; + + // pwr_z_Tbl[i][] = z^i, i=0,..,15 + __ALIGN64 U64 pwr_z_Tbl[16][P521_LEN52]; + __ALIGN64 U64 lexp[P521_LEN52]; + + MB_FUNC_NAME(mov_FE521_)(pwr_z_Tbl[0], (U64*)n521_r_mb); + MB_FUNC_NAME(mov_FE521_)(pwr_z_Tbl[1], z); + + for(i=2; i<16; i+=2) { + fe52_sqr(pwr_z_Tbl[i], pwr_z_Tbl[i/2]); + fe52_mul(pwr_z_Tbl[i+1], pwr_z_Tbl[i], z); + } + + // pwr = (n521-2) in big endian + int8u pwr[] = "\x1\xFF" + "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" + "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" + "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" + "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFA" + "\x51\x86\x87\x83\xBF\x2F\x96\x6B" + "\x7F\xCC\x01\x48\xF7\x09\xA5\xD0" + "\x3B\xB5\xC9\xB8\x89\x9C\x47\xAE" + "\xBB\x6F\xB7\x1E\x91\x38\x64\x07"; + + /* + // process 25 low bytes of the exponent: :FA 51 86 ... 64 07" + */ + /* init result */ + MB_FUNC_NAME(mov_FE521_)(lexp, (U64*)n521_r_mb); + + for(i=33; i>4) &0xF; + int lo = v & 0xF; + + fe52_sqr(lexp, lexp); + fe52_sqr(lexp, lexp); + fe52_sqr(lexp, lexp); + fe52_sqr(lexp, lexp); + if(hi) + fe52_mul(lexp, lexp, pwr_z_Tbl[hi]); + fe52_sqr(lexp, lexp); + fe52_sqr(lexp, lexp); + fe52_sqr(lexp, lexp); + fe52_sqr(lexp, lexp); + if(lo) + fe52_mul(lexp, lexp, pwr_z_Tbl[lo]); + } + + /* + // process high part of the exponent: "0x1 0xffffffffffffffff 0xffffffffffffffff 0xffffffffffffffff 0xffffffffffffffff" + */ + __ALIGN64 U64 u[P521_LEN52]; + __ALIGN64 U64 v[P521_LEN52]; + + MB_FUNC_NAME(mov_FE521_)(u, pwr_z_Tbl[15]); /* u = z^0xF */ + + fe52_sqr_pwr(v, u, 4); /* v= (z^0xF)^(2^4) = z^(0xF0) */ + fe52_mul(u, v, u); /* u = z^0xF0 * z^(0xF) = z^(0xFF) */ + + fe52_sqr_pwr(v, u, 8); /* v= (z^0xFF)^(2^8) = z^(0xFF00) */ + fe52_mul(u, v, u); /* u = z^0xFF00 * z^(0xFF) = z^(0xFFFF) */ + + fe52_sqr_pwr(v, u, 16); /* v= (z^0xFFFF)^(2^16) = z^(0xFFFF0000) */ + fe52_mul(u, v, u); /* u = z^0xFFFF0000 * z^(0xFFFF) = z^(0xFFFFFFFF) */ + + fe52_sqr_pwr(v, u, 32); /* v= (z^0xFFFFFFFF)^(2^32) = z^(0xFFFFFFFF00000000) */ + fe52_mul(u, v, u); /* u = z^0xFFFFFFFF00000000 * z^(0xFFFFFFFF) = z^(0xFFFFFFFFFFFFFFFF) */ + + fe52_sqr_pwr(v, z, 64); + fe52_mul(v, v, u); /* v = z^(0x1.FFFFFFFFFFFFFFFF) */ + + fe52_sqr_pwr(v, v, 64); + fe52_mul(v, v, u); /* v = z^(0x1.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF) */ + + fe52_sqr_pwr(v, v, 64); + fe52_mul(v, v, u); /* v = z^(0x1.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF) */ + + fe52_sqr_pwr(v, v, 64); + fe52_mul(v, v, u); /* v = z^(0x1.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF) */ + + /* combine low and high results */ + fe52_sqr_pwr(v, v, 64*4+8);/* u = z^(0x1.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.00.0000000000000000.0000000000000000.0000000000000000.0000000000000000) */ + fe52_mul(r, v, lexp); /* r = z^(0x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFA.51868783BF2F966B.7FCC0148F709A5D0.3BB5C9B8899C47AE.BB6FB71E91386407) */ +} + + +/*===================================================================== + + Specialized single operations over n521 add, sub, neg + +=====================================================================*/ +__INLINE __mb_mask MB_FUNC_NAME(lt_mbx_digit_)(const U64 a, const U64 b, const __mb_mask lt_mask) +{ + U64 d = mask_sub64(sub64(a, b), lt_mask, sub64(a, b), set1(1)); + return cmp64_mask(d, get_zero64(), _MM_CMPINT_LT); +} + +void MB_FUNC_NAME(ifma_add52_n521_)(U64 R[], const U64 A[], const U64 B[]) +{ + /* r = a + b */ + U64 r0 = add64(A[0], B[0]); + U64 r1 = add64(A[1], B[1]); + U64 r2 = add64(A[2], B[2]); + U64 r3 = add64(A[3], B[3]); + U64 r4 = add64(A[4], B[4]); + U64 r5 = add64(A[5], B[5]); + U64 r6 = add64(A[6], B[6]); + U64 r7 = add64(A[7], B[7]); + U64 r8 = add64(A[8], B[8]); + U64 r9 = add64(A[9], B[9]); + U64 r10= add64(A[10],B[10]); + + /* lt = {r0 - r10} < 2*n */ + __mb_mask + lt = MB_FUNC_NAME(lt_mbx_digit_)( r0, ((U64*)(n521_x2))[0], 0); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r1, ((U64*)(n521_x2))[1], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r2, ((U64*)(n521_x2))[2], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r3, ((U64*)(n521_x2))[3], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r4, ((U64*)(n521_x2))[4], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r5, ((U64*)(n521_x2))[5], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r6, ((U64*)(n521_x2))[6], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r7, ((U64*)(n521_x2))[7], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r8, ((U64*)(n521_x2))[8], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r9, ((U64*)(n521_x2))[9], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r10,((U64*)(n521_x2))[10],lt); + + /* {r0 - r10} -= 2*p */ + r0 = mask_sub64(r0, ~lt, r0, ((U64*)(n521_x2))[0]); + r1 = mask_sub64(r1, ~lt, r1, ((U64*)(n521_x2))[1]); + r2 = mask_sub64(r2, ~lt, r2, ((U64*)(n521_x2))[2]); + r3 = mask_sub64(r3, ~lt, r3, ((U64*)(n521_x2))[3]); + r4 = mask_sub64(r4, ~lt, r4, ((U64*)(n521_x2))[4]); + r5 = mask_sub64(r5, ~lt, r5, ((U64*)(n521_x2))[5]); + r6 = mask_sub64(r6, ~lt, r6, ((U64*)(n521_x2))[6]); + r7 = mask_sub64(r7, ~lt, r7, ((U64*)(n521_x2))[7]); + r8 = mask_sub64(r8, ~lt, r8, ((U64*)(n521_x2))[8]); + r9 = mask_sub64(r9, ~lt, r9, ((U64*)(n521_x2))[9]); + r10= mask_sub64(r10,~lt, r10,((U64*)(n521_x2))[10]); + + /* normalize r0 - r7 */ + NORM_ASHIFTR(r, 0, 1) + NORM_ASHIFTR(r, 1, 2) + NORM_ASHIFTR(r, 2, 3) + NORM_ASHIFTR(r, 3, 4) + NORM_ASHIFTR(r, 4, 5) + NORM_ASHIFTR(r, 5, 6) + NORM_ASHIFTR(r, 6, 7) + NORM_ASHIFTR(r, 7, 8) + NORM_ASHIFTR(r, 8, 9) + NORM_ASHIFTR(r, 9,10) + + R[0] = r0; + R[1] = r1; + R[2] = r2; + R[3] = r3; + R[4] = r4; + R[5] = r5; + R[6] = r6; + R[7] = r7; + R[8] = r8; + R[9] = r9; + R[10]= r10; +} + +void MB_FUNC_NAME(ifma_sub52_n521_)(U64 R[], const U64 A[], const U64 B[]) +{ + /* r = a - b */ + U64 r0 = sub64(A[0], B[0]); + U64 r1 = sub64(A[1], B[1]); + U64 r2 = sub64(A[2], B[2]); + U64 r3 = sub64(A[3], B[3]); + U64 r4 = sub64(A[4], B[4]); + U64 r5 = sub64(A[5], B[5]); + U64 r6 = sub64(A[6], B[6]); + U64 r7 = sub64(A[7], B[7]); + U64 r8 = sub64(A[8], B[8]); + U64 r9 = sub64(A[9], B[9]); + U64 r10= sub64(A[10],B[10]); + + /* lt = {r0 - r10} < 0 */ + __mb_mask + lt = MB_FUNC_NAME(lt_mbx_digit_)(r0, get_zero64(), 0); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r1, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r2, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r3, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r4, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r5, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r6, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r7, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r8, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r9, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r10,get_zero64(), lt); + + r0 = mask_add64(r0, lt, r0, ((U64*)(n521_x2))[0]); + r1 = mask_add64(r1, lt, r1, ((U64*)(n521_x2))[1]); + r2 = mask_add64(r2, lt, r2, ((U64*)(n521_x2))[2]); + r3 = mask_add64(r3, lt, r3, ((U64*)(n521_x2))[3]); + r4 = mask_add64(r4, lt, r4, ((U64*)(n521_x2))[4]); + r5 = mask_add64(r5, lt, r5, ((U64*)(n521_x2))[5]); + r6 = mask_add64(r6, lt, r6, ((U64*)(n521_x2))[6]); + r7 = mask_add64(r7, lt, r7, ((U64*)(n521_x2))[7]); + r8 = mask_add64(r8, lt, r8, ((U64*)(n521_x2))[8]); + r9 = mask_add64(r9, lt, r9, ((U64*)(n521_x2))[9]); + r10= mask_add64(r10,lt, r10,((U64*)(n521_x2))[10]); + + /* normalize r0 - r7 */ + NORM_ASHIFTR(r, 0, 1) + NORM_ASHIFTR(r, 1, 2) + NORM_ASHIFTR(r, 2, 3) + NORM_ASHIFTR(r, 3, 4) + NORM_ASHIFTR(r, 4, 5) + NORM_ASHIFTR(r, 5, 6) + NORM_ASHIFTR(r, 6, 7) + NORM_ASHIFTR(r, 7, 8) + NORM_ASHIFTR(r, 8, 9) + NORM_ASHIFTR(r, 9, 10) + + R[0] = r0; + R[1] = r1; + R[2] = r2; + R[3] = r3; + R[4] = r4; + R[5] = r5; + R[6] = r6; + R[7] = r7; + R[8] = r8; + R[9] = r9; + R[10]= r10; +} + +void MB_FUNC_NAME(ifma_neg52_n521_)(U64 R[], const U64 A[]) +{ + __mb_mask nz_mask = ~MB_FUNC_NAME(is_zero_FE521_)(A); + + /* {r0 - r10} = 2*p - A */ + U64 r0 = mask_sub64( A[0], nz_mask, ((U64*)(n521_x2))[0], A[0] ); + U64 r1 = mask_sub64( A[0], nz_mask, ((U64*)(n521_x2))[1], A[1] ); + U64 r2 = mask_sub64( A[0], nz_mask, ((U64*)(n521_x2))[2], A[2] ); + U64 r3 = mask_sub64( A[0], nz_mask, ((U64*)(n521_x2))[3], A[3] ); + U64 r4 = mask_sub64( A[0], nz_mask, ((U64*)(n521_x2))[4], A[4] ); + U64 r5 = mask_sub64( A[0], nz_mask, ((U64*)(n521_x2))[5], A[5] ); + U64 r6 = mask_sub64( A[0], nz_mask, ((U64*)(n521_x2))[6], A[6] ); + U64 r7 = mask_sub64( A[0], nz_mask, ((U64*)(n521_x2))[7], A[7] ); + U64 r8 = mask_sub64( A[0], nz_mask, ((U64*)(n521_x2))[8], A[8] ); + U64 r9 = mask_sub64( A[0], nz_mask, ((U64*)(n521_x2))[9], A[9] ); + U64 r10= mask_sub64( A[0], nz_mask, ((U64*)(n521_x2))[10],A[10]); + + /* normalize r0 - r10 */ + NORM_ASHIFTR(r, 0, 1) + NORM_ASHIFTR(r, 1, 2) + NORM_ASHIFTR(r, 2, 3) + NORM_ASHIFTR(r, 3, 4) + NORM_ASHIFTR(r, 4, 5) + NORM_ASHIFTR(r, 5, 6) + NORM_ASHIFTR(r, 6, 7) + NORM_ASHIFTR(r, 7, 8) + NORM_ASHIFTR(r, 8, 9) + NORM_ASHIFTR(r, 9, 10) + + R[0] = r0; + R[1] = r1; + R[2] = r2; + R[3] = r3; + R[4] = r4; + R[5] = r5; + R[6] = r6; + R[7] = r7; + R[8] = r8; + R[9] = r9; + R[10]= r10; +} + +// Disable optimization for VS17 +#if defined(_MSC_VER) && (_MSC_VER < 1920) && !defined(__INTEL_COMPILER) + #pragma optimize( "", off ) +#endif + +/* r = (a>=n521)? a-n521 : a */ +void MB_FUNC_NAME(ifma_fastred52_pn521_)(U64 R[], const U64 A[]) +{ + /* r = a - b */ + U64 r0 = sub64(A[0], ((U64*)(n521_mb))[0]); + U64 r1 = sub64(A[1], ((U64*)(n521_mb))[1]); + U64 r2 = sub64(A[2], ((U64*)(n521_mb))[2]); + U64 r3 = sub64(A[3], ((U64*)(n521_mb))[3]); + U64 r4 = sub64(A[4], ((U64*)(n521_mb))[4]); + U64 r5 = sub64(A[5], ((U64*)(n521_mb))[5]); + U64 r6 = sub64(A[6], ((U64*)(n521_mb))[6]); + U64 r7 = sub64(A[7], ((U64*)(n521_mb))[7]); + U64 r8 = sub64(A[8], ((U64*)(n521_mb))[8]); + U64 r9 = sub64(A[9], ((U64*)(n521_mb))[9]); + U64 r10= sub64(A[10],((U64*)(n521_mb))[10]); + + /* lt = {r0 - r10} < 0 */ + __mb_mask + lt = MB_FUNC_NAME(lt_mbx_digit_)(r0, get_zero64(), 0); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r1, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r2, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r3, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r4, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r5, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r6, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r7, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r8, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r9, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r10,get_zero64(), lt); + + r0 = mask_mov64(A[0], ~lt, r0); + r1 = mask_mov64(A[1], ~lt, r1); + r2 = mask_mov64(A[2], ~lt, r2); + r3 = mask_mov64(A[3], ~lt, r3); + r4 = mask_mov64(A[4], ~lt, r4); + r5 = mask_mov64(A[5], ~lt, r5); + r6 = mask_mov64(A[6], ~lt, r6); + r7 = mask_mov64(A[7], ~lt, r7); + r8 = mask_mov64(A[8], ~lt, r8); + r9 = mask_mov64(A[9], ~lt, r9); + r10= mask_mov64(A[10],~lt, r10); + + /* normalize r0 - r7 */ + NORM_ASHIFTR(r, 0, 1) + NORM_ASHIFTR(r, 1, 2) + NORM_ASHIFTR(r, 2, 3) + NORM_ASHIFTR(r, 3, 4) + NORM_ASHIFTR(r, 4, 5) + NORM_ASHIFTR(r, 5, 6) + NORM_ASHIFTR(r, 6, 7) + NORM_ASHIFTR(r, 7, 8) + NORM_ASHIFTR(r, 8, 9) + NORM_ASHIFTR(r, 9,10) + + R[0] = r0; + R[1] = r1; + R[2] = r2; + R[3] = r3; + R[4] = r4; + R[5] = r5; + R[6] = r6; + R[7] = r7; + R[8] = r8; + R[9] = r9; + R[10]= r10; +} + +#if defined(_MSC_VER) && (_MSC_VER < 1920) && !defined(__INTEL_COMPILER) + #pragma optimize( "", on ) +#endif + +__mb_mask MB_FUNC_NAME(ifma_cmp_lt_n521_)(const U64 a[]) +{ + return MB_FUNC_NAME(cmp_lt_FE521_)(a,(const U64 (*))n521_mb); +} + +__mb_mask MB_FUNC_NAME(ifma_check_range_n521_)(const U64 A[]) +{ + __mb_mask + mask = MB_FUNC_NAME(is_zero_FE521_)(A); + mask |= ~MB_FUNC_NAME(ifma_cmp_lt_n521_)(A); + + return mask; +} \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_arith_p256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_arith_p256.c new file mode 100644 index 0000000..421708e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_arith_p256.c @@ -0,0 +1,688 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +/* Constants */ +#define LEN52 NUMBER_OF_DIGITS(256,DIGIT_SIZE) + +/* +// prime256 = 2^256 - 2^224 + 2^192 + 2^96 -1 +// in 2^52 radix +*/ +__ALIGN64 static const int64u p256_mb[LEN52][8] = { + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x00000fffffffffff) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000001000000000) }, + { REP8_DECL(0x0000ffffffff0000) } +}; + +__ALIGN64 static const int64u p256x2_mb[LEN52][8] = { + { REP8_DECL(0x000ffffffffffffe) }, + { REP8_DECL(0x00001fffffffffff) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000002000000000) }, + { REP8_DECL(0x0001fffffffe0000) } +}; + +/* +// Note that +// k0 = -(1/p256 mod 2^DIGIT_SIZE) equals 1 +// The implementation takes this fact into account +*/ + +/* to Montgomery conversion constant +// rr = 2^((5*DIGIT_SIZE)*2) mod p256 +*/ +__ALIGN64 static const int64u p256_rr_mb[5][8] = { + { REP8_DECL(0x0000000000000300) }, + { REP8_DECL(0x000ffffffff00000) }, + { REP8_DECL(0x000ffffefffffffb) }, + { REP8_DECL(0x000fdfffffffffff) }, + { REP8_DECL(0x0000000004ffffff) } +}; + + + +/* other constants */ +__ALIGN64 static const int64u VDIGIT_MASK_[8] = + {DIGIT_MASK, DIGIT_MASK, DIGIT_MASK, DIGIT_MASK, + DIGIT_MASK, DIGIT_MASK, DIGIT_MASK, DIGIT_MASK}; +#define VDIGIT_MASK loadu64(VDIGIT_MASK_) + +#define P256_PRIME_TOP_ 0xFFFFFFFF0000LL +__ALIGN64 static const int64u VP256_PRIME_TOP_[8] = + {P256_PRIME_TOP_, P256_PRIME_TOP_, P256_PRIME_TOP_, P256_PRIME_TOP_, + P256_PRIME_TOP_, P256_PRIME_TOP_, P256_PRIME_TOP_, P256_PRIME_TOP_}; +#define VP256_PRIME_TOP loadu64(VP256_PRIME_TOP_) + + +/* Round operations */ + +#define ROUND_MUL_SRC(I, J, S_LO, R_LO, S_HI, R_HI) \ + R_LO = fma52lo(S_LO, va[I], vb[J]); \ + R_HI = fma52hi(S_HI, va[I], vb[J]); + +#define ROUND_MUL(I, J, M0, M1) \ + ROUND_MUL_SRC(I, J, M0, M0, M1, M1) + +/* Reduction round for p256 prime */ + +// Note that k == 1 for p256 curve +#ifdef IFMA_RED + +#define MUL_ADD_P256(u, res0, res1, res2, res3, res4, res5) \ + { \ + U64 u = and64_const(res0, DIGIT_MASK); /* k == 1 */ \ + res0 = fma52lo(res0, u, set64((1ULL<<52) - 1)); \ + res1 = fma52hi(res1, u, set64((1ULL<<52) - 1)); \ + res1 = add64(res1, srli64(res0, 52)); \ + res1 = fma52lo(res1, u, set64((1ULL<<44) - 1)); \ + res2 = fma52hi(res2, u, set64((1ULL<<44) - 1)); \ + res3 = fma52lo(res3, u, set64(1ULL<<36)); \ + res4 = fma52hi(res4, u, set64(1ULL<<36)); \ + res4 = fma52lo(res4, u, set64(0xFFFFFFFF0000LL)); \ + res5 = fma52hi(res5, u, set64(0xFFFFFFFF0000LL)); \ + } + +#else // IFMA_RED + +#define MUL_ADD_P256(u, res0, res1, res2, res3, res4, res5) \ + { \ + /* a * ( 2^96 - 1 ) = -a + a * 2^96 = -a + a * 2^{52+44} */ \ + U64 u = and64(res0, VDIGIT_MASK); /* k == 1 */ \ + /*res0 = sub64(res0, u);*/ /* Zero out low 52 bits */ \ + res1 = add64(res1, srli64(res0, DIGIT_SIZE)); /* Carry propagation */ \ + res1 = add64(res1, and64(slli64(u, 44), VDIGIT_MASK)); \ + res2 = add64(res2, srli64(u, DIGIT_SIZE - 44)); \ + /* ( a * 2^{36} ) * 2^{52*3} */ \ + res3 = add64(res3, and64(slli64(u, 36), VDIGIT_MASK)); \ + res4 = add64(res4, srli64(u, DIGIT_SIZE - 36)); \ + /* ( a * (2^{48} - 1) - a * (2^{16} - 1) ) * 2^{52*4} = */ \ + /* ( a * 2^{48} - a * 2^{16} ) * 2^{52*4} */ \ + res4 = fma52lo(res4, u, VP256_PRIME_TOP); \ + res5 = fma52hi(res5, u, VP256_PRIME_TOP); \ + } + +#endif // IFMA_RED + +/*===================================================================== + + Specialized single and dual operations in p256 - sqr & mul + +=====================================================================*/ + +void MB_FUNC_NAME(ifma_ams52_p256_)(U64 r[], const U64 va[]) +{ + const U64* vb = va; + + U64 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9; + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = get_zero64(); + + // Calculate full square + ROUND_MUL(0, 1, r1, r2) + ROUND_MUL(0, 2, r2, r3) + ROUND_MUL(0, 3, r3, r4) + ROUND_MUL(0, 4, r4, r5) + ROUND_MUL(1, 4, r5, r6) + ROUND_MUL(2, 4, r6, r7) + ROUND_MUL(3, 4, r7, r8) + + ROUND_MUL(1, 2, r3, r4) + ROUND_MUL(1, 3, r4, r5) + ROUND_MUL(2, 3, r5, r6) + + r1 = add64(r1, r1); + r2 = add64(r2, r2); + r3 = add64(r3, r3); + r4 = add64(r4, r4); + r5 = add64(r5, r5); + r6 = add64(r6, r6); + r7 = add64(r7, r7); + r8 = add64(r8, r8); + + ROUND_MUL(0, 0, r0, r1) + ROUND_MUL(1, 1, r2, r3) + ROUND_MUL(2, 2, r4, r5) + ROUND_MUL(3, 3, r6, r7) + ROUND_MUL(4, 4, r8, r9) + + // Reduction + MUL_ADD_P256(u0, r0, r1, r2, r3, r4, r5); + MUL_ADD_P256(u1, r1, r2, r3, r4, r5, r6); + MUL_ADD_P256(u2, r2, r3, r4, r5, r6, r7); + MUL_ADD_P256(u3, r3, r4, r5, r6, r7, r8); + MUL_ADD_P256(u4, r4, r5, r6, r7, r8, r9); + + // normalization + NORM_LSHIFTR(r, 5, 6) + NORM_LSHIFTR(r, 6, 7) + NORM_LSHIFTR(r, 7, 8) + NORM_LSHIFTR(r, 8, 9) + + r[0] = r5; + r[1] = r6; + r[2] = r7; + r[3] = r8; + r[4] = r9; +} + +void MB_FUNC_NAME(ifma_amm52_p256_)(U64 r[], const U64 va[], const U64 vb[]) +{ + U64 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9; + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = get_zero64(); + + ROUND_MUL(4, 4, r8, r9) + ROUND_MUL(3, 0, r3, r4) + ROUND_MUL(1, 2, r3, r4) + ROUND_MUL(0, 3, r3, r4) + ROUND_MUL(2, 1, r3, r4) + ROUND_MUL(2, 2, r4, r5) + ROUND_MUL(0, 4, r4, r5) + ROUND_MUL(1, 3, r4, r5) + ROUND_MUL(3, 1, r4, r5) + ROUND_MUL(4, 0, r4, r5) + ROUND_MUL(1, 4, r5, r6) + ROUND_MUL(2, 3, r5, r6) + ROUND_MUL(3, 2, r5, r6) + ROUND_MUL(4, 1, r5, r6) + ROUND_MUL(2, 4, r6, r7) + ROUND_MUL(3, 3, r6, r7) + ROUND_MUL(4, 2, r6, r7) + ROUND_MUL(0, 0, r0, r1) + ROUND_MUL(0, 1, r1, r2) + ROUND_MUL(0, 2, r2, r3) + ROUND_MUL(1, 0, r1, r2) + ROUND_MUL(1, 1, r2, r3) + ROUND_MUL(2, 0, r2, r3) + ROUND_MUL(3, 4, r7, r8) + ROUND_MUL(4, 3, r7, r8) + + // Reduction + MUL_ADD_P256(u0, r0, r1, r2, r3, r4, r5); + MUL_ADD_P256(u1, r1, r2, r3, r4, r5, r6); + MUL_ADD_P256(u2, r2, r3, r4, r5, r6, r7); + MUL_ADD_P256(u3, r3, r4, r5, r6, r7, r8); + MUL_ADD_P256(u4, r4, r5, r6, r7, r8, r9); + + // normalization + NORM_LSHIFTR(r, 5, 6) + NORM_LSHIFTR(r, 6, 7) + NORM_LSHIFTR(r, 7, 8) + NORM_LSHIFTR(r, 8, 9) + + r[0] = r5; + r[1] = r6; + r[2] = r7; + r[3] = r8; + r[4] = r9; +} + +void MB_FUNC_NAME(ifma_ams52_p256_dual_)(U64 r0[], U64 r1[], + const U64 inp0[], const U64 inp1[]) +{ + U64 *va; // = (U64 *)inp0; + U64 *vb; // = (U64 *)inp0; + + U64 r00, r01, r02, r03, r04, r05, r06, r07, r08, r09; + U64 r10, r11, r12, r13, r14, r15, r16, r17, r18, r19; + r00 = r01 = r02 = r03 = r04 = r05 = r06 = r07 = r08 = r09 = get_zero64(); + r10 = r11 = r12 = r13 = r14 = r15 = r16 = r17 = r18 = r19 = get_zero64(); + + /// Calculate full square + va = vb = (U64 *)inp0; + // Multiplication + ROUND_MUL(0, 1, r01, r02) + ROUND_MUL(0, 2, r02, r03) + ROUND_MUL(0, 3, r03, r04) + ROUND_MUL(0, 4, r04, r05) + ROUND_MUL(1, 4, r05, r06) + ROUND_MUL(2, 4, r06, r07) + ROUND_MUL(3, 4, r07, r08) + ROUND_MUL(1, 2, r03, r04) + ROUND_MUL(1, 3, r04, r05) + ROUND_MUL(2, 3, r05, r06) + // Doubling + r01 = add64(r01, r01); + r02 = add64(r02, r02); + r03 = add64(r03, r03); + r04 = add64(r04, r04); + r05 = add64(r05, r05); + r06 = add64(r06, r06); + r07 = add64(r07, r07); + r08 = add64(r08, r08); + // Adding square + ROUND_MUL(0, 0, r00, r01) + ROUND_MUL(1, 1, r02, r03) + ROUND_MUL(2, 2, r04, r05) + ROUND_MUL(3, 3, r06, r07) + ROUND_MUL(4, 4, r08, r09) + + /// Calculate full square + va = vb = (U64 *)inp1; + // Multiplication + ROUND_MUL(0, 1, r11, r12) + ROUND_MUL(0, 2, r12, r13) + ROUND_MUL(0, 3, r13, r14) + ROUND_MUL(0, 4, r14, r15) + ROUND_MUL(1, 4, r15, r16) + ROUND_MUL(2, 4, r16, r17) + ROUND_MUL(3, 4, r17, r18) + ROUND_MUL(1, 2, r13, r14) + ROUND_MUL(1, 3, r14, r15) + ROUND_MUL(2, 3, r15, r16) + // Doubling + r11 = add64(r11, r11); + r12 = add64(r12, r12); + r13 = add64(r13, r13); + r14 = add64(r14, r14); + r15 = add64(r15, r15); + r16 = add64(r16, r16); + r17 = add64(r17, r17); + r18 = add64(r18, r18); + // Adding square + ROUND_MUL(0, 0, r10, r11) + ROUND_MUL(1, 1, r12, r13) + ROUND_MUL(2, 2, r14, r15) + ROUND_MUL(3, 3, r16, r17) + ROUND_MUL(4, 4, r18, r19) + + // Reduction res0 + MUL_ADD_P256(u0, r00, r01, r02, r03, r04, r05); + MUL_ADD_P256(u1, r01, r02, r03, r04, r05, r06); + MUL_ADD_P256(u2, r02, r03, r04, r05, r06, r07); + MUL_ADD_P256(u3, r03, r04, r05, r06, r07, r08); + MUL_ADD_P256(u4, r04, r05, r06, r07, r08, r09); + + // Reduction res1 + MUL_ADD_P256(u0, r10, r11, r12, r13, r14, r15); + MUL_ADD_P256(u1, r11, r12, r13, r14, r15, r16); + MUL_ADD_P256(u2, r12, r13, r14, r15, r16, r17); + MUL_ADD_P256(u3, r13, r14, r15, r16, r17, r18); + MUL_ADD_P256(u4, r14, r15, r16, r17, r18, r19); + + // normalization res0 + NORM_LSHIFTR(r0, 5, 6) + NORM_LSHIFTR(r0, 6, 7) + NORM_LSHIFTR(r0, 7, 8) + NORM_LSHIFTR(r0, 8, 9) + + r0[0] = r05; + r0[1] = r06; + r0[2] = r07; + r0[3] = r08; + r0[4] = r09; + + // normalization res1 + NORM_LSHIFTR(r1, 5, 6) + NORM_LSHIFTR(r1, 6, 7) + NORM_LSHIFTR(r1, 7, 8) + NORM_LSHIFTR(r1, 8, 9) + + r1[0] = r15; + r1[1] = r16; + r1[2] = r17; + r1[3] = r18; + r1[4] = r19; +} + +void MB_FUNC_NAME(ifma_amm52_p256_dual_)(U64 r0[], U64 r1[], + const U64 inp0A[], const U64 inp0B[], + const U64 inp1A[], const U64 inp1B[]) +{ + U64 *va, *vb; + + U64 r00, r01, r02, r03, r04, r05, r06, r07, r08, r09; + U64 r10, r11, r12, r13, r14, r15, r16, r17, r18, r19; + r00 = r01 = r02 = r03 = r04 = r05 = r06 = r07 = r08 = r09 = get_zero64(); + r10 = r11 = r12 = r13 = r14 = r15 = r16 = r17 = r18 = r19 = get_zero64(); + + // 5x5 multiplication + va = (U64 *)inp0A; + vb = (U64 *)inp0B; + ROUND_MUL(4, 4, r08, r09) + ROUND_MUL(3, 0, r03, r04) + ROUND_MUL(1, 2, r03, r04) + ROUND_MUL(0, 3, r03, r04) + ROUND_MUL(2, 1, r03, r04) + ROUND_MUL(2, 2, r04, r05) + ROUND_MUL(0, 4, r04, r05) + ROUND_MUL(1, 3, r04, r05) + ROUND_MUL(3, 1, r04, r05) + ROUND_MUL(4, 0, r04, r05) + ROUND_MUL(1, 4, r05, r06) + ROUND_MUL(2, 3, r05, r06) + ROUND_MUL(3, 2, r05, r06) + ROUND_MUL(4, 1, r05, r06) + ROUND_MUL(2, 4, r06, r07) + ROUND_MUL(3, 3, r06, r07) + ROUND_MUL(4, 2, r06, r07) + ROUND_MUL(0, 0, r00, r01) + ROUND_MUL(0, 1, r01, r02) + ROUND_MUL(0, 2, r02, r03) + ROUND_MUL(1, 0, r01, r02) + ROUND_MUL(1, 1, r02, r03) + ROUND_MUL(2, 0, r02, r03) + ROUND_MUL(3, 4, r07, r08) + ROUND_MUL(4, 3, r07, r08) + + // 5x5 multiplication + va = (U64 *)inp1A; + vb = (U64 *)inp1B; + ROUND_MUL(4, 4, r18, r19) + ROUND_MUL(3, 0, r13, r14) + ROUND_MUL(1, 2, r13, r14) + ROUND_MUL(0, 3, r13, r14) + ROUND_MUL(2, 1, r13, r14) + ROUND_MUL(2, 2, r14, r15) + ROUND_MUL(0, 4, r14, r15) + ROUND_MUL(1, 3, r14, r15) + ROUND_MUL(3, 1, r14, r15) + ROUND_MUL(4, 0, r14, r15) + ROUND_MUL(1, 4, r15, r16) + ROUND_MUL(2, 3, r15, r16) + ROUND_MUL(3, 2, r15, r16) + ROUND_MUL(4, 1, r15, r16) + ROUND_MUL(2, 4, r16, r17) + ROUND_MUL(3, 3, r16, r17) + ROUND_MUL(4, 2, r16, r17) + ROUND_MUL(0, 0, r10, r11) + ROUND_MUL(0, 1, r11, r12) + ROUND_MUL(0, 2, r12, r13) + ROUND_MUL(1, 0, r11, r12) + ROUND_MUL(1, 1, r12, r13) + ROUND_MUL(2, 0, r12, r13) + ROUND_MUL(3, 4, r17, r18) + ROUND_MUL(4, 3, r17, r18) + + // Reduction for input 0 + MUL_ADD_P256(u0, r00, r01, r02, r03, r04, r05) + MUL_ADD_P256(u1, r01, r02, r03, r04, r05, r06) + MUL_ADD_P256(u2, r02, r03, r04, r05, r06, r07) + MUL_ADD_P256(u3, r03, r04, r05, r06, r07, r08) + MUL_ADD_P256(u4, r04, r05, r06, r07, r08, r09) + + // Reduction for input 1 + MUL_ADD_P256(u0, r10, r11, r12, r13, r14, r15) + MUL_ADD_P256(u1, r11, r12, r13, r14, r15, r16) + MUL_ADD_P256(u2, r12, r13, r14, r15, r16, r17) + MUL_ADD_P256(u3, r13, r14, r15, r16, r17, r18) + MUL_ADD_P256(u4, r14, r15, r16, r17, r18, r19) + + // normalization res0 + NORM_LSHIFTR(r0, 5, 6) + NORM_LSHIFTR(r0, 6, 7) + NORM_LSHIFTR(r0, 7, 8) + NORM_LSHIFTR(r0, 8, 9) + + r0[0] = r05; + r0[1] = r06; + r0[2] = r07; + r0[3] = r08; + r0[4] = r09; + + // normalization res1 + NORM_LSHIFTR(r1, 5, 6) + NORM_LSHIFTR(r1, 6, 7) + NORM_LSHIFTR(r1, 7, 8) + NORM_LSHIFTR(r1, 8, 9) + + r1[0] = r15; + r1[1] = r16; + r1[2] = r17; + r1[3] = r18; + r1[4] = r19; +} +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#define LEN52 NUMBER_OF_DIGITS(256,DIGIT_SIZE) + +void MB_FUNC_NAME(ifma_reduce52_p256_)(U64 R[], const U64 A[]) +{ + /* r = a - p256_mb */ + U64 r0 = sub64(A[0], ((U64*)(p256_mb))[0]); + U64 r1 = sub64(A[1], ((U64*)(p256_mb))[1]); + U64 r2 = sub64(A[2], ((U64*)(p256_mb))[2]); + U64 r3 = sub64(A[3], ((U64*)(p256_mb))[3]); + U64 r4 = sub64(A[4], ((U64*)(p256_mb))[4]); + + /* normalize r0 - r4 */ + NORM_ASHIFTR(r, 0, 1) + NORM_ASHIFTR(r, 1, 2) + NORM_ASHIFTR(r, 2, 3) + NORM_ASHIFTR(r, 3, 4) + + /* r = a> 1 */ + mask = sub64(get_zero64(), and64(t1, one)); + t0 = add64(t0, and64(base, mask)); + t0 = srli64(t0, 1); + + mask = sub64(get_zero64(), and64(t2, one)); + t1 = add64(t1, and64(base, mask)); + t1 = srli64(t1, 1); + + mask = sub64(get_zero64(), and64(t3, one)); + t2 = add64(t2, and64(base, mask)); + t2 = srli64(t2, 1); + + mask = sub64(get_zero64(), and64(t4, one)); + t3 = add64(t3, and64(base, mask)); + t3 = srli64(t3, 1); + + t4 = srli64(t4, 1); + + /* normalize t0, t1, t2, t3, t4 */ + NORM_LSHIFTR(t, 0,1) + NORM_LSHIFTR(t, 1,2) + NORM_LSHIFTR(t, 2,3) + NORM_LSHIFTR(t, 3,4) + + r[0] = t0; + r[1] = t1; + r[2] = t2; + r[3] = t3; + r[4] = t4; +} + +__mb_mask MB_FUNC_NAME(ifma_cmp_lt_p256_)(const U64 a[]) +{ + return MB_FUNC_NAME(cmp_lt_FE256_)(a,(const U64 (*))p256_mb); +} + +__mb_mask MB_FUNC_NAME(ifma_check_range_p256_)(const U64 A[]) +{ + __mb_mask + mask = MB_FUNC_NAME(is_zero_FE256_)(A); + mask |= ~MB_FUNC_NAME(ifma_cmp_lt_p256_)(A); + + return mask; +} + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_arith_p384.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_arith_p384.c new file mode 100644 index 0000000..95d8a38 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_arith_p384.c @@ -0,0 +1,638 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +/* Constants */ + +/* +// p384 = 2^384 - 2^128 - 2^96 + 2^32 - 1 +// in 2^52 radix +*/ +__ALIGN64 static const int64u p384_mb[P384_LEN52][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(0x00000000ffffffff) }, + { REP8_DECL(0x000ff00000000000) }, + { REP8_DECL(0x000ffffffeffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x00000000000fffff) } +}; + +/* 2*p384 */ +__ALIGN64 static const int64u p384_x2[P384_LEN52][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(0x00000001fffffffe) }, + { REP8_DECL(0x000fe00000000000) }, + { REP8_DECL(0x000ffffffdffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x00000000001fffff) } +}; + +/* +// Note that +// k0 = -(1/p384 mod 2^DIGIT_SIZE) equals 1 +// The implementation takes this fact into account +*/ + +/* to Montgomery conversion constant +// rr = 2^((P384_LEN52*DIGIT_SIZE)*2) mod p384 +*/ +__ALIGN64 static const int64u p384_rr_mb[P384_LEN52][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x000fe00000001000) }, + { REP8_DECL(0x0000000000ffffff) }, + { REP8_DECL(0x0000000000000020) }, + { REP8_DECL(0x0000fffffffe0000) }, + { REP8_DECL(0x0000000020000000) }, + { REP8_DECL(0x0000000000000100) }, + { REP8_DECL(0x0000000000000000) } +}; + + +/*===================================================================== + + Specialized operations over p384: sqr & mul + +=====================================================================*/ + +#define ROUND_MUL(I, J, R_LO, R_HI) \ + R_LO = fma52lo(R_LO, va[I], vb[J]); \ + R_HI = fma52hi(R_HI, va[I], vb[J]); + +#define P384_REDUCTION_ROUND(u, r0, r1, r2, r3, r4, r5, r6, r7, r8, modulus) \ +{ \ + U64 t = sub64(get_zero64(), u); \ + U64 lo = and64(t, loadu64(VMASK52)); \ + U64 hi = add64(u, srai64(t,63)); \ + \ + r0 = fma52lo(r0, u, modulus[0]); \ + r1 = fma52lo(r1, u, modulus[1]); \ + r2 = fma52lo(r2, u, modulus[2]); \ + r3 = add64(r3, lo); /*r3 = fma52lo(r3, u, modulus[3]); */ \ + r4 = add64(r4, lo); /*r4 = fma52lo(r4, u, modulus[4]); */ \ + r5 = add64(r5, lo); /*r5 = fma52lo(r5, u, modulus[5]); */ \ + r6 = add64(r6, lo); /*r6 = fma52lo(r6, u, modulus[6]); */ \ + r7 = fma52lo(r7, u, modulus[7]); \ + \ + r1 = add64(r1, srli64(r0, DIGIT_SIZE)); /* carry propagation */ \ + \ + r1 = fma52hi(r1, u, modulus[0]); \ + r2 = fma52hi(r2, u, modulus[1]); \ + r3 = fma52hi(r3, u, modulus[2]); \ + r8 = fma52hi(r8, u, modulus[7]); \ + u = and64(add64(r1, slli64(r1, 32)), loadu64(VMASK52)); /* update u = r1*k, k = (2^32 +1) */ \ + r4 = add64(r4, hi); /*r4 = fma52hi(r4, u, modulus[3]); */ \ + r5 = add64(r5, hi); /*r5 = fma52hi(r5, u, modulus[4]); */ \ + r6 = add64(r6, hi); /*r6 = fma52hi(r6, u, modulus[5]); */ \ + r7 = add64(r7, hi); /*r7 = fma52hi(r7, u, modulus[6]); */ \ +} + +void MB_FUNC_NAME(ifma_amm52_p384_)(U64 r[], const U64 va[], const U64 vb[]) +{ + U64 r0, r1, r2, r3, r4, r5, r6, r7; + int itr; + + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = get_zero64(); + + for(itr=0; itr < P384_LEN52; itr++) { + U64 Yi, T, LO, HI; + U64 Bi = loadu64(vb); + vb++; + + fma52lo_mem(r0, r0, Bi, va, SIMD_BYTES * 0); + Yi = and64(add64(r0, slli64(r0, 32)), loadu64(VMASK52)); /* k = (2^32 +1) */ + T = sub64(get_zero64(), Yi); + LO = and64(T, loadu64(VMASK52)); + HI = add64(Yi, srai64(T,63)); + fma52lo_mem(r1, r1, Bi, va, SIMD_BYTES * 1); + fma52lo_mem(r2, r2, Bi, va, SIMD_BYTES * 2); + fma52lo_mem(r3, r3, Bi, va, SIMD_BYTES * 3); + fma52lo_mem(r4, r4, Bi, va, SIMD_BYTES * 4); + fma52lo_mem(r5, r5, Bi, va, SIMD_BYTES * 5); + fma52lo_mem(r6, r6, Bi, va, SIMD_BYTES * 6); + fma52lo_mem(r7, r7, Bi, va, SIMD_BYTES * 7); + + fma52lo_mem(r0, r0, Yi, p384_mb, SIMD_BYTES * 0); + fma52lo_mem(r1, r1, Yi, p384_mb, SIMD_BYTES * 1); + r1 = add64(r1, srli64(r0, DIGIT_SIZE)); /* carry propagation */ + fma52lo_mem(r2, r2, Yi, p384_mb, SIMD_BYTES * 2); + r3 = add64(r3, LO); /* fma52lo_mem(r3, r3, Yi, p384_mb, SIMD_BYTES * 3); */ + r4 = add64(r4, LO); /* fma52lo_mem(r4, r4, Yi, p384_mb, SIMD_BYTES * 4); */ + r5 = add64(r5, LO); /* fma52lo_mem(r5, r5, Yi, p384_mb, SIMD_BYTES * 5); */ + r6 = add64(r6, LO); /* fma52lo_mem(r6, r6, Yi, p384_mb, SIMD_BYTES * 6); */ + fma52lo_mem(r7, r7, Yi, p384_mb, SIMD_BYTES * 7); + + fma52hi_mem(r0, r1, Bi, va, SIMD_BYTES * 0); + fma52hi_mem(r1, r2, Bi, va, SIMD_BYTES * 1); + fma52hi_mem(r2, r3, Bi, va, SIMD_BYTES * 2); + fma52hi_mem(r3, r4, Bi, va, SIMD_BYTES * 3); + fma52hi_mem(r4, r5, Bi, va, SIMD_BYTES * 4); + fma52hi_mem(r5, r6, Bi, va, SIMD_BYTES * 5); + fma52hi_mem(r6, r7, Bi, va, SIMD_BYTES * 6); + fma52hi_mem(r7, get_zero64(), Bi, va, SIMD_BYTES * 7); + + fma52hi_mem(r0, r0, Yi, p384_mb, SIMD_BYTES * 0); + fma52hi_mem(r1, r1, Yi, p384_mb, SIMD_BYTES * 1); + fma52hi_mem(r2, r2, Yi, p384_mb, SIMD_BYTES * 2); + r3 = add64(r3, HI); /* fma52hi_mem(r3, r3, Yi, p384_mb, SIMD_BYTES * 3); */ + r4 = add64(r4, HI); /* fma52hi_mem(r4, r4, Yi, p384_mb, SIMD_BYTES * 4); */ + r5 = add64(r5, HI); /* fma52hi_mem(r5, r5, Yi, p384_mb, SIMD_BYTES * 5); */ + r6 = add64(r6, HI); /* fma52hi_mem(r6, r6, Yi, p384_mb, SIMD_BYTES * 6); */ + fma52hi_mem(r7, r7, Yi, p384_mb, SIMD_BYTES * 7); + } + + // normalization + NORM_LSHIFTR(r, 0, 1) + NORM_LSHIFTR(r, 1, 2) + NORM_LSHIFTR(r, 2, 3) + NORM_LSHIFTR(r, 3, 4) + NORM_LSHIFTR(r, 4, 5) + NORM_LSHIFTR(r, 5, 6) + NORM_LSHIFTR(r, 6, 7) + + r[0] = r0; + r[1] = r1; + r[2] = r2; + r[3] = r3; + r[4] = r4; + r[5] = r5; + r[6] = r6; + r[7] = r7; +} + +void MB_FUNC_NAME(ifma_ams52_p384_)(U64 r[], const U64 va[]) +{ + const U64* vb = va; + U64* modulus = (U64*)p384_mb; + + U64 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15; + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = + r10 = r11 = r12 = r13 = r14 = r15 = get_zero64(); + + // full square + ROUND_MUL(0, 1, r1, r2) + ROUND_MUL(0, 2, r2, r3) + ROUND_MUL(0, 3, r3, r4) + ROUND_MUL(0, 4, r4, r5) + ROUND_MUL(0, 5, r5, r6) + ROUND_MUL(0, 6, r6, r7) + ROUND_MUL(0, 7, r7, r8) + + ROUND_MUL(1, 2, r3, r4) + ROUND_MUL(1, 3, r4, r5) + ROUND_MUL(1, 4, r5, r6) + ROUND_MUL(1, 5, r6, r7) + ROUND_MUL(1, 6, r7, r8) + ROUND_MUL(1, 7, r8, r9) + + ROUND_MUL(2, 3, r5, r6) + ROUND_MUL(2, 4, r6, r7) + ROUND_MUL(2, 5, r7, r8) + ROUND_MUL(2, 6, r8, r9) + ROUND_MUL(2, 7, r9, r10) + + ROUND_MUL(3, 4, r7, r8) + ROUND_MUL(3, 5, r8, r9) + ROUND_MUL(3, 6, r9, r10) + ROUND_MUL(3, 7, r10,r11) + + ROUND_MUL(4, 5, r9, r10) + ROUND_MUL(4, 6, r10,r11) + ROUND_MUL(4, 7, r11,r12) + + ROUND_MUL(5, 6, r11,r12) + ROUND_MUL(5, 7, r12,r13) + + ROUND_MUL(6, 7, r13,r14) + + r1 = add64(r1, r1); + r2 = add64(r2, r2); + r3 = add64(r3, r3); + r4 = add64(r4, r4); + r5 = add64(r5, r5); + r6 = add64(r6, r6); + r7 = add64(r7, r7); + r8 = add64(r8, r8); + r9 = add64(r9, r9); + r10 = add64(r10, r10); + r11 = add64(r11, r11); + r12 = add64(r12, r12); + r13 = add64(r13, r13); + r14 = add64(r14, r14); + + U64 u; + ROUND_MUL(0, 0, r0, r1) + u = and64(add64(r0, slli64(r0, 32)), loadu64(VMASK52)); + ROUND_MUL(1, 1, r2, r3) + ROUND_MUL(2, 2, r4, r5) + ROUND_MUL(3, 3, r6, r7) + ROUND_MUL(4, 4, r8, r9) + ROUND_MUL(5, 5, r10, r11) + ROUND_MUL(6, 6, r12, r13) + ROUND_MUL(7, 7, r14, r15) + + // reduction + P384_REDUCTION_ROUND(u, r0, r1, r2, r3, r4, r5, r6, r7, r8, modulus); + P384_REDUCTION_ROUND(u, r1, r2, r3, r4, r5, r6, r7, r8, r9, modulus); + P384_REDUCTION_ROUND(u, r2, r3, r4, r5, r6, r7, r8, r9, r10, modulus); + P384_REDUCTION_ROUND(u, r3, r4, r5, r6, r7, r8, r9, r10,r11, modulus); + P384_REDUCTION_ROUND(u, r4, r5, r6, r7, r8, r9, r10,r11,r12, modulus); + P384_REDUCTION_ROUND(u, r5, r6, r7, r8, r9, r10,r11,r12,r13, modulus); + P384_REDUCTION_ROUND(u, r6, r7, r8, r9, r10,r11,r12,r13,r14, modulus); + P384_REDUCTION_ROUND(u, r7, r8, r9, r10,r11,r12,r13,r14,r15, modulus); + + // normalization + NORM_LSHIFTR(r, 8, 9) + NORM_LSHIFTR(r, 9, 10) + NORM_LSHIFTR(r, 10, 11) + NORM_LSHIFTR(r, 11, 12) + NORM_LSHIFTR(r, 12, 13) + NORM_LSHIFTR(r, 13, 14) + NORM_LSHIFTR(r, 14, 15) + + r[0] = r8; + r[1] = r9; + r[2] = r10; + r[3] = r11; + r[4] = r12; + r[5] = r13; + r[6] = r14; + r[7] = r15; +} + +void MB_FUNC_NAME(ifma_reduce52_p384_)(U64 R[], const U64 A[]) +{ + /* r = a - p384_mb */ + U64 r0 = sub64(A[0], ((U64*)(p384_mb))[0]); + U64 r1 = sub64(A[1], ((U64*)(p384_mb))[1]); + U64 r2 = sub64(A[2], ((U64*)(p384_mb))[2]); + U64 r3 = sub64(A[3], ((U64*)(p384_mb))[3]); + U64 r4 = sub64(A[4], ((U64*)(p384_mb))[4]); + U64 r5 = sub64(A[5], ((U64*)(p384_mb))[5]); + U64 r6 = sub64(A[6], ((U64*)(p384_mb))[6]); + U64 r7 = sub64(A[7], ((U64*)(p384_mb))[7]); + + /* normalize r0 - r7 */ + NORM_ASHIFTR(r, 0, 1) + NORM_ASHIFTR(r, 1, 2) + NORM_ASHIFTR(r, 2, 3) + NORM_ASHIFTR(r, 3, 4) + NORM_ASHIFTR(r, 4, 5) + NORM_ASHIFTR(r, 5, 6) + NORM_ASHIFTR(r, 6, 7) + + /* r = a r = z^(0xFFFFFFFFFFFFFFFF FFFFFFFFFFFFFFFF FFFFFFFFFFFFFFFF FFFFFFFFFFFFFFFE FFFFFFFF00000000 00000000FFFFFFFD) +// +// note: z in in Montgomery domain (as soon mul() and sqr() below are amm-functions +// r in Montgomery domain too +*/ +#define fe52_sqr MB_FUNC_NAME(ifma_ams52_p384_) +#define fe52_mul MB_FUNC_NAME(ifma_amm52_p384_) + +/* r = base^(2^n) */ +__INLINE void fe52_sqr_pwr(U64 r[], const U64 base[], int n) +{ + if(r!=base) + MB_FUNC_NAME(mov_FE384_)(r, base); + for(; n>0; n--) + fe52_sqr(r,r); +} + +void MB_FUNC_NAME(ifma_aminv52_p384_)(U64 r[], const U64 z[]) +{ + __ALIGN64 U64 u[P384_LEN52]; + __ALIGN64 U64 v[P384_LEN52]; + __ALIGN64 U64 zD[P384_LEN52]; + __ALIGN64 U64 zE[P384_LEN52]; + __ALIGN64 U64 zF[P384_LEN52]; + + fe52_sqr(u, z); /* u = z^2 */ + fe52_mul(v, u, z); /* v = z^2 * z = z^3 */ + fe52_sqr_pwr(zF, v, 2); /* zF= (z^3)^(2^2) = z^12 */ + + fe52_mul(zD, zF, z); /* zD = z^12 * z = z^xD */ + fe52_mul(zE, zF, u); /* zE = z^12 * z^2 = z^xE */ + fe52_mul(zF, zF, v); /* zF = z^12 * z^3 = z^xF */ + + fe52_sqr_pwr(u, zF, 4); /* u = (z^xF)^(2^4) = z^xF0 */ + fe52_mul(zD, u, zD); /* zD = z^xF0 * z^xD = z^xFD */ + fe52_mul(zE, u, zE); /* zE = z^xF0 * z^xE = z^xFE */ + fe52_mul(zF, u, zF); /* zF = z^xF0 * z^xF = z^xFF */ + + fe52_sqr_pwr(u, zF, 8); /* u = (z^xFF)^(2^8) = z^xFF00 */ + fe52_mul(zD, u, zD); /* zD = z^xFF00 * z^xFD = z^xFFFD */ + fe52_mul(zE, u, zE); /* zE = z^xFF00 * z^xFE = z^xFFFE */ + fe52_mul(zF, u, zF); /* zF = z^xFF00 * z^xFF = z^xFFFF */ + + fe52_sqr_pwr(u, zF, 16); /* u = (z^xFFFF)^(2^16) = z^xFFFF0000 */ + fe52_mul(zD, u, zD); /* zD = z^xFFFF0000 * z^xFFFD = z^xFFFFFFFD */ + fe52_mul(zE, u, zE); /* zE = z^xFFFF0000 * z^xFFFE = z^xFFFFFFFE */ + fe52_mul(zF, u, zF); /* zF = z^xFFFF0000 * z^xFFFF = z^xFFFFFFFF */ + + fe52_sqr_pwr(u, zF, 32); /* u = (z^xFFFFFFFF)^(2^32) = z^xFFFFFFFF00000000 */ + fe52_mul(zE, u, zE); /* zE = z^xFFFFFFFF00000000 * z^xFFFFFFFE = z^xFFFFFFFFFFFFFFFE */ + fe52_mul(zF, u, zF); /* zF = z^xFFFFFFFF00000000 * z^xFFFFFFFF = z^xFFFFFFFFFFFFFFFF */ + + /* v = z^xFFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFF + = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF */ + fe52_sqr_pwr(v, zF, 64); + fe52_mul(v, v, zF); + /* v = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFF + = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFFF */ + fe52_sqr_pwr(v, v, 64); + fe52_mul(v, v, zF); + /* v = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFE + = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE */ + fe52_sqr_pwr(v, v, 64); + fe52_mul(v, v, zE); + /* v = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE.0000000000000000 * z^xFFFFFFFF00000000 + = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE.FFFFFFFF00000000 */ + fe52_sqr_pwr(v, v, 64); + fe52_mul(v, v, u); + /* r = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE.FFFFFFFF00000000.0000000000000000 * z^xFFFFFFFD + = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE.FFFFFFFF00000000.00000000FFFFFFFD */ + fe52_sqr_pwr(v, v, 64); + fe52_mul(r, v, zD); +} + +/*===================================================================== + + Specialized single operations over p384: add, sub, neg + +=====================================================================*/ +__INLINE __mb_mask MB_FUNC_NAME(lt_mbx_digit_)(const U64 a, const U64 b, const __mb_mask lt_mask) +{ + U64 d = mask_sub64(sub64(a, b), lt_mask, sub64(a, b), set1(1)); + return cmp64_mask(d, get_zero64(), _MM_CMPINT_LT); +} + +void MB_FUNC_NAME(ifma_add52_p384_)(U64 R[], const U64 A[], const U64 B[]) +{ + /* r = a + b */ + U64 r0 = add64(A[0], B[0]); + U64 r1 = add64(A[1], B[1]); + U64 r2 = add64(A[2], B[2]); + U64 r3 = add64(A[3], B[3]); + U64 r4 = add64(A[4], B[4]); + U64 r5 = add64(A[5], B[5]); + U64 r6 = add64(A[6], B[6]); + U64 r7 = add64(A[7], B[7]); + + /* lt = {r0 - r7} < 2*p */ + __mb_mask + lt = MB_FUNC_NAME(lt_mbx_digit_)( r0, ((U64*)(p384_x2))[0], 0); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r1, ((U64*)(p384_x2))[1], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r2, ((U64*)(p384_x2))[2], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r3, ((U64*)(p384_x2))[3], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r4, ((U64*)(p384_x2))[4], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r5, ((U64*)(p384_x2))[5], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r6, ((U64*)(p384_x2))[6], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r7, ((U64*)(p384_x2))[7], lt); + + /* {r0 - r7} -= 2*p */ + r0 = mask_sub64(r0, ~lt, r0, ((U64*)(p384_x2))[0]); + r1 = mask_sub64(r1, ~lt, r1, ((U64*)(p384_x2))[1]); + r2 = mask_sub64(r2, ~lt, r2, ((U64*)(p384_x2))[2]); + r3 = mask_sub64(r3, ~lt, r3, ((U64*)(p384_x2))[3]); + r4 = mask_sub64(r4, ~lt, r4, ((U64*)(p384_x2))[4]); + r5 = mask_sub64(r5, ~lt, r5, ((U64*)(p384_x2))[5]); + r6 = mask_sub64(r6, ~lt, r6, ((U64*)(p384_x2))[6]); + r7 = mask_sub64(r7, ~lt, r7, ((U64*)(p384_x2))[7]); + + /* normalize r0 - r7 */ + NORM_ASHIFTR(r, 0,1) + NORM_ASHIFTR(r, 1,2) + NORM_ASHIFTR(r, 2,3) + NORM_ASHIFTR(r, 3,4) + NORM_ASHIFTR(r, 4,5) + NORM_ASHIFTR(r, 5,6) + NORM_ASHIFTR(r, 6,7) + + R[0] = r0; + R[1] = r1; + R[2] = r2; + R[3] = r3; + R[4] = r4; + R[5] = r5; + R[6] = r6; + R[7] = r7; +} + +void MB_FUNC_NAME(ifma_sub52_p384_)(U64 R[], const U64 A[], const U64 B[]) +{ + /* r = a - b */ + U64 r0 = sub64(A[0], B[0]); + U64 r1 = sub64(A[1], B[1]); + U64 r2 = sub64(A[2], B[2]); + U64 r3 = sub64(A[3], B[3]); + U64 r4 = sub64(A[4], B[4]); + U64 r5 = sub64(A[5], B[5]); + U64 r6 = sub64(A[6], B[6]); + U64 r7 = sub64(A[7], B[7]); + + /* lt = {r0 - r7} < 0 */ + __mb_mask + lt = MB_FUNC_NAME(lt_mbx_digit_)(r0, get_zero64(), 0); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r1, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r2, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r3, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r4, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r5, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r6, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r7, get_zero64(), lt); + + r0 = mask_add64(r0, lt, r0, ((U64*)(p384_x2))[0]); + r1 = mask_add64(r1, lt, r1, ((U64*)(p384_x2))[1]); + r2 = mask_add64(r2, lt, r2, ((U64*)(p384_x2))[2]); + r3 = mask_add64(r3, lt, r3, ((U64*)(p384_x2))[3]); + r4 = mask_add64(r4, lt, r4, ((U64*)(p384_x2))[4]); + r5 = mask_add64(r5, lt, r5, ((U64*)(p384_x2))[5]); + r6 = mask_add64(r6, lt, r6, ((U64*)(p384_x2))[6]); + r7 = mask_add64(r7, lt, r7, ((U64*)(p384_x2))[7]); + + /* normalize r0 - r7 */ + NORM_ASHIFTR(r, 0,1) + NORM_ASHIFTR(r, 1,2) + NORM_ASHIFTR(r, 2,3) + NORM_ASHIFTR(r, 3,4) + NORM_ASHIFTR(r, 4,5) + NORM_ASHIFTR(r, 5,6) + NORM_ASHIFTR(r, 6,7) + + R[0] = r0; + R[1] = r1; + R[2] = r2; + R[3] = r3; + R[4] = r4; + R[5] = r5; + R[6] = r6; + R[7] = r7; +} + +void MB_FUNC_NAME(ifma_neg52_p384_)(U64 R[], const U64 A[]) +{ + __mb_mask nz_mask = ~MB_FUNC_NAME(is_zero_FE384_)(A); + + /* {r0 - r7} = 2*p - A */ + U64 r0 = mask_sub64( A[0], nz_mask, ((U64*)(p384_x2))[0], A[0] ); + U64 r1 = mask_sub64( A[0], nz_mask, ((U64*)(p384_x2))[1], A[1] ); + U64 r2 = mask_sub64( A[0], nz_mask, ((U64*)(p384_x2))[2], A[2] ); + U64 r3 = mask_sub64( A[0], nz_mask, ((U64*)(p384_x2))[3], A[3] ); + U64 r4 = mask_sub64( A[0], nz_mask, ((U64*)(p384_x2))[4], A[4] ); + U64 r5 = mask_sub64( A[0], nz_mask, ((U64*)(p384_x2))[5], A[5] ); + U64 r6 = mask_sub64( A[0], nz_mask, ((U64*)(p384_x2))[6], A[6] ); + U64 r7 = mask_sub64( A[0], nz_mask, ((U64*)(p384_x2))[7], A[7] ); + + /* normalize r0 - r7 */ + NORM_ASHIFTR(r, 0,1) + NORM_ASHIFTR(r, 1,2) + NORM_ASHIFTR(r, 2,3) + NORM_ASHIFTR(r, 3,4) + NORM_ASHIFTR(r, 4,5) + NORM_ASHIFTR(r, 5,6) + NORM_ASHIFTR(r, 6,7) + + R[0] = r0; + R[1] = r1; + R[2] = r2; + R[3] = r3; + R[4] = r4; + R[5] = r5; + R[6] = r6; + R[7] = r7; +} + +void MB_FUNC_NAME(ifma_double52_p384_)(U64 R[], const U64 A[]) +{ + MB_FUNC_NAME(ifma_add52_p384_)(R, A, A); +} + +void MB_FUNC_NAME(ifma_tripple52_p384_)(U64 R[], const U64 A[]) +{ + U64 T[P384_LEN52]; + MB_FUNC_NAME(ifma_add52_p384_)(T, A, A); + MB_FUNC_NAME(ifma_add52_p384_)(R, T, A); +} + +void MB_FUNC_NAME(ifma_half52_p384_)(U64 R[], const U64 A[]) +{ + U64 one = set1(1); + U64 base = set1(DIGIT_BASE); + + /* res = a + is_odd(a)? p384 : 0 */ + U64 mask = sub64(get_zero64(), and64(A[0], one)); + U64 t0 = add64(A[0], and64(((U64*)p384_mb)[0], mask)); + U64 t1 = add64(A[1], and64(((U64*)p384_mb)[1], mask)); + U64 t2 = add64(A[2], and64(((U64*)p384_mb)[2], mask)); + U64 t3 = add64(A[3], and64(((U64*)p384_mb)[3], mask)); + U64 t4 = add64(A[4], and64(((U64*)p384_mb)[4], mask)); + U64 t5 = add64(A[5], and64(((U64*)p384_mb)[5], mask)); + U64 t6 = add64(A[6], and64(((U64*)p384_mb)[6], mask)); + U64 t7 = add64(A[7], and64(((U64*)p384_mb)[7], mask)); + + /* t =>> 1 */ + mask = sub64(get_zero64(), and64(t1, one)); + t0 = add64(t0, and64(base, mask)); + t0 = srli64(t0, 1); + + mask = sub64(get_zero64(), and64(t2, one)); + t1 = add64(t1, and64(base, mask)); + t1 = srli64(t1, 1); + + mask = sub64(get_zero64(), and64(t3, one)); + t2 = add64(t2, and64(base, mask)); + t2 = srli64(t2, 1); + + mask = sub64(get_zero64(), and64(t4, one)); + t3 = add64(t3, and64(base, mask)); + t3 = srli64(t3, 1); + + mask = sub64(get_zero64(), and64(t5, one)); + t4 = add64(t4, and64(base, mask)); + t4 = srli64(t4, 1); + + mask = sub64(get_zero64(), and64(t6, one)); + t5 = add64(t5, and64(base, mask)); + t5 = srli64(t5, 1); + + mask = sub64(get_zero64(), and64(t7, one)); + t6 = add64(t6, and64(base, mask)); + t6 = srli64(t6, 1); + + t7 = srli64(t7, 1); + + /* normalize t0, t1, t2, t3, t4 */ + NORM_LSHIFTR(t, 0,1) + NORM_LSHIFTR(t, 1,2) + NORM_LSHIFTR(t, 2,3) + NORM_LSHIFTR(t, 3,4) + NORM_LSHIFTR(t, 4,5) + NORM_LSHIFTR(t, 5,6) + NORM_LSHIFTR(t, 6,7) + + R[0] = t0; + R[1] = t1; + R[2] = t2; + R[3] = t3; + R[4] = t4; + R[5] = t5; + R[6] = t6; + R[7] = t7; +} + +__mb_mask MB_FUNC_NAME(ifma_cmp_lt_p384_)(const U64 a[]) +{ + return MB_FUNC_NAME(cmp_lt_FE384_)(a,(const U64 (*))p384_mb); +} + +__mb_mask MB_FUNC_NAME(ifma_check_range_p384_)(const U64 A[]) +{ + __mb_mask + mask = MB_FUNC_NAME(is_zero_FE384_)(A); + mask |= ~MB_FUNC_NAME(ifma_cmp_lt_p384_)(A); + + return mask; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_arith_p521.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_arith_p521.c new file mode 100644 index 0000000..48aca53 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_arith_p521.c @@ -0,0 +1,870 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +/* Constants */ + +/* +// p521 = 2^521 - 1 +// in 2^52 radix +*/ +__ALIGN64 static const int64u p521_mb[P521_LEN52][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x0000000000000001) } +}; + +/* 2*p521 */ +__ALIGN64 static const int64u p521_x2[P521_LEN52][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(0x000ffffffffffffe) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x0000000000000003) } +}; + +/* +// Note that +// k0 = -(1/p521 mod 2^DIGIT_SIZE) equals 1 +// The implementation takes this fact into account +*/ + +/* to Montgomery conversion constant +// rr = 2^((P521_LEN52*DIGIT_SIZE)*2) mod p521 +*/ +__ALIGN64 static const int64u p521_rr_mb[P521_LEN52][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0004000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) } +}; + + +/*===================================================================== + + Specialized operations over p521: sqr & mul + +=====================================================================*/ + +#define ROUND_MUL(I, J, LO, HI) \ + LO = fma52lo(LO, va[I], vb[J]); \ + HI = fma52hi(HI, va[I], vb[J]) + +#define RED_P521_STEP(u, r0,r1, r10) { \ + U64 u = and64(r0, loadu64(VMASK52)); /* u = r0*k, k == 1 */ \ + r1 = add64(r1, srli64(r0, DIGIT_SIZE)); /* carry propagation */ \ + /* reduction */ \ + u = add64(u, u); \ + r10 = add64(r10, u); \ +} + +void MB_FUNC_NAME(ifma_amm52_p521_)(U64 r[], const U64 va[], const U64 vb[]) +{ + U64 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9; + U64 r10, r11, r12, r13, r14, r15, r16, r17, r18, r19; + U64 r20, r21; + + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = + r10 = r11 = r12 = r13 = r14 = r15 = r16 = r17 = r18 = r19= + r20 = r21 = get_zero64(); + + // full multiplication + ROUND_MUL(0, 0, r0, r1); + ROUND_MUL(1, 0, r1, r2); + ROUND_MUL(2, 0, r2, r3); + ROUND_MUL(3, 0, r3, r4); + ROUND_MUL(4, 0, r4, r5); + ROUND_MUL(5, 0, r5, r6); + ROUND_MUL(6, 0, r6, r7); + ROUND_MUL(7, 0, r7, r8); + ROUND_MUL(8, 0, r8, r9); + ROUND_MUL(9, 0, r9, r10); + ROUND_MUL(10,0, r10,r11); + + ROUND_MUL(0, 1, r1, r2); + ROUND_MUL(1, 1, r2, r3); + ROUND_MUL(2, 1, r3, r4); + ROUND_MUL(3, 1, r4, r5); + ROUND_MUL(4, 1, r5, r6); + ROUND_MUL(5, 1, r6, r7); + ROUND_MUL(6, 1, r7, r8); + ROUND_MUL(7, 1, r8, r9); + ROUND_MUL(8, 1, r9, r10); + ROUND_MUL(9, 1, r10,r11); + ROUND_MUL(10,1, r11,r12); + + ROUND_MUL(0, 2, r2, r3); + ROUND_MUL(1, 2, r3, r4); + ROUND_MUL(2, 2, r4, r5); + ROUND_MUL(3, 2, r5, r6); + ROUND_MUL(4, 2, r6, r7); + ROUND_MUL(5, 2, r7, r8); + ROUND_MUL(6, 2, r8, r9); + ROUND_MUL(7, 2, r9, r10); + ROUND_MUL(8, 2, r10,r11); + ROUND_MUL(9, 2, r11,r12); + ROUND_MUL(10,2, r12,r13); + + ROUND_MUL(0, 3, r3, r4); + ROUND_MUL(1, 3, r4, r5); + ROUND_MUL(2, 3, r5, r6); + ROUND_MUL(3, 3, r6, r7); + ROUND_MUL(4, 3, r7, r8); + ROUND_MUL(5, 3, r8, r9); + ROUND_MUL(6, 3, r9, r10); + ROUND_MUL(7, 3, r10,r11); + ROUND_MUL(8, 3, r11,r12); + ROUND_MUL(9, 3, r12,r13); + ROUND_MUL(10,3, r13,r14); + + ROUND_MUL(0, 4, r4, r5); + ROUND_MUL(1, 4, r5, r6); + ROUND_MUL(2, 4, r6, r7); + ROUND_MUL(3, 4, r7, r8); + ROUND_MUL(4, 4, r8, r9); + ROUND_MUL(5, 4, r9, r10); + ROUND_MUL(6, 4, r10,r11); + ROUND_MUL(7, 4, r11,r12); + ROUND_MUL(8, 4, r12,r13); + ROUND_MUL(9, 4, r13,r14); + ROUND_MUL(10,4, r14,r15); + + ROUND_MUL(0, 5, r5, r6); + ROUND_MUL(1, 5, r6, r7); + ROUND_MUL(2, 5, r7, r8); + ROUND_MUL(3, 5, r8, r9); + ROUND_MUL(4, 5, r9, r10); + ROUND_MUL(5, 5, r10,r11); + ROUND_MUL(6, 5, r11,r12); + ROUND_MUL(7, 5, r12,r13); + ROUND_MUL(8, 5, r13,r14); + ROUND_MUL(9, 5, r14,r15); + ROUND_MUL(10,5, r15,r16); + + ROUND_MUL(0, 6, r6, r7); + ROUND_MUL(1, 6, r7, r8); + ROUND_MUL(2, 6, r8, r9); + ROUND_MUL(3, 6, r9, r10); + ROUND_MUL(4, 6, r10,r11); + ROUND_MUL(5, 6, r11,r12); + ROUND_MUL(6, 6, r12,r13); + ROUND_MUL(7, 6, r13,r14); + ROUND_MUL(8, 6, r14,r15); + ROUND_MUL(9, 6, r15,r16); + ROUND_MUL(10,6, r16,r17); + + ROUND_MUL(0, 7, r7, r8); + ROUND_MUL(1, 7, r8, r9); + ROUND_MUL(2, 7, r9, r10); + ROUND_MUL(3, 7, r10,r11); + ROUND_MUL(4, 7, r11,r12); + ROUND_MUL(5, 7, r12,r13); + ROUND_MUL(6, 7, r13,r14); + ROUND_MUL(7, 7, r14,r15); + ROUND_MUL(8, 7, r15,r16); + ROUND_MUL(9, 7, r16,r17); + ROUND_MUL(10,7, r17,r18); + + ROUND_MUL(0, 8, r8, r9); + ROUND_MUL(1, 8, r9, r10); + ROUND_MUL(2, 8, r10,r11); + ROUND_MUL(3, 8, r11,r12); + ROUND_MUL(4, 8, r12,r13); + ROUND_MUL(5, 8, r13,r14); + ROUND_MUL(6, 8, r14,r15); + ROUND_MUL(7, 8, r15,r16); + ROUND_MUL(8, 8, r16,r17); + ROUND_MUL(9, 8, r17,r18); + ROUND_MUL(10,8, r18,r19); + + ROUND_MUL(0, 9, r9, r10); + ROUND_MUL(1, 9, r10,r11); + ROUND_MUL(2, 9, r11,r12); + ROUND_MUL(3, 9, r12,r13); + ROUND_MUL(4, 9, r13,r14); + ROUND_MUL(5, 9, r14,r15); + ROUND_MUL(6, 9, r15,r16); + ROUND_MUL(7, 9, r16,r17); + ROUND_MUL(8, 9, r17,r18); + ROUND_MUL(9, 9, r18,r19); + ROUND_MUL(10,9, r19,r20); + + ROUND_MUL(0, 10, r10,r11); + ROUND_MUL(1, 10, r11,r12); + ROUND_MUL(2, 10, r12,r13); + ROUND_MUL(3, 10, r13,r14); + ROUND_MUL(4, 10, r14,r15); + ROUND_MUL(5, 10, r15,r16); + ROUND_MUL(6, 10, r16,r17); + ROUND_MUL(7, 10, r17,r18); + ROUND_MUL(8, 10, r18,r19); + ROUND_MUL(9, 10, r19,r20); + ROUND_MUL(10,10, r20,r21); + + // reduction p = 2^521 -1 + RED_P521_STEP(u0, r0, r1, r10); + RED_P521_STEP(u1, r1, r2, r11); + RED_P521_STEP(u2, r2, r3, r12); + RED_P521_STEP(u3, r3, r4, r13); + RED_P521_STEP(u4, r4, r5, r14); + RED_P521_STEP(u5, r5, r6, r15); + RED_P521_STEP(u6, r6, r7, r16); + RED_P521_STEP(u7, r7, r8, r17); + RED_P521_STEP(u8, r8, r9, r18); + RED_P521_STEP(u9, r9, r10,r19); + RED_P521_STEP(u10,r10,r11,r20); + + // normalization + NORM_LSHIFTR(r, 11, 12) + NORM_LSHIFTR(r, 12, 13) + NORM_LSHIFTR(r, 13, 14) + NORM_LSHIFTR(r, 14, 15) + NORM_LSHIFTR(r, 15, 16) + NORM_LSHIFTR(r, 16, 17) + NORM_LSHIFTR(r, 17, 18) + NORM_LSHIFTR(r, 18, 19) + NORM_LSHIFTR(r, 19, 20) + NORM_LSHIFTR(r, 20, 21) + + r[0] = r11; + r[1] = r12; + r[2] = r13; + r[3] = r14; + r[4] = r15; + r[5] = r16; + r[6] = r17; + r[7] = r18; + r[8] = r19; + r[9] = r20; + r[10]= r21; +} + +void MB_FUNC_NAME(ifma_ams52_p521_)(U64 r[], const U64 va[]) +{ + const U64* vb = va; + + U64 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9; + U64 r10, r11, r12, r13, r14, r15, r16, r17, r18, r19; + U64 r20, r21; + + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = r10 = + r11 = r12 = r13 = r14 = r15 = r16 = r17 = r18 = r19 = r20 = r21 = get_zero64(); + + // full square + ROUND_MUL(0, 1, r1, r2); + ROUND_MUL(0, 2, r2, r3); + ROUND_MUL(0, 3, r3, r4); + ROUND_MUL(0, 4, r4, r5); + ROUND_MUL(0, 5, r5, r6); + ROUND_MUL(0, 6, r6, r7); + ROUND_MUL(0, 7, r7, r8); + ROUND_MUL(0, 8, r8, r9); + ROUND_MUL(0, 9, r9, r10); + ROUND_MUL(0, 10,r10,r11); + + ROUND_MUL(1, 2, r3, r4); + ROUND_MUL(1, 3, r4, r5); + ROUND_MUL(1, 4, r5, r6); + ROUND_MUL(1, 5, r6, r7); + ROUND_MUL(1, 6, r7, r8); + ROUND_MUL(1, 7, r8, r9); + ROUND_MUL(1, 8, r9, r10); + ROUND_MUL(1, 9, r10,r11); + ROUND_MUL(1, 10,r11,r12); + + ROUND_MUL(2, 3, r5, r6); + ROUND_MUL(2, 4, r6, r7); + ROUND_MUL(2, 5, r7, r8); + ROUND_MUL(2, 6, r8, r9); + ROUND_MUL(2, 7, r9, r10); + ROUND_MUL(2, 8, r10,r11); + ROUND_MUL(2, 9, r11,r12); + ROUND_MUL(2, 10,r12,r13); + + ROUND_MUL(3, 4, r7, r8); + ROUND_MUL(3, 5, r8, r9); + ROUND_MUL(3, 6, r9, r10); + ROUND_MUL(3, 7, r10,r11); + ROUND_MUL(3, 8, r11,r12); + ROUND_MUL(3, 9, r12,r13); + ROUND_MUL(3, 10,r13,r14); + + ROUND_MUL(4, 5, r9, r10); + ROUND_MUL(4, 6, r10,r11); + ROUND_MUL(4, 7, r11,r12); + ROUND_MUL(4, 8, r12,r13); + ROUND_MUL(4, 9, r13,r14); + ROUND_MUL(4, 10,r14,r15); + + ROUND_MUL(5, 6, r11,r12); + ROUND_MUL(5, 7, r12,r13); + ROUND_MUL(5, 8, r13,r14); + ROUND_MUL(5, 9, r14,r15); + ROUND_MUL(5, 10,r15,r16); + + ROUND_MUL(6, 7, r13,r14); + ROUND_MUL(6, 8, r14,r15); + ROUND_MUL(6, 9, r15,r16); + ROUND_MUL(6, 10,r16,r17); + + ROUND_MUL(7, 8, r15,r16); + ROUND_MUL(7, 9, r16,r17); + ROUND_MUL(7, 10,r17,r18); + + ROUND_MUL(8, 9, r17,r18); + ROUND_MUL(8, 10,r18,r19); + + ROUND_MUL(9, 10,r19,r20); + + r1 = add64(r1, r1); + r2 = add64(r2, r2); + r3 = add64(r3, r3); + r4 = add64(r4, r4); + r5 = add64(r5, r5); + r6 = add64(r6, r6); + r7 = add64(r7, r7); + r8 = add64(r8, r8); + r9 = add64(r9, r9); + r10 = add64(r10, r10); + r11 = add64(r11, r11); + r12 = add64(r12, r12); + r13 = add64(r13, r13); + r14 = add64(r14, r14); + r15 = add64(r15, r15); + r16 = add64(r16, r16); + r17 = add64(r17, r17); + r18 = add64(r18, r18); + r19 = add64(r19, r19); + r20 = add64(r20, r20); + + ROUND_MUL(0, 0, r0, r1); + ROUND_MUL(1, 1, r2, r3); + ROUND_MUL(2, 2, r4, r5); + ROUND_MUL(3, 3, r6, r7); + ROUND_MUL(4, 4, r8, r9); + ROUND_MUL(5, 5, r10, r11); + ROUND_MUL(6, 6, r12, r13); + ROUND_MUL(7, 7, r14, r15); + ROUND_MUL(8, 8, r16, r17); + ROUND_MUL(9, 9, r18, r19); + ROUND_MUL(10,10,r20, r21); + + // reduction p = 2^521 -1 + RED_P521_STEP(u0, r0, r1, r10); + RED_P521_STEP(u1, r1, r2, r11); + RED_P521_STEP(u2, r2, r3, r12); + RED_P521_STEP(u3, r3, r4, r13); + RED_P521_STEP(u4, r4, r5, r14); + RED_P521_STEP(u5, r5, r6, r15); + RED_P521_STEP(u6, r6, r7, r16); + RED_P521_STEP(u7, r7, r8, r17); + RED_P521_STEP(u8, r8, r9, r18); + RED_P521_STEP(u9, r9, r10,r19); + RED_P521_STEP(u10,r10,r11,r20); + + // normalization + NORM_LSHIFTR(r, 11, 12) + NORM_LSHIFTR(r, 12, 13) + NORM_LSHIFTR(r, 13, 14) + NORM_LSHIFTR(r, 14, 15) + NORM_LSHIFTR(r, 15, 16) + NORM_LSHIFTR(r, 16, 17) + NORM_LSHIFTR(r, 17, 18) + NORM_LSHIFTR(r, 18, 19) + NORM_LSHIFTR(r, 19, 20) + NORM_LSHIFTR(r, 20, 21) + + r[0] = r11; + r[1] = r12; + r[2] = r13; + r[3] = r14; + r[4] = r15; + r[5] = r16; + r[6] = r17; + r[7] = r18; + r[8] = r19; + r[9] = r20; + r[10]= r21; +} + +void MB_FUNC_NAME(ifma_reduce52_p521_)(U64 R[], const U64 A[]) +{ + /* r = a - p521_mb */ + U64 r0 = sub64(A[0], ((U64*)(p521_mb))[0]); + U64 r1 = sub64(A[1], ((U64*)(p521_mb))[1]); + U64 r2 = sub64(A[2], ((U64*)(p521_mb))[2]); + U64 r3 = sub64(A[3], ((U64*)(p521_mb))[3]); + U64 r4 = sub64(A[4], ((U64*)(p521_mb))[4]); + U64 r5 = sub64(A[5], ((U64*)(p521_mb))[5]); + U64 r6 = sub64(A[6], ((U64*)(p521_mb))[6]); + U64 r7 = sub64(A[7], ((U64*)(p521_mb))[7]); + U64 r8 = sub64(A[8], ((U64*)(p521_mb))[8]); + U64 r9 = sub64(A[9], ((U64*)(p521_mb))[9]); + U64 r10= sub64(A[10],((U64*)(p521_mb))[10]); + + /* normalize r0 - r10 */ + NORM_ASHIFTR(r, 0, 1) + NORM_ASHIFTR(r, 1, 2) + NORM_ASHIFTR(r, 2, 3) + NORM_ASHIFTR(r, 3, 4) + NORM_ASHIFTR(r, 4, 5) + NORM_ASHIFTR(r, 5, 6) + NORM_ASHIFTR(r, 6, 7) + NORM_ASHIFTR(r, 7, 8) + NORM_ASHIFTR(r, 8, 9) + NORM_ASHIFTR(r, 9, 10) + + /* r = a r = z^(0x1FF FFFFFFFFFFFFFFFF FFFFFFFFFFFFFFFF FFFFFFFFFFFFFFFF FFFFFFFFFFFFFFFF FFFFFFFFFFFFFFFF FFFFFFFFFFFFFFFF FFFFFFFFFFFFFFFF FFFFFFFFFFFFFFFD) +// +// note: z in in Montgomery domain (as soon mul() and sqr() below are amm-functions +// r in Montgomery domain too +*/ +#define fe52_sqr MB_FUNC_NAME(ifma_ams52_p521_) +#define fe52_mul MB_FUNC_NAME(ifma_amm52_p521_) + +/* r = base^(2^n) */ +__INLINE void fe52_sqr_pwr(U64 r[], const U64 base[], int n) +{ + if(r!=base) + MB_FUNC_NAME(mov_FE521_)(r, base); + for(; n>0; n--) + fe52_sqr(r,r); +} + +void MB_FUNC_NAME(ifma_aminv52_p521_)(U64 r[], const U64 z[]) +{ + __ALIGN64 U64 u[P521_LEN52]; + __ALIGN64 U64 v[P521_LEN52]; + __ALIGN64 U64 zD[P521_LEN52]; + __ALIGN64 U64 zF[P521_LEN52]; + __ALIGN64 U64 z1FF[P521_LEN52]; + + fe52_sqr(u, z); /* u = z^2 */ + fe52_mul(v, u, z); /* v = z^2 * z = z^3 */ + fe52_sqr_pwr(zF, v, 2); /* zF= (z^3)^(2^2) = z^12 */ + + fe52_mul(zD, zF, z); /* zD = z^12 * z = z^xD */ + fe52_mul(zF, zF, v); /* zF = z^12 * z^3 = z^xF */ + + fe52_sqr_pwr(u, zF, 4); /* u = (z^xF)^(2^4) = z^xF0 */ + fe52_mul(zD, u, zD); /* zD = z^xF0 * z^xD = z^xFD */ + fe52_mul(zF, u, zF); /* zF = z^xF0 * z^xF = z^xFF */ + + fe52_sqr(z1FF, zF); /* z1FF= (zF^2) = z^x1FE */ + fe52_mul(z1FF, z1FF, z); /* z1FF *= z = z^x1FF */ + + fe52_sqr_pwr(u, zF, 8); /* u = (z^xFF)^(2^8) = z^xFF00 */ + fe52_mul(zD, u, zD); /* zD = z^xFF00 * z^xFD = z^xFFFD */ + fe52_mul(zF, u, zF); /* zF = z^xFF00 * z^xFF = z^xFFFF */ + + fe52_sqr_pwr(u, zF, 16); /* u = (z^xFFFF)^(2^16) = z^xFFFF0000 */ + fe52_mul(zD, u, zD); /* zD = z^xFFFF0000 * z^xFFFD = z^xFFFFFFFD */ + fe52_mul(zF, u, zF); /* zF = z^xFFFF0000 * z^xFFFF = z^xFFFFFFFF */ + + fe52_sqr_pwr(u, zF, 32); /* u = (z^xFFFFFFFF)^(2^32) = z^xFFFFFFFF00000000 */ + fe52_mul(zD, u, zD); /* zD = z^xFFFFFFFF00000000 * z^xFFFFFFFD = z^xFFFFFFFFFFFFFFFD */ + fe52_mul(zF, u, zF); /* zF = z^xFFFFFFFF00000000 * z^xFFFFFFFF = z^xFFFFFFFFFFFFFFFF */ + + /* v = z^x1FF.0000000000000000 * z^xFFFFFFFFFFFFFFFF + = z^x1FF.FFFFFFFFFFFFFFFF */ + fe52_sqr_pwr(v, z1FF, 64); + fe52_mul(v, v, zF); + + /* v = z^x1FF.FFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFF + = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF */ + fe52_sqr_pwr(v, v, 64); + fe52_mul(v, v, zF); + + /* v = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFF + = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF */ + fe52_sqr_pwr(v, v, 64); + fe52_mul(v, v, zF); + + /* v = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFF + = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF */ + fe52_sqr_pwr(v, v, 64); + fe52_mul(v, v, zF); + + /* v = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFF + = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF */ + fe52_sqr_pwr(v, v, 64); + fe52_mul(v, v, zF); + + /* v = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFF + = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF */ + fe52_sqr_pwr(v, v, 64); + fe52_mul(v, v, zF); + + /* v = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFF + = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF */ + fe52_sqr_pwr(v, v, 64); + fe52_mul(v, v, zF); + + /* v = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFD + = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFD */ + fe52_sqr_pwr(v, v, 64); + fe52_mul(r, v, zD); +} + +/*===================================================================== + + Specialized single operations over p521: add, sub, neg + +=====================================================================*/ +__INLINE __mb_mask MB_FUNC_NAME(lt_mbx_digit_)(const U64 a, const U64 b, const __mb_mask lt_mask) +{ + U64 d = mask_sub64(sub64(a, b), lt_mask, sub64(a, b), set1(1)); + return cmp64_mask(d, get_zero64(), _MM_CMPINT_LT); +} + +void MB_FUNC_NAME(ifma_add52_p521_)(U64 R[], const U64 A[], const U64 B[]) +{ + /* r = a + b */ + U64 r0 = add64(A[0], B[0]); + U64 r1 = add64(A[1], B[1]); + U64 r2 = add64(A[2], B[2]); + U64 r3 = add64(A[3], B[3]); + U64 r4 = add64(A[4], B[4]); + U64 r5 = add64(A[5], B[5]); + U64 r6 = add64(A[6], B[6]); + U64 r7 = add64(A[7], B[7]); + U64 r8 = add64(A[8], B[8]); + U64 r9 = add64(A[9], B[9]); + U64 r10= add64(A[10],B[10]); + + /* lt = {r0 - r10} < 2*p */ + __mb_mask + lt = MB_FUNC_NAME(lt_mbx_digit_)( r0, ((U64*)(p521_x2))[0], 0); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r1, ((U64*)(p521_x2))[1], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r2, ((U64*)(p521_x2))[2], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r3, ((U64*)(p521_x2))[3], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r4, ((U64*)(p521_x2))[4], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r5, ((U64*)(p521_x2))[5], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r6, ((U64*)(p521_x2))[6], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r7, ((U64*)(p521_x2))[7], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r8, ((U64*)(p521_x2))[8], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r9, ((U64*)(p521_x2))[9], lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)( r10,((U64*)(p521_x2))[10],lt); + + /* {r0 - r10} -= 2*p */ + r0 = mask_sub64(r0, ~lt, r0, ((U64*)(p521_x2))[0]); + r1 = mask_sub64(r1, ~lt, r1, ((U64*)(p521_x2))[1]); + r2 = mask_sub64(r2, ~lt, r2, ((U64*)(p521_x2))[2]); + r3 = mask_sub64(r3, ~lt, r3, ((U64*)(p521_x2))[3]); + r4 = mask_sub64(r4, ~lt, r4, ((U64*)(p521_x2))[4]); + r5 = mask_sub64(r5, ~lt, r5, ((U64*)(p521_x2))[5]); + r6 = mask_sub64(r6, ~lt, r6, ((U64*)(p521_x2))[6]); + r7 = mask_sub64(r7, ~lt, r7, ((U64*)(p521_x2))[7]); + r8 = mask_sub64(r8, ~lt, r8, ((U64*)(p521_x2))[8]); + r9 = mask_sub64(r9, ~lt, r9, ((U64*)(p521_x2))[9]); + r10= mask_sub64(r10,~lt, r10,((U64*)(p521_x2))[10]); + + /* normalize r0 - r10 */ + NORM_ASHIFTR(r, 0, 1) + NORM_ASHIFTR(r, 1, 2) + NORM_ASHIFTR(r, 2, 3) + NORM_ASHIFTR(r, 3, 4) + NORM_ASHIFTR(r, 4, 5) + NORM_ASHIFTR(r, 5, 6) + NORM_ASHIFTR(r, 6, 7) + NORM_ASHIFTR(r, 7, 8) + NORM_ASHIFTR(r, 8, 9) + NORM_ASHIFTR(r, 9,10) + + R[0] = r0; + R[1] = r1; + R[2] = r2; + R[3] = r3; + R[4] = r4; + R[5] = r5; + R[6] = r6; + R[7] = r7; + R[8] = r8; + R[9] = r9; + R[10]= r10; +} + +void MB_FUNC_NAME(ifma_sub52_p521_)(U64 R[], const U64 A[], const U64 B[]) +{ + /* r = a - b */ + U64 r0 = sub64(A[0], B[0]); + U64 r1 = sub64(A[1], B[1]); + U64 r2 = sub64(A[2], B[2]); + U64 r3 = sub64(A[3], B[3]); + U64 r4 = sub64(A[4], B[4]); + U64 r5 = sub64(A[5], B[5]); + U64 r6 = sub64(A[6], B[6]); + U64 r7 = sub64(A[7], B[7]); + U64 r8 = sub64(A[8], B[8]); + U64 r9 = sub64(A[9], B[9]); + U64 r10= sub64(A[10],B[10]); + + /* lt = {r0 - r10} < 0 */ + __mb_mask + lt = MB_FUNC_NAME(lt_mbx_digit_)(r0, get_zero64(), 0); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r1, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r2, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r3, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r4, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r5, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r6, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r7, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r8, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r9, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r10,get_zero64(), lt); + + r0 = mask_add64(r0, lt, r0, ((U64*)(p521_x2))[0]); + r1 = mask_add64(r1, lt, r1, ((U64*)(p521_x2))[1]); + r2 = mask_add64(r2, lt, r2, ((U64*)(p521_x2))[2]); + r3 = mask_add64(r3, lt, r3, ((U64*)(p521_x2))[3]); + r4 = mask_add64(r4, lt, r4, ((U64*)(p521_x2))[4]); + r5 = mask_add64(r5, lt, r5, ((U64*)(p521_x2))[5]); + r6 = mask_add64(r6, lt, r6, ((U64*)(p521_x2))[6]); + r7 = mask_add64(r7, lt, r7, ((U64*)(p521_x2))[7]); + r8 = mask_add64(r8, lt, r8, ((U64*)(p521_x2))[8]); + r9 = mask_add64(r9, lt, r9, ((U64*)(p521_x2))[9]); + r10= mask_add64(r10,lt, r10,((U64*)(p521_x2))[10]); + + /* normalize r0 - r7 */ + NORM_ASHIFTR(r, 0, 1) + NORM_ASHIFTR(r, 1, 2) + NORM_ASHIFTR(r, 2, 3) + NORM_ASHIFTR(r, 3, 4) + NORM_ASHIFTR(r, 4, 5) + NORM_ASHIFTR(r, 5, 6) + NORM_ASHIFTR(r, 6, 7) + NORM_ASHIFTR(r, 7, 8) + NORM_ASHIFTR(r, 8, 9) + NORM_ASHIFTR(r, 9, 10) + + R[0] = r0; + R[1] = r1; + R[2] = r2; + R[3] = r3; + R[4] = r4; + R[5] = r5; + R[6] = r6; + R[7] = r7; + R[8] = r8; + R[9] = r9; + R[10]= r10; +} + +void MB_FUNC_NAME(ifma_neg52_p521_)(U64 R[], const U64 A[]) +{ + __mb_mask nz_mask = ~MB_FUNC_NAME(is_zero_FE521_)(A); + + /* {r0 - r10} = 2*p - A */ + U64 r0 = mask_sub64( A[0], nz_mask, ((U64*)(p521_x2))[0], A[0] ); + U64 r1 = mask_sub64( A[0], nz_mask, ((U64*)(p521_x2))[1], A[1] ); + U64 r2 = mask_sub64( A[0], nz_mask, ((U64*)(p521_x2))[2], A[2] ); + U64 r3 = mask_sub64( A[0], nz_mask, ((U64*)(p521_x2))[3], A[3] ); + U64 r4 = mask_sub64( A[0], nz_mask, ((U64*)(p521_x2))[4], A[4] ); + U64 r5 = mask_sub64( A[0], nz_mask, ((U64*)(p521_x2))[5], A[5] ); + U64 r6 = mask_sub64( A[0], nz_mask, ((U64*)(p521_x2))[6], A[6] ); + U64 r7 = mask_sub64( A[0], nz_mask, ((U64*)(p521_x2))[7], A[7] ); + U64 r8 = mask_sub64( A[0], nz_mask, ((U64*)(p521_x2))[8], A[8] ); + U64 r9 = mask_sub64( A[0], nz_mask, ((U64*)(p521_x2))[9], A[9] ); + U64 r10= mask_sub64( A[0], nz_mask, ((U64*)(p521_x2))[10],A[10]); + + /* normalize r0 - r10 */ + NORM_ASHIFTR(r, 0, 1) + NORM_ASHIFTR(r, 1, 2) + NORM_ASHIFTR(r, 2, 3) + NORM_ASHIFTR(r, 3, 4) + NORM_ASHIFTR(r, 4, 5) + NORM_ASHIFTR(r, 5, 6) + NORM_ASHIFTR(r, 6, 7) + NORM_ASHIFTR(r, 7, 8) + NORM_ASHIFTR(r, 8, 9) + NORM_ASHIFTR(r, 9, 10) + + R[0] = r0; + R[1] = r1; + R[2] = r2; + R[3] = r3; + R[4] = r4; + R[5] = r5; + R[6] = r6; + R[7] = r7; + R[8] = r8; + R[9] = r9; + R[10]= r10; +} + +void MB_FUNC_NAME(ifma_double52_p521_)(U64 R[], const U64 A[]) +{ + MB_FUNC_NAME(ifma_add52_p521_)(R, A, A); +} + +void MB_FUNC_NAME(ifma_tripple52_p521_)(U64 R[], const U64 A[]) +{ + U64 T[P521_LEN52]; + MB_FUNC_NAME(ifma_add52_p521_)(T, A, A); + MB_FUNC_NAME(ifma_add52_p521_)(R, T, A); +} + +void MB_FUNC_NAME(ifma_half52_p521_)(U64 R[], const U64 A[]) +{ + U64 one = set1(1); + U64 base = set1(DIGIT_BASE); + + /* res = a + is_odd(a)? p521 : 0 */ + U64 mask = sub64(get_zero64(), and64(A[0], one)); + U64 t0 = add64(A[0], and64(((U64*)p521_mb)[0], mask)); + U64 t1 = add64(A[1], and64(((U64*)p521_mb)[1], mask)); + U64 t2 = add64(A[2], and64(((U64*)p521_mb)[2], mask)); + U64 t3 = add64(A[3], and64(((U64*)p521_mb)[3], mask)); + U64 t4 = add64(A[4], and64(((U64*)p521_mb)[4], mask)); + U64 t5 = add64(A[5], and64(((U64*)p521_mb)[5], mask)); + U64 t6 = add64(A[6], and64(((U64*)p521_mb)[6], mask)); + U64 t7 = add64(A[7], and64(((U64*)p521_mb)[7], mask)); + U64 t8 = add64(A[8], and64(((U64*)p521_mb)[8], mask)); + U64 t9 = add64(A[9], and64(((U64*)p521_mb)[9], mask)); + U64 t10= add64(A[10],and64(((U64*)p521_mb)[10],mask)); + + /* t =>> 1 */ + mask = sub64(get_zero64(), and64(t1, one)); + t0 = add64(t0, and64(base, mask)); + t0 = srli64(t0, 1); + + mask = sub64(get_zero64(), and64(t2, one)); + t1 = add64(t1, and64(base, mask)); + t1 = srli64(t1, 1); + + mask = sub64(get_zero64(), and64(t3, one)); + t2 = add64(t2, and64(base, mask)); + t2 = srli64(t2, 1); + + mask = sub64(get_zero64(), and64(t4, one)); + t3 = add64(t3, and64(base, mask)); + t3 = srli64(t3, 1); + + mask = sub64(get_zero64(), and64(t5, one)); + t4 = add64(t4, and64(base, mask)); + t4 = srli64(t4, 1); + + mask = sub64(get_zero64(), and64(t6, one)); + t5 = add64(t5, and64(base, mask)); + t5 = srli64(t5, 1); + + mask = sub64(get_zero64(), and64(t7, one)); + t6 = add64(t6, and64(base, mask)); + t6 = srli64(t6, 1); + + mask = sub64(get_zero64(), and64(t8, one)); + t7 = add64(t7, and64(base, mask)); + t7 = srli64(t7, 1); + + mask = sub64(get_zero64(), and64(t9, one)); + t8 = add64(t8, and64(base, mask)); + t8 = srli64(t8, 1); + + mask = sub64(get_zero64(), and64(t10, one)); + t9 = add64(t9, and64(base, mask)); + t9 = srli64(t9, 1); + + t10 = srli64(t10, 1); + + /* normalize t0, t1, t2, t3, t4 */ + NORM_LSHIFTR(t, 0, 1) + NORM_LSHIFTR(t, 1, 2) + NORM_LSHIFTR(t, 2, 3) + NORM_LSHIFTR(t, 3, 4) + NORM_LSHIFTR(t, 4, 5) + NORM_LSHIFTR(t, 5, 6) + NORM_LSHIFTR(t, 6, 7) + NORM_LSHIFTR(t, 7, 8) + NORM_LSHIFTR(t, 8, 9) + NORM_LSHIFTR(t, 9,10) + + R[0] = t0; + R[1] = t1; + R[2] = t2; + R[3] = t3; + R[4] = t4; + R[5] = t5; + R[6] = t6; + R[7] = t7; + R[8] = t8; + R[9] = t9; + R[10]= t10; +} + +__mb_mask MB_FUNC_NAME(ifma_cmp_lt_p521_)(const U64 a[]) +{ + return MB_FUNC_NAME(cmp_lt_FE521_)(a,(const U64 (*))p521_mb); +} + +__mb_mask MB_FUNC_NAME(ifma_check_range_p521_)(const U64 A[]) +{ + __mb_mask + mask = MB_FUNC_NAME(is_zero_FE521_)(A); + mask |= ~MB_FUNC_NAME(ifma_cmp_lt_p521_)(A); + + return mask; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecdh_p256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecdh_p256.c new file mode 100644 index 0000000..b83e082 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecdh_p256.c @@ -0,0 +1,252 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +#include +#include +#include +#include + +#ifndef BN_OPENSSL_DISABLE +#include +#endif + +#ifndef BN_OPENSSL_DISABLE +/* +// Computes shared key +// pa_shared_key[] array of pointers to the shared keys +// pa_skey[] array of pointers to the own (ephemeral) private keys +// pa_pubx[] array of pointers to the party's public keys X-coordinates +// pa_puby[] array of pointers to the party's public keys Y-coordinates +// pa_pubz[] array of pointers to the party's public keys Z-coordinates (or NULL, if affine coordinate requested) +// pBuffer pointer to the scratch buffer +// +// Note: +// input party's public key depends on is pa_pubz[] parameter and represented either +// - in (X:Y:Z) projective Jacobian coordinates +// or +// - in (x:y) affine coordinate +*/ +DLL_PUBLIC +mbx_status mbx_nistp256_ecdh_ssl_mb8(int8u* pa_shared_key[8], + const BIGNUM* const pa_skey[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* pa_pubz!=0 means the output is in Jacobian projective coordinates */ + int use_jproj_coords = NULL!=pa_pubz; + + /* test input pointers */ + if(NULL==pa_shared_key || NULL==pa_skey || NULL==pa_pubx || NULL==pa_puby) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + int8u* shared = pa_shared_key[buf_no]; + const BIGNUM* skey = pa_skey[buf_no]; + const BIGNUM* pubx = pa_pubx[buf_no]; + const BIGNUM* puby = pa_puby[buf_no]; + const BIGNUM* pubz = use_jproj_coords? pa_pubz[buf_no] : NULL; + + /* if any of pointer NULL set error status */ + if(NULL==shared || NULL==skey|| NULL==pubx || NULL==puby || (use_jproj_coords && NULL==pubz)) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* + // processing + */ + + /* zero padded private keys */ + U64 secretz[P256_LEN64+1]; + ifma_BN_transpose_copy((int64u (*)[8])secretz, (const BIGNUM**)pa_skey, P256_BITSIZE); + secretz[P256_LEN64] = get_zero64(); + + status |= MBX_SET_STS_BY_MASK(status, is_zero(secretz, P256_LEN64+1), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + return status; + } + + P256_POINT P; + /* set party's public */ + ifma_BN_to_mb8((int64u (*)[8])P.X, (const BIGNUM* (*))pa_pubx, P256_BITSIZE); /* P-> radix 2^52 */ + ifma_BN_to_mb8((int64u (*)[8])P.Y, (const BIGNUM* (*))pa_puby, P256_BITSIZE); + if(use_jproj_coords) + ifma_BN_to_mb8((int64u (*)[8])P.Z, (const BIGNUM* (*))pa_pubz, P256_BITSIZE); + else + MB_FUNC_NAME(mov_FE256_)(P.Z, (U64*)ones); + /* convert to Montgomery */ + MB_FUNC_NAME(ifma_tomont52_p256_)(P.X, P.X); + MB_FUNC_NAME(ifma_tomont52_p256_)(P.Y, P.Y); + MB_FUNC_NAME(ifma_tomont52_p256_)(P.Z, P.Z); + + /* check if P does not belong to EC */ + __mb_mask not_on_curve_mask = ~MB_FUNC_NAME(ifma_is_on_curve_p256_)(&P, use_jproj_coords); + /* set points out of EC to infinity */ + MB_FUNC_NAME(mask_set_point_to_infinity_)(&P, not_on_curve_mask); + /* update status */ + status |= MBX_SET_STS_BY_MASK(status, not_on_curve_mask, MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + return status; + } + + P256_POINT R; + /* compute R = [secretz]*P */ + MB_FUNC_NAME(ifma_ec_nistp256_mul_point_)(&R, &P, secretz); + + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + + /* return affine R.x */ + __ALIGN64 U64 Z2[P256_LEN52]; + ifma_aminv52_p256_mb8(Z2, R.Z); /* 1/Z */ + ifma_ams52_p256_mb8(Z2, Z2); /* 1/Z^2 */ + ifma_amm52_p256_mb8(R.X, R.X, Z2); /* x = (X) * (1/Z^2) */ + /* to regular domain */ + MB_FUNC_NAME(ifma_frommont52_p256_)(R.X, R.X); + + /* store result */ + ifma_mb8_to_HexStr8(pa_shared_key, (const int64u (*)[8])R.X, P256_BITSIZE); + + /* clear computed shared keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])&R, sizeof(R)/sizeof(U64)); + + return status; +} +#endif // BN_OPENSSL_DISABLE + +DLL_PUBLIC +mbx_status mbx_nistp256_ecdh_mb8(int8u* pa_shared_key[8], + const int64u* const pa_skey[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* pa_pubz!=0 means the output is in Jacobian projective coordinates */ + int use_jproj_coords = NULL!=pa_pubz; + + /* test input pointers */ + if(NULL==pa_shared_key || NULL==pa_skey || NULL==pa_pubx || NULL==pa_puby) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + int8u* shared = pa_shared_key[buf_no]; + const int64u* skey = pa_skey[buf_no]; + const int64u* pubx = pa_pubx[buf_no]; + const int64u* puby = pa_puby[buf_no]; + const int64u* pubz = use_jproj_coords? pa_pubz[buf_no] : NULL; + + /* if any of pointer NULL set error status */ + if(NULL==shared || NULL==skey || NULL==pubx || NULL==puby || (use_jproj_coords && NULL==pubz)) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* + // processing + */ + + /* zero padded private keys */ + U64 secretz[P256_LEN64+1]; + ifma_BNU_transpose_copy((int64u (*)[8])secretz, (const int64u**)pa_skey, P256_BITSIZE); + secretz[P256_LEN64] = get_zero64(); + + status |= MBX_SET_STS_BY_MASK(status, is_zero(secretz, P256_LEN64+1), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + return status; + } + + P256_POINT P; + /* set party's public */ + ifma_BNU_to_mb8((int64u (*)[8])P.X, (const int64u* (*))pa_pubx, P256_BITSIZE); // P-> crypto_mb radix 2^52 + ifma_BNU_to_mb8((int64u (*)[8])P.Y, (const int64u* (*))pa_puby, P256_BITSIZE); + if(use_jproj_coords) + ifma_BNU_to_mb8((int64u (*)[8])P.Z, (const int64u* (*))pa_pubz, P256_BITSIZE); + else + MB_FUNC_NAME(mov_FE256_)(P.Z, (U64*)ones); + /* convert to Montgomery */ + MB_FUNC_NAME(ifma_tomont52_p256_)(P.X, P.X); + MB_FUNC_NAME(ifma_tomont52_p256_)(P.Y, P.Y); + MB_FUNC_NAME(ifma_tomont52_p256_)(P.Z, P.Z); + + /* check if P does not belong to EC */ + __mb_mask not_on_curve_mask = ~MB_FUNC_NAME(ifma_is_on_curve_p256_)(&P, use_jproj_coords); + /* set points out of EC to infinity */ + MB_FUNC_NAME(mask_set_point_to_infinity_)(&P, not_on_curve_mask); + /* update status */ + status |= MBX_SET_STS_BY_MASK(status, not_on_curve_mask, MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + return status; + } + + P256_POINT R; + /* compute R = [secretz]*P */ + MB_FUNC_NAME(ifma_ec_nistp256_mul_point_)(&R, &P, secretz); + + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + + /* return affine R.x */ + __ALIGN64 U64 Z2[P256_LEN52]; + ifma_aminv52_p256_mb8(Z2, R.Z); /* 1/Z */ + ifma_ams52_p256_mb8(Z2, Z2); /* 1/Z^2 */ + ifma_amm52_p256_mb8(R.X, R.X, Z2); /* x = (X) * (1/Z^2) */ + /* to regular domain */ + MB_FUNC_NAME(ifma_frommont52_p256_)(R.X, R.X); + + /* store result */ + ifma_mb8_to_HexStr8(pa_shared_key, (const int64u (*)[8])R.X, P256_BITSIZE); + + /* clear computed shared keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])&R, sizeof(R)/sizeof(U64)); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecdh_p384.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecdh_p384.c new file mode 100644 index 0000000..16edf1b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecdh_p384.c @@ -0,0 +1,252 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +#include +#include +#include +#include + +#ifndef BN_OPENSSL_DISABLE +#include +#endif + +#ifndef BN_OPENSSL_DISABLE +/* +// Computes shared key +// pa_shared_key[] array of pointers to the shared keys +// pa_skey[] array of pointers to the own (ephemeral) private keys +// pa_pubx[] array of pointers to the party's public keys X-coordinates +// pa_puby[] array of pointers to the party's public keys Y-coordinates +// pa_pubz[] array of pointers to the party's public keys Z-coordinates (or NULL, if affine coordinate requested) +// pBuffer pointer to the scratch buffer +// +// Note: +// input party's public key depends on is pa_pubz[] parameter and represented either +// - in (X:Y:Z) projective Jacobian coordinates +// or +// - in (x:y) affine coordinate +*/ +DLL_PUBLIC +mbx_status mbx_nistp384_ecdh_ssl_mb8(int8u* pa_shared_key[8], + const BIGNUM* const pa_skey[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* pa_pubz!=0 means the output is in Jacobian projective coordinates */ + int use_jproj_coords = NULL!=pa_pubz; + + /* test input pointers */ + if(NULL==pa_shared_key || NULL==pa_skey || NULL==pa_pubx || NULL==pa_puby) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + int8u* shared = pa_shared_key[buf_no]; + const BIGNUM* skey = pa_skey[buf_no]; + const BIGNUM* pubx = pa_pubx[buf_no]; + const BIGNUM* puby = pa_puby[buf_no]; + const BIGNUM* pubz = use_jproj_coords? pa_pubz[buf_no] : NULL; + + /* if any of pointer NULL set error status */ + if(NULL==shared || NULL==skey || NULL==pubx || NULL==puby || (use_jproj_coords && NULL==pubz)) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* + // processing + */ + + /* zero padded private keys */ + U64 secretz[P384_LEN64+1]; + ifma_BN_transpose_copy((int64u (*)[8])secretz, (const BIGNUM**)pa_skey, P384_BITSIZE); + secretz[P384_LEN64] = get_zero64(); + + status |= MBX_SET_STS_BY_MASK(status, is_zero(secretz, P384_LEN64+1), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + return status; + } + + P384_POINT P; + /* set party's public */ + ifma_BN_to_mb8((int64u (*)[8])P.X, (const BIGNUM* (*))pa_pubx, P384_BITSIZE); /* P-> radix 2^52 */ + ifma_BN_to_mb8((int64u (*)[8])P.Y, (const BIGNUM* (*))pa_puby, P384_BITSIZE); + if(use_jproj_coords) + ifma_BN_to_mb8((int64u (*)[8])P.Z, (const BIGNUM* (*))pa_pubz, P384_BITSIZE); + else + MB_FUNC_NAME(mov_FE384_)(P.Z, (U64*)ones); + /* convert to Montgomery */ + MB_FUNC_NAME(ifma_tomont52_p384_)(P.X, P.X); + MB_FUNC_NAME(ifma_tomont52_p384_)(P.Y, P.Y); + MB_FUNC_NAME(ifma_tomont52_p384_)(P.Z, P.Z); + + /* check if P does not belong to EC */ + __mb_mask not_on_curve_mask = ~MB_FUNC_NAME(ifma_is_on_curve_p384_)(&P, use_jproj_coords); + /* set points out of EC to infinity */ + MB_FUNC_NAME(mask_set_point_to_infinity_)(&P, not_on_curve_mask); + /* update status */ + status |= MBX_SET_STS_BY_MASK(status, not_on_curve_mask, MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + return status; + } + + P384_POINT R; + /* compute R = [secretz]*P */ + MB_FUNC_NAME(ifma_ec_nistp384_mul_point_)(&R, &P, secretz); + + /* clear ephemeral secret copy */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + + /* return affine R.x */ + __ALIGN64 U64 Z2[P384_LEN52]; + ifma_aminv52_p384_mb8(Z2, R.Z); /* 1/Z */ + ifma_ams52_p384_mb8(Z2, Z2); /* 1/Z^2 */ + ifma_amm52_p384_mb8(R.X, R.X, Z2); /* x = (X) * (1/Z^2) */ + /* to regular domain */ + MB_FUNC_NAME(ifma_frommont52_p384_)(R.X, R.X); + + /* store result */ + ifma_mb8_to_HexStr8(pa_shared_key, (const int64u (*)[8])R.X, P384_BITSIZE); + + /* clear shared secret */ + MB_FUNC_NAME(zero_)((int64u (*)[8])&R, sizeof(R)/sizeof(U64)); + + return status; +} +#endif // BN_OPENSSL_DISABLE + +DLL_PUBLIC +mbx_status mbx_nistp384_ecdh_mb8(int8u* pa_shared_key[8], + const int64u* const pa_skey[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* pa_pubz!=0 means the output is in Jacobian projective coordinates */ + int use_jproj_coords = NULL!=pa_pubz; + + /* test input pointers */ + if(NULL==pa_shared_key || NULL==pa_skey || NULL==pa_pubx || NULL==pa_puby) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + int8u* shared = pa_shared_key[buf_no]; + const int64u* skey = pa_skey[buf_no]; + const int64u* pubx = pa_pubx[buf_no]; + const int64u* puby = pa_puby[buf_no]; + const int64u* pubz = use_jproj_coords? pa_pubz[buf_no] : NULL; + + /* if any of pointer NULL set error status */ + if(NULL==shared || NULL==skey || NULL==pubx || NULL== puby || (use_jproj_coords && NULL==pubz)) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* + // processing + */ + + /* zero padded private keys */ + U64 secretz[P384_LEN64+1]; + ifma_BNU_transpose_copy((int64u (*)[8])secretz, (const int64u**)pa_skey, P384_BITSIZE); + secretz[P384_LEN64] = get_zero64(); + + status |= MBX_SET_STS_BY_MASK(status, is_zero(secretz, P384_LEN64+1), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + return status; + } + + P384_POINT P; + /* set party's public */ + ifma_BNU_to_mb8((int64u (*)[8])P.X, (const int64u* (*))pa_pubx, P384_BITSIZE); // P-> crypto_mb radix 2^52 + ifma_BNU_to_mb8((int64u (*)[8])P.Y, (const int64u* (*))pa_puby, P384_BITSIZE); + if(use_jproj_coords) + ifma_BNU_to_mb8((int64u (*)[8])P.Z, (const int64u* (*))pa_pubz, P384_BITSIZE); + else + MB_FUNC_NAME(mov_FE384_)(P.Z, (U64*)ones); + /* convert to Montgomery */ + MB_FUNC_NAME(ifma_tomont52_p384_)(P.X, P.X); + MB_FUNC_NAME(ifma_tomont52_p384_)(P.Y, P.Y); + MB_FUNC_NAME(ifma_tomont52_p384_)(P.Z, P.Z); + + /* check if P does not belong to EC */ + __mb_mask not_on_curve_mask = ~MB_FUNC_NAME(ifma_is_on_curve_p384_)(&P, use_jproj_coords); + /* set points out of EC to infinity */ + MB_FUNC_NAME(mask_set_point_to_infinity_)(&P, not_on_curve_mask); + /* update status */ + status |= MBX_SET_STS_BY_MASK(status, not_on_curve_mask, MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + return status; + } + + P384_POINT R; + /* compute R = [secretz]*P */ + MB_FUNC_NAME(ifma_ec_nistp384_mul_point_)(&R, &P, secretz); + + /* clear ephemeral secret copy */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + + /* return affine R.x */ + __ALIGN64 U64 Z2[P384_LEN52]; + ifma_aminv52_p384_mb8(Z2, R.Z); /* 1/Z */ + ifma_ams52_p384_mb8(Z2, Z2); /* 1/Z^2 */ + ifma_amm52_p384_mb8(R.X, R.X, Z2); /* x = (X) * (1/Z^2) */ + /* to regular domain */ + MB_FUNC_NAME(ifma_frommont52_p384_)(R.X, R.X); + + /* store result */ + ifma_mb8_to_HexStr8(pa_shared_key, (const int64u (*)[8])R.X, P384_BITSIZE); + + /* clear shared secret */ + MB_FUNC_NAME(zero_)((int64u (*)[8])&R, sizeof(R)/sizeof(U64)); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecdh_p521.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecdh_p521.c new file mode 100644 index 0000000..a5d3494 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecdh_p521.c @@ -0,0 +1,252 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +#include +#include +#include +#include + +#ifndef BN_OPENSSL_DISABLE +#include +#endif + +#ifndef BN_OPENSSL_DISABLE +/* +// Computes shared key +// pa_shared_key[] array of pointers to the shared keys +// pa_skey[] array of pointers to the own (ephemeral) private keys +// pa_pubx[] array of pointers to the party's public keys X-coordinates +// pa_puby[] array of pointers to the party's public keys Y-coordinates +// pa_pubz[] array of pointers to the party's public keys Z-coordinates (or NULL, if affine coordinate requested) +// pBuffer pointer to the scratch buffer +// +// Note: +// input party's public key depends on is pa_pubz[] parameter and represented either +// - in (X:Y:Z) projective Jacobian coordinates +// or +// - in (x:y) affine coordinate +*/ +DLL_PUBLIC +mbx_status mbx_nistp521_ecdh_ssl_mb8(int8u* pa_shared_key[8], + const BIGNUM* const pa_skey[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* pa_pubz!=0 means the output is in Jacobian projective coordinates */ + int use_jproj_coords = NULL!=pa_pubz; + + /* test input pointers */ + if(NULL==pa_shared_key || NULL==pa_skey || NULL==pa_pubx || NULL==pa_puby) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + int8u* shared = pa_shared_key[buf_no]; + const BIGNUM* skey = pa_skey[buf_no]; + const BIGNUM* pubx = pa_pubx[buf_no]; + const BIGNUM* puby = pa_puby[buf_no]; + const BIGNUM* pubz = use_jproj_coords? pa_pubz[buf_no] : NULL; + + /* if any of pointer NULL set error status */ + if(NULL==shared || NULL==skey || NULL==pubx || NULL==puby || (use_jproj_coords && NULL==pubz)) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* + // processing + */ + + /* zero padded private keys */ + U64 secretz[P521_LEN64+1]; + ifma_BN_transpose_copy((int64u (*)[8])secretz, (const BIGNUM**)pa_skey, P521_BITSIZE); + secretz[P521_LEN64] = get_zero64(); + + status |= MBX_SET_STS_BY_MASK(status, is_zero(secretz, P521_LEN64+1), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + return status; + } + + P521_POINT P; + /* set party's public */ + ifma_BN_to_mb8((int64u (*)[8])P.X, (const BIGNUM* (*))pa_pubx, P521_BITSIZE); /* P-> radix 2^52 */ + ifma_BN_to_mb8((int64u (*)[8])P.Y, (const BIGNUM* (*))pa_puby, P521_BITSIZE); + if(use_jproj_coords) + ifma_BN_to_mb8((int64u (*)[8])P.Z, (const BIGNUM* (*))pa_pubz, P521_BITSIZE); + else + MB_FUNC_NAME(mov_FE521_)(P.Z, (U64*)ones); + /* convert to Montgomery */ + MB_FUNC_NAME(ifma_tomont52_p521_)(P.X, P.X); + MB_FUNC_NAME(ifma_tomont52_p521_)(P.Y, P.Y); + MB_FUNC_NAME(ifma_tomont52_p521_)(P.Z, P.Z); + + /* check if P does not belong to EC */ + __mb_mask not_on_curve_mask = ~MB_FUNC_NAME(ifma_is_on_curve_p521_)(&P, use_jproj_coords); + /* set points out of EC to infinity */ + MB_FUNC_NAME(mask_set_point_to_infinity_)(&P, not_on_curve_mask); + /* update status */ + status |= MBX_SET_STS_BY_MASK(status, not_on_curve_mask, MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + return status; + } + + P521_POINT R; + /* compute R = [secretz]*P */ + MB_FUNC_NAME(ifma_ec_nistp521_mul_point_)(&R, &P, secretz); + + /* clear ephemeral secret copy */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + + /* return affine R.x */ + __ALIGN64 U64 Z2[P521_LEN52]; + ifma_aminv52_p521_mb8(Z2, R.Z); /* 1/Z */ + ifma_ams52_p521_mb8(Z2, Z2); /* 1/Z^2 */ + ifma_amm52_p521_mb8(R.X, R.X, Z2); /* x = (X) * (1/Z^2) */ + /* to regular domain */ + MB_FUNC_NAME(ifma_frommont52_p521_)(R.X, R.X); + + /* store result */ + ifma_mb8_to_HexStr8(pa_shared_key, (const int64u (*)[8])R.X, P521_BITSIZE); + + /* clear shared secret */ + MB_FUNC_NAME(zero_)((int64u (*)[8])&R, sizeof(R)/sizeof(U64)); + + return status; +} +#endif // BN_OPENSSL_DISABLE + +DLL_PUBLIC +mbx_status mbx_nistp521_ecdh_mb8(int8u* pa_shared_key[8], + const int64u* const pa_skey[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* pa_pubz!=0 means the output is in Jacobian projective coordinates */ + int use_jproj_coords = NULL!=pa_pubz; + + /* test input pointers */ + if(NULL==pa_shared_key || NULL==pa_skey || NULL==pa_pubx || NULL==pa_puby) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + int8u* shared = pa_shared_key[buf_no]; + const int64u* skey = pa_skey[buf_no]; + const int64u* pubx = pa_pubx[buf_no]; + const int64u* puby = pa_puby[buf_no]; + const int64u* pubz = use_jproj_coords? pa_pubz[buf_no] : NULL; + + /* if any of pointer NULL set error status */ + if(NULL==shared || NULL==skey || NULL==pubx || NULL== puby || (use_jproj_coords && NULL==pubz)) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* + // processing + */ + + /* zero padded private keys */ + U64 secretz[P521_LEN64+1]; + ifma_BNU_transpose_copy((int64u (*)[8])secretz, (const int64u**)pa_skey, P521_BITSIZE); + secretz[P521_LEN64] = get_zero64(); + + status |= MBX_SET_STS_BY_MASK(status, is_zero(secretz, P521_LEN64+1), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + return status; + } + + P521_POINT P; + /* set party's public */ + ifma_BNU_to_mb8((int64u (*)[8])P.X, (const int64u* (*))pa_pubx, P521_BITSIZE); // P-> crypto_mb radix 2^52 + ifma_BNU_to_mb8((int64u (*)[8])P.Y, (const int64u* (*))pa_puby, P521_BITSIZE); + if(use_jproj_coords) + ifma_BNU_to_mb8((int64u (*)[8])P.Z, (const int64u* (*))pa_pubz, P521_BITSIZE); + else + MB_FUNC_NAME(mov_FE521_)(P.Z, (U64*)ones); + /* convert to Montgomery */ + MB_FUNC_NAME(ifma_tomont52_p521_)(P.X, P.X); + MB_FUNC_NAME(ifma_tomont52_p521_)(P.Y, P.Y); + MB_FUNC_NAME(ifma_tomont52_p521_)(P.Z, P.Z); + + /* check if P does not belong to EC */ + __mb_mask not_on_curve_mask = ~MB_FUNC_NAME(ifma_is_on_curve_p521_)(&P, use_jproj_coords); + /* set points out of EC to infinity */ + MB_FUNC_NAME(mask_set_point_to_infinity_)(&P, not_on_curve_mask); + /* update status */ + status |= MBX_SET_STS_BY_MASK(status, not_on_curve_mask, MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + return status; + } + + P521_POINT R; + /* compute R = [secretz]*P */ + MB_FUNC_NAME(ifma_ec_nistp521_mul_point_)(&R, &P, secretz); + + /* clear ephemeral secret copy */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + + /* return affine R.x */ + __ALIGN64 U64 Z2[P521_LEN52]; + ifma_aminv52_p521_mb8(Z2, R.Z); /* 1/Z */ + ifma_ams52_p521_mb8(Z2, Z2); /* 1/Z^2 */ + ifma_amm52_p521_mb8(R.X, R.X, Z2); /* x = (X) * (1/Z^2) */ + /* to regular domain */ + MB_FUNC_NAME(ifma_frommont52_p521_)(R.X, R.X); + + /* store result */ + ifma_mb8_to_HexStr8(pa_shared_key, (const int64u (*)[8])R.X, P521_BITSIZE); + + /* clear shared secret */ + MB_FUNC_NAME(zero_)((int64u (*)[8])&R, sizeof(R)/sizeof(U64)); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecdsa_p256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecdsa_p256.c new file mode 100644 index 0000000..97a6476 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecdsa_p256.c @@ -0,0 +1,887 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +#include +#include +#include +#include + +#ifndef BN_OPENSSL_DISABLE +#include +#include +#ifdef OPENSSL_IS_BORINGSSL +#include +#endif +#endif + +/* +// common functions +*/ + +/* +// compute secret key inversion +// +// inv_skey 1/skey mod n256 +// +// note: pay attention on skey[] presenttaion +// it should be FE element of N256 basef GF +*/ +static void nistp256_ecdsa_inv_keys_mb8(U64 inv_skey[], + const U64 skey[], + int8u pBuffer[]) +{ + /* compute inversion ober n256 of secret keys */ + MB_FUNC_NAME(ifma_tomont52_n256_)(inv_skey, skey); + ifma_aminv52_n256_mb8(inv_skey, inv_skey); /* 1/skeys mod n256 */ + MB_FUNC_NAME(ifma_frommont52_n256_)(inv_skey, inv_skey); +} + +/* +// compute r-component of the ECDSA signature +// +// r = ([skey]*G).x mod n256 +// +// note: pay attention on skey[] presenttaion +// it should be transposed and zero expanded +*/ +static __mb_mask nistp256_ecdsa_sign_r_mb8(U64 sign_r[], + const U64 skey[], + int8u pBuffer[]) +{ + /* compute ephemeral public keys */ + P256_POINT P; + + MB_FUNC_NAME(ifma_ec_nistp256_mul_pointbase_)(&P, skey); + + /* extract affine P.x */ + MB_FUNC_NAME(ifma_aminv52_p256_)(P.Z, P.Z); /* 1/Z */ + MB_FUNC_NAME(ifma_ams52_p256_)(P.Z, P.Z); /* 1/Z^2 */ + MB_FUNC_NAME(ifma_amm52_p256_)(P.X, P.X, P.Z); /* x = (X) * (1/Z^2) */ + + /* convert x-coordinate to regular and then tp Montgomery n256 */ + MB_FUNC_NAME(ifma_frommont52_p256_)(P.X, P.X); + MB_FUNC_NAME(ifma_fastred52_pn256_)(sign_r, P.X); /* fast reduction p => n */ + + return MB_FUNC_NAME(is_zero_FE256_)(sign_r); +} + +/* +// compute s-component of the ECDSA signature +// +// s = (inv_eph) * (msg + prv_skey*sign_r) mod n256 +*/ +static __mb_mask nistp256_ecdsa_sign_s_mb8(U64 sign_s[], + U64 msg[], + const U64 sign_r[], + U64 inv_eph_skey[], + U64 reg_skey[], + int8u pBuffer[]) +{ + __ALIGN64 U64 tmp[P256_LEN52]; + + /* conver to Montgomery over n256 domain */ + MB_FUNC_NAME(ifma_tomont52_n256_)(inv_eph_skey, inv_eph_skey); + MB_FUNC_NAME(ifma_tomont52_n256_)(tmp, sign_r); + MB_FUNC_NAME(ifma_tomont52_n256_)(msg, msg); + MB_FUNC_NAME(ifma_tomont52_n256_)(reg_skey, reg_skey); + + /* s = (inv_eph) * (msg + prv_skey*sign_r) mod n256 */ + MB_FUNC_NAME(ifma_amm52_n256_)(sign_s, reg_skey, tmp); + MB_FUNC_NAME(ifma_add52_n256_)(sign_s, sign_s, msg); + MB_FUNC_NAME(ifma_amm52_n256_)(sign_s, sign_s, inv_eph_skey); + MB_FUNC_NAME(ifma_frommont52_n256_)(sign_s, sign_s); + + return MB_FUNC_NAME(is_zero_FE256_)(sign_s); +} + +/* +// ECDSA signature verification algorithm +*/ +static __mb_mask nistp256_ecdsa_verify_mb8(U64 sign_r[], + U64 sign_s[], + U64 msg[], + P256_POINT* W) +{ + /* convert public key coords to Montgomery */ + MB_FUNC_NAME(ifma_tomont52_p256_)(W->X, W->X); + MB_FUNC_NAME(ifma_tomont52_p256_)(W->Y, W->Y); + MB_FUNC_NAME(ifma_tomont52_p256_)(W->Z, W->Z); + + __ALIGN64 U64 h1[P256_LEN52]; + __ALIGN64 U64 h2[P256_LEN52]; + + /* h = (sign_s)^(-1) */ + MB_FUNC_NAME(ifma_tomont52_n256_)(sign_s, sign_s); + MB_FUNC_NAME(ifma_aminv52_n256_)(sign_s, sign_s); + /* h1 = msg * h */ + MB_FUNC_NAME(ifma_tomont52_n256_)(h1, msg); + MB_FUNC_NAME(ifma_amm52_n256_)(h1, h1, sign_s); + MB_FUNC_NAME(ifma_frommont52_n256_)(h1, h1); + /* h2 = sign_r * h */ + MB_FUNC_NAME(ifma_tomont52_n256_)(h2, sign_r); + MB_FUNC_NAME(ifma_amm52_n256_)(h2, h2, sign_s); + MB_FUNC_NAME(ifma_frommont52_n256_)(h2, h2); + + int64u tmp[8][P256_LEN64]; + int64u* pa_tmp[8] = { tmp[0], tmp[1], tmp[2], tmp[3], + tmp[4], tmp[5], tmp[6], tmp[7] }; + + ifma_mb8_to_BNU(pa_tmp, (const int64u(*)[8])h1, P256_BITSIZE); + ifma_BNU_transpose_copy((int64u(*)[8])h1, (const int64u(**))pa_tmp, P256_BITSIZE); + + ifma_mb8_to_BNU(pa_tmp, (const int64u(*)[8])h2, P256_BITSIZE); + ifma_BNU_transpose_copy((int64u(*)[8])h2, (const int64u(**))pa_tmp, P256_BITSIZE); + + h1[P256_LEN64] = get_zero64(); + h2[P256_LEN64] = get_zero64(); + + P256_POINT P; + + // P = h1*G + h2*W + MB_FUNC_NAME(ifma_ec_nistp256_mul_point_)(W, W, h2); + MB_FUNC_NAME(ifma_ec_nistp256_mul_pointbase_)(&P, h1); + MB_FUNC_NAME(ifma_ec_nistp256_add_point_)(&P, &P, W); + + // P != 0 + __mb_mask signature_err_mask = MB_FUNC_NAME(is_zero_point_cordinate_)(P.Z); + + /* sign_r_restored = P.X mod n */ + __ALIGN64 U64 sign_r_restored[P256_LEN52]; + MB_FUNC_NAME(get_nistp256_ec_affine_coords_)(sign_r_restored, NULL, &P); + MB_FUNC_NAME(ifma_frommont52_p256_)(sign_r_restored, sign_r_restored); + MB_FUNC_NAME(ifma_fastred52_pn256_)(sign_r_restored, sign_r_restored); + + /* sign_r_restored != sign_r */ + signature_err_mask |= ~(MB_FUNC_NAME(cmp_eq_FE256_)(sign_r_restored, sign_r)); + + return signature_err_mask; +} + +/* +// ECDSA kernels +*/ + +/* +// pre-computation of ECDSA signature +// +// pa_inv_eph_skey[] array of pointers to the inversion of signer's ephemeral private keys +// pa_sign_rp[] array of pointers to the r-components of the signatures +// pa_eph_skey[] array of pointers to the ephemeral (nonce) signer's ephemeral private keys +// pBuffer pointer to the scratch buffer +// +// function computes two values that does not depend on the message to be signed +// - inversion of signer's ephemeral (nonce) keys +// - r-component of the signature +// and are later used during the signing process +*/ +DLL_PUBLIC +mbx_status mbx_nistp256_ecdsa_sign_setup_mb8(int64u* pa_inv_eph_skey[8], + int64u* pa_sign_rp[8], + const int64u* const pa_eph_skey[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* test input pointers */ + if(NULL==pa_inv_eph_skey || NULL==pa_sign_rp || NULL==pa_eph_skey) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check data pointers */ + for(buf_no=0; buf_no<8; buf_no++) { + int64u* pinv_key = pa_inv_eph_skey[buf_no]; + int64u* psign_rp = pa_sign_rp[buf_no]; + const int64u* pkey = pa_eph_skey[buf_no]; + /* if any of pointer NULL set error status */ + if(NULL==pinv_key || NULL==psign_rp || NULL==pkey) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + if(!MBX_IS_ANY_OK_STS(status) ) + return status; + + /* convert keys into FE and compute inversion */ + U64 T[P256_LEN52]; + ifma_BNU_to_mb8((int64u (*)[8])T, pa_eph_skey, P256_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FE256_)(T), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear key's inversion */ + MB_FUNC_NAME(zero_)((int64u (*)[8])T, sizeof(T)/sizeof(U64)); + return status; + } + + nistp256_ecdsa_inv_keys_mb8(T, T, 0); + /* return results in suitable format */ + ifma_mb8_to_BNU(pa_inv_eph_skey, (const int64u(*)[8])T, P256_BITSIZE); + + /* clear key's inversion */ + MB_FUNC_NAME(zero_)((int64u (*)[8])T, sizeof(T)/sizeof(U64)); + + /* convert keys into scalars */ + U64 scalarz[P256_LEN64+1]; + ifma_BNU_transpose_copy((int64u (*)[8])scalarz, pa_eph_skey, P256_BITSIZE); + scalarz[P256_LEN64] = get_zero64(); + /* compute r-component of the DSA signature */ + int8u stt_mask = nistp256_ecdsa_sign_r_mb8(T, scalarz, pBuffer); + + /* clear copy of the ephemeral secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])scalarz, sizeof(scalarz)/sizeof(U64)); + + /* return results in suitable format */ + ifma_mb8_to_BNU(pa_sign_rp, (const int64u(*)[8])T, P256_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, stt_mask, MBX_STATUS_SIGNATURE_ERR); + return status; +} + +/* +// computes ECDSA signature +// +// pa_sign_r[] array of pointers to the r-components of the signatures +// pa_sign_s[] array of pointers to the s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// pa_sign_rp[] array of pointers to the pre-computed r-components of the signatures +// pa_inv_eph_skey[] array of pointers to the inversion of signer's ephemeral private keys +// pa_reg_skey[] array of pointers to the regular signer's ephemeral (nonce) private keys +// pBuffer pointer to the scratch buffer +*/ +DLL_PUBLIC +mbx_status mbx_nistp256_ecdsa_sign_complete_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_sign_rp[8], + const int64u* const pa_inv_eph_skey[8], + const int64u* const pa_reg_skey[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* test input pointers */ + if(NULL==pa_sign_r || NULL==pa_sign_s || NULL==pa_msg || + NULL==pa_sign_rp || NULL==pa_inv_eph_skey || NULL==pa_reg_skey) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check data pointers */ + for(buf_no=0; buf_no<8; buf_no++) { + int8u* psign_r = pa_sign_r[buf_no]; + int8u* psign_s = pa_sign_s[buf_no]; + const int8u* pmsg = pa_msg[buf_no]; + const int64u* psign_pr = pa_sign_rp[buf_no]; + const int64u* pinv_eph_key = pa_inv_eph_skey[buf_no]; + const int64u* preg_key = pa_reg_skey[buf_no]; + /* if any of pointer NULL set error status */ + if(NULL==psign_r || NULL==psign_s || NULL==pmsg || + NULL==psign_pr || NULL==pinv_eph_key || NULL==preg_key) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + if(!MBX_IS_ANY_OK_STS(status) ) + return status; + + __ALIGN64 U64 inv_eph[P256_LEN52]; + __ALIGN64 U64 reg_skey[P256_LEN52]; + __ALIGN64 U64 sign_r[P256_LEN52]; + __ALIGN64 U64 sign_s[P256_LEN52]; + __ALIGN64 U64 msg[P256_LEN52]; + + /* convert inv_eph, reg_skey, sign_r and message to mb format */ + + ifma_BNU_to_mb8((int64u (*)[8])inv_eph, pa_inv_eph_skey, P256_BITSIZE); + ifma_BNU_to_mb8((int64u (*)[8])reg_skey, pa_reg_skey, P256_BITSIZE); + ifma_BNU_to_mb8((int64u (*)[8])sign_r, pa_sign_rp, P256_BITSIZE); + ifma_HexStr8_to_mb8((int64u (*)[8])msg, pa_msg, P256_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FE256_)(inv_eph), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FE256_)(reg_skey), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FE256_)(sign_r), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_n256_)(msg), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the ephemeral secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])inv_eph, sizeof(inv_eph)/sizeof(U64)); + /* clear copy of the regular secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])reg_skey, sizeof(reg_skey)/sizeof(U64)); + return status; + } + + /* compute s- signature component: s = (inv_eph) * (msg + prv_skey*sign_r) mod n256 */ + nistp256_ecdsa_sign_s_mb8(sign_s, msg, sign_r, inv_eph, reg_skey, pBuffer); + + /* clear copy of the ephemeral secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])inv_eph, sizeof(inv_eph)/sizeof(U64)); + /* clear copy of the regular secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])reg_skey, sizeof(reg_skey)/sizeof(U64)); + + /* check if sign_r!=0 and sign_s!=0 */ + int8u stt_mask_r = MB_FUNC_NAME(is_zero_FE256_)(sign_r); + int8u stt_mask_s = MB_FUNC_NAME(is_zero_FE256_)(sign_s); + + /* convert sign_r and sing_s to strings */ + ifma_mb8_to_HexStr8(pa_sign_r, (const int64u(*)[8])sign_r, P256_BITSIZE); + ifma_mb8_to_HexStr8(pa_sign_s, (const int64u(*)[8])sign_s, P256_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, stt_mask_r, MBX_STATUS_SIGNATURE_ERR); + status |= MBX_SET_STS_BY_MASK(status, stt_mask_s, MBX_STATUS_SIGNATURE_ERR); + return status; +} + +/* +// Computes ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// pa_eph_skey[] array of pointers to the signer's ephemeral (nonce) private keys +// pa_reg_skey[] array of pointers to the signer's regular (long term) private keys +// pBuffer pointer to the scratch buffer +*/ +DLL_PUBLIC +mbx_status mbx_nistp256_ecdsa_sign_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_eph_skey[8], + const int64u* const pa_reg_skey[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* test input pointers */ + if(NULL==pa_sign_r || NULL==pa_sign_s || NULL==pa_msg || NULL==pa_eph_skey || NULL==pa_reg_skey) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + /* check data pointers */ + for(buf_no=0; buf_no<8; buf_no++) { + int8u* psign_r = pa_sign_r[buf_no]; + int8u* psign_s = pa_sign_s[buf_no]; + const int8u* pmsg = pa_msg[buf_no]; + const int64u* peph_key = pa_eph_skey[buf_no]; + const int64u* preg_key = pa_reg_skey[buf_no]; + /* if any of pointer NULL set error status */ + if(NULL==psign_r || NULL==psign_s || NULL==pmsg || NULL==peph_key || NULL==preg_key) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + if(!MBX_IS_ANY_OK_STS(status) ) + return status; + + __ALIGN64 U64 inv_eph_key[P256_LEN52]; + __ALIGN64 U64 reg_key[P256_LEN52]; + __ALIGN64 U64 sign_r[P256_LEN52]; + __ALIGN64 U64 sign_s[P256_LEN52]; + __ALIGN64 U64 scalar[P256_LEN64+1]; + __ALIGN64 U64 msg[P256_LEN52]; + + /* convert ephemeral keys into FE */ + ifma_BNU_to_mb8((int64u (*)[8])inv_eph_key, pa_eph_skey, P256_BITSIZE); + /* convert epphemeral keys into sclar */ + ifma_BNU_transpose_copy((int64u (*)[8])scalar, pa_eph_skey, P256_BITSIZE); + scalar[P256_LEN64] = get_zero64(); + /* convert reg_skey */ + ifma_BNU_to_mb8((int64u (*)[8])reg_key, pa_reg_skey, P256_BITSIZE); + /* convert message */ + ifma_HexStr8_to_mb8((int64u (*)[8])msg, pa_msg, P256_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FE256_)(inv_eph_key), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FE256_)(reg_key), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_n256_)(msg), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the ephemeral secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])inv_eph_key, sizeof(inv_eph_key)/sizeof(U64)); + MB_FUNC_NAME(zero_)((int64u (*)[8])scalar, sizeof(scalar)/sizeof(U64)); + /* clear copy of the regular secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])reg_key, sizeof(reg_key)/sizeof(U64)); + return status; + } + + /* compute inversion */ + nistp256_ecdsa_inv_keys_mb8(inv_eph_key, inv_eph_key, pBuffer); + /* compute r-component */ + nistp256_ecdsa_sign_r_mb8(sign_r, scalar, pBuffer); + /* compute s-component */ + nistp256_ecdsa_sign_s_mb8(sign_s, msg, sign_r, inv_eph_key, reg_key, pBuffer); + + /* clear copy of the ephemeral secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])inv_eph_key, sizeof(inv_eph_key)/sizeof(U64)); + MB_FUNC_NAME(zero_)((int64u (*)[8])scalar, sizeof(scalar)/sizeof(U64)); + + /* clear copy of the regular secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])reg_key, sizeof(reg_key)/sizeof(U64)); + + /* check if sign_r!=0 and sign_s!=0 */ + int8u stt_mask_r = MB_FUNC_NAME(is_zero_FE256_)(sign_r); + int8u stt_mask_s = MB_FUNC_NAME(is_zero_FE256_)(sign_s); + + /* convert singnature components to strings */ + ifma_mb8_to_HexStr8(pa_sign_r, (const int64u(*)[8])sign_r, P256_BITSIZE); + ifma_mb8_to_HexStr8(pa_sign_s, (const int64u(*)[8])sign_s, P256_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, stt_mask_r, MBX_STATUS_SIGNATURE_ERR); + status |= MBX_SET_STS_BY_MASK(status, stt_mask_s, MBX_STATUS_SIGNATURE_ERR); + return status; +} + +/* +// Verifies ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// pa_pubx[] array of pointers to the public keys X-coordinates +// pa_puby[] array of pointers to the public keys Y-coordinates +// pa_pubz[] array of pointers to the public keys Z-coordinates +// pBuffer pointer to the scratch buffer*/ +DLL_PUBLIC +mbx_status mbx_nistp256_ecdsa_verify_mb8(const int8u* const pa_sign_r[8], + const int8u* const pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + int use_jproj_coords = NULL!=pa_pubz; + + /* test input pointers */ + if(NULL==pa_pubx || NULL==pa_puby || NULL==pa_msg || NULL==pa_sign_r || NULL==pa_sign_s) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + const int64u* pubx = pa_pubx[buf_no]; + const int64u* puby = pa_puby[buf_no]; + const int64u* pubz = use_jproj_coords? pa_pubz[buf_no] : NULL; + const int8u* msg = pa_msg[buf_no]; + const int8u* r = pa_sign_r[buf_no]; + const int8u* s = pa_sign_s[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL==pubx || NULL==puby || NULL==msg || NULL== r || NULL==s || (use_jproj_coords && NULL==pubz)) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + /* if all pointers NULL exit */ + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + __ALIGN64 U64 msg[P256_LEN52]; + __ALIGN64 U64 sign_r[P256_LEN52]; + __ALIGN64 U64 sign_s[P256_LEN52]; + + /* convert input params */ + ifma_HexStr8_to_mb8((int64u (*)[8])msg, pa_msg, P256_BITSIZE); + ifma_HexStr8_to_mb8((int64u (*)[8])sign_r, pa_sign_r, P256_BITSIZE); + ifma_HexStr8_to_mb8((int64u (*)[8])sign_s, pa_sign_s, P256_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_n256_)(msg), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_n256_)(sign_r), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_n256_)(sign_s), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + P256_POINT W; + + ifma_BNU_to_mb8((int64u (*)[8])W.X, (const int64u* (*))pa_pubx, P256_BITSIZE); + ifma_BNU_to_mb8((int64u (*)[8])W.Y, (const int64u* (*))pa_puby, P256_BITSIZE); + if(use_jproj_coords) + ifma_BNU_to_mb8((int64u (*)[8])W.Z, (const int64u* (*))pa_pubz, P256_BITSIZE); + else + MB_FUNC_NAME(mov_FE256_)(W.Z, (U64*)ones); + + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_p256_)(W.X), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_p256_)(W.Y), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_p256_)(W.Z), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + __mb_mask signature_err_mask = nistp256_ecdsa_verify_mb8(sign_r,sign_s,msg, &W); + status |= MBX_SET_STS_BY_MASK(status, signature_err_mask, MBX_STATUS_SIGNATURE_ERR); + + return status; +} + +/* +// OpenSSL's specific implementations +*/ +#ifndef BN_OPENSSL_DISABLE + +static void reverse_inplace(int8u* inpout, int len) +{ + int mudpoint = len/2; + for(int n=0; n + +#include +#include +#include +#include + +#ifndef BN_OPENSSL_DISABLE +#include +#include +#ifdef OPENSSL_IS_BORINGSSL +#include +#endif +#endif + +/* +// common functions +*/ + +/* +// compute secret key inversion +// +// inv_skey 1/skey mod n384 +// +// note: pay attention on skey[] presenttaion +// it should be FE element of N384 basef GF +*/ +static void nistp384_ecdsa_inv_keys_mb8(U64 inv_skey[], + const U64 skey[], + int8u pBuffer[]) +{ + /* compute inversion ober n384 of secret keys */ + MB_FUNC_NAME(ifma_tomont52_n384_)(inv_skey, skey); + ifma_aminv52_n384_mb8(inv_skey, inv_skey); /* 1/skeys mod n384 */ + MB_FUNC_NAME(ifma_frommont52_n384_)(inv_skey, inv_skey); +} + +/* +// compute r-component of the ECDSA signature +// +// r = ([skey]*G).x mod n384 +// +// note: pay attention on skey[] presenttaion +// it should be transposed and zero expanded +*/ +static __mb_mask nistp384_ecdsa_sign_r_mb8(U64 sign_r[], + const U64 skey[], + int8u pBuffer[]) +{ + /* compute ephemeral public keys */ + P384_POINT P; + + MB_FUNC_NAME(ifma_ec_nistp384_mul_pointbase_)(&P, skey); + + /* extract affine P.x */ + MB_FUNC_NAME(ifma_aminv52_p384_)(P.Z, P.Z); /* 1/Z */ + MB_FUNC_NAME(ifma_ams52_p384_)(P.Z, P.Z); /* 1/Z^2 */ + MB_FUNC_NAME(ifma_amm52_p384_)(P.X, P.X, P.Z); /* x = (X) * (1/Z^2) */ + + /* convert x-coordinate to regular and then tp Montgomery n384 */ + MB_FUNC_NAME(ifma_frommont52_p384_)(P.X, P.X); + MB_FUNC_NAME(ifma_fastred52_pn384_)(sign_r, P.X); /* fast reduction p => n */ + + return MB_FUNC_NAME(is_zero_FE384_)(sign_r); +} + +/* +// compute s-component of the ECDSA signature +// +// s = (inv_eph) * (msg + prv_skey*sign_r) mod n384 +*/ +static __mb_mask nistp384_ecdsa_sign_s_mb8(U64 sign_s[], + U64 msg[], + const U64 sign_r[], + U64 inv_eph_skey[], + U64 reg_skey[], + int8u pBuffer[]) +{ + __ALIGN64 U64 tmp[P384_LEN52]; + + /* conver to Montgomery over n384 domain */ + MB_FUNC_NAME(ifma_tomont52_n384_)(inv_eph_skey, inv_eph_skey); + MB_FUNC_NAME(ifma_tomont52_n384_)(tmp, sign_r); + MB_FUNC_NAME(ifma_tomont52_n384_)(msg, msg); + MB_FUNC_NAME(ifma_tomont52_n384_)(reg_skey, reg_skey); + + /* s = (inv_eph) * (msg + prv_skey*sign_r) mod n384 */ + MB_FUNC_NAME(ifma_amm52_n384_)(sign_s, reg_skey, tmp); + MB_FUNC_NAME(ifma_add52_n384_)(sign_s, sign_s, msg); + MB_FUNC_NAME(ifma_amm52_n384_)(sign_s, sign_s, inv_eph_skey); + MB_FUNC_NAME(ifma_frommont52_n384_)(sign_s, sign_s); + + return MB_FUNC_NAME(is_zero_FE384_)(sign_s); +} + +/* +// ECDSA signature verification algorithm +*/ +static __mb_mask nistp384_ecdsa_verify_mb8(U64 sign_r[], + U64 sign_s[], + U64 msg[], + P384_POINT* W) +{ + /* convert public key coords to Montgomery */ + MB_FUNC_NAME(ifma_tomont52_p384_)(W->X, W->X); + MB_FUNC_NAME(ifma_tomont52_p384_)(W->Y, W->Y); + MB_FUNC_NAME(ifma_tomont52_p384_)(W->Z, W->Z); + + __ALIGN64 U64 h1[P384_LEN52]; + __ALIGN64 U64 h2[P384_LEN52]; + + /* h = (sign_s)^(-1) */ + MB_FUNC_NAME(ifma_tomont52_n384_)(sign_s, sign_s); + MB_FUNC_NAME(ifma_aminv52_n384_)(sign_s, sign_s); + /* h1 = msg * h */ + MB_FUNC_NAME(ifma_tomont52_n384_)(h1, msg); + MB_FUNC_NAME(ifma_amm52_n384_)(h1, h1, sign_s); + MB_FUNC_NAME(ifma_frommont52_n384_)(h1, h1); + /* h2 = sign_r * h */ + MB_FUNC_NAME(ifma_tomont52_n384_)(h2, sign_r); + MB_FUNC_NAME(ifma_amm52_n384_)(h2, h2, sign_s); + MB_FUNC_NAME(ifma_frommont52_n384_)(h2, h2); + + int64u tmp[8][P384_LEN64]; + int64u* pa_tmp[8] = { tmp[0], tmp[1], tmp[2], tmp[3], + tmp[4], tmp[5], tmp[6], tmp[7] }; + + ifma_mb8_to_BNU(pa_tmp, (const int64u(*)[8])h1, P384_BITSIZE); + ifma_BNU_transpose_copy((int64u(*)[8])h1, (const int64u(**))pa_tmp, P384_BITSIZE); + + ifma_mb8_to_BNU(pa_tmp, (const int64u(*)[8])h2, P384_BITSIZE); + ifma_BNU_transpose_copy((int64u(*)[8])h2, (const int64u(**))pa_tmp, P384_BITSIZE); + + h1[P384_LEN64] = get_zero64(); + h2[P384_LEN64] = get_zero64(); + + P384_POINT P; + + // P = h1*G + h2*W + MB_FUNC_NAME(ifma_ec_nistp384_mul_point_)(W, W, h2); + MB_FUNC_NAME(ifma_ec_nistp384_mul_pointbase_)(&P, h1); + MB_FUNC_NAME(ifma_ec_nistp384_add_point_)(&P, &P, W); + + // P != 0 + __mb_mask signature_err_mask = MB_FUNC_NAME(is_zero_point_cordinate_)(P.Z); + + /* sign_r_restored = P.X mod n */ + __ALIGN64 U64 sign_r_restored[P384_LEN52]; + MB_FUNC_NAME(get_nistp384_ec_affine_coords_)(sign_r_restored, NULL, &P); + MB_FUNC_NAME(ifma_frommont52_p384_)(sign_r_restored, sign_r_restored); + MB_FUNC_NAME(ifma_fastred52_pn384_)(sign_r_restored, sign_r_restored); + + /* sign_r_restored != sign_r */ + signature_err_mask |= ~(MB_FUNC_NAME(cmp_eq_FE384_)(sign_r_restored, sign_r)); + + return signature_err_mask; +} + +/* +// ECDSA kernels +*/ + +/* +// pre-computation of ECDSA signature +// +// pa_inv_eph_skey[] array of pointers to the inversion of signer's ephemeral private keys +// pa_sign_rp[] array of pointers to the r-components of the signatures +// pa_eph_skey[] array of pointers to the ephemeral (nonce) signer's ephemeral private keys +// pBuffer pointer to the scratch buffer +// +// function computes two values that does not depend on the message to be signed +// - inversion of signer's ephemeral (nonce) keys +// - r-component of the signature +// and are later used during the signing process +*/ +DLL_PUBLIC +mbx_status mbx_nistp384_ecdsa_sign_setup_mb8(int64u* pa_inv_eph_skey[8], + int64u* pa_sign_rp[8], + const int64u* const pa_eph_skey[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* test input pointers */ + if(NULL==pa_inv_eph_skey || NULL==pa_sign_rp || NULL==pa_eph_skey) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check data pointers */ + for(buf_no=0; buf_no<8; buf_no++) { + int64u* pinv_key = pa_inv_eph_skey[buf_no]; + int64u* psign_rp = pa_sign_rp[buf_no]; + const int64u* pkey = pa_eph_skey[buf_no]; + /* if any of pointer NULL set error status */ + if(NULL==pinv_key || NULL==psign_rp || NULL==pkey) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + if(!MBX_IS_ANY_OK_STS(status) ) + return status; + + /* convert keys into FE and compute inversion */ + U64 T[P384_LEN52]; + ifma_BNU_to_mb8((int64u (*)[8])T, pa_eph_skey, P384_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FE384_)(T), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear key's inversion */ + MB_FUNC_NAME(zero_)((int64u (*)[8])T, sizeof(T)/sizeof(U64)); + return status; + } + + nistp384_ecdsa_inv_keys_mb8(T, T, 0); + /* return results in suitable format */ + ifma_mb8_to_BNU(pa_inv_eph_skey, (const int64u(*)[8])T, P384_BITSIZE); + + /* clear key's inversion */ + MB_FUNC_NAME(zero_)((int64u (*)[8])T, sizeof(T)/sizeof(U64)); + + /* convert keys into scalars */ + U64 scalarz[P384_LEN64+1]; + ifma_BNU_transpose_copy((int64u (*)[8])scalarz, pa_eph_skey, P384_BITSIZE); + scalarz[P384_LEN64] = get_zero64(); + /* compute r-component of the DSA signature */ + int8u stt_mask = nistp384_ecdsa_sign_r_mb8(T, scalarz, pBuffer); + + /* clear copy of the ephemeral secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])scalarz, sizeof(scalarz)/sizeof(U64)); + + /* return results in suitable format */ + ifma_mb8_to_BNU(pa_sign_rp, (const int64u(*)[8])T, P384_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, stt_mask, MBX_STATUS_SIGNATURE_ERR); + return status; +} + +/* +// computes ECDSA signature +// +// pa_sign_r[] array of pointers to the r-components of the signatures +// pa_sign_s[] array of pointers to the s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// pa_sign_rp[] array of pointers to the pre-computed r-components of the signatures +// pa_inv_eph_skey[] array of pointers to the inversion of signer's ephemeral private keys +// pa_reg_skey[] array of pointers to the regular signer's ephemeral (nonce) private keys +// pBuffer pointer to the scratch buffer +*/ +DLL_PUBLIC +mbx_status mbx_nistp384_ecdsa_sign_complete_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_sign_rp[8], + const int64u* const pa_inv_eph_skey[8], + const int64u* const pa_reg_skey[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* test input pointers */ + if(NULL==pa_sign_r || NULL==pa_sign_s || NULL==pa_msg || + NULL==pa_sign_rp || NULL==pa_inv_eph_skey || NULL==pa_reg_skey) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check data pointers */ + for(buf_no=0; buf_no<8; buf_no++) { + int8u* psign_r = pa_sign_r[buf_no]; + int8u* psign_s = pa_sign_s[buf_no]; + const int8u* pmsg = pa_msg[buf_no]; + const int64u* psign_pr = pa_sign_rp[buf_no]; + const int64u* pinv_eph_key = pa_inv_eph_skey[buf_no]; + const int64u* preg_key = pa_reg_skey[buf_no]; + /* if any of pointer NULL set error status */ + if(NULL==psign_r || NULL==psign_s || NULL==pmsg || + NULL==psign_pr || NULL==pinv_eph_key || NULL==preg_key) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + if(!MBX_IS_ANY_OK_STS(status) ) + return status; + + __ALIGN64 U64 inv_eph[P384_LEN52]; + __ALIGN64 U64 reg_skey[P384_LEN52]; + __ALIGN64 U64 sign_r[P384_LEN52]; + __ALIGN64 U64 sign_s[P384_LEN52]; + __ALIGN64 U64 msg[P384_LEN52]; + + /* convert inv_eph, reg_skey, sign_r and message to mb format */ + ifma_BNU_to_mb8((int64u (*)[8])inv_eph, pa_inv_eph_skey, P384_BITSIZE); + ifma_BNU_to_mb8((int64u (*)[8])reg_skey, pa_reg_skey, P384_BITSIZE); + ifma_BNU_to_mb8((int64u (*)[8])sign_r, pa_sign_rp, P384_BITSIZE); + ifma_HexStr8_to_mb8((int64u (*)[8])msg, pa_msg, P384_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FE384_)(inv_eph), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FE384_)(reg_skey), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FE384_)(sign_r), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_n384_)(msg), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the ephemeral secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])inv_eph, sizeof(inv_eph)/sizeof(U64)); + /* clear copy of the regular secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])reg_skey, sizeof(reg_skey)/sizeof(U64)); + return status; + } + + /* compute s- signature component: s = (inv_eph) * (msg + prv_skey*sign_r) mod n384 */ + nistp384_ecdsa_sign_s_mb8(sign_s, msg, sign_r, inv_eph, reg_skey, pBuffer); + + /* clear copy of the ephemeral secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])inv_eph, sizeof(inv_eph)/sizeof(U64)); + /* clear copy of the regular secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])reg_skey, sizeof(reg_skey)/sizeof(U64)); + + /* check if sign_r!=0 and sign_s!=0 */ + int8u stt_mask_r = MB_FUNC_NAME(is_zero_FE384_)(sign_r); + int8u stt_mask_s = MB_FUNC_NAME(is_zero_FE384_)(sign_s); + + /* convert sign_r and sing_s to strings */ + ifma_mb8_to_HexStr8(pa_sign_r, (const int64u(*)[8])sign_r, P384_BITSIZE); + ifma_mb8_to_HexStr8(pa_sign_s, (const int64u(*)[8])sign_s, P384_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, stt_mask_r, MBX_STATUS_SIGNATURE_ERR); + status |= MBX_SET_STS_BY_MASK(status, stt_mask_s, MBX_STATUS_SIGNATURE_ERR); + return status; +} + +/* +// Computes ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// pa_eph_skey[] array of pointers to the signer's ephemeral (nonce) private keys +// pa_reg_skey[] array of pointers to the signer's regular (long term) private keys +// pBuffer pointer to the scratch buffer +*/ +DLL_PUBLIC +mbx_status mbx_nistp384_ecdsa_sign_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_eph_skey[8], + const int64u* const pa_reg_skey[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* test input pointers */ + if(NULL==pa_sign_r || NULL==pa_sign_s || NULL==pa_msg || NULL==pa_eph_skey || NULL==pa_reg_skey) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + /* check data pointers */ + for(buf_no=0; buf_no<8; buf_no++) { + int8u* psign_r = pa_sign_r[buf_no]; + int8u* psign_s = pa_sign_s[buf_no]; + const int8u* pmsg = pa_msg[buf_no]; + const int64u* peph_key = pa_eph_skey[buf_no]; + const int64u* preg_key = pa_reg_skey[buf_no]; + /* if any of pointer NULL set error status */ + if(NULL==psign_r || NULL==psign_s || NULL==pmsg || NULL==peph_key || NULL==preg_key) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + if(!MBX_IS_ANY_OK_STS(status) ) + return status; + + __ALIGN64 U64 inv_eph_key[P384_LEN52]; + __ALIGN64 U64 reg_key[P384_LEN52]; + __ALIGN64 U64 sign_r[P384_LEN52]; + __ALIGN64 U64 sign_s[P384_LEN52]; + __ALIGN64 U64 scalar[P384_LEN64+1]; + __ALIGN64 U64 msg[P384_LEN52]; + + /* convert ephemeral keys into FE */ + ifma_BNU_to_mb8((int64u (*)[8])inv_eph_key, pa_eph_skey, P384_BITSIZE); + /* convert epphemeral keys into sclar*/ + ifma_BNU_transpose_copy((int64u (*)[8])scalar, pa_eph_skey, P384_BITSIZE); + scalar[P384_LEN64] = get_zero64(); + /* convert reg_skey*/ + ifma_BNU_to_mb8((int64u (*)[8])reg_key, pa_reg_skey, P384_BITSIZE); + /* convert message */ + ifma_HexStr8_to_mb8((int64u (*)[8])msg, pa_msg, P384_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FE384_)(inv_eph_key), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FE384_)(reg_key), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_n384_)(msg), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the ephemeral secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])inv_eph_key, sizeof(inv_eph_key)/sizeof(U64)); + MB_FUNC_NAME(zero_)((int64u (*)[8])scalar, sizeof(scalar)/sizeof(U64)); + /* clear copy of the regular secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])reg_key, sizeof(reg_key)/sizeof(U64)); + return status; + } + + /* compute inversion */ + nistp384_ecdsa_inv_keys_mb8(inv_eph_key, inv_eph_key, pBuffer); + /* compute r-component */ + nistp384_ecdsa_sign_r_mb8(sign_r, scalar, pBuffer); + /* compute s-component */ + nistp384_ecdsa_sign_s_mb8(sign_s, msg, sign_r, inv_eph_key, reg_key, pBuffer); + + /* clear copy of the ephemeral secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])inv_eph_key, sizeof(inv_eph_key)/sizeof(U64)); + MB_FUNC_NAME(zero_)((int64u (*)[8])scalar, sizeof(scalar)/sizeof(U64)); + + /* clear copy of the regular secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])reg_key, sizeof(reg_key)/sizeof(U64)); + + /* check if sign_r!=0 and sign_s!=0 */ + int8u stt_mask_r = MB_FUNC_NAME(is_zero_FE384_)(sign_r); + int8u stt_mask_s = MB_FUNC_NAME(is_zero_FE384_)(sign_s); + + /* convert singnature components to strings */ + ifma_mb8_to_HexStr8(pa_sign_r, (const int64u(*)[8])sign_r, P384_BITSIZE); + ifma_mb8_to_HexStr8(pa_sign_s, (const int64u(*)[8])sign_s, P384_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, stt_mask_r, MBX_STATUS_SIGNATURE_ERR); + status |= MBX_SET_STS_BY_MASK(status, stt_mask_s, MBX_STATUS_SIGNATURE_ERR); + return status; +} + +/* +// ECDSA signature verification +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_msg[] array of pointers to the message representatives that have been signed +// pa_pubx[] array of pointers to the signer's public keys X-coordinates +// pa_puby[] array of pointers to the signer's public keys Y-coordinates +// pa_pubz[] array of pointers to the signer's public keys Z-coordinates (or NULL, if affine coordinate requested) +// pBuffer pointer to the scratch buffer +*/ +DLL_PUBLIC +mbx_status mbx_nistp384_ecdsa_verify_mb8(const int8u* const pa_sign_r[8], + const int8u* const pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + int use_jproj_coords = NULL != pa_pubz; + + /* test input pointers */ + if (NULL == pa_sign_r || NULL == pa_sign_s || NULL == pa_msg || NULL == pa_pubx || NULL == pa_puby) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check data pointers */ + for (buf_no = 0; buf_no < 8; buf_no++) { + const int8u* psign_r = pa_sign_r[buf_no]; + const int8u* psign_s = pa_sign_s[buf_no]; + const int8u* pmsg = pa_msg[buf_no]; + const int64u* pubx = pa_pubx[buf_no]; + const int64u* puby = pa_puby[buf_no]; + const int64u* pubz = use_jproj_coords ? pa_pubz[buf_no] : NULL; + + /* if any of pointer NULL set error status */ + if (NULL == psign_r || NULL == psign_s || NULL == pmsg || NULL == pubx || NULL == puby || (use_jproj_coords && NULL == pubz)) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + /* if all pointers NULL exit */ + if (!MBX_IS_ANY_OK_STS(status)) + return status; + + __ALIGN64 U64 msg[P384_LEN52]; + __ALIGN64 U64 sign_r[P384_LEN52]; + __ALIGN64 U64 sign_s[P384_LEN52]; + + /* convert input params */ + ifma_HexStr8_to_mb8((int64u(*)[8])msg, pa_msg, P384_BITSIZE); + ifma_HexStr8_to_mb8((int64u(*)[8])sign_r, pa_sign_r, P384_BITSIZE); + ifma_HexStr8_to_mb8((int64u(*)[8])sign_s, pa_sign_s, P384_BITSIZE); + + /* + // check, that sign_r and sign_s is in [1, n – 1] + // it's equivalent to sign_ != 0 (because it is unsigned value) and sign_ < n + */ + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_n384_)(msg), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_n384_)(sign_r), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_n384_)(sign_s), MBX_STATUS_MISMATCH_PARAM_ERR); + + if (!MBX_IS_ANY_OK_STS(status)) + return status; + + P384_POINT W; + + ifma_BNU_to_mb8((int64u(*)[8])W.X, (const int64u* (*))pa_pubx, P384_BITSIZE); + ifma_BNU_to_mb8((int64u(*)[8])W.Y, (const int64u* (*))pa_puby, P384_BITSIZE); + if (use_jproj_coords) + ifma_BNU_to_mb8((int64u(*)[8])W.Z, (const int64u* (*))pa_pubz, P384_BITSIZE); + else + MB_FUNC_NAME(mov_FE384_)(W.Z, (U64*)ones); + + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_p384_)(W.X), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_p384_)(W.Y), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_p384_)(W.Z), MBX_STATUS_MISMATCH_PARAM_ERR); + + if (!MBX_IS_ANY_OK_STS(status)) + return status; + + __mb_mask signature_err_mask = nistp384_ecdsa_verify_mb8(sign_r, sign_s, msg, &W); + status |= MBX_SET_STS_BY_MASK(status, signature_err_mask, MBX_STATUS_SIGNATURE_ERR); + + return status; +} + +/* +// OpenSSL's specific implementations +*/ +#ifndef BN_OPENSSL_DISABLE + +static void reverse_inplace(int8u* inpout, int len) +{ + int mudpoint = len/2; + for(int n=0; n + +#include +#include +#include +#include + +#ifndef BN_OPENSSL_DISABLE +#include +#include +#ifdef OPENSSL_IS_BORINGSSL +#include +#endif +#endif + +/* +// common functions +*/ + +/* +// compute secret key inversion +// +// inv_skey 1/skey mod n521 +// +// note: pay attention on skey[] presenttaion +// it should be FE element of N521 basef GF +*/ +static void nistp521_ecdsa_inv_keys_mb8(U64 inv_skey[], + const U64 skey[], + int8u pBuffer[]) +{ + /* compute inversion ober n521 of secret keys */ + MB_FUNC_NAME(ifma_tomont52_n521_)(inv_skey, skey); + ifma_aminv52_n521_mb8(inv_skey, inv_skey); /* 1/skeys mod n521 */ + MB_FUNC_NAME(ifma_frommont52_n521_)(inv_skey, inv_skey); +} + +/* +// compute r-component of the ECDSA signature +// +// r = ([skey]*G).x mod n521 +// +// note: pay attention on skey[] presenttaion +// it should be transposed and zero expanded +*/ +static __mb_mask nistp521_ecdsa_sign_r_mb8(U64 sign_r[], + const U64 skey[], + int8u pBuffer[]) +{ + /* compute ephemeral public keys */ + P521_POINT P; + + MB_FUNC_NAME(ifma_ec_nistp521_mul_pointbase_)(&P, skey); + + /* extract affine P.x */ + MB_FUNC_NAME(ifma_aminv52_p521_)(P.Z, P.Z); /* 1/Z */ + MB_FUNC_NAME(ifma_ams52_p521_)(P.Z, P.Z); /* 1/Z^2 */ + MB_FUNC_NAME(ifma_amm52_p521_)(P.X, P.X, P.Z); /* x = (X) * (1/Z^2) */ + + /* convert x-coordinate to regular and then tp Montgomery n521 */ + MB_FUNC_NAME(ifma_frommont52_p521_)(P.X, P.X); + MB_FUNC_NAME(ifma_fastred52_pn521_)(sign_r, P.X); /* fast reduction p => n */ + + return MB_FUNC_NAME(is_zero_FE521_)(sign_r); +} + +/* +// compute s-component of the ECDSA signature +// +// s = (inv_eph) * (msg + prv_skey*sign_r) mod n521 +*/ +static __mb_mask nistp521_ecdsa_sign_s_mb8(U64 sign_s[], + U64 msg[], + const U64 sign_r[], + U64 inv_eph_skey[], + U64 reg_skey[], + int8u pBuffer[]) +{ + __ALIGN64 U64 tmp[P521_LEN52]; + + /* conver to Montgomery over n521 domain */ + MB_FUNC_NAME(ifma_tomont52_n521_)(inv_eph_skey, inv_eph_skey); + MB_FUNC_NAME(ifma_tomont52_n521_)(tmp, sign_r); + MB_FUNC_NAME(ifma_tomont52_n521_)(msg, msg); + MB_FUNC_NAME(ifma_tomont52_n521_)(reg_skey, reg_skey); + + /* s = (inv_eph) * (msg + prv_skey*sign_r) mod n521 */ + MB_FUNC_NAME(ifma_amm52_n521_)(sign_s, reg_skey, tmp); + MB_FUNC_NAME(ifma_add52_n521_)(sign_s, sign_s, msg); + MB_FUNC_NAME(ifma_amm52_n521_)(sign_s, sign_s, inv_eph_skey); + MB_FUNC_NAME(ifma_frommont52_n521_)(sign_s, sign_s); + + return MB_FUNC_NAME(is_zero_FE521_)(sign_s); +} + +/* +// ECDSA signature verification algorithm +*/ +static __mb_mask nistp521_ecdsa_verify_mb8(U64 sign_r[], + U64 sign_s[], + U64 msg[], + P521_POINT* W) +{ + /* convert public key coords to Montgomery */ + MB_FUNC_NAME(ifma_tomont52_p521_)(W->X, W->X); + MB_FUNC_NAME(ifma_tomont52_p521_)(W->Y, W->Y); + MB_FUNC_NAME(ifma_tomont52_p521_)(W->Z, W->Z); + + __ALIGN64 U64 h1[P521_LEN52]; + __ALIGN64 U64 h2[P521_LEN52]; + + /* h = (sign_s)^(-1) */ + MB_FUNC_NAME(ifma_tomont52_n521_)(sign_s, sign_s); + MB_FUNC_NAME(ifma_aminv52_n521_)(sign_s, sign_s); + /* h1 = msg * h */ + MB_FUNC_NAME(ifma_tomont52_n521_)(h1, msg); + MB_FUNC_NAME(ifma_amm52_n521_)(h1, h1, sign_s); + MB_FUNC_NAME(ifma_frommont52_n521_)(h1,h1); + /* h2 = sign_r * h */ + MB_FUNC_NAME(ifma_tomont52_n521_)(h2, sign_r); + MB_FUNC_NAME(ifma_amm52_n521_)(h2, h2, sign_s); + MB_FUNC_NAME(ifma_frommont52_n521_)(h2,h2); + + int64u tmp[8][P521_LEN64]; + int64u* pa_tmp[8] = {tmp[0], tmp[1], tmp[2], tmp[3], + tmp[4], tmp[5], tmp[6], tmp[7]}; + + ifma_mb8_to_BNU(pa_tmp, (const int64u(*)[8])h1, P521_BITSIZE); + ifma_BNU_transpose_copy((int64u (*)[8])h1, (const int64u(**))pa_tmp, P521_BITSIZE); + + ifma_mb8_to_BNU(pa_tmp, (const int64u(*)[8])h2, P521_BITSIZE); + ifma_BNU_transpose_copy((int64u (*)[8])h2, (const int64u(**))pa_tmp, P521_BITSIZE); + + h1[P521_LEN64] = get_zero64(); + h2[P521_LEN64] = get_zero64(); + + P521_POINT P; + + // P = h1*G + h2*W + MB_FUNC_NAME(ifma_ec_nistp521_mul_point_)(W, W, h2); + MB_FUNC_NAME(ifma_ec_nistp521_mul_pointbase_)(&P, h1); + MB_FUNC_NAME(ifma_ec_nistp521_add_point_)(&P, &P, W); + + // P != 0 + __mb_mask signature_err_mask = MB_FUNC_NAME(is_zero_point_cordinate_)(P.Z); + + /* sign_r_restored = P.X mod n */ + __ALIGN64 U64 sign_r_restored[P521_LEN52]; + MB_FUNC_NAME(get_nistp521_ec_affine_coords_)(sign_r_restored, NULL, &P); + MB_FUNC_NAME(ifma_frommont52_p521_)(sign_r_restored, sign_r_restored); + MB_FUNC_NAME(ifma_fastred52_pn521_)(sign_r_restored, sign_r_restored); + + /* sign_r_restored != sign_r */ + signature_err_mask |= ~(MB_FUNC_NAME(cmp_eq_FE521_)(sign_r_restored, sign_r)); + + return signature_err_mask; +} + +/* +// ECDSA kernels +*/ + +/* +// pre-computation of ECDSA signature +// +// pa_inv_eph_skey[] array of pointers to the inversion of signer's ephemeral private keys +// pa_sign_rp[] array of pointers to the r-components of the signatures +// pa_eph_skey[] array of pointers to the ephemeral (nonce) signer's ephemeral private keys +// pBuffer pointer to the scratch buffer +// +// function computes two values that does not depend on the message to be signed +// - inversion of signer's ephemeral (nonce) keys +// - r-component of the signature +// and are later used during the signing process +*/ +DLL_PUBLIC +mbx_status mbx_nistp521_ecdsa_sign_setup_mb8(int64u* pa_inv_eph_skey[8], + int64u* pa_sign_rp[8], + const int64u* const pa_eph_skey[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* test input pointers */ + if(NULL==pa_inv_eph_skey || NULL==pa_sign_rp || NULL==pa_eph_skey) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check data pointers */ + for(buf_no=0; buf_no<8; buf_no++) { + int64u* pinv_key = pa_inv_eph_skey[buf_no]; + int64u* psign_rp = pa_sign_rp[buf_no]; + const int64u* pkey = pa_eph_skey[buf_no]; + /* if any of pointer NULL set error status */ + if(NULL==pinv_key || NULL==psign_rp || NULL==pkey) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + if(!MBX_IS_ANY_OK_STS(status) ) + return status; + + /* convert keys into FE and compute inversion */ + U64 T[P521_LEN52]; + ifma_BNU_to_mb8((int64u (*)[8])T, pa_eph_skey, P521_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FE521_)(T), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear key's inversion */ + MB_FUNC_NAME(zero_)((int64u (*)[8])T, sizeof(T)/sizeof(U64)); + return status; + } + + nistp521_ecdsa_inv_keys_mb8(T, T, 0); + /* return results in suitable format */ + ifma_mb8_to_BNU(pa_inv_eph_skey, (const int64u(*)[8])T, P521_BITSIZE); + + /* clear key's inversion */ + MB_FUNC_NAME(zero_)((int64u (*)[8])T, sizeof(T)/sizeof(U64)); + + /* convert keys into scalars */ + U64 scalarz[P521_LEN64+1]; + ifma_BNU_transpose_copy((int64u (*)[8])scalarz, pa_eph_skey, P521_BITSIZE); + scalarz[P521_LEN64] = get_zero64(); + /* compute r-component of the DSA signature */ + int8u stt_mask = nistp521_ecdsa_sign_r_mb8(T, scalarz, pBuffer); + + /* clear copy of the ephemeral secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])scalarz, sizeof(scalarz)/sizeof(U64)); + + /* return results in suitable format */ + ifma_mb8_to_BNU(pa_sign_rp, (const int64u(*)[8])T, P521_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, stt_mask, MBX_STATUS_SIGNATURE_ERR); + return status; +} + +/* +// computes ECDSA signature +// +// pa_sign_r[] array of pointers to the r-components of the signatures +// pa_sign_s[] array of pointers to the s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// pa_sign_rp[] array of pointers to the pre-computed r-components of the signatures +// pa_inv_eph_skey[] array of pointers to the inversion of signer's ephemeral private keys +// pa_reg_skey[] array of pointers to the regular signer's ephemeral (nonce) private keys +// pBuffer pointer to the scratch buffer +*/ +DLL_PUBLIC +mbx_status mbx_nistp521_ecdsa_sign_complete_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_sign_rp[8], + const int64u* const pa_inv_eph_skey[8], + const int64u* const pa_reg_skey[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* test input pointers */ + if(NULL==pa_sign_r || NULL==pa_sign_s || NULL==pa_msg || + NULL==pa_sign_rp || NULL==pa_inv_eph_skey || NULL==pa_reg_skey) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check data pointers */ + for(buf_no=0; buf_no<8; buf_no++) { + int8u* psign_r = pa_sign_r[buf_no]; + int8u* psign_s = pa_sign_s[buf_no]; + const int8u* pmsg = pa_msg[buf_no]; + const int64u* psign_pr = pa_sign_rp[buf_no]; + const int64u* pinv_eph_key = pa_inv_eph_skey[buf_no]; + const int64u* preg_key = pa_reg_skey[buf_no]; + /* if any of pointer NULL set error status */ + if(NULL==psign_r || NULL==psign_s || NULL==pmsg || + NULL==psign_pr || NULL==pinv_eph_key || NULL==preg_key) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + if(!MBX_IS_ANY_OK_STS(status) ) + return status; + + __ALIGN64 U64 inv_eph[P521_LEN52]; + __ALIGN64 U64 reg_skey[P521_LEN52]; + __ALIGN64 U64 sign_r[P521_LEN52]; + __ALIGN64 U64 sign_s[P521_LEN52]; + __ALIGN64 U64 msg[P521_LEN52]; + + /* convert inv_eph, reg_skey, sign_r and message to mb format */ + ifma_BNU_to_mb8((int64u (*)[8])inv_eph, pa_inv_eph_skey, P521_BITSIZE); + ifma_BNU_to_mb8((int64u (*)[8])reg_skey, pa_reg_skey, P521_BITSIZE); + ifma_BNU_to_mb8((int64u (*)[8])sign_r, pa_sign_rp, P521_BITSIZE); + ifma_HexStr8_to_mb8((int64u (*)[8])msg, pa_msg, P521_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FE521_)(inv_eph), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FE521_)(reg_skey), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FE521_)(sign_r), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_n521_)(msg), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the ephemeral secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])inv_eph, sizeof(inv_eph)/sizeof(U64)); + /* clear copy of the regular secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])reg_skey, sizeof(reg_skey)/sizeof(U64)); + return status; + } + + /* compute s- signature component: s = (inv_eph) * (msg + prv_skey*sign_r) mod n521 */ + nistp521_ecdsa_sign_s_mb8(sign_s, msg, sign_r, inv_eph, reg_skey, pBuffer); + + /* clear copy of the ephemeral secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])inv_eph, sizeof(inv_eph)/sizeof(U64)); + /* clear copy of the regular secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])reg_skey, sizeof(reg_skey)/sizeof(U64)); + + /* check if sign_r!=0 and sign_s!=0 */ + int8u stt_mask_r = MB_FUNC_NAME(is_zero_FE521_)(sign_r); + int8u stt_mask_s = MB_FUNC_NAME(is_zero_FE521_)(sign_s); + + /* convert sign_r and sing_s to strings */ + ifma_mb8_to_HexStr8(pa_sign_r, (const int64u(*)[8])sign_r, P521_BITSIZE); + ifma_mb8_to_HexStr8(pa_sign_s, (const int64u(*)[8])sign_s, P521_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, stt_mask_r, MBX_STATUS_SIGNATURE_ERR); + status |= MBX_SET_STS_BY_MASK(status, stt_mask_s, MBX_STATUS_SIGNATURE_ERR); + return status; +} + +/* +// Computes ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// pa_eph_skey[] array of pointers to the signer's ephemeral (nonce) private keys +// pa_reg_skey[] array of pointers to the signer's regular (long term) private keys +// pBuffer pointer to the scratch buffer +*/ +DLL_PUBLIC +mbx_status mbx_nistp521_ecdsa_sign_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_eph_skey[8], + const int64u* const pa_reg_skey[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* test input pointers */ + if(NULL==pa_sign_r || NULL==pa_sign_s || NULL==pa_msg || NULL==pa_eph_skey || NULL==pa_reg_skey) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + /* check data pointers */ + for(buf_no=0; buf_no<8; buf_no++) { + int8u* psign_r = pa_sign_r[buf_no]; + int8u* psign_s = pa_sign_s[buf_no]; + const int8u* pmsg = pa_msg[buf_no]; + const int64u* peph_key = pa_eph_skey[buf_no]; + const int64u* preg_key = pa_reg_skey[buf_no]; + /* if any of pointer NULL set error status */ + if(NULL==psign_r || NULL==psign_s || NULL==pmsg || NULL==peph_key || NULL==preg_key) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + if(!MBX_IS_ANY_OK_STS(status) ) + return status; + + __ALIGN64 U64 inv_eph_key[P521_LEN52]; + __ALIGN64 U64 reg_key[P521_LEN52]; + __ALIGN64 U64 sign_r[P521_LEN52]; + __ALIGN64 U64 sign_s[P521_LEN52]; + __ALIGN64 U64 scalar[P521_LEN64+1]; + __ALIGN64 U64 msg[P521_LEN52]; + + /* convert ephemeral keys into FE */ + ifma_BNU_to_mb8((int64u (*)[8])inv_eph_key, pa_eph_skey, P521_BITSIZE); + /* convert epphemeral keys into sclar*/ + ifma_BNU_transpose_copy((int64u (*)[8])scalar, pa_eph_skey, P521_BITSIZE); + scalar[P521_LEN64] = get_zero64(); + /* convert reg_skey*/ + ifma_BNU_to_mb8((int64u (*)[8])reg_key, pa_reg_skey, P521_BITSIZE); + /* convert message */ + ifma_HexStr8_to_mb8((int64u (*)[8])msg, pa_msg, P521_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FE521_)(inv_eph_key), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FE521_)(reg_key), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_n521_)(msg), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the ephemeral secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])inv_eph_key, sizeof(inv_eph_key)/sizeof(U64)); + MB_FUNC_NAME(zero_)((int64u (*)[8])scalar, sizeof(scalar)/sizeof(U64)); + /* clear copy of the regular secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])reg_key, sizeof(reg_key)/sizeof(U64)); + return status; + } + + /* compute inversion */ + nistp521_ecdsa_inv_keys_mb8(inv_eph_key, inv_eph_key, pBuffer); + /* compute r-component */ + nistp521_ecdsa_sign_r_mb8(sign_r, scalar, pBuffer); + /* compute s-component */ + nistp521_ecdsa_sign_s_mb8(sign_s, msg, sign_r, inv_eph_key, reg_key, pBuffer); + + /* clear copy of the ephemeral secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])inv_eph_key, sizeof(inv_eph_key)/sizeof(U64)); + MB_FUNC_NAME(zero_)((int64u (*)[8])scalar, sizeof(scalar)/sizeof(U64)); + + /* clear copy of the regular secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])reg_key, sizeof(reg_key)/sizeof(U64)); + + /* check if sign_r!=0 and sign_s!=0 */ + int8u stt_mask_r = MB_FUNC_NAME(is_zero_FE521_)(sign_r); + int8u stt_mask_s = MB_FUNC_NAME(is_zero_FE521_)(sign_s); + + /* convert singnature components to strings */ + ifma_mb8_to_HexStr8(pa_sign_r, (const int64u(*)[8])sign_r, P521_BITSIZE); + ifma_mb8_to_HexStr8(pa_sign_s, (const int64u(*)[8])sign_s, P521_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, stt_mask_r, MBX_STATUS_SIGNATURE_ERR); + status |= MBX_SET_STS_BY_MASK(status, stt_mask_s, MBX_STATUS_SIGNATURE_ERR); + return status; +} + +/* +// Verifies ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// pa_pubx[] array of pointers to the public keys X-coordinates +// pa_puby[] array of pointers to the public keys Y-coordinates +// pa_pubz[] array of pointers to the public keys Z-coordinates +// pBuffer pointer to the scratch buffer +*/ +DLL_PUBLIC +mbx_status mbx_nistp521_ecdsa_verify_mb8(const int8u* const pa_sign_r[8], + const int8u* const pa_sign_s[8], + const int8u* const pa_msg[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + int use_jproj_coords = NULL!=pa_pubz; + + /* test input pointers */ + if(NULL==pa_pubx || NULL==pa_puby || NULL==pa_msg || NULL==pa_sign_r || NULL==pa_sign_s) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + const int64u* pubx = pa_pubx[buf_no]; + const int64u* puby = pa_puby[buf_no]; + const int64u* pubz = use_jproj_coords? pa_pubz[buf_no] : NULL; + const int8u* msg = pa_msg[buf_no]; + const int8u* r = pa_sign_r[buf_no]; + const int8u* s = pa_sign_s[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL==pubx || NULL==puby || NULL==msg || NULL== r || NULL==s || (use_jproj_coords && NULL==pubz)) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + /* if all pointers NULL exit */ + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + __ALIGN64 U64 msg[P521_LEN52]; + __ALIGN64 U64 sign_r[P521_LEN52]; + __ALIGN64 U64 sign_s[P521_LEN52]; + + /* convert input params */ + ifma_HexStr8_to_mb8((int64u (*)[8])msg, pa_msg, P521_BITSIZE); + ifma_HexStr8_to_mb8((int64u (*)[8])sign_r, pa_sign_r, P521_BITSIZE); + ifma_HexStr8_to_mb8((int64u (*)[8])sign_s, pa_sign_s, P521_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_n521_)(msg), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_n521_)(sign_r), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_n521_)(sign_s), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + P521_POINT W; + + ifma_BNU_to_mb8((int64u (*)[8])W.X, (const int64u* (*))pa_pubx, P521_BITSIZE); + ifma_BNU_to_mb8((int64u (*)[8])W.Y, (const int64u* (*))pa_puby, P521_BITSIZE); + if(use_jproj_coords) + ifma_BNU_to_mb8((int64u (*)[8])W.Z, (const int64u* (*))pa_pubz, P521_BITSIZE); + else + MB_FUNC_NAME(mov_FE521_)(W.Z, (U64*)ones); + + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_p521_)(W.X), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_p521_)(W.Y), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_p521_)(W.Z), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + __mb_mask signature_err_mask = nistp521_ecdsa_verify_mb8(sign_r,sign_s,msg, &W); + status |= MBX_SET_STS_BY_MASK(status, signature_err_mask, MBX_STATUS_SIGNATURE_ERR); + + return status; +} + +/* +// OpenSSL's specific implementations +*/ +#ifndef BN_OPENSSL_DISABLE + +static void reverse_inplace(int8u* inpout, int len) +{ + int mudpoint = len/2; + for(int n=0; n +#include + + +/* simplify naming */ +#define sqr MB_FUNC_NAME(ifma_ams52_p256_) +#define mul MB_FUNC_NAME(ifma_amm52_p256_) +#define add MB_FUNC_NAME(ifma_add52_p256_) +#define sub MB_FUNC_NAME(ifma_sub52_p256_) +#define mul2 MB_FUNC_NAME(ifma_double52_p256_) +#define mul3 MB_FUNC_NAME(ifma_tripple52_p256_) +#define div2 MB_FUNC_NAME(ifma_half52_p256_) + + +/* +// Presentation of point at infinity: +// - projective (X : Y : 0) +// - affine (0 : 0) +*/ + +/* +// R(X3:Y3:Z3) = [2]P(X1:Y1:Z1) +// +// formulas: +// A = 4*X1*Y1^2 +// B = 3*(X1^2-Z1^4) +// X3= B^2 -2*A +// Y3= B*(A-X3) -8*Y1^4 +// Z3= 2*Y1*Z1 +// +// cost: 4S+4M+9A +// +*/ +void MB_FUNC_NAME(ifma_ec_nistp256_dbl_point_)(P256_POINT* r, const P256_POINT* p) +{ + __ALIGN64 U64 T[P256_LEN52]; + __ALIGN64 U64 U[P256_LEN52]; + __ALIGN64 U64 V[P256_LEN52]; + __ALIGN64 U64 A[P256_LEN52]; + __ALIGN64 U64 B[P256_LEN52]; + + const U64* X1 = p->X; /* input point */ + const U64* Y1 = p->Y; + const U64* Z1 = p->Z; + U64* X3 = r->X; /* output point */ + U64* Y3 = r->Y; + U64* Z3 = r->Z; + + mul2(T, Y1); /* T = 2*Y1 */ + + sqr(V, T); /* V = 4*Y1^2 */ /* sqr_dual */ + sqr(U, Z1); /* U = Z1^2 */ + + sub(B, X1, U); /* B = X1-Z1^2 */ + add(U, X1, U); /* U = X1+Z1^2 */ + + mul(A, V, X1); /* A = 4*X*Y1^2 */ /* mul_dual */ + mul(B, B, U); /* B = (X1^2-Z1^4) */ + + mul2(X3, A); /* X3 = 2*A */ + mul3(B, B); /* B = 3*(X1^2-Z1^4) */ + + sqr(U, B); /* U = B^2 */ /* sqr_dual */ + sqr(Y3, V); /* Y3= V^2 = 16*Y1^4 */ + + sub(X3, U, X3); /* X3=B^2 - 2*A */ + div2(Y3,Y3); /* Y3=Y3/2 = 8*Y1^4 */ + + sub(U, A, X3); /* U = A-X3 */ + + mul(Z3, T, Z1); /* Z3= 2*Y1*Z1 */ /* mul_dual */ + mul(U, U, B); /* U = B*(A-X3) */ + + sub(Y3, U, Y3); /* Y3 = B*(A-X3) -8*Y1^4 */ +} + + +/* +// R(X3:Y3:Z3) = P(X1:Y1:Z1) + Q(X2:Y2:Z2) +// +// formulas: +// A = X1*Z2^2 B = X2*Z1^2 C = Y1*Z2^3 D = Y2*Z1^3 +// E = B-A F = D-C +// X3= -E^3 -2*A*E^2 + F^2 +// Y3= -C*E^3 + F*(A*E^2 -X3) +// Z3= Z1*Z2*E +// +// cost: 4S+12M+7A +// +*/ +void MB_FUNC_NAME(ifma_ec_nistp256_add_point_)(P256_POINT* r, const P256_POINT* p, const P256_POINT* q) +{ + /* coordinates of p */ + const U64* X1 = p->X; + const U64* Y1 = p->Y; + const U64* Z1 = p->Z; + __mb_mask p_at_infinity = MB_FUNC_NAME(is_zero_point_cordinate_)(p->Z); + + /* coordinates of q */ + const U64* X2 = q->X; + const U64* Y2 = q->Y; + const U64* Z2 = q->Z; + __mb_mask q_at_infinity = MB_FUNC_NAME(is_zero_point_cordinate_)(q->Z); + + /* coordinates of temp point T(X3:Y3:Z3) */ + __ALIGN64 U64 X3[P256_LEN52]; + __ALIGN64 U64 Y3[P256_LEN52]; + __ALIGN64 U64 Z3[P256_LEN52]; + + /* temporary */ + __ALIGN64 U64 U1[P256_LEN52]; + __ALIGN64 U64 U2[P256_LEN52]; + __ALIGN64 U64 S1[P256_LEN52]; + __ALIGN64 U64 S2[P256_LEN52]; + __ALIGN64 U64 H[P256_LEN52]; + __ALIGN64 U64 R[P256_LEN52]; + + mul(S1, Y1, Z2); /* S1 = Y1*Z2 */ + sqr(U1, Z2); /* U1 = Z2^2 */ + + mul(S2, Y2, Z1); /* S2 = Y2*Z1 */ + sqr(U2, Z1); /* U2 = Z1^2 */ + + mul(S1, S1, U1); /* S1 = Y1*Z2^3 */ + mul(S2, S2, U2); /* S2 = Y2*Z1^3 */ + + mul(U1, X1, U1); /* U1 = X1*Z2^2 */ + mul(U2, X2, U2); /* U2 = X2*Z1^2 */ + + sub(R, S2, S1); /* R = S2-S1 */ + sub(H, U2, U1); /* H = U2-U1 */ + + /* check if affine (p.x:p.y) == (q.x:q.y) and and do doubling if this happens */ + __mb_mask x_are_equal = MB_FUNC_NAME(is_zero_FE256_)(H); + __mb_mask y_are_equal = MB_FUNC_NAME(is_zero_FE256_)(R); + __mb_mask points_are_equal = (x_are_equal & y_are_equal & (~p_at_infinity) & (~q_at_infinity)); + + P256_POINT P2; + MB_FUNC_NAME(set_point_to_infinity_)(&P2); + if (points_are_equal) { + MB_FUNC_NAME(ifma_ec_nistp256_dbl_point_)(&P2, p); + } + + mul(Z3, Z1, Z2); /* Z3 = Z1*Z2 */ + sqr(U2, H); /* U2 = H^2 */ + mul(Z3, Z3, H); /* Z3 = (Z1*Z2)*H */ + sqr(S2, R); /* S2 = R^2 */ + mul(H, H, U2); /* H = H^3 */ + + mul(U1, U1, U2); /* U1 = U1*H^2 */ + sub(X3, S2, H); /* X3 = R^2 - H^3 */ + mul2(U2, U1); /* U2 = 2*U1*H^2 */ + mul(S1, S1, H); /* S1 = S1*H^3 */ + sub(X3, X3, U2); /* X3 = (R^2 - H^3) -2*U1*H^2 */ + + sub(Y3, U1, X3); /* Y3 = R*(U1*H^2 - X3) -S1*H^3 */ + mul(Y3, Y3, R); + sub(Y3, Y3, S1); + + /* T = p_at_infinity? q : T */ + MB_FUNC_NAME(mask_mov_FE256_)(X3, X3, p_at_infinity, q->X); + MB_FUNC_NAME(mask_mov_FE256_)(Y3, Y3, p_at_infinity, q->Y); + MB_FUNC_NAME(mask_mov_FE256_)(Z3, Z3, p_at_infinity, q->Z); + /* T = q_at_infinity? p : T */ + MB_FUNC_NAME(mask_mov_FE256_)(X3, X3, q_at_infinity, p->X); + MB_FUNC_NAME(mask_mov_FE256_)(Y3, Y3, q_at_infinity, p->Y); + MB_FUNC_NAME(mask_mov_FE256_)(Z3, Z3, q_at_infinity, p->Z); + + /* r = points_are_equal? P2 : T */ + MB_FUNC_NAME(mask_mov_FE256_)(r->X, X3, points_are_equal, P2.X); + MB_FUNC_NAME(mask_mov_FE256_)(r->Y, Y3, points_are_equal, P2.Y); + MB_FUNC_NAME(mask_mov_FE256_)(r->Z, Z3, points_are_equal, P2.Z); +} + + +/* to Montgomery conversion constant +// r = 2^(P256_LEN52*DIGIT_SIZE) mod p256 +*/ +__ALIGN64 static const int64u p256_r_mb[P256_LEN52][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(0x0000000000000010) }, + { REP8_DECL(0x000f000000000000) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000ffeffffffffff) }, + { REP8_DECL(0x00000000000fffff) } +}; +const U64* MB_FUNC_NAME(ifma_ec_nistp256_coord_one_)(void) +{ + return (U64*)p256_r_mb; +} + + +/* +// R(X3:Y3:Z3) = P(X1:Y1:Z1) + Q(X2:Y2:Z2=1) +// +// formulas: +// A = X1*Z2^2 B = X2*Z1^2 C = Y1*Z2^3 D = Y2*Z1^3 +// E = B-A F = D-C +// X3= -E^3 -2*A*E^2 + F^2 +// Y3= -C*E^3 + F*(A*E^2 -X3) +// Z3= Z1*Z2*E +// +// if Z2=1, then +// A = X1 B = X2*Z1^2 C = Y1 D = Y2*Z1^3 +// E = B-X1 F = D-Y1 +// X3= -E^3 -2*X1*E^2 + F^2 +// Y3= -Y1*E^3 + F*(X1*E^2 -X3) +// Z3= Z1*E +// +// cost: 3S+8M+7A +*/ +void MB_FUNC_NAME(ifma_ec_nistp256_add_point_affine_)(P256_POINT* r, const P256_POINT* p, const P256_POINT_AFFINE* q) +{ + /* coordinates of p (projective) */ + const U64* X1 = p->X; + const U64* Y1 = p->Y; + const U64* Z1 = p->Z; + __mb_mask p_at_infinity = MB_FUNC_NAME(is_zero_point_cordinate_)(p->Z); + + /* coordinates of q (affine) */ + const U64* X2 = q->x; + const U64* Y2 = q->y; + __mb_mask q_at_infinity = MB_FUNC_NAME(is_zero_point_cordinate_)(q->x) + & MB_FUNC_NAME(is_zero_point_cordinate_)(q->y); + + /* coordinates of temp point T(X3:Y3:Z3) */ + __ALIGN64 U64 X3[P256_LEN52]; + __ALIGN64 U64 Y3[P256_LEN52]; + __ALIGN64 U64 Z3[P256_LEN52]; + + __ALIGN64 U64 U2[P256_LEN52]; + __ALIGN64 U64 S2[P256_LEN52]; + __ALIGN64 U64 H[P256_LEN52]; + __ALIGN64 U64 R[P256_LEN52]; + + sqr(R, Z1); // R = Z1^2 + mul(S2, Y2, Z1); // S2 = Y2*Z1 + mul(U2, X2, R); // U2 = X2*Z1^2 + mul(S2, S2, R); // S2 = Y2*Z1^3 + + sub(H, U2, X1); // H = U2-X1 + sub(R, S2, Y1); // R = S2-Y1 + + mul(Z3, H, Z1); // Z3 = H*Z1 + + sqr(U2, H); // U2 = H^2 + sqr(S2, R); // S2 = R^2 + mul(H, H, U2); // H = H^3 + + mul(U2, U2, X1); // U2 = X1*H^2 + + mul(Y3, H, Y1); // T = Y1*H^3 + + mul2(X3, U2); // X3 = 2*X1*H^2 + sub(X3, S2, X3); // X3 = R^2 - 2*X1*H^2 + sub(X3, X3, H); // X3 = R^2 - 2*X1*H^2 -H^3 + + sub(U2, U2, X3); // U2 = X1*H^2 - X3 + mul(U2, U2, R); // U2 = R*(X1*H^2 - X3) + sub(Y3, U2, Y3); // Y3 = -Y1*H^3 + R*(X1*H^2 - X3) + + /* T = p_at_infinity? q : T */ + MB_FUNC_NAME(mask_mov_FE256_)(X3, X3, p_at_infinity, q->x); + MB_FUNC_NAME(mask_mov_FE256_)(Y3, Y3, p_at_infinity, q->y); + MB_FUNC_NAME(mask_mov_FE256_)(Z3, Z3, p_at_infinity, (U64*)p256_r_mb); + /* T = q_at_infinity? p : T */ + MB_FUNC_NAME(mask_mov_FE256_)(X3, X3, q_at_infinity, p->X); + MB_FUNC_NAME(mask_mov_FE256_)(Y3, Y3, q_at_infinity, p->Y); + MB_FUNC_NAME(mask_mov_FE256_)(Z3, Z3, q_at_infinity, p->Z); + + /* r = T */ + MB_FUNC_NAME(mov_FE256_)(r->X, X3); + MB_FUNC_NAME(mov_FE256_)(r->Y, Y3); + MB_FUNC_NAME(mov_FE256_)(r->Z, Z3); +} + +void MB_FUNC_NAME(get_nistp256_ec_affine_coords_)(U64 x[], U64 y[], const P256_POINT* P) +{ + __ALIGN64 U64 invZ1[P256_LEN52]; + __ALIGN64 U64 invZn[P256_LEN52]; + + /* 1/Z and 1/Z^2 */ + MB_FUNC_NAME(ifma_aminv52_p256_)(invZ1, P->Z); + MB_FUNC_NAME(ifma_ams52_p256_)(invZn, invZ1); + + /* if affine P.x requested */ + if(x) + MB_FUNC_NAME(ifma_amm52_p256_)(x, P->X, invZn); + + /* if affine P.y requested */ + if(y) { + MB_FUNC_NAME(ifma_amm52_p256_)(invZn, invZn, invZ1); + MB_FUNC_NAME(ifma_amm52_p256_)(y, P->Y, invZn); + } +} + +////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////// + +static __NOINLINE void clear_secret_context(U64* wval, U64* dval, __mb_mask* dsign) +{ + *wval = get_zero64(); + *dval = get_zero64(); + *dsign = 0; + return; +} + +#define WIN_SIZE (4) + +/* + s = (Ipp8u)(~((wvalue >> ws) - 1)); //sign + d = (1 << (ws+1)) - wvalue - 1; // digit, win size "ws" + d = (d & s) | (wvaluen & ~s); + d = (d >> 1) + (d & 1); + *sign = s & 1; + *digit = (Ipp8u)d; +*/ +__INLINE void MB_FUNC_NAME(booth_recode_)(__mb_mask* sign, U64* dvalue, U64 wvalue) +{ + U64 one = set1(1); + U64 zero = get_zero64(); + U64 t = srli64(wvalue, WIN_SIZE); + __mb_mask s = cmp64_mask(t, zero, _MM_CMPINT_NE); + U64 d = sub64( sub64(set1(1<<(WIN_SIZE+1)), wvalue), one); + d = mask_mov64(wvalue, s, d); + U64 odd = and64(d, one); + d = add64( srli64(d, 1), odd); + + *sign = s; + *dvalue = d; +} + +/* extract point */ +static void MB_FUNC_NAME(extract_point_)(P256_POINT* r, const P256_POINT tbl[], U64 idx) +{ + /* decrenent index (the table noes not contain [0]*P */ + U64 idx_target = sub64(idx, set1(1)); + + /* assume the point at infinity is what need */ + P256_POINT R; + MB_FUNC_NAME(set_point_to_infinity_)(&R); + + /* find out what we actually need or just keep original infinity */ + int32u n; + for(n=0; n<(1<<(WIN_SIZE-1)); n++) { + U64 idx_curr = set1(n); + __mb_mask k = cmp64_mask(idx_curr, idx_target, _MM_CMPINT_EQ); + + /* R = k? tbl[] : R */ + MB_FUNC_NAME(secure_mask_mov_FE256_)(R.X, R.X, k, tbl[n].X); + MB_FUNC_NAME(secure_mask_mov_FE256_)(R.Y, R.Y, k, tbl[n].Y); + MB_FUNC_NAME(secure_mask_mov_FE256_)(R.Z, R.Z, k, tbl[n].Z); + } + MB_FUNC_NAME(mov_FE256_)(r->X, R.X); + MB_FUNC_NAME(mov_FE256_)(r->Y, R.Y); + MB_FUNC_NAME(mov_FE256_)(r->Z, R.Z); +} + +void MB_FUNC_NAME(ifma_ec_nistp256_mul_point_)(P256_POINT* r, const P256_POINT* p, const U64 scalar[]) +{ + /* pre-computed table */ + __ALIGN64 P256_POINT tbl[1<<(WIN_SIZE-1)]; + + /* + // compute tbl[] = [n]P, n=1,..,2^(WIN_SIZE-1): + // + // tbl[2*n] = tbl[2*n-1]+p + // tbl[2*n+1] = [2]*tbl[n] + */ + /* tbl[0] = p */ + MB_FUNC_NAME(mov_FE256_)(tbl[0].X, p->X); + MB_FUNC_NAME(mov_FE256_)(tbl[0].Y, p->Y); + MB_FUNC_NAME(mov_FE256_)(tbl[0].Z, p->Z); + /* tbl[1] = [2]*p */ + MB_FUNC_NAME(ifma_ec_nistp256_dbl_point_)(&tbl[1], p); + + int n; + for(n=1; n < (1<<(WIN_SIZE-1))/2; n++) { + MB_FUNC_NAME(ifma_ec_nistp256_add_point_)(&tbl[2*n], &tbl[2*n-1], p); + MB_FUNC_NAME(ifma_ec_nistp256_dbl_point_)(&tbl[2*n+1], &tbl[n]); + } + + P256_POINT R; + P256_POINT T; + U64 Ty[P256_LEN52]; + + /* + // point (LR) multiplication + */ + U64 idx_mask = set1( (1<<(WIN_SIZE+1))-1 ); + int bit = P256_BITSIZE-(P256_BITSIZE % WIN_SIZE); + int chunk_no = (bit-1)/64; + int chunk_shift = (bit-1)%64; + + /* first window */ + U64 wvalue = loadu64(&scalar[chunk_no]); + wvalue = and64( srli64(wvalue, chunk_shift), idx_mask); + + U64 dvalue; + __mb_mask dsign; + MB_FUNC_NAME(booth_recode_)(&dsign, &dvalue, wvalue); + MB_FUNC_NAME(extract_point_)(&R, tbl, dvalue); + + for(bit-=WIN_SIZE; bit>=WIN_SIZE; bit-=WIN_SIZE) { + /* doubling */ + MB_FUNC_NAME(ifma_ec_nistp256_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_nistp256_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_nistp256_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_nistp256_dbl_point_)(&R, &R); + #if (WIN_SIZE==5) + MB_FUNC_NAME(ifma_ec_nistp256_dbl_point_)(&R, &R); + #endif + + /* extract precomputed []P */ + chunk_no = (bit-1)/64; + chunk_shift = (bit-1)%64; + + wvalue = loadu64(&scalar[chunk_no]); + #if (_MSC_VER <= 1916) /* VS 2017 not supported _mm512_shrdv_epi64 */ + { + __m512i t_lo_ = _mm512_srlv_epi64(wvalue, set64(chunk_shift)); + __m512i t_hi_ = _mm512_sllv_epi64(loadu64(&scalar[chunk_no+1]), set64(64-chunk_shift)); + wvalue = or64(t_lo_, t_hi_); + } + #else + wvalue = _mm512_shrdv_epi64(wvalue, loadu64(&scalar[chunk_no+1]), set1((int32u)chunk_shift)); + #endif + wvalue = and64(wvalue, idx_mask); + + MB_FUNC_NAME(booth_recode_)(&dsign, &dvalue, wvalue); + MB_FUNC_NAME(extract_point_)(&T, tbl, dvalue); + + /* T = dsign? -T : T */ + MB_FUNC_NAME(ifma_neg52_p256_)(Ty, T.Y); + MB_FUNC_NAME(secure_mask_mov_FE256_)(T.Y, T.Y, dsign, Ty); + + /* acumulate T */ + MB_FUNC_NAME(ifma_ec_nistp256_add_point_)(&R, &R, &T); + } + + /* last window */ + MB_FUNC_NAME(ifma_ec_nistp256_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_nistp256_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_nistp256_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_nistp256_dbl_point_)(&R, &R); + #if (WIN_SIZE==5) + MB_FUNC_NAME(ifma_ec_nistp256_dbl_point_)(&R, &R); + #endif + + wvalue = loadu64(&scalar[0]); + wvalue = and64( slli64(wvalue, 1), idx_mask); + MB_FUNC_NAME(booth_recode_)(&dsign, &dvalue, wvalue); + MB_FUNC_NAME(extract_point_)(&T, tbl, dvalue); + + MB_FUNC_NAME(ifma_neg52_p256_)(Ty, T.Y); + MB_FUNC_NAME(secure_mask_mov_FE256_)(T.Y, T.Y, dsign, Ty); + + MB_FUNC_NAME(ifma_ec_nistp256_add_point_)(&R, &R, &T); + + /* r = R */ + MB_FUNC_NAME(mov_FE256_)(r->X, R.X); + MB_FUNC_NAME(mov_FE256_)(r->Y, R.Y); + MB_FUNC_NAME(mov_FE256_)(r->Z, R.Z); + + /* clear r (to fix potential secutity flaw in case of ecdh */ + MB_FUNC_NAME(zero_)((int64u (*)[8])&R, sizeof(R)/sizeof(U64)); + + /* clear stubs of secret scalar */ + clear_secret_context(&wvalue, &dvalue, &dsign); +} +#undef WIN_SIZE + + +////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////// + +#include + +#define BP_WIN_SIZE MUL_BASEPOINT_WIN_SIZE /* defined in the header above */ + +__INLINE void MB_FUNC_NAME(booth_recode_bp_)(__mb_mask* sign, U64* dvalue, U64 wvalue) +{ + U64 one = set1(1); + U64 zero = get_zero64(); + U64 t = srli64(wvalue, BP_WIN_SIZE); + __mb_mask s = cmp64_mask(t, zero, _MM_CMPINT_NE); + U64 d = sub64( sub64(set1(1<<(BP_WIN_SIZE+1)), wvalue), one); + d = mask_mov64(wvalue, s, d); + U64 odd = and64(d, one); + d = add64( srli64(d, 1), odd); + + *sign = s; + *dvalue = d; +} + +/* extract affine affine point */ +__INLINE void MB_FUNC_NAME(extract_point_affine_)(P256_POINT_AFFINE* r, const SINGLE_P256_POINT_AFFINE* tbl, U64 idx) +{ + /* decrement index (the table noes not contain [0]*P */ + U64 targIdx = sub64(idx, set1(1)); + + U64 ax0, ax1, ax2, ax3, ax4, ay0, ay1, ay2, ay3, ay4; + + /* assume the point at infinity is what need */ + ax0 = ax1 = ax2 = ax3 = ax4 = ay0 = ay1 = ay2 = ay3 = ay4 = get_zero64(); + + /* find out what we actually need or just keep original infinity */ + int n; + U64 currIdx = get_zero64(); + for(n=0; n<(1<<(BP_WIN_SIZE-1)); n++, tbl++, currIdx = add64(currIdx, set1(1))) { + __mb_mask k = cmp64_mask(currIdx, targIdx, _MM_CMPINT_EQ); + + /* R = k? set1( tbl[] ) : R */ + ax0 = mask_add64(ax0, k, ax0, set1( tbl->x[0] )); + ax1 = mask_add64(ax1, k, ax1, set1( tbl->x[1] )); + ax2 = mask_add64(ax2, k, ax2, set1( tbl->x[2] )); + ax3 = mask_add64(ax3, k, ax3, set1( tbl->x[3] )); + ax4 = mask_add64(ax4, k, ax4, set1( tbl->x[4] )); + + ay0 = mask_add64(ay0, k, ay0, set1( tbl->y[0] )); + ay1 = mask_add64(ay1, k, ay1, set1( tbl->y[1] )); + ay2 = mask_add64(ay2, k, ay2, set1( tbl->y[2] )); + ay3 = mask_add64(ay3, k, ay3, set1( tbl->y[3] )); + ay4 = mask_add64(ay4, k, ay4, set1( tbl->y[4] )); + } + + r->x[0] = ax0; + r->x[1] = ax1; + r->x[2] = ax2; + r->x[3] = ax3; + r->x[4] = ax4; + + r->y[0] = ay0; + r->y[1] = ay1; + r->y[2] = ay2; + r->y[3] = ay3; + r->y[4] = ay4; +} + +void MB_FUNC_NAME(ifma_ec_nistp256_mul_pointbase_)(P256_POINT* r, const U64 scalar[]) +{ + /* pre-computed table of base powers */ + SINGLE_P256_POINT_AFFINE* tbl = &ifma_ec_nistp256_bp_precomp[0][0]; + + P256_POINT R; + P256_POINT_AFFINE A; + U64 Ty[P256_LEN52]; + + /* R = O */ + MB_FUNC_NAME(set_point_to_infinity_)(&R); + + + /* + // base point (RL) multiplication + */ + U64 wvalue, dvalue; + __mb_mask dsign; + + U64 idx_mask = set1( (1<<(BP_WIN_SIZE+1))-1 ); + int bit = 0; + + /* first window - window[0] */ + wvalue = loadu64(&scalar[0]); + wvalue = and64( slli64(wvalue, 1), idx_mask); + MB_FUNC_NAME(booth_recode_bp_)(&dsign, &dvalue, wvalue); + MB_FUNC_NAME(extract_point_affine_)(&A, tbl, dvalue); + tbl+=BP_N_ENTRY; + + /* A = dsign? -A : A */ + MB_FUNC_NAME(ifma_neg52_p256_)(Ty, A.y); + MB_FUNC_NAME(secure_mask_mov_FE256_)(A.y, A.y, dsign, Ty); + + /* R += A */ + MB_FUNC_NAME(ifma_ec_nistp256_add_point_affine_)(&R, &R, &A); + + int chunk_no; + int chunk_shift; + for(bit+=BP_WIN_SIZE; bit<=P256_BITSIZE; bit+=BP_WIN_SIZE) { + chunk_no = (bit-1)/64; + chunk_shift = (bit-1)%64; + + wvalue = loadu64(&scalar[chunk_no]); + #if (_MSC_VER <= 1916) /* VS 2017 not supported _mm512_shrdv_epi64 */ + { + __m512i t_lo_ = _mm512_srlv_epi64(wvalue, set64(chunk_shift)); + __m512i t_hi_ = _mm512_sllv_epi64(loadu64(&scalar[chunk_no+1]), set64(64-chunk_shift)); + wvalue = or64(t_lo_, t_hi_); + } + #else + wvalue = _mm512_shrdv_epi64(wvalue, loadu64(&scalar[chunk_no+1]), set1((int32u)chunk_shift)); + #endif + wvalue = and64(wvalue, idx_mask); + + MB_FUNC_NAME(booth_recode_bp_)(&dsign, &dvalue, wvalue); + MB_FUNC_NAME(extract_point_affine_)(&A, tbl, dvalue); + tbl+=BP_N_ENTRY; + + /* A = dsign? -A : A */ + MB_FUNC_NAME(ifma_neg52_p256_)(Ty, A.y); + MB_FUNC_NAME(secure_mask_mov_FE256_)(A.y, A.y, dsign, Ty); + + /* R += A */ + MB_FUNC_NAME(ifma_ec_nistp256_add_point_affine_)(&R, &R, &A); + } + + /* r = R */ + MB_FUNC_NAME(mov_FE256_)(r->X, R.X); + MB_FUNC_NAME(mov_FE256_)(r->Y, R.Y); + MB_FUNC_NAME(mov_FE256_)(r->Z, R.Z); + + /* clear stubs of secret scalar */ + clear_secret_context(&wvalue, &dvalue, &dsign); +} +#undef BP_WIN_SIZE + + +/* P256 parameters: mont(a), mont(b) */ +__ALIGN64 static const int64u mont_a_p256_mb[P256_LEN52][8] = { + { REP8_DECL(0x000fffffffffffcf) }, + { REP8_DECL(0x00030fffffffffff) }, + { REP8_DECL(0x000000000000000) }, + { REP8_DECL(0x0000031000000000) }, + { REP8_DECL(0x0000ffffffcf0000) } +}; +__ALIGN64 static const int64u mont_b_p256_mb[P256_LEN52][8] = { + { REP8_DECL(0x000df6229c4bddfd) }, + { REP8_DECL(0x000ca8843090d89c) }, + { REP8_DECL(0x000212ed6acf005c) }, + { REP8_DECL(0x00083415a220abf7) }, + { REP8_DECL(0x0000c30061dd4874) } +}; + +/* +// We have a curve defined by a Weierstrass equation: y^2 = x^3 + a*x + b. +// +// The points are considered in Jacobian projective coordinates +// where (X, Y, Z) represents (x, y) = (X/Z^2, Y/Z^3). +// Substituting this and multiplying by Z^6 transforms the above equation into +// Y^2 = X^3 + a*X*Z^4 + b*Z^6 +// To test this, we add up the right-hand side in 'rh'. +*/ +__mb_mask MB_FUNC_NAME(ifma_is_on_curve_p256_)(const P256_POINT* p, int use_jproj_coords) +{ + U64 rh[P256_LEN52]; + U64 Z4[P256_LEN52], Z6[P256_LEN52], tmp[P256_LEN52]; + + /* rh := X^2 */ + MB_FUNC_NAME(ifma_ams52_p256_)(rh, p->X); + + /* if Z!=1, then rh = X^3 + a*X*Z^4 + b*Z^6 = X*(X^2 + a*X*Z^4) + b*Z^6 */ + if(use_jproj_coords) { + MB_FUNC_NAME(ifma_ams52_p256_)(tmp, p->Z); /* tmp = Z^2 */ + MB_FUNC_NAME(ifma_ams52_p256_)(Z4, tmp); /* Z4 = Z^4 */ + MB_FUNC_NAME(ifma_amm52_p256_)(Z6, Z4, tmp); /* Z6 = Z^6 */ + + MB_FUNC_NAME(ifma_add52_p256_)(tmp, Z4, Z4); /* tmp = 2*Z^4 */ + MB_FUNC_NAME(ifma_add52_p256_)(tmp, tmp, Z4); /* tmp = 2*Z^4 */ + MB_FUNC_NAME(ifma_sub52_p256_)(rh, rh, tmp); /* rh = X^2 + a*Z^4 */ + MB_FUNC_NAME(ifma_amm52_p256_)(rh, rh, p->X); /* rh = (X^2 + a*Z^4)*X */ + + MB_FUNC_NAME(ifma_amm52_p256_)(tmp, Z6, (U64*)mont_b_p256_mb); + MB_FUNC_NAME(ifma_add52_p256_)(rh, rh, tmp); /* rh = (X^2 + a*Z^4)*X + b*Z^6 */ + } + /* if Z==1, then rh = X^3 + a*X + b = X*(X^2 +a) b */ + else { + MB_FUNC_NAME(ifma_add52_p256_)(rh, rh, (U64*)mont_a_p256_mb); /* rh = X^2+a */ + MB_FUNC_NAME(ifma_amm52_p256_)(rh, rh, p->X); /* rh = (X^2+a)*X */ + MB_FUNC_NAME(ifma_add52_p256_)(rh, rh, (U64*)mont_b_p256_mb); /* rh = (X^2+a)*X + b */ + } + MB_FUNC_NAME(ifma_frommont52_p256_)(rh, rh); + + /* rl = tmp = Y^2 */ + MB_FUNC_NAME(ifma_ams52_p256_)(tmp, p->Y); + MB_FUNC_NAME(ifma_frommont52_p256_)(tmp, tmp); + + /* mask = rl==rh */ + __mb_mask is_on_curve_mask = MB_FUNC_NAME(cmp_eq_FE256_)(tmp, rh); + + return is_on_curve_mask; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecpoint_p384.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecpoint_p384.c new file mode 100644 index 0000000..4fdb27f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecpoint_p384.c @@ -0,0 +1,715 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + + +/* simplify naming */ +#define sqr MB_FUNC_NAME(ifma_ams52_p384_) +#define mul MB_FUNC_NAME(ifma_amm52_p384_) +#define add MB_FUNC_NAME(ifma_add52_p384_) +#define sub MB_FUNC_NAME(ifma_sub52_p384_) +#define mul2 MB_FUNC_NAME(ifma_double52_p384_) +#define mul3 MB_FUNC_NAME(ifma_tripple52_p384_) +#define div2 MB_FUNC_NAME(ifma_half52_p384_) + + +/* +// Presentation of point at infinity: +// - projective (X : Y : 0) +// - affine (0 : 0) +*/ + +/* +// R(X3:Y3:Z3) = [2]P(X1:Y1:Z1) +// +// formulas: +// A = 4*X1*Y1^2 +// B = 3*(X1^2-Z1^4) +// X3= B^2 -2*A +// Y3= B*(A-X3) -8*Y1^4 +// Z3= 2*Y1*Z1 +// +// cost: 4S+4M+9A +// +*/ +void MB_FUNC_NAME(ifma_ec_nistp384_dbl_point_)(P384_POINT* r, const P384_POINT* p) +{ + __ALIGN64 U64 T[P384_LEN52]; + __ALIGN64 U64 U[P384_LEN52]; + __ALIGN64 U64 V[P384_LEN52]; + __ALIGN64 U64 A[P384_LEN52]; + __ALIGN64 U64 B[P384_LEN52]; + + const U64* X1 = p->X; /* input point */ + const U64* Y1 = p->Y; + const U64* Z1 = p->Z; + U64* X3 = r->X; /* output point */ + U64* Y3 = r->Y; + U64* Z3 = r->Z; + + mul2(T, Y1); /* T = 2*Y1 */ + + sqr(V, T); /* V = 4*Y1^2 */ /* sqr_dual */ + sqr(U, Z1); /* U = Z1^2 */ + + sub(B, X1, U); /* B = X1-Z1^2 */ + add(U, X1, U); /* U = X1+Z1^2 */ + + mul(A, V, X1); /* A = 4*X*Y1^2 */ /* mul_dual */ + mul(B, B, U); /* B = (X1^2-Z1^4) */ + + mul2(X3, A); /* X3 = 2*A */ + mul3(B, B); /* B = 3*(X1^2-Z1^4) */ + + sqr(U, B); /* U = B^2 */ /* sqr_dual */ + sqr(Y3, V); /* Y3= V^2 = 16*Y1^4 */ + + sub(X3, U, X3); /* X3=B^2 - 2*A */ + div2(Y3,Y3); /* Y3=Y3/2 = 8*Y1^4 */ + + sub(U, A, X3); /* U = A-X3 */ + + mul(Z3, T, Z1); /* Z3= 2*Y1*Z1 */ /* mul_dual */ + mul(U, U, B); /* U = B*(A-X3) */ + + sub(Y3, U, Y3); /* Y3 = B*(A-X3) -8*Y1^4 */ +} + + +/* +// R(X3:Y3:Z3) = P(X1:Y1:Z1) + Q(X2:Y2:Z2) +// +// formulas: +// A = X1*Z2^2 B = X2*Z1^2 C = Y1*Z2^3 D = Y2*Z1^3 +// E = B-A F = D-C +// X3= -E^3 -2*A*E^2 + F^2 +// Y3= -C*E^3 + F*(A*E^2 -X3) +// Z3= Z1*Z2*E +// +// cost: 4S+12M+7A +// +*/ +void MB_FUNC_NAME(ifma_ec_nistp384_add_point_)(P384_POINT* r, const P384_POINT* p, const P384_POINT* q) +{ + /* coordinates of p */ + const U64* X1 = p->X; + const U64* Y1 = p->Y; + const U64* Z1 = p->Z; + __mb_mask p_at_infinity = MB_FUNC_NAME(is_zero_point_cordinate_)(p->Z); + + /* coordinates of q */ + const U64* X2 = q->X; + const U64* Y2 = q->Y; + const U64* Z2 = q->Z; + __mb_mask q_at_infinity = MB_FUNC_NAME(is_zero_point_cordinate_)(q->Z); + + /* coordinates of temp point T(X3:Y3:Z3) */ + __ALIGN64 U64 X3[P384_LEN52]; + __ALIGN64 U64 Y3[P384_LEN52]; + __ALIGN64 U64 Z3[P384_LEN52]; + + /* temporary */ + __ALIGN64 U64 U1[P384_LEN52]; + __ALIGN64 U64 U2[P384_LEN52]; + __ALIGN64 U64 S1[P384_LEN52]; + __ALIGN64 U64 S2[P384_LEN52]; + __ALIGN64 U64 H[P384_LEN52]; + __ALIGN64 U64 R[P384_LEN52]; + + mul(S1, Y1, Z2); /* S1 = Y1*Z2 */ + sqr(U1, Z2); /* U1 = Z2^2 */ + + mul(S2, Y2, Z1); /* S2 = Y2*Z1 */ + sqr(U2, Z1); /* U2 = Z1^2 */ + + mul(S1, S1, U1); /* S1 = Y1*Z2^3 */ + mul(S2, S2, U2); /* S2 = Y2*Z1^3 */ + + mul(U1, X1, U1); /* U1 = X1*Z2^2 */ + mul(U2, X2, U2); /* U2 = X2*Z1^2 */ + + sub(R, S2, S1); /* R = S2-S1 */ + sub(H, U2, U1); /* H = U2-U1 */ + + /* check if affine (p.x:p.y) == (q.x:q.y) and and do doubling if this happens */ + __mb_mask x_are_equal = MB_FUNC_NAME(is_zero_FE384_)(H); + __mb_mask y_are_equal = MB_FUNC_NAME(is_zero_FE384_)(R); + __mb_mask points_are_equal = (x_are_equal & y_are_equal & (~p_at_infinity) & (~q_at_infinity)); + + P384_POINT P2; + MB_FUNC_NAME(set_point_to_infinity_)(&P2); + if (points_are_equal) { + MB_FUNC_NAME(ifma_ec_nistp384_dbl_point_)(&P2, p); + } + + mul(Z3, Z1, Z2); /* Z3 = Z1*Z2 */ + sqr(U2, H); /* U2 = H^2 */ + mul(Z3, Z3, H); /* Z3 = (Z1*Z2)*H */ + sqr(S2, R); /* S2 = R^2 */ + mul(H, H, U2); /* H = H^3 */ + + mul(U1, U1, U2); /* U1 = U1*H^2 */ + sub(X3, S2, H); /* X3 = R^2 - H^3 */ + mul2(U2, U1); /* U2 = 2*U1*H^2 */ + mul(S1, S1, H); /* S1 = S1*H^3 */ + sub(X3, X3, U2); /* X3 = (R^2 - H^3) -2*U1*H^2 */ + + sub(Y3, U1, X3); /* Y3 = R*(U1*H^2 - X3) -S1*H^3 */ + mul(Y3, Y3, R); + sub(Y3, Y3, S1); + + /* T = p_at_infinity? q : T */ + MB_FUNC_NAME(mask_mov_FE384_)(X3, X3, p_at_infinity, q->X); + MB_FUNC_NAME(mask_mov_FE384_)(Y3, Y3, p_at_infinity, q->Y); + MB_FUNC_NAME(mask_mov_FE384_)(Z3, Z3, p_at_infinity, q->Z); + /* T = q_at_infinity? p : T */ + MB_FUNC_NAME(mask_mov_FE384_)(X3, X3, q_at_infinity, p->X); + MB_FUNC_NAME(mask_mov_FE384_)(Y3, Y3, q_at_infinity, p->Y); + MB_FUNC_NAME(mask_mov_FE384_)(Z3, Z3, q_at_infinity, p->Z); + + /* r = points_are_equal? P2 : T */ + MB_FUNC_NAME(mask_mov_FE384_)(r->X, X3, points_are_equal, P2.X); + MB_FUNC_NAME(mask_mov_FE384_)(r->Y, Y3, points_are_equal, P2.Y); + MB_FUNC_NAME(mask_mov_FE384_)(r->Z, Z3, points_are_equal, P2.Z); +} + + +/* Montgomery(1) +// r = 2^(P384_LEN52*DIGIT_SIZE) mod p384 +*/ +__ALIGN64 static const int64u p384_r_mb[P384_LEN52][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(0x0000000100000000) }, + { REP8_DECL(0x000ffffffffff000) }, + { REP8_DECL(0x0000000000ffffff) }, + { REP8_DECL(0x0000000000000010) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) } +}; + + +/* +// R(X3:Y3:Z3) = P(X1:Y1:Z1) + Q(X2:Y2:Z2=1) +// +// formulas: +// A = X1*Z2^2 B = X2*Z1^2 C = Y1*Z2^3 D = Y2*Z1^3 +// E = B-A F = D-C +// X3= -E^3 -2*A*E^2 + F^2 +// Y3= -C*E^3 + F*(A*E^2 -X3) +// Z3= Z1*Z2*E +// +// if Z2=1, then +// A = X1 B = X2*Z1^2 C = Y1 D = Y2*Z1^3 +// E = B-X1 F = D-Y1 +// X3= -E^3 -2*X1*E^2 + F^2 +// Y3= -Y1*E^3 + F*(X1*E^2 -X3) +// Z3= Z1*E +// +// cost: 3S+8M+7A +*/ +void MB_FUNC_NAME(ifma_ec_nistp384_add_point_affine_)(P384_POINT* r, const P384_POINT* p, const P384_POINT_AFFINE* q) +{ + /* coordinates of p (projective) */ + const U64* X1 = p->X; + const U64* Y1 = p->Y; + const U64* Z1 = p->Z; + __mb_mask p_at_infinity = MB_FUNC_NAME(is_zero_point_cordinate_)(p->Z); + + /* coordinates of q (affine) */ + const U64* X2 = q->x; + const U64* Y2 = q->y; + __mb_mask q_at_infinity = MB_FUNC_NAME(is_zero_point_cordinate_)(q->x) + & MB_FUNC_NAME(is_zero_point_cordinate_)(q->y); + + /* coordinates of temp point T(X3:Y3:Z3) */ + __ALIGN64 U64 X3[P384_LEN52]; + __ALIGN64 U64 Y3[P384_LEN52]; + __ALIGN64 U64 Z3[P384_LEN52]; + + __ALIGN64 U64 U2[P384_LEN52]; + __ALIGN64 U64 S2[P384_LEN52]; + __ALIGN64 U64 H[P384_LEN52]; + __ALIGN64 U64 R[P384_LEN52]; + + sqr(R, Z1); // R = Z1^2 + mul(S2, Y2, Z1); // S2 = Y2*Z1 + mul(U2, X2, R); // U2 = X2*Z1^2 + mul(S2, S2, R); // S2 = Y2*Z1^3 + + sub(H, U2, X1); // H = U2-X1 + sub(R, S2, Y1); // R = S2-Y1 + + mul(Z3, H, Z1); // Z3 = H*Z1 + + sqr(U2, H); // U2 = H^2 + sqr(S2, R); // S2 = R^2 + mul(H, H, U2); // H = H^3 + + mul(U2, U2, X1); // U2 = X1*H^2 + + mul(Y3, H, Y1); // T = Y1*H^3 + + mul2(X3, U2); // X3 = 2*X1*H^2 + sub(X3, S2, X3); // X3 = R^2 - 2*X1*H^2 + sub(X3, X3, H); // X3 = R^2 - 2*X1*H^2 -H^3 + + sub(U2, U2, X3); // U2 = X1*H^2 - X3 + mul(U2, U2, R); // U2 = R*(X1*H^2 - X3) + sub(Y3, U2, Y3); // Y3 = -Y1*H^3 + R*(X1*H^2 - X3) + + /* T = p_at_infinity? q : T */ + MB_FUNC_NAME(mask_mov_FE384_)(X3, X3, p_at_infinity, q->x); + MB_FUNC_NAME(mask_mov_FE384_)(Y3, Y3, p_at_infinity, q->y); + MB_FUNC_NAME(mask_mov_FE384_)(Z3, Z3, p_at_infinity, (U64*)p384_r_mb); + /* T = q_at_infinity? p : T */ + MB_FUNC_NAME(mask_mov_FE384_)(X3, X3, q_at_infinity, p->X); + MB_FUNC_NAME(mask_mov_FE384_)(Y3, Y3, q_at_infinity, p->Y); + MB_FUNC_NAME(mask_mov_FE384_)(Z3, Z3, q_at_infinity, p->Z); + + /* r = T */ + MB_FUNC_NAME(mov_FE384_)(r->X, X3); + MB_FUNC_NAME(mov_FE384_)(r->Y, Y3); + MB_FUNC_NAME(mov_FE384_)(r->Z, Z3); +} + +void MB_FUNC_NAME(get_nistp384_ec_affine_coords_)(U64 x[], U64 y[], const P384_POINT* P) +{ + __ALIGN64 U64 invZ1[P384_LEN52]; + __ALIGN64 U64 invZn[P384_LEN52]; + + /* 1/Z and 1/Z^2 */ + MB_FUNC_NAME(ifma_aminv52_p384_)(invZ1, P->Z); + MB_FUNC_NAME(ifma_ams52_p384_)(invZn, invZ1); + + /* if affine P.x requested */ + if(x) + MB_FUNC_NAME(ifma_amm52_p384_)(x, P->X, invZn); + + /* if affine P.y requested */ + if(y) { + MB_FUNC_NAME(ifma_amm52_p384_)(invZn, invZn, invZ1); + MB_FUNC_NAME(ifma_amm52_p384_)(y, P->Y, invZn); + } +} + +////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////// + +static __NOINLINE void clear_secret_context(U64* wval, U64* dval, __mb_mask* dsign) +{ + *wval = get_zero64(); + *dval = get_zero64(); + *dsign = 0; + return; +} + +#define WIN_SIZE (4) + +/* + s = (Ipp8u)(~((wvalue >> ws) - 1)); //sign + d = (1 << (ws+1)) - wvalue - 1; // digit, win size "ws" + d = (d & s) | (wvaluen & ~s); + d = (d >> 1) + (d & 1); + *sign = s & 1; + *digit = (Ipp8u)d; +*/ +__INLINE void MB_FUNC_NAME(booth_recode_)(__mb_mask* sign, U64* dvalue, U64 wvalue) +{ + U64 one = set1(1); + U64 zero = get_zero64(); + U64 t = srli64(wvalue, WIN_SIZE); + __mb_mask s = cmp64_mask(t, zero, _MM_CMPINT_NE); + U64 d = sub64( sub64(set1(1<<(WIN_SIZE+1)), wvalue), one); + d = mask_mov64(wvalue, s, d); + U64 odd = and64(d, one); + d = add64( srli64(d, 1), odd); + + *sign = s; + *dvalue = d; +} + +/* extract point */ +static void MB_FUNC_NAME(extract_point_)(P384_POINT* r, const P384_POINT tbl[], U64 idx) +{ + /* decrenent index (the table noes not contain [0]*P */ + U64 idx_target = sub64(idx, set1(1)); + + /* assume the point at infinity is what need */ + P384_POINT R; + MB_FUNC_NAME(set_point_to_infinity_)(&R); + + /* find out what we actually need or just keep original infinity */ + int32u n; + for(n=0; n<(1<<(WIN_SIZE-1)); n++) { + U64 idx_curr = set1(n); + __mb_mask k = cmp64_mask(idx_curr, idx_target, _MM_CMPINT_EQ); + + /* R = k? tbl[] : R */ + MB_FUNC_NAME(secure_mask_mov_FE384_)(R.X, R.X, k, tbl[n].X); + MB_FUNC_NAME(secure_mask_mov_FE384_)(R.Y, R.Y, k, tbl[n].Y); + MB_FUNC_NAME(secure_mask_mov_FE384_)(R.Z, R.Z, k, tbl[n].Z); + } + MB_FUNC_NAME(mov_FE384_)(r->X, R.X); + MB_FUNC_NAME(mov_FE384_)(r->Y, R.Y); + MB_FUNC_NAME(mov_FE384_)(r->Z, R.Z); +} + +void MB_FUNC_NAME(ifma_ec_nistp384_mul_point_)(P384_POINT* r, const P384_POINT* p, const U64 scalar[]) +{ + /* pre-computed table */ + __ALIGN64 P384_POINT tbl[1<<(WIN_SIZE-1)]; + + /* + // compute tbl[] = [n]P, n=1,..,2^(WIN_SIZE-1): + // + // tbl[2*n] = tbl[2*n-1]+p + // tbl[2*n+1] = [2]*tbl[n] + */ + /* tbl[0] = p */ + MB_FUNC_NAME(mov_FE384_)(tbl[0].X, p->X); + MB_FUNC_NAME(mov_FE384_)(tbl[0].Y, p->Y); + MB_FUNC_NAME(mov_FE384_)(tbl[0].Z, p->Z); + /* tbl[1] = [2]*p */ + MB_FUNC_NAME(ifma_ec_nistp384_dbl_point_)(&tbl[1], p); + + int n; + for(n=1; n < (1<<(WIN_SIZE-1))/2; n++) { + MB_FUNC_NAME(ifma_ec_nistp384_add_point_)(&tbl[2*n], &tbl[2*n-1], p); + MB_FUNC_NAME(ifma_ec_nistp384_dbl_point_)(&tbl[2*n+1], &tbl[n]); + } + + P384_POINT R; + P384_POINT T; + U64 Ty[P384_LEN52]; + + /* + // point (LR) multiplication + */ + U64 idx_mask = set1( (1<<(WIN_SIZE+1))-1 ); + int bit = P384_BITSIZE-(P384_BITSIZE % WIN_SIZE); + int chunk_no = (bit-1)/64; + int chunk_shift = (bit-1)%64; + + /* first window */ + U64 wvalue = loadu64(&scalar[chunk_no]); + wvalue = and64( srli64(wvalue, chunk_shift), idx_mask); + + U64 dvalue; + __mb_mask dsign; + MB_FUNC_NAME(booth_recode_)(&dsign, &dvalue, wvalue); + MB_FUNC_NAME(extract_point_)(&R, tbl, dvalue); + + for(bit-=WIN_SIZE; bit>=WIN_SIZE; bit-=WIN_SIZE) { + /* doubling */ + MB_FUNC_NAME(ifma_ec_nistp384_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_nistp384_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_nistp384_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_nistp384_dbl_point_)(&R, &R); + #if (WIN_SIZE==5) + MB_FUNC_NAME(ifma_ec_nistp384_dbl_point_)(&R, &R); + #endif + + /* extract precomputed []P */ + chunk_no = (bit-1)/64; + chunk_shift = (bit-1)%64; + + wvalue = loadu64(&scalar[chunk_no]); + #if (_MSC_VER <= 1916) /* VS 2017 not supported _mm512_shrdv_epi64 */ + { + __m512i t_lo_ = _mm512_srlv_epi64(wvalue, set64(chunk_shift)); + __m512i t_hi_ = _mm512_sllv_epi64(loadu64(&scalar[chunk_no+1]), set64(64-chunk_shift)); + wvalue = or64(t_lo_, t_hi_); + } + #else + wvalue = _mm512_shrdv_epi64(wvalue, loadu64(&scalar[chunk_no+1]), set1((int32u)chunk_shift)); + #endif + wvalue = and64(wvalue, idx_mask); + + MB_FUNC_NAME(booth_recode_)(&dsign, &dvalue, wvalue); + MB_FUNC_NAME(extract_point_)(&T, tbl, dvalue); + + /* T = dsign? -T : T */ + MB_FUNC_NAME(ifma_neg52_p384_)(Ty, T.Y); + MB_FUNC_NAME(secure_mask_mov_FE384_)(T.Y, T.Y, dsign, Ty); + + /* acumulate T */ + MB_FUNC_NAME(ifma_ec_nistp384_add_point_)(&R, &R, &T); + } + + /* last window */ + MB_FUNC_NAME(ifma_ec_nistp384_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_nistp384_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_nistp384_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_nistp384_dbl_point_)(&R, &R); + #if (WIN_SIZE==5) + MB_FUNC_NAME(ifma_ec_nistp384_dbl_point_)(&R, &R); + #endif + + wvalue = loadu64(&scalar[0]); + wvalue = and64( slli64(wvalue, 1), idx_mask); + MB_FUNC_NAME(booth_recode_)(&dsign, &dvalue, wvalue); + MB_FUNC_NAME(extract_point_)(&T, tbl, dvalue); + + MB_FUNC_NAME(ifma_neg52_p384_)(Ty, T.Y); + MB_FUNC_NAME(secure_mask_mov_FE384_)(T.Y, T.Y, dsign, Ty); + + MB_FUNC_NAME(ifma_ec_nistp384_add_point_)(&R, &R, &T); + + /* r = R */ + MB_FUNC_NAME(mov_FE384_)(r->X, R.X); + MB_FUNC_NAME(mov_FE384_)(r->Y, R.Y); + MB_FUNC_NAME(mov_FE384_)(r->Z, R.Z); + + /* clear r (to fix potential secutity flaw in case of ecdh */ + MB_FUNC_NAME(zero_)((int64u (*)[8])&R, sizeof(R)/sizeof(U64)); + + /* clear stubs of secret scalar */ + clear_secret_context(&wvalue, &dvalue, &dsign); +} +#undef WIN_SIZE + + +////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////// + +#include + +#define BP_WIN_SIZE MUL_BASEPOINT_WIN_SIZE /* defined in the header above */ + +__INLINE void MB_FUNC_NAME(booth_recode_bp_)(__mb_mask* sign, U64* dvalue, U64 wvalue) +{ + U64 one = set1(1); + U64 zero = get_zero64(); + U64 t = srli64(wvalue, BP_WIN_SIZE); + __mb_mask s = cmp64_mask(t, zero, _MM_CMPINT_NE); + U64 d = sub64( sub64(set1(1<<(BP_WIN_SIZE+1)), wvalue), one); + d = mask_mov64(wvalue, s, d); + U64 odd = and64(d, one); + d = add64( srli64(d, 1), odd); + + *sign = s; + *dvalue = d; +} + +/* extract affine affine point */ +__INLINE void MB_FUNC_NAME(extract_point_affine_)(P384_POINT_AFFINE* r, const SINGLE_P384_POINT_AFFINE* tbl, U64 idx) +{ + /* decrement index (the table noes not contain [0]*P */ + U64 targIdx = sub64(idx, set1(1)); + + U64 ax0, ax1, ax2, ax3, ax4, ax5, ax6, ax7; + U64 ay0, ay1, ay2, ay3, ay4, ay5, ay6, ay7; + + /* assume the point at infinity is what need */ + ax0 = ax1 = ax2 = ax3 = ax4 = ax5 = ax6 = ax7= + ay0 = ay1 = ay2 = ay3 = ay4 = ay5 = ay6 = ay7 = get_zero64(); + + /* find out what we actually need or just keep original infinity */ + int n; + U64 currIdx = get_zero64(); + for(n=0; n<(1<<(BP_WIN_SIZE-1)); n++, tbl++, currIdx = add64(currIdx, set1(1))) { + __mb_mask k = cmp64_mask(currIdx, targIdx, _MM_CMPINT_EQ); + + /* R = k? set1( tbl[] ) : R */ + ax0 = mask_add64(ax0, k, ax0, set1( tbl->x[0] )); + ax1 = mask_add64(ax1, k, ax1, set1( tbl->x[1] )); + ax2 = mask_add64(ax2, k, ax2, set1( tbl->x[2] )); + ax3 = mask_add64(ax3, k, ax3, set1( tbl->x[3] )); + ax4 = mask_add64(ax4, k, ax4, set1( tbl->x[4] )); + ax5 = mask_add64(ax5, k, ax5, set1( tbl->x[5] )); + ax6 = mask_add64(ax6, k, ax6, set1( tbl->x[6] )); + ax7 = mask_add64(ax7, k, ax7, set1( tbl->x[7] )); + + ay0 = mask_add64(ay0, k, ay0, set1( tbl->y[0] )); + ay1 = mask_add64(ay1, k, ay1, set1( tbl->y[1] )); + ay2 = mask_add64(ay2, k, ay2, set1( tbl->y[2] )); + ay3 = mask_add64(ay3, k, ay3, set1( tbl->y[3] )); + ay4 = mask_add64(ay4, k, ay4, set1( tbl->y[4] )); + ay5 = mask_add64(ay5, k, ay5, set1( tbl->y[5] )); + ay6 = mask_add64(ay6, k, ay6, set1( tbl->y[6] )); + ay7 = mask_add64(ay7, k, ay7, set1( tbl->y[7] )); + } + + r->x[0] = ax0; + r->x[1] = ax1; + r->x[2] = ax2; + r->x[3] = ax3; + r->x[4] = ax4; + r->x[5] = ax5; + r->x[6] = ax6; + r->x[7] = ax7; + + r->y[0] = ay0; + r->y[1] = ay1; + r->y[2] = ay2; + r->y[3] = ay3; + r->y[4] = ay4; + r->y[5] = ay5; + r->y[6] = ay6; + r->y[7] = ay7; +} + +void MB_FUNC_NAME(ifma_ec_nistp384_mul_pointbase_)(P384_POINT* r, const U64 scalar[]) +{ + /* pre-computed table of base powers */ + SINGLE_P384_POINT_AFFINE* tbl = &ifma_ec_nistp384_bp_precomp[0][0]; + + P384_POINT R; + P384_POINT_AFFINE A; + U64 Ty[P384_LEN52]; + + /* R = O */ + MB_FUNC_NAME(set_point_to_infinity_)(&R); + + + /* + // base point (RL) multiplication + */ + U64 wvalue, dvalue; + __mb_mask dsign; + + U64 idx_mask = set1( (1<<(BP_WIN_SIZE+1))-1 ); + int bit = 0; + + /* first window - window[0] */ + wvalue = loadu64(&scalar[0]); + wvalue = and64( slli64(wvalue, 1), idx_mask); + MB_FUNC_NAME(booth_recode_bp_)(&dsign, &dvalue, wvalue); + MB_FUNC_NAME(extract_point_affine_)(&A, tbl, dvalue); + tbl+=BP_N_ENTRY; + + /* A = dsign? -A : A */ + MB_FUNC_NAME(ifma_neg52_p384_)(Ty, A.y); + MB_FUNC_NAME(secure_mask_mov_FE384_)(A.y, A.y, dsign, Ty); + + /* R += A */ + MB_FUNC_NAME(ifma_ec_nistp384_add_point_affine_)(&R, &R, &A); + + int chunk_no; + int chunk_shift; + for(bit+=BP_WIN_SIZE; bit<=P384_BITSIZE; bit+=BP_WIN_SIZE) { + chunk_no = (bit-1)/64; + chunk_shift = (bit-1)%64; + + wvalue = loadu64(&scalar[chunk_no]); + #if (_MSC_VER <= 1916) /* VS 2017 not supported _mm512_shrdv_epi64 */ + { + __m512i t_lo_ = _mm512_srlv_epi64(wvalue, set64(chunk_shift)); + __m512i t_hi_ = _mm512_sllv_epi64(loadu64(&scalar[chunk_no+1]), set64(64-chunk_shift)); + wvalue = or64(t_lo_, t_hi_); + } + #else + wvalue = _mm512_shrdv_epi64(wvalue, loadu64(&scalar[chunk_no+1]), set1((int32u)chunk_shift)); + #endif + wvalue = and64(wvalue, idx_mask); + + MB_FUNC_NAME(booth_recode_bp_)(&dsign, &dvalue, wvalue); + MB_FUNC_NAME(extract_point_affine_)(&A, tbl, dvalue); + tbl+=BP_N_ENTRY; + + /* A = dsign? -A : A */ + MB_FUNC_NAME(ifma_neg52_p384_)(Ty, A.y); + MB_FUNC_NAME(secure_mask_mov_FE384_)(A.y, A.y, dsign, Ty); + + /* R += A */ + MB_FUNC_NAME(ifma_ec_nistp384_add_point_affine_)(&R, &R, &A); + } + + /* r = R */ + MB_FUNC_NAME(mov_FE384_)(r->X, R.X); + MB_FUNC_NAME(mov_FE384_)(r->Y, R.Y); + MB_FUNC_NAME(mov_FE384_)(r->Z, R.Z); + + /* clear stubs of secret scalar */ + clear_secret_context(&wvalue, &dvalue, &dsign); +} +#undef BP_WIN_SIZE + + +/* P384 parameters: mont(a), mont(b) */ +__ALIGN64 int64u mont_a_p384_mb[P384_LEN52][8] = { //!!! + { REP8_DECL(0x000ffffdffffffff) }, + { REP8_DECL(0x000ff00000002fff) }, + { REP8_DECL(0x000ffffffbffffff) }, + { REP8_DECL(0x000fffffffffffcf) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x00000000000fffff) } +}; +__ALIGN64 int64u mont_b_p384_mb[P384_LEN52][8] = { //!!!!! + { REP8_DECL(0x00091c81cd08114b) }, + { REP8_DECL(0x0003708118870d03) }, + { REP8_DECL(0x000431bf24475444) }, + { REP8_DECL(0x000209b1920022fc) }, + { REP8_DECL(0x000e94938ae277f2) }, + { REP8_DECL(0x000022094e3374be) }, + { REP8_DECL(0x000ff9b62b21f41f) }, + { REP8_DECL(0x00000000000604fb) }, +}; + +/* +// We have a curve defined by a Weierstrass equation: y^2 = x^3 + a*x + b. +// +// The points are considered in Jacobian projective coordinates +// where (X, Y, Z) represents (x, y) = (X/Z^2, Y/Z^3). +// Substituting this and multiplying by Z^6 transforms the above equation into +// Y^2 = X^3 + a*X*Z^4 + b*Z^6 +// To test this, we add up the right-hand side in 'rh'. +*/ +__mb_mask MB_FUNC_NAME(ifma_is_on_curve_p384_)(const P384_POINT* p, int use_jproj_coords) +{ + U64 rh[P384_LEN52]; + U64 Z4[P384_LEN52], Z6[P384_LEN52], tmp[P384_LEN52]; + + /* rh := X^2 */ + MB_FUNC_NAME(ifma_ams52_p384_)(rh, p->X); + + /* if Z!=1, then rh = X^3 + a*X*Z^4 + b*Z^6 = X*(X^2 + a*X*Z^4) + b*Z^6 */ + if(use_jproj_coords) { + MB_FUNC_NAME(ifma_ams52_p384_)(tmp, p->Z); /* tmp = Z^2 */ + MB_FUNC_NAME(ifma_ams52_p384_)(Z4, tmp); /* Z4 = Z^4 */ + MB_FUNC_NAME(ifma_amm52_p384_)(Z6, Z4, tmp); /* Z6 = Z^6 */ + + MB_FUNC_NAME(ifma_add52_p384_)(tmp, Z4, Z4); /* tmp = 2*Z^4 */ + MB_FUNC_NAME(ifma_add52_p384_)(tmp, tmp, Z4); /* tmp = 2*Z^4 */ + MB_FUNC_NAME(ifma_sub52_p384_)(rh, rh, tmp); /* rh = X^2 + a*Z^4 */ + MB_FUNC_NAME(ifma_amm52_p384_)(rh, rh, p->X); /* rh = (X^2 + a*Z^4)*X */ + + MB_FUNC_NAME(ifma_amm52_p384_)(tmp, Z6, (U64*)mont_b_p384_mb); + MB_FUNC_NAME(ifma_add52_p384_)(rh, rh, tmp); /* rh = (X^2 + a*Z^4)*X + b*Z^6 */ + } + /* if Z==1, then rh = X^3 + a*X + b = X*(X^2 +a) b */ + else { + MB_FUNC_NAME(ifma_add52_p384_)(rh, rh, (U64*)mont_a_p384_mb); /* rh = X^2+a */ + MB_FUNC_NAME(ifma_amm52_p384_)(rh, rh, p->X); /* rh = (X^2+a)*X */ + MB_FUNC_NAME(ifma_add52_p384_)(rh, rh, (U64*)mont_b_p384_mb); /* rh = (X^2+a)*X + b */ + } + MB_FUNC_NAME(ifma_frommont52_p384_)(rh, rh); + + /* rl = tmp = Y^2 */ + MB_FUNC_NAME(ifma_ams52_p384_)(tmp, p->Y); + MB_FUNC_NAME(ifma_frommont52_p384_)(tmp, tmp); + + /* mask = rl==rh */ + __mb_mask is_on_curve_mask = MB_FUNC_NAME(cmp_eq_FE384_)(tmp, rh); + + return is_on_curve_mask; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecpoint_p521.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecpoint_p521.c new file mode 100644 index 0000000..cc5b10e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecpoint_p521.c @@ -0,0 +1,737 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + + +/* simplify naming */ +#define sqr MB_FUNC_NAME(ifma_ams52_p521_) +#define mul MB_FUNC_NAME(ifma_amm52_p521_) +#define add MB_FUNC_NAME(ifma_add52_p521_) +#define sub MB_FUNC_NAME(ifma_sub52_p521_) +#define mul2 MB_FUNC_NAME(ifma_double52_p521_) +#define mul3 MB_FUNC_NAME(ifma_tripple52_p521_) +#define div2 MB_FUNC_NAME(ifma_half52_p521_) + + +/* +// Presentation of point at infinity: +// - projective (X : Y : 0) +// - affine (0 : 0) +*/ + +/* +// R(X3:Y3:Z3) = [2]P(X1:Y1:Z1) +// +// formulas: +// A = 4*X1*Y1^2 +// B = 3*(X1^2-Z1^4) +// X3= B^2 -2*A +// Y3= B*(A-X3) -8*Y1^4 +// Z3= 2*Y1*Z1 +// +// cost: 4S+4M+9A +// +*/ +void MB_FUNC_NAME(ifma_ec_nistp521_dbl_point_)(P521_POINT* r, const P521_POINT* p) +{ + __ALIGN64 U64 T[P521_LEN52]; + __ALIGN64 U64 U[P521_LEN52]; + __ALIGN64 U64 V[P521_LEN52]; + __ALIGN64 U64 A[P521_LEN52]; + __ALIGN64 U64 B[P521_LEN52]; + + const U64* X1 = p->X; /* input point */ + const U64* Y1 = p->Y; + const U64* Z1 = p->Z; + U64* X3 = r->X; /* output point */ + U64* Y3 = r->Y; + U64* Z3 = r->Z; + + mul2(T, Y1); /* T = 2*Y1 */ + + sqr(V, T); /* V = 4*Y1^2 */ /* sqr_dual */ + sqr(U, Z1); /* U = Z1^2 */ + + sub(B, X1, U); /* B = X1-Z1^2 */ + add(U, X1, U); /* U = X1+Z1^2 */ + + mul(A, V, X1); /* A = 4*X*Y1^2 */ /* mul_dual */ + mul(B, B, U); /* B = (X1^2-Z1^4) */ + + mul2(X3, A); /* X3 = 2*A */ + mul3(B, B); /* B = 3*(X1^2-Z1^4) */ + + sqr(U, B); /* U = B^2 */ /* sqr_dual */ + sqr(Y3, V); /* Y3= V^2 = 16*Y1^4 */ + + sub(X3, U, X3); /* X3=B^2 - 2*A */ + div2(Y3,Y3); /* Y3=Y3/2 = 8*Y1^4 */ + + sub(U, A, X3); /* U = A-X3 */ + + mul(Z3, T, Z1); /* Z3= 2*Y1*Z1 */ /* mul_dual */ + mul(U, U, B); /* U = B*(A-X3) */ + + sub(Y3, U, Y3); /* Y3 = B*(A-X3) -8*Y1^4 */ +} + + +/* +// R(X3:Y3:Z3) = P(X1:Y1:Z1) + Q(X2:Y2:Z2) +// +// formulas: +// A = X1*Z2^2 B = X2*Z1^2 C = Y1*Z2^3 D = Y2*Z1^3 +// E = B-A F = D-C +// X3= -E^3 -2*A*E^2 + F^2 +// Y3= -C*E^3 + F*(A*E^2 -X3) +// Z3= Z1*Z2*E +// +// cost: 4S+12M+7A +// +*/ +void MB_FUNC_NAME(ifma_ec_nistp521_add_point_)(P521_POINT* r, const P521_POINT* p, const P521_POINT* q) +{ + /* coordinates of p */ + const U64* X1 = p->X; + const U64* Y1 = p->Y; + const U64* Z1 = p->Z; + __mb_mask p_at_infinity = MB_FUNC_NAME(is_zero_point_cordinate_)(p->Z); + + /* coordinates of q */ + const U64* X2 = q->X; + const U64* Y2 = q->Y; + const U64* Z2 = q->Z; + __mb_mask q_at_infinity = MB_FUNC_NAME(is_zero_point_cordinate_)(q->Z); + + /* coordinates of temp point T(X3:Y3:Z3) */ + __ALIGN64 U64 X3[P521_LEN52]; + __ALIGN64 U64 Y3[P521_LEN52]; + __ALIGN64 U64 Z3[P521_LEN52]; + + /* temporary */ + __ALIGN64 U64 U1[P521_LEN52]; + __ALIGN64 U64 U2[P521_LEN52]; + __ALIGN64 U64 S1[P521_LEN52]; + __ALIGN64 U64 S2[P521_LEN52]; + __ALIGN64 U64 H[P521_LEN52]; + __ALIGN64 U64 R[P521_LEN52]; + + mul(S1, Y1, Z2); /* S1 = Y1*Z2 */ + sqr(U1, Z2); /* U1 = Z2^2 */ + + mul(S2, Y2, Z1); /* S2 = Y2*Z1 */ + sqr(U2, Z1); /* U2 = Z1^2 */ + + mul(S1, S1, U1); /* S1 = Y1*Z2^3 */ + mul(S2, S2, U2); /* S2 = Y2*Z1^3 */ + + mul(U1, X1, U1); /* U1 = X1*Z2^2 */ + mul(U2, X2, U2); /* U2 = X2*Z1^2 */ + + sub(R, S2, S1); /* R = S2-S1 */ + sub(H, U2, U1); /* H = U2-U1 */ + + /* check if affine (p.x:p.y) == (q.x:q.y) and and do doubling if this happens */ + __mb_mask x_are_equal = MB_FUNC_NAME(is_zero_FE521_)(H); + __mb_mask y_are_equal = MB_FUNC_NAME(is_zero_FE521_)(R); + __mb_mask points_are_equal = (x_are_equal & y_are_equal & (~p_at_infinity) & (~q_at_infinity)); + + P521_POINT P2; + MB_FUNC_NAME(set_point_to_infinity_)(&P2); + if (points_are_equal) { + MB_FUNC_NAME(ifma_ec_nistp521_dbl_point_)(&P2, p); + } + + mul(Z3, Z1, Z2); /* Z3 = Z1*Z2 */ + sqr(U2, H); /* U2 = H^2 */ + mul(Z3, Z3, H); /* Z3 = (Z1*Z2)*H */ + sqr(S2, R); /* S2 = R^2 */ + mul(H, H, U2); /* H = H^3 */ + + mul(U1, U1, U2); /* U1 = U1*H^2 */ + sub(X3, S2, H); /* X3 = R^2 - H^3 */ + mul2(U2, U1); /* U2 = 2*U1*H^2 */ + mul(S1, S1, H); /* S1 = S1*H^3 */ + sub(X3, X3, U2); /* X3 = (R^2 - H^3) -2*U1*H^2 */ + + sub(Y3, U1, X3); /* Y3 = R*(U1*H^2 - X3) -S1*H^3 */ + mul(Y3, Y3, R); + sub(Y3, Y3, S1); + + /* T = p_at_infinity? q : T */ + MB_FUNC_NAME(mask_mov_FE521_)(X3, X3, p_at_infinity, q->X); + MB_FUNC_NAME(mask_mov_FE521_)(Y3, Y3, p_at_infinity, q->Y); + MB_FUNC_NAME(mask_mov_FE521_)(Z3, Z3, p_at_infinity, q->Z); + /* T = q_at_infinity? p : T */ + MB_FUNC_NAME(mask_mov_FE521_)(X3, X3, q_at_infinity, p->X); + MB_FUNC_NAME(mask_mov_FE521_)(Y3, Y3, q_at_infinity, p->Y); + MB_FUNC_NAME(mask_mov_FE521_)(Z3, Z3, q_at_infinity, p->Z); + + /* r = T */ + /* r = points_are_equal? P2 : T */ + MB_FUNC_NAME(mask_mov_FE521_)(r->X, X3, points_are_equal, P2.X); + MB_FUNC_NAME(mask_mov_FE521_)(r->Y, Y3, points_are_equal, P2.Y); + MB_FUNC_NAME(mask_mov_FE521_)(r->Z, Z3, points_are_equal, P2.Z); +} + + +/* Montgomery(1) +// r = 2^(P521_LEN52*DIGIT_SIZE) mod p521 +*/ +__ALIGN64 static const int64u p521_r_mb[P521_LEN52][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(0x0008000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) } +}; + + +/* +// R(X3:Y3:Z3) = P(X1:Y1:Z1) + Q(X2:Y2:Z2=1) +// +// formulas: +// A = X1*Z2^2 B = X2*Z1^2 C = Y1*Z2^3 D = Y2*Z1^3 +// E = B-A F = D-C +// X3= -E^3 -2*A*E^2 + F^2 +// Y3= -C*E^3 + F*(A*E^2 -X3) +// Z3= Z1*Z2*E +// +// if Z2=1, then +// A = X1 B = X2*Z1^2 C = Y1 D = Y2*Z1^3 +// E = B-X1 F = D-Y1 +// X3= -E^3 -2*X1*E^2 + F^2 +// Y3= -Y1*E^3 + F*(X1*E^2 -X3) +// Z3= Z1*E +// +// cost: 3S+8M+7A +*/ +void MB_FUNC_NAME(ifma_ec_nistp521_add_point_affine_)(P521_POINT* r, const P521_POINT* p, const P521_POINT_AFFINE* q) +{ + /* coordinates of p (projective) */ + const U64* X1 = p->X; + const U64* Y1 = p->Y; + const U64* Z1 = p->Z; + __mb_mask p_at_infinity = MB_FUNC_NAME(is_zero_point_cordinate_)(p->Z); + + /* coordinates of q (affine) */ + const U64* X2 = q->x; + const U64* Y2 = q->y; + __mb_mask q_at_infinity = MB_FUNC_NAME(is_zero_point_cordinate_)(q->x) + & MB_FUNC_NAME(is_zero_point_cordinate_)(q->y); + + /* coordinates of temp point T(X3:Y3:Z3) */ + __ALIGN64 U64 X3[P521_LEN52]; + __ALIGN64 U64 Y3[P521_LEN52]; + __ALIGN64 U64 Z3[P521_LEN52]; + + __ALIGN64 U64 U2[P521_LEN52]; + __ALIGN64 U64 S2[P521_LEN52]; + __ALIGN64 U64 H[P521_LEN52]; + __ALIGN64 U64 R[P521_LEN52]; + + sqr(R, Z1); // R = Z1^2 + mul(S2, Y2, Z1); // S2 = Y2*Z1 + mul(U2, X2, R); // U2 = X2*Z1^2 + mul(S2, S2, R); // S2 = Y2*Z1^3 + + sub(H, U2, X1); // H = U2-X1 + sub(R, S2, Y1); // R = S2-Y1 + + mul(Z3, H, Z1); // Z3 = H*Z1 + + sqr(U2, H); // U2 = H^2 + sqr(S2, R); // S2 = R^2 + mul(H, H, U2); // H = H^3 + + mul(U2, U2, X1); // U2 = X1*H^2 + + mul(Y3, H, Y1); // T = Y1*H^3 + + mul2(X3, U2); // X3 = 2*X1*H^2 + sub(X3, S2, X3); // X3 = R^2 - 2*X1*H^2 + sub(X3, X3, H); // X3 = R^2 - 2*X1*H^2 -H^3 + + sub(U2, U2, X3); // U2 = X1*H^2 - X3 + mul(U2, U2, R); // U2 = R*(X1*H^2 - X3) + sub(Y3, U2, Y3); // Y3 = -Y1*H^3 + R*(X1*H^2 - X3) + + /* T = p_at_infinity? q : T */ + MB_FUNC_NAME(mask_mov_FE521_)(X3, X3, p_at_infinity, q->x); + MB_FUNC_NAME(mask_mov_FE521_)(Y3, Y3, p_at_infinity, q->y); + MB_FUNC_NAME(mask_mov_FE521_)(Z3, Z3, p_at_infinity, (U64*)p521_r_mb); + /* T = q_at_infinity? p : T */ + MB_FUNC_NAME(mask_mov_FE521_)(X3, X3, q_at_infinity, p->X); + MB_FUNC_NAME(mask_mov_FE521_)(Y3, Y3, q_at_infinity, p->Y); + MB_FUNC_NAME(mask_mov_FE521_)(Z3, Z3, q_at_infinity, p->Z); + + /* r = T */ + MB_FUNC_NAME(mov_FE521_)(r->X, X3); + MB_FUNC_NAME(mov_FE521_)(r->Y, Y3); + MB_FUNC_NAME(mov_FE521_)(r->Z, Z3); +} + +void MB_FUNC_NAME(get_nistp521_ec_affine_coords_)(U64 x[], U64 y[], const P521_POINT* P) +{ + __ALIGN64 U64 invZ1[P521_LEN52]; + __ALIGN64 U64 invZn[P521_LEN52]; + + /* 1/Z and 1/Z^2 */ + MB_FUNC_NAME(ifma_aminv52_p521_)(invZ1, P->Z); + MB_FUNC_NAME(ifma_ams52_p521_)(invZn, invZ1); + + /* if affine P.x requested */ + if(x) + MB_FUNC_NAME(ifma_amm52_p521_)(x, P->X, invZn); + + /* if affine P.y requested */ + if(y) { + MB_FUNC_NAME(ifma_amm52_p521_)(invZn, invZn, invZ1); + MB_FUNC_NAME(ifma_amm52_p521_)(y, P->Y, invZn); + } +} + +////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////// + +static __NOINLINE void clear_secret_context(U64* wval, U64* dval, __mb_mask* dsign) +{ + *wval = get_zero64(); + *dval = get_zero64(); + *dsign = 0; + return; +} + +#define WIN_SIZE (4) + +/* + s = (Ipp8u)(~((wvalue >> ws) - 1)); //sign + d = (1 << (ws+1)) - wvalue - 1; // digit, win size "ws" + d = (d & s) | (wvaluen & ~s); + d = (d >> 1) + (d & 1); + *sign = s & 1; + *digit = (Ipp8u)d; +*/ +__INLINE void MB_FUNC_NAME(booth_recode_)(__mb_mask* sign, U64* dvalue, U64 wvalue) +{ + U64 one = set1(1); + U64 zero = get_zero64(); + U64 t = srli64(wvalue, WIN_SIZE); + __mb_mask s = cmp64_mask(t, zero, _MM_CMPINT_NE); + U64 d = sub64( sub64(set1(1<<(WIN_SIZE+1)), wvalue), one); + d = mask_mov64(wvalue, s, d); + U64 odd = and64(d, one); + d = add64( srli64(d, 1), odd); + + *sign = s; + *dvalue = d; +} + +/* extract point */ +static void MB_FUNC_NAME(extract_point_)(P521_POINT* r, const P521_POINT tbl[], U64 idx) +{ + /* decrenent index (the table noes not contain [0]*P */ + U64 idx_target = sub64(idx, set1(1)); + + /* assume the point at infinity is what need */ + P521_POINT R; + MB_FUNC_NAME(set_point_to_infinity_)(&R); + + /* find out what we actually need or just keep original infinity */ + int32u n; + for(n=0; n<(1<<(WIN_SIZE-1)); n++) { + U64 idx_curr = set1(n); + __mb_mask k = cmp64_mask(idx_curr, idx_target, _MM_CMPINT_EQ); + + /* R = k? tbl[] : R */ + MB_FUNC_NAME(secure_mask_mov_FE521_)(R.X, R.X, k, tbl[n].X); + MB_FUNC_NAME(secure_mask_mov_FE521_)(R.Y, R.Y, k, tbl[n].Y); + MB_FUNC_NAME(secure_mask_mov_FE521_)(R.Z, R.Z, k, tbl[n].Z); + } + MB_FUNC_NAME(mov_FE521_)(r->X, R.X); + MB_FUNC_NAME(mov_FE521_)(r->Y, R.Y); + MB_FUNC_NAME(mov_FE521_)(r->Z, R.Z); +} + +void MB_FUNC_NAME(ifma_ec_nistp521_mul_point_)(P521_POINT* r, const P521_POINT* p, const U64 scalar[]) +{ + /* pre-computed table */ + __ALIGN64 P521_POINT tbl[1<<(WIN_SIZE-1)]; + + /* + // compute tbl[] = [n]P, n=1,..,2^(WIN_SIZE-1): + // + // tbl[2*n] = tbl[2*n-1]+p + // tbl[2*n+1] = [2]*tbl[n] + */ + /* tbl[0] = p */ + MB_FUNC_NAME(mov_FE521_)(tbl[0].X, p->X); + MB_FUNC_NAME(mov_FE521_)(tbl[0].Y, p->Y); + MB_FUNC_NAME(mov_FE521_)(tbl[0].Z, p->Z); + /* tbl[1] = [2]*p */ + MB_FUNC_NAME(ifma_ec_nistp521_dbl_point_)(&tbl[1], p); + + int n; + for(n=1; n < (1<<(WIN_SIZE-1))/2; n++) { + MB_FUNC_NAME(ifma_ec_nistp521_add_point_)(&tbl[2*n], &tbl[2*n-1], p); + MB_FUNC_NAME(ifma_ec_nistp521_dbl_point_)(&tbl[2*n+1], &tbl[n]); + } + + P521_POINT R; + P521_POINT T; + U64 Ty[P521_LEN52]; + + /* + // point (LR) multiplication + */ + U64 idx_mask = set1( (1<<(WIN_SIZE+1))-1 ); + int bit = P521_BITSIZE-(P521_BITSIZE % WIN_SIZE); + int chunk_no = (bit-1)/64; + int chunk_shift = (bit-1)%64; + + /* first window */ + U64 wvalue = loadu64(&scalar[chunk_no]); + wvalue = and64( srli64(wvalue, chunk_shift), idx_mask); + + U64 dvalue; + __mb_mask dsign; + MB_FUNC_NAME(booth_recode_)(&dsign, &dvalue, wvalue); + MB_FUNC_NAME(extract_point_)(&R, tbl, dvalue); + + for(bit-=WIN_SIZE; bit>=WIN_SIZE; bit-=WIN_SIZE) { + /* doubling */ + MB_FUNC_NAME(ifma_ec_nistp521_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_nistp521_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_nistp521_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_nistp521_dbl_point_)(&R, &R); + #if (WIN_SIZE==5) + MB_FUNC_NAME(ifma_ec_nistp521_dbl_point_)(&R, &R); + #endif + + /* extract precomputed []P */ + chunk_no = (bit-1)/64; + chunk_shift = (bit-1)%64; + + wvalue = loadu64(&scalar[chunk_no]); + #if (_MSC_VER <= 1916) /* VS 2017 not supported _mm512_shrdv_epi64 */ + { + __m512i t_lo_ = _mm512_srlv_epi64(wvalue, set64(chunk_shift)); + __m512i t_hi_ = _mm512_sllv_epi64(loadu64(&scalar[chunk_no+1]), set64(64-chunk_shift)); + wvalue = or64(t_lo_, t_hi_); + } + #else + wvalue = _mm512_shrdv_epi64(wvalue, loadu64(&scalar[chunk_no+1]), set1((int32u)chunk_shift)); + #endif + wvalue = and64(wvalue, idx_mask); + + MB_FUNC_NAME(booth_recode_)(&dsign, &dvalue, wvalue); + MB_FUNC_NAME(extract_point_)(&T, tbl, dvalue); + + /* T = dsign? -T : T */ + MB_FUNC_NAME(ifma_neg52_p521_)(Ty, T.Y); + MB_FUNC_NAME(secure_mask_mov_FE521_)(T.Y, T.Y, dsign, Ty); + + /* acumulate T */ + MB_FUNC_NAME(ifma_ec_nistp521_add_point_)(&R, &R, &T); + } + + /* last window */ + MB_FUNC_NAME(ifma_ec_nistp521_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_nistp521_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_nistp521_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_nistp521_dbl_point_)(&R, &R); + #if (WIN_SIZE==5) + MB_FUNC_NAME(ifma_ec_nistp521_dbl_point_)(&R, &R); + #endif + + wvalue = loadu64(&scalar[0]); + wvalue = and64( slli64(wvalue, 1), idx_mask); + MB_FUNC_NAME(booth_recode_)(&dsign, &dvalue, wvalue); + MB_FUNC_NAME(extract_point_)(&T, tbl, dvalue); + + MB_FUNC_NAME(ifma_neg52_p521_)(Ty, T.Y); + MB_FUNC_NAME(secure_mask_mov_FE521_)(T.Y, T.Y, dsign, Ty); + + MB_FUNC_NAME(ifma_ec_nistp521_add_point_)(&R, &R, &T); + + /* r = R */ + MB_FUNC_NAME(mov_FE521_)(r->X, R.X); + MB_FUNC_NAME(mov_FE521_)(r->Y, R.Y); + MB_FUNC_NAME(mov_FE521_)(r->Z, R.Z); + + /* clear r (to fix potential secutity flaw in case of ecdh */ + MB_FUNC_NAME(zero_)((int64u (*)[8])&R, sizeof(R)/sizeof(U64)); + + /* clear stubs of secret scalar */ + clear_secret_context(&wvalue, &dvalue, &dsign); +} +#undef WIN_SIZE + + +////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////// + +#include + +#define BP_WIN_SIZE MUL_BASEPOINT_WIN_SIZE /* defined in the header above */ + +__INLINE void MB_FUNC_NAME(booth_recode_bp_)(__mb_mask* sign, U64* dvalue, U64 wvalue) +{ + U64 one = set1(1); + U64 zero = get_zero64(); + U64 t = srli64(wvalue, BP_WIN_SIZE); + __mb_mask s = cmp64_mask(t, zero, _MM_CMPINT_NE); + U64 d = sub64( sub64(set1(1<<(BP_WIN_SIZE+1)), wvalue), one); + d = mask_mov64(wvalue, s, d); + U64 odd = and64(d, one); + d = add64( srli64(d, 1), odd); + + *sign = s; + *dvalue = d; +} + +/* extract affine affine point */ +__INLINE void MB_FUNC_NAME(extract_point_affine_)(P521_POINT_AFFINE* r, const SINGLE_P521_POINT_AFFINE* tbl, U64 idx) +{ + /* decrement index (the table noes not contain [0]*P */ + U64 targIdx = sub64(idx, set1(1)); + + U64 ax0, ax1, ax2, ax3, ax4, ax5, ax6, ax7, ax8, ax9, ax10; + U64 ay0, ay1, ay2, ay3, ay4, ay5, ay6, ay7, ay8, ay9, ay10; + + /* assume the point at infinity is what need */ + ax0 = ax1 = ax2 = ax3 = ax4 = ax5 = ax6 = ax7 = ax8 = ax9 = ax10 = + ay0 = ay1 = ay2 = ay3 = ay4 = ay5 = ay6 = ay7 = ay8 = ay9 = ay10 = get_zero64(); + + /* find out what we actually need or just keep original infinity */ + int n; + U64 currIdx = get_zero64(); + for(n=0; n<(1<<(BP_WIN_SIZE-1)); n++, tbl++, currIdx = add64(currIdx, set1(1))) { + __mb_mask k = cmp64_mask(currIdx, targIdx, _MM_CMPINT_EQ); + + /* R = k? set1( tbl[] ) : R */ + ax0 = mask_add64(ax0, k, ax0, set1( tbl->x[0] )); + ax1 = mask_add64(ax1, k, ax1, set1( tbl->x[1] )); + ax2 = mask_add64(ax2, k, ax2, set1( tbl->x[2] )); + ax3 = mask_add64(ax3, k, ax3, set1( tbl->x[3] )); + ax4 = mask_add64(ax4, k, ax4, set1( tbl->x[4] )); + ax5 = mask_add64(ax5, k, ax5, set1( tbl->x[5] )); + ax6 = mask_add64(ax6, k, ax6, set1( tbl->x[6] )); + ax7 = mask_add64(ax7, k, ax7, set1( tbl->x[7] )); + ax8 = mask_add64(ax8, k, ax8, set1( tbl->x[8] )); + ax9 = mask_add64(ax9, k, ax9, set1( tbl->x[9] )); + ax10= mask_add64(ax10,k, ax10,set1( tbl->x[10])); + + ay0 = mask_add64(ay0, k, ay0, set1( tbl->y[0] )); + ay1 = mask_add64(ay1, k, ay1, set1( tbl->y[1] )); + ay2 = mask_add64(ay2, k, ay2, set1( tbl->y[2] )); + ay3 = mask_add64(ay3, k, ay3, set1( tbl->y[3] )); + ay4 = mask_add64(ay4, k, ay4, set1( tbl->y[4] )); + ay5 = mask_add64(ay5, k, ay5, set1( tbl->y[5] )); + ay6 = mask_add64(ay6, k, ay6, set1( tbl->y[6] )); + ay7 = mask_add64(ay7, k, ay7, set1( tbl->y[7] )); + ay8 = mask_add64(ay8, k, ay8, set1( tbl->y[8] )); + ay9 = mask_add64(ay9, k, ay9, set1( tbl->y[9] )); + ay10= mask_add64(ay10,k, ay10,set1( tbl->y[10])); + } + + r->x[0] = ax0; + r->x[1] = ax1; + r->x[2] = ax2; + r->x[3] = ax3; + r->x[4] = ax4; + r->x[5] = ax5; + r->x[6] = ax6; + r->x[7] = ax7; + r->x[8] = ax8; + r->x[9] = ax9; + r->x[10]= ax10; + + r->y[0] = ay0; + r->y[1] = ay1; + r->y[2] = ay2; + r->y[3] = ay3; + r->y[4] = ay4; + r->y[5] = ay5; + r->y[6] = ay6; + r->y[7] = ay7; + r->y[8] = ay8; + r->y[9] = ay9; + r->y[10]= ay10; +} + +void MB_FUNC_NAME(ifma_ec_nistp521_mul_pointbase_)(P521_POINT* r, const U64 scalar[]) +{ + /* pre-computed table of base powers */ + SINGLE_P521_POINT_AFFINE* tbl = &ifma_ec_nistp521_bp_precomp[0][0]; + + P521_POINT R; + P521_POINT_AFFINE A; + U64 Ty[P521_LEN52]; + + /* R = O */ + MB_FUNC_NAME(set_point_to_infinity_)(&R); + + + /* + // base point (RL) multiplication + */ + U64 wvalue, dvalue; + __mb_mask dsign; + + U64 idx_mask = set1( (1<<(BP_WIN_SIZE+1))-1 ); + int bit = 0; + + /* first window - window[0] */ + wvalue = loadu64(&scalar[0]); + wvalue = and64( slli64(wvalue, 1), idx_mask); + MB_FUNC_NAME(booth_recode_bp_)(&dsign, &dvalue, wvalue); + MB_FUNC_NAME(extract_point_affine_)(&A, tbl, dvalue); + tbl+=BP_N_ENTRY; + + /* A = dsign? -A : A */ + MB_FUNC_NAME(ifma_neg52_p521_)(Ty, A.y); + MB_FUNC_NAME(secure_mask_mov_FE521_)(A.y, A.y, dsign, Ty); + + /* R += A */ + MB_FUNC_NAME(ifma_ec_nistp521_add_point_affine_)(&R, &R, &A); + + int chunk_no; + int chunk_shift; + for(bit+=BP_WIN_SIZE; bit<=P521_BITSIZE; bit+=BP_WIN_SIZE) { + chunk_no = (bit-1)/64; + chunk_shift = (bit-1)%64; + + wvalue = loadu64(&scalar[chunk_no]); + #if (_MSC_VER <= 1916) /* VS 2017 not supported _mm512_shrdv_epi64 */ + { + __m512i t_lo_ = _mm512_srlv_epi64(wvalue, set64(chunk_shift)); + __m512i t_hi_ = _mm512_sllv_epi64(loadu64(&scalar[chunk_no+1]), set64(64-chunk_shift)); + wvalue = or64(t_lo_, t_hi_); + } + #else + wvalue = _mm512_shrdv_epi64(wvalue, loadu64(&scalar[chunk_no+1]), set1((int32u)chunk_shift)); + #endif + wvalue = and64(wvalue, idx_mask); + + MB_FUNC_NAME(booth_recode_bp_)(&dsign, &dvalue, wvalue); + MB_FUNC_NAME(extract_point_affine_)(&A, tbl, dvalue); + tbl+=BP_N_ENTRY; + + /* A = dsign? -A : A */ + MB_FUNC_NAME(ifma_neg52_p521_)(Ty, A.y); + MB_FUNC_NAME(secure_mask_mov_FE521_)(A.y, A.y, dsign, Ty); + + /* R += A */ + MB_FUNC_NAME(ifma_ec_nistp521_add_point_affine_)(&R, &R, &A); + } + + /* r = R */ + MB_FUNC_NAME(mov_FE521_)(r->X, R.X); + MB_FUNC_NAME(mov_FE521_)(r->Y, R.Y); + MB_FUNC_NAME(mov_FE521_)(r->Z, R.Z); + + /* clear stubs of secret scalar */ + clear_secret_context(&wvalue, &dvalue, &dsign); +} +#undef BP_WIN_SIZE + + +/* P521 parameters: mont(a), mont(b) */ +__ALIGN64 int64u mont_a_p521_mb[P521_LEN52][8] = { + { REP8_DECL(0x0007ffffffffffff) }, + { REP8_DECL(0x000ffffffffffffe) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x0000000000000001) } +}; +__ALIGN64 int64u mont_b_p521_mb[P521_LEN52][8] = { + { REP8_DECL(0x00014654fae58638) }, + { REP8_DECL(0x00028fea35a81f80) }, + { REP8_DECL(0x000c41e961a78f7a) }, + { REP8_DECL(0x000dd8df839ab9ef) }, + { REP8_DECL(0x00049bd8b29605e9) }, + { REP8_DECL(0x0000ab0c9ca8f63f) }, + { REP8_DECL(0x0005a44c8c77884f) }, + { REP8_DECL(0x00092dccd98af9dc) }, + { REP8_DECL(0x0005b42a077516d3) }, + { REP8_DECL(0x000e4d0fc94d10d0) }, + { REP8_DECL(0x0000000000000000) } +}; + +/* +// We have a curve defined by a Weierstrass equation: y^2 = x^3 + a*x + b. +// +// The points are considered in Jacobian projective coordinates +// where (X, Y, Z) represents (x, y) = (X/Z^2, Y/Z^3). +// Substituting this and multiplying by Z^6 transforms the above equation into +// Y^2 = X^3 + a*X*Z^4 + b*Z^6 +// To test this, we add up the right-hand side in 'rh'. +*/ +__mb_mask MB_FUNC_NAME(ifma_is_on_curve_p521_)(const P521_POINT* p, int use_jproj_coords) +{ + U64 rh[P521_LEN52]; + U64 Z4[P521_LEN52], Z6[P521_LEN52], tmp[P521_LEN52]; + + /* rh := X^2 */ + MB_FUNC_NAME(ifma_ams52_p521_)(rh, p->X); + + /* if Z!=1, then rh = X^3 + a*X*Z^4 + b*Z^6 = X*(X^2 + a*X*Z^4) + b*Z^6 */ + if(use_jproj_coords) { + MB_FUNC_NAME(ifma_ams52_p521_)(tmp, p->Z); /* tmp = Z^2 */ + MB_FUNC_NAME(ifma_ams52_p521_)(Z4, tmp); /* Z4 = Z^4 */ + MB_FUNC_NAME(ifma_amm52_p521_)(Z6, Z4, tmp); /* Z6 = Z^6 */ + + MB_FUNC_NAME(ifma_add52_p521_)(tmp, Z4, Z4); /* tmp = 2*Z^4 */ + MB_FUNC_NAME(ifma_add52_p521_)(tmp, tmp, Z4); /* tmp = 2*Z^4 */ + MB_FUNC_NAME(ifma_sub52_p521_)(rh, rh, tmp); /* rh = X^2 + a*Z^4 */ + MB_FUNC_NAME(ifma_amm52_p521_)(rh, rh, p->X); /* rh = (X^2 + a*Z^4)*X */ + + MB_FUNC_NAME(ifma_amm52_p521_)(tmp, Z6, (U64*)mont_b_p521_mb); + MB_FUNC_NAME(ifma_add52_p521_)(rh, rh, tmp); /* rh = (X^2 + a*Z^4)*X + b*Z^6 */ + } + /* if Z==1, then rh = X^3 + a*X + b = X*(X^2 +a) b */ + else { + MB_FUNC_NAME(ifma_add52_p521_)(rh, rh, (U64*)mont_a_p521_mb); /* rh = X^2+a */ + MB_FUNC_NAME(ifma_amm52_p521_)(rh, rh, p->X); /* rh = (X^2+a)*X */ + MB_FUNC_NAME(ifma_add52_p521_)(rh, rh, (U64*)mont_b_p521_mb); /* rh = (X^2+a)*X + b */ + } + MB_FUNC_NAME(ifma_frommont52_p521_)(rh, rh); + + /* rl = tmp = Y^2 */ + MB_FUNC_NAME(ifma_ams52_p521_)(tmp, p->Y); + MB_FUNC_NAME(ifma_frommont52_p521_)(tmp, tmp); + + /* mask = rl==rh */ + __mb_mask is_on_curve_mask = MB_FUNC_NAME(cmp_eq_FE521_)(tmp, rh); + + return is_on_curve_mask; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecpubkey_p256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecpubkey_p256.c new file mode 100644 index 0000000..64f1055 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecpubkey_p256.c @@ -0,0 +1,213 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +#include +#include +#include +#include + +#ifndef BN_OPENSSL_DISABLE +#include +#endif + +#ifndef BN_OPENSSL_DISABLE +/* +// Computes public key +// pa_pubx[] array of pointers to the public keys X-coordinates +// pa_puby[] array of pointers to the public keys Y-coordinates +// pa_pubz[] array of pointers to the public keys Z-coordinates (or NULL, if affine coordinate requested) +// pa_skey[] array of pointers to the private keys +// pBuffer pointer to the scratch buffer +// +// Note: +// output public key depends on pa_pubz[] parameter and represented either +// - in (X:Y:Z) projective Jacobian coordinates if pa_pubz[] != NULL +// or +// - in (x:y) affine coordinate if pa_pubz[] == NULL +*/ +DLL_PUBLIC +mbx_status mbx_nistp256_ecpublic_key_ssl_mb8(BIGNUM* pa_pubx[8], + BIGNUM* pa_puby[8], + BIGNUM* pa_pubz[8], + const BIGNUM* const pa_skey[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* pa_bubz!=0 means the output is in Jacobian projective coordinates */ + int use_jproj_coords = NULL!=pa_pubz; + + /* test input pointers */ + if(NULL==pa_pubx || NULL==pa_puby || NULL==pa_skey) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + BIGNUM* out_x = pa_pubx[buf_no]; + BIGNUM* out_y = pa_puby[buf_no]; + BIGNUM* out_z = use_jproj_coords? pa_pubz[buf_no] : NULL; + const BIGNUM* key = pa_skey[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL==out_x || NULL==out_y || (use_jproj_coords && NULL==out_z) || NULL==key) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* + // processing + */ + + /* zero padded keys */ + U64 scalarz[P256_LEN64+1]; + ifma_BN_transpose_copy((int64u (*)[8])scalarz, pa_skey, P256_BITSIZE); + scalarz[P256_LEN64] = get_zero64(); + + status |= MBX_SET_STS_BY_MASK(status, is_zero(scalarz, P256_LEN64+1), MBX_STATUS_MISMATCH_PARAM_ERR); + + /* do not need to clear copy of secret keys before this return - all of them is NULL or zero */ + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* public key */ + P256_POINT P; + + /* compute public keys */ + MB_FUNC_NAME(ifma_ec_nistp256_mul_pointbase_)(&P, scalarz); + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])scalarz, sizeof(scalarz)/sizeof(U64)); + + if(!use_jproj_coords) + MB_FUNC_NAME(get_nistp256_ec_affine_coords_)(P.X, P.Y, &P); + + /* convert P coordinates to regular domain */ + MB_FUNC_NAME(ifma_frommont52_p256_)(P.X, P.X); + MB_FUNC_NAME(ifma_frommont52_p256_)(P.Y, P.Y); + if(use_jproj_coords) + MB_FUNC_NAME(ifma_frommont52_p256_)(P.Z, P.Z); + + /* convert public key and store BIGNUM result */ + int8u tmp[8][NUMBER_OF_DIGITS(P256_BITSIZE,8)]; + int8u* const pa_tmp[8] = {tmp[0],tmp[1],tmp[2],tmp[3],tmp[4],tmp[5],tmp[6],tmp[7]}; + + /* X */ + ifma_mb8_to_HexStr8(pa_tmp, (const int64u (*)[8])P.X, P256_BITSIZE); + for(buf_no=0; (buf_no<8) && (0==MBX_GET_STS(status, buf_no)); buf_no++) { + BN_bin2bn(pa_tmp[buf_no], NUMBER_OF_DIGITS(P256_BITSIZE,8), pa_pubx[buf_no]); + } + + /* Y */ + ifma_mb8_to_HexStr8(pa_tmp, (const int64u (*)[8])P.Y, P256_BITSIZE); + for(buf_no=0; (buf_no<8) && (0==MBX_GET_STS(status, buf_no)); buf_no++) { + BN_bin2bn(pa_tmp[buf_no], NUMBER_OF_DIGITS(P256_BITSIZE,8), pa_puby[buf_no]); + } + + /* Z */ + if(use_jproj_coords) { + ifma_mb8_to_HexStr8(pa_tmp, (const int64u (*)[8])P.Z, P256_BITSIZE); + for(buf_no=0; (buf_no<8) && (0==MBX_GET_STS(status, buf_no)); buf_no++) { + BN_bin2bn(pa_tmp[buf_no], NUMBER_OF_DIGITS(P256_BITSIZE,8), pa_pubz[buf_no]); + } + } + + return status; +} +#endif // BN_OPENSSL_DISABLE + +DLL_PUBLIC +mbx_status mbx_nistp256_ecpublic_key_mb8(int64u* pa_pubx[8], + int64u* pa_puby[8], + int64u* pa_pubz[8], + const int64u* const pa_skey[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* pa_bubz!=0 means the output is in Jacobian projective coordinates */ + int use_jproj_coords = NULL!=pa_pubz; + + /* test input pointers */ + if(NULL==pa_pubx || NULL==pa_puby || NULL==pa_skey) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + int64u* out_x = pa_pubx[buf_no]; + int64u* out_y = pa_puby[buf_no]; + int64u* out_z = use_jproj_coords? pa_pubz[buf_no] : NULL; + const int64u* key = pa_skey[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL==out_x || NULL==out_y || (use_jproj_coords && NULL==out_z) || NULL==key) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* + // processing + */ + + /* zero padded keys */ + U64 scalarz[P256_LEN64+1]; + ifma_BNU_transpose_copy((int64u (*)[8])scalarz, pa_skey, P256_BITSIZE); + scalarz[P256_LEN64] = get_zero64(); + + status |= MBX_SET_STS_BY_MASK(status, is_zero(scalarz, P256_LEN64+1), MBX_STATUS_MISMATCH_PARAM_ERR); + + /* do not need to clear copy of secret keys before this return - all of them is NULL or zero */ + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* public key */ + P256_POINT P; + + /* compute public keys */ + MB_FUNC_NAME(ifma_ec_nistp256_mul_pointbase_)(&P, scalarz); + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])scalarz, sizeof(scalarz)/sizeof(U64)); + + if(!use_jproj_coords) + MB_FUNC_NAME(get_nistp256_ec_affine_coords_)(P.X, P.Y, &P); + + /* convert P coordinates to regular domain */ + MB_FUNC_NAME(ifma_frommont52_p256_)(P.X, P.X); + MB_FUNC_NAME(ifma_frommont52_p256_)(P.Y, P.Y); + if(use_jproj_coords) + MB_FUNC_NAME(ifma_frommont52_p256_)(P.Z, P.Z); + + /* store result */ + ifma_mb8_to_BNU(pa_pubx, (const int64u (*)[8])P.X, P256_BITSIZE); + ifma_mb8_to_BNU(pa_puby, (const int64u (*)[8])P.Y, P256_BITSIZE); + if(use_jproj_coords) + ifma_mb8_to_BNU(pa_pubz, (const int64u (*)[8])P.Z, P256_BITSIZE); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecpubkey_p384.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecpubkey_p384.c new file mode 100644 index 0000000..c51d194 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecpubkey_p384.c @@ -0,0 +1,213 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +#include +#include +#include +#include + +#ifndef BN_OPENSSL_DISABLE +#include +#endif + +#ifndef BN_OPENSSL_DISABLE +/* +// Computes public key +// pa_pubx[] array of pointers to the public keys X-coordinates +// pa_puby[] array of pointers to the public keys Y-coordinates +// pa_pubz[] array of pointers to the public keys Z-coordinates (or NULL, if affine coordinate requested) +// pa_skey[] array of pointers to the private keys +// pBuffer pointer to the scratch buffer +// +// Note: +// output public key depends on pa_pubz[] parameter and represented either +// - in (X:Y:Z) projective Jacobian coordinates if pa_pubz[] != NULL +// or +// - in (x:y) affine coordinate if pa_pubz[] == NULL +*/ +DLL_PUBLIC +mbx_status mbx_nistp384_ecpublic_key_ssl_mb8(BIGNUM* pa_pubx[8], + BIGNUM* pa_puby[8], + BIGNUM* pa_pubz[8], + const BIGNUM* const pa_skey[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* pa_bubz!=0 means the output is in Jacobian projective coordinates */ + int use_jproj_coords = NULL!=pa_pubz; + + /* test input pointers */ + if(NULL==pa_pubx || NULL==pa_puby || NULL==pa_skey) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + BIGNUM* out_x = pa_pubx[buf_no]; + BIGNUM* out_y = pa_puby[buf_no]; + BIGNUM* out_z = use_jproj_coords? pa_pubz[buf_no] : NULL; + const BIGNUM* key = pa_skey[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL==out_x || NULL==out_y || (use_jproj_coords && NULL==out_z) || NULL==key) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* + // processing + */ + + /* zero padded keys */ + U64 scalarz[P384_LEN64+1]; + ifma_BN_transpose_copy((int64u (*)[8])scalarz, pa_skey, P384_BITSIZE); + scalarz[P384_LEN64] = get_zero64(); + + status |= MBX_SET_STS_BY_MASK(status, is_zero(scalarz, P384_LEN64+1), MBX_STATUS_MISMATCH_PARAM_ERR); + + /* do not need to clear copy of secret keys before this return - all of them is NULL or zero */ + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* public key */ + P384_POINT P; + + /* compute public keys */ + MB_FUNC_NAME(ifma_ec_nistp384_mul_pointbase_)(&P, scalarz); + /* clear copy of secret */ + MB_FUNC_NAME(zero_)((int64u (*)[8])scalarz, sizeof(scalarz)/sizeof(U64)); + + if(!use_jproj_coords) + MB_FUNC_NAME(get_nistp384_ec_affine_coords_)(P.X, P.Y, &P); + + /* convert P coordinates to regular domain */ + MB_FUNC_NAME(ifma_frommont52_p384_)(P.X, P.X); + MB_FUNC_NAME(ifma_frommont52_p384_)(P.Y, P.Y); + if(use_jproj_coords) + MB_FUNC_NAME(ifma_frommont52_p384_)(P.Z, P.Z); + + /* convert public key and store BIGNUM result */ + int8u tmp[8][NUMBER_OF_DIGITS(P384_BITSIZE,8)]; + int8u* const pa_tmp[8] = {tmp[0],tmp[1],tmp[2],tmp[3],tmp[4],tmp[5],tmp[6],tmp[7]}; + + /* X */ + ifma_mb8_to_HexStr8(pa_tmp, (const int64u (*)[8])P.X, P384_BITSIZE); + for(buf_no=0; (buf_no<8) && (0==MBX_GET_STS(status, buf_no)); buf_no++) { + BN_bin2bn(pa_tmp[buf_no], NUMBER_OF_DIGITS(P384_BITSIZE,8), pa_pubx[buf_no]); + } + + /* Y */ + ifma_mb8_to_HexStr8(pa_tmp, (const int64u (*)[8])P.Y, P384_BITSIZE); + for(buf_no=0; (buf_no<8) && (0==MBX_GET_STS(status, buf_no)); buf_no++) { + BN_bin2bn(pa_tmp[buf_no], NUMBER_OF_DIGITS(P384_BITSIZE,8), pa_puby[buf_no]); + } + + /* Z */ + if(use_jproj_coords) { + ifma_mb8_to_HexStr8(pa_tmp, (const int64u (*)[8])P.Z, P384_BITSIZE); + for(buf_no=0; (buf_no<8) && (0==MBX_GET_STS(status, buf_no)); buf_no++) { + BN_bin2bn(pa_tmp[buf_no], NUMBER_OF_DIGITS(P384_BITSIZE,8), pa_pubz[buf_no]); + } + } + + return status; +} +#endif // BN_OPENSSL_DISABLE + +DLL_PUBLIC +mbx_status mbx_nistp384_ecpublic_key_mb8(int64u* pa_pubx[8], + int64u* pa_puby[8], + int64u* pa_pubz[8], + const int64u* const pa_skey[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* pa_bubz!=0 means the output is in Jacobian projective coordinates */ + int use_jproj_coords = NULL!=pa_pubz; + + /* test input pointers */ + if(NULL==pa_pubx || NULL==pa_puby || NULL==pa_skey) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + int64u* out_x = pa_pubx[buf_no]; + int64u* out_y = pa_puby[buf_no]; + int64u* out_z = use_jproj_coords? pa_pubz[buf_no] : NULL; + const int64u* key = pa_skey[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL==out_x || NULL==out_y || (use_jproj_coords && NULL==out_z) || NULL==key) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* + // processing + */ + + /* zero padded keys */ + U64 scalarz[P384_LEN64+1]; + ifma_BNU_transpose_copy((int64u (*)[8])scalarz, pa_skey, P384_BITSIZE); + scalarz[P384_LEN64] = get_zero64(); + + status |= MBX_SET_STS_BY_MASK(status, is_zero(scalarz, P384_LEN64+1), MBX_STATUS_MISMATCH_PARAM_ERR); + + /* do not need to clear copy of secret keys before this return - all of them is NULL or zero */ + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* public key */ + P384_POINT P; + + /* compute public keys */ + MB_FUNC_NAME(ifma_ec_nistp384_mul_pointbase_)(&P, scalarz); + /* clear copy of secret */ + MB_FUNC_NAME(zero_)((int64u (*)[8])scalarz, sizeof(scalarz)/sizeof(U64)); + + if(!use_jproj_coords) + MB_FUNC_NAME(get_nistp384_ec_affine_coords_)(P.X, P.Y, &P); + + /* convert P coordinates to regular domain */ + MB_FUNC_NAME(ifma_frommont52_p384_)(P.X, P.X); + MB_FUNC_NAME(ifma_frommont52_p384_)(P.Y, P.Y); + if(use_jproj_coords) + MB_FUNC_NAME(ifma_frommont52_p384_)(P.Z, P.Z); + + /* store result */ + ifma_mb8_to_BNU(pa_pubx, (const int64u (*)[8])P.X, P384_BITSIZE); + ifma_mb8_to_BNU(pa_puby, (const int64u (*)[8])P.Y, P384_BITSIZE); + if(use_jproj_coords) + ifma_mb8_to_BNU(pa_pubz, (const int64u (*)[8])P.Z, P384_BITSIZE); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecpubkey_p521.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecpubkey_p521.c new file mode 100644 index 0000000..7df5bb4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ecnist/ifma_ecpubkey_p521.c @@ -0,0 +1,213 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +#include +#include +#include +#include + +#ifndef BN_OPENSSL_DISABLE +#include +#endif + +#ifndef BN_OPENSSL_DISABLE +/* +// Computes public key +// pa_pubx[] array of pointers to the public keys X-coordinates +// pa_puby[] array of pointers to the public keys Y-coordinates +// pa_pubz[] array of pointers to the public keys Z-coordinates (or NULL, if affine coordinate requested) +// pa_skey[] array of pointers to the private keys +// pBuffer pointer to the scratch buffer +// +// Note: +// output public key depends on pa_pubz[] parameter and represented either +// - in (X:Y:Z) projective Jacobian coordinates if pa_pubz[] != NULL +// or +// - in (x:y) affine coordinate if pa_pubz[] == NULL +*/ +DLL_PUBLIC +mbx_status mbx_nistp521_ecpublic_key_ssl_mb8(BIGNUM* pa_pubx[8], + BIGNUM* pa_puby[8], + BIGNUM* pa_pubz[8], + const BIGNUM* const pa_skey[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* pa_bubz!=0 means the output is in Jacobian projective coordinates */ + int use_jproj_coords = NULL!=pa_pubz; + + /* test input pointers */ + if(NULL==pa_pubx || NULL==pa_puby || NULL==pa_skey) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + BIGNUM* out_x = pa_pubx[buf_no]; + BIGNUM* out_y = pa_puby[buf_no]; + BIGNUM* out_z = use_jproj_coords? pa_pubz[buf_no] : NULL; + const BIGNUM* key = pa_skey[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL==out_x || NULL==out_y || (use_jproj_coords && NULL==out_z) || NULL==key) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* + // processing + */ + + /* zero padded keys */ + U64 scalarz[P521_LEN64+1]; + ifma_BN_transpose_copy((int64u (*)[8])scalarz, pa_skey, P521_BITSIZE); + scalarz[P521_LEN64] = get_zero64(); + + status |= MBX_SET_STS_BY_MASK(status, is_zero(scalarz, P521_LEN64+1), MBX_STATUS_MISMATCH_PARAM_ERR); + + /* do not need to clear copy of secret keys before this return - all of them is NULL or zero */ + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* public key */ + P521_POINT P; + + /* compute public keys */ + MB_FUNC_NAME(ifma_ec_nistp521_mul_pointbase_)(&P, scalarz); + /* clear copy of secret */ + MB_FUNC_NAME(zero_)((int64u (*)[8])scalarz, sizeof(scalarz)/sizeof(U64)); + + if(!use_jproj_coords) + MB_FUNC_NAME(get_nistp521_ec_affine_coords_)(P.X, P.Y, &P); + + /* convert P coordinates to regular domain */ + MB_FUNC_NAME(ifma_frommont52_p521_)(P.X, P.X); + MB_FUNC_NAME(ifma_frommont52_p521_)(P.Y, P.Y); + if(use_jproj_coords) + MB_FUNC_NAME(ifma_frommont52_p521_)(P.Z, P.Z); + + /* convert public key and store BIGNUM result */ + int8u tmp[8][NUMBER_OF_DIGITS(P521_BITSIZE,8)]; + int8u* const pa_tmp[8] = {tmp[0],tmp[1],tmp[2],tmp[3],tmp[4],tmp[5],tmp[6],tmp[7]}; + + /* X */ + ifma_mb8_to_HexStr8(pa_tmp, (const int64u (*)[8])P.X, P521_BITSIZE); + for(buf_no=0; (buf_no<8) && (0==MBX_GET_STS(status, buf_no)); buf_no++) { + BN_bin2bn(pa_tmp[buf_no], NUMBER_OF_DIGITS(P521_BITSIZE,8), pa_pubx[buf_no]); + } + + /* Y */ + ifma_mb8_to_HexStr8(pa_tmp, (const int64u (*)[8])P.Y, P521_BITSIZE); + for(buf_no=0; (buf_no<8) && (0==MBX_GET_STS(status, buf_no)); buf_no++) { + BN_bin2bn(pa_tmp[buf_no], NUMBER_OF_DIGITS(P521_BITSIZE,8), pa_puby[buf_no]); + } + + /* Z */ + if(use_jproj_coords) { + ifma_mb8_to_HexStr8(pa_tmp, (const int64u (*)[8])P.Z, P521_BITSIZE); + for(buf_no=0; (buf_no<8) && (0==MBX_GET_STS(status, buf_no)); buf_no++) { + BN_bin2bn(pa_tmp[buf_no], NUMBER_OF_DIGITS(P521_BITSIZE,8), pa_pubz[buf_no]); + } + } + + return status; +} +#endif // BN_OPENSSL_DISABLE + +DLL_PUBLIC +mbx_status mbx_nistp521_ecpublic_key_mb8(int64u* pa_pubx[8], + int64u* pa_puby[8], + int64u* pa_pubz[8], + const int64u* const pa_skey[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* pa_bubz!=0 means the output is in Jacobian projective coordinates */ + int use_jproj_coords = NULL!=pa_pubz; + + /* test input pointers */ + if(NULL==pa_pubx || NULL==pa_puby || NULL==pa_skey) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + int64u* out_x = pa_pubx[buf_no]; + int64u* out_y = pa_puby[buf_no]; + int64u* out_z = use_jproj_coords? pa_pubz[buf_no] : NULL; + const int64u* key = pa_skey[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL==out_x || NULL==out_y || (use_jproj_coords && NULL==out_z) || NULL==key) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* + // processing + */ + + /* zero padded keys */ + U64 scalarz[P521_LEN64+1]; + ifma_BNU_transpose_copy((int64u (*)[8])scalarz, pa_skey, P521_BITSIZE); + scalarz[P521_LEN64] = get_zero64(); + + status |= MBX_SET_STS_BY_MASK(status, is_zero(scalarz, P521_LEN64+1), MBX_STATUS_MISMATCH_PARAM_ERR); + + /* do not need to clear copy of secret keys before this return - all of them is NULL or zero */ + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* public key */ + P521_POINT P; + + /* compute public keys */ + MB_FUNC_NAME(ifma_ec_nistp521_mul_pointbase_)(&P, scalarz); + /* clear copy of secret */ + MB_FUNC_NAME(zero_)((int64u (*)[8])scalarz, sizeof(scalarz)/sizeof(U64)); + + if(!use_jproj_coords) + MB_FUNC_NAME(get_nistp521_ec_affine_coords_)(P.X, P.Y, &P); + + /* convert P coordinates to regular domain */ + MB_FUNC_NAME(ifma_frommont52_p521_)(P.X, P.X); + MB_FUNC_NAME(ifma_frommont52_p521_)(P.Y, P.Y); + if(use_jproj_coords) + MB_FUNC_NAME(ifma_frommont52_p521_)(P.Z, P.Z); + + /* store result */ + ifma_mb8_to_BNU(pa_pubx, (const int64u (*)[8])P.X, P521_BITSIZE); + ifma_mb8_to_BNU(pa_puby, (const int64u (*)[8])P.Y, P521_BITSIZE); + if(use_jproj_coords) + ifma_mb8_to_BNU(pa_pubz, (const int64u (*)[8])P.Z, P521_BITSIZE); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ed25519/ifma_arith_ed25519.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ed25519/ifma_arith_ed25519.c new file mode 100644 index 0000000..5d0ac48 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ed25519/ifma_arith_ed25519.c @@ -0,0 +1,540 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include +#include + +#define BP_WIN_SIZE MUL_BASEPOINT_WIN_SIZE /* defined in the header above */ + +/* +// Twisted Edwards Curve parameters +*/ + +/* d = -(121665/121666) */ +__ALIGN64 static const int64u ed25519_d[FE_LEN52][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(0x000b4dca135978a3) }, + { REP8_DECL(0x0004d4141d8ab75e) }, + { REP8_DECL(0x000779e89800700a) }, + { REP8_DECL(0x000fe738cc740797) }, + { REP8_DECL(0x000052036cee2b6f) } +}; + +/* (2*d) */ +__ALIGN64 static const int64u ed25519_d2[FE_LEN52][sizeof(U64) / sizeof(int64u)] = { + { REP8_DECL(0x00069b9426b2f159) }, + { REP8_DECL(0x0009a8283b156ebd) }, + { REP8_DECL(0x000ef3d13000e014) }, + { REP8_DECL(0x000fce7198e80f2e) }, + { REP8_DECL(0x00002406d9dc56df) } +}; + +/* 2^((p-1)/4) = 0x2b8324804fc1df0b2b4d00993dfbd7a72f431806ad2fe478c4ee1b274a0ea0b0 */ +__ALIGN64 static const int64u ed25519_2_pm1_4[FE_LEN52][sizeof(U64) / sizeof(int64u)] = { + { REP8_DECL(0x000e1b274a0ea0b0) }, + { REP8_DECL(0x00006ad2fe478c4e) }, + { REP8_DECL(0x000dfbd7a72f4318) }, + { REP8_DECL(0x000df0b2b4d00993) }, + { REP8_DECL(0x00002b8324804fc1) } +}; + +/* ext => cached */ +__INLINE void ge_ext_to_cached_mb(ge52_cached_mb *r, const ge52_ext_mb* p) +{ + fe52_add(r->YaddX, p->Y, p->X); + fe52_sub(r->YsubX, p->Y, p->X); + fe52_copy_mb(r->Z, p->Z); + fe52_mul(r->T2d, p->T, (U64*)ed25519_d2); +} + +/* +// r = p + q +*/ +static void ge52_add_precomp(ge52_p1p1_mb *r, const ge52_ext_mb *p, const ge52_precomp_mb *q) +{ + fe52_mb t0; + + fe52_add(r->X, p->Y, p->X); // X3 = Y1+X1 + fe52_sub(r->Y, p->Y, p->X); // Y3 = Y1-X1 + fe52_mul(r->Z, r->X, q->yaddx); // Z3 = X3*yplusx2 + fe52_mul(r->Y, r->Y, q->ysubx); // Y3 = Y3*yminisx2 + fe52_mul(r->T, q->t2d, p->T); // T3 = T1*xy2d2 + fe52_add(t0, p->Z, p->Z); // t0 = Z1+Z1 + fe52_sub(r->X, r->Z, r->Y); // X3 = Z3-Y3 = X3*yplusx2 - Y3*yminisx2 = (Y1+X1)*yplusx2 - (Y1-X1)*yminisx2 + fe52_add(r->Y, r->Z, r->Y); // Y3 = Z3+Y3 = X3*yplusx2 + Y3*yminisx2 = (Y1+X1)*yplusx2 + (Y1-X1)*yminisx2 + fe52_add(r->Z, t0, r->T); // Z3 = 2*Z1 + T1*xy2d2 + fe52_sub(r->T, t0, r->T); // T3 = 2*Z1 - T1*xy2d2 +} + +static void ge_add(ge52_p1p1_mb* r, const ge52_ext_mb* p, const ge52_cached_mb* q) +{ + fe52_mb t0; + + fe52_add(r->X, p->Y, p->X); + fe52_sub(r->Y, p->Y, p->X); + fe52_mul(r->Z, r->X, q->YaddX); + fe52_mul(r->Y, r->Y, q->YsubX); + fe52_mul(r->T, q->T2d, p->T); + fe52_mul(r->X, p->Z, q->Z); + fe52_add(t0, r->X, r->X); + fe52_sub(r->X, r->Z, r->Y); + fe52_add(r->Y, r->Z, r->Y); + fe52_add(r->Z, t0, r->T); + fe52_sub(r->T, t0, r->T); +} + +/* r = 2 * p */ +static void ge_dbl(ge52_p1p1_mb *r, const ge52_homo_mb* p) +{ + fe52_mb t0; + + fe52_sqr(r->X, p->X); + fe52_sqr(r->Z, p->Y); + fe52_sqr(r->T, p->Z); + fe52_add(r->T, r->T, r->T); + fe52_add(r->Y, p->X, p->Y); + fe52_sqr(t0, r->Y); + fe52_add(r->Y, r->Z, r->X); + fe52_sub(r->Z, r->Z, r->X); + fe52_sub(r->X, t0, r->Y); + fe52_sub(r->T, r->T, r->Z); +} + + +#define PARITY_SLOT ((GE25519_COMP_BITSIZE-1)/DIGIT_SIZE) +#define PARITY_BIT (1LL << ((GE25519_COMP_BITSIZE - 1) % DIGIT_SIZE)) + +#define SIGN_FE52(fe) srli64((fe)[PARITY_SLOT], ((GE25519_COMP_BITSIZE - 1) % DIGIT_SIZE)) +#define PARITY_FE52(fe) and64_const((fe)[0], 1) + +/* compress point */ +void ge52_ext_compress(fe52_mb r, const ge52_ext_mb* p) +{ + fe52_mb recip; + fe52_mb x; + fe52_mb y; + + fe52_inv(recip, p->Z); + fe52_mul(x, p->X, recip); + fe52_mul(y, p->Y, recip); + fe52_red(x, x); + fe52_red(y, y); + + __mb_mask is_negative = cmp64_mask(and64_const(x[0], 1), set1(1), _MM_CMPINT_EQ); + y[(GE25519_COMP_BITSIZE-1)/DIGIT_SIZE] = _mm512_mask_or_epi64(y[(GE25519_COMP_BITSIZE-1)/DIGIT_SIZE], is_negative, y[(GE25519_COMP_BITSIZE - 1) / DIGIT_SIZE], set1(1LL << (GE25519_COMP_BITSIZE-1)%DIGIT_SIZE)); + + fe52_copy_mb(r, y); +} + +/* decompress point */ +__mb_mask ge52_ext_decompress(ge52_ext_mb* r, const fe52_mb compressed_point) +{ + fe52_mb x; + fe52_mb y; + + fe52_mb u; + fe52_mb v; + fe52_mb v3; + fe52_mb vxx; + + /* inital values are neutral */ + neutral_ge52_ext_mb(r); + + /* y = fe and clear "x-sign" bit */ + fe52_copy_mb(y, compressed_point); + y[FE_LEN52-1] = and64_const(y[FE_LEN52-1], PRIME25519_HI); + + /* x^2 = (y^2 -1) / (d y^2 +1) = u/v. compute numerator (u) and denumerator (v) */ + fe52_sqr(u, y); + fe52_mul(v, u, (U64*)ed25519_d); + fe52_sub(u, u, r->Z); /* u = y^2-1 */ + fe52_add(v, v, r->Z); /* v = dy^2+1 */ + + /* + // compute candidate x = sqrt(u/v) = uv^3(uv^7)^((p-5)/8) + */ + fe52_sqr(v3, v); + fe52_mul(v3, v3, v); /* v3 = v^3 */ + fe52_sqr(x, v3); + fe52_mul(x, x, v); + fe52_mul(x, x, u); /* x = uv^7 */ + + fe52_p2_252m3(x, x); /* x = (uv^7)^((p-5)/8) */ + fe52_mul(x, x, v3); + fe52_mul(x, x, u); /* x = uv^3(uv^7)^((p-5)/8) */ + + fe52_sqr(vxx, x); + fe52_mul(vxx, vxx, v); /* vxx = v*x^2 */ + + /* case 1: check if v*x^2 == u */ + fe52_sub(v3, vxx, u); /* v*x^2-u */ + __mb_mask k1 = fe52_mb_is_zero(v3); + fe52_cmov_mb(r->X, r->X, k1, x); + + /* case 2: check if v*x^2 == -u */ + fe52_add(v3, vxx, u); + __mb_mask k2 = fe52_mb_is_zero(v3); + fe52_mul(x, x, (U64*)ed25519_2_pm1_4); + fe52_cmov_mb(r->X, r->X, k2, x); + + /* copy y coordinate*/ + __mb_mask k = k1|k2; + fe52_cmov_mb(r->Y, r->Y, k, y); + + /* set x to (-x) if x.parity different with compressed_point.sign */ + k1 = cmp64_mask(SIGN_FE52(compressed_point), PARITY_FE52(r->X), _MM_CMPINT_NE); + fe52_neg(v3, r->X); + fe52_cmov_mb(r->X, r->X, k1, v3); + + /* update T compomemt */ + fe52_mul(r->T, r->X, r->Y); + + return k; +} + + +/* select from the pre-computed table */ +static void extract_precomputed_basepoint_dual(ge52_precomp_mb* p0, + ge52_precomp_mb* p1, + const ge52_precomp* tbl, + U64 idx0, U64 idx1) +{ + /* set h0, h1 to neutral point */ + neutral_ge52_precomp_mb(p0); + neutral_ge52_precomp_mb(p1); + + /* indexes are considered as signed values */ + __mb_mask is_neg_idx0 = cmp64_mask(idx0, get_zero64(), _MM_CMPINT_LT); + __mb_mask is_neg_idx1 = cmp64_mask(idx1, get_zero64(), _MM_CMPINT_LT); + idx0 = mask_sub64(idx0, is_neg_idx0, get_zero64(), idx0); + idx1 = mask_sub64(idx1, is_neg_idx1, get_zero64(), idx1); + + /* select p0, p1 wrt idx0, idx1 indexes */ + ge52_cmov1_precomp_mb(p0, p0, cmp64_mask(idx0, set1(1), _MM_CMPINT_EQ), tbl); + ge52_cmov1_precomp_mb(p1, p1, cmp64_mask(idx1, set1(1), _MM_CMPINT_EQ), tbl); + tbl++; + ge52_cmov1_precomp_mb(p0, p0, cmp64_mask(idx0, set1(2), _MM_CMPINT_EQ), tbl); + ge52_cmov1_precomp_mb(p1, p1, cmp64_mask(idx1, set1(2), _MM_CMPINT_EQ), tbl); + tbl++; + ge52_cmov1_precomp_mb(p0, p0, cmp64_mask(idx0, set1(3), _MM_CMPINT_EQ), tbl); + ge52_cmov1_precomp_mb(p1, p1, cmp64_mask(idx1, set1(3), _MM_CMPINT_EQ), tbl); + tbl++; + ge52_cmov1_precomp_mb(p0, p0, cmp64_mask(idx0, set1(4), _MM_CMPINT_EQ), tbl); + ge52_cmov1_precomp_mb(p1, p1, cmp64_mask(idx1, set1(4), _MM_CMPINT_EQ), tbl); + tbl++; + ge52_cmov1_precomp_mb(p0, p0, cmp64_mask(idx0, set1(5), _MM_CMPINT_EQ), tbl); + ge52_cmov1_precomp_mb(p1, p1, cmp64_mask(idx1, set1(5), _MM_CMPINT_EQ), tbl); + tbl++; + ge52_cmov1_precomp_mb(p0, p0, cmp64_mask(idx0, set1(6), _MM_CMPINT_EQ), tbl); + ge52_cmov1_precomp_mb(p1, p1, cmp64_mask(idx1, set1(6), _MM_CMPINT_EQ), tbl); + tbl++; + ge52_cmov1_precomp_mb(p0, p0, cmp64_mask(idx0, set1(7), _MM_CMPINT_EQ), tbl); + ge52_cmov1_precomp_mb(p1, p1, cmp64_mask(idx1, set1(7), _MM_CMPINT_EQ), tbl); + tbl++; + ge52_cmov1_precomp_mb(p0, p0, cmp64_mask(idx0, set1(8), _MM_CMPINT_EQ), tbl); + ge52_cmov1_precomp_mb(p1, p1, cmp64_mask(idx1, set1(8), _MM_CMPINT_EQ), tbl); + + /* adjust for sign */ + fe52_mb neg; + fe52_neg(neg, p0->t2d); + fe52_cswap_mb(p0->ysubx, is_neg_idx0, p0->yaddx); + fe52_cmov_mb(p0->t2d, p0->t2d, is_neg_idx0, neg); + + fe52_neg(neg, p1->t2d); + fe52_cswap_mb(p1->ysubx, is_neg_idx1, p1->yaddx); + fe52_cmov_mb(p1->t2d, p1->t2d, is_neg_idx1, neg); +} + +/* +// r = [s]*G +// +// where s = s[0] + 256*s[1] +...+ 256^31*s[31] +// G is the Ed25519 base point (x,4/5) with x positive. +// +// Preconditions: +// s[31] <= 127 +*/ + +/* if msb set */ +__INLINE int32u isMsb_ct(int32u a) +{ return (int32u)0 - (a >> (sizeof(a) * 8 - 1)); } + +/* tests if a==0 */ +__INLINE int32u isZero(int32u a) +{ return isMsb_ct(~a & (a - 1)); } + +/* tests if a==b */ +__INLINE int32u isEqu(int32u a, int32u b) +{ return isZero(a ^ b); } + +void ifma_ed25519_mul_basepoint(ge52_ext_mb* r, const U64 scalar[]) +{ + /* implementation uses scalar representation over base b=16, q[i] are half-bytes */ + __ALIGN64 ge52_ext_mb r0; /* q[0]*16^0 + q[2]*16^2 + q[4]*16^4 + ...+ q[30]*16^30 */ + __ALIGN64 ge52_ext_mb r1; /* q[1]*16^1 + q[3]*16^3 + q[5]*16^5 + ...+ q[31]*16^31 */ + + /* point extracted from the pre-computed table */ + __ALIGN64 ge52_precomp_mb h0; + __ALIGN64 ge52_precomp_mb h1; + + /* temporary */ + __ALIGN64 ge52_p1p1_mb t; + __ALIGN64 ge52_homo_mb s; + + /* inital values are nuetral */ + neutral_ge52_ext_mb(&r0); + neutral_ge52_ext_mb(&r1); + + /* pre-computed basepoint table */ + ge52_precomp* tbl = &ifma_ed25519_bp_precomp[0][0]; + + __mb_mask carry = 0; /* used in convertion to signed value */ + + for(int n=0; n< FE_LEN64; n++) { + /* scalar value*/ + U64 scalarV = loadu64(&scalar[n]); + + for(int m=0; m> ws) - 1)); //sign + d = (1 << (ws+1)) - wvalue - 1; // digit, win size "ws" + d = (d & s) | (wvaluen & ~s); + d = (d >> 1) + (d & 1); + *sign = s & 1; + *digit = (Ipp8u)d; +*/ +__INLINE void booth_recode(__mb_mask* sign, U64* dvalue, U64 wvalue) +{ + U64 one = set1(1); + U64 zero = get_zero64(); + U64 t = srli64(wvalue, WIN_SIZE); + __mb_mask s = cmp64_mask(t, zero, _MM_CMPINT_NE); + U64 d = sub64(sub64(set1(1 << (WIN_SIZE + 1)), wvalue), one); + d = mask_mov64(wvalue, s, d); + U64 odd = and64(d, one); + d = add64(srli64(d, 1), odd); + + *sign = s; + *dvalue = d; +} + +/* extract point */ +static void extract_cached_point(ge52_cached_mb* r, const ge52_cached_mb tbl[], U64 idx, __mb_mask sign) +{ + /* decrement index (the table does not contain neutral element) */ + U64 idx_target = sub64(idx, set1(1)); + + neutral_ge52_cached_mb(r); + + /* find out what we actually need or just keep neutral */ + int32u n; + for(n=0; n<(1<<(WIN_SIZE-1)); n++) { + U64 idx_curr = set1(n); + __mb_mask k = cmp64_mask(idx_curr, idx_target, _MM_CMPINT_EQ); + /* r = k? tbl[] : r */ + cmov_ge52_cached_mb(r, r, k, &tbl[n]); + } + + /* adjust for sign */ + fe52_mb neg; + fe52_neg(neg, r->T2d); + fe52_cswap_mb(r->YsubX, sign, r->YaddX); + fe52_cmov_mb(r->T2d, r->T2d, sign, neg); +} + +/* +// r = [s]*P +// +// where s = s[0] + 256*s[1] +...+ 256^31*s[31] +// P is an arbitrary Ed25519 point. +*/ +void ifma_ed25519_mul_point(ge52_ext_mb* r, const ge52_ext_mb* p, const U64 scalar[]) +{ + __ALIGN64 ge52_p1p1_mb p1p1; + __ALIGN64 ge52_homo_mb homo; + __ALIGN64 ge52_cached_mb cached; + __ALIGN64 ge52_ext_mb ext; + + /* generate the table tbl[] = {p, [2]p, [3]p, [4]p, [5]p, [6]p, [7]p, [8]p} */ + __ALIGN64 ge52_cached_mb tbl[1 << (WIN_SIZE-1)]; + int n; + ge_ext_to_cached_mb(&tbl[0], p); /* tbl[0] = p */ + for(n=1; n<(1 << (WIN_SIZE-1)); n++) { + ge_add(&p1p1, p, &tbl[n-1]); /* tbl[n] = p + tbl[n-1] */ + ge52_p1p1_to_ext_mb(&ext, &p1p1); + ge_ext_to_cached_mb(&tbl[n], &ext); + } + + /* ext = neutral */ + neutral_ge52_ext_mb(&ext); + + /* + // point (LR) multiplication, 256-bit scalar + */ + U64 idx_mask = set1((1 << (WIN_SIZE + 1)) - 1); + int bit = SCALAR_BITSIZE - (SCALAR_BITSIZE% WIN_SIZE); + int chunk_no = (bit - 1) / 64; + int chunk_shift = (bit - 1) % 64; + + U64 dvalue; /* digit value */ + __mb_mask dsign; /* digit sign */ + + /* + // first window + */ + U64 wvalue = loadu64(&scalar[chunk_no]); + wvalue = and64(srli64(wvalue, chunk_shift), idx_mask); + + booth_recode(&dsign, &dvalue, wvalue); + extract_cached_point(&cached, tbl, dvalue, dsign); + + /* ext = cached */ + ge_add(&p1p1, &ext, &cached); + ge52_p1p1_to_ext_mb(&ext, &p1p1); + + /* + // other windows + */ + for(bit-=WIN_SIZE; bit>=WIN_SIZE; bit-=WIN_SIZE) { + /* 4-x doubling: ext = [16]ext */ + ge52_ext_to_homo_mb(&homo, &ext); + + ge_dbl(&p1p1, &homo); + ge52_p1p1_to_homo_mb(&homo, &p1p1); + ge_dbl(&p1p1, &homo); + ge52_p1p1_to_homo_mb(&homo, &p1p1); + ge_dbl(&p1p1, &homo); + ge52_p1p1_to_homo_mb(&homo, &p1p1); + ge_dbl(&p1p1, &homo); + ge52_p1p1_to_ext_mb(&ext, &p1p1); + + /* extract precomputed []P */ + chunk_no = (bit - 1) / 64; + chunk_shift = (bit - 1) % 64; + + wvalue = loadu64(&scalar[chunk_no]); + wvalue = _mm512_shrdv_epi64(wvalue, loadu64(&scalar[chunk_no + 1]), set1((int32u)chunk_shift)); + wvalue = and64(wvalue, idx_mask); + + booth_recode(&dsign, &dvalue, wvalue); + extract_cached_point(&cached, tbl, dvalue, dsign); + + /* acumulate ext += cached */ + ge_add(&p1p1, &ext, &cached); + ge52_p1p1_to_ext_mb(&ext, &p1p1); + } + + /* + // last window + */ + ge52_ext_to_homo_mb(&homo, &ext); + ge_dbl(&p1p1, &homo); + ge52_p1p1_to_homo_mb(&homo, &p1p1); + ge_dbl(&p1p1, &homo); + ge52_p1p1_to_homo_mb(&homo, &p1p1); + ge_dbl(&p1p1, &homo); + ge52_p1p1_to_homo_mb(&homo, &p1p1); + ge_dbl(&p1p1, &homo); + ge52_p1p1_to_ext_mb(&ext, &p1p1); + + /* extract precomputed []P */ + wvalue = loadu64(&scalar[0]); + wvalue = and64(slli64(wvalue, 1), idx_mask); + booth_recode(&dsign, &dvalue, wvalue); + extract_cached_point(&cached, tbl, dvalue, dsign); + + /* acumulate ext += cached */ + ge_add(&p1p1, &ext, &cached); + ge52_p1p1_to_ext_mb(r, &p1p1); +} +#undef SCALAR_BITSIZE +#undef WIN_SIZE + +/* R = [p]P + [g]G */ +void ifma_ed25519_prod_point(ge52_ext_mb* r, const ge52_ext_mb* p, const U64 scalarP[], const U64 scalarG[]) +{ + __ALIGN64 ge52_ext_mb T_mb; + ifma_ed25519_mul_point(r, p, scalarP); + ifma_ed25519_mul_basepoint(&T_mb, scalarG); + + __ALIGN64 ge52_cached_mb c; + __ALIGN64 ge52_p1p1_mb t; + ge_ext_to_cached_mb(&c, &T_mb); + ge_add(&t, r, &c); + ge52_p1p1_to_ext_mb(r, &t); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ed25519/ifma_arith_n25519.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ed25519/ifma_arith_n25519.c new file mode 100644 index 0000000..86dea33 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ed25519/ifma_arith_n25519.c @@ -0,0 +1,390 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +#include +#include + +/* +// ED25519 prime base point order +// n = 2^252+27742317777372353535851937790883648493 = 0x1000000000000000000000000000000014DEF9DEA2F79CD65812631A5CF5D3ED +// in 2^52 radix +*/ +__ALIGN64 static const int64u ed25519n_mb[NE_LEN52][sizeof(U64) / sizeof(int64u)] = { + { REP8_DECL(0x0002631A5CF5D3ED) }, + { REP8_DECL(0x000DEA2F79CD6581) }, + { REP8_DECL(0x000000000014DEF9) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000100000000000) } +}; + +/* mu = floor( 2^2*DIGIT_SIZE/n) = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEB2106215D086329A7ED9CE5A30A2C131B39 */ +__ALIGN64 static const int64u ed25519mu_mb[NE_LEN52+1][sizeof(U64) / sizeof(int64u)] = { + { REP8_DECL(0x0005A30A2C131B39) }, + { REP8_DECL(0x000086329A7ED9CE) }, + { REP8_DECL(0x000FFFEB2106215D) }, + { REP8_DECL(0x000FFFFFFFFFFFFF) }, + { REP8_DECL(0x000FFFFFFFFFFFFF) }, + { REP8_DECL(0x00000000000000FF) } +}; + +#define ROUND_MUL(R_LO, R_HI, A_I, B_J) \ + (R_LO) = fma52lo((R_LO), (A_I), (B_J)); \ + (R_HI) = fma52hi((R_HI), (A_I), (B_J)); + +__ALIGN64 static const int64u VBASE52[sizeof(U64)/sizeof(int64u)] = { + REP8_DECL(DIGIT_BASE) +}; + +__ALIGN64 static const int64u VMASK52[sizeof(U64)/sizeof(int64u)] = { + REP8_DECL(DIGIT_MASK) +}; + + +#define NORM_LSHIFTR(LO, HI) \ + (HI) = add64((HI), srli64((LO), DIGIT_SIZE)); \ + (LO) = and64((LO), loadu64(VMASK52)); + + +void ifma52_sub_with_borrow(U64 r[], const U64 x[], const U64 y[]) +{ + U64 base = loadu64(VBASE52); + U64 one = set1(1LL); + + U64 subtrahend = get_zero64(); + __mmask8 lt = 0; + + subtrahend = _mm512_mask_add_epi64(y[0], lt, y[0], one); + lt = _mm512_cmp_epu64_mask(x[0], subtrahend, _MM_CMPINT_LT); + r[0] = _mm512_sub_epi64(x[0], subtrahend); + r[0] = _mm512_mask_add_epi64(r[0], lt, r[0], base); + + subtrahend = _mm512_mask_add_epi64(y[1], lt, y[1], one); + lt = _mm512_cmp_epu64_mask(x[1], subtrahend, _MM_CMPINT_LT); + r[1] = _mm512_sub_epi64(x[1], subtrahend); + r[1] = _mm512_mask_add_epi64(r[1], lt, r[1], base); + + subtrahend = _mm512_mask_add_epi64(y[2], lt, y[2], one); + lt = _mm512_cmp_epu64_mask(x[2], subtrahend, _MM_CMPINT_LT); + r[2] = _mm512_sub_epi64(x[2], subtrahend); + r[2] = _mm512_mask_add_epi64(r[2], lt, r[2], base); + + subtrahend = _mm512_mask_add_epi64(y[3], lt, y[3], one); + lt = _mm512_cmp_epu64_mask(x[3], subtrahend, _MM_CMPINT_LT); + r[3] = _mm512_sub_epi64(x[3], subtrahend); + r[3] = _mm512_mask_add_epi64(r[3], lt, r[3], base); + + subtrahend = _mm512_mask_add_epi64(y[4], lt, y[4], one); + lt = _mm512_cmp_epu64_mask(x[4], subtrahend, _MM_CMPINT_LT); + r[4] = _mm512_sub_epi64(x[4], subtrahend); + r[4] = _mm512_mask_add_epi64(r[4], lt, r[4], base); + + /* the latest step is not necessary, bcause Barett algorithms guarantee that + // 0<= (r = r1-r2) <3*modulus + // r[0..4] is enough to keep 3*modulus + */ + #if 0 + subtrahend = _mm512_mask_add_epi64(y[5], lt, y[5], one); + lt = _mm512_cmp_epu64_mask(x[5], subtrahend, _MM_CMPINT_LT); + r[5] = _mm512_sub_epi64(x[5], subtrahend); + r[5] = _mm512_mask_add_epi64(r[5], lt, r[4], base); + #endif +} + +// r = x>DITIT_SIZE*(K+1) +// +// 4. r1 = x mod b^(k+1) +// 5. r2 = q3*n mod b^(k+1) +// 6. r = r1-r2, if r<0 then r+=b^(k+1) +// 7. if r>n then r -=n -- at most 2 final reductions +// r>n then r -=n +*/ +void ifma52_ed25519n_reduce(U64 r[NE_LEN52], const U64 x[NE_LEN52*2]) +{ + /* 1. q1 = x/b^(k-1) -- just (k+1) most significant digits, len(q1) = 2*k - (k-1) = k+1 */ + const U64* q1 = x + (NE_LEN52-1); + + /* + // 2. q2 = q1*mu, len(q2) = (k+1) + (k+1) = 2*k+2 + // 3. q3 = floor(q2 / b^(k+1)), len(q3) = 2*k+2 - (k+1) = k+1 + // => no necessary do compute whole (q1*mu) product, (k+1) most significant digits are enough + // + // code below represents full multiplication + // from which discarded (commented out) computation of the least significant digits + */ + U64* mu = (U64*)ed25519mu_mb; + U64 /*q3_0, q3_1, q3_2, q3_3, q3_4, */ q3_5, q3_6, q3_7, q3_8, q3_9, q3_10, q3_11; + /*q3_0= q3_1= q3_2= q3_3= q3_4= */ q3_5 = q3_6 = q3_7 = q3_8 = q3_9 = q3_10 = q3_11 = get_zero64(); + + //ROUND_MUL(q3_0, q3_1, q1[0], mu[0]); + + //ROUND_MUL(q3_1, q3_2, q1[0], mu[1]); + //ROUND_MUL(q3_1, q3_2, q1[1], mu[0]); + + //ROUND_MUL(q3_2, q3_3, q1[0], mu[2]); + //ROUND_MUL(q3_2, q3_3, q1[1], mu[1]); + //ROUND_MUL(q3_2, q3_3, q1[2], mu[0]); + + //ROUND_MUL(q3_3, q3_4, q1[0], mu[3]); + //ROUND_MUL(q3_3, q3_4, q1[1], mu[2]); + //ROUND_MUL(q3_3, q3_4, q1[2], mu[1]); + //ROUND_MUL(q3_3, q3_4, q1[3], mu[0]); + + //ROUND_MUL(q3_4, q3_5, q1[0], mu[4]); + //ROUND_MUL(q3_4, q3_5, q1[1], mu[3]); + //ROUND_MUL(q3_4, q3_5, q1[2], mu[2]); + //ROUND_MUL(q3_4, q3_5, q1[3], mu[1]); + //ROUND_MUL(q3_4, q3_5, q1[4], mu[0]); + q3_5 = fma52hi(q3_5, q1[0], mu[4]); + q3_5 = fma52hi(q3_5, q1[1], mu[3]); + q3_5 = fma52hi(q3_5, q1[2], mu[2]); + q3_5 = fma52hi(q3_5, q1[3], mu[1]); + q3_5 = fma52hi(q3_5, q1[4], mu[0]); + + ROUND_MUL(q3_5, q3_6, q1[0], mu[5]); + ROUND_MUL(q3_5, q3_6, q1[1], mu[4]); + ROUND_MUL(q3_5, q3_6, q1[2], mu[3]); + ROUND_MUL(q3_5, q3_6, q1[3], mu[2]); + ROUND_MUL(q3_5, q3_6, q1[4], mu[1]); + ROUND_MUL(q3_5, q3_6, q1[5], mu[0]); + + ROUND_MUL(q3_6, q3_7, q1[1], mu[5]); + ROUND_MUL(q3_6, q3_7, q1[2], mu[4]); + ROUND_MUL(q3_6, q3_7, q1[3], mu[3]); + ROUND_MUL(q3_6, q3_7, q1[4], mu[2]); + ROUND_MUL(q3_6, q3_7, q1[5], mu[1]); + + ROUND_MUL(q3_7, q3_8, q1[2], mu[5]); + ROUND_MUL(q3_7, q3_8, q1[3], mu[4]); + ROUND_MUL(q3_7, q3_8, q1[4], mu[3]); + ROUND_MUL(q3_7, q3_8, q1[5], mu[2]); + + ROUND_MUL(q3_8, q3_9, q1[3], mu[5]); + ROUND_MUL(q3_8, q3_9, q1[4], mu[4]); + ROUND_MUL(q3_8, q3_9, q1[5], mu[3]); + + ROUND_MUL(q3_9, q3_10, q1[4], mu[5]); + ROUND_MUL(q3_9, q3_10, q1[5], mu[4]); + + /* note, that the lates (2*k+2) digit will always be 0 (count bits presentation of x and mu) */ + ROUND_MUL(q3_10, q3_11, q1[5], mu[5]); + + /* normalization, q3 = {q3_11, q3_10, q3_9, q3_8, q3_7, q3_6}, note q3_11 is zero */ + NORM_LSHIFTR(q3_5, q3_6); + NORM_LSHIFTR(q3_6, q3_7); + NORM_LSHIFTR(q3_7, q3_8); + NORM_LSHIFTR(q3_8, q3_9); + NORM_LSHIFTR(q3_9, q3_10); + NORM_LSHIFTR(q3_10, q3_11); + + /* 4. r1 = x mod b^(k+1) is just (k+1) least significant digits of x */ + const U64* r1 = x; + + /* 5. r2 = (q3*n) mod b^(k+1) + // => no necessary do compute whole (q3*n) product, (k+1) least significant digits are enough + // + // code below represents full multiplication + // from which discarded (commented out) computation of the most significant digits + */ + U64* n = (U64*)ed25519n_mb; + U64 r2_0, r2_1, r2_2, r2_3, r2_4, r2_5; + r2_0 = r2_1 = r2_2 = r2_3 = r2_4 = r2_5 = q3_11 = get_zero64(); + + ROUND_MUL(r2_0, r2_1, q3_6, n[0]); + + ROUND_MUL(r2_1, r2_2, q3_6, n[1]); + ROUND_MUL(r2_1, r2_2, q3_7, n[0]); + + ROUND_MUL(r2_2, r2_3, q3_6, n[2]); + ROUND_MUL(r2_2, r2_3, q3_7, n[1]); + ROUND_MUL(r2_2, r2_3, q3_8, n[0]); + + ROUND_MUL(r2_3, r2_4, q3_6, n[3]); + ROUND_MUL(r2_3, r2_4, q3_7, n[2]); + ROUND_MUL(r2_3, r2_4, q3_8, n[1]); + ROUND_MUL(r2_3, r2_4, q3_9, n[0]); + + ROUND_MUL(r2_4, r2_5, q3_6, n[4]); + ROUND_MUL(r2_4, r2_5, q3_7, n[3]); + ROUND_MUL(r2_4, r2_5, q3_8, n[2]); + ROUND_MUL(r2_4, r2_5, q3_9, n[1]); + ROUND_MUL(r2_4, r2_5,q3_10, n[0]); + + r2_5 = fma52lo(r2_5, q3_7, n[4]); + r2_5 = fma52lo(r2_5, q3_8, n[3]); + r2_5 = fma52lo(r2_5, q3_9, n[2]); + r2_5 = fma52lo(r2_5,q3_10, n[1]); + r2_5 = fma52lo(r2_5,q3_11, n[0]); + + /* r2[6], r2[7], ... amd other most significal digits of full multiplication are discarded */ + //r2[6] = fma52hi(r2[5], q3_7, n[4]); + //r2[6] = fma52hi(r2[5], q3_8, n[3]); + //r2[6] = fma52hi(r2[5], q3_9, n[2]); + //r2[6] = fma52hi(r2[5],q3_10, n[1]); + //r2[6] = fma52hi(r2[5],q3_11, n[0]); + + /* normalization */ + NORM_LSHIFTR(r2_0, r2_1); + NORM_LSHIFTR(r2_1, r2_2); + NORM_LSHIFTR(r2_2, r2_3); + NORM_LSHIFTR(r2_3, r2_4); + NORM_LSHIFTR(r2_4, r2_5); + NORM_LSHIFTR(r2_5, q3_11); + /* note q3_11 is out of mod b^(k+1) */ + + U64 r2[NE_LEN52+1]; + r2[0] = r2_0; + r2[1] = r2_1; + r2[2] = r2_2; + r2[3] = r2_3; + r2[4] = r2_4; + r2[5] = r2_5; + + /* 6. r = r1-r2 */ + ifma52_sub_with_borrow(r, r1, r2); + + /* 7. reduce modulo twice */ + ifma52_sub_ed25519n(r, r); + ifma52_sub_ed25519n(r, r); +} + +// r = a*b+c %n +void ifma52_ed25519n_madd(U64 r[NE_LEN52], const U64 a[NE_LEN52], const U64 b[NE_LEN52], const U64 c[NE_LEN52]) +{ + U64 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9; + + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = get_zero64(); + + /* (a*b) */ + ROUND_MUL(r0, r1, a[0], b[0]); + ROUND_MUL(r1, r2, a[1], b[0]); + ROUND_MUL(r2, r3, a[2], b[0]); + ROUND_MUL(r3, r4, a[3], b[0]); + ROUND_MUL(r4, r5, a[4], b[0]); + + ROUND_MUL(r1, r2, a[0], b[1]); + ROUND_MUL(r2, r3, a[1], b[1]); + ROUND_MUL(r3, r4, a[2], b[1]); + ROUND_MUL(r4, r5, a[3], b[1]); + ROUND_MUL(r5, r6, a[4], b[1]); + + ROUND_MUL(r2, r3, a[0], b[2]); + ROUND_MUL(r3, r4, a[1], b[2]); + ROUND_MUL(r4, r5, a[2], b[2]); + ROUND_MUL(r5, r6, a[3], b[2]); + ROUND_MUL(r6, r7, a[4], b[2]); + + ROUND_MUL(r3, r4, a[0], b[3]); + ROUND_MUL(r4, r5, a[1], b[3]); + ROUND_MUL(r5, r6, a[2], b[3]); + ROUND_MUL(r6, r7, a[3], b[3]); + ROUND_MUL(r7, r8, a[4], b[3]); + + ROUND_MUL(r4, r5, a[0], b[4]); + ROUND_MUL(r5, r6, a[1], b[4]); + ROUND_MUL(r6, r7, a[2], b[4]); + ROUND_MUL(r7, r8, a[3], b[4]); + ROUND_MUL(r8, r9, a[4], b[4]); + + /* (a*b) + c */ + r0 = add64(r0, c[0]); + r1 = add64(r1, c[1]); + r2 = add64(r2, c[2]); + r3 = add64(r3, c[3]); + r4 = add64(r4, c[4]); + + /* normalize result */ + NORM_LSHIFTR(r0, r1); + NORM_LSHIFTR(r1, r2); + NORM_LSHIFTR(r2, r3); + NORM_LSHIFTR(r3, r4); + NORM_LSHIFTR(r4, r5); + NORM_LSHIFTR(r5, r6); + NORM_LSHIFTR(r6, r7); + NORM_LSHIFTR(r7, r8); + NORM_LSHIFTR(r8, r9); + + U64 t[NE_LEN52*2]; + t[0] = r0; + t[1] = r1; + t[2] = r2; + t[3] = r3; + t[4] = r4; + t[5] = r5; + t[6] = r6; + t[7] = r7; + t[8] = r8; + t[9] = r9; + ifma52_ed25519n_reduce(r, t); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ed25519/ifma_arith_p25519.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ed25519/ifma_arith_p25519.c new file mode 100644 index 0000000..8412e0d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ed25519/ifma_arith_p25519.c @@ -0,0 +1,527 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +__ALIGN64 static const int64u VPRIME25519_LO[sizeof(U64) / sizeof(int64u)] = { + REP8_DECL(PRIME25519_LO) +}; +__ALIGN64 static const int64u VPRIME25519_MID[sizeof(U64) / sizeof(int64u)] = { + REP8_DECL(PRIME25519_MID) +}; +__ALIGN64 static const int64u VPRIME25519_HI[sizeof(U64) / sizeof(int64u)] = { + REP8_DECL(PRIME25519_HI) +}; + + +/* normalization macros */ +#define NORM_LSHIFTR(R, I, J) \ + R##J = add64(R##J, srli64(R##I, DIGIT_SIZE)); \ + R##I = and64_const(R##I, DIGIT_MASK); +#define NORM_ASHIFTR(R, I, J) \ + R##J = add64(R##J, srai64(R##I, DIGIT_SIZE)); \ + R##I = and64_const(R##I, DIGIT_MASK); + + +//__INLINE +void fe52_mb_add_mod25519(fe52_mb vr, const fe52_mb va, const fe52_mb vb) +{ + /* r = a+b */ + U64 r0 = add64(va[0], vb[0]); + U64 r1 = add64(va[1], vb[1]); + U64 r2 = add64(va[2], vb[2]); + U64 r3 = add64(va[3], vb[3]); + U64 r4 = add64(va[4], vb[4]); + + /* t = r-modulus (2^255-19) */ + U64 t0 = sub64(r0, loadu64(VPRIME25519_LO)); + U64 t1 = sub64(r1, loadu64(VPRIME25519_MID)); + U64 t2 = sub64(r2, loadu64(VPRIME25519_MID)); + U64 t3 = sub64(r3, loadu64(VPRIME25519_MID)); + U64 t4 = sub64(r4, loadu64(VPRIME25519_HI)); + + /* normalize r0, r1, r2, r3, r4 */ + NORM_LSHIFTR(r, 0, 1) + NORM_LSHIFTR(r, 1, 2) + NORM_LSHIFTR(r, 2, 3) + NORM_LSHIFTR(r, 3, 4) + + /* normalize t0, t1, t2, t3, t4 */ + NORM_ASHIFTR(t, 0, 1) + NORM_ASHIFTR(t, 1, 2) + NORM_ASHIFTR(t, 2, 3) + NORM_ASHIFTR(t, 3, 4) + + /* condition mask t4<0? (-1) : 0 */ + __mb_mask cmask = cmp64_mask(t4, get_zero64(), _MM_CMPINT_LT); + + vr[0] = mask_mov64(t0, cmask, r0); + vr[1] = mask_mov64(t1, cmask, r1); + vr[2] = mask_mov64(t2, cmask, r2); + vr[3] = mask_mov64(t3, cmask, r3); + vr[4] = mask_mov64(t4, cmask, r4); +} + +//__INLINE +void fe52_mb_sub_mod25519(fe52_mb vr, const fe52_mb va, const fe52_mb vb) +{ + /* r = a-b */ + U64 r0 = sub64(va[0], vb[0]); + U64 r1 = sub64(va[1], vb[1]); + U64 r2 = sub64(va[2], vb[2]); + U64 r3 = sub64(va[3], vb[3]); + U64 r4 = sub64(va[4], vb[4]); + + /* t = r+modulus (2^255-19) */ + U64 t0 = add64(r0, loadu64(VPRIME25519_LO)); + U64 t1 = add64(r1, loadu64(VPRIME25519_MID)); + U64 t2 = add64(r2, loadu64(VPRIME25519_MID)); + U64 t3 = add64(r3, loadu64(VPRIME25519_MID)); + U64 t4 = add64(r4, loadu64(VPRIME25519_HI)); + + /* normalize r0, r1, r2, r3, r4 */ + NORM_ASHIFTR(r, 0, 1) + NORM_ASHIFTR(r, 1, 2) + NORM_ASHIFTR(r, 2, 3) + NORM_ASHIFTR(r, 3, 4) + + /* normalize t0, t1, t2, t3, t4 */ + NORM_ASHIFTR(t, 0, 1) + NORM_ASHIFTR(t, 1, 2) + NORM_ASHIFTR(t, 2, 3) + NORM_ASHIFTR(t, 3, 4) + + /* condition mask r4<0? (-1) : 0 */ + __mb_mask cmask = cmp64_mask(r4, get_zero64(), _MM_CMPINT_LT); + + vr[0] = mask_mov64(r0, cmask, t0); + vr[1] = mask_mov64(r1, cmask, t1); + vr[2] = mask_mov64(r2, cmask, t2); + vr[3] = mask_mov64(r3, cmask, t3); + vr[4] = mask_mov64(r4, cmask, t4); +} + +//__INLINE +void fe52_mb_neg_mod25519(fe52_mb vr, const fe52_mb va) +{ + __mb_mask non_zero = ~fe52_mb_is_zero(va); + + /* r = is_zero? a : p-a */ + U64 r0 = mask_sub64(va[0], non_zero, loadu64(VPRIME25519_LO), va[0]); + U64 r1 = mask_sub64(va[1], non_zero, loadu64(VPRIME25519_MID), va[1]); + U64 r2 = mask_sub64(va[2], non_zero, loadu64(VPRIME25519_MID), va[2]); + U64 r3 = mask_sub64(va[3], non_zero, loadu64(VPRIME25519_MID), va[3]); + U64 r4 = mask_sub64(va[4], non_zero, loadu64(VPRIME25519_HI), va[4]); + + /* normalize r0, r1, r2, r3, r4 */ + NORM_ASHIFTR(r, 0, 1) + NORM_ASHIFTR(r, 1, 2) + NORM_ASHIFTR(r, 2, 3) + NORM_ASHIFTR(r, 3, 4) + + vr[0] = r0; + vr[1] = r1; + vr[2] = r2; + vr[3] = r3; + vr[4] = r4; +} + +/* +// multiplicative operations +*/ + +/* full multiplication macros */ +#define ROUND_MUL_SRC(I, J, S_LO, R_LO, S_HI, R_HI) \ + R_LO = fma52lo(S_LO, va[I], vb[J]); \ + R_HI = fma52hi(S_HI, va[I], vb[J]); + +#define ROUND_MUL(I, J, M0, M1) \ + ROUND_MUL_SRC(I, J, M0, M0, M1, M1) + +/* reduction constants and macros */ +#define MASK47 ((1ULL << (255 - 52 * 4)) - 1) + +__ALIGN64 static const int64u MASK47_[sizeof(U64) / sizeof(int64u)] = { REP8_DECL(MASK47) }; +__ALIGN64 static const int64u MOD_2_255_[sizeof(U64) / sizeof(int64u)] = { REP8_DECL(19) }; +__ALIGN64 static const int64u MOD_2_260_[sizeof(U64) / sizeof(int64u)] = { REP8_DECL(19*32) }; + +#define MASK_47 loadu64(MASK47_) +#define MOD_2_255 loadu64(MOD_2_255_) +#define MOD_2_260 loadu64(MOD_2_260_) + +#define REDUCE_ROUND(R0, R1, R5) \ + r##R0 = fma52lo(r##R0, r##R5, MOD_2_260); \ + r##R1 = fma52lo( fma52hi(r##R1, r##R5, MOD_2_260), \ + srli64(r##R5, 52), MOD_2_260); + + +//__INLINE +void fe52_mb_mul_mod25519(fe52_mb vr, const fe52_mb va, const fe52_mb vb) +{ + U64 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9; + + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = get_zero64(); + + /* full multiplication */ + ROUND_MUL(4, 4, r8, r9); + ROUND_MUL(3, 0, r3, r4); + ROUND_MUL(1, 2, r3, r4); + ROUND_MUL(0, 3, r3, r4); + ROUND_MUL(2, 1, r3, r4); + ROUND_MUL(2, 2, r4, r5); + ROUND_MUL(0, 4, r4, r5); + ROUND_MUL(1, 3, r4, r5); + ROUND_MUL(3, 1, r4, r5); + ROUND_MUL(4, 0, r4, r5); + + ROUND_MUL(1, 4, r5, r6); + ROUND_MUL(2, 3, r5, r6); + ROUND_MUL(3, 2, r5, r6); + ROUND_MUL(4, 1, r5, r6); + ROUND_MUL(2, 4, r6, r7); + ROUND_MUL(3, 3, r6, r7); + ROUND_MUL(4, 2, r6, r7); + + ROUND_MUL(0, 0, r0, r1); + ROUND_MUL(0, 1, r1, r2); + ROUND_MUL(0, 2, r2, r3); + ROUND_MUL(1, 0, r1, r2); + ROUND_MUL(1, 1, r2, r3); + ROUND_MUL(2, 0, r2, r3); + ROUND_MUL(3, 4, r7, r8); + ROUND_MUL(4, 3, r7, r8); + + /* reduce r4 upper bits */ + r4 = fma52lo(r4, r9, MOD_2_260); + r0 = fma52lo(r0, srli64(r4, 47), MOD_2_255); + r4 = and64(r4, MASK_47); + /* rest of reduction */ + REDUCE_ROUND(0, 1, 5); + REDUCE_ROUND(1, 2, 6); + REDUCE_ROUND(2, 3, 7); + REDUCE_ROUND(3, 4, 8); + + /* normalize result */ + NORM_LSHIFTR(r, 0, 1); + NORM_LSHIFTR(r, 1, 2); + NORM_LSHIFTR(r, 2, 3); + NORM_LSHIFTR(r, 3, 4); + + vr[0] = r0; + vr[1] = r1; + vr[2] = r2; + vr[3] = r3; + vr[4] = r4; +} + +/* SQR +c=0 (0,0) +c=1 (0,1) +c=2 (0,2) (1,1) +c=3 (0,3) (1,2) +c=4 (0,4) (1,3) (2,2) +c=5 (1,4) (2,3) +c=6 (2,4) (3,3) +c=7 (3,4) +c=8 (4,4) +*/ +//__INLINE +void fe52_mb_sqr_mod25519(fe52_mb vr, const fe52_mb va) +{ + U64 *vb = (U64*)va; + + U64 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9; + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = get_zero64(); + + // Square + ROUND_MUL(0, 1, r1, r2); + ROUND_MUL(0, 2, r2, r3); + ROUND_MUL(0, 3, r3, r4); + ROUND_MUL(0, 4, r4, r5); + ROUND_MUL(1, 4, r5, r6); + ROUND_MUL(2, 4, r6, r7); + ROUND_MUL(3, 4, r7, r8); + + ROUND_MUL(1, 2, r3, r4); + ROUND_MUL(1, 3, r4, r5); + ROUND_MUL(2, 3, r5, r6); + + r1 = add64(r1, r1); + r2 = add64(r2, r2); + r3 = add64(r3, r3); + r4 = add64(r4, r4); + r5 = add64(r5, r5); + r6 = add64(r6, r6); + r7 = add64(r7, r7); + r8 = add64(r8, r8); + + ROUND_MUL(0, 0, r0, r1); + ROUND_MUL(1, 1, r2, r3); + ROUND_MUL(2, 2, r4, r5); + ROUND_MUL(3, 3, r6, r7); + ROUND_MUL(4, 4, r8, r9); + + /* reduce r4 upper bits */ + r4 = fma52lo(r4, r9, MOD_2_260); + r0 = fma52lo(r0, srli64(r4, 47), MOD_2_255); + r4 = and64(r4, MASK_47); + /* rest of reduction */ + REDUCE_ROUND(0, 1, 5); + REDUCE_ROUND(1, 2, 6); + REDUCE_ROUND(2, 3, 7); + REDUCE_ROUND(3, 4, 8); + + /* normalize result */ + NORM_LSHIFTR(r, 0, 1); + NORM_LSHIFTR(r, 1, 2); + NORM_LSHIFTR(r, 2, 3); + NORM_LSHIFTR(r, 3, 4); + + vr[0] = r0; + vr[1] = r1; + vr[2] = r2; + vr[3] = r3; + vr[4] = r4; +} + + +/* variant of ROUND_MUL_SRC() and ROUND_MUL() macros */ +#define ROUND_MUL_SRC_A(I, J, S_LO, R_LO, S_HI, R_HI) \ + R_LO = fma52lo(S_LO, a##I, a##J); \ + R_HI = fma52hi(S_HI, a##I, a##J); + +#define ROUND_MUL_A(I, J, M0, M1) \ + ROUND_MUL_SRC_A(I, J, M0, M0, M1, M1) + +void fe52_mb_sqr_mod25519_times(fe52_mb vr, const fe52_mb va, int count) +{ + U64 a0 = va[0]; + U64 a1 = va[1]; + U64 a2 = va[2]; + U64 a3 = va[3]; + U64 a4 = va[4]; + + U64 r0, r1, r2, r3, r4_1, r4, r5, r6, r7, r8, r9; + + for(int i=0; i + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +/* length of SHA512 hash in bits and bytes */ +#define SHA512_HASH_BITLENGTH (512) +#define HASH_LENGTH NUMBER_OF_DIGITS(SHA512_HASH_BITLENGTH, 8) + +static void ed25519_expand_key(int8u* pa_secret_expand[8], const ed25519_private_key* const pa_secret_key[8]) +{ + /* do hash of secret keys in sequence */ + for (int n = 0; n < 8; n++) { + const ed25519_private_key* psecret = pa_secret_key[n]; + if (psecret) { + SHA512MsgDigest(*psecret, sizeof(ed25519_private_key), pa_secret_expand[n]); + /* prune the buffer according to RFC8032 */ + pa_secret_expand[n][0] &= 0xf8; + pa_secret_expand[n][31] &= 0x7f; + pa_secret_expand[n][31] |= 0x40; + } + } +} + +/* +// Computes public key +// pa_public_key[] array of pointers to the public keys X-coordinates +// pa_secret_key[] array of pointers to the public keys Y-coordinates +*/ +DLL_PUBLIC +mbx_status MB_FUNC_NAME(mbx_ed25519_public_key_)(ed25519_public_key* pa_public_key[8], + const ed25519_private_key* const pa_private_key[8]) +{ + mbx_status status = MBX_STATUS_OK; + + /* test input pointers */ + if (NULL == pa_private_key || NULL == pa_public_key) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + int buf_no; + for (buf_no = 0; buf_no < 8; buf_no++) { + const ed25519_private_key* private_key = pa_private_key[buf_no]; + ed25519_public_key* public_key = pa_public_key[buf_no]; + + /* if any of pointer NULL set error status */ + if (NULL == private_key || NULL == public_key) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + /* continue processing if there are correct parameters */ + if (MBX_IS_ANY_OK_STS(status)) { + + int8u sha512_digest[8][HASH_LENGTH] = { 0 }; + int64u* pa_sha512_digest[] = { + (int64u*)sha512_digest[0], (int64u*)sha512_digest[1], (int64u*)sha512_digest[2], (int64u*)sha512_digest[3], + (int64u*)sha512_digest[4], (int64u*)sha512_digest[5], (int64u*)sha512_digest[6], (int64u*)sha512_digest[7] + }; + /* expand secret keys */ + ed25519_expand_key((int8u**)pa_sha512_digest, pa_private_key); + + /* convert into mb fromat */ + U64 scalar[FE_LEN64]; + ifma_BNU_transpose_copy((int64u(*)[8])scalar, (const int64u**)pa_sha512_digest, SHA512_HASH_BITLENGTH/2); + + /* r = [scalar]*G */ + ge52_ext_mb r; + ifma_ed25519_mul_basepoint(&r, scalar); + + /* compress point to public */ + fe52_mb packed_point; + ge52_ext_compress(packed_point, &r); + + /* return result */ + ifma_mb8_to_BNU((int64u * const*)pa_public_key, (const int64u(*)[8])packed_point, GE25519_COMP_BITSIZE); + + /* clear memory containing potentially secret data */ + MB_FUNC_NAME(zero_)((int64u(*)[8])scalar, sizeof(scalar)/sizeof(U64)); + MB_FUNC_NAME(zero_)((int64u(*)[8])sha512_digest, sizeof(sha512_digest)/sizeof(U64)); + } + + return status; +} + +/* +// Computes ED2519 signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_msg[] array of pointers to the messages are being signed +// msgLen[] array of messages lengths (in bytes) +// pa_private_key[] array of pointers to the signer's private keys +// pa_public_key[] array of pointers to the signer's public keys +*/ +static void ed25519_nonce(int8u* pa_nonce[8], int8u* pa_az[8], const int8u* const pa_msg[8], const int32u msgLen[8]) +{ + SHA512State pState; + + for(int n=0; n<8; n++) { + int8u* az = pa_az[n] + HASH_LENGTH/2; + const int8u* msg = pa_msg[n]; + int32u mlen = msgLen[n]; + + SHA512Init(&pState); + SHA512Update(az, 32, &pState); + if (msg) + SHA512Update(msg, mlen, &pState); + + SHA512Final(pa_nonce[n], &pState); + } +} + +static void ed25519_hash_r_pub_msg(int8u* pa_hram[8], const ed25519_sign_component* const pa_sign_r[], const ed25519_public_key* const pa_public_key[8], const int8u* const pa_msg[8], const int32u msgLen[8]) +{ + SHA512State pState; + + for (int n = 0; n < 8; n++) { + const ed25519_sign_component* sign_r = pa_sign_r[n]; + const ed25519_public_key* public = pa_public_key[n]; + const int8u* msg = pa_msg[n]; + int32u mlen = msgLen[n]; + SHA512Init(&pState); + + if(sign_r) + SHA512Update((const int8u*)sign_r, NUMBER_OF_DIGITS(GE25519_COMP_BITSIZE, 8), &pState); + if(public) + SHA512Update((const int8u*)public, sizeof(ed25519_public_key), &pState); + if(msg) + SHA512Update(msg, mlen, &pState); + SHA512Final(pa_hram[n], &pState); + } +} + +DLL_PUBLIC +mbx_status MB_FUNC_NAME(mbx_ed25519_sign_)(ed25519_sign_component* pa_sign_r[8], + ed25519_sign_component* pa_sign_s[8], + const int8u* const pa_msg[8], const int32u msgLen[8], + const ed25519_private_key* const pa_private_key[8], + const ed25519_public_key* const pa_public_key[8]) +{ + mbx_status status = MBX_STATUS_OK; + + /* test input pointers */ + if(NULL == pa_sign_r || NULL == pa_sign_s || + NULL == pa_msg || NULL == msgLen || + NULL == pa_private_key || NULL== pa_public_key) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + int buf_no; + for (buf_no = 0; buf_no < 8; buf_no++) { + ed25519_sign_component* sign_r = pa_sign_r[buf_no]; + ed25519_sign_component* sign_s = pa_sign_s[buf_no]; + const int8u* msg = pa_msg[buf_no]; + const ed25519_private_key* secret = pa_private_key[buf_no]; + const ed25519_public_key* public = pa_public_key[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL == sign_r || NULL == sign_s || NULL== msg || + NULL == secret || NULL == public) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + /* continue processing if there are correct parameters */ + if (MBX_IS_ANY_OK_STS(status)) { + //#define HASH_LENGTH NUMBER_OF_DIGITS(SHA512_DIGEST_BITLENGTH, 8) + + /* expanded secret keys */ + int8u az[8][HASH_LENGTH] = { 0 }; + int64u* pa_az[8] = { + (int64u*)az[0], (int64u*)az[1], (int64u*)az[2], (int64u*)az[3], + (int64u*)az[4], (int64u*)az[5], (int64u*)az[6], (int64u*)az[7] + }; + + /* nonce */ + int8u nonce[8][HASH_LENGTH] = { 0 }; + int64u* pa_nonce[8] = { + (int64u*)nonce[0], (int64u*)nonce[1], (int64u*)nonce[2], (int64u*)nonce[3], + (int64u*)nonce[4], (int64u*)nonce[5], (int64u*)nonce[6], (int64u*)nonce[7] + }; + + /* hram */ + int8u hram[8][HASH_LENGTH] = { 0 }; + int64u* pa_hram[8] = { + (int64u*)hram[0], (int64u*)hram[1], (int64u*)hram[2], (int64u*)hram[3], + (int64u*)hram[4], (int64u*)hram[5], (int64u*)hram[6], (int64u*)hram[7] + }; + + /* expands secret key, az = H(private_key) */ + ed25519_expand_key((int8u**)pa_az, pa_private_key); + U64 mb_az[FE_LEN52]; + ifma_BNU_to_mb8((int64u(*)[8])mb_az, (const int64u**)pa_az, SHA512_HASH_BITLENGTH/2); + + /* computes nonce, nonce = H(az+32, msg) */ + /* Note: only a second half of each pa_az[n] item is used in the ed25519_nonce() function */ + ed25519_nonce((int8u**)pa_nonce, (int8u**)pa_az, pa_msg, msgLen); + + /* reduce nonce wrt n */ + U64 mb_nonce[NUMBER_OF_DIGITS(SHA512_HASH_BITLENGTH, DIGIT_SIZE)]; + ifma_BNU_to_mb8((int64u(*)[8])mb_nonce, (const int64u**)pa_nonce, SHA512_HASH_BITLENGTH); + ifma52_ed25519n_reduce(mb_nonce, mb_nonce); + + /* computes r-component of the signature */ + ifma_mb8_to_BNU(pa_nonce, (const int64u(*)[8])mb_nonce, N25519_BITSIZE); + fe52_mb mb_scalar; + ifma_BNU_transpose_copy((int64u(*)[8])mb_scalar, (const int64u**)pa_nonce, N25519_BITSIZE); + + ge52_ext_mb R; + ifma_ed25519_mul_basepoint(&R, mb_scalar); /* R = [scalar]*G */ + fe52_mb mb_sign_r; + ge52_ext_compress(mb_sign_r, &R); /* compress point to r-component */ + + /* store r-component of the signature */ + ifma_mb8_to_BNU((int64u * const*)pa_sign_r, (const int64u(*)[8])mb_sign_r, GE25519_COMP_BITSIZE); + + /* computes hram, hram = H(r_sign, public_key, msg) */ + ed25519_hash_r_pub_msg((int8u**)pa_hram, (const ed25519_sign_component**)pa_sign_r, pa_public_key, pa_msg, msgLen); + + U64 mb_hram[NUMBER_OF_DIGITS(SHA512_HASH_BITLENGTH, DIGIT_SIZE)]; + ifma_BNU_to_mb8((int64u(*)[8])mb_hram, (const int64u**)pa_hram, SHA512_HASH_BITLENGTH); + ifma52_ed25519n_reduce(mb_hram, mb_hram); + + /* s = hram*az + nonce */ + fe52_mb mb_sign_s; + ifma52_ed25519n_madd(mb_sign_s, mb_hram, mb_az, mb_nonce); + + /* store s-component of the signature */ + ifma_mb8_to_BNU((int64u * const*)pa_sign_s, (const int64u(*)[8])mb_sign_s, N25519_BITSIZE); + + /* clear memory containing potentially secret data */ + MB_FUNC_NAME(zero_)((int64u(*)[8])az, sizeof(az)/sizeof(U64)); + MB_FUNC_NAME(zero_)((int64u(*)[8])nonce, sizeof(nonce)/sizeof(U64)); + MB_FUNC_NAME(zero_)((int64u(*)[8])mb_scalar, sizeof(mb_scalar)/sizeof(U64)); + } + + return status; +} + +/* +// ED25519 prime base point order +// n = 2^252+27742317777372353535851937790883648493 = 0x1000000000000000000000000000000014DEF9DEA2F79CD65812631A5CF5D3ED +// in 2^64 radix +*/ +__ALIGN64 static int64u ed25519n_mb64[NE_LEN64][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(0x5812631A5CF5D3ED) }, + { REP8_DECL(0x14DEF9DEA2F79CD6) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x1000000000000000) } +}; + +DLL_PUBLIC +mbx_status MB_FUNC_NAME(mbx_ed25519_verify_)(const ed25519_sign_component* const pa_sign_r[8], + const ed25519_sign_component* const pa_sign_s[8], + const int8u* const pa_msg[8], const int32u msgLen[8], + const ed25519_public_key* const pa_public_key[8]) +{ + mbx_status status = MBX_STATUS_OK; + + /* test input pointers */ + if (NULL == pa_sign_r || NULL == pa_sign_s || + NULL == pa_msg || NULL == msgLen || + NULL == pa_public_key) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + int buf_no; + for (buf_no = 0; buf_no < 8; buf_no++) { + const ed25519_sign_component* sign_r = pa_sign_r[buf_no]; + const ed25519_sign_component* sign_s = pa_sign_s[buf_no]; + const int8u* msg = pa_msg[buf_no]; + const ed25519_public_key* public = pa_public_key[buf_no]; + + /* if any of pointer NULL set error status */ + if (NULL == sign_r || NULL == sign_s || NULL == msg || + NULL == public) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + /* continue processing if there are correct parameters */ + if (MBX_IS_ANY_OK_STS(status)) { + + /* h = SHA512(sign_r || public_key || msg) */ + __ALIGN64 int8u h[8][HASH_LENGTH] = { 0 }; + int64u* pa_h[8] = { + (int64u*)h[0], (int64u*)h[1], (int64u*)h[2], (int64u*)h[3], + (int64u*)h[4], (int64u*)h[5], (int64u*)h[6], (int64u*)h[7] + }; + ed25519_hash_r_pub_msg((int8u**)pa_h, pa_sign_r, pa_public_key, pa_msg, msgLen); + + /* reduce h %= n */ + __ALIGN64 U64 h_mb[NUMBER_OF_DIGITS(SHA512_HASH_BITLENGTH, DIGIT_SIZE)]; + ifma_BNU_to_mb8((int64u(*)[8])h_mb, (const int64u**)pa_h, SHA512_HASH_BITLENGTH); + ifma52_ed25519n_reduce(h_mb, h_mb); + + /* convert h_mb: fe52 => fe64 */ + __ALIGN64 U64 h64_mb[FE_LEN64+1]; + fe52_to_fe64_mb(h64_mb, h_mb); + h64_mb[FE_LEN64] = get_zero64(); + + /* input r-components to mb */ + __ALIGN64 fe52_mb inputR_mb; + ifma_BNU_to_mb8((int64u(*)[8])inputR_mb, (const int64u**)pa_sign_r, GE25519_COMP_BITSIZE); + + /* input s-components to mb */ + __ALIGN64 U64 s_mb[NE_LEN64 + 1]; + ifma_BNU_transpose_copy((int64u(*)[8])s_mb, (const int64u**)pa_sign_s, 256); + s_mb[NE_LEN64] = get_zero64(); + + /* (comppressed) publickey to mb */ + __ALIGN64 fe52_mb pubfe_mb; + ifma_BNU_to_mb8((int64u(*)[8])pubfe_mb, (const int64u**)pa_public_key, P25519_BITSIZE + 1); + + /* check that s0; n--) { + k |= cmp64_mask(n_mb[n - 1], s_mb[n - 1], _MM_CMPINT_NLE); + } + status |= MBX_SET_STS_BY_MASK(status, ~k, MBX_STATUS_MISMATCH_PARAM_ERR); + + /* continue processing if there are correct parameters */ + if (MBX_IS_ANY_OK_STS(status)) { + /* decompress public to ext point A */ + __ALIGN64 ge52_ext_mb pubA_mb; + k = ge52_ext_decompress(&pubA_mb, pubfe_mb); + fe52_neg(pubA_mb.X, pubA_mb.X); + fe52_neg(pubA_mb.T, pubA_mb.T); + status |= MBX_SET_STS_BY_MASK(status, ~k, MBX_STATUS_SIGNATURE_ERR); + + if (MBX_IS_ANY_OK_STS(status)) { + /* R = [h]A + [s]G */ + __ALIGN64 ge52_ext_mb R_mb; + ifma_ed25519_prod_point(&R_mb, &pubA_mb, h64_mb, s_mb); + + /* recovered r-components */ + __ALIGN64 fe52_mb checkR_mb; + ge52_ext_compress(checkR_mb, &R_mb); + + /* check that recovered r- and input r- components are equal each other */ + k = fe52_mb_is_equ(checkR_mb, inputR_mb); + status |= MBX_SET_STS_BY_MASK(status, ~k, MBX_STATUS_SIGNATURE_ERR); + } + } + } + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ed25519/sha512.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ed25519/sha512.c new file mode 100644 index 0000000..bac28a2 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/ed25519/sha512.c @@ -0,0 +1,226 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +/* setup init hash value */ +static void sha512_init(int64u* pHash) +{ + pHash[0] = sha512_iv[0]; + pHash[1] = sha512_iv[1]; + pHash[2] = sha512_iv[2]; + pHash[3] = sha512_iv[3]; + pHash[4] = sha512_iv[4]; + pHash[5] = sha512_iv[5]; + pHash[6] = sha512_iv[6]; + pHash[7] = sha512_iv[7]; +} + +static void sha512_update(void* uniHash, const int8u* mblk, int mlen) +{ + int32u* data = (int32u*)mblk; + + int64u* digest = (int64u*)uniHash; + int64u* SHA512_cnt_loc = (int64u*)sha512_cnt; + + for (; mlen >= MBS_SHA512; data += MBS_SHA512 / sizeof(int32u), mlen -= MBS_SHA512) { + int64u wdat[16]; + int j; + + int64u v[8]; + + /* initialize the first 16 words in the array W (remember about endian) */ + for (j = 0; j < 16; j++) { + int32u hiX = data[2 * j]; + int32u loX = data[2 * j + 1]; + wdat[j] = MAKEDWORD(ENDIANNESS(loX), ENDIANNESS(hiX)); + } + + /* copy digest */ + CopyBlock(digest, v, SHA512_DIGEST_BITSIZE / BYTESIZE); + + for (j = 0; j < 80; j += 16) { + SHA512_STEP(0, j); + SHA512_STEP(1, j); + SHA512_STEP(2, j); + SHA512_STEP(3, j); + SHA512_STEP(4, j); + SHA512_STEP(5, j); + SHA512_STEP(6, j); + SHA512_STEP(7, j); + SHA512_STEP(8, j); + SHA512_STEP(9, j); + SHA512_STEP(10, j); + SHA512_STEP(11, j); + SHA512_STEP(12, j); + SHA512_STEP(13, j); + SHA512_STEP(14, j); + SHA512_STEP(15, j); + } + + /* update digest */ + digest[0] += v[0]; + digest[1] += v[1]; + digest[2] += v[2]; + digest[3] += v[3]; + digest[4] += v[4]; + digest[5] += v[5]; + digest[6] += v[6]; + digest[7] += v[7]; + } +} + +static void sha512_final(DigestSHA512 pHash, const int8u* inpBuffer, int inpLen, int64u lenLo, int64u lenHi) +{ + /* local buffer and it length */ + int8u buffer[MBS_SHA512 * 2]; + int bufferLen = inpLen < (MBS_SHA512 - (int)MLR_SHA512) ? MBS_SHA512 : MBS_SHA512 * 2; + + /* copy rest of message into internal buffer */ + CopyBlock(inpBuffer, buffer, inpLen); + + /* padd message */ + buffer[inpLen++] = 0x80; + PadBlock(0, buffer + inpLen, (int)(bufferLen - inpLen - (int)MLR_SHA512)); + + /* message length representation */ + lenHi = LSL64(lenHi, 3) | LSR64(lenLo, 63 - 3); + lenLo = LSL64(lenLo, 3); + ((int64u*)(buffer + bufferLen))[-2] = ENDIANNESS64(lenHi); + ((int64u*)(buffer + bufferLen))[-1] = ENDIANNESS64(lenLo); + + /* copmplete hash computation */ + sha512_update(pHash, buffer, bufferLen); +} + + +void SHA512Init(SHA512State* pState) +{ + /* zeros message length */ + HASH_LENLO(pState) = 0; + HASH_LENHI(pState) = 0; + /* message buffer is free */ + HASH_BUFFIDX(pState) = 0; + /* clear ctx buffer */ + PadBlock(0, HASH_BUFF(pState), MBS_SHA512); + + /* setup initial digest */ + sha512_init(HASH_VALUE(pState)); +} + +void SHA512Update(const int8u* pSrc, int len, SHA512State* pState) +{ + +/* +// handle non empty message +*/ +if (len) { + int procLen; + + int idx = HASH_BUFFIDX(pState); + int8u* pBuffer = HASH_BUFF(pState); + int64u lenLo = HASH_LENLO(pState) + (int64u)len; + int64u lenHi = HASH_LENHI(pState); + if (lenLo < HASH_LENLO(pState)) lenHi++; + + /* if non empty internal buffer filling */ + if (idx) { + /* copy from input stream to the internal buffer as match as possible */ + procLen = MIN(len, (MBS_SHA512 - idx)); + CopyBlock(pSrc, pBuffer + idx, procLen); + + /* update message pointer and length */ + pSrc += procLen; + len -= procLen; + idx += procLen; + + /* update digest if buffer full */ + if (MBS_SHA512 == idx) { + sha512_update(HASH_VALUE(pState), pBuffer, MBS_SHA512); + idx = 0; + } + } + + /* main message part processing */ + procLen = len & ~(MBS_SHA512 - 1); + if (procLen) { + sha512_update(HASH_VALUE(pState), pSrc, procLen); + pSrc += procLen; + len -= procLen; + } + + /* store rest of message into the internal buffer */ + if (len) { + CopyBlock(pSrc, pBuffer, len); + idx += len; + } + + /* update length of processed message */ + HASH_LENLO(pState) = lenLo; + HASH_LENHI(pState) = lenHi; + HASH_BUFFIDX(pState) = idx; +} +} + +void SHA512Final(int8u* pMD, SHA512State* pState) +{ + sha512_final(HASH_VALUE(pState), + HASH_BUFF(pState), HASH_BUFFIDX(pState), + HASH_LENLO(pState), HASH_LENHI(pState)); + /* convert hash into big endian */ + ((int64u*)pMD)[0] = ENDIANNESS64(HASH_VALUE(pState)[0]); + ((int64u*)pMD)[1] = ENDIANNESS64(HASH_VALUE(pState)[1]); + ((int64u*)pMD)[2] = ENDIANNESS64(HASH_VALUE(pState)[2]); + ((int64u*)pMD)[3] = ENDIANNESS64(HASH_VALUE(pState)[3]); + ((int64u*)pMD)[4] = ENDIANNESS64(HASH_VALUE(pState)[4]); + ((int64u*)pMD)[5] = ENDIANNESS64(HASH_VALUE(pState)[5]); + ((int64u*)pMD)[6] = ENDIANNESS64(HASH_VALUE(pState)[6]); + ((int64u*)pMD)[7] = ENDIANNESS64(HASH_VALUE(pState)[7]); + + /* re-init hash value */ + SHA512Init(pState); +} + +void SHA512MsgDigest(const int8u* pMsg, int len, int8u* pMD) +{ + /* message length in the multiple MBS and the rest */ + int msgLenBlks = len & (-MBS_SHA512); + int msgLenRest = len - msgLenBlks; + DigestSHA512 hash; + + /* init hash */ + sha512_init(hash); + + /* process main part of the message */ + if (msgLenBlks) { + sha512_update(hash, pMsg, msgLenBlks); + pMsg += msgLenBlks; + } + + sha512_final(hash, pMsg, msgLenRest, (int64u)len, 0); + hash[0] = ENDIANNESS64(hash[0]); + hash[1] = ENDIANNESS64(hash[1]); + hash[2] = ENDIANNESS64(hash[2]); + hash[3] = ENDIANNESS64(hash[3]); + hash[4] = ENDIANNESS64(hash[4]); + hash[5] = ENDIANNESS64(hash[5]); + hash[6] = ENDIANNESS64(hash[6]); + hash[7] = ENDIANNESS64(hash[7]); + + CopyBlock(hash, pMD, SHA512_DIGEST_BITSIZE / BYTESIZE); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/exp/ifma_exp1k_mb.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/exp/ifma_exp1k_mb.c new file mode 100644 index 0000000..4faa2cc --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/exp/ifma_exp1k_mb.c @@ -0,0 +1,227 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + + +#include +#include +#include + + +#define USE_AMS +#ifdef USE_AMS + #define SQUARE_52x20_mb8(out, Y, mod, k0) \ + AMS52x20_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); + + #ifdef USE_AMS_5x + #define SQUARE_5x52x20_mb8(out, Y, mod, k0) \ + AMS5x52x20_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); + #else + #define SQUARE_5x52x20_mb8(out, Y, mod, k0) \ + AMS52x20_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + AMS52x20_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x20_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x20_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x20_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); + #endif +#else + #define SQUARE_52x20_mb8(out, Y, mod, k0) \ + ifma_amm52x20_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); + #define SQUARE_5x52x20_mb8(out, Y, mod, k0) \ + ifma_amm52x20_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x20_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x20_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x20_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x20_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); +#endif + +#define BITSIZE_MODULUS (1024) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,DIGIT_SIZE)) //20 +#define LEN64 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,64)) //16 + + +typedef int64u (*arr_pint64u_x8)[LEN52][8]; // pointer to pre-computed table of base powers + +static int64u* extract_multiplier_mb8(int64u out[LEN52][8], int64u tbl[][LEN52][8], const int64u idx_mb8[8]) +{ + // Assume first element is what need + __m512i X0 = _mm512_load_si512(tbl[0][0]); + __m512i X1 = _mm512_load_si512(tbl[0][1]); + __m512i X2 = _mm512_load_si512(tbl[0][2]); + __m512i X3 = _mm512_load_si512(tbl[0][3]); + __m512i X4 = _mm512_load_si512(tbl[0][4]); + __m512i X5 = _mm512_load_si512(tbl[0][5]); + __m512i X6 = _mm512_load_si512(tbl[0][6]); + __m512i X7 = _mm512_load_si512(tbl[0][7]); + __m512i X8 = _mm512_load_si512(tbl[0][8]); + __m512i X9 = _mm512_load_si512(tbl[0][9]); + __m512i X10= _mm512_load_si512(tbl[0][10]); + __m512i X11= _mm512_load_si512(tbl[0][11]); + __m512i X12= _mm512_load_si512(tbl[0][12]); + __m512i X13= _mm512_load_si512(tbl[0][13]); + __m512i X14= _mm512_load_si512(tbl[0][14]); + __m512i X15= _mm512_load_si512(tbl[0][15]); + __m512i X16= _mm512_load_si512(tbl[0][16]); + __m512i X17= _mm512_load_si512(tbl[0][17]); + __m512i X18= _mm512_load_si512(tbl[0][18]); + __m512i X19= _mm512_load_si512(tbl[0][19]); + + __m512i idx_target = _mm512_load_si512(idx_mb8); + + int n; + // Find out what we actually need or just keep original + for(n=1; n<(1<=0; exp_bit_no-=EXP_WIN_SIZE) { + /* series of squaring */ + #if EXP_WIN_SIZE==5 + SQUARE_5x52x20_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + #else + SQUARE_52x20_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x20_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x20_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x20_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + #endif + + /* extract pre-computed multiplier from the table */ + { + exp_chunk_no = exp_bit_no/64; + exp_chunk_shift = exp_bit_no%64; + + red_table_idx = _mm512_load_si512(expz[exp_chunk_no]); + T = _mm512_load_si512(expz[exp_chunk_no+1]); + + red_table_idx = _mm512_srl_epi64(red_table_idx, _mm_set1_epi64x(exp_chunk_shift)); + T = _mm512_sll_epi64(T, _mm_set1_epi64x(64-exp_chunk_shift)); + red_table_idx = _mm512_and_si512( _mm512_xor_si512(red_table_idx, T), table_idx_mask); + + ifma_extract_amm52x20_mb8 ((int64u*)red_Y, (int64u*)red_Y, red_table, (int64u*)(&red_table_idx), + (int64u*)modulus, (int64u*)k0); + } + } + } + + /* convert result back in regular 2^52 domain */ + zero_mb8(red_X, LEN52); + _mm512_store_si512(red_X, _mm512_set1_epi64(1)); + ifma_amm52x20_mb8((int64u*)out, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); + + zero_mb8(red_Y, LEN52); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/exp/ifma_exp2k_mb.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/exp/ifma_exp2k_mb.c new file mode 100644 index 0000000..dd63627 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/exp/ifma_exp2k_mb.c @@ -0,0 +1,288 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include +#include + + +#define USE_AMS +#ifdef USE_AMS + #define SQUARE_52x40_mb8(out, Y, mod, k0) \ + AMS52x40_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); + + #ifdef USE_AMS_5x + #define SQUARE_5x52x40_mb8(out, Y, mod, k0) \ + AMS5x52x40_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); + #else + #define SQUARE_5x52x40_mb8(out, Y, mod, k0) \ + AMS52x40_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + AMS52x40_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x40_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x40_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x40_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); + #endif +#else + #define SQUARE_52x40_mb8(out, Y, mod, k0) \ + ifma_amm52x40_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); + #define SQUARE_5x52x40_mb8(out, Y, mod, k0) \ + ifma_amm52x40_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x40_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x40_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x40_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x40_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); +#endif + +#define BITSIZE_MODULUS (2048) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,DIGIT_SIZE)) //40 +#define LEN64 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,64)) //32 + + +typedef int64u (*arr_pint64u_x8)[LEN52][8]; // pointer to pre-computed table of base powers + +static int64u* extract_multiplier_mb8(int64u out[LEN52][8], int64u tbl[][LEN52][8], const int64u idx_mb8[8]) +{ + // Assume first element is what need + __m512i X0 = _mm512_load_si512(tbl[0][0]); + __m512i X1 = _mm512_load_si512(tbl[0][1]); + __m512i X2 = _mm512_load_si512(tbl[0][2]); + __m512i X3 = _mm512_load_si512(tbl[0][3]); + __m512i X4 = _mm512_load_si512(tbl[0][4]); + __m512i X5 = _mm512_load_si512(tbl[0][5]); + __m512i X6 = _mm512_load_si512(tbl[0][6]); + __m512i X7 = _mm512_load_si512(tbl[0][7]); + __m512i X8 = _mm512_load_si512(tbl[0][8]); + __m512i X9 = _mm512_load_si512(tbl[0][9]); + __m512i X10= _mm512_load_si512(tbl[0][10]); + __m512i X11= _mm512_load_si512(tbl[0][11]); + __m512i X12= _mm512_load_si512(tbl[0][12]); + __m512i X13= _mm512_load_si512(tbl[0][13]); + __m512i X14= _mm512_load_si512(tbl[0][14]); + __m512i X15= _mm512_load_si512(tbl[0][15]); + __m512i X16= _mm512_load_si512(tbl[0][16]); + __m512i X17= _mm512_load_si512(tbl[0][17]); + __m512i X18= _mm512_load_si512(tbl[0][18]); + __m512i X19= _mm512_load_si512(tbl[0][19]); + __m512i X20= _mm512_load_si512(tbl[0][20]); + __m512i X21= _mm512_load_si512(tbl[0][21]); + __m512i X22= _mm512_load_si512(tbl[0][22]); + __m512i X23= _mm512_load_si512(tbl[0][23]); + __m512i X24= _mm512_load_si512(tbl[0][24]); + __m512i X25= _mm512_load_si512(tbl[0][25]); + __m512i X26= _mm512_load_si512(tbl[0][26]); + __m512i X27= _mm512_load_si512(tbl[0][27]); + __m512i X28= _mm512_load_si512(tbl[0][28]); + __m512i X29= _mm512_load_si512(tbl[0][29]); + __m512i X30= _mm512_load_si512(tbl[0][30]); + __m512i X31= _mm512_load_si512(tbl[0][31]); + __m512i X32= _mm512_load_si512(tbl[0][32]); + __m512i X33= _mm512_load_si512(tbl[0][33]); + __m512i X34= _mm512_load_si512(tbl[0][34]); + __m512i X35= _mm512_load_si512(tbl[0][35]); + __m512i X36= _mm512_load_si512(tbl[0][36]); + __m512i X37= _mm512_load_si512(tbl[0][37]); + __m512i X38= _mm512_load_si512(tbl[0][38]); + __m512i X39= _mm512_load_si512(tbl[0][39]); + + __m512i idx_target = _mm512_load_si512(idx_mb8); + + int n; + // Find out what we actually need or just keep original + for(n=1; n<(1<=0; exp_bit_no-=EXP_WIN_SIZE) { + /* series of squaring */ + #if EXP_WIN_SIZE==5 + SQUARE_5x52x40_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + #else + SQUARE_52x40_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x40_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x40_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x40_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + #endif + + /* extract pre-computed multiplier from the table */ + { + exp_chunk_no = exp_bit_no/64; + exp_chunk_shift = exp_bit_no%64; + + red_table_idx = _mm512_load_si512(expz[exp_chunk_no]); + T = _mm512_load_si512(expz[exp_chunk_no+1]); + + red_table_idx = _mm512_srl_epi64(red_table_idx, _mm_set1_epi64x(exp_chunk_shift)); + T = _mm512_sll_epi64(T, _mm_set1_epi64x(64-exp_chunk_shift)); + red_table_idx = _mm512_and_si512( _mm512_xor_si512(red_table_idx, T), table_idx_mask); + + extract_multiplier_mb8(red_X, red_table, (int64u*)(&red_table_idx)); + } + /* and multiply */ + ifma_amm52x40_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); + } + } + + /* convert result back in regular 2^52 domain */ + zero_mb8(red_X, LEN52); + _mm512_store_si512(red_X, _mm512_set1_epi64(1)); + ifma_amm52x40_mb8((int64u*)out, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); + + zero_mb8(red_Y, LEN52); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/exp/ifma_exp3k_mb.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/exp/ifma_exp3k_mb.c new file mode 100644 index 0000000..0123c4c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/exp/ifma_exp3k_mb.c @@ -0,0 +1,343 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include +#include + + +#define USE_AMS +#ifdef USE_AMS + #define SQUARE_52x60_mb8(out, Y, mod, k0) \ + AMS52x60_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); + + #define SQUARE_5x52x60_mb8(out, Y, mod, k0) \ + AMS52x60_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + AMS52x60_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x60_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x60_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x60_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); +#else + #define SQUARE_52x60_mb8(out, Y, mod, k0) \ + ifma_amm52x60_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); + #define SQUARE_5x52x60_mb8(out, Y, mod, k0) \ + ifma_amm52x60_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x60_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x60_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x60_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x60_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); +#endif + +#define BITSIZE_MODULUS (3072) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,DIGIT_SIZE)) //60 +#define LEN64 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,64)) //48 + + +typedef int64u (*arr_pint64u_x8)[LEN52][8]; // pointer to pre-computed table of base powers + +static int64u* extract_multiplier_mb8(int64u out[LEN52][8], int64u tbl[][LEN52][8], const int64u idx_mb8[8]) +{ + // Assume first element is what need + __m512i X0 = _mm512_load_si512(tbl[0][0]); + __m512i X1 = _mm512_load_si512(tbl[0][1]); + __m512i X2 = _mm512_load_si512(tbl[0][2]); + __m512i X3 = _mm512_load_si512(tbl[0][3]); + __m512i X4 = _mm512_load_si512(tbl[0][4]); + __m512i X5 = _mm512_load_si512(tbl[0][5]); + __m512i X6 = _mm512_load_si512(tbl[0][6]); + __m512i X7 = _mm512_load_si512(tbl[0][7]); + __m512i X8 = _mm512_load_si512(tbl[0][8]); + __m512i X9 = _mm512_load_si512(tbl[0][9]); + __m512i X10= _mm512_load_si512(tbl[0][10]); + __m512i X11= _mm512_load_si512(tbl[0][11]); + __m512i X12= _mm512_load_si512(tbl[0][12]); + __m512i X13= _mm512_load_si512(tbl[0][13]); + __m512i X14= _mm512_load_si512(tbl[0][14]); + __m512i X15= _mm512_load_si512(tbl[0][15]); + __m512i X16= _mm512_load_si512(tbl[0][16]); + __m512i X17= _mm512_load_si512(tbl[0][17]); + __m512i X18= _mm512_load_si512(tbl[0][18]); + __m512i X19= _mm512_load_si512(tbl[0][19]); + __m512i X20= _mm512_load_si512(tbl[0][20]); + __m512i X21= _mm512_load_si512(tbl[0][21]); + __m512i X22= _mm512_load_si512(tbl[0][22]); + __m512i X23= _mm512_load_si512(tbl[0][23]); + __m512i X24= _mm512_load_si512(tbl[0][24]); + __m512i X25= _mm512_load_si512(tbl[0][25]); + __m512i X26= _mm512_load_si512(tbl[0][26]); + __m512i X27= _mm512_load_si512(tbl[0][27]); + __m512i X28= _mm512_load_si512(tbl[0][28]); + __m512i X29= _mm512_load_si512(tbl[0][29]); + __m512i X30= _mm512_load_si512(tbl[0][30]); + __m512i X31= _mm512_load_si512(tbl[0][31]); + __m512i X32= _mm512_load_si512(tbl[0][32]); + __m512i X33= _mm512_load_si512(tbl[0][33]); + __m512i X34= _mm512_load_si512(tbl[0][34]); + __m512i X35= _mm512_load_si512(tbl[0][35]); + __m512i X36= _mm512_load_si512(tbl[0][36]); + __m512i X37= _mm512_load_si512(tbl[0][37]); + __m512i X38= _mm512_load_si512(tbl[0][38]); + __m512i X39= _mm512_load_si512(tbl[0][39]); + __m512i X40= _mm512_load_si512(tbl[0][40]); + __m512i X41= _mm512_load_si512(tbl[0][41]); + __m512i X42= _mm512_load_si512(tbl[0][42]); + __m512i X43= _mm512_load_si512(tbl[0][43]); + __m512i X44= _mm512_load_si512(tbl[0][44]); + __m512i X45= _mm512_load_si512(tbl[0][45]); + __m512i X46= _mm512_load_si512(tbl[0][46]); + __m512i X47= _mm512_load_si512(tbl[0][47]); + __m512i X48= _mm512_load_si512(tbl[0][48]); + __m512i X49= _mm512_load_si512(tbl[0][49]); + __m512i X50= _mm512_load_si512(tbl[0][50]); + __m512i X51= _mm512_load_si512(tbl[0][51]); + __m512i X52= _mm512_load_si512(tbl[0][52]); + __m512i X53= _mm512_load_si512(tbl[0][53]); + __m512i X54= _mm512_load_si512(tbl[0][54]); + __m512i X55= _mm512_load_si512(tbl[0][55]); + __m512i X56= _mm512_load_si512(tbl[0][56]); + __m512i X57= _mm512_load_si512(tbl[0][57]); + __m512i X58= _mm512_load_si512(tbl[0][58]); + __m512i X59= _mm512_load_si512(tbl[0][59]); + + __m512i idx_target = _mm512_load_si512(idx_mb8); + + int n; + // Find out what we actually need or just keep original + for(n=1; n<(1<=0; exp_bit_no-=EXP_WIN_SIZE) { + /* series of squaring */ + #if EXP_WIN_SIZE==5 + SQUARE_5x52x60_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + #else + SQUARE_52x60_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x60_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x60_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x60_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + #endif + + /* extract pre-computed multiplier from the table */ + { + exp_chunk_no = exp_bit_no/64; + exp_chunk_shift = exp_bit_no%64; + + red_table_idx = _mm512_load_si512(expz[exp_chunk_no]); + T = _mm512_load_si512(expz[exp_chunk_no+1]); + + red_table_idx = _mm512_srl_epi64(red_table_idx, _mm_set1_epi64x(exp_chunk_shift)); + T = _mm512_sll_epi64(T, _mm_set1_epi64x(64-exp_chunk_shift)); + red_table_idx = _mm512_and_si512( _mm512_xor_si512(red_table_idx, T), table_idx_mask); + + extract_multiplier_mb8(red_X, red_table, (int64u*)(&red_table_idx)); + } + /* and multiply */ + ifma_amm52x60_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); + } + } + + /* convert result back in regular 2^52 domain */ + zero_mb8(red_X, LEN52); + _mm512_store_si512(red_X, _mm512_set1_epi64(1)); + ifma_amm52x60_mb8((int64u*)out, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); + + zero_mb8(red_Y, LEN52); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/exp/ifma_exp4k_mb.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/exp/ifma_exp4k_mb.c new file mode 100644 index 0000000..dd17af6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/exp/ifma_exp4k_mb.c @@ -0,0 +1,399 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include +#include + + +#define USE_AMS +#ifdef USE_AMS + #define SQUARE_52x79_mb8(out, Y, mod, k0) \ + AMS52x79_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); + + #define SQUARE_5x52x79_mb8(out, Y, mod, k0) \ + AMS52x79_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + AMS52x79_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x79_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x79_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x79_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); +#else + #define SQUARE_52x79_mb8(out, Y, mod, k0) \ + ifma_amm52x79_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); + #define SQUARE_5x52x79_mb8(out, Y, mod, k0) \ + ifma_amm52x79_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x79_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x79_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x79_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x79_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); +#endif + +#define BITSIZE_MODULUS (4096) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,DIGIT_SIZE)) //79 +#define LEN64 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,64)) //64 + + +typedef int64u (*arr_pint64u_x8)[LEN52][8]; // pointer to pre-computed table of base powers + +static int64u* extract_multiplier_mb8(int64u out[LEN52][8], int64u tbl[][LEN52][8], const int64u idx_mb8[8]) +{ + // Assume first element is what need + __m512i X0 = _mm512_load_si512(tbl[0][0]); + __m512i X1 = _mm512_load_si512(tbl[0][1]); + __m512i X2 = _mm512_load_si512(tbl[0][2]); + __m512i X3 = _mm512_load_si512(tbl[0][3]); + __m512i X4 = _mm512_load_si512(tbl[0][4]); + __m512i X5 = _mm512_load_si512(tbl[0][5]); + __m512i X6 = _mm512_load_si512(tbl[0][6]); + __m512i X7 = _mm512_load_si512(tbl[0][7]); + __m512i X8 = _mm512_load_si512(tbl[0][8]); + __m512i X9 = _mm512_load_si512(tbl[0][9]); + __m512i X10= _mm512_load_si512(tbl[0][10]); + __m512i X11= _mm512_load_si512(tbl[0][11]); + __m512i X12= _mm512_load_si512(tbl[0][12]); + __m512i X13= _mm512_load_si512(tbl[0][13]); + __m512i X14= _mm512_load_si512(tbl[0][14]); + __m512i X15= _mm512_load_si512(tbl[0][15]); + __m512i X16= _mm512_load_si512(tbl[0][16]); + __m512i X17= _mm512_load_si512(tbl[0][17]); + __m512i X18= _mm512_load_si512(tbl[0][18]); + __m512i X19= _mm512_load_si512(tbl[0][19]); + __m512i X20= _mm512_load_si512(tbl[0][20]); + __m512i X21= _mm512_load_si512(tbl[0][21]); + __m512i X22= _mm512_load_si512(tbl[0][22]); + __m512i X23= _mm512_load_si512(tbl[0][23]); + __m512i X24= _mm512_load_si512(tbl[0][24]); + __m512i X25= _mm512_load_si512(tbl[0][25]); + __m512i X26= _mm512_load_si512(tbl[0][26]); + __m512i X27= _mm512_load_si512(tbl[0][27]); + __m512i X28= _mm512_load_si512(tbl[0][28]); + __m512i X29= _mm512_load_si512(tbl[0][29]); + __m512i X30= _mm512_load_si512(tbl[0][30]); + __m512i X31= _mm512_load_si512(tbl[0][31]); + __m512i X32= _mm512_load_si512(tbl[0][32]); + __m512i X33= _mm512_load_si512(tbl[0][33]); + __m512i X34= _mm512_load_si512(tbl[0][34]); + __m512i X35= _mm512_load_si512(tbl[0][35]); + __m512i X36= _mm512_load_si512(tbl[0][36]); + __m512i X37= _mm512_load_si512(tbl[0][37]); + __m512i X38= _mm512_load_si512(tbl[0][38]); + __m512i X39= _mm512_load_si512(tbl[0][39]); + __m512i X40= _mm512_load_si512(tbl[0][40]); + __m512i X41= _mm512_load_si512(tbl[0][41]); + __m512i X42= _mm512_load_si512(tbl[0][42]); + __m512i X43= _mm512_load_si512(tbl[0][43]); + __m512i X44= _mm512_load_si512(tbl[0][44]); + __m512i X45= _mm512_load_si512(tbl[0][45]); + __m512i X46= _mm512_load_si512(tbl[0][46]); + __m512i X47= _mm512_load_si512(tbl[0][47]); + __m512i X48= _mm512_load_si512(tbl[0][48]); + __m512i X49= _mm512_load_si512(tbl[0][49]); + __m512i X50= _mm512_load_si512(tbl[0][50]); + __m512i X51= _mm512_load_si512(tbl[0][51]); + __m512i X52= _mm512_load_si512(tbl[0][52]); + __m512i X53= _mm512_load_si512(tbl[0][53]); + __m512i X54= _mm512_load_si512(tbl[0][54]); + __m512i X55= _mm512_load_si512(tbl[0][55]); + __m512i X56= _mm512_load_si512(tbl[0][56]); + __m512i X57= _mm512_load_si512(tbl[0][57]); + __m512i X58= _mm512_load_si512(tbl[0][58]); + __m512i X59= _mm512_load_si512(tbl[0][59]); + __m512i X60= _mm512_load_si512(tbl[0][60]); + __m512i X61= _mm512_load_si512(tbl[0][61]); + __m512i X62= _mm512_load_si512(tbl[0][62]); + __m512i X63= _mm512_load_si512(tbl[0][63]); + __m512i X64= _mm512_load_si512(tbl[0][64]); + __m512i X65= _mm512_load_si512(tbl[0][65]); + __m512i X66= _mm512_load_si512(tbl[0][66]); + __m512i X67= _mm512_load_si512(tbl[0][67]); + __m512i X68= _mm512_load_si512(tbl[0][68]); + __m512i X69= _mm512_load_si512(tbl[0][69]); + __m512i X70= _mm512_load_si512(tbl[0][70]); + __m512i X71= _mm512_load_si512(tbl[0][71]); + __m512i X72= _mm512_load_si512(tbl[0][72]); + __m512i X73= _mm512_load_si512(tbl[0][73]); + __m512i X74= _mm512_load_si512(tbl[0][74]); + __m512i X75= _mm512_load_si512(tbl[0][75]); + __m512i X76= _mm512_load_si512(tbl[0][76]); + __m512i X77= _mm512_load_si512(tbl[0][77]); + __m512i X78= _mm512_load_si512(tbl[0][78]); + + __m512i idx_target = _mm512_load_si512(idx_mb8); + + int n; + // Find out what we actually need or just keep original + for(n=1; n<(1<=0; exp_bit_no-=EXP_WIN_SIZE) { + /* series of squaring */ + #if EXP_WIN_SIZE==5 + SQUARE_5x52x79_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + #else + SQUARE_52x79_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x79_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x79_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x79_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + #endif + + /* extract pre-computed multiplier from the table */ + { + exp_chunk_no = exp_bit_no/64; + exp_chunk_shift = exp_bit_no%64; + + red_table_idx = _mm512_load_si512(expz[exp_chunk_no]); + T = _mm512_load_si512(expz[exp_chunk_no+1]); + + red_table_idx = _mm512_srl_epi64(red_table_idx, _mm_set1_epi64x(exp_chunk_shift)); + T = _mm512_sll_epi64(T, _mm_set1_epi64x(64-exp_chunk_shift)); + red_table_idx = _mm512_and_si512( _mm512_xor_si512(red_table_idx, T), table_idx_mask); + + extract_multiplier_mb8(red_X, red_table, (int64u*)(&red_table_idx)); + } + /* and multiply */ + ifma_amm52x79_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); + } + } + + /* convert result back in regular 2^52 domain */ + zero_mb8(red_X, LEN52); + _mm512_store_si512(red_X, _mm512_set1_epi64(1)); + ifma_amm52x79_mb8((int64u*)out, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); + + zero_mb8(red_Y, LEN52); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/exp/ifma_exp_mb.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/exp/ifma_exp_mb.c new file mode 100644 index 0000000..044eac5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/exp/ifma_exp_mb.c @@ -0,0 +1,251 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + + +#include +#include + +#include +#include +#include +#include +#include + + +mbx_status ifma_exp_mb(int64u* const out_pa[8], + const int64u* const base_pa[8], + const int64u* const exp_pa[8], int exp_bits, + const int64u* const mod_pa[8], int mod_bits, + exp_mb8 expfunc, + int8u* pBuffer, int bufferLen) +{ + mbx_status status = MBX_STATUS_OK; + + /* test input pointers */ + if (NULL==out_pa || NULL==base_pa || NULL==exp_pa || NULL==mod_pa || NULL==expfunc || NULL== pBuffer) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* test exp length */ + if (exp_bits > mod_bits) { + status = MBX_SET_STS_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* test length of buffer */ + int bufferMinLen = mbx_exp_BufferSize(mod_bits); + if (bufferLen < bufferMinLen) { + status = MBX_SET_STS_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* check pointers and values */ + int buf_no; + for (buf_no = 0; buf_no < 8; buf_no++) { + int64u* out = out_pa[buf_no]; + const int64u* base = base_pa[buf_no]; + const int64u* exp = exp_pa[buf_no]; + const int64u* mod = mod_pa[buf_no]; + + /* if any of pointer NULL set error status */ + if (NULL == out || NULL == base || NULL == exp || NULL == mod) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + /* + // processing + */ + if (MBX_IS_ANY_OK_STS(status)) { + int len52 = NUMBER_OF_DIGITS(mod_bits, DIGIT_SIZE); + int len64 = NUMBER_OF_DIGITS(mod_bits, 64); + + /* 64-byte aligned buffer of int64[8] */ + pint64u_x8 pBuffer_x8 = (pint64u_x8)IFMA_ALIGNED_PTR(pBuffer, 64); + + /* allocate mb8 buffers */ + pint64u_x8 k0_mb8 = pBuffer_x8; + pint64u_x8 expz_mb8 = k0_mb8 + 1; + pint64u_x8 rr_mb8 = expz_mb8 + (len64+1); + pint64u_x8 inout_mb8 = rr_mb8 + len52; + pint64u_x8 mod_mb8 = inout_mb8 + len52; + /* MULTIPLE_OF_10 because of AMS5x52x79_diagonal_mb8() implementaion specific */ + pint64u_x8 work_buffer = mod_mb8 + MULTIPLE_OF(len52, 10); + + /* convert modulus to ifma fmt */ + zero_mb8(mod_mb8, MULTIPLE_OF(len52, 10)); + ifma_BNU_to_mb8(mod_mb8, mod_pa, mod_bits); + + /* compute k0[] */ + ifma_montFactor52_mb8(k0_mb8[0], mod_mb8[0]); + + /* compute to_Montgomery domain converters */ + ifma_montRR52x_mb8(rr_mb8, mod_mb8, mod_bits); + + /* convert input to ifma fmt */ + ifma_BNU_to_mb8(inout_mb8, base_pa, mod_bits); + + /* re-arrange exps to ifma */ + zero_mb8(expz_mb8, len64+1); + ifma_BNU_transpose_copy(expz_mb8, exp_pa, exp_bits); + + /* exponentiation */ + expfunc(inout_mb8, + (const int64u(*)[8])inout_mb8, + (const int64u(*)[8])expz_mb8, exp_bits, + (const int64u(*)[8])mod_mb8, + (const int64u(*)[8])rr_mb8, + k0_mb8[0], + (int64u(*)[8])work_buffer); + + /* convert result from ifma fmt */ + ifma_mb8_to_BNU(out_pa, (const int64u(*)[8])inout_mb8, mod_bits); + + /* clear exponents */ + zero_mb8(expz_mb8, len64); + } + + return status; +} + +DLL_PUBLIC +mbx_status mbx_exp1024_mb8(int64u* const out_pa[8], + const int64u* const base_pa[8], + const int64u* const exp_pa[8], int exp_bits, + const int64u* const mod_pa[8], int mod_bits, + int8u* pBuffer, int bufferLen) +{ + mbx_status status = MBX_STATUS_OK; + + /* test exp modulus range */ + if(EXP_MODULUS_1024 != bits_range(mod_bits)) { + status = MBX_SET_STS_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* 1k exponentiation */ + return ifma_exp_mb(out_pa, base_pa, + exp_pa, exp_bits, + mod_pa, mod_bits, + ifma_modexp1024_mb, + pBuffer, bufferLen); +} + +DLL_PUBLIC +mbx_status mbx_exp2048_mb8(int64u* const out_pa[8], + const int64u* const base_pa[8], + const int64u* const exp_pa[8], int exp_bits, + const int64u* const mod_pa[8], int mod_bits, + int8u* pBuffer, int bufferLen) +{ + mbx_status status = MBX_STATUS_OK; + + /* test exp modulus range */ + if(EXP_MODULUS_2048 != bits_range(mod_bits)) { + status = MBX_SET_STS_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* 2k exponentiation */ + return ifma_exp_mb(out_pa, base_pa, + exp_pa, exp_bits, + mod_pa, mod_bits, + ifma_modexp2048_mb, + pBuffer, bufferLen); +} + +DLL_PUBLIC +mbx_status mbx_exp3072_mb8(int64u* const out_pa[8], + const int64u* const base_pa[8], + const int64u* const exp_pa[8], int exp_bits, + const int64u* const mod_pa[8], int mod_bits, + int8u* pBuffer, int bufferLen) +{ + mbx_status status = MBX_STATUS_OK; + + /* test exp modulus range */ + if(EXP_MODULUS_3072 != bits_range(mod_bits)) { + status = MBX_SET_STS_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* 3k exponentiation */ + return ifma_exp_mb(out_pa, base_pa, + exp_pa, exp_bits, + mod_pa, mod_bits, + ifma_modexp3072_mb, + pBuffer, bufferLen); +} + +DLL_PUBLIC +mbx_status mbx_exp4096_mb8(int64u* const out_pa[8], + const int64u* const base_pa[8], + const int64u* const exp_pa[8], int exp_bits, + const int64u* const mod_pa[8], int mod_bits, + int8u* pBuffer, int bufferLen) +{ + mbx_status status = MBX_STATUS_OK; + + /* test exp modulus range */ + if(EXP_MODULUS_4096 != bits_range(mod_bits)) { + status = MBX_SET_STS_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* 4k exponentiation */ + return ifma_exp_mb(out_pa, base_pa, + exp_pa, exp_bits, + mod_pa, mod_bits, + ifma_modexp4096_mb, + pBuffer, bufferLen); +} + +DLL_PUBLIC +mbx_status mbx_exp_mb8(int64u* const out_pa[8], + const int64u* const base_pa[8], + const int64u* const exp_pa[8], int exp_bits, + const int64u* const mod_pa[8], int mod_bits, + int8u* pBuffer, int bufferLen) +{ + mbx_status status = MBX_STATUS_OK; + + /* test exp modulus range */ + int modulus_range = bits_range(mod_bits); + if(0 == modulus_range) { + status = MBX_SET_STS_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + // + // processing + // + exp_mb8 expfunc = NULL; + switch (modulus_range) { + case EXP_MODULUS_1024: expfunc = ifma_modexp1024_mb; break; + case EXP_MODULUS_2048: expfunc = ifma_modexp2048_mb; break; + case EXP_MODULUS_3072: expfunc = ifma_modexp3072_mb; break; + case EXP_MODULUS_4096: expfunc = ifma_modexp4096_mb; break; + default: break; + } + + return ifma_exp_mb(out_pa, base_pa, + exp_pa, exp_bits, + mod_pa, mod_bits, + expfunc, + pBuffer, bufferLen); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/exp/ifma_exp_method.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/exp/ifma_exp_method.c new file mode 100644 index 0000000..f2cd6cc --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/exp/ifma_exp_method.c @@ -0,0 +1,69 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include +#include +#include + +/* map exp modulus bit size to exp modulus range */ +int bits_range(int modulusBits) +{ + int modulusLen = (NUMBER_OF_DIGITS(modulusBits, DIGIT_SIZE)); + int modulusLen_top = (NUMBER_OF_DIGITS(modulusBits+2, DIGIT_SIZE)); + + if(modulusLen != modulusLen_top) + return EXP_MODULUS_UNSUPPORT; + + switch (modulusLen) { + case NUMBER_OF_DIGITS(EXP_MODULUS_1024, DIGIT_SIZE): return EXP_MODULUS_1024; + case NUMBER_OF_DIGITS(EXP_MODULUS_2048, DIGIT_SIZE): return EXP_MODULUS_2048; + case NUMBER_OF_DIGITS(EXP_MODULUS_3072, DIGIT_SIZE): return EXP_MODULUS_3072; + case NUMBER_OF_DIGITS(EXP_MODULUS_4096, DIGIT_SIZE): return EXP_MODULUS_4096; + default: return EXP_MODULUS_UNSUPPORT; + } +} + +/* size of scratch bufer */ +DLL_PUBLIC +int mbx_exp_BufferSize(int modulusBits) +{ + int modulusRange = bits_range(modulusBits); + + if(modulusRange) { + int len52 = NUMBER_OF_DIGITS(modulusRange, DIGIT_SIZE); + int len64 = NUMBER_OF_DIGITS(modulusRange, 64); + int bufferSize = (8 /* alignment*/ + + /* ifma_mont_exp_mb */ + +1*8 /* k0 */ + +(len64+1+1)*8 /* expz */ + +len52*8 /* rr */ + +len52*8 /* inp_out */ + + MULTIPLE_OF(len52, 10)*8 /* modulus */ + + /* ifma_exp1/2/3/4k_mb */ + +len52*8 /* Y */ + +len52*8 /* X */ + +(1 << EXP_WIN_SIZE) * len52*8 /* pre-computed table*/ + + ) * sizeof(int64u); + return bufferSize; + } + else + return 0; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/AMS4x52x20_diagonal_stitched_with_extract_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/AMS4x52x20_diagonal_stitched_with_extract_mb8.c new file mode 100644 index 0000000..58aad0c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/AMS4x52x20_diagonal_stitched_with_extract_mb8.c @@ -0,0 +1,2049 @@ +/******************************************************************************* + * Copyright (C) 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include + +#ifdef __GNUC__ + #define ASM(a) __asm__(a); +#else + #define ASM(a) +#endif + +/* +Two independant functions are stitched: +- 4 squarings +- Extracting values from the precomputed tables MulTbl and MulTblx + + +For squaring, an optimized approach is utilized. As an example, suppose we are multiplying two 4-digit numbers: + | a3 | a2 | a1 | a0 | + | b3 | b2 | b1 | b0 | + X______________________ + + | a3 * b3 | a2 * b2 | a1 * b1 | a0 * b0 | + | a3 * b2 | a2 * b1 | a1 * b0 | + | a2 * b3 | a1 * b2 | a0 * b1 | + | a3 * b1 | a2 * b0 | + | a1 * b3 | a0 * b2 | + | a3 * b0 | + | a0 * b3 | + +This operation is realized with 4x4=16 digit-wise multiplications. When a=b (for squaring), multiplication operation is optimizes as follows: + | a3 | a2 | a1 | a0 | + | a3 | a2 | a1 | a0 | + X______________________ + + | a3 * a3 | a2 * a2 | a1 * a1 | a0 * a0 | + 2* | a3 * a2 | a2 * a1 | a1 * a0 | + + 2* | a3 * a1 | a2 * a0 | + + 2* | a3 * a0 | + +This operation is realized with 10 digit-wise multiplications. For an n-digit squaring operation, (n^2-n)/2 less digit-wise multiplications are +required. The number of digit-wise multiplications for n-digit squaring can be calculated with the following equation: + + n^2 - (n^2-n)/2 + +multiplication by 2 operations are realized with add64 instructions. +*/ +void AMS4x52x20_diagonal_stitched_with_extract_mb8( + int64u* out_mb, U64* mulb, U64* mulbx, const int64u* inpA_mb, + const int64u* inpM_mb, const int64u* k0_mb, int64u MulTbl[][redLen][ 8], + int64u MulTblx[][redLen][ 8], const int64u Idx[ 8]) { + + U64 res00, res01, res02, res03, res04, res05, res06, res07, res08, res09, + res10, res11, res12, res13, res14, res15, res16, res17, res18, res19, + res20, res21, res22, res23, res24, res25, res26, res27, res28, res29, + res30, res31, res32, res33, res34, res35, res36, res37, res38, res39; + + const U64 mont_constant = loadu64((U64*) k0_mb); + + U64* a = (U64*) inpA_mb; + U64* m = (U64*) inpM_mb; + U64* r = (U64*) out_mb; + + U64 idx_target = loadu64((U64*) Idx); + __mmask8 extract_sel_mask = cmpeq64_mask(set64(0), idx_target); + + for (int iter = 0; iter < 4; ++iter) { + res00 = res01 = res02 = res03 = res04 = res05 = res06 = res07 = res08 = res09 = + res10 = res11 = res12 = res13 = res14 = res15 = res16 = res17 = res18 = res19 = + res20 = res21 = res22 = res23 = res24 = res25 = res26 = res27 = res28 = res29 = + res30 = res31 = res32 = res33 = res34 = res35 = res36 = res37 = res38 = res39 = get_zero64(); + + // Calculate full square + // stitched with extraction code + + //*******BEGIN SQUARING CODE SEGMENT************// + res01 = fma52lo(res01, a[ 0], a[ 1]); // Sum(1) + res02 = fma52hi(res02, a[ 0], a[ 1]); // Sum(1) + res02 = fma52lo(res02, a[ 0], a[ 2]); // Sum(2) + res03 = fma52hi(res03, a[ 0], a[ 2]); // Sum(2) + res03 = fma52lo(res03, a[ 1], a[ 2]); // Sum(3) + res04 = fma52hi(res04, a[ 1], a[ 2]); // Sum(3) + res03 = fma52lo(res03, a[ 0], a[ 3]); // Sum(3) + res04 = fma52hi(res04, a[ 0], a[ 3]); // Sum(3) + res04 = fma52lo(res04, a[ 1], a[ 3]); // Sum(4) + res05 = fma52hi(res05, a[ 1], a[ 3]); // Sum(4) + res05 = fma52lo(res05, a[ 2], a[ 3]); // Sum(5) + res06 = fma52hi(res06, a[ 2], a[ 3]); // Sum(5) + res04 = fma52lo(res04, a[ 0], a[ 4]); // Sum(4) + res05 = fma52hi(res05, a[ 0], a[ 4]); // Sum(4) + res05 = fma52lo(res05, a[ 1], a[ 4]); // Sum(5) + res06 = fma52hi(res06, a[ 1], a[ 4]); // Sum(5) + res06 = fma52lo(res06, a[ 2], a[ 4]); // Sum(6) + res07 = fma52hi(res07, a[ 2], a[ 4]); // Sum(6) + res07 = fma52lo(res07, a[ 3], a[ 4]); // Sum(7) + res08 = fma52hi(res08, a[ 3], a[ 4]); // Sum(7) + res05 = fma52lo(res05, a[ 0], a[ 5]); // Sum(5) + res06 = fma52hi(res06, a[ 0], a[ 5]); // Sum(5) + res06 = fma52lo(res06, a[ 1], a[ 5]); // Sum(6) + res07 = fma52hi(res07, a[ 1], a[ 5]); // Sum(6) + res07 = fma52lo(res07, a[ 2], a[ 5]); // Sum(7) + res08 = fma52hi(res08, a[ 2], a[ 5]); // Sum(7) + res08 = fma52lo(res08, a[ 3], a[ 5]); // Sum(8) + res09 = fma52hi(res09, a[ 3], a[ 5]); // Sum(8) + res09 = fma52lo(res09, a[ 4], a[ 5]); // Sum(9) + res10 = fma52hi(res10, a[ 4], a[ 5]); // Sum(9) + res06 = fma52lo(res06, a[ 0], a[ 6]); // Sum(6) + res07 = fma52hi(res07, a[ 0], a[ 6]); // Sum(6) + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + U64 idx_curr = set64(1+4*iter); + __mmask8 extract_sel_mask_next = cmpeq64_mask(idx_curr, idx_target); + U64 temp = loadstream64(MulTbl[0+4*iter][ 0]); + mulb[ 0] = _mm512_mask_mov_epi64(mulb[ 0], extract_sel_mask, temp); + temp = loadstream64(MulTbl[0+4*iter][ 1]); + mulb[ 1] = _mm512_mask_mov_epi64(mulb[ 1], extract_sel_mask, temp); + temp = loadstream64(MulTbl[0+4*iter][ 2]); + mulb[ 2] = _mm512_mask_mov_epi64(mulb[ 2], extract_sel_mask, temp); + temp = loadstream64(MulTbl[0+4*iter][ 3]); + mulb[ 3] = _mm512_mask_mov_epi64(mulb[ 3], extract_sel_mask, temp); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res07 = fma52lo(res07, a[ 1], a[ 6]); // Sum(7) + res08 = fma52hi(res08, a[ 1], a[ 6]); // Sum(7) + res08 = fma52lo(res08, a[ 2], a[ 6]); // Sum(8) + res09 = fma52hi(res09, a[ 2], a[ 6]); // Sum(8) + res09 = fma52lo(res09, a[ 3], a[ 6]); // Sum(9) + res10 = fma52hi(res10, a[ 3], a[ 6]); // Sum(9) + res10 = fma52lo(res10, a[ 4], a[ 6]); // Sum(10) + res11 = fma52hi(res11, a[ 4], a[ 6]); // Sum(10) + res11 = fma52lo(res11, a[ 5], a[ 6]); // Sum(11) + res12 = fma52hi(res12, a[ 5], a[ 6]); // Sum(11) + res07 = fma52lo(res07, a[ 0], a[ 7]); // Sum(7) + res08 = fma52hi(res08, a[ 0], a[ 7]); // Sum(7) + res08 = fma52lo(res08, a[ 1], a[ 7]); // Sum(8) + res09 = fma52hi(res09, a[ 1], a[ 7]); // Sum(8) + res09 = fma52lo(res09, a[ 2], a[ 7]); // Sum(9) + res10 = fma52hi(res10, a[ 2], a[ 7]); // Sum(9) + res10 = fma52lo(res10, a[ 3], a[ 7]); // Sum(10) + res11 = fma52hi(res11, a[ 3], a[ 7]); // Sum(10) + res11 = fma52lo(res11, a[ 4], a[ 7]); // Sum(11) + res12 = fma52hi(res12, a[ 4], a[ 7]); // Sum(11) + res08 = fma52lo(res08, a[ 0], a[ 8]); // Sum(8) + res09 = fma52hi(res09, a[ 0], a[ 8]); // Sum(8) + res09 = fma52lo(res09, a[ 1], a[ 8]); // Sum(9) + res10 = fma52hi(res10, a[ 1], a[ 8]); // Sum(9) + res10 = fma52lo(res10, a[ 2], a[ 8]); // Sum(10) + res11 = fma52hi(res11, a[ 2], a[ 8]); // Sum(10) + res11 = fma52lo(res11, a[ 3], a[ 8]); // Sum(11) + res12 = fma52hi(res12, a[ 3], a[ 8]); // Sum(11) + res09 = fma52lo(res09, a[ 0], a[ 9]); // Sum(9) + res10 = fma52hi(res10, a[ 0], a[ 9]); // Sum(9) + res10 = fma52lo(res10, a[ 1], a[ 9]); // Sum(10) + res11 = fma52hi(res11, a[ 1], a[ 9]); // Sum(10 + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + temp = loadstream64(MulTbl[0+4*iter][ 4]); + mulb[ 4] = _mm512_mask_mov_epi64(mulb[ 4], extract_sel_mask, temp); + temp = loadstream64(MulTbl[0+4*iter][ 5]); + mulb[ 5] = _mm512_mask_mov_epi64(mulb[ 5], extract_sel_mask, temp); + temp = loadstream64(MulTbl[0+4*iter][ 6]); + mulb[ 6] = _mm512_mask_mov_epi64(mulb[ 6], extract_sel_mask, temp); + temp = loadstream64(MulTbl[0+4*iter][ 7]); + mulb[ 7] = _mm512_mask_mov_epi64(mulb[ 7], extract_sel_mask, temp); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res11 = fma52lo(res11, a[ 2], a[ 9]); // Sum(11) + res12 = fma52hi(res12, a[ 2], a[ 9]); // Sum(11) + res10 = fma52lo(res10, a[ 0], a[10]); // Sum(10) + res11 = fma52hi(res11, a[ 0], a[10]); // Sum(10) + res11 = fma52lo(res11, a[ 1], a[10]); // Sum(11) + res12 = fma52hi(res12, a[ 1], a[10]); // Sum(11) + res11 = fma52lo(res11, a[ 0], a[11]); // Sum(11) + res12 = fma52hi(res12, a[ 0], a[11]); // Sum(11) + res01 = add64(res01, res01); // Double(1) + res02 = add64(res02, res02); // Double(2) + res03 = add64(res03, res03); // Double(3) + res04 = add64(res04, res04); // Double(4) + res05 = add64(res05, res05); // Double(5) + res06 = add64(res06, res06); // Double(6) + res07 = add64(res07, res07); // Double(7) + res08 = add64(res08, res08); // Double(8) + res09 = add64(res09, res09); // Double(9) + res10 = add64(res10, res10); // Double(10) + res11 = add64(res11, res11); // Double(11) + res00 = fma52lo(res00, a[ 0], a[ 0]); // Add sqr(0) + res01 = fma52hi(res01, a[ 0], a[ 0]); // Add sqr(0) + res02 = fma52lo(res02, a[ 1], a[ 1]); // Add sqr(2) + res03 = fma52hi(res03, a[ 1], a[ 1]); // Add sqr(2) + res04 = fma52lo(res04, a[ 2], a[ 2]); // Add sqr(4) + res05 = fma52hi(res05, a[ 2], a[ 2]); // Add sqr(4) + res06 = fma52lo(res06, a[ 3], a[ 3]); // Add sqr(6) + res07 = fma52hi(res07, a[ 3], a[ 3]); // Add sqr(6) + res08 = fma52lo(res08, a[ 4], a[ 4]); // Add sqr(8) + res09 = fma52hi(res09, a[ 4], a[ 4]); // Add sqr(8) + res10 = fma52lo(res10, a[ 5], a[ 5]); // Add sqr(10) + res11 = fma52hi(res11, a[ 5], a[ 5]); // Add sqr(10) + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + temp = loadstream64(MulTbl[0+4*iter][ 8]); + mulb[ 8] = _mm512_mask_mov_epi64(mulb[ 8], extract_sel_mask, temp); + temp = loadstream64(MulTbl[0+4*iter][ 9]); + mulb[ 9] = _mm512_mask_mov_epi64(mulb[ 9], extract_sel_mask, temp); + temp = loadstream64(MulTbl[0+4*iter][10]); + mulb[10] = _mm512_mask_mov_epi64(mulb[10], extract_sel_mask, temp); + temp = loadstream64(MulTbl[0+4*iter][11]); + mulb[11] = _mm512_mask_mov_epi64(mulb[11], extract_sel_mask, temp); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res12 = fma52lo(res12, a[ 5], a[ 7]); // Sum(12) + res13 = fma52hi(res13, a[ 5], a[ 7]); // Sum(12) + res13 = fma52lo(res13, a[ 6], a[ 7]); // Sum(13) + res14 = fma52hi(res14, a[ 6], a[ 7]); // Sum(13) + res12 = fma52lo(res12, a[ 4], a[ 8]); // Sum(12) + res13 = fma52hi(res13, a[ 4], a[ 8]); // Sum(12) + res13 = fma52lo(res13, a[ 5], a[ 8]); // Sum(13) + res14 = fma52hi(res14, a[ 5], a[ 8]); // Sum(13) + res14 = fma52lo(res14, a[ 6], a[ 8]); // Sum(14) + res15 = fma52hi(res15, a[ 6], a[ 8]); // Sum(14) + res15 = fma52lo(res15, a[ 7], a[ 8]); // Sum(15) + res16 = fma52hi(res16, a[ 7], a[ 8]); // Sum(15) + res12 = fma52lo(res12, a[ 3], a[ 9]); // Sum(12) + res13 = fma52hi(res13, a[ 3], a[ 9]); // Sum(12) + res13 = fma52lo(res13, a[ 4], a[ 9]); // Sum(13) + res14 = fma52hi(res14, a[ 4], a[ 9]); // Sum(13) + res14 = fma52lo(res14, a[ 5], a[ 9]); // Sum(14) + res15 = fma52hi(res15, a[ 5], a[ 9]); // Sum(14) + res15 = fma52lo(res15, a[ 6], a[ 9]); // Sum(15) + res16 = fma52hi(res16, a[ 6], a[ 9]); // Sum(15) + res16 = fma52lo(res16, a[ 7], a[ 9]); // Sum(16) + res17 = fma52hi(res17, a[ 7], a[ 9]); // Sum(16) + res17 = fma52lo(res17, a[ 8], a[ 9]); // Sum(17) + res18 = fma52hi(res18, a[ 8], a[ 9]); // Sum(17) + res12 = fma52lo(res12, a[ 2], a[10]); // Sum(12) + res13 = fma52hi(res13, a[ 2], a[10]); // Sum(12) + res13 = fma52lo(res13, a[ 3], a[10]); // Sum(13) + res14 = fma52hi(res14, a[ 3], a[10]); // Sum(13) + res14 = fma52lo(res14, a[ 4], a[10]); // Sum(14) + res15 = fma52hi(res15, a[ 4], a[10]); // Sum(14) + res15 = fma52lo(res15, a[ 5], a[10]); // Sum(15) + res16 = fma52hi(res16, a[ 5], a[10]); // Sum(15) + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + temp = loadstream64(MulTbl[0+4*iter][12]); + mulb[12] = _mm512_mask_mov_epi64(mulb[12], extract_sel_mask, temp); + temp = loadstream64(MulTbl[0+4*iter][13]); + mulb[13] = _mm512_mask_mov_epi64(mulb[13], extract_sel_mask, temp); + temp = loadstream64(MulTbl[0+4*iter][14]); + mulb[14] = _mm512_mask_mov_epi64(mulb[14], extract_sel_mask, temp); + temp = loadstream64(MulTbl[0+4*iter][15]); + mulb[15] = _mm512_mask_mov_epi64(mulb[15], extract_sel_mask, temp); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res16 = fma52lo(res16, a[ 6], a[10]); // Sum(16) + res17 = fma52hi(res17, a[ 6], a[10]); // Sum(16) + res17 = fma52lo(res17, a[ 7], a[10]); // Sum(17) + res18 = fma52hi(res18, a[ 7], a[10]); // Sum(17) + res18 = fma52lo(res18, a[ 8], a[10]); // Sum(18) + res19 = fma52hi(res19, a[ 8], a[10]); // Sum(18) + res19 = fma52lo(res19, a[ 9], a[10]); // Sum(19) + res20 = fma52hi(res20, a[ 9], a[10]); // Sum(19) + res12 = fma52lo(res12, a[ 1], a[11]); // Sum(12) + res13 = fma52hi(res13, a[ 1], a[11]); // Sum(12) + res13 = fma52lo(res13, a[ 2], a[11]); // Sum(13) + res14 = fma52hi(res14, a[ 2], a[11]); // Sum(13) + res14 = fma52lo(res14, a[ 3], a[11]); // Sum(14) + res15 = fma52hi(res15, a[ 3], a[11]); // Sum(14) + res15 = fma52lo(res15, a[ 4], a[11]); // Sum(15) + res16 = fma52hi(res16, a[ 4], a[11]); // Sum(15) + res16 = fma52lo(res16, a[ 5], a[11]); // Sum(16) + res17 = fma52hi(res17, a[ 5], a[11]); // Sum(16) + res17 = fma52lo(res17, a[ 6], a[11]); // Sum(17) + res18 = fma52hi(res18, a[ 6], a[11]); // Sum(17) + res18 = fma52lo(res18, a[ 7], a[11]); // Sum(18) + res19 = fma52hi(res19, a[ 7], a[11]); // Sum(18) + res19 = fma52lo(res19, a[ 8], a[11]); // Sum(19) + res20 = fma52hi(res20, a[ 8], a[11]); // Sum(19) + res20 = fma52lo(res20, a[ 9], a[11]); // Sum(20) + res21 = fma52hi(res21, a[ 9], a[11]); // Sum(20) + res21 = fma52lo(res21, a[10], a[11]); // Sum(21) + res22 = fma52hi(res22, a[10], a[11]); // Sum(21) + res12 = fma52lo(res12, a[ 0], a[12]); // Sum(12) + res13 = fma52hi(res13, a[ 0], a[12]); // Sum(12) + res13 = fma52lo(res13, a[ 1], a[12]); // Sum(13) + res14 = fma52hi(res14, a[ 1], a[12]); // Sum(13) + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + temp = loadstream64(MulTbl[0+4*iter][16]); + mulb[16] = _mm512_mask_mov_epi64(mulb[16], extract_sel_mask, temp); + temp = loadstream64(MulTbl[0+4*iter][17]); + mulb[17] = _mm512_mask_mov_epi64(mulb[17], extract_sel_mask, temp); + temp = loadstream64(MulTbl[0+4*iter][18]); + mulb[18] = _mm512_mask_mov_epi64(mulb[18], extract_sel_mask, temp); + temp = loadstream64(MulTbl[0+4*iter][19]); + mulb[19] = _mm512_mask_mov_epi64(mulb[19], extract_sel_mask, temp); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res14 = fma52lo(res14, a[ 2], a[12]); // Sum(14) + res15 = fma52hi(res15, a[ 2], a[12]); // Sum(14) + res15 = fma52lo(res15, a[ 3], a[12]); // Sum(15) + res16 = fma52hi(res16, a[ 3], a[12]); // Sum(15) + res16 = fma52lo(res16, a[ 4], a[12]); // Sum(16) + res17 = fma52hi(res17, a[ 4], a[12]); // Sum(16) + res17 = fma52lo(res17, a[ 5], a[12]); // Sum(17) + res18 = fma52hi(res18, a[ 5], a[12]); // Sum(17) + res18 = fma52lo(res18, a[ 6], a[12]); // Sum(18) + res19 = fma52hi(res19, a[ 6], a[12]); // Sum(18) + res19 = fma52lo(res19, a[ 7], a[12]); // Sum(19) + res20 = fma52hi(res20, a[ 7], a[12]); // Sum(19) + res20 = fma52lo(res20, a[ 8], a[12]); // Sum(20) + res21 = fma52hi(res21, a[ 8], a[12]); // Sum(20) + res21 = fma52lo(res21, a[ 9], a[12]); // Sum(21) + res22 = fma52hi(res22, a[ 9], a[12]); // Sum(21) + res22 = fma52lo(res22, a[10], a[12]); // Sum(22) + res23 = fma52hi(res23, a[10], a[12]); // Sum(22) + res23 = fma52lo(res23, a[11], a[12]); // Sum(23) + res24 = fma52hi(res24, a[11], a[12]); // Sum(23) + res13 = fma52lo(res13, a[ 0], a[13]); // Sum(13) + res14 = fma52hi(res14, a[ 0], a[13]); // Sum(13) + res14 = fma52lo(res14, a[ 1], a[13]); // Sum(14) + res15 = fma52hi(res15, a[ 1], a[13]); // Sum(14) + res15 = fma52lo(res15, a[ 2], a[13]); // Sum(15) + res16 = fma52hi(res16, a[ 2], a[13]); // Sum(15) + res16 = fma52lo(res16, a[ 3], a[13]); // Sum(16) + res17 = fma52hi(res17, a[ 3], a[13]); // Sum(16) + res17 = fma52lo(res17, a[ 4], a[13]); // Sum(17) + res18 = fma52hi(res18, a[ 4], a[13]); // Sum(17) + res18 = fma52lo(res18, a[ 5], a[13]); // Sum(18) + res19 = fma52hi(res19, a[ 5], a[13]); // Sum(18) + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + U64 tempx = loadstream64(MulTblx[0+4*iter][ 0]); + mulbx[ 0] = _mm512_mask_mov_epi64(mulbx[ 0], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[0+4*iter][ 1]); + mulbx[ 1] = _mm512_mask_mov_epi64(mulbx[ 1], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[0+4*iter][ 2]); + mulbx[ 2] = _mm512_mask_mov_epi64(mulbx[ 2], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[0+4*iter][ 3]); + mulbx[ 3] = _mm512_mask_mov_epi64(mulbx[ 3], extract_sel_mask, tempx); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res19 = fma52lo(res19, a[ 6], a[13]); // Sum(19) + res20 = fma52hi(res20, a[ 6], a[13]); // Sum(19) + res20 = fma52lo(res20, a[ 7], a[13]); // Sum(20) + res21 = fma52hi(res21, a[ 7], a[13]); // Sum(20) + res21 = fma52lo(res21, a[ 8], a[13]); // Sum(21) + res22 = fma52hi(res22, a[ 8], a[13]); // Sum(21) + res22 = fma52lo(res22, a[ 9], a[13]); // Sum(22) + res23 = fma52hi(res23, a[ 9], a[13]); // Sum(22) + res23 = fma52lo(res23, a[10], a[13]); // Sum(23) + res24 = fma52hi(res24, a[10], a[13]); // Sum(23) + res14 = fma52lo(res14, a[ 0], a[14]); // Sum(14) + res15 = fma52hi(res15, a[ 0], a[14]); // Sum(14) + res15 = fma52lo(res15, a[ 1], a[14]); // Sum(15) + res16 = fma52hi(res16, a[ 1], a[14]); // Sum(15) + res16 = fma52lo(res16, a[ 2], a[14]); // Sum(16) + res17 = fma52hi(res17, a[ 2], a[14]); // Sum(16) + res17 = fma52lo(res17, a[ 3], a[14]); // Sum(17) + res18 = fma52hi(res18, a[ 3], a[14]); // Sum(17) + res18 = fma52lo(res18, a[ 4], a[14]); // Sum(18) + res19 = fma52hi(res19, a[ 4], a[14]); // Sum(18) + res19 = fma52lo(res19, a[ 5], a[14]); // Sum(19) + res20 = fma52hi(res20, a[ 5], a[14]); // Sum(19) + res20 = fma52lo(res20, a[ 6], a[14]); // Sum(20) + res21 = fma52hi(res21, a[ 6], a[14]); // Sum(20) + res21 = fma52lo(res21, a[ 7], a[14]); // Sum(21) + res22 = fma52hi(res22, a[ 7], a[14]); // Sum(21) + res22 = fma52lo(res22, a[ 8], a[14]); // Sum(22) + res23 = fma52hi(res23, a[ 8], a[14]); // Sum(22) + res23 = fma52lo(res23, a[ 9], a[14]); // Sum(23) + res24 = fma52hi(res24, a[ 9], a[14]); // Sum(23) + res15 = fma52lo(res15, a[ 0], a[15]); // Sum(15) + res16 = fma52hi(res16, a[ 0], a[15]); // Sum(15) + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + tempx = loadstream64(MulTblx[0+4*iter][ 4]); + mulbx[ 4] = _mm512_mask_mov_epi64(mulbx[ 4], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[0+4*iter][ 5]); + mulbx[ 5] = _mm512_mask_mov_epi64(mulbx[ 5], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[0+4*iter][ 6]); + mulbx[ 6] = _mm512_mask_mov_epi64(mulbx[ 6], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[0+4*iter][ 7]); + mulbx[ 7] = _mm512_mask_mov_epi64(mulbx[ 7], extract_sel_mask, tempx); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res16 = fma52lo(res16, a[ 1], a[15]); // Sum(16) + res17 = fma52hi(res17, a[ 1], a[15]); // Sum(16) + res17 = fma52lo(res17, a[ 2], a[15]); // Sum(17) + res18 = fma52hi(res18, a[ 2], a[15]); // Sum(17) + res18 = fma52lo(res18, a[ 3], a[15]); // Sum(18) + res19 = fma52hi(res19, a[ 3], a[15]); // Sum(18) + res19 = fma52lo(res19, a[ 4], a[15]); // Sum(19) + res20 = fma52hi(res20, a[ 4], a[15]); // Sum(19) + res20 = fma52lo(res20, a[ 5], a[15]); // Sum(20) + res21 = fma52hi(res21, a[ 5], a[15]); // Sum(20) + res21 = fma52lo(res21, a[ 6], a[15]); // Sum(21) + res22 = fma52hi(res22, a[ 6], a[15]); // Sum(21) + res22 = fma52lo(res22, a[ 7], a[15]); // Sum(22) + res23 = fma52hi(res23, a[ 7], a[15]); // Sum(22) + res23 = fma52lo(res23, a[ 8], a[15]); // Sum(23) + res24 = fma52hi(res24, a[ 8], a[15]); // Sum(23) + res16 = fma52lo(res16, a[ 0], a[16]); // Sum(16) + res17 = fma52hi(res17, a[ 0], a[16]); // Sum(16) + res17 = fma52lo(res17, a[ 1], a[16]); // Sum(17) + res18 = fma52hi(res18, a[ 1], a[16]); // Sum(17) + res18 = fma52lo(res18, a[ 2], a[16]); // Sum(18) + res19 = fma52hi(res19, a[ 2], a[16]); // Sum(18) + res19 = fma52lo(res19, a[ 3], a[16]); // Sum(19) + res20 = fma52hi(res20, a[ 3], a[16]); // Sum(19) + res20 = fma52lo(res20, a[ 4], a[16]); // Sum(20) + res21 = fma52hi(res21, a[ 4], a[16]); // Sum(20) + res21 = fma52lo(res21, a[ 5], a[16]); // Sum(21) + res22 = fma52hi(res22, a[ 5], a[16]); // Sum(21) + res22 = fma52lo(res22, a[ 6], a[16]); // Sum(22) + res23 = fma52hi(res23, a[ 6], a[16]); // Sum(22) + res23 = fma52lo(res23, a[ 7], a[16]); // Sum(23) + res24 = fma52hi(res24, a[ 7], a[16]); // Sum(23) + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + tempx = loadstream64(MulTblx[0+4*iter][ 8]); + mulbx[ 8] = _mm512_mask_mov_epi64(mulbx[ 8], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[0+4*iter][ 9]); + mulbx[ 9] = _mm512_mask_mov_epi64(mulbx[ 9], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[0+4*iter][10]); + mulbx[10] = _mm512_mask_mov_epi64(mulbx[10], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[0+4*iter][11]); + mulbx[11] = _mm512_mask_mov_epi64(mulbx[11], extract_sel_mask, tempx); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res17 = fma52lo(res17, a[ 0], a[17]); // Sum(17) + res18 = fma52hi(res18, a[ 0], a[17]); // Sum(17) + res18 = fma52lo(res18, a[ 1], a[17]); // Sum(18) + res19 = fma52hi(res19, a[ 1], a[17]); // Sum(18) + res19 = fma52lo(res19, a[ 2], a[17]); // Sum(19) + res20 = fma52hi(res20, a[ 2], a[17]); // Sum(19) + res20 = fma52lo(res20, a[ 3], a[17]); // Sum(20) + res21 = fma52hi(res21, a[ 3], a[17]); // Sum(20) + res21 = fma52lo(res21, a[ 4], a[17]); // Sum(21) + res22 = fma52hi(res22, a[ 4], a[17]); // Sum(21) + res22 = fma52lo(res22, a[ 5], a[17]); // Sum(22) + res23 = fma52hi(res23, a[ 5], a[17]); // Sum(22) + res23 = fma52lo(res23, a[ 6], a[17]); // Sum(23) + res24 = fma52hi(res24, a[ 6], a[17]); // Sum(23) + res18 = fma52lo(res18, a[ 0], a[18]); // Sum(18) + res19 = fma52hi(res19, a[ 0], a[18]); // Sum(18) + res19 = fma52lo(res19, a[ 1], a[18]); // Sum(19) + res20 = fma52hi(res20, a[ 1], a[18]); // Sum(19) + res20 = fma52lo(res20, a[ 2], a[18]); // Sum(20) + res21 = fma52hi(res21, a[ 2], a[18]); // Sum(20) + res21 = fma52lo(res21, a[ 3], a[18]); // Sum(21) + res22 = fma52hi(res22, a[ 3], a[18]); // Sum(21) + res22 = fma52lo(res22, a[ 4], a[18]); // Sum(22) + res23 = fma52hi(res23, a[ 4], a[18]); // Sum(22) + res23 = fma52lo(res23, a[ 5], a[18]); // Sum(23) + res24 = fma52hi(res24, a[ 5], a[18]); // Sum(23) + res19 = fma52lo(res19, a[ 0], a[19]); // Sum(19) + res20 = fma52hi(res20, a[ 0], a[19]); // Sum(19) + res20 = fma52lo(res20, a[ 1], a[19]); // Sum(20) + res21 = fma52hi(res21, a[ 1], a[19]); // Sum(20) + res21 = fma52lo(res21, a[ 2], a[19]); // Sum(21) + res22 = fma52hi(res22, a[ 2], a[19]); // Sum(21) + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + tempx = loadstream64(MulTblx[0+4*iter][12]); + mulbx[12] = _mm512_mask_mov_epi64(mulbx[12], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[0+4*iter][13]); + mulbx[13] = _mm512_mask_mov_epi64(mulbx[13], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[0+4*iter][14]); + mulbx[14] = _mm512_mask_mov_epi64(mulbx[14], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[0+4*iter][15]); + mulbx[15] = _mm512_mask_mov_epi64(mulbx[15], extract_sel_mask, tempx); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res22 = fma52lo(res22, a[ 3], a[19]); // Sum(22) + res23 = fma52hi(res23, a[ 3], a[19]); // Sum(22) + res23 = fma52lo(res23, a[ 4], a[19]); // Sum(23) + res24 = fma52hi(res24, a[ 4], a[19]); // Sum(23) + res12 = add64(res12, res12); // Double(12) + res13 = add64(res13, res13); // Double(13) + res14 = add64(res14, res14); // Double(14) + res15 = add64(res15, res15); // Double(15) + res16 = add64(res16, res16); // Double(16) + res17 = add64(res17, res17); // Double(17) + res18 = add64(res18, res18); // Double(18) + res19 = add64(res19, res19); // Double(19) + res20 = add64(res20, res20); // Double(20) + res21 = add64(res21, res21); // Double(21) + res22 = add64(res22, res22); // Double(22) + res23 = add64(res23, res23); // Double(23) + res12 = fma52lo(res12, a[ 6], a[ 6]); // Add sqr(12) + res13 = fma52hi(res13, a[ 6], a[ 6]); // Add sqr(12) + res14 = fma52lo(res14, a[ 7], a[ 7]); // Add sqr(14) + res15 = fma52hi(res15, a[ 7], a[ 7]); // Add sqr(14) + res16 = fma52lo(res16, a[ 8], a[ 8]); // Add sqr(16) + res17 = fma52hi(res17, a[ 8], a[ 8]); // Add sqr(16) + res18 = fma52lo(res18, a[ 9], a[ 9]); // Add sqr(18) + res19 = fma52hi(res19, a[ 9], a[ 9]); // Add sqr(18) + res20 = fma52lo(res20, a[10], a[10]); // Add sqr(20) + res21 = fma52hi(res21, a[10], a[10]); // Add sqr(20) + res22 = fma52lo(res22, a[11], a[11]); // Add sqr(22) + res23 = fma52hi(res23, a[11], a[11]); // Add sqr(22) + res24 = fma52lo(res24, a[11], a[13]); // Sum(24) + res25 = fma52hi(res25, a[11], a[13]); // Sum(24) + res25 = fma52lo(res25, a[12], a[13]); // Sum(25) + res26 = fma52hi(res26, a[12], a[13]); // Sum(25) + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + tempx = loadstream64(MulTblx[0+4*iter][16]); + mulbx[16] = _mm512_mask_mov_epi64(mulbx[16], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[0+4*iter][17]); + mulbx[17] = _mm512_mask_mov_epi64(mulbx[17], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[0+4*iter][18]); + mulbx[18] = _mm512_mask_mov_epi64(mulbx[18], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[0+4*iter][19]); + mulbx[19] = _mm512_mask_mov_epi64(mulbx[19], extract_sel_mask, tempx); + extract_sel_mask = extract_sel_mask_next; + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res24 = fma52lo(res24, a[10], a[14]); // Sum(24) + res25 = fma52hi(res25, a[10], a[14]); // Sum(24) + res25 = fma52lo(res25, a[11], a[14]); // Sum(25) + res26 = fma52hi(res26, a[11], a[14]); // Sum(25) + res26 = fma52lo(res26, a[12], a[14]); // Sum(26) + res27 = fma52hi(res27, a[12], a[14]); // Sum(26) + res27 = fma52lo(res27, a[13], a[14]); // Sum(27) + res28 = fma52hi(res28, a[13], a[14]); // Sum(27) + res24 = fma52lo(res24, a[ 9], a[15]); // Sum(24) + res25 = fma52hi(res25, a[ 9], a[15]); // Sum(24) + res25 = fma52lo(res25, a[10], a[15]); // Sum(25) + res26 = fma52hi(res26, a[10], a[15]); // Sum(25) + res26 = fma52lo(res26, a[11], a[15]); // Sum(26) + res27 = fma52hi(res27, a[11], a[15]); // Sum(26) + res27 = fma52lo(res27, a[12], a[15]); // Sum(27) + res28 = fma52hi(res28, a[12], a[15]); // Sum(27) + res28 = fma52lo(res28, a[13], a[15]); // Sum(28) + res29 = fma52hi(res29, a[13], a[15]); // Sum(28) + res29 = fma52lo(res29, a[14], a[15]); // Sum(29) + res30 = fma52hi(res30, a[14], a[15]); // Sum(29) + res24 = fma52lo(res24, a[ 8], a[16]); // Sum(24) + res25 = fma52hi(res25, a[ 8], a[16]); // Sum(24) + res25 = fma52lo(res25, a[ 9], a[16]); // Sum(25) + res26 = fma52hi(res26, a[ 9], a[16]); // Sum(25) + res26 = fma52lo(res26, a[10], a[16]); // Sum(26) + res27 = fma52hi(res27, a[10], a[16]); // Sum(26) + res27 = fma52lo(res27, a[11], a[16]); // Sum(27) + res28 = fma52hi(res28, a[11], a[16]); // Sum(27) + res28 = fma52lo(res28, a[12], a[16]); // Sum(28) + res29 = fma52hi(res29, a[12], a[16]); // Sum(28) + res29 = fma52lo(res29, a[13], a[16]); // Sum(29) + res30 = fma52hi(res30, a[13], a[16]); // Sum(29) + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + idx_curr = set64(2+4*iter); + extract_sel_mask_next = cmpeq64_mask(idx_curr, idx_target); + temp = loadstream64(MulTbl[1+4*iter][ 0]); + mulb[ 0] = _mm512_mask_mov_epi64(mulb[ 0], extract_sel_mask, temp); + temp = loadstream64(MulTbl[1+4*iter][ 1]); + mulb[ 1] = _mm512_mask_mov_epi64(mulb[ 1], extract_sel_mask, temp); + temp = loadstream64(MulTbl[1+4*iter][ 2]); + mulb[ 2] = _mm512_mask_mov_epi64(mulb[ 2], extract_sel_mask, temp); + temp = loadstream64(MulTbl[1+4*iter][ 3]); + mulb[ 3] = _mm512_mask_mov_epi64(mulb[ 3], extract_sel_mask, temp); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res30 = fma52lo(res30, a[14], a[16]); // Sum(30) + res31 = fma52hi(res31, a[14], a[16]); // Sum(30) + res31 = fma52lo(res31, a[15], a[16]); // Sum(31) + res32 = fma52hi(res32, a[15], a[16]); // Sum(31) + res24 = fma52lo(res24, a[ 7], a[17]); // Sum(24) + res25 = fma52hi(res25, a[ 7], a[17]); // Sum(24) + res25 = fma52lo(res25, a[ 8], a[17]); // Sum(25) + res26 = fma52hi(res26, a[ 8], a[17]); // Sum(25) + res26 = fma52lo(res26, a[ 9], a[17]); // Sum(26) + res27 = fma52hi(res27, a[ 9], a[17]); // Sum(26) + res27 = fma52lo(res27, a[10], a[17]); // Sum(27) + res28 = fma52hi(res28, a[10], a[17]); // Sum(27) + res28 = fma52lo(res28, a[11], a[17]); // Sum(28) + res29 = fma52hi(res29, a[11], a[17]); // Sum(28) + res29 = fma52lo(res29, a[12], a[17]); // Sum(29) + res30 = fma52hi(res30, a[12], a[17]); // Sum(29) + res30 = fma52lo(res30, a[13], a[17]); // Sum(30) + res31 = fma52hi(res31, a[13], a[17]); // Sum(30) + res31 = fma52lo(res31, a[14], a[17]); // Sum(31) + res32 = fma52hi(res32, a[14], a[17]); // Sum(31) + res32 = fma52lo(res32, a[15], a[17]); // Sum(32) + res33 = fma52hi(res33, a[15], a[17]); // Sum(32) + res33 = fma52lo(res33, a[16], a[17]); // Sum(33) + res34 = fma52hi(res34, a[16], a[17]); // Sum(33) + res24 = fma52lo(res24, a[ 6], a[18]); // Sum(24) + res25 = fma52hi(res25, a[ 6], a[18]); // Sum(24) + res25 = fma52lo(res25, a[ 7], a[18]); // Sum(25) + res26 = fma52hi(res26, a[ 7], a[18]); // Sum(25) + res26 = fma52lo(res26, a[ 8], a[18]); // Sum(26) + res27 = fma52hi(res27, a[ 8], a[18]); // Sum(26) + res27 = fma52lo(res27, a[ 9], a[18]); // Sum(27) + res28 = fma52hi(res28, a[ 9], a[18]); // Sum(27) + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + temp = loadstream64(MulTbl[1+4*iter][ 4]); + mulb[ 4] = _mm512_mask_mov_epi64(mulb[ 4], extract_sel_mask, temp); + temp = loadstream64(MulTbl[1+4*iter][ 5]); + mulb[ 5] = _mm512_mask_mov_epi64(mulb[ 5], extract_sel_mask, temp); + temp = loadstream64(MulTbl[1+4*iter][ 6]); + mulb[ 6] = _mm512_mask_mov_epi64(mulb[ 6], extract_sel_mask, temp); + temp = loadstream64(MulTbl[1+4*iter][ 7]); + mulb[ 7] = _mm512_mask_mov_epi64(mulb[ 7], extract_sel_mask, temp); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res28 = fma52lo(res28, a[10], a[18]); // Sum(28) + res29 = fma52hi(res29, a[10], a[18]); // Sum(28) + res29 = fma52lo(res29, a[11], a[18]); // Sum(29) + res30 = fma52hi(res30, a[11], a[18]); // Sum(29) + res30 = fma52lo(res30, a[12], a[18]); // Sum(30) + res31 = fma52hi(res31, a[12], a[18]); // Sum(30) + res31 = fma52lo(res31, a[13], a[18]); // Sum(31) + res32 = fma52hi(res32, a[13], a[18]); // Sum(31) + res32 = fma52lo(res32, a[14], a[18]); // Sum(32) + res33 = fma52hi(res33, a[14], a[18]); // Sum(32) + res33 = fma52lo(res33, a[15], a[18]); // Sum(33) + res34 = fma52hi(res34, a[15], a[18]); // Sum(33) + res34 = fma52lo(res34, a[16], a[18]); // Sum(34) + res35 = fma52hi(res35, a[16], a[18]); // Sum(34) + res35 = fma52lo(res35, a[17], a[18]); // Sum(35) + res36 = fma52hi(res36, a[17], a[18]); // Sum(35) + res24 = fma52lo(res24, a[ 5], a[19]); // Sum(24) + res25 = fma52hi(res25, a[ 5], a[19]); // Sum(24) + res25 = fma52lo(res25, a[ 6], a[19]); // Sum(25) + res26 = fma52hi(res26, a[ 6], a[19]); // Sum(25) + res26 = fma52lo(res26, a[ 7], a[19]); // Sum(26) + res27 = fma52hi(res27, a[ 7], a[19]); // Sum(26) + res27 = fma52lo(res27, a[ 8], a[19]); // Sum(27) + res28 = fma52hi(res28, a[ 8], a[19]); // Sum(27) + res28 = fma52lo(res28, a[ 9], a[19]); // Sum(28) + res29 = fma52hi(res29, a[ 9], a[19]); // Sum(28) + res29 = fma52lo(res29, a[10], a[19]); // Sum(29) + res30 = fma52hi(res30, a[10], a[19]); // Sum(29) + res30 = fma52lo(res30, a[11], a[19]); // Sum(30) + res31 = fma52hi(res31, a[11], a[19]); // Sum(30) + res31 = fma52lo(res31, a[12], a[19]); // Sum(31) + res32 = fma52hi(res32, a[12], a[19]); // Sum(31) + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + temp = loadstream64(MulTbl[1+4*iter][ 8]); + mulb[ 8] = _mm512_mask_mov_epi64(mulb[ 8], extract_sel_mask, temp); + temp = loadstream64(MulTbl[1+4*iter][ 9]); + mulb[ 9] = _mm512_mask_mov_epi64(mulb[ 9], extract_sel_mask, temp); + temp = loadstream64(MulTbl[1+4*iter][10]); + mulb[10] = _mm512_mask_mov_epi64(mulb[10], extract_sel_mask, temp); + temp = loadstream64(MulTbl[1+4*iter][11]); + mulb[11] = _mm512_mask_mov_epi64(mulb[11], extract_sel_mask, temp); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res32 = fma52lo(res32, a[13], a[19]); // Sum(32) + res33 = fma52hi(res33, a[13], a[19]); // Sum(32) + res33 = fma52lo(res33, a[14], a[19]); // Sum(33) + res34 = fma52hi(res34, a[14], a[19]); // Sum(33) + res34 = fma52lo(res34, a[15], a[19]); // Sum(34) + res35 = fma52hi(res35, a[15], a[19]); // Sum(34) + res35 = fma52lo(res35, a[16], a[19]); // Sum(35) + res36 = fma52hi(res36, a[16], a[19]); // Sum(35) + res24 = add64(res24, res24); // Double(24) + res25 = add64(res25, res25); // Double(25) + res26 = add64(res26, res26); // Double(26) + res27 = add64(res27, res27); // Double(27) + res28 = add64(res28, res28); // Double(28) + res29 = add64(res29, res29); // Double(29) + res30 = add64(res30, res30); // Double(30) + res31 = add64(res31, res31); // Double(31) + res32 = add64(res32, res32); // Double(32) + res33 = add64(res33, res33); // Double(33) + res34 = add64(res34, res34); // Double(34) + res35 = add64(res35, res35); // Double(35) + res24 = fma52lo(res24, a[12], a[12]); // Add sqr(24) + res25 = fma52hi(res25, a[12], a[12]); // Add sqr(24) + res26 = fma52lo(res26, a[13], a[13]); // Add sqr(26) + res27 = fma52hi(res27, a[13], a[13]); // Add sqr(26) + res28 = fma52lo(res28, a[14], a[14]); // Add sqr(28) + res29 = fma52hi(res29, a[14], a[14]); // Add sqr(28) + res30 = fma52lo(res30, a[15], a[15]); // Add sqr(30) + res31 = fma52hi(res31, a[15], a[15]); // Add sqr(30) + res32 = fma52lo(res32, a[16], a[16]); // Add sqr(32) + res33 = fma52hi(res33, a[16], a[16]); // Add sqr(32) + res34 = fma52lo(res34, a[17], a[17]); // Add sqr(34) + res35 = fma52hi(res35, a[17], a[17]); // Add sqr(34) + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + temp = loadstream64(MulTbl[1+4*iter][12]); + mulb[12] = _mm512_mask_mov_epi64(mulb[12], extract_sel_mask, temp); + temp = loadstream64(MulTbl[1+4*iter][13]); + mulb[13] = _mm512_mask_mov_epi64(mulb[13], extract_sel_mask, temp); + temp = loadstream64(MulTbl[1+4*iter][14]); + mulb[14] = _mm512_mask_mov_epi64(mulb[14], extract_sel_mask, temp); + temp = loadstream64(MulTbl[1+4*iter][15]); + mulb[15] = _mm512_mask_mov_epi64(mulb[15], extract_sel_mask, temp); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res36 = fma52lo(res36, a[17], a[19]); // Sum(36) + res37 = fma52hi(res37, a[17], a[19]); // Sum(36) + res37 = fma52lo(res37, a[18], a[19]); // Sum(37) + res38 = fma52hi(res38, a[18], a[19]); // Sum(37) + res36 = add64(res36, res36); // Double(36) + res37 = add64(res37, res37); // Double(37) + res38 = add64(res38, res38); // Double(38) + res36 = fma52lo(res36, a[18], a[18]); // Add sqr(36) + res37 = fma52hi(res37, a[18], a[18]); // Add sqr(36) + res38 = fma52lo(res38, a[19], a[19]); // Add sqr(38) + res39 = fma52hi(res39, a[19], a[19]); // Add sqr(38) + + // Begin Reduction + U64 u0 = mul52lo(res00, mont_constant); + + res00 = fma52lo(res00, u0, m[ 0]); + res01 = fma52hi(res01, u0, m[ 0]); + res01 = fma52lo(res01, u0, m[ 1]); + res02 = fma52hi(res02, u0, m[ 1]); + + res01 = add64(res01, srli64(res00, DIGIT_SIZE)); + U64 u1 = mul52lo(res01, mont_constant); // early multiplication for the reduction of res01 + + res02 = fma52lo(res02, u0, m[ 2]); + res03 = fma52hi(res03, u0, m[ 2]); + res03 = fma52lo(res03, u0, m[ 3]); + res04 = fma52hi(res04, u0, m[ 3]); + res04 = fma52lo(res04, u0, m[ 4]); + res05 = fma52hi(res05, u0, m[ 4]); + res05 = fma52lo(res05, u0, m[ 5]); + res06 = fma52hi(res06, u0, m[ 5]); + res06 = fma52lo(res06, u0, m[ 6]); + res07 = fma52hi(res07, u0, m[ 6]); + res07 = fma52lo(res07, u0, m[ 7]); + res08 = fma52hi(res08, u0, m[ 7]); + res08 = fma52lo(res08, u0, m[ 8]); + res09 = fma52hi(res09, u0, m[ 8]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + temp = loadstream64(MulTbl[1+4*iter][16]); + mulb[16] = _mm512_mask_mov_epi64(mulb[16], extract_sel_mask, temp); + temp = loadstream64(MulTbl[1+4*iter][17]); + mulb[17] = _mm512_mask_mov_epi64(mulb[17], extract_sel_mask, temp); + temp = loadstream64(MulTbl[1+4*iter][18]); + mulb[18] = _mm512_mask_mov_epi64(mulb[18], extract_sel_mask, temp); + temp = loadstream64(MulTbl[1+4*iter][19]); + mulb[19] = _mm512_mask_mov_epi64(mulb[19], extract_sel_mask, temp); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res09 = fma52lo(res09, u0, m[ 9]); + res10 = fma52hi(res10, u0, m[ 9]); + res10 = fma52lo(res10, u0, m[10]); + res11 = fma52hi(res11, u0, m[10]); + res11 = fma52lo(res11, u0, m[11]); + res12 = fma52hi(res12, u0, m[11]); + res12 = fma52lo(res12, u0, m[12]); + res13 = fma52hi(res13, u0, m[12]); + res13 = fma52lo(res13, u0, m[13]); + res14 = fma52hi(res14, u0, m[13]); + res14 = fma52lo(res14, u0, m[14]); + res15 = fma52hi(res15, u0, m[14]); + res15 = fma52lo(res15, u0, m[15]); + res16 = fma52hi(res16, u0, m[15]); + res16 = fma52lo(res16, u0, m[16]); + res17 = fma52hi(res17, u0, m[16]); + res17 = fma52lo(res17, u0, m[17]); + res18 = fma52hi(res18, u0, m[17]); + res18 = fma52lo(res18, u0, m[18]); + res19 = fma52hi(res19, u0, m[18]); + res19 = fma52lo(res19, u0, m[19]); + res20 = fma52hi(res20, u0, m[19]); + res01 = fma52lo(res01, u1, m[ 0]); + res02 = fma52hi(res02, u1, m[ 0]); + res02 = fma52lo(res02, u1, m[ 1]); + res03 = fma52hi(res03, u1, m[ 1]); + + res02 = add64(res02, srli64(res01, DIGIT_SIZE)); + U64 u2 = mul52lo(res02, mont_constant); // early multiplication for the reduction of res02 + + res03 = fma52lo(res03, u1, m[ 2]); + res04 = fma52hi(res04, u1, m[ 2]); + res04 = fma52lo(res04, u1, m[ 3]); + res05 = fma52hi(res05, u1, m[ 3]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + tempx = loadstream64(MulTblx[1+4*iter][ 0]); + mulbx[ 0] = _mm512_mask_mov_epi64(mulbx[ 0], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[1+4*iter][ 1]); + mulbx[ 1] = _mm512_mask_mov_epi64(mulbx[ 1], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[1+4*iter][ 2]); + mulbx[ 2] = _mm512_mask_mov_epi64(mulbx[ 2], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[1+4*iter][ 3]); + mulbx[ 3] = _mm512_mask_mov_epi64(mulbx[ 3], extract_sel_mask, tempx); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res05 = fma52lo(res05, u1, m[ 4]); + res06 = fma52hi(res06, u1, m[ 4]); + res06 = fma52lo(res06, u1, m[ 5]); + res07 = fma52hi(res07, u1, m[ 5]); + res07 = fma52lo(res07, u1, m[ 6]); + res08 = fma52hi(res08, u1, m[ 6]); + res08 = fma52lo(res08, u1, m[ 7]); + res09 = fma52hi(res09, u1, m[ 7]); + res09 = fma52lo(res09, u1, m[ 8]); + res10 = fma52hi(res10, u1, m[ 8]); + res10 = fma52lo(res10, u1, m[ 9]); + res11 = fma52hi(res11, u1, m[ 9]); + res11 = fma52lo(res11, u1, m[10]); + res12 = fma52hi(res12, u1, m[10]); + res12 = fma52lo(res12, u1, m[11]); + res13 = fma52hi(res13, u1, m[11]); + res13 = fma52lo(res13, u1, m[12]); + res14 = fma52hi(res14, u1, m[12]); + res14 = fma52lo(res14, u1, m[13]); + res15 = fma52hi(res15, u1, m[13]); + res15 = fma52lo(res15, u1, m[14]); + res16 = fma52hi(res16, u1, m[14]); + res16 = fma52lo(res16, u1, m[15]); + res17 = fma52hi(res17, u1, m[15]); + res17 = fma52lo(res17, u1, m[16]); + res18 = fma52hi(res18, u1, m[16]); + res18 = fma52lo(res18, u1, m[17]); + res19 = fma52hi(res19, u1, m[17]); + res19 = fma52lo(res19, u1, m[18]); + res20 = fma52hi(res20, u1, m[18]); + res20 = fma52lo(res20, u1, m[19]); + res21 = fma52hi(res21, u1, m[19]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + tempx = loadstream64(MulTblx[1+4*iter][ 4]); + mulbx[ 4] = _mm512_mask_mov_epi64(mulbx[ 4], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[1+4*iter][ 5]); + mulbx[ 5] = _mm512_mask_mov_epi64(mulbx[ 5], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[1+4*iter][ 6]); + mulbx[ 6] = _mm512_mask_mov_epi64(mulbx[ 6], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[1+4*iter][ 7]); + mulbx[ 7] = _mm512_mask_mov_epi64(mulbx[ 7], extract_sel_mask, tempx); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res02 = fma52lo(res02, u2, m[ 0]); + res03 = fma52hi(res03, u2, m[ 0]); + res03 = fma52lo(res03, u2, m[ 1]); + res04 = fma52hi(res04, u2, m[ 1]); + + res03 = add64(res03, srli64(res02, DIGIT_SIZE)); + U64 u3 = mul52lo(res03, mont_constant); // early multiplication for the reduction of res03 + + res04 = fma52lo(res04, u2, m[ 2]); + res05 = fma52hi(res05, u2, m[ 2]); + res05 = fma52lo(res05, u2, m[ 3]); + res06 = fma52hi(res06, u2, m[ 3]); + res06 = fma52lo(res06, u2, m[ 4]); + res07 = fma52hi(res07, u2, m[ 4]); + res07 = fma52lo(res07, u2, m[ 5]); + res08 = fma52hi(res08, u2, m[ 5]); + res08 = fma52lo(res08, u2, m[ 6]); + res09 = fma52hi(res09, u2, m[ 6]); + res09 = fma52lo(res09, u2, m[ 7]); + res10 = fma52hi(res10, u2, m[ 7]); + res10 = fma52lo(res10, u2, m[ 8]); + res11 = fma52hi(res11, u2, m[ 8]); + res11 = fma52lo(res11, u2, m[ 9]); + res12 = fma52hi(res12, u2, m[ 9]); + res12 = fma52lo(res12, u2, m[10]); + res13 = fma52hi(res13, u2, m[10]); + res13 = fma52lo(res13, u2, m[11]); + res14 = fma52hi(res14, u2, m[11]); + res14 = fma52lo(res14, u2, m[12]); + res15 = fma52hi(res15, u2, m[12]); + res15 = fma52lo(res15, u2, m[13]); + res16 = fma52hi(res16, u2, m[13]); + res16 = fma52lo(res16, u2, m[14]); + res17 = fma52hi(res17, u2, m[14]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + tempx = loadstream64(MulTblx[1+4*iter][ 8]); + mulbx[ 8] = _mm512_mask_mov_epi64(mulbx[ 8], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[1+4*iter][ 9]); + mulbx[ 9] = _mm512_mask_mov_epi64(mulbx[ 9], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[1+4*iter][10]); + mulbx[10] = _mm512_mask_mov_epi64(mulbx[10], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[1+4*iter][11]); + mulbx[11] = _mm512_mask_mov_epi64(mulbx[11], extract_sel_mask, tempx); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res17 = fma52lo(res17, u2, m[15]); + res18 = fma52hi(res18, u2, m[15]); + res18 = fma52lo(res18, u2, m[16]); + res19 = fma52hi(res19, u2, m[16]); + res19 = fma52lo(res19, u2, m[17]); + res20 = fma52hi(res20, u2, m[17]); + res20 = fma52lo(res20, u2, m[18]); + res21 = fma52hi(res21, u2, m[18]); + res21 = fma52lo(res21, u2, m[19]); + res22 = fma52hi(res22, u2, m[19]); + res03 = fma52lo(res03, u3, m[ 0]); + res04 = fma52hi(res04, u3, m[ 0]); + res04 = fma52lo(res04, u3, m[ 1]); + res05 = fma52hi(res05, u3, m[ 1]); + + res04 = add64(res04, srli64(res03, DIGIT_SIZE)); + U64 u4 = mul52lo(res04, mont_constant); // early multiplication for the reduction of res04 + + res05 = fma52lo(res05, u3, m[ 2]); + res06 = fma52hi(res06, u3, m[ 2]); + res06 = fma52lo(res06, u3, m[ 3]); + res07 = fma52hi(res07, u3, m[ 3]); + res07 = fma52lo(res07, u3, m[ 4]); + res08 = fma52hi(res08, u3, m[ 4]); + res08 = fma52lo(res08, u3, m[ 5]); + res09 = fma52hi(res09, u3, m[ 5]); + res09 = fma52lo(res09, u3, m[ 6]); + res10 = fma52hi(res10, u3, m[ 6]); + res10 = fma52lo(res10, u3, m[ 7]); + res11 = fma52hi(res11, u3, m[ 7]); + res11 = fma52lo(res11, u3, m[ 8]); + res12 = fma52hi(res12, u3, m[ 8]); + res12 = fma52lo(res12, u3, m[ 9]); + res13 = fma52hi(res13, u3, m[ 9]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + tempx = loadstream64(MulTblx[1+4*iter][12]); + mulbx[12] = _mm512_mask_mov_epi64(mulbx[12], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[1+4*iter][13]); + mulbx[13] = _mm512_mask_mov_epi64(mulbx[13], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[1+4*iter][14]); + mulbx[14] = _mm512_mask_mov_epi64(mulbx[14], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[1+4*iter][15]); + mulbx[15] = _mm512_mask_mov_epi64(mulbx[15], extract_sel_mask, tempx); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res13 = fma52lo(res13, u3, m[10]); + res14 = fma52hi(res14, u3, m[10]); + res14 = fma52lo(res14, u3, m[11]); + res15 = fma52hi(res15, u3, m[11]); + res15 = fma52lo(res15, u3, m[12]); + res16 = fma52hi(res16, u3, m[12]); + res16 = fma52lo(res16, u3, m[13]); + res17 = fma52hi(res17, u3, m[13]); + res17 = fma52lo(res17, u3, m[14]); + res18 = fma52hi(res18, u3, m[14]); + res18 = fma52lo(res18, u3, m[15]); + res19 = fma52hi(res19, u3, m[15]); + res19 = fma52lo(res19, u3, m[16]); + res20 = fma52hi(res20, u3, m[16]); + res20 = fma52lo(res20, u3, m[17]); + res21 = fma52hi(res21, u3, m[17]); + res21 = fma52lo(res21, u3, m[18]); + res22 = fma52hi(res22, u3, m[18]); + res22 = fma52lo(res22, u3, m[19]); + res23 = fma52hi(res23, u3, m[19]); + res04 = fma52lo(res04, u4, m[ 0]); + res05 = fma52hi(res05, u4, m[ 0]); + res05 = fma52lo(res05, u4, m[ 1]); + res06 = fma52hi(res06, u4, m[ 1]); + + res05 = add64(res05, srli64(res04, DIGIT_SIZE)); + U64 u5 = mul52lo(res05, mont_constant); // early multiplication for the reduction of res05 + + res06 = fma52lo(res06, u4, m[ 2]); + res07 = fma52hi(res07, u4, m[ 2]); + res07 = fma52lo(res07, u4, m[ 3]); + res08 = fma52hi(res08, u4, m[ 3]); + res08 = fma52lo(res08, u4, m[ 4]); + res09 = fma52hi(res09, u4, m[ 4]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + tempx = loadstream64(MulTblx[1+4*iter][16]); + mulbx[16] = _mm512_mask_mov_epi64(mulbx[16], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[1+4*iter][17]); + mulbx[17] = _mm512_mask_mov_epi64(mulbx[17], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[1+4*iter][18]); + mulbx[18] = _mm512_mask_mov_epi64(mulbx[18], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[1+4*iter][19]); + mulbx[19] = _mm512_mask_mov_epi64(mulbx[19], extract_sel_mask, tempx); + extract_sel_mask = extract_sel_mask_next; + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res09 = fma52lo(res09, u4, m[ 5]); + res10 = fma52hi(res10, u4, m[ 5]); + res10 = fma52lo(res10, u4, m[ 6]); + res11 = fma52hi(res11, u4, m[ 6]); + res11 = fma52lo(res11, u4, m[ 7]); + res12 = fma52hi(res12, u4, m[ 7]); + res12 = fma52lo(res12, u4, m[ 8]); + res13 = fma52hi(res13, u4, m[ 8]); + res13 = fma52lo(res13, u4, m[ 9]); + res14 = fma52hi(res14, u4, m[ 9]); + res14 = fma52lo(res14, u4, m[10]); + res15 = fma52hi(res15, u4, m[10]); + res15 = fma52lo(res15, u4, m[11]); + res16 = fma52hi(res16, u4, m[11]); + res16 = fma52lo(res16, u4, m[12]); + res17 = fma52hi(res17, u4, m[12]); + res17 = fma52lo(res17, u4, m[13]); + res18 = fma52hi(res18, u4, m[13]); + res18 = fma52lo(res18, u4, m[14]); + res19 = fma52hi(res19, u4, m[14]); + res19 = fma52lo(res19, u4, m[15]); + res20 = fma52hi(res20, u4, m[15]); + res20 = fma52lo(res20, u4, m[16]); + res21 = fma52hi(res21, u4, m[16]); + res21 = fma52lo(res21, u4, m[17]); + res22 = fma52hi(res22, u4, m[17]); + res22 = fma52lo(res22, u4, m[18]); + res23 = fma52hi(res23, u4, m[18]); + res23 = fma52lo(res23, u4, m[19]); + res24 = fma52hi(res24, u4, m[19]); + res05 = fma52lo(res05, u5, m[ 0]); + res06 = fma52hi(res06, u5, m[ 0]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + idx_curr = set64(3+4*iter); + extract_sel_mask_next = cmpeq64_mask(idx_curr, idx_target); + temp = loadstream64(MulTbl[2+4*iter][ 0]); + mulb[ 0] = _mm512_mask_mov_epi64(mulb[ 0], extract_sel_mask, temp); + temp = loadstream64(MulTbl[2+4*iter][ 1]); + mulb[ 1] = _mm512_mask_mov_epi64(mulb[ 1], extract_sel_mask, temp); + temp = loadstream64(MulTbl[2+4*iter][ 2]); + mulb[ 2] = _mm512_mask_mov_epi64(mulb[ 2], extract_sel_mask, temp); + temp = loadstream64(MulTbl[2+4*iter][ 3]); + mulb[ 3] = _mm512_mask_mov_epi64(mulb[ 3], extract_sel_mask, temp); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res06 = fma52lo(res06, u5, m[ 1]); + res07 = fma52hi(res07, u5, m[ 1]); + + res06 = add64(res06, srli64(res05, DIGIT_SIZE)); + U64 u6 = mul52lo(res06, mont_constant); // early multiplication for the reduction of res06 + + res07 = fma52lo(res07, u5, m[ 2]); + res08 = fma52hi(res08, u5, m[ 2]); + res08 = fma52lo(res08, u5, m[ 3]); + res09 = fma52hi(res09, u5, m[ 3]); + res09 = fma52lo(res09, u5, m[ 4]); + res10 = fma52hi(res10, u5, m[ 4]); + res10 = fma52lo(res10, u5, m[ 5]); + res11 = fma52hi(res11, u5, m[ 5]); + res11 = fma52lo(res11, u5, m[ 6]); + res12 = fma52hi(res12, u5, m[ 6]); + res12 = fma52lo(res12, u5, m[ 7]); + res13 = fma52hi(res13, u5, m[ 7]); + res13 = fma52lo(res13, u5, m[ 8]); + res14 = fma52hi(res14, u5, m[ 8]); + res14 = fma52lo(res14, u5, m[ 9]); + res15 = fma52hi(res15, u5, m[ 9]); + res15 = fma52lo(res15, u5, m[10]); + res16 = fma52hi(res16, u5, m[10]); + res16 = fma52lo(res16, u5, m[11]); + res17 = fma52hi(res17, u5, m[11]); + res17 = fma52lo(res17, u5, m[12]); + res18 = fma52hi(res18, u5, m[12]); + res18 = fma52lo(res18, u5, m[13]); + res19 = fma52hi(res19, u5, m[13]); + res19 = fma52lo(res19, u5, m[14]); + res20 = fma52hi(res20, u5, m[14]); + res20 = fma52lo(res20, u5, m[15]); + res21 = fma52hi(res21, u5, m[15]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + temp = loadstream64(MulTbl[2+4*iter][ 4]); + mulb[ 4] = _mm512_mask_mov_epi64(mulb[ 4], extract_sel_mask, temp); + temp = loadstream64(MulTbl[2+4*iter][ 5]); + mulb[ 5] = _mm512_mask_mov_epi64(mulb[ 5], extract_sel_mask, temp); + temp = loadstream64(MulTbl[2+4*iter][ 6]); + mulb[ 6] = _mm512_mask_mov_epi64(mulb[ 6], extract_sel_mask, temp); + temp = loadstream64(MulTbl[2+4*iter][ 7]); + mulb[ 7] = _mm512_mask_mov_epi64(mulb[ 7], extract_sel_mask, temp); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res21 = fma52lo(res21, u5, m[16]); + res22 = fma52hi(res22, u5, m[16]); + res22 = fma52lo(res22, u5, m[17]); + res23 = fma52hi(res23, u5, m[17]); + res23 = fma52lo(res23, u5, m[18]); + res24 = fma52hi(res24, u5, m[18]); + res24 = fma52lo(res24, u5, m[19]); + res25 = fma52hi(res25, u5, m[19]); + res06 = fma52lo(res06, u6, m[ 0]); + res07 = fma52hi(res07, u6, m[ 0]); + res07 = fma52lo(res07, u6, m[ 1]); + res08 = fma52hi(res08, u6, m[ 1]); + + res07 = add64(res07, srli64(res06, DIGIT_SIZE)); + U64 u7 = mul52lo(res07, mont_constant); // early multiplication for the reduction of res07 + + res08 = fma52lo(res08, u6, m[ 2]); + res09 = fma52hi(res09, u6, m[ 2]); + res09 = fma52lo(res09, u6, m[ 3]); + res10 = fma52hi(res10, u6, m[ 3]); + res10 = fma52lo(res10, u6, m[ 4]); + res11 = fma52hi(res11, u6, m[ 4]); + res11 = fma52lo(res11, u6, m[ 5]); + res12 = fma52hi(res12, u6, m[ 5]); + res12 = fma52lo(res12, u6, m[ 6]); + res13 = fma52hi(res13, u6, m[ 6]); + res13 = fma52lo(res13, u6, m[ 7]); + res14 = fma52hi(res14, u6, m[ 7]); + res14 = fma52lo(res14, u6, m[ 8]); + res15 = fma52hi(res15, u6, m[ 8]); + res15 = fma52lo(res15, u6, m[ 9]); + res16 = fma52hi(res16, u6, m[ 9]); + res16 = fma52lo(res16, u6, m[10]); + res17 = fma52hi(res17, u6, m[10]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + temp = loadstream64(MulTbl[2+4*iter][ 8]); + mulb[ 8] = _mm512_mask_mov_epi64(mulb[ 8], extract_sel_mask, temp); + temp = loadstream64(MulTbl[2+4*iter][ 9]); + mulb[ 9] = _mm512_mask_mov_epi64(mulb[ 9], extract_sel_mask, temp); + temp = loadstream64(MulTbl[2+4*iter][10]); + mulb[10] = _mm512_mask_mov_epi64(mulb[10], extract_sel_mask, temp); + temp = loadstream64(MulTbl[2+4*iter][11]); + mulb[11] = _mm512_mask_mov_epi64(mulb[11], extract_sel_mask, temp); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res17 = fma52lo(res17, u6, m[11]); + res18 = fma52hi(res18, u6, m[11]); + res18 = fma52lo(res18, u6, m[12]); + res19 = fma52hi(res19, u6, m[12]); + res19 = fma52lo(res19, u6, m[13]); + res20 = fma52hi(res20, u6, m[13]); + res20 = fma52lo(res20, u6, m[14]); + res21 = fma52hi(res21, u6, m[14]); + res21 = fma52lo(res21, u6, m[15]); + res22 = fma52hi(res22, u6, m[15]); + res22 = fma52lo(res22, u6, m[16]); + res23 = fma52hi(res23, u6, m[16]); + res23 = fma52lo(res23, u6, m[17]); + res24 = fma52hi(res24, u6, m[17]); + res24 = fma52lo(res24, u6, m[18]); + res25 = fma52hi(res25, u6, m[18]); + res25 = fma52lo(res25, u6, m[19]); + res26 = fma52hi(res26, u6, m[19]); + res07 = fma52lo(res07, u7, m[ 0]); + res08 = fma52hi(res08, u7, m[ 0]); + res08 = fma52lo(res08, u7, m[ 1]); + res09 = fma52hi(res09, u7, m[ 1]); + + res08 = add64(res08, srli64(res07, DIGIT_SIZE)); + U64 u8 = mul52lo(res08, mont_constant); // early multiplication for the reduction of res08 + + res09 = fma52lo(res09, u7, m[ 2]); + res10 = fma52hi(res10, u7, m[ 2]); + res10 = fma52lo(res10, u7, m[ 3]); + res11 = fma52hi(res11, u7, m[ 3]); + res11 = fma52lo(res11, u7, m[ 4]); + res12 = fma52hi(res12, u7, m[ 4]); + res12 = fma52lo(res12, u7, m[ 5]); + res13 = fma52hi(res13, u7, m[ 5]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + temp = loadstream64(MulTbl[2+4*iter][12]); + mulb[12] = _mm512_mask_mov_epi64(mulb[12], extract_sel_mask, temp); + temp = loadstream64(MulTbl[2+4*iter][13]); + mulb[13] = _mm512_mask_mov_epi64(mulb[13], extract_sel_mask, temp); + temp = loadstream64(MulTbl[2+4*iter][14]); + mulb[14] = _mm512_mask_mov_epi64(mulb[14], extract_sel_mask, temp); + temp = loadstream64(MulTbl[2+4*iter][15]); + mulb[15] = _mm512_mask_mov_epi64(mulb[15], extract_sel_mask, temp); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res13 = fma52lo(res13, u7, m[ 6]); + res14 = fma52hi(res14, u7, m[ 6]); + res14 = fma52lo(res14, u7, m[ 7]); + res15 = fma52hi(res15, u7, m[ 7]); + res15 = fma52lo(res15, u7, m[ 8]); + res16 = fma52hi(res16, u7, m[ 8]); + res16 = fma52lo(res16, u7, m[ 9]); + res17 = fma52hi(res17, u7, m[ 9]); + res17 = fma52lo(res17, u7, m[10]); + res18 = fma52hi(res18, u7, m[10]); + res18 = fma52lo(res18, u7, m[11]); + res19 = fma52hi(res19, u7, m[11]); + res19 = fma52lo(res19, u7, m[12]); + res20 = fma52hi(res20, u7, m[12]); + res20 = fma52lo(res20, u7, m[13]); + res21 = fma52hi(res21, u7, m[13]); + res21 = fma52lo(res21, u7, m[14]); + res22 = fma52hi(res22, u7, m[14]); + res22 = fma52lo(res22, u7, m[15]); + res23 = fma52hi(res23, u7, m[15]); + res23 = fma52lo(res23, u7, m[16]); + res24 = fma52hi(res24, u7, m[16]); + res24 = fma52lo(res24, u7, m[17]); + res25 = fma52hi(res25, u7, m[17]); + res25 = fma52lo(res25, u7, m[18]); + res26 = fma52hi(res26, u7, m[18]); + res26 = fma52lo(res26, u7, m[19]); + res27 = fma52hi(res27, u7, m[19]); + res08 = fma52lo(res08, u8, m[ 0]); + res09 = fma52hi(res09, u8, m[ 0]); + res09 = fma52lo(res09, u8, m[ 1]); + res10 = fma52hi(res10, u8, m[ 1]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + temp = loadstream64(MulTbl[2+4*iter][16]); + mulb[16] = _mm512_mask_mov_epi64(mulb[16], extract_sel_mask, temp); + temp = loadstream64(MulTbl[2+4*iter][17]); + mulb[17] = _mm512_mask_mov_epi64(mulb[17], extract_sel_mask, temp); + temp = loadstream64(MulTbl[2+4*iter][18]); + mulb[18] = _mm512_mask_mov_epi64(mulb[18], extract_sel_mask, temp); + temp = loadstream64(MulTbl[2+4*iter][19]); + mulb[19] = _mm512_mask_mov_epi64(mulb[19], extract_sel_mask, temp); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res09 = add64(res09, srli64(res08, DIGIT_SIZE)); + U64 u9 = mul52lo(res09, mont_constant); // early multiplication for the reduction of res09 + + res10 = fma52lo(res10, u8, m[ 2]); + res11 = fma52hi(res11, u8, m[ 2]); + res11 = fma52lo(res11, u8, m[ 3]); + res12 = fma52hi(res12, u8, m[ 3]); + res12 = fma52lo(res12, u8, m[ 4]); + res13 = fma52hi(res13, u8, m[ 4]); + res13 = fma52lo(res13, u8, m[ 5]); + res14 = fma52hi(res14, u8, m[ 5]); + res14 = fma52lo(res14, u8, m[ 6]); + res15 = fma52hi(res15, u8, m[ 6]); + res15 = fma52lo(res15, u8, m[ 7]); + res16 = fma52hi(res16, u8, m[ 7]); + res16 = fma52lo(res16, u8, m[ 8]); + res17 = fma52hi(res17, u8, m[ 8]); + res17 = fma52lo(res17, u8, m[ 9]); + res18 = fma52hi(res18, u8, m[ 9]); + res18 = fma52lo(res18, u8, m[10]); + res19 = fma52hi(res19, u8, m[10]); + res19 = fma52lo(res19, u8, m[11]); + res20 = fma52hi(res20, u8, m[11]); + res20 = fma52lo(res20, u8, m[12]); + res21 = fma52hi(res21, u8, m[12]); + res21 = fma52lo(res21, u8, m[13]); + res22 = fma52hi(res22, u8, m[13]); + res22 = fma52lo(res22, u8, m[14]); + res23 = fma52hi(res23, u8, m[14]); + res23 = fma52lo(res23, u8, m[15]); + res24 = fma52hi(res24, u8, m[15]); + res24 = fma52lo(res24, u8, m[16]); + res25 = fma52hi(res25, u8, m[16]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + tempx = loadstream64(MulTblx[2+4*iter][ 0]); + mulbx[ 0] = _mm512_mask_mov_epi64(mulbx[ 0], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[2+4*iter][ 1]); + mulbx[ 1] = _mm512_mask_mov_epi64(mulbx[ 1], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[2+4*iter][ 2]); + mulbx[ 2] = _mm512_mask_mov_epi64(mulbx[ 2], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[2+4*iter][ 3]); + mulbx[ 3] = _mm512_mask_mov_epi64(mulbx[ 3], extract_sel_mask, tempx); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res25 = fma52lo(res25, u8, m[17]); + res26 = fma52hi(res26, u8, m[17]); + res26 = fma52lo(res26, u8, m[18]); + res27 = fma52hi(res27, u8, m[18]); + res27 = fma52lo(res27, u8, m[19]); + res28 = fma52hi(res28, u8, m[19]); + res09 = fma52lo(res09, u9, m[ 0]); + res10 = fma52hi(res10, u9, m[ 0]); + res10 = fma52lo(res10, u9, m[ 1]); + res11 = fma52hi(res11, u9, m[ 1]); + + res10 = add64(res10, srli64(res09, DIGIT_SIZE)); + U64 u10 = mul52lo(res10, mont_constant); // early multiplication for the reduction of res10 + + res11 = fma52lo(res11, u9, m[ 2]); + res12 = fma52hi(res12, u9, m[ 2]); + res12 = fma52lo(res12, u9, m[ 3]); + res13 = fma52hi(res13, u9, m[ 3]); + res13 = fma52lo(res13, u9, m[ 4]); + res14 = fma52hi(res14, u9, m[ 4]); + res14 = fma52lo(res14, u9, m[ 5]); + res15 = fma52hi(res15, u9, m[ 5]); + res15 = fma52lo(res15, u9, m[ 6]); + res16 = fma52hi(res16, u9, m[ 6]); + res16 = fma52lo(res16, u9, m[ 7]); + res17 = fma52hi(res17, u9, m[ 7]); + res17 = fma52lo(res17, u9, m[ 8]); + res18 = fma52hi(res18, u9, m[ 8]); + res18 = fma52lo(res18, u9, m[ 9]); + res19 = fma52hi(res19, u9, m[ 9]); + res19 = fma52lo(res19, u9, m[10]); + res20 = fma52hi(res20, u9, m[10]); + res20 = fma52lo(res20, u9, m[11]); + res21 = fma52hi(res21, u9, m[11]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + tempx = loadstream64(MulTblx[2+4*iter][ 4]); + mulbx[ 4] = _mm512_mask_mov_epi64(mulbx[ 4], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[2+4*iter][ 5]); + mulbx[ 5] = _mm512_mask_mov_epi64(mulbx[ 5], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[2+4*iter][ 6]); + mulbx[ 6] = _mm512_mask_mov_epi64(mulbx[ 6], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[2+4*iter][ 7]); + mulbx[ 7] = _mm512_mask_mov_epi64(mulbx[ 7], extract_sel_mask, tempx); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res21 = fma52lo(res21, u9, m[12]); + res22 = fma52hi(res22, u9, m[12]); + res22 = fma52lo(res22, u9, m[13]); + res23 = fma52hi(res23, u9, m[13]); + res23 = fma52lo(res23, u9, m[14]); + res24 = fma52hi(res24, u9, m[14]); + res24 = fma52lo(res24, u9, m[15]); + res25 = fma52hi(res25, u9, m[15]); + res25 = fma52lo(res25, u9, m[16]); + res26 = fma52hi(res26, u9, m[16]); + res26 = fma52lo(res26, u9, m[17]); + res27 = fma52hi(res27, u9, m[17]); + res27 = fma52lo(res27, u9, m[18]); + res28 = fma52hi(res28, u9, m[18]); + res28 = fma52lo(res28, u9, m[19]); + res29 = fma52hi(res29, u9, m[19]); + res10 = fma52lo(res10, u10, m[ 0]); + res11 = fma52hi(res11, u10, m[ 0]); + res11 = fma52lo(res11, u10, m[ 1]); + res12 = fma52hi(res12, u10, m[ 1]); + + res11 = add64(res11, srli64(res10, DIGIT_SIZE)); + U64 u11 = mul52lo(res11, mont_constant); // early multiplication for the reduction of res11 + + res12 = fma52lo(res12, u10, m[ 2]); + res13 = fma52hi(res13, u10, m[ 2]); + res13 = fma52lo(res13, u10, m[ 3]); + res14 = fma52hi(res14, u10, m[ 3]); + res14 = fma52lo(res14, u10, m[ 4]); + res15 = fma52hi(res15, u10, m[ 4]); + res15 = fma52lo(res15, u10, m[ 5]); + res16 = fma52hi(res16, u10, m[ 5]); + res16 = fma52lo(res16, u10, m[ 6]); + res17 = fma52hi(res17, u10, m[ 6]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + tempx = loadstream64(MulTblx[2+4*iter][ 8]); + mulbx[ 8] = _mm512_mask_mov_epi64(mulbx[ 8], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[2+4*iter][ 9]); + mulbx[ 9] = _mm512_mask_mov_epi64(mulbx[ 9], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[2+4*iter][10]); + mulbx[10] = _mm512_mask_mov_epi64(mulbx[10], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[2+4*iter][11]); + mulbx[11] = _mm512_mask_mov_epi64(mulbx[11], extract_sel_mask, tempx); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res17 = fma52lo(res17, u10, m[ 7]); + res18 = fma52hi(res18, u10, m[ 7]); + res18 = fma52lo(res18, u10, m[ 8]); + res19 = fma52hi(res19, u10, m[ 8]); + res19 = fma52lo(res19, u10, m[ 9]); + res20 = fma52hi(res20, u10, m[ 9]); + res20 = fma52lo(res20, u10, m[10]); + res21 = fma52hi(res21, u10, m[10]); + res21 = fma52lo(res21, u10, m[11]); + res22 = fma52hi(res22, u10, m[11]); + res22 = fma52lo(res22, u10, m[12]); + res23 = fma52hi(res23, u10, m[12]); + res23 = fma52lo(res23, u10, m[13]); + res24 = fma52hi(res24, u10, m[13]); + res24 = fma52lo(res24, u10, m[14]); + res25 = fma52hi(res25, u10, m[14]); + res25 = fma52lo(res25, u10, m[15]); + res26 = fma52hi(res26, u10, m[15]); + res26 = fma52lo(res26, u10, m[16]); + res27 = fma52hi(res27, u10, m[16]); + res27 = fma52lo(res27, u10, m[17]); + res28 = fma52hi(res28, u10, m[17]); + res28 = fma52lo(res28, u10, m[18]); + res29 = fma52hi(res29, u10, m[18]); + res29 = fma52lo(res29, u10, m[19]); + res30 = fma52hi(res30, u10, m[19]); + res11 = fma52lo(res11, u11, m[ 0]); + res12 = fma52hi(res12, u11, m[ 0]); + res12 = fma52lo(res12, u11, m[ 1]); + res13 = fma52hi(res13, u11, m[ 1]); + + res12 = add64(res12, srli64(res11, DIGIT_SIZE)); + U64 u12 = mul52lo(res12, mont_constant); // early multiplication for the reduction of res12 + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + tempx = loadstream64(MulTblx[2+4*iter][12]); + mulbx[12] = _mm512_mask_mov_epi64(mulbx[12], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[2+4*iter][13]); + mulbx[13] = _mm512_mask_mov_epi64(mulbx[13], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[2+4*iter][14]); + mulbx[14] = _mm512_mask_mov_epi64(mulbx[14], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[2+4*iter][15]); + mulbx[15] = _mm512_mask_mov_epi64(mulbx[15], extract_sel_mask, tempx); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res13 = fma52lo(res13, u11, m[ 2]); + res14 = fma52hi(res14, u11, m[ 2]); + res14 = fma52lo(res14, u11, m[ 3]); + res15 = fma52hi(res15, u11, m[ 3]); + res15 = fma52lo(res15, u11, m[ 4]); + res16 = fma52hi(res16, u11, m[ 4]); + res16 = fma52lo(res16, u11, m[ 5]); + res17 = fma52hi(res17, u11, m[ 5]); + res17 = fma52lo(res17, u11, m[ 6]); + res18 = fma52hi(res18, u11, m[ 6]); + res18 = fma52lo(res18, u11, m[ 7]); + res19 = fma52hi(res19, u11, m[ 7]); + res19 = fma52lo(res19, u11, m[ 8]); + res20 = fma52hi(res20, u11, m[ 8]); + res20 = fma52lo(res20, u11, m[ 9]); + res21 = fma52hi(res21, u11, m[ 9]); + res21 = fma52lo(res21, u11, m[10]); + res22 = fma52hi(res22, u11, m[10]); + res22 = fma52lo(res22, u11, m[11]); + res23 = fma52hi(res23, u11, m[11]); + res23 = fma52lo(res23, u11, m[12]); + res24 = fma52hi(res24, u11, m[12]); + res24 = fma52lo(res24, u11, m[13]); + res25 = fma52hi(res25, u11, m[13]); + res25 = fma52lo(res25, u11, m[14]); + res26 = fma52hi(res26, u11, m[14]); + res26 = fma52lo(res26, u11, m[15]); + res27 = fma52hi(res27, u11, m[15]); + res27 = fma52lo(res27, u11, m[16]); + res28 = fma52hi(res28, u11, m[16]); + res28 = fma52lo(res28, u11, m[17]); + res29 = fma52hi(res29, u11, m[17]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + tempx = loadstream64(MulTblx[2+4*iter][16]); + mulbx[16] = _mm512_mask_mov_epi64(mulbx[16], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[2+4*iter][17]); + mulbx[17] = _mm512_mask_mov_epi64(mulbx[17], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[2+4*iter][18]); + mulbx[18] = _mm512_mask_mov_epi64(mulbx[18], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[2+4*iter][19]); + mulbx[19] = _mm512_mask_mov_epi64(mulbx[19], extract_sel_mask, tempx); + extract_sel_mask = extract_sel_mask_next; + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res29 = fma52lo(res29, u11, m[18]); + res30 = fma52hi(res30, u11, m[18]); + res30 = fma52lo(res30, u11, m[19]); + res31 = fma52hi(res31, u11, m[19]); + res12 = fma52lo(res12, u12, m[ 0]); + res13 = fma52hi(res13, u12, m[ 0]); + res13 = fma52lo(res13, u12, m[ 1]); + res14 = fma52hi(res14, u12, m[ 1]); + + res13 = add64(res13, srli64(res12, DIGIT_SIZE)); + U64 u13 = mul52lo(res13, mont_constant); // early multiplication for the reduction of res13 + + res14 = fma52lo(res14, u12, m[ 2]); + res15 = fma52hi(res15, u12, m[ 2]); + res15 = fma52lo(res15, u12, m[ 3]); + res16 = fma52hi(res16, u12, m[ 3]); + res16 = fma52lo(res16, u12, m[ 4]); + res17 = fma52hi(res17, u12, m[ 4]); + res17 = fma52lo(res17, u12, m[ 5]); + res18 = fma52hi(res18, u12, m[ 5]); + res18 = fma52lo(res18, u12, m[ 6]); + res19 = fma52hi(res19, u12, m[ 6]); + res19 = fma52lo(res19, u12, m[ 7]); + res20 = fma52hi(res20, u12, m[ 7]); + res20 = fma52lo(res20, u12, m[ 8]); + res21 = fma52hi(res21, u12, m[ 8]); + res21 = fma52lo(res21, u12, m[ 9]); + res22 = fma52hi(res22, u12, m[ 9]); + res22 = fma52lo(res22, u12, m[10]); + res23 = fma52hi(res23, u12, m[10]); + res23 = fma52lo(res23, u12, m[11]); + res24 = fma52hi(res24, u12, m[11]); + res24 = fma52lo(res24, u12, m[12]); + res25 = fma52hi(res25, u12, m[12]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + idx_curr = set64(4+4*iter); + extract_sel_mask_next = cmpeq64_mask(idx_curr, idx_target); + temp = loadstream64(MulTbl[3+4*iter][ 0]); + mulb[ 0] = _mm512_mask_mov_epi64(mulb[ 0], extract_sel_mask, temp); + temp = loadstream64(MulTbl[3+4*iter][ 1]); + mulb[ 1] = _mm512_mask_mov_epi64(mulb[ 1], extract_sel_mask, temp); + temp = loadstream64(MulTbl[3+4*iter][ 2]); + mulb[ 2] = _mm512_mask_mov_epi64(mulb[ 2], extract_sel_mask, temp); + temp = loadstream64(MulTbl[3+4*iter][ 3]); + mulb[ 3] = _mm512_mask_mov_epi64(mulb[ 3], extract_sel_mask, temp); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res25 = fma52lo(res25, u12, m[13]); + res26 = fma52hi(res26, u12, m[13]); + res26 = fma52lo(res26, u12, m[14]); + res27 = fma52hi(res27, u12, m[14]); + res27 = fma52lo(res27, u12, m[15]); + res28 = fma52hi(res28, u12, m[15]); + res28 = fma52lo(res28, u12, m[16]); + res29 = fma52hi(res29, u12, m[16]); + res29 = fma52lo(res29, u12, m[17]); + res30 = fma52hi(res30, u12, m[17]); + res30 = fma52lo(res30, u12, m[18]); + res31 = fma52hi(res31, u12, m[18]); + res31 = fma52lo(res31, u12, m[19]); + res32 = fma52hi(res32, u12, m[19]); + res13 = fma52lo(res13, u13, m[ 0]); + res14 = fma52hi(res14, u13, m[ 0]); + res14 = fma52lo(res14, u13, m[ 1]); + res15 = fma52hi(res15, u13, m[ 1]); + + res14 = add64(res14, srli64(res13, DIGIT_SIZE)); + U64 u14 = mul52lo(res14, mont_constant); // early multiplication for the reduction of res14 + + res15 = fma52lo(res15, u13, m[ 2]); + res16 = fma52hi(res16, u13, m[ 2]); + res16 = fma52lo(res16, u13, m[ 3]); + res17 = fma52hi(res17, u13, m[ 3]); + res17 = fma52lo(res17, u13, m[ 4]); + res18 = fma52hi(res18, u13, m[ 4]); + res18 = fma52lo(res18, u13, m[ 5]); + res19 = fma52hi(res19, u13, m[ 5]); + res19 = fma52lo(res19, u13, m[ 6]); + res20 = fma52hi(res20, u13, m[ 6]); + res20 = fma52lo(res20, u13, m[ 7]); + res21 = fma52hi(res21, u13, m[ 7]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + temp = loadstream64(MulTbl[3+4*iter][ 4]); + mulb[ 4] = _mm512_mask_mov_epi64(mulb[ 4], extract_sel_mask, temp); + temp = loadstream64(MulTbl[3+4*iter][ 5]); + mulb[ 5] = _mm512_mask_mov_epi64(mulb[ 5], extract_sel_mask, temp); + temp = loadstream64(MulTbl[3+4*iter][ 6]); + mulb[ 6] = _mm512_mask_mov_epi64(mulb[ 6], extract_sel_mask, temp); + temp = loadstream64(MulTbl[3+4*iter][ 7]); + mulb[ 7] = _mm512_mask_mov_epi64(mulb[ 7], extract_sel_mask, temp); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res21 = fma52lo(res21, u13, m[ 8]); + res22 = fma52hi(res22, u13, m[ 8]); + res22 = fma52lo(res22, u13, m[ 9]); + res23 = fma52hi(res23, u13, m[ 9]); + res23 = fma52lo(res23, u13, m[10]); + res24 = fma52hi(res24, u13, m[10]); + res24 = fma52lo(res24, u13, m[11]); + res25 = fma52hi(res25, u13, m[11]); + res25 = fma52lo(res25, u13, m[12]); + res26 = fma52hi(res26, u13, m[12]); + res26 = fma52lo(res26, u13, m[13]); + res27 = fma52hi(res27, u13, m[13]); + res27 = fma52lo(res27, u13, m[14]); + res28 = fma52hi(res28, u13, m[14]); + res28 = fma52lo(res28, u13, m[15]); + res29 = fma52hi(res29, u13, m[15]); + res29 = fma52lo(res29, u13, m[16]); + res30 = fma52hi(res30, u13, m[16]); + res30 = fma52lo(res30, u13, m[17]); + res31 = fma52hi(res31, u13, m[17]); + res31 = fma52lo(res31, u13, m[18]); + res32 = fma52hi(res32, u13, m[18]); + res32 = fma52lo(res32, u13, m[19]); + res33 = fma52hi(res33, u13, m[19]); + res14 = fma52lo(res14, u14, m[ 0]); + res15 = fma52hi(res15, u14, m[ 0]); + res15 = fma52lo(res15, u14, m[ 1]); + res16 = fma52hi(res16, u14, m[ 1]); + + res15 = add64(res15, srli64(res14, DIGIT_SIZE)); + U64 u15 = mul52lo(res15, mont_constant); // early multiplication for the reduction of res15 + + res16 = fma52lo(res16, u14, m[ 2]); + res17 = fma52hi(res17, u14, m[ 2]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + temp = loadstream64(MulTbl[3+4*iter][ 8]); + mulb[ 8] = _mm512_mask_mov_epi64(mulb[ 8], extract_sel_mask, temp); + temp = loadstream64(MulTbl[3+4*iter][ 9]); + mulb[ 9] = _mm512_mask_mov_epi64(mulb[ 9], extract_sel_mask, temp); + temp = loadstream64(MulTbl[3+4*iter][10]); + mulb[10] = _mm512_mask_mov_epi64(mulb[10], extract_sel_mask, temp); + temp = loadstream64(MulTbl[3+4*iter][11]); + mulb[11] = _mm512_mask_mov_epi64(mulb[11], extract_sel_mask, temp); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res17 = fma52lo(res17, u14, m[ 3]); + res18 = fma52hi(res18, u14, m[ 3]); + res18 = fma52lo(res18, u14, m[ 4]); + res19 = fma52hi(res19, u14, m[ 4]); + res19 = fma52lo(res19, u14, m[ 5]); + res20 = fma52hi(res20, u14, m[ 5]); + res20 = fma52lo(res20, u14, m[ 6]); + res21 = fma52hi(res21, u14, m[ 6]); + res21 = fma52lo(res21, u14, m[ 7]); + res22 = fma52hi(res22, u14, m[ 7]); + res22 = fma52lo(res22, u14, m[ 8]); + res23 = fma52hi(res23, u14, m[ 8]); + res23 = fma52lo(res23, u14, m[ 9]); + res24 = fma52hi(res24, u14, m[ 9]); + res24 = fma52lo(res24, u14, m[10]); + res25 = fma52hi(res25, u14, m[10]); + res25 = fma52lo(res25, u14, m[11]); + res26 = fma52hi(res26, u14, m[11]); + res26 = fma52lo(res26, u14, m[12]); + res27 = fma52hi(res27, u14, m[12]); + res27 = fma52lo(res27, u14, m[13]); + res28 = fma52hi(res28, u14, m[13]); + res28 = fma52lo(res28, u14, m[14]); + res29 = fma52hi(res29, u14, m[14]); + res29 = fma52lo(res29, u14, m[15]); + res30 = fma52hi(res30, u14, m[15]); + res30 = fma52lo(res30, u14, m[16]); + res31 = fma52hi(res31, u14, m[16]); + res31 = fma52lo(res31, u14, m[17]); + res32 = fma52hi(res32, u14, m[17]); + res32 = fma52lo(res32, u14, m[18]); + res33 = fma52hi(res33, u14, m[18]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + temp = loadstream64(MulTbl[3+4*iter][12]); + mulb[12] = _mm512_mask_mov_epi64(mulb[12], extract_sel_mask, temp); + temp = loadstream64(MulTbl[3+4*iter][13]); + mulb[13] = _mm512_mask_mov_epi64(mulb[13], extract_sel_mask, temp); + temp = loadstream64(MulTbl[3+4*iter][14]); + mulb[14] = _mm512_mask_mov_epi64(mulb[14], extract_sel_mask, temp); + temp = loadstream64(MulTbl[3+4*iter][15]); + mulb[15] = _mm512_mask_mov_epi64(mulb[15], extract_sel_mask, temp); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res33 = fma52lo(res33, u14, m[19]); + res34 = fma52hi(res34, u14, m[19]); + res15 = fma52lo(res15, u15, m[ 0]); + res16 = fma52hi(res16, u15, m[ 0]); + res16 = fma52lo(res16, u15, m[ 1]); + res17 = fma52hi(res17, u15, m[ 1]); + + res16 = add64(res16, srli64(res15, DIGIT_SIZE)); + U64 u16 = mul52lo(res16, mont_constant); // early multiplication for the reduction of res16 + + res17 = fma52lo(res17, u15, m[ 2]); + res18 = fma52hi(res18, u15, m[ 2]); + res18 = fma52lo(res18, u15, m[ 3]); + res19 = fma52hi(res19, u15, m[ 3]); + res19 = fma52lo(res19, u15, m[ 4]); + res20 = fma52hi(res20, u15, m[ 4]); + res20 = fma52lo(res20, u15, m[ 5]); + res21 = fma52hi(res21, u15, m[ 5]); + res21 = fma52lo(res21, u15, m[ 6]); + res22 = fma52hi(res22, u15, m[ 6]); + res22 = fma52lo(res22, u15, m[ 7]); + res23 = fma52hi(res23, u15, m[ 7]); + res23 = fma52lo(res23, u15, m[ 8]); + res24 = fma52hi(res24, u15, m[ 8]); + res24 = fma52lo(res24, u15, m[ 9]); + res25 = fma52hi(res25, u15, m[ 9]); + res25 = fma52lo(res25, u15, m[10]); + res26 = fma52hi(res26, u15, m[10]); + res26 = fma52lo(res26, u15, m[11]); + res27 = fma52hi(res27, u15, m[11]); + res27 = fma52lo(res27, u15, m[12]); + res28 = fma52hi(res28, u15, m[12]); + res28 = fma52lo(res28, u15, m[13]); + res29 = fma52hi(res29, u15, m[13]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + temp = loadstream64(MulTbl[3+4*iter][16]); + mulb[16] = _mm512_mask_mov_epi64(mulb[16], extract_sel_mask, temp); + temp = loadstream64(MulTbl[3+4*iter][17]); + mulb[17] = _mm512_mask_mov_epi64(mulb[17], extract_sel_mask, temp); + temp = loadstream64(MulTbl[3+4*iter][18]); + mulb[18] = _mm512_mask_mov_epi64(mulb[18], extract_sel_mask, temp); + temp = loadstream64(MulTbl[3+4*iter][19]); + mulb[19] = _mm512_mask_mov_epi64(mulb[19], extract_sel_mask, temp); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res29 = fma52lo(res29, u15, m[14]); + res30 = fma52hi(res30, u15, m[14]); + res30 = fma52lo(res30, u15, m[15]); + res31 = fma52hi(res31, u15, m[15]); + res31 = fma52lo(res31, u15, m[16]); + res32 = fma52hi(res32, u15, m[16]); + res32 = fma52lo(res32, u15, m[17]); + res33 = fma52hi(res33, u15, m[17]); + res33 = fma52lo(res33, u15, m[18]); + res34 = fma52hi(res34, u15, m[18]); + res34 = fma52lo(res34, u15, m[19]); + res35 = fma52hi(res35, u15, m[19]); + res16 = fma52lo(res16, u16, m[ 0]); + res17 = fma52hi(res17, u16, m[ 0]); + res17 = fma52lo(res17, u16, m[ 1]); + res18 = fma52hi(res18, u16, m[ 1]); + + res17 = add64(res17, srli64(res16, DIGIT_SIZE)); + U64 u17 = mul52lo(res17, mont_constant); // early multiplication for the reduction of res17 + + res18 = fma52lo(res18, u16, m[ 2]); + res19 = fma52hi(res19, u16, m[ 2]); + res19 = fma52lo(res19, u16, m[ 3]); + res20 = fma52hi(res20, u16, m[ 3]); + res20 = fma52lo(res20, u16, m[ 4]); + res21 = fma52hi(res21, u16, m[ 4]); + res21 = fma52lo(res21, u16, m[ 5]); + res22 = fma52hi(res22, u16, m[ 5]); + res22 = fma52lo(res22, u16, m[ 6]); + res23 = fma52hi(res23, u16, m[ 6]); + res23 = fma52lo(res23, u16, m[ 7]); + res24 = fma52hi(res24, u16, m[ 7]); + res24 = fma52lo(res24, u16, m[ 8]); + res25 = fma52hi(res25, u16, m[ 8]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + tempx = loadstream64(MulTblx[3+4*iter][ 0]); + mulbx[ 0] = _mm512_mask_mov_epi64(mulbx[ 0], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[3+4*iter][ 1]); + mulbx[ 1] = _mm512_mask_mov_epi64(mulbx[ 1], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[3+4*iter][ 2]); + mulbx[ 2] = _mm512_mask_mov_epi64(mulbx[ 2], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[3+4*iter][ 3]); + mulbx[ 3] = _mm512_mask_mov_epi64(mulbx[ 3], extract_sel_mask, tempx); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res25 = fma52lo(res25, u16, m[ 9]); + res26 = fma52hi(res26, u16, m[ 9]); + res26 = fma52lo(res26, u16, m[10]); + res27 = fma52hi(res27, u16, m[10]); + res27 = fma52lo(res27, u16, m[11]); + res28 = fma52hi(res28, u16, m[11]); + res28 = fma52lo(res28, u16, m[12]); + res29 = fma52hi(res29, u16, m[12]); + res29 = fma52lo(res29, u16, m[13]); + res30 = fma52hi(res30, u16, m[13]); + res30 = fma52lo(res30, u16, m[14]); + res31 = fma52hi(res31, u16, m[14]); + res31 = fma52lo(res31, u16, m[15]); + res32 = fma52hi(res32, u16, m[15]); + res32 = fma52lo(res32, u16, m[16]); + res33 = fma52hi(res33, u16, m[16]); + res33 = fma52lo(res33, u16, m[17]); + res34 = fma52hi(res34, u16, m[17]); + res34 = fma52lo(res34, u16, m[18]); + res35 = fma52hi(res35, u16, m[18]); + res35 = fma52lo(res35, u16, m[19]); + res36 = fma52hi(res36, u16, m[19]); + res17 = fma52lo(res17, u17, m[ 0]); + res18 = fma52hi(res18, u17, m[ 0]); + res18 = fma52lo(res18, u17, m[ 1]); + res19 = fma52hi(res19, u17, m[ 1]); + + res18 = add64(res18, srli64(res17, DIGIT_SIZE)); + U64 u18 = mul52lo(res18, mont_constant); // early multiplication for the reduction of res18 + + res19 = fma52lo(res19, u17, m[ 2]); + res20 = fma52hi(res20, u17, m[ 2]); + res20 = fma52lo(res20, u17, m[ 3]); + res21 = fma52hi(res21, u17, m[ 3]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + tempx = loadstream64(MulTblx[3+4*iter][ 4]); + mulbx[ 4] = _mm512_mask_mov_epi64(mulbx[ 4], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[3+4*iter][ 5]); + mulbx[ 5] = _mm512_mask_mov_epi64(mulbx[ 5], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[3+4*iter][ 6]); + mulbx[ 6] = _mm512_mask_mov_epi64(mulbx[ 6], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[3+4*iter][ 7]); + mulbx[ 7] = _mm512_mask_mov_epi64(mulbx[ 7], extract_sel_mask, tempx); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res21 = fma52lo(res21, u17, m[ 4]); + res22 = fma52hi(res22, u17, m[ 4]); + res22 = fma52lo(res22, u17, m[ 5]); + res23 = fma52hi(res23, u17, m[ 5]); + res23 = fma52lo(res23, u17, m[ 6]); + res24 = fma52hi(res24, u17, m[ 6]); + res24 = fma52lo(res24, u17, m[ 7]); + res25 = fma52hi(res25, u17, m[ 7]); + res25 = fma52lo(res25, u17, m[ 8]); + res26 = fma52hi(res26, u17, m[ 8]); + res26 = fma52lo(res26, u17, m[ 9]); + res27 = fma52hi(res27, u17, m[ 9]); + res27 = fma52lo(res27, u17, m[10]); + res28 = fma52hi(res28, u17, m[10]); + res28 = fma52lo(res28, u17, m[11]); + res29 = fma52hi(res29, u17, m[11]); + res29 = fma52lo(res29, u17, m[12]); + res30 = fma52hi(res30, u17, m[12]); + res30 = fma52lo(res30, u17, m[13]); + res31 = fma52hi(res31, u17, m[13]); + res31 = fma52lo(res31, u17, m[14]); + res32 = fma52hi(res32, u17, m[14]); + res32 = fma52lo(res32, u17, m[15]); + res33 = fma52hi(res33, u17, m[15]); + res33 = fma52lo(res33, u17, m[16]); + res34 = fma52hi(res34, u17, m[16]); + res34 = fma52lo(res34, u17, m[17]); + res35 = fma52hi(res35, u17, m[17]); + res35 = fma52lo(res35, u17, m[18]); + res36 = fma52hi(res36, u17, m[18]); + res36 = fma52lo(res36, u17, m[19]); + res37 = fma52hi(res37, u17, m[19]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + tempx = loadstream64(MulTblx[3+4*iter][ 8]); + mulbx[ 8] = _mm512_mask_mov_epi64(mulbx[ 8], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[3+4*iter][ 9]); + mulbx[ 9] = _mm512_mask_mov_epi64(mulbx[ 9], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[3+4*iter][10]); + mulbx[10] = _mm512_mask_mov_epi64(mulbx[10], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[3+4*iter][11]); + mulbx[11] = _mm512_mask_mov_epi64(mulbx[11], extract_sel_mask, tempx); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res18 = fma52lo(res18, u18, m[ 0]); + res19 = fma52hi(res19, u18, m[ 0]); + res19 = fma52lo(res19, u18, m[ 1]); + res20 = fma52hi(res20, u18, m[ 1]); + + res19 = add64(res19, srli64(res18, DIGIT_SIZE)); + U64 u19 = mul52lo(res19, mont_constant); // early multiplication for the reduction of res19 + + res20 = fma52lo(res20, u18, m[ 2]); + res21 = fma52hi(res21, u18, m[ 2]); + res21 = fma52lo(res21, u18, m[ 3]); + res22 = fma52hi(res22, u18, m[ 3]); + res22 = fma52lo(res22, u18, m[ 4]); + res23 = fma52hi(res23, u18, m[ 4]); + res23 = fma52lo(res23, u18, m[ 5]); + res24 = fma52hi(res24, u18, m[ 5]); + res24 = fma52lo(res24, u18, m[ 6]); + res25 = fma52hi(res25, u18, m[ 6]); + res25 = fma52lo(res25, u18, m[ 7]); + res26 = fma52hi(res26, u18, m[ 7]); + res26 = fma52lo(res26, u18, m[ 8]); + res27 = fma52hi(res27, u18, m[ 8]); + res27 = fma52lo(res27, u18, m[ 9]); + res28 = fma52hi(res28, u18, m[ 9]); + res28 = fma52lo(res28, u18, m[10]); + res29 = fma52hi(res29, u18, m[10]); + res29 = fma52lo(res29, u18, m[11]); + res30 = fma52hi(res30, u18, m[11]); + res30 = fma52lo(res30, u18, m[12]); + res31 = fma52hi(res31, u18, m[12]); + res31 = fma52lo(res31, u18, m[13]); + res32 = fma52hi(res32, u18, m[13]); + res32 = fma52lo(res32, u18, m[14]); + res33 = fma52hi(res33, u18, m[14]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + tempx = loadstream64(MulTblx[3+4*iter][12]); + mulbx[12] = _mm512_mask_mov_epi64(mulbx[12], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[3+4*iter][13]); + mulbx[13] = _mm512_mask_mov_epi64(mulbx[13], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[3+4*iter][14]); + mulbx[14] = _mm512_mask_mov_epi64(mulbx[14], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[3+4*iter][15]); + mulbx[15] = _mm512_mask_mov_epi64(mulbx[15], extract_sel_mask, tempx); + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res33 = fma52lo(res33, u18, m[15]); + res34 = fma52hi(res34, u18, m[15]); + res34 = fma52lo(res34, u18, m[16]); + res35 = fma52hi(res35, u18, m[16]); + res35 = fma52lo(res35, u18, m[17]); + res36 = fma52hi(res36, u18, m[17]); + res36 = fma52lo(res36, u18, m[18]); + res37 = fma52hi(res37, u18, m[18]); + res37 = fma52lo(res37, u18, m[19]); + res38 = fma52hi(res38, u18, m[19]); + res19 = fma52lo(res19, u19, m[ 0]); + res20 = fma52hi(res20, u19, m[ 0]); + res20 = fma52lo(res20, u19, m[ 1]); + res21 = fma52hi(res21, u19, m[ 1]); + res20 = add64(res20, srli64(res19, DIGIT_SIZE)); + res21 = fma52lo(res21, u19, m[ 2]); + res22 = fma52hi(res22, u19, m[ 2]); + res22 = fma52lo(res22, u19, m[ 3]); + res23 = fma52hi(res23, u19, m[ 3]); + res23 = fma52lo(res23, u19, m[ 4]); + res24 = fma52hi(res24, u19, m[ 4]); + res24 = fma52lo(res24, u19, m[ 5]); + res25 = fma52hi(res25, u19, m[ 5]); + res25 = fma52lo(res25, u19, m[ 6]); + res26 = fma52hi(res26, u19, m[ 6]); + res26 = fma52lo(res26, u19, m[ 7]); + res27 = fma52hi(res27, u19, m[ 7]); + res27 = fma52lo(res27, u19, m[ 8]); + res28 = fma52hi(res28, u19, m[ 8]); + res28 = fma52lo(res28, u19, m[ 9]); + res29 = fma52hi(res29, u19, m[ 9]); + res29 = fma52lo(res29, u19, m[10]); + //*******END SQUARING CODE SEGMENT**************// + + //*******BEGIN EXTRACTION CODE SEGMENT****************************// + tempx = loadstream64(MulTblx[3+4*iter][16]); + mulbx[16] = _mm512_mask_mov_epi64(mulbx[16], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[3+4*iter][17]); + mulbx[17] = _mm512_mask_mov_epi64(mulbx[17], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[3+4*iter][18]); + mulbx[18] = _mm512_mask_mov_epi64(mulbx[18], extract_sel_mask, tempx); + tempx = loadstream64(MulTblx[3+4*iter][19]); + mulbx[19] = _mm512_mask_mov_epi64(mulbx[19], extract_sel_mask, tempx); + extract_sel_mask = extract_sel_mask_next; + //*******END EXTRACTION CODE SEGMENT******************************// + + //*******BEGIN SQUARING CODE SEGMENT************// + res30 = fma52hi(res30, u19, m[10]); + res30 = fma52lo(res30, u19, m[11]); + res31 = fma52hi(res31, u19, m[11]); + res31 = fma52lo(res31, u19, m[12]); + res32 = fma52hi(res32, u19, m[12]); + res32 = fma52lo(res32, u19, m[13]); + res33 = fma52hi(res33, u19, m[13]); + res33 = fma52lo(res33, u19, m[14]); + res34 = fma52hi(res34, u19, m[14]); + res34 = fma52lo(res34, u19, m[15]); + res35 = fma52hi(res35, u19, m[15]); + res35 = fma52lo(res35, u19, m[16]); + res36 = fma52hi(res36, u19, m[16]); + res36 = fma52lo(res36, u19, m[17]); + res37 = fma52hi(res37, u19, m[17]); + res37 = fma52lo(res37, u19, m[18]); + res38 = fma52hi(res38, u19, m[18]); + res38 = fma52lo(res38, u19, m[19]); + res39 = fma52hi(res39, u19, m[19]); + + // Normalization + // We do not need to zero out the top bits + // They will not be used by the fma53lo/hi instructios + r[ 0] = res20; + res21 = add64(res21, srli64(res20, DIGIT_SIZE)); + r[ 1] = res21; + res22 = add64(res22, srli64(res21, DIGIT_SIZE)); + r[ 2] = res22; + res23 = add64(res23, srli64(res22, DIGIT_SIZE)); + r[ 3] = res23; + res24 = add64(res24, srli64(res23, DIGIT_SIZE)); + r[ 4] = res24; + res25 = add64(res25, srli64(res24, DIGIT_SIZE)); + r[ 5] = res25; + res26 = add64(res26, srli64(res25, DIGIT_SIZE)); + r[ 6] = res26; + res27 = add64(res27, srli64(res26, DIGIT_SIZE)); + r[ 7] = res27; + res28 = add64(res28, srli64(res27, DIGIT_SIZE)); + r[ 8] = res28; + res29 = add64(res29, srli64(res28, DIGIT_SIZE)); + r[ 9] = res29; + res30 = add64(res30, srli64(res29, DIGIT_SIZE)); + r[10] = res30; + res31 = add64(res31, srli64(res30, DIGIT_SIZE)); + r[11] = res31; + res32 = add64(res32, srli64(res31, DIGIT_SIZE)); + r[12] = res32; + res33 = add64(res33, srli64(res32, DIGIT_SIZE)); + r[13] = res33; + res34 = add64(res34, srli64(res33, DIGIT_SIZE)); + r[14] = res34; + res35 = add64(res35, srli64(res34, DIGIT_SIZE)); + r[15] = res35; + res36 = add64(res36, srli64(res35, DIGIT_SIZE)); + r[16] = res36; + res37 = add64(res37, srli64(res36, DIGIT_SIZE)); + r[17] = res37; + res38 = add64(res38, srli64(res37, DIGIT_SIZE)); + r[18] = res38; + res39 = add64(res39, srli64(res38, DIGIT_SIZE)); + r[19] = res39; + a = (U64*) out_mb; + //*******END SQUARING CODE SEGMENT**************// + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ahmm52x20_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ahmm52x20_mb8.c new file mode 100644 index 0000000..c221f94 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ahmm52x20_mb8.c @@ -0,0 +1,291 @@ +/******************************************************************************* + * Copyright (C) 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include + +/* +Almost Half Montgomery Multiplication (AHMM) + +Output: + out_mb : c = A*B*(2^(-20*52)) mod M +Inputs: + inpA_mb : A + inpB_mb : B + inpBx_mb : K = B*(2^(-10*52)) mod M + inpM_mb : M + k0_mb : mont_constant = (-M^(-1) mod 2^(52)) + + AL=A[ 9: 0] + AH=A[19:10] + + C=0 + for i from 0 to 9: + C=C+AL[i]*K+AH[i]*B + T=C[0]*mu // discard T[1] + C=C+T[0]*M // C[0] is zero + C=C>>52 // at each step of the for loop, divide the result by 2^52 + return C +*/ + +void ifma_ahmm52x20_mb8( + int64u *out_mb, const int64u *inpA_mb, const int64u* inpB_mb, const int64u* inpBx_mb, + const int64u* inpM_mb, const int64u* k0_mb) { + + // Temporary Registers to hold 20 52-bit intermediate results + U64 res00, res01, res02, res03, res04, res05, res06, res07, res08, res09, + res10, res11, res12, res13, res14, res15, res16, res17, res18, res19, + res20; + + // Precomputed Montgomery constant (-M^(-1) mod 2^(52)) + const U64 mont_constant = loadu64(k0_mb); + + // C = 0 + res00 = res01 = res02 = res03 = res04 = res05 = res06 = res07 = res08 = res09 = + res10 = res11 = res12 = res13 = res14 = res15 = res16 = res17 = res18 = res19 = + res20 = get_zero64(); + + U64* A = (U64*) inpA_mb; + U64* M = (U64*) inpM_mb; + U64* mulb = (U64*) inpB_mb; + U64* mulbx = (U64*) inpBx_mb; + + for (int itr = 0; itr < 10; itr++) { + //****************************************************** + // C=C+AL[i]*K+AH[i]*B + //****************************************************** + + // C=C+AH[i]*B + // load AH[i] + const U64 AHi = A[itr+10]; // AH=A[19:10] + res00 = fma52lo(res00, AHi, mulb[ 0]); + res01 = fma52hi(res01, AHi, mulb[ 0]); + res01 = fma52lo(res01, AHi, mulb[ 1]); + res02 = fma52hi(res02, AHi, mulb[ 1]); + res02 = fma52lo(res02, AHi, mulb[ 2]); + res03 = fma52hi(res03, AHi, mulb[ 2]); + res03 = fma52lo(res03, AHi, mulb[ 3]); + res04 = fma52hi(res04, AHi, mulb[ 3]); + res04 = fma52lo(res04, AHi, mulb[ 4]); + res05 = fma52hi(res05, AHi, mulb[ 4]); + res05 = fma52lo(res05, AHi, mulb[ 5]); + res06 = fma52hi(res06, AHi, mulb[ 5]); + res06 = fma52lo(res06, AHi, mulb[ 6]); + res07 = fma52hi(res07, AHi, mulb[ 6]); + res07 = fma52lo(res07, AHi, mulb[ 7]); + res08 = fma52hi(res08, AHi, mulb[ 7]); + res08 = fma52lo(res08, AHi, mulb[ 8]); + res09 = fma52hi(res09, AHi, mulb[ 8]); + res09 = fma52lo(res09, AHi, mulb[ 9]); + res10 = fma52hi(res10, AHi, mulb[ 9]); + res10 = fma52lo(res10, AHi, mulb[10]); + res11 = fma52hi(res11, AHi, mulb[10]); + res11 = fma52lo(res11, AHi, mulb[11]); + res12 = fma52hi(res12, AHi, mulb[11]); + res12 = fma52lo(res12, AHi, mulb[12]); + res13 = fma52hi(res13, AHi, mulb[12]); + res13 = fma52lo(res13, AHi, mulb[13]); + res14 = fma52hi(res14, AHi, mulb[13]); + res14 = fma52lo(res14, AHi, mulb[14]); + res15 = fma52hi(res15, AHi, mulb[14]); + res15 = fma52lo(res15, AHi, mulb[15]); + res16 = fma52hi(res16, AHi, mulb[15]); + res16 = fma52lo(res16, AHi, mulb[16]); + res17 = fma52hi(res17, AHi, mulb[16]); + res17 = fma52lo(res17, AHi, mulb[17]); + res18 = fma52hi(res18, AHi, mulb[17]); + res18 = fma52lo(res18, AHi, mulb[18]); + res19 = fma52hi(res19, AHi, mulb[18]); + res19 = fma52lo(res19, AHi, mulb[19]); + res20 = fma52hi(get_zero64(), AHi, mulb[19]); + + // C=C+AL[i]*K (low part) + // load AL[i] + const U64 ALi = A[itr]; // AL=A[ 9: 0] + res00 = fma52lo(res00, ALi, mulbx[ 0]); + res01 = fma52hi(res01, ALi, mulbx[ 0]); + res01 = fma52lo(res01, ALi, mulbx[ 1]); + res02 = fma52hi(res02, ALi, mulbx[ 1]); + res02 = fma52lo(res02, ALi, mulbx[ 2]); + res03 = fma52hi(res03, ALi, mulbx[ 2]); + res03 = fma52lo(res03, ALi, mulbx[ 3]); + res04 = fma52hi(res04, ALi, mulbx[ 3]); + res04 = fma52lo(res04, ALi, mulbx[ 4]); + res05 = fma52hi(res05, ALi, mulbx[ 4]); + res05 = fma52lo(res05, ALi, mulbx[ 5]); + res06 = fma52hi(res06, ALi, mulbx[ 5]); + res06 = fma52lo(res06, ALi, mulbx[ 6]); + res07 = fma52hi(res07, ALi, mulbx[ 6]); + res07 = fma52lo(res07, ALi, mulbx[ 7]); + res08 = fma52hi(res08, ALi, mulbx[ 7]); + res08 = fma52lo(res08, ALi, mulbx[ 8]); + res09 = fma52hi(res09, ALi, mulbx[ 8]); + res09 = fma52lo(res09, ALi, mulbx[ 9]); + res10 = fma52hi(res10, ALi, mulbx[ 9]); + res10 = fma52lo(res10, ALi, mulbx[10]); + res11 = fma52hi(res11, ALi, mulbx[10]); + res11 = fma52lo(res11, ALi, mulbx[11]); + res12 = fma52hi(res12, ALi, mulbx[11]); + res12 = fma52lo(res12, ALi, mulbx[12]); + res13 = fma52hi(res13, ALi, mulbx[12]); + res13 = fma52lo(res13, ALi, mulbx[13]); + res14 = fma52hi(res14, ALi, mulbx[13]); + res14 = fma52lo(res14, ALi, mulbx[14]); + res15 = fma52hi(res15, ALi, mulbx[14]); + res15 = fma52lo(res15, ALi, mulbx[15]); + res16 = fma52hi(res16, ALi, mulbx[15]); + res16 = fma52lo(res16, ALi, mulbx[16]); + res17 = fma52hi(res17, ALi, mulbx[16]); + res17 = fma52lo(res17, ALi, mulbx[17]); + res18 = fma52hi(res18, ALi, mulbx[17]); + res18 = fma52lo(res18, ALi, mulbx[18]); + res19 = fma52hi(res19, ALi, mulbx[18]); + res19 = fma52lo(res19, ALi, mulbx[19]); + res20 = fma52hi(res20, ALi, mulbx[19]); + // done: C=C+AL[i]*K+AH[i]*B + //****************************************************** + + // T=C[0]*mu + U64 T = fma52lo(get_zero64(), res00, mont_constant); + + // C=C+T[0]*M (low part) + res00 = fma52lo(res00, T, M[ 0]); + + // low 52 (DIGIT_SIZE) bits of res00 are 0 + // high 12 bits are accumulated to res01 (same bit-weight) + res00 = srli64(res00, DIGIT_SIZE); + res01 = add64(res01, res00); + + res01 = fma52lo(res01, T, M[ 1]); + res00 = fma52hi(res01, T, M[ 0]); + res02 = fma52lo(res02, T, M[ 2]); + res01 = fma52hi(res02, T, M[ 1]); + res03 = fma52lo(res03, T, M[ 3]); + res02 = fma52hi(res03, T, M[ 2]); + res04 = fma52lo(res04, T, M[ 4]); + res03 = fma52hi(res04, T, M[ 3]); + res05 = fma52lo(res05, T, M[ 5]); + res04 = fma52hi(res05, T, M[ 4]); + res06 = fma52lo(res06, T, M[ 6]); + res05 = fma52hi(res06, T, M[ 5]); + res07 = fma52lo(res07, T, M[ 7]); + res06 = fma52hi(res07, T, M[ 6]); + res08 = fma52lo(res08, T, M[ 8]); + res07 = fma52hi(res08, T, M[ 7]); + res09 = fma52lo(res09, T, M[ 9]); + res08 = fma52hi(res09, T, M[ 8]); + res10 = fma52lo(res10, T, M[10]); + res09 = fma52hi(res10, T, M[ 9]); + res11 = fma52lo(res11, T, M[11]); + res10 = fma52hi(res11, T, M[10]); + res12 = fma52lo(res12, T, M[12]); + res11 = fma52hi(res12, T, M[11]); + res13 = fma52lo(res13, T, M[13]); + res12 = fma52hi(res13, T, M[12]); + res14 = fma52lo(res14, T, M[14]); + res13 = fma52hi(res14, T, M[13]); + res15 = fma52lo(res15, T, M[15]); + res14 = fma52hi(res15, T, M[14]); + res16 = fma52lo(res16, T, M[16]); + res15 = fma52hi(res16, T, M[15]); + res17 = fma52lo(res17, T, M[17]); + res16 = fma52hi(res17, T, M[16]); + res18 = fma52lo(res18, T, M[18]); + res17 = fma52hi(res18, T, M[17]); + res19 = fma52lo(res19, T, M[19]); + res18 = fma52hi(res19, T, M[18]); + res19 = fma52hi(res20, T, M[19]); + } + + // Normalization + U64 High_Extra_Bits = srli64(res00, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 0, res00); + + res01 = add64(res01, High_Extra_Bits); + High_Extra_Bits = srli64(res01, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 1, res01); + + res02 = add64(res02, High_Extra_Bits); + High_Extra_Bits = srli64(res02, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 2, res02); + + res03 = add64(res03, High_Extra_Bits); + High_Extra_Bits = srli64(res03, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 3, res03); + + res04 = add64(res04, High_Extra_Bits); + High_Extra_Bits = srli64(res04, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 4, res04); + + res05 = add64(res05, High_Extra_Bits); + High_Extra_Bits = srli64(res05, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 5, res05); + + res06 = add64(res06, High_Extra_Bits); + High_Extra_Bits = srli64(res06, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 6, res06); + + res07 = add64(res07, High_Extra_Bits); + High_Extra_Bits = srli64(res07, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 7, res07); + + res08 = add64(res08, High_Extra_Bits); + High_Extra_Bits = srli64(res08, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 8, res08); + + res09 = add64(res09, High_Extra_Bits); + High_Extra_Bits = srli64(res09, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 9, res09); + + res10 = add64(res10, High_Extra_Bits); + High_Extra_Bits = srli64(res10, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 10, res10); + + res11 = add64(res11, High_Extra_Bits); + High_Extra_Bits = srli64(res11, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 11, res11); + + res12 = add64(res12, High_Extra_Bits); + High_Extra_Bits = srli64(res12, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 12, res12); + + res13 = add64(res13, High_Extra_Bits); + High_Extra_Bits = srli64(res13, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 13, res13); + + res14 = add64(res14, High_Extra_Bits); + High_Extra_Bits = srli64(res14, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 14, res14); + + res15 = add64(res15, High_Extra_Bits); + High_Extra_Bits = srli64(res15, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 15, res15); + + res16 = add64(res16, High_Extra_Bits); + High_Extra_Bits = srli64(res16, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 16, res16); + + res17 = add64(res17, High_Extra_Bits); + High_Extra_Bits = srli64(res17, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 17, res17); + + res18 = add64(res18, High_Extra_Bits); + High_Extra_Bits = srli64(res18, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 18, res18); + + res19 = add64(res19, High_Extra_Bits); + storeu64(out_mb + MB_WIDTH * 19, res19); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ahmr52x20_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ahmr52x20_mb8.c new file mode 100644 index 0000000..6f166f1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ahmr52x20_mb8.c @@ -0,0 +1,229 @@ +/******************************************************************************* + * Copyright (C) 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +/* +Almost Half Montgomery Reduction (AHMR) + +Output: + out_mb : C = A*(2^(-10*52)) mod M +Inputs: + inpA_mb : A + inpM_mb : M + k0_mb : mont_constant = (-M^(-1) mod 2^(52)) + + C=A + for i from 0 to 9: + T=C[0]*mu // discard T[1] + C=C+T[0]*M // C[0] is zero + C=C>>52 // at each step of the for loop, divide the result by 2^52 + return C +*/ + + +#include + +void ifma_ahmr52x20_mb8( + int64u* out_mb, const int64u* inpA_mb, const int64u* inpM_mb, const int64u* k0_mb) { + + U64 res00, res01, res02, res03, res04, res05, res06, res07, res08, res09, + res10, res11, res12, res13, res14, res15, res16, res17, res18, res19; + + const U64 mont_constant = loadu64(k0_mb); + + res00 = res01 = res02 = res03 = res04 = res05 = res06 = res07 = res08 = res09 = + res10 = res11 = res12 = res13 = res14 = res15 = res16 = res17 = res18 = res19 = get_zero64(); + + // top 12 bits of 64-bit digits of the input A are junk data + // they need to be masked out before the reduction operation + const U64 MASK52 = set64(DIGIT_MASK); + + U64* M = (U64*) inpM_mb; + + // C=A + // res00:res19 are temporary registers holding intermediate result C + res00 = loadu64(inpA_mb + 0*MB_WIDTH); + res00 = and64(res00, MASK52); + res01 = loadu64(inpA_mb + 1*MB_WIDTH); + res01 = and64(res01, MASK52); + res02 = loadu64(inpA_mb + 2*MB_WIDTH); + res02 = and64(res02, MASK52); + res03 = loadu64(inpA_mb + 3*MB_WIDTH); + res03 = and64(res03, MASK52); + res04 = loadu64(inpA_mb + 4*MB_WIDTH); + res04 = and64(res04, MASK52); + res05 = loadu64(inpA_mb + 5*MB_WIDTH); + res05 = and64(res05, MASK52); + res06 = loadu64(inpA_mb + 6*MB_WIDTH); + res06 = and64(res06, MASK52); + res07 = loadu64(inpA_mb + 7*MB_WIDTH); + res07 = and64(res07, MASK52); + res08 = loadu64(inpA_mb + 8*MB_WIDTH); + res08 = and64(res08, MASK52); + res09 = loadu64(inpA_mb + 9*MB_WIDTH); + res09 = and64(res09, MASK52); + res10 = loadu64(inpA_mb +10*MB_WIDTH); + res10 = and64(res10, MASK52); + res11 = loadu64(inpA_mb +11*MB_WIDTH); + res11 = and64(res11, MASK52); + res12 = loadu64(inpA_mb +12*MB_WIDTH); + res12 = and64(res12, MASK52); + res13 = loadu64(inpA_mb +13*MB_WIDTH); + res13 = and64(res13, MASK52); + res14 = loadu64(inpA_mb +14*MB_WIDTH); + res14 = and64(res14, MASK52); + res15 = loadu64(inpA_mb +15*MB_WIDTH); + res15 = and64(res15, MASK52); + res16 = loadu64(inpA_mb +16*MB_WIDTH); + res16 = and64(res16, MASK52); + res17 = loadu64(inpA_mb +17*MB_WIDTH); + res17 = and64(res17, MASK52); + res18 = loadu64(inpA_mb +18*MB_WIDTH); + res18 = and64(res18, MASK52); + res19 = loadu64(inpA_mb +19*MB_WIDTH); + res19 = and64(res19, MASK52); + + for (int itr = 0; itr < 10; itr++) { + // T=C[0]*mu + const U64 T = fma52lo(get_zero64(), res00, mont_constant); + + // C=C+T[0]*M (low part) + res00 = fma52lo(res00, T, M[ 0]); + + // low 52 (DIGIT_SIZE) bits of res00 are 0 + // high 12 bits are accumulated to res01 (same bit-weight) + res00 = srli64(res00, DIGIT_SIZE); + res01 = add64(res01, res00); + + res01 = fma52lo(res01, T, M[ 1]); + res00 = fma52hi(res01, T, M[ 0]); + res02 = fma52lo(res02, T, M[ 2]); + res01 = fma52hi(res02, T, M[ 1]); + res03 = fma52lo(res03, T, M[ 3]); + res02 = fma52hi(res03, T, M[ 2]); + res04 = fma52lo(res04, T, M[ 4]); + res03 = fma52hi(res04, T, M[ 3]); + res05 = fma52lo(res05, T, M[ 5]); + res04 = fma52hi(res05, T, M[ 4]); + res06 = fma52lo(res06, T, M[ 6]); + res05 = fma52hi(res06, T, M[ 5]); + res07 = fma52lo(res07, T, M[ 7]); + res06 = fma52hi(res07, T, M[ 6]); + res08 = fma52lo(res08, T, M[ 8]); + res07 = fma52hi(res08, T, M[ 7]); + res09 = fma52lo(res09, T, M[ 9]); + res08 = fma52hi(res09, T, M[ 8]); + res10 = fma52lo(res10, T, M[10]); + res09 = fma52hi(res10, T, M[ 9]); + res11 = fma52lo(res11, T, M[11]); + res10 = fma52hi(res11, T, M[10]); + res12 = fma52lo(res12, T, M[12]); + res11 = fma52hi(res12, T, M[11]); + res13 = fma52lo(res13, T, M[13]); + res12 = fma52hi(res13, T, M[12]); + res14 = fma52lo(res14, T, M[14]); + res13 = fma52hi(res14, T, M[13]); + res15 = fma52lo(res15, T, M[15]); + res14 = fma52hi(res15, T, M[14]); + res16 = fma52lo(res16, T, M[16]); + res15 = fma52hi(res16, T, M[15]); + res17 = fma52lo(res17, T, M[17]); + res16 = fma52hi(res17, T, M[16]); + res18 = fma52lo(res18, T, M[18]); + res17 = fma52hi(res18, T, M[17]); + res19 = fma52lo(res19, T, M[19]); + res18 = fma52hi(res19, T, M[18]); + res19 = fma52hi(get_zero64(), T, M[19]); + } + // Normalization + U64 High_Extra_Bits = srli64(res00, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 0, res00); + + res01 = add64(res01, High_Extra_Bits); + High_Extra_Bits = srli64(res01, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 1, res01); + + res02 = add64(res02, High_Extra_Bits); + High_Extra_Bits = srli64(res02, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 2, res02); + + res03 = add64(res03, High_Extra_Bits); + High_Extra_Bits = srli64(res03, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 3, res03); + + res04 = add64(res04, High_Extra_Bits); + High_Extra_Bits = srli64(res04, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 4, res04); + + res05 = add64(res05, High_Extra_Bits); + High_Extra_Bits = srli64(res05, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 5, res05); + + res06 = add64(res06, High_Extra_Bits); + High_Extra_Bits = srli64(res06, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 6, res06); + + res07 = add64(res07, High_Extra_Bits); + High_Extra_Bits = srli64(res07, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 7, res07); + + res08 = add64(res08, High_Extra_Bits); + High_Extra_Bits = srli64(res08, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 8, res08); + + res09 = add64(res09, High_Extra_Bits); + High_Extra_Bits = srli64(res09, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 9, res09); + + res10 = add64(res10, High_Extra_Bits); + High_Extra_Bits = srli64(res10, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 10, res10); + + res11 = add64(res11, High_Extra_Bits); + High_Extra_Bits = srli64(res11, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 11, res11); + + res12 = add64(res12, High_Extra_Bits); + High_Extra_Bits = srli64(res12, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 12, res12); + + res13 = add64(res13, High_Extra_Bits); + High_Extra_Bits = srli64(res13, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 13, res13); + + res14 = add64(res14, High_Extra_Bits); + High_Extra_Bits = srli64(res14, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 14, res14); + + res15 = add64(res15, High_Extra_Bits); + High_Extra_Bits = srli64(res15, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 15, res15); + + res16 = add64(res16, High_Extra_Bits); + High_Extra_Bits = srli64(res16, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 16, res16); + + res17 = add64(res17, High_Extra_Bits); + High_Extra_Bits = srli64(res17, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 17, res17); + + res18 = add64(res18, High_Extra_Bits); + High_Extra_Bits = srli64(res18, DIGIT_SIZE); + storeu64(out_mb + MB_WIDTH * 18, res18); + + res19 = add64(res19, High_Extra_Bits); + storeu64(out_mb + MB_WIDTH * 19, res19); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_amm52x10_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_amm52x10_mb8.c new file mode 100644 index 0000000..e97ecf5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_amm52x10_mb8.c @@ -0,0 +1,120 @@ +/******************************************************************************* + * Copyright (C) 2019 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include + +void ifma_amm52x10_mb8(int64u *out_mb, const int64u *inpA_mb, + const int64u *inpB_mb, const int64u *inpM_mb, + const int64u *k0_mb) { + U64 res00, res01, res02, res03, res04, res05, res06, res07, res08, res09; + U64 K = loadu64(k0_mb); + int itr; + res00 = res01 = res02 = res03 = res04 = res05 = res06 = res07 = res08 = + res09 = get_zero64(); + + for (itr = 0; itr < 10; itr++) { + U64 Yi; + U64 Bi = loadu64(inpB_mb); + inpB_mb += MB_WIDTH; + fma52lo_mem(res00, res00, Bi, inpA_mb, SIMD_BYTES * 0); + fma52lo_mem(res01, res01, Bi, inpA_mb, SIMD_BYTES * 1); + fma52lo_mem(res02, res02, Bi, inpA_mb, SIMD_BYTES * 2); + fma52lo_mem(res03, res03, Bi, inpA_mb, SIMD_BYTES * 3); + fma52lo_mem(res04, res04, Bi, inpA_mb, SIMD_BYTES * 4); + fma52lo_mem(res05, res05, Bi, inpA_mb, SIMD_BYTES * 5); + fma52lo_mem(res06, res06, Bi, inpA_mb, SIMD_BYTES * 6); + fma52lo_mem(res07, res07, Bi, inpA_mb, SIMD_BYTES * 7); + fma52lo_mem(res08, res08, Bi, inpA_mb, SIMD_BYTES * 8); + fma52lo_mem(res09, res09, Bi, inpA_mb, SIMD_BYTES * 9); + Yi = fma52lo(get_zero64(), res00, K); + fma52lo_mem(res00, res00, Yi, inpM_mb, SIMD_BYTES * 0); + fma52lo_mem(res01, res01, Yi, inpM_mb, SIMD_BYTES * 1); + fma52lo_mem(res02, res02, Yi, inpM_mb, SIMD_BYTES * 2); + fma52lo_mem(res03, res03, Yi, inpM_mb, SIMD_BYTES * 3); + fma52lo_mem(res04, res04, Yi, inpM_mb, SIMD_BYTES * 4); + fma52lo_mem(res05, res05, Yi, inpM_mb, SIMD_BYTES * 5); + fma52lo_mem(res06, res06, Yi, inpM_mb, SIMD_BYTES * 6); + fma52lo_mem(res07, res07, Yi, inpM_mb, SIMD_BYTES * 7); + fma52lo_mem(res08, res08, Yi, inpM_mb, SIMD_BYTES * 8); + fma52lo_mem(res09, res09, Yi, inpM_mb, SIMD_BYTES * 9); + res00 = srli64(res00, DIGIT_SIZE); + res01 = add64(res01, res00); + fma52hi_mem(res00, res01, Bi, inpA_mb, SIMD_BYTES * 0); + fma52hi_mem(res01, res02, Bi, inpA_mb, SIMD_BYTES * 1); + fma52hi_mem(res02, res03, Bi, inpA_mb, SIMD_BYTES * 2); + fma52hi_mem(res03, res04, Bi, inpA_mb, SIMD_BYTES * 3); + fma52hi_mem(res04, res05, Bi, inpA_mb, SIMD_BYTES * 4); + fma52hi_mem(res05, res06, Bi, inpA_mb, SIMD_BYTES * 5); + fma52hi_mem(res06, res07, Bi, inpA_mb, SIMD_BYTES * 6); + fma52hi_mem(res07, res08, Bi, inpA_mb, SIMD_BYTES * 7); + fma52hi_mem(res08, res09, Bi, inpA_mb, SIMD_BYTES * 8); + fma52hi_mem(res09, get_zero64(), Bi, inpA_mb, SIMD_BYTES * 9); + fma52hi_mem(res00, res00, Yi, inpM_mb, SIMD_BYTES * 0); + fma52hi_mem(res01, res01, Yi, inpM_mb, SIMD_BYTES * 1); + fma52hi_mem(res02, res02, Yi, inpM_mb, SIMD_BYTES * 2); + fma52hi_mem(res03, res03, Yi, inpM_mb, SIMD_BYTES * 3); + fma52hi_mem(res04, res04, Yi, inpM_mb, SIMD_BYTES * 4); + fma52hi_mem(res05, res05, Yi, inpM_mb, SIMD_BYTES * 5); + fma52hi_mem(res06, res06, Yi, inpM_mb, SIMD_BYTES * 6); + fma52hi_mem(res07, res07, Yi, inpM_mb, SIMD_BYTES * 7); + fma52hi_mem(res08, res08, Yi, inpM_mb, SIMD_BYTES * 8); + fma52hi_mem(res09, res09, Yi, inpM_mb, SIMD_BYTES * 9); + } + // Normalization + { + U64 T = get_zero64(); + U64 MASK = set64(DIGIT_MASK); + T = srli64(res00, DIGIT_SIZE); + res00 = and64(res00, MASK); + storeu64(out_mb + MB_WIDTH * 0, res00); + res01 = add64(res01, T); + T = srli64(res01, DIGIT_SIZE); + res01 = and64(res01, MASK); + storeu64(out_mb + MB_WIDTH * 1, res01); + res02 = add64(res02, T); + T = srli64(res02, DIGIT_SIZE); + res02 = and64(res02, MASK); + storeu64(out_mb + MB_WIDTH * 2, res02); + res03 = add64(res03, T); + T = srli64(res03, DIGIT_SIZE); + res03 = and64(res03, MASK); + storeu64(out_mb + MB_WIDTH * 3, res03); + res04 = add64(res04, T); + T = srli64(res04, DIGIT_SIZE); + res04 = and64(res04, MASK); + storeu64(out_mb + MB_WIDTH * 4, res04); + res05 = add64(res05, T); + T = srli64(res05, DIGIT_SIZE); + res05 = and64(res05, MASK); + storeu64(out_mb + MB_WIDTH * 5, res05); + res06 = add64(res06, T); + T = srli64(res06, DIGIT_SIZE); + res06 = and64(res06, MASK); + storeu64(out_mb + MB_WIDTH * 6, res06); + res07 = add64(res07, T); + T = srli64(res07, DIGIT_SIZE); + res07 = and64(res07, MASK); + storeu64(out_mb + MB_WIDTH * 7, res07); + res08 = add64(res08, T); + T = srli64(res08, DIGIT_SIZE); + res08 = and64(res08, MASK); + storeu64(out_mb + MB_WIDTH * 8, res08); + res09 = add64(res09, T); + res09 = and64(res09, MASK); + storeu64(out_mb + MB_WIDTH * 9, res09); + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_amm52x20_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_amm52x20_mb8.c new file mode 100644 index 0000000..7a513c9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_amm52x20_mb8.c @@ -0,0 +1,202 @@ +/******************************************************************************* + * Copyright (C) 2019 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include + +void ifma_amm52x20_mb8(int64u *out_mb, const int64u *inpA_mb, + const int64u *inpB_mb, const int64u *inpM_mb, + const int64u *k0_mb) { + U64 res00, res01, res02, res03, res04, res05, res06, res07, res08, res09, + res10, res11, res12, res13, res14, res15, res16, res17, res18, res19; + U64 K = loadu64(k0_mb); + int itr; + res00 = res01 = res02 = res03 = res04 = res05 = res06 = res07 = res08 = + res09 = res10 = res11 = res12 = res13 = res14 = res15 = res16 = res17 = + res18 = res19 = get_zero64(); + + for (itr = 0; itr < 20; itr++) { + U64 Yi; + U64 Bi = loadu64(inpB_mb); + inpB_mb += MB_WIDTH; + fma52lo_mem(res00, res00, Bi, inpA_mb, SIMD_BYTES * 0); + fma52lo_mem(res01, res01, Bi, inpA_mb, SIMD_BYTES * 1); + fma52lo_mem(res02, res02, Bi, inpA_mb, SIMD_BYTES * 2); + fma52lo_mem(res03, res03, Bi, inpA_mb, SIMD_BYTES * 3); + fma52lo_mem(res04, res04, Bi, inpA_mb, SIMD_BYTES * 4); + fma52lo_mem(res05, res05, Bi, inpA_mb, SIMD_BYTES * 5); + fma52lo_mem(res06, res06, Bi, inpA_mb, SIMD_BYTES * 6); + fma52lo_mem(res07, res07, Bi, inpA_mb, SIMD_BYTES * 7); + fma52lo_mem(res08, res08, Bi, inpA_mb, SIMD_BYTES * 8); + fma52lo_mem(res09, res09, Bi, inpA_mb, SIMD_BYTES * 9); + fma52lo_mem(res10, res10, Bi, inpA_mb, SIMD_BYTES * 10); + fma52lo_mem(res11, res11, Bi, inpA_mb, SIMD_BYTES * 11); + fma52lo_mem(res12, res12, Bi, inpA_mb, SIMD_BYTES * 12); + fma52lo_mem(res13, res13, Bi, inpA_mb, SIMD_BYTES * 13); + fma52lo_mem(res14, res14, Bi, inpA_mb, SIMD_BYTES * 14); + fma52lo_mem(res15, res15, Bi, inpA_mb, SIMD_BYTES * 15); + fma52lo_mem(res16, res16, Bi, inpA_mb, SIMD_BYTES * 16); + fma52lo_mem(res17, res17, Bi, inpA_mb, SIMD_BYTES * 17); + fma52lo_mem(res18, res18, Bi, inpA_mb, SIMD_BYTES * 18); + fma52lo_mem(res19, res19, Bi, inpA_mb, SIMD_BYTES * 19); + Yi = fma52lo(get_zero64(), res00, K); + fma52lo_mem(res00, res00, Yi, inpM_mb, SIMD_BYTES * 0); + fma52lo_mem(res01, res01, Yi, inpM_mb, SIMD_BYTES * 1); + fma52lo_mem(res02, res02, Yi, inpM_mb, SIMD_BYTES * 2); + fma52lo_mem(res03, res03, Yi, inpM_mb, SIMD_BYTES * 3); + fma52lo_mem(res04, res04, Yi, inpM_mb, SIMD_BYTES * 4); + fma52lo_mem(res05, res05, Yi, inpM_mb, SIMD_BYTES * 5); + fma52lo_mem(res06, res06, Yi, inpM_mb, SIMD_BYTES * 6); + fma52lo_mem(res07, res07, Yi, inpM_mb, SIMD_BYTES * 7); + fma52lo_mem(res08, res08, Yi, inpM_mb, SIMD_BYTES * 8); + fma52lo_mem(res09, res09, Yi, inpM_mb, SIMD_BYTES * 9); + fma52lo_mem(res10, res10, Yi, inpM_mb, SIMD_BYTES * 10); + fma52lo_mem(res11, res11, Yi, inpM_mb, SIMD_BYTES * 11); + fma52lo_mem(res12, res12, Yi, inpM_mb, SIMD_BYTES * 12); + fma52lo_mem(res13, res13, Yi, inpM_mb, SIMD_BYTES * 13); + fma52lo_mem(res14, res14, Yi, inpM_mb, SIMD_BYTES * 14); + fma52lo_mem(res15, res15, Yi, inpM_mb, SIMD_BYTES * 15); + fma52lo_mem(res16, res16, Yi, inpM_mb, SIMD_BYTES * 16); + fma52lo_mem(res17, res17, Yi, inpM_mb, SIMD_BYTES * 17); + fma52lo_mem(res18, res18, Yi, inpM_mb, SIMD_BYTES * 18); + fma52lo_mem(res19, res19, Yi, inpM_mb, SIMD_BYTES * 19); + res00 = srli64(res00, DIGIT_SIZE); + res01 = add64(res01, res00); + fma52hi_mem(res00, res01, Bi, inpA_mb, SIMD_BYTES * 0); + fma52hi_mem(res01, res02, Bi, inpA_mb, SIMD_BYTES * 1); + fma52hi_mem(res02, res03, Bi, inpA_mb, SIMD_BYTES * 2); + fma52hi_mem(res03, res04, Bi, inpA_mb, SIMD_BYTES * 3); + fma52hi_mem(res04, res05, Bi, inpA_mb, SIMD_BYTES * 4); + fma52hi_mem(res05, res06, Bi, inpA_mb, SIMD_BYTES * 5); + fma52hi_mem(res06, res07, Bi, inpA_mb, SIMD_BYTES * 6); + fma52hi_mem(res07, res08, Bi, inpA_mb, SIMD_BYTES * 7); + fma52hi_mem(res08, res09, Bi, inpA_mb, SIMD_BYTES * 8); + fma52hi_mem(res09, res10, Bi, inpA_mb, SIMD_BYTES * 9); + fma52hi_mem(res10, res11, Bi, inpA_mb, SIMD_BYTES * 10); + fma52hi_mem(res11, res12, Bi, inpA_mb, SIMD_BYTES * 11); + fma52hi_mem(res12, res13, Bi, inpA_mb, SIMD_BYTES * 12); + fma52hi_mem(res13, res14, Bi, inpA_mb, SIMD_BYTES * 13); + fma52hi_mem(res14, res15, Bi, inpA_mb, SIMD_BYTES * 14); + fma52hi_mem(res15, res16, Bi, inpA_mb, SIMD_BYTES * 15); + fma52hi_mem(res16, res17, Bi, inpA_mb, SIMD_BYTES * 16); + fma52hi_mem(res17, res18, Bi, inpA_mb, SIMD_BYTES * 17); + fma52hi_mem(res18, res19, Bi, inpA_mb, SIMD_BYTES * 18); + fma52hi_mem(res19, get_zero64(), Bi, inpA_mb, SIMD_BYTES * 19); + fma52hi_mem(res00, res00, Yi, inpM_mb, SIMD_BYTES * 0); + fma52hi_mem(res01, res01, Yi, inpM_mb, SIMD_BYTES * 1); + fma52hi_mem(res02, res02, Yi, inpM_mb, SIMD_BYTES * 2); + fma52hi_mem(res03, res03, Yi, inpM_mb, SIMD_BYTES * 3); + fma52hi_mem(res04, res04, Yi, inpM_mb, SIMD_BYTES * 4); + fma52hi_mem(res05, res05, Yi, inpM_mb, SIMD_BYTES * 5); + fma52hi_mem(res06, res06, Yi, inpM_mb, SIMD_BYTES * 6); + fma52hi_mem(res07, res07, Yi, inpM_mb, SIMD_BYTES * 7); + fma52hi_mem(res08, res08, Yi, inpM_mb, SIMD_BYTES * 8); + fma52hi_mem(res09, res09, Yi, inpM_mb, SIMD_BYTES * 9); + fma52hi_mem(res10, res10, Yi, inpM_mb, SIMD_BYTES * 10); + fma52hi_mem(res11, res11, Yi, inpM_mb, SIMD_BYTES * 11); + fma52hi_mem(res12, res12, Yi, inpM_mb, SIMD_BYTES * 12); + fma52hi_mem(res13, res13, Yi, inpM_mb, SIMD_BYTES * 13); + fma52hi_mem(res14, res14, Yi, inpM_mb, SIMD_BYTES * 14); + fma52hi_mem(res15, res15, Yi, inpM_mb, SIMD_BYTES * 15); + fma52hi_mem(res16, res16, Yi, inpM_mb, SIMD_BYTES * 16); + fma52hi_mem(res17, res17, Yi, inpM_mb, SIMD_BYTES * 17); + fma52hi_mem(res18, res18, Yi, inpM_mb, SIMD_BYTES * 18); + fma52hi_mem(res19, res19, Yi, inpM_mb, SIMD_BYTES * 19); + } + // Normalization + { + U64 T = get_zero64(); + U64 MASK = set64(DIGIT_MASK); + T = srli64(res00, DIGIT_SIZE); + res00 = and64(res00, MASK); + storeu64(out_mb + MB_WIDTH * 0, res00); + res01 = add64(res01, T); + T = srli64(res01, DIGIT_SIZE); + res01 = and64(res01, MASK); + storeu64(out_mb + MB_WIDTH * 1, res01); + res02 = add64(res02, T); + T = srli64(res02, DIGIT_SIZE); + res02 = and64(res02, MASK); + storeu64(out_mb + MB_WIDTH * 2, res02); + res03 = add64(res03, T); + T = srli64(res03, DIGIT_SIZE); + res03 = and64(res03, MASK); + storeu64(out_mb + MB_WIDTH * 3, res03); + res04 = add64(res04, T); + T = srli64(res04, DIGIT_SIZE); + res04 = and64(res04, MASK); + storeu64(out_mb + MB_WIDTH * 4, res04); + res05 = add64(res05, T); + T = srli64(res05, DIGIT_SIZE); + res05 = and64(res05, MASK); + storeu64(out_mb + MB_WIDTH * 5, res05); + res06 = add64(res06, T); + T = srli64(res06, DIGIT_SIZE); + res06 = and64(res06, MASK); + storeu64(out_mb + MB_WIDTH * 6, res06); + res07 = add64(res07, T); + T = srli64(res07, DIGIT_SIZE); + res07 = and64(res07, MASK); + storeu64(out_mb + MB_WIDTH * 7, res07); + res08 = add64(res08, T); + T = srli64(res08, DIGIT_SIZE); + res08 = and64(res08, MASK); + storeu64(out_mb + MB_WIDTH * 8, res08); + res09 = add64(res09, T); + T = srli64(res09, DIGIT_SIZE); + res09 = and64(res09, MASK); + storeu64(out_mb + MB_WIDTH * 9, res09); + res10 = add64(res10, T); + T = srli64(res10, DIGIT_SIZE); + res10 = and64(res10, MASK); + storeu64(out_mb + MB_WIDTH * 10, res10); + res11 = add64(res11, T); + T = srli64(res11, DIGIT_SIZE); + res11 = and64(res11, MASK); + storeu64(out_mb + MB_WIDTH * 11, res11); + res12 = add64(res12, T); + T = srli64(res12, DIGIT_SIZE); + res12 = and64(res12, MASK); + storeu64(out_mb + MB_WIDTH * 12, res12); + res13 = add64(res13, T); + T = srli64(res13, DIGIT_SIZE); + res13 = and64(res13, MASK); + storeu64(out_mb + MB_WIDTH * 13, res13); + res14 = add64(res14, T); + T = srli64(res14, DIGIT_SIZE); + res14 = and64(res14, MASK); + storeu64(out_mb + MB_WIDTH * 14, res14); + res15 = add64(res15, T); + T = srli64(res15, DIGIT_SIZE); + res15 = and64(res15, MASK); + storeu64(out_mb + MB_WIDTH * 15, res15); + res16 = add64(res16, T); + T = srli64(res16, DIGIT_SIZE); + res16 = and64(res16, MASK); + storeu64(out_mb + MB_WIDTH * 16, res16); + res17 = add64(res17, T); + T = srli64(res17, DIGIT_SIZE); + res17 = and64(res17, MASK); + storeu64(out_mb + MB_WIDTH * 17, res17); + res18 = add64(res18, T); + T = srli64(res18, DIGIT_SIZE); + res18 = and64(res18, MASK); + storeu64(out_mb + MB_WIDTH * 18, res18); + res19 = add64(res19, T); + res19 = and64(res19, MASK); + storeu64(out_mb + MB_WIDTH * 19, res19); + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_amm52x30_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_amm52x30_mb8.c new file mode 100644 index 0000000..2f94c67 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_amm52x30_mb8.c @@ -0,0 +1,284 @@ +/******************************************************************************* + * Copyright (C) 2019 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include + +void ifma_amm52x30_mb8(int64u *out_mb, const int64u *inpA_mb, + const int64u *inpB_mb, const int64u *inpM_mb, + const int64u *k0_mb) { + U64 res00, res01, res02, res03, res04, res05, res06, res07, res08, res09, + res10, res11, res12, res13, res14, res15, res16, res17, res18, res19, + res20, res21, res22, res23, res24, res25, res26, res27, res28, res29; + U64 K = loadu64(k0_mb); + int itr; + res00 = res01 = res02 = res03 = res04 = res05 = res06 = res07 = res08 = + res09 = res10 = res11 = res12 = res13 = res14 = res15 = res16 = res17 = + res18 = res19 = res20 = res21 = res22 = res23 = res24 = res25 = + res26 = res27 = res28 = res29 = get_zero64(); + + for (itr = 0; itr < 30; itr++) { + U64 Yi; + U64 Bi = loadu64(inpB_mb); + inpB_mb += MB_WIDTH; + fma52lo_mem(res00, res00, Bi, inpA_mb, SIMD_BYTES * 0); + fma52lo_mem(res01, res01, Bi, inpA_mb, SIMD_BYTES * 1); + fma52lo_mem(res02, res02, Bi, inpA_mb, SIMD_BYTES * 2); + fma52lo_mem(res03, res03, Bi, inpA_mb, SIMD_BYTES * 3); + fma52lo_mem(res04, res04, Bi, inpA_mb, SIMD_BYTES * 4); + fma52lo_mem(res05, res05, Bi, inpA_mb, SIMD_BYTES * 5); + fma52lo_mem(res06, res06, Bi, inpA_mb, SIMD_BYTES * 6); + fma52lo_mem(res07, res07, Bi, inpA_mb, SIMD_BYTES * 7); + fma52lo_mem(res08, res08, Bi, inpA_mb, SIMD_BYTES * 8); + fma52lo_mem(res09, res09, Bi, inpA_mb, SIMD_BYTES * 9); + fma52lo_mem(res10, res10, Bi, inpA_mb, SIMD_BYTES * 10); + fma52lo_mem(res11, res11, Bi, inpA_mb, SIMD_BYTES * 11); + fma52lo_mem(res12, res12, Bi, inpA_mb, SIMD_BYTES * 12); + fma52lo_mem(res13, res13, Bi, inpA_mb, SIMD_BYTES * 13); + fma52lo_mem(res14, res14, Bi, inpA_mb, SIMD_BYTES * 14); + fma52lo_mem(res15, res15, Bi, inpA_mb, SIMD_BYTES * 15); + fma52lo_mem(res16, res16, Bi, inpA_mb, SIMD_BYTES * 16); + fma52lo_mem(res17, res17, Bi, inpA_mb, SIMD_BYTES * 17); + fma52lo_mem(res18, res18, Bi, inpA_mb, SIMD_BYTES * 18); + fma52lo_mem(res19, res19, Bi, inpA_mb, SIMD_BYTES * 19); + fma52lo_mem(res20, res20, Bi, inpA_mb, SIMD_BYTES * 20); + fma52lo_mem(res21, res21, Bi, inpA_mb, SIMD_BYTES * 21); + fma52lo_mem(res22, res22, Bi, inpA_mb, SIMD_BYTES * 22); + fma52lo_mem(res23, res23, Bi, inpA_mb, SIMD_BYTES * 23); + fma52lo_mem(res24, res24, Bi, inpA_mb, SIMD_BYTES * 24); + fma52lo_mem(res25, res25, Bi, inpA_mb, SIMD_BYTES * 25); + fma52lo_mem(res26, res26, Bi, inpA_mb, SIMD_BYTES * 26); + fma52lo_mem(res27, res27, Bi, inpA_mb, SIMD_BYTES * 27); + fma52lo_mem(res28, res28, Bi, inpA_mb, SIMD_BYTES * 28); + fma52lo_mem(res29, res29, Bi, inpA_mb, SIMD_BYTES * 29); + Yi = fma52lo(get_zero64(), res00, K); + fma52lo_mem(res00, res00, Yi, inpM_mb, SIMD_BYTES * 0); + fma52lo_mem(res01, res01, Yi, inpM_mb, SIMD_BYTES * 1); + fma52lo_mem(res02, res02, Yi, inpM_mb, SIMD_BYTES * 2); + fma52lo_mem(res03, res03, Yi, inpM_mb, SIMD_BYTES * 3); + fma52lo_mem(res04, res04, Yi, inpM_mb, SIMD_BYTES * 4); + fma52lo_mem(res05, res05, Yi, inpM_mb, SIMD_BYTES * 5); + fma52lo_mem(res06, res06, Yi, inpM_mb, SIMD_BYTES * 6); + fma52lo_mem(res07, res07, Yi, inpM_mb, SIMD_BYTES * 7); + fma52lo_mem(res08, res08, Yi, inpM_mb, SIMD_BYTES * 8); + fma52lo_mem(res09, res09, Yi, inpM_mb, SIMD_BYTES * 9); + fma52lo_mem(res10, res10, Yi, inpM_mb, SIMD_BYTES * 10); + fma52lo_mem(res11, res11, Yi, inpM_mb, SIMD_BYTES * 11); + fma52lo_mem(res12, res12, Yi, inpM_mb, SIMD_BYTES * 12); + fma52lo_mem(res13, res13, Yi, inpM_mb, SIMD_BYTES * 13); + fma52lo_mem(res14, res14, Yi, inpM_mb, SIMD_BYTES * 14); + fma52lo_mem(res15, res15, Yi, inpM_mb, SIMD_BYTES * 15); + fma52lo_mem(res16, res16, Yi, inpM_mb, SIMD_BYTES * 16); + fma52lo_mem(res17, res17, Yi, inpM_mb, SIMD_BYTES * 17); + fma52lo_mem(res18, res18, Yi, inpM_mb, SIMD_BYTES * 18); + fma52lo_mem(res19, res19, Yi, inpM_mb, SIMD_BYTES * 19); + fma52lo_mem(res20, res20, Yi, inpM_mb, SIMD_BYTES * 20); + fma52lo_mem(res21, res21, Yi, inpM_mb, SIMD_BYTES * 21); + fma52lo_mem(res22, res22, Yi, inpM_mb, SIMD_BYTES * 22); + fma52lo_mem(res23, res23, Yi, inpM_mb, SIMD_BYTES * 23); + fma52lo_mem(res24, res24, Yi, inpM_mb, SIMD_BYTES * 24); + fma52lo_mem(res25, res25, Yi, inpM_mb, SIMD_BYTES * 25); + fma52lo_mem(res26, res26, Yi, inpM_mb, SIMD_BYTES * 26); + fma52lo_mem(res27, res27, Yi, inpM_mb, SIMD_BYTES * 27); + fma52lo_mem(res28, res28, Yi, inpM_mb, SIMD_BYTES * 28); + fma52lo_mem(res29, res29, Yi, inpM_mb, SIMD_BYTES * 29); + res00 = srli64(res00, DIGIT_SIZE); + res01 = add64(res01, res00); + fma52hi_mem(res00, res01, Bi, inpA_mb, SIMD_BYTES * 0); + fma52hi_mem(res01, res02, Bi, inpA_mb, SIMD_BYTES * 1); + fma52hi_mem(res02, res03, Bi, inpA_mb, SIMD_BYTES * 2); + fma52hi_mem(res03, res04, Bi, inpA_mb, SIMD_BYTES * 3); + fma52hi_mem(res04, res05, Bi, inpA_mb, SIMD_BYTES * 4); + fma52hi_mem(res05, res06, Bi, inpA_mb, SIMD_BYTES * 5); + fma52hi_mem(res06, res07, Bi, inpA_mb, SIMD_BYTES * 6); + fma52hi_mem(res07, res08, Bi, inpA_mb, SIMD_BYTES * 7); + fma52hi_mem(res08, res09, Bi, inpA_mb, SIMD_BYTES * 8); + fma52hi_mem(res09, res10, Bi, inpA_mb, SIMD_BYTES * 9); + fma52hi_mem(res10, res11, Bi, inpA_mb, SIMD_BYTES * 10); + fma52hi_mem(res11, res12, Bi, inpA_mb, SIMD_BYTES * 11); + fma52hi_mem(res12, res13, Bi, inpA_mb, SIMD_BYTES * 12); + fma52hi_mem(res13, res14, Bi, inpA_mb, SIMD_BYTES * 13); + fma52hi_mem(res14, res15, Bi, inpA_mb, SIMD_BYTES * 14); + fma52hi_mem(res15, res16, Bi, inpA_mb, SIMD_BYTES * 15); + fma52hi_mem(res16, res17, Bi, inpA_mb, SIMD_BYTES * 16); + fma52hi_mem(res17, res18, Bi, inpA_mb, SIMD_BYTES * 17); + fma52hi_mem(res18, res19, Bi, inpA_mb, SIMD_BYTES * 18); + fma52hi_mem(res19, res20, Bi, inpA_mb, SIMD_BYTES * 19); + fma52hi_mem(res20, res21, Bi, inpA_mb, SIMD_BYTES * 20); + fma52hi_mem(res21, res22, Bi, inpA_mb, SIMD_BYTES * 21); + fma52hi_mem(res22, res23, Bi, inpA_mb, SIMD_BYTES * 22); + fma52hi_mem(res23, res24, Bi, inpA_mb, SIMD_BYTES * 23); + fma52hi_mem(res24, res25, Bi, inpA_mb, SIMD_BYTES * 24); + fma52hi_mem(res25, res26, Bi, inpA_mb, SIMD_BYTES * 25); + fma52hi_mem(res26, res27, Bi, inpA_mb, SIMD_BYTES * 26); + fma52hi_mem(res27, res28, Bi, inpA_mb, SIMD_BYTES * 27); + fma52hi_mem(res28, res29, Bi, inpA_mb, SIMD_BYTES * 28); + fma52hi_mem(res29, get_zero64(), Bi, inpA_mb, SIMD_BYTES * 29); + fma52hi_mem(res00, res00, Yi, inpM_mb, SIMD_BYTES * 0); + fma52hi_mem(res01, res01, Yi, inpM_mb, SIMD_BYTES * 1); + fma52hi_mem(res02, res02, Yi, inpM_mb, SIMD_BYTES * 2); + fma52hi_mem(res03, res03, Yi, inpM_mb, SIMD_BYTES * 3); + fma52hi_mem(res04, res04, Yi, inpM_mb, SIMD_BYTES * 4); + fma52hi_mem(res05, res05, Yi, inpM_mb, SIMD_BYTES * 5); + fma52hi_mem(res06, res06, Yi, inpM_mb, SIMD_BYTES * 6); + fma52hi_mem(res07, res07, Yi, inpM_mb, SIMD_BYTES * 7); + fma52hi_mem(res08, res08, Yi, inpM_mb, SIMD_BYTES * 8); + fma52hi_mem(res09, res09, Yi, inpM_mb, SIMD_BYTES * 9); + fma52hi_mem(res10, res10, Yi, inpM_mb, SIMD_BYTES * 10); + fma52hi_mem(res11, res11, Yi, inpM_mb, SIMD_BYTES * 11); + fma52hi_mem(res12, res12, Yi, inpM_mb, SIMD_BYTES * 12); + fma52hi_mem(res13, res13, Yi, inpM_mb, SIMD_BYTES * 13); + fma52hi_mem(res14, res14, Yi, inpM_mb, SIMD_BYTES * 14); + fma52hi_mem(res15, res15, Yi, inpM_mb, SIMD_BYTES * 15); + fma52hi_mem(res16, res16, Yi, inpM_mb, SIMD_BYTES * 16); + fma52hi_mem(res17, res17, Yi, inpM_mb, SIMD_BYTES * 17); + fma52hi_mem(res18, res18, Yi, inpM_mb, SIMD_BYTES * 18); + fma52hi_mem(res19, res19, Yi, inpM_mb, SIMD_BYTES * 19); + fma52hi_mem(res20, res20, Yi, inpM_mb, SIMD_BYTES * 20); + fma52hi_mem(res21, res21, Yi, inpM_mb, SIMD_BYTES * 21); + fma52hi_mem(res22, res22, Yi, inpM_mb, SIMD_BYTES * 22); + fma52hi_mem(res23, res23, Yi, inpM_mb, SIMD_BYTES * 23); + fma52hi_mem(res24, res24, Yi, inpM_mb, SIMD_BYTES * 24); + fma52hi_mem(res25, res25, Yi, inpM_mb, SIMD_BYTES * 25); + fma52hi_mem(res26, res26, Yi, inpM_mb, SIMD_BYTES * 26); + fma52hi_mem(res27, res27, Yi, inpM_mb, SIMD_BYTES * 27); + fma52hi_mem(res28, res28, Yi, inpM_mb, SIMD_BYTES * 28); + fma52hi_mem(res29, res29, Yi, inpM_mb, SIMD_BYTES * 29); + } + // Normalization + { + U64 T = get_zero64(); + U64 MASK = set64(DIGIT_MASK); + T = srli64(res00, DIGIT_SIZE); + res00 = and64(res00, MASK); + storeu64(out_mb + MB_WIDTH * 0, res00); + res01 = add64(res01, T); + T = srli64(res01, DIGIT_SIZE); + res01 = and64(res01, MASK); + storeu64(out_mb + MB_WIDTH * 1, res01); + res02 = add64(res02, T); + T = srli64(res02, DIGIT_SIZE); + res02 = and64(res02, MASK); + storeu64(out_mb + MB_WIDTH * 2, res02); + res03 = add64(res03, T); + T = srli64(res03, DIGIT_SIZE); + res03 = and64(res03, MASK); + storeu64(out_mb + MB_WIDTH * 3, res03); + res04 = add64(res04, T); + T = srli64(res04, DIGIT_SIZE); + res04 = and64(res04, MASK); + storeu64(out_mb + MB_WIDTH * 4, res04); + res05 = add64(res05, T); + T = srli64(res05, DIGIT_SIZE); + res05 = and64(res05, MASK); + storeu64(out_mb + MB_WIDTH * 5, res05); + res06 = add64(res06, T); + T = srli64(res06, DIGIT_SIZE); + res06 = and64(res06, MASK); + storeu64(out_mb + MB_WIDTH * 6, res06); + res07 = add64(res07, T); + T = srli64(res07, DIGIT_SIZE); + res07 = and64(res07, MASK); + storeu64(out_mb + MB_WIDTH * 7, res07); + res08 = add64(res08, T); + T = srli64(res08, DIGIT_SIZE); + res08 = and64(res08, MASK); + storeu64(out_mb + MB_WIDTH * 8, res08); + res09 = add64(res09, T); + T = srli64(res09, DIGIT_SIZE); + res09 = and64(res09, MASK); + storeu64(out_mb + MB_WIDTH * 9, res09); + res10 = add64(res10, T); + T = srli64(res10, DIGIT_SIZE); + res10 = and64(res10, MASK); + storeu64(out_mb + MB_WIDTH * 10, res10); + res11 = add64(res11, T); + T = srli64(res11, DIGIT_SIZE); + res11 = and64(res11, MASK); + storeu64(out_mb + MB_WIDTH * 11, res11); + res12 = add64(res12, T); + T = srli64(res12, DIGIT_SIZE); + res12 = and64(res12, MASK); + storeu64(out_mb + MB_WIDTH * 12, res12); + res13 = add64(res13, T); + T = srli64(res13, DIGIT_SIZE); + res13 = and64(res13, MASK); + storeu64(out_mb + MB_WIDTH * 13, res13); + res14 = add64(res14, T); + T = srli64(res14, DIGIT_SIZE); + res14 = and64(res14, MASK); + storeu64(out_mb + MB_WIDTH * 14, res14); + res15 = add64(res15, T); + T = srli64(res15, DIGIT_SIZE); + res15 = and64(res15, MASK); + storeu64(out_mb + MB_WIDTH * 15, res15); + res16 = add64(res16, T); + T = srli64(res16, DIGIT_SIZE); + res16 = and64(res16, MASK); + storeu64(out_mb + MB_WIDTH * 16, res16); + res17 = add64(res17, T); + T = srli64(res17, DIGIT_SIZE); + res17 = and64(res17, MASK); + storeu64(out_mb + MB_WIDTH * 17, res17); + res18 = add64(res18, T); + T = srli64(res18, DIGIT_SIZE); + res18 = and64(res18, MASK); + storeu64(out_mb + MB_WIDTH * 18, res18); + res19 = add64(res19, T); + T = srli64(res19, DIGIT_SIZE); + res19 = and64(res19, MASK); + storeu64(out_mb + MB_WIDTH * 19, res19); + res20 = add64(res20, T); + T = srli64(res20, DIGIT_SIZE); + res20 = and64(res20, MASK); + storeu64(out_mb + MB_WIDTH * 20, res20); + res21 = add64(res21, T); + T = srli64(res21, DIGIT_SIZE); + res21 = and64(res21, MASK); + storeu64(out_mb + MB_WIDTH * 21, res21); + res22 = add64(res22, T); + T = srli64(res22, DIGIT_SIZE); + res22 = and64(res22, MASK); + storeu64(out_mb + MB_WIDTH * 22, res22); + res23 = add64(res23, T); + T = srli64(res23, DIGIT_SIZE); + res23 = and64(res23, MASK); + storeu64(out_mb + MB_WIDTH * 23, res23); + res24 = add64(res24, T); + T = srli64(res24, DIGIT_SIZE); + res24 = and64(res24, MASK); + storeu64(out_mb + MB_WIDTH * 24, res24); + res25 = add64(res25, T); + T = srli64(res25, DIGIT_SIZE); + res25 = and64(res25, MASK); + storeu64(out_mb + MB_WIDTH * 25, res25); + res26 = add64(res26, T); + T = srli64(res26, DIGIT_SIZE); + res26 = and64(res26, MASK); + storeu64(out_mb + MB_WIDTH * 26, res26); + res27 = add64(res27, T); + T = srli64(res27, DIGIT_SIZE); + res27 = and64(res27, MASK); + storeu64(out_mb + MB_WIDTH * 27, res27); + res28 = add64(res28, T); + T = srli64(res28, DIGIT_SIZE); + res28 = and64(res28, MASK); + storeu64(out_mb + MB_WIDTH * 28, res28); + res29 = add64(res29, T); + res29 = and64(res29, MASK); + storeu64(out_mb + MB_WIDTH * 29, res29); + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_amm52x40_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_amm52x40_mb8.c new file mode 100644 index 0000000..9812ec6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_amm52x40_mb8.c @@ -0,0 +1,366 @@ +/******************************************************************************* + * Copyright (C) 2019 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include + +void ifma_amm52x40_mb8(int64u *out_mb, const int64u *inpA_mb, + const int64u *inpB_mb, const int64u *inpM_mb, + const int64u *k0_mb) { + U64 res00, res01, res02, res03, res04, res05, res06, res07, res08, res09, + res10, res11, res12, res13, res14, res15, res16, res17, res18, res19, + res20, res21, res22, res23, res24, res25, res26, res27, res28, res29, + res30, res31, res32, res33, res34, res35, res36, res37, res38, res39; + U64 K = loadu64(k0_mb); + int itr; + res00 = res01 = res02 = res03 = res04 = res05 = res06 = res07 = res08 = + res09 = res10 = res11 = res12 = res13 = res14 = res15 = res16 = res17 = + res18 = res19 = res20 = res21 = res22 = res23 = res24 = res25 = + res26 = res27 = res28 = res29 = res30 = res31 = res32 = res33 = + res34 = res35 = res36 = res37 = res38 = res39 = get_zero64(); + + for (itr = 0; itr < 40; itr++) { + U64 Yi; + U64 Bi = loadu64(inpB_mb); + inpB_mb += MB_WIDTH; + fma52lo_mem(res00, res00, Bi, inpA_mb, SIMD_BYTES * 0); + fma52lo_mem(res01, res01, Bi, inpA_mb, SIMD_BYTES * 1); + fma52lo_mem(res02, res02, Bi, inpA_mb, SIMD_BYTES * 2); + fma52lo_mem(res03, res03, Bi, inpA_mb, SIMD_BYTES * 3); + fma52lo_mem(res04, res04, Bi, inpA_mb, SIMD_BYTES * 4); + fma52lo_mem(res05, res05, Bi, inpA_mb, SIMD_BYTES * 5); + fma52lo_mem(res06, res06, Bi, inpA_mb, SIMD_BYTES * 6); + fma52lo_mem(res07, res07, Bi, inpA_mb, SIMD_BYTES * 7); + fma52lo_mem(res08, res08, Bi, inpA_mb, SIMD_BYTES * 8); + fma52lo_mem(res09, res09, Bi, inpA_mb, SIMD_BYTES * 9); + fma52lo_mem(res10, res10, Bi, inpA_mb, SIMD_BYTES * 10); + fma52lo_mem(res11, res11, Bi, inpA_mb, SIMD_BYTES * 11); + fma52lo_mem(res12, res12, Bi, inpA_mb, SIMD_BYTES * 12); + fma52lo_mem(res13, res13, Bi, inpA_mb, SIMD_BYTES * 13); + fma52lo_mem(res14, res14, Bi, inpA_mb, SIMD_BYTES * 14); + fma52lo_mem(res15, res15, Bi, inpA_mb, SIMD_BYTES * 15); + fma52lo_mem(res16, res16, Bi, inpA_mb, SIMD_BYTES * 16); + fma52lo_mem(res17, res17, Bi, inpA_mb, SIMD_BYTES * 17); + fma52lo_mem(res18, res18, Bi, inpA_mb, SIMD_BYTES * 18); + fma52lo_mem(res19, res19, Bi, inpA_mb, SIMD_BYTES * 19); + fma52lo_mem(res20, res20, Bi, inpA_mb, SIMD_BYTES * 20); + fma52lo_mem(res21, res21, Bi, inpA_mb, SIMD_BYTES * 21); + fma52lo_mem(res22, res22, Bi, inpA_mb, SIMD_BYTES * 22); + fma52lo_mem(res23, res23, Bi, inpA_mb, SIMD_BYTES * 23); + fma52lo_mem(res24, res24, Bi, inpA_mb, SIMD_BYTES * 24); + fma52lo_mem(res25, res25, Bi, inpA_mb, SIMD_BYTES * 25); + fma52lo_mem(res26, res26, Bi, inpA_mb, SIMD_BYTES * 26); + fma52lo_mem(res27, res27, Bi, inpA_mb, SIMD_BYTES * 27); + fma52lo_mem(res28, res28, Bi, inpA_mb, SIMD_BYTES * 28); + fma52lo_mem(res29, res29, Bi, inpA_mb, SIMD_BYTES * 29); + fma52lo_mem(res30, res30, Bi, inpA_mb, SIMD_BYTES * 30); + fma52lo_mem(res31, res31, Bi, inpA_mb, SIMD_BYTES * 31); + fma52lo_mem(res32, res32, Bi, inpA_mb, SIMD_BYTES * 32); + fma52lo_mem(res33, res33, Bi, inpA_mb, SIMD_BYTES * 33); + fma52lo_mem(res34, res34, Bi, inpA_mb, SIMD_BYTES * 34); + fma52lo_mem(res35, res35, Bi, inpA_mb, SIMD_BYTES * 35); + fma52lo_mem(res36, res36, Bi, inpA_mb, SIMD_BYTES * 36); + fma52lo_mem(res37, res37, Bi, inpA_mb, SIMD_BYTES * 37); + fma52lo_mem(res38, res38, Bi, inpA_mb, SIMD_BYTES * 38); + fma52lo_mem(res39, res39, Bi, inpA_mb, SIMD_BYTES * 39); + Yi = fma52lo(get_zero64(), res00, K); + fma52lo_mem(res00, res00, Yi, inpM_mb, SIMD_BYTES * 0); + fma52lo_mem(res01, res01, Yi, inpM_mb, SIMD_BYTES * 1); + fma52lo_mem(res02, res02, Yi, inpM_mb, SIMD_BYTES * 2); + fma52lo_mem(res03, res03, Yi, inpM_mb, SIMD_BYTES * 3); + fma52lo_mem(res04, res04, Yi, inpM_mb, SIMD_BYTES * 4); + fma52lo_mem(res05, res05, Yi, inpM_mb, SIMD_BYTES * 5); + fma52lo_mem(res06, res06, Yi, inpM_mb, SIMD_BYTES * 6); + fma52lo_mem(res07, res07, Yi, inpM_mb, SIMD_BYTES * 7); + fma52lo_mem(res08, res08, Yi, inpM_mb, SIMD_BYTES * 8); + fma52lo_mem(res09, res09, Yi, inpM_mb, SIMD_BYTES * 9); + fma52lo_mem(res10, res10, Yi, inpM_mb, SIMD_BYTES * 10); + fma52lo_mem(res11, res11, Yi, inpM_mb, SIMD_BYTES * 11); + fma52lo_mem(res12, res12, Yi, inpM_mb, SIMD_BYTES * 12); + fma52lo_mem(res13, res13, Yi, inpM_mb, SIMD_BYTES * 13); + fma52lo_mem(res14, res14, Yi, inpM_mb, SIMD_BYTES * 14); + fma52lo_mem(res15, res15, Yi, inpM_mb, SIMD_BYTES * 15); + fma52lo_mem(res16, res16, Yi, inpM_mb, SIMD_BYTES * 16); + fma52lo_mem(res17, res17, Yi, inpM_mb, SIMD_BYTES * 17); + fma52lo_mem(res18, res18, Yi, inpM_mb, SIMD_BYTES * 18); + fma52lo_mem(res19, res19, Yi, inpM_mb, SIMD_BYTES * 19); + fma52lo_mem(res20, res20, Yi, inpM_mb, SIMD_BYTES * 20); + fma52lo_mem(res21, res21, Yi, inpM_mb, SIMD_BYTES * 21); + fma52lo_mem(res22, res22, Yi, inpM_mb, SIMD_BYTES * 22); + fma52lo_mem(res23, res23, Yi, inpM_mb, SIMD_BYTES * 23); + fma52lo_mem(res24, res24, Yi, inpM_mb, SIMD_BYTES * 24); + fma52lo_mem(res25, res25, Yi, inpM_mb, SIMD_BYTES * 25); + fma52lo_mem(res26, res26, Yi, inpM_mb, SIMD_BYTES * 26); + fma52lo_mem(res27, res27, Yi, inpM_mb, SIMD_BYTES * 27); + fma52lo_mem(res28, res28, Yi, inpM_mb, SIMD_BYTES * 28); + fma52lo_mem(res29, res29, Yi, inpM_mb, SIMD_BYTES * 29); + fma52lo_mem(res30, res30, Yi, inpM_mb, SIMD_BYTES * 30); + fma52lo_mem(res31, res31, Yi, inpM_mb, SIMD_BYTES * 31); + fma52lo_mem(res32, res32, Yi, inpM_mb, SIMD_BYTES * 32); + fma52lo_mem(res33, res33, Yi, inpM_mb, SIMD_BYTES * 33); + fma52lo_mem(res34, res34, Yi, inpM_mb, SIMD_BYTES * 34); + fma52lo_mem(res35, res35, Yi, inpM_mb, SIMD_BYTES * 35); + fma52lo_mem(res36, res36, Yi, inpM_mb, SIMD_BYTES * 36); + fma52lo_mem(res37, res37, Yi, inpM_mb, SIMD_BYTES * 37); + fma52lo_mem(res38, res38, Yi, inpM_mb, SIMD_BYTES * 38); + fma52lo_mem(res39, res39, Yi, inpM_mb, SIMD_BYTES * 39); + res00 = srli64(res00, DIGIT_SIZE); + res01 = add64(res01, res00); + fma52hi_mem(res00, res01, Bi, inpA_mb, SIMD_BYTES * 0); + fma52hi_mem(res01, res02, Bi, inpA_mb, SIMD_BYTES * 1); + fma52hi_mem(res02, res03, Bi, inpA_mb, SIMD_BYTES * 2); + fma52hi_mem(res03, res04, Bi, inpA_mb, SIMD_BYTES * 3); + fma52hi_mem(res04, res05, Bi, inpA_mb, SIMD_BYTES * 4); + fma52hi_mem(res05, res06, Bi, inpA_mb, SIMD_BYTES * 5); + fma52hi_mem(res06, res07, Bi, inpA_mb, SIMD_BYTES * 6); + fma52hi_mem(res07, res08, Bi, inpA_mb, SIMD_BYTES * 7); + fma52hi_mem(res08, res09, Bi, inpA_mb, SIMD_BYTES * 8); + fma52hi_mem(res09, res10, Bi, inpA_mb, SIMD_BYTES * 9); + fma52hi_mem(res10, res11, Bi, inpA_mb, SIMD_BYTES * 10); + fma52hi_mem(res11, res12, Bi, inpA_mb, SIMD_BYTES * 11); + fma52hi_mem(res12, res13, Bi, inpA_mb, SIMD_BYTES * 12); + fma52hi_mem(res13, res14, Bi, inpA_mb, SIMD_BYTES * 13); + fma52hi_mem(res14, res15, Bi, inpA_mb, SIMD_BYTES * 14); + fma52hi_mem(res15, res16, Bi, inpA_mb, SIMD_BYTES * 15); + fma52hi_mem(res16, res17, Bi, inpA_mb, SIMD_BYTES * 16); + fma52hi_mem(res17, res18, Bi, inpA_mb, SIMD_BYTES * 17); + fma52hi_mem(res18, res19, Bi, inpA_mb, SIMD_BYTES * 18); + fma52hi_mem(res19, res20, Bi, inpA_mb, SIMD_BYTES * 19); + fma52hi_mem(res20, res21, Bi, inpA_mb, SIMD_BYTES * 20); + fma52hi_mem(res21, res22, Bi, inpA_mb, SIMD_BYTES * 21); + fma52hi_mem(res22, res23, Bi, inpA_mb, SIMD_BYTES * 22); + fma52hi_mem(res23, res24, Bi, inpA_mb, SIMD_BYTES * 23); + fma52hi_mem(res24, res25, Bi, inpA_mb, SIMD_BYTES * 24); + fma52hi_mem(res25, res26, Bi, inpA_mb, SIMD_BYTES * 25); + fma52hi_mem(res26, res27, Bi, inpA_mb, SIMD_BYTES * 26); + fma52hi_mem(res27, res28, Bi, inpA_mb, SIMD_BYTES * 27); + fma52hi_mem(res28, res29, Bi, inpA_mb, SIMD_BYTES * 28); + fma52hi_mem(res29, res30, Bi, inpA_mb, SIMD_BYTES * 29); + fma52hi_mem(res30, res31, Bi, inpA_mb, SIMD_BYTES * 30); + fma52hi_mem(res31, res32, Bi, inpA_mb, SIMD_BYTES * 31); + fma52hi_mem(res32, res33, Bi, inpA_mb, SIMD_BYTES * 32); + fma52hi_mem(res33, res34, Bi, inpA_mb, SIMD_BYTES * 33); + fma52hi_mem(res34, res35, Bi, inpA_mb, SIMD_BYTES * 34); + fma52hi_mem(res35, res36, Bi, inpA_mb, SIMD_BYTES * 35); + fma52hi_mem(res36, res37, Bi, inpA_mb, SIMD_BYTES * 36); + fma52hi_mem(res37, res38, Bi, inpA_mb, SIMD_BYTES * 37); + fma52hi_mem(res38, res39, Bi, inpA_mb, SIMD_BYTES * 38); + fma52hi_mem(res39, get_zero64(), Bi, inpA_mb, SIMD_BYTES * 39); + fma52hi_mem(res00, res00, Yi, inpM_mb, SIMD_BYTES * 0); + fma52hi_mem(res01, res01, Yi, inpM_mb, SIMD_BYTES * 1); + fma52hi_mem(res02, res02, Yi, inpM_mb, SIMD_BYTES * 2); + fma52hi_mem(res03, res03, Yi, inpM_mb, SIMD_BYTES * 3); + fma52hi_mem(res04, res04, Yi, inpM_mb, SIMD_BYTES * 4); + fma52hi_mem(res05, res05, Yi, inpM_mb, SIMD_BYTES * 5); + fma52hi_mem(res06, res06, Yi, inpM_mb, SIMD_BYTES * 6); + fma52hi_mem(res07, res07, Yi, inpM_mb, SIMD_BYTES * 7); + fma52hi_mem(res08, res08, Yi, inpM_mb, SIMD_BYTES * 8); + fma52hi_mem(res09, res09, Yi, inpM_mb, SIMD_BYTES * 9); + fma52hi_mem(res10, res10, Yi, inpM_mb, SIMD_BYTES * 10); + fma52hi_mem(res11, res11, Yi, inpM_mb, SIMD_BYTES * 11); + fma52hi_mem(res12, res12, Yi, inpM_mb, SIMD_BYTES * 12); + fma52hi_mem(res13, res13, Yi, inpM_mb, SIMD_BYTES * 13); + fma52hi_mem(res14, res14, Yi, inpM_mb, SIMD_BYTES * 14); + fma52hi_mem(res15, res15, Yi, inpM_mb, SIMD_BYTES * 15); + fma52hi_mem(res16, res16, Yi, inpM_mb, SIMD_BYTES * 16); + fma52hi_mem(res17, res17, Yi, inpM_mb, SIMD_BYTES * 17); + fma52hi_mem(res18, res18, Yi, inpM_mb, SIMD_BYTES * 18); + fma52hi_mem(res19, res19, Yi, inpM_mb, SIMD_BYTES * 19); + fma52hi_mem(res20, res20, Yi, inpM_mb, SIMD_BYTES * 20); + fma52hi_mem(res21, res21, Yi, inpM_mb, SIMD_BYTES * 21); + fma52hi_mem(res22, res22, Yi, inpM_mb, SIMD_BYTES * 22); + fma52hi_mem(res23, res23, Yi, inpM_mb, SIMD_BYTES * 23); + fma52hi_mem(res24, res24, Yi, inpM_mb, SIMD_BYTES * 24); + fma52hi_mem(res25, res25, Yi, inpM_mb, SIMD_BYTES * 25); + fma52hi_mem(res26, res26, Yi, inpM_mb, SIMD_BYTES * 26); + fma52hi_mem(res27, res27, Yi, inpM_mb, SIMD_BYTES * 27); + fma52hi_mem(res28, res28, Yi, inpM_mb, SIMD_BYTES * 28); + fma52hi_mem(res29, res29, Yi, inpM_mb, SIMD_BYTES * 29); + fma52hi_mem(res30, res30, Yi, inpM_mb, SIMD_BYTES * 30); + fma52hi_mem(res31, res31, Yi, inpM_mb, SIMD_BYTES * 31); + fma52hi_mem(res32, res32, Yi, inpM_mb, SIMD_BYTES * 32); + fma52hi_mem(res33, res33, Yi, inpM_mb, SIMD_BYTES * 33); + fma52hi_mem(res34, res34, Yi, inpM_mb, SIMD_BYTES * 34); + fma52hi_mem(res35, res35, Yi, inpM_mb, SIMD_BYTES * 35); + fma52hi_mem(res36, res36, Yi, inpM_mb, SIMD_BYTES * 36); + fma52hi_mem(res37, res37, Yi, inpM_mb, SIMD_BYTES * 37); + fma52hi_mem(res38, res38, Yi, inpM_mb, SIMD_BYTES * 38); + fma52hi_mem(res39, res39, Yi, inpM_mb, SIMD_BYTES * 39); + } + // Normalization + { + U64 T = get_zero64(); + U64 MASK = set64(DIGIT_MASK); + T = srli64(res00, DIGIT_SIZE); + res00 = and64(res00, MASK); + storeu64(out_mb + MB_WIDTH * 0, res00); + res01 = add64(res01, T); + T = srli64(res01, DIGIT_SIZE); + res01 = and64(res01, MASK); + storeu64(out_mb + MB_WIDTH * 1, res01); + res02 = add64(res02, T); + T = srli64(res02, DIGIT_SIZE); + res02 = and64(res02, MASK); + storeu64(out_mb + MB_WIDTH * 2, res02); + res03 = add64(res03, T); + T = srli64(res03, DIGIT_SIZE); + res03 = and64(res03, MASK); + storeu64(out_mb + MB_WIDTH * 3, res03); + res04 = add64(res04, T); + T = srli64(res04, DIGIT_SIZE); + res04 = and64(res04, MASK); + storeu64(out_mb + MB_WIDTH * 4, res04); + res05 = add64(res05, T); + T = srli64(res05, DIGIT_SIZE); + res05 = and64(res05, MASK); + storeu64(out_mb + MB_WIDTH * 5, res05); + res06 = add64(res06, T); + T = srli64(res06, DIGIT_SIZE); + res06 = and64(res06, MASK); + storeu64(out_mb + MB_WIDTH * 6, res06); + res07 = add64(res07, T); + T = srli64(res07, DIGIT_SIZE); + res07 = and64(res07, MASK); + storeu64(out_mb + MB_WIDTH * 7, res07); + res08 = add64(res08, T); + T = srli64(res08, DIGIT_SIZE); + res08 = and64(res08, MASK); + storeu64(out_mb + MB_WIDTH * 8, res08); + res09 = add64(res09, T); + T = srli64(res09, DIGIT_SIZE); + res09 = and64(res09, MASK); + storeu64(out_mb + MB_WIDTH * 9, res09); + res10 = add64(res10, T); + T = srli64(res10, DIGIT_SIZE); + res10 = and64(res10, MASK); + storeu64(out_mb + MB_WIDTH * 10, res10); + res11 = add64(res11, T); + T = srli64(res11, DIGIT_SIZE); + res11 = and64(res11, MASK); + storeu64(out_mb + MB_WIDTH * 11, res11); + res12 = add64(res12, T); + T = srli64(res12, DIGIT_SIZE); + res12 = and64(res12, MASK); + storeu64(out_mb + MB_WIDTH * 12, res12); + res13 = add64(res13, T); + T = srli64(res13, DIGIT_SIZE); + res13 = and64(res13, MASK); + storeu64(out_mb + MB_WIDTH * 13, res13); + res14 = add64(res14, T); + T = srli64(res14, DIGIT_SIZE); + res14 = and64(res14, MASK); + storeu64(out_mb + MB_WIDTH * 14, res14); + res15 = add64(res15, T); + T = srli64(res15, DIGIT_SIZE); + res15 = and64(res15, MASK); + storeu64(out_mb + MB_WIDTH * 15, res15); + res16 = add64(res16, T); + T = srli64(res16, DIGIT_SIZE); + res16 = and64(res16, MASK); + storeu64(out_mb + MB_WIDTH * 16, res16); + res17 = add64(res17, T); + T = srli64(res17, DIGIT_SIZE); + res17 = and64(res17, MASK); + storeu64(out_mb + MB_WIDTH * 17, res17); + res18 = add64(res18, T); + T = srli64(res18, DIGIT_SIZE); + res18 = and64(res18, MASK); + storeu64(out_mb + MB_WIDTH * 18, res18); + res19 = add64(res19, T); + T = srli64(res19, DIGIT_SIZE); + res19 = and64(res19, MASK); + storeu64(out_mb + MB_WIDTH * 19, res19); + res20 = add64(res20, T); + T = srli64(res20, DIGIT_SIZE); + res20 = and64(res20, MASK); + storeu64(out_mb + MB_WIDTH * 20, res20); + res21 = add64(res21, T); + T = srli64(res21, DIGIT_SIZE); + res21 = and64(res21, MASK); + storeu64(out_mb + MB_WIDTH * 21, res21); + res22 = add64(res22, T); + T = srli64(res22, DIGIT_SIZE); + res22 = and64(res22, MASK); + storeu64(out_mb + MB_WIDTH * 22, res22); + res23 = add64(res23, T); + T = srli64(res23, DIGIT_SIZE); + res23 = and64(res23, MASK); + storeu64(out_mb + MB_WIDTH * 23, res23); + res24 = add64(res24, T); + T = srli64(res24, DIGIT_SIZE); + res24 = and64(res24, MASK); + storeu64(out_mb + MB_WIDTH * 24, res24); + res25 = add64(res25, T); + T = srli64(res25, DIGIT_SIZE); + res25 = and64(res25, MASK); + storeu64(out_mb + MB_WIDTH * 25, res25); + res26 = add64(res26, T); + T = srli64(res26, DIGIT_SIZE); + res26 = and64(res26, MASK); + storeu64(out_mb + MB_WIDTH * 26, res26); + res27 = add64(res27, T); + T = srli64(res27, DIGIT_SIZE); + res27 = and64(res27, MASK); + storeu64(out_mb + MB_WIDTH * 27, res27); + res28 = add64(res28, T); + T = srli64(res28, DIGIT_SIZE); + res28 = and64(res28, MASK); + storeu64(out_mb + MB_WIDTH * 28, res28); + res29 = add64(res29, T); + T = srli64(res29, DIGIT_SIZE); + res29 = and64(res29, MASK); + storeu64(out_mb + MB_WIDTH * 29, res29); + res30 = add64(res30, T); + T = srli64(res30, DIGIT_SIZE); + res30 = and64(res30, MASK); + storeu64(out_mb + MB_WIDTH * 30, res30); + res31 = add64(res31, T); + T = srli64(res31, DIGIT_SIZE); + res31 = and64(res31, MASK); + storeu64(out_mb + MB_WIDTH * 31, res31); + res32 = add64(res32, T); + T = srli64(res32, DIGIT_SIZE); + res32 = and64(res32, MASK); + storeu64(out_mb + MB_WIDTH * 32, res32); + res33 = add64(res33, T); + T = srli64(res33, DIGIT_SIZE); + res33 = and64(res33, MASK); + storeu64(out_mb + MB_WIDTH * 33, res33); + res34 = add64(res34, T); + T = srli64(res34, DIGIT_SIZE); + res34 = and64(res34, MASK); + storeu64(out_mb + MB_WIDTH * 34, res34); + res35 = add64(res35, T); + T = srli64(res35, DIGIT_SIZE); + res35 = and64(res35, MASK); + storeu64(out_mb + MB_WIDTH * 35, res35); + res36 = add64(res36, T); + T = srli64(res36, DIGIT_SIZE); + res36 = and64(res36, MASK); + storeu64(out_mb + MB_WIDTH * 36, res36); + res37 = add64(res37, T); + T = srli64(res37, DIGIT_SIZE); + res37 = and64(res37, MASK); + storeu64(out_mb + MB_WIDTH * 37, res37); + res38 = add64(res38, T); + T = srli64(res38, DIGIT_SIZE); + res38 = and64(res38, MASK); + storeu64(out_mb + MB_WIDTH * 38, res38); + res39 = add64(res39, T); + res39 = and64(res39, MASK); + storeu64(out_mb + MB_WIDTH * 39, res39); + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_amm52x60_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_amm52x60_mb8.c new file mode 100644 index 0000000..2b6f9fb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_amm52x60_mb8.c @@ -0,0 +1,532 @@ +/******************************************************************************* + * Copyright (C) 2019 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include + +void ifma_amm52x60_mb8(int64u *out_mb, const int64u *inpA_mb, + const int64u *inpB_mb, const int64u *inpM_mb, + const int64u *k0_mb) { + U64 res00, res01, res02, res03, res04, res05, res06, res07, res08, res09, + res10, res11, res12, res13, res14, res15, res16, res17, res18, res19, + res20, res21, res22, res23, res24, res25, res26, res27, res28, res29, + res30, res31, res32, res33, res34, res35, res36, res37, res38, res39, + res40, res41, res42, res43, res44, res45, res46, res47, res48, res49, + res50, res51, res52, res53, res54, res55, res56, res57, res58, res59; + U64 K = loadu64(k0_mb); + int itr; + res00 = res01 = res02 = res03 = res04 = res05 = res06 = res07 = res08 = + res09 = res10 = res11 = res12 = res13 = res14 = res15 = res16 = res17 = + res18 = res19 = res20 = res21 = res22 = res23 = res24 = res25 = + res26 = res27 = res28 = res29 = res30 = res31 = res32 = res33 = + res34 = res35 = res36 = res37 = res38 = res39 = res40 = + res41 = res42 = res43 = res44 = res45 = res46 = res47 = + res48 = res49 = res50 = res51 = res52 = res53 = + res54 = res55 = res56 = res57 = res58 = res59 = + get_zero64(); + + for (itr = 0; itr < 60; itr++) { + U64 Yi; + U64 Bi = loadu64(inpB_mb); + inpB_mb += MB_WIDTH; + fma52lo_mem(res00, res00, Bi, inpA_mb, SIMD_BYTES * 0); + fma52lo_mem(res01, res01, Bi, inpA_mb, SIMD_BYTES * 1); + fma52lo_mem(res02, res02, Bi, inpA_mb, SIMD_BYTES * 2); + fma52lo_mem(res03, res03, Bi, inpA_mb, SIMD_BYTES * 3); + fma52lo_mem(res04, res04, Bi, inpA_mb, SIMD_BYTES * 4); + fma52lo_mem(res05, res05, Bi, inpA_mb, SIMD_BYTES * 5); + fma52lo_mem(res06, res06, Bi, inpA_mb, SIMD_BYTES * 6); + fma52lo_mem(res07, res07, Bi, inpA_mb, SIMD_BYTES * 7); + fma52lo_mem(res08, res08, Bi, inpA_mb, SIMD_BYTES * 8); + fma52lo_mem(res09, res09, Bi, inpA_mb, SIMD_BYTES * 9); + fma52lo_mem(res10, res10, Bi, inpA_mb, SIMD_BYTES * 10); + fma52lo_mem(res11, res11, Bi, inpA_mb, SIMD_BYTES * 11); + fma52lo_mem(res12, res12, Bi, inpA_mb, SIMD_BYTES * 12); + fma52lo_mem(res13, res13, Bi, inpA_mb, SIMD_BYTES * 13); + fma52lo_mem(res14, res14, Bi, inpA_mb, SIMD_BYTES * 14); + fma52lo_mem(res15, res15, Bi, inpA_mb, SIMD_BYTES * 15); + fma52lo_mem(res16, res16, Bi, inpA_mb, SIMD_BYTES * 16); + fma52lo_mem(res17, res17, Bi, inpA_mb, SIMD_BYTES * 17); + fma52lo_mem(res18, res18, Bi, inpA_mb, SIMD_BYTES * 18); + fma52lo_mem(res19, res19, Bi, inpA_mb, SIMD_BYTES * 19); + fma52lo_mem(res20, res20, Bi, inpA_mb, SIMD_BYTES * 20); + fma52lo_mem(res21, res21, Bi, inpA_mb, SIMD_BYTES * 21); + fma52lo_mem(res22, res22, Bi, inpA_mb, SIMD_BYTES * 22); + fma52lo_mem(res23, res23, Bi, inpA_mb, SIMD_BYTES * 23); + fma52lo_mem(res24, res24, Bi, inpA_mb, SIMD_BYTES * 24); + fma52lo_mem(res25, res25, Bi, inpA_mb, SIMD_BYTES * 25); + fma52lo_mem(res26, res26, Bi, inpA_mb, SIMD_BYTES * 26); + fma52lo_mem(res27, res27, Bi, inpA_mb, SIMD_BYTES * 27); + fma52lo_mem(res28, res28, Bi, inpA_mb, SIMD_BYTES * 28); + fma52lo_mem(res29, res29, Bi, inpA_mb, SIMD_BYTES * 29); + fma52lo_mem(res30, res30, Bi, inpA_mb, SIMD_BYTES * 30); + fma52lo_mem(res31, res31, Bi, inpA_mb, SIMD_BYTES * 31); + fma52lo_mem(res32, res32, Bi, inpA_mb, SIMD_BYTES * 32); + fma52lo_mem(res33, res33, Bi, inpA_mb, SIMD_BYTES * 33); + fma52lo_mem(res34, res34, Bi, inpA_mb, SIMD_BYTES * 34); + fma52lo_mem(res35, res35, Bi, inpA_mb, SIMD_BYTES * 35); + fma52lo_mem(res36, res36, Bi, inpA_mb, SIMD_BYTES * 36); + fma52lo_mem(res37, res37, Bi, inpA_mb, SIMD_BYTES * 37); + fma52lo_mem(res38, res38, Bi, inpA_mb, SIMD_BYTES * 38); + fma52lo_mem(res39, res39, Bi, inpA_mb, SIMD_BYTES * 39); + fma52lo_mem(res40, res40, Bi, inpA_mb, SIMD_BYTES * 40); + fma52lo_mem(res41, res41, Bi, inpA_mb, SIMD_BYTES * 41); + fma52lo_mem(res42, res42, Bi, inpA_mb, SIMD_BYTES * 42); + fma52lo_mem(res43, res43, Bi, inpA_mb, SIMD_BYTES * 43); + fma52lo_mem(res44, res44, Bi, inpA_mb, SIMD_BYTES * 44); + fma52lo_mem(res45, res45, Bi, inpA_mb, SIMD_BYTES * 45); + fma52lo_mem(res46, res46, Bi, inpA_mb, SIMD_BYTES * 46); + fma52lo_mem(res47, res47, Bi, inpA_mb, SIMD_BYTES * 47); + fma52lo_mem(res48, res48, Bi, inpA_mb, SIMD_BYTES * 48); + fma52lo_mem(res49, res49, Bi, inpA_mb, SIMD_BYTES * 49); + fma52lo_mem(res50, res50, Bi, inpA_mb, SIMD_BYTES * 50); + fma52lo_mem(res51, res51, Bi, inpA_mb, SIMD_BYTES * 51); + fma52lo_mem(res52, res52, Bi, inpA_mb, SIMD_BYTES * 52); + fma52lo_mem(res53, res53, Bi, inpA_mb, SIMD_BYTES * 53); + fma52lo_mem(res54, res54, Bi, inpA_mb, SIMD_BYTES * 54); + fma52lo_mem(res55, res55, Bi, inpA_mb, SIMD_BYTES * 55); + fma52lo_mem(res56, res56, Bi, inpA_mb, SIMD_BYTES * 56); + fma52lo_mem(res57, res57, Bi, inpA_mb, SIMD_BYTES * 57); + fma52lo_mem(res58, res58, Bi, inpA_mb, SIMD_BYTES * 58); + fma52lo_mem(res59, res59, Bi, inpA_mb, SIMD_BYTES * 59); + Yi = fma52lo(get_zero64(), res00, K); + fma52lo_mem(res00, res00, Yi, inpM_mb, SIMD_BYTES * 0); + fma52lo_mem(res01, res01, Yi, inpM_mb, SIMD_BYTES * 1); + fma52lo_mem(res02, res02, Yi, inpM_mb, SIMD_BYTES * 2); + fma52lo_mem(res03, res03, Yi, inpM_mb, SIMD_BYTES * 3); + fma52lo_mem(res04, res04, Yi, inpM_mb, SIMD_BYTES * 4); + fma52lo_mem(res05, res05, Yi, inpM_mb, SIMD_BYTES * 5); + fma52lo_mem(res06, res06, Yi, inpM_mb, SIMD_BYTES * 6); + fma52lo_mem(res07, res07, Yi, inpM_mb, SIMD_BYTES * 7); + fma52lo_mem(res08, res08, Yi, inpM_mb, SIMD_BYTES * 8); + fma52lo_mem(res09, res09, Yi, inpM_mb, SIMD_BYTES * 9); + fma52lo_mem(res10, res10, Yi, inpM_mb, SIMD_BYTES * 10); + fma52lo_mem(res11, res11, Yi, inpM_mb, SIMD_BYTES * 11); + fma52lo_mem(res12, res12, Yi, inpM_mb, SIMD_BYTES * 12); + fma52lo_mem(res13, res13, Yi, inpM_mb, SIMD_BYTES * 13); + fma52lo_mem(res14, res14, Yi, inpM_mb, SIMD_BYTES * 14); + fma52lo_mem(res15, res15, Yi, inpM_mb, SIMD_BYTES * 15); + fma52lo_mem(res16, res16, Yi, inpM_mb, SIMD_BYTES * 16); + fma52lo_mem(res17, res17, Yi, inpM_mb, SIMD_BYTES * 17); + fma52lo_mem(res18, res18, Yi, inpM_mb, SIMD_BYTES * 18); + fma52lo_mem(res19, res19, Yi, inpM_mb, SIMD_BYTES * 19); + fma52lo_mem(res20, res20, Yi, inpM_mb, SIMD_BYTES * 20); + fma52lo_mem(res21, res21, Yi, inpM_mb, SIMD_BYTES * 21); + fma52lo_mem(res22, res22, Yi, inpM_mb, SIMD_BYTES * 22); + fma52lo_mem(res23, res23, Yi, inpM_mb, SIMD_BYTES * 23); + fma52lo_mem(res24, res24, Yi, inpM_mb, SIMD_BYTES * 24); + fma52lo_mem(res25, res25, Yi, inpM_mb, SIMD_BYTES * 25); + fma52lo_mem(res26, res26, Yi, inpM_mb, SIMD_BYTES * 26); + fma52lo_mem(res27, res27, Yi, inpM_mb, SIMD_BYTES * 27); + fma52lo_mem(res28, res28, Yi, inpM_mb, SIMD_BYTES * 28); + fma52lo_mem(res29, res29, Yi, inpM_mb, SIMD_BYTES * 29); + fma52lo_mem(res30, res30, Yi, inpM_mb, SIMD_BYTES * 30); + fma52lo_mem(res31, res31, Yi, inpM_mb, SIMD_BYTES * 31); + fma52lo_mem(res32, res32, Yi, inpM_mb, SIMD_BYTES * 32); + fma52lo_mem(res33, res33, Yi, inpM_mb, SIMD_BYTES * 33); + fma52lo_mem(res34, res34, Yi, inpM_mb, SIMD_BYTES * 34); + fma52lo_mem(res35, res35, Yi, inpM_mb, SIMD_BYTES * 35); + fma52lo_mem(res36, res36, Yi, inpM_mb, SIMD_BYTES * 36); + fma52lo_mem(res37, res37, Yi, inpM_mb, SIMD_BYTES * 37); + fma52lo_mem(res38, res38, Yi, inpM_mb, SIMD_BYTES * 38); + fma52lo_mem(res39, res39, Yi, inpM_mb, SIMD_BYTES * 39); + fma52lo_mem(res40, res40, Yi, inpM_mb, SIMD_BYTES * 40); + fma52lo_mem(res41, res41, Yi, inpM_mb, SIMD_BYTES * 41); + fma52lo_mem(res42, res42, Yi, inpM_mb, SIMD_BYTES * 42); + fma52lo_mem(res43, res43, Yi, inpM_mb, SIMD_BYTES * 43); + fma52lo_mem(res44, res44, Yi, inpM_mb, SIMD_BYTES * 44); + fma52lo_mem(res45, res45, Yi, inpM_mb, SIMD_BYTES * 45); + fma52lo_mem(res46, res46, Yi, inpM_mb, SIMD_BYTES * 46); + fma52lo_mem(res47, res47, Yi, inpM_mb, SIMD_BYTES * 47); + fma52lo_mem(res48, res48, Yi, inpM_mb, SIMD_BYTES * 48); + fma52lo_mem(res49, res49, Yi, inpM_mb, SIMD_BYTES * 49); + fma52lo_mem(res50, res50, Yi, inpM_mb, SIMD_BYTES * 50); + fma52lo_mem(res51, res51, Yi, inpM_mb, SIMD_BYTES * 51); + fma52lo_mem(res52, res52, Yi, inpM_mb, SIMD_BYTES * 52); + fma52lo_mem(res53, res53, Yi, inpM_mb, SIMD_BYTES * 53); + fma52lo_mem(res54, res54, Yi, inpM_mb, SIMD_BYTES * 54); + fma52lo_mem(res55, res55, Yi, inpM_mb, SIMD_BYTES * 55); + fma52lo_mem(res56, res56, Yi, inpM_mb, SIMD_BYTES * 56); + fma52lo_mem(res57, res57, Yi, inpM_mb, SIMD_BYTES * 57); + fma52lo_mem(res58, res58, Yi, inpM_mb, SIMD_BYTES * 58); + fma52lo_mem(res59, res59, Yi, inpM_mb, SIMD_BYTES * 59); + res00 = srli64(res00, DIGIT_SIZE); + res01 = add64(res01, res00); + fma52hi_mem(res00, res01, Bi, inpA_mb, SIMD_BYTES * 0); + fma52hi_mem(res01, res02, Bi, inpA_mb, SIMD_BYTES * 1); + fma52hi_mem(res02, res03, Bi, inpA_mb, SIMD_BYTES * 2); + fma52hi_mem(res03, res04, Bi, inpA_mb, SIMD_BYTES * 3); + fma52hi_mem(res04, res05, Bi, inpA_mb, SIMD_BYTES * 4); + fma52hi_mem(res05, res06, Bi, inpA_mb, SIMD_BYTES * 5); + fma52hi_mem(res06, res07, Bi, inpA_mb, SIMD_BYTES * 6); + fma52hi_mem(res07, res08, Bi, inpA_mb, SIMD_BYTES * 7); + fma52hi_mem(res08, res09, Bi, inpA_mb, SIMD_BYTES * 8); + fma52hi_mem(res09, res10, Bi, inpA_mb, SIMD_BYTES * 9); + fma52hi_mem(res10, res11, Bi, inpA_mb, SIMD_BYTES * 10); + fma52hi_mem(res11, res12, Bi, inpA_mb, SIMD_BYTES * 11); + fma52hi_mem(res12, res13, Bi, inpA_mb, SIMD_BYTES * 12); + fma52hi_mem(res13, res14, Bi, inpA_mb, SIMD_BYTES * 13); + fma52hi_mem(res14, res15, Bi, inpA_mb, SIMD_BYTES * 14); + fma52hi_mem(res15, res16, Bi, inpA_mb, SIMD_BYTES * 15); + fma52hi_mem(res16, res17, Bi, inpA_mb, SIMD_BYTES * 16); + fma52hi_mem(res17, res18, Bi, inpA_mb, SIMD_BYTES * 17); + fma52hi_mem(res18, res19, Bi, inpA_mb, SIMD_BYTES * 18); + fma52hi_mem(res19, res20, Bi, inpA_mb, SIMD_BYTES * 19); + fma52hi_mem(res20, res21, Bi, inpA_mb, SIMD_BYTES * 20); + fma52hi_mem(res21, res22, Bi, inpA_mb, SIMD_BYTES * 21); + fma52hi_mem(res22, res23, Bi, inpA_mb, SIMD_BYTES * 22); + fma52hi_mem(res23, res24, Bi, inpA_mb, SIMD_BYTES * 23); + fma52hi_mem(res24, res25, Bi, inpA_mb, SIMD_BYTES * 24); + fma52hi_mem(res25, res26, Bi, inpA_mb, SIMD_BYTES * 25); + fma52hi_mem(res26, res27, Bi, inpA_mb, SIMD_BYTES * 26); + fma52hi_mem(res27, res28, Bi, inpA_mb, SIMD_BYTES * 27); + fma52hi_mem(res28, res29, Bi, inpA_mb, SIMD_BYTES * 28); + fma52hi_mem(res29, res30, Bi, inpA_mb, SIMD_BYTES * 29); + fma52hi_mem(res30, res31, Bi, inpA_mb, SIMD_BYTES * 30); + fma52hi_mem(res31, res32, Bi, inpA_mb, SIMD_BYTES * 31); + fma52hi_mem(res32, res33, Bi, inpA_mb, SIMD_BYTES * 32); + fma52hi_mem(res33, res34, Bi, inpA_mb, SIMD_BYTES * 33); + fma52hi_mem(res34, res35, Bi, inpA_mb, SIMD_BYTES * 34); + fma52hi_mem(res35, res36, Bi, inpA_mb, SIMD_BYTES * 35); + fma52hi_mem(res36, res37, Bi, inpA_mb, SIMD_BYTES * 36); + fma52hi_mem(res37, res38, Bi, inpA_mb, SIMD_BYTES * 37); + fma52hi_mem(res38, res39, Bi, inpA_mb, SIMD_BYTES * 38); + fma52hi_mem(res39, res40, Bi, inpA_mb, SIMD_BYTES * 39); + fma52hi_mem(res40, res41, Bi, inpA_mb, SIMD_BYTES * 40); + fma52hi_mem(res41, res42, Bi, inpA_mb, SIMD_BYTES * 41); + fma52hi_mem(res42, res43, Bi, inpA_mb, SIMD_BYTES * 42); + fma52hi_mem(res43, res44, Bi, inpA_mb, SIMD_BYTES * 43); + fma52hi_mem(res44, res45, Bi, inpA_mb, SIMD_BYTES * 44); + fma52hi_mem(res45, res46, Bi, inpA_mb, SIMD_BYTES * 45); + fma52hi_mem(res46, res47, Bi, inpA_mb, SIMD_BYTES * 46); + fma52hi_mem(res47, res48, Bi, inpA_mb, SIMD_BYTES * 47); + fma52hi_mem(res48, res49, Bi, inpA_mb, SIMD_BYTES * 48); + fma52hi_mem(res49, res50, Bi, inpA_mb, SIMD_BYTES * 49); + fma52hi_mem(res50, res51, Bi, inpA_mb, SIMD_BYTES * 50); + fma52hi_mem(res51, res52, Bi, inpA_mb, SIMD_BYTES * 51); + fma52hi_mem(res52, res53, Bi, inpA_mb, SIMD_BYTES * 52); + fma52hi_mem(res53, res54, Bi, inpA_mb, SIMD_BYTES * 53); + fma52hi_mem(res54, res55, Bi, inpA_mb, SIMD_BYTES * 54); + fma52hi_mem(res55, res56, Bi, inpA_mb, SIMD_BYTES * 55); + fma52hi_mem(res56, res57, Bi, inpA_mb, SIMD_BYTES * 56); + fma52hi_mem(res57, res58, Bi, inpA_mb, SIMD_BYTES * 57); + fma52hi_mem(res58, res59, Bi, inpA_mb, SIMD_BYTES * 58); + fma52hi_mem(res59, get_zero64(), Bi, inpA_mb, SIMD_BYTES * 59); + fma52hi_mem(res00, res00, Yi, inpM_mb, SIMD_BYTES * 0); + fma52hi_mem(res01, res01, Yi, inpM_mb, SIMD_BYTES * 1); + fma52hi_mem(res02, res02, Yi, inpM_mb, SIMD_BYTES * 2); + fma52hi_mem(res03, res03, Yi, inpM_mb, SIMD_BYTES * 3); + fma52hi_mem(res04, res04, Yi, inpM_mb, SIMD_BYTES * 4); + fma52hi_mem(res05, res05, Yi, inpM_mb, SIMD_BYTES * 5); + fma52hi_mem(res06, res06, Yi, inpM_mb, SIMD_BYTES * 6); + fma52hi_mem(res07, res07, Yi, inpM_mb, SIMD_BYTES * 7); + fma52hi_mem(res08, res08, Yi, inpM_mb, SIMD_BYTES * 8); + fma52hi_mem(res09, res09, Yi, inpM_mb, SIMD_BYTES * 9); + fma52hi_mem(res10, res10, Yi, inpM_mb, SIMD_BYTES * 10); + fma52hi_mem(res11, res11, Yi, inpM_mb, SIMD_BYTES * 11); + fma52hi_mem(res12, res12, Yi, inpM_mb, SIMD_BYTES * 12); + fma52hi_mem(res13, res13, Yi, inpM_mb, SIMD_BYTES * 13); + fma52hi_mem(res14, res14, Yi, inpM_mb, SIMD_BYTES * 14); + fma52hi_mem(res15, res15, Yi, inpM_mb, SIMD_BYTES * 15); + fma52hi_mem(res16, res16, Yi, inpM_mb, SIMD_BYTES * 16); + fma52hi_mem(res17, res17, Yi, inpM_mb, SIMD_BYTES * 17); + fma52hi_mem(res18, res18, Yi, inpM_mb, SIMD_BYTES * 18); + fma52hi_mem(res19, res19, Yi, inpM_mb, SIMD_BYTES * 19); + fma52hi_mem(res20, res20, Yi, inpM_mb, SIMD_BYTES * 20); + fma52hi_mem(res21, res21, Yi, inpM_mb, SIMD_BYTES * 21); + fma52hi_mem(res22, res22, Yi, inpM_mb, SIMD_BYTES * 22); + fma52hi_mem(res23, res23, Yi, inpM_mb, SIMD_BYTES * 23); + fma52hi_mem(res24, res24, Yi, inpM_mb, SIMD_BYTES * 24); + fma52hi_mem(res25, res25, Yi, inpM_mb, SIMD_BYTES * 25); + fma52hi_mem(res26, res26, Yi, inpM_mb, SIMD_BYTES * 26); + fma52hi_mem(res27, res27, Yi, inpM_mb, SIMD_BYTES * 27); + fma52hi_mem(res28, res28, Yi, inpM_mb, SIMD_BYTES * 28); + fma52hi_mem(res29, res29, Yi, inpM_mb, SIMD_BYTES * 29); + fma52hi_mem(res30, res30, Yi, inpM_mb, SIMD_BYTES * 30); + fma52hi_mem(res31, res31, Yi, inpM_mb, SIMD_BYTES * 31); + fma52hi_mem(res32, res32, Yi, inpM_mb, SIMD_BYTES * 32); + fma52hi_mem(res33, res33, Yi, inpM_mb, SIMD_BYTES * 33); + fma52hi_mem(res34, res34, Yi, inpM_mb, SIMD_BYTES * 34); + fma52hi_mem(res35, res35, Yi, inpM_mb, SIMD_BYTES * 35); + fma52hi_mem(res36, res36, Yi, inpM_mb, SIMD_BYTES * 36); + fma52hi_mem(res37, res37, Yi, inpM_mb, SIMD_BYTES * 37); + fma52hi_mem(res38, res38, Yi, inpM_mb, SIMD_BYTES * 38); + fma52hi_mem(res39, res39, Yi, inpM_mb, SIMD_BYTES * 39); + fma52hi_mem(res40, res40, Yi, inpM_mb, SIMD_BYTES * 40); + fma52hi_mem(res41, res41, Yi, inpM_mb, SIMD_BYTES * 41); + fma52hi_mem(res42, res42, Yi, inpM_mb, SIMD_BYTES * 42); + fma52hi_mem(res43, res43, Yi, inpM_mb, SIMD_BYTES * 43); + fma52hi_mem(res44, res44, Yi, inpM_mb, SIMD_BYTES * 44); + fma52hi_mem(res45, res45, Yi, inpM_mb, SIMD_BYTES * 45); + fma52hi_mem(res46, res46, Yi, inpM_mb, SIMD_BYTES * 46); + fma52hi_mem(res47, res47, Yi, inpM_mb, SIMD_BYTES * 47); + fma52hi_mem(res48, res48, Yi, inpM_mb, SIMD_BYTES * 48); + fma52hi_mem(res49, res49, Yi, inpM_mb, SIMD_BYTES * 49); + fma52hi_mem(res50, res50, Yi, inpM_mb, SIMD_BYTES * 50); + fma52hi_mem(res51, res51, Yi, inpM_mb, SIMD_BYTES * 51); + fma52hi_mem(res52, res52, Yi, inpM_mb, SIMD_BYTES * 52); + fma52hi_mem(res53, res53, Yi, inpM_mb, SIMD_BYTES * 53); + fma52hi_mem(res54, res54, Yi, inpM_mb, SIMD_BYTES * 54); + fma52hi_mem(res55, res55, Yi, inpM_mb, SIMD_BYTES * 55); + fma52hi_mem(res56, res56, Yi, inpM_mb, SIMD_BYTES * 56); + fma52hi_mem(res57, res57, Yi, inpM_mb, SIMD_BYTES * 57); + fma52hi_mem(res58, res58, Yi, inpM_mb, SIMD_BYTES * 58); + fma52hi_mem(res59, res59, Yi, inpM_mb, SIMD_BYTES * 59); + } + // Normalization + { + U64 T = get_zero64(); + U64 MASK = set64(DIGIT_MASK); + T = srli64(res00, DIGIT_SIZE); + res00 = and64(res00, MASK); + storeu64(out_mb + MB_WIDTH * 0, res00); + res01 = add64(res01, T); + T = srli64(res01, DIGIT_SIZE); + res01 = and64(res01, MASK); + storeu64(out_mb + MB_WIDTH * 1, res01); + res02 = add64(res02, T); + T = srli64(res02, DIGIT_SIZE); + res02 = and64(res02, MASK); + storeu64(out_mb + MB_WIDTH * 2, res02); + res03 = add64(res03, T); + T = srli64(res03, DIGIT_SIZE); + res03 = and64(res03, MASK); + storeu64(out_mb + MB_WIDTH * 3, res03); + res04 = add64(res04, T); + T = srli64(res04, DIGIT_SIZE); + res04 = and64(res04, MASK); + storeu64(out_mb + MB_WIDTH * 4, res04); + res05 = add64(res05, T); + T = srli64(res05, DIGIT_SIZE); + res05 = and64(res05, MASK); + storeu64(out_mb + MB_WIDTH * 5, res05); + res06 = add64(res06, T); + T = srli64(res06, DIGIT_SIZE); + res06 = and64(res06, MASK); + storeu64(out_mb + MB_WIDTH * 6, res06); + res07 = add64(res07, T); + T = srli64(res07, DIGIT_SIZE); + res07 = and64(res07, MASK); + storeu64(out_mb + MB_WIDTH * 7, res07); + res08 = add64(res08, T); + T = srli64(res08, DIGIT_SIZE); + res08 = and64(res08, MASK); + storeu64(out_mb + MB_WIDTH * 8, res08); + res09 = add64(res09, T); + T = srli64(res09, DIGIT_SIZE); + res09 = and64(res09, MASK); + storeu64(out_mb + MB_WIDTH * 9, res09); + res10 = add64(res10, T); + T = srli64(res10, DIGIT_SIZE); + res10 = and64(res10, MASK); + storeu64(out_mb + MB_WIDTH * 10, res10); + res11 = add64(res11, T); + T = srli64(res11, DIGIT_SIZE); + res11 = and64(res11, MASK); + storeu64(out_mb + MB_WIDTH * 11, res11); + res12 = add64(res12, T); + T = srli64(res12, DIGIT_SIZE); + res12 = and64(res12, MASK); + storeu64(out_mb + MB_WIDTH * 12, res12); + res13 = add64(res13, T); + T = srli64(res13, DIGIT_SIZE); + res13 = and64(res13, MASK); + storeu64(out_mb + MB_WIDTH * 13, res13); + res14 = add64(res14, T); + T = srli64(res14, DIGIT_SIZE); + res14 = and64(res14, MASK); + storeu64(out_mb + MB_WIDTH * 14, res14); + res15 = add64(res15, T); + T = srli64(res15, DIGIT_SIZE); + res15 = and64(res15, MASK); + storeu64(out_mb + MB_WIDTH * 15, res15); + res16 = add64(res16, T); + T = srli64(res16, DIGIT_SIZE); + res16 = and64(res16, MASK); + storeu64(out_mb + MB_WIDTH * 16, res16); + res17 = add64(res17, T); + T = srli64(res17, DIGIT_SIZE); + res17 = and64(res17, MASK); + storeu64(out_mb + MB_WIDTH * 17, res17); + res18 = add64(res18, T); + T = srli64(res18, DIGIT_SIZE); + res18 = and64(res18, MASK); + storeu64(out_mb + MB_WIDTH * 18, res18); + res19 = add64(res19, T); + T = srli64(res19, DIGIT_SIZE); + res19 = and64(res19, MASK); + storeu64(out_mb + MB_WIDTH * 19, res19); + res20 = add64(res20, T); + T = srli64(res20, DIGIT_SIZE); + res20 = and64(res20, MASK); + storeu64(out_mb + MB_WIDTH * 20, res20); + res21 = add64(res21, T); + T = srli64(res21, DIGIT_SIZE); + res21 = and64(res21, MASK); + storeu64(out_mb + MB_WIDTH * 21, res21); + res22 = add64(res22, T); + T = srli64(res22, DIGIT_SIZE); + res22 = and64(res22, MASK); + storeu64(out_mb + MB_WIDTH * 22, res22); + res23 = add64(res23, T); + T = srli64(res23, DIGIT_SIZE); + res23 = and64(res23, MASK); + storeu64(out_mb + MB_WIDTH * 23, res23); + res24 = add64(res24, T); + T = srli64(res24, DIGIT_SIZE); + res24 = and64(res24, MASK); + storeu64(out_mb + MB_WIDTH * 24, res24); + res25 = add64(res25, T); + T = srli64(res25, DIGIT_SIZE); + res25 = and64(res25, MASK); + storeu64(out_mb + MB_WIDTH * 25, res25); + res26 = add64(res26, T); + T = srli64(res26, DIGIT_SIZE); + res26 = and64(res26, MASK); + storeu64(out_mb + MB_WIDTH * 26, res26); + res27 = add64(res27, T); + T = srli64(res27, DIGIT_SIZE); + res27 = and64(res27, MASK); + storeu64(out_mb + MB_WIDTH * 27, res27); + res28 = add64(res28, T); + T = srli64(res28, DIGIT_SIZE); + res28 = and64(res28, MASK); + storeu64(out_mb + MB_WIDTH * 28, res28); + res29 = add64(res29, T); + T = srli64(res29, DIGIT_SIZE); + res29 = and64(res29, MASK); + storeu64(out_mb + MB_WIDTH * 29, res29); + res30 = add64(res30, T); + T = srli64(res30, DIGIT_SIZE); + res30 = and64(res30, MASK); + storeu64(out_mb + MB_WIDTH * 30, res30); + res31 = add64(res31, T); + T = srli64(res31, DIGIT_SIZE); + res31 = and64(res31, MASK); + storeu64(out_mb + MB_WIDTH * 31, res31); + res32 = add64(res32, T); + T = srli64(res32, DIGIT_SIZE); + res32 = and64(res32, MASK); + storeu64(out_mb + MB_WIDTH * 32, res32); + res33 = add64(res33, T); + T = srli64(res33, DIGIT_SIZE); + res33 = and64(res33, MASK); + storeu64(out_mb + MB_WIDTH * 33, res33); + res34 = add64(res34, T); + T = srli64(res34, DIGIT_SIZE); + res34 = and64(res34, MASK); + storeu64(out_mb + MB_WIDTH * 34, res34); + res35 = add64(res35, T); + T = srli64(res35, DIGIT_SIZE); + res35 = and64(res35, MASK); + storeu64(out_mb + MB_WIDTH * 35, res35); + res36 = add64(res36, T); + T = srli64(res36, DIGIT_SIZE); + res36 = and64(res36, MASK); + storeu64(out_mb + MB_WIDTH * 36, res36); + res37 = add64(res37, T); + T = srli64(res37, DIGIT_SIZE); + res37 = and64(res37, MASK); + storeu64(out_mb + MB_WIDTH * 37, res37); + res38 = add64(res38, T); + T = srli64(res38, DIGIT_SIZE); + res38 = and64(res38, MASK); + storeu64(out_mb + MB_WIDTH * 38, res38); + res39 = add64(res39, T); + T = srli64(res39, DIGIT_SIZE); + res39 = and64(res39, MASK); + storeu64(out_mb + MB_WIDTH * 39, res39); + res40 = add64(res40, T); + T = srli64(res40, DIGIT_SIZE); + res40 = and64(res40, MASK); + storeu64(out_mb + MB_WIDTH * 40, res40); + res41 = add64(res41, T); + T = srli64(res41, DIGIT_SIZE); + res41 = and64(res41, MASK); + storeu64(out_mb + MB_WIDTH * 41, res41); + res42 = add64(res42, T); + T = srli64(res42, DIGIT_SIZE); + res42 = and64(res42, MASK); + storeu64(out_mb + MB_WIDTH * 42, res42); + res43 = add64(res43, T); + T = srli64(res43, DIGIT_SIZE); + res43 = and64(res43, MASK); + storeu64(out_mb + MB_WIDTH * 43, res43); + res44 = add64(res44, T); + T = srli64(res44, DIGIT_SIZE); + res44 = and64(res44, MASK); + storeu64(out_mb + MB_WIDTH * 44, res44); + res45 = add64(res45, T); + T = srli64(res45, DIGIT_SIZE); + res45 = and64(res45, MASK); + storeu64(out_mb + MB_WIDTH * 45, res45); + res46 = add64(res46, T); + T = srli64(res46, DIGIT_SIZE); + res46 = and64(res46, MASK); + storeu64(out_mb + MB_WIDTH * 46, res46); + res47 = add64(res47, T); + T = srli64(res47, DIGIT_SIZE); + res47 = and64(res47, MASK); + storeu64(out_mb + MB_WIDTH * 47, res47); + res48 = add64(res48, T); + T = srli64(res48, DIGIT_SIZE); + res48 = and64(res48, MASK); + storeu64(out_mb + MB_WIDTH * 48, res48); + res49 = add64(res49, T); + T = srli64(res49, DIGIT_SIZE); + res49 = and64(res49, MASK); + storeu64(out_mb + MB_WIDTH * 49, res49); + res50 = add64(res50, T); + T = srli64(res50, DIGIT_SIZE); + res50 = and64(res50, MASK); + storeu64(out_mb + MB_WIDTH * 50, res50); + res51 = add64(res51, T); + T = srli64(res51, DIGIT_SIZE); + res51 = and64(res51, MASK); + storeu64(out_mb + MB_WIDTH * 51, res51); + res52 = add64(res52, T); + T = srli64(res52, DIGIT_SIZE); + res52 = and64(res52, MASK); + storeu64(out_mb + MB_WIDTH * 52, res52); + res53 = add64(res53, T); + T = srli64(res53, DIGIT_SIZE); + res53 = and64(res53, MASK); + storeu64(out_mb + MB_WIDTH * 53, res53); + res54 = add64(res54, T); + T = srli64(res54, DIGIT_SIZE); + res54 = and64(res54, MASK); + storeu64(out_mb + MB_WIDTH * 54, res54); + res55 = add64(res55, T); + T = srli64(res55, DIGIT_SIZE); + res55 = and64(res55, MASK); + storeu64(out_mb + MB_WIDTH * 55, res55); + res56 = add64(res56, T); + T = srli64(res56, DIGIT_SIZE); + res56 = and64(res56, MASK); + storeu64(out_mb + MB_WIDTH * 56, res56); + res57 = add64(res57, T); + T = srli64(res57, DIGIT_SIZE); + res57 = and64(res57, MASK); + storeu64(out_mb + MB_WIDTH * 57, res57); + res58 = add64(res58, T); + T = srli64(res58, DIGIT_SIZE); + res58 = and64(res58, MASK); + storeu64(out_mb + MB_WIDTH * 58, res58); + res59 = add64(res59, T); + res59 = and64(res59, MASK); + storeu64(out_mb + MB_WIDTH * 59, res59); + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_amm52x79_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_amm52x79_mb8.c new file mode 100644 index 0000000..d0a7e59 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_amm52x79_mb8.c @@ -0,0 +1,690 @@ +/******************************************************************************* + * Copyright (C) 2019 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include + +void ifma_amm52x79_mb8(int64u *out_mb, const int64u *inpA_mb, + const int64u *inpB_mb, const int64u *inpM_mb, + const int64u *k0_mb) { + U64 res00, res01, res02, res03, res04, res05, res06, res07, res08, res09, + res10, res11, res12, res13, res14, res15, res16, res17, res18, res19, + res20, res21, res22, res23, res24, res25, res26, res27, res28, res29, + res30, res31, res32, res33, res34, res35, res36, res37, res38, res39, + res40, res41, res42, res43, res44, res45, res46, res47, res48, res49, + res50, res51, res52, res53, res54, res55, res56, res57, res58, res59, + res60, res61, res62, res63, res64, res65, res66, res67, res68, res69, + res70, res71, res72, res73, res74, res75, res76, res77, res78; + U64 K = loadu64(k0_mb); + int itr; + res00 = res01 = res02 = res03 = res04 = res05 = res06 = res07 = res08 = + res09 = res10 = res11 = res12 = res13 = res14 = res15 = res16 = res17 = + res18 = res19 = res20 = res21 = res22 = res23 = res24 = res25 = + res26 = res27 = res28 = res29 = res30 = res31 = res32 = res33 = + res34 = res35 = res36 = res37 = res38 = res39 = res40 = + res41 = res42 = res43 = res44 = res45 = res46 = res47 = + res48 = res49 = res50 = res51 = res52 = res53 = + res54 = res55 = res56 = res57 = res58 = res59 = + res60 = res61 = res62 = res63 = res64 = + res65 = res66 = res67 = res68 = res69 = + res70 = res71 = res72 = res73 = + res74 = res75 = res76 = res77 = + res78 = get_zero64(); + + for (itr = 0; itr < 79; itr++) { + U64 Yi; + U64 Bi = loadu64(inpB_mb); + inpB_mb += MB_WIDTH; + fma52lo_mem(res00, res00, Bi, inpA_mb, SIMD_BYTES * 0); + fma52lo_mem(res01, res01, Bi, inpA_mb, SIMD_BYTES * 1); + fma52lo_mem(res02, res02, Bi, inpA_mb, SIMD_BYTES * 2); + fma52lo_mem(res03, res03, Bi, inpA_mb, SIMD_BYTES * 3); + fma52lo_mem(res04, res04, Bi, inpA_mb, SIMD_BYTES * 4); + fma52lo_mem(res05, res05, Bi, inpA_mb, SIMD_BYTES * 5); + fma52lo_mem(res06, res06, Bi, inpA_mb, SIMD_BYTES * 6); + fma52lo_mem(res07, res07, Bi, inpA_mb, SIMD_BYTES * 7); + fma52lo_mem(res08, res08, Bi, inpA_mb, SIMD_BYTES * 8); + fma52lo_mem(res09, res09, Bi, inpA_mb, SIMD_BYTES * 9); + fma52lo_mem(res10, res10, Bi, inpA_mb, SIMD_BYTES * 10); + fma52lo_mem(res11, res11, Bi, inpA_mb, SIMD_BYTES * 11); + fma52lo_mem(res12, res12, Bi, inpA_mb, SIMD_BYTES * 12); + fma52lo_mem(res13, res13, Bi, inpA_mb, SIMD_BYTES * 13); + fma52lo_mem(res14, res14, Bi, inpA_mb, SIMD_BYTES * 14); + fma52lo_mem(res15, res15, Bi, inpA_mb, SIMD_BYTES * 15); + fma52lo_mem(res16, res16, Bi, inpA_mb, SIMD_BYTES * 16); + fma52lo_mem(res17, res17, Bi, inpA_mb, SIMD_BYTES * 17); + fma52lo_mem(res18, res18, Bi, inpA_mb, SIMD_BYTES * 18); + fma52lo_mem(res19, res19, Bi, inpA_mb, SIMD_BYTES * 19); + fma52lo_mem(res20, res20, Bi, inpA_mb, SIMD_BYTES * 20); + fma52lo_mem(res21, res21, Bi, inpA_mb, SIMD_BYTES * 21); + fma52lo_mem(res22, res22, Bi, inpA_mb, SIMD_BYTES * 22); + fma52lo_mem(res23, res23, Bi, inpA_mb, SIMD_BYTES * 23); + fma52lo_mem(res24, res24, Bi, inpA_mb, SIMD_BYTES * 24); + fma52lo_mem(res25, res25, Bi, inpA_mb, SIMD_BYTES * 25); + fma52lo_mem(res26, res26, Bi, inpA_mb, SIMD_BYTES * 26); + fma52lo_mem(res27, res27, Bi, inpA_mb, SIMD_BYTES * 27); + fma52lo_mem(res28, res28, Bi, inpA_mb, SIMD_BYTES * 28); + fma52lo_mem(res29, res29, Bi, inpA_mb, SIMD_BYTES * 29); + fma52lo_mem(res30, res30, Bi, inpA_mb, SIMD_BYTES * 30); + fma52lo_mem(res31, res31, Bi, inpA_mb, SIMD_BYTES * 31); + fma52lo_mem(res32, res32, Bi, inpA_mb, SIMD_BYTES * 32); + fma52lo_mem(res33, res33, Bi, inpA_mb, SIMD_BYTES * 33); + fma52lo_mem(res34, res34, Bi, inpA_mb, SIMD_BYTES * 34); + fma52lo_mem(res35, res35, Bi, inpA_mb, SIMD_BYTES * 35); + fma52lo_mem(res36, res36, Bi, inpA_mb, SIMD_BYTES * 36); + fma52lo_mem(res37, res37, Bi, inpA_mb, SIMD_BYTES * 37); + fma52lo_mem(res38, res38, Bi, inpA_mb, SIMD_BYTES * 38); + fma52lo_mem(res39, res39, Bi, inpA_mb, SIMD_BYTES * 39); + fma52lo_mem(res40, res40, Bi, inpA_mb, SIMD_BYTES * 40); + fma52lo_mem(res41, res41, Bi, inpA_mb, SIMD_BYTES * 41); + fma52lo_mem(res42, res42, Bi, inpA_mb, SIMD_BYTES * 42); + fma52lo_mem(res43, res43, Bi, inpA_mb, SIMD_BYTES * 43); + fma52lo_mem(res44, res44, Bi, inpA_mb, SIMD_BYTES * 44); + fma52lo_mem(res45, res45, Bi, inpA_mb, SIMD_BYTES * 45); + fma52lo_mem(res46, res46, Bi, inpA_mb, SIMD_BYTES * 46); + fma52lo_mem(res47, res47, Bi, inpA_mb, SIMD_BYTES * 47); + fma52lo_mem(res48, res48, Bi, inpA_mb, SIMD_BYTES * 48); + fma52lo_mem(res49, res49, Bi, inpA_mb, SIMD_BYTES * 49); + fma52lo_mem(res50, res50, Bi, inpA_mb, SIMD_BYTES * 50); + fma52lo_mem(res51, res51, Bi, inpA_mb, SIMD_BYTES * 51); + fma52lo_mem(res52, res52, Bi, inpA_mb, SIMD_BYTES * 52); + fma52lo_mem(res53, res53, Bi, inpA_mb, SIMD_BYTES * 53); + fma52lo_mem(res54, res54, Bi, inpA_mb, SIMD_BYTES * 54); + fma52lo_mem(res55, res55, Bi, inpA_mb, SIMD_BYTES * 55); + fma52lo_mem(res56, res56, Bi, inpA_mb, SIMD_BYTES * 56); + fma52lo_mem(res57, res57, Bi, inpA_mb, SIMD_BYTES * 57); + fma52lo_mem(res58, res58, Bi, inpA_mb, SIMD_BYTES * 58); + fma52lo_mem(res59, res59, Bi, inpA_mb, SIMD_BYTES * 59); + fma52lo_mem(res60, res60, Bi, inpA_mb, SIMD_BYTES * 60); + fma52lo_mem(res61, res61, Bi, inpA_mb, SIMD_BYTES * 61); + fma52lo_mem(res62, res62, Bi, inpA_mb, SIMD_BYTES * 62); + fma52lo_mem(res63, res63, Bi, inpA_mb, SIMD_BYTES * 63); + fma52lo_mem(res64, res64, Bi, inpA_mb, SIMD_BYTES * 64); + fma52lo_mem(res65, res65, Bi, inpA_mb, SIMD_BYTES * 65); + fma52lo_mem(res66, res66, Bi, inpA_mb, SIMD_BYTES * 66); + fma52lo_mem(res67, res67, Bi, inpA_mb, SIMD_BYTES * 67); + fma52lo_mem(res68, res68, Bi, inpA_mb, SIMD_BYTES * 68); + fma52lo_mem(res69, res69, Bi, inpA_mb, SIMD_BYTES * 69); + fma52lo_mem(res70, res70, Bi, inpA_mb, SIMD_BYTES * 70); + fma52lo_mem(res71, res71, Bi, inpA_mb, SIMD_BYTES * 71); + fma52lo_mem(res72, res72, Bi, inpA_mb, SIMD_BYTES * 72); + fma52lo_mem(res73, res73, Bi, inpA_mb, SIMD_BYTES * 73); + fma52lo_mem(res74, res74, Bi, inpA_mb, SIMD_BYTES * 74); + fma52lo_mem(res75, res75, Bi, inpA_mb, SIMD_BYTES * 75); + fma52lo_mem(res76, res76, Bi, inpA_mb, SIMD_BYTES * 76); + fma52lo_mem(res77, res77, Bi, inpA_mb, SIMD_BYTES * 77); + fma52lo_mem(res78, res78, Bi, inpA_mb, SIMD_BYTES * 78); + Yi = fma52lo(get_zero64(), res00, K); + fma52lo_mem(res00, res00, Yi, inpM_mb, SIMD_BYTES * 0); + fma52lo_mem(res01, res01, Yi, inpM_mb, SIMD_BYTES * 1); + fma52lo_mem(res02, res02, Yi, inpM_mb, SIMD_BYTES * 2); + fma52lo_mem(res03, res03, Yi, inpM_mb, SIMD_BYTES * 3); + fma52lo_mem(res04, res04, Yi, inpM_mb, SIMD_BYTES * 4); + fma52lo_mem(res05, res05, Yi, inpM_mb, SIMD_BYTES * 5); + fma52lo_mem(res06, res06, Yi, inpM_mb, SIMD_BYTES * 6); + fma52lo_mem(res07, res07, Yi, inpM_mb, SIMD_BYTES * 7); + fma52lo_mem(res08, res08, Yi, inpM_mb, SIMD_BYTES * 8); + fma52lo_mem(res09, res09, Yi, inpM_mb, SIMD_BYTES * 9); + fma52lo_mem(res10, res10, Yi, inpM_mb, SIMD_BYTES * 10); + fma52lo_mem(res11, res11, Yi, inpM_mb, SIMD_BYTES * 11); + fma52lo_mem(res12, res12, Yi, inpM_mb, SIMD_BYTES * 12); + fma52lo_mem(res13, res13, Yi, inpM_mb, SIMD_BYTES * 13); + fma52lo_mem(res14, res14, Yi, inpM_mb, SIMD_BYTES * 14); + fma52lo_mem(res15, res15, Yi, inpM_mb, SIMD_BYTES * 15); + fma52lo_mem(res16, res16, Yi, inpM_mb, SIMD_BYTES * 16); + fma52lo_mem(res17, res17, Yi, inpM_mb, SIMD_BYTES * 17); + fma52lo_mem(res18, res18, Yi, inpM_mb, SIMD_BYTES * 18); + fma52lo_mem(res19, res19, Yi, inpM_mb, SIMD_BYTES * 19); + fma52lo_mem(res20, res20, Yi, inpM_mb, SIMD_BYTES * 20); + fma52lo_mem(res21, res21, Yi, inpM_mb, SIMD_BYTES * 21); + fma52lo_mem(res22, res22, Yi, inpM_mb, SIMD_BYTES * 22); + fma52lo_mem(res23, res23, Yi, inpM_mb, SIMD_BYTES * 23); + fma52lo_mem(res24, res24, Yi, inpM_mb, SIMD_BYTES * 24); + fma52lo_mem(res25, res25, Yi, inpM_mb, SIMD_BYTES * 25); + fma52lo_mem(res26, res26, Yi, inpM_mb, SIMD_BYTES * 26); + fma52lo_mem(res27, res27, Yi, inpM_mb, SIMD_BYTES * 27); + fma52lo_mem(res28, res28, Yi, inpM_mb, SIMD_BYTES * 28); + fma52lo_mem(res29, res29, Yi, inpM_mb, SIMD_BYTES * 29); + fma52lo_mem(res30, res30, Yi, inpM_mb, SIMD_BYTES * 30); + fma52lo_mem(res31, res31, Yi, inpM_mb, SIMD_BYTES * 31); + fma52lo_mem(res32, res32, Yi, inpM_mb, SIMD_BYTES * 32); + fma52lo_mem(res33, res33, Yi, inpM_mb, SIMD_BYTES * 33); + fma52lo_mem(res34, res34, Yi, inpM_mb, SIMD_BYTES * 34); + fma52lo_mem(res35, res35, Yi, inpM_mb, SIMD_BYTES * 35); + fma52lo_mem(res36, res36, Yi, inpM_mb, SIMD_BYTES * 36); + fma52lo_mem(res37, res37, Yi, inpM_mb, SIMD_BYTES * 37); + fma52lo_mem(res38, res38, Yi, inpM_mb, SIMD_BYTES * 38); + fma52lo_mem(res39, res39, Yi, inpM_mb, SIMD_BYTES * 39); + fma52lo_mem(res40, res40, Yi, inpM_mb, SIMD_BYTES * 40); + fma52lo_mem(res41, res41, Yi, inpM_mb, SIMD_BYTES * 41); + fma52lo_mem(res42, res42, Yi, inpM_mb, SIMD_BYTES * 42); + fma52lo_mem(res43, res43, Yi, inpM_mb, SIMD_BYTES * 43); + fma52lo_mem(res44, res44, Yi, inpM_mb, SIMD_BYTES * 44); + fma52lo_mem(res45, res45, Yi, inpM_mb, SIMD_BYTES * 45); + fma52lo_mem(res46, res46, Yi, inpM_mb, SIMD_BYTES * 46); + fma52lo_mem(res47, res47, Yi, inpM_mb, SIMD_BYTES * 47); + fma52lo_mem(res48, res48, Yi, inpM_mb, SIMD_BYTES * 48); + fma52lo_mem(res49, res49, Yi, inpM_mb, SIMD_BYTES * 49); + fma52lo_mem(res50, res50, Yi, inpM_mb, SIMD_BYTES * 50); + fma52lo_mem(res51, res51, Yi, inpM_mb, SIMD_BYTES * 51); + fma52lo_mem(res52, res52, Yi, inpM_mb, SIMD_BYTES * 52); + fma52lo_mem(res53, res53, Yi, inpM_mb, SIMD_BYTES * 53); + fma52lo_mem(res54, res54, Yi, inpM_mb, SIMD_BYTES * 54); + fma52lo_mem(res55, res55, Yi, inpM_mb, SIMD_BYTES * 55); + fma52lo_mem(res56, res56, Yi, inpM_mb, SIMD_BYTES * 56); + fma52lo_mem(res57, res57, Yi, inpM_mb, SIMD_BYTES * 57); + fma52lo_mem(res58, res58, Yi, inpM_mb, SIMD_BYTES * 58); + fma52lo_mem(res59, res59, Yi, inpM_mb, SIMD_BYTES * 59); + fma52lo_mem(res60, res60, Yi, inpM_mb, SIMD_BYTES * 60); + fma52lo_mem(res61, res61, Yi, inpM_mb, SIMD_BYTES * 61); + fma52lo_mem(res62, res62, Yi, inpM_mb, SIMD_BYTES * 62); + fma52lo_mem(res63, res63, Yi, inpM_mb, SIMD_BYTES * 63); + fma52lo_mem(res64, res64, Yi, inpM_mb, SIMD_BYTES * 64); + fma52lo_mem(res65, res65, Yi, inpM_mb, SIMD_BYTES * 65); + fma52lo_mem(res66, res66, Yi, inpM_mb, SIMD_BYTES * 66); + fma52lo_mem(res67, res67, Yi, inpM_mb, SIMD_BYTES * 67); + fma52lo_mem(res68, res68, Yi, inpM_mb, SIMD_BYTES * 68); + fma52lo_mem(res69, res69, Yi, inpM_mb, SIMD_BYTES * 69); + fma52lo_mem(res70, res70, Yi, inpM_mb, SIMD_BYTES * 70); + fma52lo_mem(res71, res71, Yi, inpM_mb, SIMD_BYTES * 71); + fma52lo_mem(res72, res72, Yi, inpM_mb, SIMD_BYTES * 72); + fma52lo_mem(res73, res73, Yi, inpM_mb, SIMD_BYTES * 73); + fma52lo_mem(res74, res74, Yi, inpM_mb, SIMD_BYTES * 74); + fma52lo_mem(res75, res75, Yi, inpM_mb, SIMD_BYTES * 75); + fma52lo_mem(res76, res76, Yi, inpM_mb, SIMD_BYTES * 76); + fma52lo_mem(res77, res77, Yi, inpM_mb, SIMD_BYTES * 77); + fma52lo_mem(res78, res78, Yi, inpM_mb, SIMD_BYTES * 78); + res00 = srli64(res00, DIGIT_SIZE); + res01 = add64(res01, res00); + fma52hi_mem(res00, res01, Bi, inpA_mb, SIMD_BYTES * 0); + fma52hi_mem(res01, res02, Bi, inpA_mb, SIMD_BYTES * 1); + fma52hi_mem(res02, res03, Bi, inpA_mb, SIMD_BYTES * 2); + fma52hi_mem(res03, res04, Bi, inpA_mb, SIMD_BYTES * 3); + fma52hi_mem(res04, res05, Bi, inpA_mb, SIMD_BYTES * 4); + fma52hi_mem(res05, res06, Bi, inpA_mb, SIMD_BYTES * 5); + fma52hi_mem(res06, res07, Bi, inpA_mb, SIMD_BYTES * 6); + fma52hi_mem(res07, res08, Bi, inpA_mb, SIMD_BYTES * 7); + fma52hi_mem(res08, res09, Bi, inpA_mb, SIMD_BYTES * 8); + fma52hi_mem(res09, res10, Bi, inpA_mb, SIMD_BYTES * 9); + fma52hi_mem(res10, res11, Bi, inpA_mb, SIMD_BYTES * 10); + fma52hi_mem(res11, res12, Bi, inpA_mb, SIMD_BYTES * 11); + fma52hi_mem(res12, res13, Bi, inpA_mb, SIMD_BYTES * 12); + fma52hi_mem(res13, res14, Bi, inpA_mb, SIMD_BYTES * 13); + fma52hi_mem(res14, res15, Bi, inpA_mb, SIMD_BYTES * 14); + fma52hi_mem(res15, res16, Bi, inpA_mb, SIMD_BYTES * 15); + fma52hi_mem(res16, res17, Bi, inpA_mb, SIMD_BYTES * 16); + fma52hi_mem(res17, res18, Bi, inpA_mb, SIMD_BYTES * 17); + fma52hi_mem(res18, res19, Bi, inpA_mb, SIMD_BYTES * 18); + fma52hi_mem(res19, res20, Bi, inpA_mb, SIMD_BYTES * 19); + fma52hi_mem(res20, res21, Bi, inpA_mb, SIMD_BYTES * 20); + fma52hi_mem(res21, res22, Bi, inpA_mb, SIMD_BYTES * 21); + fma52hi_mem(res22, res23, Bi, inpA_mb, SIMD_BYTES * 22); + fma52hi_mem(res23, res24, Bi, inpA_mb, SIMD_BYTES * 23); + fma52hi_mem(res24, res25, Bi, inpA_mb, SIMD_BYTES * 24); + fma52hi_mem(res25, res26, Bi, inpA_mb, SIMD_BYTES * 25); + fma52hi_mem(res26, res27, Bi, inpA_mb, SIMD_BYTES * 26); + fma52hi_mem(res27, res28, Bi, inpA_mb, SIMD_BYTES * 27); + fma52hi_mem(res28, res29, Bi, inpA_mb, SIMD_BYTES * 28); + fma52hi_mem(res29, res30, Bi, inpA_mb, SIMD_BYTES * 29); + fma52hi_mem(res30, res31, Bi, inpA_mb, SIMD_BYTES * 30); + fma52hi_mem(res31, res32, Bi, inpA_mb, SIMD_BYTES * 31); + fma52hi_mem(res32, res33, Bi, inpA_mb, SIMD_BYTES * 32); + fma52hi_mem(res33, res34, Bi, inpA_mb, SIMD_BYTES * 33); + fma52hi_mem(res34, res35, Bi, inpA_mb, SIMD_BYTES * 34); + fma52hi_mem(res35, res36, Bi, inpA_mb, SIMD_BYTES * 35); + fma52hi_mem(res36, res37, Bi, inpA_mb, SIMD_BYTES * 36); + fma52hi_mem(res37, res38, Bi, inpA_mb, SIMD_BYTES * 37); + fma52hi_mem(res38, res39, Bi, inpA_mb, SIMD_BYTES * 38); + fma52hi_mem(res39, res40, Bi, inpA_mb, SIMD_BYTES * 39); + fma52hi_mem(res40, res41, Bi, inpA_mb, SIMD_BYTES * 40); + fma52hi_mem(res41, res42, Bi, inpA_mb, SIMD_BYTES * 41); + fma52hi_mem(res42, res43, Bi, inpA_mb, SIMD_BYTES * 42); + fma52hi_mem(res43, res44, Bi, inpA_mb, SIMD_BYTES * 43); + fma52hi_mem(res44, res45, Bi, inpA_mb, SIMD_BYTES * 44); + fma52hi_mem(res45, res46, Bi, inpA_mb, SIMD_BYTES * 45); + fma52hi_mem(res46, res47, Bi, inpA_mb, SIMD_BYTES * 46); + fma52hi_mem(res47, res48, Bi, inpA_mb, SIMD_BYTES * 47); + fma52hi_mem(res48, res49, Bi, inpA_mb, SIMD_BYTES * 48); + fma52hi_mem(res49, res50, Bi, inpA_mb, SIMD_BYTES * 49); + fma52hi_mem(res50, res51, Bi, inpA_mb, SIMD_BYTES * 50); + fma52hi_mem(res51, res52, Bi, inpA_mb, SIMD_BYTES * 51); + fma52hi_mem(res52, res53, Bi, inpA_mb, SIMD_BYTES * 52); + fma52hi_mem(res53, res54, Bi, inpA_mb, SIMD_BYTES * 53); + fma52hi_mem(res54, res55, Bi, inpA_mb, SIMD_BYTES * 54); + fma52hi_mem(res55, res56, Bi, inpA_mb, SIMD_BYTES * 55); + fma52hi_mem(res56, res57, Bi, inpA_mb, SIMD_BYTES * 56); + fma52hi_mem(res57, res58, Bi, inpA_mb, SIMD_BYTES * 57); + fma52hi_mem(res58, res59, Bi, inpA_mb, SIMD_BYTES * 58); + fma52hi_mem(res59, res60, Bi, inpA_mb, SIMD_BYTES * 59); + fma52hi_mem(res60, res61, Bi, inpA_mb, SIMD_BYTES * 60); + fma52hi_mem(res61, res62, Bi, inpA_mb, SIMD_BYTES * 61); + fma52hi_mem(res62, res63, Bi, inpA_mb, SIMD_BYTES * 62); + fma52hi_mem(res63, res64, Bi, inpA_mb, SIMD_BYTES * 63); + fma52hi_mem(res64, res65, Bi, inpA_mb, SIMD_BYTES * 64); + fma52hi_mem(res65, res66, Bi, inpA_mb, SIMD_BYTES * 65); + fma52hi_mem(res66, res67, Bi, inpA_mb, SIMD_BYTES * 66); + fma52hi_mem(res67, res68, Bi, inpA_mb, SIMD_BYTES * 67); + fma52hi_mem(res68, res69, Bi, inpA_mb, SIMD_BYTES * 68); + fma52hi_mem(res69, res70, Bi, inpA_mb, SIMD_BYTES * 69); + fma52hi_mem(res70, res71, Bi, inpA_mb, SIMD_BYTES * 70); + fma52hi_mem(res71, res72, Bi, inpA_mb, SIMD_BYTES * 71); + fma52hi_mem(res72, res73, Bi, inpA_mb, SIMD_BYTES * 72); + fma52hi_mem(res73, res74, Bi, inpA_mb, SIMD_BYTES * 73); + fma52hi_mem(res74, res75, Bi, inpA_mb, SIMD_BYTES * 74); + fma52hi_mem(res75, res76, Bi, inpA_mb, SIMD_BYTES * 75); + fma52hi_mem(res76, res77, Bi, inpA_mb, SIMD_BYTES * 76); + fma52hi_mem(res77, res78, Bi, inpA_mb, SIMD_BYTES * 77); + fma52hi_mem(res78, get_zero64(), Bi, inpA_mb, SIMD_BYTES * 78); + fma52hi_mem(res00, res00, Yi, inpM_mb, SIMD_BYTES * 0); + fma52hi_mem(res01, res01, Yi, inpM_mb, SIMD_BYTES * 1); + fma52hi_mem(res02, res02, Yi, inpM_mb, SIMD_BYTES * 2); + fma52hi_mem(res03, res03, Yi, inpM_mb, SIMD_BYTES * 3); + fma52hi_mem(res04, res04, Yi, inpM_mb, SIMD_BYTES * 4); + fma52hi_mem(res05, res05, Yi, inpM_mb, SIMD_BYTES * 5); + fma52hi_mem(res06, res06, Yi, inpM_mb, SIMD_BYTES * 6); + fma52hi_mem(res07, res07, Yi, inpM_mb, SIMD_BYTES * 7); + fma52hi_mem(res08, res08, Yi, inpM_mb, SIMD_BYTES * 8); + fma52hi_mem(res09, res09, Yi, inpM_mb, SIMD_BYTES * 9); + fma52hi_mem(res10, res10, Yi, inpM_mb, SIMD_BYTES * 10); + fma52hi_mem(res11, res11, Yi, inpM_mb, SIMD_BYTES * 11); + fma52hi_mem(res12, res12, Yi, inpM_mb, SIMD_BYTES * 12); + fma52hi_mem(res13, res13, Yi, inpM_mb, SIMD_BYTES * 13); + fma52hi_mem(res14, res14, Yi, inpM_mb, SIMD_BYTES * 14); + fma52hi_mem(res15, res15, Yi, inpM_mb, SIMD_BYTES * 15); + fma52hi_mem(res16, res16, Yi, inpM_mb, SIMD_BYTES * 16); + fma52hi_mem(res17, res17, Yi, inpM_mb, SIMD_BYTES * 17); + fma52hi_mem(res18, res18, Yi, inpM_mb, SIMD_BYTES * 18); + fma52hi_mem(res19, res19, Yi, inpM_mb, SIMD_BYTES * 19); + fma52hi_mem(res20, res20, Yi, inpM_mb, SIMD_BYTES * 20); + fma52hi_mem(res21, res21, Yi, inpM_mb, SIMD_BYTES * 21); + fma52hi_mem(res22, res22, Yi, inpM_mb, SIMD_BYTES * 22); + fma52hi_mem(res23, res23, Yi, inpM_mb, SIMD_BYTES * 23); + fma52hi_mem(res24, res24, Yi, inpM_mb, SIMD_BYTES * 24); + fma52hi_mem(res25, res25, Yi, inpM_mb, SIMD_BYTES * 25); + fma52hi_mem(res26, res26, Yi, inpM_mb, SIMD_BYTES * 26); + fma52hi_mem(res27, res27, Yi, inpM_mb, SIMD_BYTES * 27); + fma52hi_mem(res28, res28, Yi, inpM_mb, SIMD_BYTES * 28); + fma52hi_mem(res29, res29, Yi, inpM_mb, SIMD_BYTES * 29); + fma52hi_mem(res30, res30, Yi, inpM_mb, SIMD_BYTES * 30); + fma52hi_mem(res31, res31, Yi, inpM_mb, SIMD_BYTES * 31); + fma52hi_mem(res32, res32, Yi, inpM_mb, SIMD_BYTES * 32); + fma52hi_mem(res33, res33, Yi, inpM_mb, SIMD_BYTES * 33); + fma52hi_mem(res34, res34, Yi, inpM_mb, SIMD_BYTES * 34); + fma52hi_mem(res35, res35, Yi, inpM_mb, SIMD_BYTES * 35); + fma52hi_mem(res36, res36, Yi, inpM_mb, SIMD_BYTES * 36); + fma52hi_mem(res37, res37, Yi, inpM_mb, SIMD_BYTES * 37); + fma52hi_mem(res38, res38, Yi, inpM_mb, SIMD_BYTES * 38); + fma52hi_mem(res39, res39, Yi, inpM_mb, SIMD_BYTES * 39); + fma52hi_mem(res40, res40, Yi, inpM_mb, SIMD_BYTES * 40); + fma52hi_mem(res41, res41, Yi, inpM_mb, SIMD_BYTES * 41); + fma52hi_mem(res42, res42, Yi, inpM_mb, SIMD_BYTES * 42); + fma52hi_mem(res43, res43, Yi, inpM_mb, SIMD_BYTES * 43); + fma52hi_mem(res44, res44, Yi, inpM_mb, SIMD_BYTES * 44); + fma52hi_mem(res45, res45, Yi, inpM_mb, SIMD_BYTES * 45); + fma52hi_mem(res46, res46, Yi, inpM_mb, SIMD_BYTES * 46); + fma52hi_mem(res47, res47, Yi, inpM_mb, SIMD_BYTES * 47); + fma52hi_mem(res48, res48, Yi, inpM_mb, SIMD_BYTES * 48); + fma52hi_mem(res49, res49, Yi, inpM_mb, SIMD_BYTES * 49); + fma52hi_mem(res50, res50, Yi, inpM_mb, SIMD_BYTES * 50); + fma52hi_mem(res51, res51, Yi, inpM_mb, SIMD_BYTES * 51); + fma52hi_mem(res52, res52, Yi, inpM_mb, SIMD_BYTES * 52); + fma52hi_mem(res53, res53, Yi, inpM_mb, SIMD_BYTES * 53); + fma52hi_mem(res54, res54, Yi, inpM_mb, SIMD_BYTES * 54); + fma52hi_mem(res55, res55, Yi, inpM_mb, SIMD_BYTES * 55); + fma52hi_mem(res56, res56, Yi, inpM_mb, SIMD_BYTES * 56); + fma52hi_mem(res57, res57, Yi, inpM_mb, SIMD_BYTES * 57); + fma52hi_mem(res58, res58, Yi, inpM_mb, SIMD_BYTES * 58); + fma52hi_mem(res59, res59, Yi, inpM_mb, SIMD_BYTES * 59); + fma52hi_mem(res60, res60, Yi, inpM_mb, SIMD_BYTES * 60); + fma52hi_mem(res61, res61, Yi, inpM_mb, SIMD_BYTES * 61); + fma52hi_mem(res62, res62, Yi, inpM_mb, SIMD_BYTES * 62); + fma52hi_mem(res63, res63, Yi, inpM_mb, SIMD_BYTES * 63); + fma52hi_mem(res64, res64, Yi, inpM_mb, SIMD_BYTES * 64); + fma52hi_mem(res65, res65, Yi, inpM_mb, SIMD_BYTES * 65); + fma52hi_mem(res66, res66, Yi, inpM_mb, SIMD_BYTES * 66); + fma52hi_mem(res67, res67, Yi, inpM_mb, SIMD_BYTES * 67); + fma52hi_mem(res68, res68, Yi, inpM_mb, SIMD_BYTES * 68); + fma52hi_mem(res69, res69, Yi, inpM_mb, SIMD_BYTES * 69); + fma52hi_mem(res70, res70, Yi, inpM_mb, SIMD_BYTES * 70); + fma52hi_mem(res71, res71, Yi, inpM_mb, SIMD_BYTES * 71); + fma52hi_mem(res72, res72, Yi, inpM_mb, SIMD_BYTES * 72); + fma52hi_mem(res73, res73, Yi, inpM_mb, SIMD_BYTES * 73); + fma52hi_mem(res74, res74, Yi, inpM_mb, SIMD_BYTES * 74); + fma52hi_mem(res75, res75, Yi, inpM_mb, SIMD_BYTES * 75); + fma52hi_mem(res76, res76, Yi, inpM_mb, SIMD_BYTES * 76); + fma52hi_mem(res77, res77, Yi, inpM_mb, SIMD_BYTES * 77); + fma52hi_mem(res78, res78, Yi, inpM_mb, SIMD_BYTES * 78); + } + // Normalization + { + U64 T = get_zero64(); + U64 MASK = set64(DIGIT_MASK); + T = srli64(res00, DIGIT_SIZE); + res00 = and64(res00, MASK); + storeu64(out_mb + MB_WIDTH * 0, res00); + res01 = add64(res01, T); + T = srli64(res01, DIGIT_SIZE); + res01 = and64(res01, MASK); + storeu64(out_mb + MB_WIDTH * 1, res01); + res02 = add64(res02, T); + T = srli64(res02, DIGIT_SIZE); + res02 = and64(res02, MASK); + storeu64(out_mb + MB_WIDTH * 2, res02); + res03 = add64(res03, T); + T = srli64(res03, DIGIT_SIZE); + res03 = and64(res03, MASK); + storeu64(out_mb + MB_WIDTH * 3, res03); + res04 = add64(res04, T); + T = srli64(res04, DIGIT_SIZE); + res04 = and64(res04, MASK); + storeu64(out_mb + MB_WIDTH * 4, res04); + res05 = add64(res05, T); + T = srli64(res05, DIGIT_SIZE); + res05 = and64(res05, MASK); + storeu64(out_mb + MB_WIDTH * 5, res05); + res06 = add64(res06, T); + T = srli64(res06, DIGIT_SIZE); + res06 = and64(res06, MASK); + storeu64(out_mb + MB_WIDTH * 6, res06); + res07 = add64(res07, T); + T = srli64(res07, DIGIT_SIZE); + res07 = and64(res07, MASK); + storeu64(out_mb + MB_WIDTH * 7, res07); + res08 = add64(res08, T); + T = srli64(res08, DIGIT_SIZE); + res08 = and64(res08, MASK); + storeu64(out_mb + MB_WIDTH * 8, res08); + res09 = add64(res09, T); + T = srli64(res09, DIGIT_SIZE); + res09 = and64(res09, MASK); + storeu64(out_mb + MB_WIDTH * 9, res09); + res10 = add64(res10, T); + T = srli64(res10, DIGIT_SIZE); + res10 = and64(res10, MASK); + storeu64(out_mb + MB_WIDTH * 10, res10); + res11 = add64(res11, T); + T = srli64(res11, DIGIT_SIZE); + res11 = and64(res11, MASK); + storeu64(out_mb + MB_WIDTH * 11, res11); + res12 = add64(res12, T); + T = srli64(res12, DIGIT_SIZE); + res12 = and64(res12, MASK); + storeu64(out_mb + MB_WIDTH * 12, res12); + res13 = add64(res13, T); + T = srli64(res13, DIGIT_SIZE); + res13 = and64(res13, MASK); + storeu64(out_mb + MB_WIDTH * 13, res13); + res14 = add64(res14, T); + T = srli64(res14, DIGIT_SIZE); + res14 = and64(res14, MASK); + storeu64(out_mb + MB_WIDTH * 14, res14); + res15 = add64(res15, T); + T = srli64(res15, DIGIT_SIZE); + res15 = and64(res15, MASK); + storeu64(out_mb + MB_WIDTH * 15, res15); + res16 = add64(res16, T); + T = srli64(res16, DIGIT_SIZE); + res16 = and64(res16, MASK); + storeu64(out_mb + MB_WIDTH * 16, res16); + res17 = add64(res17, T); + T = srli64(res17, DIGIT_SIZE); + res17 = and64(res17, MASK); + storeu64(out_mb + MB_WIDTH * 17, res17); + res18 = add64(res18, T); + T = srli64(res18, DIGIT_SIZE); + res18 = and64(res18, MASK); + storeu64(out_mb + MB_WIDTH * 18, res18); + res19 = add64(res19, T); + T = srli64(res19, DIGIT_SIZE); + res19 = and64(res19, MASK); + storeu64(out_mb + MB_WIDTH * 19, res19); + res20 = add64(res20, T); + T = srli64(res20, DIGIT_SIZE); + res20 = and64(res20, MASK); + storeu64(out_mb + MB_WIDTH * 20, res20); + res21 = add64(res21, T); + T = srli64(res21, DIGIT_SIZE); + res21 = and64(res21, MASK); + storeu64(out_mb + MB_WIDTH * 21, res21); + res22 = add64(res22, T); + T = srli64(res22, DIGIT_SIZE); + res22 = and64(res22, MASK); + storeu64(out_mb + MB_WIDTH * 22, res22); + res23 = add64(res23, T); + T = srli64(res23, DIGIT_SIZE); + res23 = and64(res23, MASK); + storeu64(out_mb + MB_WIDTH * 23, res23); + res24 = add64(res24, T); + T = srli64(res24, DIGIT_SIZE); + res24 = and64(res24, MASK); + storeu64(out_mb + MB_WIDTH * 24, res24); + res25 = add64(res25, T); + T = srli64(res25, DIGIT_SIZE); + res25 = and64(res25, MASK); + storeu64(out_mb + MB_WIDTH * 25, res25); + res26 = add64(res26, T); + T = srli64(res26, DIGIT_SIZE); + res26 = and64(res26, MASK); + storeu64(out_mb + MB_WIDTH * 26, res26); + res27 = add64(res27, T); + T = srli64(res27, DIGIT_SIZE); + res27 = and64(res27, MASK); + storeu64(out_mb + MB_WIDTH * 27, res27); + res28 = add64(res28, T); + T = srli64(res28, DIGIT_SIZE); + res28 = and64(res28, MASK); + storeu64(out_mb + MB_WIDTH * 28, res28); + res29 = add64(res29, T); + T = srli64(res29, DIGIT_SIZE); + res29 = and64(res29, MASK); + storeu64(out_mb + MB_WIDTH * 29, res29); + res30 = add64(res30, T); + T = srli64(res30, DIGIT_SIZE); + res30 = and64(res30, MASK); + storeu64(out_mb + MB_WIDTH * 30, res30); + res31 = add64(res31, T); + T = srli64(res31, DIGIT_SIZE); + res31 = and64(res31, MASK); + storeu64(out_mb + MB_WIDTH * 31, res31); + res32 = add64(res32, T); + T = srli64(res32, DIGIT_SIZE); + res32 = and64(res32, MASK); + storeu64(out_mb + MB_WIDTH * 32, res32); + res33 = add64(res33, T); + T = srli64(res33, DIGIT_SIZE); + res33 = and64(res33, MASK); + storeu64(out_mb + MB_WIDTH * 33, res33); + res34 = add64(res34, T); + T = srli64(res34, DIGIT_SIZE); + res34 = and64(res34, MASK); + storeu64(out_mb + MB_WIDTH * 34, res34); + res35 = add64(res35, T); + T = srli64(res35, DIGIT_SIZE); + res35 = and64(res35, MASK); + storeu64(out_mb + MB_WIDTH * 35, res35); + res36 = add64(res36, T); + T = srli64(res36, DIGIT_SIZE); + res36 = and64(res36, MASK); + storeu64(out_mb + MB_WIDTH * 36, res36); + res37 = add64(res37, T); + T = srli64(res37, DIGIT_SIZE); + res37 = and64(res37, MASK); + storeu64(out_mb + MB_WIDTH * 37, res37); + res38 = add64(res38, T); + T = srli64(res38, DIGIT_SIZE); + res38 = and64(res38, MASK); + storeu64(out_mb + MB_WIDTH * 38, res38); + res39 = add64(res39, T); + T = srli64(res39, DIGIT_SIZE); + res39 = and64(res39, MASK); + storeu64(out_mb + MB_WIDTH * 39, res39); + res40 = add64(res40, T); + T = srli64(res40, DIGIT_SIZE); + res40 = and64(res40, MASK); + storeu64(out_mb + MB_WIDTH * 40, res40); + res41 = add64(res41, T); + T = srli64(res41, DIGIT_SIZE); + res41 = and64(res41, MASK); + storeu64(out_mb + MB_WIDTH * 41, res41); + res42 = add64(res42, T); + T = srli64(res42, DIGIT_SIZE); + res42 = and64(res42, MASK); + storeu64(out_mb + MB_WIDTH * 42, res42); + res43 = add64(res43, T); + T = srli64(res43, DIGIT_SIZE); + res43 = and64(res43, MASK); + storeu64(out_mb + MB_WIDTH * 43, res43); + res44 = add64(res44, T); + T = srli64(res44, DIGIT_SIZE); + res44 = and64(res44, MASK); + storeu64(out_mb + MB_WIDTH * 44, res44); + res45 = add64(res45, T); + T = srli64(res45, DIGIT_SIZE); + res45 = and64(res45, MASK); + storeu64(out_mb + MB_WIDTH * 45, res45); + res46 = add64(res46, T); + T = srli64(res46, DIGIT_SIZE); + res46 = and64(res46, MASK); + storeu64(out_mb + MB_WIDTH * 46, res46); + res47 = add64(res47, T); + T = srli64(res47, DIGIT_SIZE); + res47 = and64(res47, MASK); + storeu64(out_mb + MB_WIDTH * 47, res47); + res48 = add64(res48, T); + T = srli64(res48, DIGIT_SIZE); + res48 = and64(res48, MASK); + storeu64(out_mb + MB_WIDTH * 48, res48); + res49 = add64(res49, T); + T = srli64(res49, DIGIT_SIZE); + res49 = and64(res49, MASK); + storeu64(out_mb + MB_WIDTH * 49, res49); + res50 = add64(res50, T); + T = srli64(res50, DIGIT_SIZE); + res50 = and64(res50, MASK); + storeu64(out_mb + MB_WIDTH * 50, res50); + res51 = add64(res51, T); + T = srli64(res51, DIGIT_SIZE); + res51 = and64(res51, MASK); + storeu64(out_mb + MB_WIDTH * 51, res51); + res52 = add64(res52, T); + T = srli64(res52, DIGIT_SIZE); + res52 = and64(res52, MASK); + storeu64(out_mb + MB_WIDTH * 52, res52); + res53 = add64(res53, T); + T = srli64(res53, DIGIT_SIZE); + res53 = and64(res53, MASK); + storeu64(out_mb + MB_WIDTH * 53, res53); + res54 = add64(res54, T); + T = srli64(res54, DIGIT_SIZE); + res54 = and64(res54, MASK); + storeu64(out_mb + MB_WIDTH * 54, res54); + res55 = add64(res55, T); + T = srli64(res55, DIGIT_SIZE); + res55 = and64(res55, MASK); + storeu64(out_mb + MB_WIDTH * 55, res55); + res56 = add64(res56, T); + T = srli64(res56, DIGIT_SIZE); + res56 = and64(res56, MASK); + storeu64(out_mb + MB_WIDTH * 56, res56); + res57 = add64(res57, T); + T = srli64(res57, DIGIT_SIZE); + res57 = and64(res57, MASK); + storeu64(out_mb + MB_WIDTH * 57, res57); + res58 = add64(res58, T); + T = srli64(res58, DIGIT_SIZE); + res58 = and64(res58, MASK); + storeu64(out_mb + MB_WIDTH * 58, res58); + res59 = add64(res59, T); + T = srli64(res59, DIGIT_SIZE); + res59 = and64(res59, MASK); + storeu64(out_mb + MB_WIDTH * 59, res59); + res60 = add64(res60, T); + T = srli64(res60, DIGIT_SIZE); + res60 = and64(res60, MASK); + storeu64(out_mb + MB_WIDTH * 60, res60); + res61 = add64(res61, T); + T = srli64(res61, DIGIT_SIZE); + res61 = and64(res61, MASK); + storeu64(out_mb + MB_WIDTH * 61, res61); + res62 = add64(res62, T); + T = srli64(res62, DIGIT_SIZE); + res62 = and64(res62, MASK); + storeu64(out_mb + MB_WIDTH * 62, res62); + res63 = add64(res63, T); + T = srli64(res63, DIGIT_SIZE); + res63 = and64(res63, MASK); + storeu64(out_mb + MB_WIDTH * 63, res63); + res64 = add64(res64, T); + T = srli64(res64, DIGIT_SIZE); + res64 = and64(res64, MASK); + storeu64(out_mb + MB_WIDTH * 64, res64); + res65 = add64(res65, T); + T = srli64(res65, DIGIT_SIZE); + res65 = and64(res65, MASK); + storeu64(out_mb + MB_WIDTH * 65, res65); + res66 = add64(res66, T); + T = srli64(res66, DIGIT_SIZE); + res66 = and64(res66, MASK); + storeu64(out_mb + MB_WIDTH * 66, res66); + res67 = add64(res67, T); + T = srli64(res67, DIGIT_SIZE); + res67 = and64(res67, MASK); + storeu64(out_mb + MB_WIDTH * 67, res67); + res68 = add64(res68, T); + T = srli64(res68, DIGIT_SIZE); + res68 = and64(res68, MASK); + storeu64(out_mb + MB_WIDTH * 68, res68); + res69 = add64(res69, T); + T = srli64(res69, DIGIT_SIZE); + res69 = and64(res69, MASK); + storeu64(out_mb + MB_WIDTH * 69, res69); + res70 = add64(res70, T); + T = srli64(res70, DIGIT_SIZE); + res70 = and64(res70, MASK); + storeu64(out_mb + MB_WIDTH * 70, res70); + res71 = add64(res71, T); + T = srli64(res71, DIGIT_SIZE); + res71 = and64(res71, MASK); + storeu64(out_mb + MB_WIDTH * 71, res71); + res72 = add64(res72, T); + T = srli64(res72, DIGIT_SIZE); + res72 = and64(res72, MASK); + storeu64(out_mb + MB_WIDTH * 72, res72); + res73 = add64(res73, T); + T = srli64(res73, DIGIT_SIZE); + res73 = and64(res73, MASK); + storeu64(out_mb + MB_WIDTH * 73, res73); + res74 = add64(res74, T); + T = srli64(res74, DIGIT_SIZE); + res74 = and64(res74, MASK); + storeu64(out_mb + MB_WIDTH * 74, res74); + res75 = add64(res75, T); + T = srli64(res75, DIGIT_SIZE); + res75 = and64(res75, MASK); + storeu64(out_mb + MB_WIDTH * 75, res75); + res76 = add64(res76, T); + T = srli64(res76, DIGIT_SIZE); + res76 = and64(res76, MASK); + storeu64(out_mb + MB_WIDTH * 76, res76); + res77 = add64(res77, T); + T = srli64(res77, DIGIT_SIZE); + res77 = and64(res77, MASK); + storeu64(out_mb + MB_WIDTH * 77, res77); + res78 = add64(res78, T); + res78 = and64(res78, MASK); + storeu64(out_mb + MB_WIDTH * 78, res78); + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams52x10_diagonal_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams52x10_diagonal_mb8.c new file mode 100644 index 0000000..a25ae6d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams52x10_diagonal_mb8.c @@ -0,0 +1,436 @@ +/******************************************************************************* + * Copyright (C) 2019 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include + +#ifdef __GNUC__ +#define ASM(a) __asm__(a); +#else +#define ASM(a) +#endif + +void AMS52x10_diagonal_mb8(int64u *out_mb, const int64u *inpA_mb, + const int64u *inpM_mb, const int64u *k0_mb) { + U64 res0, res1, res2, res3, res4, res5, res6, res7, res8, res9, res10, res11, + res12, res13, res14, res15, res16, res17, res18, res19; + + U64 *a = (U64 *)inpA_mb; + U64 *m = (U64 *)inpM_mb; + U64 *r = (U64 *)out_mb; + U64 k = loadu64((U64 *)k0_mb); + + res0 = res1 = res2 = res3 = res4 = res5 = res6 = res7 = res8 = res9 = res10 = + res11 = res12 = res13 = res14 = res15 = res16 = res17 = res18 = res19 = + get_zero64(); + res1 = fma52lo(res1, a[0], a[1]); // Sum(1) + res2 = fma52hi(res2, a[0], a[1]); // Sum(1) + res2 = fma52lo(res2, a[0], a[2]); // Sum(2) + res3 = fma52hi(res3, a[0], a[2]); // Sum(2) + res3 = fma52lo(res3, a[1], a[2]); // Sum(3) + res4 = fma52hi(res4, a[1], a[2]); // Sum(3) + res3 = fma52lo(res3, a[0], a[3]); // Sum(3) + res4 = fma52hi(res4, a[0], a[3]); // Sum(3) + res4 = fma52lo(res4, a[1], a[3]); // Sum(4) + res5 = fma52hi(res5, a[1], a[3]); // Sum(4) + res5 = fma52lo(res5, a[2], a[3]); // Sum(5) + res6 = fma52hi(res6, a[2], a[3]); // Sum(5) + res4 = fma52lo(res4, a[0], a[4]); // Sum(4) + res5 = fma52hi(res5, a[0], a[4]); // Sum(4) + res5 = fma52lo(res5, a[1], a[4]); // Sum(5) + res6 = fma52hi(res6, a[1], a[4]); // Sum(5) + res6 = fma52lo(res6, a[2], a[4]); // Sum(6) + res7 = fma52hi(res7, a[2], a[4]); // Sum(6) + res7 = fma52lo(res7, a[3], a[4]); // Sum(7) + res8 = fma52hi(res8, a[3], a[4]); // Sum(7) + res5 = fma52lo(res5, a[0], a[5]); // Sum(5) + res6 = fma52hi(res6, a[0], a[5]); // Sum(5) + res6 = fma52lo(res6, a[1], a[5]); // Sum(6) + res7 = fma52hi(res7, a[1], a[5]); // Sum(6) + res7 = fma52lo(res7, a[2], a[5]); // Sum(7) + res8 = fma52hi(res8, a[2], a[5]); // Sum(7) + res8 = fma52lo(res8, a[3], a[5]); // Sum(8) + res9 = fma52hi(res9, a[3], a[5]); // Sum(8) + res9 = fma52lo(res9, a[4], a[5]); // Sum(9) + res10 = fma52hi(res10, a[4], a[5]); // Sum(9) + res6 = fma52lo(res6, a[0], a[6]); // Sum(6) + res7 = fma52hi(res7, a[0], a[6]); // Sum(6) + res7 = fma52lo(res7, a[1], a[6]); // Sum(7) + res8 = fma52hi(res8, a[1], a[6]); // Sum(7) + res8 = fma52lo(res8, a[2], a[6]); // Sum(8) + res9 = fma52hi(res9, a[2], a[6]); // Sum(8) + res9 = fma52lo(res9, a[3], a[6]); // Sum(9) + res10 = fma52hi(res10, a[3], a[6]); // Sum(9) + res10 = fma52lo(res10, a[4], a[6]); // Sum(10) + res11 = fma52hi(res11, a[4], a[6]); // Sum(10) + res11 = fma52lo(res11, a[5], a[6]); // Sum(11) + res12 = fma52hi(res12, a[5], a[6]); // Sum(11) + res7 = fma52lo(res7, a[0], a[7]); // Sum(7) + res8 = fma52hi(res8, a[0], a[7]); // Sum(7) + res8 = fma52lo(res8, a[1], a[7]); // Sum(8) + res9 = fma52hi(res9, a[1], a[7]); // Sum(8) + res9 = fma52lo(res9, a[2], a[7]); // Sum(9) + res10 = fma52hi(res10, a[2], a[7]); // Sum(9) + res10 = fma52lo(res10, a[3], a[7]); // Sum(10) + res11 = fma52hi(res11, a[3], a[7]); // Sum(10) + res11 = fma52lo(res11, a[4], a[7]); // Sum(11) + res12 = fma52hi(res12, a[4], a[7]); // Sum(11) + res8 = fma52lo(res8, a[0], a[8]); // Sum(8) + res9 = fma52hi(res9, a[0], a[8]); // Sum(8) + res9 = fma52lo(res9, a[1], a[8]); // Sum(9) + res10 = fma52hi(res10, a[1], a[8]); // Sum(9) + res10 = fma52lo(res10, a[2], a[8]); // Sum(10) + res11 = fma52hi(res11, a[2], a[8]); // Sum(10) + res11 = fma52lo(res11, a[3], a[8]); // Sum(11) + res12 = fma52hi(res12, a[3], a[8]); // Sum(11) + res9 = fma52lo(res9, a[0], a[9]); // Sum(9) + res10 = fma52hi(res10, a[0], a[9]); // Sum(9) + res10 = fma52lo(res10, a[1], a[9]); // Sum(10) + res11 = fma52hi(res11, a[1], a[9]); // Sum(10) + res11 = fma52lo(res11, a[2], a[9]); // Sum(11) + res12 = fma52hi(res12, a[2], a[9]); // Sum(11) + res0 = add64(res0, res0); // Double(0) + res1 = add64(res1, res1); // Double(1) + res2 = add64(res2, res2); // Double(2) + res3 = add64(res3, res3); // Double(3) + res4 = add64(res4, res4); // Double(4) + res5 = add64(res5, res5); // Double(5) + res6 = add64(res6, res6); // Double(6) + res7 = add64(res7, res7); // Double(7) + res8 = add64(res8, res8); // Double(8) + res9 = add64(res9, res9); // Double(9) + res10 = add64(res10, res10); // Double(10) + res11 = add64(res11, res11); // Double(11) + res0 = fma52lo(res0, a[0], a[0]); // Add sqr(0) + res1 = fma52hi(res1, a[0], a[0]); // Add sqr(0) + res2 = fma52lo(res2, a[1], a[1]); // Add sqr(2) + res3 = fma52hi(res3, a[1], a[1]); // Add sqr(2) + res4 = fma52lo(res4, a[2], a[2]); // Add sqr(4) + res5 = fma52hi(res5, a[2], a[2]); // Add sqr(4) + res6 = fma52lo(res6, a[3], a[3]); // Add sqr(6) + res7 = fma52hi(res7, a[3], a[3]); // Add sqr(6) + res8 = fma52lo(res8, a[4], a[4]); // Add sqr(8) + res9 = fma52hi(res9, a[4], a[4]); // Add sqr(8) + res10 = fma52lo(res10, a[5], a[5]); // Add sqr(10) + res11 = fma52hi(res11, a[5], a[5]); // Add sqr(10) + res12 = fma52lo(res12, a[5], a[7]); // Sum(12) + res13 = fma52hi(res13, a[5], a[7]); // Sum(12) + res13 = fma52lo(res13, a[6], a[7]); // Sum(13) + res14 = fma52hi(res14, a[6], a[7]); // Sum(13) + res12 = fma52lo(res12, a[4], a[8]); // Sum(12) + res13 = fma52hi(res13, a[4], a[8]); // Sum(12) + res13 = fma52lo(res13, a[5], a[8]); // Sum(13) + res14 = fma52hi(res14, a[5], a[8]); // Sum(13) + res14 = fma52lo(res14, a[6], a[8]); // Sum(14) + res15 = fma52hi(res15, a[6], a[8]); // Sum(14) + res15 = fma52lo(res15, a[7], a[8]); // Sum(15) + res16 = fma52hi(res16, a[7], a[8]); // Sum(15) + res12 = fma52lo(res12, a[3], a[9]); // Sum(12) + res13 = fma52hi(res13, a[3], a[9]); // Sum(12) + res13 = fma52lo(res13, a[4], a[9]); // Sum(13) + res14 = fma52hi(res14, a[4], a[9]); // Sum(13) + res14 = fma52lo(res14, a[5], a[9]); // Sum(14) + res15 = fma52hi(res15, a[5], a[9]); // Sum(14) + res15 = fma52lo(res15, a[6], a[9]); // Sum(15) + res16 = fma52hi(res16, a[6], a[9]); // Sum(15) + res16 = fma52lo(res16, a[7], a[9]); // Sum(16) + res17 = fma52hi(res17, a[7], a[9]); // Sum(16) + res17 = fma52lo(res17, a[8], a[9]); // Sum(17) + res18 = fma52hi(res18, a[8], a[9]); // Sum(17) + res12 = add64(res12, res12); // Double(12) + res13 = add64(res13, res13); // Double(13) + res14 = add64(res14, res14); // Double(14) + res15 = add64(res15, res15); // Double(15) + res16 = add64(res16, res16); // Double(16) + res17 = add64(res17, res17); // Double(17) + res18 = add64(res18, res18); // Double(18) + res12 = fma52lo(res12, a[6], a[6]); // Add sqr(12) + res13 = fma52hi(res13, a[6], a[6]); // Add sqr(12) + res14 = fma52lo(res14, a[7], a[7]); // Add sqr(14) + res15 = fma52hi(res15, a[7], a[7]); // Add sqr(14) + res16 = fma52lo(res16, a[8], a[8]); // Add sqr(16) + res17 = fma52hi(res17, a[8], a[8]); // Add sqr(16) + res18 = fma52lo(res18, a[9], a[9]); // Add sqr(18) + res19 = fma52hi(res19, a[9], a[9]); // Add sqr(18) + + // Generate u_i + U64 u0 = mul52lo(res0, k); + ASM("jmp l0\nl0:\n"); + + // Create u0 + fma52lo_mem(res0, res0, u0, m, SIMD_BYTES * 0); + fma52hi_mem(res1, res1, u0, m, SIMD_BYTES * 0); + res1 = fma52lo(res1, u0, m[1]); + res2 = fma52hi(res2, u0, m[1]); + res1 = add64(res1, srli64(res0, DIGIT_SIZE)); + U64 u1 = mul52lo(res1, k); + fma52lo_mem(res2, res2, u0, m, SIMD_BYTES * 2); + fma52hi_mem(res3, res3, u0, m, SIMD_BYTES * 2); + res3 = fma52lo(res3, u0, m[3]); + res4 = fma52hi(res4, u0, m[3]); + fma52lo_mem(res4, res4, u0, m, SIMD_BYTES * 4); + fma52hi_mem(res5, res5, u0, m, SIMD_BYTES * 4); + res5 = fma52lo(res5, u0, m[5]); + res6 = fma52hi(res6, u0, m[5]); + fma52lo_mem(res6, res6, u0, m, SIMD_BYTES * 6); + fma52hi_mem(res7, res7, u0, m, SIMD_BYTES * 6); + res7 = fma52lo(res7, u0, m[7]); + res8 = fma52hi(res8, u0, m[7]); + fma52lo_mem(res8, res8, u0, m, SIMD_BYTES * 8); + fma52hi_mem(res9, res9, u0, m, SIMD_BYTES * 8); + res9 = fma52lo(res9, u0, m[9]); + res10 = fma52hi(res10, u0, m[9]); + + // Create u1 + fma52lo_mem(res1, res1, u1, m, SIMD_BYTES * 0); + fma52hi_mem(res2, res2, u1, m, SIMD_BYTES * 0); + res2 = fma52lo(res2, u1, m[1]); + res3 = fma52hi(res3, u1, m[1]); + res2 = add64(res2, srli64(res1, DIGIT_SIZE)); + U64 u2 = mul52lo(res2, k); + fma52lo_mem(res3, res3, u1, m, SIMD_BYTES * 2); + fma52hi_mem(res4, res4, u1, m, SIMD_BYTES * 2); + res4 = fma52lo(res4, u1, m[3]); + res5 = fma52hi(res5, u1, m[3]); + fma52lo_mem(res5, res5, u1, m, SIMD_BYTES * 4); + fma52hi_mem(res6, res6, u1, m, SIMD_BYTES * 4); + res6 = fma52lo(res6, u1, m[5]); + res7 = fma52hi(res7, u1, m[5]); + fma52lo_mem(res7, res7, u1, m, SIMD_BYTES * 6); + fma52hi_mem(res8, res8, u1, m, SIMD_BYTES * 6); + res8 = fma52lo(res8, u1, m[7]); + res9 = fma52hi(res9, u1, m[7]); + fma52lo_mem(res9, res9, u1, m, SIMD_BYTES * 8); + fma52hi_mem(res10, res10, u1, m, SIMD_BYTES * 8); + res10 = fma52lo(res10, u1, m[9]); + res11 = fma52hi(res11, u1, m[9]); + ASM("jmp l2\nl2:\n"); + + // Create u2 + fma52lo_mem(res2, res2, u2, m, SIMD_BYTES * 0); + fma52hi_mem(res3, res3, u2, m, SIMD_BYTES * 0); + res3 = fma52lo(res3, u2, m[1]); + res4 = fma52hi(res4, u2, m[1]); + res3 = add64(res3, srli64(res2, DIGIT_SIZE)); + U64 u3 = mul52lo(res3, k); + fma52lo_mem(res4, res4, u2, m, SIMD_BYTES * 2); + fma52hi_mem(res5, res5, u2, m, SIMD_BYTES * 2); + res5 = fma52lo(res5, u2, m[3]); + res6 = fma52hi(res6, u2, m[3]); + fma52lo_mem(res6, res6, u2, m, SIMD_BYTES * 4); + fma52hi_mem(res7, res7, u2, m, SIMD_BYTES * 4); + res7 = fma52lo(res7, u2, m[5]); + res8 = fma52hi(res8, u2, m[5]); + fma52lo_mem(res8, res8, u2, m, SIMD_BYTES * 6); + fma52hi_mem(res9, res9, u2, m, SIMD_BYTES * 6); + res9 = fma52lo(res9, u2, m[7]); + res10 = fma52hi(res10, u2, m[7]); + fma52lo_mem(res10, res10, u2, m, SIMD_BYTES * 8); + fma52hi_mem(res11, res11, u2, m, SIMD_BYTES * 8); + res11 = fma52lo(res11, u2, m[9]); + res12 = fma52hi(res12, u2, m[9]); + + // Create u3 + fma52lo_mem(res3, res3, u3, m, SIMD_BYTES * 0); + fma52hi_mem(res4, res4, u3, m, SIMD_BYTES * 0); + res4 = fma52lo(res4, u3, m[1]); + res5 = fma52hi(res5, u3, m[1]); + res4 = add64(res4, srli64(res3, DIGIT_SIZE)); + U64 u4 = mul52lo(res4, k); + fma52lo_mem(res5, res5, u3, m, SIMD_BYTES * 2); + fma52hi_mem(res6, res6, u3, m, SIMD_BYTES * 2); + res6 = fma52lo(res6, u3, m[3]); + res7 = fma52hi(res7, u3, m[3]); + fma52lo_mem(res7, res7, u3, m, SIMD_BYTES * 4); + fma52hi_mem(res8, res8, u3, m, SIMD_BYTES * 4); + res8 = fma52lo(res8, u3, m[5]); + res9 = fma52hi(res9, u3, m[5]); + fma52lo_mem(res9, res9, u3, m, SIMD_BYTES * 6); + fma52hi_mem(res10, res10, u3, m, SIMD_BYTES * 6); + res10 = fma52lo(res10, u3, m[7]); + res11 = fma52hi(res11, u3, m[7]); + fma52lo_mem(res11, res11, u3, m, SIMD_BYTES * 8); + fma52hi_mem(res12, res12, u3, m, SIMD_BYTES * 8); + res12 = fma52lo(res12, u3, m[9]); + res13 = fma52hi(res13, u3, m[9]); + ASM("jmp l4\nl4:\n"); + + // Create u4 + fma52lo_mem(res4, res4, u4, m, SIMD_BYTES * 0); + fma52hi_mem(res5, res5, u4, m, SIMD_BYTES * 0); + res5 = fma52lo(res5, u4, m[1]); + res6 = fma52hi(res6, u4, m[1]); + res5 = add64(res5, srli64(res4, DIGIT_SIZE)); + U64 u5 = mul52lo(res5, k); + fma52lo_mem(res6, res6, u4, m, SIMD_BYTES * 2); + fma52hi_mem(res7, res7, u4, m, SIMD_BYTES * 2); + res7 = fma52lo(res7, u4, m[3]); + res8 = fma52hi(res8, u4, m[3]); + fma52lo_mem(res8, res8, u4, m, SIMD_BYTES * 4); + fma52hi_mem(res9, res9, u4, m, SIMD_BYTES * 4); + res9 = fma52lo(res9, u4, m[5]); + res10 = fma52hi(res10, u4, m[5]); + fma52lo_mem(res10, res10, u4, m, SIMD_BYTES * 6); + fma52hi_mem(res11, res11, u4, m, SIMD_BYTES * 6); + res11 = fma52lo(res11, u4, m[7]); + res12 = fma52hi(res12, u4, m[7]); + fma52lo_mem(res12, res12, u4, m, SIMD_BYTES * 8); + fma52hi_mem(res13, res13, u4, m, SIMD_BYTES * 8); + res13 = fma52lo(res13, u4, m[9]); + res14 = fma52hi(res14, u4, m[9]); + + // Create u5 + fma52lo_mem(res5, res5, u5, m, SIMD_BYTES * 0); + fma52hi_mem(res6, res6, u5, m, SIMD_BYTES * 0); + res6 = fma52lo(res6, u5, m[1]); + res7 = fma52hi(res7, u5, m[1]); + res6 = add64(res6, srli64(res5, DIGIT_SIZE)); + U64 u6 = mul52lo(res6, k); + fma52lo_mem(res7, res7, u5, m, SIMD_BYTES * 2); + fma52hi_mem(res8, res8, u5, m, SIMD_BYTES * 2); + res8 = fma52lo(res8, u5, m[3]); + res9 = fma52hi(res9, u5, m[3]); + fma52lo_mem(res9, res9, u5, m, SIMD_BYTES * 4); + fma52hi_mem(res10, res10, u5, m, SIMD_BYTES * 4); + res10 = fma52lo(res10, u5, m[5]); + res11 = fma52hi(res11, u5, m[5]); + fma52lo_mem(res11, res11, u5, m, SIMD_BYTES * 6); + fma52hi_mem(res12, res12, u5, m, SIMD_BYTES * 6); + res12 = fma52lo(res12, u5, m[7]); + res13 = fma52hi(res13, u5, m[7]); + fma52lo_mem(res13, res13, u5, m, SIMD_BYTES * 8); + fma52hi_mem(res14, res14, u5, m, SIMD_BYTES * 8); + res14 = fma52lo(res14, u5, m[9]); + res15 = fma52hi(res15, u5, m[9]); + ASM("jmp l6\nl6:\n"); + + // Create u6 + fma52lo_mem(res6, res6, u6, m, SIMD_BYTES * 0); + fma52hi_mem(res7, res7, u6, m, SIMD_BYTES * 0); + res7 = fma52lo(res7, u6, m[1]); + res8 = fma52hi(res8, u6, m[1]); + res7 = add64(res7, srli64(res6, DIGIT_SIZE)); + U64 u7 = mul52lo(res7, k); + fma52lo_mem(res8, res8, u6, m, SIMD_BYTES * 2); + fma52hi_mem(res9, res9, u6, m, SIMD_BYTES * 2); + res9 = fma52lo(res9, u6, m[3]); + res10 = fma52hi(res10, u6, m[3]); + fma52lo_mem(res10, res10, u6, m, SIMD_BYTES * 4); + fma52hi_mem(res11, res11, u6, m, SIMD_BYTES * 4); + res11 = fma52lo(res11, u6, m[5]); + res12 = fma52hi(res12, u6, m[5]); + fma52lo_mem(res12, res12, u6, m, SIMD_BYTES * 6); + fma52hi_mem(res13, res13, u6, m, SIMD_BYTES * 6); + res13 = fma52lo(res13, u6, m[7]); + res14 = fma52hi(res14, u6, m[7]); + fma52lo_mem(res14, res14, u6, m, SIMD_BYTES * 8); + fma52hi_mem(res15, res15, u6, m, SIMD_BYTES * 8); + res15 = fma52lo(res15, u6, m[9]); + res16 = fma52hi(res16, u6, m[9]); + + // Create u7 + fma52lo_mem(res7, res7, u7, m, SIMD_BYTES * 0); + fma52hi_mem(res8, res8, u7, m, SIMD_BYTES * 0); + res8 = fma52lo(res8, u7, m[1]); + res9 = fma52hi(res9, u7, m[1]); + res8 = add64(res8, srli64(res7, DIGIT_SIZE)); + U64 u8 = mul52lo(res8, k); + fma52lo_mem(res9, res9, u7, m, SIMD_BYTES * 2); + fma52hi_mem(res10, res10, u7, m, SIMD_BYTES * 2); + res10 = fma52lo(res10, u7, m[3]); + res11 = fma52hi(res11, u7, m[3]); + fma52lo_mem(res11, res11, u7, m, SIMD_BYTES * 4); + fma52hi_mem(res12, res12, u7, m, SIMD_BYTES * 4); + res12 = fma52lo(res12, u7, m[5]); + res13 = fma52hi(res13, u7, m[5]); + fma52lo_mem(res13, res13, u7, m, SIMD_BYTES * 6); + fma52hi_mem(res14, res14, u7, m, SIMD_BYTES * 6); + res14 = fma52lo(res14, u7, m[7]); + res15 = fma52hi(res15, u7, m[7]); + fma52lo_mem(res15, res15, u7, m, SIMD_BYTES * 8); + fma52hi_mem(res16, res16, u7, m, SIMD_BYTES * 8); + res16 = fma52lo(res16, u7, m[9]); + res17 = fma52hi(res17, u7, m[9]); + ASM("jmp l8\nl8:\n"); + + // Create u8 + fma52lo_mem(res8, res8, u8, m, SIMD_BYTES * 0); + fma52hi_mem(res9, res9, u8, m, SIMD_BYTES * 0); + res9 = fma52lo(res9, u8, m[1]); + res10 = fma52hi(res10, u8, m[1]); + res9 = add64(res9, srli64(res8, DIGIT_SIZE)); + U64 u9 = mul52lo(res9, k); + fma52lo_mem(res10, res10, u8, m, SIMD_BYTES * 2); + fma52hi_mem(res11, res11, u8, m, SIMD_BYTES * 2); + res11 = fma52lo(res11, u8, m[3]); + res12 = fma52hi(res12, u8, m[3]); + fma52lo_mem(res12, res12, u8, m, SIMD_BYTES * 4); + fma52hi_mem(res13, res13, u8, m, SIMD_BYTES * 4); + res13 = fma52lo(res13, u8, m[5]); + res14 = fma52hi(res14, u8, m[5]); + fma52lo_mem(res14, res14, u8, m, SIMD_BYTES * 6); + fma52hi_mem(res15, res15, u8, m, SIMD_BYTES * 6); + res15 = fma52lo(res15, u8, m[7]); + res16 = fma52hi(res16, u8, m[7]); + fma52lo_mem(res16, res16, u8, m, SIMD_BYTES * 8); + fma52hi_mem(res17, res17, u8, m, SIMD_BYTES * 8); + res17 = fma52lo(res17, u8, m[9]); + res18 = fma52hi(res18, u8, m[9]); + + // Create u9 + fma52lo_mem(res9, res9, u9, m, SIMD_BYTES * 0); + fma52hi_mem(res10, res10, u9, m, SIMD_BYTES * 0); + res10 = fma52lo(res10, u9, m[1]); + res11 = fma52hi(res11, u9, m[1]); + res10 = add64(res10, srli64(res9, DIGIT_SIZE)); + fma52lo_mem(res11, res11, u9, m, SIMD_BYTES * 2); + fma52hi_mem(res12, res12, u9, m, SIMD_BYTES * 2); + res12 = fma52lo(res12, u9, m[3]); + res13 = fma52hi(res13, u9, m[3]); + fma52lo_mem(res13, res13, u9, m, SIMD_BYTES * 4); + fma52hi_mem(res14, res14, u9, m, SIMD_BYTES * 4); + res14 = fma52lo(res14, u9, m[5]); + res15 = fma52hi(res15, u9, m[5]); + fma52lo_mem(res15, res15, u9, m, SIMD_BYTES * 6); + fma52hi_mem(res16, res16, u9, m, SIMD_BYTES * 6); + res16 = fma52lo(res16, u9, m[7]); + res17 = fma52hi(res17, u9, m[7]); + fma52lo_mem(res17, res17, u9, m, SIMD_BYTES * 8); + fma52hi_mem(res18, res18, u9, m, SIMD_BYTES * 8); + res18 = fma52lo(res18, u9, m[9]); + res19 = fma52hi(res19, u9, m[9]); + + // Normalization + r[0] = and64_const(res10, DIGIT_MASK); + res11 = add64(res11, srli64(res10, DIGIT_SIZE)); + r[1] = and64_const(res11, DIGIT_MASK); + res12 = add64(res12, srli64(res11, DIGIT_SIZE)); + r[2] = and64_const(res12, DIGIT_MASK); + res13 = add64(res13, srli64(res12, DIGIT_SIZE)); + r[3] = and64_const(res13, DIGIT_MASK); + res14 = add64(res14, srli64(res13, DIGIT_SIZE)); + r[4] = and64_const(res14, DIGIT_MASK); + res15 = add64(res15, srli64(res14, DIGIT_SIZE)); + r[5] = and64_const(res15, DIGIT_MASK); + res16 = add64(res16, srli64(res15, DIGIT_SIZE)); + r[6] = and64_const(res16, DIGIT_MASK); + res17 = add64(res17, srli64(res16, DIGIT_SIZE)); + r[7] = and64_const(res17, DIGIT_MASK); + res18 = add64(res18, srli64(res17, DIGIT_SIZE)); + r[8] = and64_const(res18, DIGIT_MASK); + res19 = add64(res19, srli64(res18, DIGIT_SIZE)); + r[9] = and64_const(res19, DIGIT_MASK); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams52x20_diagonal_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams52x20_diagonal_mb8.c new file mode 100644 index 0000000..2f7bf77 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams52x20_diagonal_mb8.c @@ -0,0 +1,1435 @@ +/******************************************************************************* + * Copyright (C) 2019 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include + +#ifdef __GNUC__ +#define ASM(a) __asm__(a); +#else +#define ASM(a) +#endif + +void AMS52x20_diagonal_mb8(int64u *out_mb, const int64u *inpA_mb, + const int64u *inpM_mb, const int64u *k0_mb) { + U64 res0, res1, res2, res3, res4, res5, res6, res7, res8, res9, res10, res11, + res12, res13, res14, res15, res16, res17, res18, res19, res20, res21, + res22, res23, res24, res25, res26, res27, res28, res29, res30, res31, + res32, res33, res34, res35, res36, res37, res38, res39; + + U64 *a = (U64 *)inpA_mb; + U64 *m = (U64 *)inpM_mb; + U64 *r = (U64 *)out_mb; + U64 k = loadu64((U64 *)k0_mb); + + res0 = res1 = res2 = res3 = res4 = res5 = res6 = res7 = res8 = res9 = res10 = + res11 = res12 = res13 = res14 = res15 = res16 = res17 = res18 = res19 = + res20 = res21 = res22 = res23 = res24 = res25 = res26 = res27 = + res28 = res29 = res30 = res31 = res32 = res33 = res34 = res35 = + res36 = res37 = res38 = res39 = get_zero64(); + res1 = fma52lo(res1, a[0], a[1]); // Sum(1) + res2 = fma52hi(res2, a[0], a[1]); // Sum(1) + res2 = fma52lo(res2, a[0], a[2]); // Sum(2) + res3 = fma52hi(res3, a[0], a[2]); // Sum(2) + res3 = fma52lo(res3, a[1], a[2]); // Sum(3) + res4 = fma52hi(res4, a[1], a[2]); // Sum(3) + res3 = fma52lo(res3, a[0], a[3]); // Sum(3) + res4 = fma52hi(res4, a[0], a[3]); // Sum(3) + res4 = fma52lo(res4, a[1], a[3]); // Sum(4) + res5 = fma52hi(res5, a[1], a[3]); // Sum(4) + res5 = fma52lo(res5, a[2], a[3]); // Sum(5) + res6 = fma52hi(res6, a[2], a[3]); // Sum(5) + res4 = fma52lo(res4, a[0], a[4]); // Sum(4) + res5 = fma52hi(res5, a[0], a[4]); // Sum(4) + res5 = fma52lo(res5, a[1], a[4]); // Sum(5) + res6 = fma52hi(res6, a[1], a[4]); // Sum(5) + res6 = fma52lo(res6, a[2], a[4]); // Sum(6) + res7 = fma52hi(res7, a[2], a[4]); // Sum(6) + res7 = fma52lo(res7, a[3], a[4]); // Sum(7) + res8 = fma52hi(res8, a[3], a[4]); // Sum(7) + res5 = fma52lo(res5, a[0], a[5]); // Sum(5) + res6 = fma52hi(res6, a[0], a[5]); // Sum(5) + res6 = fma52lo(res6, a[1], a[5]); // Sum(6) + res7 = fma52hi(res7, a[1], a[5]); // Sum(6) + res7 = fma52lo(res7, a[2], a[5]); // Sum(7) + res8 = fma52hi(res8, a[2], a[5]); // Sum(7) + res8 = fma52lo(res8, a[3], a[5]); // Sum(8) + res9 = fma52hi(res9, a[3], a[5]); // Sum(8) + res9 = fma52lo(res9, a[4], a[5]); // Sum(9) + res10 = fma52hi(res10, a[4], a[5]); // Sum(9) + res6 = fma52lo(res6, a[0], a[6]); // Sum(6) + res7 = fma52hi(res7, a[0], a[6]); // Sum(6) + res7 = fma52lo(res7, a[1], a[6]); // Sum(7) + res8 = fma52hi(res8, a[1], a[6]); // Sum(7) + res8 = fma52lo(res8, a[2], a[6]); // Sum(8) + res9 = fma52hi(res9, a[2], a[6]); // Sum(8) + res9 = fma52lo(res9, a[3], a[6]); // Sum(9) + res10 = fma52hi(res10, a[3], a[6]); // Sum(9) + res10 = fma52lo(res10, a[4], a[6]); // Sum(10) + res11 = fma52hi(res11, a[4], a[6]); // Sum(10) + res11 = fma52lo(res11, a[5], a[6]); // Sum(11) + res12 = fma52hi(res12, a[5], a[6]); // Sum(11) + res7 = fma52lo(res7, a[0], a[7]); // Sum(7) + res8 = fma52hi(res8, a[0], a[7]); // Sum(7) + res8 = fma52lo(res8, a[1], a[7]); // Sum(8) + res9 = fma52hi(res9, a[1], a[7]); // Sum(8) + res9 = fma52lo(res9, a[2], a[7]); // Sum(9) + res10 = fma52hi(res10, a[2], a[7]); // Sum(9) + res10 = fma52lo(res10, a[3], a[7]); // Sum(10) + res11 = fma52hi(res11, a[3], a[7]); // Sum(10) + res11 = fma52lo(res11, a[4], a[7]); // Sum(11) + res12 = fma52hi(res12, a[4], a[7]); // Sum(11) + res8 = fma52lo(res8, a[0], a[8]); // Sum(8) + res9 = fma52hi(res9, a[0], a[8]); // Sum(8) + res9 = fma52lo(res9, a[1], a[8]); // Sum(9) + res10 = fma52hi(res10, a[1], a[8]); // Sum(9) + res10 = fma52lo(res10, a[2], a[8]); // Sum(10) + res11 = fma52hi(res11, a[2], a[8]); // Sum(10) + res11 = fma52lo(res11, a[3], a[8]); // Sum(11) + res12 = fma52hi(res12, a[3], a[8]); // Sum(11) + res9 = fma52lo(res9, a[0], a[9]); // Sum(9) + res10 = fma52hi(res10, a[0], a[9]); // Sum(9) + res10 = fma52lo(res10, a[1], a[9]); // Sum(10) + res11 = fma52hi(res11, a[1], a[9]); // Sum(10) + res11 = fma52lo(res11, a[2], a[9]); // Sum(11) + res12 = fma52hi(res12, a[2], a[9]); // Sum(11) + res10 = fma52lo(res10, a[0], a[10]); // Sum(10) + res11 = fma52hi(res11, a[0], a[10]); // Sum(10) + res11 = fma52lo(res11, a[1], a[10]); // Sum(11) + res12 = fma52hi(res12, a[1], a[10]); // Sum(11) + res11 = fma52lo(res11, a[0], a[11]); // Sum(11) + res12 = fma52hi(res12, a[0], a[11]); // Sum(11) + res0 = add64(res0, res0); // Double(0) + res1 = add64(res1, res1); // Double(1) + res2 = add64(res2, res2); // Double(2) + res3 = add64(res3, res3); // Double(3) + res4 = add64(res4, res4); // Double(4) + res5 = add64(res5, res5); // Double(5) + res6 = add64(res6, res6); // Double(6) + res7 = add64(res7, res7); // Double(7) + res8 = add64(res8, res8); // Double(8) + res9 = add64(res9, res9); // Double(9) + res10 = add64(res10, res10); // Double(10) + res11 = add64(res11, res11); // Double(11) + res0 = fma52lo(res0, a[0], a[0]); // Add sqr(0) + res1 = fma52hi(res1, a[0], a[0]); // Add sqr(0) + res2 = fma52lo(res2, a[1], a[1]); // Add sqr(2) + res3 = fma52hi(res3, a[1], a[1]); // Add sqr(2) + res4 = fma52lo(res4, a[2], a[2]); // Add sqr(4) + res5 = fma52hi(res5, a[2], a[2]); // Add sqr(4) + res6 = fma52lo(res6, a[3], a[3]); // Add sqr(6) + res7 = fma52hi(res7, a[3], a[3]); // Add sqr(6) + res8 = fma52lo(res8, a[4], a[4]); // Add sqr(8) + res9 = fma52hi(res9, a[4], a[4]); // Add sqr(8) + res10 = fma52lo(res10, a[5], a[5]); // Add sqr(10) + res11 = fma52hi(res11, a[5], a[5]); // Add sqr(10) + res12 = fma52lo(res12, a[5], a[7]); // Sum(12) + res13 = fma52hi(res13, a[5], a[7]); // Sum(12) + res13 = fma52lo(res13, a[6], a[7]); // Sum(13) + res14 = fma52hi(res14, a[6], a[7]); // Sum(13) + res12 = fma52lo(res12, a[4], a[8]); // Sum(12) + res13 = fma52hi(res13, a[4], a[8]); // Sum(12) + res13 = fma52lo(res13, a[5], a[8]); // Sum(13) + res14 = fma52hi(res14, a[5], a[8]); // Sum(13) + res14 = fma52lo(res14, a[6], a[8]); // Sum(14) + res15 = fma52hi(res15, a[6], a[8]); // Sum(14) + res15 = fma52lo(res15, a[7], a[8]); // Sum(15) + res16 = fma52hi(res16, a[7], a[8]); // Sum(15) + res12 = fma52lo(res12, a[3], a[9]); // Sum(12) + res13 = fma52hi(res13, a[3], a[9]); // Sum(12) + res13 = fma52lo(res13, a[4], a[9]); // Sum(13) + res14 = fma52hi(res14, a[4], a[9]); // Sum(13) + res14 = fma52lo(res14, a[5], a[9]); // Sum(14) + res15 = fma52hi(res15, a[5], a[9]); // Sum(14) + res15 = fma52lo(res15, a[6], a[9]); // Sum(15) + res16 = fma52hi(res16, a[6], a[9]); // Sum(15) + res16 = fma52lo(res16, a[7], a[9]); // Sum(16) + res17 = fma52hi(res17, a[7], a[9]); // Sum(16) + res17 = fma52lo(res17, a[8], a[9]); // Sum(17) + res18 = fma52hi(res18, a[8], a[9]); // Sum(17) + res12 = fma52lo(res12, a[2], a[10]); // Sum(12) + res13 = fma52hi(res13, a[2], a[10]); // Sum(12) + res13 = fma52lo(res13, a[3], a[10]); // Sum(13) + res14 = fma52hi(res14, a[3], a[10]); // Sum(13) + res14 = fma52lo(res14, a[4], a[10]); // Sum(14) + res15 = fma52hi(res15, a[4], a[10]); // Sum(14) + res15 = fma52lo(res15, a[5], a[10]); // Sum(15) + res16 = fma52hi(res16, a[5], a[10]); // Sum(15) + res16 = fma52lo(res16, a[6], a[10]); // Sum(16) + res17 = fma52hi(res17, a[6], a[10]); // Sum(16) + res17 = fma52lo(res17, a[7], a[10]); // Sum(17) + res18 = fma52hi(res18, a[7], a[10]); // Sum(17) + res18 = fma52lo(res18, a[8], a[10]); // Sum(18) + res19 = fma52hi(res19, a[8], a[10]); // Sum(18) + res19 = fma52lo(res19, a[9], a[10]); // Sum(19) + res20 = fma52hi(res20, a[9], a[10]); // Sum(19) + res12 = fma52lo(res12, a[1], a[11]); // Sum(12) + res13 = fma52hi(res13, a[1], a[11]); // Sum(12) + res13 = fma52lo(res13, a[2], a[11]); // Sum(13) + res14 = fma52hi(res14, a[2], a[11]); // Sum(13) + res14 = fma52lo(res14, a[3], a[11]); // Sum(14) + res15 = fma52hi(res15, a[3], a[11]); // Sum(14) + res15 = fma52lo(res15, a[4], a[11]); // Sum(15) + res16 = fma52hi(res16, a[4], a[11]); // Sum(15) + res16 = fma52lo(res16, a[5], a[11]); // Sum(16) + res17 = fma52hi(res17, a[5], a[11]); // Sum(16) + res17 = fma52lo(res17, a[6], a[11]); // Sum(17) + res18 = fma52hi(res18, a[6], a[11]); // Sum(17) + res18 = fma52lo(res18, a[7], a[11]); // Sum(18) + res19 = fma52hi(res19, a[7], a[11]); // Sum(18) + res19 = fma52lo(res19, a[8], a[11]); // Sum(19) + res20 = fma52hi(res20, a[8], a[11]); // Sum(19) + res20 = fma52lo(res20, a[9], a[11]); // Sum(20) + res21 = fma52hi(res21, a[9], a[11]); // Sum(20) + res21 = fma52lo(res21, a[10], a[11]); // Sum(21) + res22 = fma52hi(res22, a[10], a[11]); // Sum(21) + res12 = fma52lo(res12, a[0], a[12]); // Sum(12) + res13 = fma52hi(res13, a[0], a[12]); // Sum(12) + res13 = fma52lo(res13, a[1], a[12]); // Sum(13) + res14 = fma52hi(res14, a[1], a[12]); // Sum(13) + res14 = fma52lo(res14, a[2], a[12]); // Sum(14) + res15 = fma52hi(res15, a[2], a[12]); // Sum(14) + res15 = fma52lo(res15, a[3], a[12]); // Sum(15) + res16 = fma52hi(res16, a[3], a[12]); // Sum(15) + res16 = fma52lo(res16, a[4], a[12]); // Sum(16) + res17 = fma52hi(res17, a[4], a[12]); // Sum(16) + res17 = fma52lo(res17, a[5], a[12]); // Sum(17) + res18 = fma52hi(res18, a[5], a[12]); // Sum(17) + res18 = fma52lo(res18, a[6], a[12]); // Sum(18) + res19 = fma52hi(res19, a[6], a[12]); // Sum(18) + res19 = fma52lo(res19, a[7], a[12]); // Sum(19) + res20 = fma52hi(res20, a[7], a[12]); // Sum(19) + res20 = fma52lo(res20, a[8], a[12]); // Sum(20) + res21 = fma52hi(res21, a[8], a[12]); // Sum(20) + res21 = fma52lo(res21, a[9], a[12]); // Sum(21) + res22 = fma52hi(res22, a[9], a[12]); // Sum(21) + res22 = fma52lo(res22, a[10], a[12]); // Sum(22) + res23 = fma52hi(res23, a[10], a[12]); // Sum(22) + res23 = fma52lo(res23, a[11], a[12]); // Sum(23) + res24 = fma52hi(res24, a[11], a[12]); // Sum(23) + res13 = fma52lo(res13, a[0], a[13]); // Sum(13) + res14 = fma52hi(res14, a[0], a[13]); // Sum(13) + res14 = fma52lo(res14, a[1], a[13]); // Sum(14) + res15 = fma52hi(res15, a[1], a[13]); // Sum(14) + res15 = fma52lo(res15, a[2], a[13]); // Sum(15) + res16 = fma52hi(res16, a[2], a[13]); // Sum(15) + res16 = fma52lo(res16, a[3], a[13]); // Sum(16) + res17 = fma52hi(res17, a[3], a[13]); // Sum(16) + res17 = fma52lo(res17, a[4], a[13]); // Sum(17) + res18 = fma52hi(res18, a[4], a[13]); // Sum(17) + res18 = fma52lo(res18, a[5], a[13]); // Sum(18) + res19 = fma52hi(res19, a[5], a[13]); // Sum(18) + res19 = fma52lo(res19, a[6], a[13]); // Sum(19) + res20 = fma52hi(res20, a[6], a[13]); // Sum(19) + res20 = fma52lo(res20, a[7], a[13]); // Sum(20) + res21 = fma52hi(res21, a[7], a[13]); // Sum(20) + res21 = fma52lo(res21, a[8], a[13]); // Sum(21) + res22 = fma52hi(res22, a[8], a[13]); // Sum(21) + res22 = fma52lo(res22, a[9], a[13]); // Sum(22) + res23 = fma52hi(res23, a[9], a[13]); // Sum(22) + res23 = fma52lo(res23, a[10], a[13]); // Sum(23) + res24 = fma52hi(res24, a[10], a[13]); // Sum(23) + res14 = fma52lo(res14, a[0], a[14]); // Sum(14) + res15 = fma52hi(res15, a[0], a[14]); // Sum(14) + res15 = fma52lo(res15, a[1], a[14]); // Sum(15) + res16 = fma52hi(res16, a[1], a[14]); // Sum(15) + res16 = fma52lo(res16, a[2], a[14]); // Sum(16) + res17 = fma52hi(res17, a[2], a[14]); // Sum(16) + res17 = fma52lo(res17, a[3], a[14]); // Sum(17) + res18 = fma52hi(res18, a[3], a[14]); // Sum(17) + res18 = fma52lo(res18, a[4], a[14]); // Sum(18) + res19 = fma52hi(res19, a[4], a[14]); // Sum(18) + res19 = fma52lo(res19, a[5], a[14]); // Sum(19) + res20 = fma52hi(res20, a[5], a[14]); // Sum(19) + res20 = fma52lo(res20, a[6], a[14]); // Sum(20) + res21 = fma52hi(res21, a[6], a[14]); // Sum(20) + res21 = fma52lo(res21, a[7], a[14]); // Sum(21) + res22 = fma52hi(res22, a[7], a[14]); // Sum(21) + res22 = fma52lo(res22, a[8], a[14]); // Sum(22) + res23 = fma52hi(res23, a[8], a[14]); // Sum(22) + res23 = fma52lo(res23, a[9], a[14]); // Sum(23) + res24 = fma52hi(res24, a[9], a[14]); // Sum(23) + res15 = fma52lo(res15, a[0], a[15]); // Sum(15) + res16 = fma52hi(res16, a[0], a[15]); // Sum(15) + res16 = fma52lo(res16, a[1], a[15]); // Sum(16) + res17 = fma52hi(res17, a[1], a[15]); // Sum(16) + res17 = fma52lo(res17, a[2], a[15]); // Sum(17) + res18 = fma52hi(res18, a[2], a[15]); // Sum(17) + res18 = fma52lo(res18, a[3], a[15]); // Sum(18) + res19 = fma52hi(res19, a[3], a[15]); // Sum(18) + res19 = fma52lo(res19, a[4], a[15]); // Sum(19) + res20 = fma52hi(res20, a[4], a[15]); // Sum(19) + res20 = fma52lo(res20, a[5], a[15]); // Sum(20) + res21 = fma52hi(res21, a[5], a[15]); // Sum(20) + res21 = fma52lo(res21, a[6], a[15]); // Sum(21) + res22 = fma52hi(res22, a[6], a[15]); // Sum(21) + res22 = fma52lo(res22, a[7], a[15]); // Sum(22) + res23 = fma52hi(res23, a[7], a[15]); // Sum(22) + res23 = fma52lo(res23, a[8], a[15]); // Sum(23) + res24 = fma52hi(res24, a[8], a[15]); // Sum(23) + res16 = fma52lo(res16, a[0], a[16]); // Sum(16) + res17 = fma52hi(res17, a[0], a[16]); // Sum(16) + res17 = fma52lo(res17, a[1], a[16]); // Sum(17) + res18 = fma52hi(res18, a[1], a[16]); // Sum(17) + res18 = fma52lo(res18, a[2], a[16]); // Sum(18) + res19 = fma52hi(res19, a[2], a[16]); // Sum(18) + res19 = fma52lo(res19, a[3], a[16]); // Sum(19) + res20 = fma52hi(res20, a[3], a[16]); // Sum(19) + res20 = fma52lo(res20, a[4], a[16]); // Sum(20) + res21 = fma52hi(res21, a[4], a[16]); // Sum(20) + res21 = fma52lo(res21, a[5], a[16]); // Sum(21) + res22 = fma52hi(res22, a[5], a[16]); // Sum(21) + res22 = fma52lo(res22, a[6], a[16]); // Sum(22) + res23 = fma52hi(res23, a[6], a[16]); // Sum(22) + res23 = fma52lo(res23, a[7], a[16]); // Sum(23) + res24 = fma52hi(res24, a[7], a[16]); // Sum(23) + res17 = fma52lo(res17, a[0], a[17]); // Sum(17) + res18 = fma52hi(res18, a[0], a[17]); // Sum(17) + res18 = fma52lo(res18, a[1], a[17]); // Sum(18) + res19 = fma52hi(res19, a[1], a[17]); // Sum(18) + res19 = fma52lo(res19, a[2], a[17]); // Sum(19) + res20 = fma52hi(res20, a[2], a[17]); // Sum(19) + res20 = fma52lo(res20, a[3], a[17]); // Sum(20) + res21 = fma52hi(res21, a[3], a[17]); // Sum(20) + res21 = fma52lo(res21, a[4], a[17]); // Sum(21) + res22 = fma52hi(res22, a[4], a[17]); // Sum(21) + res22 = fma52lo(res22, a[5], a[17]); // Sum(22) + res23 = fma52hi(res23, a[5], a[17]); // Sum(22) + res23 = fma52lo(res23, a[6], a[17]); // Sum(23) + res24 = fma52hi(res24, a[6], a[17]); // Sum(23) + res18 = fma52lo(res18, a[0], a[18]); // Sum(18) + res19 = fma52hi(res19, a[0], a[18]); // Sum(18) + res19 = fma52lo(res19, a[1], a[18]); // Sum(19) + res20 = fma52hi(res20, a[1], a[18]); // Sum(19) + res20 = fma52lo(res20, a[2], a[18]); // Sum(20) + res21 = fma52hi(res21, a[2], a[18]); // Sum(20) + res21 = fma52lo(res21, a[3], a[18]); // Sum(21) + res22 = fma52hi(res22, a[3], a[18]); // Sum(21) + res22 = fma52lo(res22, a[4], a[18]); // Sum(22) + res23 = fma52hi(res23, a[4], a[18]); // Sum(22) + res23 = fma52lo(res23, a[5], a[18]); // Sum(23) + res24 = fma52hi(res24, a[5], a[18]); // Sum(23) + res19 = fma52lo(res19, a[0], a[19]); // Sum(19) + res20 = fma52hi(res20, a[0], a[19]); // Sum(19) + res20 = fma52lo(res20, a[1], a[19]); // Sum(20) + res21 = fma52hi(res21, a[1], a[19]); // Sum(20) + res21 = fma52lo(res21, a[2], a[19]); // Sum(21) + res22 = fma52hi(res22, a[2], a[19]); // Sum(21) + res22 = fma52lo(res22, a[3], a[19]); // Sum(22) + res23 = fma52hi(res23, a[3], a[19]); // Sum(22) + res23 = fma52lo(res23, a[4], a[19]); // Sum(23) + res24 = fma52hi(res24, a[4], a[19]); // Sum(23) + res12 = add64(res12, res12); // Double(12) + res13 = add64(res13, res13); // Double(13) + res14 = add64(res14, res14); // Double(14) + res15 = add64(res15, res15); // Double(15) + res16 = add64(res16, res16); // Double(16) + res17 = add64(res17, res17); // Double(17) + res18 = add64(res18, res18); // Double(18) + res19 = add64(res19, res19); // Double(19) + res20 = add64(res20, res20); // Double(20) + res21 = add64(res21, res21); // Double(21) + res22 = add64(res22, res22); // Double(22) + res23 = add64(res23, res23); // Double(23) + res12 = fma52lo(res12, a[6], a[6]); // Add sqr(12) + res13 = fma52hi(res13, a[6], a[6]); // Add sqr(12) + res14 = fma52lo(res14, a[7], a[7]); // Add sqr(14) + res15 = fma52hi(res15, a[7], a[7]); // Add sqr(14) + res16 = fma52lo(res16, a[8], a[8]); // Add sqr(16) + res17 = fma52hi(res17, a[8], a[8]); // Add sqr(16) + res18 = fma52lo(res18, a[9], a[9]); // Add sqr(18) + res19 = fma52hi(res19, a[9], a[9]); // Add sqr(18) + res20 = fma52lo(res20, a[10], a[10]); // Add sqr(20) + res21 = fma52hi(res21, a[10], a[10]); // Add sqr(20) + res22 = fma52lo(res22, a[11], a[11]); // Add sqr(22) + res23 = fma52hi(res23, a[11], a[11]); // Add sqr(22) + res24 = fma52lo(res24, a[11], a[13]); // Sum(24) + res25 = fma52hi(res25, a[11], a[13]); // Sum(24) + res25 = fma52lo(res25, a[12], a[13]); // Sum(25) + res26 = fma52hi(res26, a[12], a[13]); // Sum(25) + res24 = fma52lo(res24, a[10], a[14]); // Sum(24) + res25 = fma52hi(res25, a[10], a[14]); // Sum(24) + res25 = fma52lo(res25, a[11], a[14]); // Sum(25) + res26 = fma52hi(res26, a[11], a[14]); // Sum(25) + res26 = fma52lo(res26, a[12], a[14]); // Sum(26) + res27 = fma52hi(res27, a[12], a[14]); // Sum(26) + res27 = fma52lo(res27, a[13], a[14]); // Sum(27) + res28 = fma52hi(res28, a[13], a[14]); // Sum(27) + res24 = fma52lo(res24, a[9], a[15]); // Sum(24) + res25 = fma52hi(res25, a[9], a[15]); // Sum(24) + res25 = fma52lo(res25, a[10], a[15]); // Sum(25) + res26 = fma52hi(res26, a[10], a[15]); // Sum(25) + res26 = fma52lo(res26, a[11], a[15]); // Sum(26) + res27 = fma52hi(res27, a[11], a[15]); // Sum(26) + res27 = fma52lo(res27, a[12], a[15]); // Sum(27) + res28 = fma52hi(res28, a[12], a[15]); // Sum(27) + res28 = fma52lo(res28, a[13], a[15]); // Sum(28) + res29 = fma52hi(res29, a[13], a[15]); // Sum(28) + res29 = fma52lo(res29, a[14], a[15]); // Sum(29) + res30 = fma52hi(res30, a[14], a[15]); // Sum(29) + res24 = fma52lo(res24, a[8], a[16]); // Sum(24) + res25 = fma52hi(res25, a[8], a[16]); // Sum(24) + res25 = fma52lo(res25, a[9], a[16]); // Sum(25) + res26 = fma52hi(res26, a[9], a[16]); // Sum(25) + res26 = fma52lo(res26, a[10], a[16]); // Sum(26) + res27 = fma52hi(res27, a[10], a[16]); // Sum(26) + res27 = fma52lo(res27, a[11], a[16]); // Sum(27) + res28 = fma52hi(res28, a[11], a[16]); // Sum(27) + res28 = fma52lo(res28, a[12], a[16]); // Sum(28) + res29 = fma52hi(res29, a[12], a[16]); // Sum(28) + res29 = fma52lo(res29, a[13], a[16]); // Sum(29) + res30 = fma52hi(res30, a[13], a[16]); // Sum(29) + res30 = fma52lo(res30, a[14], a[16]); // Sum(30) + res31 = fma52hi(res31, a[14], a[16]); // Sum(30) + res31 = fma52lo(res31, a[15], a[16]); // Sum(31) + res32 = fma52hi(res32, a[15], a[16]); // Sum(31) + res24 = fma52lo(res24, a[7], a[17]); // Sum(24) + res25 = fma52hi(res25, a[7], a[17]); // Sum(24) + res25 = fma52lo(res25, a[8], a[17]); // Sum(25) + res26 = fma52hi(res26, a[8], a[17]); // Sum(25) + res26 = fma52lo(res26, a[9], a[17]); // Sum(26) + res27 = fma52hi(res27, a[9], a[17]); // Sum(26) + res27 = fma52lo(res27, a[10], a[17]); // Sum(27) + res28 = fma52hi(res28, a[10], a[17]); // Sum(27) + res28 = fma52lo(res28, a[11], a[17]); // Sum(28) + res29 = fma52hi(res29, a[11], a[17]); // Sum(28) + res29 = fma52lo(res29, a[12], a[17]); // Sum(29) + res30 = fma52hi(res30, a[12], a[17]); // Sum(29) + res30 = fma52lo(res30, a[13], a[17]); // Sum(30) + res31 = fma52hi(res31, a[13], a[17]); // Sum(30) + res31 = fma52lo(res31, a[14], a[17]); // Sum(31) + res32 = fma52hi(res32, a[14], a[17]); // Sum(31) + res32 = fma52lo(res32, a[15], a[17]); // Sum(32) + res33 = fma52hi(res33, a[15], a[17]); // Sum(32) + res33 = fma52lo(res33, a[16], a[17]); // Sum(33) + res34 = fma52hi(res34, a[16], a[17]); // Sum(33) + res24 = fma52lo(res24, a[6], a[18]); // Sum(24) + res25 = fma52hi(res25, a[6], a[18]); // Sum(24) + res25 = fma52lo(res25, a[7], a[18]); // Sum(25) + res26 = fma52hi(res26, a[7], a[18]); // Sum(25) + res26 = fma52lo(res26, a[8], a[18]); // Sum(26) + res27 = fma52hi(res27, a[8], a[18]); // Sum(26) + res27 = fma52lo(res27, a[9], a[18]); // Sum(27) + res28 = fma52hi(res28, a[9], a[18]); // Sum(27) + res28 = fma52lo(res28, a[10], a[18]); // Sum(28) + res29 = fma52hi(res29, a[10], a[18]); // Sum(28) + res29 = fma52lo(res29, a[11], a[18]); // Sum(29) + res30 = fma52hi(res30, a[11], a[18]); // Sum(29) + res30 = fma52lo(res30, a[12], a[18]); // Sum(30) + res31 = fma52hi(res31, a[12], a[18]); // Sum(30) + res31 = fma52lo(res31, a[13], a[18]); // Sum(31) + res32 = fma52hi(res32, a[13], a[18]); // Sum(31) + res32 = fma52lo(res32, a[14], a[18]); // Sum(32) + res33 = fma52hi(res33, a[14], a[18]); // Sum(32) + res33 = fma52lo(res33, a[15], a[18]); // Sum(33) + res34 = fma52hi(res34, a[15], a[18]); // Sum(33) + res34 = fma52lo(res34, a[16], a[18]); // Sum(34) + res35 = fma52hi(res35, a[16], a[18]); // Sum(34) + res35 = fma52lo(res35, a[17], a[18]); // Sum(35) + res36 = fma52hi(res36, a[17], a[18]); // Sum(35) + res24 = fma52lo(res24, a[5], a[19]); // Sum(24) + res25 = fma52hi(res25, a[5], a[19]); // Sum(24) + res25 = fma52lo(res25, a[6], a[19]); // Sum(25) + res26 = fma52hi(res26, a[6], a[19]); // Sum(25) + res26 = fma52lo(res26, a[7], a[19]); // Sum(26) + res27 = fma52hi(res27, a[7], a[19]); // Sum(26) + res27 = fma52lo(res27, a[8], a[19]); // Sum(27) + res28 = fma52hi(res28, a[8], a[19]); // Sum(27) + res28 = fma52lo(res28, a[9], a[19]); // Sum(28) + res29 = fma52hi(res29, a[9], a[19]); // Sum(28) + res29 = fma52lo(res29, a[10], a[19]); // Sum(29) + res30 = fma52hi(res30, a[10], a[19]); // Sum(29) + res30 = fma52lo(res30, a[11], a[19]); // Sum(30) + res31 = fma52hi(res31, a[11], a[19]); // Sum(30) + res31 = fma52lo(res31, a[12], a[19]); // Sum(31) + res32 = fma52hi(res32, a[12], a[19]); // Sum(31) + res32 = fma52lo(res32, a[13], a[19]); // Sum(32) + res33 = fma52hi(res33, a[13], a[19]); // Sum(32) + res33 = fma52lo(res33, a[14], a[19]); // Sum(33) + res34 = fma52hi(res34, a[14], a[19]); // Sum(33) + res34 = fma52lo(res34, a[15], a[19]); // Sum(34) + res35 = fma52hi(res35, a[15], a[19]); // Sum(34) + res35 = fma52lo(res35, a[16], a[19]); // Sum(35) + res36 = fma52hi(res36, a[16], a[19]); // Sum(35) + res24 = add64(res24, res24); // Double(24) + res25 = add64(res25, res25); // Double(25) + res26 = add64(res26, res26); // Double(26) + res27 = add64(res27, res27); // Double(27) + res28 = add64(res28, res28); // Double(28) + res29 = add64(res29, res29); // Double(29) + res30 = add64(res30, res30); // Double(30) + res31 = add64(res31, res31); // Double(31) + res32 = add64(res32, res32); // Double(32) + res33 = add64(res33, res33); // Double(33) + res34 = add64(res34, res34); // Double(34) + res35 = add64(res35, res35); // Double(35) + res24 = fma52lo(res24, a[12], a[12]); // Add sqr(24) + res25 = fma52hi(res25, a[12], a[12]); // Add sqr(24) + res26 = fma52lo(res26, a[13], a[13]); // Add sqr(26) + res27 = fma52hi(res27, a[13], a[13]); // Add sqr(26) + res28 = fma52lo(res28, a[14], a[14]); // Add sqr(28) + res29 = fma52hi(res29, a[14], a[14]); // Add sqr(28) + res30 = fma52lo(res30, a[15], a[15]); // Add sqr(30) + res31 = fma52hi(res31, a[15], a[15]); // Add sqr(30) + res32 = fma52lo(res32, a[16], a[16]); // Add sqr(32) + res33 = fma52hi(res33, a[16], a[16]); // Add sqr(32) + res34 = fma52lo(res34, a[17], a[17]); // Add sqr(34) + res35 = fma52hi(res35, a[17], a[17]); // Add sqr(34) + res36 = fma52lo(res36, a[17], a[19]); // Sum(36) + res37 = fma52hi(res37, a[17], a[19]); // Sum(36) + res37 = fma52lo(res37, a[18], a[19]); // Sum(37) + res38 = fma52hi(res38, a[18], a[19]); // Sum(37) + res36 = add64(res36, res36); // Double(36) + res37 = add64(res37, res37); // Double(37) + res38 = add64(res38, res38); // Double(38) + res36 = fma52lo(res36, a[18], a[18]); // Add sqr(36) + res37 = fma52hi(res37, a[18], a[18]); // Add sqr(36) + res38 = fma52lo(res38, a[19], a[19]); // Add sqr(38) + res39 = fma52hi(res39, a[19], a[19]); // Add sqr(38) + + // Generate u_i + U64 u0 = mul52lo(res0, k); + ASM("jmp l0\nl0:\n"); + + // Create u0 + fma52lo_mem(res0, res0, u0, m, SIMD_BYTES * 0); + fma52hi_mem(res1, res1, u0, m, SIMD_BYTES * 0); + res1 = fma52lo(res1, u0, m[1]); + res2 = fma52hi(res2, u0, m[1]); + res1 = add64(res1, srli64(res0, DIGIT_SIZE)); + U64 u1 = mul52lo(res1, k); + fma52lo_mem(res2, res2, u0, m, SIMD_BYTES * 2); + fma52hi_mem(res3, res3, u0, m, SIMD_BYTES * 2); + res3 = fma52lo(res3, u0, m[3]); + res4 = fma52hi(res4, u0, m[3]); + fma52lo_mem(res4, res4, u0, m, SIMD_BYTES * 4); + fma52hi_mem(res5, res5, u0, m, SIMD_BYTES * 4); + res5 = fma52lo(res5, u0, m[5]); + res6 = fma52hi(res6, u0, m[5]); + fma52lo_mem(res6, res6, u0, m, SIMD_BYTES * 6); + fma52hi_mem(res7, res7, u0, m, SIMD_BYTES * 6); + res7 = fma52lo(res7, u0, m[7]); + res8 = fma52hi(res8, u0, m[7]); + fma52lo_mem(res8, res8, u0, m, SIMD_BYTES * 8); + fma52hi_mem(res9, res9, u0, m, SIMD_BYTES * 8); + res9 = fma52lo(res9, u0, m[9]); + res10 = fma52hi(res10, u0, m[9]); + fma52lo_mem(res10, res10, u0, m, SIMD_BYTES * 10); + fma52hi_mem(res11, res11, u0, m, SIMD_BYTES * 10); + res11 = fma52lo(res11, u0, m[11]); + res12 = fma52hi(res12, u0, m[11]); + fma52lo_mem(res12, res12, u0, m, SIMD_BYTES * 12); + fma52hi_mem(res13, res13, u0, m, SIMD_BYTES * 12); + res13 = fma52lo(res13, u0, m[13]); + res14 = fma52hi(res14, u0, m[13]); + fma52lo_mem(res14, res14, u0, m, SIMD_BYTES * 14); + fma52hi_mem(res15, res15, u0, m, SIMD_BYTES * 14); + res15 = fma52lo(res15, u0, m[15]); + res16 = fma52hi(res16, u0, m[15]); + fma52lo_mem(res16, res16, u0, m, SIMD_BYTES * 16); + fma52hi_mem(res17, res17, u0, m, SIMD_BYTES * 16); + res17 = fma52lo(res17, u0, m[17]); + res18 = fma52hi(res18, u0, m[17]); + fma52lo_mem(res18, res18, u0, m, SIMD_BYTES * 18); + fma52hi_mem(res19, res19, u0, m, SIMD_BYTES * 18); + res19 = fma52lo(res19, u0, m[19]); + res20 = fma52hi(res20, u0, m[19]); + + // Create u1 + fma52lo_mem(res1, res1, u1, m, SIMD_BYTES * 0); + fma52hi_mem(res2, res2, u1, m, SIMD_BYTES * 0); + res2 = fma52lo(res2, u1, m[1]); + res3 = fma52hi(res3, u1, m[1]); + res2 = add64(res2, srli64(res1, DIGIT_SIZE)); + U64 u2 = mul52lo(res2, k); + fma52lo_mem(res3, res3, u1, m, SIMD_BYTES * 2); + fma52hi_mem(res4, res4, u1, m, SIMD_BYTES * 2); + res4 = fma52lo(res4, u1, m[3]); + res5 = fma52hi(res5, u1, m[3]); + fma52lo_mem(res5, res5, u1, m, SIMD_BYTES * 4); + fma52hi_mem(res6, res6, u1, m, SIMD_BYTES * 4); + res6 = fma52lo(res6, u1, m[5]); + res7 = fma52hi(res7, u1, m[5]); + fma52lo_mem(res7, res7, u1, m, SIMD_BYTES * 6); + fma52hi_mem(res8, res8, u1, m, SIMD_BYTES * 6); + res8 = fma52lo(res8, u1, m[7]); + res9 = fma52hi(res9, u1, m[7]); + fma52lo_mem(res9, res9, u1, m, SIMD_BYTES * 8); + fma52hi_mem(res10, res10, u1, m, SIMD_BYTES * 8); + res10 = fma52lo(res10, u1, m[9]); + res11 = fma52hi(res11, u1, m[9]); + fma52lo_mem(res11, res11, u1, m, SIMD_BYTES * 10); + fma52hi_mem(res12, res12, u1, m, SIMD_BYTES * 10); + res12 = fma52lo(res12, u1, m[11]); + res13 = fma52hi(res13, u1, m[11]); + fma52lo_mem(res13, res13, u1, m, SIMD_BYTES * 12); + fma52hi_mem(res14, res14, u1, m, SIMD_BYTES * 12); + res14 = fma52lo(res14, u1, m[13]); + res15 = fma52hi(res15, u1, m[13]); + fma52lo_mem(res15, res15, u1, m, SIMD_BYTES * 14); + fma52hi_mem(res16, res16, u1, m, SIMD_BYTES * 14); + res16 = fma52lo(res16, u1, m[15]); + res17 = fma52hi(res17, u1, m[15]); + fma52lo_mem(res17, res17, u1, m, SIMD_BYTES * 16); + fma52hi_mem(res18, res18, u1, m, SIMD_BYTES * 16); + res18 = fma52lo(res18, u1, m[17]); + res19 = fma52hi(res19, u1, m[17]); + fma52lo_mem(res19, res19, u1, m, SIMD_BYTES * 18); + fma52hi_mem(res20, res20, u1, m, SIMD_BYTES * 18); + res20 = fma52lo(res20, u1, m[19]); + res21 = fma52hi(res21, u1, m[19]); + ASM("jmp l2\nl2:\n"); + + // Create u2 + fma52lo_mem(res2, res2, u2, m, SIMD_BYTES * 0); + fma52hi_mem(res3, res3, u2, m, SIMD_BYTES * 0); + res3 = fma52lo(res3, u2, m[1]); + res4 = fma52hi(res4, u2, m[1]); + res3 = add64(res3, srli64(res2, DIGIT_SIZE)); + U64 u3 = mul52lo(res3, k); + fma52lo_mem(res4, res4, u2, m, SIMD_BYTES * 2); + fma52hi_mem(res5, res5, u2, m, SIMD_BYTES * 2); + res5 = fma52lo(res5, u2, m[3]); + res6 = fma52hi(res6, u2, m[3]); + fma52lo_mem(res6, res6, u2, m, SIMD_BYTES * 4); + fma52hi_mem(res7, res7, u2, m, SIMD_BYTES * 4); + res7 = fma52lo(res7, u2, m[5]); + res8 = fma52hi(res8, u2, m[5]); + fma52lo_mem(res8, res8, u2, m, SIMD_BYTES * 6); + fma52hi_mem(res9, res9, u2, m, SIMD_BYTES * 6); + res9 = fma52lo(res9, u2, m[7]); + res10 = fma52hi(res10, u2, m[7]); + fma52lo_mem(res10, res10, u2, m, SIMD_BYTES * 8); + fma52hi_mem(res11, res11, u2, m, SIMD_BYTES * 8); + res11 = fma52lo(res11, u2, m[9]); + res12 = fma52hi(res12, u2, m[9]); + fma52lo_mem(res12, res12, u2, m, SIMD_BYTES * 10); + fma52hi_mem(res13, res13, u2, m, SIMD_BYTES * 10); + res13 = fma52lo(res13, u2, m[11]); + res14 = fma52hi(res14, u2, m[11]); + fma52lo_mem(res14, res14, u2, m, SIMD_BYTES * 12); + fma52hi_mem(res15, res15, u2, m, SIMD_BYTES * 12); + res15 = fma52lo(res15, u2, m[13]); + res16 = fma52hi(res16, u2, m[13]); + fma52lo_mem(res16, res16, u2, m, SIMD_BYTES * 14); + fma52hi_mem(res17, res17, u2, m, SIMD_BYTES * 14); + res17 = fma52lo(res17, u2, m[15]); + res18 = fma52hi(res18, u2, m[15]); + fma52lo_mem(res18, res18, u2, m, SIMD_BYTES * 16); + fma52hi_mem(res19, res19, u2, m, SIMD_BYTES * 16); + res19 = fma52lo(res19, u2, m[17]); + res20 = fma52hi(res20, u2, m[17]); + fma52lo_mem(res20, res20, u2, m, SIMD_BYTES * 18); + fma52hi_mem(res21, res21, u2, m, SIMD_BYTES * 18); + res21 = fma52lo(res21, u2, m[19]); + res22 = fma52hi(res22, u2, m[19]); + + // Create u3 + fma52lo_mem(res3, res3, u3, m, SIMD_BYTES * 0); + fma52hi_mem(res4, res4, u3, m, SIMD_BYTES * 0); + res4 = fma52lo(res4, u3, m[1]); + res5 = fma52hi(res5, u3, m[1]); + res4 = add64(res4, srli64(res3, DIGIT_SIZE)); + U64 u4 = mul52lo(res4, k); + fma52lo_mem(res5, res5, u3, m, SIMD_BYTES * 2); + fma52hi_mem(res6, res6, u3, m, SIMD_BYTES * 2); + res6 = fma52lo(res6, u3, m[3]); + res7 = fma52hi(res7, u3, m[3]); + fma52lo_mem(res7, res7, u3, m, SIMD_BYTES * 4); + fma52hi_mem(res8, res8, u3, m, SIMD_BYTES * 4); + res8 = fma52lo(res8, u3, m[5]); + res9 = fma52hi(res9, u3, m[5]); + fma52lo_mem(res9, res9, u3, m, SIMD_BYTES * 6); + fma52hi_mem(res10, res10, u3, m, SIMD_BYTES * 6); + res10 = fma52lo(res10, u3, m[7]); + res11 = fma52hi(res11, u3, m[7]); + fma52lo_mem(res11, res11, u3, m, SIMD_BYTES * 8); + fma52hi_mem(res12, res12, u3, m, SIMD_BYTES * 8); + res12 = fma52lo(res12, u3, m[9]); + res13 = fma52hi(res13, u3, m[9]); + fma52lo_mem(res13, res13, u3, m, SIMD_BYTES * 10); + fma52hi_mem(res14, res14, u3, m, SIMD_BYTES * 10); + res14 = fma52lo(res14, u3, m[11]); + res15 = fma52hi(res15, u3, m[11]); + fma52lo_mem(res15, res15, u3, m, SIMD_BYTES * 12); + fma52hi_mem(res16, res16, u3, m, SIMD_BYTES * 12); + res16 = fma52lo(res16, u3, m[13]); + res17 = fma52hi(res17, u3, m[13]); + fma52lo_mem(res17, res17, u3, m, SIMD_BYTES * 14); + fma52hi_mem(res18, res18, u3, m, SIMD_BYTES * 14); + res18 = fma52lo(res18, u3, m[15]); + res19 = fma52hi(res19, u3, m[15]); + fma52lo_mem(res19, res19, u3, m, SIMD_BYTES * 16); + fma52hi_mem(res20, res20, u3, m, SIMD_BYTES * 16); + res20 = fma52lo(res20, u3, m[17]); + res21 = fma52hi(res21, u3, m[17]); + fma52lo_mem(res21, res21, u3, m, SIMD_BYTES * 18); + fma52hi_mem(res22, res22, u3, m, SIMD_BYTES * 18); + res22 = fma52lo(res22, u3, m[19]); + res23 = fma52hi(res23, u3, m[19]); + ASM("jmp l4\nl4:\n"); + + // Create u4 + fma52lo_mem(res4, res4, u4, m, SIMD_BYTES * 0); + fma52hi_mem(res5, res5, u4, m, SIMD_BYTES * 0); + res5 = fma52lo(res5, u4, m[1]); + res6 = fma52hi(res6, u4, m[1]); + res5 = add64(res5, srli64(res4, DIGIT_SIZE)); + U64 u5 = mul52lo(res5, k); + fma52lo_mem(res6, res6, u4, m, SIMD_BYTES * 2); + fma52hi_mem(res7, res7, u4, m, SIMD_BYTES * 2); + res7 = fma52lo(res7, u4, m[3]); + res8 = fma52hi(res8, u4, m[3]); + fma52lo_mem(res8, res8, u4, m, SIMD_BYTES * 4); + fma52hi_mem(res9, res9, u4, m, SIMD_BYTES * 4); + res9 = fma52lo(res9, u4, m[5]); + res10 = fma52hi(res10, u4, m[5]); + fma52lo_mem(res10, res10, u4, m, SIMD_BYTES * 6); + fma52hi_mem(res11, res11, u4, m, SIMD_BYTES * 6); + res11 = fma52lo(res11, u4, m[7]); + res12 = fma52hi(res12, u4, m[7]); + fma52lo_mem(res12, res12, u4, m, SIMD_BYTES * 8); + fma52hi_mem(res13, res13, u4, m, SIMD_BYTES * 8); + res13 = fma52lo(res13, u4, m[9]); + res14 = fma52hi(res14, u4, m[9]); + fma52lo_mem(res14, res14, u4, m, SIMD_BYTES * 10); + fma52hi_mem(res15, res15, u4, m, SIMD_BYTES * 10); + res15 = fma52lo(res15, u4, m[11]); + res16 = fma52hi(res16, u4, m[11]); + fma52lo_mem(res16, res16, u4, m, SIMD_BYTES * 12); + fma52hi_mem(res17, res17, u4, m, SIMD_BYTES * 12); + res17 = fma52lo(res17, u4, m[13]); + res18 = fma52hi(res18, u4, m[13]); + fma52lo_mem(res18, res18, u4, m, SIMD_BYTES * 14); + fma52hi_mem(res19, res19, u4, m, SIMD_BYTES * 14); + res19 = fma52lo(res19, u4, m[15]); + res20 = fma52hi(res20, u4, m[15]); + fma52lo_mem(res20, res20, u4, m, SIMD_BYTES * 16); + fma52hi_mem(res21, res21, u4, m, SIMD_BYTES * 16); + res21 = fma52lo(res21, u4, m[17]); + res22 = fma52hi(res22, u4, m[17]); + fma52lo_mem(res22, res22, u4, m, SIMD_BYTES * 18); + fma52hi_mem(res23, res23, u4, m, SIMD_BYTES * 18); + res23 = fma52lo(res23, u4, m[19]); + res24 = fma52hi(res24, u4, m[19]); + + // Create u5 + fma52lo_mem(res5, res5, u5, m, SIMD_BYTES * 0); + fma52hi_mem(res6, res6, u5, m, SIMD_BYTES * 0); + res6 = fma52lo(res6, u5, m[1]); + res7 = fma52hi(res7, u5, m[1]); + res6 = add64(res6, srli64(res5, DIGIT_SIZE)); + U64 u6 = mul52lo(res6, k); + fma52lo_mem(res7, res7, u5, m, SIMD_BYTES * 2); + fma52hi_mem(res8, res8, u5, m, SIMD_BYTES * 2); + res8 = fma52lo(res8, u5, m[3]); + res9 = fma52hi(res9, u5, m[3]); + fma52lo_mem(res9, res9, u5, m, SIMD_BYTES * 4); + fma52hi_mem(res10, res10, u5, m, SIMD_BYTES * 4); + res10 = fma52lo(res10, u5, m[5]); + res11 = fma52hi(res11, u5, m[5]); + fma52lo_mem(res11, res11, u5, m, SIMD_BYTES * 6); + fma52hi_mem(res12, res12, u5, m, SIMD_BYTES * 6); + res12 = fma52lo(res12, u5, m[7]); + res13 = fma52hi(res13, u5, m[7]); + fma52lo_mem(res13, res13, u5, m, SIMD_BYTES * 8); + fma52hi_mem(res14, res14, u5, m, SIMD_BYTES * 8); + res14 = fma52lo(res14, u5, m[9]); + res15 = fma52hi(res15, u5, m[9]); + fma52lo_mem(res15, res15, u5, m, SIMD_BYTES * 10); + fma52hi_mem(res16, res16, u5, m, SIMD_BYTES * 10); + res16 = fma52lo(res16, u5, m[11]); + res17 = fma52hi(res17, u5, m[11]); + fma52lo_mem(res17, res17, u5, m, SIMD_BYTES * 12); + fma52hi_mem(res18, res18, u5, m, SIMD_BYTES * 12); + res18 = fma52lo(res18, u5, m[13]); + res19 = fma52hi(res19, u5, m[13]); + fma52lo_mem(res19, res19, u5, m, SIMD_BYTES * 14); + fma52hi_mem(res20, res20, u5, m, SIMD_BYTES * 14); + res20 = fma52lo(res20, u5, m[15]); + res21 = fma52hi(res21, u5, m[15]); + fma52lo_mem(res21, res21, u5, m, SIMD_BYTES * 16); + fma52hi_mem(res22, res22, u5, m, SIMD_BYTES * 16); + res22 = fma52lo(res22, u5, m[17]); + res23 = fma52hi(res23, u5, m[17]); + fma52lo_mem(res23, res23, u5, m, SIMD_BYTES * 18); + fma52hi_mem(res24, res24, u5, m, SIMD_BYTES * 18); + res24 = fma52lo(res24, u5, m[19]); + res25 = fma52hi(res25, u5, m[19]); + ASM("jmp l6\nl6:\n"); + + // Create u6 + fma52lo_mem(res6, res6, u6, m, SIMD_BYTES * 0); + fma52hi_mem(res7, res7, u6, m, SIMD_BYTES * 0); + res7 = fma52lo(res7, u6, m[1]); + res8 = fma52hi(res8, u6, m[1]); + res7 = add64(res7, srli64(res6, DIGIT_SIZE)); + U64 u7 = mul52lo(res7, k); + fma52lo_mem(res8, res8, u6, m, SIMD_BYTES * 2); + fma52hi_mem(res9, res9, u6, m, SIMD_BYTES * 2); + res9 = fma52lo(res9, u6, m[3]); + res10 = fma52hi(res10, u6, m[3]); + fma52lo_mem(res10, res10, u6, m, SIMD_BYTES * 4); + fma52hi_mem(res11, res11, u6, m, SIMD_BYTES * 4); + res11 = fma52lo(res11, u6, m[5]); + res12 = fma52hi(res12, u6, m[5]); + fma52lo_mem(res12, res12, u6, m, SIMD_BYTES * 6); + fma52hi_mem(res13, res13, u6, m, SIMD_BYTES * 6); + res13 = fma52lo(res13, u6, m[7]); + res14 = fma52hi(res14, u6, m[7]); + fma52lo_mem(res14, res14, u6, m, SIMD_BYTES * 8); + fma52hi_mem(res15, res15, u6, m, SIMD_BYTES * 8); + res15 = fma52lo(res15, u6, m[9]); + res16 = fma52hi(res16, u6, m[9]); + fma52lo_mem(res16, res16, u6, m, SIMD_BYTES * 10); + fma52hi_mem(res17, res17, u6, m, SIMD_BYTES * 10); + res17 = fma52lo(res17, u6, m[11]); + res18 = fma52hi(res18, u6, m[11]); + fma52lo_mem(res18, res18, u6, m, SIMD_BYTES * 12); + fma52hi_mem(res19, res19, u6, m, SIMD_BYTES * 12); + res19 = fma52lo(res19, u6, m[13]); + res20 = fma52hi(res20, u6, m[13]); + fma52lo_mem(res20, res20, u6, m, SIMD_BYTES * 14); + fma52hi_mem(res21, res21, u6, m, SIMD_BYTES * 14); + res21 = fma52lo(res21, u6, m[15]); + res22 = fma52hi(res22, u6, m[15]); + fma52lo_mem(res22, res22, u6, m, SIMD_BYTES * 16); + fma52hi_mem(res23, res23, u6, m, SIMD_BYTES * 16); + res23 = fma52lo(res23, u6, m[17]); + res24 = fma52hi(res24, u6, m[17]); + fma52lo_mem(res24, res24, u6, m, SIMD_BYTES * 18); + fma52hi_mem(res25, res25, u6, m, SIMD_BYTES * 18); + res25 = fma52lo(res25, u6, m[19]); + res26 = fma52hi(res26, u6, m[19]); + + // Create u7 + fma52lo_mem(res7, res7, u7, m, SIMD_BYTES * 0); + fma52hi_mem(res8, res8, u7, m, SIMD_BYTES * 0); + res8 = fma52lo(res8, u7, m[1]); + res9 = fma52hi(res9, u7, m[1]); + res8 = add64(res8, srli64(res7, DIGIT_SIZE)); + U64 u8 = mul52lo(res8, k); + fma52lo_mem(res9, res9, u7, m, SIMD_BYTES * 2); + fma52hi_mem(res10, res10, u7, m, SIMD_BYTES * 2); + res10 = fma52lo(res10, u7, m[3]); + res11 = fma52hi(res11, u7, m[3]); + fma52lo_mem(res11, res11, u7, m, SIMD_BYTES * 4); + fma52hi_mem(res12, res12, u7, m, SIMD_BYTES * 4); + res12 = fma52lo(res12, u7, m[5]); + res13 = fma52hi(res13, u7, m[5]); + fma52lo_mem(res13, res13, u7, m, SIMD_BYTES * 6); + fma52hi_mem(res14, res14, u7, m, SIMD_BYTES * 6); + res14 = fma52lo(res14, u7, m[7]); + res15 = fma52hi(res15, u7, m[7]); + fma52lo_mem(res15, res15, u7, m, SIMD_BYTES * 8); + fma52hi_mem(res16, res16, u7, m, SIMD_BYTES * 8); + res16 = fma52lo(res16, u7, m[9]); + res17 = fma52hi(res17, u7, m[9]); + fma52lo_mem(res17, res17, u7, m, SIMD_BYTES * 10); + fma52hi_mem(res18, res18, u7, m, SIMD_BYTES * 10); + res18 = fma52lo(res18, u7, m[11]); + res19 = fma52hi(res19, u7, m[11]); + fma52lo_mem(res19, res19, u7, m, SIMD_BYTES * 12); + fma52hi_mem(res20, res20, u7, m, SIMD_BYTES * 12); + res20 = fma52lo(res20, u7, m[13]); + res21 = fma52hi(res21, u7, m[13]); + fma52lo_mem(res21, res21, u7, m, SIMD_BYTES * 14); + fma52hi_mem(res22, res22, u7, m, SIMD_BYTES * 14); + res22 = fma52lo(res22, u7, m[15]); + res23 = fma52hi(res23, u7, m[15]); + fma52lo_mem(res23, res23, u7, m, SIMD_BYTES * 16); + fma52hi_mem(res24, res24, u7, m, SIMD_BYTES * 16); + res24 = fma52lo(res24, u7, m[17]); + res25 = fma52hi(res25, u7, m[17]); + fma52lo_mem(res25, res25, u7, m, SIMD_BYTES * 18); + fma52hi_mem(res26, res26, u7, m, SIMD_BYTES * 18); + res26 = fma52lo(res26, u7, m[19]); + res27 = fma52hi(res27, u7, m[19]); + ASM("jmp l8\nl8:\n"); + + // Create u8 + fma52lo_mem(res8, res8, u8, m, SIMD_BYTES * 0); + fma52hi_mem(res9, res9, u8, m, SIMD_BYTES * 0); + res9 = fma52lo(res9, u8, m[1]); + res10 = fma52hi(res10, u8, m[1]); + res9 = add64(res9, srli64(res8, DIGIT_SIZE)); + U64 u9 = mul52lo(res9, k); + fma52lo_mem(res10, res10, u8, m, SIMD_BYTES * 2); + fma52hi_mem(res11, res11, u8, m, SIMD_BYTES * 2); + res11 = fma52lo(res11, u8, m[3]); + res12 = fma52hi(res12, u8, m[3]); + fma52lo_mem(res12, res12, u8, m, SIMD_BYTES * 4); + fma52hi_mem(res13, res13, u8, m, SIMD_BYTES * 4); + res13 = fma52lo(res13, u8, m[5]); + res14 = fma52hi(res14, u8, m[5]); + fma52lo_mem(res14, res14, u8, m, SIMD_BYTES * 6); + fma52hi_mem(res15, res15, u8, m, SIMD_BYTES * 6); + res15 = fma52lo(res15, u8, m[7]); + res16 = fma52hi(res16, u8, m[7]); + fma52lo_mem(res16, res16, u8, m, SIMD_BYTES * 8); + fma52hi_mem(res17, res17, u8, m, SIMD_BYTES * 8); + res17 = fma52lo(res17, u8, m[9]); + res18 = fma52hi(res18, u8, m[9]); + fma52lo_mem(res18, res18, u8, m, SIMD_BYTES * 10); + fma52hi_mem(res19, res19, u8, m, SIMD_BYTES * 10); + res19 = fma52lo(res19, u8, m[11]); + res20 = fma52hi(res20, u8, m[11]); + fma52lo_mem(res20, res20, u8, m, SIMD_BYTES * 12); + fma52hi_mem(res21, res21, u8, m, SIMD_BYTES * 12); + res21 = fma52lo(res21, u8, m[13]); + res22 = fma52hi(res22, u8, m[13]); + fma52lo_mem(res22, res22, u8, m, SIMD_BYTES * 14); + fma52hi_mem(res23, res23, u8, m, SIMD_BYTES * 14); + res23 = fma52lo(res23, u8, m[15]); + res24 = fma52hi(res24, u8, m[15]); + fma52lo_mem(res24, res24, u8, m, SIMD_BYTES * 16); + fma52hi_mem(res25, res25, u8, m, SIMD_BYTES * 16); + res25 = fma52lo(res25, u8, m[17]); + res26 = fma52hi(res26, u8, m[17]); + fma52lo_mem(res26, res26, u8, m, SIMD_BYTES * 18); + fma52hi_mem(res27, res27, u8, m, SIMD_BYTES * 18); + res27 = fma52lo(res27, u8, m[19]); + res28 = fma52hi(res28, u8, m[19]); + + // Create u9 + fma52lo_mem(res9, res9, u9, m, SIMD_BYTES * 0); + fma52hi_mem(res10, res10, u9, m, SIMD_BYTES * 0); + res10 = fma52lo(res10, u9, m[1]); + res11 = fma52hi(res11, u9, m[1]); + res10 = add64(res10, srli64(res9, DIGIT_SIZE)); + U64 u10 = mul52lo(res10, k); + fma52lo_mem(res11, res11, u9, m, SIMD_BYTES * 2); + fma52hi_mem(res12, res12, u9, m, SIMD_BYTES * 2); + res12 = fma52lo(res12, u9, m[3]); + res13 = fma52hi(res13, u9, m[3]); + fma52lo_mem(res13, res13, u9, m, SIMD_BYTES * 4); + fma52hi_mem(res14, res14, u9, m, SIMD_BYTES * 4); + res14 = fma52lo(res14, u9, m[5]); + res15 = fma52hi(res15, u9, m[5]); + fma52lo_mem(res15, res15, u9, m, SIMD_BYTES * 6); + fma52hi_mem(res16, res16, u9, m, SIMD_BYTES * 6); + res16 = fma52lo(res16, u9, m[7]); + res17 = fma52hi(res17, u9, m[7]); + fma52lo_mem(res17, res17, u9, m, SIMD_BYTES * 8); + fma52hi_mem(res18, res18, u9, m, SIMD_BYTES * 8); + res18 = fma52lo(res18, u9, m[9]); + res19 = fma52hi(res19, u9, m[9]); + fma52lo_mem(res19, res19, u9, m, SIMD_BYTES * 10); + fma52hi_mem(res20, res20, u9, m, SIMD_BYTES * 10); + res20 = fma52lo(res20, u9, m[11]); + res21 = fma52hi(res21, u9, m[11]); + fma52lo_mem(res21, res21, u9, m, SIMD_BYTES * 12); + fma52hi_mem(res22, res22, u9, m, SIMD_BYTES * 12); + res22 = fma52lo(res22, u9, m[13]); + res23 = fma52hi(res23, u9, m[13]); + fma52lo_mem(res23, res23, u9, m, SIMD_BYTES * 14); + fma52hi_mem(res24, res24, u9, m, SIMD_BYTES * 14); + res24 = fma52lo(res24, u9, m[15]); + res25 = fma52hi(res25, u9, m[15]); + fma52lo_mem(res25, res25, u9, m, SIMD_BYTES * 16); + fma52hi_mem(res26, res26, u9, m, SIMD_BYTES * 16); + res26 = fma52lo(res26, u9, m[17]); + res27 = fma52hi(res27, u9, m[17]); + fma52lo_mem(res27, res27, u9, m, SIMD_BYTES * 18); + fma52hi_mem(res28, res28, u9, m, SIMD_BYTES * 18); + res28 = fma52lo(res28, u9, m[19]); + res29 = fma52hi(res29, u9, m[19]); + ASM("jmp l10\nl10:\n"); + + // Create u10 + fma52lo_mem(res10, res10, u10, m, SIMD_BYTES * 0); + fma52hi_mem(res11, res11, u10, m, SIMD_BYTES * 0); + res11 = fma52lo(res11, u10, m[1]); + res12 = fma52hi(res12, u10, m[1]); + res11 = add64(res11, srli64(res10, DIGIT_SIZE)); + U64 u11 = mul52lo(res11, k); + fma52lo_mem(res12, res12, u10, m, SIMD_BYTES * 2); + fma52hi_mem(res13, res13, u10, m, SIMD_BYTES * 2); + res13 = fma52lo(res13, u10, m[3]); + res14 = fma52hi(res14, u10, m[3]); + fma52lo_mem(res14, res14, u10, m, SIMD_BYTES * 4); + fma52hi_mem(res15, res15, u10, m, SIMD_BYTES * 4); + res15 = fma52lo(res15, u10, m[5]); + res16 = fma52hi(res16, u10, m[5]); + fma52lo_mem(res16, res16, u10, m, SIMD_BYTES * 6); + fma52hi_mem(res17, res17, u10, m, SIMD_BYTES * 6); + res17 = fma52lo(res17, u10, m[7]); + res18 = fma52hi(res18, u10, m[7]); + fma52lo_mem(res18, res18, u10, m, SIMD_BYTES * 8); + fma52hi_mem(res19, res19, u10, m, SIMD_BYTES * 8); + res19 = fma52lo(res19, u10, m[9]); + res20 = fma52hi(res20, u10, m[9]); + fma52lo_mem(res20, res20, u10, m, SIMD_BYTES * 10); + fma52hi_mem(res21, res21, u10, m, SIMD_BYTES * 10); + res21 = fma52lo(res21, u10, m[11]); + res22 = fma52hi(res22, u10, m[11]); + fma52lo_mem(res22, res22, u10, m, SIMD_BYTES * 12); + fma52hi_mem(res23, res23, u10, m, SIMD_BYTES * 12); + res23 = fma52lo(res23, u10, m[13]); + res24 = fma52hi(res24, u10, m[13]); + fma52lo_mem(res24, res24, u10, m, SIMD_BYTES * 14); + fma52hi_mem(res25, res25, u10, m, SIMD_BYTES * 14); + res25 = fma52lo(res25, u10, m[15]); + res26 = fma52hi(res26, u10, m[15]); + fma52lo_mem(res26, res26, u10, m, SIMD_BYTES * 16); + fma52hi_mem(res27, res27, u10, m, SIMD_BYTES * 16); + res27 = fma52lo(res27, u10, m[17]); + res28 = fma52hi(res28, u10, m[17]); + fma52lo_mem(res28, res28, u10, m, SIMD_BYTES * 18); + fma52hi_mem(res29, res29, u10, m, SIMD_BYTES * 18); + res29 = fma52lo(res29, u10, m[19]); + res30 = fma52hi(res30, u10, m[19]); + + // Create u11 + fma52lo_mem(res11, res11, u11, m, SIMD_BYTES * 0); + fma52hi_mem(res12, res12, u11, m, SIMD_BYTES * 0); + res12 = fma52lo(res12, u11, m[1]); + res13 = fma52hi(res13, u11, m[1]); + res12 = add64(res12, srli64(res11, DIGIT_SIZE)); + U64 u12 = mul52lo(res12, k); + fma52lo_mem(res13, res13, u11, m, SIMD_BYTES * 2); + fma52hi_mem(res14, res14, u11, m, SIMD_BYTES * 2); + res14 = fma52lo(res14, u11, m[3]); + res15 = fma52hi(res15, u11, m[3]); + fma52lo_mem(res15, res15, u11, m, SIMD_BYTES * 4); + fma52hi_mem(res16, res16, u11, m, SIMD_BYTES * 4); + res16 = fma52lo(res16, u11, m[5]); + res17 = fma52hi(res17, u11, m[5]); + fma52lo_mem(res17, res17, u11, m, SIMD_BYTES * 6); + fma52hi_mem(res18, res18, u11, m, SIMD_BYTES * 6); + res18 = fma52lo(res18, u11, m[7]); + res19 = fma52hi(res19, u11, m[7]); + fma52lo_mem(res19, res19, u11, m, SIMD_BYTES * 8); + fma52hi_mem(res20, res20, u11, m, SIMD_BYTES * 8); + res20 = fma52lo(res20, u11, m[9]); + res21 = fma52hi(res21, u11, m[9]); + fma52lo_mem(res21, res21, u11, m, SIMD_BYTES * 10); + fma52hi_mem(res22, res22, u11, m, SIMD_BYTES * 10); + res22 = fma52lo(res22, u11, m[11]); + res23 = fma52hi(res23, u11, m[11]); + fma52lo_mem(res23, res23, u11, m, SIMD_BYTES * 12); + fma52hi_mem(res24, res24, u11, m, SIMD_BYTES * 12); + res24 = fma52lo(res24, u11, m[13]); + res25 = fma52hi(res25, u11, m[13]); + fma52lo_mem(res25, res25, u11, m, SIMD_BYTES * 14); + fma52hi_mem(res26, res26, u11, m, SIMD_BYTES * 14); + res26 = fma52lo(res26, u11, m[15]); + res27 = fma52hi(res27, u11, m[15]); + fma52lo_mem(res27, res27, u11, m, SIMD_BYTES * 16); + fma52hi_mem(res28, res28, u11, m, SIMD_BYTES * 16); + res28 = fma52lo(res28, u11, m[17]); + res29 = fma52hi(res29, u11, m[17]); + fma52lo_mem(res29, res29, u11, m, SIMD_BYTES * 18); + fma52hi_mem(res30, res30, u11, m, SIMD_BYTES * 18); + res30 = fma52lo(res30, u11, m[19]); + res31 = fma52hi(res31, u11, m[19]); + ASM("jmp l12\nl12:\n"); + + // Create u12 + fma52lo_mem(res12, res12, u12, m, SIMD_BYTES * 0); + fma52hi_mem(res13, res13, u12, m, SIMD_BYTES * 0); + res13 = fma52lo(res13, u12, m[1]); + res14 = fma52hi(res14, u12, m[1]); + res13 = add64(res13, srli64(res12, DIGIT_SIZE)); + U64 u13 = mul52lo(res13, k); + fma52lo_mem(res14, res14, u12, m, SIMD_BYTES * 2); + fma52hi_mem(res15, res15, u12, m, SIMD_BYTES * 2); + res15 = fma52lo(res15, u12, m[3]); + res16 = fma52hi(res16, u12, m[3]); + fma52lo_mem(res16, res16, u12, m, SIMD_BYTES * 4); + fma52hi_mem(res17, res17, u12, m, SIMD_BYTES * 4); + res17 = fma52lo(res17, u12, m[5]); + res18 = fma52hi(res18, u12, m[5]); + fma52lo_mem(res18, res18, u12, m, SIMD_BYTES * 6); + fma52hi_mem(res19, res19, u12, m, SIMD_BYTES * 6); + res19 = fma52lo(res19, u12, m[7]); + res20 = fma52hi(res20, u12, m[7]); + fma52lo_mem(res20, res20, u12, m, SIMD_BYTES * 8); + fma52hi_mem(res21, res21, u12, m, SIMD_BYTES * 8); + res21 = fma52lo(res21, u12, m[9]); + res22 = fma52hi(res22, u12, m[9]); + fma52lo_mem(res22, res22, u12, m, SIMD_BYTES * 10); + fma52hi_mem(res23, res23, u12, m, SIMD_BYTES * 10); + res23 = fma52lo(res23, u12, m[11]); + res24 = fma52hi(res24, u12, m[11]); + fma52lo_mem(res24, res24, u12, m, SIMD_BYTES * 12); + fma52hi_mem(res25, res25, u12, m, SIMD_BYTES * 12); + res25 = fma52lo(res25, u12, m[13]); + res26 = fma52hi(res26, u12, m[13]); + fma52lo_mem(res26, res26, u12, m, SIMD_BYTES * 14); + fma52hi_mem(res27, res27, u12, m, SIMD_BYTES * 14); + res27 = fma52lo(res27, u12, m[15]); + res28 = fma52hi(res28, u12, m[15]); + fma52lo_mem(res28, res28, u12, m, SIMD_BYTES * 16); + fma52hi_mem(res29, res29, u12, m, SIMD_BYTES * 16); + res29 = fma52lo(res29, u12, m[17]); + res30 = fma52hi(res30, u12, m[17]); + fma52lo_mem(res30, res30, u12, m, SIMD_BYTES * 18); + fma52hi_mem(res31, res31, u12, m, SIMD_BYTES * 18); + res31 = fma52lo(res31, u12, m[19]); + res32 = fma52hi(res32, u12, m[19]); + + // Create u13 + fma52lo_mem(res13, res13, u13, m, SIMD_BYTES * 0); + fma52hi_mem(res14, res14, u13, m, SIMD_BYTES * 0); + res14 = fma52lo(res14, u13, m[1]); + res15 = fma52hi(res15, u13, m[1]); + res14 = add64(res14, srli64(res13, DIGIT_SIZE)); + U64 u14 = mul52lo(res14, k); + fma52lo_mem(res15, res15, u13, m, SIMD_BYTES * 2); + fma52hi_mem(res16, res16, u13, m, SIMD_BYTES * 2); + res16 = fma52lo(res16, u13, m[3]); + res17 = fma52hi(res17, u13, m[3]); + fma52lo_mem(res17, res17, u13, m, SIMD_BYTES * 4); + fma52hi_mem(res18, res18, u13, m, SIMD_BYTES * 4); + res18 = fma52lo(res18, u13, m[5]); + res19 = fma52hi(res19, u13, m[5]); + fma52lo_mem(res19, res19, u13, m, SIMD_BYTES * 6); + fma52hi_mem(res20, res20, u13, m, SIMD_BYTES * 6); + res20 = fma52lo(res20, u13, m[7]); + res21 = fma52hi(res21, u13, m[7]); + fma52lo_mem(res21, res21, u13, m, SIMD_BYTES * 8); + fma52hi_mem(res22, res22, u13, m, SIMD_BYTES * 8); + res22 = fma52lo(res22, u13, m[9]); + res23 = fma52hi(res23, u13, m[9]); + fma52lo_mem(res23, res23, u13, m, SIMD_BYTES * 10); + fma52hi_mem(res24, res24, u13, m, SIMD_BYTES * 10); + res24 = fma52lo(res24, u13, m[11]); + res25 = fma52hi(res25, u13, m[11]); + fma52lo_mem(res25, res25, u13, m, SIMD_BYTES * 12); + fma52hi_mem(res26, res26, u13, m, SIMD_BYTES * 12); + res26 = fma52lo(res26, u13, m[13]); + res27 = fma52hi(res27, u13, m[13]); + fma52lo_mem(res27, res27, u13, m, SIMD_BYTES * 14); + fma52hi_mem(res28, res28, u13, m, SIMD_BYTES * 14); + res28 = fma52lo(res28, u13, m[15]); + res29 = fma52hi(res29, u13, m[15]); + fma52lo_mem(res29, res29, u13, m, SIMD_BYTES * 16); + fma52hi_mem(res30, res30, u13, m, SIMD_BYTES * 16); + res30 = fma52lo(res30, u13, m[17]); + res31 = fma52hi(res31, u13, m[17]); + fma52lo_mem(res31, res31, u13, m, SIMD_BYTES * 18); + fma52hi_mem(res32, res32, u13, m, SIMD_BYTES * 18); + res32 = fma52lo(res32, u13, m[19]); + res33 = fma52hi(res33, u13, m[19]); + ASM("jmp l14\nl14:\n"); + + // Create u14 + fma52lo_mem(res14, res14, u14, m, SIMD_BYTES * 0); + fma52hi_mem(res15, res15, u14, m, SIMD_BYTES * 0); + res15 = fma52lo(res15, u14, m[1]); + res16 = fma52hi(res16, u14, m[1]); + res15 = add64(res15, srli64(res14, DIGIT_SIZE)); + U64 u15 = mul52lo(res15, k); + fma52lo_mem(res16, res16, u14, m, SIMD_BYTES * 2); + fma52hi_mem(res17, res17, u14, m, SIMD_BYTES * 2); + res17 = fma52lo(res17, u14, m[3]); + res18 = fma52hi(res18, u14, m[3]); + fma52lo_mem(res18, res18, u14, m, SIMD_BYTES * 4); + fma52hi_mem(res19, res19, u14, m, SIMD_BYTES * 4); + res19 = fma52lo(res19, u14, m[5]); + res20 = fma52hi(res20, u14, m[5]); + fma52lo_mem(res20, res20, u14, m, SIMD_BYTES * 6); + fma52hi_mem(res21, res21, u14, m, SIMD_BYTES * 6); + res21 = fma52lo(res21, u14, m[7]); + res22 = fma52hi(res22, u14, m[7]); + fma52lo_mem(res22, res22, u14, m, SIMD_BYTES * 8); + fma52hi_mem(res23, res23, u14, m, SIMD_BYTES * 8); + res23 = fma52lo(res23, u14, m[9]); + res24 = fma52hi(res24, u14, m[9]); + fma52lo_mem(res24, res24, u14, m, SIMD_BYTES * 10); + fma52hi_mem(res25, res25, u14, m, SIMD_BYTES * 10); + res25 = fma52lo(res25, u14, m[11]); + res26 = fma52hi(res26, u14, m[11]); + fma52lo_mem(res26, res26, u14, m, SIMD_BYTES * 12); + fma52hi_mem(res27, res27, u14, m, SIMD_BYTES * 12); + res27 = fma52lo(res27, u14, m[13]); + res28 = fma52hi(res28, u14, m[13]); + fma52lo_mem(res28, res28, u14, m, SIMD_BYTES * 14); + fma52hi_mem(res29, res29, u14, m, SIMD_BYTES * 14); + res29 = fma52lo(res29, u14, m[15]); + res30 = fma52hi(res30, u14, m[15]); + fma52lo_mem(res30, res30, u14, m, SIMD_BYTES * 16); + fma52hi_mem(res31, res31, u14, m, SIMD_BYTES * 16); + res31 = fma52lo(res31, u14, m[17]); + res32 = fma52hi(res32, u14, m[17]); + fma52lo_mem(res32, res32, u14, m, SIMD_BYTES * 18); + fma52hi_mem(res33, res33, u14, m, SIMD_BYTES * 18); + res33 = fma52lo(res33, u14, m[19]); + res34 = fma52hi(res34, u14, m[19]); + + // Create u15 + fma52lo_mem(res15, res15, u15, m, SIMD_BYTES * 0); + fma52hi_mem(res16, res16, u15, m, SIMD_BYTES * 0); + res16 = fma52lo(res16, u15, m[1]); + res17 = fma52hi(res17, u15, m[1]); + res16 = add64(res16, srli64(res15, DIGIT_SIZE)); + U64 u16 = mul52lo(res16, k); + fma52lo_mem(res17, res17, u15, m, SIMD_BYTES * 2); + fma52hi_mem(res18, res18, u15, m, SIMD_BYTES * 2); + res18 = fma52lo(res18, u15, m[3]); + res19 = fma52hi(res19, u15, m[3]); + fma52lo_mem(res19, res19, u15, m, SIMD_BYTES * 4); + fma52hi_mem(res20, res20, u15, m, SIMD_BYTES * 4); + res20 = fma52lo(res20, u15, m[5]); + res21 = fma52hi(res21, u15, m[5]); + fma52lo_mem(res21, res21, u15, m, SIMD_BYTES * 6); + fma52hi_mem(res22, res22, u15, m, SIMD_BYTES * 6); + res22 = fma52lo(res22, u15, m[7]); + res23 = fma52hi(res23, u15, m[7]); + fma52lo_mem(res23, res23, u15, m, SIMD_BYTES * 8); + fma52hi_mem(res24, res24, u15, m, SIMD_BYTES * 8); + res24 = fma52lo(res24, u15, m[9]); + res25 = fma52hi(res25, u15, m[9]); + fma52lo_mem(res25, res25, u15, m, SIMD_BYTES * 10); + fma52hi_mem(res26, res26, u15, m, SIMD_BYTES * 10); + res26 = fma52lo(res26, u15, m[11]); + res27 = fma52hi(res27, u15, m[11]); + fma52lo_mem(res27, res27, u15, m, SIMD_BYTES * 12); + fma52hi_mem(res28, res28, u15, m, SIMD_BYTES * 12); + res28 = fma52lo(res28, u15, m[13]); + res29 = fma52hi(res29, u15, m[13]); + fma52lo_mem(res29, res29, u15, m, SIMD_BYTES * 14); + fma52hi_mem(res30, res30, u15, m, SIMD_BYTES * 14); + res30 = fma52lo(res30, u15, m[15]); + res31 = fma52hi(res31, u15, m[15]); + fma52lo_mem(res31, res31, u15, m, SIMD_BYTES * 16); + fma52hi_mem(res32, res32, u15, m, SIMD_BYTES * 16); + res32 = fma52lo(res32, u15, m[17]); + res33 = fma52hi(res33, u15, m[17]); + fma52lo_mem(res33, res33, u15, m, SIMD_BYTES * 18); + fma52hi_mem(res34, res34, u15, m, SIMD_BYTES * 18); + res34 = fma52lo(res34, u15, m[19]); + res35 = fma52hi(res35, u15, m[19]); + ASM("jmp l16\nl16:\n"); + + // Create u16 + fma52lo_mem(res16, res16, u16, m, SIMD_BYTES * 0); + fma52hi_mem(res17, res17, u16, m, SIMD_BYTES * 0); + res17 = fma52lo(res17, u16, m[1]); + res18 = fma52hi(res18, u16, m[1]); + res17 = add64(res17, srli64(res16, DIGIT_SIZE)); + U64 u17 = mul52lo(res17, k); + fma52lo_mem(res18, res18, u16, m, SIMD_BYTES * 2); + fma52hi_mem(res19, res19, u16, m, SIMD_BYTES * 2); + res19 = fma52lo(res19, u16, m[3]); + res20 = fma52hi(res20, u16, m[3]); + fma52lo_mem(res20, res20, u16, m, SIMD_BYTES * 4); + fma52hi_mem(res21, res21, u16, m, SIMD_BYTES * 4); + res21 = fma52lo(res21, u16, m[5]); + res22 = fma52hi(res22, u16, m[5]); + fma52lo_mem(res22, res22, u16, m, SIMD_BYTES * 6); + fma52hi_mem(res23, res23, u16, m, SIMD_BYTES * 6); + res23 = fma52lo(res23, u16, m[7]); + res24 = fma52hi(res24, u16, m[7]); + fma52lo_mem(res24, res24, u16, m, SIMD_BYTES * 8); + fma52hi_mem(res25, res25, u16, m, SIMD_BYTES * 8); + res25 = fma52lo(res25, u16, m[9]); + res26 = fma52hi(res26, u16, m[9]); + fma52lo_mem(res26, res26, u16, m, SIMD_BYTES * 10); + fma52hi_mem(res27, res27, u16, m, SIMD_BYTES * 10); + res27 = fma52lo(res27, u16, m[11]); + res28 = fma52hi(res28, u16, m[11]); + fma52lo_mem(res28, res28, u16, m, SIMD_BYTES * 12); + fma52hi_mem(res29, res29, u16, m, SIMD_BYTES * 12); + res29 = fma52lo(res29, u16, m[13]); + res30 = fma52hi(res30, u16, m[13]); + fma52lo_mem(res30, res30, u16, m, SIMD_BYTES * 14); + fma52hi_mem(res31, res31, u16, m, SIMD_BYTES * 14); + res31 = fma52lo(res31, u16, m[15]); + res32 = fma52hi(res32, u16, m[15]); + fma52lo_mem(res32, res32, u16, m, SIMD_BYTES * 16); + fma52hi_mem(res33, res33, u16, m, SIMD_BYTES * 16); + res33 = fma52lo(res33, u16, m[17]); + res34 = fma52hi(res34, u16, m[17]); + fma52lo_mem(res34, res34, u16, m, SIMD_BYTES * 18); + fma52hi_mem(res35, res35, u16, m, SIMD_BYTES * 18); + res35 = fma52lo(res35, u16, m[19]); + res36 = fma52hi(res36, u16, m[19]); + + // Create u17 + fma52lo_mem(res17, res17, u17, m, SIMD_BYTES * 0); + fma52hi_mem(res18, res18, u17, m, SIMD_BYTES * 0); + res18 = fma52lo(res18, u17, m[1]); + res19 = fma52hi(res19, u17, m[1]); + res18 = add64(res18, srli64(res17, DIGIT_SIZE)); + U64 u18 = mul52lo(res18, k); + fma52lo_mem(res19, res19, u17, m, SIMD_BYTES * 2); + fma52hi_mem(res20, res20, u17, m, SIMD_BYTES * 2); + res20 = fma52lo(res20, u17, m[3]); + res21 = fma52hi(res21, u17, m[3]); + fma52lo_mem(res21, res21, u17, m, SIMD_BYTES * 4); + fma52hi_mem(res22, res22, u17, m, SIMD_BYTES * 4); + res22 = fma52lo(res22, u17, m[5]); + res23 = fma52hi(res23, u17, m[5]); + fma52lo_mem(res23, res23, u17, m, SIMD_BYTES * 6); + fma52hi_mem(res24, res24, u17, m, SIMD_BYTES * 6); + res24 = fma52lo(res24, u17, m[7]); + res25 = fma52hi(res25, u17, m[7]); + fma52lo_mem(res25, res25, u17, m, SIMD_BYTES * 8); + fma52hi_mem(res26, res26, u17, m, SIMD_BYTES * 8); + res26 = fma52lo(res26, u17, m[9]); + res27 = fma52hi(res27, u17, m[9]); + fma52lo_mem(res27, res27, u17, m, SIMD_BYTES * 10); + fma52hi_mem(res28, res28, u17, m, SIMD_BYTES * 10); + res28 = fma52lo(res28, u17, m[11]); + res29 = fma52hi(res29, u17, m[11]); + fma52lo_mem(res29, res29, u17, m, SIMD_BYTES * 12); + fma52hi_mem(res30, res30, u17, m, SIMD_BYTES * 12); + res30 = fma52lo(res30, u17, m[13]); + res31 = fma52hi(res31, u17, m[13]); + fma52lo_mem(res31, res31, u17, m, SIMD_BYTES * 14); + fma52hi_mem(res32, res32, u17, m, SIMD_BYTES * 14); + res32 = fma52lo(res32, u17, m[15]); + res33 = fma52hi(res33, u17, m[15]); + fma52lo_mem(res33, res33, u17, m, SIMD_BYTES * 16); + fma52hi_mem(res34, res34, u17, m, SIMD_BYTES * 16); + res34 = fma52lo(res34, u17, m[17]); + res35 = fma52hi(res35, u17, m[17]); + fma52lo_mem(res35, res35, u17, m, SIMD_BYTES * 18); + fma52hi_mem(res36, res36, u17, m, SIMD_BYTES * 18); + res36 = fma52lo(res36, u17, m[19]); + res37 = fma52hi(res37, u17, m[19]); + ASM("jmp l18\nl18:\n"); + + // Create u18 + fma52lo_mem(res18, res18, u18, m, SIMD_BYTES * 0); + fma52hi_mem(res19, res19, u18, m, SIMD_BYTES * 0); + res19 = fma52lo(res19, u18, m[1]); + res20 = fma52hi(res20, u18, m[1]); + res19 = add64(res19, srli64(res18, DIGIT_SIZE)); + U64 u19 = mul52lo(res19, k); + fma52lo_mem(res20, res20, u18, m, SIMD_BYTES * 2); + fma52hi_mem(res21, res21, u18, m, SIMD_BYTES * 2); + res21 = fma52lo(res21, u18, m[3]); + res22 = fma52hi(res22, u18, m[3]); + fma52lo_mem(res22, res22, u18, m, SIMD_BYTES * 4); + fma52hi_mem(res23, res23, u18, m, SIMD_BYTES * 4); + res23 = fma52lo(res23, u18, m[5]); + res24 = fma52hi(res24, u18, m[5]); + fma52lo_mem(res24, res24, u18, m, SIMD_BYTES * 6); + fma52hi_mem(res25, res25, u18, m, SIMD_BYTES * 6); + res25 = fma52lo(res25, u18, m[7]); + res26 = fma52hi(res26, u18, m[7]); + fma52lo_mem(res26, res26, u18, m, SIMD_BYTES * 8); + fma52hi_mem(res27, res27, u18, m, SIMD_BYTES * 8); + res27 = fma52lo(res27, u18, m[9]); + res28 = fma52hi(res28, u18, m[9]); + fma52lo_mem(res28, res28, u18, m, SIMD_BYTES * 10); + fma52hi_mem(res29, res29, u18, m, SIMD_BYTES * 10); + res29 = fma52lo(res29, u18, m[11]); + res30 = fma52hi(res30, u18, m[11]); + fma52lo_mem(res30, res30, u18, m, SIMD_BYTES * 12); + fma52hi_mem(res31, res31, u18, m, SIMD_BYTES * 12); + res31 = fma52lo(res31, u18, m[13]); + res32 = fma52hi(res32, u18, m[13]); + fma52lo_mem(res32, res32, u18, m, SIMD_BYTES * 14); + fma52hi_mem(res33, res33, u18, m, SIMD_BYTES * 14); + res33 = fma52lo(res33, u18, m[15]); + res34 = fma52hi(res34, u18, m[15]); + fma52lo_mem(res34, res34, u18, m, SIMD_BYTES * 16); + fma52hi_mem(res35, res35, u18, m, SIMD_BYTES * 16); + res35 = fma52lo(res35, u18, m[17]); + res36 = fma52hi(res36, u18, m[17]); + fma52lo_mem(res36, res36, u18, m, SIMD_BYTES * 18); + fma52hi_mem(res37, res37, u18, m, SIMD_BYTES * 18); + res37 = fma52lo(res37, u18, m[19]); + res38 = fma52hi(res38, u18, m[19]); + + // Create u19 + fma52lo_mem(res19, res19, u19, m, SIMD_BYTES * 0); + fma52hi_mem(res20, res20, u19, m, SIMD_BYTES * 0); + res20 = fma52lo(res20, u19, m[1]); + res21 = fma52hi(res21, u19, m[1]); + res20 = add64(res20, srli64(res19, DIGIT_SIZE)); + fma52lo_mem(res21, res21, u19, m, SIMD_BYTES * 2); + fma52hi_mem(res22, res22, u19, m, SIMD_BYTES * 2); + res22 = fma52lo(res22, u19, m[3]); + res23 = fma52hi(res23, u19, m[3]); + fma52lo_mem(res23, res23, u19, m, SIMD_BYTES * 4); + fma52hi_mem(res24, res24, u19, m, SIMD_BYTES * 4); + res24 = fma52lo(res24, u19, m[5]); + res25 = fma52hi(res25, u19, m[5]); + fma52lo_mem(res25, res25, u19, m, SIMD_BYTES * 6); + fma52hi_mem(res26, res26, u19, m, SIMD_BYTES * 6); + res26 = fma52lo(res26, u19, m[7]); + res27 = fma52hi(res27, u19, m[7]); + fma52lo_mem(res27, res27, u19, m, SIMD_BYTES * 8); + fma52hi_mem(res28, res28, u19, m, SIMD_BYTES * 8); + res28 = fma52lo(res28, u19, m[9]); + res29 = fma52hi(res29, u19, m[9]); + fma52lo_mem(res29, res29, u19, m, SIMD_BYTES * 10); + fma52hi_mem(res30, res30, u19, m, SIMD_BYTES * 10); + res30 = fma52lo(res30, u19, m[11]); + res31 = fma52hi(res31, u19, m[11]); + fma52lo_mem(res31, res31, u19, m, SIMD_BYTES * 12); + fma52hi_mem(res32, res32, u19, m, SIMD_BYTES * 12); + res32 = fma52lo(res32, u19, m[13]); + res33 = fma52hi(res33, u19, m[13]); + fma52lo_mem(res33, res33, u19, m, SIMD_BYTES * 14); + fma52hi_mem(res34, res34, u19, m, SIMD_BYTES * 14); + res34 = fma52lo(res34, u19, m[15]); + res35 = fma52hi(res35, u19, m[15]); + fma52lo_mem(res35, res35, u19, m, SIMD_BYTES * 16); + fma52hi_mem(res36, res36, u19, m, SIMD_BYTES * 16); + res36 = fma52lo(res36, u19, m[17]); + res37 = fma52hi(res37, u19, m[17]); + fma52lo_mem(res37, res37, u19, m, SIMD_BYTES * 18); + fma52hi_mem(res38, res38, u19, m, SIMD_BYTES * 18); + res38 = fma52lo(res38, u19, m[19]); + res39 = fma52hi(res39, u19, m[19]); + + // Normalization + r[0] = and64_const(res20, DIGIT_MASK); + res21 = add64(res21, srli64(res20, DIGIT_SIZE)); + r[1] = and64_const(res21, DIGIT_MASK); + res22 = add64(res22, srli64(res21, DIGIT_SIZE)); + r[2] = and64_const(res22, DIGIT_MASK); + res23 = add64(res23, srli64(res22, DIGIT_SIZE)); + r[3] = and64_const(res23, DIGIT_MASK); + res24 = add64(res24, srli64(res23, DIGIT_SIZE)); + r[4] = and64_const(res24, DIGIT_MASK); + res25 = add64(res25, srli64(res24, DIGIT_SIZE)); + r[5] = and64_const(res25, DIGIT_MASK); + res26 = add64(res26, srli64(res25, DIGIT_SIZE)); + r[6] = and64_const(res26, DIGIT_MASK); + res27 = add64(res27, srli64(res26, DIGIT_SIZE)); + r[7] = and64_const(res27, DIGIT_MASK); + res28 = add64(res28, srli64(res27, DIGIT_SIZE)); + r[8] = and64_const(res28, DIGIT_MASK); + res29 = add64(res29, srli64(res28, DIGIT_SIZE)); + r[9] = and64_const(res29, DIGIT_MASK); + res30 = add64(res30, srli64(res29, DIGIT_SIZE)); + r[10] = and64_const(res30, DIGIT_MASK); + res31 = add64(res31, srli64(res30, DIGIT_SIZE)); + r[11] = and64_const(res31, DIGIT_MASK); + res32 = add64(res32, srli64(res31, DIGIT_SIZE)); + r[12] = and64_const(res32, DIGIT_MASK); + res33 = add64(res33, srli64(res32, DIGIT_SIZE)); + r[13] = and64_const(res33, DIGIT_MASK); + res34 = add64(res34, srli64(res33, DIGIT_SIZE)); + r[14] = and64_const(res34, DIGIT_MASK); + res35 = add64(res35, srli64(res34, DIGIT_SIZE)); + r[15] = and64_const(res35, DIGIT_MASK); + res36 = add64(res36, srli64(res35, DIGIT_SIZE)); + r[16] = and64_const(res36, DIGIT_MASK); + res37 = add64(res37, srli64(res36, DIGIT_SIZE)); + r[17] = and64_const(res37, DIGIT_MASK); + res38 = add64(res38, srli64(res37, DIGIT_SIZE)); + r[18] = and64_const(res38, DIGIT_MASK); + res39 = add64(res39, srli64(res38, DIGIT_SIZE)); + r[19] = and64_const(res39, DIGIT_MASK); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams52x30_diagonal_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams52x30_diagonal_mb8.c new file mode 100644 index 0000000..9a51a82 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams52x30_diagonal_mb8.c @@ -0,0 +1,1522 @@ +/******************************************************************************* + * Copyright (C) 2019 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include + +#ifdef __GNUC__ +#define ASM(a) __asm__(a); +#else +#define ASM(a) +#endif + +void AMS52x30_diagonal_mb8(int64u *out_mb, const int64u *inpA_mb, + const int64u *inpM_mb, const int64u *k0_mb) { + + __ALIGN64 U64 res[60]; + __ALIGN64 U64 u[30]; + U64 k; + U64 *a = (U64 *)inpA_mb; + U64 *m = (U64 *)inpM_mb; + U64 *r = (U64 *)out_mb; + + k = loadu64((U64 *)k0_mb); + int i; + for (i = 0; i < 60; ++i) + res[i] = get_zero64(); + + res[1] = fma52lo(res[1], a[0], a[1]); // Sum(1) + res[2] = fma52hi(res[2], a[0], a[1]); // Sum(1) + res[2] = fma52lo(res[2], a[0], a[2]); // Sum(2) + res[3] = fma52hi(res[3], a[0], a[2]); // Sum(2) + res[3] = fma52lo(res[3], a[1], a[2]); // Sum(3) + res[4] = fma52hi(res[4], a[1], a[2]); // Sum(3) + res[3] = fma52lo(res[3], a[0], a[3]); // Sum(3) + res[4] = fma52hi(res[4], a[0], a[3]); // Sum(3) + res[4] = fma52lo(res[4], a[1], a[3]); // Sum(4) + res[5] = fma52hi(res[5], a[1], a[3]); // Sum(4) + res[5] = fma52lo(res[5], a[2], a[3]); // Sum(5) + res[6] = fma52hi(res[6], a[2], a[3]); // Sum(5) + res[4] = fma52lo(res[4], a[0], a[4]); // Sum(4) + res[5] = fma52hi(res[5], a[0], a[4]); // Sum(4) + res[5] = fma52lo(res[5], a[1], a[4]); // Sum(5) + res[6] = fma52hi(res[6], a[1], a[4]); // Sum(5) + res[6] = fma52lo(res[6], a[2], a[4]); // Sum(6) + res[7] = fma52hi(res[7], a[2], a[4]); // Sum(6) + res[7] = fma52lo(res[7], a[3], a[4]); // Sum(7) + res[8] = fma52hi(res[8], a[3], a[4]); // Sum(7) + res[5] = fma52lo(res[5], a[0], a[5]); // Sum(5) + res[6] = fma52hi(res[6], a[0], a[5]); // Sum(5) + res[6] = fma52lo(res[6], a[1], a[5]); // Sum(6) + res[7] = fma52hi(res[7], a[1], a[5]); // Sum(6) + res[7] = fma52lo(res[7], a[2], a[5]); // Sum(7) + res[8] = fma52hi(res[8], a[2], a[5]); // Sum(7) + res[8] = fma52lo(res[8], a[3], a[5]); // Sum(8) + res[9] = fma52hi(res[9], a[3], a[5]); // Sum(8) + res[9] = fma52lo(res[9], a[4], a[5]); // Sum(9) + res[10] = fma52hi(res[10], a[4], a[5]); // Sum(9) + res[6] = fma52lo(res[6], a[0], a[6]); // Sum(6) + res[7] = fma52hi(res[7], a[0], a[6]); // Sum(6) + res[7] = fma52lo(res[7], a[1], a[6]); // Sum(7) + res[8] = fma52hi(res[8], a[1], a[6]); // Sum(7) + res[8] = fma52lo(res[8], a[2], a[6]); // Sum(8) + res[9] = fma52hi(res[9], a[2], a[6]); // Sum(8) + res[9] = fma52lo(res[9], a[3], a[6]); // Sum(9) + res[10] = fma52hi(res[10], a[3], a[6]); // Sum(9) + res[10] = fma52lo(res[10], a[4], a[6]); // Sum(10) + res[11] = fma52hi(res[11], a[4], a[6]); // Sum(10) + res[11] = fma52lo(res[11], a[5], a[6]); // Sum(11) + res[12] = fma52hi(res[12], a[5], a[6]); // Sum(11) + res[7] = fma52lo(res[7], a[0], a[7]); // Sum(7) + res[8] = fma52hi(res[8], a[0], a[7]); // Sum(7) + res[8] = fma52lo(res[8], a[1], a[7]); // Sum(8) + res[9] = fma52hi(res[9], a[1], a[7]); // Sum(8) + res[9] = fma52lo(res[9], a[2], a[7]); // Sum(9) + res[10] = fma52hi(res[10], a[2], a[7]); // Sum(9) + res[10] = fma52lo(res[10], a[3], a[7]); // Sum(10) + res[11] = fma52hi(res[11], a[3], a[7]); // Sum(10) + res[11] = fma52lo(res[11], a[4], a[7]); // Sum(11) + res[12] = fma52hi(res[12], a[4], a[7]); // Sum(11) + res[8] = fma52lo(res[8], a[0], a[8]); // Sum(8) + res[9] = fma52hi(res[9], a[0], a[8]); // Sum(8) + res[9] = fma52lo(res[9], a[1], a[8]); // Sum(9) + res[10] = fma52hi(res[10], a[1], a[8]); // Sum(9) + res[10] = fma52lo(res[10], a[2], a[8]); // Sum(10) + res[11] = fma52hi(res[11], a[2], a[8]); // Sum(10) + res[11] = fma52lo(res[11], a[3], a[8]); // Sum(11) + res[12] = fma52hi(res[12], a[3], a[8]); // Sum(11) + res[9] = fma52lo(res[9], a[0], a[9]); // Sum(9) + res[10] = fma52hi(res[10], a[0], a[9]); // Sum(9) + res[10] = fma52lo(res[10], a[1], a[9]); // Sum(10) + res[11] = fma52hi(res[11], a[1], a[9]); // Sum(10) + res[11] = fma52lo(res[11], a[2], a[9]); // Sum(11) + res[12] = fma52hi(res[12], a[2], a[9]); // Sum(11) + res[10] = fma52lo(res[10], a[0], a[10]); // Sum(10) + res[11] = fma52hi(res[11], a[0], a[10]); // Sum(10) + res[11] = fma52lo(res[11], a[1], a[10]); // Sum(11) + res[12] = fma52hi(res[12], a[1], a[10]); // Sum(11) + res[11] = fma52lo(res[11], a[0], a[11]); // Sum(11) + res[12] = fma52hi(res[12], a[0], a[11]); // Sum(11) + res[0] = add64(res[0], res[0]); // Double(0) + res[1] = add64(res[1], res[1]); // Double(1) + res[2] = add64(res[2], res[2]); // Double(2) + res[3] = add64(res[3], res[3]); // Double(3) + res[4] = add64(res[4], res[4]); // Double(4) + res[5] = add64(res[5], res[5]); // Double(5) + res[6] = add64(res[6], res[6]); // Double(6) + res[7] = add64(res[7], res[7]); // Double(7) + res[8] = add64(res[8], res[8]); // Double(8) + res[9] = add64(res[9], res[9]); // Double(9) + res[10] = add64(res[10], res[10]); // Double(10) + res[11] = add64(res[11], res[11]); // Double(11) + res[0] = fma52lo(res[0], a[0], a[0]); // Add sqr(0) + res[1] = fma52hi(res[1], a[0], a[0]); // Add sqr(0) + res[2] = fma52lo(res[2], a[1], a[1]); // Add sqr(2) + res[3] = fma52hi(res[3], a[1], a[1]); // Add sqr(2) + res[4] = fma52lo(res[4], a[2], a[2]); // Add sqr(4) + res[5] = fma52hi(res[5], a[2], a[2]); // Add sqr(4) + res[6] = fma52lo(res[6], a[3], a[3]); // Add sqr(6) + res[7] = fma52hi(res[7], a[3], a[3]); // Add sqr(6) + res[8] = fma52lo(res[8], a[4], a[4]); // Add sqr(8) + res[9] = fma52hi(res[9], a[4], a[4]); // Add sqr(8) + res[10] = fma52lo(res[10], a[5], a[5]); // Add sqr(10) + res[11] = fma52hi(res[11], a[5], a[5]); // Add sqr(10) + res[12] = fma52lo(res[12], a[5], a[7]); // Sum(12) + res[13] = fma52hi(res[13], a[5], a[7]); // Sum(12) + res[13] = fma52lo(res[13], a[6], a[7]); // Sum(13) + res[14] = fma52hi(res[14], a[6], a[7]); // Sum(13) + res[12] = fma52lo(res[12], a[4], a[8]); // Sum(12) + res[13] = fma52hi(res[13], a[4], a[8]); // Sum(12) + res[13] = fma52lo(res[13], a[5], a[8]); // Sum(13) + res[14] = fma52hi(res[14], a[5], a[8]); // Sum(13) + res[14] = fma52lo(res[14], a[6], a[8]); // Sum(14) + res[15] = fma52hi(res[15], a[6], a[8]); // Sum(14) + res[15] = fma52lo(res[15], a[7], a[8]); // Sum(15) + res[16] = fma52hi(res[16], a[7], a[8]); // Sum(15) + res[12] = fma52lo(res[12], a[3], a[9]); // Sum(12) + res[13] = fma52hi(res[13], a[3], a[9]); // Sum(12) + res[13] = fma52lo(res[13], a[4], a[9]); // Sum(13) + res[14] = fma52hi(res[14], a[4], a[9]); // Sum(13) + res[14] = fma52lo(res[14], a[5], a[9]); // Sum(14) + res[15] = fma52hi(res[15], a[5], a[9]); // Sum(14) + res[15] = fma52lo(res[15], a[6], a[9]); // Sum(15) + res[16] = fma52hi(res[16], a[6], a[9]); // Sum(15) + res[16] = fma52lo(res[16], a[7], a[9]); // Sum(16) + res[17] = fma52hi(res[17], a[7], a[9]); // Sum(16) + res[17] = fma52lo(res[17], a[8], a[9]); // Sum(17) + res[18] = fma52hi(res[18], a[8], a[9]); // Sum(17) + res[12] = fma52lo(res[12], a[2], a[10]); // Sum(12) + res[13] = fma52hi(res[13], a[2], a[10]); // Sum(12) + res[13] = fma52lo(res[13], a[3], a[10]); // Sum(13) + res[14] = fma52hi(res[14], a[3], a[10]); // Sum(13) + res[14] = fma52lo(res[14], a[4], a[10]); // Sum(14) + res[15] = fma52hi(res[15], a[4], a[10]); // Sum(14) + res[15] = fma52lo(res[15], a[5], a[10]); // Sum(15) + res[16] = fma52hi(res[16], a[5], a[10]); // Sum(15) + res[16] = fma52lo(res[16], a[6], a[10]); // Sum(16) + res[17] = fma52hi(res[17], a[6], a[10]); // Sum(16) + res[17] = fma52lo(res[17], a[7], a[10]); // Sum(17) + res[18] = fma52hi(res[18], a[7], a[10]); // Sum(17) + res[18] = fma52lo(res[18], a[8], a[10]); // Sum(18) + res[19] = fma52hi(res[19], a[8], a[10]); // Sum(18) + res[19] = fma52lo(res[19], a[9], a[10]); // Sum(19) + res[20] = fma52hi(res[20], a[9], a[10]); // Sum(19) + res[12] = fma52lo(res[12], a[1], a[11]); // Sum(12) + res[13] = fma52hi(res[13], a[1], a[11]); // Sum(12) + res[13] = fma52lo(res[13], a[2], a[11]); // Sum(13) + res[14] = fma52hi(res[14], a[2], a[11]); // Sum(13) + res[14] = fma52lo(res[14], a[3], a[11]); // Sum(14) + res[15] = fma52hi(res[15], a[3], a[11]); // Sum(14) + res[15] = fma52lo(res[15], a[4], a[11]); // Sum(15) + res[16] = fma52hi(res[16], a[4], a[11]); // Sum(15) + res[16] = fma52lo(res[16], a[5], a[11]); // Sum(16) + res[17] = fma52hi(res[17], a[5], a[11]); // Sum(16) + res[17] = fma52lo(res[17], a[6], a[11]); // Sum(17) + res[18] = fma52hi(res[18], a[6], a[11]); // Sum(17) + res[18] = fma52lo(res[18], a[7], a[11]); // Sum(18) + res[19] = fma52hi(res[19], a[7], a[11]); // Sum(18) + res[19] = fma52lo(res[19], a[8], a[11]); // Sum(19) + res[20] = fma52hi(res[20], a[8], a[11]); // Sum(19) + res[20] = fma52lo(res[20], a[9], a[11]); // Sum(20) + res[21] = fma52hi(res[21], a[9], a[11]); // Sum(20) + res[21] = fma52lo(res[21], a[10], a[11]); // Sum(21) + res[22] = fma52hi(res[22], a[10], a[11]); // Sum(21) + res[12] = fma52lo(res[12], a[0], a[12]); // Sum(12) + res[13] = fma52hi(res[13], a[0], a[12]); // Sum(12) + res[13] = fma52lo(res[13], a[1], a[12]); // Sum(13) + res[14] = fma52hi(res[14], a[1], a[12]); // Sum(13) + res[14] = fma52lo(res[14], a[2], a[12]); // Sum(14) + res[15] = fma52hi(res[15], a[2], a[12]); // Sum(14) + res[15] = fma52lo(res[15], a[3], a[12]); // Sum(15) + res[16] = fma52hi(res[16], a[3], a[12]); // Sum(15) + res[16] = fma52lo(res[16], a[4], a[12]); // Sum(16) + res[17] = fma52hi(res[17], a[4], a[12]); // Sum(16) + res[17] = fma52lo(res[17], a[5], a[12]); // Sum(17) + res[18] = fma52hi(res[18], a[5], a[12]); // Sum(17) + res[18] = fma52lo(res[18], a[6], a[12]); // Sum(18) + res[19] = fma52hi(res[19], a[6], a[12]); // Sum(18) + res[19] = fma52lo(res[19], a[7], a[12]); // Sum(19) + res[20] = fma52hi(res[20], a[7], a[12]); // Sum(19) + res[20] = fma52lo(res[20], a[8], a[12]); // Sum(20) + res[21] = fma52hi(res[21], a[8], a[12]); // Sum(20) + res[21] = fma52lo(res[21], a[9], a[12]); // Sum(21) + res[22] = fma52hi(res[22], a[9], a[12]); // Sum(21) + res[22] = fma52lo(res[22], a[10], a[12]); // Sum(22) + res[23] = fma52hi(res[23], a[10], a[12]); // Sum(22) + res[23] = fma52lo(res[23], a[11], a[12]); // Sum(23) + res[24] = fma52hi(res[24], a[11], a[12]); // Sum(23) + res[13] = fma52lo(res[13], a[0], a[13]); // Sum(13) + res[14] = fma52hi(res[14], a[0], a[13]); // Sum(13) + res[14] = fma52lo(res[14], a[1], a[13]); // Sum(14) + res[15] = fma52hi(res[15], a[1], a[13]); // Sum(14) + res[15] = fma52lo(res[15], a[2], a[13]); // Sum(15) + res[16] = fma52hi(res[16], a[2], a[13]); // Sum(15) + res[16] = fma52lo(res[16], a[3], a[13]); // Sum(16) + res[17] = fma52hi(res[17], a[3], a[13]); // Sum(16) + res[17] = fma52lo(res[17], a[4], a[13]); // Sum(17) + res[18] = fma52hi(res[18], a[4], a[13]); // Sum(17) + res[18] = fma52lo(res[18], a[5], a[13]); // Sum(18) + res[19] = fma52hi(res[19], a[5], a[13]); // Sum(18) + res[19] = fma52lo(res[19], a[6], a[13]); // Sum(19) + res[20] = fma52hi(res[20], a[6], a[13]); // Sum(19) + res[20] = fma52lo(res[20], a[7], a[13]); // Sum(20) + res[21] = fma52hi(res[21], a[7], a[13]); // Sum(20) + res[21] = fma52lo(res[21], a[8], a[13]); // Sum(21) + res[22] = fma52hi(res[22], a[8], a[13]); // Sum(21) + res[22] = fma52lo(res[22], a[9], a[13]); // Sum(22) + res[23] = fma52hi(res[23], a[9], a[13]); // Sum(22) + res[23] = fma52lo(res[23], a[10], a[13]); // Sum(23) + res[24] = fma52hi(res[24], a[10], a[13]); // Sum(23) + res[14] = fma52lo(res[14], a[0], a[14]); // Sum(14) + res[15] = fma52hi(res[15], a[0], a[14]); // Sum(14) + res[15] = fma52lo(res[15], a[1], a[14]); // Sum(15) + res[16] = fma52hi(res[16], a[1], a[14]); // Sum(15) + res[16] = fma52lo(res[16], a[2], a[14]); // Sum(16) + res[17] = fma52hi(res[17], a[2], a[14]); // Sum(16) + res[17] = fma52lo(res[17], a[3], a[14]); // Sum(17) + res[18] = fma52hi(res[18], a[3], a[14]); // Sum(17) + res[18] = fma52lo(res[18], a[4], a[14]); // Sum(18) + res[19] = fma52hi(res[19], a[4], a[14]); // Sum(18) + res[19] = fma52lo(res[19], a[5], a[14]); // Sum(19) + res[20] = fma52hi(res[20], a[5], a[14]); // Sum(19) + res[20] = fma52lo(res[20], a[6], a[14]); // Sum(20) + res[21] = fma52hi(res[21], a[6], a[14]); // Sum(20) + res[21] = fma52lo(res[21], a[7], a[14]); // Sum(21) + res[22] = fma52hi(res[22], a[7], a[14]); // Sum(21) + res[22] = fma52lo(res[22], a[8], a[14]); // Sum(22) + res[23] = fma52hi(res[23], a[8], a[14]); // Sum(22) + res[23] = fma52lo(res[23], a[9], a[14]); // Sum(23) + res[24] = fma52hi(res[24], a[9], a[14]); // Sum(23) + res[15] = fma52lo(res[15], a[0], a[15]); // Sum(15) + res[16] = fma52hi(res[16], a[0], a[15]); // Sum(15) + res[16] = fma52lo(res[16], a[1], a[15]); // Sum(16) + res[17] = fma52hi(res[17], a[1], a[15]); // Sum(16) + res[17] = fma52lo(res[17], a[2], a[15]); // Sum(17) + res[18] = fma52hi(res[18], a[2], a[15]); // Sum(17) + res[18] = fma52lo(res[18], a[3], a[15]); // Sum(18) + res[19] = fma52hi(res[19], a[3], a[15]); // Sum(18) + res[19] = fma52lo(res[19], a[4], a[15]); // Sum(19) + res[20] = fma52hi(res[20], a[4], a[15]); // Sum(19) + res[20] = fma52lo(res[20], a[5], a[15]); // Sum(20) + res[21] = fma52hi(res[21], a[5], a[15]); // Sum(20) + res[21] = fma52lo(res[21], a[6], a[15]); // Sum(21) + res[22] = fma52hi(res[22], a[6], a[15]); // Sum(21) + res[22] = fma52lo(res[22], a[7], a[15]); // Sum(22) + res[23] = fma52hi(res[23], a[7], a[15]); // Sum(22) + res[23] = fma52lo(res[23], a[8], a[15]); // Sum(23) + res[24] = fma52hi(res[24], a[8], a[15]); // Sum(23) + res[16] = fma52lo(res[16], a[0], a[16]); // Sum(16) + res[17] = fma52hi(res[17], a[0], a[16]); // Sum(16) + res[17] = fma52lo(res[17], a[1], a[16]); // Sum(17) + res[18] = fma52hi(res[18], a[1], a[16]); // Sum(17) + res[18] = fma52lo(res[18], a[2], a[16]); // Sum(18) + res[19] = fma52hi(res[19], a[2], a[16]); // Sum(18) + res[19] = fma52lo(res[19], a[3], a[16]); // Sum(19) + res[20] = fma52hi(res[20], a[3], a[16]); // Sum(19) + res[20] = fma52lo(res[20], a[4], a[16]); // Sum(20) + res[21] = fma52hi(res[21], a[4], a[16]); // Sum(20) + res[21] = fma52lo(res[21], a[5], a[16]); // Sum(21) + res[22] = fma52hi(res[22], a[5], a[16]); // Sum(21) + res[22] = fma52lo(res[22], a[6], a[16]); // Sum(22) + res[23] = fma52hi(res[23], a[6], a[16]); // Sum(22) + res[23] = fma52lo(res[23], a[7], a[16]); // Sum(23) + res[24] = fma52hi(res[24], a[7], a[16]); // Sum(23) + res[17] = fma52lo(res[17], a[0], a[17]); // Sum(17) + res[18] = fma52hi(res[18], a[0], a[17]); // Sum(17) + res[18] = fma52lo(res[18], a[1], a[17]); // Sum(18) + res[19] = fma52hi(res[19], a[1], a[17]); // Sum(18) + res[19] = fma52lo(res[19], a[2], a[17]); // Sum(19) + res[20] = fma52hi(res[20], a[2], a[17]); // Sum(19) + res[20] = fma52lo(res[20], a[3], a[17]); // Sum(20) + res[21] = fma52hi(res[21], a[3], a[17]); // Sum(20) + res[21] = fma52lo(res[21], a[4], a[17]); // Sum(21) + res[22] = fma52hi(res[22], a[4], a[17]); // Sum(21) + res[22] = fma52lo(res[22], a[5], a[17]); // Sum(22) + res[23] = fma52hi(res[23], a[5], a[17]); // Sum(22) + res[23] = fma52lo(res[23], a[6], a[17]); // Sum(23) + res[24] = fma52hi(res[24], a[6], a[17]); // Sum(23) + res[18] = fma52lo(res[18], a[0], a[18]); // Sum(18) + res[19] = fma52hi(res[19], a[0], a[18]); // Sum(18) + res[19] = fma52lo(res[19], a[1], a[18]); // Sum(19) + res[20] = fma52hi(res[20], a[1], a[18]); // Sum(19) + res[20] = fma52lo(res[20], a[2], a[18]); // Sum(20) + res[21] = fma52hi(res[21], a[2], a[18]); // Sum(20) + res[21] = fma52lo(res[21], a[3], a[18]); // Sum(21) + res[22] = fma52hi(res[22], a[3], a[18]); // Sum(21) + res[22] = fma52lo(res[22], a[4], a[18]); // Sum(22) + res[23] = fma52hi(res[23], a[4], a[18]); // Sum(22) + res[23] = fma52lo(res[23], a[5], a[18]); // Sum(23) + res[24] = fma52hi(res[24], a[5], a[18]); // Sum(23) + res[19] = fma52lo(res[19], a[0], a[19]); // Sum(19) + res[20] = fma52hi(res[20], a[0], a[19]); // Sum(19) + res[20] = fma52lo(res[20], a[1], a[19]); // Sum(20) + res[21] = fma52hi(res[21], a[1], a[19]); // Sum(20) + res[21] = fma52lo(res[21], a[2], a[19]); // Sum(21) + res[22] = fma52hi(res[22], a[2], a[19]); // Sum(21) + res[22] = fma52lo(res[22], a[3], a[19]); // Sum(22) + res[23] = fma52hi(res[23], a[3], a[19]); // Sum(22) + res[23] = fma52lo(res[23], a[4], a[19]); // Sum(23) + res[24] = fma52hi(res[24], a[4], a[19]); // Sum(23) + res[20] = fma52lo(res[20], a[0], a[20]); // Sum(20) + res[21] = fma52hi(res[21], a[0], a[20]); // Sum(20) + res[21] = fma52lo(res[21], a[1], a[20]); // Sum(21) + res[22] = fma52hi(res[22], a[1], a[20]); // Sum(21) + res[22] = fma52lo(res[22], a[2], a[20]); // Sum(22) + res[23] = fma52hi(res[23], a[2], a[20]); // Sum(22) + res[23] = fma52lo(res[23], a[3], a[20]); // Sum(23) + res[24] = fma52hi(res[24], a[3], a[20]); // Sum(23) + res[21] = fma52lo(res[21], a[0], a[21]); // Sum(21) + res[22] = fma52hi(res[22], a[0], a[21]); // Sum(21) + res[22] = fma52lo(res[22], a[1], a[21]); // Sum(22) + res[23] = fma52hi(res[23], a[1], a[21]); // Sum(22) + res[23] = fma52lo(res[23], a[2], a[21]); // Sum(23) + res[24] = fma52hi(res[24], a[2], a[21]); // Sum(23) + res[22] = fma52lo(res[22], a[0], a[22]); // Sum(22) + res[23] = fma52hi(res[23], a[0], a[22]); // Sum(22) + res[23] = fma52lo(res[23], a[1], a[22]); // Sum(23) + res[24] = fma52hi(res[24], a[1], a[22]); // Sum(23) + res[23] = fma52lo(res[23], a[0], a[23]); // Sum(23) + res[24] = fma52hi(res[24], a[0], a[23]); // Sum(23) + res[12] = add64(res[12], res[12]); // Double(12) + res[13] = add64(res[13], res[13]); // Double(13) + res[14] = add64(res[14], res[14]); // Double(14) + res[15] = add64(res[15], res[15]); // Double(15) + res[16] = add64(res[16], res[16]); // Double(16) + res[17] = add64(res[17], res[17]); // Double(17) + res[18] = add64(res[18], res[18]); // Double(18) + res[19] = add64(res[19], res[19]); // Double(19) + res[20] = add64(res[20], res[20]); // Double(20) + res[21] = add64(res[21], res[21]); // Double(21) + res[22] = add64(res[22], res[22]); // Double(22) + res[23] = add64(res[23], res[23]); // Double(23) + res[12] = fma52lo(res[12], a[6], a[6]); // Add sqr(12) + res[13] = fma52hi(res[13], a[6], a[6]); // Add sqr(12) + res[14] = fma52lo(res[14], a[7], a[7]); // Add sqr(14) + res[15] = fma52hi(res[15], a[7], a[7]); // Add sqr(14) + res[16] = fma52lo(res[16], a[8], a[8]); // Add sqr(16) + res[17] = fma52hi(res[17], a[8], a[8]); // Add sqr(16) + res[18] = fma52lo(res[18], a[9], a[9]); // Add sqr(18) + res[19] = fma52hi(res[19], a[9], a[9]); // Add sqr(18) + res[20] = fma52lo(res[20], a[10], a[10]); // Add sqr(20) + res[21] = fma52hi(res[21], a[10], a[10]); // Add sqr(20) + res[22] = fma52lo(res[22], a[11], a[11]); // Add sqr(22) + res[23] = fma52hi(res[23], a[11], a[11]); // Add sqr(22) + res[24] = fma52lo(res[24], a[11], a[13]); // Sum(24) + res[25] = fma52hi(res[25], a[11], a[13]); // Sum(24) + res[25] = fma52lo(res[25], a[12], a[13]); // Sum(25) + res[26] = fma52hi(res[26], a[12], a[13]); // Sum(25) + res[24] = fma52lo(res[24], a[10], a[14]); // Sum(24) + res[25] = fma52hi(res[25], a[10], a[14]); // Sum(24) + res[25] = fma52lo(res[25], a[11], a[14]); // Sum(25) + res[26] = fma52hi(res[26], a[11], a[14]); // Sum(25) + res[26] = fma52lo(res[26], a[12], a[14]); // Sum(26) + res[27] = fma52hi(res[27], a[12], a[14]); // Sum(26) + res[27] = fma52lo(res[27], a[13], a[14]); // Sum(27) + res[28] = fma52hi(res[28], a[13], a[14]); // Sum(27) + res[24] = fma52lo(res[24], a[9], a[15]); // Sum(24) + res[25] = fma52hi(res[25], a[9], a[15]); // Sum(24) + res[25] = fma52lo(res[25], a[10], a[15]); // Sum(25) + res[26] = fma52hi(res[26], a[10], a[15]); // Sum(25) + res[26] = fma52lo(res[26], a[11], a[15]); // Sum(26) + res[27] = fma52hi(res[27], a[11], a[15]); // Sum(26) + res[27] = fma52lo(res[27], a[12], a[15]); // Sum(27) + res[28] = fma52hi(res[28], a[12], a[15]); // Sum(27) + res[28] = fma52lo(res[28], a[13], a[15]); // Sum(28) + res[29] = fma52hi(res[29], a[13], a[15]); // Sum(28) + res[29] = fma52lo(res[29], a[14], a[15]); // Sum(29) + res[30] = fma52hi(res[30], a[14], a[15]); // Sum(29) + res[24] = fma52lo(res[24], a[8], a[16]); // Sum(24) + res[25] = fma52hi(res[25], a[8], a[16]); // Sum(24) + res[25] = fma52lo(res[25], a[9], a[16]); // Sum(25) + res[26] = fma52hi(res[26], a[9], a[16]); // Sum(25) + res[26] = fma52lo(res[26], a[10], a[16]); // Sum(26) + res[27] = fma52hi(res[27], a[10], a[16]); // Sum(26) + res[27] = fma52lo(res[27], a[11], a[16]); // Sum(27) + res[28] = fma52hi(res[28], a[11], a[16]); // Sum(27) + res[28] = fma52lo(res[28], a[12], a[16]); // Sum(28) + res[29] = fma52hi(res[29], a[12], a[16]); // Sum(28) + res[29] = fma52lo(res[29], a[13], a[16]); // Sum(29) + res[30] = fma52hi(res[30], a[13], a[16]); // Sum(29) + res[30] = fma52lo(res[30], a[14], a[16]); // Sum(30) + res[31] = fma52hi(res[31], a[14], a[16]); // Sum(30) + res[31] = fma52lo(res[31], a[15], a[16]); // Sum(31) + res[32] = fma52hi(res[32], a[15], a[16]); // Sum(31) + res[24] = fma52lo(res[24], a[7], a[17]); // Sum(24) + res[25] = fma52hi(res[25], a[7], a[17]); // Sum(24) + res[25] = fma52lo(res[25], a[8], a[17]); // Sum(25) + res[26] = fma52hi(res[26], a[8], a[17]); // Sum(25) + res[26] = fma52lo(res[26], a[9], a[17]); // Sum(26) + res[27] = fma52hi(res[27], a[9], a[17]); // Sum(26) + res[27] = fma52lo(res[27], a[10], a[17]); // Sum(27) + res[28] = fma52hi(res[28], a[10], a[17]); // Sum(27) + res[28] = fma52lo(res[28], a[11], a[17]); // Sum(28) + res[29] = fma52hi(res[29], a[11], a[17]); // Sum(28) + res[29] = fma52lo(res[29], a[12], a[17]); // Sum(29) + res[30] = fma52hi(res[30], a[12], a[17]); // Sum(29) + res[30] = fma52lo(res[30], a[13], a[17]); // Sum(30) + res[31] = fma52hi(res[31], a[13], a[17]); // Sum(30) + res[31] = fma52lo(res[31], a[14], a[17]); // Sum(31) + res[32] = fma52hi(res[32], a[14], a[17]); // Sum(31) + res[32] = fma52lo(res[32], a[15], a[17]); // Sum(32) + res[33] = fma52hi(res[33], a[15], a[17]); // Sum(32) + res[33] = fma52lo(res[33], a[16], a[17]); // Sum(33) + res[34] = fma52hi(res[34], a[16], a[17]); // Sum(33) + res[24] = fma52lo(res[24], a[6], a[18]); // Sum(24) + res[25] = fma52hi(res[25], a[6], a[18]); // Sum(24) + res[25] = fma52lo(res[25], a[7], a[18]); // Sum(25) + res[26] = fma52hi(res[26], a[7], a[18]); // Sum(25) + res[26] = fma52lo(res[26], a[8], a[18]); // Sum(26) + res[27] = fma52hi(res[27], a[8], a[18]); // Sum(26) + res[27] = fma52lo(res[27], a[9], a[18]); // Sum(27) + res[28] = fma52hi(res[28], a[9], a[18]); // Sum(27) + res[28] = fma52lo(res[28], a[10], a[18]); // Sum(28) + res[29] = fma52hi(res[29], a[10], a[18]); // Sum(28) + res[29] = fma52lo(res[29], a[11], a[18]); // Sum(29) + res[30] = fma52hi(res[30], a[11], a[18]); // Sum(29) + res[30] = fma52lo(res[30], a[12], a[18]); // Sum(30) + res[31] = fma52hi(res[31], a[12], a[18]); // Sum(30) + res[31] = fma52lo(res[31], a[13], a[18]); // Sum(31) + res[32] = fma52hi(res[32], a[13], a[18]); // Sum(31) + res[32] = fma52lo(res[32], a[14], a[18]); // Sum(32) + res[33] = fma52hi(res[33], a[14], a[18]); // Sum(32) + res[33] = fma52lo(res[33], a[15], a[18]); // Sum(33) + res[34] = fma52hi(res[34], a[15], a[18]); // Sum(33) + res[34] = fma52lo(res[34], a[16], a[18]); // Sum(34) + res[35] = fma52hi(res[35], a[16], a[18]); // Sum(34) + res[35] = fma52lo(res[35], a[17], a[18]); // Sum(35) + res[36] = fma52hi(res[36], a[17], a[18]); // Sum(35) + res[24] = fma52lo(res[24], a[5], a[19]); // Sum(24) + res[25] = fma52hi(res[25], a[5], a[19]); // Sum(24) + res[25] = fma52lo(res[25], a[6], a[19]); // Sum(25) + res[26] = fma52hi(res[26], a[6], a[19]); // Sum(25) + res[26] = fma52lo(res[26], a[7], a[19]); // Sum(26) + res[27] = fma52hi(res[27], a[7], a[19]); // Sum(26) + res[27] = fma52lo(res[27], a[8], a[19]); // Sum(27) + res[28] = fma52hi(res[28], a[8], a[19]); // Sum(27) + res[28] = fma52lo(res[28], a[9], a[19]); // Sum(28) + res[29] = fma52hi(res[29], a[9], a[19]); // Sum(28) + res[29] = fma52lo(res[29], a[10], a[19]); // Sum(29) + res[30] = fma52hi(res[30], a[10], a[19]); // Sum(29) + res[30] = fma52lo(res[30], a[11], a[19]); // Sum(30) + res[31] = fma52hi(res[31], a[11], a[19]); // Sum(30) + res[31] = fma52lo(res[31], a[12], a[19]); // Sum(31) + res[32] = fma52hi(res[32], a[12], a[19]); // Sum(31) + res[32] = fma52lo(res[32], a[13], a[19]); // Sum(32) + res[33] = fma52hi(res[33], a[13], a[19]); // Sum(32) + res[33] = fma52lo(res[33], a[14], a[19]); // Sum(33) + res[34] = fma52hi(res[34], a[14], a[19]); // Sum(33) + res[34] = fma52lo(res[34], a[15], a[19]); // Sum(34) + res[35] = fma52hi(res[35], a[15], a[19]); // Sum(34) + res[35] = fma52lo(res[35], a[16], a[19]); // Sum(35) + res[36] = fma52hi(res[36], a[16], a[19]); // Sum(35) + res[24] = fma52lo(res[24], a[4], a[20]); // Sum(24) + res[25] = fma52hi(res[25], a[4], a[20]); // Sum(24) + res[25] = fma52lo(res[25], a[5], a[20]); // Sum(25) + res[26] = fma52hi(res[26], a[5], a[20]); // Sum(25) + res[26] = fma52lo(res[26], a[6], a[20]); // Sum(26) + res[27] = fma52hi(res[27], a[6], a[20]); // Sum(26) + res[27] = fma52lo(res[27], a[7], a[20]); // Sum(27) + res[28] = fma52hi(res[28], a[7], a[20]); // Sum(27) + res[28] = fma52lo(res[28], a[8], a[20]); // Sum(28) + res[29] = fma52hi(res[29], a[8], a[20]); // Sum(28) + res[29] = fma52lo(res[29], a[9], a[20]); // Sum(29) + res[30] = fma52hi(res[30], a[9], a[20]); // Sum(29) + res[30] = fma52lo(res[30], a[10], a[20]); // Sum(30) + res[31] = fma52hi(res[31], a[10], a[20]); // Sum(30) + res[31] = fma52lo(res[31], a[11], a[20]); // Sum(31) + res[32] = fma52hi(res[32], a[11], a[20]); // Sum(31) + res[32] = fma52lo(res[32], a[12], a[20]); // Sum(32) + res[33] = fma52hi(res[33], a[12], a[20]); // Sum(32) + res[33] = fma52lo(res[33], a[13], a[20]); // Sum(33) + res[34] = fma52hi(res[34], a[13], a[20]); // Sum(33) + res[34] = fma52lo(res[34], a[14], a[20]); // Sum(34) + res[35] = fma52hi(res[35], a[14], a[20]); // Sum(34) + res[35] = fma52lo(res[35], a[15], a[20]); // Sum(35) + res[36] = fma52hi(res[36], a[15], a[20]); // Sum(35) + res[24] = fma52lo(res[24], a[3], a[21]); // Sum(24) + res[25] = fma52hi(res[25], a[3], a[21]); // Sum(24) + res[25] = fma52lo(res[25], a[4], a[21]); // Sum(25) + res[26] = fma52hi(res[26], a[4], a[21]); // Sum(25) + res[26] = fma52lo(res[26], a[5], a[21]); // Sum(26) + res[27] = fma52hi(res[27], a[5], a[21]); // Sum(26) + res[27] = fma52lo(res[27], a[6], a[21]); // Sum(27) + res[28] = fma52hi(res[28], a[6], a[21]); // Sum(27) + res[28] = fma52lo(res[28], a[7], a[21]); // Sum(28) + res[29] = fma52hi(res[29], a[7], a[21]); // Sum(28) + res[29] = fma52lo(res[29], a[8], a[21]); // Sum(29) + res[30] = fma52hi(res[30], a[8], a[21]); // Sum(29) + res[30] = fma52lo(res[30], a[9], a[21]); // Sum(30) + res[31] = fma52hi(res[31], a[9], a[21]); // Sum(30) + res[31] = fma52lo(res[31], a[10], a[21]); // Sum(31) + res[32] = fma52hi(res[32], a[10], a[21]); // Sum(31) + res[32] = fma52lo(res[32], a[11], a[21]); // Sum(32) + res[33] = fma52hi(res[33], a[11], a[21]); // Sum(32) + res[33] = fma52lo(res[33], a[12], a[21]); // Sum(33) + res[34] = fma52hi(res[34], a[12], a[21]); // Sum(33) + res[34] = fma52lo(res[34], a[13], a[21]); // Sum(34) + res[35] = fma52hi(res[35], a[13], a[21]); // Sum(34) + res[35] = fma52lo(res[35], a[14], a[21]); // Sum(35) + res[36] = fma52hi(res[36], a[14], a[21]); // Sum(35) + res[24] = fma52lo(res[24], a[2], a[22]); // Sum(24) + res[25] = fma52hi(res[25], a[2], a[22]); // Sum(24) + res[25] = fma52lo(res[25], a[3], a[22]); // Sum(25) + res[26] = fma52hi(res[26], a[3], a[22]); // Sum(25) + res[26] = fma52lo(res[26], a[4], a[22]); // Sum(26) + res[27] = fma52hi(res[27], a[4], a[22]); // Sum(26) + res[27] = fma52lo(res[27], a[5], a[22]); // Sum(27) + res[28] = fma52hi(res[28], a[5], a[22]); // Sum(27) + res[28] = fma52lo(res[28], a[6], a[22]); // Sum(28) + res[29] = fma52hi(res[29], a[6], a[22]); // Sum(28) + res[29] = fma52lo(res[29], a[7], a[22]); // Sum(29) + res[30] = fma52hi(res[30], a[7], a[22]); // Sum(29) + res[30] = fma52lo(res[30], a[8], a[22]); // Sum(30) + res[31] = fma52hi(res[31], a[8], a[22]); // Sum(30) + res[31] = fma52lo(res[31], a[9], a[22]); // Sum(31) + res[32] = fma52hi(res[32], a[9], a[22]); // Sum(31) + res[32] = fma52lo(res[32], a[10], a[22]); // Sum(32) + res[33] = fma52hi(res[33], a[10], a[22]); // Sum(32) + res[33] = fma52lo(res[33], a[11], a[22]); // Sum(33) + res[34] = fma52hi(res[34], a[11], a[22]); // Sum(33) + res[34] = fma52lo(res[34], a[12], a[22]); // Sum(34) + res[35] = fma52hi(res[35], a[12], a[22]); // Sum(34) + res[35] = fma52lo(res[35], a[13], a[22]); // Sum(35) + res[36] = fma52hi(res[36], a[13], a[22]); // Sum(35) + res[24] = fma52lo(res[24], a[1], a[23]); // Sum(24) + res[25] = fma52hi(res[25], a[1], a[23]); // Sum(24) + res[25] = fma52lo(res[25], a[2], a[23]); // Sum(25) + res[26] = fma52hi(res[26], a[2], a[23]); // Sum(25) + res[26] = fma52lo(res[26], a[3], a[23]); // Sum(26) + res[27] = fma52hi(res[27], a[3], a[23]); // Sum(26) + res[27] = fma52lo(res[27], a[4], a[23]); // Sum(27) + res[28] = fma52hi(res[28], a[4], a[23]); // Sum(27) + res[28] = fma52lo(res[28], a[5], a[23]); // Sum(28) + res[29] = fma52hi(res[29], a[5], a[23]); // Sum(28) + res[29] = fma52lo(res[29], a[6], a[23]); // Sum(29) + res[30] = fma52hi(res[30], a[6], a[23]); // Sum(29) + res[30] = fma52lo(res[30], a[7], a[23]); // Sum(30) + res[31] = fma52hi(res[31], a[7], a[23]); // Sum(30) + res[31] = fma52lo(res[31], a[8], a[23]); // Sum(31) + res[32] = fma52hi(res[32], a[8], a[23]); // Sum(31) + res[32] = fma52lo(res[32], a[9], a[23]); // Sum(32) + res[33] = fma52hi(res[33], a[9], a[23]); // Sum(32) + res[33] = fma52lo(res[33], a[10], a[23]); // Sum(33) + res[34] = fma52hi(res[34], a[10], a[23]); // Sum(33) + res[34] = fma52lo(res[34], a[11], a[23]); // Sum(34) + res[35] = fma52hi(res[35], a[11], a[23]); // Sum(34) + res[35] = fma52lo(res[35], a[12], a[23]); // Sum(35) + res[36] = fma52hi(res[36], a[12], a[23]); // Sum(35) + res[24] = fma52lo(res[24], a[0], a[24]); // Sum(24) + res[25] = fma52hi(res[25], a[0], a[24]); // Sum(24) + res[25] = fma52lo(res[25], a[1], a[24]); // Sum(25) + res[26] = fma52hi(res[26], a[1], a[24]); // Sum(25) + res[26] = fma52lo(res[26], a[2], a[24]); // Sum(26) + res[27] = fma52hi(res[27], a[2], a[24]); // Sum(26) + res[27] = fma52lo(res[27], a[3], a[24]); // Sum(27) + res[28] = fma52hi(res[28], a[3], a[24]); // Sum(27) + res[28] = fma52lo(res[28], a[4], a[24]); // Sum(28) + res[29] = fma52hi(res[29], a[4], a[24]); // Sum(28) + res[29] = fma52lo(res[29], a[5], a[24]); // Sum(29) + res[30] = fma52hi(res[30], a[5], a[24]); // Sum(29) + res[30] = fma52lo(res[30], a[6], a[24]); // Sum(30) + res[31] = fma52hi(res[31], a[6], a[24]); // Sum(30) + res[31] = fma52lo(res[31], a[7], a[24]); // Sum(31) + res[32] = fma52hi(res[32], a[7], a[24]); // Sum(31) + res[32] = fma52lo(res[32], a[8], a[24]); // Sum(32) + res[33] = fma52hi(res[33], a[8], a[24]); // Sum(32) + res[33] = fma52lo(res[33], a[9], a[24]); // Sum(33) + res[34] = fma52hi(res[34], a[9], a[24]); // Sum(33) + res[34] = fma52lo(res[34], a[10], a[24]); // Sum(34) + res[35] = fma52hi(res[35], a[10], a[24]); // Sum(34) + res[35] = fma52lo(res[35], a[11], a[24]); // Sum(35) + res[36] = fma52hi(res[36], a[11], a[24]); // Sum(35) + res[25] = fma52lo(res[25], a[0], a[25]); // Sum(25) + res[26] = fma52hi(res[26], a[0], a[25]); // Sum(25) + res[26] = fma52lo(res[26], a[1], a[25]); // Sum(26) + res[27] = fma52hi(res[27], a[1], a[25]); // Sum(26) + res[27] = fma52lo(res[27], a[2], a[25]); // Sum(27) + res[28] = fma52hi(res[28], a[2], a[25]); // Sum(27) + res[28] = fma52lo(res[28], a[3], a[25]); // Sum(28) + res[29] = fma52hi(res[29], a[3], a[25]); // Sum(28) + res[29] = fma52lo(res[29], a[4], a[25]); // Sum(29) + res[30] = fma52hi(res[30], a[4], a[25]); // Sum(29) + res[30] = fma52lo(res[30], a[5], a[25]); // Sum(30) + res[31] = fma52hi(res[31], a[5], a[25]); // Sum(30) + res[31] = fma52lo(res[31], a[6], a[25]); // Sum(31) + res[32] = fma52hi(res[32], a[6], a[25]); // Sum(31) + res[32] = fma52lo(res[32], a[7], a[25]); // Sum(32) + res[33] = fma52hi(res[33], a[7], a[25]); // Sum(32) + res[33] = fma52lo(res[33], a[8], a[25]); // Sum(33) + res[34] = fma52hi(res[34], a[8], a[25]); // Sum(33) + res[34] = fma52lo(res[34], a[9], a[25]); // Sum(34) + res[35] = fma52hi(res[35], a[9], a[25]); // Sum(34) + res[35] = fma52lo(res[35], a[10], a[25]); // Sum(35) + res[36] = fma52hi(res[36], a[10], a[25]); // Sum(35) + res[26] = fma52lo(res[26], a[0], a[26]); // Sum(26) + res[27] = fma52hi(res[27], a[0], a[26]); // Sum(26) + res[27] = fma52lo(res[27], a[1], a[26]); // Sum(27) + res[28] = fma52hi(res[28], a[1], a[26]); // Sum(27) + res[28] = fma52lo(res[28], a[2], a[26]); // Sum(28) + res[29] = fma52hi(res[29], a[2], a[26]); // Sum(28) + res[29] = fma52lo(res[29], a[3], a[26]); // Sum(29) + res[30] = fma52hi(res[30], a[3], a[26]); // Sum(29) + res[30] = fma52lo(res[30], a[4], a[26]); // Sum(30) + res[31] = fma52hi(res[31], a[4], a[26]); // Sum(30) + res[31] = fma52lo(res[31], a[5], a[26]); // Sum(31) + res[32] = fma52hi(res[32], a[5], a[26]); // Sum(31) + res[32] = fma52lo(res[32], a[6], a[26]); // Sum(32) + res[33] = fma52hi(res[33], a[6], a[26]); // Sum(32) + res[33] = fma52lo(res[33], a[7], a[26]); // Sum(33) + res[34] = fma52hi(res[34], a[7], a[26]); // Sum(33) + res[34] = fma52lo(res[34], a[8], a[26]); // Sum(34) + res[35] = fma52hi(res[35], a[8], a[26]); // Sum(34) + res[35] = fma52lo(res[35], a[9], a[26]); // Sum(35) + res[36] = fma52hi(res[36], a[9], a[26]); // Sum(35) + res[27] = fma52lo(res[27], a[0], a[27]); // Sum(27) + res[28] = fma52hi(res[28], a[0], a[27]); // Sum(27) + res[28] = fma52lo(res[28], a[1], a[27]); // Sum(28) + res[29] = fma52hi(res[29], a[1], a[27]); // Sum(28) + res[29] = fma52lo(res[29], a[2], a[27]); // Sum(29) + res[30] = fma52hi(res[30], a[2], a[27]); // Sum(29) + res[30] = fma52lo(res[30], a[3], a[27]); // Sum(30) + res[31] = fma52hi(res[31], a[3], a[27]); // Sum(30) + res[31] = fma52lo(res[31], a[4], a[27]); // Sum(31) + res[32] = fma52hi(res[32], a[4], a[27]); // Sum(31) + res[32] = fma52lo(res[32], a[5], a[27]); // Sum(32) + res[33] = fma52hi(res[33], a[5], a[27]); // Sum(32) + res[33] = fma52lo(res[33], a[6], a[27]); // Sum(33) + res[34] = fma52hi(res[34], a[6], a[27]); // Sum(33) + res[34] = fma52lo(res[34], a[7], a[27]); // Sum(34) + res[35] = fma52hi(res[35], a[7], a[27]); // Sum(34) + res[35] = fma52lo(res[35], a[8], a[27]); // Sum(35) + res[36] = fma52hi(res[36], a[8], a[27]); // Sum(35) + res[28] = fma52lo(res[28], a[0], a[28]); // Sum(28) + res[29] = fma52hi(res[29], a[0], a[28]); // Sum(28) + res[29] = fma52lo(res[29], a[1], a[28]); // Sum(29) + res[30] = fma52hi(res[30], a[1], a[28]); // Sum(29) + res[30] = fma52lo(res[30], a[2], a[28]); // Sum(30) + res[31] = fma52hi(res[31], a[2], a[28]); // Sum(30) + res[31] = fma52lo(res[31], a[3], a[28]); // Sum(31) + res[32] = fma52hi(res[32], a[3], a[28]); // Sum(31) + res[32] = fma52lo(res[32], a[4], a[28]); // Sum(32) + res[33] = fma52hi(res[33], a[4], a[28]); // Sum(32) + res[33] = fma52lo(res[33], a[5], a[28]); // Sum(33) + res[34] = fma52hi(res[34], a[5], a[28]); // Sum(33) + res[34] = fma52lo(res[34], a[6], a[28]); // Sum(34) + res[35] = fma52hi(res[35], a[6], a[28]); // Sum(34) + res[35] = fma52lo(res[35], a[7], a[28]); // Sum(35) + res[36] = fma52hi(res[36], a[7], a[28]); // Sum(35) + res[29] = fma52lo(res[29], a[0], a[29]); // Sum(29) + res[30] = fma52hi(res[30], a[0], a[29]); // Sum(29) + res[30] = fma52lo(res[30], a[1], a[29]); // Sum(30) + res[31] = fma52hi(res[31], a[1], a[29]); // Sum(30) + res[31] = fma52lo(res[31], a[2], a[29]); // Sum(31) + res[32] = fma52hi(res[32], a[2], a[29]); // Sum(31) + res[32] = fma52lo(res[32], a[3], a[29]); // Sum(32) + res[33] = fma52hi(res[33], a[3], a[29]); // Sum(32) + res[33] = fma52lo(res[33], a[4], a[29]); // Sum(33) + res[34] = fma52hi(res[34], a[4], a[29]); // Sum(33) + res[34] = fma52lo(res[34], a[5], a[29]); // Sum(34) + res[35] = fma52hi(res[35], a[5], a[29]); // Sum(34) + res[35] = fma52lo(res[35], a[6], a[29]); // Sum(35) + res[36] = fma52hi(res[36], a[6], a[29]); // Sum(35) + res[24] = add64(res[24], res[24]); // Double(24) + res[25] = add64(res[25], res[25]); // Double(25) + res[26] = add64(res[26], res[26]); // Double(26) + res[27] = add64(res[27], res[27]); // Double(27) + res[28] = add64(res[28], res[28]); // Double(28) + res[29] = add64(res[29], res[29]); // Double(29) + res[30] = add64(res[30], res[30]); // Double(30) + res[31] = add64(res[31], res[31]); // Double(31) + res[32] = add64(res[32], res[32]); // Double(32) + res[33] = add64(res[33], res[33]); // Double(33) + res[34] = add64(res[34], res[34]); // Double(34) + res[35] = add64(res[35], res[35]); // Double(35) + res[24] = fma52lo(res[24], a[12], a[12]); // Add sqr(24) + res[25] = fma52hi(res[25], a[12], a[12]); // Add sqr(24) + res[26] = fma52lo(res[26], a[13], a[13]); // Add sqr(26) + res[27] = fma52hi(res[27], a[13], a[13]); // Add sqr(26) + res[28] = fma52lo(res[28], a[14], a[14]); // Add sqr(28) + res[29] = fma52hi(res[29], a[14], a[14]); // Add sqr(28) + res[30] = fma52lo(res[30], a[15], a[15]); // Add sqr(30) + res[31] = fma52hi(res[31], a[15], a[15]); // Add sqr(30) + res[32] = fma52lo(res[32], a[16], a[16]); // Add sqr(32) + res[33] = fma52hi(res[33], a[16], a[16]); // Add sqr(32) + res[34] = fma52lo(res[34], a[17], a[17]); // Add sqr(34) + res[35] = fma52hi(res[35], a[17], a[17]); // Add sqr(34) + res[36] = fma52lo(res[36], a[17], a[19]); // Sum(36) + res[37] = fma52hi(res[37], a[17], a[19]); // Sum(36) + res[37] = fma52lo(res[37], a[18], a[19]); // Sum(37) + res[38] = fma52hi(res[38], a[18], a[19]); // Sum(37) + res[36] = fma52lo(res[36], a[16], a[20]); // Sum(36) + res[37] = fma52hi(res[37], a[16], a[20]); // Sum(36) + res[37] = fma52lo(res[37], a[17], a[20]); // Sum(37) + res[38] = fma52hi(res[38], a[17], a[20]); // Sum(37) + res[38] = fma52lo(res[38], a[18], a[20]); // Sum(38) + res[39] = fma52hi(res[39], a[18], a[20]); // Sum(38) + res[39] = fma52lo(res[39], a[19], a[20]); // Sum(39) + res[40] = fma52hi(res[40], a[19], a[20]); // Sum(39) + res[36] = fma52lo(res[36], a[15], a[21]); // Sum(36) + res[37] = fma52hi(res[37], a[15], a[21]); // Sum(36) + res[37] = fma52lo(res[37], a[16], a[21]); // Sum(37) + res[38] = fma52hi(res[38], a[16], a[21]); // Sum(37) + res[38] = fma52lo(res[38], a[17], a[21]); // Sum(38) + res[39] = fma52hi(res[39], a[17], a[21]); // Sum(38) + res[39] = fma52lo(res[39], a[18], a[21]); // Sum(39) + res[40] = fma52hi(res[40], a[18], a[21]); // Sum(39) + res[40] = fma52lo(res[40], a[19], a[21]); // Sum(40) + res[41] = fma52hi(res[41], a[19], a[21]); // Sum(40) + res[41] = fma52lo(res[41], a[20], a[21]); // Sum(41) + res[42] = fma52hi(res[42], a[20], a[21]); // Sum(41) + res[36] = fma52lo(res[36], a[14], a[22]); // Sum(36) + res[37] = fma52hi(res[37], a[14], a[22]); // Sum(36) + res[37] = fma52lo(res[37], a[15], a[22]); // Sum(37) + res[38] = fma52hi(res[38], a[15], a[22]); // Sum(37) + res[38] = fma52lo(res[38], a[16], a[22]); // Sum(38) + res[39] = fma52hi(res[39], a[16], a[22]); // Sum(38) + res[39] = fma52lo(res[39], a[17], a[22]); // Sum(39) + res[40] = fma52hi(res[40], a[17], a[22]); // Sum(39) + res[40] = fma52lo(res[40], a[18], a[22]); // Sum(40) + res[41] = fma52hi(res[41], a[18], a[22]); // Sum(40) + res[41] = fma52lo(res[41], a[19], a[22]); // Sum(41) + res[42] = fma52hi(res[42], a[19], a[22]); // Sum(41) + res[42] = fma52lo(res[42], a[20], a[22]); // Sum(42) + res[43] = fma52hi(res[43], a[20], a[22]); // Sum(42) + res[43] = fma52lo(res[43], a[21], a[22]); // Sum(43) + res[44] = fma52hi(res[44], a[21], a[22]); // Sum(43) + res[36] = fma52lo(res[36], a[13], a[23]); // Sum(36) + res[37] = fma52hi(res[37], a[13], a[23]); // Sum(36) + res[37] = fma52lo(res[37], a[14], a[23]); // Sum(37) + res[38] = fma52hi(res[38], a[14], a[23]); // Sum(37) + res[38] = fma52lo(res[38], a[15], a[23]); // Sum(38) + res[39] = fma52hi(res[39], a[15], a[23]); // Sum(38) + res[39] = fma52lo(res[39], a[16], a[23]); // Sum(39) + res[40] = fma52hi(res[40], a[16], a[23]); // Sum(39) + res[40] = fma52lo(res[40], a[17], a[23]); // Sum(40) + res[41] = fma52hi(res[41], a[17], a[23]); // Sum(40) + res[41] = fma52lo(res[41], a[18], a[23]); // Sum(41) + res[42] = fma52hi(res[42], a[18], a[23]); // Sum(41) + res[42] = fma52lo(res[42], a[19], a[23]); // Sum(42) + res[43] = fma52hi(res[43], a[19], a[23]); // Sum(42) + res[43] = fma52lo(res[43], a[20], a[23]); // Sum(43) + res[44] = fma52hi(res[44], a[20], a[23]); // Sum(43) + res[44] = fma52lo(res[44], a[21], a[23]); // Sum(44) + res[45] = fma52hi(res[45], a[21], a[23]); // Sum(44) + res[45] = fma52lo(res[45], a[22], a[23]); // Sum(45) + res[46] = fma52hi(res[46], a[22], a[23]); // Sum(45) + res[36] = fma52lo(res[36], a[12], a[24]); // Sum(36) + res[37] = fma52hi(res[37], a[12], a[24]); // Sum(36) + res[37] = fma52lo(res[37], a[13], a[24]); // Sum(37) + res[38] = fma52hi(res[38], a[13], a[24]); // Sum(37) + res[38] = fma52lo(res[38], a[14], a[24]); // Sum(38) + res[39] = fma52hi(res[39], a[14], a[24]); // Sum(38) + res[39] = fma52lo(res[39], a[15], a[24]); // Sum(39) + res[40] = fma52hi(res[40], a[15], a[24]); // Sum(39) + res[40] = fma52lo(res[40], a[16], a[24]); // Sum(40) + res[41] = fma52hi(res[41], a[16], a[24]); // Sum(40) + res[41] = fma52lo(res[41], a[17], a[24]); // Sum(41) + res[42] = fma52hi(res[42], a[17], a[24]); // Sum(41) + res[42] = fma52lo(res[42], a[18], a[24]); // Sum(42) + res[43] = fma52hi(res[43], a[18], a[24]); // Sum(42) + res[43] = fma52lo(res[43], a[19], a[24]); // Sum(43) + res[44] = fma52hi(res[44], a[19], a[24]); // Sum(43) + res[44] = fma52lo(res[44], a[20], a[24]); // Sum(44) + res[45] = fma52hi(res[45], a[20], a[24]); // Sum(44) + res[45] = fma52lo(res[45], a[21], a[24]); // Sum(45) + res[46] = fma52hi(res[46], a[21], a[24]); // Sum(45) + res[46] = fma52lo(res[46], a[22], a[24]); // Sum(46) + res[47] = fma52hi(res[47], a[22], a[24]); // Sum(46) + res[47] = fma52lo(res[47], a[23], a[24]); // Sum(47) + res[48] = fma52hi(res[48], a[23], a[24]); // Sum(47) + res[36] = fma52lo(res[36], a[11], a[25]); // Sum(36) + res[37] = fma52hi(res[37], a[11], a[25]); // Sum(36) + res[37] = fma52lo(res[37], a[12], a[25]); // Sum(37) + res[38] = fma52hi(res[38], a[12], a[25]); // Sum(37) + res[38] = fma52lo(res[38], a[13], a[25]); // Sum(38) + res[39] = fma52hi(res[39], a[13], a[25]); // Sum(38) + res[39] = fma52lo(res[39], a[14], a[25]); // Sum(39) + res[40] = fma52hi(res[40], a[14], a[25]); // Sum(39) + res[40] = fma52lo(res[40], a[15], a[25]); // Sum(40) + res[41] = fma52hi(res[41], a[15], a[25]); // Sum(40) + res[41] = fma52lo(res[41], a[16], a[25]); // Sum(41) + res[42] = fma52hi(res[42], a[16], a[25]); // Sum(41) + res[42] = fma52lo(res[42], a[17], a[25]); // Sum(42) + res[43] = fma52hi(res[43], a[17], a[25]); // Sum(42) + res[43] = fma52lo(res[43], a[18], a[25]); // Sum(43) + res[44] = fma52hi(res[44], a[18], a[25]); // Sum(43) + res[44] = fma52lo(res[44], a[19], a[25]); // Sum(44) + res[45] = fma52hi(res[45], a[19], a[25]); // Sum(44) + res[45] = fma52lo(res[45], a[20], a[25]); // Sum(45) + res[46] = fma52hi(res[46], a[20], a[25]); // Sum(45) + res[46] = fma52lo(res[46], a[21], a[25]); // Sum(46) + res[47] = fma52hi(res[47], a[21], a[25]); // Sum(46) + res[47] = fma52lo(res[47], a[22], a[25]); // Sum(47) + res[48] = fma52hi(res[48], a[22], a[25]); // Sum(47) + res[36] = fma52lo(res[36], a[10], a[26]); // Sum(36) + res[37] = fma52hi(res[37], a[10], a[26]); // Sum(36) + res[37] = fma52lo(res[37], a[11], a[26]); // Sum(37) + res[38] = fma52hi(res[38], a[11], a[26]); // Sum(37) + res[38] = fma52lo(res[38], a[12], a[26]); // Sum(38) + res[39] = fma52hi(res[39], a[12], a[26]); // Sum(38) + res[39] = fma52lo(res[39], a[13], a[26]); // Sum(39) + res[40] = fma52hi(res[40], a[13], a[26]); // Sum(39) + res[40] = fma52lo(res[40], a[14], a[26]); // Sum(40) + res[41] = fma52hi(res[41], a[14], a[26]); // Sum(40) + res[41] = fma52lo(res[41], a[15], a[26]); // Sum(41) + res[42] = fma52hi(res[42], a[15], a[26]); // Sum(41) + res[42] = fma52lo(res[42], a[16], a[26]); // Sum(42) + res[43] = fma52hi(res[43], a[16], a[26]); // Sum(42) + res[43] = fma52lo(res[43], a[17], a[26]); // Sum(43) + res[44] = fma52hi(res[44], a[17], a[26]); // Sum(43) + res[44] = fma52lo(res[44], a[18], a[26]); // Sum(44) + res[45] = fma52hi(res[45], a[18], a[26]); // Sum(44) + res[45] = fma52lo(res[45], a[19], a[26]); // Sum(45) + res[46] = fma52hi(res[46], a[19], a[26]); // Sum(45) + res[46] = fma52lo(res[46], a[20], a[26]); // Sum(46) + res[47] = fma52hi(res[47], a[20], a[26]); // Sum(46) + res[47] = fma52lo(res[47], a[21], a[26]); // Sum(47) + res[48] = fma52hi(res[48], a[21], a[26]); // Sum(47) + res[36] = fma52lo(res[36], a[9], a[27]); // Sum(36) + res[37] = fma52hi(res[37], a[9], a[27]); // Sum(36) + res[37] = fma52lo(res[37], a[10], a[27]); // Sum(37) + res[38] = fma52hi(res[38], a[10], a[27]); // Sum(37) + res[38] = fma52lo(res[38], a[11], a[27]); // Sum(38) + res[39] = fma52hi(res[39], a[11], a[27]); // Sum(38) + res[39] = fma52lo(res[39], a[12], a[27]); // Sum(39) + res[40] = fma52hi(res[40], a[12], a[27]); // Sum(39) + res[40] = fma52lo(res[40], a[13], a[27]); // Sum(40) + res[41] = fma52hi(res[41], a[13], a[27]); // Sum(40) + res[41] = fma52lo(res[41], a[14], a[27]); // Sum(41) + res[42] = fma52hi(res[42], a[14], a[27]); // Sum(41) + res[42] = fma52lo(res[42], a[15], a[27]); // Sum(42) + res[43] = fma52hi(res[43], a[15], a[27]); // Sum(42) + res[43] = fma52lo(res[43], a[16], a[27]); // Sum(43) + res[44] = fma52hi(res[44], a[16], a[27]); // Sum(43) + res[44] = fma52lo(res[44], a[17], a[27]); // Sum(44) + res[45] = fma52hi(res[45], a[17], a[27]); // Sum(44) + res[45] = fma52lo(res[45], a[18], a[27]); // Sum(45) + res[46] = fma52hi(res[46], a[18], a[27]); // Sum(45) + res[46] = fma52lo(res[46], a[19], a[27]); // Sum(46) + res[47] = fma52hi(res[47], a[19], a[27]); // Sum(46) + res[47] = fma52lo(res[47], a[20], a[27]); // Sum(47) + res[48] = fma52hi(res[48], a[20], a[27]); // Sum(47) + res[36] = fma52lo(res[36], a[8], a[28]); // Sum(36) + res[37] = fma52hi(res[37], a[8], a[28]); // Sum(36) + res[37] = fma52lo(res[37], a[9], a[28]); // Sum(37) + res[38] = fma52hi(res[38], a[9], a[28]); // Sum(37) + res[38] = fma52lo(res[38], a[10], a[28]); // Sum(38) + res[39] = fma52hi(res[39], a[10], a[28]); // Sum(38) + res[39] = fma52lo(res[39], a[11], a[28]); // Sum(39) + res[40] = fma52hi(res[40], a[11], a[28]); // Sum(39) + res[40] = fma52lo(res[40], a[12], a[28]); // Sum(40) + res[41] = fma52hi(res[41], a[12], a[28]); // Sum(40) + res[41] = fma52lo(res[41], a[13], a[28]); // Sum(41) + res[42] = fma52hi(res[42], a[13], a[28]); // Sum(41) + res[42] = fma52lo(res[42], a[14], a[28]); // Sum(42) + res[43] = fma52hi(res[43], a[14], a[28]); // Sum(42) + res[43] = fma52lo(res[43], a[15], a[28]); // Sum(43) + res[44] = fma52hi(res[44], a[15], a[28]); // Sum(43) + res[44] = fma52lo(res[44], a[16], a[28]); // Sum(44) + res[45] = fma52hi(res[45], a[16], a[28]); // Sum(44) + res[45] = fma52lo(res[45], a[17], a[28]); // Sum(45) + res[46] = fma52hi(res[46], a[17], a[28]); // Sum(45) + res[46] = fma52lo(res[46], a[18], a[28]); // Sum(46) + res[47] = fma52hi(res[47], a[18], a[28]); // Sum(46) + res[47] = fma52lo(res[47], a[19], a[28]); // Sum(47) + res[48] = fma52hi(res[48], a[19], a[28]); // Sum(47) + res[36] = fma52lo(res[36], a[7], a[29]); // Sum(36) + res[37] = fma52hi(res[37], a[7], a[29]); // Sum(36) + res[37] = fma52lo(res[37], a[8], a[29]); // Sum(37) + res[38] = fma52hi(res[38], a[8], a[29]); // Sum(37) + res[38] = fma52lo(res[38], a[9], a[29]); // Sum(38) + res[39] = fma52hi(res[39], a[9], a[29]); // Sum(38) + res[39] = fma52lo(res[39], a[10], a[29]); // Sum(39) + res[40] = fma52hi(res[40], a[10], a[29]); // Sum(39) + res[40] = fma52lo(res[40], a[11], a[29]); // Sum(40) + res[41] = fma52hi(res[41], a[11], a[29]); // Sum(40) + res[41] = fma52lo(res[41], a[12], a[29]); // Sum(41) + res[42] = fma52hi(res[42], a[12], a[29]); // Sum(41) + res[42] = fma52lo(res[42], a[13], a[29]); // Sum(42) + res[43] = fma52hi(res[43], a[13], a[29]); // Sum(42) + res[43] = fma52lo(res[43], a[14], a[29]); // Sum(43) + res[44] = fma52hi(res[44], a[14], a[29]); // Sum(43) + res[44] = fma52lo(res[44], a[15], a[29]); // Sum(44) + res[45] = fma52hi(res[45], a[15], a[29]); // Sum(44) + res[45] = fma52lo(res[45], a[16], a[29]); // Sum(45) + res[46] = fma52hi(res[46], a[16], a[29]); // Sum(45) + res[46] = fma52lo(res[46], a[17], a[29]); // Sum(46) + res[47] = fma52hi(res[47], a[17], a[29]); // Sum(46) + res[47] = fma52lo(res[47], a[18], a[29]); // Sum(47) + res[48] = fma52hi(res[48], a[18], a[29]); // Sum(47) + res[36] = add64(res[36], res[36]); // Double(36) + res[37] = add64(res[37], res[37]); // Double(37) + res[38] = add64(res[38], res[38]); // Double(38) + res[39] = add64(res[39], res[39]); // Double(39) + res[40] = add64(res[40], res[40]); // Double(40) + res[41] = add64(res[41], res[41]); // Double(41) + res[42] = add64(res[42], res[42]); // Double(42) + res[43] = add64(res[43], res[43]); // Double(43) + res[44] = add64(res[44], res[44]); // Double(44) + res[45] = add64(res[45], res[45]); // Double(45) + res[46] = add64(res[46], res[46]); // Double(46) + res[47] = add64(res[47], res[47]); // Double(47) + res[36] = fma52lo(res[36], a[18], a[18]); // Add sqr(36) + res[37] = fma52hi(res[37], a[18], a[18]); // Add sqr(36) + res[38] = fma52lo(res[38], a[19], a[19]); // Add sqr(38) + res[39] = fma52hi(res[39], a[19], a[19]); // Add sqr(38) + res[40] = fma52lo(res[40], a[20], a[20]); // Add sqr(40) + res[41] = fma52hi(res[41], a[20], a[20]); // Add sqr(40) + res[42] = fma52lo(res[42], a[21], a[21]); // Add sqr(42) + res[43] = fma52hi(res[43], a[21], a[21]); // Add sqr(42) + res[44] = fma52lo(res[44], a[22], a[22]); // Add sqr(44) + res[45] = fma52hi(res[45], a[22], a[22]); // Add sqr(44) + res[46] = fma52lo(res[46], a[23], a[23]); // Add sqr(46) + res[47] = fma52hi(res[47], a[23], a[23]); // Add sqr(46) + res[48] = fma52lo(res[48], a[23], a[25]); // Sum(48) + res[49] = fma52hi(res[49], a[23], a[25]); // Sum(48) + res[49] = fma52lo(res[49], a[24], a[25]); // Sum(49) + res[50] = fma52hi(res[50], a[24], a[25]); // Sum(49) + res[48] = fma52lo(res[48], a[22], a[26]); // Sum(48) + res[49] = fma52hi(res[49], a[22], a[26]); // Sum(48) + res[49] = fma52lo(res[49], a[23], a[26]); // Sum(49) + res[50] = fma52hi(res[50], a[23], a[26]); // Sum(49) + res[50] = fma52lo(res[50], a[24], a[26]); // Sum(50) + res[51] = fma52hi(res[51], a[24], a[26]); // Sum(50) + res[51] = fma52lo(res[51], a[25], a[26]); // Sum(51) + res[52] = fma52hi(res[52], a[25], a[26]); // Sum(51) + res[48] = fma52lo(res[48], a[21], a[27]); // Sum(48) + res[49] = fma52hi(res[49], a[21], a[27]); // Sum(48) + res[49] = fma52lo(res[49], a[22], a[27]); // Sum(49) + res[50] = fma52hi(res[50], a[22], a[27]); // Sum(49) + res[50] = fma52lo(res[50], a[23], a[27]); // Sum(50) + res[51] = fma52hi(res[51], a[23], a[27]); // Sum(50) + res[51] = fma52lo(res[51], a[24], a[27]); // Sum(51) + res[52] = fma52hi(res[52], a[24], a[27]); // Sum(51) + res[52] = fma52lo(res[52], a[25], a[27]); // Sum(52) + res[53] = fma52hi(res[53], a[25], a[27]); // Sum(52) + res[53] = fma52lo(res[53], a[26], a[27]); // Sum(53) + res[54] = fma52hi(res[54], a[26], a[27]); // Sum(53) + res[48] = fma52lo(res[48], a[20], a[28]); // Sum(48) + res[49] = fma52hi(res[49], a[20], a[28]); // Sum(48) + res[49] = fma52lo(res[49], a[21], a[28]); // Sum(49) + res[50] = fma52hi(res[50], a[21], a[28]); // Sum(49) + res[50] = fma52lo(res[50], a[22], a[28]); // Sum(50) + res[51] = fma52hi(res[51], a[22], a[28]); // Sum(50) + res[51] = fma52lo(res[51], a[23], a[28]); // Sum(51) + res[52] = fma52hi(res[52], a[23], a[28]); // Sum(51) + res[52] = fma52lo(res[52], a[24], a[28]); // Sum(52) + res[53] = fma52hi(res[53], a[24], a[28]); // Sum(52) + res[53] = fma52lo(res[53], a[25], a[28]); // Sum(53) + res[54] = fma52hi(res[54], a[25], a[28]); // Sum(53) + res[54] = fma52lo(res[54], a[26], a[28]); // Sum(54) + res[55] = fma52hi(res[55], a[26], a[28]); // Sum(54) + res[55] = fma52lo(res[55], a[27], a[28]); // Sum(55) + res[56] = fma52hi(res[56], a[27], a[28]); // Sum(55) + res[48] = fma52lo(res[48], a[19], a[29]); // Sum(48) + res[49] = fma52hi(res[49], a[19], a[29]); // Sum(48) + res[49] = fma52lo(res[49], a[20], a[29]); // Sum(49) + res[50] = fma52hi(res[50], a[20], a[29]); // Sum(49) + res[50] = fma52lo(res[50], a[21], a[29]); // Sum(50) + res[51] = fma52hi(res[51], a[21], a[29]); // Sum(50) + res[51] = fma52lo(res[51], a[22], a[29]); // Sum(51) + res[52] = fma52hi(res[52], a[22], a[29]); // Sum(51) + res[52] = fma52lo(res[52], a[23], a[29]); // Sum(52) + res[53] = fma52hi(res[53], a[23], a[29]); // Sum(52) + res[53] = fma52lo(res[53], a[24], a[29]); // Sum(53) + res[54] = fma52hi(res[54], a[24], a[29]); // Sum(53) + res[54] = fma52lo(res[54], a[25], a[29]); // Sum(54) + res[55] = fma52hi(res[55], a[25], a[29]); // Sum(54) + res[55] = fma52lo(res[55], a[26], a[29]); // Sum(55) + res[56] = fma52hi(res[56], a[26], a[29]); // Sum(55) + res[56] = fma52lo(res[56], a[27], a[29]); // Sum(56) + res[57] = fma52hi(res[57], a[27], a[29]); // Sum(56) + res[57] = fma52lo(res[57], a[28], a[29]); // Sum(57) + res[58] = fma52hi(res[58], a[28], a[29]); // Sum(57) + res[48] = add64(res[48], res[48]); // Double(48) + res[49] = add64(res[49], res[49]); // Double(49) + res[50] = add64(res[50], res[50]); // Double(50) + res[51] = add64(res[51], res[51]); // Double(51) + res[52] = add64(res[52], res[52]); // Double(52) + res[53] = add64(res[53], res[53]); // Double(53) + res[54] = add64(res[54], res[54]); // Double(54) + res[55] = add64(res[55], res[55]); // Double(55) + res[56] = add64(res[56], res[56]); // Double(56) + res[57] = add64(res[57], res[57]); // Double(57) + res[58] = add64(res[58], res[58]); // Double(58) + res[48] = fma52lo(res[48], a[24], a[24]); // Add sqr(48) + res[49] = fma52hi(res[49], a[24], a[24]); // Add sqr(48) + res[50] = fma52lo(res[50], a[25], a[25]); // Add sqr(50) + res[51] = fma52hi(res[51], a[25], a[25]); // Add sqr(50) + res[52] = fma52lo(res[52], a[26], a[26]); // Add sqr(52) + res[53] = fma52hi(res[53], a[26], a[26]); // Add sqr(52) + res[54] = fma52lo(res[54], a[27], a[27]); // Add sqr(54) + res[55] = fma52hi(res[55], a[27], a[27]); // Add sqr(54) + res[56] = fma52lo(res[56], a[28], a[28]); // Add sqr(56) + res[57] = fma52hi(res[57], a[28], a[28]); // Add sqr(56) + res[58] = fma52lo(res[58], a[29], a[29]); // Add sqr(58) + res[59] = fma52hi(res[59], a[29], a[29]); // Add sqr(58) + + // Montgomery Reduction + int it; + for (it = 0; it < 30; it += 10) { // Reduction step + int jt = 0; + if ((it + 0) > 0) + res[it + 0] = add64(res[it + 0], srli64(res[it + -1], DIGIT_SIZE)); + u[it + 0] = mul52lo(res[it + 0], k); + res[it + jt + 0] = fma52lo(res[it + jt + 0], u[it + 0], m[jt + 0]); + res[it + jt + 1] = fma52hi(res[it + jt + 1], u[it + 0], m[jt + 0]); + res[it + jt + 1] = fma52lo(res[it + jt + 1], u[it + 0], m[jt + 1]); + res[it + jt + 2] = fma52hi(res[it + jt + 2], u[it + 0], m[jt + 1]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 0], m[jt + 2]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 0], m[jt + 2]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 0], m[jt + 3]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 0], m[jt + 3]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 0], m[jt + 4]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 0], m[jt + 4]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 0], m[jt + 5]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 0], m[jt + 5]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 0], m[jt + 6]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 0], m[jt + 6]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 0], m[jt + 7]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 0], m[jt + 7]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 0], m[jt + 8]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 0], m[jt + 8]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 0], m[jt + 9]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 0], m[jt + 9]); + res[it + 1] = add64(res[it + 1], srli64(res[it + 0], DIGIT_SIZE)); + u[it + 1] = mul52lo(res[it + 1], k); + res[it + jt + 1] = fma52lo(res[it + jt + 1], u[it + 1], m[jt + 0]); + res[it + jt + 2] = fma52hi(res[it + jt + 2], u[it + 1], m[jt + 0]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 1], m[jt + 1]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 1], m[jt + 1]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 1], m[jt + 2]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 1], m[jt + 2]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 1], m[jt + 3]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 1], m[jt + 3]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 1], m[jt + 4]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 1], m[jt + 4]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 1], m[jt + 5]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 1], m[jt + 5]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 1], m[jt + 6]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 1], m[jt + 6]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 1], m[jt + 7]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 1], m[jt + 7]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 1], m[jt + 8]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 1], m[jt + 8]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 1], m[jt + 9]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 1], m[jt + 9]); + res[it + 2] = add64(res[it + 2], srli64(res[it + 1], DIGIT_SIZE)); + u[it + 2] = mul52lo(res[it + 2], k); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 2], m[jt + 0]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 2], m[jt + 0]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 2], m[jt + 1]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 2], m[jt + 1]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 2], m[jt + 2]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 2], m[jt + 2]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 2], m[jt + 3]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 2], m[jt + 3]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 2], m[jt + 4]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 2], m[jt + 4]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 2], m[jt + 5]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 2], m[jt + 5]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 2], m[jt + 6]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 2], m[jt + 6]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 2], m[jt + 7]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 2], m[jt + 7]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 2], m[jt + 8]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 2], m[jt + 8]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 2], m[jt + 9]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 2], m[jt + 9]); + res[it + 3] = add64(res[it + 3], srli64(res[it + 2], DIGIT_SIZE)); + u[it + 3] = mul52lo(res[it + 3], k); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 3], m[jt + 0]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 3], m[jt + 0]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 3], m[jt + 1]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 3], m[jt + 1]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 3], m[jt + 2]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 3], m[jt + 2]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 3], m[jt + 3]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 3], m[jt + 3]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 3], m[jt + 4]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 3], m[jt + 4]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 3], m[jt + 5]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 3], m[jt + 5]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 3], m[jt + 6]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 3], m[jt + 6]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 3], m[jt + 7]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 3], m[jt + 7]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 3], m[jt + 8]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 3], m[jt + 8]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 3], m[jt + 9]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 3], m[jt + 9]); + res[it + 4] = add64(res[it + 4], srli64(res[it + 3], DIGIT_SIZE)); + u[it + 4] = mul52lo(res[it + 4], k); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 4], m[jt + 0]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 4], m[jt + 0]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 4], m[jt + 1]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 4], m[jt + 1]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 4], m[jt + 2]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 4], m[jt + 2]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 4], m[jt + 3]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 4], m[jt + 3]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 4], m[jt + 4]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 4], m[jt + 4]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 4], m[jt + 5]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 4], m[jt + 5]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 4], m[jt + 6]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 4], m[jt + 6]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 4], m[jt + 7]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 4], m[jt + 7]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 4], m[jt + 8]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 4], m[jt + 8]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 4], m[jt + 9]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 4], m[jt + 9]); + res[it + 5] = add64(res[it + 5], srli64(res[it + 4], DIGIT_SIZE)); + u[it + 5] = mul52lo(res[it + 5], k); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 5], m[jt + 0]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 5], m[jt + 0]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 5], m[jt + 1]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 5], m[jt + 1]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 5], m[jt + 2]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 5], m[jt + 2]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 5], m[jt + 3]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 5], m[jt + 3]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 5], m[jt + 4]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 5], m[jt + 4]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 5], m[jt + 5]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 5], m[jt + 5]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 5], m[jt + 6]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 5], m[jt + 6]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 5], m[jt + 7]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 5], m[jt + 7]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 5], m[jt + 8]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 5], m[jt + 8]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 5], m[jt + 9]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 5], m[jt + 9]); + res[it + 6] = add64(res[it + 6], srli64(res[it + 5], DIGIT_SIZE)); + u[it + 6] = mul52lo(res[it + 6], k); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 6], m[jt + 0]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 6], m[jt + 0]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 6], m[jt + 1]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 6], m[jt + 1]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 6], m[jt + 2]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 6], m[jt + 2]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 6], m[jt + 3]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 6], m[jt + 3]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 6], m[jt + 4]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 6], m[jt + 4]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 6], m[jt + 5]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 6], m[jt + 5]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 6], m[jt + 6]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 6], m[jt + 6]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 6], m[jt + 7]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 6], m[jt + 7]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 6], m[jt + 8]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 6], m[jt + 8]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 6], m[jt + 9]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 6], m[jt + 9]); + res[it + 7] = add64(res[it + 7], srli64(res[it + 6], DIGIT_SIZE)); + u[it + 7] = mul52lo(res[it + 7], k); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 7], m[jt + 0]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 7], m[jt + 0]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 7], m[jt + 1]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 7], m[jt + 1]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 7], m[jt + 2]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 7], m[jt + 2]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 7], m[jt + 3]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 7], m[jt + 3]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 7], m[jt + 4]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 7], m[jt + 4]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 7], m[jt + 5]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 7], m[jt + 5]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 7], m[jt + 6]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 7], m[jt + 6]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 7], m[jt + 7]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 7], m[jt + 7]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 7], m[jt + 8]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 7], m[jt + 8]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 7], m[jt + 9]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 7], m[jt + 9]); + res[it + 8] = add64(res[it + 8], srli64(res[it + 7], DIGIT_SIZE)); + u[it + 8] = mul52lo(res[it + 8], k); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 8], m[jt + 0]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 8], m[jt + 0]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 8], m[jt + 1]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 8], m[jt + 1]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 8], m[jt + 2]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 8], m[jt + 2]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 8], m[jt + 3]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 8], m[jt + 3]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 8], m[jt + 4]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 8], m[jt + 4]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 8], m[jt + 5]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 8], m[jt + 5]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 8], m[jt + 6]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 8], m[jt + 6]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 8], m[jt + 7]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 8], m[jt + 7]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 8], m[jt + 8]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 8], m[jt + 8]); + res[it + jt + 17] = fma52lo(res[it + jt + 17], u[it + 8], m[jt + 9]); + res[it + jt + 18] = fma52hi(res[it + jt + 18], u[it + 8], m[jt + 9]); + res[it + 9] = add64(res[it + 9], srli64(res[it + 8], DIGIT_SIZE)); + u[it + 9] = mul52lo(res[it + 9], k); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 9], m[jt + 0]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 9], m[jt + 0]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 9], m[jt + 1]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 9], m[jt + 1]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 9], m[jt + 2]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 9], m[jt + 2]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 9], m[jt + 3]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 9], m[jt + 3]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 9], m[jt + 4]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 9], m[jt + 4]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 9], m[jt + 5]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 9], m[jt + 5]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 9], m[jt + 6]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 9], m[jt + 6]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 9], m[jt + 7]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 9], m[jt + 7]); + res[it + jt + 17] = fma52lo(res[it + jt + 17], u[it + 9], m[jt + 8]); + res[it + jt + 18] = fma52hi(res[it + jt + 18], u[it + 9], m[jt + 8]); + res[it + jt + 18] = fma52lo(res[it + jt + 18], u[it + 9], m[jt + 9]); + res[it + jt + 19] = fma52hi(res[it + jt + 19], u[it + 9], m[jt + 9]); + + for (jt = 10; jt < 30; jt += 10) { // Poly tile + res[it + jt + 0] = fma52lo(res[it + jt + 0], u[it + 0], m[jt + 0]); + res[it + jt + 1] = fma52hi(res[it + jt + 1], u[it + 0], m[jt + 0]); + res[it + jt + 1] = fma52lo(res[it + jt + 1], u[it + 0], m[jt + 1]); + res[it + jt + 2] = fma52hi(res[it + jt + 2], u[it + 0], m[jt + 1]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 0], m[jt + 2]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 0], m[jt + 2]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 0], m[jt + 3]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 0], m[jt + 3]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 0], m[jt + 4]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 0], m[jt + 4]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 0], m[jt + 5]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 0], m[jt + 5]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 0], m[jt + 6]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 0], m[jt + 6]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 0], m[jt + 7]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 0], m[jt + 7]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 0], m[jt + 8]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 0], m[jt + 8]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 0], m[jt + 9]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 0], m[jt + 9]); + res[it + jt + 1] = fma52lo(res[it + jt + 1], u[it + 1], m[jt + 0]); + res[it + jt + 2] = fma52hi(res[it + jt + 2], u[it + 1], m[jt + 0]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 1], m[jt + 1]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 1], m[jt + 1]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 1], m[jt + 2]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 1], m[jt + 2]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 1], m[jt + 3]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 1], m[jt + 3]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 1], m[jt + 4]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 1], m[jt + 4]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 1], m[jt + 5]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 1], m[jt + 5]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 1], m[jt + 6]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 1], m[jt + 6]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 1], m[jt + 7]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 1], m[jt + 7]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 1], m[jt + 8]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 1], m[jt + 8]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 1], m[jt + 9]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 1], m[jt + 9]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 2], m[jt + 0]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 2], m[jt + 0]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 2], m[jt + 1]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 2], m[jt + 1]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 2], m[jt + 2]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 2], m[jt + 2]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 2], m[jt + 3]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 2], m[jt + 3]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 2], m[jt + 4]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 2], m[jt + 4]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 2], m[jt + 5]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 2], m[jt + 5]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 2], m[jt + 6]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 2], m[jt + 6]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 2], m[jt + 7]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 2], m[jt + 7]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 2], m[jt + 8]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 2], m[jt + 8]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 2], m[jt + 9]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 2], m[jt + 9]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 3], m[jt + 0]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 3], m[jt + 0]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 3], m[jt + 1]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 3], m[jt + 1]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 3], m[jt + 2]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 3], m[jt + 2]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 3], m[jt + 3]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 3], m[jt + 3]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 3], m[jt + 4]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 3], m[jt + 4]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 3], m[jt + 5]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 3], m[jt + 5]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 3], m[jt + 6]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 3], m[jt + 6]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 3], m[jt + 7]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 3], m[jt + 7]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 3], m[jt + 8]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 3], m[jt + 8]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 3], m[jt + 9]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 3], m[jt + 9]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 4], m[jt + 0]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 4], m[jt + 0]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 4], m[jt + 1]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 4], m[jt + 1]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 4], m[jt + 2]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 4], m[jt + 2]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 4], m[jt + 3]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 4], m[jt + 3]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 4], m[jt + 4]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 4], m[jt + 4]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 4], m[jt + 5]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 4], m[jt + 5]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 4], m[jt + 6]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 4], m[jt + 6]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 4], m[jt + 7]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 4], m[jt + 7]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 4], m[jt + 8]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 4], m[jt + 8]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 4], m[jt + 9]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 4], m[jt + 9]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 5], m[jt + 0]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 5], m[jt + 0]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 5], m[jt + 1]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 5], m[jt + 1]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 5], m[jt + 2]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 5], m[jt + 2]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 5], m[jt + 3]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 5], m[jt + 3]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 5], m[jt + 4]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 5], m[jt + 4]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 5], m[jt + 5]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 5], m[jt + 5]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 5], m[jt + 6]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 5], m[jt + 6]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 5], m[jt + 7]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 5], m[jt + 7]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 5], m[jt + 8]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 5], m[jt + 8]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 5], m[jt + 9]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 5], m[jt + 9]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 6], m[jt + 0]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 6], m[jt + 0]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 6], m[jt + 1]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 6], m[jt + 1]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 6], m[jt + 2]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 6], m[jt + 2]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 6], m[jt + 3]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 6], m[jt + 3]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 6], m[jt + 4]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 6], m[jt + 4]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 6], m[jt + 5]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 6], m[jt + 5]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 6], m[jt + 6]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 6], m[jt + 6]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 6], m[jt + 7]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 6], m[jt + 7]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 6], m[jt + 8]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 6], m[jt + 8]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 6], m[jt + 9]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 6], m[jt + 9]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 7], m[jt + 0]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 7], m[jt + 0]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 7], m[jt + 1]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 7], m[jt + 1]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 7], m[jt + 2]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 7], m[jt + 2]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 7], m[jt + 3]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 7], m[jt + 3]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 7], m[jt + 4]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 7], m[jt + 4]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 7], m[jt + 5]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 7], m[jt + 5]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 7], m[jt + 6]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 7], m[jt + 6]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 7], m[jt + 7]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 7], m[jt + 7]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 7], m[jt + 8]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 7], m[jt + 8]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 7], m[jt + 9]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 7], m[jt + 9]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 8], m[jt + 0]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 8], m[jt + 0]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 8], m[jt + 1]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 8], m[jt + 1]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 8], m[jt + 2]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 8], m[jt + 2]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 8], m[jt + 3]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 8], m[jt + 3]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 8], m[jt + 4]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 8], m[jt + 4]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 8], m[jt + 5]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 8], m[jt + 5]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 8], m[jt + 6]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 8], m[jt + 6]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 8], m[jt + 7]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 8], m[jt + 7]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 8], m[jt + 8]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 8], m[jt + 8]); + res[it + jt + 17] = fma52lo(res[it + jt + 17], u[it + 8], m[jt + 9]); + res[it + jt + 18] = fma52hi(res[it + jt + 18], u[it + 8], m[jt + 9]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 9], m[jt + 0]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 9], m[jt + 0]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 9], m[jt + 1]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 9], m[jt + 1]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 9], m[jt + 2]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 9], m[jt + 2]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 9], m[jt + 3]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 9], m[jt + 3]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 9], m[jt + 4]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 9], m[jt + 4]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 9], m[jt + 5]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 9], m[jt + 5]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 9], m[jt + 6]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 9], m[jt + 6]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 9], m[jt + 7]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 9], m[jt + 7]); + res[it + jt + 17] = fma52lo(res[it + jt + 17], u[it + 9], m[jt + 8]); + res[it + jt + 18] = fma52hi(res[it + jt + 18], u[it + 9], m[jt + 8]); + res[it + jt + 18] = fma52lo(res[it + jt + 18], u[it + 9], m[jt + 9]); + res[it + jt + 19] = fma52hi(res[it + jt + 19], u[it + 9], m[jt + 9]); + } + } + + // Normalization + res[30] = add64(res[30], srli64(res[29], DIGIT_SIZE)); + r[0] = and64_const(res[30], DIGIT_MASK); + res[31] = add64(res[31], srli64(res[30], DIGIT_SIZE)); + r[1] = and64_const(res[31], DIGIT_MASK); + res[32] = add64(res[32], srli64(res[31], DIGIT_SIZE)); + r[2] = and64_const(res[32], DIGIT_MASK); + res[33] = add64(res[33], srli64(res[32], DIGIT_SIZE)); + r[3] = and64_const(res[33], DIGIT_MASK); + res[34] = add64(res[34], srli64(res[33], DIGIT_SIZE)); + r[4] = and64_const(res[34], DIGIT_MASK); + res[35] = add64(res[35], srli64(res[34], DIGIT_SIZE)); + r[5] = and64_const(res[35], DIGIT_MASK); + res[36] = add64(res[36], srli64(res[35], DIGIT_SIZE)); + r[6] = and64_const(res[36], DIGIT_MASK); + res[37] = add64(res[37], srli64(res[36], DIGIT_SIZE)); + r[7] = and64_const(res[37], DIGIT_MASK); + res[38] = add64(res[38], srli64(res[37], DIGIT_SIZE)); + r[8] = and64_const(res[38], DIGIT_MASK); + res[39] = add64(res[39], srli64(res[38], DIGIT_SIZE)); + r[9] = and64_const(res[39], DIGIT_MASK); + res[40] = add64(res[40], srli64(res[39], DIGIT_SIZE)); + r[10] = and64_const(res[40], DIGIT_MASK); + res[41] = add64(res[41], srli64(res[40], DIGIT_SIZE)); + r[11] = and64_const(res[41], DIGIT_MASK); + res[42] = add64(res[42], srli64(res[41], DIGIT_SIZE)); + r[12] = and64_const(res[42], DIGIT_MASK); + res[43] = add64(res[43], srli64(res[42], DIGIT_SIZE)); + r[13] = and64_const(res[43], DIGIT_MASK); + res[44] = add64(res[44], srli64(res[43], DIGIT_SIZE)); + r[14] = and64_const(res[44], DIGIT_MASK); + res[45] = add64(res[45], srli64(res[44], DIGIT_SIZE)); + r[15] = and64_const(res[45], DIGIT_MASK); + res[46] = add64(res[46], srli64(res[45], DIGIT_SIZE)); + r[16] = and64_const(res[46], DIGIT_MASK); + res[47] = add64(res[47], srli64(res[46], DIGIT_SIZE)); + r[17] = and64_const(res[47], DIGIT_MASK); + res[48] = add64(res[48], srli64(res[47], DIGIT_SIZE)); + r[18] = and64_const(res[48], DIGIT_MASK); + res[49] = add64(res[49], srli64(res[48], DIGIT_SIZE)); + r[19] = and64_const(res[49], DIGIT_MASK); + res[50] = add64(res[50], srli64(res[49], DIGIT_SIZE)); + r[20] = and64_const(res[50], DIGIT_MASK); + res[51] = add64(res[51], srli64(res[50], DIGIT_SIZE)); + r[21] = and64_const(res[51], DIGIT_MASK); + res[52] = add64(res[52], srli64(res[51], DIGIT_SIZE)); + r[22] = and64_const(res[52], DIGIT_MASK); + res[53] = add64(res[53], srli64(res[52], DIGIT_SIZE)); + r[23] = and64_const(res[53], DIGIT_MASK); + res[54] = add64(res[54], srli64(res[53], DIGIT_SIZE)); + r[24] = and64_const(res[54], DIGIT_MASK); + res[55] = add64(res[55], srli64(res[54], DIGIT_SIZE)); + r[25] = and64_const(res[55], DIGIT_MASK); + res[56] = add64(res[56], srli64(res[55], DIGIT_SIZE)); + r[26] = and64_const(res[56], DIGIT_MASK); + res[57] = add64(res[57], srli64(res[56], DIGIT_SIZE)); + r[27] = and64_const(res[57], DIGIT_MASK); + res[58] = add64(res[58], srli64(res[57], DIGIT_SIZE)); + r[28] = and64_const(res[58], DIGIT_MASK); + res[59] = add64(res[59], srli64(res[58], DIGIT_SIZE)); + r[29] = and64_const(res[59], DIGIT_MASK); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams52x40_diagonal_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams52x40_diagonal_mb8.c new file mode 100644 index 0000000..c75b627 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams52x40_diagonal_mb8.c @@ -0,0 +1,2271 @@ +/******************************************************************************* + * Copyright (C) 2019 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include + +#ifdef __GNUC__ +#define ASM(a) __asm__(a); +#else +#define ASM(a) +#endif + +void AMS52x40_diagonal_mb8(int64u *out_mb, const int64u *inpA_mb, + const int64u *inpM_mb, const int64u *k0_mb) { + __ALIGN64 U64 res[80]; + __ALIGN64 U64 u[40]; + U64 k; + U64 *a = (U64 *)inpA_mb; + U64 *m = (U64 *)inpM_mb; + U64 *r = (U64 *)out_mb; + + k = loadu64((U64 *)k0_mb); + int i; + for (i = 0; i < 80; ++i) + res[i] = get_zero64(); + + res[1] = fma52lo(res[1], a[0], a[1]); // Sum(1) + res[2] = fma52hi(res[2], a[0], a[1]); // Sum(1) + res[2] = fma52lo(res[2], a[0], a[2]); // Sum(2) + res[3] = fma52hi(res[3], a[0], a[2]); // Sum(2) + res[3] = fma52lo(res[3], a[1], a[2]); // Sum(3) + res[4] = fma52hi(res[4], a[1], a[2]); // Sum(3) + res[3] = fma52lo(res[3], a[0], a[3]); // Sum(3) + res[4] = fma52hi(res[4], a[0], a[3]); // Sum(3) + res[4] = fma52lo(res[4], a[1], a[3]); // Sum(4) + res[5] = fma52hi(res[5], a[1], a[3]); // Sum(4) + res[5] = fma52lo(res[5], a[2], a[3]); // Sum(5) + res[6] = fma52hi(res[6], a[2], a[3]); // Sum(5) + res[4] = fma52lo(res[4], a[0], a[4]); // Sum(4) + res[5] = fma52hi(res[5], a[0], a[4]); // Sum(4) + res[5] = fma52lo(res[5], a[1], a[4]); // Sum(5) + res[6] = fma52hi(res[6], a[1], a[4]); // Sum(5) + res[6] = fma52lo(res[6], a[2], a[4]); // Sum(6) + res[7] = fma52hi(res[7], a[2], a[4]); // Sum(6) + res[7] = fma52lo(res[7], a[3], a[4]); // Sum(7) + res[8] = fma52hi(res[8], a[3], a[4]); // Sum(7) + res[5] = fma52lo(res[5], a[0], a[5]); // Sum(5) + res[6] = fma52hi(res[6], a[0], a[5]); // Sum(5) + res[6] = fma52lo(res[6], a[1], a[5]); // Sum(6) + res[7] = fma52hi(res[7], a[1], a[5]); // Sum(6) + res[7] = fma52lo(res[7], a[2], a[5]); // Sum(7) + res[8] = fma52hi(res[8], a[2], a[5]); // Sum(7) + res[8] = fma52lo(res[8], a[3], a[5]); // Sum(8) + res[9] = fma52hi(res[9], a[3], a[5]); // Sum(8) + res[9] = fma52lo(res[9], a[4], a[5]); // Sum(9) + res[10] = fma52hi(res[10], a[4], a[5]); // Sum(9) + res[6] = fma52lo(res[6], a[0], a[6]); // Sum(6) + res[7] = fma52hi(res[7], a[0], a[6]); // Sum(6) + res[7] = fma52lo(res[7], a[1], a[6]); // Sum(7) + res[8] = fma52hi(res[8], a[1], a[6]); // Sum(7) + res[8] = fma52lo(res[8], a[2], a[6]); // Sum(8) + res[9] = fma52hi(res[9], a[2], a[6]); // Sum(8) + res[9] = fma52lo(res[9], a[3], a[6]); // Sum(9) + res[10] = fma52hi(res[10], a[3], a[6]); // Sum(9) + res[10] = fma52lo(res[10], a[4], a[6]); // Sum(10) + res[11] = fma52hi(res[11], a[4], a[6]); // Sum(10) + res[11] = fma52lo(res[11], a[5], a[6]); // Sum(11) + res[12] = fma52hi(res[12], a[5], a[6]); // Sum(11) + res[7] = fma52lo(res[7], a[0], a[7]); // Sum(7) + res[8] = fma52hi(res[8], a[0], a[7]); // Sum(7) + res[8] = fma52lo(res[8], a[1], a[7]); // Sum(8) + res[9] = fma52hi(res[9], a[1], a[7]); // Sum(8) + res[9] = fma52lo(res[9], a[2], a[7]); // Sum(9) + res[10] = fma52hi(res[10], a[2], a[7]); // Sum(9) + res[10] = fma52lo(res[10], a[3], a[7]); // Sum(10) + res[11] = fma52hi(res[11], a[3], a[7]); // Sum(10) + res[11] = fma52lo(res[11], a[4], a[7]); // Sum(11) + res[12] = fma52hi(res[12], a[4], a[7]); // Sum(11) + res[8] = fma52lo(res[8], a[0], a[8]); // Sum(8) + res[9] = fma52hi(res[9], a[0], a[8]); // Sum(8) + res[9] = fma52lo(res[9], a[1], a[8]); // Sum(9) + res[10] = fma52hi(res[10], a[1], a[8]); // Sum(9) + res[10] = fma52lo(res[10], a[2], a[8]); // Sum(10) + res[11] = fma52hi(res[11], a[2], a[8]); // Sum(10) + res[11] = fma52lo(res[11], a[3], a[8]); // Sum(11) + res[12] = fma52hi(res[12], a[3], a[8]); // Sum(11) + res[9] = fma52lo(res[9], a[0], a[9]); // Sum(9) + res[10] = fma52hi(res[10], a[0], a[9]); // Sum(9) + res[10] = fma52lo(res[10], a[1], a[9]); // Sum(10) + res[11] = fma52hi(res[11], a[1], a[9]); // Sum(10) + res[11] = fma52lo(res[11], a[2], a[9]); // Sum(11) + res[12] = fma52hi(res[12], a[2], a[9]); // Sum(11) + res[10] = fma52lo(res[10], a[0], a[10]); // Sum(10) + res[11] = fma52hi(res[11], a[0], a[10]); // Sum(10) + res[11] = fma52lo(res[11], a[1], a[10]); // Sum(11) + res[12] = fma52hi(res[12], a[1], a[10]); // Sum(11) + res[11] = fma52lo(res[11], a[0], a[11]); // Sum(11) + res[12] = fma52hi(res[12], a[0], a[11]); // Sum(11) + res[0] = add64(res[0], res[0]); // Double(0) + res[1] = add64(res[1], res[1]); // Double(1) + res[2] = add64(res[2], res[2]); // Double(2) + res[3] = add64(res[3], res[3]); // Double(3) + res[4] = add64(res[4], res[4]); // Double(4) + res[5] = add64(res[5], res[5]); // Double(5) + res[6] = add64(res[6], res[6]); // Double(6) + res[7] = add64(res[7], res[7]); // Double(7) + res[8] = add64(res[8], res[8]); // Double(8) + res[9] = add64(res[9], res[9]); // Double(9) + res[10] = add64(res[10], res[10]); // Double(10) + res[11] = add64(res[11], res[11]); // Double(11) + res[0] = fma52lo(res[0], a[0], a[0]); // Add sqr(0) + res[1] = fma52hi(res[1], a[0], a[0]); // Add sqr(0) + res[2] = fma52lo(res[2], a[1], a[1]); // Add sqr(2) + res[3] = fma52hi(res[3], a[1], a[1]); // Add sqr(2) + res[4] = fma52lo(res[4], a[2], a[2]); // Add sqr(4) + res[5] = fma52hi(res[5], a[2], a[2]); // Add sqr(4) + res[6] = fma52lo(res[6], a[3], a[3]); // Add sqr(6) + res[7] = fma52hi(res[7], a[3], a[3]); // Add sqr(6) + res[8] = fma52lo(res[8], a[4], a[4]); // Add sqr(8) + res[9] = fma52hi(res[9], a[4], a[4]); // Add sqr(8) + res[10] = fma52lo(res[10], a[5], a[5]); // Add sqr(10) + res[11] = fma52hi(res[11], a[5], a[5]); // Add sqr(10) + res[12] = fma52lo(res[12], a[5], a[7]); // Sum(12) + res[13] = fma52hi(res[13], a[5], a[7]); // Sum(12) + res[13] = fma52lo(res[13], a[6], a[7]); // Sum(13) + res[14] = fma52hi(res[14], a[6], a[7]); // Sum(13) + res[12] = fma52lo(res[12], a[4], a[8]); // Sum(12) + res[13] = fma52hi(res[13], a[4], a[8]); // Sum(12) + res[13] = fma52lo(res[13], a[5], a[8]); // Sum(13) + res[14] = fma52hi(res[14], a[5], a[8]); // Sum(13) + res[14] = fma52lo(res[14], a[6], a[8]); // Sum(14) + res[15] = fma52hi(res[15], a[6], a[8]); // Sum(14) + res[15] = fma52lo(res[15], a[7], a[8]); // Sum(15) + res[16] = fma52hi(res[16], a[7], a[8]); // Sum(15) + res[12] = fma52lo(res[12], a[3], a[9]); // Sum(12) + res[13] = fma52hi(res[13], a[3], a[9]); // Sum(12) + res[13] = fma52lo(res[13], a[4], a[9]); // Sum(13) + res[14] = fma52hi(res[14], a[4], a[9]); // Sum(13) + res[14] = fma52lo(res[14], a[5], a[9]); // Sum(14) + res[15] = fma52hi(res[15], a[5], a[9]); // Sum(14) + res[15] = fma52lo(res[15], a[6], a[9]); // Sum(15) + res[16] = fma52hi(res[16], a[6], a[9]); // Sum(15) + res[16] = fma52lo(res[16], a[7], a[9]); // Sum(16) + res[17] = fma52hi(res[17], a[7], a[9]); // Sum(16) + res[17] = fma52lo(res[17], a[8], a[9]); // Sum(17) + res[18] = fma52hi(res[18], a[8], a[9]); // Sum(17) + res[12] = fma52lo(res[12], a[2], a[10]); // Sum(12) + res[13] = fma52hi(res[13], a[2], a[10]); // Sum(12) + res[13] = fma52lo(res[13], a[3], a[10]); // Sum(13) + res[14] = fma52hi(res[14], a[3], a[10]); // Sum(13) + res[14] = fma52lo(res[14], a[4], a[10]); // Sum(14) + res[15] = fma52hi(res[15], a[4], a[10]); // Sum(14) + res[15] = fma52lo(res[15], a[5], a[10]); // Sum(15) + res[16] = fma52hi(res[16], a[5], a[10]); // Sum(15) + res[16] = fma52lo(res[16], a[6], a[10]); // Sum(16) + res[17] = fma52hi(res[17], a[6], a[10]); // Sum(16) + res[17] = fma52lo(res[17], a[7], a[10]); // Sum(17) + res[18] = fma52hi(res[18], a[7], a[10]); // Sum(17) + res[18] = fma52lo(res[18], a[8], a[10]); // Sum(18) + res[19] = fma52hi(res[19], a[8], a[10]); // Sum(18) + res[19] = fma52lo(res[19], a[9], a[10]); // Sum(19) + res[20] = fma52hi(res[20], a[9], a[10]); // Sum(19) + res[12] = fma52lo(res[12], a[1], a[11]); // Sum(12) + res[13] = fma52hi(res[13], a[1], a[11]); // Sum(12) + res[13] = fma52lo(res[13], a[2], a[11]); // Sum(13) + res[14] = fma52hi(res[14], a[2], a[11]); // Sum(13) + res[14] = fma52lo(res[14], a[3], a[11]); // Sum(14) + res[15] = fma52hi(res[15], a[3], a[11]); // Sum(14) + res[15] = fma52lo(res[15], a[4], a[11]); // Sum(15) + res[16] = fma52hi(res[16], a[4], a[11]); // Sum(15) + res[16] = fma52lo(res[16], a[5], a[11]); // Sum(16) + res[17] = fma52hi(res[17], a[5], a[11]); // Sum(16) + res[17] = fma52lo(res[17], a[6], a[11]); // Sum(17) + res[18] = fma52hi(res[18], a[6], a[11]); // Sum(17) + res[18] = fma52lo(res[18], a[7], a[11]); // Sum(18) + res[19] = fma52hi(res[19], a[7], a[11]); // Sum(18) + res[19] = fma52lo(res[19], a[8], a[11]); // Sum(19) + res[20] = fma52hi(res[20], a[8], a[11]); // Sum(19) + res[20] = fma52lo(res[20], a[9], a[11]); // Sum(20) + res[21] = fma52hi(res[21], a[9], a[11]); // Sum(20) + res[21] = fma52lo(res[21], a[10], a[11]); // Sum(21) + res[22] = fma52hi(res[22], a[10], a[11]); // Sum(21) + res[12] = fma52lo(res[12], a[0], a[12]); // Sum(12) + res[13] = fma52hi(res[13], a[0], a[12]); // Sum(12) + res[13] = fma52lo(res[13], a[1], a[12]); // Sum(13) + res[14] = fma52hi(res[14], a[1], a[12]); // Sum(13) + res[14] = fma52lo(res[14], a[2], a[12]); // Sum(14) + res[15] = fma52hi(res[15], a[2], a[12]); // Sum(14) + res[15] = fma52lo(res[15], a[3], a[12]); // Sum(15) + res[16] = fma52hi(res[16], a[3], a[12]); // Sum(15) + res[16] = fma52lo(res[16], a[4], a[12]); // Sum(16) + res[17] = fma52hi(res[17], a[4], a[12]); // Sum(16) + res[17] = fma52lo(res[17], a[5], a[12]); // Sum(17) + res[18] = fma52hi(res[18], a[5], a[12]); // Sum(17) + res[18] = fma52lo(res[18], a[6], a[12]); // Sum(18) + res[19] = fma52hi(res[19], a[6], a[12]); // Sum(18) + res[19] = fma52lo(res[19], a[7], a[12]); // Sum(19) + res[20] = fma52hi(res[20], a[7], a[12]); // Sum(19) + res[20] = fma52lo(res[20], a[8], a[12]); // Sum(20) + res[21] = fma52hi(res[21], a[8], a[12]); // Sum(20) + res[21] = fma52lo(res[21], a[9], a[12]); // Sum(21) + res[22] = fma52hi(res[22], a[9], a[12]); // Sum(21) + res[22] = fma52lo(res[22], a[10], a[12]); // Sum(22) + res[23] = fma52hi(res[23], a[10], a[12]); // Sum(22) + res[23] = fma52lo(res[23], a[11], a[12]); // Sum(23) + res[24] = fma52hi(res[24], a[11], a[12]); // Sum(23) + res[13] = fma52lo(res[13], a[0], a[13]); // Sum(13) + res[14] = fma52hi(res[14], a[0], a[13]); // Sum(13) + res[14] = fma52lo(res[14], a[1], a[13]); // Sum(14) + res[15] = fma52hi(res[15], a[1], a[13]); // Sum(14) + res[15] = fma52lo(res[15], a[2], a[13]); // Sum(15) + res[16] = fma52hi(res[16], a[2], a[13]); // Sum(15) + res[16] = fma52lo(res[16], a[3], a[13]); // Sum(16) + res[17] = fma52hi(res[17], a[3], a[13]); // Sum(16) + res[17] = fma52lo(res[17], a[4], a[13]); // Sum(17) + res[18] = fma52hi(res[18], a[4], a[13]); // Sum(17) + res[18] = fma52lo(res[18], a[5], a[13]); // Sum(18) + res[19] = fma52hi(res[19], a[5], a[13]); // Sum(18) + res[19] = fma52lo(res[19], a[6], a[13]); // Sum(19) + res[20] = fma52hi(res[20], a[6], a[13]); // Sum(19) + res[20] = fma52lo(res[20], a[7], a[13]); // Sum(20) + res[21] = fma52hi(res[21], a[7], a[13]); // Sum(20) + res[21] = fma52lo(res[21], a[8], a[13]); // Sum(21) + res[22] = fma52hi(res[22], a[8], a[13]); // Sum(21) + res[22] = fma52lo(res[22], a[9], a[13]); // Sum(22) + res[23] = fma52hi(res[23], a[9], a[13]); // Sum(22) + res[23] = fma52lo(res[23], a[10], a[13]); // Sum(23) + res[24] = fma52hi(res[24], a[10], a[13]); // Sum(23) + res[14] = fma52lo(res[14], a[0], a[14]); // Sum(14) + res[15] = fma52hi(res[15], a[0], a[14]); // Sum(14) + res[15] = fma52lo(res[15], a[1], a[14]); // Sum(15) + res[16] = fma52hi(res[16], a[1], a[14]); // Sum(15) + res[16] = fma52lo(res[16], a[2], a[14]); // Sum(16) + res[17] = fma52hi(res[17], a[2], a[14]); // Sum(16) + res[17] = fma52lo(res[17], a[3], a[14]); // Sum(17) + res[18] = fma52hi(res[18], a[3], a[14]); // Sum(17) + res[18] = fma52lo(res[18], a[4], a[14]); // Sum(18) + res[19] = fma52hi(res[19], a[4], a[14]); // Sum(18) + res[19] = fma52lo(res[19], a[5], a[14]); // Sum(19) + res[20] = fma52hi(res[20], a[5], a[14]); // Sum(19) + res[20] = fma52lo(res[20], a[6], a[14]); // Sum(20) + res[21] = fma52hi(res[21], a[6], a[14]); // Sum(20) + res[21] = fma52lo(res[21], a[7], a[14]); // Sum(21) + res[22] = fma52hi(res[22], a[7], a[14]); // Sum(21) + res[22] = fma52lo(res[22], a[8], a[14]); // Sum(22) + res[23] = fma52hi(res[23], a[8], a[14]); // Sum(22) + res[23] = fma52lo(res[23], a[9], a[14]); // Sum(23) + res[24] = fma52hi(res[24], a[9], a[14]); // Sum(23) + res[15] = fma52lo(res[15], a[0], a[15]); // Sum(15) + res[16] = fma52hi(res[16], a[0], a[15]); // Sum(15) + res[16] = fma52lo(res[16], a[1], a[15]); // Sum(16) + res[17] = fma52hi(res[17], a[1], a[15]); // Sum(16) + res[17] = fma52lo(res[17], a[2], a[15]); // Sum(17) + res[18] = fma52hi(res[18], a[2], a[15]); // Sum(17) + res[18] = fma52lo(res[18], a[3], a[15]); // Sum(18) + res[19] = fma52hi(res[19], a[3], a[15]); // Sum(18) + res[19] = fma52lo(res[19], a[4], a[15]); // Sum(19) + res[20] = fma52hi(res[20], a[4], a[15]); // Sum(19) + res[20] = fma52lo(res[20], a[5], a[15]); // Sum(20) + res[21] = fma52hi(res[21], a[5], a[15]); // Sum(20) + res[21] = fma52lo(res[21], a[6], a[15]); // Sum(21) + res[22] = fma52hi(res[22], a[6], a[15]); // Sum(21) + res[22] = fma52lo(res[22], a[7], a[15]); // Sum(22) + res[23] = fma52hi(res[23], a[7], a[15]); // Sum(22) + res[23] = fma52lo(res[23], a[8], a[15]); // Sum(23) + res[24] = fma52hi(res[24], a[8], a[15]); // Sum(23) + res[16] = fma52lo(res[16], a[0], a[16]); // Sum(16) + res[17] = fma52hi(res[17], a[0], a[16]); // Sum(16) + res[17] = fma52lo(res[17], a[1], a[16]); // Sum(17) + res[18] = fma52hi(res[18], a[1], a[16]); // Sum(17) + res[18] = fma52lo(res[18], a[2], a[16]); // Sum(18) + res[19] = fma52hi(res[19], a[2], a[16]); // Sum(18) + res[19] = fma52lo(res[19], a[3], a[16]); // Sum(19) + res[20] = fma52hi(res[20], a[3], a[16]); // Sum(19) + res[20] = fma52lo(res[20], a[4], a[16]); // Sum(20) + res[21] = fma52hi(res[21], a[4], a[16]); // Sum(20) + res[21] = fma52lo(res[21], a[5], a[16]); // Sum(21) + res[22] = fma52hi(res[22], a[5], a[16]); // Sum(21) + res[22] = fma52lo(res[22], a[6], a[16]); // Sum(22) + res[23] = fma52hi(res[23], a[6], a[16]); // Sum(22) + res[23] = fma52lo(res[23], a[7], a[16]); // Sum(23) + res[24] = fma52hi(res[24], a[7], a[16]); // Sum(23) + res[17] = fma52lo(res[17], a[0], a[17]); // Sum(17) + res[18] = fma52hi(res[18], a[0], a[17]); // Sum(17) + res[18] = fma52lo(res[18], a[1], a[17]); // Sum(18) + res[19] = fma52hi(res[19], a[1], a[17]); // Sum(18) + res[19] = fma52lo(res[19], a[2], a[17]); // Sum(19) + res[20] = fma52hi(res[20], a[2], a[17]); // Sum(19) + res[20] = fma52lo(res[20], a[3], a[17]); // Sum(20) + res[21] = fma52hi(res[21], a[3], a[17]); // Sum(20) + res[21] = fma52lo(res[21], a[4], a[17]); // Sum(21) + res[22] = fma52hi(res[22], a[4], a[17]); // Sum(21) + res[22] = fma52lo(res[22], a[5], a[17]); // Sum(22) + res[23] = fma52hi(res[23], a[5], a[17]); // Sum(22) + res[23] = fma52lo(res[23], a[6], a[17]); // Sum(23) + res[24] = fma52hi(res[24], a[6], a[17]); // Sum(23) + res[18] = fma52lo(res[18], a[0], a[18]); // Sum(18) + res[19] = fma52hi(res[19], a[0], a[18]); // Sum(18) + res[19] = fma52lo(res[19], a[1], a[18]); // Sum(19) + res[20] = fma52hi(res[20], a[1], a[18]); // Sum(19) + res[20] = fma52lo(res[20], a[2], a[18]); // Sum(20) + res[21] = fma52hi(res[21], a[2], a[18]); // Sum(20) + res[21] = fma52lo(res[21], a[3], a[18]); // Sum(21) + res[22] = fma52hi(res[22], a[3], a[18]); // Sum(21) + res[22] = fma52lo(res[22], a[4], a[18]); // Sum(22) + res[23] = fma52hi(res[23], a[4], a[18]); // Sum(22) + res[23] = fma52lo(res[23], a[5], a[18]); // Sum(23) + res[24] = fma52hi(res[24], a[5], a[18]); // Sum(23) + res[19] = fma52lo(res[19], a[0], a[19]); // Sum(19) + res[20] = fma52hi(res[20], a[0], a[19]); // Sum(19) + res[20] = fma52lo(res[20], a[1], a[19]); // Sum(20) + res[21] = fma52hi(res[21], a[1], a[19]); // Sum(20) + res[21] = fma52lo(res[21], a[2], a[19]); // Sum(21) + res[22] = fma52hi(res[22], a[2], a[19]); // Sum(21) + res[22] = fma52lo(res[22], a[3], a[19]); // Sum(22) + res[23] = fma52hi(res[23], a[3], a[19]); // Sum(22) + res[23] = fma52lo(res[23], a[4], a[19]); // Sum(23) + res[24] = fma52hi(res[24], a[4], a[19]); // Sum(23) + res[20] = fma52lo(res[20], a[0], a[20]); // Sum(20) + res[21] = fma52hi(res[21], a[0], a[20]); // Sum(20) + res[21] = fma52lo(res[21], a[1], a[20]); // Sum(21) + res[22] = fma52hi(res[22], a[1], a[20]); // Sum(21) + res[22] = fma52lo(res[22], a[2], a[20]); // Sum(22) + res[23] = fma52hi(res[23], a[2], a[20]); // Sum(22) + res[23] = fma52lo(res[23], a[3], a[20]); // Sum(23) + res[24] = fma52hi(res[24], a[3], a[20]); // Sum(23) + res[21] = fma52lo(res[21], a[0], a[21]); // Sum(21) + res[22] = fma52hi(res[22], a[0], a[21]); // Sum(21) + res[22] = fma52lo(res[22], a[1], a[21]); // Sum(22) + res[23] = fma52hi(res[23], a[1], a[21]); // Sum(22) + res[23] = fma52lo(res[23], a[2], a[21]); // Sum(23) + res[24] = fma52hi(res[24], a[2], a[21]); // Sum(23) + res[22] = fma52lo(res[22], a[0], a[22]); // Sum(22) + res[23] = fma52hi(res[23], a[0], a[22]); // Sum(22) + res[23] = fma52lo(res[23], a[1], a[22]); // Sum(23) + res[24] = fma52hi(res[24], a[1], a[22]); // Sum(23) + res[23] = fma52lo(res[23], a[0], a[23]); // Sum(23) + res[24] = fma52hi(res[24], a[0], a[23]); // Sum(23) + res[12] = add64(res[12], res[12]); // Double(12) + res[13] = add64(res[13], res[13]); // Double(13) + res[14] = add64(res[14], res[14]); // Double(14) + res[15] = add64(res[15], res[15]); // Double(15) + res[16] = add64(res[16], res[16]); // Double(16) + res[17] = add64(res[17], res[17]); // Double(17) + res[18] = add64(res[18], res[18]); // Double(18) + res[19] = add64(res[19], res[19]); // Double(19) + res[20] = add64(res[20], res[20]); // Double(20) + res[21] = add64(res[21], res[21]); // Double(21) + res[22] = add64(res[22], res[22]); // Double(22) + res[23] = add64(res[23], res[23]); // Double(23) + res[12] = fma52lo(res[12], a[6], a[6]); // Add sqr(12) + res[13] = fma52hi(res[13], a[6], a[6]); // Add sqr(12) + res[14] = fma52lo(res[14], a[7], a[7]); // Add sqr(14) + res[15] = fma52hi(res[15], a[7], a[7]); // Add sqr(14) + res[16] = fma52lo(res[16], a[8], a[8]); // Add sqr(16) + res[17] = fma52hi(res[17], a[8], a[8]); // Add sqr(16) + res[18] = fma52lo(res[18], a[9], a[9]); // Add sqr(18) + res[19] = fma52hi(res[19], a[9], a[9]); // Add sqr(18) + res[20] = fma52lo(res[20], a[10], a[10]); // Add sqr(20) + res[21] = fma52hi(res[21], a[10], a[10]); // Add sqr(20) + res[22] = fma52lo(res[22], a[11], a[11]); // Add sqr(22) + res[23] = fma52hi(res[23], a[11], a[11]); // Add sqr(22) + res[24] = fma52lo(res[24], a[11], a[13]); // Sum(24) + res[25] = fma52hi(res[25], a[11], a[13]); // Sum(24) + res[25] = fma52lo(res[25], a[12], a[13]); // Sum(25) + res[26] = fma52hi(res[26], a[12], a[13]); // Sum(25) + res[24] = fma52lo(res[24], a[10], a[14]); // Sum(24) + res[25] = fma52hi(res[25], a[10], a[14]); // Sum(24) + res[25] = fma52lo(res[25], a[11], a[14]); // Sum(25) + res[26] = fma52hi(res[26], a[11], a[14]); // Sum(25) + res[26] = fma52lo(res[26], a[12], a[14]); // Sum(26) + res[27] = fma52hi(res[27], a[12], a[14]); // Sum(26) + res[27] = fma52lo(res[27], a[13], a[14]); // Sum(27) + res[28] = fma52hi(res[28], a[13], a[14]); // Sum(27) + res[24] = fma52lo(res[24], a[9], a[15]); // Sum(24) + res[25] = fma52hi(res[25], a[9], a[15]); // Sum(24) + res[25] = fma52lo(res[25], a[10], a[15]); // Sum(25) + res[26] = fma52hi(res[26], a[10], a[15]); // Sum(25) + res[26] = fma52lo(res[26], a[11], a[15]); // Sum(26) + res[27] = fma52hi(res[27], a[11], a[15]); // Sum(26) + res[27] = fma52lo(res[27], a[12], a[15]); // Sum(27) + res[28] = fma52hi(res[28], a[12], a[15]); // Sum(27) + res[28] = fma52lo(res[28], a[13], a[15]); // Sum(28) + res[29] = fma52hi(res[29], a[13], a[15]); // Sum(28) + res[29] = fma52lo(res[29], a[14], a[15]); // Sum(29) + res[30] = fma52hi(res[30], a[14], a[15]); // Sum(29) + res[24] = fma52lo(res[24], a[8], a[16]); // Sum(24) + res[25] = fma52hi(res[25], a[8], a[16]); // Sum(24) + res[25] = fma52lo(res[25], a[9], a[16]); // Sum(25) + res[26] = fma52hi(res[26], a[9], a[16]); // Sum(25) + res[26] = fma52lo(res[26], a[10], a[16]); // Sum(26) + res[27] = fma52hi(res[27], a[10], a[16]); // Sum(26) + res[27] = fma52lo(res[27], a[11], a[16]); // Sum(27) + res[28] = fma52hi(res[28], a[11], a[16]); // Sum(27) + res[28] = fma52lo(res[28], a[12], a[16]); // Sum(28) + res[29] = fma52hi(res[29], a[12], a[16]); // Sum(28) + res[29] = fma52lo(res[29], a[13], a[16]); // Sum(29) + res[30] = fma52hi(res[30], a[13], a[16]); // Sum(29) + res[30] = fma52lo(res[30], a[14], a[16]); // Sum(30) + res[31] = fma52hi(res[31], a[14], a[16]); // Sum(30) + res[31] = fma52lo(res[31], a[15], a[16]); // Sum(31) + res[32] = fma52hi(res[32], a[15], a[16]); // Sum(31) + res[24] = fma52lo(res[24], a[7], a[17]); // Sum(24) + res[25] = fma52hi(res[25], a[7], a[17]); // Sum(24) + res[25] = fma52lo(res[25], a[8], a[17]); // Sum(25) + res[26] = fma52hi(res[26], a[8], a[17]); // Sum(25) + res[26] = fma52lo(res[26], a[9], a[17]); // Sum(26) + res[27] = fma52hi(res[27], a[9], a[17]); // Sum(26) + res[27] = fma52lo(res[27], a[10], a[17]); // Sum(27) + res[28] = fma52hi(res[28], a[10], a[17]); // Sum(27) + res[28] = fma52lo(res[28], a[11], a[17]); // Sum(28) + res[29] = fma52hi(res[29], a[11], a[17]); // Sum(28) + res[29] = fma52lo(res[29], a[12], a[17]); // Sum(29) + res[30] = fma52hi(res[30], a[12], a[17]); // Sum(29) + res[30] = fma52lo(res[30], a[13], a[17]); // Sum(30) + res[31] = fma52hi(res[31], a[13], a[17]); // Sum(30) + res[31] = fma52lo(res[31], a[14], a[17]); // Sum(31) + res[32] = fma52hi(res[32], a[14], a[17]); // Sum(31) + res[32] = fma52lo(res[32], a[15], a[17]); // Sum(32) + res[33] = fma52hi(res[33], a[15], a[17]); // Sum(32) + res[33] = fma52lo(res[33], a[16], a[17]); // Sum(33) + res[34] = fma52hi(res[34], a[16], a[17]); // Sum(33) + res[24] = fma52lo(res[24], a[6], a[18]); // Sum(24) + res[25] = fma52hi(res[25], a[6], a[18]); // Sum(24) + res[25] = fma52lo(res[25], a[7], a[18]); // Sum(25) + res[26] = fma52hi(res[26], a[7], a[18]); // Sum(25) + res[26] = fma52lo(res[26], a[8], a[18]); // Sum(26) + res[27] = fma52hi(res[27], a[8], a[18]); // Sum(26) + res[27] = fma52lo(res[27], a[9], a[18]); // Sum(27) + res[28] = fma52hi(res[28], a[9], a[18]); // Sum(27) + res[28] = fma52lo(res[28], a[10], a[18]); // Sum(28) + res[29] = fma52hi(res[29], a[10], a[18]); // Sum(28) + res[29] = fma52lo(res[29], a[11], a[18]); // Sum(29) + res[30] = fma52hi(res[30], a[11], a[18]); // Sum(29) + res[30] = fma52lo(res[30], a[12], a[18]); // Sum(30) + res[31] = fma52hi(res[31], a[12], a[18]); // Sum(30) + res[31] = fma52lo(res[31], a[13], a[18]); // Sum(31) + res[32] = fma52hi(res[32], a[13], a[18]); // Sum(31) + res[32] = fma52lo(res[32], a[14], a[18]); // Sum(32) + res[33] = fma52hi(res[33], a[14], a[18]); // Sum(32) + res[33] = fma52lo(res[33], a[15], a[18]); // Sum(33) + res[34] = fma52hi(res[34], a[15], a[18]); // Sum(33) + res[34] = fma52lo(res[34], a[16], a[18]); // Sum(34) + res[35] = fma52hi(res[35], a[16], a[18]); // Sum(34) + res[35] = fma52lo(res[35], a[17], a[18]); // Sum(35) + res[36] = fma52hi(res[36], a[17], a[18]); // Sum(35) + res[24] = fma52lo(res[24], a[5], a[19]); // Sum(24) + res[25] = fma52hi(res[25], a[5], a[19]); // Sum(24) + res[25] = fma52lo(res[25], a[6], a[19]); // Sum(25) + res[26] = fma52hi(res[26], a[6], a[19]); // Sum(25) + res[26] = fma52lo(res[26], a[7], a[19]); // Sum(26) + res[27] = fma52hi(res[27], a[7], a[19]); // Sum(26) + res[27] = fma52lo(res[27], a[8], a[19]); // Sum(27) + res[28] = fma52hi(res[28], a[8], a[19]); // Sum(27) + res[28] = fma52lo(res[28], a[9], a[19]); // Sum(28) + res[29] = fma52hi(res[29], a[9], a[19]); // Sum(28) + res[29] = fma52lo(res[29], a[10], a[19]); // Sum(29) + res[30] = fma52hi(res[30], a[10], a[19]); // Sum(29) + res[30] = fma52lo(res[30], a[11], a[19]); // Sum(30) + res[31] = fma52hi(res[31], a[11], a[19]); // Sum(30) + res[31] = fma52lo(res[31], a[12], a[19]); // Sum(31) + res[32] = fma52hi(res[32], a[12], a[19]); // Sum(31) + res[32] = fma52lo(res[32], a[13], a[19]); // Sum(32) + res[33] = fma52hi(res[33], a[13], a[19]); // Sum(32) + res[33] = fma52lo(res[33], a[14], a[19]); // Sum(33) + res[34] = fma52hi(res[34], a[14], a[19]); // Sum(33) + res[34] = fma52lo(res[34], a[15], a[19]); // Sum(34) + res[35] = fma52hi(res[35], a[15], a[19]); // Sum(34) + res[35] = fma52lo(res[35], a[16], a[19]); // Sum(35) + res[36] = fma52hi(res[36], a[16], a[19]); // Sum(35) + res[24] = fma52lo(res[24], a[4], a[20]); // Sum(24) + res[25] = fma52hi(res[25], a[4], a[20]); // Sum(24) + res[25] = fma52lo(res[25], a[5], a[20]); // Sum(25) + res[26] = fma52hi(res[26], a[5], a[20]); // Sum(25) + res[26] = fma52lo(res[26], a[6], a[20]); // Sum(26) + res[27] = fma52hi(res[27], a[6], a[20]); // Sum(26) + res[27] = fma52lo(res[27], a[7], a[20]); // Sum(27) + res[28] = fma52hi(res[28], a[7], a[20]); // Sum(27) + res[28] = fma52lo(res[28], a[8], a[20]); // Sum(28) + res[29] = fma52hi(res[29], a[8], a[20]); // Sum(28) + res[29] = fma52lo(res[29], a[9], a[20]); // Sum(29) + res[30] = fma52hi(res[30], a[9], a[20]); // Sum(29) + res[30] = fma52lo(res[30], a[10], a[20]); // Sum(30) + res[31] = fma52hi(res[31], a[10], a[20]); // Sum(30) + res[31] = fma52lo(res[31], a[11], a[20]); // Sum(31) + res[32] = fma52hi(res[32], a[11], a[20]); // Sum(31) + res[32] = fma52lo(res[32], a[12], a[20]); // Sum(32) + res[33] = fma52hi(res[33], a[12], a[20]); // Sum(32) + res[33] = fma52lo(res[33], a[13], a[20]); // Sum(33) + res[34] = fma52hi(res[34], a[13], a[20]); // Sum(33) + res[34] = fma52lo(res[34], a[14], a[20]); // Sum(34) + res[35] = fma52hi(res[35], a[14], a[20]); // Sum(34) + res[35] = fma52lo(res[35], a[15], a[20]); // Sum(35) + res[36] = fma52hi(res[36], a[15], a[20]); // Sum(35) + res[24] = fma52lo(res[24], a[3], a[21]); // Sum(24) + res[25] = fma52hi(res[25], a[3], a[21]); // Sum(24) + res[25] = fma52lo(res[25], a[4], a[21]); // Sum(25) + res[26] = fma52hi(res[26], a[4], a[21]); // Sum(25) + res[26] = fma52lo(res[26], a[5], a[21]); // Sum(26) + res[27] = fma52hi(res[27], a[5], a[21]); // Sum(26) + res[27] = fma52lo(res[27], a[6], a[21]); // Sum(27) + res[28] = fma52hi(res[28], a[6], a[21]); // Sum(27) + res[28] = fma52lo(res[28], a[7], a[21]); // Sum(28) + res[29] = fma52hi(res[29], a[7], a[21]); // Sum(28) + res[29] = fma52lo(res[29], a[8], a[21]); // Sum(29) + res[30] = fma52hi(res[30], a[8], a[21]); // Sum(29) + res[30] = fma52lo(res[30], a[9], a[21]); // Sum(30) + res[31] = fma52hi(res[31], a[9], a[21]); // Sum(30) + res[31] = fma52lo(res[31], a[10], a[21]); // Sum(31) + res[32] = fma52hi(res[32], a[10], a[21]); // Sum(31) + res[32] = fma52lo(res[32], a[11], a[21]); // Sum(32) + res[33] = fma52hi(res[33], a[11], a[21]); // Sum(32) + res[33] = fma52lo(res[33], a[12], a[21]); // Sum(33) + res[34] = fma52hi(res[34], a[12], a[21]); // Sum(33) + res[34] = fma52lo(res[34], a[13], a[21]); // Sum(34) + res[35] = fma52hi(res[35], a[13], a[21]); // Sum(34) + res[35] = fma52lo(res[35], a[14], a[21]); // Sum(35) + res[36] = fma52hi(res[36], a[14], a[21]); // Sum(35) + res[24] = fma52lo(res[24], a[2], a[22]); // Sum(24) + res[25] = fma52hi(res[25], a[2], a[22]); // Sum(24) + res[25] = fma52lo(res[25], a[3], a[22]); // Sum(25) + res[26] = fma52hi(res[26], a[3], a[22]); // Sum(25) + res[26] = fma52lo(res[26], a[4], a[22]); // Sum(26) + res[27] = fma52hi(res[27], a[4], a[22]); // Sum(26) + res[27] = fma52lo(res[27], a[5], a[22]); // Sum(27) + res[28] = fma52hi(res[28], a[5], a[22]); // Sum(27) + res[28] = fma52lo(res[28], a[6], a[22]); // Sum(28) + res[29] = fma52hi(res[29], a[6], a[22]); // Sum(28) + res[29] = fma52lo(res[29], a[7], a[22]); // Sum(29) + res[30] = fma52hi(res[30], a[7], a[22]); // Sum(29) + res[30] = fma52lo(res[30], a[8], a[22]); // Sum(30) + res[31] = fma52hi(res[31], a[8], a[22]); // Sum(30) + res[31] = fma52lo(res[31], a[9], a[22]); // Sum(31) + res[32] = fma52hi(res[32], a[9], a[22]); // Sum(31) + res[32] = fma52lo(res[32], a[10], a[22]); // Sum(32) + res[33] = fma52hi(res[33], a[10], a[22]); // Sum(32) + res[33] = fma52lo(res[33], a[11], a[22]); // Sum(33) + res[34] = fma52hi(res[34], a[11], a[22]); // Sum(33) + res[34] = fma52lo(res[34], a[12], a[22]); // Sum(34) + res[35] = fma52hi(res[35], a[12], a[22]); // Sum(34) + res[35] = fma52lo(res[35], a[13], a[22]); // Sum(35) + res[36] = fma52hi(res[36], a[13], a[22]); // Sum(35) + res[24] = fma52lo(res[24], a[1], a[23]); // Sum(24) + res[25] = fma52hi(res[25], a[1], a[23]); // Sum(24) + res[25] = fma52lo(res[25], a[2], a[23]); // Sum(25) + res[26] = fma52hi(res[26], a[2], a[23]); // Sum(25) + res[26] = fma52lo(res[26], a[3], a[23]); // Sum(26) + res[27] = fma52hi(res[27], a[3], a[23]); // Sum(26) + res[27] = fma52lo(res[27], a[4], a[23]); // Sum(27) + res[28] = fma52hi(res[28], a[4], a[23]); // Sum(27) + res[28] = fma52lo(res[28], a[5], a[23]); // Sum(28) + res[29] = fma52hi(res[29], a[5], a[23]); // Sum(28) + res[29] = fma52lo(res[29], a[6], a[23]); // Sum(29) + res[30] = fma52hi(res[30], a[6], a[23]); // Sum(29) + res[30] = fma52lo(res[30], a[7], a[23]); // Sum(30) + res[31] = fma52hi(res[31], a[7], a[23]); // Sum(30) + res[31] = fma52lo(res[31], a[8], a[23]); // Sum(31) + res[32] = fma52hi(res[32], a[8], a[23]); // Sum(31) + res[32] = fma52lo(res[32], a[9], a[23]); // Sum(32) + res[33] = fma52hi(res[33], a[9], a[23]); // Sum(32) + res[33] = fma52lo(res[33], a[10], a[23]); // Sum(33) + res[34] = fma52hi(res[34], a[10], a[23]); // Sum(33) + res[34] = fma52lo(res[34], a[11], a[23]); // Sum(34) + res[35] = fma52hi(res[35], a[11], a[23]); // Sum(34) + res[35] = fma52lo(res[35], a[12], a[23]); // Sum(35) + res[36] = fma52hi(res[36], a[12], a[23]); // Sum(35) + res[24] = fma52lo(res[24], a[0], a[24]); // Sum(24) + res[25] = fma52hi(res[25], a[0], a[24]); // Sum(24) + res[25] = fma52lo(res[25], a[1], a[24]); // Sum(25) + res[26] = fma52hi(res[26], a[1], a[24]); // Sum(25) + res[26] = fma52lo(res[26], a[2], a[24]); // Sum(26) + res[27] = fma52hi(res[27], a[2], a[24]); // Sum(26) + res[27] = fma52lo(res[27], a[3], a[24]); // Sum(27) + res[28] = fma52hi(res[28], a[3], a[24]); // Sum(27) + res[28] = fma52lo(res[28], a[4], a[24]); // Sum(28) + res[29] = fma52hi(res[29], a[4], a[24]); // Sum(28) + res[29] = fma52lo(res[29], a[5], a[24]); // Sum(29) + res[30] = fma52hi(res[30], a[5], a[24]); // Sum(29) + res[30] = fma52lo(res[30], a[6], a[24]); // Sum(30) + res[31] = fma52hi(res[31], a[6], a[24]); // Sum(30) + res[31] = fma52lo(res[31], a[7], a[24]); // Sum(31) + res[32] = fma52hi(res[32], a[7], a[24]); // Sum(31) + res[32] = fma52lo(res[32], a[8], a[24]); // Sum(32) + res[33] = fma52hi(res[33], a[8], a[24]); // Sum(32) + res[33] = fma52lo(res[33], a[9], a[24]); // Sum(33) + res[34] = fma52hi(res[34], a[9], a[24]); // Sum(33) + res[34] = fma52lo(res[34], a[10], a[24]); // Sum(34) + res[35] = fma52hi(res[35], a[10], a[24]); // Sum(34) + res[35] = fma52lo(res[35], a[11], a[24]); // Sum(35) + res[36] = fma52hi(res[36], a[11], a[24]); // Sum(35) + res[25] = fma52lo(res[25], a[0], a[25]); // Sum(25) + res[26] = fma52hi(res[26], a[0], a[25]); // Sum(25) + res[26] = fma52lo(res[26], a[1], a[25]); // Sum(26) + res[27] = fma52hi(res[27], a[1], a[25]); // Sum(26) + res[27] = fma52lo(res[27], a[2], a[25]); // Sum(27) + res[28] = fma52hi(res[28], a[2], a[25]); // Sum(27) + res[28] = fma52lo(res[28], a[3], a[25]); // Sum(28) + res[29] = fma52hi(res[29], a[3], a[25]); // Sum(28) + res[29] = fma52lo(res[29], a[4], a[25]); // Sum(29) + res[30] = fma52hi(res[30], a[4], a[25]); // Sum(29) + res[30] = fma52lo(res[30], a[5], a[25]); // Sum(30) + res[31] = fma52hi(res[31], a[5], a[25]); // Sum(30) + res[31] = fma52lo(res[31], a[6], a[25]); // Sum(31) + res[32] = fma52hi(res[32], a[6], a[25]); // Sum(31) + res[32] = fma52lo(res[32], a[7], a[25]); // Sum(32) + res[33] = fma52hi(res[33], a[7], a[25]); // Sum(32) + res[33] = fma52lo(res[33], a[8], a[25]); // Sum(33) + res[34] = fma52hi(res[34], a[8], a[25]); // Sum(33) + res[34] = fma52lo(res[34], a[9], a[25]); // Sum(34) + res[35] = fma52hi(res[35], a[9], a[25]); // Sum(34) + res[35] = fma52lo(res[35], a[10], a[25]); // Sum(35) + res[36] = fma52hi(res[36], a[10], a[25]); // Sum(35) + res[26] = fma52lo(res[26], a[0], a[26]); // Sum(26) + res[27] = fma52hi(res[27], a[0], a[26]); // Sum(26) + res[27] = fma52lo(res[27], a[1], a[26]); // Sum(27) + res[28] = fma52hi(res[28], a[1], a[26]); // Sum(27) + res[28] = fma52lo(res[28], a[2], a[26]); // Sum(28) + res[29] = fma52hi(res[29], a[2], a[26]); // Sum(28) + res[29] = fma52lo(res[29], a[3], a[26]); // Sum(29) + res[30] = fma52hi(res[30], a[3], a[26]); // Sum(29) + res[30] = fma52lo(res[30], a[4], a[26]); // Sum(30) + res[31] = fma52hi(res[31], a[4], a[26]); // Sum(30) + res[31] = fma52lo(res[31], a[5], a[26]); // Sum(31) + res[32] = fma52hi(res[32], a[5], a[26]); // Sum(31) + res[32] = fma52lo(res[32], a[6], a[26]); // Sum(32) + res[33] = fma52hi(res[33], a[6], a[26]); // Sum(32) + res[33] = fma52lo(res[33], a[7], a[26]); // Sum(33) + res[34] = fma52hi(res[34], a[7], a[26]); // Sum(33) + res[34] = fma52lo(res[34], a[8], a[26]); // Sum(34) + res[35] = fma52hi(res[35], a[8], a[26]); // Sum(34) + res[35] = fma52lo(res[35], a[9], a[26]); // Sum(35) + res[36] = fma52hi(res[36], a[9], a[26]); // Sum(35) + res[27] = fma52lo(res[27], a[0], a[27]); // Sum(27) + res[28] = fma52hi(res[28], a[0], a[27]); // Sum(27) + res[28] = fma52lo(res[28], a[1], a[27]); // Sum(28) + res[29] = fma52hi(res[29], a[1], a[27]); // Sum(28) + res[29] = fma52lo(res[29], a[2], a[27]); // Sum(29) + res[30] = fma52hi(res[30], a[2], a[27]); // Sum(29) + res[30] = fma52lo(res[30], a[3], a[27]); // Sum(30) + res[31] = fma52hi(res[31], a[3], a[27]); // Sum(30) + res[31] = fma52lo(res[31], a[4], a[27]); // Sum(31) + res[32] = fma52hi(res[32], a[4], a[27]); // Sum(31) + res[32] = fma52lo(res[32], a[5], a[27]); // Sum(32) + res[33] = fma52hi(res[33], a[5], a[27]); // Sum(32) + res[33] = fma52lo(res[33], a[6], a[27]); // Sum(33) + res[34] = fma52hi(res[34], a[6], a[27]); // Sum(33) + res[34] = fma52lo(res[34], a[7], a[27]); // Sum(34) + res[35] = fma52hi(res[35], a[7], a[27]); // Sum(34) + res[35] = fma52lo(res[35], a[8], a[27]); // Sum(35) + res[36] = fma52hi(res[36], a[8], a[27]); // Sum(35) + res[28] = fma52lo(res[28], a[0], a[28]); // Sum(28) + res[29] = fma52hi(res[29], a[0], a[28]); // Sum(28) + res[29] = fma52lo(res[29], a[1], a[28]); // Sum(29) + res[30] = fma52hi(res[30], a[1], a[28]); // Sum(29) + res[30] = fma52lo(res[30], a[2], a[28]); // Sum(30) + res[31] = fma52hi(res[31], a[2], a[28]); // Sum(30) + res[31] = fma52lo(res[31], a[3], a[28]); // Sum(31) + res[32] = fma52hi(res[32], a[3], a[28]); // Sum(31) + res[32] = fma52lo(res[32], a[4], a[28]); // Sum(32) + res[33] = fma52hi(res[33], a[4], a[28]); // Sum(32) + res[33] = fma52lo(res[33], a[5], a[28]); // Sum(33) + res[34] = fma52hi(res[34], a[5], a[28]); // Sum(33) + res[34] = fma52lo(res[34], a[6], a[28]); // Sum(34) + res[35] = fma52hi(res[35], a[6], a[28]); // Sum(34) + res[35] = fma52lo(res[35], a[7], a[28]); // Sum(35) + res[36] = fma52hi(res[36], a[7], a[28]); // Sum(35) + res[29] = fma52lo(res[29], a[0], a[29]); // Sum(29) + res[30] = fma52hi(res[30], a[0], a[29]); // Sum(29) + res[30] = fma52lo(res[30], a[1], a[29]); // Sum(30) + res[31] = fma52hi(res[31], a[1], a[29]); // Sum(30) + res[31] = fma52lo(res[31], a[2], a[29]); // Sum(31) + res[32] = fma52hi(res[32], a[2], a[29]); // Sum(31) + res[32] = fma52lo(res[32], a[3], a[29]); // Sum(32) + res[33] = fma52hi(res[33], a[3], a[29]); // Sum(32) + res[33] = fma52lo(res[33], a[4], a[29]); // Sum(33) + res[34] = fma52hi(res[34], a[4], a[29]); // Sum(33) + res[34] = fma52lo(res[34], a[5], a[29]); // Sum(34) + res[35] = fma52hi(res[35], a[5], a[29]); // Sum(34) + res[35] = fma52lo(res[35], a[6], a[29]); // Sum(35) + res[36] = fma52hi(res[36], a[6], a[29]); // Sum(35) + res[30] = fma52lo(res[30], a[0], a[30]); // Sum(30) + res[31] = fma52hi(res[31], a[0], a[30]); // Sum(30) + res[31] = fma52lo(res[31], a[1], a[30]); // Sum(31) + res[32] = fma52hi(res[32], a[1], a[30]); // Sum(31) + res[32] = fma52lo(res[32], a[2], a[30]); // Sum(32) + res[33] = fma52hi(res[33], a[2], a[30]); // Sum(32) + res[33] = fma52lo(res[33], a[3], a[30]); // Sum(33) + res[34] = fma52hi(res[34], a[3], a[30]); // Sum(33) + res[34] = fma52lo(res[34], a[4], a[30]); // Sum(34) + res[35] = fma52hi(res[35], a[4], a[30]); // Sum(34) + res[35] = fma52lo(res[35], a[5], a[30]); // Sum(35) + res[36] = fma52hi(res[36], a[5], a[30]); // Sum(35) + res[31] = fma52lo(res[31], a[0], a[31]); // Sum(31) + res[32] = fma52hi(res[32], a[0], a[31]); // Sum(31) + res[32] = fma52lo(res[32], a[1], a[31]); // Sum(32) + res[33] = fma52hi(res[33], a[1], a[31]); // Sum(32) + res[33] = fma52lo(res[33], a[2], a[31]); // Sum(33) + res[34] = fma52hi(res[34], a[2], a[31]); // Sum(33) + res[34] = fma52lo(res[34], a[3], a[31]); // Sum(34) + res[35] = fma52hi(res[35], a[3], a[31]); // Sum(34) + res[35] = fma52lo(res[35], a[4], a[31]); // Sum(35) + res[36] = fma52hi(res[36], a[4], a[31]); // Sum(35) + res[32] = fma52lo(res[32], a[0], a[32]); // Sum(32) + res[33] = fma52hi(res[33], a[0], a[32]); // Sum(32) + res[33] = fma52lo(res[33], a[1], a[32]); // Sum(33) + res[34] = fma52hi(res[34], a[1], a[32]); // Sum(33) + res[34] = fma52lo(res[34], a[2], a[32]); // Sum(34) + res[35] = fma52hi(res[35], a[2], a[32]); // Sum(34) + res[35] = fma52lo(res[35], a[3], a[32]); // Sum(35) + res[36] = fma52hi(res[36], a[3], a[32]); // Sum(35) + res[33] = fma52lo(res[33], a[0], a[33]); // Sum(33) + res[34] = fma52hi(res[34], a[0], a[33]); // Sum(33) + res[34] = fma52lo(res[34], a[1], a[33]); // Sum(34) + res[35] = fma52hi(res[35], a[1], a[33]); // Sum(34) + res[35] = fma52lo(res[35], a[2], a[33]); // Sum(35) + res[36] = fma52hi(res[36], a[2], a[33]); // Sum(35) + res[34] = fma52lo(res[34], a[0], a[34]); // Sum(34) + res[35] = fma52hi(res[35], a[0], a[34]); // Sum(34) + res[35] = fma52lo(res[35], a[1], a[34]); // Sum(35) + res[36] = fma52hi(res[36], a[1], a[34]); // Sum(35) + res[35] = fma52lo(res[35], a[0], a[35]); // Sum(35) + res[36] = fma52hi(res[36], a[0], a[35]); // Sum(35) + res[24] = add64(res[24], res[24]); // Double(24) + res[25] = add64(res[25], res[25]); // Double(25) + res[26] = add64(res[26], res[26]); // Double(26) + res[27] = add64(res[27], res[27]); // Double(27) + res[28] = add64(res[28], res[28]); // Double(28) + res[29] = add64(res[29], res[29]); // Double(29) + res[30] = add64(res[30], res[30]); // Double(30) + res[31] = add64(res[31], res[31]); // Double(31) + res[32] = add64(res[32], res[32]); // Double(32) + res[33] = add64(res[33], res[33]); // Double(33) + res[34] = add64(res[34], res[34]); // Double(34) + res[35] = add64(res[35], res[35]); // Double(35) + res[24] = fma52lo(res[24], a[12], a[12]); // Add sqr(24) + res[25] = fma52hi(res[25], a[12], a[12]); // Add sqr(24) + res[26] = fma52lo(res[26], a[13], a[13]); // Add sqr(26) + res[27] = fma52hi(res[27], a[13], a[13]); // Add sqr(26) + res[28] = fma52lo(res[28], a[14], a[14]); // Add sqr(28) + res[29] = fma52hi(res[29], a[14], a[14]); // Add sqr(28) + res[30] = fma52lo(res[30], a[15], a[15]); // Add sqr(30) + res[31] = fma52hi(res[31], a[15], a[15]); // Add sqr(30) + res[32] = fma52lo(res[32], a[16], a[16]); // Add sqr(32) + res[33] = fma52hi(res[33], a[16], a[16]); // Add sqr(32) + res[34] = fma52lo(res[34], a[17], a[17]); // Add sqr(34) + res[35] = fma52hi(res[35], a[17], a[17]); // Add sqr(34) + res[36] = fma52lo(res[36], a[17], a[19]); // Sum(36) + res[37] = fma52hi(res[37], a[17], a[19]); // Sum(36) + res[37] = fma52lo(res[37], a[18], a[19]); // Sum(37) + res[38] = fma52hi(res[38], a[18], a[19]); // Sum(37) + res[36] = fma52lo(res[36], a[16], a[20]); // Sum(36) + res[37] = fma52hi(res[37], a[16], a[20]); // Sum(36) + res[37] = fma52lo(res[37], a[17], a[20]); // Sum(37) + res[38] = fma52hi(res[38], a[17], a[20]); // Sum(37) + res[38] = fma52lo(res[38], a[18], a[20]); // Sum(38) + res[39] = fma52hi(res[39], a[18], a[20]); // Sum(38) + res[39] = fma52lo(res[39], a[19], a[20]); // Sum(39) + res[40] = fma52hi(res[40], a[19], a[20]); // Sum(39) + res[36] = fma52lo(res[36], a[15], a[21]); // Sum(36) + res[37] = fma52hi(res[37], a[15], a[21]); // Sum(36) + res[37] = fma52lo(res[37], a[16], a[21]); // Sum(37) + res[38] = fma52hi(res[38], a[16], a[21]); // Sum(37) + res[38] = fma52lo(res[38], a[17], a[21]); // Sum(38) + res[39] = fma52hi(res[39], a[17], a[21]); // Sum(38) + res[39] = fma52lo(res[39], a[18], a[21]); // Sum(39) + res[40] = fma52hi(res[40], a[18], a[21]); // Sum(39) + res[40] = fma52lo(res[40], a[19], a[21]); // Sum(40) + res[41] = fma52hi(res[41], a[19], a[21]); // Sum(40) + res[41] = fma52lo(res[41], a[20], a[21]); // Sum(41) + res[42] = fma52hi(res[42], a[20], a[21]); // Sum(41) + res[36] = fma52lo(res[36], a[14], a[22]); // Sum(36) + res[37] = fma52hi(res[37], a[14], a[22]); // Sum(36) + res[37] = fma52lo(res[37], a[15], a[22]); // Sum(37) + res[38] = fma52hi(res[38], a[15], a[22]); // Sum(37) + res[38] = fma52lo(res[38], a[16], a[22]); // Sum(38) + res[39] = fma52hi(res[39], a[16], a[22]); // Sum(38) + res[39] = fma52lo(res[39], a[17], a[22]); // Sum(39) + res[40] = fma52hi(res[40], a[17], a[22]); // Sum(39) + res[40] = fma52lo(res[40], a[18], a[22]); // Sum(40) + res[41] = fma52hi(res[41], a[18], a[22]); // Sum(40) + res[41] = fma52lo(res[41], a[19], a[22]); // Sum(41) + res[42] = fma52hi(res[42], a[19], a[22]); // Sum(41) + res[42] = fma52lo(res[42], a[20], a[22]); // Sum(42) + res[43] = fma52hi(res[43], a[20], a[22]); // Sum(42) + res[43] = fma52lo(res[43], a[21], a[22]); // Sum(43) + res[44] = fma52hi(res[44], a[21], a[22]); // Sum(43) + res[36] = fma52lo(res[36], a[13], a[23]); // Sum(36) + res[37] = fma52hi(res[37], a[13], a[23]); // Sum(36) + res[37] = fma52lo(res[37], a[14], a[23]); // Sum(37) + res[38] = fma52hi(res[38], a[14], a[23]); // Sum(37) + res[38] = fma52lo(res[38], a[15], a[23]); // Sum(38) + res[39] = fma52hi(res[39], a[15], a[23]); // Sum(38) + res[39] = fma52lo(res[39], a[16], a[23]); // Sum(39) + res[40] = fma52hi(res[40], a[16], a[23]); // Sum(39) + res[40] = fma52lo(res[40], a[17], a[23]); // Sum(40) + res[41] = fma52hi(res[41], a[17], a[23]); // Sum(40) + res[41] = fma52lo(res[41], a[18], a[23]); // Sum(41) + res[42] = fma52hi(res[42], a[18], a[23]); // Sum(41) + res[42] = fma52lo(res[42], a[19], a[23]); // Sum(42) + res[43] = fma52hi(res[43], a[19], a[23]); // Sum(42) + res[43] = fma52lo(res[43], a[20], a[23]); // Sum(43) + res[44] = fma52hi(res[44], a[20], a[23]); // Sum(43) + res[44] = fma52lo(res[44], a[21], a[23]); // Sum(44) + res[45] = fma52hi(res[45], a[21], a[23]); // Sum(44) + res[45] = fma52lo(res[45], a[22], a[23]); // Sum(45) + res[46] = fma52hi(res[46], a[22], a[23]); // Sum(45) + res[36] = fma52lo(res[36], a[12], a[24]); // Sum(36) + res[37] = fma52hi(res[37], a[12], a[24]); // Sum(36) + res[37] = fma52lo(res[37], a[13], a[24]); // Sum(37) + res[38] = fma52hi(res[38], a[13], a[24]); // Sum(37) + res[38] = fma52lo(res[38], a[14], a[24]); // Sum(38) + res[39] = fma52hi(res[39], a[14], a[24]); // Sum(38) + res[39] = fma52lo(res[39], a[15], a[24]); // Sum(39) + res[40] = fma52hi(res[40], a[15], a[24]); // Sum(39) + res[40] = fma52lo(res[40], a[16], a[24]); // Sum(40) + res[41] = fma52hi(res[41], a[16], a[24]); // Sum(40) + res[41] = fma52lo(res[41], a[17], a[24]); // Sum(41) + res[42] = fma52hi(res[42], a[17], a[24]); // Sum(41) + res[42] = fma52lo(res[42], a[18], a[24]); // Sum(42) + res[43] = fma52hi(res[43], a[18], a[24]); // Sum(42) + res[43] = fma52lo(res[43], a[19], a[24]); // Sum(43) + res[44] = fma52hi(res[44], a[19], a[24]); // Sum(43) + res[44] = fma52lo(res[44], a[20], a[24]); // Sum(44) + res[45] = fma52hi(res[45], a[20], a[24]); // Sum(44) + res[45] = fma52lo(res[45], a[21], a[24]); // Sum(45) + res[46] = fma52hi(res[46], a[21], a[24]); // Sum(45) + res[46] = fma52lo(res[46], a[22], a[24]); // Sum(46) + res[47] = fma52hi(res[47], a[22], a[24]); // Sum(46) + res[47] = fma52lo(res[47], a[23], a[24]); // Sum(47) + res[48] = fma52hi(res[48], a[23], a[24]); // Sum(47) + res[36] = fma52lo(res[36], a[11], a[25]); // Sum(36) + res[37] = fma52hi(res[37], a[11], a[25]); // Sum(36) + res[37] = fma52lo(res[37], a[12], a[25]); // Sum(37) + res[38] = fma52hi(res[38], a[12], a[25]); // Sum(37) + res[38] = fma52lo(res[38], a[13], a[25]); // Sum(38) + res[39] = fma52hi(res[39], a[13], a[25]); // Sum(38) + res[39] = fma52lo(res[39], a[14], a[25]); // Sum(39) + res[40] = fma52hi(res[40], a[14], a[25]); // Sum(39) + res[40] = fma52lo(res[40], a[15], a[25]); // Sum(40) + res[41] = fma52hi(res[41], a[15], a[25]); // Sum(40) + res[41] = fma52lo(res[41], a[16], a[25]); // Sum(41) + res[42] = fma52hi(res[42], a[16], a[25]); // Sum(41) + res[42] = fma52lo(res[42], a[17], a[25]); // Sum(42) + res[43] = fma52hi(res[43], a[17], a[25]); // Sum(42) + res[43] = fma52lo(res[43], a[18], a[25]); // Sum(43) + res[44] = fma52hi(res[44], a[18], a[25]); // Sum(43) + res[44] = fma52lo(res[44], a[19], a[25]); // Sum(44) + res[45] = fma52hi(res[45], a[19], a[25]); // Sum(44) + res[45] = fma52lo(res[45], a[20], a[25]); // Sum(45) + res[46] = fma52hi(res[46], a[20], a[25]); // Sum(45) + res[46] = fma52lo(res[46], a[21], a[25]); // Sum(46) + res[47] = fma52hi(res[47], a[21], a[25]); // Sum(46) + res[47] = fma52lo(res[47], a[22], a[25]); // Sum(47) + res[48] = fma52hi(res[48], a[22], a[25]); // Sum(47) + res[36] = fma52lo(res[36], a[10], a[26]); // Sum(36) + res[37] = fma52hi(res[37], a[10], a[26]); // Sum(36) + res[37] = fma52lo(res[37], a[11], a[26]); // Sum(37) + res[38] = fma52hi(res[38], a[11], a[26]); // Sum(37) + res[38] = fma52lo(res[38], a[12], a[26]); // Sum(38) + res[39] = fma52hi(res[39], a[12], a[26]); // Sum(38) + res[39] = fma52lo(res[39], a[13], a[26]); // Sum(39) + res[40] = fma52hi(res[40], a[13], a[26]); // Sum(39) + res[40] = fma52lo(res[40], a[14], a[26]); // Sum(40) + res[41] = fma52hi(res[41], a[14], a[26]); // Sum(40) + res[41] = fma52lo(res[41], a[15], a[26]); // Sum(41) + res[42] = fma52hi(res[42], a[15], a[26]); // Sum(41) + res[42] = fma52lo(res[42], a[16], a[26]); // Sum(42) + res[43] = fma52hi(res[43], a[16], a[26]); // Sum(42) + res[43] = fma52lo(res[43], a[17], a[26]); // Sum(43) + res[44] = fma52hi(res[44], a[17], a[26]); // Sum(43) + res[44] = fma52lo(res[44], a[18], a[26]); // Sum(44) + res[45] = fma52hi(res[45], a[18], a[26]); // Sum(44) + res[45] = fma52lo(res[45], a[19], a[26]); // Sum(45) + res[46] = fma52hi(res[46], a[19], a[26]); // Sum(45) + res[46] = fma52lo(res[46], a[20], a[26]); // Sum(46) + res[47] = fma52hi(res[47], a[20], a[26]); // Sum(46) + res[47] = fma52lo(res[47], a[21], a[26]); // Sum(47) + res[48] = fma52hi(res[48], a[21], a[26]); // Sum(47) + res[36] = fma52lo(res[36], a[9], a[27]); // Sum(36) + res[37] = fma52hi(res[37], a[9], a[27]); // Sum(36) + res[37] = fma52lo(res[37], a[10], a[27]); // Sum(37) + res[38] = fma52hi(res[38], a[10], a[27]); // Sum(37) + res[38] = fma52lo(res[38], a[11], a[27]); // Sum(38) + res[39] = fma52hi(res[39], a[11], a[27]); // Sum(38) + res[39] = fma52lo(res[39], a[12], a[27]); // Sum(39) + res[40] = fma52hi(res[40], a[12], a[27]); // Sum(39) + res[40] = fma52lo(res[40], a[13], a[27]); // Sum(40) + res[41] = fma52hi(res[41], a[13], a[27]); // Sum(40) + res[41] = fma52lo(res[41], a[14], a[27]); // Sum(41) + res[42] = fma52hi(res[42], a[14], a[27]); // Sum(41) + res[42] = fma52lo(res[42], a[15], a[27]); // Sum(42) + res[43] = fma52hi(res[43], a[15], a[27]); // Sum(42) + res[43] = fma52lo(res[43], a[16], a[27]); // Sum(43) + res[44] = fma52hi(res[44], a[16], a[27]); // Sum(43) + res[44] = fma52lo(res[44], a[17], a[27]); // Sum(44) + res[45] = fma52hi(res[45], a[17], a[27]); // Sum(44) + res[45] = fma52lo(res[45], a[18], a[27]); // Sum(45) + res[46] = fma52hi(res[46], a[18], a[27]); // Sum(45) + res[46] = fma52lo(res[46], a[19], a[27]); // Sum(46) + res[47] = fma52hi(res[47], a[19], a[27]); // Sum(46) + res[47] = fma52lo(res[47], a[20], a[27]); // Sum(47) + res[48] = fma52hi(res[48], a[20], a[27]); // Sum(47) + res[36] = fma52lo(res[36], a[8], a[28]); // Sum(36) + res[37] = fma52hi(res[37], a[8], a[28]); // Sum(36) + res[37] = fma52lo(res[37], a[9], a[28]); // Sum(37) + res[38] = fma52hi(res[38], a[9], a[28]); // Sum(37) + res[38] = fma52lo(res[38], a[10], a[28]); // Sum(38) + res[39] = fma52hi(res[39], a[10], a[28]); // Sum(38) + res[39] = fma52lo(res[39], a[11], a[28]); // Sum(39) + res[40] = fma52hi(res[40], a[11], a[28]); // Sum(39) + res[40] = fma52lo(res[40], a[12], a[28]); // Sum(40) + res[41] = fma52hi(res[41], a[12], a[28]); // Sum(40) + res[41] = fma52lo(res[41], a[13], a[28]); // Sum(41) + res[42] = fma52hi(res[42], a[13], a[28]); // Sum(41) + res[42] = fma52lo(res[42], a[14], a[28]); // Sum(42) + res[43] = fma52hi(res[43], a[14], a[28]); // Sum(42) + res[43] = fma52lo(res[43], a[15], a[28]); // Sum(43) + res[44] = fma52hi(res[44], a[15], a[28]); // Sum(43) + res[44] = fma52lo(res[44], a[16], a[28]); // Sum(44) + res[45] = fma52hi(res[45], a[16], a[28]); // Sum(44) + res[45] = fma52lo(res[45], a[17], a[28]); // Sum(45) + res[46] = fma52hi(res[46], a[17], a[28]); // Sum(45) + res[46] = fma52lo(res[46], a[18], a[28]); // Sum(46) + res[47] = fma52hi(res[47], a[18], a[28]); // Sum(46) + res[47] = fma52lo(res[47], a[19], a[28]); // Sum(47) + res[48] = fma52hi(res[48], a[19], a[28]); // Sum(47) + res[36] = fma52lo(res[36], a[7], a[29]); // Sum(36) + res[37] = fma52hi(res[37], a[7], a[29]); // Sum(36) + res[37] = fma52lo(res[37], a[8], a[29]); // Sum(37) + res[38] = fma52hi(res[38], a[8], a[29]); // Sum(37) + res[38] = fma52lo(res[38], a[9], a[29]); // Sum(38) + res[39] = fma52hi(res[39], a[9], a[29]); // Sum(38) + res[39] = fma52lo(res[39], a[10], a[29]); // Sum(39) + res[40] = fma52hi(res[40], a[10], a[29]); // Sum(39) + res[40] = fma52lo(res[40], a[11], a[29]); // Sum(40) + res[41] = fma52hi(res[41], a[11], a[29]); // Sum(40) + res[41] = fma52lo(res[41], a[12], a[29]); // Sum(41) + res[42] = fma52hi(res[42], a[12], a[29]); // Sum(41) + res[42] = fma52lo(res[42], a[13], a[29]); // Sum(42) + res[43] = fma52hi(res[43], a[13], a[29]); // Sum(42) + res[43] = fma52lo(res[43], a[14], a[29]); // Sum(43) + res[44] = fma52hi(res[44], a[14], a[29]); // Sum(43) + res[44] = fma52lo(res[44], a[15], a[29]); // Sum(44) + res[45] = fma52hi(res[45], a[15], a[29]); // Sum(44) + res[45] = fma52lo(res[45], a[16], a[29]); // Sum(45) + res[46] = fma52hi(res[46], a[16], a[29]); // Sum(45) + res[46] = fma52lo(res[46], a[17], a[29]); // Sum(46) + res[47] = fma52hi(res[47], a[17], a[29]); // Sum(46) + res[47] = fma52lo(res[47], a[18], a[29]); // Sum(47) + res[48] = fma52hi(res[48], a[18], a[29]); // Sum(47) + res[36] = fma52lo(res[36], a[6], a[30]); // Sum(36) + res[37] = fma52hi(res[37], a[6], a[30]); // Sum(36) + res[37] = fma52lo(res[37], a[7], a[30]); // Sum(37) + res[38] = fma52hi(res[38], a[7], a[30]); // Sum(37) + res[38] = fma52lo(res[38], a[8], a[30]); // Sum(38) + res[39] = fma52hi(res[39], a[8], a[30]); // Sum(38) + res[39] = fma52lo(res[39], a[9], a[30]); // Sum(39) + res[40] = fma52hi(res[40], a[9], a[30]); // Sum(39) + res[40] = fma52lo(res[40], a[10], a[30]); // Sum(40) + res[41] = fma52hi(res[41], a[10], a[30]); // Sum(40) + res[41] = fma52lo(res[41], a[11], a[30]); // Sum(41) + res[42] = fma52hi(res[42], a[11], a[30]); // Sum(41) + res[42] = fma52lo(res[42], a[12], a[30]); // Sum(42) + res[43] = fma52hi(res[43], a[12], a[30]); // Sum(42) + res[43] = fma52lo(res[43], a[13], a[30]); // Sum(43) + res[44] = fma52hi(res[44], a[13], a[30]); // Sum(43) + res[44] = fma52lo(res[44], a[14], a[30]); // Sum(44) + res[45] = fma52hi(res[45], a[14], a[30]); // Sum(44) + res[45] = fma52lo(res[45], a[15], a[30]); // Sum(45) + res[46] = fma52hi(res[46], a[15], a[30]); // Sum(45) + res[46] = fma52lo(res[46], a[16], a[30]); // Sum(46) + res[47] = fma52hi(res[47], a[16], a[30]); // Sum(46) + res[47] = fma52lo(res[47], a[17], a[30]); // Sum(47) + res[48] = fma52hi(res[48], a[17], a[30]); // Sum(47) + res[36] = fma52lo(res[36], a[5], a[31]); // Sum(36) + res[37] = fma52hi(res[37], a[5], a[31]); // Sum(36) + res[37] = fma52lo(res[37], a[6], a[31]); // Sum(37) + res[38] = fma52hi(res[38], a[6], a[31]); // Sum(37) + res[38] = fma52lo(res[38], a[7], a[31]); // Sum(38) + res[39] = fma52hi(res[39], a[7], a[31]); // Sum(38) + res[39] = fma52lo(res[39], a[8], a[31]); // Sum(39) + res[40] = fma52hi(res[40], a[8], a[31]); // Sum(39) + res[40] = fma52lo(res[40], a[9], a[31]); // Sum(40) + res[41] = fma52hi(res[41], a[9], a[31]); // Sum(40) + res[41] = fma52lo(res[41], a[10], a[31]); // Sum(41) + res[42] = fma52hi(res[42], a[10], a[31]); // Sum(41) + res[42] = fma52lo(res[42], a[11], a[31]); // Sum(42) + res[43] = fma52hi(res[43], a[11], a[31]); // Sum(42) + res[43] = fma52lo(res[43], a[12], a[31]); // Sum(43) + res[44] = fma52hi(res[44], a[12], a[31]); // Sum(43) + res[44] = fma52lo(res[44], a[13], a[31]); // Sum(44) + res[45] = fma52hi(res[45], a[13], a[31]); // Sum(44) + res[45] = fma52lo(res[45], a[14], a[31]); // Sum(45) + res[46] = fma52hi(res[46], a[14], a[31]); // Sum(45) + res[46] = fma52lo(res[46], a[15], a[31]); // Sum(46) + res[47] = fma52hi(res[47], a[15], a[31]); // Sum(46) + res[47] = fma52lo(res[47], a[16], a[31]); // Sum(47) + res[48] = fma52hi(res[48], a[16], a[31]); // Sum(47) + res[36] = fma52lo(res[36], a[4], a[32]); // Sum(36) + res[37] = fma52hi(res[37], a[4], a[32]); // Sum(36) + res[37] = fma52lo(res[37], a[5], a[32]); // Sum(37) + res[38] = fma52hi(res[38], a[5], a[32]); // Sum(37) + res[38] = fma52lo(res[38], a[6], a[32]); // Sum(38) + res[39] = fma52hi(res[39], a[6], a[32]); // Sum(38) + res[39] = fma52lo(res[39], a[7], a[32]); // Sum(39) + res[40] = fma52hi(res[40], a[7], a[32]); // Sum(39) + res[40] = fma52lo(res[40], a[8], a[32]); // Sum(40) + res[41] = fma52hi(res[41], a[8], a[32]); // Sum(40) + res[41] = fma52lo(res[41], a[9], a[32]); // Sum(41) + res[42] = fma52hi(res[42], a[9], a[32]); // Sum(41) + res[42] = fma52lo(res[42], a[10], a[32]); // Sum(42) + res[43] = fma52hi(res[43], a[10], a[32]); // Sum(42) + res[43] = fma52lo(res[43], a[11], a[32]); // Sum(43) + res[44] = fma52hi(res[44], a[11], a[32]); // Sum(43) + res[44] = fma52lo(res[44], a[12], a[32]); // Sum(44) + res[45] = fma52hi(res[45], a[12], a[32]); // Sum(44) + res[45] = fma52lo(res[45], a[13], a[32]); // Sum(45) + res[46] = fma52hi(res[46], a[13], a[32]); // Sum(45) + res[46] = fma52lo(res[46], a[14], a[32]); // Sum(46) + res[47] = fma52hi(res[47], a[14], a[32]); // Sum(46) + res[47] = fma52lo(res[47], a[15], a[32]); // Sum(47) + res[48] = fma52hi(res[48], a[15], a[32]); // Sum(47) + res[36] = fma52lo(res[36], a[3], a[33]); // Sum(36) + res[37] = fma52hi(res[37], a[3], a[33]); // Sum(36) + res[37] = fma52lo(res[37], a[4], a[33]); // Sum(37) + res[38] = fma52hi(res[38], a[4], a[33]); // Sum(37) + res[38] = fma52lo(res[38], a[5], a[33]); // Sum(38) + res[39] = fma52hi(res[39], a[5], a[33]); // Sum(38) + res[39] = fma52lo(res[39], a[6], a[33]); // Sum(39) + res[40] = fma52hi(res[40], a[6], a[33]); // Sum(39) + res[40] = fma52lo(res[40], a[7], a[33]); // Sum(40) + res[41] = fma52hi(res[41], a[7], a[33]); // Sum(40) + res[41] = fma52lo(res[41], a[8], a[33]); // Sum(41) + res[42] = fma52hi(res[42], a[8], a[33]); // Sum(41) + res[42] = fma52lo(res[42], a[9], a[33]); // Sum(42) + res[43] = fma52hi(res[43], a[9], a[33]); // Sum(42) + res[43] = fma52lo(res[43], a[10], a[33]); // Sum(43) + res[44] = fma52hi(res[44], a[10], a[33]); // Sum(43) + res[44] = fma52lo(res[44], a[11], a[33]); // Sum(44) + res[45] = fma52hi(res[45], a[11], a[33]); // Sum(44) + res[45] = fma52lo(res[45], a[12], a[33]); // Sum(45) + res[46] = fma52hi(res[46], a[12], a[33]); // Sum(45) + res[46] = fma52lo(res[46], a[13], a[33]); // Sum(46) + res[47] = fma52hi(res[47], a[13], a[33]); // Sum(46) + res[47] = fma52lo(res[47], a[14], a[33]); // Sum(47) + res[48] = fma52hi(res[48], a[14], a[33]); // Sum(47) + res[36] = fma52lo(res[36], a[2], a[34]); // Sum(36) + res[37] = fma52hi(res[37], a[2], a[34]); // Sum(36) + res[37] = fma52lo(res[37], a[3], a[34]); // Sum(37) + res[38] = fma52hi(res[38], a[3], a[34]); // Sum(37) + res[38] = fma52lo(res[38], a[4], a[34]); // Sum(38) + res[39] = fma52hi(res[39], a[4], a[34]); // Sum(38) + res[39] = fma52lo(res[39], a[5], a[34]); // Sum(39) + res[40] = fma52hi(res[40], a[5], a[34]); // Sum(39) + res[40] = fma52lo(res[40], a[6], a[34]); // Sum(40) + res[41] = fma52hi(res[41], a[6], a[34]); // Sum(40) + res[41] = fma52lo(res[41], a[7], a[34]); // Sum(41) + res[42] = fma52hi(res[42], a[7], a[34]); // Sum(41) + res[42] = fma52lo(res[42], a[8], a[34]); // Sum(42) + res[43] = fma52hi(res[43], a[8], a[34]); // Sum(42) + res[43] = fma52lo(res[43], a[9], a[34]); // Sum(43) + res[44] = fma52hi(res[44], a[9], a[34]); // Sum(43) + res[44] = fma52lo(res[44], a[10], a[34]); // Sum(44) + res[45] = fma52hi(res[45], a[10], a[34]); // Sum(44) + res[45] = fma52lo(res[45], a[11], a[34]); // Sum(45) + res[46] = fma52hi(res[46], a[11], a[34]); // Sum(45) + res[46] = fma52lo(res[46], a[12], a[34]); // Sum(46) + res[47] = fma52hi(res[47], a[12], a[34]); // Sum(46) + res[47] = fma52lo(res[47], a[13], a[34]); // Sum(47) + res[48] = fma52hi(res[48], a[13], a[34]); // Sum(47) + res[36] = fma52lo(res[36], a[1], a[35]); // Sum(36) + res[37] = fma52hi(res[37], a[1], a[35]); // Sum(36) + res[37] = fma52lo(res[37], a[2], a[35]); // Sum(37) + res[38] = fma52hi(res[38], a[2], a[35]); // Sum(37) + res[38] = fma52lo(res[38], a[3], a[35]); // Sum(38) + res[39] = fma52hi(res[39], a[3], a[35]); // Sum(38) + res[39] = fma52lo(res[39], a[4], a[35]); // Sum(39) + res[40] = fma52hi(res[40], a[4], a[35]); // Sum(39) + res[40] = fma52lo(res[40], a[5], a[35]); // Sum(40) + res[41] = fma52hi(res[41], a[5], a[35]); // Sum(40) + res[41] = fma52lo(res[41], a[6], a[35]); // Sum(41) + res[42] = fma52hi(res[42], a[6], a[35]); // Sum(41) + res[42] = fma52lo(res[42], a[7], a[35]); // Sum(42) + res[43] = fma52hi(res[43], a[7], a[35]); // Sum(42) + res[43] = fma52lo(res[43], a[8], a[35]); // Sum(43) + res[44] = fma52hi(res[44], a[8], a[35]); // Sum(43) + res[44] = fma52lo(res[44], a[9], a[35]); // Sum(44) + res[45] = fma52hi(res[45], a[9], a[35]); // Sum(44) + res[45] = fma52lo(res[45], a[10], a[35]); // Sum(45) + res[46] = fma52hi(res[46], a[10], a[35]); // Sum(45) + res[46] = fma52lo(res[46], a[11], a[35]); // Sum(46) + res[47] = fma52hi(res[47], a[11], a[35]); // Sum(46) + res[47] = fma52lo(res[47], a[12], a[35]); // Sum(47) + res[48] = fma52hi(res[48], a[12], a[35]); // Sum(47) + res[36] = fma52lo(res[36], a[0], a[36]); // Sum(36) + res[37] = fma52hi(res[37], a[0], a[36]); // Sum(36) + res[37] = fma52lo(res[37], a[1], a[36]); // Sum(37) + res[38] = fma52hi(res[38], a[1], a[36]); // Sum(37) + res[38] = fma52lo(res[38], a[2], a[36]); // Sum(38) + res[39] = fma52hi(res[39], a[2], a[36]); // Sum(38) + res[39] = fma52lo(res[39], a[3], a[36]); // Sum(39) + res[40] = fma52hi(res[40], a[3], a[36]); // Sum(39) + res[40] = fma52lo(res[40], a[4], a[36]); // Sum(40) + res[41] = fma52hi(res[41], a[4], a[36]); // Sum(40) + res[41] = fma52lo(res[41], a[5], a[36]); // Sum(41) + res[42] = fma52hi(res[42], a[5], a[36]); // Sum(41) + res[42] = fma52lo(res[42], a[6], a[36]); // Sum(42) + res[43] = fma52hi(res[43], a[6], a[36]); // Sum(42) + res[43] = fma52lo(res[43], a[7], a[36]); // Sum(43) + res[44] = fma52hi(res[44], a[7], a[36]); // Sum(43) + res[44] = fma52lo(res[44], a[8], a[36]); // Sum(44) + res[45] = fma52hi(res[45], a[8], a[36]); // Sum(44) + res[45] = fma52lo(res[45], a[9], a[36]); // Sum(45) + res[46] = fma52hi(res[46], a[9], a[36]); // Sum(45) + res[46] = fma52lo(res[46], a[10], a[36]); // Sum(46) + res[47] = fma52hi(res[47], a[10], a[36]); // Sum(46) + res[47] = fma52lo(res[47], a[11], a[36]); // Sum(47) + res[48] = fma52hi(res[48], a[11], a[36]); // Sum(47) + res[37] = fma52lo(res[37], a[0], a[37]); // Sum(37) + res[38] = fma52hi(res[38], a[0], a[37]); // Sum(37) + res[38] = fma52lo(res[38], a[1], a[37]); // Sum(38) + res[39] = fma52hi(res[39], a[1], a[37]); // Sum(38) + res[39] = fma52lo(res[39], a[2], a[37]); // Sum(39) + res[40] = fma52hi(res[40], a[2], a[37]); // Sum(39) + res[40] = fma52lo(res[40], a[3], a[37]); // Sum(40) + res[41] = fma52hi(res[41], a[3], a[37]); // Sum(40) + res[41] = fma52lo(res[41], a[4], a[37]); // Sum(41) + res[42] = fma52hi(res[42], a[4], a[37]); // Sum(41) + res[42] = fma52lo(res[42], a[5], a[37]); // Sum(42) + res[43] = fma52hi(res[43], a[5], a[37]); // Sum(42) + res[43] = fma52lo(res[43], a[6], a[37]); // Sum(43) + res[44] = fma52hi(res[44], a[6], a[37]); // Sum(43) + res[44] = fma52lo(res[44], a[7], a[37]); // Sum(44) + res[45] = fma52hi(res[45], a[7], a[37]); // Sum(44) + res[45] = fma52lo(res[45], a[8], a[37]); // Sum(45) + res[46] = fma52hi(res[46], a[8], a[37]); // Sum(45) + res[46] = fma52lo(res[46], a[9], a[37]); // Sum(46) + res[47] = fma52hi(res[47], a[9], a[37]); // Sum(46) + res[47] = fma52lo(res[47], a[10], a[37]); // Sum(47) + res[48] = fma52hi(res[48], a[10], a[37]); // Sum(47) + res[38] = fma52lo(res[38], a[0], a[38]); // Sum(38) + res[39] = fma52hi(res[39], a[0], a[38]); // Sum(38) + res[39] = fma52lo(res[39], a[1], a[38]); // Sum(39) + res[40] = fma52hi(res[40], a[1], a[38]); // Sum(39) + res[40] = fma52lo(res[40], a[2], a[38]); // Sum(40) + res[41] = fma52hi(res[41], a[2], a[38]); // Sum(40) + res[41] = fma52lo(res[41], a[3], a[38]); // Sum(41) + res[42] = fma52hi(res[42], a[3], a[38]); // Sum(41) + res[42] = fma52lo(res[42], a[4], a[38]); // Sum(42) + res[43] = fma52hi(res[43], a[4], a[38]); // Sum(42) + res[43] = fma52lo(res[43], a[5], a[38]); // Sum(43) + res[44] = fma52hi(res[44], a[5], a[38]); // Sum(43) + res[44] = fma52lo(res[44], a[6], a[38]); // Sum(44) + res[45] = fma52hi(res[45], a[6], a[38]); // Sum(44) + res[45] = fma52lo(res[45], a[7], a[38]); // Sum(45) + res[46] = fma52hi(res[46], a[7], a[38]); // Sum(45) + res[46] = fma52lo(res[46], a[8], a[38]); // Sum(46) + res[47] = fma52hi(res[47], a[8], a[38]); // Sum(46) + res[47] = fma52lo(res[47], a[9], a[38]); // Sum(47) + res[48] = fma52hi(res[48], a[9], a[38]); // Sum(47) + res[39] = fma52lo(res[39], a[0], a[39]); // Sum(39) + res[40] = fma52hi(res[40], a[0], a[39]); // Sum(39) + res[40] = fma52lo(res[40], a[1], a[39]); // Sum(40) + res[41] = fma52hi(res[41], a[1], a[39]); // Sum(40) + res[41] = fma52lo(res[41], a[2], a[39]); // Sum(41) + res[42] = fma52hi(res[42], a[2], a[39]); // Sum(41) + res[42] = fma52lo(res[42], a[3], a[39]); // Sum(42) + res[43] = fma52hi(res[43], a[3], a[39]); // Sum(42) + res[43] = fma52lo(res[43], a[4], a[39]); // Sum(43) + res[44] = fma52hi(res[44], a[4], a[39]); // Sum(43) + res[44] = fma52lo(res[44], a[5], a[39]); // Sum(44) + res[45] = fma52hi(res[45], a[5], a[39]); // Sum(44) + res[45] = fma52lo(res[45], a[6], a[39]); // Sum(45) + res[46] = fma52hi(res[46], a[6], a[39]); // Sum(45) + res[46] = fma52lo(res[46], a[7], a[39]); // Sum(46) + res[47] = fma52hi(res[47], a[7], a[39]); // Sum(46) + res[47] = fma52lo(res[47], a[8], a[39]); // Sum(47) + res[48] = fma52hi(res[48], a[8], a[39]); // Sum(47) + res[36] = add64(res[36], res[36]); // Double(36) + res[37] = add64(res[37], res[37]); // Double(37) + res[38] = add64(res[38], res[38]); // Double(38) + res[39] = add64(res[39], res[39]); // Double(39) + res[40] = add64(res[40], res[40]); // Double(40) + res[41] = add64(res[41], res[41]); // Double(41) + res[42] = add64(res[42], res[42]); // Double(42) + res[43] = add64(res[43], res[43]); // Double(43) + res[44] = add64(res[44], res[44]); // Double(44) + res[45] = add64(res[45], res[45]); // Double(45) + res[46] = add64(res[46], res[46]); // Double(46) + res[47] = add64(res[47], res[47]); // Double(47) + res[36] = fma52lo(res[36], a[18], a[18]); // Add sqr(36) + res[37] = fma52hi(res[37], a[18], a[18]); // Add sqr(36) + res[38] = fma52lo(res[38], a[19], a[19]); // Add sqr(38) + res[39] = fma52hi(res[39], a[19], a[19]); // Add sqr(38) + res[40] = fma52lo(res[40], a[20], a[20]); // Add sqr(40) + res[41] = fma52hi(res[41], a[20], a[20]); // Add sqr(40) + res[42] = fma52lo(res[42], a[21], a[21]); // Add sqr(42) + res[43] = fma52hi(res[43], a[21], a[21]); // Add sqr(42) + res[44] = fma52lo(res[44], a[22], a[22]); // Add sqr(44) + res[45] = fma52hi(res[45], a[22], a[22]); // Add sqr(44) + res[46] = fma52lo(res[46], a[23], a[23]); // Add sqr(46) + res[47] = fma52hi(res[47], a[23], a[23]); // Add sqr(46) + res[48] = fma52lo(res[48], a[23], a[25]); // Sum(48) + res[49] = fma52hi(res[49], a[23], a[25]); // Sum(48) + res[49] = fma52lo(res[49], a[24], a[25]); // Sum(49) + res[50] = fma52hi(res[50], a[24], a[25]); // Sum(49) + res[48] = fma52lo(res[48], a[22], a[26]); // Sum(48) + res[49] = fma52hi(res[49], a[22], a[26]); // Sum(48) + res[49] = fma52lo(res[49], a[23], a[26]); // Sum(49) + res[50] = fma52hi(res[50], a[23], a[26]); // Sum(49) + res[50] = fma52lo(res[50], a[24], a[26]); // Sum(50) + res[51] = fma52hi(res[51], a[24], a[26]); // Sum(50) + res[51] = fma52lo(res[51], a[25], a[26]); // Sum(51) + res[52] = fma52hi(res[52], a[25], a[26]); // Sum(51) + res[48] = fma52lo(res[48], a[21], a[27]); // Sum(48) + res[49] = fma52hi(res[49], a[21], a[27]); // Sum(48) + res[49] = fma52lo(res[49], a[22], a[27]); // Sum(49) + res[50] = fma52hi(res[50], a[22], a[27]); // Sum(49) + res[50] = fma52lo(res[50], a[23], a[27]); // Sum(50) + res[51] = fma52hi(res[51], a[23], a[27]); // Sum(50) + res[51] = fma52lo(res[51], a[24], a[27]); // Sum(51) + res[52] = fma52hi(res[52], a[24], a[27]); // Sum(51) + res[52] = fma52lo(res[52], a[25], a[27]); // Sum(52) + res[53] = fma52hi(res[53], a[25], a[27]); // Sum(52) + res[53] = fma52lo(res[53], a[26], a[27]); // Sum(53) + res[54] = fma52hi(res[54], a[26], a[27]); // Sum(53) + res[48] = fma52lo(res[48], a[20], a[28]); // Sum(48) + res[49] = fma52hi(res[49], a[20], a[28]); // Sum(48) + res[49] = fma52lo(res[49], a[21], a[28]); // Sum(49) + res[50] = fma52hi(res[50], a[21], a[28]); // Sum(49) + res[50] = fma52lo(res[50], a[22], a[28]); // Sum(50) + res[51] = fma52hi(res[51], a[22], a[28]); // Sum(50) + res[51] = fma52lo(res[51], a[23], a[28]); // Sum(51) + res[52] = fma52hi(res[52], a[23], a[28]); // Sum(51) + res[52] = fma52lo(res[52], a[24], a[28]); // Sum(52) + res[53] = fma52hi(res[53], a[24], a[28]); // Sum(52) + res[53] = fma52lo(res[53], a[25], a[28]); // Sum(53) + res[54] = fma52hi(res[54], a[25], a[28]); // Sum(53) + res[54] = fma52lo(res[54], a[26], a[28]); // Sum(54) + res[55] = fma52hi(res[55], a[26], a[28]); // Sum(54) + res[55] = fma52lo(res[55], a[27], a[28]); // Sum(55) + res[56] = fma52hi(res[56], a[27], a[28]); // Sum(55) + res[48] = fma52lo(res[48], a[19], a[29]); // Sum(48) + res[49] = fma52hi(res[49], a[19], a[29]); // Sum(48) + res[49] = fma52lo(res[49], a[20], a[29]); // Sum(49) + res[50] = fma52hi(res[50], a[20], a[29]); // Sum(49) + res[50] = fma52lo(res[50], a[21], a[29]); // Sum(50) + res[51] = fma52hi(res[51], a[21], a[29]); // Sum(50) + res[51] = fma52lo(res[51], a[22], a[29]); // Sum(51) + res[52] = fma52hi(res[52], a[22], a[29]); // Sum(51) + res[52] = fma52lo(res[52], a[23], a[29]); // Sum(52) + res[53] = fma52hi(res[53], a[23], a[29]); // Sum(52) + res[53] = fma52lo(res[53], a[24], a[29]); // Sum(53) + res[54] = fma52hi(res[54], a[24], a[29]); // Sum(53) + res[54] = fma52lo(res[54], a[25], a[29]); // Sum(54) + res[55] = fma52hi(res[55], a[25], a[29]); // Sum(54) + res[55] = fma52lo(res[55], a[26], a[29]); // Sum(55) + res[56] = fma52hi(res[56], a[26], a[29]); // Sum(55) + res[56] = fma52lo(res[56], a[27], a[29]); // Sum(56) + res[57] = fma52hi(res[57], a[27], a[29]); // Sum(56) + res[57] = fma52lo(res[57], a[28], a[29]); // Sum(57) + res[58] = fma52hi(res[58], a[28], a[29]); // Sum(57) + res[48] = fma52lo(res[48], a[18], a[30]); // Sum(48) + res[49] = fma52hi(res[49], a[18], a[30]); // Sum(48) + res[49] = fma52lo(res[49], a[19], a[30]); // Sum(49) + res[50] = fma52hi(res[50], a[19], a[30]); // Sum(49) + res[50] = fma52lo(res[50], a[20], a[30]); // Sum(50) + res[51] = fma52hi(res[51], a[20], a[30]); // Sum(50) + res[51] = fma52lo(res[51], a[21], a[30]); // Sum(51) + res[52] = fma52hi(res[52], a[21], a[30]); // Sum(51) + res[52] = fma52lo(res[52], a[22], a[30]); // Sum(52) + res[53] = fma52hi(res[53], a[22], a[30]); // Sum(52) + res[53] = fma52lo(res[53], a[23], a[30]); // Sum(53) + res[54] = fma52hi(res[54], a[23], a[30]); // Sum(53) + res[54] = fma52lo(res[54], a[24], a[30]); // Sum(54) + res[55] = fma52hi(res[55], a[24], a[30]); // Sum(54) + res[55] = fma52lo(res[55], a[25], a[30]); // Sum(55) + res[56] = fma52hi(res[56], a[25], a[30]); // Sum(55) + res[56] = fma52lo(res[56], a[26], a[30]); // Sum(56) + res[57] = fma52hi(res[57], a[26], a[30]); // Sum(56) + res[57] = fma52lo(res[57], a[27], a[30]); // Sum(57) + res[58] = fma52hi(res[58], a[27], a[30]); // Sum(57) + res[58] = fma52lo(res[58], a[28], a[30]); // Sum(58) + res[59] = fma52hi(res[59], a[28], a[30]); // Sum(58) + res[59] = fma52lo(res[59], a[29], a[30]); // Sum(59) + res[60] = fma52hi(res[60], a[29], a[30]); // Sum(59) + res[48] = fma52lo(res[48], a[17], a[31]); // Sum(48) + res[49] = fma52hi(res[49], a[17], a[31]); // Sum(48) + res[49] = fma52lo(res[49], a[18], a[31]); // Sum(49) + res[50] = fma52hi(res[50], a[18], a[31]); // Sum(49) + res[50] = fma52lo(res[50], a[19], a[31]); // Sum(50) + res[51] = fma52hi(res[51], a[19], a[31]); // Sum(50) + res[51] = fma52lo(res[51], a[20], a[31]); // Sum(51) + res[52] = fma52hi(res[52], a[20], a[31]); // Sum(51) + res[52] = fma52lo(res[52], a[21], a[31]); // Sum(52) + res[53] = fma52hi(res[53], a[21], a[31]); // Sum(52) + res[53] = fma52lo(res[53], a[22], a[31]); // Sum(53) + res[54] = fma52hi(res[54], a[22], a[31]); // Sum(53) + res[54] = fma52lo(res[54], a[23], a[31]); // Sum(54) + res[55] = fma52hi(res[55], a[23], a[31]); // Sum(54) + res[55] = fma52lo(res[55], a[24], a[31]); // Sum(55) + res[56] = fma52hi(res[56], a[24], a[31]); // Sum(55) + res[56] = fma52lo(res[56], a[25], a[31]); // Sum(56) + res[57] = fma52hi(res[57], a[25], a[31]); // Sum(56) + res[57] = fma52lo(res[57], a[26], a[31]); // Sum(57) + res[58] = fma52hi(res[58], a[26], a[31]); // Sum(57) + res[58] = fma52lo(res[58], a[27], a[31]); // Sum(58) + res[59] = fma52hi(res[59], a[27], a[31]); // Sum(58) + res[59] = fma52lo(res[59], a[28], a[31]); // Sum(59) + res[60] = fma52hi(res[60], a[28], a[31]); // Sum(59) + res[48] = fma52lo(res[48], a[16], a[32]); // Sum(48) + res[49] = fma52hi(res[49], a[16], a[32]); // Sum(48) + res[49] = fma52lo(res[49], a[17], a[32]); // Sum(49) + res[50] = fma52hi(res[50], a[17], a[32]); // Sum(49) + res[50] = fma52lo(res[50], a[18], a[32]); // Sum(50) + res[51] = fma52hi(res[51], a[18], a[32]); // Sum(50) + res[51] = fma52lo(res[51], a[19], a[32]); // Sum(51) + res[52] = fma52hi(res[52], a[19], a[32]); // Sum(51) + res[52] = fma52lo(res[52], a[20], a[32]); // Sum(52) + res[53] = fma52hi(res[53], a[20], a[32]); // Sum(52) + res[53] = fma52lo(res[53], a[21], a[32]); // Sum(53) + res[54] = fma52hi(res[54], a[21], a[32]); // Sum(53) + res[54] = fma52lo(res[54], a[22], a[32]); // Sum(54) + res[55] = fma52hi(res[55], a[22], a[32]); // Sum(54) + res[55] = fma52lo(res[55], a[23], a[32]); // Sum(55) + res[56] = fma52hi(res[56], a[23], a[32]); // Sum(55) + res[56] = fma52lo(res[56], a[24], a[32]); // Sum(56) + res[57] = fma52hi(res[57], a[24], a[32]); // Sum(56) + res[57] = fma52lo(res[57], a[25], a[32]); // Sum(57) + res[58] = fma52hi(res[58], a[25], a[32]); // Sum(57) + res[58] = fma52lo(res[58], a[26], a[32]); // Sum(58) + res[59] = fma52hi(res[59], a[26], a[32]); // Sum(58) + res[59] = fma52lo(res[59], a[27], a[32]); // Sum(59) + res[60] = fma52hi(res[60], a[27], a[32]); // Sum(59) + res[48] = fma52lo(res[48], a[15], a[33]); // Sum(48) + res[49] = fma52hi(res[49], a[15], a[33]); // Sum(48) + res[49] = fma52lo(res[49], a[16], a[33]); // Sum(49) + res[50] = fma52hi(res[50], a[16], a[33]); // Sum(49) + res[50] = fma52lo(res[50], a[17], a[33]); // Sum(50) + res[51] = fma52hi(res[51], a[17], a[33]); // Sum(50) + res[51] = fma52lo(res[51], a[18], a[33]); // Sum(51) + res[52] = fma52hi(res[52], a[18], a[33]); // Sum(51) + res[52] = fma52lo(res[52], a[19], a[33]); // Sum(52) + res[53] = fma52hi(res[53], a[19], a[33]); // Sum(52) + res[53] = fma52lo(res[53], a[20], a[33]); // Sum(53) + res[54] = fma52hi(res[54], a[20], a[33]); // Sum(53) + res[54] = fma52lo(res[54], a[21], a[33]); // Sum(54) + res[55] = fma52hi(res[55], a[21], a[33]); // Sum(54) + res[55] = fma52lo(res[55], a[22], a[33]); // Sum(55) + res[56] = fma52hi(res[56], a[22], a[33]); // Sum(55) + res[56] = fma52lo(res[56], a[23], a[33]); // Sum(56) + res[57] = fma52hi(res[57], a[23], a[33]); // Sum(56) + res[57] = fma52lo(res[57], a[24], a[33]); // Sum(57) + res[58] = fma52hi(res[58], a[24], a[33]); // Sum(57) + res[58] = fma52lo(res[58], a[25], a[33]); // Sum(58) + res[59] = fma52hi(res[59], a[25], a[33]); // Sum(58) + res[59] = fma52lo(res[59], a[26], a[33]); // Sum(59) + res[60] = fma52hi(res[60], a[26], a[33]); // Sum(59) + res[48] = fma52lo(res[48], a[14], a[34]); // Sum(48) + res[49] = fma52hi(res[49], a[14], a[34]); // Sum(48) + res[49] = fma52lo(res[49], a[15], a[34]); // Sum(49) + res[50] = fma52hi(res[50], a[15], a[34]); // Sum(49) + res[50] = fma52lo(res[50], a[16], a[34]); // Sum(50) + res[51] = fma52hi(res[51], a[16], a[34]); // Sum(50) + res[51] = fma52lo(res[51], a[17], a[34]); // Sum(51) + res[52] = fma52hi(res[52], a[17], a[34]); // Sum(51) + res[52] = fma52lo(res[52], a[18], a[34]); // Sum(52) + res[53] = fma52hi(res[53], a[18], a[34]); // Sum(52) + res[53] = fma52lo(res[53], a[19], a[34]); // Sum(53) + res[54] = fma52hi(res[54], a[19], a[34]); // Sum(53) + res[54] = fma52lo(res[54], a[20], a[34]); // Sum(54) + res[55] = fma52hi(res[55], a[20], a[34]); // Sum(54) + res[55] = fma52lo(res[55], a[21], a[34]); // Sum(55) + res[56] = fma52hi(res[56], a[21], a[34]); // Sum(55) + res[56] = fma52lo(res[56], a[22], a[34]); // Sum(56) + res[57] = fma52hi(res[57], a[22], a[34]); // Sum(56) + res[57] = fma52lo(res[57], a[23], a[34]); // Sum(57) + res[58] = fma52hi(res[58], a[23], a[34]); // Sum(57) + res[58] = fma52lo(res[58], a[24], a[34]); // Sum(58) + res[59] = fma52hi(res[59], a[24], a[34]); // Sum(58) + res[59] = fma52lo(res[59], a[25], a[34]); // Sum(59) + res[60] = fma52hi(res[60], a[25], a[34]); // Sum(59) + res[48] = fma52lo(res[48], a[13], a[35]); // Sum(48) + res[49] = fma52hi(res[49], a[13], a[35]); // Sum(48) + res[49] = fma52lo(res[49], a[14], a[35]); // Sum(49) + res[50] = fma52hi(res[50], a[14], a[35]); // Sum(49) + res[50] = fma52lo(res[50], a[15], a[35]); // Sum(50) + res[51] = fma52hi(res[51], a[15], a[35]); // Sum(50) + res[51] = fma52lo(res[51], a[16], a[35]); // Sum(51) + res[52] = fma52hi(res[52], a[16], a[35]); // Sum(51) + res[52] = fma52lo(res[52], a[17], a[35]); // Sum(52) + res[53] = fma52hi(res[53], a[17], a[35]); // Sum(52) + res[53] = fma52lo(res[53], a[18], a[35]); // Sum(53) + res[54] = fma52hi(res[54], a[18], a[35]); // Sum(53) + res[54] = fma52lo(res[54], a[19], a[35]); // Sum(54) + res[55] = fma52hi(res[55], a[19], a[35]); // Sum(54) + res[55] = fma52lo(res[55], a[20], a[35]); // Sum(55) + res[56] = fma52hi(res[56], a[20], a[35]); // Sum(55) + res[56] = fma52lo(res[56], a[21], a[35]); // Sum(56) + res[57] = fma52hi(res[57], a[21], a[35]); // Sum(56) + res[57] = fma52lo(res[57], a[22], a[35]); // Sum(57) + res[58] = fma52hi(res[58], a[22], a[35]); // Sum(57) + res[58] = fma52lo(res[58], a[23], a[35]); // Sum(58) + res[59] = fma52hi(res[59], a[23], a[35]); // Sum(58) + res[59] = fma52lo(res[59], a[24], a[35]); // Sum(59) + res[60] = fma52hi(res[60], a[24], a[35]); // Sum(59) + res[48] = fma52lo(res[48], a[12], a[36]); // Sum(48) + res[49] = fma52hi(res[49], a[12], a[36]); // Sum(48) + res[49] = fma52lo(res[49], a[13], a[36]); // Sum(49) + res[50] = fma52hi(res[50], a[13], a[36]); // Sum(49) + res[50] = fma52lo(res[50], a[14], a[36]); // Sum(50) + res[51] = fma52hi(res[51], a[14], a[36]); // Sum(50) + res[51] = fma52lo(res[51], a[15], a[36]); // Sum(51) + res[52] = fma52hi(res[52], a[15], a[36]); // Sum(51) + res[52] = fma52lo(res[52], a[16], a[36]); // Sum(52) + res[53] = fma52hi(res[53], a[16], a[36]); // Sum(52) + res[53] = fma52lo(res[53], a[17], a[36]); // Sum(53) + res[54] = fma52hi(res[54], a[17], a[36]); // Sum(53) + res[54] = fma52lo(res[54], a[18], a[36]); // Sum(54) + res[55] = fma52hi(res[55], a[18], a[36]); // Sum(54) + res[55] = fma52lo(res[55], a[19], a[36]); // Sum(55) + res[56] = fma52hi(res[56], a[19], a[36]); // Sum(55) + res[56] = fma52lo(res[56], a[20], a[36]); // Sum(56) + res[57] = fma52hi(res[57], a[20], a[36]); // Sum(56) + res[57] = fma52lo(res[57], a[21], a[36]); // Sum(57) + res[58] = fma52hi(res[58], a[21], a[36]); // Sum(57) + res[58] = fma52lo(res[58], a[22], a[36]); // Sum(58) + res[59] = fma52hi(res[59], a[22], a[36]); // Sum(58) + res[59] = fma52lo(res[59], a[23], a[36]); // Sum(59) + res[60] = fma52hi(res[60], a[23], a[36]); // Sum(59) + res[48] = fma52lo(res[48], a[11], a[37]); // Sum(48) + res[49] = fma52hi(res[49], a[11], a[37]); // Sum(48) + res[49] = fma52lo(res[49], a[12], a[37]); // Sum(49) + res[50] = fma52hi(res[50], a[12], a[37]); // Sum(49) + res[50] = fma52lo(res[50], a[13], a[37]); // Sum(50) + res[51] = fma52hi(res[51], a[13], a[37]); // Sum(50) + res[51] = fma52lo(res[51], a[14], a[37]); // Sum(51) + res[52] = fma52hi(res[52], a[14], a[37]); // Sum(51) + res[52] = fma52lo(res[52], a[15], a[37]); // Sum(52) + res[53] = fma52hi(res[53], a[15], a[37]); // Sum(52) + res[53] = fma52lo(res[53], a[16], a[37]); // Sum(53) + res[54] = fma52hi(res[54], a[16], a[37]); // Sum(53) + res[54] = fma52lo(res[54], a[17], a[37]); // Sum(54) + res[55] = fma52hi(res[55], a[17], a[37]); // Sum(54) + res[55] = fma52lo(res[55], a[18], a[37]); // Sum(55) + res[56] = fma52hi(res[56], a[18], a[37]); // Sum(55) + res[56] = fma52lo(res[56], a[19], a[37]); // Sum(56) + res[57] = fma52hi(res[57], a[19], a[37]); // Sum(56) + res[57] = fma52lo(res[57], a[20], a[37]); // Sum(57) + res[58] = fma52hi(res[58], a[20], a[37]); // Sum(57) + res[58] = fma52lo(res[58], a[21], a[37]); // Sum(58) + res[59] = fma52hi(res[59], a[21], a[37]); // Sum(58) + res[59] = fma52lo(res[59], a[22], a[37]); // Sum(59) + res[60] = fma52hi(res[60], a[22], a[37]); // Sum(59) + res[48] = fma52lo(res[48], a[10], a[38]); // Sum(48) + res[49] = fma52hi(res[49], a[10], a[38]); // Sum(48) + res[49] = fma52lo(res[49], a[11], a[38]); // Sum(49) + res[50] = fma52hi(res[50], a[11], a[38]); // Sum(49) + res[50] = fma52lo(res[50], a[12], a[38]); // Sum(50) + res[51] = fma52hi(res[51], a[12], a[38]); // Sum(50) + res[51] = fma52lo(res[51], a[13], a[38]); // Sum(51) + res[52] = fma52hi(res[52], a[13], a[38]); // Sum(51) + res[52] = fma52lo(res[52], a[14], a[38]); // Sum(52) + res[53] = fma52hi(res[53], a[14], a[38]); // Sum(52) + res[53] = fma52lo(res[53], a[15], a[38]); // Sum(53) + res[54] = fma52hi(res[54], a[15], a[38]); // Sum(53) + res[54] = fma52lo(res[54], a[16], a[38]); // Sum(54) + res[55] = fma52hi(res[55], a[16], a[38]); // Sum(54) + res[55] = fma52lo(res[55], a[17], a[38]); // Sum(55) + res[56] = fma52hi(res[56], a[17], a[38]); // Sum(55) + res[56] = fma52lo(res[56], a[18], a[38]); // Sum(56) + res[57] = fma52hi(res[57], a[18], a[38]); // Sum(56) + res[57] = fma52lo(res[57], a[19], a[38]); // Sum(57) + res[58] = fma52hi(res[58], a[19], a[38]); // Sum(57) + res[58] = fma52lo(res[58], a[20], a[38]); // Sum(58) + res[59] = fma52hi(res[59], a[20], a[38]); // Sum(58) + res[59] = fma52lo(res[59], a[21], a[38]); // Sum(59) + res[60] = fma52hi(res[60], a[21], a[38]); // Sum(59) + res[48] = fma52lo(res[48], a[9], a[39]); // Sum(48) + res[49] = fma52hi(res[49], a[9], a[39]); // Sum(48) + res[49] = fma52lo(res[49], a[10], a[39]); // Sum(49) + res[50] = fma52hi(res[50], a[10], a[39]); // Sum(49) + res[50] = fma52lo(res[50], a[11], a[39]); // Sum(50) + res[51] = fma52hi(res[51], a[11], a[39]); // Sum(50) + res[51] = fma52lo(res[51], a[12], a[39]); // Sum(51) + res[52] = fma52hi(res[52], a[12], a[39]); // Sum(51) + res[52] = fma52lo(res[52], a[13], a[39]); // Sum(52) + res[53] = fma52hi(res[53], a[13], a[39]); // Sum(52) + res[53] = fma52lo(res[53], a[14], a[39]); // Sum(53) + res[54] = fma52hi(res[54], a[14], a[39]); // Sum(53) + res[54] = fma52lo(res[54], a[15], a[39]); // Sum(54) + res[55] = fma52hi(res[55], a[15], a[39]); // Sum(54) + res[55] = fma52lo(res[55], a[16], a[39]); // Sum(55) + res[56] = fma52hi(res[56], a[16], a[39]); // Sum(55) + res[56] = fma52lo(res[56], a[17], a[39]); // Sum(56) + res[57] = fma52hi(res[57], a[17], a[39]); // Sum(56) + res[57] = fma52lo(res[57], a[18], a[39]); // Sum(57) + res[58] = fma52hi(res[58], a[18], a[39]); // Sum(57) + res[58] = fma52lo(res[58], a[19], a[39]); // Sum(58) + res[59] = fma52hi(res[59], a[19], a[39]); // Sum(58) + res[59] = fma52lo(res[59], a[20], a[39]); // Sum(59) + res[60] = fma52hi(res[60], a[20], a[39]); // Sum(59) + res[48] = add64(res[48], res[48]); // Double(48) + res[49] = add64(res[49], res[49]); // Double(49) + res[50] = add64(res[50], res[50]); // Double(50) + res[51] = add64(res[51], res[51]); // Double(51) + res[52] = add64(res[52], res[52]); // Double(52) + res[53] = add64(res[53], res[53]); // Double(53) + res[54] = add64(res[54], res[54]); // Double(54) + res[55] = add64(res[55], res[55]); // Double(55) + res[56] = add64(res[56], res[56]); // Double(56) + res[57] = add64(res[57], res[57]); // Double(57) + res[58] = add64(res[58], res[58]); // Double(58) + res[59] = add64(res[59], res[59]); // Double(59) + res[48] = fma52lo(res[48], a[24], a[24]); // Add sqr(48) + res[49] = fma52hi(res[49], a[24], a[24]); // Add sqr(48) + res[50] = fma52lo(res[50], a[25], a[25]); // Add sqr(50) + res[51] = fma52hi(res[51], a[25], a[25]); // Add sqr(50) + res[52] = fma52lo(res[52], a[26], a[26]); // Add sqr(52) + res[53] = fma52hi(res[53], a[26], a[26]); // Add sqr(52) + res[54] = fma52lo(res[54], a[27], a[27]); // Add sqr(54) + res[55] = fma52hi(res[55], a[27], a[27]); // Add sqr(54) + res[56] = fma52lo(res[56], a[28], a[28]); // Add sqr(56) + res[57] = fma52hi(res[57], a[28], a[28]); // Add sqr(56) + res[58] = fma52lo(res[58], a[29], a[29]); // Add sqr(58) + res[59] = fma52hi(res[59], a[29], a[29]); // Add sqr(58) + res[60] = fma52lo(res[60], a[29], a[31]); // Sum(60) + res[61] = fma52hi(res[61], a[29], a[31]); // Sum(60) + res[61] = fma52lo(res[61], a[30], a[31]); // Sum(61) + res[62] = fma52hi(res[62], a[30], a[31]); // Sum(61) + res[60] = fma52lo(res[60], a[28], a[32]); // Sum(60) + res[61] = fma52hi(res[61], a[28], a[32]); // Sum(60) + res[61] = fma52lo(res[61], a[29], a[32]); // Sum(61) + res[62] = fma52hi(res[62], a[29], a[32]); // Sum(61) + res[62] = fma52lo(res[62], a[30], a[32]); // Sum(62) + res[63] = fma52hi(res[63], a[30], a[32]); // Sum(62) + res[63] = fma52lo(res[63], a[31], a[32]); // Sum(63) + res[64] = fma52hi(res[64], a[31], a[32]); // Sum(63) + res[60] = fma52lo(res[60], a[27], a[33]); // Sum(60) + res[61] = fma52hi(res[61], a[27], a[33]); // Sum(60) + res[61] = fma52lo(res[61], a[28], a[33]); // Sum(61) + res[62] = fma52hi(res[62], a[28], a[33]); // Sum(61) + res[62] = fma52lo(res[62], a[29], a[33]); // Sum(62) + res[63] = fma52hi(res[63], a[29], a[33]); // Sum(62) + res[63] = fma52lo(res[63], a[30], a[33]); // Sum(63) + res[64] = fma52hi(res[64], a[30], a[33]); // Sum(63) + res[64] = fma52lo(res[64], a[31], a[33]); // Sum(64) + res[65] = fma52hi(res[65], a[31], a[33]); // Sum(64) + res[65] = fma52lo(res[65], a[32], a[33]); // Sum(65) + res[66] = fma52hi(res[66], a[32], a[33]); // Sum(65) + res[60] = fma52lo(res[60], a[26], a[34]); // Sum(60) + res[61] = fma52hi(res[61], a[26], a[34]); // Sum(60) + res[61] = fma52lo(res[61], a[27], a[34]); // Sum(61) + res[62] = fma52hi(res[62], a[27], a[34]); // Sum(61) + res[62] = fma52lo(res[62], a[28], a[34]); // Sum(62) + res[63] = fma52hi(res[63], a[28], a[34]); // Sum(62) + res[63] = fma52lo(res[63], a[29], a[34]); // Sum(63) + res[64] = fma52hi(res[64], a[29], a[34]); // Sum(63) + res[64] = fma52lo(res[64], a[30], a[34]); // Sum(64) + res[65] = fma52hi(res[65], a[30], a[34]); // Sum(64) + res[65] = fma52lo(res[65], a[31], a[34]); // Sum(65) + res[66] = fma52hi(res[66], a[31], a[34]); // Sum(65) + res[66] = fma52lo(res[66], a[32], a[34]); // Sum(66) + res[67] = fma52hi(res[67], a[32], a[34]); // Sum(66) + res[67] = fma52lo(res[67], a[33], a[34]); // Sum(67) + res[68] = fma52hi(res[68], a[33], a[34]); // Sum(67) + res[60] = fma52lo(res[60], a[25], a[35]); // Sum(60) + res[61] = fma52hi(res[61], a[25], a[35]); // Sum(60) + res[61] = fma52lo(res[61], a[26], a[35]); // Sum(61) + res[62] = fma52hi(res[62], a[26], a[35]); // Sum(61) + res[62] = fma52lo(res[62], a[27], a[35]); // Sum(62) + res[63] = fma52hi(res[63], a[27], a[35]); // Sum(62) + res[63] = fma52lo(res[63], a[28], a[35]); // Sum(63) + res[64] = fma52hi(res[64], a[28], a[35]); // Sum(63) + res[64] = fma52lo(res[64], a[29], a[35]); // Sum(64) + res[65] = fma52hi(res[65], a[29], a[35]); // Sum(64) + res[65] = fma52lo(res[65], a[30], a[35]); // Sum(65) + res[66] = fma52hi(res[66], a[30], a[35]); // Sum(65) + res[66] = fma52lo(res[66], a[31], a[35]); // Sum(66) + res[67] = fma52hi(res[67], a[31], a[35]); // Sum(66) + res[67] = fma52lo(res[67], a[32], a[35]); // Sum(67) + res[68] = fma52hi(res[68], a[32], a[35]); // Sum(67) + res[68] = fma52lo(res[68], a[33], a[35]); // Sum(68) + res[69] = fma52hi(res[69], a[33], a[35]); // Sum(68) + res[69] = fma52lo(res[69], a[34], a[35]); // Sum(69) + res[70] = fma52hi(res[70], a[34], a[35]); // Sum(69) + res[60] = fma52lo(res[60], a[24], a[36]); // Sum(60) + res[61] = fma52hi(res[61], a[24], a[36]); // Sum(60) + res[61] = fma52lo(res[61], a[25], a[36]); // Sum(61) + res[62] = fma52hi(res[62], a[25], a[36]); // Sum(61) + res[62] = fma52lo(res[62], a[26], a[36]); // Sum(62) + res[63] = fma52hi(res[63], a[26], a[36]); // Sum(62) + res[63] = fma52lo(res[63], a[27], a[36]); // Sum(63) + res[64] = fma52hi(res[64], a[27], a[36]); // Sum(63) + res[64] = fma52lo(res[64], a[28], a[36]); // Sum(64) + res[65] = fma52hi(res[65], a[28], a[36]); // Sum(64) + res[65] = fma52lo(res[65], a[29], a[36]); // Sum(65) + res[66] = fma52hi(res[66], a[29], a[36]); // Sum(65) + res[66] = fma52lo(res[66], a[30], a[36]); // Sum(66) + res[67] = fma52hi(res[67], a[30], a[36]); // Sum(66) + res[67] = fma52lo(res[67], a[31], a[36]); // Sum(67) + res[68] = fma52hi(res[68], a[31], a[36]); // Sum(67) + res[68] = fma52lo(res[68], a[32], a[36]); // Sum(68) + res[69] = fma52hi(res[69], a[32], a[36]); // Sum(68) + res[69] = fma52lo(res[69], a[33], a[36]); // Sum(69) + res[70] = fma52hi(res[70], a[33], a[36]); // Sum(69) + res[70] = fma52lo(res[70], a[34], a[36]); // Sum(70) + res[71] = fma52hi(res[71], a[34], a[36]); // Sum(70) + res[71] = fma52lo(res[71], a[35], a[36]); // Sum(71) + res[72] = fma52hi(res[72], a[35], a[36]); // Sum(71) + res[60] = fma52lo(res[60], a[23], a[37]); // Sum(60) + res[61] = fma52hi(res[61], a[23], a[37]); // Sum(60) + res[61] = fma52lo(res[61], a[24], a[37]); // Sum(61) + res[62] = fma52hi(res[62], a[24], a[37]); // Sum(61) + res[62] = fma52lo(res[62], a[25], a[37]); // Sum(62) + res[63] = fma52hi(res[63], a[25], a[37]); // Sum(62) + res[63] = fma52lo(res[63], a[26], a[37]); // Sum(63) + res[64] = fma52hi(res[64], a[26], a[37]); // Sum(63) + res[64] = fma52lo(res[64], a[27], a[37]); // Sum(64) + res[65] = fma52hi(res[65], a[27], a[37]); // Sum(64) + res[65] = fma52lo(res[65], a[28], a[37]); // Sum(65) + res[66] = fma52hi(res[66], a[28], a[37]); // Sum(65) + res[66] = fma52lo(res[66], a[29], a[37]); // Sum(66) + res[67] = fma52hi(res[67], a[29], a[37]); // Sum(66) + res[67] = fma52lo(res[67], a[30], a[37]); // Sum(67) + res[68] = fma52hi(res[68], a[30], a[37]); // Sum(67) + res[68] = fma52lo(res[68], a[31], a[37]); // Sum(68) + res[69] = fma52hi(res[69], a[31], a[37]); // Sum(68) + res[69] = fma52lo(res[69], a[32], a[37]); // Sum(69) + res[70] = fma52hi(res[70], a[32], a[37]); // Sum(69) + res[70] = fma52lo(res[70], a[33], a[37]); // Sum(70) + res[71] = fma52hi(res[71], a[33], a[37]); // Sum(70) + res[71] = fma52lo(res[71], a[34], a[37]); // Sum(71) + res[72] = fma52hi(res[72], a[34], a[37]); // Sum(71) + res[60] = fma52lo(res[60], a[22], a[38]); // Sum(60) + res[61] = fma52hi(res[61], a[22], a[38]); // Sum(60) + res[61] = fma52lo(res[61], a[23], a[38]); // Sum(61) + res[62] = fma52hi(res[62], a[23], a[38]); // Sum(61) + res[62] = fma52lo(res[62], a[24], a[38]); // Sum(62) + res[63] = fma52hi(res[63], a[24], a[38]); // Sum(62) + res[63] = fma52lo(res[63], a[25], a[38]); // Sum(63) + res[64] = fma52hi(res[64], a[25], a[38]); // Sum(63) + res[64] = fma52lo(res[64], a[26], a[38]); // Sum(64) + res[65] = fma52hi(res[65], a[26], a[38]); // Sum(64) + res[65] = fma52lo(res[65], a[27], a[38]); // Sum(65) + res[66] = fma52hi(res[66], a[27], a[38]); // Sum(65) + res[66] = fma52lo(res[66], a[28], a[38]); // Sum(66) + res[67] = fma52hi(res[67], a[28], a[38]); // Sum(66) + res[67] = fma52lo(res[67], a[29], a[38]); // Sum(67) + res[68] = fma52hi(res[68], a[29], a[38]); // Sum(67) + res[68] = fma52lo(res[68], a[30], a[38]); // Sum(68) + res[69] = fma52hi(res[69], a[30], a[38]); // Sum(68) + res[69] = fma52lo(res[69], a[31], a[38]); // Sum(69) + res[70] = fma52hi(res[70], a[31], a[38]); // Sum(69) + res[70] = fma52lo(res[70], a[32], a[38]); // Sum(70) + res[71] = fma52hi(res[71], a[32], a[38]); // Sum(70) + res[71] = fma52lo(res[71], a[33], a[38]); // Sum(71) + res[72] = fma52hi(res[72], a[33], a[38]); // Sum(71) + res[60] = fma52lo(res[60], a[21], a[39]); // Sum(60) + res[61] = fma52hi(res[61], a[21], a[39]); // Sum(60) + res[61] = fma52lo(res[61], a[22], a[39]); // Sum(61) + res[62] = fma52hi(res[62], a[22], a[39]); // Sum(61) + res[62] = fma52lo(res[62], a[23], a[39]); // Sum(62) + res[63] = fma52hi(res[63], a[23], a[39]); // Sum(62) + res[63] = fma52lo(res[63], a[24], a[39]); // Sum(63) + res[64] = fma52hi(res[64], a[24], a[39]); // Sum(63) + res[64] = fma52lo(res[64], a[25], a[39]); // Sum(64) + res[65] = fma52hi(res[65], a[25], a[39]); // Sum(64) + res[65] = fma52lo(res[65], a[26], a[39]); // Sum(65) + res[66] = fma52hi(res[66], a[26], a[39]); // Sum(65) + res[66] = fma52lo(res[66], a[27], a[39]); // Sum(66) + res[67] = fma52hi(res[67], a[27], a[39]); // Sum(66) + res[67] = fma52lo(res[67], a[28], a[39]); // Sum(67) + res[68] = fma52hi(res[68], a[28], a[39]); // Sum(67) + res[68] = fma52lo(res[68], a[29], a[39]); // Sum(68) + res[69] = fma52hi(res[69], a[29], a[39]); // Sum(68) + res[69] = fma52lo(res[69], a[30], a[39]); // Sum(69) + res[70] = fma52hi(res[70], a[30], a[39]); // Sum(69) + res[70] = fma52lo(res[70], a[31], a[39]); // Sum(70) + res[71] = fma52hi(res[71], a[31], a[39]); // Sum(70) + res[71] = fma52lo(res[71], a[32], a[39]); // Sum(71) + res[72] = fma52hi(res[72], a[32], a[39]); // Sum(71) + res[60] = add64(res[60], res[60]); // Double(60) + res[61] = add64(res[61], res[61]); // Double(61) + res[62] = add64(res[62], res[62]); // Double(62) + res[63] = add64(res[63], res[63]); // Double(63) + res[64] = add64(res[64], res[64]); // Double(64) + res[65] = add64(res[65], res[65]); // Double(65) + res[66] = add64(res[66], res[66]); // Double(66) + res[67] = add64(res[67], res[67]); // Double(67) + res[68] = add64(res[68], res[68]); // Double(68) + res[69] = add64(res[69], res[69]); // Double(69) + res[70] = add64(res[70], res[70]); // Double(70) + res[71] = add64(res[71], res[71]); // Double(71) + res[60] = fma52lo(res[60], a[30], a[30]); // Add sqr(60) + res[61] = fma52hi(res[61], a[30], a[30]); // Add sqr(60) + res[62] = fma52lo(res[62], a[31], a[31]); // Add sqr(62) + res[63] = fma52hi(res[63], a[31], a[31]); // Add sqr(62) + res[64] = fma52lo(res[64], a[32], a[32]); // Add sqr(64) + res[65] = fma52hi(res[65], a[32], a[32]); // Add sqr(64) + res[66] = fma52lo(res[66], a[33], a[33]); // Add sqr(66) + res[67] = fma52hi(res[67], a[33], a[33]); // Add sqr(66) + res[68] = fma52lo(res[68], a[34], a[34]); // Add sqr(68) + res[69] = fma52hi(res[69], a[34], a[34]); // Add sqr(68) + res[70] = fma52lo(res[70], a[35], a[35]); // Add sqr(70) + res[71] = fma52hi(res[71], a[35], a[35]); // Add sqr(70) + res[72] = fma52lo(res[72], a[35], a[37]); // Sum(72) + res[73] = fma52hi(res[73], a[35], a[37]); // Sum(72) + res[73] = fma52lo(res[73], a[36], a[37]); // Sum(73) + res[74] = fma52hi(res[74], a[36], a[37]); // Sum(73) + res[72] = fma52lo(res[72], a[34], a[38]); // Sum(72) + res[73] = fma52hi(res[73], a[34], a[38]); // Sum(72) + res[73] = fma52lo(res[73], a[35], a[38]); // Sum(73) + res[74] = fma52hi(res[74], a[35], a[38]); // Sum(73) + res[74] = fma52lo(res[74], a[36], a[38]); // Sum(74) + res[75] = fma52hi(res[75], a[36], a[38]); // Sum(74) + res[75] = fma52lo(res[75], a[37], a[38]); // Sum(75) + res[76] = fma52hi(res[76], a[37], a[38]); // Sum(75) + res[72] = fma52lo(res[72], a[33], a[39]); // Sum(72) + res[73] = fma52hi(res[73], a[33], a[39]); // Sum(72) + res[73] = fma52lo(res[73], a[34], a[39]); // Sum(73) + res[74] = fma52hi(res[74], a[34], a[39]); // Sum(73) + res[74] = fma52lo(res[74], a[35], a[39]); // Sum(74) + res[75] = fma52hi(res[75], a[35], a[39]); // Sum(74) + res[75] = fma52lo(res[75], a[36], a[39]); // Sum(75) + res[76] = fma52hi(res[76], a[36], a[39]); // Sum(75) + res[76] = fma52lo(res[76], a[37], a[39]); // Sum(76) + res[77] = fma52hi(res[77], a[37], a[39]); // Sum(76) + res[77] = fma52lo(res[77], a[38], a[39]); // Sum(77) + res[78] = fma52hi(res[78], a[38], a[39]); // Sum(77) + res[72] = add64(res[72], res[72]); // Double(72) + res[73] = add64(res[73], res[73]); // Double(73) + res[74] = add64(res[74], res[74]); // Double(74) + res[75] = add64(res[75], res[75]); // Double(75) + res[76] = add64(res[76], res[76]); // Double(76) + res[77] = add64(res[77], res[77]); // Double(77) + res[78] = add64(res[78], res[78]); // Double(78) + res[72] = fma52lo(res[72], a[36], a[36]); // Add sqr(72) + res[73] = fma52hi(res[73], a[36], a[36]); // Add sqr(72) + res[74] = fma52lo(res[74], a[37], a[37]); // Add sqr(74) + res[75] = fma52hi(res[75], a[37], a[37]); // Add sqr(74) + res[76] = fma52lo(res[76], a[38], a[38]); // Add sqr(76) + res[77] = fma52hi(res[77], a[38], a[38]); // Add sqr(76) + res[78] = fma52lo(res[78], a[39], a[39]); // Add sqr(78) + res[79] = fma52hi(res[79], a[39], a[39]); // Add sqr(78) + + // Montgomery Reduction + int it; + for (it = 0; it < 40; it += 10) { // Reduction step + int jt = 0; + if ((it + 0) > 0) + res[it + 0] = add64(res[it + 0], srli64(res[it + -1], DIGIT_SIZE)); + u[it + 0] = mul52lo(res[it + 0], k); + res[it + jt + 0] = fma52lo(res[it + jt + 0], u[it + 0], m[jt + 0]); + res[it + jt + 1] = fma52hi(res[it + jt + 1], u[it + 0], m[jt + 0]); + res[it + jt + 1] = fma52lo(res[it + jt + 1], u[it + 0], m[jt + 1]); + res[it + jt + 2] = fma52hi(res[it + jt + 2], u[it + 0], m[jt + 1]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 0], m[jt + 2]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 0], m[jt + 2]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 0], m[jt + 3]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 0], m[jt + 3]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 0], m[jt + 4]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 0], m[jt + 4]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 0], m[jt + 5]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 0], m[jt + 5]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 0], m[jt + 6]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 0], m[jt + 6]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 0], m[jt + 7]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 0], m[jt + 7]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 0], m[jt + 8]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 0], m[jt + 8]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 0], m[jt + 9]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 0], m[jt + 9]); + res[it + 1] = add64(res[it + 1], srli64(res[it + 0], DIGIT_SIZE)); + u[it + 1] = mul52lo(res[it + 1], k); + res[it + jt + 1] = fma52lo(res[it + jt + 1], u[it + 1], m[jt + 0]); + res[it + jt + 2] = fma52hi(res[it + jt + 2], u[it + 1], m[jt + 0]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 1], m[jt + 1]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 1], m[jt + 1]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 1], m[jt + 2]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 1], m[jt + 2]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 1], m[jt + 3]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 1], m[jt + 3]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 1], m[jt + 4]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 1], m[jt + 4]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 1], m[jt + 5]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 1], m[jt + 5]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 1], m[jt + 6]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 1], m[jt + 6]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 1], m[jt + 7]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 1], m[jt + 7]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 1], m[jt + 8]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 1], m[jt + 8]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 1], m[jt + 9]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 1], m[jt + 9]); + res[it + 2] = add64(res[it + 2], srli64(res[it + 1], DIGIT_SIZE)); + u[it + 2] = mul52lo(res[it + 2], k); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 2], m[jt + 0]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 2], m[jt + 0]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 2], m[jt + 1]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 2], m[jt + 1]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 2], m[jt + 2]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 2], m[jt + 2]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 2], m[jt + 3]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 2], m[jt + 3]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 2], m[jt + 4]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 2], m[jt + 4]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 2], m[jt + 5]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 2], m[jt + 5]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 2], m[jt + 6]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 2], m[jt + 6]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 2], m[jt + 7]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 2], m[jt + 7]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 2], m[jt + 8]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 2], m[jt + 8]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 2], m[jt + 9]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 2], m[jt + 9]); + res[it + 3] = add64(res[it + 3], srli64(res[it + 2], DIGIT_SIZE)); + u[it + 3] = mul52lo(res[it + 3], k); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 3], m[jt + 0]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 3], m[jt + 0]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 3], m[jt + 1]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 3], m[jt + 1]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 3], m[jt + 2]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 3], m[jt + 2]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 3], m[jt + 3]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 3], m[jt + 3]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 3], m[jt + 4]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 3], m[jt + 4]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 3], m[jt + 5]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 3], m[jt + 5]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 3], m[jt + 6]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 3], m[jt + 6]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 3], m[jt + 7]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 3], m[jt + 7]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 3], m[jt + 8]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 3], m[jt + 8]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 3], m[jt + 9]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 3], m[jt + 9]); + res[it + 4] = add64(res[it + 4], srli64(res[it + 3], DIGIT_SIZE)); + u[it + 4] = mul52lo(res[it + 4], k); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 4], m[jt + 0]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 4], m[jt + 0]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 4], m[jt + 1]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 4], m[jt + 1]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 4], m[jt + 2]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 4], m[jt + 2]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 4], m[jt + 3]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 4], m[jt + 3]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 4], m[jt + 4]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 4], m[jt + 4]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 4], m[jt + 5]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 4], m[jt + 5]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 4], m[jt + 6]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 4], m[jt + 6]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 4], m[jt + 7]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 4], m[jt + 7]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 4], m[jt + 8]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 4], m[jt + 8]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 4], m[jt + 9]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 4], m[jt + 9]); + res[it + 5] = add64(res[it + 5], srli64(res[it + 4], DIGIT_SIZE)); + u[it + 5] = mul52lo(res[it + 5], k); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 5], m[jt + 0]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 5], m[jt + 0]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 5], m[jt + 1]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 5], m[jt + 1]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 5], m[jt + 2]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 5], m[jt + 2]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 5], m[jt + 3]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 5], m[jt + 3]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 5], m[jt + 4]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 5], m[jt + 4]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 5], m[jt + 5]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 5], m[jt + 5]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 5], m[jt + 6]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 5], m[jt + 6]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 5], m[jt + 7]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 5], m[jt + 7]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 5], m[jt + 8]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 5], m[jt + 8]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 5], m[jt + 9]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 5], m[jt + 9]); + res[it + 6] = add64(res[it + 6], srli64(res[it + 5], DIGIT_SIZE)); + u[it + 6] = mul52lo(res[it + 6], k); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 6], m[jt + 0]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 6], m[jt + 0]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 6], m[jt + 1]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 6], m[jt + 1]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 6], m[jt + 2]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 6], m[jt + 2]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 6], m[jt + 3]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 6], m[jt + 3]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 6], m[jt + 4]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 6], m[jt + 4]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 6], m[jt + 5]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 6], m[jt + 5]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 6], m[jt + 6]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 6], m[jt + 6]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 6], m[jt + 7]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 6], m[jt + 7]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 6], m[jt + 8]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 6], m[jt + 8]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 6], m[jt + 9]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 6], m[jt + 9]); + res[it + 7] = add64(res[it + 7], srli64(res[it + 6], DIGIT_SIZE)); + u[it + 7] = mul52lo(res[it + 7], k); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 7], m[jt + 0]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 7], m[jt + 0]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 7], m[jt + 1]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 7], m[jt + 1]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 7], m[jt + 2]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 7], m[jt + 2]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 7], m[jt + 3]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 7], m[jt + 3]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 7], m[jt + 4]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 7], m[jt + 4]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 7], m[jt + 5]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 7], m[jt + 5]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 7], m[jt + 6]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 7], m[jt + 6]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 7], m[jt + 7]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 7], m[jt + 7]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 7], m[jt + 8]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 7], m[jt + 8]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 7], m[jt + 9]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 7], m[jt + 9]); + res[it + 8] = add64(res[it + 8], srli64(res[it + 7], DIGIT_SIZE)); + u[it + 8] = mul52lo(res[it + 8], k); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 8], m[jt + 0]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 8], m[jt + 0]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 8], m[jt + 1]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 8], m[jt + 1]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 8], m[jt + 2]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 8], m[jt + 2]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 8], m[jt + 3]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 8], m[jt + 3]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 8], m[jt + 4]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 8], m[jt + 4]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 8], m[jt + 5]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 8], m[jt + 5]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 8], m[jt + 6]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 8], m[jt + 6]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 8], m[jt + 7]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 8], m[jt + 7]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 8], m[jt + 8]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 8], m[jt + 8]); + res[it + jt + 17] = fma52lo(res[it + jt + 17], u[it + 8], m[jt + 9]); + res[it + jt + 18] = fma52hi(res[it + jt + 18], u[it + 8], m[jt + 9]); + res[it + 9] = add64(res[it + 9], srli64(res[it + 8], DIGIT_SIZE)); + u[it + 9] = mul52lo(res[it + 9], k); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 9], m[jt + 0]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 9], m[jt + 0]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 9], m[jt + 1]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 9], m[jt + 1]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 9], m[jt + 2]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 9], m[jt + 2]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 9], m[jt + 3]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 9], m[jt + 3]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 9], m[jt + 4]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 9], m[jt + 4]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 9], m[jt + 5]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 9], m[jt + 5]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 9], m[jt + 6]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 9], m[jt + 6]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 9], m[jt + 7]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 9], m[jt + 7]); + res[it + jt + 17] = fma52lo(res[it + jt + 17], u[it + 9], m[jt + 8]); + res[it + jt + 18] = fma52hi(res[it + jt + 18], u[it + 9], m[jt + 8]); + res[it + jt + 18] = fma52lo(res[it + jt + 18], u[it + 9], m[jt + 9]); + res[it + jt + 19] = fma52hi(res[it + jt + 19], u[it + 9], m[jt + 9]); + + for (jt = 10; jt < 40; jt += 10) { // Poly tile + res[it + jt + 0] = fma52lo(res[it + jt + 0], u[it + 0], m[jt + 0]); + res[it + jt + 1] = fma52hi(res[it + jt + 1], u[it + 0], m[jt + 0]); + res[it + jt + 1] = fma52lo(res[it + jt + 1], u[it + 0], m[jt + 1]); + res[it + jt + 2] = fma52hi(res[it + jt + 2], u[it + 0], m[jt + 1]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 0], m[jt + 2]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 0], m[jt + 2]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 0], m[jt + 3]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 0], m[jt + 3]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 0], m[jt + 4]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 0], m[jt + 4]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 0], m[jt + 5]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 0], m[jt + 5]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 0], m[jt + 6]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 0], m[jt + 6]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 0], m[jt + 7]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 0], m[jt + 7]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 0], m[jt + 8]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 0], m[jt + 8]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 0], m[jt + 9]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 0], m[jt + 9]); + res[it + jt + 1] = fma52lo(res[it + jt + 1], u[it + 1], m[jt + 0]); + res[it + jt + 2] = fma52hi(res[it + jt + 2], u[it + 1], m[jt + 0]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 1], m[jt + 1]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 1], m[jt + 1]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 1], m[jt + 2]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 1], m[jt + 2]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 1], m[jt + 3]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 1], m[jt + 3]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 1], m[jt + 4]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 1], m[jt + 4]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 1], m[jt + 5]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 1], m[jt + 5]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 1], m[jt + 6]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 1], m[jt + 6]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 1], m[jt + 7]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 1], m[jt + 7]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 1], m[jt + 8]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 1], m[jt + 8]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 1], m[jt + 9]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 1], m[jt + 9]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 2], m[jt + 0]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 2], m[jt + 0]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 2], m[jt + 1]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 2], m[jt + 1]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 2], m[jt + 2]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 2], m[jt + 2]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 2], m[jt + 3]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 2], m[jt + 3]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 2], m[jt + 4]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 2], m[jt + 4]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 2], m[jt + 5]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 2], m[jt + 5]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 2], m[jt + 6]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 2], m[jt + 6]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 2], m[jt + 7]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 2], m[jt + 7]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 2], m[jt + 8]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 2], m[jt + 8]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 2], m[jt + 9]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 2], m[jt + 9]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 3], m[jt + 0]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 3], m[jt + 0]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 3], m[jt + 1]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 3], m[jt + 1]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 3], m[jt + 2]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 3], m[jt + 2]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 3], m[jt + 3]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 3], m[jt + 3]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 3], m[jt + 4]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 3], m[jt + 4]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 3], m[jt + 5]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 3], m[jt + 5]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 3], m[jt + 6]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 3], m[jt + 6]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 3], m[jt + 7]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 3], m[jt + 7]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 3], m[jt + 8]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 3], m[jt + 8]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 3], m[jt + 9]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 3], m[jt + 9]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 4], m[jt + 0]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 4], m[jt + 0]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 4], m[jt + 1]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 4], m[jt + 1]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 4], m[jt + 2]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 4], m[jt + 2]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 4], m[jt + 3]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 4], m[jt + 3]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 4], m[jt + 4]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 4], m[jt + 4]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 4], m[jt + 5]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 4], m[jt + 5]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 4], m[jt + 6]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 4], m[jt + 6]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 4], m[jt + 7]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 4], m[jt + 7]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 4], m[jt + 8]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 4], m[jt + 8]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 4], m[jt + 9]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 4], m[jt + 9]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 5], m[jt + 0]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 5], m[jt + 0]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 5], m[jt + 1]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 5], m[jt + 1]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 5], m[jt + 2]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 5], m[jt + 2]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 5], m[jt + 3]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 5], m[jt + 3]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 5], m[jt + 4]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 5], m[jt + 4]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 5], m[jt + 5]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 5], m[jt + 5]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 5], m[jt + 6]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 5], m[jt + 6]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 5], m[jt + 7]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 5], m[jt + 7]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 5], m[jt + 8]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 5], m[jt + 8]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 5], m[jt + 9]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 5], m[jt + 9]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 6], m[jt + 0]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 6], m[jt + 0]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 6], m[jt + 1]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 6], m[jt + 1]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 6], m[jt + 2]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 6], m[jt + 2]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 6], m[jt + 3]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 6], m[jt + 3]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 6], m[jt + 4]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 6], m[jt + 4]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 6], m[jt + 5]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 6], m[jt + 5]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 6], m[jt + 6]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 6], m[jt + 6]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 6], m[jt + 7]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 6], m[jt + 7]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 6], m[jt + 8]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 6], m[jt + 8]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 6], m[jt + 9]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 6], m[jt + 9]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 7], m[jt + 0]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 7], m[jt + 0]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 7], m[jt + 1]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 7], m[jt + 1]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 7], m[jt + 2]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 7], m[jt + 2]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 7], m[jt + 3]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 7], m[jt + 3]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 7], m[jt + 4]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 7], m[jt + 4]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 7], m[jt + 5]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 7], m[jt + 5]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 7], m[jt + 6]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 7], m[jt + 6]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 7], m[jt + 7]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 7], m[jt + 7]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 7], m[jt + 8]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 7], m[jt + 8]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 7], m[jt + 9]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 7], m[jt + 9]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 8], m[jt + 0]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 8], m[jt + 0]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 8], m[jt + 1]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 8], m[jt + 1]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 8], m[jt + 2]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 8], m[jt + 2]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 8], m[jt + 3]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 8], m[jt + 3]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 8], m[jt + 4]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 8], m[jt + 4]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 8], m[jt + 5]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 8], m[jt + 5]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 8], m[jt + 6]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 8], m[jt + 6]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 8], m[jt + 7]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 8], m[jt + 7]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 8], m[jt + 8]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 8], m[jt + 8]); + res[it + jt + 17] = fma52lo(res[it + jt + 17], u[it + 8], m[jt + 9]); + res[it + jt + 18] = fma52hi(res[it + jt + 18], u[it + 8], m[jt + 9]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 9], m[jt + 0]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 9], m[jt + 0]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 9], m[jt + 1]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 9], m[jt + 1]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 9], m[jt + 2]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 9], m[jt + 2]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 9], m[jt + 3]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 9], m[jt + 3]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 9], m[jt + 4]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 9], m[jt + 4]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 9], m[jt + 5]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 9], m[jt + 5]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 9], m[jt + 6]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 9], m[jt + 6]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 9], m[jt + 7]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 9], m[jt + 7]); + res[it + jt + 17] = fma52lo(res[it + jt + 17], u[it + 9], m[jt + 8]); + res[it + jt + 18] = fma52hi(res[it + jt + 18], u[it + 9], m[jt + 8]); + res[it + jt + 18] = fma52lo(res[it + jt + 18], u[it + 9], m[jt + 9]); + res[it + jt + 19] = fma52hi(res[it + jt + 19], u[it + 9], m[jt + 9]); + } + } + + // Normalization + res[40] = add64(res[40], srli64(res[39], DIGIT_SIZE)); + r[0] = and64_const(res[40], DIGIT_MASK); + res[41] = add64(res[41], srli64(res[40], DIGIT_SIZE)); + r[1] = and64_const(res[41], DIGIT_MASK); + res[42] = add64(res[42], srli64(res[41], DIGIT_SIZE)); + r[2] = and64_const(res[42], DIGIT_MASK); + res[43] = add64(res[43], srli64(res[42], DIGIT_SIZE)); + r[3] = and64_const(res[43], DIGIT_MASK); + res[44] = add64(res[44], srli64(res[43], DIGIT_SIZE)); + r[4] = and64_const(res[44], DIGIT_MASK); + res[45] = add64(res[45], srli64(res[44], DIGIT_SIZE)); + r[5] = and64_const(res[45], DIGIT_MASK); + res[46] = add64(res[46], srli64(res[45], DIGIT_SIZE)); + r[6] = and64_const(res[46], DIGIT_MASK); + res[47] = add64(res[47], srli64(res[46], DIGIT_SIZE)); + r[7] = and64_const(res[47], DIGIT_MASK); + res[48] = add64(res[48], srli64(res[47], DIGIT_SIZE)); + r[8] = and64_const(res[48], DIGIT_MASK); + res[49] = add64(res[49], srli64(res[48], DIGIT_SIZE)); + r[9] = and64_const(res[49], DIGIT_MASK); + res[50] = add64(res[50], srli64(res[49], DIGIT_SIZE)); + r[10] = and64_const(res[50], DIGIT_MASK); + res[51] = add64(res[51], srli64(res[50], DIGIT_SIZE)); + r[11] = and64_const(res[51], DIGIT_MASK); + res[52] = add64(res[52], srli64(res[51], DIGIT_SIZE)); + r[12] = and64_const(res[52], DIGIT_MASK); + res[53] = add64(res[53], srli64(res[52], DIGIT_SIZE)); + r[13] = and64_const(res[53], DIGIT_MASK); + res[54] = add64(res[54], srli64(res[53], DIGIT_SIZE)); + r[14] = and64_const(res[54], DIGIT_MASK); + res[55] = add64(res[55], srli64(res[54], DIGIT_SIZE)); + r[15] = and64_const(res[55], DIGIT_MASK); + res[56] = add64(res[56], srli64(res[55], DIGIT_SIZE)); + r[16] = and64_const(res[56], DIGIT_MASK); + res[57] = add64(res[57], srli64(res[56], DIGIT_SIZE)); + r[17] = and64_const(res[57], DIGIT_MASK); + res[58] = add64(res[58], srli64(res[57], DIGIT_SIZE)); + r[18] = and64_const(res[58], DIGIT_MASK); + res[59] = add64(res[59], srli64(res[58], DIGIT_SIZE)); + r[19] = and64_const(res[59], DIGIT_MASK); + res[60] = add64(res[60], srli64(res[59], DIGIT_SIZE)); + r[20] = and64_const(res[60], DIGIT_MASK); + res[61] = add64(res[61], srli64(res[60], DIGIT_SIZE)); + r[21] = and64_const(res[61], DIGIT_MASK); + res[62] = add64(res[62], srli64(res[61], DIGIT_SIZE)); + r[22] = and64_const(res[62], DIGIT_MASK); + res[63] = add64(res[63], srli64(res[62], DIGIT_SIZE)); + r[23] = and64_const(res[63], DIGIT_MASK); + res[64] = add64(res[64], srli64(res[63], DIGIT_SIZE)); + r[24] = and64_const(res[64], DIGIT_MASK); + res[65] = add64(res[65], srli64(res[64], DIGIT_SIZE)); + r[25] = and64_const(res[65], DIGIT_MASK); + res[66] = add64(res[66], srli64(res[65], DIGIT_SIZE)); + r[26] = and64_const(res[66], DIGIT_MASK); + res[67] = add64(res[67], srli64(res[66], DIGIT_SIZE)); + r[27] = and64_const(res[67], DIGIT_MASK); + res[68] = add64(res[68], srli64(res[67], DIGIT_SIZE)); + r[28] = and64_const(res[68], DIGIT_MASK); + res[69] = add64(res[69], srli64(res[68], DIGIT_SIZE)); + r[29] = and64_const(res[69], DIGIT_MASK); + res[70] = add64(res[70], srli64(res[69], DIGIT_SIZE)); + r[30] = and64_const(res[70], DIGIT_MASK); + res[71] = add64(res[71], srli64(res[70], DIGIT_SIZE)); + r[31] = and64_const(res[71], DIGIT_MASK); + res[72] = add64(res[72], srli64(res[71], DIGIT_SIZE)); + r[32] = and64_const(res[72], DIGIT_MASK); + res[73] = add64(res[73], srli64(res[72], DIGIT_SIZE)); + r[33] = and64_const(res[73], DIGIT_MASK); + res[74] = add64(res[74], srli64(res[73], DIGIT_SIZE)); + r[34] = and64_const(res[74], DIGIT_MASK); + res[75] = add64(res[75], srli64(res[74], DIGIT_SIZE)); + r[35] = and64_const(res[75], DIGIT_MASK); + res[76] = add64(res[76], srli64(res[75], DIGIT_SIZE)); + r[36] = and64_const(res[76], DIGIT_MASK); + res[77] = add64(res[77], srli64(res[76], DIGIT_SIZE)); + r[37] = and64_const(res[77], DIGIT_MASK); + res[78] = add64(res[78], srli64(res[77], DIGIT_SIZE)); + r[38] = and64_const(res[78], DIGIT_MASK); + res[79] = add64(res[79], srli64(res[78], DIGIT_SIZE)); + r[39] = and64_const(res[79], DIGIT_MASK); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams52x60_diagonal_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams52x60_diagonal_mb8.c new file mode 100644 index 0000000..9b26439 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams52x60_diagonal_mb8.c @@ -0,0 +1,4371 @@ +/******************************************************************************* + * Copyright (C) 2019 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include + +#ifdef __GNUC__ +#define ASM(a) __asm__(a); +#else +#define ASM(a) +#endif + +void AMS52x60_diagonal_mb8(int64u *out_mb, const int64u *inpA_mb, + const int64u *inpM_mb, const int64u *k0_mb) { + __ALIGN64 U64 res[120]; + __ALIGN64 U64 u[60]; + U64 k; + U64 *a = (U64 *)inpA_mb; + U64 *m = (U64 *)inpM_mb; + U64 *r = (U64 *)out_mb; + + k = loadu64((U64 *)k0_mb); + int i; + for (i = 0; i < 120; ++i) + res[i] = get_zero64(); + + res[1] = fma52lo(res[1], a[0], a[1]); // Sum(1) + res[2] = fma52hi(res[2], a[0], a[1]); // Sum(1) + res[2] = fma52lo(res[2], a[0], a[2]); // Sum(2) + res[3] = fma52hi(res[3], a[0], a[2]); // Sum(2) + res[3] = fma52lo(res[3], a[1], a[2]); // Sum(3) + res[4] = fma52hi(res[4], a[1], a[2]); // Sum(3) + res[3] = fma52lo(res[3], a[0], a[3]); // Sum(3) + res[4] = fma52hi(res[4], a[0], a[3]); // Sum(3) + res[4] = fma52lo(res[4], a[1], a[3]); // Sum(4) + res[5] = fma52hi(res[5], a[1], a[3]); // Sum(4) + res[5] = fma52lo(res[5], a[2], a[3]); // Sum(5) + res[6] = fma52hi(res[6], a[2], a[3]); // Sum(5) + res[4] = fma52lo(res[4], a[0], a[4]); // Sum(4) + res[5] = fma52hi(res[5], a[0], a[4]); // Sum(4) + res[5] = fma52lo(res[5], a[1], a[4]); // Sum(5) + res[6] = fma52hi(res[6], a[1], a[4]); // Sum(5) + res[6] = fma52lo(res[6], a[2], a[4]); // Sum(6) + res[7] = fma52hi(res[7], a[2], a[4]); // Sum(6) + res[7] = fma52lo(res[7], a[3], a[4]); // Sum(7) + res[8] = fma52hi(res[8], a[3], a[4]); // Sum(7) + res[5] = fma52lo(res[5], a[0], a[5]); // Sum(5) + res[6] = fma52hi(res[6], a[0], a[5]); // Sum(5) + res[6] = fma52lo(res[6], a[1], a[5]); // Sum(6) + res[7] = fma52hi(res[7], a[1], a[5]); // Sum(6) + res[7] = fma52lo(res[7], a[2], a[5]); // Sum(7) + res[8] = fma52hi(res[8], a[2], a[5]); // Sum(7) + res[8] = fma52lo(res[8], a[3], a[5]); // Sum(8) + res[9] = fma52hi(res[9], a[3], a[5]); // Sum(8) + res[9] = fma52lo(res[9], a[4], a[5]); // Sum(9) + res[10] = fma52hi(res[10], a[4], a[5]); // Sum(9) + res[6] = fma52lo(res[6], a[0], a[6]); // Sum(6) + res[7] = fma52hi(res[7], a[0], a[6]); // Sum(6) + res[7] = fma52lo(res[7], a[1], a[6]); // Sum(7) + res[8] = fma52hi(res[8], a[1], a[6]); // Sum(7) + res[8] = fma52lo(res[8], a[2], a[6]); // Sum(8) + res[9] = fma52hi(res[9], a[2], a[6]); // Sum(8) + res[9] = fma52lo(res[9], a[3], a[6]); // Sum(9) + res[10] = fma52hi(res[10], a[3], a[6]); // Sum(9) + res[10] = fma52lo(res[10], a[4], a[6]); // Sum(10) + res[11] = fma52hi(res[11], a[4], a[6]); // Sum(10) + res[11] = fma52lo(res[11], a[5], a[6]); // Sum(11) + res[12] = fma52hi(res[12], a[5], a[6]); // Sum(11) + res[7] = fma52lo(res[7], a[0], a[7]); // Sum(7) + res[8] = fma52hi(res[8], a[0], a[7]); // Sum(7) + res[8] = fma52lo(res[8], a[1], a[7]); // Sum(8) + res[9] = fma52hi(res[9], a[1], a[7]); // Sum(8) + res[9] = fma52lo(res[9], a[2], a[7]); // Sum(9) + res[10] = fma52hi(res[10], a[2], a[7]); // Sum(9) + res[10] = fma52lo(res[10], a[3], a[7]); // Sum(10) + res[11] = fma52hi(res[11], a[3], a[7]); // Sum(10) + res[11] = fma52lo(res[11], a[4], a[7]); // Sum(11) + res[12] = fma52hi(res[12], a[4], a[7]); // Sum(11) + res[8] = fma52lo(res[8], a[0], a[8]); // Sum(8) + res[9] = fma52hi(res[9], a[0], a[8]); // Sum(8) + res[9] = fma52lo(res[9], a[1], a[8]); // Sum(9) + res[10] = fma52hi(res[10], a[1], a[8]); // Sum(9) + res[10] = fma52lo(res[10], a[2], a[8]); // Sum(10) + res[11] = fma52hi(res[11], a[2], a[8]); // Sum(10) + res[11] = fma52lo(res[11], a[3], a[8]); // Sum(11) + res[12] = fma52hi(res[12], a[3], a[8]); // Sum(11) + res[9] = fma52lo(res[9], a[0], a[9]); // Sum(9) + res[10] = fma52hi(res[10], a[0], a[9]); // Sum(9) + res[10] = fma52lo(res[10], a[1], a[9]); // Sum(10) + res[11] = fma52hi(res[11], a[1], a[9]); // Sum(10) + res[11] = fma52lo(res[11], a[2], a[9]); // Sum(11) + res[12] = fma52hi(res[12], a[2], a[9]); // Sum(11) + res[10] = fma52lo(res[10], a[0], a[10]); // Sum(10) + res[11] = fma52hi(res[11], a[0], a[10]); // Sum(10) + res[11] = fma52lo(res[11], a[1], a[10]); // Sum(11) + res[12] = fma52hi(res[12], a[1], a[10]); // Sum(11) + res[11] = fma52lo(res[11], a[0], a[11]); // Sum(11) + res[12] = fma52hi(res[12], a[0], a[11]); // Sum(11) + res[0] = add64(res[0], res[0]); // Double(0) + res[1] = add64(res[1], res[1]); // Double(1) + res[2] = add64(res[2], res[2]); // Double(2) + res[3] = add64(res[3], res[3]); // Double(3) + res[4] = add64(res[4], res[4]); // Double(4) + res[5] = add64(res[5], res[5]); // Double(5) + res[6] = add64(res[6], res[6]); // Double(6) + res[7] = add64(res[7], res[7]); // Double(7) + res[8] = add64(res[8], res[8]); // Double(8) + res[9] = add64(res[9], res[9]); // Double(9) + res[10] = add64(res[10], res[10]); // Double(10) + res[11] = add64(res[11], res[11]); // Double(11) + res[0] = fma52lo(res[0], a[0], a[0]); // Add sqr(0) + res[1] = fma52hi(res[1], a[0], a[0]); // Add sqr(0) + res[2] = fma52lo(res[2], a[1], a[1]); // Add sqr(2) + res[3] = fma52hi(res[3], a[1], a[1]); // Add sqr(2) + res[4] = fma52lo(res[4], a[2], a[2]); // Add sqr(4) + res[5] = fma52hi(res[5], a[2], a[2]); // Add sqr(4) + res[6] = fma52lo(res[6], a[3], a[3]); // Add sqr(6) + res[7] = fma52hi(res[7], a[3], a[3]); // Add sqr(6) + res[8] = fma52lo(res[8], a[4], a[4]); // Add sqr(8) + res[9] = fma52hi(res[9], a[4], a[4]); // Add sqr(8) + res[10] = fma52lo(res[10], a[5], a[5]); // Add sqr(10) + res[11] = fma52hi(res[11], a[5], a[5]); // Add sqr(10) + res[12] = fma52lo(res[12], a[5], a[7]); // Sum(12) + res[13] = fma52hi(res[13], a[5], a[7]); // Sum(12) + res[13] = fma52lo(res[13], a[6], a[7]); // Sum(13) + res[14] = fma52hi(res[14], a[6], a[7]); // Sum(13) + res[12] = fma52lo(res[12], a[4], a[8]); // Sum(12) + res[13] = fma52hi(res[13], a[4], a[8]); // Sum(12) + res[13] = fma52lo(res[13], a[5], a[8]); // Sum(13) + res[14] = fma52hi(res[14], a[5], a[8]); // Sum(13) + res[14] = fma52lo(res[14], a[6], a[8]); // Sum(14) + res[15] = fma52hi(res[15], a[6], a[8]); // Sum(14) + res[15] = fma52lo(res[15], a[7], a[8]); // Sum(15) + res[16] = fma52hi(res[16], a[7], a[8]); // Sum(15) + res[12] = fma52lo(res[12], a[3], a[9]); // Sum(12) + res[13] = fma52hi(res[13], a[3], a[9]); // Sum(12) + res[13] = fma52lo(res[13], a[4], a[9]); // Sum(13) + res[14] = fma52hi(res[14], a[4], a[9]); // Sum(13) + res[14] = fma52lo(res[14], a[5], a[9]); // Sum(14) + res[15] = fma52hi(res[15], a[5], a[9]); // Sum(14) + res[15] = fma52lo(res[15], a[6], a[9]); // Sum(15) + res[16] = fma52hi(res[16], a[6], a[9]); // Sum(15) + res[16] = fma52lo(res[16], a[7], a[9]); // Sum(16) + res[17] = fma52hi(res[17], a[7], a[9]); // Sum(16) + res[17] = fma52lo(res[17], a[8], a[9]); // Sum(17) + res[18] = fma52hi(res[18], a[8], a[9]); // Sum(17) + res[12] = fma52lo(res[12], a[2], a[10]); // Sum(12) + res[13] = fma52hi(res[13], a[2], a[10]); // Sum(12) + res[13] = fma52lo(res[13], a[3], a[10]); // Sum(13) + res[14] = fma52hi(res[14], a[3], a[10]); // Sum(13) + res[14] = fma52lo(res[14], a[4], a[10]); // Sum(14) + res[15] = fma52hi(res[15], a[4], a[10]); // Sum(14) + res[15] = fma52lo(res[15], a[5], a[10]); // Sum(15) + res[16] = fma52hi(res[16], a[5], a[10]); // Sum(15) + res[16] = fma52lo(res[16], a[6], a[10]); // Sum(16) + res[17] = fma52hi(res[17], a[6], a[10]); // Sum(16) + res[17] = fma52lo(res[17], a[7], a[10]); // Sum(17) + res[18] = fma52hi(res[18], a[7], a[10]); // Sum(17) + res[18] = fma52lo(res[18], a[8], a[10]); // Sum(18) + res[19] = fma52hi(res[19], a[8], a[10]); // Sum(18) + res[19] = fma52lo(res[19], a[9], a[10]); // Sum(19) + res[20] = fma52hi(res[20], a[9], a[10]); // Sum(19) + res[12] = fma52lo(res[12], a[1], a[11]); // Sum(12) + res[13] = fma52hi(res[13], a[1], a[11]); // Sum(12) + res[13] = fma52lo(res[13], a[2], a[11]); // Sum(13) + res[14] = fma52hi(res[14], a[2], a[11]); // Sum(13) + res[14] = fma52lo(res[14], a[3], a[11]); // Sum(14) + res[15] = fma52hi(res[15], a[3], a[11]); // Sum(14) + res[15] = fma52lo(res[15], a[4], a[11]); // Sum(15) + res[16] = fma52hi(res[16], a[4], a[11]); // Sum(15) + res[16] = fma52lo(res[16], a[5], a[11]); // Sum(16) + res[17] = fma52hi(res[17], a[5], a[11]); // Sum(16) + res[17] = fma52lo(res[17], a[6], a[11]); // Sum(17) + res[18] = fma52hi(res[18], a[6], a[11]); // Sum(17) + res[18] = fma52lo(res[18], a[7], a[11]); // Sum(18) + res[19] = fma52hi(res[19], a[7], a[11]); // Sum(18) + res[19] = fma52lo(res[19], a[8], a[11]); // Sum(19) + res[20] = fma52hi(res[20], a[8], a[11]); // Sum(19) + res[20] = fma52lo(res[20], a[9], a[11]); // Sum(20) + res[21] = fma52hi(res[21], a[9], a[11]); // Sum(20) + res[21] = fma52lo(res[21], a[10], a[11]); // Sum(21) + res[22] = fma52hi(res[22], a[10], a[11]); // Sum(21) + res[12] = fma52lo(res[12], a[0], a[12]); // Sum(12) + res[13] = fma52hi(res[13], a[0], a[12]); // Sum(12) + res[13] = fma52lo(res[13], a[1], a[12]); // Sum(13) + res[14] = fma52hi(res[14], a[1], a[12]); // Sum(13) + res[14] = fma52lo(res[14], a[2], a[12]); // Sum(14) + res[15] = fma52hi(res[15], a[2], a[12]); // Sum(14) + res[15] = fma52lo(res[15], a[3], a[12]); // Sum(15) + res[16] = fma52hi(res[16], a[3], a[12]); // Sum(15) + res[16] = fma52lo(res[16], a[4], a[12]); // Sum(16) + res[17] = fma52hi(res[17], a[4], a[12]); // Sum(16) + res[17] = fma52lo(res[17], a[5], a[12]); // Sum(17) + res[18] = fma52hi(res[18], a[5], a[12]); // Sum(17) + res[18] = fma52lo(res[18], a[6], a[12]); // Sum(18) + res[19] = fma52hi(res[19], a[6], a[12]); // Sum(18) + res[19] = fma52lo(res[19], a[7], a[12]); // Sum(19) + res[20] = fma52hi(res[20], a[7], a[12]); // Sum(19) + res[20] = fma52lo(res[20], a[8], a[12]); // Sum(20) + res[21] = fma52hi(res[21], a[8], a[12]); // Sum(20) + res[21] = fma52lo(res[21], a[9], a[12]); // Sum(21) + res[22] = fma52hi(res[22], a[9], a[12]); // Sum(21) + res[22] = fma52lo(res[22], a[10], a[12]); // Sum(22) + res[23] = fma52hi(res[23], a[10], a[12]); // Sum(22) + res[23] = fma52lo(res[23], a[11], a[12]); // Sum(23) + res[24] = fma52hi(res[24], a[11], a[12]); // Sum(23) + res[13] = fma52lo(res[13], a[0], a[13]); // Sum(13) + res[14] = fma52hi(res[14], a[0], a[13]); // Sum(13) + res[14] = fma52lo(res[14], a[1], a[13]); // Sum(14) + res[15] = fma52hi(res[15], a[1], a[13]); // Sum(14) + res[15] = fma52lo(res[15], a[2], a[13]); // Sum(15) + res[16] = fma52hi(res[16], a[2], a[13]); // Sum(15) + res[16] = fma52lo(res[16], a[3], a[13]); // Sum(16) + res[17] = fma52hi(res[17], a[3], a[13]); // Sum(16) + res[17] = fma52lo(res[17], a[4], a[13]); // Sum(17) + res[18] = fma52hi(res[18], a[4], a[13]); // Sum(17) + res[18] = fma52lo(res[18], a[5], a[13]); // Sum(18) + res[19] = fma52hi(res[19], a[5], a[13]); // Sum(18) + res[19] = fma52lo(res[19], a[6], a[13]); // Sum(19) + res[20] = fma52hi(res[20], a[6], a[13]); // Sum(19) + res[20] = fma52lo(res[20], a[7], a[13]); // Sum(20) + res[21] = fma52hi(res[21], a[7], a[13]); // Sum(20) + res[21] = fma52lo(res[21], a[8], a[13]); // Sum(21) + res[22] = fma52hi(res[22], a[8], a[13]); // Sum(21) + res[22] = fma52lo(res[22], a[9], a[13]); // Sum(22) + res[23] = fma52hi(res[23], a[9], a[13]); // Sum(22) + res[23] = fma52lo(res[23], a[10], a[13]); // Sum(23) + res[24] = fma52hi(res[24], a[10], a[13]); // Sum(23) + res[14] = fma52lo(res[14], a[0], a[14]); // Sum(14) + res[15] = fma52hi(res[15], a[0], a[14]); // Sum(14) + res[15] = fma52lo(res[15], a[1], a[14]); // Sum(15) + res[16] = fma52hi(res[16], a[1], a[14]); // Sum(15) + res[16] = fma52lo(res[16], a[2], a[14]); // Sum(16) + res[17] = fma52hi(res[17], a[2], a[14]); // Sum(16) + res[17] = fma52lo(res[17], a[3], a[14]); // Sum(17) + res[18] = fma52hi(res[18], a[3], a[14]); // Sum(17) + res[18] = fma52lo(res[18], a[4], a[14]); // Sum(18) + res[19] = fma52hi(res[19], a[4], a[14]); // Sum(18) + res[19] = fma52lo(res[19], a[5], a[14]); // Sum(19) + res[20] = fma52hi(res[20], a[5], a[14]); // Sum(19) + res[20] = fma52lo(res[20], a[6], a[14]); // Sum(20) + res[21] = fma52hi(res[21], a[6], a[14]); // Sum(20) + res[21] = fma52lo(res[21], a[7], a[14]); // Sum(21) + res[22] = fma52hi(res[22], a[7], a[14]); // Sum(21) + res[22] = fma52lo(res[22], a[8], a[14]); // Sum(22) + res[23] = fma52hi(res[23], a[8], a[14]); // Sum(22) + res[23] = fma52lo(res[23], a[9], a[14]); // Sum(23) + res[24] = fma52hi(res[24], a[9], a[14]); // Sum(23) + res[15] = fma52lo(res[15], a[0], a[15]); // Sum(15) + res[16] = fma52hi(res[16], a[0], a[15]); // Sum(15) + res[16] = fma52lo(res[16], a[1], a[15]); // Sum(16) + res[17] = fma52hi(res[17], a[1], a[15]); // Sum(16) + res[17] = fma52lo(res[17], a[2], a[15]); // Sum(17) + res[18] = fma52hi(res[18], a[2], a[15]); // Sum(17) + res[18] = fma52lo(res[18], a[3], a[15]); // Sum(18) + res[19] = fma52hi(res[19], a[3], a[15]); // Sum(18) + res[19] = fma52lo(res[19], a[4], a[15]); // Sum(19) + res[20] = fma52hi(res[20], a[4], a[15]); // Sum(19) + res[20] = fma52lo(res[20], a[5], a[15]); // Sum(20) + res[21] = fma52hi(res[21], a[5], a[15]); // Sum(20) + res[21] = fma52lo(res[21], a[6], a[15]); // Sum(21) + res[22] = fma52hi(res[22], a[6], a[15]); // Sum(21) + res[22] = fma52lo(res[22], a[7], a[15]); // Sum(22) + res[23] = fma52hi(res[23], a[7], a[15]); // Sum(22) + res[23] = fma52lo(res[23], a[8], a[15]); // Sum(23) + res[24] = fma52hi(res[24], a[8], a[15]); // Sum(23) + res[16] = fma52lo(res[16], a[0], a[16]); // Sum(16) + res[17] = fma52hi(res[17], a[0], a[16]); // Sum(16) + res[17] = fma52lo(res[17], a[1], a[16]); // Sum(17) + res[18] = fma52hi(res[18], a[1], a[16]); // Sum(17) + res[18] = fma52lo(res[18], a[2], a[16]); // Sum(18) + res[19] = fma52hi(res[19], a[2], a[16]); // Sum(18) + res[19] = fma52lo(res[19], a[3], a[16]); // Sum(19) + res[20] = fma52hi(res[20], a[3], a[16]); // Sum(19) + res[20] = fma52lo(res[20], a[4], a[16]); // Sum(20) + res[21] = fma52hi(res[21], a[4], a[16]); // Sum(20) + res[21] = fma52lo(res[21], a[5], a[16]); // Sum(21) + res[22] = fma52hi(res[22], a[5], a[16]); // Sum(21) + res[22] = fma52lo(res[22], a[6], a[16]); // Sum(22) + res[23] = fma52hi(res[23], a[6], a[16]); // Sum(22) + res[23] = fma52lo(res[23], a[7], a[16]); // Sum(23) + res[24] = fma52hi(res[24], a[7], a[16]); // Sum(23) + res[17] = fma52lo(res[17], a[0], a[17]); // Sum(17) + res[18] = fma52hi(res[18], a[0], a[17]); // Sum(17) + res[18] = fma52lo(res[18], a[1], a[17]); // Sum(18) + res[19] = fma52hi(res[19], a[1], a[17]); // Sum(18) + res[19] = fma52lo(res[19], a[2], a[17]); // Sum(19) + res[20] = fma52hi(res[20], a[2], a[17]); // Sum(19) + res[20] = fma52lo(res[20], a[3], a[17]); // Sum(20) + res[21] = fma52hi(res[21], a[3], a[17]); // Sum(20) + res[21] = fma52lo(res[21], a[4], a[17]); // Sum(21) + res[22] = fma52hi(res[22], a[4], a[17]); // Sum(21) + res[22] = fma52lo(res[22], a[5], a[17]); // Sum(22) + res[23] = fma52hi(res[23], a[5], a[17]); // Sum(22) + res[23] = fma52lo(res[23], a[6], a[17]); // Sum(23) + res[24] = fma52hi(res[24], a[6], a[17]); // Sum(23) + res[18] = fma52lo(res[18], a[0], a[18]); // Sum(18) + res[19] = fma52hi(res[19], a[0], a[18]); // Sum(18) + res[19] = fma52lo(res[19], a[1], a[18]); // Sum(19) + res[20] = fma52hi(res[20], a[1], a[18]); // Sum(19) + res[20] = fma52lo(res[20], a[2], a[18]); // Sum(20) + res[21] = fma52hi(res[21], a[2], a[18]); // Sum(20) + res[21] = fma52lo(res[21], a[3], a[18]); // Sum(21) + res[22] = fma52hi(res[22], a[3], a[18]); // Sum(21) + res[22] = fma52lo(res[22], a[4], a[18]); // Sum(22) + res[23] = fma52hi(res[23], a[4], a[18]); // Sum(22) + res[23] = fma52lo(res[23], a[5], a[18]); // Sum(23) + res[24] = fma52hi(res[24], a[5], a[18]); // Sum(23) + res[19] = fma52lo(res[19], a[0], a[19]); // Sum(19) + res[20] = fma52hi(res[20], a[0], a[19]); // Sum(19) + res[20] = fma52lo(res[20], a[1], a[19]); // Sum(20) + res[21] = fma52hi(res[21], a[1], a[19]); // Sum(20) + res[21] = fma52lo(res[21], a[2], a[19]); // Sum(21) + res[22] = fma52hi(res[22], a[2], a[19]); // Sum(21) + res[22] = fma52lo(res[22], a[3], a[19]); // Sum(22) + res[23] = fma52hi(res[23], a[3], a[19]); // Sum(22) + res[23] = fma52lo(res[23], a[4], a[19]); // Sum(23) + res[24] = fma52hi(res[24], a[4], a[19]); // Sum(23) + res[20] = fma52lo(res[20], a[0], a[20]); // Sum(20) + res[21] = fma52hi(res[21], a[0], a[20]); // Sum(20) + res[21] = fma52lo(res[21], a[1], a[20]); // Sum(21) + res[22] = fma52hi(res[22], a[1], a[20]); // Sum(21) + res[22] = fma52lo(res[22], a[2], a[20]); // Sum(22) + res[23] = fma52hi(res[23], a[2], a[20]); // Sum(22) + res[23] = fma52lo(res[23], a[3], a[20]); // Sum(23) + res[24] = fma52hi(res[24], a[3], a[20]); // Sum(23) + res[21] = fma52lo(res[21], a[0], a[21]); // Sum(21) + res[22] = fma52hi(res[22], a[0], a[21]); // Sum(21) + res[22] = fma52lo(res[22], a[1], a[21]); // Sum(22) + res[23] = fma52hi(res[23], a[1], a[21]); // Sum(22) + res[23] = fma52lo(res[23], a[2], a[21]); // Sum(23) + res[24] = fma52hi(res[24], a[2], a[21]); // Sum(23) + res[22] = fma52lo(res[22], a[0], a[22]); // Sum(22) + res[23] = fma52hi(res[23], a[0], a[22]); // Sum(22) + res[23] = fma52lo(res[23], a[1], a[22]); // Sum(23) + res[24] = fma52hi(res[24], a[1], a[22]); // Sum(23) + res[23] = fma52lo(res[23], a[0], a[23]); // Sum(23) + res[24] = fma52hi(res[24], a[0], a[23]); // Sum(23) + res[12] = add64(res[12], res[12]); // Double(12) + res[13] = add64(res[13], res[13]); // Double(13) + res[14] = add64(res[14], res[14]); // Double(14) + res[15] = add64(res[15], res[15]); // Double(15) + res[16] = add64(res[16], res[16]); // Double(16) + res[17] = add64(res[17], res[17]); // Double(17) + res[18] = add64(res[18], res[18]); // Double(18) + res[19] = add64(res[19], res[19]); // Double(19) + res[20] = add64(res[20], res[20]); // Double(20) + res[21] = add64(res[21], res[21]); // Double(21) + res[22] = add64(res[22], res[22]); // Double(22) + res[23] = add64(res[23], res[23]); // Double(23) + res[12] = fma52lo(res[12], a[6], a[6]); // Add sqr(12) + res[13] = fma52hi(res[13], a[6], a[6]); // Add sqr(12) + res[14] = fma52lo(res[14], a[7], a[7]); // Add sqr(14) + res[15] = fma52hi(res[15], a[7], a[7]); // Add sqr(14) + res[16] = fma52lo(res[16], a[8], a[8]); // Add sqr(16) + res[17] = fma52hi(res[17], a[8], a[8]); // Add sqr(16) + res[18] = fma52lo(res[18], a[9], a[9]); // Add sqr(18) + res[19] = fma52hi(res[19], a[9], a[9]); // Add sqr(18) + res[20] = fma52lo(res[20], a[10], a[10]); // Add sqr(20) + res[21] = fma52hi(res[21], a[10], a[10]); // Add sqr(20) + res[22] = fma52lo(res[22], a[11], a[11]); // Add sqr(22) + res[23] = fma52hi(res[23], a[11], a[11]); // Add sqr(22) + res[24] = fma52lo(res[24], a[11], a[13]); // Sum(24) + res[25] = fma52hi(res[25], a[11], a[13]); // Sum(24) + res[25] = fma52lo(res[25], a[12], a[13]); // Sum(25) + res[26] = fma52hi(res[26], a[12], a[13]); // Sum(25) + res[24] = fma52lo(res[24], a[10], a[14]); // Sum(24) + res[25] = fma52hi(res[25], a[10], a[14]); // Sum(24) + res[25] = fma52lo(res[25], a[11], a[14]); // Sum(25) + res[26] = fma52hi(res[26], a[11], a[14]); // Sum(25) + res[26] = fma52lo(res[26], a[12], a[14]); // Sum(26) + res[27] = fma52hi(res[27], a[12], a[14]); // Sum(26) + res[27] = fma52lo(res[27], a[13], a[14]); // Sum(27) + res[28] = fma52hi(res[28], a[13], a[14]); // Sum(27) + res[24] = fma52lo(res[24], a[9], a[15]); // Sum(24) + res[25] = fma52hi(res[25], a[9], a[15]); // Sum(24) + res[25] = fma52lo(res[25], a[10], a[15]); // Sum(25) + res[26] = fma52hi(res[26], a[10], a[15]); // Sum(25) + res[26] = fma52lo(res[26], a[11], a[15]); // Sum(26) + res[27] = fma52hi(res[27], a[11], a[15]); // Sum(26) + res[27] = fma52lo(res[27], a[12], a[15]); // Sum(27) + res[28] = fma52hi(res[28], a[12], a[15]); // Sum(27) + res[28] = fma52lo(res[28], a[13], a[15]); // Sum(28) + res[29] = fma52hi(res[29], a[13], a[15]); // Sum(28) + res[29] = fma52lo(res[29], a[14], a[15]); // Sum(29) + res[30] = fma52hi(res[30], a[14], a[15]); // Sum(29) + res[24] = fma52lo(res[24], a[8], a[16]); // Sum(24) + res[25] = fma52hi(res[25], a[8], a[16]); // Sum(24) + res[25] = fma52lo(res[25], a[9], a[16]); // Sum(25) + res[26] = fma52hi(res[26], a[9], a[16]); // Sum(25) + res[26] = fma52lo(res[26], a[10], a[16]); // Sum(26) + res[27] = fma52hi(res[27], a[10], a[16]); // Sum(26) + res[27] = fma52lo(res[27], a[11], a[16]); // Sum(27) + res[28] = fma52hi(res[28], a[11], a[16]); // Sum(27) + res[28] = fma52lo(res[28], a[12], a[16]); // Sum(28) + res[29] = fma52hi(res[29], a[12], a[16]); // Sum(28) + res[29] = fma52lo(res[29], a[13], a[16]); // Sum(29) + res[30] = fma52hi(res[30], a[13], a[16]); // Sum(29) + res[30] = fma52lo(res[30], a[14], a[16]); // Sum(30) + res[31] = fma52hi(res[31], a[14], a[16]); // Sum(30) + res[31] = fma52lo(res[31], a[15], a[16]); // Sum(31) + res[32] = fma52hi(res[32], a[15], a[16]); // Sum(31) + res[24] = fma52lo(res[24], a[7], a[17]); // Sum(24) + res[25] = fma52hi(res[25], a[7], a[17]); // Sum(24) + res[25] = fma52lo(res[25], a[8], a[17]); // Sum(25) + res[26] = fma52hi(res[26], a[8], a[17]); // Sum(25) + res[26] = fma52lo(res[26], a[9], a[17]); // Sum(26) + res[27] = fma52hi(res[27], a[9], a[17]); // Sum(26) + res[27] = fma52lo(res[27], a[10], a[17]); // Sum(27) + res[28] = fma52hi(res[28], a[10], a[17]); // Sum(27) + res[28] = fma52lo(res[28], a[11], a[17]); // Sum(28) + res[29] = fma52hi(res[29], a[11], a[17]); // Sum(28) + res[29] = fma52lo(res[29], a[12], a[17]); // Sum(29) + res[30] = fma52hi(res[30], a[12], a[17]); // Sum(29) + res[30] = fma52lo(res[30], a[13], a[17]); // Sum(30) + res[31] = fma52hi(res[31], a[13], a[17]); // Sum(30) + res[31] = fma52lo(res[31], a[14], a[17]); // Sum(31) + res[32] = fma52hi(res[32], a[14], a[17]); // Sum(31) + res[32] = fma52lo(res[32], a[15], a[17]); // Sum(32) + res[33] = fma52hi(res[33], a[15], a[17]); // Sum(32) + res[33] = fma52lo(res[33], a[16], a[17]); // Sum(33) + res[34] = fma52hi(res[34], a[16], a[17]); // Sum(33) + res[24] = fma52lo(res[24], a[6], a[18]); // Sum(24) + res[25] = fma52hi(res[25], a[6], a[18]); // Sum(24) + res[25] = fma52lo(res[25], a[7], a[18]); // Sum(25) + res[26] = fma52hi(res[26], a[7], a[18]); // Sum(25) + res[26] = fma52lo(res[26], a[8], a[18]); // Sum(26) + res[27] = fma52hi(res[27], a[8], a[18]); // Sum(26) + res[27] = fma52lo(res[27], a[9], a[18]); // Sum(27) + res[28] = fma52hi(res[28], a[9], a[18]); // Sum(27) + res[28] = fma52lo(res[28], a[10], a[18]); // Sum(28) + res[29] = fma52hi(res[29], a[10], a[18]); // Sum(28) + res[29] = fma52lo(res[29], a[11], a[18]); // Sum(29) + res[30] = fma52hi(res[30], a[11], a[18]); // Sum(29) + res[30] = fma52lo(res[30], a[12], a[18]); // Sum(30) + res[31] = fma52hi(res[31], a[12], a[18]); // Sum(30) + res[31] = fma52lo(res[31], a[13], a[18]); // Sum(31) + res[32] = fma52hi(res[32], a[13], a[18]); // Sum(31) + res[32] = fma52lo(res[32], a[14], a[18]); // Sum(32) + res[33] = fma52hi(res[33], a[14], a[18]); // Sum(32) + res[33] = fma52lo(res[33], a[15], a[18]); // Sum(33) + res[34] = fma52hi(res[34], a[15], a[18]); // Sum(33) + res[34] = fma52lo(res[34], a[16], a[18]); // Sum(34) + res[35] = fma52hi(res[35], a[16], a[18]); // Sum(34) + res[35] = fma52lo(res[35], a[17], a[18]); // Sum(35) + res[36] = fma52hi(res[36], a[17], a[18]); // Sum(35) + res[24] = fma52lo(res[24], a[5], a[19]); // Sum(24) + res[25] = fma52hi(res[25], a[5], a[19]); // Sum(24) + res[25] = fma52lo(res[25], a[6], a[19]); // Sum(25) + res[26] = fma52hi(res[26], a[6], a[19]); // Sum(25) + res[26] = fma52lo(res[26], a[7], a[19]); // Sum(26) + res[27] = fma52hi(res[27], a[7], a[19]); // Sum(26) + res[27] = fma52lo(res[27], a[8], a[19]); // Sum(27) + res[28] = fma52hi(res[28], a[8], a[19]); // Sum(27) + res[28] = fma52lo(res[28], a[9], a[19]); // Sum(28) + res[29] = fma52hi(res[29], a[9], a[19]); // Sum(28) + res[29] = fma52lo(res[29], a[10], a[19]); // Sum(29) + res[30] = fma52hi(res[30], a[10], a[19]); // Sum(29) + res[30] = fma52lo(res[30], a[11], a[19]); // Sum(30) + res[31] = fma52hi(res[31], a[11], a[19]); // Sum(30) + res[31] = fma52lo(res[31], a[12], a[19]); // Sum(31) + res[32] = fma52hi(res[32], a[12], a[19]); // Sum(31) + res[32] = fma52lo(res[32], a[13], a[19]); // Sum(32) + res[33] = fma52hi(res[33], a[13], a[19]); // Sum(32) + res[33] = fma52lo(res[33], a[14], a[19]); // Sum(33) + res[34] = fma52hi(res[34], a[14], a[19]); // Sum(33) + res[34] = fma52lo(res[34], a[15], a[19]); // Sum(34) + res[35] = fma52hi(res[35], a[15], a[19]); // Sum(34) + res[35] = fma52lo(res[35], a[16], a[19]); // Sum(35) + res[36] = fma52hi(res[36], a[16], a[19]); // Sum(35) + res[24] = fma52lo(res[24], a[4], a[20]); // Sum(24) + res[25] = fma52hi(res[25], a[4], a[20]); // Sum(24) + res[25] = fma52lo(res[25], a[5], a[20]); // Sum(25) + res[26] = fma52hi(res[26], a[5], a[20]); // Sum(25) + res[26] = fma52lo(res[26], a[6], a[20]); // Sum(26) + res[27] = fma52hi(res[27], a[6], a[20]); // Sum(26) + res[27] = fma52lo(res[27], a[7], a[20]); // Sum(27) + res[28] = fma52hi(res[28], a[7], a[20]); // Sum(27) + res[28] = fma52lo(res[28], a[8], a[20]); // Sum(28) + res[29] = fma52hi(res[29], a[8], a[20]); // Sum(28) + res[29] = fma52lo(res[29], a[9], a[20]); // Sum(29) + res[30] = fma52hi(res[30], a[9], a[20]); // Sum(29) + res[30] = fma52lo(res[30], a[10], a[20]); // Sum(30) + res[31] = fma52hi(res[31], a[10], a[20]); // Sum(30) + res[31] = fma52lo(res[31], a[11], a[20]); // Sum(31) + res[32] = fma52hi(res[32], a[11], a[20]); // Sum(31) + res[32] = fma52lo(res[32], a[12], a[20]); // Sum(32) + res[33] = fma52hi(res[33], a[12], a[20]); // Sum(32) + res[33] = fma52lo(res[33], a[13], a[20]); // Sum(33) + res[34] = fma52hi(res[34], a[13], a[20]); // Sum(33) + res[34] = fma52lo(res[34], a[14], a[20]); // Sum(34) + res[35] = fma52hi(res[35], a[14], a[20]); // Sum(34) + res[35] = fma52lo(res[35], a[15], a[20]); // Sum(35) + res[36] = fma52hi(res[36], a[15], a[20]); // Sum(35) + res[24] = fma52lo(res[24], a[3], a[21]); // Sum(24) + res[25] = fma52hi(res[25], a[3], a[21]); // Sum(24) + res[25] = fma52lo(res[25], a[4], a[21]); // Sum(25) + res[26] = fma52hi(res[26], a[4], a[21]); // Sum(25) + res[26] = fma52lo(res[26], a[5], a[21]); // Sum(26) + res[27] = fma52hi(res[27], a[5], a[21]); // Sum(26) + res[27] = fma52lo(res[27], a[6], a[21]); // Sum(27) + res[28] = fma52hi(res[28], a[6], a[21]); // Sum(27) + res[28] = fma52lo(res[28], a[7], a[21]); // Sum(28) + res[29] = fma52hi(res[29], a[7], a[21]); // Sum(28) + res[29] = fma52lo(res[29], a[8], a[21]); // Sum(29) + res[30] = fma52hi(res[30], a[8], a[21]); // Sum(29) + res[30] = fma52lo(res[30], a[9], a[21]); // Sum(30) + res[31] = fma52hi(res[31], a[9], a[21]); // Sum(30) + res[31] = fma52lo(res[31], a[10], a[21]); // Sum(31) + res[32] = fma52hi(res[32], a[10], a[21]); // Sum(31) + res[32] = fma52lo(res[32], a[11], a[21]); // Sum(32) + res[33] = fma52hi(res[33], a[11], a[21]); // Sum(32) + res[33] = fma52lo(res[33], a[12], a[21]); // Sum(33) + res[34] = fma52hi(res[34], a[12], a[21]); // Sum(33) + res[34] = fma52lo(res[34], a[13], a[21]); // Sum(34) + res[35] = fma52hi(res[35], a[13], a[21]); // Sum(34) + res[35] = fma52lo(res[35], a[14], a[21]); // Sum(35) + res[36] = fma52hi(res[36], a[14], a[21]); // Sum(35) + res[24] = fma52lo(res[24], a[2], a[22]); // Sum(24) + res[25] = fma52hi(res[25], a[2], a[22]); // Sum(24) + res[25] = fma52lo(res[25], a[3], a[22]); // Sum(25) + res[26] = fma52hi(res[26], a[3], a[22]); // Sum(25) + res[26] = fma52lo(res[26], a[4], a[22]); // Sum(26) + res[27] = fma52hi(res[27], a[4], a[22]); // Sum(26) + res[27] = fma52lo(res[27], a[5], a[22]); // Sum(27) + res[28] = fma52hi(res[28], a[5], a[22]); // Sum(27) + res[28] = fma52lo(res[28], a[6], a[22]); // Sum(28) + res[29] = fma52hi(res[29], a[6], a[22]); // Sum(28) + res[29] = fma52lo(res[29], a[7], a[22]); // Sum(29) + res[30] = fma52hi(res[30], a[7], a[22]); // Sum(29) + res[30] = fma52lo(res[30], a[8], a[22]); // Sum(30) + res[31] = fma52hi(res[31], a[8], a[22]); // Sum(30) + res[31] = fma52lo(res[31], a[9], a[22]); // Sum(31) + res[32] = fma52hi(res[32], a[9], a[22]); // Sum(31) + res[32] = fma52lo(res[32], a[10], a[22]); // Sum(32) + res[33] = fma52hi(res[33], a[10], a[22]); // Sum(32) + res[33] = fma52lo(res[33], a[11], a[22]); // Sum(33) + res[34] = fma52hi(res[34], a[11], a[22]); // Sum(33) + res[34] = fma52lo(res[34], a[12], a[22]); // Sum(34) + res[35] = fma52hi(res[35], a[12], a[22]); // Sum(34) + res[35] = fma52lo(res[35], a[13], a[22]); // Sum(35) + res[36] = fma52hi(res[36], a[13], a[22]); // Sum(35) + res[24] = fma52lo(res[24], a[1], a[23]); // Sum(24) + res[25] = fma52hi(res[25], a[1], a[23]); // Sum(24) + res[25] = fma52lo(res[25], a[2], a[23]); // Sum(25) + res[26] = fma52hi(res[26], a[2], a[23]); // Sum(25) + res[26] = fma52lo(res[26], a[3], a[23]); // Sum(26) + res[27] = fma52hi(res[27], a[3], a[23]); // Sum(26) + res[27] = fma52lo(res[27], a[4], a[23]); // Sum(27) + res[28] = fma52hi(res[28], a[4], a[23]); // Sum(27) + res[28] = fma52lo(res[28], a[5], a[23]); // Sum(28) + res[29] = fma52hi(res[29], a[5], a[23]); // Sum(28) + res[29] = fma52lo(res[29], a[6], a[23]); // Sum(29) + res[30] = fma52hi(res[30], a[6], a[23]); // Sum(29) + res[30] = fma52lo(res[30], a[7], a[23]); // Sum(30) + res[31] = fma52hi(res[31], a[7], a[23]); // Sum(30) + res[31] = fma52lo(res[31], a[8], a[23]); // Sum(31) + res[32] = fma52hi(res[32], a[8], a[23]); // Sum(31) + res[32] = fma52lo(res[32], a[9], a[23]); // Sum(32) + res[33] = fma52hi(res[33], a[9], a[23]); // Sum(32) + res[33] = fma52lo(res[33], a[10], a[23]); // Sum(33) + res[34] = fma52hi(res[34], a[10], a[23]); // Sum(33) + res[34] = fma52lo(res[34], a[11], a[23]); // Sum(34) + res[35] = fma52hi(res[35], a[11], a[23]); // Sum(34) + res[35] = fma52lo(res[35], a[12], a[23]); // Sum(35) + res[36] = fma52hi(res[36], a[12], a[23]); // Sum(35) + res[24] = fma52lo(res[24], a[0], a[24]); // Sum(24) + res[25] = fma52hi(res[25], a[0], a[24]); // Sum(24) + res[25] = fma52lo(res[25], a[1], a[24]); // Sum(25) + res[26] = fma52hi(res[26], a[1], a[24]); // Sum(25) + res[26] = fma52lo(res[26], a[2], a[24]); // Sum(26) + res[27] = fma52hi(res[27], a[2], a[24]); // Sum(26) + res[27] = fma52lo(res[27], a[3], a[24]); // Sum(27) + res[28] = fma52hi(res[28], a[3], a[24]); // Sum(27) + res[28] = fma52lo(res[28], a[4], a[24]); // Sum(28) + res[29] = fma52hi(res[29], a[4], a[24]); // Sum(28) + res[29] = fma52lo(res[29], a[5], a[24]); // Sum(29) + res[30] = fma52hi(res[30], a[5], a[24]); // Sum(29) + res[30] = fma52lo(res[30], a[6], a[24]); // Sum(30) + res[31] = fma52hi(res[31], a[6], a[24]); // Sum(30) + res[31] = fma52lo(res[31], a[7], a[24]); // Sum(31) + res[32] = fma52hi(res[32], a[7], a[24]); // Sum(31) + res[32] = fma52lo(res[32], a[8], a[24]); // Sum(32) + res[33] = fma52hi(res[33], a[8], a[24]); // Sum(32) + res[33] = fma52lo(res[33], a[9], a[24]); // Sum(33) + res[34] = fma52hi(res[34], a[9], a[24]); // Sum(33) + res[34] = fma52lo(res[34], a[10], a[24]); // Sum(34) + res[35] = fma52hi(res[35], a[10], a[24]); // Sum(34) + res[35] = fma52lo(res[35], a[11], a[24]); // Sum(35) + res[36] = fma52hi(res[36], a[11], a[24]); // Sum(35) + res[25] = fma52lo(res[25], a[0], a[25]); // Sum(25) + res[26] = fma52hi(res[26], a[0], a[25]); // Sum(25) + res[26] = fma52lo(res[26], a[1], a[25]); // Sum(26) + res[27] = fma52hi(res[27], a[1], a[25]); // Sum(26) + res[27] = fma52lo(res[27], a[2], a[25]); // Sum(27) + res[28] = fma52hi(res[28], a[2], a[25]); // Sum(27) + res[28] = fma52lo(res[28], a[3], a[25]); // Sum(28) + res[29] = fma52hi(res[29], a[3], a[25]); // Sum(28) + res[29] = fma52lo(res[29], a[4], a[25]); // Sum(29) + res[30] = fma52hi(res[30], a[4], a[25]); // Sum(29) + res[30] = fma52lo(res[30], a[5], a[25]); // Sum(30) + res[31] = fma52hi(res[31], a[5], a[25]); // Sum(30) + res[31] = fma52lo(res[31], a[6], a[25]); // Sum(31) + res[32] = fma52hi(res[32], a[6], a[25]); // Sum(31) + res[32] = fma52lo(res[32], a[7], a[25]); // Sum(32) + res[33] = fma52hi(res[33], a[7], a[25]); // Sum(32) + res[33] = fma52lo(res[33], a[8], a[25]); // Sum(33) + res[34] = fma52hi(res[34], a[8], a[25]); // Sum(33) + res[34] = fma52lo(res[34], a[9], a[25]); // Sum(34) + res[35] = fma52hi(res[35], a[9], a[25]); // Sum(34) + res[35] = fma52lo(res[35], a[10], a[25]); // Sum(35) + res[36] = fma52hi(res[36], a[10], a[25]); // Sum(35) + res[26] = fma52lo(res[26], a[0], a[26]); // Sum(26) + res[27] = fma52hi(res[27], a[0], a[26]); // Sum(26) + res[27] = fma52lo(res[27], a[1], a[26]); // Sum(27) + res[28] = fma52hi(res[28], a[1], a[26]); // Sum(27) + res[28] = fma52lo(res[28], a[2], a[26]); // Sum(28) + res[29] = fma52hi(res[29], a[2], a[26]); // Sum(28) + res[29] = fma52lo(res[29], a[3], a[26]); // Sum(29) + res[30] = fma52hi(res[30], a[3], a[26]); // Sum(29) + res[30] = fma52lo(res[30], a[4], a[26]); // Sum(30) + res[31] = fma52hi(res[31], a[4], a[26]); // Sum(30) + res[31] = fma52lo(res[31], a[5], a[26]); // Sum(31) + res[32] = fma52hi(res[32], a[5], a[26]); // Sum(31) + res[32] = fma52lo(res[32], a[6], a[26]); // Sum(32) + res[33] = fma52hi(res[33], a[6], a[26]); // Sum(32) + res[33] = fma52lo(res[33], a[7], a[26]); // Sum(33) + res[34] = fma52hi(res[34], a[7], a[26]); // Sum(33) + res[34] = fma52lo(res[34], a[8], a[26]); // Sum(34) + res[35] = fma52hi(res[35], a[8], a[26]); // Sum(34) + res[35] = fma52lo(res[35], a[9], a[26]); // Sum(35) + res[36] = fma52hi(res[36], a[9], a[26]); // Sum(35) + res[27] = fma52lo(res[27], a[0], a[27]); // Sum(27) + res[28] = fma52hi(res[28], a[0], a[27]); // Sum(27) + res[28] = fma52lo(res[28], a[1], a[27]); // Sum(28) + res[29] = fma52hi(res[29], a[1], a[27]); // Sum(28) + res[29] = fma52lo(res[29], a[2], a[27]); // Sum(29) + res[30] = fma52hi(res[30], a[2], a[27]); // Sum(29) + res[30] = fma52lo(res[30], a[3], a[27]); // Sum(30) + res[31] = fma52hi(res[31], a[3], a[27]); // Sum(30) + res[31] = fma52lo(res[31], a[4], a[27]); // Sum(31) + res[32] = fma52hi(res[32], a[4], a[27]); // Sum(31) + res[32] = fma52lo(res[32], a[5], a[27]); // Sum(32) + res[33] = fma52hi(res[33], a[5], a[27]); // Sum(32) + res[33] = fma52lo(res[33], a[6], a[27]); // Sum(33) + res[34] = fma52hi(res[34], a[6], a[27]); // Sum(33) + res[34] = fma52lo(res[34], a[7], a[27]); // Sum(34) + res[35] = fma52hi(res[35], a[7], a[27]); // Sum(34) + res[35] = fma52lo(res[35], a[8], a[27]); // Sum(35) + res[36] = fma52hi(res[36], a[8], a[27]); // Sum(35) + res[28] = fma52lo(res[28], a[0], a[28]); // Sum(28) + res[29] = fma52hi(res[29], a[0], a[28]); // Sum(28) + res[29] = fma52lo(res[29], a[1], a[28]); // Sum(29) + res[30] = fma52hi(res[30], a[1], a[28]); // Sum(29) + res[30] = fma52lo(res[30], a[2], a[28]); // Sum(30) + res[31] = fma52hi(res[31], a[2], a[28]); // Sum(30) + res[31] = fma52lo(res[31], a[3], a[28]); // Sum(31) + res[32] = fma52hi(res[32], a[3], a[28]); // Sum(31) + res[32] = fma52lo(res[32], a[4], a[28]); // Sum(32) + res[33] = fma52hi(res[33], a[4], a[28]); // Sum(32) + res[33] = fma52lo(res[33], a[5], a[28]); // Sum(33) + res[34] = fma52hi(res[34], a[5], a[28]); // Sum(33) + res[34] = fma52lo(res[34], a[6], a[28]); // Sum(34) + res[35] = fma52hi(res[35], a[6], a[28]); // Sum(34) + res[35] = fma52lo(res[35], a[7], a[28]); // Sum(35) + res[36] = fma52hi(res[36], a[7], a[28]); // Sum(35) + res[29] = fma52lo(res[29], a[0], a[29]); // Sum(29) + res[30] = fma52hi(res[30], a[0], a[29]); // Sum(29) + res[30] = fma52lo(res[30], a[1], a[29]); // Sum(30) + res[31] = fma52hi(res[31], a[1], a[29]); // Sum(30) + res[31] = fma52lo(res[31], a[2], a[29]); // Sum(31) + res[32] = fma52hi(res[32], a[2], a[29]); // Sum(31) + res[32] = fma52lo(res[32], a[3], a[29]); // Sum(32) + res[33] = fma52hi(res[33], a[3], a[29]); // Sum(32) + res[33] = fma52lo(res[33], a[4], a[29]); // Sum(33) + res[34] = fma52hi(res[34], a[4], a[29]); // Sum(33) + res[34] = fma52lo(res[34], a[5], a[29]); // Sum(34) + res[35] = fma52hi(res[35], a[5], a[29]); // Sum(34) + res[35] = fma52lo(res[35], a[6], a[29]); // Sum(35) + res[36] = fma52hi(res[36], a[6], a[29]); // Sum(35) + res[30] = fma52lo(res[30], a[0], a[30]); // Sum(30) + res[31] = fma52hi(res[31], a[0], a[30]); // Sum(30) + res[31] = fma52lo(res[31], a[1], a[30]); // Sum(31) + res[32] = fma52hi(res[32], a[1], a[30]); // Sum(31) + res[32] = fma52lo(res[32], a[2], a[30]); // Sum(32) + res[33] = fma52hi(res[33], a[2], a[30]); // Sum(32) + res[33] = fma52lo(res[33], a[3], a[30]); // Sum(33) + res[34] = fma52hi(res[34], a[3], a[30]); // Sum(33) + res[34] = fma52lo(res[34], a[4], a[30]); // Sum(34) + res[35] = fma52hi(res[35], a[4], a[30]); // Sum(34) + res[35] = fma52lo(res[35], a[5], a[30]); // Sum(35) + res[36] = fma52hi(res[36], a[5], a[30]); // Sum(35) + res[31] = fma52lo(res[31], a[0], a[31]); // Sum(31) + res[32] = fma52hi(res[32], a[0], a[31]); // Sum(31) + res[32] = fma52lo(res[32], a[1], a[31]); // Sum(32) + res[33] = fma52hi(res[33], a[1], a[31]); // Sum(32) + res[33] = fma52lo(res[33], a[2], a[31]); // Sum(33) + res[34] = fma52hi(res[34], a[2], a[31]); // Sum(33) + res[34] = fma52lo(res[34], a[3], a[31]); // Sum(34) + res[35] = fma52hi(res[35], a[3], a[31]); // Sum(34) + res[35] = fma52lo(res[35], a[4], a[31]); // Sum(35) + res[36] = fma52hi(res[36], a[4], a[31]); // Sum(35) + res[32] = fma52lo(res[32], a[0], a[32]); // Sum(32) + res[33] = fma52hi(res[33], a[0], a[32]); // Sum(32) + res[33] = fma52lo(res[33], a[1], a[32]); // Sum(33) + res[34] = fma52hi(res[34], a[1], a[32]); // Sum(33) + res[34] = fma52lo(res[34], a[2], a[32]); // Sum(34) + res[35] = fma52hi(res[35], a[2], a[32]); // Sum(34) + res[35] = fma52lo(res[35], a[3], a[32]); // Sum(35) + res[36] = fma52hi(res[36], a[3], a[32]); // Sum(35) + res[33] = fma52lo(res[33], a[0], a[33]); // Sum(33) + res[34] = fma52hi(res[34], a[0], a[33]); // Sum(33) + res[34] = fma52lo(res[34], a[1], a[33]); // Sum(34) + res[35] = fma52hi(res[35], a[1], a[33]); // Sum(34) + res[35] = fma52lo(res[35], a[2], a[33]); // Sum(35) + res[36] = fma52hi(res[36], a[2], a[33]); // Sum(35) + res[34] = fma52lo(res[34], a[0], a[34]); // Sum(34) + res[35] = fma52hi(res[35], a[0], a[34]); // Sum(34) + res[35] = fma52lo(res[35], a[1], a[34]); // Sum(35) + res[36] = fma52hi(res[36], a[1], a[34]); // Sum(35) + res[35] = fma52lo(res[35], a[0], a[35]); // Sum(35) + res[36] = fma52hi(res[36], a[0], a[35]); // Sum(35) + res[24] = add64(res[24], res[24]); // Double(24) + res[25] = add64(res[25], res[25]); // Double(25) + res[26] = add64(res[26], res[26]); // Double(26) + res[27] = add64(res[27], res[27]); // Double(27) + res[28] = add64(res[28], res[28]); // Double(28) + res[29] = add64(res[29], res[29]); // Double(29) + res[30] = add64(res[30], res[30]); // Double(30) + res[31] = add64(res[31], res[31]); // Double(31) + res[32] = add64(res[32], res[32]); // Double(32) + res[33] = add64(res[33], res[33]); // Double(33) + res[34] = add64(res[34], res[34]); // Double(34) + res[35] = add64(res[35], res[35]); // Double(35) + res[24] = fma52lo(res[24], a[12], a[12]); // Add sqr(24) + res[25] = fma52hi(res[25], a[12], a[12]); // Add sqr(24) + res[26] = fma52lo(res[26], a[13], a[13]); // Add sqr(26) + res[27] = fma52hi(res[27], a[13], a[13]); // Add sqr(26) + res[28] = fma52lo(res[28], a[14], a[14]); // Add sqr(28) + res[29] = fma52hi(res[29], a[14], a[14]); // Add sqr(28) + res[30] = fma52lo(res[30], a[15], a[15]); // Add sqr(30) + res[31] = fma52hi(res[31], a[15], a[15]); // Add sqr(30) + res[32] = fma52lo(res[32], a[16], a[16]); // Add sqr(32) + res[33] = fma52hi(res[33], a[16], a[16]); // Add sqr(32) + res[34] = fma52lo(res[34], a[17], a[17]); // Add sqr(34) + res[35] = fma52hi(res[35], a[17], a[17]); // Add sqr(34) + res[36] = fma52lo(res[36], a[17], a[19]); // Sum(36) + res[37] = fma52hi(res[37], a[17], a[19]); // Sum(36) + res[37] = fma52lo(res[37], a[18], a[19]); // Sum(37) + res[38] = fma52hi(res[38], a[18], a[19]); // Sum(37) + res[36] = fma52lo(res[36], a[16], a[20]); // Sum(36) + res[37] = fma52hi(res[37], a[16], a[20]); // Sum(36) + res[37] = fma52lo(res[37], a[17], a[20]); // Sum(37) + res[38] = fma52hi(res[38], a[17], a[20]); // Sum(37) + res[38] = fma52lo(res[38], a[18], a[20]); // Sum(38) + res[39] = fma52hi(res[39], a[18], a[20]); // Sum(38) + res[39] = fma52lo(res[39], a[19], a[20]); // Sum(39) + res[40] = fma52hi(res[40], a[19], a[20]); // Sum(39) + res[36] = fma52lo(res[36], a[15], a[21]); // Sum(36) + res[37] = fma52hi(res[37], a[15], a[21]); // Sum(36) + res[37] = fma52lo(res[37], a[16], a[21]); // Sum(37) + res[38] = fma52hi(res[38], a[16], a[21]); // Sum(37) + res[38] = fma52lo(res[38], a[17], a[21]); // Sum(38) + res[39] = fma52hi(res[39], a[17], a[21]); // Sum(38) + res[39] = fma52lo(res[39], a[18], a[21]); // Sum(39) + res[40] = fma52hi(res[40], a[18], a[21]); // Sum(39) + res[40] = fma52lo(res[40], a[19], a[21]); // Sum(40) + res[41] = fma52hi(res[41], a[19], a[21]); // Sum(40) + res[41] = fma52lo(res[41], a[20], a[21]); // Sum(41) + res[42] = fma52hi(res[42], a[20], a[21]); // Sum(41) + res[36] = fma52lo(res[36], a[14], a[22]); // Sum(36) + res[37] = fma52hi(res[37], a[14], a[22]); // Sum(36) + res[37] = fma52lo(res[37], a[15], a[22]); // Sum(37) + res[38] = fma52hi(res[38], a[15], a[22]); // Sum(37) + res[38] = fma52lo(res[38], a[16], a[22]); // Sum(38) + res[39] = fma52hi(res[39], a[16], a[22]); // Sum(38) + res[39] = fma52lo(res[39], a[17], a[22]); // Sum(39) + res[40] = fma52hi(res[40], a[17], a[22]); // Sum(39) + res[40] = fma52lo(res[40], a[18], a[22]); // Sum(40) + res[41] = fma52hi(res[41], a[18], a[22]); // Sum(40) + res[41] = fma52lo(res[41], a[19], a[22]); // Sum(41) + res[42] = fma52hi(res[42], a[19], a[22]); // Sum(41) + res[42] = fma52lo(res[42], a[20], a[22]); // Sum(42) + res[43] = fma52hi(res[43], a[20], a[22]); // Sum(42) + res[43] = fma52lo(res[43], a[21], a[22]); // Sum(43) + res[44] = fma52hi(res[44], a[21], a[22]); // Sum(43) + res[36] = fma52lo(res[36], a[13], a[23]); // Sum(36) + res[37] = fma52hi(res[37], a[13], a[23]); // Sum(36) + res[37] = fma52lo(res[37], a[14], a[23]); // Sum(37) + res[38] = fma52hi(res[38], a[14], a[23]); // Sum(37) + res[38] = fma52lo(res[38], a[15], a[23]); // Sum(38) + res[39] = fma52hi(res[39], a[15], a[23]); // Sum(38) + res[39] = fma52lo(res[39], a[16], a[23]); // Sum(39) + res[40] = fma52hi(res[40], a[16], a[23]); // Sum(39) + res[40] = fma52lo(res[40], a[17], a[23]); // Sum(40) + res[41] = fma52hi(res[41], a[17], a[23]); // Sum(40) + res[41] = fma52lo(res[41], a[18], a[23]); // Sum(41) + res[42] = fma52hi(res[42], a[18], a[23]); // Sum(41) + res[42] = fma52lo(res[42], a[19], a[23]); // Sum(42) + res[43] = fma52hi(res[43], a[19], a[23]); // Sum(42) + res[43] = fma52lo(res[43], a[20], a[23]); // Sum(43) + res[44] = fma52hi(res[44], a[20], a[23]); // Sum(43) + res[44] = fma52lo(res[44], a[21], a[23]); // Sum(44) + res[45] = fma52hi(res[45], a[21], a[23]); // Sum(44) + res[45] = fma52lo(res[45], a[22], a[23]); // Sum(45) + res[46] = fma52hi(res[46], a[22], a[23]); // Sum(45) + res[36] = fma52lo(res[36], a[12], a[24]); // Sum(36) + res[37] = fma52hi(res[37], a[12], a[24]); // Sum(36) + res[37] = fma52lo(res[37], a[13], a[24]); // Sum(37) + res[38] = fma52hi(res[38], a[13], a[24]); // Sum(37) + res[38] = fma52lo(res[38], a[14], a[24]); // Sum(38) + res[39] = fma52hi(res[39], a[14], a[24]); // Sum(38) + res[39] = fma52lo(res[39], a[15], a[24]); // Sum(39) + res[40] = fma52hi(res[40], a[15], a[24]); // Sum(39) + res[40] = fma52lo(res[40], a[16], a[24]); // Sum(40) + res[41] = fma52hi(res[41], a[16], a[24]); // Sum(40) + res[41] = fma52lo(res[41], a[17], a[24]); // Sum(41) + res[42] = fma52hi(res[42], a[17], a[24]); // Sum(41) + res[42] = fma52lo(res[42], a[18], a[24]); // Sum(42) + res[43] = fma52hi(res[43], a[18], a[24]); // Sum(42) + res[43] = fma52lo(res[43], a[19], a[24]); // Sum(43) + res[44] = fma52hi(res[44], a[19], a[24]); // Sum(43) + res[44] = fma52lo(res[44], a[20], a[24]); // Sum(44) + res[45] = fma52hi(res[45], a[20], a[24]); // Sum(44) + res[45] = fma52lo(res[45], a[21], a[24]); // Sum(45) + res[46] = fma52hi(res[46], a[21], a[24]); // Sum(45) + res[46] = fma52lo(res[46], a[22], a[24]); // Sum(46) + res[47] = fma52hi(res[47], a[22], a[24]); // Sum(46) + res[47] = fma52lo(res[47], a[23], a[24]); // Sum(47) + res[48] = fma52hi(res[48], a[23], a[24]); // Sum(47) + res[36] = fma52lo(res[36], a[11], a[25]); // Sum(36) + res[37] = fma52hi(res[37], a[11], a[25]); // Sum(36) + res[37] = fma52lo(res[37], a[12], a[25]); // Sum(37) + res[38] = fma52hi(res[38], a[12], a[25]); // Sum(37) + res[38] = fma52lo(res[38], a[13], a[25]); // Sum(38) + res[39] = fma52hi(res[39], a[13], a[25]); // Sum(38) + res[39] = fma52lo(res[39], a[14], a[25]); // Sum(39) + res[40] = fma52hi(res[40], a[14], a[25]); // Sum(39) + res[40] = fma52lo(res[40], a[15], a[25]); // Sum(40) + res[41] = fma52hi(res[41], a[15], a[25]); // Sum(40) + res[41] = fma52lo(res[41], a[16], a[25]); // Sum(41) + res[42] = fma52hi(res[42], a[16], a[25]); // Sum(41) + res[42] = fma52lo(res[42], a[17], a[25]); // Sum(42) + res[43] = fma52hi(res[43], a[17], a[25]); // Sum(42) + res[43] = fma52lo(res[43], a[18], a[25]); // Sum(43) + res[44] = fma52hi(res[44], a[18], a[25]); // Sum(43) + res[44] = fma52lo(res[44], a[19], a[25]); // Sum(44) + res[45] = fma52hi(res[45], a[19], a[25]); // Sum(44) + res[45] = fma52lo(res[45], a[20], a[25]); // Sum(45) + res[46] = fma52hi(res[46], a[20], a[25]); // Sum(45) + res[46] = fma52lo(res[46], a[21], a[25]); // Sum(46) + res[47] = fma52hi(res[47], a[21], a[25]); // Sum(46) + res[47] = fma52lo(res[47], a[22], a[25]); // Sum(47) + res[48] = fma52hi(res[48], a[22], a[25]); // Sum(47) + res[36] = fma52lo(res[36], a[10], a[26]); // Sum(36) + res[37] = fma52hi(res[37], a[10], a[26]); // Sum(36) + res[37] = fma52lo(res[37], a[11], a[26]); // Sum(37) + res[38] = fma52hi(res[38], a[11], a[26]); // Sum(37) + res[38] = fma52lo(res[38], a[12], a[26]); // Sum(38) + res[39] = fma52hi(res[39], a[12], a[26]); // Sum(38) + res[39] = fma52lo(res[39], a[13], a[26]); // Sum(39) + res[40] = fma52hi(res[40], a[13], a[26]); // Sum(39) + res[40] = fma52lo(res[40], a[14], a[26]); // Sum(40) + res[41] = fma52hi(res[41], a[14], a[26]); // Sum(40) + res[41] = fma52lo(res[41], a[15], a[26]); // Sum(41) + res[42] = fma52hi(res[42], a[15], a[26]); // Sum(41) + res[42] = fma52lo(res[42], a[16], a[26]); // Sum(42) + res[43] = fma52hi(res[43], a[16], a[26]); // Sum(42) + res[43] = fma52lo(res[43], a[17], a[26]); // Sum(43) + res[44] = fma52hi(res[44], a[17], a[26]); // Sum(43) + res[44] = fma52lo(res[44], a[18], a[26]); // Sum(44) + res[45] = fma52hi(res[45], a[18], a[26]); // Sum(44) + res[45] = fma52lo(res[45], a[19], a[26]); // Sum(45) + res[46] = fma52hi(res[46], a[19], a[26]); // Sum(45) + res[46] = fma52lo(res[46], a[20], a[26]); // Sum(46) + res[47] = fma52hi(res[47], a[20], a[26]); // Sum(46) + res[47] = fma52lo(res[47], a[21], a[26]); // Sum(47) + res[48] = fma52hi(res[48], a[21], a[26]); // Sum(47) + res[36] = fma52lo(res[36], a[9], a[27]); // Sum(36) + res[37] = fma52hi(res[37], a[9], a[27]); // Sum(36) + res[37] = fma52lo(res[37], a[10], a[27]); // Sum(37) + res[38] = fma52hi(res[38], a[10], a[27]); // Sum(37) + res[38] = fma52lo(res[38], a[11], a[27]); // Sum(38) + res[39] = fma52hi(res[39], a[11], a[27]); // Sum(38) + res[39] = fma52lo(res[39], a[12], a[27]); // Sum(39) + res[40] = fma52hi(res[40], a[12], a[27]); // Sum(39) + res[40] = fma52lo(res[40], a[13], a[27]); // Sum(40) + res[41] = fma52hi(res[41], a[13], a[27]); // Sum(40) + res[41] = fma52lo(res[41], a[14], a[27]); // Sum(41) + res[42] = fma52hi(res[42], a[14], a[27]); // Sum(41) + res[42] = fma52lo(res[42], a[15], a[27]); // Sum(42) + res[43] = fma52hi(res[43], a[15], a[27]); // Sum(42) + res[43] = fma52lo(res[43], a[16], a[27]); // Sum(43) + res[44] = fma52hi(res[44], a[16], a[27]); // Sum(43) + res[44] = fma52lo(res[44], a[17], a[27]); // Sum(44) + res[45] = fma52hi(res[45], a[17], a[27]); // Sum(44) + res[45] = fma52lo(res[45], a[18], a[27]); // Sum(45) + res[46] = fma52hi(res[46], a[18], a[27]); // Sum(45) + res[46] = fma52lo(res[46], a[19], a[27]); // Sum(46) + res[47] = fma52hi(res[47], a[19], a[27]); // Sum(46) + res[47] = fma52lo(res[47], a[20], a[27]); // Sum(47) + res[48] = fma52hi(res[48], a[20], a[27]); // Sum(47) + res[36] = fma52lo(res[36], a[8], a[28]); // Sum(36) + res[37] = fma52hi(res[37], a[8], a[28]); // Sum(36) + res[37] = fma52lo(res[37], a[9], a[28]); // Sum(37) + res[38] = fma52hi(res[38], a[9], a[28]); // Sum(37) + res[38] = fma52lo(res[38], a[10], a[28]); // Sum(38) + res[39] = fma52hi(res[39], a[10], a[28]); // Sum(38) + res[39] = fma52lo(res[39], a[11], a[28]); // Sum(39) + res[40] = fma52hi(res[40], a[11], a[28]); // Sum(39) + res[40] = fma52lo(res[40], a[12], a[28]); // Sum(40) + res[41] = fma52hi(res[41], a[12], a[28]); // Sum(40) + res[41] = fma52lo(res[41], a[13], a[28]); // Sum(41) + res[42] = fma52hi(res[42], a[13], a[28]); // Sum(41) + res[42] = fma52lo(res[42], a[14], a[28]); // Sum(42) + res[43] = fma52hi(res[43], a[14], a[28]); // Sum(42) + res[43] = fma52lo(res[43], a[15], a[28]); // Sum(43) + res[44] = fma52hi(res[44], a[15], a[28]); // Sum(43) + res[44] = fma52lo(res[44], a[16], a[28]); // Sum(44) + res[45] = fma52hi(res[45], a[16], a[28]); // Sum(44) + res[45] = fma52lo(res[45], a[17], a[28]); // Sum(45) + res[46] = fma52hi(res[46], a[17], a[28]); // Sum(45) + res[46] = fma52lo(res[46], a[18], a[28]); // Sum(46) + res[47] = fma52hi(res[47], a[18], a[28]); // Sum(46) + res[47] = fma52lo(res[47], a[19], a[28]); // Sum(47) + res[48] = fma52hi(res[48], a[19], a[28]); // Sum(47) + res[36] = fma52lo(res[36], a[7], a[29]); // Sum(36) + res[37] = fma52hi(res[37], a[7], a[29]); // Sum(36) + res[37] = fma52lo(res[37], a[8], a[29]); // Sum(37) + res[38] = fma52hi(res[38], a[8], a[29]); // Sum(37) + res[38] = fma52lo(res[38], a[9], a[29]); // Sum(38) + res[39] = fma52hi(res[39], a[9], a[29]); // Sum(38) + res[39] = fma52lo(res[39], a[10], a[29]); // Sum(39) + res[40] = fma52hi(res[40], a[10], a[29]); // Sum(39) + res[40] = fma52lo(res[40], a[11], a[29]); // Sum(40) + res[41] = fma52hi(res[41], a[11], a[29]); // Sum(40) + res[41] = fma52lo(res[41], a[12], a[29]); // Sum(41) + res[42] = fma52hi(res[42], a[12], a[29]); // Sum(41) + res[42] = fma52lo(res[42], a[13], a[29]); // Sum(42) + res[43] = fma52hi(res[43], a[13], a[29]); // Sum(42) + res[43] = fma52lo(res[43], a[14], a[29]); // Sum(43) + res[44] = fma52hi(res[44], a[14], a[29]); // Sum(43) + res[44] = fma52lo(res[44], a[15], a[29]); // Sum(44) + res[45] = fma52hi(res[45], a[15], a[29]); // Sum(44) + res[45] = fma52lo(res[45], a[16], a[29]); // Sum(45) + res[46] = fma52hi(res[46], a[16], a[29]); // Sum(45) + res[46] = fma52lo(res[46], a[17], a[29]); // Sum(46) + res[47] = fma52hi(res[47], a[17], a[29]); // Sum(46) + res[47] = fma52lo(res[47], a[18], a[29]); // Sum(47) + res[48] = fma52hi(res[48], a[18], a[29]); // Sum(47) + res[36] = fma52lo(res[36], a[6], a[30]); // Sum(36) + res[37] = fma52hi(res[37], a[6], a[30]); // Sum(36) + res[37] = fma52lo(res[37], a[7], a[30]); // Sum(37) + res[38] = fma52hi(res[38], a[7], a[30]); // Sum(37) + res[38] = fma52lo(res[38], a[8], a[30]); // Sum(38) + res[39] = fma52hi(res[39], a[8], a[30]); // Sum(38) + res[39] = fma52lo(res[39], a[9], a[30]); // Sum(39) + res[40] = fma52hi(res[40], a[9], a[30]); // Sum(39) + res[40] = fma52lo(res[40], a[10], a[30]); // Sum(40) + res[41] = fma52hi(res[41], a[10], a[30]); // Sum(40) + res[41] = fma52lo(res[41], a[11], a[30]); // Sum(41) + res[42] = fma52hi(res[42], a[11], a[30]); // Sum(41) + res[42] = fma52lo(res[42], a[12], a[30]); // Sum(42) + res[43] = fma52hi(res[43], a[12], a[30]); // Sum(42) + res[43] = fma52lo(res[43], a[13], a[30]); // Sum(43) + res[44] = fma52hi(res[44], a[13], a[30]); // Sum(43) + res[44] = fma52lo(res[44], a[14], a[30]); // Sum(44) + res[45] = fma52hi(res[45], a[14], a[30]); // Sum(44) + res[45] = fma52lo(res[45], a[15], a[30]); // Sum(45) + res[46] = fma52hi(res[46], a[15], a[30]); // Sum(45) + res[46] = fma52lo(res[46], a[16], a[30]); // Sum(46) + res[47] = fma52hi(res[47], a[16], a[30]); // Sum(46) + res[47] = fma52lo(res[47], a[17], a[30]); // Sum(47) + res[48] = fma52hi(res[48], a[17], a[30]); // Sum(47) + res[36] = fma52lo(res[36], a[5], a[31]); // Sum(36) + res[37] = fma52hi(res[37], a[5], a[31]); // Sum(36) + res[37] = fma52lo(res[37], a[6], a[31]); // Sum(37) + res[38] = fma52hi(res[38], a[6], a[31]); // Sum(37) + res[38] = fma52lo(res[38], a[7], a[31]); // Sum(38) + res[39] = fma52hi(res[39], a[7], a[31]); // Sum(38) + res[39] = fma52lo(res[39], a[8], a[31]); // Sum(39) + res[40] = fma52hi(res[40], a[8], a[31]); // Sum(39) + res[40] = fma52lo(res[40], a[9], a[31]); // Sum(40) + res[41] = fma52hi(res[41], a[9], a[31]); // Sum(40) + res[41] = fma52lo(res[41], a[10], a[31]); // Sum(41) + res[42] = fma52hi(res[42], a[10], a[31]); // Sum(41) + res[42] = fma52lo(res[42], a[11], a[31]); // Sum(42) + res[43] = fma52hi(res[43], a[11], a[31]); // Sum(42) + res[43] = fma52lo(res[43], a[12], a[31]); // Sum(43) + res[44] = fma52hi(res[44], a[12], a[31]); // Sum(43) + res[44] = fma52lo(res[44], a[13], a[31]); // Sum(44) + res[45] = fma52hi(res[45], a[13], a[31]); // Sum(44) + res[45] = fma52lo(res[45], a[14], a[31]); // Sum(45) + res[46] = fma52hi(res[46], a[14], a[31]); // Sum(45) + res[46] = fma52lo(res[46], a[15], a[31]); // Sum(46) + res[47] = fma52hi(res[47], a[15], a[31]); // Sum(46) + res[47] = fma52lo(res[47], a[16], a[31]); // Sum(47) + res[48] = fma52hi(res[48], a[16], a[31]); // Sum(47) + res[36] = fma52lo(res[36], a[4], a[32]); // Sum(36) + res[37] = fma52hi(res[37], a[4], a[32]); // Sum(36) + res[37] = fma52lo(res[37], a[5], a[32]); // Sum(37) + res[38] = fma52hi(res[38], a[5], a[32]); // Sum(37) + res[38] = fma52lo(res[38], a[6], a[32]); // Sum(38) + res[39] = fma52hi(res[39], a[6], a[32]); // Sum(38) + res[39] = fma52lo(res[39], a[7], a[32]); // Sum(39) + res[40] = fma52hi(res[40], a[7], a[32]); // Sum(39) + res[40] = fma52lo(res[40], a[8], a[32]); // Sum(40) + res[41] = fma52hi(res[41], a[8], a[32]); // Sum(40) + res[41] = fma52lo(res[41], a[9], a[32]); // Sum(41) + res[42] = fma52hi(res[42], a[9], a[32]); // Sum(41) + res[42] = fma52lo(res[42], a[10], a[32]); // Sum(42) + res[43] = fma52hi(res[43], a[10], a[32]); // Sum(42) + res[43] = fma52lo(res[43], a[11], a[32]); // Sum(43) + res[44] = fma52hi(res[44], a[11], a[32]); // Sum(43) + res[44] = fma52lo(res[44], a[12], a[32]); // Sum(44) + res[45] = fma52hi(res[45], a[12], a[32]); // Sum(44) + res[45] = fma52lo(res[45], a[13], a[32]); // Sum(45) + res[46] = fma52hi(res[46], a[13], a[32]); // Sum(45) + res[46] = fma52lo(res[46], a[14], a[32]); // Sum(46) + res[47] = fma52hi(res[47], a[14], a[32]); // Sum(46) + res[47] = fma52lo(res[47], a[15], a[32]); // Sum(47) + res[48] = fma52hi(res[48], a[15], a[32]); // Sum(47) + res[36] = fma52lo(res[36], a[3], a[33]); // Sum(36) + res[37] = fma52hi(res[37], a[3], a[33]); // Sum(36) + res[37] = fma52lo(res[37], a[4], a[33]); // Sum(37) + res[38] = fma52hi(res[38], a[4], a[33]); // Sum(37) + res[38] = fma52lo(res[38], a[5], a[33]); // Sum(38) + res[39] = fma52hi(res[39], a[5], a[33]); // Sum(38) + res[39] = fma52lo(res[39], a[6], a[33]); // Sum(39) + res[40] = fma52hi(res[40], a[6], a[33]); // Sum(39) + res[40] = fma52lo(res[40], a[7], a[33]); // Sum(40) + res[41] = fma52hi(res[41], a[7], a[33]); // Sum(40) + res[41] = fma52lo(res[41], a[8], a[33]); // Sum(41) + res[42] = fma52hi(res[42], a[8], a[33]); // Sum(41) + res[42] = fma52lo(res[42], a[9], a[33]); // Sum(42) + res[43] = fma52hi(res[43], a[9], a[33]); // Sum(42) + res[43] = fma52lo(res[43], a[10], a[33]); // Sum(43) + res[44] = fma52hi(res[44], a[10], a[33]); // Sum(43) + res[44] = fma52lo(res[44], a[11], a[33]); // Sum(44) + res[45] = fma52hi(res[45], a[11], a[33]); // Sum(44) + res[45] = fma52lo(res[45], a[12], a[33]); // Sum(45) + res[46] = fma52hi(res[46], a[12], a[33]); // Sum(45) + res[46] = fma52lo(res[46], a[13], a[33]); // Sum(46) + res[47] = fma52hi(res[47], a[13], a[33]); // Sum(46) + res[47] = fma52lo(res[47], a[14], a[33]); // Sum(47) + res[48] = fma52hi(res[48], a[14], a[33]); // Sum(47) + res[36] = fma52lo(res[36], a[2], a[34]); // Sum(36) + res[37] = fma52hi(res[37], a[2], a[34]); // Sum(36) + res[37] = fma52lo(res[37], a[3], a[34]); // Sum(37) + res[38] = fma52hi(res[38], a[3], a[34]); // Sum(37) + res[38] = fma52lo(res[38], a[4], a[34]); // Sum(38) + res[39] = fma52hi(res[39], a[4], a[34]); // Sum(38) + res[39] = fma52lo(res[39], a[5], a[34]); // Sum(39) + res[40] = fma52hi(res[40], a[5], a[34]); // Sum(39) + res[40] = fma52lo(res[40], a[6], a[34]); // Sum(40) + res[41] = fma52hi(res[41], a[6], a[34]); // Sum(40) + res[41] = fma52lo(res[41], a[7], a[34]); // Sum(41) + res[42] = fma52hi(res[42], a[7], a[34]); // Sum(41) + res[42] = fma52lo(res[42], a[8], a[34]); // Sum(42) + res[43] = fma52hi(res[43], a[8], a[34]); // Sum(42) + res[43] = fma52lo(res[43], a[9], a[34]); // Sum(43) + res[44] = fma52hi(res[44], a[9], a[34]); // Sum(43) + res[44] = fma52lo(res[44], a[10], a[34]); // Sum(44) + res[45] = fma52hi(res[45], a[10], a[34]); // Sum(44) + res[45] = fma52lo(res[45], a[11], a[34]); // Sum(45) + res[46] = fma52hi(res[46], a[11], a[34]); // Sum(45) + res[46] = fma52lo(res[46], a[12], a[34]); // Sum(46) + res[47] = fma52hi(res[47], a[12], a[34]); // Sum(46) + res[47] = fma52lo(res[47], a[13], a[34]); // Sum(47) + res[48] = fma52hi(res[48], a[13], a[34]); // Sum(47) + res[36] = fma52lo(res[36], a[1], a[35]); // Sum(36) + res[37] = fma52hi(res[37], a[1], a[35]); // Sum(36) + res[37] = fma52lo(res[37], a[2], a[35]); // Sum(37) + res[38] = fma52hi(res[38], a[2], a[35]); // Sum(37) + res[38] = fma52lo(res[38], a[3], a[35]); // Sum(38) + res[39] = fma52hi(res[39], a[3], a[35]); // Sum(38) + res[39] = fma52lo(res[39], a[4], a[35]); // Sum(39) + res[40] = fma52hi(res[40], a[4], a[35]); // Sum(39) + res[40] = fma52lo(res[40], a[5], a[35]); // Sum(40) + res[41] = fma52hi(res[41], a[5], a[35]); // Sum(40) + res[41] = fma52lo(res[41], a[6], a[35]); // Sum(41) + res[42] = fma52hi(res[42], a[6], a[35]); // Sum(41) + res[42] = fma52lo(res[42], a[7], a[35]); // Sum(42) + res[43] = fma52hi(res[43], a[7], a[35]); // Sum(42) + res[43] = fma52lo(res[43], a[8], a[35]); // Sum(43) + res[44] = fma52hi(res[44], a[8], a[35]); // Sum(43) + res[44] = fma52lo(res[44], a[9], a[35]); // Sum(44) + res[45] = fma52hi(res[45], a[9], a[35]); // Sum(44) + res[45] = fma52lo(res[45], a[10], a[35]); // Sum(45) + res[46] = fma52hi(res[46], a[10], a[35]); // Sum(45) + res[46] = fma52lo(res[46], a[11], a[35]); // Sum(46) + res[47] = fma52hi(res[47], a[11], a[35]); // Sum(46) + res[47] = fma52lo(res[47], a[12], a[35]); // Sum(47) + res[48] = fma52hi(res[48], a[12], a[35]); // Sum(47) + res[36] = fma52lo(res[36], a[0], a[36]); // Sum(36) + res[37] = fma52hi(res[37], a[0], a[36]); // Sum(36) + res[37] = fma52lo(res[37], a[1], a[36]); // Sum(37) + res[38] = fma52hi(res[38], a[1], a[36]); // Sum(37) + res[38] = fma52lo(res[38], a[2], a[36]); // Sum(38) + res[39] = fma52hi(res[39], a[2], a[36]); // Sum(38) + res[39] = fma52lo(res[39], a[3], a[36]); // Sum(39) + res[40] = fma52hi(res[40], a[3], a[36]); // Sum(39) + res[40] = fma52lo(res[40], a[4], a[36]); // Sum(40) + res[41] = fma52hi(res[41], a[4], a[36]); // Sum(40) + res[41] = fma52lo(res[41], a[5], a[36]); // Sum(41) + res[42] = fma52hi(res[42], a[5], a[36]); // Sum(41) + res[42] = fma52lo(res[42], a[6], a[36]); // Sum(42) + res[43] = fma52hi(res[43], a[6], a[36]); // Sum(42) + res[43] = fma52lo(res[43], a[7], a[36]); // Sum(43) + res[44] = fma52hi(res[44], a[7], a[36]); // Sum(43) + res[44] = fma52lo(res[44], a[8], a[36]); // Sum(44) + res[45] = fma52hi(res[45], a[8], a[36]); // Sum(44) + res[45] = fma52lo(res[45], a[9], a[36]); // Sum(45) + res[46] = fma52hi(res[46], a[9], a[36]); // Sum(45) + res[46] = fma52lo(res[46], a[10], a[36]); // Sum(46) + res[47] = fma52hi(res[47], a[10], a[36]); // Sum(46) + res[47] = fma52lo(res[47], a[11], a[36]); // Sum(47) + res[48] = fma52hi(res[48], a[11], a[36]); // Sum(47) + res[37] = fma52lo(res[37], a[0], a[37]); // Sum(37) + res[38] = fma52hi(res[38], a[0], a[37]); // Sum(37) + res[38] = fma52lo(res[38], a[1], a[37]); // Sum(38) + res[39] = fma52hi(res[39], a[1], a[37]); // Sum(38) + res[39] = fma52lo(res[39], a[2], a[37]); // Sum(39) + res[40] = fma52hi(res[40], a[2], a[37]); // Sum(39) + res[40] = fma52lo(res[40], a[3], a[37]); // Sum(40) + res[41] = fma52hi(res[41], a[3], a[37]); // Sum(40) + res[41] = fma52lo(res[41], a[4], a[37]); // Sum(41) + res[42] = fma52hi(res[42], a[4], a[37]); // Sum(41) + res[42] = fma52lo(res[42], a[5], a[37]); // Sum(42) + res[43] = fma52hi(res[43], a[5], a[37]); // Sum(42) + res[43] = fma52lo(res[43], a[6], a[37]); // Sum(43) + res[44] = fma52hi(res[44], a[6], a[37]); // Sum(43) + res[44] = fma52lo(res[44], a[7], a[37]); // Sum(44) + res[45] = fma52hi(res[45], a[7], a[37]); // Sum(44) + res[45] = fma52lo(res[45], a[8], a[37]); // Sum(45) + res[46] = fma52hi(res[46], a[8], a[37]); // Sum(45) + res[46] = fma52lo(res[46], a[9], a[37]); // Sum(46) + res[47] = fma52hi(res[47], a[9], a[37]); // Sum(46) + res[47] = fma52lo(res[47], a[10], a[37]); // Sum(47) + res[48] = fma52hi(res[48], a[10], a[37]); // Sum(47) + res[38] = fma52lo(res[38], a[0], a[38]); // Sum(38) + res[39] = fma52hi(res[39], a[0], a[38]); // Sum(38) + res[39] = fma52lo(res[39], a[1], a[38]); // Sum(39) + res[40] = fma52hi(res[40], a[1], a[38]); // Sum(39) + res[40] = fma52lo(res[40], a[2], a[38]); // Sum(40) + res[41] = fma52hi(res[41], a[2], a[38]); // Sum(40) + res[41] = fma52lo(res[41], a[3], a[38]); // Sum(41) + res[42] = fma52hi(res[42], a[3], a[38]); // Sum(41) + res[42] = fma52lo(res[42], a[4], a[38]); // Sum(42) + res[43] = fma52hi(res[43], a[4], a[38]); // Sum(42) + res[43] = fma52lo(res[43], a[5], a[38]); // Sum(43) + res[44] = fma52hi(res[44], a[5], a[38]); // Sum(43) + res[44] = fma52lo(res[44], a[6], a[38]); // Sum(44) + res[45] = fma52hi(res[45], a[6], a[38]); // Sum(44) + res[45] = fma52lo(res[45], a[7], a[38]); // Sum(45) + res[46] = fma52hi(res[46], a[7], a[38]); // Sum(45) + res[46] = fma52lo(res[46], a[8], a[38]); // Sum(46) + res[47] = fma52hi(res[47], a[8], a[38]); // Sum(46) + res[47] = fma52lo(res[47], a[9], a[38]); // Sum(47) + res[48] = fma52hi(res[48], a[9], a[38]); // Sum(47) + res[39] = fma52lo(res[39], a[0], a[39]); // Sum(39) + res[40] = fma52hi(res[40], a[0], a[39]); // Sum(39) + res[40] = fma52lo(res[40], a[1], a[39]); // Sum(40) + res[41] = fma52hi(res[41], a[1], a[39]); // Sum(40) + res[41] = fma52lo(res[41], a[2], a[39]); // Sum(41) + res[42] = fma52hi(res[42], a[2], a[39]); // Sum(41) + res[42] = fma52lo(res[42], a[3], a[39]); // Sum(42) + res[43] = fma52hi(res[43], a[3], a[39]); // Sum(42) + res[43] = fma52lo(res[43], a[4], a[39]); // Sum(43) + res[44] = fma52hi(res[44], a[4], a[39]); // Sum(43) + res[44] = fma52lo(res[44], a[5], a[39]); // Sum(44) + res[45] = fma52hi(res[45], a[5], a[39]); // Sum(44) + res[45] = fma52lo(res[45], a[6], a[39]); // Sum(45) + res[46] = fma52hi(res[46], a[6], a[39]); // Sum(45) + res[46] = fma52lo(res[46], a[7], a[39]); // Sum(46) + res[47] = fma52hi(res[47], a[7], a[39]); // Sum(46) + res[47] = fma52lo(res[47], a[8], a[39]); // Sum(47) + res[48] = fma52hi(res[48], a[8], a[39]); // Sum(47) + res[40] = fma52lo(res[40], a[0], a[40]); // Sum(40) + res[41] = fma52hi(res[41], a[0], a[40]); // Sum(40) + res[41] = fma52lo(res[41], a[1], a[40]); // Sum(41) + res[42] = fma52hi(res[42], a[1], a[40]); // Sum(41) + res[42] = fma52lo(res[42], a[2], a[40]); // Sum(42) + res[43] = fma52hi(res[43], a[2], a[40]); // Sum(42) + res[43] = fma52lo(res[43], a[3], a[40]); // Sum(43) + res[44] = fma52hi(res[44], a[3], a[40]); // Sum(43) + res[44] = fma52lo(res[44], a[4], a[40]); // Sum(44) + res[45] = fma52hi(res[45], a[4], a[40]); // Sum(44) + res[45] = fma52lo(res[45], a[5], a[40]); // Sum(45) + res[46] = fma52hi(res[46], a[5], a[40]); // Sum(45) + res[46] = fma52lo(res[46], a[6], a[40]); // Sum(46) + res[47] = fma52hi(res[47], a[6], a[40]); // Sum(46) + res[47] = fma52lo(res[47], a[7], a[40]); // Sum(47) + res[48] = fma52hi(res[48], a[7], a[40]); // Sum(47) + res[41] = fma52lo(res[41], a[0], a[41]); // Sum(41) + res[42] = fma52hi(res[42], a[0], a[41]); // Sum(41) + res[42] = fma52lo(res[42], a[1], a[41]); // Sum(42) + res[43] = fma52hi(res[43], a[1], a[41]); // Sum(42) + res[43] = fma52lo(res[43], a[2], a[41]); // Sum(43) + res[44] = fma52hi(res[44], a[2], a[41]); // Sum(43) + res[44] = fma52lo(res[44], a[3], a[41]); // Sum(44) + res[45] = fma52hi(res[45], a[3], a[41]); // Sum(44) + res[45] = fma52lo(res[45], a[4], a[41]); // Sum(45) + res[46] = fma52hi(res[46], a[4], a[41]); // Sum(45) + res[46] = fma52lo(res[46], a[5], a[41]); // Sum(46) + res[47] = fma52hi(res[47], a[5], a[41]); // Sum(46) + res[47] = fma52lo(res[47], a[6], a[41]); // Sum(47) + res[48] = fma52hi(res[48], a[6], a[41]); // Sum(47) + res[42] = fma52lo(res[42], a[0], a[42]); // Sum(42) + res[43] = fma52hi(res[43], a[0], a[42]); // Sum(42) + res[43] = fma52lo(res[43], a[1], a[42]); // Sum(43) + res[44] = fma52hi(res[44], a[1], a[42]); // Sum(43) + res[44] = fma52lo(res[44], a[2], a[42]); // Sum(44) + res[45] = fma52hi(res[45], a[2], a[42]); // Sum(44) + res[45] = fma52lo(res[45], a[3], a[42]); // Sum(45) + res[46] = fma52hi(res[46], a[3], a[42]); // Sum(45) + res[46] = fma52lo(res[46], a[4], a[42]); // Sum(46) + res[47] = fma52hi(res[47], a[4], a[42]); // Sum(46) + res[47] = fma52lo(res[47], a[5], a[42]); // Sum(47) + res[48] = fma52hi(res[48], a[5], a[42]); // Sum(47) + res[43] = fma52lo(res[43], a[0], a[43]); // Sum(43) + res[44] = fma52hi(res[44], a[0], a[43]); // Sum(43) + res[44] = fma52lo(res[44], a[1], a[43]); // Sum(44) + res[45] = fma52hi(res[45], a[1], a[43]); // Sum(44) + res[45] = fma52lo(res[45], a[2], a[43]); // Sum(45) + res[46] = fma52hi(res[46], a[2], a[43]); // Sum(45) + res[46] = fma52lo(res[46], a[3], a[43]); // Sum(46) + res[47] = fma52hi(res[47], a[3], a[43]); // Sum(46) + res[47] = fma52lo(res[47], a[4], a[43]); // Sum(47) + res[48] = fma52hi(res[48], a[4], a[43]); // Sum(47) + res[44] = fma52lo(res[44], a[0], a[44]); // Sum(44) + res[45] = fma52hi(res[45], a[0], a[44]); // Sum(44) + res[45] = fma52lo(res[45], a[1], a[44]); // Sum(45) + res[46] = fma52hi(res[46], a[1], a[44]); // Sum(45) + res[46] = fma52lo(res[46], a[2], a[44]); // Sum(46) + res[47] = fma52hi(res[47], a[2], a[44]); // Sum(46) + res[47] = fma52lo(res[47], a[3], a[44]); // Sum(47) + res[48] = fma52hi(res[48], a[3], a[44]); // Sum(47) + res[45] = fma52lo(res[45], a[0], a[45]); // Sum(45) + res[46] = fma52hi(res[46], a[0], a[45]); // Sum(45) + res[46] = fma52lo(res[46], a[1], a[45]); // Sum(46) + res[47] = fma52hi(res[47], a[1], a[45]); // Sum(46) + res[47] = fma52lo(res[47], a[2], a[45]); // Sum(47) + res[48] = fma52hi(res[48], a[2], a[45]); // Sum(47) + res[46] = fma52lo(res[46], a[0], a[46]); // Sum(46) + res[47] = fma52hi(res[47], a[0], a[46]); // Sum(46) + res[47] = fma52lo(res[47], a[1], a[46]); // Sum(47) + res[48] = fma52hi(res[48], a[1], a[46]); // Sum(47) + res[47] = fma52lo(res[47], a[0], a[47]); // Sum(47) + res[48] = fma52hi(res[48], a[0], a[47]); // Sum(47) + res[36] = add64(res[36], res[36]); // Double(36) + res[37] = add64(res[37], res[37]); // Double(37) + res[38] = add64(res[38], res[38]); // Double(38) + res[39] = add64(res[39], res[39]); // Double(39) + res[40] = add64(res[40], res[40]); // Double(40) + res[41] = add64(res[41], res[41]); // Double(41) + res[42] = add64(res[42], res[42]); // Double(42) + res[43] = add64(res[43], res[43]); // Double(43) + res[44] = add64(res[44], res[44]); // Double(44) + res[45] = add64(res[45], res[45]); // Double(45) + res[46] = add64(res[46], res[46]); // Double(46) + res[47] = add64(res[47], res[47]); // Double(47) + res[36] = fma52lo(res[36], a[18], a[18]); // Add sqr(36) + res[37] = fma52hi(res[37], a[18], a[18]); // Add sqr(36) + res[38] = fma52lo(res[38], a[19], a[19]); // Add sqr(38) + res[39] = fma52hi(res[39], a[19], a[19]); // Add sqr(38) + res[40] = fma52lo(res[40], a[20], a[20]); // Add sqr(40) + res[41] = fma52hi(res[41], a[20], a[20]); // Add sqr(40) + res[42] = fma52lo(res[42], a[21], a[21]); // Add sqr(42) + res[43] = fma52hi(res[43], a[21], a[21]); // Add sqr(42) + res[44] = fma52lo(res[44], a[22], a[22]); // Add sqr(44) + res[45] = fma52hi(res[45], a[22], a[22]); // Add sqr(44) + res[46] = fma52lo(res[46], a[23], a[23]); // Add sqr(46) + res[47] = fma52hi(res[47], a[23], a[23]); // Add sqr(46) + res[48] = fma52lo(res[48], a[23], a[25]); // Sum(48) + res[49] = fma52hi(res[49], a[23], a[25]); // Sum(48) + res[49] = fma52lo(res[49], a[24], a[25]); // Sum(49) + res[50] = fma52hi(res[50], a[24], a[25]); // Sum(49) + res[48] = fma52lo(res[48], a[22], a[26]); // Sum(48) + res[49] = fma52hi(res[49], a[22], a[26]); // Sum(48) + res[49] = fma52lo(res[49], a[23], a[26]); // Sum(49) + res[50] = fma52hi(res[50], a[23], a[26]); // Sum(49) + res[50] = fma52lo(res[50], a[24], a[26]); // Sum(50) + res[51] = fma52hi(res[51], a[24], a[26]); // Sum(50) + res[51] = fma52lo(res[51], a[25], a[26]); // Sum(51) + res[52] = fma52hi(res[52], a[25], a[26]); // Sum(51) + res[48] = fma52lo(res[48], a[21], a[27]); // Sum(48) + res[49] = fma52hi(res[49], a[21], a[27]); // Sum(48) + res[49] = fma52lo(res[49], a[22], a[27]); // Sum(49) + res[50] = fma52hi(res[50], a[22], a[27]); // Sum(49) + res[50] = fma52lo(res[50], a[23], a[27]); // Sum(50) + res[51] = fma52hi(res[51], a[23], a[27]); // Sum(50) + res[51] = fma52lo(res[51], a[24], a[27]); // Sum(51) + res[52] = fma52hi(res[52], a[24], a[27]); // Sum(51) + res[52] = fma52lo(res[52], a[25], a[27]); // Sum(52) + res[53] = fma52hi(res[53], a[25], a[27]); // Sum(52) + res[53] = fma52lo(res[53], a[26], a[27]); // Sum(53) + res[54] = fma52hi(res[54], a[26], a[27]); // Sum(53) + res[48] = fma52lo(res[48], a[20], a[28]); // Sum(48) + res[49] = fma52hi(res[49], a[20], a[28]); // Sum(48) + res[49] = fma52lo(res[49], a[21], a[28]); // Sum(49) + res[50] = fma52hi(res[50], a[21], a[28]); // Sum(49) + res[50] = fma52lo(res[50], a[22], a[28]); // Sum(50) + res[51] = fma52hi(res[51], a[22], a[28]); // Sum(50) + res[51] = fma52lo(res[51], a[23], a[28]); // Sum(51) + res[52] = fma52hi(res[52], a[23], a[28]); // Sum(51) + res[52] = fma52lo(res[52], a[24], a[28]); // Sum(52) + res[53] = fma52hi(res[53], a[24], a[28]); // Sum(52) + res[53] = fma52lo(res[53], a[25], a[28]); // Sum(53) + res[54] = fma52hi(res[54], a[25], a[28]); // Sum(53) + res[54] = fma52lo(res[54], a[26], a[28]); // Sum(54) + res[55] = fma52hi(res[55], a[26], a[28]); // Sum(54) + res[55] = fma52lo(res[55], a[27], a[28]); // Sum(55) + res[56] = fma52hi(res[56], a[27], a[28]); // Sum(55) + res[48] = fma52lo(res[48], a[19], a[29]); // Sum(48) + res[49] = fma52hi(res[49], a[19], a[29]); // Sum(48) + res[49] = fma52lo(res[49], a[20], a[29]); // Sum(49) + res[50] = fma52hi(res[50], a[20], a[29]); // Sum(49) + res[50] = fma52lo(res[50], a[21], a[29]); // Sum(50) + res[51] = fma52hi(res[51], a[21], a[29]); // Sum(50) + res[51] = fma52lo(res[51], a[22], a[29]); // Sum(51) + res[52] = fma52hi(res[52], a[22], a[29]); // Sum(51) + res[52] = fma52lo(res[52], a[23], a[29]); // Sum(52) + res[53] = fma52hi(res[53], a[23], a[29]); // Sum(52) + res[53] = fma52lo(res[53], a[24], a[29]); // Sum(53) + res[54] = fma52hi(res[54], a[24], a[29]); // Sum(53) + res[54] = fma52lo(res[54], a[25], a[29]); // Sum(54) + res[55] = fma52hi(res[55], a[25], a[29]); // Sum(54) + res[55] = fma52lo(res[55], a[26], a[29]); // Sum(55) + res[56] = fma52hi(res[56], a[26], a[29]); // Sum(55) + res[56] = fma52lo(res[56], a[27], a[29]); // Sum(56) + res[57] = fma52hi(res[57], a[27], a[29]); // Sum(56) + res[57] = fma52lo(res[57], a[28], a[29]); // Sum(57) + res[58] = fma52hi(res[58], a[28], a[29]); // Sum(57) + res[48] = fma52lo(res[48], a[18], a[30]); // Sum(48) + res[49] = fma52hi(res[49], a[18], a[30]); // Sum(48) + res[49] = fma52lo(res[49], a[19], a[30]); // Sum(49) + res[50] = fma52hi(res[50], a[19], a[30]); // Sum(49) + res[50] = fma52lo(res[50], a[20], a[30]); // Sum(50) + res[51] = fma52hi(res[51], a[20], a[30]); // Sum(50) + res[51] = fma52lo(res[51], a[21], a[30]); // Sum(51) + res[52] = fma52hi(res[52], a[21], a[30]); // Sum(51) + res[52] = fma52lo(res[52], a[22], a[30]); // Sum(52) + res[53] = fma52hi(res[53], a[22], a[30]); // Sum(52) + res[53] = fma52lo(res[53], a[23], a[30]); // Sum(53) + res[54] = fma52hi(res[54], a[23], a[30]); // Sum(53) + res[54] = fma52lo(res[54], a[24], a[30]); // Sum(54) + res[55] = fma52hi(res[55], a[24], a[30]); // Sum(54) + res[55] = fma52lo(res[55], a[25], a[30]); // Sum(55) + res[56] = fma52hi(res[56], a[25], a[30]); // Sum(55) + res[56] = fma52lo(res[56], a[26], a[30]); // Sum(56) + res[57] = fma52hi(res[57], a[26], a[30]); // Sum(56) + res[57] = fma52lo(res[57], a[27], a[30]); // Sum(57) + res[58] = fma52hi(res[58], a[27], a[30]); // Sum(57) + res[58] = fma52lo(res[58], a[28], a[30]); // Sum(58) + res[59] = fma52hi(res[59], a[28], a[30]); // Sum(58) + res[59] = fma52lo(res[59], a[29], a[30]); // Sum(59) + res[60] = fma52hi(res[60], a[29], a[30]); // Sum(59) + res[48] = fma52lo(res[48], a[17], a[31]); // Sum(48) + res[49] = fma52hi(res[49], a[17], a[31]); // Sum(48) + res[49] = fma52lo(res[49], a[18], a[31]); // Sum(49) + res[50] = fma52hi(res[50], a[18], a[31]); // Sum(49) + res[50] = fma52lo(res[50], a[19], a[31]); // Sum(50) + res[51] = fma52hi(res[51], a[19], a[31]); // Sum(50) + res[51] = fma52lo(res[51], a[20], a[31]); // Sum(51) + res[52] = fma52hi(res[52], a[20], a[31]); // Sum(51) + res[52] = fma52lo(res[52], a[21], a[31]); // Sum(52) + res[53] = fma52hi(res[53], a[21], a[31]); // Sum(52) + res[53] = fma52lo(res[53], a[22], a[31]); // Sum(53) + res[54] = fma52hi(res[54], a[22], a[31]); // Sum(53) + res[54] = fma52lo(res[54], a[23], a[31]); // Sum(54) + res[55] = fma52hi(res[55], a[23], a[31]); // Sum(54) + res[55] = fma52lo(res[55], a[24], a[31]); // Sum(55) + res[56] = fma52hi(res[56], a[24], a[31]); // Sum(55) + res[56] = fma52lo(res[56], a[25], a[31]); // Sum(56) + res[57] = fma52hi(res[57], a[25], a[31]); // Sum(56) + res[57] = fma52lo(res[57], a[26], a[31]); // Sum(57) + res[58] = fma52hi(res[58], a[26], a[31]); // Sum(57) + res[58] = fma52lo(res[58], a[27], a[31]); // Sum(58) + res[59] = fma52hi(res[59], a[27], a[31]); // Sum(58) + res[59] = fma52lo(res[59], a[28], a[31]); // Sum(59) + res[60] = fma52hi(res[60], a[28], a[31]); // Sum(59) + res[48] = fma52lo(res[48], a[16], a[32]); // Sum(48) + res[49] = fma52hi(res[49], a[16], a[32]); // Sum(48) + res[49] = fma52lo(res[49], a[17], a[32]); // Sum(49) + res[50] = fma52hi(res[50], a[17], a[32]); // Sum(49) + res[50] = fma52lo(res[50], a[18], a[32]); // Sum(50) + res[51] = fma52hi(res[51], a[18], a[32]); // Sum(50) + res[51] = fma52lo(res[51], a[19], a[32]); // Sum(51) + res[52] = fma52hi(res[52], a[19], a[32]); // Sum(51) + res[52] = fma52lo(res[52], a[20], a[32]); // Sum(52) + res[53] = fma52hi(res[53], a[20], a[32]); // Sum(52) + res[53] = fma52lo(res[53], a[21], a[32]); // Sum(53) + res[54] = fma52hi(res[54], a[21], a[32]); // Sum(53) + res[54] = fma52lo(res[54], a[22], a[32]); // Sum(54) + res[55] = fma52hi(res[55], a[22], a[32]); // Sum(54) + res[55] = fma52lo(res[55], a[23], a[32]); // Sum(55) + res[56] = fma52hi(res[56], a[23], a[32]); // Sum(55) + res[56] = fma52lo(res[56], a[24], a[32]); // Sum(56) + res[57] = fma52hi(res[57], a[24], a[32]); // Sum(56) + res[57] = fma52lo(res[57], a[25], a[32]); // Sum(57) + res[58] = fma52hi(res[58], a[25], a[32]); // Sum(57) + res[58] = fma52lo(res[58], a[26], a[32]); // Sum(58) + res[59] = fma52hi(res[59], a[26], a[32]); // Sum(58) + res[59] = fma52lo(res[59], a[27], a[32]); // Sum(59) + res[60] = fma52hi(res[60], a[27], a[32]); // Sum(59) + res[48] = fma52lo(res[48], a[15], a[33]); // Sum(48) + res[49] = fma52hi(res[49], a[15], a[33]); // Sum(48) + res[49] = fma52lo(res[49], a[16], a[33]); // Sum(49) + res[50] = fma52hi(res[50], a[16], a[33]); // Sum(49) + res[50] = fma52lo(res[50], a[17], a[33]); // Sum(50) + res[51] = fma52hi(res[51], a[17], a[33]); // Sum(50) + res[51] = fma52lo(res[51], a[18], a[33]); // Sum(51) + res[52] = fma52hi(res[52], a[18], a[33]); // Sum(51) + res[52] = fma52lo(res[52], a[19], a[33]); // Sum(52) + res[53] = fma52hi(res[53], a[19], a[33]); // Sum(52) + res[53] = fma52lo(res[53], a[20], a[33]); // Sum(53) + res[54] = fma52hi(res[54], a[20], a[33]); // Sum(53) + res[54] = fma52lo(res[54], a[21], a[33]); // Sum(54) + res[55] = fma52hi(res[55], a[21], a[33]); // Sum(54) + res[55] = fma52lo(res[55], a[22], a[33]); // Sum(55) + res[56] = fma52hi(res[56], a[22], a[33]); // Sum(55) + res[56] = fma52lo(res[56], a[23], a[33]); // Sum(56) + res[57] = fma52hi(res[57], a[23], a[33]); // Sum(56) + res[57] = fma52lo(res[57], a[24], a[33]); // Sum(57) + res[58] = fma52hi(res[58], a[24], a[33]); // Sum(57) + res[58] = fma52lo(res[58], a[25], a[33]); // Sum(58) + res[59] = fma52hi(res[59], a[25], a[33]); // Sum(58) + res[59] = fma52lo(res[59], a[26], a[33]); // Sum(59) + res[60] = fma52hi(res[60], a[26], a[33]); // Sum(59) + res[48] = fma52lo(res[48], a[14], a[34]); // Sum(48) + res[49] = fma52hi(res[49], a[14], a[34]); // Sum(48) + res[49] = fma52lo(res[49], a[15], a[34]); // Sum(49) + res[50] = fma52hi(res[50], a[15], a[34]); // Sum(49) + res[50] = fma52lo(res[50], a[16], a[34]); // Sum(50) + res[51] = fma52hi(res[51], a[16], a[34]); // Sum(50) + res[51] = fma52lo(res[51], a[17], a[34]); // Sum(51) + res[52] = fma52hi(res[52], a[17], a[34]); // Sum(51) + res[52] = fma52lo(res[52], a[18], a[34]); // Sum(52) + res[53] = fma52hi(res[53], a[18], a[34]); // Sum(52) + res[53] = fma52lo(res[53], a[19], a[34]); // Sum(53) + res[54] = fma52hi(res[54], a[19], a[34]); // Sum(53) + res[54] = fma52lo(res[54], a[20], a[34]); // Sum(54) + res[55] = fma52hi(res[55], a[20], a[34]); // Sum(54) + res[55] = fma52lo(res[55], a[21], a[34]); // Sum(55) + res[56] = fma52hi(res[56], a[21], a[34]); // Sum(55) + res[56] = fma52lo(res[56], a[22], a[34]); // Sum(56) + res[57] = fma52hi(res[57], a[22], a[34]); // Sum(56) + res[57] = fma52lo(res[57], a[23], a[34]); // Sum(57) + res[58] = fma52hi(res[58], a[23], a[34]); // Sum(57) + res[58] = fma52lo(res[58], a[24], a[34]); // Sum(58) + res[59] = fma52hi(res[59], a[24], a[34]); // Sum(58) + res[59] = fma52lo(res[59], a[25], a[34]); // Sum(59) + res[60] = fma52hi(res[60], a[25], a[34]); // Sum(59) + res[48] = fma52lo(res[48], a[13], a[35]); // Sum(48) + res[49] = fma52hi(res[49], a[13], a[35]); // Sum(48) + res[49] = fma52lo(res[49], a[14], a[35]); // Sum(49) + res[50] = fma52hi(res[50], a[14], a[35]); // Sum(49) + res[50] = fma52lo(res[50], a[15], a[35]); // Sum(50) + res[51] = fma52hi(res[51], a[15], a[35]); // Sum(50) + res[51] = fma52lo(res[51], a[16], a[35]); // Sum(51) + res[52] = fma52hi(res[52], a[16], a[35]); // Sum(51) + res[52] = fma52lo(res[52], a[17], a[35]); // Sum(52) + res[53] = fma52hi(res[53], a[17], a[35]); // Sum(52) + res[53] = fma52lo(res[53], a[18], a[35]); // Sum(53) + res[54] = fma52hi(res[54], a[18], a[35]); // Sum(53) + res[54] = fma52lo(res[54], a[19], a[35]); // Sum(54) + res[55] = fma52hi(res[55], a[19], a[35]); // Sum(54) + res[55] = fma52lo(res[55], a[20], a[35]); // Sum(55) + res[56] = fma52hi(res[56], a[20], a[35]); // Sum(55) + res[56] = fma52lo(res[56], a[21], a[35]); // Sum(56) + res[57] = fma52hi(res[57], a[21], a[35]); // Sum(56) + res[57] = fma52lo(res[57], a[22], a[35]); // Sum(57) + res[58] = fma52hi(res[58], a[22], a[35]); // Sum(57) + res[58] = fma52lo(res[58], a[23], a[35]); // Sum(58) + res[59] = fma52hi(res[59], a[23], a[35]); // Sum(58) + res[59] = fma52lo(res[59], a[24], a[35]); // Sum(59) + res[60] = fma52hi(res[60], a[24], a[35]); // Sum(59) + res[48] = fma52lo(res[48], a[12], a[36]); // Sum(48) + res[49] = fma52hi(res[49], a[12], a[36]); // Sum(48) + res[49] = fma52lo(res[49], a[13], a[36]); // Sum(49) + res[50] = fma52hi(res[50], a[13], a[36]); // Sum(49) + res[50] = fma52lo(res[50], a[14], a[36]); // Sum(50) + res[51] = fma52hi(res[51], a[14], a[36]); // Sum(50) + res[51] = fma52lo(res[51], a[15], a[36]); // Sum(51) + res[52] = fma52hi(res[52], a[15], a[36]); // Sum(51) + res[52] = fma52lo(res[52], a[16], a[36]); // Sum(52) + res[53] = fma52hi(res[53], a[16], a[36]); // Sum(52) + res[53] = fma52lo(res[53], a[17], a[36]); // Sum(53) + res[54] = fma52hi(res[54], a[17], a[36]); // Sum(53) + res[54] = fma52lo(res[54], a[18], a[36]); // Sum(54) + res[55] = fma52hi(res[55], a[18], a[36]); // Sum(54) + res[55] = fma52lo(res[55], a[19], a[36]); // Sum(55) + res[56] = fma52hi(res[56], a[19], a[36]); // Sum(55) + res[56] = fma52lo(res[56], a[20], a[36]); // Sum(56) + res[57] = fma52hi(res[57], a[20], a[36]); // Sum(56) + res[57] = fma52lo(res[57], a[21], a[36]); // Sum(57) + res[58] = fma52hi(res[58], a[21], a[36]); // Sum(57) + res[58] = fma52lo(res[58], a[22], a[36]); // Sum(58) + res[59] = fma52hi(res[59], a[22], a[36]); // Sum(58) + res[59] = fma52lo(res[59], a[23], a[36]); // Sum(59) + res[60] = fma52hi(res[60], a[23], a[36]); // Sum(59) + res[48] = fma52lo(res[48], a[11], a[37]); // Sum(48) + res[49] = fma52hi(res[49], a[11], a[37]); // Sum(48) + res[49] = fma52lo(res[49], a[12], a[37]); // Sum(49) + res[50] = fma52hi(res[50], a[12], a[37]); // Sum(49) + res[50] = fma52lo(res[50], a[13], a[37]); // Sum(50) + res[51] = fma52hi(res[51], a[13], a[37]); // Sum(50) + res[51] = fma52lo(res[51], a[14], a[37]); // Sum(51) + res[52] = fma52hi(res[52], a[14], a[37]); // Sum(51) + res[52] = fma52lo(res[52], a[15], a[37]); // Sum(52) + res[53] = fma52hi(res[53], a[15], a[37]); // Sum(52) + res[53] = fma52lo(res[53], a[16], a[37]); // Sum(53) + res[54] = fma52hi(res[54], a[16], a[37]); // Sum(53) + res[54] = fma52lo(res[54], a[17], a[37]); // Sum(54) + res[55] = fma52hi(res[55], a[17], a[37]); // Sum(54) + res[55] = fma52lo(res[55], a[18], a[37]); // Sum(55) + res[56] = fma52hi(res[56], a[18], a[37]); // Sum(55) + res[56] = fma52lo(res[56], a[19], a[37]); // Sum(56) + res[57] = fma52hi(res[57], a[19], a[37]); // Sum(56) + res[57] = fma52lo(res[57], a[20], a[37]); // Sum(57) + res[58] = fma52hi(res[58], a[20], a[37]); // Sum(57) + res[58] = fma52lo(res[58], a[21], a[37]); // Sum(58) + res[59] = fma52hi(res[59], a[21], a[37]); // Sum(58) + res[59] = fma52lo(res[59], a[22], a[37]); // Sum(59) + res[60] = fma52hi(res[60], a[22], a[37]); // Sum(59) + res[48] = fma52lo(res[48], a[10], a[38]); // Sum(48) + res[49] = fma52hi(res[49], a[10], a[38]); // Sum(48) + res[49] = fma52lo(res[49], a[11], a[38]); // Sum(49) + res[50] = fma52hi(res[50], a[11], a[38]); // Sum(49) + res[50] = fma52lo(res[50], a[12], a[38]); // Sum(50) + res[51] = fma52hi(res[51], a[12], a[38]); // Sum(50) + res[51] = fma52lo(res[51], a[13], a[38]); // Sum(51) + res[52] = fma52hi(res[52], a[13], a[38]); // Sum(51) + res[52] = fma52lo(res[52], a[14], a[38]); // Sum(52) + res[53] = fma52hi(res[53], a[14], a[38]); // Sum(52) + res[53] = fma52lo(res[53], a[15], a[38]); // Sum(53) + res[54] = fma52hi(res[54], a[15], a[38]); // Sum(53) + res[54] = fma52lo(res[54], a[16], a[38]); // Sum(54) + res[55] = fma52hi(res[55], a[16], a[38]); // Sum(54) + res[55] = fma52lo(res[55], a[17], a[38]); // Sum(55) + res[56] = fma52hi(res[56], a[17], a[38]); // Sum(55) + res[56] = fma52lo(res[56], a[18], a[38]); // Sum(56) + res[57] = fma52hi(res[57], a[18], a[38]); // Sum(56) + res[57] = fma52lo(res[57], a[19], a[38]); // Sum(57) + res[58] = fma52hi(res[58], a[19], a[38]); // Sum(57) + res[58] = fma52lo(res[58], a[20], a[38]); // Sum(58) + res[59] = fma52hi(res[59], a[20], a[38]); // Sum(58) + res[59] = fma52lo(res[59], a[21], a[38]); // Sum(59) + res[60] = fma52hi(res[60], a[21], a[38]); // Sum(59) + res[48] = fma52lo(res[48], a[9], a[39]); // Sum(48) + res[49] = fma52hi(res[49], a[9], a[39]); // Sum(48) + res[49] = fma52lo(res[49], a[10], a[39]); // Sum(49) + res[50] = fma52hi(res[50], a[10], a[39]); // Sum(49) + res[50] = fma52lo(res[50], a[11], a[39]); // Sum(50) + res[51] = fma52hi(res[51], a[11], a[39]); // Sum(50) + res[51] = fma52lo(res[51], a[12], a[39]); // Sum(51) + res[52] = fma52hi(res[52], a[12], a[39]); // Sum(51) + res[52] = fma52lo(res[52], a[13], a[39]); // Sum(52) + res[53] = fma52hi(res[53], a[13], a[39]); // Sum(52) + res[53] = fma52lo(res[53], a[14], a[39]); // Sum(53) + res[54] = fma52hi(res[54], a[14], a[39]); // Sum(53) + res[54] = fma52lo(res[54], a[15], a[39]); // Sum(54) + res[55] = fma52hi(res[55], a[15], a[39]); // Sum(54) + res[55] = fma52lo(res[55], a[16], a[39]); // Sum(55) + res[56] = fma52hi(res[56], a[16], a[39]); // Sum(55) + res[56] = fma52lo(res[56], a[17], a[39]); // Sum(56) + res[57] = fma52hi(res[57], a[17], a[39]); // Sum(56) + res[57] = fma52lo(res[57], a[18], a[39]); // Sum(57) + res[58] = fma52hi(res[58], a[18], a[39]); // Sum(57) + res[58] = fma52lo(res[58], a[19], a[39]); // Sum(58) + res[59] = fma52hi(res[59], a[19], a[39]); // Sum(58) + res[59] = fma52lo(res[59], a[20], a[39]); // Sum(59) + res[60] = fma52hi(res[60], a[20], a[39]); // Sum(59) + res[48] = fma52lo(res[48], a[8], a[40]); // Sum(48) + res[49] = fma52hi(res[49], a[8], a[40]); // Sum(48) + res[49] = fma52lo(res[49], a[9], a[40]); // Sum(49) + res[50] = fma52hi(res[50], a[9], a[40]); // Sum(49) + res[50] = fma52lo(res[50], a[10], a[40]); // Sum(50) + res[51] = fma52hi(res[51], a[10], a[40]); // Sum(50) + res[51] = fma52lo(res[51], a[11], a[40]); // Sum(51) + res[52] = fma52hi(res[52], a[11], a[40]); // Sum(51) + res[52] = fma52lo(res[52], a[12], a[40]); // Sum(52) + res[53] = fma52hi(res[53], a[12], a[40]); // Sum(52) + res[53] = fma52lo(res[53], a[13], a[40]); // Sum(53) + res[54] = fma52hi(res[54], a[13], a[40]); // Sum(53) + res[54] = fma52lo(res[54], a[14], a[40]); // Sum(54) + res[55] = fma52hi(res[55], a[14], a[40]); // Sum(54) + res[55] = fma52lo(res[55], a[15], a[40]); // Sum(55) + res[56] = fma52hi(res[56], a[15], a[40]); // Sum(55) + res[56] = fma52lo(res[56], a[16], a[40]); // Sum(56) + res[57] = fma52hi(res[57], a[16], a[40]); // Sum(56) + res[57] = fma52lo(res[57], a[17], a[40]); // Sum(57) + res[58] = fma52hi(res[58], a[17], a[40]); // Sum(57) + res[58] = fma52lo(res[58], a[18], a[40]); // Sum(58) + res[59] = fma52hi(res[59], a[18], a[40]); // Sum(58) + res[59] = fma52lo(res[59], a[19], a[40]); // Sum(59) + res[60] = fma52hi(res[60], a[19], a[40]); // Sum(59) + res[48] = fma52lo(res[48], a[7], a[41]); // Sum(48) + res[49] = fma52hi(res[49], a[7], a[41]); // Sum(48) + res[49] = fma52lo(res[49], a[8], a[41]); // Sum(49) + res[50] = fma52hi(res[50], a[8], a[41]); // Sum(49) + res[50] = fma52lo(res[50], a[9], a[41]); // Sum(50) + res[51] = fma52hi(res[51], a[9], a[41]); // Sum(50) + res[51] = fma52lo(res[51], a[10], a[41]); // Sum(51) + res[52] = fma52hi(res[52], a[10], a[41]); // Sum(51) + res[52] = fma52lo(res[52], a[11], a[41]); // Sum(52) + res[53] = fma52hi(res[53], a[11], a[41]); // Sum(52) + res[53] = fma52lo(res[53], a[12], a[41]); // Sum(53) + res[54] = fma52hi(res[54], a[12], a[41]); // Sum(53) + res[54] = fma52lo(res[54], a[13], a[41]); // Sum(54) + res[55] = fma52hi(res[55], a[13], a[41]); // Sum(54) + res[55] = fma52lo(res[55], a[14], a[41]); // Sum(55) + res[56] = fma52hi(res[56], a[14], a[41]); // Sum(55) + res[56] = fma52lo(res[56], a[15], a[41]); // Sum(56) + res[57] = fma52hi(res[57], a[15], a[41]); // Sum(56) + res[57] = fma52lo(res[57], a[16], a[41]); // Sum(57) + res[58] = fma52hi(res[58], a[16], a[41]); // Sum(57) + res[58] = fma52lo(res[58], a[17], a[41]); // Sum(58) + res[59] = fma52hi(res[59], a[17], a[41]); // Sum(58) + res[59] = fma52lo(res[59], a[18], a[41]); // Sum(59) + res[60] = fma52hi(res[60], a[18], a[41]); // Sum(59) + res[48] = fma52lo(res[48], a[6], a[42]); // Sum(48) + res[49] = fma52hi(res[49], a[6], a[42]); // Sum(48) + res[49] = fma52lo(res[49], a[7], a[42]); // Sum(49) + res[50] = fma52hi(res[50], a[7], a[42]); // Sum(49) + res[50] = fma52lo(res[50], a[8], a[42]); // Sum(50) + res[51] = fma52hi(res[51], a[8], a[42]); // Sum(50) + res[51] = fma52lo(res[51], a[9], a[42]); // Sum(51) + res[52] = fma52hi(res[52], a[9], a[42]); // Sum(51) + res[52] = fma52lo(res[52], a[10], a[42]); // Sum(52) + res[53] = fma52hi(res[53], a[10], a[42]); // Sum(52) + res[53] = fma52lo(res[53], a[11], a[42]); // Sum(53) + res[54] = fma52hi(res[54], a[11], a[42]); // Sum(53) + res[54] = fma52lo(res[54], a[12], a[42]); // Sum(54) + res[55] = fma52hi(res[55], a[12], a[42]); // Sum(54) + res[55] = fma52lo(res[55], a[13], a[42]); // Sum(55) + res[56] = fma52hi(res[56], a[13], a[42]); // Sum(55) + res[56] = fma52lo(res[56], a[14], a[42]); // Sum(56) + res[57] = fma52hi(res[57], a[14], a[42]); // Sum(56) + res[57] = fma52lo(res[57], a[15], a[42]); // Sum(57) + res[58] = fma52hi(res[58], a[15], a[42]); // Sum(57) + res[58] = fma52lo(res[58], a[16], a[42]); // Sum(58) + res[59] = fma52hi(res[59], a[16], a[42]); // Sum(58) + res[59] = fma52lo(res[59], a[17], a[42]); // Sum(59) + res[60] = fma52hi(res[60], a[17], a[42]); // Sum(59) + res[48] = fma52lo(res[48], a[5], a[43]); // Sum(48) + res[49] = fma52hi(res[49], a[5], a[43]); // Sum(48) + res[49] = fma52lo(res[49], a[6], a[43]); // Sum(49) + res[50] = fma52hi(res[50], a[6], a[43]); // Sum(49) + res[50] = fma52lo(res[50], a[7], a[43]); // Sum(50) + res[51] = fma52hi(res[51], a[7], a[43]); // Sum(50) + res[51] = fma52lo(res[51], a[8], a[43]); // Sum(51) + res[52] = fma52hi(res[52], a[8], a[43]); // Sum(51) + res[52] = fma52lo(res[52], a[9], a[43]); // Sum(52) + res[53] = fma52hi(res[53], a[9], a[43]); // Sum(52) + res[53] = fma52lo(res[53], a[10], a[43]); // Sum(53) + res[54] = fma52hi(res[54], a[10], a[43]); // Sum(53) + res[54] = fma52lo(res[54], a[11], a[43]); // Sum(54) + res[55] = fma52hi(res[55], a[11], a[43]); // Sum(54) + res[55] = fma52lo(res[55], a[12], a[43]); // Sum(55) + res[56] = fma52hi(res[56], a[12], a[43]); // Sum(55) + res[56] = fma52lo(res[56], a[13], a[43]); // Sum(56) + res[57] = fma52hi(res[57], a[13], a[43]); // Sum(56) + res[57] = fma52lo(res[57], a[14], a[43]); // Sum(57) + res[58] = fma52hi(res[58], a[14], a[43]); // Sum(57) + res[58] = fma52lo(res[58], a[15], a[43]); // Sum(58) + res[59] = fma52hi(res[59], a[15], a[43]); // Sum(58) + res[59] = fma52lo(res[59], a[16], a[43]); // Sum(59) + res[60] = fma52hi(res[60], a[16], a[43]); // Sum(59) + res[48] = fma52lo(res[48], a[4], a[44]); // Sum(48) + res[49] = fma52hi(res[49], a[4], a[44]); // Sum(48) + res[49] = fma52lo(res[49], a[5], a[44]); // Sum(49) + res[50] = fma52hi(res[50], a[5], a[44]); // Sum(49) + res[50] = fma52lo(res[50], a[6], a[44]); // Sum(50) + res[51] = fma52hi(res[51], a[6], a[44]); // Sum(50) + res[51] = fma52lo(res[51], a[7], a[44]); // Sum(51) + res[52] = fma52hi(res[52], a[7], a[44]); // Sum(51) + res[52] = fma52lo(res[52], a[8], a[44]); // Sum(52) + res[53] = fma52hi(res[53], a[8], a[44]); // Sum(52) + res[53] = fma52lo(res[53], a[9], a[44]); // Sum(53) + res[54] = fma52hi(res[54], a[9], a[44]); // Sum(53) + res[54] = fma52lo(res[54], a[10], a[44]); // Sum(54) + res[55] = fma52hi(res[55], a[10], a[44]); // Sum(54) + res[55] = fma52lo(res[55], a[11], a[44]); // Sum(55) + res[56] = fma52hi(res[56], a[11], a[44]); // Sum(55) + res[56] = fma52lo(res[56], a[12], a[44]); // Sum(56) + res[57] = fma52hi(res[57], a[12], a[44]); // Sum(56) + res[57] = fma52lo(res[57], a[13], a[44]); // Sum(57) + res[58] = fma52hi(res[58], a[13], a[44]); // Sum(57) + res[58] = fma52lo(res[58], a[14], a[44]); // Sum(58) + res[59] = fma52hi(res[59], a[14], a[44]); // Sum(58) + res[59] = fma52lo(res[59], a[15], a[44]); // Sum(59) + res[60] = fma52hi(res[60], a[15], a[44]); // Sum(59) + res[48] = fma52lo(res[48], a[3], a[45]); // Sum(48) + res[49] = fma52hi(res[49], a[3], a[45]); // Sum(48) + res[49] = fma52lo(res[49], a[4], a[45]); // Sum(49) + res[50] = fma52hi(res[50], a[4], a[45]); // Sum(49) + res[50] = fma52lo(res[50], a[5], a[45]); // Sum(50) + res[51] = fma52hi(res[51], a[5], a[45]); // Sum(50) + res[51] = fma52lo(res[51], a[6], a[45]); // Sum(51) + res[52] = fma52hi(res[52], a[6], a[45]); // Sum(51) + res[52] = fma52lo(res[52], a[7], a[45]); // Sum(52) + res[53] = fma52hi(res[53], a[7], a[45]); // Sum(52) + res[53] = fma52lo(res[53], a[8], a[45]); // Sum(53) + res[54] = fma52hi(res[54], a[8], a[45]); // Sum(53) + res[54] = fma52lo(res[54], a[9], a[45]); // Sum(54) + res[55] = fma52hi(res[55], a[9], a[45]); // Sum(54) + res[55] = fma52lo(res[55], a[10], a[45]); // Sum(55) + res[56] = fma52hi(res[56], a[10], a[45]); // Sum(55) + res[56] = fma52lo(res[56], a[11], a[45]); // Sum(56) + res[57] = fma52hi(res[57], a[11], a[45]); // Sum(56) + res[57] = fma52lo(res[57], a[12], a[45]); // Sum(57) + res[58] = fma52hi(res[58], a[12], a[45]); // Sum(57) + res[58] = fma52lo(res[58], a[13], a[45]); // Sum(58) + res[59] = fma52hi(res[59], a[13], a[45]); // Sum(58) + res[59] = fma52lo(res[59], a[14], a[45]); // Sum(59) + res[60] = fma52hi(res[60], a[14], a[45]); // Sum(59) + res[48] = fma52lo(res[48], a[2], a[46]); // Sum(48) + res[49] = fma52hi(res[49], a[2], a[46]); // Sum(48) + res[49] = fma52lo(res[49], a[3], a[46]); // Sum(49) + res[50] = fma52hi(res[50], a[3], a[46]); // Sum(49) + res[50] = fma52lo(res[50], a[4], a[46]); // Sum(50) + res[51] = fma52hi(res[51], a[4], a[46]); // Sum(50) + res[51] = fma52lo(res[51], a[5], a[46]); // Sum(51) + res[52] = fma52hi(res[52], a[5], a[46]); // Sum(51) + res[52] = fma52lo(res[52], a[6], a[46]); // Sum(52) + res[53] = fma52hi(res[53], a[6], a[46]); // Sum(52) + res[53] = fma52lo(res[53], a[7], a[46]); // Sum(53) + res[54] = fma52hi(res[54], a[7], a[46]); // Sum(53) + res[54] = fma52lo(res[54], a[8], a[46]); // Sum(54) + res[55] = fma52hi(res[55], a[8], a[46]); // Sum(54) + res[55] = fma52lo(res[55], a[9], a[46]); // Sum(55) + res[56] = fma52hi(res[56], a[9], a[46]); // Sum(55) + res[56] = fma52lo(res[56], a[10], a[46]); // Sum(56) + res[57] = fma52hi(res[57], a[10], a[46]); // Sum(56) + res[57] = fma52lo(res[57], a[11], a[46]); // Sum(57) + res[58] = fma52hi(res[58], a[11], a[46]); // Sum(57) + res[58] = fma52lo(res[58], a[12], a[46]); // Sum(58) + res[59] = fma52hi(res[59], a[12], a[46]); // Sum(58) + res[59] = fma52lo(res[59], a[13], a[46]); // Sum(59) + res[60] = fma52hi(res[60], a[13], a[46]); // Sum(59) + res[48] = fma52lo(res[48], a[1], a[47]); // Sum(48) + res[49] = fma52hi(res[49], a[1], a[47]); // Sum(48) + res[49] = fma52lo(res[49], a[2], a[47]); // Sum(49) + res[50] = fma52hi(res[50], a[2], a[47]); // Sum(49) + res[50] = fma52lo(res[50], a[3], a[47]); // Sum(50) + res[51] = fma52hi(res[51], a[3], a[47]); // Sum(50) + res[51] = fma52lo(res[51], a[4], a[47]); // Sum(51) + res[52] = fma52hi(res[52], a[4], a[47]); // Sum(51) + res[52] = fma52lo(res[52], a[5], a[47]); // Sum(52) + res[53] = fma52hi(res[53], a[5], a[47]); // Sum(52) + res[53] = fma52lo(res[53], a[6], a[47]); // Sum(53) + res[54] = fma52hi(res[54], a[6], a[47]); // Sum(53) + res[54] = fma52lo(res[54], a[7], a[47]); // Sum(54) + res[55] = fma52hi(res[55], a[7], a[47]); // Sum(54) + res[55] = fma52lo(res[55], a[8], a[47]); // Sum(55) + res[56] = fma52hi(res[56], a[8], a[47]); // Sum(55) + res[56] = fma52lo(res[56], a[9], a[47]); // Sum(56) + res[57] = fma52hi(res[57], a[9], a[47]); // Sum(56) + res[57] = fma52lo(res[57], a[10], a[47]); // Sum(57) + res[58] = fma52hi(res[58], a[10], a[47]); // Sum(57) + res[58] = fma52lo(res[58], a[11], a[47]); // Sum(58) + res[59] = fma52hi(res[59], a[11], a[47]); // Sum(58) + res[59] = fma52lo(res[59], a[12], a[47]); // Sum(59) + res[60] = fma52hi(res[60], a[12], a[47]); // Sum(59) + res[48] = fma52lo(res[48], a[0], a[48]); // Sum(48) + res[49] = fma52hi(res[49], a[0], a[48]); // Sum(48) + res[49] = fma52lo(res[49], a[1], a[48]); // Sum(49) + res[50] = fma52hi(res[50], a[1], a[48]); // Sum(49) + res[50] = fma52lo(res[50], a[2], a[48]); // Sum(50) + res[51] = fma52hi(res[51], a[2], a[48]); // Sum(50) + res[51] = fma52lo(res[51], a[3], a[48]); // Sum(51) + res[52] = fma52hi(res[52], a[3], a[48]); // Sum(51) + res[52] = fma52lo(res[52], a[4], a[48]); // Sum(52) + res[53] = fma52hi(res[53], a[4], a[48]); // Sum(52) + res[53] = fma52lo(res[53], a[5], a[48]); // Sum(53) + res[54] = fma52hi(res[54], a[5], a[48]); // Sum(53) + res[54] = fma52lo(res[54], a[6], a[48]); // Sum(54) + res[55] = fma52hi(res[55], a[6], a[48]); // Sum(54) + res[55] = fma52lo(res[55], a[7], a[48]); // Sum(55) + res[56] = fma52hi(res[56], a[7], a[48]); // Sum(55) + res[56] = fma52lo(res[56], a[8], a[48]); // Sum(56) + res[57] = fma52hi(res[57], a[8], a[48]); // Sum(56) + res[57] = fma52lo(res[57], a[9], a[48]); // Sum(57) + res[58] = fma52hi(res[58], a[9], a[48]); // Sum(57) + res[58] = fma52lo(res[58], a[10], a[48]); // Sum(58) + res[59] = fma52hi(res[59], a[10], a[48]); // Sum(58) + res[59] = fma52lo(res[59], a[11], a[48]); // Sum(59) + res[60] = fma52hi(res[60], a[11], a[48]); // Sum(59) + res[49] = fma52lo(res[49], a[0], a[49]); // Sum(49) + res[50] = fma52hi(res[50], a[0], a[49]); // Sum(49) + res[50] = fma52lo(res[50], a[1], a[49]); // Sum(50) + res[51] = fma52hi(res[51], a[1], a[49]); // Sum(50) + res[51] = fma52lo(res[51], a[2], a[49]); // Sum(51) + res[52] = fma52hi(res[52], a[2], a[49]); // Sum(51) + res[52] = fma52lo(res[52], a[3], a[49]); // Sum(52) + res[53] = fma52hi(res[53], a[3], a[49]); // Sum(52) + res[53] = fma52lo(res[53], a[4], a[49]); // Sum(53) + res[54] = fma52hi(res[54], a[4], a[49]); // Sum(53) + res[54] = fma52lo(res[54], a[5], a[49]); // Sum(54) + res[55] = fma52hi(res[55], a[5], a[49]); // Sum(54) + res[55] = fma52lo(res[55], a[6], a[49]); // Sum(55) + res[56] = fma52hi(res[56], a[6], a[49]); // Sum(55) + res[56] = fma52lo(res[56], a[7], a[49]); // Sum(56) + res[57] = fma52hi(res[57], a[7], a[49]); // Sum(56) + res[57] = fma52lo(res[57], a[8], a[49]); // Sum(57) + res[58] = fma52hi(res[58], a[8], a[49]); // Sum(57) + res[58] = fma52lo(res[58], a[9], a[49]); // Sum(58) + res[59] = fma52hi(res[59], a[9], a[49]); // Sum(58) + res[59] = fma52lo(res[59], a[10], a[49]); // Sum(59) + res[60] = fma52hi(res[60], a[10], a[49]); // Sum(59) + res[50] = fma52lo(res[50], a[0], a[50]); // Sum(50) + res[51] = fma52hi(res[51], a[0], a[50]); // Sum(50) + res[51] = fma52lo(res[51], a[1], a[50]); // Sum(51) + res[52] = fma52hi(res[52], a[1], a[50]); // Sum(51) + res[52] = fma52lo(res[52], a[2], a[50]); // Sum(52) + res[53] = fma52hi(res[53], a[2], a[50]); // Sum(52) + res[53] = fma52lo(res[53], a[3], a[50]); // Sum(53) + res[54] = fma52hi(res[54], a[3], a[50]); // Sum(53) + res[54] = fma52lo(res[54], a[4], a[50]); // Sum(54) + res[55] = fma52hi(res[55], a[4], a[50]); // Sum(54) + res[55] = fma52lo(res[55], a[5], a[50]); // Sum(55) + res[56] = fma52hi(res[56], a[5], a[50]); // Sum(55) + res[56] = fma52lo(res[56], a[6], a[50]); // Sum(56) + res[57] = fma52hi(res[57], a[6], a[50]); // Sum(56) + res[57] = fma52lo(res[57], a[7], a[50]); // Sum(57) + res[58] = fma52hi(res[58], a[7], a[50]); // Sum(57) + res[58] = fma52lo(res[58], a[8], a[50]); // Sum(58) + res[59] = fma52hi(res[59], a[8], a[50]); // Sum(58) + res[59] = fma52lo(res[59], a[9], a[50]); // Sum(59) + res[60] = fma52hi(res[60], a[9], a[50]); // Sum(59) + res[51] = fma52lo(res[51], a[0], a[51]); // Sum(51) + res[52] = fma52hi(res[52], a[0], a[51]); // Sum(51) + res[52] = fma52lo(res[52], a[1], a[51]); // Sum(52) + res[53] = fma52hi(res[53], a[1], a[51]); // Sum(52) + res[53] = fma52lo(res[53], a[2], a[51]); // Sum(53) + res[54] = fma52hi(res[54], a[2], a[51]); // Sum(53) + res[54] = fma52lo(res[54], a[3], a[51]); // Sum(54) + res[55] = fma52hi(res[55], a[3], a[51]); // Sum(54) + res[55] = fma52lo(res[55], a[4], a[51]); // Sum(55) + res[56] = fma52hi(res[56], a[4], a[51]); // Sum(55) + res[56] = fma52lo(res[56], a[5], a[51]); // Sum(56) + res[57] = fma52hi(res[57], a[5], a[51]); // Sum(56) + res[57] = fma52lo(res[57], a[6], a[51]); // Sum(57) + res[58] = fma52hi(res[58], a[6], a[51]); // Sum(57) + res[58] = fma52lo(res[58], a[7], a[51]); // Sum(58) + res[59] = fma52hi(res[59], a[7], a[51]); // Sum(58) + res[59] = fma52lo(res[59], a[8], a[51]); // Sum(59) + res[60] = fma52hi(res[60], a[8], a[51]); // Sum(59) + res[52] = fma52lo(res[52], a[0], a[52]); // Sum(52) + res[53] = fma52hi(res[53], a[0], a[52]); // Sum(52) + res[53] = fma52lo(res[53], a[1], a[52]); // Sum(53) + res[54] = fma52hi(res[54], a[1], a[52]); // Sum(53) + res[54] = fma52lo(res[54], a[2], a[52]); // Sum(54) + res[55] = fma52hi(res[55], a[2], a[52]); // Sum(54) + res[55] = fma52lo(res[55], a[3], a[52]); // Sum(55) + res[56] = fma52hi(res[56], a[3], a[52]); // Sum(55) + res[56] = fma52lo(res[56], a[4], a[52]); // Sum(56) + res[57] = fma52hi(res[57], a[4], a[52]); // Sum(56) + res[57] = fma52lo(res[57], a[5], a[52]); // Sum(57) + res[58] = fma52hi(res[58], a[5], a[52]); // Sum(57) + res[58] = fma52lo(res[58], a[6], a[52]); // Sum(58) + res[59] = fma52hi(res[59], a[6], a[52]); // Sum(58) + res[59] = fma52lo(res[59], a[7], a[52]); // Sum(59) + res[60] = fma52hi(res[60], a[7], a[52]); // Sum(59) + res[53] = fma52lo(res[53], a[0], a[53]); // Sum(53) + res[54] = fma52hi(res[54], a[0], a[53]); // Sum(53) + res[54] = fma52lo(res[54], a[1], a[53]); // Sum(54) + res[55] = fma52hi(res[55], a[1], a[53]); // Sum(54) + res[55] = fma52lo(res[55], a[2], a[53]); // Sum(55) + res[56] = fma52hi(res[56], a[2], a[53]); // Sum(55) + res[56] = fma52lo(res[56], a[3], a[53]); // Sum(56) + res[57] = fma52hi(res[57], a[3], a[53]); // Sum(56) + res[57] = fma52lo(res[57], a[4], a[53]); // Sum(57) + res[58] = fma52hi(res[58], a[4], a[53]); // Sum(57) + res[58] = fma52lo(res[58], a[5], a[53]); // Sum(58) + res[59] = fma52hi(res[59], a[5], a[53]); // Sum(58) + res[59] = fma52lo(res[59], a[6], a[53]); // Sum(59) + res[60] = fma52hi(res[60], a[6], a[53]); // Sum(59) + res[54] = fma52lo(res[54], a[0], a[54]); // Sum(54) + res[55] = fma52hi(res[55], a[0], a[54]); // Sum(54) + res[55] = fma52lo(res[55], a[1], a[54]); // Sum(55) + res[56] = fma52hi(res[56], a[1], a[54]); // Sum(55) + res[56] = fma52lo(res[56], a[2], a[54]); // Sum(56) + res[57] = fma52hi(res[57], a[2], a[54]); // Sum(56) + res[57] = fma52lo(res[57], a[3], a[54]); // Sum(57) + res[58] = fma52hi(res[58], a[3], a[54]); // Sum(57) + res[58] = fma52lo(res[58], a[4], a[54]); // Sum(58) + res[59] = fma52hi(res[59], a[4], a[54]); // Sum(58) + res[59] = fma52lo(res[59], a[5], a[54]); // Sum(59) + res[60] = fma52hi(res[60], a[5], a[54]); // Sum(59) + res[55] = fma52lo(res[55], a[0], a[55]); // Sum(55) + res[56] = fma52hi(res[56], a[0], a[55]); // Sum(55) + res[56] = fma52lo(res[56], a[1], a[55]); // Sum(56) + res[57] = fma52hi(res[57], a[1], a[55]); // Sum(56) + res[57] = fma52lo(res[57], a[2], a[55]); // Sum(57) + res[58] = fma52hi(res[58], a[2], a[55]); // Sum(57) + res[58] = fma52lo(res[58], a[3], a[55]); // Sum(58) + res[59] = fma52hi(res[59], a[3], a[55]); // Sum(58) + res[59] = fma52lo(res[59], a[4], a[55]); // Sum(59) + res[60] = fma52hi(res[60], a[4], a[55]); // Sum(59) + res[56] = fma52lo(res[56], a[0], a[56]); // Sum(56) + res[57] = fma52hi(res[57], a[0], a[56]); // Sum(56) + res[57] = fma52lo(res[57], a[1], a[56]); // Sum(57) + res[58] = fma52hi(res[58], a[1], a[56]); // Sum(57) + res[58] = fma52lo(res[58], a[2], a[56]); // Sum(58) + res[59] = fma52hi(res[59], a[2], a[56]); // Sum(58) + res[59] = fma52lo(res[59], a[3], a[56]); // Sum(59) + res[60] = fma52hi(res[60], a[3], a[56]); // Sum(59) + res[57] = fma52lo(res[57], a[0], a[57]); // Sum(57) + res[58] = fma52hi(res[58], a[0], a[57]); // Sum(57) + res[58] = fma52lo(res[58], a[1], a[57]); // Sum(58) + res[59] = fma52hi(res[59], a[1], a[57]); // Sum(58) + res[59] = fma52lo(res[59], a[2], a[57]); // Sum(59) + res[60] = fma52hi(res[60], a[2], a[57]); // Sum(59) + res[58] = fma52lo(res[58], a[0], a[58]); // Sum(58) + res[59] = fma52hi(res[59], a[0], a[58]); // Sum(58) + res[59] = fma52lo(res[59], a[1], a[58]); // Sum(59) + res[60] = fma52hi(res[60], a[1], a[58]); // Sum(59) + res[59] = fma52lo(res[59], a[0], a[59]); // Sum(59) + res[60] = fma52hi(res[60], a[0], a[59]); // Sum(59) + res[48] = add64(res[48], res[48]); // Double(48) + res[49] = add64(res[49], res[49]); // Double(49) + res[50] = add64(res[50], res[50]); // Double(50) + res[51] = add64(res[51], res[51]); // Double(51) + res[52] = add64(res[52], res[52]); // Double(52) + res[53] = add64(res[53], res[53]); // Double(53) + res[54] = add64(res[54], res[54]); // Double(54) + res[55] = add64(res[55], res[55]); // Double(55) + res[56] = add64(res[56], res[56]); // Double(56) + res[57] = add64(res[57], res[57]); // Double(57) + res[58] = add64(res[58], res[58]); // Double(58) + res[59] = add64(res[59], res[59]); // Double(59) + res[48] = fma52lo(res[48], a[24], a[24]); // Add sqr(48) + res[49] = fma52hi(res[49], a[24], a[24]); // Add sqr(48) + res[50] = fma52lo(res[50], a[25], a[25]); // Add sqr(50) + res[51] = fma52hi(res[51], a[25], a[25]); // Add sqr(50) + res[52] = fma52lo(res[52], a[26], a[26]); // Add sqr(52) + res[53] = fma52hi(res[53], a[26], a[26]); // Add sqr(52) + res[54] = fma52lo(res[54], a[27], a[27]); // Add sqr(54) + res[55] = fma52hi(res[55], a[27], a[27]); // Add sqr(54) + res[56] = fma52lo(res[56], a[28], a[28]); // Add sqr(56) + res[57] = fma52hi(res[57], a[28], a[28]); // Add sqr(56) + res[58] = fma52lo(res[58], a[29], a[29]); // Add sqr(58) + res[59] = fma52hi(res[59], a[29], a[29]); // Add sqr(58) + res[60] = fma52lo(res[60], a[29], a[31]); // Sum(60) + res[61] = fma52hi(res[61], a[29], a[31]); // Sum(60) + res[61] = fma52lo(res[61], a[30], a[31]); // Sum(61) + res[62] = fma52hi(res[62], a[30], a[31]); // Sum(61) + res[60] = fma52lo(res[60], a[28], a[32]); // Sum(60) + res[61] = fma52hi(res[61], a[28], a[32]); // Sum(60) + res[61] = fma52lo(res[61], a[29], a[32]); // Sum(61) + res[62] = fma52hi(res[62], a[29], a[32]); // Sum(61) + res[62] = fma52lo(res[62], a[30], a[32]); // Sum(62) + res[63] = fma52hi(res[63], a[30], a[32]); // Sum(62) + res[63] = fma52lo(res[63], a[31], a[32]); // Sum(63) + res[64] = fma52hi(res[64], a[31], a[32]); // Sum(63) + res[60] = fma52lo(res[60], a[27], a[33]); // Sum(60) + res[61] = fma52hi(res[61], a[27], a[33]); // Sum(60) + res[61] = fma52lo(res[61], a[28], a[33]); // Sum(61) + res[62] = fma52hi(res[62], a[28], a[33]); // Sum(61) + res[62] = fma52lo(res[62], a[29], a[33]); // Sum(62) + res[63] = fma52hi(res[63], a[29], a[33]); // Sum(62) + res[63] = fma52lo(res[63], a[30], a[33]); // Sum(63) + res[64] = fma52hi(res[64], a[30], a[33]); // Sum(63) + res[64] = fma52lo(res[64], a[31], a[33]); // Sum(64) + res[65] = fma52hi(res[65], a[31], a[33]); // Sum(64) + res[65] = fma52lo(res[65], a[32], a[33]); // Sum(65) + res[66] = fma52hi(res[66], a[32], a[33]); // Sum(65) + res[60] = fma52lo(res[60], a[26], a[34]); // Sum(60) + res[61] = fma52hi(res[61], a[26], a[34]); // Sum(60) + res[61] = fma52lo(res[61], a[27], a[34]); // Sum(61) + res[62] = fma52hi(res[62], a[27], a[34]); // Sum(61) + res[62] = fma52lo(res[62], a[28], a[34]); // Sum(62) + res[63] = fma52hi(res[63], a[28], a[34]); // Sum(62) + res[63] = fma52lo(res[63], a[29], a[34]); // Sum(63) + res[64] = fma52hi(res[64], a[29], a[34]); // Sum(63) + res[64] = fma52lo(res[64], a[30], a[34]); // Sum(64) + res[65] = fma52hi(res[65], a[30], a[34]); // Sum(64) + res[65] = fma52lo(res[65], a[31], a[34]); // Sum(65) + res[66] = fma52hi(res[66], a[31], a[34]); // Sum(65) + res[66] = fma52lo(res[66], a[32], a[34]); // Sum(66) + res[67] = fma52hi(res[67], a[32], a[34]); // Sum(66) + res[67] = fma52lo(res[67], a[33], a[34]); // Sum(67) + res[68] = fma52hi(res[68], a[33], a[34]); // Sum(67) + res[60] = fma52lo(res[60], a[25], a[35]); // Sum(60) + res[61] = fma52hi(res[61], a[25], a[35]); // Sum(60) + res[61] = fma52lo(res[61], a[26], a[35]); // Sum(61) + res[62] = fma52hi(res[62], a[26], a[35]); // Sum(61) + res[62] = fma52lo(res[62], a[27], a[35]); // Sum(62) + res[63] = fma52hi(res[63], a[27], a[35]); // Sum(62) + res[63] = fma52lo(res[63], a[28], a[35]); // Sum(63) + res[64] = fma52hi(res[64], a[28], a[35]); // Sum(63) + res[64] = fma52lo(res[64], a[29], a[35]); // Sum(64) + res[65] = fma52hi(res[65], a[29], a[35]); // Sum(64) + res[65] = fma52lo(res[65], a[30], a[35]); // Sum(65) + res[66] = fma52hi(res[66], a[30], a[35]); // Sum(65) + res[66] = fma52lo(res[66], a[31], a[35]); // Sum(66) + res[67] = fma52hi(res[67], a[31], a[35]); // Sum(66) + res[67] = fma52lo(res[67], a[32], a[35]); // Sum(67) + res[68] = fma52hi(res[68], a[32], a[35]); // Sum(67) + res[68] = fma52lo(res[68], a[33], a[35]); // Sum(68) + res[69] = fma52hi(res[69], a[33], a[35]); // Sum(68) + res[69] = fma52lo(res[69], a[34], a[35]); // Sum(69) + res[70] = fma52hi(res[70], a[34], a[35]); // Sum(69) + res[60] = fma52lo(res[60], a[24], a[36]); // Sum(60) + res[61] = fma52hi(res[61], a[24], a[36]); // Sum(60) + res[61] = fma52lo(res[61], a[25], a[36]); // Sum(61) + res[62] = fma52hi(res[62], a[25], a[36]); // Sum(61) + res[62] = fma52lo(res[62], a[26], a[36]); // Sum(62) + res[63] = fma52hi(res[63], a[26], a[36]); // Sum(62) + res[63] = fma52lo(res[63], a[27], a[36]); // Sum(63) + res[64] = fma52hi(res[64], a[27], a[36]); // Sum(63) + res[64] = fma52lo(res[64], a[28], a[36]); // Sum(64) + res[65] = fma52hi(res[65], a[28], a[36]); // Sum(64) + res[65] = fma52lo(res[65], a[29], a[36]); // Sum(65) + res[66] = fma52hi(res[66], a[29], a[36]); // Sum(65) + res[66] = fma52lo(res[66], a[30], a[36]); // Sum(66) + res[67] = fma52hi(res[67], a[30], a[36]); // Sum(66) + res[67] = fma52lo(res[67], a[31], a[36]); // Sum(67) + res[68] = fma52hi(res[68], a[31], a[36]); // Sum(67) + res[68] = fma52lo(res[68], a[32], a[36]); // Sum(68) + res[69] = fma52hi(res[69], a[32], a[36]); // Sum(68) + res[69] = fma52lo(res[69], a[33], a[36]); // Sum(69) + res[70] = fma52hi(res[70], a[33], a[36]); // Sum(69) + res[70] = fma52lo(res[70], a[34], a[36]); // Sum(70) + res[71] = fma52hi(res[71], a[34], a[36]); // Sum(70) + res[71] = fma52lo(res[71], a[35], a[36]); // Sum(71) + res[72] = fma52hi(res[72], a[35], a[36]); // Sum(71) + res[60] = fma52lo(res[60], a[23], a[37]); // Sum(60) + res[61] = fma52hi(res[61], a[23], a[37]); // Sum(60) + res[61] = fma52lo(res[61], a[24], a[37]); // Sum(61) + res[62] = fma52hi(res[62], a[24], a[37]); // Sum(61) + res[62] = fma52lo(res[62], a[25], a[37]); // Sum(62) + res[63] = fma52hi(res[63], a[25], a[37]); // Sum(62) + res[63] = fma52lo(res[63], a[26], a[37]); // Sum(63) + res[64] = fma52hi(res[64], a[26], a[37]); // Sum(63) + res[64] = fma52lo(res[64], a[27], a[37]); // Sum(64) + res[65] = fma52hi(res[65], a[27], a[37]); // Sum(64) + res[65] = fma52lo(res[65], a[28], a[37]); // Sum(65) + res[66] = fma52hi(res[66], a[28], a[37]); // Sum(65) + res[66] = fma52lo(res[66], a[29], a[37]); // Sum(66) + res[67] = fma52hi(res[67], a[29], a[37]); // Sum(66) + res[67] = fma52lo(res[67], a[30], a[37]); // Sum(67) + res[68] = fma52hi(res[68], a[30], a[37]); // Sum(67) + res[68] = fma52lo(res[68], a[31], a[37]); // Sum(68) + res[69] = fma52hi(res[69], a[31], a[37]); // Sum(68) + res[69] = fma52lo(res[69], a[32], a[37]); // Sum(69) + res[70] = fma52hi(res[70], a[32], a[37]); // Sum(69) + res[70] = fma52lo(res[70], a[33], a[37]); // Sum(70) + res[71] = fma52hi(res[71], a[33], a[37]); // Sum(70) + res[71] = fma52lo(res[71], a[34], a[37]); // Sum(71) + res[72] = fma52hi(res[72], a[34], a[37]); // Sum(71) + res[60] = fma52lo(res[60], a[22], a[38]); // Sum(60) + res[61] = fma52hi(res[61], a[22], a[38]); // Sum(60) + res[61] = fma52lo(res[61], a[23], a[38]); // Sum(61) + res[62] = fma52hi(res[62], a[23], a[38]); // Sum(61) + res[62] = fma52lo(res[62], a[24], a[38]); // Sum(62) + res[63] = fma52hi(res[63], a[24], a[38]); // Sum(62) + res[63] = fma52lo(res[63], a[25], a[38]); // Sum(63) + res[64] = fma52hi(res[64], a[25], a[38]); // Sum(63) + res[64] = fma52lo(res[64], a[26], a[38]); // Sum(64) + res[65] = fma52hi(res[65], a[26], a[38]); // Sum(64) + res[65] = fma52lo(res[65], a[27], a[38]); // Sum(65) + res[66] = fma52hi(res[66], a[27], a[38]); // Sum(65) + res[66] = fma52lo(res[66], a[28], a[38]); // Sum(66) + res[67] = fma52hi(res[67], a[28], a[38]); // Sum(66) + res[67] = fma52lo(res[67], a[29], a[38]); // Sum(67) + res[68] = fma52hi(res[68], a[29], a[38]); // Sum(67) + res[68] = fma52lo(res[68], a[30], a[38]); // Sum(68) + res[69] = fma52hi(res[69], a[30], a[38]); // Sum(68) + res[69] = fma52lo(res[69], a[31], a[38]); // Sum(69) + res[70] = fma52hi(res[70], a[31], a[38]); // Sum(69) + res[70] = fma52lo(res[70], a[32], a[38]); // Sum(70) + res[71] = fma52hi(res[71], a[32], a[38]); // Sum(70) + res[71] = fma52lo(res[71], a[33], a[38]); // Sum(71) + res[72] = fma52hi(res[72], a[33], a[38]); // Sum(71) + res[60] = fma52lo(res[60], a[21], a[39]); // Sum(60) + res[61] = fma52hi(res[61], a[21], a[39]); // Sum(60) + res[61] = fma52lo(res[61], a[22], a[39]); // Sum(61) + res[62] = fma52hi(res[62], a[22], a[39]); // Sum(61) + res[62] = fma52lo(res[62], a[23], a[39]); // Sum(62) + res[63] = fma52hi(res[63], a[23], a[39]); // Sum(62) + res[63] = fma52lo(res[63], a[24], a[39]); // Sum(63) + res[64] = fma52hi(res[64], a[24], a[39]); // Sum(63) + res[64] = fma52lo(res[64], a[25], a[39]); // Sum(64) + res[65] = fma52hi(res[65], a[25], a[39]); // Sum(64) + res[65] = fma52lo(res[65], a[26], a[39]); // Sum(65) + res[66] = fma52hi(res[66], a[26], a[39]); // Sum(65) + res[66] = fma52lo(res[66], a[27], a[39]); // Sum(66) + res[67] = fma52hi(res[67], a[27], a[39]); // Sum(66) + res[67] = fma52lo(res[67], a[28], a[39]); // Sum(67) + res[68] = fma52hi(res[68], a[28], a[39]); // Sum(67) + res[68] = fma52lo(res[68], a[29], a[39]); // Sum(68) + res[69] = fma52hi(res[69], a[29], a[39]); // Sum(68) + res[69] = fma52lo(res[69], a[30], a[39]); // Sum(69) + res[70] = fma52hi(res[70], a[30], a[39]); // Sum(69) + res[70] = fma52lo(res[70], a[31], a[39]); // Sum(70) + res[71] = fma52hi(res[71], a[31], a[39]); // Sum(70) + res[71] = fma52lo(res[71], a[32], a[39]); // Sum(71) + res[72] = fma52hi(res[72], a[32], a[39]); // Sum(71) + res[60] = fma52lo(res[60], a[20], a[40]); // Sum(60) + res[61] = fma52hi(res[61], a[20], a[40]); // Sum(60) + res[61] = fma52lo(res[61], a[21], a[40]); // Sum(61) + res[62] = fma52hi(res[62], a[21], a[40]); // Sum(61) + res[62] = fma52lo(res[62], a[22], a[40]); // Sum(62) + res[63] = fma52hi(res[63], a[22], a[40]); // Sum(62) + res[63] = fma52lo(res[63], a[23], a[40]); // Sum(63) + res[64] = fma52hi(res[64], a[23], a[40]); // Sum(63) + res[64] = fma52lo(res[64], a[24], a[40]); // Sum(64) + res[65] = fma52hi(res[65], a[24], a[40]); // Sum(64) + res[65] = fma52lo(res[65], a[25], a[40]); // Sum(65) + res[66] = fma52hi(res[66], a[25], a[40]); // Sum(65) + res[66] = fma52lo(res[66], a[26], a[40]); // Sum(66) + res[67] = fma52hi(res[67], a[26], a[40]); // Sum(66) + res[67] = fma52lo(res[67], a[27], a[40]); // Sum(67) + res[68] = fma52hi(res[68], a[27], a[40]); // Sum(67) + res[68] = fma52lo(res[68], a[28], a[40]); // Sum(68) + res[69] = fma52hi(res[69], a[28], a[40]); // Sum(68) + res[69] = fma52lo(res[69], a[29], a[40]); // Sum(69) + res[70] = fma52hi(res[70], a[29], a[40]); // Sum(69) + res[70] = fma52lo(res[70], a[30], a[40]); // Sum(70) + res[71] = fma52hi(res[71], a[30], a[40]); // Sum(70) + res[71] = fma52lo(res[71], a[31], a[40]); // Sum(71) + res[72] = fma52hi(res[72], a[31], a[40]); // Sum(71) + res[60] = fma52lo(res[60], a[19], a[41]); // Sum(60) + res[61] = fma52hi(res[61], a[19], a[41]); // Sum(60) + res[61] = fma52lo(res[61], a[20], a[41]); // Sum(61) + res[62] = fma52hi(res[62], a[20], a[41]); // Sum(61) + res[62] = fma52lo(res[62], a[21], a[41]); // Sum(62) + res[63] = fma52hi(res[63], a[21], a[41]); // Sum(62) + res[63] = fma52lo(res[63], a[22], a[41]); // Sum(63) + res[64] = fma52hi(res[64], a[22], a[41]); // Sum(63) + res[64] = fma52lo(res[64], a[23], a[41]); // Sum(64) + res[65] = fma52hi(res[65], a[23], a[41]); // Sum(64) + res[65] = fma52lo(res[65], a[24], a[41]); // Sum(65) + res[66] = fma52hi(res[66], a[24], a[41]); // Sum(65) + res[66] = fma52lo(res[66], a[25], a[41]); // Sum(66) + res[67] = fma52hi(res[67], a[25], a[41]); // Sum(66) + res[67] = fma52lo(res[67], a[26], a[41]); // Sum(67) + res[68] = fma52hi(res[68], a[26], a[41]); // Sum(67) + res[68] = fma52lo(res[68], a[27], a[41]); // Sum(68) + res[69] = fma52hi(res[69], a[27], a[41]); // Sum(68) + res[69] = fma52lo(res[69], a[28], a[41]); // Sum(69) + res[70] = fma52hi(res[70], a[28], a[41]); // Sum(69) + res[70] = fma52lo(res[70], a[29], a[41]); // Sum(70) + res[71] = fma52hi(res[71], a[29], a[41]); // Sum(70) + res[71] = fma52lo(res[71], a[30], a[41]); // Sum(71) + res[72] = fma52hi(res[72], a[30], a[41]); // Sum(71) + res[60] = fma52lo(res[60], a[18], a[42]); // Sum(60) + res[61] = fma52hi(res[61], a[18], a[42]); // Sum(60) + res[61] = fma52lo(res[61], a[19], a[42]); // Sum(61) + res[62] = fma52hi(res[62], a[19], a[42]); // Sum(61) + res[62] = fma52lo(res[62], a[20], a[42]); // Sum(62) + res[63] = fma52hi(res[63], a[20], a[42]); // Sum(62) + res[63] = fma52lo(res[63], a[21], a[42]); // Sum(63) + res[64] = fma52hi(res[64], a[21], a[42]); // Sum(63) + res[64] = fma52lo(res[64], a[22], a[42]); // Sum(64) + res[65] = fma52hi(res[65], a[22], a[42]); // Sum(64) + res[65] = fma52lo(res[65], a[23], a[42]); // Sum(65) + res[66] = fma52hi(res[66], a[23], a[42]); // Sum(65) + res[66] = fma52lo(res[66], a[24], a[42]); // Sum(66) + res[67] = fma52hi(res[67], a[24], a[42]); // Sum(66) + res[67] = fma52lo(res[67], a[25], a[42]); // Sum(67) + res[68] = fma52hi(res[68], a[25], a[42]); // Sum(67) + res[68] = fma52lo(res[68], a[26], a[42]); // Sum(68) + res[69] = fma52hi(res[69], a[26], a[42]); // Sum(68) + res[69] = fma52lo(res[69], a[27], a[42]); // Sum(69) + res[70] = fma52hi(res[70], a[27], a[42]); // Sum(69) + res[70] = fma52lo(res[70], a[28], a[42]); // Sum(70) + res[71] = fma52hi(res[71], a[28], a[42]); // Sum(70) + res[71] = fma52lo(res[71], a[29], a[42]); // Sum(71) + res[72] = fma52hi(res[72], a[29], a[42]); // Sum(71) + res[60] = fma52lo(res[60], a[17], a[43]); // Sum(60) + res[61] = fma52hi(res[61], a[17], a[43]); // Sum(60) + res[61] = fma52lo(res[61], a[18], a[43]); // Sum(61) + res[62] = fma52hi(res[62], a[18], a[43]); // Sum(61) + res[62] = fma52lo(res[62], a[19], a[43]); // Sum(62) + res[63] = fma52hi(res[63], a[19], a[43]); // Sum(62) + res[63] = fma52lo(res[63], a[20], a[43]); // Sum(63) + res[64] = fma52hi(res[64], a[20], a[43]); // Sum(63) + res[64] = fma52lo(res[64], a[21], a[43]); // Sum(64) + res[65] = fma52hi(res[65], a[21], a[43]); // Sum(64) + res[65] = fma52lo(res[65], a[22], a[43]); // Sum(65) + res[66] = fma52hi(res[66], a[22], a[43]); // Sum(65) + res[66] = fma52lo(res[66], a[23], a[43]); // Sum(66) + res[67] = fma52hi(res[67], a[23], a[43]); // Sum(66) + res[67] = fma52lo(res[67], a[24], a[43]); // Sum(67) + res[68] = fma52hi(res[68], a[24], a[43]); // Sum(67) + res[68] = fma52lo(res[68], a[25], a[43]); // Sum(68) + res[69] = fma52hi(res[69], a[25], a[43]); // Sum(68) + res[69] = fma52lo(res[69], a[26], a[43]); // Sum(69) + res[70] = fma52hi(res[70], a[26], a[43]); // Sum(69) + res[70] = fma52lo(res[70], a[27], a[43]); // Sum(70) + res[71] = fma52hi(res[71], a[27], a[43]); // Sum(70) + res[71] = fma52lo(res[71], a[28], a[43]); // Sum(71) + res[72] = fma52hi(res[72], a[28], a[43]); // Sum(71) + res[60] = fma52lo(res[60], a[16], a[44]); // Sum(60) + res[61] = fma52hi(res[61], a[16], a[44]); // Sum(60) + res[61] = fma52lo(res[61], a[17], a[44]); // Sum(61) + res[62] = fma52hi(res[62], a[17], a[44]); // Sum(61) + res[62] = fma52lo(res[62], a[18], a[44]); // Sum(62) + res[63] = fma52hi(res[63], a[18], a[44]); // Sum(62) + res[63] = fma52lo(res[63], a[19], a[44]); // Sum(63) + res[64] = fma52hi(res[64], a[19], a[44]); // Sum(63) + res[64] = fma52lo(res[64], a[20], a[44]); // Sum(64) + res[65] = fma52hi(res[65], a[20], a[44]); // Sum(64) + res[65] = fma52lo(res[65], a[21], a[44]); // Sum(65) + res[66] = fma52hi(res[66], a[21], a[44]); // Sum(65) + res[66] = fma52lo(res[66], a[22], a[44]); // Sum(66) + res[67] = fma52hi(res[67], a[22], a[44]); // Sum(66) + res[67] = fma52lo(res[67], a[23], a[44]); // Sum(67) + res[68] = fma52hi(res[68], a[23], a[44]); // Sum(67) + res[68] = fma52lo(res[68], a[24], a[44]); // Sum(68) + res[69] = fma52hi(res[69], a[24], a[44]); // Sum(68) + res[69] = fma52lo(res[69], a[25], a[44]); // Sum(69) + res[70] = fma52hi(res[70], a[25], a[44]); // Sum(69) + res[70] = fma52lo(res[70], a[26], a[44]); // Sum(70) + res[71] = fma52hi(res[71], a[26], a[44]); // Sum(70) + res[71] = fma52lo(res[71], a[27], a[44]); // Sum(71) + res[72] = fma52hi(res[72], a[27], a[44]); // Sum(71) + res[60] = fma52lo(res[60], a[15], a[45]); // Sum(60) + res[61] = fma52hi(res[61], a[15], a[45]); // Sum(60) + res[61] = fma52lo(res[61], a[16], a[45]); // Sum(61) + res[62] = fma52hi(res[62], a[16], a[45]); // Sum(61) + res[62] = fma52lo(res[62], a[17], a[45]); // Sum(62) + res[63] = fma52hi(res[63], a[17], a[45]); // Sum(62) + res[63] = fma52lo(res[63], a[18], a[45]); // Sum(63) + res[64] = fma52hi(res[64], a[18], a[45]); // Sum(63) + res[64] = fma52lo(res[64], a[19], a[45]); // Sum(64) + res[65] = fma52hi(res[65], a[19], a[45]); // Sum(64) + res[65] = fma52lo(res[65], a[20], a[45]); // Sum(65) + res[66] = fma52hi(res[66], a[20], a[45]); // Sum(65) + res[66] = fma52lo(res[66], a[21], a[45]); // Sum(66) + res[67] = fma52hi(res[67], a[21], a[45]); // Sum(66) + res[67] = fma52lo(res[67], a[22], a[45]); // Sum(67) + res[68] = fma52hi(res[68], a[22], a[45]); // Sum(67) + res[68] = fma52lo(res[68], a[23], a[45]); // Sum(68) + res[69] = fma52hi(res[69], a[23], a[45]); // Sum(68) + res[69] = fma52lo(res[69], a[24], a[45]); // Sum(69) + res[70] = fma52hi(res[70], a[24], a[45]); // Sum(69) + res[70] = fma52lo(res[70], a[25], a[45]); // Sum(70) + res[71] = fma52hi(res[71], a[25], a[45]); // Sum(70) + res[71] = fma52lo(res[71], a[26], a[45]); // Sum(71) + res[72] = fma52hi(res[72], a[26], a[45]); // Sum(71) + res[60] = fma52lo(res[60], a[14], a[46]); // Sum(60) + res[61] = fma52hi(res[61], a[14], a[46]); // Sum(60) + res[61] = fma52lo(res[61], a[15], a[46]); // Sum(61) + res[62] = fma52hi(res[62], a[15], a[46]); // Sum(61) + res[62] = fma52lo(res[62], a[16], a[46]); // Sum(62) + res[63] = fma52hi(res[63], a[16], a[46]); // Sum(62) + res[63] = fma52lo(res[63], a[17], a[46]); // Sum(63) + res[64] = fma52hi(res[64], a[17], a[46]); // Sum(63) + res[64] = fma52lo(res[64], a[18], a[46]); // Sum(64) + res[65] = fma52hi(res[65], a[18], a[46]); // Sum(64) + res[65] = fma52lo(res[65], a[19], a[46]); // Sum(65) + res[66] = fma52hi(res[66], a[19], a[46]); // Sum(65) + res[66] = fma52lo(res[66], a[20], a[46]); // Sum(66) + res[67] = fma52hi(res[67], a[20], a[46]); // Sum(66) + res[67] = fma52lo(res[67], a[21], a[46]); // Sum(67) + res[68] = fma52hi(res[68], a[21], a[46]); // Sum(67) + res[68] = fma52lo(res[68], a[22], a[46]); // Sum(68) + res[69] = fma52hi(res[69], a[22], a[46]); // Sum(68) + res[69] = fma52lo(res[69], a[23], a[46]); // Sum(69) + res[70] = fma52hi(res[70], a[23], a[46]); // Sum(69) + res[70] = fma52lo(res[70], a[24], a[46]); // Sum(70) + res[71] = fma52hi(res[71], a[24], a[46]); // Sum(70) + res[71] = fma52lo(res[71], a[25], a[46]); // Sum(71) + res[72] = fma52hi(res[72], a[25], a[46]); // Sum(71) + res[60] = fma52lo(res[60], a[13], a[47]); // Sum(60) + res[61] = fma52hi(res[61], a[13], a[47]); // Sum(60) + res[61] = fma52lo(res[61], a[14], a[47]); // Sum(61) + res[62] = fma52hi(res[62], a[14], a[47]); // Sum(61) + res[62] = fma52lo(res[62], a[15], a[47]); // Sum(62) + res[63] = fma52hi(res[63], a[15], a[47]); // Sum(62) + res[63] = fma52lo(res[63], a[16], a[47]); // Sum(63) + res[64] = fma52hi(res[64], a[16], a[47]); // Sum(63) + res[64] = fma52lo(res[64], a[17], a[47]); // Sum(64) + res[65] = fma52hi(res[65], a[17], a[47]); // Sum(64) + res[65] = fma52lo(res[65], a[18], a[47]); // Sum(65) + res[66] = fma52hi(res[66], a[18], a[47]); // Sum(65) + res[66] = fma52lo(res[66], a[19], a[47]); // Sum(66) + res[67] = fma52hi(res[67], a[19], a[47]); // Sum(66) + res[67] = fma52lo(res[67], a[20], a[47]); // Sum(67) + res[68] = fma52hi(res[68], a[20], a[47]); // Sum(67) + res[68] = fma52lo(res[68], a[21], a[47]); // Sum(68) + res[69] = fma52hi(res[69], a[21], a[47]); // Sum(68) + res[69] = fma52lo(res[69], a[22], a[47]); // Sum(69) + res[70] = fma52hi(res[70], a[22], a[47]); // Sum(69) + res[70] = fma52lo(res[70], a[23], a[47]); // Sum(70) + res[71] = fma52hi(res[71], a[23], a[47]); // Sum(70) + res[71] = fma52lo(res[71], a[24], a[47]); // Sum(71) + res[72] = fma52hi(res[72], a[24], a[47]); // Sum(71) + res[60] = fma52lo(res[60], a[12], a[48]); // Sum(60) + res[61] = fma52hi(res[61], a[12], a[48]); // Sum(60) + res[61] = fma52lo(res[61], a[13], a[48]); // Sum(61) + res[62] = fma52hi(res[62], a[13], a[48]); // Sum(61) + res[62] = fma52lo(res[62], a[14], a[48]); // Sum(62) + res[63] = fma52hi(res[63], a[14], a[48]); // Sum(62) + res[63] = fma52lo(res[63], a[15], a[48]); // Sum(63) + res[64] = fma52hi(res[64], a[15], a[48]); // Sum(63) + res[64] = fma52lo(res[64], a[16], a[48]); // Sum(64) + res[65] = fma52hi(res[65], a[16], a[48]); // Sum(64) + res[65] = fma52lo(res[65], a[17], a[48]); // Sum(65) + res[66] = fma52hi(res[66], a[17], a[48]); // Sum(65) + res[66] = fma52lo(res[66], a[18], a[48]); // Sum(66) + res[67] = fma52hi(res[67], a[18], a[48]); // Sum(66) + res[67] = fma52lo(res[67], a[19], a[48]); // Sum(67) + res[68] = fma52hi(res[68], a[19], a[48]); // Sum(67) + res[68] = fma52lo(res[68], a[20], a[48]); // Sum(68) + res[69] = fma52hi(res[69], a[20], a[48]); // Sum(68) + res[69] = fma52lo(res[69], a[21], a[48]); // Sum(69) + res[70] = fma52hi(res[70], a[21], a[48]); // Sum(69) + res[70] = fma52lo(res[70], a[22], a[48]); // Sum(70) + res[71] = fma52hi(res[71], a[22], a[48]); // Sum(70) + res[71] = fma52lo(res[71], a[23], a[48]); // Sum(71) + res[72] = fma52hi(res[72], a[23], a[48]); // Sum(71) + res[60] = fma52lo(res[60], a[11], a[49]); // Sum(60) + res[61] = fma52hi(res[61], a[11], a[49]); // Sum(60) + res[61] = fma52lo(res[61], a[12], a[49]); // Sum(61) + res[62] = fma52hi(res[62], a[12], a[49]); // Sum(61) + res[62] = fma52lo(res[62], a[13], a[49]); // Sum(62) + res[63] = fma52hi(res[63], a[13], a[49]); // Sum(62) + res[63] = fma52lo(res[63], a[14], a[49]); // Sum(63) + res[64] = fma52hi(res[64], a[14], a[49]); // Sum(63) + res[64] = fma52lo(res[64], a[15], a[49]); // Sum(64) + res[65] = fma52hi(res[65], a[15], a[49]); // Sum(64) + res[65] = fma52lo(res[65], a[16], a[49]); // Sum(65) + res[66] = fma52hi(res[66], a[16], a[49]); // Sum(65) + res[66] = fma52lo(res[66], a[17], a[49]); // Sum(66) + res[67] = fma52hi(res[67], a[17], a[49]); // Sum(66) + res[67] = fma52lo(res[67], a[18], a[49]); // Sum(67) + res[68] = fma52hi(res[68], a[18], a[49]); // Sum(67) + res[68] = fma52lo(res[68], a[19], a[49]); // Sum(68) + res[69] = fma52hi(res[69], a[19], a[49]); // Sum(68) + res[69] = fma52lo(res[69], a[20], a[49]); // Sum(69) + res[70] = fma52hi(res[70], a[20], a[49]); // Sum(69) + res[70] = fma52lo(res[70], a[21], a[49]); // Sum(70) + res[71] = fma52hi(res[71], a[21], a[49]); // Sum(70) + res[71] = fma52lo(res[71], a[22], a[49]); // Sum(71) + res[72] = fma52hi(res[72], a[22], a[49]); // Sum(71) + res[60] = fma52lo(res[60], a[10], a[50]); // Sum(60) + res[61] = fma52hi(res[61], a[10], a[50]); // Sum(60) + res[61] = fma52lo(res[61], a[11], a[50]); // Sum(61) + res[62] = fma52hi(res[62], a[11], a[50]); // Sum(61) + res[62] = fma52lo(res[62], a[12], a[50]); // Sum(62) + res[63] = fma52hi(res[63], a[12], a[50]); // Sum(62) + res[63] = fma52lo(res[63], a[13], a[50]); // Sum(63) + res[64] = fma52hi(res[64], a[13], a[50]); // Sum(63) + res[64] = fma52lo(res[64], a[14], a[50]); // Sum(64) + res[65] = fma52hi(res[65], a[14], a[50]); // Sum(64) + res[65] = fma52lo(res[65], a[15], a[50]); // Sum(65) + res[66] = fma52hi(res[66], a[15], a[50]); // Sum(65) + res[66] = fma52lo(res[66], a[16], a[50]); // Sum(66) + res[67] = fma52hi(res[67], a[16], a[50]); // Sum(66) + res[67] = fma52lo(res[67], a[17], a[50]); // Sum(67) + res[68] = fma52hi(res[68], a[17], a[50]); // Sum(67) + res[68] = fma52lo(res[68], a[18], a[50]); // Sum(68) + res[69] = fma52hi(res[69], a[18], a[50]); // Sum(68) + res[69] = fma52lo(res[69], a[19], a[50]); // Sum(69) + res[70] = fma52hi(res[70], a[19], a[50]); // Sum(69) + res[70] = fma52lo(res[70], a[20], a[50]); // Sum(70) + res[71] = fma52hi(res[71], a[20], a[50]); // Sum(70) + res[71] = fma52lo(res[71], a[21], a[50]); // Sum(71) + res[72] = fma52hi(res[72], a[21], a[50]); // Sum(71) + res[60] = fma52lo(res[60], a[9], a[51]); // Sum(60) + res[61] = fma52hi(res[61], a[9], a[51]); // Sum(60) + res[61] = fma52lo(res[61], a[10], a[51]); // Sum(61) + res[62] = fma52hi(res[62], a[10], a[51]); // Sum(61) + res[62] = fma52lo(res[62], a[11], a[51]); // Sum(62) + res[63] = fma52hi(res[63], a[11], a[51]); // Sum(62) + res[63] = fma52lo(res[63], a[12], a[51]); // Sum(63) + res[64] = fma52hi(res[64], a[12], a[51]); // Sum(63) + res[64] = fma52lo(res[64], a[13], a[51]); // Sum(64) + res[65] = fma52hi(res[65], a[13], a[51]); // Sum(64) + res[65] = fma52lo(res[65], a[14], a[51]); // Sum(65) + res[66] = fma52hi(res[66], a[14], a[51]); // Sum(65) + res[66] = fma52lo(res[66], a[15], a[51]); // Sum(66) + res[67] = fma52hi(res[67], a[15], a[51]); // Sum(66) + res[67] = fma52lo(res[67], a[16], a[51]); // Sum(67) + res[68] = fma52hi(res[68], a[16], a[51]); // Sum(67) + res[68] = fma52lo(res[68], a[17], a[51]); // Sum(68) + res[69] = fma52hi(res[69], a[17], a[51]); // Sum(68) + res[69] = fma52lo(res[69], a[18], a[51]); // Sum(69) + res[70] = fma52hi(res[70], a[18], a[51]); // Sum(69) + res[70] = fma52lo(res[70], a[19], a[51]); // Sum(70) + res[71] = fma52hi(res[71], a[19], a[51]); // Sum(70) + res[71] = fma52lo(res[71], a[20], a[51]); // Sum(71) + res[72] = fma52hi(res[72], a[20], a[51]); // Sum(71) + res[60] = fma52lo(res[60], a[8], a[52]); // Sum(60) + res[61] = fma52hi(res[61], a[8], a[52]); // Sum(60) + res[61] = fma52lo(res[61], a[9], a[52]); // Sum(61) + res[62] = fma52hi(res[62], a[9], a[52]); // Sum(61) + res[62] = fma52lo(res[62], a[10], a[52]); // Sum(62) + res[63] = fma52hi(res[63], a[10], a[52]); // Sum(62) + res[63] = fma52lo(res[63], a[11], a[52]); // Sum(63) + res[64] = fma52hi(res[64], a[11], a[52]); // Sum(63) + res[64] = fma52lo(res[64], a[12], a[52]); // Sum(64) + res[65] = fma52hi(res[65], a[12], a[52]); // Sum(64) + res[65] = fma52lo(res[65], a[13], a[52]); // Sum(65) + res[66] = fma52hi(res[66], a[13], a[52]); // Sum(65) + res[66] = fma52lo(res[66], a[14], a[52]); // Sum(66) + res[67] = fma52hi(res[67], a[14], a[52]); // Sum(66) + res[67] = fma52lo(res[67], a[15], a[52]); // Sum(67) + res[68] = fma52hi(res[68], a[15], a[52]); // Sum(67) + res[68] = fma52lo(res[68], a[16], a[52]); // Sum(68) + res[69] = fma52hi(res[69], a[16], a[52]); // Sum(68) + res[69] = fma52lo(res[69], a[17], a[52]); // Sum(69) + res[70] = fma52hi(res[70], a[17], a[52]); // Sum(69) + res[70] = fma52lo(res[70], a[18], a[52]); // Sum(70) + res[71] = fma52hi(res[71], a[18], a[52]); // Sum(70) + res[71] = fma52lo(res[71], a[19], a[52]); // Sum(71) + res[72] = fma52hi(res[72], a[19], a[52]); // Sum(71) + res[60] = fma52lo(res[60], a[7], a[53]); // Sum(60) + res[61] = fma52hi(res[61], a[7], a[53]); // Sum(60) + res[61] = fma52lo(res[61], a[8], a[53]); // Sum(61) + res[62] = fma52hi(res[62], a[8], a[53]); // Sum(61) + res[62] = fma52lo(res[62], a[9], a[53]); // Sum(62) + res[63] = fma52hi(res[63], a[9], a[53]); // Sum(62) + res[63] = fma52lo(res[63], a[10], a[53]); // Sum(63) + res[64] = fma52hi(res[64], a[10], a[53]); // Sum(63) + res[64] = fma52lo(res[64], a[11], a[53]); // Sum(64) + res[65] = fma52hi(res[65], a[11], a[53]); // Sum(64) + res[65] = fma52lo(res[65], a[12], a[53]); // Sum(65) + res[66] = fma52hi(res[66], a[12], a[53]); // Sum(65) + res[66] = fma52lo(res[66], a[13], a[53]); // Sum(66) + res[67] = fma52hi(res[67], a[13], a[53]); // Sum(66) + res[67] = fma52lo(res[67], a[14], a[53]); // Sum(67) + res[68] = fma52hi(res[68], a[14], a[53]); // Sum(67) + res[68] = fma52lo(res[68], a[15], a[53]); // Sum(68) + res[69] = fma52hi(res[69], a[15], a[53]); // Sum(68) + res[69] = fma52lo(res[69], a[16], a[53]); // Sum(69) + res[70] = fma52hi(res[70], a[16], a[53]); // Sum(69) + res[70] = fma52lo(res[70], a[17], a[53]); // Sum(70) + res[71] = fma52hi(res[71], a[17], a[53]); // Sum(70) + res[71] = fma52lo(res[71], a[18], a[53]); // Sum(71) + res[72] = fma52hi(res[72], a[18], a[53]); // Sum(71) + res[60] = fma52lo(res[60], a[6], a[54]); // Sum(60) + res[61] = fma52hi(res[61], a[6], a[54]); // Sum(60) + res[61] = fma52lo(res[61], a[7], a[54]); // Sum(61) + res[62] = fma52hi(res[62], a[7], a[54]); // Sum(61) + res[62] = fma52lo(res[62], a[8], a[54]); // Sum(62) + res[63] = fma52hi(res[63], a[8], a[54]); // Sum(62) + res[63] = fma52lo(res[63], a[9], a[54]); // Sum(63) + res[64] = fma52hi(res[64], a[9], a[54]); // Sum(63) + res[64] = fma52lo(res[64], a[10], a[54]); // Sum(64) + res[65] = fma52hi(res[65], a[10], a[54]); // Sum(64) + res[65] = fma52lo(res[65], a[11], a[54]); // Sum(65) + res[66] = fma52hi(res[66], a[11], a[54]); // Sum(65) + res[66] = fma52lo(res[66], a[12], a[54]); // Sum(66) + res[67] = fma52hi(res[67], a[12], a[54]); // Sum(66) + res[67] = fma52lo(res[67], a[13], a[54]); // Sum(67) + res[68] = fma52hi(res[68], a[13], a[54]); // Sum(67) + res[68] = fma52lo(res[68], a[14], a[54]); // Sum(68) + res[69] = fma52hi(res[69], a[14], a[54]); // Sum(68) + res[69] = fma52lo(res[69], a[15], a[54]); // Sum(69) + res[70] = fma52hi(res[70], a[15], a[54]); // Sum(69) + res[70] = fma52lo(res[70], a[16], a[54]); // Sum(70) + res[71] = fma52hi(res[71], a[16], a[54]); // Sum(70) + res[71] = fma52lo(res[71], a[17], a[54]); // Sum(71) + res[72] = fma52hi(res[72], a[17], a[54]); // Sum(71) + res[60] = fma52lo(res[60], a[5], a[55]); // Sum(60) + res[61] = fma52hi(res[61], a[5], a[55]); // Sum(60) + res[61] = fma52lo(res[61], a[6], a[55]); // Sum(61) + res[62] = fma52hi(res[62], a[6], a[55]); // Sum(61) + res[62] = fma52lo(res[62], a[7], a[55]); // Sum(62) + res[63] = fma52hi(res[63], a[7], a[55]); // Sum(62) + res[63] = fma52lo(res[63], a[8], a[55]); // Sum(63) + res[64] = fma52hi(res[64], a[8], a[55]); // Sum(63) + res[64] = fma52lo(res[64], a[9], a[55]); // Sum(64) + res[65] = fma52hi(res[65], a[9], a[55]); // Sum(64) + res[65] = fma52lo(res[65], a[10], a[55]); // Sum(65) + res[66] = fma52hi(res[66], a[10], a[55]); // Sum(65) + res[66] = fma52lo(res[66], a[11], a[55]); // Sum(66) + res[67] = fma52hi(res[67], a[11], a[55]); // Sum(66) + res[67] = fma52lo(res[67], a[12], a[55]); // Sum(67) + res[68] = fma52hi(res[68], a[12], a[55]); // Sum(67) + res[68] = fma52lo(res[68], a[13], a[55]); // Sum(68) + res[69] = fma52hi(res[69], a[13], a[55]); // Sum(68) + res[69] = fma52lo(res[69], a[14], a[55]); // Sum(69) + res[70] = fma52hi(res[70], a[14], a[55]); // Sum(69) + res[70] = fma52lo(res[70], a[15], a[55]); // Sum(70) + res[71] = fma52hi(res[71], a[15], a[55]); // Sum(70) + res[71] = fma52lo(res[71], a[16], a[55]); // Sum(71) + res[72] = fma52hi(res[72], a[16], a[55]); // Sum(71) + res[60] = fma52lo(res[60], a[4], a[56]); // Sum(60) + res[61] = fma52hi(res[61], a[4], a[56]); // Sum(60) + res[61] = fma52lo(res[61], a[5], a[56]); // Sum(61) + res[62] = fma52hi(res[62], a[5], a[56]); // Sum(61) + res[62] = fma52lo(res[62], a[6], a[56]); // Sum(62) + res[63] = fma52hi(res[63], a[6], a[56]); // Sum(62) + res[63] = fma52lo(res[63], a[7], a[56]); // Sum(63) + res[64] = fma52hi(res[64], a[7], a[56]); // Sum(63) + res[64] = fma52lo(res[64], a[8], a[56]); // Sum(64) + res[65] = fma52hi(res[65], a[8], a[56]); // Sum(64) + res[65] = fma52lo(res[65], a[9], a[56]); // Sum(65) + res[66] = fma52hi(res[66], a[9], a[56]); // Sum(65) + res[66] = fma52lo(res[66], a[10], a[56]); // Sum(66) + res[67] = fma52hi(res[67], a[10], a[56]); // Sum(66) + res[67] = fma52lo(res[67], a[11], a[56]); // Sum(67) + res[68] = fma52hi(res[68], a[11], a[56]); // Sum(67) + res[68] = fma52lo(res[68], a[12], a[56]); // Sum(68) + res[69] = fma52hi(res[69], a[12], a[56]); // Sum(68) + res[69] = fma52lo(res[69], a[13], a[56]); // Sum(69) + res[70] = fma52hi(res[70], a[13], a[56]); // Sum(69) + res[70] = fma52lo(res[70], a[14], a[56]); // Sum(70) + res[71] = fma52hi(res[71], a[14], a[56]); // Sum(70) + res[71] = fma52lo(res[71], a[15], a[56]); // Sum(71) + res[72] = fma52hi(res[72], a[15], a[56]); // Sum(71) + res[60] = fma52lo(res[60], a[3], a[57]); // Sum(60) + res[61] = fma52hi(res[61], a[3], a[57]); // Sum(60) + res[61] = fma52lo(res[61], a[4], a[57]); // Sum(61) + res[62] = fma52hi(res[62], a[4], a[57]); // Sum(61) + res[62] = fma52lo(res[62], a[5], a[57]); // Sum(62) + res[63] = fma52hi(res[63], a[5], a[57]); // Sum(62) + res[63] = fma52lo(res[63], a[6], a[57]); // Sum(63) + res[64] = fma52hi(res[64], a[6], a[57]); // Sum(63) + res[64] = fma52lo(res[64], a[7], a[57]); // Sum(64) + res[65] = fma52hi(res[65], a[7], a[57]); // Sum(64) + res[65] = fma52lo(res[65], a[8], a[57]); // Sum(65) + res[66] = fma52hi(res[66], a[8], a[57]); // Sum(65) + res[66] = fma52lo(res[66], a[9], a[57]); // Sum(66) + res[67] = fma52hi(res[67], a[9], a[57]); // Sum(66) + res[67] = fma52lo(res[67], a[10], a[57]); // Sum(67) + res[68] = fma52hi(res[68], a[10], a[57]); // Sum(67) + res[68] = fma52lo(res[68], a[11], a[57]); // Sum(68) + res[69] = fma52hi(res[69], a[11], a[57]); // Sum(68) + res[69] = fma52lo(res[69], a[12], a[57]); // Sum(69) + res[70] = fma52hi(res[70], a[12], a[57]); // Sum(69) + res[70] = fma52lo(res[70], a[13], a[57]); // Sum(70) + res[71] = fma52hi(res[71], a[13], a[57]); // Sum(70) + res[71] = fma52lo(res[71], a[14], a[57]); // Sum(71) + res[72] = fma52hi(res[72], a[14], a[57]); // Sum(71) + res[60] = fma52lo(res[60], a[2], a[58]); // Sum(60) + res[61] = fma52hi(res[61], a[2], a[58]); // Sum(60) + res[61] = fma52lo(res[61], a[3], a[58]); // Sum(61) + res[62] = fma52hi(res[62], a[3], a[58]); // Sum(61) + res[62] = fma52lo(res[62], a[4], a[58]); // Sum(62) + res[63] = fma52hi(res[63], a[4], a[58]); // Sum(62) + res[63] = fma52lo(res[63], a[5], a[58]); // Sum(63) + res[64] = fma52hi(res[64], a[5], a[58]); // Sum(63) + res[64] = fma52lo(res[64], a[6], a[58]); // Sum(64) + res[65] = fma52hi(res[65], a[6], a[58]); // Sum(64) + res[65] = fma52lo(res[65], a[7], a[58]); // Sum(65) + res[66] = fma52hi(res[66], a[7], a[58]); // Sum(65) + res[66] = fma52lo(res[66], a[8], a[58]); // Sum(66) + res[67] = fma52hi(res[67], a[8], a[58]); // Sum(66) + res[67] = fma52lo(res[67], a[9], a[58]); // Sum(67) + res[68] = fma52hi(res[68], a[9], a[58]); // Sum(67) + res[68] = fma52lo(res[68], a[10], a[58]); // Sum(68) + res[69] = fma52hi(res[69], a[10], a[58]); // Sum(68) + res[69] = fma52lo(res[69], a[11], a[58]); // Sum(69) + res[70] = fma52hi(res[70], a[11], a[58]); // Sum(69) + res[70] = fma52lo(res[70], a[12], a[58]); // Sum(70) + res[71] = fma52hi(res[71], a[12], a[58]); // Sum(70) + res[71] = fma52lo(res[71], a[13], a[58]); // Sum(71) + res[72] = fma52hi(res[72], a[13], a[58]); // Sum(71) + res[60] = fma52lo(res[60], a[1], a[59]); // Sum(60) + res[61] = fma52hi(res[61], a[1], a[59]); // Sum(60) + res[61] = fma52lo(res[61], a[2], a[59]); // Sum(61) + res[62] = fma52hi(res[62], a[2], a[59]); // Sum(61) + res[62] = fma52lo(res[62], a[3], a[59]); // Sum(62) + res[63] = fma52hi(res[63], a[3], a[59]); // Sum(62) + res[63] = fma52lo(res[63], a[4], a[59]); // Sum(63) + res[64] = fma52hi(res[64], a[4], a[59]); // Sum(63) + res[64] = fma52lo(res[64], a[5], a[59]); // Sum(64) + res[65] = fma52hi(res[65], a[5], a[59]); // Sum(64) + res[65] = fma52lo(res[65], a[6], a[59]); // Sum(65) + res[66] = fma52hi(res[66], a[6], a[59]); // Sum(65) + res[66] = fma52lo(res[66], a[7], a[59]); // Sum(66) + res[67] = fma52hi(res[67], a[7], a[59]); // Sum(66) + res[67] = fma52lo(res[67], a[8], a[59]); // Sum(67) + res[68] = fma52hi(res[68], a[8], a[59]); // Sum(67) + res[68] = fma52lo(res[68], a[9], a[59]); // Sum(68) + res[69] = fma52hi(res[69], a[9], a[59]); // Sum(68) + res[69] = fma52lo(res[69], a[10], a[59]); // Sum(69) + res[70] = fma52hi(res[70], a[10], a[59]); // Sum(69) + res[70] = fma52lo(res[70], a[11], a[59]); // Sum(70) + res[71] = fma52hi(res[71], a[11], a[59]); // Sum(70) + res[71] = fma52lo(res[71], a[12], a[59]); // Sum(71) + res[72] = fma52hi(res[72], a[12], a[59]); // Sum(71) + res[60] = add64(res[60], res[60]); // Double(60) + res[61] = add64(res[61], res[61]); // Double(61) + res[62] = add64(res[62], res[62]); // Double(62) + res[63] = add64(res[63], res[63]); // Double(63) + res[64] = add64(res[64], res[64]); // Double(64) + res[65] = add64(res[65], res[65]); // Double(65) + res[66] = add64(res[66], res[66]); // Double(66) + res[67] = add64(res[67], res[67]); // Double(67) + res[68] = add64(res[68], res[68]); // Double(68) + res[69] = add64(res[69], res[69]); // Double(69) + res[70] = add64(res[70], res[70]); // Double(70) + res[71] = add64(res[71], res[71]); // Double(71) + res[60] = fma52lo(res[60], a[30], a[30]); // Add sqr(60) + res[61] = fma52hi(res[61], a[30], a[30]); // Add sqr(60) + res[62] = fma52lo(res[62], a[31], a[31]); // Add sqr(62) + res[63] = fma52hi(res[63], a[31], a[31]); // Add sqr(62) + res[64] = fma52lo(res[64], a[32], a[32]); // Add sqr(64) + res[65] = fma52hi(res[65], a[32], a[32]); // Add sqr(64) + res[66] = fma52lo(res[66], a[33], a[33]); // Add sqr(66) + res[67] = fma52hi(res[67], a[33], a[33]); // Add sqr(66) + res[68] = fma52lo(res[68], a[34], a[34]); // Add sqr(68) + res[69] = fma52hi(res[69], a[34], a[34]); // Add sqr(68) + res[70] = fma52lo(res[70], a[35], a[35]); // Add sqr(70) + res[71] = fma52hi(res[71], a[35], a[35]); // Add sqr(70) + res[72] = fma52lo(res[72], a[35], a[37]); // Sum(72) + res[73] = fma52hi(res[73], a[35], a[37]); // Sum(72) + res[73] = fma52lo(res[73], a[36], a[37]); // Sum(73) + res[74] = fma52hi(res[74], a[36], a[37]); // Sum(73) + res[72] = fma52lo(res[72], a[34], a[38]); // Sum(72) + res[73] = fma52hi(res[73], a[34], a[38]); // Sum(72) + res[73] = fma52lo(res[73], a[35], a[38]); // Sum(73) + res[74] = fma52hi(res[74], a[35], a[38]); // Sum(73) + res[74] = fma52lo(res[74], a[36], a[38]); // Sum(74) + res[75] = fma52hi(res[75], a[36], a[38]); // Sum(74) + res[75] = fma52lo(res[75], a[37], a[38]); // Sum(75) + res[76] = fma52hi(res[76], a[37], a[38]); // Sum(75) + res[72] = fma52lo(res[72], a[33], a[39]); // Sum(72) + res[73] = fma52hi(res[73], a[33], a[39]); // Sum(72) + res[73] = fma52lo(res[73], a[34], a[39]); // Sum(73) + res[74] = fma52hi(res[74], a[34], a[39]); // Sum(73) + res[74] = fma52lo(res[74], a[35], a[39]); // Sum(74) + res[75] = fma52hi(res[75], a[35], a[39]); // Sum(74) + res[75] = fma52lo(res[75], a[36], a[39]); // Sum(75) + res[76] = fma52hi(res[76], a[36], a[39]); // Sum(75) + res[76] = fma52lo(res[76], a[37], a[39]); // Sum(76) + res[77] = fma52hi(res[77], a[37], a[39]); // Sum(76) + res[77] = fma52lo(res[77], a[38], a[39]); // Sum(77) + res[78] = fma52hi(res[78], a[38], a[39]); // Sum(77) + res[72] = fma52lo(res[72], a[32], a[40]); // Sum(72) + res[73] = fma52hi(res[73], a[32], a[40]); // Sum(72) + res[73] = fma52lo(res[73], a[33], a[40]); // Sum(73) + res[74] = fma52hi(res[74], a[33], a[40]); // Sum(73) + res[74] = fma52lo(res[74], a[34], a[40]); // Sum(74) + res[75] = fma52hi(res[75], a[34], a[40]); // Sum(74) + res[75] = fma52lo(res[75], a[35], a[40]); // Sum(75) + res[76] = fma52hi(res[76], a[35], a[40]); // Sum(75) + res[76] = fma52lo(res[76], a[36], a[40]); // Sum(76) + res[77] = fma52hi(res[77], a[36], a[40]); // Sum(76) + res[77] = fma52lo(res[77], a[37], a[40]); // Sum(77) + res[78] = fma52hi(res[78], a[37], a[40]); // Sum(77) + res[78] = fma52lo(res[78], a[38], a[40]); // Sum(78) + res[79] = fma52hi(res[79], a[38], a[40]); // Sum(78) + res[79] = fma52lo(res[79], a[39], a[40]); // Sum(79) + res[80] = fma52hi(res[80], a[39], a[40]); // Sum(79) + res[72] = fma52lo(res[72], a[31], a[41]); // Sum(72) + res[73] = fma52hi(res[73], a[31], a[41]); // Sum(72) + res[73] = fma52lo(res[73], a[32], a[41]); // Sum(73) + res[74] = fma52hi(res[74], a[32], a[41]); // Sum(73) + res[74] = fma52lo(res[74], a[33], a[41]); // Sum(74) + res[75] = fma52hi(res[75], a[33], a[41]); // Sum(74) + res[75] = fma52lo(res[75], a[34], a[41]); // Sum(75) + res[76] = fma52hi(res[76], a[34], a[41]); // Sum(75) + res[76] = fma52lo(res[76], a[35], a[41]); // Sum(76) + res[77] = fma52hi(res[77], a[35], a[41]); // Sum(76) + res[77] = fma52lo(res[77], a[36], a[41]); // Sum(77) + res[78] = fma52hi(res[78], a[36], a[41]); // Sum(77) + res[78] = fma52lo(res[78], a[37], a[41]); // Sum(78) + res[79] = fma52hi(res[79], a[37], a[41]); // Sum(78) + res[79] = fma52lo(res[79], a[38], a[41]); // Sum(79) + res[80] = fma52hi(res[80], a[38], a[41]); // Sum(79) + res[80] = fma52lo(res[80], a[39], a[41]); // Sum(80) + res[81] = fma52hi(res[81], a[39], a[41]); // Sum(80) + res[81] = fma52lo(res[81], a[40], a[41]); // Sum(81) + res[82] = fma52hi(res[82], a[40], a[41]); // Sum(81) + res[72] = fma52lo(res[72], a[30], a[42]); // Sum(72) + res[73] = fma52hi(res[73], a[30], a[42]); // Sum(72) + res[73] = fma52lo(res[73], a[31], a[42]); // Sum(73) + res[74] = fma52hi(res[74], a[31], a[42]); // Sum(73) + res[74] = fma52lo(res[74], a[32], a[42]); // Sum(74) + res[75] = fma52hi(res[75], a[32], a[42]); // Sum(74) + res[75] = fma52lo(res[75], a[33], a[42]); // Sum(75) + res[76] = fma52hi(res[76], a[33], a[42]); // Sum(75) + res[76] = fma52lo(res[76], a[34], a[42]); // Sum(76) + res[77] = fma52hi(res[77], a[34], a[42]); // Sum(76) + res[77] = fma52lo(res[77], a[35], a[42]); // Sum(77) + res[78] = fma52hi(res[78], a[35], a[42]); // Sum(77) + res[78] = fma52lo(res[78], a[36], a[42]); // Sum(78) + res[79] = fma52hi(res[79], a[36], a[42]); // Sum(78) + res[79] = fma52lo(res[79], a[37], a[42]); // Sum(79) + res[80] = fma52hi(res[80], a[37], a[42]); // Sum(79) + res[80] = fma52lo(res[80], a[38], a[42]); // Sum(80) + res[81] = fma52hi(res[81], a[38], a[42]); // Sum(80) + res[81] = fma52lo(res[81], a[39], a[42]); // Sum(81) + res[82] = fma52hi(res[82], a[39], a[42]); // Sum(81) + res[82] = fma52lo(res[82], a[40], a[42]); // Sum(82) + res[83] = fma52hi(res[83], a[40], a[42]); // Sum(82) + res[83] = fma52lo(res[83], a[41], a[42]); // Sum(83) + res[84] = fma52hi(res[84], a[41], a[42]); // Sum(83) + res[72] = fma52lo(res[72], a[29], a[43]); // Sum(72) + res[73] = fma52hi(res[73], a[29], a[43]); // Sum(72) + res[73] = fma52lo(res[73], a[30], a[43]); // Sum(73) + res[74] = fma52hi(res[74], a[30], a[43]); // Sum(73) + res[74] = fma52lo(res[74], a[31], a[43]); // Sum(74) + res[75] = fma52hi(res[75], a[31], a[43]); // Sum(74) + res[75] = fma52lo(res[75], a[32], a[43]); // Sum(75) + res[76] = fma52hi(res[76], a[32], a[43]); // Sum(75) + res[76] = fma52lo(res[76], a[33], a[43]); // Sum(76) + res[77] = fma52hi(res[77], a[33], a[43]); // Sum(76) + res[77] = fma52lo(res[77], a[34], a[43]); // Sum(77) + res[78] = fma52hi(res[78], a[34], a[43]); // Sum(77) + res[78] = fma52lo(res[78], a[35], a[43]); // Sum(78) + res[79] = fma52hi(res[79], a[35], a[43]); // Sum(78) + res[79] = fma52lo(res[79], a[36], a[43]); // Sum(79) + res[80] = fma52hi(res[80], a[36], a[43]); // Sum(79) + res[80] = fma52lo(res[80], a[37], a[43]); // Sum(80) + res[81] = fma52hi(res[81], a[37], a[43]); // Sum(80) + res[81] = fma52lo(res[81], a[38], a[43]); // Sum(81) + res[82] = fma52hi(res[82], a[38], a[43]); // Sum(81) + res[82] = fma52lo(res[82], a[39], a[43]); // Sum(82) + res[83] = fma52hi(res[83], a[39], a[43]); // Sum(82) + res[83] = fma52lo(res[83], a[40], a[43]); // Sum(83) + res[84] = fma52hi(res[84], a[40], a[43]); // Sum(83) + res[72] = fma52lo(res[72], a[28], a[44]); // Sum(72) + res[73] = fma52hi(res[73], a[28], a[44]); // Sum(72) + res[73] = fma52lo(res[73], a[29], a[44]); // Sum(73) + res[74] = fma52hi(res[74], a[29], a[44]); // Sum(73) + res[74] = fma52lo(res[74], a[30], a[44]); // Sum(74) + res[75] = fma52hi(res[75], a[30], a[44]); // Sum(74) + res[75] = fma52lo(res[75], a[31], a[44]); // Sum(75) + res[76] = fma52hi(res[76], a[31], a[44]); // Sum(75) + res[76] = fma52lo(res[76], a[32], a[44]); // Sum(76) + res[77] = fma52hi(res[77], a[32], a[44]); // Sum(76) + res[77] = fma52lo(res[77], a[33], a[44]); // Sum(77) + res[78] = fma52hi(res[78], a[33], a[44]); // Sum(77) + res[78] = fma52lo(res[78], a[34], a[44]); // Sum(78) + res[79] = fma52hi(res[79], a[34], a[44]); // Sum(78) + res[79] = fma52lo(res[79], a[35], a[44]); // Sum(79) + res[80] = fma52hi(res[80], a[35], a[44]); // Sum(79) + res[80] = fma52lo(res[80], a[36], a[44]); // Sum(80) + res[81] = fma52hi(res[81], a[36], a[44]); // Sum(80) + res[81] = fma52lo(res[81], a[37], a[44]); // Sum(81) + res[82] = fma52hi(res[82], a[37], a[44]); // Sum(81) + res[82] = fma52lo(res[82], a[38], a[44]); // Sum(82) + res[83] = fma52hi(res[83], a[38], a[44]); // Sum(82) + res[83] = fma52lo(res[83], a[39], a[44]); // Sum(83) + res[84] = fma52hi(res[84], a[39], a[44]); // Sum(83) + res[72] = fma52lo(res[72], a[27], a[45]); // Sum(72) + res[73] = fma52hi(res[73], a[27], a[45]); // Sum(72) + res[73] = fma52lo(res[73], a[28], a[45]); // Sum(73) + res[74] = fma52hi(res[74], a[28], a[45]); // Sum(73) + res[74] = fma52lo(res[74], a[29], a[45]); // Sum(74) + res[75] = fma52hi(res[75], a[29], a[45]); // Sum(74) + res[75] = fma52lo(res[75], a[30], a[45]); // Sum(75) + res[76] = fma52hi(res[76], a[30], a[45]); // Sum(75) + res[76] = fma52lo(res[76], a[31], a[45]); // Sum(76) + res[77] = fma52hi(res[77], a[31], a[45]); // Sum(76) + res[77] = fma52lo(res[77], a[32], a[45]); // Sum(77) + res[78] = fma52hi(res[78], a[32], a[45]); // Sum(77) + res[78] = fma52lo(res[78], a[33], a[45]); // Sum(78) + res[79] = fma52hi(res[79], a[33], a[45]); // Sum(78) + res[79] = fma52lo(res[79], a[34], a[45]); // Sum(79) + res[80] = fma52hi(res[80], a[34], a[45]); // Sum(79) + res[80] = fma52lo(res[80], a[35], a[45]); // Sum(80) + res[81] = fma52hi(res[81], a[35], a[45]); // Sum(80) + res[81] = fma52lo(res[81], a[36], a[45]); // Sum(81) + res[82] = fma52hi(res[82], a[36], a[45]); // Sum(81) + res[82] = fma52lo(res[82], a[37], a[45]); // Sum(82) + res[83] = fma52hi(res[83], a[37], a[45]); // Sum(82) + res[83] = fma52lo(res[83], a[38], a[45]); // Sum(83) + res[84] = fma52hi(res[84], a[38], a[45]); // Sum(83) + res[72] = fma52lo(res[72], a[26], a[46]); // Sum(72) + res[73] = fma52hi(res[73], a[26], a[46]); // Sum(72) + res[73] = fma52lo(res[73], a[27], a[46]); // Sum(73) + res[74] = fma52hi(res[74], a[27], a[46]); // Sum(73) + res[74] = fma52lo(res[74], a[28], a[46]); // Sum(74) + res[75] = fma52hi(res[75], a[28], a[46]); // Sum(74) + res[75] = fma52lo(res[75], a[29], a[46]); // Sum(75) + res[76] = fma52hi(res[76], a[29], a[46]); // Sum(75) + res[76] = fma52lo(res[76], a[30], a[46]); // Sum(76) + res[77] = fma52hi(res[77], a[30], a[46]); // Sum(76) + res[77] = fma52lo(res[77], a[31], a[46]); // Sum(77) + res[78] = fma52hi(res[78], a[31], a[46]); // Sum(77) + res[78] = fma52lo(res[78], a[32], a[46]); // Sum(78) + res[79] = fma52hi(res[79], a[32], a[46]); // Sum(78) + res[79] = fma52lo(res[79], a[33], a[46]); // Sum(79) + res[80] = fma52hi(res[80], a[33], a[46]); // Sum(79) + res[80] = fma52lo(res[80], a[34], a[46]); // Sum(80) + res[81] = fma52hi(res[81], a[34], a[46]); // Sum(80) + res[81] = fma52lo(res[81], a[35], a[46]); // Sum(81) + res[82] = fma52hi(res[82], a[35], a[46]); // Sum(81) + res[82] = fma52lo(res[82], a[36], a[46]); // Sum(82) + res[83] = fma52hi(res[83], a[36], a[46]); // Sum(82) + res[83] = fma52lo(res[83], a[37], a[46]); // Sum(83) + res[84] = fma52hi(res[84], a[37], a[46]); // Sum(83) + res[72] = fma52lo(res[72], a[25], a[47]); // Sum(72) + res[73] = fma52hi(res[73], a[25], a[47]); // Sum(72) + res[73] = fma52lo(res[73], a[26], a[47]); // Sum(73) + res[74] = fma52hi(res[74], a[26], a[47]); // Sum(73) + res[74] = fma52lo(res[74], a[27], a[47]); // Sum(74) + res[75] = fma52hi(res[75], a[27], a[47]); // Sum(74) + res[75] = fma52lo(res[75], a[28], a[47]); // Sum(75) + res[76] = fma52hi(res[76], a[28], a[47]); // Sum(75) + res[76] = fma52lo(res[76], a[29], a[47]); // Sum(76) + res[77] = fma52hi(res[77], a[29], a[47]); // Sum(76) + res[77] = fma52lo(res[77], a[30], a[47]); // Sum(77) + res[78] = fma52hi(res[78], a[30], a[47]); // Sum(77) + res[78] = fma52lo(res[78], a[31], a[47]); // Sum(78) + res[79] = fma52hi(res[79], a[31], a[47]); // Sum(78) + res[79] = fma52lo(res[79], a[32], a[47]); // Sum(79) + res[80] = fma52hi(res[80], a[32], a[47]); // Sum(79) + res[80] = fma52lo(res[80], a[33], a[47]); // Sum(80) + res[81] = fma52hi(res[81], a[33], a[47]); // Sum(80) + res[81] = fma52lo(res[81], a[34], a[47]); // Sum(81) + res[82] = fma52hi(res[82], a[34], a[47]); // Sum(81) + res[82] = fma52lo(res[82], a[35], a[47]); // Sum(82) + res[83] = fma52hi(res[83], a[35], a[47]); // Sum(82) + res[83] = fma52lo(res[83], a[36], a[47]); // Sum(83) + res[84] = fma52hi(res[84], a[36], a[47]); // Sum(83) + res[72] = fma52lo(res[72], a[24], a[48]); // Sum(72) + res[73] = fma52hi(res[73], a[24], a[48]); // Sum(72) + res[73] = fma52lo(res[73], a[25], a[48]); // Sum(73) + res[74] = fma52hi(res[74], a[25], a[48]); // Sum(73) + res[74] = fma52lo(res[74], a[26], a[48]); // Sum(74) + res[75] = fma52hi(res[75], a[26], a[48]); // Sum(74) + res[75] = fma52lo(res[75], a[27], a[48]); // Sum(75) + res[76] = fma52hi(res[76], a[27], a[48]); // Sum(75) + res[76] = fma52lo(res[76], a[28], a[48]); // Sum(76) + res[77] = fma52hi(res[77], a[28], a[48]); // Sum(76) + res[77] = fma52lo(res[77], a[29], a[48]); // Sum(77) + res[78] = fma52hi(res[78], a[29], a[48]); // Sum(77) + res[78] = fma52lo(res[78], a[30], a[48]); // Sum(78) + res[79] = fma52hi(res[79], a[30], a[48]); // Sum(78) + res[79] = fma52lo(res[79], a[31], a[48]); // Sum(79) + res[80] = fma52hi(res[80], a[31], a[48]); // Sum(79) + res[80] = fma52lo(res[80], a[32], a[48]); // Sum(80) + res[81] = fma52hi(res[81], a[32], a[48]); // Sum(80) + res[81] = fma52lo(res[81], a[33], a[48]); // Sum(81) + res[82] = fma52hi(res[82], a[33], a[48]); // Sum(81) + res[82] = fma52lo(res[82], a[34], a[48]); // Sum(82) + res[83] = fma52hi(res[83], a[34], a[48]); // Sum(82) + res[83] = fma52lo(res[83], a[35], a[48]); // Sum(83) + res[84] = fma52hi(res[84], a[35], a[48]); // Sum(83) + res[72] = fma52lo(res[72], a[23], a[49]); // Sum(72) + res[73] = fma52hi(res[73], a[23], a[49]); // Sum(72) + res[73] = fma52lo(res[73], a[24], a[49]); // Sum(73) + res[74] = fma52hi(res[74], a[24], a[49]); // Sum(73) + res[74] = fma52lo(res[74], a[25], a[49]); // Sum(74) + res[75] = fma52hi(res[75], a[25], a[49]); // Sum(74) + res[75] = fma52lo(res[75], a[26], a[49]); // Sum(75) + res[76] = fma52hi(res[76], a[26], a[49]); // Sum(75) + res[76] = fma52lo(res[76], a[27], a[49]); // Sum(76) + res[77] = fma52hi(res[77], a[27], a[49]); // Sum(76) + res[77] = fma52lo(res[77], a[28], a[49]); // Sum(77) + res[78] = fma52hi(res[78], a[28], a[49]); // Sum(77) + res[78] = fma52lo(res[78], a[29], a[49]); // Sum(78) + res[79] = fma52hi(res[79], a[29], a[49]); // Sum(78) + res[79] = fma52lo(res[79], a[30], a[49]); // Sum(79) + res[80] = fma52hi(res[80], a[30], a[49]); // Sum(79) + res[80] = fma52lo(res[80], a[31], a[49]); // Sum(80) + res[81] = fma52hi(res[81], a[31], a[49]); // Sum(80) + res[81] = fma52lo(res[81], a[32], a[49]); // Sum(81) + res[82] = fma52hi(res[82], a[32], a[49]); // Sum(81) + res[82] = fma52lo(res[82], a[33], a[49]); // Sum(82) + res[83] = fma52hi(res[83], a[33], a[49]); // Sum(82) + res[83] = fma52lo(res[83], a[34], a[49]); // Sum(83) + res[84] = fma52hi(res[84], a[34], a[49]); // Sum(83) + res[72] = fma52lo(res[72], a[22], a[50]); // Sum(72) + res[73] = fma52hi(res[73], a[22], a[50]); // Sum(72) + res[73] = fma52lo(res[73], a[23], a[50]); // Sum(73) + res[74] = fma52hi(res[74], a[23], a[50]); // Sum(73) + res[74] = fma52lo(res[74], a[24], a[50]); // Sum(74) + res[75] = fma52hi(res[75], a[24], a[50]); // Sum(74) + res[75] = fma52lo(res[75], a[25], a[50]); // Sum(75) + res[76] = fma52hi(res[76], a[25], a[50]); // Sum(75) + res[76] = fma52lo(res[76], a[26], a[50]); // Sum(76) + res[77] = fma52hi(res[77], a[26], a[50]); // Sum(76) + res[77] = fma52lo(res[77], a[27], a[50]); // Sum(77) + res[78] = fma52hi(res[78], a[27], a[50]); // Sum(77) + res[78] = fma52lo(res[78], a[28], a[50]); // Sum(78) + res[79] = fma52hi(res[79], a[28], a[50]); // Sum(78) + res[79] = fma52lo(res[79], a[29], a[50]); // Sum(79) + res[80] = fma52hi(res[80], a[29], a[50]); // Sum(79) + res[80] = fma52lo(res[80], a[30], a[50]); // Sum(80) + res[81] = fma52hi(res[81], a[30], a[50]); // Sum(80) + res[81] = fma52lo(res[81], a[31], a[50]); // Sum(81) + res[82] = fma52hi(res[82], a[31], a[50]); // Sum(81) + res[82] = fma52lo(res[82], a[32], a[50]); // Sum(82) + res[83] = fma52hi(res[83], a[32], a[50]); // Sum(82) + res[83] = fma52lo(res[83], a[33], a[50]); // Sum(83) + res[84] = fma52hi(res[84], a[33], a[50]); // Sum(83) + res[72] = fma52lo(res[72], a[21], a[51]); // Sum(72) + res[73] = fma52hi(res[73], a[21], a[51]); // Sum(72) + res[73] = fma52lo(res[73], a[22], a[51]); // Sum(73) + res[74] = fma52hi(res[74], a[22], a[51]); // Sum(73) + res[74] = fma52lo(res[74], a[23], a[51]); // Sum(74) + res[75] = fma52hi(res[75], a[23], a[51]); // Sum(74) + res[75] = fma52lo(res[75], a[24], a[51]); // Sum(75) + res[76] = fma52hi(res[76], a[24], a[51]); // Sum(75) + res[76] = fma52lo(res[76], a[25], a[51]); // Sum(76) + res[77] = fma52hi(res[77], a[25], a[51]); // Sum(76) + res[77] = fma52lo(res[77], a[26], a[51]); // Sum(77) + res[78] = fma52hi(res[78], a[26], a[51]); // Sum(77) + res[78] = fma52lo(res[78], a[27], a[51]); // Sum(78) + res[79] = fma52hi(res[79], a[27], a[51]); // Sum(78) + res[79] = fma52lo(res[79], a[28], a[51]); // Sum(79) + res[80] = fma52hi(res[80], a[28], a[51]); // Sum(79) + res[80] = fma52lo(res[80], a[29], a[51]); // Sum(80) + res[81] = fma52hi(res[81], a[29], a[51]); // Sum(80) + res[81] = fma52lo(res[81], a[30], a[51]); // Sum(81) + res[82] = fma52hi(res[82], a[30], a[51]); // Sum(81) + res[82] = fma52lo(res[82], a[31], a[51]); // Sum(82) + res[83] = fma52hi(res[83], a[31], a[51]); // Sum(82) + res[83] = fma52lo(res[83], a[32], a[51]); // Sum(83) + res[84] = fma52hi(res[84], a[32], a[51]); // Sum(83) + res[72] = fma52lo(res[72], a[20], a[52]); // Sum(72) + res[73] = fma52hi(res[73], a[20], a[52]); // Sum(72) + res[73] = fma52lo(res[73], a[21], a[52]); // Sum(73) + res[74] = fma52hi(res[74], a[21], a[52]); // Sum(73) + res[74] = fma52lo(res[74], a[22], a[52]); // Sum(74) + res[75] = fma52hi(res[75], a[22], a[52]); // Sum(74) + res[75] = fma52lo(res[75], a[23], a[52]); // Sum(75) + res[76] = fma52hi(res[76], a[23], a[52]); // Sum(75) + res[76] = fma52lo(res[76], a[24], a[52]); // Sum(76) + res[77] = fma52hi(res[77], a[24], a[52]); // Sum(76) + res[77] = fma52lo(res[77], a[25], a[52]); // Sum(77) + res[78] = fma52hi(res[78], a[25], a[52]); // Sum(77) + res[78] = fma52lo(res[78], a[26], a[52]); // Sum(78) + res[79] = fma52hi(res[79], a[26], a[52]); // Sum(78) + res[79] = fma52lo(res[79], a[27], a[52]); // Sum(79) + res[80] = fma52hi(res[80], a[27], a[52]); // Sum(79) + res[80] = fma52lo(res[80], a[28], a[52]); // Sum(80) + res[81] = fma52hi(res[81], a[28], a[52]); // Sum(80) + res[81] = fma52lo(res[81], a[29], a[52]); // Sum(81) + res[82] = fma52hi(res[82], a[29], a[52]); // Sum(81) + res[82] = fma52lo(res[82], a[30], a[52]); // Sum(82) + res[83] = fma52hi(res[83], a[30], a[52]); // Sum(82) + res[83] = fma52lo(res[83], a[31], a[52]); // Sum(83) + res[84] = fma52hi(res[84], a[31], a[52]); // Sum(83) + res[72] = fma52lo(res[72], a[19], a[53]); // Sum(72) + res[73] = fma52hi(res[73], a[19], a[53]); // Sum(72) + res[73] = fma52lo(res[73], a[20], a[53]); // Sum(73) + res[74] = fma52hi(res[74], a[20], a[53]); // Sum(73) + res[74] = fma52lo(res[74], a[21], a[53]); // Sum(74) + res[75] = fma52hi(res[75], a[21], a[53]); // Sum(74) + res[75] = fma52lo(res[75], a[22], a[53]); // Sum(75) + res[76] = fma52hi(res[76], a[22], a[53]); // Sum(75) + res[76] = fma52lo(res[76], a[23], a[53]); // Sum(76) + res[77] = fma52hi(res[77], a[23], a[53]); // Sum(76) + res[77] = fma52lo(res[77], a[24], a[53]); // Sum(77) + res[78] = fma52hi(res[78], a[24], a[53]); // Sum(77) + res[78] = fma52lo(res[78], a[25], a[53]); // Sum(78) + res[79] = fma52hi(res[79], a[25], a[53]); // Sum(78) + res[79] = fma52lo(res[79], a[26], a[53]); // Sum(79) + res[80] = fma52hi(res[80], a[26], a[53]); // Sum(79) + res[80] = fma52lo(res[80], a[27], a[53]); // Sum(80) + res[81] = fma52hi(res[81], a[27], a[53]); // Sum(80) + res[81] = fma52lo(res[81], a[28], a[53]); // Sum(81) + res[82] = fma52hi(res[82], a[28], a[53]); // Sum(81) + res[82] = fma52lo(res[82], a[29], a[53]); // Sum(82) + res[83] = fma52hi(res[83], a[29], a[53]); // Sum(82) + res[83] = fma52lo(res[83], a[30], a[53]); // Sum(83) + res[84] = fma52hi(res[84], a[30], a[53]); // Sum(83) + res[72] = fma52lo(res[72], a[18], a[54]); // Sum(72) + res[73] = fma52hi(res[73], a[18], a[54]); // Sum(72) + res[73] = fma52lo(res[73], a[19], a[54]); // Sum(73) + res[74] = fma52hi(res[74], a[19], a[54]); // Sum(73) + res[74] = fma52lo(res[74], a[20], a[54]); // Sum(74) + res[75] = fma52hi(res[75], a[20], a[54]); // Sum(74) + res[75] = fma52lo(res[75], a[21], a[54]); // Sum(75) + res[76] = fma52hi(res[76], a[21], a[54]); // Sum(75) + res[76] = fma52lo(res[76], a[22], a[54]); // Sum(76) + res[77] = fma52hi(res[77], a[22], a[54]); // Sum(76) + res[77] = fma52lo(res[77], a[23], a[54]); // Sum(77) + res[78] = fma52hi(res[78], a[23], a[54]); // Sum(77) + res[78] = fma52lo(res[78], a[24], a[54]); // Sum(78) + res[79] = fma52hi(res[79], a[24], a[54]); // Sum(78) + res[79] = fma52lo(res[79], a[25], a[54]); // Sum(79) + res[80] = fma52hi(res[80], a[25], a[54]); // Sum(79) + res[80] = fma52lo(res[80], a[26], a[54]); // Sum(80) + res[81] = fma52hi(res[81], a[26], a[54]); // Sum(80) + res[81] = fma52lo(res[81], a[27], a[54]); // Sum(81) + res[82] = fma52hi(res[82], a[27], a[54]); // Sum(81) + res[82] = fma52lo(res[82], a[28], a[54]); // Sum(82) + res[83] = fma52hi(res[83], a[28], a[54]); // Sum(82) + res[83] = fma52lo(res[83], a[29], a[54]); // Sum(83) + res[84] = fma52hi(res[84], a[29], a[54]); // Sum(83) + res[72] = fma52lo(res[72], a[17], a[55]); // Sum(72) + res[73] = fma52hi(res[73], a[17], a[55]); // Sum(72) + res[73] = fma52lo(res[73], a[18], a[55]); // Sum(73) + res[74] = fma52hi(res[74], a[18], a[55]); // Sum(73) + res[74] = fma52lo(res[74], a[19], a[55]); // Sum(74) + res[75] = fma52hi(res[75], a[19], a[55]); // Sum(74) + res[75] = fma52lo(res[75], a[20], a[55]); // Sum(75) + res[76] = fma52hi(res[76], a[20], a[55]); // Sum(75) + res[76] = fma52lo(res[76], a[21], a[55]); // Sum(76) + res[77] = fma52hi(res[77], a[21], a[55]); // Sum(76) + res[77] = fma52lo(res[77], a[22], a[55]); // Sum(77) + res[78] = fma52hi(res[78], a[22], a[55]); // Sum(77) + res[78] = fma52lo(res[78], a[23], a[55]); // Sum(78) + res[79] = fma52hi(res[79], a[23], a[55]); // Sum(78) + res[79] = fma52lo(res[79], a[24], a[55]); // Sum(79) + res[80] = fma52hi(res[80], a[24], a[55]); // Sum(79) + res[80] = fma52lo(res[80], a[25], a[55]); // Sum(80) + res[81] = fma52hi(res[81], a[25], a[55]); // Sum(80) + res[81] = fma52lo(res[81], a[26], a[55]); // Sum(81) + res[82] = fma52hi(res[82], a[26], a[55]); // Sum(81) + res[82] = fma52lo(res[82], a[27], a[55]); // Sum(82) + res[83] = fma52hi(res[83], a[27], a[55]); // Sum(82) + res[83] = fma52lo(res[83], a[28], a[55]); // Sum(83) + res[84] = fma52hi(res[84], a[28], a[55]); // Sum(83) + res[72] = fma52lo(res[72], a[16], a[56]); // Sum(72) + res[73] = fma52hi(res[73], a[16], a[56]); // Sum(72) + res[73] = fma52lo(res[73], a[17], a[56]); // Sum(73) + res[74] = fma52hi(res[74], a[17], a[56]); // Sum(73) + res[74] = fma52lo(res[74], a[18], a[56]); // Sum(74) + res[75] = fma52hi(res[75], a[18], a[56]); // Sum(74) + res[75] = fma52lo(res[75], a[19], a[56]); // Sum(75) + res[76] = fma52hi(res[76], a[19], a[56]); // Sum(75) + res[76] = fma52lo(res[76], a[20], a[56]); // Sum(76) + res[77] = fma52hi(res[77], a[20], a[56]); // Sum(76) + res[77] = fma52lo(res[77], a[21], a[56]); // Sum(77) + res[78] = fma52hi(res[78], a[21], a[56]); // Sum(77) + res[78] = fma52lo(res[78], a[22], a[56]); // Sum(78) + res[79] = fma52hi(res[79], a[22], a[56]); // Sum(78) + res[79] = fma52lo(res[79], a[23], a[56]); // Sum(79) + res[80] = fma52hi(res[80], a[23], a[56]); // Sum(79) + res[80] = fma52lo(res[80], a[24], a[56]); // Sum(80) + res[81] = fma52hi(res[81], a[24], a[56]); // Sum(80) + res[81] = fma52lo(res[81], a[25], a[56]); // Sum(81) + res[82] = fma52hi(res[82], a[25], a[56]); // Sum(81) + res[82] = fma52lo(res[82], a[26], a[56]); // Sum(82) + res[83] = fma52hi(res[83], a[26], a[56]); // Sum(82) + res[83] = fma52lo(res[83], a[27], a[56]); // Sum(83) + res[84] = fma52hi(res[84], a[27], a[56]); // Sum(83) + res[72] = fma52lo(res[72], a[15], a[57]); // Sum(72) + res[73] = fma52hi(res[73], a[15], a[57]); // Sum(72) + res[73] = fma52lo(res[73], a[16], a[57]); // Sum(73) + res[74] = fma52hi(res[74], a[16], a[57]); // Sum(73) + res[74] = fma52lo(res[74], a[17], a[57]); // Sum(74) + res[75] = fma52hi(res[75], a[17], a[57]); // Sum(74) + res[75] = fma52lo(res[75], a[18], a[57]); // Sum(75) + res[76] = fma52hi(res[76], a[18], a[57]); // Sum(75) + res[76] = fma52lo(res[76], a[19], a[57]); // Sum(76) + res[77] = fma52hi(res[77], a[19], a[57]); // Sum(76) + res[77] = fma52lo(res[77], a[20], a[57]); // Sum(77) + res[78] = fma52hi(res[78], a[20], a[57]); // Sum(77) + res[78] = fma52lo(res[78], a[21], a[57]); // Sum(78) + res[79] = fma52hi(res[79], a[21], a[57]); // Sum(78) + res[79] = fma52lo(res[79], a[22], a[57]); // Sum(79) + res[80] = fma52hi(res[80], a[22], a[57]); // Sum(79) + res[80] = fma52lo(res[80], a[23], a[57]); // Sum(80) + res[81] = fma52hi(res[81], a[23], a[57]); // Sum(80) + res[81] = fma52lo(res[81], a[24], a[57]); // Sum(81) + res[82] = fma52hi(res[82], a[24], a[57]); // Sum(81) + res[82] = fma52lo(res[82], a[25], a[57]); // Sum(82) + res[83] = fma52hi(res[83], a[25], a[57]); // Sum(82) + res[83] = fma52lo(res[83], a[26], a[57]); // Sum(83) + res[84] = fma52hi(res[84], a[26], a[57]); // Sum(83) + res[72] = fma52lo(res[72], a[14], a[58]); // Sum(72) + res[73] = fma52hi(res[73], a[14], a[58]); // Sum(72) + res[73] = fma52lo(res[73], a[15], a[58]); // Sum(73) + res[74] = fma52hi(res[74], a[15], a[58]); // Sum(73) + res[74] = fma52lo(res[74], a[16], a[58]); // Sum(74) + res[75] = fma52hi(res[75], a[16], a[58]); // Sum(74) + res[75] = fma52lo(res[75], a[17], a[58]); // Sum(75) + res[76] = fma52hi(res[76], a[17], a[58]); // Sum(75) + res[76] = fma52lo(res[76], a[18], a[58]); // Sum(76) + res[77] = fma52hi(res[77], a[18], a[58]); // Sum(76) + res[77] = fma52lo(res[77], a[19], a[58]); // Sum(77) + res[78] = fma52hi(res[78], a[19], a[58]); // Sum(77) + res[78] = fma52lo(res[78], a[20], a[58]); // Sum(78) + res[79] = fma52hi(res[79], a[20], a[58]); // Sum(78) + res[79] = fma52lo(res[79], a[21], a[58]); // Sum(79) + res[80] = fma52hi(res[80], a[21], a[58]); // Sum(79) + res[80] = fma52lo(res[80], a[22], a[58]); // Sum(80) + res[81] = fma52hi(res[81], a[22], a[58]); // Sum(80) + res[81] = fma52lo(res[81], a[23], a[58]); // Sum(81) + res[82] = fma52hi(res[82], a[23], a[58]); // Sum(81) + res[82] = fma52lo(res[82], a[24], a[58]); // Sum(82) + res[83] = fma52hi(res[83], a[24], a[58]); // Sum(82) + res[83] = fma52lo(res[83], a[25], a[58]); // Sum(83) + res[84] = fma52hi(res[84], a[25], a[58]); // Sum(83) + res[72] = fma52lo(res[72], a[13], a[59]); // Sum(72) + res[73] = fma52hi(res[73], a[13], a[59]); // Sum(72) + res[73] = fma52lo(res[73], a[14], a[59]); // Sum(73) + res[74] = fma52hi(res[74], a[14], a[59]); // Sum(73) + res[74] = fma52lo(res[74], a[15], a[59]); // Sum(74) + res[75] = fma52hi(res[75], a[15], a[59]); // Sum(74) + res[75] = fma52lo(res[75], a[16], a[59]); // Sum(75) + res[76] = fma52hi(res[76], a[16], a[59]); // Sum(75) + res[76] = fma52lo(res[76], a[17], a[59]); // Sum(76) + res[77] = fma52hi(res[77], a[17], a[59]); // Sum(76) + res[77] = fma52lo(res[77], a[18], a[59]); // Sum(77) + res[78] = fma52hi(res[78], a[18], a[59]); // Sum(77) + res[78] = fma52lo(res[78], a[19], a[59]); // Sum(78) + res[79] = fma52hi(res[79], a[19], a[59]); // Sum(78) + res[79] = fma52lo(res[79], a[20], a[59]); // Sum(79) + res[80] = fma52hi(res[80], a[20], a[59]); // Sum(79) + res[80] = fma52lo(res[80], a[21], a[59]); // Sum(80) + res[81] = fma52hi(res[81], a[21], a[59]); // Sum(80) + res[81] = fma52lo(res[81], a[22], a[59]); // Sum(81) + res[82] = fma52hi(res[82], a[22], a[59]); // Sum(81) + res[82] = fma52lo(res[82], a[23], a[59]); // Sum(82) + res[83] = fma52hi(res[83], a[23], a[59]); // Sum(82) + res[83] = fma52lo(res[83], a[24], a[59]); // Sum(83) + res[84] = fma52hi(res[84], a[24], a[59]); // Sum(83) + res[72] = add64(res[72], res[72]); // Double(72) + res[73] = add64(res[73], res[73]); // Double(73) + res[74] = add64(res[74], res[74]); // Double(74) + res[75] = add64(res[75], res[75]); // Double(75) + res[76] = add64(res[76], res[76]); // Double(76) + res[77] = add64(res[77], res[77]); // Double(77) + res[78] = add64(res[78], res[78]); // Double(78) + res[79] = add64(res[79], res[79]); // Double(79) + res[80] = add64(res[80], res[80]); // Double(80) + res[81] = add64(res[81], res[81]); // Double(81) + res[82] = add64(res[82], res[82]); // Double(82) + res[83] = add64(res[83], res[83]); // Double(83) + res[72] = fma52lo(res[72], a[36], a[36]); // Add sqr(72) + res[73] = fma52hi(res[73], a[36], a[36]); // Add sqr(72) + res[74] = fma52lo(res[74], a[37], a[37]); // Add sqr(74) + res[75] = fma52hi(res[75], a[37], a[37]); // Add sqr(74) + res[76] = fma52lo(res[76], a[38], a[38]); // Add sqr(76) + res[77] = fma52hi(res[77], a[38], a[38]); // Add sqr(76) + res[78] = fma52lo(res[78], a[39], a[39]); // Add sqr(78) + res[79] = fma52hi(res[79], a[39], a[39]); // Add sqr(78) + res[80] = fma52lo(res[80], a[40], a[40]); // Add sqr(80) + res[81] = fma52hi(res[81], a[40], a[40]); // Add sqr(80) + res[82] = fma52lo(res[82], a[41], a[41]); // Add sqr(82) + res[83] = fma52hi(res[83], a[41], a[41]); // Add sqr(82) + res[84] = fma52lo(res[84], a[41], a[43]); // Sum(84) + res[85] = fma52hi(res[85], a[41], a[43]); // Sum(84) + res[85] = fma52lo(res[85], a[42], a[43]); // Sum(85) + res[86] = fma52hi(res[86], a[42], a[43]); // Sum(85) + res[84] = fma52lo(res[84], a[40], a[44]); // Sum(84) + res[85] = fma52hi(res[85], a[40], a[44]); // Sum(84) + res[85] = fma52lo(res[85], a[41], a[44]); // Sum(85) + res[86] = fma52hi(res[86], a[41], a[44]); // Sum(85) + res[86] = fma52lo(res[86], a[42], a[44]); // Sum(86) + res[87] = fma52hi(res[87], a[42], a[44]); // Sum(86) + res[87] = fma52lo(res[87], a[43], a[44]); // Sum(87) + res[88] = fma52hi(res[88], a[43], a[44]); // Sum(87) + res[84] = fma52lo(res[84], a[39], a[45]); // Sum(84) + res[85] = fma52hi(res[85], a[39], a[45]); // Sum(84) + res[85] = fma52lo(res[85], a[40], a[45]); // Sum(85) + res[86] = fma52hi(res[86], a[40], a[45]); // Sum(85) + res[86] = fma52lo(res[86], a[41], a[45]); // Sum(86) + res[87] = fma52hi(res[87], a[41], a[45]); // Sum(86) + res[87] = fma52lo(res[87], a[42], a[45]); // Sum(87) + res[88] = fma52hi(res[88], a[42], a[45]); // Sum(87) + res[88] = fma52lo(res[88], a[43], a[45]); // Sum(88) + res[89] = fma52hi(res[89], a[43], a[45]); // Sum(88) + res[89] = fma52lo(res[89], a[44], a[45]); // Sum(89) + res[90] = fma52hi(res[90], a[44], a[45]); // Sum(89) + res[84] = fma52lo(res[84], a[38], a[46]); // Sum(84) + res[85] = fma52hi(res[85], a[38], a[46]); // Sum(84) + res[85] = fma52lo(res[85], a[39], a[46]); // Sum(85) + res[86] = fma52hi(res[86], a[39], a[46]); // Sum(85) + res[86] = fma52lo(res[86], a[40], a[46]); // Sum(86) + res[87] = fma52hi(res[87], a[40], a[46]); // Sum(86) + res[87] = fma52lo(res[87], a[41], a[46]); // Sum(87) + res[88] = fma52hi(res[88], a[41], a[46]); // Sum(87) + res[88] = fma52lo(res[88], a[42], a[46]); // Sum(88) + res[89] = fma52hi(res[89], a[42], a[46]); // Sum(88) + res[89] = fma52lo(res[89], a[43], a[46]); // Sum(89) + res[90] = fma52hi(res[90], a[43], a[46]); // Sum(89) + res[90] = fma52lo(res[90], a[44], a[46]); // Sum(90) + res[91] = fma52hi(res[91], a[44], a[46]); // Sum(90) + res[91] = fma52lo(res[91], a[45], a[46]); // Sum(91) + res[92] = fma52hi(res[92], a[45], a[46]); // Sum(91) + res[84] = fma52lo(res[84], a[37], a[47]); // Sum(84) + res[85] = fma52hi(res[85], a[37], a[47]); // Sum(84) + res[85] = fma52lo(res[85], a[38], a[47]); // Sum(85) + res[86] = fma52hi(res[86], a[38], a[47]); // Sum(85) + res[86] = fma52lo(res[86], a[39], a[47]); // Sum(86) + res[87] = fma52hi(res[87], a[39], a[47]); // Sum(86) + res[87] = fma52lo(res[87], a[40], a[47]); // Sum(87) + res[88] = fma52hi(res[88], a[40], a[47]); // Sum(87) + res[88] = fma52lo(res[88], a[41], a[47]); // Sum(88) + res[89] = fma52hi(res[89], a[41], a[47]); // Sum(88) + res[89] = fma52lo(res[89], a[42], a[47]); // Sum(89) + res[90] = fma52hi(res[90], a[42], a[47]); // Sum(89) + res[90] = fma52lo(res[90], a[43], a[47]); // Sum(90) + res[91] = fma52hi(res[91], a[43], a[47]); // Sum(90) + res[91] = fma52lo(res[91], a[44], a[47]); // Sum(91) + res[92] = fma52hi(res[92], a[44], a[47]); // Sum(91) + res[92] = fma52lo(res[92], a[45], a[47]); // Sum(92) + res[93] = fma52hi(res[93], a[45], a[47]); // Sum(92) + res[93] = fma52lo(res[93], a[46], a[47]); // Sum(93) + res[94] = fma52hi(res[94], a[46], a[47]); // Sum(93) + res[84] = fma52lo(res[84], a[36], a[48]); // Sum(84) + res[85] = fma52hi(res[85], a[36], a[48]); // Sum(84) + res[85] = fma52lo(res[85], a[37], a[48]); // Sum(85) + res[86] = fma52hi(res[86], a[37], a[48]); // Sum(85) + res[86] = fma52lo(res[86], a[38], a[48]); // Sum(86) + res[87] = fma52hi(res[87], a[38], a[48]); // Sum(86) + res[87] = fma52lo(res[87], a[39], a[48]); // Sum(87) + res[88] = fma52hi(res[88], a[39], a[48]); // Sum(87) + res[88] = fma52lo(res[88], a[40], a[48]); // Sum(88) + res[89] = fma52hi(res[89], a[40], a[48]); // Sum(88) + res[89] = fma52lo(res[89], a[41], a[48]); // Sum(89) + res[90] = fma52hi(res[90], a[41], a[48]); // Sum(89) + res[90] = fma52lo(res[90], a[42], a[48]); // Sum(90) + res[91] = fma52hi(res[91], a[42], a[48]); // Sum(90) + res[91] = fma52lo(res[91], a[43], a[48]); // Sum(91) + res[92] = fma52hi(res[92], a[43], a[48]); // Sum(91) + res[92] = fma52lo(res[92], a[44], a[48]); // Sum(92) + res[93] = fma52hi(res[93], a[44], a[48]); // Sum(92) + res[93] = fma52lo(res[93], a[45], a[48]); // Sum(93) + res[94] = fma52hi(res[94], a[45], a[48]); // Sum(93) + res[94] = fma52lo(res[94], a[46], a[48]); // Sum(94) + res[95] = fma52hi(res[95], a[46], a[48]); // Sum(94) + res[95] = fma52lo(res[95], a[47], a[48]); // Sum(95) + res[96] = fma52hi(res[96], a[47], a[48]); // Sum(95) + res[84] = fma52lo(res[84], a[35], a[49]); // Sum(84) + res[85] = fma52hi(res[85], a[35], a[49]); // Sum(84) + res[85] = fma52lo(res[85], a[36], a[49]); // Sum(85) + res[86] = fma52hi(res[86], a[36], a[49]); // Sum(85) + res[86] = fma52lo(res[86], a[37], a[49]); // Sum(86) + res[87] = fma52hi(res[87], a[37], a[49]); // Sum(86) + res[87] = fma52lo(res[87], a[38], a[49]); // Sum(87) + res[88] = fma52hi(res[88], a[38], a[49]); // Sum(87) + res[88] = fma52lo(res[88], a[39], a[49]); // Sum(88) + res[89] = fma52hi(res[89], a[39], a[49]); // Sum(88) + res[89] = fma52lo(res[89], a[40], a[49]); // Sum(89) + res[90] = fma52hi(res[90], a[40], a[49]); // Sum(89) + res[90] = fma52lo(res[90], a[41], a[49]); // Sum(90) + res[91] = fma52hi(res[91], a[41], a[49]); // Sum(90) + res[91] = fma52lo(res[91], a[42], a[49]); // Sum(91) + res[92] = fma52hi(res[92], a[42], a[49]); // Sum(91) + res[92] = fma52lo(res[92], a[43], a[49]); // Sum(92) + res[93] = fma52hi(res[93], a[43], a[49]); // Sum(92) + res[93] = fma52lo(res[93], a[44], a[49]); // Sum(93) + res[94] = fma52hi(res[94], a[44], a[49]); // Sum(93) + res[94] = fma52lo(res[94], a[45], a[49]); // Sum(94) + res[95] = fma52hi(res[95], a[45], a[49]); // Sum(94) + res[95] = fma52lo(res[95], a[46], a[49]); // Sum(95) + res[96] = fma52hi(res[96], a[46], a[49]); // Sum(95) + res[84] = fma52lo(res[84], a[34], a[50]); // Sum(84) + res[85] = fma52hi(res[85], a[34], a[50]); // Sum(84) + res[85] = fma52lo(res[85], a[35], a[50]); // Sum(85) + res[86] = fma52hi(res[86], a[35], a[50]); // Sum(85) + res[86] = fma52lo(res[86], a[36], a[50]); // Sum(86) + res[87] = fma52hi(res[87], a[36], a[50]); // Sum(86) + res[87] = fma52lo(res[87], a[37], a[50]); // Sum(87) + res[88] = fma52hi(res[88], a[37], a[50]); // Sum(87) + res[88] = fma52lo(res[88], a[38], a[50]); // Sum(88) + res[89] = fma52hi(res[89], a[38], a[50]); // Sum(88) + res[89] = fma52lo(res[89], a[39], a[50]); // Sum(89) + res[90] = fma52hi(res[90], a[39], a[50]); // Sum(89) + res[90] = fma52lo(res[90], a[40], a[50]); // Sum(90) + res[91] = fma52hi(res[91], a[40], a[50]); // Sum(90) + res[91] = fma52lo(res[91], a[41], a[50]); // Sum(91) + res[92] = fma52hi(res[92], a[41], a[50]); // Sum(91) + res[92] = fma52lo(res[92], a[42], a[50]); // Sum(92) + res[93] = fma52hi(res[93], a[42], a[50]); // Sum(92) + res[93] = fma52lo(res[93], a[43], a[50]); // Sum(93) + res[94] = fma52hi(res[94], a[43], a[50]); // Sum(93) + res[94] = fma52lo(res[94], a[44], a[50]); // Sum(94) + res[95] = fma52hi(res[95], a[44], a[50]); // Sum(94) + res[95] = fma52lo(res[95], a[45], a[50]); // Sum(95) + res[96] = fma52hi(res[96], a[45], a[50]); // Sum(95) + res[84] = fma52lo(res[84], a[33], a[51]); // Sum(84) + res[85] = fma52hi(res[85], a[33], a[51]); // Sum(84) + res[85] = fma52lo(res[85], a[34], a[51]); // Sum(85) + res[86] = fma52hi(res[86], a[34], a[51]); // Sum(85) + res[86] = fma52lo(res[86], a[35], a[51]); // Sum(86) + res[87] = fma52hi(res[87], a[35], a[51]); // Sum(86) + res[87] = fma52lo(res[87], a[36], a[51]); // Sum(87) + res[88] = fma52hi(res[88], a[36], a[51]); // Sum(87) + res[88] = fma52lo(res[88], a[37], a[51]); // Sum(88) + res[89] = fma52hi(res[89], a[37], a[51]); // Sum(88) + res[89] = fma52lo(res[89], a[38], a[51]); // Sum(89) + res[90] = fma52hi(res[90], a[38], a[51]); // Sum(89) + res[90] = fma52lo(res[90], a[39], a[51]); // Sum(90) + res[91] = fma52hi(res[91], a[39], a[51]); // Sum(90) + res[91] = fma52lo(res[91], a[40], a[51]); // Sum(91) + res[92] = fma52hi(res[92], a[40], a[51]); // Sum(91) + res[92] = fma52lo(res[92], a[41], a[51]); // Sum(92) + res[93] = fma52hi(res[93], a[41], a[51]); // Sum(92) + res[93] = fma52lo(res[93], a[42], a[51]); // Sum(93) + res[94] = fma52hi(res[94], a[42], a[51]); // Sum(93) + res[94] = fma52lo(res[94], a[43], a[51]); // Sum(94) + res[95] = fma52hi(res[95], a[43], a[51]); // Sum(94) + res[95] = fma52lo(res[95], a[44], a[51]); // Sum(95) + res[96] = fma52hi(res[96], a[44], a[51]); // Sum(95) + res[84] = fma52lo(res[84], a[32], a[52]); // Sum(84) + res[85] = fma52hi(res[85], a[32], a[52]); // Sum(84) + res[85] = fma52lo(res[85], a[33], a[52]); // Sum(85) + res[86] = fma52hi(res[86], a[33], a[52]); // Sum(85) + res[86] = fma52lo(res[86], a[34], a[52]); // Sum(86) + res[87] = fma52hi(res[87], a[34], a[52]); // Sum(86) + res[87] = fma52lo(res[87], a[35], a[52]); // Sum(87) + res[88] = fma52hi(res[88], a[35], a[52]); // Sum(87) + res[88] = fma52lo(res[88], a[36], a[52]); // Sum(88) + res[89] = fma52hi(res[89], a[36], a[52]); // Sum(88) + res[89] = fma52lo(res[89], a[37], a[52]); // Sum(89) + res[90] = fma52hi(res[90], a[37], a[52]); // Sum(89) + res[90] = fma52lo(res[90], a[38], a[52]); // Sum(90) + res[91] = fma52hi(res[91], a[38], a[52]); // Sum(90) + res[91] = fma52lo(res[91], a[39], a[52]); // Sum(91) + res[92] = fma52hi(res[92], a[39], a[52]); // Sum(91) + res[92] = fma52lo(res[92], a[40], a[52]); // Sum(92) + res[93] = fma52hi(res[93], a[40], a[52]); // Sum(92) + res[93] = fma52lo(res[93], a[41], a[52]); // Sum(93) + res[94] = fma52hi(res[94], a[41], a[52]); // Sum(93) + res[94] = fma52lo(res[94], a[42], a[52]); // Sum(94) + res[95] = fma52hi(res[95], a[42], a[52]); // Sum(94) + res[95] = fma52lo(res[95], a[43], a[52]); // Sum(95) + res[96] = fma52hi(res[96], a[43], a[52]); // Sum(95) + res[84] = fma52lo(res[84], a[31], a[53]); // Sum(84) + res[85] = fma52hi(res[85], a[31], a[53]); // Sum(84) + res[85] = fma52lo(res[85], a[32], a[53]); // Sum(85) + res[86] = fma52hi(res[86], a[32], a[53]); // Sum(85) + res[86] = fma52lo(res[86], a[33], a[53]); // Sum(86) + res[87] = fma52hi(res[87], a[33], a[53]); // Sum(86) + res[87] = fma52lo(res[87], a[34], a[53]); // Sum(87) + res[88] = fma52hi(res[88], a[34], a[53]); // Sum(87) + res[88] = fma52lo(res[88], a[35], a[53]); // Sum(88) + res[89] = fma52hi(res[89], a[35], a[53]); // Sum(88) + res[89] = fma52lo(res[89], a[36], a[53]); // Sum(89) + res[90] = fma52hi(res[90], a[36], a[53]); // Sum(89) + res[90] = fma52lo(res[90], a[37], a[53]); // Sum(90) + res[91] = fma52hi(res[91], a[37], a[53]); // Sum(90) + res[91] = fma52lo(res[91], a[38], a[53]); // Sum(91) + res[92] = fma52hi(res[92], a[38], a[53]); // Sum(91) + res[92] = fma52lo(res[92], a[39], a[53]); // Sum(92) + res[93] = fma52hi(res[93], a[39], a[53]); // Sum(92) + res[93] = fma52lo(res[93], a[40], a[53]); // Sum(93) + res[94] = fma52hi(res[94], a[40], a[53]); // Sum(93) + res[94] = fma52lo(res[94], a[41], a[53]); // Sum(94) + res[95] = fma52hi(res[95], a[41], a[53]); // Sum(94) + res[95] = fma52lo(res[95], a[42], a[53]); // Sum(95) + res[96] = fma52hi(res[96], a[42], a[53]); // Sum(95) + res[84] = fma52lo(res[84], a[30], a[54]); // Sum(84) + res[85] = fma52hi(res[85], a[30], a[54]); // Sum(84) + res[85] = fma52lo(res[85], a[31], a[54]); // Sum(85) + res[86] = fma52hi(res[86], a[31], a[54]); // Sum(85) + res[86] = fma52lo(res[86], a[32], a[54]); // Sum(86) + res[87] = fma52hi(res[87], a[32], a[54]); // Sum(86) + res[87] = fma52lo(res[87], a[33], a[54]); // Sum(87) + res[88] = fma52hi(res[88], a[33], a[54]); // Sum(87) + res[88] = fma52lo(res[88], a[34], a[54]); // Sum(88) + res[89] = fma52hi(res[89], a[34], a[54]); // Sum(88) + res[89] = fma52lo(res[89], a[35], a[54]); // Sum(89) + res[90] = fma52hi(res[90], a[35], a[54]); // Sum(89) + res[90] = fma52lo(res[90], a[36], a[54]); // Sum(90) + res[91] = fma52hi(res[91], a[36], a[54]); // Sum(90) + res[91] = fma52lo(res[91], a[37], a[54]); // Sum(91) + res[92] = fma52hi(res[92], a[37], a[54]); // Sum(91) + res[92] = fma52lo(res[92], a[38], a[54]); // Sum(92) + res[93] = fma52hi(res[93], a[38], a[54]); // Sum(92) + res[93] = fma52lo(res[93], a[39], a[54]); // Sum(93) + res[94] = fma52hi(res[94], a[39], a[54]); // Sum(93) + res[94] = fma52lo(res[94], a[40], a[54]); // Sum(94) + res[95] = fma52hi(res[95], a[40], a[54]); // Sum(94) + res[95] = fma52lo(res[95], a[41], a[54]); // Sum(95) + res[96] = fma52hi(res[96], a[41], a[54]); // Sum(95) + res[84] = fma52lo(res[84], a[29], a[55]); // Sum(84) + res[85] = fma52hi(res[85], a[29], a[55]); // Sum(84) + res[85] = fma52lo(res[85], a[30], a[55]); // Sum(85) + res[86] = fma52hi(res[86], a[30], a[55]); // Sum(85) + res[86] = fma52lo(res[86], a[31], a[55]); // Sum(86) + res[87] = fma52hi(res[87], a[31], a[55]); // Sum(86) + res[87] = fma52lo(res[87], a[32], a[55]); // Sum(87) + res[88] = fma52hi(res[88], a[32], a[55]); // Sum(87) + res[88] = fma52lo(res[88], a[33], a[55]); // Sum(88) + res[89] = fma52hi(res[89], a[33], a[55]); // Sum(88) + res[89] = fma52lo(res[89], a[34], a[55]); // Sum(89) + res[90] = fma52hi(res[90], a[34], a[55]); // Sum(89) + res[90] = fma52lo(res[90], a[35], a[55]); // Sum(90) + res[91] = fma52hi(res[91], a[35], a[55]); // Sum(90) + res[91] = fma52lo(res[91], a[36], a[55]); // Sum(91) + res[92] = fma52hi(res[92], a[36], a[55]); // Sum(91) + res[92] = fma52lo(res[92], a[37], a[55]); // Sum(92) + res[93] = fma52hi(res[93], a[37], a[55]); // Sum(92) + res[93] = fma52lo(res[93], a[38], a[55]); // Sum(93) + res[94] = fma52hi(res[94], a[38], a[55]); // Sum(93) + res[94] = fma52lo(res[94], a[39], a[55]); // Sum(94) + res[95] = fma52hi(res[95], a[39], a[55]); // Sum(94) + res[95] = fma52lo(res[95], a[40], a[55]); // Sum(95) + res[96] = fma52hi(res[96], a[40], a[55]); // Sum(95) + res[84] = fma52lo(res[84], a[28], a[56]); // Sum(84) + res[85] = fma52hi(res[85], a[28], a[56]); // Sum(84) + res[85] = fma52lo(res[85], a[29], a[56]); // Sum(85) + res[86] = fma52hi(res[86], a[29], a[56]); // Sum(85) + res[86] = fma52lo(res[86], a[30], a[56]); // Sum(86) + res[87] = fma52hi(res[87], a[30], a[56]); // Sum(86) + res[87] = fma52lo(res[87], a[31], a[56]); // Sum(87) + res[88] = fma52hi(res[88], a[31], a[56]); // Sum(87) + res[88] = fma52lo(res[88], a[32], a[56]); // Sum(88) + res[89] = fma52hi(res[89], a[32], a[56]); // Sum(88) + res[89] = fma52lo(res[89], a[33], a[56]); // Sum(89) + res[90] = fma52hi(res[90], a[33], a[56]); // Sum(89) + res[90] = fma52lo(res[90], a[34], a[56]); // Sum(90) + res[91] = fma52hi(res[91], a[34], a[56]); // Sum(90) + res[91] = fma52lo(res[91], a[35], a[56]); // Sum(91) + res[92] = fma52hi(res[92], a[35], a[56]); // Sum(91) + res[92] = fma52lo(res[92], a[36], a[56]); // Sum(92) + res[93] = fma52hi(res[93], a[36], a[56]); // Sum(92) + res[93] = fma52lo(res[93], a[37], a[56]); // Sum(93) + res[94] = fma52hi(res[94], a[37], a[56]); // Sum(93) + res[94] = fma52lo(res[94], a[38], a[56]); // Sum(94) + res[95] = fma52hi(res[95], a[38], a[56]); // Sum(94) + res[95] = fma52lo(res[95], a[39], a[56]); // Sum(95) + res[96] = fma52hi(res[96], a[39], a[56]); // Sum(95) + res[84] = fma52lo(res[84], a[27], a[57]); // Sum(84) + res[85] = fma52hi(res[85], a[27], a[57]); // Sum(84) + res[85] = fma52lo(res[85], a[28], a[57]); // Sum(85) + res[86] = fma52hi(res[86], a[28], a[57]); // Sum(85) + res[86] = fma52lo(res[86], a[29], a[57]); // Sum(86) + res[87] = fma52hi(res[87], a[29], a[57]); // Sum(86) + res[87] = fma52lo(res[87], a[30], a[57]); // Sum(87) + res[88] = fma52hi(res[88], a[30], a[57]); // Sum(87) + res[88] = fma52lo(res[88], a[31], a[57]); // Sum(88) + res[89] = fma52hi(res[89], a[31], a[57]); // Sum(88) + res[89] = fma52lo(res[89], a[32], a[57]); // Sum(89) + res[90] = fma52hi(res[90], a[32], a[57]); // Sum(89) + res[90] = fma52lo(res[90], a[33], a[57]); // Sum(90) + res[91] = fma52hi(res[91], a[33], a[57]); // Sum(90) + res[91] = fma52lo(res[91], a[34], a[57]); // Sum(91) + res[92] = fma52hi(res[92], a[34], a[57]); // Sum(91) + res[92] = fma52lo(res[92], a[35], a[57]); // Sum(92) + res[93] = fma52hi(res[93], a[35], a[57]); // Sum(92) + res[93] = fma52lo(res[93], a[36], a[57]); // Sum(93) + res[94] = fma52hi(res[94], a[36], a[57]); // Sum(93) + res[94] = fma52lo(res[94], a[37], a[57]); // Sum(94) + res[95] = fma52hi(res[95], a[37], a[57]); // Sum(94) + res[95] = fma52lo(res[95], a[38], a[57]); // Sum(95) + res[96] = fma52hi(res[96], a[38], a[57]); // Sum(95) + res[84] = fma52lo(res[84], a[26], a[58]); // Sum(84) + res[85] = fma52hi(res[85], a[26], a[58]); // Sum(84) + res[85] = fma52lo(res[85], a[27], a[58]); // Sum(85) + res[86] = fma52hi(res[86], a[27], a[58]); // Sum(85) + res[86] = fma52lo(res[86], a[28], a[58]); // Sum(86) + res[87] = fma52hi(res[87], a[28], a[58]); // Sum(86) + res[87] = fma52lo(res[87], a[29], a[58]); // Sum(87) + res[88] = fma52hi(res[88], a[29], a[58]); // Sum(87) + res[88] = fma52lo(res[88], a[30], a[58]); // Sum(88) + res[89] = fma52hi(res[89], a[30], a[58]); // Sum(88) + res[89] = fma52lo(res[89], a[31], a[58]); // Sum(89) + res[90] = fma52hi(res[90], a[31], a[58]); // Sum(89) + res[90] = fma52lo(res[90], a[32], a[58]); // Sum(90) + res[91] = fma52hi(res[91], a[32], a[58]); // Sum(90) + res[91] = fma52lo(res[91], a[33], a[58]); // Sum(91) + res[92] = fma52hi(res[92], a[33], a[58]); // Sum(91) + res[92] = fma52lo(res[92], a[34], a[58]); // Sum(92) + res[93] = fma52hi(res[93], a[34], a[58]); // Sum(92) + res[93] = fma52lo(res[93], a[35], a[58]); // Sum(93) + res[94] = fma52hi(res[94], a[35], a[58]); // Sum(93) + res[94] = fma52lo(res[94], a[36], a[58]); // Sum(94) + res[95] = fma52hi(res[95], a[36], a[58]); // Sum(94) + res[95] = fma52lo(res[95], a[37], a[58]); // Sum(95) + res[96] = fma52hi(res[96], a[37], a[58]); // Sum(95) + res[84] = fma52lo(res[84], a[25], a[59]); // Sum(84) + res[85] = fma52hi(res[85], a[25], a[59]); // Sum(84) + res[85] = fma52lo(res[85], a[26], a[59]); // Sum(85) + res[86] = fma52hi(res[86], a[26], a[59]); // Sum(85) + res[86] = fma52lo(res[86], a[27], a[59]); // Sum(86) + res[87] = fma52hi(res[87], a[27], a[59]); // Sum(86) + res[87] = fma52lo(res[87], a[28], a[59]); // Sum(87) + res[88] = fma52hi(res[88], a[28], a[59]); // Sum(87) + res[88] = fma52lo(res[88], a[29], a[59]); // Sum(88) + res[89] = fma52hi(res[89], a[29], a[59]); // Sum(88) + res[89] = fma52lo(res[89], a[30], a[59]); // Sum(89) + res[90] = fma52hi(res[90], a[30], a[59]); // Sum(89) + res[90] = fma52lo(res[90], a[31], a[59]); // Sum(90) + res[91] = fma52hi(res[91], a[31], a[59]); // Sum(90) + res[91] = fma52lo(res[91], a[32], a[59]); // Sum(91) + res[92] = fma52hi(res[92], a[32], a[59]); // Sum(91) + res[92] = fma52lo(res[92], a[33], a[59]); // Sum(92) + res[93] = fma52hi(res[93], a[33], a[59]); // Sum(92) + res[93] = fma52lo(res[93], a[34], a[59]); // Sum(93) + res[94] = fma52hi(res[94], a[34], a[59]); // Sum(93) + res[94] = fma52lo(res[94], a[35], a[59]); // Sum(94) + res[95] = fma52hi(res[95], a[35], a[59]); // Sum(94) + res[95] = fma52lo(res[95], a[36], a[59]); // Sum(95) + res[96] = fma52hi(res[96], a[36], a[59]); // Sum(95) + res[84] = add64(res[84], res[84]); // Double(84) + res[85] = add64(res[85], res[85]); // Double(85) + res[86] = add64(res[86], res[86]); // Double(86) + res[87] = add64(res[87], res[87]); // Double(87) + res[88] = add64(res[88], res[88]); // Double(88) + res[89] = add64(res[89], res[89]); // Double(89) + res[90] = add64(res[90], res[90]); // Double(90) + res[91] = add64(res[91], res[91]); // Double(91) + res[92] = add64(res[92], res[92]); // Double(92) + res[93] = add64(res[93], res[93]); // Double(93) + res[94] = add64(res[94], res[94]); // Double(94) + res[95] = add64(res[95], res[95]); // Double(95) + res[84] = fma52lo(res[84], a[42], a[42]); // Add sqr(84) + res[85] = fma52hi(res[85], a[42], a[42]); // Add sqr(84) + res[86] = fma52lo(res[86], a[43], a[43]); // Add sqr(86) + res[87] = fma52hi(res[87], a[43], a[43]); // Add sqr(86) + res[88] = fma52lo(res[88], a[44], a[44]); // Add sqr(88) + res[89] = fma52hi(res[89], a[44], a[44]); // Add sqr(88) + res[90] = fma52lo(res[90], a[45], a[45]); // Add sqr(90) + res[91] = fma52hi(res[91], a[45], a[45]); // Add sqr(90) + res[92] = fma52lo(res[92], a[46], a[46]); // Add sqr(92) + res[93] = fma52hi(res[93], a[46], a[46]); // Add sqr(92) + res[94] = fma52lo(res[94], a[47], a[47]); // Add sqr(94) + res[95] = fma52hi(res[95], a[47], a[47]); // Add sqr(94) + res[96] = fma52lo(res[96], a[47], a[49]); // Sum(96) + res[97] = fma52hi(res[97], a[47], a[49]); // Sum(96) + res[97] = fma52lo(res[97], a[48], a[49]); // Sum(97) + res[98] = fma52hi(res[98], a[48], a[49]); // Sum(97) + res[96] = fma52lo(res[96], a[46], a[50]); // Sum(96) + res[97] = fma52hi(res[97], a[46], a[50]); // Sum(96) + res[97] = fma52lo(res[97], a[47], a[50]); // Sum(97) + res[98] = fma52hi(res[98], a[47], a[50]); // Sum(97) + res[98] = fma52lo(res[98], a[48], a[50]); // Sum(98) + res[99] = fma52hi(res[99], a[48], a[50]); // Sum(98) + res[99] = fma52lo(res[99], a[49], a[50]); // Sum(99) + res[100] = fma52hi(res[100], a[49], a[50]); // Sum(99) + res[96] = fma52lo(res[96], a[45], a[51]); // Sum(96) + res[97] = fma52hi(res[97], a[45], a[51]); // Sum(96) + res[97] = fma52lo(res[97], a[46], a[51]); // Sum(97) + res[98] = fma52hi(res[98], a[46], a[51]); // Sum(97) + res[98] = fma52lo(res[98], a[47], a[51]); // Sum(98) + res[99] = fma52hi(res[99], a[47], a[51]); // Sum(98) + res[99] = fma52lo(res[99], a[48], a[51]); // Sum(99) + res[100] = fma52hi(res[100], a[48], a[51]); // Sum(99) + res[100] = fma52lo(res[100], a[49], a[51]); // Sum(100) + res[101] = fma52hi(res[101], a[49], a[51]); // Sum(100) + res[101] = fma52lo(res[101], a[50], a[51]); // Sum(101) + res[102] = fma52hi(res[102], a[50], a[51]); // Sum(101) + res[96] = fma52lo(res[96], a[44], a[52]); // Sum(96) + res[97] = fma52hi(res[97], a[44], a[52]); // Sum(96) + res[97] = fma52lo(res[97], a[45], a[52]); // Sum(97) + res[98] = fma52hi(res[98], a[45], a[52]); // Sum(97) + res[98] = fma52lo(res[98], a[46], a[52]); // Sum(98) + res[99] = fma52hi(res[99], a[46], a[52]); // Sum(98) + res[99] = fma52lo(res[99], a[47], a[52]); // Sum(99) + res[100] = fma52hi(res[100], a[47], a[52]); // Sum(99) + res[100] = fma52lo(res[100], a[48], a[52]); // Sum(100) + res[101] = fma52hi(res[101], a[48], a[52]); // Sum(100) + res[101] = fma52lo(res[101], a[49], a[52]); // Sum(101) + res[102] = fma52hi(res[102], a[49], a[52]); // Sum(101) + res[102] = fma52lo(res[102], a[50], a[52]); // Sum(102) + res[103] = fma52hi(res[103], a[50], a[52]); // Sum(102) + res[103] = fma52lo(res[103], a[51], a[52]); // Sum(103) + res[104] = fma52hi(res[104], a[51], a[52]); // Sum(103) + res[96] = fma52lo(res[96], a[43], a[53]); // Sum(96) + res[97] = fma52hi(res[97], a[43], a[53]); // Sum(96) + res[97] = fma52lo(res[97], a[44], a[53]); // Sum(97) + res[98] = fma52hi(res[98], a[44], a[53]); // Sum(97) + res[98] = fma52lo(res[98], a[45], a[53]); // Sum(98) + res[99] = fma52hi(res[99], a[45], a[53]); // Sum(98) + res[99] = fma52lo(res[99], a[46], a[53]); // Sum(99) + res[100] = fma52hi(res[100], a[46], a[53]); // Sum(99) + res[100] = fma52lo(res[100], a[47], a[53]); // Sum(100) + res[101] = fma52hi(res[101], a[47], a[53]); // Sum(100) + res[101] = fma52lo(res[101], a[48], a[53]); // Sum(101) + res[102] = fma52hi(res[102], a[48], a[53]); // Sum(101) + res[102] = fma52lo(res[102], a[49], a[53]); // Sum(102) + res[103] = fma52hi(res[103], a[49], a[53]); // Sum(102) + res[103] = fma52lo(res[103], a[50], a[53]); // Sum(103) + res[104] = fma52hi(res[104], a[50], a[53]); // Sum(103) + res[104] = fma52lo(res[104], a[51], a[53]); // Sum(104) + res[105] = fma52hi(res[105], a[51], a[53]); // Sum(104) + res[105] = fma52lo(res[105], a[52], a[53]); // Sum(105) + res[106] = fma52hi(res[106], a[52], a[53]); // Sum(105) + res[96] = fma52lo(res[96], a[42], a[54]); // Sum(96) + res[97] = fma52hi(res[97], a[42], a[54]); // Sum(96) + res[97] = fma52lo(res[97], a[43], a[54]); // Sum(97) + res[98] = fma52hi(res[98], a[43], a[54]); // Sum(97) + res[98] = fma52lo(res[98], a[44], a[54]); // Sum(98) + res[99] = fma52hi(res[99], a[44], a[54]); // Sum(98) + res[99] = fma52lo(res[99], a[45], a[54]); // Sum(99) + res[100] = fma52hi(res[100], a[45], a[54]); // Sum(99) + res[100] = fma52lo(res[100], a[46], a[54]); // Sum(100) + res[101] = fma52hi(res[101], a[46], a[54]); // Sum(100) + res[101] = fma52lo(res[101], a[47], a[54]); // Sum(101) + res[102] = fma52hi(res[102], a[47], a[54]); // Sum(101) + res[102] = fma52lo(res[102], a[48], a[54]); // Sum(102) + res[103] = fma52hi(res[103], a[48], a[54]); // Sum(102) + res[103] = fma52lo(res[103], a[49], a[54]); // Sum(103) + res[104] = fma52hi(res[104], a[49], a[54]); // Sum(103) + res[104] = fma52lo(res[104], a[50], a[54]); // Sum(104) + res[105] = fma52hi(res[105], a[50], a[54]); // Sum(104) + res[105] = fma52lo(res[105], a[51], a[54]); // Sum(105) + res[106] = fma52hi(res[106], a[51], a[54]); // Sum(105) + res[106] = fma52lo(res[106], a[52], a[54]); // Sum(106) + res[107] = fma52hi(res[107], a[52], a[54]); // Sum(106) + res[107] = fma52lo(res[107], a[53], a[54]); // Sum(107) + res[108] = fma52hi(res[108], a[53], a[54]); // Sum(107) + res[96] = fma52lo(res[96], a[41], a[55]); // Sum(96) + res[97] = fma52hi(res[97], a[41], a[55]); // Sum(96) + res[97] = fma52lo(res[97], a[42], a[55]); // Sum(97) + res[98] = fma52hi(res[98], a[42], a[55]); // Sum(97) + res[98] = fma52lo(res[98], a[43], a[55]); // Sum(98) + res[99] = fma52hi(res[99], a[43], a[55]); // Sum(98) + res[99] = fma52lo(res[99], a[44], a[55]); // Sum(99) + res[100] = fma52hi(res[100], a[44], a[55]); // Sum(99) + res[100] = fma52lo(res[100], a[45], a[55]); // Sum(100) + res[101] = fma52hi(res[101], a[45], a[55]); // Sum(100) + res[101] = fma52lo(res[101], a[46], a[55]); // Sum(101) + res[102] = fma52hi(res[102], a[46], a[55]); // Sum(101) + res[102] = fma52lo(res[102], a[47], a[55]); // Sum(102) + res[103] = fma52hi(res[103], a[47], a[55]); // Sum(102) + res[103] = fma52lo(res[103], a[48], a[55]); // Sum(103) + res[104] = fma52hi(res[104], a[48], a[55]); // Sum(103) + res[104] = fma52lo(res[104], a[49], a[55]); // Sum(104) + res[105] = fma52hi(res[105], a[49], a[55]); // Sum(104) + res[105] = fma52lo(res[105], a[50], a[55]); // Sum(105) + res[106] = fma52hi(res[106], a[50], a[55]); // Sum(105) + res[106] = fma52lo(res[106], a[51], a[55]); // Sum(106) + res[107] = fma52hi(res[107], a[51], a[55]); // Sum(106) + res[107] = fma52lo(res[107], a[52], a[55]); // Sum(107) + res[108] = fma52hi(res[108], a[52], a[55]); // Sum(107) + res[96] = fma52lo(res[96], a[40], a[56]); // Sum(96) + res[97] = fma52hi(res[97], a[40], a[56]); // Sum(96) + res[97] = fma52lo(res[97], a[41], a[56]); // Sum(97) + res[98] = fma52hi(res[98], a[41], a[56]); // Sum(97) + res[98] = fma52lo(res[98], a[42], a[56]); // Sum(98) + res[99] = fma52hi(res[99], a[42], a[56]); // Sum(98) + res[99] = fma52lo(res[99], a[43], a[56]); // Sum(99) + res[100] = fma52hi(res[100], a[43], a[56]); // Sum(99) + res[100] = fma52lo(res[100], a[44], a[56]); // Sum(100) + res[101] = fma52hi(res[101], a[44], a[56]); // Sum(100) + res[101] = fma52lo(res[101], a[45], a[56]); // Sum(101) + res[102] = fma52hi(res[102], a[45], a[56]); // Sum(101) + res[102] = fma52lo(res[102], a[46], a[56]); // Sum(102) + res[103] = fma52hi(res[103], a[46], a[56]); // Sum(102) + res[103] = fma52lo(res[103], a[47], a[56]); // Sum(103) + res[104] = fma52hi(res[104], a[47], a[56]); // Sum(103) + res[104] = fma52lo(res[104], a[48], a[56]); // Sum(104) + res[105] = fma52hi(res[105], a[48], a[56]); // Sum(104) + res[105] = fma52lo(res[105], a[49], a[56]); // Sum(105) + res[106] = fma52hi(res[106], a[49], a[56]); // Sum(105) + res[106] = fma52lo(res[106], a[50], a[56]); // Sum(106) + res[107] = fma52hi(res[107], a[50], a[56]); // Sum(106) + res[107] = fma52lo(res[107], a[51], a[56]); // Sum(107) + res[108] = fma52hi(res[108], a[51], a[56]); // Sum(107) + res[96] = fma52lo(res[96], a[39], a[57]); // Sum(96) + res[97] = fma52hi(res[97], a[39], a[57]); // Sum(96) + res[97] = fma52lo(res[97], a[40], a[57]); // Sum(97) + res[98] = fma52hi(res[98], a[40], a[57]); // Sum(97) + res[98] = fma52lo(res[98], a[41], a[57]); // Sum(98) + res[99] = fma52hi(res[99], a[41], a[57]); // Sum(98) + res[99] = fma52lo(res[99], a[42], a[57]); // Sum(99) + res[100] = fma52hi(res[100], a[42], a[57]); // Sum(99) + res[100] = fma52lo(res[100], a[43], a[57]); // Sum(100) + res[101] = fma52hi(res[101], a[43], a[57]); // Sum(100) + res[101] = fma52lo(res[101], a[44], a[57]); // Sum(101) + res[102] = fma52hi(res[102], a[44], a[57]); // Sum(101) + res[102] = fma52lo(res[102], a[45], a[57]); // Sum(102) + res[103] = fma52hi(res[103], a[45], a[57]); // Sum(102) + res[103] = fma52lo(res[103], a[46], a[57]); // Sum(103) + res[104] = fma52hi(res[104], a[46], a[57]); // Sum(103) + res[104] = fma52lo(res[104], a[47], a[57]); // Sum(104) + res[105] = fma52hi(res[105], a[47], a[57]); // Sum(104) + res[105] = fma52lo(res[105], a[48], a[57]); // Sum(105) + res[106] = fma52hi(res[106], a[48], a[57]); // Sum(105) + res[106] = fma52lo(res[106], a[49], a[57]); // Sum(106) + res[107] = fma52hi(res[107], a[49], a[57]); // Sum(106) + res[107] = fma52lo(res[107], a[50], a[57]); // Sum(107) + res[108] = fma52hi(res[108], a[50], a[57]); // Sum(107) + res[96] = fma52lo(res[96], a[38], a[58]); // Sum(96) + res[97] = fma52hi(res[97], a[38], a[58]); // Sum(96) + res[97] = fma52lo(res[97], a[39], a[58]); // Sum(97) + res[98] = fma52hi(res[98], a[39], a[58]); // Sum(97) + res[98] = fma52lo(res[98], a[40], a[58]); // Sum(98) + res[99] = fma52hi(res[99], a[40], a[58]); // Sum(98) + res[99] = fma52lo(res[99], a[41], a[58]); // Sum(99) + res[100] = fma52hi(res[100], a[41], a[58]); // Sum(99) + res[100] = fma52lo(res[100], a[42], a[58]); // Sum(100) + res[101] = fma52hi(res[101], a[42], a[58]); // Sum(100) + res[101] = fma52lo(res[101], a[43], a[58]); // Sum(101) + res[102] = fma52hi(res[102], a[43], a[58]); // Sum(101) + res[102] = fma52lo(res[102], a[44], a[58]); // Sum(102) + res[103] = fma52hi(res[103], a[44], a[58]); // Sum(102) + res[103] = fma52lo(res[103], a[45], a[58]); // Sum(103) + res[104] = fma52hi(res[104], a[45], a[58]); // Sum(103) + res[104] = fma52lo(res[104], a[46], a[58]); // Sum(104) + res[105] = fma52hi(res[105], a[46], a[58]); // Sum(104) + res[105] = fma52lo(res[105], a[47], a[58]); // Sum(105) + res[106] = fma52hi(res[106], a[47], a[58]); // Sum(105) + res[106] = fma52lo(res[106], a[48], a[58]); // Sum(106) + res[107] = fma52hi(res[107], a[48], a[58]); // Sum(106) + res[107] = fma52lo(res[107], a[49], a[58]); // Sum(107) + res[108] = fma52hi(res[108], a[49], a[58]); // Sum(107) + res[96] = fma52lo(res[96], a[37], a[59]); // Sum(96) + res[97] = fma52hi(res[97], a[37], a[59]); // Sum(96) + res[97] = fma52lo(res[97], a[38], a[59]); // Sum(97) + res[98] = fma52hi(res[98], a[38], a[59]); // Sum(97) + res[98] = fma52lo(res[98], a[39], a[59]); // Sum(98) + res[99] = fma52hi(res[99], a[39], a[59]); // Sum(98) + res[99] = fma52lo(res[99], a[40], a[59]); // Sum(99) + res[100] = fma52hi(res[100], a[40], a[59]); // Sum(99) + res[100] = fma52lo(res[100], a[41], a[59]); // Sum(100) + res[101] = fma52hi(res[101], a[41], a[59]); // Sum(100) + res[101] = fma52lo(res[101], a[42], a[59]); // Sum(101) + res[102] = fma52hi(res[102], a[42], a[59]); // Sum(101) + res[102] = fma52lo(res[102], a[43], a[59]); // Sum(102) + res[103] = fma52hi(res[103], a[43], a[59]); // Sum(102) + res[103] = fma52lo(res[103], a[44], a[59]); // Sum(103) + res[104] = fma52hi(res[104], a[44], a[59]); // Sum(103) + res[104] = fma52lo(res[104], a[45], a[59]); // Sum(104) + res[105] = fma52hi(res[105], a[45], a[59]); // Sum(104) + res[105] = fma52lo(res[105], a[46], a[59]); // Sum(105) + res[106] = fma52hi(res[106], a[46], a[59]); // Sum(105) + res[106] = fma52lo(res[106], a[47], a[59]); // Sum(106) + res[107] = fma52hi(res[107], a[47], a[59]); // Sum(106) + res[107] = fma52lo(res[107], a[48], a[59]); // Sum(107) + res[108] = fma52hi(res[108], a[48], a[59]); // Sum(107) + res[96] = add64(res[96], res[96]); // Double(96) + res[97] = add64(res[97], res[97]); // Double(97) + res[98] = add64(res[98], res[98]); // Double(98) + res[99] = add64(res[99], res[99]); // Double(99) + res[100] = add64(res[100], res[100]); // Double(100) + res[101] = add64(res[101], res[101]); // Double(101) + res[102] = add64(res[102], res[102]); // Double(102) + res[103] = add64(res[103], res[103]); // Double(103) + res[104] = add64(res[104], res[104]); // Double(104) + res[105] = add64(res[105], res[105]); // Double(105) + res[106] = add64(res[106], res[106]); // Double(106) + res[107] = add64(res[107], res[107]); // Double(107) + res[96] = fma52lo(res[96], a[48], a[48]); // Add sqr(96) + res[97] = fma52hi(res[97], a[48], a[48]); // Add sqr(96) + res[98] = fma52lo(res[98], a[49], a[49]); // Add sqr(98) + res[99] = fma52hi(res[99], a[49], a[49]); // Add sqr(98) + res[100] = fma52lo(res[100], a[50], a[50]); // Add sqr(100) + res[101] = fma52hi(res[101], a[50], a[50]); // Add sqr(100) + res[102] = fma52lo(res[102], a[51], a[51]); // Add sqr(102) + res[103] = fma52hi(res[103], a[51], a[51]); // Add sqr(102) + res[104] = fma52lo(res[104], a[52], a[52]); // Add sqr(104) + res[105] = fma52hi(res[105], a[52], a[52]); // Add sqr(104) + res[106] = fma52lo(res[106], a[53], a[53]); // Add sqr(106) + res[107] = fma52hi(res[107], a[53], a[53]); // Add sqr(106) + res[108] = fma52lo(res[108], a[53], a[55]); // Sum(108) + res[109] = fma52hi(res[109], a[53], a[55]); // Sum(108) + res[109] = fma52lo(res[109], a[54], a[55]); // Sum(109) + res[110] = fma52hi(res[110], a[54], a[55]); // Sum(109) + res[108] = fma52lo(res[108], a[52], a[56]); // Sum(108) + res[109] = fma52hi(res[109], a[52], a[56]); // Sum(108) + res[109] = fma52lo(res[109], a[53], a[56]); // Sum(109) + res[110] = fma52hi(res[110], a[53], a[56]); // Sum(109) + res[110] = fma52lo(res[110], a[54], a[56]); // Sum(110) + res[111] = fma52hi(res[111], a[54], a[56]); // Sum(110) + res[111] = fma52lo(res[111], a[55], a[56]); // Sum(111) + res[112] = fma52hi(res[112], a[55], a[56]); // Sum(111) + res[108] = fma52lo(res[108], a[51], a[57]); // Sum(108) + res[109] = fma52hi(res[109], a[51], a[57]); // Sum(108) + res[109] = fma52lo(res[109], a[52], a[57]); // Sum(109) + res[110] = fma52hi(res[110], a[52], a[57]); // Sum(109) + res[110] = fma52lo(res[110], a[53], a[57]); // Sum(110) + res[111] = fma52hi(res[111], a[53], a[57]); // Sum(110) + res[111] = fma52lo(res[111], a[54], a[57]); // Sum(111) + res[112] = fma52hi(res[112], a[54], a[57]); // Sum(111) + res[112] = fma52lo(res[112], a[55], a[57]); // Sum(112) + res[113] = fma52hi(res[113], a[55], a[57]); // Sum(112) + res[113] = fma52lo(res[113], a[56], a[57]); // Sum(113) + res[114] = fma52hi(res[114], a[56], a[57]); // Sum(113) + res[108] = fma52lo(res[108], a[50], a[58]); // Sum(108) + res[109] = fma52hi(res[109], a[50], a[58]); // Sum(108) + res[109] = fma52lo(res[109], a[51], a[58]); // Sum(109) + res[110] = fma52hi(res[110], a[51], a[58]); // Sum(109) + res[110] = fma52lo(res[110], a[52], a[58]); // Sum(110) + res[111] = fma52hi(res[111], a[52], a[58]); // Sum(110) + res[111] = fma52lo(res[111], a[53], a[58]); // Sum(111) + res[112] = fma52hi(res[112], a[53], a[58]); // Sum(111) + res[112] = fma52lo(res[112], a[54], a[58]); // Sum(112) + res[113] = fma52hi(res[113], a[54], a[58]); // Sum(112) + res[113] = fma52lo(res[113], a[55], a[58]); // Sum(113) + res[114] = fma52hi(res[114], a[55], a[58]); // Sum(113) + res[114] = fma52lo(res[114], a[56], a[58]); // Sum(114) + res[115] = fma52hi(res[115], a[56], a[58]); // Sum(114) + res[115] = fma52lo(res[115], a[57], a[58]); // Sum(115) + res[116] = fma52hi(res[116], a[57], a[58]); // Sum(115) + res[108] = fma52lo(res[108], a[49], a[59]); // Sum(108) + res[109] = fma52hi(res[109], a[49], a[59]); // Sum(108) + res[109] = fma52lo(res[109], a[50], a[59]); // Sum(109) + res[110] = fma52hi(res[110], a[50], a[59]); // Sum(109) + res[110] = fma52lo(res[110], a[51], a[59]); // Sum(110) + res[111] = fma52hi(res[111], a[51], a[59]); // Sum(110) + res[111] = fma52lo(res[111], a[52], a[59]); // Sum(111) + res[112] = fma52hi(res[112], a[52], a[59]); // Sum(111) + res[112] = fma52lo(res[112], a[53], a[59]); // Sum(112) + res[113] = fma52hi(res[113], a[53], a[59]); // Sum(112) + res[113] = fma52lo(res[113], a[54], a[59]); // Sum(113) + res[114] = fma52hi(res[114], a[54], a[59]); // Sum(113) + res[114] = fma52lo(res[114], a[55], a[59]); // Sum(114) + res[115] = fma52hi(res[115], a[55], a[59]); // Sum(114) + res[115] = fma52lo(res[115], a[56], a[59]); // Sum(115) + res[116] = fma52hi(res[116], a[56], a[59]); // Sum(115) + res[116] = fma52lo(res[116], a[57], a[59]); // Sum(116) + res[117] = fma52hi(res[117], a[57], a[59]); // Sum(116) + res[117] = fma52lo(res[117], a[58], a[59]); // Sum(117) + res[118] = fma52hi(res[118], a[58], a[59]); // Sum(117) + res[108] = add64(res[108], res[108]); // Double(108) + res[109] = add64(res[109], res[109]); // Double(109) + res[110] = add64(res[110], res[110]); // Double(110) + res[111] = add64(res[111], res[111]); // Double(111) + res[112] = add64(res[112], res[112]); // Double(112) + res[113] = add64(res[113], res[113]); // Double(113) + res[114] = add64(res[114], res[114]); // Double(114) + res[115] = add64(res[115], res[115]); // Double(115) + res[116] = add64(res[116], res[116]); // Double(116) + res[117] = add64(res[117], res[117]); // Double(117) + res[118] = add64(res[118], res[118]); // Double(118) + res[108] = fma52lo(res[108], a[54], a[54]); // Add sqr(108) + res[109] = fma52hi(res[109], a[54], a[54]); // Add sqr(108) + res[110] = fma52lo(res[110], a[55], a[55]); // Add sqr(110) + res[111] = fma52hi(res[111], a[55], a[55]); // Add sqr(110) + res[112] = fma52lo(res[112], a[56], a[56]); // Add sqr(112) + res[113] = fma52hi(res[113], a[56], a[56]); // Add sqr(112) + res[114] = fma52lo(res[114], a[57], a[57]); // Add sqr(114) + res[115] = fma52hi(res[115], a[57], a[57]); // Add sqr(114) + res[116] = fma52lo(res[116], a[58], a[58]); // Add sqr(116) + res[117] = fma52hi(res[117], a[58], a[58]); // Add sqr(116) + res[118] = fma52lo(res[118], a[59], a[59]); // Add sqr(118) + res[119] = fma52hi(res[119], a[59], a[59]); // Add sqr(118) + + // Montgomery Reduction + int it; + for (it = 0; it < 60; it += 10) { // Reduction step + int jt = 0; + if ((it + 0) > 0) + res[it + 0] = add64(res[it + 0], srli64(res[it + -1], DIGIT_SIZE)); + u[it + 0] = mul52lo(res[it + 0], k); + res[it + jt + 0] = fma52lo(res[it + jt + 0], u[it + 0], m[jt + 0]); + res[it + jt + 1] = fma52hi(res[it + jt + 1], u[it + 0], m[jt + 0]); + res[it + jt + 1] = fma52lo(res[it + jt + 1], u[it + 0], m[jt + 1]); + res[it + jt + 2] = fma52hi(res[it + jt + 2], u[it + 0], m[jt + 1]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 0], m[jt + 2]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 0], m[jt + 2]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 0], m[jt + 3]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 0], m[jt + 3]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 0], m[jt + 4]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 0], m[jt + 4]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 0], m[jt + 5]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 0], m[jt + 5]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 0], m[jt + 6]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 0], m[jt + 6]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 0], m[jt + 7]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 0], m[jt + 7]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 0], m[jt + 8]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 0], m[jt + 8]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 0], m[jt + 9]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 0], m[jt + 9]); + res[it + 1] = add64(res[it + 1], srli64(res[it + 0], DIGIT_SIZE)); + u[it + 1] = mul52lo(res[it + 1], k); + res[it + jt + 1] = fma52lo(res[it + jt + 1], u[it + 1], m[jt + 0]); + res[it + jt + 2] = fma52hi(res[it + jt + 2], u[it + 1], m[jt + 0]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 1], m[jt + 1]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 1], m[jt + 1]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 1], m[jt + 2]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 1], m[jt + 2]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 1], m[jt + 3]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 1], m[jt + 3]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 1], m[jt + 4]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 1], m[jt + 4]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 1], m[jt + 5]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 1], m[jt + 5]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 1], m[jt + 6]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 1], m[jt + 6]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 1], m[jt + 7]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 1], m[jt + 7]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 1], m[jt + 8]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 1], m[jt + 8]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 1], m[jt + 9]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 1], m[jt + 9]); + res[it + 2] = add64(res[it + 2], srli64(res[it + 1], DIGIT_SIZE)); + u[it + 2] = mul52lo(res[it + 2], k); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 2], m[jt + 0]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 2], m[jt + 0]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 2], m[jt + 1]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 2], m[jt + 1]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 2], m[jt + 2]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 2], m[jt + 2]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 2], m[jt + 3]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 2], m[jt + 3]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 2], m[jt + 4]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 2], m[jt + 4]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 2], m[jt + 5]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 2], m[jt + 5]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 2], m[jt + 6]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 2], m[jt + 6]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 2], m[jt + 7]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 2], m[jt + 7]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 2], m[jt + 8]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 2], m[jt + 8]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 2], m[jt + 9]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 2], m[jt + 9]); + res[it + 3] = add64(res[it + 3], srli64(res[it + 2], DIGIT_SIZE)); + u[it + 3] = mul52lo(res[it + 3], k); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 3], m[jt + 0]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 3], m[jt + 0]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 3], m[jt + 1]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 3], m[jt + 1]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 3], m[jt + 2]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 3], m[jt + 2]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 3], m[jt + 3]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 3], m[jt + 3]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 3], m[jt + 4]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 3], m[jt + 4]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 3], m[jt + 5]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 3], m[jt + 5]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 3], m[jt + 6]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 3], m[jt + 6]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 3], m[jt + 7]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 3], m[jt + 7]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 3], m[jt + 8]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 3], m[jt + 8]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 3], m[jt + 9]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 3], m[jt + 9]); + res[it + 4] = add64(res[it + 4], srli64(res[it + 3], DIGIT_SIZE)); + u[it + 4] = mul52lo(res[it + 4], k); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 4], m[jt + 0]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 4], m[jt + 0]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 4], m[jt + 1]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 4], m[jt + 1]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 4], m[jt + 2]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 4], m[jt + 2]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 4], m[jt + 3]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 4], m[jt + 3]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 4], m[jt + 4]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 4], m[jt + 4]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 4], m[jt + 5]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 4], m[jt + 5]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 4], m[jt + 6]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 4], m[jt + 6]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 4], m[jt + 7]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 4], m[jt + 7]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 4], m[jt + 8]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 4], m[jt + 8]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 4], m[jt + 9]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 4], m[jt + 9]); + res[it + 5] = add64(res[it + 5], srli64(res[it + 4], DIGIT_SIZE)); + u[it + 5] = mul52lo(res[it + 5], k); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 5], m[jt + 0]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 5], m[jt + 0]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 5], m[jt + 1]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 5], m[jt + 1]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 5], m[jt + 2]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 5], m[jt + 2]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 5], m[jt + 3]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 5], m[jt + 3]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 5], m[jt + 4]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 5], m[jt + 4]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 5], m[jt + 5]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 5], m[jt + 5]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 5], m[jt + 6]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 5], m[jt + 6]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 5], m[jt + 7]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 5], m[jt + 7]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 5], m[jt + 8]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 5], m[jt + 8]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 5], m[jt + 9]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 5], m[jt + 9]); + res[it + 6] = add64(res[it + 6], srli64(res[it + 5], DIGIT_SIZE)); + u[it + 6] = mul52lo(res[it + 6], k); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 6], m[jt + 0]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 6], m[jt + 0]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 6], m[jt + 1]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 6], m[jt + 1]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 6], m[jt + 2]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 6], m[jt + 2]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 6], m[jt + 3]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 6], m[jt + 3]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 6], m[jt + 4]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 6], m[jt + 4]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 6], m[jt + 5]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 6], m[jt + 5]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 6], m[jt + 6]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 6], m[jt + 6]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 6], m[jt + 7]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 6], m[jt + 7]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 6], m[jt + 8]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 6], m[jt + 8]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 6], m[jt + 9]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 6], m[jt + 9]); + res[it + 7] = add64(res[it + 7], srli64(res[it + 6], DIGIT_SIZE)); + u[it + 7] = mul52lo(res[it + 7], k); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 7], m[jt + 0]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 7], m[jt + 0]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 7], m[jt + 1]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 7], m[jt + 1]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 7], m[jt + 2]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 7], m[jt + 2]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 7], m[jt + 3]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 7], m[jt + 3]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 7], m[jt + 4]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 7], m[jt + 4]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 7], m[jt + 5]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 7], m[jt + 5]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 7], m[jt + 6]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 7], m[jt + 6]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 7], m[jt + 7]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 7], m[jt + 7]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 7], m[jt + 8]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 7], m[jt + 8]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 7], m[jt + 9]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 7], m[jt + 9]); + res[it + 8] = add64(res[it + 8], srli64(res[it + 7], DIGIT_SIZE)); + u[it + 8] = mul52lo(res[it + 8], k); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 8], m[jt + 0]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 8], m[jt + 0]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 8], m[jt + 1]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 8], m[jt + 1]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 8], m[jt + 2]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 8], m[jt + 2]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 8], m[jt + 3]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 8], m[jt + 3]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 8], m[jt + 4]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 8], m[jt + 4]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 8], m[jt + 5]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 8], m[jt + 5]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 8], m[jt + 6]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 8], m[jt + 6]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 8], m[jt + 7]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 8], m[jt + 7]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 8], m[jt + 8]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 8], m[jt + 8]); + res[it + jt + 17] = fma52lo(res[it + jt + 17], u[it + 8], m[jt + 9]); + res[it + jt + 18] = fma52hi(res[it + jt + 18], u[it + 8], m[jt + 9]); + res[it + 9] = add64(res[it + 9], srli64(res[it + 8], DIGIT_SIZE)); + u[it + 9] = mul52lo(res[it + 9], k); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 9], m[jt + 0]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 9], m[jt + 0]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 9], m[jt + 1]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 9], m[jt + 1]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 9], m[jt + 2]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 9], m[jt + 2]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 9], m[jt + 3]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 9], m[jt + 3]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 9], m[jt + 4]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 9], m[jt + 4]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 9], m[jt + 5]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 9], m[jt + 5]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 9], m[jt + 6]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 9], m[jt + 6]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 9], m[jt + 7]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 9], m[jt + 7]); + res[it + jt + 17] = fma52lo(res[it + jt + 17], u[it + 9], m[jt + 8]); + res[it + jt + 18] = fma52hi(res[it + jt + 18], u[it + 9], m[jt + 8]); + res[it + jt + 18] = fma52lo(res[it + jt + 18], u[it + 9], m[jt + 9]); + res[it + jt + 19] = fma52hi(res[it + jt + 19], u[it + 9], m[jt + 9]); + + for (jt = 10; jt < 60; jt += 10) { // Poly tile + res[it + jt + 0] = fma52lo(res[it + jt + 0], u[it + 0], m[jt + 0]); + res[it + jt + 1] = fma52hi(res[it + jt + 1], u[it + 0], m[jt + 0]); + res[it + jt + 1] = fma52lo(res[it + jt + 1], u[it + 0], m[jt + 1]); + res[it + jt + 2] = fma52hi(res[it + jt + 2], u[it + 0], m[jt + 1]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 0], m[jt + 2]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 0], m[jt + 2]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 0], m[jt + 3]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 0], m[jt + 3]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 0], m[jt + 4]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 0], m[jt + 4]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 0], m[jt + 5]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 0], m[jt + 5]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 0], m[jt + 6]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 0], m[jt + 6]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 0], m[jt + 7]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 0], m[jt + 7]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 0], m[jt + 8]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 0], m[jt + 8]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 0], m[jt + 9]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 0], m[jt + 9]); + res[it + jt + 1] = fma52lo(res[it + jt + 1], u[it + 1], m[jt + 0]); + res[it + jt + 2] = fma52hi(res[it + jt + 2], u[it + 1], m[jt + 0]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 1], m[jt + 1]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 1], m[jt + 1]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 1], m[jt + 2]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 1], m[jt + 2]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 1], m[jt + 3]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 1], m[jt + 3]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 1], m[jt + 4]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 1], m[jt + 4]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 1], m[jt + 5]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 1], m[jt + 5]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 1], m[jt + 6]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 1], m[jt + 6]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 1], m[jt + 7]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 1], m[jt + 7]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 1], m[jt + 8]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 1], m[jt + 8]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 1], m[jt + 9]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 1], m[jt + 9]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 2], m[jt + 0]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 2], m[jt + 0]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 2], m[jt + 1]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 2], m[jt + 1]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 2], m[jt + 2]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 2], m[jt + 2]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 2], m[jt + 3]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 2], m[jt + 3]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 2], m[jt + 4]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 2], m[jt + 4]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 2], m[jt + 5]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 2], m[jt + 5]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 2], m[jt + 6]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 2], m[jt + 6]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 2], m[jt + 7]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 2], m[jt + 7]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 2], m[jt + 8]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 2], m[jt + 8]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 2], m[jt + 9]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 2], m[jt + 9]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 3], m[jt + 0]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 3], m[jt + 0]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 3], m[jt + 1]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 3], m[jt + 1]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 3], m[jt + 2]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 3], m[jt + 2]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 3], m[jt + 3]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 3], m[jt + 3]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 3], m[jt + 4]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 3], m[jt + 4]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 3], m[jt + 5]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 3], m[jt + 5]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 3], m[jt + 6]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 3], m[jt + 6]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 3], m[jt + 7]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 3], m[jt + 7]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 3], m[jt + 8]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 3], m[jt + 8]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 3], m[jt + 9]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 3], m[jt + 9]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 4], m[jt + 0]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 4], m[jt + 0]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 4], m[jt + 1]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 4], m[jt + 1]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 4], m[jt + 2]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 4], m[jt + 2]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 4], m[jt + 3]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 4], m[jt + 3]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 4], m[jt + 4]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 4], m[jt + 4]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 4], m[jt + 5]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 4], m[jt + 5]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 4], m[jt + 6]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 4], m[jt + 6]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 4], m[jt + 7]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 4], m[jt + 7]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 4], m[jt + 8]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 4], m[jt + 8]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 4], m[jt + 9]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 4], m[jt + 9]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 5], m[jt + 0]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 5], m[jt + 0]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 5], m[jt + 1]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 5], m[jt + 1]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 5], m[jt + 2]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 5], m[jt + 2]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 5], m[jt + 3]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 5], m[jt + 3]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 5], m[jt + 4]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 5], m[jt + 4]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 5], m[jt + 5]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 5], m[jt + 5]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 5], m[jt + 6]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 5], m[jt + 6]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 5], m[jt + 7]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 5], m[jt + 7]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 5], m[jt + 8]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 5], m[jt + 8]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 5], m[jt + 9]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 5], m[jt + 9]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 6], m[jt + 0]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 6], m[jt + 0]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 6], m[jt + 1]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 6], m[jt + 1]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 6], m[jt + 2]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 6], m[jt + 2]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 6], m[jt + 3]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 6], m[jt + 3]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 6], m[jt + 4]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 6], m[jt + 4]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 6], m[jt + 5]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 6], m[jt + 5]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 6], m[jt + 6]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 6], m[jt + 6]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 6], m[jt + 7]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 6], m[jt + 7]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 6], m[jt + 8]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 6], m[jt + 8]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 6], m[jt + 9]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 6], m[jt + 9]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 7], m[jt + 0]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 7], m[jt + 0]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 7], m[jt + 1]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 7], m[jt + 1]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 7], m[jt + 2]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 7], m[jt + 2]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 7], m[jt + 3]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 7], m[jt + 3]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 7], m[jt + 4]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 7], m[jt + 4]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 7], m[jt + 5]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 7], m[jt + 5]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 7], m[jt + 6]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 7], m[jt + 6]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 7], m[jt + 7]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 7], m[jt + 7]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 7], m[jt + 8]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 7], m[jt + 8]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 7], m[jt + 9]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 7], m[jt + 9]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 8], m[jt + 0]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 8], m[jt + 0]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 8], m[jt + 1]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 8], m[jt + 1]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 8], m[jt + 2]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 8], m[jt + 2]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 8], m[jt + 3]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 8], m[jt + 3]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 8], m[jt + 4]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 8], m[jt + 4]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 8], m[jt + 5]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 8], m[jt + 5]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 8], m[jt + 6]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 8], m[jt + 6]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 8], m[jt + 7]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 8], m[jt + 7]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 8], m[jt + 8]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 8], m[jt + 8]); + res[it + jt + 17] = fma52lo(res[it + jt + 17], u[it + 8], m[jt + 9]); + res[it + jt + 18] = fma52hi(res[it + jt + 18], u[it + 8], m[jt + 9]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 9], m[jt + 0]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 9], m[jt + 0]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 9], m[jt + 1]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 9], m[jt + 1]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 9], m[jt + 2]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 9], m[jt + 2]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 9], m[jt + 3]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 9], m[jt + 3]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 9], m[jt + 4]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 9], m[jt + 4]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 9], m[jt + 5]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 9], m[jt + 5]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 9], m[jt + 6]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 9], m[jt + 6]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 9], m[jt + 7]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 9], m[jt + 7]); + res[it + jt + 17] = fma52lo(res[it + jt + 17], u[it + 9], m[jt + 8]); + res[it + jt + 18] = fma52hi(res[it + jt + 18], u[it + 9], m[jt + 8]); + res[it + jt + 18] = fma52lo(res[it + jt + 18], u[it + 9], m[jt + 9]); + res[it + jt + 19] = fma52hi(res[it + jt + 19], u[it + 9], m[jt + 9]); + } + } + + // Normalization + res[60] = add64(res[60], srli64(res[59], DIGIT_SIZE)); + r[0] = and64_const(res[60], DIGIT_MASK); + res[61] = add64(res[61], srli64(res[60], DIGIT_SIZE)); + r[1] = and64_const(res[61], DIGIT_MASK); + res[62] = add64(res[62], srli64(res[61], DIGIT_SIZE)); + r[2] = and64_const(res[62], DIGIT_MASK); + res[63] = add64(res[63], srli64(res[62], DIGIT_SIZE)); + r[3] = and64_const(res[63], DIGIT_MASK); + res[64] = add64(res[64], srli64(res[63], DIGIT_SIZE)); + r[4] = and64_const(res[64], DIGIT_MASK); + res[65] = add64(res[65], srli64(res[64], DIGIT_SIZE)); + r[5] = and64_const(res[65], DIGIT_MASK); + res[66] = add64(res[66], srli64(res[65], DIGIT_SIZE)); + r[6] = and64_const(res[66], DIGIT_MASK); + res[67] = add64(res[67], srli64(res[66], DIGIT_SIZE)); + r[7] = and64_const(res[67], DIGIT_MASK); + res[68] = add64(res[68], srli64(res[67], DIGIT_SIZE)); + r[8] = and64_const(res[68], DIGIT_MASK); + res[69] = add64(res[69], srli64(res[68], DIGIT_SIZE)); + r[9] = and64_const(res[69], DIGIT_MASK); + res[70] = add64(res[70], srli64(res[69], DIGIT_SIZE)); + r[10] = and64_const(res[70], DIGIT_MASK); + res[71] = add64(res[71], srli64(res[70], DIGIT_SIZE)); + r[11] = and64_const(res[71], DIGIT_MASK); + res[72] = add64(res[72], srli64(res[71], DIGIT_SIZE)); + r[12] = and64_const(res[72], DIGIT_MASK); + res[73] = add64(res[73], srli64(res[72], DIGIT_SIZE)); + r[13] = and64_const(res[73], DIGIT_MASK); + res[74] = add64(res[74], srli64(res[73], DIGIT_SIZE)); + r[14] = and64_const(res[74], DIGIT_MASK); + res[75] = add64(res[75], srli64(res[74], DIGIT_SIZE)); + r[15] = and64_const(res[75], DIGIT_MASK); + res[76] = add64(res[76], srli64(res[75], DIGIT_SIZE)); + r[16] = and64_const(res[76], DIGIT_MASK); + res[77] = add64(res[77], srli64(res[76], DIGIT_SIZE)); + r[17] = and64_const(res[77], DIGIT_MASK); + res[78] = add64(res[78], srli64(res[77], DIGIT_SIZE)); + r[18] = and64_const(res[78], DIGIT_MASK); + res[79] = add64(res[79], srli64(res[78], DIGIT_SIZE)); + r[19] = and64_const(res[79], DIGIT_MASK); + res[80] = add64(res[80], srli64(res[79], DIGIT_SIZE)); + r[20] = and64_const(res[80], DIGIT_MASK); + res[81] = add64(res[81], srli64(res[80], DIGIT_SIZE)); + r[21] = and64_const(res[81], DIGIT_MASK); + res[82] = add64(res[82], srli64(res[81], DIGIT_SIZE)); + r[22] = and64_const(res[82], DIGIT_MASK); + res[83] = add64(res[83], srli64(res[82], DIGIT_SIZE)); + r[23] = and64_const(res[83], DIGIT_MASK); + res[84] = add64(res[84], srli64(res[83], DIGIT_SIZE)); + r[24] = and64_const(res[84], DIGIT_MASK); + res[85] = add64(res[85], srli64(res[84], DIGIT_SIZE)); + r[25] = and64_const(res[85], DIGIT_MASK); + res[86] = add64(res[86], srli64(res[85], DIGIT_SIZE)); + r[26] = and64_const(res[86], DIGIT_MASK); + res[87] = add64(res[87], srli64(res[86], DIGIT_SIZE)); + r[27] = and64_const(res[87], DIGIT_MASK); + res[88] = add64(res[88], srli64(res[87], DIGIT_SIZE)); + r[28] = and64_const(res[88], DIGIT_MASK); + res[89] = add64(res[89], srli64(res[88], DIGIT_SIZE)); + r[29] = and64_const(res[89], DIGIT_MASK); + res[90] = add64(res[90], srli64(res[89], DIGIT_SIZE)); + r[30] = and64_const(res[90], DIGIT_MASK); + res[91] = add64(res[91], srli64(res[90], DIGIT_SIZE)); + r[31] = and64_const(res[91], DIGIT_MASK); + res[92] = add64(res[92], srli64(res[91], DIGIT_SIZE)); + r[32] = and64_const(res[92], DIGIT_MASK); + res[93] = add64(res[93], srli64(res[92], DIGIT_SIZE)); + r[33] = and64_const(res[93], DIGIT_MASK); + res[94] = add64(res[94], srli64(res[93], DIGIT_SIZE)); + r[34] = and64_const(res[94], DIGIT_MASK); + res[95] = add64(res[95], srli64(res[94], DIGIT_SIZE)); + r[35] = and64_const(res[95], DIGIT_MASK); + res[96] = add64(res[96], srli64(res[95], DIGIT_SIZE)); + r[36] = and64_const(res[96], DIGIT_MASK); + res[97] = add64(res[97], srli64(res[96], DIGIT_SIZE)); + r[37] = and64_const(res[97], DIGIT_MASK); + res[98] = add64(res[98], srli64(res[97], DIGIT_SIZE)); + r[38] = and64_const(res[98], DIGIT_MASK); + res[99] = add64(res[99], srli64(res[98], DIGIT_SIZE)); + r[39] = and64_const(res[99], DIGIT_MASK); + res[100] = add64(res[100], srli64(res[99], DIGIT_SIZE)); + r[40] = and64_const(res[100], DIGIT_MASK); + res[101] = add64(res[101], srli64(res[100], DIGIT_SIZE)); + r[41] = and64_const(res[101], DIGIT_MASK); + res[102] = add64(res[102], srli64(res[101], DIGIT_SIZE)); + r[42] = and64_const(res[102], DIGIT_MASK); + res[103] = add64(res[103], srli64(res[102], DIGIT_SIZE)); + r[43] = and64_const(res[103], DIGIT_MASK); + res[104] = add64(res[104], srli64(res[103], DIGIT_SIZE)); + r[44] = and64_const(res[104], DIGIT_MASK); + res[105] = add64(res[105], srli64(res[104], DIGIT_SIZE)); + r[45] = and64_const(res[105], DIGIT_MASK); + res[106] = add64(res[106], srli64(res[105], DIGIT_SIZE)); + r[46] = and64_const(res[106], DIGIT_MASK); + res[107] = add64(res[107], srli64(res[106], DIGIT_SIZE)); + r[47] = and64_const(res[107], DIGIT_MASK); + res[108] = add64(res[108], srli64(res[107], DIGIT_SIZE)); + r[48] = and64_const(res[108], DIGIT_MASK); + res[109] = add64(res[109], srli64(res[108], DIGIT_SIZE)); + r[49] = and64_const(res[109], DIGIT_MASK); + res[110] = add64(res[110], srli64(res[109], DIGIT_SIZE)); + r[50] = and64_const(res[110], DIGIT_MASK); + res[111] = add64(res[111], srli64(res[110], DIGIT_SIZE)); + r[51] = and64_const(res[111], DIGIT_MASK); + res[112] = add64(res[112], srli64(res[111], DIGIT_SIZE)); + r[52] = and64_const(res[112], DIGIT_MASK); + res[113] = add64(res[113], srli64(res[112], DIGIT_SIZE)); + r[53] = and64_const(res[113], DIGIT_MASK); + res[114] = add64(res[114], srli64(res[113], DIGIT_SIZE)); + r[54] = and64_const(res[114], DIGIT_MASK); + res[115] = add64(res[115], srli64(res[114], DIGIT_SIZE)); + r[55] = and64_const(res[115], DIGIT_MASK); + res[116] = add64(res[116], srli64(res[115], DIGIT_SIZE)); + r[56] = and64_const(res[116], DIGIT_MASK); + res[117] = add64(res[117], srli64(res[116], DIGIT_SIZE)); + r[57] = and64_const(res[117], DIGIT_MASK); + res[118] = add64(res[118], srli64(res[117], DIGIT_SIZE)); + r[58] = and64_const(res[118], DIGIT_MASK); + res[119] = add64(res[119], srli64(res[118], DIGIT_SIZE)); + r[59] = and64_const(res[119], DIGIT_MASK); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams52x79_diagonal_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams52x79_diagonal_mb8.c new file mode 100644 index 0000000..dbcc664 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams52x79_diagonal_mb8.c @@ -0,0 +1,7108 @@ +/******************************************************************************* + * Copyright (C) 2019 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include + +#ifdef __GNUC__ +#define ASM(a) __asm__(a); +#else +#define ASM(a) +#endif + +void AMS52x79_diagonal_mb8(int64u *out_mb, const int64u *inpA_mb, + const int64u *inpM_mb, const int64u *k0_mb) { + + __ALIGN64 U64 res[160]; + __ALIGN64 U64 u[80]; + U64 k; + U64 *a = (U64 *)inpA_mb; + U64 *m = (U64 *)inpM_mb; + U64 *r = (U64 *)out_mb; + + k = loadu64((U64 *)k0_mb); + int i; + for (i = 0; i < 160; ++i) + res[i] = get_zero64(); + + // Calculate full square + res[1] = fma52lo(res[1], a[0], a[1]); // Sum(1) + res[2] = fma52hi(res[2], a[0], a[1]); // Sum(1) + res[2] = fma52lo(res[2], a[0], a[2]); // Sum(2) + res[3] = fma52hi(res[3], a[0], a[2]); // Sum(2) + res[3] = fma52lo(res[3], a[1], a[2]); // Sum(3) + res[4] = fma52hi(res[4], a[1], a[2]); // Sum(3) + res[3] = fma52lo(res[3], a[0], a[3]); // Sum(3) + res[4] = fma52hi(res[4], a[0], a[3]); // Sum(3) + res[4] = fma52lo(res[4], a[1], a[3]); // Sum(4) + res[5] = fma52hi(res[5], a[1], a[3]); // Sum(4) + res[5] = fma52lo(res[5], a[2], a[3]); // Sum(5) + res[6] = fma52hi(res[6], a[2], a[3]); // Sum(5) + res[4] = fma52lo(res[4], a[0], a[4]); // Sum(4) + res[5] = fma52hi(res[5], a[0], a[4]); // Sum(4) + res[5] = fma52lo(res[5], a[1], a[4]); // Sum(5) + res[6] = fma52hi(res[6], a[1], a[4]); // Sum(5) + res[6] = fma52lo(res[6], a[2], a[4]); // Sum(6) + res[7] = fma52hi(res[7], a[2], a[4]); // Sum(6) + res[7] = fma52lo(res[7], a[3], a[4]); // Sum(7) + res[8] = fma52hi(res[8], a[3], a[4]); // Sum(7) + res[5] = fma52lo(res[5], a[0], a[5]); // Sum(5) + res[6] = fma52hi(res[6], a[0], a[5]); // Sum(5) + res[6] = fma52lo(res[6], a[1], a[5]); // Sum(6) + res[7] = fma52hi(res[7], a[1], a[5]); // Sum(6) + res[7] = fma52lo(res[7], a[2], a[5]); // Sum(7) + res[8] = fma52hi(res[8], a[2], a[5]); // Sum(7) + res[8] = fma52lo(res[8], a[3], a[5]); // Sum(8) + res[9] = fma52hi(res[9], a[3], a[5]); // Sum(8) + res[9] = fma52lo(res[9], a[4], a[5]); // Sum(9) + res[10] = fma52hi(res[10], a[4], a[5]); // Sum(9) + res[6] = fma52lo(res[6], a[0], a[6]); // Sum(6) + res[7] = fma52hi(res[7], a[0], a[6]); // Sum(6) + res[7] = fma52lo(res[7], a[1], a[6]); // Sum(7) + res[8] = fma52hi(res[8], a[1], a[6]); // Sum(7) + res[8] = fma52lo(res[8], a[2], a[6]); // Sum(8) + res[9] = fma52hi(res[9], a[2], a[6]); // Sum(8) + res[9] = fma52lo(res[9], a[3], a[6]); // Sum(9) + res[10] = fma52hi(res[10], a[3], a[6]); // Sum(9) + res[10] = fma52lo(res[10], a[4], a[6]); // Sum(10) + res[11] = fma52hi(res[11], a[4], a[6]); // Sum(10) + res[11] = fma52lo(res[11], a[5], a[6]); // Sum(11) + res[12] = fma52hi(res[12], a[5], a[6]); // Sum(11) + res[7] = fma52lo(res[7], a[0], a[7]); // Sum(7) + res[8] = fma52hi(res[8], a[0], a[7]); // Sum(7) + res[8] = fma52lo(res[8], a[1], a[7]); // Sum(8) + res[9] = fma52hi(res[9], a[1], a[7]); // Sum(8) + res[9] = fma52lo(res[9], a[2], a[7]); // Sum(9) + res[10] = fma52hi(res[10], a[2], a[7]); // Sum(9) + res[10] = fma52lo(res[10], a[3], a[7]); // Sum(10) + res[11] = fma52hi(res[11], a[3], a[7]); // Sum(10) + res[11] = fma52lo(res[11], a[4], a[7]); // Sum(11) + res[12] = fma52hi(res[12], a[4], a[7]); // Sum(11) + res[8] = fma52lo(res[8], a[0], a[8]); // Sum(8) + res[9] = fma52hi(res[9], a[0], a[8]); // Sum(8) + res[9] = fma52lo(res[9], a[1], a[8]); // Sum(9) + res[10] = fma52hi(res[10], a[1], a[8]); // Sum(9) + res[10] = fma52lo(res[10], a[2], a[8]); // Sum(10) + res[11] = fma52hi(res[11], a[2], a[8]); // Sum(10) + res[11] = fma52lo(res[11], a[3], a[8]); // Sum(11) + res[12] = fma52hi(res[12], a[3], a[8]); // Sum(11) + res[9] = fma52lo(res[9], a[0], a[9]); // Sum(9) + res[10] = fma52hi(res[10], a[0], a[9]); // Sum(9) + res[10] = fma52lo(res[10], a[1], a[9]); // Sum(10) + res[11] = fma52hi(res[11], a[1], a[9]); // Sum(10) + res[11] = fma52lo(res[11], a[2], a[9]); // Sum(11) + res[12] = fma52hi(res[12], a[2], a[9]); // Sum(11) + res[10] = fma52lo(res[10], a[0], a[10]); // Sum(10) + res[11] = fma52hi(res[11], a[0], a[10]); // Sum(10) + res[11] = fma52lo(res[11], a[1], a[10]); // Sum(11) + res[12] = fma52hi(res[12], a[1], a[10]); // Sum(11) + res[11] = fma52lo(res[11], a[0], a[11]); // Sum(11) + res[12] = fma52hi(res[12], a[0], a[11]); // Sum(11) + res[0] = add64(res[0], res[0]); // Double(0) + res[1] = add64(res[1], res[1]); // Double(1) + res[2] = add64(res[2], res[2]); // Double(2) + res[3] = add64(res[3], res[3]); // Double(3) + res[4] = add64(res[4], res[4]); // Double(4) + res[5] = add64(res[5], res[5]); // Double(5) + res[6] = add64(res[6], res[6]); // Double(6) + res[7] = add64(res[7], res[7]); // Double(7) + res[8] = add64(res[8], res[8]); // Double(8) + res[9] = add64(res[9], res[9]); // Double(9) + res[10] = add64(res[10], res[10]); // Double(10) + res[11] = add64(res[11], res[11]); // Double(11) + res[0] = fma52lo(res[0], a[0], a[0]); // Add sqr(0) + res[1] = fma52hi(res[1], a[0], a[0]); // Add sqr(0) + res[2] = fma52lo(res[2], a[1], a[1]); // Add sqr(2) + res[3] = fma52hi(res[3], a[1], a[1]); // Add sqr(2) + res[4] = fma52lo(res[4], a[2], a[2]); // Add sqr(4) + res[5] = fma52hi(res[5], a[2], a[2]); // Add sqr(4) + res[6] = fma52lo(res[6], a[3], a[3]); // Add sqr(6) + res[7] = fma52hi(res[7], a[3], a[3]); // Add sqr(6) + res[8] = fma52lo(res[8], a[4], a[4]); // Add sqr(8) + res[9] = fma52hi(res[9], a[4], a[4]); // Add sqr(8) + res[10] = fma52lo(res[10], a[5], a[5]); // Add sqr(10) + res[11] = fma52hi(res[11], a[5], a[5]); // Add sqr(10) + res[12] = fma52lo(res[12], a[5], a[7]); // Sum(12) + res[13] = fma52hi(res[13], a[5], a[7]); // Sum(12) + res[13] = fma52lo(res[13], a[6], a[7]); // Sum(13) + res[14] = fma52hi(res[14], a[6], a[7]); // Sum(13) + res[12] = fma52lo(res[12], a[4], a[8]); // Sum(12) + res[13] = fma52hi(res[13], a[4], a[8]); // Sum(12) + res[13] = fma52lo(res[13], a[5], a[8]); // Sum(13) + res[14] = fma52hi(res[14], a[5], a[8]); // Sum(13) + res[14] = fma52lo(res[14], a[6], a[8]); // Sum(14) + res[15] = fma52hi(res[15], a[6], a[8]); // Sum(14) + res[15] = fma52lo(res[15], a[7], a[8]); // Sum(15) + res[16] = fma52hi(res[16], a[7], a[8]); // Sum(15) + res[12] = fma52lo(res[12], a[3], a[9]); // Sum(12) + res[13] = fma52hi(res[13], a[3], a[9]); // Sum(12) + res[13] = fma52lo(res[13], a[4], a[9]); // Sum(13) + res[14] = fma52hi(res[14], a[4], a[9]); // Sum(13) + res[14] = fma52lo(res[14], a[5], a[9]); // Sum(14) + res[15] = fma52hi(res[15], a[5], a[9]); // Sum(14) + res[15] = fma52lo(res[15], a[6], a[9]); // Sum(15) + res[16] = fma52hi(res[16], a[6], a[9]); // Sum(15) + res[16] = fma52lo(res[16], a[7], a[9]); // Sum(16) + res[17] = fma52hi(res[17], a[7], a[9]); // Sum(16) + res[17] = fma52lo(res[17], a[8], a[9]); // Sum(17) + res[18] = fma52hi(res[18], a[8], a[9]); // Sum(17) + res[12] = fma52lo(res[12], a[2], a[10]); // Sum(12) + res[13] = fma52hi(res[13], a[2], a[10]); // Sum(12) + res[13] = fma52lo(res[13], a[3], a[10]); // Sum(13) + res[14] = fma52hi(res[14], a[3], a[10]); // Sum(13) + res[14] = fma52lo(res[14], a[4], a[10]); // Sum(14) + res[15] = fma52hi(res[15], a[4], a[10]); // Sum(14) + res[15] = fma52lo(res[15], a[5], a[10]); // Sum(15) + res[16] = fma52hi(res[16], a[5], a[10]); // Sum(15) + res[16] = fma52lo(res[16], a[6], a[10]); // Sum(16) + res[17] = fma52hi(res[17], a[6], a[10]); // Sum(16) + res[17] = fma52lo(res[17], a[7], a[10]); // Sum(17) + res[18] = fma52hi(res[18], a[7], a[10]); // Sum(17) + res[18] = fma52lo(res[18], a[8], a[10]); // Sum(18) + res[19] = fma52hi(res[19], a[8], a[10]); // Sum(18) + res[19] = fma52lo(res[19], a[9], a[10]); // Sum(19) + res[20] = fma52hi(res[20], a[9], a[10]); // Sum(19) + res[12] = fma52lo(res[12], a[1], a[11]); // Sum(12) + res[13] = fma52hi(res[13], a[1], a[11]); // Sum(12) + res[13] = fma52lo(res[13], a[2], a[11]); // Sum(13) + res[14] = fma52hi(res[14], a[2], a[11]); // Sum(13) + res[14] = fma52lo(res[14], a[3], a[11]); // Sum(14) + res[15] = fma52hi(res[15], a[3], a[11]); // Sum(14) + res[15] = fma52lo(res[15], a[4], a[11]); // Sum(15) + res[16] = fma52hi(res[16], a[4], a[11]); // Sum(15) + res[16] = fma52lo(res[16], a[5], a[11]); // Sum(16) + res[17] = fma52hi(res[17], a[5], a[11]); // Sum(16) + res[17] = fma52lo(res[17], a[6], a[11]); // Sum(17) + res[18] = fma52hi(res[18], a[6], a[11]); // Sum(17) + res[18] = fma52lo(res[18], a[7], a[11]); // Sum(18) + res[19] = fma52hi(res[19], a[7], a[11]); // Sum(18) + res[19] = fma52lo(res[19], a[8], a[11]); // Sum(19) + res[20] = fma52hi(res[20], a[8], a[11]); // Sum(19) + res[20] = fma52lo(res[20], a[9], a[11]); // Sum(20) + res[21] = fma52hi(res[21], a[9], a[11]); // Sum(20) + res[21] = fma52lo(res[21], a[10], a[11]); // Sum(21) + res[22] = fma52hi(res[22], a[10], a[11]); // Sum(21) + res[12] = fma52lo(res[12], a[0], a[12]); // Sum(12) + res[13] = fma52hi(res[13], a[0], a[12]); // Sum(12) + res[13] = fma52lo(res[13], a[1], a[12]); // Sum(13) + res[14] = fma52hi(res[14], a[1], a[12]); // Sum(13) + res[14] = fma52lo(res[14], a[2], a[12]); // Sum(14) + res[15] = fma52hi(res[15], a[2], a[12]); // Sum(14) + res[15] = fma52lo(res[15], a[3], a[12]); // Sum(15) + res[16] = fma52hi(res[16], a[3], a[12]); // Sum(15) + res[16] = fma52lo(res[16], a[4], a[12]); // Sum(16) + res[17] = fma52hi(res[17], a[4], a[12]); // Sum(16) + res[17] = fma52lo(res[17], a[5], a[12]); // Sum(17) + res[18] = fma52hi(res[18], a[5], a[12]); // Sum(17) + res[18] = fma52lo(res[18], a[6], a[12]); // Sum(18) + res[19] = fma52hi(res[19], a[6], a[12]); // Sum(18) + res[19] = fma52lo(res[19], a[7], a[12]); // Sum(19) + res[20] = fma52hi(res[20], a[7], a[12]); // Sum(19) + res[20] = fma52lo(res[20], a[8], a[12]); // Sum(20) + res[21] = fma52hi(res[21], a[8], a[12]); // Sum(20) + res[21] = fma52lo(res[21], a[9], a[12]); // Sum(21) + res[22] = fma52hi(res[22], a[9], a[12]); // Sum(21) + res[22] = fma52lo(res[22], a[10], a[12]); // Sum(22) + res[23] = fma52hi(res[23], a[10], a[12]); // Sum(22) + res[23] = fma52lo(res[23], a[11], a[12]); // Sum(23) + res[24] = fma52hi(res[24], a[11], a[12]); // Sum(23) + res[13] = fma52lo(res[13], a[0], a[13]); // Sum(13) + res[14] = fma52hi(res[14], a[0], a[13]); // Sum(13) + res[14] = fma52lo(res[14], a[1], a[13]); // Sum(14) + res[15] = fma52hi(res[15], a[1], a[13]); // Sum(14) + res[15] = fma52lo(res[15], a[2], a[13]); // Sum(15) + res[16] = fma52hi(res[16], a[2], a[13]); // Sum(15) + res[16] = fma52lo(res[16], a[3], a[13]); // Sum(16) + res[17] = fma52hi(res[17], a[3], a[13]); // Sum(16) + res[17] = fma52lo(res[17], a[4], a[13]); // Sum(17) + res[18] = fma52hi(res[18], a[4], a[13]); // Sum(17) + res[18] = fma52lo(res[18], a[5], a[13]); // Sum(18) + res[19] = fma52hi(res[19], a[5], a[13]); // Sum(18) + res[19] = fma52lo(res[19], a[6], a[13]); // Sum(19) + res[20] = fma52hi(res[20], a[6], a[13]); // Sum(19) + res[20] = fma52lo(res[20], a[7], a[13]); // Sum(20) + res[21] = fma52hi(res[21], a[7], a[13]); // Sum(20) + res[21] = fma52lo(res[21], a[8], a[13]); // Sum(21) + res[22] = fma52hi(res[22], a[8], a[13]); // Sum(21) + res[22] = fma52lo(res[22], a[9], a[13]); // Sum(22) + res[23] = fma52hi(res[23], a[9], a[13]); // Sum(22) + res[23] = fma52lo(res[23], a[10], a[13]); // Sum(23) + res[24] = fma52hi(res[24], a[10], a[13]); // Sum(23) + res[14] = fma52lo(res[14], a[0], a[14]); // Sum(14) + res[15] = fma52hi(res[15], a[0], a[14]); // Sum(14) + res[15] = fma52lo(res[15], a[1], a[14]); // Sum(15) + res[16] = fma52hi(res[16], a[1], a[14]); // Sum(15) + res[16] = fma52lo(res[16], a[2], a[14]); // Sum(16) + res[17] = fma52hi(res[17], a[2], a[14]); // Sum(16) + res[17] = fma52lo(res[17], a[3], a[14]); // Sum(17) + res[18] = fma52hi(res[18], a[3], a[14]); // Sum(17) + res[18] = fma52lo(res[18], a[4], a[14]); // Sum(18) + res[19] = fma52hi(res[19], a[4], a[14]); // Sum(18) + res[19] = fma52lo(res[19], a[5], a[14]); // Sum(19) + res[20] = fma52hi(res[20], a[5], a[14]); // Sum(19) + res[20] = fma52lo(res[20], a[6], a[14]); // Sum(20) + res[21] = fma52hi(res[21], a[6], a[14]); // Sum(20) + res[21] = fma52lo(res[21], a[7], a[14]); // Sum(21) + res[22] = fma52hi(res[22], a[7], a[14]); // Sum(21) + res[22] = fma52lo(res[22], a[8], a[14]); // Sum(22) + res[23] = fma52hi(res[23], a[8], a[14]); // Sum(22) + res[23] = fma52lo(res[23], a[9], a[14]); // Sum(23) + res[24] = fma52hi(res[24], a[9], a[14]); // Sum(23) + res[15] = fma52lo(res[15], a[0], a[15]); // Sum(15) + res[16] = fma52hi(res[16], a[0], a[15]); // Sum(15) + res[16] = fma52lo(res[16], a[1], a[15]); // Sum(16) + res[17] = fma52hi(res[17], a[1], a[15]); // Sum(16) + res[17] = fma52lo(res[17], a[2], a[15]); // Sum(17) + res[18] = fma52hi(res[18], a[2], a[15]); // Sum(17) + res[18] = fma52lo(res[18], a[3], a[15]); // Sum(18) + res[19] = fma52hi(res[19], a[3], a[15]); // Sum(18) + res[19] = fma52lo(res[19], a[4], a[15]); // Sum(19) + res[20] = fma52hi(res[20], a[4], a[15]); // Sum(19) + res[20] = fma52lo(res[20], a[5], a[15]); // Sum(20) + res[21] = fma52hi(res[21], a[5], a[15]); // Sum(20) + res[21] = fma52lo(res[21], a[6], a[15]); // Sum(21) + res[22] = fma52hi(res[22], a[6], a[15]); // Sum(21) + res[22] = fma52lo(res[22], a[7], a[15]); // Sum(22) + res[23] = fma52hi(res[23], a[7], a[15]); // Sum(22) + res[23] = fma52lo(res[23], a[8], a[15]); // Sum(23) + res[24] = fma52hi(res[24], a[8], a[15]); // Sum(23) + res[16] = fma52lo(res[16], a[0], a[16]); // Sum(16) + res[17] = fma52hi(res[17], a[0], a[16]); // Sum(16) + res[17] = fma52lo(res[17], a[1], a[16]); // Sum(17) + res[18] = fma52hi(res[18], a[1], a[16]); // Sum(17) + res[18] = fma52lo(res[18], a[2], a[16]); // Sum(18) + res[19] = fma52hi(res[19], a[2], a[16]); // Sum(18) + res[19] = fma52lo(res[19], a[3], a[16]); // Sum(19) + res[20] = fma52hi(res[20], a[3], a[16]); // Sum(19) + res[20] = fma52lo(res[20], a[4], a[16]); // Sum(20) + res[21] = fma52hi(res[21], a[4], a[16]); // Sum(20) + res[21] = fma52lo(res[21], a[5], a[16]); // Sum(21) + res[22] = fma52hi(res[22], a[5], a[16]); // Sum(21) + res[22] = fma52lo(res[22], a[6], a[16]); // Sum(22) + res[23] = fma52hi(res[23], a[6], a[16]); // Sum(22) + res[23] = fma52lo(res[23], a[7], a[16]); // Sum(23) + res[24] = fma52hi(res[24], a[7], a[16]); // Sum(23) + res[17] = fma52lo(res[17], a[0], a[17]); // Sum(17) + res[18] = fma52hi(res[18], a[0], a[17]); // Sum(17) + res[18] = fma52lo(res[18], a[1], a[17]); // Sum(18) + res[19] = fma52hi(res[19], a[1], a[17]); // Sum(18) + res[19] = fma52lo(res[19], a[2], a[17]); // Sum(19) + res[20] = fma52hi(res[20], a[2], a[17]); // Sum(19) + res[20] = fma52lo(res[20], a[3], a[17]); // Sum(20) + res[21] = fma52hi(res[21], a[3], a[17]); // Sum(20) + res[21] = fma52lo(res[21], a[4], a[17]); // Sum(21) + res[22] = fma52hi(res[22], a[4], a[17]); // Sum(21) + res[22] = fma52lo(res[22], a[5], a[17]); // Sum(22) + res[23] = fma52hi(res[23], a[5], a[17]); // Sum(22) + res[23] = fma52lo(res[23], a[6], a[17]); // Sum(23) + res[24] = fma52hi(res[24], a[6], a[17]); // Sum(23) + res[18] = fma52lo(res[18], a[0], a[18]); // Sum(18) + res[19] = fma52hi(res[19], a[0], a[18]); // Sum(18) + res[19] = fma52lo(res[19], a[1], a[18]); // Sum(19) + res[20] = fma52hi(res[20], a[1], a[18]); // Sum(19) + res[20] = fma52lo(res[20], a[2], a[18]); // Sum(20) + res[21] = fma52hi(res[21], a[2], a[18]); // Sum(20) + res[21] = fma52lo(res[21], a[3], a[18]); // Sum(21) + res[22] = fma52hi(res[22], a[3], a[18]); // Sum(21) + res[22] = fma52lo(res[22], a[4], a[18]); // Sum(22) + res[23] = fma52hi(res[23], a[4], a[18]); // Sum(22) + res[23] = fma52lo(res[23], a[5], a[18]); // Sum(23) + res[24] = fma52hi(res[24], a[5], a[18]); // Sum(23) + res[19] = fma52lo(res[19], a[0], a[19]); // Sum(19) + res[20] = fma52hi(res[20], a[0], a[19]); // Sum(19) + res[20] = fma52lo(res[20], a[1], a[19]); // Sum(20) + res[21] = fma52hi(res[21], a[1], a[19]); // Sum(20) + res[21] = fma52lo(res[21], a[2], a[19]); // Sum(21) + res[22] = fma52hi(res[22], a[2], a[19]); // Sum(21) + res[22] = fma52lo(res[22], a[3], a[19]); // Sum(22) + res[23] = fma52hi(res[23], a[3], a[19]); // Sum(22) + res[23] = fma52lo(res[23], a[4], a[19]); // Sum(23) + res[24] = fma52hi(res[24], a[4], a[19]); // Sum(23) + res[20] = fma52lo(res[20], a[0], a[20]); // Sum(20) + res[21] = fma52hi(res[21], a[0], a[20]); // Sum(20) + res[21] = fma52lo(res[21], a[1], a[20]); // Sum(21) + res[22] = fma52hi(res[22], a[1], a[20]); // Sum(21) + res[22] = fma52lo(res[22], a[2], a[20]); // Sum(22) + res[23] = fma52hi(res[23], a[2], a[20]); // Sum(22) + res[23] = fma52lo(res[23], a[3], a[20]); // Sum(23) + res[24] = fma52hi(res[24], a[3], a[20]); // Sum(23) + res[21] = fma52lo(res[21], a[0], a[21]); // Sum(21) + res[22] = fma52hi(res[22], a[0], a[21]); // Sum(21) + res[22] = fma52lo(res[22], a[1], a[21]); // Sum(22) + res[23] = fma52hi(res[23], a[1], a[21]); // Sum(22) + res[23] = fma52lo(res[23], a[2], a[21]); // Sum(23) + res[24] = fma52hi(res[24], a[2], a[21]); // Sum(23) + res[22] = fma52lo(res[22], a[0], a[22]); // Sum(22) + res[23] = fma52hi(res[23], a[0], a[22]); // Sum(22) + res[23] = fma52lo(res[23], a[1], a[22]); // Sum(23) + res[24] = fma52hi(res[24], a[1], a[22]); // Sum(23) + res[23] = fma52lo(res[23], a[0], a[23]); // Sum(23) + res[24] = fma52hi(res[24], a[0], a[23]); // Sum(23) + res[12] = add64(res[12], res[12]); // Double(12) + res[13] = add64(res[13], res[13]); // Double(13) + res[14] = add64(res[14], res[14]); // Double(14) + res[15] = add64(res[15], res[15]); // Double(15) + res[16] = add64(res[16], res[16]); // Double(16) + res[17] = add64(res[17], res[17]); // Double(17) + res[18] = add64(res[18], res[18]); // Double(18) + res[19] = add64(res[19], res[19]); // Double(19) + res[20] = add64(res[20], res[20]); // Double(20) + res[21] = add64(res[21], res[21]); // Double(21) + res[22] = add64(res[22], res[22]); // Double(22) + res[23] = add64(res[23], res[23]); // Double(23) + res[12] = fma52lo(res[12], a[6], a[6]); // Add sqr(12) + res[13] = fma52hi(res[13], a[6], a[6]); // Add sqr(12) + res[14] = fma52lo(res[14], a[7], a[7]); // Add sqr(14) + res[15] = fma52hi(res[15], a[7], a[7]); // Add sqr(14) + res[16] = fma52lo(res[16], a[8], a[8]); // Add sqr(16) + res[17] = fma52hi(res[17], a[8], a[8]); // Add sqr(16) + res[18] = fma52lo(res[18], a[9], a[9]); // Add sqr(18) + res[19] = fma52hi(res[19], a[9], a[9]); // Add sqr(18) + res[20] = fma52lo(res[20], a[10], a[10]); // Add sqr(20) + res[21] = fma52hi(res[21], a[10], a[10]); // Add sqr(20) + res[22] = fma52lo(res[22], a[11], a[11]); // Add sqr(22) + res[23] = fma52hi(res[23], a[11], a[11]); // Add sqr(22) + res[24] = fma52lo(res[24], a[11], a[13]); // Sum(24) + res[25] = fma52hi(res[25], a[11], a[13]); // Sum(24) + res[25] = fma52lo(res[25], a[12], a[13]); // Sum(25) + res[26] = fma52hi(res[26], a[12], a[13]); // Sum(25) + res[24] = fma52lo(res[24], a[10], a[14]); // Sum(24) + res[25] = fma52hi(res[25], a[10], a[14]); // Sum(24) + res[25] = fma52lo(res[25], a[11], a[14]); // Sum(25) + res[26] = fma52hi(res[26], a[11], a[14]); // Sum(25) + res[26] = fma52lo(res[26], a[12], a[14]); // Sum(26) + res[27] = fma52hi(res[27], a[12], a[14]); // Sum(26) + res[27] = fma52lo(res[27], a[13], a[14]); // Sum(27) + res[28] = fma52hi(res[28], a[13], a[14]); // Sum(27) + res[24] = fma52lo(res[24], a[9], a[15]); // Sum(24) + res[25] = fma52hi(res[25], a[9], a[15]); // Sum(24) + res[25] = fma52lo(res[25], a[10], a[15]); // Sum(25) + res[26] = fma52hi(res[26], a[10], a[15]); // Sum(25) + res[26] = fma52lo(res[26], a[11], a[15]); // Sum(26) + res[27] = fma52hi(res[27], a[11], a[15]); // Sum(26) + res[27] = fma52lo(res[27], a[12], a[15]); // Sum(27) + res[28] = fma52hi(res[28], a[12], a[15]); // Sum(27) + res[28] = fma52lo(res[28], a[13], a[15]); // Sum(28) + res[29] = fma52hi(res[29], a[13], a[15]); // Sum(28) + res[29] = fma52lo(res[29], a[14], a[15]); // Sum(29) + res[30] = fma52hi(res[30], a[14], a[15]); // Sum(29) + res[24] = fma52lo(res[24], a[8], a[16]); // Sum(24) + res[25] = fma52hi(res[25], a[8], a[16]); // Sum(24) + res[25] = fma52lo(res[25], a[9], a[16]); // Sum(25) + res[26] = fma52hi(res[26], a[9], a[16]); // Sum(25) + res[26] = fma52lo(res[26], a[10], a[16]); // Sum(26) + res[27] = fma52hi(res[27], a[10], a[16]); // Sum(26) + res[27] = fma52lo(res[27], a[11], a[16]); // Sum(27) + res[28] = fma52hi(res[28], a[11], a[16]); // Sum(27) + res[28] = fma52lo(res[28], a[12], a[16]); // Sum(28) + res[29] = fma52hi(res[29], a[12], a[16]); // Sum(28) + res[29] = fma52lo(res[29], a[13], a[16]); // Sum(29) + res[30] = fma52hi(res[30], a[13], a[16]); // Sum(29) + res[30] = fma52lo(res[30], a[14], a[16]); // Sum(30) + res[31] = fma52hi(res[31], a[14], a[16]); // Sum(30) + res[31] = fma52lo(res[31], a[15], a[16]); // Sum(31) + res[32] = fma52hi(res[32], a[15], a[16]); // Sum(31) + res[24] = fma52lo(res[24], a[7], a[17]); // Sum(24) + res[25] = fma52hi(res[25], a[7], a[17]); // Sum(24) + res[25] = fma52lo(res[25], a[8], a[17]); // Sum(25) + res[26] = fma52hi(res[26], a[8], a[17]); // Sum(25) + res[26] = fma52lo(res[26], a[9], a[17]); // Sum(26) + res[27] = fma52hi(res[27], a[9], a[17]); // Sum(26) + res[27] = fma52lo(res[27], a[10], a[17]); // Sum(27) + res[28] = fma52hi(res[28], a[10], a[17]); // Sum(27) + res[28] = fma52lo(res[28], a[11], a[17]); // Sum(28) + res[29] = fma52hi(res[29], a[11], a[17]); // Sum(28) + res[29] = fma52lo(res[29], a[12], a[17]); // Sum(29) + res[30] = fma52hi(res[30], a[12], a[17]); // Sum(29) + res[30] = fma52lo(res[30], a[13], a[17]); // Sum(30) + res[31] = fma52hi(res[31], a[13], a[17]); // Sum(30) + res[31] = fma52lo(res[31], a[14], a[17]); // Sum(31) + res[32] = fma52hi(res[32], a[14], a[17]); // Sum(31) + res[32] = fma52lo(res[32], a[15], a[17]); // Sum(32) + res[33] = fma52hi(res[33], a[15], a[17]); // Sum(32) + res[33] = fma52lo(res[33], a[16], a[17]); // Sum(33) + res[34] = fma52hi(res[34], a[16], a[17]); // Sum(33) + res[24] = fma52lo(res[24], a[6], a[18]); // Sum(24) + res[25] = fma52hi(res[25], a[6], a[18]); // Sum(24) + res[25] = fma52lo(res[25], a[7], a[18]); // Sum(25) + res[26] = fma52hi(res[26], a[7], a[18]); // Sum(25) + res[26] = fma52lo(res[26], a[8], a[18]); // Sum(26) + res[27] = fma52hi(res[27], a[8], a[18]); // Sum(26) + res[27] = fma52lo(res[27], a[9], a[18]); // Sum(27) + res[28] = fma52hi(res[28], a[9], a[18]); // Sum(27) + res[28] = fma52lo(res[28], a[10], a[18]); // Sum(28) + res[29] = fma52hi(res[29], a[10], a[18]); // Sum(28) + res[29] = fma52lo(res[29], a[11], a[18]); // Sum(29) + res[30] = fma52hi(res[30], a[11], a[18]); // Sum(29) + res[30] = fma52lo(res[30], a[12], a[18]); // Sum(30) + res[31] = fma52hi(res[31], a[12], a[18]); // Sum(30) + res[31] = fma52lo(res[31], a[13], a[18]); // Sum(31) + res[32] = fma52hi(res[32], a[13], a[18]); // Sum(31) + res[32] = fma52lo(res[32], a[14], a[18]); // Sum(32) + res[33] = fma52hi(res[33], a[14], a[18]); // Sum(32) + res[33] = fma52lo(res[33], a[15], a[18]); // Sum(33) + res[34] = fma52hi(res[34], a[15], a[18]); // Sum(33) + res[34] = fma52lo(res[34], a[16], a[18]); // Sum(34) + res[35] = fma52hi(res[35], a[16], a[18]); // Sum(34) + res[35] = fma52lo(res[35], a[17], a[18]); // Sum(35) + res[36] = fma52hi(res[36], a[17], a[18]); // Sum(35) + res[24] = fma52lo(res[24], a[5], a[19]); // Sum(24) + res[25] = fma52hi(res[25], a[5], a[19]); // Sum(24) + res[25] = fma52lo(res[25], a[6], a[19]); // Sum(25) + res[26] = fma52hi(res[26], a[6], a[19]); // Sum(25) + res[26] = fma52lo(res[26], a[7], a[19]); // Sum(26) + res[27] = fma52hi(res[27], a[7], a[19]); // Sum(26) + res[27] = fma52lo(res[27], a[8], a[19]); // Sum(27) + res[28] = fma52hi(res[28], a[8], a[19]); // Sum(27) + res[28] = fma52lo(res[28], a[9], a[19]); // Sum(28) + res[29] = fma52hi(res[29], a[9], a[19]); // Sum(28) + res[29] = fma52lo(res[29], a[10], a[19]); // Sum(29) + res[30] = fma52hi(res[30], a[10], a[19]); // Sum(29) + res[30] = fma52lo(res[30], a[11], a[19]); // Sum(30) + res[31] = fma52hi(res[31], a[11], a[19]); // Sum(30) + res[31] = fma52lo(res[31], a[12], a[19]); // Sum(31) + res[32] = fma52hi(res[32], a[12], a[19]); // Sum(31) + res[32] = fma52lo(res[32], a[13], a[19]); // Sum(32) + res[33] = fma52hi(res[33], a[13], a[19]); // Sum(32) + res[33] = fma52lo(res[33], a[14], a[19]); // Sum(33) + res[34] = fma52hi(res[34], a[14], a[19]); // Sum(33) + res[34] = fma52lo(res[34], a[15], a[19]); // Sum(34) + res[35] = fma52hi(res[35], a[15], a[19]); // Sum(34) + res[35] = fma52lo(res[35], a[16], a[19]); // Sum(35) + res[36] = fma52hi(res[36], a[16], a[19]); // Sum(35) + res[24] = fma52lo(res[24], a[4], a[20]); // Sum(24) + res[25] = fma52hi(res[25], a[4], a[20]); // Sum(24) + res[25] = fma52lo(res[25], a[5], a[20]); // Sum(25) + res[26] = fma52hi(res[26], a[5], a[20]); // Sum(25) + res[26] = fma52lo(res[26], a[6], a[20]); // Sum(26) + res[27] = fma52hi(res[27], a[6], a[20]); // Sum(26) + res[27] = fma52lo(res[27], a[7], a[20]); // Sum(27) + res[28] = fma52hi(res[28], a[7], a[20]); // Sum(27) + res[28] = fma52lo(res[28], a[8], a[20]); // Sum(28) + res[29] = fma52hi(res[29], a[8], a[20]); // Sum(28) + res[29] = fma52lo(res[29], a[9], a[20]); // Sum(29) + res[30] = fma52hi(res[30], a[9], a[20]); // Sum(29) + res[30] = fma52lo(res[30], a[10], a[20]); // Sum(30) + res[31] = fma52hi(res[31], a[10], a[20]); // Sum(30) + res[31] = fma52lo(res[31], a[11], a[20]); // Sum(31) + res[32] = fma52hi(res[32], a[11], a[20]); // Sum(31) + res[32] = fma52lo(res[32], a[12], a[20]); // Sum(32) + res[33] = fma52hi(res[33], a[12], a[20]); // Sum(32) + res[33] = fma52lo(res[33], a[13], a[20]); // Sum(33) + res[34] = fma52hi(res[34], a[13], a[20]); // Sum(33) + res[34] = fma52lo(res[34], a[14], a[20]); // Sum(34) + res[35] = fma52hi(res[35], a[14], a[20]); // Sum(34) + res[35] = fma52lo(res[35], a[15], a[20]); // Sum(35) + res[36] = fma52hi(res[36], a[15], a[20]); // Sum(35) + res[24] = fma52lo(res[24], a[3], a[21]); // Sum(24) + res[25] = fma52hi(res[25], a[3], a[21]); // Sum(24) + res[25] = fma52lo(res[25], a[4], a[21]); // Sum(25) + res[26] = fma52hi(res[26], a[4], a[21]); // Sum(25) + res[26] = fma52lo(res[26], a[5], a[21]); // Sum(26) + res[27] = fma52hi(res[27], a[5], a[21]); // Sum(26) + res[27] = fma52lo(res[27], a[6], a[21]); // Sum(27) + res[28] = fma52hi(res[28], a[6], a[21]); // Sum(27) + res[28] = fma52lo(res[28], a[7], a[21]); // Sum(28) + res[29] = fma52hi(res[29], a[7], a[21]); // Sum(28) + res[29] = fma52lo(res[29], a[8], a[21]); // Sum(29) + res[30] = fma52hi(res[30], a[8], a[21]); // Sum(29) + res[30] = fma52lo(res[30], a[9], a[21]); // Sum(30) + res[31] = fma52hi(res[31], a[9], a[21]); // Sum(30) + res[31] = fma52lo(res[31], a[10], a[21]); // Sum(31) + res[32] = fma52hi(res[32], a[10], a[21]); // Sum(31) + res[32] = fma52lo(res[32], a[11], a[21]); // Sum(32) + res[33] = fma52hi(res[33], a[11], a[21]); // Sum(32) + res[33] = fma52lo(res[33], a[12], a[21]); // Sum(33) + res[34] = fma52hi(res[34], a[12], a[21]); // Sum(33) + res[34] = fma52lo(res[34], a[13], a[21]); // Sum(34) + res[35] = fma52hi(res[35], a[13], a[21]); // Sum(34) + res[35] = fma52lo(res[35], a[14], a[21]); // Sum(35) + res[36] = fma52hi(res[36], a[14], a[21]); // Sum(35) + res[24] = fma52lo(res[24], a[2], a[22]); // Sum(24) + res[25] = fma52hi(res[25], a[2], a[22]); // Sum(24) + res[25] = fma52lo(res[25], a[3], a[22]); // Sum(25) + res[26] = fma52hi(res[26], a[3], a[22]); // Sum(25) + res[26] = fma52lo(res[26], a[4], a[22]); // Sum(26) + res[27] = fma52hi(res[27], a[4], a[22]); // Sum(26) + res[27] = fma52lo(res[27], a[5], a[22]); // Sum(27) + res[28] = fma52hi(res[28], a[5], a[22]); // Sum(27) + res[28] = fma52lo(res[28], a[6], a[22]); // Sum(28) + res[29] = fma52hi(res[29], a[6], a[22]); // Sum(28) + res[29] = fma52lo(res[29], a[7], a[22]); // Sum(29) + res[30] = fma52hi(res[30], a[7], a[22]); // Sum(29) + res[30] = fma52lo(res[30], a[8], a[22]); // Sum(30) + res[31] = fma52hi(res[31], a[8], a[22]); // Sum(30) + res[31] = fma52lo(res[31], a[9], a[22]); // Sum(31) + res[32] = fma52hi(res[32], a[9], a[22]); // Sum(31) + res[32] = fma52lo(res[32], a[10], a[22]); // Sum(32) + res[33] = fma52hi(res[33], a[10], a[22]); // Sum(32) + res[33] = fma52lo(res[33], a[11], a[22]); // Sum(33) + res[34] = fma52hi(res[34], a[11], a[22]); // Sum(33) + res[34] = fma52lo(res[34], a[12], a[22]); // Sum(34) + res[35] = fma52hi(res[35], a[12], a[22]); // Sum(34) + res[35] = fma52lo(res[35], a[13], a[22]); // Sum(35) + res[36] = fma52hi(res[36], a[13], a[22]); // Sum(35) + res[24] = fma52lo(res[24], a[1], a[23]); // Sum(24) + res[25] = fma52hi(res[25], a[1], a[23]); // Sum(24) + res[25] = fma52lo(res[25], a[2], a[23]); // Sum(25) + res[26] = fma52hi(res[26], a[2], a[23]); // Sum(25) + res[26] = fma52lo(res[26], a[3], a[23]); // Sum(26) + res[27] = fma52hi(res[27], a[3], a[23]); // Sum(26) + res[27] = fma52lo(res[27], a[4], a[23]); // Sum(27) + res[28] = fma52hi(res[28], a[4], a[23]); // Sum(27) + res[28] = fma52lo(res[28], a[5], a[23]); // Sum(28) + res[29] = fma52hi(res[29], a[5], a[23]); // Sum(28) + res[29] = fma52lo(res[29], a[6], a[23]); // Sum(29) + res[30] = fma52hi(res[30], a[6], a[23]); // Sum(29) + res[30] = fma52lo(res[30], a[7], a[23]); // Sum(30) + res[31] = fma52hi(res[31], a[7], a[23]); // Sum(30) + res[31] = fma52lo(res[31], a[8], a[23]); // Sum(31) + res[32] = fma52hi(res[32], a[8], a[23]); // Sum(31) + res[32] = fma52lo(res[32], a[9], a[23]); // Sum(32) + res[33] = fma52hi(res[33], a[9], a[23]); // Sum(32) + res[33] = fma52lo(res[33], a[10], a[23]); // Sum(33) + res[34] = fma52hi(res[34], a[10], a[23]); // Sum(33) + res[34] = fma52lo(res[34], a[11], a[23]); // Sum(34) + res[35] = fma52hi(res[35], a[11], a[23]); // Sum(34) + res[35] = fma52lo(res[35], a[12], a[23]); // Sum(35) + res[36] = fma52hi(res[36], a[12], a[23]); // Sum(35) + res[24] = fma52lo(res[24], a[0], a[24]); // Sum(24) + res[25] = fma52hi(res[25], a[0], a[24]); // Sum(24) + res[25] = fma52lo(res[25], a[1], a[24]); // Sum(25) + res[26] = fma52hi(res[26], a[1], a[24]); // Sum(25) + res[26] = fma52lo(res[26], a[2], a[24]); // Sum(26) + res[27] = fma52hi(res[27], a[2], a[24]); // Sum(26) + res[27] = fma52lo(res[27], a[3], a[24]); // Sum(27) + res[28] = fma52hi(res[28], a[3], a[24]); // Sum(27) + res[28] = fma52lo(res[28], a[4], a[24]); // Sum(28) + res[29] = fma52hi(res[29], a[4], a[24]); // Sum(28) + res[29] = fma52lo(res[29], a[5], a[24]); // Sum(29) + res[30] = fma52hi(res[30], a[5], a[24]); // Sum(29) + res[30] = fma52lo(res[30], a[6], a[24]); // Sum(30) + res[31] = fma52hi(res[31], a[6], a[24]); // Sum(30) + res[31] = fma52lo(res[31], a[7], a[24]); // Sum(31) + res[32] = fma52hi(res[32], a[7], a[24]); // Sum(31) + res[32] = fma52lo(res[32], a[8], a[24]); // Sum(32) + res[33] = fma52hi(res[33], a[8], a[24]); // Sum(32) + res[33] = fma52lo(res[33], a[9], a[24]); // Sum(33) + res[34] = fma52hi(res[34], a[9], a[24]); // Sum(33) + res[34] = fma52lo(res[34], a[10], a[24]); // Sum(34) + res[35] = fma52hi(res[35], a[10], a[24]); // Sum(34) + res[35] = fma52lo(res[35], a[11], a[24]); // Sum(35) + res[36] = fma52hi(res[36], a[11], a[24]); // Sum(35) + res[25] = fma52lo(res[25], a[0], a[25]); // Sum(25) + res[26] = fma52hi(res[26], a[0], a[25]); // Sum(25) + res[26] = fma52lo(res[26], a[1], a[25]); // Sum(26) + res[27] = fma52hi(res[27], a[1], a[25]); // Sum(26) + res[27] = fma52lo(res[27], a[2], a[25]); // Sum(27) + res[28] = fma52hi(res[28], a[2], a[25]); // Sum(27) + res[28] = fma52lo(res[28], a[3], a[25]); // Sum(28) + res[29] = fma52hi(res[29], a[3], a[25]); // Sum(28) + res[29] = fma52lo(res[29], a[4], a[25]); // Sum(29) + res[30] = fma52hi(res[30], a[4], a[25]); // Sum(29) + res[30] = fma52lo(res[30], a[5], a[25]); // Sum(30) + res[31] = fma52hi(res[31], a[5], a[25]); // Sum(30) + res[31] = fma52lo(res[31], a[6], a[25]); // Sum(31) + res[32] = fma52hi(res[32], a[6], a[25]); // Sum(31) + res[32] = fma52lo(res[32], a[7], a[25]); // Sum(32) + res[33] = fma52hi(res[33], a[7], a[25]); // Sum(32) + res[33] = fma52lo(res[33], a[8], a[25]); // Sum(33) + res[34] = fma52hi(res[34], a[8], a[25]); // Sum(33) + res[34] = fma52lo(res[34], a[9], a[25]); // Sum(34) + res[35] = fma52hi(res[35], a[9], a[25]); // Sum(34) + res[35] = fma52lo(res[35], a[10], a[25]); // Sum(35) + res[36] = fma52hi(res[36], a[10], a[25]); // Sum(35) + res[26] = fma52lo(res[26], a[0], a[26]); // Sum(26) + res[27] = fma52hi(res[27], a[0], a[26]); // Sum(26) + res[27] = fma52lo(res[27], a[1], a[26]); // Sum(27) + res[28] = fma52hi(res[28], a[1], a[26]); // Sum(27) + res[28] = fma52lo(res[28], a[2], a[26]); // Sum(28) + res[29] = fma52hi(res[29], a[2], a[26]); // Sum(28) + res[29] = fma52lo(res[29], a[3], a[26]); // Sum(29) + res[30] = fma52hi(res[30], a[3], a[26]); // Sum(29) + res[30] = fma52lo(res[30], a[4], a[26]); // Sum(30) + res[31] = fma52hi(res[31], a[4], a[26]); // Sum(30) + res[31] = fma52lo(res[31], a[5], a[26]); // Sum(31) + res[32] = fma52hi(res[32], a[5], a[26]); // Sum(31) + res[32] = fma52lo(res[32], a[6], a[26]); // Sum(32) + res[33] = fma52hi(res[33], a[6], a[26]); // Sum(32) + res[33] = fma52lo(res[33], a[7], a[26]); // Sum(33) + res[34] = fma52hi(res[34], a[7], a[26]); // Sum(33) + res[34] = fma52lo(res[34], a[8], a[26]); // Sum(34) + res[35] = fma52hi(res[35], a[8], a[26]); // Sum(34) + res[35] = fma52lo(res[35], a[9], a[26]); // Sum(35) + res[36] = fma52hi(res[36], a[9], a[26]); // Sum(35) + res[27] = fma52lo(res[27], a[0], a[27]); // Sum(27) + res[28] = fma52hi(res[28], a[0], a[27]); // Sum(27) + res[28] = fma52lo(res[28], a[1], a[27]); // Sum(28) + res[29] = fma52hi(res[29], a[1], a[27]); // Sum(28) + res[29] = fma52lo(res[29], a[2], a[27]); // Sum(29) + res[30] = fma52hi(res[30], a[2], a[27]); // Sum(29) + res[30] = fma52lo(res[30], a[3], a[27]); // Sum(30) + res[31] = fma52hi(res[31], a[3], a[27]); // Sum(30) + res[31] = fma52lo(res[31], a[4], a[27]); // Sum(31) + res[32] = fma52hi(res[32], a[4], a[27]); // Sum(31) + res[32] = fma52lo(res[32], a[5], a[27]); // Sum(32) + res[33] = fma52hi(res[33], a[5], a[27]); // Sum(32) + res[33] = fma52lo(res[33], a[6], a[27]); // Sum(33) + res[34] = fma52hi(res[34], a[6], a[27]); // Sum(33) + res[34] = fma52lo(res[34], a[7], a[27]); // Sum(34) + res[35] = fma52hi(res[35], a[7], a[27]); // Sum(34) + res[35] = fma52lo(res[35], a[8], a[27]); // Sum(35) + res[36] = fma52hi(res[36], a[8], a[27]); // Sum(35) + res[28] = fma52lo(res[28], a[0], a[28]); // Sum(28) + res[29] = fma52hi(res[29], a[0], a[28]); // Sum(28) + res[29] = fma52lo(res[29], a[1], a[28]); // Sum(29) + res[30] = fma52hi(res[30], a[1], a[28]); // Sum(29) + res[30] = fma52lo(res[30], a[2], a[28]); // Sum(30) + res[31] = fma52hi(res[31], a[2], a[28]); // Sum(30) + res[31] = fma52lo(res[31], a[3], a[28]); // Sum(31) + res[32] = fma52hi(res[32], a[3], a[28]); // Sum(31) + res[32] = fma52lo(res[32], a[4], a[28]); // Sum(32) + res[33] = fma52hi(res[33], a[4], a[28]); // Sum(32) + res[33] = fma52lo(res[33], a[5], a[28]); // Sum(33) + res[34] = fma52hi(res[34], a[5], a[28]); // Sum(33) + res[34] = fma52lo(res[34], a[6], a[28]); // Sum(34) + res[35] = fma52hi(res[35], a[6], a[28]); // Sum(34) + res[35] = fma52lo(res[35], a[7], a[28]); // Sum(35) + res[36] = fma52hi(res[36], a[7], a[28]); // Sum(35) + res[29] = fma52lo(res[29], a[0], a[29]); // Sum(29) + res[30] = fma52hi(res[30], a[0], a[29]); // Sum(29) + res[30] = fma52lo(res[30], a[1], a[29]); // Sum(30) + res[31] = fma52hi(res[31], a[1], a[29]); // Sum(30) + res[31] = fma52lo(res[31], a[2], a[29]); // Sum(31) + res[32] = fma52hi(res[32], a[2], a[29]); // Sum(31) + res[32] = fma52lo(res[32], a[3], a[29]); // Sum(32) + res[33] = fma52hi(res[33], a[3], a[29]); // Sum(32) + res[33] = fma52lo(res[33], a[4], a[29]); // Sum(33) + res[34] = fma52hi(res[34], a[4], a[29]); // Sum(33) + res[34] = fma52lo(res[34], a[5], a[29]); // Sum(34) + res[35] = fma52hi(res[35], a[5], a[29]); // Sum(34) + res[35] = fma52lo(res[35], a[6], a[29]); // Sum(35) + res[36] = fma52hi(res[36], a[6], a[29]); // Sum(35) + res[30] = fma52lo(res[30], a[0], a[30]); // Sum(30) + res[31] = fma52hi(res[31], a[0], a[30]); // Sum(30) + res[31] = fma52lo(res[31], a[1], a[30]); // Sum(31) + res[32] = fma52hi(res[32], a[1], a[30]); // Sum(31) + res[32] = fma52lo(res[32], a[2], a[30]); // Sum(32) + res[33] = fma52hi(res[33], a[2], a[30]); // Sum(32) + res[33] = fma52lo(res[33], a[3], a[30]); // Sum(33) + res[34] = fma52hi(res[34], a[3], a[30]); // Sum(33) + res[34] = fma52lo(res[34], a[4], a[30]); // Sum(34) + res[35] = fma52hi(res[35], a[4], a[30]); // Sum(34) + res[35] = fma52lo(res[35], a[5], a[30]); // Sum(35) + res[36] = fma52hi(res[36], a[5], a[30]); // Sum(35) + res[31] = fma52lo(res[31], a[0], a[31]); // Sum(31) + res[32] = fma52hi(res[32], a[0], a[31]); // Sum(31) + res[32] = fma52lo(res[32], a[1], a[31]); // Sum(32) + res[33] = fma52hi(res[33], a[1], a[31]); // Sum(32) + res[33] = fma52lo(res[33], a[2], a[31]); // Sum(33) + res[34] = fma52hi(res[34], a[2], a[31]); // Sum(33) + res[34] = fma52lo(res[34], a[3], a[31]); // Sum(34) + res[35] = fma52hi(res[35], a[3], a[31]); // Sum(34) + res[35] = fma52lo(res[35], a[4], a[31]); // Sum(35) + res[36] = fma52hi(res[36], a[4], a[31]); // Sum(35) + res[32] = fma52lo(res[32], a[0], a[32]); // Sum(32) + res[33] = fma52hi(res[33], a[0], a[32]); // Sum(32) + res[33] = fma52lo(res[33], a[1], a[32]); // Sum(33) + res[34] = fma52hi(res[34], a[1], a[32]); // Sum(33) + res[34] = fma52lo(res[34], a[2], a[32]); // Sum(34) + res[35] = fma52hi(res[35], a[2], a[32]); // Sum(34) + res[35] = fma52lo(res[35], a[3], a[32]); // Sum(35) + res[36] = fma52hi(res[36], a[3], a[32]); // Sum(35) + res[33] = fma52lo(res[33], a[0], a[33]); // Sum(33) + res[34] = fma52hi(res[34], a[0], a[33]); // Sum(33) + res[34] = fma52lo(res[34], a[1], a[33]); // Sum(34) + res[35] = fma52hi(res[35], a[1], a[33]); // Sum(34) + res[35] = fma52lo(res[35], a[2], a[33]); // Sum(35) + res[36] = fma52hi(res[36], a[2], a[33]); // Sum(35) + res[34] = fma52lo(res[34], a[0], a[34]); // Sum(34) + res[35] = fma52hi(res[35], a[0], a[34]); // Sum(34) + res[35] = fma52lo(res[35], a[1], a[34]); // Sum(35) + res[36] = fma52hi(res[36], a[1], a[34]); // Sum(35) + res[35] = fma52lo(res[35], a[0], a[35]); // Sum(35) + res[36] = fma52hi(res[36], a[0], a[35]); // Sum(35) + res[24] = add64(res[24], res[24]); // Double(24) + res[25] = add64(res[25], res[25]); // Double(25) + res[26] = add64(res[26], res[26]); // Double(26) + res[27] = add64(res[27], res[27]); // Double(27) + res[28] = add64(res[28], res[28]); // Double(28) + res[29] = add64(res[29], res[29]); // Double(29) + res[30] = add64(res[30], res[30]); // Double(30) + res[31] = add64(res[31], res[31]); // Double(31) + res[32] = add64(res[32], res[32]); // Double(32) + res[33] = add64(res[33], res[33]); // Double(33) + res[34] = add64(res[34], res[34]); // Double(34) + res[35] = add64(res[35], res[35]); // Double(35) + res[24] = fma52lo(res[24], a[12], a[12]); // Add sqr(24) + res[25] = fma52hi(res[25], a[12], a[12]); // Add sqr(24) + res[26] = fma52lo(res[26], a[13], a[13]); // Add sqr(26) + res[27] = fma52hi(res[27], a[13], a[13]); // Add sqr(26) + res[28] = fma52lo(res[28], a[14], a[14]); // Add sqr(28) + res[29] = fma52hi(res[29], a[14], a[14]); // Add sqr(28) + res[30] = fma52lo(res[30], a[15], a[15]); // Add sqr(30) + res[31] = fma52hi(res[31], a[15], a[15]); // Add sqr(30) + res[32] = fma52lo(res[32], a[16], a[16]); // Add sqr(32) + res[33] = fma52hi(res[33], a[16], a[16]); // Add sqr(32) + res[34] = fma52lo(res[34], a[17], a[17]); // Add sqr(34) + res[35] = fma52hi(res[35], a[17], a[17]); // Add sqr(34) + res[36] = fma52lo(res[36], a[17], a[19]); // Sum(36) + res[37] = fma52hi(res[37], a[17], a[19]); // Sum(36) + res[37] = fma52lo(res[37], a[18], a[19]); // Sum(37) + res[38] = fma52hi(res[38], a[18], a[19]); // Sum(37) + res[36] = fma52lo(res[36], a[16], a[20]); // Sum(36) + res[37] = fma52hi(res[37], a[16], a[20]); // Sum(36) + res[37] = fma52lo(res[37], a[17], a[20]); // Sum(37) + res[38] = fma52hi(res[38], a[17], a[20]); // Sum(37) + res[38] = fma52lo(res[38], a[18], a[20]); // Sum(38) + res[39] = fma52hi(res[39], a[18], a[20]); // Sum(38) + res[39] = fma52lo(res[39], a[19], a[20]); // Sum(39) + res[40] = fma52hi(res[40], a[19], a[20]); // Sum(39) + res[36] = fma52lo(res[36], a[15], a[21]); // Sum(36) + res[37] = fma52hi(res[37], a[15], a[21]); // Sum(36) + res[37] = fma52lo(res[37], a[16], a[21]); // Sum(37) + res[38] = fma52hi(res[38], a[16], a[21]); // Sum(37) + res[38] = fma52lo(res[38], a[17], a[21]); // Sum(38) + res[39] = fma52hi(res[39], a[17], a[21]); // Sum(38) + res[39] = fma52lo(res[39], a[18], a[21]); // Sum(39) + res[40] = fma52hi(res[40], a[18], a[21]); // Sum(39) + res[40] = fma52lo(res[40], a[19], a[21]); // Sum(40) + res[41] = fma52hi(res[41], a[19], a[21]); // Sum(40) + res[41] = fma52lo(res[41], a[20], a[21]); // Sum(41) + res[42] = fma52hi(res[42], a[20], a[21]); // Sum(41) + res[36] = fma52lo(res[36], a[14], a[22]); // Sum(36) + res[37] = fma52hi(res[37], a[14], a[22]); // Sum(36) + res[37] = fma52lo(res[37], a[15], a[22]); // Sum(37) + res[38] = fma52hi(res[38], a[15], a[22]); // Sum(37) + res[38] = fma52lo(res[38], a[16], a[22]); // Sum(38) + res[39] = fma52hi(res[39], a[16], a[22]); // Sum(38) + res[39] = fma52lo(res[39], a[17], a[22]); // Sum(39) + res[40] = fma52hi(res[40], a[17], a[22]); // Sum(39) + res[40] = fma52lo(res[40], a[18], a[22]); // Sum(40) + res[41] = fma52hi(res[41], a[18], a[22]); // Sum(40) + res[41] = fma52lo(res[41], a[19], a[22]); // Sum(41) + res[42] = fma52hi(res[42], a[19], a[22]); // Sum(41) + res[42] = fma52lo(res[42], a[20], a[22]); // Sum(42) + res[43] = fma52hi(res[43], a[20], a[22]); // Sum(42) + res[43] = fma52lo(res[43], a[21], a[22]); // Sum(43) + res[44] = fma52hi(res[44], a[21], a[22]); // Sum(43) + res[36] = fma52lo(res[36], a[13], a[23]); // Sum(36) + res[37] = fma52hi(res[37], a[13], a[23]); // Sum(36) + res[37] = fma52lo(res[37], a[14], a[23]); // Sum(37) + res[38] = fma52hi(res[38], a[14], a[23]); // Sum(37) + res[38] = fma52lo(res[38], a[15], a[23]); // Sum(38) + res[39] = fma52hi(res[39], a[15], a[23]); // Sum(38) + res[39] = fma52lo(res[39], a[16], a[23]); // Sum(39) + res[40] = fma52hi(res[40], a[16], a[23]); // Sum(39) + res[40] = fma52lo(res[40], a[17], a[23]); // Sum(40) + res[41] = fma52hi(res[41], a[17], a[23]); // Sum(40) + res[41] = fma52lo(res[41], a[18], a[23]); // Sum(41) + res[42] = fma52hi(res[42], a[18], a[23]); // Sum(41) + res[42] = fma52lo(res[42], a[19], a[23]); // Sum(42) + res[43] = fma52hi(res[43], a[19], a[23]); // Sum(42) + res[43] = fma52lo(res[43], a[20], a[23]); // Sum(43) + res[44] = fma52hi(res[44], a[20], a[23]); // Sum(43) + res[44] = fma52lo(res[44], a[21], a[23]); // Sum(44) + res[45] = fma52hi(res[45], a[21], a[23]); // Sum(44) + res[45] = fma52lo(res[45], a[22], a[23]); // Sum(45) + res[46] = fma52hi(res[46], a[22], a[23]); // Sum(45) + res[36] = fma52lo(res[36], a[12], a[24]); // Sum(36) + res[37] = fma52hi(res[37], a[12], a[24]); // Sum(36) + res[37] = fma52lo(res[37], a[13], a[24]); // Sum(37) + res[38] = fma52hi(res[38], a[13], a[24]); // Sum(37) + res[38] = fma52lo(res[38], a[14], a[24]); // Sum(38) + res[39] = fma52hi(res[39], a[14], a[24]); // Sum(38) + res[39] = fma52lo(res[39], a[15], a[24]); // Sum(39) + res[40] = fma52hi(res[40], a[15], a[24]); // Sum(39) + res[40] = fma52lo(res[40], a[16], a[24]); // Sum(40) + res[41] = fma52hi(res[41], a[16], a[24]); // Sum(40) + res[41] = fma52lo(res[41], a[17], a[24]); // Sum(41) + res[42] = fma52hi(res[42], a[17], a[24]); // Sum(41) + res[42] = fma52lo(res[42], a[18], a[24]); // Sum(42) + res[43] = fma52hi(res[43], a[18], a[24]); // Sum(42) + res[43] = fma52lo(res[43], a[19], a[24]); // Sum(43) + res[44] = fma52hi(res[44], a[19], a[24]); // Sum(43) + res[44] = fma52lo(res[44], a[20], a[24]); // Sum(44) + res[45] = fma52hi(res[45], a[20], a[24]); // Sum(44) + res[45] = fma52lo(res[45], a[21], a[24]); // Sum(45) + res[46] = fma52hi(res[46], a[21], a[24]); // Sum(45) + res[46] = fma52lo(res[46], a[22], a[24]); // Sum(46) + res[47] = fma52hi(res[47], a[22], a[24]); // Sum(46) + res[47] = fma52lo(res[47], a[23], a[24]); // Sum(47) + res[48] = fma52hi(res[48], a[23], a[24]); // Sum(47) + res[36] = fma52lo(res[36], a[11], a[25]); // Sum(36) + res[37] = fma52hi(res[37], a[11], a[25]); // Sum(36) + res[37] = fma52lo(res[37], a[12], a[25]); // Sum(37) + res[38] = fma52hi(res[38], a[12], a[25]); // Sum(37) + res[38] = fma52lo(res[38], a[13], a[25]); // Sum(38) + res[39] = fma52hi(res[39], a[13], a[25]); // Sum(38) + res[39] = fma52lo(res[39], a[14], a[25]); // Sum(39) + res[40] = fma52hi(res[40], a[14], a[25]); // Sum(39) + res[40] = fma52lo(res[40], a[15], a[25]); // Sum(40) + res[41] = fma52hi(res[41], a[15], a[25]); // Sum(40) + res[41] = fma52lo(res[41], a[16], a[25]); // Sum(41) + res[42] = fma52hi(res[42], a[16], a[25]); // Sum(41) + res[42] = fma52lo(res[42], a[17], a[25]); // Sum(42) + res[43] = fma52hi(res[43], a[17], a[25]); // Sum(42) + res[43] = fma52lo(res[43], a[18], a[25]); // Sum(43) + res[44] = fma52hi(res[44], a[18], a[25]); // Sum(43) + res[44] = fma52lo(res[44], a[19], a[25]); // Sum(44) + res[45] = fma52hi(res[45], a[19], a[25]); // Sum(44) + res[45] = fma52lo(res[45], a[20], a[25]); // Sum(45) + res[46] = fma52hi(res[46], a[20], a[25]); // Sum(45) + res[46] = fma52lo(res[46], a[21], a[25]); // Sum(46) + res[47] = fma52hi(res[47], a[21], a[25]); // Sum(46) + res[47] = fma52lo(res[47], a[22], a[25]); // Sum(47) + res[48] = fma52hi(res[48], a[22], a[25]); // Sum(47) + res[36] = fma52lo(res[36], a[10], a[26]); // Sum(36) + res[37] = fma52hi(res[37], a[10], a[26]); // Sum(36) + res[37] = fma52lo(res[37], a[11], a[26]); // Sum(37) + res[38] = fma52hi(res[38], a[11], a[26]); // Sum(37) + res[38] = fma52lo(res[38], a[12], a[26]); // Sum(38) + res[39] = fma52hi(res[39], a[12], a[26]); // Sum(38) + res[39] = fma52lo(res[39], a[13], a[26]); // Sum(39) + res[40] = fma52hi(res[40], a[13], a[26]); // Sum(39) + res[40] = fma52lo(res[40], a[14], a[26]); // Sum(40) + res[41] = fma52hi(res[41], a[14], a[26]); // Sum(40) + res[41] = fma52lo(res[41], a[15], a[26]); // Sum(41) + res[42] = fma52hi(res[42], a[15], a[26]); // Sum(41) + res[42] = fma52lo(res[42], a[16], a[26]); // Sum(42) + res[43] = fma52hi(res[43], a[16], a[26]); // Sum(42) + res[43] = fma52lo(res[43], a[17], a[26]); // Sum(43) + res[44] = fma52hi(res[44], a[17], a[26]); // Sum(43) + res[44] = fma52lo(res[44], a[18], a[26]); // Sum(44) + res[45] = fma52hi(res[45], a[18], a[26]); // Sum(44) + res[45] = fma52lo(res[45], a[19], a[26]); // Sum(45) + res[46] = fma52hi(res[46], a[19], a[26]); // Sum(45) + res[46] = fma52lo(res[46], a[20], a[26]); // Sum(46) + res[47] = fma52hi(res[47], a[20], a[26]); // Sum(46) + res[47] = fma52lo(res[47], a[21], a[26]); // Sum(47) + res[48] = fma52hi(res[48], a[21], a[26]); // Sum(47) + res[36] = fma52lo(res[36], a[9], a[27]); // Sum(36) + res[37] = fma52hi(res[37], a[9], a[27]); // Sum(36) + res[37] = fma52lo(res[37], a[10], a[27]); // Sum(37) + res[38] = fma52hi(res[38], a[10], a[27]); // Sum(37) + res[38] = fma52lo(res[38], a[11], a[27]); // Sum(38) + res[39] = fma52hi(res[39], a[11], a[27]); // Sum(38) + res[39] = fma52lo(res[39], a[12], a[27]); // Sum(39) + res[40] = fma52hi(res[40], a[12], a[27]); // Sum(39) + res[40] = fma52lo(res[40], a[13], a[27]); // Sum(40) + res[41] = fma52hi(res[41], a[13], a[27]); // Sum(40) + res[41] = fma52lo(res[41], a[14], a[27]); // Sum(41) + res[42] = fma52hi(res[42], a[14], a[27]); // Sum(41) + res[42] = fma52lo(res[42], a[15], a[27]); // Sum(42) + res[43] = fma52hi(res[43], a[15], a[27]); // Sum(42) + res[43] = fma52lo(res[43], a[16], a[27]); // Sum(43) + res[44] = fma52hi(res[44], a[16], a[27]); // Sum(43) + res[44] = fma52lo(res[44], a[17], a[27]); // Sum(44) + res[45] = fma52hi(res[45], a[17], a[27]); // Sum(44) + res[45] = fma52lo(res[45], a[18], a[27]); // Sum(45) + res[46] = fma52hi(res[46], a[18], a[27]); // Sum(45) + res[46] = fma52lo(res[46], a[19], a[27]); // Sum(46) + res[47] = fma52hi(res[47], a[19], a[27]); // Sum(46) + res[47] = fma52lo(res[47], a[20], a[27]); // Sum(47) + res[48] = fma52hi(res[48], a[20], a[27]); // Sum(47) + res[36] = fma52lo(res[36], a[8], a[28]); // Sum(36) + res[37] = fma52hi(res[37], a[8], a[28]); // Sum(36) + res[37] = fma52lo(res[37], a[9], a[28]); // Sum(37) + res[38] = fma52hi(res[38], a[9], a[28]); // Sum(37) + res[38] = fma52lo(res[38], a[10], a[28]); // Sum(38) + res[39] = fma52hi(res[39], a[10], a[28]); // Sum(38) + res[39] = fma52lo(res[39], a[11], a[28]); // Sum(39) + res[40] = fma52hi(res[40], a[11], a[28]); // Sum(39) + res[40] = fma52lo(res[40], a[12], a[28]); // Sum(40) + res[41] = fma52hi(res[41], a[12], a[28]); // Sum(40) + res[41] = fma52lo(res[41], a[13], a[28]); // Sum(41) + res[42] = fma52hi(res[42], a[13], a[28]); // Sum(41) + res[42] = fma52lo(res[42], a[14], a[28]); // Sum(42) + res[43] = fma52hi(res[43], a[14], a[28]); // Sum(42) + res[43] = fma52lo(res[43], a[15], a[28]); // Sum(43) + res[44] = fma52hi(res[44], a[15], a[28]); // Sum(43) + res[44] = fma52lo(res[44], a[16], a[28]); // Sum(44) + res[45] = fma52hi(res[45], a[16], a[28]); // Sum(44) + res[45] = fma52lo(res[45], a[17], a[28]); // Sum(45) + res[46] = fma52hi(res[46], a[17], a[28]); // Sum(45) + res[46] = fma52lo(res[46], a[18], a[28]); // Sum(46) + res[47] = fma52hi(res[47], a[18], a[28]); // Sum(46) + res[47] = fma52lo(res[47], a[19], a[28]); // Sum(47) + res[48] = fma52hi(res[48], a[19], a[28]); // Sum(47) + res[36] = fma52lo(res[36], a[7], a[29]); // Sum(36) + res[37] = fma52hi(res[37], a[7], a[29]); // Sum(36) + res[37] = fma52lo(res[37], a[8], a[29]); // Sum(37) + res[38] = fma52hi(res[38], a[8], a[29]); // Sum(37) + res[38] = fma52lo(res[38], a[9], a[29]); // Sum(38) + res[39] = fma52hi(res[39], a[9], a[29]); // Sum(38) + res[39] = fma52lo(res[39], a[10], a[29]); // Sum(39) + res[40] = fma52hi(res[40], a[10], a[29]); // Sum(39) + res[40] = fma52lo(res[40], a[11], a[29]); // Sum(40) + res[41] = fma52hi(res[41], a[11], a[29]); // Sum(40) + res[41] = fma52lo(res[41], a[12], a[29]); // Sum(41) + res[42] = fma52hi(res[42], a[12], a[29]); // Sum(41) + res[42] = fma52lo(res[42], a[13], a[29]); // Sum(42) + res[43] = fma52hi(res[43], a[13], a[29]); // Sum(42) + res[43] = fma52lo(res[43], a[14], a[29]); // Sum(43) + res[44] = fma52hi(res[44], a[14], a[29]); // Sum(43) + res[44] = fma52lo(res[44], a[15], a[29]); // Sum(44) + res[45] = fma52hi(res[45], a[15], a[29]); // Sum(44) + res[45] = fma52lo(res[45], a[16], a[29]); // Sum(45) + res[46] = fma52hi(res[46], a[16], a[29]); // Sum(45) + res[46] = fma52lo(res[46], a[17], a[29]); // Sum(46) + res[47] = fma52hi(res[47], a[17], a[29]); // Sum(46) + res[47] = fma52lo(res[47], a[18], a[29]); // Sum(47) + res[48] = fma52hi(res[48], a[18], a[29]); // Sum(47) + res[36] = fma52lo(res[36], a[6], a[30]); // Sum(36) + res[37] = fma52hi(res[37], a[6], a[30]); // Sum(36) + res[37] = fma52lo(res[37], a[7], a[30]); // Sum(37) + res[38] = fma52hi(res[38], a[7], a[30]); // Sum(37) + res[38] = fma52lo(res[38], a[8], a[30]); // Sum(38) + res[39] = fma52hi(res[39], a[8], a[30]); // Sum(38) + res[39] = fma52lo(res[39], a[9], a[30]); // Sum(39) + res[40] = fma52hi(res[40], a[9], a[30]); // Sum(39) + res[40] = fma52lo(res[40], a[10], a[30]); // Sum(40) + res[41] = fma52hi(res[41], a[10], a[30]); // Sum(40) + res[41] = fma52lo(res[41], a[11], a[30]); // Sum(41) + res[42] = fma52hi(res[42], a[11], a[30]); // Sum(41) + res[42] = fma52lo(res[42], a[12], a[30]); // Sum(42) + res[43] = fma52hi(res[43], a[12], a[30]); // Sum(42) + res[43] = fma52lo(res[43], a[13], a[30]); // Sum(43) + res[44] = fma52hi(res[44], a[13], a[30]); // Sum(43) + res[44] = fma52lo(res[44], a[14], a[30]); // Sum(44) + res[45] = fma52hi(res[45], a[14], a[30]); // Sum(44) + res[45] = fma52lo(res[45], a[15], a[30]); // Sum(45) + res[46] = fma52hi(res[46], a[15], a[30]); // Sum(45) + res[46] = fma52lo(res[46], a[16], a[30]); // Sum(46) + res[47] = fma52hi(res[47], a[16], a[30]); // Sum(46) + res[47] = fma52lo(res[47], a[17], a[30]); // Sum(47) + res[48] = fma52hi(res[48], a[17], a[30]); // Sum(47) + res[36] = fma52lo(res[36], a[5], a[31]); // Sum(36) + res[37] = fma52hi(res[37], a[5], a[31]); // Sum(36) + res[37] = fma52lo(res[37], a[6], a[31]); // Sum(37) + res[38] = fma52hi(res[38], a[6], a[31]); // Sum(37) + res[38] = fma52lo(res[38], a[7], a[31]); // Sum(38) + res[39] = fma52hi(res[39], a[7], a[31]); // Sum(38) + res[39] = fma52lo(res[39], a[8], a[31]); // Sum(39) + res[40] = fma52hi(res[40], a[8], a[31]); // Sum(39) + res[40] = fma52lo(res[40], a[9], a[31]); // Sum(40) + res[41] = fma52hi(res[41], a[9], a[31]); // Sum(40) + res[41] = fma52lo(res[41], a[10], a[31]); // Sum(41) + res[42] = fma52hi(res[42], a[10], a[31]); // Sum(41) + res[42] = fma52lo(res[42], a[11], a[31]); // Sum(42) + res[43] = fma52hi(res[43], a[11], a[31]); // Sum(42) + res[43] = fma52lo(res[43], a[12], a[31]); // Sum(43) + res[44] = fma52hi(res[44], a[12], a[31]); // Sum(43) + res[44] = fma52lo(res[44], a[13], a[31]); // Sum(44) + res[45] = fma52hi(res[45], a[13], a[31]); // Sum(44) + res[45] = fma52lo(res[45], a[14], a[31]); // Sum(45) + res[46] = fma52hi(res[46], a[14], a[31]); // Sum(45) + res[46] = fma52lo(res[46], a[15], a[31]); // Sum(46) + res[47] = fma52hi(res[47], a[15], a[31]); // Sum(46) + res[47] = fma52lo(res[47], a[16], a[31]); // Sum(47) + res[48] = fma52hi(res[48], a[16], a[31]); // Sum(47) + res[36] = fma52lo(res[36], a[4], a[32]); // Sum(36) + res[37] = fma52hi(res[37], a[4], a[32]); // Sum(36) + res[37] = fma52lo(res[37], a[5], a[32]); // Sum(37) + res[38] = fma52hi(res[38], a[5], a[32]); // Sum(37) + res[38] = fma52lo(res[38], a[6], a[32]); // Sum(38) + res[39] = fma52hi(res[39], a[6], a[32]); // Sum(38) + res[39] = fma52lo(res[39], a[7], a[32]); // Sum(39) + res[40] = fma52hi(res[40], a[7], a[32]); // Sum(39) + res[40] = fma52lo(res[40], a[8], a[32]); // Sum(40) + res[41] = fma52hi(res[41], a[8], a[32]); // Sum(40) + res[41] = fma52lo(res[41], a[9], a[32]); // Sum(41) + res[42] = fma52hi(res[42], a[9], a[32]); // Sum(41) + res[42] = fma52lo(res[42], a[10], a[32]); // Sum(42) + res[43] = fma52hi(res[43], a[10], a[32]); // Sum(42) + res[43] = fma52lo(res[43], a[11], a[32]); // Sum(43) + res[44] = fma52hi(res[44], a[11], a[32]); // Sum(43) + res[44] = fma52lo(res[44], a[12], a[32]); // Sum(44) + res[45] = fma52hi(res[45], a[12], a[32]); // Sum(44) + res[45] = fma52lo(res[45], a[13], a[32]); // Sum(45) + res[46] = fma52hi(res[46], a[13], a[32]); // Sum(45) + res[46] = fma52lo(res[46], a[14], a[32]); // Sum(46) + res[47] = fma52hi(res[47], a[14], a[32]); // Sum(46) + res[47] = fma52lo(res[47], a[15], a[32]); // Sum(47) + res[48] = fma52hi(res[48], a[15], a[32]); // Sum(47) + res[36] = fma52lo(res[36], a[3], a[33]); // Sum(36) + res[37] = fma52hi(res[37], a[3], a[33]); // Sum(36) + res[37] = fma52lo(res[37], a[4], a[33]); // Sum(37) + res[38] = fma52hi(res[38], a[4], a[33]); // Sum(37) + res[38] = fma52lo(res[38], a[5], a[33]); // Sum(38) + res[39] = fma52hi(res[39], a[5], a[33]); // Sum(38) + res[39] = fma52lo(res[39], a[6], a[33]); // Sum(39) + res[40] = fma52hi(res[40], a[6], a[33]); // Sum(39) + res[40] = fma52lo(res[40], a[7], a[33]); // Sum(40) + res[41] = fma52hi(res[41], a[7], a[33]); // Sum(40) + res[41] = fma52lo(res[41], a[8], a[33]); // Sum(41) + res[42] = fma52hi(res[42], a[8], a[33]); // Sum(41) + res[42] = fma52lo(res[42], a[9], a[33]); // Sum(42) + res[43] = fma52hi(res[43], a[9], a[33]); // Sum(42) + res[43] = fma52lo(res[43], a[10], a[33]); // Sum(43) + res[44] = fma52hi(res[44], a[10], a[33]); // Sum(43) + res[44] = fma52lo(res[44], a[11], a[33]); // Sum(44) + res[45] = fma52hi(res[45], a[11], a[33]); // Sum(44) + res[45] = fma52lo(res[45], a[12], a[33]); // Sum(45) + res[46] = fma52hi(res[46], a[12], a[33]); // Sum(45) + res[46] = fma52lo(res[46], a[13], a[33]); // Sum(46) + res[47] = fma52hi(res[47], a[13], a[33]); // Sum(46) + res[47] = fma52lo(res[47], a[14], a[33]); // Sum(47) + res[48] = fma52hi(res[48], a[14], a[33]); // Sum(47) + res[36] = fma52lo(res[36], a[2], a[34]); // Sum(36) + res[37] = fma52hi(res[37], a[2], a[34]); // Sum(36) + res[37] = fma52lo(res[37], a[3], a[34]); // Sum(37) + res[38] = fma52hi(res[38], a[3], a[34]); // Sum(37) + res[38] = fma52lo(res[38], a[4], a[34]); // Sum(38) + res[39] = fma52hi(res[39], a[4], a[34]); // Sum(38) + res[39] = fma52lo(res[39], a[5], a[34]); // Sum(39) + res[40] = fma52hi(res[40], a[5], a[34]); // Sum(39) + res[40] = fma52lo(res[40], a[6], a[34]); // Sum(40) + res[41] = fma52hi(res[41], a[6], a[34]); // Sum(40) + res[41] = fma52lo(res[41], a[7], a[34]); // Sum(41) + res[42] = fma52hi(res[42], a[7], a[34]); // Sum(41) + res[42] = fma52lo(res[42], a[8], a[34]); // Sum(42) + res[43] = fma52hi(res[43], a[8], a[34]); // Sum(42) + res[43] = fma52lo(res[43], a[9], a[34]); // Sum(43) + res[44] = fma52hi(res[44], a[9], a[34]); // Sum(43) + res[44] = fma52lo(res[44], a[10], a[34]); // Sum(44) + res[45] = fma52hi(res[45], a[10], a[34]); // Sum(44) + res[45] = fma52lo(res[45], a[11], a[34]); // Sum(45) + res[46] = fma52hi(res[46], a[11], a[34]); // Sum(45) + res[46] = fma52lo(res[46], a[12], a[34]); // Sum(46) + res[47] = fma52hi(res[47], a[12], a[34]); // Sum(46) + res[47] = fma52lo(res[47], a[13], a[34]); // Sum(47) + res[48] = fma52hi(res[48], a[13], a[34]); // Sum(47) + res[36] = fma52lo(res[36], a[1], a[35]); // Sum(36) + res[37] = fma52hi(res[37], a[1], a[35]); // Sum(36) + res[37] = fma52lo(res[37], a[2], a[35]); // Sum(37) + res[38] = fma52hi(res[38], a[2], a[35]); // Sum(37) + res[38] = fma52lo(res[38], a[3], a[35]); // Sum(38) + res[39] = fma52hi(res[39], a[3], a[35]); // Sum(38) + res[39] = fma52lo(res[39], a[4], a[35]); // Sum(39) + res[40] = fma52hi(res[40], a[4], a[35]); // Sum(39) + res[40] = fma52lo(res[40], a[5], a[35]); // Sum(40) + res[41] = fma52hi(res[41], a[5], a[35]); // Sum(40) + res[41] = fma52lo(res[41], a[6], a[35]); // Sum(41) + res[42] = fma52hi(res[42], a[6], a[35]); // Sum(41) + res[42] = fma52lo(res[42], a[7], a[35]); // Sum(42) + res[43] = fma52hi(res[43], a[7], a[35]); // Sum(42) + res[43] = fma52lo(res[43], a[8], a[35]); // Sum(43) + res[44] = fma52hi(res[44], a[8], a[35]); // Sum(43) + res[44] = fma52lo(res[44], a[9], a[35]); // Sum(44) + res[45] = fma52hi(res[45], a[9], a[35]); // Sum(44) + res[45] = fma52lo(res[45], a[10], a[35]); // Sum(45) + res[46] = fma52hi(res[46], a[10], a[35]); // Sum(45) + res[46] = fma52lo(res[46], a[11], a[35]); // Sum(46) + res[47] = fma52hi(res[47], a[11], a[35]); // Sum(46) + res[47] = fma52lo(res[47], a[12], a[35]); // Sum(47) + res[48] = fma52hi(res[48], a[12], a[35]); // Sum(47) + res[36] = fma52lo(res[36], a[0], a[36]); // Sum(36) + res[37] = fma52hi(res[37], a[0], a[36]); // Sum(36) + res[37] = fma52lo(res[37], a[1], a[36]); // Sum(37) + res[38] = fma52hi(res[38], a[1], a[36]); // Sum(37) + res[38] = fma52lo(res[38], a[2], a[36]); // Sum(38) + res[39] = fma52hi(res[39], a[2], a[36]); // Sum(38) + res[39] = fma52lo(res[39], a[3], a[36]); // Sum(39) + res[40] = fma52hi(res[40], a[3], a[36]); // Sum(39) + res[40] = fma52lo(res[40], a[4], a[36]); // Sum(40) + res[41] = fma52hi(res[41], a[4], a[36]); // Sum(40) + res[41] = fma52lo(res[41], a[5], a[36]); // Sum(41) + res[42] = fma52hi(res[42], a[5], a[36]); // Sum(41) + res[42] = fma52lo(res[42], a[6], a[36]); // Sum(42) + res[43] = fma52hi(res[43], a[6], a[36]); // Sum(42) + res[43] = fma52lo(res[43], a[7], a[36]); // Sum(43) + res[44] = fma52hi(res[44], a[7], a[36]); // Sum(43) + res[44] = fma52lo(res[44], a[8], a[36]); // Sum(44) + res[45] = fma52hi(res[45], a[8], a[36]); // Sum(44) + res[45] = fma52lo(res[45], a[9], a[36]); // Sum(45) + res[46] = fma52hi(res[46], a[9], a[36]); // Sum(45) + res[46] = fma52lo(res[46], a[10], a[36]); // Sum(46) + res[47] = fma52hi(res[47], a[10], a[36]); // Sum(46) + res[47] = fma52lo(res[47], a[11], a[36]); // Sum(47) + res[48] = fma52hi(res[48], a[11], a[36]); // Sum(47) + res[37] = fma52lo(res[37], a[0], a[37]); // Sum(37) + res[38] = fma52hi(res[38], a[0], a[37]); // Sum(37) + res[38] = fma52lo(res[38], a[1], a[37]); // Sum(38) + res[39] = fma52hi(res[39], a[1], a[37]); // Sum(38) + res[39] = fma52lo(res[39], a[2], a[37]); // Sum(39) + res[40] = fma52hi(res[40], a[2], a[37]); // Sum(39) + res[40] = fma52lo(res[40], a[3], a[37]); // Sum(40) + res[41] = fma52hi(res[41], a[3], a[37]); // Sum(40) + res[41] = fma52lo(res[41], a[4], a[37]); // Sum(41) + res[42] = fma52hi(res[42], a[4], a[37]); // Sum(41) + res[42] = fma52lo(res[42], a[5], a[37]); // Sum(42) + res[43] = fma52hi(res[43], a[5], a[37]); // Sum(42) + res[43] = fma52lo(res[43], a[6], a[37]); // Sum(43) + res[44] = fma52hi(res[44], a[6], a[37]); // Sum(43) + res[44] = fma52lo(res[44], a[7], a[37]); // Sum(44) + res[45] = fma52hi(res[45], a[7], a[37]); // Sum(44) + res[45] = fma52lo(res[45], a[8], a[37]); // Sum(45) + res[46] = fma52hi(res[46], a[8], a[37]); // Sum(45) + res[46] = fma52lo(res[46], a[9], a[37]); // Sum(46) + res[47] = fma52hi(res[47], a[9], a[37]); // Sum(46) + res[47] = fma52lo(res[47], a[10], a[37]); // Sum(47) + res[48] = fma52hi(res[48], a[10], a[37]); // Sum(47) + res[38] = fma52lo(res[38], a[0], a[38]); // Sum(38) + res[39] = fma52hi(res[39], a[0], a[38]); // Sum(38) + res[39] = fma52lo(res[39], a[1], a[38]); // Sum(39) + res[40] = fma52hi(res[40], a[1], a[38]); // Sum(39) + res[40] = fma52lo(res[40], a[2], a[38]); // Sum(40) + res[41] = fma52hi(res[41], a[2], a[38]); // Sum(40) + res[41] = fma52lo(res[41], a[3], a[38]); // Sum(41) + res[42] = fma52hi(res[42], a[3], a[38]); // Sum(41) + res[42] = fma52lo(res[42], a[4], a[38]); // Sum(42) + res[43] = fma52hi(res[43], a[4], a[38]); // Sum(42) + res[43] = fma52lo(res[43], a[5], a[38]); // Sum(43) + res[44] = fma52hi(res[44], a[5], a[38]); // Sum(43) + res[44] = fma52lo(res[44], a[6], a[38]); // Sum(44) + res[45] = fma52hi(res[45], a[6], a[38]); // Sum(44) + res[45] = fma52lo(res[45], a[7], a[38]); // Sum(45) + res[46] = fma52hi(res[46], a[7], a[38]); // Sum(45) + res[46] = fma52lo(res[46], a[8], a[38]); // Sum(46) + res[47] = fma52hi(res[47], a[8], a[38]); // Sum(46) + res[47] = fma52lo(res[47], a[9], a[38]); // Sum(47) + res[48] = fma52hi(res[48], a[9], a[38]); // Sum(47) + res[39] = fma52lo(res[39], a[0], a[39]); // Sum(39) + res[40] = fma52hi(res[40], a[0], a[39]); // Sum(39) + res[40] = fma52lo(res[40], a[1], a[39]); // Sum(40) + res[41] = fma52hi(res[41], a[1], a[39]); // Sum(40) + res[41] = fma52lo(res[41], a[2], a[39]); // Sum(41) + res[42] = fma52hi(res[42], a[2], a[39]); // Sum(41) + res[42] = fma52lo(res[42], a[3], a[39]); // Sum(42) + res[43] = fma52hi(res[43], a[3], a[39]); // Sum(42) + res[43] = fma52lo(res[43], a[4], a[39]); // Sum(43) + res[44] = fma52hi(res[44], a[4], a[39]); // Sum(43) + res[44] = fma52lo(res[44], a[5], a[39]); // Sum(44) + res[45] = fma52hi(res[45], a[5], a[39]); // Sum(44) + res[45] = fma52lo(res[45], a[6], a[39]); // Sum(45) + res[46] = fma52hi(res[46], a[6], a[39]); // Sum(45) + res[46] = fma52lo(res[46], a[7], a[39]); // Sum(46) + res[47] = fma52hi(res[47], a[7], a[39]); // Sum(46) + res[47] = fma52lo(res[47], a[8], a[39]); // Sum(47) + res[48] = fma52hi(res[48], a[8], a[39]); // Sum(47) + res[40] = fma52lo(res[40], a[0], a[40]); // Sum(40) + res[41] = fma52hi(res[41], a[0], a[40]); // Sum(40) + res[41] = fma52lo(res[41], a[1], a[40]); // Sum(41) + res[42] = fma52hi(res[42], a[1], a[40]); // Sum(41) + res[42] = fma52lo(res[42], a[2], a[40]); // Sum(42) + res[43] = fma52hi(res[43], a[2], a[40]); // Sum(42) + res[43] = fma52lo(res[43], a[3], a[40]); // Sum(43) + res[44] = fma52hi(res[44], a[3], a[40]); // Sum(43) + res[44] = fma52lo(res[44], a[4], a[40]); // Sum(44) + res[45] = fma52hi(res[45], a[4], a[40]); // Sum(44) + res[45] = fma52lo(res[45], a[5], a[40]); // Sum(45) + res[46] = fma52hi(res[46], a[5], a[40]); // Sum(45) + res[46] = fma52lo(res[46], a[6], a[40]); // Sum(46) + res[47] = fma52hi(res[47], a[6], a[40]); // Sum(46) + res[47] = fma52lo(res[47], a[7], a[40]); // Sum(47) + res[48] = fma52hi(res[48], a[7], a[40]); // Sum(47) + res[41] = fma52lo(res[41], a[0], a[41]); // Sum(41) + res[42] = fma52hi(res[42], a[0], a[41]); // Sum(41) + res[42] = fma52lo(res[42], a[1], a[41]); // Sum(42) + res[43] = fma52hi(res[43], a[1], a[41]); // Sum(42) + res[43] = fma52lo(res[43], a[2], a[41]); // Sum(43) + res[44] = fma52hi(res[44], a[2], a[41]); // Sum(43) + res[44] = fma52lo(res[44], a[3], a[41]); // Sum(44) + res[45] = fma52hi(res[45], a[3], a[41]); // Sum(44) + res[45] = fma52lo(res[45], a[4], a[41]); // Sum(45) + res[46] = fma52hi(res[46], a[4], a[41]); // Sum(45) + res[46] = fma52lo(res[46], a[5], a[41]); // Sum(46) + res[47] = fma52hi(res[47], a[5], a[41]); // Sum(46) + res[47] = fma52lo(res[47], a[6], a[41]); // Sum(47) + res[48] = fma52hi(res[48], a[6], a[41]); // Sum(47) + res[42] = fma52lo(res[42], a[0], a[42]); // Sum(42) + res[43] = fma52hi(res[43], a[0], a[42]); // Sum(42) + res[43] = fma52lo(res[43], a[1], a[42]); // Sum(43) + res[44] = fma52hi(res[44], a[1], a[42]); // Sum(43) + res[44] = fma52lo(res[44], a[2], a[42]); // Sum(44) + res[45] = fma52hi(res[45], a[2], a[42]); // Sum(44) + res[45] = fma52lo(res[45], a[3], a[42]); // Sum(45) + res[46] = fma52hi(res[46], a[3], a[42]); // Sum(45) + res[46] = fma52lo(res[46], a[4], a[42]); // Sum(46) + res[47] = fma52hi(res[47], a[4], a[42]); // Sum(46) + res[47] = fma52lo(res[47], a[5], a[42]); // Sum(47) + res[48] = fma52hi(res[48], a[5], a[42]); // Sum(47) + res[43] = fma52lo(res[43], a[0], a[43]); // Sum(43) + res[44] = fma52hi(res[44], a[0], a[43]); // Sum(43) + res[44] = fma52lo(res[44], a[1], a[43]); // Sum(44) + res[45] = fma52hi(res[45], a[1], a[43]); // Sum(44) + res[45] = fma52lo(res[45], a[2], a[43]); // Sum(45) + res[46] = fma52hi(res[46], a[2], a[43]); // Sum(45) + res[46] = fma52lo(res[46], a[3], a[43]); // Sum(46) + res[47] = fma52hi(res[47], a[3], a[43]); // Sum(46) + res[47] = fma52lo(res[47], a[4], a[43]); // Sum(47) + res[48] = fma52hi(res[48], a[4], a[43]); // Sum(47) + res[44] = fma52lo(res[44], a[0], a[44]); // Sum(44) + res[45] = fma52hi(res[45], a[0], a[44]); // Sum(44) + res[45] = fma52lo(res[45], a[1], a[44]); // Sum(45) + res[46] = fma52hi(res[46], a[1], a[44]); // Sum(45) + res[46] = fma52lo(res[46], a[2], a[44]); // Sum(46) + res[47] = fma52hi(res[47], a[2], a[44]); // Sum(46) + res[47] = fma52lo(res[47], a[3], a[44]); // Sum(47) + res[48] = fma52hi(res[48], a[3], a[44]); // Sum(47) + res[45] = fma52lo(res[45], a[0], a[45]); // Sum(45) + res[46] = fma52hi(res[46], a[0], a[45]); // Sum(45) + res[46] = fma52lo(res[46], a[1], a[45]); // Sum(46) + res[47] = fma52hi(res[47], a[1], a[45]); // Sum(46) + res[47] = fma52lo(res[47], a[2], a[45]); // Sum(47) + res[48] = fma52hi(res[48], a[2], a[45]); // Sum(47) + res[46] = fma52lo(res[46], a[0], a[46]); // Sum(46) + res[47] = fma52hi(res[47], a[0], a[46]); // Sum(46) + res[47] = fma52lo(res[47], a[1], a[46]); // Sum(47) + res[48] = fma52hi(res[48], a[1], a[46]); // Sum(47) + res[47] = fma52lo(res[47], a[0], a[47]); // Sum(47) + res[48] = fma52hi(res[48], a[0], a[47]); // Sum(47) + res[36] = add64(res[36], res[36]); // Double(36) + res[37] = add64(res[37], res[37]); // Double(37) + res[38] = add64(res[38], res[38]); // Double(38) + res[39] = add64(res[39], res[39]); // Double(39) + res[40] = add64(res[40], res[40]); // Double(40) + res[41] = add64(res[41], res[41]); // Double(41) + res[42] = add64(res[42], res[42]); // Double(42) + res[43] = add64(res[43], res[43]); // Double(43) + res[44] = add64(res[44], res[44]); // Double(44) + res[45] = add64(res[45], res[45]); // Double(45) + res[46] = add64(res[46], res[46]); // Double(46) + res[47] = add64(res[47], res[47]); // Double(47) + res[36] = fma52lo(res[36], a[18], a[18]); // Add sqr(36) + res[37] = fma52hi(res[37], a[18], a[18]); // Add sqr(36) + res[38] = fma52lo(res[38], a[19], a[19]); // Add sqr(38) + res[39] = fma52hi(res[39], a[19], a[19]); // Add sqr(38) + res[40] = fma52lo(res[40], a[20], a[20]); // Add sqr(40) + res[41] = fma52hi(res[41], a[20], a[20]); // Add sqr(40) + res[42] = fma52lo(res[42], a[21], a[21]); // Add sqr(42) + res[43] = fma52hi(res[43], a[21], a[21]); // Add sqr(42) + res[44] = fma52lo(res[44], a[22], a[22]); // Add sqr(44) + res[45] = fma52hi(res[45], a[22], a[22]); // Add sqr(44) + res[46] = fma52lo(res[46], a[23], a[23]); // Add sqr(46) + res[47] = fma52hi(res[47], a[23], a[23]); // Add sqr(46) + res[48] = fma52lo(res[48], a[23], a[25]); // Sum(48) + res[49] = fma52hi(res[49], a[23], a[25]); // Sum(48) + res[49] = fma52lo(res[49], a[24], a[25]); // Sum(49) + res[50] = fma52hi(res[50], a[24], a[25]); // Sum(49) + res[48] = fma52lo(res[48], a[22], a[26]); // Sum(48) + res[49] = fma52hi(res[49], a[22], a[26]); // Sum(48) + res[49] = fma52lo(res[49], a[23], a[26]); // Sum(49) + res[50] = fma52hi(res[50], a[23], a[26]); // Sum(49) + res[50] = fma52lo(res[50], a[24], a[26]); // Sum(50) + res[51] = fma52hi(res[51], a[24], a[26]); // Sum(50) + res[51] = fma52lo(res[51], a[25], a[26]); // Sum(51) + res[52] = fma52hi(res[52], a[25], a[26]); // Sum(51) + res[48] = fma52lo(res[48], a[21], a[27]); // Sum(48) + res[49] = fma52hi(res[49], a[21], a[27]); // Sum(48) + res[49] = fma52lo(res[49], a[22], a[27]); // Sum(49) + res[50] = fma52hi(res[50], a[22], a[27]); // Sum(49) + res[50] = fma52lo(res[50], a[23], a[27]); // Sum(50) + res[51] = fma52hi(res[51], a[23], a[27]); // Sum(50) + res[51] = fma52lo(res[51], a[24], a[27]); // Sum(51) + res[52] = fma52hi(res[52], a[24], a[27]); // Sum(51) + res[52] = fma52lo(res[52], a[25], a[27]); // Sum(52) + res[53] = fma52hi(res[53], a[25], a[27]); // Sum(52) + res[53] = fma52lo(res[53], a[26], a[27]); // Sum(53) + res[54] = fma52hi(res[54], a[26], a[27]); // Sum(53) + res[48] = fma52lo(res[48], a[20], a[28]); // Sum(48) + res[49] = fma52hi(res[49], a[20], a[28]); // Sum(48) + res[49] = fma52lo(res[49], a[21], a[28]); // Sum(49) + res[50] = fma52hi(res[50], a[21], a[28]); // Sum(49) + res[50] = fma52lo(res[50], a[22], a[28]); // Sum(50) + res[51] = fma52hi(res[51], a[22], a[28]); // Sum(50) + res[51] = fma52lo(res[51], a[23], a[28]); // Sum(51) + res[52] = fma52hi(res[52], a[23], a[28]); // Sum(51) + res[52] = fma52lo(res[52], a[24], a[28]); // Sum(52) + res[53] = fma52hi(res[53], a[24], a[28]); // Sum(52) + res[53] = fma52lo(res[53], a[25], a[28]); // Sum(53) + res[54] = fma52hi(res[54], a[25], a[28]); // Sum(53) + res[54] = fma52lo(res[54], a[26], a[28]); // Sum(54) + res[55] = fma52hi(res[55], a[26], a[28]); // Sum(54) + res[55] = fma52lo(res[55], a[27], a[28]); // Sum(55) + res[56] = fma52hi(res[56], a[27], a[28]); // Sum(55) + res[48] = fma52lo(res[48], a[19], a[29]); // Sum(48) + res[49] = fma52hi(res[49], a[19], a[29]); // Sum(48) + res[49] = fma52lo(res[49], a[20], a[29]); // Sum(49) + res[50] = fma52hi(res[50], a[20], a[29]); // Sum(49) + res[50] = fma52lo(res[50], a[21], a[29]); // Sum(50) + res[51] = fma52hi(res[51], a[21], a[29]); // Sum(50) + res[51] = fma52lo(res[51], a[22], a[29]); // Sum(51) + res[52] = fma52hi(res[52], a[22], a[29]); // Sum(51) + res[52] = fma52lo(res[52], a[23], a[29]); // Sum(52) + res[53] = fma52hi(res[53], a[23], a[29]); // Sum(52) + res[53] = fma52lo(res[53], a[24], a[29]); // Sum(53) + res[54] = fma52hi(res[54], a[24], a[29]); // Sum(53) + res[54] = fma52lo(res[54], a[25], a[29]); // Sum(54) + res[55] = fma52hi(res[55], a[25], a[29]); // Sum(54) + res[55] = fma52lo(res[55], a[26], a[29]); // Sum(55) + res[56] = fma52hi(res[56], a[26], a[29]); // Sum(55) + res[56] = fma52lo(res[56], a[27], a[29]); // Sum(56) + res[57] = fma52hi(res[57], a[27], a[29]); // Sum(56) + res[57] = fma52lo(res[57], a[28], a[29]); // Sum(57) + res[58] = fma52hi(res[58], a[28], a[29]); // Sum(57) + res[48] = fma52lo(res[48], a[18], a[30]); // Sum(48) + res[49] = fma52hi(res[49], a[18], a[30]); // Sum(48) + res[49] = fma52lo(res[49], a[19], a[30]); // Sum(49) + res[50] = fma52hi(res[50], a[19], a[30]); // Sum(49) + res[50] = fma52lo(res[50], a[20], a[30]); // Sum(50) + res[51] = fma52hi(res[51], a[20], a[30]); // Sum(50) + res[51] = fma52lo(res[51], a[21], a[30]); // Sum(51) + res[52] = fma52hi(res[52], a[21], a[30]); // Sum(51) + res[52] = fma52lo(res[52], a[22], a[30]); // Sum(52) + res[53] = fma52hi(res[53], a[22], a[30]); // Sum(52) + res[53] = fma52lo(res[53], a[23], a[30]); // Sum(53) + res[54] = fma52hi(res[54], a[23], a[30]); // Sum(53) + res[54] = fma52lo(res[54], a[24], a[30]); // Sum(54) + res[55] = fma52hi(res[55], a[24], a[30]); // Sum(54) + res[55] = fma52lo(res[55], a[25], a[30]); // Sum(55) + res[56] = fma52hi(res[56], a[25], a[30]); // Sum(55) + res[56] = fma52lo(res[56], a[26], a[30]); // Sum(56) + res[57] = fma52hi(res[57], a[26], a[30]); // Sum(56) + res[57] = fma52lo(res[57], a[27], a[30]); // Sum(57) + res[58] = fma52hi(res[58], a[27], a[30]); // Sum(57) + res[58] = fma52lo(res[58], a[28], a[30]); // Sum(58) + res[59] = fma52hi(res[59], a[28], a[30]); // Sum(58) + res[59] = fma52lo(res[59], a[29], a[30]); // Sum(59) + res[60] = fma52hi(res[60], a[29], a[30]); // Sum(59) + res[48] = fma52lo(res[48], a[17], a[31]); // Sum(48) + res[49] = fma52hi(res[49], a[17], a[31]); // Sum(48) + res[49] = fma52lo(res[49], a[18], a[31]); // Sum(49) + res[50] = fma52hi(res[50], a[18], a[31]); // Sum(49) + res[50] = fma52lo(res[50], a[19], a[31]); // Sum(50) + res[51] = fma52hi(res[51], a[19], a[31]); // Sum(50) + res[51] = fma52lo(res[51], a[20], a[31]); // Sum(51) + res[52] = fma52hi(res[52], a[20], a[31]); // Sum(51) + res[52] = fma52lo(res[52], a[21], a[31]); // Sum(52) + res[53] = fma52hi(res[53], a[21], a[31]); // Sum(52) + res[53] = fma52lo(res[53], a[22], a[31]); // Sum(53) + res[54] = fma52hi(res[54], a[22], a[31]); // Sum(53) + res[54] = fma52lo(res[54], a[23], a[31]); // Sum(54) + res[55] = fma52hi(res[55], a[23], a[31]); // Sum(54) + res[55] = fma52lo(res[55], a[24], a[31]); // Sum(55) + res[56] = fma52hi(res[56], a[24], a[31]); // Sum(55) + res[56] = fma52lo(res[56], a[25], a[31]); // Sum(56) + res[57] = fma52hi(res[57], a[25], a[31]); // Sum(56) + res[57] = fma52lo(res[57], a[26], a[31]); // Sum(57) + res[58] = fma52hi(res[58], a[26], a[31]); // Sum(57) + res[58] = fma52lo(res[58], a[27], a[31]); // Sum(58) + res[59] = fma52hi(res[59], a[27], a[31]); // Sum(58) + res[59] = fma52lo(res[59], a[28], a[31]); // Sum(59) + res[60] = fma52hi(res[60], a[28], a[31]); // Sum(59) + res[48] = fma52lo(res[48], a[16], a[32]); // Sum(48) + res[49] = fma52hi(res[49], a[16], a[32]); // Sum(48) + res[49] = fma52lo(res[49], a[17], a[32]); // Sum(49) + res[50] = fma52hi(res[50], a[17], a[32]); // Sum(49) + res[50] = fma52lo(res[50], a[18], a[32]); // Sum(50) + res[51] = fma52hi(res[51], a[18], a[32]); // Sum(50) + res[51] = fma52lo(res[51], a[19], a[32]); // Sum(51) + res[52] = fma52hi(res[52], a[19], a[32]); // Sum(51) + res[52] = fma52lo(res[52], a[20], a[32]); // Sum(52) + res[53] = fma52hi(res[53], a[20], a[32]); // Sum(52) + res[53] = fma52lo(res[53], a[21], a[32]); // Sum(53) + res[54] = fma52hi(res[54], a[21], a[32]); // Sum(53) + res[54] = fma52lo(res[54], a[22], a[32]); // Sum(54) + res[55] = fma52hi(res[55], a[22], a[32]); // Sum(54) + res[55] = fma52lo(res[55], a[23], a[32]); // Sum(55) + res[56] = fma52hi(res[56], a[23], a[32]); // Sum(55) + res[56] = fma52lo(res[56], a[24], a[32]); // Sum(56) + res[57] = fma52hi(res[57], a[24], a[32]); // Sum(56) + res[57] = fma52lo(res[57], a[25], a[32]); // Sum(57) + res[58] = fma52hi(res[58], a[25], a[32]); // Sum(57) + res[58] = fma52lo(res[58], a[26], a[32]); // Sum(58) + res[59] = fma52hi(res[59], a[26], a[32]); // Sum(58) + res[59] = fma52lo(res[59], a[27], a[32]); // Sum(59) + res[60] = fma52hi(res[60], a[27], a[32]); // Sum(59) + res[48] = fma52lo(res[48], a[15], a[33]); // Sum(48) + res[49] = fma52hi(res[49], a[15], a[33]); // Sum(48) + res[49] = fma52lo(res[49], a[16], a[33]); // Sum(49) + res[50] = fma52hi(res[50], a[16], a[33]); // Sum(49) + res[50] = fma52lo(res[50], a[17], a[33]); // Sum(50) + res[51] = fma52hi(res[51], a[17], a[33]); // Sum(50) + res[51] = fma52lo(res[51], a[18], a[33]); // Sum(51) + res[52] = fma52hi(res[52], a[18], a[33]); // Sum(51) + res[52] = fma52lo(res[52], a[19], a[33]); // Sum(52) + res[53] = fma52hi(res[53], a[19], a[33]); // Sum(52) + res[53] = fma52lo(res[53], a[20], a[33]); // Sum(53) + res[54] = fma52hi(res[54], a[20], a[33]); // Sum(53) + res[54] = fma52lo(res[54], a[21], a[33]); // Sum(54) + res[55] = fma52hi(res[55], a[21], a[33]); // Sum(54) + res[55] = fma52lo(res[55], a[22], a[33]); // Sum(55) + res[56] = fma52hi(res[56], a[22], a[33]); // Sum(55) + res[56] = fma52lo(res[56], a[23], a[33]); // Sum(56) + res[57] = fma52hi(res[57], a[23], a[33]); // Sum(56) + res[57] = fma52lo(res[57], a[24], a[33]); // Sum(57) + res[58] = fma52hi(res[58], a[24], a[33]); // Sum(57) + res[58] = fma52lo(res[58], a[25], a[33]); // Sum(58) + res[59] = fma52hi(res[59], a[25], a[33]); // Sum(58) + res[59] = fma52lo(res[59], a[26], a[33]); // Sum(59) + res[60] = fma52hi(res[60], a[26], a[33]); // Sum(59) + res[48] = fma52lo(res[48], a[14], a[34]); // Sum(48) + res[49] = fma52hi(res[49], a[14], a[34]); // Sum(48) + res[49] = fma52lo(res[49], a[15], a[34]); // Sum(49) + res[50] = fma52hi(res[50], a[15], a[34]); // Sum(49) + res[50] = fma52lo(res[50], a[16], a[34]); // Sum(50) + res[51] = fma52hi(res[51], a[16], a[34]); // Sum(50) + res[51] = fma52lo(res[51], a[17], a[34]); // Sum(51) + res[52] = fma52hi(res[52], a[17], a[34]); // Sum(51) + res[52] = fma52lo(res[52], a[18], a[34]); // Sum(52) + res[53] = fma52hi(res[53], a[18], a[34]); // Sum(52) + res[53] = fma52lo(res[53], a[19], a[34]); // Sum(53) + res[54] = fma52hi(res[54], a[19], a[34]); // Sum(53) + res[54] = fma52lo(res[54], a[20], a[34]); // Sum(54) + res[55] = fma52hi(res[55], a[20], a[34]); // Sum(54) + res[55] = fma52lo(res[55], a[21], a[34]); // Sum(55) + res[56] = fma52hi(res[56], a[21], a[34]); // Sum(55) + res[56] = fma52lo(res[56], a[22], a[34]); // Sum(56) + res[57] = fma52hi(res[57], a[22], a[34]); // Sum(56) + res[57] = fma52lo(res[57], a[23], a[34]); // Sum(57) + res[58] = fma52hi(res[58], a[23], a[34]); // Sum(57) + res[58] = fma52lo(res[58], a[24], a[34]); // Sum(58) + res[59] = fma52hi(res[59], a[24], a[34]); // Sum(58) + res[59] = fma52lo(res[59], a[25], a[34]); // Sum(59) + res[60] = fma52hi(res[60], a[25], a[34]); // Sum(59) + res[48] = fma52lo(res[48], a[13], a[35]); // Sum(48) + res[49] = fma52hi(res[49], a[13], a[35]); // Sum(48) + res[49] = fma52lo(res[49], a[14], a[35]); // Sum(49) + res[50] = fma52hi(res[50], a[14], a[35]); // Sum(49) + res[50] = fma52lo(res[50], a[15], a[35]); // Sum(50) + res[51] = fma52hi(res[51], a[15], a[35]); // Sum(50) + res[51] = fma52lo(res[51], a[16], a[35]); // Sum(51) + res[52] = fma52hi(res[52], a[16], a[35]); // Sum(51) + res[52] = fma52lo(res[52], a[17], a[35]); // Sum(52) + res[53] = fma52hi(res[53], a[17], a[35]); // Sum(52) + res[53] = fma52lo(res[53], a[18], a[35]); // Sum(53) + res[54] = fma52hi(res[54], a[18], a[35]); // Sum(53) + res[54] = fma52lo(res[54], a[19], a[35]); // Sum(54) + res[55] = fma52hi(res[55], a[19], a[35]); // Sum(54) + res[55] = fma52lo(res[55], a[20], a[35]); // Sum(55) + res[56] = fma52hi(res[56], a[20], a[35]); // Sum(55) + res[56] = fma52lo(res[56], a[21], a[35]); // Sum(56) + res[57] = fma52hi(res[57], a[21], a[35]); // Sum(56) + res[57] = fma52lo(res[57], a[22], a[35]); // Sum(57) + res[58] = fma52hi(res[58], a[22], a[35]); // Sum(57) + res[58] = fma52lo(res[58], a[23], a[35]); // Sum(58) + res[59] = fma52hi(res[59], a[23], a[35]); // Sum(58) + res[59] = fma52lo(res[59], a[24], a[35]); // Sum(59) + res[60] = fma52hi(res[60], a[24], a[35]); // Sum(59) + res[48] = fma52lo(res[48], a[12], a[36]); // Sum(48) + res[49] = fma52hi(res[49], a[12], a[36]); // Sum(48) + res[49] = fma52lo(res[49], a[13], a[36]); // Sum(49) + res[50] = fma52hi(res[50], a[13], a[36]); // Sum(49) + res[50] = fma52lo(res[50], a[14], a[36]); // Sum(50) + res[51] = fma52hi(res[51], a[14], a[36]); // Sum(50) + res[51] = fma52lo(res[51], a[15], a[36]); // Sum(51) + res[52] = fma52hi(res[52], a[15], a[36]); // Sum(51) + res[52] = fma52lo(res[52], a[16], a[36]); // Sum(52) + res[53] = fma52hi(res[53], a[16], a[36]); // Sum(52) + res[53] = fma52lo(res[53], a[17], a[36]); // Sum(53) + res[54] = fma52hi(res[54], a[17], a[36]); // Sum(53) + res[54] = fma52lo(res[54], a[18], a[36]); // Sum(54) + res[55] = fma52hi(res[55], a[18], a[36]); // Sum(54) + res[55] = fma52lo(res[55], a[19], a[36]); // Sum(55) + res[56] = fma52hi(res[56], a[19], a[36]); // Sum(55) + res[56] = fma52lo(res[56], a[20], a[36]); // Sum(56) + res[57] = fma52hi(res[57], a[20], a[36]); // Sum(56) + res[57] = fma52lo(res[57], a[21], a[36]); // Sum(57) + res[58] = fma52hi(res[58], a[21], a[36]); // Sum(57) + res[58] = fma52lo(res[58], a[22], a[36]); // Sum(58) + res[59] = fma52hi(res[59], a[22], a[36]); // Sum(58) + res[59] = fma52lo(res[59], a[23], a[36]); // Sum(59) + res[60] = fma52hi(res[60], a[23], a[36]); // Sum(59) + res[48] = fma52lo(res[48], a[11], a[37]); // Sum(48) + res[49] = fma52hi(res[49], a[11], a[37]); // Sum(48) + res[49] = fma52lo(res[49], a[12], a[37]); // Sum(49) + res[50] = fma52hi(res[50], a[12], a[37]); // Sum(49) + res[50] = fma52lo(res[50], a[13], a[37]); // Sum(50) + res[51] = fma52hi(res[51], a[13], a[37]); // Sum(50) + res[51] = fma52lo(res[51], a[14], a[37]); // Sum(51) + res[52] = fma52hi(res[52], a[14], a[37]); // Sum(51) + res[52] = fma52lo(res[52], a[15], a[37]); // Sum(52) + res[53] = fma52hi(res[53], a[15], a[37]); // Sum(52) + res[53] = fma52lo(res[53], a[16], a[37]); // Sum(53) + res[54] = fma52hi(res[54], a[16], a[37]); // Sum(53) + res[54] = fma52lo(res[54], a[17], a[37]); // Sum(54) + res[55] = fma52hi(res[55], a[17], a[37]); // Sum(54) + res[55] = fma52lo(res[55], a[18], a[37]); // Sum(55) + res[56] = fma52hi(res[56], a[18], a[37]); // Sum(55) + res[56] = fma52lo(res[56], a[19], a[37]); // Sum(56) + res[57] = fma52hi(res[57], a[19], a[37]); // Sum(56) + res[57] = fma52lo(res[57], a[20], a[37]); // Sum(57) + res[58] = fma52hi(res[58], a[20], a[37]); // Sum(57) + res[58] = fma52lo(res[58], a[21], a[37]); // Sum(58) + res[59] = fma52hi(res[59], a[21], a[37]); // Sum(58) + res[59] = fma52lo(res[59], a[22], a[37]); // Sum(59) + res[60] = fma52hi(res[60], a[22], a[37]); // Sum(59) + res[48] = fma52lo(res[48], a[10], a[38]); // Sum(48) + res[49] = fma52hi(res[49], a[10], a[38]); // Sum(48) + res[49] = fma52lo(res[49], a[11], a[38]); // Sum(49) + res[50] = fma52hi(res[50], a[11], a[38]); // Sum(49) + res[50] = fma52lo(res[50], a[12], a[38]); // Sum(50) + res[51] = fma52hi(res[51], a[12], a[38]); // Sum(50) + res[51] = fma52lo(res[51], a[13], a[38]); // Sum(51) + res[52] = fma52hi(res[52], a[13], a[38]); // Sum(51) + res[52] = fma52lo(res[52], a[14], a[38]); // Sum(52) + res[53] = fma52hi(res[53], a[14], a[38]); // Sum(52) + res[53] = fma52lo(res[53], a[15], a[38]); // Sum(53) + res[54] = fma52hi(res[54], a[15], a[38]); // Sum(53) + res[54] = fma52lo(res[54], a[16], a[38]); // Sum(54) + res[55] = fma52hi(res[55], a[16], a[38]); // Sum(54) + res[55] = fma52lo(res[55], a[17], a[38]); // Sum(55) + res[56] = fma52hi(res[56], a[17], a[38]); // Sum(55) + res[56] = fma52lo(res[56], a[18], a[38]); // Sum(56) + res[57] = fma52hi(res[57], a[18], a[38]); // Sum(56) + res[57] = fma52lo(res[57], a[19], a[38]); // Sum(57) + res[58] = fma52hi(res[58], a[19], a[38]); // Sum(57) + res[58] = fma52lo(res[58], a[20], a[38]); // Sum(58) + res[59] = fma52hi(res[59], a[20], a[38]); // Sum(58) + res[59] = fma52lo(res[59], a[21], a[38]); // Sum(59) + res[60] = fma52hi(res[60], a[21], a[38]); // Sum(59) + res[48] = fma52lo(res[48], a[9], a[39]); // Sum(48) + res[49] = fma52hi(res[49], a[9], a[39]); // Sum(48) + res[49] = fma52lo(res[49], a[10], a[39]); // Sum(49) + res[50] = fma52hi(res[50], a[10], a[39]); // Sum(49) + res[50] = fma52lo(res[50], a[11], a[39]); // Sum(50) + res[51] = fma52hi(res[51], a[11], a[39]); // Sum(50) + res[51] = fma52lo(res[51], a[12], a[39]); // Sum(51) + res[52] = fma52hi(res[52], a[12], a[39]); // Sum(51) + res[52] = fma52lo(res[52], a[13], a[39]); // Sum(52) + res[53] = fma52hi(res[53], a[13], a[39]); // Sum(52) + res[53] = fma52lo(res[53], a[14], a[39]); // Sum(53) + res[54] = fma52hi(res[54], a[14], a[39]); // Sum(53) + res[54] = fma52lo(res[54], a[15], a[39]); // Sum(54) + res[55] = fma52hi(res[55], a[15], a[39]); // Sum(54) + res[55] = fma52lo(res[55], a[16], a[39]); // Sum(55) + res[56] = fma52hi(res[56], a[16], a[39]); // Sum(55) + res[56] = fma52lo(res[56], a[17], a[39]); // Sum(56) + res[57] = fma52hi(res[57], a[17], a[39]); // Sum(56) + res[57] = fma52lo(res[57], a[18], a[39]); // Sum(57) + res[58] = fma52hi(res[58], a[18], a[39]); // Sum(57) + res[58] = fma52lo(res[58], a[19], a[39]); // Sum(58) + res[59] = fma52hi(res[59], a[19], a[39]); // Sum(58) + res[59] = fma52lo(res[59], a[20], a[39]); // Sum(59) + res[60] = fma52hi(res[60], a[20], a[39]); // Sum(59) + res[48] = fma52lo(res[48], a[8], a[40]); // Sum(48) + res[49] = fma52hi(res[49], a[8], a[40]); // Sum(48) + res[49] = fma52lo(res[49], a[9], a[40]); // Sum(49) + res[50] = fma52hi(res[50], a[9], a[40]); // Sum(49) + res[50] = fma52lo(res[50], a[10], a[40]); // Sum(50) + res[51] = fma52hi(res[51], a[10], a[40]); // Sum(50) + res[51] = fma52lo(res[51], a[11], a[40]); // Sum(51) + res[52] = fma52hi(res[52], a[11], a[40]); // Sum(51) + res[52] = fma52lo(res[52], a[12], a[40]); // Sum(52) + res[53] = fma52hi(res[53], a[12], a[40]); // Sum(52) + res[53] = fma52lo(res[53], a[13], a[40]); // Sum(53) + res[54] = fma52hi(res[54], a[13], a[40]); // Sum(53) + res[54] = fma52lo(res[54], a[14], a[40]); // Sum(54) + res[55] = fma52hi(res[55], a[14], a[40]); // Sum(54) + res[55] = fma52lo(res[55], a[15], a[40]); // Sum(55) + res[56] = fma52hi(res[56], a[15], a[40]); // Sum(55) + res[56] = fma52lo(res[56], a[16], a[40]); // Sum(56) + res[57] = fma52hi(res[57], a[16], a[40]); // Sum(56) + res[57] = fma52lo(res[57], a[17], a[40]); // Sum(57) + res[58] = fma52hi(res[58], a[17], a[40]); // Sum(57) + res[58] = fma52lo(res[58], a[18], a[40]); // Sum(58) + res[59] = fma52hi(res[59], a[18], a[40]); // Sum(58) + res[59] = fma52lo(res[59], a[19], a[40]); // Sum(59) + res[60] = fma52hi(res[60], a[19], a[40]); // Sum(59) + res[48] = fma52lo(res[48], a[7], a[41]); // Sum(48) + res[49] = fma52hi(res[49], a[7], a[41]); // Sum(48) + res[49] = fma52lo(res[49], a[8], a[41]); // Sum(49) + res[50] = fma52hi(res[50], a[8], a[41]); // Sum(49) + res[50] = fma52lo(res[50], a[9], a[41]); // Sum(50) + res[51] = fma52hi(res[51], a[9], a[41]); // Sum(50) + res[51] = fma52lo(res[51], a[10], a[41]); // Sum(51) + res[52] = fma52hi(res[52], a[10], a[41]); // Sum(51) + res[52] = fma52lo(res[52], a[11], a[41]); // Sum(52) + res[53] = fma52hi(res[53], a[11], a[41]); // Sum(52) + res[53] = fma52lo(res[53], a[12], a[41]); // Sum(53) + res[54] = fma52hi(res[54], a[12], a[41]); // Sum(53) + res[54] = fma52lo(res[54], a[13], a[41]); // Sum(54) + res[55] = fma52hi(res[55], a[13], a[41]); // Sum(54) + res[55] = fma52lo(res[55], a[14], a[41]); // Sum(55) + res[56] = fma52hi(res[56], a[14], a[41]); // Sum(55) + res[56] = fma52lo(res[56], a[15], a[41]); // Sum(56) + res[57] = fma52hi(res[57], a[15], a[41]); // Sum(56) + res[57] = fma52lo(res[57], a[16], a[41]); // Sum(57) + res[58] = fma52hi(res[58], a[16], a[41]); // Sum(57) + res[58] = fma52lo(res[58], a[17], a[41]); // Sum(58) + res[59] = fma52hi(res[59], a[17], a[41]); // Sum(58) + res[59] = fma52lo(res[59], a[18], a[41]); // Sum(59) + res[60] = fma52hi(res[60], a[18], a[41]); // Sum(59) + res[48] = fma52lo(res[48], a[6], a[42]); // Sum(48) + res[49] = fma52hi(res[49], a[6], a[42]); // Sum(48) + res[49] = fma52lo(res[49], a[7], a[42]); // Sum(49) + res[50] = fma52hi(res[50], a[7], a[42]); // Sum(49) + res[50] = fma52lo(res[50], a[8], a[42]); // Sum(50) + res[51] = fma52hi(res[51], a[8], a[42]); // Sum(50) + res[51] = fma52lo(res[51], a[9], a[42]); // Sum(51) + res[52] = fma52hi(res[52], a[9], a[42]); // Sum(51) + res[52] = fma52lo(res[52], a[10], a[42]); // Sum(52) + res[53] = fma52hi(res[53], a[10], a[42]); // Sum(52) + res[53] = fma52lo(res[53], a[11], a[42]); // Sum(53) + res[54] = fma52hi(res[54], a[11], a[42]); // Sum(53) + res[54] = fma52lo(res[54], a[12], a[42]); // Sum(54) + res[55] = fma52hi(res[55], a[12], a[42]); // Sum(54) + res[55] = fma52lo(res[55], a[13], a[42]); // Sum(55) + res[56] = fma52hi(res[56], a[13], a[42]); // Sum(55) + res[56] = fma52lo(res[56], a[14], a[42]); // Sum(56) + res[57] = fma52hi(res[57], a[14], a[42]); // Sum(56) + res[57] = fma52lo(res[57], a[15], a[42]); // Sum(57) + res[58] = fma52hi(res[58], a[15], a[42]); // Sum(57) + res[58] = fma52lo(res[58], a[16], a[42]); // Sum(58) + res[59] = fma52hi(res[59], a[16], a[42]); // Sum(58) + res[59] = fma52lo(res[59], a[17], a[42]); // Sum(59) + res[60] = fma52hi(res[60], a[17], a[42]); // Sum(59) + res[48] = fma52lo(res[48], a[5], a[43]); // Sum(48) + res[49] = fma52hi(res[49], a[5], a[43]); // Sum(48) + res[49] = fma52lo(res[49], a[6], a[43]); // Sum(49) + res[50] = fma52hi(res[50], a[6], a[43]); // Sum(49) + res[50] = fma52lo(res[50], a[7], a[43]); // Sum(50) + res[51] = fma52hi(res[51], a[7], a[43]); // Sum(50) + res[51] = fma52lo(res[51], a[8], a[43]); // Sum(51) + res[52] = fma52hi(res[52], a[8], a[43]); // Sum(51) + res[52] = fma52lo(res[52], a[9], a[43]); // Sum(52) + res[53] = fma52hi(res[53], a[9], a[43]); // Sum(52) + res[53] = fma52lo(res[53], a[10], a[43]); // Sum(53) + res[54] = fma52hi(res[54], a[10], a[43]); // Sum(53) + res[54] = fma52lo(res[54], a[11], a[43]); // Sum(54) + res[55] = fma52hi(res[55], a[11], a[43]); // Sum(54) + res[55] = fma52lo(res[55], a[12], a[43]); // Sum(55) + res[56] = fma52hi(res[56], a[12], a[43]); // Sum(55) + res[56] = fma52lo(res[56], a[13], a[43]); // Sum(56) + res[57] = fma52hi(res[57], a[13], a[43]); // Sum(56) + res[57] = fma52lo(res[57], a[14], a[43]); // Sum(57) + res[58] = fma52hi(res[58], a[14], a[43]); // Sum(57) + res[58] = fma52lo(res[58], a[15], a[43]); // Sum(58) + res[59] = fma52hi(res[59], a[15], a[43]); // Sum(58) + res[59] = fma52lo(res[59], a[16], a[43]); // Sum(59) + res[60] = fma52hi(res[60], a[16], a[43]); // Sum(59) + res[48] = fma52lo(res[48], a[4], a[44]); // Sum(48) + res[49] = fma52hi(res[49], a[4], a[44]); // Sum(48) + res[49] = fma52lo(res[49], a[5], a[44]); // Sum(49) + res[50] = fma52hi(res[50], a[5], a[44]); // Sum(49) + res[50] = fma52lo(res[50], a[6], a[44]); // Sum(50) + res[51] = fma52hi(res[51], a[6], a[44]); // Sum(50) + res[51] = fma52lo(res[51], a[7], a[44]); // Sum(51) + res[52] = fma52hi(res[52], a[7], a[44]); // Sum(51) + res[52] = fma52lo(res[52], a[8], a[44]); // Sum(52) + res[53] = fma52hi(res[53], a[8], a[44]); // Sum(52) + res[53] = fma52lo(res[53], a[9], a[44]); // Sum(53) + res[54] = fma52hi(res[54], a[9], a[44]); // Sum(53) + res[54] = fma52lo(res[54], a[10], a[44]); // Sum(54) + res[55] = fma52hi(res[55], a[10], a[44]); // Sum(54) + res[55] = fma52lo(res[55], a[11], a[44]); // Sum(55) + res[56] = fma52hi(res[56], a[11], a[44]); // Sum(55) + res[56] = fma52lo(res[56], a[12], a[44]); // Sum(56) + res[57] = fma52hi(res[57], a[12], a[44]); // Sum(56) + res[57] = fma52lo(res[57], a[13], a[44]); // Sum(57) + res[58] = fma52hi(res[58], a[13], a[44]); // Sum(57) + res[58] = fma52lo(res[58], a[14], a[44]); // Sum(58) + res[59] = fma52hi(res[59], a[14], a[44]); // Sum(58) + res[59] = fma52lo(res[59], a[15], a[44]); // Sum(59) + res[60] = fma52hi(res[60], a[15], a[44]); // Sum(59) + res[48] = fma52lo(res[48], a[3], a[45]); // Sum(48) + res[49] = fma52hi(res[49], a[3], a[45]); // Sum(48) + res[49] = fma52lo(res[49], a[4], a[45]); // Sum(49) + res[50] = fma52hi(res[50], a[4], a[45]); // Sum(49) + res[50] = fma52lo(res[50], a[5], a[45]); // Sum(50) + res[51] = fma52hi(res[51], a[5], a[45]); // Sum(50) + res[51] = fma52lo(res[51], a[6], a[45]); // Sum(51) + res[52] = fma52hi(res[52], a[6], a[45]); // Sum(51) + res[52] = fma52lo(res[52], a[7], a[45]); // Sum(52) + res[53] = fma52hi(res[53], a[7], a[45]); // Sum(52) + res[53] = fma52lo(res[53], a[8], a[45]); // Sum(53) + res[54] = fma52hi(res[54], a[8], a[45]); // Sum(53) + res[54] = fma52lo(res[54], a[9], a[45]); // Sum(54) + res[55] = fma52hi(res[55], a[9], a[45]); // Sum(54) + res[55] = fma52lo(res[55], a[10], a[45]); // Sum(55) + res[56] = fma52hi(res[56], a[10], a[45]); // Sum(55) + res[56] = fma52lo(res[56], a[11], a[45]); // Sum(56) + res[57] = fma52hi(res[57], a[11], a[45]); // Sum(56) + res[57] = fma52lo(res[57], a[12], a[45]); // Sum(57) + res[58] = fma52hi(res[58], a[12], a[45]); // Sum(57) + res[58] = fma52lo(res[58], a[13], a[45]); // Sum(58) + res[59] = fma52hi(res[59], a[13], a[45]); // Sum(58) + res[59] = fma52lo(res[59], a[14], a[45]); // Sum(59) + res[60] = fma52hi(res[60], a[14], a[45]); // Sum(59) + res[48] = fma52lo(res[48], a[2], a[46]); // Sum(48) + res[49] = fma52hi(res[49], a[2], a[46]); // Sum(48) + res[49] = fma52lo(res[49], a[3], a[46]); // Sum(49) + res[50] = fma52hi(res[50], a[3], a[46]); // Sum(49) + res[50] = fma52lo(res[50], a[4], a[46]); // Sum(50) + res[51] = fma52hi(res[51], a[4], a[46]); // Sum(50) + res[51] = fma52lo(res[51], a[5], a[46]); // Sum(51) + res[52] = fma52hi(res[52], a[5], a[46]); // Sum(51) + res[52] = fma52lo(res[52], a[6], a[46]); // Sum(52) + res[53] = fma52hi(res[53], a[6], a[46]); // Sum(52) + res[53] = fma52lo(res[53], a[7], a[46]); // Sum(53) + res[54] = fma52hi(res[54], a[7], a[46]); // Sum(53) + res[54] = fma52lo(res[54], a[8], a[46]); // Sum(54) + res[55] = fma52hi(res[55], a[8], a[46]); // Sum(54) + res[55] = fma52lo(res[55], a[9], a[46]); // Sum(55) + res[56] = fma52hi(res[56], a[9], a[46]); // Sum(55) + res[56] = fma52lo(res[56], a[10], a[46]); // Sum(56) + res[57] = fma52hi(res[57], a[10], a[46]); // Sum(56) + res[57] = fma52lo(res[57], a[11], a[46]); // Sum(57) + res[58] = fma52hi(res[58], a[11], a[46]); // Sum(57) + res[58] = fma52lo(res[58], a[12], a[46]); // Sum(58) + res[59] = fma52hi(res[59], a[12], a[46]); // Sum(58) + res[59] = fma52lo(res[59], a[13], a[46]); // Sum(59) + res[60] = fma52hi(res[60], a[13], a[46]); // Sum(59) + res[48] = fma52lo(res[48], a[1], a[47]); // Sum(48) + res[49] = fma52hi(res[49], a[1], a[47]); // Sum(48) + res[49] = fma52lo(res[49], a[2], a[47]); // Sum(49) + res[50] = fma52hi(res[50], a[2], a[47]); // Sum(49) + res[50] = fma52lo(res[50], a[3], a[47]); // Sum(50) + res[51] = fma52hi(res[51], a[3], a[47]); // Sum(50) + res[51] = fma52lo(res[51], a[4], a[47]); // Sum(51) + res[52] = fma52hi(res[52], a[4], a[47]); // Sum(51) + res[52] = fma52lo(res[52], a[5], a[47]); // Sum(52) + res[53] = fma52hi(res[53], a[5], a[47]); // Sum(52) + res[53] = fma52lo(res[53], a[6], a[47]); // Sum(53) + res[54] = fma52hi(res[54], a[6], a[47]); // Sum(53) + res[54] = fma52lo(res[54], a[7], a[47]); // Sum(54) + res[55] = fma52hi(res[55], a[7], a[47]); // Sum(54) + res[55] = fma52lo(res[55], a[8], a[47]); // Sum(55) + res[56] = fma52hi(res[56], a[8], a[47]); // Sum(55) + res[56] = fma52lo(res[56], a[9], a[47]); // Sum(56) + res[57] = fma52hi(res[57], a[9], a[47]); // Sum(56) + res[57] = fma52lo(res[57], a[10], a[47]); // Sum(57) + res[58] = fma52hi(res[58], a[10], a[47]); // Sum(57) + res[58] = fma52lo(res[58], a[11], a[47]); // Sum(58) + res[59] = fma52hi(res[59], a[11], a[47]); // Sum(58) + res[59] = fma52lo(res[59], a[12], a[47]); // Sum(59) + res[60] = fma52hi(res[60], a[12], a[47]); // Sum(59) + res[48] = fma52lo(res[48], a[0], a[48]); // Sum(48) + res[49] = fma52hi(res[49], a[0], a[48]); // Sum(48) + res[49] = fma52lo(res[49], a[1], a[48]); // Sum(49) + res[50] = fma52hi(res[50], a[1], a[48]); // Sum(49) + res[50] = fma52lo(res[50], a[2], a[48]); // Sum(50) + res[51] = fma52hi(res[51], a[2], a[48]); // Sum(50) + res[51] = fma52lo(res[51], a[3], a[48]); // Sum(51) + res[52] = fma52hi(res[52], a[3], a[48]); // Sum(51) + res[52] = fma52lo(res[52], a[4], a[48]); // Sum(52) + res[53] = fma52hi(res[53], a[4], a[48]); // Sum(52) + res[53] = fma52lo(res[53], a[5], a[48]); // Sum(53) + res[54] = fma52hi(res[54], a[5], a[48]); // Sum(53) + res[54] = fma52lo(res[54], a[6], a[48]); // Sum(54) + res[55] = fma52hi(res[55], a[6], a[48]); // Sum(54) + res[55] = fma52lo(res[55], a[7], a[48]); // Sum(55) + res[56] = fma52hi(res[56], a[7], a[48]); // Sum(55) + res[56] = fma52lo(res[56], a[8], a[48]); // Sum(56) + res[57] = fma52hi(res[57], a[8], a[48]); // Sum(56) + res[57] = fma52lo(res[57], a[9], a[48]); // Sum(57) + res[58] = fma52hi(res[58], a[9], a[48]); // Sum(57) + res[58] = fma52lo(res[58], a[10], a[48]); // Sum(58) + res[59] = fma52hi(res[59], a[10], a[48]); // Sum(58) + res[59] = fma52lo(res[59], a[11], a[48]); // Sum(59) + res[60] = fma52hi(res[60], a[11], a[48]); // Sum(59) + res[49] = fma52lo(res[49], a[0], a[49]); // Sum(49) + res[50] = fma52hi(res[50], a[0], a[49]); // Sum(49) + res[50] = fma52lo(res[50], a[1], a[49]); // Sum(50) + res[51] = fma52hi(res[51], a[1], a[49]); // Sum(50) + res[51] = fma52lo(res[51], a[2], a[49]); // Sum(51) + res[52] = fma52hi(res[52], a[2], a[49]); // Sum(51) + res[52] = fma52lo(res[52], a[3], a[49]); // Sum(52) + res[53] = fma52hi(res[53], a[3], a[49]); // Sum(52) + res[53] = fma52lo(res[53], a[4], a[49]); // Sum(53) + res[54] = fma52hi(res[54], a[4], a[49]); // Sum(53) + res[54] = fma52lo(res[54], a[5], a[49]); // Sum(54) + res[55] = fma52hi(res[55], a[5], a[49]); // Sum(54) + res[55] = fma52lo(res[55], a[6], a[49]); // Sum(55) + res[56] = fma52hi(res[56], a[6], a[49]); // Sum(55) + res[56] = fma52lo(res[56], a[7], a[49]); // Sum(56) + res[57] = fma52hi(res[57], a[7], a[49]); // Sum(56) + res[57] = fma52lo(res[57], a[8], a[49]); // Sum(57) + res[58] = fma52hi(res[58], a[8], a[49]); // Sum(57) + res[58] = fma52lo(res[58], a[9], a[49]); // Sum(58) + res[59] = fma52hi(res[59], a[9], a[49]); // Sum(58) + res[59] = fma52lo(res[59], a[10], a[49]); // Sum(59) + res[60] = fma52hi(res[60], a[10], a[49]); // Sum(59) + res[50] = fma52lo(res[50], a[0], a[50]); // Sum(50) + res[51] = fma52hi(res[51], a[0], a[50]); // Sum(50) + res[51] = fma52lo(res[51], a[1], a[50]); // Sum(51) + res[52] = fma52hi(res[52], a[1], a[50]); // Sum(51) + res[52] = fma52lo(res[52], a[2], a[50]); // Sum(52) + res[53] = fma52hi(res[53], a[2], a[50]); // Sum(52) + res[53] = fma52lo(res[53], a[3], a[50]); // Sum(53) + res[54] = fma52hi(res[54], a[3], a[50]); // Sum(53) + res[54] = fma52lo(res[54], a[4], a[50]); // Sum(54) + res[55] = fma52hi(res[55], a[4], a[50]); // Sum(54) + res[55] = fma52lo(res[55], a[5], a[50]); // Sum(55) + res[56] = fma52hi(res[56], a[5], a[50]); // Sum(55) + res[56] = fma52lo(res[56], a[6], a[50]); // Sum(56) + res[57] = fma52hi(res[57], a[6], a[50]); // Sum(56) + res[57] = fma52lo(res[57], a[7], a[50]); // Sum(57) + res[58] = fma52hi(res[58], a[7], a[50]); // Sum(57) + res[58] = fma52lo(res[58], a[8], a[50]); // Sum(58) + res[59] = fma52hi(res[59], a[8], a[50]); // Sum(58) + res[59] = fma52lo(res[59], a[9], a[50]); // Sum(59) + res[60] = fma52hi(res[60], a[9], a[50]); // Sum(59) + res[51] = fma52lo(res[51], a[0], a[51]); // Sum(51) + res[52] = fma52hi(res[52], a[0], a[51]); // Sum(51) + res[52] = fma52lo(res[52], a[1], a[51]); // Sum(52) + res[53] = fma52hi(res[53], a[1], a[51]); // Sum(52) + res[53] = fma52lo(res[53], a[2], a[51]); // Sum(53) + res[54] = fma52hi(res[54], a[2], a[51]); // Sum(53) + res[54] = fma52lo(res[54], a[3], a[51]); // Sum(54) + res[55] = fma52hi(res[55], a[3], a[51]); // Sum(54) + res[55] = fma52lo(res[55], a[4], a[51]); // Sum(55) + res[56] = fma52hi(res[56], a[4], a[51]); // Sum(55) + res[56] = fma52lo(res[56], a[5], a[51]); // Sum(56) + res[57] = fma52hi(res[57], a[5], a[51]); // Sum(56) + res[57] = fma52lo(res[57], a[6], a[51]); // Sum(57) + res[58] = fma52hi(res[58], a[6], a[51]); // Sum(57) + res[58] = fma52lo(res[58], a[7], a[51]); // Sum(58) + res[59] = fma52hi(res[59], a[7], a[51]); // Sum(58) + res[59] = fma52lo(res[59], a[8], a[51]); // Sum(59) + res[60] = fma52hi(res[60], a[8], a[51]); // Sum(59) + res[52] = fma52lo(res[52], a[0], a[52]); // Sum(52) + res[53] = fma52hi(res[53], a[0], a[52]); // Sum(52) + res[53] = fma52lo(res[53], a[1], a[52]); // Sum(53) + res[54] = fma52hi(res[54], a[1], a[52]); // Sum(53) + res[54] = fma52lo(res[54], a[2], a[52]); // Sum(54) + res[55] = fma52hi(res[55], a[2], a[52]); // Sum(54) + res[55] = fma52lo(res[55], a[3], a[52]); // Sum(55) + res[56] = fma52hi(res[56], a[3], a[52]); // Sum(55) + res[56] = fma52lo(res[56], a[4], a[52]); // Sum(56) + res[57] = fma52hi(res[57], a[4], a[52]); // Sum(56) + res[57] = fma52lo(res[57], a[5], a[52]); // Sum(57) + res[58] = fma52hi(res[58], a[5], a[52]); // Sum(57) + res[58] = fma52lo(res[58], a[6], a[52]); // Sum(58) + res[59] = fma52hi(res[59], a[6], a[52]); // Sum(58) + res[59] = fma52lo(res[59], a[7], a[52]); // Sum(59) + res[60] = fma52hi(res[60], a[7], a[52]); // Sum(59) + res[53] = fma52lo(res[53], a[0], a[53]); // Sum(53) + res[54] = fma52hi(res[54], a[0], a[53]); // Sum(53) + res[54] = fma52lo(res[54], a[1], a[53]); // Sum(54) + res[55] = fma52hi(res[55], a[1], a[53]); // Sum(54) + res[55] = fma52lo(res[55], a[2], a[53]); // Sum(55) + res[56] = fma52hi(res[56], a[2], a[53]); // Sum(55) + res[56] = fma52lo(res[56], a[3], a[53]); // Sum(56) + res[57] = fma52hi(res[57], a[3], a[53]); // Sum(56) + res[57] = fma52lo(res[57], a[4], a[53]); // Sum(57) + res[58] = fma52hi(res[58], a[4], a[53]); // Sum(57) + res[58] = fma52lo(res[58], a[5], a[53]); // Sum(58) + res[59] = fma52hi(res[59], a[5], a[53]); // Sum(58) + res[59] = fma52lo(res[59], a[6], a[53]); // Sum(59) + res[60] = fma52hi(res[60], a[6], a[53]); // Sum(59) + res[54] = fma52lo(res[54], a[0], a[54]); // Sum(54) + res[55] = fma52hi(res[55], a[0], a[54]); // Sum(54) + res[55] = fma52lo(res[55], a[1], a[54]); // Sum(55) + res[56] = fma52hi(res[56], a[1], a[54]); // Sum(55) + res[56] = fma52lo(res[56], a[2], a[54]); // Sum(56) + res[57] = fma52hi(res[57], a[2], a[54]); // Sum(56) + res[57] = fma52lo(res[57], a[3], a[54]); // Sum(57) + res[58] = fma52hi(res[58], a[3], a[54]); // Sum(57) + res[58] = fma52lo(res[58], a[4], a[54]); // Sum(58) + res[59] = fma52hi(res[59], a[4], a[54]); // Sum(58) + res[59] = fma52lo(res[59], a[5], a[54]); // Sum(59) + res[60] = fma52hi(res[60], a[5], a[54]); // Sum(59) + res[55] = fma52lo(res[55], a[0], a[55]); // Sum(55) + res[56] = fma52hi(res[56], a[0], a[55]); // Sum(55) + res[56] = fma52lo(res[56], a[1], a[55]); // Sum(56) + res[57] = fma52hi(res[57], a[1], a[55]); // Sum(56) + res[57] = fma52lo(res[57], a[2], a[55]); // Sum(57) + res[58] = fma52hi(res[58], a[2], a[55]); // Sum(57) + res[58] = fma52lo(res[58], a[3], a[55]); // Sum(58) + res[59] = fma52hi(res[59], a[3], a[55]); // Sum(58) + res[59] = fma52lo(res[59], a[4], a[55]); // Sum(59) + res[60] = fma52hi(res[60], a[4], a[55]); // Sum(59) + res[56] = fma52lo(res[56], a[0], a[56]); // Sum(56) + res[57] = fma52hi(res[57], a[0], a[56]); // Sum(56) + res[57] = fma52lo(res[57], a[1], a[56]); // Sum(57) + res[58] = fma52hi(res[58], a[1], a[56]); // Sum(57) + res[58] = fma52lo(res[58], a[2], a[56]); // Sum(58) + res[59] = fma52hi(res[59], a[2], a[56]); // Sum(58) + res[59] = fma52lo(res[59], a[3], a[56]); // Sum(59) + res[60] = fma52hi(res[60], a[3], a[56]); // Sum(59) + res[57] = fma52lo(res[57], a[0], a[57]); // Sum(57) + res[58] = fma52hi(res[58], a[0], a[57]); // Sum(57) + res[58] = fma52lo(res[58], a[1], a[57]); // Sum(58) + res[59] = fma52hi(res[59], a[1], a[57]); // Sum(58) + res[59] = fma52lo(res[59], a[2], a[57]); // Sum(59) + res[60] = fma52hi(res[60], a[2], a[57]); // Sum(59) + res[58] = fma52lo(res[58], a[0], a[58]); // Sum(58) + res[59] = fma52hi(res[59], a[0], a[58]); // Sum(58) + res[59] = fma52lo(res[59], a[1], a[58]); // Sum(59) + res[60] = fma52hi(res[60], a[1], a[58]); // Sum(59) + res[59] = fma52lo(res[59], a[0], a[59]); // Sum(59) + res[60] = fma52hi(res[60], a[0], a[59]); // Sum(59) + res[48] = add64(res[48], res[48]); // Double(48) + res[49] = add64(res[49], res[49]); // Double(49) + res[50] = add64(res[50], res[50]); // Double(50) + res[51] = add64(res[51], res[51]); // Double(51) + res[52] = add64(res[52], res[52]); // Double(52) + res[53] = add64(res[53], res[53]); // Double(53) + res[54] = add64(res[54], res[54]); // Double(54) + res[55] = add64(res[55], res[55]); // Double(55) + res[56] = add64(res[56], res[56]); // Double(56) + res[57] = add64(res[57], res[57]); // Double(57) + res[58] = add64(res[58], res[58]); // Double(58) + res[59] = add64(res[59], res[59]); // Double(59) + res[48] = fma52lo(res[48], a[24], a[24]); // Add sqr(48) + res[49] = fma52hi(res[49], a[24], a[24]); // Add sqr(48) + res[50] = fma52lo(res[50], a[25], a[25]); // Add sqr(50) + res[51] = fma52hi(res[51], a[25], a[25]); // Add sqr(50) + res[52] = fma52lo(res[52], a[26], a[26]); // Add sqr(52) + res[53] = fma52hi(res[53], a[26], a[26]); // Add sqr(52) + res[54] = fma52lo(res[54], a[27], a[27]); // Add sqr(54) + res[55] = fma52hi(res[55], a[27], a[27]); // Add sqr(54) + res[56] = fma52lo(res[56], a[28], a[28]); // Add sqr(56) + res[57] = fma52hi(res[57], a[28], a[28]); // Add sqr(56) + res[58] = fma52lo(res[58], a[29], a[29]); // Add sqr(58) + res[59] = fma52hi(res[59], a[29], a[29]); // Add sqr(58) + res[60] = fma52lo(res[60], a[29], a[31]); // Sum(60) + res[61] = fma52hi(res[61], a[29], a[31]); // Sum(60) + res[61] = fma52lo(res[61], a[30], a[31]); // Sum(61) + res[62] = fma52hi(res[62], a[30], a[31]); // Sum(61) + res[60] = fma52lo(res[60], a[28], a[32]); // Sum(60) + res[61] = fma52hi(res[61], a[28], a[32]); // Sum(60) + res[61] = fma52lo(res[61], a[29], a[32]); // Sum(61) + res[62] = fma52hi(res[62], a[29], a[32]); // Sum(61) + res[62] = fma52lo(res[62], a[30], a[32]); // Sum(62) + res[63] = fma52hi(res[63], a[30], a[32]); // Sum(62) + res[63] = fma52lo(res[63], a[31], a[32]); // Sum(63) + res[64] = fma52hi(res[64], a[31], a[32]); // Sum(63) + res[60] = fma52lo(res[60], a[27], a[33]); // Sum(60) + res[61] = fma52hi(res[61], a[27], a[33]); // Sum(60) + res[61] = fma52lo(res[61], a[28], a[33]); // Sum(61) + res[62] = fma52hi(res[62], a[28], a[33]); // Sum(61) + res[62] = fma52lo(res[62], a[29], a[33]); // Sum(62) + res[63] = fma52hi(res[63], a[29], a[33]); // Sum(62) + res[63] = fma52lo(res[63], a[30], a[33]); // Sum(63) + res[64] = fma52hi(res[64], a[30], a[33]); // Sum(63) + res[64] = fma52lo(res[64], a[31], a[33]); // Sum(64) + res[65] = fma52hi(res[65], a[31], a[33]); // Sum(64) + res[65] = fma52lo(res[65], a[32], a[33]); // Sum(65) + res[66] = fma52hi(res[66], a[32], a[33]); // Sum(65) + res[60] = fma52lo(res[60], a[26], a[34]); // Sum(60) + res[61] = fma52hi(res[61], a[26], a[34]); // Sum(60) + res[61] = fma52lo(res[61], a[27], a[34]); // Sum(61) + res[62] = fma52hi(res[62], a[27], a[34]); // Sum(61) + res[62] = fma52lo(res[62], a[28], a[34]); // Sum(62) + res[63] = fma52hi(res[63], a[28], a[34]); // Sum(62) + res[63] = fma52lo(res[63], a[29], a[34]); // Sum(63) + res[64] = fma52hi(res[64], a[29], a[34]); // Sum(63) + res[64] = fma52lo(res[64], a[30], a[34]); // Sum(64) + res[65] = fma52hi(res[65], a[30], a[34]); // Sum(64) + res[65] = fma52lo(res[65], a[31], a[34]); // Sum(65) + res[66] = fma52hi(res[66], a[31], a[34]); // Sum(65) + res[66] = fma52lo(res[66], a[32], a[34]); // Sum(66) + res[67] = fma52hi(res[67], a[32], a[34]); // Sum(66) + res[67] = fma52lo(res[67], a[33], a[34]); // Sum(67) + res[68] = fma52hi(res[68], a[33], a[34]); // Sum(67) + res[60] = fma52lo(res[60], a[25], a[35]); // Sum(60) + res[61] = fma52hi(res[61], a[25], a[35]); // Sum(60) + res[61] = fma52lo(res[61], a[26], a[35]); // Sum(61) + res[62] = fma52hi(res[62], a[26], a[35]); // Sum(61) + res[62] = fma52lo(res[62], a[27], a[35]); // Sum(62) + res[63] = fma52hi(res[63], a[27], a[35]); // Sum(62) + res[63] = fma52lo(res[63], a[28], a[35]); // Sum(63) + res[64] = fma52hi(res[64], a[28], a[35]); // Sum(63) + res[64] = fma52lo(res[64], a[29], a[35]); // Sum(64) + res[65] = fma52hi(res[65], a[29], a[35]); // Sum(64) + res[65] = fma52lo(res[65], a[30], a[35]); // Sum(65) + res[66] = fma52hi(res[66], a[30], a[35]); // Sum(65) + res[66] = fma52lo(res[66], a[31], a[35]); // Sum(66) + res[67] = fma52hi(res[67], a[31], a[35]); // Sum(66) + res[67] = fma52lo(res[67], a[32], a[35]); // Sum(67) + res[68] = fma52hi(res[68], a[32], a[35]); // Sum(67) + res[68] = fma52lo(res[68], a[33], a[35]); // Sum(68) + res[69] = fma52hi(res[69], a[33], a[35]); // Sum(68) + res[69] = fma52lo(res[69], a[34], a[35]); // Sum(69) + res[70] = fma52hi(res[70], a[34], a[35]); // Sum(69) + res[60] = fma52lo(res[60], a[24], a[36]); // Sum(60) + res[61] = fma52hi(res[61], a[24], a[36]); // Sum(60) + res[61] = fma52lo(res[61], a[25], a[36]); // Sum(61) + res[62] = fma52hi(res[62], a[25], a[36]); // Sum(61) + res[62] = fma52lo(res[62], a[26], a[36]); // Sum(62) + res[63] = fma52hi(res[63], a[26], a[36]); // Sum(62) + res[63] = fma52lo(res[63], a[27], a[36]); // Sum(63) + res[64] = fma52hi(res[64], a[27], a[36]); // Sum(63) + res[64] = fma52lo(res[64], a[28], a[36]); // Sum(64) + res[65] = fma52hi(res[65], a[28], a[36]); // Sum(64) + res[65] = fma52lo(res[65], a[29], a[36]); // Sum(65) + res[66] = fma52hi(res[66], a[29], a[36]); // Sum(65) + res[66] = fma52lo(res[66], a[30], a[36]); // Sum(66) + res[67] = fma52hi(res[67], a[30], a[36]); // Sum(66) + res[67] = fma52lo(res[67], a[31], a[36]); // Sum(67) + res[68] = fma52hi(res[68], a[31], a[36]); // Sum(67) + res[68] = fma52lo(res[68], a[32], a[36]); // Sum(68) + res[69] = fma52hi(res[69], a[32], a[36]); // Sum(68) + res[69] = fma52lo(res[69], a[33], a[36]); // Sum(69) + res[70] = fma52hi(res[70], a[33], a[36]); // Sum(69) + res[70] = fma52lo(res[70], a[34], a[36]); // Sum(70) + res[71] = fma52hi(res[71], a[34], a[36]); // Sum(70) + res[71] = fma52lo(res[71], a[35], a[36]); // Sum(71) + res[72] = fma52hi(res[72], a[35], a[36]); // Sum(71) + res[60] = fma52lo(res[60], a[23], a[37]); // Sum(60) + res[61] = fma52hi(res[61], a[23], a[37]); // Sum(60) + res[61] = fma52lo(res[61], a[24], a[37]); // Sum(61) + res[62] = fma52hi(res[62], a[24], a[37]); // Sum(61) + res[62] = fma52lo(res[62], a[25], a[37]); // Sum(62) + res[63] = fma52hi(res[63], a[25], a[37]); // Sum(62) + res[63] = fma52lo(res[63], a[26], a[37]); // Sum(63) + res[64] = fma52hi(res[64], a[26], a[37]); // Sum(63) + res[64] = fma52lo(res[64], a[27], a[37]); // Sum(64) + res[65] = fma52hi(res[65], a[27], a[37]); // Sum(64) + res[65] = fma52lo(res[65], a[28], a[37]); // Sum(65) + res[66] = fma52hi(res[66], a[28], a[37]); // Sum(65) + res[66] = fma52lo(res[66], a[29], a[37]); // Sum(66) + res[67] = fma52hi(res[67], a[29], a[37]); // Sum(66) + res[67] = fma52lo(res[67], a[30], a[37]); // Sum(67) + res[68] = fma52hi(res[68], a[30], a[37]); // Sum(67) + res[68] = fma52lo(res[68], a[31], a[37]); // Sum(68) + res[69] = fma52hi(res[69], a[31], a[37]); // Sum(68) + res[69] = fma52lo(res[69], a[32], a[37]); // Sum(69) + res[70] = fma52hi(res[70], a[32], a[37]); // Sum(69) + res[70] = fma52lo(res[70], a[33], a[37]); // Sum(70) + res[71] = fma52hi(res[71], a[33], a[37]); // Sum(70) + res[71] = fma52lo(res[71], a[34], a[37]); // Sum(71) + res[72] = fma52hi(res[72], a[34], a[37]); // Sum(71) + res[60] = fma52lo(res[60], a[22], a[38]); // Sum(60) + res[61] = fma52hi(res[61], a[22], a[38]); // Sum(60) + res[61] = fma52lo(res[61], a[23], a[38]); // Sum(61) + res[62] = fma52hi(res[62], a[23], a[38]); // Sum(61) + res[62] = fma52lo(res[62], a[24], a[38]); // Sum(62) + res[63] = fma52hi(res[63], a[24], a[38]); // Sum(62) + res[63] = fma52lo(res[63], a[25], a[38]); // Sum(63) + res[64] = fma52hi(res[64], a[25], a[38]); // Sum(63) + res[64] = fma52lo(res[64], a[26], a[38]); // Sum(64) + res[65] = fma52hi(res[65], a[26], a[38]); // Sum(64) + res[65] = fma52lo(res[65], a[27], a[38]); // Sum(65) + res[66] = fma52hi(res[66], a[27], a[38]); // Sum(65) + res[66] = fma52lo(res[66], a[28], a[38]); // Sum(66) + res[67] = fma52hi(res[67], a[28], a[38]); // Sum(66) + res[67] = fma52lo(res[67], a[29], a[38]); // Sum(67) + res[68] = fma52hi(res[68], a[29], a[38]); // Sum(67) + res[68] = fma52lo(res[68], a[30], a[38]); // Sum(68) + res[69] = fma52hi(res[69], a[30], a[38]); // Sum(68) + res[69] = fma52lo(res[69], a[31], a[38]); // Sum(69) + res[70] = fma52hi(res[70], a[31], a[38]); // Sum(69) + res[70] = fma52lo(res[70], a[32], a[38]); // Sum(70) + res[71] = fma52hi(res[71], a[32], a[38]); // Sum(70) + res[71] = fma52lo(res[71], a[33], a[38]); // Sum(71) + res[72] = fma52hi(res[72], a[33], a[38]); // Sum(71) + res[60] = fma52lo(res[60], a[21], a[39]); // Sum(60) + res[61] = fma52hi(res[61], a[21], a[39]); // Sum(60) + res[61] = fma52lo(res[61], a[22], a[39]); // Sum(61) + res[62] = fma52hi(res[62], a[22], a[39]); // Sum(61) + res[62] = fma52lo(res[62], a[23], a[39]); // Sum(62) + res[63] = fma52hi(res[63], a[23], a[39]); // Sum(62) + res[63] = fma52lo(res[63], a[24], a[39]); // Sum(63) + res[64] = fma52hi(res[64], a[24], a[39]); // Sum(63) + res[64] = fma52lo(res[64], a[25], a[39]); // Sum(64) + res[65] = fma52hi(res[65], a[25], a[39]); // Sum(64) + res[65] = fma52lo(res[65], a[26], a[39]); // Sum(65) + res[66] = fma52hi(res[66], a[26], a[39]); // Sum(65) + res[66] = fma52lo(res[66], a[27], a[39]); // Sum(66) + res[67] = fma52hi(res[67], a[27], a[39]); // Sum(66) + res[67] = fma52lo(res[67], a[28], a[39]); // Sum(67) + res[68] = fma52hi(res[68], a[28], a[39]); // Sum(67) + res[68] = fma52lo(res[68], a[29], a[39]); // Sum(68) + res[69] = fma52hi(res[69], a[29], a[39]); // Sum(68) + res[69] = fma52lo(res[69], a[30], a[39]); // Sum(69) + res[70] = fma52hi(res[70], a[30], a[39]); // Sum(69) + res[70] = fma52lo(res[70], a[31], a[39]); // Sum(70) + res[71] = fma52hi(res[71], a[31], a[39]); // Sum(70) + res[71] = fma52lo(res[71], a[32], a[39]); // Sum(71) + res[72] = fma52hi(res[72], a[32], a[39]); // Sum(71) + res[60] = fma52lo(res[60], a[20], a[40]); // Sum(60) + res[61] = fma52hi(res[61], a[20], a[40]); // Sum(60) + res[61] = fma52lo(res[61], a[21], a[40]); // Sum(61) + res[62] = fma52hi(res[62], a[21], a[40]); // Sum(61) + res[62] = fma52lo(res[62], a[22], a[40]); // Sum(62) + res[63] = fma52hi(res[63], a[22], a[40]); // Sum(62) + res[63] = fma52lo(res[63], a[23], a[40]); // Sum(63) + res[64] = fma52hi(res[64], a[23], a[40]); // Sum(63) + res[64] = fma52lo(res[64], a[24], a[40]); // Sum(64) + res[65] = fma52hi(res[65], a[24], a[40]); // Sum(64) + res[65] = fma52lo(res[65], a[25], a[40]); // Sum(65) + res[66] = fma52hi(res[66], a[25], a[40]); // Sum(65) + res[66] = fma52lo(res[66], a[26], a[40]); // Sum(66) + res[67] = fma52hi(res[67], a[26], a[40]); // Sum(66) + res[67] = fma52lo(res[67], a[27], a[40]); // Sum(67) + res[68] = fma52hi(res[68], a[27], a[40]); // Sum(67) + res[68] = fma52lo(res[68], a[28], a[40]); // Sum(68) + res[69] = fma52hi(res[69], a[28], a[40]); // Sum(68) + res[69] = fma52lo(res[69], a[29], a[40]); // Sum(69) + res[70] = fma52hi(res[70], a[29], a[40]); // Sum(69) + res[70] = fma52lo(res[70], a[30], a[40]); // Sum(70) + res[71] = fma52hi(res[71], a[30], a[40]); // Sum(70) + res[71] = fma52lo(res[71], a[31], a[40]); // Sum(71) + res[72] = fma52hi(res[72], a[31], a[40]); // Sum(71) + res[60] = fma52lo(res[60], a[19], a[41]); // Sum(60) + res[61] = fma52hi(res[61], a[19], a[41]); // Sum(60) + res[61] = fma52lo(res[61], a[20], a[41]); // Sum(61) + res[62] = fma52hi(res[62], a[20], a[41]); // Sum(61) + res[62] = fma52lo(res[62], a[21], a[41]); // Sum(62) + res[63] = fma52hi(res[63], a[21], a[41]); // Sum(62) + res[63] = fma52lo(res[63], a[22], a[41]); // Sum(63) + res[64] = fma52hi(res[64], a[22], a[41]); // Sum(63) + res[64] = fma52lo(res[64], a[23], a[41]); // Sum(64) + res[65] = fma52hi(res[65], a[23], a[41]); // Sum(64) + res[65] = fma52lo(res[65], a[24], a[41]); // Sum(65) + res[66] = fma52hi(res[66], a[24], a[41]); // Sum(65) + res[66] = fma52lo(res[66], a[25], a[41]); // Sum(66) + res[67] = fma52hi(res[67], a[25], a[41]); // Sum(66) + res[67] = fma52lo(res[67], a[26], a[41]); // Sum(67) + res[68] = fma52hi(res[68], a[26], a[41]); // Sum(67) + res[68] = fma52lo(res[68], a[27], a[41]); // Sum(68) + res[69] = fma52hi(res[69], a[27], a[41]); // Sum(68) + res[69] = fma52lo(res[69], a[28], a[41]); // Sum(69) + res[70] = fma52hi(res[70], a[28], a[41]); // Sum(69) + res[70] = fma52lo(res[70], a[29], a[41]); // Sum(70) + res[71] = fma52hi(res[71], a[29], a[41]); // Sum(70) + res[71] = fma52lo(res[71], a[30], a[41]); // Sum(71) + res[72] = fma52hi(res[72], a[30], a[41]); // Sum(71) + res[60] = fma52lo(res[60], a[18], a[42]); // Sum(60) + res[61] = fma52hi(res[61], a[18], a[42]); // Sum(60) + res[61] = fma52lo(res[61], a[19], a[42]); // Sum(61) + res[62] = fma52hi(res[62], a[19], a[42]); // Sum(61) + res[62] = fma52lo(res[62], a[20], a[42]); // Sum(62) + res[63] = fma52hi(res[63], a[20], a[42]); // Sum(62) + res[63] = fma52lo(res[63], a[21], a[42]); // Sum(63) + res[64] = fma52hi(res[64], a[21], a[42]); // Sum(63) + res[64] = fma52lo(res[64], a[22], a[42]); // Sum(64) + res[65] = fma52hi(res[65], a[22], a[42]); // Sum(64) + res[65] = fma52lo(res[65], a[23], a[42]); // Sum(65) + res[66] = fma52hi(res[66], a[23], a[42]); // Sum(65) + res[66] = fma52lo(res[66], a[24], a[42]); // Sum(66) + res[67] = fma52hi(res[67], a[24], a[42]); // Sum(66) + res[67] = fma52lo(res[67], a[25], a[42]); // Sum(67) + res[68] = fma52hi(res[68], a[25], a[42]); // Sum(67) + res[68] = fma52lo(res[68], a[26], a[42]); // Sum(68) + res[69] = fma52hi(res[69], a[26], a[42]); // Sum(68) + res[69] = fma52lo(res[69], a[27], a[42]); // Sum(69) + res[70] = fma52hi(res[70], a[27], a[42]); // Sum(69) + res[70] = fma52lo(res[70], a[28], a[42]); // Sum(70) + res[71] = fma52hi(res[71], a[28], a[42]); // Sum(70) + res[71] = fma52lo(res[71], a[29], a[42]); // Sum(71) + res[72] = fma52hi(res[72], a[29], a[42]); // Sum(71) + res[60] = fma52lo(res[60], a[17], a[43]); // Sum(60) + res[61] = fma52hi(res[61], a[17], a[43]); // Sum(60) + res[61] = fma52lo(res[61], a[18], a[43]); // Sum(61) + res[62] = fma52hi(res[62], a[18], a[43]); // Sum(61) + res[62] = fma52lo(res[62], a[19], a[43]); // Sum(62) + res[63] = fma52hi(res[63], a[19], a[43]); // Sum(62) + res[63] = fma52lo(res[63], a[20], a[43]); // Sum(63) + res[64] = fma52hi(res[64], a[20], a[43]); // Sum(63) + res[64] = fma52lo(res[64], a[21], a[43]); // Sum(64) + res[65] = fma52hi(res[65], a[21], a[43]); // Sum(64) + res[65] = fma52lo(res[65], a[22], a[43]); // Sum(65) + res[66] = fma52hi(res[66], a[22], a[43]); // Sum(65) + res[66] = fma52lo(res[66], a[23], a[43]); // Sum(66) + res[67] = fma52hi(res[67], a[23], a[43]); // Sum(66) + res[67] = fma52lo(res[67], a[24], a[43]); // Sum(67) + res[68] = fma52hi(res[68], a[24], a[43]); // Sum(67) + res[68] = fma52lo(res[68], a[25], a[43]); // Sum(68) + res[69] = fma52hi(res[69], a[25], a[43]); // Sum(68) + res[69] = fma52lo(res[69], a[26], a[43]); // Sum(69) + res[70] = fma52hi(res[70], a[26], a[43]); // Sum(69) + res[70] = fma52lo(res[70], a[27], a[43]); // Sum(70) + res[71] = fma52hi(res[71], a[27], a[43]); // Sum(70) + res[71] = fma52lo(res[71], a[28], a[43]); // Sum(71) + res[72] = fma52hi(res[72], a[28], a[43]); // Sum(71) + res[60] = fma52lo(res[60], a[16], a[44]); // Sum(60) + res[61] = fma52hi(res[61], a[16], a[44]); // Sum(60) + res[61] = fma52lo(res[61], a[17], a[44]); // Sum(61) + res[62] = fma52hi(res[62], a[17], a[44]); // Sum(61) + res[62] = fma52lo(res[62], a[18], a[44]); // Sum(62) + res[63] = fma52hi(res[63], a[18], a[44]); // Sum(62) + res[63] = fma52lo(res[63], a[19], a[44]); // Sum(63) + res[64] = fma52hi(res[64], a[19], a[44]); // Sum(63) + res[64] = fma52lo(res[64], a[20], a[44]); // Sum(64) + res[65] = fma52hi(res[65], a[20], a[44]); // Sum(64) + res[65] = fma52lo(res[65], a[21], a[44]); // Sum(65) + res[66] = fma52hi(res[66], a[21], a[44]); // Sum(65) + res[66] = fma52lo(res[66], a[22], a[44]); // Sum(66) + res[67] = fma52hi(res[67], a[22], a[44]); // Sum(66) + res[67] = fma52lo(res[67], a[23], a[44]); // Sum(67) + res[68] = fma52hi(res[68], a[23], a[44]); // Sum(67) + res[68] = fma52lo(res[68], a[24], a[44]); // Sum(68) + res[69] = fma52hi(res[69], a[24], a[44]); // Sum(68) + res[69] = fma52lo(res[69], a[25], a[44]); // Sum(69) + res[70] = fma52hi(res[70], a[25], a[44]); // Sum(69) + res[70] = fma52lo(res[70], a[26], a[44]); // Sum(70) + res[71] = fma52hi(res[71], a[26], a[44]); // Sum(70) + res[71] = fma52lo(res[71], a[27], a[44]); // Sum(71) + res[72] = fma52hi(res[72], a[27], a[44]); // Sum(71) + res[60] = fma52lo(res[60], a[15], a[45]); // Sum(60) + res[61] = fma52hi(res[61], a[15], a[45]); // Sum(60) + res[61] = fma52lo(res[61], a[16], a[45]); // Sum(61) + res[62] = fma52hi(res[62], a[16], a[45]); // Sum(61) + res[62] = fma52lo(res[62], a[17], a[45]); // Sum(62) + res[63] = fma52hi(res[63], a[17], a[45]); // Sum(62) + res[63] = fma52lo(res[63], a[18], a[45]); // Sum(63) + res[64] = fma52hi(res[64], a[18], a[45]); // Sum(63) + res[64] = fma52lo(res[64], a[19], a[45]); // Sum(64) + res[65] = fma52hi(res[65], a[19], a[45]); // Sum(64) + res[65] = fma52lo(res[65], a[20], a[45]); // Sum(65) + res[66] = fma52hi(res[66], a[20], a[45]); // Sum(65) + res[66] = fma52lo(res[66], a[21], a[45]); // Sum(66) + res[67] = fma52hi(res[67], a[21], a[45]); // Sum(66) + res[67] = fma52lo(res[67], a[22], a[45]); // Sum(67) + res[68] = fma52hi(res[68], a[22], a[45]); // Sum(67) + res[68] = fma52lo(res[68], a[23], a[45]); // Sum(68) + res[69] = fma52hi(res[69], a[23], a[45]); // Sum(68) + res[69] = fma52lo(res[69], a[24], a[45]); // Sum(69) + res[70] = fma52hi(res[70], a[24], a[45]); // Sum(69) + res[70] = fma52lo(res[70], a[25], a[45]); // Sum(70) + res[71] = fma52hi(res[71], a[25], a[45]); // Sum(70) + res[71] = fma52lo(res[71], a[26], a[45]); // Sum(71) + res[72] = fma52hi(res[72], a[26], a[45]); // Sum(71) + res[60] = fma52lo(res[60], a[14], a[46]); // Sum(60) + res[61] = fma52hi(res[61], a[14], a[46]); // Sum(60) + res[61] = fma52lo(res[61], a[15], a[46]); // Sum(61) + res[62] = fma52hi(res[62], a[15], a[46]); // Sum(61) + res[62] = fma52lo(res[62], a[16], a[46]); // Sum(62) + res[63] = fma52hi(res[63], a[16], a[46]); // Sum(62) + res[63] = fma52lo(res[63], a[17], a[46]); // Sum(63) + res[64] = fma52hi(res[64], a[17], a[46]); // Sum(63) + res[64] = fma52lo(res[64], a[18], a[46]); // Sum(64) + res[65] = fma52hi(res[65], a[18], a[46]); // Sum(64) + res[65] = fma52lo(res[65], a[19], a[46]); // Sum(65) + res[66] = fma52hi(res[66], a[19], a[46]); // Sum(65) + res[66] = fma52lo(res[66], a[20], a[46]); // Sum(66) + res[67] = fma52hi(res[67], a[20], a[46]); // Sum(66) + res[67] = fma52lo(res[67], a[21], a[46]); // Sum(67) + res[68] = fma52hi(res[68], a[21], a[46]); // Sum(67) + res[68] = fma52lo(res[68], a[22], a[46]); // Sum(68) + res[69] = fma52hi(res[69], a[22], a[46]); // Sum(68) + res[69] = fma52lo(res[69], a[23], a[46]); // Sum(69) + res[70] = fma52hi(res[70], a[23], a[46]); // Sum(69) + res[70] = fma52lo(res[70], a[24], a[46]); // Sum(70) + res[71] = fma52hi(res[71], a[24], a[46]); // Sum(70) + res[71] = fma52lo(res[71], a[25], a[46]); // Sum(71) + res[72] = fma52hi(res[72], a[25], a[46]); // Sum(71) + res[60] = fma52lo(res[60], a[13], a[47]); // Sum(60) + res[61] = fma52hi(res[61], a[13], a[47]); // Sum(60) + res[61] = fma52lo(res[61], a[14], a[47]); // Sum(61) + res[62] = fma52hi(res[62], a[14], a[47]); // Sum(61) + res[62] = fma52lo(res[62], a[15], a[47]); // Sum(62) + res[63] = fma52hi(res[63], a[15], a[47]); // Sum(62) + res[63] = fma52lo(res[63], a[16], a[47]); // Sum(63) + res[64] = fma52hi(res[64], a[16], a[47]); // Sum(63) + res[64] = fma52lo(res[64], a[17], a[47]); // Sum(64) + res[65] = fma52hi(res[65], a[17], a[47]); // Sum(64) + res[65] = fma52lo(res[65], a[18], a[47]); // Sum(65) + res[66] = fma52hi(res[66], a[18], a[47]); // Sum(65) + res[66] = fma52lo(res[66], a[19], a[47]); // Sum(66) + res[67] = fma52hi(res[67], a[19], a[47]); // Sum(66) + res[67] = fma52lo(res[67], a[20], a[47]); // Sum(67) + res[68] = fma52hi(res[68], a[20], a[47]); // Sum(67) + res[68] = fma52lo(res[68], a[21], a[47]); // Sum(68) + res[69] = fma52hi(res[69], a[21], a[47]); // Sum(68) + res[69] = fma52lo(res[69], a[22], a[47]); // Sum(69) + res[70] = fma52hi(res[70], a[22], a[47]); // Sum(69) + res[70] = fma52lo(res[70], a[23], a[47]); // Sum(70) + res[71] = fma52hi(res[71], a[23], a[47]); // Sum(70) + res[71] = fma52lo(res[71], a[24], a[47]); // Sum(71) + res[72] = fma52hi(res[72], a[24], a[47]); // Sum(71) + res[60] = fma52lo(res[60], a[12], a[48]); // Sum(60) + res[61] = fma52hi(res[61], a[12], a[48]); // Sum(60) + res[61] = fma52lo(res[61], a[13], a[48]); // Sum(61) + res[62] = fma52hi(res[62], a[13], a[48]); // Sum(61) + res[62] = fma52lo(res[62], a[14], a[48]); // Sum(62) + res[63] = fma52hi(res[63], a[14], a[48]); // Sum(62) + res[63] = fma52lo(res[63], a[15], a[48]); // Sum(63) + res[64] = fma52hi(res[64], a[15], a[48]); // Sum(63) + res[64] = fma52lo(res[64], a[16], a[48]); // Sum(64) + res[65] = fma52hi(res[65], a[16], a[48]); // Sum(64) + res[65] = fma52lo(res[65], a[17], a[48]); // Sum(65) + res[66] = fma52hi(res[66], a[17], a[48]); // Sum(65) + res[66] = fma52lo(res[66], a[18], a[48]); // Sum(66) + res[67] = fma52hi(res[67], a[18], a[48]); // Sum(66) + res[67] = fma52lo(res[67], a[19], a[48]); // Sum(67) + res[68] = fma52hi(res[68], a[19], a[48]); // Sum(67) + res[68] = fma52lo(res[68], a[20], a[48]); // Sum(68) + res[69] = fma52hi(res[69], a[20], a[48]); // Sum(68) + res[69] = fma52lo(res[69], a[21], a[48]); // Sum(69) + res[70] = fma52hi(res[70], a[21], a[48]); // Sum(69) + res[70] = fma52lo(res[70], a[22], a[48]); // Sum(70) + res[71] = fma52hi(res[71], a[22], a[48]); // Sum(70) + res[71] = fma52lo(res[71], a[23], a[48]); // Sum(71) + res[72] = fma52hi(res[72], a[23], a[48]); // Sum(71) + res[60] = fma52lo(res[60], a[11], a[49]); // Sum(60) + res[61] = fma52hi(res[61], a[11], a[49]); // Sum(60) + res[61] = fma52lo(res[61], a[12], a[49]); // Sum(61) + res[62] = fma52hi(res[62], a[12], a[49]); // Sum(61) + res[62] = fma52lo(res[62], a[13], a[49]); // Sum(62) + res[63] = fma52hi(res[63], a[13], a[49]); // Sum(62) + res[63] = fma52lo(res[63], a[14], a[49]); // Sum(63) + res[64] = fma52hi(res[64], a[14], a[49]); // Sum(63) + res[64] = fma52lo(res[64], a[15], a[49]); // Sum(64) + res[65] = fma52hi(res[65], a[15], a[49]); // Sum(64) + res[65] = fma52lo(res[65], a[16], a[49]); // Sum(65) + res[66] = fma52hi(res[66], a[16], a[49]); // Sum(65) + res[66] = fma52lo(res[66], a[17], a[49]); // Sum(66) + res[67] = fma52hi(res[67], a[17], a[49]); // Sum(66) + res[67] = fma52lo(res[67], a[18], a[49]); // Sum(67) + res[68] = fma52hi(res[68], a[18], a[49]); // Sum(67) + res[68] = fma52lo(res[68], a[19], a[49]); // Sum(68) + res[69] = fma52hi(res[69], a[19], a[49]); // Sum(68) + res[69] = fma52lo(res[69], a[20], a[49]); // Sum(69) + res[70] = fma52hi(res[70], a[20], a[49]); // Sum(69) + res[70] = fma52lo(res[70], a[21], a[49]); // Sum(70) + res[71] = fma52hi(res[71], a[21], a[49]); // Sum(70) + res[71] = fma52lo(res[71], a[22], a[49]); // Sum(71) + res[72] = fma52hi(res[72], a[22], a[49]); // Sum(71) + res[60] = fma52lo(res[60], a[10], a[50]); // Sum(60) + res[61] = fma52hi(res[61], a[10], a[50]); // Sum(60) + res[61] = fma52lo(res[61], a[11], a[50]); // Sum(61) + res[62] = fma52hi(res[62], a[11], a[50]); // Sum(61) + res[62] = fma52lo(res[62], a[12], a[50]); // Sum(62) + res[63] = fma52hi(res[63], a[12], a[50]); // Sum(62) + res[63] = fma52lo(res[63], a[13], a[50]); // Sum(63) + res[64] = fma52hi(res[64], a[13], a[50]); // Sum(63) + res[64] = fma52lo(res[64], a[14], a[50]); // Sum(64) + res[65] = fma52hi(res[65], a[14], a[50]); // Sum(64) + res[65] = fma52lo(res[65], a[15], a[50]); // Sum(65) + res[66] = fma52hi(res[66], a[15], a[50]); // Sum(65) + res[66] = fma52lo(res[66], a[16], a[50]); // Sum(66) + res[67] = fma52hi(res[67], a[16], a[50]); // Sum(66) + res[67] = fma52lo(res[67], a[17], a[50]); // Sum(67) + res[68] = fma52hi(res[68], a[17], a[50]); // Sum(67) + res[68] = fma52lo(res[68], a[18], a[50]); // Sum(68) + res[69] = fma52hi(res[69], a[18], a[50]); // Sum(68) + res[69] = fma52lo(res[69], a[19], a[50]); // Sum(69) + res[70] = fma52hi(res[70], a[19], a[50]); // Sum(69) + res[70] = fma52lo(res[70], a[20], a[50]); // Sum(70) + res[71] = fma52hi(res[71], a[20], a[50]); // Sum(70) + res[71] = fma52lo(res[71], a[21], a[50]); // Sum(71) + res[72] = fma52hi(res[72], a[21], a[50]); // Sum(71) + res[60] = fma52lo(res[60], a[9], a[51]); // Sum(60) + res[61] = fma52hi(res[61], a[9], a[51]); // Sum(60) + res[61] = fma52lo(res[61], a[10], a[51]); // Sum(61) + res[62] = fma52hi(res[62], a[10], a[51]); // Sum(61) + res[62] = fma52lo(res[62], a[11], a[51]); // Sum(62) + res[63] = fma52hi(res[63], a[11], a[51]); // Sum(62) + res[63] = fma52lo(res[63], a[12], a[51]); // Sum(63) + res[64] = fma52hi(res[64], a[12], a[51]); // Sum(63) + res[64] = fma52lo(res[64], a[13], a[51]); // Sum(64) + res[65] = fma52hi(res[65], a[13], a[51]); // Sum(64) + res[65] = fma52lo(res[65], a[14], a[51]); // Sum(65) + res[66] = fma52hi(res[66], a[14], a[51]); // Sum(65) + res[66] = fma52lo(res[66], a[15], a[51]); // Sum(66) + res[67] = fma52hi(res[67], a[15], a[51]); // Sum(66) + res[67] = fma52lo(res[67], a[16], a[51]); // Sum(67) + res[68] = fma52hi(res[68], a[16], a[51]); // Sum(67) + res[68] = fma52lo(res[68], a[17], a[51]); // Sum(68) + res[69] = fma52hi(res[69], a[17], a[51]); // Sum(68) + res[69] = fma52lo(res[69], a[18], a[51]); // Sum(69) + res[70] = fma52hi(res[70], a[18], a[51]); // Sum(69) + res[70] = fma52lo(res[70], a[19], a[51]); // Sum(70) + res[71] = fma52hi(res[71], a[19], a[51]); // Sum(70) + res[71] = fma52lo(res[71], a[20], a[51]); // Sum(71) + res[72] = fma52hi(res[72], a[20], a[51]); // Sum(71) + res[60] = fma52lo(res[60], a[8], a[52]); // Sum(60) + res[61] = fma52hi(res[61], a[8], a[52]); // Sum(60) + res[61] = fma52lo(res[61], a[9], a[52]); // Sum(61) + res[62] = fma52hi(res[62], a[9], a[52]); // Sum(61) + res[62] = fma52lo(res[62], a[10], a[52]); // Sum(62) + res[63] = fma52hi(res[63], a[10], a[52]); // Sum(62) + res[63] = fma52lo(res[63], a[11], a[52]); // Sum(63) + res[64] = fma52hi(res[64], a[11], a[52]); // Sum(63) + res[64] = fma52lo(res[64], a[12], a[52]); // Sum(64) + res[65] = fma52hi(res[65], a[12], a[52]); // Sum(64) + res[65] = fma52lo(res[65], a[13], a[52]); // Sum(65) + res[66] = fma52hi(res[66], a[13], a[52]); // Sum(65) + res[66] = fma52lo(res[66], a[14], a[52]); // Sum(66) + res[67] = fma52hi(res[67], a[14], a[52]); // Sum(66) + res[67] = fma52lo(res[67], a[15], a[52]); // Sum(67) + res[68] = fma52hi(res[68], a[15], a[52]); // Sum(67) + res[68] = fma52lo(res[68], a[16], a[52]); // Sum(68) + res[69] = fma52hi(res[69], a[16], a[52]); // Sum(68) + res[69] = fma52lo(res[69], a[17], a[52]); // Sum(69) + res[70] = fma52hi(res[70], a[17], a[52]); // Sum(69) + res[70] = fma52lo(res[70], a[18], a[52]); // Sum(70) + res[71] = fma52hi(res[71], a[18], a[52]); // Sum(70) + res[71] = fma52lo(res[71], a[19], a[52]); // Sum(71) + res[72] = fma52hi(res[72], a[19], a[52]); // Sum(71) + res[60] = fma52lo(res[60], a[7], a[53]); // Sum(60) + res[61] = fma52hi(res[61], a[7], a[53]); // Sum(60) + res[61] = fma52lo(res[61], a[8], a[53]); // Sum(61) + res[62] = fma52hi(res[62], a[8], a[53]); // Sum(61) + res[62] = fma52lo(res[62], a[9], a[53]); // Sum(62) + res[63] = fma52hi(res[63], a[9], a[53]); // Sum(62) + res[63] = fma52lo(res[63], a[10], a[53]); // Sum(63) + res[64] = fma52hi(res[64], a[10], a[53]); // Sum(63) + res[64] = fma52lo(res[64], a[11], a[53]); // Sum(64) + res[65] = fma52hi(res[65], a[11], a[53]); // Sum(64) + res[65] = fma52lo(res[65], a[12], a[53]); // Sum(65) + res[66] = fma52hi(res[66], a[12], a[53]); // Sum(65) + res[66] = fma52lo(res[66], a[13], a[53]); // Sum(66) + res[67] = fma52hi(res[67], a[13], a[53]); // Sum(66) + res[67] = fma52lo(res[67], a[14], a[53]); // Sum(67) + res[68] = fma52hi(res[68], a[14], a[53]); // Sum(67) + res[68] = fma52lo(res[68], a[15], a[53]); // Sum(68) + res[69] = fma52hi(res[69], a[15], a[53]); // Sum(68) + res[69] = fma52lo(res[69], a[16], a[53]); // Sum(69) + res[70] = fma52hi(res[70], a[16], a[53]); // Sum(69) + res[70] = fma52lo(res[70], a[17], a[53]); // Sum(70) + res[71] = fma52hi(res[71], a[17], a[53]); // Sum(70) + res[71] = fma52lo(res[71], a[18], a[53]); // Sum(71) + res[72] = fma52hi(res[72], a[18], a[53]); // Sum(71) + res[60] = fma52lo(res[60], a[6], a[54]); // Sum(60) + res[61] = fma52hi(res[61], a[6], a[54]); // Sum(60) + res[61] = fma52lo(res[61], a[7], a[54]); // Sum(61) + res[62] = fma52hi(res[62], a[7], a[54]); // Sum(61) + res[62] = fma52lo(res[62], a[8], a[54]); // Sum(62) + res[63] = fma52hi(res[63], a[8], a[54]); // Sum(62) + res[63] = fma52lo(res[63], a[9], a[54]); // Sum(63) + res[64] = fma52hi(res[64], a[9], a[54]); // Sum(63) + res[64] = fma52lo(res[64], a[10], a[54]); // Sum(64) + res[65] = fma52hi(res[65], a[10], a[54]); // Sum(64) + res[65] = fma52lo(res[65], a[11], a[54]); // Sum(65) + res[66] = fma52hi(res[66], a[11], a[54]); // Sum(65) + res[66] = fma52lo(res[66], a[12], a[54]); // Sum(66) + res[67] = fma52hi(res[67], a[12], a[54]); // Sum(66) + res[67] = fma52lo(res[67], a[13], a[54]); // Sum(67) + res[68] = fma52hi(res[68], a[13], a[54]); // Sum(67) + res[68] = fma52lo(res[68], a[14], a[54]); // Sum(68) + res[69] = fma52hi(res[69], a[14], a[54]); // Sum(68) + res[69] = fma52lo(res[69], a[15], a[54]); // Sum(69) + res[70] = fma52hi(res[70], a[15], a[54]); // Sum(69) + res[70] = fma52lo(res[70], a[16], a[54]); // Sum(70) + res[71] = fma52hi(res[71], a[16], a[54]); // Sum(70) + res[71] = fma52lo(res[71], a[17], a[54]); // Sum(71) + res[72] = fma52hi(res[72], a[17], a[54]); // Sum(71) + res[60] = fma52lo(res[60], a[5], a[55]); // Sum(60) + res[61] = fma52hi(res[61], a[5], a[55]); // Sum(60) + res[61] = fma52lo(res[61], a[6], a[55]); // Sum(61) + res[62] = fma52hi(res[62], a[6], a[55]); // Sum(61) + res[62] = fma52lo(res[62], a[7], a[55]); // Sum(62) + res[63] = fma52hi(res[63], a[7], a[55]); // Sum(62) + res[63] = fma52lo(res[63], a[8], a[55]); // Sum(63) + res[64] = fma52hi(res[64], a[8], a[55]); // Sum(63) + res[64] = fma52lo(res[64], a[9], a[55]); // Sum(64) + res[65] = fma52hi(res[65], a[9], a[55]); // Sum(64) + res[65] = fma52lo(res[65], a[10], a[55]); // Sum(65) + res[66] = fma52hi(res[66], a[10], a[55]); // Sum(65) + res[66] = fma52lo(res[66], a[11], a[55]); // Sum(66) + res[67] = fma52hi(res[67], a[11], a[55]); // Sum(66) + res[67] = fma52lo(res[67], a[12], a[55]); // Sum(67) + res[68] = fma52hi(res[68], a[12], a[55]); // Sum(67) + res[68] = fma52lo(res[68], a[13], a[55]); // Sum(68) + res[69] = fma52hi(res[69], a[13], a[55]); // Sum(68) + res[69] = fma52lo(res[69], a[14], a[55]); // Sum(69) + res[70] = fma52hi(res[70], a[14], a[55]); // Sum(69) + res[70] = fma52lo(res[70], a[15], a[55]); // Sum(70) + res[71] = fma52hi(res[71], a[15], a[55]); // Sum(70) + res[71] = fma52lo(res[71], a[16], a[55]); // Sum(71) + res[72] = fma52hi(res[72], a[16], a[55]); // Sum(71) + res[60] = fma52lo(res[60], a[4], a[56]); // Sum(60) + res[61] = fma52hi(res[61], a[4], a[56]); // Sum(60) + res[61] = fma52lo(res[61], a[5], a[56]); // Sum(61) + res[62] = fma52hi(res[62], a[5], a[56]); // Sum(61) + res[62] = fma52lo(res[62], a[6], a[56]); // Sum(62) + res[63] = fma52hi(res[63], a[6], a[56]); // Sum(62) + res[63] = fma52lo(res[63], a[7], a[56]); // Sum(63) + res[64] = fma52hi(res[64], a[7], a[56]); // Sum(63) + res[64] = fma52lo(res[64], a[8], a[56]); // Sum(64) + res[65] = fma52hi(res[65], a[8], a[56]); // Sum(64) + res[65] = fma52lo(res[65], a[9], a[56]); // Sum(65) + res[66] = fma52hi(res[66], a[9], a[56]); // Sum(65) + res[66] = fma52lo(res[66], a[10], a[56]); // Sum(66) + res[67] = fma52hi(res[67], a[10], a[56]); // Sum(66) + res[67] = fma52lo(res[67], a[11], a[56]); // Sum(67) + res[68] = fma52hi(res[68], a[11], a[56]); // Sum(67) + res[68] = fma52lo(res[68], a[12], a[56]); // Sum(68) + res[69] = fma52hi(res[69], a[12], a[56]); // Sum(68) + res[69] = fma52lo(res[69], a[13], a[56]); // Sum(69) + res[70] = fma52hi(res[70], a[13], a[56]); // Sum(69) + res[70] = fma52lo(res[70], a[14], a[56]); // Sum(70) + res[71] = fma52hi(res[71], a[14], a[56]); // Sum(70) + res[71] = fma52lo(res[71], a[15], a[56]); // Sum(71) + res[72] = fma52hi(res[72], a[15], a[56]); // Sum(71) + res[60] = fma52lo(res[60], a[3], a[57]); // Sum(60) + res[61] = fma52hi(res[61], a[3], a[57]); // Sum(60) + res[61] = fma52lo(res[61], a[4], a[57]); // Sum(61) + res[62] = fma52hi(res[62], a[4], a[57]); // Sum(61) + res[62] = fma52lo(res[62], a[5], a[57]); // Sum(62) + res[63] = fma52hi(res[63], a[5], a[57]); // Sum(62) + res[63] = fma52lo(res[63], a[6], a[57]); // Sum(63) + res[64] = fma52hi(res[64], a[6], a[57]); // Sum(63) + res[64] = fma52lo(res[64], a[7], a[57]); // Sum(64) + res[65] = fma52hi(res[65], a[7], a[57]); // Sum(64) + res[65] = fma52lo(res[65], a[8], a[57]); // Sum(65) + res[66] = fma52hi(res[66], a[8], a[57]); // Sum(65) + res[66] = fma52lo(res[66], a[9], a[57]); // Sum(66) + res[67] = fma52hi(res[67], a[9], a[57]); // Sum(66) + res[67] = fma52lo(res[67], a[10], a[57]); // Sum(67) + res[68] = fma52hi(res[68], a[10], a[57]); // Sum(67) + res[68] = fma52lo(res[68], a[11], a[57]); // Sum(68) + res[69] = fma52hi(res[69], a[11], a[57]); // Sum(68) + res[69] = fma52lo(res[69], a[12], a[57]); // Sum(69) + res[70] = fma52hi(res[70], a[12], a[57]); // Sum(69) + res[70] = fma52lo(res[70], a[13], a[57]); // Sum(70) + res[71] = fma52hi(res[71], a[13], a[57]); // Sum(70) + res[71] = fma52lo(res[71], a[14], a[57]); // Sum(71) + res[72] = fma52hi(res[72], a[14], a[57]); // Sum(71) + res[60] = fma52lo(res[60], a[2], a[58]); // Sum(60) + res[61] = fma52hi(res[61], a[2], a[58]); // Sum(60) + res[61] = fma52lo(res[61], a[3], a[58]); // Sum(61) + res[62] = fma52hi(res[62], a[3], a[58]); // Sum(61) + res[62] = fma52lo(res[62], a[4], a[58]); // Sum(62) + res[63] = fma52hi(res[63], a[4], a[58]); // Sum(62) + res[63] = fma52lo(res[63], a[5], a[58]); // Sum(63) + res[64] = fma52hi(res[64], a[5], a[58]); // Sum(63) + res[64] = fma52lo(res[64], a[6], a[58]); // Sum(64) + res[65] = fma52hi(res[65], a[6], a[58]); // Sum(64) + res[65] = fma52lo(res[65], a[7], a[58]); // Sum(65) + res[66] = fma52hi(res[66], a[7], a[58]); // Sum(65) + res[66] = fma52lo(res[66], a[8], a[58]); // Sum(66) + res[67] = fma52hi(res[67], a[8], a[58]); // Sum(66) + res[67] = fma52lo(res[67], a[9], a[58]); // Sum(67) + res[68] = fma52hi(res[68], a[9], a[58]); // Sum(67) + res[68] = fma52lo(res[68], a[10], a[58]); // Sum(68) + res[69] = fma52hi(res[69], a[10], a[58]); // Sum(68) + res[69] = fma52lo(res[69], a[11], a[58]); // Sum(69) + res[70] = fma52hi(res[70], a[11], a[58]); // Sum(69) + res[70] = fma52lo(res[70], a[12], a[58]); // Sum(70) + res[71] = fma52hi(res[71], a[12], a[58]); // Sum(70) + res[71] = fma52lo(res[71], a[13], a[58]); // Sum(71) + res[72] = fma52hi(res[72], a[13], a[58]); // Sum(71) + res[60] = fma52lo(res[60], a[1], a[59]); // Sum(60) + res[61] = fma52hi(res[61], a[1], a[59]); // Sum(60) + res[61] = fma52lo(res[61], a[2], a[59]); // Sum(61) + res[62] = fma52hi(res[62], a[2], a[59]); // Sum(61) + res[62] = fma52lo(res[62], a[3], a[59]); // Sum(62) + res[63] = fma52hi(res[63], a[3], a[59]); // Sum(62) + res[63] = fma52lo(res[63], a[4], a[59]); // Sum(63) + res[64] = fma52hi(res[64], a[4], a[59]); // Sum(63) + res[64] = fma52lo(res[64], a[5], a[59]); // Sum(64) + res[65] = fma52hi(res[65], a[5], a[59]); // Sum(64) + res[65] = fma52lo(res[65], a[6], a[59]); // Sum(65) + res[66] = fma52hi(res[66], a[6], a[59]); // Sum(65) + res[66] = fma52lo(res[66], a[7], a[59]); // Sum(66) + res[67] = fma52hi(res[67], a[7], a[59]); // Sum(66) + res[67] = fma52lo(res[67], a[8], a[59]); // Sum(67) + res[68] = fma52hi(res[68], a[8], a[59]); // Sum(67) + res[68] = fma52lo(res[68], a[9], a[59]); // Sum(68) + res[69] = fma52hi(res[69], a[9], a[59]); // Sum(68) + res[69] = fma52lo(res[69], a[10], a[59]); // Sum(69) + res[70] = fma52hi(res[70], a[10], a[59]); // Sum(69) + res[70] = fma52lo(res[70], a[11], a[59]); // Sum(70) + res[71] = fma52hi(res[71], a[11], a[59]); // Sum(70) + res[71] = fma52lo(res[71], a[12], a[59]); // Sum(71) + res[72] = fma52hi(res[72], a[12], a[59]); // Sum(71) + res[60] = fma52lo(res[60], a[0], a[60]); // Sum(60) + res[61] = fma52hi(res[61], a[0], a[60]); // Sum(60) + res[61] = fma52lo(res[61], a[1], a[60]); // Sum(61) + res[62] = fma52hi(res[62], a[1], a[60]); // Sum(61) + res[62] = fma52lo(res[62], a[2], a[60]); // Sum(62) + res[63] = fma52hi(res[63], a[2], a[60]); // Sum(62) + res[63] = fma52lo(res[63], a[3], a[60]); // Sum(63) + res[64] = fma52hi(res[64], a[3], a[60]); // Sum(63) + res[64] = fma52lo(res[64], a[4], a[60]); // Sum(64) + res[65] = fma52hi(res[65], a[4], a[60]); // Sum(64) + res[65] = fma52lo(res[65], a[5], a[60]); // Sum(65) + res[66] = fma52hi(res[66], a[5], a[60]); // Sum(65) + res[66] = fma52lo(res[66], a[6], a[60]); // Sum(66) + res[67] = fma52hi(res[67], a[6], a[60]); // Sum(66) + res[67] = fma52lo(res[67], a[7], a[60]); // Sum(67) + res[68] = fma52hi(res[68], a[7], a[60]); // Sum(67) + res[68] = fma52lo(res[68], a[8], a[60]); // Sum(68) + res[69] = fma52hi(res[69], a[8], a[60]); // Sum(68) + res[69] = fma52lo(res[69], a[9], a[60]); // Sum(69) + res[70] = fma52hi(res[70], a[9], a[60]); // Sum(69) + res[70] = fma52lo(res[70], a[10], a[60]); // Sum(70) + res[71] = fma52hi(res[71], a[10], a[60]); // Sum(70) + res[71] = fma52lo(res[71], a[11], a[60]); // Sum(71) + res[72] = fma52hi(res[72], a[11], a[60]); // Sum(71) + res[61] = fma52lo(res[61], a[0], a[61]); // Sum(61) + res[62] = fma52hi(res[62], a[0], a[61]); // Sum(61) + res[62] = fma52lo(res[62], a[1], a[61]); // Sum(62) + res[63] = fma52hi(res[63], a[1], a[61]); // Sum(62) + res[63] = fma52lo(res[63], a[2], a[61]); // Sum(63) + res[64] = fma52hi(res[64], a[2], a[61]); // Sum(63) + res[64] = fma52lo(res[64], a[3], a[61]); // Sum(64) + res[65] = fma52hi(res[65], a[3], a[61]); // Sum(64) + res[65] = fma52lo(res[65], a[4], a[61]); // Sum(65) + res[66] = fma52hi(res[66], a[4], a[61]); // Sum(65) + res[66] = fma52lo(res[66], a[5], a[61]); // Sum(66) + res[67] = fma52hi(res[67], a[5], a[61]); // Sum(66) + res[67] = fma52lo(res[67], a[6], a[61]); // Sum(67) + res[68] = fma52hi(res[68], a[6], a[61]); // Sum(67) + res[68] = fma52lo(res[68], a[7], a[61]); // Sum(68) + res[69] = fma52hi(res[69], a[7], a[61]); // Sum(68) + res[69] = fma52lo(res[69], a[8], a[61]); // Sum(69) + res[70] = fma52hi(res[70], a[8], a[61]); // Sum(69) + res[70] = fma52lo(res[70], a[9], a[61]); // Sum(70) + res[71] = fma52hi(res[71], a[9], a[61]); // Sum(70) + res[71] = fma52lo(res[71], a[10], a[61]); // Sum(71) + res[72] = fma52hi(res[72], a[10], a[61]); // Sum(71) + res[62] = fma52lo(res[62], a[0], a[62]); // Sum(62) + res[63] = fma52hi(res[63], a[0], a[62]); // Sum(62) + res[63] = fma52lo(res[63], a[1], a[62]); // Sum(63) + res[64] = fma52hi(res[64], a[1], a[62]); // Sum(63) + res[64] = fma52lo(res[64], a[2], a[62]); // Sum(64) + res[65] = fma52hi(res[65], a[2], a[62]); // Sum(64) + res[65] = fma52lo(res[65], a[3], a[62]); // Sum(65) + res[66] = fma52hi(res[66], a[3], a[62]); // Sum(65) + res[66] = fma52lo(res[66], a[4], a[62]); // Sum(66) + res[67] = fma52hi(res[67], a[4], a[62]); // Sum(66) + res[67] = fma52lo(res[67], a[5], a[62]); // Sum(67) + res[68] = fma52hi(res[68], a[5], a[62]); // Sum(67) + res[68] = fma52lo(res[68], a[6], a[62]); // Sum(68) + res[69] = fma52hi(res[69], a[6], a[62]); // Sum(68) + res[69] = fma52lo(res[69], a[7], a[62]); // Sum(69) + res[70] = fma52hi(res[70], a[7], a[62]); // Sum(69) + res[70] = fma52lo(res[70], a[8], a[62]); // Sum(70) + res[71] = fma52hi(res[71], a[8], a[62]); // Sum(70) + res[71] = fma52lo(res[71], a[9], a[62]); // Sum(71) + res[72] = fma52hi(res[72], a[9], a[62]); // Sum(71) + res[63] = fma52lo(res[63], a[0], a[63]); // Sum(63) + res[64] = fma52hi(res[64], a[0], a[63]); // Sum(63) + res[64] = fma52lo(res[64], a[1], a[63]); // Sum(64) + res[65] = fma52hi(res[65], a[1], a[63]); // Sum(64) + res[65] = fma52lo(res[65], a[2], a[63]); // Sum(65) + res[66] = fma52hi(res[66], a[2], a[63]); // Sum(65) + res[66] = fma52lo(res[66], a[3], a[63]); // Sum(66) + res[67] = fma52hi(res[67], a[3], a[63]); // Sum(66) + res[67] = fma52lo(res[67], a[4], a[63]); // Sum(67) + res[68] = fma52hi(res[68], a[4], a[63]); // Sum(67) + res[68] = fma52lo(res[68], a[5], a[63]); // Sum(68) + res[69] = fma52hi(res[69], a[5], a[63]); // Sum(68) + res[69] = fma52lo(res[69], a[6], a[63]); // Sum(69) + res[70] = fma52hi(res[70], a[6], a[63]); // Sum(69) + res[70] = fma52lo(res[70], a[7], a[63]); // Sum(70) + res[71] = fma52hi(res[71], a[7], a[63]); // Sum(70) + res[71] = fma52lo(res[71], a[8], a[63]); // Sum(71) + res[72] = fma52hi(res[72], a[8], a[63]); // Sum(71) + res[64] = fma52lo(res[64], a[0], a[64]); // Sum(64) + res[65] = fma52hi(res[65], a[0], a[64]); // Sum(64) + res[65] = fma52lo(res[65], a[1], a[64]); // Sum(65) + res[66] = fma52hi(res[66], a[1], a[64]); // Sum(65) + res[66] = fma52lo(res[66], a[2], a[64]); // Sum(66) + res[67] = fma52hi(res[67], a[2], a[64]); // Sum(66) + res[67] = fma52lo(res[67], a[3], a[64]); // Sum(67) + res[68] = fma52hi(res[68], a[3], a[64]); // Sum(67) + res[68] = fma52lo(res[68], a[4], a[64]); // Sum(68) + res[69] = fma52hi(res[69], a[4], a[64]); // Sum(68) + res[69] = fma52lo(res[69], a[5], a[64]); // Sum(69) + res[70] = fma52hi(res[70], a[5], a[64]); // Sum(69) + res[70] = fma52lo(res[70], a[6], a[64]); // Sum(70) + res[71] = fma52hi(res[71], a[6], a[64]); // Sum(70) + res[71] = fma52lo(res[71], a[7], a[64]); // Sum(71) + res[72] = fma52hi(res[72], a[7], a[64]); // Sum(71) + res[65] = fma52lo(res[65], a[0], a[65]); // Sum(65) + res[66] = fma52hi(res[66], a[0], a[65]); // Sum(65) + res[66] = fma52lo(res[66], a[1], a[65]); // Sum(66) + res[67] = fma52hi(res[67], a[1], a[65]); // Sum(66) + res[67] = fma52lo(res[67], a[2], a[65]); // Sum(67) + res[68] = fma52hi(res[68], a[2], a[65]); // Sum(67) + res[68] = fma52lo(res[68], a[3], a[65]); // Sum(68) + res[69] = fma52hi(res[69], a[3], a[65]); // Sum(68) + res[69] = fma52lo(res[69], a[4], a[65]); // Sum(69) + res[70] = fma52hi(res[70], a[4], a[65]); // Sum(69) + res[70] = fma52lo(res[70], a[5], a[65]); // Sum(70) + res[71] = fma52hi(res[71], a[5], a[65]); // Sum(70) + res[71] = fma52lo(res[71], a[6], a[65]); // Sum(71) + res[72] = fma52hi(res[72], a[6], a[65]); // Sum(71) + res[66] = fma52lo(res[66], a[0], a[66]); // Sum(66) + res[67] = fma52hi(res[67], a[0], a[66]); // Sum(66) + res[67] = fma52lo(res[67], a[1], a[66]); // Sum(67) + res[68] = fma52hi(res[68], a[1], a[66]); // Sum(67) + res[68] = fma52lo(res[68], a[2], a[66]); // Sum(68) + res[69] = fma52hi(res[69], a[2], a[66]); // Sum(68) + res[69] = fma52lo(res[69], a[3], a[66]); // Sum(69) + res[70] = fma52hi(res[70], a[3], a[66]); // Sum(69) + res[70] = fma52lo(res[70], a[4], a[66]); // Sum(70) + res[71] = fma52hi(res[71], a[4], a[66]); // Sum(70) + res[71] = fma52lo(res[71], a[5], a[66]); // Sum(71) + res[72] = fma52hi(res[72], a[5], a[66]); // Sum(71) + res[67] = fma52lo(res[67], a[0], a[67]); // Sum(67) + res[68] = fma52hi(res[68], a[0], a[67]); // Sum(67) + res[68] = fma52lo(res[68], a[1], a[67]); // Sum(68) + res[69] = fma52hi(res[69], a[1], a[67]); // Sum(68) + res[69] = fma52lo(res[69], a[2], a[67]); // Sum(69) + res[70] = fma52hi(res[70], a[2], a[67]); // Sum(69) + res[70] = fma52lo(res[70], a[3], a[67]); // Sum(70) + res[71] = fma52hi(res[71], a[3], a[67]); // Sum(70) + res[71] = fma52lo(res[71], a[4], a[67]); // Sum(71) + res[72] = fma52hi(res[72], a[4], a[67]); // Sum(71) + res[68] = fma52lo(res[68], a[0], a[68]); // Sum(68) + res[69] = fma52hi(res[69], a[0], a[68]); // Sum(68) + res[69] = fma52lo(res[69], a[1], a[68]); // Sum(69) + res[70] = fma52hi(res[70], a[1], a[68]); // Sum(69) + res[70] = fma52lo(res[70], a[2], a[68]); // Sum(70) + res[71] = fma52hi(res[71], a[2], a[68]); // Sum(70) + res[71] = fma52lo(res[71], a[3], a[68]); // Sum(71) + res[72] = fma52hi(res[72], a[3], a[68]); // Sum(71) + res[69] = fma52lo(res[69], a[0], a[69]); // Sum(69) + res[70] = fma52hi(res[70], a[0], a[69]); // Sum(69) + res[70] = fma52lo(res[70], a[1], a[69]); // Sum(70) + res[71] = fma52hi(res[71], a[1], a[69]); // Sum(70) + res[71] = fma52lo(res[71], a[2], a[69]); // Sum(71) + res[72] = fma52hi(res[72], a[2], a[69]); // Sum(71) + res[70] = fma52lo(res[70], a[0], a[70]); // Sum(70) + res[71] = fma52hi(res[71], a[0], a[70]); // Sum(70) + res[71] = fma52lo(res[71], a[1], a[70]); // Sum(71) + res[72] = fma52hi(res[72], a[1], a[70]); // Sum(71) + res[71] = fma52lo(res[71], a[0], a[71]); // Sum(71) + res[72] = fma52hi(res[72], a[0], a[71]); // Sum(71) + res[60] = add64(res[60], res[60]); // Double(60) + res[61] = add64(res[61], res[61]); // Double(61) + res[62] = add64(res[62], res[62]); // Double(62) + res[63] = add64(res[63], res[63]); // Double(63) + res[64] = add64(res[64], res[64]); // Double(64) + res[65] = add64(res[65], res[65]); // Double(65) + res[66] = add64(res[66], res[66]); // Double(66) + res[67] = add64(res[67], res[67]); // Double(67) + res[68] = add64(res[68], res[68]); // Double(68) + res[69] = add64(res[69], res[69]); // Double(69) + res[70] = add64(res[70], res[70]); // Double(70) + res[71] = add64(res[71], res[71]); // Double(71) + res[60] = fma52lo(res[60], a[30], a[30]); // Add sqr(60) + res[61] = fma52hi(res[61], a[30], a[30]); // Add sqr(60) + res[62] = fma52lo(res[62], a[31], a[31]); // Add sqr(62) + res[63] = fma52hi(res[63], a[31], a[31]); // Add sqr(62) + res[64] = fma52lo(res[64], a[32], a[32]); // Add sqr(64) + res[65] = fma52hi(res[65], a[32], a[32]); // Add sqr(64) + res[66] = fma52lo(res[66], a[33], a[33]); // Add sqr(66) + res[67] = fma52hi(res[67], a[33], a[33]); // Add sqr(66) + res[68] = fma52lo(res[68], a[34], a[34]); // Add sqr(68) + res[69] = fma52hi(res[69], a[34], a[34]); // Add sqr(68) + res[70] = fma52lo(res[70], a[35], a[35]); // Add sqr(70) + res[71] = fma52hi(res[71], a[35], a[35]); // Add sqr(70) + res[72] = fma52lo(res[72], a[35], a[37]); // Sum(72) + res[73] = fma52hi(res[73], a[35], a[37]); // Sum(72) + res[73] = fma52lo(res[73], a[36], a[37]); // Sum(73) + res[74] = fma52hi(res[74], a[36], a[37]); // Sum(73) + res[72] = fma52lo(res[72], a[34], a[38]); // Sum(72) + res[73] = fma52hi(res[73], a[34], a[38]); // Sum(72) + res[73] = fma52lo(res[73], a[35], a[38]); // Sum(73) + res[74] = fma52hi(res[74], a[35], a[38]); // Sum(73) + res[74] = fma52lo(res[74], a[36], a[38]); // Sum(74) + res[75] = fma52hi(res[75], a[36], a[38]); // Sum(74) + res[75] = fma52lo(res[75], a[37], a[38]); // Sum(75) + res[76] = fma52hi(res[76], a[37], a[38]); // Sum(75) + res[72] = fma52lo(res[72], a[33], a[39]); // Sum(72) + res[73] = fma52hi(res[73], a[33], a[39]); // Sum(72) + res[73] = fma52lo(res[73], a[34], a[39]); // Sum(73) + res[74] = fma52hi(res[74], a[34], a[39]); // Sum(73) + res[74] = fma52lo(res[74], a[35], a[39]); // Sum(74) + res[75] = fma52hi(res[75], a[35], a[39]); // Sum(74) + res[75] = fma52lo(res[75], a[36], a[39]); // Sum(75) + res[76] = fma52hi(res[76], a[36], a[39]); // Sum(75) + res[76] = fma52lo(res[76], a[37], a[39]); // Sum(76) + res[77] = fma52hi(res[77], a[37], a[39]); // Sum(76) + res[77] = fma52lo(res[77], a[38], a[39]); // Sum(77) + res[78] = fma52hi(res[78], a[38], a[39]); // Sum(77) + res[72] = fma52lo(res[72], a[32], a[40]); // Sum(72) + res[73] = fma52hi(res[73], a[32], a[40]); // Sum(72) + res[73] = fma52lo(res[73], a[33], a[40]); // Sum(73) + res[74] = fma52hi(res[74], a[33], a[40]); // Sum(73) + res[74] = fma52lo(res[74], a[34], a[40]); // Sum(74) + res[75] = fma52hi(res[75], a[34], a[40]); // Sum(74) + res[75] = fma52lo(res[75], a[35], a[40]); // Sum(75) + res[76] = fma52hi(res[76], a[35], a[40]); // Sum(75) + res[76] = fma52lo(res[76], a[36], a[40]); // Sum(76) + res[77] = fma52hi(res[77], a[36], a[40]); // Sum(76) + res[77] = fma52lo(res[77], a[37], a[40]); // Sum(77) + res[78] = fma52hi(res[78], a[37], a[40]); // Sum(77) + res[78] = fma52lo(res[78], a[38], a[40]); // Sum(78) + res[79] = fma52hi(res[79], a[38], a[40]); // Sum(78) + res[79] = fma52lo(res[79], a[39], a[40]); // Sum(79) + res[80] = fma52hi(res[80], a[39], a[40]); // Sum(79) + res[72] = fma52lo(res[72], a[31], a[41]); // Sum(72) + res[73] = fma52hi(res[73], a[31], a[41]); // Sum(72) + res[73] = fma52lo(res[73], a[32], a[41]); // Sum(73) + res[74] = fma52hi(res[74], a[32], a[41]); // Sum(73) + res[74] = fma52lo(res[74], a[33], a[41]); // Sum(74) + res[75] = fma52hi(res[75], a[33], a[41]); // Sum(74) + res[75] = fma52lo(res[75], a[34], a[41]); // Sum(75) + res[76] = fma52hi(res[76], a[34], a[41]); // Sum(75) + res[76] = fma52lo(res[76], a[35], a[41]); // Sum(76) + res[77] = fma52hi(res[77], a[35], a[41]); // Sum(76) + res[77] = fma52lo(res[77], a[36], a[41]); // Sum(77) + res[78] = fma52hi(res[78], a[36], a[41]); // Sum(77) + res[78] = fma52lo(res[78], a[37], a[41]); // Sum(78) + res[79] = fma52hi(res[79], a[37], a[41]); // Sum(78) + res[79] = fma52lo(res[79], a[38], a[41]); // Sum(79) + res[80] = fma52hi(res[80], a[38], a[41]); // Sum(79) + res[80] = fma52lo(res[80], a[39], a[41]); // Sum(80) + res[81] = fma52hi(res[81], a[39], a[41]); // Sum(80) + res[81] = fma52lo(res[81], a[40], a[41]); // Sum(81) + res[82] = fma52hi(res[82], a[40], a[41]); // Sum(81) + res[72] = fma52lo(res[72], a[30], a[42]); // Sum(72) + res[73] = fma52hi(res[73], a[30], a[42]); // Sum(72) + res[73] = fma52lo(res[73], a[31], a[42]); // Sum(73) + res[74] = fma52hi(res[74], a[31], a[42]); // Sum(73) + res[74] = fma52lo(res[74], a[32], a[42]); // Sum(74) + res[75] = fma52hi(res[75], a[32], a[42]); // Sum(74) + res[75] = fma52lo(res[75], a[33], a[42]); // Sum(75) + res[76] = fma52hi(res[76], a[33], a[42]); // Sum(75) + res[76] = fma52lo(res[76], a[34], a[42]); // Sum(76) + res[77] = fma52hi(res[77], a[34], a[42]); // Sum(76) + res[77] = fma52lo(res[77], a[35], a[42]); // Sum(77) + res[78] = fma52hi(res[78], a[35], a[42]); // Sum(77) + res[78] = fma52lo(res[78], a[36], a[42]); // Sum(78) + res[79] = fma52hi(res[79], a[36], a[42]); // Sum(78) + res[79] = fma52lo(res[79], a[37], a[42]); // Sum(79) + res[80] = fma52hi(res[80], a[37], a[42]); // Sum(79) + res[80] = fma52lo(res[80], a[38], a[42]); // Sum(80) + res[81] = fma52hi(res[81], a[38], a[42]); // Sum(80) + res[81] = fma52lo(res[81], a[39], a[42]); // Sum(81) + res[82] = fma52hi(res[82], a[39], a[42]); // Sum(81) + res[82] = fma52lo(res[82], a[40], a[42]); // Sum(82) + res[83] = fma52hi(res[83], a[40], a[42]); // Sum(82) + res[83] = fma52lo(res[83], a[41], a[42]); // Sum(83) + res[84] = fma52hi(res[84], a[41], a[42]); // Sum(83) + res[72] = fma52lo(res[72], a[29], a[43]); // Sum(72) + res[73] = fma52hi(res[73], a[29], a[43]); // Sum(72) + res[73] = fma52lo(res[73], a[30], a[43]); // Sum(73) + res[74] = fma52hi(res[74], a[30], a[43]); // Sum(73) + res[74] = fma52lo(res[74], a[31], a[43]); // Sum(74) + res[75] = fma52hi(res[75], a[31], a[43]); // Sum(74) + res[75] = fma52lo(res[75], a[32], a[43]); // Sum(75) + res[76] = fma52hi(res[76], a[32], a[43]); // Sum(75) + res[76] = fma52lo(res[76], a[33], a[43]); // Sum(76) + res[77] = fma52hi(res[77], a[33], a[43]); // Sum(76) + res[77] = fma52lo(res[77], a[34], a[43]); // Sum(77) + res[78] = fma52hi(res[78], a[34], a[43]); // Sum(77) + res[78] = fma52lo(res[78], a[35], a[43]); // Sum(78) + res[79] = fma52hi(res[79], a[35], a[43]); // Sum(78) + res[79] = fma52lo(res[79], a[36], a[43]); // Sum(79) + res[80] = fma52hi(res[80], a[36], a[43]); // Sum(79) + res[80] = fma52lo(res[80], a[37], a[43]); // Sum(80) + res[81] = fma52hi(res[81], a[37], a[43]); // Sum(80) + res[81] = fma52lo(res[81], a[38], a[43]); // Sum(81) + res[82] = fma52hi(res[82], a[38], a[43]); // Sum(81) + res[82] = fma52lo(res[82], a[39], a[43]); // Sum(82) + res[83] = fma52hi(res[83], a[39], a[43]); // Sum(82) + res[83] = fma52lo(res[83], a[40], a[43]); // Sum(83) + res[84] = fma52hi(res[84], a[40], a[43]); // Sum(83) + res[72] = fma52lo(res[72], a[28], a[44]); // Sum(72) + res[73] = fma52hi(res[73], a[28], a[44]); // Sum(72) + res[73] = fma52lo(res[73], a[29], a[44]); // Sum(73) + res[74] = fma52hi(res[74], a[29], a[44]); // Sum(73) + res[74] = fma52lo(res[74], a[30], a[44]); // Sum(74) + res[75] = fma52hi(res[75], a[30], a[44]); // Sum(74) + res[75] = fma52lo(res[75], a[31], a[44]); // Sum(75) + res[76] = fma52hi(res[76], a[31], a[44]); // Sum(75) + res[76] = fma52lo(res[76], a[32], a[44]); // Sum(76) + res[77] = fma52hi(res[77], a[32], a[44]); // Sum(76) + res[77] = fma52lo(res[77], a[33], a[44]); // Sum(77) + res[78] = fma52hi(res[78], a[33], a[44]); // Sum(77) + res[78] = fma52lo(res[78], a[34], a[44]); // Sum(78) + res[79] = fma52hi(res[79], a[34], a[44]); // Sum(78) + res[79] = fma52lo(res[79], a[35], a[44]); // Sum(79) + res[80] = fma52hi(res[80], a[35], a[44]); // Sum(79) + res[80] = fma52lo(res[80], a[36], a[44]); // Sum(80) + res[81] = fma52hi(res[81], a[36], a[44]); // Sum(80) + res[81] = fma52lo(res[81], a[37], a[44]); // Sum(81) + res[82] = fma52hi(res[82], a[37], a[44]); // Sum(81) + res[82] = fma52lo(res[82], a[38], a[44]); // Sum(82) + res[83] = fma52hi(res[83], a[38], a[44]); // Sum(82) + res[83] = fma52lo(res[83], a[39], a[44]); // Sum(83) + res[84] = fma52hi(res[84], a[39], a[44]); // Sum(83) + res[72] = fma52lo(res[72], a[27], a[45]); // Sum(72) + res[73] = fma52hi(res[73], a[27], a[45]); // Sum(72) + res[73] = fma52lo(res[73], a[28], a[45]); // Sum(73) + res[74] = fma52hi(res[74], a[28], a[45]); // Sum(73) + res[74] = fma52lo(res[74], a[29], a[45]); // Sum(74) + res[75] = fma52hi(res[75], a[29], a[45]); // Sum(74) + res[75] = fma52lo(res[75], a[30], a[45]); // Sum(75) + res[76] = fma52hi(res[76], a[30], a[45]); // Sum(75) + res[76] = fma52lo(res[76], a[31], a[45]); // Sum(76) + res[77] = fma52hi(res[77], a[31], a[45]); // Sum(76) + res[77] = fma52lo(res[77], a[32], a[45]); // Sum(77) + res[78] = fma52hi(res[78], a[32], a[45]); // Sum(77) + res[78] = fma52lo(res[78], a[33], a[45]); // Sum(78) + res[79] = fma52hi(res[79], a[33], a[45]); // Sum(78) + res[79] = fma52lo(res[79], a[34], a[45]); // Sum(79) + res[80] = fma52hi(res[80], a[34], a[45]); // Sum(79) + res[80] = fma52lo(res[80], a[35], a[45]); // Sum(80) + res[81] = fma52hi(res[81], a[35], a[45]); // Sum(80) + res[81] = fma52lo(res[81], a[36], a[45]); // Sum(81) + res[82] = fma52hi(res[82], a[36], a[45]); // Sum(81) + res[82] = fma52lo(res[82], a[37], a[45]); // Sum(82) + res[83] = fma52hi(res[83], a[37], a[45]); // Sum(82) + res[83] = fma52lo(res[83], a[38], a[45]); // Sum(83) + res[84] = fma52hi(res[84], a[38], a[45]); // Sum(83) + res[72] = fma52lo(res[72], a[26], a[46]); // Sum(72) + res[73] = fma52hi(res[73], a[26], a[46]); // Sum(72) + res[73] = fma52lo(res[73], a[27], a[46]); // Sum(73) + res[74] = fma52hi(res[74], a[27], a[46]); // Sum(73) + res[74] = fma52lo(res[74], a[28], a[46]); // Sum(74) + res[75] = fma52hi(res[75], a[28], a[46]); // Sum(74) + res[75] = fma52lo(res[75], a[29], a[46]); // Sum(75) + res[76] = fma52hi(res[76], a[29], a[46]); // Sum(75) + res[76] = fma52lo(res[76], a[30], a[46]); // Sum(76) + res[77] = fma52hi(res[77], a[30], a[46]); // Sum(76) + res[77] = fma52lo(res[77], a[31], a[46]); // Sum(77) + res[78] = fma52hi(res[78], a[31], a[46]); // Sum(77) + res[78] = fma52lo(res[78], a[32], a[46]); // Sum(78) + res[79] = fma52hi(res[79], a[32], a[46]); // Sum(78) + res[79] = fma52lo(res[79], a[33], a[46]); // Sum(79) + res[80] = fma52hi(res[80], a[33], a[46]); // Sum(79) + res[80] = fma52lo(res[80], a[34], a[46]); // Sum(80) + res[81] = fma52hi(res[81], a[34], a[46]); // Sum(80) + res[81] = fma52lo(res[81], a[35], a[46]); // Sum(81) + res[82] = fma52hi(res[82], a[35], a[46]); // Sum(81) + res[82] = fma52lo(res[82], a[36], a[46]); // Sum(82) + res[83] = fma52hi(res[83], a[36], a[46]); // Sum(82) + res[83] = fma52lo(res[83], a[37], a[46]); // Sum(83) + res[84] = fma52hi(res[84], a[37], a[46]); // Sum(83) + res[72] = fma52lo(res[72], a[25], a[47]); // Sum(72) + res[73] = fma52hi(res[73], a[25], a[47]); // Sum(72) + res[73] = fma52lo(res[73], a[26], a[47]); // Sum(73) + res[74] = fma52hi(res[74], a[26], a[47]); // Sum(73) + res[74] = fma52lo(res[74], a[27], a[47]); // Sum(74) + res[75] = fma52hi(res[75], a[27], a[47]); // Sum(74) + res[75] = fma52lo(res[75], a[28], a[47]); // Sum(75) + res[76] = fma52hi(res[76], a[28], a[47]); // Sum(75) + res[76] = fma52lo(res[76], a[29], a[47]); // Sum(76) + res[77] = fma52hi(res[77], a[29], a[47]); // Sum(76) + res[77] = fma52lo(res[77], a[30], a[47]); // Sum(77) + res[78] = fma52hi(res[78], a[30], a[47]); // Sum(77) + res[78] = fma52lo(res[78], a[31], a[47]); // Sum(78) + res[79] = fma52hi(res[79], a[31], a[47]); // Sum(78) + res[79] = fma52lo(res[79], a[32], a[47]); // Sum(79) + res[80] = fma52hi(res[80], a[32], a[47]); // Sum(79) + res[80] = fma52lo(res[80], a[33], a[47]); // Sum(80) + res[81] = fma52hi(res[81], a[33], a[47]); // Sum(80) + res[81] = fma52lo(res[81], a[34], a[47]); // Sum(81) + res[82] = fma52hi(res[82], a[34], a[47]); // Sum(81) + res[82] = fma52lo(res[82], a[35], a[47]); // Sum(82) + res[83] = fma52hi(res[83], a[35], a[47]); // Sum(82) + res[83] = fma52lo(res[83], a[36], a[47]); // Sum(83) + res[84] = fma52hi(res[84], a[36], a[47]); // Sum(83) + res[72] = fma52lo(res[72], a[24], a[48]); // Sum(72) + res[73] = fma52hi(res[73], a[24], a[48]); // Sum(72) + res[73] = fma52lo(res[73], a[25], a[48]); // Sum(73) + res[74] = fma52hi(res[74], a[25], a[48]); // Sum(73) + res[74] = fma52lo(res[74], a[26], a[48]); // Sum(74) + res[75] = fma52hi(res[75], a[26], a[48]); // Sum(74) + res[75] = fma52lo(res[75], a[27], a[48]); // Sum(75) + res[76] = fma52hi(res[76], a[27], a[48]); // Sum(75) + res[76] = fma52lo(res[76], a[28], a[48]); // Sum(76) + res[77] = fma52hi(res[77], a[28], a[48]); // Sum(76) + res[77] = fma52lo(res[77], a[29], a[48]); // Sum(77) + res[78] = fma52hi(res[78], a[29], a[48]); // Sum(77) + res[78] = fma52lo(res[78], a[30], a[48]); // Sum(78) + res[79] = fma52hi(res[79], a[30], a[48]); // Sum(78) + res[79] = fma52lo(res[79], a[31], a[48]); // Sum(79) + res[80] = fma52hi(res[80], a[31], a[48]); // Sum(79) + res[80] = fma52lo(res[80], a[32], a[48]); // Sum(80) + res[81] = fma52hi(res[81], a[32], a[48]); // Sum(80) + res[81] = fma52lo(res[81], a[33], a[48]); // Sum(81) + res[82] = fma52hi(res[82], a[33], a[48]); // Sum(81) + res[82] = fma52lo(res[82], a[34], a[48]); // Sum(82) + res[83] = fma52hi(res[83], a[34], a[48]); // Sum(82) + res[83] = fma52lo(res[83], a[35], a[48]); // Sum(83) + res[84] = fma52hi(res[84], a[35], a[48]); // Sum(83) + res[72] = fma52lo(res[72], a[23], a[49]); // Sum(72) + res[73] = fma52hi(res[73], a[23], a[49]); // Sum(72) + res[73] = fma52lo(res[73], a[24], a[49]); // Sum(73) + res[74] = fma52hi(res[74], a[24], a[49]); // Sum(73) + res[74] = fma52lo(res[74], a[25], a[49]); // Sum(74) + res[75] = fma52hi(res[75], a[25], a[49]); // Sum(74) + res[75] = fma52lo(res[75], a[26], a[49]); // Sum(75) + res[76] = fma52hi(res[76], a[26], a[49]); // Sum(75) + res[76] = fma52lo(res[76], a[27], a[49]); // Sum(76) + res[77] = fma52hi(res[77], a[27], a[49]); // Sum(76) + res[77] = fma52lo(res[77], a[28], a[49]); // Sum(77) + res[78] = fma52hi(res[78], a[28], a[49]); // Sum(77) + res[78] = fma52lo(res[78], a[29], a[49]); // Sum(78) + res[79] = fma52hi(res[79], a[29], a[49]); // Sum(78) + res[79] = fma52lo(res[79], a[30], a[49]); // Sum(79) + res[80] = fma52hi(res[80], a[30], a[49]); // Sum(79) + res[80] = fma52lo(res[80], a[31], a[49]); // Sum(80) + res[81] = fma52hi(res[81], a[31], a[49]); // Sum(80) + res[81] = fma52lo(res[81], a[32], a[49]); // Sum(81) + res[82] = fma52hi(res[82], a[32], a[49]); // Sum(81) + res[82] = fma52lo(res[82], a[33], a[49]); // Sum(82) + res[83] = fma52hi(res[83], a[33], a[49]); // Sum(82) + res[83] = fma52lo(res[83], a[34], a[49]); // Sum(83) + res[84] = fma52hi(res[84], a[34], a[49]); // Sum(83) + res[72] = fma52lo(res[72], a[22], a[50]); // Sum(72) + res[73] = fma52hi(res[73], a[22], a[50]); // Sum(72) + res[73] = fma52lo(res[73], a[23], a[50]); // Sum(73) + res[74] = fma52hi(res[74], a[23], a[50]); // Sum(73) + res[74] = fma52lo(res[74], a[24], a[50]); // Sum(74) + res[75] = fma52hi(res[75], a[24], a[50]); // Sum(74) + res[75] = fma52lo(res[75], a[25], a[50]); // Sum(75) + res[76] = fma52hi(res[76], a[25], a[50]); // Sum(75) + res[76] = fma52lo(res[76], a[26], a[50]); // Sum(76) + res[77] = fma52hi(res[77], a[26], a[50]); // Sum(76) + res[77] = fma52lo(res[77], a[27], a[50]); // Sum(77) + res[78] = fma52hi(res[78], a[27], a[50]); // Sum(77) + res[78] = fma52lo(res[78], a[28], a[50]); // Sum(78) + res[79] = fma52hi(res[79], a[28], a[50]); // Sum(78) + res[79] = fma52lo(res[79], a[29], a[50]); // Sum(79) + res[80] = fma52hi(res[80], a[29], a[50]); // Sum(79) + res[80] = fma52lo(res[80], a[30], a[50]); // Sum(80) + res[81] = fma52hi(res[81], a[30], a[50]); // Sum(80) + res[81] = fma52lo(res[81], a[31], a[50]); // Sum(81) + res[82] = fma52hi(res[82], a[31], a[50]); // Sum(81) + res[82] = fma52lo(res[82], a[32], a[50]); // Sum(82) + res[83] = fma52hi(res[83], a[32], a[50]); // Sum(82) + res[83] = fma52lo(res[83], a[33], a[50]); // Sum(83) + res[84] = fma52hi(res[84], a[33], a[50]); // Sum(83) + res[72] = fma52lo(res[72], a[21], a[51]); // Sum(72) + res[73] = fma52hi(res[73], a[21], a[51]); // Sum(72) + res[73] = fma52lo(res[73], a[22], a[51]); // Sum(73) + res[74] = fma52hi(res[74], a[22], a[51]); // Sum(73) + res[74] = fma52lo(res[74], a[23], a[51]); // Sum(74) + res[75] = fma52hi(res[75], a[23], a[51]); // Sum(74) + res[75] = fma52lo(res[75], a[24], a[51]); // Sum(75) + res[76] = fma52hi(res[76], a[24], a[51]); // Sum(75) + res[76] = fma52lo(res[76], a[25], a[51]); // Sum(76) + res[77] = fma52hi(res[77], a[25], a[51]); // Sum(76) + res[77] = fma52lo(res[77], a[26], a[51]); // Sum(77) + res[78] = fma52hi(res[78], a[26], a[51]); // Sum(77) + res[78] = fma52lo(res[78], a[27], a[51]); // Sum(78) + res[79] = fma52hi(res[79], a[27], a[51]); // Sum(78) + res[79] = fma52lo(res[79], a[28], a[51]); // Sum(79) + res[80] = fma52hi(res[80], a[28], a[51]); // Sum(79) + res[80] = fma52lo(res[80], a[29], a[51]); // Sum(80) + res[81] = fma52hi(res[81], a[29], a[51]); // Sum(80) + res[81] = fma52lo(res[81], a[30], a[51]); // Sum(81) + res[82] = fma52hi(res[82], a[30], a[51]); // Sum(81) + res[82] = fma52lo(res[82], a[31], a[51]); // Sum(82) + res[83] = fma52hi(res[83], a[31], a[51]); // Sum(82) + res[83] = fma52lo(res[83], a[32], a[51]); // Sum(83) + res[84] = fma52hi(res[84], a[32], a[51]); // Sum(83) + res[72] = fma52lo(res[72], a[20], a[52]); // Sum(72) + res[73] = fma52hi(res[73], a[20], a[52]); // Sum(72) + res[73] = fma52lo(res[73], a[21], a[52]); // Sum(73) + res[74] = fma52hi(res[74], a[21], a[52]); // Sum(73) + res[74] = fma52lo(res[74], a[22], a[52]); // Sum(74) + res[75] = fma52hi(res[75], a[22], a[52]); // Sum(74) + res[75] = fma52lo(res[75], a[23], a[52]); // Sum(75) + res[76] = fma52hi(res[76], a[23], a[52]); // Sum(75) + res[76] = fma52lo(res[76], a[24], a[52]); // Sum(76) + res[77] = fma52hi(res[77], a[24], a[52]); // Sum(76) + res[77] = fma52lo(res[77], a[25], a[52]); // Sum(77) + res[78] = fma52hi(res[78], a[25], a[52]); // Sum(77) + res[78] = fma52lo(res[78], a[26], a[52]); // Sum(78) + res[79] = fma52hi(res[79], a[26], a[52]); // Sum(78) + res[79] = fma52lo(res[79], a[27], a[52]); // Sum(79) + res[80] = fma52hi(res[80], a[27], a[52]); // Sum(79) + res[80] = fma52lo(res[80], a[28], a[52]); // Sum(80) + res[81] = fma52hi(res[81], a[28], a[52]); // Sum(80) + res[81] = fma52lo(res[81], a[29], a[52]); // Sum(81) + res[82] = fma52hi(res[82], a[29], a[52]); // Sum(81) + res[82] = fma52lo(res[82], a[30], a[52]); // Sum(82) + res[83] = fma52hi(res[83], a[30], a[52]); // Sum(82) + res[83] = fma52lo(res[83], a[31], a[52]); // Sum(83) + res[84] = fma52hi(res[84], a[31], a[52]); // Sum(83) + res[72] = fma52lo(res[72], a[19], a[53]); // Sum(72) + res[73] = fma52hi(res[73], a[19], a[53]); // Sum(72) + res[73] = fma52lo(res[73], a[20], a[53]); // Sum(73) + res[74] = fma52hi(res[74], a[20], a[53]); // Sum(73) + res[74] = fma52lo(res[74], a[21], a[53]); // Sum(74) + res[75] = fma52hi(res[75], a[21], a[53]); // Sum(74) + res[75] = fma52lo(res[75], a[22], a[53]); // Sum(75) + res[76] = fma52hi(res[76], a[22], a[53]); // Sum(75) + res[76] = fma52lo(res[76], a[23], a[53]); // Sum(76) + res[77] = fma52hi(res[77], a[23], a[53]); // Sum(76) + res[77] = fma52lo(res[77], a[24], a[53]); // Sum(77) + res[78] = fma52hi(res[78], a[24], a[53]); // Sum(77) + res[78] = fma52lo(res[78], a[25], a[53]); // Sum(78) + res[79] = fma52hi(res[79], a[25], a[53]); // Sum(78) + res[79] = fma52lo(res[79], a[26], a[53]); // Sum(79) + res[80] = fma52hi(res[80], a[26], a[53]); // Sum(79) + res[80] = fma52lo(res[80], a[27], a[53]); // Sum(80) + res[81] = fma52hi(res[81], a[27], a[53]); // Sum(80) + res[81] = fma52lo(res[81], a[28], a[53]); // Sum(81) + res[82] = fma52hi(res[82], a[28], a[53]); // Sum(81) + res[82] = fma52lo(res[82], a[29], a[53]); // Sum(82) + res[83] = fma52hi(res[83], a[29], a[53]); // Sum(82) + res[83] = fma52lo(res[83], a[30], a[53]); // Sum(83) + res[84] = fma52hi(res[84], a[30], a[53]); // Sum(83) + res[72] = fma52lo(res[72], a[18], a[54]); // Sum(72) + res[73] = fma52hi(res[73], a[18], a[54]); // Sum(72) + res[73] = fma52lo(res[73], a[19], a[54]); // Sum(73) + res[74] = fma52hi(res[74], a[19], a[54]); // Sum(73) + res[74] = fma52lo(res[74], a[20], a[54]); // Sum(74) + res[75] = fma52hi(res[75], a[20], a[54]); // Sum(74) + res[75] = fma52lo(res[75], a[21], a[54]); // Sum(75) + res[76] = fma52hi(res[76], a[21], a[54]); // Sum(75) + res[76] = fma52lo(res[76], a[22], a[54]); // Sum(76) + res[77] = fma52hi(res[77], a[22], a[54]); // Sum(76) + res[77] = fma52lo(res[77], a[23], a[54]); // Sum(77) + res[78] = fma52hi(res[78], a[23], a[54]); // Sum(77) + res[78] = fma52lo(res[78], a[24], a[54]); // Sum(78) + res[79] = fma52hi(res[79], a[24], a[54]); // Sum(78) + res[79] = fma52lo(res[79], a[25], a[54]); // Sum(79) + res[80] = fma52hi(res[80], a[25], a[54]); // Sum(79) + res[80] = fma52lo(res[80], a[26], a[54]); // Sum(80) + res[81] = fma52hi(res[81], a[26], a[54]); // Sum(80) + res[81] = fma52lo(res[81], a[27], a[54]); // Sum(81) + res[82] = fma52hi(res[82], a[27], a[54]); // Sum(81) + res[82] = fma52lo(res[82], a[28], a[54]); // Sum(82) + res[83] = fma52hi(res[83], a[28], a[54]); // Sum(82) + res[83] = fma52lo(res[83], a[29], a[54]); // Sum(83) + res[84] = fma52hi(res[84], a[29], a[54]); // Sum(83) + res[72] = fma52lo(res[72], a[17], a[55]); // Sum(72) + res[73] = fma52hi(res[73], a[17], a[55]); // Sum(72) + res[73] = fma52lo(res[73], a[18], a[55]); // Sum(73) + res[74] = fma52hi(res[74], a[18], a[55]); // Sum(73) + res[74] = fma52lo(res[74], a[19], a[55]); // Sum(74) + res[75] = fma52hi(res[75], a[19], a[55]); // Sum(74) + res[75] = fma52lo(res[75], a[20], a[55]); // Sum(75) + res[76] = fma52hi(res[76], a[20], a[55]); // Sum(75) + res[76] = fma52lo(res[76], a[21], a[55]); // Sum(76) + res[77] = fma52hi(res[77], a[21], a[55]); // Sum(76) + res[77] = fma52lo(res[77], a[22], a[55]); // Sum(77) + res[78] = fma52hi(res[78], a[22], a[55]); // Sum(77) + res[78] = fma52lo(res[78], a[23], a[55]); // Sum(78) + res[79] = fma52hi(res[79], a[23], a[55]); // Sum(78) + res[79] = fma52lo(res[79], a[24], a[55]); // Sum(79) + res[80] = fma52hi(res[80], a[24], a[55]); // Sum(79) + res[80] = fma52lo(res[80], a[25], a[55]); // Sum(80) + res[81] = fma52hi(res[81], a[25], a[55]); // Sum(80) + res[81] = fma52lo(res[81], a[26], a[55]); // Sum(81) + res[82] = fma52hi(res[82], a[26], a[55]); // Sum(81) + res[82] = fma52lo(res[82], a[27], a[55]); // Sum(82) + res[83] = fma52hi(res[83], a[27], a[55]); // Sum(82) + res[83] = fma52lo(res[83], a[28], a[55]); // Sum(83) + res[84] = fma52hi(res[84], a[28], a[55]); // Sum(83) + res[72] = fma52lo(res[72], a[16], a[56]); // Sum(72) + res[73] = fma52hi(res[73], a[16], a[56]); // Sum(72) + res[73] = fma52lo(res[73], a[17], a[56]); // Sum(73) + res[74] = fma52hi(res[74], a[17], a[56]); // Sum(73) + res[74] = fma52lo(res[74], a[18], a[56]); // Sum(74) + res[75] = fma52hi(res[75], a[18], a[56]); // Sum(74) + res[75] = fma52lo(res[75], a[19], a[56]); // Sum(75) + res[76] = fma52hi(res[76], a[19], a[56]); // Sum(75) + res[76] = fma52lo(res[76], a[20], a[56]); // Sum(76) + res[77] = fma52hi(res[77], a[20], a[56]); // Sum(76) + res[77] = fma52lo(res[77], a[21], a[56]); // Sum(77) + res[78] = fma52hi(res[78], a[21], a[56]); // Sum(77) + res[78] = fma52lo(res[78], a[22], a[56]); // Sum(78) + res[79] = fma52hi(res[79], a[22], a[56]); // Sum(78) + res[79] = fma52lo(res[79], a[23], a[56]); // Sum(79) + res[80] = fma52hi(res[80], a[23], a[56]); // Sum(79) + res[80] = fma52lo(res[80], a[24], a[56]); // Sum(80) + res[81] = fma52hi(res[81], a[24], a[56]); // Sum(80) + res[81] = fma52lo(res[81], a[25], a[56]); // Sum(81) + res[82] = fma52hi(res[82], a[25], a[56]); // Sum(81) + res[82] = fma52lo(res[82], a[26], a[56]); // Sum(82) + res[83] = fma52hi(res[83], a[26], a[56]); // Sum(82) + res[83] = fma52lo(res[83], a[27], a[56]); // Sum(83) + res[84] = fma52hi(res[84], a[27], a[56]); // Sum(83) + res[72] = fma52lo(res[72], a[15], a[57]); // Sum(72) + res[73] = fma52hi(res[73], a[15], a[57]); // Sum(72) + res[73] = fma52lo(res[73], a[16], a[57]); // Sum(73) + res[74] = fma52hi(res[74], a[16], a[57]); // Sum(73) + res[74] = fma52lo(res[74], a[17], a[57]); // Sum(74) + res[75] = fma52hi(res[75], a[17], a[57]); // Sum(74) + res[75] = fma52lo(res[75], a[18], a[57]); // Sum(75) + res[76] = fma52hi(res[76], a[18], a[57]); // Sum(75) + res[76] = fma52lo(res[76], a[19], a[57]); // Sum(76) + res[77] = fma52hi(res[77], a[19], a[57]); // Sum(76) + res[77] = fma52lo(res[77], a[20], a[57]); // Sum(77) + res[78] = fma52hi(res[78], a[20], a[57]); // Sum(77) + res[78] = fma52lo(res[78], a[21], a[57]); // Sum(78) + res[79] = fma52hi(res[79], a[21], a[57]); // Sum(78) + res[79] = fma52lo(res[79], a[22], a[57]); // Sum(79) + res[80] = fma52hi(res[80], a[22], a[57]); // Sum(79) + res[80] = fma52lo(res[80], a[23], a[57]); // Sum(80) + res[81] = fma52hi(res[81], a[23], a[57]); // Sum(80) + res[81] = fma52lo(res[81], a[24], a[57]); // Sum(81) + res[82] = fma52hi(res[82], a[24], a[57]); // Sum(81) + res[82] = fma52lo(res[82], a[25], a[57]); // Sum(82) + res[83] = fma52hi(res[83], a[25], a[57]); // Sum(82) + res[83] = fma52lo(res[83], a[26], a[57]); // Sum(83) + res[84] = fma52hi(res[84], a[26], a[57]); // Sum(83) + res[72] = fma52lo(res[72], a[14], a[58]); // Sum(72) + res[73] = fma52hi(res[73], a[14], a[58]); // Sum(72) + res[73] = fma52lo(res[73], a[15], a[58]); // Sum(73) + res[74] = fma52hi(res[74], a[15], a[58]); // Sum(73) + res[74] = fma52lo(res[74], a[16], a[58]); // Sum(74) + res[75] = fma52hi(res[75], a[16], a[58]); // Sum(74) + res[75] = fma52lo(res[75], a[17], a[58]); // Sum(75) + res[76] = fma52hi(res[76], a[17], a[58]); // Sum(75) + res[76] = fma52lo(res[76], a[18], a[58]); // Sum(76) + res[77] = fma52hi(res[77], a[18], a[58]); // Sum(76) + res[77] = fma52lo(res[77], a[19], a[58]); // Sum(77) + res[78] = fma52hi(res[78], a[19], a[58]); // Sum(77) + res[78] = fma52lo(res[78], a[20], a[58]); // Sum(78) + res[79] = fma52hi(res[79], a[20], a[58]); // Sum(78) + res[79] = fma52lo(res[79], a[21], a[58]); // Sum(79) + res[80] = fma52hi(res[80], a[21], a[58]); // Sum(79) + res[80] = fma52lo(res[80], a[22], a[58]); // Sum(80) + res[81] = fma52hi(res[81], a[22], a[58]); // Sum(80) + res[81] = fma52lo(res[81], a[23], a[58]); // Sum(81) + res[82] = fma52hi(res[82], a[23], a[58]); // Sum(81) + res[82] = fma52lo(res[82], a[24], a[58]); // Sum(82) + res[83] = fma52hi(res[83], a[24], a[58]); // Sum(82) + res[83] = fma52lo(res[83], a[25], a[58]); // Sum(83) + res[84] = fma52hi(res[84], a[25], a[58]); // Sum(83) + res[72] = fma52lo(res[72], a[13], a[59]); // Sum(72) + res[73] = fma52hi(res[73], a[13], a[59]); // Sum(72) + res[73] = fma52lo(res[73], a[14], a[59]); // Sum(73) + res[74] = fma52hi(res[74], a[14], a[59]); // Sum(73) + res[74] = fma52lo(res[74], a[15], a[59]); // Sum(74) + res[75] = fma52hi(res[75], a[15], a[59]); // Sum(74) + res[75] = fma52lo(res[75], a[16], a[59]); // Sum(75) + res[76] = fma52hi(res[76], a[16], a[59]); // Sum(75) + res[76] = fma52lo(res[76], a[17], a[59]); // Sum(76) + res[77] = fma52hi(res[77], a[17], a[59]); // Sum(76) + res[77] = fma52lo(res[77], a[18], a[59]); // Sum(77) + res[78] = fma52hi(res[78], a[18], a[59]); // Sum(77) + res[78] = fma52lo(res[78], a[19], a[59]); // Sum(78) + res[79] = fma52hi(res[79], a[19], a[59]); // Sum(78) + res[79] = fma52lo(res[79], a[20], a[59]); // Sum(79) + res[80] = fma52hi(res[80], a[20], a[59]); // Sum(79) + res[80] = fma52lo(res[80], a[21], a[59]); // Sum(80) + res[81] = fma52hi(res[81], a[21], a[59]); // Sum(80) + res[81] = fma52lo(res[81], a[22], a[59]); // Sum(81) + res[82] = fma52hi(res[82], a[22], a[59]); // Sum(81) + res[82] = fma52lo(res[82], a[23], a[59]); // Sum(82) + res[83] = fma52hi(res[83], a[23], a[59]); // Sum(82) + res[83] = fma52lo(res[83], a[24], a[59]); // Sum(83) + res[84] = fma52hi(res[84], a[24], a[59]); // Sum(83) + res[72] = fma52lo(res[72], a[12], a[60]); // Sum(72) + res[73] = fma52hi(res[73], a[12], a[60]); // Sum(72) + res[73] = fma52lo(res[73], a[13], a[60]); // Sum(73) + res[74] = fma52hi(res[74], a[13], a[60]); // Sum(73) + res[74] = fma52lo(res[74], a[14], a[60]); // Sum(74) + res[75] = fma52hi(res[75], a[14], a[60]); // Sum(74) + res[75] = fma52lo(res[75], a[15], a[60]); // Sum(75) + res[76] = fma52hi(res[76], a[15], a[60]); // Sum(75) + res[76] = fma52lo(res[76], a[16], a[60]); // Sum(76) + res[77] = fma52hi(res[77], a[16], a[60]); // Sum(76) + res[77] = fma52lo(res[77], a[17], a[60]); // Sum(77) + res[78] = fma52hi(res[78], a[17], a[60]); // Sum(77) + res[78] = fma52lo(res[78], a[18], a[60]); // Sum(78) + res[79] = fma52hi(res[79], a[18], a[60]); // Sum(78) + res[79] = fma52lo(res[79], a[19], a[60]); // Sum(79) + res[80] = fma52hi(res[80], a[19], a[60]); // Sum(79) + res[80] = fma52lo(res[80], a[20], a[60]); // Sum(80) + res[81] = fma52hi(res[81], a[20], a[60]); // Sum(80) + res[81] = fma52lo(res[81], a[21], a[60]); // Sum(81) + res[82] = fma52hi(res[82], a[21], a[60]); // Sum(81) + res[82] = fma52lo(res[82], a[22], a[60]); // Sum(82) + res[83] = fma52hi(res[83], a[22], a[60]); // Sum(82) + res[83] = fma52lo(res[83], a[23], a[60]); // Sum(83) + res[84] = fma52hi(res[84], a[23], a[60]); // Sum(83) + res[72] = fma52lo(res[72], a[11], a[61]); // Sum(72) + res[73] = fma52hi(res[73], a[11], a[61]); // Sum(72) + res[73] = fma52lo(res[73], a[12], a[61]); // Sum(73) + res[74] = fma52hi(res[74], a[12], a[61]); // Sum(73) + res[74] = fma52lo(res[74], a[13], a[61]); // Sum(74) + res[75] = fma52hi(res[75], a[13], a[61]); // Sum(74) + res[75] = fma52lo(res[75], a[14], a[61]); // Sum(75) + res[76] = fma52hi(res[76], a[14], a[61]); // Sum(75) + res[76] = fma52lo(res[76], a[15], a[61]); // Sum(76) + res[77] = fma52hi(res[77], a[15], a[61]); // Sum(76) + res[77] = fma52lo(res[77], a[16], a[61]); // Sum(77) + res[78] = fma52hi(res[78], a[16], a[61]); // Sum(77) + res[78] = fma52lo(res[78], a[17], a[61]); // Sum(78) + res[79] = fma52hi(res[79], a[17], a[61]); // Sum(78) + res[79] = fma52lo(res[79], a[18], a[61]); // Sum(79) + res[80] = fma52hi(res[80], a[18], a[61]); // Sum(79) + res[80] = fma52lo(res[80], a[19], a[61]); // Sum(80) + res[81] = fma52hi(res[81], a[19], a[61]); // Sum(80) + res[81] = fma52lo(res[81], a[20], a[61]); // Sum(81) + res[82] = fma52hi(res[82], a[20], a[61]); // Sum(81) + res[82] = fma52lo(res[82], a[21], a[61]); // Sum(82) + res[83] = fma52hi(res[83], a[21], a[61]); // Sum(82) + res[83] = fma52lo(res[83], a[22], a[61]); // Sum(83) + res[84] = fma52hi(res[84], a[22], a[61]); // Sum(83) + res[72] = fma52lo(res[72], a[10], a[62]); // Sum(72) + res[73] = fma52hi(res[73], a[10], a[62]); // Sum(72) + res[73] = fma52lo(res[73], a[11], a[62]); // Sum(73) + res[74] = fma52hi(res[74], a[11], a[62]); // Sum(73) + res[74] = fma52lo(res[74], a[12], a[62]); // Sum(74) + res[75] = fma52hi(res[75], a[12], a[62]); // Sum(74) + res[75] = fma52lo(res[75], a[13], a[62]); // Sum(75) + res[76] = fma52hi(res[76], a[13], a[62]); // Sum(75) + res[76] = fma52lo(res[76], a[14], a[62]); // Sum(76) + res[77] = fma52hi(res[77], a[14], a[62]); // Sum(76) + res[77] = fma52lo(res[77], a[15], a[62]); // Sum(77) + res[78] = fma52hi(res[78], a[15], a[62]); // Sum(77) + res[78] = fma52lo(res[78], a[16], a[62]); // Sum(78) + res[79] = fma52hi(res[79], a[16], a[62]); // Sum(78) + res[79] = fma52lo(res[79], a[17], a[62]); // Sum(79) + res[80] = fma52hi(res[80], a[17], a[62]); // Sum(79) + res[80] = fma52lo(res[80], a[18], a[62]); // Sum(80) + res[81] = fma52hi(res[81], a[18], a[62]); // Sum(80) + res[81] = fma52lo(res[81], a[19], a[62]); // Sum(81) + res[82] = fma52hi(res[82], a[19], a[62]); // Sum(81) + res[82] = fma52lo(res[82], a[20], a[62]); // Sum(82) + res[83] = fma52hi(res[83], a[20], a[62]); // Sum(82) + res[83] = fma52lo(res[83], a[21], a[62]); // Sum(83) + res[84] = fma52hi(res[84], a[21], a[62]); // Sum(83) + res[72] = fma52lo(res[72], a[9], a[63]); // Sum(72) + res[73] = fma52hi(res[73], a[9], a[63]); // Sum(72) + res[73] = fma52lo(res[73], a[10], a[63]); // Sum(73) + res[74] = fma52hi(res[74], a[10], a[63]); // Sum(73) + res[74] = fma52lo(res[74], a[11], a[63]); // Sum(74) + res[75] = fma52hi(res[75], a[11], a[63]); // Sum(74) + res[75] = fma52lo(res[75], a[12], a[63]); // Sum(75) + res[76] = fma52hi(res[76], a[12], a[63]); // Sum(75) + res[76] = fma52lo(res[76], a[13], a[63]); // Sum(76) + res[77] = fma52hi(res[77], a[13], a[63]); // Sum(76) + res[77] = fma52lo(res[77], a[14], a[63]); // Sum(77) + res[78] = fma52hi(res[78], a[14], a[63]); // Sum(77) + res[78] = fma52lo(res[78], a[15], a[63]); // Sum(78) + res[79] = fma52hi(res[79], a[15], a[63]); // Sum(78) + res[79] = fma52lo(res[79], a[16], a[63]); // Sum(79) + res[80] = fma52hi(res[80], a[16], a[63]); // Sum(79) + res[80] = fma52lo(res[80], a[17], a[63]); // Sum(80) + res[81] = fma52hi(res[81], a[17], a[63]); // Sum(80) + res[81] = fma52lo(res[81], a[18], a[63]); // Sum(81) + res[82] = fma52hi(res[82], a[18], a[63]); // Sum(81) + res[82] = fma52lo(res[82], a[19], a[63]); // Sum(82) + res[83] = fma52hi(res[83], a[19], a[63]); // Sum(82) + res[83] = fma52lo(res[83], a[20], a[63]); // Sum(83) + res[84] = fma52hi(res[84], a[20], a[63]); // Sum(83) + res[72] = fma52lo(res[72], a[8], a[64]); // Sum(72) + res[73] = fma52hi(res[73], a[8], a[64]); // Sum(72) + res[73] = fma52lo(res[73], a[9], a[64]); // Sum(73) + res[74] = fma52hi(res[74], a[9], a[64]); // Sum(73) + res[74] = fma52lo(res[74], a[10], a[64]); // Sum(74) + res[75] = fma52hi(res[75], a[10], a[64]); // Sum(74) + res[75] = fma52lo(res[75], a[11], a[64]); // Sum(75) + res[76] = fma52hi(res[76], a[11], a[64]); // Sum(75) + res[76] = fma52lo(res[76], a[12], a[64]); // Sum(76) + res[77] = fma52hi(res[77], a[12], a[64]); // Sum(76) + res[77] = fma52lo(res[77], a[13], a[64]); // Sum(77) + res[78] = fma52hi(res[78], a[13], a[64]); // Sum(77) + res[78] = fma52lo(res[78], a[14], a[64]); // Sum(78) + res[79] = fma52hi(res[79], a[14], a[64]); // Sum(78) + res[79] = fma52lo(res[79], a[15], a[64]); // Sum(79) + res[80] = fma52hi(res[80], a[15], a[64]); // Sum(79) + res[80] = fma52lo(res[80], a[16], a[64]); // Sum(80) + res[81] = fma52hi(res[81], a[16], a[64]); // Sum(80) + res[81] = fma52lo(res[81], a[17], a[64]); // Sum(81) + res[82] = fma52hi(res[82], a[17], a[64]); // Sum(81) + res[82] = fma52lo(res[82], a[18], a[64]); // Sum(82) + res[83] = fma52hi(res[83], a[18], a[64]); // Sum(82) + res[83] = fma52lo(res[83], a[19], a[64]); // Sum(83) + res[84] = fma52hi(res[84], a[19], a[64]); // Sum(83) + res[72] = fma52lo(res[72], a[7], a[65]); // Sum(72) + res[73] = fma52hi(res[73], a[7], a[65]); // Sum(72) + res[73] = fma52lo(res[73], a[8], a[65]); // Sum(73) + res[74] = fma52hi(res[74], a[8], a[65]); // Sum(73) + res[74] = fma52lo(res[74], a[9], a[65]); // Sum(74) + res[75] = fma52hi(res[75], a[9], a[65]); // Sum(74) + res[75] = fma52lo(res[75], a[10], a[65]); // Sum(75) + res[76] = fma52hi(res[76], a[10], a[65]); // Sum(75) + res[76] = fma52lo(res[76], a[11], a[65]); // Sum(76) + res[77] = fma52hi(res[77], a[11], a[65]); // Sum(76) + res[77] = fma52lo(res[77], a[12], a[65]); // Sum(77) + res[78] = fma52hi(res[78], a[12], a[65]); // Sum(77) + res[78] = fma52lo(res[78], a[13], a[65]); // Sum(78) + res[79] = fma52hi(res[79], a[13], a[65]); // Sum(78) + res[79] = fma52lo(res[79], a[14], a[65]); // Sum(79) + res[80] = fma52hi(res[80], a[14], a[65]); // Sum(79) + res[80] = fma52lo(res[80], a[15], a[65]); // Sum(80) + res[81] = fma52hi(res[81], a[15], a[65]); // Sum(80) + res[81] = fma52lo(res[81], a[16], a[65]); // Sum(81) + res[82] = fma52hi(res[82], a[16], a[65]); // Sum(81) + res[82] = fma52lo(res[82], a[17], a[65]); // Sum(82) + res[83] = fma52hi(res[83], a[17], a[65]); // Sum(82) + res[83] = fma52lo(res[83], a[18], a[65]); // Sum(83) + res[84] = fma52hi(res[84], a[18], a[65]); // Sum(83) + res[72] = fma52lo(res[72], a[6], a[66]); // Sum(72) + res[73] = fma52hi(res[73], a[6], a[66]); // Sum(72) + res[73] = fma52lo(res[73], a[7], a[66]); // Sum(73) + res[74] = fma52hi(res[74], a[7], a[66]); // Sum(73) + res[74] = fma52lo(res[74], a[8], a[66]); // Sum(74) + res[75] = fma52hi(res[75], a[8], a[66]); // Sum(74) + res[75] = fma52lo(res[75], a[9], a[66]); // Sum(75) + res[76] = fma52hi(res[76], a[9], a[66]); // Sum(75) + res[76] = fma52lo(res[76], a[10], a[66]); // Sum(76) + res[77] = fma52hi(res[77], a[10], a[66]); // Sum(76) + res[77] = fma52lo(res[77], a[11], a[66]); // Sum(77) + res[78] = fma52hi(res[78], a[11], a[66]); // Sum(77) + res[78] = fma52lo(res[78], a[12], a[66]); // Sum(78) + res[79] = fma52hi(res[79], a[12], a[66]); // Sum(78) + res[79] = fma52lo(res[79], a[13], a[66]); // Sum(79) + res[80] = fma52hi(res[80], a[13], a[66]); // Sum(79) + res[80] = fma52lo(res[80], a[14], a[66]); // Sum(80) + res[81] = fma52hi(res[81], a[14], a[66]); // Sum(80) + res[81] = fma52lo(res[81], a[15], a[66]); // Sum(81) + res[82] = fma52hi(res[82], a[15], a[66]); // Sum(81) + res[82] = fma52lo(res[82], a[16], a[66]); // Sum(82) + res[83] = fma52hi(res[83], a[16], a[66]); // Sum(82) + res[83] = fma52lo(res[83], a[17], a[66]); // Sum(83) + res[84] = fma52hi(res[84], a[17], a[66]); // Sum(83) + res[72] = fma52lo(res[72], a[5], a[67]); // Sum(72) + res[73] = fma52hi(res[73], a[5], a[67]); // Sum(72) + res[73] = fma52lo(res[73], a[6], a[67]); // Sum(73) + res[74] = fma52hi(res[74], a[6], a[67]); // Sum(73) + res[74] = fma52lo(res[74], a[7], a[67]); // Sum(74) + res[75] = fma52hi(res[75], a[7], a[67]); // Sum(74) + res[75] = fma52lo(res[75], a[8], a[67]); // Sum(75) + res[76] = fma52hi(res[76], a[8], a[67]); // Sum(75) + res[76] = fma52lo(res[76], a[9], a[67]); // Sum(76) + res[77] = fma52hi(res[77], a[9], a[67]); // Sum(76) + res[77] = fma52lo(res[77], a[10], a[67]); // Sum(77) + res[78] = fma52hi(res[78], a[10], a[67]); // Sum(77) + res[78] = fma52lo(res[78], a[11], a[67]); // Sum(78) + res[79] = fma52hi(res[79], a[11], a[67]); // Sum(78) + res[79] = fma52lo(res[79], a[12], a[67]); // Sum(79) + res[80] = fma52hi(res[80], a[12], a[67]); // Sum(79) + res[80] = fma52lo(res[80], a[13], a[67]); // Sum(80) + res[81] = fma52hi(res[81], a[13], a[67]); // Sum(80) + res[81] = fma52lo(res[81], a[14], a[67]); // Sum(81) + res[82] = fma52hi(res[82], a[14], a[67]); // Sum(81) + res[82] = fma52lo(res[82], a[15], a[67]); // Sum(82) + res[83] = fma52hi(res[83], a[15], a[67]); // Sum(82) + res[83] = fma52lo(res[83], a[16], a[67]); // Sum(83) + res[84] = fma52hi(res[84], a[16], a[67]); // Sum(83) + res[72] = fma52lo(res[72], a[4], a[68]); // Sum(72) + res[73] = fma52hi(res[73], a[4], a[68]); // Sum(72) + res[73] = fma52lo(res[73], a[5], a[68]); // Sum(73) + res[74] = fma52hi(res[74], a[5], a[68]); // Sum(73) + res[74] = fma52lo(res[74], a[6], a[68]); // Sum(74) + res[75] = fma52hi(res[75], a[6], a[68]); // Sum(74) + res[75] = fma52lo(res[75], a[7], a[68]); // Sum(75) + res[76] = fma52hi(res[76], a[7], a[68]); // Sum(75) + res[76] = fma52lo(res[76], a[8], a[68]); // Sum(76) + res[77] = fma52hi(res[77], a[8], a[68]); // Sum(76) + res[77] = fma52lo(res[77], a[9], a[68]); // Sum(77) + res[78] = fma52hi(res[78], a[9], a[68]); // Sum(77) + res[78] = fma52lo(res[78], a[10], a[68]); // Sum(78) + res[79] = fma52hi(res[79], a[10], a[68]); // Sum(78) + res[79] = fma52lo(res[79], a[11], a[68]); // Sum(79) + res[80] = fma52hi(res[80], a[11], a[68]); // Sum(79) + res[80] = fma52lo(res[80], a[12], a[68]); // Sum(80) + res[81] = fma52hi(res[81], a[12], a[68]); // Sum(80) + res[81] = fma52lo(res[81], a[13], a[68]); // Sum(81) + res[82] = fma52hi(res[82], a[13], a[68]); // Sum(81) + res[82] = fma52lo(res[82], a[14], a[68]); // Sum(82) + res[83] = fma52hi(res[83], a[14], a[68]); // Sum(82) + res[83] = fma52lo(res[83], a[15], a[68]); // Sum(83) + res[84] = fma52hi(res[84], a[15], a[68]); // Sum(83) + res[72] = fma52lo(res[72], a[3], a[69]); // Sum(72) + res[73] = fma52hi(res[73], a[3], a[69]); // Sum(72) + res[73] = fma52lo(res[73], a[4], a[69]); // Sum(73) + res[74] = fma52hi(res[74], a[4], a[69]); // Sum(73) + res[74] = fma52lo(res[74], a[5], a[69]); // Sum(74) + res[75] = fma52hi(res[75], a[5], a[69]); // Sum(74) + res[75] = fma52lo(res[75], a[6], a[69]); // Sum(75) + res[76] = fma52hi(res[76], a[6], a[69]); // Sum(75) + res[76] = fma52lo(res[76], a[7], a[69]); // Sum(76) + res[77] = fma52hi(res[77], a[7], a[69]); // Sum(76) + res[77] = fma52lo(res[77], a[8], a[69]); // Sum(77) + res[78] = fma52hi(res[78], a[8], a[69]); // Sum(77) + res[78] = fma52lo(res[78], a[9], a[69]); // Sum(78) + res[79] = fma52hi(res[79], a[9], a[69]); // Sum(78) + res[79] = fma52lo(res[79], a[10], a[69]); // Sum(79) + res[80] = fma52hi(res[80], a[10], a[69]); // Sum(79) + res[80] = fma52lo(res[80], a[11], a[69]); // Sum(80) + res[81] = fma52hi(res[81], a[11], a[69]); // Sum(80) + res[81] = fma52lo(res[81], a[12], a[69]); // Sum(81) + res[82] = fma52hi(res[82], a[12], a[69]); // Sum(81) + res[82] = fma52lo(res[82], a[13], a[69]); // Sum(82) + res[83] = fma52hi(res[83], a[13], a[69]); // Sum(82) + res[83] = fma52lo(res[83], a[14], a[69]); // Sum(83) + res[84] = fma52hi(res[84], a[14], a[69]); // Sum(83) + res[72] = fma52lo(res[72], a[2], a[70]); // Sum(72) + res[73] = fma52hi(res[73], a[2], a[70]); // Sum(72) + res[73] = fma52lo(res[73], a[3], a[70]); // Sum(73) + res[74] = fma52hi(res[74], a[3], a[70]); // Sum(73) + res[74] = fma52lo(res[74], a[4], a[70]); // Sum(74) + res[75] = fma52hi(res[75], a[4], a[70]); // Sum(74) + res[75] = fma52lo(res[75], a[5], a[70]); // Sum(75) + res[76] = fma52hi(res[76], a[5], a[70]); // Sum(75) + res[76] = fma52lo(res[76], a[6], a[70]); // Sum(76) + res[77] = fma52hi(res[77], a[6], a[70]); // Sum(76) + res[77] = fma52lo(res[77], a[7], a[70]); // Sum(77) + res[78] = fma52hi(res[78], a[7], a[70]); // Sum(77) + res[78] = fma52lo(res[78], a[8], a[70]); // Sum(78) + res[79] = fma52hi(res[79], a[8], a[70]); // Sum(78) + res[79] = fma52lo(res[79], a[9], a[70]); // Sum(79) + res[80] = fma52hi(res[80], a[9], a[70]); // Sum(79) + res[80] = fma52lo(res[80], a[10], a[70]); // Sum(80) + res[81] = fma52hi(res[81], a[10], a[70]); // Sum(80) + res[81] = fma52lo(res[81], a[11], a[70]); // Sum(81) + res[82] = fma52hi(res[82], a[11], a[70]); // Sum(81) + res[82] = fma52lo(res[82], a[12], a[70]); // Sum(82) + res[83] = fma52hi(res[83], a[12], a[70]); // Sum(82) + res[83] = fma52lo(res[83], a[13], a[70]); // Sum(83) + res[84] = fma52hi(res[84], a[13], a[70]); // Sum(83) + res[72] = fma52lo(res[72], a[1], a[71]); // Sum(72) + res[73] = fma52hi(res[73], a[1], a[71]); // Sum(72) + res[73] = fma52lo(res[73], a[2], a[71]); // Sum(73) + res[74] = fma52hi(res[74], a[2], a[71]); // Sum(73) + res[74] = fma52lo(res[74], a[3], a[71]); // Sum(74) + res[75] = fma52hi(res[75], a[3], a[71]); // Sum(74) + res[75] = fma52lo(res[75], a[4], a[71]); // Sum(75) + res[76] = fma52hi(res[76], a[4], a[71]); // Sum(75) + res[76] = fma52lo(res[76], a[5], a[71]); // Sum(76) + res[77] = fma52hi(res[77], a[5], a[71]); // Sum(76) + res[77] = fma52lo(res[77], a[6], a[71]); // Sum(77) + res[78] = fma52hi(res[78], a[6], a[71]); // Sum(77) + res[78] = fma52lo(res[78], a[7], a[71]); // Sum(78) + res[79] = fma52hi(res[79], a[7], a[71]); // Sum(78) + res[79] = fma52lo(res[79], a[8], a[71]); // Sum(79) + res[80] = fma52hi(res[80], a[8], a[71]); // Sum(79) + res[80] = fma52lo(res[80], a[9], a[71]); // Sum(80) + res[81] = fma52hi(res[81], a[9], a[71]); // Sum(80) + res[81] = fma52lo(res[81], a[10], a[71]); // Sum(81) + res[82] = fma52hi(res[82], a[10], a[71]); // Sum(81) + res[82] = fma52lo(res[82], a[11], a[71]); // Sum(82) + res[83] = fma52hi(res[83], a[11], a[71]); // Sum(82) + res[83] = fma52lo(res[83], a[12], a[71]); // Sum(83) + res[84] = fma52hi(res[84], a[12], a[71]); // Sum(83) + res[72] = fma52lo(res[72], a[0], a[72]); // Sum(72) + res[73] = fma52hi(res[73], a[0], a[72]); // Sum(72) + res[73] = fma52lo(res[73], a[1], a[72]); // Sum(73) + res[74] = fma52hi(res[74], a[1], a[72]); // Sum(73) + res[74] = fma52lo(res[74], a[2], a[72]); // Sum(74) + res[75] = fma52hi(res[75], a[2], a[72]); // Sum(74) + res[75] = fma52lo(res[75], a[3], a[72]); // Sum(75) + res[76] = fma52hi(res[76], a[3], a[72]); // Sum(75) + res[76] = fma52lo(res[76], a[4], a[72]); // Sum(76) + res[77] = fma52hi(res[77], a[4], a[72]); // Sum(76) + res[77] = fma52lo(res[77], a[5], a[72]); // Sum(77) + res[78] = fma52hi(res[78], a[5], a[72]); // Sum(77) + res[78] = fma52lo(res[78], a[6], a[72]); // Sum(78) + res[79] = fma52hi(res[79], a[6], a[72]); // Sum(78) + res[79] = fma52lo(res[79], a[7], a[72]); // Sum(79) + res[80] = fma52hi(res[80], a[7], a[72]); // Sum(79) + res[80] = fma52lo(res[80], a[8], a[72]); // Sum(80) + res[81] = fma52hi(res[81], a[8], a[72]); // Sum(80) + res[81] = fma52lo(res[81], a[9], a[72]); // Sum(81) + res[82] = fma52hi(res[82], a[9], a[72]); // Sum(81) + res[82] = fma52lo(res[82], a[10], a[72]); // Sum(82) + res[83] = fma52hi(res[83], a[10], a[72]); // Sum(82) + res[83] = fma52lo(res[83], a[11], a[72]); // Sum(83) + res[84] = fma52hi(res[84], a[11], a[72]); // Sum(83) + res[73] = fma52lo(res[73], a[0], a[73]); // Sum(73) + res[74] = fma52hi(res[74], a[0], a[73]); // Sum(73) + res[74] = fma52lo(res[74], a[1], a[73]); // Sum(74) + res[75] = fma52hi(res[75], a[1], a[73]); // Sum(74) + res[75] = fma52lo(res[75], a[2], a[73]); // Sum(75) + res[76] = fma52hi(res[76], a[2], a[73]); // Sum(75) + res[76] = fma52lo(res[76], a[3], a[73]); // Sum(76) + res[77] = fma52hi(res[77], a[3], a[73]); // Sum(76) + res[77] = fma52lo(res[77], a[4], a[73]); // Sum(77) + res[78] = fma52hi(res[78], a[4], a[73]); // Sum(77) + res[78] = fma52lo(res[78], a[5], a[73]); // Sum(78) + res[79] = fma52hi(res[79], a[5], a[73]); // Sum(78) + res[79] = fma52lo(res[79], a[6], a[73]); // Sum(79) + res[80] = fma52hi(res[80], a[6], a[73]); // Sum(79) + res[80] = fma52lo(res[80], a[7], a[73]); // Sum(80) + res[81] = fma52hi(res[81], a[7], a[73]); // Sum(80) + res[81] = fma52lo(res[81], a[8], a[73]); // Sum(81) + res[82] = fma52hi(res[82], a[8], a[73]); // Sum(81) + res[82] = fma52lo(res[82], a[9], a[73]); // Sum(82) + res[83] = fma52hi(res[83], a[9], a[73]); // Sum(82) + res[83] = fma52lo(res[83], a[10], a[73]); // Sum(83) + res[84] = fma52hi(res[84], a[10], a[73]); // Sum(83) + res[74] = fma52lo(res[74], a[0], a[74]); // Sum(74) + res[75] = fma52hi(res[75], a[0], a[74]); // Sum(74) + res[75] = fma52lo(res[75], a[1], a[74]); // Sum(75) + res[76] = fma52hi(res[76], a[1], a[74]); // Sum(75) + res[76] = fma52lo(res[76], a[2], a[74]); // Sum(76) + res[77] = fma52hi(res[77], a[2], a[74]); // Sum(76) + res[77] = fma52lo(res[77], a[3], a[74]); // Sum(77) + res[78] = fma52hi(res[78], a[3], a[74]); // Sum(77) + res[78] = fma52lo(res[78], a[4], a[74]); // Sum(78) + res[79] = fma52hi(res[79], a[4], a[74]); // Sum(78) + res[79] = fma52lo(res[79], a[5], a[74]); // Sum(79) + res[80] = fma52hi(res[80], a[5], a[74]); // Sum(79) + res[80] = fma52lo(res[80], a[6], a[74]); // Sum(80) + res[81] = fma52hi(res[81], a[6], a[74]); // Sum(80) + res[81] = fma52lo(res[81], a[7], a[74]); // Sum(81) + res[82] = fma52hi(res[82], a[7], a[74]); // Sum(81) + res[82] = fma52lo(res[82], a[8], a[74]); // Sum(82) + res[83] = fma52hi(res[83], a[8], a[74]); // Sum(82) + res[83] = fma52lo(res[83], a[9], a[74]); // Sum(83) + res[84] = fma52hi(res[84], a[9], a[74]); // Sum(83) + res[75] = fma52lo(res[75], a[0], a[75]); // Sum(75) + res[76] = fma52hi(res[76], a[0], a[75]); // Sum(75) + res[76] = fma52lo(res[76], a[1], a[75]); // Sum(76) + res[77] = fma52hi(res[77], a[1], a[75]); // Sum(76) + res[77] = fma52lo(res[77], a[2], a[75]); // Sum(77) + res[78] = fma52hi(res[78], a[2], a[75]); // Sum(77) + res[78] = fma52lo(res[78], a[3], a[75]); // Sum(78) + res[79] = fma52hi(res[79], a[3], a[75]); // Sum(78) + res[79] = fma52lo(res[79], a[4], a[75]); // Sum(79) + res[80] = fma52hi(res[80], a[4], a[75]); // Sum(79) + res[80] = fma52lo(res[80], a[5], a[75]); // Sum(80) + res[81] = fma52hi(res[81], a[5], a[75]); // Sum(80) + res[81] = fma52lo(res[81], a[6], a[75]); // Sum(81) + res[82] = fma52hi(res[82], a[6], a[75]); // Sum(81) + res[82] = fma52lo(res[82], a[7], a[75]); // Sum(82) + res[83] = fma52hi(res[83], a[7], a[75]); // Sum(82) + res[83] = fma52lo(res[83], a[8], a[75]); // Sum(83) + res[84] = fma52hi(res[84], a[8], a[75]); // Sum(83) + res[76] = fma52lo(res[76], a[0], a[76]); // Sum(76) + res[77] = fma52hi(res[77], a[0], a[76]); // Sum(76) + res[77] = fma52lo(res[77], a[1], a[76]); // Sum(77) + res[78] = fma52hi(res[78], a[1], a[76]); // Sum(77) + res[78] = fma52lo(res[78], a[2], a[76]); // Sum(78) + res[79] = fma52hi(res[79], a[2], a[76]); // Sum(78) + res[79] = fma52lo(res[79], a[3], a[76]); // Sum(79) + res[80] = fma52hi(res[80], a[3], a[76]); // Sum(79) + res[80] = fma52lo(res[80], a[4], a[76]); // Sum(80) + res[81] = fma52hi(res[81], a[4], a[76]); // Sum(80) + res[81] = fma52lo(res[81], a[5], a[76]); // Sum(81) + res[82] = fma52hi(res[82], a[5], a[76]); // Sum(81) + res[82] = fma52lo(res[82], a[6], a[76]); // Sum(82) + res[83] = fma52hi(res[83], a[6], a[76]); // Sum(82) + res[83] = fma52lo(res[83], a[7], a[76]); // Sum(83) + res[84] = fma52hi(res[84], a[7], a[76]); // Sum(83) + res[77] = fma52lo(res[77], a[0], a[77]); // Sum(77) + res[78] = fma52hi(res[78], a[0], a[77]); // Sum(77) + res[78] = fma52lo(res[78], a[1], a[77]); // Sum(78) + res[79] = fma52hi(res[79], a[1], a[77]); // Sum(78) + res[79] = fma52lo(res[79], a[2], a[77]); // Sum(79) + res[80] = fma52hi(res[80], a[2], a[77]); // Sum(79) + res[80] = fma52lo(res[80], a[3], a[77]); // Sum(80) + res[81] = fma52hi(res[81], a[3], a[77]); // Sum(80) + res[81] = fma52lo(res[81], a[4], a[77]); // Sum(81) + res[82] = fma52hi(res[82], a[4], a[77]); // Sum(81) + res[82] = fma52lo(res[82], a[5], a[77]); // Sum(82) + res[83] = fma52hi(res[83], a[5], a[77]); // Sum(82) + res[83] = fma52lo(res[83], a[6], a[77]); // Sum(83) + res[84] = fma52hi(res[84], a[6], a[77]); // Sum(83) + res[78] = fma52lo(res[78], a[0], a[78]); // Sum(78) + res[79] = fma52hi(res[79], a[0], a[78]); // Sum(78) + res[79] = fma52lo(res[79], a[1], a[78]); // Sum(79) + res[80] = fma52hi(res[80], a[1], a[78]); // Sum(79) + res[80] = fma52lo(res[80], a[2], a[78]); // Sum(80) + res[81] = fma52hi(res[81], a[2], a[78]); // Sum(80) + res[81] = fma52lo(res[81], a[3], a[78]); // Sum(81) + res[82] = fma52hi(res[82], a[3], a[78]); // Sum(81) + res[82] = fma52lo(res[82], a[4], a[78]); // Sum(82) + res[83] = fma52hi(res[83], a[4], a[78]); // Sum(82) + res[83] = fma52lo(res[83], a[5], a[78]); // Sum(83) + res[84] = fma52hi(res[84], a[5], a[78]); // Sum(83) + res[72] = add64(res[72], res[72]); // Double(72) + res[73] = add64(res[73], res[73]); // Double(73) + res[74] = add64(res[74], res[74]); // Double(74) + res[75] = add64(res[75], res[75]); // Double(75) + res[76] = add64(res[76], res[76]); // Double(76) + res[77] = add64(res[77], res[77]); // Double(77) + res[78] = add64(res[78], res[78]); // Double(78) + res[79] = add64(res[79], res[79]); // Double(79) + res[80] = add64(res[80], res[80]); // Double(80) + res[81] = add64(res[81], res[81]); // Double(81) + res[82] = add64(res[82], res[82]); // Double(82) + res[83] = add64(res[83], res[83]); // Double(83) + res[72] = fma52lo(res[72], a[36], a[36]); // Add sqr(72) + res[73] = fma52hi(res[73], a[36], a[36]); // Add sqr(72) + res[74] = fma52lo(res[74], a[37], a[37]); // Add sqr(74) + res[75] = fma52hi(res[75], a[37], a[37]); // Add sqr(74) + res[76] = fma52lo(res[76], a[38], a[38]); // Add sqr(76) + res[77] = fma52hi(res[77], a[38], a[38]); // Add sqr(76) + res[78] = fma52lo(res[78], a[39], a[39]); // Add sqr(78) + res[79] = fma52hi(res[79], a[39], a[39]); // Add sqr(78) + res[80] = fma52lo(res[80], a[40], a[40]); // Add sqr(80) + res[81] = fma52hi(res[81], a[40], a[40]); // Add sqr(80) + res[82] = fma52lo(res[82], a[41], a[41]); // Add sqr(82) + res[83] = fma52hi(res[83], a[41], a[41]); // Add sqr(82) + res[84] = fma52lo(res[84], a[41], a[43]); // Sum(84) + res[85] = fma52hi(res[85], a[41], a[43]); // Sum(84) + res[85] = fma52lo(res[85], a[42], a[43]); // Sum(85) + res[86] = fma52hi(res[86], a[42], a[43]); // Sum(85) + res[84] = fma52lo(res[84], a[40], a[44]); // Sum(84) + res[85] = fma52hi(res[85], a[40], a[44]); // Sum(84) + res[85] = fma52lo(res[85], a[41], a[44]); // Sum(85) + res[86] = fma52hi(res[86], a[41], a[44]); // Sum(85) + res[86] = fma52lo(res[86], a[42], a[44]); // Sum(86) + res[87] = fma52hi(res[87], a[42], a[44]); // Sum(86) + res[87] = fma52lo(res[87], a[43], a[44]); // Sum(87) + res[88] = fma52hi(res[88], a[43], a[44]); // Sum(87) + res[84] = fma52lo(res[84], a[39], a[45]); // Sum(84) + res[85] = fma52hi(res[85], a[39], a[45]); // Sum(84) + res[85] = fma52lo(res[85], a[40], a[45]); // Sum(85) + res[86] = fma52hi(res[86], a[40], a[45]); // Sum(85) + res[86] = fma52lo(res[86], a[41], a[45]); // Sum(86) + res[87] = fma52hi(res[87], a[41], a[45]); // Sum(86) + res[87] = fma52lo(res[87], a[42], a[45]); // Sum(87) + res[88] = fma52hi(res[88], a[42], a[45]); // Sum(87) + res[88] = fma52lo(res[88], a[43], a[45]); // Sum(88) + res[89] = fma52hi(res[89], a[43], a[45]); // Sum(88) + res[89] = fma52lo(res[89], a[44], a[45]); // Sum(89) + res[90] = fma52hi(res[90], a[44], a[45]); // Sum(89) + res[84] = fma52lo(res[84], a[38], a[46]); // Sum(84) + res[85] = fma52hi(res[85], a[38], a[46]); // Sum(84) + res[85] = fma52lo(res[85], a[39], a[46]); // Sum(85) + res[86] = fma52hi(res[86], a[39], a[46]); // Sum(85) + res[86] = fma52lo(res[86], a[40], a[46]); // Sum(86) + res[87] = fma52hi(res[87], a[40], a[46]); // Sum(86) + res[87] = fma52lo(res[87], a[41], a[46]); // Sum(87) + res[88] = fma52hi(res[88], a[41], a[46]); // Sum(87) + res[88] = fma52lo(res[88], a[42], a[46]); // Sum(88) + res[89] = fma52hi(res[89], a[42], a[46]); // Sum(88) + res[89] = fma52lo(res[89], a[43], a[46]); // Sum(89) + res[90] = fma52hi(res[90], a[43], a[46]); // Sum(89) + res[90] = fma52lo(res[90], a[44], a[46]); // Sum(90) + res[91] = fma52hi(res[91], a[44], a[46]); // Sum(90) + res[91] = fma52lo(res[91], a[45], a[46]); // Sum(91) + res[92] = fma52hi(res[92], a[45], a[46]); // Sum(91) + res[84] = fma52lo(res[84], a[37], a[47]); // Sum(84) + res[85] = fma52hi(res[85], a[37], a[47]); // Sum(84) + res[85] = fma52lo(res[85], a[38], a[47]); // Sum(85) + res[86] = fma52hi(res[86], a[38], a[47]); // Sum(85) + res[86] = fma52lo(res[86], a[39], a[47]); // Sum(86) + res[87] = fma52hi(res[87], a[39], a[47]); // Sum(86) + res[87] = fma52lo(res[87], a[40], a[47]); // Sum(87) + res[88] = fma52hi(res[88], a[40], a[47]); // Sum(87) + res[88] = fma52lo(res[88], a[41], a[47]); // Sum(88) + res[89] = fma52hi(res[89], a[41], a[47]); // Sum(88) + res[89] = fma52lo(res[89], a[42], a[47]); // Sum(89) + res[90] = fma52hi(res[90], a[42], a[47]); // Sum(89) + res[90] = fma52lo(res[90], a[43], a[47]); // Sum(90) + res[91] = fma52hi(res[91], a[43], a[47]); // Sum(90) + res[91] = fma52lo(res[91], a[44], a[47]); // Sum(91) + res[92] = fma52hi(res[92], a[44], a[47]); // Sum(91) + res[92] = fma52lo(res[92], a[45], a[47]); // Sum(92) + res[93] = fma52hi(res[93], a[45], a[47]); // Sum(92) + res[93] = fma52lo(res[93], a[46], a[47]); // Sum(93) + res[94] = fma52hi(res[94], a[46], a[47]); // Sum(93) + res[84] = fma52lo(res[84], a[36], a[48]); // Sum(84) + res[85] = fma52hi(res[85], a[36], a[48]); // Sum(84) + res[85] = fma52lo(res[85], a[37], a[48]); // Sum(85) + res[86] = fma52hi(res[86], a[37], a[48]); // Sum(85) + res[86] = fma52lo(res[86], a[38], a[48]); // Sum(86) + res[87] = fma52hi(res[87], a[38], a[48]); // Sum(86) + res[87] = fma52lo(res[87], a[39], a[48]); // Sum(87) + res[88] = fma52hi(res[88], a[39], a[48]); // Sum(87) + res[88] = fma52lo(res[88], a[40], a[48]); // Sum(88) + res[89] = fma52hi(res[89], a[40], a[48]); // Sum(88) + res[89] = fma52lo(res[89], a[41], a[48]); // Sum(89) + res[90] = fma52hi(res[90], a[41], a[48]); // Sum(89) + res[90] = fma52lo(res[90], a[42], a[48]); // Sum(90) + res[91] = fma52hi(res[91], a[42], a[48]); // Sum(90) + res[91] = fma52lo(res[91], a[43], a[48]); // Sum(91) + res[92] = fma52hi(res[92], a[43], a[48]); // Sum(91) + res[92] = fma52lo(res[92], a[44], a[48]); // Sum(92) + res[93] = fma52hi(res[93], a[44], a[48]); // Sum(92) + res[93] = fma52lo(res[93], a[45], a[48]); // Sum(93) + res[94] = fma52hi(res[94], a[45], a[48]); // Sum(93) + res[94] = fma52lo(res[94], a[46], a[48]); // Sum(94) + res[95] = fma52hi(res[95], a[46], a[48]); // Sum(94) + res[95] = fma52lo(res[95], a[47], a[48]); // Sum(95) + res[96] = fma52hi(res[96], a[47], a[48]); // Sum(95) + res[84] = fma52lo(res[84], a[35], a[49]); // Sum(84) + res[85] = fma52hi(res[85], a[35], a[49]); // Sum(84) + res[85] = fma52lo(res[85], a[36], a[49]); // Sum(85) + res[86] = fma52hi(res[86], a[36], a[49]); // Sum(85) + res[86] = fma52lo(res[86], a[37], a[49]); // Sum(86) + res[87] = fma52hi(res[87], a[37], a[49]); // Sum(86) + res[87] = fma52lo(res[87], a[38], a[49]); // Sum(87) + res[88] = fma52hi(res[88], a[38], a[49]); // Sum(87) + res[88] = fma52lo(res[88], a[39], a[49]); // Sum(88) + res[89] = fma52hi(res[89], a[39], a[49]); // Sum(88) + res[89] = fma52lo(res[89], a[40], a[49]); // Sum(89) + res[90] = fma52hi(res[90], a[40], a[49]); // Sum(89) + res[90] = fma52lo(res[90], a[41], a[49]); // Sum(90) + res[91] = fma52hi(res[91], a[41], a[49]); // Sum(90) + res[91] = fma52lo(res[91], a[42], a[49]); // Sum(91) + res[92] = fma52hi(res[92], a[42], a[49]); // Sum(91) + res[92] = fma52lo(res[92], a[43], a[49]); // Sum(92) + res[93] = fma52hi(res[93], a[43], a[49]); // Sum(92) + res[93] = fma52lo(res[93], a[44], a[49]); // Sum(93) + res[94] = fma52hi(res[94], a[44], a[49]); // Sum(93) + res[94] = fma52lo(res[94], a[45], a[49]); // Sum(94) + res[95] = fma52hi(res[95], a[45], a[49]); // Sum(94) + res[95] = fma52lo(res[95], a[46], a[49]); // Sum(95) + res[96] = fma52hi(res[96], a[46], a[49]); // Sum(95) + res[84] = fma52lo(res[84], a[34], a[50]); // Sum(84) + res[85] = fma52hi(res[85], a[34], a[50]); // Sum(84) + res[85] = fma52lo(res[85], a[35], a[50]); // Sum(85) + res[86] = fma52hi(res[86], a[35], a[50]); // Sum(85) + res[86] = fma52lo(res[86], a[36], a[50]); // Sum(86) + res[87] = fma52hi(res[87], a[36], a[50]); // Sum(86) + res[87] = fma52lo(res[87], a[37], a[50]); // Sum(87) + res[88] = fma52hi(res[88], a[37], a[50]); // Sum(87) + res[88] = fma52lo(res[88], a[38], a[50]); // Sum(88) + res[89] = fma52hi(res[89], a[38], a[50]); // Sum(88) + res[89] = fma52lo(res[89], a[39], a[50]); // Sum(89) + res[90] = fma52hi(res[90], a[39], a[50]); // Sum(89) + res[90] = fma52lo(res[90], a[40], a[50]); // Sum(90) + res[91] = fma52hi(res[91], a[40], a[50]); // Sum(90) + res[91] = fma52lo(res[91], a[41], a[50]); // Sum(91) + res[92] = fma52hi(res[92], a[41], a[50]); // Sum(91) + res[92] = fma52lo(res[92], a[42], a[50]); // Sum(92) + res[93] = fma52hi(res[93], a[42], a[50]); // Sum(92) + res[93] = fma52lo(res[93], a[43], a[50]); // Sum(93) + res[94] = fma52hi(res[94], a[43], a[50]); // Sum(93) + res[94] = fma52lo(res[94], a[44], a[50]); // Sum(94) + res[95] = fma52hi(res[95], a[44], a[50]); // Sum(94) + res[95] = fma52lo(res[95], a[45], a[50]); // Sum(95) + res[96] = fma52hi(res[96], a[45], a[50]); // Sum(95) + res[84] = fma52lo(res[84], a[33], a[51]); // Sum(84) + res[85] = fma52hi(res[85], a[33], a[51]); // Sum(84) + res[85] = fma52lo(res[85], a[34], a[51]); // Sum(85) + res[86] = fma52hi(res[86], a[34], a[51]); // Sum(85) + res[86] = fma52lo(res[86], a[35], a[51]); // Sum(86) + res[87] = fma52hi(res[87], a[35], a[51]); // Sum(86) + res[87] = fma52lo(res[87], a[36], a[51]); // Sum(87) + res[88] = fma52hi(res[88], a[36], a[51]); // Sum(87) + res[88] = fma52lo(res[88], a[37], a[51]); // Sum(88) + res[89] = fma52hi(res[89], a[37], a[51]); // Sum(88) + res[89] = fma52lo(res[89], a[38], a[51]); // Sum(89) + res[90] = fma52hi(res[90], a[38], a[51]); // Sum(89) + res[90] = fma52lo(res[90], a[39], a[51]); // Sum(90) + res[91] = fma52hi(res[91], a[39], a[51]); // Sum(90) + res[91] = fma52lo(res[91], a[40], a[51]); // Sum(91) + res[92] = fma52hi(res[92], a[40], a[51]); // Sum(91) + res[92] = fma52lo(res[92], a[41], a[51]); // Sum(92) + res[93] = fma52hi(res[93], a[41], a[51]); // Sum(92) + res[93] = fma52lo(res[93], a[42], a[51]); // Sum(93) + res[94] = fma52hi(res[94], a[42], a[51]); // Sum(93) + res[94] = fma52lo(res[94], a[43], a[51]); // Sum(94) + res[95] = fma52hi(res[95], a[43], a[51]); // Sum(94) + res[95] = fma52lo(res[95], a[44], a[51]); // Sum(95) + res[96] = fma52hi(res[96], a[44], a[51]); // Sum(95) + res[84] = fma52lo(res[84], a[32], a[52]); // Sum(84) + res[85] = fma52hi(res[85], a[32], a[52]); // Sum(84) + res[85] = fma52lo(res[85], a[33], a[52]); // Sum(85) + res[86] = fma52hi(res[86], a[33], a[52]); // Sum(85) + res[86] = fma52lo(res[86], a[34], a[52]); // Sum(86) + res[87] = fma52hi(res[87], a[34], a[52]); // Sum(86) + res[87] = fma52lo(res[87], a[35], a[52]); // Sum(87) + res[88] = fma52hi(res[88], a[35], a[52]); // Sum(87) + res[88] = fma52lo(res[88], a[36], a[52]); // Sum(88) + res[89] = fma52hi(res[89], a[36], a[52]); // Sum(88) + res[89] = fma52lo(res[89], a[37], a[52]); // Sum(89) + res[90] = fma52hi(res[90], a[37], a[52]); // Sum(89) + res[90] = fma52lo(res[90], a[38], a[52]); // Sum(90) + res[91] = fma52hi(res[91], a[38], a[52]); // Sum(90) + res[91] = fma52lo(res[91], a[39], a[52]); // Sum(91) + res[92] = fma52hi(res[92], a[39], a[52]); // Sum(91) + res[92] = fma52lo(res[92], a[40], a[52]); // Sum(92) + res[93] = fma52hi(res[93], a[40], a[52]); // Sum(92) + res[93] = fma52lo(res[93], a[41], a[52]); // Sum(93) + res[94] = fma52hi(res[94], a[41], a[52]); // Sum(93) + res[94] = fma52lo(res[94], a[42], a[52]); // Sum(94) + res[95] = fma52hi(res[95], a[42], a[52]); // Sum(94) + res[95] = fma52lo(res[95], a[43], a[52]); // Sum(95) + res[96] = fma52hi(res[96], a[43], a[52]); // Sum(95) + res[84] = fma52lo(res[84], a[31], a[53]); // Sum(84) + res[85] = fma52hi(res[85], a[31], a[53]); // Sum(84) + res[85] = fma52lo(res[85], a[32], a[53]); // Sum(85) + res[86] = fma52hi(res[86], a[32], a[53]); // Sum(85) + res[86] = fma52lo(res[86], a[33], a[53]); // Sum(86) + res[87] = fma52hi(res[87], a[33], a[53]); // Sum(86) + res[87] = fma52lo(res[87], a[34], a[53]); // Sum(87) + res[88] = fma52hi(res[88], a[34], a[53]); // Sum(87) + res[88] = fma52lo(res[88], a[35], a[53]); // Sum(88) + res[89] = fma52hi(res[89], a[35], a[53]); // Sum(88) + res[89] = fma52lo(res[89], a[36], a[53]); // Sum(89) + res[90] = fma52hi(res[90], a[36], a[53]); // Sum(89) + res[90] = fma52lo(res[90], a[37], a[53]); // Sum(90) + res[91] = fma52hi(res[91], a[37], a[53]); // Sum(90) + res[91] = fma52lo(res[91], a[38], a[53]); // Sum(91) + res[92] = fma52hi(res[92], a[38], a[53]); // Sum(91) + res[92] = fma52lo(res[92], a[39], a[53]); // Sum(92) + res[93] = fma52hi(res[93], a[39], a[53]); // Sum(92) + res[93] = fma52lo(res[93], a[40], a[53]); // Sum(93) + res[94] = fma52hi(res[94], a[40], a[53]); // Sum(93) + res[94] = fma52lo(res[94], a[41], a[53]); // Sum(94) + res[95] = fma52hi(res[95], a[41], a[53]); // Sum(94) + res[95] = fma52lo(res[95], a[42], a[53]); // Sum(95) + res[96] = fma52hi(res[96], a[42], a[53]); // Sum(95) + res[84] = fma52lo(res[84], a[30], a[54]); // Sum(84) + res[85] = fma52hi(res[85], a[30], a[54]); // Sum(84) + res[85] = fma52lo(res[85], a[31], a[54]); // Sum(85) + res[86] = fma52hi(res[86], a[31], a[54]); // Sum(85) + res[86] = fma52lo(res[86], a[32], a[54]); // Sum(86) + res[87] = fma52hi(res[87], a[32], a[54]); // Sum(86) + res[87] = fma52lo(res[87], a[33], a[54]); // Sum(87) + res[88] = fma52hi(res[88], a[33], a[54]); // Sum(87) + res[88] = fma52lo(res[88], a[34], a[54]); // Sum(88) + res[89] = fma52hi(res[89], a[34], a[54]); // Sum(88) + res[89] = fma52lo(res[89], a[35], a[54]); // Sum(89) + res[90] = fma52hi(res[90], a[35], a[54]); // Sum(89) + res[90] = fma52lo(res[90], a[36], a[54]); // Sum(90) + res[91] = fma52hi(res[91], a[36], a[54]); // Sum(90) + res[91] = fma52lo(res[91], a[37], a[54]); // Sum(91) + res[92] = fma52hi(res[92], a[37], a[54]); // Sum(91) + res[92] = fma52lo(res[92], a[38], a[54]); // Sum(92) + res[93] = fma52hi(res[93], a[38], a[54]); // Sum(92) + res[93] = fma52lo(res[93], a[39], a[54]); // Sum(93) + res[94] = fma52hi(res[94], a[39], a[54]); // Sum(93) + res[94] = fma52lo(res[94], a[40], a[54]); // Sum(94) + res[95] = fma52hi(res[95], a[40], a[54]); // Sum(94) + res[95] = fma52lo(res[95], a[41], a[54]); // Sum(95) + res[96] = fma52hi(res[96], a[41], a[54]); // Sum(95) + res[84] = fma52lo(res[84], a[29], a[55]); // Sum(84) + res[85] = fma52hi(res[85], a[29], a[55]); // Sum(84) + res[85] = fma52lo(res[85], a[30], a[55]); // Sum(85) + res[86] = fma52hi(res[86], a[30], a[55]); // Sum(85) + res[86] = fma52lo(res[86], a[31], a[55]); // Sum(86) + res[87] = fma52hi(res[87], a[31], a[55]); // Sum(86) + res[87] = fma52lo(res[87], a[32], a[55]); // Sum(87) + res[88] = fma52hi(res[88], a[32], a[55]); // Sum(87) + res[88] = fma52lo(res[88], a[33], a[55]); // Sum(88) + res[89] = fma52hi(res[89], a[33], a[55]); // Sum(88) + res[89] = fma52lo(res[89], a[34], a[55]); // Sum(89) + res[90] = fma52hi(res[90], a[34], a[55]); // Sum(89) + res[90] = fma52lo(res[90], a[35], a[55]); // Sum(90) + res[91] = fma52hi(res[91], a[35], a[55]); // Sum(90) + res[91] = fma52lo(res[91], a[36], a[55]); // Sum(91) + res[92] = fma52hi(res[92], a[36], a[55]); // Sum(91) + res[92] = fma52lo(res[92], a[37], a[55]); // Sum(92) + res[93] = fma52hi(res[93], a[37], a[55]); // Sum(92) + res[93] = fma52lo(res[93], a[38], a[55]); // Sum(93) + res[94] = fma52hi(res[94], a[38], a[55]); // Sum(93) + res[94] = fma52lo(res[94], a[39], a[55]); // Sum(94) + res[95] = fma52hi(res[95], a[39], a[55]); // Sum(94) + res[95] = fma52lo(res[95], a[40], a[55]); // Sum(95) + res[96] = fma52hi(res[96], a[40], a[55]); // Sum(95) + res[84] = fma52lo(res[84], a[28], a[56]); // Sum(84) + res[85] = fma52hi(res[85], a[28], a[56]); // Sum(84) + res[85] = fma52lo(res[85], a[29], a[56]); // Sum(85) + res[86] = fma52hi(res[86], a[29], a[56]); // Sum(85) + res[86] = fma52lo(res[86], a[30], a[56]); // Sum(86) + res[87] = fma52hi(res[87], a[30], a[56]); // Sum(86) + res[87] = fma52lo(res[87], a[31], a[56]); // Sum(87) + res[88] = fma52hi(res[88], a[31], a[56]); // Sum(87) + res[88] = fma52lo(res[88], a[32], a[56]); // Sum(88) + res[89] = fma52hi(res[89], a[32], a[56]); // Sum(88) + res[89] = fma52lo(res[89], a[33], a[56]); // Sum(89) + res[90] = fma52hi(res[90], a[33], a[56]); // Sum(89) + res[90] = fma52lo(res[90], a[34], a[56]); // Sum(90) + res[91] = fma52hi(res[91], a[34], a[56]); // Sum(90) + res[91] = fma52lo(res[91], a[35], a[56]); // Sum(91) + res[92] = fma52hi(res[92], a[35], a[56]); // Sum(91) + res[92] = fma52lo(res[92], a[36], a[56]); // Sum(92) + res[93] = fma52hi(res[93], a[36], a[56]); // Sum(92) + res[93] = fma52lo(res[93], a[37], a[56]); // Sum(93) + res[94] = fma52hi(res[94], a[37], a[56]); // Sum(93) + res[94] = fma52lo(res[94], a[38], a[56]); // Sum(94) + res[95] = fma52hi(res[95], a[38], a[56]); // Sum(94) + res[95] = fma52lo(res[95], a[39], a[56]); // Sum(95) + res[96] = fma52hi(res[96], a[39], a[56]); // Sum(95) + res[84] = fma52lo(res[84], a[27], a[57]); // Sum(84) + res[85] = fma52hi(res[85], a[27], a[57]); // Sum(84) + res[85] = fma52lo(res[85], a[28], a[57]); // Sum(85) + res[86] = fma52hi(res[86], a[28], a[57]); // Sum(85) + res[86] = fma52lo(res[86], a[29], a[57]); // Sum(86) + res[87] = fma52hi(res[87], a[29], a[57]); // Sum(86) + res[87] = fma52lo(res[87], a[30], a[57]); // Sum(87) + res[88] = fma52hi(res[88], a[30], a[57]); // Sum(87) + res[88] = fma52lo(res[88], a[31], a[57]); // Sum(88) + res[89] = fma52hi(res[89], a[31], a[57]); // Sum(88) + res[89] = fma52lo(res[89], a[32], a[57]); // Sum(89) + res[90] = fma52hi(res[90], a[32], a[57]); // Sum(89) + res[90] = fma52lo(res[90], a[33], a[57]); // Sum(90) + res[91] = fma52hi(res[91], a[33], a[57]); // Sum(90) + res[91] = fma52lo(res[91], a[34], a[57]); // Sum(91) + res[92] = fma52hi(res[92], a[34], a[57]); // Sum(91) + res[92] = fma52lo(res[92], a[35], a[57]); // Sum(92) + res[93] = fma52hi(res[93], a[35], a[57]); // Sum(92) + res[93] = fma52lo(res[93], a[36], a[57]); // Sum(93) + res[94] = fma52hi(res[94], a[36], a[57]); // Sum(93) + res[94] = fma52lo(res[94], a[37], a[57]); // Sum(94) + res[95] = fma52hi(res[95], a[37], a[57]); // Sum(94) + res[95] = fma52lo(res[95], a[38], a[57]); // Sum(95) + res[96] = fma52hi(res[96], a[38], a[57]); // Sum(95) + res[84] = fma52lo(res[84], a[26], a[58]); // Sum(84) + res[85] = fma52hi(res[85], a[26], a[58]); // Sum(84) + res[85] = fma52lo(res[85], a[27], a[58]); // Sum(85) + res[86] = fma52hi(res[86], a[27], a[58]); // Sum(85) + res[86] = fma52lo(res[86], a[28], a[58]); // Sum(86) + res[87] = fma52hi(res[87], a[28], a[58]); // Sum(86) + res[87] = fma52lo(res[87], a[29], a[58]); // Sum(87) + res[88] = fma52hi(res[88], a[29], a[58]); // Sum(87) + res[88] = fma52lo(res[88], a[30], a[58]); // Sum(88) + res[89] = fma52hi(res[89], a[30], a[58]); // Sum(88) + res[89] = fma52lo(res[89], a[31], a[58]); // Sum(89) + res[90] = fma52hi(res[90], a[31], a[58]); // Sum(89) + res[90] = fma52lo(res[90], a[32], a[58]); // Sum(90) + res[91] = fma52hi(res[91], a[32], a[58]); // Sum(90) + res[91] = fma52lo(res[91], a[33], a[58]); // Sum(91) + res[92] = fma52hi(res[92], a[33], a[58]); // Sum(91) + res[92] = fma52lo(res[92], a[34], a[58]); // Sum(92) + res[93] = fma52hi(res[93], a[34], a[58]); // Sum(92) + res[93] = fma52lo(res[93], a[35], a[58]); // Sum(93) + res[94] = fma52hi(res[94], a[35], a[58]); // Sum(93) + res[94] = fma52lo(res[94], a[36], a[58]); // Sum(94) + res[95] = fma52hi(res[95], a[36], a[58]); // Sum(94) + res[95] = fma52lo(res[95], a[37], a[58]); // Sum(95) + res[96] = fma52hi(res[96], a[37], a[58]); // Sum(95) + res[84] = fma52lo(res[84], a[25], a[59]); // Sum(84) + res[85] = fma52hi(res[85], a[25], a[59]); // Sum(84) + res[85] = fma52lo(res[85], a[26], a[59]); // Sum(85) + res[86] = fma52hi(res[86], a[26], a[59]); // Sum(85) + res[86] = fma52lo(res[86], a[27], a[59]); // Sum(86) + res[87] = fma52hi(res[87], a[27], a[59]); // Sum(86) + res[87] = fma52lo(res[87], a[28], a[59]); // Sum(87) + res[88] = fma52hi(res[88], a[28], a[59]); // Sum(87) + res[88] = fma52lo(res[88], a[29], a[59]); // Sum(88) + res[89] = fma52hi(res[89], a[29], a[59]); // Sum(88) + res[89] = fma52lo(res[89], a[30], a[59]); // Sum(89) + res[90] = fma52hi(res[90], a[30], a[59]); // Sum(89) + res[90] = fma52lo(res[90], a[31], a[59]); // Sum(90) + res[91] = fma52hi(res[91], a[31], a[59]); // Sum(90) + res[91] = fma52lo(res[91], a[32], a[59]); // Sum(91) + res[92] = fma52hi(res[92], a[32], a[59]); // Sum(91) + res[92] = fma52lo(res[92], a[33], a[59]); // Sum(92) + res[93] = fma52hi(res[93], a[33], a[59]); // Sum(92) + res[93] = fma52lo(res[93], a[34], a[59]); // Sum(93) + res[94] = fma52hi(res[94], a[34], a[59]); // Sum(93) + res[94] = fma52lo(res[94], a[35], a[59]); // Sum(94) + res[95] = fma52hi(res[95], a[35], a[59]); // Sum(94) + res[95] = fma52lo(res[95], a[36], a[59]); // Sum(95) + res[96] = fma52hi(res[96], a[36], a[59]); // Sum(95) + res[84] = fma52lo(res[84], a[24], a[60]); // Sum(84) + res[85] = fma52hi(res[85], a[24], a[60]); // Sum(84) + res[85] = fma52lo(res[85], a[25], a[60]); // Sum(85) + res[86] = fma52hi(res[86], a[25], a[60]); // Sum(85) + res[86] = fma52lo(res[86], a[26], a[60]); // Sum(86) + res[87] = fma52hi(res[87], a[26], a[60]); // Sum(86) + res[87] = fma52lo(res[87], a[27], a[60]); // Sum(87) + res[88] = fma52hi(res[88], a[27], a[60]); // Sum(87) + res[88] = fma52lo(res[88], a[28], a[60]); // Sum(88) + res[89] = fma52hi(res[89], a[28], a[60]); // Sum(88) + res[89] = fma52lo(res[89], a[29], a[60]); // Sum(89) + res[90] = fma52hi(res[90], a[29], a[60]); // Sum(89) + res[90] = fma52lo(res[90], a[30], a[60]); // Sum(90) + res[91] = fma52hi(res[91], a[30], a[60]); // Sum(90) + res[91] = fma52lo(res[91], a[31], a[60]); // Sum(91) + res[92] = fma52hi(res[92], a[31], a[60]); // Sum(91) + res[92] = fma52lo(res[92], a[32], a[60]); // Sum(92) + res[93] = fma52hi(res[93], a[32], a[60]); // Sum(92) + res[93] = fma52lo(res[93], a[33], a[60]); // Sum(93) + res[94] = fma52hi(res[94], a[33], a[60]); // Sum(93) + res[94] = fma52lo(res[94], a[34], a[60]); // Sum(94) + res[95] = fma52hi(res[95], a[34], a[60]); // Sum(94) + res[95] = fma52lo(res[95], a[35], a[60]); // Sum(95) + res[96] = fma52hi(res[96], a[35], a[60]); // Sum(95) + res[84] = fma52lo(res[84], a[23], a[61]); // Sum(84) + res[85] = fma52hi(res[85], a[23], a[61]); // Sum(84) + res[85] = fma52lo(res[85], a[24], a[61]); // Sum(85) + res[86] = fma52hi(res[86], a[24], a[61]); // Sum(85) + res[86] = fma52lo(res[86], a[25], a[61]); // Sum(86) + res[87] = fma52hi(res[87], a[25], a[61]); // Sum(86) + res[87] = fma52lo(res[87], a[26], a[61]); // Sum(87) + res[88] = fma52hi(res[88], a[26], a[61]); // Sum(87) + res[88] = fma52lo(res[88], a[27], a[61]); // Sum(88) + res[89] = fma52hi(res[89], a[27], a[61]); // Sum(88) + res[89] = fma52lo(res[89], a[28], a[61]); // Sum(89) + res[90] = fma52hi(res[90], a[28], a[61]); // Sum(89) + res[90] = fma52lo(res[90], a[29], a[61]); // Sum(90) + res[91] = fma52hi(res[91], a[29], a[61]); // Sum(90) + res[91] = fma52lo(res[91], a[30], a[61]); // Sum(91) + res[92] = fma52hi(res[92], a[30], a[61]); // Sum(91) + res[92] = fma52lo(res[92], a[31], a[61]); // Sum(92) + res[93] = fma52hi(res[93], a[31], a[61]); // Sum(92) + res[93] = fma52lo(res[93], a[32], a[61]); // Sum(93) + res[94] = fma52hi(res[94], a[32], a[61]); // Sum(93) + res[94] = fma52lo(res[94], a[33], a[61]); // Sum(94) + res[95] = fma52hi(res[95], a[33], a[61]); // Sum(94) + res[95] = fma52lo(res[95], a[34], a[61]); // Sum(95) + res[96] = fma52hi(res[96], a[34], a[61]); // Sum(95) + res[84] = fma52lo(res[84], a[22], a[62]); // Sum(84) + res[85] = fma52hi(res[85], a[22], a[62]); // Sum(84) + res[85] = fma52lo(res[85], a[23], a[62]); // Sum(85) + res[86] = fma52hi(res[86], a[23], a[62]); // Sum(85) + res[86] = fma52lo(res[86], a[24], a[62]); // Sum(86) + res[87] = fma52hi(res[87], a[24], a[62]); // Sum(86) + res[87] = fma52lo(res[87], a[25], a[62]); // Sum(87) + res[88] = fma52hi(res[88], a[25], a[62]); // Sum(87) + res[88] = fma52lo(res[88], a[26], a[62]); // Sum(88) + res[89] = fma52hi(res[89], a[26], a[62]); // Sum(88) + res[89] = fma52lo(res[89], a[27], a[62]); // Sum(89) + res[90] = fma52hi(res[90], a[27], a[62]); // Sum(89) + res[90] = fma52lo(res[90], a[28], a[62]); // Sum(90) + res[91] = fma52hi(res[91], a[28], a[62]); // Sum(90) + res[91] = fma52lo(res[91], a[29], a[62]); // Sum(91) + res[92] = fma52hi(res[92], a[29], a[62]); // Sum(91) + res[92] = fma52lo(res[92], a[30], a[62]); // Sum(92) + res[93] = fma52hi(res[93], a[30], a[62]); // Sum(92) + res[93] = fma52lo(res[93], a[31], a[62]); // Sum(93) + res[94] = fma52hi(res[94], a[31], a[62]); // Sum(93) + res[94] = fma52lo(res[94], a[32], a[62]); // Sum(94) + res[95] = fma52hi(res[95], a[32], a[62]); // Sum(94) + res[95] = fma52lo(res[95], a[33], a[62]); // Sum(95) + res[96] = fma52hi(res[96], a[33], a[62]); // Sum(95) + res[84] = fma52lo(res[84], a[21], a[63]); // Sum(84) + res[85] = fma52hi(res[85], a[21], a[63]); // Sum(84) + res[85] = fma52lo(res[85], a[22], a[63]); // Sum(85) + res[86] = fma52hi(res[86], a[22], a[63]); // Sum(85) + res[86] = fma52lo(res[86], a[23], a[63]); // Sum(86) + res[87] = fma52hi(res[87], a[23], a[63]); // Sum(86) + res[87] = fma52lo(res[87], a[24], a[63]); // Sum(87) + res[88] = fma52hi(res[88], a[24], a[63]); // Sum(87) + res[88] = fma52lo(res[88], a[25], a[63]); // Sum(88) + res[89] = fma52hi(res[89], a[25], a[63]); // Sum(88) + res[89] = fma52lo(res[89], a[26], a[63]); // Sum(89) + res[90] = fma52hi(res[90], a[26], a[63]); // Sum(89) + res[90] = fma52lo(res[90], a[27], a[63]); // Sum(90) + res[91] = fma52hi(res[91], a[27], a[63]); // Sum(90) + res[91] = fma52lo(res[91], a[28], a[63]); // Sum(91) + res[92] = fma52hi(res[92], a[28], a[63]); // Sum(91) + res[92] = fma52lo(res[92], a[29], a[63]); // Sum(92) + res[93] = fma52hi(res[93], a[29], a[63]); // Sum(92) + res[93] = fma52lo(res[93], a[30], a[63]); // Sum(93) + res[94] = fma52hi(res[94], a[30], a[63]); // Sum(93) + res[94] = fma52lo(res[94], a[31], a[63]); // Sum(94) + res[95] = fma52hi(res[95], a[31], a[63]); // Sum(94) + res[95] = fma52lo(res[95], a[32], a[63]); // Sum(95) + res[96] = fma52hi(res[96], a[32], a[63]); // Sum(95) + res[84] = fma52lo(res[84], a[20], a[64]); // Sum(84) + res[85] = fma52hi(res[85], a[20], a[64]); // Sum(84) + res[85] = fma52lo(res[85], a[21], a[64]); // Sum(85) + res[86] = fma52hi(res[86], a[21], a[64]); // Sum(85) + res[86] = fma52lo(res[86], a[22], a[64]); // Sum(86) + res[87] = fma52hi(res[87], a[22], a[64]); // Sum(86) + res[87] = fma52lo(res[87], a[23], a[64]); // Sum(87) + res[88] = fma52hi(res[88], a[23], a[64]); // Sum(87) + res[88] = fma52lo(res[88], a[24], a[64]); // Sum(88) + res[89] = fma52hi(res[89], a[24], a[64]); // Sum(88) + res[89] = fma52lo(res[89], a[25], a[64]); // Sum(89) + res[90] = fma52hi(res[90], a[25], a[64]); // Sum(89) + res[90] = fma52lo(res[90], a[26], a[64]); // Sum(90) + res[91] = fma52hi(res[91], a[26], a[64]); // Sum(90) + res[91] = fma52lo(res[91], a[27], a[64]); // Sum(91) + res[92] = fma52hi(res[92], a[27], a[64]); // Sum(91) + res[92] = fma52lo(res[92], a[28], a[64]); // Sum(92) + res[93] = fma52hi(res[93], a[28], a[64]); // Sum(92) + res[93] = fma52lo(res[93], a[29], a[64]); // Sum(93) + res[94] = fma52hi(res[94], a[29], a[64]); // Sum(93) + res[94] = fma52lo(res[94], a[30], a[64]); // Sum(94) + res[95] = fma52hi(res[95], a[30], a[64]); // Sum(94) + res[95] = fma52lo(res[95], a[31], a[64]); // Sum(95) + res[96] = fma52hi(res[96], a[31], a[64]); // Sum(95) + res[84] = fma52lo(res[84], a[19], a[65]); // Sum(84) + res[85] = fma52hi(res[85], a[19], a[65]); // Sum(84) + res[85] = fma52lo(res[85], a[20], a[65]); // Sum(85) + res[86] = fma52hi(res[86], a[20], a[65]); // Sum(85) + res[86] = fma52lo(res[86], a[21], a[65]); // Sum(86) + res[87] = fma52hi(res[87], a[21], a[65]); // Sum(86) + res[87] = fma52lo(res[87], a[22], a[65]); // Sum(87) + res[88] = fma52hi(res[88], a[22], a[65]); // Sum(87) + res[88] = fma52lo(res[88], a[23], a[65]); // Sum(88) + res[89] = fma52hi(res[89], a[23], a[65]); // Sum(88) + res[89] = fma52lo(res[89], a[24], a[65]); // Sum(89) + res[90] = fma52hi(res[90], a[24], a[65]); // Sum(89) + res[90] = fma52lo(res[90], a[25], a[65]); // Sum(90) + res[91] = fma52hi(res[91], a[25], a[65]); // Sum(90) + res[91] = fma52lo(res[91], a[26], a[65]); // Sum(91) + res[92] = fma52hi(res[92], a[26], a[65]); // Sum(91) + res[92] = fma52lo(res[92], a[27], a[65]); // Sum(92) + res[93] = fma52hi(res[93], a[27], a[65]); // Sum(92) + res[93] = fma52lo(res[93], a[28], a[65]); // Sum(93) + res[94] = fma52hi(res[94], a[28], a[65]); // Sum(93) + res[94] = fma52lo(res[94], a[29], a[65]); // Sum(94) + res[95] = fma52hi(res[95], a[29], a[65]); // Sum(94) + res[95] = fma52lo(res[95], a[30], a[65]); // Sum(95) + res[96] = fma52hi(res[96], a[30], a[65]); // Sum(95) + res[84] = fma52lo(res[84], a[18], a[66]); // Sum(84) + res[85] = fma52hi(res[85], a[18], a[66]); // Sum(84) + res[85] = fma52lo(res[85], a[19], a[66]); // Sum(85) + res[86] = fma52hi(res[86], a[19], a[66]); // Sum(85) + res[86] = fma52lo(res[86], a[20], a[66]); // Sum(86) + res[87] = fma52hi(res[87], a[20], a[66]); // Sum(86) + res[87] = fma52lo(res[87], a[21], a[66]); // Sum(87) + res[88] = fma52hi(res[88], a[21], a[66]); // Sum(87) + res[88] = fma52lo(res[88], a[22], a[66]); // Sum(88) + res[89] = fma52hi(res[89], a[22], a[66]); // Sum(88) + res[89] = fma52lo(res[89], a[23], a[66]); // Sum(89) + res[90] = fma52hi(res[90], a[23], a[66]); // Sum(89) + res[90] = fma52lo(res[90], a[24], a[66]); // Sum(90) + res[91] = fma52hi(res[91], a[24], a[66]); // Sum(90) + res[91] = fma52lo(res[91], a[25], a[66]); // Sum(91) + res[92] = fma52hi(res[92], a[25], a[66]); // Sum(91) + res[92] = fma52lo(res[92], a[26], a[66]); // Sum(92) + res[93] = fma52hi(res[93], a[26], a[66]); // Sum(92) + res[93] = fma52lo(res[93], a[27], a[66]); // Sum(93) + res[94] = fma52hi(res[94], a[27], a[66]); // Sum(93) + res[94] = fma52lo(res[94], a[28], a[66]); // Sum(94) + res[95] = fma52hi(res[95], a[28], a[66]); // Sum(94) + res[95] = fma52lo(res[95], a[29], a[66]); // Sum(95) + res[96] = fma52hi(res[96], a[29], a[66]); // Sum(95) + res[84] = fma52lo(res[84], a[17], a[67]); // Sum(84) + res[85] = fma52hi(res[85], a[17], a[67]); // Sum(84) + res[85] = fma52lo(res[85], a[18], a[67]); // Sum(85) + res[86] = fma52hi(res[86], a[18], a[67]); // Sum(85) + res[86] = fma52lo(res[86], a[19], a[67]); // Sum(86) + res[87] = fma52hi(res[87], a[19], a[67]); // Sum(86) + res[87] = fma52lo(res[87], a[20], a[67]); // Sum(87) + res[88] = fma52hi(res[88], a[20], a[67]); // Sum(87) + res[88] = fma52lo(res[88], a[21], a[67]); // Sum(88) + res[89] = fma52hi(res[89], a[21], a[67]); // Sum(88) + res[89] = fma52lo(res[89], a[22], a[67]); // Sum(89) + res[90] = fma52hi(res[90], a[22], a[67]); // Sum(89) + res[90] = fma52lo(res[90], a[23], a[67]); // Sum(90) + res[91] = fma52hi(res[91], a[23], a[67]); // Sum(90) + res[91] = fma52lo(res[91], a[24], a[67]); // Sum(91) + res[92] = fma52hi(res[92], a[24], a[67]); // Sum(91) + res[92] = fma52lo(res[92], a[25], a[67]); // Sum(92) + res[93] = fma52hi(res[93], a[25], a[67]); // Sum(92) + res[93] = fma52lo(res[93], a[26], a[67]); // Sum(93) + res[94] = fma52hi(res[94], a[26], a[67]); // Sum(93) + res[94] = fma52lo(res[94], a[27], a[67]); // Sum(94) + res[95] = fma52hi(res[95], a[27], a[67]); // Sum(94) + res[95] = fma52lo(res[95], a[28], a[67]); // Sum(95) + res[96] = fma52hi(res[96], a[28], a[67]); // Sum(95) + res[84] = fma52lo(res[84], a[16], a[68]); // Sum(84) + res[85] = fma52hi(res[85], a[16], a[68]); // Sum(84) + res[85] = fma52lo(res[85], a[17], a[68]); // Sum(85) + res[86] = fma52hi(res[86], a[17], a[68]); // Sum(85) + res[86] = fma52lo(res[86], a[18], a[68]); // Sum(86) + res[87] = fma52hi(res[87], a[18], a[68]); // Sum(86) + res[87] = fma52lo(res[87], a[19], a[68]); // Sum(87) + res[88] = fma52hi(res[88], a[19], a[68]); // Sum(87) + res[88] = fma52lo(res[88], a[20], a[68]); // Sum(88) + res[89] = fma52hi(res[89], a[20], a[68]); // Sum(88) + res[89] = fma52lo(res[89], a[21], a[68]); // Sum(89) + res[90] = fma52hi(res[90], a[21], a[68]); // Sum(89) + res[90] = fma52lo(res[90], a[22], a[68]); // Sum(90) + res[91] = fma52hi(res[91], a[22], a[68]); // Sum(90) + res[91] = fma52lo(res[91], a[23], a[68]); // Sum(91) + res[92] = fma52hi(res[92], a[23], a[68]); // Sum(91) + res[92] = fma52lo(res[92], a[24], a[68]); // Sum(92) + res[93] = fma52hi(res[93], a[24], a[68]); // Sum(92) + res[93] = fma52lo(res[93], a[25], a[68]); // Sum(93) + res[94] = fma52hi(res[94], a[25], a[68]); // Sum(93) + res[94] = fma52lo(res[94], a[26], a[68]); // Sum(94) + res[95] = fma52hi(res[95], a[26], a[68]); // Sum(94) + res[95] = fma52lo(res[95], a[27], a[68]); // Sum(95) + res[96] = fma52hi(res[96], a[27], a[68]); // Sum(95) + res[84] = fma52lo(res[84], a[15], a[69]); // Sum(84) + res[85] = fma52hi(res[85], a[15], a[69]); // Sum(84) + res[85] = fma52lo(res[85], a[16], a[69]); // Sum(85) + res[86] = fma52hi(res[86], a[16], a[69]); // Sum(85) + res[86] = fma52lo(res[86], a[17], a[69]); // Sum(86) + res[87] = fma52hi(res[87], a[17], a[69]); // Sum(86) + res[87] = fma52lo(res[87], a[18], a[69]); // Sum(87) + res[88] = fma52hi(res[88], a[18], a[69]); // Sum(87) + res[88] = fma52lo(res[88], a[19], a[69]); // Sum(88) + res[89] = fma52hi(res[89], a[19], a[69]); // Sum(88) + res[89] = fma52lo(res[89], a[20], a[69]); // Sum(89) + res[90] = fma52hi(res[90], a[20], a[69]); // Sum(89) + res[90] = fma52lo(res[90], a[21], a[69]); // Sum(90) + res[91] = fma52hi(res[91], a[21], a[69]); // Sum(90) + res[91] = fma52lo(res[91], a[22], a[69]); // Sum(91) + res[92] = fma52hi(res[92], a[22], a[69]); // Sum(91) + res[92] = fma52lo(res[92], a[23], a[69]); // Sum(92) + res[93] = fma52hi(res[93], a[23], a[69]); // Sum(92) + res[93] = fma52lo(res[93], a[24], a[69]); // Sum(93) + res[94] = fma52hi(res[94], a[24], a[69]); // Sum(93) + res[94] = fma52lo(res[94], a[25], a[69]); // Sum(94) + res[95] = fma52hi(res[95], a[25], a[69]); // Sum(94) + res[95] = fma52lo(res[95], a[26], a[69]); // Sum(95) + res[96] = fma52hi(res[96], a[26], a[69]); // Sum(95) + res[84] = fma52lo(res[84], a[14], a[70]); // Sum(84) + res[85] = fma52hi(res[85], a[14], a[70]); // Sum(84) + res[85] = fma52lo(res[85], a[15], a[70]); // Sum(85) + res[86] = fma52hi(res[86], a[15], a[70]); // Sum(85) + res[86] = fma52lo(res[86], a[16], a[70]); // Sum(86) + res[87] = fma52hi(res[87], a[16], a[70]); // Sum(86) + res[87] = fma52lo(res[87], a[17], a[70]); // Sum(87) + res[88] = fma52hi(res[88], a[17], a[70]); // Sum(87) + res[88] = fma52lo(res[88], a[18], a[70]); // Sum(88) + res[89] = fma52hi(res[89], a[18], a[70]); // Sum(88) + res[89] = fma52lo(res[89], a[19], a[70]); // Sum(89) + res[90] = fma52hi(res[90], a[19], a[70]); // Sum(89) + res[90] = fma52lo(res[90], a[20], a[70]); // Sum(90) + res[91] = fma52hi(res[91], a[20], a[70]); // Sum(90) + res[91] = fma52lo(res[91], a[21], a[70]); // Sum(91) + res[92] = fma52hi(res[92], a[21], a[70]); // Sum(91) + res[92] = fma52lo(res[92], a[22], a[70]); // Sum(92) + res[93] = fma52hi(res[93], a[22], a[70]); // Sum(92) + res[93] = fma52lo(res[93], a[23], a[70]); // Sum(93) + res[94] = fma52hi(res[94], a[23], a[70]); // Sum(93) + res[94] = fma52lo(res[94], a[24], a[70]); // Sum(94) + res[95] = fma52hi(res[95], a[24], a[70]); // Sum(94) + res[95] = fma52lo(res[95], a[25], a[70]); // Sum(95) + res[96] = fma52hi(res[96], a[25], a[70]); // Sum(95) + res[84] = fma52lo(res[84], a[13], a[71]); // Sum(84) + res[85] = fma52hi(res[85], a[13], a[71]); // Sum(84) + res[85] = fma52lo(res[85], a[14], a[71]); // Sum(85) + res[86] = fma52hi(res[86], a[14], a[71]); // Sum(85) + res[86] = fma52lo(res[86], a[15], a[71]); // Sum(86) + res[87] = fma52hi(res[87], a[15], a[71]); // Sum(86) + res[87] = fma52lo(res[87], a[16], a[71]); // Sum(87) + res[88] = fma52hi(res[88], a[16], a[71]); // Sum(87) + res[88] = fma52lo(res[88], a[17], a[71]); // Sum(88) + res[89] = fma52hi(res[89], a[17], a[71]); // Sum(88) + res[89] = fma52lo(res[89], a[18], a[71]); // Sum(89) + res[90] = fma52hi(res[90], a[18], a[71]); // Sum(89) + res[90] = fma52lo(res[90], a[19], a[71]); // Sum(90) + res[91] = fma52hi(res[91], a[19], a[71]); // Sum(90) + res[91] = fma52lo(res[91], a[20], a[71]); // Sum(91) + res[92] = fma52hi(res[92], a[20], a[71]); // Sum(91) + res[92] = fma52lo(res[92], a[21], a[71]); // Sum(92) + res[93] = fma52hi(res[93], a[21], a[71]); // Sum(92) + res[93] = fma52lo(res[93], a[22], a[71]); // Sum(93) + res[94] = fma52hi(res[94], a[22], a[71]); // Sum(93) + res[94] = fma52lo(res[94], a[23], a[71]); // Sum(94) + res[95] = fma52hi(res[95], a[23], a[71]); // Sum(94) + res[95] = fma52lo(res[95], a[24], a[71]); // Sum(95) + res[96] = fma52hi(res[96], a[24], a[71]); // Sum(95) + res[84] = fma52lo(res[84], a[12], a[72]); // Sum(84) + res[85] = fma52hi(res[85], a[12], a[72]); // Sum(84) + res[85] = fma52lo(res[85], a[13], a[72]); // Sum(85) + res[86] = fma52hi(res[86], a[13], a[72]); // Sum(85) + res[86] = fma52lo(res[86], a[14], a[72]); // Sum(86) + res[87] = fma52hi(res[87], a[14], a[72]); // Sum(86) + res[87] = fma52lo(res[87], a[15], a[72]); // Sum(87) + res[88] = fma52hi(res[88], a[15], a[72]); // Sum(87) + res[88] = fma52lo(res[88], a[16], a[72]); // Sum(88) + res[89] = fma52hi(res[89], a[16], a[72]); // Sum(88) + res[89] = fma52lo(res[89], a[17], a[72]); // Sum(89) + res[90] = fma52hi(res[90], a[17], a[72]); // Sum(89) + res[90] = fma52lo(res[90], a[18], a[72]); // Sum(90) + res[91] = fma52hi(res[91], a[18], a[72]); // Sum(90) + res[91] = fma52lo(res[91], a[19], a[72]); // Sum(91) + res[92] = fma52hi(res[92], a[19], a[72]); // Sum(91) + res[92] = fma52lo(res[92], a[20], a[72]); // Sum(92) + res[93] = fma52hi(res[93], a[20], a[72]); // Sum(92) + res[93] = fma52lo(res[93], a[21], a[72]); // Sum(93) + res[94] = fma52hi(res[94], a[21], a[72]); // Sum(93) + res[94] = fma52lo(res[94], a[22], a[72]); // Sum(94) + res[95] = fma52hi(res[95], a[22], a[72]); // Sum(94) + res[95] = fma52lo(res[95], a[23], a[72]); // Sum(95) + res[96] = fma52hi(res[96], a[23], a[72]); // Sum(95) + res[84] = fma52lo(res[84], a[11], a[73]); // Sum(84) + res[85] = fma52hi(res[85], a[11], a[73]); // Sum(84) + res[85] = fma52lo(res[85], a[12], a[73]); // Sum(85) + res[86] = fma52hi(res[86], a[12], a[73]); // Sum(85) + res[86] = fma52lo(res[86], a[13], a[73]); // Sum(86) + res[87] = fma52hi(res[87], a[13], a[73]); // Sum(86) + res[87] = fma52lo(res[87], a[14], a[73]); // Sum(87) + res[88] = fma52hi(res[88], a[14], a[73]); // Sum(87) + res[88] = fma52lo(res[88], a[15], a[73]); // Sum(88) + res[89] = fma52hi(res[89], a[15], a[73]); // Sum(88) + res[89] = fma52lo(res[89], a[16], a[73]); // Sum(89) + res[90] = fma52hi(res[90], a[16], a[73]); // Sum(89) + res[90] = fma52lo(res[90], a[17], a[73]); // Sum(90) + res[91] = fma52hi(res[91], a[17], a[73]); // Sum(90) + res[91] = fma52lo(res[91], a[18], a[73]); // Sum(91) + res[92] = fma52hi(res[92], a[18], a[73]); // Sum(91) + res[92] = fma52lo(res[92], a[19], a[73]); // Sum(92) + res[93] = fma52hi(res[93], a[19], a[73]); // Sum(92) + res[93] = fma52lo(res[93], a[20], a[73]); // Sum(93) + res[94] = fma52hi(res[94], a[20], a[73]); // Sum(93) + res[94] = fma52lo(res[94], a[21], a[73]); // Sum(94) + res[95] = fma52hi(res[95], a[21], a[73]); // Sum(94) + res[95] = fma52lo(res[95], a[22], a[73]); // Sum(95) + res[96] = fma52hi(res[96], a[22], a[73]); // Sum(95) + res[84] = fma52lo(res[84], a[10], a[74]); // Sum(84) + res[85] = fma52hi(res[85], a[10], a[74]); // Sum(84) + res[85] = fma52lo(res[85], a[11], a[74]); // Sum(85) + res[86] = fma52hi(res[86], a[11], a[74]); // Sum(85) + res[86] = fma52lo(res[86], a[12], a[74]); // Sum(86) + res[87] = fma52hi(res[87], a[12], a[74]); // Sum(86) + res[87] = fma52lo(res[87], a[13], a[74]); // Sum(87) + res[88] = fma52hi(res[88], a[13], a[74]); // Sum(87) + res[88] = fma52lo(res[88], a[14], a[74]); // Sum(88) + res[89] = fma52hi(res[89], a[14], a[74]); // Sum(88) + res[89] = fma52lo(res[89], a[15], a[74]); // Sum(89) + res[90] = fma52hi(res[90], a[15], a[74]); // Sum(89) + res[90] = fma52lo(res[90], a[16], a[74]); // Sum(90) + res[91] = fma52hi(res[91], a[16], a[74]); // Sum(90) + res[91] = fma52lo(res[91], a[17], a[74]); // Sum(91) + res[92] = fma52hi(res[92], a[17], a[74]); // Sum(91) + res[92] = fma52lo(res[92], a[18], a[74]); // Sum(92) + res[93] = fma52hi(res[93], a[18], a[74]); // Sum(92) + res[93] = fma52lo(res[93], a[19], a[74]); // Sum(93) + res[94] = fma52hi(res[94], a[19], a[74]); // Sum(93) + res[94] = fma52lo(res[94], a[20], a[74]); // Sum(94) + res[95] = fma52hi(res[95], a[20], a[74]); // Sum(94) + res[95] = fma52lo(res[95], a[21], a[74]); // Sum(95) + res[96] = fma52hi(res[96], a[21], a[74]); // Sum(95) + res[84] = fma52lo(res[84], a[9], a[75]); // Sum(84) + res[85] = fma52hi(res[85], a[9], a[75]); // Sum(84) + res[85] = fma52lo(res[85], a[10], a[75]); // Sum(85) + res[86] = fma52hi(res[86], a[10], a[75]); // Sum(85) + res[86] = fma52lo(res[86], a[11], a[75]); // Sum(86) + res[87] = fma52hi(res[87], a[11], a[75]); // Sum(86) + res[87] = fma52lo(res[87], a[12], a[75]); // Sum(87) + res[88] = fma52hi(res[88], a[12], a[75]); // Sum(87) + res[88] = fma52lo(res[88], a[13], a[75]); // Sum(88) + res[89] = fma52hi(res[89], a[13], a[75]); // Sum(88) + res[89] = fma52lo(res[89], a[14], a[75]); // Sum(89) + res[90] = fma52hi(res[90], a[14], a[75]); // Sum(89) + res[90] = fma52lo(res[90], a[15], a[75]); // Sum(90) + res[91] = fma52hi(res[91], a[15], a[75]); // Sum(90) + res[91] = fma52lo(res[91], a[16], a[75]); // Sum(91) + res[92] = fma52hi(res[92], a[16], a[75]); // Sum(91) + res[92] = fma52lo(res[92], a[17], a[75]); // Sum(92) + res[93] = fma52hi(res[93], a[17], a[75]); // Sum(92) + res[93] = fma52lo(res[93], a[18], a[75]); // Sum(93) + res[94] = fma52hi(res[94], a[18], a[75]); // Sum(93) + res[94] = fma52lo(res[94], a[19], a[75]); // Sum(94) + res[95] = fma52hi(res[95], a[19], a[75]); // Sum(94) + res[95] = fma52lo(res[95], a[20], a[75]); // Sum(95) + res[96] = fma52hi(res[96], a[20], a[75]); // Sum(95) + res[84] = fma52lo(res[84], a[8], a[76]); // Sum(84) + res[85] = fma52hi(res[85], a[8], a[76]); // Sum(84) + res[85] = fma52lo(res[85], a[9], a[76]); // Sum(85) + res[86] = fma52hi(res[86], a[9], a[76]); // Sum(85) + res[86] = fma52lo(res[86], a[10], a[76]); // Sum(86) + res[87] = fma52hi(res[87], a[10], a[76]); // Sum(86) + res[87] = fma52lo(res[87], a[11], a[76]); // Sum(87) + res[88] = fma52hi(res[88], a[11], a[76]); // Sum(87) + res[88] = fma52lo(res[88], a[12], a[76]); // Sum(88) + res[89] = fma52hi(res[89], a[12], a[76]); // Sum(88) + res[89] = fma52lo(res[89], a[13], a[76]); // Sum(89) + res[90] = fma52hi(res[90], a[13], a[76]); // Sum(89) + res[90] = fma52lo(res[90], a[14], a[76]); // Sum(90) + res[91] = fma52hi(res[91], a[14], a[76]); // Sum(90) + res[91] = fma52lo(res[91], a[15], a[76]); // Sum(91) + res[92] = fma52hi(res[92], a[15], a[76]); // Sum(91) + res[92] = fma52lo(res[92], a[16], a[76]); // Sum(92) + res[93] = fma52hi(res[93], a[16], a[76]); // Sum(92) + res[93] = fma52lo(res[93], a[17], a[76]); // Sum(93) + res[94] = fma52hi(res[94], a[17], a[76]); // Sum(93) + res[94] = fma52lo(res[94], a[18], a[76]); // Sum(94) + res[95] = fma52hi(res[95], a[18], a[76]); // Sum(94) + res[95] = fma52lo(res[95], a[19], a[76]); // Sum(95) + res[96] = fma52hi(res[96], a[19], a[76]); // Sum(95) + res[84] = fma52lo(res[84], a[7], a[77]); // Sum(84) + res[85] = fma52hi(res[85], a[7], a[77]); // Sum(84) + res[85] = fma52lo(res[85], a[8], a[77]); // Sum(85) + res[86] = fma52hi(res[86], a[8], a[77]); // Sum(85) + res[86] = fma52lo(res[86], a[9], a[77]); // Sum(86) + res[87] = fma52hi(res[87], a[9], a[77]); // Sum(86) + res[87] = fma52lo(res[87], a[10], a[77]); // Sum(87) + res[88] = fma52hi(res[88], a[10], a[77]); // Sum(87) + res[88] = fma52lo(res[88], a[11], a[77]); // Sum(88) + res[89] = fma52hi(res[89], a[11], a[77]); // Sum(88) + res[89] = fma52lo(res[89], a[12], a[77]); // Sum(89) + res[90] = fma52hi(res[90], a[12], a[77]); // Sum(89) + res[90] = fma52lo(res[90], a[13], a[77]); // Sum(90) + res[91] = fma52hi(res[91], a[13], a[77]); // Sum(90) + res[91] = fma52lo(res[91], a[14], a[77]); // Sum(91) + res[92] = fma52hi(res[92], a[14], a[77]); // Sum(91) + res[92] = fma52lo(res[92], a[15], a[77]); // Sum(92) + res[93] = fma52hi(res[93], a[15], a[77]); // Sum(92) + res[93] = fma52lo(res[93], a[16], a[77]); // Sum(93) + res[94] = fma52hi(res[94], a[16], a[77]); // Sum(93) + res[94] = fma52lo(res[94], a[17], a[77]); // Sum(94) + res[95] = fma52hi(res[95], a[17], a[77]); // Sum(94) + res[95] = fma52lo(res[95], a[18], a[77]); // Sum(95) + res[96] = fma52hi(res[96], a[18], a[77]); // Sum(95) + res[84] = fma52lo(res[84], a[6], a[78]); // Sum(84) + res[85] = fma52hi(res[85], a[6], a[78]); // Sum(84) + res[85] = fma52lo(res[85], a[7], a[78]); // Sum(85) + res[86] = fma52hi(res[86], a[7], a[78]); // Sum(85) + res[86] = fma52lo(res[86], a[8], a[78]); // Sum(86) + res[87] = fma52hi(res[87], a[8], a[78]); // Sum(86) + res[87] = fma52lo(res[87], a[9], a[78]); // Sum(87) + res[88] = fma52hi(res[88], a[9], a[78]); // Sum(87) + res[88] = fma52lo(res[88], a[10], a[78]); // Sum(88) + res[89] = fma52hi(res[89], a[10], a[78]); // Sum(88) + res[89] = fma52lo(res[89], a[11], a[78]); // Sum(89) + res[90] = fma52hi(res[90], a[11], a[78]); // Sum(89) + res[90] = fma52lo(res[90], a[12], a[78]); // Sum(90) + res[91] = fma52hi(res[91], a[12], a[78]); // Sum(90) + res[91] = fma52lo(res[91], a[13], a[78]); // Sum(91) + res[92] = fma52hi(res[92], a[13], a[78]); // Sum(91) + res[92] = fma52lo(res[92], a[14], a[78]); // Sum(92) + res[93] = fma52hi(res[93], a[14], a[78]); // Sum(92) + res[93] = fma52lo(res[93], a[15], a[78]); // Sum(93) + res[94] = fma52hi(res[94], a[15], a[78]); // Sum(93) + res[94] = fma52lo(res[94], a[16], a[78]); // Sum(94) + res[95] = fma52hi(res[95], a[16], a[78]); // Sum(94) + res[95] = fma52lo(res[95], a[17], a[78]); // Sum(95) + res[96] = fma52hi(res[96], a[17], a[78]); // Sum(95) + res[84] = add64(res[84], res[84]); // Double(84) + res[85] = add64(res[85], res[85]); // Double(85) + res[86] = add64(res[86], res[86]); // Double(86) + res[87] = add64(res[87], res[87]); // Double(87) + res[88] = add64(res[88], res[88]); // Double(88) + res[89] = add64(res[89], res[89]); // Double(89) + res[90] = add64(res[90], res[90]); // Double(90) + res[91] = add64(res[91], res[91]); // Double(91) + res[92] = add64(res[92], res[92]); // Double(92) + res[93] = add64(res[93], res[93]); // Double(93) + res[94] = add64(res[94], res[94]); // Double(94) + res[95] = add64(res[95], res[95]); // Double(95) + res[84] = fma52lo(res[84], a[42], a[42]); // Add sqr(84) + res[85] = fma52hi(res[85], a[42], a[42]); // Add sqr(84) + res[86] = fma52lo(res[86], a[43], a[43]); // Add sqr(86) + res[87] = fma52hi(res[87], a[43], a[43]); // Add sqr(86) + res[88] = fma52lo(res[88], a[44], a[44]); // Add sqr(88) + res[89] = fma52hi(res[89], a[44], a[44]); // Add sqr(88) + res[90] = fma52lo(res[90], a[45], a[45]); // Add sqr(90) + res[91] = fma52hi(res[91], a[45], a[45]); // Add sqr(90) + res[92] = fma52lo(res[92], a[46], a[46]); // Add sqr(92) + res[93] = fma52hi(res[93], a[46], a[46]); // Add sqr(92) + res[94] = fma52lo(res[94], a[47], a[47]); // Add sqr(94) + res[95] = fma52hi(res[95], a[47], a[47]); // Add sqr(94) + res[96] = fma52lo(res[96], a[47], a[49]); // Sum(96) + res[97] = fma52hi(res[97], a[47], a[49]); // Sum(96) + res[97] = fma52lo(res[97], a[48], a[49]); // Sum(97) + res[98] = fma52hi(res[98], a[48], a[49]); // Sum(97) + res[96] = fma52lo(res[96], a[46], a[50]); // Sum(96) + res[97] = fma52hi(res[97], a[46], a[50]); // Sum(96) + res[97] = fma52lo(res[97], a[47], a[50]); // Sum(97) + res[98] = fma52hi(res[98], a[47], a[50]); // Sum(97) + res[98] = fma52lo(res[98], a[48], a[50]); // Sum(98) + res[99] = fma52hi(res[99], a[48], a[50]); // Sum(98) + res[99] = fma52lo(res[99], a[49], a[50]); // Sum(99) + res[100] = fma52hi(res[100], a[49], a[50]); // Sum(99) + res[96] = fma52lo(res[96], a[45], a[51]); // Sum(96) + res[97] = fma52hi(res[97], a[45], a[51]); // Sum(96) + res[97] = fma52lo(res[97], a[46], a[51]); // Sum(97) + res[98] = fma52hi(res[98], a[46], a[51]); // Sum(97) + res[98] = fma52lo(res[98], a[47], a[51]); // Sum(98) + res[99] = fma52hi(res[99], a[47], a[51]); // Sum(98) + res[99] = fma52lo(res[99], a[48], a[51]); // Sum(99) + res[100] = fma52hi(res[100], a[48], a[51]); // Sum(99) + res[100] = fma52lo(res[100], a[49], a[51]); // Sum(100) + res[101] = fma52hi(res[101], a[49], a[51]); // Sum(100) + res[101] = fma52lo(res[101], a[50], a[51]); // Sum(101) + res[102] = fma52hi(res[102], a[50], a[51]); // Sum(101) + res[96] = fma52lo(res[96], a[44], a[52]); // Sum(96) + res[97] = fma52hi(res[97], a[44], a[52]); // Sum(96) + res[97] = fma52lo(res[97], a[45], a[52]); // Sum(97) + res[98] = fma52hi(res[98], a[45], a[52]); // Sum(97) + res[98] = fma52lo(res[98], a[46], a[52]); // Sum(98) + res[99] = fma52hi(res[99], a[46], a[52]); // Sum(98) + res[99] = fma52lo(res[99], a[47], a[52]); // Sum(99) + res[100] = fma52hi(res[100], a[47], a[52]); // Sum(99) + res[100] = fma52lo(res[100], a[48], a[52]); // Sum(100) + res[101] = fma52hi(res[101], a[48], a[52]); // Sum(100) + res[101] = fma52lo(res[101], a[49], a[52]); // Sum(101) + res[102] = fma52hi(res[102], a[49], a[52]); // Sum(101) + res[102] = fma52lo(res[102], a[50], a[52]); // Sum(102) + res[103] = fma52hi(res[103], a[50], a[52]); // Sum(102) + res[103] = fma52lo(res[103], a[51], a[52]); // Sum(103) + res[104] = fma52hi(res[104], a[51], a[52]); // Sum(103) + res[96] = fma52lo(res[96], a[43], a[53]); // Sum(96) + res[97] = fma52hi(res[97], a[43], a[53]); // Sum(96) + res[97] = fma52lo(res[97], a[44], a[53]); // Sum(97) + res[98] = fma52hi(res[98], a[44], a[53]); // Sum(97) + res[98] = fma52lo(res[98], a[45], a[53]); // Sum(98) + res[99] = fma52hi(res[99], a[45], a[53]); // Sum(98) + res[99] = fma52lo(res[99], a[46], a[53]); // Sum(99) + res[100] = fma52hi(res[100], a[46], a[53]); // Sum(99) + res[100] = fma52lo(res[100], a[47], a[53]); // Sum(100) + res[101] = fma52hi(res[101], a[47], a[53]); // Sum(100) + res[101] = fma52lo(res[101], a[48], a[53]); // Sum(101) + res[102] = fma52hi(res[102], a[48], a[53]); // Sum(101) + res[102] = fma52lo(res[102], a[49], a[53]); // Sum(102) + res[103] = fma52hi(res[103], a[49], a[53]); // Sum(102) + res[103] = fma52lo(res[103], a[50], a[53]); // Sum(103) + res[104] = fma52hi(res[104], a[50], a[53]); // Sum(103) + res[104] = fma52lo(res[104], a[51], a[53]); // Sum(104) + res[105] = fma52hi(res[105], a[51], a[53]); // Sum(104) + res[105] = fma52lo(res[105], a[52], a[53]); // Sum(105) + res[106] = fma52hi(res[106], a[52], a[53]); // Sum(105) + res[96] = fma52lo(res[96], a[42], a[54]); // Sum(96) + res[97] = fma52hi(res[97], a[42], a[54]); // Sum(96) + res[97] = fma52lo(res[97], a[43], a[54]); // Sum(97) + res[98] = fma52hi(res[98], a[43], a[54]); // Sum(97) + res[98] = fma52lo(res[98], a[44], a[54]); // Sum(98) + res[99] = fma52hi(res[99], a[44], a[54]); // Sum(98) + res[99] = fma52lo(res[99], a[45], a[54]); // Sum(99) + res[100] = fma52hi(res[100], a[45], a[54]); // Sum(99) + res[100] = fma52lo(res[100], a[46], a[54]); // Sum(100) + res[101] = fma52hi(res[101], a[46], a[54]); // Sum(100) + res[101] = fma52lo(res[101], a[47], a[54]); // Sum(101) + res[102] = fma52hi(res[102], a[47], a[54]); // Sum(101) + res[102] = fma52lo(res[102], a[48], a[54]); // Sum(102) + res[103] = fma52hi(res[103], a[48], a[54]); // Sum(102) + res[103] = fma52lo(res[103], a[49], a[54]); // Sum(103) + res[104] = fma52hi(res[104], a[49], a[54]); // Sum(103) + res[104] = fma52lo(res[104], a[50], a[54]); // Sum(104) + res[105] = fma52hi(res[105], a[50], a[54]); // Sum(104) + res[105] = fma52lo(res[105], a[51], a[54]); // Sum(105) + res[106] = fma52hi(res[106], a[51], a[54]); // Sum(105) + res[106] = fma52lo(res[106], a[52], a[54]); // Sum(106) + res[107] = fma52hi(res[107], a[52], a[54]); // Sum(106) + res[107] = fma52lo(res[107], a[53], a[54]); // Sum(107) + res[108] = fma52hi(res[108], a[53], a[54]); // Sum(107) + res[96] = fma52lo(res[96], a[41], a[55]); // Sum(96) + res[97] = fma52hi(res[97], a[41], a[55]); // Sum(96) + res[97] = fma52lo(res[97], a[42], a[55]); // Sum(97) + res[98] = fma52hi(res[98], a[42], a[55]); // Sum(97) + res[98] = fma52lo(res[98], a[43], a[55]); // Sum(98) + res[99] = fma52hi(res[99], a[43], a[55]); // Sum(98) + res[99] = fma52lo(res[99], a[44], a[55]); // Sum(99) + res[100] = fma52hi(res[100], a[44], a[55]); // Sum(99) + res[100] = fma52lo(res[100], a[45], a[55]); // Sum(100) + res[101] = fma52hi(res[101], a[45], a[55]); // Sum(100) + res[101] = fma52lo(res[101], a[46], a[55]); // Sum(101) + res[102] = fma52hi(res[102], a[46], a[55]); // Sum(101) + res[102] = fma52lo(res[102], a[47], a[55]); // Sum(102) + res[103] = fma52hi(res[103], a[47], a[55]); // Sum(102) + res[103] = fma52lo(res[103], a[48], a[55]); // Sum(103) + res[104] = fma52hi(res[104], a[48], a[55]); // Sum(103) + res[104] = fma52lo(res[104], a[49], a[55]); // Sum(104) + res[105] = fma52hi(res[105], a[49], a[55]); // Sum(104) + res[105] = fma52lo(res[105], a[50], a[55]); // Sum(105) + res[106] = fma52hi(res[106], a[50], a[55]); // Sum(105) + res[106] = fma52lo(res[106], a[51], a[55]); // Sum(106) + res[107] = fma52hi(res[107], a[51], a[55]); // Sum(106) + res[107] = fma52lo(res[107], a[52], a[55]); // Sum(107) + res[108] = fma52hi(res[108], a[52], a[55]); // Sum(107) + res[96] = fma52lo(res[96], a[40], a[56]); // Sum(96) + res[97] = fma52hi(res[97], a[40], a[56]); // Sum(96) + res[97] = fma52lo(res[97], a[41], a[56]); // Sum(97) + res[98] = fma52hi(res[98], a[41], a[56]); // Sum(97) + res[98] = fma52lo(res[98], a[42], a[56]); // Sum(98) + res[99] = fma52hi(res[99], a[42], a[56]); // Sum(98) + res[99] = fma52lo(res[99], a[43], a[56]); // Sum(99) + res[100] = fma52hi(res[100], a[43], a[56]); // Sum(99) + res[100] = fma52lo(res[100], a[44], a[56]); // Sum(100) + res[101] = fma52hi(res[101], a[44], a[56]); // Sum(100) + res[101] = fma52lo(res[101], a[45], a[56]); // Sum(101) + res[102] = fma52hi(res[102], a[45], a[56]); // Sum(101) + res[102] = fma52lo(res[102], a[46], a[56]); // Sum(102) + res[103] = fma52hi(res[103], a[46], a[56]); // Sum(102) + res[103] = fma52lo(res[103], a[47], a[56]); // Sum(103) + res[104] = fma52hi(res[104], a[47], a[56]); // Sum(103) + res[104] = fma52lo(res[104], a[48], a[56]); // Sum(104) + res[105] = fma52hi(res[105], a[48], a[56]); // Sum(104) + res[105] = fma52lo(res[105], a[49], a[56]); // Sum(105) + res[106] = fma52hi(res[106], a[49], a[56]); // Sum(105) + res[106] = fma52lo(res[106], a[50], a[56]); // Sum(106) + res[107] = fma52hi(res[107], a[50], a[56]); // Sum(106) + res[107] = fma52lo(res[107], a[51], a[56]); // Sum(107) + res[108] = fma52hi(res[108], a[51], a[56]); // Sum(107) + res[96] = fma52lo(res[96], a[39], a[57]); // Sum(96) + res[97] = fma52hi(res[97], a[39], a[57]); // Sum(96) + res[97] = fma52lo(res[97], a[40], a[57]); // Sum(97) + res[98] = fma52hi(res[98], a[40], a[57]); // Sum(97) + res[98] = fma52lo(res[98], a[41], a[57]); // Sum(98) + res[99] = fma52hi(res[99], a[41], a[57]); // Sum(98) + res[99] = fma52lo(res[99], a[42], a[57]); // Sum(99) + res[100] = fma52hi(res[100], a[42], a[57]); // Sum(99) + res[100] = fma52lo(res[100], a[43], a[57]); // Sum(100) + res[101] = fma52hi(res[101], a[43], a[57]); // Sum(100) + res[101] = fma52lo(res[101], a[44], a[57]); // Sum(101) + res[102] = fma52hi(res[102], a[44], a[57]); // Sum(101) + res[102] = fma52lo(res[102], a[45], a[57]); // Sum(102) + res[103] = fma52hi(res[103], a[45], a[57]); // Sum(102) + res[103] = fma52lo(res[103], a[46], a[57]); // Sum(103) + res[104] = fma52hi(res[104], a[46], a[57]); // Sum(103) + res[104] = fma52lo(res[104], a[47], a[57]); // Sum(104) + res[105] = fma52hi(res[105], a[47], a[57]); // Sum(104) + res[105] = fma52lo(res[105], a[48], a[57]); // Sum(105) + res[106] = fma52hi(res[106], a[48], a[57]); // Sum(105) + res[106] = fma52lo(res[106], a[49], a[57]); // Sum(106) + res[107] = fma52hi(res[107], a[49], a[57]); // Sum(106) + res[107] = fma52lo(res[107], a[50], a[57]); // Sum(107) + res[108] = fma52hi(res[108], a[50], a[57]); // Sum(107) + res[96] = fma52lo(res[96], a[38], a[58]); // Sum(96) + res[97] = fma52hi(res[97], a[38], a[58]); // Sum(96) + res[97] = fma52lo(res[97], a[39], a[58]); // Sum(97) + res[98] = fma52hi(res[98], a[39], a[58]); // Sum(97) + res[98] = fma52lo(res[98], a[40], a[58]); // Sum(98) + res[99] = fma52hi(res[99], a[40], a[58]); // Sum(98) + res[99] = fma52lo(res[99], a[41], a[58]); // Sum(99) + res[100] = fma52hi(res[100], a[41], a[58]); // Sum(99) + res[100] = fma52lo(res[100], a[42], a[58]); // Sum(100) + res[101] = fma52hi(res[101], a[42], a[58]); // Sum(100) + res[101] = fma52lo(res[101], a[43], a[58]); // Sum(101) + res[102] = fma52hi(res[102], a[43], a[58]); // Sum(101) + res[102] = fma52lo(res[102], a[44], a[58]); // Sum(102) + res[103] = fma52hi(res[103], a[44], a[58]); // Sum(102) + res[103] = fma52lo(res[103], a[45], a[58]); // Sum(103) + res[104] = fma52hi(res[104], a[45], a[58]); // Sum(103) + res[104] = fma52lo(res[104], a[46], a[58]); // Sum(104) + res[105] = fma52hi(res[105], a[46], a[58]); // Sum(104) + res[105] = fma52lo(res[105], a[47], a[58]); // Sum(105) + res[106] = fma52hi(res[106], a[47], a[58]); // Sum(105) + res[106] = fma52lo(res[106], a[48], a[58]); // Sum(106) + res[107] = fma52hi(res[107], a[48], a[58]); // Sum(106) + res[107] = fma52lo(res[107], a[49], a[58]); // Sum(107) + res[108] = fma52hi(res[108], a[49], a[58]); // Sum(107) + res[96] = fma52lo(res[96], a[37], a[59]); // Sum(96) + res[97] = fma52hi(res[97], a[37], a[59]); // Sum(96) + res[97] = fma52lo(res[97], a[38], a[59]); // Sum(97) + res[98] = fma52hi(res[98], a[38], a[59]); // Sum(97) + res[98] = fma52lo(res[98], a[39], a[59]); // Sum(98) + res[99] = fma52hi(res[99], a[39], a[59]); // Sum(98) + res[99] = fma52lo(res[99], a[40], a[59]); // Sum(99) + res[100] = fma52hi(res[100], a[40], a[59]); // Sum(99) + res[100] = fma52lo(res[100], a[41], a[59]); // Sum(100) + res[101] = fma52hi(res[101], a[41], a[59]); // Sum(100) + res[101] = fma52lo(res[101], a[42], a[59]); // Sum(101) + res[102] = fma52hi(res[102], a[42], a[59]); // Sum(101) + res[102] = fma52lo(res[102], a[43], a[59]); // Sum(102) + res[103] = fma52hi(res[103], a[43], a[59]); // Sum(102) + res[103] = fma52lo(res[103], a[44], a[59]); // Sum(103) + res[104] = fma52hi(res[104], a[44], a[59]); // Sum(103) + res[104] = fma52lo(res[104], a[45], a[59]); // Sum(104) + res[105] = fma52hi(res[105], a[45], a[59]); // Sum(104) + res[105] = fma52lo(res[105], a[46], a[59]); // Sum(105) + res[106] = fma52hi(res[106], a[46], a[59]); // Sum(105) + res[106] = fma52lo(res[106], a[47], a[59]); // Sum(106) + res[107] = fma52hi(res[107], a[47], a[59]); // Sum(106) + res[107] = fma52lo(res[107], a[48], a[59]); // Sum(107) + res[108] = fma52hi(res[108], a[48], a[59]); // Sum(107) + res[96] = fma52lo(res[96], a[36], a[60]); // Sum(96) + res[97] = fma52hi(res[97], a[36], a[60]); // Sum(96) + res[97] = fma52lo(res[97], a[37], a[60]); // Sum(97) + res[98] = fma52hi(res[98], a[37], a[60]); // Sum(97) + res[98] = fma52lo(res[98], a[38], a[60]); // Sum(98) + res[99] = fma52hi(res[99], a[38], a[60]); // Sum(98) + res[99] = fma52lo(res[99], a[39], a[60]); // Sum(99) + res[100] = fma52hi(res[100], a[39], a[60]); // Sum(99) + res[100] = fma52lo(res[100], a[40], a[60]); // Sum(100) + res[101] = fma52hi(res[101], a[40], a[60]); // Sum(100) + res[101] = fma52lo(res[101], a[41], a[60]); // Sum(101) + res[102] = fma52hi(res[102], a[41], a[60]); // Sum(101) + res[102] = fma52lo(res[102], a[42], a[60]); // Sum(102) + res[103] = fma52hi(res[103], a[42], a[60]); // Sum(102) + res[103] = fma52lo(res[103], a[43], a[60]); // Sum(103) + res[104] = fma52hi(res[104], a[43], a[60]); // Sum(103) + res[104] = fma52lo(res[104], a[44], a[60]); // Sum(104) + res[105] = fma52hi(res[105], a[44], a[60]); // Sum(104) + res[105] = fma52lo(res[105], a[45], a[60]); // Sum(105) + res[106] = fma52hi(res[106], a[45], a[60]); // Sum(105) + res[106] = fma52lo(res[106], a[46], a[60]); // Sum(106) + res[107] = fma52hi(res[107], a[46], a[60]); // Sum(106) + res[107] = fma52lo(res[107], a[47], a[60]); // Sum(107) + res[108] = fma52hi(res[108], a[47], a[60]); // Sum(107) + res[96] = fma52lo(res[96], a[35], a[61]); // Sum(96) + res[97] = fma52hi(res[97], a[35], a[61]); // Sum(96) + res[97] = fma52lo(res[97], a[36], a[61]); // Sum(97) + res[98] = fma52hi(res[98], a[36], a[61]); // Sum(97) + res[98] = fma52lo(res[98], a[37], a[61]); // Sum(98) + res[99] = fma52hi(res[99], a[37], a[61]); // Sum(98) + res[99] = fma52lo(res[99], a[38], a[61]); // Sum(99) + res[100] = fma52hi(res[100], a[38], a[61]); // Sum(99) + res[100] = fma52lo(res[100], a[39], a[61]); // Sum(100) + res[101] = fma52hi(res[101], a[39], a[61]); // Sum(100) + res[101] = fma52lo(res[101], a[40], a[61]); // Sum(101) + res[102] = fma52hi(res[102], a[40], a[61]); // Sum(101) + res[102] = fma52lo(res[102], a[41], a[61]); // Sum(102) + res[103] = fma52hi(res[103], a[41], a[61]); // Sum(102) + res[103] = fma52lo(res[103], a[42], a[61]); // Sum(103) + res[104] = fma52hi(res[104], a[42], a[61]); // Sum(103) + res[104] = fma52lo(res[104], a[43], a[61]); // Sum(104) + res[105] = fma52hi(res[105], a[43], a[61]); // Sum(104) + res[105] = fma52lo(res[105], a[44], a[61]); // Sum(105) + res[106] = fma52hi(res[106], a[44], a[61]); // Sum(105) + res[106] = fma52lo(res[106], a[45], a[61]); // Sum(106) + res[107] = fma52hi(res[107], a[45], a[61]); // Sum(106) + res[107] = fma52lo(res[107], a[46], a[61]); // Sum(107) + res[108] = fma52hi(res[108], a[46], a[61]); // Sum(107) + res[96] = fma52lo(res[96], a[34], a[62]); // Sum(96) + res[97] = fma52hi(res[97], a[34], a[62]); // Sum(96) + res[97] = fma52lo(res[97], a[35], a[62]); // Sum(97) + res[98] = fma52hi(res[98], a[35], a[62]); // Sum(97) + res[98] = fma52lo(res[98], a[36], a[62]); // Sum(98) + res[99] = fma52hi(res[99], a[36], a[62]); // Sum(98) + res[99] = fma52lo(res[99], a[37], a[62]); // Sum(99) + res[100] = fma52hi(res[100], a[37], a[62]); // Sum(99) + res[100] = fma52lo(res[100], a[38], a[62]); // Sum(100) + res[101] = fma52hi(res[101], a[38], a[62]); // Sum(100) + res[101] = fma52lo(res[101], a[39], a[62]); // Sum(101) + res[102] = fma52hi(res[102], a[39], a[62]); // Sum(101) + res[102] = fma52lo(res[102], a[40], a[62]); // Sum(102) + res[103] = fma52hi(res[103], a[40], a[62]); // Sum(102) + res[103] = fma52lo(res[103], a[41], a[62]); // Sum(103) + res[104] = fma52hi(res[104], a[41], a[62]); // Sum(103) + res[104] = fma52lo(res[104], a[42], a[62]); // Sum(104) + res[105] = fma52hi(res[105], a[42], a[62]); // Sum(104) + res[105] = fma52lo(res[105], a[43], a[62]); // Sum(105) + res[106] = fma52hi(res[106], a[43], a[62]); // Sum(105) + res[106] = fma52lo(res[106], a[44], a[62]); // Sum(106) + res[107] = fma52hi(res[107], a[44], a[62]); // Sum(106) + res[107] = fma52lo(res[107], a[45], a[62]); // Sum(107) + res[108] = fma52hi(res[108], a[45], a[62]); // Sum(107) + res[96] = fma52lo(res[96], a[33], a[63]); // Sum(96) + res[97] = fma52hi(res[97], a[33], a[63]); // Sum(96) + res[97] = fma52lo(res[97], a[34], a[63]); // Sum(97) + res[98] = fma52hi(res[98], a[34], a[63]); // Sum(97) + res[98] = fma52lo(res[98], a[35], a[63]); // Sum(98) + res[99] = fma52hi(res[99], a[35], a[63]); // Sum(98) + res[99] = fma52lo(res[99], a[36], a[63]); // Sum(99) + res[100] = fma52hi(res[100], a[36], a[63]); // Sum(99) + res[100] = fma52lo(res[100], a[37], a[63]); // Sum(100) + res[101] = fma52hi(res[101], a[37], a[63]); // Sum(100) + res[101] = fma52lo(res[101], a[38], a[63]); // Sum(101) + res[102] = fma52hi(res[102], a[38], a[63]); // Sum(101) + res[102] = fma52lo(res[102], a[39], a[63]); // Sum(102) + res[103] = fma52hi(res[103], a[39], a[63]); // Sum(102) + res[103] = fma52lo(res[103], a[40], a[63]); // Sum(103) + res[104] = fma52hi(res[104], a[40], a[63]); // Sum(103) + res[104] = fma52lo(res[104], a[41], a[63]); // Sum(104) + res[105] = fma52hi(res[105], a[41], a[63]); // Sum(104) + res[105] = fma52lo(res[105], a[42], a[63]); // Sum(105) + res[106] = fma52hi(res[106], a[42], a[63]); // Sum(105) + res[106] = fma52lo(res[106], a[43], a[63]); // Sum(106) + res[107] = fma52hi(res[107], a[43], a[63]); // Sum(106) + res[107] = fma52lo(res[107], a[44], a[63]); // Sum(107) + res[108] = fma52hi(res[108], a[44], a[63]); // Sum(107) + res[96] = fma52lo(res[96], a[32], a[64]); // Sum(96) + res[97] = fma52hi(res[97], a[32], a[64]); // Sum(96) + res[97] = fma52lo(res[97], a[33], a[64]); // Sum(97) + res[98] = fma52hi(res[98], a[33], a[64]); // Sum(97) + res[98] = fma52lo(res[98], a[34], a[64]); // Sum(98) + res[99] = fma52hi(res[99], a[34], a[64]); // Sum(98) + res[99] = fma52lo(res[99], a[35], a[64]); // Sum(99) + res[100] = fma52hi(res[100], a[35], a[64]); // Sum(99) + res[100] = fma52lo(res[100], a[36], a[64]); // Sum(100) + res[101] = fma52hi(res[101], a[36], a[64]); // Sum(100) + res[101] = fma52lo(res[101], a[37], a[64]); // Sum(101) + res[102] = fma52hi(res[102], a[37], a[64]); // Sum(101) + res[102] = fma52lo(res[102], a[38], a[64]); // Sum(102) + res[103] = fma52hi(res[103], a[38], a[64]); // Sum(102) + res[103] = fma52lo(res[103], a[39], a[64]); // Sum(103) + res[104] = fma52hi(res[104], a[39], a[64]); // Sum(103) + res[104] = fma52lo(res[104], a[40], a[64]); // Sum(104) + res[105] = fma52hi(res[105], a[40], a[64]); // Sum(104) + res[105] = fma52lo(res[105], a[41], a[64]); // Sum(105) + res[106] = fma52hi(res[106], a[41], a[64]); // Sum(105) + res[106] = fma52lo(res[106], a[42], a[64]); // Sum(106) + res[107] = fma52hi(res[107], a[42], a[64]); // Sum(106) + res[107] = fma52lo(res[107], a[43], a[64]); // Sum(107) + res[108] = fma52hi(res[108], a[43], a[64]); // Sum(107) + res[96] = fma52lo(res[96], a[31], a[65]); // Sum(96) + res[97] = fma52hi(res[97], a[31], a[65]); // Sum(96) + res[97] = fma52lo(res[97], a[32], a[65]); // Sum(97) + res[98] = fma52hi(res[98], a[32], a[65]); // Sum(97) + res[98] = fma52lo(res[98], a[33], a[65]); // Sum(98) + res[99] = fma52hi(res[99], a[33], a[65]); // Sum(98) + res[99] = fma52lo(res[99], a[34], a[65]); // Sum(99) + res[100] = fma52hi(res[100], a[34], a[65]); // Sum(99) + res[100] = fma52lo(res[100], a[35], a[65]); // Sum(100) + res[101] = fma52hi(res[101], a[35], a[65]); // Sum(100) + res[101] = fma52lo(res[101], a[36], a[65]); // Sum(101) + res[102] = fma52hi(res[102], a[36], a[65]); // Sum(101) + res[102] = fma52lo(res[102], a[37], a[65]); // Sum(102) + res[103] = fma52hi(res[103], a[37], a[65]); // Sum(102) + res[103] = fma52lo(res[103], a[38], a[65]); // Sum(103) + res[104] = fma52hi(res[104], a[38], a[65]); // Sum(103) + res[104] = fma52lo(res[104], a[39], a[65]); // Sum(104) + res[105] = fma52hi(res[105], a[39], a[65]); // Sum(104) + res[105] = fma52lo(res[105], a[40], a[65]); // Sum(105) + res[106] = fma52hi(res[106], a[40], a[65]); // Sum(105) + res[106] = fma52lo(res[106], a[41], a[65]); // Sum(106) + res[107] = fma52hi(res[107], a[41], a[65]); // Sum(106) + res[107] = fma52lo(res[107], a[42], a[65]); // Sum(107) + res[108] = fma52hi(res[108], a[42], a[65]); // Sum(107) + res[96] = fma52lo(res[96], a[30], a[66]); // Sum(96) + res[97] = fma52hi(res[97], a[30], a[66]); // Sum(96) + res[97] = fma52lo(res[97], a[31], a[66]); // Sum(97) + res[98] = fma52hi(res[98], a[31], a[66]); // Sum(97) + res[98] = fma52lo(res[98], a[32], a[66]); // Sum(98) + res[99] = fma52hi(res[99], a[32], a[66]); // Sum(98) + res[99] = fma52lo(res[99], a[33], a[66]); // Sum(99) + res[100] = fma52hi(res[100], a[33], a[66]); // Sum(99) + res[100] = fma52lo(res[100], a[34], a[66]); // Sum(100) + res[101] = fma52hi(res[101], a[34], a[66]); // Sum(100) + res[101] = fma52lo(res[101], a[35], a[66]); // Sum(101) + res[102] = fma52hi(res[102], a[35], a[66]); // Sum(101) + res[102] = fma52lo(res[102], a[36], a[66]); // Sum(102) + res[103] = fma52hi(res[103], a[36], a[66]); // Sum(102) + res[103] = fma52lo(res[103], a[37], a[66]); // Sum(103) + res[104] = fma52hi(res[104], a[37], a[66]); // Sum(103) + res[104] = fma52lo(res[104], a[38], a[66]); // Sum(104) + res[105] = fma52hi(res[105], a[38], a[66]); // Sum(104) + res[105] = fma52lo(res[105], a[39], a[66]); // Sum(105) + res[106] = fma52hi(res[106], a[39], a[66]); // Sum(105) + res[106] = fma52lo(res[106], a[40], a[66]); // Sum(106) + res[107] = fma52hi(res[107], a[40], a[66]); // Sum(106) + res[107] = fma52lo(res[107], a[41], a[66]); // Sum(107) + res[108] = fma52hi(res[108], a[41], a[66]); // Sum(107) + res[96] = fma52lo(res[96], a[29], a[67]); // Sum(96) + res[97] = fma52hi(res[97], a[29], a[67]); // Sum(96) + res[97] = fma52lo(res[97], a[30], a[67]); // Sum(97) + res[98] = fma52hi(res[98], a[30], a[67]); // Sum(97) + res[98] = fma52lo(res[98], a[31], a[67]); // Sum(98) + res[99] = fma52hi(res[99], a[31], a[67]); // Sum(98) + res[99] = fma52lo(res[99], a[32], a[67]); // Sum(99) + res[100] = fma52hi(res[100], a[32], a[67]); // Sum(99) + res[100] = fma52lo(res[100], a[33], a[67]); // Sum(100) + res[101] = fma52hi(res[101], a[33], a[67]); // Sum(100) + res[101] = fma52lo(res[101], a[34], a[67]); // Sum(101) + res[102] = fma52hi(res[102], a[34], a[67]); // Sum(101) + res[102] = fma52lo(res[102], a[35], a[67]); // Sum(102) + res[103] = fma52hi(res[103], a[35], a[67]); // Sum(102) + res[103] = fma52lo(res[103], a[36], a[67]); // Sum(103) + res[104] = fma52hi(res[104], a[36], a[67]); // Sum(103) + res[104] = fma52lo(res[104], a[37], a[67]); // Sum(104) + res[105] = fma52hi(res[105], a[37], a[67]); // Sum(104) + res[105] = fma52lo(res[105], a[38], a[67]); // Sum(105) + res[106] = fma52hi(res[106], a[38], a[67]); // Sum(105) + res[106] = fma52lo(res[106], a[39], a[67]); // Sum(106) + res[107] = fma52hi(res[107], a[39], a[67]); // Sum(106) + res[107] = fma52lo(res[107], a[40], a[67]); // Sum(107) + res[108] = fma52hi(res[108], a[40], a[67]); // Sum(107) + res[96] = fma52lo(res[96], a[28], a[68]); // Sum(96) + res[97] = fma52hi(res[97], a[28], a[68]); // Sum(96) + res[97] = fma52lo(res[97], a[29], a[68]); // Sum(97) + res[98] = fma52hi(res[98], a[29], a[68]); // Sum(97) + res[98] = fma52lo(res[98], a[30], a[68]); // Sum(98) + res[99] = fma52hi(res[99], a[30], a[68]); // Sum(98) + res[99] = fma52lo(res[99], a[31], a[68]); // Sum(99) + res[100] = fma52hi(res[100], a[31], a[68]); // Sum(99) + res[100] = fma52lo(res[100], a[32], a[68]); // Sum(100) + res[101] = fma52hi(res[101], a[32], a[68]); // Sum(100) + res[101] = fma52lo(res[101], a[33], a[68]); // Sum(101) + res[102] = fma52hi(res[102], a[33], a[68]); // Sum(101) + res[102] = fma52lo(res[102], a[34], a[68]); // Sum(102) + res[103] = fma52hi(res[103], a[34], a[68]); // Sum(102) + res[103] = fma52lo(res[103], a[35], a[68]); // Sum(103) + res[104] = fma52hi(res[104], a[35], a[68]); // Sum(103) + res[104] = fma52lo(res[104], a[36], a[68]); // Sum(104) + res[105] = fma52hi(res[105], a[36], a[68]); // Sum(104) + res[105] = fma52lo(res[105], a[37], a[68]); // Sum(105) + res[106] = fma52hi(res[106], a[37], a[68]); // Sum(105) + res[106] = fma52lo(res[106], a[38], a[68]); // Sum(106) + res[107] = fma52hi(res[107], a[38], a[68]); // Sum(106) + res[107] = fma52lo(res[107], a[39], a[68]); // Sum(107) + res[108] = fma52hi(res[108], a[39], a[68]); // Sum(107) + res[96] = fma52lo(res[96], a[27], a[69]); // Sum(96) + res[97] = fma52hi(res[97], a[27], a[69]); // Sum(96) + res[97] = fma52lo(res[97], a[28], a[69]); // Sum(97) + res[98] = fma52hi(res[98], a[28], a[69]); // Sum(97) + res[98] = fma52lo(res[98], a[29], a[69]); // Sum(98) + res[99] = fma52hi(res[99], a[29], a[69]); // Sum(98) + res[99] = fma52lo(res[99], a[30], a[69]); // Sum(99) + res[100] = fma52hi(res[100], a[30], a[69]); // Sum(99) + res[100] = fma52lo(res[100], a[31], a[69]); // Sum(100) + res[101] = fma52hi(res[101], a[31], a[69]); // Sum(100) + res[101] = fma52lo(res[101], a[32], a[69]); // Sum(101) + res[102] = fma52hi(res[102], a[32], a[69]); // Sum(101) + res[102] = fma52lo(res[102], a[33], a[69]); // Sum(102) + res[103] = fma52hi(res[103], a[33], a[69]); // Sum(102) + res[103] = fma52lo(res[103], a[34], a[69]); // Sum(103) + res[104] = fma52hi(res[104], a[34], a[69]); // Sum(103) + res[104] = fma52lo(res[104], a[35], a[69]); // Sum(104) + res[105] = fma52hi(res[105], a[35], a[69]); // Sum(104) + res[105] = fma52lo(res[105], a[36], a[69]); // Sum(105) + res[106] = fma52hi(res[106], a[36], a[69]); // Sum(105) + res[106] = fma52lo(res[106], a[37], a[69]); // Sum(106) + res[107] = fma52hi(res[107], a[37], a[69]); // Sum(106) + res[107] = fma52lo(res[107], a[38], a[69]); // Sum(107) + res[108] = fma52hi(res[108], a[38], a[69]); // Sum(107) + res[96] = fma52lo(res[96], a[26], a[70]); // Sum(96) + res[97] = fma52hi(res[97], a[26], a[70]); // Sum(96) + res[97] = fma52lo(res[97], a[27], a[70]); // Sum(97) + res[98] = fma52hi(res[98], a[27], a[70]); // Sum(97) + res[98] = fma52lo(res[98], a[28], a[70]); // Sum(98) + res[99] = fma52hi(res[99], a[28], a[70]); // Sum(98) + res[99] = fma52lo(res[99], a[29], a[70]); // Sum(99) + res[100] = fma52hi(res[100], a[29], a[70]); // Sum(99) + res[100] = fma52lo(res[100], a[30], a[70]); // Sum(100) + res[101] = fma52hi(res[101], a[30], a[70]); // Sum(100) + res[101] = fma52lo(res[101], a[31], a[70]); // Sum(101) + res[102] = fma52hi(res[102], a[31], a[70]); // Sum(101) + res[102] = fma52lo(res[102], a[32], a[70]); // Sum(102) + res[103] = fma52hi(res[103], a[32], a[70]); // Sum(102) + res[103] = fma52lo(res[103], a[33], a[70]); // Sum(103) + res[104] = fma52hi(res[104], a[33], a[70]); // Sum(103) + res[104] = fma52lo(res[104], a[34], a[70]); // Sum(104) + res[105] = fma52hi(res[105], a[34], a[70]); // Sum(104) + res[105] = fma52lo(res[105], a[35], a[70]); // Sum(105) + res[106] = fma52hi(res[106], a[35], a[70]); // Sum(105) + res[106] = fma52lo(res[106], a[36], a[70]); // Sum(106) + res[107] = fma52hi(res[107], a[36], a[70]); // Sum(106) + res[107] = fma52lo(res[107], a[37], a[70]); // Sum(107) + res[108] = fma52hi(res[108], a[37], a[70]); // Sum(107) + res[96] = fma52lo(res[96], a[25], a[71]); // Sum(96) + res[97] = fma52hi(res[97], a[25], a[71]); // Sum(96) + res[97] = fma52lo(res[97], a[26], a[71]); // Sum(97) + res[98] = fma52hi(res[98], a[26], a[71]); // Sum(97) + res[98] = fma52lo(res[98], a[27], a[71]); // Sum(98) + res[99] = fma52hi(res[99], a[27], a[71]); // Sum(98) + res[99] = fma52lo(res[99], a[28], a[71]); // Sum(99) + res[100] = fma52hi(res[100], a[28], a[71]); // Sum(99) + res[100] = fma52lo(res[100], a[29], a[71]); // Sum(100) + res[101] = fma52hi(res[101], a[29], a[71]); // Sum(100) + res[101] = fma52lo(res[101], a[30], a[71]); // Sum(101) + res[102] = fma52hi(res[102], a[30], a[71]); // Sum(101) + res[102] = fma52lo(res[102], a[31], a[71]); // Sum(102) + res[103] = fma52hi(res[103], a[31], a[71]); // Sum(102) + res[103] = fma52lo(res[103], a[32], a[71]); // Sum(103) + res[104] = fma52hi(res[104], a[32], a[71]); // Sum(103) + res[104] = fma52lo(res[104], a[33], a[71]); // Sum(104) + res[105] = fma52hi(res[105], a[33], a[71]); // Sum(104) + res[105] = fma52lo(res[105], a[34], a[71]); // Sum(105) + res[106] = fma52hi(res[106], a[34], a[71]); // Sum(105) + res[106] = fma52lo(res[106], a[35], a[71]); // Sum(106) + res[107] = fma52hi(res[107], a[35], a[71]); // Sum(106) + res[107] = fma52lo(res[107], a[36], a[71]); // Sum(107) + res[108] = fma52hi(res[108], a[36], a[71]); // Sum(107) + res[96] = fma52lo(res[96], a[24], a[72]); // Sum(96) + res[97] = fma52hi(res[97], a[24], a[72]); // Sum(96) + res[97] = fma52lo(res[97], a[25], a[72]); // Sum(97) + res[98] = fma52hi(res[98], a[25], a[72]); // Sum(97) + res[98] = fma52lo(res[98], a[26], a[72]); // Sum(98) + res[99] = fma52hi(res[99], a[26], a[72]); // Sum(98) + res[99] = fma52lo(res[99], a[27], a[72]); // Sum(99) + res[100] = fma52hi(res[100], a[27], a[72]); // Sum(99) + res[100] = fma52lo(res[100], a[28], a[72]); // Sum(100) + res[101] = fma52hi(res[101], a[28], a[72]); // Sum(100) + res[101] = fma52lo(res[101], a[29], a[72]); // Sum(101) + res[102] = fma52hi(res[102], a[29], a[72]); // Sum(101) + res[102] = fma52lo(res[102], a[30], a[72]); // Sum(102) + res[103] = fma52hi(res[103], a[30], a[72]); // Sum(102) + res[103] = fma52lo(res[103], a[31], a[72]); // Sum(103) + res[104] = fma52hi(res[104], a[31], a[72]); // Sum(103) + res[104] = fma52lo(res[104], a[32], a[72]); // Sum(104) + res[105] = fma52hi(res[105], a[32], a[72]); // Sum(104) + res[105] = fma52lo(res[105], a[33], a[72]); // Sum(105) + res[106] = fma52hi(res[106], a[33], a[72]); // Sum(105) + res[106] = fma52lo(res[106], a[34], a[72]); // Sum(106) + res[107] = fma52hi(res[107], a[34], a[72]); // Sum(106) + res[107] = fma52lo(res[107], a[35], a[72]); // Sum(107) + res[108] = fma52hi(res[108], a[35], a[72]); // Sum(107) + res[96] = fma52lo(res[96], a[23], a[73]); // Sum(96) + res[97] = fma52hi(res[97], a[23], a[73]); // Sum(96) + res[97] = fma52lo(res[97], a[24], a[73]); // Sum(97) + res[98] = fma52hi(res[98], a[24], a[73]); // Sum(97) + res[98] = fma52lo(res[98], a[25], a[73]); // Sum(98) + res[99] = fma52hi(res[99], a[25], a[73]); // Sum(98) + res[99] = fma52lo(res[99], a[26], a[73]); // Sum(99) + res[100] = fma52hi(res[100], a[26], a[73]); // Sum(99) + res[100] = fma52lo(res[100], a[27], a[73]); // Sum(100) + res[101] = fma52hi(res[101], a[27], a[73]); // Sum(100) + res[101] = fma52lo(res[101], a[28], a[73]); // Sum(101) + res[102] = fma52hi(res[102], a[28], a[73]); // Sum(101) + res[102] = fma52lo(res[102], a[29], a[73]); // Sum(102) + res[103] = fma52hi(res[103], a[29], a[73]); // Sum(102) + res[103] = fma52lo(res[103], a[30], a[73]); // Sum(103) + res[104] = fma52hi(res[104], a[30], a[73]); // Sum(103) + res[104] = fma52lo(res[104], a[31], a[73]); // Sum(104) + res[105] = fma52hi(res[105], a[31], a[73]); // Sum(104) + res[105] = fma52lo(res[105], a[32], a[73]); // Sum(105) + res[106] = fma52hi(res[106], a[32], a[73]); // Sum(105) + res[106] = fma52lo(res[106], a[33], a[73]); // Sum(106) + res[107] = fma52hi(res[107], a[33], a[73]); // Sum(106) + res[107] = fma52lo(res[107], a[34], a[73]); // Sum(107) + res[108] = fma52hi(res[108], a[34], a[73]); // Sum(107) + res[96] = fma52lo(res[96], a[22], a[74]); // Sum(96) + res[97] = fma52hi(res[97], a[22], a[74]); // Sum(96) + res[97] = fma52lo(res[97], a[23], a[74]); // Sum(97) + res[98] = fma52hi(res[98], a[23], a[74]); // Sum(97) + res[98] = fma52lo(res[98], a[24], a[74]); // Sum(98) + res[99] = fma52hi(res[99], a[24], a[74]); // Sum(98) + res[99] = fma52lo(res[99], a[25], a[74]); // Sum(99) + res[100] = fma52hi(res[100], a[25], a[74]); // Sum(99) + res[100] = fma52lo(res[100], a[26], a[74]); // Sum(100) + res[101] = fma52hi(res[101], a[26], a[74]); // Sum(100) + res[101] = fma52lo(res[101], a[27], a[74]); // Sum(101) + res[102] = fma52hi(res[102], a[27], a[74]); // Sum(101) + res[102] = fma52lo(res[102], a[28], a[74]); // Sum(102) + res[103] = fma52hi(res[103], a[28], a[74]); // Sum(102) + res[103] = fma52lo(res[103], a[29], a[74]); // Sum(103) + res[104] = fma52hi(res[104], a[29], a[74]); // Sum(103) + res[104] = fma52lo(res[104], a[30], a[74]); // Sum(104) + res[105] = fma52hi(res[105], a[30], a[74]); // Sum(104) + res[105] = fma52lo(res[105], a[31], a[74]); // Sum(105) + res[106] = fma52hi(res[106], a[31], a[74]); // Sum(105) + res[106] = fma52lo(res[106], a[32], a[74]); // Sum(106) + res[107] = fma52hi(res[107], a[32], a[74]); // Sum(106) + res[107] = fma52lo(res[107], a[33], a[74]); // Sum(107) + res[108] = fma52hi(res[108], a[33], a[74]); // Sum(107) + res[96] = fma52lo(res[96], a[21], a[75]); // Sum(96) + res[97] = fma52hi(res[97], a[21], a[75]); // Sum(96) + res[97] = fma52lo(res[97], a[22], a[75]); // Sum(97) + res[98] = fma52hi(res[98], a[22], a[75]); // Sum(97) + res[98] = fma52lo(res[98], a[23], a[75]); // Sum(98) + res[99] = fma52hi(res[99], a[23], a[75]); // Sum(98) + res[99] = fma52lo(res[99], a[24], a[75]); // Sum(99) + res[100] = fma52hi(res[100], a[24], a[75]); // Sum(99) + res[100] = fma52lo(res[100], a[25], a[75]); // Sum(100) + res[101] = fma52hi(res[101], a[25], a[75]); // Sum(100) + res[101] = fma52lo(res[101], a[26], a[75]); // Sum(101) + res[102] = fma52hi(res[102], a[26], a[75]); // Sum(101) + res[102] = fma52lo(res[102], a[27], a[75]); // Sum(102) + res[103] = fma52hi(res[103], a[27], a[75]); // Sum(102) + res[103] = fma52lo(res[103], a[28], a[75]); // Sum(103) + res[104] = fma52hi(res[104], a[28], a[75]); // Sum(103) + res[104] = fma52lo(res[104], a[29], a[75]); // Sum(104) + res[105] = fma52hi(res[105], a[29], a[75]); // Sum(104) + res[105] = fma52lo(res[105], a[30], a[75]); // Sum(105) + res[106] = fma52hi(res[106], a[30], a[75]); // Sum(105) + res[106] = fma52lo(res[106], a[31], a[75]); // Sum(106) + res[107] = fma52hi(res[107], a[31], a[75]); // Sum(106) + res[107] = fma52lo(res[107], a[32], a[75]); // Sum(107) + res[108] = fma52hi(res[108], a[32], a[75]); // Sum(107) + res[96] = fma52lo(res[96], a[20], a[76]); // Sum(96) + res[97] = fma52hi(res[97], a[20], a[76]); // Sum(96) + res[97] = fma52lo(res[97], a[21], a[76]); // Sum(97) + res[98] = fma52hi(res[98], a[21], a[76]); // Sum(97) + res[98] = fma52lo(res[98], a[22], a[76]); // Sum(98) + res[99] = fma52hi(res[99], a[22], a[76]); // Sum(98) + res[99] = fma52lo(res[99], a[23], a[76]); // Sum(99) + res[100] = fma52hi(res[100], a[23], a[76]); // Sum(99) + res[100] = fma52lo(res[100], a[24], a[76]); // Sum(100) + res[101] = fma52hi(res[101], a[24], a[76]); // Sum(100) + res[101] = fma52lo(res[101], a[25], a[76]); // Sum(101) + res[102] = fma52hi(res[102], a[25], a[76]); // Sum(101) + res[102] = fma52lo(res[102], a[26], a[76]); // Sum(102) + res[103] = fma52hi(res[103], a[26], a[76]); // Sum(102) + res[103] = fma52lo(res[103], a[27], a[76]); // Sum(103) + res[104] = fma52hi(res[104], a[27], a[76]); // Sum(103) + res[104] = fma52lo(res[104], a[28], a[76]); // Sum(104) + res[105] = fma52hi(res[105], a[28], a[76]); // Sum(104) + res[105] = fma52lo(res[105], a[29], a[76]); // Sum(105) + res[106] = fma52hi(res[106], a[29], a[76]); // Sum(105) + res[106] = fma52lo(res[106], a[30], a[76]); // Sum(106) + res[107] = fma52hi(res[107], a[30], a[76]); // Sum(106) + res[107] = fma52lo(res[107], a[31], a[76]); // Sum(107) + res[108] = fma52hi(res[108], a[31], a[76]); // Sum(107) + res[96] = fma52lo(res[96], a[19], a[77]); // Sum(96) + res[97] = fma52hi(res[97], a[19], a[77]); // Sum(96) + res[97] = fma52lo(res[97], a[20], a[77]); // Sum(97) + res[98] = fma52hi(res[98], a[20], a[77]); // Sum(97) + res[98] = fma52lo(res[98], a[21], a[77]); // Sum(98) + res[99] = fma52hi(res[99], a[21], a[77]); // Sum(98) + res[99] = fma52lo(res[99], a[22], a[77]); // Sum(99) + res[100] = fma52hi(res[100], a[22], a[77]); // Sum(99) + res[100] = fma52lo(res[100], a[23], a[77]); // Sum(100) + res[101] = fma52hi(res[101], a[23], a[77]); // Sum(100) + res[101] = fma52lo(res[101], a[24], a[77]); // Sum(101) + res[102] = fma52hi(res[102], a[24], a[77]); // Sum(101) + res[102] = fma52lo(res[102], a[25], a[77]); // Sum(102) + res[103] = fma52hi(res[103], a[25], a[77]); // Sum(102) + res[103] = fma52lo(res[103], a[26], a[77]); // Sum(103) + res[104] = fma52hi(res[104], a[26], a[77]); // Sum(103) + res[104] = fma52lo(res[104], a[27], a[77]); // Sum(104) + res[105] = fma52hi(res[105], a[27], a[77]); // Sum(104) + res[105] = fma52lo(res[105], a[28], a[77]); // Sum(105) + res[106] = fma52hi(res[106], a[28], a[77]); // Sum(105) + res[106] = fma52lo(res[106], a[29], a[77]); // Sum(106) + res[107] = fma52hi(res[107], a[29], a[77]); // Sum(106) + res[107] = fma52lo(res[107], a[30], a[77]); // Sum(107) + res[108] = fma52hi(res[108], a[30], a[77]); // Sum(107) + res[96] = fma52lo(res[96], a[18], a[78]); // Sum(96) + res[97] = fma52hi(res[97], a[18], a[78]); // Sum(96) + res[97] = fma52lo(res[97], a[19], a[78]); // Sum(97) + res[98] = fma52hi(res[98], a[19], a[78]); // Sum(97) + res[98] = fma52lo(res[98], a[20], a[78]); // Sum(98) + res[99] = fma52hi(res[99], a[20], a[78]); // Sum(98) + res[99] = fma52lo(res[99], a[21], a[78]); // Sum(99) + res[100] = fma52hi(res[100], a[21], a[78]); // Sum(99) + res[100] = fma52lo(res[100], a[22], a[78]); // Sum(100) + res[101] = fma52hi(res[101], a[22], a[78]); // Sum(100) + res[101] = fma52lo(res[101], a[23], a[78]); // Sum(101) + res[102] = fma52hi(res[102], a[23], a[78]); // Sum(101) + res[102] = fma52lo(res[102], a[24], a[78]); // Sum(102) + res[103] = fma52hi(res[103], a[24], a[78]); // Sum(102) + res[103] = fma52lo(res[103], a[25], a[78]); // Sum(103) + res[104] = fma52hi(res[104], a[25], a[78]); // Sum(103) + res[104] = fma52lo(res[104], a[26], a[78]); // Sum(104) + res[105] = fma52hi(res[105], a[26], a[78]); // Sum(104) + res[105] = fma52lo(res[105], a[27], a[78]); // Sum(105) + res[106] = fma52hi(res[106], a[27], a[78]); // Sum(105) + res[106] = fma52lo(res[106], a[28], a[78]); // Sum(106) + res[107] = fma52hi(res[107], a[28], a[78]); // Sum(106) + res[107] = fma52lo(res[107], a[29], a[78]); // Sum(107) + res[108] = fma52hi(res[108], a[29], a[78]); // Sum(107) + res[96] = add64(res[96], res[96]); // Double(96) + res[97] = add64(res[97], res[97]); // Double(97) + res[98] = add64(res[98], res[98]); // Double(98) + res[99] = add64(res[99], res[99]); // Double(99) + res[100] = add64(res[100], res[100]); // Double(100) + res[101] = add64(res[101], res[101]); // Double(101) + res[102] = add64(res[102], res[102]); // Double(102) + res[103] = add64(res[103], res[103]); // Double(103) + res[104] = add64(res[104], res[104]); // Double(104) + res[105] = add64(res[105], res[105]); // Double(105) + res[106] = add64(res[106], res[106]); // Double(106) + res[107] = add64(res[107], res[107]); // Double(107) + res[96] = fma52lo(res[96], a[48], a[48]); // Add sqr(96) + res[97] = fma52hi(res[97], a[48], a[48]); // Add sqr(96) + res[98] = fma52lo(res[98], a[49], a[49]); // Add sqr(98) + res[99] = fma52hi(res[99], a[49], a[49]); // Add sqr(98) + res[100] = fma52lo(res[100], a[50], a[50]); // Add sqr(100) + res[101] = fma52hi(res[101], a[50], a[50]); // Add sqr(100) + res[102] = fma52lo(res[102], a[51], a[51]); // Add sqr(102) + res[103] = fma52hi(res[103], a[51], a[51]); // Add sqr(102) + res[104] = fma52lo(res[104], a[52], a[52]); // Add sqr(104) + res[105] = fma52hi(res[105], a[52], a[52]); // Add sqr(104) + res[106] = fma52lo(res[106], a[53], a[53]); // Add sqr(106) + res[107] = fma52hi(res[107], a[53], a[53]); // Add sqr(106) + res[108] = fma52lo(res[108], a[53], a[55]); // Sum(108) + res[109] = fma52hi(res[109], a[53], a[55]); // Sum(108) + res[109] = fma52lo(res[109], a[54], a[55]); // Sum(109) + res[110] = fma52hi(res[110], a[54], a[55]); // Sum(109) + res[108] = fma52lo(res[108], a[52], a[56]); // Sum(108) + res[109] = fma52hi(res[109], a[52], a[56]); // Sum(108) + res[109] = fma52lo(res[109], a[53], a[56]); // Sum(109) + res[110] = fma52hi(res[110], a[53], a[56]); // Sum(109) + res[110] = fma52lo(res[110], a[54], a[56]); // Sum(110) + res[111] = fma52hi(res[111], a[54], a[56]); // Sum(110) + res[111] = fma52lo(res[111], a[55], a[56]); // Sum(111) + res[112] = fma52hi(res[112], a[55], a[56]); // Sum(111) + res[108] = fma52lo(res[108], a[51], a[57]); // Sum(108) + res[109] = fma52hi(res[109], a[51], a[57]); // Sum(108) + res[109] = fma52lo(res[109], a[52], a[57]); // Sum(109) + res[110] = fma52hi(res[110], a[52], a[57]); // Sum(109) + res[110] = fma52lo(res[110], a[53], a[57]); // Sum(110) + res[111] = fma52hi(res[111], a[53], a[57]); // Sum(110) + res[111] = fma52lo(res[111], a[54], a[57]); // Sum(111) + res[112] = fma52hi(res[112], a[54], a[57]); // Sum(111) + res[112] = fma52lo(res[112], a[55], a[57]); // Sum(112) + res[113] = fma52hi(res[113], a[55], a[57]); // Sum(112) + res[113] = fma52lo(res[113], a[56], a[57]); // Sum(113) + res[114] = fma52hi(res[114], a[56], a[57]); // Sum(113) + res[108] = fma52lo(res[108], a[50], a[58]); // Sum(108) + res[109] = fma52hi(res[109], a[50], a[58]); // Sum(108) + res[109] = fma52lo(res[109], a[51], a[58]); // Sum(109) + res[110] = fma52hi(res[110], a[51], a[58]); // Sum(109) + res[110] = fma52lo(res[110], a[52], a[58]); // Sum(110) + res[111] = fma52hi(res[111], a[52], a[58]); // Sum(110) + res[111] = fma52lo(res[111], a[53], a[58]); // Sum(111) + res[112] = fma52hi(res[112], a[53], a[58]); // Sum(111) + res[112] = fma52lo(res[112], a[54], a[58]); // Sum(112) + res[113] = fma52hi(res[113], a[54], a[58]); // Sum(112) + res[113] = fma52lo(res[113], a[55], a[58]); // Sum(113) + res[114] = fma52hi(res[114], a[55], a[58]); // Sum(113) + res[114] = fma52lo(res[114], a[56], a[58]); // Sum(114) + res[115] = fma52hi(res[115], a[56], a[58]); // Sum(114) + res[115] = fma52lo(res[115], a[57], a[58]); // Sum(115) + res[116] = fma52hi(res[116], a[57], a[58]); // Sum(115) + res[108] = fma52lo(res[108], a[49], a[59]); // Sum(108) + res[109] = fma52hi(res[109], a[49], a[59]); // Sum(108) + res[109] = fma52lo(res[109], a[50], a[59]); // Sum(109) + res[110] = fma52hi(res[110], a[50], a[59]); // Sum(109) + res[110] = fma52lo(res[110], a[51], a[59]); // Sum(110) + res[111] = fma52hi(res[111], a[51], a[59]); // Sum(110) + res[111] = fma52lo(res[111], a[52], a[59]); // Sum(111) + res[112] = fma52hi(res[112], a[52], a[59]); // Sum(111) + res[112] = fma52lo(res[112], a[53], a[59]); // Sum(112) + res[113] = fma52hi(res[113], a[53], a[59]); // Sum(112) + res[113] = fma52lo(res[113], a[54], a[59]); // Sum(113) + res[114] = fma52hi(res[114], a[54], a[59]); // Sum(113) + res[114] = fma52lo(res[114], a[55], a[59]); // Sum(114) + res[115] = fma52hi(res[115], a[55], a[59]); // Sum(114) + res[115] = fma52lo(res[115], a[56], a[59]); // Sum(115) + res[116] = fma52hi(res[116], a[56], a[59]); // Sum(115) + res[116] = fma52lo(res[116], a[57], a[59]); // Sum(116) + res[117] = fma52hi(res[117], a[57], a[59]); // Sum(116) + res[117] = fma52lo(res[117], a[58], a[59]); // Sum(117) + res[118] = fma52hi(res[118], a[58], a[59]); // Sum(117) + res[108] = fma52lo(res[108], a[48], a[60]); // Sum(108) + res[109] = fma52hi(res[109], a[48], a[60]); // Sum(108) + res[109] = fma52lo(res[109], a[49], a[60]); // Sum(109) + res[110] = fma52hi(res[110], a[49], a[60]); // Sum(109) + res[110] = fma52lo(res[110], a[50], a[60]); // Sum(110) + res[111] = fma52hi(res[111], a[50], a[60]); // Sum(110) + res[111] = fma52lo(res[111], a[51], a[60]); // Sum(111) + res[112] = fma52hi(res[112], a[51], a[60]); // Sum(111) + res[112] = fma52lo(res[112], a[52], a[60]); // Sum(112) + res[113] = fma52hi(res[113], a[52], a[60]); // Sum(112) + res[113] = fma52lo(res[113], a[53], a[60]); // Sum(113) + res[114] = fma52hi(res[114], a[53], a[60]); // Sum(113) + res[114] = fma52lo(res[114], a[54], a[60]); // Sum(114) + res[115] = fma52hi(res[115], a[54], a[60]); // Sum(114) + res[115] = fma52lo(res[115], a[55], a[60]); // Sum(115) + res[116] = fma52hi(res[116], a[55], a[60]); // Sum(115) + res[116] = fma52lo(res[116], a[56], a[60]); // Sum(116) + res[117] = fma52hi(res[117], a[56], a[60]); // Sum(116) + res[117] = fma52lo(res[117], a[57], a[60]); // Sum(117) + res[118] = fma52hi(res[118], a[57], a[60]); // Sum(117) + res[118] = fma52lo(res[118], a[58], a[60]); // Sum(118) + res[119] = fma52hi(res[119], a[58], a[60]); // Sum(118) + res[119] = fma52lo(res[119], a[59], a[60]); // Sum(119) + res[120] = fma52hi(res[120], a[59], a[60]); // Sum(119) + res[108] = fma52lo(res[108], a[47], a[61]); // Sum(108) + res[109] = fma52hi(res[109], a[47], a[61]); // Sum(108) + res[109] = fma52lo(res[109], a[48], a[61]); // Sum(109) + res[110] = fma52hi(res[110], a[48], a[61]); // Sum(109) + res[110] = fma52lo(res[110], a[49], a[61]); // Sum(110) + res[111] = fma52hi(res[111], a[49], a[61]); // Sum(110) + res[111] = fma52lo(res[111], a[50], a[61]); // Sum(111) + res[112] = fma52hi(res[112], a[50], a[61]); // Sum(111) + res[112] = fma52lo(res[112], a[51], a[61]); // Sum(112) + res[113] = fma52hi(res[113], a[51], a[61]); // Sum(112) + res[113] = fma52lo(res[113], a[52], a[61]); // Sum(113) + res[114] = fma52hi(res[114], a[52], a[61]); // Sum(113) + res[114] = fma52lo(res[114], a[53], a[61]); // Sum(114) + res[115] = fma52hi(res[115], a[53], a[61]); // Sum(114) + res[115] = fma52lo(res[115], a[54], a[61]); // Sum(115) + res[116] = fma52hi(res[116], a[54], a[61]); // Sum(115) + res[116] = fma52lo(res[116], a[55], a[61]); // Sum(116) + res[117] = fma52hi(res[117], a[55], a[61]); // Sum(116) + res[117] = fma52lo(res[117], a[56], a[61]); // Sum(117) + res[118] = fma52hi(res[118], a[56], a[61]); // Sum(117) + res[118] = fma52lo(res[118], a[57], a[61]); // Sum(118) + res[119] = fma52hi(res[119], a[57], a[61]); // Sum(118) + res[119] = fma52lo(res[119], a[58], a[61]); // Sum(119) + res[120] = fma52hi(res[120], a[58], a[61]); // Sum(119) + res[108] = fma52lo(res[108], a[46], a[62]); // Sum(108) + res[109] = fma52hi(res[109], a[46], a[62]); // Sum(108) + res[109] = fma52lo(res[109], a[47], a[62]); // Sum(109) + res[110] = fma52hi(res[110], a[47], a[62]); // Sum(109) + res[110] = fma52lo(res[110], a[48], a[62]); // Sum(110) + res[111] = fma52hi(res[111], a[48], a[62]); // Sum(110) + res[111] = fma52lo(res[111], a[49], a[62]); // Sum(111) + res[112] = fma52hi(res[112], a[49], a[62]); // Sum(111) + res[112] = fma52lo(res[112], a[50], a[62]); // Sum(112) + res[113] = fma52hi(res[113], a[50], a[62]); // Sum(112) + res[113] = fma52lo(res[113], a[51], a[62]); // Sum(113) + res[114] = fma52hi(res[114], a[51], a[62]); // Sum(113) + res[114] = fma52lo(res[114], a[52], a[62]); // Sum(114) + res[115] = fma52hi(res[115], a[52], a[62]); // Sum(114) + res[115] = fma52lo(res[115], a[53], a[62]); // Sum(115) + res[116] = fma52hi(res[116], a[53], a[62]); // Sum(115) + res[116] = fma52lo(res[116], a[54], a[62]); // Sum(116) + res[117] = fma52hi(res[117], a[54], a[62]); // Sum(116) + res[117] = fma52lo(res[117], a[55], a[62]); // Sum(117) + res[118] = fma52hi(res[118], a[55], a[62]); // Sum(117) + res[118] = fma52lo(res[118], a[56], a[62]); // Sum(118) + res[119] = fma52hi(res[119], a[56], a[62]); // Sum(118) + res[119] = fma52lo(res[119], a[57], a[62]); // Sum(119) + res[120] = fma52hi(res[120], a[57], a[62]); // Sum(119) + res[108] = fma52lo(res[108], a[45], a[63]); // Sum(108) + res[109] = fma52hi(res[109], a[45], a[63]); // Sum(108) + res[109] = fma52lo(res[109], a[46], a[63]); // Sum(109) + res[110] = fma52hi(res[110], a[46], a[63]); // Sum(109) + res[110] = fma52lo(res[110], a[47], a[63]); // Sum(110) + res[111] = fma52hi(res[111], a[47], a[63]); // Sum(110) + res[111] = fma52lo(res[111], a[48], a[63]); // Sum(111) + res[112] = fma52hi(res[112], a[48], a[63]); // Sum(111) + res[112] = fma52lo(res[112], a[49], a[63]); // Sum(112) + res[113] = fma52hi(res[113], a[49], a[63]); // Sum(112) + res[113] = fma52lo(res[113], a[50], a[63]); // Sum(113) + res[114] = fma52hi(res[114], a[50], a[63]); // Sum(113) + res[114] = fma52lo(res[114], a[51], a[63]); // Sum(114) + res[115] = fma52hi(res[115], a[51], a[63]); // Sum(114) + res[115] = fma52lo(res[115], a[52], a[63]); // Sum(115) + res[116] = fma52hi(res[116], a[52], a[63]); // Sum(115) + res[116] = fma52lo(res[116], a[53], a[63]); // Sum(116) + res[117] = fma52hi(res[117], a[53], a[63]); // Sum(116) + res[117] = fma52lo(res[117], a[54], a[63]); // Sum(117) + res[118] = fma52hi(res[118], a[54], a[63]); // Sum(117) + res[118] = fma52lo(res[118], a[55], a[63]); // Sum(118) + res[119] = fma52hi(res[119], a[55], a[63]); // Sum(118) + res[119] = fma52lo(res[119], a[56], a[63]); // Sum(119) + res[120] = fma52hi(res[120], a[56], a[63]); // Sum(119) + res[108] = fma52lo(res[108], a[44], a[64]); // Sum(108) + res[109] = fma52hi(res[109], a[44], a[64]); // Sum(108) + res[109] = fma52lo(res[109], a[45], a[64]); // Sum(109) + res[110] = fma52hi(res[110], a[45], a[64]); // Sum(109) + res[110] = fma52lo(res[110], a[46], a[64]); // Sum(110) + res[111] = fma52hi(res[111], a[46], a[64]); // Sum(110) + res[111] = fma52lo(res[111], a[47], a[64]); // Sum(111) + res[112] = fma52hi(res[112], a[47], a[64]); // Sum(111) + res[112] = fma52lo(res[112], a[48], a[64]); // Sum(112) + res[113] = fma52hi(res[113], a[48], a[64]); // Sum(112) + res[113] = fma52lo(res[113], a[49], a[64]); // Sum(113) + res[114] = fma52hi(res[114], a[49], a[64]); // Sum(113) + res[114] = fma52lo(res[114], a[50], a[64]); // Sum(114) + res[115] = fma52hi(res[115], a[50], a[64]); // Sum(114) + res[115] = fma52lo(res[115], a[51], a[64]); // Sum(115) + res[116] = fma52hi(res[116], a[51], a[64]); // Sum(115) + res[116] = fma52lo(res[116], a[52], a[64]); // Sum(116) + res[117] = fma52hi(res[117], a[52], a[64]); // Sum(116) + res[117] = fma52lo(res[117], a[53], a[64]); // Sum(117) + res[118] = fma52hi(res[118], a[53], a[64]); // Sum(117) + res[118] = fma52lo(res[118], a[54], a[64]); // Sum(118) + res[119] = fma52hi(res[119], a[54], a[64]); // Sum(118) + res[119] = fma52lo(res[119], a[55], a[64]); // Sum(119) + res[120] = fma52hi(res[120], a[55], a[64]); // Sum(119) + res[108] = fma52lo(res[108], a[43], a[65]); // Sum(108) + res[109] = fma52hi(res[109], a[43], a[65]); // Sum(108) + res[109] = fma52lo(res[109], a[44], a[65]); // Sum(109) + res[110] = fma52hi(res[110], a[44], a[65]); // Sum(109) + res[110] = fma52lo(res[110], a[45], a[65]); // Sum(110) + res[111] = fma52hi(res[111], a[45], a[65]); // Sum(110) + res[111] = fma52lo(res[111], a[46], a[65]); // Sum(111) + res[112] = fma52hi(res[112], a[46], a[65]); // Sum(111) + res[112] = fma52lo(res[112], a[47], a[65]); // Sum(112) + res[113] = fma52hi(res[113], a[47], a[65]); // Sum(112) + res[113] = fma52lo(res[113], a[48], a[65]); // Sum(113) + res[114] = fma52hi(res[114], a[48], a[65]); // Sum(113) + res[114] = fma52lo(res[114], a[49], a[65]); // Sum(114) + res[115] = fma52hi(res[115], a[49], a[65]); // Sum(114) + res[115] = fma52lo(res[115], a[50], a[65]); // Sum(115) + res[116] = fma52hi(res[116], a[50], a[65]); // Sum(115) + res[116] = fma52lo(res[116], a[51], a[65]); // Sum(116) + res[117] = fma52hi(res[117], a[51], a[65]); // Sum(116) + res[117] = fma52lo(res[117], a[52], a[65]); // Sum(117) + res[118] = fma52hi(res[118], a[52], a[65]); // Sum(117) + res[118] = fma52lo(res[118], a[53], a[65]); // Sum(118) + res[119] = fma52hi(res[119], a[53], a[65]); // Sum(118) + res[119] = fma52lo(res[119], a[54], a[65]); // Sum(119) + res[120] = fma52hi(res[120], a[54], a[65]); // Sum(119) + res[108] = fma52lo(res[108], a[42], a[66]); // Sum(108) + res[109] = fma52hi(res[109], a[42], a[66]); // Sum(108) + res[109] = fma52lo(res[109], a[43], a[66]); // Sum(109) + res[110] = fma52hi(res[110], a[43], a[66]); // Sum(109) + res[110] = fma52lo(res[110], a[44], a[66]); // Sum(110) + res[111] = fma52hi(res[111], a[44], a[66]); // Sum(110) + res[111] = fma52lo(res[111], a[45], a[66]); // Sum(111) + res[112] = fma52hi(res[112], a[45], a[66]); // Sum(111) + res[112] = fma52lo(res[112], a[46], a[66]); // Sum(112) + res[113] = fma52hi(res[113], a[46], a[66]); // Sum(112) + res[113] = fma52lo(res[113], a[47], a[66]); // Sum(113) + res[114] = fma52hi(res[114], a[47], a[66]); // Sum(113) + res[114] = fma52lo(res[114], a[48], a[66]); // Sum(114) + res[115] = fma52hi(res[115], a[48], a[66]); // Sum(114) + res[115] = fma52lo(res[115], a[49], a[66]); // Sum(115) + res[116] = fma52hi(res[116], a[49], a[66]); // Sum(115) + res[116] = fma52lo(res[116], a[50], a[66]); // Sum(116) + res[117] = fma52hi(res[117], a[50], a[66]); // Sum(116) + res[117] = fma52lo(res[117], a[51], a[66]); // Sum(117) + res[118] = fma52hi(res[118], a[51], a[66]); // Sum(117) + res[118] = fma52lo(res[118], a[52], a[66]); // Sum(118) + res[119] = fma52hi(res[119], a[52], a[66]); // Sum(118) + res[119] = fma52lo(res[119], a[53], a[66]); // Sum(119) + res[120] = fma52hi(res[120], a[53], a[66]); // Sum(119) + res[108] = fma52lo(res[108], a[41], a[67]); // Sum(108) + res[109] = fma52hi(res[109], a[41], a[67]); // Sum(108) + res[109] = fma52lo(res[109], a[42], a[67]); // Sum(109) + res[110] = fma52hi(res[110], a[42], a[67]); // Sum(109) + res[110] = fma52lo(res[110], a[43], a[67]); // Sum(110) + res[111] = fma52hi(res[111], a[43], a[67]); // Sum(110) + res[111] = fma52lo(res[111], a[44], a[67]); // Sum(111) + res[112] = fma52hi(res[112], a[44], a[67]); // Sum(111) + res[112] = fma52lo(res[112], a[45], a[67]); // Sum(112) + res[113] = fma52hi(res[113], a[45], a[67]); // Sum(112) + res[113] = fma52lo(res[113], a[46], a[67]); // Sum(113) + res[114] = fma52hi(res[114], a[46], a[67]); // Sum(113) + res[114] = fma52lo(res[114], a[47], a[67]); // Sum(114) + res[115] = fma52hi(res[115], a[47], a[67]); // Sum(114) + res[115] = fma52lo(res[115], a[48], a[67]); // Sum(115) + res[116] = fma52hi(res[116], a[48], a[67]); // Sum(115) + res[116] = fma52lo(res[116], a[49], a[67]); // Sum(116) + res[117] = fma52hi(res[117], a[49], a[67]); // Sum(116) + res[117] = fma52lo(res[117], a[50], a[67]); // Sum(117) + res[118] = fma52hi(res[118], a[50], a[67]); // Sum(117) + res[118] = fma52lo(res[118], a[51], a[67]); // Sum(118) + res[119] = fma52hi(res[119], a[51], a[67]); // Sum(118) + res[119] = fma52lo(res[119], a[52], a[67]); // Sum(119) + res[120] = fma52hi(res[120], a[52], a[67]); // Sum(119) + res[108] = fma52lo(res[108], a[40], a[68]); // Sum(108) + res[109] = fma52hi(res[109], a[40], a[68]); // Sum(108) + res[109] = fma52lo(res[109], a[41], a[68]); // Sum(109) + res[110] = fma52hi(res[110], a[41], a[68]); // Sum(109) + res[110] = fma52lo(res[110], a[42], a[68]); // Sum(110) + res[111] = fma52hi(res[111], a[42], a[68]); // Sum(110) + res[111] = fma52lo(res[111], a[43], a[68]); // Sum(111) + res[112] = fma52hi(res[112], a[43], a[68]); // Sum(111) + res[112] = fma52lo(res[112], a[44], a[68]); // Sum(112) + res[113] = fma52hi(res[113], a[44], a[68]); // Sum(112) + res[113] = fma52lo(res[113], a[45], a[68]); // Sum(113) + res[114] = fma52hi(res[114], a[45], a[68]); // Sum(113) + res[114] = fma52lo(res[114], a[46], a[68]); // Sum(114) + res[115] = fma52hi(res[115], a[46], a[68]); // Sum(114) + res[115] = fma52lo(res[115], a[47], a[68]); // Sum(115) + res[116] = fma52hi(res[116], a[47], a[68]); // Sum(115) + res[116] = fma52lo(res[116], a[48], a[68]); // Sum(116) + res[117] = fma52hi(res[117], a[48], a[68]); // Sum(116) + res[117] = fma52lo(res[117], a[49], a[68]); // Sum(117) + res[118] = fma52hi(res[118], a[49], a[68]); // Sum(117) + res[118] = fma52lo(res[118], a[50], a[68]); // Sum(118) + res[119] = fma52hi(res[119], a[50], a[68]); // Sum(118) + res[119] = fma52lo(res[119], a[51], a[68]); // Sum(119) + res[120] = fma52hi(res[120], a[51], a[68]); // Sum(119) + res[108] = fma52lo(res[108], a[39], a[69]); // Sum(108) + res[109] = fma52hi(res[109], a[39], a[69]); // Sum(108) + res[109] = fma52lo(res[109], a[40], a[69]); // Sum(109) + res[110] = fma52hi(res[110], a[40], a[69]); // Sum(109) + res[110] = fma52lo(res[110], a[41], a[69]); // Sum(110) + res[111] = fma52hi(res[111], a[41], a[69]); // Sum(110) + res[111] = fma52lo(res[111], a[42], a[69]); // Sum(111) + res[112] = fma52hi(res[112], a[42], a[69]); // Sum(111) + res[112] = fma52lo(res[112], a[43], a[69]); // Sum(112) + res[113] = fma52hi(res[113], a[43], a[69]); // Sum(112) + res[113] = fma52lo(res[113], a[44], a[69]); // Sum(113) + res[114] = fma52hi(res[114], a[44], a[69]); // Sum(113) + res[114] = fma52lo(res[114], a[45], a[69]); // Sum(114) + res[115] = fma52hi(res[115], a[45], a[69]); // Sum(114) + res[115] = fma52lo(res[115], a[46], a[69]); // Sum(115) + res[116] = fma52hi(res[116], a[46], a[69]); // Sum(115) + res[116] = fma52lo(res[116], a[47], a[69]); // Sum(116) + res[117] = fma52hi(res[117], a[47], a[69]); // Sum(116) + res[117] = fma52lo(res[117], a[48], a[69]); // Sum(117) + res[118] = fma52hi(res[118], a[48], a[69]); // Sum(117) + res[118] = fma52lo(res[118], a[49], a[69]); // Sum(118) + res[119] = fma52hi(res[119], a[49], a[69]); // Sum(118) + res[119] = fma52lo(res[119], a[50], a[69]); // Sum(119) + res[120] = fma52hi(res[120], a[50], a[69]); // Sum(119) + res[108] = fma52lo(res[108], a[38], a[70]); // Sum(108) + res[109] = fma52hi(res[109], a[38], a[70]); // Sum(108) + res[109] = fma52lo(res[109], a[39], a[70]); // Sum(109) + res[110] = fma52hi(res[110], a[39], a[70]); // Sum(109) + res[110] = fma52lo(res[110], a[40], a[70]); // Sum(110) + res[111] = fma52hi(res[111], a[40], a[70]); // Sum(110) + res[111] = fma52lo(res[111], a[41], a[70]); // Sum(111) + res[112] = fma52hi(res[112], a[41], a[70]); // Sum(111) + res[112] = fma52lo(res[112], a[42], a[70]); // Sum(112) + res[113] = fma52hi(res[113], a[42], a[70]); // Sum(112) + res[113] = fma52lo(res[113], a[43], a[70]); // Sum(113) + res[114] = fma52hi(res[114], a[43], a[70]); // Sum(113) + res[114] = fma52lo(res[114], a[44], a[70]); // Sum(114) + res[115] = fma52hi(res[115], a[44], a[70]); // Sum(114) + res[115] = fma52lo(res[115], a[45], a[70]); // Sum(115) + res[116] = fma52hi(res[116], a[45], a[70]); // Sum(115) + res[116] = fma52lo(res[116], a[46], a[70]); // Sum(116) + res[117] = fma52hi(res[117], a[46], a[70]); // Sum(116) + res[117] = fma52lo(res[117], a[47], a[70]); // Sum(117) + res[118] = fma52hi(res[118], a[47], a[70]); // Sum(117) + res[118] = fma52lo(res[118], a[48], a[70]); // Sum(118) + res[119] = fma52hi(res[119], a[48], a[70]); // Sum(118) + res[119] = fma52lo(res[119], a[49], a[70]); // Sum(119) + res[120] = fma52hi(res[120], a[49], a[70]); // Sum(119) + res[108] = fma52lo(res[108], a[37], a[71]); // Sum(108) + res[109] = fma52hi(res[109], a[37], a[71]); // Sum(108) + res[109] = fma52lo(res[109], a[38], a[71]); // Sum(109) + res[110] = fma52hi(res[110], a[38], a[71]); // Sum(109) + res[110] = fma52lo(res[110], a[39], a[71]); // Sum(110) + res[111] = fma52hi(res[111], a[39], a[71]); // Sum(110) + res[111] = fma52lo(res[111], a[40], a[71]); // Sum(111) + res[112] = fma52hi(res[112], a[40], a[71]); // Sum(111) + res[112] = fma52lo(res[112], a[41], a[71]); // Sum(112) + res[113] = fma52hi(res[113], a[41], a[71]); // Sum(112) + res[113] = fma52lo(res[113], a[42], a[71]); // Sum(113) + res[114] = fma52hi(res[114], a[42], a[71]); // Sum(113) + res[114] = fma52lo(res[114], a[43], a[71]); // Sum(114) + res[115] = fma52hi(res[115], a[43], a[71]); // Sum(114) + res[115] = fma52lo(res[115], a[44], a[71]); // Sum(115) + res[116] = fma52hi(res[116], a[44], a[71]); // Sum(115) + res[116] = fma52lo(res[116], a[45], a[71]); // Sum(116) + res[117] = fma52hi(res[117], a[45], a[71]); // Sum(116) + res[117] = fma52lo(res[117], a[46], a[71]); // Sum(117) + res[118] = fma52hi(res[118], a[46], a[71]); // Sum(117) + res[118] = fma52lo(res[118], a[47], a[71]); // Sum(118) + res[119] = fma52hi(res[119], a[47], a[71]); // Sum(118) + res[119] = fma52lo(res[119], a[48], a[71]); // Sum(119) + res[120] = fma52hi(res[120], a[48], a[71]); // Sum(119) + res[108] = fma52lo(res[108], a[36], a[72]); // Sum(108) + res[109] = fma52hi(res[109], a[36], a[72]); // Sum(108) + res[109] = fma52lo(res[109], a[37], a[72]); // Sum(109) + res[110] = fma52hi(res[110], a[37], a[72]); // Sum(109) + res[110] = fma52lo(res[110], a[38], a[72]); // Sum(110) + res[111] = fma52hi(res[111], a[38], a[72]); // Sum(110) + res[111] = fma52lo(res[111], a[39], a[72]); // Sum(111) + res[112] = fma52hi(res[112], a[39], a[72]); // Sum(111) + res[112] = fma52lo(res[112], a[40], a[72]); // Sum(112) + res[113] = fma52hi(res[113], a[40], a[72]); // Sum(112) + res[113] = fma52lo(res[113], a[41], a[72]); // Sum(113) + res[114] = fma52hi(res[114], a[41], a[72]); // Sum(113) + res[114] = fma52lo(res[114], a[42], a[72]); // Sum(114) + res[115] = fma52hi(res[115], a[42], a[72]); // Sum(114) + res[115] = fma52lo(res[115], a[43], a[72]); // Sum(115) + res[116] = fma52hi(res[116], a[43], a[72]); // Sum(115) + res[116] = fma52lo(res[116], a[44], a[72]); // Sum(116) + res[117] = fma52hi(res[117], a[44], a[72]); // Sum(116) + res[117] = fma52lo(res[117], a[45], a[72]); // Sum(117) + res[118] = fma52hi(res[118], a[45], a[72]); // Sum(117) + res[118] = fma52lo(res[118], a[46], a[72]); // Sum(118) + res[119] = fma52hi(res[119], a[46], a[72]); // Sum(118) + res[119] = fma52lo(res[119], a[47], a[72]); // Sum(119) + res[120] = fma52hi(res[120], a[47], a[72]); // Sum(119) + res[108] = fma52lo(res[108], a[35], a[73]); // Sum(108) + res[109] = fma52hi(res[109], a[35], a[73]); // Sum(108) + res[109] = fma52lo(res[109], a[36], a[73]); // Sum(109) + res[110] = fma52hi(res[110], a[36], a[73]); // Sum(109) + res[110] = fma52lo(res[110], a[37], a[73]); // Sum(110) + res[111] = fma52hi(res[111], a[37], a[73]); // Sum(110) + res[111] = fma52lo(res[111], a[38], a[73]); // Sum(111) + res[112] = fma52hi(res[112], a[38], a[73]); // Sum(111) + res[112] = fma52lo(res[112], a[39], a[73]); // Sum(112) + res[113] = fma52hi(res[113], a[39], a[73]); // Sum(112) + res[113] = fma52lo(res[113], a[40], a[73]); // Sum(113) + res[114] = fma52hi(res[114], a[40], a[73]); // Sum(113) + res[114] = fma52lo(res[114], a[41], a[73]); // Sum(114) + res[115] = fma52hi(res[115], a[41], a[73]); // Sum(114) + res[115] = fma52lo(res[115], a[42], a[73]); // Sum(115) + res[116] = fma52hi(res[116], a[42], a[73]); // Sum(115) + res[116] = fma52lo(res[116], a[43], a[73]); // Sum(116) + res[117] = fma52hi(res[117], a[43], a[73]); // Sum(116) + res[117] = fma52lo(res[117], a[44], a[73]); // Sum(117) + res[118] = fma52hi(res[118], a[44], a[73]); // Sum(117) + res[118] = fma52lo(res[118], a[45], a[73]); // Sum(118) + res[119] = fma52hi(res[119], a[45], a[73]); // Sum(118) + res[119] = fma52lo(res[119], a[46], a[73]); // Sum(119) + res[120] = fma52hi(res[120], a[46], a[73]); // Sum(119) + res[108] = fma52lo(res[108], a[34], a[74]); // Sum(108) + res[109] = fma52hi(res[109], a[34], a[74]); // Sum(108) + res[109] = fma52lo(res[109], a[35], a[74]); // Sum(109) + res[110] = fma52hi(res[110], a[35], a[74]); // Sum(109) + res[110] = fma52lo(res[110], a[36], a[74]); // Sum(110) + res[111] = fma52hi(res[111], a[36], a[74]); // Sum(110) + res[111] = fma52lo(res[111], a[37], a[74]); // Sum(111) + res[112] = fma52hi(res[112], a[37], a[74]); // Sum(111) + res[112] = fma52lo(res[112], a[38], a[74]); // Sum(112) + res[113] = fma52hi(res[113], a[38], a[74]); // Sum(112) + res[113] = fma52lo(res[113], a[39], a[74]); // Sum(113) + res[114] = fma52hi(res[114], a[39], a[74]); // Sum(113) + res[114] = fma52lo(res[114], a[40], a[74]); // Sum(114) + res[115] = fma52hi(res[115], a[40], a[74]); // Sum(114) + res[115] = fma52lo(res[115], a[41], a[74]); // Sum(115) + res[116] = fma52hi(res[116], a[41], a[74]); // Sum(115) + res[116] = fma52lo(res[116], a[42], a[74]); // Sum(116) + res[117] = fma52hi(res[117], a[42], a[74]); // Sum(116) + res[117] = fma52lo(res[117], a[43], a[74]); // Sum(117) + res[118] = fma52hi(res[118], a[43], a[74]); // Sum(117) + res[118] = fma52lo(res[118], a[44], a[74]); // Sum(118) + res[119] = fma52hi(res[119], a[44], a[74]); // Sum(118) + res[119] = fma52lo(res[119], a[45], a[74]); // Sum(119) + res[120] = fma52hi(res[120], a[45], a[74]); // Sum(119) + res[108] = fma52lo(res[108], a[33], a[75]); // Sum(108) + res[109] = fma52hi(res[109], a[33], a[75]); // Sum(108) + res[109] = fma52lo(res[109], a[34], a[75]); // Sum(109) + res[110] = fma52hi(res[110], a[34], a[75]); // Sum(109) + res[110] = fma52lo(res[110], a[35], a[75]); // Sum(110) + res[111] = fma52hi(res[111], a[35], a[75]); // Sum(110) + res[111] = fma52lo(res[111], a[36], a[75]); // Sum(111) + res[112] = fma52hi(res[112], a[36], a[75]); // Sum(111) + res[112] = fma52lo(res[112], a[37], a[75]); // Sum(112) + res[113] = fma52hi(res[113], a[37], a[75]); // Sum(112) + res[113] = fma52lo(res[113], a[38], a[75]); // Sum(113) + res[114] = fma52hi(res[114], a[38], a[75]); // Sum(113) + res[114] = fma52lo(res[114], a[39], a[75]); // Sum(114) + res[115] = fma52hi(res[115], a[39], a[75]); // Sum(114) + res[115] = fma52lo(res[115], a[40], a[75]); // Sum(115) + res[116] = fma52hi(res[116], a[40], a[75]); // Sum(115) + res[116] = fma52lo(res[116], a[41], a[75]); // Sum(116) + res[117] = fma52hi(res[117], a[41], a[75]); // Sum(116) + res[117] = fma52lo(res[117], a[42], a[75]); // Sum(117) + res[118] = fma52hi(res[118], a[42], a[75]); // Sum(117) + res[118] = fma52lo(res[118], a[43], a[75]); // Sum(118) + res[119] = fma52hi(res[119], a[43], a[75]); // Sum(118) + res[119] = fma52lo(res[119], a[44], a[75]); // Sum(119) + res[120] = fma52hi(res[120], a[44], a[75]); // Sum(119) + res[108] = fma52lo(res[108], a[32], a[76]); // Sum(108) + res[109] = fma52hi(res[109], a[32], a[76]); // Sum(108) + res[109] = fma52lo(res[109], a[33], a[76]); // Sum(109) + res[110] = fma52hi(res[110], a[33], a[76]); // Sum(109) + res[110] = fma52lo(res[110], a[34], a[76]); // Sum(110) + res[111] = fma52hi(res[111], a[34], a[76]); // Sum(110) + res[111] = fma52lo(res[111], a[35], a[76]); // Sum(111) + res[112] = fma52hi(res[112], a[35], a[76]); // Sum(111) + res[112] = fma52lo(res[112], a[36], a[76]); // Sum(112) + res[113] = fma52hi(res[113], a[36], a[76]); // Sum(112) + res[113] = fma52lo(res[113], a[37], a[76]); // Sum(113) + res[114] = fma52hi(res[114], a[37], a[76]); // Sum(113) + res[114] = fma52lo(res[114], a[38], a[76]); // Sum(114) + res[115] = fma52hi(res[115], a[38], a[76]); // Sum(114) + res[115] = fma52lo(res[115], a[39], a[76]); // Sum(115) + res[116] = fma52hi(res[116], a[39], a[76]); // Sum(115) + res[116] = fma52lo(res[116], a[40], a[76]); // Sum(116) + res[117] = fma52hi(res[117], a[40], a[76]); // Sum(116) + res[117] = fma52lo(res[117], a[41], a[76]); // Sum(117) + res[118] = fma52hi(res[118], a[41], a[76]); // Sum(117) + res[118] = fma52lo(res[118], a[42], a[76]); // Sum(118) + res[119] = fma52hi(res[119], a[42], a[76]); // Sum(118) + res[119] = fma52lo(res[119], a[43], a[76]); // Sum(119) + res[120] = fma52hi(res[120], a[43], a[76]); // Sum(119) + res[108] = fma52lo(res[108], a[31], a[77]); // Sum(108) + res[109] = fma52hi(res[109], a[31], a[77]); // Sum(108) + res[109] = fma52lo(res[109], a[32], a[77]); // Sum(109) + res[110] = fma52hi(res[110], a[32], a[77]); // Sum(109) + res[110] = fma52lo(res[110], a[33], a[77]); // Sum(110) + res[111] = fma52hi(res[111], a[33], a[77]); // Sum(110) + res[111] = fma52lo(res[111], a[34], a[77]); // Sum(111) + res[112] = fma52hi(res[112], a[34], a[77]); // Sum(111) + res[112] = fma52lo(res[112], a[35], a[77]); // Sum(112) + res[113] = fma52hi(res[113], a[35], a[77]); // Sum(112) + res[113] = fma52lo(res[113], a[36], a[77]); // Sum(113) + res[114] = fma52hi(res[114], a[36], a[77]); // Sum(113) + res[114] = fma52lo(res[114], a[37], a[77]); // Sum(114) + res[115] = fma52hi(res[115], a[37], a[77]); // Sum(114) + res[115] = fma52lo(res[115], a[38], a[77]); // Sum(115) + res[116] = fma52hi(res[116], a[38], a[77]); // Sum(115) + res[116] = fma52lo(res[116], a[39], a[77]); // Sum(116) + res[117] = fma52hi(res[117], a[39], a[77]); // Sum(116) + res[117] = fma52lo(res[117], a[40], a[77]); // Sum(117) + res[118] = fma52hi(res[118], a[40], a[77]); // Sum(117) + res[118] = fma52lo(res[118], a[41], a[77]); // Sum(118) + res[119] = fma52hi(res[119], a[41], a[77]); // Sum(118) + res[119] = fma52lo(res[119], a[42], a[77]); // Sum(119) + res[120] = fma52hi(res[120], a[42], a[77]); // Sum(119) + res[108] = fma52lo(res[108], a[30], a[78]); // Sum(108) + res[109] = fma52hi(res[109], a[30], a[78]); // Sum(108) + res[109] = fma52lo(res[109], a[31], a[78]); // Sum(109) + res[110] = fma52hi(res[110], a[31], a[78]); // Sum(109) + res[110] = fma52lo(res[110], a[32], a[78]); // Sum(110) + res[111] = fma52hi(res[111], a[32], a[78]); // Sum(110) + res[111] = fma52lo(res[111], a[33], a[78]); // Sum(111) + res[112] = fma52hi(res[112], a[33], a[78]); // Sum(111) + res[112] = fma52lo(res[112], a[34], a[78]); // Sum(112) + res[113] = fma52hi(res[113], a[34], a[78]); // Sum(112) + res[113] = fma52lo(res[113], a[35], a[78]); // Sum(113) + res[114] = fma52hi(res[114], a[35], a[78]); // Sum(113) + res[114] = fma52lo(res[114], a[36], a[78]); // Sum(114) + res[115] = fma52hi(res[115], a[36], a[78]); // Sum(114) + res[115] = fma52lo(res[115], a[37], a[78]); // Sum(115) + res[116] = fma52hi(res[116], a[37], a[78]); // Sum(115) + res[116] = fma52lo(res[116], a[38], a[78]); // Sum(116) + res[117] = fma52hi(res[117], a[38], a[78]); // Sum(116) + res[117] = fma52lo(res[117], a[39], a[78]); // Sum(117) + res[118] = fma52hi(res[118], a[39], a[78]); // Sum(117) + res[118] = fma52lo(res[118], a[40], a[78]); // Sum(118) + res[119] = fma52hi(res[119], a[40], a[78]); // Sum(118) + res[119] = fma52lo(res[119], a[41], a[78]); // Sum(119) + res[120] = fma52hi(res[120], a[41], a[78]); // Sum(119) + res[108] = add64(res[108], res[108]); // Double(108) + res[109] = add64(res[109], res[109]); // Double(109) + res[110] = add64(res[110], res[110]); // Double(110) + res[111] = add64(res[111], res[111]); // Double(111) + res[112] = add64(res[112], res[112]); // Double(112) + res[113] = add64(res[113], res[113]); // Double(113) + res[114] = add64(res[114], res[114]); // Double(114) + res[115] = add64(res[115], res[115]); // Double(115) + res[116] = add64(res[116], res[116]); // Double(116) + res[117] = add64(res[117], res[117]); // Double(117) + res[118] = add64(res[118], res[118]); // Double(118) + res[119] = add64(res[119], res[119]); // Double(119) + res[108] = fma52lo(res[108], a[54], a[54]); // Add sqr(108) + res[109] = fma52hi(res[109], a[54], a[54]); // Add sqr(108) + res[110] = fma52lo(res[110], a[55], a[55]); // Add sqr(110) + res[111] = fma52hi(res[111], a[55], a[55]); // Add sqr(110) + res[112] = fma52lo(res[112], a[56], a[56]); // Add sqr(112) + res[113] = fma52hi(res[113], a[56], a[56]); // Add sqr(112) + res[114] = fma52lo(res[114], a[57], a[57]); // Add sqr(114) + res[115] = fma52hi(res[115], a[57], a[57]); // Add sqr(114) + res[116] = fma52lo(res[116], a[58], a[58]); // Add sqr(116) + res[117] = fma52hi(res[117], a[58], a[58]); // Add sqr(116) + res[118] = fma52lo(res[118], a[59], a[59]); // Add sqr(118) + res[119] = fma52hi(res[119], a[59], a[59]); // Add sqr(118) + res[120] = fma52lo(res[120], a[59], a[61]); // Sum(120) + res[121] = fma52hi(res[121], a[59], a[61]); // Sum(120) + res[121] = fma52lo(res[121], a[60], a[61]); // Sum(121) + res[122] = fma52hi(res[122], a[60], a[61]); // Sum(121) + res[120] = fma52lo(res[120], a[58], a[62]); // Sum(120) + res[121] = fma52hi(res[121], a[58], a[62]); // Sum(120) + res[121] = fma52lo(res[121], a[59], a[62]); // Sum(121) + res[122] = fma52hi(res[122], a[59], a[62]); // Sum(121) + res[122] = fma52lo(res[122], a[60], a[62]); // Sum(122) + res[123] = fma52hi(res[123], a[60], a[62]); // Sum(122) + res[123] = fma52lo(res[123], a[61], a[62]); // Sum(123) + res[124] = fma52hi(res[124], a[61], a[62]); // Sum(123) + res[120] = fma52lo(res[120], a[57], a[63]); // Sum(120) + res[121] = fma52hi(res[121], a[57], a[63]); // Sum(120) + res[121] = fma52lo(res[121], a[58], a[63]); // Sum(121) + res[122] = fma52hi(res[122], a[58], a[63]); // Sum(121) + res[122] = fma52lo(res[122], a[59], a[63]); // Sum(122) + res[123] = fma52hi(res[123], a[59], a[63]); // Sum(122) + res[123] = fma52lo(res[123], a[60], a[63]); // Sum(123) + res[124] = fma52hi(res[124], a[60], a[63]); // Sum(123) + res[124] = fma52lo(res[124], a[61], a[63]); // Sum(124) + res[125] = fma52hi(res[125], a[61], a[63]); // Sum(124) + res[125] = fma52lo(res[125], a[62], a[63]); // Sum(125) + res[126] = fma52hi(res[126], a[62], a[63]); // Sum(125) + res[120] = fma52lo(res[120], a[56], a[64]); // Sum(120) + res[121] = fma52hi(res[121], a[56], a[64]); // Sum(120) + res[121] = fma52lo(res[121], a[57], a[64]); // Sum(121) + res[122] = fma52hi(res[122], a[57], a[64]); // Sum(121) + res[122] = fma52lo(res[122], a[58], a[64]); // Sum(122) + res[123] = fma52hi(res[123], a[58], a[64]); // Sum(122) + res[123] = fma52lo(res[123], a[59], a[64]); // Sum(123) + res[124] = fma52hi(res[124], a[59], a[64]); // Sum(123) + res[124] = fma52lo(res[124], a[60], a[64]); // Sum(124) + res[125] = fma52hi(res[125], a[60], a[64]); // Sum(124) + res[125] = fma52lo(res[125], a[61], a[64]); // Sum(125) + res[126] = fma52hi(res[126], a[61], a[64]); // Sum(125) + res[126] = fma52lo(res[126], a[62], a[64]); // Sum(126) + res[127] = fma52hi(res[127], a[62], a[64]); // Sum(126) + res[127] = fma52lo(res[127], a[63], a[64]); // Sum(127) + res[128] = fma52hi(res[128], a[63], a[64]); // Sum(127) + res[120] = fma52lo(res[120], a[55], a[65]); // Sum(120) + res[121] = fma52hi(res[121], a[55], a[65]); // Sum(120) + res[121] = fma52lo(res[121], a[56], a[65]); // Sum(121) + res[122] = fma52hi(res[122], a[56], a[65]); // Sum(121) + res[122] = fma52lo(res[122], a[57], a[65]); // Sum(122) + res[123] = fma52hi(res[123], a[57], a[65]); // Sum(122) + res[123] = fma52lo(res[123], a[58], a[65]); // Sum(123) + res[124] = fma52hi(res[124], a[58], a[65]); // Sum(123) + res[124] = fma52lo(res[124], a[59], a[65]); // Sum(124) + res[125] = fma52hi(res[125], a[59], a[65]); // Sum(124) + res[125] = fma52lo(res[125], a[60], a[65]); // Sum(125) + res[126] = fma52hi(res[126], a[60], a[65]); // Sum(125) + res[126] = fma52lo(res[126], a[61], a[65]); // Sum(126) + res[127] = fma52hi(res[127], a[61], a[65]); // Sum(126) + res[127] = fma52lo(res[127], a[62], a[65]); // Sum(127) + res[128] = fma52hi(res[128], a[62], a[65]); // Sum(127) + res[128] = fma52lo(res[128], a[63], a[65]); // Sum(128) + res[129] = fma52hi(res[129], a[63], a[65]); // Sum(128) + res[129] = fma52lo(res[129], a[64], a[65]); // Sum(129) + res[130] = fma52hi(res[130], a[64], a[65]); // Sum(129) + res[120] = fma52lo(res[120], a[54], a[66]); // Sum(120) + res[121] = fma52hi(res[121], a[54], a[66]); // Sum(120) + res[121] = fma52lo(res[121], a[55], a[66]); // Sum(121) + res[122] = fma52hi(res[122], a[55], a[66]); // Sum(121) + res[122] = fma52lo(res[122], a[56], a[66]); // Sum(122) + res[123] = fma52hi(res[123], a[56], a[66]); // Sum(122) + res[123] = fma52lo(res[123], a[57], a[66]); // Sum(123) + res[124] = fma52hi(res[124], a[57], a[66]); // Sum(123) + res[124] = fma52lo(res[124], a[58], a[66]); // Sum(124) + res[125] = fma52hi(res[125], a[58], a[66]); // Sum(124) + res[125] = fma52lo(res[125], a[59], a[66]); // Sum(125) + res[126] = fma52hi(res[126], a[59], a[66]); // Sum(125) + res[126] = fma52lo(res[126], a[60], a[66]); // Sum(126) + res[127] = fma52hi(res[127], a[60], a[66]); // Sum(126) + res[127] = fma52lo(res[127], a[61], a[66]); // Sum(127) + res[128] = fma52hi(res[128], a[61], a[66]); // Sum(127) + res[128] = fma52lo(res[128], a[62], a[66]); // Sum(128) + res[129] = fma52hi(res[129], a[62], a[66]); // Sum(128) + res[129] = fma52lo(res[129], a[63], a[66]); // Sum(129) + res[130] = fma52hi(res[130], a[63], a[66]); // Sum(129) + res[130] = fma52lo(res[130], a[64], a[66]); // Sum(130) + res[131] = fma52hi(res[131], a[64], a[66]); // Sum(130) + res[131] = fma52lo(res[131], a[65], a[66]); // Sum(131) + res[132] = fma52hi(res[132], a[65], a[66]); // Sum(131) + res[120] = fma52lo(res[120], a[53], a[67]); // Sum(120) + res[121] = fma52hi(res[121], a[53], a[67]); // Sum(120) + res[121] = fma52lo(res[121], a[54], a[67]); // Sum(121) + res[122] = fma52hi(res[122], a[54], a[67]); // Sum(121) + res[122] = fma52lo(res[122], a[55], a[67]); // Sum(122) + res[123] = fma52hi(res[123], a[55], a[67]); // Sum(122) + res[123] = fma52lo(res[123], a[56], a[67]); // Sum(123) + res[124] = fma52hi(res[124], a[56], a[67]); // Sum(123) + res[124] = fma52lo(res[124], a[57], a[67]); // Sum(124) + res[125] = fma52hi(res[125], a[57], a[67]); // Sum(124) + res[125] = fma52lo(res[125], a[58], a[67]); // Sum(125) + res[126] = fma52hi(res[126], a[58], a[67]); // Sum(125) + res[126] = fma52lo(res[126], a[59], a[67]); // Sum(126) + res[127] = fma52hi(res[127], a[59], a[67]); // Sum(126) + res[127] = fma52lo(res[127], a[60], a[67]); // Sum(127) + res[128] = fma52hi(res[128], a[60], a[67]); // Sum(127) + res[128] = fma52lo(res[128], a[61], a[67]); // Sum(128) + res[129] = fma52hi(res[129], a[61], a[67]); // Sum(128) + res[129] = fma52lo(res[129], a[62], a[67]); // Sum(129) + res[130] = fma52hi(res[130], a[62], a[67]); // Sum(129) + res[130] = fma52lo(res[130], a[63], a[67]); // Sum(130) + res[131] = fma52hi(res[131], a[63], a[67]); // Sum(130) + res[131] = fma52lo(res[131], a[64], a[67]); // Sum(131) + res[132] = fma52hi(res[132], a[64], a[67]); // Sum(131) + res[120] = fma52lo(res[120], a[52], a[68]); // Sum(120) + res[121] = fma52hi(res[121], a[52], a[68]); // Sum(120) + res[121] = fma52lo(res[121], a[53], a[68]); // Sum(121) + res[122] = fma52hi(res[122], a[53], a[68]); // Sum(121) + res[122] = fma52lo(res[122], a[54], a[68]); // Sum(122) + res[123] = fma52hi(res[123], a[54], a[68]); // Sum(122) + res[123] = fma52lo(res[123], a[55], a[68]); // Sum(123) + res[124] = fma52hi(res[124], a[55], a[68]); // Sum(123) + res[124] = fma52lo(res[124], a[56], a[68]); // Sum(124) + res[125] = fma52hi(res[125], a[56], a[68]); // Sum(124) + res[125] = fma52lo(res[125], a[57], a[68]); // Sum(125) + res[126] = fma52hi(res[126], a[57], a[68]); // Sum(125) + res[126] = fma52lo(res[126], a[58], a[68]); // Sum(126) + res[127] = fma52hi(res[127], a[58], a[68]); // Sum(126) + res[127] = fma52lo(res[127], a[59], a[68]); // Sum(127) + res[128] = fma52hi(res[128], a[59], a[68]); // Sum(127) + res[128] = fma52lo(res[128], a[60], a[68]); // Sum(128) + res[129] = fma52hi(res[129], a[60], a[68]); // Sum(128) + res[129] = fma52lo(res[129], a[61], a[68]); // Sum(129) + res[130] = fma52hi(res[130], a[61], a[68]); // Sum(129) + res[130] = fma52lo(res[130], a[62], a[68]); // Sum(130) + res[131] = fma52hi(res[131], a[62], a[68]); // Sum(130) + res[131] = fma52lo(res[131], a[63], a[68]); // Sum(131) + res[132] = fma52hi(res[132], a[63], a[68]); // Sum(131) + res[120] = fma52lo(res[120], a[51], a[69]); // Sum(120) + res[121] = fma52hi(res[121], a[51], a[69]); // Sum(120) + res[121] = fma52lo(res[121], a[52], a[69]); // Sum(121) + res[122] = fma52hi(res[122], a[52], a[69]); // Sum(121) + res[122] = fma52lo(res[122], a[53], a[69]); // Sum(122) + res[123] = fma52hi(res[123], a[53], a[69]); // Sum(122) + res[123] = fma52lo(res[123], a[54], a[69]); // Sum(123) + res[124] = fma52hi(res[124], a[54], a[69]); // Sum(123) + res[124] = fma52lo(res[124], a[55], a[69]); // Sum(124) + res[125] = fma52hi(res[125], a[55], a[69]); // Sum(124) + res[125] = fma52lo(res[125], a[56], a[69]); // Sum(125) + res[126] = fma52hi(res[126], a[56], a[69]); // Sum(125) + res[126] = fma52lo(res[126], a[57], a[69]); // Sum(126) + res[127] = fma52hi(res[127], a[57], a[69]); // Sum(126) + res[127] = fma52lo(res[127], a[58], a[69]); // Sum(127) + res[128] = fma52hi(res[128], a[58], a[69]); // Sum(127) + res[128] = fma52lo(res[128], a[59], a[69]); // Sum(128) + res[129] = fma52hi(res[129], a[59], a[69]); // Sum(128) + res[129] = fma52lo(res[129], a[60], a[69]); // Sum(129) + res[130] = fma52hi(res[130], a[60], a[69]); // Sum(129) + res[130] = fma52lo(res[130], a[61], a[69]); // Sum(130) + res[131] = fma52hi(res[131], a[61], a[69]); // Sum(130) + res[131] = fma52lo(res[131], a[62], a[69]); // Sum(131) + res[132] = fma52hi(res[132], a[62], a[69]); // Sum(131) + res[120] = fma52lo(res[120], a[50], a[70]); // Sum(120) + res[121] = fma52hi(res[121], a[50], a[70]); // Sum(120) + res[121] = fma52lo(res[121], a[51], a[70]); // Sum(121) + res[122] = fma52hi(res[122], a[51], a[70]); // Sum(121) + res[122] = fma52lo(res[122], a[52], a[70]); // Sum(122) + res[123] = fma52hi(res[123], a[52], a[70]); // Sum(122) + res[123] = fma52lo(res[123], a[53], a[70]); // Sum(123) + res[124] = fma52hi(res[124], a[53], a[70]); // Sum(123) + res[124] = fma52lo(res[124], a[54], a[70]); // Sum(124) + res[125] = fma52hi(res[125], a[54], a[70]); // Sum(124) + res[125] = fma52lo(res[125], a[55], a[70]); // Sum(125) + res[126] = fma52hi(res[126], a[55], a[70]); // Sum(125) + res[126] = fma52lo(res[126], a[56], a[70]); // Sum(126) + res[127] = fma52hi(res[127], a[56], a[70]); // Sum(126) + res[127] = fma52lo(res[127], a[57], a[70]); // Sum(127) + res[128] = fma52hi(res[128], a[57], a[70]); // Sum(127) + res[128] = fma52lo(res[128], a[58], a[70]); // Sum(128) + res[129] = fma52hi(res[129], a[58], a[70]); // Sum(128) + res[129] = fma52lo(res[129], a[59], a[70]); // Sum(129) + res[130] = fma52hi(res[130], a[59], a[70]); // Sum(129) + res[130] = fma52lo(res[130], a[60], a[70]); // Sum(130) + res[131] = fma52hi(res[131], a[60], a[70]); // Sum(130) + res[131] = fma52lo(res[131], a[61], a[70]); // Sum(131) + res[132] = fma52hi(res[132], a[61], a[70]); // Sum(131) + res[120] = fma52lo(res[120], a[49], a[71]); // Sum(120) + res[121] = fma52hi(res[121], a[49], a[71]); // Sum(120) + res[121] = fma52lo(res[121], a[50], a[71]); // Sum(121) + res[122] = fma52hi(res[122], a[50], a[71]); // Sum(121) + res[122] = fma52lo(res[122], a[51], a[71]); // Sum(122) + res[123] = fma52hi(res[123], a[51], a[71]); // Sum(122) + res[123] = fma52lo(res[123], a[52], a[71]); // Sum(123) + res[124] = fma52hi(res[124], a[52], a[71]); // Sum(123) + res[124] = fma52lo(res[124], a[53], a[71]); // Sum(124) + res[125] = fma52hi(res[125], a[53], a[71]); // Sum(124) + res[125] = fma52lo(res[125], a[54], a[71]); // Sum(125) + res[126] = fma52hi(res[126], a[54], a[71]); // Sum(125) + res[126] = fma52lo(res[126], a[55], a[71]); // Sum(126) + res[127] = fma52hi(res[127], a[55], a[71]); // Sum(126) + res[127] = fma52lo(res[127], a[56], a[71]); // Sum(127) + res[128] = fma52hi(res[128], a[56], a[71]); // Sum(127) + res[128] = fma52lo(res[128], a[57], a[71]); // Sum(128) + res[129] = fma52hi(res[129], a[57], a[71]); // Sum(128) + res[129] = fma52lo(res[129], a[58], a[71]); // Sum(129) + res[130] = fma52hi(res[130], a[58], a[71]); // Sum(129) + res[130] = fma52lo(res[130], a[59], a[71]); // Sum(130) + res[131] = fma52hi(res[131], a[59], a[71]); // Sum(130) + res[131] = fma52lo(res[131], a[60], a[71]); // Sum(131) + res[132] = fma52hi(res[132], a[60], a[71]); // Sum(131) + res[120] = fma52lo(res[120], a[48], a[72]); // Sum(120) + res[121] = fma52hi(res[121], a[48], a[72]); // Sum(120) + res[121] = fma52lo(res[121], a[49], a[72]); // Sum(121) + res[122] = fma52hi(res[122], a[49], a[72]); // Sum(121) + res[122] = fma52lo(res[122], a[50], a[72]); // Sum(122) + res[123] = fma52hi(res[123], a[50], a[72]); // Sum(122) + res[123] = fma52lo(res[123], a[51], a[72]); // Sum(123) + res[124] = fma52hi(res[124], a[51], a[72]); // Sum(123) + res[124] = fma52lo(res[124], a[52], a[72]); // Sum(124) + res[125] = fma52hi(res[125], a[52], a[72]); // Sum(124) + res[125] = fma52lo(res[125], a[53], a[72]); // Sum(125) + res[126] = fma52hi(res[126], a[53], a[72]); // Sum(125) + res[126] = fma52lo(res[126], a[54], a[72]); // Sum(126) + res[127] = fma52hi(res[127], a[54], a[72]); // Sum(126) + res[127] = fma52lo(res[127], a[55], a[72]); // Sum(127) + res[128] = fma52hi(res[128], a[55], a[72]); // Sum(127) + res[128] = fma52lo(res[128], a[56], a[72]); // Sum(128) + res[129] = fma52hi(res[129], a[56], a[72]); // Sum(128) + res[129] = fma52lo(res[129], a[57], a[72]); // Sum(129) + res[130] = fma52hi(res[130], a[57], a[72]); // Sum(129) + res[130] = fma52lo(res[130], a[58], a[72]); // Sum(130) + res[131] = fma52hi(res[131], a[58], a[72]); // Sum(130) + res[131] = fma52lo(res[131], a[59], a[72]); // Sum(131) + res[132] = fma52hi(res[132], a[59], a[72]); // Sum(131) + res[120] = fma52lo(res[120], a[47], a[73]); // Sum(120) + res[121] = fma52hi(res[121], a[47], a[73]); // Sum(120) + res[121] = fma52lo(res[121], a[48], a[73]); // Sum(121) + res[122] = fma52hi(res[122], a[48], a[73]); // Sum(121) + res[122] = fma52lo(res[122], a[49], a[73]); // Sum(122) + res[123] = fma52hi(res[123], a[49], a[73]); // Sum(122) + res[123] = fma52lo(res[123], a[50], a[73]); // Sum(123) + res[124] = fma52hi(res[124], a[50], a[73]); // Sum(123) + res[124] = fma52lo(res[124], a[51], a[73]); // Sum(124) + res[125] = fma52hi(res[125], a[51], a[73]); // Sum(124) + res[125] = fma52lo(res[125], a[52], a[73]); // Sum(125) + res[126] = fma52hi(res[126], a[52], a[73]); // Sum(125) + res[126] = fma52lo(res[126], a[53], a[73]); // Sum(126) + res[127] = fma52hi(res[127], a[53], a[73]); // Sum(126) + res[127] = fma52lo(res[127], a[54], a[73]); // Sum(127) + res[128] = fma52hi(res[128], a[54], a[73]); // Sum(127) + res[128] = fma52lo(res[128], a[55], a[73]); // Sum(128) + res[129] = fma52hi(res[129], a[55], a[73]); // Sum(128) + res[129] = fma52lo(res[129], a[56], a[73]); // Sum(129) + res[130] = fma52hi(res[130], a[56], a[73]); // Sum(129) + res[130] = fma52lo(res[130], a[57], a[73]); // Sum(130) + res[131] = fma52hi(res[131], a[57], a[73]); // Sum(130) + res[131] = fma52lo(res[131], a[58], a[73]); // Sum(131) + res[132] = fma52hi(res[132], a[58], a[73]); // Sum(131) + res[120] = fma52lo(res[120], a[46], a[74]); // Sum(120) + res[121] = fma52hi(res[121], a[46], a[74]); // Sum(120) + res[121] = fma52lo(res[121], a[47], a[74]); // Sum(121) + res[122] = fma52hi(res[122], a[47], a[74]); // Sum(121) + res[122] = fma52lo(res[122], a[48], a[74]); // Sum(122) + res[123] = fma52hi(res[123], a[48], a[74]); // Sum(122) + res[123] = fma52lo(res[123], a[49], a[74]); // Sum(123) + res[124] = fma52hi(res[124], a[49], a[74]); // Sum(123) + res[124] = fma52lo(res[124], a[50], a[74]); // Sum(124) + res[125] = fma52hi(res[125], a[50], a[74]); // Sum(124) + res[125] = fma52lo(res[125], a[51], a[74]); // Sum(125) + res[126] = fma52hi(res[126], a[51], a[74]); // Sum(125) + res[126] = fma52lo(res[126], a[52], a[74]); // Sum(126) + res[127] = fma52hi(res[127], a[52], a[74]); // Sum(126) + res[127] = fma52lo(res[127], a[53], a[74]); // Sum(127) + res[128] = fma52hi(res[128], a[53], a[74]); // Sum(127) + res[128] = fma52lo(res[128], a[54], a[74]); // Sum(128) + res[129] = fma52hi(res[129], a[54], a[74]); // Sum(128) + res[129] = fma52lo(res[129], a[55], a[74]); // Sum(129) + res[130] = fma52hi(res[130], a[55], a[74]); // Sum(129) + res[130] = fma52lo(res[130], a[56], a[74]); // Sum(130) + res[131] = fma52hi(res[131], a[56], a[74]); // Sum(130) + res[131] = fma52lo(res[131], a[57], a[74]); // Sum(131) + res[132] = fma52hi(res[132], a[57], a[74]); // Sum(131) + res[120] = fma52lo(res[120], a[45], a[75]); // Sum(120) + res[121] = fma52hi(res[121], a[45], a[75]); // Sum(120) + res[121] = fma52lo(res[121], a[46], a[75]); // Sum(121) + res[122] = fma52hi(res[122], a[46], a[75]); // Sum(121) + res[122] = fma52lo(res[122], a[47], a[75]); // Sum(122) + res[123] = fma52hi(res[123], a[47], a[75]); // Sum(122) + res[123] = fma52lo(res[123], a[48], a[75]); // Sum(123) + res[124] = fma52hi(res[124], a[48], a[75]); // Sum(123) + res[124] = fma52lo(res[124], a[49], a[75]); // Sum(124) + res[125] = fma52hi(res[125], a[49], a[75]); // Sum(124) + res[125] = fma52lo(res[125], a[50], a[75]); // Sum(125) + res[126] = fma52hi(res[126], a[50], a[75]); // Sum(125) + res[126] = fma52lo(res[126], a[51], a[75]); // Sum(126) + res[127] = fma52hi(res[127], a[51], a[75]); // Sum(126) + res[127] = fma52lo(res[127], a[52], a[75]); // Sum(127) + res[128] = fma52hi(res[128], a[52], a[75]); // Sum(127) + res[128] = fma52lo(res[128], a[53], a[75]); // Sum(128) + res[129] = fma52hi(res[129], a[53], a[75]); // Sum(128) + res[129] = fma52lo(res[129], a[54], a[75]); // Sum(129) + res[130] = fma52hi(res[130], a[54], a[75]); // Sum(129) + res[130] = fma52lo(res[130], a[55], a[75]); // Sum(130) + res[131] = fma52hi(res[131], a[55], a[75]); // Sum(130) + res[131] = fma52lo(res[131], a[56], a[75]); // Sum(131) + res[132] = fma52hi(res[132], a[56], a[75]); // Sum(131) + res[120] = fma52lo(res[120], a[44], a[76]); // Sum(120) + res[121] = fma52hi(res[121], a[44], a[76]); // Sum(120) + res[121] = fma52lo(res[121], a[45], a[76]); // Sum(121) + res[122] = fma52hi(res[122], a[45], a[76]); // Sum(121) + res[122] = fma52lo(res[122], a[46], a[76]); // Sum(122) + res[123] = fma52hi(res[123], a[46], a[76]); // Sum(122) + res[123] = fma52lo(res[123], a[47], a[76]); // Sum(123) + res[124] = fma52hi(res[124], a[47], a[76]); // Sum(123) + res[124] = fma52lo(res[124], a[48], a[76]); // Sum(124) + res[125] = fma52hi(res[125], a[48], a[76]); // Sum(124) + res[125] = fma52lo(res[125], a[49], a[76]); // Sum(125) + res[126] = fma52hi(res[126], a[49], a[76]); // Sum(125) + res[126] = fma52lo(res[126], a[50], a[76]); // Sum(126) + res[127] = fma52hi(res[127], a[50], a[76]); // Sum(126) + res[127] = fma52lo(res[127], a[51], a[76]); // Sum(127) + res[128] = fma52hi(res[128], a[51], a[76]); // Sum(127) + res[128] = fma52lo(res[128], a[52], a[76]); // Sum(128) + res[129] = fma52hi(res[129], a[52], a[76]); // Sum(128) + res[129] = fma52lo(res[129], a[53], a[76]); // Sum(129) + res[130] = fma52hi(res[130], a[53], a[76]); // Sum(129) + res[130] = fma52lo(res[130], a[54], a[76]); // Sum(130) + res[131] = fma52hi(res[131], a[54], a[76]); // Sum(130) + res[131] = fma52lo(res[131], a[55], a[76]); // Sum(131) + res[132] = fma52hi(res[132], a[55], a[76]); // Sum(131) + res[120] = fma52lo(res[120], a[43], a[77]); // Sum(120) + res[121] = fma52hi(res[121], a[43], a[77]); // Sum(120) + res[121] = fma52lo(res[121], a[44], a[77]); // Sum(121) + res[122] = fma52hi(res[122], a[44], a[77]); // Sum(121) + res[122] = fma52lo(res[122], a[45], a[77]); // Sum(122) + res[123] = fma52hi(res[123], a[45], a[77]); // Sum(122) + res[123] = fma52lo(res[123], a[46], a[77]); // Sum(123) + res[124] = fma52hi(res[124], a[46], a[77]); // Sum(123) + res[124] = fma52lo(res[124], a[47], a[77]); // Sum(124) + res[125] = fma52hi(res[125], a[47], a[77]); // Sum(124) + res[125] = fma52lo(res[125], a[48], a[77]); // Sum(125) + res[126] = fma52hi(res[126], a[48], a[77]); // Sum(125) + res[126] = fma52lo(res[126], a[49], a[77]); // Sum(126) + res[127] = fma52hi(res[127], a[49], a[77]); // Sum(126) + res[127] = fma52lo(res[127], a[50], a[77]); // Sum(127) + res[128] = fma52hi(res[128], a[50], a[77]); // Sum(127) + res[128] = fma52lo(res[128], a[51], a[77]); // Sum(128) + res[129] = fma52hi(res[129], a[51], a[77]); // Sum(128) + res[129] = fma52lo(res[129], a[52], a[77]); // Sum(129) + res[130] = fma52hi(res[130], a[52], a[77]); // Sum(129) + res[130] = fma52lo(res[130], a[53], a[77]); // Sum(130) + res[131] = fma52hi(res[131], a[53], a[77]); // Sum(130) + res[131] = fma52lo(res[131], a[54], a[77]); // Sum(131) + res[132] = fma52hi(res[132], a[54], a[77]); // Sum(131) + res[120] = fma52lo(res[120], a[42], a[78]); // Sum(120) + res[121] = fma52hi(res[121], a[42], a[78]); // Sum(120) + res[121] = fma52lo(res[121], a[43], a[78]); // Sum(121) + res[122] = fma52hi(res[122], a[43], a[78]); // Sum(121) + res[122] = fma52lo(res[122], a[44], a[78]); // Sum(122) + res[123] = fma52hi(res[123], a[44], a[78]); // Sum(122) + res[123] = fma52lo(res[123], a[45], a[78]); // Sum(123) + res[124] = fma52hi(res[124], a[45], a[78]); // Sum(123) + res[124] = fma52lo(res[124], a[46], a[78]); // Sum(124) + res[125] = fma52hi(res[125], a[46], a[78]); // Sum(124) + res[125] = fma52lo(res[125], a[47], a[78]); // Sum(125) + res[126] = fma52hi(res[126], a[47], a[78]); // Sum(125) + res[126] = fma52lo(res[126], a[48], a[78]); // Sum(126) + res[127] = fma52hi(res[127], a[48], a[78]); // Sum(126) + res[127] = fma52lo(res[127], a[49], a[78]); // Sum(127) + res[128] = fma52hi(res[128], a[49], a[78]); // Sum(127) + res[128] = fma52lo(res[128], a[50], a[78]); // Sum(128) + res[129] = fma52hi(res[129], a[50], a[78]); // Sum(128) + res[129] = fma52lo(res[129], a[51], a[78]); // Sum(129) + res[130] = fma52hi(res[130], a[51], a[78]); // Sum(129) + res[130] = fma52lo(res[130], a[52], a[78]); // Sum(130) + res[131] = fma52hi(res[131], a[52], a[78]); // Sum(130) + res[131] = fma52lo(res[131], a[53], a[78]); // Sum(131) + res[132] = fma52hi(res[132], a[53], a[78]); // Sum(131) + res[120] = add64(res[120], res[120]); // Double(120) + res[121] = add64(res[121], res[121]); // Double(121) + res[122] = add64(res[122], res[122]); // Double(122) + res[123] = add64(res[123], res[123]); // Double(123) + res[124] = add64(res[124], res[124]); // Double(124) + res[125] = add64(res[125], res[125]); // Double(125) + res[126] = add64(res[126], res[126]); // Double(126) + res[127] = add64(res[127], res[127]); // Double(127) + res[128] = add64(res[128], res[128]); // Double(128) + res[129] = add64(res[129], res[129]); // Double(129) + res[130] = add64(res[130], res[130]); // Double(130) + res[131] = add64(res[131], res[131]); // Double(131) + res[120] = fma52lo(res[120], a[60], a[60]); // Add sqr(120) + res[121] = fma52hi(res[121], a[60], a[60]); // Add sqr(120) + res[122] = fma52lo(res[122], a[61], a[61]); // Add sqr(122) + res[123] = fma52hi(res[123], a[61], a[61]); // Add sqr(122) + res[124] = fma52lo(res[124], a[62], a[62]); // Add sqr(124) + res[125] = fma52hi(res[125], a[62], a[62]); // Add sqr(124) + res[126] = fma52lo(res[126], a[63], a[63]); // Add sqr(126) + res[127] = fma52hi(res[127], a[63], a[63]); // Add sqr(126) + res[128] = fma52lo(res[128], a[64], a[64]); // Add sqr(128) + res[129] = fma52hi(res[129], a[64], a[64]); // Add sqr(128) + res[130] = fma52lo(res[130], a[65], a[65]); // Add sqr(130) + res[131] = fma52hi(res[131], a[65], a[65]); // Add sqr(130) + res[132] = fma52lo(res[132], a[65], a[67]); // Sum(132) + res[133] = fma52hi(res[133], a[65], a[67]); // Sum(132) + res[133] = fma52lo(res[133], a[66], a[67]); // Sum(133) + res[134] = fma52hi(res[134], a[66], a[67]); // Sum(133) + res[132] = fma52lo(res[132], a[64], a[68]); // Sum(132) + res[133] = fma52hi(res[133], a[64], a[68]); // Sum(132) + res[133] = fma52lo(res[133], a[65], a[68]); // Sum(133) + res[134] = fma52hi(res[134], a[65], a[68]); // Sum(133) + res[134] = fma52lo(res[134], a[66], a[68]); // Sum(134) + res[135] = fma52hi(res[135], a[66], a[68]); // Sum(134) + res[135] = fma52lo(res[135], a[67], a[68]); // Sum(135) + res[136] = fma52hi(res[136], a[67], a[68]); // Sum(135) + res[132] = fma52lo(res[132], a[63], a[69]); // Sum(132) + res[133] = fma52hi(res[133], a[63], a[69]); // Sum(132) + res[133] = fma52lo(res[133], a[64], a[69]); // Sum(133) + res[134] = fma52hi(res[134], a[64], a[69]); // Sum(133) + res[134] = fma52lo(res[134], a[65], a[69]); // Sum(134) + res[135] = fma52hi(res[135], a[65], a[69]); // Sum(134) + res[135] = fma52lo(res[135], a[66], a[69]); // Sum(135) + res[136] = fma52hi(res[136], a[66], a[69]); // Sum(135) + res[136] = fma52lo(res[136], a[67], a[69]); // Sum(136) + res[137] = fma52hi(res[137], a[67], a[69]); // Sum(136) + res[137] = fma52lo(res[137], a[68], a[69]); // Sum(137) + res[138] = fma52hi(res[138], a[68], a[69]); // Sum(137) + res[132] = fma52lo(res[132], a[62], a[70]); // Sum(132) + res[133] = fma52hi(res[133], a[62], a[70]); // Sum(132) + res[133] = fma52lo(res[133], a[63], a[70]); // Sum(133) + res[134] = fma52hi(res[134], a[63], a[70]); // Sum(133) + res[134] = fma52lo(res[134], a[64], a[70]); // Sum(134) + res[135] = fma52hi(res[135], a[64], a[70]); // Sum(134) + res[135] = fma52lo(res[135], a[65], a[70]); // Sum(135) + res[136] = fma52hi(res[136], a[65], a[70]); // Sum(135) + res[136] = fma52lo(res[136], a[66], a[70]); // Sum(136) + res[137] = fma52hi(res[137], a[66], a[70]); // Sum(136) + res[137] = fma52lo(res[137], a[67], a[70]); // Sum(137) + res[138] = fma52hi(res[138], a[67], a[70]); // Sum(137) + res[138] = fma52lo(res[138], a[68], a[70]); // Sum(138) + res[139] = fma52hi(res[139], a[68], a[70]); // Sum(138) + res[139] = fma52lo(res[139], a[69], a[70]); // Sum(139) + res[140] = fma52hi(res[140], a[69], a[70]); // Sum(139) + res[132] = fma52lo(res[132], a[61], a[71]); // Sum(132) + res[133] = fma52hi(res[133], a[61], a[71]); // Sum(132) + res[133] = fma52lo(res[133], a[62], a[71]); // Sum(133) + res[134] = fma52hi(res[134], a[62], a[71]); // Sum(133) + res[134] = fma52lo(res[134], a[63], a[71]); // Sum(134) + res[135] = fma52hi(res[135], a[63], a[71]); // Sum(134) + res[135] = fma52lo(res[135], a[64], a[71]); // Sum(135) + res[136] = fma52hi(res[136], a[64], a[71]); // Sum(135) + res[136] = fma52lo(res[136], a[65], a[71]); // Sum(136) + res[137] = fma52hi(res[137], a[65], a[71]); // Sum(136) + res[137] = fma52lo(res[137], a[66], a[71]); // Sum(137) + res[138] = fma52hi(res[138], a[66], a[71]); // Sum(137) + res[138] = fma52lo(res[138], a[67], a[71]); // Sum(138) + res[139] = fma52hi(res[139], a[67], a[71]); // Sum(138) + res[139] = fma52lo(res[139], a[68], a[71]); // Sum(139) + res[140] = fma52hi(res[140], a[68], a[71]); // Sum(139) + res[140] = fma52lo(res[140], a[69], a[71]); // Sum(140) + res[141] = fma52hi(res[141], a[69], a[71]); // Sum(140) + res[141] = fma52lo(res[141], a[70], a[71]); // Sum(141) + res[142] = fma52hi(res[142], a[70], a[71]); // Sum(141) + res[132] = fma52lo(res[132], a[60], a[72]); // Sum(132) + res[133] = fma52hi(res[133], a[60], a[72]); // Sum(132) + res[133] = fma52lo(res[133], a[61], a[72]); // Sum(133) + res[134] = fma52hi(res[134], a[61], a[72]); // Sum(133) + res[134] = fma52lo(res[134], a[62], a[72]); // Sum(134) + res[135] = fma52hi(res[135], a[62], a[72]); // Sum(134) + res[135] = fma52lo(res[135], a[63], a[72]); // Sum(135) + res[136] = fma52hi(res[136], a[63], a[72]); // Sum(135) + res[136] = fma52lo(res[136], a[64], a[72]); // Sum(136) + res[137] = fma52hi(res[137], a[64], a[72]); // Sum(136) + res[137] = fma52lo(res[137], a[65], a[72]); // Sum(137) + res[138] = fma52hi(res[138], a[65], a[72]); // Sum(137) + res[138] = fma52lo(res[138], a[66], a[72]); // Sum(138) + res[139] = fma52hi(res[139], a[66], a[72]); // Sum(138) + res[139] = fma52lo(res[139], a[67], a[72]); // Sum(139) + res[140] = fma52hi(res[140], a[67], a[72]); // Sum(139) + res[140] = fma52lo(res[140], a[68], a[72]); // Sum(140) + res[141] = fma52hi(res[141], a[68], a[72]); // Sum(140) + res[141] = fma52lo(res[141], a[69], a[72]); // Sum(141) + res[142] = fma52hi(res[142], a[69], a[72]); // Sum(141) + res[142] = fma52lo(res[142], a[70], a[72]); // Sum(142) + res[143] = fma52hi(res[143], a[70], a[72]); // Sum(142) + res[143] = fma52lo(res[143], a[71], a[72]); // Sum(143) + res[144] = fma52hi(res[144], a[71], a[72]); // Sum(143) + res[132] = fma52lo(res[132], a[59], a[73]); // Sum(132) + res[133] = fma52hi(res[133], a[59], a[73]); // Sum(132) + res[133] = fma52lo(res[133], a[60], a[73]); // Sum(133) + res[134] = fma52hi(res[134], a[60], a[73]); // Sum(133) + res[134] = fma52lo(res[134], a[61], a[73]); // Sum(134) + res[135] = fma52hi(res[135], a[61], a[73]); // Sum(134) + res[135] = fma52lo(res[135], a[62], a[73]); // Sum(135) + res[136] = fma52hi(res[136], a[62], a[73]); // Sum(135) + res[136] = fma52lo(res[136], a[63], a[73]); // Sum(136) + res[137] = fma52hi(res[137], a[63], a[73]); // Sum(136) + res[137] = fma52lo(res[137], a[64], a[73]); // Sum(137) + res[138] = fma52hi(res[138], a[64], a[73]); // Sum(137) + res[138] = fma52lo(res[138], a[65], a[73]); // Sum(138) + res[139] = fma52hi(res[139], a[65], a[73]); // Sum(138) + res[139] = fma52lo(res[139], a[66], a[73]); // Sum(139) + res[140] = fma52hi(res[140], a[66], a[73]); // Sum(139) + res[140] = fma52lo(res[140], a[67], a[73]); // Sum(140) + res[141] = fma52hi(res[141], a[67], a[73]); // Sum(140) + res[141] = fma52lo(res[141], a[68], a[73]); // Sum(141) + res[142] = fma52hi(res[142], a[68], a[73]); // Sum(141) + res[142] = fma52lo(res[142], a[69], a[73]); // Sum(142) + res[143] = fma52hi(res[143], a[69], a[73]); // Sum(142) + res[143] = fma52lo(res[143], a[70], a[73]); // Sum(143) + res[144] = fma52hi(res[144], a[70], a[73]); // Sum(143) + res[132] = fma52lo(res[132], a[58], a[74]); // Sum(132) + res[133] = fma52hi(res[133], a[58], a[74]); // Sum(132) + res[133] = fma52lo(res[133], a[59], a[74]); // Sum(133) + res[134] = fma52hi(res[134], a[59], a[74]); // Sum(133) + res[134] = fma52lo(res[134], a[60], a[74]); // Sum(134) + res[135] = fma52hi(res[135], a[60], a[74]); // Sum(134) + res[135] = fma52lo(res[135], a[61], a[74]); // Sum(135) + res[136] = fma52hi(res[136], a[61], a[74]); // Sum(135) + res[136] = fma52lo(res[136], a[62], a[74]); // Sum(136) + res[137] = fma52hi(res[137], a[62], a[74]); // Sum(136) + res[137] = fma52lo(res[137], a[63], a[74]); // Sum(137) + res[138] = fma52hi(res[138], a[63], a[74]); // Sum(137) + res[138] = fma52lo(res[138], a[64], a[74]); // Sum(138) + res[139] = fma52hi(res[139], a[64], a[74]); // Sum(138) + res[139] = fma52lo(res[139], a[65], a[74]); // Sum(139) + res[140] = fma52hi(res[140], a[65], a[74]); // Sum(139) + res[140] = fma52lo(res[140], a[66], a[74]); // Sum(140) + res[141] = fma52hi(res[141], a[66], a[74]); // Sum(140) + res[141] = fma52lo(res[141], a[67], a[74]); // Sum(141) + res[142] = fma52hi(res[142], a[67], a[74]); // Sum(141) + res[142] = fma52lo(res[142], a[68], a[74]); // Sum(142) + res[143] = fma52hi(res[143], a[68], a[74]); // Sum(142) + res[143] = fma52lo(res[143], a[69], a[74]); // Sum(143) + res[144] = fma52hi(res[144], a[69], a[74]); // Sum(143) + res[132] = fma52lo(res[132], a[57], a[75]); // Sum(132) + res[133] = fma52hi(res[133], a[57], a[75]); // Sum(132) + res[133] = fma52lo(res[133], a[58], a[75]); // Sum(133) + res[134] = fma52hi(res[134], a[58], a[75]); // Sum(133) + res[134] = fma52lo(res[134], a[59], a[75]); // Sum(134) + res[135] = fma52hi(res[135], a[59], a[75]); // Sum(134) + res[135] = fma52lo(res[135], a[60], a[75]); // Sum(135) + res[136] = fma52hi(res[136], a[60], a[75]); // Sum(135) + res[136] = fma52lo(res[136], a[61], a[75]); // Sum(136) + res[137] = fma52hi(res[137], a[61], a[75]); // Sum(136) + res[137] = fma52lo(res[137], a[62], a[75]); // Sum(137) + res[138] = fma52hi(res[138], a[62], a[75]); // Sum(137) + res[138] = fma52lo(res[138], a[63], a[75]); // Sum(138) + res[139] = fma52hi(res[139], a[63], a[75]); // Sum(138) + res[139] = fma52lo(res[139], a[64], a[75]); // Sum(139) + res[140] = fma52hi(res[140], a[64], a[75]); // Sum(139) + res[140] = fma52lo(res[140], a[65], a[75]); // Sum(140) + res[141] = fma52hi(res[141], a[65], a[75]); // Sum(140) + res[141] = fma52lo(res[141], a[66], a[75]); // Sum(141) + res[142] = fma52hi(res[142], a[66], a[75]); // Sum(141) + res[142] = fma52lo(res[142], a[67], a[75]); // Sum(142) + res[143] = fma52hi(res[143], a[67], a[75]); // Sum(142) + res[143] = fma52lo(res[143], a[68], a[75]); // Sum(143) + res[144] = fma52hi(res[144], a[68], a[75]); // Sum(143) + res[132] = fma52lo(res[132], a[56], a[76]); // Sum(132) + res[133] = fma52hi(res[133], a[56], a[76]); // Sum(132) + res[133] = fma52lo(res[133], a[57], a[76]); // Sum(133) + res[134] = fma52hi(res[134], a[57], a[76]); // Sum(133) + res[134] = fma52lo(res[134], a[58], a[76]); // Sum(134) + res[135] = fma52hi(res[135], a[58], a[76]); // Sum(134) + res[135] = fma52lo(res[135], a[59], a[76]); // Sum(135) + res[136] = fma52hi(res[136], a[59], a[76]); // Sum(135) + res[136] = fma52lo(res[136], a[60], a[76]); // Sum(136) + res[137] = fma52hi(res[137], a[60], a[76]); // Sum(136) + res[137] = fma52lo(res[137], a[61], a[76]); // Sum(137) + res[138] = fma52hi(res[138], a[61], a[76]); // Sum(137) + res[138] = fma52lo(res[138], a[62], a[76]); // Sum(138) + res[139] = fma52hi(res[139], a[62], a[76]); // Sum(138) + res[139] = fma52lo(res[139], a[63], a[76]); // Sum(139) + res[140] = fma52hi(res[140], a[63], a[76]); // Sum(139) + res[140] = fma52lo(res[140], a[64], a[76]); // Sum(140) + res[141] = fma52hi(res[141], a[64], a[76]); // Sum(140) + res[141] = fma52lo(res[141], a[65], a[76]); // Sum(141) + res[142] = fma52hi(res[142], a[65], a[76]); // Sum(141) + res[142] = fma52lo(res[142], a[66], a[76]); // Sum(142) + res[143] = fma52hi(res[143], a[66], a[76]); // Sum(142) + res[143] = fma52lo(res[143], a[67], a[76]); // Sum(143) + res[144] = fma52hi(res[144], a[67], a[76]); // Sum(143) + res[132] = fma52lo(res[132], a[55], a[77]); // Sum(132) + res[133] = fma52hi(res[133], a[55], a[77]); // Sum(132) + res[133] = fma52lo(res[133], a[56], a[77]); // Sum(133) + res[134] = fma52hi(res[134], a[56], a[77]); // Sum(133) + res[134] = fma52lo(res[134], a[57], a[77]); // Sum(134) + res[135] = fma52hi(res[135], a[57], a[77]); // Sum(134) + res[135] = fma52lo(res[135], a[58], a[77]); // Sum(135) + res[136] = fma52hi(res[136], a[58], a[77]); // Sum(135) + res[136] = fma52lo(res[136], a[59], a[77]); // Sum(136) + res[137] = fma52hi(res[137], a[59], a[77]); // Sum(136) + res[137] = fma52lo(res[137], a[60], a[77]); // Sum(137) + res[138] = fma52hi(res[138], a[60], a[77]); // Sum(137) + res[138] = fma52lo(res[138], a[61], a[77]); // Sum(138) + res[139] = fma52hi(res[139], a[61], a[77]); // Sum(138) + res[139] = fma52lo(res[139], a[62], a[77]); // Sum(139) + res[140] = fma52hi(res[140], a[62], a[77]); // Sum(139) + res[140] = fma52lo(res[140], a[63], a[77]); // Sum(140) + res[141] = fma52hi(res[141], a[63], a[77]); // Sum(140) + res[141] = fma52lo(res[141], a[64], a[77]); // Sum(141) + res[142] = fma52hi(res[142], a[64], a[77]); // Sum(141) + res[142] = fma52lo(res[142], a[65], a[77]); // Sum(142) + res[143] = fma52hi(res[143], a[65], a[77]); // Sum(142) + res[143] = fma52lo(res[143], a[66], a[77]); // Sum(143) + res[144] = fma52hi(res[144], a[66], a[77]); // Sum(143) + res[132] = fma52lo(res[132], a[54], a[78]); // Sum(132) + res[133] = fma52hi(res[133], a[54], a[78]); // Sum(132) + res[133] = fma52lo(res[133], a[55], a[78]); // Sum(133) + res[134] = fma52hi(res[134], a[55], a[78]); // Sum(133) + res[134] = fma52lo(res[134], a[56], a[78]); // Sum(134) + res[135] = fma52hi(res[135], a[56], a[78]); // Sum(134) + res[135] = fma52lo(res[135], a[57], a[78]); // Sum(135) + res[136] = fma52hi(res[136], a[57], a[78]); // Sum(135) + res[136] = fma52lo(res[136], a[58], a[78]); // Sum(136) + res[137] = fma52hi(res[137], a[58], a[78]); // Sum(136) + res[137] = fma52lo(res[137], a[59], a[78]); // Sum(137) + res[138] = fma52hi(res[138], a[59], a[78]); // Sum(137) + res[138] = fma52lo(res[138], a[60], a[78]); // Sum(138) + res[139] = fma52hi(res[139], a[60], a[78]); // Sum(138) + res[139] = fma52lo(res[139], a[61], a[78]); // Sum(139) + res[140] = fma52hi(res[140], a[61], a[78]); // Sum(139) + res[140] = fma52lo(res[140], a[62], a[78]); // Sum(140) + res[141] = fma52hi(res[141], a[62], a[78]); // Sum(140) + res[141] = fma52lo(res[141], a[63], a[78]); // Sum(141) + res[142] = fma52hi(res[142], a[63], a[78]); // Sum(141) + res[142] = fma52lo(res[142], a[64], a[78]); // Sum(142) + res[143] = fma52hi(res[143], a[64], a[78]); // Sum(142) + res[143] = fma52lo(res[143], a[65], a[78]); // Sum(143) + res[144] = fma52hi(res[144], a[65], a[78]); // Sum(143) + res[132] = add64(res[132], res[132]); // Double(132) + res[133] = add64(res[133], res[133]); // Double(133) + res[134] = add64(res[134], res[134]); // Double(134) + res[135] = add64(res[135], res[135]); // Double(135) + res[136] = add64(res[136], res[136]); // Double(136) + res[137] = add64(res[137], res[137]); // Double(137) + res[138] = add64(res[138], res[138]); // Double(138) + res[139] = add64(res[139], res[139]); // Double(139) + res[140] = add64(res[140], res[140]); // Double(140) + res[141] = add64(res[141], res[141]); // Double(141) + res[142] = add64(res[142], res[142]); // Double(142) + res[143] = add64(res[143], res[143]); // Double(143) + res[132] = fma52lo(res[132], a[66], a[66]); // Add sqr(132) + res[133] = fma52hi(res[133], a[66], a[66]); // Add sqr(132) + res[134] = fma52lo(res[134], a[67], a[67]); // Add sqr(134) + res[135] = fma52hi(res[135], a[67], a[67]); // Add sqr(134) + res[136] = fma52lo(res[136], a[68], a[68]); // Add sqr(136) + res[137] = fma52hi(res[137], a[68], a[68]); // Add sqr(136) + res[138] = fma52lo(res[138], a[69], a[69]); // Add sqr(138) + res[139] = fma52hi(res[139], a[69], a[69]); // Add sqr(138) + res[140] = fma52lo(res[140], a[70], a[70]); // Add sqr(140) + res[141] = fma52hi(res[141], a[70], a[70]); // Add sqr(140) + res[142] = fma52lo(res[142], a[71], a[71]); // Add sqr(142) + res[143] = fma52hi(res[143], a[71], a[71]); // Add sqr(142) + res[144] = fma52lo(res[144], a[71], a[73]); // Sum(144) + res[145] = fma52hi(res[145], a[71], a[73]); // Sum(144) + res[145] = fma52lo(res[145], a[72], a[73]); // Sum(145) + res[146] = fma52hi(res[146], a[72], a[73]); // Sum(145) + res[144] = fma52lo(res[144], a[70], a[74]); // Sum(144) + res[145] = fma52hi(res[145], a[70], a[74]); // Sum(144) + res[145] = fma52lo(res[145], a[71], a[74]); // Sum(145) + res[146] = fma52hi(res[146], a[71], a[74]); // Sum(145) + res[146] = fma52lo(res[146], a[72], a[74]); // Sum(146) + res[147] = fma52hi(res[147], a[72], a[74]); // Sum(146) + res[147] = fma52lo(res[147], a[73], a[74]); // Sum(147) + res[148] = fma52hi(res[148], a[73], a[74]); // Sum(147) + res[144] = fma52lo(res[144], a[69], a[75]); // Sum(144) + res[145] = fma52hi(res[145], a[69], a[75]); // Sum(144) + res[145] = fma52lo(res[145], a[70], a[75]); // Sum(145) + res[146] = fma52hi(res[146], a[70], a[75]); // Sum(145) + res[146] = fma52lo(res[146], a[71], a[75]); // Sum(146) + res[147] = fma52hi(res[147], a[71], a[75]); // Sum(146) + res[147] = fma52lo(res[147], a[72], a[75]); // Sum(147) + res[148] = fma52hi(res[148], a[72], a[75]); // Sum(147) + res[148] = fma52lo(res[148], a[73], a[75]); // Sum(148) + res[149] = fma52hi(res[149], a[73], a[75]); // Sum(148) + res[149] = fma52lo(res[149], a[74], a[75]); // Sum(149) + res[150] = fma52hi(res[150], a[74], a[75]); // Sum(149) + res[144] = fma52lo(res[144], a[68], a[76]); // Sum(144) + res[145] = fma52hi(res[145], a[68], a[76]); // Sum(144) + res[145] = fma52lo(res[145], a[69], a[76]); // Sum(145) + res[146] = fma52hi(res[146], a[69], a[76]); // Sum(145) + res[146] = fma52lo(res[146], a[70], a[76]); // Sum(146) + res[147] = fma52hi(res[147], a[70], a[76]); // Sum(146) + res[147] = fma52lo(res[147], a[71], a[76]); // Sum(147) + res[148] = fma52hi(res[148], a[71], a[76]); // Sum(147) + res[148] = fma52lo(res[148], a[72], a[76]); // Sum(148) + res[149] = fma52hi(res[149], a[72], a[76]); // Sum(148) + res[149] = fma52lo(res[149], a[73], a[76]); // Sum(149) + res[150] = fma52hi(res[150], a[73], a[76]); // Sum(149) + res[150] = fma52lo(res[150], a[74], a[76]); // Sum(150) + res[151] = fma52hi(res[151], a[74], a[76]); // Sum(150) + res[151] = fma52lo(res[151], a[75], a[76]); // Sum(151) + res[152] = fma52hi(res[152], a[75], a[76]); // Sum(151) + res[144] = fma52lo(res[144], a[67], a[77]); // Sum(144) + res[145] = fma52hi(res[145], a[67], a[77]); // Sum(144) + res[145] = fma52lo(res[145], a[68], a[77]); // Sum(145) + res[146] = fma52hi(res[146], a[68], a[77]); // Sum(145) + res[146] = fma52lo(res[146], a[69], a[77]); // Sum(146) + res[147] = fma52hi(res[147], a[69], a[77]); // Sum(146) + res[147] = fma52lo(res[147], a[70], a[77]); // Sum(147) + res[148] = fma52hi(res[148], a[70], a[77]); // Sum(147) + res[148] = fma52lo(res[148], a[71], a[77]); // Sum(148) + res[149] = fma52hi(res[149], a[71], a[77]); // Sum(148) + res[149] = fma52lo(res[149], a[72], a[77]); // Sum(149) + res[150] = fma52hi(res[150], a[72], a[77]); // Sum(149) + res[150] = fma52lo(res[150], a[73], a[77]); // Sum(150) + res[151] = fma52hi(res[151], a[73], a[77]); // Sum(150) + res[151] = fma52lo(res[151], a[74], a[77]); // Sum(151) + res[152] = fma52hi(res[152], a[74], a[77]); // Sum(151) + res[152] = fma52lo(res[152], a[75], a[77]); // Sum(152) + res[153] = fma52hi(res[153], a[75], a[77]); // Sum(152) + res[153] = fma52lo(res[153], a[76], a[77]); // Sum(153) + res[154] = fma52hi(res[154], a[76], a[77]); // Sum(153) + res[144] = fma52lo(res[144], a[66], a[78]); // Sum(144) + res[145] = fma52hi(res[145], a[66], a[78]); // Sum(144) + res[145] = fma52lo(res[145], a[67], a[78]); // Sum(145) + res[146] = fma52hi(res[146], a[67], a[78]); // Sum(145) + res[146] = fma52lo(res[146], a[68], a[78]); // Sum(146) + res[147] = fma52hi(res[147], a[68], a[78]); // Sum(146) + res[147] = fma52lo(res[147], a[69], a[78]); // Sum(147) + res[148] = fma52hi(res[148], a[69], a[78]); // Sum(147) + res[148] = fma52lo(res[148], a[70], a[78]); // Sum(148) + res[149] = fma52hi(res[149], a[70], a[78]); // Sum(148) + res[149] = fma52lo(res[149], a[71], a[78]); // Sum(149) + res[150] = fma52hi(res[150], a[71], a[78]); // Sum(149) + res[150] = fma52lo(res[150], a[72], a[78]); // Sum(150) + res[151] = fma52hi(res[151], a[72], a[78]); // Sum(150) + res[151] = fma52lo(res[151], a[73], a[78]); // Sum(151) + res[152] = fma52hi(res[152], a[73], a[78]); // Sum(151) + res[152] = fma52lo(res[152], a[74], a[78]); // Sum(152) + res[153] = fma52hi(res[153], a[74], a[78]); // Sum(152) + res[153] = fma52lo(res[153], a[75], a[78]); // Sum(153) + res[154] = fma52hi(res[154], a[75], a[78]); // Sum(153) + res[154] = fma52lo(res[154], a[76], a[78]); // Sum(154) + res[155] = fma52hi(res[155], a[76], a[78]); // Sum(154) + res[155] = fma52lo(res[155], a[77], a[78]); // Sum(155) + res[156] = fma52hi(res[156], a[77], a[78]); // Sum(155) + res[144] = add64(res[144], res[144]); // Double(144) + res[145] = add64(res[145], res[145]); // Double(145) + res[146] = add64(res[146], res[146]); // Double(146) + res[147] = add64(res[147], res[147]); // Double(147) + res[148] = add64(res[148], res[148]); // Double(148) + res[149] = add64(res[149], res[149]); // Double(149) + res[150] = add64(res[150], res[150]); // Double(150) + res[151] = add64(res[151], res[151]); // Double(151) + res[152] = add64(res[152], res[152]); // Double(152) + res[153] = add64(res[153], res[153]); // Double(153) + res[154] = add64(res[154], res[154]); // Double(154) + res[155] = add64(res[155], res[155]); // Double(155) + res[144] = fma52lo(res[144], a[72], a[72]); // Add sqr(144) + res[145] = fma52hi(res[145], a[72], a[72]); // Add sqr(144) + res[146] = fma52lo(res[146], a[73], a[73]); // Add sqr(146) + res[147] = fma52hi(res[147], a[73], a[73]); // Add sqr(146) + res[148] = fma52lo(res[148], a[74], a[74]); // Add sqr(148) + res[149] = fma52hi(res[149], a[74], a[74]); // Add sqr(148) + res[150] = fma52lo(res[150], a[75], a[75]); // Add sqr(150) + res[151] = fma52hi(res[151], a[75], a[75]); // Add sqr(150) + res[152] = fma52lo(res[152], a[76], a[76]); // Add sqr(152) + res[153] = fma52hi(res[153], a[76], a[76]); // Add sqr(152) + res[154] = fma52lo(res[154], a[77], a[77]); // Add sqr(154) + res[155] = fma52hi(res[155], a[77], a[77]); // Add sqr(154) + res[156] = add64(res[156], res[156]); // Double(156) + res[156] = fma52lo(res[156], a[78], a[78]); // Add sqr(156) + res[157] = fma52hi(res[157], a[78], a[78]); // Add sqr(156) + + // Montgomery Reduction + int it; + for (it = 0; it < 80; it += 10) { // Reduction step + int jt = 0; + if ((it + 0) > 0) + res[it + 0] = add64(res[it + 0], srli64(res[it + -1], DIGIT_SIZE)); + u[it + 0] = mul52lo(res[it + 0], k); + res[it + jt + 0] = fma52lo(res[it + jt + 0], u[it + 0], m[jt + 0]); + res[it + jt + 1] = fma52hi(res[it + jt + 1], u[it + 0], m[jt + 0]); + res[it + jt + 1] = fma52lo(res[it + jt + 1], u[it + 0], m[jt + 1]); + res[it + jt + 2] = fma52hi(res[it + jt + 2], u[it + 0], m[jt + 1]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 0], m[jt + 2]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 0], m[jt + 2]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 0], m[jt + 3]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 0], m[jt + 3]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 0], m[jt + 4]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 0], m[jt + 4]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 0], m[jt + 5]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 0], m[jt + 5]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 0], m[jt + 6]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 0], m[jt + 6]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 0], m[jt + 7]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 0], m[jt + 7]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 0], m[jt + 8]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 0], m[jt + 8]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 0], m[jt + 9]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 0], m[jt + 9]); + res[it + 1] = add64(res[it + 1], srli64(res[it + 0], DIGIT_SIZE)); + u[it + 1] = mul52lo(res[it + 1], k); + res[it + jt + 1] = fma52lo(res[it + jt + 1], u[it + 1], m[jt + 0]); + res[it + jt + 2] = fma52hi(res[it + jt + 2], u[it + 1], m[jt + 0]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 1], m[jt + 1]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 1], m[jt + 1]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 1], m[jt + 2]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 1], m[jt + 2]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 1], m[jt + 3]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 1], m[jt + 3]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 1], m[jt + 4]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 1], m[jt + 4]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 1], m[jt + 5]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 1], m[jt + 5]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 1], m[jt + 6]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 1], m[jt + 6]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 1], m[jt + 7]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 1], m[jt + 7]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 1], m[jt + 8]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 1], m[jt + 8]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 1], m[jt + 9]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 1], m[jt + 9]); + res[it + 2] = add64(res[it + 2], srli64(res[it + 1], DIGIT_SIZE)); + u[it + 2] = mul52lo(res[it + 2], k); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 2], m[jt + 0]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 2], m[jt + 0]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 2], m[jt + 1]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 2], m[jt + 1]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 2], m[jt + 2]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 2], m[jt + 2]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 2], m[jt + 3]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 2], m[jt + 3]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 2], m[jt + 4]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 2], m[jt + 4]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 2], m[jt + 5]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 2], m[jt + 5]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 2], m[jt + 6]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 2], m[jt + 6]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 2], m[jt + 7]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 2], m[jt + 7]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 2], m[jt + 8]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 2], m[jt + 8]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 2], m[jt + 9]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 2], m[jt + 9]); + res[it + 3] = add64(res[it + 3], srli64(res[it + 2], DIGIT_SIZE)); + u[it + 3] = mul52lo(res[it + 3], k); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 3], m[jt + 0]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 3], m[jt + 0]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 3], m[jt + 1]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 3], m[jt + 1]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 3], m[jt + 2]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 3], m[jt + 2]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 3], m[jt + 3]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 3], m[jt + 3]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 3], m[jt + 4]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 3], m[jt + 4]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 3], m[jt + 5]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 3], m[jt + 5]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 3], m[jt + 6]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 3], m[jt + 6]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 3], m[jt + 7]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 3], m[jt + 7]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 3], m[jt + 8]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 3], m[jt + 8]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 3], m[jt + 9]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 3], m[jt + 9]); + res[it + 4] = add64(res[it + 4], srli64(res[it + 3], DIGIT_SIZE)); + u[it + 4] = mul52lo(res[it + 4], k); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 4], m[jt + 0]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 4], m[jt + 0]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 4], m[jt + 1]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 4], m[jt + 1]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 4], m[jt + 2]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 4], m[jt + 2]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 4], m[jt + 3]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 4], m[jt + 3]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 4], m[jt + 4]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 4], m[jt + 4]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 4], m[jt + 5]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 4], m[jt + 5]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 4], m[jt + 6]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 4], m[jt + 6]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 4], m[jt + 7]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 4], m[jt + 7]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 4], m[jt + 8]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 4], m[jt + 8]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 4], m[jt + 9]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 4], m[jt + 9]); + res[it + 5] = add64(res[it + 5], srli64(res[it + 4], DIGIT_SIZE)); + u[it + 5] = mul52lo(res[it + 5], k); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 5], m[jt + 0]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 5], m[jt + 0]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 5], m[jt + 1]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 5], m[jt + 1]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 5], m[jt + 2]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 5], m[jt + 2]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 5], m[jt + 3]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 5], m[jt + 3]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 5], m[jt + 4]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 5], m[jt + 4]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 5], m[jt + 5]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 5], m[jt + 5]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 5], m[jt + 6]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 5], m[jt + 6]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 5], m[jt + 7]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 5], m[jt + 7]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 5], m[jt + 8]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 5], m[jt + 8]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 5], m[jt + 9]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 5], m[jt + 9]); + res[it + 6] = add64(res[it + 6], srli64(res[it + 5], DIGIT_SIZE)); + u[it + 6] = mul52lo(res[it + 6], k); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 6], m[jt + 0]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 6], m[jt + 0]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 6], m[jt + 1]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 6], m[jt + 1]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 6], m[jt + 2]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 6], m[jt + 2]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 6], m[jt + 3]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 6], m[jt + 3]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 6], m[jt + 4]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 6], m[jt + 4]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 6], m[jt + 5]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 6], m[jt + 5]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 6], m[jt + 6]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 6], m[jt + 6]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 6], m[jt + 7]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 6], m[jt + 7]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 6], m[jt + 8]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 6], m[jt + 8]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 6], m[jt + 9]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 6], m[jt + 9]); + res[it + 7] = add64(res[it + 7], srli64(res[it + 6], DIGIT_SIZE)); + u[it + 7] = mul52lo(res[it + 7], k); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 7], m[jt + 0]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 7], m[jt + 0]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 7], m[jt + 1]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 7], m[jt + 1]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 7], m[jt + 2]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 7], m[jt + 2]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 7], m[jt + 3]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 7], m[jt + 3]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 7], m[jt + 4]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 7], m[jt + 4]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 7], m[jt + 5]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 7], m[jt + 5]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 7], m[jt + 6]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 7], m[jt + 6]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 7], m[jt + 7]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 7], m[jt + 7]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 7], m[jt + 8]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 7], m[jt + 8]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 7], m[jt + 9]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 7], m[jt + 9]); + res[it + 8] = add64(res[it + 8], srli64(res[it + 7], DIGIT_SIZE)); + u[it + 8] = mul52lo(res[it + 8], k); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 8], m[jt + 0]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 8], m[jt + 0]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 8], m[jt + 1]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 8], m[jt + 1]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 8], m[jt + 2]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 8], m[jt + 2]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 8], m[jt + 3]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 8], m[jt + 3]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 8], m[jt + 4]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 8], m[jt + 4]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 8], m[jt + 5]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 8], m[jt + 5]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 8], m[jt + 6]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 8], m[jt + 6]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 8], m[jt + 7]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 8], m[jt + 7]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 8], m[jt + 8]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 8], m[jt + 8]); + res[it + jt + 17] = fma52lo(res[it + jt + 17], u[it + 8], m[jt + 9]); + res[it + jt + 18] = fma52hi(res[it + jt + 18], u[it + 8], m[jt + 9]); + res[it + 9] = add64(res[it + 9], srli64(res[it + 8], DIGIT_SIZE)); + u[it + 9] = (it + 9 < 79) ? mul52lo(res[it + 9], k) : get_zero64(); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 9], m[jt + 0]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 9], m[jt + 0]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 9], m[jt + 1]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 9], m[jt + 1]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 9], m[jt + 2]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 9], m[jt + 2]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 9], m[jt + 3]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 9], m[jt + 3]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 9], m[jt + 4]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 9], m[jt + 4]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 9], m[jt + 5]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 9], m[jt + 5]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 9], m[jt + 6]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 9], m[jt + 6]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 9], m[jt + 7]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 9], m[jt + 7]); + res[it + jt + 17] = fma52lo(res[it + jt + 17], u[it + 9], m[jt + 8]); + res[it + jt + 18] = fma52hi(res[it + jt + 18], u[it + 9], m[jt + 8]); + res[it + jt + 18] = fma52lo(res[it + jt + 18], u[it + 9], m[jt + 9]); + res[it + jt + 19] = fma52hi(res[it + jt + 19], u[it + 9], m[jt + 9]); + + for (jt = 10; jt < 80; jt += 10) { // Poly tile + res[it + jt + 0] = fma52lo(res[it + jt + 0], u[it + 0], m[jt + 0]); + res[it + jt + 1] = fma52hi(res[it + jt + 1], u[it + 0], m[jt + 0]); + res[it + jt + 1] = fma52lo(res[it + jt + 1], u[it + 0], m[jt + 1]); + res[it + jt + 2] = fma52hi(res[it + jt + 2], u[it + 0], m[jt + 1]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 0], m[jt + 2]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 0], m[jt + 2]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 0], m[jt + 3]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 0], m[jt + 3]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 0], m[jt + 4]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 0], m[jt + 4]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 0], m[jt + 5]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 0], m[jt + 5]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 0], m[jt + 6]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 0], m[jt + 6]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 0], m[jt + 7]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 0], m[jt + 7]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 0], m[jt + 8]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 0], m[jt + 8]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 0], m[jt + 9]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 0], m[jt + 9]); + res[it + jt + 1] = fma52lo(res[it + jt + 1], u[it + 1], m[jt + 0]); + res[it + jt + 2] = fma52hi(res[it + jt + 2], u[it + 1], m[jt + 0]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 1], m[jt + 1]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 1], m[jt + 1]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 1], m[jt + 2]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 1], m[jt + 2]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 1], m[jt + 3]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 1], m[jt + 3]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 1], m[jt + 4]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 1], m[jt + 4]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 1], m[jt + 5]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 1], m[jt + 5]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 1], m[jt + 6]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 1], m[jt + 6]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 1], m[jt + 7]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 1], m[jt + 7]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 1], m[jt + 8]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 1], m[jt + 8]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 1], m[jt + 9]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 1], m[jt + 9]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 2], m[jt + 0]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 2], m[jt + 0]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 2], m[jt + 1]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 2], m[jt + 1]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 2], m[jt + 2]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 2], m[jt + 2]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 2], m[jt + 3]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 2], m[jt + 3]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 2], m[jt + 4]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 2], m[jt + 4]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 2], m[jt + 5]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 2], m[jt + 5]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 2], m[jt + 6]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 2], m[jt + 6]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 2], m[jt + 7]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 2], m[jt + 7]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 2], m[jt + 8]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 2], m[jt + 8]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 2], m[jt + 9]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 2], m[jt + 9]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 3], m[jt + 0]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 3], m[jt + 0]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 3], m[jt + 1]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 3], m[jt + 1]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 3], m[jt + 2]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 3], m[jt + 2]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 3], m[jt + 3]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 3], m[jt + 3]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 3], m[jt + 4]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 3], m[jt + 4]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 3], m[jt + 5]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 3], m[jt + 5]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 3], m[jt + 6]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 3], m[jt + 6]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 3], m[jt + 7]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 3], m[jt + 7]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 3], m[jt + 8]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 3], m[jt + 8]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 3], m[jt + 9]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 3], m[jt + 9]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 4], m[jt + 0]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 4], m[jt + 0]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 4], m[jt + 1]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 4], m[jt + 1]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 4], m[jt + 2]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 4], m[jt + 2]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 4], m[jt + 3]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 4], m[jt + 3]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 4], m[jt + 4]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 4], m[jt + 4]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 4], m[jt + 5]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 4], m[jt + 5]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 4], m[jt + 6]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 4], m[jt + 6]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 4], m[jt + 7]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 4], m[jt + 7]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 4], m[jt + 8]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 4], m[jt + 8]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 4], m[jt + 9]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 4], m[jt + 9]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 5], m[jt + 0]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 5], m[jt + 0]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 5], m[jt + 1]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 5], m[jt + 1]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 5], m[jt + 2]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 5], m[jt + 2]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 5], m[jt + 3]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 5], m[jt + 3]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 5], m[jt + 4]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 5], m[jt + 4]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 5], m[jt + 5]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 5], m[jt + 5]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 5], m[jt + 6]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 5], m[jt + 6]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 5], m[jt + 7]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 5], m[jt + 7]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 5], m[jt + 8]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 5], m[jt + 8]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 5], m[jt + 9]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 5], m[jt + 9]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 6], m[jt + 0]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 6], m[jt + 0]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 6], m[jt + 1]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 6], m[jt + 1]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 6], m[jt + 2]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 6], m[jt + 2]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 6], m[jt + 3]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 6], m[jt + 3]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 6], m[jt + 4]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 6], m[jt + 4]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 6], m[jt + 5]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 6], m[jt + 5]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 6], m[jt + 6]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 6], m[jt + 6]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 6], m[jt + 7]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 6], m[jt + 7]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 6], m[jt + 8]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 6], m[jt + 8]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 6], m[jt + 9]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 6], m[jt + 9]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 7], m[jt + 0]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 7], m[jt + 0]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 7], m[jt + 1]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 7], m[jt + 1]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 7], m[jt + 2]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 7], m[jt + 2]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 7], m[jt + 3]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 7], m[jt + 3]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 7], m[jt + 4]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 7], m[jt + 4]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 7], m[jt + 5]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 7], m[jt + 5]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 7], m[jt + 6]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 7], m[jt + 6]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 7], m[jt + 7]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 7], m[jt + 7]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 7], m[jt + 8]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 7], m[jt + 8]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 7], m[jt + 9]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 7], m[jt + 9]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 8], m[jt + 0]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 8], m[jt + 0]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 8], m[jt + 1]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 8], m[jt + 1]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 8], m[jt + 2]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 8], m[jt + 2]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 8], m[jt + 3]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 8], m[jt + 3]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 8], m[jt + 4]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 8], m[jt + 4]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 8], m[jt + 5]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 8], m[jt + 5]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 8], m[jt + 6]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 8], m[jt + 6]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 8], m[jt + 7]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 8], m[jt + 7]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 8], m[jt + 8]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 8], m[jt + 8]); + res[it + jt + 17] = fma52lo(res[it + jt + 17], u[it + 8], m[jt + 9]); + res[it + jt + 18] = fma52hi(res[it + jt + 18], u[it + 8], m[jt + 9]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 9], m[jt + 0]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 9], m[jt + 0]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 9], m[jt + 1]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 9], m[jt + 1]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 9], m[jt + 2]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 9], m[jt + 2]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 9], m[jt + 3]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 9], m[jt + 3]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 9], m[jt + 4]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 9], m[jt + 4]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 9], m[jt + 5]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 9], m[jt + 5]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 9], m[jt + 6]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 9], m[jt + 6]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 9], m[jt + 7]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 9], m[jt + 7]); + res[it + jt + 17] = fma52lo(res[it + jt + 17], u[it + 9], m[jt + 8]); + res[it + jt + 18] = fma52hi(res[it + jt + 18], u[it + 9], m[jt + 8]); + res[it + jt + 18] = fma52lo(res[it + jt + 18], u[it + 9], m[jt + 9]); + res[it + jt + 19] = fma52hi(res[it + jt + 19], u[it + 9], m[jt + 9]); + } + } + + // Normalization + r[0] = and64_const(res[79], DIGIT_MASK); + res[80] = add64(res[80], srli64(res[79], DIGIT_SIZE)); + r[1] = and64_const(res[80], DIGIT_MASK); + res[81] = add64(res[81], srli64(res[80], DIGIT_SIZE)); + r[2] = and64_const(res[81], DIGIT_MASK); + res[82] = add64(res[82], srli64(res[81], DIGIT_SIZE)); + r[3] = and64_const(res[82], DIGIT_MASK); + res[83] = add64(res[83], srli64(res[82], DIGIT_SIZE)); + r[4] = and64_const(res[83], DIGIT_MASK); + res[84] = add64(res[84], srli64(res[83], DIGIT_SIZE)); + r[5] = and64_const(res[84], DIGIT_MASK); + res[85] = add64(res[85], srli64(res[84], DIGIT_SIZE)); + r[6] = and64_const(res[85], DIGIT_MASK); + res[86] = add64(res[86], srli64(res[85], DIGIT_SIZE)); + r[7] = and64_const(res[86], DIGIT_MASK); + res[87] = add64(res[87], srli64(res[86], DIGIT_SIZE)); + r[8] = and64_const(res[87], DIGIT_MASK); + res[88] = add64(res[88], srli64(res[87], DIGIT_SIZE)); + r[9] = and64_const(res[88], DIGIT_MASK); + res[89] = add64(res[89], srli64(res[88], DIGIT_SIZE)); + r[10] = and64_const(res[89], DIGIT_MASK); + res[90] = add64(res[90], srli64(res[89], DIGIT_SIZE)); + r[11] = and64_const(res[90], DIGIT_MASK); + res[91] = add64(res[91], srli64(res[90], DIGIT_SIZE)); + r[12] = and64_const(res[91], DIGIT_MASK); + res[92] = add64(res[92], srli64(res[91], DIGIT_SIZE)); + r[13] = and64_const(res[92], DIGIT_MASK); + res[93] = add64(res[93], srli64(res[92], DIGIT_SIZE)); + r[14] = and64_const(res[93], DIGIT_MASK); + res[94] = add64(res[94], srli64(res[93], DIGIT_SIZE)); + r[15] = and64_const(res[94], DIGIT_MASK); + res[95] = add64(res[95], srli64(res[94], DIGIT_SIZE)); + r[16] = and64_const(res[95], DIGIT_MASK); + res[96] = add64(res[96], srli64(res[95], DIGIT_SIZE)); + r[17] = and64_const(res[96], DIGIT_MASK); + res[97] = add64(res[97], srli64(res[96], DIGIT_SIZE)); + r[18] = and64_const(res[97], DIGIT_MASK); + res[98] = add64(res[98], srli64(res[97], DIGIT_SIZE)); + r[19] = and64_const(res[98], DIGIT_MASK); + res[99] = add64(res[99], srli64(res[98], DIGIT_SIZE)); + r[20] = and64_const(res[99], DIGIT_MASK); + res[100] = add64(res[100], srli64(res[99], DIGIT_SIZE)); + r[21] = and64_const(res[100], DIGIT_MASK); + res[101] = add64(res[101], srli64(res[100], DIGIT_SIZE)); + r[22] = and64_const(res[101], DIGIT_MASK); + res[102] = add64(res[102], srli64(res[101], DIGIT_SIZE)); + r[23] = and64_const(res[102], DIGIT_MASK); + res[103] = add64(res[103], srli64(res[102], DIGIT_SIZE)); + r[24] = and64_const(res[103], DIGIT_MASK); + res[104] = add64(res[104], srli64(res[103], DIGIT_SIZE)); + r[25] = and64_const(res[104], DIGIT_MASK); + res[105] = add64(res[105], srli64(res[104], DIGIT_SIZE)); + r[26] = and64_const(res[105], DIGIT_MASK); + res[106] = add64(res[106], srli64(res[105], DIGIT_SIZE)); + r[27] = and64_const(res[106], DIGIT_MASK); + res[107] = add64(res[107], srli64(res[106], DIGIT_SIZE)); + r[28] = and64_const(res[107], DIGIT_MASK); + res[108] = add64(res[108], srli64(res[107], DIGIT_SIZE)); + r[29] = and64_const(res[108], DIGIT_MASK); + res[109] = add64(res[109], srli64(res[108], DIGIT_SIZE)); + r[30] = and64_const(res[109], DIGIT_MASK); + res[110] = add64(res[110], srli64(res[109], DIGIT_SIZE)); + r[31] = and64_const(res[110], DIGIT_MASK); + res[111] = add64(res[111], srli64(res[110], DIGIT_SIZE)); + r[32] = and64_const(res[111], DIGIT_MASK); + res[112] = add64(res[112], srli64(res[111], DIGIT_SIZE)); + r[33] = and64_const(res[112], DIGIT_MASK); + res[113] = add64(res[113], srli64(res[112], DIGIT_SIZE)); + r[34] = and64_const(res[113], DIGIT_MASK); + res[114] = add64(res[114], srli64(res[113], DIGIT_SIZE)); + r[35] = and64_const(res[114], DIGIT_MASK); + res[115] = add64(res[115], srli64(res[114], DIGIT_SIZE)); + r[36] = and64_const(res[115], DIGIT_MASK); + res[116] = add64(res[116], srli64(res[115], DIGIT_SIZE)); + r[37] = and64_const(res[116], DIGIT_MASK); + res[117] = add64(res[117], srli64(res[116], DIGIT_SIZE)); + r[38] = and64_const(res[117], DIGIT_MASK); + res[118] = add64(res[118], srli64(res[117], DIGIT_SIZE)); + r[39] = and64_const(res[118], DIGIT_MASK); + res[119] = add64(res[119], srli64(res[118], DIGIT_SIZE)); + r[40] = and64_const(res[119], DIGIT_MASK); + res[120] = add64(res[120], srli64(res[119], DIGIT_SIZE)); + r[41] = and64_const(res[120], DIGIT_MASK); + res[121] = add64(res[121], srli64(res[120], DIGIT_SIZE)); + r[42] = and64_const(res[121], DIGIT_MASK); + res[122] = add64(res[122], srli64(res[121], DIGIT_SIZE)); + r[43] = and64_const(res[122], DIGIT_MASK); + res[123] = add64(res[123], srli64(res[122], DIGIT_SIZE)); + r[44] = and64_const(res[123], DIGIT_MASK); + res[124] = add64(res[124], srli64(res[123], DIGIT_SIZE)); + r[45] = and64_const(res[124], DIGIT_MASK); + res[125] = add64(res[125], srli64(res[124], DIGIT_SIZE)); + r[46] = and64_const(res[125], DIGIT_MASK); + res[126] = add64(res[126], srli64(res[125], DIGIT_SIZE)); + r[47] = and64_const(res[126], DIGIT_MASK); + res[127] = add64(res[127], srli64(res[126], DIGIT_SIZE)); + r[48] = and64_const(res[127], DIGIT_MASK); + res[128] = add64(res[128], srli64(res[127], DIGIT_SIZE)); + r[49] = and64_const(res[128], DIGIT_MASK); + res[129] = add64(res[129], srli64(res[128], DIGIT_SIZE)); + r[50] = and64_const(res[129], DIGIT_MASK); + res[130] = add64(res[130], srli64(res[129], DIGIT_SIZE)); + r[51] = and64_const(res[130], DIGIT_MASK); + res[131] = add64(res[131], srli64(res[130], DIGIT_SIZE)); + r[52] = and64_const(res[131], DIGIT_MASK); + res[132] = add64(res[132], srli64(res[131], DIGIT_SIZE)); + r[53] = and64_const(res[132], DIGIT_MASK); + res[133] = add64(res[133], srli64(res[132], DIGIT_SIZE)); + r[54] = and64_const(res[133], DIGIT_MASK); + res[134] = add64(res[134], srli64(res[133], DIGIT_SIZE)); + r[55] = and64_const(res[134], DIGIT_MASK); + res[135] = add64(res[135], srli64(res[134], DIGIT_SIZE)); + r[56] = and64_const(res[135], DIGIT_MASK); + res[136] = add64(res[136], srli64(res[135], DIGIT_SIZE)); + r[57] = and64_const(res[136], DIGIT_MASK); + res[137] = add64(res[137], srli64(res[136], DIGIT_SIZE)); + r[58] = and64_const(res[137], DIGIT_MASK); + res[138] = add64(res[138], srli64(res[137], DIGIT_SIZE)); + r[59] = and64_const(res[138], DIGIT_MASK); + res[139] = add64(res[139], srli64(res[138], DIGIT_SIZE)); + r[60] = and64_const(res[139], DIGIT_MASK); + res[140] = add64(res[140], srli64(res[139], DIGIT_SIZE)); + r[61] = and64_const(res[140], DIGIT_MASK); + res[141] = add64(res[141], srli64(res[140], DIGIT_SIZE)); + r[62] = and64_const(res[141], DIGIT_MASK); + res[142] = add64(res[142], srli64(res[141], DIGIT_SIZE)); + r[63] = and64_const(res[142], DIGIT_MASK); + res[143] = add64(res[143], srli64(res[142], DIGIT_SIZE)); + r[64] = and64_const(res[143], DIGIT_MASK); + res[144] = add64(res[144], srli64(res[143], DIGIT_SIZE)); + r[65] = and64_const(res[144], DIGIT_MASK); + res[145] = add64(res[145], srli64(res[144], DIGIT_SIZE)); + r[66] = and64_const(res[145], DIGIT_MASK); + res[146] = add64(res[146], srli64(res[145], DIGIT_SIZE)); + r[67] = and64_const(res[146], DIGIT_MASK); + res[147] = add64(res[147], srli64(res[146], DIGIT_SIZE)); + r[68] = and64_const(res[147], DIGIT_MASK); + res[148] = add64(res[148], srli64(res[147], DIGIT_SIZE)); + r[69] = and64_const(res[148], DIGIT_MASK); + res[149] = add64(res[149], srli64(res[148], DIGIT_SIZE)); + r[70] = and64_const(res[149], DIGIT_MASK); + res[150] = add64(res[150], srli64(res[149], DIGIT_SIZE)); + r[71] = and64_const(res[150], DIGIT_MASK); + res[151] = add64(res[151], srli64(res[150], DIGIT_SIZE)); + r[72] = and64_const(res[151], DIGIT_MASK); + res[152] = add64(res[152], srli64(res[151], DIGIT_SIZE)); + r[73] = and64_const(res[152], DIGIT_MASK); + res[153] = add64(res[153], srli64(res[152], DIGIT_SIZE)); + r[74] = and64_const(res[153], DIGIT_MASK); + res[154] = add64(res[154], srli64(res[153], DIGIT_SIZE)); + r[75] = and64_const(res[154], DIGIT_MASK); + res[155] = add64(res[155], srli64(res[154], DIGIT_SIZE)); + r[76] = and64_const(res[155], DIGIT_MASK); + res[156] = add64(res[156], srli64(res[155], DIGIT_SIZE)); + r[77] = and64_const(res[156], DIGIT_MASK); + res[157] = add64(res[157], srli64(res[156], DIGIT_SIZE)); + r[78] = and64_const(res[157], DIGIT_MASK); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams5x52x10_diagonal_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams5x52x10_diagonal_mb8.c new file mode 100644 index 0000000..08ea979 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams5x52x10_diagonal_mb8.c @@ -0,0 +1,442 @@ +/******************************************************************************* + * Copyright (C) 2019 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include + +#ifdef __GNUC__ +#define ASM(a) __asm__(a); +#else +#define ASM(a) +#endif + +void AMS5x52x10_diagonal_mb8(int64u *out_mb, const int64u *inpA_mb, + const int64u *inpM_mb, const int64u *k0_mb) { + U64 res0, res1, res2, res3, res4, res5, res6, res7, res8, res9, res10, res11, + res12, res13, res14, res15, res16, res17, res18, res19; + U64 k; + U64 *a = (U64 *)inpA_mb; + U64 *m = (U64 *)inpM_mb; + U64 *r = (U64 *)out_mb; + int iter; + const int iters = 5; + + k = loadu64((U64 *)k0_mb); + for (iter = 0; iter < iters; ++iter) { + res0 = res1 = res2 = res3 = res4 = res5 = res6 = res7 = res8 = res9 = + res10 = res11 = res12 = res13 = res14 = res15 = res16 = res17 = res18 = + res19 = get_zero64(); + // Calculate full square + res1 = fma52lo(res1, a[0], a[1]); // Sum(1) + res2 = fma52hi(res2, a[0], a[1]); // Sum(1) + res2 = fma52lo(res2, a[0], a[2]); // Sum(2) + res3 = fma52hi(res3, a[0], a[2]); // Sum(2) + res3 = fma52lo(res3, a[1], a[2]); // Sum(3) + res4 = fma52hi(res4, a[1], a[2]); // Sum(3) + res3 = fma52lo(res3, a[0], a[3]); // Sum(3) + res4 = fma52hi(res4, a[0], a[3]); // Sum(3) + res4 = fma52lo(res4, a[1], a[3]); // Sum(4) + res5 = fma52hi(res5, a[1], a[3]); // Sum(4) + res5 = fma52lo(res5, a[2], a[3]); // Sum(5) + res6 = fma52hi(res6, a[2], a[3]); // Sum(5) + res4 = fma52lo(res4, a[0], a[4]); // Sum(4) + res5 = fma52hi(res5, a[0], a[4]); // Sum(4) + res5 = fma52lo(res5, a[1], a[4]); // Sum(5) + res6 = fma52hi(res6, a[1], a[4]); // Sum(5) + res6 = fma52lo(res6, a[2], a[4]); // Sum(6) + res7 = fma52hi(res7, a[2], a[4]); // Sum(6) + res7 = fma52lo(res7, a[3], a[4]); // Sum(7) + res8 = fma52hi(res8, a[3], a[4]); // Sum(7) + res5 = fma52lo(res5, a[0], a[5]); // Sum(5) + res6 = fma52hi(res6, a[0], a[5]); // Sum(5) + res6 = fma52lo(res6, a[1], a[5]); // Sum(6) + res7 = fma52hi(res7, a[1], a[5]); // Sum(6) + res7 = fma52lo(res7, a[2], a[5]); // Sum(7) + res8 = fma52hi(res8, a[2], a[5]); // Sum(7) + res8 = fma52lo(res8, a[3], a[5]); // Sum(8) + res9 = fma52hi(res9, a[3], a[5]); // Sum(8) + res9 = fma52lo(res9, a[4], a[5]); // Sum(9) + res10 = fma52hi(res10, a[4], a[5]); // Sum(9) + res6 = fma52lo(res6, a[0], a[6]); // Sum(6) + res7 = fma52hi(res7, a[0], a[6]); // Sum(6) + res7 = fma52lo(res7, a[1], a[6]); // Sum(7) + res8 = fma52hi(res8, a[1], a[6]); // Sum(7) + res8 = fma52lo(res8, a[2], a[6]); // Sum(8) + res9 = fma52hi(res9, a[2], a[6]); // Sum(8) + res9 = fma52lo(res9, a[3], a[6]); // Sum(9) + res10 = fma52hi(res10, a[3], a[6]); // Sum(9) + res10 = fma52lo(res10, a[4], a[6]); // Sum(10) + res11 = fma52hi(res11, a[4], a[6]); // Sum(10) + res11 = fma52lo(res11, a[5], a[6]); // Sum(11) + res12 = fma52hi(res12, a[5], a[6]); // Sum(11) + res7 = fma52lo(res7, a[0], a[7]); // Sum(7) + res8 = fma52hi(res8, a[0], a[7]); // Sum(7) + res8 = fma52lo(res8, a[1], a[7]); // Sum(8) + res9 = fma52hi(res9, a[1], a[7]); // Sum(8) + res9 = fma52lo(res9, a[2], a[7]); // Sum(9) + res10 = fma52hi(res10, a[2], a[7]); // Sum(9) + res10 = fma52lo(res10, a[3], a[7]); // Sum(10) + res11 = fma52hi(res11, a[3], a[7]); // Sum(10) + res11 = fma52lo(res11, a[4], a[7]); // Sum(11) + res12 = fma52hi(res12, a[4], a[7]); // Sum(11) + res8 = fma52lo(res8, a[0], a[8]); // Sum(8) + res9 = fma52hi(res9, a[0], a[8]); // Sum(8) + res9 = fma52lo(res9, a[1], a[8]); // Sum(9) + res10 = fma52hi(res10, a[1], a[8]); // Sum(9) + res10 = fma52lo(res10, a[2], a[8]); // Sum(10) + res11 = fma52hi(res11, a[2], a[8]); // Sum(10) + res11 = fma52lo(res11, a[3], a[8]); // Sum(11) + res12 = fma52hi(res12, a[3], a[8]); // Sum(11) + res9 = fma52lo(res9, a[0], a[9]); // Sum(9) + res10 = fma52hi(res10, a[0], a[9]); // Sum(9) + res10 = fma52lo(res10, a[1], a[9]); // Sum(10) + res11 = fma52hi(res11, a[1], a[9]); // Sum(10) + res11 = fma52lo(res11, a[2], a[9]); // Sum(11) + res12 = fma52hi(res12, a[2], a[9]); // Sum(11) + res0 = add64(res0, res0); // Double(0) + res1 = add64(res1, res1); // Double(1) + res2 = add64(res2, res2); // Double(2) + res3 = add64(res3, res3); // Double(3) + res4 = add64(res4, res4); // Double(4) + res5 = add64(res5, res5); // Double(5) + res6 = add64(res6, res6); // Double(6) + res7 = add64(res7, res7); // Double(7) + res8 = add64(res8, res8); // Double(8) + res9 = add64(res9, res9); // Double(9) + res10 = add64(res10, res10); // Double(10) + res11 = add64(res11, res11); // Double(11) + res0 = fma52lo(res0, a[0], a[0]); // Add sqr(0) + res1 = fma52hi(res1, a[0], a[0]); // Add sqr(0) + res2 = fma52lo(res2, a[1], a[1]); // Add sqr(2) + res3 = fma52hi(res3, a[1], a[1]); // Add sqr(2) + res4 = fma52lo(res4, a[2], a[2]); // Add sqr(4) + res5 = fma52hi(res5, a[2], a[2]); // Add sqr(4) + res6 = fma52lo(res6, a[3], a[3]); // Add sqr(6) + res7 = fma52hi(res7, a[3], a[3]); // Add sqr(6) + res8 = fma52lo(res8, a[4], a[4]); // Add sqr(8) + res9 = fma52hi(res9, a[4], a[4]); // Add sqr(8) + res10 = fma52lo(res10, a[5], a[5]); // Add sqr(10) + res11 = fma52hi(res11, a[5], a[5]); // Add sqr(10) + res12 = fma52lo(res12, a[5], a[7]); // Sum(12) + res13 = fma52hi(res13, a[5], a[7]); // Sum(12) + res13 = fma52lo(res13, a[6], a[7]); // Sum(13) + res14 = fma52hi(res14, a[6], a[7]); // Sum(13) + res12 = fma52lo(res12, a[4], a[8]); // Sum(12) + res13 = fma52hi(res13, a[4], a[8]); // Sum(12) + res13 = fma52lo(res13, a[5], a[8]); // Sum(13) + res14 = fma52hi(res14, a[5], a[8]); // Sum(13) + res14 = fma52lo(res14, a[6], a[8]); // Sum(14) + res15 = fma52hi(res15, a[6], a[8]); // Sum(14) + res15 = fma52lo(res15, a[7], a[8]); // Sum(15) + res16 = fma52hi(res16, a[7], a[8]); // Sum(15) + res12 = fma52lo(res12, a[3], a[9]); // Sum(12) + res13 = fma52hi(res13, a[3], a[9]); // Sum(12) + res13 = fma52lo(res13, a[4], a[9]); // Sum(13) + res14 = fma52hi(res14, a[4], a[9]); // Sum(13) + res14 = fma52lo(res14, a[5], a[9]); // Sum(14) + res15 = fma52hi(res15, a[5], a[9]); // Sum(14) + res15 = fma52lo(res15, a[6], a[9]); // Sum(15) + res16 = fma52hi(res16, a[6], a[9]); // Sum(15) + res16 = fma52lo(res16, a[7], a[9]); // Sum(16) + res17 = fma52hi(res17, a[7], a[9]); // Sum(16) + res17 = fma52lo(res17, a[8], a[9]); // Sum(17) + res18 = fma52hi(res18, a[8], a[9]); // Sum(17) + res12 = add64(res12, res12); // Double(12) + res13 = add64(res13, res13); // Double(13) + res14 = add64(res14, res14); // Double(14) + res15 = add64(res15, res15); // Double(15) + res16 = add64(res16, res16); // Double(16) + res17 = add64(res17, res17); // Double(17) + res18 = add64(res18, res18); // Double(18) + res12 = fma52lo(res12, a[6], a[6]); // Add sqr(12) + res13 = fma52hi(res13, a[6], a[6]); // Add sqr(12) + res14 = fma52lo(res14, a[7], a[7]); // Add sqr(14) + res15 = fma52hi(res15, a[7], a[7]); // Add sqr(14) + res16 = fma52lo(res16, a[8], a[8]); // Add sqr(16) + res17 = fma52hi(res17, a[8], a[8]); // Add sqr(16) + res18 = fma52lo(res18, a[9], a[9]); // Add sqr(18) + res19 = fma52hi(res19, a[9], a[9]); // Add sqr(18) + + // Generate u_i + U64 u0 = mul52lo(res0, k); + ASM("jmp l0\nl0:\n"); + + // Create u0 + fma52lo_mem(res0, res0, u0, m, SIMD_BYTES * 0); + fma52hi_mem(res1, res1, u0, m, SIMD_BYTES * 0); + res1 = fma52lo(res1, u0, m[1]); + res2 = fma52hi(res2, u0, m[1]); + res1 = add64(res1, srli64(res0, DIGIT_SIZE)); + U64 u1 = mul52lo(res1, k); + fma52lo_mem(res2, res2, u0, m, SIMD_BYTES * 2); + fma52hi_mem(res3, res3, u0, m, SIMD_BYTES * 2); + res3 = fma52lo(res3, u0, m[3]); + res4 = fma52hi(res4, u0, m[3]); + fma52lo_mem(res4, res4, u0, m, SIMD_BYTES * 4); + fma52hi_mem(res5, res5, u0, m, SIMD_BYTES * 4); + res5 = fma52lo(res5, u0, m[5]); + res6 = fma52hi(res6, u0, m[5]); + fma52lo_mem(res6, res6, u0, m, SIMD_BYTES * 6); + fma52hi_mem(res7, res7, u0, m, SIMD_BYTES * 6); + res7 = fma52lo(res7, u0, m[7]); + res8 = fma52hi(res8, u0, m[7]); + fma52lo_mem(res8, res8, u0, m, SIMD_BYTES * 8); + fma52hi_mem(res9, res9, u0, m, SIMD_BYTES * 8); + res9 = fma52lo(res9, u0, m[9]); + res10 = fma52hi(res10, u0, m[9]); + + // Create u1 + fma52lo_mem(res1, res1, u1, m, SIMD_BYTES * 0); + fma52hi_mem(res2, res2, u1, m, SIMD_BYTES * 0); + res2 = fma52lo(res2, u1, m[1]); + res3 = fma52hi(res3, u1, m[1]); + res2 = add64(res2, srli64(res1, DIGIT_SIZE)); + U64 u2 = mul52lo(res2, k); + fma52lo_mem(res3, res3, u1, m, SIMD_BYTES * 2); + fma52hi_mem(res4, res4, u1, m, SIMD_BYTES * 2); + res4 = fma52lo(res4, u1, m[3]); + res5 = fma52hi(res5, u1, m[3]); + fma52lo_mem(res5, res5, u1, m, SIMD_BYTES * 4); + fma52hi_mem(res6, res6, u1, m, SIMD_BYTES * 4); + res6 = fma52lo(res6, u1, m[5]); + res7 = fma52hi(res7, u1, m[5]); + fma52lo_mem(res7, res7, u1, m, SIMD_BYTES * 6); + fma52hi_mem(res8, res8, u1, m, SIMD_BYTES * 6); + res8 = fma52lo(res8, u1, m[7]); + res9 = fma52hi(res9, u1, m[7]); + fma52lo_mem(res9, res9, u1, m, SIMD_BYTES * 8); + fma52hi_mem(res10, res10, u1, m, SIMD_BYTES * 8); + res10 = fma52lo(res10, u1, m[9]); + res11 = fma52hi(res11, u1, m[9]); + ASM("jmp l2\nl2:\n"); + + // Create u2 + fma52lo_mem(res2, res2, u2, m, SIMD_BYTES * 0); + fma52hi_mem(res3, res3, u2, m, SIMD_BYTES * 0); + res3 = fma52lo(res3, u2, m[1]); + res4 = fma52hi(res4, u2, m[1]); + res3 = add64(res3, srli64(res2, DIGIT_SIZE)); + U64 u3 = mul52lo(res3, k); + fma52lo_mem(res4, res4, u2, m, SIMD_BYTES * 2); + fma52hi_mem(res5, res5, u2, m, SIMD_BYTES * 2); + res5 = fma52lo(res5, u2, m[3]); + res6 = fma52hi(res6, u2, m[3]); + fma52lo_mem(res6, res6, u2, m, SIMD_BYTES * 4); + fma52hi_mem(res7, res7, u2, m, SIMD_BYTES * 4); + res7 = fma52lo(res7, u2, m[5]); + res8 = fma52hi(res8, u2, m[5]); + fma52lo_mem(res8, res8, u2, m, SIMD_BYTES * 6); + fma52hi_mem(res9, res9, u2, m, SIMD_BYTES * 6); + res9 = fma52lo(res9, u2, m[7]); + res10 = fma52hi(res10, u2, m[7]); + fma52lo_mem(res10, res10, u2, m, SIMD_BYTES * 8); + fma52hi_mem(res11, res11, u2, m, SIMD_BYTES * 8); + res11 = fma52lo(res11, u2, m[9]); + res12 = fma52hi(res12, u2, m[9]); + + // Create u3 + fma52lo_mem(res3, res3, u3, m, SIMD_BYTES * 0); + fma52hi_mem(res4, res4, u3, m, SIMD_BYTES * 0); + res4 = fma52lo(res4, u3, m[1]); + res5 = fma52hi(res5, u3, m[1]); + res4 = add64(res4, srli64(res3, DIGIT_SIZE)); + U64 u4 = mul52lo(res4, k); + fma52lo_mem(res5, res5, u3, m, SIMD_BYTES * 2); + fma52hi_mem(res6, res6, u3, m, SIMD_BYTES * 2); + res6 = fma52lo(res6, u3, m[3]); + res7 = fma52hi(res7, u3, m[3]); + fma52lo_mem(res7, res7, u3, m, SIMD_BYTES * 4); + fma52hi_mem(res8, res8, u3, m, SIMD_BYTES * 4); + res8 = fma52lo(res8, u3, m[5]); + res9 = fma52hi(res9, u3, m[5]); + fma52lo_mem(res9, res9, u3, m, SIMD_BYTES * 6); + fma52hi_mem(res10, res10, u3, m, SIMD_BYTES * 6); + res10 = fma52lo(res10, u3, m[7]); + res11 = fma52hi(res11, u3, m[7]); + fma52lo_mem(res11, res11, u3, m, SIMD_BYTES * 8); + fma52hi_mem(res12, res12, u3, m, SIMD_BYTES * 8); + res12 = fma52lo(res12, u3, m[9]); + res13 = fma52hi(res13, u3, m[9]); + ASM("jmp l4\nl4:\n"); + + // Create u4 + fma52lo_mem(res4, res4, u4, m, SIMD_BYTES * 0); + fma52hi_mem(res5, res5, u4, m, SIMD_BYTES * 0); + res5 = fma52lo(res5, u4, m[1]); + res6 = fma52hi(res6, u4, m[1]); + res5 = add64(res5, srli64(res4, DIGIT_SIZE)); + U64 u5 = mul52lo(res5, k); + fma52lo_mem(res6, res6, u4, m, SIMD_BYTES * 2); + fma52hi_mem(res7, res7, u4, m, SIMD_BYTES * 2); + res7 = fma52lo(res7, u4, m[3]); + res8 = fma52hi(res8, u4, m[3]); + fma52lo_mem(res8, res8, u4, m, SIMD_BYTES * 4); + fma52hi_mem(res9, res9, u4, m, SIMD_BYTES * 4); + res9 = fma52lo(res9, u4, m[5]); + res10 = fma52hi(res10, u4, m[5]); + fma52lo_mem(res10, res10, u4, m, SIMD_BYTES * 6); + fma52hi_mem(res11, res11, u4, m, SIMD_BYTES * 6); + res11 = fma52lo(res11, u4, m[7]); + res12 = fma52hi(res12, u4, m[7]); + fma52lo_mem(res12, res12, u4, m, SIMD_BYTES * 8); + fma52hi_mem(res13, res13, u4, m, SIMD_BYTES * 8); + res13 = fma52lo(res13, u4, m[9]); + res14 = fma52hi(res14, u4, m[9]); + + // Create u5 + fma52lo_mem(res5, res5, u5, m, SIMD_BYTES * 0); + fma52hi_mem(res6, res6, u5, m, SIMD_BYTES * 0); + res6 = fma52lo(res6, u5, m[1]); + res7 = fma52hi(res7, u5, m[1]); + res6 = add64(res6, srli64(res5, DIGIT_SIZE)); + U64 u6 = mul52lo(res6, k); + fma52lo_mem(res7, res7, u5, m, SIMD_BYTES * 2); + fma52hi_mem(res8, res8, u5, m, SIMD_BYTES * 2); + res8 = fma52lo(res8, u5, m[3]); + res9 = fma52hi(res9, u5, m[3]); + fma52lo_mem(res9, res9, u5, m, SIMD_BYTES * 4); + fma52hi_mem(res10, res10, u5, m, SIMD_BYTES * 4); + res10 = fma52lo(res10, u5, m[5]); + res11 = fma52hi(res11, u5, m[5]); + fma52lo_mem(res11, res11, u5, m, SIMD_BYTES * 6); + fma52hi_mem(res12, res12, u5, m, SIMD_BYTES * 6); + res12 = fma52lo(res12, u5, m[7]); + res13 = fma52hi(res13, u5, m[7]); + fma52lo_mem(res13, res13, u5, m, SIMD_BYTES * 8); + fma52hi_mem(res14, res14, u5, m, SIMD_BYTES * 8); + res14 = fma52lo(res14, u5, m[9]); + res15 = fma52hi(res15, u5, m[9]); + ASM("jmp l6\nl6:\n"); + + // Create u6 + fma52lo_mem(res6, res6, u6, m, SIMD_BYTES * 0); + fma52hi_mem(res7, res7, u6, m, SIMD_BYTES * 0); + res7 = fma52lo(res7, u6, m[1]); + res8 = fma52hi(res8, u6, m[1]); + res7 = add64(res7, srli64(res6, DIGIT_SIZE)); + U64 u7 = mul52lo(res7, k); + fma52lo_mem(res8, res8, u6, m, SIMD_BYTES * 2); + fma52hi_mem(res9, res9, u6, m, SIMD_BYTES * 2); + res9 = fma52lo(res9, u6, m[3]); + res10 = fma52hi(res10, u6, m[3]); + fma52lo_mem(res10, res10, u6, m, SIMD_BYTES * 4); + fma52hi_mem(res11, res11, u6, m, SIMD_BYTES * 4); + res11 = fma52lo(res11, u6, m[5]); + res12 = fma52hi(res12, u6, m[5]); + fma52lo_mem(res12, res12, u6, m, SIMD_BYTES * 6); + fma52hi_mem(res13, res13, u6, m, SIMD_BYTES * 6); + res13 = fma52lo(res13, u6, m[7]); + res14 = fma52hi(res14, u6, m[7]); + fma52lo_mem(res14, res14, u6, m, SIMD_BYTES * 8); + fma52hi_mem(res15, res15, u6, m, SIMD_BYTES * 8); + res15 = fma52lo(res15, u6, m[9]); + res16 = fma52hi(res16, u6, m[9]); + + // Create u7 + fma52lo_mem(res7, res7, u7, m, SIMD_BYTES * 0); + fma52hi_mem(res8, res8, u7, m, SIMD_BYTES * 0); + res8 = fma52lo(res8, u7, m[1]); + res9 = fma52hi(res9, u7, m[1]); + res8 = add64(res8, srli64(res7, DIGIT_SIZE)); + U64 u8 = mul52lo(res8, k); + fma52lo_mem(res9, res9, u7, m, SIMD_BYTES * 2); + fma52hi_mem(res10, res10, u7, m, SIMD_BYTES * 2); + res10 = fma52lo(res10, u7, m[3]); + res11 = fma52hi(res11, u7, m[3]); + fma52lo_mem(res11, res11, u7, m, SIMD_BYTES * 4); + fma52hi_mem(res12, res12, u7, m, SIMD_BYTES * 4); + res12 = fma52lo(res12, u7, m[5]); + res13 = fma52hi(res13, u7, m[5]); + fma52lo_mem(res13, res13, u7, m, SIMD_BYTES * 6); + fma52hi_mem(res14, res14, u7, m, SIMD_BYTES * 6); + res14 = fma52lo(res14, u7, m[7]); + res15 = fma52hi(res15, u7, m[7]); + fma52lo_mem(res15, res15, u7, m, SIMD_BYTES * 8); + fma52hi_mem(res16, res16, u7, m, SIMD_BYTES * 8); + res16 = fma52lo(res16, u7, m[9]); + res17 = fma52hi(res17, u7, m[9]); + ASM("jmp l8\nl8:\n"); + + // Create u8 + fma52lo_mem(res8, res8, u8, m, SIMD_BYTES * 0); + fma52hi_mem(res9, res9, u8, m, SIMD_BYTES * 0); + res9 = fma52lo(res9, u8, m[1]); + res10 = fma52hi(res10, u8, m[1]); + res9 = add64(res9, srli64(res8, DIGIT_SIZE)); + U64 u9 = mul52lo(res9, k); + fma52lo_mem(res10, res10, u8, m, SIMD_BYTES * 2); + fma52hi_mem(res11, res11, u8, m, SIMD_BYTES * 2); + res11 = fma52lo(res11, u8, m[3]); + res12 = fma52hi(res12, u8, m[3]); + fma52lo_mem(res12, res12, u8, m, SIMD_BYTES * 4); + fma52hi_mem(res13, res13, u8, m, SIMD_BYTES * 4); + res13 = fma52lo(res13, u8, m[5]); + res14 = fma52hi(res14, u8, m[5]); + fma52lo_mem(res14, res14, u8, m, SIMD_BYTES * 6); + fma52hi_mem(res15, res15, u8, m, SIMD_BYTES * 6); + res15 = fma52lo(res15, u8, m[7]); + res16 = fma52hi(res16, u8, m[7]); + fma52lo_mem(res16, res16, u8, m, SIMD_BYTES * 8); + fma52hi_mem(res17, res17, u8, m, SIMD_BYTES * 8); + res17 = fma52lo(res17, u8, m[9]); + res18 = fma52hi(res18, u8, m[9]); + + // Create u9 + fma52lo_mem(res9, res9, u9, m, SIMD_BYTES * 0); + fma52hi_mem(res10, res10, u9, m, SIMD_BYTES * 0); + res10 = fma52lo(res10, u9, m[1]); + res11 = fma52hi(res11, u9, m[1]); + res10 = add64(res10, srli64(res9, DIGIT_SIZE)); + fma52lo_mem(res11, res11, u9, m, SIMD_BYTES * 2); + fma52hi_mem(res12, res12, u9, m, SIMD_BYTES * 2); + res12 = fma52lo(res12, u9, m[3]); + res13 = fma52hi(res13, u9, m[3]); + fma52lo_mem(res13, res13, u9, m, SIMD_BYTES * 4); + fma52hi_mem(res14, res14, u9, m, SIMD_BYTES * 4); + res14 = fma52lo(res14, u9, m[5]); + res15 = fma52hi(res15, u9, m[5]); + fma52lo_mem(res15, res15, u9, m, SIMD_BYTES * 6); + fma52hi_mem(res16, res16, u9, m, SIMD_BYTES * 6); + res16 = fma52lo(res16, u9, m[7]); + res17 = fma52hi(res17, u9, m[7]); + fma52lo_mem(res17, res17, u9, m, SIMD_BYTES * 8); + fma52hi_mem(res18, res18, u9, m, SIMD_BYTES * 8); + res18 = fma52lo(res18, u9, m[9]); + res19 = fma52hi(res19, u9, m[9]); + + // Normalization + r[0] = res10; + res11 = add64(res11, srli64(res10, DIGIT_SIZE)); + r[1] = res11; + res12 = add64(res12, srli64(res11, DIGIT_SIZE)); + r[2] = res12; + res13 = add64(res13, srli64(res12, DIGIT_SIZE)); + r[3] = res13; + res14 = add64(res14, srli64(res13, DIGIT_SIZE)); + r[4] = res14; + res15 = add64(res15, srli64(res14, DIGIT_SIZE)); + r[5] = res15; + res16 = add64(res16, srli64(res15, DIGIT_SIZE)); + r[6] = res16; + res17 = add64(res17, srli64(res16, DIGIT_SIZE)); + r[7] = res17; + res18 = add64(res18, srli64(res17, DIGIT_SIZE)); + r[8] = res18; + res19 = add64(res19, srli64(res18, DIGIT_SIZE)); + r[9] = res19; + a = (U64 *)out_mb; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams5x52x20_diagonal_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams5x52x20_diagonal_mb8.c new file mode 100644 index 0000000..90d23e2 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams5x52x20_diagonal_mb8.c @@ -0,0 +1,1441 @@ +/******************************************************************************* + * Copyright (C) 2019 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include + +#ifdef __GNUC__ +#define ASM(a) __asm__(a); +#else +#define ASM(a) +#endif + +void AMS5x52x20_diagonal_mb8(int64u *out_mb, const int64u *inpA_mb, + const int64u *inpM_mb, const int64u *k0_mb) { + U64 res0, res1, res2, res3, res4, res5, res6, res7, res8, res9, res10, res11, + res12, res13, res14, res15, res16, res17, res18, res19, res20, res21, + res22, res23, res24, res25, res26, res27, res28, res29, res30, res31, + res32, res33, res34, res35, res36, res37, res38, res39; + U64 k; + U64 *a = (U64 *)inpA_mb; + U64 *m = (U64 *)inpM_mb; + U64 *r = (U64 *)out_mb; + int iter; + const int iters = 5; + + k = loadu64((U64 *)k0_mb); + for (iter = 0; iter < iters; ++iter) { + res0 = res1 = res2 = res3 = res4 = res5 = res6 = res7 = res8 = res9 = + res10 = res11 = res12 = res13 = res14 = res15 = res16 = res17 = res18 = + res19 = res20 = res21 = res22 = res23 = res24 = res25 = res26 = + res27 = res28 = res29 = res30 = res31 = res32 = res33 = res34 = + res35 = res36 = res37 = res38 = res39 = get_zero64(); + // Calculate full square + res1 = fma52lo(res1, a[0], a[1]); // Sum(1) + res2 = fma52hi(res2, a[0], a[1]); // Sum(1) + res2 = fma52lo(res2, a[0], a[2]); // Sum(2) + res3 = fma52hi(res3, a[0], a[2]); // Sum(2) + res3 = fma52lo(res3, a[1], a[2]); // Sum(3) + res4 = fma52hi(res4, a[1], a[2]); // Sum(3) + res3 = fma52lo(res3, a[0], a[3]); // Sum(3) + res4 = fma52hi(res4, a[0], a[3]); // Sum(3) + res4 = fma52lo(res4, a[1], a[3]); // Sum(4) + res5 = fma52hi(res5, a[1], a[3]); // Sum(4) + res5 = fma52lo(res5, a[2], a[3]); // Sum(5) + res6 = fma52hi(res6, a[2], a[3]); // Sum(5) + res4 = fma52lo(res4, a[0], a[4]); // Sum(4) + res5 = fma52hi(res5, a[0], a[4]); // Sum(4) + res5 = fma52lo(res5, a[1], a[4]); // Sum(5) + res6 = fma52hi(res6, a[1], a[4]); // Sum(5) + res6 = fma52lo(res6, a[2], a[4]); // Sum(6) + res7 = fma52hi(res7, a[2], a[4]); // Sum(6) + res7 = fma52lo(res7, a[3], a[4]); // Sum(7) + res8 = fma52hi(res8, a[3], a[4]); // Sum(7) + res5 = fma52lo(res5, a[0], a[5]); // Sum(5) + res6 = fma52hi(res6, a[0], a[5]); // Sum(5) + res6 = fma52lo(res6, a[1], a[5]); // Sum(6) + res7 = fma52hi(res7, a[1], a[5]); // Sum(6) + res7 = fma52lo(res7, a[2], a[5]); // Sum(7) + res8 = fma52hi(res8, a[2], a[5]); // Sum(7) + res8 = fma52lo(res8, a[3], a[5]); // Sum(8) + res9 = fma52hi(res9, a[3], a[5]); // Sum(8) + res9 = fma52lo(res9, a[4], a[5]); // Sum(9) + res10 = fma52hi(res10, a[4], a[5]); // Sum(9) + res6 = fma52lo(res6, a[0], a[6]); // Sum(6) + res7 = fma52hi(res7, a[0], a[6]); // Sum(6) + res7 = fma52lo(res7, a[1], a[6]); // Sum(7) + res8 = fma52hi(res8, a[1], a[6]); // Sum(7) + res8 = fma52lo(res8, a[2], a[6]); // Sum(8) + res9 = fma52hi(res9, a[2], a[6]); // Sum(8) + res9 = fma52lo(res9, a[3], a[6]); // Sum(9) + res10 = fma52hi(res10, a[3], a[6]); // Sum(9) + res10 = fma52lo(res10, a[4], a[6]); // Sum(10) + res11 = fma52hi(res11, a[4], a[6]); // Sum(10) + res11 = fma52lo(res11, a[5], a[6]); // Sum(11) + res12 = fma52hi(res12, a[5], a[6]); // Sum(11) + res7 = fma52lo(res7, a[0], a[7]); // Sum(7) + res8 = fma52hi(res8, a[0], a[7]); // Sum(7) + res8 = fma52lo(res8, a[1], a[7]); // Sum(8) + res9 = fma52hi(res9, a[1], a[7]); // Sum(8) + res9 = fma52lo(res9, a[2], a[7]); // Sum(9) + res10 = fma52hi(res10, a[2], a[7]); // Sum(9) + res10 = fma52lo(res10, a[3], a[7]); // Sum(10) + res11 = fma52hi(res11, a[3], a[7]); // Sum(10) + res11 = fma52lo(res11, a[4], a[7]); // Sum(11) + res12 = fma52hi(res12, a[4], a[7]); // Sum(11) + res8 = fma52lo(res8, a[0], a[8]); // Sum(8) + res9 = fma52hi(res9, a[0], a[8]); // Sum(8) + res9 = fma52lo(res9, a[1], a[8]); // Sum(9) + res10 = fma52hi(res10, a[1], a[8]); // Sum(9) + res10 = fma52lo(res10, a[2], a[8]); // Sum(10) + res11 = fma52hi(res11, a[2], a[8]); // Sum(10) + res11 = fma52lo(res11, a[3], a[8]); // Sum(11) + res12 = fma52hi(res12, a[3], a[8]); // Sum(11) + res9 = fma52lo(res9, a[0], a[9]); // Sum(9) + res10 = fma52hi(res10, a[0], a[9]); // Sum(9) + res10 = fma52lo(res10, a[1], a[9]); // Sum(10) + res11 = fma52hi(res11, a[1], a[9]); // Sum(10) + res11 = fma52lo(res11, a[2], a[9]); // Sum(11) + res12 = fma52hi(res12, a[2], a[9]); // Sum(11) + res10 = fma52lo(res10, a[0], a[10]); // Sum(10) + res11 = fma52hi(res11, a[0], a[10]); // Sum(10) + res11 = fma52lo(res11, a[1], a[10]); // Sum(11) + res12 = fma52hi(res12, a[1], a[10]); // Sum(11) + res11 = fma52lo(res11, a[0], a[11]); // Sum(11) + res12 = fma52hi(res12, a[0], a[11]); // Sum(11) + res0 = add64(res0, res0); // Double(0) + res1 = add64(res1, res1); // Double(1) + res2 = add64(res2, res2); // Double(2) + res3 = add64(res3, res3); // Double(3) + res4 = add64(res4, res4); // Double(4) + res5 = add64(res5, res5); // Double(5) + res6 = add64(res6, res6); // Double(6) + res7 = add64(res7, res7); // Double(7) + res8 = add64(res8, res8); // Double(8) + res9 = add64(res9, res9); // Double(9) + res10 = add64(res10, res10); // Double(10) + res11 = add64(res11, res11); // Double(11) + res0 = fma52lo(res0, a[0], a[0]); // Add sqr(0) + res1 = fma52hi(res1, a[0], a[0]); // Add sqr(0) + res2 = fma52lo(res2, a[1], a[1]); // Add sqr(2) + res3 = fma52hi(res3, a[1], a[1]); // Add sqr(2) + res4 = fma52lo(res4, a[2], a[2]); // Add sqr(4) + res5 = fma52hi(res5, a[2], a[2]); // Add sqr(4) + res6 = fma52lo(res6, a[3], a[3]); // Add sqr(6) + res7 = fma52hi(res7, a[3], a[3]); // Add sqr(6) + res8 = fma52lo(res8, a[4], a[4]); // Add sqr(8) + res9 = fma52hi(res9, a[4], a[4]); // Add sqr(8) + res10 = fma52lo(res10, a[5], a[5]); // Add sqr(10) + res11 = fma52hi(res11, a[5], a[5]); // Add sqr(10) + res12 = fma52lo(res12, a[5], a[7]); // Sum(12) + res13 = fma52hi(res13, a[5], a[7]); // Sum(12) + res13 = fma52lo(res13, a[6], a[7]); // Sum(13) + res14 = fma52hi(res14, a[6], a[7]); // Sum(13) + res12 = fma52lo(res12, a[4], a[8]); // Sum(12) + res13 = fma52hi(res13, a[4], a[8]); // Sum(12) + res13 = fma52lo(res13, a[5], a[8]); // Sum(13) + res14 = fma52hi(res14, a[5], a[8]); // Sum(13) + res14 = fma52lo(res14, a[6], a[8]); // Sum(14) + res15 = fma52hi(res15, a[6], a[8]); // Sum(14) + res15 = fma52lo(res15, a[7], a[8]); // Sum(15) + res16 = fma52hi(res16, a[7], a[8]); // Sum(15) + res12 = fma52lo(res12, a[3], a[9]); // Sum(12) + res13 = fma52hi(res13, a[3], a[9]); // Sum(12) + res13 = fma52lo(res13, a[4], a[9]); // Sum(13) + res14 = fma52hi(res14, a[4], a[9]); // Sum(13) + res14 = fma52lo(res14, a[5], a[9]); // Sum(14) + res15 = fma52hi(res15, a[5], a[9]); // Sum(14) + res15 = fma52lo(res15, a[6], a[9]); // Sum(15) + res16 = fma52hi(res16, a[6], a[9]); // Sum(15) + res16 = fma52lo(res16, a[7], a[9]); // Sum(16) + res17 = fma52hi(res17, a[7], a[9]); // Sum(16) + res17 = fma52lo(res17, a[8], a[9]); // Sum(17) + res18 = fma52hi(res18, a[8], a[9]); // Sum(17) + res12 = fma52lo(res12, a[2], a[10]); // Sum(12) + res13 = fma52hi(res13, a[2], a[10]); // Sum(12) + res13 = fma52lo(res13, a[3], a[10]); // Sum(13) + res14 = fma52hi(res14, a[3], a[10]); // Sum(13) + res14 = fma52lo(res14, a[4], a[10]); // Sum(14) + res15 = fma52hi(res15, a[4], a[10]); // Sum(14) + res15 = fma52lo(res15, a[5], a[10]); // Sum(15) + res16 = fma52hi(res16, a[5], a[10]); // Sum(15) + res16 = fma52lo(res16, a[6], a[10]); // Sum(16) + res17 = fma52hi(res17, a[6], a[10]); // Sum(16) + res17 = fma52lo(res17, a[7], a[10]); // Sum(17) + res18 = fma52hi(res18, a[7], a[10]); // Sum(17) + res18 = fma52lo(res18, a[8], a[10]); // Sum(18) + res19 = fma52hi(res19, a[8], a[10]); // Sum(18) + res19 = fma52lo(res19, a[9], a[10]); // Sum(19) + res20 = fma52hi(res20, a[9], a[10]); // Sum(19) + res12 = fma52lo(res12, a[1], a[11]); // Sum(12) + res13 = fma52hi(res13, a[1], a[11]); // Sum(12) + res13 = fma52lo(res13, a[2], a[11]); // Sum(13) + res14 = fma52hi(res14, a[2], a[11]); // Sum(13) + res14 = fma52lo(res14, a[3], a[11]); // Sum(14) + res15 = fma52hi(res15, a[3], a[11]); // Sum(14) + res15 = fma52lo(res15, a[4], a[11]); // Sum(15) + res16 = fma52hi(res16, a[4], a[11]); // Sum(15) + res16 = fma52lo(res16, a[5], a[11]); // Sum(16) + res17 = fma52hi(res17, a[5], a[11]); // Sum(16) + res17 = fma52lo(res17, a[6], a[11]); // Sum(17) + res18 = fma52hi(res18, a[6], a[11]); // Sum(17) + res18 = fma52lo(res18, a[7], a[11]); // Sum(18) + res19 = fma52hi(res19, a[7], a[11]); // Sum(18) + res19 = fma52lo(res19, a[8], a[11]); // Sum(19) + res20 = fma52hi(res20, a[8], a[11]); // Sum(19) + res20 = fma52lo(res20, a[9], a[11]); // Sum(20) + res21 = fma52hi(res21, a[9], a[11]); // Sum(20) + res21 = fma52lo(res21, a[10], a[11]); // Sum(21) + res22 = fma52hi(res22, a[10], a[11]); // Sum(21) + res12 = fma52lo(res12, a[0], a[12]); // Sum(12) + res13 = fma52hi(res13, a[0], a[12]); // Sum(12) + res13 = fma52lo(res13, a[1], a[12]); // Sum(13) + res14 = fma52hi(res14, a[1], a[12]); // Sum(13) + res14 = fma52lo(res14, a[2], a[12]); // Sum(14) + res15 = fma52hi(res15, a[2], a[12]); // Sum(14) + res15 = fma52lo(res15, a[3], a[12]); // Sum(15) + res16 = fma52hi(res16, a[3], a[12]); // Sum(15) + res16 = fma52lo(res16, a[4], a[12]); // Sum(16) + res17 = fma52hi(res17, a[4], a[12]); // Sum(16) + res17 = fma52lo(res17, a[5], a[12]); // Sum(17) + res18 = fma52hi(res18, a[5], a[12]); // Sum(17) + res18 = fma52lo(res18, a[6], a[12]); // Sum(18) + res19 = fma52hi(res19, a[6], a[12]); // Sum(18) + res19 = fma52lo(res19, a[7], a[12]); // Sum(19) + res20 = fma52hi(res20, a[7], a[12]); // Sum(19) + res20 = fma52lo(res20, a[8], a[12]); // Sum(20) + res21 = fma52hi(res21, a[8], a[12]); // Sum(20) + res21 = fma52lo(res21, a[9], a[12]); // Sum(21) + res22 = fma52hi(res22, a[9], a[12]); // Sum(21) + res22 = fma52lo(res22, a[10], a[12]); // Sum(22) + res23 = fma52hi(res23, a[10], a[12]); // Sum(22) + res23 = fma52lo(res23, a[11], a[12]); // Sum(23) + res24 = fma52hi(res24, a[11], a[12]); // Sum(23) + res13 = fma52lo(res13, a[0], a[13]); // Sum(13) + res14 = fma52hi(res14, a[0], a[13]); // Sum(13) + res14 = fma52lo(res14, a[1], a[13]); // Sum(14) + res15 = fma52hi(res15, a[1], a[13]); // Sum(14) + res15 = fma52lo(res15, a[2], a[13]); // Sum(15) + res16 = fma52hi(res16, a[2], a[13]); // Sum(15) + res16 = fma52lo(res16, a[3], a[13]); // Sum(16) + res17 = fma52hi(res17, a[3], a[13]); // Sum(16) + res17 = fma52lo(res17, a[4], a[13]); // Sum(17) + res18 = fma52hi(res18, a[4], a[13]); // Sum(17) + res18 = fma52lo(res18, a[5], a[13]); // Sum(18) + res19 = fma52hi(res19, a[5], a[13]); // Sum(18) + res19 = fma52lo(res19, a[6], a[13]); // Sum(19) + res20 = fma52hi(res20, a[6], a[13]); // Sum(19) + res20 = fma52lo(res20, a[7], a[13]); // Sum(20) + res21 = fma52hi(res21, a[7], a[13]); // Sum(20) + res21 = fma52lo(res21, a[8], a[13]); // Sum(21) + res22 = fma52hi(res22, a[8], a[13]); // Sum(21) + res22 = fma52lo(res22, a[9], a[13]); // Sum(22) + res23 = fma52hi(res23, a[9], a[13]); // Sum(22) + res23 = fma52lo(res23, a[10], a[13]); // Sum(23) + res24 = fma52hi(res24, a[10], a[13]); // Sum(23) + res14 = fma52lo(res14, a[0], a[14]); // Sum(14) + res15 = fma52hi(res15, a[0], a[14]); // Sum(14) + res15 = fma52lo(res15, a[1], a[14]); // Sum(15) + res16 = fma52hi(res16, a[1], a[14]); // Sum(15) + res16 = fma52lo(res16, a[2], a[14]); // Sum(16) + res17 = fma52hi(res17, a[2], a[14]); // Sum(16) + res17 = fma52lo(res17, a[3], a[14]); // Sum(17) + res18 = fma52hi(res18, a[3], a[14]); // Sum(17) + res18 = fma52lo(res18, a[4], a[14]); // Sum(18) + res19 = fma52hi(res19, a[4], a[14]); // Sum(18) + res19 = fma52lo(res19, a[5], a[14]); // Sum(19) + res20 = fma52hi(res20, a[5], a[14]); // Sum(19) + res20 = fma52lo(res20, a[6], a[14]); // Sum(20) + res21 = fma52hi(res21, a[6], a[14]); // Sum(20) + res21 = fma52lo(res21, a[7], a[14]); // Sum(21) + res22 = fma52hi(res22, a[7], a[14]); // Sum(21) + res22 = fma52lo(res22, a[8], a[14]); // Sum(22) + res23 = fma52hi(res23, a[8], a[14]); // Sum(22) + res23 = fma52lo(res23, a[9], a[14]); // Sum(23) + res24 = fma52hi(res24, a[9], a[14]); // Sum(23) + res15 = fma52lo(res15, a[0], a[15]); // Sum(15) + res16 = fma52hi(res16, a[0], a[15]); // Sum(15) + res16 = fma52lo(res16, a[1], a[15]); // Sum(16) + res17 = fma52hi(res17, a[1], a[15]); // Sum(16) + res17 = fma52lo(res17, a[2], a[15]); // Sum(17) + res18 = fma52hi(res18, a[2], a[15]); // Sum(17) + res18 = fma52lo(res18, a[3], a[15]); // Sum(18) + res19 = fma52hi(res19, a[3], a[15]); // Sum(18) + res19 = fma52lo(res19, a[4], a[15]); // Sum(19) + res20 = fma52hi(res20, a[4], a[15]); // Sum(19) + res20 = fma52lo(res20, a[5], a[15]); // Sum(20) + res21 = fma52hi(res21, a[5], a[15]); // Sum(20) + res21 = fma52lo(res21, a[6], a[15]); // Sum(21) + res22 = fma52hi(res22, a[6], a[15]); // Sum(21) + res22 = fma52lo(res22, a[7], a[15]); // Sum(22) + res23 = fma52hi(res23, a[7], a[15]); // Sum(22) + res23 = fma52lo(res23, a[8], a[15]); // Sum(23) + res24 = fma52hi(res24, a[8], a[15]); // Sum(23) + res16 = fma52lo(res16, a[0], a[16]); // Sum(16) + res17 = fma52hi(res17, a[0], a[16]); // Sum(16) + res17 = fma52lo(res17, a[1], a[16]); // Sum(17) + res18 = fma52hi(res18, a[1], a[16]); // Sum(17) + res18 = fma52lo(res18, a[2], a[16]); // Sum(18) + res19 = fma52hi(res19, a[2], a[16]); // Sum(18) + res19 = fma52lo(res19, a[3], a[16]); // Sum(19) + res20 = fma52hi(res20, a[3], a[16]); // Sum(19) + res20 = fma52lo(res20, a[4], a[16]); // Sum(20) + res21 = fma52hi(res21, a[4], a[16]); // Sum(20) + res21 = fma52lo(res21, a[5], a[16]); // Sum(21) + res22 = fma52hi(res22, a[5], a[16]); // Sum(21) + res22 = fma52lo(res22, a[6], a[16]); // Sum(22) + res23 = fma52hi(res23, a[6], a[16]); // Sum(22) + res23 = fma52lo(res23, a[7], a[16]); // Sum(23) + res24 = fma52hi(res24, a[7], a[16]); // Sum(23) + res17 = fma52lo(res17, a[0], a[17]); // Sum(17) + res18 = fma52hi(res18, a[0], a[17]); // Sum(17) + res18 = fma52lo(res18, a[1], a[17]); // Sum(18) + res19 = fma52hi(res19, a[1], a[17]); // Sum(18) + res19 = fma52lo(res19, a[2], a[17]); // Sum(19) + res20 = fma52hi(res20, a[2], a[17]); // Sum(19) + res20 = fma52lo(res20, a[3], a[17]); // Sum(20) + res21 = fma52hi(res21, a[3], a[17]); // Sum(20) + res21 = fma52lo(res21, a[4], a[17]); // Sum(21) + res22 = fma52hi(res22, a[4], a[17]); // Sum(21) + res22 = fma52lo(res22, a[5], a[17]); // Sum(22) + res23 = fma52hi(res23, a[5], a[17]); // Sum(22) + res23 = fma52lo(res23, a[6], a[17]); // Sum(23) + res24 = fma52hi(res24, a[6], a[17]); // Sum(23) + res18 = fma52lo(res18, a[0], a[18]); // Sum(18) + res19 = fma52hi(res19, a[0], a[18]); // Sum(18) + res19 = fma52lo(res19, a[1], a[18]); // Sum(19) + res20 = fma52hi(res20, a[1], a[18]); // Sum(19) + res20 = fma52lo(res20, a[2], a[18]); // Sum(20) + res21 = fma52hi(res21, a[2], a[18]); // Sum(20) + res21 = fma52lo(res21, a[3], a[18]); // Sum(21) + res22 = fma52hi(res22, a[3], a[18]); // Sum(21) + res22 = fma52lo(res22, a[4], a[18]); // Sum(22) + res23 = fma52hi(res23, a[4], a[18]); // Sum(22) + res23 = fma52lo(res23, a[5], a[18]); // Sum(23) + res24 = fma52hi(res24, a[5], a[18]); // Sum(23) + res19 = fma52lo(res19, a[0], a[19]); // Sum(19) + res20 = fma52hi(res20, a[0], a[19]); // Sum(19) + res20 = fma52lo(res20, a[1], a[19]); // Sum(20) + res21 = fma52hi(res21, a[1], a[19]); // Sum(20) + res21 = fma52lo(res21, a[2], a[19]); // Sum(21) + res22 = fma52hi(res22, a[2], a[19]); // Sum(21) + res22 = fma52lo(res22, a[3], a[19]); // Sum(22) + res23 = fma52hi(res23, a[3], a[19]); // Sum(22) + res23 = fma52lo(res23, a[4], a[19]); // Sum(23) + res24 = fma52hi(res24, a[4], a[19]); // Sum(23) + res12 = add64(res12, res12); // Double(12) + res13 = add64(res13, res13); // Double(13) + res14 = add64(res14, res14); // Double(14) + res15 = add64(res15, res15); // Double(15) + res16 = add64(res16, res16); // Double(16) + res17 = add64(res17, res17); // Double(17) + res18 = add64(res18, res18); // Double(18) + res19 = add64(res19, res19); // Double(19) + res20 = add64(res20, res20); // Double(20) + res21 = add64(res21, res21); // Double(21) + res22 = add64(res22, res22); // Double(22) + res23 = add64(res23, res23); // Double(23) + res12 = fma52lo(res12, a[6], a[6]); // Add sqr(12) + res13 = fma52hi(res13, a[6], a[6]); // Add sqr(12) + res14 = fma52lo(res14, a[7], a[7]); // Add sqr(14) + res15 = fma52hi(res15, a[7], a[7]); // Add sqr(14) + res16 = fma52lo(res16, a[8], a[8]); // Add sqr(16) + res17 = fma52hi(res17, a[8], a[8]); // Add sqr(16) + res18 = fma52lo(res18, a[9], a[9]); // Add sqr(18) + res19 = fma52hi(res19, a[9], a[9]); // Add sqr(18) + res20 = fma52lo(res20, a[10], a[10]); // Add sqr(20) + res21 = fma52hi(res21, a[10], a[10]); // Add sqr(20) + res22 = fma52lo(res22, a[11], a[11]); // Add sqr(22) + res23 = fma52hi(res23, a[11], a[11]); // Add sqr(22) + res24 = fma52lo(res24, a[11], a[13]); // Sum(24) + res25 = fma52hi(res25, a[11], a[13]); // Sum(24) + res25 = fma52lo(res25, a[12], a[13]); // Sum(25) + res26 = fma52hi(res26, a[12], a[13]); // Sum(25) + res24 = fma52lo(res24, a[10], a[14]); // Sum(24) + res25 = fma52hi(res25, a[10], a[14]); // Sum(24) + res25 = fma52lo(res25, a[11], a[14]); // Sum(25) + res26 = fma52hi(res26, a[11], a[14]); // Sum(25) + res26 = fma52lo(res26, a[12], a[14]); // Sum(26) + res27 = fma52hi(res27, a[12], a[14]); // Sum(26) + res27 = fma52lo(res27, a[13], a[14]); // Sum(27) + res28 = fma52hi(res28, a[13], a[14]); // Sum(27) + res24 = fma52lo(res24, a[9], a[15]); // Sum(24) + res25 = fma52hi(res25, a[9], a[15]); // Sum(24) + res25 = fma52lo(res25, a[10], a[15]); // Sum(25) + res26 = fma52hi(res26, a[10], a[15]); // Sum(25) + res26 = fma52lo(res26, a[11], a[15]); // Sum(26) + res27 = fma52hi(res27, a[11], a[15]); // Sum(26) + res27 = fma52lo(res27, a[12], a[15]); // Sum(27) + res28 = fma52hi(res28, a[12], a[15]); // Sum(27) + res28 = fma52lo(res28, a[13], a[15]); // Sum(28) + res29 = fma52hi(res29, a[13], a[15]); // Sum(28) + res29 = fma52lo(res29, a[14], a[15]); // Sum(29) + res30 = fma52hi(res30, a[14], a[15]); // Sum(29) + res24 = fma52lo(res24, a[8], a[16]); // Sum(24) + res25 = fma52hi(res25, a[8], a[16]); // Sum(24) + res25 = fma52lo(res25, a[9], a[16]); // Sum(25) + res26 = fma52hi(res26, a[9], a[16]); // Sum(25) + res26 = fma52lo(res26, a[10], a[16]); // Sum(26) + res27 = fma52hi(res27, a[10], a[16]); // Sum(26) + res27 = fma52lo(res27, a[11], a[16]); // Sum(27) + res28 = fma52hi(res28, a[11], a[16]); // Sum(27) + res28 = fma52lo(res28, a[12], a[16]); // Sum(28) + res29 = fma52hi(res29, a[12], a[16]); // Sum(28) + res29 = fma52lo(res29, a[13], a[16]); // Sum(29) + res30 = fma52hi(res30, a[13], a[16]); // Sum(29) + res30 = fma52lo(res30, a[14], a[16]); // Sum(30) + res31 = fma52hi(res31, a[14], a[16]); // Sum(30) + res31 = fma52lo(res31, a[15], a[16]); // Sum(31) + res32 = fma52hi(res32, a[15], a[16]); // Sum(31) + res24 = fma52lo(res24, a[7], a[17]); // Sum(24) + res25 = fma52hi(res25, a[7], a[17]); // Sum(24) + res25 = fma52lo(res25, a[8], a[17]); // Sum(25) + res26 = fma52hi(res26, a[8], a[17]); // Sum(25) + res26 = fma52lo(res26, a[9], a[17]); // Sum(26) + res27 = fma52hi(res27, a[9], a[17]); // Sum(26) + res27 = fma52lo(res27, a[10], a[17]); // Sum(27) + res28 = fma52hi(res28, a[10], a[17]); // Sum(27) + res28 = fma52lo(res28, a[11], a[17]); // Sum(28) + res29 = fma52hi(res29, a[11], a[17]); // Sum(28) + res29 = fma52lo(res29, a[12], a[17]); // Sum(29) + res30 = fma52hi(res30, a[12], a[17]); // Sum(29) + res30 = fma52lo(res30, a[13], a[17]); // Sum(30) + res31 = fma52hi(res31, a[13], a[17]); // Sum(30) + res31 = fma52lo(res31, a[14], a[17]); // Sum(31) + res32 = fma52hi(res32, a[14], a[17]); // Sum(31) + res32 = fma52lo(res32, a[15], a[17]); // Sum(32) + res33 = fma52hi(res33, a[15], a[17]); // Sum(32) + res33 = fma52lo(res33, a[16], a[17]); // Sum(33) + res34 = fma52hi(res34, a[16], a[17]); // Sum(33) + res24 = fma52lo(res24, a[6], a[18]); // Sum(24) + res25 = fma52hi(res25, a[6], a[18]); // Sum(24) + res25 = fma52lo(res25, a[7], a[18]); // Sum(25) + res26 = fma52hi(res26, a[7], a[18]); // Sum(25) + res26 = fma52lo(res26, a[8], a[18]); // Sum(26) + res27 = fma52hi(res27, a[8], a[18]); // Sum(26) + res27 = fma52lo(res27, a[9], a[18]); // Sum(27) + res28 = fma52hi(res28, a[9], a[18]); // Sum(27) + res28 = fma52lo(res28, a[10], a[18]); // Sum(28) + res29 = fma52hi(res29, a[10], a[18]); // Sum(28) + res29 = fma52lo(res29, a[11], a[18]); // Sum(29) + res30 = fma52hi(res30, a[11], a[18]); // Sum(29) + res30 = fma52lo(res30, a[12], a[18]); // Sum(30) + res31 = fma52hi(res31, a[12], a[18]); // Sum(30) + res31 = fma52lo(res31, a[13], a[18]); // Sum(31) + res32 = fma52hi(res32, a[13], a[18]); // Sum(31) + res32 = fma52lo(res32, a[14], a[18]); // Sum(32) + res33 = fma52hi(res33, a[14], a[18]); // Sum(32) + res33 = fma52lo(res33, a[15], a[18]); // Sum(33) + res34 = fma52hi(res34, a[15], a[18]); // Sum(33) + res34 = fma52lo(res34, a[16], a[18]); // Sum(34) + res35 = fma52hi(res35, a[16], a[18]); // Sum(34) + res35 = fma52lo(res35, a[17], a[18]); // Sum(35) + res36 = fma52hi(res36, a[17], a[18]); // Sum(35) + res24 = fma52lo(res24, a[5], a[19]); // Sum(24) + res25 = fma52hi(res25, a[5], a[19]); // Sum(24) + res25 = fma52lo(res25, a[6], a[19]); // Sum(25) + res26 = fma52hi(res26, a[6], a[19]); // Sum(25) + res26 = fma52lo(res26, a[7], a[19]); // Sum(26) + res27 = fma52hi(res27, a[7], a[19]); // Sum(26) + res27 = fma52lo(res27, a[8], a[19]); // Sum(27) + res28 = fma52hi(res28, a[8], a[19]); // Sum(27) + res28 = fma52lo(res28, a[9], a[19]); // Sum(28) + res29 = fma52hi(res29, a[9], a[19]); // Sum(28) + res29 = fma52lo(res29, a[10], a[19]); // Sum(29) + res30 = fma52hi(res30, a[10], a[19]); // Sum(29) + res30 = fma52lo(res30, a[11], a[19]); // Sum(30) + res31 = fma52hi(res31, a[11], a[19]); // Sum(30) + res31 = fma52lo(res31, a[12], a[19]); // Sum(31) + res32 = fma52hi(res32, a[12], a[19]); // Sum(31) + res32 = fma52lo(res32, a[13], a[19]); // Sum(32) + res33 = fma52hi(res33, a[13], a[19]); // Sum(32) + res33 = fma52lo(res33, a[14], a[19]); // Sum(33) + res34 = fma52hi(res34, a[14], a[19]); // Sum(33) + res34 = fma52lo(res34, a[15], a[19]); // Sum(34) + res35 = fma52hi(res35, a[15], a[19]); // Sum(34) + res35 = fma52lo(res35, a[16], a[19]); // Sum(35) + res36 = fma52hi(res36, a[16], a[19]); // Sum(35) + res24 = add64(res24, res24); // Double(24) + res25 = add64(res25, res25); // Double(25) + res26 = add64(res26, res26); // Double(26) + res27 = add64(res27, res27); // Double(27) + res28 = add64(res28, res28); // Double(28) + res29 = add64(res29, res29); // Double(29) + res30 = add64(res30, res30); // Double(30) + res31 = add64(res31, res31); // Double(31) + res32 = add64(res32, res32); // Double(32) + res33 = add64(res33, res33); // Double(33) + res34 = add64(res34, res34); // Double(34) + res35 = add64(res35, res35); // Double(35) + res24 = fma52lo(res24, a[12], a[12]); // Add sqr(24) + res25 = fma52hi(res25, a[12], a[12]); // Add sqr(24) + res26 = fma52lo(res26, a[13], a[13]); // Add sqr(26) + res27 = fma52hi(res27, a[13], a[13]); // Add sqr(26) + res28 = fma52lo(res28, a[14], a[14]); // Add sqr(28) + res29 = fma52hi(res29, a[14], a[14]); // Add sqr(28) + res30 = fma52lo(res30, a[15], a[15]); // Add sqr(30) + res31 = fma52hi(res31, a[15], a[15]); // Add sqr(30) + res32 = fma52lo(res32, a[16], a[16]); // Add sqr(32) + res33 = fma52hi(res33, a[16], a[16]); // Add sqr(32) + res34 = fma52lo(res34, a[17], a[17]); // Add sqr(34) + res35 = fma52hi(res35, a[17], a[17]); // Add sqr(34) + res36 = fma52lo(res36, a[17], a[19]); // Sum(36) + res37 = fma52hi(res37, a[17], a[19]); // Sum(36) + res37 = fma52lo(res37, a[18], a[19]); // Sum(37) + res38 = fma52hi(res38, a[18], a[19]); // Sum(37) + res36 = add64(res36, res36); // Double(36) + res37 = add64(res37, res37); // Double(37) + res38 = add64(res38, res38); // Double(38) + res36 = fma52lo(res36, a[18], a[18]); // Add sqr(36) + res37 = fma52hi(res37, a[18], a[18]); // Add sqr(36) + res38 = fma52lo(res38, a[19], a[19]); // Add sqr(38) + res39 = fma52hi(res39, a[19], a[19]); // Add sqr(38) + + // Generate u_i + U64 u0 = mul52lo(res0, k); + ASM("jmp l0\nl0:\n"); + + // Create u0 + fma52lo_mem(res0, res0, u0, m, SIMD_BYTES * 0); + fma52hi_mem(res1, res1, u0, m, SIMD_BYTES * 0); + res1 = fma52lo(res1, u0, m[1]); + res2 = fma52hi(res2, u0, m[1]); + res1 = add64(res1, srli64(res0, DIGIT_SIZE)); + U64 u1 = mul52lo(res1, k); + fma52lo_mem(res2, res2, u0, m, SIMD_BYTES * 2); + fma52hi_mem(res3, res3, u0, m, SIMD_BYTES * 2); + res3 = fma52lo(res3, u0, m[3]); + res4 = fma52hi(res4, u0, m[3]); + fma52lo_mem(res4, res4, u0, m, SIMD_BYTES * 4); + fma52hi_mem(res5, res5, u0, m, SIMD_BYTES * 4); + res5 = fma52lo(res5, u0, m[5]); + res6 = fma52hi(res6, u0, m[5]); + fma52lo_mem(res6, res6, u0, m, SIMD_BYTES * 6); + fma52hi_mem(res7, res7, u0, m, SIMD_BYTES * 6); + res7 = fma52lo(res7, u0, m[7]); + res8 = fma52hi(res8, u0, m[7]); + fma52lo_mem(res8, res8, u0, m, SIMD_BYTES * 8); + fma52hi_mem(res9, res9, u0, m, SIMD_BYTES * 8); + res9 = fma52lo(res9, u0, m[9]); + res10 = fma52hi(res10, u0, m[9]); + fma52lo_mem(res10, res10, u0, m, SIMD_BYTES * 10); + fma52hi_mem(res11, res11, u0, m, SIMD_BYTES * 10); + res11 = fma52lo(res11, u0, m[11]); + res12 = fma52hi(res12, u0, m[11]); + fma52lo_mem(res12, res12, u0, m, SIMD_BYTES * 12); + fma52hi_mem(res13, res13, u0, m, SIMD_BYTES * 12); + res13 = fma52lo(res13, u0, m[13]); + res14 = fma52hi(res14, u0, m[13]); + fma52lo_mem(res14, res14, u0, m, SIMD_BYTES * 14); + fma52hi_mem(res15, res15, u0, m, SIMD_BYTES * 14); + res15 = fma52lo(res15, u0, m[15]); + res16 = fma52hi(res16, u0, m[15]); + fma52lo_mem(res16, res16, u0, m, SIMD_BYTES * 16); + fma52hi_mem(res17, res17, u0, m, SIMD_BYTES * 16); + res17 = fma52lo(res17, u0, m[17]); + res18 = fma52hi(res18, u0, m[17]); + fma52lo_mem(res18, res18, u0, m, SIMD_BYTES * 18); + fma52hi_mem(res19, res19, u0, m, SIMD_BYTES * 18); + res19 = fma52lo(res19, u0, m[19]); + res20 = fma52hi(res20, u0, m[19]); + + // Create u1 + fma52lo_mem(res1, res1, u1, m, SIMD_BYTES * 0); + fma52hi_mem(res2, res2, u1, m, SIMD_BYTES * 0); + res2 = fma52lo(res2, u1, m[1]); + res3 = fma52hi(res3, u1, m[1]); + res2 = add64(res2, srli64(res1, DIGIT_SIZE)); + U64 u2 = mul52lo(res2, k); + fma52lo_mem(res3, res3, u1, m, SIMD_BYTES * 2); + fma52hi_mem(res4, res4, u1, m, SIMD_BYTES * 2); + res4 = fma52lo(res4, u1, m[3]); + res5 = fma52hi(res5, u1, m[3]); + fma52lo_mem(res5, res5, u1, m, SIMD_BYTES * 4); + fma52hi_mem(res6, res6, u1, m, SIMD_BYTES * 4); + res6 = fma52lo(res6, u1, m[5]); + res7 = fma52hi(res7, u1, m[5]); + fma52lo_mem(res7, res7, u1, m, SIMD_BYTES * 6); + fma52hi_mem(res8, res8, u1, m, SIMD_BYTES * 6); + res8 = fma52lo(res8, u1, m[7]); + res9 = fma52hi(res9, u1, m[7]); + fma52lo_mem(res9, res9, u1, m, SIMD_BYTES * 8); + fma52hi_mem(res10, res10, u1, m, SIMD_BYTES * 8); + res10 = fma52lo(res10, u1, m[9]); + res11 = fma52hi(res11, u1, m[9]); + fma52lo_mem(res11, res11, u1, m, SIMD_BYTES * 10); + fma52hi_mem(res12, res12, u1, m, SIMD_BYTES * 10); + res12 = fma52lo(res12, u1, m[11]); + res13 = fma52hi(res13, u1, m[11]); + fma52lo_mem(res13, res13, u1, m, SIMD_BYTES * 12); + fma52hi_mem(res14, res14, u1, m, SIMD_BYTES * 12); + res14 = fma52lo(res14, u1, m[13]); + res15 = fma52hi(res15, u1, m[13]); + fma52lo_mem(res15, res15, u1, m, SIMD_BYTES * 14); + fma52hi_mem(res16, res16, u1, m, SIMD_BYTES * 14); + res16 = fma52lo(res16, u1, m[15]); + res17 = fma52hi(res17, u1, m[15]); + fma52lo_mem(res17, res17, u1, m, SIMD_BYTES * 16); + fma52hi_mem(res18, res18, u1, m, SIMD_BYTES * 16); + res18 = fma52lo(res18, u1, m[17]); + res19 = fma52hi(res19, u1, m[17]); + fma52lo_mem(res19, res19, u1, m, SIMD_BYTES * 18); + fma52hi_mem(res20, res20, u1, m, SIMD_BYTES * 18); + res20 = fma52lo(res20, u1, m[19]); + res21 = fma52hi(res21, u1, m[19]); + ASM("jmp l2\nl2:\n"); + + // Create u2 + fma52lo_mem(res2, res2, u2, m, SIMD_BYTES * 0); + fma52hi_mem(res3, res3, u2, m, SIMD_BYTES * 0); + res3 = fma52lo(res3, u2, m[1]); + res4 = fma52hi(res4, u2, m[1]); + res3 = add64(res3, srli64(res2, DIGIT_SIZE)); + U64 u3 = mul52lo(res3, k); + fma52lo_mem(res4, res4, u2, m, SIMD_BYTES * 2); + fma52hi_mem(res5, res5, u2, m, SIMD_BYTES * 2); + res5 = fma52lo(res5, u2, m[3]); + res6 = fma52hi(res6, u2, m[3]); + fma52lo_mem(res6, res6, u2, m, SIMD_BYTES * 4); + fma52hi_mem(res7, res7, u2, m, SIMD_BYTES * 4); + res7 = fma52lo(res7, u2, m[5]); + res8 = fma52hi(res8, u2, m[5]); + fma52lo_mem(res8, res8, u2, m, SIMD_BYTES * 6); + fma52hi_mem(res9, res9, u2, m, SIMD_BYTES * 6); + res9 = fma52lo(res9, u2, m[7]); + res10 = fma52hi(res10, u2, m[7]); + fma52lo_mem(res10, res10, u2, m, SIMD_BYTES * 8); + fma52hi_mem(res11, res11, u2, m, SIMD_BYTES * 8); + res11 = fma52lo(res11, u2, m[9]); + res12 = fma52hi(res12, u2, m[9]); + fma52lo_mem(res12, res12, u2, m, SIMD_BYTES * 10); + fma52hi_mem(res13, res13, u2, m, SIMD_BYTES * 10); + res13 = fma52lo(res13, u2, m[11]); + res14 = fma52hi(res14, u2, m[11]); + fma52lo_mem(res14, res14, u2, m, SIMD_BYTES * 12); + fma52hi_mem(res15, res15, u2, m, SIMD_BYTES * 12); + res15 = fma52lo(res15, u2, m[13]); + res16 = fma52hi(res16, u2, m[13]); + fma52lo_mem(res16, res16, u2, m, SIMD_BYTES * 14); + fma52hi_mem(res17, res17, u2, m, SIMD_BYTES * 14); + res17 = fma52lo(res17, u2, m[15]); + res18 = fma52hi(res18, u2, m[15]); + fma52lo_mem(res18, res18, u2, m, SIMD_BYTES * 16); + fma52hi_mem(res19, res19, u2, m, SIMD_BYTES * 16); + res19 = fma52lo(res19, u2, m[17]); + res20 = fma52hi(res20, u2, m[17]); + fma52lo_mem(res20, res20, u2, m, SIMD_BYTES * 18); + fma52hi_mem(res21, res21, u2, m, SIMD_BYTES * 18); + res21 = fma52lo(res21, u2, m[19]); + res22 = fma52hi(res22, u2, m[19]); + + // Create u3 + fma52lo_mem(res3, res3, u3, m, SIMD_BYTES * 0); + fma52hi_mem(res4, res4, u3, m, SIMD_BYTES * 0); + res4 = fma52lo(res4, u3, m[1]); + res5 = fma52hi(res5, u3, m[1]); + res4 = add64(res4, srli64(res3, DIGIT_SIZE)); + U64 u4 = mul52lo(res4, k); + fma52lo_mem(res5, res5, u3, m, SIMD_BYTES * 2); + fma52hi_mem(res6, res6, u3, m, SIMD_BYTES * 2); + res6 = fma52lo(res6, u3, m[3]); + res7 = fma52hi(res7, u3, m[3]); + fma52lo_mem(res7, res7, u3, m, SIMD_BYTES * 4); + fma52hi_mem(res8, res8, u3, m, SIMD_BYTES * 4); + res8 = fma52lo(res8, u3, m[5]); + res9 = fma52hi(res9, u3, m[5]); + fma52lo_mem(res9, res9, u3, m, SIMD_BYTES * 6); + fma52hi_mem(res10, res10, u3, m, SIMD_BYTES * 6); + res10 = fma52lo(res10, u3, m[7]); + res11 = fma52hi(res11, u3, m[7]); + fma52lo_mem(res11, res11, u3, m, SIMD_BYTES * 8); + fma52hi_mem(res12, res12, u3, m, SIMD_BYTES * 8); + res12 = fma52lo(res12, u3, m[9]); + res13 = fma52hi(res13, u3, m[9]); + fma52lo_mem(res13, res13, u3, m, SIMD_BYTES * 10); + fma52hi_mem(res14, res14, u3, m, SIMD_BYTES * 10); + res14 = fma52lo(res14, u3, m[11]); + res15 = fma52hi(res15, u3, m[11]); + fma52lo_mem(res15, res15, u3, m, SIMD_BYTES * 12); + fma52hi_mem(res16, res16, u3, m, SIMD_BYTES * 12); + res16 = fma52lo(res16, u3, m[13]); + res17 = fma52hi(res17, u3, m[13]); + fma52lo_mem(res17, res17, u3, m, SIMD_BYTES * 14); + fma52hi_mem(res18, res18, u3, m, SIMD_BYTES * 14); + res18 = fma52lo(res18, u3, m[15]); + res19 = fma52hi(res19, u3, m[15]); + fma52lo_mem(res19, res19, u3, m, SIMD_BYTES * 16); + fma52hi_mem(res20, res20, u3, m, SIMD_BYTES * 16); + res20 = fma52lo(res20, u3, m[17]); + res21 = fma52hi(res21, u3, m[17]); + fma52lo_mem(res21, res21, u3, m, SIMD_BYTES * 18); + fma52hi_mem(res22, res22, u3, m, SIMD_BYTES * 18); + res22 = fma52lo(res22, u3, m[19]); + res23 = fma52hi(res23, u3, m[19]); + ASM("jmp l4\nl4:\n"); + + // Create u4 + fma52lo_mem(res4, res4, u4, m, SIMD_BYTES * 0); + fma52hi_mem(res5, res5, u4, m, SIMD_BYTES * 0); + res5 = fma52lo(res5, u4, m[1]); + res6 = fma52hi(res6, u4, m[1]); + res5 = add64(res5, srli64(res4, DIGIT_SIZE)); + U64 u5 = mul52lo(res5, k); + fma52lo_mem(res6, res6, u4, m, SIMD_BYTES * 2); + fma52hi_mem(res7, res7, u4, m, SIMD_BYTES * 2); + res7 = fma52lo(res7, u4, m[3]); + res8 = fma52hi(res8, u4, m[3]); + fma52lo_mem(res8, res8, u4, m, SIMD_BYTES * 4); + fma52hi_mem(res9, res9, u4, m, SIMD_BYTES * 4); + res9 = fma52lo(res9, u4, m[5]); + res10 = fma52hi(res10, u4, m[5]); + fma52lo_mem(res10, res10, u4, m, SIMD_BYTES * 6); + fma52hi_mem(res11, res11, u4, m, SIMD_BYTES * 6); + res11 = fma52lo(res11, u4, m[7]); + res12 = fma52hi(res12, u4, m[7]); + fma52lo_mem(res12, res12, u4, m, SIMD_BYTES * 8); + fma52hi_mem(res13, res13, u4, m, SIMD_BYTES * 8); + res13 = fma52lo(res13, u4, m[9]); + res14 = fma52hi(res14, u4, m[9]); + fma52lo_mem(res14, res14, u4, m, SIMD_BYTES * 10); + fma52hi_mem(res15, res15, u4, m, SIMD_BYTES * 10); + res15 = fma52lo(res15, u4, m[11]); + res16 = fma52hi(res16, u4, m[11]); + fma52lo_mem(res16, res16, u4, m, SIMD_BYTES * 12); + fma52hi_mem(res17, res17, u4, m, SIMD_BYTES * 12); + res17 = fma52lo(res17, u4, m[13]); + res18 = fma52hi(res18, u4, m[13]); + fma52lo_mem(res18, res18, u4, m, SIMD_BYTES * 14); + fma52hi_mem(res19, res19, u4, m, SIMD_BYTES * 14); + res19 = fma52lo(res19, u4, m[15]); + res20 = fma52hi(res20, u4, m[15]); + fma52lo_mem(res20, res20, u4, m, SIMD_BYTES * 16); + fma52hi_mem(res21, res21, u4, m, SIMD_BYTES * 16); + res21 = fma52lo(res21, u4, m[17]); + res22 = fma52hi(res22, u4, m[17]); + fma52lo_mem(res22, res22, u4, m, SIMD_BYTES * 18); + fma52hi_mem(res23, res23, u4, m, SIMD_BYTES * 18); + res23 = fma52lo(res23, u4, m[19]); + res24 = fma52hi(res24, u4, m[19]); + + // Create u5 + fma52lo_mem(res5, res5, u5, m, SIMD_BYTES * 0); + fma52hi_mem(res6, res6, u5, m, SIMD_BYTES * 0); + res6 = fma52lo(res6, u5, m[1]); + res7 = fma52hi(res7, u5, m[1]); + res6 = add64(res6, srli64(res5, DIGIT_SIZE)); + U64 u6 = mul52lo(res6, k); + fma52lo_mem(res7, res7, u5, m, SIMD_BYTES * 2); + fma52hi_mem(res8, res8, u5, m, SIMD_BYTES * 2); + res8 = fma52lo(res8, u5, m[3]); + res9 = fma52hi(res9, u5, m[3]); + fma52lo_mem(res9, res9, u5, m, SIMD_BYTES * 4); + fma52hi_mem(res10, res10, u5, m, SIMD_BYTES * 4); + res10 = fma52lo(res10, u5, m[5]); + res11 = fma52hi(res11, u5, m[5]); + fma52lo_mem(res11, res11, u5, m, SIMD_BYTES * 6); + fma52hi_mem(res12, res12, u5, m, SIMD_BYTES * 6); + res12 = fma52lo(res12, u5, m[7]); + res13 = fma52hi(res13, u5, m[7]); + fma52lo_mem(res13, res13, u5, m, SIMD_BYTES * 8); + fma52hi_mem(res14, res14, u5, m, SIMD_BYTES * 8); + res14 = fma52lo(res14, u5, m[9]); + res15 = fma52hi(res15, u5, m[9]); + fma52lo_mem(res15, res15, u5, m, SIMD_BYTES * 10); + fma52hi_mem(res16, res16, u5, m, SIMD_BYTES * 10); + res16 = fma52lo(res16, u5, m[11]); + res17 = fma52hi(res17, u5, m[11]); + fma52lo_mem(res17, res17, u5, m, SIMD_BYTES * 12); + fma52hi_mem(res18, res18, u5, m, SIMD_BYTES * 12); + res18 = fma52lo(res18, u5, m[13]); + res19 = fma52hi(res19, u5, m[13]); + fma52lo_mem(res19, res19, u5, m, SIMD_BYTES * 14); + fma52hi_mem(res20, res20, u5, m, SIMD_BYTES * 14); + res20 = fma52lo(res20, u5, m[15]); + res21 = fma52hi(res21, u5, m[15]); + fma52lo_mem(res21, res21, u5, m, SIMD_BYTES * 16); + fma52hi_mem(res22, res22, u5, m, SIMD_BYTES * 16); + res22 = fma52lo(res22, u5, m[17]); + res23 = fma52hi(res23, u5, m[17]); + fma52lo_mem(res23, res23, u5, m, SIMD_BYTES * 18); + fma52hi_mem(res24, res24, u5, m, SIMD_BYTES * 18); + res24 = fma52lo(res24, u5, m[19]); + res25 = fma52hi(res25, u5, m[19]); + ASM("jmp l6\nl6:\n"); + + // Create u6 + fma52lo_mem(res6, res6, u6, m, SIMD_BYTES * 0); + fma52hi_mem(res7, res7, u6, m, SIMD_BYTES * 0); + res7 = fma52lo(res7, u6, m[1]); + res8 = fma52hi(res8, u6, m[1]); + res7 = add64(res7, srli64(res6, DIGIT_SIZE)); + U64 u7 = mul52lo(res7, k); + fma52lo_mem(res8, res8, u6, m, SIMD_BYTES * 2); + fma52hi_mem(res9, res9, u6, m, SIMD_BYTES * 2); + res9 = fma52lo(res9, u6, m[3]); + res10 = fma52hi(res10, u6, m[3]); + fma52lo_mem(res10, res10, u6, m, SIMD_BYTES * 4); + fma52hi_mem(res11, res11, u6, m, SIMD_BYTES * 4); + res11 = fma52lo(res11, u6, m[5]); + res12 = fma52hi(res12, u6, m[5]); + fma52lo_mem(res12, res12, u6, m, SIMD_BYTES * 6); + fma52hi_mem(res13, res13, u6, m, SIMD_BYTES * 6); + res13 = fma52lo(res13, u6, m[7]); + res14 = fma52hi(res14, u6, m[7]); + fma52lo_mem(res14, res14, u6, m, SIMD_BYTES * 8); + fma52hi_mem(res15, res15, u6, m, SIMD_BYTES * 8); + res15 = fma52lo(res15, u6, m[9]); + res16 = fma52hi(res16, u6, m[9]); + fma52lo_mem(res16, res16, u6, m, SIMD_BYTES * 10); + fma52hi_mem(res17, res17, u6, m, SIMD_BYTES * 10); + res17 = fma52lo(res17, u6, m[11]); + res18 = fma52hi(res18, u6, m[11]); + fma52lo_mem(res18, res18, u6, m, SIMD_BYTES * 12); + fma52hi_mem(res19, res19, u6, m, SIMD_BYTES * 12); + res19 = fma52lo(res19, u6, m[13]); + res20 = fma52hi(res20, u6, m[13]); + fma52lo_mem(res20, res20, u6, m, SIMD_BYTES * 14); + fma52hi_mem(res21, res21, u6, m, SIMD_BYTES * 14); + res21 = fma52lo(res21, u6, m[15]); + res22 = fma52hi(res22, u6, m[15]); + fma52lo_mem(res22, res22, u6, m, SIMD_BYTES * 16); + fma52hi_mem(res23, res23, u6, m, SIMD_BYTES * 16); + res23 = fma52lo(res23, u6, m[17]); + res24 = fma52hi(res24, u6, m[17]); + fma52lo_mem(res24, res24, u6, m, SIMD_BYTES * 18); + fma52hi_mem(res25, res25, u6, m, SIMD_BYTES * 18); + res25 = fma52lo(res25, u6, m[19]); + res26 = fma52hi(res26, u6, m[19]); + + // Create u7 + fma52lo_mem(res7, res7, u7, m, SIMD_BYTES * 0); + fma52hi_mem(res8, res8, u7, m, SIMD_BYTES * 0); + res8 = fma52lo(res8, u7, m[1]); + res9 = fma52hi(res9, u7, m[1]); + res8 = add64(res8, srli64(res7, DIGIT_SIZE)); + U64 u8 = mul52lo(res8, k); + fma52lo_mem(res9, res9, u7, m, SIMD_BYTES * 2); + fma52hi_mem(res10, res10, u7, m, SIMD_BYTES * 2); + res10 = fma52lo(res10, u7, m[3]); + res11 = fma52hi(res11, u7, m[3]); + fma52lo_mem(res11, res11, u7, m, SIMD_BYTES * 4); + fma52hi_mem(res12, res12, u7, m, SIMD_BYTES * 4); + res12 = fma52lo(res12, u7, m[5]); + res13 = fma52hi(res13, u7, m[5]); + fma52lo_mem(res13, res13, u7, m, SIMD_BYTES * 6); + fma52hi_mem(res14, res14, u7, m, SIMD_BYTES * 6); + res14 = fma52lo(res14, u7, m[7]); + res15 = fma52hi(res15, u7, m[7]); + fma52lo_mem(res15, res15, u7, m, SIMD_BYTES * 8); + fma52hi_mem(res16, res16, u7, m, SIMD_BYTES * 8); + res16 = fma52lo(res16, u7, m[9]); + res17 = fma52hi(res17, u7, m[9]); + fma52lo_mem(res17, res17, u7, m, SIMD_BYTES * 10); + fma52hi_mem(res18, res18, u7, m, SIMD_BYTES * 10); + res18 = fma52lo(res18, u7, m[11]); + res19 = fma52hi(res19, u7, m[11]); + fma52lo_mem(res19, res19, u7, m, SIMD_BYTES * 12); + fma52hi_mem(res20, res20, u7, m, SIMD_BYTES * 12); + res20 = fma52lo(res20, u7, m[13]); + res21 = fma52hi(res21, u7, m[13]); + fma52lo_mem(res21, res21, u7, m, SIMD_BYTES * 14); + fma52hi_mem(res22, res22, u7, m, SIMD_BYTES * 14); + res22 = fma52lo(res22, u7, m[15]); + res23 = fma52hi(res23, u7, m[15]); + fma52lo_mem(res23, res23, u7, m, SIMD_BYTES * 16); + fma52hi_mem(res24, res24, u7, m, SIMD_BYTES * 16); + res24 = fma52lo(res24, u7, m[17]); + res25 = fma52hi(res25, u7, m[17]); + fma52lo_mem(res25, res25, u7, m, SIMD_BYTES * 18); + fma52hi_mem(res26, res26, u7, m, SIMD_BYTES * 18); + res26 = fma52lo(res26, u7, m[19]); + res27 = fma52hi(res27, u7, m[19]); + ASM("jmp l8\nl8:\n"); + + // Create u8 + fma52lo_mem(res8, res8, u8, m, SIMD_BYTES * 0); + fma52hi_mem(res9, res9, u8, m, SIMD_BYTES * 0); + res9 = fma52lo(res9, u8, m[1]); + res10 = fma52hi(res10, u8, m[1]); + res9 = add64(res9, srli64(res8, DIGIT_SIZE)); + U64 u9 = mul52lo(res9, k); + fma52lo_mem(res10, res10, u8, m, SIMD_BYTES * 2); + fma52hi_mem(res11, res11, u8, m, SIMD_BYTES * 2); + res11 = fma52lo(res11, u8, m[3]); + res12 = fma52hi(res12, u8, m[3]); + fma52lo_mem(res12, res12, u8, m, SIMD_BYTES * 4); + fma52hi_mem(res13, res13, u8, m, SIMD_BYTES * 4); + res13 = fma52lo(res13, u8, m[5]); + res14 = fma52hi(res14, u8, m[5]); + fma52lo_mem(res14, res14, u8, m, SIMD_BYTES * 6); + fma52hi_mem(res15, res15, u8, m, SIMD_BYTES * 6); + res15 = fma52lo(res15, u8, m[7]); + res16 = fma52hi(res16, u8, m[7]); + fma52lo_mem(res16, res16, u8, m, SIMD_BYTES * 8); + fma52hi_mem(res17, res17, u8, m, SIMD_BYTES * 8); + res17 = fma52lo(res17, u8, m[9]); + res18 = fma52hi(res18, u8, m[9]); + fma52lo_mem(res18, res18, u8, m, SIMD_BYTES * 10); + fma52hi_mem(res19, res19, u8, m, SIMD_BYTES * 10); + res19 = fma52lo(res19, u8, m[11]); + res20 = fma52hi(res20, u8, m[11]); + fma52lo_mem(res20, res20, u8, m, SIMD_BYTES * 12); + fma52hi_mem(res21, res21, u8, m, SIMD_BYTES * 12); + res21 = fma52lo(res21, u8, m[13]); + res22 = fma52hi(res22, u8, m[13]); + fma52lo_mem(res22, res22, u8, m, SIMD_BYTES * 14); + fma52hi_mem(res23, res23, u8, m, SIMD_BYTES * 14); + res23 = fma52lo(res23, u8, m[15]); + res24 = fma52hi(res24, u8, m[15]); + fma52lo_mem(res24, res24, u8, m, SIMD_BYTES * 16); + fma52hi_mem(res25, res25, u8, m, SIMD_BYTES * 16); + res25 = fma52lo(res25, u8, m[17]); + res26 = fma52hi(res26, u8, m[17]); + fma52lo_mem(res26, res26, u8, m, SIMD_BYTES * 18); + fma52hi_mem(res27, res27, u8, m, SIMD_BYTES * 18); + res27 = fma52lo(res27, u8, m[19]); + res28 = fma52hi(res28, u8, m[19]); + + // Create u9 + fma52lo_mem(res9, res9, u9, m, SIMD_BYTES * 0); + fma52hi_mem(res10, res10, u9, m, SIMD_BYTES * 0); + res10 = fma52lo(res10, u9, m[1]); + res11 = fma52hi(res11, u9, m[1]); + res10 = add64(res10, srli64(res9, DIGIT_SIZE)); + U64 u10 = mul52lo(res10, k); + fma52lo_mem(res11, res11, u9, m, SIMD_BYTES * 2); + fma52hi_mem(res12, res12, u9, m, SIMD_BYTES * 2); + res12 = fma52lo(res12, u9, m[3]); + res13 = fma52hi(res13, u9, m[3]); + fma52lo_mem(res13, res13, u9, m, SIMD_BYTES * 4); + fma52hi_mem(res14, res14, u9, m, SIMD_BYTES * 4); + res14 = fma52lo(res14, u9, m[5]); + res15 = fma52hi(res15, u9, m[5]); + fma52lo_mem(res15, res15, u9, m, SIMD_BYTES * 6); + fma52hi_mem(res16, res16, u9, m, SIMD_BYTES * 6); + res16 = fma52lo(res16, u9, m[7]); + res17 = fma52hi(res17, u9, m[7]); + fma52lo_mem(res17, res17, u9, m, SIMD_BYTES * 8); + fma52hi_mem(res18, res18, u9, m, SIMD_BYTES * 8); + res18 = fma52lo(res18, u9, m[9]); + res19 = fma52hi(res19, u9, m[9]); + fma52lo_mem(res19, res19, u9, m, SIMD_BYTES * 10); + fma52hi_mem(res20, res20, u9, m, SIMD_BYTES * 10); + res20 = fma52lo(res20, u9, m[11]); + res21 = fma52hi(res21, u9, m[11]); + fma52lo_mem(res21, res21, u9, m, SIMD_BYTES * 12); + fma52hi_mem(res22, res22, u9, m, SIMD_BYTES * 12); + res22 = fma52lo(res22, u9, m[13]); + res23 = fma52hi(res23, u9, m[13]); + fma52lo_mem(res23, res23, u9, m, SIMD_BYTES * 14); + fma52hi_mem(res24, res24, u9, m, SIMD_BYTES * 14); + res24 = fma52lo(res24, u9, m[15]); + res25 = fma52hi(res25, u9, m[15]); + fma52lo_mem(res25, res25, u9, m, SIMD_BYTES * 16); + fma52hi_mem(res26, res26, u9, m, SIMD_BYTES * 16); + res26 = fma52lo(res26, u9, m[17]); + res27 = fma52hi(res27, u9, m[17]); + fma52lo_mem(res27, res27, u9, m, SIMD_BYTES * 18); + fma52hi_mem(res28, res28, u9, m, SIMD_BYTES * 18); + res28 = fma52lo(res28, u9, m[19]); + res29 = fma52hi(res29, u9, m[19]); + ASM("jmp l10\nl10:\n"); + + // Create u10 + fma52lo_mem(res10, res10, u10, m, SIMD_BYTES * 0); + fma52hi_mem(res11, res11, u10, m, SIMD_BYTES * 0); + res11 = fma52lo(res11, u10, m[1]); + res12 = fma52hi(res12, u10, m[1]); + res11 = add64(res11, srli64(res10, DIGIT_SIZE)); + U64 u11 = mul52lo(res11, k); + fma52lo_mem(res12, res12, u10, m, SIMD_BYTES * 2); + fma52hi_mem(res13, res13, u10, m, SIMD_BYTES * 2); + res13 = fma52lo(res13, u10, m[3]); + res14 = fma52hi(res14, u10, m[3]); + fma52lo_mem(res14, res14, u10, m, SIMD_BYTES * 4); + fma52hi_mem(res15, res15, u10, m, SIMD_BYTES * 4); + res15 = fma52lo(res15, u10, m[5]); + res16 = fma52hi(res16, u10, m[5]); + fma52lo_mem(res16, res16, u10, m, SIMD_BYTES * 6); + fma52hi_mem(res17, res17, u10, m, SIMD_BYTES * 6); + res17 = fma52lo(res17, u10, m[7]); + res18 = fma52hi(res18, u10, m[7]); + fma52lo_mem(res18, res18, u10, m, SIMD_BYTES * 8); + fma52hi_mem(res19, res19, u10, m, SIMD_BYTES * 8); + res19 = fma52lo(res19, u10, m[9]); + res20 = fma52hi(res20, u10, m[9]); + fma52lo_mem(res20, res20, u10, m, SIMD_BYTES * 10); + fma52hi_mem(res21, res21, u10, m, SIMD_BYTES * 10); + res21 = fma52lo(res21, u10, m[11]); + res22 = fma52hi(res22, u10, m[11]); + fma52lo_mem(res22, res22, u10, m, SIMD_BYTES * 12); + fma52hi_mem(res23, res23, u10, m, SIMD_BYTES * 12); + res23 = fma52lo(res23, u10, m[13]); + res24 = fma52hi(res24, u10, m[13]); + fma52lo_mem(res24, res24, u10, m, SIMD_BYTES * 14); + fma52hi_mem(res25, res25, u10, m, SIMD_BYTES * 14); + res25 = fma52lo(res25, u10, m[15]); + res26 = fma52hi(res26, u10, m[15]); + fma52lo_mem(res26, res26, u10, m, SIMD_BYTES * 16); + fma52hi_mem(res27, res27, u10, m, SIMD_BYTES * 16); + res27 = fma52lo(res27, u10, m[17]); + res28 = fma52hi(res28, u10, m[17]); + fma52lo_mem(res28, res28, u10, m, SIMD_BYTES * 18); + fma52hi_mem(res29, res29, u10, m, SIMD_BYTES * 18); + res29 = fma52lo(res29, u10, m[19]); + res30 = fma52hi(res30, u10, m[19]); + + // Create u11 + fma52lo_mem(res11, res11, u11, m, SIMD_BYTES * 0); + fma52hi_mem(res12, res12, u11, m, SIMD_BYTES * 0); + res12 = fma52lo(res12, u11, m[1]); + res13 = fma52hi(res13, u11, m[1]); + res12 = add64(res12, srli64(res11, DIGIT_SIZE)); + U64 u12 = mul52lo(res12, k); + fma52lo_mem(res13, res13, u11, m, SIMD_BYTES * 2); + fma52hi_mem(res14, res14, u11, m, SIMD_BYTES * 2); + res14 = fma52lo(res14, u11, m[3]); + res15 = fma52hi(res15, u11, m[3]); + fma52lo_mem(res15, res15, u11, m, SIMD_BYTES * 4); + fma52hi_mem(res16, res16, u11, m, SIMD_BYTES * 4); + res16 = fma52lo(res16, u11, m[5]); + res17 = fma52hi(res17, u11, m[5]); + fma52lo_mem(res17, res17, u11, m, SIMD_BYTES * 6); + fma52hi_mem(res18, res18, u11, m, SIMD_BYTES * 6); + res18 = fma52lo(res18, u11, m[7]); + res19 = fma52hi(res19, u11, m[7]); + fma52lo_mem(res19, res19, u11, m, SIMD_BYTES * 8); + fma52hi_mem(res20, res20, u11, m, SIMD_BYTES * 8); + res20 = fma52lo(res20, u11, m[9]); + res21 = fma52hi(res21, u11, m[9]); + fma52lo_mem(res21, res21, u11, m, SIMD_BYTES * 10); + fma52hi_mem(res22, res22, u11, m, SIMD_BYTES * 10); + res22 = fma52lo(res22, u11, m[11]); + res23 = fma52hi(res23, u11, m[11]); + fma52lo_mem(res23, res23, u11, m, SIMD_BYTES * 12); + fma52hi_mem(res24, res24, u11, m, SIMD_BYTES * 12); + res24 = fma52lo(res24, u11, m[13]); + res25 = fma52hi(res25, u11, m[13]); + fma52lo_mem(res25, res25, u11, m, SIMD_BYTES * 14); + fma52hi_mem(res26, res26, u11, m, SIMD_BYTES * 14); + res26 = fma52lo(res26, u11, m[15]); + res27 = fma52hi(res27, u11, m[15]); + fma52lo_mem(res27, res27, u11, m, SIMD_BYTES * 16); + fma52hi_mem(res28, res28, u11, m, SIMD_BYTES * 16); + res28 = fma52lo(res28, u11, m[17]); + res29 = fma52hi(res29, u11, m[17]); + fma52lo_mem(res29, res29, u11, m, SIMD_BYTES * 18); + fma52hi_mem(res30, res30, u11, m, SIMD_BYTES * 18); + res30 = fma52lo(res30, u11, m[19]); + res31 = fma52hi(res31, u11, m[19]); + ASM("jmp l12\nl12:\n"); + + // Create u12 + fma52lo_mem(res12, res12, u12, m, SIMD_BYTES * 0); + fma52hi_mem(res13, res13, u12, m, SIMD_BYTES * 0); + res13 = fma52lo(res13, u12, m[1]); + res14 = fma52hi(res14, u12, m[1]); + res13 = add64(res13, srli64(res12, DIGIT_SIZE)); + U64 u13 = mul52lo(res13, k); + fma52lo_mem(res14, res14, u12, m, SIMD_BYTES * 2); + fma52hi_mem(res15, res15, u12, m, SIMD_BYTES * 2); + res15 = fma52lo(res15, u12, m[3]); + res16 = fma52hi(res16, u12, m[3]); + fma52lo_mem(res16, res16, u12, m, SIMD_BYTES * 4); + fma52hi_mem(res17, res17, u12, m, SIMD_BYTES * 4); + res17 = fma52lo(res17, u12, m[5]); + res18 = fma52hi(res18, u12, m[5]); + fma52lo_mem(res18, res18, u12, m, SIMD_BYTES * 6); + fma52hi_mem(res19, res19, u12, m, SIMD_BYTES * 6); + res19 = fma52lo(res19, u12, m[7]); + res20 = fma52hi(res20, u12, m[7]); + fma52lo_mem(res20, res20, u12, m, SIMD_BYTES * 8); + fma52hi_mem(res21, res21, u12, m, SIMD_BYTES * 8); + res21 = fma52lo(res21, u12, m[9]); + res22 = fma52hi(res22, u12, m[9]); + fma52lo_mem(res22, res22, u12, m, SIMD_BYTES * 10); + fma52hi_mem(res23, res23, u12, m, SIMD_BYTES * 10); + res23 = fma52lo(res23, u12, m[11]); + res24 = fma52hi(res24, u12, m[11]); + fma52lo_mem(res24, res24, u12, m, SIMD_BYTES * 12); + fma52hi_mem(res25, res25, u12, m, SIMD_BYTES * 12); + res25 = fma52lo(res25, u12, m[13]); + res26 = fma52hi(res26, u12, m[13]); + fma52lo_mem(res26, res26, u12, m, SIMD_BYTES * 14); + fma52hi_mem(res27, res27, u12, m, SIMD_BYTES * 14); + res27 = fma52lo(res27, u12, m[15]); + res28 = fma52hi(res28, u12, m[15]); + fma52lo_mem(res28, res28, u12, m, SIMD_BYTES * 16); + fma52hi_mem(res29, res29, u12, m, SIMD_BYTES * 16); + res29 = fma52lo(res29, u12, m[17]); + res30 = fma52hi(res30, u12, m[17]); + fma52lo_mem(res30, res30, u12, m, SIMD_BYTES * 18); + fma52hi_mem(res31, res31, u12, m, SIMD_BYTES * 18); + res31 = fma52lo(res31, u12, m[19]); + res32 = fma52hi(res32, u12, m[19]); + + // Create u13 + fma52lo_mem(res13, res13, u13, m, SIMD_BYTES * 0); + fma52hi_mem(res14, res14, u13, m, SIMD_BYTES * 0); + res14 = fma52lo(res14, u13, m[1]); + res15 = fma52hi(res15, u13, m[1]); + res14 = add64(res14, srli64(res13, DIGIT_SIZE)); + U64 u14 = mul52lo(res14, k); + fma52lo_mem(res15, res15, u13, m, SIMD_BYTES * 2); + fma52hi_mem(res16, res16, u13, m, SIMD_BYTES * 2); + res16 = fma52lo(res16, u13, m[3]); + res17 = fma52hi(res17, u13, m[3]); + fma52lo_mem(res17, res17, u13, m, SIMD_BYTES * 4); + fma52hi_mem(res18, res18, u13, m, SIMD_BYTES * 4); + res18 = fma52lo(res18, u13, m[5]); + res19 = fma52hi(res19, u13, m[5]); + fma52lo_mem(res19, res19, u13, m, SIMD_BYTES * 6); + fma52hi_mem(res20, res20, u13, m, SIMD_BYTES * 6); + res20 = fma52lo(res20, u13, m[7]); + res21 = fma52hi(res21, u13, m[7]); + fma52lo_mem(res21, res21, u13, m, SIMD_BYTES * 8); + fma52hi_mem(res22, res22, u13, m, SIMD_BYTES * 8); + res22 = fma52lo(res22, u13, m[9]); + res23 = fma52hi(res23, u13, m[9]); + fma52lo_mem(res23, res23, u13, m, SIMD_BYTES * 10); + fma52hi_mem(res24, res24, u13, m, SIMD_BYTES * 10); + res24 = fma52lo(res24, u13, m[11]); + res25 = fma52hi(res25, u13, m[11]); + fma52lo_mem(res25, res25, u13, m, SIMD_BYTES * 12); + fma52hi_mem(res26, res26, u13, m, SIMD_BYTES * 12); + res26 = fma52lo(res26, u13, m[13]); + res27 = fma52hi(res27, u13, m[13]); + fma52lo_mem(res27, res27, u13, m, SIMD_BYTES * 14); + fma52hi_mem(res28, res28, u13, m, SIMD_BYTES * 14); + res28 = fma52lo(res28, u13, m[15]); + res29 = fma52hi(res29, u13, m[15]); + fma52lo_mem(res29, res29, u13, m, SIMD_BYTES * 16); + fma52hi_mem(res30, res30, u13, m, SIMD_BYTES * 16); + res30 = fma52lo(res30, u13, m[17]); + res31 = fma52hi(res31, u13, m[17]); + fma52lo_mem(res31, res31, u13, m, SIMD_BYTES * 18); + fma52hi_mem(res32, res32, u13, m, SIMD_BYTES * 18); + res32 = fma52lo(res32, u13, m[19]); + res33 = fma52hi(res33, u13, m[19]); + ASM("jmp l14\nl14:\n"); + + // Create u14 + fma52lo_mem(res14, res14, u14, m, SIMD_BYTES * 0); + fma52hi_mem(res15, res15, u14, m, SIMD_BYTES * 0); + res15 = fma52lo(res15, u14, m[1]); + res16 = fma52hi(res16, u14, m[1]); + res15 = add64(res15, srli64(res14, DIGIT_SIZE)); + U64 u15 = mul52lo(res15, k); + fma52lo_mem(res16, res16, u14, m, SIMD_BYTES * 2); + fma52hi_mem(res17, res17, u14, m, SIMD_BYTES * 2); + res17 = fma52lo(res17, u14, m[3]); + res18 = fma52hi(res18, u14, m[3]); + fma52lo_mem(res18, res18, u14, m, SIMD_BYTES * 4); + fma52hi_mem(res19, res19, u14, m, SIMD_BYTES * 4); + res19 = fma52lo(res19, u14, m[5]); + res20 = fma52hi(res20, u14, m[5]); + fma52lo_mem(res20, res20, u14, m, SIMD_BYTES * 6); + fma52hi_mem(res21, res21, u14, m, SIMD_BYTES * 6); + res21 = fma52lo(res21, u14, m[7]); + res22 = fma52hi(res22, u14, m[7]); + fma52lo_mem(res22, res22, u14, m, SIMD_BYTES * 8); + fma52hi_mem(res23, res23, u14, m, SIMD_BYTES * 8); + res23 = fma52lo(res23, u14, m[9]); + res24 = fma52hi(res24, u14, m[9]); + fma52lo_mem(res24, res24, u14, m, SIMD_BYTES * 10); + fma52hi_mem(res25, res25, u14, m, SIMD_BYTES * 10); + res25 = fma52lo(res25, u14, m[11]); + res26 = fma52hi(res26, u14, m[11]); + fma52lo_mem(res26, res26, u14, m, SIMD_BYTES * 12); + fma52hi_mem(res27, res27, u14, m, SIMD_BYTES * 12); + res27 = fma52lo(res27, u14, m[13]); + res28 = fma52hi(res28, u14, m[13]); + fma52lo_mem(res28, res28, u14, m, SIMD_BYTES * 14); + fma52hi_mem(res29, res29, u14, m, SIMD_BYTES * 14); + res29 = fma52lo(res29, u14, m[15]); + res30 = fma52hi(res30, u14, m[15]); + fma52lo_mem(res30, res30, u14, m, SIMD_BYTES * 16); + fma52hi_mem(res31, res31, u14, m, SIMD_BYTES * 16); + res31 = fma52lo(res31, u14, m[17]); + res32 = fma52hi(res32, u14, m[17]); + fma52lo_mem(res32, res32, u14, m, SIMD_BYTES * 18); + fma52hi_mem(res33, res33, u14, m, SIMD_BYTES * 18); + res33 = fma52lo(res33, u14, m[19]); + res34 = fma52hi(res34, u14, m[19]); + + // Create u15 + fma52lo_mem(res15, res15, u15, m, SIMD_BYTES * 0); + fma52hi_mem(res16, res16, u15, m, SIMD_BYTES * 0); + res16 = fma52lo(res16, u15, m[1]); + res17 = fma52hi(res17, u15, m[1]); + res16 = add64(res16, srli64(res15, DIGIT_SIZE)); + U64 u16 = mul52lo(res16, k); + fma52lo_mem(res17, res17, u15, m, SIMD_BYTES * 2); + fma52hi_mem(res18, res18, u15, m, SIMD_BYTES * 2); + res18 = fma52lo(res18, u15, m[3]); + res19 = fma52hi(res19, u15, m[3]); + fma52lo_mem(res19, res19, u15, m, SIMD_BYTES * 4); + fma52hi_mem(res20, res20, u15, m, SIMD_BYTES * 4); + res20 = fma52lo(res20, u15, m[5]); + res21 = fma52hi(res21, u15, m[5]); + fma52lo_mem(res21, res21, u15, m, SIMD_BYTES * 6); + fma52hi_mem(res22, res22, u15, m, SIMD_BYTES * 6); + res22 = fma52lo(res22, u15, m[7]); + res23 = fma52hi(res23, u15, m[7]); + fma52lo_mem(res23, res23, u15, m, SIMD_BYTES * 8); + fma52hi_mem(res24, res24, u15, m, SIMD_BYTES * 8); + res24 = fma52lo(res24, u15, m[9]); + res25 = fma52hi(res25, u15, m[9]); + fma52lo_mem(res25, res25, u15, m, SIMD_BYTES * 10); + fma52hi_mem(res26, res26, u15, m, SIMD_BYTES * 10); + res26 = fma52lo(res26, u15, m[11]); + res27 = fma52hi(res27, u15, m[11]); + fma52lo_mem(res27, res27, u15, m, SIMD_BYTES * 12); + fma52hi_mem(res28, res28, u15, m, SIMD_BYTES * 12); + res28 = fma52lo(res28, u15, m[13]); + res29 = fma52hi(res29, u15, m[13]); + fma52lo_mem(res29, res29, u15, m, SIMD_BYTES * 14); + fma52hi_mem(res30, res30, u15, m, SIMD_BYTES * 14); + res30 = fma52lo(res30, u15, m[15]); + res31 = fma52hi(res31, u15, m[15]); + fma52lo_mem(res31, res31, u15, m, SIMD_BYTES * 16); + fma52hi_mem(res32, res32, u15, m, SIMD_BYTES * 16); + res32 = fma52lo(res32, u15, m[17]); + res33 = fma52hi(res33, u15, m[17]); + fma52lo_mem(res33, res33, u15, m, SIMD_BYTES * 18); + fma52hi_mem(res34, res34, u15, m, SIMD_BYTES * 18); + res34 = fma52lo(res34, u15, m[19]); + res35 = fma52hi(res35, u15, m[19]); + ASM("jmp l16\nl16:\n"); + + // Create u16 + fma52lo_mem(res16, res16, u16, m, SIMD_BYTES * 0); + fma52hi_mem(res17, res17, u16, m, SIMD_BYTES * 0); + res17 = fma52lo(res17, u16, m[1]); + res18 = fma52hi(res18, u16, m[1]); + res17 = add64(res17, srli64(res16, DIGIT_SIZE)); + U64 u17 = mul52lo(res17, k); + fma52lo_mem(res18, res18, u16, m, SIMD_BYTES * 2); + fma52hi_mem(res19, res19, u16, m, SIMD_BYTES * 2); + res19 = fma52lo(res19, u16, m[3]); + res20 = fma52hi(res20, u16, m[3]); + fma52lo_mem(res20, res20, u16, m, SIMD_BYTES * 4); + fma52hi_mem(res21, res21, u16, m, SIMD_BYTES * 4); + res21 = fma52lo(res21, u16, m[5]); + res22 = fma52hi(res22, u16, m[5]); + fma52lo_mem(res22, res22, u16, m, SIMD_BYTES * 6); + fma52hi_mem(res23, res23, u16, m, SIMD_BYTES * 6); + res23 = fma52lo(res23, u16, m[7]); + res24 = fma52hi(res24, u16, m[7]); + fma52lo_mem(res24, res24, u16, m, SIMD_BYTES * 8); + fma52hi_mem(res25, res25, u16, m, SIMD_BYTES * 8); + res25 = fma52lo(res25, u16, m[9]); + res26 = fma52hi(res26, u16, m[9]); + fma52lo_mem(res26, res26, u16, m, SIMD_BYTES * 10); + fma52hi_mem(res27, res27, u16, m, SIMD_BYTES * 10); + res27 = fma52lo(res27, u16, m[11]); + res28 = fma52hi(res28, u16, m[11]); + fma52lo_mem(res28, res28, u16, m, SIMD_BYTES * 12); + fma52hi_mem(res29, res29, u16, m, SIMD_BYTES * 12); + res29 = fma52lo(res29, u16, m[13]); + res30 = fma52hi(res30, u16, m[13]); + fma52lo_mem(res30, res30, u16, m, SIMD_BYTES * 14); + fma52hi_mem(res31, res31, u16, m, SIMD_BYTES * 14); + res31 = fma52lo(res31, u16, m[15]); + res32 = fma52hi(res32, u16, m[15]); + fma52lo_mem(res32, res32, u16, m, SIMD_BYTES * 16); + fma52hi_mem(res33, res33, u16, m, SIMD_BYTES * 16); + res33 = fma52lo(res33, u16, m[17]); + res34 = fma52hi(res34, u16, m[17]); + fma52lo_mem(res34, res34, u16, m, SIMD_BYTES * 18); + fma52hi_mem(res35, res35, u16, m, SIMD_BYTES * 18); + res35 = fma52lo(res35, u16, m[19]); + res36 = fma52hi(res36, u16, m[19]); + + // Create u17 + fma52lo_mem(res17, res17, u17, m, SIMD_BYTES * 0); + fma52hi_mem(res18, res18, u17, m, SIMD_BYTES * 0); + res18 = fma52lo(res18, u17, m[1]); + res19 = fma52hi(res19, u17, m[1]); + res18 = add64(res18, srli64(res17, DIGIT_SIZE)); + U64 u18 = mul52lo(res18, k); + fma52lo_mem(res19, res19, u17, m, SIMD_BYTES * 2); + fma52hi_mem(res20, res20, u17, m, SIMD_BYTES * 2); + res20 = fma52lo(res20, u17, m[3]); + res21 = fma52hi(res21, u17, m[3]); + fma52lo_mem(res21, res21, u17, m, SIMD_BYTES * 4); + fma52hi_mem(res22, res22, u17, m, SIMD_BYTES * 4); + res22 = fma52lo(res22, u17, m[5]); + res23 = fma52hi(res23, u17, m[5]); + fma52lo_mem(res23, res23, u17, m, SIMD_BYTES * 6); + fma52hi_mem(res24, res24, u17, m, SIMD_BYTES * 6); + res24 = fma52lo(res24, u17, m[7]); + res25 = fma52hi(res25, u17, m[7]); + fma52lo_mem(res25, res25, u17, m, SIMD_BYTES * 8); + fma52hi_mem(res26, res26, u17, m, SIMD_BYTES * 8); + res26 = fma52lo(res26, u17, m[9]); + res27 = fma52hi(res27, u17, m[9]); + fma52lo_mem(res27, res27, u17, m, SIMD_BYTES * 10); + fma52hi_mem(res28, res28, u17, m, SIMD_BYTES * 10); + res28 = fma52lo(res28, u17, m[11]); + res29 = fma52hi(res29, u17, m[11]); + fma52lo_mem(res29, res29, u17, m, SIMD_BYTES * 12); + fma52hi_mem(res30, res30, u17, m, SIMD_BYTES * 12); + res30 = fma52lo(res30, u17, m[13]); + res31 = fma52hi(res31, u17, m[13]); + fma52lo_mem(res31, res31, u17, m, SIMD_BYTES * 14); + fma52hi_mem(res32, res32, u17, m, SIMD_BYTES * 14); + res32 = fma52lo(res32, u17, m[15]); + res33 = fma52hi(res33, u17, m[15]); + fma52lo_mem(res33, res33, u17, m, SIMD_BYTES * 16); + fma52hi_mem(res34, res34, u17, m, SIMD_BYTES * 16); + res34 = fma52lo(res34, u17, m[17]); + res35 = fma52hi(res35, u17, m[17]); + fma52lo_mem(res35, res35, u17, m, SIMD_BYTES * 18); + fma52hi_mem(res36, res36, u17, m, SIMD_BYTES * 18); + res36 = fma52lo(res36, u17, m[19]); + res37 = fma52hi(res37, u17, m[19]); + ASM("jmp l18\nl18:\n"); + + // Create u18 + fma52lo_mem(res18, res18, u18, m, SIMD_BYTES * 0); + fma52hi_mem(res19, res19, u18, m, SIMD_BYTES * 0); + res19 = fma52lo(res19, u18, m[1]); + res20 = fma52hi(res20, u18, m[1]); + res19 = add64(res19, srli64(res18, DIGIT_SIZE)); + U64 u19 = mul52lo(res19, k); + fma52lo_mem(res20, res20, u18, m, SIMD_BYTES * 2); + fma52hi_mem(res21, res21, u18, m, SIMD_BYTES * 2); + res21 = fma52lo(res21, u18, m[3]); + res22 = fma52hi(res22, u18, m[3]); + fma52lo_mem(res22, res22, u18, m, SIMD_BYTES * 4); + fma52hi_mem(res23, res23, u18, m, SIMD_BYTES * 4); + res23 = fma52lo(res23, u18, m[5]); + res24 = fma52hi(res24, u18, m[5]); + fma52lo_mem(res24, res24, u18, m, SIMD_BYTES * 6); + fma52hi_mem(res25, res25, u18, m, SIMD_BYTES * 6); + res25 = fma52lo(res25, u18, m[7]); + res26 = fma52hi(res26, u18, m[7]); + fma52lo_mem(res26, res26, u18, m, SIMD_BYTES * 8); + fma52hi_mem(res27, res27, u18, m, SIMD_BYTES * 8); + res27 = fma52lo(res27, u18, m[9]); + res28 = fma52hi(res28, u18, m[9]); + fma52lo_mem(res28, res28, u18, m, SIMD_BYTES * 10); + fma52hi_mem(res29, res29, u18, m, SIMD_BYTES * 10); + res29 = fma52lo(res29, u18, m[11]); + res30 = fma52hi(res30, u18, m[11]); + fma52lo_mem(res30, res30, u18, m, SIMD_BYTES * 12); + fma52hi_mem(res31, res31, u18, m, SIMD_BYTES * 12); + res31 = fma52lo(res31, u18, m[13]); + res32 = fma52hi(res32, u18, m[13]); + fma52lo_mem(res32, res32, u18, m, SIMD_BYTES * 14); + fma52hi_mem(res33, res33, u18, m, SIMD_BYTES * 14); + res33 = fma52lo(res33, u18, m[15]); + res34 = fma52hi(res34, u18, m[15]); + fma52lo_mem(res34, res34, u18, m, SIMD_BYTES * 16); + fma52hi_mem(res35, res35, u18, m, SIMD_BYTES * 16); + res35 = fma52lo(res35, u18, m[17]); + res36 = fma52hi(res36, u18, m[17]); + fma52lo_mem(res36, res36, u18, m, SIMD_BYTES * 18); + fma52hi_mem(res37, res37, u18, m, SIMD_BYTES * 18); + res37 = fma52lo(res37, u18, m[19]); + res38 = fma52hi(res38, u18, m[19]); + + // Create u19 + fma52lo_mem(res19, res19, u19, m, SIMD_BYTES * 0); + fma52hi_mem(res20, res20, u19, m, SIMD_BYTES * 0); + res20 = fma52lo(res20, u19, m[1]); + res21 = fma52hi(res21, u19, m[1]); + res20 = add64(res20, srli64(res19, DIGIT_SIZE)); + fma52lo_mem(res21, res21, u19, m, SIMD_BYTES * 2); + fma52hi_mem(res22, res22, u19, m, SIMD_BYTES * 2); + res22 = fma52lo(res22, u19, m[3]); + res23 = fma52hi(res23, u19, m[3]); + fma52lo_mem(res23, res23, u19, m, SIMD_BYTES * 4); + fma52hi_mem(res24, res24, u19, m, SIMD_BYTES * 4); + res24 = fma52lo(res24, u19, m[5]); + res25 = fma52hi(res25, u19, m[5]); + fma52lo_mem(res25, res25, u19, m, SIMD_BYTES * 6); + fma52hi_mem(res26, res26, u19, m, SIMD_BYTES * 6); + res26 = fma52lo(res26, u19, m[7]); + res27 = fma52hi(res27, u19, m[7]); + fma52lo_mem(res27, res27, u19, m, SIMD_BYTES * 8); + fma52hi_mem(res28, res28, u19, m, SIMD_BYTES * 8); + res28 = fma52lo(res28, u19, m[9]); + res29 = fma52hi(res29, u19, m[9]); + fma52lo_mem(res29, res29, u19, m, SIMD_BYTES * 10); + fma52hi_mem(res30, res30, u19, m, SIMD_BYTES * 10); + res30 = fma52lo(res30, u19, m[11]); + res31 = fma52hi(res31, u19, m[11]); + fma52lo_mem(res31, res31, u19, m, SIMD_BYTES * 12); + fma52hi_mem(res32, res32, u19, m, SIMD_BYTES * 12); + res32 = fma52lo(res32, u19, m[13]); + res33 = fma52hi(res33, u19, m[13]); + fma52lo_mem(res33, res33, u19, m, SIMD_BYTES * 14); + fma52hi_mem(res34, res34, u19, m, SIMD_BYTES * 14); + res34 = fma52lo(res34, u19, m[15]); + res35 = fma52hi(res35, u19, m[15]); + fma52lo_mem(res35, res35, u19, m, SIMD_BYTES * 16); + fma52hi_mem(res36, res36, u19, m, SIMD_BYTES * 16); + res36 = fma52lo(res36, u19, m[17]); + res37 = fma52hi(res37, u19, m[17]); + fma52lo_mem(res37, res37, u19, m, SIMD_BYTES * 18); + fma52hi_mem(res38, res38, u19, m, SIMD_BYTES * 18); + res38 = fma52lo(res38, u19, m[19]); + res39 = fma52hi(res39, u19, m[19]); + + // Normalization + r[0] = res20; + res21 = add64(res21, srli64(res20, DIGIT_SIZE)); + r[1] = res21; + res22 = add64(res22, srli64(res21, DIGIT_SIZE)); + r[2] = res22; + res23 = add64(res23, srli64(res22, DIGIT_SIZE)); + r[3] = res23; + res24 = add64(res24, srli64(res23, DIGIT_SIZE)); + r[4] = res24; + res25 = add64(res25, srli64(res24, DIGIT_SIZE)); + r[5] = res25; + res26 = add64(res26, srli64(res25, DIGIT_SIZE)); + r[6] = res26; + res27 = add64(res27, srli64(res26, DIGIT_SIZE)); + r[7] = res27; + res28 = add64(res28, srli64(res27, DIGIT_SIZE)); + r[8] = res28; + res29 = add64(res29, srli64(res28, DIGIT_SIZE)); + r[9] = res29; + res30 = add64(res30, srli64(res29, DIGIT_SIZE)); + r[10] = res30; + res31 = add64(res31, srli64(res30, DIGIT_SIZE)); + r[11] = res31; + res32 = add64(res32, srli64(res31, DIGIT_SIZE)); + r[12] = res32; + res33 = add64(res33, srli64(res32, DIGIT_SIZE)); + r[13] = res33; + res34 = add64(res34, srli64(res33, DIGIT_SIZE)); + r[14] = res34; + res35 = add64(res35, srli64(res34, DIGIT_SIZE)); + r[15] = res35; + res36 = add64(res36, srli64(res35, DIGIT_SIZE)); + r[16] = res36; + res37 = add64(res37, srli64(res36, DIGIT_SIZE)); + r[17] = res37; + res38 = add64(res38, srli64(res37, DIGIT_SIZE)); + r[18] = res38; + res39 = add64(res39, srli64(res38, DIGIT_SIZE)); + r[19] = res39; + a = (U64 *)out_mb; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams5x52x40_diagonal_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams5x52x40_diagonal_mb8.c new file mode 100644 index 0000000..7434c09 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_ams5x52x40_diagonal_mb8.c @@ -0,0 +1,2277 @@ +/******************************************************************************* + * Copyright (C) 2019 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include + +#ifdef __GNUC__ +#define ASM(a) __asm__(a); +#else +#define ASM(a) +#endif + +void AMS5x52x40_diagonal_mb8(int64u *out_mb, const int64u *inpA_mb, + const int64u *inpM_mb, const int64u *k0_mb) { + __ALIGN64 U64 res[80]; + __ALIGN64 U64 u[40]; + U64 k; + U64 *a = (U64 *)inpA_mb; + U64 *m = (U64 *)inpM_mb; + U64 *r = (U64 *)out_mb; + int iter; + const int iters = 5; + + k = loadu64((U64 *)k0_mb); + for (iter = 0; iter < iters; ++iter) { + int i; + for (i = 0; i < 80; ++i) + res[i] = get_zero64(); + + // Calculate full square + res[1] = fma52lo(res[1], a[0], a[1]); // Sum(1) + res[2] = fma52hi(res[2], a[0], a[1]); // Sum(1) + res[2] = fma52lo(res[2], a[0], a[2]); // Sum(2) + res[3] = fma52hi(res[3], a[0], a[2]); // Sum(2) + res[3] = fma52lo(res[3], a[1], a[2]); // Sum(3) + res[4] = fma52hi(res[4], a[1], a[2]); // Sum(3) + res[3] = fma52lo(res[3], a[0], a[3]); // Sum(3) + res[4] = fma52hi(res[4], a[0], a[3]); // Sum(3) + res[4] = fma52lo(res[4], a[1], a[3]); // Sum(4) + res[5] = fma52hi(res[5], a[1], a[3]); // Sum(4) + res[5] = fma52lo(res[5], a[2], a[3]); // Sum(5) + res[6] = fma52hi(res[6], a[2], a[3]); // Sum(5) + res[4] = fma52lo(res[4], a[0], a[4]); // Sum(4) + res[5] = fma52hi(res[5], a[0], a[4]); // Sum(4) + res[5] = fma52lo(res[5], a[1], a[4]); // Sum(5) + res[6] = fma52hi(res[6], a[1], a[4]); // Sum(5) + res[6] = fma52lo(res[6], a[2], a[4]); // Sum(6) + res[7] = fma52hi(res[7], a[2], a[4]); // Sum(6) + res[7] = fma52lo(res[7], a[3], a[4]); // Sum(7) + res[8] = fma52hi(res[8], a[3], a[4]); // Sum(7) + res[5] = fma52lo(res[5], a[0], a[5]); // Sum(5) + res[6] = fma52hi(res[6], a[0], a[5]); // Sum(5) + res[6] = fma52lo(res[6], a[1], a[5]); // Sum(6) + res[7] = fma52hi(res[7], a[1], a[5]); // Sum(6) + res[7] = fma52lo(res[7], a[2], a[5]); // Sum(7) + res[8] = fma52hi(res[8], a[2], a[5]); // Sum(7) + res[8] = fma52lo(res[8], a[3], a[5]); // Sum(8) + res[9] = fma52hi(res[9], a[3], a[5]); // Sum(8) + res[9] = fma52lo(res[9], a[4], a[5]); // Sum(9) + res[10] = fma52hi(res[10], a[4], a[5]); // Sum(9) + res[6] = fma52lo(res[6], a[0], a[6]); // Sum(6) + res[7] = fma52hi(res[7], a[0], a[6]); // Sum(6) + res[7] = fma52lo(res[7], a[1], a[6]); // Sum(7) + res[8] = fma52hi(res[8], a[1], a[6]); // Sum(7) + res[8] = fma52lo(res[8], a[2], a[6]); // Sum(8) + res[9] = fma52hi(res[9], a[2], a[6]); // Sum(8) + res[9] = fma52lo(res[9], a[3], a[6]); // Sum(9) + res[10] = fma52hi(res[10], a[3], a[6]); // Sum(9) + res[10] = fma52lo(res[10], a[4], a[6]); // Sum(10) + res[11] = fma52hi(res[11], a[4], a[6]); // Sum(10) + res[11] = fma52lo(res[11], a[5], a[6]); // Sum(11) + res[12] = fma52hi(res[12], a[5], a[6]); // Sum(11) + res[7] = fma52lo(res[7], a[0], a[7]); // Sum(7) + res[8] = fma52hi(res[8], a[0], a[7]); // Sum(7) + res[8] = fma52lo(res[8], a[1], a[7]); // Sum(8) + res[9] = fma52hi(res[9], a[1], a[7]); // Sum(8) + res[9] = fma52lo(res[9], a[2], a[7]); // Sum(9) + res[10] = fma52hi(res[10], a[2], a[7]); // Sum(9) + res[10] = fma52lo(res[10], a[3], a[7]); // Sum(10) + res[11] = fma52hi(res[11], a[3], a[7]); // Sum(10) + res[11] = fma52lo(res[11], a[4], a[7]); // Sum(11) + res[12] = fma52hi(res[12], a[4], a[7]); // Sum(11) + res[8] = fma52lo(res[8], a[0], a[8]); // Sum(8) + res[9] = fma52hi(res[9], a[0], a[8]); // Sum(8) + res[9] = fma52lo(res[9], a[1], a[8]); // Sum(9) + res[10] = fma52hi(res[10], a[1], a[8]); // Sum(9) + res[10] = fma52lo(res[10], a[2], a[8]); // Sum(10) + res[11] = fma52hi(res[11], a[2], a[8]); // Sum(10) + res[11] = fma52lo(res[11], a[3], a[8]); // Sum(11) + res[12] = fma52hi(res[12], a[3], a[8]); // Sum(11) + res[9] = fma52lo(res[9], a[0], a[9]); // Sum(9) + res[10] = fma52hi(res[10], a[0], a[9]); // Sum(9) + res[10] = fma52lo(res[10], a[1], a[9]); // Sum(10) + res[11] = fma52hi(res[11], a[1], a[9]); // Sum(10) + res[11] = fma52lo(res[11], a[2], a[9]); // Sum(11) + res[12] = fma52hi(res[12], a[2], a[9]); // Sum(11) + res[10] = fma52lo(res[10], a[0], a[10]); // Sum(10) + res[11] = fma52hi(res[11], a[0], a[10]); // Sum(10) + res[11] = fma52lo(res[11], a[1], a[10]); // Sum(11) + res[12] = fma52hi(res[12], a[1], a[10]); // Sum(11) + res[11] = fma52lo(res[11], a[0], a[11]); // Sum(11) + res[12] = fma52hi(res[12], a[0], a[11]); // Sum(11) + res[0] = add64(res[0], res[0]); // Double(0) + res[1] = add64(res[1], res[1]); // Double(1) + res[2] = add64(res[2], res[2]); // Double(2) + res[3] = add64(res[3], res[3]); // Double(3) + res[4] = add64(res[4], res[4]); // Double(4) + res[5] = add64(res[5], res[5]); // Double(5) + res[6] = add64(res[6], res[6]); // Double(6) + res[7] = add64(res[7], res[7]); // Double(7) + res[8] = add64(res[8], res[8]); // Double(8) + res[9] = add64(res[9], res[9]); // Double(9) + res[10] = add64(res[10], res[10]); // Double(10) + res[11] = add64(res[11], res[11]); // Double(11) + res[0] = fma52lo(res[0], a[0], a[0]); // Add sqr(0) + res[1] = fma52hi(res[1], a[0], a[0]); // Add sqr(0) + res[2] = fma52lo(res[2], a[1], a[1]); // Add sqr(2) + res[3] = fma52hi(res[3], a[1], a[1]); // Add sqr(2) + res[4] = fma52lo(res[4], a[2], a[2]); // Add sqr(4) + res[5] = fma52hi(res[5], a[2], a[2]); // Add sqr(4) + res[6] = fma52lo(res[6], a[3], a[3]); // Add sqr(6) + res[7] = fma52hi(res[7], a[3], a[3]); // Add sqr(6) + res[8] = fma52lo(res[8], a[4], a[4]); // Add sqr(8) + res[9] = fma52hi(res[9], a[4], a[4]); // Add sqr(8) + res[10] = fma52lo(res[10], a[5], a[5]); // Add sqr(10) + res[11] = fma52hi(res[11], a[5], a[5]); // Add sqr(10) + res[12] = fma52lo(res[12], a[5], a[7]); // Sum(12) + res[13] = fma52hi(res[13], a[5], a[7]); // Sum(12) + res[13] = fma52lo(res[13], a[6], a[7]); // Sum(13) + res[14] = fma52hi(res[14], a[6], a[7]); // Sum(13) + res[12] = fma52lo(res[12], a[4], a[8]); // Sum(12) + res[13] = fma52hi(res[13], a[4], a[8]); // Sum(12) + res[13] = fma52lo(res[13], a[5], a[8]); // Sum(13) + res[14] = fma52hi(res[14], a[5], a[8]); // Sum(13) + res[14] = fma52lo(res[14], a[6], a[8]); // Sum(14) + res[15] = fma52hi(res[15], a[6], a[8]); // Sum(14) + res[15] = fma52lo(res[15], a[7], a[8]); // Sum(15) + res[16] = fma52hi(res[16], a[7], a[8]); // Sum(15) + res[12] = fma52lo(res[12], a[3], a[9]); // Sum(12) + res[13] = fma52hi(res[13], a[3], a[9]); // Sum(12) + res[13] = fma52lo(res[13], a[4], a[9]); // Sum(13) + res[14] = fma52hi(res[14], a[4], a[9]); // Sum(13) + res[14] = fma52lo(res[14], a[5], a[9]); // Sum(14) + res[15] = fma52hi(res[15], a[5], a[9]); // Sum(14) + res[15] = fma52lo(res[15], a[6], a[9]); // Sum(15) + res[16] = fma52hi(res[16], a[6], a[9]); // Sum(15) + res[16] = fma52lo(res[16], a[7], a[9]); // Sum(16) + res[17] = fma52hi(res[17], a[7], a[9]); // Sum(16) + res[17] = fma52lo(res[17], a[8], a[9]); // Sum(17) + res[18] = fma52hi(res[18], a[8], a[9]); // Sum(17) + res[12] = fma52lo(res[12], a[2], a[10]); // Sum(12) + res[13] = fma52hi(res[13], a[2], a[10]); // Sum(12) + res[13] = fma52lo(res[13], a[3], a[10]); // Sum(13) + res[14] = fma52hi(res[14], a[3], a[10]); // Sum(13) + res[14] = fma52lo(res[14], a[4], a[10]); // Sum(14) + res[15] = fma52hi(res[15], a[4], a[10]); // Sum(14) + res[15] = fma52lo(res[15], a[5], a[10]); // Sum(15) + res[16] = fma52hi(res[16], a[5], a[10]); // Sum(15) + res[16] = fma52lo(res[16], a[6], a[10]); // Sum(16) + res[17] = fma52hi(res[17], a[6], a[10]); // Sum(16) + res[17] = fma52lo(res[17], a[7], a[10]); // Sum(17) + res[18] = fma52hi(res[18], a[7], a[10]); // Sum(17) + res[18] = fma52lo(res[18], a[8], a[10]); // Sum(18) + res[19] = fma52hi(res[19], a[8], a[10]); // Sum(18) + res[19] = fma52lo(res[19], a[9], a[10]); // Sum(19) + res[20] = fma52hi(res[20], a[9], a[10]); // Sum(19) + res[12] = fma52lo(res[12], a[1], a[11]); // Sum(12) + res[13] = fma52hi(res[13], a[1], a[11]); // Sum(12) + res[13] = fma52lo(res[13], a[2], a[11]); // Sum(13) + res[14] = fma52hi(res[14], a[2], a[11]); // Sum(13) + res[14] = fma52lo(res[14], a[3], a[11]); // Sum(14) + res[15] = fma52hi(res[15], a[3], a[11]); // Sum(14) + res[15] = fma52lo(res[15], a[4], a[11]); // Sum(15) + res[16] = fma52hi(res[16], a[4], a[11]); // Sum(15) + res[16] = fma52lo(res[16], a[5], a[11]); // Sum(16) + res[17] = fma52hi(res[17], a[5], a[11]); // Sum(16) + res[17] = fma52lo(res[17], a[6], a[11]); // Sum(17) + res[18] = fma52hi(res[18], a[6], a[11]); // Sum(17) + res[18] = fma52lo(res[18], a[7], a[11]); // Sum(18) + res[19] = fma52hi(res[19], a[7], a[11]); // Sum(18) + res[19] = fma52lo(res[19], a[8], a[11]); // Sum(19) + res[20] = fma52hi(res[20], a[8], a[11]); // Sum(19) + res[20] = fma52lo(res[20], a[9], a[11]); // Sum(20) + res[21] = fma52hi(res[21], a[9], a[11]); // Sum(20) + res[21] = fma52lo(res[21], a[10], a[11]); // Sum(21) + res[22] = fma52hi(res[22], a[10], a[11]); // Sum(21) + res[12] = fma52lo(res[12], a[0], a[12]); // Sum(12) + res[13] = fma52hi(res[13], a[0], a[12]); // Sum(12) + res[13] = fma52lo(res[13], a[1], a[12]); // Sum(13) + res[14] = fma52hi(res[14], a[1], a[12]); // Sum(13) + res[14] = fma52lo(res[14], a[2], a[12]); // Sum(14) + res[15] = fma52hi(res[15], a[2], a[12]); // Sum(14) + res[15] = fma52lo(res[15], a[3], a[12]); // Sum(15) + res[16] = fma52hi(res[16], a[3], a[12]); // Sum(15) + res[16] = fma52lo(res[16], a[4], a[12]); // Sum(16) + res[17] = fma52hi(res[17], a[4], a[12]); // Sum(16) + res[17] = fma52lo(res[17], a[5], a[12]); // Sum(17) + res[18] = fma52hi(res[18], a[5], a[12]); // Sum(17) + res[18] = fma52lo(res[18], a[6], a[12]); // Sum(18) + res[19] = fma52hi(res[19], a[6], a[12]); // Sum(18) + res[19] = fma52lo(res[19], a[7], a[12]); // Sum(19) + res[20] = fma52hi(res[20], a[7], a[12]); // Sum(19) + res[20] = fma52lo(res[20], a[8], a[12]); // Sum(20) + res[21] = fma52hi(res[21], a[8], a[12]); // Sum(20) + res[21] = fma52lo(res[21], a[9], a[12]); // Sum(21) + res[22] = fma52hi(res[22], a[9], a[12]); // Sum(21) + res[22] = fma52lo(res[22], a[10], a[12]); // Sum(22) + res[23] = fma52hi(res[23], a[10], a[12]); // Sum(22) + res[23] = fma52lo(res[23], a[11], a[12]); // Sum(23) + res[24] = fma52hi(res[24], a[11], a[12]); // Sum(23) + res[13] = fma52lo(res[13], a[0], a[13]); // Sum(13) + res[14] = fma52hi(res[14], a[0], a[13]); // Sum(13) + res[14] = fma52lo(res[14], a[1], a[13]); // Sum(14) + res[15] = fma52hi(res[15], a[1], a[13]); // Sum(14) + res[15] = fma52lo(res[15], a[2], a[13]); // Sum(15) + res[16] = fma52hi(res[16], a[2], a[13]); // Sum(15) + res[16] = fma52lo(res[16], a[3], a[13]); // Sum(16) + res[17] = fma52hi(res[17], a[3], a[13]); // Sum(16) + res[17] = fma52lo(res[17], a[4], a[13]); // Sum(17) + res[18] = fma52hi(res[18], a[4], a[13]); // Sum(17) + res[18] = fma52lo(res[18], a[5], a[13]); // Sum(18) + res[19] = fma52hi(res[19], a[5], a[13]); // Sum(18) + res[19] = fma52lo(res[19], a[6], a[13]); // Sum(19) + res[20] = fma52hi(res[20], a[6], a[13]); // Sum(19) + res[20] = fma52lo(res[20], a[7], a[13]); // Sum(20) + res[21] = fma52hi(res[21], a[7], a[13]); // Sum(20) + res[21] = fma52lo(res[21], a[8], a[13]); // Sum(21) + res[22] = fma52hi(res[22], a[8], a[13]); // Sum(21) + res[22] = fma52lo(res[22], a[9], a[13]); // Sum(22) + res[23] = fma52hi(res[23], a[9], a[13]); // Sum(22) + res[23] = fma52lo(res[23], a[10], a[13]); // Sum(23) + res[24] = fma52hi(res[24], a[10], a[13]); // Sum(23) + res[14] = fma52lo(res[14], a[0], a[14]); // Sum(14) + res[15] = fma52hi(res[15], a[0], a[14]); // Sum(14) + res[15] = fma52lo(res[15], a[1], a[14]); // Sum(15) + res[16] = fma52hi(res[16], a[1], a[14]); // Sum(15) + res[16] = fma52lo(res[16], a[2], a[14]); // Sum(16) + res[17] = fma52hi(res[17], a[2], a[14]); // Sum(16) + res[17] = fma52lo(res[17], a[3], a[14]); // Sum(17) + res[18] = fma52hi(res[18], a[3], a[14]); // Sum(17) + res[18] = fma52lo(res[18], a[4], a[14]); // Sum(18) + res[19] = fma52hi(res[19], a[4], a[14]); // Sum(18) + res[19] = fma52lo(res[19], a[5], a[14]); // Sum(19) + res[20] = fma52hi(res[20], a[5], a[14]); // Sum(19) + res[20] = fma52lo(res[20], a[6], a[14]); // Sum(20) + res[21] = fma52hi(res[21], a[6], a[14]); // Sum(20) + res[21] = fma52lo(res[21], a[7], a[14]); // Sum(21) + res[22] = fma52hi(res[22], a[7], a[14]); // Sum(21) + res[22] = fma52lo(res[22], a[8], a[14]); // Sum(22) + res[23] = fma52hi(res[23], a[8], a[14]); // Sum(22) + res[23] = fma52lo(res[23], a[9], a[14]); // Sum(23) + res[24] = fma52hi(res[24], a[9], a[14]); // Sum(23) + res[15] = fma52lo(res[15], a[0], a[15]); // Sum(15) + res[16] = fma52hi(res[16], a[0], a[15]); // Sum(15) + res[16] = fma52lo(res[16], a[1], a[15]); // Sum(16) + res[17] = fma52hi(res[17], a[1], a[15]); // Sum(16) + res[17] = fma52lo(res[17], a[2], a[15]); // Sum(17) + res[18] = fma52hi(res[18], a[2], a[15]); // Sum(17) + res[18] = fma52lo(res[18], a[3], a[15]); // Sum(18) + res[19] = fma52hi(res[19], a[3], a[15]); // Sum(18) + res[19] = fma52lo(res[19], a[4], a[15]); // Sum(19) + res[20] = fma52hi(res[20], a[4], a[15]); // Sum(19) + res[20] = fma52lo(res[20], a[5], a[15]); // Sum(20) + res[21] = fma52hi(res[21], a[5], a[15]); // Sum(20) + res[21] = fma52lo(res[21], a[6], a[15]); // Sum(21) + res[22] = fma52hi(res[22], a[6], a[15]); // Sum(21) + res[22] = fma52lo(res[22], a[7], a[15]); // Sum(22) + res[23] = fma52hi(res[23], a[7], a[15]); // Sum(22) + res[23] = fma52lo(res[23], a[8], a[15]); // Sum(23) + res[24] = fma52hi(res[24], a[8], a[15]); // Sum(23) + res[16] = fma52lo(res[16], a[0], a[16]); // Sum(16) + res[17] = fma52hi(res[17], a[0], a[16]); // Sum(16) + res[17] = fma52lo(res[17], a[1], a[16]); // Sum(17) + res[18] = fma52hi(res[18], a[1], a[16]); // Sum(17) + res[18] = fma52lo(res[18], a[2], a[16]); // Sum(18) + res[19] = fma52hi(res[19], a[2], a[16]); // Sum(18) + res[19] = fma52lo(res[19], a[3], a[16]); // Sum(19) + res[20] = fma52hi(res[20], a[3], a[16]); // Sum(19) + res[20] = fma52lo(res[20], a[4], a[16]); // Sum(20) + res[21] = fma52hi(res[21], a[4], a[16]); // Sum(20) + res[21] = fma52lo(res[21], a[5], a[16]); // Sum(21) + res[22] = fma52hi(res[22], a[5], a[16]); // Sum(21) + res[22] = fma52lo(res[22], a[6], a[16]); // Sum(22) + res[23] = fma52hi(res[23], a[6], a[16]); // Sum(22) + res[23] = fma52lo(res[23], a[7], a[16]); // Sum(23) + res[24] = fma52hi(res[24], a[7], a[16]); // Sum(23) + res[17] = fma52lo(res[17], a[0], a[17]); // Sum(17) + res[18] = fma52hi(res[18], a[0], a[17]); // Sum(17) + res[18] = fma52lo(res[18], a[1], a[17]); // Sum(18) + res[19] = fma52hi(res[19], a[1], a[17]); // Sum(18) + res[19] = fma52lo(res[19], a[2], a[17]); // Sum(19) + res[20] = fma52hi(res[20], a[2], a[17]); // Sum(19) + res[20] = fma52lo(res[20], a[3], a[17]); // Sum(20) + res[21] = fma52hi(res[21], a[3], a[17]); // Sum(20) + res[21] = fma52lo(res[21], a[4], a[17]); // Sum(21) + res[22] = fma52hi(res[22], a[4], a[17]); // Sum(21) + res[22] = fma52lo(res[22], a[5], a[17]); // Sum(22) + res[23] = fma52hi(res[23], a[5], a[17]); // Sum(22) + res[23] = fma52lo(res[23], a[6], a[17]); // Sum(23) + res[24] = fma52hi(res[24], a[6], a[17]); // Sum(23) + res[18] = fma52lo(res[18], a[0], a[18]); // Sum(18) + res[19] = fma52hi(res[19], a[0], a[18]); // Sum(18) + res[19] = fma52lo(res[19], a[1], a[18]); // Sum(19) + res[20] = fma52hi(res[20], a[1], a[18]); // Sum(19) + res[20] = fma52lo(res[20], a[2], a[18]); // Sum(20) + res[21] = fma52hi(res[21], a[2], a[18]); // Sum(20) + res[21] = fma52lo(res[21], a[3], a[18]); // Sum(21) + res[22] = fma52hi(res[22], a[3], a[18]); // Sum(21) + res[22] = fma52lo(res[22], a[4], a[18]); // Sum(22) + res[23] = fma52hi(res[23], a[4], a[18]); // Sum(22) + res[23] = fma52lo(res[23], a[5], a[18]); // Sum(23) + res[24] = fma52hi(res[24], a[5], a[18]); // Sum(23) + res[19] = fma52lo(res[19], a[0], a[19]); // Sum(19) + res[20] = fma52hi(res[20], a[0], a[19]); // Sum(19) + res[20] = fma52lo(res[20], a[1], a[19]); // Sum(20) + res[21] = fma52hi(res[21], a[1], a[19]); // Sum(20) + res[21] = fma52lo(res[21], a[2], a[19]); // Sum(21) + res[22] = fma52hi(res[22], a[2], a[19]); // Sum(21) + res[22] = fma52lo(res[22], a[3], a[19]); // Sum(22) + res[23] = fma52hi(res[23], a[3], a[19]); // Sum(22) + res[23] = fma52lo(res[23], a[4], a[19]); // Sum(23) + res[24] = fma52hi(res[24], a[4], a[19]); // Sum(23) + res[20] = fma52lo(res[20], a[0], a[20]); // Sum(20) + res[21] = fma52hi(res[21], a[0], a[20]); // Sum(20) + res[21] = fma52lo(res[21], a[1], a[20]); // Sum(21) + res[22] = fma52hi(res[22], a[1], a[20]); // Sum(21) + res[22] = fma52lo(res[22], a[2], a[20]); // Sum(22) + res[23] = fma52hi(res[23], a[2], a[20]); // Sum(22) + res[23] = fma52lo(res[23], a[3], a[20]); // Sum(23) + res[24] = fma52hi(res[24], a[3], a[20]); // Sum(23) + res[21] = fma52lo(res[21], a[0], a[21]); // Sum(21) + res[22] = fma52hi(res[22], a[0], a[21]); // Sum(21) + res[22] = fma52lo(res[22], a[1], a[21]); // Sum(22) + res[23] = fma52hi(res[23], a[1], a[21]); // Sum(22) + res[23] = fma52lo(res[23], a[2], a[21]); // Sum(23) + res[24] = fma52hi(res[24], a[2], a[21]); // Sum(23) + res[22] = fma52lo(res[22], a[0], a[22]); // Sum(22) + res[23] = fma52hi(res[23], a[0], a[22]); // Sum(22) + res[23] = fma52lo(res[23], a[1], a[22]); // Sum(23) + res[24] = fma52hi(res[24], a[1], a[22]); // Sum(23) + res[23] = fma52lo(res[23], a[0], a[23]); // Sum(23) + res[24] = fma52hi(res[24], a[0], a[23]); // Sum(23) + res[12] = add64(res[12], res[12]); // Double(12) + res[13] = add64(res[13], res[13]); // Double(13) + res[14] = add64(res[14], res[14]); // Double(14) + res[15] = add64(res[15], res[15]); // Double(15) + res[16] = add64(res[16], res[16]); // Double(16) + res[17] = add64(res[17], res[17]); // Double(17) + res[18] = add64(res[18], res[18]); // Double(18) + res[19] = add64(res[19], res[19]); // Double(19) + res[20] = add64(res[20], res[20]); // Double(20) + res[21] = add64(res[21], res[21]); // Double(21) + res[22] = add64(res[22], res[22]); // Double(22) + res[23] = add64(res[23], res[23]); // Double(23) + res[12] = fma52lo(res[12], a[6], a[6]); // Add sqr(12) + res[13] = fma52hi(res[13], a[6], a[6]); // Add sqr(12) + res[14] = fma52lo(res[14], a[7], a[7]); // Add sqr(14) + res[15] = fma52hi(res[15], a[7], a[7]); // Add sqr(14) + res[16] = fma52lo(res[16], a[8], a[8]); // Add sqr(16) + res[17] = fma52hi(res[17], a[8], a[8]); // Add sqr(16) + res[18] = fma52lo(res[18], a[9], a[9]); // Add sqr(18) + res[19] = fma52hi(res[19], a[9], a[9]); // Add sqr(18) + res[20] = fma52lo(res[20], a[10], a[10]); // Add sqr(20) + res[21] = fma52hi(res[21], a[10], a[10]); // Add sqr(20) + res[22] = fma52lo(res[22], a[11], a[11]); // Add sqr(22) + res[23] = fma52hi(res[23], a[11], a[11]); // Add sqr(22) + res[24] = fma52lo(res[24], a[11], a[13]); // Sum(24) + res[25] = fma52hi(res[25], a[11], a[13]); // Sum(24) + res[25] = fma52lo(res[25], a[12], a[13]); // Sum(25) + res[26] = fma52hi(res[26], a[12], a[13]); // Sum(25) + res[24] = fma52lo(res[24], a[10], a[14]); // Sum(24) + res[25] = fma52hi(res[25], a[10], a[14]); // Sum(24) + res[25] = fma52lo(res[25], a[11], a[14]); // Sum(25) + res[26] = fma52hi(res[26], a[11], a[14]); // Sum(25) + res[26] = fma52lo(res[26], a[12], a[14]); // Sum(26) + res[27] = fma52hi(res[27], a[12], a[14]); // Sum(26) + res[27] = fma52lo(res[27], a[13], a[14]); // Sum(27) + res[28] = fma52hi(res[28], a[13], a[14]); // Sum(27) + res[24] = fma52lo(res[24], a[9], a[15]); // Sum(24) + res[25] = fma52hi(res[25], a[9], a[15]); // Sum(24) + res[25] = fma52lo(res[25], a[10], a[15]); // Sum(25) + res[26] = fma52hi(res[26], a[10], a[15]); // Sum(25) + res[26] = fma52lo(res[26], a[11], a[15]); // Sum(26) + res[27] = fma52hi(res[27], a[11], a[15]); // Sum(26) + res[27] = fma52lo(res[27], a[12], a[15]); // Sum(27) + res[28] = fma52hi(res[28], a[12], a[15]); // Sum(27) + res[28] = fma52lo(res[28], a[13], a[15]); // Sum(28) + res[29] = fma52hi(res[29], a[13], a[15]); // Sum(28) + res[29] = fma52lo(res[29], a[14], a[15]); // Sum(29) + res[30] = fma52hi(res[30], a[14], a[15]); // Sum(29) + res[24] = fma52lo(res[24], a[8], a[16]); // Sum(24) + res[25] = fma52hi(res[25], a[8], a[16]); // Sum(24) + res[25] = fma52lo(res[25], a[9], a[16]); // Sum(25) + res[26] = fma52hi(res[26], a[9], a[16]); // Sum(25) + res[26] = fma52lo(res[26], a[10], a[16]); // Sum(26) + res[27] = fma52hi(res[27], a[10], a[16]); // Sum(26) + res[27] = fma52lo(res[27], a[11], a[16]); // Sum(27) + res[28] = fma52hi(res[28], a[11], a[16]); // Sum(27) + res[28] = fma52lo(res[28], a[12], a[16]); // Sum(28) + res[29] = fma52hi(res[29], a[12], a[16]); // Sum(28) + res[29] = fma52lo(res[29], a[13], a[16]); // Sum(29) + res[30] = fma52hi(res[30], a[13], a[16]); // Sum(29) + res[30] = fma52lo(res[30], a[14], a[16]); // Sum(30) + res[31] = fma52hi(res[31], a[14], a[16]); // Sum(30) + res[31] = fma52lo(res[31], a[15], a[16]); // Sum(31) + res[32] = fma52hi(res[32], a[15], a[16]); // Sum(31) + res[24] = fma52lo(res[24], a[7], a[17]); // Sum(24) + res[25] = fma52hi(res[25], a[7], a[17]); // Sum(24) + res[25] = fma52lo(res[25], a[8], a[17]); // Sum(25) + res[26] = fma52hi(res[26], a[8], a[17]); // Sum(25) + res[26] = fma52lo(res[26], a[9], a[17]); // Sum(26) + res[27] = fma52hi(res[27], a[9], a[17]); // Sum(26) + res[27] = fma52lo(res[27], a[10], a[17]); // Sum(27) + res[28] = fma52hi(res[28], a[10], a[17]); // Sum(27) + res[28] = fma52lo(res[28], a[11], a[17]); // Sum(28) + res[29] = fma52hi(res[29], a[11], a[17]); // Sum(28) + res[29] = fma52lo(res[29], a[12], a[17]); // Sum(29) + res[30] = fma52hi(res[30], a[12], a[17]); // Sum(29) + res[30] = fma52lo(res[30], a[13], a[17]); // Sum(30) + res[31] = fma52hi(res[31], a[13], a[17]); // Sum(30) + res[31] = fma52lo(res[31], a[14], a[17]); // Sum(31) + res[32] = fma52hi(res[32], a[14], a[17]); // Sum(31) + res[32] = fma52lo(res[32], a[15], a[17]); // Sum(32) + res[33] = fma52hi(res[33], a[15], a[17]); // Sum(32) + res[33] = fma52lo(res[33], a[16], a[17]); // Sum(33) + res[34] = fma52hi(res[34], a[16], a[17]); // Sum(33) + res[24] = fma52lo(res[24], a[6], a[18]); // Sum(24) + res[25] = fma52hi(res[25], a[6], a[18]); // Sum(24) + res[25] = fma52lo(res[25], a[7], a[18]); // Sum(25) + res[26] = fma52hi(res[26], a[7], a[18]); // Sum(25) + res[26] = fma52lo(res[26], a[8], a[18]); // Sum(26) + res[27] = fma52hi(res[27], a[8], a[18]); // Sum(26) + res[27] = fma52lo(res[27], a[9], a[18]); // Sum(27) + res[28] = fma52hi(res[28], a[9], a[18]); // Sum(27) + res[28] = fma52lo(res[28], a[10], a[18]); // Sum(28) + res[29] = fma52hi(res[29], a[10], a[18]); // Sum(28) + res[29] = fma52lo(res[29], a[11], a[18]); // Sum(29) + res[30] = fma52hi(res[30], a[11], a[18]); // Sum(29) + res[30] = fma52lo(res[30], a[12], a[18]); // Sum(30) + res[31] = fma52hi(res[31], a[12], a[18]); // Sum(30) + res[31] = fma52lo(res[31], a[13], a[18]); // Sum(31) + res[32] = fma52hi(res[32], a[13], a[18]); // Sum(31) + res[32] = fma52lo(res[32], a[14], a[18]); // Sum(32) + res[33] = fma52hi(res[33], a[14], a[18]); // Sum(32) + res[33] = fma52lo(res[33], a[15], a[18]); // Sum(33) + res[34] = fma52hi(res[34], a[15], a[18]); // Sum(33) + res[34] = fma52lo(res[34], a[16], a[18]); // Sum(34) + res[35] = fma52hi(res[35], a[16], a[18]); // Sum(34) + res[35] = fma52lo(res[35], a[17], a[18]); // Sum(35) + res[36] = fma52hi(res[36], a[17], a[18]); // Sum(35) + res[24] = fma52lo(res[24], a[5], a[19]); // Sum(24) + res[25] = fma52hi(res[25], a[5], a[19]); // Sum(24) + res[25] = fma52lo(res[25], a[6], a[19]); // Sum(25) + res[26] = fma52hi(res[26], a[6], a[19]); // Sum(25) + res[26] = fma52lo(res[26], a[7], a[19]); // Sum(26) + res[27] = fma52hi(res[27], a[7], a[19]); // Sum(26) + res[27] = fma52lo(res[27], a[8], a[19]); // Sum(27) + res[28] = fma52hi(res[28], a[8], a[19]); // Sum(27) + res[28] = fma52lo(res[28], a[9], a[19]); // Sum(28) + res[29] = fma52hi(res[29], a[9], a[19]); // Sum(28) + res[29] = fma52lo(res[29], a[10], a[19]); // Sum(29) + res[30] = fma52hi(res[30], a[10], a[19]); // Sum(29) + res[30] = fma52lo(res[30], a[11], a[19]); // Sum(30) + res[31] = fma52hi(res[31], a[11], a[19]); // Sum(30) + res[31] = fma52lo(res[31], a[12], a[19]); // Sum(31) + res[32] = fma52hi(res[32], a[12], a[19]); // Sum(31) + res[32] = fma52lo(res[32], a[13], a[19]); // Sum(32) + res[33] = fma52hi(res[33], a[13], a[19]); // Sum(32) + res[33] = fma52lo(res[33], a[14], a[19]); // Sum(33) + res[34] = fma52hi(res[34], a[14], a[19]); // Sum(33) + res[34] = fma52lo(res[34], a[15], a[19]); // Sum(34) + res[35] = fma52hi(res[35], a[15], a[19]); // Sum(34) + res[35] = fma52lo(res[35], a[16], a[19]); // Sum(35) + res[36] = fma52hi(res[36], a[16], a[19]); // Sum(35) + res[24] = fma52lo(res[24], a[4], a[20]); // Sum(24) + res[25] = fma52hi(res[25], a[4], a[20]); // Sum(24) + res[25] = fma52lo(res[25], a[5], a[20]); // Sum(25) + res[26] = fma52hi(res[26], a[5], a[20]); // Sum(25) + res[26] = fma52lo(res[26], a[6], a[20]); // Sum(26) + res[27] = fma52hi(res[27], a[6], a[20]); // Sum(26) + res[27] = fma52lo(res[27], a[7], a[20]); // Sum(27) + res[28] = fma52hi(res[28], a[7], a[20]); // Sum(27) + res[28] = fma52lo(res[28], a[8], a[20]); // Sum(28) + res[29] = fma52hi(res[29], a[8], a[20]); // Sum(28) + res[29] = fma52lo(res[29], a[9], a[20]); // Sum(29) + res[30] = fma52hi(res[30], a[9], a[20]); // Sum(29) + res[30] = fma52lo(res[30], a[10], a[20]); // Sum(30) + res[31] = fma52hi(res[31], a[10], a[20]); // Sum(30) + res[31] = fma52lo(res[31], a[11], a[20]); // Sum(31) + res[32] = fma52hi(res[32], a[11], a[20]); // Sum(31) + res[32] = fma52lo(res[32], a[12], a[20]); // Sum(32) + res[33] = fma52hi(res[33], a[12], a[20]); // Sum(32) + res[33] = fma52lo(res[33], a[13], a[20]); // Sum(33) + res[34] = fma52hi(res[34], a[13], a[20]); // Sum(33) + res[34] = fma52lo(res[34], a[14], a[20]); // Sum(34) + res[35] = fma52hi(res[35], a[14], a[20]); // Sum(34) + res[35] = fma52lo(res[35], a[15], a[20]); // Sum(35) + res[36] = fma52hi(res[36], a[15], a[20]); // Sum(35) + res[24] = fma52lo(res[24], a[3], a[21]); // Sum(24) + res[25] = fma52hi(res[25], a[3], a[21]); // Sum(24) + res[25] = fma52lo(res[25], a[4], a[21]); // Sum(25) + res[26] = fma52hi(res[26], a[4], a[21]); // Sum(25) + res[26] = fma52lo(res[26], a[5], a[21]); // Sum(26) + res[27] = fma52hi(res[27], a[5], a[21]); // Sum(26) + res[27] = fma52lo(res[27], a[6], a[21]); // Sum(27) + res[28] = fma52hi(res[28], a[6], a[21]); // Sum(27) + res[28] = fma52lo(res[28], a[7], a[21]); // Sum(28) + res[29] = fma52hi(res[29], a[7], a[21]); // Sum(28) + res[29] = fma52lo(res[29], a[8], a[21]); // Sum(29) + res[30] = fma52hi(res[30], a[8], a[21]); // Sum(29) + res[30] = fma52lo(res[30], a[9], a[21]); // Sum(30) + res[31] = fma52hi(res[31], a[9], a[21]); // Sum(30) + res[31] = fma52lo(res[31], a[10], a[21]); // Sum(31) + res[32] = fma52hi(res[32], a[10], a[21]); // Sum(31) + res[32] = fma52lo(res[32], a[11], a[21]); // Sum(32) + res[33] = fma52hi(res[33], a[11], a[21]); // Sum(32) + res[33] = fma52lo(res[33], a[12], a[21]); // Sum(33) + res[34] = fma52hi(res[34], a[12], a[21]); // Sum(33) + res[34] = fma52lo(res[34], a[13], a[21]); // Sum(34) + res[35] = fma52hi(res[35], a[13], a[21]); // Sum(34) + res[35] = fma52lo(res[35], a[14], a[21]); // Sum(35) + res[36] = fma52hi(res[36], a[14], a[21]); // Sum(35) + res[24] = fma52lo(res[24], a[2], a[22]); // Sum(24) + res[25] = fma52hi(res[25], a[2], a[22]); // Sum(24) + res[25] = fma52lo(res[25], a[3], a[22]); // Sum(25) + res[26] = fma52hi(res[26], a[3], a[22]); // Sum(25) + res[26] = fma52lo(res[26], a[4], a[22]); // Sum(26) + res[27] = fma52hi(res[27], a[4], a[22]); // Sum(26) + res[27] = fma52lo(res[27], a[5], a[22]); // Sum(27) + res[28] = fma52hi(res[28], a[5], a[22]); // Sum(27) + res[28] = fma52lo(res[28], a[6], a[22]); // Sum(28) + res[29] = fma52hi(res[29], a[6], a[22]); // Sum(28) + res[29] = fma52lo(res[29], a[7], a[22]); // Sum(29) + res[30] = fma52hi(res[30], a[7], a[22]); // Sum(29) + res[30] = fma52lo(res[30], a[8], a[22]); // Sum(30) + res[31] = fma52hi(res[31], a[8], a[22]); // Sum(30) + res[31] = fma52lo(res[31], a[9], a[22]); // Sum(31) + res[32] = fma52hi(res[32], a[9], a[22]); // Sum(31) + res[32] = fma52lo(res[32], a[10], a[22]); // Sum(32) + res[33] = fma52hi(res[33], a[10], a[22]); // Sum(32) + res[33] = fma52lo(res[33], a[11], a[22]); // Sum(33) + res[34] = fma52hi(res[34], a[11], a[22]); // Sum(33) + res[34] = fma52lo(res[34], a[12], a[22]); // Sum(34) + res[35] = fma52hi(res[35], a[12], a[22]); // Sum(34) + res[35] = fma52lo(res[35], a[13], a[22]); // Sum(35) + res[36] = fma52hi(res[36], a[13], a[22]); // Sum(35) + res[24] = fma52lo(res[24], a[1], a[23]); // Sum(24) + res[25] = fma52hi(res[25], a[1], a[23]); // Sum(24) + res[25] = fma52lo(res[25], a[2], a[23]); // Sum(25) + res[26] = fma52hi(res[26], a[2], a[23]); // Sum(25) + res[26] = fma52lo(res[26], a[3], a[23]); // Sum(26) + res[27] = fma52hi(res[27], a[3], a[23]); // Sum(26) + res[27] = fma52lo(res[27], a[4], a[23]); // Sum(27) + res[28] = fma52hi(res[28], a[4], a[23]); // Sum(27) + res[28] = fma52lo(res[28], a[5], a[23]); // Sum(28) + res[29] = fma52hi(res[29], a[5], a[23]); // Sum(28) + res[29] = fma52lo(res[29], a[6], a[23]); // Sum(29) + res[30] = fma52hi(res[30], a[6], a[23]); // Sum(29) + res[30] = fma52lo(res[30], a[7], a[23]); // Sum(30) + res[31] = fma52hi(res[31], a[7], a[23]); // Sum(30) + res[31] = fma52lo(res[31], a[8], a[23]); // Sum(31) + res[32] = fma52hi(res[32], a[8], a[23]); // Sum(31) + res[32] = fma52lo(res[32], a[9], a[23]); // Sum(32) + res[33] = fma52hi(res[33], a[9], a[23]); // Sum(32) + res[33] = fma52lo(res[33], a[10], a[23]); // Sum(33) + res[34] = fma52hi(res[34], a[10], a[23]); // Sum(33) + res[34] = fma52lo(res[34], a[11], a[23]); // Sum(34) + res[35] = fma52hi(res[35], a[11], a[23]); // Sum(34) + res[35] = fma52lo(res[35], a[12], a[23]); // Sum(35) + res[36] = fma52hi(res[36], a[12], a[23]); // Sum(35) + res[24] = fma52lo(res[24], a[0], a[24]); // Sum(24) + res[25] = fma52hi(res[25], a[0], a[24]); // Sum(24) + res[25] = fma52lo(res[25], a[1], a[24]); // Sum(25) + res[26] = fma52hi(res[26], a[1], a[24]); // Sum(25) + res[26] = fma52lo(res[26], a[2], a[24]); // Sum(26) + res[27] = fma52hi(res[27], a[2], a[24]); // Sum(26) + res[27] = fma52lo(res[27], a[3], a[24]); // Sum(27) + res[28] = fma52hi(res[28], a[3], a[24]); // Sum(27) + res[28] = fma52lo(res[28], a[4], a[24]); // Sum(28) + res[29] = fma52hi(res[29], a[4], a[24]); // Sum(28) + res[29] = fma52lo(res[29], a[5], a[24]); // Sum(29) + res[30] = fma52hi(res[30], a[5], a[24]); // Sum(29) + res[30] = fma52lo(res[30], a[6], a[24]); // Sum(30) + res[31] = fma52hi(res[31], a[6], a[24]); // Sum(30) + res[31] = fma52lo(res[31], a[7], a[24]); // Sum(31) + res[32] = fma52hi(res[32], a[7], a[24]); // Sum(31) + res[32] = fma52lo(res[32], a[8], a[24]); // Sum(32) + res[33] = fma52hi(res[33], a[8], a[24]); // Sum(32) + res[33] = fma52lo(res[33], a[9], a[24]); // Sum(33) + res[34] = fma52hi(res[34], a[9], a[24]); // Sum(33) + res[34] = fma52lo(res[34], a[10], a[24]); // Sum(34) + res[35] = fma52hi(res[35], a[10], a[24]); // Sum(34) + res[35] = fma52lo(res[35], a[11], a[24]); // Sum(35) + res[36] = fma52hi(res[36], a[11], a[24]); // Sum(35) + res[25] = fma52lo(res[25], a[0], a[25]); // Sum(25) + res[26] = fma52hi(res[26], a[0], a[25]); // Sum(25) + res[26] = fma52lo(res[26], a[1], a[25]); // Sum(26) + res[27] = fma52hi(res[27], a[1], a[25]); // Sum(26) + res[27] = fma52lo(res[27], a[2], a[25]); // Sum(27) + res[28] = fma52hi(res[28], a[2], a[25]); // Sum(27) + res[28] = fma52lo(res[28], a[3], a[25]); // Sum(28) + res[29] = fma52hi(res[29], a[3], a[25]); // Sum(28) + res[29] = fma52lo(res[29], a[4], a[25]); // Sum(29) + res[30] = fma52hi(res[30], a[4], a[25]); // Sum(29) + res[30] = fma52lo(res[30], a[5], a[25]); // Sum(30) + res[31] = fma52hi(res[31], a[5], a[25]); // Sum(30) + res[31] = fma52lo(res[31], a[6], a[25]); // Sum(31) + res[32] = fma52hi(res[32], a[6], a[25]); // Sum(31) + res[32] = fma52lo(res[32], a[7], a[25]); // Sum(32) + res[33] = fma52hi(res[33], a[7], a[25]); // Sum(32) + res[33] = fma52lo(res[33], a[8], a[25]); // Sum(33) + res[34] = fma52hi(res[34], a[8], a[25]); // Sum(33) + res[34] = fma52lo(res[34], a[9], a[25]); // Sum(34) + res[35] = fma52hi(res[35], a[9], a[25]); // Sum(34) + res[35] = fma52lo(res[35], a[10], a[25]); // Sum(35) + res[36] = fma52hi(res[36], a[10], a[25]); // Sum(35) + res[26] = fma52lo(res[26], a[0], a[26]); // Sum(26) + res[27] = fma52hi(res[27], a[0], a[26]); // Sum(26) + res[27] = fma52lo(res[27], a[1], a[26]); // Sum(27) + res[28] = fma52hi(res[28], a[1], a[26]); // Sum(27) + res[28] = fma52lo(res[28], a[2], a[26]); // Sum(28) + res[29] = fma52hi(res[29], a[2], a[26]); // Sum(28) + res[29] = fma52lo(res[29], a[3], a[26]); // Sum(29) + res[30] = fma52hi(res[30], a[3], a[26]); // Sum(29) + res[30] = fma52lo(res[30], a[4], a[26]); // Sum(30) + res[31] = fma52hi(res[31], a[4], a[26]); // Sum(30) + res[31] = fma52lo(res[31], a[5], a[26]); // Sum(31) + res[32] = fma52hi(res[32], a[5], a[26]); // Sum(31) + res[32] = fma52lo(res[32], a[6], a[26]); // Sum(32) + res[33] = fma52hi(res[33], a[6], a[26]); // Sum(32) + res[33] = fma52lo(res[33], a[7], a[26]); // Sum(33) + res[34] = fma52hi(res[34], a[7], a[26]); // Sum(33) + res[34] = fma52lo(res[34], a[8], a[26]); // Sum(34) + res[35] = fma52hi(res[35], a[8], a[26]); // Sum(34) + res[35] = fma52lo(res[35], a[9], a[26]); // Sum(35) + res[36] = fma52hi(res[36], a[9], a[26]); // Sum(35) + res[27] = fma52lo(res[27], a[0], a[27]); // Sum(27) + res[28] = fma52hi(res[28], a[0], a[27]); // Sum(27) + res[28] = fma52lo(res[28], a[1], a[27]); // Sum(28) + res[29] = fma52hi(res[29], a[1], a[27]); // Sum(28) + res[29] = fma52lo(res[29], a[2], a[27]); // Sum(29) + res[30] = fma52hi(res[30], a[2], a[27]); // Sum(29) + res[30] = fma52lo(res[30], a[3], a[27]); // Sum(30) + res[31] = fma52hi(res[31], a[3], a[27]); // Sum(30) + res[31] = fma52lo(res[31], a[4], a[27]); // Sum(31) + res[32] = fma52hi(res[32], a[4], a[27]); // Sum(31) + res[32] = fma52lo(res[32], a[5], a[27]); // Sum(32) + res[33] = fma52hi(res[33], a[5], a[27]); // Sum(32) + res[33] = fma52lo(res[33], a[6], a[27]); // Sum(33) + res[34] = fma52hi(res[34], a[6], a[27]); // Sum(33) + res[34] = fma52lo(res[34], a[7], a[27]); // Sum(34) + res[35] = fma52hi(res[35], a[7], a[27]); // Sum(34) + res[35] = fma52lo(res[35], a[8], a[27]); // Sum(35) + res[36] = fma52hi(res[36], a[8], a[27]); // Sum(35) + res[28] = fma52lo(res[28], a[0], a[28]); // Sum(28) + res[29] = fma52hi(res[29], a[0], a[28]); // Sum(28) + res[29] = fma52lo(res[29], a[1], a[28]); // Sum(29) + res[30] = fma52hi(res[30], a[1], a[28]); // Sum(29) + res[30] = fma52lo(res[30], a[2], a[28]); // Sum(30) + res[31] = fma52hi(res[31], a[2], a[28]); // Sum(30) + res[31] = fma52lo(res[31], a[3], a[28]); // Sum(31) + res[32] = fma52hi(res[32], a[3], a[28]); // Sum(31) + res[32] = fma52lo(res[32], a[4], a[28]); // Sum(32) + res[33] = fma52hi(res[33], a[4], a[28]); // Sum(32) + res[33] = fma52lo(res[33], a[5], a[28]); // Sum(33) + res[34] = fma52hi(res[34], a[5], a[28]); // Sum(33) + res[34] = fma52lo(res[34], a[6], a[28]); // Sum(34) + res[35] = fma52hi(res[35], a[6], a[28]); // Sum(34) + res[35] = fma52lo(res[35], a[7], a[28]); // Sum(35) + res[36] = fma52hi(res[36], a[7], a[28]); // Sum(35) + res[29] = fma52lo(res[29], a[0], a[29]); // Sum(29) + res[30] = fma52hi(res[30], a[0], a[29]); // Sum(29) + res[30] = fma52lo(res[30], a[1], a[29]); // Sum(30) + res[31] = fma52hi(res[31], a[1], a[29]); // Sum(30) + res[31] = fma52lo(res[31], a[2], a[29]); // Sum(31) + res[32] = fma52hi(res[32], a[2], a[29]); // Sum(31) + res[32] = fma52lo(res[32], a[3], a[29]); // Sum(32) + res[33] = fma52hi(res[33], a[3], a[29]); // Sum(32) + res[33] = fma52lo(res[33], a[4], a[29]); // Sum(33) + res[34] = fma52hi(res[34], a[4], a[29]); // Sum(33) + res[34] = fma52lo(res[34], a[5], a[29]); // Sum(34) + res[35] = fma52hi(res[35], a[5], a[29]); // Sum(34) + res[35] = fma52lo(res[35], a[6], a[29]); // Sum(35) + res[36] = fma52hi(res[36], a[6], a[29]); // Sum(35) + res[30] = fma52lo(res[30], a[0], a[30]); // Sum(30) + res[31] = fma52hi(res[31], a[0], a[30]); // Sum(30) + res[31] = fma52lo(res[31], a[1], a[30]); // Sum(31) + res[32] = fma52hi(res[32], a[1], a[30]); // Sum(31) + res[32] = fma52lo(res[32], a[2], a[30]); // Sum(32) + res[33] = fma52hi(res[33], a[2], a[30]); // Sum(32) + res[33] = fma52lo(res[33], a[3], a[30]); // Sum(33) + res[34] = fma52hi(res[34], a[3], a[30]); // Sum(33) + res[34] = fma52lo(res[34], a[4], a[30]); // Sum(34) + res[35] = fma52hi(res[35], a[4], a[30]); // Sum(34) + res[35] = fma52lo(res[35], a[5], a[30]); // Sum(35) + res[36] = fma52hi(res[36], a[5], a[30]); // Sum(35) + res[31] = fma52lo(res[31], a[0], a[31]); // Sum(31) + res[32] = fma52hi(res[32], a[0], a[31]); // Sum(31) + res[32] = fma52lo(res[32], a[1], a[31]); // Sum(32) + res[33] = fma52hi(res[33], a[1], a[31]); // Sum(32) + res[33] = fma52lo(res[33], a[2], a[31]); // Sum(33) + res[34] = fma52hi(res[34], a[2], a[31]); // Sum(33) + res[34] = fma52lo(res[34], a[3], a[31]); // Sum(34) + res[35] = fma52hi(res[35], a[3], a[31]); // Sum(34) + res[35] = fma52lo(res[35], a[4], a[31]); // Sum(35) + res[36] = fma52hi(res[36], a[4], a[31]); // Sum(35) + res[32] = fma52lo(res[32], a[0], a[32]); // Sum(32) + res[33] = fma52hi(res[33], a[0], a[32]); // Sum(32) + res[33] = fma52lo(res[33], a[1], a[32]); // Sum(33) + res[34] = fma52hi(res[34], a[1], a[32]); // Sum(33) + res[34] = fma52lo(res[34], a[2], a[32]); // Sum(34) + res[35] = fma52hi(res[35], a[2], a[32]); // Sum(34) + res[35] = fma52lo(res[35], a[3], a[32]); // Sum(35) + res[36] = fma52hi(res[36], a[3], a[32]); // Sum(35) + res[33] = fma52lo(res[33], a[0], a[33]); // Sum(33) + res[34] = fma52hi(res[34], a[0], a[33]); // Sum(33) + res[34] = fma52lo(res[34], a[1], a[33]); // Sum(34) + res[35] = fma52hi(res[35], a[1], a[33]); // Sum(34) + res[35] = fma52lo(res[35], a[2], a[33]); // Sum(35) + res[36] = fma52hi(res[36], a[2], a[33]); // Sum(35) + res[34] = fma52lo(res[34], a[0], a[34]); // Sum(34) + res[35] = fma52hi(res[35], a[0], a[34]); // Sum(34) + res[35] = fma52lo(res[35], a[1], a[34]); // Sum(35) + res[36] = fma52hi(res[36], a[1], a[34]); // Sum(35) + res[35] = fma52lo(res[35], a[0], a[35]); // Sum(35) + res[36] = fma52hi(res[36], a[0], a[35]); // Sum(35) + res[24] = add64(res[24], res[24]); // Double(24) + res[25] = add64(res[25], res[25]); // Double(25) + res[26] = add64(res[26], res[26]); // Double(26) + res[27] = add64(res[27], res[27]); // Double(27) + res[28] = add64(res[28], res[28]); // Double(28) + res[29] = add64(res[29], res[29]); // Double(29) + res[30] = add64(res[30], res[30]); // Double(30) + res[31] = add64(res[31], res[31]); // Double(31) + res[32] = add64(res[32], res[32]); // Double(32) + res[33] = add64(res[33], res[33]); // Double(33) + res[34] = add64(res[34], res[34]); // Double(34) + res[35] = add64(res[35], res[35]); // Double(35) + res[24] = fma52lo(res[24], a[12], a[12]); // Add sqr(24) + res[25] = fma52hi(res[25], a[12], a[12]); // Add sqr(24) + res[26] = fma52lo(res[26], a[13], a[13]); // Add sqr(26) + res[27] = fma52hi(res[27], a[13], a[13]); // Add sqr(26) + res[28] = fma52lo(res[28], a[14], a[14]); // Add sqr(28) + res[29] = fma52hi(res[29], a[14], a[14]); // Add sqr(28) + res[30] = fma52lo(res[30], a[15], a[15]); // Add sqr(30) + res[31] = fma52hi(res[31], a[15], a[15]); // Add sqr(30) + res[32] = fma52lo(res[32], a[16], a[16]); // Add sqr(32) + res[33] = fma52hi(res[33], a[16], a[16]); // Add sqr(32) + res[34] = fma52lo(res[34], a[17], a[17]); // Add sqr(34) + res[35] = fma52hi(res[35], a[17], a[17]); // Add sqr(34) + res[36] = fma52lo(res[36], a[17], a[19]); // Sum(36) + res[37] = fma52hi(res[37], a[17], a[19]); // Sum(36) + res[37] = fma52lo(res[37], a[18], a[19]); // Sum(37) + res[38] = fma52hi(res[38], a[18], a[19]); // Sum(37) + res[36] = fma52lo(res[36], a[16], a[20]); // Sum(36) + res[37] = fma52hi(res[37], a[16], a[20]); // Sum(36) + res[37] = fma52lo(res[37], a[17], a[20]); // Sum(37) + res[38] = fma52hi(res[38], a[17], a[20]); // Sum(37) + res[38] = fma52lo(res[38], a[18], a[20]); // Sum(38) + res[39] = fma52hi(res[39], a[18], a[20]); // Sum(38) + res[39] = fma52lo(res[39], a[19], a[20]); // Sum(39) + res[40] = fma52hi(res[40], a[19], a[20]); // Sum(39) + res[36] = fma52lo(res[36], a[15], a[21]); // Sum(36) + res[37] = fma52hi(res[37], a[15], a[21]); // Sum(36) + res[37] = fma52lo(res[37], a[16], a[21]); // Sum(37) + res[38] = fma52hi(res[38], a[16], a[21]); // Sum(37) + res[38] = fma52lo(res[38], a[17], a[21]); // Sum(38) + res[39] = fma52hi(res[39], a[17], a[21]); // Sum(38) + res[39] = fma52lo(res[39], a[18], a[21]); // Sum(39) + res[40] = fma52hi(res[40], a[18], a[21]); // Sum(39) + res[40] = fma52lo(res[40], a[19], a[21]); // Sum(40) + res[41] = fma52hi(res[41], a[19], a[21]); // Sum(40) + res[41] = fma52lo(res[41], a[20], a[21]); // Sum(41) + res[42] = fma52hi(res[42], a[20], a[21]); // Sum(41) + res[36] = fma52lo(res[36], a[14], a[22]); // Sum(36) + res[37] = fma52hi(res[37], a[14], a[22]); // Sum(36) + res[37] = fma52lo(res[37], a[15], a[22]); // Sum(37) + res[38] = fma52hi(res[38], a[15], a[22]); // Sum(37) + res[38] = fma52lo(res[38], a[16], a[22]); // Sum(38) + res[39] = fma52hi(res[39], a[16], a[22]); // Sum(38) + res[39] = fma52lo(res[39], a[17], a[22]); // Sum(39) + res[40] = fma52hi(res[40], a[17], a[22]); // Sum(39) + res[40] = fma52lo(res[40], a[18], a[22]); // Sum(40) + res[41] = fma52hi(res[41], a[18], a[22]); // Sum(40) + res[41] = fma52lo(res[41], a[19], a[22]); // Sum(41) + res[42] = fma52hi(res[42], a[19], a[22]); // Sum(41) + res[42] = fma52lo(res[42], a[20], a[22]); // Sum(42) + res[43] = fma52hi(res[43], a[20], a[22]); // Sum(42) + res[43] = fma52lo(res[43], a[21], a[22]); // Sum(43) + res[44] = fma52hi(res[44], a[21], a[22]); // Sum(43) + res[36] = fma52lo(res[36], a[13], a[23]); // Sum(36) + res[37] = fma52hi(res[37], a[13], a[23]); // Sum(36) + res[37] = fma52lo(res[37], a[14], a[23]); // Sum(37) + res[38] = fma52hi(res[38], a[14], a[23]); // Sum(37) + res[38] = fma52lo(res[38], a[15], a[23]); // Sum(38) + res[39] = fma52hi(res[39], a[15], a[23]); // Sum(38) + res[39] = fma52lo(res[39], a[16], a[23]); // Sum(39) + res[40] = fma52hi(res[40], a[16], a[23]); // Sum(39) + res[40] = fma52lo(res[40], a[17], a[23]); // Sum(40) + res[41] = fma52hi(res[41], a[17], a[23]); // Sum(40) + res[41] = fma52lo(res[41], a[18], a[23]); // Sum(41) + res[42] = fma52hi(res[42], a[18], a[23]); // Sum(41) + res[42] = fma52lo(res[42], a[19], a[23]); // Sum(42) + res[43] = fma52hi(res[43], a[19], a[23]); // Sum(42) + res[43] = fma52lo(res[43], a[20], a[23]); // Sum(43) + res[44] = fma52hi(res[44], a[20], a[23]); // Sum(43) + res[44] = fma52lo(res[44], a[21], a[23]); // Sum(44) + res[45] = fma52hi(res[45], a[21], a[23]); // Sum(44) + res[45] = fma52lo(res[45], a[22], a[23]); // Sum(45) + res[46] = fma52hi(res[46], a[22], a[23]); // Sum(45) + res[36] = fma52lo(res[36], a[12], a[24]); // Sum(36) + res[37] = fma52hi(res[37], a[12], a[24]); // Sum(36) + res[37] = fma52lo(res[37], a[13], a[24]); // Sum(37) + res[38] = fma52hi(res[38], a[13], a[24]); // Sum(37) + res[38] = fma52lo(res[38], a[14], a[24]); // Sum(38) + res[39] = fma52hi(res[39], a[14], a[24]); // Sum(38) + res[39] = fma52lo(res[39], a[15], a[24]); // Sum(39) + res[40] = fma52hi(res[40], a[15], a[24]); // Sum(39) + res[40] = fma52lo(res[40], a[16], a[24]); // Sum(40) + res[41] = fma52hi(res[41], a[16], a[24]); // Sum(40) + res[41] = fma52lo(res[41], a[17], a[24]); // Sum(41) + res[42] = fma52hi(res[42], a[17], a[24]); // Sum(41) + res[42] = fma52lo(res[42], a[18], a[24]); // Sum(42) + res[43] = fma52hi(res[43], a[18], a[24]); // Sum(42) + res[43] = fma52lo(res[43], a[19], a[24]); // Sum(43) + res[44] = fma52hi(res[44], a[19], a[24]); // Sum(43) + res[44] = fma52lo(res[44], a[20], a[24]); // Sum(44) + res[45] = fma52hi(res[45], a[20], a[24]); // Sum(44) + res[45] = fma52lo(res[45], a[21], a[24]); // Sum(45) + res[46] = fma52hi(res[46], a[21], a[24]); // Sum(45) + res[46] = fma52lo(res[46], a[22], a[24]); // Sum(46) + res[47] = fma52hi(res[47], a[22], a[24]); // Sum(46) + res[47] = fma52lo(res[47], a[23], a[24]); // Sum(47) + res[48] = fma52hi(res[48], a[23], a[24]); // Sum(47) + res[36] = fma52lo(res[36], a[11], a[25]); // Sum(36) + res[37] = fma52hi(res[37], a[11], a[25]); // Sum(36) + res[37] = fma52lo(res[37], a[12], a[25]); // Sum(37) + res[38] = fma52hi(res[38], a[12], a[25]); // Sum(37) + res[38] = fma52lo(res[38], a[13], a[25]); // Sum(38) + res[39] = fma52hi(res[39], a[13], a[25]); // Sum(38) + res[39] = fma52lo(res[39], a[14], a[25]); // Sum(39) + res[40] = fma52hi(res[40], a[14], a[25]); // Sum(39) + res[40] = fma52lo(res[40], a[15], a[25]); // Sum(40) + res[41] = fma52hi(res[41], a[15], a[25]); // Sum(40) + res[41] = fma52lo(res[41], a[16], a[25]); // Sum(41) + res[42] = fma52hi(res[42], a[16], a[25]); // Sum(41) + res[42] = fma52lo(res[42], a[17], a[25]); // Sum(42) + res[43] = fma52hi(res[43], a[17], a[25]); // Sum(42) + res[43] = fma52lo(res[43], a[18], a[25]); // Sum(43) + res[44] = fma52hi(res[44], a[18], a[25]); // Sum(43) + res[44] = fma52lo(res[44], a[19], a[25]); // Sum(44) + res[45] = fma52hi(res[45], a[19], a[25]); // Sum(44) + res[45] = fma52lo(res[45], a[20], a[25]); // Sum(45) + res[46] = fma52hi(res[46], a[20], a[25]); // Sum(45) + res[46] = fma52lo(res[46], a[21], a[25]); // Sum(46) + res[47] = fma52hi(res[47], a[21], a[25]); // Sum(46) + res[47] = fma52lo(res[47], a[22], a[25]); // Sum(47) + res[48] = fma52hi(res[48], a[22], a[25]); // Sum(47) + res[36] = fma52lo(res[36], a[10], a[26]); // Sum(36) + res[37] = fma52hi(res[37], a[10], a[26]); // Sum(36) + res[37] = fma52lo(res[37], a[11], a[26]); // Sum(37) + res[38] = fma52hi(res[38], a[11], a[26]); // Sum(37) + res[38] = fma52lo(res[38], a[12], a[26]); // Sum(38) + res[39] = fma52hi(res[39], a[12], a[26]); // Sum(38) + res[39] = fma52lo(res[39], a[13], a[26]); // Sum(39) + res[40] = fma52hi(res[40], a[13], a[26]); // Sum(39) + res[40] = fma52lo(res[40], a[14], a[26]); // Sum(40) + res[41] = fma52hi(res[41], a[14], a[26]); // Sum(40) + res[41] = fma52lo(res[41], a[15], a[26]); // Sum(41) + res[42] = fma52hi(res[42], a[15], a[26]); // Sum(41) + res[42] = fma52lo(res[42], a[16], a[26]); // Sum(42) + res[43] = fma52hi(res[43], a[16], a[26]); // Sum(42) + res[43] = fma52lo(res[43], a[17], a[26]); // Sum(43) + res[44] = fma52hi(res[44], a[17], a[26]); // Sum(43) + res[44] = fma52lo(res[44], a[18], a[26]); // Sum(44) + res[45] = fma52hi(res[45], a[18], a[26]); // Sum(44) + res[45] = fma52lo(res[45], a[19], a[26]); // Sum(45) + res[46] = fma52hi(res[46], a[19], a[26]); // Sum(45) + res[46] = fma52lo(res[46], a[20], a[26]); // Sum(46) + res[47] = fma52hi(res[47], a[20], a[26]); // Sum(46) + res[47] = fma52lo(res[47], a[21], a[26]); // Sum(47) + res[48] = fma52hi(res[48], a[21], a[26]); // Sum(47) + res[36] = fma52lo(res[36], a[9], a[27]); // Sum(36) + res[37] = fma52hi(res[37], a[9], a[27]); // Sum(36) + res[37] = fma52lo(res[37], a[10], a[27]); // Sum(37) + res[38] = fma52hi(res[38], a[10], a[27]); // Sum(37) + res[38] = fma52lo(res[38], a[11], a[27]); // Sum(38) + res[39] = fma52hi(res[39], a[11], a[27]); // Sum(38) + res[39] = fma52lo(res[39], a[12], a[27]); // Sum(39) + res[40] = fma52hi(res[40], a[12], a[27]); // Sum(39) + res[40] = fma52lo(res[40], a[13], a[27]); // Sum(40) + res[41] = fma52hi(res[41], a[13], a[27]); // Sum(40) + res[41] = fma52lo(res[41], a[14], a[27]); // Sum(41) + res[42] = fma52hi(res[42], a[14], a[27]); // Sum(41) + res[42] = fma52lo(res[42], a[15], a[27]); // Sum(42) + res[43] = fma52hi(res[43], a[15], a[27]); // Sum(42) + res[43] = fma52lo(res[43], a[16], a[27]); // Sum(43) + res[44] = fma52hi(res[44], a[16], a[27]); // Sum(43) + res[44] = fma52lo(res[44], a[17], a[27]); // Sum(44) + res[45] = fma52hi(res[45], a[17], a[27]); // Sum(44) + res[45] = fma52lo(res[45], a[18], a[27]); // Sum(45) + res[46] = fma52hi(res[46], a[18], a[27]); // Sum(45) + res[46] = fma52lo(res[46], a[19], a[27]); // Sum(46) + res[47] = fma52hi(res[47], a[19], a[27]); // Sum(46) + res[47] = fma52lo(res[47], a[20], a[27]); // Sum(47) + res[48] = fma52hi(res[48], a[20], a[27]); // Sum(47) + res[36] = fma52lo(res[36], a[8], a[28]); // Sum(36) + res[37] = fma52hi(res[37], a[8], a[28]); // Sum(36) + res[37] = fma52lo(res[37], a[9], a[28]); // Sum(37) + res[38] = fma52hi(res[38], a[9], a[28]); // Sum(37) + res[38] = fma52lo(res[38], a[10], a[28]); // Sum(38) + res[39] = fma52hi(res[39], a[10], a[28]); // Sum(38) + res[39] = fma52lo(res[39], a[11], a[28]); // Sum(39) + res[40] = fma52hi(res[40], a[11], a[28]); // Sum(39) + res[40] = fma52lo(res[40], a[12], a[28]); // Sum(40) + res[41] = fma52hi(res[41], a[12], a[28]); // Sum(40) + res[41] = fma52lo(res[41], a[13], a[28]); // Sum(41) + res[42] = fma52hi(res[42], a[13], a[28]); // Sum(41) + res[42] = fma52lo(res[42], a[14], a[28]); // Sum(42) + res[43] = fma52hi(res[43], a[14], a[28]); // Sum(42) + res[43] = fma52lo(res[43], a[15], a[28]); // Sum(43) + res[44] = fma52hi(res[44], a[15], a[28]); // Sum(43) + res[44] = fma52lo(res[44], a[16], a[28]); // Sum(44) + res[45] = fma52hi(res[45], a[16], a[28]); // Sum(44) + res[45] = fma52lo(res[45], a[17], a[28]); // Sum(45) + res[46] = fma52hi(res[46], a[17], a[28]); // Sum(45) + res[46] = fma52lo(res[46], a[18], a[28]); // Sum(46) + res[47] = fma52hi(res[47], a[18], a[28]); // Sum(46) + res[47] = fma52lo(res[47], a[19], a[28]); // Sum(47) + res[48] = fma52hi(res[48], a[19], a[28]); // Sum(47) + res[36] = fma52lo(res[36], a[7], a[29]); // Sum(36) + res[37] = fma52hi(res[37], a[7], a[29]); // Sum(36) + res[37] = fma52lo(res[37], a[8], a[29]); // Sum(37) + res[38] = fma52hi(res[38], a[8], a[29]); // Sum(37) + res[38] = fma52lo(res[38], a[9], a[29]); // Sum(38) + res[39] = fma52hi(res[39], a[9], a[29]); // Sum(38) + res[39] = fma52lo(res[39], a[10], a[29]); // Sum(39) + res[40] = fma52hi(res[40], a[10], a[29]); // Sum(39) + res[40] = fma52lo(res[40], a[11], a[29]); // Sum(40) + res[41] = fma52hi(res[41], a[11], a[29]); // Sum(40) + res[41] = fma52lo(res[41], a[12], a[29]); // Sum(41) + res[42] = fma52hi(res[42], a[12], a[29]); // Sum(41) + res[42] = fma52lo(res[42], a[13], a[29]); // Sum(42) + res[43] = fma52hi(res[43], a[13], a[29]); // Sum(42) + res[43] = fma52lo(res[43], a[14], a[29]); // Sum(43) + res[44] = fma52hi(res[44], a[14], a[29]); // Sum(43) + res[44] = fma52lo(res[44], a[15], a[29]); // Sum(44) + res[45] = fma52hi(res[45], a[15], a[29]); // Sum(44) + res[45] = fma52lo(res[45], a[16], a[29]); // Sum(45) + res[46] = fma52hi(res[46], a[16], a[29]); // Sum(45) + res[46] = fma52lo(res[46], a[17], a[29]); // Sum(46) + res[47] = fma52hi(res[47], a[17], a[29]); // Sum(46) + res[47] = fma52lo(res[47], a[18], a[29]); // Sum(47) + res[48] = fma52hi(res[48], a[18], a[29]); // Sum(47) + res[36] = fma52lo(res[36], a[6], a[30]); // Sum(36) + res[37] = fma52hi(res[37], a[6], a[30]); // Sum(36) + res[37] = fma52lo(res[37], a[7], a[30]); // Sum(37) + res[38] = fma52hi(res[38], a[7], a[30]); // Sum(37) + res[38] = fma52lo(res[38], a[8], a[30]); // Sum(38) + res[39] = fma52hi(res[39], a[8], a[30]); // Sum(38) + res[39] = fma52lo(res[39], a[9], a[30]); // Sum(39) + res[40] = fma52hi(res[40], a[9], a[30]); // Sum(39) + res[40] = fma52lo(res[40], a[10], a[30]); // Sum(40) + res[41] = fma52hi(res[41], a[10], a[30]); // Sum(40) + res[41] = fma52lo(res[41], a[11], a[30]); // Sum(41) + res[42] = fma52hi(res[42], a[11], a[30]); // Sum(41) + res[42] = fma52lo(res[42], a[12], a[30]); // Sum(42) + res[43] = fma52hi(res[43], a[12], a[30]); // Sum(42) + res[43] = fma52lo(res[43], a[13], a[30]); // Sum(43) + res[44] = fma52hi(res[44], a[13], a[30]); // Sum(43) + res[44] = fma52lo(res[44], a[14], a[30]); // Sum(44) + res[45] = fma52hi(res[45], a[14], a[30]); // Sum(44) + res[45] = fma52lo(res[45], a[15], a[30]); // Sum(45) + res[46] = fma52hi(res[46], a[15], a[30]); // Sum(45) + res[46] = fma52lo(res[46], a[16], a[30]); // Sum(46) + res[47] = fma52hi(res[47], a[16], a[30]); // Sum(46) + res[47] = fma52lo(res[47], a[17], a[30]); // Sum(47) + res[48] = fma52hi(res[48], a[17], a[30]); // Sum(47) + res[36] = fma52lo(res[36], a[5], a[31]); // Sum(36) + res[37] = fma52hi(res[37], a[5], a[31]); // Sum(36) + res[37] = fma52lo(res[37], a[6], a[31]); // Sum(37) + res[38] = fma52hi(res[38], a[6], a[31]); // Sum(37) + res[38] = fma52lo(res[38], a[7], a[31]); // Sum(38) + res[39] = fma52hi(res[39], a[7], a[31]); // Sum(38) + res[39] = fma52lo(res[39], a[8], a[31]); // Sum(39) + res[40] = fma52hi(res[40], a[8], a[31]); // Sum(39) + res[40] = fma52lo(res[40], a[9], a[31]); // Sum(40) + res[41] = fma52hi(res[41], a[9], a[31]); // Sum(40) + res[41] = fma52lo(res[41], a[10], a[31]); // Sum(41) + res[42] = fma52hi(res[42], a[10], a[31]); // Sum(41) + res[42] = fma52lo(res[42], a[11], a[31]); // Sum(42) + res[43] = fma52hi(res[43], a[11], a[31]); // Sum(42) + res[43] = fma52lo(res[43], a[12], a[31]); // Sum(43) + res[44] = fma52hi(res[44], a[12], a[31]); // Sum(43) + res[44] = fma52lo(res[44], a[13], a[31]); // Sum(44) + res[45] = fma52hi(res[45], a[13], a[31]); // Sum(44) + res[45] = fma52lo(res[45], a[14], a[31]); // Sum(45) + res[46] = fma52hi(res[46], a[14], a[31]); // Sum(45) + res[46] = fma52lo(res[46], a[15], a[31]); // Sum(46) + res[47] = fma52hi(res[47], a[15], a[31]); // Sum(46) + res[47] = fma52lo(res[47], a[16], a[31]); // Sum(47) + res[48] = fma52hi(res[48], a[16], a[31]); // Sum(47) + res[36] = fma52lo(res[36], a[4], a[32]); // Sum(36) + res[37] = fma52hi(res[37], a[4], a[32]); // Sum(36) + res[37] = fma52lo(res[37], a[5], a[32]); // Sum(37) + res[38] = fma52hi(res[38], a[5], a[32]); // Sum(37) + res[38] = fma52lo(res[38], a[6], a[32]); // Sum(38) + res[39] = fma52hi(res[39], a[6], a[32]); // Sum(38) + res[39] = fma52lo(res[39], a[7], a[32]); // Sum(39) + res[40] = fma52hi(res[40], a[7], a[32]); // Sum(39) + res[40] = fma52lo(res[40], a[8], a[32]); // Sum(40) + res[41] = fma52hi(res[41], a[8], a[32]); // Sum(40) + res[41] = fma52lo(res[41], a[9], a[32]); // Sum(41) + res[42] = fma52hi(res[42], a[9], a[32]); // Sum(41) + res[42] = fma52lo(res[42], a[10], a[32]); // Sum(42) + res[43] = fma52hi(res[43], a[10], a[32]); // Sum(42) + res[43] = fma52lo(res[43], a[11], a[32]); // Sum(43) + res[44] = fma52hi(res[44], a[11], a[32]); // Sum(43) + res[44] = fma52lo(res[44], a[12], a[32]); // Sum(44) + res[45] = fma52hi(res[45], a[12], a[32]); // Sum(44) + res[45] = fma52lo(res[45], a[13], a[32]); // Sum(45) + res[46] = fma52hi(res[46], a[13], a[32]); // Sum(45) + res[46] = fma52lo(res[46], a[14], a[32]); // Sum(46) + res[47] = fma52hi(res[47], a[14], a[32]); // Sum(46) + res[47] = fma52lo(res[47], a[15], a[32]); // Sum(47) + res[48] = fma52hi(res[48], a[15], a[32]); // Sum(47) + res[36] = fma52lo(res[36], a[3], a[33]); // Sum(36) + res[37] = fma52hi(res[37], a[3], a[33]); // Sum(36) + res[37] = fma52lo(res[37], a[4], a[33]); // Sum(37) + res[38] = fma52hi(res[38], a[4], a[33]); // Sum(37) + res[38] = fma52lo(res[38], a[5], a[33]); // Sum(38) + res[39] = fma52hi(res[39], a[5], a[33]); // Sum(38) + res[39] = fma52lo(res[39], a[6], a[33]); // Sum(39) + res[40] = fma52hi(res[40], a[6], a[33]); // Sum(39) + res[40] = fma52lo(res[40], a[7], a[33]); // Sum(40) + res[41] = fma52hi(res[41], a[7], a[33]); // Sum(40) + res[41] = fma52lo(res[41], a[8], a[33]); // Sum(41) + res[42] = fma52hi(res[42], a[8], a[33]); // Sum(41) + res[42] = fma52lo(res[42], a[9], a[33]); // Sum(42) + res[43] = fma52hi(res[43], a[9], a[33]); // Sum(42) + res[43] = fma52lo(res[43], a[10], a[33]); // Sum(43) + res[44] = fma52hi(res[44], a[10], a[33]); // Sum(43) + res[44] = fma52lo(res[44], a[11], a[33]); // Sum(44) + res[45] = fma52hi(res[45], a[11], a[33]); // Sum(44) + res[45] = fma52lo(res[45], a[12], a[33]); // Sum(45) + res[46] = fma52hi(res[46], a[12], a[33]); // Sum(45) + res[46] = fma52lo(res[46], a[13], a[33]); // Sum(46) + res[47] = fma52hi(res[47], a[13], a[33]); // Sum(46) + res[47] = fma52lo(res[47], a[14], a[33]); // Sum(47) + res[48] = fma52hi(res[48], a[14], a[33]); // Sum(47) + res[36] = fma52lo(res[36], a[2], a[34]); // Sum(36) + res[37] = fma52hi(res[37], a[2], a[34]); // Sum(36) + res[37] = fma52lo(res[37], a[3], a[34]); // Sum(37) + res[38] = fma52hi(res[38], a[3], a[34]); // Sum(37) + res[38] = fma52lo(res[38], a[4], a[34]); // Sum(38) + res[39] = fma52hi(res[39], a[4], a[34]); // Sum(38) + res[39] = fma52lo(res[39], a[5], a[34]); // Sum(39) + res[40] = fma52hi(res[40], a[5], a[34]); // Sum(39) + res[40] = fma52lo(res[40], a[6], a[34]); // Sum(40) + res[41] = fma52hi(res[41], a[6], a[34]); // Sum(40) + res[41] = fma52lo(res[41], a[7], a[34]); // Sum(41) + res[42] = fma52hi(res[42], a[7], a[34]); // Sum(41) + res[42] = fma52lo(res[42], a[8], a[34]); // Sum(42) + res[43] = fma52hi(res[43], a[8], a[34]); // Sum(42) + res[43] = fma52lo(res[43], a[9], a[34]); // Sum(43) + res[44] = fma52hi(res[44], a[9], a[34]); // Sum(43) + res[44] = fma52lo(res[44], a[10], a[34]); // Sum(44) + res[45] = fma52hi(res[45], a[10], a[34]); // Sum(44) + res[45] = fma52lo(res[45], a[11], a[34]); // Sum(45) + res[46] = fma52hi(res[46], a[11], a[34]); // Sum(45) + res[46] = fma52lo(res[46], a[12], a[34]); // Sum(46) + res[47] = fma52hi(res[47], a[12], a[34]); // Sum(46) + res[47] = fma52lo(res[47], a[13], a[34]); // Sum(47) + res[48] = fma52hi(res[48], a[13], a[34]); // Sum(47) + res[36] = fma52lo(res[36], a[1], a[35]); // Sum(36) + res[37] = fma52hi(res[37], a[1], a[35]); // Sum(36) + res[37] = fma52lo(res[37], a[2], a[35]); // Sum(37) + res[38] = fma52hi(res[38], a[2], a[35]); // Sum(37) + res[38] = fma52lo(res[38], a[3], a[35]); // Sum(38) + res[39] = fma52hi(res[39], a[3], a[35]); // Sum(38) + res[39] = fma52lo(res[39], a[4], a[35]); // Sum(39) + res[40] = fma52hi(res[40], a[4], a[35]); // Sum(39) + res[40] = fma52lo(res[40], a[5], a[35]); // Sum(40) + res[41] = fma52hi(res[41], a[5], a[35]); // Sum(40) + res[41] = fma52lo(res[41], a[6], a[35]); // Sum(41) + res[42] = fma52hi(res[42], a[6], a[35]); // Sum(41) + res[42] = fma52lo(res[42], a[7], a[35]); // Sum(42) + res[43] = fma52hi(res[43], a[7], a[35]); // Sum(42) + res[43] = fma52lo(res[43], a[8], a[35]); // Sum(43) + res[44] = fma52hi(res[44], a[8], a[35]); // Sum(43) + res[44] = fma52lo(res[44], a[9], a[35]); // Sum(44) + res[45] = fma52hi(res[45], a[9], a[35]); // Sum(44) + res[45] = fma52lo(res[45], a[10], a[35]); // Sum(45) + res[46] = fma52hi(res[46], a[10], a[35]); // Sum(45) + res[46] = fma52lo(res[46], a[11], a[35]); // Sum(46) + res[47] = fma52hi(res[47], a[11], a[35]); // Sum(46) + res[47] = fma52lo(res[47], a[12], a[35]); // Sum(47) + res[48] = fma52hi(res[48], a[12], a[35]); // Sum(47) + res[36] = fma52lo(res[36], a[0], a[36]); // Sum(36) + res[37] = fma52hi(res[37], a[0], a[36]); // Sum(36) + res[37] = fma52lo(res[37], a[1], a[36]); // Sum(37) + res[38] = fma52hi(res[38], a[1], a[36]); // Sum(37) + res[38] = fma52lo(res[38], a[2], a[36]); // Sum(38) + res[39] = fma52hi(res[39], a[2], a[36]); // Sum(38) + res[39] = fma52lo(res[39], a[3], a[36]); // Sum(39) + res[40] = fma52hi(res[40], a[3], a[36]); // Sum(39) + res[40] = fma52lo(res[40], a[4], a[36]); // Sum(40) + res[41] = fma52hi(res[41], a[4], a[36]); // Sum(40) + res[41] = fma52lo(res[41], a[5], a[36]); // Sum(41) + res[42] = fma52hi(res[42], a[5], a[36]); // Sum(41) + res[42] = fma52lo(res[42], a[6], a[36]); // Sum(42) + res[43] = fma52hi(res[43], a[6], a[36]); // Sum(42) + res[43] = fma52lo(res[43], a[7], a[36]); // Sum(43) + res[44] = fma52hi(res[44], a[7], a[36]); // Sum(43) + res[44] = fma52lo(res[44], a[8], a[36]); // Sum(44) + res[45] = fma52hi(res[45], a[8], a[36]); // Sum(44) + res[45] = fma52lo(res[45], a[9], a[36]); // Sum(45) + res[46] = fma52hi(res[46], a[9], a[36]); // Sum(45) + res[46] = fma52lo(res[46], a[10], a[36]); // Sum(46) + res[47] = fma52hi(res[47], a[10], a[36]); // Sum(46) + res[47] = fma52lo(res[47], a[11], a[36]); // Sum(47) + res[48] = fma52hi(res[48], a[11], a[36]); // Sum(47) + res[37] = fma52lo(res[37], a[0], a[37]); // Sum(37) + res[38] = fma52hi(res[38], a[0], a[37]); // Sum(37) + res[38] = fma52lo(res[38], a[1], a[37]); // Sum(38) + res[39] = fma52hi(res[39], a[1], a[37]); // Sum(38) + res[39] = fma52lo(res[39], a[2], a[37]); // Sum(39) + res[40] = fma52hi(res[40], a[2], a[37]); // Sum(39) + res[40] = fma52lo(res[40], a[3], a[37]); // Sum(40) + res[41] = fma52hi(res[41], a[3], a[37]); // Sum(40) + res[41] = fma52lo(res[41], a[4], a[37]); // Sum(41) + res[42] = fma52hi(res[42], a[4], a[37]); // Sum(41) + res[42] = fma52lo(res[42], a[5], a[37]); // Sum(42) + res[43] = fma52hi(res[43], a[5], a[37]); // Sum(42) + res[43] = fma52lo(res[43], a[6], a[37]); // Sum(43) + res[44] = fma52hi(res[44], a[6], a[37]); // Sum(43) + res[44] = fma52lo(res[44], a[7], a[37]); // Sum(44) + res[45] = fma52hi(res[45], a[7], a[37]); // Sum(44) + res[45] = fma52lo(res[45], a[8], a[37]); // Sum(45) + res[46] = fma52hi(res[46], a[8], a[37]); // Sum(45) + res[46] = fma52lo(res[46], a[9], a[37]); // Sum(46) + res[47] = fma52hi(res[47], a[9], a[37]); // Sum(46) + res[47] = fma52lo(res[47], a[10], a[37]); // Sum(47) + res[48] = fma52hi(res[48], a[10], a[37]); // Sum(47) + res[38] = fma52lo(res[38], a[0], a[38]); // Sum(38) + res[39] = fma52hi(res[39], a[0], a[38]); // Sum(38) + res[39] = fma52lo(res[39], a[1], a[38]); // Sum(39) + res[40] = fma52hi(res[40], a[1], a[38]); // Sum(39) + res[40] = fma52lo(res[40], a[2], a[38]); // Sum(40) + res[41] = fma52hi(res[41], a[2], a[38]); // Sum(40) + res[41] = fma52lo(res[41], a[3], a[38]); // Sum(41) + res[42] = fma52hi(res[42], a[3], a[38]); // Sum(41) + res[42] = fma52lo(res[42], a[4], a[38]); // Sum(42) + res[43] = fma52hi(res[43], a[4], a[38]); // Sum(42) + res[43] = fma52lo(res[43], a[5], a[38]); // Sum(43) + res[44] = fma52hi(res[44], a[5], a[38]); // Sum(43) + res[44] = fma52lo(res[44], a[6], a[38]); // Sum(44) + res[45] = fma52hi(res[45], a[6], a[38]); // Sum(44) + res[45] = fma52lo(res[45], a[7], a[38]); // Sum(45) + res[46] = fma52hi(res[46], a[7], a[38]); // Sum(45) + res[46] = fma52lo(res[46], a[8], a[38]); // Sum(46) + res[47] = fma52hi(res[47], a[8], a[38]); // Sum(46) + res[47] = fma52lo(res[47], a[9], a[38]); // Sum(47) + res[48] = fma52hi(res[48], a[9], a[38]); // Sum(47) + res[39] = fma52lo(res[39], a[0], a[39]); // Sum(39) + res[40] = fma52hi(res[40], a[0], a[39]); // Sum(39) + res[40] = fma52lo(res[40], a[1], a[39]); // Sum(40) + res[41] = fma52hi(res[41], a[1], a[39]); // Sum(40) + res[41] = fma52lo(res[41], a[2], a[39]); // Sum(41) + res[42] = fma52hi(res[42], a[2], a[39]); // Sum(41) + res[42] = fma52lo(res[42], a[3], a[39]); // Sum(42) + res[43] = fma52hi(res[43], a[3], a[39]); // Sum(42) + res[43] = fma52lo(res[43], a[4], a[39]); // Sum(43) + res[44] = fma52hi(res[44], a[4], a[39]); // Sum(43) + res[44] = fma52lo(res[44], a[5], a[39]); // Sum(44) + res[45] = fma52hi(res[45], a[5], a[39]); // Sum(44) + res[45] = fma52lo(res[45], a[6], a[39]); // Sum(45) + res[46] = fma52hi(res[46], a[6], a[39]); // Sum(45) + res[46] = fma52lo(res[46], a[7], a[39]); // Sum(46) + res[47] = fma52hi(res[47], a[7], a[39]); // Sum(46) + res[47] = fma52lo(res[47], a[8], a[39]); // Sum(47) + res[48] = fma52hi(res[48], a[8], a[39]); // Sum(47) + res[36] = add64(res[36], res[36]); // Double(36) + res[37] = add64(res[37], res[37]); // Double(37) + res[38] = add64(res[38], res[38]); // Double(38) + res[39] = add64(res[39], res[39]); // Double(39) + res[40] = add64(res[40], res[40]); // Double(40) + res[41] = add64(res[41], res[41]); // Double(41) + res[42] = add64(res[42], res[42]); // Double(42) + res[43] = add64(res[43], res[43]); // Double(43) + res[44] = add64(res[44], res[44]); // Double(44) + res[45] = add64(res[45], res[45]); // Double(45) + res[46] = add64(res[46], res[46]); // Double(46) + res[47] = add64(res[47], res[47]); // Double(47) + res[36] = fma52lo(res[36], a[18], a[18]); // Add sqr(36) + res[37] = fma52hi(res[37], a[18], a[18]); // Add sqr(36) + res[38] = fma52lo(res[38], a[19], a[19]); // Add sqr(38) + res[39] = fma52hi(res[39], a[19], a[19]); // Add sqr(38) + res[40] = fma52lo(res[40], a[20], a[20]); // Add sqr(40) + res[41] = fma52hi(res[41], a[20], a[20]); // Add sqr(40) + res[42] = fma52lo(res[42], a[21], a[21]); // Add sqr(42) + res[43] = fma52hi(res[43], a[21], a[21]); // Add sqr(42) + res[44] = fma52lo(res[44], a[22], a[22]); // Add sqr(44) + res[45] = fma52hi(res[45], a[22], a[22]); // Add sqr(44) + res[46] = fma52lo(res[46], a[23], a[23]); // Add sqr(46) + res[47] = fma52hi(res[47], a[23], a[23]); // Add sqr(46) + res[48] = fma52lo(res[48], a[23], a[25]); // Sum(48) + res[49] = fma52hi(res[49], a[23], a[25]); // Sum(48) + res[49] = fma52lo(res[49], a[24], a[25]); // Sum(49) + res[50] = fma52hi(res[50], a[24], a[25]); // Sum(49) + res[48] = fma52lo(res[48], a[22], a[26]); // Sum(48) + res[49] = fma52hi(res[49], a[22], a[26]); // Sum(48) + res[49] = fma52lo(res[49], a[23], a[26]); // Sum(49) + res[50] = fma52hi(res[50], a[23], a[26]); // Sum(49) + res[50] = fma52lo(res[50], a[24], a[26]); // Sum(50) + res[51] = fma52hi(res[51], a[24], a[26]); // Sum(50) + res[51] = fma52lo(res[51], a[25], a[26]); // Sum(51) + res[52] = fma52hi(res[52], a[25], a[26]); // Sum(51) + res[48] = fma52lo(res[48], a[21], a[27]); // Sum(48) + res[49] = fma52hi(res[49], a[21], a[27]); // Sum(48) + res[49] = fma52lo(res[49], a[22], a[27]); // Sum(49) + res[50] = fma52hi(res[50], a[22], a[27]); // Sum(49) + res[50] = fma52lo(res[50], a[23], a[27]); // Sum(50) + res[51] = fma52hi(res[51], a[23], a[27]); // Sum(50) + res[51] = fma52lo(res[51], a[24], a[27]); // Sum(51) + res[52] = fma52hi(res[52], a[24], a[27]); // Sum(51) + res[52] = fma52lo(res[52], a[25], a[27]); // Sum(52) + res[53] = fma52hi(res[53], a[25], a[27]); // Sum(52) + res[53] = fma52lo(res[53], a[26], a[27]); // Sum(53) + res[54] = fma52hi(res[54], a[26], a[27]); // Sum(53) + res[48] = fma52lo(res[48], a[20], a[28]); // Sum(48) + res[49] = fma52hi(res[49], a[20], a[28]); // Sum(48) + res[49] = fma52lo(res[49], a[21], a[28]); // Sum(49) + res[50] = fma52hi(res[50], a[21], a[28]); // Sum(49) + res[50] = fma52lo(res[50], a[22], a[28]); // Sum(50) + res[51] = fma52hi(res[51], a[22], a[28]); // Sum(50) + res[51] = fma52lo(res[51], a[23], a[28]); // Sum(51) + res[52] = fma52hi(res[52], a[23], a[28]); // Sum(51) + res[52] = fma52lo(res[52], a[24], a[28]); // Sum(52) + res[53] = fma52hi(res[53], a[24], a[28]); // Sum(52) + res[53] = fma52lo(res[53], a[25], a[28]); // Sum(53) + res[54] = fma52hi(res[54], a[25], a[28]); // Sum(53) + res[54] = fma52lo(res[54], a[26], a[28]); // Sum(54) + res[55] = fma52hi(res[55], a[26], a[28]); // Sum(54) + res[55] = fma52lo(res[55], a[27], a[28]); // Sum(55) + res[56] = fma52hi(res[56], a[27], a[28]); // Sum(55) + res[48] = fma52lo(res[48], a[19], a[29]); // Sum(48) + res[49] = fma52hi(res[49], a[19], a[29]); // Sum(48) + res[49] = fma52lo(res[49], a[20], a[29]); // Sum(49) + res[50] = fma52hi(res[50], a[20], a[29]); // Sum(49) + res[50] = fma52lo(res[50], a[21], a[29]); // Sum(50) + res[51] = fma52hi(res[51], a[21], a[29]); // Sum(50) + res[51] = fma52lo(res[51], a[22], a[29]); // Sum(51) + res[52] = fma52hi(res[52], a[22], a[29]); // Sum(51) + res[52] = fma52lo(res[52], a[23], a[29]); // Sum(52) + res[53] = fma52hi(res[53], a[23], a[29]); // Sum(52) + res[53] = fma52lo(res[53], a[24], a[29]); // Sum(53) + res[54] = fma52hi(res[54], a[24], a[29]); // Sum(53) + res[54] = fma52lo(res[54], a[25], a[29]); // Sum(54) + res[55] = fma52hi(res[55], a[25], a[29]); // Sum(54) + res[55] = fma52lo(res[55], a[26], a[29]); // Sum(55) + res[56] = fma52hi(res[56], a[26], a[29]); // Sum(55) + res[56] = fma52lo(res[56], a[27], a[29]); // Sum(56) + res[57] = fma52hi(res[57], a[27], a[29]); // Sum(56) + res[57] = fma52lo(res[57], a[28], a[29]); // Sum(57) + res[58] = fma52hi(res[58], a[28], a[29]); // Sum(57) + res[48] = fma52lo(res[48], a[18], a[30]); // Sum(48) + res[49] = fma52hi(res[49], a[18], a[30]); // Sum(48) + res[49] = fma52lo(res[49], a[19], a[30]); // Sum(49) + res[50] = fma52hi(res[50], a[19], a[30]); // Sum(49) + res[50] = fma52lo(res[50], a[20], a[30]); // Sum(50) + res[51] = fma52hi(res[51], a[20], a[30]); // Sum(50) + res[51] = fma52lo(res[51], a[21], a[30]); // Sum(51) + res[52] = fma52hi(res[52], a[21], a[30]); // Sum(51) + res[52] = fma52lo(res[52], a[22], a[30]); // Sum(52) + res[53] = fma52hi(res[53], a[22], a[30]); // Sum(52) + res[53] = fma52lo(res[53], a[23], a[30]); // Sum(53) + res[54] = fma52hi(res[54], a[23], a[30]); // Sum(53) + res[54] = fma52lo(res[54], a[24], a[30]); // Sum(54) + res[55] = fma52hi(res[55], a[24], a[30]); // Sum(54) + res[55] = fma52lo(res[55], a[25], a[30]); // Sum(55) + res[56] = fma52hi(res[56], a[25], a[30]); // Sum(55) + res[56] = fma52lo(res[56], a[26], a[30]); // Sum(56) + res[57] = fma52hi(res[57], a[26], a[30]); // Sum(56) + res[57] = fma52lo(res[57], a[27], a[30]); // Sum(57) + res[58] = fma52hi(res[58], a[27], a[30]); // Sum(57) + res[58] = fma52lo(res[58], a[28], a[30]); // Sum(58) + res[59] = fma52hi(res[59], a[28], a[30]); // Sum(58) + res[59] = fma52lo(res[59], a[29], a[30]); // Sum(59) + res[60] = fma52hi(res[60], a[29], a[30]); // Sum(59) + res[48] = fma52lo(res[48], a[17], a[31]); // Sum(48) + res[49] = fma52hi(res[49], a[17], a[31]); // Sum(48) + res[49] = fma52lo(res[49], a[18], a[31]); // Sum(49) + res[50] = fma52hi(res[50], a[18], a[31]); // Sum(49) + res[50] = fma52lo(res[50], a[19], a[31]); // Sum(50) + res[51] = fma52hi(res[51], a[19], a[31]); // Sum(50) + res[51] = fma52lo(res[51], a[20], a[31]); // Sum(51) + res[52] = fma52hi(res[52], a[20], a[31]); // Sum(51) + res[52] = fma52lo(res[52], a[21], a[31]); // Sum(52) + res[53] = fma52hi(res[53], a[21], a[31]); // Sum(52) + res[53] = fma52lo(res[53], a[22], a[31]); // Sum(53) + res[54] = fma52hi(res[54], a[22], a[31]); // Sum(53) + res[54] = fma52lo(res[54], a[23], a[31]); // Sum(54) + res[55] = fma52hi(res[55], a[23], a[31]); // Sum(54) + res[55] = fma52lo(res[55], a[24], a[31]); // Sum(55) + res[56] = fma52hi(res[56], a[24], a[31]); // Sum(55) + res[56] = fma52lo(res[56], a[25], a[31]); // Sum(56) + res[57] = fma52hi(res[57], a[25], a[31]); // Sum(56) + res[57] = fma52lo(res[57], a[26], a[31]); // Sum(57) + res[58] = fma52hi(res[58], a[26], a[31]); // Sum(57) + res[58] = fma52lo(res[58], a[27], a[31]); // Sum(58) + res[59] = fma52hi(res[59], a[27], a[31]); // Sum(58) + res[59] = fma52lo(res[59], a[28], a[31]); // Sum(59) + res[60] = fma52hi(res[60], a[28], a[31]); // Sum(59) + res[48] = fma52lo(res[48], a[16], a[32]); // Sum(48) + res[49] = fma52hi(res[49], a[16], a[32]); // Sum(48) + res[49] = fma52lo(res[49], a[17], a[32]); // Sum(49) + res[50] = fma52hi(res[50], a[17], a[32]); // Sum(49) + res[50] = fma52lo(res[50], a[18], a[32]); // Sum(50) + res[51] = fma52hi(res[51], a[18], a[32]); // Sum(50) + res[51] = fma52lo(res[51], a[19], a[32]); // Sum(51) + res[52] = fma52hi(res[52], a[19], a[32]); // Sum(51) + res[52] = fma52lo(res[52], a[20], a[32]); // Sum(52) + res[53] = fma52hi(res[53], a[20], a[32]); // Sum(52) + res[53] = fma52lo(res[53], a[21], a[32]); // Sum(53) + res[54] = fma52hi(res[54], a[21], a[32]); // Sum(53) + res[54] = fma52lo(res[54], a[22], a[32]); // Sum(54) + res[55] = fma52hi(res[55], a[22], a[32]); // Sum(54) + res[55] = fma52lo(res[55], a[23], a[32]); // Sum(55) + res[56] = fma52hi(res[56], a[23], a[32]); // Sum(55) + res[56] = fma52lo(res[56], a[24], a[32]); // Sum(56) + res[57] = fma52hi(res[57], a[24], a[32]); // Sum(56) + res[57] = fma52lo(res[57], a[25], a[32]); // Sum(57) + res[58] = fma52hi(res[58], a[25], a[32]); // Sum(57) + res[58] = fma52lo(res[58], a[26], a[32]); // Sum(58) + res[59] = fma52hi(res[59], a[26], a[32]); // Sum(58) + res[59] = fma52lo(res[59], a[27], a[32]); // Sum(59) + res[60] = fma52hi(res[60], a[27], a[32]); // Sum(59) + res[48] = fma52lo(res[48], a[15], a[33]); // Sum(48) + res[49] = fma52hi(res[49], a[15], a[33]); // Sum(48) + res[49] = fma52lo(res[49], a[16], a[33]); // Sum(49) + res[50] = fma52hi(res[50], a[16], a[33]); // Sum(49) + res[50] = fma52lo(res[50], a[17], a[33]); // Sum(50) + res[51] = fma52hi(res[51], a[17], a[33]); // Sum(50) + res[51] = fma52lo(res[51], a[18], a[33]); // Sum(51) + res[52] = fma52hi(res[52], a[18], a[33]); // Sum(51) + res[52] = fma52lo(res[52], a[19], a[33]); // Sum(52) + res[53] = fma52hi(res[53], a[19], a[33]); // Sum(52) + res[53] = fma52lo(res[53], a[20], a[33]); // Sum(53) + res[54] = fma52hi(res[54], a[20], a[33]); // Sum(53) + res[54] = fma52lo(res[54], a[21], a[33]); // Sum(54) + res[55] = fma52hi(res[55], a[21], a[33]); // Sum(54) + res[55] = fma52lo(res[55], a[22], a[33]); // Sum(55) + res[56] = fma52hi(res[56], a[22], a[33]); // Sum(55) + res[56] = fma52lo(res[56], a[23], a[33]); // Sum(56) + res[57] = fma52hi(res[57], a[23], a[33]); // Sum(56) + res[57] = fma52lo(res[57], a[24], a[33]); // Sum(57) + res[58] = fma52hi(res[58], a[24], a[33]); // Sum(57) + res[58] = fma52lo(res[58], a[25], a[33]); // Sum(58) + res[59] = fma52hi(res[59], a[25], a[33]); // Sum(58) + res[59] = fma52lo(res[59], a[26], a[33]); // Sum(59) + res[60] = fma52hi(res[60], a[26], a[33]); // Sum(59) + res[48] = fma52lo(res[48], a[14], a[34]); // Sum(48) + res[49] = fma52hi(res[49], a[14], a[34]); // Sum(48) + res[49] = fma52lo(res[49], a[15], a[34]); // Sum(49) + res[50] = fma52hi(res[50], a[15], a[34]); // Sum(49) + res[50] = fma52lo(res[50], a[16], a[34]); // Sum(50) + res[51] = fma52hi(res[51], a[16], a[34]); // Sum(50) + res[51] = fma52lo(res[51], a[17], a[34]); // Sum(51) + res[52] = fma52hi(res[52], a[17], a[34]); // Sum(51) + res[52] = fma52lo(res[52], a[18], a[34]); // Sum(52) + res[53] = fma52hi(res[53], a[18], a[34]); // Sum(52) + res[53] = fma52lo(res[53], a[19], a[34]); // Sum(53) + res[54] = fma52hi(res[54], a[19], a[34]); // Sum(53) + res[54] = fma52lo(res[54], a[20], a[34]); // Sum(54) + res[55] = fma52hi(res[55], a[20], a[34]); // Sum(54) + res[55] = fma52lo(res[55], a[21], a[34]); // Sum(55) + res[56] = fma52hi(res[56], a[21], a[34]); // Sum(55) + res[56] = fma52lo(res[56], a[22], a[34]); // Sum(56) + res[57] = fma52hi(res[57], a[22], a[34]); // Sum(56) + res[57] = fma52lo(res[57], a[23], a[34]); // Sum(57) + res[58] = fma52hi(res[58], a[23], a[34]); // Sum(57) + res[58] = fma52lo(res[58], a[24], a[34]); // Sum(58) + res[59] = fma52hi(res[59], a[24], a[34]); // Sum(58) + res[59] = fma52lo(res[59], a[25], a[34]); // Sum(59) + res[60] = fma52hi(res[60], a[25], a[34]); // Sum(59) + res[48] = fma52lo(res[48], a[13], a[35]); // Sum(48) + res[49] = fma52hi(res[49], a[13], a[35]); // Sum(48) + res[49] = fma52lo(res[49], a[14], a[35]); // Sum(49) + res[50] = fma52hi(res[50], a[14], a[35]); // Sum(49) + res[50] = fma52lo(res[50], a[15], a[35]); // Sum(50) + res[51] = fma52hi(res[51], a[15], a[35]); // Sum(50) + res[51] = fma52lo(res[51], a[16], a[35]); // Sum(51) + res[52] = fma52hi(res[52], a[16], a[35]); // Sum(51) + res[52] = fma52lo(res[52], a[17], a[35]); // Sum(52) + res[53] = fma52hi(res[53], a[17], a[35]); // Sum(52) + res[53] = fma52lo(res[53], a[18], a[35]); // Sum(53) + res[54] = fma52hi(res[54], a[18], a[35]); // Sum(53) + res[54] = fma52lo(res[54], a[19], a[35]); // Sum(54) + res[55] = fma52hi(res[55], a[19], a[35]); // Sum(54) + res[55] = fma52lo(res[55], a[20], a[35]); // Sum(55) + res[56] = fma52hi(res[56], a[20], a[35]); // Sum(55) + res[56] = fma52lo(res[56], a[21], a[35]); // Sum(56) + res[57] = fma52hi(res[57], a[21], a[35]); // Sum(56) + res[57] = fma52lo(res[57], a[22], a[35]); // Sum(57) + res[58] = fma52hi(res[58], a[22], a[35]); // Sum(57) + res[58] = fma52lo(res[58], a[23], a[35]); // Sum(58) + res[59] = fma52hi(res[59], a[23], a[35]); // Sum(58) + res[59] = fma52lo(res[59], a[24], a[35]); // Sum(59) + res[60] = fma52hi(res[60], a[24], a[35]); // Sum(59) + res[48] = fma52lo(res[48], a[12], a[36]); // Sum(48) + res[49] = fma52hi(res[49], a[12], a[36]); // Sum(48) + res[49] = fma52lo(res[49], a[13], a[36]); // Sum(49) + res[50] = fma52hi(res[50], a[13], a[36]); // Sum(49) + res[50] = fma52lo(res[50], a[14], a[36]); // Sum(50) + res[51] = fma52hi(res[51], a[14], a[36]); // Sum(50) + res[51] = fma52lo(res[51], a[15], a[36]); // Sum(51) + res[52] = fma52hi(res[52], a[15], a[36]); // Sum(51) + res[52] = fma52lo(res[52], a[16], a[36]); // Sum(52) + res[53] = fma52hi(res[53], a[16], a[36]); // Sum(52) + res[53] = fma52lo(res[53], a[17], a[36]); // Sum(53) + res[54] = fma52hi(res[54], a[17], a[36]); // Sum(53) + res[54] = fma52lo(res[54], a[18], a[36]); // Sum(54) + res[55] = fma52hi(res[55], a[18], a[36]); // Sum(54) + res[55] = fma52lo(res[55], a[19], a[36]); // Sum(55) + res[56] = fma52hi(res[56], a[19], a[36]); // Sum(55) + res[56] = fma52lo(res[56], a[20], a[36]); // Sum(56) + res[57] = fma52hi(res[57], a[20], a[36]); // Sum(56) + res[57] = fma52lo(res[57], a[21], a[36]); // Sum(57) + res[58] = fma52hi(res[58], a[21], a[36]); // Sum(57) + res[58] = fma52lo(res[58], a[22], a[36]); // Sum(58) + res[59] = fma52hi(res[59], a[22], a[36]); // Sum(58) + res[59] = fma52lo(res[59], a[23], a[36]); // Sum(59) + res[60] = fma52hi(res[60], a[23], a[36]); // Sum(59) + res[48] = fma52lo(res[48], a[11], a[37]); // Sum(48) + res[49] = fma52hi(res[49], a[11], a[37]); // Sum(48) + res[49] = fma52lo(res[49], a[12], a[37]); // Sum(49) + res[50] = fma52hi(res[50], a[12], a[37]); // Sum(49) + res[50] = fma52lo(res[50], a[13], a[37]); // Sum(50) + res[51] = fma52hi(res[51], a[13], a[37]); // Sum(50) + res[51] = fma52lo(res[51], a[14], a[37]); // Sum(51) + res[52] = fma52hi(res[52], a[14], a[37]); // Sum(51) + res[52] = fma52lo(res[52], a[15], a[37]); // Sum(52) + res[53] = fma52hi(res[53], a[15], a[37]); // Sum(52) + res[53] = fma52lo(res[53], a[16], a[37]); // Sum(53) + res[54] = fma52hi(res[54], a[16], a[37]); // Sum(53) + res[54] = fma52lo(res[54], a[17], a[37]); // Sum(54) + res[55] = fma52hi(res[55], a[17], a[37]); // Sum(54) + res[55] = fma52lo(res[55], a[18], a[37]); // Sum(55) + res[56] = fma52hi(res[56], a[18], a[37]); // Sum(55) + res[56] = fma52lo(res[56], a[19], a[37]); // Sum(56) + res[57] = fma52hi(res[57], a[19], a[37]); // Sum(56) + res[57] = fma52lo(res[57], a[20], a[37]); // Sum(57) + res[58] = fma52hi(res[58], a[20], a[37]); // Sum(57) + res[58] = fma52lo(res[58], a[21], a[37]); // Sum(58) + res[59] = fma52hi(res[59], a[21], a[37]); // Sum(58) + res[59] = fma52lo(res[59], a[22], a[37]); // Sum(59) + res[60] = fma52hi(res[60], a[22], a[37]); // Sum(59) + res[48] = fma52lo(res[48], a[10], a[38]); // Sum(48) + res[49] = fma52hi(res[49], a[10], a[38]); // Sum(48) + res[49] = fma52lo(res[49], a[11], a[38]); // Sum(49) + res[50] = fma52hi(res[50], a[11], a[38]); // Sum(49) + res[50] = fma52lo(res[50], a[12], a[38]); // Sum(50) + res[51] = fma52hi(res[51], a[12], a[38]); // Sum(50) + res[51] = fma52lo(res[51], a[13], a[38]); // Sum(51) + res[52] = fma52hi(res[52], a[13], a[38]); // Sum(51) + res[52] = fma52lo(res[52], a[14], a[38]); // Sum(52) + res[53] = fma52hi(res[53], a[14], a[38]); // Sum(52) + res[53] = fma52lo(res[53], a[15], a[38]); // Sum(53) + res[54] = fma52hi(res[54], a[15], a[38]); // Sum(53) + res[54] = fma52lo(res[54], a[16], a[38]); // Sum(54) + res[55] = fma52hi(res[55], a[16], a[38]); // Sum(54) + res[55] = fma52lo(res[55], a[17], a[38]); // Sum(55) + res[56] = fma52hi(res[56], a[17], a[38]); // Sum(55) + res[56] = fma52lo(res[56], a[18], a[38]); // Sum(56) + res[57] = fma52hi(res[57], a[18], a[38]); // Sum(56) + res[57] = fma52lo(res[57], a[19], a[38]); // Sum(57) + res[58] = fma52hi(res[58], a[19], a[38]); // Sum(57) + res[58] = fma52lo(res[58], a[20], a[38]); // Sum(58) + res[59] = fma52hi(res[59], a[20], a[38]); // Sum(58) + res[59] = fma52lo(res[59], a[21], a[38]); // Sum(59) + res[60] = fma52hi(res[60], a[21], a[38]); // Sum(59) + res[48] = fma52lo(res[48], a[9], a[39]); // Sum(48) + res[49] = fma52hi(res[49], a[9], a[39]); // Sum(48) + res[49] = fma52lo(res[49], a[10], a[39]); // Sum(49) + res[50] = fma52hi(res[50], a[10], a[39]); // Sum(49) + res[50] = fma52lo(res[50], a[11], a[39]); // Sum(50) + res[51] = fma52hi(res[51], a[11], a[39]); // Sum(50) + res[51] = fma52lo(res[51], a[12], a[39]); // Sum(51) + res[52] = fma52hi(res[52], a[12], a[39]); // Sum(51) + res[52] = fma52lo(res[52], a[13], a[39]); // Sum(52) + res[53] = fma52hi(res[53], a[13], a[39]); // Sum(52) + res[53] = fma52lo(res[53], a[14], a[39]); // Sum(53) + res[54] = fma52hi(res[54], a[14], a[39]); // Sum(53) + res[54] = fma52lo(res[54], a[15], a[39]); // Sum(54) + res[55] = fma52hi(res[55], a[15], a[39]); // Sum(54) + res[55] = fma52lo(res[55], a[16], a[39]); // Sum(55) + res[56] = fma52hi(res[56], a[16], a[39]); // Sum(55) + res[56] = fma52lo(res[56], a[17], a[39]); // Sum(56) + res[57] = fma52hi(res[57], a[17], a[39]); // Sum(56) + res[57] = fma52lo(res[57], a[18], a[39]); // Sum(57) + res[58] = fma52hi(res[58], a[18], a[39]); // Sum(57) + res[58] = fma52lo(res[58], a[19], a[39]); // Sum(58) + res[59] = fma52hi(res[59], a[19], a[39]); // Sum(58) + res[59] = fma52lo(res[59], a[20], a[39]); // Sum(59) + res[60] = fma52hi(res[60], a[20], a[39]); // Sum(59) + res[48] = add64(res[48], res[48]); // Double(48) + res[49] = add64(res[49], res[49]); // Double(49) + res[50] = add64(res[50], res[50]); // Double(50) + res[51] = add64(res[51], res[51]); // Double(51) + res[52] = add64(res[52], res[52]); // Double(52) + res[53] = add64(res[53], res[53]); // Double(53) + res[54] = add64(res[54], res[54]); // Double(54) + res[55] = add64(res[55], res[55]); // Double(55) + res[56] = add64(res[56], res[56]); // Double(56) + res[57] = add64(res[57], res[57]); // Double(57) + res[58] = add64(res[58], res[58]); // Double(58) + res[59] = add64(res[59], res[59]); // Double(59) + res[48] = fma52lo(res[48], a[24], a[24]); // Add sqr(48) + res[49] = fma52hi(res[49], a[24], a[24]); // Add sqr(48) + res[50] = fma52lo(res[50], a[25], a[25]); // Add sqr(50) + res[51] = fma52hi(res[51], a[25], a[25]); // Add sqr(50) + res[52] = fma52lo(res[52], a[26], a[26]); // Add sqr(52) + res[53] = fma52hi(res[53], a[26], a[26]); // Add sqr(52) + res[54] = fma52lo(res[54], a[27], a[27]); // Add sqr(54) + res[55] = fma52hi(res[55], a[27], a[27]); // Add sqr(54) + res[56] = fma52lo(res[56], a[28], a[28]); // Add sqr(56) + res[57] = fma52hi(res[57], a[28], a[28]); // Add sqr(56) + res[58] = fma52lo(res[58], a[29], a[29]); // Add sqr(58) + res[59] = fma52hi(res[59], a[29], a[29]); // Add sqr(58) + res[60] = fma52lo(res[60], a[29], a[31]); // Sum(60) + res[61] = fma52hi(res[61], a[29], a[31]); // Sum(60) + res[61] = fma52lo(res[61], a[30], a[31]); // Sum(61) + res[62] = fma52hi(res[62], a[30], a[31]); // Sum(61) + res[60] = fma52lo(res[60], a[28], a[32]); // Sum(60) + res[61] = fma52hi(res[61], a[28], a[32]); // Sum(60) + res[61] = fma52lo(res[61], a[29], a[32]); // Sum(61) + res[62] = fma52hi(res[62], a[29], a[32]); // Sum(61) + res[62] = fma52lo(res[62], a[30], a[32]); // Sum(62) + res[63] = fma52hi(res[63], a[30], a[32]); // Sum(62) + res[63] = fma52lo(res[63], a[31], a[32]); // Sum(63) + res[64] = fma52hi(res[64], a[31], a[32]); // Sum(63) + res[60] = fma52lo(res[60], a[27], a[33]); // Sum(60) + res[61] = fma52hi(res[61], a[27], a[33]); // Sum(60) + res[61] = fma52lo(res[61], a[28], a[33]); // Sum(61) + res[62] = fma52hi(res[62], a[28], a[33]); // Sum(61) + res[62] = fma52lo(res[62], a[29], a[33]); // Sum(62) + res[63] = fma52hi(res[63], a[29], a[33]); // Sum(62) + res[63] = fma52lo(res[63], a[30], a[33]); // Sum(63) + res[64] = fma52hi(res[64], a[30], a[33]); // Sum(63) + res[64] = fma52lo(res[64], a[31], a[33]); // Sum(64) + res[65] = fma52hi(res[65], a[31], a[33]); // Sum(64) + res[65] = fma52lo(res[65], a[32], a[33]); // Sum(65) + res[66] = fma52hi(res[66], a[32], a[33]); // Sum(65) + res[60] = fma52lo(res[60], a[26], a[34]); // Sum(60) + res[61] = fma52hi(res[61], a[26], a[34]); // Sum(60) + res[61] = fma52lo(res[61], a[27], a[34]); // Sum(61) + res[62] = fma52hi(res[62], a[27], a[34]); // Sum(61) + res[62] = fma52lo(res[62], a[28], a[34]); // Sum(62) + res[63] = fma52hi(res[63], a[28], a[34]); // Sum(62) + res[63] = fma52lo(res[63], a[29], a[34]); // Sum(63) + res[64] = fma52hi(res[64], a[29], a[34]); // Sum(63) + res[64] = fma52lo(res[64], a[30], a[34]); // Sum(64) + res[65] = fma52hi(res[65], a[30], a[34]); // Sum(64) + res[65] = fma52lo(res[65], a[31], a[34]); // Sum(65) + res[66] = fma52hi(res[66], a[31], a[34]); // Sum(65) + res[66] = fma52lo(res[66], a[32], a[34]); // Sum(66) + res[67] = fma52hi(res[67], a[32], a[34]); // Sum(66) + res[67] = fma52lo(res[67], a[33], a[34]); // Sum(67) + res[68] = fma52hi(res[68], a[33], a[34]); // Sum(67) + res[60] = fma52lo(res[60], a[25], a[35]); // Sum(60) + res[61] = fma52hi(res[61], a[25], a[35]); // Sum(60) + res[61] = fma52lo(res[61], a[26], a[35]); // Sum(61) + res[62] = fma52hi(res[62], a[26], a[35]); // Sum(61) + res[62] = fma52lo(res[62], a[27], a[35]); // Sum(62) + res[63] = fma52hi(res[63], a[27], a[35]); // Sum(62) + res[63] = fma52lo(res[63], a[28], a[35]); // Sum(63) + res[64] = fma52hi(res[64], a[28], a[35]); // Sum(63) + res[64] = fma52lo(res[64], a[29], a[35]); // Sum(64) + res[65] = fma52hi(res[65], a[29], a[35]); // Sum(64) + res[65] = fma52lo(res[65], a[30], a[35]); // Sum(65) + res[66] = fma52hi(res[66], a[30], a[35]); // Sum(65) + res[66] = fma52lo(res[66], a[31], a[35]); // Sum(66) + res[67] = fma52hi(res[67], a[31], a[35]); // Sum(66) + res[67] = fma52lo(res[67], a[32], a[35]); // Sum(67) + res[68] = fma52hi(res[68], a[32], a[35]); // Sum(67) + res[68] = fma52lo(res[68], a[33], a[35]); // Sum(68) + res[69] = fma52hi(res[69], a[33], a[35]); // Sum(68) + res[69] = fma52lo(res[69], a[34], a[35]); // Sum(69) + res[70] = fma52hi(res[70], a[34], a[35]); // Sum(69) + res[60] = fma52lo(res[60], a[24], a[36]); // Sum(60) + res[61] = fma52hi(res[61], a[24], a[36]); // Sum(60) + res[61] = fma52lo(res[61], a[25], a[36]); // Sum(61) + res[62] = fma52hi(res[62], a[25], a[36]); // Sum(61) + res[62] = fma52lo(res[62], a[26], a[36]); // Sum(62) + res[63] = fma52hi(res[63], a[26], a[36]); // Sum(62) + res[63] = fma52lo(res[63], a[27], a[36]); // Sum(63) + res[64] = fma52hi(res[64], a[27], a[36]); // Sum(63) + res[64] = fma52lo(res[64], a[28], a[36]); // Sum(64) + res[65] = fma52hi(res[65], a[28], a[36]); // Sum(64) + res[65] = fma52lo(res[65], a[29], a[36]); // Sum(65) + res[66] = fma52hi(res[66], a[29], a[36]); // Sum(65) + res[66] = fma52lo(res[66], a[30], a[36]); // Sum(66) + res[67] = fma52hi(res[67], a[30], a[36]); // Sum(66) + res[67] = fma52lo(res[67], a[31], a[36]); // Sum(67) + res[68] = fma52hi(res[68], a[31], a[36]); // Sum(67) + res[68] = fma52lo(res[68], a[32], a[36]); // Sum(68) + res[69] = fma52hi(res[69], a[32], a[36]); // Sum(68) + res[69] = fma52lo(res[69], a[33], a[36]); // Sum(69) + res[70] = fma52hi(res[70], a[33], a[36]); // Sum(69) + res[70] = fma52lo(res[70], a[34], a[36]); // Sum(70) + res[71] = fma52hi(res[71], a[34], a[36]); // Sum(70) + res[71] = fma52lo(res[71], a[35], a[36]); // Sum(71) + res[72] = fma52hi(res[72], a[35], a[36]); // Sum(71) + res[60] = fma52lo(res[60], a[23], a[37]); // Sum(60) + res[61] = fma52hi(res[61], a[23], a[37]); // Sum(60) + res[61] = fma52lo(res[61], a[24], a[37]); // Sum(61) + res[62] = fma52hi(res[62], a[24], a[37]); // Sum(61) + res[62] = fma52lo(res[62], a[25], a[37]); // Sum(62) + res[63] = fma52hi(res[63], a[25], a[37]); // Sum(62) + res[63] = fma52lo(res[63], a[26], a[37]); // Sum(63) + res[64] = fma52hi(res[64], a[26], a[37]); // Sum(63) + res[64] = fma52lo(res[64], a[27], a[37]); // Sum(64) + res[65] = fma52hi(res[65], a[27], a[37]); // Sum(64) + res[65] = fma52lo(res[65], a[28], a[37]); // Sum(65) + res[66] = fma52hi(res[66], a[28], a[37]); // Sum(65) + res[66] = fma52lo(res[66], a[29], a[37]); // Sum(66) + res[67] = fma52hi(res[67], a[29], a[37]); // Sum(66) + res[67] = fma52lo(res[67], a[30], a[37]); // Sum(67) + res[68] = fma52hi(res[68], a[30], a[37]); // Sum(67) + res[68] = fma52lo(res[68], a[31], a[37]); // Sum(68) + res[69] = fma52hi(res[69], a[31], a[37]); // Sum(68) + res[69] = fma52lo(res[69], a[32], a[37]); // Sum(69) + res[70] = fma52hi(res[70], a[32], a[37]); // Sum(69) + res[70] = fma52lo(res[70], a[33], a[37]); // Sum(70) + res[71] = fma52hi(res[71], a[33], a[37]); // Sum(70) + res[71] = fma52lo(res[71], a[34], a[37]); // Sum(71) + res[72] = fma52hi(res[72], a[34], a[37]); // Sum(71) + res[60] = fma52lo(res[60], a[22], a[38]); // Sum(60) + res[61] = fma52hi(res[61], a[22], a[38]); // Sum(60) + res[61] = fma52lo(res[61], a[23], a[38]); // Sum(61) + res[62] = fma52hi(res[62], a[23], a[38]); // Sum(61) + res[62] = fma52lo(res[62], a[24], a[38]); // Sum(62) + res[63] = fma52hi(res[63], a[24], a[38]); // Sum(62) + res[63] = fma52lo(res[63], a[25], a[38]); // Sum(63) + res[64] = fma52hi(res[64], a[25], a[38]); // Sum(63) + res[64] = fma52lo(res[64], a[26], a[38]); // Sum(64) + res[65] = fma52hi(res[65], a[26], a[38]); // Sum(64) + res[65] = fma52lo(res[65], a[27], a[38]); // Sum(65) + res[66] = fma52hi(res[66], a[27], a[38]); // Sum(65) + res[66] = fma52lo(res[66], a[28], a[38]); // Sum(66) + res[67] = fma52hi(res[67], a[28], a[38]); // Sum(66) + res[67] = fma52lo(res[67], a[29], a[38]); // Sum(67) + res[68] = fma52hi(res[68], a[29], a[38]); // Sum(67) + res[68] = fma52lo(res[68], a[30], a[38]); // Sum(68) + res[69] = fma52hi(res[69], a[30], a[38]); // Sum(68) + res[69] = fma52lo(res[69], a[31], a[38]); // Sum(69) + res[70] = fma52hi(res[70], a[31], a[38]); // Sum(69) + res[70] = fma52lo(res[70], a[32], a[38]); // Sum(70) + res[71] = fma52hi(res[71], a[32], a[38]); // Sum(70) + res[71] = fma52lo(res[71], a[33], a[38]); // Sum(71) + res[72] = fma52hi(res[72], a[33], a[38]); // Sum(71) + res[60] = fma52lo(res[60], a[21], a[39]); // Sum(60) + res[61] = fma52hi(res[61], a[21], a[39]); // Sum(60) + res[61] = fma52lo(res[61], a[22], a[39]); // Sum(61) + res[62] = fma52hi(res[62], a[22], a[39]); // Sum(61) + res[62] = fma52lo(res[62], a[23], a[39]); // Sum(62) + res[63] = fma52hi(res[63], a[23], a[39]); // Sum(62) + res[63] = fma52lo(res[63], a[24], a[39]); // Sum(63) + res[64] = fma52hi(res[64], a[24], a[39]); // Sum(63) + res[64] = fma52lo(res[64], a[25], a[39]); // Sum(64) + res[65] = fma52hi(res[65], a[25], a[39]); // Sum(64) + res[65] = fma52lo(res[65], a[26], a[39]); // Sum(65) + res[66] = fma52hi(res[66], a[26], a[39]); // Sum(65) + res[66] = fma52lo(res[66], a[27], a[39]); // Sum(66) + res[67] = fma52hi(res[67], a[27], a[39]); // Sum(66) + res[67] = fma52lo(res[67], a[28], a[39]); // Sum(67) + res[68] = fma52hi(res[68], a[28], a[39]); // Sum(67) + res[68] = fma52lo(res[68], a[29], a[39]); // Sum(68) + res[69] = fma52hi(res[69], a[29], a[39]); // Sum(68) + res[69] = fma52lo(res[69], a[30], a[39]); // Sum(69) + res[70] = fma52hi(res[70], a[30], a[39]); // Sum(69) + res[70] = fma52lo(res[70], a[31], a[39]); // Sum(70) + res[71] = fma52hi(res[71], a[31], a[39]); // Sum(70) + res[71] = fma52lo(res[71], a[32], a[39]); // Sum(71) + res[72] = fma52hi(res[72], a[32], a[39]); // Sum(71) + res[60] = add64(res[60], res[60]); // Double(60) + res[61] = add64(res[61], res[61]); // Double(61) + res[62] = add64(res[62], res[62]); // Double(62) + res[63] = add64(res[63], res[63]); // Double(63) + res[64] = add64(res[64], res[64]); // Double(64) + res[65] = add64(res[65], res[65]); // Double(65) + res[66] = add64(res[66], res[66]); // Double(66) + res[67] = add64(res[67], res[67]); // Double(67) + res[68] = add64(res[68], res[68]); // Double(68) + res[69] = add64(res[69], res[69]); // Double(69) + res[70] = add64(res[70], res[70]); // Double(70) + res[71] = add64(res[71], res[71]); // Double(71) + res[60] = fma52lo(res[60], a[30], a[30]); // Add sqr(60) + res[61] = fma52hi(res[61], a[30], a[30]); // Add sqr(60) + res[62] = fma52lo(res[62], a[31], a[31]); // Add sqr(62) + res[63] = fma52hi(res[63], a[31], a[31]); // Add sqr(62) + res[64] = fma52lo(res[64], a[32], a[32]); // Add sqr(64) + res[65] = fma52hi(res[65], a[32], a[32]); // Add sqr(64) + res[66] = fma52lo(res[66], a[33], a[33]); // Add sqr(66) + res[67] = fma52hi(res[67], a[33], a[33]); // Add sqr(66) + res[68] = fma52lo(res[68], a[34], a[34]); // Add sqr(68) + res[69] = fma52hi(res[69], a[34], a[34]); // Add sqr(68) + res[70] = fma52lo(res[70], a[35], a[35]); // Add sqr(70) + res[71] = fma52hi(res[71], a[35], a[35]); // Add sqr(70) + res[72] = fma52lo(res[72], a[35], a[37]); // Sum(72) + res[73] = fma52hi(res[73], a[35], a[37]); // Sum(72) + res[73] = fma52lo(res[73], a[36], a[37]); // Sum(73) + res[74] = fma52hi(res[74], a[36], a[37]); // Sum(73) + res[72] = fma52lo(res[72], a[34], a[38]); // Sum(72) + res[73] = fma52hi(res[73], a[34], a[38]); // Sum(72) + res[73] = fma52lo(res[73], a[35], a[38]); // Sum(73) + res[74] = fma52hi(res[74], a[35], a[38]); // Sum(73) + res[74] = fma52lo(res[74], a[36], a[38]); // Sum(74) + res[75] = fma52hi(res[75], a[36], a[38]); // Sum(74) + res[75] = fma52lo(res[75], a[37], a[38]); // Sum(75) + res[76] = fma52hi(res[76], a[37], a[38]); // Sum(75) + res[72] = fma52lo(res[72], a[33], a[39]); // Sum(72) + res[73] = fma52hi(res[73], a[33], a[39]); // Sum(72) + res[73] = fma52lo(res[73], a[34], a[39]); // Sum(73) + res[74] = fma52hi(res[74], a[34], a[39]); // Sum(73) + res[74] = fma52lo(res[74], a[35], a[39]); // Sum(74) + res[75] = fma52hi(res[75], a[35], a[39]); // Sum(74) + res[75] = fma52lo(res[75], a[36], a[39]); // Sum(75) + res[76] = fma52hi(res[76], a[36], a[39]); // Sum(75) + res[76] = fma52lo(res[76], a[37], a[39]); // Sum(76) + res[77] = fma52hi(res[77], a[37], a[39]); // Sum(76) + res[77] = fma52lo(res[77], a[38], a[39]); // Sum(77) + res[78] = fma52hi(res[78], a[38], a[39]); // Sum(77) + res[72] = add64(res[72], res[72]); // Double(72) + res[73] = add64(res[73], res[73]); // Double(73) + res[74] = add64(res[74], res[74]); // Double(74) + res[75] = add64(res[75], res[75]); // Double(75) + res[76] = add64(res[76], res[76]); // Double(76) + res[77] = add64(res[77], res[77]); // Double(77) + res[78] = add64(res[78], res[78]); // Double(78) + res[72] = fma52lo(res[72], a[36], a[36]); // Add sqr(72) + res[73] = fma52hi(res[73], a[36], a[36]); // Add sqr(72) + res[74] = fma52lo(res[74], a[37], a[37]); // Add sqr(74) + res[75] = fma52hi(res[75], a[37], a[37]); // Add sqr(74) + res[76] = fma52lo(res[76], a[38], a[38]); // Add sqr(76) + res[77] = fma52hi(res[77], a[38], a[38]); // Add sqr(76) + res[78] = fma52lo(res[78], a[39], a[39]); // Add sqr(78) + res[79] = fma52hi(res[79], a[39], a[39]); // Add sqr(78) + + // Montgomery Reduction + int it; + for (it = 0; it < 40; it += 10) { // Reduction step + int jt = 0; + if ((it + 0) > 0) + res[it + 0] = add64(res[it + 0], srli64(res[it + -1], DIGIT_SIZE)); + u[it + 0] = mul52lo(res[it + 0], k); + res[it + jt + 0] = fma52lo(res[it + jt + 0], u[it + 0], m[jt + 0]); + res[it + jt + 1] = fma52hi(res[it + jt + 1], u[it + 0], m[jt + 0]); + res[it + jt + 1] = fma52lo(res[it + jt + 1], u[it + 0], m[jt + 1]); + res[it + jt + 2] = fma52hi(res[it + jt + 2], u[it + 0], m[jt + 1]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 0], m[jt + 2]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 0], m[jt + 2]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 0], m[jt + 3]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 0], m[jt + 3]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 0], m[jt + 4]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 0], m[jt + 4]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 0], m[jt + 5]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 0], m[jt + 5]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 0], m[jt + 6]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 0], m[jt + 6]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 0], m[jt + 7]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 0], m[jt + 7]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 0], m[jt + 8]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 0], m[jt + 8]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 0], m[jt + 9]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 0], m[jt + 9]); + res[it + 1] = add64(res[it + 1], srli64(res[it + 0], DIGIT_SIZE)); + u[it + 1] = mul52lo(res[it + 1], k); + res[it + jt + 1] = fma52lo(res[it + jt + 1], u[it + 1], m[jt + 0]); + res[it + jt + 2] = fma52hi(res[it + jt + 2], u[it + 1], m[jt + 0]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 1], m[jt + 1]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 1], m[jt + 1]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 1], m[jt + 2]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 1], m[jt + 2]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 1], m[jt + 3]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 1], m[jt + 3]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 1], m[jt + 4]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 1], m[jt + 4]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 1], m[jt + 5]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 1], m[jt + 5]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 1], m[jt + 6]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 1], m[jt + 6]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 1], m[jt + 7]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 1], m[jt + 7]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 1], m[jt + 8]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 1], m[jt + 8]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 1], m[jt + 9]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 1], m[jt + 9]); + res[it + 2] = add64(res[it + 2], srli64(res[it + 1], DIGIT_SIZE)); + u[it + 2] = mul52lo(res[it + 2], k); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 2], m[jt + 0]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 2], m[jt + 0]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 2], m[jt + 1]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 2], m[jt + 1]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 2], m[jt + 2]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 2], m[jt + 2]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 2], m[jt + 3]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 2], m[jt + 3]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 2], m[jt + 4]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 2], m[jt + 4]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 2], m[jt + 5]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 2], m[jt + 5]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 2], m[jt + 6]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 2], m[jt + 6]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 2], m[jt + 7]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 2], m[jt + 7]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 2], m[jt + 8]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 2], m[jt + 8]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 2], m[jt + 9]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 2], m[jt + 9]); + res[it + 3] = add64(res[it + 3], srli64(res[it + 2], DIGIT_SIZE)); + u[it + 3] = mul52lo(res[it + 3], k); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 3], m[jt + 0]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 3], m[jt + 0]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 3], m[jt + 1]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 3], m[jt + 1]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 3], m[jt + 2]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 3], m[jt + 2]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 3], m[jt + 3]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 3], m[jt + 3]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 3], m[jt + 4]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 3], m[jt + 4]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 3], m[jt + 5]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 3], m[jt + 5]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 3], m[jt + 6]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 3], m[jt + 6]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 3], m[jt + 7]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 3], m[jt + 7]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 3], m[jt + 8]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 3], m[jt + 8]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 3], m[jt + 9]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 3], m[jt + 9]); + res[it + 4] = add64(res[it + 4], srli64(res[it + 3], DIGIT_SIZE)); + u[it + 4] = mul52lo(res[it + 4], k); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 4], m[jt + 0]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 4], m[jt + 0]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 4], m[jt + 1]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 4], m[jt + 1]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 4], m[jt + 2]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 4], m[jt + 2]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 4], m[jt + 3]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 4], m[jt + 3]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 4], m[jt + 4]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 4], m[jt + 4]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 4], m[jt + 5]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 4], m[jt + 5]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 4], m[jt + 6]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 4], m[jt + 6]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 4], m[jt + 7]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 4], m[jt + 7]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 4], m[jt + 8]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 4], m[jt + 8]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 4], m[jt + 9]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 4], m[jt + 9]); + res[it + 5] = add64(res[it + 5], srli64(res[it + 4], DIGIT_SIZE)); + u[it + 5] = mul52lo(res[it + 5], k); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 5], m[jt + 0]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 5], m[jt + 0]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 5], m[jt + 1]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 5], m[jt + 1]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 5], m[jt + 2]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 5], m[jt + 2]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 5], m[jt + 3]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 5], m[jt + 3]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 5], m[jt + 4]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 5], m[jt + 4]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 5], m[jt + 5]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 5], m[jt + 5]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 5], m[jt + 6]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 5], m[jt + 6]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 5], m[jt + 7]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 5], m[jt + 7]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 5], m[jt + 8]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 5], m[jt + 8]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 5], m[jt + 9]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 5], m[jt + 9]); + res[it + 6] = add64(res[it + 6], srli64(res[it + 5], DIGIT_SIZE)); + u[it + 6] = mul52lo(res[it + 6], k); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 6], m[jt + 0]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 6], m[jt + 0]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 6], m[jt + 1]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 6], m[jt + 1]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 6], m[jt + 2]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 6], m[jt + 2]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 6], m[jt + 3]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 6], m[jt + 3]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 6], m[jt + 4]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 6], m[jt + 4]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 6], m[jt + 5]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 6], m[jt + 5]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 6], m[jt + 6]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 6], m[jt + 6]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 6], m[jt + 7]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 6], m[jt + 7]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 6], m[jt + 8]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 6], m[jt + 8]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 6], m[jt + 9]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 6], m[jt + 9]); + res[it + 7] = add64(res[it + 7], srli64(res[it + 6], DIGIT_SIZE)); + u[it + 7] = mul52lo(res[it + 7], k); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 7], m[jt + 0]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 7], m[jt + 0]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 7], m[jt + 1]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 7], m[jt + 1]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 7], m[jt + 2]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 7], m[jt + 2]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 7], m[jt + 3]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 7], m[jt + 3]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 7], m[jt + 4]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 7], m[jt + 4]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 7], m[jt + 5]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 7], m[jt + 5]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 7], m[jt + 6]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 7], m[jt + 6]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 7], m[jt + 7]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 7], m[jt + 7]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 7], m[jt + 8]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 7], m[jt + 8]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 7], m[jt + 9]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 7], m[jt + 9]); + res[it + 8] = add64(res[it + 8], srli64(res[it + 7], DIGIT_SIZE)); + u[it + 8] = mul52lo(res[it + 8], k); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 8], m[jt + 0]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 8], m[jt + 0]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 8], m[jt + 1]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 8], m[jt + 1]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 8], m[jt + 2]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 8], m[jt + 2]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 8], m[jt + 3]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 8], m[jt + 3]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 8], m[jt + 4]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 8], m[jt + 4]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 8], m[jt + 5]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 8], m[jt + 5]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 8], m[jt + 6]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 8], m[jt + 6]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 8], m[jt + 7]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 8], m[jt + 7]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 8], m[jt + 8]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 8], m[jt + 8]); + res[it + jt + 17] = fma52lo(res[it + jt + 17], u[it + 8], m[jt + 9]); + res[it + jt + 18] = fma52hi(res[it + jt + 18], u[it + 8], m[jt + 9]); + res[it + 9] = add64(res[it + 9], srli64(res[it + 8], DIGIT_SIZE)); + u[it + 9] = mul52lo(res[it + 9], k); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 9], m[jt + 0]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 9], m[jt + 0]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 9], m[jt + 1]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 9], m[jt + 1]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 9], m[jt + 2]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 9], m[jt + 2]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 9], m[jt + 3]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 9], m[jt + 3]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 9], m[jt + 4]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 9], m[jt + 4]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 9], m[jt + 5]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 9], m[jt + 5]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 9], m[jt + 6]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 9], m[jt + 6]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 9], m[jt + 7]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 9], m[jt + 7]); + res[it + jt + 17] = fma52lo(res[it + jt + 17], u[it + 9], m[jt + 8]); + res[it + jt + 18] = fma52hi(res[it + jt + 18], u[it + 9], m[jt + 8]); + res[it + jt + 18] = fma52lo(res[it + jt + 18], u[it + 9], m[jt + 9]); + res[it + jt + 19] = fma52hi(res[it + jt + 19], u[it + 9], m[jt + 9]); + + for (jt = 10; jt < 40; jt += 10) { // Poly tile + res[it + jt + 0] = fma52lo(res[it + jt + 0], u[it + 0], m[jt + 0]); + res[it + jt + 1] = fma52hi(res[it + jt + 1], u[it + 0], m[jt + 0]); + res[it + jt + 1] = fma52lo(res[it + jt + 1], u[it + 0], m[jt + 1]); + res[it + jt + 2] = fma52hi(res[it + jt + 2], u[it + 0], m[jt + 1]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 0], m[jt + 2]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 0], m[jt + 2]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 0], m[jt + 3]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 0], m[jt + 3]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 0], m[jt + 4]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 0], m[jt + 4]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 0], m[jt + 5]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 0], m[jt + 5]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 0], m[jt + 6]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 0], m[jt + 6]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 0], m[jt + 7]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 0], m[jt + 7]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 0], m[jt + 8]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 0], m[jt + 8]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 0], m[jt + 9]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 0], m[jt + 9]); + res[it + jt + 1] = fma52lo(res[it + jt + 1], u[it + 1], m[jt + 0]); + res[it + jt + 2] = fma52hi(res[it + jt + 2], u[it + 1], m[jt + 0]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 1], m[jt + 1]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 1], m[jt + 1]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 1], m[jt + 2]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 1], m[jt + 2]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 1], m[jt + 3]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 1], m[jt + 3]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 1], m[jt + 4]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 1], m[jt + 4]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 1], m[jt + 5]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 1], m[jt + 5]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 1], m[jt + 6]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 1], m[jt + 6]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 1], m[jt + 7]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 1], m[jt + 7]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 1], m[jt + 8]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 1], m[jt + 8]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 1], m[jt + 9]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 1], m[jt + 9]); + res[it + jt + 2] = fma52lo(res[it + jt + 2], u[it + 2], m[jt + 0]); + res[it + jt + 3] = fma52hi(res[it + jt + 3], u[it + 2], m[jt + 0]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 2], m[jt + 1]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 2], m[jt + 1]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 2], m[jt + 2]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 2], m[jt + 2]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 2], m[jt + 3]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 2], m[jt + 3]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 2], m[jt + 4]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 2], m[jt + 4]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 2], m[jt + 5]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 2], m[jt + 5]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 2], m[jt + 6]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 2], m[jt + 6]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 2], m[jt + 7]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 2], m[jt + 7]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 2], m[jt + 8]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 2], m[jt + 8]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 2], m[jt + 9]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 2], m[jt + 9]); + res[it + jt + 3] = fma52lo(res[it + jt + 3], u[it + 3], m[jt + 0]); + res[it + jt + 4] = fma52hi(res[it + jt + 4], u[it + 3], m[jt + 0]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 3], m[jt + 1]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 3], m[jt + 1]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 3], m[jt + 2]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 3], m[jt + 2]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 3], m[jt + 3]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 3], m[jt + 3]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 3], m[jt + 4]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 3], m[jt + 4]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 3], m[jt + 5]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 3], m[jt + 5]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 3], m[jt + 6]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 3], m[jt + 6]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 3], m[jt + 7]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 3], m[jt + 7]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 3], m[jt + 8]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 3], m[jt + 8]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 3], m[jt + 9]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 3], m[jt + 9]); + res[it + jt + 4] = fma52lo(res[it + jt + 4], u[it + 4], m[jt + 0]); + res[it + jt + 5] = fma52hi(res[it + jt + 5], u[it + 4], m[jt + 0]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 4], m[jt + 1]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 4], m[jt + 1]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 4], m[jt + 2]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 4], m[jt + 2]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 4], m[jt + 3]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 4], m[jt + 3]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 4], m[jt + 4]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 4], m[jt + 4]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 4], m[jt + 5]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 4], m[jt + 5]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 4], m[jt + 6]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 4], m[jt + 6]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 4], m[jt + 7]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 4], m[jt + 7]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 4], m[jt + 8]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 4], m[jt + 8]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 4], m[jt + 9]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 4], m[jt + 9]); + res[it + jt + 5] = fma52lo(res[it + jt + 5], u[it + 5], m[jt + 0]); + res[it + jt + 6] = fma52hi(res[it + jt + 6], u[it + 5], m[jt + 0]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 5], m[jt + 1]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 5], m[jt + 1]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 5], m[jt + 2]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 5], m[jt + 2]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 5], m[jt + 3]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 5], m[jt + 3]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 5], m[jt + 4]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 5], m[jt + 4]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 5], m[jt + 5]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 5], m[jt + 5]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 5], m[jt + 6]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 5], m[jt + 6]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 5], m[jt + 7]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 5], m[jt + 7]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 5], m[jt + 8]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 5], m[jt + 8]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 5], m[jt + 9]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 5], m[jt + 9]); + res[it + jt + 6] = fma52lo(res[it + jt + 6], u[it + 6], m[jt + 0]); + res[it + jt + 7] = fma52hi(res[it + jt + 7], u[it + 6], m[jt + 0]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 6], m[jt + 1]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 6], m[jt + 1]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 6], m[jt + 2]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 6], m[jt + 2]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 6], m[jt + 3]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 6], m[jt + 3]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 6], m[jt + 4]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 6], m[jt + 4]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 6], m[jt + 5]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 6], m[jt + 5]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 6], m[jt + 6]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 6], m[jt + 6]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 6], m[jt + 7]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 6], m[jt + 7]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 6], m[jt + 8]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 6], m[jt + 8]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 6], m[jt + 9]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 6], m[jt + 9]); + res[it + jt + 7] = fma52lo(res[it + jt + 7], u[it + 7], m[jt + 0]); + res[it + jt + 8] = fma52hi(res[it + jt + 8], u[it + 7], m[jt + 0]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 7], m[jt + 1]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 7], m[jt + 1]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 7], m[jt + 2]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 7], m[jt + 2]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 7], m[jt + 3]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 7], m[jt + 3]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 7], m[jt + 4]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 7], m[jt + 4]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 7], m[jt + 5]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 7], m[jt + 5]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 7], m[jt + 6]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 7], m[jt + 6]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 7], m[jt + 7]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 7], m[jt + 7]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 7], m[jt + 8]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 7], m[jt + 8]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 7], m[jt + 9]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 7], m[jt + 9]); + res[it + jt + 8] = fma52lo(res[it + jt + 8], u[it + 8], m[jt + 0]); + res[it + jt + 9] = fma52hi(res[it + jt + 9], u[it + 8], m[jt + 0]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 8], m[jt + 1]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 8], m[jt + 1]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 8], m[jt + 2]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 8], m[jt + 2]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 8], m[jt + 3]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 8], m[jt + 3]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 8], m[jt + 4]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 8], m[jt + 4]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 8], m[jt + 5]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 8], m[jt + 5]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 8], m[jt + 6]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 8], m[jt + 6]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 8], m[jt + 7]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 8], m[jt + 7]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 8], m[jt + 8]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 8], m[jt + 8]); + res[it + jt + 17] = fma52lo(res[it + jt + 17], u[it + 8], m[jt + 9]); + res[it + jt + 18] = fma52hi(res[it + jt + 18], u[it + 8], m[jt + 9]); + res[it + jt + 9] = fma52lo(res[it + jt + 9], u[it + 9], m[jt + 0]); + res[it + jt + 10] = fma52hi(res[it + jt + 10], u[it + 9], m[jt + 0]); + res[it + jt + 10] = fma52lo(res[it + jt + 10], u[it + 9], m[jt + 1]); + res[it + jt + 11] = fma52hi(res[it + jt + 11], u[it + 9], m[jt + 1]); + res[it + jt + 11] = fma52lo(res[it + jt + 11], u[it + 9], m[jt + 2]); + res[it + jt + 12] = fma52hi(res[it + jt + 12], u[it + 9], m[jt + 2]); + res[it + jt + 12] = fma52lo(res[it + jt + 12], u[it + 9], m[jt + 3]); + res[it + jt + 13] = fma52hi(res[it + jt + 13], u[it + 9], m[jt + 3]); + res[it + jt + 13] = fma52lo(res[it + jt + 13], u[it + 9], m[jt + 4]); + res[it + jt + 14] = fma52hi(res[it + jt + 14], u[it + 9], m[jt + 4]); + res[it + jt + 14] = fma52lo(res[it + jt + 14], u[it + 9], m[jt + 5]); + res[it + jt + 15] = fma52hi(res[it + jt + 15], u[it + 9], m[jt + 5]); + res[it + jt + 15] = fma52lo(res[it + jt + 15], u[it + 9], m[jt + 6]); + res[it + jt + 16] = fma52hi(res[it + jt + 16], u[it + 9], m[jt + 6]); + res[it + jt + 16] = fma52lo(res[it + jt + 16], u[it + 9], m[jt + 7]); + res[it + jt + 17] = fma52hi(res[it + jt + 17], u[it + 9], m[jt + 7]); + res[it + jt + 17] = fma52lo(res[it + jt + 17], u[it + 9], m[jt + 8]); + res[it + jt + 18] = fma52hi(res[it + jt + 18], u[it + 9], m[jt + 8]); + res[it + jt + 18] = fma52lo(res[it + jt + 18], u[it + 9], m[jt + 9]); + res[it + jt + 19] = fma52hi(res[it + jt + 19], u[it + 9], m[jt + 9]); + } + } + + // Normalization + res[40] = add64(res[40], srli64(res[39], DIGIT_SIZE)); + r[0] = res[40]; + res[41] = add64(res[41], srli64(res[40], DIGIT_SIZE)); + r[1] = res[41]; + res[42] = add64(res[42], srli64(res[41], DIGIT_SIZE)); + r[2] = res[42]; + res[43] = add64(res[43], srli64(res[42], DIGIT_SIZE)); + r[3] = res[43]; + res[44] = add64(res[44], srli64(res[43], DIGIT_SIZE)); + r[4] = res[44]; + res[45] = add64(res[45], srli64(res[44], DIGIT_SIZE)); + r[5] = res[45]; + res[46] = add64(res[46], srli64(res[45], DIGIT_SIZE)); + r[6] = res[46]; + res[47] = add64(res[47], srli64(res[46], DIGIT_SIZE)); + r[7] = res[47]; + res[48] = add64(res[48], srli64(res[47], DIGIT_SIZE)); + r[8] = res[48]; + res[49] = add64(res[49], srli64(res[48], DIGIT_SIZE)); + r[9] = res[49]; + res[50] = add64(res[50], srli64(res[49], DIGIT_SIZE)); + r[10] = res[50]; + res[51] = add64(res[51], srli64(res[50], DIGIT_SIZE)); + r[11] = res[51]; + res[52] = add64(res[52], srli64(res[51], DIGIT_SIZE)); + r[12] = res[52]; + res[53] = add64(res[53], srli64(res[52], DIGIT_SIZE)); + r[13] = res[53]; + res[54] = add64(res[54], srli64(res[53], DIGIT_SIZE)); + r[14] = res[54]; + res[55] = add64(res[55], srli64(res[54], DIGIT_SIZE)); + r[15] = res[55]; + res[56] = add64(res[56], srli64(res[55], DIGIT_SIZE)); + r[16] = res[56]; + res[57] = add64(res[57], srli64(res[56], DIGIT_SIZE)); + r[17] = res[57]; + res[58] = add64(res[58], srli64(res[57], DIGIT_SIZE)); + r[18] = res[58]; + res[59] = add64(res[59], srli64(res[58], DIGIT_SIZE)); + r[19] = res[59]; + res[60] = add64(res[60], srli64(res[59], DIGIT_SIZE)); + r[20] = res[60]; + res[61] = add64(res[61], srli64(res[60], DIGIT_SIZE)); + r[21] = res[61]; + res[62] = add64(res[62], srli64(res[61], DIGIT_SIZE)); + r[22] = res[62]; + res[63] = add64(res[63], srli64(res[62], DIGIT_SIZE)); + r[23] = res[63]; + res[64] = add64(res[64], srli64(res[63], DIGIT_SIZE)); + r[24] = res[64]; + res[65] = add64(res[65], srli64(res[64], DIGIT_SIZE)); + r[25] = res[65]; + res[66] = add64(res[66], srli64(res[65], DIGIT_SIZE)); + r[26] = res[66]; + res[67] = add64(res[67], srli64(res[66], DIGIT_SIZE)); + r[27] = res[67]; + res[68] = add64(res[68], srli64(res[67], DIGIT_SIZE)); + r[28] = res[68]; + res[69] = add64(res[69], srli64(res[68], DIGIT_SIZE)); + r[29] = res[69]; + res[70] = add64(res[70], srli64(res[69], DIGIT_SIZE)); + r[30] = res[70]; + res[71] = add64(res[71], srli64(res[70], DIGIT_SIZE)); + r[31] = res[71]; + res[72] = add64(res[72], srli64(res[71], DIGIT_SIZE)); + r[32] = res[72]; + res[73] = add64(res[73], srli64(res[72], DIGIT_SIZE)); + r[33] = res[73]; + res[74] = add64(res[74], srli64(res[73], DIGIT_SIZE)); + r[34] = res[74]; + res[75] = add64(res[75], srli64(res[74], DIGIT_SIZE)); + r[35] = res[75]; + res[76] = add64(res[76], srli64(res[75], DIGIT_SIZE)); + r[36] = res[76]; + res[77] = add64(res[77], srli64(res[76], DIGIT_SIZE)); + r[37] = res[77]; + res[78] = add64(res[78], srli64(res[77], DIGIT_SIZE)); + r[38] = res[78]; + res[79] = add64(res[79], srli64(res[78], DIGIT_SIZE)); + r[39] = res[79]; + a = (U64 *)out_mb; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_extract_amm52x20_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_extract_amm52x20_mb8.c new file mode 100644 index 0000000..ad481b0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/avx512_primitives/ifma_extract_amm52x20_mb8.c @@ -0,0 +1,255 @@ +/******************************************************************************* + * Copyright (C) 2019 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include + +void ifma_extract_amm52x20_mb8(int64u *out_mb8, const int64u *inpA_mb8, + int64u MulTbl[][redLen][8], const int64u Idx[8], + const int64u *inpM_mb8, const int64u *k0_mb8) { + U64 res00, res01, res02, res03, res04, res05, res06, res07, res08, res09, + res10, res11, res12, res13, res14, res15, res16, res17, res18, res19; + U64 mulB00, mulB01, mulB02, mulB03, mulB04, mulB05, mulB06, mulB07, mulB08, + mulB09, mulB10, mulB11, mulB12, mulB13, mulB14, mulB15, mulB16, mulB17, + mulB18, mulB19; + U64 K = loadu64(k0_mb8); /* k0[] */ + __mmask8 k; + __ALIGN64 U64 inpB_mb8[20]; + int itr; + res00 = res01 = res02 = res03 = res04 = res05 = res06 = res07 = res08 = + res09 = res10 = res11 = res12 = res13 = res14 = res15 = res16 = res17 = + res18 = res19 = get_zero64(); + + U64 idx_target = loadu64((U64 *)Idx); + k = cmpeq64_mask(set64(1), idx_target); + mulB00 = loadu64(MulTbl[0][0]); + mulB01 = loadu64(MulTbl[0][1]); + mulB02 = loadu64(MulTbl[0][2]); + mulB03 = loadu64(MulTbl[0][3]); + mulB04 = loadu64(MulTbl[0][4]); + mulB05 = loadu64(MulTbl[0][5]); + mulB06 = loadu64(MulTbl[0][6]); + mulB07 = loadu64(MulTbl[0][7]); + mulB08 = loadu64(MulTbl[0][8]); + mulB09 = loadu64(MulTbl[0][9]); + mulB10 = loadu64(MulTbl[0][10]); + mulB11 = loadu64(MulTbl[0][11]); + mulB12 = loadu64(MulTbl[0][12]); + mulB13 = loadu64(MulTbl[0][13]); + mulB14 = loadu64(MulTbl[0][14]); + mulB15 = loadu64(MulTbl[0][15]); + mulB16 = loadu64(MulTbl[0][16]); + mulB17 = loadu64(MulTbl[0][17]); + mulB18 = loadu64(MulTbl[0][18]); + mulB19 = loadu64(MulTbl[0][19]); + for (itr = 1; itr < (1 << 5); ++itr) { + U64 idx_curr = set64(itr + 1); + __mmask8 k_new = cmpeq64_mask(idx_curr, idx_target); + mulB00 = select64(k, mulB00, (U64 *) MulTbl[itr][0]); + mulB01 = select64(k, mulB01, (U64 *) MulTbl[itr][1]); + mulB02 = select64(k, mulB02, (U64 *) MulTbl[itr][2]); + mulB03 = select64(k, mulB03, (U64 *) MulTbl[itr][3]); + mulB04 = select64(k, mulB04, (U64 *) MulTbl[itr][4]); + mulB05 = select64(k, mulB05, (U64 *) MulTbl[itr][5]); + mulB06 = select64(k, mulB06, (U64 *) MulTbl[itr][6]); + mulB07 = select64(k, mulB07, (U64 *) MulTbl[itr][7]); + mulB08 = select64(k, mulB08, (U64 *) MulTbl[itr][8]); + mulB09 = select64(k, mulB09, (U64 *) MulTbl[itr][9]); + mulB10 = select64(k, mulB10, (U64 *) MulTbl[itr][10]); + mulB11 = select64(k, mulB11, (U64 *) MulTbl[itr][11]); + mulB12 = select64(k, mulB12, (U64 *) MulTbl[itr][12]); + mulB13 = select64(k, mulB13, (U64 *) MulTbl[itr][13]); + mulB14 = select64(k, mulB14, (U64 *) MulTbl[itr][14]); + mulB15 = select64(k, mulB15, (U64 *) MulTbl[itr][15]); + mulB16 = select64(k, mulB16, (U64 *) MulTbl[itr][16]); + mulB17 = select64(k, mulB17, (U64 *) MulTbl[itr][17]); + mulB18 = select64(k, mulB18, (U64 *) MulTbl[itr][18]); + mulB19 = select64(k, mulB19, (U64 *) MulTbl[itr][19]); + k = k_new; + } + inpB_mb8[0] = mulB00; + inpB_mb8[1] = mulB01; + inpB_mb8[2] = mulB02; + inpB_mb8[3] = mulB03; + inpB_mb8[4] = mulB04; + inpB_mb8[5] = mulB05; + inpB_mb8[6] = mulB06; + inpB_mb8[7] = mulB07; + inpB_mb8[8] = mulB08; + inpB_mb8[9] = mulB09; + inpB_mb8[10] = mulB10; + inpB_mb8[11] = mulB11; + inpB_mb8[12] = mulB12; + inpB_mb8[13] = mulB13; + inpB_mb8[14] = mulB14; + inpB_mb8[15] = mulB15; + inpB_mb8[16] = mulB16; + inpB_mb8[17] = mulB17; + inpB_mb8[18] = mulB18; + inpB_mb8[19] = mulB19; + + for (itr = 0; itr < 20; itr++) { + U64 Yi; + U64 Bi = inpB_mb8[itr]; + fma52lo_mem(res00, res00, Bi, inpA_mb8, 64 * 0); + fma52lo_mem(res01, res01, Bi, inpA_mb8, 64 * 1); + fma52lo_mem(res02, res02, Bi, inpA_mb8, 64 * 2); + fma52lo_mem(res03, res03, Bi, inpA_mb8, 64 * 3); + fma52lo_mem(res04, res04, Bi, inpA_mb8, 64 * 4); + fma52lo_mem(res05, res05, Bi, inpA_mb8, 64 * 5); + fma52lo_mem(res06, res06, Bi, inpA_mb8, 64 * 6); + fma52lo_mem(res07, res07, Bi, inpA_mb8, 64 * 7); + fma52lo_mem(res08, res08, Bi, inpA_mb8, 64 * 8); + fma52lo_mem(res09, res09, Bi, inpA_mb8, 64 * 9); + fma52lo_mem(res10, res10, Bi, inpA_mb8, 64 * 10); + fma52lo_mem(res11, res11, Bi, inpA_mb8, 64 * 11); + fma52lo_mem(res12, res12, Bi, inpA_mb8, 64 * 12); + fma52lo_mem(res13, res13, Bi, inpA_mb8, 64 * 13); + fma52lo_mem(res14, res14, Bi, inpA_mb8, 64 * 14); + fma52lo_mem(res15, res15, Bi, inpA_mb8, 64 * 15); + fma52lo_mem(res16, res16, Bi, inpA_mb8, 64 * 16); + fma52lo_mem(res17, res17, Bi, inpA_mb8, 64 * 17); + fma52lo_mem(res18, res18, Bi, inpA_mb8, 64 * 18); + fma52lo_mem(res19, res19, Bi, inpA_mb8, 64 * 19); + Yi = fma52lo(get_zero64(), res00, K); + fma52lo_mem(res00, res00, Yi, inpM_mb8, 64 * 0); + fma52lo_mem(res01, res01, Yi, inpM_mb8, 64 * 1); + fma52lo_mem(res02, res02, Yi, inpM_mb8, 64 * 2); + fma52lo_mem(res03, res03, Yi, inpM_mb8, 64 * 3); + fma52lo_mem(res04, res04, Yi, inpM_mb8, 64 * 4); + fma52lo_mem(res05, res05, Yi, inpM_mb8, 64 * 5); + fma52lo_mem(res06, res06, Yi, inpM_mb8, 64 * 6); + fma52lo_mem(res07, res07, Yi, inpM_mb8, 64 * 7); + fma52lo_mem(res08, res08, Yi, inpM_mb8, 64 * 8); + fma52lo_mem(res09, res09, Yi, inpM_mb8, 64 * 9); + fma52lo_mem(res10, res10, Yi, inpM_mb8, 64 * 10); + fma52lo_mem(res11, res11, Yi, inpM_mb8, 64 * 11); + fma52lo_mem(res12, res12, Yi, inpM_mb8, 64 * 12); + fma52lo_mem(res13, res13, Yi, inpM_mb8, 64 * 13); + fma52lo_mem(res14, res14, Yi, inpM_mb8, 64 * 14); + fma52lo_mem(res15, res15, Yi, inpM_mb8, 64 * 15); + fma52lo_mem(res16, res16, Yi, inpM_mb8, 64 * 16); + fma52lo_mem(res17, res17, Yi, inpM_mb8, 64 * 17); + fma52lo_mem(res18, res18, Yi, inpM_mb8, 64 * 18); + fma52lo_mem(res19, res19, Yi, inpM_mb8, 64 * 19); + res00 = srli64(res00, DIGIT_SIZE); + res01 = add64(res01, res00); + fma52hi_mem(res00, res01, Bi, inpA_mb8, 64 * 0); + fma52hi_mem(res01, res02, Bi, inpA_mb8, 64 * 1); + fma52hi_mem(res02, res03, Bi, inpA_mb8, 64 * 2); + fma52hi_mem(res03, res04, Bi, inpA_mb8, 64 * 3); + fma52hi_mem(res04, res05, Bi, inpA_mb8, 64 * 4); + fma52hi_mem(res05, res06, Bi, inpA_mb8, 64 * 5); + fma52hi_mem(res06, res07, Bi, inpA_mb8, 64 * 6); + fma52hi_mem(res07, res08, Bi, inpA_mb8, 64 * 7); + fma52hi_mem(res08, res09, Bi, inpA_mb8, 64 * 8); + fma52hi_mem(res09, res10, Bi, inpA_mb8, 64 * 9); + fma52hi_mem(res10, res11, Bi, inpA_mb8, 64 * 10); + fma52hi_mem(res11, res12, Bi, inpA_mb8, 64 * 11); + fma52hi_mem(res12, res13, Bi, inpA_mb8, 64 * 12); + fma52hi_mem(res13, res14, Bi, inpA_mb8, 64 * 13); + fma52hi_mem(res14, res15, Bi, inpA_mb8, 64 * 14); + fma52hi_mem(res15, res16, Bi, inpA_mb8, 64 * 15); + fma52hi_mem(res16, res17, Bi, inpA_mb8, 64 * 16); + fma52hi_mem(res17, res18, Bi, inpA_mb8, 64 * 17); + fma52hi_mem(res18, res19, Bi, inpA_mb8, 64 * 18); + fma52hi_mem(res19, get_zero64(), Bi, inpA_mb8, 64 * 19); + fma52hi_mem(res00, res00, Yi, inpM_mb8, 64 * 0); + fma52hi_mem(res01, res01, Yi, inpM_mb8, 64 * 1); + fma52hi_mem(res02, res02, Yi, inpM_mb8, 64 * 2); + fma52hi_mem(res03, res03, Yi, inpM_mb8, 64 * 3); + fma52hi_mem(res04, res04, Yi, inpM_mb8, 64 * 4); + fma52hi_mem(res05, res05, Yi, inpM_mb8, 64 * 5); + fma52hi_mem(res06, res06, Yi, inpM_mb8, 64 * 6); + fma52hi_mem(res07, res07, Yi, inpM_mb8, 64 * 7); + fma52hi_mem(res08, res08, Yi, inpM_mb8, 64 * 8); + fma52hi_mem(res09, res09, Yi, inpM_mb8, 64 * 9); + fma52hi_mem(res10, res10, Yi, inpM_mb8, 64 * 10); + fma52hi_mem(res11, res11, Yi, inpM_mb8, 64 * 11); + fma52hi_mem(res12, res12, Yi, inpM_mb8, 64 * 12); + fma52hi_mem(res13, res13, Yi, inpM_mb8, 64 * 13); + fma52hi_mem(res14, res14, Yi, inpM_mb8, 64 * 14); + fma52hi_mem(res15, res15, Yi, inpM_mb8, 64 * 15); + fma52hi_mem(res16, res16, Yi, inpM_mb8, 64 * 16); + fma52hi_mem(res17, res17, Yi, inpM_mb8, 64 * 17); + fma52hi_mem(res18, res18, Yi, inpM_mb8, 64 * 18); + fma52hi_mem(res19, res19, Yi, inpM_mb8, 64 * 19); + } + // Normalization + { + U64 T = get_zero64(); + //U64 MASK = set64(DIGIT_MASK); + T = srli64(res00, DIGIT_SIZE); + storeu64(out_mb8 + 8 * 0, res00); + res01 = add64(res01, T); + T = srli64(res01, DIGIT_SIZE); + storeu64(out_mb8 + 8 * 1, res01); + res02 = add64(res02, T); + T = srli64(res02, DIGIT_SIZE); + storeu64(out_mb8 + 8 * 2, res02); + res03 = add64(res03, T); + T = srli64(res03, DIGIT_SIZE); + storeu64(out_mb8 + 8 * 3, res03); + res04 = add64(res04, T); + T = srli64(res04, DIGIT_SIZE); + storeu64(out_mb8 + 8 * 4, res04); + res05 = add64(res05, T); + T = srli64(res05, DIGIT_SIZE); + storeu64(out_mb8 + 8 * 5, res05); + res06 = add64(res06, T); + T = srli64(res06, DIGIT_SIZE); + storeu64(out_mb8 + 8 * 6, res06); + res07 = add64(res07, T); + T = srli64(res07, DIGIT_SIZE); + storeu64(out_mb8 + 8 * 7, res07); + res08 = add64(res08, T); + T = srli64(res08, DIGIT_SIZE); + storeu64(out_mb8 + 8 * 8, res08); + res09 = add64(res09, T); + T = srli64(res09, DIGIT_SIZE); + storeu64(out_mb8 + 8 * 9, res09); + res10 = add64(res10, T); + T = srli64(res10, DIGIT_SIZE); + storeu64(out_mb8 + 8 * 10, res10); + res11 = add64(res11, T); + T = srli64(res11, DIGIT_SIZE); + storeu64(out_mb8 + 8 * 11, res11); + res12 = add64(res12, T); + T = srli64(res12, DIGIT_SIZE); + storeu64(out_mb8 + 8 * 12, res12); + res13 = add64(res13, T); + T = srli64(res13, DIGIT_SIZE); + storeu64(out_mb8 + 8 * 13, res13); + res14 = add64(res14, T); + T = srli64(res14, DIGIT_SIZE); + storeu64(out_mb8 + 8 * 14, res14); + res15 = add64(res15, T); + T = srli64(res15, DIGIT_SIZE); + storeu64(out_mb8 + 8 * 15, res15); + res16 = add64(res16, T); + T = srli64(res16, DIGIT_SIZE); + storeu64(out_mb8 + 8 * 16, res16); + res17 = add64(res17, T); + T = srli64(res17, DIGIT_SIZE); + storeu64(out_mb8 + 8 * 17, res17); + res18 = add64(res18, T); + T = srli64(res18, DIGIT_SIZE); + storeu64(out_mb8 + 8 * 18, res18); + res19 = add64(res19, T); + storeu64(out_mb8 + 8 * 19, res19); + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_div_104_by_52.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_div_104_by_52.c new file mode 100644 index 0000000..842eb24 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_div_104_by_52.c @@ -0,0 +1,124 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + + +static ALIGNSPEC const UINT64 __mask52[] = { DUP8_DECL(0x00ffffffffffffffull) }; +static ALIGNSPEC const UINT64 __one[] = { DUP8_DECL(0x3ff0000000000000ull) }; +static ALIGNSPEC const UINT64 __two52[] = { DUP8_DECL(0x4330000000000000ull) }; +//static ALIGNSPEC const UINT64 __zero[] = { DUP8_DECL(0x0000000000000000ull) }; +static ALIGNSPEC const UINT64 __one_l[] = { DUP8_DECL(0x0000000000000001ull) }; + + + +// each 64-bit SIMD element should be in the [0, 2^52) range +// perform division of (Ah[k]*2^52 + Al[k])/B[k], k=0..7 +// Return quotients (as well as remainders via pointer) +// Note: There is no checking for B==0 +VUINT64 __div_104_by_52(VUINT64 Ah, VUINT64 Al, VUINT64 B, VUINT64* prem) +{ + VUINT64 msk52, Rem, one; + VUINT64 sgn_mask, B_corr, Qh, Ql; + VDOUBLE db, dah, dal, dbr, d_one; + VDOUBLE dqh, dql, two52, dqh2; + VMASK_L corr_gt_mask; + + // limit range of Ah, Al, B inputs to 52 bits (optional?) + msk52 = VLOAD_L(__mask52); + B = VAND_L(B, msk52); + Ah = VAND_L(Ah, msk52); + Al = VAND_L(Al, msk52); + + // convert inputs to double precision + // conversion will be exact + db = VCVT_L2D(B); + dah = VCVT_L2D(Ah); + dal = VCVT_L2D(Al); + + d_one = VLOAD_D(__one); + // get reciprocal of B, RZ mode + dbr = VDIV_RU_D(d_one, db); + + // estimate quotient Qh = (dah*dbr)_RZ ~ Ah/B + dqh = VMUL_RZ_D(dah, dbr); + // truncate dqh to integral + dqh = VROUND_RZ_D(dqh); + // get remainder term: dah_new -= dqh*dbh + // 0 <= dah_new <= B (due to rounding direction and rounding error magnitude in dbr, dqh) + dah = VQFMR_D(db, dqh, dah); + + // estimate quotient Ql = (dal*dbr)_RZ ~ Al/B + dql = VMUL_RZ_D(dal, dbr); + // truncate dqh to integral + dql = VROUND_RZ_D(dql); + // get remainder term: dal_new -= dql*dbh + // 0 <= dal_new <= B (similar to above) + dal = VQFMR_D(db, dql, dal); + // convert remainder term back to (signed) 64-bit integer + Al = VCVT_D2L(dal); + + // scale dah by 2^52 + two52 = VLOAD_D(__two52); + dah = VMUL_D(dah, two52); + // estimate quotient Qh2 = ((2^52)*dah)_RU/db + dqh2 = VMUL_RZ_D(dah, dbr); + // truncate dqh2 to integral + dqh2 = VROUND_RZ_D(dqh2); + // remainder term: dah_new2 = dah - db*dqh2 + // dah_new2 in (-B, 2B) + dah = VQFMR_D(db, dqh2, dah); + // convert remainder term back to (signed) 64-bit integer + Ah = VCVT_D2L(dah); + + // add and convert low quotients: Qh2+Ql + dql = VADD_D(dql, dqh2); + Ql = VCVT_D2L(dql); + // convert high quotient + Qh = VCVT_D2L(dqh); + + // remainder term in (-B, 3B) + Rem = VADD_L(Ah, Al); + + // Rem < 0? + sgn_mask = VSAR_L(Rem, 63); + // Rem > B? + corr_gt_mask = VCMP_GE_L(Rem, B); + B_corr = VAND_L(B, sgn_mask); + // apply correction if B<0 + Rem = VADD_L(Rem, B_corr); + Ql = VADD_L(Ql, sgn_mask); + + // if Rem> B, apply correction + Rem = VMSUB_L(corr_gt_mask, Rem, B); + one = VLOAD_L(__one_l); + Ql = VMADD_L(corr_gt_mask, Ql, one); + + // Rem > B? + corr_gt_mask = VCMP_GE_L(Rem, B); + // if Rem> B, apply correction + // this is the final remainder + *prem = VMSUB_L(corr_gt_mask, Rem, B); + Ql = VMADD_L(corr_gt_mask, Ql, one); + + // now add the high part of the quotient + Qh = VSHL_L(Qh, 52); + Ql = VADD_L(Ql, Qh); + + return Ql; + +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x10_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x10_mb8.c new file mode 100644 index 0000000..7baa2b1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x10_mb8.c @@ -0,0 +1,198 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + + +#include +#include + +#define USE_AMS +#ifdef USE_AMS + #define SQUARE_52x10_mb8(out, Y, mod, k0) \ + AMS52x10_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); + + #ifdef USE_AMS_5x + #define SQUARE_5x52x10_mb8(out, Y, mod, k0) \ + AMS5x52x10_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); + #else + #define SQUARE_5x52x10_mb8(out, Y, mod, k0) \ + AMS52x10_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + AMS52x10_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x10_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x10_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x10_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); + #endif +#else + #define SQUARE_52x10_mb8(out, Y, mod, k0) \ + ifma_amm52x10_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); + #define SQUARE_5x52x10_mb8(out, Y, mod, k0) \ + ifma_amm52x10_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x10_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x10_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x10_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x10_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); +#endif + +#define BITSIZE_MODULUS (512) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,DIGIT_SIZE)) //10 +#define LEN64 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,64)) //8 + +#define EXP_WIN_SIZE (5) //(4) +#define EXP_WIN_MASK ((1<=0; exp_bit_no-=EXP_WIN_SIZE) { + /* series of squaring */ + #if EXP_WIN_SIZE==5 + SQUARE_5x52x10_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + #else + SQUARE_52x10_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x10_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x10_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x10_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + #endif + + /* extract pre-computed multiplier from the table */ + { + __m512i T; + exp_chunk_no = exp_bit_no/64; + exp_chunk_shift = exp_bit_no%64; + + red_table_idx = _mm512_load_si512(expz[exp_chunk_no]); + T = _mm512_load_si512(expz[exp_chunk_no+1]); + + red_table_idx = _mm512_srl_epi64(red_table_idx, _mm_set1_epi64x(exp_chunk_shift)); + T = _mm512_sll_epi64(T, _mm_set1_epi64x(64-exp_chunk_shift)); + red_table_idx = _mm512_and_si512( _mm512_xor_si512(red_table_idx, T), table_idx_mask); + + extract_multiplier_mb8(red_X, red_table, (int64u*)(&red_table_idx)); + } + /* and multiply */ + ifma_amm52x10_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); + } + } + + /* clear exponents */ + zero_mb8(expz, LEN64); + + /* convert result back in regular 2^52 domain */ + zero_mb8(red_X, LEN52); + _mm512_store_si512(red_X, _mm512_set1_epi64(1)); + ifma_amm52x10_mb8((int64u*)out, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x20_65537_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x20_65537_mb8.c new file mode 100644 index 0000000..a3a95ed --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x20_65537_mb8.c @@ -0,0 +1,77 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + + +#define USE_AMS +#ifdef USE_AMS + #define SQUARE_52x20_mb8(out, Y, mod, k0) \ + AMS52x20_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); + + #ifdef USE_AMS_5x + #define SQUARE_5x52x20_mb8(out, Y, mod, k0) \ + AMS5x52x20_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); + #else + #define SQUARE_5x52x20_mb8(out, Y, mod, k0) \ + AMS52x20_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + AMS52x20_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x20_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x20_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x20_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); + #endif +#else + #define SQUARE_52x20_mb8(out, Y, mod, k0) \ + ifma_amm52x20_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); + #define SQUARE_5x52x20_mb8(out, Y, mod, k0) \ + ifma_amm52x20_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x20_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x20_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x20_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x20_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); +#endif + +#define BITSIZE_MODULUS (RSA_1K) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,DIGIT_SIZE)) //20 + +void EXP52x20_pub65537_mb8(int64u out[][8], + const int64u base[][8], // already in ifma fmt + const int64u modulus[][8], // already in ifma fmt + const int64u toMont[][8], // already in ifma fmt + const int64u k0[8], + int64u work_buffer[][8]) +{ + /* allocate red(undant) result Y and multiplier X */ + pint64u_x8 red_Y = (pint64u_x8)work_buffer; + pint64u_x8 red_X = (pint64u_x8)(red_Y + LEN52); + + /* convert base into redundant Montgomery domain */ + ifma_amm52x20_mb8((int64u*)red_X, (int64u*)base, (int64u*)toMont, (int64u*)modulus, (int64u*)k0); + + /* exponentition 65537 = 0x10001 */ + SQUARE_52x20_mb8((int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); + SQUARE_5x52x20_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_5x52x20_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_5x52x20_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + ifma_amm52x20_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); + + /* convert result back in regular 2^52 domain */ + zero_mb8(red_X, LEN52); + _mm512_store_si512(red_X, _mm512_set1_epi64(1)); + ifma_amm52x20_mb8((int64u*)out, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x20_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x20_mb8.c new file mode 100644 index 0000000..e7ab459 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x20_mb8.c @@ -0,0 +1,217 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + + +#include +#include + +#define USE_AMS +#ifdef USE_AMS + #define SQUARE_52x20_mb8(out, Y, mod, k0) \ + AMS52x20_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*) k0); +#else + #define SQUARE_52x20_mb8(out, Y, mod, k0) \ + ifma_amm52x20_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*) k0); +#endif + +#define BITSIZE_MODULUS (1024) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,DIGIT_SIZE)) //20 +#define LEN64 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,64)) //16 + +// only valid value is 4 +// not removed to make code more readable +#define EXP_WIN_SIZE (4) +#define EXP_WIN_MASK ((1<=0; exp_bit_no-=EXP_WIN_SIZE) { + __m512i T; + exp_chunk_no = exp_bit_no/64; + exp_chunk_shift = exp_bit_no%64; + + red_table_idx = _mm512_load_si512(expz[exp_chunk_no]); + T = _mm512_load_si512(expz[exp_chunk_no+1]); + + red_table_idx = _mm512_srl_epi64(red_table_idx, _mm_set1_epi64x(exp_chunk_shift)); + T = _mm512_sll_epi64(T, _mm_set1_epi64x(64-exp_chunk_shift)); + red_table_idx = _mm512_and_si512( _mm512_xor_si512(red_table_idx, T), table_idx_mask); + + /* 4 squarings stitched with extracting pre-computed multiplier from the table */ + AMS4x52x20_diagonal_stitched_with_extract_mb8((int64u*) red_Y, (U64*) mulb, (U64*) mulbx, (int64u*)red_Y, (int64u*) modulus, (int64u*) k0, red_table, redx_table, (int64u*) (&red_table_idx)); + + ifma_ahmm52x20_mb8((int64u*) red_Y, (int64u*) red_Y, (int64u*) mulb, (int64u*) mulbx, (int64u*) modulus, (int64u*) k0); + } + + /* clear exponents */ + zero_mb8(expz, LEN64); + + /* convert result back in regular 2^DIGIT_SIZE domain */ + zero_mb8(red_X, LEN52); + _mm512_store_si512(red_X, _mm512_set1_epi64(1)); + ifma_amm52x20_mb8((int64u*) out, (int64u*) red_Y, (int64u*) red_X, (int64u*) modulus, (int64u*) k0); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x30_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x30_mb8.c new file mode 100644 index 0000000..83c1c6d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x30_mb8.c @@ -0,0 +1,251 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#define USE_AMS +#ifdef USE_AMS + #define SQUARE_52x30_mb8(out, Y, mod, k0) \ + AMS52x30_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); + + #define SQUARE_5x52x30_mb8(out, Y, mod, k0) \ + ifma_amm52x30_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x30_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x30_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x30_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x30_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); +#else + #define SQUARE_52x30_mb8(out, Y, mod, k0) \ + ifma_amm52x30_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); + #define SQUARE_5x52x30_mb8(out, Y, mod, k0) \ + ifma_amm52x30_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x30_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x30_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x30_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x30_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); +#endif + +#define BITSIZE_MODULUS (1536) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,DIGIT_SIZE)) //30 +#define LEN64 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,64)) //24 + +#define EXP_WIN_SIZE (5) //(4) +#define EXP_WIN_MASK ((1<=0; exp_bit_no-=EXP_WIN_SIZE) { + /* series of squaring */ + #if EXP_WIN_SIZE==5 + SQUARE_5x52x30_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + #else + SQUARE_52x30_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x30_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x30_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x30_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + #endif + + /* extract pre-computed multiplier from the table */ + { + __m512i T; + exp_chunk_no = exp_bit_no/64; + exp_chunk_shift = exp_bit_no%64; + + red_table_idx = _mm512_load_si512(expz[exp_chunk_no]); + T = _mm512_load_si512(expz[exp_chunk_no+1]); + + red_table_idx = _mm512_srl_epi64(red_table_idx, _mm_set1_epi64x(exp_chunk_shift)); + T = _mm512_sll_epi64(T, _mm_set1_epi64x(64-exp_chunk_shift)); + red_table_idx = _mm512_and_si512( _mm512_xor_si512(red_table_idx, T), table_idx_mask); + + extract_multiplier_mb8(red_X, red_table, (int64u*)(&red_table_idx)); + } + /* and multiply */ + ifma_amm52x30_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); + } + } + + /* clear exponents */ + zero_mb8(expz, LEN64); + + /* convert result back in regular 2^52 domain */ + zero_mb8(red_X, LEN52); + _mm512_store_si512(red_X, _mm512_set1_epi64(1)); + ifma_amm52x30_mb8((int64u*)out, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x40_65537_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x40_65537_mb8.c new file mode 100644 index 0000000..d7955eb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x40_65537_mb8.c @@ -0,0 +1,77 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + + +#define USE_AMS +#ifdef USE_AMS + #define SQUARE_52x40_mb8(out, Y, mod, k0) \ + AMS52x40_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); + + #ifdef USE_AMS_5x + #define SQUARE_5x52x40_mb8(out, Y, mod, k0) \ + AMS5x52x40_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); + #else + #define SQUARE_5x52x40_mb8(out, Y, mod, k0) \ + AMS52x40_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + AMS52x40_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x40_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x40_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x40_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); + #endif +#else + #define SQUARE_52x40_mb8(out, Y, mod, k0) \ + ifma_amm52x40_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); + #define SQUARE_5x52x40_mb8(out, Y, mod, k0) \ + ifma_amm52x40_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x40_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x40_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x40_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x40_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); +#endif + +#define BITSIZE_MODULUS (RSA_2K) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,DIGIT_SIZE)) //40 + +void EXP52x40_pub65537_mb8(int64u out[][8], + const int64u base[][8], + const int64u modulus[][8], + const int64u toMont[][8], + const int64u k0[8], + int64u work_buffer[][8]) +{ + /* allocate red(undant) result Y and multiplier X */ + pint64u_x8 red_Y = (pint64u_x8)work_buffer; + pint64u_x8 red_X = (pint64u_x8)(work_buffer + LEN52); + + /* convert base into redundant domain */ + ifma_amm52x40_mb8((int64u*)red_X, (int64u*)base, (int64u*)toMont, (int64u*)modulus, (int64u*)k0); + + /* exponentition 65537 = 0x10001 */ + SQUARE_52x40_mb8((int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); + SQUARE_5x52x40_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_5x52x40_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_5x52x40_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + ifma_amm52x40_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); + + /* convert result back in regular 2^52 domain */ + zero_mb8(red_X, LEN52); + _mm512_store_si512(red_X, _mm512_set1_epi64(1)); + ifma_amm52x40_mb8((int64u*)out, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x40_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x40_mb8.c new file mode 100644 index 0000000..8e96cdf --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x40_mb8.c @@ -0,0 +1,287 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#define USE_AMS +#ifdef USE_AMS + #define SQUARE_52x40_mb8(out, Y, mod, k0) \ + AMS52x40_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); + + #ifdef USE_AMS_5x + #define SQUARE_5x52x40_mb8(out, Y, mod, k0) \ + AMS5x52x40_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); + #else + #define SQUARE_5x52x40_mb8(out, Y, mod, k0) \ + AMS52x40_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + AMS52x40_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x40_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x40_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x40_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); + #endif +#else + #define SQUARE_52x40_mb8(out, Y, mod, k0) \ + ifma_amm52x40_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); + #define SQUARE_5x52x40_mb8(out, Y, mod, k0) \ + ifma_amm52x40_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x40_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x40_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x40_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x40_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); +#endif + +#define BITSIZE_MODULUS (2048) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,DIGIT_SIZE)) //40 +#define LEN64 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,64)) //32 + +#define EXP_WIN_SIZE (5) //(4) +#define EXP_WIN_MASK ((1<=0; exp_bit_no-=EXP_WIN_SIZE) { + /* series of squaring */ + #if EXP_WIN_SIZE==5 + SQUARE_5x52x40_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + #else + SQUARE_52x40_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x40_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x40_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x40_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + #endif + + /* extract pre-computed multiplier from the table */ + { + __m512i T; + exp_chunk_no = exp_bit_no/64; + exp_chunk_shift = exp_bit_no%64; + + red_table_idx = _mm512_load_si512(expz[exp_chunk_no]); + T = _mm512_load_si512(expz[exp_chunk_no+1]); + + red_table_idx = _mm512_srl_epi64(red_table_idx, _mm_set1_epi64x(exp_chunk_shift)); + T = _mm512_sll_epi64(T, _mm_set1_epi64x(64-exp_chunk_shift)); + red_table_idx = _mm512_and_si512( _mm512_xor_si512(red_table_idx, T), table_idx_mask); + + extract_multiplier_mb8(red_X, red_table, (int64u*)(&red_table_idx)); + } + /* and multiply */ + ifma_amm52x40_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); + } + } + + /* clear exponents */ + zero_mb8(expz, LEN64); + + /* convert result back in regular 2^52 domain */ + zero_mb8(red_X, LEN52); + _mm512_store_si512(red_X, _mm512_set1_epi64(1)); + ifma_amm52x40_mb8((int64u*)out, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x60_65537_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x60_65537_mb8.c new file mode 100644 index 0000000..7089b52 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x60_65537_mb8.c @@ -0,0 +1,72 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + + +#define USE_AMS +#ifdef USE_AMS + #define SQUARE_52x60_mb8(out, Y, mod, k0) \ + AMS52x60_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); + + #define SQUARE_5x52x60_mb8(out, Y, mod, k0) \ + AMS52x60_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + AMS52x60_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x60_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x60_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x60_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); +#else + #define SQUARE_52x60_mb8(out, Y, mod, k0) \ + ifma_amm52x60_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); + #define SQUARE_5x52x60_mb8(out, Y, mod, k0) \ + ifma_amm52x60_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x60_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x60_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x60_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x60_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); +#endif + +#define BITSIZE_MODULUS (RSA_3K) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,DIGIT_SIZE)) //60 + +void EXP52x60_pub65537_mb8(int64u out[][8], + const int64u base[][8], + const int64u modulus[][8], + const int64u toMont[][8], + const int64u k0[8], + int64u work_buffer[][8]) +{ + /* allocate red(undant) result Y and multiplier X */ + pint64u_x8 red_Y = (pint64u_x8)work_buffer; + pint64u_x8 red_X = (pint64u_x8)(work_buffer + LEN52); + + /* convert base into redundant domain */ + ifma_amm52x60_mb8((int64u*)red_X, (int64u*)base, (int64u*)toMont, (int64u*)modulus, (int64u*)k0); + + /* exponentition 65537 = 0x10001 */ + SQUARE_52x60_mb8((int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); + SQUARE_5x52x60_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_5x52x60_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_5x52x60_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + ifma_amm52x60_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); + + /* convert result back in regular 2^52 domain */ + zero_mb8(red_X, LEN52); + _mm512_store_si512(red_X, _mm512_set1_epi64(1)); + ifma_amm52x60_mb8((int64u*)out, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x60_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x60_mb8.c new file mode 100644 index 0000000..0aa5752 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x60_mb8.c @@ -0,0 +1,342 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#define USE_AMS +#ifdef USE_AMS + #define SQUARE_52x60_mb8(out, Y, mod, k0) \ + AMS52x60_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); + + #define SQUARE_5x52x60_mb8(out, Y, mod, k0) \ + AMS52x60_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + AMS52x60_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x60_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x60_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x60_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); +#else + #define SQUARE_52x60_mb8(out, Y, mod, k0) \ + ifma_amm52x60_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); + #define SQUARE_5x52x60_mb8(out, Y, mod, k0) \ + ifma_amm52x60_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x60_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x60_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x60_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x60_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); +#endif + +#define BITSIZE_MODULUS (3072) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,DIGIT_SIZE)) //60 +#define LEN64 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,64)) //48 + +#define EXP_WIN_SIZE (5) //(4) +#define EXP_WIN_MASK ((1<=0; exp_bit_no-=EXP_WIN_SIZE) { + /* series of squaring */ + #if EXP_WIN_SIZE==5 + SQUARE_5x52x60_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + #else + SQUARE_52x60_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x60_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x60_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x60_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + #endif + + /* extract pre-computed multiplier from the table */ + { + __m512i T; + exp_chunk_no = exp_bit_no/64; + exp_chunk_shift = exp_bit_no%64; + + red_table_idx = _mm512_load_si512(expz[exp_chunk_no]); + T = _mm512_load_si512(expz[exp_chunk_no+1]); + + red_table_idx = _mm512_srl_epi64(red_table_idx, _mm_set1_epi64x(exp_chunk_shift)); + T = _mm512_sll_epi64(T, _mm_set1_epi64x(64-exp_chunk_shift)); + red_table_idx = _mm512_and_si512( _mm512_xor_si512(red_table_idx, T), table_idx_mask); + + extract_multiplier_mb8(red_X, red_table, (int64u*)(&red_table_idx)); + } + /* and multiply */ + ifma_amm52x60_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); + } + } + + /* clear exponents */ + zero_mb8(expz, LEN64); + + /* convert result back in regular 2^52 domain */ + zero_mb8(red_X, LEN52); + _mm512_store_si512(red_X, _mm512_set1_epi64(1)); + ifma_amm52x60_mb8((int64u*)out, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x79_65537_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x79_65537_mb8.c new file mode 100644 index 0000000..c9b9a70 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x79_65537_mb8.c @@ -0,0 +1,74 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + + +#define USE_AMS +#ifdef USE_AMS + #define SQUARE_52x79_mb8(out, Y, mod, k0) \ + AMS52x79_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); + + #define SQUARE_5x52x79_mb8(out, Y, mod, k0) \ + AMS52x79_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + AMS52x79_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x79_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x79_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x79_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); +#else + #define SQUARE_52x79_mb8(out, Y, mod, k0) \ + ifma_amm52x79_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); + #define SQUARE_5x52x79_mb8(out, Y, mod, k0) \ + ifma_amm52x79_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x79_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x79_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x79_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x79_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); +#endif + +#define BITSIZE_MODULUS (RSA_4K) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,DIGIT_SIZE)) //79 + +void EXP52x79_pub65537_mb8(int64u out[][8], + const int64u base[][8], + const int64u modulus[][8], + const int64u toMont[][8], + const int64u k0[8], + int64u work_buffer[][8]) +{ + /* allocate red(undant) result Y and multiplier X */ + pint64u_x8 red_Y = (pint64u_x8)(work_buffer + 1); + pint64u_x8 red_X = (pint64u_x8)(red_Y + LEN52); + + /* convert base into redundant domain */ + zero_mb8(red_X, LEN52); + ifma_amm52x79_mb8((int64u*)red_X, (int64u*)base, (int64u*)toMont, (int64u*)modulus, (int64u*)k0); + + /* exponentition 65537 = 0x10001 */ + SQUARE_52x79_mb8((int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); + SQUARE_5x52x79_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_5x52x79_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_5x52x79_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + ifma_amm52x79_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); + + /* convert result back in regular 2^52 domain */ + zero_mb8(red_X, LEN52); + _mm512_store_si512(red_X, _mm512_set1_epi64(1)); + ifma_amm52x79_mb8((int64u*)out, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); +} + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x79_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x79_mb8.c new file mode 100644 index 0000000..99b18b7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_exp52x79_mb8.c @@ -0,0 +1,398 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#define USE_AMS +#ifdef USE_AMS + #define SQUARE_52x79_mb8(out, Y, mod, k0) \ + AMS52x79_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); + + #define SQUARE_5x52x79_mb8(out, Y, mod, k0) \ + AMS52x79_diagonal_mb8((int64u*)out, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + AMS52x79_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x79_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x79_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + AMS52x79_diagonal_mb8((int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); +#else + #define SQUARE_52x79_mb8(out, Y, mod, k0) \ + ifma_amm52x79_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); + #define SQUARE_5x52x79_mb8(out, Y, mod, k0) \ + ifma_amm52x79_mb8((int64u*)out, (int64u*)Y, (int64u*)Y, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x79_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x79_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x79_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); \ + ifma_amm52x79_mb8((int64u*)out, (int64u*)out, (int64u*)out, (int64u*)mod, (int64u*)k0); +#endif + +#define BITSIZE_MODULUS (4096) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,DIGIT_SIZE)) //79 +#define LEN64 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,64)) //64 + +#define EXP_WIN_SIZE (5) //(4) +#define EXP_WIN_MASK ((1<=0; exp_bit_no-=EXP_WIN_SIZE) { + /* series of squaring */ + #if EXP_WIN_SIZE==5 + SQUARE_5x52x79_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + #else + SQUARE_52x79_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x79_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x79_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + SQUARE_52x79_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)modulus, (int64u*)k0); + #endif + + /* extract pre-computed multiplier from the table */ + { + __m512i T; + exp_chunk_no = exp_bit_no/64; + exp_chunk_shift = exp_bit_no%64; + + red_table_idx = _mm512_load_si512(expz[exp_chunk_no]); + T = _mm512_load_si512(expz[exp_chunk_no+1]); + + red_table_idx = _mm512_srl_epi64(red_table_idx, _mm_set1_epi64x(exp_chunk_shift)); + T = _mm512_sll_epi64(T, _mm_set1_epi64x(64-exp_chunk_shift)); + red_table_idx = _mm512_and_si512( _mm512_xor_si512(red_table_idx, T), table_idx_mask); + + extract_multiplier_mb8(red_X, red_table, (int64u*)(&red_table_idx)); + } + /* and multiply */ + ifma_amm52x79_mb8((int64u*)red_Y, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); + } + } + + /* clear exponents */ + zero_mb8(expz, LEN64); + + /* convert result back in regular 2^52 domain */ + zero_mb8(red_X, LEN52); + _mm512_store_si512(red_X, _mm512_set1_epi64(1)); + ifma_amm52x79_mb8((int64u*)out, (int64u*)red_Y, (int64u*)red_X, (int64u*)modulus, (int64u*)k0); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_method.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_method.c new file mode 100644 index 0000000..d1992ee --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_method.c @@ -0,0 +1,377 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include +#include +#include + +#define EXP_WIN_SIZE (5) //(4) + +/* +// rsa public key methods +*/ +DLL_PUBLIC +const mbx_RSA_Method* mbx_RSA1K_pub65537_Method(void) +{ + #define RSA_BITLEN (RSA_1K) + #define LEN52 (NUMBER_OF_DIGITS(RSA_BITLEN, DIGIT_SIZE)) + static mbx_RSA_Method m = { + RSA_ID(RSA_PUB_KEY,RSA_BITLEN), + RSA_BITLEN, + 64 + (8 + LEN52*8 + LEN52*8 + MULTIPLE_OF(LEN52,10)*8 + (LEN52*8)*2) * sizeof(int64u), /* buffer */ + //ifma_BNU_to_mb8, + //NULL, + EXP52x20_pub65537_mb8, + NULL, + NULL, + NULL, + NULL, + NULL + }; + return &m; + #undef RSA_BITLEN + #undef LEN52 +} + +DLL_PUBLIC +const mbx_RSA_Method* mbx_RSA2K_pub65537_Method(void) +{ +#define RSA_BITLEN (RSA_2K) +#define LEN52 (NUMBER_OF_DIGITS(RSA_BITLEN, DIGIT_SIZE)) + static mbx_RSA_Method m = { + RSA_ID(RSA_PUB_KEY,RSA_BITLEN), + RSA_BITLEN, + 64 + (8 + LEN52*8 + LEN52*8 + MULTIPLE_OF(LEN52,10)*8 + (LEN52*8)*2) * sizeof(int64u), /* buffer */ + //ifma_BNU_to_mb8, + //NULL, + EXP52x40_pub65537_mb8, + NULL, + NULL, + NULL, + NULL, + NULL + }; + return &m; +#undef RSA_BITLEN +#undef LEN52 +} + +DLL_PUBLIC +const mbx_RSA_Method* mbx_RSA3K_pub65537_Method(void) +{ +#define RSA_BITLEN (RSA_3K) +#define LEN52 (NUMBER_OF_DIGITS(RSA_BITLEN, DIGIT_SIZE)) + static mbx_RSA_Method m = { + RSA_ID(RSA_PUB_KEY,RSA_BITLEN), + RSA_BITLEN, + 64 + (8 + LEN52*8 + LEN52*8 + MULTIPLE_OF(LEN52,10)*8 + (LEN52*8)*2) * sizeof(int64u), /* buffer */ + //ifma_BNU_to_mb8, + //NULL, + EXP52x60_pub65537_mb8, + NULL, + NULL, + NULL, + NULL, + NULL + }; + return &m; +#undef RSA_BITLEN +#undef LEN52 +} + +DLL_PUBLIC +const mbx_RSA_Method* mbx_RSA4K_pub65537_Method(void) +{ +#define RSA_BITLEN (RSA_4K) +#define LEN52 (NUMBER_OF_DIGITS(RSA_BITLEN, DIGIT_SIZE)) + static mbx_RSA_Method m = { + RSA_ID(RSA_PUB_KEY,RSA_BITLEN), + RSA_BITLEN, + 64 + (8 + LEN52*8 + LEN52*8 + MULTIPLE_OF(LEN52,10)*8 + (LEN52*8)*2 + 1) * sizeof(int64u), /* buffer */ + //ifma_BNU_to_mb8, + //NULL, + EXP52x79_pub65537_mb8, + NULL, + NULL, + NULL, + NULL, + NULL + }; + return &m; +#undef RSA_BITLEN +#undef LEN52 +} + +DLL_PUBLIC +const mbx_RSA_Method* mbx_RSA_pub65537_Method(int rsaBitsize) +{ + switch (rsaBitsize) { + case RSA_1K: return mbx_RSA1K_pub65537_Method(); + case RSA_2K: return mbx_RSA2K_pub65537_Method(); + case RSA_3K: return mbx_RSA3K_pub65537_Method(); + case RSA_4K: return mbx_RSA4K_pub65537_Method(); + default: return NULL; + } +} + + +/* +// rsa private key methods +*/ +DLL_PUBLIC +const mbx_RSA_Method* mbx_RSA1K_private_Method(void) +{ + #define RSA_BITLEN (RSA_1K) + #define LEN52 (NUMBER_OF_DIGITS(RSA_BITLEN, DIGIT_SIZE)) + #define LEN64 (NUMBER_OF_DIGITS(RSA_BITLEN, 64)) + static mbx_RSA_Method m = { + RSA_ID(RSA_PRV2_KEY,RSA_BITLEN), + RSA_BITLEN, + 64 + (8 + LEN64*8 + LEN52*8 + LEN52*8 + MULTIPLE_OF(LEN52,10)*8 + (LEN52*8)*2 + (LEN64+1)*8 + (1<buffSize : 0; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_other52x_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_other52x_mb8.c new file mode 100644 index 0000000..5bf5a51 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_other52x_mb8.c @@ -0,0 +1,2399 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include +#include + +#ifdef __GNUC__ +#define ASM(a) __asm__(a); +#else +#define ASM(a) +#endif + +__NOINLINE +void zero_mb8(int64u (*out)[8], int len) +{ +#if defined(__GNUC__) + // Avoid dead code elimination for GNU compilers + ASM(""); +#endif + __m512i T = _mm512_setzero_si512(); + int i; + for(i=0; i x < ((m0*y) mod (2*x)) */ + y = _mm512_mask_add_epi64(y, k, y,x); /* y+=x */ + /* mask = 2*x-1 */ + x = nx; + nx = _mm512_add_epi64(nx, nx); + mask = _mm512_sub_epi64(nx, _mm512_set1_epi64(1)); + } + y = _mm512_sub_epi64(_mm512_setzero_si512(), y); /* return (0-y) */ + y = _mm512_and_si512(y, _mm512_set1_epi64(DIGIT_MASK)); + + _mm512_storeu_si512(k0_mb8, y); +} + +/* r = (a-b) mod m */ +void ifma_modsub52x10_mb8(int64u res[][8], const int64u inpA[][8], const int64u inpB[][8], const int64u inpM[][8]) +{ +#define BITSIZE (512) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE,52)) +#define MSD_MSK (MS_DIGIT_MASK(BITSIZE,52)) + + __m512i* pr = (__m512i*)res; + __m512i* pa = (__m512i*)inpA; + __m512i* pb = (__m512i*)inpB; + __m512i* pm = (__m512i*)inpM; + + __m512i DIG_MASK = _mm512_set1_epi64(DIGIT_MASK); + __m512i MSD_MASK = _mm512_set1_epi64(MSD_MSK); + + __m512i T[LEN52]; + __m512i R, CARRY, BORROW; + int n; + + /* T[] = a[] - b[] */ + BORROW = _mm512_setzero_si512(); + + for (n = 0; n < LEN52; n++) { + R = _mm512_sub_epi64(pa[n], pb[n]); + R = _mm512_sub_epi64(R, BORROW); + BORROW = _mm512_srli_epi64(R, (64 - 1)); + R = _mm512_and_epi64(R, DIG_MASK); + _mm512_store_si512(T + n, R); + } + + /* correct last digit */ + R = _mm512_and_epi64(R, MSD_MASK); + _mm512_store_si512(T + LEN52 - 1, R); + + /* masked modulus add: r[] = T[] + BORROW? m[] : 0 */ + CARRY = _mm512_setzero_si512(); + BORROW = _mm512_sub_epi64(CARRY, BORROW); /* RORROW -> mask */ + + for (n = 0; n < LEN52; n++) { + R = _mm512_and_epi64(BORROW, pm[n]); + R = _mm512_add_epi64(R, T[n]); + R = _mm512_add_epi64(R, CARRY); + CARRY = _mm512_srli_epi64(R, DIGIT_SIZE); + R = _mm512_and_epi64(R, DIG_MASK); + _mm512_store_si512(pr + n, R); + } + /* correct last digit */ + R = _mm512_and_epi64(R, MSD_MASK); + _mm512_store_si512(pr + LEN52 - 1, R); + +#undef BITSIZE +#undef LEN52 +#undef MSD_MSK +} + +void ifma_modsub52x20_mb8(int64u res[][8], const int64u inpA[][8], const int64u inpB[][8], const int64u inpM[][8]) +{ +#define BITSIZE (1024) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE,52)) +#define MSD_MSK (MS_DIGIT_MASK(BITSIZE,52)) + + __m512i* pr = (__m512i*)res; + __m512i* pa = (__m512i*)inpA; + __m512i* pb = (__m512i*)inpB; + __m512i* pm = (__m512i*)inpM; + + __m512i DIG_MASK = _mm512_set1_epi64(DIGIT_MASK); + __m512i MSD_MASK = _mm512_set1_epi64(MSD_MSK); + + __m512i T[LEN52]; + __m512i R, CARRY, BORROW; + int n; + + /* T[] = a[] - b[] */ + BORROW = _mm512_setzero_si512(); + + for (n = 0; n < LEN52; n++) { + R = _mm512_sub_epi64(pa[n], pb[n]); + R = _mm512_sub_epi64(R, BORROW); + BORROW = _mm512_srli_epi64(R, (64 - 1)); + R = _mm512_and_epi64(R, DIG_MASK); + _mm512_store_si512(T + n, R); + } + + /* correct last digit */ + R = _mm512_and_epi64(R, MSD_MASK); + _mm512_store_si512(T + LEN52 - 1, R); + + /* masked modulus add: r[] = T[] + BORROW? m[] : 0 */ + CARRY = _mm512_setzero_si512(); + BORROW = _mm512_sub_epi64(CARRY, BORROW); /* RORROW -> mask */ + + for (n = 0; n < LEN52; n++) { + R = _mm512_and_epi64(BORROW, pm[n]); + R = _mm512_add_epi64(R, T[n]); + R = _mm512_add_epi64(R, CARRY); + CARRY = _mm512_srli_epi64(R, DIGIT_SIZE); + R = _mm512_and_epi64(R, DIG_MASK); + _mm512_store_si512(pr + n, R); + } + /* correct last digit */ + R = _mm512_and_epi64(R, MSD_MASK); + _mm512_store_si512(pr + LEN52 - 1, R); + + #undef BITSIZE + #undef LEN52 + #undef MSD_MSK +} + +void ifma_modsub52x30_mb8(int64u res[][8], const int64u inpA[][8], const int64u inpB[][8], const int64u inpM[][8]) +{ + #define BITSIZE (1536) + #define LEN52 (NUMBER_OF_DIGITS(BITSIZE,52)) + #define MSD_MSK (MS_DIGIT_MASK(BITSIZE,52)) + + __m512i* pr = (__m512i*)res; + __m512i* pa = (__m512i*)inpA; + __m512i* pb = (__m512i*)inpB; + __m512i* pm = (__m512i*)inpM; + + __m512i DIG_MASK = _mm512_set1_epi64(DIGIT_MASK); + __m512i MSD_MASK = _mm512_set1_epi64(MSD_MSK); + + __m512i T[LEN52]; + __m512i R, CARRY, BORROW; + int n; + + /* T[] = a[] - b[] */ + BORROW = _mm512_setzero_si512(); + + for (n = 0; n < LEN52; n++) { + R = _mm512_sub_epi64(pa[n], pb[n]); + R = _mm512_sub_epi64(R, BORROW); + BORROW = _mm512_srli_epi64(R, (64 - 1)); + R = _mm512_and_epi64(R, DIG_MASK); + _mm512_store_si512(T + n, R); + } + + /* correct last digit */ + R = _mm512_and_epi64(R, MSD_MASK); + _mm512_store_si512(T + LEN52 - 1, R); + + /* masked modulus add: r[] = T[] + BORROW? m[] : 0 */ + CARRY = _mm512_setzero_si512(); + BORROW = _mm512_sub_epi64(CARRY, BORROW); /* RORROW -> mask */ + + for (n = 0; n < LEN52; n++) { + R = _mm512_and_epi64(BORROW, pm[n]); + R = _mm512_add_epi64(R, T[n]); + R = _mm512_add_epi64(R, CARRY); + CARRY = _mm512_srli_epi64(R, DIGIT_SIZE); + R = _mm512_and_epi64(R, DIG_MASK); + _mm512_store_si512(pr + n, R); + } + /* correct last digit */ + R = _mm512_and_epi64(R, MSD_MASK); + _mm512_store_si512(pr + LEN52 - 1, R); + + #undef BITSIZE + #undef LEN52 + #undef MSD_MSK +} + +void ifma_modsub52x40_mb8(int64u res[][8], const int64u inpA[][8], const int64u inpB[][8], const int64u inpM[][8]) +{ +#define BITSIZE (2048) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE,52)) +#define MSD_MSK (MS_DIGIT_MASK(BITSIZE,52)) + + __m512i* pr = (__m512i*)res; + __m512i* pa = (__m512i*)inpA; + __m512i* pb = (__m512i*)inpB; + __m512i* pm = (__m512i*)inpM; + + __m512i DIG_MASK = _mm512_set1_epi64(DIGIT_MASK); + __m512i MSD_MASK = _mm512_set1_epi64(MSD_MSK); + + __m512i T[LEN52]; + __m512i R, CARRY, BORROW; + int n; + + /* T[] = a[] - b[] */ + BORROW = _mm512_setzero_si512(); + + for (n = 0; n < LEN52; n++) { + R = _mm512_sub_epi64(pa[n], pb[n]); + R = _mm512_sub_epi64(R, BORROW); + BORROW = _mm512_srli_epi64(R, (64 - 1)); + R = _mm512_and_epi64(R, DIG_MASK); + _mm512_store_si512(T + n, R); + } + + /* correct last digit */ + R = _mm512_and_epi64(R, MSD_MASK); + _mm512_store_si512(T + LEN52 - 1, R); + + /* masked modulus add: r[] = T[] + BORROW? m[] : 0 */ + CARRY = _mm512_setzero_si512(); + BORROW = _mm512_sub_epi64(CARRY, BORROW); /* RORROW -> mask */ + + for (n = 0; n < LEN52; n++) { + R = _mm512_and_epi64(BORROW, pm[n]); + R = _mm512_add_epi64(R, T[n]); + R = _mm512_add_epi64(R, CARRY); + CARRY = _mm512_srli_epi64(R, DIGIT_SIZE); + R = _mm512_and_epi64(R, DIG_MASK); + _mm512_store_si512(pr + n, R); + } + /* correct last digit */ + R = _mm512_and_epi64(R, MSD_MASK); + _mm512_store_si512(pr + LEN52 - 1, R); + +#undef BITSIZE +#undef LEN52 +#undef MSD_MSK +} + + +/* r += a*b */ +void ifma_addmul52x10_mb8(int64u pRes[][8], const int64u inpA[][8], const int64u inpB[][8]) +{ +#define BITSIZE (RSA_1K/2) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE,52)) +#define MSD_MSK (MS_DIGIT_MASK(RSA_2K,52)) + + __m512i* pr = (__m512i*)pRes; + __m512i* pa = (__m512i*)inpA; + __m512i* pb = (__m512i*)inpB; + + __m512i DIG_MASK = _mm512_set1_epi64(DIGIT_MASK); + + __m512i R00 = _mm512_load_si512(pr + 0); /* load pr[nsM-1],...,R[0] */ + __m512i R01 = _mm512_load_si512(pr + 1); + __m512i R02 = _mm512_load_si512(pr + 2); + __m512i R03 = _mm512_load_si512(pr + 3); + __m512i R04 = _mm512_load_si512(pr + 4); + __m512i R05 = _mm512_load_si512(pr + 5); + __m512i R06 = _mm512_load_si512(pr + 6); + __m512i R07 = _mm512_load_si512(pr + 7); + __m512i R08 = _mm512_load_si512(pr + 8); + __m512i R09 = _mm512_load_si512(pr + 9); + + int itr; + for (itr = 0; itr < LEN52; itr++) { + __m512i Bi = _mm512_load_si512(pb); + __m512i nxtR = _mm512_load_si512(pr + LEN52); + pb++; + + _mm512_madd52lo_epu64_(R00, R00, Bi, pa, 64 * 0); + _mm512_madd52lo_epu64_(R01, R01, Bi, pa, 64 * 1); + _mm512_madd52lo_epu64_(R02, R02, Bi, pa, 64 * 2); + _mm512_madd52lo_epu64_(R03, R03, Bi, pa, 64 * 3); + _mm512_madd52lo_epu64_(R04, R04, Bi, pa, 64 * 4); + _mm512_madd52lo_epu64_(R05, R05, Bi, pa, 64 * 5); + _mm512_madd52lo_epu64_(R06, R06, Bi, pa, 64 * 6); + _mm512_madd52lo_epu64_(R07, R07, Bi, pa, 64 * 7); + _mm512_madd52lo_epu64_(R08, R08, Bi, pa, 64 * 8); + _mm512_madd52lo_epu64_(R09, R09, Bi, pa, 64 * 9); + + _mm512_store_si512(pr, _mm512_and_epi64(R00, DIG_MASK)); /* store normalized result */ + pr++; + + R00 = _mm512_srli_epi64(R00, DIGIT_SIZE); + R01 = _mm512_add_epi64(R01, R00); + + _mm512_madd52hi_epu64_(R00, R01, Bi, pa, 64 * 0); + _mm512_madd52hi_epu64_(R01, R02, Bi, pa, 64 * 1); + _mm512_madd52hi_epu64_(R02, R03, Bi, pa, 64 * 2); + _mm512_madd52hi_epu64_(R03, R04, Bi, pa, 64 * 3); + _mm512_madd52hi_epu64_(R04, R05, Bi, pa, 64 * 4); + _mm512_madd52hi_epu64_(R05, R06, Bi, pa, 64 * 5); + _mm512_madd52hi_epu64_(R06, R07, Bi, pa, 64 * 6); + _mm512_madd52hi_epu64_(R07, R08, Bi, pa, 64 * 7); + _mm512_madd52hi_epu64_(R08, R09, Bi, pa, 64 * 8); + _mm512_madd52hi_epu64_(R09, nxtR, Bi, pa, 64 * 9); + } + /* normalization */ + { + __m512i + T = _mm512_srli_epi64(R00, DIGIT_SIZE); + R00 = _mm512_and_epi64(R00, DIG_MASK); + _mm512_store_si512(pr + 0, R00); + + R01 = _mm512_add_epi64(R01, T); + T = _mm512_srli_epi64(R01, DIGIT_SIZE); + R01 = _mm512_and_epi64(R01, DIG_MASK); + _mm512_store_si512(pr + 1, R01); + + R02 = _mm512_add_epi64(R02, T); + T = _mm512_srli_epi64(R02, DIGIT_SIZE); + R02 = _mm512_and_epi64(R02, DIG_MASK); + _mm512_store_si512(pr + 2, R02); + + R03 = _mm512_add_epi64(R03, T); + T = _mm512_srli_epi64(R03, DIGIT_SIZE); + R03 = _mm512_and_epi64(R03, DIG_MASK); + _mm512_store_si512(pr + 3, R03); + + R04 = _mm512_add_epi64(R04, T); + T = _mm512_srli_epi64(R04, DIGIT_SIZE); + R04 = _mm512_and_epi64(R04, DIG_MASK); + _mm512_store_si512(pr + 4, R04); + + R05 = _mm512_add_epi64(R05, T); + T = _mm512_srli_epi64(R05, DIGIT_SIZE); + R05 = _mm512_and_epi64(R05, DIG_MASK); + _mm512_store_si512(pr + 5, R05); + + R06 = _mm512_add_epi64(R06, T); + T = _mm512_srli_epi64(R06, DIGIT_SIZE); + R06 = _mm512_and_epi64(R06, DIG_MASK); + _mm512_store_si512(pr + 6, R06); + + R07 = _mm512_add_epi64(R07, T); + T = _mm512_srli_epi64(R07, DIGIT_SIZE); + R07 = _mm512_and_epi64(R07, DIG_MASK); + _mm512_store_si512(pr + 7, R07); + + R08 = _mm512_add_epi64(R08, T); + T = _mm512_srli_epi64(R08, DIGIT_SIZE); + R08 = _mm512_and_epi64(R08, DIG_MASK); + _mm512_store_si512(pr + 8, R08); + + R09 = _mm512_add_epi64(R09, T); + T = _mm512_srli_epi64(R09, DIGIT_SIZE); + R09 = _mm512_and_epi64(R09, DIG_MASK); + _mm512_store_si512(pr + 9, R09); + } + +#undef BITSIZE +#undef LEN52 +#undef MSD_MSK +} + +void ifma_addmul52x20_mb8(int64u pRes[][8], const int64u inpA[][8], const int64u inpB[][8]) +{ + #define BITSIZE (RSA_2K/2) + #define LEN52 (NUMBER_OF_DIGITS(BITSIZE,52)) + #define MSD_MSK (MS_DIGIT_MASK(RSA_2K,52)) + + __m512i* pr = (__m512i*)pRes; + __m512i* pa = (__m512i*)inpA; + __m512i* pb = (__m512i*)inpB; + + __m512i DIG_MASK = _mm512_set1_epi64(DIGIT_MASK); + __m512i MS_DIG_MASK = _mm512_set1_epi64(MSD_MSK); + + __m512i R00 = _mm512_load_si512(pr + 0); /* load pr[nsM-1],...,R[0] */ + __m512i R01 = _mm512_load_si512(pr + 1); + __m512i R02 = _mm512_load_si512(pr + 2); + __m512i R03 = _mm512_load_si512(pr + 3); + __m512i R04 = _mm512_load_si512(pr + 4); + __m512i R05 = _mm512_load_si512(pr + 5); + __m512i R06 = _mm512_load_si512(pr + 6); + __m512i R07 = _mm512_load_si512(pr + 7); + __m512i R08 = _mm512_load_si512(pr + 8); + __m512i R09 = _mm512_load_si512(pr + 9); + __m512i R10 = _mm512_load_si512(pr + 10); + __m512i R11 = _mm512_load_si512(pr + 11); + __m512i R12 = _mm512_load_si512(pr + 12); + __m512i R13 = _mm512_load_si512(pr + 13); + __m512i R14 = _mm512_load_si512(pr + 14); + __m512i R15 = _mm512_load_si512(pr + 15); + __m512i R16 = _mm512_load_si512(pr + 16); + __m512i R17 = _mm512_load_si512(pr + 17); + __m512i R18 = _mm512_load_si512(pr + 18); + __m512i R19 = _mm512_load_si512(pr + 19); + + int itr; + for (itr = 0; itr < LEN52; itr++) { + __m512i Bi = _mm512_load_si512(pb); + __m512i nxtR = _mm512_load_si512(pr + LEN52); + pb++; + + _mm512_madd52lo_epu64_(R00, R00, Bi, pa, 64 * 0); + _mm512_madd52lo_epu64_(R01, R01, Bi, pa, 64 * 1); + _mm512_madd52lo_epu64_(R02, R02, Bi, pa, 64 * 2); + _mm512_madd52lo_epu64_(R03, R03, Bi, pa, 64 * 3); + _mm512_madd52lo_epu64_(R04, R04, Bi, pa, 64 * 4); + _mm512_madd52lo_epu64_(R05, R05, Bi, pa, 64 * 5); + _mm512_madd52lo_epu64_(R06, R06, Bi, pa, 64 * 6); + _mm512_madd52lo_epu64_(R07, R07, Bi, pa, 64 * 7); + _mm512_madd52lo_epu64_(R08, R08, Bi, pa, 64 * 8); + _mm512_madd52lo_epu64_(R09, R09, Bi, pa, 64 * 9); + _mm512_madd52lo_epu64_(R10, R10, Bi, pa, 64 * 10); + _mm512_madd52lo_epu64_(R11, R11, Bi, pa, 64 * 11); + _mm512_madd52lo_epu64_(R12, R12, Bi, pa, 64 * 12); + _mm512_madd52lo_epu64_(R13, R13, Bi, pa, 64 * 13); + _mm512_madd52lo_epu64_(R14, R14, Bi, pa, 64 * 14); + _mm512_madd52lo_epu64_(R15, R15, Bi, pa, 64 * 15); + _mm512_madd52lo_epu64_(R16, R16, Bi, pa, 64 * 16); + _mm512_madd52lo_epu64_(R17, R17, Bi, pa, 64 * 17); + _mm512_madd52lo_epu64_(R18, R18, Bi, pa, 64 * 18); + _mm512_madd52lo_epu64_(R19, R19, Bi, pa, 64 * 19); + + _mm512_store_si512(pr, _mm512_and_epi64(R00, DIG_MASK)); /* store normalized result */ + pr++; + + R00 = _mm512_srli_epi64(R00, DIGIT_SIZE); + R01 = _mm512_add_epi64(R01, R00); + + _mm512_madd52hi_epu64_(R00, R01, Bi, pa, 64 * 0); + _mm512_madd52hi_epu64_(R01, R02, Bi, pa, 64 * 1); + _mm512_madd52hi_epu64_(R02, R03, Bi, pa, 64 * 2); + _mm512_madd52hi_epu64_(R03, R04, Bi, pa, 64 * 3); + _mm512_madd52hi_epu64_(R04, R05, Bi, pa, 64 * 4); + _mm512_madd52hi_epu64_(R05, R06, Bi, pa, 64 * 5); + _mm512_madd52hi_epu64_(R06, R07, Bi, pa, 64 * 6); + _mm512_madd52hi_epu64_(R07, R08, Bi, pa, 64 * 7); + _mm512_madd52hi_epu64_(R08, R09, Bi, pa, 64 * 8); + _mm512_madd52hi_epu64_(R09, R10, Bi, pa, 64 * 9); + _mm512_madd52hi_epu64_(R10, R11, Bi, pa, 64 * 10); + _mm512_madd52hi_epu64_(R11, R12, Bi, pa, 64 * 11); + _mm512_madd52hi_epu64_(R12, R13, Bi, pa, 64 * 12); + _mm512_madd52hi_epu64_(R13, R14, Bi, pa, 64 * 13); + _mm512_madd52hi_epu64_(R14, R15, Bi, pa, 64 * 14); + _mm512_madd52hi_epu64_(R15, R16, Bi, pa, 64 * 15); + _mm512_madd52hi_epu64_(R16, R17, Bi, pa, 64 * 16); + _mm512_madd52hi_epu64_(R17, R18, Bi, pa, 64 * 17); + _mm512_madd52hi_epu64_(R18, R19, Bi, pa, 64 * 18); + _mm512_madd52hi_epu64_(R19, nxtR, Bi, pa, 64 * 19); + } + /* normalization */ + { + __m512i + T = _mm512_srli_epi64(R00, DIGIT_SIZE); + R00 = _mm512_and_epi64(R00, DIG_MASK); + _mm512_store_si512(pr + 0, R00); + + R01 = _mm512_add_epi64(R01, T); + T = _mm512_srli_epi64(R01, DIGIT_SIZE); + R01 = _mm512_and_epi64(R01, DIG_MASK); + _mm512_store_si512(pr + 1, R01); + + R02 = _mm512_add_epi64(R02, T); + T = _mm512_srli_epi64(R02, DIGIT_SIZE); + R02 = _mm512_and_epi64(R02, DIG_MASK); + _mm512_store_si512(pr + 2, R02); + + R03 = _mm512_add_epi64(R03, T); + T = _mm512_srli_epi64(R03, DIGIT_SIZE); + R03 = _mm512_and_epi64(R03, DIG_MASK); + _mm512_store_si512(pr + 3, R03); + + R04 = _mm512_add_epi64(R04, T); + T = _mm512_srli_epi64(R04, DIGIT_SIZE); + R04 = _mm512_and_epi64(R04, DIG_MASK); + _mm512_store_si512(pr + 4, R04); + + R05 = _mm512_add_epi64(R05, T); + T = _mm512_srli_epi64(R05, DIGIT_SIZE); + R05 = _mm512_and_epi64(R05, DIG_MASK); + _mm512_store_si512(pr + 5, R05); + + R06 = _mm512_add_epi64(R06, T); + T = _mm512_srli_epi64(R06, DIGIT_SIZE); + R06 = _mm512_and_epi64(R06, DIG_MASK); + _mm512_store_si512(pr + 6, R06); + + R07 = _mm512_add_epi64(R07, T); + T = _mm512_srli_epi64(R07, DIGIT_SIZE); + R07 = _mm512_and_epi64(R07, DIG_MASK); + _mm512_store_si512(pr + 7, R07); + + R08 = _mm512_add_epi64(R08, T); + T = _mm512_srli_epi64(R08, DIGIT_SIZE); + R08 = _mm512_and_epi64(R08, DIG_MASK); + _mm512_store_si512(pr + 8, R08); + + R09 = _mm512_add_epi64(R09, T); + T = _mm512_srli_epi64(R09, DIGIT_SIZE); + R09 = _mm512_and_epi64(R09, DIG_MASK); + _mm512_store_si512(pr + 9, R09); + + R10 = _mm512_add_epi64(R10, T); + T = _mm512_srli_epi64(R10, DIGIT_SIZE); + R10 = _mm512_and_epi64(R10, DIG_MASK); + _mm512_store_si512(pr + 10, R10); + + R11 = _mm512_add_epi64(R11, T); + T = _mm512_srli_epi64(R11, DIGIT_SIZE); + R11 = _mm512_and_epi64(R11, DIG_MASK); + _mm512_store_si512(pr + 11, R11); + + R12 = _mm512_add_epi64(R12, T); + T = _mm512_srli_epi64(R12, DIGIT_SIZE); + R12 = _mm512_and_epi64(R12, DIG_MASK); + _mm512_store_si512(pr + 12, R12); + + R13 = _mm512_add_epi64(R13, T); + T = _mm512_srli_epi64(R13, DIGIT_SIZE); + R13 = _mm512_and_epi64(R13, DIG_MASK); + _mm512_store_si512(pr + 13, R13); + + R14 = _mm512_add_epi64(R14, T); + T = _mm512_srli_epi64(R14, DIGIT_SIZE); + R14 = _mm512_and_epi64(R14, DIG_MASK); + _mm512_store_si512(pr + 14, R14); + + R15 = _mm512_add_epi64(R15, T); + T = _mm512_srli_epi64(R15, DIGIT_SIZE); + R15 = _mm512_and_epi64(R15, DIG_MASK); + _mm512_store_si512(pr + 15, R15); + + R16 = _mm512_add_epi64(R16, T); + T = _mm512_srli_epi64(R16, DIGIT_SIZE); + R16 = _mm512_and_epi64(R16, DIG_MASK); + _mm512_store_si512(pr + 16, R16); + + R17 = _mm512_add_epi64(R17, T); + T = _mm512_srli_epi64(R17, DIGIT_SIZE); + R17 = _mm512_and_epi64(R17, DIG_MASK); + _mm512_store_si512(pr + 17, R17); + + R18 = _mm512_add_epi64(R18, T); + T = _mm512_srli_epi64(R18, DIGIT_SIZE); + R18 = _mm512_and_epi64(R18, DIG_MASK); + _mm512_store_si512(pr + 18, R18); + + R19 = _mm512_add_epi64(R19, T); + T = _mm512_srli_epi64(R19, DIGIT_SIZE); + R19 = _mm512_and_epi64(R19, MS_DIG_MASK); + _mm512_store_si512(pr + 19, R19); + } + + #undef BITSIZE + #undef LEN52 + #undef MSD_MSK +} + +void ifma_addmul52x30_mb8(int64u pRes[][8], const int64u inpA[][8], const int64u inpB[][8]) +{ +#define BITSIZE (RSA_3K/2) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE,52)) +#define MSD_MSK (MS_DIGIT_MASK(RSA_3K,52)) + + __m512i* pr = (__m512i*)pRes; + __m512i* pa = (__m512i*)inpA; + __m512i* pb = (__m512i*)inpB; + + __m512i DIG_MASK = _mm512_set1_epi64(DIGIT_MASK); + __m512i MS_DIG_MASK = _mm512_set1_epi64(MSD_MSK); + + __m512i R00 = _mm512_load_si512(pr + 0); /* load pr[nsM-1],...,R[0] */ + __m512i R01 = _mm512_load_si512(pr + 1); + __m512i R02 = _mm512_load_si512(pr + 2); + __m512i R03 = _mm512_load_si512(pr + 3); + __m512i R04 = _mm512_load_si512(pr + 4); + __m512i R05 = _mm512_load_si512(pr + 5); + __m512i R06 = _mm512_load_si512(pr + 6); + __m512i R07 = _mm512_load_si512(pr + 7); + __m512i R08 = _mm512_load_si512(pr + 8); + __m512i R09 = _mm512_load_si512(pr + 9); + __m512i R10 = _mm512_load_si512(pr + 10); + __m512i R11 = _mm512_load_si512(pr + 11); + __m512i R12 = _mm512_load_si512(pr + 12); + __m512i R13 = _mm512_load_si512(pr + 13); + __m512i R14 = _mm512_load_si512(pr + 14); + __m512i R15 = _mm512_load_si512(pr + 15); + __m512i R16 = _mm512_load_si512(pr + 16); + __m512i R17 = _mm512_load_si512(pr + 17); + __m512i R18 = _mm512_load_si512(pr + 18); + __m512i R19 = _mm512_load_si512(pr + 19); + __m512i R20 = _mm512_load_si512(pr + 20); + __m512i R21 = _mm512_load_si512(pr + 21); + __m512i R22 = _mm512_load_si512(pr + 22); + __m512i R23 = _mm512_load_si512(pr + 23); + __m512i R24 = _mm512_load_si512(pr + 24); + __m512i R25 = _mm512_load_si512(pr + 25); + __m512i R26 = _mm512_load_si512(pr + 26); + __m512i R27 = _mm512_load_si512(pr + 27); + __m512i R28 = _mm512_load_si512(pr + 28); + __m512i R29 = _mm512_load_si512(pr + 29); + + int itr; + for (itr = 0; itr < LEN52; itr++) { + __m512i Bi = _mm512_load_si512(pb); + __m512i nxtR = _mm512_load_si512(pr + LEN52); + pb++; + + _mm512_madd52lo_epu64_(R00, R00, Bi, pa, 64 * 0); + _mm512_madd52lo_epu64_(R01, R01, Bi, pa, 64 * 1); + _mm512_madd52lo_epu64_(R02, R02, Bi, pa, 64 * 2); + _mm512_madd52lo_epu64_(R03, R03, Bi, pa, 64 * 3); + _mm512_madd52lo_epu64_(R04, R04, Bi, pa, 64 * 4); + _mm512_madd52lo_epu64_(R05, R05, Bi, pa, 64 * 5); + _mm512_madd52lo_epu64_(R06, R06, Bi, pa, 64 * 6); + _mm512_madd52lo_epu64_(R07, R07, Bi, pa, 64 * 7); + _mm512_madd52lo_epu64_(R08, R08, Bi, pa, 64 * 8); + _mm512_madd52lo_epu64_(R09, R09, Bi, pa, 64 * 9); + _mm512_madd52lo_epu64_(R10, R10, Bi, pa, 64 * 10); + _mm512_madd52lo_epu64_(R11, R11, Bi, pa, 64 * 11); + _mm512_madd52lo_epu64_(R12, R12, Bi, pa, 64 * 12); + _mm512_madd52lo_epu64_(R13, R13, Bi, pa, 64 * 13); + _mm512_madd52lo_epu64_(R14, R14, Bi, pa, 64 * 14); + _mm512_madd52lo_epu64_(R15, R15, Bi, pa, 64 * 15); + _mm512_madd52lo_epu64_(R16, R16, Bi, pa, 64 * 16); + _mm512_madd52lo_epu64_(R17, R17, Bi, pa, 64 * 17); + _mm512_madd52lo_epu64_(R18, R18, Bi, pa, 64 * 18); + _mm512_madd52lo_epu64_(R19, R19, Bi, pa, 64 * 19); + _mm512_madd52lo_epu64_(R20, R20, Bi, pa, 64 * 20); + _mm512_madd52lo_epu64_(R21, R21, Bi, pa, 64 * 21); + _mm512_madd52lo_epu64_(R22, R22, Bi, pa, 64 * 22); + _mm512_madd52lo_epu64_(R23, R23, Bi, pa, 64 * 23); + _mm512_madd52lo_epu64_(R24, R24, Bi, pa, 64 * 24); + _mm512_madd52lo_epu64_(R25, R25, Bi, pa, 64 * 25); + _mm512_madd52lo_epu64_(R26, R26, Bi, pa, 64 * 26); + _mm512_madd52lo_epu64_(R27, R27, Bi, pa, 64 * 27); + _mm512_madd52lo_epu64_(R28, R28, Bi, pa, 64 * 28); + _mm512_madd52lo_epu64_(R29, R29, Bi, pa, 64 * 29); + + _mm512_store_si512(pr, _mm512_and_epi64(R00, DIG_MASK)); /* store normalized result */ + pr++; + + R00 = _mm512_srli_epi64(R00, DIGIT_SIZE); + R01 = _mm512_add_epi64(R01, R00); + + _mm512_madd52hi_epu64_(R00, R01, Bi, pa, 64 * 0); + _mm512_madd52hi_epu64_(R01, R02, Bi, pa, 64 * 1); + _mm512_madd52hi_epu64_(R02, R03, Bi, pa, 64 * 2); + _mm512_madd52hi_epu64_(R03, R04, Bi, pa, 64 * 3); + _mm512_madd52hi_epu64_(R04, R05, Bi, pa, 64 * 4); + _mm512_madd52hi_epu64_(R05, R06, Bi, pa, 64 * 5); + _mm512_madd52hi_epu64_(R06, R07, Bi, pa, 64 * 6); + _mm512_madd52hi_epu64_(R07, R08, Bi, pa, 64 * 7); + _mm512_madd52hi_epu64_(R08, R09, Bi, pa, 64 * 8); + _mm512_madd52hi_epu64_(R09, R10, Bi, pa, 64 * 9); + _mm512_madd52hi_epu64_(R10, R11, Bi, pa, 64 * 10); + _mm512_madd52hi_epu64_(R11, R12, Bi, pa, 64 * 11); + _mm512_madd52hi_epu64_(R12, R13, Bi, pa, 64 * 12); + _mm512_madd52hi_epu64_(R13, R14, Bi, pa, 64 * 13); + _mm512_madd52hi_epu64_(R14, R15, Bi, pa, 64 * 14); + _mm512_madd52hi_epu64_(R15, R16, Bi, pa, 64 * 15); + _mm512_madd52hi_epu64_(R16, R17, Bi, pa, 64 * 16); + _mm512_madd52hi_epu64_(R17, R18, Bi, pa, 64 * 17); + _mm512_madd52hi_epu64_(R18, R19, Bi, pa, 64 * 18); + _mm512_madd52hi_epu64_(R19, R20, Bi, pa, 64 * 19); + _mm512_madd52hi_epu64_(R20, R21, Bi, pa, 64 * 20); + _mm512_madd52hi_epu64_(R21, R22, Bi, pa, 64 * 21); + _mm512_madd52hi_epu64_(R22, R23, Bi, pa, 64 * 22); + _mm512_madd52hi_epu64_(R23, R24, Bi, pa, 64 * 23); + _mm512_madd52hi_epu64_(R24, R25, Bi, pa, 64 * 24); + _mm512_madd52hi_epu64_(R25, R26, Bi, pa, 64 * 25); + _mm512_madd52hi_epu64_(R26, R27, Bi, pa, 64 * 26); + _mm512_madd52hi_epu64_(R27, R28, Bi, pa, 64 * 27); + _mm512_madd52hi_epu64_(R28, R29, Bi, pa, 64 * 28); + _mm512_madd52hi_epu64_(R29, nxtR, Bi, pa, 64 * 29); + } + /* normalization */ + { + __m512i + T = _mm512_srli_epi64(R00, DIGIT_SIZE); + R00 = _mm512_and_epi64(R00, DIG_MASK); + _mm512_store_si512(pr + 0, R00); + + R01 = _mm512_add_epi64(R01, T); + T = _mm512_srli_epi64(R01, DIGIT_SIZE); + R01 = _mm512_and_epi64(R01, DIG_MASK); + _mm512_store_si512(pr + 1, R01); + + R02 = _mm512_add_epi64(R02, T); + T = _mm512_srli_epi64(R02, DIGIT_SIZE); + R02 = _mm512_and_epi64(R02, DIG_MASK); + _mm512_store_si512(pr + 2, R02); + + R03 = _mm512_add_epi64(R03, T); + T = _mm512_srli_epi64(R03, DIGIT_SIZE); + R03 = _mm512_and_epi64(R03, DIG_MASK); + _mm512_store_si512(pr + 3, R03); + + R04 = _mm512_add_epi64(R04, T); + T = _mm512_srli_epi64(R04, DIGIT_SIZE); + R04 = _mm512_and_epi64(R04, DIG_MASK); + _mm512_store_si512(pr + 4, R04); + + R05 = _mm512_add_epi64(R05, T); + T = _mm512_srli_epi64(R05, DIGIT_SIZE); + R05 = _mm512_and_epi64(R05, DIG_MASK); + _mm512_store_si512(pr + 5, R05); + + R06 = _mm512_add_epi64(R06, T); + T = _mm512_srli_epi64(R06, DIGIT_SIZE); + R06 = _mm512_and_epi64(R06, DIG_MASK); + _mm512_store_si512(pr + 6, R06); + + R07 = _mm512_add_epi64(R07, T); + T = _mm512_srli_epi64(R07, DIGIT_SIZE); + R07 = _mm512_and_epi64(R07, DIG_MASK); + _mm512_store_si512(pr + 7, R07); + + R08 = _mm512_add_epi64(R08, T); + T = _mm512_srli_epi64(R08, DIGIT_SIZE); + R08 = _mm512_and_epi64(R08, DIG_MASK); + _mm512_store_si512(pr + 8, R08); + + R09 = _mm512_add_epi64(R09, T); + T = _mm512_srli_epi64(R09, DIGIT_SIZE); + R09 = _mm512_and_epi64(R09, DIG_MASK); + _mm512_store_si512(pr + 9, R09); + + R10 = _mm512_add_epi64(R10, T); + T = _mm512_srli_epi64(R10, DIGIT_SIZE); + R10 = _mm512_and_epi64(R10, DIG_MASK); + _mm512_store_si512(pr + 10, R10); + + R11 = _mm512_add_epi64(R11, T); + T = _mm512_srli_epi64(R11, DIGIT_SIZE); + R11 = _mm512_and_epi64(R11, DIG_MASK); + _mm512_store_si512(pr + 11, R11); + + R12 = _mm512_add_epi64(R12, T); + T = _mm512_srli_epi64(R12, DIGIT_SIZE); + R12 = _mm512_and_epi64(R12, DIG_MASK); + _mm512_store_si512(pr + 12, R12); + + R13 = _mm512_add_epi64(R13, T); + T = _mm512_srli_epi64(R13, DIGIT_SIZE); + R13 = _mm512_and_epi64(R13, DIG_MASK); + _mm512_store_si512(pr + 13, R13); + + R14 = _mm512_add_epi64(R14, T); + T = _mm512_srli_epi64(R14, DIGIT_SIZE); + R14 = _mm512_and_epi64(R14, DIG_MASK); + _mm512_store_si512(pr + 14, R14); + + R15 = _mm512_add_epi64(R15, T); + T = _mm512_srli_epi64(R15, DIGIT_SIZE); + R15 = _mm512_and_epi64(R15, DIG_MASK); + _mm512_store_si512(pr + 15, R15); + + R16 = _mm512_add_epi64(R16, T); + T = _mm512_srli_epi64(R16, DIGIT_SIZE); + R16 = _mm512_and_epi64(R16, DIG_MASK); + _mm512_store_si512(pr + 16, R16); + + R17 = _mm512_add_epi64(R17, T); + T = _mm512_srli_epi64(R17, DIGIT_SIZE); + R17 = _mm512_and_epi64(R17, DIG_MASK); + _mm512_store_si512(pr + 17, R17); + + R18 = _mm512_add_epi64(R18, T); + T = _mm512_srli_epi64(R18, DIGIT_SIZE); + R18 = _mm512_and_epi64(R18, DIG_MASK); + _mm512_store_si512(pr + 18, R18); + + R19 = _mm512_add_epi64(R19, T); + T = _mm512_srli_epi64(R19, DIGIT_SIZE); + R19 = _mm512_and_epi64(R19, DIG_MASK); + _mm512_store_si512(pr + 19, R19); + + R20 = _mm512_add_epi64(R20, T); + T = _mm512_srli_epi64(R20, DIGIT_SIZE); + R20 = _mm512_and_epi64(R20, DIG_MASK); + _mm512_store_si512(pr + 20, R20); + + R21 = _mm512_add_epi64(R21, T); + T = _mm512_srli_epi64(R21, DIGIT_SIZE); + R21 = _mm512_and_epi64(R21, DIG_MASK); + _mm512_store_si512(pr + 21, R21); + + R22 = _mm512_add_epi64(R22, T); + T = _mm512_srli_epi64(R22, DIGIT_SIZE); + R22 = _mm512_and_epi64(R22, DIG_MASK); + _mm512_store_si512(pr + 22, R22); + + R23 = _mm512_add_epi64(R23, T); + T = _mm512_srli_epi64(R23, DIGIT_SIZE); + R23 = _mm512_and_epi64(R23, DIG_MASK); + _mm512_store_si512(pr + 23, R23); + + R24 = _mm512_add_epi64(R24, T); + T = _mm512_srli_epi64(R24, DIGIT_SIZE); + R24 = _mm512_and_epi64(R24, DIG_MASK); + _mm512_store_si512(pr + 24, R24); + + R25 = _mm512_add_epi64(R25, T); + T = _mm512_srli_epi64(R25, DIGIT_SIZE); + R25 = _mm512_and_epi64(R25, DIG_MASK); + _mm512_store_si512(pr + 25, R25); + + R26 = _mm512_add_epi64(R26, T); + T = _mm512_srli_epi64(R26, DIGIT_SIZE); + R26 = _mm512_and_epi64(R26, DIG_MASK); + _mm512_store_si512(pr + 26, R26); + + R27 = _mm512_add_epi64(R27, T); + T = _mm512_srli_epi64(R27, DIGIT_SIZE); + R27 = _mm512_and_epi64(R27, DIG_MASK); + _mm512_store_si512(pr + 27, R27); + + R28 = _mm512_add_epi64(R28, T); + T = _mm512_srli_epi64(R28, DIGIT_SIZE); + R28 = _mm512_and_epi64(R28, DIG_MASK); + _mm512_store_si512(pr + 28, R28); + + R29 = _mm512_add_epi64(R29, T); + T = _mm512_srli_epi64(R29, DIGIT_SIZE); + R29 = _mm512_and_epi64(R29, MS_DIG_MASK); + _mm512_store_si512(pr + 29, R29); + } + + #undef BITSIZE + #undef LEN52 + #undef MSD_MSK +} + + +void ifma_addmul52x40_mb8(int64u pRes[][8], const int64u inpA[][8], const int64u inpB[][8]) +{ +#define BITSIZE (RSA_4K/2) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE,52)) +#define MSD_MSK (MS_DIGIT_MASK(RSA_4K,52)) + + __m512i* pr = (__m512i*)pRes; + __m512i* pa = (__m512i*)inpA; + __m512i* pb = (__m512i*)inpB; + + __m512i DIG_MASK = _mm512_set1_epi64(DIGIT_MASK); + __m512i MS_DIG_MASK = _mm512_set1_epi64(MSD_MSK); + + __m512i R00 = _mm512_load_si512(pr + 0); /* load pr[nsM-1],...,R[0] */ + __m512i R01 = _mm512_load_si512(pr + 1); + __m512i R02 = _mm512_load_si512(pr + 2); + __m512i R03 = _mm512_load_si512(pr + 3); + __m512i R04 = _mm512_load_si512(pr + 4); + __m512i R05 = _mm512_load_si512(pr + 5); + __m512i R06 = _mm512_load_si512(pr + 6); + __m512i R07 = _mm512_load_si512(pr + 7); + __m512i R08 = _mm512_load_si512(pr + 8); + __m512i R09 = _mm512_load_si512(pr + 9); + __m512i R10 = _mm512_load_si512(pr + 10); + __m512i R11 = _mm512_load_si512(pr + 11); + __m512i R12 = _mm512_load_si512(pr + 12); + __m512i R13 = _mm512_load_si512(pr + 13); + __m512i R14 = _mm512_load_si512(pr + 14); + __m512i R15 = _mm512_load_si512(pr + 15); + __m512i R16 = _mm512_load_si512(pr + 16); + __m512i R17 = _mm512_load_si512(pr + 17); + __m512i R18 = _mm512_load_si512(pr + 18); + __m512i R19 = _mm512_load_si512(pr + 19); + __m512i R20 = _mm512_load_si512(pr + 20); + __m512i R21 = _mm512_load_si512(pr + 21); + __m512i R22 = _mm512_load_si512(pr + 22); + __m512i R23 = _mm512_load_si512(pr + 23); + __m512i R24 = _mm512_load_si512(pr + 24); + __m512i R25 = _mm512_load_si512(pr + 25); + __m512i R26 = _mm512_load_si512(pr + 26); + __m512i R27 = _mm512_load_si512(pr + 27); + __m512i R28 = _mm512_load_si512(pr + 28); + __m512i R29 = _mm512_load_si512(pr + 29); + __m512i R30 = _mm512_load_si512(pr + 30); + __m512i R31 = _mm512_load_si512(pr + 31); + __m512i R32 = _mm512_load_si512(pr + 32); + __m512i R33 = _mm512_load_si512(pr + 33); + __m512i R34 = _mm512_load_si512(pr + 34); + __m512i R35 = _mm512_load_si512(pr + 35); + __m512i R36 = _mm512_load_si512(pr + 36); + __m512i R37 = _mm512_load_si512(pr + 37); + __m512i R38 = _mm512_load_si512(pr + 38); + __m512i R39 = _mm512_load_si512(pr + 39); + + int itr; + for (itr = 0; itr < LEN52; itr++) { + __m512i Bi = _mm512_load_si512(pb); + __m512i nxtR = _mm512_load_si512(pr + LEN52); + pb++; + + _mm512_madd52lo_epu64_(R00, R00, Bi, pa, 64 * 0); + _mm512_madd52lo_epu64_(R01, R01, Bi, pa, 64 * 1); + _mm512_madd52lo_epu64_(R02, R02, Bi, pa, 64 * 2); + _mm512_madd52lo_epu64_(R03, R03, Bi, pa, 64 * 3); + _mm512_madd52lo_epu64_(R04, R04, Bi, pa, 64 * 4); + _mm512_madd52lo_epu64_(R05, R05, Bi, pa, 64 * 5); + _mm512_madd52lo_epu64_(R06, R06, Bi, pa, 64 * 6); + _mm512_madd52lo_epu64_(R07, R07, Bi, pa, 64 * 7); + _mm512_madd52lo_epu64_(R08, R08, Bi, pa, 64 * 8); + _mm512_madd52lo_epu64_(R09, R09, Bi, pa, 64 * 9); + _mm512_madd52lo_epu64_(R10, R10, Bi, pa, 64 * 10); + _mm512_madd52lo_epu64_(R11, R11, Bi, pa, 64 * 11); + _mm512_madd52lo_epu64_(R12, R12, Bi, pa, 64 * 12); + _mm512_madd52lo_epu64_(R13, R13, Bi, pa, 64 * 13); + _mm512_madd52lo_epu64_(R14, R14, Bi, pa, 64 * 14); + _mm512_madd52lo_epu64_(R15, R15, Bi, pa, 64 * 15); + _mm512_madd52lo_epu64_(R16, R16, Bi, pa, 64 * 16); + _mm512_madd52lo_epu64_(R17, R17, Bi, pa, 64 * 17); + _mm512_madd52lo_epu64_(R18, R18, Bi, pa, 64 * 18); + _mm512_madd52lo_epu64_(R19, R19, Bi, pa, 64 * 19); + _mm512_madd52lo_epu64_(R20, R20, Bi, pa, 64 * 20); + _mm512_madd52lo_epu64_(R21, R21, Bi, pa, 64 * 21); + _mm512_madd52lo_epu64_(R22, R22, Bi, pa, 64 * 22); + _mm512_madd52lo_epu64_(R23, R23, Bi, pa, 64 * 23); + _mm512_madd52lo_epu64_(R24, R24, Bi, pa, 64 * 24); + _mm512_madd52lo_epu64_(R25, R25, Bi, pa, 64 * 25); + _mm512_madd52lo_epu64_(R26, R26, Bi, pa, 64 * 26); + _mm512_madd52lo_epu64_(R27, R27, Bi, pa, 64 * 27); + _mm512_madd52lo_epu64_(R28, R28, Bi, pa, 64 * 28); + _mm512_madd52lo_epu64_(R29, R29, Bi, pa, 64 * 29); + _mm512_madd52lo_epu64_(R30, R30, Bi, pa, 64 * 30); + _mm512_madd52lo_epu64_(R31, R31, Bi, pa, 64 * 31); + _mm512_madd52lo_epu64_(R32, R32, Bi, pa, 64 * 32); + _mm512_madd52lo_epu64_(R33, R33, Bi, pa, 64 * 33); + _mm512_madd52lo_epu64_(R34, R34, Bi, pa, 64 * 34); + _mm512_madd52lo_epu64_(R35, R35, Bi, pa, 64 * 35); + _mm512_madd52lo_epu64_(R36, R36, Bi, pa, 64 * 36); + _mm512_madd52lo_epu64_(R37, R37, Bi, pa, 64 * 37); + _mm512_madd52lo_epu64_(R38, R38, Bi, pa, 64 * 38); + _mm512_madd52lo_epu64_(R39, R39, Bi, pa, 64 * 39); + + _mm512_store_si512(pr, _mm512_and_epi64(R00, DIG_MASK)); /* store normalized result */ + pr++; + + R00 = _mm512_srli_epi64(R00, DIGIT_SIZE); + R01 = _mm512_add_epi64(R01, R00); + + _mm512_madd52hi_epu64_(R00, R01, Bi, pa, 64 * 0); + _mm512_madd52hi_epu64_(R01, R02, Bi, pa, 64 * 1); + _mm512_madd52hi_epu64_(R02, R03, Bi, pa, 64 * 2); + _mm512_madd52hi_epu64_(R03, R04, Bi, pa, 64 * 3); + _mm512_madd52hi_epu64_(R04, R05, Bi, pa, 64 * 4); + _mm512_madd52hi_epu64_(R05, R06, Bi, pa, 64 * 5); + _mm512_madd52hi_epu64_(R06, R07, Bi, pa, 64 * 6); + _mm512_madd52hi_epu64_(R07, R08, Bi, pa, 64 * 7); + _mm512_madd52hi_epu64_(R08, R09, Bi, pa, 64 * 8); + _mm512_madd52hi_epu64_(R09, R10, Bi, pa, 64 * 9); + _mm512_madd52hi_epu64_(R10, R11, Bi, pa, 64 * 10); + _mm512_madd52hi_epu64_(R11, R12, Bi, pa, 64 * 11); + _mm512_madd52hi_epu64_(R12, R13, Bi, pa, 64 * 12); + _mm512_madd52hi_epu64_(R13, R14, Bi, pa, 64 * 13); + _mm512_madd52hi_epu64_(R14, R15, Bi, pa, 64 * 14); + _mm512_madd52hi_epu64_(R15, R16, Bi, pa, 64 * 15); + _mm512_madd52hi_epu64_(R16, R17, Bi, pa, 64 * 16); + _mm512_madd52hi_epu64_(R17, R18, Bi, pa, 64 * 17); + _mm512_madd52hi_epu64_(R18, R19, Bi, pa, 64 * 18); + _mm512_madd52hi_epu64_(R19, R20, Bi, pa, 64 * 19); + _mm512_madd52hi_epu64_(R20, R21, Bi, pa, 64 * 20); + _mm512_madd52hi_epu64_(R21, R22, Bi, pa, 64 * 21); + _mm512_madd52hi_epu64_(R22, R23, Bi, pa, 64 * 22); + _mm512_madd52hi_epu64_(R23, R24, Bi, pa, 64 * 23); + _mm512_madd52hi_epu64_(R24, R25, Bi, pa, 64 * 24); + _mm512_madd52hi_epu64_(R25, R26, Bi, pa, 64 * 25); + _mm512_madd52hi_epu64_(R26, R27, Bi, pa, 64 * 26); + _mm512_madd52hi_epu64_(R27, R28, Bi, pa, 64 * 27); + _mm512_madd52hi_epu64_(R28, R29, Bi, pa, 64 * 28); + _mm512_madd52hi_epu64_(R29, R30, Bi, pa, 64 * 29); + _mm512_madd52hi_epu64_(R30, R31, Bi, pa, 64 * 30); + _mm512_madd52hi_epu64_(R31, R32, Bi, pa, 64 * 31); + _mm512_madd52hi_epu64_(R32, R33, Bi, pa, 64 * 32); + _mm512_madd52hi_epu64_(R33, R34, Bi, pa, 64 * 33); + _mm512_madd52hi_epu64_(R34, R35, Bi, pa, 64 * 34); + _mm512_madd52hi_epu64_(R35, R36, Bi, pa, 64 * 35); + _mm512_madd52hi_epu64_(R36, R37, Bi, pa, 64 * 36); + _mm512_madd52hi_epu64_(R37, R38, Bi, pa, 64 * 37); + _mm512_madd52hi_epu64_(R38, R39, Bi, pa, 64 * 38); + _mm512_madd52hi_epu64_(R39, nxtR, Bi, pa, 64 * 39); + + } + /* normalization */ + { + __m512i + T = _mm512_srli_epi64(R00, DIGIT_SIZE); + R00 = _mm512_and_epi64(R00, DIG_MASK); + _mm512_store_si512(pr + 0, R00); + + R01 = _mm512_add_epi64(R01, T); + T = _mm512_srli_epi64(R01, DIGIT_SIZE); + R01 = _mm512_and_epi64(R01, DIG_MASK); + _mm512_store_si512(pr + 1, R01); + + R02 = _mm512_add_epi64(R02, T); + T = _mm512_srli_epi64(R02, DIGIT_SIZE); + R02 = _mm512_and_epi64(R02, DIG_MASK); + _mm512_store_si512(pr + 2, R02); + + R03 = _mm512_add_epi64(R03, T); + T = _mm512_srli_epi64(R03, DIGIT_SIZE); + R03 = _mm512_and_epi64(R03, DIG_MASK); + _mm512_store_si512(pr + 3, R03); + + R04 = _mm512_add_epi64(R04, T); + T = _mm512_srli_epi64(R04, DIGIT_SIZE); + R04 = _mm512_and_epi64(R04, DIG_MASK); + _mm512_store_si512(pr + 4, R04); + + R05 = _mm512_add_epi64(R05, T); + T = _mm512_srli_epi64(R05, DIGIT_SIZE); + R05 = _mm512_and_epi64(R05, DIG_MASK); + _mm512_store_si512(pr + 5, R05); + + R06 = _mm512_add_epi64(R06, T); + T = _mm512_srli_epi64(R06, DIGIT_SIZE); + R06 = _mm512_and_epi64(R06, DIG_MASK); + _mm512_store_si512(pr + 6, R06); + + R07 = _mm512_add_epi64(R07, T); + T = _mm512_srli_epi64(R07, DIGIT_SIZE); + R07 = _mm512_and_epi64(R07, DIG_MASK); + _mm512_store_si512(pr + 7, R07); + + R08 = _mm512_add_epi64(R08, T); + T = _mm512_srli_epi64(R08, DIGIT_SIZE); + R08 = _mm512_and_epi64(R08, DIG_MASK); + _mm512_store_si512(pr + 8, R08); + + R09 = _mm512_add_epi64(R09, T); + T = _mm512_srli_epi64(R09, DIGIT_SIZE); + R09 = _mm512_and_epi64(R09, DIG_MASK); + _mm512_store_si512(pr + 9, R09); + + R10 = _mm512_add_epi64(R10, T); + T = _mm512_srli_epi64(R10, DIGIT_SIZE); + R10 = _mm512_and_epi64(R10, DIG_MASK); + _mm512_store_si512(pr + 10, R10); + + R11 = _mm512_add_epi64(R11, T); + T = _mm512_srli_epi64(R11, DIGIT_SIZE); + R11 = _mm512_and_epi64(R11, DIG_MASK); + _mm512_store_si512(pr + 11, R11); + + R12 = _mm512_add_epi64(R12, T); + T = _mm512_srli_epi64(R12, DIGIT_SIZE); + R12 = _mm512_and_epi64(R12, DIG_MASK); + _mm512_store_si512(pr + 12, R12); + + R13 = _mm512_add_epi64(R13, T); + T = _mm512_srli_epi64(R13, DIGIT_SIZE); + R13 = _mm512_and_epi64(R13, DIG_MASK); + _mm512_store_si512(pr + 13, R13); + + R14 = _mm512_add_epi64(R14, T); + T = _mm512_srli_epi64(R14, DIGIT_SIZE); + R14 = _mm512_and_epi64(R14, DIG_MASK); + _mm512_store_si512(pr + 14, R14); + + R15 = _mm512_add_epi64(R15, T); + T = _mm512_srli_epi64(R15, DIGIT_SIZE); + R15 = _mm512_and_epi64(R15, DIG_MASK); + _mm512_store_si512(pr + 15, R15); + + R16 = _mm512_add_epi64(R16, T); + T = _mm512_srli_epi64(R16, DIGIT_SIZE); + R16 = _mm512_and_epi64(R16, DIG_MASK); + _mm512_store_si512(pr + 16, R16); + + R17 = _mm512_add_epi64(R17, T); + T = _mm512_srli_epi64(R17, DIGIT_SIZE); + R17 = _mm512_and_epi64(R17, DIG_MASK); + _mm512_store_si512(pr + 17, R17); + + R18 = _mm512_add_epi64(R18, T); + T = _mm512_srli_epi64(R18, DIGIT_SIZE); + R18 = _mm512_and_epi64(R18, DIG_MASK); + _mm512_store_si512(pr + 18, R18); + + R19 = _mm512_add_epi64(R19, T); + T = _mm512_srli_epi64(R19, DIGIT_SIZE); + R19 = _mm512_and_epi64(R19, DIG_MASK); + _mm512_store_si512(pr + 19, R19); + + R20 = _mm512_add_epi64(R20, T); + T = _mm512_srli_epi64(R20, DIGIT_SIZE); + R20 = _mm512_and_epi64(R20, DIG_MASK); + _mm512_store_si512(pr + 20, R20); + + R21 = _mm512_add_epi64(R21, T); + T = _mm512_srli_epi64(R21, DIGIT_SIZE); + R21 = _mm512_and_epi64(R21, DIG_MASK); + _mm512_store_si512(pr + 21, R21); + + R22 = _mm512_add_epi64(R22, T); + T = _mm512_srli_epi64(R22, DIGIT_SIZE); + R22 = _mm512_and_epi64(R22, DIG_MASK); + _mm512_store_si512(pr + 22, R22); + + R23 = _mm512_add_epi64(R23, T); + T = _mm512_srli_epi64(R23, DIGIT_SIZE); + R23 = _mm512_and_epi64(R23, DIG_MASK); + _mm512_store_si512(pr + 23, R23); + + R24 = _mm512_add_epi64(R24, T); + T = _mm512_srli_epi64(R24, DIGIT_SIZE); + R24 = _mm512_and_epi64(R24, DIG_MASK); + _mm512_store_si512(pr + 24, R24); + + R25 = _mm512_add_epi64(R25, T); + T = _mm512_srli_epi64(R25, DIGIT_SIZE); + R25 = _mm512_and_epi64(R25, DIG_MASK); + _mm512_store_si512(pr + 25, R25); + + R26 = _mm512_add_epi64(R26, T); + T = _mm512_srli_epi64(R26, DIGIT_SIZE); + R26 = _mm512_and_epi64(R26, DIG_MASK); + _mm512_store_si512(pr + 26, R26); + + R27 = _mm512_add_epi64(R27, T); + T = _mm512_srli_epi64(R27, DIGIT_SIZE); + R27 = _mm512_and_epi64(R27, DIG_MASK); + _mm512_store_si512(pr + 27, R27); + + R28 = _mm512_add_epi64(R28, T); + T = _mm512_srli_epi64(R28, DIGIT_SIZE); + R28 = _mm512_and_epi64(R28, DIG_MASK); + _mm512_store_si512(pr + 28, R28); + + R29 = _mm512_add_epi64(R29, T); + T = _mm512_srli_epi64(R29, DIGIT_SIZE); + R29 = _mm512_and_epi64(R29, DIG_MASK); + _mm512_store_si512(pr + 29, R29); + + R30 = _mm512_add_epi64(R30, T); + T = _mm512_srli_epi64(R30, DIGIT_SIZE); + R30 = _mm512_and_epi64(R30, DIG_MASK); + _mm512_store_si512(pr + 30, R30); + + R31 = _mm512_add_epi64(R31, T); + T = _mm512_srli_epi64(R31, DIGIT_SIZE); + R31 = _mm512_and_epi64(R31, DIG_MASK); + _mm512_store_si512(pr + 31, R31); + + R32 = _mm512_add_epi64(R32, T); + T = _mm512_srli_epi64(R32, DIGIT_SIZE); + R32 = _mm512_and_epi64(R32, DIG_MASK); + _mm512_store_si512(pr + 32, R32); + + R33 = _mm512_add_epi64(R33, T); + T = _mm512_srli_epi64(R33, DIGIT_SIZE); + R33 = _mm512_and_epi64(R33, DIG_MASK); + _mm512_store_si512(pr + 33, R33); + + R34 = _mm512_add_epi64(R34, T); + T = _mm512_srli_epi64(R34, DIGIT_SIZE); + R34 = _mm512_and_epi64(R34, DIG_MASK); + _mm512_store_si512(pr + 34, R34); + + R35 = _mm512_add_epi64(R35, T); + T = _mm512_srli_epi64(R35, DIGIT_SIZE); + R35 = _mm512_and_epi64(R35, DIG_MASK); + _mm512_store_si512(pr + 35, R35); + + R36 = _mm512_add_epi64(R36, T); + T = _mm512_srli_epi64(R36, DIGIT_SIZE); + R36 = _mm512_and_epi64(R36, DIG_MASK); + _mm512_store_si512(pr + 36, R36); + + R37 = _mm512_add_epi64(R37, T); + T = _mm512_srli_epi64(R37, DIGIT_SIZE); + R37 = _mm512_and_epi64(R37, DIG_MASK); + _mm512_store_si512(pr + 37, R37); + + R38 = _mm512_add_epi64(R38, T); + T = _mm512_srli_epi64(R38, DIGIT_SIZE); + R38 = _mm512_and_epi64(R38, MS_DIG_MASK); + _mm512_store_si512(pr + 38, R38); + } + + #undef BITSIZE + #undef LEN52 + #undef MSD_MSK +} + + +/* r = x * (R^-1) mod q */ +void ifma_amred52x10_mb8(int64u res[][8], + const int64u inpA[][8], /* int nsA == 2*nsM */ + const int64u inpM[][8], /* int nsM ==10 */ + const int64u k0[8]) +{ + int64u* pA = (int64u*)inpA; + int64u* pM = (int64u*)inpM; + int64u* pR = (int64u*)res; + + __m512i K = _mm512_load_si512(k0); /* k0[] */ + + __m512i R00 = _mm512_load_si512(pA + 8 * 0); /* load A[nsM-1],...,A[0] */ + __m512i R01 = _mm512_load_si512(pA + 8 * 1); + __m512i R02 = _mm512_load_si512(pA + 8 * 2); + __m512i R03 = _mm512_load_si512(pA + 8 * 3); + __m512i R04 = _mm512_load_si512(pA + 8 * 4); + __m512i R05 = _mm512_load_si512(pA + 8 * 5); + __m512i R06 = _mm512_load_si512(pA + 8 * 6); + __m512i R07 = _mm512_load_si512(pA + 8 * 7); + __m512i R08 = _mm512_load_si512(pA + 8 * 8); + __m512i R09 = _mm512_load_si512(pA + 8 * 9); + + int itr; + for (itr = 0, pA += 8 * 10; itr < 10; itr++, pA += 8) { + __m512i Yi = _mm512_madd52lo_epu64(_mm512_setzero_si512(), R00, K); + __m512i nxtA = _mm512_load_si512(pA); + + _mm512_madd52lo_epu64_(R00, R00, Yi, pM, 64 * 0); + _mm512_madd52lo_epu64_(R01, R01, Yi, pM, 64 * 1); + _mm512_madd52lo_epu64_(R02, R02, Yi, pM, 64 * 2); + _mm512_madd52lo_epu64_(R03, R03, Yi, pM, 64 * 3); + _mm512_madd52lo_epu64_(R04, R04, Yi, pM, 64 * 4); + _mm512_madd52lo_epu64_(R05, R05, Yi, pM, 64 * 5); + _mm512_madd52lo_epu64_(R06, R06, Yi, pM, 64 * 6); + _mm512_madd52lo_epu64_(R07, R07, Yi, pM, 64 * 7); + _mm512_madd52lo_epu64_(R08, R08, Yi, pM, 64 * 8); + _mm512_madd52lo_epu64_(R09, R09, Yi, pM, 64 * 9); + + R00 = _mm512_srli_epi64(R00, DIGIT_SIZE); + R01 = _mm512_add_epi64(R01, R00); + + _mm512_madd52hi_epu64_(R00, R01, Yi, pM, 64 * 0); + _mm512_madd52hi_epu64_(R01, R02, Yi, pM, 64 * 1); + _mm512_madd52hi_epu64_(R02, R03, Yi, pM, 64 * 2); + _mm512_madd52hi_epu64_(R03, R04, Yi, pM, 64 * 3); + _mm512_madd52hi_epu64_(R04, R05, Yi, pM, 64 * 4); + _mm512_madd52hi_epu64_(R05, R06, Yi, pM, 64 * 5); + _mm512_madd52hi_epu64_(R06, R07, Yi, pM, 64 * 6); + _mm512_madd52hi_epu64_(R07, R08, Yi, pM, 64 * 7); + _mm512_madd52hi_epu64_(R08, R09, Yi, pM, 64 * 8); + _mm512_madd52hi_epu64_(R09, nxtA, Yi, pM, 64 * 9); + } + + /* normalization */ + { + __m512i MASK = _mm512_set1_epi64(DIGIT_MASK); + + __m512i + T = _mm512_srli_epi64(R00, DIGIT_SIZE); + R00 = _mm512_and_epi64(R00, MASK); + _mm512_store_si512(pR + 8 * 0, R00); + + R01 = _mm512_add_epi64(R01, T); + T = _mm512_srli_epi64(R01, DIGIT_SIZE); + R01 = _mm512_and_epi64(R01, MASK); + _mm512_store_si512(pR + 8 * 1, R01); + + R02 = _mm512_add_epi64(R02, T); + T = _mm512_srli_epi64(R02, DIGIT_SIZE); + R02 = _mm512_and_epi64(R02, MASK); + _mm512_store_si512(pR + 8 * 2, R02); + + R03 = _mm512_add_epi64(R03, T); + T = _mm512_srli_epi64(R03, DIGIT_SIZE); + R03 = _mm512_and_epi64(R03, MASK); + _mm512_store_si512(pR + 8 * 3, R03); + + R04 = _mm512_add_epi64(R04, T); + T = _mm512_srli_epi64(R04, DIGIT_SIZE); + R04 = _mm512_and_epi64(R04, MASK); + _mm512_store_si512(pR + 8 * 4, R04); + + R05 = _mm512_add_epi64(R05, T); + T = _mm512_srli_epi64(R05, DIGIT_SIZE); + R05 = _mm512_and_epi64(R05, MASK); + _mm512_store_si512(pR + 8 * 5, R05); + + R06 = _mm512_add_epi64(R06, T); + T = _mm512_srli_epi64(R06, DIGIT_SIZE); + R06 = _mm512_and_epi64(R06, MASK); + _mm512_store_si512(pR + 8 * 6, R06); + + R07 = _mm512_add_epi64(R07, T); + T = _mm512_srli_epi64(R07, DIGIT_SIZE); + R07 = _mm512_and_epi64(R07, MASK); + _mm512_store_si512(pR + 8 * 7, R07); + + R08 = _mm512_add_epi64(R08, T); + T = _mm512_srli_epi64(R08, DIGIT_SIZE); + R08 = _mm512_and_epi64(R08, MASK); + _mm512_store_si512(pR + 8 * 8, R08); + + R09 = _mm512_add_epi64(R09, T); + T = _mm512_srli_epi64(R09, DIGIT_SIZE); + R09 = _mm512_and_epi64(R09, MASK); + _mm512_store_si512(pR + 8 * 9, R09); + } +} + +void ifma_amred52x20_mb8(int64u res[][8], + const int64u inpA[][8], /* int nsA == 2*nsM */ + const int64u inpM[][8], /* int nsM ==20 */ + const int64u k0[8]) +{ + int64u* pA = (int64u*)inpA; + int64u* pM = (int64u*)inpM; + int64u* pR = (int64u*)res; + + __m512i K = _mm512_load_si512(k0); /* k0[] */ + + __m512i R00 = _mm512_load_si512(pA+8*0); /* load A[nsM-1],...,A[0] */ + __m512i R01 = _mm512_load_si512(pA+8*1); + __m512i R02 = _mm512_load_si512(pA+8*2); + __m512i R03 = _mm512_load_si512(pA+8*3); + __m512i R04 = _mm512_load_si512(pA+8*4); + __m512i R05 = _mm512_load_si512(pA+8*5); + __m512i R06 = _mm512_load_si512(pA+8*6); + __m512i R07 = _mm512_load_si512(pA+8*7); + __m512i R08 = _mm512_load_si512(pA+8*8); + __m512i R09 = _mm512_load_si512(pA+8*9); + __m512i R10 = _mm512_load_si512(pA+8*10); + __m512i R11 = _mm512_load_si512(pA+8*11); + __m512i R12 = _mm512_load_si512(pA+8*12); + __m512i R13 = _mm512_load_si512(pA+8*13); + __m512i R14 = _mm512_load_si512(pA+8*14); + __m512i R15 = _mm512_load_si512(pA+8*15); + __m512i R16 = _mm512_load_si512(pA+8*16); + __m512i R17 = _mm512_load_si512(pA+8*17); + __m512i R18 = _mm512_load_si512(pA+8*18); + __m512i R19 = _mm512_load_si512(pA+8*19); + + int itr; + for(itr=0, pA+=8*20; itr<20; itr++, pA+=8) { + __m512i Yi = _mm512_madd52lo_epu64(_mm512_setzero_si512(), R00, K); + __m512i nxtA = _mm512_load_si512(pA); + + _mm512_madd52lo_epu64_(R00, R00, Yi, pM, 64*0); + _mm512_madd52lo_epu64_(R01, R01, Yi, pM, 64*1); + _mm512_madd52lo_epu64_(R02, R02, Yi, pM, 64*2); + _mm512_madd52lo_epu64_(R03, R03, Yi, pM, 64*3); + _mm512_madd52lo_epu64_(R04, R04, Yi, pM, 64*4); + _mm512_madd52lo_epu64_(R05, R05, Yi, pM, 64*5); + _mm512_madd52lo_epu64_(R06, R06, Yi, pM, 64*6); + _mm512_madd52lo_epu64_(R07, R07, Yi, pM, 64*7); + _mm512_madd52lo_epu64_(R08, R08, Yi, pM, 64*8); + _mm512_madd52lo_epu64_(R09, R09, Yi, pM, 64*9); + _mm512_madd52lo_epu64_(R10, R10, Yi, pM, 64*10); + _mm512_madd52lo_epu64_(R11, R11, Yi, pM, 64*11); + _mm512_madd52lo_epu64_(R12, R12, Yi, pM, 64*12); + _mm512_madd52lo_epu64_(R13, R13, Yi, pM, 64*13); + _mm512_madd52lo_epu64_(R14, R14, Yi, pM, 64*14); + _mm512_madd52lo_epu64_(R15, R15, Yi, pM, 64*15); + _mm512_madd52lo_epu64_(R16, R16, Yi, pM, 64*16); + _mm512_madd52lo_epu64_(R17, R17, Yi, pM, 64*17); + _mm512_madd52lo_epu64_(R18, R18, Yi, pM, 64*18); + _mm512_madd52lo_epu64_(R19, R19, Yi, pM, 64*19); + + R00 = _mm512_srli_epi64(R00, DIGIT_SIZE); + R01 = _mm512_add_epi64(R01, R00); + + _mm512_madd52hi_epu64_(R00, R01, Yi, pM, 64*0); + _mm512_madd52hi_epu64_(R01, R02, Yi, pM, 64*1); + _mm512_madd52hi_epu64_(R02, R03, Yi, pM, 64*2); + _mm512_madd52hi_epu64_(R03, R04, Yi, pM, 64*3); + _mm512_madd52hi_epu64_(R04, R05, Yi, pM, 64*4); + _mm512_madd52hi_epu64_(R05, R06, Yi, pM, 64*5); + _mm512_madd52hi_epu64_(R06, R07, Yi, pM, 64*6); + _mm512_madd52hi_epu64_(R07, R08, Yi, pM, 64*7); + _mm512_madd52hi_epu64_(R08, R09, Yi, pM, 64*8); + _mm512_madd52hi_epu64_(R09, R10, Yi, pM, 64*9); + _mm512_madd52hi_epu64_(R10, R11, Yi, pM, 64*10); + _mm512_madd52hi_epu64_(R11, R12, Yi, pM, 64*11); + _mm512_madd52hi_epu64_(R12, R13, Yi, pM, 64*12); + _mm512_madd52hi_epu64_(R13, R14, Yi, pM, 64*13); + _mm512_madd52hi_epu64_(R14, R15, Yi, pM, 64*14); + _mm512_madd52hi_epu64_(R15, R16, Yi, pM, 64*15); + _mm512_madd52hi_epu64_(R16, R17, Yi, pM, 64*16); + _mm512_madd52hi_epu64_(R17, R18, Yi, pM, 64*17); + _mm512_madd52hi_epu64_(R18, R19, Yi, pM, 64*18); + _mm512_madd52hi_epu64_(R19, nxtA,Yi, pM, 64*19); + } + + /* normalization */ + { + __m512i MASK = _mm512_set1_epi64(DIGIT_MASK); + + __m512i + T = _mm512_srli_epi64(R00, DIGIT_SIZE); + R00 = _mm512_and_epi64(R00, MASK); + _mm512_store_si512(pR+8*0, R00); + + R01 = _mm512_add_epi64(R01, T); + T = _mm512_srli_epi64(R01, DIGIT_SIZE); + R01 = _mm512_and_epi64(R01, MASK); + _mm512_store_si512(pR+8*1, R01); + + R02 = _mm512_add_epi64(R02, T); + T = _mm512_srli_epi64(R02, DIGIT_SIZE); + R02 = _mm512_and_epi64(R02, MASK); + _mm512_store_si512(pR+8*2, R02); + + R03 = _mm512_add_epi64(R03, T); + T = _mm512_srli_epi64(R03, DIGIT_SIZE); + R03 = _mm512_and_epi64(R03, MASK); + _mm512_store_si512(pR+8*3, R03); + + R04 = _mm512_add_epi64(R04, T); + T = _mm512_srli_epi64(R04, DIGIT_SIZE); + R04 = _mm512_and_epi64(R04, MASK); + _mm512_store_si512(pR+8*4, R04); + + R05 = _mm512_add_epi64(R05, T); + T = _mm512_srli_epi64(R05, DIGIT_SIZE); + R05 = _mm512_and_epi64(R05, MASK); + _mm512_store_si512(pR+8*5, R05); + + R06 = _mm512_add_epi64(R06, T); + T = _mm512_srli_epi64(R06, DIGIT_SIZE); + R06 = _mm512_and_epi64(R06, MASK); + _mm512_store_si512(pR+8*6, R06); + + R07 = _mm512_add_epi64(R07, T); + T = _mm512_srli_epi64(R07, DIGIT_SIZE); + R07 = _mm512_and_epi64(R07, MASK); + _mm512_store_si512(pR+8*7, R07); + + R08 = _mm512_add_epi64(R08, T); + T = _mm512_srli_epi64(R08, DIGIT_SIZE); + R08 = _mm512_and_epi64(R08, MASK); + _mm512_store_si512(pR+8*8, R08); + + R09 = _mm512_add_epi64(R09, T); + T = _mm512_srli_epi64(R09, DIGIT_SIZE); + R09 = _mm512_and_epi64(R09, MASK); + _mm512_store_si512(pR+8*9, R09); + + R10 = _mm512_add_epi64(R10, T); + T = _mm512_srli_epi64(R10, DIGIT_SIZE); + R10 = _mm512_and_epi64(R10, MASK); + _mm512_store_si512(pR+8*10, R10); + + R11 = _mm512_add_epi64(R11, T); + T = _mm512_srli_epi64(R11, DIGIT_SIZE); + R11 = _mm512_and_epi64(R11, MASK); + _mm512_store_si512(pR+8*11, R11); + + R12 = _mm512_add_epi64(R12, T); + T = _mm512_srli_epi64(R12, DIGIT_SIZE); + R12 = _mm512_and_epi64(R12, MASK); + _mm512_store_si512(pR+8*12, R12); + + R13 = _mm512_add_epi64(R13, T); + T = _mm512_srli_epi64(R13, DIGIT_SIZE); + R13 = _mm512_and_epi64(R13, MASK); + _mm512_store_si512(pR+8*13, R13); + + R14 = _mm512_add_epi64(R14, T); + T = _mm512_srli_epi64(R14, DIGIT_SIZE); + R14 = _mm512_and_epi64(R14, MASK); + _mm512_store_si512(pR+8*14, R14); + + R15 = _mm512_add_epi64(R15, T); + T = _mm512_srli_epi64(R15, DIGIT_SIZE); + R15 = _mm512_and_epi64(R15, MASK); + _mm512_store_si512(pR+8*15, R15); + + R16 = _mm512_add_epi64(R16, T); + T = _mm512_srli_epi64(R16, DIGIT_SIZE); + R16 = _mm512_and_epi64(R16, MASK); + _mm512_store_si512(pR+8*16, R16); + + R17 = _mm512_add_epi64(R17, T); + T = _mm512_srli_epi64(R17, DIGIT_SIZE); + R17 = _mm512_and_epi64(R17, MASK); + _mm512_store_si512(pR+8*17, R17); + + R18 = _mm512_add_epi64(R18, T); + T = _mm512_srli_epi64(R18, DIGIT_SIZE); + R18 = _mm512_and_epi64(R18, MASK); + _mm512_store_si512(pR+8*18, R18); + + R19 = _mm512_add_epi64(R19, T); + T = _mm512_srli_epi64(R19, DIGIT_SIZE); + R19 = _mm512_and_epi64(R19, MASK); + _mm512_store_si512(pR+8*19, R19); + } +} + +void ifma_amred52x30_mb8(int64u res[][8], + const int64u inpA[][8], /* int nsA == 2*nsM */ + const int64u inpM[][8], /* int nsM ==20 */ + const int64u k0[8]) +{ + int64u* pA = (int64u*)inpA; + int64u* pM = (int64u*)inpM; + int64u* pR = (int64u*)res; + + __m512i K = _mm512_load_si512(k0); /* k0[] */ + + __m512i R00 = _mm512_load_si512(pA + 8 * 0); /* load A[nsM-1],...,A[0] */ + __m512i R01 = _mm512_load_si512(pA + 8 * 1); + __m512i R02 = _mm512_load_si512(pA + 8 * 2); + __m512i R03 = _mm512_load_si512(pA + 8 * 3); + __m512i R04 = _mm512_load_si512(pA + 8 * 4); + __m512i R05 = _mm512_load_si512(pA + 8 * 5); + __m512i R06 = _mm512_load_si512(pA + 8 * 6); + __m512i R07 = _mm512_load_si512(pA + 8 * 7); + __m512i R08 = _mm512_load_si512(pA + 8 * 8); + __m512i R09 = _mm512_load_si512(pA + 8 * 9); + __m512i R10 = _mm512_load_si512(pA + 8 * 10); + __m512i R11 = _mm512_load_si512(pA + 8 * 11); + __m512i R12 = _mm512_load_si512(pA + 8 * 12); + __m512i R13 = _mm512_load_si512(pA + 8 * 13); + __m512i R14 = _mm512_load_si512(pA + 8 * 14); + __m512i R15 = _mm512_load_si512(pA + 8 * 15); + __m512i R16 = _mm512_load_si512(pA + 8 * 16); + __m512i R17 = _mm512_load_si512(pA + 8 * 17); + __m512i R18 = _mm512_load_si512(pA + 8 * 18); + __m512i R19 = _mm512_load_si512(pA + 8 * 19); + __m512i R20 = _mm512_load_si512(pA + 8 * 20); + __m512i R21 = _mm512_load_si512(pA + 8 * 21); + __m512i R22 = _mm512_load_si512(pA + 8 * 22); + __m512i R23 = _mm512_load_si512(pA + 8 * 23); + __m512i R24 = _mm512_load_si512(pA + 8 * 24); + __m512i R25 = _mm512_load_si512(pA + 8 * 25); + __m512i R26 = _mm512_load_si512(pA + 8 * 26); + __m512i R27 = _mm512_load_si512(pA + 8 * 27); + __m512i R28 = _mm512_load_si512(pA + 8 * 28); + __m512i R29 = _mm512_load_si512(pA + 8 * 29); + + int itr; + for (itr = 0, pA += 8 * 30; itr < 30; itr++, pA += 8) { + __m512i Yi = _mm512_madd52lo_epu64(_mm512_setzero_si512(), R00, K); + __m512i nxtA = _mm512_load_si512(pA); + + _mm512_madd52lo_epu64_(R00, R00, Yi, pM, 64 * 0); + _mm512_madd52lo_epu64_(R01, R01, Yi, pM, 64 * 1); + _mm512_madd52lo_epu64_(R02, R02, Yi, pM, 64 * 2); + _mm512_madd52lo_epu64_(R03, R03, Yi, pM, 64 * 3); + _mm512_madd52lo_epu64_(R04, R04, Yi, pM, 64 * 4); + _mm512_madd52lo_epu64_(R05, R05, Yi, pM, 64 * 5); + _mm512_madd52lo_epu64_(R06, R06, Yi, pM, 64 * 6); + _mm512_madd52lo_epu64_(R07, R07, Yi, pM, 64 * 7); + _mm512_madd52lo_epu64_(R08, R08, Yi, pM, 64 * 8); + _mm512_madd52lo_epu64_(R09, R09, Yi, pM, 64 * 9); + _mm512_madd52lo_epu64_(R10, R10, Yi, pM, 64 * 10); + _mm512_madd52lo_epu64_(R11, R11, Yi, pM, 64 * 11); + _mm512_madd52lo_epu64_(R12, R12, Yi, pM, 64 * 12); + _mm512_madd52lo_epu64_(R13, R13, Yi, pM, 64 * 13); + _mm512_madd52lo_epu64_(R14, R14, Yi, pM, 64 * 14); + _mm512_madd52lo_epu64_(R15, R15, Yi, pM, 64 * 15); + _mm512_madd52lo_epu64_(R16, R16, Yi, pM, 64 * 16); + _mm512_madd52lo_epu64_(R17, R17, Yi, pM, 64 * 17); + _mm512_madd52lo_epu64_(R18, R18, Yi, pM, 64 * 18); + _mm512_madd52lo_epu64_(R19, R19, Yi, pM, 64 * 19); + _mm512_madd52lo_epu64_(R20, R20, Yi, pM, 64 * 20); + _mm512_madd52lo_epu64_(R21, R21, Yi, pM, 64 * 21); + _mm512_madd52lo_epu64_(R22, R22, Yi, pM, 64 * 22); + _mm512_madd52lo_epu64_(R23, R23, Yi, pM, 64 * 23); + _mm512_madd52lo_epu64_(R24, R24, Yi, pM, 64 * 24); + _mm512_madd52lo_epu64_(R25, R25, Yi, pM, 64 * 25); + _mm512_madd52lo_epu64_(R26, R26, Yi, pM, 64 * 26); + _mm512_madd52lo_epu64_(R27, R27, Yi, pM, 64 * 27); + _mm512_madd52lo_epu64_(R28, R28, Yi, pM, 64 * 28); + _mm512_madd52lo_epu64_(R29, R29, Yi, pM, 64 * 29); + + R00 = _mm512_srli_epi64(R00, DIGIT_SIZE); + R01 = _mm512_add_epi64(R01, R00); + + _mm512_madd52hi_epu64_(R00, R01, Yi, pM, 64 * 0); + _mm512_madd52hi_epu64_(R01, R02, Yi, pM, 64 * 1); + _mm512_madd52hi_epu64_(R02, R03, Yi, pM, 64 * 2); + _mm512_madd52hi_epu64_(R03, R04, Yi, pM, 64 * 3); + _mm512_madd52hi_epu64_(R04, R05, Yi, pM, 64 * 4); + _mm512_madd52hi_epu64_(R05, R06, Yi, pM, 64 * 5); + _mm512_madd52hi_epu64_(R06, R07, Yi, pM, 64 * 6); + _mm512_madd52hi_epu64_(R07, R08, Yi, pM, 64 * 7); + _mm512_madd52hi_epu64_(R08, R09, Yi, pM, 64 * 8); + _mm512_madd52hi_epu64_(R09, R10, Yi, pM, 64 * 9); + _mm512_madd52hi_epu64_(R10, R11, Yi, pM, 64 * 10); + _mm512_madd52hi_epu64_(R11, R12, Yi, pM, 64 * 11); + _mm512_madd52hi_epu64_(R12, R13, Yi, pM, 64 * 12); + _mm512_madd52hi_epu64_(R13, R14, Yi, pM, 64 * 13); + _mm512_madd52hi_epu64_(R14, R15, Yi, pM, 64 * 14); + _mm512_madd52hi_epu64_(R15, R16, Yi, pM, 64 * 15); + _mm512_madd52hi_epu64_(R16, R17, Yi, pM, 64 * 16); + _mm512_madd52hi_epu64_(R17, R18, Yi, pM, 64 * 17); + _mm512_madd52hi_epu64_(R18, R19, Yi, pM, 64 * 18); + _mm512_madd52hi_epu64_(R19, R20, Yi, pM, 64 * 19); + _mm512_madd52hi_epu64_(R20, R21, Yi, pM, 64 * 20); + _mm512_madd52hi_epu64_(R21, R22, Yi, pM, 64 * 21); + _mm512_madd52hi_epu64_(R22, R23, Yi, pM, 64 * 22); + _mm512_madd52hi_epu64_(R23, R24, Yi, pM, 64 * 23); + _mm512_madd52hi_epu64_(R24, R25, Yi, pM, 64 * 24); + _mm512_madd52hi_epu64_(R25, R26, Yi, pM, 64 * 25); + _mm512_madd52hi_epu64_(R26, R27, Yi, pM, 64 * 26); + _mm512_madd52hi_epu64_(R27, R28, Yi, pM, 64 * 27); + _mm512_madd52hi_epu64_(R28, R29, Yi, pM, 64 * 28); + _mm512_madd52hi_epu64_(R29, nxtA, Yi, pM, 64 * 29); + } + + /* normalization */ + { + __m512i MASK = _mm512_set1_epi64(DIGIT_MASK); + + __m512i + T = _mm512_srli_epi64(R00, DIGIT_SIZE); + R00 = _mm512_and_epi64(R00, MASK); + _mm512_store_si512(pR + 8 * 0, R00); + + R01 = _mm512_add_epi64(R01, T); + T = _mm512_srli_epi64(R01, DIGIT_SIZE); + R01 = _mm512_and_epi64(R01, MASK); + _mm512_store_si512(pR + 8 * 1, R01); + + R02 = _mm512_add_epi64(R02, T); + T = _mm512_srli_epi64(R02, DIGIT_SIZE); + R02 = _mm512_and_epi64(R02, MASK); + _mm512_store_si512(pR + 8 * 2, R02); + + R03 = _mm512_add_epi64(R03, T); + T = _mm512_srli_epi64(R03, DIGIT_SIZE); + R03 = _mm512_and_epi64(R03, MASK); + _mm512_store_si512(pR + 8 * 3, R03); + + R04 = _mm512_add_epi64(R04, T); + T = _mm512_srli_epi64(R04, DIGIT_SIZE); + R04 = _mm512_and_epi64(R04, MASK); + _mm512_store_si512(pR + 8 * 4, R04); + + R05 = _mm512_add_epi64(R05, T); + T = _mm512_srli_epi64(R05, DIGIT_SIZE); + R05 = _mm512_and_epi64(R05, MASK); + _mm512_store_si512(pR + 8 * 5, R05); + + R06 = _mm512_add_epi64(R06, T); + T = _mm512_srli_epi64(R06, DIGIT_SIZE); + R06 = _mm512_and_epi64(R06, MASK); + _mm512_store_si512(pR + 8 * 6, R06); + + R07 = _mm512_add_epi64(R07, T); + T = _mm512_srli_epi64(R07, DIGIT_SIZE); + R07 = _mm512_and_epi64(R07, MASK); + _mm512_store_si512(pR + 8 * 7, R07); + + R08 = _mm512_add_epi64(R08, T); + T = _mm512_srli_epi64(R08, DIGIT_SIZE); + R08 = _mm512_and_epi64(R08, MASK); + _mm512_store_si512(pR + 8 * 8, R08); + + R09 = _mm512_add_epi64(R09, T); + T = _mm512_srli_epi64(R09, DIGIT_SIZE); + R09 = _mm512_and_epi64(R09, MASK); + _mm512_store_si512(pR + 8 * 9, R09); + + R10 = _mm512_add_epi64(R10, T); + T = _mm512_srli_epi64(R10, DIGIT_SIZE); + R10 = _mm512_and_epi64(R10, MASK); + _mm512_store_si512(pR + 8 * 10, R10); + + R11 = _mm512_add_epi64(R11, T); + T = _mm512_srli_epi64(R11, DIGIT_SIZE); + R11 = _mm512_and_epi64(R11, MASK); + _mm512_store_si512(pR + 8 * 11, R11); + + R12 = _mm512_add_epi64(R12, T); + T = _mm512_srli_epi64(R12, DIGIT_SIZE); + R12 = _mm512_and_epi64(R12, MASK); + _mm512_store_si512(pR + 8 * 12, R12); + + R13 = _mm512_add_epi64(R13, T); + T = _mm512_srli_epi64(R13, DIGIT_SIZE); + R13 = _mm512_and_epi64(R13, MASK); + _mm512_store_si512(pR + 8 * 13, R13); + + R14 = _mm512_add_epi64(R14, T); + T = _mm512_srli_epi64(R14, DIGIT_SIZE); + R14 = _mm512_and_epi64(R14, MASK); + _mm512_store_si512(pR + 8 * 14, R14); + + R15 = _mm512_add_epi64(R15, T); + T = _mm512_srli_epi64(R15, DIGIT_SIZE); + R15 = _mm512_and_epi64(R15, MASK); + _mm512_store_si512(pR + 8 * 15, R15); + + R16 = _mm512_add_epi64(R16, T); + T = _mm512_srli_epi64(R16, DIGIT_SIZE); + R16 = _mm512_and_epi64(R16, MASK); + _mm512_store_si512(pR + 8 * 16, R16); + + R17 = _mm512_add_epi64(R17, T); + T = _mm512_srli_epi64(R17, DIGIT_SIZE); + R17 = _mm512_and_epi64(R17, MASK); + _mm512_store_si512(pR + 8 * 17, R17); + + R18 = _mm512_add_epi64(R18, T); + T = _mm512_srli_epi64(R18, DIGIT_SIZE); + R18 = _mm512_and_epi64(R18, MASK); + _mm512_store_si512(pR + 8 * 18, R18); + + R19 = _mm512_add_epi64(R19, T); + T = _mm512_srli_epi64(R19, DIGIT_SIZE); + R19 = _mm512_and_epi64(R19, MASK); + _mm512_store_si512(pR + 8 * 19, R19); + + R20 = _mm512_add_epi64(R20, T); + T = _mm512_srli_epi64(R20, DIGIT_SIZE); + R20 = _mm512_and_epi64(R20, MASK); + _mm512_store_si512(pR + 8 * 20, R20); + + R21 = _mm512_add_epi64(R21, T); + T = _mm512_srli_epi64(R21, DIGIT_SIZE); + R21 = _mm512_and_epi64(R21, MASK); + _mm512_store_si512(pR + 8 * 21, R21); + + R22 = _mm512_add_epi64(R22, T); + T = _mm512_srli_epi64(R22, DIGIT_SIZE); + R22 = _mm512_and_epi64(R22, MASK); + _mm512_store_si512(pR + 8 * 22, R22); + + R23 = _mm512_add_epi64(R23, T); + T = _mm512_srli_epi64(R23, DIGIT_SIZE); + R23 = _mm512_and_epi64(R23, MASK); + _mm512_store_si512(pR + 8 * 23, R23); + + R24 = _mm512_add_epi64(R24, T); + T = _mm512_srli_epi64(R24, DIGIT_SIZE); + R24 = _mm512_and_epi64(R24, MASK); + _mm512_store_si512(pR + 8 * 24, R24); + + R25 = _mm512_add_epi64(R25, T); + T = _mm512_srli_epi64(R25, DIGIT_SIZE); + R25 = _mm512_and_epi64(R25, MASK); + _mm512_store_si512(pR + 8 * 25, R25); + + R26 = _mm512_add_epi64(R26, T); + T = _mm512_srli_epi64(R26, DIGIT_SIZE); + R26 = _mm512_and_epi64(R26, MASK); + _mm512_store_si512(pR + 8 * 26, R26); + + R27 = _mm512_add_epi64(R27, T); + T = _mm512_srli_epi64(R27, DIGIT_SIZE); + R27 = _mm512_and_epi64(R27, MASK); + _mm512_store_si512(pR + 8 * 27, R27); + + R28 = _mm512_add_epi64(R28, T); + T = _mm512_srli_epi64(R28, DIGIT_SIZE); + R28 = _mm512_and_epi64(R28, MASK); + _mm512_store_si512(pR + 8 * 28, R28); + + R29 = _mm512_add_epi64(R29, T); + T = _mm512_srli_epi64(R29, DIGIT_SIZE); + R29 = _mm512_and_epi64(R29, MASK); + _mm512_store_si512(pR + 8 * 29, R29); + } +} + +void ifma_amred52x40_mb8(int64u res[][8], + const int64u inpA[][8], /* int nsA == 2*nsM */ + const int64u inpM[][8], /* int nsM ==20 */ + const int64u k0[8]) +{ + int64u* pA = (int64u*)inpA; + int64u* pM = (int64u*)inpM; + int64u* pR = (int64u*)res; + + __m512i K = _mm512_load_si512(k0); /* k0[] */ + + __m512i R00 = _mm512_load_si512(pA + 8 * 0); /* load A[nsM-1],...,A[0] */ + __m512i R01 = _mm512_load_si512(pA + 8 * 1); + __m512i R02 = _mm512_load_si512(pA + 8 * 2); + __m512i R03 = _mm512_load_si512(pA + 8 * 3); + __m512i R04 = _mm512_load_si512(pA + 8 * 4); + __m512i R05 = _mm512_load_si512(pA + 8 * 5); + __m512i R06 = _mm512_load_si512(pA + 8 * 6); + __m512i R07 = _mm512_load_si512(pA + 8 * 7); + __m512i R08 = _mm512_load_si512(pA + 8 * 8); + __m512i R09 = _mm512_load_si512(pA + 8 * 9); + __m512i R10 = _mm512_load_si512(pA + 8 * 10); + __m512i R11 = _mm512_load_si512(pA + 8 * 11); + __m512i R12 = _mm512_load_si512(pA + 8 * 12); + __m512i R13 = _mm512_load_si512(pA + 8 * 13); + __m512i R14 = _mm512_load_si512(pA + 8 * 14); + __m512i R15 = _mm512_load_si512(pA + 8 * 15); + __m512i R16 = _mm512_load_si512(pA + 8 * 16); + __m512i R17 = _mm512_load_si512(pA + 8 * 17); + __m512i R18 = _mm512_load_si512(pA + 8 * 18); + __m512i R19 = _mm512_load_si512(pA + 8 * 19); + __m512i R20 = _mm512_load_si512(pA + 8 * 20); + __m512i R21 = _mm512_load_si512(pA + 8 * 21); + __m512i R22 = _mm512_load_si512(pA + 8 * 22); + __m512i R23 = _mm512_load_si512(pA + 8 * 23); + __m512i R24 = _mm512_load_si512(pA + 8 * 24); + __m512i R25 = _mm512_load_si512(pA + 8 * 25); + __m512i R26 = _mm512_load_si512(pA + 8 * 26); + __m512i R27 = _mm512_load_si512(pA + 8 * 27); + __m512i R28 = _mm512_load_si512(pA + 8 * 28); + __m512i R29 = _mm512_load_si512(pA + 8 * 29); + __m512i R30 = _mm512_load_si512(pA + 8 * 30); + __m512i R31 = _mm512_load_si512(pA + 8 * 31); + __m512i R32 = _mm512_load_si512(pA + 8 * 32); + __m512i R33 = _mm512_load_si512(pA + 8 * 33); + __m512i R34 = _mm512_load_si512(pA + 8 * 34); + __m512i R35 = _mm512_load_si512(pA + 8 * 35); + __m512i R36 = _mm512_load_si512(pA + 8 * 36); + __m512i R37 = _mm512_load_si512(pA + 8 * 37); + __m512i R38 = _mm512_load_si512(pA + 8 * 38); + __m512i R39 = _mm512_load_si512(pA + 8 * 39); + + int itr; + for (itr = 0, pA += 8 * 40; itr < 40; itr++, pA += 8) { + __m512i Yi = _mm512_madd52lo_epu64(_mm512_setzero_si512(), R00, K); + __m512i nxtA = _mm512_load_si512(pA); + + _mm512_madd52lo_epu64_(R00, R00, Yi, pM, 64 * 0); + _mm512_madd52lo_epu64_(R01, R01, Yi, pM, 64 * 1); + _mm512_madd52lo_epu64_(R02, R02, Yi, pM, 64 * 2); + _mm512_madd52lo_epu64_(R03, R03, Yi, pM, 64 * 3); + _mm512_madd52lo_epu64_(R04, R04, Yi, pM, 64 * 4); + _mm512_madd52lo_epu64_(R05, R05, Yi, pM, 64 * 5); + _mm512_madd52lo_epu64_(R06, R06, Yi, pM, 64 * 6); + _mm512_madd52lo_epu64_(R07, R07, Yi, pM, 64 * 7); + _mm512_madd52lo_epu64_(R08, R08, Yi, pM, 64 * 8); + _mm512_madd52lo_epu64_(R09, R09, Yi, pM, 64 * 9); + _mm512_madd52lo_epu64_(R10, R10, Yi, pM, 64 * 10); + _mm512_madd52lo_epu64_(R11, R11, Yi, pM, 64 * 11); + _mm512_madd52lo_epu64_(R12, R12, Yi, pM, 64 * 12); + _mm512_madd52lo_epu64_(R13, R13, Yi, pM, 64 * 13); + _mm512_madd52lo_epu64_(R14, R14, Yi, pM, 64 * 14); + _mm512_madd52lo_epu64_(R15, R15, Yi, pM, 64 * 15); + _mm512_madd52lo_epu64_(R16, R16, Yi, pM, 64 * 16); + _mm512_madd52lo_epu64_(R17, R17, Yi, pM, 64 * 17); + _mm512_madd52lo_epu64_(R18, R18, Yi, pM, 64 * 18); + _mm512_madd52lo_epu64_(R19, R19, Yi, pM, 64 * 19); + _mm512_madd52lo_epu64_(R20, R20, Yi, pM, 64 * 20); + _mm512_madd52lo_epu64_(R21, R21, Yi, pM, 64 * 21); + _mm512_madd52lo_epu64_(R22, R22, Yi, pM, 64 * 22); + _mm512_madd52lo_epu64_(R23, R23, Yi, pM, 64 * 23); + _mm512_madd52lo_epu64_(R24, R24, Yi, pM, 64 * 24); + _mm512_madd52lo_epu64_(R25, R25, Yi, pM, 64 * 25); + _mm512_madd52lo_epu64_(R26, R26, Yi, pM, 64 * 26); + _mm512_madd52lo_epu64_(R27, R27, Yi, pM, 64 * 27); + _mm512_madd52lo_epu64_(R28, R28, Yi, pM, 64 * 28); + _mm512_madd52lo_epu64_(R29, R29, Yi, pM, 64 * 29); + _mm512_madd52lo_epu64_(R30, R30, Yi, pM, 64 * 30); + _mm512_madd52lo_epu64_(R31, R31, Yi, pM, 64 * 31); + _mm512_madd52lo_epu64_(R32, R32, Yi, pM, 64 * 32); + _mm512_madd52lo_epu64_(R33, R33, Yi, pM, 64 * 33); + _mm512_madd52lo_epu64_(R34, R34, Yi, pM, 64 * 34); + _mm512_madd52lo_epu64_(R35, R35, Yi, pM, 64 * 35); + _mm512_madd52lo_epu64_(R36, R36, Yi, pM, 64 * 36); + _mm512_madd52lo_epu64_(R37, R37, Yi, pM, 64 * 37); + _mm512_madd52lo_epu64_(R38, R38, Yi, pM, 64 * 38); + _mm512_madd52lo_epu64_(R39, R39, Yi, pM, 64 * 39); + + R00 = _mm512_srli_epi64(R00, DIGIT_SIZE); + R01 = _mm512_add_epi64(R01, R00); + + _mm512_madd52hi_epu64_(R00, R01, Yi, pM, 64 * 0); + _mm512_madd52hi_epu64_(R01, R02, Yi, pM, 64 * 1); + _mm512_madd52hi_epu64_(R02, R03, Yi, pM, 64 * 2); + _mm512_madd52hi_epu64_(R03, R04, Yi, pM, 64 * 3); + _mm512_madd52hi_epu64_(R04, R05, Yi, pM, 64 * 4); + _mm512_madd52hi_epu64_(R05, R06, Yi, pM, 64 * 5); + _mm512_madd52hi_epu64_(R06, R07, Yi, pM, 64 * 6); + _mm512_madd52hi_epu64_(R07, R08, Yi, pM, 64 * 7); + _mm512_madd52hi_epu64_(R08, R09, Yi, pM, 64 * 8); + _mm512_madd52hi_epu64_(R09, R10, Yi, pM, 64 * 9); + _mm512_madd52hi_epu64_(R10, R11, Yi, pM, 64 * 10); + _mm512_madd52hi_epu64_(R11, R12, Yi, pM, 64 * 11); + _mm512_madd52hi_epu64_(R12, R13, Yi, pM, 64 * 12); + _mm512_madd52hi_epu64_(R13, R14, Yi, pM, 64 * 13); + _mm512_madd52hi_epu64_(R14, R15, Yi, pM, 64 * 14); + _mm512_madd52hi_epu64_(R15, R16, Yi, pM, 64 * 15); + _mm512_madd52hi_epu64_(R16, R17, Yi, pM, 64 * 16); + _mm512_madd52hi_epu64_(R17, R18, Yi, pM, 64 * 17); + _mm512_madd52hi_epu64_(R18, R19, Yi, pM, 64 * 18); + _mm512_madd52hi_epu64_(R19, R20, Yi, pM, 64 * 19); + _mm512_madd52hi_epu64_(R20, R21, Yi, pM, 64 * 20); + _mm512_madd52hi_epu64_(R21, R22, Yi, pM, 64 * 21); + _mm512_madd52hi_epu64_(R22, R23, Yi, pM, 64 * 22); + _mm512_madd52hi_epu64_(R23, R24, Yi, pM, 64 * 23); + _mm512_madd52hi_epu64_(R24, R25, Yi, pM, 64 * 24); + _mm512_madd52hi_epu64_(R25, R26, Yi, pM, 64 * 25); + _mm512_madd52hi_epu64_(R26, R27, Yi, pM, 64 * 26); + _mm512_madd52hi_epu64_(R27, R28, Yi, pM, 64 * 27); + _mm512_madd52hi_epu64_(R28, R29, Yi, pM, 64 * 28); + _mm512_madd52hi_epu64_(R29, R30, Yi, pM, 64 * 29); + _mm512_madd52hi_epu64_(R30, R31, Yi, pM, 64 * 30); + _mm512_madd52hi_epu64_(R31, R32, Yi, pM, 64 * 31); + _mm512_madd52hi_epu64_(R32, R33, Yi, pM, 64 * 32); + _mm512_madd52hi_epu64_(R33, R34, Yi, pM, 64 * 33); + _mm512_madd52hi_epu64_(R34, R35, Yi, pM, 64 * 34); + _mm512_madd52hi_epu64_(R35, R36, Yi, pM, 64 * 35); + _mm512_madd52hi_epu64_(R36, R37, Yi, pM, 64 * 36); + _mm512_madd52hi_epu64_(R37, R38, Yi, pM, 64 * 37); + _mm512_madd52hi_epu64_(R38, R39, Yi, pM, 64 * 38); + _mm512_madd52hi_epu64_(R39, nxtA, Yi, pM, 64 * 39); + } + + /* normalization */ + { + __m512i MASK = _mm512_set1_epi64(DIGIT_MASK); + + __m512i + T = _mm512_srli_epi64(R00, DIGIT_SIZE); + R00 = _mm512_and_epi64(R00, MASK); + _mm512_store_si512(pR + 8 * 0, R00); + + R01 = _mm512_add_epi64(R01, T); + T = _mm512_srli_epi64(R01, DIGIT_SIZE); + R01 = _mm512_and_epi64(R01, MASK); + _mm512_store_si512(pR + 8 * 1, R01); + + R02 = _mm512_add_epi64(R02, T); + T = _mm512_srli_epi64(R02, DIGIT_SIZE); + R02 = _mm512_and_epi64(R02, MASK); + _mm512_store_si512(pR + 8 * 2, R02); + + R03 = _mm512_add_epi64(R03, T); + T = _mm512_srli_epi64(R03, DIGIT_SIZE); + R03 = _mm512_and_epi64(R03, MASK); + _mm512_store_si512(pR + 8 * 3, R03); + + R04 = _mm512_add_epi64(R04, T); + T = _mm512_srli_epi64(R04, DIGIT_SIZE); + R04 = _mm512_and_epi64(R04, MASK); + _mm512_store_si512(pR + 8 * 4, R04); + + R05 = _mm512_add_epi64(R05, T); + T = _mm512_srli_epi64(R05, DIGIT_SIZE); + R05 = _mm512_and_epi64(R05, MASK); + _mm512_store_si512(pR + 8 * 5, R05); + + R06 = _mm512_add_epi64(R06, T); + T = _mm512_srli_epi64(R06, DIGIT_SIZE); + R06 = _mm512_and_epi64(R06, MASK); + _mm512_store_si512(pR + 8 * 6, R06); + + R07 = _mm512_add_epi64(R07, T); + T = _mm512_srli_epi64(R07, DIGIT_SIZE); + R07 = _mm512_and_epi64(R07, MASK); + _mm512_store_si512(pR + 8 * 7, R07); + + R08 = _mm512_add_epi64(R08, T); + T = _mm512_srli_epi64(R08, DIGIT_SIZE); + R08 = _mm512_and_epi64(R08, MASK); + _mm512_store_si512(pR + 8 * 8, R08); + + R09 = _mm512_add_epi64(R09, T); + T = _mm512_srli_epi64(R09, DIGIT_SIZE); + R09 = _mm512_and_epi64(R09, MASK); + _mm512_store_si512(pR + 8 * 9, R09); + + R10 = _mm512_add_epi64(R10, T); + T = _mm512_srli_epi64(R10, DIGIT_SIZE); + R10 = _mm512_and_epi64(R10, MASK); + _mm512_store_si512(pR + 8 * 10, R10); + + R11 = _mm512_add_epi64(R11, T); + T = _mm512_srli_epi64(R11, DIGIT_SIZE); + R11 = _mm512_and_epi64(R11, MASK); + _mm512_store_si512(pR + 8 * 11, R11); + + R12 = _mm512_add_epi64(R12, T); + T = _mm512_srli_epi64(R12, DIGIT_SIZE); + R12 = _mm512_and_epi64(R12, MASK); + _mm512_store_si512(pR + 8 * 12, R12); + + R13 = _mm512_add_epi64(R13, T); + T = _mm512_srli_epi64(R13, DIGIT_SIZE); + R13 = _mm512_and_epi64(R13, MASK); + _mm512_store_si512(pR + 8 * 13, R13); + + R14 = _mm512_add_epi64(R14, T); + T = _mm512_srli_epi64(R14, DIGIT_SIZE); + R14 = _mm512_and_epi64(R14, MASK); + _mm512_store_si512(pR + 8 * 14, R14); + + R15 = _mm512_add_epi64(R15, T); + T = _mm512_srli_epi64(R15, DIGIT_SIZE); + R15 = _mm512_and_epi64(R15, MASK); + _mm512_store_si512(pR + 8 * 15, R15); + + R16 = _mm512_add_epi64(R16, T); + T = _mm512_srli_epi64(R16, DIGIT_SIZE); + R16 = _mm512_and_epi64(R16, MASK); + _mm512_store_si512(pR + 8 * 16, R16); + + R17 = _mm512_add_epi64(R17, T); + T = _mm512_srli_epi64(R17, DIGIT_SIZE); + R17 = _mm512_and_epi64(R17, MASK); + _mm512_store_si512(pR + 8 * 17, R17); + + R18 = _mm512_add_epi64(R18, T); + T = _mm512_srli_epi64(R18, DIGIT_SIZE); + R18 = _mm512_and_epi64(R18, MASK); + _mm512_store_si512(pR + 8 * 18, R18); + + R19 = _mm512_add_epi64(R19, T); + T = _mm512_srli_epi64(R19, DIGIT_SIZE); + R19 = _mm512_and_epi64(R19, MASK); + _mm512_store_si512(pR + 8 * 19, R19); + + R20 = _mm512_add_epi64(R20, T); + T = _mm512_srli_epi64(R20, DIGIT_SIZE); + R20 = _mm512_and_epi64(R20, MASK); + _mm512_store_si512(pR + 8 * 20, R20); + + R21 = _mm512_add_epi64(R21, T); + T = _mm512_srli_epi64(R21, DIGIT_SIZE); + R21 = _mm512_and_epi64(R21, MASK); + _mm512_store_si512(pR + 8 * 21, R21); + + R22 = _mm512_add_epi64(R22, T); + T = _mm512_srli_epi64(R22, DIGIT_SIZE); + R22 = _mm512_and_epi64(R22, MASK); + _mm512_store_si512(pR + 8 * 22, R22); + + R23 = _mm512_add_epi64(R23, T); + T = _mm512_srli_epi64(R23, DIGIT_SIZE); + R23 = _mm512_and_epi64(R23, MASK); + _mm512_store_si512(pR + 8 * 23, R23); + + R24 = _mm512_add_epi64(R24, T); + T = _mm512_srli_epi64(R24, DIGIT_SIZE); + R24 = _mm512_and_epi64(R24, MASK); + _mm512_store_si512(pR + 8 * 24, R24); + + R25 = _mm512_add_epi64(R25, T); + T = _mm512_srli_epi64(R25, DIGIT_SIZE); + R25 = _mm512_and_epi64(R25, MASK); + _mm512_store_si512(pR + 8 * 25, R25); + + R26 = _mm512_add_epi64(R26, T); + T = _mm512_srli_epi64(R26, DIGIT_SIZE); + R26 = _mm512_and_epi64(R26, MASK); + _mm512_store_si512(pR + 8 * 26, R26); + + R27 = _mm512_add_epi64(R27, T); + T = _mm512_srli_epi64(R27, DIGIT_SIZE); + R27 = _mm512_and_epi64(R27, MASK); + _mm512_store_si512(pR + 8 * 27, R27); + + R28 = _mm512_add_epi64(R28, T); + T = _mm512_srli_epi64(R28, DIGIT_SIZE); + R28 = _mm512_and_epi64(R28, MASK); + _mm512_store_si512(pR + 8 * 28, R28); + + R29 = _mm512_add_epi64(R29, T); + T = _mm512_srli_epi64(R29, DIGIT_SIZE); + R29 = _mm512_and_epi64(R29, MASK); + _mm512_store_si512(pR + 8 * 29, R29); + + R30 = _mm512_add_epi64(R30, T); + T = _mm512_srli_epi64(R30, DIGIT_SIZE); + R30 = _mm512_and_epi64(R30, MASK); + _mm512_store_si512(pR + 8 * 30, R30); + + R31 = _mm512_add_epi64(R31, T); + T = _mm512_srli_epi64(R31, DIGIT_SIZE); + R31 = _mm512_and_epi64(R31, MASK); + _mm512_store_si512(pR + 8 * 31, R31); + + R32 = _mm512_add_epi64(R32, T); + T = _mm512_srli_epi64(R32, DIGIT_SIZE); + R32 = _mm512_and_epi64(R32, MASK); + _mm512_store_si512(pR + 8 * 32, R32); + + R33 = _mm512_add_epi64(R33, T); + T = _mm512_srli_epi64(R33, DIGIT_SIZE); + R33 = _mm512_and_epi64(R33, MASK); + _mm512_store_si512(pR + 8 * 33, R33); + + R34 = _mm512_add_epi64(R34, T); + T = _mm512_srli_epi64(R34, DIGIT_SIZE); + R34 = _mm512_and_epi64(R34, MASK); + _mm512_store_si512(pR + 8 * 34, R34); + + R35 = _mm512_add_epi64(R35, T); + T = _mm512_srli_epi64(R35, DIGIT_SIZE); + R35 = _mm512_and_epi64(R35, MASK); + _mm512_store_si512(pR + 8 * 35, R35); + + R36 = _mm512_add_epi64(R36, T); + T = _mm512_srli_epi64(R36, DIGIT_SIZE); + R36 = _mm512_and_epi64(R36, MASK); + _mm512_store_si512(pR + 8 * 36, R36); + + R37 = _mm512_add_epi64(R37, T); + T = _mm512_srli_epi64(R37, DIGIT_SIZE); + R37 = _mm512_and_epi64(R37, MASK); + _mm512_store_si512(pR + 8 * 37, R37); + + R38 = _mm512_add_epi64(R38, T); + T = _mm512_srli_epi64(R38, DIGIT_SIZE); + R38 = _mm512_and_epi64(R38, MASK); + _mm512_store_si512(pR + 8 * 38, R38); + + R39 = _mm512_add_epi64(R39, T); + T = _mm512_srli_epi64(R39, DIGIT_SIZE); + R39 = _mm512_and_epi64(R39, MASK); + _mm512_store_si512(pR + 8 * 39, R39); + } +} +////////////////////////////////////////////////////////////////////// + +/* +// out[] = inp[] <>nbit */ +static void rshift52x_mb8(int64u pOut[][8], int64u pInp[][8], int ns, __m512i sbiR) +{ + __m512i sbiL = _mm512_sub_epi64(_mm512_set1_epi64(DIGIT_SIZE), sbiR); + __m512i digMask = _mm512_set1_epi64(DIGIT_MASK); + + __m512i shiftedL = _mm512_setzero_si512(); + int n; + for(n=ns; n>0; n--) { + __m512i inp = _mm512_load_si512(pInp[n-1]); + __m512i out = _mm512_and_si512(_mm512_or_si512(shiftedL, _mm512_srlv_epi64(inp,sbiR)), digMask); + _mm512_store_si512(pOut[n-1], out); + shiftedL = _mm512_sllv_epi64(inp, sbiL); + } +} + +/* +// believe the following works: +// +// right = {r1:r0} - high and low +// left = {l1:l0} - high and low +// +// if(r1!=l1) +// return r1>l1; +// else +// return r0>l0; +*/ +static __mmask8 left_gt_right_mb8(__m512i left_hi, __m512i left_lo, __m512i right_hi, __m512i right_lo) +{ + __mmask8 k0 = _mm512_cmpneq_epi64_mask(left_hi, right_hi); + __mmask8 k1 = ( k0 & _mm512_cmpgt_epu64_mask(left_hi, right_hi)) + | (~k0 & _mm512_cmpgt_epu64_mask(left_lo, right_lo)); + return k1; +} + +__m512i tz; + +/* sub multiplied_by_digit +// maskk8 reports whether addition performed inside +*/ +static __mmask8 ifma_sub_muldig52x_mb8(__m512i* pRes, const __m512i* pM, int nsM, __m512i dig) +{ + __m512i MASK = _mm512_set1_epi64(DIGIT_MASK); + + __m512i prodLO = _mm512_setzero_si512(); + __m512i prodHI = _mm512_setzero_si512(); + __m512i cf = _mm512_setzero_si512(); + __m512i T; + __mmask8 k1; + + int n; + + /* multiply dig*pM[] and sub from pRes[] */ + for(n=0; n pair of digits {r:x2} + + // k1 = left>right + k1 = left_gt_right_mb8(left_hi, left_lo, rem, x2); + + // if(left>right) + quo = _mm512_mask_sub_epi64(quo, k1, quo, one); // quo -= 1 + rem = _mm512_mask_add_epi64(rem, k1, rem, y0); // rem += y0 , note "right" increased at the same time + + k0 = _mm512_mask_cmpgt_epu64_mask(k1, y1, left_lo); // k0 = left_lo < y1 + left_lo = _mm512_mask_sub_epi64(left_lo, k1, left_lo, y1); // left -= y1 + left_hi = _mm512_mask_sub_epi64(left_hi, (k1&k0), left_hi, one); + + // k0 = rem < B + k0 = _mm512_cmplt_epu64_mask(rem, base52); + // k1 = left>right + k1 = k0 & left_gt_right_mb8(left_hi, left_lo, rem, x2); + + quo = _mm512_mask_sub_epi64(quo, k1, quo, one); // quo -= 1 + + return quo; +} + +/* x = x % m */ +void ifma_mreduce52x_mb8(int64u pX[][8], int nsX, int64u pM[][8], int nsM) +{ + /* usually divider have to be normilized */ + __m512i* pMtop = (__m512i*)(pM[nsM-1]); /* top of M */ + __m512i normBits = _mm512_sub_epi64( + _mm512_lzcnt_epi64(pMtop[0]), + _mm512_set1_epi64(64-DIGIT_SIZE) + ); + tz = _mm512_setzero_si512(); + /* normalize both divisor and shift dividentr */ + lshift52x_mb8(pM, pM, nsM, normBits); + /* expand and shift X */ + _mm512_store_si512(pX[nsX], _mm512_setzero_si512()); + lshift52x_mb8(pX, pX, nsX+1, normBits); + + // division + { + __m512i* pXtop = (__m512i*)(pX[nsX]); /* top of X (zero value) -- &X[nsX] */ + __m512i* pXbot = (__m512i*)(pX[nsX-nsM]); /* bot of X -- &X[nsQ-1] */ + + while(pXbot>=(__m512i*)pX) { + /* compute estimation of quotient digit q */ + __m512i q = estimateq_mb8(pXtop, pMtop); + + /* multiply and sub */ + ifma_sub_muldig52x_mb8(pXbot, (__m512i*)pM, nsM, q); + + pXtop--; + pXbot--; + } + } + + rshift52x_mb8(pX, pX, nsM, normBits); + rshift52x_mb8(pM, pM, nsM, normBits); +} + +/* bitsize of 2^64 representation => bitsize of 2^52 representation */ +#define BASE52_BITSIZE(b64bitsize) ((b64bitsize) + ((DIGIT_SIZE - ((b64bitsize) % DIGIT_SIZE)) % DIGIT_SIZE)) + +/* rr = 2^(2*ifmaBitLen) mod m */ +void ifma_montRR52x_mb8 (int64u pRR[][8], int64u pM[][8], int convBitLen) +{ + #define MAX_IFMA_MODULUS_BITLEN BASE52_BITSIZE(RSA_4K) + + /* buffer to hold 2^(2*MAX_IFMA_MODULUS_BITLEN) */ + __ALIGN64 int64u pwr2_mb8[(NUMBER_OF_DIGITS(2 * MAX_IFMA_MODULUS_BITLEN + 1, DIGIT_SIZE)) + 1][8]; /* +1 is necessary extension for ifma_mreduce52x_mb8() purpose */ + + int ifmaBitLen = BASE52_BITSIZE(convBitLen); + int ifmaLen = NUMBER_OF_DIGITS(ifmaBitLen, DIGIT_SIZE); + + int pwr = 2*ifmaBitLen; + int s = pwr - ((pwr/DIGIT_SIZE) * DIGIT_SIZE); + int pwrLen = NUMBER_OF_DIGITS(pwr + 1, DIGIT_SIZE); + + /* set 2^(ifmaBitLen*2) */ + zero_mb8(pwr2_mb8, pwrLen); + _mm512_store_si512(pwr2_mb8[pwrLen-1], _mm512_slli_epi64(_mm512_set1_epi64(1), s) ); + + /* 2^(ifmaBitLen*2) mod M */ + ifma_mreduce52x_mb8(pwr2_mb8, pwrLen, pM, ifmaLen); + + /* copy result */ + for(s=0; s +#include +#include +#include + +/* +// public exponent e=65537 implied +*/ +void ifma_cp_rsa_pub_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const int64u* const n_pa[8], + int rsaBitlen, + const mbx_RSA_Method* m, + int8u* pBuffer) +{ + int len52 = NUMBER_OF_DIGITS(rsaBitlen, DIGIT_SIZE); + + /* 64-byte aligned buffer of int64[8] */ + pint64u_x8 pBuffer_x8 = (pint64u_x8)IFMA_ALIGNED_PTR(pBuffer,64); + + /* allocate mb8 buffers */ + pint64u_x8 k0_mb8 = pBuffer_x8; + pint64u_x8 rr_mb8 = k0_mb8 +1; + pint64u_x8 inout_mb8 = rr_mb8 +len52; + pint64u_x8 n_mb8 = inout_mb8 +len52; + pint64u_x8 work_buffer = n_mb8 + len52; + + /* convert modulus to ifma fmt */ + zero_mb8(n_mb8, MULTIPLE_OF(len52, 10)); + ifma_BNU_to_mb8(n_mb8, n_pa, rsaBitlen); + + /* compute k0[] */ + ifma_montFactor52_mb8(k0_mb8[0], n_mb8[0]); + + /* compute to_Montgomery domain converters */ + ifma_montRR52x_mb8(rr_mb8, n_mb8, rsaBitlen); + + /* convert input to ifma fmt */ + ifma_HexStr8_to_mb8(inout_mb8, from_pa, rsaBitlen); + + /* exponentiation */ + m->expfunc65537(inout_mb8, + (const int64u (*)[8])inout_mb8, + (const int64u (*)[8])n_mb8, + (const int64u (*)[8])rr_mb8, + k0_mb8[0], + (int64u (*)[8])work_buffer); + + /* convert result from ifma fmt */ + ifma_mb8_to_HexStr8(to_pa, (const int64u(*)[8])inout_mb8, rsaBitlen); +} + +/* +// private key +*/ +void ifma_cp_rsa_prv2_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const int64u* const d_pa[8], + const int64u* const n_pa[8], + int rsaBitlen, + const mbx_RSA_Method* m, + int8u* pBuffer) + +{ + int len52 = NUMBER_OF_DIGITS(rsaBitlen, DIGIT_SIZE); + int len64 = NUMBER_OF_DIGITS(rsaBitlen, 64); + + /* 64-byte aligned buffer of int64[8] */ + pint64u_x8 pBuffer_x8 = (pint64u_x8)IFMA_ALIGNED_PTR(pBuffer,64); + + /* allocate mb8 buffers */ + pint64u_x8 k0_mb8 = pBuffer_x8; + pint64u_x8 d_mb8 = k0_mb8 +1; + pint64u_x8 rr_mb8 = d_mb8 +len64; + pint64u_x8 inout_mb8 = rr_mb8 +len52; + pint64u_x8 n_mb8 = inout_mb8 +len52; + pint64u_x8 work_buffer = n_mb8 + len52; + + /* convert modulus to ifma fmt */ + zero_mb8(n_mb8, MULTIPLE_OF(len52, 10)); + ifma_BNU_to_mb8(n_mb8, n_pa, rsaBitlen); + + /* compute k0[] */ + ifma_montFactor52_mb8(k0_mb8[0], n_mb8[0]); + + /* compute to_Montgomery domain converters */ + ifma_montRR52x_mb8(rr_mb8, n_mb8, rsaBitlen); + + /* convert input to ifma fmt */ + ifma_HexStr8_to_mb8(inout_mb8, from_pa, rsaBitlen); + + /* re-arrange exps to ifma */ + ifma_BNU_transpose_copy(d_mb8, d_pa, rsaBitlen); + + /* exponentiation */ + m->expfun(inout_mb8, + (const int64u(*)[8])inout_mb8, + (const int64u(*)[8])d_mb8, + (const int64u(*)[8])n_mb8, + (const int64u(*)[8])rr_mb8, + k0_mb8[0], + (int64u (*)[8])work_buffer); + + /* convert result from ifma fmt */ + ifma_mb8_to_HexStr8(to_pa, (const int64u(*)[8])inout_mb8, rsaBitlen); + + /* clear exponents */ + zero_mb8(d_mb8, len64); +} + +/* +// private key (ctr) +*/ +void ifma_cp_rsa_prv5_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const int64u* const p_pa[8], + const int64u* const q_pa[8], + const int64u* const dp_pa[8], + const int64u* const dq_pa[8], + const int64u* const iq_pa[8], + int rsaBitlen, + const mbx_RSA_Method* m, + int8u* pBuffer) + +{ + int factorBitlen = rsaBitlen/2; + int len52 = NUMBER_OF_DIGITS(factorBitlen, DIGIT_SIZE); + int len64 = NUMBER_OF_DIGITS(factorBitlen, 64); + + /* 64-byte aligned buffer of int64[8] */ + pint64u_x8 pBuffer_x8 = (pint64u_x8)IFMA_ALIGNED_PTR(pBuffer,64); + + /* allocate mb8 buffers */ + pint64u_x8 k0_mb8 = pBuffer_x8; + pint64u_x8 p_mb8 = k0_mb8 +1; + pint64u_x8 q_mb8 = p_mb8 +len52; + pint64u_x8 d_mb8 = q_mb8 +len52; + pint64u_x8 rr_mb8 = d_mb8 +len64; + pint64u_x8 xp_mb8 = rr_mb8 +len52; + pint64u_x8 xq_mb8 = xp_mb8 +len52; + pint64u_x8 inp_mb8 = xq_mb8 +len52; + pint64u_x8 work_buffer = inp_mb8 + len52*2; + + /* convert input to ifma fmt */ + zero_mb8(inp_mb8, len52*2); + ifma_HexStr8_to_mb8(inp_mb8, from_pa, rsaBitlen); + + /* + // q exponentiation + */ + + /* convert modulus to ifma fmt */ + ifma_BNU_to_mb8(q_mb8, q_pa, factorBitlen); + /* compute k0[] */ + ifma_montFactor52_mb8(k0_mb8[0], q_mb8[0]); + /* compute to_Montgomery domain converters */ + ifma_montRR52x_mb8(rr_mb8, q_mb8, factorBitlen); + /* xq = x mod q */ + m->amred52x(xq_mb8, (const int64u(*)[8])inp_mb8, (const int64u(*)[8])q_mb8, k0_mb8[0]); + m->ammul52x((int64u*)xq_mb8, (int64u*)xq_mb8, (int64u*)rr_mb8, (int64u*)q_mb8, k0_mb8[0]); + m->modsub52x(xq_mb8, (const int64u(*)[8])xq_mb8, (const int64u(*)[8])q_mb8, (const int64u(*)[8])q_mb8); // ?? + /* re-arrange exps to ifma */ + ifma_BNU_transpose_copy(d_mb8, dq_pa, factorBitlen); + + m->expfun(xq_mb8, + (const int64u(*)[8])xq_mb8, + (const int64u(*)[8])d_mb8, + (const int64u(*)[8])q_mb8, + (const int64u(*)[8])rr_mb8, + k0_mb8[0], + (int64u (*)[8])work_buffer); + + /* + // p exponentiation + */ + + /* convert modulus to ifma fmt */ + ifma_BNU_to_mb8(p_mb8, p_pa, factorBitlen); + /* compute k0[] */ + ifma_montFactor52_mb8(k0_mb8[0], p_mb8[0]); + /* compute to_Montgomery domain converters */ + ifma_montRR52x_mb8(rr_mb8, p_mb8, factorBitlen); + /* xq = x mod q */ + m->amred52x(xp_mb8, (const int64u(*)[8])inp_mb8, (const int64u(*)[8])p_mb8, k0_mb8[0]); + m->ammul52x((int64u*)xp_mb8, (int64u*)xp_mb8, (int64u*)rr_mb8, (int64u*)p_mb8, k0_mb8[0]); + m->modsub52x(xp_mb8, (const int64u(*)[8])xp_mb8, (const int64u(*)[8])p_mb8, (const int64u(*)[8])p_mb8); // ?? + /* re-arrange exps to ifma */ + ifma_BNU_transpose_copy(d_mb8, dp_pa, factorBitlen); + + m->expfun(xp_mb8, + (const int64u(*)[8])xp_mb8, + (const int64u(*)[8])d_mb8, + (const int64u(*)[8])p_mb8, + (const int64u(*)[8])rr_mb8, + k0_mb8[0], + (int64u (*)[8])work_buffer); + + /* + // crt recombination + */ + + /* xp = (xp-xq) mod p */ + m->modsub52x(inp_mb8,(const int64u(*)[8])xq_mb8, (const int64u(*)[8])p_mb8, (const int64u(*)[8])p_mb8); /* for specific case pmodsub52x(xp_mb8, (const int64u(*)[8])xp_mb8, (const int64u(*)[8])inp_mb8, (const int64u(*)[8])p_mb8); + + /* xp = (xp*coef) mod p */ + ifma_BNU_to_mb8(inp_mb8, iq_pa, factorBitlen); /* coef */ + m->ammul52x((int64u*)xp_mb8, (int64u*)xp_mb8, (int64u*)rr_mb8, (int64u*)p_mb8, k0_mb8[0]); /* -> mont domain */ + m->ammul52x((int64u*)xp_mb8, (int64u*)xp_mb8, (int64u*)inp_mb8, (int64u*)p_mb8, k0_mb8[0]); /* mmul */ + m->modsub52x(xp_mb8, (const int64u(*)[8])xp_mb8, (const int64u(*)[8])p_mb8, (const int64u(*)[8])p_mb8);/* correction */ + + /* xp = (xp*q + xq) */ + zero_mb8(inp_mb8, len52*2); + copy_mb8(inp_mb8, (const int64u(*)[8])xq_mb8, len52); + m->mla52x(inp_mb8, (const int64u(*)[8])xp_mb8, (const int64u(*)[8])q_mb8); + + /* convert result from ifma fmt */ + ifma_mb8_to_HexStr8(to_pa, (const int64u(*)[8])inp_mb8, rsaBitlen); + + /* clear exponents, p, q */ + zero_mb8(d_mb8, len64); + zero_mb8(q_mb8, len52); + zero_mb8(p_mb8, len52); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_rsa_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_rsa_mb8.c new file mode 100644 index 0000000..ff1a574 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_rsa_mb8.c @@ -0,0 +1,314 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + + +#include +#include + +#include +#include +#include +#include + +#if !defined(NO_USE_MALLOC) +#include +#endif + +// y = x^65537 mod n +DLL_PUBLIC +mbx_status mbx_rsa_public_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const int64u* const n_pa[8], + int expected_rsa_bitsize, + const mbx_RSA_Method* m, + int8u* pBuffer) +{ + const mbx_RSA_Method* meth = m; + + mbx_status status = 0; + int buf_no; + + /* test input pointers */ + if(NULL==from_pa || NULL==to_pa || NULL==n_pa) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + /* test rsa modulus size */ + if(RSA_1K != expected_rsa_bitsize && RSA_2K != expected_rsa_bitsize && + RSA_3K != expected_rsa_bitsize && RSA_4K != expected_rsa_bitsize) { + status = MBX_SET_STS_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + const int8u* inp = from_pa[buf_no]; + int8u* out = to_pa[buf_no]; + const int64u* n = n_pa[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL==inp || NULL==out || NULL==n) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + continue; + } + } + + /* test method */ + if(NULL==meth) { + meth = mbx_RSA_pub65537_Method(expected_rsa_bitsize); + if(NULL==meth) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + } + /* check if requested operation matched to method's */ + if(RSA_PUB_KEY != OP_RSA_ID(meth->id)) { + status = MBX_SET_STS_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + /* check if requested RSA matched to method's */ + if(expected_rsa_bitsize != BISIZE_RSA_ID(meth->id)) { + status = MBX_SET_STS_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* + // processing + */ + if( MBX_IS_ANY_OK_STS(status) ) { + int8u* buffer = pBuffer; + + #if !defined(NO_USE_MALLOC) + int allocated_buf = 0; + + /* check if allocated buffer) */ + if(NULL==buffer) { + buffer = (int8u*)( malloc(meth->buffSize) ); + if(NULL==buffer) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + allocated_buf = 1; + } + #endif + + ifma_cp_rsa_pub_layer_mb8(from_pa, to_pa, n_pa, + expected_rsa_bitsize, meth, + buffer); + + #if !defined(NO_USE_MALLOC) + /* release buffer */ + if(allocated_buf) + free(buffer); + #endif + } + + return status; +} + +DLL_PUBLIC +mbx_status mbx_rsa_private_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const int64u* const d_pa[8], + const int64u* const n_pa[8], + int expected_rsa_bitsize, + const mbx_RSA_Method* m, + int8u* pBuffer) +{ + const mbx_RSA_Method* meth = m; + + mbx_status status = 0; + int buf_no; + + /* test input pointers */ + if(NULL==from_pa || NULL==to_pa || NULL==d_pa || NULL==n_pa) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + /* test rsa modulus size */ + if(RSA_1K != expected_rsa_bitsize && RSA_2K != expected_rsa_bitsize && + RSA_3K != expected_rsa_bitsize && RSA_4K != expected_rsa_bitsize) { + status = MBX_SET_STS_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + const int8u* inp = from_pa[buf_no]; + int8u* out = to_pa[buf_no]; + const int64u* d = d_pa[buf_no]; + const int64u* n = n_pa[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL==inp || NULL==out || NULL==d || NULL==n) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + continue; + } + } + + /* test method */ + if(NULL==meth) { + meth = mbx_RSA_private_Method(expected_rsa_bitsize); + if(NULL==meth) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + } + /* check if requested operation matched to method's */ + if(RSA_PRV2_KEY != OP_RSA_ID(meth->id)) { + status = MBX_SET_STS_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + /* check if requested RSA matched to method's */ + if(expected_rsa_bitsize != BISIZE_RSA_ID(meth->id)) { + status = MBX_SET_STS_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* + // processing + */ + if( MBX_IS_ANY_OK_STS(status) ) { + int8u* buffer = pBuffer; + + #if !defined(NO_USE_MALLOC) + int allocated_buf = 0; + + /* check if allocated buffer) */ + if(NULL==buffer) { + buffer = (int8u*)( malloc(meth->buffSize) ); + if(NULL==buffer) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + allocated_buf = 1; + } + #endif + + ifma_cp_rsa_prv2_layer_mb8(from_pa, to_pa, d_pa, n_pa, + expected_rsa_bitsize, meth, + buffer); + + #if !defined(NO_USE_MALLOC) + /* release buffer */ + if(allocated_buf) + free(buffer); + #endif + } + + return status; +} + +DLL_PUBLIC +mbx_status mbx_rsa_private_crt_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const int64u* const p_pa[8], + const int64u* const q_pa[8], + const int64u* const dp_pa[8], + const int64u* const dq_pa[8], + const int64u* const iq_pa[8], + int expected_rsa_bitsize, + const mbx_RSA_Method* m, + int8u* pBuffer) +{ + const mbx_RSA_Method* meth = m; + + mbx_status status = 0; + int buf_no; + + /* test input pointers */ + if(NULL==from_pa || NULL==to_pa || + NULL==p_pa || NULL==q_pa || NULL==dp_pa || NULL==dq_pa || NULL==iq_pa) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + /* test rsa modulus size */ + if(RSA_1K != expected_rsa_bitsize && RSA_2K != expected_rsa_bitsize && + RSA_3K != expected_rsa_bitsize && RSA_4K != expected_rsa_bitsize) { + status = MBX_SET_STS_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + const int8u* inp = from_pa[buf_no]; + int8u* out = to_pa[buf_no]; + const int64u* p = p_pa[buf_no]; + const int64u* q = q_pa[buf_no]; + const int64u* dp = dp_pa[buf_no]; + const int64u* dq = dq_pa[buf_no]; + const int64u* iq = iq_pa[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL==inp || NULL==out || NULL==q || NULL==p || NULL==dq || NULL==dp || NULL==iq) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + continue; + } + } + + /* test method */ + if(NULL==meth) { + meth = mbx_RSA_private_crt_Method(expected_rsa_bitsize); + if(NULL==meth) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + } + /* check if requested operation matched to method's */ + if(RSA_PRV5_KEY != OP_RSA_ID(meth->id)) { + status = MBX_SET_STS_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + /* check if requested RSA matched to method's */ + if(expected_rsa_bitsize != BISIZE_RSA_ID(meth->id)) { + status = MBX_SET_STS_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* + // processing + */ + if( MBX_IS_ANY_OK_STS(status) ) { + int8u* buffer = pBuffer; + + #if !defined(NO_USE_MALLOC) + int allocated_buf = 0; + + /* check if allocated buffer) */ + if(NULL==buffer) { + buffer = (int8u*)( malloc(meth->buffSize) ); + if(NULL==buffer) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + allocated_buf = 1; + } + #endif + + ifma_cp_rsa_prv5_layer_mb8(from_pa, to_pa, p_pa, q_pa, dp_pa, dq_pa, iq_pa, + expected_rsa_bitsize, meth, + buffer); + + #if !defined(NO_USE_MALLOC) + /* release buffer */ + if(allocated_buf) + free(buffer); + #endif + } + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_rsa_ssl_prv2_layer_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_rsa_ssl_prv2_layer_mb8.c new file mode 100644 index 0000000..e15814f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_rsa_ssl_prv2_layer_mb8.c @@ -0,0 +1,258 @@ +typedef int to_avoid_translation_unit_is_empty_warning; + +#ifndef BN_OPENSSL_DISABLE + +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +#include +#include +#include + +#define EXP_WIN_SIZE (5) //(4) +#define EXP_WIN_MASK ((1< + +#include + +#include +#include +#include + + +// y = x^d mod n +DLL_PUBLIC +mbx_status mbx_rsa_private_ssl_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const d_pa[8], + const BIGNUM* const n_pa[8], + int expected_rsa_bitsize) +{ + mbx_status status = 0; + int buf_no; + + /* test input pointers */ + if(NULL==from_pa || NULL==to_pa || NULL==d_pa || NULL==n_pa) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + /* test rsa modulus size */ + if(RSA_1K != expected_rsa_bitsize && RSA_2K != expected_rsa_bitsize && + RSA_3K != expected_rsa_bitsize && RSA_4K != expected_rsa_bitsize) { + status = MBX_SET_STS_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + const int8u* inp = from_pa[buf_no]; + int8u* out = to_pa[buf_no]; + const BIGNUM* d = d_pa[buf_no]; + const BIGNUM* n = n_pa[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL==inp || NULL==out || NULL==d || NULL==n) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + continue; + } + + /* check rsa size */ + if(expected_rsa_bitsize != BN_num_bits(n)) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + continue; + } + } + + /* continue processing if there are correct parameters */ + if( MBX_IS_ANY_OK_STS(status) ) { + /* select exponentiation */ + switch (expected_rsa_bitsize) { + case RSA_1K: ifma_ssl_rsa1K_prv2_layer_mb8(from_pa, to_pa, d_pa, n_pa); break; + case RSA_2K: ifma_ssl_rsa2K_prv2_layer_mb8(from_pa, to_pa, d_pa, n_pa); break; + case RSA_3K: ifma_ssl_rsa3K_prv2_layer_mb8(from_pa, to_pa, d_pa, n_pa); break; + case RSA_4K: ifma_ssl_rsa4K_prv2_layer_mb8(from_pa, to_pa, d_pa, n_pa); break; + } + } + + return status; +} + +#endif /* BN_OPENSSL_DISABLE */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_rsa_ssl_prv5_layer_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_rsa_ssl_prv5_layer_mb8.c new file mode 100644 index 0000000..4e7db9a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_rsa_ssl_prv5_layer_mb8.c @@ -0,0 +1,486 @@ +typedef int to_avoid_translation_unit_is_empty_warning; + +#ifndef BN_OPENSSL_DISABLE + +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +#include +#include +#include + +#define EXP_WIN_SIZE (5) //(4) +#define EXP_WIN_MASK ((1< mont domain */ + ifma_amm52x10_mb8((int64u*)xp_mb8, (int64u*)xp_mb8, (int64u*)inp_mb8, (int64u*)p_mb8, k0_mb8); /* mmul */ + ifma_modsub52x10_mb8(xp_mb8, (const int64u(*)[8])xp_mb8, (const int64u(*)[8])p_mb8, (const int64u(*)[8])p_mb8);/* correction */ + + /* xp = (xp*q + xq) */ + zero_mb8(inp_mb8, LEN52*2); + copy_mb8(inp_mb8, (const int64u(*)[8])xq_mb8, LEN52); + ifma_addmul52x10_mb8(inp_mb8, (const int64u(*)[8])xp_mb8, (const int64u(*)[8])q_mb8); + + /* convert result from ifma fmt */ + ifma_mb8_to_HexStr8(to_pa, (const int64u(*)[8])inp_mb8, RSA_BITLEN); + + /* clear exponents, p, q */ + zero_mb8(d_mb8, LEN64); + zero_mb8(q_mb8, LEN52); + zero_mb8(p_mb8, LEN52); + + #undef RSA_BITLEN + #undef FACTOR_BITLEN + #undef LEN52 + #undef LEN64 +} + + +void ifma_ssl_rsa2K_prv5_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const p_pa[8], + const BIGNUM* const q_pa[8], + const BIGNUM* const dp_pa[8], + const BIGNUM* const dq_pa[8], + const BIGNUM* const iq_pa[8]) +{ + #define RSA_BITLEN (RSA_2K) + #define FACTOR_BITLEN (RSA_BITLEN/2) + #define LEN52 (NUMBER_OF_DIGITS(FACTOR_BITLEN, DIGIT_SIZE)) + #define LEN64 (NUMBER_OF_DIGITS(FACTOR_BITLEN, 64)) + + /* allocate mb8 buffers */ + __ALIGN64 int64u k0_mb8[8]; + __ALIGN64 int64u p_mb8[LEN52][8]; + __ALIGN64 int64u q_mb8[LEN52][8]; + __ALIGN64 int64u d_mb8[LEN64][8]; + __ALIGN64 int64u rr_mb8[LEN52][8]; + __ALIGN64 int64u xp_mb8[LEN52][8]; + __ALIGN64 int64u xq_mb8[LEN52][8]; + __ALIGN64 int64u inp_mb8[LEN52*2][8]; + /* allocate stack for red(undant) result, multiplier, for exponent X and for pre-computed table of base powers */ + __ALIGN64 int64u work_buffer[LEN52*2 + 1 + (LEN64 + 1) + (1 << EXP_WIN_SIZE)*LEN52][8]; + + /* convert input to ifma fmt */ + zero_mb8(inp_mb8, LEN52*2); + ifma_HexStr8_to_mb8(inp_mb8, from_pa, RSA_BITLEN); + + /* + // q exponentiation + */ + + /* convert modulus to ifma fmt */ + ifma_BN_to_mb8(q_mb8, q_pa, FACTOR_BITLEN); + /* compute k0[] */ + ifma_montFactor52_mb8(k0_mb8, q_mb8[0]); + /* compute to_Montgomery domain converters */ + ifma_montRR52x_mb8(rr_mb8, q_mb8, FACTOR_BITLEN); + /* xq = x mod q */ + ifma_amred52x20_mb8(xq_mb8, (const int64u(*)[8])inp_mb8, (const int64u(*)[8])q_mb8, k0_mb8); + ifma_amm52x20_mb8((int64u*)xq_mb8, (int64u*)xq_mb8, (int64u*)rr_mb8, (int64u*)q_mb8, k0_mb8); + ifma_modsub52x20_mb8(xq_mb8, (const int64u(*)[8])xq_mb8, (const int64u(*)[8])q_mb8, (const int64u(*)[8])q_mb8); // ?? + /* re-arrange exps to ifma */ + ifma_BN_transpose_copy(d_mb8, dq_pa, FACTOR_BITLEN); + + EXP52x20_mb8(xq_mb8, + (const int64u(*)[8])xq_mb8, + (const int64u(*)[8])d_mb8, + (const int64u(*)[8])q_mb8, + (const int64u(*)[8])rr_mb8, + k0_mb8, + (int64u(*)[8])work_buffer); + + /* + // p exponentiation + */ + + /* convert modulus to ifma fmt */ + ifma_BN_to_mb8(p_mb8, p_pa, FACTOR_BITLEN); + /* compute k0[] */ + ifma_montFactor52_mb8(k0_mb8, p_mb8[0]); + /* compute to_Montgomery domain converters */ + ifma_montRR52x_mb8(rr_mb8, p_mb8, FACTOR_BITLEN); + /* xp = x mod p */ + ifma_amred52x20_mb8(xp_mb8, (const int64u(*)[8])inp_mb8, (const int64u(*)[8])p_mb8, k0_mb8); + ifma_amm52x20_mb8((int64u*)xp_mb8, (int64u*)xp_mb8, (int64u*)rr_mb8, (int64u*)p_mb8, k0_mb8); + ifma_modsub52x20_mb8(xp_mb8, (const int64u(*)[8])xp_mb8, (const int64u(*)[8])p_mb8, (const int64u(*)[8])p_mb8); // ?? + /* re-arrange exps to ifma */ + ifma_BN_transpose_copy(d_mb8, dp_pa, FACTOR_BITLEN); + + EXP52x20_mb8(xp_mb8, + (const int64u(*)[8])xp_mb8, + (const int64u(*)[8])d_mb8, + (const int64u(*)[8])p_mb8, + (const int64u(*)[8])rr_mb8, + k0_mb8, + (int64u(*)[8])work_buffer); + + /* + // crt recombination + */ + + /* xp = (xp-xq) mod p */ + ifma_modsub52x20_mb8(inp_mb8,(const int64u(*)[8])xq_mb8, (const int64u(*)[8])p_mb8, (const int64u(*)[8])p_mb8); /* for specific case p mont domain */ + ifma_amm52x20_mb8((int64u*)xp_mb8, (int64u*)xp_mb8, (int64u*)inp_mb8, (int64u*)p_mb8, k0_mb8); /* mmul */ + ifma_modsub52x20_mb8(xp_mb8, (const int64u(*)[8])xp_mb8, (const int64u(*)[8])p_mb8, (const int64u(*)[8])p_mb8);/* correction */ + + /* xp = (xp*q + xq) */ + zero_mb8(inp_mb8, LEN52*2); + copy_mb8(inp_mb8, (const int64u(*)[8])xq_mb8, LEN52); + ifma_addmul52x20_mb8(inp_mb8, (const int64u(*)[8])xp_mb8, (const int64u(*)[8])q_mb8); + + /* convert result from ifma fmt */ + ifma_mb8_to_HexStr8(to_pa, (const int64u(*)[8])inp_mb8, RSA_BITLEN); + + /* clear exponents, p, q */ + zero_mb8(d_mb8, LEN64); + zero_mb8(q_mb8, LEN52); + zero_mb8(p_mb8, LEN52); + + #undef RSA_BITLEN + #undef FACTOR_BITLEN + #undef LEN52 + #undef LEN64 +} + + +void ifma_ssl_rsa3K_prv5_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const p_pa[8], + const BIGNUM* const q_pa[8], + const BIGNUM* const dp_pa[8], + const BIGNUM* const dq_pa[8], + const BIGNUM* const iq_pa[8]) +{ + #define RSA_BITLEN (RSA_3K) + #define FACTOR_BITLEN (RSA_BITLEN/2) + #define LEN52 (NUMBER_OF_DIGITS(FACTOR_BITLEN, DIGIT_SIZE)) + #define LEN64 (NUMBER_OF_DIGITS(FACTOR_BITLEN, 64)) + + /* allocate mb8 buffers */ + __ALIGN64 int64u k0_mb8[8]; + __ALIGN64 int64u p_mb8[LEN52][8]; + __ALIGN64 int64u q_mb8[LEN52][8]; + __ALIGN64 int64u d_mb8[LEN64][8]; + __ALIGN64 int64u rr_mb8[LEN52][8]; + __ALIGN64 int64u xp_mb8[LEN52][8]; + __ALIGN64 int64u xq_mb8[LEN52][8]; + __ALIGN64 int64u inp_mb8[LEN52 * 2][8]; + /* allocate stack for red(undant) result, multiplier, for exponent X and for pre-computed table of base powers */ + __ALIGN64 int64u work_buffer[LEN52*2 + 1 + (LEN64 + 1) + (1 << EXP_WIN_SIZE)*LEN52][8]; + + /* convert input to ifma fmt */ + zero_mb8(inp_mb8, LEN52 * 2); + ifma_HexStr8_to_mb8(inp_mb8, from_pa, RSA_BITLEN); + + /* + // q exponentiation + */ + + /* convert modulus to ifma fmt */ + ifma_BN_to_mb8(q_mb8, q_pa, FACTOR_BITLEN); + /* compute k0[] */ + ifma_montFactor52_mb8(k0_mb8, q_mb8[0]); + /* compute to_Montgomery domain converters */ + ifma_montRR52x_mb8(rr_mb8, q_mb8, FACTOR_BITLEN); + /* xq = x mod q */ + ifma_amred52x30_mb8(xq_mb8, (const int64u(*)[8])inp_mb8, (const int64u(*)[8])q_mb8, k0_mb8); + ifma_amm52x30_mb8((int64u*)xq_mb8, (int64u*)xq_mb8, (int64u*)rr_mb8, (int64u*)q_mb8, k0_mb8); + ifma_modsub52x30_mb8(xq_mb8, (const int64u(*)[8])xq_mb8, (const int64u(*)[8])q_mb8, (const int64u(*)[8])q_mb8); // ?? + /* re-arrange exps to ifma */ + ifma_BN_transpose_copy(d_mb8, dq_pa, FACTOR_BITLEN); + + EXP52x30_mb8(xq_mb8, + (const int64u(*)[8])xq_mb8, + (const int64u(*)[8])d_mb8, + (const int64u(*)[8])q_mb8, + (const int64u(*)[8])rr_mb8, + k0_mb8, + (int64u(*)[8])work_buffer); + + /* + // p exponentiation + */ + + /* convert modulus to ifma fmt */ + ifma_BN_to_mb8(p_mb8, p_pa, FACTOR_BITLEN); + /* compute k0[] */ + ifma_montFactor52_mb8(k0_mb8, p_mb8[0]); + /* compute to_Montgomery domain converters */ + ifma_montRR52x_mb8(rr_mb8, p_mb8, FACTOR_BITLEN); + /* xp = x mod p */ + ifma_amred52x30_mb8(xp_mb8, (const int64u(*)[8])inp_mb8, (const int64u(*)[8])p_mb8, k0_mb8); + ifma_amm52x30_mb8((int64u*)xp_mb8, (int64u*)xp_mb8, (int64u*)rr_mb8, (int64u*)p_mb8, k0_mb8); + ifma_modsub52x30_mb8(xp_mb8, (const int64u(*)[8])xp_mb8, (const int64u(*)[8])p_mb8, (const int64u(*)[8])p_mb8); // ?? + /* re-arrange exps to ifma */ + ifma_BN_transpose_copy(d_mb8, dp_pa, FACTOR_BITLEN); + + EXP52x30_mb8(xp_mb8, + (const int64u(*)[8])xp_mb8, + (const int64u(*)[8])d_mb8, + (const int64u(*)[8])p_mb8, + (const int64u(*)[8])rr_mb8, + k0_mb8, + (int64u(*)[8])work_buffer); + + /* + // crt recombination + */ + + /* xp = (xp-xq) mod p */ + ifma_modsub52x30_mb8(inp_mb8,(const int64u(*)[8])xq_mb8, (const int64u(*)[8])p_mb8, (const int64u(*)[8])p_mb8); /* for specific case p mont domain */ + ifma_amm52x30_mb8((int64u*)xp_mb8, (int64u*)xp_mb8, (int64u*)inp_mb8, (int64u*)p_mb8, k0_mb8); /* mmul */ + ifma_modsub52x30_mb8(xp_mb8, (const int64u(*)[8])xp_mb8, (const int64u(*)[8])p_mb8, (const int64u(*)[8])p_mb8);/* correction */ + + /* xp = (xp*q + xq) */ + zero_mb8(inp_mb8, LEN52 * 2); + copy_mb8(inp_mb8, (const int64u(*)[8])xq_mb8, LEN52); + ifma_addmul52x30_mb8(inp_mb8, (const int64u(*)[8])xp_mb8, (const int64u(*)[8])q_mb8); + + /* convert result from ifma fmt */ + ifma_mb8_to_HexStr8(to_pa, (const int64u(*)[8])inp_mb8, RSA_BITLEN); + + /* clear exponents, p, q */ + zero_mb8(d_mb8, LEN64); + zero_mb8(q_mb8, LEN52); + zero_mb8(p_mb8, LEN52); + + #undef RSA_BITLEN + #undef FACTOR_BITLEN + #undef LEN52 + #undef LEN64 +} + + +void ifma_ssl_rsa4K_prv5_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const p_pa[8], + const BIGNUM* const q_pa[8], + const BIGNUM* const dp_pa[8], + const BIGNUM* const dq_pa[8], + const BIGNUM* const iq_pa[8]) +{ + #define RSA_BITLEN (RSA_4K) + #define FACTOR_BITLEN (RSA_BITLEN/2) + #define LEN52 (NUMBER_OF_DIGITS(FACTOR_BITLEN, DIGIT_SIZE)) + #define LEN64 (NUMBER_OF_DIGITS(FACTOR_BITLEN, 64)) + + /* allocate mb8 buffers */ + __ALIGN64 int64u k0_mb8[8]; + __ALIGN64 int64u p_mb8[LEN52][8]; + __ALIGN64 int64u q_mb8[LEN52][8]; + __ALIGN64 int64u d_mb8[LEN64][8]; + __ALIGN64 int64u rr_mb8[LEN52][8]; + __ALIGN64 int64u xp_mb8[LEN52][8]; + __ALIGN64 int64u xq_mb8[LEN52][8]; + __ALIGN64 int64u inp_mb8[LEN52*2][8]; + /* allocate stack for red(undant) result, multiplier, for exponent X and for pre-computed table of base powers */ + __ALIGN64 int64u work_buffer[LEN52*2 + 1 + (LEN64 + 1) + (1 << EXP_WIN_SIZE)*LEN52][8]; + + /* convert input to ifma fmt */ + zero_mb8(inp_mb8, LEN52 * 2); + ifma_HexStr8_to_mb8(inp_mb8, from_pa, RSA_BITLEN); + + /* + // q exponentiation + */ + + /* convert modulus to ifma fmt */ + ifma_BN_to_mb8(q_mb8, q_pa, FACTOR_BITLEN); + /* compute k0[] */ + ifma_montFactor52_mb8(k0_mb8, q_mb8[0]); + /* compute to_Montgomery domain converters */ + ifma_montRR52x_mb8(rr_mb8, q_mb8, FACTOR_BITLEN); + /* xq = x mod q */ + ifma_amred52x40_mb8(xq_mb8, (const int64u(*)[8])inp_mb8, (const int64u(*)[8])q_mb8, k0_mb8); + ifma_amm52x40_mb8((int64u*)xq_mb8, (int64u*)xq_mb8, (int64u*)rr_mb8, (int64u*)q_mb8, k0_mb8); + ifma_modsub52x40_mb8(xq_mb8, (const int64u(*)[8])xq_mb8, (const int64u(*)[8])q_mb8, (const int64u(*)[8])q_mb8); // ?? + /* re-arrange exps to ifma */ + ifma_BN_transpose_copy(d_mb8, dq_pa, FACTOR_BITLEN); + + EXP52x40_mb8(xq_mb8, + (const int64u(*)[8])xq_mb8, + (const int64u(*)[8])d_mb8, + (const int64u(*)[8])q_mb8, + (const int64u(*)[8])rr_mb8, + k0_mb8, + (int64u(*)[8])work_buffer); + + /* + // p exponentiation + */ + + /* convert modulus to ifma fmt */ + ifma_BN_to_mb8(p_mb8, p_pa, FACTOR_BITLEN); + /* compute k0[] */ + ifma_montFactor52_mb8(k0_mb8, p_mb8[0]); + /* compute to_Montgomery domain converters */ + ifma_montRR52x_mb8(rr_mb8, p_mb8, FACTOR_BITLEN); + /* xp = x mod p */ + ifma_amred52x40_mb8(xp_mb8, (const int64u(*)[8])inp_mb8, (const int64u(*)[8])p_mb8, k0_mb8); + ifma_amm52x40_mb8((int64u*)xp_mb8, (int64u*)xp_mb8, (int64u*)rr_mb8, (int64u*)p_mb8, k0_mb8); + ifma_modsub52x40_mb8(xp_mb8, (const int64u(*)[8])xp_mb8, (const int64u(*)[8])p_mb8, (const int64u(*)[8])p_mb8); // ?? + /* re-arrange exps to ifma */ + ifma_BN_transpose_copy(d_mb8, dp_pa, FACTOR_BITLEN); + + EXP52x40_mb8(xp_mb8, + (const int64u(*)[8])xp_mb8, + (const int64u(*)[8])d_mb8, + (const int64u(*)[8])p_mb8, + (const int64u(*)[8])rr_mb8, + k0_mb8, + (int64u(*)[8])work_buffer); + + /* + // crt recombination + */ + + /* xp = (xp-xq) mod p */ + ifma_modsub52x40_mb8(inp_mb8,(const int64u(*)[8])xq_mb8, (const int64u(*)[8])p_mb8, (const int64u(*)[8])p_mb8); /* for specific case p mont domain */ + ifma_amm52x40_mb8((int64u*)xp_mb8, (int64u*)xp_mb8, (int64u*)inp_mb8, (int64u*)p_mb8, k0_mb8); /* mmul */ + ifma_modsub52x40_mb8(xp_mb8, (const int64u(*)[8])xp_mb8, (const int64u(*)[8])p_mb8, (const int64u(*)[8])p_mb8);/* correction */ + + /* xp = (xp*q + xq) */ + zero_mb8(inp_mb8, LEN52 * 2); + copy_mb8(inp_mb8, (const int64u(*)[8])xq_mb8, LEN52); + ifma_addmul52x40_mb8(inp_mb8, (const int64u(*)[8])xp_mb8, (const int64u(*)[8])q_mb8); + + /* convert result from ifma fmt */ + ifma_mb8_to_HexStr8(to_pa, (const int64u(*)[8])inp_mb8, RSA_BITLEN); + + /* clear exponents, p, q */ + zero_mb8(d_mb8, LEN64); + zero_mb8(q_mb8, LEN52); + zero_mb8(p_mb8, LEN52); + + #undef RSA_BITLEN + #undef FACTOR_BITLEN + #undef LEN52 + #undef LEN64 +} + +#endif /* BN_OPENSSL_DISABLE */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_rsa_ssl_prv5_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_rsa_ssl_prv5_mb8.c new file mode 100644 index 0000000..e3fb0ae --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_rsa_ssl_prv5_mb8.c @@ -0,0 +1,97 @@ +typedef int to_avoid_translation_unit_is_empty_warning; + +#ifndef BN_OPENSSL_DISABLE + +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +#include + +#include +#include +#include + + +// y = x^d mod n (crt) +DLL_PUBLIC +mbx_status mbx_rsa_private_crt_ssl_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const p_pa[8], + const BIGNUM* const q_pa[8], + const BIGNUM* const dp_pa[8], + const BIGNUM* const dq_pa[8], + const BIGNUM* const iq_pa[8], + int expected_rsa_bitsize) +{ + mbx_status status = 0; + int buf_no; + + /* test input pointers */ + if(NULL==from_pa || NULL==to_pa || + NULL==p_pa || NULL==q_pa || NULL==dp_pa || NULL==dq_pa || NULL==iq_pa) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + /* test rsa modulus size */ + if(RSA_1K != expected_rsa_bitsize && RSA_2K != expected_rsa_bitsize && + RSA_3K != expected_rsa_bitsize && RSA_4K != expected_rsa_bitsize) { + status = MBX_SET_STS_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + const int8u* inp = from_pa[buf_no]; + int8u* out = to_pa[buf_no]; + const BIGNUM* q = q_pa[buf_no]; + const BIGNUM* p = p_pa[buf_no]; + const BIGNUM* dq = dq_pa[buf_no]; + const BIGNUM* dp = dp_pa[buf_no]; + const BIGNUM* iq = iq_pa[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL==inp || NULL==out || NULL==q || NULL==p || NULL==dq || NULL==dp || NULL==iq) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + continue; + } + + /* check rsa's factors */ + if(((expected_rsa_bitsize/2) != BN_num_bits(p)) || ((expected_rsa_bitsize/2) != BN_num_bits(q))) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + continue; + } + } + + /* continue processing if there are correct parameters */ + if( MBX_IS_ANY_OK_STS(status) ) { + //int factor_bitsize = expected_rsa_bitsize/2; + + /* select exponentiation and other functions */ + switch (expected_rsa_bitsize) { + case RSA_1K: ifma_ssl_rsa1K_prv5_layer_mb8(from_pa, to_pa, p_pa, q_pa, dp_pa, dq_pa, iq_pa); break; + case RSA_2K: ifma_ssl_rsa2K_prv5_layer_mb8(from_pa, to_pa, p_pa, q_pa, dp_pa, dq_pa, iq_pa); break; + case RSA_3K: ifma_ssl_rsa3K_prv5_layer_mb8(from_pa, to_pa, p_pa, q_pa, dp_pa, dq_pa, iq_pa); break; + case RSA_4K: ifma_ssl_rsa4K_prv5_layer_mb8(from_pa, to_pa, p_pa, q_pa, dp_pa, dq_pa, iq_pa); break; + } + } + + return status; +} + +#endif /* BN_OPENSSL_DISABLE */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_rsa_ssl_pub65537_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_rsa_ssl_pub65537_mb8.c new file mode 100644 index 0000000..2034885 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_rsa_ssl_pub65537_mb8.c @@ -0,0 +1,97 @@ +typedef int to_avoid_translation_unit_is_empty_warning; + +#ifndef BN_OPENSSL_DISABLE + +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +#include + +#include +#include +#include + + +// y = x^65537 mod n +DLL_PUBLIC +mbx_status mbx_rsa_public_ssl_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const e_pa[8], + const BIGNUM* const n_pa[8], + int expected_rsa_bitsize) +{ + /* check pointers and values */ + const int64u expected_pub_exp = 65537; + + mbx_status status = 0; + int buf_no; + + /* test input pointers */ + if(NULL==from_pa || NULL==to_pa || NULL==e_pa || NULL==n_pa) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + /* test rsa modulus size */ + if(RSA_1K!=expected_rsa_bitsize && RSA_2K!=expected_rsa_bitsize && + RSA_3K != expected_rsa_bitsize && RSA_4K != expected_rsa_bitsize) { + status = MBX_SET_STS_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + const int8u* inp = from_pa[buf_no]; + int8u* out = to_pa[buf_no]; + const BIGNUM* e = e_pa[buf_no]; + const BIGNUM* n = n_pa[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL==inp || NULL==out || NULL==e || NULL==n) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + continue; + } + + /* check public exponent */ + if(!BN_is_word(e, expected_pub_exp)) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + continue; + } + + /* check rsa size */ + if(expected_rsa_bitsize != BN_num_bits(n)) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + continue; + } + } + + /* continue processing if there are correct parameters */ + if( MBX_IS_ANY_OK_STS(status) ) { + /* use suitable implementation */ + switch(expected_rsa_bitsize) { + case RSA_1K: ifma_ssl_rsa1K_pub_layer_mb8(from_pa, to_pa, n_pa); break; + case RSA_2K: ifma_ssl_rsa2K_pub_layer_mb8(from_pa, to_pa, n_pa); break; + case RSA_3K: ifma_ssl_rsa3K_pub_layer_mb8(from_pa, to_pa, n_pa); break; + case RSA_4K: ifma_ssl_rsa4K_pub_layer_mb8(from_pa, to_pa, n_pa); break; + } + } + + return status; +} + +#endif /* BN_OPENSSL_DISABLE */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_rsa_ssl_pub_layer_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_rsa_ssl_pub_layer_mb8.c new file mode 100644 index 0000000..dc68fc7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/rsa/ifma_rsa_ssl_pub_layer_mb8.c @@ -0,0 +1,212 @@ +typedef int to_avoid_translation_unit_is_empty_warning; + +#ifndef BN_OPENSSL_DISABLE + +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +#include +#include +#include + + +/* +// public exponent e=65537 implied +*/ + +void ifma_ssl_rsa1K_pub_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const n_pa[8]) +{ + #define RSA_BITLEN (RSA_1K) + #define LEN52 (NUMBER_OF_DIGITS(RSA_BITLEN, DIGIT_SIZE)) + + /* allocate mb8 buffers */ + __ALIGN64 int64u k0_mb8[8]; + __ALIGN64 int64u rr_mb8[LEN52][8]; + __ALIGN64 int64u inout_mb8[LEN52][8]; + /* MULTIPLE_OF_10 because of AMS5x52x79_diagonal_mb8() implementaion specific */ + __ALIGN64 int64u n_mb8[MULTIPLE_OF(LEN52, 10)][8]; + /* allocate stack for red(undant) result and multiplier */ + __ALIGN64 int64u work_buffer[LEN52*2][8]; + + /* convert modulus to ifma fmt */ + zero_mb8(n_mb8, MULTIPLE_OF(LEN52, 10)); + ifma_BN_to_mb8(n_mb8, n_pa, RSA_BITLEN); + + /* compute k0[] */ + ifma_montFactor52_mb8(k0_mb8, n_mb8[0]); + + /* compute to_Montgomery domain converters */ + ifma_montRR52x_mb8(rr_mb8, n_mb8, RSA_BITLEN); + + /* convert input to ifma fmt */ + ifma_HexStr8_to_mb8(inout_mb8, from_pa, RSA_BITLEN); + + /* exponentiation */ + EXP52x20_pub65537_mb8(inout_mb8, + (const int64u (*)[8])inout_mb8, + (const int64u (*)[8])n_mb8, + (const int64u (*)[8])rr_mb8, + k0_mb8, + (int64u(*)[8])work_buffer); + + /* convert result from ifma fmt */ + ifma_mb8_to_HexStr8(to_pa, (const int64u(*)[8])inout_mb8, RSA_BITLEN); + + #undef RSA_BITLEN + #undef LEN52 +} + + +void ifma_ssl_rsa2K_pub_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const n_pa[8]) +{ + #define RSA_BITLEN (RSA_2K) + #define LEN52 (NUMBER_OF_DIGITS(RSA_BITLEN, DIGIT_SIZE)) + + /* allocate mb8 buffers */ + __ALIGN64 int64u k0_mb8[8]; + __ALIGN64 int64u rr_mb8[LEN52][8]; + __ALIGN64 int64u inout_mb8[LEN52][8]; + /* MULTIPLE_OF_10 because of AMS5x52x79_diagonal_mb8() implementaion specific */ + __ALIGN64 int64u n_mb8[MULTIPLE_OF(LEN52, 10)][8]; + /* allocate stack for red(undant) result and multiplier */ + __ALIGN64 int64u work_buffer[LEN52*2][8]; + + /* convert modulus to ifma fmt */ + zero_mb8(n_mb8, MULTIPLE_OF(LEN52, 10)); + ifma_BN_to_mb8(n_mb8, n_pa, RSA_BITLEN); + + /* compute k0[] */ + ifma_montFactor52_mb8(k0_mb8, n_mb8[0]); + + /* compute to_Montgomery domain converters */ + ifma_montRR52x_mb8(rr_mb8, n_mb8, RSA_BITLEN); + + /* convert input to ifma fmt */ + ifma_HexStr8_to_mb8(inout_mb8, from_pa, RSA_BITLEN); + + /* exponentiation */ + EXP52x40_pub65537_mb8(inout_mb8, + (const int64u(*)[8])inout_mb8, + (const int64u(*)[8])n_mb8, + (const int64u(*)[8])rr_mb8, + k0_mb8, + (int64u(*)[8])work_buffer); + + /* convert result from ifma fmt */ + ifma_mb8_to_HexStr8(to_pa, (const int64u(*)[8])inout_mb8, RSA_BITLEN); + + #undef RSA_BITLEN + #undef LEN52 +} + + +void ifma_ssl_rsa3K_pub_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const n_pa[8]) +{ + #define RSA_BITLEN (RSA_3K) + #define LEN52 (NUMBER_OF_DIGITS(RSA_BITLEN, DIGIT_SIZE)) + + /* allocate mb8 buffers */ + __ALIGN64 int64u k0_mb8[8]; + __ALIGN64 int64u rr_mb8[LEN52][8]; + __ALIGN64 int64u inout_mb8[LEN52][8]; + /* MULTIPLE_OF_10 because of AMS5x52x79_diagonal_mb8() implementaion specific */ + __ALIGN64 int64u n_mb8[MULTIPLE_OF(LEN52, 10)][8]; + /* allocate stack for red(undant) result and multiplier */ + __ALIGN64 int64u work_buffer[LEN52*2][8]; + + /* convert modulus to ifma fmt */ + zero_mb8(n_mb8, MULTIPLE_OF(LEN52, 10)); + ifma_BN_to_mb8(n_mb8, n_pa, RSA_BITLEN); + + /* compute k0[] */ + ifma_montFactor52_mb8(k0_mb8, n_mb8[0]); + + /* compute to_Montgomery domain converters */ + ifma_montRR52x_mb8(rr_mb8, n_mb8, RSA_BITLEN); + + /* convert input to ifma fmt */ + ifma_HexStr8_to_mb8(inout_mb8, from_pa, RSA_BITLEN); + + /* exponentiation */ + EXP52x60_pub65537_mb8(inout_mb8, + (const int64u(*)[8])inout_mb8, + (const int64u(*)[8])n_mb8, + (const int64u(*)[8])rr_mb8, + k0_mb8, + (int64u(*)[8])work_buffer); + + /* convert result from ifma fmt */ + ifma_mb8_to_HexStr8(to_pa, (const int64u(*)[8])inout_mb8, RSA_BITLEN); + + #undef RSA_BITLEN + #undef LEN52 +} + + +void ifma_ssl_rsa4K_pub_layer_mb8(const int8u* const from_pa[8], + int8u* const to_pa[8], + const BIGNUM* const n_pa[8]) +{ + #define RSA_BITLEN (RSA_4K) + #define LEN52 (NUMBER_OF_DIGITS(RSA_BITLEN, DIGIT_SIZE)) + + /* allocate mb8 buffers */ + __ALIGN64 int64u k0_mb8[8]; + __ALIGN64 int64u rr_mb8[LEN52][8]; + __ALIGN64 int64u inout_mb8[LEN52][8]; + /* MULTIPLE_OF_10 because of AMS5x52x79_diagonal_mb8() implementaion specific */ + __ALIGN64 int64u n_mb8[MULTIPLE_OF(LEN52, 10)][8]; + /* allocate stack for red(undant) result and multiplier */ + __ALIGN64 int64u work_buffer[LEN52*2+1][8]; + + /* convert modulus to ifma fmt */ + zero_mb8(n_mb8, MULTIPLE_OF(LEN52, 10)); + ifma_BN_to_mb8(n_mb8, n_pa, RSA_BITLEN); + + /* compute k0[] */ + ifma_montFactor52_mb8(k0_mb8, n_mb8[0]); + + /* compute to_Montgomery domain converters */ + ifma_montRR52x_mb8(rr_mb8, n_mb8, RSA_BITLEN); + + /* convert input to ifma fmt */ + ifma_HexStr8_to_mb8(inout_mb8, from_pa, RSA_BITLEN); + + /* exponentiation */ + EXP52x79_pub65537_mb8(inout_mb8, + (const int64u(*)[8])inout_mb8, + (const int64u(*)[8])n_mb8, + (const int64u(*)[8])rr_mb8, + k0_mb8, + (int64u(*)[8])work_buffer); + + /* convert result from ifma fmt */ + ifma_mb8_to_HexStr8(to_pa, (const int64u(*)[8])inout_mb8, RSA_BITLEN); + + #undef RSA_BITLEN + #undef LEN52 +} + +#endif /* BN_OPENSSL_DISABLE */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm2/ifma_arith_nsm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm2/ifma_arith_nsm2.c new file mode 100644 index 0000000..964d779 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm2/ifma_arith_nsm2.c @@ -0,0 +1,219 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +/* Constants */ +#define LEN52 NUMBER_OF_DIGITS(256,DIGIT_SIZE) + +/* +// EC SM2 prime base point order +// in 2^52 radix +*/ +__ALIGN64 static const int64u nsm2_mb[LEN52][8] = { + { REP8_DECL(0x000bf40939d54123) }, + { REP8_DECL(0x0006b21c6052b53b) }, + { REP8_DECL(0x000fffffff7203df) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x0000fffffffeffff) } +}; + +__ALIGN64 static const int64u nsm2x2_mb[LEN52][8] = { + { REP8_DECL(0x0007e81273aa8246) }, + { REP8_DECL(0x000d6438c0a56a77) }, + { REP8_DECL(0x000ffffffee407be) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x0001fffffffdffff) } +}; + +/* k0 = -( (1/nsm2 mod 2^DIGIT_SIZE) ) mod 2^DIGIT_SIZE */ +__ALIGN64 static const int64u nsm2_k0_mb[8] = { + REP8_DECL(0x000f9e8872350975) +}; + +/* to Montgomery conversion constant +// rr = 2^((LEN52*DIGIT_SIZE)*2) mod nsm2 +*/ +__ALIGN64 static const int64u nsm2_rr_mb[LEN52][8] = { + { REP8_DECL(0x000c16674a517de6) }, + { REP8_DECL(0x000507a6e5f7c418) }, + { REP8_DECL(0x000fe0d44507dc1c) }, + { REP8_DECL(0x0003b620fc84c3af) }, + { REP8_DECL(0x0000b5e412c02b3d) } +}; + +/*===================================================================== + + Specialized single operations in nsm2 - sqr & mul + +=====================================================================*/ +EXTERN_C U64* MB_FUNC_NAME(ifma_nsm2_)(void) +{ + return (U64*)nsm2_mb; +} + +void MB_FUNC_NAME(ifma_ams52_nsm2_)(U64 r[], const U64 a[]) +{ + MB_FUNC_NAME(ifma_ams52x5_)(r, a, (U64*)nsm2_mb, nsm2_k0_mb); +} + +void MB_FUNC_NAME(ifma_amm52_nsm2_)(U64 r[], const U64 a[], const U64 b[]) +{ + MB_FUNC_NAME(ifma_amm52x5_)(r, a, b, (U64*)nsm2_mb, nsm2_k0_mb); +} + +void MB_FUNC_NAME(ifma_tomont52_nsm2_)(U64 r[], const U64 a[]) +{ + MB_FUNC_NAME(ifma_amm52x5_)(r, a, (U64*)nsm2_rr_mb, (U64*)nsm2_mb, nsm2_k0_mb); +} + +void MB_FUNC_NAME(ifma_frommont52_nsm2_)(U64 r[], const U64 a[]) +{ + MB_FUNC_NAME(ifma_amm52_nsm2_)(r, a, (U64*)ones); + MB_FUNC_NAME(ifma_fastred52_pnsm2_)(r, r); +} + +/* +// computes r = 1/z = z^(nsm2-2) mod nsm2 +// +// note: z in in Montgomery domain (as soon mul() and sqr() below are amm-functions +// r in Montgomery domain too +*/ +#define sqr_nsm2 MB_FUNC_NAME(ifma_ams52_nsm2_) +#define mul_nsm2 MB_FUNC_NAME(ifma_amm52_nsm2_) + +void MB_FUNC_NAME(ifma_aminv52_nsm2_)(U64 r[], const U64 z[]) +{ + int i; + + // pwr_z_Tbl[i][] = z^i, i=0,..,15 + __ALIGN64 U64 pwr_z_Tbl[16][LEN52]; + + MB_FUNC_NAME(ifma_tomont52_nsm2_)(pwr_z_Tbl[0], (U64*)ones); + MB_FUNC_NAME(mov_FESM2_)(pwr_z_Tbl[1], z); + + for(i=2; i<16; i+=2) { + sqr_nsm2(pwr_z_Tbl[i], pwr_z_Tbl[i/2]); + mul_nsm2(pwr_z_Tbl[i+1], pwr_z_Tbl[i], z); + } + + // pwr = (nsm2-2) in big endian + int8u pwr[] = "\xFF\xFF\xFF\xFE\xFF\xFF\xFF\xFF" + "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" + "\x72\x03\xDF\x6B\x21\xC6\x05\x2B" + "\x53\xBB\xF4\x09\x39\xD5\x41\x21"; + // init r = 1 + MB_FUNC_NAME(mov_FESM2_)(r, pwr_z_Tbl[0]); + + for(i=0; i<32; i++) { + int v = pwr[i]; + int hi = (v>>4) &0xF; + int lo = v & 0xF; + + sqr_nsm2(r, r); + sqr_nsm2(r, r); + sqr_nsm2(r, r); + sqr_nsm2(r, r); + if(hi) + mul_nsm2(r, r, pwr_z_Tbl[hi]); + sqr_nsm2(r, r); + sqr_nsm2(r, r); + sqr_nsm2(r, r); + sqr_nsm2(r, r); + if(lo) + mul_nsm2(r, r, pwr_z_Tbl[lo]); + } +} + +/*===================================================================== + + Specialized single operations in nsm2 - add, sub & neg + +=====================================================================*/ + +void MB_FUNC_NAME(ifma_add52_nsm2_)(U64 r[], const U64 a[], const U64 b[]) +{ + MB_FUNC_NAME(ifma_add52x5_)(r, a, b, (U64*)nsm2x2_mb); +} + +void MB_FUNC_NAME(ifma_sub52_nsm2_)(U64 r[], const U64 a[], const U64 b[]) +{ + MB_FUNC_NAME(ifma_sub52x5_)(r, a, b, (U64*)nsm2x2_mb); +} + +void MB_FUNC_NAME(ifma_neg52_nsm2_)(U64 r[], const U64 a[]) +{ + MB_FUNC_NAME(ifma_neg52x5_)(r, a, (U64*)nsm2x2_mb); +} + +static __mb_mask MB_FUNC_NAME(lt_mbx_digit_)(const U64 a, const U64 b, const __mb_mask lt_mask) +{ + U64 d = mask_sub64(sub64(a, b), lt_mask, sub64(a, b), set1(1)); + return cmp64_mask(d, get_zero64(), _MM_CMPINT_LT); +} + +/* r = (a>=nsm2)? a-nsm2 : a */ +void MB_FUNC_NAME(ifma_fastred52_pnsm2_)(U64 R[], const U64 A[]) +{ + /* r = a - b */ + U64 r0 = sub64(A[0], ((U64*)(nsm2_mb))[0]); + U64 r1 = sub64(A[1], ((U64*)(nsm2_mb))[1]); + U64 r2 = sub64(A[2], ((U64*)(nsm2_mb))[2]); + U64 r3 = sub64(A[3], ((U64*)(nsm2_mb))[3]); + U64 r4 = sub64(A[4], ((U64*)(nsm2_mb))[4]); + + /* lt = {r0 - r4} < 0 */ + __mb_mask + lt = MB_FUNC_NAME(lt_mbx_digit_)(r0, get_zero64(), 0); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r1, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r2, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r3, get_zero64(), lt); + lt = MB_FUNC_NAME(lt_mbx_digit_)(r4, get_zero64(), lt); + + r0 = mask_mov64(A[0], ~lt, r0); + r1 = mask_mov64(A[1], ~lt, r1); + r2 = mask_mov64(A[2], ~lt, r2); + r3 = mask_mov64(A[3], ~lt, r3); + r4 = mask_mov64(A[4], ~lt, r4); + + /* normalize r0 - r4 */ + NORM_ASHIFTR(r, 0,1) + NORM_ASHIFTR(r, 1,2) + NORM_ASHIFTR(r, 2,3) + NORM_ASHIFTR(r, 3,4) + + R[0] = r0; + R[1] = r1; + R[2] = r2; + R[3] = r3; + R[4] = r4; +} + +__mb_mask MB_FUNC_NAME(ifma_cmp_lt_nsm2_)(const U64 a[]) +{ + return MB_FUNC_NAME(cmp_lt_FESM2_)(a,(const U64 (*))nsm2_mb); +} + +__mb_mask MB_FUNC_NAME(ifma_check_range_nsm2_)(const U64 A[]) +{ + __mb_mask + mask = MB_FUNC_NAME(is_zero_FESM2_)(A); + mask |= ~MB_FUNC_NAME(ifma_cmp_lt_nsm2_)(A); + + return mask; +} + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm2/ifma_arith_psm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm2/ifma_arith_psm2.c new file mode 100644 index 0000000..f51d5a0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm2/ifma_arith_psm2.c @@ -0,0 +1,503 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +/* +// prime256 = 2^256 - 2^224 - 2^96 + 2^64 - 1 +// in 2^52 radix +*/ +__ALIGN64 static const int64u psm2_mb[PSM2_LEN52][8] = { + { REP8_DECL(0xfffffffffffff) }, + { REP8_DECL(0xff00000000fff) }, + { REP8_DECL(0xfffffffffffff) }, + { REP8_DECL(0xfffffffffffff) }, + { REP8_DECL(0x0fffffffeffff) } +}; + +__ALIGN64 static const int64u psm2x2_mb[PSM2_LEN52][8] = { + { REP8_DECL(0xffffffffffffe) }, + { REP8_DECL(0xfe00000001fff) }, + { REP8_DECL(0xfffffffffffff) }, + { REP8_DECL(0xfffffffffffff) }, + { REP8_DECL(0x1fffffffdffff) } +}; + +/* to Montgomery conversion constant +// rr = 2^((PSM2_LEN52*DIGIT_SIZE)*2) mod psm2 +*/ +__ALIGN64 static const int64u psm2_rr_mb[PSM2_LEN52][8] = { + { REP8_DECL(0x0020000000300) }, + { REP8_DECL(0xffffffff00000) }, + { REP8_DECL(0x0000100000002) }, + { REP8_DECL(0x0200000001000) }, + { REP8_DECL(0x0000004000000) } +}; + +/* other constants */ +__ALIGN64 static const int64u VDIGIT_MASK_[8] = + {DIGIT_MASK, DIGIT_MASK, DIGIT_MASK, DIGIT_MASK, + DIGIT_MASK, DIGIT_MASK, DIGIT_MASK, DIGIT_MASK}; +#define VDIGIT_MASK loadu64(VDIGIT_MASK_) + +#define ROUND_MUL(I, J, LO, HI) \ + LO = fma52lo(LO, va[I], vb[J]); \ + HI = fma52hi(HI, va[I], vb[J]) + + +//TODO: reduction +#define MUL_ADD_PSM2(u, res0, res1, res2, res3, res4, res5) \ +{ \ + /* a * ( 2^52 - 1 ) = 2^52 * a - a */ \ + U64 u = and64(res0, VDIGIT_MASK); \ + /* res0 = sub64(res0, u); */ /* Zero out low 52 bits */ \ + res1 = add64(res1, srli64(res0, DIGIT_SIZE)); /* Carry propagation */ \ + res1 = add64(res1, u); \ + res1 = fma52lo(res1, u, ((U64*)psm2_mb)[1]); \ + res2 = fma52hi(res2, u, ((U64*)psm2_mb)[1]); \ + res2 = sub64(res2, u); \ + res4 = add64(res4, u); \ + res4 = fma52lo(res4, u, ((U64*)psm2_mb)[4]); \ + res5 = fma52hi(res5, u, ((U64*)psm2_mb)[4]); \ +} + +void MB_FUNC_NAME(ifma_amm52_psm2_)(U64 r[], const U64 va[], const U64 vb[]) +{ + U64 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9; + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = get_zero64(); + + // full multiplication + ROUND_MUL(0, 0, r0, r1); + ROUND_MUL(1, 0, r1, r2); + ROUND_MUL(2, 0, r2, r3); + ROUND_MUL(3, 0, r3, r4); + ROUND_MUL(4, 0, r4, r5); + + ROUND_MUL(0, 1, r1, r2); + ROUND_MUL(1, 1, r2, r3); + ROUND_MUL(2, 1, r3, r4); + ROUND_MUL(3, 1, r4, r5); + ROUND_MUL(4, 1, r5, r6); + + ROUND_MUL(0, 2, r2, r3); + ROUND_MUL(1, 2, r3, r4); + ROUND_MUL(2, 2, r4, r5); + ROUND_MUL(3, 2, r5, r6); + ROUND_MUL(4, 2, r6, r7); + + ROUND_MUL(0, 3, r3, r4); + ROUND_MUL(1, 3, r4, r5); + ROUND_MUL(2, 3, r5, r6); + ROUND_MUL(3, 3, r6, r7); + ROUND_MUL(4, 3, r7, r8); + + ROUND_MUL(0, 4, r4, r5); + ROUND_MUL(1, 4, r5, r6); + ROUND_MUL(2, 4, r6, r7); + ROUND_MUL(3, 4, r7, r8); + ROUND_MUL(4, 4, r8, r9); + + //reduction + MUL_ADD_PSM2(u0, r0, r1, r2, r3, r4, r5); + MUL_ADD_PSM2(u1, r1, r2, r3, r4, r5, r6); + MUL_ADD_PSM2(u2, r2, r3, r4, r5, r6, r7); + MUL_ADD_PSM2(u3, r3, r4, r5, r6, r7, r8); + MUL_ADD_PSM2(u4, r4, r5, r6, r7, r8, r9); + + NORM_LSHIFTR(r, 5, 6) + NORM_LSHIFTR(r, 6, 7) + NORM_LSHIFTR(r, 7, 8) + NORM_LSHIFTR(r, 8, 9) + + r[0] = r5; + r[1] = r6; + r[2] = r7; + r[3] = r8; + r[4] = r9; +} + +void MB_FUNC_NAME(ifma_ams52_psm2_)(U64 r[], const U64 va[]) +{ + U64 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9; + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = get_zero64(); + + const U64* vb = va; + + ROUND_MUL(0, 1, r1, r2); + ROUND_MUL(0, 2, r2, r3); + ROUND_MUL(0, 3, r3, r4); + ROUND_MUL(0, 4, r4, r5); + ROUND_MUL(1, 4, r5, r6); + ROUND_MUL(2, 4, r6, r7); + ROUND_MUL(3, 4, r7, r8); + + ROUND_MUL(1, 2, r3, r4); + ROUND_MUL(1, 3, r4, r5); + + ROUND_MUL(2, 3, r5, r6); + + r1 = add64(r1, r1); + r2 = add64(r2, r2); + r3 = add64(r3, r3); + r4 = add64(r4, r4); + r5 = add64(r5, r5); + r6 = add64(r6, r6); + r7 = add64(r7, r7); + r8 = add64(r8, r8); + + ROUND_MUL(0, 0, r0, r1); + ROUND_MUL(1, 1, r2, r3); + ROUND_MUL(2, 2, r4, r5); + ROUND_MUL(3, 3, r6, r7); + ROUND_MUL(4, 4, r8, r9); + + //reduction + MUL_ADD_PSM2(u0, r0, r1, r2, r3, r4, r5); + MUL_ADD_PSM2(u1, r1, r2, r3, r4, r5, r6); + MUL_ADD_PSM2(u2, r2, r3, r4, r5, r6, r7); + MUL_ADD_PSM2(u3, r3, r4, r5, r6, r7, r8); + MUL_ADD_PSM2(u4, r4, r5, r6, r7, r8, r9); + + NORM_LSHIFTR(r, 5, 6) + NORM_LSHIFTR(r, 6, 7) + NORM_LSHIFTR(r, 7, 8) + NORM_LSHIFTR(r, 8, 9) + + r[0] = r5; + r[1] = r6; + r[2] = r7; + r[3] = r8; + r[4] = r9; + +} + +#define sqr_psm2 MB_FUNC_NAME(ifma_ams52_psm2_) +#define mul_psm2 MB_FUNC_NAME(ifma_amm52_psm2_) + +#define sqr_psm2_x2(res,target) \ +{ \ + sqr_psm2(res,target); \ + sqr_psm2(res,res); \ +} + +#define sqr_psm2_x4(res,target) \ +{ \ + sqr_psm2_x2(res,target); \ + sqr_psm2_x2(res,res); \ +} + +#define sqr_psm2_x8(res,target) \ +{ \ + sqr_psm2_x4(res,target); \ + sqr_psm2_x4(res,res); \ +} + +#define sqr_psm2_x16(res,target) \ +{ \ + sqr_psm2_x8(res,target); \ + sqr_psm2_x8(res,res); \ +} + +#define sqr_psm2_x32(res,target) \ +{ \ + sqr_psm2_x16(res,target); \ + sqr_psm2_x16(res,res); \ +} + +#define sqr_psm2_x64(res,target) \ +{ \ + sqr_psm2_x32(res,target); \ + sqr_psm2_x32(res,res); \ +} + +void MB_FUNC_NAME(ifma_aminv52_psm2_)(U64 r[], const U64 z[]) +{ + __ALIGN64 U64 tmp1[PSM2_LEN52]; + __ALIGN64 U64 tmp2[PSM2_LEN52]; + + __ALIGN64 U64 D[PSM2_LEN52]; + __ALIGN64 U64 E[PSM2_LEN52]; + __ALIGN64 U64 F[PSM2_LEN52]; + + sqr_psm2(tmp1,z); + mul_psm2(F,tmp1,z); /* F = z^3 */ + + /* tmp2 = z^0xC */ + sqr_psm2_x2(tmp2,F); + + mul_psm2(D, tmp2, z); /* D = z^0xD */ + mul_psm2(E, tmp2, tmp1); /* E = z^0xE */ + mul_psm2(F, tmp2, F); /* F = z^0xF */ + + /* tmp2 = z^0xF0 */ + sqr_psm2_x4(tmp2,F); + + mul_psm2(D, tmp2, D); /* D = z^0xFD */ + mul_psm2(E, tmp2, E); /* E = z^0xFE */ + mul_psm2(F, tmp2, F); /* F = z^0xFF */ + + /* tmp2 = z^0xFF00 */ + sqr_psm2_x8(tmp2,F); + + mul_psm2(D, tmp2, D); /* D = z^0xFFFD */ + mul_psm2(E, tmp2, E); /* E = z^0xFFFE */ + mul_psm2(F, tmp2, F); /* F = z^0xFFFF */ + + /* tmp2 = z^0xFFFF0000 */ + sqr_psm2_x16(tmp2,F); + + mul_psm2(D, tmp2, D); /* D = z^0xFFFFFFFD */ + mul_psm2(E, tmp2, E); /* E = z^0xFFFFFFFE */ + mul_psm2(F, tmp2, F); /* F = z^0xFFFFFFFF */ + + /* z ^ FFFFFFFE 00000000 */ + sqr_psm2_x32(r,E); + /* z ^ FFFFFFFE FFFFFFFF */ + mul_psm2(r,r,F); + /* z ^ FFFFFFFE FFFFFFFF 00000000 */ + sqr_psm2_x32(r,r); + /* z ^ FFFFFFFE FFFFFFFF FFFFFFFF */ + mul_psm2(r,r,F); + /* z ^ FFFFFFFE FFFFFFFF FFFFFFFF 00000000 */ + sqr_psm2_x32(r,r); + /* z ^ FFFFFFFE FFFFFFFF FFFFFFFF FFFFFFFF */ + mul_psm2(r,r,F); + /* z ^ FFFFFFFE FFFFFFFF FFFFFFFF FFFFFFFF 00000000 */ + sqr_psm2_x32(r,r); + /* z ^ FFFFFFFE FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF */ + mul_psm2(r,r,F); + /* z ^ FFFFFFFE FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF 00000000 00000000 */ + sqr_psm2_x64(r,r); + /* z ^ FFFFFFFE FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF 00000000 FFFFFFFF */ + mul_psm2(r,r,F); + /* z ^ FFFFFFFE FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF 00000000 FFFFFFFF 00000000 */ + sqr_psm2_x32(r,r); + /* z ^ FFFFFFFE FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF 00000000 FFFFFFFF FFFFFFFD */ + mul_psm2(r,r,D); + +} + +void MB_FUNC_NAME(ifma_reduce52_psm2_)(U64 R[], const U64 A[]) +{ + /* r = a - p384_mb */ + U64 r0 = sub64(A[0], ((U64*)(psm2_mb))[0]); + U64 r1 = sub64(A[1], ((U64*)(psm2_mb))[1]); + U64 r2 = sub64(A[2], ((U64*)(psm2_mb))[2]); + U64 r3 = sub64(A[3], ((U64*)(psm2_mb))[3]); + U64 r4 = sub64(A[4], ((U64*)(psm2_mb))[4]); + + /* normalize r0 - r4 */ + NORM_ASHIFTR(r, 0, 1) + NORM_ASHIFTR(r, 1, 2) + NORM_ASHIFTR(r, 2, 3) + NORM_ASHIFTR(r, 3, 4) + + /* r = a> 1 */ + mask = sub64(get_zero64(), and64(t1, one)); + t0 = add64(t0, and64(base, mask)); + t0 = srli64(t0, 1); + + mask = sub64(get_zero64(), and64(t2, one)); + t1 = add64(t1, and64(base, mask)); + t1 = srli64(t1, 1); + + mask = sub64(get_zero64(), and64(t3, one)); + t2 = add64(t2, and64(base, mask)); + t2 = srli64(t2, 1); + + mask = sub64(get_zero64(), and64(t4, one)); + t3 = add64(t3, and64(base, mask)); + t3 = srli64(t3, 1); + + t4 = srli64(t4, 1); + + /* normalize t0, t1, t2, t3, t4 */ + NORM_LSHIFTR(t, 0,1) + NORM_LSHIFTR(t, 1,2) + NORM_LSHIFTR(t, 2,3) + NORM_LSHIFTR(t, 3,4) + + r[0] = t0; + r[1] = t1; + r[2] = t2; + r[3] = t3; + r[4] = t4; +} + +__mb_mask MB_FUNC_NAME(ifma_cmp_lt_psm2_)(const U64 a[]) +{ + return MB_FUNC_NAME(cmp_lt_FESM2_)(a,(const U64 (*))psm2_mb); +} + +__mb_mask MB_FUNC_NAME(ifma_check_range_psm2_)(const U64 A[]) +{ + __mb_mask + mask = MB_FUNC_NAME(is_zero_FESM2_)(A); + mask |= ~MB_FUNC_NAME(ifma_cmp_lt_psm2_)(A); + + return mask; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm2/ifma_ecdh_sm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm2/ifma_ecdh_sm2.c new file mode 100644 index 0000000..0637fd1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm2/ifma_ecdh_sm2.c @@ -0,0 +1,256 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +#include +#include +#include +#include + +#ifndef BN_OPENSSL_DISABLE +#include +#endif + +#ifndef BN_OPENSSL_DISABLE +/* +// Computes shared key +// pa_shared_key[] array of pointers to the shared keys +// pa_skey[] array of pointers to the own (ephemeral) private keys +// pa_pubx[] array of pointers to the party's public keys X-coordinates +// pa_puby[] array of pointers to the party's public keys Y-coordinates +// pa_pubz[] array of pointers to the party's public keys Z-coordinates (or NULL, if affine coordinate requested) +// pBuffer pointer to the scratch buffer +// +// Note: +// input party's public key depends on is pa_pubz[] parameter and represented either +// - in (X:Y:Z) projective Jacobian coordinates +// or +// - in (x:y) affine coordinate +*/ +DLL_PUBLIC +mbx_status mbx_sm2_ecdh_ssl_mb8(int8u* pa_shared_key[8], + const BIGNUM* const pa_skey[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* pa_pubz!=0 means the output is in Jacobian projective coordinates */ + int use_jproj_coords = NULL!=pa_pubz; + + /* test input pointers */ + if(NULL==pa_shared_key || NULL==pa_skey || NULL==pa_pubx || NULL==pa_puby) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + int8u* shared = pa_shared_key[buf_no]; + const BIGNUM* skey = pa_skey[buf_no]; + const BIGNUM* pubx = pa_pubx[buf_no]; + const BIGNUM* puby = pa_puby[buf_no]; + const BIGNUM* pubz = use_jproj_coords? pa_pubz[buf_no] : NULL; + + /* if any of pointer NULL set error status */ + if(NULL==shared || NULL==skey|| NULL==pubx || NULL==puby || (use_jproj_coords && NULL==pubz)) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* + // processing + */ + + /* zero padded private keys */ + U64 secretz[PSM2_LEN64+1]; + ifma_BN_transpose_copy((int64u (*)[8])secretz, (const BIGNUM**)pa_skey, PSM2_BITSIZE); + secretz[PSM2_LEN64] = get_zero64(); + + status |= MBX_SET_STS_BY_MASK(status, is_zero(secretz, PSM2_LEN64+1), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + return status; + } + + SM2_POINT P; + + /* set party's public */ + ifma_BN_to_mb8((int64u (*)[8])P.X, (const BIGNUM* (*))pa_pubx, PSM2_BITSIZE); /* P-> radix 2^52 */ + ifma_BN_to_mb8((int64u (*)[8])P.Y, (const BIGNUM* (*))pa_puby, PSM2_BITSIZE); + if(use_jproj_coords) + ifma_BN_to_mb8((int64u (*)[8])P.Z, (const BIGNUM* (*))pa_pubz, PSM2_BITSIZE); + else + MB_FUNC_NAME(mov_FESM2_)(P.Z, (U64*)ones); + /* convert to Montgomery */ + MB_FUNC_NAME(ifma_tomont52_psm2_)(P.X, P.X); + MB_FUNC_NAME(ifma_tomont52_psm2_)(P.Y, P.Y); + MB_FUNC_NAME(ifma_tomont52_psm2_)(P.Z, P.Z); + + /* check if P does not belong to EC */ + __mb_mask not_on_curve_mask = ~MB_FUNC_NAME(ifma_is_on_curve_psm2_)(&P, use_jproj_coords); + /* set points out of EC to infinity */ + MB_FUNC_NAME(mask_set_point_to_infinity_)(&P, not_on_curve_mask); + /* update status */ + status |= MBX_SET_STS_BY_MASK(status, not_on_curve_mask, MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + return status; + } + + SM2_POINT R; + + /* compute R = [secretz]*P */ + MB_FUNC_NAME(ifma_ec_sm2_mul_point_)(&R, &P, secretz); + + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + + /* return affine R.x */ + __ALIGN64 U64 Z2[PSM2_LEN52]; + ifma_aminv52_psm2_mb8(Z2, R.Z); /* 1/Z */ + ifma_ams52_psm2_mb8(Z2, Z2); /* 1/Z^2 */ + ifma_amm52_psm2_mb8(R.X, R.X, Z2); /* x = (X) * (1/Z^2) */ + /* to regular domain */ + MB_FUNC_NAME(ifma_frommont52_psm2_)(R.X, R.X); + + /* store result */ + ifma_mb8_to_HexStr8(pa_shared_key, (const int64u (*)[8])R.X, PSM2_BITSIZE); + + /* clear computed shared keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])&R, sizeof(R)/sizeof(U64)); + + return status; +} +#endif // BN_OPENSSL_DISABLE + +DLL_PUBLIC +mbx_status mbx_sm2_ecdh_mb8(int8u* pa_shared_key[8], + const int64u* const pa_skey[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* pa_pubz!=0 means the output is in Jacobian projective coordinates */ + int use_jproj_coords = NULL!=pa_pubz; + + /* test input pointers */ + if(NULL==pa_shared_key || NULL==pa_skey || NULL==pa_pubx || NULL==pa_puby) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + int8u* shared = pa_shared_key[buf_no]; + const int64u* skey = pa_skey[buf_no]; + const int64u* pubx = pa_pubx[buf_no]; + const int64u* puby = pa_puby[buf_no]; + const int64u* pubz = use_jproj_coords? pa_pubz[buf_no] : NULL; + + /* if any of pointer NULL set error status */ + if(NULL==shared || NULL==skey || NULL==pubx || NULL==puby || (use_jproj_coords && NULL==pubz)) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* + // processing + */ + + /* zero padded private keys */ + U64 secretz[PSM2_LEN64+1]; + ifma_BNU_transpose_copy((int64u (*)[8])secretz, (const int64u**)pa_skey, PSM2_BITSIZE); + secretz[PSM2_LEN64] = get_zero64(); + + status |= MBX_SET_STS_BY_MASK(status, is_zero(secretz, PSM2_LEN64+1), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + return status; + } + + SM2_POINT P; + + /* set party's public */ + ifma_BNU_to_mb8((int64u (*)[8])P.X, (const int64u* (*))pa_pubx, PSM2_BITSIZE); // P-> crypto_mb radix 2^52 + ifma_BNU_to_mb8((int64u (*)[8])P.Y, (const int64u* (*))pa_puby, PSM2_BITSIZE); + if(use_jproj_coords) + ifma_BNU_to_mb8((int64u (*)[8])P.Z, (const int64u* (*))pa_pubz, PSM2_BITSIZE); + else + MB_FUNC_NAME(mov_FESM2_)(P.Z, (U64*)ones); + /* convert to Montgomery */ + MB_FUNC_NAME(ifma_tomont52_psm2_)(P.X, P.X); + MB_FUNC_NAME(ifma_tomont52_psm2_)(P.Y, P.Y); + MB_FUNC_NAME(ifma_tomont52_psm2_)(P.Z, P.Z); + + /* check if P does not belong to EC */ + __mb_mask not_on_curve_mask = ~MB_FUNC_NAME(ifma_is_on_curve_psm2_)(&P, use_jproj_coords); + /* set points out of EC to infinity */ + MB_FUNC_NAME(mask_set_point_to_infinity_)(&P, not_on_curve_mask); + /* update status */ + status |= MBX_SET_STS_BY_MASK(status, not_on_curve_mask, MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + return status; + } + + SM2_POINT R; + + /* compute R = [secretz]*P */ + MB_FUNC_NAME(ifma_ec_sm2_mul_point_)(&R, &P, secretz); + + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])secretz, sizeof(secretz)/sizeof(U64)); + + /* return affine R.x */ + __ALIGN64 U64 Z2[PSM2_LEN52]; + ifma_aminv52_psm2_mb8(Z2, R.Z); /* 1/Z */ + ifma_ams52_psm2_mb8(Z2, Z2); /* 1/Z^2 */ + ifma_amm52_psm2_mb8(R.X, R.X, Z2); /* x = (X) * (1/Z^2) */ + /* to regular domain */ + MB_FUNC_NAME(ifma_frommont52_psm2_)(R.X, R.X); + + /* store result */ + ifma_mb8_to_HexStr8(pa_shared_key, (const int64u (*)[8])R.X, PSM2_BITSIZE); + + /* clear computed shared keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])&R, sizeof(R)/sizeof(U64)); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm2/ifma_ecdsa_sm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm2/ifma_ecdsa_sm2.c new file mode 100755 index 0000000..a7c8d71 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm2/ifma_ecdsa_sm2.c @@ -0,0 +1,884 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include +#include +#include +#include + +#ifndef BN_OPENSSL_DISABLE +#include +#include +#ifdef OPENSSL_IS_BORINGSSL +#include +#endif +#endif + +/* constants for Z digest computation */ +/* EC SM2 equation coeficient a, big endian */ +static const int8u a[] = "\xFF\xFF\xFF\xFE\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFC"; +/* EC SM2 equation coeficient b, big endian */ +static const int8u b[] = "\x28\xE9\xFA\x9E\x9D\x9F\x5E\x34\x4D\x5A\x9E\x4B\xCF\x65\x09\xA7\xF3\x97\x89\xF5\x15\xAB\x8F\x92\xDD\xBC\xBD\x41\x4D\x94\x0E\x93"; +/* x coordinate of the EC SM2 generator point in affine coordinates, big endian */ +static const int8u xG[] = "\x32\xC4\xAE\x2C\x1F\x19\x81\x19\x5F\x99\x04\x46\x6A\x39\xC9\x94\x8F\xE3\x0B\xBF\xF2\x66\x0B\xE1\x71\x5A\x45\x89\x33\x4C\x74\xC7"; +/* y coordinate of the EC SM2 generator point in affine coordinates, big endian */ +static const int8u yG[] = "\xBC\x37\x36\xA2\xF4\xF6\x77\x9C\x59\xBD\xCE\xE3\x6B\x69\x21\x53\xD0\xA9\x87\x7C\xC6\x2A\x47\x40\x02\xDF\x32\xE5\x21\x39\xF0\xA0"; + +static const int8u * pa_a[] = {REP8_DECL( a)}; +static const int8u * pa_b[] = {REP8_DECL( b)}; +static const int8u * pa_xG[] = {REP8_DECL(xG)}; +static const int8u * pa_yG[] = {REP8_DECL(yG)}; + +static int len_2[8] = {REP8_DECL( 2)}; +static int len_32[8] = {REP8_DECL(32)}; + +/* +// common functions +*/ + +/* compute Z digest = SM3( ENTL || ID || a || b ||xG || yG || xA || yA ) */ +static void sm2_ecdsa_compute_z_digest(int8u* pa_z_digest[8], + const int8u* pa_user_id[8], + const int user_id_len[8], + const int8u* pa_pubx[8], + const int8u* pa_puby[8]) +{ + SM3_CTX_mb8 ctx; + SM3_CTX_mb8* p_ctx = &ctx; + + sm3_init_mb8(p_ctx); + + int8u entl_data[8][2]; + const int8u* pa_entl[8]; + + for (int i = 0; i < 8; ++i) + { + int entl = ((user_id_len[i] * 8) & 0xFFFF); + + entl_data[i][1] = entl & 0xFF; + entl_data[i][0] = entl >> 8; + + pa_entl[i] = &(entl_data[i][0]); + } + + sm3_update_mb8(pa_entl, len_2, p_ctx); + + sm3_update_mb8(pa_user_id, (int *)user_id_len, p_ctx); + + sm3_update_mb8(pa_a, len_32, p_ctx); + sm3_update_mb8(pa_b, len_32, p_ctx); + sm3_update_mb8(pa_xG, len_32, p_ctx); + sm3_update_mb8(pa_yG, len_32, p_ctx); + + sm3_update_mb8(pa_pubx, len_32, p_ctx); + sm3_update_mb8(pa_puby, len_32, p_ctx); + + sm3_final_mb8(pa_z_digest, p_ctx); + return; +} + +static void sm2_ecdsa_compute_msg_digest(int8u* pa_msg_digest[8], + const int8u* pa_z_digest[8], + const int8u* pa_msg[8], + const int msg_len[8]) +{ + SM3_CTX_mb8 ctx; + SM3_CTX_mb8* p_ctx = &ctx; + + sm3_init_mb8(p_ctx); + + sm3_update_mb8(pa_z_digest, len_32, p_ctx); + sm3_update_mb8(pa_msg, (int*)msg_len, p_ctx); + + sm3_final_mb8(pa_msg_digest, p_ctx); + return; +} + +static void reverse_inplace(int8u* inp, int len) +{ + for(int i = 0; i < len/2; ++i) { + int8u a = inp[i]; + inp[i] = inp[len - 1 - i]; + inp[len - 1 - i] = a; + } + return; +} + +/* +// SM2_POINT* P - Public key in Jacobian projective coordinates, coordinates are in Montgomery over p +// int64u* pa_rev_bytes_pubX[8] - X coordinate (affine), bytes reversed for future hashing +// int64u* pa_rev_bytes_pubY[8] - Y coordinate (affine), bytes reversed for future hashing +*/ +static mbx_status sm2_ecdsa_process_pubkeys(SM2_POINT* P, + int64u* pa_rev_bytes_pubX[8], + int64u* pa_rev_bytes_pubY[8], + int use_jproj_coords, + mbx_status current_status) +{ + mbx_status status = current_status; + + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_psm2_)(P->X), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_psm2_)(P->Y), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(use_jproj_coords) { + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_psm2_)(P->Z), MBX_STATUS_MISMATCH_PARAM_ERR); + + MB_FUNC_NAME(ifma_tomont52_psm2_)(P->X, P->X); + MB_FUNC_NAME(ifma_tomont52_psm2_)(P->Y, P->Y); + MB_FUNC_NAME(ifma_tomont52_psm2_)(P->Z, P->Z); + + __ALIGN64 U64 X[PSM2_LEN52]; + __ALIGN64 U64 Y[PSM2_LEN52]; + + MB_FUNC_NAME(get_sm2_ec_affine_coords_)(X, Y, P); + + /* convert P coordinates to regular domain */ + MB_FUNC_NAME(ifma_frommont52_psm2_)(X, X); + MB_FUNC_NAME(ifma_frommont52_psm2_)(Y, Y); + + /* convert P coordinates to radix 64 */ + ifma_mb8_to_BNU(pa_rev_bytes_pubX, (const int64u (*)[8])X, PSM2_BITSIZE); + ifma_mb8_to_BNU(pa_rev_bytes_pubY, (const int64u (*)[8])Y, PSM2_BITSIZE); + } else { + ifma_mb8_to_BNU(pa_rev_bytes_pubX, (const int64u (*)[8])P->X, PSM2_BITSIZE); + ifma_mb8_to_BNU(pa_rev_bytes_pubY, (const int64u (*)[8])P->Y, PSM2_BITSIZE); + + MB_FUNC_NAME(ifma_tomont52_psm2_)(P->X, P->X); + MB_FUNC_NAME(ifma_tomont52_psm2_)(P->Y, P->Y); + MB_FUNC_NAME(ifma_tomont52_psm2_)(P->Z, (U64*)ones); + } + + for(int i = 0; i < 8; ++i) { + reverse_inplace((int8u*)pa_rev_bytes_pubX[i], PSM2_LEN8); + reverse_inplace((int8u*)pa_rev_bytes_pubY[i], PSM2_LEN8); + } + + return status; +} + +/* compute and check signature components */ +static mbx_status sm2_ecdsa_sign_mb8(U64 sign_r[], + U64 sign_s[], + U64 msg_digest[], + U64 scalar_eph_skey[], + U64 eph_skey[], + U64 reg_skey[], + mbx_status current_status) +{ + mbx_status status = current_status; + + /* compute public keys from ephemeral private keys */ + SM2_POINT P_eph; + + /* (x1, y1) = [k]G */ + MB_FUNC_NAME(ifma_ec_sm2_mul_pointbase_)(&P_eph, scalar_eph_skey); + + /* extract affine P.x */ + MB_FUNC_NAME(get_sm2_ec_affine_coords_)(/* x = */ P_eph.X, /* y = */ NULL, &P_eph); + + MB_FUNC_NAME(ifma_frommont52_psm2_)(P_eph.X, P_eph.X); + /* x = x mod n */ + MB_FUNC_NAME(ifma_fastred52_pnsm2_)(P_eph.X, P_eph.X); + + /* r = (e + x1) mod n */ + MB_FUNC_NAME(ifma_tomont52_nsm2_)(sign_r, P_eph.X); + MB_FUNC_NAME(ifma_tomont52_nsm2_)(msg_digest, msg_digest); + + MB_FUNC_NAME(ifma_add52_nsm2_)(sign_r, sign_r, msg_digest); /* sign_r = (e + x1) */ + + /* check r component */ + MB_FUNC_NAME(ifma_tomont52_nsm2_)(eph_skey, eph_skey); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FESM2_)(sign_r), MBX_STATUS_SIGNATURE_ERR); + + __ALIGN64 U64 tmp[PSM2_LEN52]; + + /* sign_r + eph_skey == n */ + MB_FUNC_NAME(ifma_add52_nsm2_)(tmp, sign_r, eph_skey); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FESM2_)(tmp), MBX_STATUS_SIGNATURE_ERR); + + if(!MBX_IS_ANY_OK_STS(status) ) + return status; + + MB_FUNC_NAME(zero_)((int64u (*)[8])tmp, sizeof(tmp)/sizeof(U64)); + + /* s = ( (1 + da)^-(1) * (k - r*da) ) */ + MB_FUNC_NAME(ifma_tomont52_nsm2_)(tmp, (U64*)ones); + MB_FUNC_NAME(ifma_tomont52_nsm2_)(reg_skey, reg_skey); + + MB_FUNC_NAME(ifma_amm52_nsm2_)(sign_s, sign_r, reg_skey); /* sign_s = r * da */ + MB_FUNC_NAME(ifma_add52_nsm2_)(reg_skey, reg_skey, tmp); /* reg_skey = 1 + da */ + MB_FUNC_NAME(ifma_aminv52_nsm2_)(reg_skey, reg_skey); /* reg_skey = (1 + da)^(-1) */ + MB_FUNC_NAME(ifma_sub52_nsm2_)(sign_s, eph_skey, sign_s); /* sign_s = k - r * da */ + MB_FUNC_NAME(ifma_amm52_nsm2_)(sign_s, sign_s, reg_skey); /* sign_s = ( (1 + da)^-(1) * (k - r*da) ) */ + + MB_FUNC_NAME(ifma_frommont52_nsm2_)(sign_r, sign_r); + MB_FUNC_NAME(ifma_frommont52_nsm2_)(sign_s, sign_s); + + /* check s component */ + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FESM2_)(sign_s), MBX_STATUS_SIGNATURE_ERR); + + return status; +} + +/* verify signature components */ +static mbx_status sm2_ecdsa_verify_mb8(U64 sign_r[], + U64 sign_s[], + U64 msg_digest[], + SM2_POINT* P, + mbx_status current_status) +{ + mbx_status status = current_status; + + __ALIGN64 U64 t[PSM2_LEN52]; + __ALIGN64 U64 h[PSM2_LEN52]; + /* t = (r + s) mod n */ + MB_FUNC_NAME(ifma_tomont52_nsm2_)(t, sign_r); + MB_FUNC_NAME(ifma_tomont52_nsm2_)(h, sign_s); + + MB_FUNC_NAME(ifma_add52_nsm2_)(t, h, t); /* t = (r + s) */ + + MB_FUNC_NAME(ifma_frommont52_nsm2_)(t, t); + + /* check t != 0 */ + __mb_mask signature_err_mask = MB_FUNC_NAME(is_zero_FE256_)(t); + + __ALIGN64 int64u tmp[8][PSM2_LEN64]; + int64u* pa_tmp[8] = {tmp[0], tmp[1], tmp[2], tmp[3], + tmp[4], tmp[5], tmp[6], tmp[7]}; + + /* (x1`, y1`) = [s]G + [t]P */ + /* convert sign_s to scalar - s */ + ifma_mb8_to_BNU(pa_tmp, (const int64u(*)[8])sign_s, PSM2_BITSIZE); + ifma_BNU_transpose_copy((int64u (*)[8])sign_s, (const int64u(**))pa_tmp, PSM2_BITSIZE); + sign_s[PSM2_LEN64] = get_zero64(); + + SM2_POINT sG; + MB_FUNC_NAME(ifma_ec_sm2_mul_pointbase_)(&sG, sign_s); /* [s]G */ + + /* convert t to scalar - t */ + ifma_mb8_to_BNU(pa_tmp, (const int64u(*)[8])t, PSM2_BITSIZE); + ifma_BNU_transpose_copy((int64u(*)[8])t, (const int64u(**))pa_tmp, PSM2_BITSIZE); + t[PSM2_LEN64] = get_zero64(); + + MB_FUNC_NAME(ifma_ec_sm2_mul_point_)(P, P, t); /* [t]P */ + MB_FUNC_NAME(ifma_ec_sm2_add_point_)(P, P, &sG); /* P = [s]G + [t]P */ + + __ALIGN64 U64 sign_r_restored[PSM2_LEN52]; + + /* extract affine P.x */ + MB_FUNC_NAME(get_sm2_ec_affine_coords_)(/* x = */ sign_r_restored, /* y = */ NULL, P); + + /* form Montgomery domain | x = x mod n */ + MB_FUNC_NAME(ifma_frommont52_psm2_)(sign_r_restored, sign_r_restored); + MB_FUNC_NAME(ifma_fastred52_pnsm2_)(sign_r_restored, sign_r_restored); + + /* r = (e + x1) mod n */ + MB_FUNC_NAME(ifma_tomont52_nsm2_)(sign_r_restored, sign_r_restored); + MB_FUNC_NAME(ifma_tomont52_nsm2_)(msg_digest, msg_digest); + + MB_FUNC_NAME(ifma_add52_nsm2_)(sign_r_restored, sign_r_restored, msg_digest); + + MB_FUNC_NAME(ifma_frommont52_nsm2_)(sign_r_restored, sign_r_restored); + + /* check equal */ + signature_err_mask |= ~(MB_FUNC_NAME(cmp_eq_FESM2_)(sign_r_restored, sign_r)); + + status |= MBX_SET_STS_BY_MASK(status, signature_err_mask, MBX_STATUS_SIGNATURE_ERR); + return status; +} + +/* +// Computes SM2 ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_user_id[] array of pointers to the users ID +// user_id_len[] array of users ID length +// pa_msg[] array of pointers to the messages are being signed +// msg_len[] array of messages length +// pa_eph_skey[] array of pointers to the signer's ephemeral private keys +// pa_reg_skey[] array of pointers to the signer's regular private keys +// pa_pubx[] array of pointers to the party's public keys X-coordinates +// pa_puby[] array of pointers to the party's public keys Y-coordinates +// pa_pubz[] array of pointers to the party's public keys Z-coordinates +// pBuffer pointer to the scratch buffer +*/ +DLL_PUBLIC +mbx_status mbx_sm2_ecdsa_sign_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_user_id[8], + const int user_id_len[8], + const int8u* const pa_msg[8], + const int msg_len[8], + const int64u* const pa_eph_skey[8], + const int64u* const pa_reg_skey[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int use_jproj_coords = NULL!=pa_pubz; + + /* test input pointers */ + if(NULL==pa_sign_r || NULL==pa_sign_s || NULL==pa_user_id || NULL==user_id_len || NULL==pa_msg || + NULL==msg_len || NULL==pa_eph_skey || NULL==pa_reg_skey || NULL==pa_pubx || NULL==pa_puby) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + int user_id_len_checked[8]; + + /* check pointers and values */ + for(int buf_no=0; buf_no<8; buf_no++) { + const int8u* r = pa_sign_r[buf_no]; + const int8u* s = pa_sign_s[buf_no]; + const int8u* id = pa_user_id[buf_no]; + const int8u* msg = pa_msg[buf_no]; + const int64u* eph_key = pa_eph_skey[buf_no]; + const int64u* reg_key = pa_reg_skey[buf_no]; + const int64u* pubx = pa_pubx[buf_no]; + const int64u* puby = pa_puby[buf_no]; + const int64u* pubz = use_jproj_coords? pa_pubz[buf_no] : NULL; + + user_id_len_checked[buf_no] = user_id_len[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL==r || NULL==s || NULL==id || NULL==msg || NULL==eph_key || + NULL==reg_key || NULL==pubx || NULL==puby || (use_jproj_coords && NULL==pubz)) { + status |= MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + if (msg_len[buf_no] < 0) { + status |= MBX_SET_STS(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + } + if ((user_id_len[buf_no] > 0xFFFF) || (user_id_len[buf_no] < 0)) { + status |= MBX_SET_STS(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + user_id_len_checked[buf_no] = 0; + } + } + + if(!MBX_IS_ANY_OK_STS(status) ) + return status; + + /* load and check secret keys */ + + __ALIGN64 U64 reg_skey[PSM2_LEN52]; + __ALIGN64 U64 eph_skey[PSM2_LEN52]; + + ifma_BNU_to_mb8((int64u (*)[8])reg_skey, pa_reg_skey, PSM2_BITSIZE); + ifma_BNU_to_mb8((int64u (*)[8])eph_skey, pa_eph_skey, PSM2_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FESM2_)(reg_skey), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FESM2_)(eph_skey), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])reg_skey, sizeof(reg_skey)/sizeof(U64)); + MB_FUNC_NAME(zero_)((int64u (*)[8])eph_skey, sizeof(eph_skey)/sizeof(U64)); + return status; + } + + __ALIGN64 int64u rev_bytes_pubX[8][PSM2_LEN64]; + int64u* pa_rev_bytes_pubX[8] = {rev_bytes_pubX[0], rev_bytes_pubX[1], rev_bytes_pubX[2], rev_bytes_pubX[3], rev_bytes_pubX[4], rev_bytes_pubX[5], rev_bytes_pubX[6], rev_bytes_pubX[7]}; + + __ALIGN64 int64u rev_bytes_pubY[8][PSM2_LEN64]; + int64u* pa_rev_bytes_pubY[8] = {rev_bytes_pubY[0], rev_bytes_pubY[1], rev_bytes_pubY[2], rev_bytes_pubY[3], rev_bytes_pubY[4], rev_bytes_pubY[5], rev_bytes_pubY[6], rev_bytes_pubY[7]}; + + SM2_POINT P; + + ifma_BNU_to_mb8((int64u (*)[8])P.X, (const int64u* (*))pa_pubx, PSM2_BITSIZE); + ifma_BNU_to_mb8((int64u (*)[8])P.Y, (const int64u* (*))pa_puby, PSM2_BITSIZE); + + if(use_jproj_coords) { + ifma_BNU_to_mb8((int64u (*)[8])P.Z, (const int64u* (*))pa_pubz, PSM2_BITSIZE); + } + + status = sm2_ecdsa_process_pubkeys(&P, + pa_rev_bytes_pubX, + pa_rev_bytes_pubY, + use_jproj_coords, + status); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])reg_skey, sizeof(reg_skey)/sizeof(U64)); + MB_FUNC_NAME(zero_)((int64u (*)[8])eph_skey, sizeof(eph_skey)/sizeof(U64)); + return status; + } + + __ALIGN64 int8u msg_digest[8][PSM2_LEN8]; + int8u* pa_msg_digest[8] = {msg_digest[0], msg_digest[1], msg_digest[2], msg_digest[3], msg_digest[4], msg_digest[5], msg_digest[6], msg_digest[7]}; + + /* compute z digest */ + sm2_ecdsa_compute_z_digest(pa_msg_digest, + (const int8u **)pa_user_id, + (const int *)user_id_len_checked, + (const int8u **)pa_rev_bytes_pubX, + (const int8u **)pa_rev_bytes_pubY); + + /* compute msg digest */ + sm2_ecdsa_compute_msg_digest(pa_msg_digest, + (const int8u **)pa_msg_digest, + (const int8u **)pa_msg, + (const int *)msg_len); + + /* zero padded keys */ + U64 scalar_eph_skey[PSM2_LEN64+1]; + ifma_BNU_transpose_copy((int64u (*)[8])scalar_eph_skey, pa_eph_skey, PSM2_BITSIZE); + scalar_eph_skey[PSM2_LEN64] = get_zero64(); + + __ALIGN64 U64 sign_r[PSM2_LEN52]; + __ALIGN64 U64 sign_s[PSM2_LEN52]; + + __ALIGN64 U64 msg[PSM2_LEN52]; + ifma_HexStr8_to_mb8((int64u (*)[8])msg, (const int8u* const*)pa_msg_digest, PSM2_BITSIZE); + + /* compute and check signature components */ + status = sm2_ecdsa_sign_mb8(sign_r, + sign_s, + msg, + scalar_eph_skey, + eph_skey, + reg_skey, + status); + + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])scalar_eph_skey, sizeof(scalar_eph_skey)/sizeof(U64)); + MB_FUNC_NAME(zero_)((int64u (*)[8])reg_skey, sizeof(reg_skey)/sizeof(U64)); + MB_FUNC_NAME(zero_)((int64u (*)[8])eph_skey, sizeof(eph_skey)/sizeof(U64)); + + if(!MBX_IS_ANY_OK_STS(status) ) + return status; + + ifma_mb8_to_HexStr8(pa_sign_r, (const int64u(*)[8])sign_r, PSM2_BITSIZE); + ifma_mb8_to_HexStr8(pa_sign_s, (const int64u(*)[8])sign_s, PSM2_BITSIZE); + + return status; +} + +/* +// Verifies SM2 ECDSA signature +// pa_sign_r[] array of pointers to the computed r-components of the signatures +// pa_sign_s[] array of pointers to the computed s-components of the signatures +// pa_msg[] array of pointers to the messages that have been signed +// pa_user_id[] array of pointers to the users ID +// user_id_len[] array of users ID length +// pa_msg[] array of pointers to the messages are being signed +// msg_len[] array of messages length +// pa_pubx[] array of pointers to the signer's public keys X-coordinates +// pa_puby[] array of pointers to the signer's public keys Y-coordinates +// pa_pubz[] array of pointers to the signer's public keys Z-coordinates (or NULL, if affine coordinate requested) +// pBuffer pointer to the scratch buffer +*/ +DLL_PUBLIC +mbx_status mbx_sm2_ecdsa_verify_mb8(const int8u* const pa_sign_r[8], + const int8u* const pa_sign_s[8], + const int8u* const pa_user_id[8], + const int user_id_len[8], + const int8u* const pa_msg[8], + const int msg_len[8], + const int64u* const pa_pubx[8], + const int64u* const pa_puby[8], + const int64u* const pa_pubz[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int use_jproj_coords = NULL!=pa_pubz; + + /* test input pointers */ + if(NULL==pa_sign_r || NULL==pa_sign_s || NULL==pa_user_id || NULL==user_id_len || + NULL==pa_msg || NULL==msg_len || NULL==pa_pubx || NULL==pa_puby) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + int user_id_len_checked[8]; + + /* check pointers and values */ + for(int buf_no=0; buf_no<8; buf_no++) { + const int8u* r = pa_sign_r[buf_no]; + const int8u* s = pa_sign_s[buf_no]; + const int8u* id = pa_user_id[buf_no]; + const int8u* msg = pa_msg[buf_no]; + const int64u* pubx = pa_pubx[buf_no]; + const int64u* puby = pa_puby[buf_no]; + const int64u* pubz = use_jproj_coords? pa_pubz[buf_no] : NULL; + + user_id_len_checked[buf_no] = user_id_len[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL==r || NULL==s || NULL==id || NULL==msg || + NULL==pubx || NULL==puby || (use_jproj_coords && NULL==pubz)) { + status |= MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + if (msg_len[buf_no] < 0) { + status |= MBX_SET_STS(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + } + if ((user_id_len[buf_no] > 0xFFFF) || (user_id_len[buf_no] < 0)) { + status |= MBX_SET_STS(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + user_id_len_checked[buf_no] = 0; + } + } + + if(!MBX_IS_ANY_OK_STS(status) ) + return status; + + __ALIGN64 U64 sign_r[PSM2_LEN52]; + __ALIGN64 U64 sign_s[PSM2_LEN52]; + + ifma_HexStr8_to_mb8((int64u (*)[8])sign_r, (const int8u* const*)pa_sign_r, PSM2_BITSIZE); + ifma_HexStr8_to_mb8((int64u (*)[8])sign_s, (const int8u* const*)pa_sign_s, PSM2_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_nsm2_)(sign_r), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_nsm2_)(sign_s), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + __ALIGN64 int64u rev_bytes_pubX[8][PSM2_LEN64]; + int64u* pa_rev_bytes_pubX[8] = {rev_bytes_pubX[0], rev_bytes_pubX[1], rev_bytes_pubX[2], rev_bytes_pubX[3], rev_bytes_pubX[4], rev_bytes_pubX[5], rev_bytes_pubX[6], rev_bytes_pubX[7]}; + + __ALIGN64 int64u rev_bytes_pubY[8][PSM2_LEN64]; + int64u* pa_rev_bytes_pubY[8] = {rev_bytes_pubY[0], rev_bytes_pubY[1], rev_bytes_pubY[2], rev_bytes_pubY[3], rev_bytes_pubY[4], rev_bytes_pubY[5], rev_bytes_pubY[6], rev_bytes_pubY[7]}; + + SM2_POINT P; + + ifma_BNU_to_mb8((int64u (*)[8])P.X, (const int64u* (*))pa_pubx, PSM2_BITSIZE); + ifma_BNU_to_mb8((int64u (*)[8])P.Y, (const int64u* (*))pa_puby, PSM2_BITSIZE); + + if(use_jproj_coords) { + ifma_BNU_to_mb8((int64u (*)[8])P.Z, (const int64u* (*))pa_pubz, PSM2_BITSIZE); + } + + status = sm2_ecdsa_process_pubkeys(&P, + pa_rev_bytes_pubX, + pa_rev_bytes_pubY, + use_jproj_coords, + status); + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + __ALIGN64 int8u msg_digest[8][PSM2_LEN8]; + int8u* pa_msg_digest[8] = {msg_digest[0], msg_digest[1], msg_digest[2], msg_digest[3], msg_digest[4], msg_digest[5], msg_digest[6], msg_digest[7]}; + + /* compute z digest */ + sm2_ecdsa_compute_z_digest(pa_msg_digest, + (const int8u **)pa_user_id, + (const int *)user_id_len_checked, + (const int8u **)pa_rev_bytes_pubX, + (const int8u **)pa_rev_bytes_pubY); + + /* compute msg digest */ + sm2_ecdsa_compute_msg_digest(pa_msg_digest, + (const int8u **)pa_msg_digest, + (const int8u **)pa_msg, + (const int *)msg_len); + + __ALIGN64 U64 msg[PSM2_LEN52]; + ifma_HexStr8_to_mb8((int64u (*)[8])msg, (const int8u* const*)pa_msg_digest, PSM2_BITSIZE); + + status = sm2_ecdsa_verify_mb8(sign_r, sign_s, msg, &P, status); + + return status; +} + +/* +// OpenSSL's specific implementations +*/ +#ifndef BN_OPENSSL_DISABLE + +DLL_PUBLIC +mbx_status mbx_sm2_ecdsa_sign_ssl_mb8(int8u* pa_sign_r[8], + int8u* pa_sign_s[8], + const int8u* const pa_user_id[8], + const int user_id_len[8], + const int8u* const pa_msg[8], + const int msg_len[8], + const BIGNUM* const pa_eph_skey[8], + const BIGNUM* const pa_reg_skey[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int use_jproj_coords = NULL!=pa_pubz; + + /* test input pointers */ + if(NULL==pa_sign_r || NULL==pa_sign_s || NULL==pa_user_id || NULL==user_id_len || NULL==pa_msg || + NULL==msg_len || NULL==pa_eph_skey || NULL==pa_reg_skey || NULL==pa_pubx || NULL==pa_puby) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + int user_id_len_checked[8]; + + /* check pointers and values */ + for(int buf_no=0; buf_no<8; buf_no++) { + const int8u* r = pa_sign_r[buf_no]; + const int8u* s = pa_sign_s[buf_no]; + const int8u* id = pa_user_id[buf_no]; + const int8u* msg = pa_msg[buf_no]; + const BIGNUM* eph_key = pa_eph_skey[buf_no]; + const BIGNUM* reg_key = pa_reg_skey[buf_no]; + const BIGNUM* pubx = pa_pubx[buf_no]; + const BIGNUM* puby = pa_puby[buf_no]; + const BIGNUM* pubz = use_jproj_coords? pa_pubz[buf_no] : NULL; + + user_id_len_checked[buf_no] = user_id_len[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL==r || NULL==s || NULL==id || NULL==msg || NULL==eph_key || + NULL==reg_key || NULL==pubx || NULL==puby || (use_jproj_coords && NULL==pubz)) { + status |= MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + if (msg_len[buf_no] < 0) { + status |= MBX_SET_STS(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + } + if ((user_id_len[buf_no] > 0xFFFF) || (user_id_len[buf_no] < 0)) { + status |= MBX_SET_STS(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + user_id_len_checked[buf_no] = 0; + } + } + + if(!MBX_IS_ANY_OK_STS(status) ) + return status; + + /* load and check secret keys */ + + __ALIGN64 U64 reg_skey[PSM2_LEN52]; + __ALIGN64 U64 eph_skey[PSM2_LEN52]; + + ifma_BN_to_mb8((int64u (*)[8])reg_skey, pa_reg_skey, PSM2_BITSIZE); + ifma_BN_to_mb8((int64u (*)[8])eph_skey, pa_eph_skey, PSM2_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FESM2_)(reg_skey), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(is_zero_FESM2_)(eph_skey), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])reg_skey, sizeof(reg_skey)/sizeof(U64)); + MB_FUNC_NAME(zero_)((int64u (*)[8])eph_skey, sizeof(eph_skey)/sizeof(U64)); + return status; + } + + __ALIGN64 int64u rev_bytes_pubX[8][PSM2_LEN64]; + int64u* pa_rev_bytes_pubX[8] = {rev_bytes_pubX[0], rev_bytes_pubX[1], rev_bytes_pubX[2], rev_bytes_pubX[3], rev_bytes_pubX[4], rev_bytes_pubX[5], rev_bytes_pubX[6], rev_bytes_pubX[7]}; + + __ALIGN64 int64u rev_bytes_pubY[8][PSM2_LEN64]; + int64u* pa_rev_bytes_pubY[8] = {rev_bytes_pubY[0], rev_bytes_pubY[1], rev_bytes_pubY[2], rev_bytes_pubY[3], rev_bytes_pubY[4], rev_bytes_pubY[5], rev_bytes_pubY[6], rev_bytes_pubY[7]}; + + SM2_POINT P; + + ifma_BN_to_mb8((int64u (*)[8])P.X, pa_pubx, PSM2_BITSIZE); + ifma_BN_to_mb8((int64u (*)[8])P.Y, pa_puby, PSM2_BITSIZE); + + if(use_jproj_coords) { + ifma_BN_to_mb8((int64u (*)[8])P.Z, pa_pubz, PSM2_BITSIZE); + } + + status = sm2_ecdsa_process_pubkeys(&P, + pa_rev_bytes_pubX, + pa_rev_bytes_pubY, + use_jproj_coords, + status); + + if(!MBX_IS_ANY_OK_STS(status)) { + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])reg_skey, sizeof(reg_skey)/sizeof(U64)); + MB_FUNC_NAME(zero_)((int64u (*)[8])eph_skey, sizeof(eph_skey)/sizeof(U64)); + return status; + } + + __ALIGN64 int8u msg_digest[8][PSM2_LEN8]; + int8u* pa_msg_digest[8] = {msg_digest[0], msg_digest[1], msg_digest[2], msg_digest[3], msg_digest[4], msg_digest[5], msg_digest[6], msg_digest[7]}; + + /* compute z digest */ + sm2_ecdsa_compute_z_digest(pa_msg_digest, + (const int8u **)pa_user_id, + (const int *)user_id_len_checked, + (const int8u **)pa_rev_bytes_pubX, + (const int8u **)pa_rev_bytes_pubY); + + /* compute msg digest */ + sm2_ecdsa_compute_msg_digest(pa_msg_digest, + (const int8u **)pa_msg_digest, + (const int8u **)pa_msg, + (const int *)msg_len); + + /* zero padded keys */ + U64 scalar_eph_skey[PSM2_LEN64+1]; + ifma_BN_transpose_copy((int64u (*)[8])scalar_eph_skey, pa_eph_skey, PSM2_BITSIZE); + scalar_eph_skey[PSM2_LEN64] = get_zero64(); + + __ALIGN64 U64 sign_r[PSM2_LEN52]; + __ALIGN64 U64 sign_s[PSM2_LEN52]; + + __ALIGN64 U64 msg[PSM2_LEN52]; + ifma_HexStr8_to_mb8((int64u (*)[8])msg, (const int8u* const*)pa_msg_digest, PSM2_BITSIZE); + + /* compute and check signature components */ + status = sm2_ecdsa_sign_mb8(sign_r, + sign_s, + msg, + scalar_eph_skey, + eph_skey, + reg_skey, + status); + + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])scalar_eph_skey, sizeof(scalar_eph_skey)/sizeof(U64)); + MB_FUNC_NAME(zero_)((int64u (*)[8])reg_skey, sizeof(reg_skey)/sizeof(U64)); + MB_FUNC_NAME(zero_)((int64u (*)[8])eph_skey, sizeof(eph_skey)/sizeof(U64)); + + if(!MBX_IS_ANY_OK_STS(status) ) + return status; + + ifma_mb8_to_HexStr8(pa_sign_r, (const int64u(*)[8])sign_r, PSM2_BITSIZE); + ifma_mb8_to_HexStr8(pa_sign_s, (const int64u(*)[8])sign_s, PSM2_BITSIZE); + + return status; +} + +DLL_PUBLIC +mbx_status mbx_sm2_ecdsa_verify_ssl_mb8(const ECDSA_SIG* const pa_sig[8], + const int8u* const pa_user_id[8], + const int user_id_len[8], + const int8u* const pa_msg[8], + const int msg_len[8], + const BIGNUM* const pa_pubx[8], + const BIGNUM* const pa_puby[8], + const BIGNUM* const pa_pubz[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int use_jproj_coords = NULL!=pa_pubz; + + /* test input pointers */ + if(NULL==pa_sig || NULL==pa_user_id || NULL==user_id_len || NULL==pa_msg || + NULL==msg_len || NULL==pa_pubx || NULL==pa_puby) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + int user_id_len_checked[8]; + + /* check pointers and values */ + for(int buf_no=0; buf_no<8; buf_no++) { + const ECDSA_SIG* sig = pa_sig[buf_no]; + const int8u* id = pa_user_id[buf_no]; + const int8u* msg = pa_msg[buf_no]; + const BIGNUM* pubx = pa_pubx[buf_no]; + const BIGNUM* puby = pa_puby[buf_no]; + const BIGNUM* pubz = use_jproj_coords? pa_pubz[buf_no] : NULL; + + user_id_len_checked[buf_no] = user_id_len[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL==sig || NULL==id || NULL==msg || + NULL==pubx || NULL==puby || (use_jproj_coords && NULL==pubz)) { + status |= MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + if (msg_len[buf_no] < 0) { + status |= MBX_SET_STS(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + } + if ((user_id_len[buf_no] > 0xFFFF) || (user_id_len[buf_no] < 0)) { + status |= MBX_SET_STS(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + user_id_len_checked[buf_no] = 0; + } + } + + if(!MBX_IS_ANY_OK_STS(status) ) + return status; + + BIGNUM* pa_sign_r[8] = { 0,0,0,0,0,0,0,0 }; + BIGNUM* pa_sign_s[8] = { 0,0,0,0,0,0,0,0 }; + + for (int buf_no = 0; buf_no < 8; buf_no++) + { + if(pa_sig[buf_no] != NULL) + { + ECDSA_SIG_get0(pa_sig[buf_no], (const BIGNUM(**))pa_sign_r + buf_no, + (const BIGNUM(**))pa_sign_s + buf_no); + } + } + + __ALIGN64 U64 sign_r[PSM2_LEN52]; + __ALIGN64 U64 sign_s[PSM2_LEN52]; + + ifma_BN_to_mb8((int64u (*)[8])sign_r, (const BIGNUM(**))pa_sign_r, PSM2_BITSIZE); + ifma_BN_to_mb8((int64u (*)[8])sign_s, (const BIGNUM(**))pa_sign_s, PSM2_BITSIZE); + + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_nsm2_)(sign_r), MBX_STATUS_MISMATCH_PARAM_ERR); + status |= MBX_SET_STS_BY_MASK(status, MB_FUNC_NAME(ifma_check_range_nsm2_)(sign_s), MBX_STATUS_MISMATCH_PARAM_ERR); + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + __ALIGN64 int64u rev_bytes_pubX[8][PSM2_LEN64]; + int64u* pa_rev_bytes_pubX[8] = {rev_bytes_pubX[0], rev_bytes_pubX[1], rev_bytes_pubX[2], rev_bytes_pubX[3], rev_bytes_pubX[4], rev_bytes_pubX[5], rev_bytes_pubX[6], rev_bytes_pubX[7]}; + + __ALIGN64 int64u rev_bytes_pubY[8][PSM2_LEN64]; + int64u* pa_rev_bytes_pubY[8] = {rev_bytes_pubY[0], rev_bytes_pubY[1], rev_bytes_pubY[2], rev_bytes_pubY[3], rev_bytes_pubY[4], rev_bytes_pubY[5], rev_bytes_pubY[6], rev_bytes_pubY[7]}; + + SM2_POINT P; + + ifma_BN_to_mb8((int64u (*)[8])P.X, pa_pubx, PSM2_BITSIZE); + ifma_BN_to_mb8((int64u (*)[8])P.Y, pa_puby, PSM2_BITSIZE); + + if(use_jproj_coords) { + ifma_BN_to_mb8((int64u (*)[8])P.Z, pa_pubz, PSM2_BITSIZE); + } + + status = sm2_ecdsa_process_pubkeys(&P, + pa_rev_bytes_pubX, + pa_rev_bytes_pubY, + use_jproj_coords, + status); + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + __ALIGN64 int8u msg_digest[8][PSM2_LEN8]; + int8u* pa_msg_digest[8] = {msg_digest[0], msg_digest[1], msg_digest[2], msg_digest[3], msg_digest[4], msg_digest[5], msg_digest[6], msg_digest[7]}; + + /* compute z digest */ + sm2_ecdsa_compute_z_digest(pa_msg_digest, + (const int8u **)pa_user_id, + (const int *)user_id_len_checked, + (const int8u **)pa_rev_bytes_pubX, + (const int8u **)pa_rev_bytes_pubY); + + /* compute msg digest */ + sm2_ecdsa_compute_msg_digest(pa_msg_digest, + (const int8u **)pa_msg_digest, + (const int8u **)pa_msg, + (const int *)msg_len); + + __ALIGN64 U64 msg[PSM2_LEN52]; + ifma_HexStr8_to_mb8((int64u (*)[8])msg, (const int8u* const*)pa_msg_digest, PSM2_BITSIZE); + + status = sm2_ecdsa_verify_mb8(sign_r, sign_s, msg, &P, status); + + return status; +} + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm2/ifma_ecpoint_sm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm2/ifma_ecpoint_sm2.c new file mode 100644 index 0000000..65dc228 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm2/ifma_ecpoint_sm2.c @@ -0,0 +1,695 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include +#include + +/* simplify naming */ +#define sqr MB_FUNC_NAME(ifma_ams52_psm2_) +#define mul MB_FUNC_NAME(ifma_amm52_psm2_) +#define add MB_FUNC_NAME(ifma_add52_psm2_) +#define sub MB_FUNC_NAME(ifma_sub52_psm2_) +#define mul2 MB_FUNC_NAME(ifma_double52_psm2_) +#define mul3 MB_FUNC_NAME(ifma_tripple52_psm2_) +#define div2 MB_FUNC_NAME(ifma_half52_psm2_) + +/* +// Presentation of point at infinity: +// - projective (X : Y : 0) +// - affine (0 : 0) +*/ + +/* +// R(X3:Y3:Z3) = [2]P(X1:Y1:Z1) +// +// formulas: +// A = 4*X1*Y1^2 +// B = 3*(X1^2-Z1^4) +// X3= B^2 -2*A +// Y3= B*(A-X3) -8*Y1^4 +// Z3= 2*Y1*Z1 +// +// cost: 4S+4M+9A +// +*/ +void MB_FUNC_NAME(ifma_ec_sm2_dbl_point_)(SM2_POINT* r, const SM2_POINT* p) +{ + __ALIGN64 U64 T[PSM2_LEN52]; + __ALIGN64 U64 U[PSM2_LEN52]; + __ALIGN64 U64 V[PSM2_LEN52]; + __ALIGN64 U64 A[PSM2_LEN52]; + __ALIGN64 U64 B[PSM2_LEN52]; + + const U64* X1 = p->X; /* input point */ + const U64* Y1 = p->Y; + const U64* Z1 = p->Z; + U64* X3 = r->X; /* output point */ + U64* Y3 = r->Y; + U64* Z3 = r->Z; + + mul2(T, Y1); /* T = 2*Y1 */ + + sqr(V, T); /* V = 4*Y1^2 */ /* sqr_dual */ + sqr(U, Z1); /* U = Z1^2 */ + + sub(B, X1, U); /* B = X1-Z1^2 */ + add(U, X1, U); /* U = X1+Z1^2 */ + + mul(A, V, X1); /* A = 4*X*Y1^2 */ /* mul_dual */ + mul(B, B, U); /* B = (X1^2-Z1^4) */ + + mul2(X3, A); /* X3 = 2*A */ + mul3(B, B); /* B = 3*(X1^2-Z1^4) */ + + sqr(U, B); /* U = B^2 */ /* sqr_dual */ + sqr(Y3, V); /* Y3= V^2 = 16*Y1^4 */ + + sub(X3, U, X3); /* X3=B^2 - 2*A */ + div2(Y3,Y3); /* Y3=Y3/2 = 8*Y1^4 */ + + sub(U, A, X3); /* U = A-X3 */ + + mul(Z3, T, Z1); /* Z3= 2*Y1*Z1 */ /* mul_dual */ + mul(U, U, B); /* U = B*(A-X3) */ + + sub(Y3, U, Y3); /* Y3 = B*(A-X3) -8*Y1^4 */ +} + + +/* +// R(X3:Y3:Z3) = P(X1:Y1:Z1) + Q(X2:Y2:Z2) +// +// formulas: +// A = X1*Z2^2 B = X2*Z1^2 C = Y1*Z2^3 D = Y2*Z1^3 +// E = B-A F = D-C +// X3= -E^3 -2*A*E^2 + F^2 +// Y3= -C*E^3 + F*(A*E^2 -X3) +// Z3= Z1*Z2*E +// +// cost: 4S+12M+7A +// +*/ +void MB_FUNC_NAME(ifma_ec_sm2_add_point_)(SM2_POINT* r, const SM2_POINT* p, const SM2_POINT* q) +{ + /* coordinates of p */ + const U64* X1 = p->X; + const U64* Y1 = p->Y; + const U64* Z1 = p->Z; + __mb_mask p_at_infinity = MB_FUNC_NAME(is_zero_point_cordinate_)(p->Z); + + /* coordinates of q */ + const U64* X2 = q->X; + const U64* Y2 = q->Y; + const U64* Z2 = q->Z; + __mb_mask q_at_infinity = MB_FUNC_NAME(is_zero_point_cordinate_)(q->Z); + + /* coordinates of temp point T(X3:Y3:Z3) */ + __ALIGN64 U64 X3[PSM2_LEN52]; + __ALIGN64 U64 Y3[PSM2_LEN52]; + __ALIGN64 U64 Z3[PSM2_LEN52]; + + /* temporary */ + __ALIGN64 U64 U1[PSM2_LEN52]; + __ALIGN64 U64 U2[PSM2_LEN52]; + __ALIGN64 U64 S1[PSM2_LEN52]; + __ALIGN64 U64 S2[PSM2_LEN52]; + __ALIGN64 U64 H[PSM2_LEN52]; + __ALIGN64 U64 R[PSM2_LEN52]; + + mul(S1, Y1, Z2); /* S1 = Y1*Z2 */ + sqr(U1, Z2); /* U1 = Z2^2 */ + + mul(S2, Y2, Z1); /* S2 = Y2*Z1 */ + sqr(U2, Z1); /* U2 = Z1^2 */ + + mul(S1, S1, U1); /* S1 = Y1*Z2^3 */ + mul(S2, S2, U2); /* S2 = Y2*Z1^3 */ + + mul(U1, X1, U1); /* U1 = X1*Z2^2 */ + mul(U2, X2, U2); /* U2 = X2*Z1^2 */ + + sub(R, S2, S1); /* R = S2-S1 */ + sub(H, U2, U1); /* H = U2-U1 */ + + + /* check if affine (p.x:p.y) == (q.x:q.y) and and do doubling if this happens */ + __mb_mask x_are_equal = MB_FUNC_NAME(is_zero_FESM2_)(H); + __mb_mask y_are_equal = MB_FUNC_NAME(is_zero_FESM2_)(R); + __mb_mask points_are_equal = (x_are_equal & y_are_equal & (~p_at_infinity) & (~q_at_infinity)); + + SM2_POINT P2; + MB_FUNC_NAME(set_point_to_infinity_)(&P2); + if (points_are_equal) { + MB_FUNC_NAME(ifma_ec_sm2_dbl_point_)(&P2, p); + } + + mul(Z3, Z1, Z2); /* Z3 = Z1*Z2 */ + sqr(U2, H); /* U2 = H^2 */ + mul(Z3, Z3, H); /* Z3 = (Z1*Z2)*H */ + sqr(S2, R); /* S2 = R^2 */ + mul(H, H, U2); /* H = H^3 */ + + mul(U1, U1, U2); /* U1 = U1*H^2 */ + sub(X3, S2, H); /* X3 = R^2 - H^3 */ + mul2(U2, U1); /* U2 = 2*U1*H^2 */ + mul(S1, S1, H); /* S1 = S1*H^3 */ + sub(X3, X3, U2); /* X3 = (R^2 - H^3) -2*U1*H^2 */ + + sub(Y3, U1, X3); /* Y3 = R*(U1*H^2 - X3) -S1*H^3 */ + mul(Y3, Y3, R); + sub(Y3, Y3, S1); + + /* T = p_at_infinity? q : T */ + MB_FUNC_NAME(mask_mov_FESM2_)(X3, X3, p_at_infinity, q->X); + MB_FUNC_NAME(mask_mov_FESM2_)(Y3, Y3, p_at_infinity, q->Y); + MB_FUNC_NAME(mask_mov_FESM2_)(Z3, Z3, p_at_infinity, q->Z); + /* T = q_at_infinity? p : T */ + MB_FUNC_NAME(mask_mov_FESM2_)(X3, X3, q_at_infinity, p->X); + MB_FUNC_NAME(mask_mov_FESM2_)(Y3, Y3, q_at_infinity, p->Y); + MB_FUNC_NAME(mask_mov_FESM2_)(Z3, Z3, q_at_infinity, p->Z); + + /* r = points_are_equal? P2 : T */ + MB_FUNC_NAME(mask_mov_FESM2_)(r->X, X3, points_are_equal, P2.X); + MB_FUNC_NAME(mask_mov_FESM2_)(r->Y, Y3, points_are_equal, P2.Y); + MB_FUNC_NAME(mask_mov_FESM2_)(r->Z, Z3, points_are_equal, P2.Z); +} + + +/* to Montgomery conversion constant +// r = 2^(PSM2_LEN52*DIGIT_SIZE) mod psm2 +*/ +__ALIGN64 static const int64u psm2_r_mb[PSM2_LEN52][sizeof(U64)/sizeof(int64u)] = { + { REP8_DECL(0x0000000000000010) }, + { REP8_DECL(0x0000ffffffff0000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000000000) }, + { REP8_DECL(0x0000000000100000) } +}; +const U64* MB_FUNC_NAME(ifma_ec_sm2_coord_one_)(void) +{ + return (U64*)psm2_r_mb; +} + + +/* +// R(X3:Y3:Z3) = P(X1:Y1:Z1) + Q(X2:Y2:Z2=1) +// +// formulas: +// A = X1*Z2^2 B = X2*Z1^2 C = Y1*Z2^3 D = Y2*Z1^3 +// E = B-A F = D-C +// X3= -E^3 -2*A*E^2 + F^2 +// Y3= -C*E^3 + F*(A*E^2 -X3) +// Z3= Z1*Z2*E +// +// if Z2=1, then +// A = X1 B = X2*Z1^2 C = Y1 D = Y2*Z1^3 +// E = B-X1 F = D-Y1 +// X3= -E^3 -2*X1*E^2 + F^2 +// Y3= -Y1*E^3 + F*(X1*E^2 -X3) +// Z3= Z1*E +// +// cost: 3S+8M+7A +*/ +void MB_FUNC_NAME(ifma_ec_sm2_add_point_affine_)(SM2_POINT* r, const SM2_POINT* p, const SM2_POINT_AFFINE* q) +{ + /* coordinates of p (projective) */ + const U64* X1 = p->X; + const U64* Y1 = p->Y; + const U64* Z1 = p->Z; + __mb_mask p_at_infinity = MB_FUNC_NAME(is_zero_point_cordinate_)(p->Z); + + /* coordinates of q (affine) */ + const U64* X2 = q->x; + const U64* Y2 = q->y; + __mb_mask q_at_infinity = MB_FUNC_NAME(is_zero_point_cordinate_)(q->x) + & MB_FUNC_NAME(is_zero_point_cordinate_)(q->y); + + /* coordinates of temp point T(X3:Y3:Z3) */ + __ALIGN64 U64 X3[PSM2_LEN52]; + __ALIGN64 U64 Y3[PSM2_LEN52]; + __ALIGN64 U64 Z3[PSM2_LEN52]; + + __ALIGN64 U64 U2[PSM2_LEN52]; + __ALIGN64 U64 S2[PSM2_LEN52]; + __ALIGN64 U64 H[PSM2_LEN52]; + __ALIGN64 U64 R[PSM2_LEN52]; + + sqr(R, Z1); // R = Z1^2 + mul(S2, Y2, Z1); // S2 = Y2*Z1 + mul(U2, X2, R); // U2 = X2*Z1^2 + mul(S2, S2, R); // S2 = Y2*Z1^3 + + sub(H, U2, X1); // H = U2-X1 + sub(R, S2, Y1); // R = S2-Y1 + + mul(Z3, H, Z1); // Z3 = H*Z1 + + sqr(U2, H); // U2 = H^2 + sqr(S2, R); // S2 = R^2 + mul(H, H, U2); // H = H^3 + + mul(U2, U2, X1); // U2 = X1*H^2 + + mul(Y3, H, Y1); // T = Y1*H^3 + + mul2(X3, U2); // X3 = 2*X1*H^2 + sub(X3, S2, X3); // X3 = R^2 - 2*X1*H^2 + sub(X3, X3, H); // X3 = R^2 - 2*X1*H^2 -H^3 + + sub(U2, U2, X3); // U2 = X1*H^2 - X3 + mul(U2, U2, R); // U2 = R*(X1*H^2 - X3) + sub(Y3, U2, Y3); // Y3 = -Y1*H^3 + R*(X1*H^2 - X3) + + /* T = p_at_infinity? q : T */ + MB_FUNC_NAME(mask_mov_FESM2_)(X3, X3, p_at_infinity, q->x); + MB_FUNC_NAME(mask_mov_FESM2_)(Y3, Y3, p_at_infinity, q->y); + MB_FUNC_NAME(mask_mov_FESM2_)(Z3, Z3, p_at_infinity, (U64*)psm2_r_mb); + /* T = q_at_infinity? p : T */ + MB_FUNC_NAME(mask_mov_FESM2_)(X3, X3, q_at_infinity, p->X); + MB_FUNC_NAME(mask_mov_FESM2_)(Y3, Y3, q_at_infinity, p->Y); + MB_FUNC_NAME(mask_mov_FESM2_)(Z3, Z3, q_at_infinity, p->Z); + + /* r = T */ + MB_FUNC_NAME(mov_FESM2_)(r->X, X3); + MB_FUNC_NAME(mov_FESM2_)(r->Y, Y3); + MB_FUNC_NAME(mov_FESM2_)(r->Z, Z3); +} + +void MB_FUNC_NAME(get_sm2_ec_affine_coords_)(U64 x[], U64 y[], const SM2_POINT* P) +{ + __ALIGN64 U64 invZ1[PSM2_LEN52]; + __ALIGN64 U64 invZn[PSM2_LEN52]; + + /* 1/Z and 1/Z^2 */ + MB_FUNC_NAME(ifma_aminv52_psm2_)(invZ1, P->Z); + MB_FUNC_NAME(ifma_ams52_psm2_)(invZn, invZ1); + + /* if affine P.x requested */ + if(x) + MB_FUNC_NAME(ifma_amm52_psm2_)(x, P->X, invZn); + + /* if affine P.y requested */ + if(y) { + MB_FUNC_NAME(ifma_amm52_psm2_)(invZn, invZn, invZ1); + MB_FUNC_NAME(ifma_amm52_psm2_)(y, P->Y, invZn); + } +} + +////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////// + +static __NOINLINE void clear_secret_context(U64* wval, U64* dval, __mb_mask* dsign) +{ + *wval = get_zero64(); + *dval = get_zero64(); + *dsign = 0; + return; +} + +#define WIN_SIZE (4) + +/* + s = (Ipp8u)(~((wvalue >> ws) - 1)); //sign + d = (1 << (ws+1)) - wvalue - 1; // digit, win size "ws" + d = (d & s) | (wvaluen & ~s); + d = (d >> 1) + (d & 1); + *sign = s & 1; + *digit = (Ipp8u)d; +*/ +__INLINE void MB_FUNC_NAME(booth_recode_)(__mb_mask* sign, U64* dvalue, U64 wvalue) +{ + U64 one = set1(1); + U64 zero = get_zero64(); + U64 t = srli64(wvalue, WIN_SIZE); + __mb_mask s = cmp64_mask(t, zero, _MM_CMPINT_NE); + U64 d = sub64( sub64(set1(1<<(WIN_SIZE+1)), wvalue), one); + d = mask_mov64(wvalue, s, d); + U64 odd = and64(d, one); + d = add64( srli64(d, 1), odd); + + *sign = s; + *dvalue = d; +} + +/* extract point */ +static void MB_FUNC_NAME(extract_point_)(SM2_POINT* r, const SM2_POINT tbl[], U64 idx) +{ + /* decrenent index (the table noes not contain [0]*P */ + U64 idx_target = sub64(idx, set1(1)); + + /* assume the point at infinity is what need */ + SM2_POINT R; + MB_FUNC_NAME(set_point_to_infinity_)(&R); + + /* find out what we actually need or just keep original infinity */ + int32u n; + for(n=0; n<(1<<(WIN_SIZE-1)); n++) { + U64 idx_curr = set1(n); + __mb_mask k = cmp64_mask(idx_curr, idx_target, _MM_CMPINT_EQ); + + /* R = k? tbl[] : R */ + MB_FUNC_NAME(secure_mask_mov_FESM2_)(R.X, R.X, k, tbl[n].X); + MB_FUNC_NAME(secure_mask_mov_FESM2_)(R.Y, R.Y, k, tbl[n].Y); + MB_FUNC_NAME(secure_mask_mov_FESM2_)(R.Z, R.Z, k, tbl[n].Z); + } + MB_FUNC_NAME(mov_FESM2_)(r->X, R.X); + MB_FUNC_NAME(mov_FESM2_)(r->Y, R.Y); + MB_FUNC_NAME(mov_FESM2_)(r->Z, R.Z); +} + +void MB_FUNC_NAME(ifma_ec_sm2_mul_point_)(SM2_POINT* r, const SM2_POINT* p, const U64 scalar[]) +{ + /* pre-computed table */ + __ALIGN64 SM2_POINT tbl[1<<(WIN_SIZE-1)]; + + /* + // compute tbl[] = [n]P, n=1,..,2^(WIN_SIZE-1): + // + // tbl[2*n] = tbl[2*n-1]+p + // tbl[2*n+1] = [2]*tbl[n] + */ + /* tbl[0] = p */ + MB_FUNC_NAME(mov_FESM2_)(tbl[0].X, p->X); + MB_FUNC_NAME(mov_FESM2_)(tbl[0].Y, p->Y); + MB_FUNC_NAME(mov_FESM2_)(tbl[0].Z, p->Z); + /* tbl[1] = [2]*p */ + MB_FUNC_NAME(ifma_ec_sm2_dbl_point_)(&tbl[1], p); + + int n; + for(n=1; n < (1<<(WIN_SIZE-1))/2; n++) { + MB_FUNC_NAME(ifma_ec_sm2_add_point_)(&tbl[2*n], &tbl[2*n-1], p); + MB_FUNC_NAME(ifma_ec_sm2_dbl_point_)(&tbl[2*n+1], &tbl[n]); + } + + SM2_POINT R; + SM2_POINT T; + U64 Ty[PSM2_LEN52]; + + /* + // point (LR) multiplication + */ + U64 idx_mask = set1( (1<<(WIN_SIZE+1))-1 ); + int bit = PSM2_BITSIZE-(PSM2_BITSIZE % WIN_SIZE); + int chunk_no = (bit-1)/64; + int chunk_shift = (bit-1)%64; + + /* first window */ + U64 wvalue = loadu64(&scalar[chunk_no]); + wvalue = and64( srli64(wvalue, chunk_shift), idx_mask); + + U64 dvalue; + __mb_mask dsign; + MB_FUNC_NAME(booth_recode_)(&dsign, &dvalue, wvalue); + MB_FUNC_NAME(extract_point_)(&R, tbl, dvalue); + + for(bit-=WIN_SIZE; bit>=WIN_SIZE; bit-=WIN_SIZE) { + /* doubling */ + MB_FUNC_NAME(ifma_ec_sm2_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_sm2_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_sm2_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_sm2_dbl_point_)(&R, &R); + #if (WIN_SIZE==5) + MB_FUNC_NAME(ifma_ec_sm2_dbl_point_)(&R, &R); + #endif + + /* extract precomputed []P */ + chunk_no = (bit-1)/64; + chunk_shift = (bit-1)%64; + + wvalue = loadu64(&scalar[chunk_no]); + #if (_MSC_VER <= 1916) /* VS 2017 not supported _mm512_shrdv_epi64 */ + { + __m512i t_lo_ = _mm512_srlv_epi64(wvalue, set64(chunk_shift)); + __m512i t_hi_ = _mm512_sllv_epi64(loadu64(&scalar[chunk_no+1]), set64(64-chunk_shift)); + wvalue = or64(t_lo_, t_hi_); + } + #else + wvalue = _mm512_shrdv_epi64(wvalue, loadu64(&scalar[chunk_no+1]), set1((int32u)chunk_shift)); + #endif + wvalue = and64(wvalue, idx_mask); + + MB_FUNC_NAME(booth_recode_)(&dsign, &dvalue, wvalue); + MB_FUNC_NAME(extract_point_)(&T, tbl, dvalue); + + /* T = dsign? -T : T */ + MB_FUNC_NAME(ifma_neg52_psm2_)(Ty, T.Y); + MB_FUNC_NAME(secure_mask_mov_FESM2_)(T.Y, T.Y, dsign, Ty); + + /* acumulate T */ + MB_FUNC_NAME(ifma_ec_sm2_add_point_)(&R, &R, &T); + } + + /* last window */ + MB_FUNC_NAME(ifma_ec_sm2_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_sm2_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_sm2_dbl_point_)(&R, &R); + MB_FUNC_NAME(ifma_ec_sm2_dbl_point_)(&R, &R); + #if (WIN_SIZE==5) + MB_FUNC_NAME(ifma_ec_sm2_dbl_point_)(&R, &R); + #endif + + wvalue = loadu64(&scalar[0]); + wvalue = and64( slli64(wvalue, 1), idx_mask); + MB_FUNC_NAME(booth_recode_)(&dsign, &dvalue, wvalue); + MB_FUNC_NAME(extract_point_)(&T, tbl, dvalue); + + MB_FUNC_NAME(ifma_neg52_psm2_)(Ty, T.Y); + MB_FUNC_NAME(secure_mask_mov_FESM2_)(T.Y, T.Y, dsign, Ty); + + MB_FUNC_NAME(ifma_ec_sm2_add_point_)(&R, &R, &T); + + /* r = R */ + MB_FUNC_NAME(mov_FESM2_)(r->X, R.X); + MB_FUNC_NAME(mov_FESM2_)(r->Y, R.Y); + MB_FUNC_NAME(mov_FESM2_)(r->Z, R.Z); + + /* clear r (to fix potential secutity flaw in case of ecdh */ + MB_FUNC_NAME(zero_)((int64u (*)[8])&R, sizeof(R)/sizeof(U64)); + + /* clear stubs of secret scalar */ + clear_secret_context(&wvalue, &dvalue, &dsign); +} +#undef WIN_SIZE + + +////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////// + +#include + +#define BP_WIN_SIZE MUL_BASEPOINT_WIN_SIZE /* defined in the header above */ + +__INLINE void MB_FUNC_NAME(booth_recode_bp_)(__mb_mask* sign, U64* dvalue, U64 wvalue) +{ + U64 one = set1(1); + U64 zero = get_zero64(); + U64 t = srli64(wvalue, BP_WIN_SIZE); + __mb_mask s = cmp64_mask(t, zero, _MM_CMPINT_NE); + U64 d = sub64( sub64(set1(1<<(BP_WIN_SIZE+1)), wvalue), one); + d = mask_mov64(wvalue, s, d); + U64 odd = and64(d, one); + d = add64( srli64(d, 1), odd); + + *sign = s; + *dvalue = d; +} + +/* extract affine affine point */ +__INLINE void MB_FUNC_NAME(extract_point_affine_)(SM2_POINT_AFFINE* r, const SINGLE_SM2_POINT_AFFINE* tbl, U64 idx) +{ + /* decrement index (the table noes not contain [0]*P */ + U64 targIdx = sub64(idx, set1(1)); + + U64 ax0, ax1, ax2, ax3, ax4, ay0, ay1, ay2, ay3, ay4; + + /* assume the point at infinity is what need */ + ax0 = ax1 = ax2 = ax3 = ax4 = ay0 = ay1 = ay2 = ay3 = ay4 = get_zero64(); + + /* find out what we actually need or just keep original infinity */ + int n; + U64 currIdx = get_zero64(); + for(n=0; n<(1<<(BP_WIN_SIZE-1)); n++, tbl++, currIdx = add64(currIdx, set1(1))) { + __mb_mask k = cmp64_mask(currIdx, targIdx, _MM_CMPINT_EQ); + + /* R = k? set1( tbl[] ) : R */ + ax0 = mask_add64(ax0, k, ax0, set1( tbl->x[0] )); + ax1 = mask_add64(ax1, k, ax1, set1( tbl->x[1] )); + ax2 = mask_add64(ax2, k, ax2, set1( tbl->x[2] )); + ax3 = mask_add64(ax3, k, ax3, set1( tbl->x[3] )); + ax4 = mask_add64(ax4, k, ax4, set1( tbl->x[4] )); + + ay0 = mask_add64(ay0, k, ay0, set1( tbl->y[0] )); + ay1 = mask_add64(ay1, k, ay1, set1( tbl->y[1] )); + ay2 = mask_add64(ay2, k, ay2, set1( tbl->y[2] )); + ay3 = mask_add64(ay3, k, ay3, set1( tbl->y[3] )); + ay4 = mask_add64(ay4, k, ay4, set1( tbl->y[4] )); + } + + r->x[0] = ax0; + r->x[1] = ax1; + r->x[2] = ax2; + r->x[3] = ax3; + r->x[4] = ax4; + + r->y[0] = ay0; + r->y[1] = ay1; + r->y[2] = ay2; + r->y[3] = ay3; + r->y[4] = ay4; +} + +void MB_FUNC_NAME(ifma_ec_sm2_mul_pointbase_)(SM2_POINT* r, const U64 scalar[]) +{ + /* pre-computed table of base powers */ + SINGLE_SM2_POINT_AFFINE* tbl = &ifma_ec_sm2_bp_precomp[0][0]; + + SM2_POINT R; + SM2_POINT_AFFINE A; + U64 Ty[PSM2_LEN52]; + + /* R = O */ + MB_FUNC_NAME(set_point_to_infinity_)(&R); + + + /* + // base point (RL) multiplication + */ + U64 wvalue, dvalue; + __mb_mask dsign; + + U64 idx_mask = set1( (1<<(BP_WIN_SIZE+1))-1 ); + int bit = 0; + + /* first window - window[0] */ + wvalue = loadu64(&scalar[0]); + wvalue = and64( slli64(wvalue, 1), idx_mask); + MB_FUNC_NAME(booth_recode_bp_)(&dsign, &dvalue, wvalue); + MB_FUNC_NAME(extract_point_affine_)(&A, tbl, dvalue); + tbl+=BP_N_ENTRY; + + /* A = dsign? -A : A */ + MB_FUNC_NAME(ifma_neg52_psm2_)(Ty, A.y); + MB_FUNC_NAME(secure_mask_mov_FESM2_)(A.y, A.y, dsign, Ty); + + /* R += A */ + MB_FUNC_NAME(ifma_ec_sm2_add_point_affine_)(&R, &R, &A); + + int chunk_no; + int chunk_shift; + for(bit+=BP_WIN_SIZE; bit<=PSM2_BITSIZE; bit+=BP_WIN_SIZE) { + chunk_no = (bit-1)/64; + chunk_shift = (bit-1)%64; + + wvalue = loadu64(&scalar[chunk_no]); + #if (_MSC_VER <= 1916) /* VS 2017 not supported _mm512_shrdv_epi64 */ + { + __m512i t_lo_ = _mm512_srlv_epi64(wvalue, set64(chunk_shift)); + __m512i t_hi_ = _mm512_sllv_epi64(loadu64(&scalar[chunk_no+1]), set64(64-chunk_shift)); + wvalue = or64(t_lo_, t_hi_); + } + #else + wvalue = _mm512_shrdv_epi64(wvalue, loadu64(&scalar[chunk_no+1]), set1((int32u)chunk_shift)); + #endif + wvalue = and64(wvalue, idx_mask); + + MB_FUNC_NAME(booth_recode_bp_)(&dsign, &dvalue, wvalue); + MB_FUNC_NAME(extract_point_affine_)(&A, tbl, dvalue); + tbl+=BP_N_ENTRY; + + /* A = dsign? -A : A */ + MB_FUNC_NAME(ifma_neg52_psm2_)(Ty, A.y); + MB_FUNC_NAME(secure_mask_mov_FESM2_)(A.y, A.y, dsign, Ty); + + /* R += A */ + MB_FUNC_NAME(ifma_ec_sm2_add_point_affine_)(&R, &R, &A); + } + + /* r = R */ + MB_FUNC_NAME(mov_FESM2_)(r->X, R.X); + MB_FUNC_NAME(mov_FESM2_)(r->Y, R.Y); + MB_FUNC_NAME(mov_FESM2_)(r->Z, R.Z); + + /* clear stubs of secret scalar */ + clear_secret_context(&wvalue, &dvalue, &dsign); +} +#undef BP_WIN_SIZE + +/* SM2 parameters: mont(a), mont(b) */ +__ALIGN64 static const int64u mont_a_psm2_mb[PSM2_LEN52][8] = { + { REP8_DECL(0x000fffffffffffcf) }, + { REP8_DECL(0x000cf00000030fff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x000fffffffffffff) }, + { REP8_DECL(0x0000ffffffceffff) } +}; +__ALIGN64 static const int64u mont_b_psm2_mb[PSM2_LEN52][8] = { + { REP8_DECL(0x00030632bc0dd422) }, + { REP8_DECL(0x000b09b537ab70d2) }, + { REP8_DECL(0x000a51c3c71cf379) }, + { REP8_DECL(0x0002c8527981505e) }, + { REP8_DECL(0x000040fe188da20e) } +}; + +/* +// We have a curve defined by a Weierstrass equation: y^2 = x^3 + a*x + b. +// +// The points are considered in Jacobian projective coordinates +// where (X, Y, Z) represents (x, y) = (X/Z^2, Y/Z^3). +// Substituting this and multiplying by Z^6 transforms the above equation into +// Y^2 = X^3 + a*X*Z^4 + b*Z^6 +// To test this, we add up the right-hand side in 'rh'. +*/ +__mb_mask MB_FUNC_NAME(ifma_is_on_curve_psm2_)(const SM2_POINT* p, int use_jproj_coords) +{ + U64 rh[PSM2_LEN52]; + U64 Z4[PSM2_LEN52], Z6[PSM2_LEN52], tmp[PSM2_LEN52]; + + /* rh := X^2 */ + MB_FUNC_NAME(ifma_ams52_psm2_)(rh, p->X); + + /* if Z!=1, then rh = X^3 + a*X*Z^4 + b*Z^6 = X*(X^2 + a*X*Z^4) + b*Z^6 */ + if(use_jproj_coords) { + MB_FUNC_NAME(ifma_ams52_psm2_)(tmp, p->Z); /* tmp = Z^2 */ + MB_FUNC_NAME(ifma_ams52_psm2_)(Z4, tmp); /* Z4 = Z^4 */ + MB_FUNC_NAME(ifma_amm52_psm2_)(Z6, Z4, tmp); /* Z6 = Z^6 */ + + MB_FUNC_NAME(ifma_add52_psm2_)(tmp, Z4, Z4); /* tmp = 2*Z^4 */ + MB_FUNC_NAME(ifma_add52_psm2_)(tmp, tmp, Z4); /* tmp = 2*Z^4 */ + MB_FUNC_NAME(ifma_sub52_psm2_)(rh, rh, tmp); /* rh = X^2 + a*Z^4 */ + MB_FUNC_NAME(ifma_amm52_psm2_)(rh, rh, p->X); /* rh = (X^2 + a*Z^4)*X */ + + MB_FUNC_NAME(ifma_amm52_psm2_)(tmp, Z6, (U64*)mont_b_psm2_mb); + MB_FUNC_NAME(ifma_add52_psm2_)(rh, rh, tmp); /* rh = (X^2 + a*Z^4)*X + b*Z^6 */ + } + /* if Z==1, then rh = X^3 + a*X + b = X*(X^2 +a) b */ + else { + MB_FUNC_NAME(ifma_add52_psm2_)(rh, rh, (U64*)mont_a_psm2_mb); /* rh = X^2+a */ + MB_FUNC_NAME(ifma_amm52_psm2_)(rh, rh, p->X); /* rh = (X^2+a)*X */ + MB_FUNC_NAME(ifma_add52_psm2_)(rh, rh, (U64*)mont_b_psm2_mb); /* rh = (X^2+a)*X + b */ + } + MB_FUNC_NAME(ifma_frommont52_psm2_)(rh, rh); + + /* rl = tmp = Y^2 */ + MB_FUNC_NAME(ifma_ams52_psm2_)(tmp, p->Y); + MB_FUNC_NAME(ifma_frommont52_psm2_)(tmp, tmp); + + /* mask = rl==rh */ + __mb_mask is_on_curve_mask = MB_FUNC_NAME(cmp_eq_FESM2_)(tmp, rh); + + return is_on_curve_mask; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm2/ifma_ecpubkey_sm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm2/ifma_ecpubkey_sm2.c new file mode 100644 index 0000000..36f73c1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm2/ifma_ecpubkey_sm2.c @@ -0,0 +1,214 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +#include +#include +#include +#include + +#ifndef BN_OPENSSL_DISABLE +#include +#endif + +#ifndef BN_OPENSSL_DISABLE +/* +// Computes public key +// pa_pubx[] array of pointers to the public keys X-coordinates +// pa_puby[] array of pointers to the public keys Y-coordinates +// pa_pubz[] array of pointers to the public keys Z-coordinates (or NULL, if affine coordinate requested) +// pa_skey[] array of pointers to the private keys +// pBuffer pointer to the scratch buffer +// +// Note: +// output public key depends on pa_pubz[] parameter and represented either +// - in (X:Y:Z) projective Jacobian coordinates if pa_pubz[] != NULL +// or +// - in (x:y) affine coordinate if pa_pubz[] == NULL +*/ +DLL_PUBLIC +mbx_status mbx_sm2_ecpublic_key_ssl_mb8(BIGNUM* pa_pubx[8], + BIGNUM* pa_puby[8], + BIGNUM* pa_pubz[8], + const BIGNUM* const pa_skey[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* pa_bubz!=0 means the output is in Jacobian projective coordinates */ + int use_jproj_coords = NULL!=pa_pubz; + + /* test input pointers */ + if(NULL==pa_pubx || NULL==pa_puby || NULL==pa_skey) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + BIGNUM* out_x = pa_pubx[buf_no]; + BIGNUM* out_y = pa_puby[buf_no]; + BIGNUM* out_z = use_jproj_coords? pa_pubz[buf_no] : NULL; + const BIGNUM* key = pa_skey[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL==out_x || NULL==out_y || (use_jproj_coords && NULL==out_z) || NULL==key) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* + // processing + */ + + /* zero padded keys */ + U64 scalarz[PSM2_LEN64+1]; + ifma_BN_transpose_copy((int64u (*)[8])scalarz, pa_skey, PSM2_BITSIZE); + scalarz[PSM2_LEN64] = get_zero64(); + + status |= MBX_SET_STS_BY_MASK(status, is_zero(scalarz, PSM2_LEN64+1), MBX_STATUS_MISMATCH_PARAM_ERR); + + /* do not need to clear copy of secret keys before this return - all of them is NULL or zero */ + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* public key */ + SM2_POINT P; + + /* compute public keys */ + MB_FUNC_NAME(ifma_ec_sm2_mul_pointbase_)(&P, scalarz); + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])scalarz, sizeof(scalarz)/sizeof(U64)); + + if(!use_jproj_coords) + MB_FUNC_NAME(get_sm2_ec_affine_coords_)(P.X, P.Y, &P); + + /* convert P coordinates to regular domain */ + MB_FUNC_NAME(ifma_frommont52_psm2_)(P.X, P.X); + MB_FUNC_NAME(ifma_frommont52_psm2_)(P.Y, P.Y); + if(use_jproj_coords) + MB_FUNC_NAME(ifma_frommont52_psm2_)(P.Z, P.Z); + + /* convert public key and store BIGNUM result */ + const int len_coord = NUMBER_OF_DIGITS(PSM2_BITSIZE,8); + int8u tmp[8][NUMBER_OF_DIGITS(PSM2_BITSIZE,8)]; /* need set inplace - MSVC compiler problem */ + int8u* const pa_tmp[8] = {tmp[0],tmp[1],tmp[2],tmp[3],tmp[4],tmp[5],tmp[6],tmp[7]}; + + /* X */ + ifma_mb8_to_HexStr8(pa_tmp, (const int64u (*)[8])P.X, PSM2_BITSIZE); + for(buf_no=0; (buf_no<8) && (0==MBX_GET_STS(status, buf_no)); buf_no++) { + BN_bin2bn(pa_tmp[buf_no], len_coord, pa_pubx[buf_no]); + } + + /* Y */ + ifma_mb8_to_HexStr8(pa_tmp, (const int64u (*)[8])P.Y, PSM2_BITSIZE); + for(buf_no=0; (buf_no<8) && (0==MBX_GET_STS(status, buf_no)); buf_no++) { + BN_bin2bn(pa_tmp[buf_no], len_coord, pa_puby[buf_no]); + } + + /* Z */ + if(use_jproj_coords) { + ifma_mb8_to_HexStr8(pa_tmp, (const int64u (*)[8])P.Z, PSM2_BITSIZE); + for(buf_no=0; (buf_no<8) && (0==MBX_GET_STS(status, buf_no)); buf_no++) { + BN_bin2bn(pa_tmp[buf_no], len_coord, pa_pubz[buf_no]); + } + } + + return status; +} +#endif // BN_OPENSSL_DISABLE + +DLL_PUBLIC +mbx_status mbx_sm2_ecpublic_key_mb8(int64u* pa_pubx[8], + int64u* pa_puby[8], + int64u* pa_pubz[8], + const int64u* const pa_skey[8], + int8u* pBuffer) +{ + mbx_status status = 0; + int buf_no; + + /* pa_bubz!=0 means the output is in Jacobian projective coordinates */ + int use_jproj_coords = NULL!=pa_pubz; + + /* test input pointers */ + if(NULL==pa_pubx || NULL==pa_puby || NULL==pa_skey) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + int64u* out_x = pa_pubx[buf_no]; + int64u* out_y = pa_puby[buf_no]; + int64u* out_z = use_jproj_coords? pa_pubz[buf_no] : NULL; + const int64u* key = pa_skey[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL==out_x || NULL==out_y || (use_jproj_coords && NULL==out_z) || NULL==key) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + } + } + + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* + // processing + */ + + /* zero padded keys */ + U64 scalarz[PSM2_LEN64+1]; + ifma_BNU_transpose_copy((int64u (*)[8])scalarz, pa_skey, PSM2_BITSIZE); + scalarz[PSM2_LEN64] = get_zero64(); + + status |= MBX_SET_STS_BY_MASK(status, is_zero(scalarz, PSM2_LEN64+1), MBX_STATUS_MISMATCH_PARAM_ERR); + + /* do not need to clear copy of secret keys before this return - all of them is NULL or zero */ + if(!MBX_IS_ANY_OK_STS(status)) + return status; + + /* public key */ + SM2_POINT P; + + /* compute public keys */ + MB_FUNC_NAME(ifma_ec_sm2_mul_pointbase_)(&P, scalarz); + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])scalarz, sizeof(scalarz)/sizeof(U64)); + + if(!use_jproj_coords) + MB_FUNC_NAME(get_sm2_ec_affine_coords_)(P.X, P.Y, &P); + + /* convert P coordinates to regular domain */ + MB_FUNC_NAME(ifma_frommont52_psm2_)(P.X, P.X); + MB_FUNC_NAME(ifma_frommont52_psm2_)(P.Y, P.Y); + if(use_jproj_coords) + MB_FUNC_NAME(ifma_frommont52_psm2_)(P.Z, P.Z); + + /* store result */ + ifma_mb8_to_BNU(pa_pubx, (const int64u (*)[8])P.X, PSM2_BITSIZE); + ifma_mb8_to_BNU(pa_puby, (const int64u (*)[8])P.Y, PSM2_BITSIZE); + if(use_jproj_coords) + ifma_mb8_to_BNU(pa_pubz, (const int64u (*)[8])P.Z, PSM2_BITSIZE); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_avx512_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_avx512_mb16.c new file mode 100644 index 0000000..f118c01 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_avx512_mb16.c @@ -0,0 +1,330 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +__INLINE void TRANSPOSE_16X16_I32(int32u out[][16], const int32u* const inp[16]) +{ + __m512i r0 = _mm512_loadu_si512(inp[0]); + __m512i r1 = _mm512_loadu_si512(inp[1]); + __m512i r2 = _mm512_loadu_si512(inp[2]); + __m512i r3 = _mm512_loadu_si512(inp[3]); + __m512i r4 = _mm512_loadu_si512(inp[4]); + __m512i r5 = _mm512_loadu_si512(inp[5]); + __m512i r6 = _mm512_loadu_si512(inp[6]); + __m512i r7 = _mm512_loadu_si512(inp[7]); + __m512i r8 = _mm512_loadu_si512(inp[8]); + __m512i r9 = _mm512_loadu_si512(inp[9]); + __m512i r10 = _mm512_loadu_si512(inp[10]); + __m512i r11 = _mm512_loadu_si512(inp[11]); + __m512i r12 = _mm512_loadu_si512(inp[12]); + __m512i r13 = _mm512_loadu_si512(inp[13]); + __m512i r14 = _mm512_loadu_si512(inp[14]); + __m512i r15 = _mm512_loadu_si512(inp[15]); + + // tansposition + __m512i t0 = _mm512_unpacklo_epi32(r0, r1); // 0 16 1 17 4 20 5 21 8 24 9 25 12 28 13 29 + __m512i t1 = _mm512_unpackhi_epi32(r0, r1); // 2 18 3 19 6 22 7 23 10 26 11 27 14 30 15 31 + __m512i t2 = _mm512_unpacklo_epi32(r2, r3); // 32 48 33 49 ... + __m512i t3 = _mm512_unpackhi_epi32(r2, r3); // 34 50 35 51 ... + __m512i t4 = _mm512_unpacklo_epi32(r4, r5); // 64 80 65 81 ... + __m512i t5 = _mm512_unpackhi_epi32(r4, r5); // 66 82 67 83 ... + __m512i t6 = _mm512_unpacklo_epi32(r6, r7); // 96 112 97 113 ... + __m512i t7 = _mm512_unpackhi_epi32(r6, r7); // 98 114 99 115 ... + __m512i t8 = _mm512_unpacklo_epi32(r8, r9); // 128 ... + __m512i t9 = _mm512_unpackhi_epi32(r8, r9); // 130 ... + __m512i t10 = _mm512_unpacklo_epi32(r10, r11); // 160 ... + __m512i t11 = _mm512_unpackhi_epi32(r10, r11); // 162 ... + __m512i t12 = _mm512_unpacklo_epi32(r12, r13); // 196 ... + __m512i t13 = _mm512_unpackhi_epi32(r12, r13); // 198 ... + __m512i t14 = _mm512_unpacklo_epi32(r14, r15); // 228 ... + __m512i t15 = _mm512_unpackhi_epi32(r14, r15); // 230 ... + + r0 = _mm512_unpacklo_epi64(t0, t2); // 0 16 32 48 ... + r1 = _mm512_unpackhi_epi64(t0, t2); // 1 17 33 49 ... + r2 = _mm512_unpacklo_epi64(t1, t3); // 2 18 34 49 ... + r3 = _mm512_unpackhi_epi64(t1, t3); // 3 19 35 51 ... + r4 = _mm512_unpacklo_epi64(t4, t6); // 64 80 96 112 ... + r5 = _mm512_unpackhi_epi64(t4, t6); // 65 81 97 114 ... + r6 = _mm512_unpacklo_epi64(t5, t7); // 66 82 98 113 ... + r7 = _mm512_unpackhi_epi64(t5, t7); // 67 83 99 115 ... + r8 = _mm512_unpacklo_epi64(t8, t10); // 128 144 160 176 ... + r9 = _mm512_unpackhi_epi64(t8, t10); // 129 145 161 178 ... + r10 = _mm512_unpacklo_epi64(t9, t11); // 130 146 162 177 ... + r11 = _mm512_unpackhi_epi64(t9, t11); // 131 147 163 179 ... + r12 = _mm512_unpacklo_epi64(t12, t14); // 192 208 228 240 ... + r13 = _mm512_unpackhi_epi64(t12, t14); // 193 209 229 241 ... + r14 = _mm512_unpacklo_epi64(t13, t15); // 194 210 230 242 ... + r15 = _mm512_unpackhi_epi64(t13, t15); // 195 211 231 243 ... + + t0 = _mm512_shuffle_i32x4(r0, r4, 0x88); // 0 16 32 48 8 24 40 56 64 80 96 112 ... + t1 = _mm512_shuffle_i32x4(r1, r5, 0x88); // 1 17 33 49 ... + t2 = _mm512_shuffle_i32x4(r2, r6, 0x88); // 2 18 34 50 ... + t3 = _mm512_shuffle_i32x4(r3, r7, 0x88); // 3 19 35 51 ... + t4 = _mm512_shuffle_i32x4(r0, r4, 0xdd); // 4 20 36 52 ... + t5 = _mm512_shuffle_i32x4(r1, r5, 0xdd); // 5 21 37 53 ... + t6 = _mm512_shuffle_i32x4(r2, r6, 0xdd); // 6 22 38 54 ... + t7 = _mm512_shuffle_i32x4(r3, r7, 0xdd); // 7 23 39 55 ... + t8 = _mm512_shuffle_i32x4(r8, r12, 0x88); // 128 144 160 176 ... + t9 = _mm512_shuffle_i32x4(r9, r13, 0x88); // 129 145 161 177 ... + t10 = _mm512_shuffle_i32x4(r10, r14, 0x88); // 130 146 162 178 ... + t11 = _mm512_shuffle_i32x4(r11, r15, 0x88); // 131 147 163 179 ... + t12 = _mm512_shuffle_i32x4(r8, r12, 0xdd); // 132 148 164 180 ... + t13 = _mm512_shuffle_i32x4(r9, r13, 0xdd); // 133 149 165 181 ... + t14 = _mm512_shuffle_i32x4(r10, r14, 0xdd); // 134 150 166 182 ... + t15 = _mm512_shuffle_i32x4(r11, r15, 0xdd); // 135 151 167 183 ... + + r0 = _mm512_shuffle_i32x4(t0, t8, 0x88); // 0 16 32 48 64 80 96 112 ... 240 + r1 = _mm512_shuffle_i32x4(t1, t9, 0x88); // 1 17 33 49 66 81 97 113 ... 241 + r2 = _mm512_shuffle_i32x4(t2, t10, 0x88); // 2 18 34 50 67 82 98 114 ... 242 + r3 = _mm512_shuffle_i32x4(t3, t11, 0x88); // 3 19 35 51 68 83 99 115 ... 243 + r4 = _mm512_shuffle_i32x4(t4, t12, 0x88); // 4 ... + r5 = _mm512_shuffle_i32x4(t5, t13, 0x88); // 5 ... + r6 = _mm512_shuffle_i32x4(t6, t14, 0x88); // 6 ... + r7 = _mm512_shuffle_i32x4(t7, t15, 0x88); // 7 ... + r8 = _mm512_shuffle_i32x4(t0, t8, 0xdd); // 8 ... + r9 = _mm512_shuffle_i32x4(t1, t9, 0xdd); // 9 ... + r10 = _mm512_shuffle_i32x4(t2, t10, 0xdd); // 10 ... + r11 = _mm512_shuffle_i32x4(t3, t11, 0xdd); // 11 ... + r12 = _mm512_shuffle_i32x4(t4, t12, 0xdd); // 12 ... + r13 = _mm512_shuffle_i32x4(t5, t13, 0xdd); // 13 ... + r14 = _mm512_shuffle_i32x4(t6, t14, 0xdd); // 14 ... + r15 = _mm512_shuffle_i32x4(t7, t15, 0xdd); // 15 31 47 63 79 96 111 127 ... 255 + + _mm512_storeu_si512(out[0], r0); + _mm512_storeu_si512(out[1], r1); + _mm512_storeu_si512(out[2], r2); + _mm512_storeu_si512(out[3], r3); + _mm512_storeu_si512(out[4], r4); + _mm512_storeu_si512(out[5], r5); + _mm512_storeu_si512(out[6], r6); + _mm512_storeu_si512(out[7], r7); + _mm512_storeu_si512(out[8], r8); + _mm512_storeu_si512(out[9], r9); + _mm512_storeu_si512(out[10], r10); + _mm512_storeu_si512(out[11], r11); + _mm512_storeu_si512(out[12], r12); + _mm512_storeu_si512(out[13], r13); + _mm512_storeu_si512(out[14], r14); + _mm512_storeu_si512(out[15], r15); +} + +/* Boolean functions (0<=nr<16) */ +#define FF1(X,Y,Z) (_mm512_xor_epi32(_mm512_xor_epi32(X,Y), Z)) +#define GG1(X,Y,Z) (_mm512_xor_epi32(_mm512_xor_epi32(X,Y), Z)) +/* Boolean functions (16<=nr<64) */ +#define FF2(X,Y,Z) (_mm512_or_epi32(_mm512_or_epi32(_mm512_and_epi32(X,Y),_mm512_and_epi32(X,Z)),_mm512_and_epi32(Y,Z))) +#define GG2(X,Y,Z) (_mm512_or_epi32(_mm512_and_epi32(X,Y),_mm512_andnot_epi32(X,Z))) + +/* P0 permutation: */ +#define P0(X) (_mm512_xor_epi32(_mm512_xor_epi32(X, _mm512_rol_epi32 (X, 9)), _mm512_rol_epi32 (X, 17))) +/* P1 permutation: */ +#define P1(X) (_mm512_xor_epi32(_mm512_xor_epi32(X, _mm512_rol_epi32 (X, 15)), _mm512_rol_epi32 (X, 23))) + +/* Update W */ +#define WUPDATE(nr, W) (_mm512_xor_epi32(_mm512_xor_epi32(P1(_mm512_xor_epi32(_mm512_xor_epi32(W[(nr-16)&15], W[(nr-9)&15]), _mm512_rol_epi32 (W[(nr-3)&15], 15))), _mm512_rol_epi32(W[(nr-13)&15],7)), W[(nr-6)&15])) + +// SM3 steps +/* (0<=nr<16) */ +#define STEP1_SM3(nr, A,B,C,D,E,F,G,H, Tj, W) {\ + __m512i SS1 = _mm512_rol_epi32(_mm512_add_epi32(_mm512_add_epi32(_mm512_rol_epi32(A, 12), E), _mm512_set1_epi32((int)Tj)),7);\ + __m512i SS2 = _mm512_xor_epi32(SS1, _mm512_rol_epi32(A, 12));\ + __m512i TT1 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(FF1(A, B, C), D), SS2), _mm512_xor_epi32(W[nr&15], W[(nr+4)&15]));\ + __m512i TT2 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(GG1(E, F, G), H), SS1), W[nr&15]);\ + D = _mm512_load_epi32((void*)&C); \ + C = _mm512_rol_epi32(B, 9); \ + B = _mm512_load_epi32((void*)&A); \ + A = _mm512_load_epi32((void*)&TT1);\ + H = _mm512_load_epi32((void*)&G); \ + G = _mm512_rol_epi32(F, 19); \ + F = _mm512_load_epi32((void*)&E); \ + E = P0(TT2); \ + W[(nr)&15]=WUPDATE(nr, W); \ +} + +/* (16<=nr<64) */ +#define STEP2_SM3(nr, A,B,C,D,E,F,G,H, Tj, W) {\ + __m512i SS1 = _mm512_rol_epi32(_mm512_add_epi32(_mm512_add_epi32(_mm512_rol_epi32(A, 12), E), _mm512_set1_epi32((int)Tj)),7);\ + __m512i SS2 = _mm512_xor_epi32(SS1, _mm512_rol_epi32(A, 12));\ + __m512i TT1 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(FF2(A, B, C), D), SS2), _mm512_xor_epi32(W[nr&15], W[(nr+4)&15]));\ + __m512i TT2 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(GG2(E, F, G), H), SS1), W[nr&15]);\ + D = _mm512_load_epi32((void*)&C); \ + C = _mm512_rol_epi32(B, 9); \ + B = _mm512_load_epi32((void*)&A); \ + A = _mm512_load_epi32((void*)&TT1);\ + H = _mm512_load_epi32((void*)&G); \ + G = _mm512_rol_epi32(F, 19); \ + F = _mm512_load_epi32((void*)&E); \ + E = P0(TT2); \ + W[(nr)&15]=WUPDATE(nr, W); \ +} + + +void sm3_avx512_mb16(int32u hash_pa[][16], const int8u* const msg_pa[16], int len[16]) +{ + int i; + + __ALIGN64 int32u* loc_data[SM3_NUM_BUFFERS]; + __ALIGN64 int loc_len[SM3_NUM_BUFFERS]; + + __m512i W[16]; + __m512i Vi[8]; + __m512i A, B, C, D, E, F, G, H; + + /* Allocate memory to handle numBuffers < 16, set data in not valid buffers to zero */ + __m512i zero_buffer = _mm512_setzero_si512(); + + /* Load processing mask */ + __mmask16 mb_mask = _mm512_cmp_epi32_mask(_mm512_loadu_si512(len), zero_buffer, _MM_CMPINT_NLE); + + /* Load data and set the data to zero in not valid buffers */ + M512(loc_len) = _mm512_loadu_si512(len); + + _mm512_storeu_si512(loc_data, _mm512_mask_loadu_epi64(_mm512_set1_epi64((long long)&zero_buffer), (__mmask8)mb_mask, msg_pa)); + _mm512_storeu_si512(loc_data+8, _mm512_mask_loadu_epi64(_mm512_set1_epi64((long long)&zero_buffer), *((__mmask8*)&mb_mask + 1), msg_pa + 8)); + + /* Load hash value */ + A = _mm512_loadu_si512(hash_pa); + B = _mm512_loadu_si512(hash_pa + 1); + C = _mm512_loadu_si512(hash_pa + 2); + D = _mm512_loadu_si512(hash_pa + 3); + E = _mm512_loadu_si512(hash_pa + 4); + F = _mm512_loadu_si512(hash_pa + 5); + G = _mm512_loadu_si512(hash_pa + 6); + H = _mm512_loadu_si512(hash_pa + 7); + + /* Loop over the message */ + while (mb_mask){ + /* Transpose the message data */ + TRANSPOSE_16X16_I32((int32u(*)[16])W, (const int32u**)loc_data); + + /* Init W (remember about endian) */ + for (i = 0; i < 16; i++) { + W[i] = SIMD_ENDIANNESS32(W[i]); + } + + /* Store previous hash for xor operation V(i+1) = ABCDEFGH XOR V(i) */ + Vi[0] = _mm512_load_epi32((void*)&A); + Vi[1] = _mm512_load_epi32((void*)&B); + Vi[2] = _mm512_load_epi32((void*)&C); + Vi[3] = _mm512_load_epi32((void*)&D); + Vi[4] = _mm512_load_epi32((void*)&E); + Vi[5] = _mm512_load_epi32((void*)&F); + Vi[6] = _mm512_load_epi32((void*)&G); + Vi[7] = _mm512_load_epi32((void*)&H); + + /* Compression function */ + { + STEP1_SM3(0, A, B, C, D, E, F, G, H, tj_calculated[0], W); + STEP1_SM3(1, A, B, C, D, E, F, G, H, tj_calculated[1], W); + STEP1_SM3(2, A, B, C, D, E, F, G, H, tj_calculated[2], W); + STEP1_SM3(3, A, B, C, D, E, F, G, H, tj_calculated[3], W); + STEP1_SM3(4, A, B, C, D, E, F, G, H, tj_calculated[4], W); + STEP1_SM3(5, A, B, C, D, E, F, G, H, tj_calculated[5], W); + STEP1_SM3(6, A, B, C, D, E, F, G, H, tj_calculated[6], W); + STEP1_SM3(7, A, B, C, D, E, F, G, H, tj_calculated[7], W); + + STEP1_SM3(8, A, B, C, D, E, F, G, H, tj_calculated[8], W); + STEP1_SM3(9, A, B, C, D, E, F, G, H, tj_calculated[9], W); + STEP1_SM3(10, A, B, C, D, E, F, G, H, tj_calculated[10], W); + STEP1_SM3(11, A, B, C, D, E, F, G, H, tj_calculated[11], W); + STEP1_SM3(12, A, B, C, D, E, F, G, H, tj_calculated[12], W); + STEP1_SM3(13, A, B, C, D, E, F, G, H, tj_calculated[13], W); + STEP1_SM3(14, A, B, C, D, E, F, G, H, tj_calculated[14], W); + STEP1_SM3(15, A, B, C, D, E, F, G, H, tj_calculated[15], W); + + STEP2_SM3(16, A, B, C, D, E, F, G, H, tj_calculated[16], W); + STEP2_SM3(17, A, B, C, D, E, F, G, H, tj_calculated[17], W); + STEP2_SM3(18, A, B, C, D, E, F, G, H, tj_calculated[18], W); + STEP2_SM3(19, A, B, C, D, E, F, G, H, tj_calculated[19], W); + STEP2_SM3(20, A, B, C, D, E, F, G, H, tj_calculated[20], W); + STEP2_SM3(21, A, B, C, D, E, F, G, H, tj_calculated[21], W); + STEP2_SM3(22, A, B, C, D, E, F, G, H, tj_calculated[22], W); + STEP2_SM3(23, A, B, C, D, E, F, G, H, tj_calculated[23], W); + + STEP2_SM3(24, A, B, C, D, E, F, G, H, tj_calculated[24], W); + STEP2_SM3(25, A, B, C, D, E, F, G, H, tj_calculated[25], W); + STEP2_SM3(26, A, B, C, D, E, F, G, H, tj_calculated[26], W); + STEP2_SM3(27, A, B, C, D, E, F, G, H, tj_calculated[27], W); + STEP2_SM3(28, A, B, C, D, E, F, G, H, tj_calculated[28], W); + STEP2_SM3(29, A, B, C, D, E, F, G, H, tj_calculated[29], W); + STEP2_SM3(30, A, B, C, D, E, F, G, H, tj_calculated[30], W); + STEP2_SM3(31, A, B, C, D, E, F, G, H, tj_calculated[31], W); + + STEP2_SM3(32, A, B, C, D, E, F, G, H, tj_calculated[32], W); + STEP2_SM3(33, A, B, C, D, E, F, G, H, tj_calculated[33], W); + STEP2_SM3(34, A, B, C, D, E, F, G, H, tj_calculated[34], W); + STEP2_SM3(35, A, B, C, D, E, F, G, H, tj_calculated[35], W); + STEP2_SM3(36, A, B, C, D, E, F, G, H, tj_calculated[36], W); + STEP2_SM3(37, A, B, C, D, E, F, G, H, tj_calculated[37], W); + STEP2_SM3(38, A, B, C, D, E, F, G, H, tj_calculated[38], W); + STEP2_SM3(39, A, B, C, D, E, F, G, H, tj_calculated[39], W); + + STEP2_SM3(40, A, B, C, D, E, F, G, H, tj_calculated[40], W); + STEP2_SM3(41, A, B, C, D, E, F, G, H, tj_calculated[41], W); + STEP2_SM3(42, A, B, C, D, E, F, G, H, tj_calculated[42], W); + STEP2_SM3(43, A, B, C, D, E, F, G, H, tj_calculated[43], W); + STEP2_SM3(44, A, B, C, D, E, F, G, H, tj_calculated[44], W); + STEP2_SM3(45, A, B, C, D, E, F, G, H, tj_calculated[45], W); + STEP2_SM3(46, A, B, C, D, E, F, G, H, tj_calculated[46], W); + STEP2_SM3(47, A, B, C, D, E, F, G, H, tj_calculated[47], W); + + STEP2_SM3(48, A, B, C, D, E, F, G, H, tj_calculated[16], W); + STEP2_SM3(49, A, B, C, D, E, F, G, H, tj_calculated[17], W); + STEP2_SM3(50, A, B, C, D, E, F, G, H, tj_calculated[18], W); + STEP2_SM3(51, A, B, C, D, E, F, G, H, tj_calculated[19], W); + STEP2_SM3(52, A, B, C, D, E, F, G, H, tj_calculated[20], W); + STEP2_SM3(53, A, B, C, D, E, F, G, H, tj_calculated[21], W); + STEP2_SM3(54, A, B, C, D, E, F, G, H, tj_calculated[22], W); + STEP2_SM3(55, A, B, C, D, E, F, G, H, tj_calculated[23], W); + + STEP2_SM3(56, A, B, C, D, E, F, G, H, tj_calculated[24], W); + STEP2_SM3(57, A, B, C, D, E, F, G, H, tj_calculated[25], W); + STEP2_SM3(58, A, B, C, D, E, F, G, H, tj_calculated[26], W); + STEP2_SM3(59, A, B, C, D, E, F, G, H, tj_calculated[27], W); + STEP2_SM3(60, A, B, C, D, E, F, G, H, tj_calculated[28], W); + STEP2_SM3(61, A, B, C, D, E, F, G, H, tj_calculated[29], W); + STEP2_SM3(62, A, B, C, D, E, F, G, H, tj_calculated[30], W); + STEP2_SM3(63, A, B, C, D, E, F, G, H, tj_calculated[31], W); + } + + A = _mm512_mask_xor_epi32(Vi[0], mb_mask, A, Vi[0]); + B = _mm512_mask_xor_epi32(Vi[1], mb_mask, B, Vi[1]); + C = _mm512_mask_xor_epi32(Vi[2], mb_mask, C, Vi[2]); + D = _mm512_mask_xor_epi32(Vi[3], mb_mask, D, Vi[3]); + E = _mm512_mask_xor_epi32(Vi[4], mb_mask, E, Vi[4]); + F = _mm512_mask_xor_epi32(Vi[5], mb_mask, F, Vi[5]); + G = _mm512_mask_xor_epi32(Vi[6], mb_mask, G, Vi[6]); + H = _mm512_mask_xor_epi32(Vi[7], mb_mask, H, Vi[7]); + + _mm512_storeu_si512(hash_pa, A); + _mm512_storeu_si512(hash_pa + 1, B); + _mm512_storeu_si512(hash_pa + 2, C); + _mm512_storeu_si512(hash_pa + 3, D); + _mm512_storeu_si512(hash_pa + 4, E); + _mm512_storeu_si512(hash_pa + 5, F); + _mm512_storeu_si512(hash_pa + 6, G); + _mm512_storeu_si512(hash_pa + 7, H); + + /* Update pointers to data, local lengths and mask */ + _mm512_storeu_si512(loc_data, _mm512_mask_add_epi64(_mm512_set1_epi64((long long)&zero_buffer), (__mmask8)mb_mask, _mm512_loadu_si512(loc_data), _mm512_set1_epi64(SM3_MSG_BLOCK_SIZE))); + _mm512_storeu_si512(loc_data + 8, _mm512_mask_add_epi64(_mm512_set1_epi64((long long)&zero_buffer), *((__mmask8*)&mb_mask + 1), _mm512_loadu_si512(loc_data+8), _mm512_set1_epi64(SM3_MSG_BLOCK_SIZE))); + + M512(loc_len) = _mm512_mask_sub_epi32(zero_buffer, mb_mask, _mm512_loadu_si512(loc_len), _mm512_set1_epi32(SM3_MSG_BLOCK_SIZE)); + mb_mask = _mm512_cmp_epi32_mask(_mm512_loadu_si512(loc_len), zero_buffer, _MM_CMPINT_NLE); + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_avx512_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_avx512_mb8.c new file mode 100644 index 0000000..be8ed61 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_avx512_mb8.c @@ -0,0 +1,224 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +/* Boolean functions (0<=nr<16) */ +#define FF1(X,Y,Z) (_mm256_xor_si256(_mm256_xor_si256(X,Y), Z)) +#define GG1(X,Y,Z) (_mm256_xor_si256(_mm256_xor_si256(X,Y), Z)) +/* Boolean functions (16<=nr<64) */ +#define FF2(X,Y,Z) (_mm256_or_si256(_mm256_or_si256(_mm256_and_si256(X,Y),_mm256_and_si256(X,Z)),_mm256_and_si256(Y,Z))) +#define GG2(X,Y,Z) (_mm256_or_si256(_mm256_and_si256(X,Y),_mm256_andnot_si256(X,Z))) + +/* P0 permutation: */ +#define P0(X) (_mm256_xor_si256(_mm256_xor_si256(X, _mm256_rol_epi32 (X, 9)), _mm256_rol_epi32 (X, 17))) +/* P1 permutation: */ +#define P1(X) (_mm256_xor_si256(_mm256_xor_si256(X, _mm256_rol_epi32 (X, 15)), _mm256_rol_epi32 (X, 23))) + +/* Update W */ +#define WUPDATE(nr, W) (_mm256_xor_si256(_mm256_xor_si256(P1(_mm256_xor_si256(_mm256_xor_si256(W[(nr-16)&15], W[(nr-9)&15]), _mm256_rol_epi32 (W[(nr-3)&15], 15))), _mm256_rol_epi32(W[(nr-13)&15],7)), W[(nr-6)&15])) + +// SM3 steps +/* (0<=nr<16) */ +#define STEP1_SM3(nr, A,B,C,D,E,F,G,H, Tj, W) {\ + __m256i SS1 = _mm256_rol_epi32(_mm256_add_epi32(_mm256_add_epi32(_mm256_rol_epi32(A, 12), E), _mm256_set1_epi32((int)Tj)),7);\ + __m256i SS2 = _mm256_xor_si256(SS1, _mm256_rol_epi32(A, 12));\ + __m256i TT1 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(FF1(A, B, C), D), SS2), _mm256_xor_si256(W[nr&15], W[(nr+4)&15]));\ + __m256i TT2 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(GG1(E, F, G), H), SS1), W[nr&15]);\ + D = _mm256_loadu_si256((void*)&C); \ + C = _mm256_rol_epi32(B, 9); \ + B = _mm256_loadu_si256((void*)&A); \ + A = _mm256_loadu_si256((void*)&TT1);\ + H = _mm256_loadu_si256((void*)&G); \ + G = _mm256_rol_epi32(F, 19); \ + F = _mm256_loadu_si256((void*)&E); \ + E = P0(TT2); \ + W[(nr)&15]=WUPDATE(nr, W); \ +} + +/* (16<=nr<64) */ +#define STEP2_SM3(nr, A,B,C,D,E,F,G,H, Tj, W) {\ + __m256i SS1 = _mm256_rol_epi32(_mm256_add_epi32(_mm256_add_epi32(_mm256_rol_epi32(A, 12), E), _mm256_set1_epi32((int)Tj)),7);\ + __m256i SS2 = _mm256_xor_si256(SS1, _mm256_rol_epi32(A, 12));\ + __m256i TT1 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(FF2(A, B, C), D), SS2), _mm256_xor_si256(W[nr&15], W[(nr+4)&15]));\ + __m256i TT2 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(GG2(E, F, G), H), SS1), W[nr&15]);\ + D = _mm256_loadu_si256((void*)&C); \ + C = _mm256_rol_epi32(B, 9); \ + B = _mm256_loadu_si256((void*)&A); \ + A = _mm256_loadu_si256((void*)&TT1);\ + H = _mm256_loadu_si256((void*)&G); \ + G = _mm256_rol_epi32(F, 19); \ + F = _mm256_loadu_si256((void*)&E); \ + E = P0(TT2); \ + W[(nr)&15]=WUPDATE(nr, W); \ +} + + +void sm3_avx512_mb8(int32u hash_pa[][8], const int8u* const msg_pa[8], int len[8]) +{ + int i; + + __ALIGN64 int32u* loc_data[SM3_NUM_BUFFERS8]; + __ALIGN64 int loc_len[SM3_NUM_BUFFERS8]; + + __m256i W[16]; + int32u* p_W[16] = { (int32u*)&W[0], (int32u*)&W[1], (int32u*)&W[2], (int32u*)&W[3], (int32u*)&W[4], (int32u*)&W[5], (int32u*)&W[6], (int32u*)&W[7], + (int32u*)&W[8], (int32u*)&W[9], (int32u*)&W[10], (int32u*)&W[11], (int32u*)&W[12], (int32u*)&W[13], (int32u*)&W[14], (int32u*)&W[15] }; + + __m256i Vi[8]; + __m256i A, B, C, D, E, F, G, H; + + /* Allocate memory to handle numBuffers < 16, set data in not valid buffers to zero */ + __m512i zero_buffer = _mm512_setzero_si512(); + + /* Load processing mask */ + __mmask8 mb_mask = _mm256_cmp_epi32_mask(_mm256_loadu_si256((__m256i*)len), M256(&zero_buffer), _MM_CMPINT_NE); + + /* Load data and set the data to zero in not valid buffers */ + M256(loc_len) = _mm256_loadu_si256((__m256i*)len); + + _mm512_storeu_si512(loc_data, _mm512_mask_loadu_epi64(_mm512_set1_epi64((long long)&zero_buffer), mb_mask, msg_pa)); + + /* Load hash value */ + A = _mm256_loadu_si256((__m256i*)hash_pa); + B = _mm256_loadu_si256((__m256i*)(hash_pa + 1)); + C = _mm256_loadu_si256((__m256i*)(hash_pa + 2)); + D = _mm256_loadu_si256((__m256i*)(hash_pa + 3)); + E = _mm256_loadu_si256((__m256i*)(hash_pa + 4)); + F = _mm256_loadu_si256((__m256i*)(hash_pa + 5)); + G = _mm256_loadu_si256((__m256i*)(hash_pa + 6)); + H = _mm256_loadu_si256((__m256i*)(hash_pa + 7)); + + /* Loop over the message */ + while (mb_mask){ + /* Transpose the message data */ + TRANSPOSE_8X16_I32(p_W, (const int32u**)loc_data, 0xFFFF); + + /* Init W (remember about endian) */ + for (i = 0; i < 16; i++) { + W[i] = SIMD_ENDIANNESS32(W[i]); + } + + /* Store previous hash for xor operation V(i+1) = ABCDEFGH XOR V(i) */ + Vi[0] = _mm256_loadu_si256((void*)&A); + Vi[1] = _mm256_loadu_si256((void*)&B); + Vi[2] = _mm256_loadu_si256((void*)&C); + Vi[3] = _mm256_loadu_si256((void*)&D); + Vi[4] = _mm256_loadu_si256((void*)&E); + Vi[5] = _mm256_loadu_si256((void*)&F); + Vi[6] = _mm256_loadu_si256((void*)&G); + Vi[7] = _mm256_loadu_si256((void*)&H); + + /* Compression function */ + { + STEP1_SM3(0, A, B, C, D, E, F, G, H, tj_calculated[0], W); + STEP1_SM3(1, A, B, C, D, E, F, G, H, tj_calculated[1], W); + STEP1_SM3(2, A, B, C, D, E, F, G, H, tj_calculated[2], W); + STEP1_SM3(3, A, B, C, D, E, F, G, H, tj_calculated[3], W); + STEP1_SM3(4, A, B, C, D, E, F, G, H, tj_calculated[4], W); + STEP1_SM3(5, A, B, C, D, E, F, G, H, tj_calculated[5], W); + STEP1_SM3(6, A, B, C, D, E, F, G, H, tj_calculated[6], W); + STEP1_SM3(7, A, B, C, D, E, F, G, H, tj_calculated[7], W); + + STEP1_SM3(8, A, B, C, D, E, F, G, H, tj_calculated[8], W); + STEP1_SM3(9, A, B, C, D, E, F, G, H, tj_calculated[9], W); + STEP1_SM3(10, A, B, C, D, E, F, G, H, tj_calculated[10], W); + STEP1_SM3(11, A, B, C, D, E, F, G, H, tj_calculated[11], W); + STEP1_SM3(12, A, B, C, D, E, F, G, H, tj_calculated[12], W); + STEP1_SM3(13, A, B, C, D, E, F, G, H, tj_calculated[13], W); + STEP1_SM3(14, A, B, C, D, E, F, G, H, tj_calculated[14], W); + STEP1_SM3(15, A, B, C, D, E, F, G, H, tj_calculated[15], W); + + STEP2_SM3(16, A, B, C, D, E, F, G, H, tj_calculated[16], W); + STEP2_SM3(17, A, B, C, D, E, F, G, H, tj_calculated[17], W); + STEP2_SM3(18, A, B, C, D, E, F, G, H, tj_calculated[18], W); + STEP2_SM3(19, A, B, C, D, E, F, G, H, tj_calculated[19], W); + STEP2_SM3(20, A, B, C, D, E, F, G, H, tj_calculated[20], W); + STEP2_SM3(21, A, B, C, D, E, F, G, H, tj_calculated[21], W); + STEP2_SM3(22, A, B, C, D, E, F, G, H, tj_calculated[22], W); + STEP2_SM3(23, A, B, C, D, E, F, G, H, tj_calculated[23], W); + + STEP2_SM3(24, A, B, C, D, E, F, G, H, tj_calculated[24], W); + STEP2_SM3(25, A, B, C, D, E, F, G, H, tj_calculated[25], W); + STEP2_SM3(26, A, B, C, D, E, F, G, H, tj_calculated[26], W); + STEP2_SM3(27, A, B, C, D, E, F, G, H, tj_calculated[27], W); + STEP2_SM3(28, A, B, C, D, E, F, G, H, tj_calculated[28], W); + STEP2_SM3(29, A, B, C, D, E, F, G, H, tj_calculated[29], W); + STEP2_SM3(30, A, B, C, D, E, F, G, H, tj_calculated[30], W); + STEP2_SM3(31, A, B, C, D, E, F, G, H, tj_calculated[31], W); + + STEP2_SM3(32, A, B, C, D, E, F, G, H, tj_calculated[32], W); + STEP2_SM3(33, A, B, C, D, E, F, G, H, tj_calculated[33], W); + STEP2_SM3(34, A, B, C, D, E, F, G, H, tj_calculated[34], W); + STEP2_SM3(35, A, B, C, D, E, F, G, H, tj_calculated[35], W); + STEP2_SM3(36, A, B, C, D, E, F, G, H, tj_calculated[36], W); + STEP2_SM3(37, A, B, C, D, E, F, G, H, tj_calculated[37], W); + STEP2_SM3(38, A, B, C, D, E, F, G, H, tj_calculated[38], W); + STEP2_SM3(39, A, B, C, D, E, F, G, H, tj_calculated[39], W); + + STEP2_SM3(40, A, B, C, D, E, F, G, H, tj_calculated[40], W); + STEP2_SM3(41, A, B, C, D, E, F, G, H, tj_calculated[41], W); + STEP2_SM3(42, A, B, C, D, E, F, G, H, tj_calculated[42], W); + STEP2_SM3(43, A, B, C, D, E, F, G, H, tj_calculated[43], W); + STEP2_SM3(44, A, B, C, D, E, F, G, H, tj_calculated[44], W); + STEP2_SM3(45, A, B, C, D, E, F, G, H, tj_calculated[45], W); + STEP2_SM3(46, A, B, C, D, E, F, G, H, tj_calculated[46], W); + STEP2_SM3(47, A, B, C, D, E, F, G, H, tj_calculated[47], W); + + STEP2_SM3(48, A, B, C, D, E, F, G, H, tj_calculated[16], W); + STEP2_SM3(49, A, B, C, D, E, F, G, H, tj_calculated[17], W); + STEP2_SM3(50, A, B, C, D, E, F, G, H, tj_calculated[18], W); + STEP2_SM3(51, A, B, C, D, E, F, G, H, tj_calculated[19], W); + STEP2_SM3(52, A, B, C, D, E, F, G, H, tj_calculated[20], W); + STEP2_SM3(53, A, B, C, D, E, F, G, H, tj_calculated[21], W); + STEP2_SM3(54, A, B, C, D, E, F, G, H, tj_calculated[22], W); + STEP2_SM3(55, A, B, C, D, E, F, G, H, tj_calculated[23], W); + + STEP2_SM3(56, A, B, C, D, E, F, G, H, tj_calculated[24], W); + STEP2_SM3(57, A, B, C, D, E, F, G, H, tj_calculated[25], W); + STEP2_SM3(58, A, B, C, D, E, F, G, H, tj_calculated[26], W); + STEP2_SM3(59, A, B, C, D, E, F, G, H, tj_calculated[27], W); + STEP2_SM3(60, A, B, C, D, E, F, G, H, tj_calculated[28], W); + STEP2_SM3(61, A, B, C, D, E, F, G, H, tj_calculated[29], W); + STEP2_SM3(62, A, B, C, D, E, F, G, H, tj_calculated[30], W); + STEP2_SM3(63, A, B, C, D, E, F, G, H, tj_calculated[31], W); + } + + A = _mm256_mask_xor_epi32(Vi[0], mb_mask, A, Vi[0]); + B = _mm256_mask_xor_epi32(Vi[1], mb_mask, B, Vi[1]); + C = _mm256_mask_xor_epi32(Vi[2], mb_mask, C, Vi[2]); + D = _mm256_mask_xor_epi32(Vi[3], mb_mask, D, Vi[3]); + E = _mm256_mask_xor_epi32(Vi[4], mb_mask, E, Vi[4]); + F = _mm256_mask_xor_epi32(Vi[5], mb_mask, F, Vi[5]); + G = _mm256_mask_xor_epi32(Vi[6], mb_mask, G, Vi[6]); + H = _mm256_mask_xor_epi32(Vi[7], mb_mask, H, Vi[7]); + + _mm256_storeu_si256((__m256i*)hash_pa, A); + _mm256_storeu_si256((__m256i*)(hash_pa + 1), B); + _mm256_storeu_si256((__m256i*)(hash_pa + 2), C); + _mm256_storeu_si256((__m256i*)(hash_pa + 3), D); + _mm256_storeu_si256((__m256i*)(hash_pa + 4), E); + _mm256_storeu_si256((__m256i*)(hash_pa + 5), F); + _mm256_storeu_si256((__m256i*)(hash_pa + 6), G); + _mm256_storeu_si256((__m256i*)(hash_pa + 7), H); + + /* Update pointers to data, local lengths and mask */ + M512(loc_data) = _mm512_mask_add_epi64(_mm512_set1_epi64((long long)&zero_buffer), (__mmask8)mb_mask, _mm512_loadu_si512(loc_data), _mm512_set1_epi64(SM3_MSG_BLOCK_SIZE)); + M256(loc_len) = _mm256_mask_sub_epi32(M256(&zero_buffer), mb_mask, _mm256_loadu_si256((__m256i*)loc_len), _mm256_set1_epi32(SM3_MSG_BLOCK_SIZE)); + mb_mask = _mm256_cmp_epi32_mask(_mm256_loadu_si256((__m256i*)loc_len), M256(&zero_buffer), _MM_CMPINT_NE); + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_final_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_final_mb16.c new file mode 100644 index 0000000..bc23819 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_final_mb16.c @@ -0,0 +1,115 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include + +DLL_PUBLIC +mbx_status16 mbx_sm3_final_mb16(int8u* hash_pa[16], + SM3_CTX_mb16* p_state) +{ + int i; + mbx_status16 status = 0; + + /* test input pointers */ + if(NULL==hash_pa || NULL==p_state) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + __ALIGN64 int input_len[SM3_NUM_BUFFERS]; + __ALIGN64 int buffer_len[SM3_NUM_BUFFERS]; + __ALIGN64 int64u sum_msg_len[SM3_NUM_BUFFERS]; + + /* allocate local buffer */ + __ALIGN64 int8u loc_buffer[SM3_NUM_BUFFERS][SM3_MSG_BLOCK_SIZE*2]; + const int8u* buffer_pa[SM3_NUM_BUFFERS] = { loc_buffer[0], loc_buffer[1], loc_buffer[2], loc_buffer[3], + loc_buffer[4], loc_buffer[5], loc_buffer[6], loc_buffer[7], + loc_buffer[8], loc_buffer[9], loc_buffer[10], loc_buffer[11], + loc_buffer[12], loc_buffer[13], loc_buffer[14], loc_buffer[15] }; + + __m512i zero_buffer = _mm512_setzero_si512(); + + /* + // create __mmask8 and __mmask16 based on input hash_pa + // corresponding element in mask = 0 if hash_pa[i] = 0 + */ + __mmask8 mb_mask8[2]; + mb_mask8[0] = _mm512_cmp_epi64_mask(_mm512_loadu_si512(hash_pa), zero_buffer, _MM_CMPINT_NE); + mb_mask8[1] = _mm512_cmp_epi64_mask(_mm512_loadu_si512(hash_pa + 8), zero_buffer, _MM_CMPINT_NE); + __mmask16 mb_mask16 = *(__mmask16*)mb_mask8; + + M512(sum_msg_len) = _mm512_maskz_loadu_epi64(mb_mask8[0], MSG_LEN(p_state)); + M512(sum_msg_len + 8) = _mm512_maskz_loadu_epi64(mb_mask8[1], MSG_LEN(p_state) + 8); + + /* put processed message length in bits */ + M512(sum_msg_len) = _mm512_rol_epi64(M512(sum_msg_len), 3); + M512(sum_msg_len + 8) = _mm512_rol_epi64(M512(sum_msg_len+8), 3); + M512(sum_msg_len) = _mm512_shuffle_epi8(M512(sum_msg_len), M512(swapBytes)); + M512(sum_msg_len +8) = _mm512_shuffle_epi8(M512(sum_msg_len +8), M512(swapBytes)); + + M512(input_len) = _mm512_maskz_loadu_epi32(mb_mask16, HASH_BUFFIDX(p_state)); + + __mmask16 tmp_mask = _mm512_cmplt_epi32_mask(M512(input_len), _mm512_set1_epi32(SM3_MSG_BLOCK_SIZE - (int)SM3_MSG_LEN_REPR)); + M512(buffer_len) = _mm512_mask_set1_epi32(_mm512_set1_epi32(SM3_MSG_BLOCK_SIZE * 2), tmp_mask, SM3_MSG_BLOCK_SIZE); + M512(buffer_len) = _mm512_mask_set1_epi32(M512(buffer_len), ~mb_mask16, 0); + M512(buffer_len) = _mm512_mask_set1_epi32(M512(buffer_len), ~mb_mask16, 0); + + for (i = 0; i < SM3_NUM_BUFFERS; i++) { + /* Copy rest of message into internal buffer */ + if ((mb_mask16 >> i) & 0x1) { + __mmask64 mb_mask64 = ~(0xFFFFFFFFFFFFFFFF << input_len[i]); + M512(loc_buffer[i]) = _mm512_maskz_loadu_epi8(mb_mask64, HASH_BUFF(p_state)[i]); + + /* Pad message */ + loc_buffer[i][input_len[i]++] = 0x80; + pad_block(0, loc_buffer[i] + input_len[i], (int)(buffer_len[i] - input_len[i] - (int)SM3_MSG_LEN_REPR)); + ((int64u*)(loc_buffer[i] + buffer_len[i]))[-1] = sum_msg_len[i]; + } + } + + /* Copmplete hash computation */ + sm3_avx512_mb16(HASH_VALUE(p_state), buffer_pa, buffer_len); + + /* Convert hash into big endian */ + __m512i T[8]; + const int32u* p_T[8] = { (int32u*)&T[0], (int32u*)&T[1], (int32u*)&T[2], (int32u*)&T[3], (int32u*)&T[4], (int32u*)&T[5], (int32u*)&T[6], (int32u*)&T[7] }; + + T[0] = SIMD_ENDIANNESS32(_mm512_loadu_si512(HASH_VALUE(p_state)[0])); + T[1] = SIMD_ENDIANNESS32(_mm512_loadu_si512(HASH_VALUE(p_state)[1])); + T[2] = SIMD_ENDIANNESS32(_mm512_loadu_si512(HASH_VALUE(p_state)[2])); + T[3] = SIMD_ENDIANNESS32(_mm512_loadu_si512(HASH_VALUE(p_state)[3])); + T[4] = SIMD_ENDIANNESS32(_mm512_loadu_si512(HASH_VALUE(p_state)[4])); + T[5] = SIMD_ENDIANNESS32(_mm512_loadu_si512(HASH_VALUE(p_state)[5])); + T[6] = SIMD_ENDIANNESS32(_mm512_loadu_si512(HASH_VALUE(p_state)[6])); + T[7] = SIMD_ENDIANNESS32(_mm512_loadu_si512(HASH_VALUE(p_state)[7])); + + /* Transpose hash and store in array with pointers to hash values */ + TRANSPOSE_8X16_I32((int32u**)hash_pa, p_T, mb_mask16); + + /* re-init hash value using mb masks */ + _mm512_storeu_si512(MSG_LEN(p_state), _mm512_mask_set1_epi64(_mm512_loadu_si512(MSG_LEN(p_state)), mb_mask8[0], 0)); + _mm512_storeu_si512(MSG_LEN(p_state)+8, _mm512_mask_set1_epi64(_mm512_loadu_si512(MSG_LEN(p_state)+8), mb_mask8[1], 0)); + _mm512_storeu_si512(HASH_BUFFIDX(p_state), _mm512_mask_set1_epi32(_mm512_loadu_si512(HASH_BUFFIDX(p_state)), mb_mask16, 0)); + + sm3_mask_init_mb16(p_state, mb_mask16); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_final_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_final_mb8.c new file mode 100644 index 0000000..230a3dc --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_final_mb8.c @@ -0,0 +1,101 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include + +mbx_status sm3_final_mb8(int8u* hash_pa[8], SM3_CTX_mb8* p_state) +{ + int i; + mbx_status status = 0; + + /* test input pointers */ + if(NULL==hash_pa || NULL==p_state) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + __ALIGN64 int input_len[SM3_NUM_BUFFERS8]; + __ALIGN64 int buffer_len[SM3_NUM_BUFFERS8]; + __ALIGN64 int64u sum_msg_len[SM3_NUM_BUFFERS8]; + + /* allocate local buffer */ + __ALIGN64 int8u loc_buffer[SM3_NUM_BUFFERS8][SM3_MSG_BLOCK_SIZE*2]; + const int8u* buffer_pa[SM3_NUM_BUFFERS8] = { loc_buffer[0], loc_buffer[1], loc_buffer[2], loc_buffer[3], + loc_buffer[4], loc_buffer[5], loc_buffer[6], loc_buffer[7] }; + + /* + // create __mmask8 based on input hash_pa + // corresponding element in mask = 0 if hash_pa[i] = 0 + */ + __m512i zero_buffer = _mm512_setzero_si512(); + __mmask8 mb_mask8 = _mm512_cmp_epi64_mask(_mm512_loadu_si512(hash_pa), zero_buffer, _MM_CMPINT_NE); + + M512(sum_msg_len) = _mm512_maskz_loadu_epi64(mb_mask8, MSG_LEN(p_state)); + + /* put processed message length in bits */ + M512(sum_msg_len) = _mm512_rol_epi64(M512(sum_msg_len), 3); + M512(sum_msg_len) = _mm512_shuffle_epi8(M512(sum_msg_len), M512(swapBytes)); + + M256(input_len) = _mm256_maskz_loadu_epi32(mb_mask8, HASH_BUFFIDX(p_state)); + + __mmask8 tmp_mask = _mm256_cmplt_epi32_mask(M256(input_len), _mm256_set1_epi32(SM3_MSG_BLOCK_SIZE - (int)SM3_MSG_LEN_REPR)); + M256(buffer_len) = _mm256_mask_set1_epi32(_mm256_set1_epi32(SM3_MSG_BLOCK_SIZE * 2), tmp_mask, SM3_MSG_BLOCK_SIZE); + M256(buffer_len) = _mm256_mask_set1_epi32(M256(buffer_len), ~mb_mask8, 0); + + __mmask64 mb_mask64; + for (i = 0; i < SM3_NUM_BUFFERS8; i++) { + /* Copy rest of message into internal buffer */ + mb_mask64 = ~(0xFFFFFFFFFFFFFFFF << input_len[i]); + M512(loc_buffer[i]) = _mm512_maskz_loadu_epi8(mb_mask64, HASH_BUFF(p_state)[i]); + + /* Padd message */ + loc_buffer[i][input_len[i]++] = 0x80; + pad_block(0, loc_buffer[i] + input_len[i], (int)(buffer_len[i] - input_len[i] - (int)SM3_MSG_LEN_REPR)); + ((int64u*)(loc_buffer[i] + buffer_len[i]))[-1] = sum_msg_len[i]; + } + + /* Copmplete hash computation */ + sm3_avx512_mb8(HASH_VALUE(p_state), buffer_pa, buffer_len); + + /* Convert hash into big endian */ + __m256i T[8]; + const int32u* p_T[8] = { (int32u*)&T[0], (int32u*)&T[1], (int32u*)&T[2], (int32u*)&T[3], (int32u*)&T[4], (int32u*)&T[5], (int32u*)&T[6], (int32u*)&T[7] }; + + T[0] = SIMD_ENDIANNESS32(_mm256_loadu_si256((__m256i*)HASH_VALUE(p_state)[0])); + T[1] = SIMD_ENDIANNESS32(_mm256_loadu_si256((__m256i*)HASH_VALUE(p_state)[1])); + T[2] = SIMD_ENDIANNESS32(_mm256_loadu_si256((__m256i*)HASH_VALUE(p_state)[2])); + T[3] = SIMD_ENDIANNESS32(_mm256_loadu_si256((__m256i*)HASH_VALUE(p_state)[3])); + T[4] = SIMD_ENDIANNESS32(_mm256_loadu_si256((__m256i*)HASH_VALUE(p_state)[4])); + T[5] = SIMD_ENDIANNESS32(_mm256_loadu_si256((__m256i*)HASH_VALUE(p_state)[5])); + T[6] = SIMD_ENDIANNESS32(_mm256_loadu_si256((__m256i*)HASH_VALUE(p_state)[6])); + T[7] = SIMD_ENDIANNESS32(_mm256_loadu_si256((__m256i*)HASH_VALUE(p_state)[7])); + + /* Transpose hash and store in array with pointers to hash values */ + MASK_TRANSPOSE_8X8_I32((int32u**)hash_pa, p_T, mb_mask8); + + /* re-init hash value using mb masks */ + _mm512_storeu_si512(MSG_LEN(p_state), _mm512_mask_set1_epi64(_mm512_loadu_si512(MSG_LEN(p_state)), mb_mask8, 0)); + _mm256_storeu_si256((__m256i*)HASH_BUFFIDX(p_state), _mm256_mask_set1_epi32(_mm256_loadu_si256((__m256i*)HASH_BUFFIDX(p_state)), mb_mask8, 0)); + + sm3_mask_init_mb8(p_state, mb_mask8); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_init_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_init_mb16.c new file mode 100644 index 0000000..118561d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_init_mb16.c @@ -0,0 +1,70 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include + +void sm3_mask_init_mb16(SM3_CTX_mb16 * p_state, __mmask16 mb_mask) +{ + __mmask8 mb_mask8[2]; + mb_mask8[0] = (__mmask8)mb_mask; + mb_mask8[1] = *((__mmask8*)&mb_mask + 1); + + /* clear buffer index */ + _mm512_storeu_si512(HASH_BUFFIDX(p_state), _mm512_mask_set1_epi32(_mm512_loadu_si512(HASH_BUFFIDX(p_state)), mb_mask, 0)); + + /* clear summary message length */ + _mm512_storeu_si512(MSG_LEN(p_state), _mm512_maskz_loadu_epi64(~mb_mask8[0], MSG_LEN(p_state))); + _mm512_storeu_si512(MSG_LEN(p_state) + 8, _mm512_maskz_loadu_epi64(~mb_mask8[1], MSG_LEN(p_state) + 8)); + + + /* clear buffer */ + for (int i = 0; i < SM3_NUM_BUFFERS; i++) { + if ((mb_mask >> i) & 1) + _mm512_storeu_si512(HASH_BUFF(p_state)[i], _mm512_setzero_si512()); + } + + /* setup initial digest in multi-buffer format */ + _mm512_storeu_si512(HASH_VALUE(p_state)[0], _mm512_mask_set1_epi32(_mm512_loadu_si512(HASH_VALUE(p_state)[0]), mb_mask, (int)sm3_iv[0])); + _mm512_storeu_si512(HASH_VALUE(p_state)[1], _mm512_mask_set1_epi32(_mm512_loadu_si512(HASH_VALUE(p_state)[1]), mb_mask, (int)sm3_iv[1])); + _mm512_storeu_si512(HASH_VALUE(p_state)[2], _mm512_mask_set1_epi32(_mm512_loadu_si512(HASH_VALUE(p_state)[2]), mb_mask, (int)sm3_iv[2])); + _mm512_storeu_si512(HASH_VALUE(p_state)[3], _mm512_mask_set1_epi32(_mm512_loadu_si512(HASH_VALUE(p_state)[3]), mb_mask, (int)sm3_iv[3])); + _mm512_storeu_si512(HASH_VALUE(p_state)[4], _mm512_mask_set1_epi32(_mm512_loadu_si512(HASH_VALUE(p_state)[4]), mb_mask, (int)sm3_iv[4])); + _mm512_storeu_si512(HASH_VALUE(p_state)[5], _mm512_mask_set1_epi32(_mm512_loadu_si512(HASH_VALUE(p_state)[5]), mb_mask, (int)sm3_iv[5])); + _mm512_storeu_si512(HASH_VALUE(p_state)[6], _mm512_mask_set1_epi32(_mm512_loadu_si512(HASH_VALUE(p_state)[6]), mb_mask, (int)sm3_iv[6])); + _mm512_storeu_si512(HASH_VALUE(p_state)[7], _mm512_mask_set1_epi32(_mm512_loadu_si512(HASH_VALUE(p_state)[7]), mb_mask, (int)sm3_iv[7])); +} + + +DLL_PUBLIC +mbx_status16 mbx_sm3_init_mb16(SM3_CTX_mb16 * p_state) +{ + mbx_status16 status = 0; + + /* test state pointer */ + if(NULL==p_state) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + sm3_mask_init_mb16(p_state, 0xFFFF); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_init_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_init_mb8.c new file mode 100644 index 0000000..183550f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_init_mb8.c @@ -0,0 +1,62 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include + +void sm3_mask_init_mb8(SM3_CTX_mb8* p_state, __mmask8 mb_mask) +{ + /* clear buffer index */ + _mm256_storeu_si256((__m256i *)HASH_BUFFIDX(p_state), _mm256_mask_set1_epi32(_mm256_loadu_si256((__m256i *)HASH_BUFFIDX(p_state)), mb_mask, 0)); + + /* clear summary message length */ + _mm512_storeu_si512(MSG_LEN(p_state), _mm512_maskz_loadu_epi64(~mb_mask, MSG_LEN(p_state))); + + /* clear buffer */ + for (int i = 0; i < SM3_NUM_BUFFERS8; i++) { + if ((mb_mask >> i) & 1) + _mm512_storeu_si512(HASH_BUFF(p_state)[i], _mm512_setzero_si512()); + } + + /* setup initial digest in multi-buffer format */ + _mm256_storeu_si256((__m256i *)HASH_VALUE(p_state)[0], _mm256_mask_set1_epi32(_mm256_loadu_si256((__m256i *)HASH_VALUE(p_state)[0]), mb_mask, (int)sm3_iv[0])); + _mm256_storeu_si256((__m256i *)HASH_VALUE(p_state)[1], _mm256_mask_set1_epi32(_mm256_loadu_si256((__m256i *)HASH_VALUE(p_state)[1]), mb_mask, (int)sm3_iv[1])); + _mm256_storeu_si256((__m256i *)HASH_VALUE(p_state)[2], _mm256_mask_set1_epi32(_mm256_loadu_si256((__m256i *)HASH_VALUE(p_state)[2]), mb_mask, (int)sm3_iv[2])); + _mm256_storeu_si256((__m256i *)HASH_VALUE(p_state)[3], _mm256_mask_set1_epi32(_mm256_loadu_si256((__m256i *)HASH_VALUE(p_state)[3]), mb_mask, (int)sm3_iv[3])); + _mm256_storeu_si256((__m256i *)HASH_VALUE(p_state)[4], _mm256_mask_set1_epi32(_mm256_loadu_si256((__m256i *)HASH_VALUE(p_state)[4]), mb_mask, (int)sm3_iv[4])); + _mm256_storeu_si256((__m256i *)HASH_VALUE(p_state)[5], _mm256_mask_set1_epi32(_mm256_loadu_si256((__m256i *)HASH_VALUE(p_state)[5]), mb_mask, (int)sm3_iv[5])); + _mm256_storeu_si256((__m256i *)HASH_VALUE(p_state)[6], _mm256_mask_set1_epi32(_mm256_loadu_si256((__m256i *)HASH_VALUE(p_state)[6]), mb_mask, (int)sm3_iv[6])); + _mm256_storeu_si256((__m256i *)HASH_VALUE(p_state)[7], _mm256_mask_set1_epi32(_mm256_loadu_si256((__m256i *)HASH_VALUE(p_state)[7]), mb_mask, (int)sm3_iv[7])); +} + +mbx_status sm3_init_mb8(SM3_CTX_mb8* p_state) +{ + mbx_status status = 0; + + /* test state pointer */ + if(NULL==p_state) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + sm3_mask_init_mb8(p_state, 0xFF); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_messagedigest_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_messagedigest_mb16.c new file mode 100644 index 0000000..155d3ea --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_messagedigest_mb16.c @@ -0,0 +1,58 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include + +DLL_PUBLIC +mbx_status16 mbx_sm3_msg_digest_mb16(const int8u* const msg_pa[16], + int len[16], + int8u* hash_pa[16]) +{ + int buf_no; + mbx_status16 status = 0; + + /* test input pointers */ + if(NULL==msg_pa || NULL==len || NULL==hash_pa) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + for (buf_no = 0; buf_no < SM3_NUM_BUFFERS; buf_no++) { + if ((len[buf_no] && !hash_pa[buf_no]) || (len[buf_no] && !msg_pa[buf_no])) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + return status; + } + } + + /* initialize the context of SM3 hash */ + SM3_CTX_mb16 p_state; + mbx_sm3_init_mb16(&p_state); + + /* process main part of the message */ + status = mbx_sm3_update_mb16(msg_pa, len, &p_state); + + if(MBX_IS_ANY_OK_STS16(status)) { + /* finalize message processing */ + status = mbx_sm3_final_mb16(hash_pa, &p_state); + } + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_messagedigest_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_messagedigest_mb8.c new file mode 100644 index 0000000..93f3e46 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_messagedigest_mb8.c @@ -0,0 +1,55 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include + +mbx_status sm3_msg_digest_mb8(const int8u* const msg_pa[8], int len[8], int8u* hash_pa[8]) +{ + int buf_no; + mbx_status status = 0; + + /* test input pointers */ + if(NULL==msg_pa || NULL==len || NULL==hash_pa) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + for (buf_no = 0; buf_no < SM3_NUM_BUFFERS8; buf_no++) { + if ((len[buf_no] && !hash_pa[buf_no]) || (len[buf_no] && !msg_pa[buf_no])) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + return status; + } + } + + /* initialize the context of SM3 hash */ + SM3_CTX_mb8 p_state; + sm3_init_mb8(&p_state); + + /* process main part of the message */ + status = sm3_update_mb8(msg_pa, len, &p_state); + + if(MBX_IS_ANY_OK_STS(status)) { + /* finalize message processing */ + status = sm3_final_mb8(hash_pa, &p_state); + } + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_update_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_update_mb16.c new file mode 100644 index 0000000..5d34162 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_update_mb16.c @@ -0,0 +1,150 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include + +// Disable optimization for VS17 +#if defined(_MSC_VER) && (_MSC_VER < 1920) && !defined(__INTEL_COMPILER) + #pragma optimize( "", off ) +#endif + +DLL_PUBLIC +mbx_status16 mbx_sm3_update_mb16(const int8u* const msg_pa[16], + int len[16], + SM3_CTX_mb16* p_state) +{ + int i; + mbx_status16 status = 0; + + /* test input pointers */ + if (NULL == msg_pa || NULL == len || NULL == p_state) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + __m512i loc_len = _mm512_loadu_si512(len); + int* p_loc_len = (int*)&loc_len; + + /* generate mask based on array with messages lengths */ + __m512i zero_buffer = _mm512_setzero_si512(); + __mmask16 mb_mask16 = _mm512_cmp_epi32_mask(loc_len, zero_buffer, _MM_CMPINT_NE); + + /* generate mask based on array with pointers to messages */ + __mmask8 mb_mask8[2]; + mb_mask8[0] = _mm512_cmp_epi64_mask(_mm512_loadu_si512(msg_pa), zero_buffer, _MM_CMPINT_NE); + mb_mask8[1] = _mm512_cmp_epi64_mask(_mm512_loadu_si512(msg_pa + 8), zero_buffer, _MM_CMPINT_NE); + + /* don't process the data from i buffer if in msg_pa[i] == 0 or len[i] == 0 */ + mb_mask16 &= *(__mmask16*)mb_mask8; + mb_mask8[0] = (__mmask8)mb_mask16; + mb_mask8[1] = *((__mmask8*)&mb_mask16 + 1); + + /* handle non empty request */ + if (mb_mask16) { + __ALIGN64 const int8u* loc_src[SM3_NUM_BUFFERS]; + _mm512_storeu_si512((void*)loc_src, _mm512_mask_loadu_epi64(_mm512_set1_epi64((long long)&zero_buffer), mb_mask8[0], msg_pa)); + _mm512_storeu_si512((void *)(loc_src + 8), _mm512_mask_loadu_epi64(_mm512_set1_epi64((long long)&zero_buffer), mb_mask8[1], msg_pa + 8)); + + __m512i proc_len; + __m512i idx = _mm512_loadu_si512(HASH_BUFFIDX(p_state)); + + int* p_proc_len = (int*)&proc_len; + int* p_idx = (int*)&idx; + + __ALIGN64 + int64u sum_msg_len[SM3_NUM_BUFFERS] = { (int64u)p_loc_len[0], (int64u)p_loc_len[1], (int64u)p_loc_len[2], (int64u)p_loc_len[3], + (int64u)p_loc_len[4], (int64u)p_loc_len[5], (int64u)p_loc_len[6], (int64u)p_loc_len[7], + (int64u)p_loc_len[8], (int64u)p_loc_len[9], (int64u)p_loc_len[10], (int64u)p_loc_len[11], + (int64u)p_loc_len[12], (int64u)p_loc_len[13], (int64u)p_loc_len[14], (int64u)p_loc_len[15] }; + + __ALIGN64 + int8u* p_buffer[SM3_NUM_BUFFERS] = { HASH_BUFF(p_state)[0], HASH_BUFF(p_state)[1], HASH_BUFF(p_state)[2], HASH_BUFF(p_state)[3], + HASH_BUFF(p_state)[4], HASH_BUFF(p_state)[5], HASH_BUFF(p_state)[6], HASH_BUFF(p_state)[7], + HASH_BUFF(p_state)[8], HASH_BUFF(p_state)[9], HASH_BUFF(p_state)[10], HASH_BUFF(p_state)[11], + HASH_BUFF(p_state)[12], HASH_BUFF(p_state)[13], HASH_BUFF(p_state)[14], HASH_BUFF(p_state)[15] }; + + __mmask16 processed_mask = _mm512_cmp_epi32_mask(idx, zero_buffer, _MM_CMPINT_NE); + + M512(sum_msg_len) = _mm512_mask_add_epi64(M512(sum_msg_len), mb_mask8[0], _mm512_loadu_si512(MSG_LEN(p_state)), M512(sum_msg_len)); + M512(sum_msg_len + 8) = _mm512_mask_add_epi64(M512(sum_msg_len + 8), mb_mask8[1], _mm512_loadu_si512(MSG_LEN(p_state) + 8), M512(sum_msg_len + 8)); + + /* if non empty internal buffer filling */ + if (processed_mask) { + /* calculate how many bytes need to be added in the internal buffer */ + __m512i empty_bytes_buffer = _mm512_sub_epi32(_mm512_set1_epi32(SM3_MSG_BLOCK_SIZE), idx); + processed_mask = _mm512_cmp_epi32_mask(_mm512_sub_epi32(loc_len, empty_bytes_buffer), zero_buffer, _MM_CMPINT_LT); + proc_len = _mm512_mask_loadu_epi32(empty_bytes_buffer, processed_mask, p_loc_len); + + /* copy from valid input streams to the internal buffers as much as possible */ + for (i = 0; i < SM3_NUM_BUFFERS; i++) { + if ((mb_mask16 >> i) & 0x1) { + __mmask64 mb_mask64 = 0xFFFFFFFFFFFFFFFF >> (SM3_MSG_BLOCK_SIZE - p_proc_len[i]); + _mm512_storeu_si512(p_buffer[i] + p_idx[i], _mm512_mask_loadu_epi8(_mm512_loadu_si512(p_buffer[i] + p_idx[i]), mb_mask64, loc_src[i])); + } + } + + idx = _mm512_add_epi32(idx, proc_len); + loc_len = _mm512_sub_epi32(loc_len, proc_len); + + M512(loc_src) = _mm512_add_epi64(M512(loc_src), _mm512_cvtepu32_epi64(M256(p_proc_len))); + M512(loc_src+8) = _mm512_add_epi64(M512(loc_src+8), _mm512_cvtepu32_epi64(M256(p_proc_len+8))); + + processed_mask = _mm512_cmp_epi32_mask(idx, _mm512_set1_epi32(SM3_MSG_BLOCK_SIZE), _MM_CMPINT_EQ); + proc_len = _mm512_maskz_set1_epi32(processed_mask, SM3_MSG_BLOCK_SIZE); + + /* update digest if at least one buffer is full */ + if (processed_mask) { + sm3_avx512_mb16(HASH_VALUE(p_state), (const int8u **)p_buffer, p_proc_len); + idx = _mm512_mask_set1_epi32(idx, ~_mm512_cmp_epi32_mask(proc_len, zero_buffer, _MM_CMPINT_LE), 0); + } + } + + /* main message part processing */ + proc_len = _mm512_and_epi32(loc_len, _mm512_set1_epi32(-SM3_MSG_BLOCK_SIZE)); + processed_mask = _mm512_cmp_epi32_mask(proc_len, zero_buffer, _MM_CMPINT_NLT); + + if (processed_mask) + sm3_avx512_mb16(HASH_VALUE(p_state), loc_src, p_proc_len); + + loc_len = _mm512_sub_epi32(loc_len, proc_len); + + M512(loc_src) = _mm512_add_epi64(M512(loc_src), _mm512_cvtepu32_epi64(M256(p_proc_len))); + M512(loc_src + 8) = _mm512_add_epi64(M512(loc_src + 8), _mm512_cvtepu32_epi64(M256(p_proc_len + 8))); + processed_mask = _mm512_cmp_epi32_mask(loc_len, zero_buffer, _MM_CMPINT_NLE); + + /* store rest of message into the internal buffer */ + for (i = 0; i < SM3_NUM_BUFFERS; i++) { + if ((processed_mask >> i) & 0x1) { + __mmask64 mb_mask64 = ~(0xFFFFFFFFFFFFFFFF << *(p_loc_len + i)); + _mm512_storeu_si512(p_buffer[i], _mm512_maskz_loadu_epi8(mb_mask64, loc_src[i])); + } + } + + idx = _mm512_add_epi32(idx, loc_len); + + /* Update length of processed message */ + _mm512_storeu_si512(MSG_LEN(p_state), _mm512_mask_loadu_epi64(_mm512_loadu_si512(MSG_LEN(p_state)), mb_mask8[0], sum_msg_len)); + _mm512_storeu_si512(MSG_LEN(p_state) + 8, _mm512_mask_loadu_epi64(_mm512_loadu_si512(MSG_LEN(p_state) + 8), mb_mask8[1], sum_msg_len + 8)); + _mm512_storeu_si512(HASH_BUFFIDX(p_state), _mm512_mask_loadu_epi32(_mm512_loadu_si512(HASH_BUFFIDX(p_state)), mb_mask16, p_idx)); + } + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_update_mb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_update_mb8.c new file mode 100644 index 0000000..b30bbae --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm3/sm3_update_mb8.c @@ -0,0 +1,136 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include + +// Disable optimization for VS17 +#if defined(_MSC_VER) && (_MSC_VER < 1920) && !defined(__INTEL_COMPILER) + #pragma optimize( "", off ) +#endif + +mbx_status sm3_update_mb8(const int8u* const msg_pa[8], int len[8], SM3_CTX_mb8* p_state) +{ + int i; + mbx_status status = 0; + + /* test input pointers */ + if (NULL == msg_pa || NULL == len || NULL == p_state) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + __ALIGN64 const int8u* loc_src[SM3_NUM_BUFFERS8]; + + __m256i loc_len = _mm256_loadu_si256((__m256i*)len); + int* p_loc_len = (int*)&loc_len; + + __m512i zero_buffer = _mm512_setzero_si512(); + + /* generate mask based on array with messages lengths */ + __mmask8 mb_mask = _mm256_cmp_epi32_mask(loc_len, M256(&zero_buffer), _MM_CMPINT_NE); + + /* generate mask based on msg_pa[]. Don't process the data from i buffer if in msg_pa[i] == 0 or len[i] == 0 */ + mb_mask &= _mm512_cmp_epi64_mask(_mm512_loadu_si512(msg_pa), zero_buffer, _MM_CMPINT_NE); + + /* handle non empty message */ + if (mb_mask) { + _mm512_storeu_si512((void *)loc_src, _mm512_mask_loadu_epi64(_mm512_set1_epi64((long long)&zero_buffer), mb_mask, msg_pa)); + + __m256i proc_len; + __m256i idx = _mm256_loadu_si256((__m256i*)HASH_BUFFIDX(p_state)); + + int* p_proc_len = (int*)&proc_len; + int* p_idx = (int*)&idx; + + __ALIGN64 + int64u sum_msg_len[SM3_NUM_BUFFERS8] = { (int64u)p_loc_len[0], (int64u)p_loc_len[1], (int64u)p_loc_len[2], (int64u)p_loc_len[3], + (int64u)p_loc_len[4], (int64u)p_loc_len[5], (int64u)p_loc_len[6], (int64u)p_loc_len[7] }; + + __ALIGN64 + int8u* p_buffer[SM3_NUM_BUFFERS8] = { HASH_BUFF(p_state)[0], HASH_BUFF(p_state)[1], HASH_BUFF(p_state)[2], HASH_BUFF(p_state)[3], + HASH_BUFF(p_state)[4], HASH_BUFF(p_state)[5], HASH_BUFF(p_state)[6], HASH_BUFF(p_state)[7] }; + + __mmask8 processed_mask = _mm256_cmp_epi32_mask(idx, M256(&zero_buffer), _MM_CMPINT_NE); + + M512(sum_msg_len) = _mm512_mask_add_epi64(M512(sum_msg_len), mb_mask, _mm512_loadu_si512(MSG_LEN(p_state)), M512(sum_msg_len)); + + /* if non empty internal buffer filling */ + if (processed_mask) { + + __m256i tmp = _mm256_sub_epi32(_mm256_set1_epi32(SM3_MSG_BLOCK_SIZE), idx); + processed_mask = _mm256_cmp_epi32_mask(_mm256_sub_epi32(loc_len, tmp), M256(&zero_buffer), _MM_CMPINT_LT); + + /* p_proc_len[i] = MIN(p_loc_len[i], (SM3_MSG_BLOCK_SIZE - p_idx[i])) */ + proc_len = _mm256_mask_loadu_epi32(tmp, processed_mask, p_loc_len); + + /* copy from input stream to the internal buffer as match as possible */ + for (i = 0; i < SM3_NUM_BUFFERS8; i++) { + /* copy from input stream to the internal buffer as match as possible */ + __mmask64 mb_mask64 = ~(0xFFFFFFFFFFFFFFFF << p_proc_len[i]); + _mm512_storeu_si512(p_buffer[i] + p_idx[i], _mm512_mask_loadu_epi8(_mm512_loadu_si512(p_buffer[i] + p_idx[i]), mb_mask64, loc_src[i])); + } + + idx = _mm256_add_epi32(idx, proc_len); + loc_len = _mm256_sub_epi32(loc_len, proc_len); + + M512(loc_src) = _mm512_add_epi64(M512(loc_src), _mm512_cvtepu32_epi64(M256(p_proc_len))); + + processed_mask = _mm256_cmp_epi32_mask(idx, _mm256_set1_epi32(SM3_MSG_BLOCK_SIZE), _MM_CMPINT_EQ); + proc_len = _mm256_mask_set1_epi32(proc_len, processed_mask, SM3_MSG_BLOCK_SIZE); + + /* update digest if at least one buffer is full */ + if (processed_mask) { + sm3_avx512_mb8(HASH_VALUE(p_state), (const int8u **)p_buffer, p_proc_len); + idx = _mm256_mask_set1_epi32(idx, ~_mm256_cmp_epi32_mask(proc_len, M256(&zero_buffer), 2), _MM_CMPINT_EQ); + } + } + + /* main message part processing */ + proc_len = _mm256_and_si256(loc_len, _mm256_set1_epi32(-SM3_MSG_BLOCK_SIZE)); + processed_mask = _mm256_cmp_epi32_mask(proc_len, M256(&zero_buffer), _MM_CMPINT_NLT); + + if (processed_mask) + sm3_avx512_mb8(HASH_VALUE(p_state), loc_src, p_proc_len); + + loc_len = _mm256_sub_epi32(loc_len, proc_len); + + M512(loc_src) = _mm512_add_epi64(M512(loc_src), _mm512_cvtepu32_epi64(M256(p_proc_len))); + + processed_mask = _mm256_cmp_epi32_mask(loc_len, M256(&zero_buffer), _MM_CMPINT_NLE); + + /* store rest of message into the internal buffer */ + if (processed_mask) { + for (i = 0; i < SM3_NUM_BUFFERS8; i++) { + /* copy from input stream to the internal buffer as match as possible */ + __mmask64 mb_mask64 = ~(0xFFFFFFFFFFFFFFFF << *(p_loc_len + i)); + _mm512_storeu_si512(p_buffer[i], _mm512_maskz_loadu_epi8(mb_mask64, loc_src[i])); + } + + idx = _mm256_maskz_add_epi32(0xFF, idx, loc_len); + } + + /* Update length of processed message */ + _mm512_storeu_si512(MSG_LEN(p_state), _mm512_mask_loadu_epi64(_mm512_loadu_si512(MSG_LEN(p_state)), mb_mask, sum_msg_len)); + _mm512_storeu_si512(HASH_BUFFIDX(p_state), _mm512_mask_loadu_epi32(_mm512_loadu_si512(HASH_BUFFIDX(p_state)), mb_mask, p_idx)); + } + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/internal/sm4_ccm_decrypt_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/internal/sm4_ccm_decrypt_mb16.c new file mode 100644 index 0000000..ac84db6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/internal/sm4_ccm_decrypt_mb16.c @@ -0,0 +1,91 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include +#include + +/* + * This function performs decryption with CTR and then authentication with CBC-MAC on plaintext. + * Function returns mask where bit is set to 1 if length of given data for buffer is overflowed. +*/ + +void sm4_ccm_decrypt_mb16(int8u *pa_out[SM4_LINES], + const int8u *const pa_in[SM4_LINES], + const int in_len[SM4_LINES], + __mmask16 mb_mask, + SM4_CCM_CTX_mb16 *p_context) +{ + __m128i *hash = SM4_CCM_CONTEXT_HASH(p_context); + __m128i *ctr = SM4_CCM_CONTEXT_CTR(p_context); + + const int8u *hash_ptrs[SM4_LINES]; + int8u *pa_ctr[SM4_LINES]; + unsigned i; + int full_hash_len[SM4_LINES]; + int partial_hash_len[SM4_LINES]; + int8u padded_hash[SM4_LINES][SM4_BLOCK_SIZE]; + int8u *pa_padded_hash[SM4_LINES]; + __mmask16 partial_block_mask = 0; + + /* No AAD processed */ + if (SM4_CCM_CONTEXT_STATE(p_context) == sm4_ccm_update_aad) { + int aad_lens[SM4_LINES]; + + PadBlock(0, aad_lens, sizeof(aad_lens)); + sm4_ccm_update_aad_mb16(NULL, aad_lens, 0xFFFF, p_context); + } + + /* Switch context state to decryption */ + SM4_CCM_CONTEXT_STATE(p_context) = sm4_ccm_dec; + + for (i = 0; i < SM4_LINES; i++) { + hash_ptrs[i] = (int8u *) &hash[i]; + pa_ctr[i] = (int8u *) &ctr[i]; + full_hash_len[i] = in_len[i] & 0xFFFFFFF0; + } + + /* Decrypt first */ + sm4_ctr128_kernel_mb16(pa_out, (const int8u **) pa_in, in_len, (const int32u **)SM4_CCM_CONTEXT_KEY(p_context), mb_mask, pa_ctr); + + /* Authenticate the plaintext */ + sm4_cbc_mac_kernel_mb16(hash, (const int8u *const *) pa_out, + full_hash_len, + (const int32u **)SM4_CCM_CONTEXT_KEY(p_context), + mb_mask, + hash_ptrs); + + /* Handle partial blocks of plaintext */ + for (i = 0; i < SM4_LINES; i++) { + partial_hash_len[i] = in_len[i] & 0xF; + if (partial_hash_len[i] == 0) + continue; + pa_padded_hash[i] = padded_hash[i]; + PadBlock(0, pa_padded_hash[i], SM4_BLOCK_SIZE); + CopyBlock(pa_out[i] + full_hash_len[i], pa_padded_hash[i], partial_hash_len[i]); + full_hash_len[i] = SM4_BLOCK_SIZE; + partial_block_mask |= (1 << i); + } + if (partial_block_mask != 0) { + sm4_cbc_mac_kernel_mb16(hash, (const int8u *const *) pa_padded_hash, + full_hash_len, + (const int32u **)SM4_CCM_CONTEXT_KEY(p_context), + partial_block_mask, + hash_ptrs); + SM4_CCM_CONTEXT_STATE(p_context) = sm4_ccm_get_tag; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/internal/sm4_ccm_encrypt_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/internal/sm4_ccm_encrypt_mb16.c new file mode 100644 index 0000000..01bd604 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/internal/sm4_ccm_encrypt_mb16.c @@ -0,0 +1,86 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include +#include + +/* + * This function performs authentication with CBC-MAC on plaintext and encryption with CTR. +*/ + +void sm4_ccm_encrypt_mb16(int8u *pa_out[SM4_LINES], + const int8u *const pa_in[SM4_LINES], + const int in_len[SM4_LINES], + __mmask16 mb_mask, + SM4_CCM_CTX_mb16 *p_context) +{ + __m128i *hash = SM4_CCM_CONTEXT_HASH(p_context); + __m128i *ctr = SM4_CCM_CONTEXT_CTR(p_context); + + const int8u *hash_ptrs[SM4_LINES]; + int8u *pa_ctr[SM4_LINES]; + unsigned i; + int full_hash_len[SM4_LINES]; + int partial_hash_len[SM4_LINES]; + int8u padded_hash[SM4_LINES][SM4_BLOCK_SIZE]; + int8u *pa_padded_hash[SM4_LINES]; + __mmask16 partial_block_mask = 0; + + /* No AAD processed */ + if (SM4_CCM_CONTEXT_STATE(p_context) == sm4_ccm_update_aad) { + int aad_lens[SM4_LINES]; + + PadBlock(0, aad_lens, sizeof(aad_lens)); + sm4_ccm_update_aad_mb16(NULL, aad_lens, 0xFFFF, p_context); + } + + /* Switch context state to encryption */ + SM4_CCM_CONTEXT_STATE(p_context) = sm4_ccm_enc; + + for (i = 0; i < SM4_LINES; i++) { + hash_ptrs[i] = (int8u *) &hash[i]; + pa_ctr[i] = (int8u *) &ctr[i]; + full_hash_len[i] = in_len[i] & 0xFFFFFFF0; /* Get full block lengths */ + } + sm4_cbc_mac_kernel_mb16(hash, (const int8u *const *) pa_in, + full_hash_len, + (const int32u **)SM4_CCM_CONTEXT_KEY(p_context), + mb_mask, + hash_ptrs); + + /* Handle partial blocks of plaintext */ + for (i = 0; i < SM4_LINES; i++) { + partial_hash_len[i] = in_len[i] & 0xF; /* Retrieve partial block lengths */ + if (partial_hash_len[i] == 0) + continue; + pa_padded_hash[i] = padded_hash[i]; + PadBlock(0, pa_padded_hash[i], SM4_BLOCK_SIZE); + CopyBlock(pa_in[i] + full_hash_len[i], pa_padded_hash[i], partial_hash_len[i]); + full_hash_len[i] = SM4_BLOCK_SIZE; + partial_block_mask |= (1 << i); + } + if (partial_block_mask != 0) { + sm4_cbc_mac_kernel_mb16(hash, (const int8u *const *) pa_padded_hash, + full_hash_len, + (const int32u **)SM4_CCM_CONTEXT_KEY(p_context), + partial_block_mask, + hash_ptrs); + SM4_CCM_CONTEXT_STATE(p_context) = sm4_ccm_get_tag; + } + sm4_ctr128_kernel_mb16(pa_out, (const int8u **) pa_in, in_len, (const int32u **)SM4_CCM_CONTEXT_KEY(p_context), mb_mask, pa_ctr); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/internal/sm4_ccm_get_tag_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/internal/sm4_ccm_get_tag_mb16.c new file mode 100644 index 0000000..5993fcb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/internal/sm4_ccm_get_tag_mb16.c @@ -0,0 +1,102 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include +#include /* for zero_mb8 */ + +static void sm4_encrypt_ctr0_mb16(SM4_CCM_CTX_mb16 *p_context, __m512i *s0_blocks) +{ + const mbx_sm4_key_schedule *key_sched = (const mbx_sm4_key_schedule *)SM4_CCM_CONTEXT_KEY(p_context); + + __m128i *ctr0 = SM4_CCM_CONTEXT_CTR0(p_context); + + const int8u *pa_inp[SM4_LINES]; + + for (int i = 0; i < SM4_LINES; i++) + pa_inp[i] = (unsigned char *)(ctr0 + i); + + TRANSPOSE_16x4_I32_EPI32(&s0_blocks[0], &s0_blocks[1], &s0_blocks[2], &s0_blocks[3], pa_inp, 0xFFFF); + + const __m512i *p_rk = (const __m512i *)key_sched; + + __m512i tmp; + for (int itr = 0; itr < SM4_ROUNDS; itr += 4, p_rk += 4) + SM4_FOUR_ROUNDS(s0_blocks[0], s0_blocks[1], s0_blocks[2], s0_blocks[3], tmp, p_rk, 1); + + __m512i T1_0 = unpacklo_epi32(s0_blocks[0], s0_blocks[1]); + __m512i T1_1 = unpackhi_epi32(s0_blocks[0], s0_blocks[1]); + __m512i T1_2 = unpacklo_epi32(s0_blocks[2], s0_blocks[3]); + __m512i T1_3 = unpackhi_epi32(s0_blocks[2], s0_blocks[3]); + + s0_blocks[0] = unpacklo_epi64(T1_0, T1_2); + s0_blocks[1] = unpackhi_epi64(T1_0, T1_2); + s0_blocks[2] = unpacklo_epi64(T1_1, T1_3); + s0_blocks[3] = unpackhi_epi64(T1_1, T1_3); + + s0_blocks[0] = shuffle_epi8(s0_blocks[0], M512(swapEndianness)); + s0_blocks[1] = shuffle_epi8(s0_blocks[1], M512(swapEndianness)); + s0_blocks[2] = shuffle_epi8(s0_blocks[2], M512(swapEndianness)); + s0_blocks[3] = shuffle_epi8(s0_blocks[3], M512(swapEndianness)); + + T1_0 = _mm512_shuffle_i64x2(s0_blocks[0], s0_blocks[1], 0x44); + T1_1 = _mm512_shuffle_i64x2(s0_blocks[0], s0_blocks[1], 0xEE); + T1_2 = _mm512_shuffle_i64x2(s0_blocks[2], s0_blocks[3], 0x44); + T1_3 = _mm512_shuffle_i64x2(s0_blocks[2], s0_blocks[3], 0xEE); + + s0_blocks[0] = _mm512_shuffle_i64x2(T1_0, T1_2, 0x88); + s0_blocks[1] = _mm512_shuffle_i64x2(T1_0, T1_2, 0xDD); + s0_blocks[2] = _mm512_shuffle_i64x2(T1_1, T1_3, 0x88); + s0_blocks[3] = _mm512_shuffle_i64x2(T1_1, T1_3, 0xDD); +} + +void sm4_ccm_get_tag_mb16(int8u *pa_out[SM4_LINES], const int tag_len[SM4_LINES], __mmask16 mb_mask, SM4_CCM_CTX_mb16 *p_context) +{ + __m512i s0_blocks[4]; + __m512i hash_blocks[4]; + + __m128i *hash = SM4_CCM_CONTEXT_HASH(p_context); + + /* Calculate S0 */ + sm4_encrypt_ctr0_mb16(p_context, s0_blocks); + + hash_blocks[0] = loadu(hash); + hash_blocks[1] = loadu(hash + 4); + hash_blocks[2] = loadu(hash + 4*2); + hash_blocks[3] = loadu(hash + 4*3); + + __m512i tag_blocks[4]; + + /* XOR with previously encrypted J0 */ + tag_blocks[0] = xor(hash_blocks[0], s0_blocks[0]); + tag_blocks[1] = xor(hash_blocks[1], s0_blocks[1]); + tag_blocks[2] = xor(hash_blocks[2], s0_blocks[2]); + tag_blocks[3] = xor(hash_blocks[3], s0_blocks[3]); + + /* Store result */ + for (int i = 0; i < SM4_LINES; i++) { + __m128i one_block = M128((__m128i *)tag_blocks + i); + + __mmask16 tagMask = ~(0xFFFF << (tag_len[i])) * ((mb_mask >> i) & 0x1); + _mm_mask_storeu_epi8((void *)(pa_out[i]), tagMask, one_block); + } + + /* Clear local copy of sensitive data */ + zero_mb8((int64u(*)[8])tag_blocks, sizeof(tag_blocks) / sizeof(tag_blocks[0])); + zero_mb8((int64u(*)[8])hash_blocks, sizeof(hash_blocks) / sizeof(hash_blocks[0])); + zero_mb8((int64u(*)[8])s0_blocks, sizeof(s0_blocks) / sizeof(s0_blocks[0])); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/internal/sm4_ccm_set_msg_len_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/internal/sm4_ccm_set_msg_len_mb16.c new file mode 100644 index 0000000..644c78a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/internal/sm4_ccm_set_msg_len_mb16.c @@ -0,0 +1,28 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include + +void sm4_ccm_set_msg_len_mb16(const int64u msg_len[SM4_LINES], __mmask16 mb_mask, SM4_CCM_CTX_mb16 *p_context) +{ + int64u *msg_len_ctx = SM4_CCM_CONTEXT_MSG_LEN(p_context); + unsigned i; + + for (i = 0; i < SM4_LINES; i++) + if (mb_mask & (1 << i)) + msg_len_ctx[i] = msg_len[i]; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/internal/sm4_ccm_set_tag_len_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/internal/sm4_ccm_set_tag_len_mb16.c new file mode 100644 index 0000000..d0ea494 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/internal/sm4_ccm_set_tag_len_mb16.c @@ -0,0 +1,30 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include + +void sm4_ccm_set_tag_len_mb16(const int tag_len[SM4_LINES], __mmask16 mb_mask, SM4_CCM_CTX_mb16 *p_context) +{ + int *tag_len_ctx = SM4_CCM_CONTEXT_TAG_LEN(p_context); + unsigned i; + + for (i = 0; i < SM4_LINES; i++) { + if (mb_mask & (1 << i)) { + tag_len_ctx[i] = tag_len[i]; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/internal/sm4_ccm_update_aad_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/internal/sm4_ccm_update_aad_mb16.c new file mode 100644 index 0000000..fd6f05d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/internal/sm4_ccm_update_aad_mb16.c @@ -0,0 +1,161 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include +#include + +#define MAX_AAD_SIZE_BLOCKS_0_3 46 +/* + * This function process 16 buffers with additional authentication data (AAD), + * up to 2^16 - 2^8 bytes +*/ + +void sm4_ccm_update_aad_mb16(const int8u *const pa_aad[SM4_LINES], + const int aad_len[SM4_LINES], + __mmask16 mb_mask, + SM4_CCM_CTX_mb16 *p_context) +{ + __m128i *ctr0 = SM4_CCM_CONTEXT_CTR0(p_context); + int *tag_len = SM4_CCM_CONTEXT_TAG_LEN(p_context); + int *iv_len = SM4_CCM_CONTEXT_IV_LEN(p_context); + int64u *msg_len = SM4_CCM_CONTEXT_MSG_LEN(p_context); + __m128i *hash = SM4_CCM_CONTEXT_HASH(p_context); + /* Scratch memory to construct up to 4 blocks to authenticate (up to 46 byte of AAD) */ + int8u tmp[SM4_LINES][SM4_BLOCK_SIZE*4]; + const int8u *block_ptrs[SM4_LINES]; + const int8u *hash_ptrs[SM4_LINES]; + int auth_lens[SM4_LINES]; + unsigned i, j; + int8u iv[SM4_LINES][SM4_BLOCK_SIZE]; + const int8u *iv_ptrs[SM4_LINES]; + int additional_lens[SM4_LINES]; + + __m512i max_msg16b_len = set1_epi64(0xFFFF); + __mmask16 additional_len_mask = _mm512_cmp_epi32_mask(set1_epi32(MAX_AAD_SIZE_BLOCKS_0_3), loadu(aad_len), _MM_CMPINT_LT); + __mmask8 msg_len_overflow_lo_mask = _mm512_cmp_epi64_mask(max_msg16b_len, loadu(msg_len), _MM_CMPINT_LE); + __mmask8 msg_len_overflow_hi_mask = _mm512_cmp_epi64_mask(max_msg16b_len, loadu(msg_len + 8), _MM_CMPINT_LE); + __mmask16 msg_len_overflow_mask = (__mmask16) msg_len_overflow_hi_mask << 8 | (__mmask16) msg_len_overflow_lo_mask; + + additional_len_mask = _mm512_kand(additional_len_mask, mb_mask); + + PadBlock(0, tmp, sizeof(tmp)); + PadBlock(0, iv, sizeof(iv)); + + for (i = 0; i < SM4_LINES; i++) { + + additional_lens[i] = 0; + + if ((mb_mask & (1 << i)) == 0) { + auth_lens[i] = 0; + block_ptrs[i] = NULL; + hash_ptrs[i] = NULL; + iv_ptrs[i] = NULL; + continue; + } + + CopyBlock(&ctr0[i], tmp[i], SM4_BLOCK_SIZE); + + int8u flags = tmp[i][0]; + + int8u tag_len_enc = (tag_len[i] - 2) >> 1; + + flags |= (tag_len_enc) << 3; + auth_lens[i] = SM4_BLOCK_SIZE; + block_ptrs[i] = tmp[i]; + hash_ptrs[i] = (int8u *) &hash[i]; + iv_ptrs[i] = iv[i]; + if (aad_len[i]) { + int len; + + if (aad_len[i] > MAX_AAD_SIZE_BLOCKS_0_3) { + len = MAX_AAD_SIZE_BLOCKS_0_3; + additional_lens[i] = aad_len[i] - MAX_AAD_SIZE_BLOCKS_0_3; + } else + len = aad_len[i]; + flags |= 1 << 6; + /* Copy AAD length to first 2 bytes of B_1 */ + tmp[i][16] = (int8u) (aad_len[i] >> 8); + tmp[i][17] = (int8u) aad_len[i]; + /* Copy AAD afterwards */ + CopyBlock(pa_aad[i], &tmp[i][18], len); + auth_lens[i] += (2 + 15 + len); + auth_lens[i] &= 0xfff0; /* Multiple of 16 bytes */ + } + tmp[i][0] = flags; + tmp[i][15] = (int8u) msg_len[i]; + tmp[i][14] = (int8u) (msg_len[i] >> 8); + } + + /* Check if message is longer than 2^16 - 1 and set the length appropriately in block 0 */ + if (msg_len_overflow_mask) { + for (i = 0; i < SM4_LINES; i++) { + const unsigned num_bits_msg_len = 15 - iv_len[i]; + const int64u max_len = (num_bits_msg_len == 8) ? 0xFFFFFFFFFFFFFFFF : + (1ULL << (num_bits_msg_len << 3)); + + if (msg_len[i] < max_len) { + for (j = 2; j < num_bits_msg_len; j++) + tmp[i][15-j] = (int8u) (msg_len[i] >> (8*j)); + } + } + } + sm4_cbc_mac_kernel_mb16(hash, (const int8u *const *) block_ptrs, + auth_lens, + (const int32u **)SM4_CCM_CONTEXT_KEY(p_context), + mb_mask, iv_ptrs); + + /* More AAD to process */ + if (additional_len_mask) { + for (i = 0; i < SM4_LINES; i++) { + if (additional_lens[i]) { + block_ptrs[i] = pa_aad[i] + MAX_AAD_SIZE_BLOCKS_0_3; /* First 46 bytes have been processed */ + auth_lens[i] = additional_lens[i] & 0xfff0; /* Multiple of 16 bytes */ + additional_lens[i] -= auth_lens[i]; + if (auth_lens[i] == 0) + additional_len_mask &= ~(1 << i); + } + } + + sm4_cbc_mac_kernel_mb16(hash, (const int8u *const *) block_ptrs, + auth_lens, + (const int32u **)SM4_CCM_CONTEXT_KEY(p_context), + additional_len_mask, hash_ptrs); + + additional_len_mask = _mm512_cmp_epi32_mask(set1_epi32(0), + loadu(additional_lens), + _MM_CMPINT_NE); + /* Process last blocks (up to 15 bytes) */ + for (i = 0; i < SM4_LINES; i++) { + if (additional_lens[i]) { + block_ptrs[i] = tmp[i]; /* First 46 bytes have been processed */ + PadBlock(0, tmp[i], SM4_BLOCK_SIZE); + CopyBlock(pa_aad[i] + MAX_AAD_SIZE_BLOCKS_0_3 + auth_lens[i], tmp[i], + additional_lens[i]); + auth_lens[i] = SM4_BLOCK_SIZE; + } + } + sm4_cbc_mac_kernel_mb16(hash, (const int8u *const *) block_ptrs, + auth_lens, + (const int32u **)SM4_CCM_CONTEXT_KEY(p_context), + additional_len_mask, hash_ptrs); + + } + + if (cmp_epi32_mask(loadu(aad_len), setzero(), _MM_CMPINT_EQ) != 0xFFFF) + SM4_CCM_CONTEXT_STATE(p_context) = sm4_ccm_start_encdec; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/internal/sm4_ccm_update_iv_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/internal/sm4_ccm_update_iv_mb16.c new file mode 100644 index 0000000..796441e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/internal/sm4_ccm_update_iv_mb16.c @@ -0,0 +1,55 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include + +/* +// This function process 16 buffers with initialization vector (IV) data +*/ +void sm4_ccm_update_iv_mb16(const int8u *const pa_iv[SM4_LINES], const int iv_len[SM4_LINES], __mmask16 mb_mask, SM4_CCM_CTX_mb16 *p_context) +{ + int *iv_len_ctx = SM4_CCM_CONTEXT_IV_LEN(p_context); + __m128i *ctr0 = SM4_CCM_CONTEXT_CTR0(p_context); + __m128i *ctr = SM4_CCM_CONTEXT_CTR(p_context); + + unsigned i; + + for (i = 0; i < SM4_LINES; i++) { + if (mb_mask & (1 << i)) { + int8u flags; + unsigned L; + int iv_len_i = iv_len[i]; + + L = 15 - iv_len_i; + flags = L - 1; + /* Update CTR0 for each lane */ + int8u *ctr0_nonce_ptr = (int8u *) &(ctr0[i]); + PadBlock(0, ctr0_nonce_ptr, 16); + CopyBlock(pa_iv[i], ctr0_nonce_ptr + 1, iv_len_i); + ctr0_nonce_ptr[0] = flags; + + /* Update CTR for each lane (counter = 1 to start encryption) */ + int8u *ctr_nonce_ptr = (int8u *) &(ctr[i]); + PadBlock(0, ctr_nonce_ptr, 16); + CopyBlock(pa_iv[i], ctr_nonce_ptr + 1, iv_len_i); + ctr_nonce_ptr[0] = flags; + ctr_nonce_ptr[15] = 1; + iv_len_ctx[i] = iv_len[i]; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/sm4_ccm_decrypt_mb16_api.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/sm4_ccm_decrypt_mb16_api.c new file mode 100644 index 0000000..ab02511 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/sm4_ccm_decrypt_mb16_api.c @@ -0,0 +1,70 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include + +DLL_PUBLIC +mbx_status16 +mbx_sm4_ccm_decrypt_mb16(int8u *pa_out[SM4_LINES], const int8u *const pa_in[SM4_LINES], const int in_len[SM4_LINES], SM4_CCM_CTX_mb16 *p_context) +{ + int buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + int64u *processed_len = SM4_CCM_CONTEXT_PROCESSED_LEN(p_context); + int64u *msg_len = SM4_CCM_CONTEXT_MSG_LEN(p_context); + + /* Test input pointers */ + if (NULL == pa_out || NULL == pa_in || NULL == in_len || NULL == p_context) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* Check state */ + if (sm4_ccm_update_aad != SM4_CCM_CONTEXT_STATE(p_context) && + sm4_ccm_start_encdec != SM4_CCM_CONTEXT_STATE(p_context) && + sm4_ccm_dec != SM4_CCM_CONTEXT_STATE(p_context)) { + + status = MBX_SET_STS16_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + /* Don't process buffers with NULL pointers or wrong input length */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_out[buf_no] == NULL || pa_in[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + mb_mask &= ~(0x1 << buf_no); + continue; + } + + if (in_len[buf_no] < 0) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + mb_mask &= ~(0x1 << buf_no); + continue; + } + processed_len[buf_no] += in_len[buf_no]; + /* Check if total processed length will exceed the total message length passed at init */ + if (processed_len[buf_no] > msg_len[buf_no]) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + mb_mask &= ~(0x1 << buf_no); + } + } + + if (MBX_IS_ANY_OK_STS16(status)) + sm4_ccm_decrypt_mb16(pa_out, pa_in, in_len, mb_mask, p_context); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/sm4_ccm_encrypt_mb16_api.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/sm4_ccm_encrypt_mb16_api.c new file mode 100644 index 0000000..c1976a2 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/sm4_ccm_encrypt_mb16_api.c @@ -0,0 +1,72 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include + +DLL_PUBLIC +mbx_status16 +mbx_sm4_ccm_encrypt_mb16(int8u *pa_out[SM4_LINES], const int8u *const pa_in[SM4_LINES], const int in_len[SM4_LINES], SM4_CCM_CTX_mb16 *p_context) +{ + int buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + int64u *processed_len = SM4_CCM_CONTEXT_PROCESSED_LEN(p_context); + int64u *msg_len = SM4_CCM_CONTEXT_MSG_LEN(p_context); + + /* Test input pointers */ + if (NULL == pa_out || NULL == pa_in || NULL == in_len || NULL == p_context) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* Check state */ + if (sm4_ccm_update_aad != SM4_CCM_CONTEXT_STATE(p_context) && + sm4_ccm_start_encdec != SM4_CCM_CONTEXT_STATE(p_context) && + sm4_ccm_enc != SM4_CCM_CONTEXT_STATE(p_context)) { + + status = MBX_SET_STS16_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* Don't process buffers with NULL pointers or invalid lengths */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_out[buf_no] == NULL || pa_in[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + mb_mask &= ~(0x1 << buf_no); + continue; + } + + if (in_len[buf_no] < 0) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + mb_mask &= ~(0x1 << buf_no); + continue; + } + + processed_len[buf_no] += in_len[buf_no]; + /* Check if total processed length will exceed the total message length passed at init */ + if (processed_len[buf_no] > msg_len[buf_no]) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + mb_mask &= ~(0x1 << buf_no); + } + } + + if (MBX_IS_ANY_OK_STS16(status)) + sm4_ccm_encrypt_mb16(pa_out, pa_in, in_len, mb_mask, p_context); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/sm4_ccm_get_tag_mb16_api.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/sm4_ccm_get_tag_mb16_api.c new file mode 100644 index 0000000..24a9b95 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/sm4_ccm_get_tag_mb16_api.c @@ -0,0 +1,69 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include + +DLL_PUBLIC +mbx_status16 mbx_sm4_ccm_get_tag_mb16(int8u *pa_tag[SM4_LINES], const int tag_len[SM4_LINES], SM4_CCM_CTX_mb16 *p_context) +{ + int buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + int64u *processed_len = SM4_CCM_CONTEXT_PROCESSED_LEN(p_context); + int64u *msg_len = SM4_CCM_CONTEXT_MSG_LEN(p_context); + + /* Test input pointers */ + if (NULL == pa_tag || NULL == tag_len || NULL == p_context) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* Check state */ + if (sm4_ccm_update_aad != SM4_CCM_CONTEXT_STATE(p_context) && sm4_ccm_start_encdec != SM4_CCM_CONTEXT_STATE(p_context) && + sm4_ccm_enc != SM4_CCM_CONTEXT_STATE(p_context) && sm4_ccm_dec != SM4_CCM_CONTEXT_STATE(p_context) && + sm4_ccm_get_tag != SM4_CCM_CONTEXT_STATE(p_context)) { + + status = MBX_SET_STS16_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* Don't process buffers with input pointers equal to zero and set bad status for tags of invalid length */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_tag[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + mb_mask &= ~(0x1 << buf_no); + } else { + if (tag_len[buf_no] < 4 || tag_len[buf_no] > 16 || tag_len[buf_no] & 0x1) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + mb_mask &= ~(0x1 << buf_no); + } else { + /* Check if total processed length is not equal to total message length passed at init */ + if (processed_len[buf_no] != msg_len[buf_no]) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + mb_mask &= ~(0x1 << buf_no); + } + } + } + } + + if (MBX_IS_ANY_OK_STS16(status)) { + sm4_ccm_get_tag_mb16(pa_tag, tag_len, mb_mask, p_context); + } + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/sm4_ccm_init_mb16_api.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/sm4_ccm_init_mb16_api.c new file mode 100644 index 0000000..9da9225 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/sm4_ccm_init_mb16_api.c @@ -0,0 +1,106 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include +#include + +DLL_PUBLIC +mbx_status16 mbx_sm4_ccm_init_mb16(const sm4_key *const pa_key[SM4_LINES], + const int8u *const pa_iv[SM4_LINES], + const int iv_len[SM4_LINES], + const int tag_len[SM4_LINES], + const int64u msg_len[SM4_LINES], + SM4_CCM_CTX_mb16 *p_context) +{ + int buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + + /* Test input pointers */ + if (NULL == pa_key || NULL == pa_iv || NULL == iv_len || + NULL == tag_len || NULL == msg_len || NULL == p_context) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* Don't process buffers with input pointers equal to zero and set bad status for IV with zero length */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_key[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + mb_mask &= ~(0x1 << buf_no); + continue; + } + if (pa_iv[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + mb_mask &= ~(0x1 << buf_no); + continue; + } + if ((iv_len[buf_no] < MIN_CCM_IV_LENGTH || iv_len[buf_no] > MAX_CCM_IV_LENGTH)) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + mb_mask &= ~(0x1 << buf_no); + continue; + } + if ((tag_len[buf_no] < MIN_CCM_TAG_LENGTH) || (tag_len[buf_no] > MAX_CCM_TAG_LENGTH) || + (tag_len[buf_no] & 0x1)) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + mb_mask &= ~(0x1 << buf_no); + continue; + } + + /* Check maximum message length allowed, given the number of bytes to encode message length */ + int q = 15 - iv_len[buf_no]; + int64u max_len = (q == 8) ? 0xFFFFFFFFFFFFFFFF : ((1ULL << (q << 3)) - 1); /* (2^(q * 8) - 1 */ + + if (msg_len[buf_no] > max_len) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + mb_mask &= ~(0x1 << buf_no); + } + } + + if (MBX_IS_ANY_OK_STS16(status)) { + + /* + // Compute SM4 keys + // initialize int32u mbx_sm4_key_schedule[SM4_ROUNDS][SM4_LINES] buffer in context + // keys layout for each round: + // key0 key4 key8 key12 key1 key5 key9 key13 key2 key6 key10 key14 key3 key7 key11 key15 + */ + + sm4_set_round_keys_mb16((int32u **)SM4_CCM_CONTEXT_KEY(p_context), (const int8u **)pa_key, mb_mask); + + /* Process IV */ + sm4_ccm_update_iv_mb16(pa_iv, iv_len, mb_mask, p_context); + + /* Zero initial msg and tag lengths */ + PadBlock(0, SM4_CCM_CONTEXT_MSG_LEN(p_context), sizeof(int64u)*SM4_LINES); + PadBlock(0, SM4_CCM_CONTEXT_PROCESSED_LEN(p_context), sizeof(int64u)*SM4_LINES); + PadBlock(0, SM4_CCM_CONTEXT_TAG_LEN(p_context), sizeof(int)*SM4_LINES); + + /* Process msg and tag lengths */ + sm4_ccm_set_msg_len_mb16(msg_len, mb_mask, p_context); + + sm4_ccm_set_tag_len_mb16(tag_len, mb_mask, p_context); + + /* Zero initial hash values */ + PadBlock(0, SM4_CCM_CONTEXT_HASH(p_context), 16*SM4_LINES); + + SM4_CCM_CONTEXT_STATE(p_context) = sm4_ccm_update_aad; + } + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/sm4_ccm_update_aad_mb16_api.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/sm4_ccm_update_aad_mb16_api.c new file mode 100644 index 0000000..05e60b5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/ccm/sm4_ccm_update_aad_mb16_api.c @@ -0,0 +1,59 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include + +DLL_PUBLIC +mbx_status16 mbx_sm4_ccm_update_aad_mb16(const int8u *const pa_aad[SM4_LINES], const int aad_len[SM4_LINES], SM4_CCM_CTX_mb16 *p_context) +{ + int buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + + /* Test input pointers */ + if (NULL == pa_aad || NULL == aad_len || NULL == p_context) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* Check state */ + if (sm4_ccm_update_aad != SM4_CCM_CONTEXT_STATE(p_context)) { + status = MBX_SET_STS16_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* Don't process buffers with input pointers equal to zero */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_aad[buf_no] == NULL) { + if (aad_len[buf_no] != 0) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + mb_mask &= ~(0x1 << buf_no); + continue; + } + } + if (aad_len[buf_no] < 0 || aad_len[buf_no] > MAX_CCM_AAD_LENGTH) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + mb_mask &= ~(0x1 << buf_no); + } + } + + if (MBX_IS_ANY_OK_STS16(status)) + sm4_ccm_update_aad_mb16(pa_aad, aad_len, mb_mask, p_context); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_decrypt_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_decrypt_mb16.c new file mode 100644 index 0000000..8ab502e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_decrypt_mb16.c @@ -0,0 +1,101 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include + +/* +// This function performs decryption of given data and updates ghash with given data. +// Function returns mask where bit is set to 1 if length of given data for buffer is overflowed. +*/ + +__mmask16 sm4_gcm_decrypt_mb16(int8u *pa_out[SM4_LINES], + const int8u *const pa_in[SM4_LINES], + const int in_len[SM4_LINES], + __mmask16 mb_mask, + SM4_GCM_CTX_mb16 *p_context) +{ + if (SM4_GCM_CONTEXT_STATE(p_context) == sm4_gcm_update_iv) { + /* Finalize IVs */ + sm4_gcm_finalize_iv_mb16(NULL, mb_mask, p_context); + } + + /* Switch context state to decryption */ + SM4_GCM_CONTEXT_STATE(p_context) = sm4_gcm_dec; + + const int8u *loc_pa_in[SM4_LINES]; + int in_len_rearranged[SM4_LINES]; + + /* Rearrange input pointers and lengths to required layout */ + rearrange(loc_pa_in, pa_in); + rearrange(in_len_rearranged, in_len); + __m512i loc_in_len = loadu(in_len_rearranged); + + __mmask16 overflow_mask = 0x0000; + __m512i max_txt_len = set1_epi64(0xFFFFFFFE0); /* (2^39 - 256) div 8 */ + + /* + // Update txt length + // + // TXT length is passed as 32 bit integer + // TXT length is used to construct last block for ghash computation as follow: + // 64 bits with AAD length | 64 bits with TXT length + // Length of AAD and TXT is stored in context in this form + // + // Code below transforms 32 bit input integers into the following block: + // 64 bits with TXT length | 64 zero bits + // and add it to length stored in context + // + // The whole operation is the following for each buffer: + // 64 bits with AAD len in context | 64 bits with TXT len in context + // + + // 64 zero bits | 32 zero bits | 32 bits with input TXT len + */ + + for (int i = 0; i < 4; i++) { + __m512i len_updade = maskz_expandloadu_epi32(0x1111, (void *)(in_len_rearranged + i * 4)); /* Load txt len to high part of registry */ + __m512i len_context = loadu(BUFFER_REG_NUM(SM4_GCM_CONTEXT_LEN(p_context), i)); + + len_context = add_epi64(len_context, len_updade); + + storeu(BUFFER_REG_NUM(SM4_GCM_CONTEXT_LEN(p_context), i), len_context); + + __mmask8 overflow_mask_part = cmp_epi64_mask(max_txt_len, len_context, _MM_CMPINT_LE); + + overflow_mask_part = + (overflow_mask_part & 0x01) | (overflow_mask_part & 0x04) >> 1 | (overflow_mask_part & 0x10) >> 2 | (overflow_mask_part & 0x40) >> 3; + overflow_mask = overflow_mask | overflow_mask_part << (i * 4); + } + + /* Update intermediate ghash value with full blocks of given data */ + sm4_gcm_update_ghash_full_blocks_mb16(SM4_GCM_CONTEXT_GHASH(p_context), loc_pa_in, &loc_in_len, SM4_GCM_CONTEXT_HASHKEY(p_context), mb_mask); + + if (cmp_epi32_mask(loc_in_len, setzero(), _MM_CMPINT_EQ) != 0xFFFF) { + /* Update intermediate ghash value with partial blocks of given data */ + sm4_gcm_update_ghash_partial_blocks_mb16( + SM4_GCM_CONTEXT_GHASH(p_context), loc_pa_in, &loc_in_len, SM4_GCM_CONTEXT_HASHKEY(p_context)[0], mb_mask); + /* Switch context state to tag computation to prevent decryption after any partial blocks are processed */ + SM4_GCM_CONTEXT_STATE(p_context) = sm4_gcm_get_tag; + } + + const mbx_sm4_key_schedule *key_sched = (const mbx_sm4_key_schedule *)SM4_GCM_CONTEXT_KEY(p_context); + + /* Decrypt */ + sm4_gctr_kernel_mb16(pa_out, pa_in, in_len, (const int32u **)key_sched, mb_mask, p_context); + + return overflow_mask; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_encrypt_j0_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_encrypt_j0_mb16.c new file mode 100644 index 0000000..2f1c878 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_encrypt_j0_mb16.c @@ -0,0 +1,74 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include + +/* +// This function encrypts J0 to use it for tag computation +// Encrypted J0 value XOR'ed with accumulated GHASH value in function sm4_gcm_get_tag_mb16 +*/ + +void sm4_encrypt_j0_mb16(SM4_GCM_CTX_mb16 *p_context) +{ + const mbx_sm4_key_schedule *key_sched = (const mbx_sm4_key_schedule *)SM4_GCM_CONTEXT_KEY(context); + + __m512i j0_blocks[4]; + __m128i *j0 = SM4_GCM_CONTEXT_J0(p_context); + int tagRearrange[] = { 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15 }; + + const int8u *pa_inp[SM4_LINES]; + + for (int i = 0; i < SM4_LINES; i++) { + pa_inp[i] = (unsigned char *)(j0 + tagRearrange[i]); + } + + TRANSPOSE_16x4_I32_EPI32(&j0_blocks[0], &j0_blocks[1], &j0_blocks[2], &j0_blocks[3], pa_inp, 0xFFFF); + + const __m512i *p_rk = (const __m512i *)key_sched; + + __m512i tmp; + for (int itr = 0; itr < SM4_ROUNDS; itr += 4, p_rk += 4) { + SM4_FOUR_ROUNDS(j0_blocks[0], j0_blocks[1], j0_blocks[2], j0_blocks[3], tmp, p_rk, 1); + } + + tmp = j0_blocks[0]; + j0_blocks[0] = j0_blocks[3]; + j0_blocks[3] = tmp; + tmp = j0_blocks[1]; + j0_blocks[1] = j0_blocks[2]; + j0_blocks[2] = tmp; + + __m512i T1_0 = unpacklo_epi32(j0_blocks[0], j0_blocks[1]); + __m512i T1_1 = unpackhi_epi32(j0_blocks[0], j0_blocks[1]); + __m512i T1_2 = unpacklo_epi32(j0_blocks[2], j0_blocks[3]); + __m512i T1_3 = unpackhi_epi32(j0_blocks[2], j0_blocks[3]); + + j0_blocks[0] = unpacklo_epi64(T1_0, T1_2); + j0_blocks[1] = unpackhi_epi64(T1_0, T1_2); + j0_blocks[2] = unpacklo_epi64(T1_1, T1_3); + j0_blocks[3] = unpackhi_epi64(T1_1, T1_3); + + j0_blocks[0] = shuffle_epi8(j0_blocks[0], M512(swapBytes)); + j0_blocks[1] = shuffle_epi8(j0_blocks[1], M512(swapBytes)); + j0_blocks[2] = shuffle_epi8(j0_blocks[2], M512(swapBytes)); + j0_blocks[3] = shuffle_epi8(j0_blocks[3], M512(swapBytes)); + + storeu(j0 + 0, j0_blocks[0]); + storeu(j0 + 4, j0_blocks[1]); + storeu(j0 + 8, j0_blocks[2]); + storeu(j0 + 12, j0_blocks[3]); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_encrypt_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_encrypt_mb16.c new file mode 100644 index 0000000..4288323 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_encrypt_mb16.c @@ -0,0 +1,102 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include + +/* +// This function performs encryption of given data and updates ghash with encrypted data. +// Function returns mask where bit is set to 1 if length of given data for buffer is overflowed. +*/ + +__mmask16 sm4_gcm_encrypt_mb16(int8u *pa_out[SM4_LINES], + const int8u *const pa_in[SM4_LINES], + const int in_len[SM4_LINES], + __mmask16 mb_mask, + SM4_GCM_CTX_mb16 *p_context) +{ + if (SM4_GCM_CONTEXT_STATE(p_context) == sm4_gcm_update_iv) { + /* Finalize IVs */ + sm4_gcm_finalize_iv_mb16(NULL, mb_mask, p_context); + } + + /* Switch context state to encryption */ + SM4_GCM_CONTEXT_STATE(p_context) = sm4_gcm_enc; + + const mbx_sm4_key_schedule *key_sched = (const mbx_sm4_key_schedule *)SM4_GCM_CONTEXT_KEY(p_context); + + /* Encrypt */ + sm4_gctr_kernel_mb16(pa_out, pa_in, in_len, (const int32u **)key_sched, mb_mask, p_context); + + int8u *loc_pa_out[SM4_LINES]; + int in_len_rearranged[SM4_LINES]; + + /* Rearrange input pointers and lengths to required layout */ + rearrange(loc_pa_out, pa_out); + rearrange(in_len_rearranged, in_len); + __m512i loc_in_len = loadu(in_len_rearranged); + + __mmask16 overflow_mask = 0x0000; + __m512i max_txt_len = set1_epi64(0xFFFFFFFE0); /* (2^39 - 256) div 8 */ + + /* + // Update txt length + // + // TXT length is passed as 32 bit integer + // TXT length is used to construct last block for ghash computation as follow: + // 64 bits with AAD length | 64 bits with TXT length + // Length of AAD and TXT is stored in context in this form + // + // Code below transforms 32 bit input integers into the following block: + // 64 bits with TXT length | 64 zero bits + // and add it to length stored in context + // + // The whole operation is the following for each buffer: + // 64 bits with AAD len in context | 64 bits with TXT len in context + // + + // 64 zero bits | 32 zero bits | 32 bits with input TXT len + */ + + for (int i = 0; i < 4; i++) { + __m512i len_updade = maskz_expandloadu_epi32(0x1111, (void *)(in_len_rearranged + i * 4)); /* Load txt len to high part of _m512iistry */ + __m512i len_context = loadu(BUFFER_REG_NUM(SM4_GCM_CONTEXT_LEN(p_context), i)); + + len_context = add_epi64(len_context, len_updade); + + storeu(BUFFER_REG_NUM(SM4_GCM_CONTEXT_LEN(p_context), i), len_context); + + __mmask8 overflow_mask_part = cmp_epi64_mask(max_txt_len, len_context, _MM_CMPINT_LE); + + overflow_mask_part = + (overflow_mask_part & 0x01) | (overflow_mask_part & 0x04) >> 1 | (overflow_mask_part & 0x10) >> 2 | (overflow_mask_part & 0x40) >> 3; + overflow_mask = overflow_mask | overflow_mask_part << (i * 4); + } + + /* Update intermediate ghash value with full blocks of encrypted data */ + sm4_gcm_update_ghash_full_blocks_mb16( + SM4_GCM_CONTEXT_GHASH(p_context), (const int8u **)loc_pa_out, &loc_in_len, SM4_GCM_CONTEXT_HASHKEY(p_context), mb_mask); + + if (cmp_epi32_mask(loc_in_len, setzero(), _MM_CMPINT_EQ) != 0xFFFF) { + /* Update intermediate ghash value with partial blocks of encrypted data */ + sm4_gcm_update_ghash_partial_blocks_mb16( + SM4_GCM_CONTEXT_GHASH(p_context), (const int8u **)loc_pa_out, &loc_in_len, SM4_GCM_CONTEXT_HASHKEY(p_context)[0], mb_mask); + /* Switch context state to tag computation to prevent decryption after any partial blocks are processed */ + SM4_GCM_CONTEXT_STATE(p_context) = sm4_gcm_get_tag; + } + + return overflow_mask; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_finalize_iv_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_finalize_iv_mb16.c new file mode 100644 index 0000000..f76df89 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_finalize_iv_mb16.c @@ -0,0 +1,175 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include + +/* +// This function performs IV finalization (computes J0) in the follow way: +// If bitlen(IV) == 96, then let J0 = IV || 0^31 ||1. +// If bitlen(IV) != 96, then let s = 128 * [bitlen(IV) / 128] - bitlen(IV), and let J0 = GHASH(IV || 0^(s+64) || bitlen(IV)). +// +// 0^s means the bit string that consists of s '0' bits here +// [x] means the least integer that is not less than the real number x here +// +// This function also encrypts J0 by calling sm4_encrypt_j0_mb16(), to use it later for tag computation +*/ + +void sm4_gcm_finalize_iv_mb16(const int8u *const pa_iv[SM4_LINES], __mmask16 mb_mask, SM4_GCM_CTX_mb16 *p_context) +{ + __m128i *ctr = SM4_GCM_CONTEXT_CTR(p_context); + __m128i *j0 = SM4_GCM_CONTEXT_J0(p_context); + __m128i *hashkey = SM4_GCM_CONTEXT_HASHKEY(p_context)[0]; + + int64u *iv_len = SM4_GCM_CONTEXT_LEN(p_context); + + __m512i hashkeys_4_0, hashkeys_4_1, hashkeys_4_2, hashkeys_4_3; + __m512i *hashkeys[] = { &hashkeys_4_0, &hashkeys_4_1, &hashkeys_4_2, &hashkeys_4_3 }; + + __m512i iv_blocks_4_0, iv_blocks_4_1, iv_blocks_4_2, iv_blocks_4_3; + __m512i *data_blocks[] = { &iv_blocks_4_0, &iv_blocks_4_1, &iv_blocks_4_2, &iv_blocks_4_3 }; + + __m512i len_hi = loadu(iv_len); + __m512i len_lo = loadu(iv_len + 8); + + __mmask8 eq_12_mask_hi = cmp_epi64_mask(len_hi, set1_epi64(12), _MM_CMPINT_EQ); + __mmask8 eq_12_mask_lo = cmp_epi64_mask(len_lo, set1_epi64(12), _MM_CMPINT_EQ); + + __mmask8 eq_0_mask_hi = cmp_epi64_mask(len_hi, setzero(), _MM_CMPINT_EQ); + __mmask8 eq_0_mask_lo = cmp_epi64_mask(len_lo, setzero(), _MM_CMPINT_EQ); + + __mmask16 eq_12_mask = eq_12_mask_lo << 8 | eq_12_mask_hi; + __mmask16 eq_0_mask = eq_0_mask_lo << 8 | eq_0_mask_hi; + + __mmask16 load_mask = ~eq_12_mask | ~eq_0_mask; + + /* Finalize IVs of length != 96 bit */ + if (load_mask) { + + hashkeys_4_0 = loadu(hashkey + 0); + hashkeys_4_1 = loadu(hashkey + 4); + hashkeys_4_2 = loadu(hashkey + 8); + hashkeys_4_3 = loadu(hashkey + 12); + + iv_blocks_4_0 = setzero(); + iv_blocks_4_1 = setzero(); + iv_blocks_4_2 = setzero(); + iv_blocks_4_3 = setzero(); + + /* + // Loop with 4 iterations here does not unrolled by some compilers + // Unrolled explicitly because insert32x4 require the last parameter to be const + */ + + /* Begin of explicitly unrolled loop */ + __m128i input_block_0 = _mm_maskz_set1_epi64(1, *(iv_len + (0 + 4 * 0)) << 3); + __m128i input_block_1 = _mm_maskz_set1_epi64(1, *(iv_len + (0 + 4 * 1)) << 3); + __m128i input_block_2 = _mm_maskz_set1_epi64(1, *(iv_len + (0 + 4 * 2)) << 3); + __m128i input_block_3 = _mm_maskz_set1_epi64(1, *(iv_len + (0 + 4 * 3)) << 3); + + iv_blocks_4_0 = insert32x4(iv_blocks_4_0, input_block_0, 0); + iv_blocks_4_1 = insert32x4(iv_blocks_4_1, input_block_1, 0); + iv_blocks_4_2 = insert32x4(iv_blocks_4_2, input_block_2, 0); + iv_blocks_4_3 = insert32x4(iv_blocks_4_3, input_block_3, 0); + + input_block_0 = _mm_maskz_set1_epi64(1, *(iv_len + (1 + 4 * 0)) << 3); + input_block_1 = _mm_maskz_set1_epi64(1, *(iv_len + (1 + 4 * 1)) << 3); + input_block_2 = _mm_maskz_set1_epi64(1, *(iv_len + (1 + 4 * 2)) << 3); + input_block_3 = _mm_maskz_set1_epi64(1, *(iv_len + (1 + 4 * 3)) << 3); + + iv_blocks_4_0 = insert32x4(iv_blocks_4_0, input_block_0, 1); + iv_blocks_4_1 = insert32x4(iv_blocks_4_1, input_block_1, 1); + iv_blocks_4_2 = insert32x4(iv_blocks_4_2, input_block_2, 1); + iv_blocks_4_3 = insert32x4(iv_blocks_4_3, input_block_3, 1); + + input_block_0 = _mm_maskz_set1_epi64(1, *(iv_len + (2 + 4 * 0)) << 3); + input_block_1 = _mm_maskz_set1_epi64(1, *(iv_len + (2 + 4 * 1)) << 3); + input_block_2 = _mm_maskz_set1_epi64(1, *(iv_len + (2 + 4 * 2)) << 3); + input_block_3 = _mm_maskz_set1_epi64(1, *(iv_len + (2 + 4 * 3)) << 3); + + iv_blocks_4_0 = insert32x4(iv_blocks_4_0, input_block_0, 2); + iv_blocks_4_1 = insert32x4(iv_blocks_4_1, input_block_1, 2); + iv_blocks_4_2 = insert32x4(iv_blocks_4_2, input_block_2, 2); + iv_blocks_4_3 = insert32x4(iv_blocks_4_3, input_block_3, 2); + + input_block_0 = _mm_maskz_set1_epi64(1, *(iv_len + (3 + 4 * 0)) << 3); + input_block_1 = _mm_maskz_set1_epi64(1, *(iv_len + (3 + 4 * 1)) << 3); + input_block_2 = _mm_maskz_set1_epi64(1, *(iv_len + (3 + 4 * 2)) << 3); + input_block_3 = _mm_maskz_set1_epi64(1, *(iv_len + (3 + 4 * 3)) << 3); + + iv_blocks_4_0 = insert32x4(iv_blocks_4_0, input_block_0, 3); + iv_blocks_4_1 = insert32x4(iv_blocks_4_1, input_block_1, 3); + iv_blocks_4_2 = insert32x4(iv_blocks_4_2, input_block_2, 3); + iv_blocks_4_3 = insert32x4(iv_blocks_4_3, input_block_3, 3); + + /* End of explicitly unrolled loop */ + + iv_blocks_4_0 = xor(iv_blocks_4_0, M512(j0 + 0)); + iv_blocks_4_1 = xor(iv_blocks_4_1, M512(j0 + 4)); + iv_blocks_4_2 = xor(iv_blocks_4_2, M512(j0 + 8)); + iv_blocks_4_3 = xor(iv_blocks_4_3, M512(j0 + 12)); + + sm4_gcm_ghash_mul_single_block_mb16(data_blocks, hashkeys); + + iv_blocks_4_0 = shuffle_epi8(iv_blocks_4_0, M512(swapEndianness)); + iv_blocks_4_1 = shuffle_epi8(iv_blocks_4_1, M512(swapEndianness)); + iv_blocks_4_2 = shuffle_epi8(iv_blocks_4_2, M512(swapEndianness)); + iv_blocks_4_3 = shuffle_epi8(iv_blocks_4_3, M512(swapEndianness)); + + __mmask8 store_mask_0 = 0x03 * (0x1 & ((load_mask >> 0 * 4) >> 0)) | 0x0C * (0x1 & ((load_mask >> 0 * 4) >> 1)) | + 0x30 * (0x1 & ((load_mask >> 0 * 4) >> 2)) | 0xC0 * (0x1 & ((load_mask >> 0 * 4) >> 3)); + + __mmask8 store_mask_1 = 0x03 * (0x1 & ((load_mask >> 1 * 4) >> 0)) | 0x0C * (0x1 & ((load_mask >> 1 * 4) >> 1)) | + 0x30 * (0x1 & ((load_mask >> 1 * 4) >> 2)) | 0xC0 * (0x1 & ((load_mask >> 1 * 4) >> 3)); + + __mmask8 store_mask_2 = 0x03 * (0x1 & ((load_mask >> 2 * 4) >> 0)) | 0x0C * (0x1 & ((load_mask >> 2 * 4) >> 1)) | + 0x30 * (0x1 & ((load_mask >> 2 * 4) >> 2)) | 0xC0 * (0x1 & ((load_mask >> 2 * 4) >> 3)); + + __mmask8 store_mask_3 = 0x03 * (0x1 & ((load_mask >> 3 * 4) >> 0)) | 0x0C * (0x1 & ((load_mask >> 3 * 4) >> 1)) | + 0x30 * (0x1 & ((load_mask >> 3 * 4) >> 2)) | 0xC0 * (0x1 & ((load_mask >> 3 * 4) >> 3)); + + mask_storeu_epi64(j0 + 0, store_mask_0, iv_blocks_4_0); + mask_storeu_epi64(j0 + 4, store_mask_1, iv_blocks_4_1); + mask_storeu_epi64(j0 + 8, store_mask_2, iv_blocks_4_2); + mask_storeu_epi64(j0 + 12, store_mask_3, iv_blocks_4_3); + } + + /* Finalize IVs of length == 96 bit */ + if (eq_12_mask != 0 && pa_iv != NULL) { + __m128i iv_block; + + for (int i = 0; i < SM4_LINES; i++) { + iv_block = _mm_mask_loadu_epi8(M128(one_f), 0x0FFF * (0x1 & eq_12_mask), (void *)pa_iv[i]); + _mm_mask_storeu_epi8(j0 + i, 0xFFFF * (0x1 & eq_12_mask), iv_block); + eq_12_mask >>= 1; + } + } + + /* Store initial counter */ + storeu(ctr + 0, inc_block32(shuffle_epi8(loadu(j0 + 0), M512(swapEndianness)), initialInc)); + storeu(ctr + 4, inc_block32(shuffle_epi8(loadu(j0 + 4), M512(swapEndianness)), initialInc)); + storeu(ctr + 8, inc_block32(shuffle_epi8(loadu(j0 + 8), M512(swapEndianness)), initialInc)); + storeu(ctr + 12, inc_block32(shuffle_epi8(loadu(j0 + 12), M512(swapEndianness)), initialInc)); + + sm4_encrypt_j0_mb16(p_context); + + /* Clear length buffer to reuse it for TXT and AAD length */ + SM4_GCM_CLEAR_LEN(BUFFER_REG_NUM(SM4_GCM_CONTEXT_LEN(p_context), 0)); + SM4_GCM_CLEAR_LEN(BUFFER_REG_NUM(SM4_GCM_CONTEXT_LEN(p_context), 1)); + SM4_GCM_CLEAR_LEN(BUFFER_REG_NUM(SM4_GCM_CONTEXT_LEN(p_context), 2)); + SM4_GCM_CLEAR_LEN(BUFFER_REG_NUM(SM4_GCM_CONTEXT_LEN(p_context), 3)); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_gctr_kernel_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_gctr_kernel_mb16.c new file mode 100644 index 0000000..5ac0765 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_gctr_kernel_mb16.c @@ -0,0 +1,339 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include +#include /* for zero_mb8 */ + +/* +// These functions performs GCTR encryption/decryption +// Implementation is the same with SM4-CTR +*/ + +__INLINE __m128i IncBlock128(__m128i x, int32u increment) { return _mm_add_epi32(x, _mm_maskz_loadu_epi32(1, &increment)); } + +static void sm4_gctr_mask_kernel_mb16(__m512i *CTR, + const __m512i *p_rk, + __m512i loc_len, + const int8u **loc_inp, + int8u **loc_out, + int8u *inc, + __mmask16 tmp_mask, + __mmask16 mb_mask) +{ + __m512i TMP[20]; + while (tmp_mask) { + *CTR = inc_block32(*CTR, inc); + *(CTR + 1) = inc_block32(*(CTR + 1), inc); + *(CTR + 2) = inc_block32(*(CTR + 2), inc); + *(CTR + 3) = inc_block32(*(CTR + 3), inc); + TMP[0] = shuffle_epi8(*CTR, M512(swapWordsOrder)); + TMP[1] = shuffle_epi8(*(CTR + 1), M512(swapWordsOrder)); + TMP[2] = shuffle_epi8(*(CTR + 2), M512(swapWordsOrder)); + TMP[3] = shuffle_epi8(*(CTR + 3), M512(swapWordsOrder)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + *(CTR + 4) = inc_block32(*(CTR + 4), inc); + *(CTR + 5) = inc_block32(*(CTR + 5), inc); + *(CTR + 6) = inc_block32(*(CTR + 6), inc); + *(CTR + 7) = inc_block32(*(CTR + 7), inc); + TMP[0] = shuffle_epi8(*(CTR + 4), M512(swapWordsOrder)); + TMP[1] = shuffle_epi8(*(CTR + 5), M512(swapWordsOrder)); + TMP[2] = shuffle_epi8(*(CTR + 6), M512(swapWordsOrder)); + TMP[3] = shuffle_epi8(*(CTR + 7), M512(swapWordsOrder)); + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], TMP[0], TMP[1], TMP[2], TMP[3]); + + *(CTR + 8) = inc_block32(*(CTR + 8), inc); + *(CTR + 9) = inc_block32(*(CTR + 9), inc); + *(CTR + 10) = inc_block32(*(CTR + 10), inc); + *(CTR + 11) = inc_block32(*(CTR + 11), inc); + TMP[0] = shuffle_epi8(*(CTR + 8), M512(swapWordsOrder)); + TMP[1] = shuffle_epi8(*(CTR + 9), M512(swapWordsOrder)); + TMP[2] = shuffle_epi8(*(CTR + 10), M512(swapWordsOrder)); + TMP[3] = shuffle_epi8(*(CTR + 11), M512(swapWordsOrder)); + TRANSPOSE_INP_512(TMP[12], TMP[13], TMP[14], TMP[15], TMP[0], TMP[1], TMP[2], TMP[3]); + + *(CTR + 12) = inc_block32(*(CTR + 12), inc); + *(CTR + 13) = inc_block32(*(CTR + 13), inc); + *(CTR + 14) = inc_block32(*(CTR + 14), inc); + *(CTR + 15) = inc_block32(*(CTR + 15), inc); + TMP[0] = shuffle_epi8(*(CTR + 12), M512(swapWordsOrder)); + TMP[1] = shuffle_epi8(*(CTR + 13), M512(swapWordsOrder)); + TMP[2] = shuffle_epi8(*(CTR + 14), M512(swapWordsOrder)); + TMP[3] = shuffle_epi8(*(CTR + 15), M512(swapWordsOrder)); + TRANSPOSE_INP_512(TMP[16], TMP[17], TMP[18], TMP[19], TMP[0], TMP[1], TMP[2], TMP[3]); + + SM4_KERNEL(TMP, p_rk, 1); + p_rk -= SM4_ROUNDS; + + /* Mask for data loading */ + __mmask64 stream_mask; + int *p_loc_len = (int *)&loc_len; + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = shuffle_epi8(TMP[3], M512(swapBytes)); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + mask_storeu_epi8((__m512i *)loc_out[0], stream_mask, xor(TMP[0], maskz_loadu_epi8(stream_mask, loc_inp[0]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + mask_storeu_epi8((__m512i *)loc_out[1], stream_mask, xor(TMP[1], maskz_loadu_epi8(stream_mask, loc_inp[1]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + mask_storeu_epi8((__m512i *)loc_out[2], stream_mask, xor(TMP[2], maskz_loadu_epi8(stream_mask, loc_inp[2]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + mask_storeu_epi8((__m512i *)loc_out[3], stream_mask, xor(TMP[3], maskz_loadu_epi8(stream_mask, loc_inp[3]))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[0] = shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = shuffle_epi8(TMP[3], M512(swapBytes)); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + mask_storeu_epi8((__m512i *)loc_out[4], stream_mask, xor(TMP[0], maskz_loadu_epi8(stream_mask, loc_inp[4]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + mask_storeu_epi8((__m512i *)loc_out[5], stream_mask, xor(TMP[1], maskz_loadu_epi8(stream_mask, loc_inp[5]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + mask_storeu_epi8((__m512i *)loc_out[6], stream_mask, xor(TMP[2], maskz_loadu_epi8(stream_mask, loc_inp[6]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + mask_storeu_epi8((__m512i *)loc_out[7], stream_mask, xor(TMP[3], maskz_loadu_epi8(stream_mask, loc_inp[7]))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[12], TMP[13], TMP[14], TMP[15]); + TMP[0] = shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = shuffle_epi8(TMP[3], M512(swapBytes)); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + mask_storeu_epi8((__m512i *)loc_out[8], stream_mask, xor(TMP[0], maskz_loadu_epi8(stream_mask, loc_inp[8]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + mask_storeu_epi8((__m512i *)loc_out[9], stream_mask, xor(TMP[1], maskz_loadu_epi8(stream_mask, loc_inp[9]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + mask_storeu_epi8((__m512i *)loc_out[10], stream_mask, xor(TMP[2], maskz_loadu_epi8(stream_mask, loc_inp[10]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + mask_storeu_epi8((__m512i *)loc_out[11], stream_mask, xor(TMP[3], maskz_loadu_epi8(stream_mask, loc_inp[11]))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[16], TMP[17], TMP[18], TMP[19]); + TMP[0] = shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = shuffle_epi8(TMP[3], M512(swapBytes)); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + mask_storeu_epi8((__m512i *)loc_out[12], stream_mask, xor(TMP[0], maskz_loadu_epi8(stream_mask, loc_inp[12]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + mask_storeu_epi8((__m512i *)loc_out[13], stream_mask, xor(TMP[1], maskz_loadu_epi8(stream_mask, loc_inp[13]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + mask_storeu_epi8((__m512i *)loc_out[14], stream_mask, xor(TMP[2], maskz_loadu_epi8(stream_mask, loc_inp[14]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + mask_storeu_epi8((__m512i *)loc_out[15], stream_mask, xor(TMP[3], maskz_loadu_epi8(stream_mask, loc_inp[15]))); + + /* Update pointers to data */ + M512(loc_inp) = add_epi64(loadu(loc_inp), set1_epi64(4 * SM4_BLOCK_SIZE)); + M512(loc_inp + 8) = add_epi64(loadu(loc_inp + 8), set1_epi64(4 * SM4_BLOCK_SIZE)); + + M512(loc_out) = add_epi64(loadu(loc_out), set1_epi64(4 * SM4_BLOCK_SIZE)); + M512(loc_out + 8) = add_epi64(loadu(loc_out + 8), set1_epi64(4 * SM4_BLOCK_SIZE)); + + /* Update number of blocks left and processing mask */ + loc_len = sub_epi32(loc_len, set1_epi32(4 * SM4_BLOCK_SIZE)); + tmp_mask = mask_cmp_epi32_mask(mb_mask, loc_len, set1_epi32(0), _MM_CMPINT_NLE); + inc = (int8u *)nextInc; + } + + /* clear local copy of sensitive data */ + zero_mb8((int64u(*)[8])TMP, sizeof(TMP) / sizeof(TMP[0])); +} + +void sm4_gctr_kernel_mb16(int8u *pa_out[SM4_LINES], + const int8u *const pa_inp[SM4_LINES], + const int len[SM4_LINES], + const int32u *key_sched[SM4_ROUNDS], + __mmask16 mb_mask, + SM4_GCM_CTX_mb16 *p_context) +{ + __mmask16 loc_mb_mask = 0; + + for (int i = 0; i < SM4_LINES; i++) { + __mmask16 tmp = mb_mask & (0x1 << rearrangeOrder[i]); + tmp = tmp >> rearrangeOrder[i]; + loc_mb_mask = loc_mb_mask | tmp << i; + } + + mb_mask = loc_mb_mask; + + const int8u *loc_inp[SM4_LINES]; + int8u *loc_out[SM4_LINES]; + + /* Create the local copy of the input data length in bytes and set it to zero for non-valid buffers */ + __m512i loc_len; + loc_len = loadu(len); + loc_len = mask_set1_epi32(loc_len, ~mb_mask, 0); + + /* input blocks loc_blks[] = ceil(loc_len[]/SM4_BLOCK_SIZE) */ + int32u loc_blks[SM4_LINES]; + storeu(loc_blks, srli_epi32(add_epi32(loc_len, set1_epi32(SM4_BLOCK_SIZE - 1)), 4)); + + /* Local copies of the pointers to input and output buffers */ + storeu((void *)loc_inp, loadu(pa_inp)); + storeu((void *)(loc_inp + 8), loadu(pa_inp + 8)); + + storeu(loc_out, loadu(pa_out)); + storeu(loc_out + 8, loadu(pa_out + 8)); + + /* Pointer p_rk is set to the beginning of the key schedule */ + const __m512i *p_rk = (const __m512i *)key_sched; + + /* TMP[] - temporary buffer for processing */ + /* CTR - store CTR values */ + __m512i TMP[20]; + __m512i CTR[SM4_LINES]; + __m128i loc_ctr[SM4_LINES]; + + /* Load CTR value from valid buffers and rearrange it */ + mb_mask = mask_cmp_epi32_mask(mb_mask, loc_len, setzero(), _MM_CMPINT_NLE); + int ctrRearrange[] = { 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15 }; + for (int i = 0; i < SM4_LINES; i++) { + if (0x1 & (mb_mask >> i)) { + loc_ctr[i] = _mm_loadu_si128(SM4_GCM_CONTEXT_CTR(p_context) + ctrRearrange[i]); + } else { + loc_ctr[i] = _mm_setzero_si128(); + } + + CTR[i] = broadcast_i64x2(loc_ctr[i]); + } + + /* Generate the mask to process 4 blocks from each buffer */ + __mmask16 tmp_mask = mask_cmp_epi32_mask(mb_mask, loc_len, set1_epi32(4 * SM4_BLOCK_SIZE), _MM_CMPINT_NLT); + + int8u *inc = (int8u *)firstInc; + + /* Go to this loop if all 16 buffers contain at least 4 blocks each */ + while (tmp_mask == 0xFFFF) { + CTR[0] = inc_block32(CTR[0], inc); + CTR[1] = inc_block32(CTR[1], inc); + CTR[2] = inc_block32(CTR[2], inc); + CTR[3] = inc_block32(CTR[3], inc); + TMP[0] = shuffle_epi8(CTR[0], M512(swapWordsOrder)); + TMP[1] = shuffle_epi8(CTR[1], M512(swapWordsOrder)); + TMP[2] = shuffle_epi8(CTR[2], M512(swapWordsOrder)); + TMP[3] = shuffle_epi8(CTR[3], M512(swapWordsOrder)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + CTR[4] = inc_block32(CTR[4], inc); + CTR[5] = inc_block32(CTR[5], inc); + CTR[6] = inc_block32(CTR[6], inc); + CTR[7] = inc_block32(CTR[7], inc); + TMP[0] = shuffle_epi8(CTR[4], M512(swapWordsOrder)); + TMP[1] = shuffle_epi8(CTR[5], M512(swapWordsOrder)); + TMP[2] = shuffle_epi8(CTR[6], M512(swapWordsOrder)); + TMP[3] = shuffle_epi8(CTR[7], M512(swapWordsOrder)); + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], TMP[0], TMP[1], TMP[2], TMP[3]); + + CTR[8] = inc_block32(CTR[8], inc); + CTR[9] = inc_block32(CTR[9], inc); + CTR[10] = inc_block32(CTR[10], inc); + CTR[11] = inc_block32(CTR[11], inc); + TMP[0] = shuffle_epi8(CTR[8], M512(swapWordsOrder)); + TMP[1] = shuffle_epi8(CTR[9], M512(swapWordsOrder)); + TMP[2] = shuffle_epi8(CTR[10], M512(swapWordsOrder)); + TMP[3] = shuffle_epi8(CTR[11], M512(swapWordsOrder)); + TRANSPOSE_INP_512(TMP[12], TMP[13], TMP[14], TMP[15], TMP[0], TMP[1], TMP[2], TMP[3]); + + CTR[12] = inc_block32(CTR[12], inc); + CTR[13] = inc_block32(CTR[13], inc); + CTR[14] = inc_block32(CTR[14], inc); + CTR[15] = inc_block32(CTR[15], inc); + TMP[0] = shuffle_epi8(CTR[12], M512(swapWordsOrder)); + TMP[1] = shuffle_epi8(CTR[13], M512(swapWordsOrder)); + TMP[2] = shuffle_epi8(CTR[14], M512(swapWordsOrder)); + TMP[3] = shuffle_epi8(CTR[15], M512(swapWordsOrder)); + TRANSPOSE_INP_512(TMP[16], TMP[17], TMP[18], TMP[19], TMP[0], TMP[1], TMP[2], TMP[3]); + + SM4_KERNEL(TMP, p_rk, 1); + p_rk -= SM4_ROUNDS; + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = shuffle_epi8(TMP[3], M512(swapBytes)); + storeu((__m512i *)loc_out[0], xor(TMP[0], loadu(loc_inp[0]))); + storeu((__m512i *)loc_out[1], xor(TMP[1], loadu(loc_inp[1]))); + storeu((__m512i *)loc_out[2], xor(TMP[2], loadu(loc_inp[2]))); + storeu((__m512i *)loc_out[3], xor(TMP[3], loadu(loc_inp[3]))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[0] = shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = shuffle_epi8(TMP[3], M512(swapBytes)); + storeu((__m512i *)loc_out[4], xor(TMP[0], loadu(loc_inp[4]))); + storeu((__m512i *)loc_out[5], xor(TMP[1], loadu(loc_inp[5]))); + storeu((__m512i *)loc_out[6], xor(TMP[2], loadu(loc_inp[6]))); + storeu((__m512i *)loc_out[7], xor(TMP[3], loadu(loc_inp[7]))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[12], TMP[13], TMP[14], TMP[15]); + TMP[0] = shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = shuffle_epi8(TMP[3], M512(swapBytes)); + storeu((__m512i *)loc_out[8], xor(TMP[0], loadu(loc_inp[8]))); + storeu((__m512i *)loc_out[9], xor(TMP[1], loadu(loc_inp[9]))); + storeu((__m512i *)loc_out[10], xor(TMP[2], loadu(loc_inp[10]))); + storeu((__m512i *)loc_out[11], xor(TMP[3], loadu(loc_inp[11]))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[16], TMP[17], TMP[18], TMP[19]); + TMP[0] = shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = shuffle_epi8(TMP[3], M512(swapBytes)); + storeu((__m512i *)loc_out[12], xor(TMP[0], loadu(loc_inp[12]))); + storeu((__m512i *)loc_out[13], xor(TMP[1], loadu(loc_inp[13]))); + storeu((__m512i *)loc_out[14], xor(TMP[2], loadu(loc_inp[14]))); + storeu((__m512i *)loc_out[15], xor(TMP[3], loadu(loc_inp[15]))); + + /* Update pointers to data */ + M512(loc_inp) = add_epi64(loadu(loc_inp), set1_epi64(4 * SM4_BLOCK_SIZE)); + M512(loc_inp + 8) = add_epi64(loadu(loc_inp + 8), set1_epi64(4 * SM4_BLOCK_SIZE)); + + M512(loc_out) = add_epi64(loadu(loc_out), set1_epi64(4 * SM4_BLOCK_SIZE)); + M512(loc_out + 8) = add_epi64(loadu(loc_out + 8), set1_epi64(4 * SM4_BLOCK_SIZE)); + + /* Update number of blocks left and processing mask */ + loc_len = sub_epi32(loc_len, set1_epi32(4 * SM4_BLOCK_SIZE)); + tmp_mask = mask_cmp_epi32_mask(mb_mask, loc_len, set1_epi32(4 * SM4_BLOCK_SIZE), _MM_CMPINT_NLT); + inc = (int8u *)nextInc; + } + + /* Check if we have any data */ + tmp_mask = mask_cmp_epi32_mask(mb_mask, loc_len, setzero(), _MM_CMPINT_NLE); + if (tmp_mask) { + sm4_gctr_mask_kernel_mb16(CTR, p_rk, loc_len, loc_inp, loc_out, inc, tmp_mask, mb_mask); + } + + /* update and store counters */ + for (int i = 0; i < SM4_LINES; i++) { + if (0x1 & (mb_mask >> i)) { + loc_ctr[i] = IncBlock128(loc_ctr[i], loc_blks[i]); + _mm_storeu_si128(SM4_GCM_CONTEXT_CTR(p_context) + ctrRearrange[i], loc_ctr[i]); + loc_ctr[i] = _mm_setzero_si128(); + } + } + + /* clear local copy of sensitive data */ + zero_mb8((int64u(*)[8])TMP, sizeof(TMP) / sizeof(TMP[0])); + zero_mb8((int64u(*)[8])CTR, sizeof(CTR) / sizeof(CTR[0])); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_get_tag_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_get_tag_mb16.c new file mode 100644 index 0000000..2c3cbf1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_get_tag_mb16.c @@ -0,0 +1,98 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include +#include /* for zero_mb8 */ + +/* +// This function performs tag computation as follow: +// v = 128 * [bitlen(AAD) / 128] - bitlen(AAD) +// u = 128 * [bitlen(TXT) / 128] - bitlen(TXT) +// S = GHASH (AAD || 0^v || ciphTXT || 0^u || bitlen(AAD) || bitlen(ciphTXT)). +// tag = S xor J0 +// +// J0 is previously encrypted +// +// 0^s means the bit string that consists of s '0' bits here +// [x] means the least integer that is not less than the real number x here +*/ + +void sm4_gcm_get_tag_mb16(int8u *pa_out[SM4_LINES], const int tag_len[SM4_LINES], __mmask16 mb_mask, SM4_GCM_CTX_mb16 *p_context) +{ + __m512i hashkeys_4_0, hashkeys_4_1, hashkeys_4_2, hashkeys_4_3; + __m512i *hashkeys[] = { &hashkeys_4_0, &hashkeys_4_1, &hashkeys_4_2, &hashkeys_4_3 }; + + __m512i data_len_blocks_4_0, data_len_blocks_4_1, data_len_blocks_4_2, data_len_blocks_4_3; + __m512i *data_blocks[] = { &data_len_blocks_4_0, &data_len_blocks_4_1, &data_len_blocks_4_2, &data_len_blocks_4_3 }; + + __m128i *hashkey = SM4_GCM_CONTEXT_HASHKEY(p_context)[0]; + hashkeys_4_0 = loadu(hashkey + 0); + hashkeys_4_1 = loadu(hashkey + 4); + hashkeys_4_2 = loadu(hashkey + 8); + hashkeys_4_3 = loadu(hashkey + 12); + + /* Convert length in bytes to length in bits */ + data_len_blocks_4_0 = sll_epi32(loadu(BUFFER_REG_NUM(SM4_GCM_CONTEXT_LEN(p_context), 0)), M128(bytes_to_bits_shift)); + data_len_blocks_4_1 = sll_epi32(loadu(BUFFER_REG_NUM(SM4_GCM_CONTEXT_LEN(p_context), 1)), M128(bytes_to_bits_shift)); + data_len_blocks_4_2 = sll_epi32(loadu(BUFFER_REG_NUM(SM4_GCM_CONTEXT_LEN(p_context), 2)), M128(bytes_to_bits_shift)); + data_len_blocks_4_3 = sll_epi32(loadu(BUFFER_REG_NUM(SM4_GCM_CONTEXT_LEN(p_context), 3)), M128(bytes_to_bits_shift)); + + /* XOR with accumulated GHASH value */ + __m128i *ghash = SM4_GCM_CONTEXT_GHASH(p_context); + + data_len_blocks_4_0 = xor(data_len_blocks_4_0, M512(ghash + 0)); + data_len_blocks_4_1 = xor(data_len_blocks_4_1, M512(ghash + 4)); + data_len_blocks_4_2 = xor(data_len_blocks_4_2, M512(ghash + 8)); + data_len_blocks_4_3 = xor(data_len_blocks_4_3, M512(ghash + 12)); + + /* Update GHASH value */ + sm4_gcm_ghash_mul_single_block_mb16(data_blocks, hashkeys); + + data_len_blocks_4_0 = shuffle_epi8(data_len_blocks_4_0, M512(swapEndianness)); + data_len_blocks_4_1 = shuffle_epi8(data_len_blocks_4_1, M512(swapEndianness)); + data_len_blocks_4_2 = shuffle_epi8(data_len_blocks_4_2, M512(swapEndianness)); + data_len_blocks_4_3 = shuffle_epi8(data_len_blocks_4_3, M512(swapEndianness)); + + __m512i j0_blocks[4]; + + __m128i *j0 = SM4_GCM_CONTEXT_J0(p_context); + + j0_blocks[0] = loadu(j0 + 0); + j0_blocks[1] = loadu(j0 + 4); + j0_blocks[2] = loadu(j0 + 8); + j0_blocks[3] = loadu(j0 + 12); + + __m512i tag_blocks[4]; + + /* XOR with previously encrypted J0 */ + tag_blocks[0] = xor(data_len_blocks_4_0, j0_blocks[0]); + tag_blocks[1] = xor(data_len_blocks_4_1, j0_blocks[1]); + tag_blocks[2] = xor(data_len_blocks_4_2, j0_blocks[2]); + tag_blocks[3] = xor(data_len_blocks_4_3, j0_blocks[3]); + + /* Store result */ + for (int i = 0; i < SM4_LINES; i++) { + __m128i one_block = M128((__m128i *)tag_blocks + i); + + __mmask16 tagMask = ~(0xFFFF << (tag_len[rearrangeOrder[i]])) * ((mb_mask >> i) & 0x1); + _mm_mask_storeu_epi8((void *)(pa_out[rearrangeOrder[i]]), tagMask, one_block); + } + + /* clear local copy of sensitive data */ + zero_mb8((int64u(*)[8])tag_blocks, sizeof(tag_blocks) / sizeof(tag_blocks[0])); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_ghash_mul_single_block_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_ghash_mul_single_block_mb16.c new file mode 100644 index 0000000..35073f3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_ghash_mul_single_block_mb16.c @@ -0,0 +1,131 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include + +/* +// This function performs GHASH multiplication: +// Multiplication operation for the binary Galois (finite) field of 2^128 elements. +// +// Details about this operation can be found in NIST SP 800-38D +// +// This code is a port of GCM part of Intel(R) IPSec AES-GCM code +// Code modified to work with multi-buffer approach +*/ + +void sm4_gcm_ghash_mul_single_block_mb16(__m512i *data_blocks[], __m512i *hashkeys[]) +{ + __m512i T1_0, T2_0, T3_0; + __m512i T1_1, T2_1, T3_1; + __m512i T1_2, T2_2, T3_2; + __m512i T1_3, T2_3, T3_3; + + T1_0 = clmul(*(hashkeys[0]), *(data_blocks[0]), 0x11); // T1 = a1*b1 + T1_1 = clmul(*(hashkeys[1]), *(data_blocks[1]), 0x11); + T1_2 = clmul(*(hashkeys[2]), *(data_blocks[2]), 0x11); + T1_3 = clmul(*(hashkeys[3]), *(data_blocks[3]), 0x11); + + T2_0 = clmul(*(hashkeys[0]), *(data_blocks[0]), 0x00); // T1 = a0*b0 + T2_1 = clmul(*(hashkeys[1]), *(data_blocks[1]), 0x00); + T2_2 = clmul(*(hashkeys[2]), *(data_blocks[2]), 0x00); + T2_3 = clmul(*(hashkeys[3]), *(data_blocks[3]), 0x00); + + T3_0 = clmul(*(hashkeys[0]), *(data_blocks[0]), 0x01); // T3 = a1*b0 + T3_1 = clmul(*(hashkeys[1]), *(data_blocks[1]), 0x01); + T3_2 = clmul(*(hashkeys[2]), *(data_blocks[2]), 0x01); + T3_3 = clmul(*(hashkeys[3]), *(data_blocks[3]), 0x01); + + *(data_blocks[0]) = clmul(*(hashkeys[0]), *(data_blocks[0]), 0x10); // T3 = a0*b1 + *(data_blocks[1]) = clmul(*(hashkeys[1]), *(data_blocks[1]), 0x10); + *(data_blocks[2]) = clmul(*(hashkeys[2]), *(data_blocks[2]), 0x10); + *(data_blocks[3]) = clmul(*(hashkeys[3]), *(data_blocks[3]), 0x10); + + *(data_blocks[0]) = xor(*(data_blocks[0]), T3_0); + *(data_blocks[1]) = xor(*(data_blocks[1]), T3_1); + *(data_blocks[2]) = xor(*(data_blocks[2]), T3_2); + *(data_blocks[3]) = xor(*(data_blocks[3]), T3_3); + + T3_0 = bsrli_epi128(*(data_blocks[0]), 8); + T3_1 = bsrli_epi128(*(data_blocks[1]), 8); + T3_2 = bsrli_epi128(*(data_blocks[2]), 8); + T3_3 = bsrli_epi128(*(data_blocks[3]), 8); + + *(data_blocks[0]) = bslli_epi128(*(data_blocks[0]), 8); + *(data_blocks[1]) = bslli_epi128(*(data_blocks[1]), 8); + *(data_blocks[2]) = bslli_epi128(*(data_blocks[2]), 8); + *(data_blocks[3]) = bslli_epi128(*(data_blocks[3]), 8); + + T1_0 = xor(T1_0, T3_0); + T1_1 = xor(T1_1, T3_1); + T1_2 = xor(T1_2, T3_2); + T1_3 = xor(T1_3, T3_3); + + *(data_blocks[0]) = xor(*(data_blocks[0]), T2_0); + *(data_blocks[1]) = xor(*(data_blocks[1]), T2_1); + *(data_blocks[2]) = xor(*(data_blocks[2]), T2_2); + *(data_blocks[3]) = xor(*(data_blocks[3]), T2_3); + + /* first phase of the reduction */ + + T2_0 = clmul(M512(gcm_poly2), *(data_blocks[0]), 0x01); + T2_1 = clmul(M512(gcm_poly2), *(data_blocks[1]), 0x01); + T2_2 = clmul(M512(gcm_poly2), *(data_blocks[2]), 0x01); + T2_3 = clmul(M512(gcm_poly2), *(data_blocks[3]), 0x01); + + T2_0 = bslli_epi128(T2_0, 8); + T2_1 = bslli_epi128(T2_1, 8); + T2_2 = bslli_epi128(T2_2, 8); + T2_3 = bslli_epi128(T2_3, 8); + + *(data_blocks[0]) = xor(*(data_blocks[0]), T2_0); + *(data_blocks[1]) = xor(*(data_blocks[1]), T2_1); + *(data_blocks[2]) = xor(*(data_blocks[2]), T2_2); + *(data_blocks[3]) = xor(*(data_blocks[3]), T2_3); + + /* second phase of the reduction */ + + T2_0 = clmul(M512(gcm_poly2), *(data_blocks[0]), 0x00); + T2_1 = clmul(M512(gcm_poly2), *(data_blocks[1]), 0x00); + T2_2 = clmul(M512(gcm_poly2), *(data_blocks[2]), 0x00); + T2_3 = clmul(M512(gcm_poly2), *(data_blocks[3]), 0x00); + + T2_0 = bsrli_epi128(T2_0, 4); + T2_1 = bsrli_epi128(T2_1, 4); + T2_2 = bsrli_epi128(T2_2, 4); + T2_3 = bsrli_epi128(T2_3, 4); + + *(data_blocks[0]) = clmul(M512(gcm_poly2), *(data_blocks[0]), 0x10); + *(data_blocks[1]) = clmul(M512(gcm_poly2), *(data_blocks[1]), 0x10); + *(data_blocks[2]) = clmul(M512(gcm_poly2), *(data_blocks[2]), 0x10); + *(data_blocks[3]) = clmul(M512(gcm_poly2), *(data_blocks[3]), 0x10); + + *(data_blocks[0]) = bslli_epi128(*(data_blocks[0]), 4); + *(data_blocks[1]) = bslli_epi128(*(data_blocks[1]), 4); + *(data_blocks[2]) = bslli_epi128(*(data_blocks[2]), 4); + *(data_blocks[3]) = bslli_epi128(*(data_blocks[3]), 4); + + *(data_blocks[0]) = xor(*(data_blocks[0]), T2_0); + *(data_blocks[1]) = xor(*(data_blocks[1]), T2_1); + *(data_blocks[2]) = xor(*(data_blocks[2]), T2_2); + *(data_blocks[3]) = xor(*(data_blocks[3]), T2_3); + + *(data_blocks[0]) = xor(*(data_blocks[0]), T1_0); + *(data_blocks[1]) = xor(*(data_blocks[1]), T1_1); + *(data_blocks[2]) = xor(*(data_blocks[2]), T1_2); + *(data_blocks[3]) = xor(*(data_blocks[3]), T1_3); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_precompute_hashkey_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_precompute_hashkey_mb16.c new file mode 100644 index 0000000..fef770a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_precompute_hashkey_mb16.c @@ -0,0 +1,140 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include + + +/* +// This function precomputes haskeys for delayed reduction: +// hashkeys >> 1 mod poly and hashkeys ^ 2 >> 1 mod poly ... hashkeys ^ 7 >> 1 mod poly +// +// This code is a port of GCM part of Intel(R) IPSec AES-GCM code +// Code modified to work with multi-buffer approach +*/ + +void sm4_gcm_precompute_hashkey_mb16(const mbx_sm4_key_schedule *key_sched, SM4_GCM_CTX_mb16 *p_context) +{ + const __m512i *p_rk = (const __m512i *)key_sched; + __m512i tmp; + + /* Encrypt zero blocks */ + __m512i hashkey_blocks_4_0 = setzero(); + __m512i hashkey_blocks_4_1 = setzero(); + __m512i hashkey_blocks_4_2 = setzero(); + __m512i hashkey_blocks_4_3 = setzero(); + + for (int itr = 0; itr < SM4_ROUNDS; itr += 4, p_rk += 4) { + SM4_FOUR_ROUNDS(hashkey_blocks_4_0, hashkey_blocks_4_1, hashkey_blocks_4_2, hashkey_blocks_4_3, tmp, p_rk, 1); + } + + tmp = hashkey_blocks_4_0; + hashkey_blocks_4_0 = hashkey_blocks_4_3; + hashkey_blocks_4_3 = tmp; + tmp = hashkey_blocks_4_1; + hashkey_blocks_4_1 = hashkey_blocks_4_2; + hashkey_blocks_4_2 = tmp; + + /* Get the right endianness */ + __m512i T1_0 = unpacklo_epi32(hashkey_blocks_4_0, hashkey_blocks_4_1); + __m512i T1_1 = unpackhi_epi32(hashkey_blocks_4_0, hashkey_blocks_4_1); + __m512i T1_2 = unpacklo_epi32(hashkey_blocks_4_2, hashkey_blocks_4_3); + __m512i T1_3 = unpackhi_epi32(hashkey_blocks_4_2, hashkey_blocks_4_3); + + hashkey_blocks_4_0 = unpacklo_epi64(T1_0, T1_2); + hashkey_blocks_4_1 = unpackhi_epi64(T1_0, T1_2); + hashkey_blocks_4_2 = unpacklo_epi64(T1_1, T1_3); + hashkey_blocks_4_3 = unpackhi_epi64(T1_1, T1_3); + + hashkey_blocks_4_0 = shuffle_epi8(hashkey_blocks_4_0, M512(swapWordsOrder)); + hashkey_blocks_4_1 = shuffle_epi8(hashkey_blocks_4_1, M512(swapWordsOrder)); + hashkey_blocks_4_2 = shuffle_epi8(hashkey_blocks_4_2, M512(swapWordsOrder)); + hashkey_blocks_4_3 = shuffle_epi8(hashkey_blocks_4_3, M512(swapWordsOrder)); + + /* compute hashkeys >> 1 mod poly */ + T1_0 = srli_epi64(hashkey_blocks_4_0, 63); + T1_1 = srli_epi64(hashkey_blocks_4_1, 63); + T1_2 = srli_epi64(hashkey_blocks_4_2, 63); + T1_3 = srli_epi64(hashkey_blocks_4_3, 63); + hashkey_blocks_4_0 = slli_epi64(hashkey_blocks_4_0, 1); + hashkey_blocks_4_1 = slli_epi64(hashkey_blocks_4_1, 1); + hashkey_blocks_4_2 = slli_epi64(hashkey_blocks_4_2, 1); + hashkey_blocks_4_3 = slli_epi64(hashkey_blocks_4_3, 1); + + __m512i T2_0 = bsrli_epi128(T1_0, 8); + __m512i T2_1 = bsrli_epi128(T1_1, 8); + __m512i T2_2 = bsrli_epi128(T1_2, 8); + __m512i T2_3 = bsrli_epi128(T1_3, 8); + T1_0 = bslli_epi128(T1_0, 8); + T1_1 = bslli_epi128(T1_1, 8); + T1_2 = bslli_epi128(T1_2, 8); + T1_3 = bslli_epi128(T1_3, 8); + + hashkey_blocks_4_0 = or (hashkey_blocks_4_0, T1_0); + hashkey_blocks_4_1 = or (hashkey_blocks_4_1, T1_1); + hashkey_blocks_4_2 = or (hashkey_blocks_4_2, T1_2); + hashkey_blocks_4_3 = or (hashkey_blocks_4_3, T1_3); + + T1_0 = shuffle_epi32(T2_0, 0b00100100); + T1_1 = shuffle_epi32(T2_1, 0b00100100); + T1_2 = shuffle_epi32(T2_2, 0b00100100); + T1_3 = shuffle_epi32(T2_3, 0b00100100); + + __mmask16 cmp_mask_0 = cmpeq_epi32_mask(T1_0, M512(two_one)); + __mmask16 cmp_mask_1 = cmpeq_epi32_mask(T1_1, M512(two_one)); + __mmask16 cmp_mask_2 = cmpeq_epi32_mask(T1_2, M512(two_one)); + __mmask16 cmp_mask_3 = cmpeq_epi32_mask(T1_3, M512(two_one)); + T1_0 = mask_set1_epi32(T1_0, cmp_mask_0, 0xFFFFFFFF); + T1_1 = mask_set1_epi32(T1_1, cmp_mask_1, 0xFFFFFFFF); + T1_2 = mask_set1_epi32(T1_2, cmp_mask_2, 0xFFFFFFFF); + T1_3 = mask_set1_epi32(T1_3, cmp_mask_3, 0xFFFFFFFF); + + T1_0 = and(T1_0, M512(gcm_poly)); + T1_1 = and(T1_1, M512(gcm_poly)); + T1_2 = and(T1_2, M512(gcm_poly)); + T1_3 = and(T1_3, M512(gcm_poly)); + hashkey_blocks_4_0 = xor(hashkey_blocks_4_0, T1_0); + hashkey_blocks_4_1 = xor(hashkey_blocks_4_1, T1_1); + hashkey_blocks_4_2 = xor(hashkey_blocks_4_2, T1_2); + hashkey_blocks_4_3 = xor(hashkey_blocks_4_3, T1_3); + + __m128i *p_hashkey = (__m128i *)SM4_GCM_CONTEXT_HASHKEY(p_context)[0]; + + storeu(p_hashkey + 0, hashkey_blocks_4_0); + storeu(p_hashkey + 4, hashkey_blocks_4_1); + storeu(p_hashkey + 8, hashkey_blocks_4_2); + storeu(p_hashkey + 12, hashkey_blocks_4_3); + + /* compute hashkeys ^ 2 >> 1 mod poly ... hashkeys ^ 7 >> 1 mod poly */ + __m512i hashkey_pwr_blocks_4_0 = hashkey_blocks_4_0; + __m512i hashkey_pwr_blocks_4_1 = hashkey_blocks_4_1; + __m512i hashkey_pwr_blocks_4_2 = hashkey_blocks_4_2; + __m512i hashkey_pwr_blocks_4_3 = hashkey_blocks_4_3; + + __m512i *hashkey[] = { &hashkey_blocks_4_0, &hashkey_blocks_4_1, &hashkey_blocks_4_2, &hashkey_blocks_4_3 }; + __m512i *hashkey_pwr[] = { &hashkey_pwr_blocks_4_0, &hashkey_pwr_blocks_4_1, &hashkey_pwr_blocks_4_2, &hashkey_pwr_blocks_4_3 }; + + for (int i = 1; i < 8; i++) { + sm4_gcm_ghash_mul_single_block_mb16(hashkey_pwr, hashkey); + + p_hashkey = (__m128i *)SM4_GCM_CONTEXT_HASHKEY(p_context)[i]; + + storeu(p_hashkey + 0, hashkey_pwr_blocks_4_0); + storeu(p_hashkey + 4, hashkey_pwr_blocks_4_1); + storeu(p_hashkey + 8, hashkey_pwr_blocks_4_2); + storeu(p_hashkey + 12, hashkey_pwr_blocks_4_3); + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_update_aad_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_update_aad_mb16.c new file mode 100644 index 0000000..5dbc8c0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_update_aad_mb16.c @@ -0,0 +1,90 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include + +/* +// This function process 16 buffers with additional authentication data (AAD) +*/ + +__mmask16 sm4_gcm_update_aad_mb16(const int8u *const pa_aad[SM4_LINES], const int aad_len[SM4_LINES], __mmask16 mb_mask, SM4_GCM_CTX_mb16 *p_context) +{ + if (SM4_GCM_CONTEXT_STATE(p_context) == sm4_gcm_update_iv) { + /* Finalize IVs */ + sm4_gcm_finalize_iv_mb16(NULL, mb_mask, p_context); + + SM4_GCM_CONTEXT_STATE(p_context) = sm4_gcm_update_aad; + } + + __m128i *ghash = SM4_GCM_CONTEXT_GHASH(p_context); + + const int8u *loc_pa_aad[SM4_LINES]; + int aad_len_rearranged[SM4_LINES]; + + /* Rearrange pointers and lengths to right layout */ + rearrange(loc_pa_aad, pa_aad); + rearrange(aad_len_rearranged, aad_len); + + __m512i loc_aad_len = loadu(aad_len_rearranged); + + __mmask16 overflow_mask = 0x0000; + __m512i max_aad_len = set1_epi64(0x1FFFFFFFFFFFFFFF); /* (2^64 - 1) div 8 */ + + /* + // Update aad len + // + // AAD length is passed as 32 bit integer + // AAD lenght is used to construct last block for ghahs computation as follow: + // 64 bits with AAD length | 64 bits with TXT length + // Length of AAD and TXT is stored in context in this form + // + // Code below transforms 32 bit input integers into the following block: + // 64 bits with AAD length | 64 zero bits + // and add it to length stored in context + // + // The whole operation is the following for each buffer: + // 64 bits with AAD len in context | 64 bits with TXT len in context + // + + // 32 zero bits | 32 bits with input AAD len | 64 zero bits + */ + + for (int i = 0; i < 4; i++) { + __m512i len_updade = maskz_expandloadu_epi32(0x4444, (void *)(aad_len_rearranged + i * 4)); /* Load aad len to low part of _m512iistry */ + __m512i len_context = loadu(BUFFER_REG_NUM(SM4_GCM_CONTEXT_LEN(p_context), i)); + + len_context = add_epi64(len_context, len_updade); + storeu(BUFFER_REG_NUM(SM4_GCM_CONTEXT_LEN(p_context), i), len_context); + + __mmask8 overflow_mask_part = cmp_epi64_mask(max_aad_len, len_context, _MM_CMPINT_LT); + + overflow_mask_part = + (overflow_mask_part & 0x02) >> 1 | (overflow_mask_part & 0x08) >> 2 | (overflow_mask_part & 0x20) >> 3 | (overflow_mask_part & 0x80) >> 4; + overflow_mask = overflow_mask | overflow_mask_part << (i * 4); + } + + /* Process full blocks of AADs */ + sm4_gcm_update_ghash_full_blocks_mb16(ghash, loc_pa_aad, &loc_aad_len, SM4_GCM_CONTEXT_HASHKEY(p_context), mb_mask); + + if (cmp_epi32_mask(loc_aad_len, setzero(), _MM_CMPINT_EQ) != 0xFFFF) { + /* Process partial blocks of AADs */ + sm4_gcm_update_ghash_partial_blocks_mb16(ghash, loc_pa_aad, &loc_aad_len, SM4_GCM_CONTEXT_HASHKEY(p_context)[0], mb_mask); + SM4_GCM_CONTEXT_STATE(p_context) = sm4_gcm_start_encdec; + } + + return overflow_mask; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_update_ghash_full_blocks_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_update_ghash_full_blocks_mb16.c new file mode 100644 index 0000000..415c065 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_update_ghash_full_blocks_mb16.c @@ -0,0 +1,691 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include + +__INLINE void read_first(__m512i *data_blocks[4], const int8u *const pa_input[SM4_LINES], __mmask16 load_mask) +{ + __mmask16 load_mask_0 = load_mask >> 0 * 4; + __mmask16 load_mask_1 = load_mask >> 1 * 4; + __mmask16 load_mask_2 = load_mask >> 2 * 4; + __mmask16 load_mask_3 = load_mask >> 3 * 4; + + *(data_blocks[0]) = setzero(); + *(data_blocks[1]) = setzero(); + *(data_blocks[2]) = setzero(); + *(data_blocks[3]) = setzero(); + + /* + // Loop with 4 iterations here does not unrolled by some compilers + // Unrolled explicitly because insert32x4 require the last parameter to be const + */ + + /* Begin of explicitly unrolled loop */ + + __m128i input_block_0 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_0), (void *)pa_input[0 + 4 * 0]); + __m128i input_block_1 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_1), (void *)pa_input[0 + 4 * 1]); + __m128i input_block_2 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_2), (void *)pa_input[0 + 4 * 2]); + __m128i input_block_3 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_3), (void *)pa_input[0 + 4 * 3]); + + input_block_0 = _mm_shuffle_epi8(input_block_0, M128(swapEndianness)); + input_block_1 = _mm_shuffle_epi8(input_block_1, M128(swapEndianness)); + input_block_2 = _mm_shuffle_epi8(input_block_2, M128(swapEndianness)); + input_block_3 = _mm_shuffle_epi8(input_block_3, M128(swapEndianness)); + + *(data_blocks[0]) = insert32x4(*(data_blocks[0]), input_block_0, 0); + *(data_blocks[1]) = insert32x4(*(data_blocks[1]), input_block_1, 0); + *(data_blocks[2]) = insert32x4(*(data_blocks[2]), input_block_2, 0); + *(data_blocks[3]) = insert32x4(*(data_blocks[3]), input_block_3, 0); + + load_mask_0 >>= 1; + load_mask_1 >>= 1; + load_mask_2 >>= 1; + load_mask_3 >>= 1; + + input_block_0 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_0), (void *)pa_input[1 + 4 * 0]); + input_block_1 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_1), (void *)pa_input[1 + 4 * 1]); + input_block_2 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_2), (void *)pa_input[1 + 4 * 2]); + input_block_3 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_3), (void *)pa_input[1 + 4 * 3]); + + input_block_0 = _mm_shuffle_epi8(input_block_0, M128(swapEndianness)); + input_block_1 = _mm_shuffle_epi8(input_block_1, M128(swapEndianness)); + input_block_2 = _mm_shuffle_epi8(input_block_2, M128(swapEndianness)); + input_block_3 = _mm_shuffle_epi8(input_block_3, M128(swapEndianness)); + + *(data_blocks[0]) = insert32x4(*(data_blocks[0]), input_block_0, 1); + *(data_blocks[1]) = insert32x4(*(data_blocks[1]), input_block_1, 1); + *(data_blocks[2]) = insert32x4(*(data_blocks[2]), input_block_2, 1); + *(data_blocks[3]) = insert32x4(*(data_blocks[3]), input_block_3, 1); + + load_mask_0 >>= 1; + load_mask_1 >>= 1; + load_mask_2 >>= 1; + load_mask_3 >>= 1; + + input_block_0 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_0), (void *)pa_input[2 + 4 * 0]); + input_block_1 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_1), (void *)pa_input[2 + 4 * 1]); + input_block_2 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_2), (void *)pa_input[2 + 4 * 2]); + input_block_3 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_3), (void *)pa_input[2 + 4 * 3]); + + input_block_0 = _mm_shuffle_epi8(input_block_0, M128(swapEndianness)); + input_block_1 = _mm_shuffle_epi8(input_block_1, M128(swapEndianness)); + input_block_2 = _mm_shuffle_epi8(input_block_2, M128(swapEndianness)); + input_block_3 = _mm_shuffle_epi8(input_block_3, M128(swapEndianness)); + + *(data_blocks[0]) = insert32x4(*(data_blocks[0]), input_block_0, 2); + *(data_blocks[1]) = insert32x4(*(data_blocks[1]), input_block_1, 2); + *(data_blocks[2]) = insert32x4(*(data_blocks[2]), input_block_2, 2); + *(data_blocks[3]) = insert32x4(*(data_blocks[3]), input_block_3, 2); + + load_mask_0 >>= 1; + load_mask_1 >>= 1; + load_mask_2 >>= 1; + load_mask_3 >>= 1; + + input_block_0 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_0), (void *)pa_input[3 + 4 * 0]); + input_block_1 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_1), (void *)pa_input[3 + 4 * 1]); + input_block_2 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_2), (void *)pa_input[3 + 4 * 2]); + input_block_3 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_3), (void *)pa_input[3 + 4 * 3]); + + input_block_0 = _mm_shuffle_epi8(input_block_0, M128(swapEndianness)); + input_block_1 = _mm_shuffle_epi8(input_block_1, M128(swapEndianness)); + input_block_2 = _mm_shuffle_epi8(input_block_2, M128(swapEndianness)); + input_block_3 = _mm_shuffle_epi8(input_block_3, M128(swapEndianness)); + + *(data_blocks[0]) = insert32x4(*(data_blocks[0]), input_block_0, 3); + *(data_blocks[1]) = insert32x4(*(data_blocks[1]), input_block_1, 3); + *(data_blocks[2]) = insert32x4(*(data_blocks[2]), input_block_2, 3); + *(data_blocks[3]) = insert32x4(*(data_blocks[3]), input_block_3, 3); + + /* End of explicitly unrolled loop */ +} + +__INLINE void read_next(__m512i *data_blocks[4], const int8u *const pa_input[SM4_LINES], int block_number, __mmask16 load_mask) +{ + __mmask16 load_mask_0 = load_mask >> 0 * 4; + __mmask16 load_mask_1 = load_mask >> 1 * 4; + __mmask16 load_mask_2 = load_mask >> 2 * 4; + __mmask16 load_mask_3 = load_mask >> 3 * 4; + + *(data_blocks[0]) = setzero(); + *(data_blocks[1]) = setzero(); + *(data_blocks[2]) = setzero(); + *(data_blocks[3]) = setzero(); + + /* + // Loop with 4 iterations here does not unrolled by some compilers + // Unrolled explicitly because insert32x4 require the last parameter to be const + */ + + /* Begin of explicitly unrolled loop */ + + __m128i input_block_0 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_0), (void *)(pa_input[0 + 4 * 0] + block_number * 16)); + __m128i input_block_1 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_1), (void *)(pa_input[0 + 4 * 1] + block_number * 16)); + __m128i input_block_2 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_2), (void *)(pa_input[0 + 4 * 2] + block_number * 16)); + __m128i input_block_3 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_3), (void *)(pa_input[0 + 4 * 3] + block_number * 16)); + + input_block_0 = _mm_shuffle_epi8(input_block_0, M128(swapEndianness)); + input_block_1 = _mm_shuffle_epi8(input_block_1, M128(swapEndianness)); + input_block_2 = _mm_shuffle_epi8(input_block_2, M128(swapEndianness)); + input_block_3 = _mm_shuffle_epi8(input_block_3, M128(swapEndianness)); + + *(data_blocks[0]) = insert32x4(*(data_blocks[0]), input_block_0, 0); + *(data_blocks[1]) = insert32x4(*(data_blocks[1]), input_block_1, 0); + *(data_blocks[2]) = insert32x4(*(data_blocks[2]), input_block_2, 0); + *(data_blocks[3]) = insert32x4(*(data_blocks[3]), input_block_3, 0); + + load_mask_0 >>= 1; + load_mask_1 >>= 1; + load_mask_2 >>= 1; + load_mask_3 >>= 1; + + input_block_0 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_0), (void *)(pa_input[1 + 4 * 0] + block_number * 16)); + input_block_1 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_1), (void *)(pa_input[1 + 4 * 1] + block_number * 16)); + input_block_2 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_2), (void *)(pa_input[1 + 4 * 2] + block_number * 16)); + input_block_3 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_3), (void *)(pa_input[1 + 4 * 3] + block_number * 16)); + + input_block_0 = _mm_shuffle_epi8(input_block_0, M128(swapEndianness)); + input_block_1 = _mm_shuffle_epi8(input_block_1, M128(swapEndianness)); + input_block_2 = _mm_shuffle_epi8(input_block_2, M128(swapEndianness)); + input_block_3 = _mm_shuffle_epi8(input_block_3, M128(swapEndianness)); + + *(data_blocks[0]) = insert32x4(*(data_blocks[0]), input_block_0, 1); + *(data_blocks[1]) = insert32x4(*(data_blocks[1]), input_block_1, 1); + *(data_blocks[2]) = insert32x4(*(data_blocks[2]), input_block_2, 1); + *(data_blocks[3]) = insert32x4(*(data_blocks[3]), input_block_3, 1); + + load_mask_0 >>= 1; + load_mask_1 >>= 1; + load_mask_2 >>= 1; + load_mask_3 >>= 1; + + input_block_0 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_0), (void *)(pa_input[2 + 4 * 0] + block_number * 16)); + input_block_1 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_1), (void *)(pa_input[2 + 4 * 1] + block_number * 16)); + input_block_2 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_2), (void *)(pa_input[2 + 4 * 2] + block_number * 16)); + input_block_3 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_3), (void *)(pa_input[2 + 4 * 3] + block_number * 16)); + + input_block_0 = _mm_shuffle_epi8(input_block_0, M128(swapEndianness)); + input_block_1 = _mm_shuffle_epi8(input_block_1, M128(swapEndianness)); + input_block_2 = _mm_shuffle_epi8(input_block_2, M128(swapEndianness)); + input_block_3 = _mm_shuffle_epi8(input_block_3, M128(swapEndianness)); + + *(data_blocks[0]) = insert32x4(*(data_blocks[0]), input_block_0, 2); + *(data_blocks[1]) = insert32x4(*(data_blocks[1]), input_block_1, 2); + *(data_blocks[2]) = insert32x4(*(data_blocks[2]), input_block_2, 2); + *(data_blocks[3]) = insert32x4(*(data_blocks[3]), input_block_3, 2); + + load_mask_0 >>= 1; + load_mask_1 >>= 1; + load_mask_2 >>= 1; + load_mask_3 >>= 1; + + input_block_0 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_0), (void *)(pa_input[3 + 4 * 0] + block_number * 16)); + input_block_1 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_1), (void *)(pa_input[3 + 4 * 1] + block_number * 16)); + input_block_2 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_2), (void *)(pa_input[3 + 4 * 2] + block_number * 16)); + input_block_3 = _mm_maskz_loadu_epi8(0xFFFF * (0x1 & load_mask_3), (void *)(pa_input[3 + 4 * 3] + block_number * 16)); + + input_block_0 = _mm_shuffle_epi8(input_block_0, M128(swapEndianness)); + input_block_1 = _mm_shuffle_epi8(input_block_1, M128(swapEndianness)); + input_block_2 = _mm_shuffle_epi8(input_block_2, M128(swapEndianness)); + input_block_3 = _mm_shuffle_epi8(input_block_3, M128(swapEndianness)); + + *(data_blocks[0]) = insert32x4(*(data_blocks[0]), input_block_0, 3); + *(data_blocks[1]) = insert32x4(*(data_blocks[1]), input_block_1, 3); + *(data_blocks[2]) = insert32x4(*(data_blocks[2]), input_block_2, 3); + *(data_blocks[3]) = insert32x4(*(data_blocks[3]), input_block_3, 3); + + /* End of explicitly unrolled loop */ +} + +/* +// This function updates 16 blocks of ghash by processing full blocks (128 bit blocks) of data +// +// This code is a port of GCM part of Intel(R) IPSec AES-GCM code +// Code modified to work with multi-buffer approach +*/ +void sm4_gcm_update_ghash_full_blocks_mb16(__m128i ghash[SM4_LINES], + const int8u *const pa_input[SM4_LINES], + __m512i *input_len, + __m128i hashkey[SM4_GCM_HASHKEY_PWR_NUM][SM4_LINES], + __mmask16 mb_mask) +{ + + int stream_len = 0; + __mmask16 load_mask = 0; + + __m512i hashkeys_4_0, hashkeys_4_1, hashkeys_4_2, hashkeys_4_3; + __m512i data_blocks_4_0, data_blocks_4_1, data_blocks_4_2, data_blocks_4_3; + __m512i *data_blocks[] = { &data_blocks_4_0, &data_blocks_4_1, &data_blocks_4_2, &data_blocks_4_3 }; + + __m512i T1_0, T2_0, T3_0, T4_0; + __m512i T1_1, T2_1, T3_1, T4_1; + __m512i T1_2, T2_2, T3_2, T4_2; + __m512i T1_3, T2_3, T3_3, T4_3; + + stream_len = 8; + cmp_epi32_mask((*input_len), set1_epi32(16 * 8), _MM_CMPINT_GE); + + load_mask &= mb_mask; + + while (load_mask) { + + /* Process first block using hashkey of highest power */ + hashkeys_4_0 = loadu(hashkey[stream_len - 1] + 0); + hashkeys_4_1 = loadu(hashkey[stream_len - 1] + 4); + hashkeys_4_2 = loadu(hashkey[stream_len - 1] + 8); + hashkeys_4_3 = loadu(hashkey[stream_len - 1] + 12); + + read_first(data_blocks, pa_input, load_mask); + + data_blocks_4_0 = xor(data_blocks_4_0, M512(ghash + 0)); + data_blocks_4_1 = xor(data_blocks_4_1, M512(ghash + 4)); + data_blocks_4_2 = xor(data_blocks_4_2, M512(ghash + 8)); + data_blocks_4_3 = xor(data_blocks_4_3, M512(ghash + 12)); + + T1_0 = clmul(hashkeys_4_0, data_blocks_4_0, 0x11); + T1_1 = clmul(hashkeys_4_1, data_blocks_4_1, 0x11); + T1_2 = clmul(hashkeys_4_2, data_blocks_4_2, 0x11); + T1_3 = clmul(hashkeys_4_3, data_blocks_4_3, 0x11); + + T2_0 = clmul(hashkeys_4_0, data_blocks_4_0, 0x00); + T2_1 = clmul(hashkeys_4_1, data_blocks_4_1, 0x00); + T2_2 = clmul(hashkeys_4_2, data_blocks_4_2, 0x00); + T2_3 = clmul(hashkeys_4_3, data_blocks_4_3, 0x00); + + T3_0 = clmul(hashkeys_4_0, data_blocks_4_0, 0x01); + T3_1 = clmul(hashkeys_4_1, data_blocks_4_1, 0x01); + T3_2 = clmul(hashkeys_4_2, data_blocks_4_2, 0x01); + T3_3 = clmul(hashkeys_4_3, data_blocks_4_3, 0x01); + + T4_0 = clmul(hashkeys_4_0, data_blocks_4_0, 0x10); + T4_1 = clmul(hashkeys_4_1, data_blocks_4_1, 0x10); + T4_2 = clmul(hashkeys_4_2, data_blocks_4_2, 0x10); + T4_3 = clmul(hashkeys_4_3, data_blocks_4_3, 0x10); + + T3_0 = xor(T4_0, T3_0); + T3_1 = xor(T4_1, T3_1); + T3_2 = xor(T4_2, T3_2); + T3_3 = xor(T4_3, T3_3); + + /* Process the rest of blocks */ + for (int i = 1; i < stream_len; i++) { + + read_next(data_blocks, pa_input, i, load_mask); + + hashkeys_4_0 = loadu(hashkey[stream_len - i - 1] + 0); + hashkeys_4_1 = loadu(hashkey[stream_len - i - 1] + 4); + hashkeys_4_2 = loadu(hashkey[stream_len - i - 1] + 8); + hashkeys_4_3 = loadu(hashkey[stream_len - i - 1] + 12); + + T4_0 = clmul(hashkeys_4_0, data_blocks_4_0, 0x11); + T4_1 = clmul(hashkeys_4_1, data_blocks_4_1, 0x11); + T4_2 = clmul(hashkeys_4_2, data_blocks_4_2, 0x11); + T4_3 = clmul(hashkeys_4_3, data_blocks_4_3, 0x11); + + T1_0 = xor(T4_0, T1_0); + T1_1 = xor(T4_1, T1_1); + T1_2 = xor(T4_2, T1_2); + T1_3 = xor(T4_3, T1_3); + + T4_0 = clmul(hashkeys_4_0, data_blocks_4_0, 0x00); + T4_1 = clmul(hashkeys_4_1, data_blocks_4_1, 0x00); + T4_2 = clmul(hashkeys_4_2, data_blocks_4_2, 0x00); + T4_3 = clmul(hashkeys_4_3, data_blocks_4_3, 0x00); + + T2_0 = xor(T4_0, T2_0); + T2_1 = xor(T4_1, T2_1); + T2_2 = xor(T4_2, T2_2); + T2_3 = xor(T4_3, T2_3); + + T4_0 = clmul(hashkeys_4_0, data_blocks_4_0, 0x01); + T4_1 = clmul(hashkeys_4_1, data_blocks_4_1, 0x01); + T4_2 = clmul(hashkeys_4_2, data_blocks_4_2, 0x01); + T4_3 = clmul(hashkeys_4_3, data_blocks_4_3, 0x01); + + T3_0 = xor(T4_0, T3_0); + T3_1 = xor(T4_1, T3_1); + T3_2 = xor(T4_2, T3_2); + T3_3 = xor(T4_3, T3_3); + + T4_0 = clmul(hashkeys_4_0, data_blocks_4_0, 0x10); + T4_1 = clmul(hashkeys_4_1, data_blocks_4_1, 0x10); + T4_2 = clmul(hashkeys_4_2, data_blocks_4_2, 0x10); + T4_3 = clmul(hashkeys_4_3, data_blocks_4_3, 0x10); + + T3_0 = xor(T4_0, T3_0); + T3_1 = xor(T4_1, T3_1); + T3_2 = xor(T4_2, T3_2); + T3_3 = xor(T4_3, T3_3); + } + + /* Accumulate non-redused result */ + + T4_0 = bslli_epi128(T3_0, 8); + T4_1 = bslli_epi128(T3_1, 8); + T4_2 = bslli_epi128(T3_2, 8); + T4_3 = bslli_epi128(T3_3, 8); + + T3_0 = bsrli_epi128(T3_0, 8); + T3_1 = bsrli_epi128(T3_1, 8); + T3_2 = bsrli_epi128(T3_2, 8); + T3_3 = bsrli_epi128(T3_3, 8); + + T2_0 = xor(T2_0, T4_0); + T2_1 = xor(T2_1, T4_1); + T2_2 = xor(T2_2, T4_2); + T2_3 = xor(T2_3, T4_3); + + T1_0 = xor(T1_0, T3_0); + T1_1 = xor(T1_1, T3_1); + T1_2 = xor(T1_2, T3_2); + T1_3 = xor(T1_3, T3_3); + + /* First phase of the reduction */ + + T3_0 = clmul(M512(gcm_poly2), T2_0, 0x01); + T3_1 = clmul(M512(gcm_poly2), T2_1, 0x01); + T3_2 = clmul(M512(gcm_poly2), T2_2, 0x01); + T3_3 = clmul(M512(gcm_poly2), T2_3, 0x01); + + T3_0 = bslli_epi128(T3_0, 8); + T3_1 = bslli_epi128(T3_1, 8); + T3_2 = bslli_epi128(T3_2, 8); + T3_3 = bslli_epi128(T3_3, 8); + + T2_0 = xor(T3_0, T2_0); + T2_1 = xor(T3_1, T2_1); + T2_2 = xor(T3_2, T2_2); + T2_3 = xor(T3_3, T2_3); + + /* Second phase of the reduction */ + + T3_0 = clmul(M512(gcm_poly2), T2_0, 0x00); + T3_1 = clmul(M512(gcm_poly2), T2_1, 0x00); + T3_2 = clmul(M512(gcm_poly2), T2_2, 0x00); + T3_3 = clmul(M512(gcm_poly2), T2_3, 0x00); + + T3_0 = bsrli_epi128(T3_0, 4); + T3_1 = bsrli_epi128(T3_1, 4); + T3_2 = bsrli_epi128(T3_2, 4); + T3_3 = bsrli_epi128(T3_3, 4); + + T4_0 = clmul(M512(gcm_poly2), T2_0, 0x10); + T4_1 = clmul(M512(gcm_poly2), T2_1, 0x10); + T4_2 = clmul(M512(gcm_poly2), T2_2, 0x10); + T4_3 = clmul(M512(gcm_poly2), T2_3, 0x10); + + T4_0 = bslli_epi128(T4_0, 4); + T4_1 = bslli_epi128(T4_1, 4); + T4_2 = bslli_epi128(T4_2, 4); + T4_3 = bslli_epi128(T4_3, 4); + + T4_0 = xor(T4_0, T3_0); + T4_1 = xor(T4_1, T3_1); + T4_2 = xor(T4_2, T3_2); + T4_3 = xor(T4_3, T3_3); + + T4_0 = xor(T4_0, T1_0); + T4_1 = xor(T4_1, T1_1); + T4_2 = xor(T4_2, T1_2); + T4_3 = xor(T4_3, T1_3); + + /* Reduction complete, store result */ + + __mmask8 store_mask_0 = 0x03 * (0x1 & ((load_mask >> 0 * 4) >> 0)) | // + 0x0C * (0x1 & ((load_mask >> 0 * 4) >> 1)) | // + 0x30 * (0x1 & ((load_mask >> 0 * 4) >> 2)) | // + 0xC0 * (0x1 & ((load_mask >> 0 * 4) >> 3)); // + + __mmask8 store_mask_1 = 0x03 * (0x1 & ((load_mask >> 1 * 4) >> 0)) | // + 0x0C * (0x1 & ((load_mask >> 1 * 4) >> 1)) | // + 0x30 * (0x1 & ((load_mask >> 1 * 4) >> 2)) | // + 0xC0 * (0x1 & ((load_mask >> 1 * 4) >> 3)); // + + __mmask8 store_mask_2 = 0x03 * (0x1 & ((load_mask >> 2 * 4) >> 0)) | // + 0x0C * (0x1 & ((load_mask >> 2 * 4) >> 1)) | // + 0x30 * (0x1 & ((load_mask >> 2 * 4) >> 2)) | // + 0xC0 * (0x1 & ((load_mask >> 2 * 4) >> 3)); // + + __mmask8 store_mask_3 = 0x03 * (0x1 & ((load_mask >> 3 * 4) >> 0)) | // + 0x0C * (0x1 & ((load_mask >> 3 * 4) >> 1)) | // + 0x30 * (0x1 & ((load_mask >> 3 * 4) >> 2)) | // + 0xC0 * (0x1 & ((load_mask >> 3 * 4) >> 3)); // + + mask_storeu_epi64(ghash + 0, store_mask_0, T4_0); + mask_storeu_epi64(ghash + 4, store_mask_1, T4_1); + mask_storeu_epi64(ghash + 8, store_mask_2, T4_2); + mask_storeu_epi64(ghash + 12, store_mask_3, T4_3); + + /* Update pointers and lengths */ + + (*input_len) = mask_sub_epi32((*input_len), load_mask, (*input_len), set1_epi32(16 * stream_len)); + + __mmask8 ptr_update_mask_lo = load_mask & 0xFF; + __mmask8 ptr_update_mask_hi = (load_mask >> 8) & 0xFF; + + __m512i loc_pa_input_lo = loadu(pa_input); + __m512i loc_pa_input_hi = loadu(pa_input + 8); + + loc_pa_input_lo = mask_add_epi64(loc_pa_input_lo, ptr_update_mask_lo, loc_pa_input_lo, set1_epi64(16 * stream_len)); + loc_pa_input_hi = mask_add_epi64(loc_pa_input_hi, ptr_update_mask_hi, loc_pa_input_hi, set1_epi64(16 * stream_len)); + + storeu((void *)pa_input, loc_pa_input_lo); + storeu((void *)(pa_input + 8), loc_pa_input_hi); + + load_mask = cmp_epi32_mask((*input_len), set1_epi32(16 * 8), _MM_CMPINT_GE); + + load_mask &= mb_mask; + } + + stream_len = 0; + load_mask = 0; + + /* Calculate how many blocks can be processed with delayed reduction */ + for (int i = 8; i > 0; i--) { + __mmask16 max_stream_len_mask = cmp_epi32_mask((*input_len), set1_epi32(16 * i), _MM_CMPINT_GE); + __mmask16 less_16_mask = cmp_epi32_mask((*input_len), set1_epi32(16), _MM_CMPINT_LT); + __mmask16 stream_mask = max_stream_len_mask | less_16_mask; + + load_mask = load_mask * (load_mask != 0) | max_stream_len_mask * (stream_mask == 0xFFFF) * (load_mask == 0) * (less_16_mask != 0xFFFF); + stream_len = stream_len + i * (stream_mask == 0xFFFF) * (stream_len == 0) * (less_16_mask != 0xFFFF); + } + + load_mask &= mb_mask; + + while (load_mask) { + + /* Process first block using hashkey of highest power */ + + hashkeys_4_0 = loadu(hashkey[stream_len - 1] + 0); + hashkeys_4_1 = loadu(hashkey[stream_len - 1] + 4); + hashkeys_4_2 = loadu(hashkey[stream_len - 1] + 8); + hashkeys_4_3 = loadu(hashkey[stream_len - 1] + 12); + + read_first(data_blocks, pa_input, load_mask); + + data_blocks_4_0 = xor(data_blocks_4_0, M512(ghash + 0)); + data_blocks_4_1 = xor(data_blocks_4_1, M512(ghash + 4)); + data_blocks_4_2 = xor(data_blocks_4_2, M512(ghash + 8)); + data_blocks_4_3 = xor(data_blocks_4_3, M512(ghash + 12)); + + T1_0 = clmul(hashkeys_4_0, data_blocks_4_0, 0x11); + T1_1 = clmul(hashkeys_4_1, data_blocks_4_1, 0x11); + T1_2 = clmul(hashkeys_4_2, data_blocks_4_2, 0x11); + T1_3 = clmul(hashkeys_4_3, data_blocks_4_3, 0x11); + + T2_0 = clmul(hashkeys_4_0, data_blocks_4_0, 0x00); + T2_1 = clmul(hashkeys_4_1, data_blocks_4_1, 0x00); + T2_2 = clmul(hashkeys_4_2, data_blocks_4_2, 0x00); + T2_3 = clmul(hashkeys_4_3, data_blocks_4_3, 0x00); + + T3_0 = clmul(hashkeys_4_0, data_blocks_4_0, 0x01); + T3_1 = clmul(hashkeys_4_1, data_blocks_4_1, 0x01); + T3_2 = clmul(hashkeys_4_2, data_blocks_4_2, 0x01); + T3_3 = clmul(hashkeys_4_3, data_blocks_4_3, 0x01); + + T4_0 = clmul(hashkeys_4_0, data_blocks_4_0, 0x10); + T4_1 = clmul(hashkeys_4_1, data_blocks_4_1, 0x10); + T4_2 = clmul(hashkeys_4_2, data_blocks_4_2, 0x10); + T4_3 = clmul(hashkeys_4_3, data_blocks_4_3, 0x10); + + T3_0 = xor(T4_0, T3_0); + T3_1 = xor(T4_1, T3_1); + T3_2 = xor(T4_2, T3_2); + T3_3 = xor(T4_3, T3_3); + + /* Process the rest of blocks */ + for (int i = 1; i < stream_len; i++) { + + read_next(data_blocks, pa_input, i, load_mask); + + hashkeys_4_0 = loadu(hashkey[stream_len - i - 1] + 0); + hashkeys_4_1 = loadu(hashkey[stream_len - i - 1] + 4); + hashkeys_4_2 = loadu(hashkey[stream_len - i - 1] + 8); + hashkeys_4_3 = loadu(hashkey[stream_len - i - 1] + 12); + + T4_0 = clmul(hashkeys_4_0, data_blocks_4_0, 0x11); + T4_1 = clmul(hashkeys_4_1, data_blocks_4_1, 0x11); + T4_2 = clmul(hashkeys_4_2, data_blocks_4_2, 0x11); + T4_3 = clmul(hashkeys_4_3, data_blocks_4_3, 0x11); + + T1_0 = xor(T4_0, T1_0); + T1_1 = xor(T4_1, T1_1); + T1_2 = xor(T4_2, T1_2); + T1_3 = xor(T4_3, T1_3); + + T4_0 = clmul(hashkeys_4_0, data_blocks_4_0, 0x00); + T4_1 = clmul(hashkeys_4_1, data_blocks_4_1, 0x00); + T4_2 = clmul(hashkeys_4_2, data_blocks_4_2, 0x00); + T4_3 = clmul(hashkeys_4_3, data_blocks_4_3, 0x00); + + T2_0 = xor(T4_0, T2_0); + T2_1 = xor(T4_1, T2_1); + T2_2 = xor(T4_2, T2_2); + T2_3 = xor(T4_3, T2_3); + + T4_0 = clmul(hashkeys_4_0, data_blocks_4_0, 0x01); + T4_1 = clmul(hashkeys_4_1, data_blocks_4_1, 0x01); + T4_2 = clmul(hashkeys_4_2, data_blocks_4_2, 0x01); + T4_3 = clmul(hashkeys_4_3, data_blocks_4_3, 0x01); + + T3_0 = xor(T4_0, T3_0); + T3_1 = xor(T4_1, T3_1); + T3_2 = xor(T4_2, T3_2); + T3_3 = xor(T4_3, T3_3); + + T4_0 = clmul(hashkeys_4_0, data_blocks_4_0, 0x10); + T4_1 = clmul(hashkeys_4_1, data_blocks_4_1, 0x10); + T4_2 = clmul(hashkeys_4_2, data_blocks_4_2, 0x10); + T4_3 = clmul(hashkeys_4_3, data_blocks_4_3, 0x10); + + T3_0 = xor(T4_0, T3_0); + T3_1 = xor(T4_1, T3_1); + T3_2 = xor(T4_2, T3_2); + T3_3 = xor(T4_3, T3_3); + } + + /* Accumulate non-redused result */ + + T4_0 = bslli_epi128(T3_0, 8); + T4_1 = bslli_epi128(T3_1, 8); + T4_2 = bslli_epi128(T3_2, 8); + T4_3 = bslli_epi128(T3_3, 8); + + T3_0 = bsrli_epi128(T3_0, 8); + T3_1 = bsrli_epi128(T3_1, 8); + T3_2 = bsrli_epi128(T3_2, 8); + T3_3 = bsrli_epi128(T3_3, 8); + + T2_0 = xor(T2_0, T4_0); + T2_1 = xor(T2_1, T4_1); + T2_2 = xor(T2_2, T4_2); + T2_3 = xor(T2_3, T4_3); + + T1_0 = xor(T1_0, T3_0); + T1_1 = xor(T1_1, T3_1); + T1_2 = xor(T1_2, T3_2); + T1_3 = xor(T1_3, T3_3); + + /* First phase of the reduction */ + + T3_0 = clmul(M512(gcm_poly2), T2_0, 0x01); + T3_1 = clmul(M512(gcm_poly2), T2_1, 0x01); + T3_2 = clmul(M512(gcm_poly2), T2_2, 0x01); + T3_3 = clmul(M512(gcm_poly2), T2_3, 0x01); + + T3_0 = bslli_epi128(T3_0, 8); + T3_1 = bslli_epi128(T3_1, 8); + T3_2 = bslli_epi128(T3_2, 8); + T3_3 = bslli_epi128(T3_3, 8); + + T2_0 = xor(T3_0, T2_0); + T2_1 = xor(T3_1, T2_1); + T2_2 = xor(T3_2, T2_2); + T2_3 = xor(T3_3, T2_3); + + /* Second phase of the reduction */ + + T3_0 = clmul(M512(gcm_poly2), T2_0, 0x00); + T3_1 = clmul(M512(gcm_poly2), T2_1, 0x00); + T3_2 = clmul(M512(gcm_poly2), T2_2, 0x00); + T3_3 = clmul(M512(gcm_poly2), T2_3, 0x00); + + T3_0 = bsrli_epi128(T3_0, 4); + T3_1 = bsrli_epi128(T3_1, 4); + T3_2 = bsrli_epi128(T3_2, 4); + T3_3 = bsrli_epi128(T3_3, 4); + + T4_0 = clmul(M512(gcm_poly2), T2_0, 0x10); + T4_1 = clmul(M512(gcm_poly2), T2_1, 0x10); + T4_2 = clmul(M512(gcm_poly2), T2_2, 0x10); + T4_3 = clmul(M512(gcm_poly2), T2_3, 0x10); + + T4_0 = bslli_epi128(T4_0, 4); + T4_1 = bslli_epi128(T4_1, 4); + T4_2 = bslli_epi128(T4_2, 4); + T4_3 = bslli_epi128(T4_3, 4); + + T4_0 = xor(T4_0, T3_0); + T4_1 = xor(T4_1, T3_1); + T4_2 = xor(T4_2, T3_2); + T4_3 = xor(T4_3, T3_3); + + T4_0 = xor(T4_0, T1_0); + T4_1 = xor(T4_1, T1_1); + T4_2 = xor(T4_2, T1_2); + T4_3 = xor(T4_3, T1_3); + + /* Reduction complete, store result */ + + __mmask8 store_mask_0 = 0x03 * (0x1 & ((load_mask >> 0 * 4) >> 0)) | // + 0x0C * (0x1 & ((load_mask >> 0 * 4) >> 1)) | // + 0x30 * (0x1 & ((load_mask >> 0 * 4) >> 2)) | // + 0xC0 * (0x1 & ((load_mask >> 0 * 4) >> 3)); // + + __mmask8 store_mask_1 = 0x03 * (0x1 & ((load_mask >> 1 * 4) >> 0)) | // + 0x0C * (0x1 & ((load_mask >> 1 * 4) >> 1)) | // + 0x30 * (0x1 & ((load_mask >> 1 * 4) >> 2)) | // + 0xC0 * (0x1 & ((load_mask >> 1 * 4) >> 3)); // + + __mmask8 store_mask_2 = 0x03 * (0x1 & ((load_mask >> 2 * 4) >> 0)) | // + 0x0C * (0x1 & ((load_mask >> 2 * 4) >> 1)) | // + 0x30 * (0x1 & ((load_mask >> 2 * 4) >> 2)) | // + 0xC0 * (0x1 & ((load_mask >> 2 * 4) >> 3)); // + + __mmask8 store_mask_3 = 0x03 * (0x1 & ((load_mask >> 3 * 4) >> 0)) | // + 0x0C * (0x1 & ((load_mask >> 3 * 4) >> 1)) | // + 0x30 * (0x1 & ((load_mask >> 3 * 4) >> 2)) | // + 0xC0 * (0x1 & ((load_mask >> 3 * 4) >> 3)); // + + mask_storeu_epi64(ghash + 0, store_mask_0, T4_0); + mask_storeu_epi64(ghash + 4, store_mask_1, T4_1); + mask_storeu_epi64(ghash + 8, store_mask_2, T4_2); + mask_storeu_epi64(ghash + 12, store_mask_3, T4_3); + + /* Update pointers and lengths */ + + (*input_len) = mask_sub_epi32((*input_len), load_mask, (*input_len), set1_epi32(16 * stream_len)); + + __mmask8 ptr_update_mask_lo = load_mask & 0xFF; + __mmask8 ptr_update_mask_hi = (load_mask >> 8) & 0xFF; + + __m512i loc_pa_input_lo = loadu(pa_input); + __m512i loc_pa_input_hi = loadu(pa_input + 8); + + loc_pa_input_lo = mask_add_epi64(loc_pa_input_lo, ptr_update_mask_lo, loc_pa_input_lo, set1_epi64(16 * stream_len)); + loc_pa_input_hi = mask_add_epi64(loc_pa_input_hi, ptr_update_mask_hi, loc_pa_input_hi, set1_epi64(16 * stream_len)); + + storeu((void *)pa_input, loc_pa_input_lo); + storeu((void *)(pa_input + 8), loc_pa_input_hi); + + stream_len = 0; + load_mask = 0; + + /* Calculate how many blocks can be processed with delayed reduction */ + for (int i = 8; i > 0; i--) { + __mmask16 max_stream_len_mask = cmp_epi32_mask((*input_len), set1_epi32(16 * i), _MM_CMPINT_GE); + __mmask16 less_16_mask = cmp_epi32_mask((*input_len), set1_epi32(16), _MM_CMPINT_LT); + __mmask16 stream_mask = max_stream_len_mask | less_16_mask; + + load_mask = load_mask * (load_mask != 0) | max_stream_len_mask * (stream_mask == 0xFFFF) * (load_mask == 0) * (less_16_mask != 0xFFFF); + stream_len = stream_len + i * (stream_mask == 0xFFFF) * (stream_len == 0) * (less_16_mask != 0xFFFF); + } + + load_mask &= mb_mask; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_update_ghash_partial_blocks_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_update_ghash_partial_blocks_mb16.c new file mode 100644 index 0000000..4d1662e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_update_ghash_partial_blocks_mb16.c @@ -0,0 +1,183 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include + +__INLINE void read_first(__m512i *data_blocks[4], const int8u *const pa_input[SM4_LINES], __m512i *input_len, __mmask16 load_mask) +{ + __mmask16 load_mask_0 = load_mask >> 0 * 4; + __mmask16 load_mask_1 = load_mask >> 1 * 4; + __mmask16 load_mask_2 = load_mask >> 2 * 4; + __mmask16 load_mask_3 = load_mask >> 3 * 4; + + *(data_blocks[0]) = setzero(); + *(data_blocks[1]) = setzero(); + *(data_blocks[2]) = setzero(); + *(data_blocks[3]) = setzero(); + + /* + // Loop with 4 iterations here does not unrolled by some compilers + // Unrolled explicitly because insert32x4 require the last parameter to be const + */ + + /* Begin of explicitly unrolled loop */ + + __m128i input_block_0 = + _mm_maskz_loadu_epi8((0xFFFF >> (16 - *((int *)input_len + (0 + 4 * 0)))) * (0x1 & load_mask_0), (void *)pa_input[0 + 4 * 0]); + __m128i input_block_1 = + _mm_maskz_loadu_epi8((0xFFFF >> (16 - *((int *)input_len + (0 + 4 * 1)))) * (0x1 & load_mask_1), (void *)pa_input[0 + 4 * 1]); + __m128i input_block_2 = + _mm_maskz_loadu_epi8((0xFFFF >> (16 - *((int *)input_len + (0 + 4 * 2)))) * (0x1 & load_mask_2), (void *)pa_input[0 + 4 * 2]); + __m128i input_block_3 = + _mm_maskz_loadu_epi8((0xFFFF >> (16 - *((int *)input_len + (0 + 4 * 3)))) * (0x1 & load_mask_3), (void *)pa_input[0 + 4 * 3]); + + input_block_0 = _mm_shuffle_epi8(input_block_0, M128(swapEndianness)); + input_block_1 = _mm_shuffle_epi8(input_block_1, M128(swapEndianness)); + input_block_2 = _mm_shuffle_epi8(input_block_2, M128(swapEndianness)); + input_block_3 = _mm_shuffle_epi8(input_block_3, M128(swapEndianness)); + + *(data_blocks[0]) = insert32x4(*(data_blocks[0]), input_block_0, 0); + *(data_blocks[1]) = insert32x4(*(data_blocks[1]), input_block_1, 0); + *(data_blocks[2]) = insert32x4(*(data_blocks[2]), input_block_2, 0); + *(data_blocks[3]) = insert32x4(*(data_blocks[3]), input_block_3, 0); + + load_mask_0 >>= 1; + load_mask_1 >>= 1; + load_mask_2 >>= 1; + load_mask_3 >>= 1; + + input_block_0 = _mm_maskz_loadu_epi8((0xFFFF >> (16 - *((int *)input_len + (1 + 4 * 0)))) * (0x1 & load_mask_0), (void *)pa_input[1 + 4 * 0]); + input_block_1 = _mm_maskz_loadu_epi8((0xFFFF >> (16 - *((int *)input_len + (1 + 4 * 1)))) * (0x1 & load_mask_1), (void *)pa_input[1 + 4 * 1]); + input_block_2 = _mm_maskz_loadu_epi8((0xFFFF >> (16 - *((int *)input_len + (1 + 4 * 2)))) * (0x1 & load_mask_2), (void *)pa_input[1 + 4 * 2]); + input_block_3 = _mm_maskz_loadu_epi8((0xFFFF >> (16 - *((int *)input_len + (1 + 4 * 3)))) * (0x1 & load_mask_3), (void *)pa_input[1 + 4 * 3]); + + input_block_0 = _mm_shuffle_epi8(input_block_0, M128(swapEndianness)); + input_block_1 = _mm_shuffle_epi8(input_block_1, M128(swapEndianness)); + input_block_2 = _mm_shuffle_epi8(input_block_2, M128(swapEndianness)); + input_block_3 = _mm_shuffle_epi8(input_block_3, M128(swapEndianness)); + + *(data_blocks[0]) = insert32x4(*(data_blocks[0]), input_block_0, 1); + *(data_blocks[1]) = insert32x4(*(data_blocks[1]), input_block_1, 1); + *(data_blocks[2]) = insert32x4(*(data_blocks[2]), input_block_2, 1); + *(data_blocks[3]) = insert32x4(*(data_blocks[3]), input_block_3, 1); + + load_mask_0 >>= 1; + load_mask_1 >>= 1; + load_mask_2 >>= 1; + load_mask_3 >>= 1; + + input_block_0 = _mm_maskz_loadu_epi8((0xFFFF >> (16 - *((int *)input_len + (2 + 4 * 0)))) * (0x1 & load_mask_0), (void *)pa_input[2 + 4 * 0]); + input_block_1 = _mm_maskz_loadu_epi8((0xFFFF >> (16 - *((int *)input_len + (2 + 4 * 1)))) * (0x1 & load_mask_1), (void *)pa_input[2 + 4 * 1]); + input_block_2 = _mm_maskz_loadu_epi8((0xFFFF >> (16 - *((int *)input_len + (2 + 4 * 2)))) * (0x1 & load_mask_2), (void *)pa_input[2 + 4 * 2]); + input_block_3 = _mm_maskz_loadu_epi8((0xFFFF >> (16 - *((int *)input_len + (2 + 4 * 3)))) * (0x1 & load_mask_3), (void *)pa_input[2 + 4 * 3]); + + input_block_0 = _mm_shuffle_epi8(input_block_0, M128(swapEndianness)); + input_block_1 = _mm_shuffle_epi8(input_block_1, M128(swapEndianness)); + input_block_2 = _mm_shuffle_epi8(input_block_2, M128(swapEndianness)); + input_block_3 = _mm_shuffle_epi8(input_block_3, M128(swapEndianness)); + + *(data_blocks[0]) = insert32x4(*(data_blocks[0]), input_block_0, 2); + *(data_blocks[1]) = insert32x4(*(data_blocks[1]), input_block_1, 2); + *(data_blocks[2]) = insert32x4(*(data_blocks[2]), input_block_2, 2); + *(data_blocks[3]) = insert32x4(*(data_blocks[3]), input_block_3, 2); + + load_mask_0 >>= 1; + load_mask_1 >>= 1; + load_mask_2 >>= 1; + load_mask_3 >>= 1; + + input_block_0 = _mm_maskz_loadu_epi8((0xFFFF >> (16 - *((int *)input_len + (3 + 4 * 0)))) * (0x1 & load_mask_0), (void *)pa_input[3 + 4 * 0]); + input_block_1 = _mm_maskz_loadu_epi8((0xFFFF >> (16 - *((int *)input_len + (3 + 4 * 1)))) * (0x1 & load_mask_1), (void *)pa_input[3 + 4 * 1]); + input_block_2 = _mm_maskz_loadu_epi8((0xFFFF >> (16 - *((int *)input_len + (3 + 4 * 2)))) * (0x1 & load_mask_2), (void *)pa_input[3 + 4 * 2]); + input_block_3 = _mm_maskz_loadu_epi8((0xFFFF >> (16 - *((int *)input_len + (3 + 4 * 3)))) * (0x1 & load_mask_3), (void *)pa_input[3 + 4 * 3]); + + input_block_0 = _mm_shuffle_epi8(input_block_0, M128(swapEndianness)); + input_block_1 = _mm_shuffle_epi8(input_block_1, M128(swapEndianness)); + input_block_2 = _mm_shuffle_epi8(input_block_2, M128(swapEndianness)); + input_block_3 = _mm_shuffle_epi8(input_block_3, M128(swapEndianness)); + + *(data_blocks[0]) = insert32x4(*(data_blocks[0]), input_block_0, 3); + *(data_blocks[1]) = insert32x4(*(data_blocks[1]), input_block_1, 3); + *(data_blocks[2]) = insert32x4(*(data_blocks[2]), input_block_2, 3); + *(data_blocks[3]) = insert32x4(*(data_blocks[3]), input_block_3, 3); + + /* End of explicitly unrolled loop */ +} + +/* +// This function updates 16 blocks of ghash by processing partial blocks ( < 128 bit blocks) of data +// +// This code is a port of GCM part of Intel(R) IPSec AES-GCM code +// Code modified to work with multi-buffer approach +*/ +void sm4_gcm_update_ghash_partial_blocks_mb16(__m128i ghash[SM4_LINES], + const int8u *const pa_input[SM4_LINES], + __m512i *input_len, + __m128i hashkey[SM4_LINES], + __mmask16 mb_mask) +{ + __m512i hashkeys_4_0, hashkeys_4_1, hashkeys_4_2, hashkeys_4_3; + __m512i *hashkeys[] = { &hashkeys_4_0, &hashkeys_4_1, &hashkeys_4_2, &hashkeys_4_3 }; + + __m512i data_blocks_4_0, data_blocks_4_1, data_blocks_4_2, data_blocks_4_3; + __m512i *data_blocks[] = { &data_blocks_4_0, &data_blocks_4_1, &data_blocks_4_2, &data_blocks_4_3 }; + + __mmask16 load_mask = ~cmp_epi32_mask((*input_len), setzero(), _MM_CMPINT_EQ) & mb_mask; + + if (load_mask) { + + hashkeys_4_0 = loadu(hashkey + 0); + hashkeys_4_1 = loadu(hashkey + 4); + hashkeys_4_2 = loadu(hashkey + 8); + hashkeys_4_3 = loadu(hashkey + 12); + + read_first(data_blocks, pa_input, input_len, load_mask); + + data_blocks_4_0 = xor(data_blocks_4_0, M512(ghash + 0)); + data_blocks_4_1 = xor(data_blocks_4_1, M512(ghash + 4)); + data_blocks_4_2 = xor(data_blocks_4_2, M512(ghash + 8)); + data_blocks_4_3 = xor(data_blocks_4_3, M512(ghash + 12)); + + sm4_gcm_ghash_mul_single_block_mb16(data_blocks, hashkeys); + + __mmask8 store_mask_0 = 0x03 * (0x1 & ((load_mask >> 0 * 4) >> 0)) | // + 0x0C * (0x1 & ((load_mask >> 0 * 4) >> 1)) | // + 0x30 * (0x1 & ((load_mask >> 0 * 4) >> 2)) | // + 0xC0 * (0x1 & ((load_mask >> 0 * 4) >> 3)); // + + __mmask8 store_mask_1 = 0x03 * (0x1 & ((load_mask >> 1 * 4) >> 0)) | // + 0x0C * (0x1 & ((load_mask >> 1 * 4) >> 1)) | // + 0x30 * (0x1 & ((load_mask >> 1 * 4) >> 2)) | // + 0xC0 * (0x1 & ((load_mask >> 1 * 4) >> 3)); // + + __mmask8 store_mask_2 = 0x03 * (0x1 & ((load_mask >> 2 * 4) >> 0)) | // + 0x0C * (0x1 & ((load_mask >> 2 * 4) >> 1)) | // + 0x30 * (0x1 & ((load_mask >> 2 * 4) >> 2)) | // + 0xC0 * (0x1 & ((load_mask >> 2 * 4) >> 3)); // + + __mmask8 store_mask_3 = 0x03 * (0x1 & ((load_mask >> 3 * 4) >> 0)) | // + 0x0C * (0x1 & ((load_mask >> 3 * 4) >> 1)) | // + 0x30 * (0x1 & ((load_mask >> 3 * 4) >> 2)) | // + 0xC0 * (0x1 & ((load_mask >> 3 * 4) >> 3)); // + + mask_storeu_epi64(ghash + 0, store_mask_0, data_blocks_4_0); + mask_storeu_epi64(ghash + 4, store_mask_1, data_blocks_4_1); + mask_storeu_epi64(ghash + 8, store_mask_2, data_blocks_4_2); + mask_storeu_epi64(ghash + 12, store_mask_3, data_blocks_4_3); + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_update_iv_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_update_iv_mb16.c new file mode 100644 index 0000000..fb84911 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/internal/sm4_gcm_update_iv_mb16.c @@ -0,0 +1,86 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include + +/* +// This function process 16 buffers with initialization vector (IV) data +*/ +__mmask16 sm4_gcm_update_iv_mb16(const int8u *const pa_iv[SM4_LINES], const int iv_len[SM4_LINES], __mmask16 mb_mask, SM4_GCM_CTX_mb16 *p_context) +{ + SM4_GCM_CONTEXT_STATE(p_context) = sm4_gcm_update_iv; + + __m128i *j0 = SM4_GCM_CONTEXT_J0(p_context); + + const int8u *loc_pa_iv[SM4_LINES]; + int iv_len_rearranged[SM4_LINES]; + + /* Rearrange pointers and lengths to right layout */ + rearrange(loc_pa_iv, pa_iv); + rearrange(iv_len_rearranged, iv_len); + + __m512i loc_iv_len = loadu(iv_len_rearranged); + __m512i max_iv_len = set1_epi64(0x1FFFFFFFFFFFFFFF); /* (2^64 - 1) div 8 */ + + /* + // Update full IV length + // + // IV length is passed as 32 bit integer + // IV length is used to construct last block for ghash computation as follow: + // 64 zero bits | 64 IV len bits + // Length of IV is stored in context in 64 bits + // + // Code below transforms 32 bit input integers into the following block: + // 32 zero bits | 32 IV len bits + // and add it to length stored in context + // + // The whole operation is following for each buffer: + // 64 bits with IV len in context + // + + // 32 zero bits | 32 bits with input IV len + // + // Last J0 block is constructed in IV finalization + */ + + __m512i full_iv_len = maskz_expandloadu_epi32(0x5555, iv_len_rearranged); + full_iv_len = add_epi64(full_iv_len, loadu(BUFFER_REG_NUM(SM4_GCM_CONTEXT_LEN(p_context), 0))); + storeu(BUFFER_REG_NUM(SM4_GCM_CONTEXT_LEN(p_context), 0), full_iv_len); + + __mmask8 mask_overflow_lo = cmp_epi64_mask(max_iv_len, full_iv_len, _MM_CMPINT_LT); + + full_iv_len = maskz_expandloadu_epi32(0x5555, iv_len_rearranged + 8); + full_iv_len = add_epi64(full_iv_len, loadu(BUFFER_REG_NUM(SM4_GCM_CONTEXT_LEN(p_context), 1))); + storeu(BUFFER_REG_NUM(SM4_GCM_CONTEXT_LEN(p_context), 1), full_iv_len); + + __mmask8 mask_overflow_hi = cmp_epi64_mask(max_iv_len, full_iv_len, _MM_CMPINT_LT); + + __mmask16 mask_overflow = mask_overflow_hi << 8 | mask_overflow_lo; + + /* Process full blocks of IVs */ + sm4_gcm_update_ghash_full_blocks_mb16(j0, loc_pa_iv, &loc_iv_len, SM4_GCM_CONTEXT_HASHKEY(p_context), mb_mask); + + if (cmp_epi32_mask(loc_iv_len, setzero(), _MM_CMPINT_EQ) != 0xFFFF) { + /* Process partial blocks of IVs and finalize IVs */ + sm4_gcm_update_ghash_partial_blocks_mb16(j0, loc_pa_iv, &loc_iv_len, SM4_GCM_CONTEXT_HASHKEY(p_context)[0], mb_mask); + sm4_gcm_finalize_iv_mb16(loc_pa_iv, mb_mask, p_context); + + SM4_GCM_CONTEXT_STATE(p_context) = sm4_gcm_update_aad; + } + + return mask_overflow; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/sm4_gcm_decrypt_mb16_api.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/sm4_gcm_decrypt_mb16_api.c new file mode 100644 index 0000000..effcefd --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/sm4_gcm_decrypt_mb16_api.c @@ -0,0 +1,77 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include + +DLL_PUBLIC +mbx_status16 +mbx_sm4_gcm_decrypt_mb16(int8u *pa_out[SM4_LINES], const int8u *const pa_in[SM4_LINES], const int in_len[SM4_LINES], SM4_GCM_CTX_mb16 *p_context) +{ + int buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + + /* Test input pointers */ + if (NULL == pa_out || NULL == pa_in || NULL == in_len || NULL == p_context) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* Check state */ + if (sm4_gcm_update_iv != SM4_GCM_CONTEXT_STATE(p_context) && sm4_gcm_update_aad != SM4_GCM_CONTEXT_STATE(p_context) && + sm4_gcm_start_encdec != SM4_GCM_CONTEXT_STATE(p_context) && sm4_gcm_dec != SM4_GCM_CONTEXT_STATE(p_context)) { + + status = MBX_SET_STS16_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* Don't process buffers with input pointers equal to zero */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_out[buf_no] == NULL || pa_in[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + mb_mask &= ~(0x1 << rearrangeOrder[buf_no]); + } + if (in_len[buf_no] < 0) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + mb_mask &= ~(0x1 << rearrangeOrder[buf_no]); + } + /* We take elements from the SM4_GCM_CONTEXT_LEN(p_context) array by the formula [1+buf_no*2], because of the + layout of length data in the context. Refer to sources/ippcp/crypto_mb/include/crypto_mb/sm4_gcm.h file for details. */ + int length_data_position = 1+buf_no*2; + if( ((int64u)in_len[buf_no] >= MAX_TXT_LEN) || + (SM4_GCM_CONTEXT_LEN(p_context)[length_data_position] >= MAX_TXT_LEN - (int64u)in_len[buf_no]) || + ((SM4_GCM_CONTEXT_LEN(p_context)[length_data_position] + (int64u)in_len[buf_no]) < (int64u)in_len[buf_no])) { + + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + mb_mask &= ~(0x1 << rearrangeOrder[buf_no]); + } + } + + if (MBX_IS_ANY_OK_STS16(status)) { + __mmask16 overflow_mask = sm4_gcm_decrypt_mb16(pa_out, pa_in, in_len, mb_mask, p_context); + + /* Set bad status for buffers with overflowed lengths */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (overflow_mask >> buf_no & 1) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + } + } + } + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/sm4_gcm_encrypt_mb16_api.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/sm4_gcm_encrypt_mb16_api.c new file mode 100644 index 0000000..04baf35 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/sm4_gcm_encrypt_mb16_api.c @@ -0,0 +1,77 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include + +DLL_PUBLIC +mbx_status16 +mbx_sm4_gcm_encrypt_mb16(int8u *pa_out[SM4_LINES], const int8u *const pa_in[SM4_LINES], const int in_len[SM4_LINES], SM4_GCM_CTX_mb16 *p_context) +{ + int buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + + /* Test input pointers */ + if (NULL == pa_out || NULL == pa_in || NULL == in_len || NULL == p_context) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* Check state */ + if (sm4_gcm_update_iv != SM4_GCM_CONTEXT_STATE(p_context) && sm4_gcm_update_aad != SM4_GCM_CONTEXT_STATE(p_context) && + sm4_gcm_start_encdec != SM4_GCM_CONTEXT_STATE(p_context) && sm4_gcm_enc != SM4_GCM_CONTEXT_STATE(p_context)) { + + status = MBX_SET_STS16_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* Don't process buffers with input pointers equal to zero */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_out[buf_no] == NULL || pa_in[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + mb_mask &= ~(0x1 << rearrangeOrder[buf_no]); + } + if (in_len[buf_no] < 0) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + mb_mask &= ~(0x1 << rearrangeOrder[buf_no]); + } + /* We take elements from the SM4_GCM_CONTEXT_LEN(p_context) array by the formula [1+buf_no*2], because of the + layout of length data in the context. Refer to sources/ippcp/crypto_mb/include/crypto_mb/sm4_gcm.h file for details. */ + int length_data_position = 1+buf_no*2; + if( ((int64u)in_len[buf_no] >= MAX_TXT_LEN) || + (SM4_GCM_CONTEXT_LEN(p_context)[length_data_position] >= MAX_TXT_LEN - (int64u)in_len[buf_no]) || + ((SM4_GCM_CONTEXT_LEN(p_context)[length_data_position] + (int64u)in_len[buf_no]) < (int64u)in_len[buf_no])) { + + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + mb_mask &= ~(0x1 << rearrangeOrder[buf_no]); + } + } + + if (MBX_IS_ANY_OK_STS16(status)) { + __mmask16 overflow_mask = sm4_gcm_encrypt_mb16(pa_out, pa_in, in_len, mb_mask, p_context); + + /* Set bad status for buffers with overflowed lengths */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (overflow_mask >> buf_no & 1) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + } + } + } + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/sm4_gcm_get_tag_mb16_api.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/sm4_gcm_get_tag_mb16_api.c new file mode 100644 index 0000000..b385d7d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/sm4_gcm_get_tag_mb16_api.c @@ -0,0 +1,59 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include + +DLL_PUBLIC +mbx_status16 mbx_sm4_gcm_get_tag_mb16(int8u *pa_tag[SM4_LINES], const int tag_len[SM4_LINES], SM4_GCM_CTX_mb16 *p_context) +{ + int buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + + /* Test input pointers */ + if (NULL == pa_tag || NULL == tag_len || NULL == p_context) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* Check state */ + if (sm4_gcm_update_aad != SM4_GCM_CONTEXT_STATE(p_context) && sm4_gcm_start_encdec != SM4_GCM_CONTEXT_STATE(p_context) && + sm4_gcm_enc != SM4_GCM_CONTEXT_STATE(p_context) && sm4_gcm_dec != SM4_GCM_CONTEXT_STATE(p_context) && + sm4_gcm_get_tag != SM4_GCM_CONTEXT_STATE(p_context)) { + + status = MBX_SET_STS16_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* Don't process buffers with input pointers equal to zero and set bad status for tags of invalid length */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_tag[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + mb_mask &= ~(0x1 << rearrangeOrder[buf_no]); + } + if (tag_len[buf_no] < 0 || tag_len[buf_no] > 16) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + } + } + + if (MBX_IS_ANY_OK_STS16(status)) { + sm4_gcm_get_tag_mb16(pa_tag, tag_len, mb_mask, p_context); + } + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/sm4_gcm_init_mb16_api.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/sm4_gcm_init_mb16_api.c new file mode 100644 index 0000000..8f146ab --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/sm4_gcm_init_mb16_api.c @@ -0,0 +1,97 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include + +DLL_PUBLIC +mbx_status16 mbx_sm4_gcm_init_mb16(const sm4_key *const pa_key[SM4_LINES], + const int8u *const pa_iv[SM4_LINES], + const int iv_len[SM4_LINES], + SM4_GCM_CTX_mb16 *p_context) +{ + int buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + __mmask16 mb_mask_rearranged = 0xFFFF; + + /* Test input pointers */ + if (NULL == pa_key || NULL == pa_iv || NULL == iv_len || NULL == p_context) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* Don't process buffers with input pointers equal to zero and set bad status for IV with zero length */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_key[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + mb_mask &= ~(0x1 << buf_no); + } + if (pa_iv[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + mb_mask_rearranged &= ~(0x1 << rearrangeOrder[buf_no]); + } + if (iv_len[buf_no] <= 0) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + mb_mask_rearranged &= ~(0x1 << rearrangeOrder[buf_no]); + } + } + + if (MBX_IS_ANY_OK_STS16(status)) { + + /* Clear buffers */ + + SM4_GCM_CLEAR_BUFFER((SM4_GCM_CONTEXT_BUFFER_SLOT_TYPE *)SM4_GCM_CONTEXT_J0(p_context)); + SM4_GCM_CLEAR_BUFFER((SM4_GCM_CONTEXT_BUFFER_SLOT_TYPE *)SM4_GCM_CONTEXT_GHASH(p_context)); + + + SM4_GCM_CLEAR_LEN(BUFFER_REG_NUM(SM4_GCM_CONTEXT_LEN(p_context), 0)); + SM4_GCM_CLEAR_LEN(BUFFER_REG_NUM(SM4_GCM_CONTEXT_LEN(p_context), 1)); + SM4_GCM_CLEAR_LEN(BUFFER_REG_NUM(SM4_GCM_CONTEXT_LEN(p_context), 2)); + SM4_GCM_CLEAR_LEN(BUFFER_REG_NUM(SM4_GCM_CONTEXT_LEN(p_context), 3)); + + /* + // Compute SM4 keys + // initialize int32u mbx_sm4_key_schedule[SM4_ROUNDS][SM4_LINES] buffer in context + // keys layout for each round: + // key0 key4 key8 key12 key1 key5 key9 key13 key2 key6 key10 key14 key3 key7 key11 key15 + */ + + sm4_set_round_keys_mb16((int32u **)SM4_GCM_CONTEXT_KEY(p_context), (const int8u **)pa_key, mb_mask); + + /* + // Compute hashkeys + // initialize __m128i hashkey[SM4_GCM_HASHKEY_PWR_NUM][SM4_LINES] buffer in context + // hashkeys layout for each hashkey power: + // hashkey0 hashkey4 hashkey8 hashkey12 hashkey1 hashkey5 hashkey9 hashkey13 hashkey2 hashkey6 hashkey10 hashkey14 hashkey3 hashkey7 hashkey11 + hashkey15 + */ + sm4_gcm_precompute_hashkey_mb16((const mbx_sm4_key_schedule *)SM4_GCM_CONTEXT_KEY(p_context), p_context); + + /* Process IV */ + __mmask16 overflow_mask = sm4_gcm_update_iv_mb16(pa_iv, iv_len, mb_mask_rearranged, p_context); + + /* Set bad status for buffers with overflowed lengths */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (overflow_mask >> buf_no & 1) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + } + } + } + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/sm4_gcm_update_aad_mb16_api.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/sm4_gcm_update_aad_mb16_api.c new file mode 100644 index 0000000..3c698c8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/sm4_gcm_update_aad_mb16_api.c @@ -0,0 +1,64 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include + +DLL_PUBLIC +mbx_status16 mbx_sm4_gcm_update_aad_mb16(const int8u *const pa_aad[SM4_LINES], const int aad_len[SM4_LINES], SM4_GCM_CTX_mb16 *p_context) +{ + int buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + + /* Test input pointers */ + if (NULL == pa_aad || NULL == aad_len || NULL == p_context) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* Check state */ + if (sm4_gcm_update_iv != SM4_GCM_CONTEXT_STATE(p_context) && sm4_gcm_update_aad != SM4_GCM_CONTEXT_STATE(p_context)) { + status = MBX_SET_STS16_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* Don't process buffers with input pointers equal to zero */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_aad[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + mb_mask &= ~(0x1 << rearrangeOrder[buf_no]); + } + if (aad_len[buf_no] < 0) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + mb_mask &= ~(0x1 << rearrangeOrder[buf_no]); + } + } + + if (MBX_IS_ANY_OK_STS16(status)) { + __mmask16 overflow_mask = sm4_gcm_update_aad_mb16(pa_aad, aad_len, mb_mask, p_context); + + /* Set bad status for buffers with overflowed lengths */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (overflow_mask >> buf_no & 1) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + } + } + } + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/sm4_gcm_update_iv_mb16_api.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/sm4_gcm_update_iv_mb16_api.c new file mode 100644 index 0000000..d190e90 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/gcm/sm4_gcm_update_iv_mb16_api.c @@ -0,0 +1,64 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include +#include + +DLL_PUBLIC +mbx_status16 mbx_sm4_gcm_update_iv_mb16(const int8u *const pa_iv[SM4_LINES], const int iv_len[SM4_LINES], SM4_GCM_CTX_mb16 *p_context) +{ + int buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + + /* Check input pointers */ + if (NULL == pa_iv || NULL == iv_len || NULL == p_context) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* Check state */ + if (sm4_gcm_update_iv != SM4_GCM_CONTEXT_STATE(p_context)) { + status = MBX_SET_STS16_ALL(MBX_STATUS_MISMATCH_PARAM_ERR); + return status; + } + + /* Don't process buffers with input pointers equal to zero and set bad status for IV with zero length */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_iv[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + mb_mask &= ~(0x1 << rearrangeOrder[buf_no]); + } + if (iv_len[buf_no] < 0) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + mb_mask &= ~(0x1 << rearrangeOrder[buf_no]); + } + } + + if (MBX_IS_ANY_OK_STS16(status)) { + __mmask16 overflow_mask = sm4_gcm_update_iv_mb16(pa_iv, iv_len, mb_mask, p_context); + + /* Set bad status for buffers with overflowed lengths */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (overflow_mask >> buf_no & 1) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + } + } + } + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_cbc_dec_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_cbc_dec_mb16.c new file mode 100644 index 0000000..a1c3629 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_cbc_dec_mb16.c @@ -0,0 +1,55 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include + +DLL_PUBLIC +mbx_status16 mbx_sm4_decrypt_cbc_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, const int8u* pa_iv[SM4_LINES]) +{ + int buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + + /* Test input pointers */ + if (NULL == pa_out || NULL == pa_inp || NULL == len || NULL == key_sched || NULL == pa_iv) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* Test input data length and integrity for each buffer */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_out[buf_no] == NULL || pa_inp[buf_no] == NULL || pa_iv[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + /* Do not process empty buffers */ + mb_mask &= ~(0x1 << buf_no); + } + if ((len[buf_no] < 0) || (len[buf_no]&(SM4_BLOCK_SIZE-1))) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + /* Do not process non-valid buffers */ + mb_mask &= ~(0x1 << buf_no); + } + } + + if (MBX_IS_ANY_OK_STS16(status)) + sm4_cbc_dec_kernel_mb16(pa_out, pa_inp, len, (const int32u**)key_sched, mb_mask, pa_iv); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_cbc_enc_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_cbc_enc_mb16.c new file mode 100644 index 0000000..e67820c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_cbc_enc_mb16.c @@ -0,0 +1,55 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include + +DLL_PUBLIC +mbx_status16 mbx_sm4_encrypt_cbc_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, const int8u* pa_iv[SM4_LINES]) +{ + int buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + + /* Test input pointers */ + if (NULL == pa_out || NULL == pa_inp || NULL == len || NULL == key_sched || NULL == pa_iv) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* Test input data length and integrity for each buffer */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_out[buf_no] == NULL || pa_inp[buf_no] == NULL || pa_iv[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + /* Do not process empty buffers */ + mb_mask &= ~(0x1 << buf_no); + } + if ((len[buf_no] < 0) || (len[buf_no]&(SM4_BLOCK_SIZE-1))) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + /* Do not process non-valid buffers */ + mb_mask &= ~(0x1 << buf_no); + } + } + + if (MBX_IS_ANY_OK_STS16(status)) + sm4_cbc_enc_kernel_mb16(pa_out, pa_inp, len, (const int32u**)key_sched, mb_mask, pa_iv); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_cbc_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_cbc_mb16.c new file mode 100644 index 0000000..2f2ac77 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_cbc_mb16.c @@ -0,0 +1,559 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include +#include + +void sm4_cbc_enc_kernel_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const int32u* key_sched[SM4_ROUNDS], __mmask16 mb_mask, const int8u* pa_iv[SM4_LINES]) +{ + __ALIGN64 const int8u* loc_inp[SM4_LINES]; + __ALIGN64 int8u* loc_out[SM4_LINES]; + + /* Length of the input data in 128-bit chunks - number of SM4 blocks */ + __m512i num_blocks; + GET_NUM_BLOCKS(num_blocks, len, SM4_BLOCK_SIZE); + + /* Local copies of the pointers to input and output buffers */ + _mm512_storeu_si512((void*)loc_inp, _mm512_loadu_si512(pa_inp)); + _mm512_storeu_si512((void*)(loc_inp + 8), _mm512_loadu_si512(pa_inp + 8)); + + _mm512_storeu_si512(loc_out, _mm512_loadu_si512(pa_out)); + _mm512_storeu_si512(loc_out + 8, _mm512_loadu_si512(pa_out + 8)); + + /* Set p_rk pointer to the beginning of the key schedule */ + const __m512i* p_rk = (const __m512i*)key_sched; + + /* Check if we have any data */ + __mmask16 tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, num_blocks, _mm512_setzero_si512(), _MM_CMPINT_NLE); + + __m512i iv0, iv1, iv2, iv3; + __m512i z0, z1, z2, z3, xi; + + /* Load and transpose iv */ + TRANSPOSE_16x4_I32_EPI32(&iv0, &iv1, &iv2, &iv3, pa_iv, tmp_mask); + + while (tmp_mask) { + /* Transpose input data */ + TRANSPOSE_16x4_I32_EPI32(&z0, &z1, &z2, &z3, loc_inp, tmp_mask); + + z0 = _mm512_xor_epi32(z0, iv0); + z1 = _mm512_xor_epi32(z1, iv1); + z2 = _mm512_xor_epi32(z2, iv2); + z3 = _mm512_xor_epi32(z3, iv3); + + for (int itr = 0; itr < SM4_ROUNDS; itr += 4, p_rk += 4) + SM4_FOUR_ROUNDS(z0, z1, z2, z3, xi, p_rk, 1); + + p_rk -= SM4_ROUNDS; + + iv0 = z3; + iv1 = z2; + iv2 = z1; + iv3 = z0; + + /* Transpose and store encrypted blocks */ + TRANSPOSE_4x16_I32_EPI32(&z0, &z1, &z2, &z3, loc_out, tmp_mask); + + /* Update pointers to data */ + _mm512_storeu_si512((void*)loc_inp, _mm512_add_epi64(_mm512_loadu_si512(loc_inp), _mm512_set1_epi64(SM4_BLOCK_SIZE))); + _mm512_storeu_si512((void*)(loc_inp + 8), _mm512_add_epi64(_mm512_loadu_si512(loc_inp + 8), _mm512_set1_epi64(SM4_BLOCK_SIZE))); + + _mm512_storeu_si512(loc_out, _mm512_add_epi64(_mm512_loadu_si512(loc_out), _mm512_set1_epi64(SM4_BLOCK_SIZE))); + _mm512_storeu_si512((loc_out + 8), _mm512_add_epi64(_mm512_loadu_si512(loc_out + 8), _mm512_set1_epi64(SM4_BLOCK_SIZE))); + + /* Update number of blocks left and processing mask */ + num_blocks = _mm512_sub_epi32(num_blocks, _mm512_set1_epi32(1)); + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, num_blocks, _mm512_setzero_si512(), _MM_CMPINT_NLE); + } + + /* clear local copy of sensitive data */ + zero_mb8((int64u(*)[8])&z0, 1); + zero_mb8((int64u(*)[8])&z1, 1); + zero_mb8((int64u(*)[8])&z2, 1); + zero_mb8((int64u(*)[8])&z3, 1); + zero_mb8((int64u(*)[8])&xi, 1); +} + +/* + * Perform SM4-CBC-MAC on 16 buffers and generate their authentication tag. + * + * @param[out] pa_out Array of pointers to authentication tag + * @param[in] pa_in Array of pointers to input buffers + * @param[in] len Array of buffer lengths + * @param[in] key_sched Array of SM4 scheduled keys + * @param[in] mb_mask Bitmask selecting which lines to generate tag + * @param[in] pa_iv Array of IV pointers + */ +void sm4_cbc_mac_kernel_mb16(__m128i pa_out[SM4_LINES], const int8u *const pa_inp[SM4_LINES], + const int len[SM4_LINES], const int32u* key_sched[SM4_ROUNDS], + __mmask16 mb_mask, const int8u *pa_iv[SM4_LINES]) +{ + __ALIGN64 const int8u* loc_inp[SM4_LINES]; + + /* Length of the input data in 128-bit chunks - number of SM4 blocks */ + __m512i num_blocks; + GET_NUM_BLOCKS(num_blocks, len, SM4_BLOCK_SIZE); + + /* Local copies of the pointers to input buffers */ + _mm512_storeu_si512((void*)loc_inp, _mm512_loadu_si512(pa_inp)); + _mm512_storeu_si512((void*)(loc_inp + 8), _mm512_loadu_si512(pa_inp + 8)); + + /* Set p_rk pointer to the beginning of the key schedule */ + const __m512i* p_rk = (const __m512i*)key_sched; + + /* Check if we have any data */ + __mmask16 tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, num_blocks, _mm512_setzero_si512(), _MM_CMPINT_NLE); + __m512i iv0, iv1, iv2, iv3; + __m512i z0, z1, z2, z3, xi; + __m512i in0, in1, in2, in3; + + mb_mask = tmp_mask; + + z0 = _mm512_setzero_si512(); + z1 = _mm512_setzero_si512(); + z2 = _mm512_setzero_si512(); + z3 = _mm512_setzero_si512(); + + /* Load and transpose iv */ + TRANSPOSE_16x4_I32_EPI32(&iv0, &iv1, &iv2, &iv3, pa_iv, tmp_mask); + + while (tmp_mask) { + /* Transpose input data */ + TRANSPOSE_16x4_I32_EPI32(&in0, &in1, &in2, &in3, loc_inp, tmp_mask); + + z0 = _mm512_mask_xor_epi32(z0, tmp_mask, in0, iv0); + z1 = _mm512_mask_xor_epi32(z1, tmp_mask, in1, iv1); + z2 = _mm512_mask_xor_epi32(z2, tmp_mask, in2, iv2); + z3 = _mm512_mask_xor_epi32(z3, tmp_mask, in3, iv3); + + for (int itr = 0; itr < SM4_ROUNDS; itr += 4, p_rk += 4) + SM4_FOUR_ROUNDS_MASKED(z0, z1, z2, z3, xi, tmp_mask, p_rk, 1); + + p_rk -= SM4_ROUNDS; + + iv0 = z3; + iv1 = z2; + iv2 = z1; + iv3 = z0; + + /* Update pointers to data */ + _mm512_storeu_si512((void*)loc_inp, _mm512_add_epi64(_mm512_loadu_si512(loc_inp), _mm512_set1_epi64(SM4_BLOCK_SIZE))); + _mm512_storeu_si512((void*)(loc_inp + 8), _mm512_add_epi64(_mm512_loadu_si512(loc_inp + 8), _mm512_set1_epi64(SM4_BLOCK_SIZE))); + + /* Update number of blocks left and processing mask */ + num_blocks = _mm512_sub_epi32(num_blocks, _mm512_set1_epi32(1)); + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, num_blocks, _mm512_setzero_si512(), _MM_CMPINT_NLE); + } + + /* Transpose and store result MAC */ + TRANSPOSE_4x16_I32_O128_EPI32(&z0, &z1, &z2, &z3, pa_out, mb_mask); + + /* clear local copy of sensitive data */ + zero_mb8((int64u(*)[8])&z0, 1); + zero_mb8((int64u(*)[8])&z1, 1); + zero_mb8((int64u(*)[8])&z2, 1); + zero_mb8((int64u(*)[8])&z3, 1); + zero_mb8((int64u(*)[8])&xi, 1); +} + +static void sm4_cbc_dec_incomplete_buff_mb16(const int8u* loc_inp[SM4_LINES], int8u* loc_out[SM4_LINES], + __m512i num_blocks, const __m512i* p_rk, + __mmask16 mb_mask, + __m512i TMP[20], __m512i STORED_CT[16]); + +void sm4_cbc_dec_kernel_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const int32u* key_sched[SM4_ROUNDS], __mmask16 mb_mask, const int8u* pa_iv[SM4_LINES]) +{ + const int8u* loc_inp[SM4_LINES]; + int8u* loc_out[SM4_LINES]; + + /* Load the constant value */ + const __m512i swap_m512i = _mm512_loadu_si512(swapBytes); + + /* Registers to store ciphertext blocks to be XOR'ed with output of SM4 cipher stage */ + __m512i STORED_CT[16]; + + /* Length of the input data in 128-bit chunks - number of SM4 blocks */ + __m512i num_blocks; + GET_NUM_BLOCKS(num_blocks, len, SM4_BLOCK_SIZE); + + /* Don't process empty buffers */ + mb_mask = _mm512_mask_cmp_epi32_mask(mb_mask, num_blocks, _mm512_setzero_si512(), _MM_CMPINT_NE); + + /* Local copies of the pointers to input and output buffers */ + _mm512_storeu_si512((void*)loc_inp, _mm512_loadu_si512(pa_inp)); + _mm512_storeu_si512((void*)(loc_inp + 8), _mm512_loadu_si512(pa_inp + 8)); + + _mm512_storeu_si512(loc_out, _mm512_loadu_si512(pa_out)); + _mm512_storeu_si512(loc_out + 8, _mm512_loadu_si512(pa_out + 8)); + + /* Set p_rk pointer to the end of the key schedule */ + const __m512i* p_rk = (const __m512i*)key_sched + (SM4_ROUNDS - 1); + + __ALIGN64 __m512i TMP[20]; + __mmask16 loc_mb_mask = mb_mask; + + /* Store first ciphertext block for next round */ + for (int i = 0; i < SM4_LINES; i++) { + STORED_CT[i] = _mm512_setzero_epi32(); + __m128i data_block = _mm_maskz_loadu_epi32(0x000F * (0x1&loc_mb_mask), loc_inp[i]); + STORED_CT[i] = _mm512_inserti64x2(STORED_CT[i], data_block, 3); + loc_mb_mask >>= 1; + } + + /* Process the first block from each buffer, because it contains IV specific */ + /* Load and transpose input data */ + TRANSPOSE_16x4_I32_EPI32(&TMP[0], &TMP[1], &TMP[2], &TMP[3], loc_inp, mb_mask); + + for (int itr = 0; itr < SM4_ROUNDS; itr += 4, p_rk -= 4) + SM4_FOUR_ROUNDS(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], p_rk, -1); + p_rk += SM4_ROUNDS; + + /* Transpose and store first encrypted block for each buffer */ + TRANSPOSE_AND_XOR_4x16_I32_EPI32(&TMP[0], &TMP[1], &TMP[2], &TMP[3], loc_out, pa_iv, mb_mask); + + /* Update pointers to data */ + _mm512_storeu_si512((void*)loc_inp, _mm512_add_epi64(_mm512_loadu_si512(loc_inp), _mm512_set1_epi64(SM4_BLOCK_SIZE))); + _mm512_storeu_si512((void*)(loc_inp + 8), _mm512_add_epi64(_mm512_loadu_si512(loc_inp + 8), _mm512_set1_epi64(SM4_BLOCK_SIZE))); + + _mm512_storeu_si512(loc_out, _mm512_add_epi64(_mm512_loadu_si512(loc_out), _mm512_set1_epi64(SM4_BLOCK_SIZE))); + _mm512_storeu_si512((loc_out + 8), _mm512_add_epi64(_mm512_loadu_si512(loc_out + 8), _mm512_set1_epi64(SM4_BLOCK_SIZE))); + + num_blocks = _mm512_sub_epi32(num_blocks, _mm512_set1_epi32(1)); + + /* Generate the mask to process 4 blocks from each buffer */ + __mmask16 tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, num_blocks, _mm512_set1_epi32(4), _MM_CMPINT_NLT); + + /* Go to this loop if all 16 buffers contain at least 4 blocks each */ + while (tmp_mask == 0xFFFF) { + TMP[0] = _mm512_loadu_si512(loc_inp[0]); + TMP[1] = _mm512_loadu_si512((__m512i*)(loc_inp[1])); + TMP[2] = _mm512_loadu_si512((__m512i*)(loc_inp[2])); + TMP[3] = _mm512_loadu_si512((__m512i*)(loc_inp[3])); + TMP[0] = _mm512_shuffle_epi8(TMP[0], swap_m512i); + TMP[1] = _mm512_shuffle_epi8(TMP[1], swap_m512i); + TMP[2] = _mm512_shuffle_epi8(TMP[2], swap_m512i); + TMP[3] = _mm512_shuffle_epi8(TMP[3], swap_m512i); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_loadu_si512((__m512i*)(loc_inp[4])); + TMP[1] = _mm512_loadu_si512((__m512i*)(loc_inp[5])); + TMP[2] = _mm512_loadu_si512((__m512i*)(loc_inp[6])); + TMP[3] = _mm512_loadu_si512((__m512i*)(loc_inp[7])); + TMP[0] = _mm512_shuffle_epi8(TMP[0], swap_m512i); + TMP[1] = _mm512_shuffle_epi8(TMP[1], swap_m512i); + TMP[2] = _mm512_shuffle_epi8(TMP[2], swap_m512i); + TMP[3] = _mm512_shuffle_epi8(TMP[3], swap_m512i); + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_loadu_si512((__m512i*)(loc_inp[8])); + TMP[1] = _mm512_loadu_si512((__m512i*)(loc_inp[9])); + TMP[2] = _mm512_loadu_si512((__m512i*)(loc_inp[10])); + TMP[3] = _mm512_loadu_si512((__m512i*)(loc_inp[11])); + TMP[0] = _mm512_shuffle_epi8(TMP[0], swap_m512i); + TMP[1] = _mm512_shuffle_epi8(TMP[1], swap_m512i); + TMP[2] = _mm512_shuffle_epi8(TMP[2], swap_m512i); + TMP[3] = _mm512_shuffle_epi8(TMP[3], swap_m512i); + TRANSPOSE_INP_512(TMP[12], TMP[13], TMP[14], TMP[15], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_loadu_si512((__m512i*)(loc_inp[12])); + TMP[1] = _mm512_loadu_si512((__m512i*)(loc_inp[13])); + TMP[2] = _mm512_loadu_si512((__m512i*)(loc_inp[14])); + TMP[3] = _mm512_loadu_si512((__m512i*)(loc_inp[15])); + TMP[0] = _mm512_shuffle_epi8(TMP[0], swap_m512i); + TMP[1] = _mm512_shuffle_epi8(TMP[1], swap_m512i); + TMP[2] = _mm512_shuffle_epi8(TMP[2], swap_m512i); + TMP[3] = _mm512_shuffle_epi8(TMP[3], swap_m512i); + TRANSPOSE_INP_512(TMP[16], TMP[17], TMP[18], TMP[19], TMP[0], TMP[1], TMP[2], TMP[3]); + + SM4_KERNEL(TMP, p_rk, -1); + + p_rk += SM4_ROUNDS; + + for (int i = 0; i < SM4_LINES; i++) + STORED_CT[i] = _mm512_alignr_epi64(_mm512_loadu_si512(loc_inp[i]), STORED_CT[i], 6); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], swap_m512i); + TMP[1] = _mm512_shuffle_epi8(TMP[1], swap_m512i); + TMP[2] = _mm512_shuffle_epi8(TMP[2], swap_m512i); + TMP[3] = _mm512_shuffle_epi8(TMP[3], swap_m512i); + /* xor with IVs */ + TMP[4] = _mm512_xor_si512(TMP[0], STORED_CT[0]); + TMP[5] = _mm512_xor_si512(TMP[1], STORED_CT[1]); + TMP[6] = _mm512_xor_si512(TMP[2], STORED_CT[2]); + TMP[7] = _mm512_xor_si512(TMP[3], STORED_CT[3]); + /* Store last block of ciphertext for next iteration */ + for (int i = 0; i < 4; i++) + STORED_CT[i] = _mm512_inserti64x2(STORED_CT[i], _mm_loadu_si128((__m128i const*)loc_inp[i] + 3), 3); + _mm512_storeu_si512((__m512i*)(loc_out[0]), TMP[4]); + _mm512_storeu_si512((__m512i*)(loc_out[1]), TMP[5]); + _mm512_storeu_si512((__m512i*)(loc_out[2]), TMP[6]); + _mm512_storeu_si512((__m512i*)(loc_out[3]), TMP[7]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], swap_m512i); + TMP[1] = _mm512_shuffle_epi8(TMP[1], swap_m512i); + TMP[2] = _mm512_shuffle_epi8(TMP[2], swap_m512i); + TMP[3] = _mm512_shuffle_epi8(TMP[3], swap_m512i); + /* xor with IVs */ + TMP[8] = _mm512_xor_si512(TMP[0], STORED_CT[4]); + TMP[9] = _mm512_xor_si512(TMP[1], STORED_CT[5]); + TMP[10] = _mm512_xor_si512(TMP[2], STORED_CT[6]); + TMP[11] = _mm512_xor_si512(TMP[3], STORED_CT[7]); + /* Store last block of ciphertext for next iteration */ + for (int i = 4; i < 8; i++) + STORED_CT[i] = _mm512_inserti64x2(STORED_CT[i], _mm_loadu_si128((__m128i const*)loc_inp[i] + 3), 3); + _mm512_storeu_si512((__m512i*)(loc_out[4]), TMP[8]); + _mm512_storeu_si512((__m512i*)(loc_out[5]), TMP[9]); + _mm512_storeu_si512((__m512i*)(loc_out[6]), TMP[10]); + _mm512_storeu_si512((__m512i*)(loc_out[7]), TMP[11]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[12], TMP[13], TMP[14], TMP[15]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], swap_m512i); + TMP[1] = _mm512_shuffle_epi8(TMP[1], swap_m512i); + TMP[2] = _mm512_shuffle_epi8(TMP[2], swap_m512i); + TMP[3] = _mm512_shuffle_epi8(TMP[3], swap_m512i); + /* xor with IVs */ + TMP[12] = _mm512_xor_si512(TMP[0], STORED_CT[8]); + TMP[13] = _mm512_xor_si512(TMP[1], STORED_CT[9]); + TMP[14] = _mm512_xor_si512(TMP[2], STORED_CT[10]); + TMP[15] = _mm512_xor_si512(TMP[3], STORED_CT[11]); + /* Store last block of ciphertext for next iteration */ + for (int i = 8; i < 12; i++) + STORED_CT[i] = _mm512_inserti64x2(STORED_CT[i], _mm_loadu_si128((__m128i const*)loc_inp[i] + 3), 3); + _mm512_storeu_si512((__m512i*)(loc_out[8]), TMP[12]); + _mm512_storeu_si512((__m512i*)(loc_out[9]), TMP[13]); + _mm512_storeu_si512((__m512i*)(loc_out[10]), TMP[14]); + _mm512_storeu_si512((__m512i*)(loc_out[11]), TMP[15]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[16], TMP[17], TMP[18], TMP[19]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], swap_m512i); + TMP[1] = _mm512_shuffle_epi8(TMP[1], swap_m512i); + TMP[2] = _mm512_shuffle_epi8(TMP[2], swap_m512i); + TMP[3] = _mm512_shuffle_epi8(TMP[3], swap_m512i); + /* xor with IVs */ + TMP[16] = _mm512_xor_si512(TMP[0], STORED_CT[12]); + TMP[17] = _mm512_xor_si512(TMP[1], STORED_CT[13]); + TMP[18] = _mm512_xor_si512(TMP[2], STORED_CT[14]); + TMP[19] = _mm512_xor_si512(TMP[3], STORED_CT[15]); + for (int i = 12; i < SM4_LINES; i++) + STORED_CT[i] = _mm512_inserti64x2(STORED_CT[i], _mm_loadu_si128((__m128i const*)loc_inp[i] + 3), 3); + _mm512_storeu_si512((__m512i*)(loc_out[12]), TMP[16]); + _mm512_storeu_si512((__m512i*)(loc_out[13]), TMP[17]); + _mm512_storeu_si512((__m512i*)(loc_out[14]), TMP[18]); + _mm512_storeu_si512((__m512i*)(loc_out[15]), TMP[19]); + + /* Update pointers to data */ + _mm512_storeu_si512((void*)loc_inp, _mm512_add_epi64(_mm512_loadu_si512(loc_inp), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE))); + _mm512_storeu_si512((void*)(loc_inp + 8), _mm512_add_epi64(_mm512_loadu_si512(loc_inp + 8), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE))); + + _mm512_storeu_si512(loc_out, _mm512_add_epi64(_mm512_loadu_si512(loc_out), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE))); + _mm512_storeu_si512((loc_out + 8), _mm512_add_epi64(_mm512_loadu_si512(loc_out + 8), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE))); + + /* Update number of blocks left and processing mask */ + num_blocks = _mm512_sub_epi32(num_blocks, _mm512_set1_epi32(4)); + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, num_blocks, _mm512_set1_epi32(4), _MM_CMPINT_NLT); + } + + /* compute incomplete buffer loading */ + sm4_cbc_dec_incomplete_buff_mb16(loc_inp, loc_out, + num_blocks, p_rk, + mb_mask, + TMP, STORED_CT); + /* clear local copy of sensitive data */ + zero_mb8((int64u (*)[8])TMP, sizeof(TMP)/sizeof(TMP[0])); +} + +// Disable optimization for VS19 (>= 19.27) +OPTIMIZE_OFF_VS19 + +static void sm4_cbc_dec_incomplete_buff_mb16(const int8u* loc_inp[SM4_LINES], int8u* loc_out[SM4_LINES], + __m512i num_blocks, const __m512i* p_rk, + __mmask16 mb_mask, + __m512i TMP[20], + __m512i STORED_CT[16]){ + /* Load the constant value */ + const __m512i swap_m512i = _mm512_loadu_si512(swapBytes); + /* Check if we have any data */ + __mmask16 tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, num_blocks, _mm512_setzero_si512(), _MM_CMPINT_NLE); + + while (tmp_mask) { + /* Generate the array of masks for data loading. 0 - 4 blocks will be can load from each buffer - depend on the amount of remaining data */ + __ALIGN64 __mmask8 block_mask[16]; + + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, num_blocks, _mm512_set1_epi32(4), _MM_CMPINT_NLT); + /* Will be loaded 4 blocks of data */ + M128(block_mask) = _mm_maskz_set1_epi8(tmp_mask, 0xFF); + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, num_blocks, _mm512_set1_epi32(3), _MM_CMPINT_EQ); + /* Will be loaded 3 blocks of data */ + M128(block_mask) = _mm_mask_set1_epi8(M128(block_mask), tmp_mask, 0x3F); + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, num_blocks, _mm512_set1_epi32(2), _MM_CMPINT_EQ); + /* Will be loaded 2 blocks of data */ + M128(block_mask) = _mm_mask_set1_epi8(M128(block_mask), tmp_mask, 0xF); + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, num_blocks, _mm512_set1_epi32(1), _MM_CMPINT_EQ); + /* Will be loaded 1 block of data */ + M128(block_mask) = _mm_mask_set1_epi8(M128(block_mask), tmp_mask, 0x3); + + TMP[0] = _mm512_maskz_loadu_epi64(block_mask[0], loc_inp[0]); + TMP[1] = _mm512_maskz_loadu_epi64(block_mask[1], loc_inp[1]); + TMP[2] = _mm512_maskz_loadu_epi64(block_mask[2], loc_inp[2]); + TMP[3] = _mm512_maskz_loadu_epi64(block_mask[3], loc_inp[3]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], swap_m512i); + TMP[1] = _mm512_shuffle_epi8(TMP[1], swap_m512i); + TMP[2] = _mm512_shuffle_epi8(TMP[2], swap_m512i); + TMP[3] = _mm512_shuffle_epi8(TMP[3], swap_m512i); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_maskz_loadu_epi64(block_mask[4], loc_inp[4]); + TMP[1] = _mm512_maskz_loadu_epi64(block_mask[5], loc_inp[5]); + TMP[2] = _mm512_maskz_loadu_epi64(block_mask[6], loc_inp[6]); + TMP[3] = _mm512_maskz_loadu_epi64(block_mask[7], loc_inp[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], swap_m512i); + TMP[1] = _mm512_shuffle_epi8(TMP[1], swap_m512i); + TMP[2] = _mm512_shuffle_epi8(TMP[2], swap_m512i); + TMP[3] = _mm512_shuffle_epi8(TMP[3], swap_m512i); + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_maskz_loadu_epi64(block_mask[8], loc_inp[8]); + TMP[1] = _mm512_maskz_loadu_epi64(block_mask[9], loc_inp[9]); + TMP[2] = _mm512_maskz_loadu_epi64(block_mask[10], loc_inp[10]); + TMP[3] = _mm512_maskz_loadu_epi64(block_mask[11], loc_inp[11]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], swap_m512i); + TMP[1] = _mm512_shuffle_epi8(TMP[1], swap_m512i); + TMP[2] = _mm512_shuffle_epi8(TMP[2], swap_m512i); + TMP[3] = _mm512_shuffle_epi8(TMP[3], swap_m512i); + TRANSPOSE_INP_512(TMP[12], TMP[13], TMP[14], TMP[15], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_maskz_loadu_epi64(block_mask[12], loc_inp[12]); + TMP[1] = _mm512_maskz_loadu_epi64(block_mask[13], loc_inp[13]); + TMP[2] = _mm512_maskz_loadu_epi64(block_mask[14], loc_inp[14]); + TMP[3] = _mm512_maskz_loadu_epi64(block_mask[15], loc_inp[15]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], swap_m512i); + TMP[1] = _mm512_shuffle_epi8(TMP[1], swap_m512i); + TMP[2] = _mm512_shuffle_epi8(TMP[2], swap_m512i); + TMP[3] = _mm512_shuffle_epi8(TMP[3], swap_m512i); + TRANSPOSE_INP_512(TMP[16], TMP[17], TMP[18], TMP[19], TMP[0], TMP[1], TMP[2], TMP[3]); + + SM4_KERNEL(TMP, p_rk, -1); + p_rk += SM4_ROUNDS; + + for (int i = 0; i < SM4_LINES; i++) + STORED_CT[i] = _mm512_alignr_epi64(_mm512_maskz_loadu_epi64(block_mask[i], (__m128i *const)loc_inp[i]), STORED_CT[i], 6); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], swap_m512i); + TMP[1] = _mm512_shuffle_epi8(TMP[1], swap_m512i); + TMP[2] = _mm512_shuffle_epi8(TMP[2], swap_m512i); + TMP[3] = _mm512_shuffle_epi8(TMP[3], swap_m512i); + /* xor with IVs */ + TMP[4] = _mm512_xor_si512(TMP[0], STORED_CT[0]); + TMP[5] = _mm512_xor_si512(TMP[1], STORED_CT[1]); + TMP[6] = _mm512_xor_si512(TMP[2], STORED_CT[2]); + TMP[7] = _mm512_xor_si512(TMP[3], STORED_CT[3]); + /* Store last block of ciphertext for next iteration */ + for (int i = 0; i < 4; i++) { + __mmask8 write_block_mask = _kshiftri_mask8(block_mask[i], 6); + __m128i data_block = _mm_maskz_loadu_epi64(write_block_mask, (__m128i const*)loc_inp[i] + 3); + STORED_CT[i] = _mm512_mask_inserti64x2(STORED_CT[i], block_mask[i], STORED_CT[i], data_block, 3); + } + _mm512_mask_storeu_epi64(loc_out[0], block_mask[0], TMP[4]); + _mm512_mask_storeu_epi64(loc_out[1], block_mask[1], TMP[5]); + _mm512_mask_storeu_epi64(loc_out[2], block_mask[2], TMP[6]); + _mm512_mask_storeu_epi64(loc_out[3], block_mask[3], TMP[7]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], swap_m512i); + TMP[1] = _mm512_shuffle_epi8(TMP[1], swap_m512i); + TMP[2] = _mm512_shuffle_epi8(TMP[2], swap_m512i); + TMP[3] = _mm512_shuffle_epi8(TMP[3], swap_m512i); + /* xor with IVs */ + TMP[8] = _mm512_xor_si512(TMP[0], STORED_CT[4]); + TMP[9] = _mm512_xor_si512(TMP[1], STORED_CT[5]); + TMP[10] = _mm512_xor_si512(TMP[2], STORED_CT[6]); + TMP[11] = _mm512_xor_si512(TMP[3], STORED_CT[7]); + /* Store last block of ciphertext for next iteration */ + for (int i = 4; i < 8; i++) { + __mmask8 write_block_mask = _kshiftri_mask8(block_mask[i], 6); + __m128i data_block = _mm_maskz_loadu_epi64(write_block_mask, (__m128i const*)loc_inp[i] + 3); + STORED_CT[i] = _mm512_mask_inserti64x2(STORED_CT[i], block_mask[i], STORED_CT[i], data_block, 3); + } + _mm512_mask_storeu_epi64(loc_out[4], block_mask[4], TMP[8]); + _mm512_mask_storeu_epi64(loc_out[5], block_mask[5], TMP[9]); + _mm512_mask_storeu_epi64(loc_out[6], block_mask[6], TMP[10]); + _mm512_mask_storeu_epi64(loc_out[7], block_mask[7], TMP[11]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[12], TMP[13], TMP[14], TMP[15]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], swap_m512i); + TMP[1] = _mm512_shuffle_epi8(TMP[1], swap_m512i); + TMP[2] = _mm512_shuffle_epi8(TMP[2], swap_m512i); + TMP[3] = _mm512_shuffle_epi8(TMP[3], swap_m512i); + /* xor with IVs */ + TMP[12] = _mm512_xor_si512(TMP[0], STORED_CT[8]); + TMP[13] = _mm512_xor_si512(TMP[1], STORED_CT[9]); + TMP[14] = _mm512_xor_si512(TMP[2], STORED_CT[10]); + TMP[15] = _mm512_xor_si512(TMP[3], STORED_CT[11]); + /* Store last block of ciphertext for next iteration */ + for (int i = 8; i < 12; i++) { + __mmask8 write_block_mask = _kshiftri_mask8(block_mask[i], 6); + __m128i data_block = _mm_maskz_loadu_epi64(write_block_mask, (__m128i const*)loc_inp[i] + 3); + STORED_CT[i] = _mm512_mask_inserti64x2(STORED_CT[i], block_mask[i], STORED_CT[i], data_block, 3); + } + _mm512_mask_storeu_epi64(loc_out[8], block_mask[8], TMP[12]); + _mm512_mask_storeu_epi64(loc_out[9], block_mask[9], TMP[13]); + _mm512_mask_storeu_epi64(loc_out[10], block_mask[10], TMP[14]); + _mm512_mask_storeu_epi64(loc_out[11], block_mask[11], TMP[15]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[16], TMP[17], TMP[18], TMP[19]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], swap_m512i); + TMP[1] = _mm512_shuffle_epi8(TMP[1], swap_m512i); + TMP[2] = _mm512_shuffle_epi8(TMP[2], swap_m512i); + TMP[3] = _mm512_shuffle_epi8(TMP[3], swap_m512i); + /* xor with IVs */ + TMP[16] = _mm512_xor_si512(TMP[0], STORED_CT[12]); + TMP[17] = _mm512_xor_si512(TMP[1], STORED_CT[13]); + TMP[18] = _mm512_xor_si512(TMP[2], STORED_CT[14]); + TMP[19] = _mm512_xor_si512(TMP[3], STORED_CT[15]); + /* Store last block of ciphertext for next iteration */ + for (int i = 12; i < SM4_LINES; i++) { + __mmask8 write_block_mask = _kshiftri_mask8(block_mask[i], 6); + __m128i data_block = _mm_maskz_loadu_epi64(write_block_mask, (__m128i const*)loc_inp[i] + 3); + STORED_CT[i] = _mm512_mask_inserti64x2(STORED_CT[i], block_mask[i], STORED_CT[i], data_block, 3); + } + _mm512_mask_storeu_epi64(loc_out[12], block_mask[12], TMP[16]); + _mm512_mask_storeu_epi64(loc_out[13], block_mask[13], TMP[17]); + _mm512_mask_storeu_epi64(loc_out[14], block_mask[14], TMP[18]); + _mm512_mask_storeu_epi64(loc_out[15], block_mask[15], TMP[19]); + + /* Update pointers to data */ + _mm512_storeu_si512((void*)loc_inp, _mm512_add_epi64(_mm512_loadu_si512(loc_inp), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE))); + _mm512_storeu_si512((void*)(loc_inp + 8), _mm512_add_epi64(_mm512_loadu_si512(loc_inp + 8), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE))); + + _mm512_storeu_si512(loc_out, _mm512_add_epi64(_mm512_loadu_si512(loc_out), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE))); + _mm512_storeu_si512((loc_out + 8), _mm512_add_epi64(_mm512_loadu_si512(loc_out + 8), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE))); + + /* Update the number of blocks. For some buffers, the value can become zero or a negative number - these buffers will not be processed */ + num_blocks = _mm512_sub_epi32(num_blocks, _mm512_set1_epi32(4)); + + /* Check if we have any data */ + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, num_blocks, _mm512_setzero_si512(), _MM_CMPINT_NLE); + } + return; +} + +// Enable optimization for VS19 (>= 19.27) +OPTIMIZE_ON_VS19 diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_cfb_dec_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_cfb_dec_mb16.c new file mode 100644 index 0000000..2f689ed --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_cfb_dec_mb16.c @@ -0,0 +1,55 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include + +DLL_PUBLIC +mbx_status16 mbx_sm4_decrypt_cfb128_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, const int8u* pa_iv[SM4_LINES]) +{ + int buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + + /* Test input pointers */ + if (NULL == pa_out || NULL == pa_inp || NULL == len || NULL == key_sched || NULL == pa_iv) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* Test input data length, input pointers and stream integrity*/ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_out[buf_no] == NULL || pa_inp[buf_no] == NULL || pa_iv[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + /* Do not process empty buffers */ + mb_mask &= ~(0x1 << buf_no); + } + if ((len[buf_no] < 0) || (len[buf_no] & (SM4_BLOCK_SIZE - 1))) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + /* Do not process non-valid buffers */ + mb_mask &= ~(0x1 << buf_no); + } + } + + if (MBX_IS_ANY_OK_STS16(status)) + sm4_cfb128_dec_kernel_mb16(pa_out, (const int8u**)pa_inp, (const int*)len, (const int32u**)key_sched, pa_iv, mb_mask); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_cfb_enc_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_cfb_enc_mb16.c new file mode 100644 index 0000000..3d44efe --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_cfb_enc_mb16.c @@ -0,0 +1,55 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include + +DLL_PUBLIC +mbx_status16 mbx_sm4_encrypt_cfb128_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, const int8u* pa_iv[SM4_LINES]) +{ + int buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + + /* Test input pointers */ + if (NULL == pa_out || NULL == pa_inp || NULL == len || NULL == key_sched || NULL == pa_iv) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* Test input data length, input pointers and stream integrity*/ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_out[buf_no] == NULL || pa_inp[buf_no] == NULL || pa_iv[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + /* Do not process empty buffers */ + mb_mask &= ~(0x1 << buf_no); + } + if ((len[buf_no] < 0) || (len[buf_no] & (SM4_BLOCK_SIZE - 1))) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + /* Do not process non-valid buffers */ + mb_mask &= ~(0x1 << buf_no); + } + } + + if (MBX_IS_ANY_OK_STS16(status)) + sm4_cfb128_enc_kernel_mb16(pa_out, (const int8u**)pa_inp, (const int*)len, (const int32u**)key_sched, pa_iv, mb_mask); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_cfb_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_cfb_mb16.c new file mode 100644 index 0000000..fcf26b6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_cfb_mb16.c @@ -0,0 +1,427 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +void sm4_cfb128_enc_kernel_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const int32u* key_sched[SM4_ROUNDS], const int8u* pa_iv[SM4_LINES], __mmask16 mb_mask) +{ + __ALIGN64 const int8u* loc_inp[SM4_LINES]; + __ALIGN64 int8u* loc_out[SM4_LINES]; + + /* Get the copy of input data lengths in bytes */ + __m512i loc_len = _mm512_loadu_si512(len); + int* p_loc_len = (int*)&loc_len; + + /* Local copies of the pointers to input and output buffers */ + _mm512_storeu_si512((void*)loc_inp, _mm512_loadu_si512(pa_inp)); + _mm512_storeu_si512((void*)(loc_inp + 8), _mm512_loadu_si512(pa_inp + 8)); + + _mm512_storeu_si512(loc_out, _mm512_loadu_si512(pa_out)); + _mm512_storeu_si512(loc_out + 8, _mm512_loadu_si512(pa_out + 8)); + + /* Set p_rk pointer to the beginning of the key schedule */ + const __m512i* p_rk = (const __m512i*)key_sched; + + /* Check if we have any data */ + __mmask16 tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, loc_len, _mm512_setzero_si512(), _MM_CMPINT_NLE); + + /* Load and transpose iv */ + __m512i iv0, iv1, iv2, iv3; + TRANSPOSE_16x4_I32_EPI32(&iv0, &iv1, &iv2, &iv3, (const int8u**)pa_iv, tmp_mask); + + /* Main loop */ + __m512i z0, z1, z2, z3, tmp; + while (tmp_mask) { + for (int itr = 0; itr < SM4_ROUNDS; itr += 4, p_rk += 4) + SM4_FOUR_ROUNDS(iv0, iv1, iv2, iv3, tmp, p_rk, 1); + + p_rk -= SM4_ROUNDS; + + /* Load and transpose plaintext */ + TRANSPOSE_16x4_I32_EPI32(&z0, &z1, &z2, &z3, (const int8u**)loc_inp, tmp_mask); + + /* Change the order of blocks (Y0, Y1, Y2, Y3) = R(X32, X33, X34, X35) = (X35, X34, X33, X32) and xor with plain text */ + tmp = iv0; + iv0 = _mm512_xor_epi32(iv3, z0); + iv3 = _mm512_xor_epi32(tmp, z3); + tmp = iv1; + iv1 = _mm512_xor_epi32(iv2, z1); + iv2 = _mm512_xor_epi32(tmp, z2); + + /* Transpose and store encrypted blocks by bytes */ + TRANSPOSE_4x16_I32_EPI8(iv0, iv1, iv2, iv3, loc_out, p_loc_len, tmp_mask); + + /* Update pointers to data */ + M512(loc_inp) = _mm512_add_epi64(_mm512_loadu_si512(loc_inp), _mm512_set1_epi64(SM4_BLOCK_SIZE)); + M512(loc_inp + 8) = _mm512_add_epi64(_mm512_loadu_si512(loc_inp + 8), _mm512_set1_epi64(SM4_BLOCK_SIZE)); + + M512(loc_out) = _mm512_add_epi64(_mm512_loadu_si512(loc_out), _mm512_set1_epi64(SM4_BLOCK_SIZE)); + M512(loc_out + 8) = _mm512_add_epi64(_mm512_loadu_si512(loc_out + 8), _mm512_set1_epi64(SM4_BLOCK_SIZE)); + + /* Update number of blocks left and processing mask */ + loc_len = _mm512_sub_epi32(loc_len, _mm512_set1_epi32(SM4_BLOCK_SIZE)); + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, loc_len, _mm512_setzero_si512(), _MM_CMPINT_NLE); + } + + /* clear local copy of sensitive data */ + zero_mb8((int64u(*)[8])&iv0, 1); + zero_mb8((int64u(*)[8])&iv1, 1); + zero_mb8((int64u(*)[8])&iv2, 1); + zero_mb8((int64u(*)[8])&iv3, 1); + + zero_mb8((int64u(*)[8])&z0, 1); + zero_mb8((int64u(*)[8])&z1, 1); + zero_mb8((int64u(*)[8])&z2, 1); + zero_mb8((int64u(*)[8])&z3, 1); + zero_mb8((int64u(*)[8])&tmp, 1); +} + +static void sm4_cfb128_mask_dec_kernel_mb16(const int8u** loc_inp, int8u** loc_out, __m512i loc_len, const __m512i* p_rk, __mmask16 tmp_mask, __m512i STORED_CT[16]) +{ + __m512i TMP[20]; + while (tmp_mask) { + /* Generate the array of masks for data loading */ + __mmask64 stream_mask[16]; + int* p_loc_len = (int*)&loc_len; + for (int i = 0; i < SM4_LINES; i++) { + UPDATE_STREAM_MASK_64(stream_mask[i], p_loc_len); + } + + for (int i = 0; i < SM4_LINES; i++) + STORED_CT[i] = _mm512_alignr_epi64(_mm512_maskz_loadu_epi8(stream_mask[i], (__m128i *const)loc_inp[i]), STORED_CT[i], 6); + + TMP[0] = _mm512_shuffle_epi8(STORED_CT[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(STORED_CT[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(STORED_CT[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(STORED_CT[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_shuffle_epi8(STORED_CT[4], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(STORED_CT[5], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(STORED_CT[6], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(STORED_CT[7], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_shuffle_epi8(STORED_CT[8], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(STORED_CT[9], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(STORED_CT[10], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(STORED_CT[11], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[12], TMP[13], TMP[14], TMP[15], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_shuffle_epi8(STORED_CT[12], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(STORED_CT[13], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(STORED_CT[14], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(STORED_CT[15], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[16], TMP[17], TMP[18], TMP[19], TMP[0], TMP[1], TMP[2], TMP[3]); + + SM4_KERNEL(TMP, p_rk, 1); + p_rk -= SM4_ROUNDS; + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + /* xor with ciphertext */ + TMP[4] = _mm512_xor_si512(TMP[0], _mm512_maskz_loadu_epi8(stream_mask[0], loc_inp[0])); + TMP[5] = _mm512_xor_si512(TMP[1], _mm512_maskz_loadu_epi8(stream_mask[1], loc_inp[1])); + TMP[6] = _mm512_xor_si512(TMP[2], _mm512_maskz_loadu_epi8(stream_mask[2], loc_inp[2])); + TMP[7] = _mm512_xor_si512(TMP[3], _mm512_maskz_loadu_epi8(stream_mask[3], loc_inp[3])); + /* Store last block of ciphertext for next iteration */ + for (int i = 0; i < 4; i++) { + __mmask64 write_stream_mask = _kshiftri_mask64(stream_mask[i], 8*6); + __m128i data_block = _mm_maskz_loadu_epi8((__mmask16) write_stream_mask, (__m128i const*)loc_inp[i] + 3); + STORED_CT[i] = _mm512_inserti64x2(STORED_CT[i], data_block, 3); + } + _mm512_mask_storeu_epi8(loc_out[0], stream_mask[0], TMP[4]); + _mm512_mask_storeu_epi8(loc_out[1], stream_mask[1], TMP[5]); + _mm512_mask_storeu_epi8(loc_out[2], stream_mask[2], TMP[6]); + _mm512_mask_storeu_epi8(loc_out[3], stream_mask[3], TMP[7]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + /* xor with ciphertext */ + TMP[8] = _mm512_xor_si512(TMP[0], _mm512_maskz_loadu_epi8(stream_mask[4], loc_inp[4])); + TMP[9] = _mm512_xor_si512(TMP[1], _mm512_maskz_loadu_epi8(stream_mask[5], loc_inp[5])); + TMP[10] = _mm512_xor_si512(TMP[2], _mm512_maskz_loadu_epi8(stream_mask[6], loc_inp[6])); + TMP[11] = _mm512_xor_si512(TMP[3], _mm512_maskz_loadu_epi8(stream_mask[7], loc_inp[7])); + /* Store last block of ciphertext for next iteration */ + for (int i = 4; i < 8; i++) { + __mmask64 write_stream_mask = _kshiftri_mask64(stream_mask[i], 8*6); + __m128i data_block = _mm_maskz_loadu_epi8((__mmask16) write_stream_mask, (__m128i const*)loc_inp[i] + 3); + STORED_CT[i] = _mm512_inserti64x2(STORED_CT[i], data_block, 3); + } + _mm512_mask_storeu_epi8(loc_out[4], stream_mask[4], TMP[8]); + _mm512_mask_storeu_epi8(loc_out[5], stream_mask[5], TMP[9]); + _mm512_mask_storeu_epi8(loc_out[6], stream_mask[6], TMP[10]); + _mm512_mask_storeu_epi8(loc_out[7], stream_mask[7], TMP[11]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[12], TMP[13], TMP[14], TMP[15]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + /* xor with ciphertext */ + TMP[12] = _mm512_xor_si512(TMP[0], _mm512_maskz_loadu_epi8(stream_mask[8], loc_inp[8])); + TMP[13] = _mm512_xor_si512(TMP[1], _mm512_maskz_loadu_epi8(stream_mask[9], loc_inp[9])); + TMP[14] = _mm512_xor_si512(TMP[2], _mm512_maskz_loadu_epi8(stream_mask[10], loc_inp[10])); + TMP[15] = _mm512_xor_si512(TMP[3], _mm512_maskz_loadu_epi8(stream_mask[11], loc_inp[11])); + /* Store last block of ciphertext for next iteration */ + for (int i = 8; i < 12; i++) { + __mmask64 write_stream_mask = _kshiftri_mask64(stream_mask[i], 8*6); + __m128i data_block = _mm_maskz_loadu_epi8((__mmask16) write_stream_mask, (__m128i const*)loc_inp[i] + 3); + STORED_CT[i] = _mm512_inserti64x2(STORED_CT[i], data_block, 3); + } + _mm512_mask_storeu_epi8(loc_out[8], stream_mask[8], TMP[12]); + _mm512_mask_storeu_epi8(loc_out[9], stream_mask[9], TMP[13]); + _mm512_mask_storeu_epi8(loc_out[10], stream_mask[10], TMP[14]); + _mm512_mask_storeu_epi8(loc_out[11], stream_mask[11], TMP[15]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[16], TMP[17], TMP[18], TMP[19]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + /* xor with ciphertext */ + TMP[16] = _mm512_xor_si512(TMP[0], _mm512_maskz_loadu_epi8(stream_mask[12], loc_inp[12])); + TMP[17] = _mm512_xor_si512(TMP[1], _mm512_maskz_loadu_epi8(stream_mask[13], loc_inp[13])); + TMP[18] = _mm512_xor_si512(TMP[2], _mm512_maskz_loadu_epi8(stream_mask[14], loc_inp[14])); + TMP[19] = _mm512_xor_si512(TMP[3], _mm512_maskz_loadu_epi8(stream_mask[15], loc_inp[15])); + /* Store last block of ciphertext for next iteration */ + for (int i = 12; i < SM4_LINES; i++) { + __mmask64 write_stream_mask = _kshiftri_mask64(stream_mask[i], 8*6); + __m128i data_block = _mm_maskz_loadu_epi8((__mmask16) write_stream_mask, (__m128i const*)loc_inp[i] + 3); + STORED_CT[i] = _mm512_inserti64x2(STORED_CT[i], data_block, 3); + } + _mm512_mask_storeu_epi8(loc_out[12], stream_mask[12], TMP[16]); + _mm512_mask_storeu_epi8(loc_out[13], stream_mask[13], TMP[17]); + _mm512_mask_storeu_epi8(loc_out[14], stream_mask[14], TMP[18]); + _mm512_mask_storeu_epi8(loc_out[15], stream_mask[15], TMP[19]); + + /* Update pointers to data */ + M512(loc_inp) = _mm512_add_epi64(_mm512_loadu_si512(loc_inp), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + M512(loc_inp + 8) = _mm512_add_epi64(_mm512_loadu_si512(loc_inp + 8), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + + M512(loc_out) = _mm512_add_epi64(_mm512_loadu_si512(loc_out), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + M512(loc_out + 8) = _mm512_add_epi64(_mm512_loadu_si512(loc_out + 8), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + + /* Update the number of blocks. For some buffers, the value can become zero or a negative number - these buffers will not be processed */ + loc_len = _mm512_sub_epi32(loc_len, _mm512_set1_epi32(4 * SM4_BLOCK_SIZE)); + + /* Check if we have any data */ + tmp_mask = _mm512_mask_cmp_epi32_mask(tmp_mask, loc_len, _mm512_setzero_si512(), _MM_CMPINT_NLE); + } + + /* clear local copy of sensitive data */ + zero_mb8((int64u(*)[8])TMP, sizeof(TMP) / sizeof(TMP[0])); +} + +void sm4_cfb128_dec_kernel_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const int32u* key_sched[SM4_ROUNDS], const int8u* pa_iv[SM4_LINES], __mmask16 mb_mask) { + __ALIGN64 const int8u* loc_inp[SM4_LINES]; + __ALIGN64 int8u* loc_out[SM4_LINES]; + /* Registers to store ciphertext blocks to be XOR'ed with output of SM4 cipher stage */ + __m512i STORED_CT[16]; + + /* Get the copy of input data lengths in bytes */ + __m512i loc_len = _mm512_loadu_si512(len); + + /* Don't process empty buffers */ + mb_mask = _mm512_mask_cmp_epi32_mask(mb_mask, loc_len, _mm512_setzero_si512(), _MM_CMPINT_NLE); + + /* Local copies of the pointers to input and output buffers */ + _mm512_storeu_si512((void*)loc_inp, _mm512_loadu_si512(pa_inp)); + _mm512_storeu_si512((void*)(loc_inp + 8), _mm512_loadu_si512(pa_inp + 8)); + + _mm512_storeu_si512(loc_out, _mm512_loadu_si512(pa_out)); + _mm512_storeu_si512(loc_out + 8, _mm512_loadu_si512(pa_out + 8)); + + /* Set p_rk pointer to the beginnig of the key schedule */ + const __m512i* p_rk = (const __m512i*)key_sched; + + __m512i TMP[20]; + __mmask16 loc_mb_mask = mb_mask; + + /* Store first ciphertext block for next round */ + for (int i = 0; i < SM4_LINES; i++) { + STORED_CT[i] = _mm512_setzero_epi32(); + __m128i data_block = _mm_maskz_loadu_epi32(0x000F * (0x1&loc_mb_mask), loc_inp[i]); + STORED_CT[i] = _mm512_inserti64x2(STORED_CT[i], data_block, 3); + loc_mb_mask >>= 1; + } + + /* Process the first block from each buffer, because it contains IV specific */ + /* Load and transpose iv */ + TRANSPOSE_16x4_I32_EPI32(&TMP[4], &TMP[5], &TMP[6], &TMP[7], (const int8u**)pa_iv, mb_mask); + + for (int itr = 0; itr < SM4_ROUNDS; itr += 4, p_rk += 4) + SM4_FOUR_ROUNDS(TMP[4], TMP[5], TMP[6], TMP[7], TMP[8], p_rk, 1); + p_rk = (const __m512i*)key_sched; + /* Load and transpose ciphertext */ + TRANSPOSE_16x4_I32_EPI32(&TMP[9], &TMP[10], &TMP[11], &TMP[12], (const int8u**)loc_inp, mb_mask); + + /* Change the order of blocks (Y0, Y1, Y2, Y3) = R(X32, X33, X34, X35) = (X35, X34, X33, X32) and xor with plain text */ + TMP[8] = TMP[4]; + TMP[4] = _mm512_xor_epi32(TMP[7], TMP[9]); + TMP[7] = _mm512_xor_epi32(TMP[8], TMP[12]); + TMP[8] = TMP[5]; + TMP[5] = _mm512_xor_epi32(TMP[6], TMP[10]); + TMP[6] = _mm512_xor_epi32(TMP[8], TMP[11]); + + /* Transpose and store encrypted blocks by bytes */ + int *p_loc_len = (int*)&loc_len; + TRANSPOSE_4x16_I32_EPI8(TMP[4], TMP[5], TMP[6], TMP[7], loc_out, p_loc_len, mb_mask); + + /* Update pointers to data */ + M512(loc_inp) = _mm512_add_epi64(_mm512_loadu_si512(loc_inp), _mm512_set1_epi64(SM4_BLOCK_SIZE)); + M512(loc_inp + 8) = _mm512_add_epi64(_mm512_loadu_si512(loc_inp + 8), _mm512_set1_epi64(SM4_BLOCK_SIZE)); + + M512(loc_out) = _mm512_add_epi64(_mm512_loadu_si512(loc_out), _mm512_set1_epi64(SM4_BLOCK_SIZE)); + M512(loc_out + 8) = _mm512_add_epi64(_mm512_loadu_si512(loc_out + 8), _mm512_set1_epi64(SM4_BLOCK_SIZE)); + + loc_len = _mm512_sub_epi32(loc_len, _mm512_set1_epi32(SM4_BLOCK_SIZE)); + + /* Generate the mask to process 4 blocks from each buffer */ + __mmask16 tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, loc_len, _mm512_set1_epi32(4 * SM4_BLOCK_SIZE), _MM_CMPINT_NLT); + + /* Go to this loop if all 16 buffers contain at least 4 blocks each */ + while (tmp_mask == 0xFFFF) { + for (int i = 0; i < SM4_LINES; i++) + STORED_CT[i] = _mm512_alignr_epi64(_mm512_loadu_si512(loc_inp[i]), STORED_CT[i], 6); + + TMP[0] = _mm512_shuffle_epi8(STORED_CT[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(STORED_CT[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(STORED_CT[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(STORED_CT[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_shuffle_epi8(STORED_CT[4], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(STORED_CT[5], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(STORED_CT[6], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(STORED_CT[7], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_shuffle_epi8(STORED_CT[8], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(STORED_CT[9], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(STORED_CT[10], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(STORED_CT[11], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[12], TMP[13], TMP[14], TMP[15], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_shuffle_epi8(STORED_CT[12], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(STORED_CT[13], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(STORED_CT[14], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(STORED_CT[15], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[16], TMP[17], TMP[18], TMP[19], TMP[0], TMP[1], TMP[2], TMP[3]); + + SM4_KERNEL(TMP, p_rk, 1); + p_rk -= SM4_ROUNDS; + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + /* xor with ciphertext */ + TMP[4] = _mm512_xor_si512(TMP[0], _mm512_loadu_si512(loc_inp[0])); + TMP[5] = _mm512_xor_si512(TMP[1], _mm512_loadu_si512(loc_inp[1])); + TMP[6] = _mm512_xor_si512(TMP[2], _mm512_loadu_si512(loc_inp[2])); + TMP[7] = _mm512_xor_si512(TMP[3], _mm512_loadu_si512(loc_inp[3])); + /* Store last block of ciphertext for next iteration */ + for (int i = 0; i < 4; i++) + STORED_CT[i] = _mm512_inserti64x2(STORED_CT[i], _mm_loadu_si128((__m128i const*)loc_inp[i] + 3), 3); + _mm512_storeu_si512((__m512i*)(loc_out[0]), TMP[4]); + _mm512_storeu_si512((__m512i*)(loc_out[1]), TMP[5]); + _mm512_storeu_si512((__m512i*)(loc_out[2]), TMP[6]); + _mm512_storeu_si512((__m512i*)(loc_out[3]), TMP[7]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + /* xor with ciphertext */ + TMP[8] = _mm512_xor_si512(TMP[0], _mm512_loadu_si512(loc_inp[4])); + TMP[9] = _mm512_xor_si512(TMP[1], _mm512_loadu_si512(loc_inp[5])); + TMP[10] = _mm512_xor_si512(TMP[2], _mm512_loadu_si512(loc_inp[6])); + TMP[11] = _mm512_xor_si512(TMP[3], _mm512_loadu_si512(loc_inp[7])); + /* Store last block of ciphertext for next iteration */ + for (int i = 4; i < 8; i++) + STORED_CT[i] = _mm512_inserti64x2(STORED_CT[i], _mm_loadu_si128((__m128i const*)loc_inp[i] + 3), 3); + _mm512_storeu_si512((__m512i*)(loc_out[4]), TMP[8]); + _mm512_storeu_si512((__m512i*)(loc_out[5]), TMP[9]); + _mm512_storeu_si512((__m512i*)(loc_out[6]), TMP[10]); + _mm512_storeu_si512((__m512i*)(loc_out[7]), TMP[11]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[12], TMP[13], TMP[14], TMP[15]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + /* xor with ciphertext */ + TMP[12] = _mm512_xor_si512(TMP[0], _mm512_loadu_si512(loc_inp[8])); + TMP[13] = _mm512_xor_si512(TMP[1], _mm512_loadu_si512(loc_inp[9])); + TMP[14] = _mm512_xor_si512(TMP[2], _mm512_loadu_si512(loc_inp[10])); + TMP[15] = _mm512_xor_si512(TMP[3], _mm512_loadu_si512(loc_inp[11])); + /* Store last block of ciphertext for next iteration */ + for (int i = 8; i < 12; i++) + STORED_CT[i] = _mm512_inserti64x2(STORED_CT[i], _mm_loadu_si128((__m128i const*)loc_inp[i] + 3), 3); + _mm512_storeu_si512((__m512i*)(loc_out[8]), TMP[12]); + _mm512_storeu_si512((__m512i*)(loc_out[9]), TMP[13]); + _mm512_storeu_si512((__m512i*)(loc_out[10]), TMP[14]); + _mm512_storeu_si512((__m512i*)(loc_out[11]), TMP[15]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[16], TMP[17], TMP[18], TMP[19]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + /* xor with ciphertext */ + TMP[16] = _mm512_xor_si512(TMP[0], _mm512_loadu_si512(loc_inp[12])); + TMP[17] = _mm512_xor_si512(TMP[1], _mm512_loadu_si512(loc_inp[13])); + TMP[18] = _mm512_xor_si512(TMP[2], _mm512_loadu_si512(loc_inp[14])); + TMP[19] = _mm512_xor_si512(TMP[3], _mm512_loadu_si512(loc_inp[15])); + for (int i = 12; i < SM4_LINES; i++) + STORED_CT[i] = _mm512_inserti64x2(STORED_CT[i], _mm_loadu_si128((__m128i const*)loc_inp[i] + 3), 3); + _mm512_storeu_si512((__m512i*)(loc_out[12]), TMP[16]); + _mm512_storeu_si512((__m512i*)(loc_out[13]), TMP[17]); + _mm512_storeu_si512((__m512i*)(loc_out[14]), TMP[18]); + _mm512_storeu_si512((__m512i*)(loc_out[15]), TMP[19]); + + /* Update pointers to data */ + M512(loc_inp) = _mm512_add_epi64(_mm512_loadu_si512(loc_inp), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + M512(loc_inp + 8) = _mm512_add_epi64(_mm512_loadu_si512(loc_inp + 8), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + + M512(loc_out) = _mm512_add_epi64(_mm512_loadu_si512(loc_out), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + M512(loc_out + 8) = _mm512_add_epi64(_mm512_loadu_si512(loc_out + 8), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + + /* Update number of blocks left and processing mask */ + loc_len = _mm512_sub_epi32(loc_len, _mm512_set1_epi32(4 * SM4_BLOCK_SIZE)); + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, loc_len, _mm512_set1_epi32(4 * SM4_BLOCK_SIZE), _MM_CMPINT_NLT); + } + + /* Check if we have any data */ + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, loc_len, _mm512_setzero_si512(), _MM_CMPINT_NLE); + if (tmp_mask) + sm4_cfb128_mask_dec_kernel_mb16(loc_inp, loc_out, loc_len, p_rk, tmp_mask, STORED_CT); + + /* clear local copy of sensitive data */ + zero_mb8((int64u(*)[8])TMP, sizeof(TMP) / sizeof(TMP[0])); +} + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ctr_dec_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ctr_dec_mb16.c new file mode 100644 index 0000000..f8bb336 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ctr_dec_mb16.c @@ -0,0 +1,28 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include + +DLL_PUBLIC +mbx_status16 mbx_sm4_decrypt_ctr128_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, int8u* pa_ctr[SM4_LINES]) +{ + return mbx_sm4_encrypt_ctr128_mb16(pa_out, pa_inp, len, key_sched, pa_ctr); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ctr_enc_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ctr_enc_mb16.c new file mode 100644 index 0000000..30bd59a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ctr_enc_mb16.c @@ -0,0 +1,55 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include + +DLL_PUBLIC +mbx_status16 mbx_sm4_encrypt_ctr128_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, int8u* pa_ctr[SM4_LINES]) +{ + int buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + + /* Test input pointers */ + if (NULL == pa_out || NULL == pa_inp || NULL == len || NULL == key_sched || NULL == pa_ctr) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* Test input data length and input pointers */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_out[buf_no] == NULL || pa_inp[buf_no] == NULL || pa_ctr[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + /* Do not process empty buffers */ + mb_mask &= ~(0x1 << buf_no); + } + if (len[buf_no] < 0) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + /* Do not process non-valid buffers */ + mb_mask &= ~(0x1 << buf_no); + } + } + + if (MBX_IS_ANY_OK_STS16(status)) + sm4_ctr128_kernel_mb16(pa_out, (const int8u**)pa_inp, (const int*)len, (const int32u**)key_sched, mb_mask, pa_ctr); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ctr_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ctr_mb16.c new file mode 100644 index 0000000..1a2c3ab --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ctr_mb16.c @@ -0,0 +1,321 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +static void sm4_ctr128_mask_kernel_mb16(__m512i* CTR, const __m512i* p_rk, __m512i loc_len, const int8u** loc_inp, int8u** loc_out, int8u* inc, __mmask16 tmp_mask, __mmask16 mb_mask) +{ + __m512i TMP[20]; + while (tmp_mask) { + *CTR = IncBlock512(*CTR, inc); + *(CTR + 1) = IncBlock512(*(CTR + 1), inc); + *(CTR + 2) = IncBlock512(*(CTR + 2), inc); + *(CTR + 3) = IncBlock512(*(CTR + 3), inc); + TMP[0] = _mm512_shuffle_epi8(*CTR, M512(swapWordsOrder)); + TMP[1] = _mm512_shuffle_epi8(*(CTR + 1), M512(swapWordsOrder)); + TMP[2] = _mm512_shuffle_epi8(*(CTR + 2), M512(swapWordsOrder)); + TMP[3] = _mm512_shuffle_epi8(*(CTR + 3), M512(swapWordsOrder)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + *(CTR + 4) = IncBlock512(*(CTR + 4), inc); + *(CTR + 5) = IncBlock512(*(CTR + 5), inc); + *(CTR + 6) = IncBlock512(*(CTR + 6), inc); + *(CTR + 7) = IncBlock512(*(CTR + 7), inc); + TMP[0] = _mm512_shuffle_epi8(*(CTR + 4), M512(swapWordsOrder)); + TMP[1] = _mm512_shuffle_epi8(*(CTR + 5), M512(swapWordsOrder)); + TMP[2] = _mm512_shuffle_epi8(*(CTR + 6), M512(swapWordsOrder)); + TMP[3] = _mm512_shuffle_epi8(*(CTR + 7), M512(swapWordsOrder)); + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], TMP[0], TMP[1], TMP[2], TMP[3]); + + *(CTR + 8) = IncBlock512(*(CTR + 8), inc); + *(CTR + 9) = IncBlock512(*(CTR + 9), inc); + *(CTR + 10) = IncBlock512(*(CTR + 10), inc); + *(CTR + 11) = IncBlock512(*(CTR + 11), inc); + TMP[0] = _mm512_shuffle_epi8(*(CTR + 8), M512(swapWordsOrder)); + TMP[1] = _mm512_shuffle_epi8(*(CTR + 9), M512(swapWordsOrder)); + TMP[2] = _mm512_shuffle_epi8(*(CTR + 10), M512(swapWordsOrder)); + TMP[3] = _mm512_shuffle_epi8(*(CTR + 11), M512(swapWordsOrder)); + TRANSPOSE_INP_512(TMP[12], TMP[13], TMP[14], TMP[15], TMP[0], TMP[1], TMP[2], TMP[3]); + + *(CTR + 12) = IncBlock512(*(CTR + 12), inc); + *(CTR + 13) = IncBlock512(*(CTR + 13), inc); + *(CTR + 14) = IncBlock512(*(CTR + 14), inc); + *(CTR + 15) = IncBlock512(*(CTR + 15), inc); + TMP[0] = _mm512_shuffle_epi8(*(CTR + 12), M512(swapWordsOrder)); + TMP[1] = _mm512_shuffle_epi8(*(CTR + 13), M512(swapWordsOrder)); + TMP[2] = _mm512_shuffle_epi8(*(CTR + 14), M512(swapWordsOrder)); + TMP[3] = _mm512_shuffle_epi8(*(CTR + 15), M512(swapWordsOrder)); + TRANSPOSE_INP_512(TMP[16], TMP[17], TMP[18], TMP[19], TMP[0], TMP[1], TMP[2], TMP[3]); + + SM4_KERNEL(TMP, p_rk, 1); + p_rk -= SM4_ROUNDS; + + /* Mask for data loading */ + __mmask64 stream_mask; + int* p_loc_len = (int*)&loc_len; + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + _mm512_mask_storeu_epi8((__m512i*)loc_out[0], stream_mask, _mm512_xor_si512(TMP[0], _mm512_maskz_loadu_epi8(stream_mask, loc_inp[0]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + _mm512_mask_storeu_epi8((__m512i*)loc_out[1], stream_mask, _mm512_xor_si512(TMP[1], _mm512_maskz_loadu_epi8(stream_mask, loc_inp[1]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + _mm512_mask_storeu_epi8((__m512i*)loc_out[2], stream_mask, _mm512_xor_si512(TMP[2], _mm512_maskz_loadu_epi8(stream_mask, loc_inp[2]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + _mm512_mask_storeu_epi8((__m512i*)loc_out[3], stream_mask, _mm512_xor_si512(TMP[3], _mm512_maskz_loadu_epi8(stream_mask, loc_inp[3]))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + _mm512_mask_storeu_epi8((__m512i*)loc_out[4], stream_mask, _mm512_xor_si512(TMP[0], _mm512_maskz_loadu_epi8(stream_mask, loc_inp[4]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + _mm512_mask_storeu_epi8((__m512i*)loc_out[5], stream_mask, _mm512_xor_si512(TMP[1], _mm512_maskz_loadu_epi8(stream_mask, loc_inp[5]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + _mm512_mask_storeu_epi8((__m512i*)loc_out[6], stream_mask, _mm512_xor_si512(TMP[2], _mm512_maskz_loadu_epi8(stream_mask, loc_inp[6]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + _mm512_mask_storeu_epi8((__m512i*)loc_out[7], stream_mask, _mm512_xor_si512(TMP[3], _mm512_maskz_loadu_epi8(stream_mask, loc_inp[7]))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[12], TMP[13], TMP[14], TMP[15]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + _mm512_mask_storeu_epi8((__m512i*)loc_out[8], stream_mask, _mm512_xor_si512(TMP[0], _mm512_maskz_loadu_epi8(stream_mask, loc_inp[8]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + _mm512_mask_storeu_epi8((__m512i*)loc_out[9], stream_mask, _mm512_xor_si512(TMP[1], _mm512_maskz_loadu_epi8(stream_mask, loc_inp[9]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + _mm512_mask_storeu_epi8((__m512i*)loc_out[10], stream_mask, _mm512_xor_si512(TMP[2], _mm512_maskz_loadu_epi8(stream_mask, loc_inp[10]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + _mm512_mask_storeu_epi8((__m512i*)loc_out[11], stream_mask, _mm512_xor_si512(TMP[3], _mm512_maskz_loadu_epi8(stream_mask, loc_inp[11]))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[16], TMP[17], TMP[18], TMP[19]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + _mm512_mask_storeu_epi8((__m512i*)loc_out[12], stream_mask, _mm512_xor_si512(TMP[0], _mm512_maskz_loadu_epi8(stream_mask, loc_inp[12]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + _mm512_mask_storeu_epi8((__m512i*)loc_out[13], stream_mask, _mm512_xor_si512(TMP[1], _mm512_maskz_loadu_epi8(stream_mask, loc_inp[13]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + _mm512_mask_storeu_epi8((__m512i*)loc_out[14], stream_mask, _mm512_xor_si512(TMP[2], _mm512_maskz_loadu_epi8(stream_mask, loc_inp[14]))); + UPDATE_STREAM_MASK_64(stream_mask, p_loc_len) + _mm512_mask_storeu_epi8((__m512i*)loc_out[15], stream_mask, _mm512_xor_si512(TMP[3], _mm512_maskz_loadu_epi8(stream_mask, loc_inp[15]))); + + /* Update pointers to data */ + M512(loc_inp) = _mm512_add_epi64(_mm512_loadu_si512(loc_inp), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + M512(loc_inp + 8) = _mm512_add_epi64(_mm512_loadu_si512(loc_inp + 8), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + + M512(loc_out) = _mm512_add_epi64(_mm512_loadu_si512(loc_out), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + M512(loc_out + 8) = _mm512_add_epi64(_mm512_loadu_si512(loc_out + 8), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + + /* Update number of blocks left and processing mask */ + loc_len = _mm512_sub_epi32(loc_len, _mm512_set1_epi32(4 * SM4_BLOCK_SIZE)); + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, loc_len, _mm512_set1_epi32(0), _MM_CMPINT_NLE); + inc = (int8u*)nextInc; + } + + /* clear local copy of sensitive data */ + zero_mb8((int64u(*)[8])TMP, sizeof(TMP) / sizeof(TMP[0])); +} + + +__INLINE __m128i IncBlock128(__m128i x, int32u increment) +{ + __m128i t = _mm_add_epi64(x, _mm_maskz_loadu_epi32(1, &increment)); + __mmask8 carryMask = _mm_cmplt_epu64_mask(t, x); + carryMask = (__mmask8)(carryMask << 1); + t = _mm_add_epi64(t, _mm_mask_set1_epi64(_mm_setzero_si128(), carryMask, 1)); + + return t; +} + +void sm4_ctr128_kernel_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const int32u* key_sched[SM4_ROUNDS], __mmask16 mb_mask, int8u* pa_ctr[SM4_LINES]) +{ + __ALIGN64 const int8u* loc_inp[SM4_LINES]; + __ALIGN64 int8u* loc_out[SM4_LINES]; + + /* Create the local copy of the input data length in bytes and set it to zero for non-valid buffers */ + __m512i loc_len; + loc_len = _mm512_loadu_si512(len); + loc_len = _mm512_mask_set1_epi32(loc_len, ~mb_mask, 0); + + /* input blocks loc_blks[] = ceil(loc_len[]/SM4_BLOCK_SIZE) */ + int32u loc_blks[SM4_LINES]; + _mm512_storeu_si512(loc_blks, _mm512_srli_epi32(_mm512_add_epi32(loc_len, _mm512_set1_epi32(SM4_BLOCK_SIZE - 1)), 4)); + + /* Local copies of the pointers to input and otput buffers */ + _mm512_storeu_si512((void*)loc_inp, _mm512_loadu_si512(pa_inp)); + _mm512_storeu_si512((void*)(loc_inp + 8), _mm512_loadu_si512(pa_inp + 8)); + + _mm512_storeu_si512(loc_out, _mm512_loadu_si512(pa_out)); + _mm512_storeu_si512(loc_out + 8, _mm512_loadu_si512(pa_out + 8)); + + /* Pointer p_rk is set to the beginning of the key schedule */ + const __m512i* p_rk = (const __m512i*)key_sched; + + /* TMP[] - temporary buffer for processing */ + /* CTR - store CTR values */ + __m512i TMP[20]; + __m512i CTR[SM4_LINES]; + __m128i loc_ctr[SM4_LINES]; + + /* Load CTR value from valid buffers */ + mb_mask = _mm512_mask_cmp_epi32_mask(mb_mask, loc_len, _mm512_setzero_si512(), _MM_CMPINT_NLE); + for (int i = 0; i < SM4_LINES; i++) { + if (0x1 & (mb_mask >> i)) { + loc_ctr[i] = _mm_loadu_si128((__m128i*)pa_ctr[i]); + /* Read string counter and convert to numerical */ + loc_ctr[i] = _mm_shuffle_epi8(loc_ctr[i], M128(swapEndianness)); + } + else + loc_ctr[i] = _mm_setzero_si128(); + + CTR[i] = _mm512_broadcast_i64x2(loc_ctr[i]); + } + + /* Generate the mask to process 4 blocks from each buffer */ + __mmask16 tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, loc_len, _mm512_set1_epi32(4 * SM4_BLOCK_SIZE), _MM_CMPINT_NLT); + + int8u* inc = (int8u*)firstInc; + + /* Go to this loop if all 16 buffers contain at least 4 blocks each */ + while (tmp_mask == 0xFFFF) { + CTR[0] = IncBlock512(CTR[0], inc); + CTR[1] = IncBlock512(CTR[1], inc); + CTR[2] = IncBlock512(CTR[2], inc); + CTR[3] = IncBlock512(CTR[3], inc); + TMP[0] = _mm512_shuffle_epi8(CTR[0], M512(swapWordsOrder)); + TMP[1] = _mm512_shuffle_epi8(CTR[1], M512(swapWordsOrder)); + TMP[2] = _mm512_shuffle_epi8(CTR[2], M512(swapWordsOrder)); + TMP[3] = _mm512_shuffle_epi8(CTR[3], M512(swapWordsOrder)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + CTR[4] = IncBlock512(CTR[4], inc); + CTR[5] = IncBlock512(CTR[5], inc); + CTR[6] = IncBlock512(CTR[6], inc); + CTR[7] = IncBlock512(CTR[7], inc); + TMP[0] = _mm512_shuffle_epi8(CTR[4], M512(swapWordsOrder)); + TMP[1] = _mm512_shuffle_epi8(CTR[5], M512(swapWordsOrder)); + TMP[2] = _mm512_shuffle_epi8(CTR[6], M512(swapWordsOrder)); + TMP[3] = _mm512_shuffle_epi8(CTR[7], M512(swapWordsOrder)); + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], TMP[0], TMP[1], TMP[2], TMP[3]); + + CTR[8] = IncBlock512(CTR[8], inc); + CTR[9] = IncBlock512(CTR[9], inc); + CTR[10] = IncBlock512(CTR[10], inc); + CTR[11] = IncBlock512(CTR[11], inc); + TMP[0] = _mm512_shuffle_epi8(CTR[8], M512(swapWordsOrder)); + TMP[1] = _mm512_shuffle_epi8(CTR[9], M512(swapWordsOrder)); + TMP[2] = _mm512_shuffle_epi8(CTR[10], M512(swapWordsOrder)); + TMP[3] = _mm512_shuffle_epi8(CTR[11], M512(swapWordsOrder)); + TRANSPOSE_INP_512(TMP[12], TMP[13], TMP[14], TMP[15], TMP[0], TMP[1], TMP[2], TMP[3]); + + CTR[12] = IncBlock512(CTR[12], inc); + CTR[13] = IncBlock512(CTR[13], inc); + CTR[14] = IncBlock512(CTR[14], inc); + CTR[15] = IncBlock512(CTR[15], inc); + TMP[0] = _mm512_shuffle_epi8(CTR[12], M512(swapWordsOrder)); + TMP[1] = _mm512_shuffle_epi8(CTR[13], M512(swapWordsOrder)); + TMP[2] = _mm512_shuffle_epi8(CTR[14], M512(swapWordsOrder)); + TMP[3] = _mm512_shuffle_epi8(CTR[15], M512(swapWordsOrder)); + TRANSPOSE_INP_512(TMP[16], TMP[17], TMP[18], TMP[19], TMP[0], TMP[1], TMP[2], TMP[3]); + + SM4_KERNEL(TMP, p_rk, 1); + p_rk -= SM4_ROUNDS; + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)loc_out[0], _mm512_xor_si512(TMP[0], _mm512_loadu_si512(loc_inp[0]))); + _mm512_storeu_si512((__m512i*)loc_out[1], _mm512_xor_si512(TMP[1], _mm512_loadu_si512(loc_inp[1]))); + _mm512_storeu_si512((__m512i*)loc_out[2], _mm512_xor_si512(TMP[2], _mm512_loadu_si512(loc_inp[2]))); + _mm512_storeu_si512((__m512i*)loc_out[3], _mm512_xor_si512(TMP[3], _mm512_loadu_si512(loc_inp[3]))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)loc_out[4], _mm512_xor_si512(TMP[0], _mm512_loadu_si512(loc_inp[4]))); + _mm512_storeu_si512((__m512i*)loc_out[5], _mm512_xor_si512(TMP[1], _mm512_loadu_si512(loc_inp[5]))); + _mm512_storeu_si512((__m512i*)loc_out[6], _mm512_xor_si512(TMP[2], _mm512_loadu_si512(loc_inp[6]))); + _mm512_storeu_si512((__m512i*)loc_out[7], _mm512_xor_si512(TMP[3], _mm512_loadu_si512(loc_inp[7]))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[12], TMP[13], TMP[14], TMP[15]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)loc_out[8], _mm512_xor_si512(TMP[0], _mm512_loadu_si512(loc_inp[8]))); + _mm512_storeu_si512((__m512i*)loc_out[9], _mm512_xor_si512(TMP[1], _mm512_loadu_si512(loc_inp[9]))); + _mm512_storeu_si512((__m512i*)loc_out[10], _mm512_xor_si512(TMP[2], _mm512_loadu_si512(loc_inp[10]))); + _mm512_storeu_si512((__m512i*)loc_out[11], _mm512_xor_si512(TMP[3], _mm512_loadu_si512(loc_inp[11]))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[16], TMP[17], TMP[18], TMP[19]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)loc_out[12], _mm512_xor_si512(TMP[0], _mm512_loadu_si512(loc_inp[12]))); + _mm512_storeu_si512((__m512i*)loc_out[13], _mm512_xor_si512(TMP[1], _mm512_loadu_si512(loc_inp[13]))); + _mm512_storeu_si512((__m512i*)loc_out[14], _mm512_xor_si512(TMP[2], _mm512_loadu_si512(loc_inp[14]))); + _mm512_storeu_si512((__m512i*)loc_out[15], _mm512_xor_si512(TMP[3], _mm512_loadu_si512(loc_inp[15]))); + + /* Update pointers to data */ + M512(loc_inp) = _mm512_add_epi64(_mm512_loadu_si512(loc_inp), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + M512(loc_inp + 8) = _mm512_add_epi64(_mm512_loadu_si512(loc_inp + 8), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + + M512(loc_out) = _mm512_add_epi64(_mm512_loadu_si512(loc_out), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + M512(loc_out + 8) = _mm512_add_epi64(_mm512_loadu_si512(loc_out + 8), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + + /* Update number of blocks left and processing mask */ + loc_len = _mm512_sub_epi32(loc_len, _mm512_set1_epi32(4 * SM4_BLOCK_SIZE)); + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, loc_len, _mm512_set1_epi32(4 * SM4_BLOCK_SIZE), _MM_CMPINT_NLT); + inc = (int8u*)nextInc; + } + + /* Check if we have any data */ + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, loc_len, _mm512_setzero_si512(), _MM_CMPINT_NLE); + if (tmp_mask) + sm4_ctr128_mask_kernel_mb16(CTR, p_rk, loc_len, loc_inp, loc_out, inc, tmp_mask, mb_mask); + + /* update and store counters */ + for (int i = 0; i < SM4_LINES; i++) { + if (0x1 & (mb_mask >> i)) { + loc_ctr[i] = IncBlock128(loc_ctr[i], loc_blks[i]); + loc_ctr[i] = _mm_shuffle_epi8(loc_ctr[i], M128(swapEndianness)); + _mm_storeu_si128((__m128i*)pa_ctr[i], loc_ctr[i]); + loc_ctr[i] = _mm_setzero_si128(); + } + } + + /* clear local copy of sensitive data */ + zero_mb8((int64u(*)[8])TMP, sizeof(TMP) / sizeof(TMP[0])); + zero_mb8((int64u(*)[8])CTR, sizeof(CTR) / sizeof(CTR[0])); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ecb_dec_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ecb_dec_mb16.c new file mode 100644 index 0000000..01847fd --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ecb_dec_mb16.c @@ -0,0 +1,55 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include + +DLL_PUBLIC +mbx_status16 mbx_sm4_decrypt_ecb_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched) +{ + int buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + + /* Test input pointers */ + if (NULL == pa_out || NULL == pa_inp || NULL == len || NULL == key_sched) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* Test input data length and integrity for each buffer */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_out[buf_no] == NULL || pa_inp[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + /* Do not process empty buffers */ + mb_mask &= ~(0x1 << buf_no); + } + if ((len[buf_no] < 0) || (len[buf_no]&(SM4_BLOCK_SIZE-1))) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + /* Do not process non-valid buffers */ + mb_mask &= ~(0x1 << buf_no); + } + } + + if (MBX_IS_ANY_OK_STS16(status)) + sm4_ecb_kernel_mb16(pa_out, (const int8u**)pa_inp, (const int*)len, (const int32u**)key_sched, mb_mask, SM4_DEC); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ecb_enc_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ecb_enc_mb16.c new file mode 100644 index 0000000..7fcbe05 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ecb_enc_mb16.c @@ -0,0 +1,55 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include + +DLL_PUBLIC +mbx_status16 mbx_sm4_encrypt_ecb_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched) +{ + int buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + + /* Test input pointers */ + if (NULL == pa_out || NULL == pa_inp || NULL == len || NULL == key_sched) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* Test input data length and integrity for each buffer */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_out[buf_no] == NULL || pa_inp[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + /* Do not process empty buffers */ + mb_mask &= ~(0x1 << buf_no); + } + if ((len[buf_no] < 0) || (len[buf_no]&(SM4_BLOCK_SIZE-1))) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + /* Do not process non-valid buffers */ + mb_mask &= ~(0x1 << buf_no); + } + } + + if (MBX_IS_ANY_OK_STS16(status)) + sm4_ecb_kernel_mb16(pa_out, (const int8u**)pa_inp, (const int*)len, (const int32u**)key_sched, mb_mask, SM4_ENC); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ecb_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ecb_mb16.c new file mode 100644 index 0000000..298c4a8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ecb_mb16.c @@ -0,0 +1,284 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include +#include + +static void sm4_ecb_incomplete_buff_mb16(const int8u* loc_inp[SM4_LINES], int8u* loc_out[SM4_LINES], + __m512i num_blocks, const __m512i* p_rk, const int sign, + __mmask16 mb_mask, + __m512i TMP[20]); + +void sm4_ecb_kernel_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const int32u* key_sched[SM4_ROUNDS], __mmask16 mb_mask, int operation) +{ + __ALIGN64 const int8u* loc_inp[SM4_LINES]; + __ALIGN64 int8u* loc_out[SM4_LINES]; + + /* Length of the input data in 128-bit chunks - number of SM4 blocks */ + __m512i num_blocks; + GET_NUM_BLOCKS(num_blocks, len, SM4_BLOCK_SIZE); + + /* Local copies of the pointers to input and output buffers */ + _mm512_storeu_si512((void*)loc_inp, _mm512_loadu_si512(pa_inp)); + _mm512_storeu_si512((void*)(loc_inp + 8), _mm512_loadu_si512(pa_inp + 8)); + + _mm512_storeu_si512(loc_out, _mm512_loadu_si512(pa_out)); + _mm512_storeu_si512(loc_out + 8, _mm512_loadu_si512(pa_out + 8)); + + /* p_rk set to the beginning or to the end of the key schedule */ + const __m512i* p_rk = (operation == SM4_ENC) ? (const __m512i*)key_sched : ((const __m512i*)key_sched + (SM4_ROUNDS - 1)); + + __ALIGN64 __m512i TMP[20]; + + /* Generate the mask to process 4 blocks from each buffer */ + __mmask16 tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, num_blocks, _mm512_set1_epi32(4), _MM_CMPINT_NLT); + + /* Go to this loop if all 16 buffers contain at least 4 blocks each */ + while (tmp_mask == 0xFFFF) { + TMP[0] = _mm512_loadu_si512(loc_inp[0]); + TMP[1] = _mm512_loadu_si512((__m512i*)(loc_inp[1])); + TMP[2] = _mm512_loadu_si512((__m512i*)(loc_inp[2])); + TMP[3] = _mm512_loadu_si512((__m512i*)(loc_inp[3])); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_loadu_si512((__m512i*)(loc_inp[4])); + TMP[1] = _mm512_loadu_si512((__m512i*)(loc_inp[5])); + TMP[2] = _mm512_loadu_si512((__m512i*)(loc_inp[6])); + TMP[3] = _mm512_loadu_si512((__m512i*)(loc_inp[7])); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_loadu_si512((__m512i*)(loc_inp[8])); + TMP[1] = _mm512_loadu_si512((__m512i*)(loc_inp[9])); + TMP[2] = _mm512_loadu_si512((__m512i*)(loc_inp[10])); + TMP[3] = _mm512_loadu_si512((__m512i*)(loc_inp[11])); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[12], TMP[13], TMP[14], TMP[15], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_loadu_si512((__m512i*)(loc_inp[12])); + TMP[1] = _mm512_loadu_si512((__m512i*)(loc_inp[13])); + TMP[2] = _mm512_loadu_si512((__m512i*)(loc_inp[14])); + TMP[3] = _mm512_loadu_si512((__m512i*)(loc_inp[15])); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[16], TMP[17], TMP[18], TMP[19], TMP[0], TMP[1], TMP[2], TMP[3]); + + SM4_KERNEL(TMP, p_rk, operation); + p_rk -= operation*SM4_ROUNDS; + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(loc_out[0]), TMP[0]); + _mm512_storeu_si512((__m512i*)(loc_out[1]), TMP[1]); + _mm512_storeu_si512((__m512i*)(loc_out[2]), TMP[2]); + _mm512_storeu_si512((__m512i*)(loc_out[3]), TMP[3]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(loc_out[4]), TMP[0]); + _mm512_storeu_si512((__m512i*)(loc_out[5]), TMP[1]); + _mm512_storeu_si512((__m512i*)(loc_out[6]), TMP[2]); + _mm512_storeu_si512((__m512i*)(loc_out[7]), TMP[3]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[12], TMP[13], TMP[14], TMP[15]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(loc_out[8]), TMP[0]); + _mm512_storeu_si512((__m512i*)(loc_out[9]), TMP[1]); + _mm512_storeu_si512((__m512i*)(loc_out[10]), TMP[2]); + _mm512_storeu_si512((__m512i*)(loc_out[11]), TMP[3]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[16], TMP[17], TMP[18], TMP[19]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(loc_out[12]), TMP[0]); + _mm512_storeu_si512((__m512i*)(loc_out[13]), TMP[1]); + _mm512_storeu_si512((__m512i*)(loc_out[14]), TMP[2]); + _mm512_storeu_si512((__m512i*)(loc_out[15]), TMP[3]); + + /* Update pointers to data */ + M512(loc_inp) = _mm512_add_epi64(_mm512_loadu_si512(loc_inp), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + M512(loc_inp + 8) = _mm512_add_epi64(_mm512_loadu_si512(loc_inp + 8), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + + M512(loc_out) = _mm512_add_epi64(_mm512_loadu_si512(loc_out), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + M512(loc_out + 8) = _mm512_add_epi64(_mm512_loadu_si512(loc_out + 8), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + + /* Update number of blocks left and processing mask */ + num_blocks = _mm512_sub_epi32(num_blocks, _mm512_set1_epi32(4)); + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, num_blocks, _mm512_set1_epi32(4), _MM_CMPINT_NLT); + } + + /* compute incomplete buffer loading */ + sm4_ecb_incomplete_buff_mb16(loc_inp, loc_out, + num_blocks, p_rk, operation, + mb_mask, + TMP); + /* clear local copy of sensitive data */ + zero_mb8((int64u(*)[8])TMP, sizeof(TMP) / sizeof(TMP[0])); +} + +// Disable optimization for VS19 (>= 19.27) +OPTIMIZE_OFF_VS19 + +static void sm4_ecb_incomplete_buff_mb16(const int8u* loc_inp[SM4_LINES], int8u* loc_out[SM4_LINES], + __m512i num_blocks, const __m512i* p_rk, const int sign, + __mmask16 mb_mask, + __m512i TMP[20]){ + /* Check if we have any data */ + __mmask16 tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, num_blocks, _mm512_setzero_si512(), _MM_CMPINT_NLE); + + while (tmp_mask) { + /* Generate the array of masks for data loading. 0 - 4 blocks will be can load from each buffer - depend on the amount of remaining data */ + __ALIGN64 __mmask8 block_mask[SM4_LINES]; + + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, num_blocks, _mm512_set1_epi32(4), _MM_CMPINT_NLT); + /* Will be loaded 4 blocks of data */ + M128(block_mask) = _mm_maskz_set1_epi8(tmp_mask, 0xFF); + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, num_blocks, _mm512_set1_epi32(3), _MM_CMPINT_EQ); + /* Will be loaded 3 blocks of data */ + M128(block_mask) = _mm_mask_set1_epi8(M128(block_mask), tmp_mask, 0x3F); + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, num_blocks, _mm512_set1_epi32(2), _MM_CMPINT_EQ); + /* Will be loaded 2 blocks of data */ + M128(block_mask) = _mm_mask_set1_epi8(M128(block_mask), tmp_mask, 0xF); + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, num_blocks, _mm512_set1_epi32(1), _MM_CMPINT_EQ); + /* Will be loaded 1 block of data */ + M128(block_mask) = _mm_mask_set1_epi8(M128(block_mask), tmp_mask, 0x3); + + TMP[0] = _mm512_maskz_loadu_epi64(block_mask[0], loc_inp[0]); + TMP[1] = _mm512_maskz_loadu_epi64(block_mask[1], loc_inp[1]); + TMP[2] = _mm512_maskz_loadu_epi64(block_mask[2], loc_inp[2]); + TMP[3] = _mm512_maskz_loadu_epi64(block_mask[3], loc_inp[3]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_maskz_loadu_epi64(block_mask[4], loc_inp[4]); + TMP[1] = _mm512_maskz_loadu_epi64(block_mask[5], loc_inp[5]); + TMP[2] = _mm512_maskz_loadu_epi64(block_mask[6], loc_inp[6]); + TMP[3] = _mm512_maskz_loadu_epi64(block_mask[7], loc_inp[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_maskz_loadu_epi64(block_mask[8], loc_inp[8]); + TMP[1] = _mm512_maskz_loadu_epi64(block_mask[9], loc_inp[9]); + TMP[2] = _mm512_maskz_loadu_epi64(block_mask[10], loc_inp[10]); + TMP[3] = _mm512_maskz_loadu_epi64(block_mask[11], loc_inp[11]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[12], TMP[13], TMP[14], TMP[15], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_maskz_loadu_epi64(block_mask[12], loc_inp[12]); + TMP[1] = _mm512_maskz_loadu_epi64(block_mask[13], loc_inp[13]); + TMP[2] = _mm512_maskz_loadu_epi64(block_mask[14], loc_inp[14]); + TMP[3] = _mm512_maskz_loadu_epi64(block_mask[15], loc_inp[15]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[16], TMP[17], TMP[18], TMP[19], TMP[0], TMP[1], TMP[2], TMP[3]); + + SM4_KERNEL(TMP, p_rk, sign); + p_rk -= sign*SM4_ROUNDS; + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_mask_storeu_epi64(loc_out[0], block_mask[0], TMP[0]); + _mm512_mask_storeu_epi64(loc_out[1], block_mask[1], TMP[1]); + _mm512_mask_storeu_epi64(loc_out[2], block_mask[2], TMP[2]); + _mm512_mask_storeu_epi64(loc_out[3], block_mask[3], TMP[3]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_mask_storeu_epi64(loc_out[4], block_mask[4], TMP[0]); + _mm512_mask_storeu_epi64(loc_out[5], block_mask[5], TMP[1]); + _mm512_mask_storeu_epi64(loc_out[6], block_mask[6], TMP[2]); + _mm512_mask_storeu_epi64(loc_out[7], block_mask[7], TMP[3]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[12], TMP[13], TMP[14], TMP[15]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_mask_storeu_epi64(loc_out[8], block_mask[8], TMP[0]); + _mm512_mask_storeu_epi64(loc_out[9], block_mask[9], TMP[1]); + _mm512_mask_storeu_epi64(loc_out[10], block_mask[10], TMP[2]); + _mm512_mask_storeu_epi64(loc_out[11], block_mask[11], TMP[3]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[16], TMP[17], TMP[18], TMP[19]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_mask_storeu_epi64(loc_out[12], block_mask[12], TMP[0]); + _mm512_mask_storeu_epi64(loc_out[13], block_mask[13], TMP[1]); + _mm512_mask_storeu_epi64(loc_out[14], block_mask[14], TMP[2]); + _mm512_mask_storeu_epi64(loc_out[15], block_mask[15], TMP[3]); + + /* Update pointers to data */ + M512(loc_inp) = _mm512_add_epi64(_mm512_loadu_si512(loc_inp), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + M512(loc_inp + 8) = _mm512_add_epi64(_mm512_loadu_si512(loc_inp + 8), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + + M512(loc_out) = _mm512_add_epi64(_mm512_loadu_si512(loc_out), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + M512(loc_out + 8) = _mm512_add_epi64(_mm512_loadu_si512(loc_out + 8), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + + /* Update the number of blocks. For some buffers, the value can become zero or a negative number - these buffers will not be processed */ + num_blocks = _mm512_sub_epi32(num_blocks, _mm512_set1_epi32(4)); + + /* Check if we have any data */ + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, num_blocks, _mm512_setzero_si512(), _MM_CMPINT_NLE); + } + return; +} + +// Enable optimization for VS19 (>= 19.27) +OPTIMIZE_ON_VS19 diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ofb_dec_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ofb_dec_mb16.c new file mode 100644 index 0000000..0be0161 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ofb_dec_mb16.c @@ -0,0 +1,28 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include + +DLL_PUBLIC +mbx_status16 mbx_sm4_decrypt_ofb_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, int8u* pa_iv[SM4_LINES]) +{ + return mbx_sm4_encrypt_ofb_mb16(pa_out, pa_inp, len, key_sched, pa_iv); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ofb_enc_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ofb_enc_mb16.c new file mode 100644 index 0000000..5d7835c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ofb_enc_mb16.c @@ -0,0 +1,55 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include + +DLL_PUBLIC +mbx_status16 mbx_sm4_encrypt_ofb_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched, int8u* pa_iv[SM4_LINES]) +{ + int buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + + /* Test input pointers */ + if (NULL == pa_out || NULL == pa_inp || NULL == len || NULL == key_sched || NULL == pa_iv) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* Test input data length, input pointers and stream integrity*/ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_out[buf_no] == NULL || pa_inp[buf_no] == NULL || pa_iv[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + /* Do not process empty buffers */ + mb_mask &= ~(0x1 << buf_no); + } + if (len[buf_no] < 0) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + /* Do not process non-valid buffers */ + mb_mask &= ~(0x1 << buf_no); + } + } + + if (MBX_IS_ANY_OK_STS16(status)) + sm4_ofb_kernel_mb16(pa_out, (const int8u**)pa_inp, (const int*)len, (const int32u**)key_sched, mb_mask, pa_iv); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ofb_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ofb_mb16.c new file mode 100644 index 0000000..c4c9242 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_ofb_mb16.c @@ -0,0 +1,86 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +void sm4_ofb_kernel_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], const int32u* key_sched[SM4_ROUNDS], __mmask16 mb_mask, int8u* pa_iv[SM4_LINES]) +{ + __ALIGN64 const int8u* loc_inp[SM4_LINES]; + __ALIGN64 int8u* loc_out[SM4_LINES]; + + /* Get the copy of input data lengths in bytes */ + __m512i loc_len = _mm512_loadu_si512(len); + int* p_loc_len = (int*)&loc_len; + + /* Local copies of the pointers to input and output buffers */ + _mm512_storeu_si512((void*)loc_inp, _mm512_loadu_si512(pa_inp)); + _mm512_storeu_si512((void*)(loc_inp + 8), _mm512_loadu_si512(pa_inp + 8)); + + _mm512_storeu_si512(loc_out, _mm512_loadu_si512(pa_out)); + _mm512_storeu_si512(loc_out + 8, _mm512_loadu_si512(pa_out + 8)); + + /* Set p_rk pointer to the beginning of the key schedule */ + const __m512i* p_rk = (const __m512i*)key_sched; + + /* Check if we have any data */ + __mmask16 tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, loc_len, _mm512_setzero_si512(), _MM_CMPINT_NLE); + + __m512i iv0, iv1, iv2, iv3; + + /* Load and transpose iv */ + TRANSPOSE_16x4_I32_EPI32(&iv0, &iv1, &iv2, &iv3, (const int8u**)pa_iv, tmp_mask); + + /* Main loop */ + __m512i tmp; + while (tmp_mask) { + for (int itr = 0; itr < SM4_ROUNDS; itr += 4, p_rk += 4) + SM4_FOUR_ROUNDS(iv0, iv1, iv2, iv3, tmp, p_rk, 1); + + p_rk -= SM4_ROUNDS; + + /* Change the order of blocks (Y0, Y1, Y2, Y3) = R(X32, X33, X34, X35) = (X35, X34, X33, X32) */ + tmp = iv0; + iv0 = iv3; iv3 = tmp; + tmp = iv1; + iv1 = iv2; iv2 = tmp; + + /* Transpose and store encrypted blocks by bytes */ + TRANSPOSE_AND_XOR_4x16_I32_EPI8(iv0, iv1, iv2, iv3, loc_out, loc_inp, p_loc_len, tmp_mask); + + /* Update pointers to data */ + M512(loc_inp) = _mm512_add_epi64(_mm512_loadu_si512(loc_inp), _mm512_set1_epi64(SM4_BLOCK_SIZE)); + M512(loc_inp + 8) = _mm512_add_epi64(_mm512_loadu_si512(loc_inp + 8), _mm512_set1_epi64(SM4_BLOCK_SIZE)); + + M512(loc_out) = _mm512_add_epi64(_mm512_loadu_si512(loc_out), _mm512_set1_epi64(SM4_BLOCK_SIZE)); + M512(loc_out + 8) = _mm512_add_epi64(_mm512_loadu_si512(loc_out + 8), _mm512_set1_epi64(SM4_BLOCK_SIZE)); + + /* Update number of blocks left and processing mask */ + loc_len = _mm512_sub_epi32(loc_len, _mm512_set1_epi32(SM4_BLOCK_SIZE)); + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, loc_len, _mm512_setzero_si512(), _MM_CMPINT_NLE); + } + + /* Update ofb values */ + TRANSPOSE_4x16_I32_EPI32(&iv0, &iv1, &iv2, &iv3, pa_iv, tmp_mask); + + /* clear local copy of sensitive data */ + zero_mb8((int64u(*)[8])&iv0, 1); + zero_mb8((int64u(*)[8])&iv1, 1); + zero_mb8((int64u(*)[8])&iv2, 1); + zero_mb8((int64u(*)[8])&iv3, 1); + zero_mb8((int64u(*)[8])&tmp, 1); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_setkey_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_setkey_mb16.c new file mode 100644 index 0000000..e616332 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_setkey_mb16.c @@ -0,0 +1,151 @@ +/******************************************************************************* +* Copyright (C) 2021 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include +#include + +/* FK[] constants */ +static const int32u SM4_FK[4] = { + 0xA3B1BAC6,0x56AA3350,0x677D9197,0xB27022DC +}; + +/* CK[] constants */ +static const int32u SM4_CK[32] = { + 0x00070E15,0x1C232A31,0x383F464D,0x545B6269, + 0x70777E85,0x8C939AA1,0xA8AFB6BD,0xC4CBD2D9, + 0xE0E7EEF5,0xFC030A11,0x181F262D,0x343B4249, + 0x50575E65,0x6C737A81,0x888F969D,0xA4ABB2B9, + 0xC0C7CED5,0xDCE3EAF1,0xF8FF060D,0x141B2229, + 0x30373E45,0x4C535A61,0x686F767D,0x848B9299, + 0xA0A7AEB5,0xBCC3CAD1,0xD8DFE6ED,0xF4FB0209, + 0x10171E25,0x2C333A41,0x484F565D,0x646B7279 +}; + +#define SM4_ONE_RK(K0, K1, K2, K3, TMP, CK, OUT) { \ + /* (Ki+1 ^ Ki+2 ^ Ki+3 ^ CKi) */ \ + TMP = _mm512_xor_epi32(_mm512_xor_epi32(_mm512_xor_epi32(K1, K2), K3), _mm512_set1_epi32(CK)); \ + /* T'(Ki+1 ^ Ki+2 ^ Ki+3 ^ CKi) */ \ + TMP = sBox512(TMP); \ + TMP = _mm512_xor_epi32(TMP, Lkey512(TMP)); \ + /* Ki+4 = Ki ^ T'(Ki+1 ^ Ki+2 ^ Ki+3 ^ CKi) */ \ + K0 = _mm512_xor_epi32(K0, TMP); \ + _mm512_storeu_si512((void*)OUT, K0); \ +} + +#define SM4_FOUR_RK(K0, K1, K2, K3, TMP, CK, OUT) { \ + SM4_ONE_RK(K0, K1, K2, K3, TMP, CK[0], OUT); \ + SM4_ONE_RK(K1, K2, K3, K0, TMP, CK[1], (OUT + 1)); \ + SM4_ONE_RK(K2, K3, K0, K1, TMP, CK[2], (OUT + 2)); \ + SM4_ONE_RK(K3, K0, K1, K2, TMP, CK[3], (OUT + 3)); \ +} + + +void sm4_set_round_keys_mb16(int32u* key_sched[SM4_ROUNDS], const int8u* pa_inp_key[SM4_LINES], __mmask16 mb_mask) +{ + __m512i rki = _mm512_setzero_si512(); + __m512i z0, z1, z2, z3; + + TRANSPOSE_16x4_I32_EPI32(&z0, &z1, &z2, &z3, pa_inp_key, mb_mask); + + /* (K0, K1, K2, K3) = (MK0 ^ FK0, MK1 ^ FK1, MK2 ^ FK2, MK3 ^ FK3) */ + z0 = _mm512_xor_epi32(z0, _mm512_set1_epi32(SM4_FK[0])); + z1 = _mm512_xor_epi32(z1, _mm512_set1_epi32(SM4_FK[1])); + z2 = _mm512_xor_epi32(z2, _mm512_set1_epi32(SM4_FK[2])); + z3 = _mm512_xor_epi32(z3, _mm512_set1_epi32(SM4_FK[3])); + + const int32u* pCK = SM4_CK; + const __m512i* p_rk = (const __m512i*)key_sched; + + int itr; + for (itr = 0; itr < SM4_ROUNDS; itr += 4, pCK += 4, p_rk += 4) + SM4_FOUR_RK(z0, z1, z2, z3, rki, pCK, p_rk); + + /* clear copies of sensitive data and round keys */ + zero_mb8((int64u(*)[8])&z0, 1); + zero_mb8((int64u(*)[8])&z1, 1); + zero_mb8((int64u(*)[8])&z2, 1); + zero_mb8((int64u(*)[8])&z3, 1); + zero_mb8((int64u(*)[8])&rki, 1); +} + +DLL_PUBLIC +mbx_status16 mbx_sm4_set_key_mb16(mbx_sm4_key_schedule* key_sched, const sm4_key* pa_key[SM4_LINES]) +{ + int buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + + /* Test input pointers */ + if (NULL == key_sched || NULL == pa_key) { + status = MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* Don't process buffers with input pointers equal to zero */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_key[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + mb_mask &= ~(0x1 << buf_no); + } + } + + if (MBX_IS_ANY_OK_STS16(status)) + sm4_set_round_keys_mb16((int32u**)key_sched, (const int8u**)pa_key, mb_mask); + + return status; +} + +DLL_PUBLIC +mbx_status16 mbx_sm4_xts_set_keys_mb16(mbx_sm4_key_schedule* key_sched1, + mbx_sm4_key_schedule* key_sched2, + const sm4_xts_key* pa_key[SM4_LINES]) +{ + int buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + + /* Test input pointers */ + if (NULL == key_sched1 || NULL == key_sched2 || NULL == pa_key) + return MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + + /* Don't process buffers with input pointers equal to zero */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_key[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + mb_mask &= ~(0x1 << buf_no); + } + } + + if (MBX_IS_ANY_OK_STS16(status)) { + /* Generate round keys for key1 */ + sm4_set_round_keys_mb16((int32u**)key_sched1, (const int8u**)pa_key, mb_mask); + + const sm4_key* pa_key2[SM4_LINES]; + + for (int i = 0; i < SM4_LINES; i++) + pa_key2[i] = (const sm4_key*)&((int8u*)pa_key[i])[16]; + + /* Generate round keys for key2 */ + sm4_set_round_keys_mb16((int32u**)key_sched2, (const int8u**)pa_key2, mb_mask); + } + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_xts_dec_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_xts_dec_mb16.c new file mode 100644 index 0000000..1b1530e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_xts_dec_mb16.c @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2023 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include + +DLL_PUBLIC +mbx_status16 mbx_sm4_xts_decrypt_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], + const mbx_sm4_key_schedule* key_sched1, + const mbx_sm4_key_schedule* key_sched2, + const int8u* pa_tweak[SM4_LINES]) +{ + unsigned buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + + /* Test input pointers */ + if (NULL == pa_out || NULL == pa_inp || NULL == len || + NULL == key_sched1 || NULL == key_sched2 || NULL == pa_tweak) + return MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + + /* Test input data length and input pointers */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_out[buf_no] == NULL || pa_inp[buf_no] == NULL || pa_tweak[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + /* Do not process empty buffers */ + mb_mask &= ~(0x1 << buf_no); + } + if (len[buf_no] < SM4_BLOCK_SIZE) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + /* Do not process non-valid buffers */ + mb_mask &= ~(0x1 << buf_no); + } + if (len[buf_no] > SM4_XTS_MAX_SIZE) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + /* Do not process non-valid buffers */ + mb_mask &= ~(0x1 << buf_no); + } + } + + if (MBX_IS_ANY_OK_STS16(status)) + sm4_xts_kernel_mb16(pa_out, (const int8u**)pa_inp, (const int*)len, + (const int32u**)key_sched1, (const int32u**)key_sched2, + pa_tweak, mb_mask, SM4_DEC); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_xts_enc_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_xts_enc_mb16.c new file mode 100644 index 0000000..6622b7b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_xts_enc_mb16.c @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2023 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#include +#include + +DLL_PUBLIC +mbx_status16 mbx_sm4_xts_encrypt_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], + const int len[SM4_LINES], const mbx_sm4_key_schedule* key_sched1, + const mbx_sm4_key_schedule* key_sched2, + const int8u* pa_tweak[SM4_LINES]) +{ + unsigned buf_no; + mbx_status16 status = 0; + __mmask16 mb_mask = 0xFFFF; + + /* Test input pointers */ + if (NULL == pa_out || NULL == pa_inp || NULL == len || + NULL == key_sched1 || NULL == key_sched2 || NULL == pa_tweak) + return MBX_SET_STS16_ALL(MBX_STATUS_NULL_PARAM_ERR); + + /* Test input data length and input pointers */ + for (buf_no = 0; buf_no < SM4_LINES; buf_no++) { + if (pa_out[buf_no] == NULL || pa_inp[buf_no] == NULL || pa_tweak[buf_no] == NULL) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + /* Do not process empty buffers */ + mb_mask &= ~(0x1 << buf_no); + } + if (len[buf_no] < SM4_BLOCK_SIZE) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + /* Do not process non-valid buffers */ + mb_mask &= ~(0x1 << buf_no); + } + if (len[buf_no] > SM4_XTS_MAX_SIZE) { + status = MBX_SET_STS16(status, buf_no, MBX_STATUS_MISMATCH_PARAM_ERR); + /* Do not process non-valid buffers */ + mb_mask &= ~(0x1 << buf_no); + } + } + + if (MBX_IS_ANY_OK_STS16(status)) + sm4_xts_kernel_mb16(pa_out, (const int8u**)pa_inp, (const int*)len, + (const int32u**)key_sched1, (const int32u**)key_sched2, + pa_tweak, mb_mask, SM4_ENC); + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_xts_mb16.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_xts_mb16.c new file mode 100644 index 0000000..02db407 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/sm4/sm4_xts_mb16.c @@ -0,0 +1,529 @@ +/******************************************************************************* +* Copyright (C) 2023 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include +#include + +#define FIRST_TWEAKS 1 +#define NEXT_TWEAKS 0 + +/* Generate the next 4 tweaks for a given buffer */ +static void generate_next_4_tweaks(const __m512i *PREV_TWEAK, __m512i *NEXT_TWEAK, + const __m512i z_shuf_mask, const __m512i z_poly, + const int first_tweaks) +{ + __m512i TMP1, TMP2, TMP3, TMP4; + const __mmask8 xor_mask = _cvtu32_mask8(0xAA); + + TMP1 = _mm512_shuffle_epi8(*PREV_TWEAK, z_shuf_mask); + /* + * In case of the first 4 tweaks, the shifts are variable, + * as we are start from tweak 1 in all 128-bit lanes, to construct + * tweaks 1, 2, 3 and 4 + */ + if (first_tweaks) { + const __m512i z_dq3210 = _mm512_loadu_si512(xts_const_dq3210); + const __m512i z_dq5678 = _mm512_loadu_si512(xts_const_dq5678); + + TMP2 = _mm512_sllv_epi64(*PREV_TWEAK, z_dq3210); + TMP3 = _mm512_srlv_epi64(TMP1, z_dq5678); + /* + * For following tweaks, the shifts are constant, + * as we calculate the next 4 tweaks, parting from tweaks N-4, N-3, N-2 and N, + * to construct tweaks N, N+1, N+2, N+3 + */ + } else { + TMP2 = _mm512_slli_epi64(*PREV_TWEAK, 4); + TMP3 = _mm512_srli_epi64(TMP1, 4); + } + TMP4 = _mm512_clmulepi64_epi128(TMP3, z_poly, 0); + TMP2 = _mm512_mask_xor_epi64(TMP2, xor_mask, TMP2, TMP3); + *NEXT_TWEAK = _mm512_xor_epi32(TMP4, TMP2); +} + +/* Prepare the last tweaks for a given buffer, if it has a partial block */ +static void prepare_last_tweaks(__m512i *TWEAK, __m512i *NEXT_TWEAK, + const int operation, int num_remain_full_blocks) +{ + /* + * For the encryption case, we need to prepare the tweak + * for the partial block to be at the beginning of NEXT_TWEAK, + * so depending on the number of remaining full blocks, its position + * will vary, so the permute mask will be different. In case, there are 4 full blocks, + * the newly generated NEXT_TWEAK will be positioned correctly. + */ + if (operation == SM4_ENC) { + if (num_remain_full_blocks == 1) + *NEXT_TWEAK = _mm512_permutexvar_epi64(_mm512_loadu_si512(&xts_next_tweak_permq_enc[0]), *TWEAK); + else if (num_remain_full_blocks == 2) + *NEXT_TWEAK = _mm512_permutexvar_epi64(_mm512_loadu_si512(&xts_next_tweak_permq_enc[1*8]), *TWEAK); + else if (num_remain_full_blocks == 3) + *NEXT_TWEAK = _mm512_permutexvar_epi64(_mm512_loadu_si512(&xts_next_tweak_permq_enc[2*8]), *TWEAK); + /* + * For the decryption case, it is a bit more complicated. + * In case of a partial block, the last two tweaks (the last tweak of the last full block) + * and the tweak of the last block, need to be interchanged. + * TWEAK will have the tweaks for the last FULL blocks and *NEXT_TWEAK, + * as earlier, will have the tweak for the last partial block. + */ + } else { + if (num_remain_full_blocks == 1) { + *NEXT_TWEAK = _mm512_permutexvar_epi64(_mm512_loadu_si512(&xts_next_tweak_permq[0]), *TWEAK); + *TWEAK = _mm512_permutexvar_epi64(_mm512_loadu_si512(&xts_tweak_permq[0]), *TWEAK); + } else if (num_remain_full_blocks == 2) { + *NEXT_TWEAK = _mm512_permutexvar_epi64(_mm512_loadu_si512(&xts_next_tweak_permq[1*8]), *TWEAK); + *TWEAK = _mm512_permutexvar_epi64(_mm512_loadu_si512(&xts_tweak_permq[1*8]), *TWEAK); + } else if (num_remain_full_blocks == 3) { + *NEXT_TWEAK = _mm512_permutexvar_epi64(_mm512_loadu_si512(&xts_next_tweak_permq[2*8]), *TWEAK); + *TWEAK = _mm512_permutexvar_epi64(_mm512_loadu_si512(&xts_tweak_permq[2*8]), *TWEAK); + } else if (num_remain_full_blocks == 4) { + *NEXT_TWEAK = _mm512_permutex2var_epi64(*NEXT_TWEAK, _mm512_loadu_si512(&xts_next_tweak_permq[3*8]), *TWEAK); + *TWEAK = _mm512_permutex2var_epi64(*TWEAK, _mm512_loadu_si512(&xts_tweak_permq[3*8]), *NEXT_TWEAK); + } + } +} + +static void sm4_xts_mask_kernel_mb16(__m512i* NEXT_TWEAK, const __m512i* p_rk, __m512i loc_len32, + const int8u** loc_inp, int8u** loc_out, + __mmask16 mb_mask, const int operation) +{ + __m512i TMP[20]; + const __m512i z_poly = _mm512_loadu_si512(xts_poly); + const __m512i z_partial_block_mask = _mm512_loadu_si512(xts_partial_block_mask); + const __m512i z_full_block_mask = _mm512_loadu_si512(xts_full_block_mask); + const __m512i z_shuf_mask = _mm512_loadu_si512(xts_shuf_mask); + /* Length in bytes of partial blocks for all buffers */ + const __m512i partial_len32 = _mm512_and_si512(loc_len32, z_partial_block_mask); + /* Length in bytes of full blocks for all buffers */ + loc_len32 = _mm512_and_si512(loc_len32, z_full_block_mask); + + __mmask16 ge_64_mask = _mm512_mask_cmp_epi32_mask(mb_mask, loc_len32, _mm512_set1_epi32(4 * SM4_BLOCK_SIZE), _MM_CMPINT_NLT); + __mmask8 ge_64_mask_0_7 = (__mmask8) ge_64_mask; + __mmask8 ge_64_mask_8_15 = (__mmask8) _kshiftri_mask16(ge_64_mask, 8); + /* Expand 32-bit lengths to 64-bit lengths for 16 buffers */ + const __mmask16 expand_mask = _cvtu32_mask16(0x5555); + __m512i remain_len64_0_7 = _mm512_maskz_permutexvar_epi32(expand_mask, _mm512_loadu_si512(xts_dw0_7_to_qw_idx), loc_len32); + __m512i remain_len64_8_15 = _mm512_maskz_permutexvar_epi32(expand_mask, _mm512_loadu_si512(xts_dw8_15_to_qw_idx), loc_len32); + __m512i processed_len64_0_7; + __m512i processed_len64_8_15; + __m512i TWEAK[SM4_LINES]; + __m512i num_remain_full_blocks = _mm512_srli_epi32(loc_len32, 4); + /* Calculate bitmask of buffers with at least one full block */ + __mmask16 tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, loc_len32, _mm512_set1_epi32(0), _MM_CMPINT_NLE); + + /* + * While there is at least one full block in any of the buffer, keep encrypting + * (this loop only handles full blocks, but some buffers will have here + * less than 4 full blocks) + */ + while (tmp_mask) { + /* Mask for data loading */ + __mmask64 stream_mask[16]; + int i; + + int* p_loc_len32 = (int*)&loc_len32; + int* p_num_remain_full_blocks = (int*)&num_remain_full_blocks; + int* p_partial_block = (int*)&partial_len32; + + /* Generate tweaks for next rounds */ + for (i = 0; i < SM4_LINES; i++) { + TWEAK[i] = NEXT_TWEAK[i]; + /* + * If there are at least 4 more full blocks to process, + * at least one more tweak will be needed (for more full blocks or + * for a last partial block) + */ + if (p_num_remain_full_blocks[i] >= 4) + generate_next_4_tweaks(&TWEAK[i], &NEXT_TWEAK[i], z_shuf_mask, z_poly, NEXT_TWEAKS); + + /* If there is a partial block, tweaks need to be rearranged depending on cipher direction */ + if ((p_partial_block[i] > 0) & (p_num_remain_full_blocks[i] <= 4)) + prepare_last_tweaks(&TWEAK[i], &NEXT_TWEAK[i], operation, p_num_remain_full_blocks[i]); + } + + num_remain_full_blocks = _mm512_sub_epi32(num_remain_full_blocks, _mm512_set1_epi32(4)); + + /* + * XOR plaintext from each lane with the 4 tweaks and transpose to prepare for encryption. + * Since some buffers will have less than 4 full blocks, + * a bitmask is required to load less than 64 bytes (stream_mask) + */ + UPDATE_STREAM_MASK_64(stream_mask[0], p_loc_len32) + TMP[0] = _mm512_xor_si512(TWEAK[0], _mm512_maskz_loadu_epi8(stream_mask[0], loc_inp[0])); + UPDATE_STREAM_MASK_64(stream_mask[1], p_loc_len32) + TMP[1] = _mm512_xor_si512(TWEAK[1], _mm512_maskz_loadu_epi8(stream_mask[1], loc_inp[1])); + UPDATE_STREAM_MASK_64(stream_mask[2], p_loc_len32) + TMP[2] = _mm512_xor_si512(TWEAK[2], _mm512_maskz_loadu_epi8(stream_mask[2], loc_inp[2])); + UPDATE_STREAM_MASK_64(stream_mask[3], p_loc_len32) + TMP[3] = _mm512_xor_si512(TWEAK[3], _mm512_maskz_loadu_epi8(stream_mask[3], loc_inp[3])); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + UPDATE_STREAM_MASK_64(stream_mask[4], p_loc_len32) + TMP[0] = _mm512_xor_si512(TWEAK[4], _mm512_maskz_loadu_epi8(stream_mask[4], loc_inp[4])); + UPDATE_STREAM_MASK_64(stream_mask[5], p_loc_len32) + TMP[1] = _mm512_xor_si512(TWEAK[5], _mm512_maskz_loadu_epi8(stream_mask[5], loc_inp[5])); + UPDATE_STREAM_MASK_64(stream_mask[6], p_loc_len32) + TMP[2] = _mm512_xor_si512(TWEAK[6], _mm512_maskz_loadu_epi8(stream_mask[6], loc_inp[6])); + UPDATE_STREAM_MASK_64(stream_mask[7], p_loc_len32) + TMP[3] = _mm512_xor_si512(TWEAK[7], _mm512_maskz_loadu_epi8(stream_mask[7], loc_inp[7])); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], TMP[0], TMP[1], TMP[2], TMP[3]); + + UPDATE_STREAM_MASK_64(stream_mask[8], p_loc_len32) + TMP[0] = _mm512_xor_si512(TWEAK[8], _mm512_maskz_loadu_epi8(stream_mask[8], loc_inp[8])); + UPDATE_STREAM_MASK_64(stream_mask[9], p_loc_len32) + TMP[1] = _mm512_xor_si512(TWEAK[9], _mm512_maskz_loadu_epi8(stream_mask[9], loc_inp[9])); + UPDATE_STREAM_MASK_64(stream_mask[10], p_loc_len32) + TMP[2] = _mm512_xor_si512(TWEAK[10], _mm512_maskz_loadu_epi8(stream_mask[10], loc_inp[10])); + UPDATE_STREAM_MASK_64(stream_mask[11], p_loc_len32) + TMP[3] = _mm512_xor_si512(TWEAK[11], _mm512_maskz_loadu_epi8(stream_mask[11], loc_inp[11])); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + + TRANSPOSE_INP_512(TMP[12], TMP[13], TMP[14], TMP[15], TMP[0], TMP[1], TMP[2], TMP[3]); + + UPDATE_STREAM_MASK_64(stream_mask[12], p_loc_len32) + TMP[0] = _mm512_xor_si512(TWEAK[12], _mm512_maskz_loadu_epi8(stream_mask[12], loc_inp[12])); + UPDATE_STREAM_MASK_64(stream_mask[13], p_loc_len32) + TMP[1] = _mm512_xor_si512(TWEAK[13], _mm512_maskz_loadu_epi8(stream_mask[13], loc_inp[13])); + UPDATE_STREAM_MASK_64(stream_mask[14], p_loc_len32) + TMP[2] = _mm512_xor_si512(TWEAK[14], _mm512_maskz_loadu_epi8(stream_mask[14], loc_inp[14])); + UPDATE_STREAM_MASK_64(stream_mask[15], p_loc_len32) + TMP[3] = _mm512_xor_si512(TWEAK[15], _mm512_maskz_loadu_epi8(stream_mask[15], loc_inp[15])); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + + TRANSPOSE_INP_512(TMP[16], TMP[17], TMP[18], TMP[19], TMP[0], TMP[1], TMP[2], TMP[3]); + + SM4_KERNEL(TMP, p_rk, operation); + p_rk -= operation*SM4_ROUNDS; + + /* Transpose, XOR with the tweaks again and write data out */ + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_mask_storeu_epi8((__m512i*)loc_out[0], stream_mask[0], _mm512_xor_si512(TMP[0], TWEAK[0])); + _mm512_mask_storeu_epi8((__m512i*)loc_out[1], stream_mask[1], _mm512_xor_si512(TMP[1], TWEAK[1])); + _mm512_mask_storeu_epi8((__m512i*)loc_out[2], stream_mask[2], _mm512_xor_si512(TMP[2], TWEAK[2])); + _mm512_mask_storeu_epi8((__m512i*)loc_out[3], stream_mask[3], _mm512_xor_si512(TMP[3], TWEAK[3])); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_mask_storeu_epi8((__m512i*)loc_out[4], stream_mask[4], _mm512_xor_si512(TMP[0], TWEAK[4])); + _mm512_mask_storeu_epi8((__m512i*)loc_out[5], stream_mask[5], _mm512_xor_si512(TMP[1], TWEAK[5])); + _mm512_mask_storeu_epi8((__m512i*)loc_out[6], stream_mask[6], _mm512_xor_si512(TMP[2], TWEAK[6])); + _mm512_mask_storeu_epi8((__m512i*)loc_out[7], stream_mask[7], _mm512_xor_si512(TMP[3], TWEAK[7])); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[12], TMP[13], TMP[14], TMP[15]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_mask_storeu_epi8((__m512i*)loc_out[8], stream_mask[8], _mm512_xor_si512(TMP[0], TWEAK[8])); + _mm512_mask_storeu_epi8((__m512i*)loc_out[9], stream_mask[9], _mm512_xor_si512(TMP[1], TWEAK[9])); + _mm512_mask_storeu_epi8((__m512i*)loc_out[10], stream_mask[10], _mm512_xor_si512(TMP[2],TWEAK[10])); + _mm512_mask_storeu_epi8((__m512i*)loc_out[11], stream_mask[11], _mm512_xor_si512(TMP[3],TWEAK[11])); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[16], TMP[17], TMP[18], TMP[19]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_mask_storeu_epi8((__m512i*)loc_out[12], stream_mask[12], _mm512_xor_si512(TMP[0], TWEAK[12])); + _mm512_mask_storeu_epi8((__m512i*)loc_out[13], stream_mask[13], _mm512_xor_si512(TMP[1], TWEAK[13])); + _mm512_mask_storeu_epi8((__m512i*)loc_out[14], stream_mask[14], _mm512_xor_si512(TMP[2], TWEAK[14])); + _mm512_mask_storeu_epi8((__m512i*)loc_out[15], stream_mask[15], _mm512_xor_si512(TMP[3], TWEAK[15])); + + /* Update input/output pointers to data */ + processed_len64_0_7 = _mm512_mask_blend_epi64(ge_64_mask_0_7, remain_len64_0_7, _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + processed_len64_8_15 = _mm512_mask_blend_epi64(ge_64_mask_8_15, remain_len64_8_15, _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + M512(loc_inp) = _mm512_add_epi64(_mm512_loadu_si512(loc_inp), processed_len64_0_7); + M512(loc_inp + 8) = _mm512_add_epi64(_mm512_loadu_si512(loc_inp + 8), processed_len64_8_15); + + M512(loc_out) = _mm512_add_epi64(_mm512_loadu_si512(loc_out), processed_len64_0_7); + M512(loc_out + 8) = _mm512_add_epi64(_mm512_loadu_si512(loc_out + 8), processed_len64_8_15); + + /* Update number of blocks left and processing mask */ + remain_len64_0_7 = _mm512_sub_epi64(remain_len64_0_7, processed_len64_0_7); + remain_len64_8_15 = _mm512_sub_epi64(remain_len64_8_15, processed_len64_8_15); + loc_len32 = _mm512_sub_epi32(loc_len32, _mm512_set1_epi32(4 * SM4_BLOCK_SIZE)); + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, loc_len32, _mm512_set1_epi32(0), _MM_CMPINT_NLE); + ge_64_mask_0_7 = _mm512_cmp_epi64_mask(remain_len64_0_7, _mm512_set1_epi64(4 * SM4_BLOCK_SIZE), _MM_CMPINT_NLT); + ge_64_mask_8_15 = _mm512_cmp_epi64_mask(remain_len64_8_15, _mm512_set1_epi64(4 * SM4_BLOCK_SIZE), _MM_CMPINT_NLT); + } + + /* At this stage, all buffers have at most 15 bytes (a partial block) */ + + /* Calculate bitmask of buffers with a partial block */ + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, partial_len32, _mm512_set1_epi32(0), _MM_CMPINT_NLE); + + if (tmp_mask) { + /* Encrypt last plaintext using bytes from previous ciphertext block */ + __mmask64 stream_mask[16]; + int* p_loc_len32 = (int*)&partial_len32; + __m128i XTMP[16]; + int i; + + for (i = 0; i < SM4_LINES; i++) { + /* Get right tweak (position tweak in last 16 bytes of ZMM register) */ + UPDATE_STREAM_MASK_64(stream_mask[i], p_loc_len32); + /* Read final bytes of input partial block */ + XTMP[i] = _mm_maskz_loadu_epi8((__mmask16)stream_mask[i], loc_inp[i]); + /* + * Read last bytes of previous output block to form 16 bytes + * (only if there is a partial block at the end of the buffer) + */ + if (stream_mask[i] == 0) + continue; + __m128i XOUT = _mm_maskz_loadu_epi8((__mmask16)~stream_mask[i], (loc_out[i] - 16)); + XTMP[i] = _mm_or_si128(XTMP[i], XOUT); + /* Initial XOR of new constructed input with tweak */ + XTMP[i] = _mm_xor_si128(XTMP[i], _mm512_castsi512_si128(NEXT_TWEAK[i])); + } + + /* Encrypt final block from all lanes, compressing the 16 XMMs into 4 ZMMs */ + TRANSPOSE_16x4_I32_XMM_EPI32(&TMP[0], &TMP[1], &TMP[2], &TMP[3], XTMP); + for (i = 0; i < SM4_ROUNDS; i += 4, p_rk += 4*operation) + SM4_FOUR_ROUNDS(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], p_rk, operation); + + p_rk -= operation*SM4_ROUNDS; + + /* Spread out the 4 ZMMs into 16 XMMs */ + TRANSPOSE_4x16_I32_XMM_EPI32(&TMP[0], &TMP[1], &TMP[2], &TMP[3], XTMP); + for (i = 0; i < SM4_LINES; i++) { + /* Skip the buffer if there is no partial block left */ + if (stream_mask[i] == 0) + continue; + /* + * Final XOR of output with tweak (it will be always + * in the beginning of NEXT_TWEAK, hence the cast) + */ + XTMP[i] = _mm_xor_si128(XTMP[i], _mm512_castsi512_si128(NEXT_TWEAK[i])); + /* Write first bytes of previous output block as the output of the partial block */ + __m128i XOUT = _mm_maskz_loadu_epi8((__mmask16)stream_mask[i], (loc_out[i] - 16)); + _mm_mask_storeu_epi8(loc_out[i], (__mmask16)stream_mask[i], XOUT); + /* Write last output as the output of the previous block */ + _mm_storeu_si128((__m128i*)(loc_out[i] - 16), XTMP[i]); + } + } + /* clear local copy of sensitive data */ + zero_mb8((int64u(*)[8])TMP, sizeof(TMP) / sizeof(TMP[0])); +} + +void sm4_xts_kernel_mb16(int8u* pa_out[SM4_LINES], const int8u* pa_inp[SM4_LINES], const int len[SM4_LINES], + const int32u* key_sched1[SM4_ROUNDS], const int32u* key_sched2[SM4_ROUNDS], + const int8u* pa_tweak[SM4_LINES], __mmask16 mb_mask, const int operation) +{ + __ALIGN64 const int8u* loc_inp[SM4_LINES]; + __ALIGN64 int8u* loc_out[SM4_LINES]; + + /* Create the local copy of the input data length in bytes and set it to zero for non-valid buffers */ + __m512i loc_len; + loc_len = _mm512_loadu_si512(len); + loc_len = _mm512_mask_set1_epi32(loc_len, ~mb_mask, 0); + + /* Local copies of the pointers to input and otput buffers */ + _mm512_storeu_si512((void*)loc_inp, _mm512_loadu_si512(pa_inp)); + _mm512_storeu_si512((void*)(loc_inp + 8), _mm512_loadu_si512(pa_inp + 8)); + + _mm512_storeu_si512(loc_out, _mm512_loadu_si512(pa_out)); + _mm512_storeu_si512(loc_out + 8, _mm512_loadu_si512(pa_out + 8)); + + /* Depending on the operation(enc or dec): sign allows to go up and down on the key schedule + * p_rk set to the beginning or to the end of the key schedule */ + const __m512i* p_rk1 = (operation == SM4_ENC) ? (const __m512i*)key_sched1 : ((const __m512i*)key_sched1 + (SM4_ROUNDS - 1)); + /* Pointer p_rk2 is set to the beginning of the key schedule, + * as it always encrypts the tweak, regardless the direction */ + const __m512i* p_rk2 = (const __m512i*)key_sched2; + + /* TMP[] - temporary buffer for processing */ + /* TWEAK - tweak values for current blocks (4 blocks per buffer) */ + /* NEXT_TWEAK - tweak values for following blocks (4 blocks per buffer) */ + /* inital_tweak - first tweak for all buffers */ + __m512i TMP[20]; + __m512i TWEAK[SM4_LINES]; + __m512i NEXT_TWEAK[SM4_LINES]; + __m128i initial_tweak[SM4_LINES]; + int i; + + const __m512i z_poly = _mm512_loadu_si512(xts_poly); + const __m512i z_shuf_mask = _mm512_loadu_si512(xts_shuf_mask); + + /* Encrypt initial tweak */ + TRANSPOSE_16x4_I32_EPI32(&TMP[0], &TMP[1], &TMP[2], &TMP[3], pa_tweak, mb_mask); + + for (i = 0; i < SM4_ROUNDS; i += 4, p_rk2 += 4) + SM4_FOUR_ROUNDS(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], p_rk2, SM4_ENC); + + p_rk2 -= SM4_ROUNDS; + + TRANSPOSE_4x16_I32_O128_EPI32(&TMP[0], &TMP[1], &TMP[2], &TMP[3], initial_tweak, mb_mask); + + /* Load TWEAK value from valid buffers and generate first 4 values */ + for (i = 0; i < SM4_LINES; i++) { + TWEAK[i] = _mm512_broadcast_i64x2(initial_tweak[i]); + generate_next_4_tweaks(&TWEAK[i], &NEXT_TWEAK[i], z_shuf_mask, z_poly, FIRST_TWEAKS); + } + + /* + * Generate the mask to process 4 full blocks from each buffer. + * Less than 5 full blocks requires sm4_xts_mask_kernel_mb16 to handle it, + * as it is the function that can handle partial blocks. + */ + __mmask16 tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, loc_len, _mm512_set1_epi32(5 * SM4_BLOCK_SIZE), _MM_CMPINT_NLT); + + /* Go to this loop if all 16 buffers contain at least 5 full blocks each */ + while (tmp_mask == 0xFFFF) { + for (i = 0; i < SM4_LINES; i++) { + TWEAK[i] = NEXT_TWEAK[i]; + + /* Update tweaks for next rounds */ + generate_next_4_tweaks(&TWEAK[i], &NEXT_TWEAK[i], z_shuf_mask, z_poly, NEXT_TWEAKS); + } + + /* XOR plaintext from each lane with the 4 tweaks and transpose to prepare for encryption */ + TMP[0] = _mm512_xor_si512(TWEAK[0], _mm512_loadu_si512(loc_inp[0])); + TMP[1] = _mm512_xor_si512(TWEAK[1], _mm512_loadu_si512(loc_inp[1])); + TMP[2] = _mm512_xor_si512(TWEAK[2], _mm512_loadu_si512(loc_inp[2])); + TMP[3] = _mm512_xor_si512(TWEAK[3], _mm512_loadu_si512(loc_inp[3])); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_xor_si512(TWEAK[4], _mm512_loadu_si512(loc_inp[4])); + TMP[1] = _mm512_xor_si512(TWEAK[5], _mm512_loadu_si512(loc_inp[5])); + TMP[2] = _mm512_xor_si512(TWEAK[6], _mm512_loadu_si512(loc_inp[6])); + TMP[3] = _mm512_xor_si512(TWEAK[7], _mm512_loadu_si512(loc_inp[7])); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_xor_si512(TWEAK[8], _mm512_loadu_si512(loc_inp[8])); + TMP[1] = _mm512_xor_si512(TWEAK[9], _mm512_loadu_si512(loc_inp[9])); + TMP[2] = _mm512_xor_si512(TWEAK[10], _mm512_loadu_si512(loc_inp[10])); + TMP[3] = _mm512_xor_si512(TWEAK[11], _mm512_loadu_si512(loc_inp[11])); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + + TRANSPOSE_INP_512(TMP[12], TMP[13], TMP[14], TMP[15], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_xor_si512(TWEAK[12], _mm512_loadu_si512(loc_inp[12])); + TMP[1] = _mm512_xor_si512(TWEAK[13], _mm512_loadu_si512(loc_inp[13])); + TMP[2] = _mm512_xor_si512(TWEAK[14], _mm512_loadu_si512(loc_inp[14])); + TMP[3] = _mm512_xor_si512(TWEAK[15], _mm512_loadu_si512(loc_inp[15])); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + + TRANSPOSE_INP_512(TMP[16], TMP[17], TMP[18], TMP[19], TMP[0], TMP[1], TMP[2], TMP[3]); + + SM4_KERNEL(TMP, p_rk1, operation); + p_rk1 -= operation*SM4_ROUNDS; + + /* Transpose, XOR with the tweaks again and write data out */ + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)loc_out[0], _mm512_xor_si512(TMP[0], TWEAK[0])); + _mm512_storeu_si512((__m512i*)loc_out[1], _mm512_xor_si512(TMP[1], TWEAK[1])); + _mm512_storeu_si512((__m512i*)loc_out[2], _mm512_xor_si512(TMP[2], TWEAK[2])); + _mm512_storeu_si512((__m512i*)loc_out[3], _mm512_xor_si512(TMP[3], TWEAK[3])); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)loc_out[4], _mm512_xor_si512(TMP[0], TWEAK[4])); + _mm512_storeu_si512((__m512i*)loc_out[5], _mm512_xor_si512(TMP[1], TWEAK[5])); + _mm512_storeu_si512((__m512i*)loc_out[6], _mm512_xor_si512(TMP[2], TWEAK[6])); + _mm512_storeu_si512((__m512i*)loc_out[7], _mm512_xor_si512(TMP[3], TWEAK[7])); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[12], TMP[13], TMP[14], TMP[15]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)loc_out[8], _mm512_xor_si512(TMP[0], TWEAK[8])); + _mm512_storeu_si512((__m512i*)loc_out[9], _mm512_xor_si512(TMP[1], TWEAK[9])); + _mm512_storeu_si512((__m512i*)loc_out[10], _mm512_xor_si512(TMP[2], TWEAK[10])); + _mm512_storeu_si512((__m512i*)loc_out[11], _mm512_xor_si512(TMP[3], TWEAK[11])); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[16], TMP[17], TMP[18], TMP[19]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)loc_out[13], _mm512_xor_si512(TMP[1], TWEAK[13])); + _mm512_storeu_si512((__m512i*)loc_out[13], _mm512_xor_si512(TMP[1], TWEAK[13])); + _mm512_storeu_si512((__m512i*)loc_out[14], _mm512_xor_si512(TMP[2], TWEAK[14])); + _mm512_storeu_si512((__m512i*)loc_out[15], _mm512_xor_si512(TMP[3], TWEAK[15])); + + /* Update input/output pointers to data */ + M512(loc_inp) = _mm512_add_epi64(_mm512_loadu_si512(loc_inp), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + M512(loc_inp + 8) = _mm512_add_epi64(_mm512_loadu_si512(loc_inp + 8), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + + M512(loc_out) = _mm512_add_epi64(_mm512_loadu_si512(loc_out), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + M512(loc_out + 8) = _mm512_add_epi64(_mm512_loadu_si512(loc_out + 8), _mm512_set1_epi64(4 * SM4_BLOCK_SIZE)); + + /* Update number of blocks left and processing mask */ + loc_len = _mm512_sub_epi32(loc_len, _mm512_set1_epi32(4 * SM4_BLOCK_SIZE)); + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, loc_len, _mm512_set1_epi32(5 * SM4_BLOCK_SIZE), _MM_CMPINT_NLT); + } + + /* Check if we have any data left on any of the buffers */ + tmp_mask = _mm512_mask_cmp_epi32_mask(mb_mask, loc_len, _mm512_setzero_si512(), _MM_CMPINT_NLE); + /* + * At this point, at least one buffer has less than 5 full blocks, + * so dealing with a partial block might be needed. + */ + if (tmp_mask) + sm4_xts_mask_kernel_mb16(NEXT_TWEAK, p_rk1, loc_len, loc_inp, loc_out, mb_mask, operation); + + /* clear local copy of sensitive data */ + zero_mb8((int64u(*)[8])TMP, sizeof(TMP) / sizeof(TMP[0])); + zero_mb8((int64u(*)[8])TWEAK, sizeof(TWEAK) / sizeof(TWEAK[0])); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/x25519/ifma_x25519.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/x25519/ifma_x25519.c new file mode 100644 index 0000000..d837e1b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/crypto_mb/src/x25519/ifma_x25519.c @@ -0,0 +1,1713 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include + +#include +#include +#include +#include + +#ifndef __GNUC__ +#pragma warning(disable:4013) +#endif + +#define MASK47 ((1ULL << (255 - 52 * 4)) - 1) + +__ALIGN64 static const int64u MASK47_[8] = {MASK47, MASK47, MASK47, MASK47, + MASK47, MASK47, MASK47, MASK47}; +__ALIGN64 static const int64u MOD_2_255_[8] = {19, 19, 19, 19, 19, 19, 19, 19}; +__ALIGN64 static const int64u MOD_2_260_[8] = {19*32, 19*32, 19*32, 19*32, + 19*32, 19*32, 19*32, 19*32}; +#define MASK_47 loadu64(MASK47_) +#define MOD_2_255 loadu64(MOD_2_255_) +#define MOD_2_260 loadu64(MOD_2_260_) + +#define ROUND_MUL_SRC(I, J, S_LO, R_LO, S_HI, R_HI) \ + R_LO = fma52lo(S_LO, va[I], vb[J]); \ + R_HI = fma52hi(S_HI, va[I], vb[J]); + +#define ROUND_MUL(I, J, M0, M1) \ + ROUND_MUL_SRC(I, J, M0, M0, M1, M1) + +#define REDUCE_ROUND(R0, R1, R5) \ + r##R0 = fma52lo(r##R0, r##R5, MOD_2_260); \ + r##R1 = fma52lo( fma52hi(r##R1, r##R5, MOD_2_260), \ + srli64(r##R5, 52), MOD_2_260); + +#define NORM(I, J) \ + r##J = add64(r##J, srli64(r##I, 52)); \ + r##I = and64_const(r##I, (1ULL << 52) - 1); + +//////////////////////////////////////////////////////////// + +__INLINE void ed25519_mul(U64 out[], const U64 a[], const U64 b[]) { + U64 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9; + + U64 *va = (U64*) a; + U64 *vb = (U64*) b; + U64 *vr = (U64*) out; + + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = get_zero64(); + + // Full multiplication + ROUND_MUL(4, 4, r8, r9) + + ROUND_MUL(3, 0, r3, r4) + ROUND_MUL(1, 2, r3, r4) + ROUND_MUL(0, 3, r3, r4) + ROUND_MUL(2, 1, r3, r4) + ROUND_MUL(2, 2, r4, r5) + ROUND_MUL(0, 4, r4, r5) + ROUND_MUL(1, 3, r4, r5) + ROUND_MUL(3, 1, r4, r5) + ROUND_MUL(4, 0, r4, r5) + + ROUND_MUL(1, 4, r5, r6) + ROUND_MUL(2, 3, r5, r6) + ROUND_MUL(3, 2, r5, r6) + ROUND_MUL(4, 1, r5, r6) + ROUND_MUL(2, 4, r6, r7) + ROUND_MUL(3, 3, r6, r7) + ROUND_MUL(4, 2, r6, r7) + + ROUND_MUL(0, 0, r0, r1) + ROUND_MUL(0, 1, r1, r2) + ROUND_MUL(0, 2, r2, r3) + ROUND_MUL(1, 0, r1, r2) + ROUND_MUL(1, 1, r2, r3) + ROUND_MUL(2, 0, r2, r3) + ROUND_MUL(3, 4, r7, r8) + ROUND_MUL(4, 3, r7, r8) + + r4 = fma52lo(r4, r9, MOD_2_260); + r0 = fma52lo(r0, srli64(r4, 47), MOD_2_255); + r4 = and64(r4, MASK_47); + + REDUCE_ROUND(0, 1, 5); + REDUCE_ROUND(1, 2, 6); + REDUCE_ROUND(2, 3, 7); + REDUCE_ROUND(3, 4, 8); + + // Normalize result + NORM(0,1) + NORM(1,2) + NORM(2,3) + NORM(3,4) + + storeu64(&vr[0], r0); + storeu64(&vr[1], r1); + storeu64(&vr[2], r2); + storeu64(&vr[3], r3); + storeu64(&vr[4], r4); +} + +/* SQR +c=0 (0,0) +c=1 (0,1) +c=2 (0,2) (1,1) +c=3 (0,3) (1,2) +c=4 (0,4) (1,3) (2,2) +c=5 (1,4) (2,3) +c=6 (2,4) (3,3) +c=7 (3,4) +c=8 (4,4) +*/ + +__INLINE void ed25519_sqr(U64 out[], const U64 a[]) { + U64 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9; + + U64 *va = (U64*) a; + U64 *vb = (U64*) a; + U64 *vr = (U64*) out; + + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = get_zero64(); + + // Square + ROUND_MUL(0, 1, r1, r2) + ROUND_MUL(0, 2, r2, r3) + ROUND_MUL(0, 3, r3, r4) + ROUND_MUL(0, 4, r4, r5) + ROUND_MUL(1, 4, r5, r6) + ROUND_MUL(2, 4, r6, r7) + ROUND_MUL(3, 4, r7, r8) + + ROUND_MUL(1, 2, r3, r4) + ROUND_MUL(1, 3, r4, r5) + ROUND_MUL(2, 3, r5, r6) + + r1 = add64(r1, r1); + r2 = add64(r2, r2); + r3 = add64(r3, r3); + r4 = add64(r4, r4); + r5 = add64(r5, r5); + r6 = add64(r6, r6); + r7 = add64(r7, r7); + r8 = add64(r8, r8); + + ROUND_MUL(0, 0, r0, r1) + ROUND_MUL(1, 1, r2, r3) + ROUND_MUL(2, 2, r4, r5) + ROUND_MUL(3, 3, r6, r7) + ROUND_MUL(4, 4, r8, r9) + + // Reduce r4 upper bits + r4 = fma52lo(r4, r9, MOD_2_260); + r0 = fma52lo(r0, srli64(r4, 47), MOD_2_255); + r4 = and64(r4, MASK_47); + + REDUCE_ROUND(0, 1, 5); + REDUCE_ROUND(1, 2, 6); + REDUCE_ROUND(2, 3, 7); + REDUCE_ROUND(3, 4, 8); + + // Normalize result + NORM(0,1) + NORM(1,2) + NORM(2,3) + NORM(3,4) + + storeu64(&vr[0], r0); + storeu64(&vr[1], r1); + storeu64(&vr[2], r2); + storeu64(&vr[3], r3); + storeu64(&vr[4], r4); +} + +#define ROUND_MUL_SRC_A(I, J, S_LO, R_LO, S_HI, R_HI) \ + R_LO = fma52lo(S_LO, a##I, a##J); \ + R_HI = fma52hi(S_HI, a##I, a##J); + +#define ROUND_MUL_A(I, J, M0, M1) \ + ROUND_MUL_SRC_A(I, J, M0, M0, M1, M1) + +#define NORM(I, J) \ + r##J = add64(r##J, srli64(r##I, 52)); \ + r##I = and64_const(r##I, (1ULL << 52) - 1); + + +static void MB_FUNC_NAME(ed25519_sqr_latency_)(U64 out[], const U64 a[], int count) { + U64 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9; + U64 a0, a1, a2, a3, a4; + U64 r4_1; + int i; + + U64 *vr = (U64*) out; + + a0 = a[0]; + a1 = a[1]; + a2 = a[2]; + a3 = a[3]; + a4 = a[4]; + for (i = 0; i < count; ++i) + { + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = get_zero64(); + r4_1 = get_zero64(); + + // Square + ROUND_MUL_A(0, 1, r1, r2) + ROUND_MUL_A(0, 2, r2, r3) + ROUND_MUL_A(0, 3, r3, r4_1) + ROUND_MUL_A(0, 4, r4_1, r5) + ROUND_MUL_A(1, 4, r5, r6) + ROUND_MUL_A(2, 4, r6, r7) + ROUND_MUL_A(3, 4, r7, r8) + + ROUND_MUL_A(1, 2, r3, r4) + ROUND_MUL_A(1, 3, r4, r5) + ROUND_MUL_A(2, 3, r5, r6) + + r1 = add64(r1, r1); + r2 = add64(r2, r2); + r3 = add64(r3, r3); + + r4 = add64(r4, r4_1); + r4 = add64(r4, r4); + + r5 = add64(r5, r5); + r6 = add64(r6, r6); + r7 = add64(r7, r7); + r8 = add64(r8, r8); + + ROUND_MUL_A(0, 0, r0, r1) + ROUND_MUL_A(1, 1, r2, r3) + ROUND_MUL_A(2, 2, r4, r5) + ROUND_MUL_A(3, 3, r6, r7) + ROUND_MUL_A(4, 4, r8, r9) + + r4 = fma52lo(r4, r9, MOD_2_260); + r0 = fma52lo(r0, srli64(r4, 47), MOD_2_255); + r4 = and64(r4, MASK_47); + + REDUCE_ROUND(0, 1, 5); + REDUCE_ROUND(1, 2, 6); + REDUCE_ROUND(2, 3, 7); + REDUCE_ROUND(3, 4, 8); + + // Normalize result + NORM(0,1) + NORM(1,2) + NORM(2,3) + NORM(3,4) + + a0 = r0; + a1 = r1; + a2 = r2; + a3 = r3; + a4 = r4; + } + + storeu64(&vr[0], r0); + storeu64(&vr[1], r1); + storeu64(&vr[2], r2); + storeu64(&vr[3], r3); + storeu64(&vr[4], r4); +} + +#define MASK_R4 ((1ULL << (255 - 52 * 4)) - 1) +static const int64u VMASK_R4[8] = {MASK_R4, MASK_R4, MASK_R4, MASK_R4, + MASK_R4, MASK_R4, MASK_R4, MASK_R4}; + +#define MASK52 ((1ULL << 52) - 1) +static const int64u VMASK52[8] = {MASK52, MASK52, MASK52, MASK52, + MASK52, MASK52, MASK52, MASK52}; + +#define REDUCE_ROUND_(R, R0, R1, R5) \ + R##R0 = fma52lo(R##R0, R##R5, MOD_2_260); \ + R##R1 = fma52lo( fma52hi(R##R1, R##R5, MOD_2_260), \ + srli64(R##R5, 52), MOD_2_260); + +#define NORM_(R, I, J) \ + R##J = add64(R##J, srli64(R##I, 52)); \ + R##I = and64(R##I, loadu64(VMASK52)); + +#define REDUCE_R4_N_R9(R) \ + R##4 = fma52lo(R##4, R##9, MOD_2_260); \ + R##0 = fma52lo(R##0, srli64(R##4, 47), MOD_2_255); \ + R##4 = and64(R##4, loadu64(VMASK_R4)); + +__INLINE void ed25519_mul_dual(U64 out0[], U64 out1[], + const U64 a0[], const U64 b0[], + const U64 a1[], const U64 b1[]) { + + U64 r00, r01, r02, r03, r04, r05, r06, r07, r08, r09; + U64 r10, r11, r12, r13, r14, r15, r16, r17, r18, r19; + + U64 *vr0 = (U64*) out0; + U64 *vr1 = (U64*) out1; + + r00 = r01 = r02 = r03 = r04 = r05 = r06 = r07 = r08 = r09 = get_zero64(); + r10 = r11 = r12 = r13 = r14 = r15 = r16 = r17 = r18 = r19 = get_zero64(); + + // Full multiplication + U64 *va = (U64*) a0; + U64 *vb = (U64*) b0; + ROUND_MUL(4, 4, r08, r09) + ROUND_MUL(3, 0, r03, r04) + ROUND_MUL(1, 2, r03, r04) + ROUND_MUL(0, 3, r03, r04) + ROUND_MUL(2, 1, r03, r04) + ROUND_MUL(2, 2, r04, r05) + ROUND_MUL(0, 4, r04, r05) + ROUND_MUL(1, 3, r04, r05) + ROUND_MUL(3, 1, r04, r05) + ROUND_MUL(4, 0, r04, r05) + ROUND_MUL(1, 4, r05, r06) + ROUND_MUL(2, 3, r05, r06) + ROUND_MUL(3, 2, r05, r06) + ROUND_MUL(4, 1, r05, r06) + ROUND_MUL(2, 4, r06, r07) + ROUND_MUL(3, 3, r06, r07) + ROUND_MUL(4, 2, r06, r07) + ROUND_MUL(0, 0, r00, r01) + ROUND_MUL(0, 1, r01, r02) + ROUND_MUL(0, 2, r02, r03) + ROUND_MUL(1, 0, r01, r02) + ROUND_MUL(1, 1, r02, r03) + ROUND_MUL(2, 0, r02, r03) + ROUND_MUL(3, 4, r07, r08) + ROUND_MUL(4, 3, r07, r08) + + va = (U64*) a1; + vb = (U64*) b1; + ROUND_MUL(4, 4, r18, r19) + ROUND_MUL(3, 0, r13, r14) + ROUND_MUL(1, 2, r13, r14) + ROUND_MUL(0, 3, r13, r14) + ROUND_MUL(2, 1, r13, r14) + ROUND_MUL(2, 2, r14, r15) + ROUND_MUL(0, 4, r14, r15) + ROUND_MUL(1, 3, r14, r15) + ROUND_MUL(3, 1, r14, r15) + ROUND_MUL(4, 0, r14, r15) + ROUND_MUL(1, 4, r15, r16) + ROUND_MUL(2, 3, r15, r16) + ROUND_MUL(3, 2, r15, r16) + ROUND_MUL(4, 1, r15, r16) + ROUND_MUL(2, 4, r16, r17) + ROUND_MUL(3, 3, r16, r17) + ROUND_MUL(4, 2, r16, r17) + ROUND_MUL(0, 0, r10, r11) + ROUND_MUL(0, 1, r11, r12) + ROUND_MUL(0, 2, r12, r13) + ROUND_MUL(1, 0, r11, r12) + ROUND_MUL(1, 1, r12, r13) + ROUND_MUL(2, 0, r12, r13) + ROUND_MUL(3, 4, r17, r18) + ROUND_MUL(4, 3, r17, r18) + + REDUCE_R4_N_R9(r0) + REDUCE_R4_N_R9(r1) + + REDUCE_ROUND_(r0, 0, 1, 5); + REDUCE_ROUND_(r0, 1, 2, 6); + REDUCE_ROUND_(r0, 2, 3, 7); + REDUCE_ROUND_(r0, 3, 4, 8); + + REDUCE_ROUND_(r1, 0, 1, 5); + REDUCE_ROUND_(r1, 1, 2, 6); + REDUCE_ROUND_(r1, 2, 3, 7); + REDUCE_ROUND_(r1, 3, 4, 8); + + // Normalize result + NORM_(r0, 0,1) + NORM_(r0, 1,2) + NORM_(r0, 2,3) + NORM_(r0, 3,4) + + NORM_(r1, 0,1) + NORM_(r1, 1,2) + NORM_(r1, 2,3) + NORM_(r1, 3,4) + + storeu64(&vr0[0], r00); + storeu64(&vr0[1], r01); + storeu64(&vr0[2], r02); + storeu64(&vr0[3], r03); + storeu64(&vr0[4], r04); + + storeu64(&vr1[0], r10); + storeu64(&vr1[1], r11); + storeu64(&vr1[2], r12); + storeu64(&vr1[3], r13); + storeu64(&vr1[4], r14); +} + +__INLINE void ed25519_sqr_dual(U64 out0[], U64 out1[], + const U64 a0[], const U64 a1[]) { + + U64 r00, r01, r02, r03, r04, r05, r06, r07, r08, r09; + U64 r10, r11, r12, r13, r14, r15, r16, r17, r18, r19; + + U64 *vr0 = (U64*) out0; + U64 *vr1 = (U64*) out1; + + r00 = r01 = r02 = r03 = r04 = r05 = r06 = r07 = r08 = r09 = get_zero64(); + r10 = r11 = r12 = r13 = r14 = r15 = r16 = r17 = r18 = r19 = get_zero64(); + + // Square + U64 *va = (U64*) a0; + U64 *vb = (U64*) a0; + ROUND_MUL(0, 1, r01, r02) + ROUND_MUL(0, 2, r02, r03) + ROUND_MUL(0, 3, r03, r04) + ROUND_MUL(0, 4, r04, r05) + ROUND_MUL(1, 4, r05, r06) + ROUND_MUL(2, 4, r06, r07) + ROUND_MUL(3, 4, r07, r08) + ROUND_MUL(1, 2, r03, r04) + ROUND_MUL(1, 3, r04, r05) + ROUND_MUL(2, 3, r05, r06) + + r01 = add64(r01, r01); + r02 = add64(r02, r02); + r03 = add64(r03, r03); + r04 = add64(r04, r04); + r05 = add64(r05, r05); + r06 = add64(r06, r06); + r07 = add64(r07, r07); + r08 = add64(r08, r08); + + ROUND_MUL(0, 0, r00, r01) + ROUND_MUL(1, 1, r02, r03) + ROUND_MUL(2, 2, r04, r05) + ROUND_MUL(3, 3, r06, r07) + ROUND_MUL(4, 4, r08, r09) + + va = (U64*) a1; + vb = (U64*) a1; + ROUND_MUL(0, 1, r11, r12) + ROUND_MUL(0, 2, r12, r13) + ROUND_MUL(0, 3, r13, r14) + ROUND_MUL(0, 4, r14, r15) + ROUND_MUL(1, 4, r15, r16) + ROUND_MUL(2, 4, r16, r17) + ROUND_MUL(3, 4, r17, r18) + ROUND_MUL(1, 2, r13, r14) + ROUND_MUL(1, 3, r14, r15) + ROUND_MUL(2, 3, r15, r16) + + r11 = add64(r11, r11); + r12 = add64(r12, r12); + r13 = add64(r13, r13); + r14 = add64(r14, r14); + r15 = add64(r15, r15); + r16 = add64(r16, r16); + r17 = add64(r17, r17); + r18 = add64(r18, r18); + + ROUND_MUL(0, 0, r10, r11) + ROUND_MUL(1, 1, r12, r13) + ROUND_MUL(2, 2, r14, r15) + ROUND_MUL(3, 3, r16, r17) + ROUND_MUL(4, 4, r18, r19) + + REDUCE_R4_N_R9(r0) + REDUCE_R4_N_R9(r1) + + REDUCE_ROUND_(r0, 0, 1, 5); + REDUCE_ROUND_(r0, 1, 2, 6); + REDUCE_ROUND_(r0, 2, 3, 7); + REDUCE_ROUND_(r0, 3, 4, 8); + + REDUCE_ROUND_(r1, 0, 1, 5); + REDUCE_ROUND_(r1, 1, 2, 6); + REDUCE_ROUND_(r1, 2, 3, 7); + REDUCE_ROUND_(r1, 3, 4, 8); + + // Normalize result + NORM_(r0, 0,1) + NORM_(r0, 1,2) + NORM_(r0, 2,3) + NORM_(r0, 3,4) + + NORM_(r1, 0,1) + NORM_(r1, 1,2) + NORM_(r1, 2,3) + NORM_(r1, 3,4) + + storeu64(&vr0[0], r00); + storeu64(&vr0[1], r01); + storeu64(&vr0[2], r02); + storeu64(&vr0[3], r03); + storeu64(&vr0[4], r04); + + storeu64(&vr1[0], r10); + storeu64(&vr1[1], r11); + storeu64(&vr1[2], r12); + storeu64(&vr1[3], r13); + storeu64(&vr1[4], r14); +} +////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////// + +__INLINE void fe52mb8_set(U64 out[], int64u value) +{ + storeu64(&out[0], set64((long long)value)); + storeu64(&out[1], get_zero64()); + storeu64(&out[2], get_zero64()); + storeu64(&out[3], get_zero64()); + storeu64(&out[4], get_zero64()); +} +__INLINE void fe52mb8_copy(U64 out[], const U64 in[]) +{ + storeu64(&out[0], loadu64(&in[0])); + storeu64(&out[1], loadu64(&in[1])); + storeu64(&out[2], loadu64(&in[2])); + storeu64(&out[3], loadu64(&in[3])); + storeu64(&out[4], loadu64(&in[4])); +} + +// Clang warning -Wunused-function +#if(0) +__INLINE void fe52mb8_mul_mod25519(U64 vr[], const U64 va[], const U64 vb[]) +{ + U64 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9; + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = get_zero64(); + + // Full multiplication + ROUND_MUL(4, 4, r8, r9) + + ROUND_MUL(3, 0, r3, r4) + ROUND_MUL(1, 2, r3, r4) + ROUND_MUL(0, 3, r3, r4) + ROUND_MUL(2, 1, r3, r4) + ROUND_MUL(2, 2, r4, r5) + ROUND_MUL(0, 4, r4, r5) + ROUND_MUL(1, 3, r4, r5) + ROUND_MUL(3, 1, r4, r5) + ROUND_MUL(4, 0, r4, r5) + ROUND_MUL(1, 4, r5, r6) + ROUND_MUL(2, 3, r5, r6) + + ROUND_MUL(3, 2, r5, r6) + ROUND_MUL(4, 1, r5, r6) + ROUND_MUL(2, 4, r6, r7) + ROUND_MUL(3, 3, r6, r7) + ROUND_MUL(4, 2, r6, r7) + + ROUND_MUL(0, 0, r0, r1) + ROUND_MUL(0, 1, r1, r2) + ROUND_MUL(0, 2, r2, r3) + ROUND_MUL(1, 0, r1, r2) + ROUND_MUL(1, 1, r2, r3) + ROUND_MUL(2, 0, r2, r3) + ROUND_MUL(3, 4, r7, r8) + ROUND_MUL(4, 3, r7, r8) + + //REDUCE_ROUND(4, 5, 9); + r4 = fma52lo(r4, r9, MOD_2_260); //r9 always contributes 0 to r5 (if input normalized?) + REDUCE_ROUND(3, 4, 8); + REDUCE_ROUND(2, 3, 7); + REDUCE_ROUND(1, 2, 6); + REDUCE_ROUND(0, 1, 5); + + // Reduce r4 upper bits + r0 = fma52lo(r0, srli64(r4, 47), MOD_2_255); + + // Trim top r4 bits that were already reduced above + r4 = and64(r4, MASK_47); + + // Normalize result + NORM(0,1) + NORM(1,2) + NORM(2,3) + NORM(3,4) + + storeu64(&vr[0], r0); + storeu64(&vr[1], r1); + storeu64(&vr[2], r2); + storeu64(&vr[3], r3); + storeu64(&vr[4], r4); +} + +__INLINE void fe52mb8_sqr_mod25519(U64 out[], const U64 a[]) +{ + fe52mb8_mul_mod25519(out, a, a); +} +#endif + +__INLINE void fe52mb8_mul121666_mod25519(U64 vr[], const U64 va[]) +{ + U64 multiplier = set64(121666); + + U64 r0, r1, r2, r3, r4, r5; + r0 = r1 = r2 = r3 = r4 = r5 = get_zero64(); + + // multiply + r0 = fma52lo(r0, va[0], multiplier); + r1 = fma52lo(r1, va[1], multiplier); + r2 = fma52lo(r2, va[2], multiplier); + r3 = fma52lo(r3, va[3], multiplier); + r4 = fma52lo(r4, va[4], multiplier); + + r5 = fma52hi(r5, va[4], multiplier); + r1 = fma52hi(r1, va[0], multiplier); + r2 = fma52hi(r2, va[1], multiplier); + r3 = fma52hi(r3, va[2], multiplier); + r4 = fma52hi(r4, va[3], multiplier); + + // reduce + REDUCE_ROUND(0, 1, 5); + r0 = fma52lo(r0, srli64(r4, 47), MOD_2_255); + + // trim top r4 bits that were already reduced above + r4 = and64(r4, MASK_47); + + // normalize + NORM(0,1) + NORM(1,2) + NORM(2,3) + NORM(3,4) + + storeu64(&vr[0], r0); + storeu64(&vr[1], r1); + storeu64(&vr[2], r2); + storeu64(&vr[3], r3); + storeu64(&vr[4], r4); +} + +#define PRIME25519_LO 0x000FFFFFFFFFFFED +#define PRIME25519_MID 0x000FFFFFFFFFFFFF +#define PRIME25519_HI 0x00007FFFFFFFFFFF + +// __ALIGN64 static const int64u prime25519[5] = { +// PRIME25519_LO, PRIME25519_MID, PRIME25519_MID, PRIME25519_MID, PRIME25519_HI}; + +__ALIGN64 static const int64u VPRIME25519_LO[8] = + { PRIME25519_LO, PRIME25519_LO, PRIME25519_LO, PRIME25519_LO, + PRIME25519_LO, PRIME25519_LO, PRIME25519_LO, PRIME25519_LO }; + +__ALIGN64 static const int64u VPRIME25519_MID[8] = + { PRIME25519_MID, PRIME25519_MID, PRIME25519_MID, PRIME25519_MID, + PRIME25519_MID, PRIME25519_MID, PRIME25519_MID, PRIME25519_MID }; + +__ALIGN64 static const int64u VPRIME25519_HI[8] = + { PRIME25519_HI, PRIME25519_HI, PRIME25519_HI, PRIME25519_HI, + PRIME25519_HI, PRIME25519_HI, PRIME25519_HI, PRIME25519_HI }; + + +__INLINE U64 cmov_U64(U64 a, U64 b, __mb_mask kmask) +{ return mask_mov64 (a, kmask, b); } + +#define NORM_ASHIFTR(R, I, J) \ + R##J = add64(R##J, srai64(R##I, DIGIT_SIZE)); \ + R##I = and64(R##I, loadu64(VMASK52)); + +#define NORM_LSHIFTR(R, I, J) \ + R##J = add64(R##J, srli64(R##I, DIGIT_SIZE)); \ + R##I = and64(R##I, loadu64(VMASK52)); + +__INLINE void fe52mb8_add_mod25519(U64 vr[], const U64 va[], const U64 vb[]) +{ + /* r = a+b */ + U64 r0 = add64(va[0], vb[0]); + U64 r1 = add64(va[1], vb[1]); + U64 r2 = add64(va[2], vb[2]); + U64 r3 = add64(va[3], vb[3]); + U64 r4 = add64(va[4], vb[4]); + + /* t = r-modulus (2^255-19) */ + U64 t0 = sub64(r0, loadu64(VPRIME25519_LO )); + U64 t1 = sub64(r1, loadu64(VPRIME25519_MID)); + U64 t2 = sub64(r2, loadu64(VPRIME25519_MID)); + U64 t3 = sub64(r3, loadu64(VPRIME25519_MID)); + U64 t4 = sub64(r4, loadu64(VPRIME25519_HI )); + + /* normalize r0, r1, r2, r3, r4 */ + NORM_LSHIFTR(r, 0,1) + NORM_LSHIFTR(r, 1,2) + NORM_LSHIFTR(r, 2,3) + NORM_LSHIFTR(r, 3,4) + + /* normalize t0, t1, t2, t3, t4 */ + NORM_ASHIFTR(t, 0,1) + NORM_ASHIFTR(t, 1,2) + NORM_ASHIFTR(t, 2,3) + NORM_ASHIFTR(t, 3,4) + + /* condition mask t4<0? (-1) : 0 */ + __mb_mask cmask = cmp64_mask(t4, get_zero64(), _MM_CMPINT_LT); + + storeu64(&vr[0], cmov_U64(t0, r0, cmask)); + storeu64(&vr[1], cmov_U64(t1, r1, cmask)); + storeu64(&vr[2], cmov_U64(t2, r2, cmask)); + storeu64(&vr[3], cmov_U64(t3, r3, cmask)); + storeu64(&vr[4], cmov_U64(t4, r4, cmask)); +} + +__INLINE void fe52mb8_sub_mod25519(U64 vr[], const U64 va[], const U64 vb[]) +{ + /* r = a-b */ + U64 r0 = sub64(va[0], vb[0]); + U64 r1 = sub64(va[1], vb[1]); + U64 r2 = sub64(va[2], vb[2]); + U64 r3 = sub64(va[3], vb[3]); + U64 r4 = sub64(va[4], vb[4]); + + /* t = r+modulus (2^255-19) */ + U64 t0 = add64(r0, loadu64(VPRIME25519_LO )); + U64 t1 = add64(r1, loadu64(VPRIME25519_MID)); + U64 t2 = add64(r2, loadu64(VPRIME25519_MID)); + U64 t3 = add64(r3, loadu64(VPRIME25519_MID)); + U64 t4 = add64(r4, loadu64(VPRIME25519_HI )); + + /* normalize r0, r1, r2, r3, r4 */ + NORM_ASHIFTR(r, 0,1) + NORM_ASHIFTR(r, 1,2) + NORM_ASHIFTR(r, 2,3) + NORM_ASHIFTR(r, 3,4) + + /* normalize t0, t1, t2, t3, t4 */ + NORM_ASHIFTR(t, 0,1) + NORM_ASHIFTR(t, 1,2) + NORM_ASHIFTR(t, 2,3) + NORM_ASHIFTR(t, 3,4) + + /* condition mask r4<0? (-1) : 0 */ + __mb_mask cmask = cmp64_mask(r4, get_zero64(), _MM_CMPINT_LT); + + storeu64(&vr[0], cmov_U64(r0, t0, cmask)); + storeu64(&vr[1], cmov_U64(r1, t1, cmask)); + storeu64(&vr[2], cmov_U64(r2, t2, cmask)); + storeu64(&vr[3], cmov_U64(r3, t3, cmask)); + storeu64(&vr[4], cmov_U64(r4, t4, cmask)); +} + +__INLINE void fe52mb8_red_p25519(U64 vr[], const U64 va[]) +{ + /* r = a-p */ + U64 r0 = sub64(va[0], loadu64(VPRIME25519_LO)); + U64 r1 = sub64(va[1], loadu64(VPRIME25519_MID)); + U64 r2 = sub64(va[2], loadu64(VPRIME25519_MID)); + U64 r3 = sub64(va[3], loadu64(VPRIME25519_MID)); + U64 r4 = sub64(va[4], loadu64(VPRIME25519_HI)); + + /* normalize r0, r1, r2, r3, r4 */ + NORM_ASHIFTR(r, 0, 1) + NORM_ASHIFTR(r, 1, 2) + NORM_ASHIFTR(r, 2, 3) + NORM_ASHIFTR(r, 3, 4) + + /* condition mask r4<0? (-1) : 0 */ + __mb_mask cmask = cmp64_mask(r4, get_zero64(), _MM_CMPINT_LT); + + storeu64(&vr[0], cmov_U64(r0, va[0], cmask)); + storeu64(&vr[1], cmov_U64(r1, va[1], cmask)); + storeu64(&vr[2], cmov_U64(r2, va[2], cmask)); + storeu64(&vr[3], cmov_U64(r3, va[3], cmask)); + storeu64(&vr[4], cmov_U64(r4, va[4], cmask)); +} + +// #define USE_DUAL_MUL_SQR +// #define fe52_mul fe52mb8_mul_mod25519 +// #define fe52_sqr fe52mb8_sqr_mod25519 +#define fe52_mul ed25519_mul +#define fe52_sqr ed25519_sqr +#define fe52_add fe52mb8_add_mod25519 +#define fe52_sub fe52mb8_sub_mod25519 +#define fe52_mul121666 fe52mb8_mul121666_mod25519 +#define fe52_sqr_power MB_FUNC_NAME(ed25519_sqr_latency_) + + +/* + Compute 1/z = z^(2^255 - 19 - 2) + considering the exponent as + 2^255 - 21 = (2^5) * (2^250 - 1) + 11. +*/ +__INLINE void fe52mb8_inv_mod25519(U64 out[], const U64 z[]) +{ + __ALIGN64 U64 t0[5]; + __ALIGN64 U64 t1[5]; + __ALIGN64 U64 t2[5]; + __ALIGN64 U64 t3[5]; + + /* t0 = z ** 2 */ + fe52_sqr(t0, z); + + /* t1 = t0 ** (2 ** 2) = z ** 8 */ + fe52_sqr(t1, t0); + fe52_sqr(t1, t1); + + /* t1 = z * t1 = z ** 9 */ + fe52_mul(t1, z, t1); + /* t0 = t0 * t1 = z ** 11 -- stash t0 away for the end. */ + fe52_mul(t0, t0, t1); + + /* t2 = t0 ** 2 = z ** 22 */ + fe52_sqr(t2, t0); + + /* t1 = t1 * t2 = z ** (2 ** 5 - 1) */ + fe52_mul(t1, t1, t2); + + /* t2 = t1 ** (2 ** 5) = z ** ((2 ** 5) * (2 ** 5 - 1)) */ + fe52_sqr_power(t2, t1, 5); + + /* t1 = t1 * t2 = z ** ((2 ** 5 + 1) * (2 ** 5 - 1)) = z ** (2 ** 10 - 1) */ + fe52_mul(t1, t2, t1); + + /* Continuing similarly... */ + + /* t2 = z ** (2 ** 20 - 1) */ + fe52_sqr_power(t2, t1, 10); + + fe52_mul(t2, t2, t1); + + /* t2 = z ** (2 ** 40 - 1) */ + fe52_sqr_power(t3, t2, 20); + + fe52_mul(t2, t3, t2); + + /* t2 = z ** (2 ** 10) * (2 ** 40 - 1) */ + fe52_sqr_power(t2, t2, 10); + + /* t1 = z ** (2 ** 50 - 1) */ + fe52_mul(t1, t2, t1); + + /* t2 = z ** (2 ** 100 - 1) */ + fe52_sqr_power(t2, t1, 50); + + fe52_mul(t2, t2, t1); + + /* t2 = z ** (2 ** 200 - 1) */ + fe52_sqr_power(t3, t2, 100); + + fe52_mul(t2, t3, t2); + + /* t2 = z ** ((2 ** 50) * (2 ** 200 - 1) */ + fe52_sqr_power(t2, t2, 50); + + /* t1 = z ** (2 ** 250 - 1) */ + fe52_mul(t1, t2, t1); + + /* t1 = z ** ((2 ** 5) * (2 ** 250 - 1)) */ + fe52_sqr_power(t1, t1, 5); + + /* Recall t0 = z ** 11; out = z ** (2 ** 255 - 21) */ + fe52_mul(out, t1, t0); +} + +#define cswap_U64(a, b, kmask) { \ + U64 ta = mask_mov64((a), (kmask), (b)); \ + (b) = mask_mov64((b), (kmask), (a)); \ + (a) = ta; \ +} + +static void fe52mb8_cswap(U64 a[], U64 b[], __mb_mask k) +{ + cswap_U64(a[0], b[0], k) + cswap_U64(a[1], b[1], k) + cswap_U64(a[2], b[2], k) + cswap_U64(a[3], b[3], k) + cswap_U64(a[4], b[4], k) +} + +#if 0 +static void x25519_scalar_mul(U64 out[], U64 scalar[], U64 point[]) +{ + __ALIGN64 U64 x1[5], x2[5], x3[5]; + __ALIGN64 U64 z2[5], z3[5]; + __ALIGN64 U64 tmp0[5], tmp1[5]; + + fe52mb8_copy(x1, point); + fe52mb8_set(x2, 1); + fe52mb8_set(z2, 0); + fe52mb8_copy(x3, x1); + fe52mb8_set(z3, 1); + + /* read high and remove (zero) bit 63 */ + U64 e = loadu64(&scalar[3]); + e = slli64(e, 1); + + __mb_mask swap = get_mask(0); + int bitpos; + for (bitpos=254; bitpos>= 0; bitpos--) { + if(63==(bitpos%64)) + e = loadu64(&scalar[bitpos/64]); + + __mb_mask b = cmp64_mask(e, get_zero64(), _MM_CMPINT_LT); + + swap = mask_xor (swap, b); + fe52mb8_cswap(x2, x3, swap); + fe52mb8_cswap(z2, z3, swap); + swap = b; + fe52_sub(tmp0, x3, z3); + fe52_sub(tmp1, x2, z2); + fe52_add(x2, x2, z2); + fe52_add(z2, x3, z3); + + #ifdef USE_DUAL_MUL_SQR + ed25519_mul_dual(z3, z2, x2, tmp0, z2, tmp1); + #else + fe52_mul(z3, x2, tmp0); + fe52_mul(z2, z2, tmp1); + #endif + + #ifdef USE_DUAL_MUL_SQR + ed25519_sqr_dual(tmp0, tmp1, tmp1, x2); + #else + fe52_sqr(tmp0, tmp1); + fe52_sqr(tmp1, x2); + #endif + + fe52_add(x3, z3, z2); + fe52_sub(z2, z3, z2); + fe52_mul(x2, tmp1, tmp0); + fe52_sub(tmp1, tmp1, tmp0); + fe52_sqr(z2, z2); + fe52_mul121666(z3, tmp1); + fe52_sqr(x3, x3); + fe52_add(tmp0, tmp0, z3); + + #ifdef USE_DUAL_MUL_SQR + ed25519_mul_dual(z3, z2, x1, z2, tmp1, tmp0); + #else + fe52_mul(z3, x1, z2); + fe52_mul(z2, tmp1, tmp0); + #endif + + e = slli64(e, 1); + } + + fe52mb8_inv_mod25519(z2, z2); + fe52_mul(out, x2, z2); +} +#endif + +////////////////////////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////////////////////////// + +__INLINE void ed25519_mul_dual_wonorm(U64 out0[], U64 out1[], + const U64 a0[], const U64 b0[], + const U64 a1[], const U64 b1[]) { + + U64 r00, r01, r02, r03, r04, r05, r06, r07, r08, r09; + U64 r10, r11, r12, r13, r14, r15, r16, r17, r18, r19; + + U64 *vr0 = (U64*) out0; + U64 *vr1 = (U64*) out1; + + r00 = r01 = r02 = r03 = r04 = r05 = r06 = r07 = r08 = r09 = get_zero64(); + r10 = r11 = r12 = r13 = r14 = r15 = r16 = r17 = r18 = r19 = get_zero64(); + + // Full multiplication + U64 *va = (U64*) a0; + U64 *vb = (U64*) b0; + ROUND_MUL(4, 4, r08, r09) + ROUND_MUL(3, 0, r03, r04) + ROUND_MUL(1, 2, r03, r04) + ROUND_MUL(0, 3, r03, r04) + ROUND_MUL(2, 1, r03, r04) + ROUND_MUL(2, 2, r04, r05) + ROUND_MUL(0, 4, r04, r05) + ROUND_MUL(1, 3, r04, r05) + ROUND_MUL(3, 1, r04, r05) + ROUND_MUL(4, 0, r04, r05) + ROUND_MUL(1, 4, r05, r06) + ROUND_MUL(2, 3, r05, r06) + ROUND_MUL(3, 2, r05, r06) + ROUND_MUL(4, 1, r05, r06) + ROUND_MUL(2, 4, r06, r07) + ROUND_MUL(3, 3, r06, r07) + ROUND_MUL(4, 2, r06, r07) + ROUND_MUL(0, 0, r00, r01) + ROUND_MUL(0, 1, r01, r02) + ROUND_MUL(0, 2, r02, r03) + ROUND_MUL(1, 0, r01, r02) + ROUND_MUL(1, 1, r02, r03) + ROUND_MUL(2, 0, r02, r03) + ROUND_MUL(3, 4, r07, r08) + ROUND_MUL(4, 3, r07, r08) + + va = (U64*) a1; + vb = (U64*) b1; + ROUND_MUL(4, 4, r18, r19) + ROUND_MUL(3, 0, r13, r14) + ROUND_MUL(1, 2, r13, r14) + ROUND_MUL(0, 3, r13, r14) + ROUND_MUL(2, 1, r13, r14) + ROUND_MUL(2, 2, r14, r15) + ROUND_MUL(0, 4, r14, r15) + ROUND_MUL(1, 3, r14, r15) + ROUND_MUL(3, 1, r14, r15) + ROUND_MUL(4, 0, r14, r15) + ROUND_MUL(1, 4, r15, r16) + ROUND_MUL(2, 3, r15, r16) + ROUND_MUL(3, 2, r15, r16) + ROUND_MUL(4, 1, r15, r16) + ROUND_MUL(2, 4, r16, r17) + ROUND_MUL(3, 3, r16, r17) + ROUND_MUL(4, 2, r16, r17) + ROUND_MUL(0, 0, r10, r11) + ROUND_MUL(0, 1, r11, r12) + ROUND_MUL(0, 2, r12, r13) + ROUND_MUL(1, 0, r11, r12) + ROUND_MUL(1, 1, r12, r13) + ROUND_MUL(2, 0, r12, r13) + ROUND_MUL(3, 4, r17, r18) + ROUND_MUL(4, 3, r17, r18) + + REDUCE_R4_N_R9(r0) + REDUCE_R4_N_R9(r1) + + REDUCE_ROUND_(r0, 0, 1, 5); + REDUCE_ROUND_(r0, 1, 2, 6); + REDUCE_ROUND_(r0, 2, 3, 7); + REDUCE_ROUND_(r0, 3, 4, 8); + + REDUCE_ROUND_(r1, 0, 1, 5); + REDUCE_ROUND_(r1, 1, 2, 6); + REDUCE_ROUND_(r1, 2, 3, 7); + REDUCE_ROUND_(r1, 3, 4, 8); + + storeu64(&vr0[0], r00); + storeu64(&vr0[1], r01); + storeu64(&vr0[2], r02); + storeu64(&vr0[3], r03); + storeu64(&vr0[4], r04); + + storeu64(&vr1[0], r10); + storeu64(&vr1[1], r11); + storeu64(&vr1[2], r12); + storeu64(&vr1[3], r13); + storeu64(&vr1[4], r14); +} + +__INLINE void fe52mb8_mul_mod25519_wonorm(U64 vr[], const U64 va[], const U64 vb[]) +{ + U64 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9; + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = get_zero64(); + + // Full multiplication + ROUND_MUL(4, 4, r8, r9) + + ROUND_MUL(3, 0, r3, r4) + ROUND_MUL(1, 2, r3, r4) + ROUND_MUL(0, 3, r3, r4) + ROUND_MUL(2, 1, r3, r4) + ROUND_MUL(2, 2, r4, r5) + ROUND_MUL(0, 4, r4, r5) + ROUND_MUL(1, 3, r4, r5) + ROUND_MUL(3, 1, r4, r5) + ROUND_MUL(4, 0, r4, r5) + ROUND_MUL(1, 4, r5, r6) + ROUND_MUL(2, 3, r5, r6) + + ROUND_MUL(3, 2, r5, r6) + ROUND_MUL(4, 1, r5, r6) + ROUND_MUL(2, 4, r6, r7) + ROUND_MUL(3, 3, r6, r7) + ROUND_MUL(4, 2, r6, r7) + + ROUND_MUL(0, 0, r0, r1) + ROUND_MUL(0, 1, r1, r2) + ROUND_MUL(0, 2, r2, r3) + ROUND_MUL(1, 0, r1, r2) + ROUND_MUL(1, 1, r2, r3) + ROUND_MUL(2, 0, r2, r3) + ROUND_MUL(3, 4, r7, r8) + ROUND_MUL(4, 3, r7, r8) + + //REDUCE_ROUND(4, 5, 9); + r4 = fma52lo(r4, r9, MOD_2_260); //r9 always contributes 0 to r5 (if input normalized?) + REDUCE_ROUND(3, 4, 8); + REDUCE_ROUND(2, 3, 7); + REDUCE_ROUND(1, 2, 6); + REDUCE_ROUND(0, 1, 5); + + // Reduce r4 upper bits + r0 = fma52lo(r0, srli64(r4, 47), MOD_2_255); + + // Trim top r4 bits that were already reduced above + r4 = and64(r4, MASK_47); + + storeu64(&vr[0], r0); + storeu64(&vr[1], r1); + storeu64(&vr[2], r2); + storeu64(&vr[3], r3); + storeu64(&vr[4], r4); +} + +__INLINE void fe52mb8_mul121666_mod25519_wonorm(U64 vr[], const U64 va[]) +{ + U64 multiplier = set64(121666); + + U64 r0, r1, r2, r3, r4, r5; + r0 = r1 = r2 = r3 = r4 = r5 = get_zero64(); + + // multiply + r0 = fma52lo(r0, va[0], multiplier); + r1 = fma52lo(r1, va[1], multiplier); + r2 = fma52lo(r2, va[2], multiplier); + r3 = fma52lo(r3, va[3], multiplier); + r4 = fma52lo(r4, va[4], multiplier); + + r5 = fma52hi(r5, va[4], multiplier); + r1 = fma52hi(r1, va[0], multiplier); + r2 = fma52hi(r2, va[1], multiplier); + r3 = fma52hi(r3, va[2], multiplier); + r4 = fma52hi(r4, va[3], multiplier); + + // reduce + REDUCE_ROUND(0, 1, 5); + r0 = fma52lo(r0, srli64(r4, 47), MOD_2_255); + + // trim top r4 bits that were already reduced above + r4 = and64(r4, MASK_47); + + storeu64(&vr[0], r0); + storeu64(&vr[1], r1); + storeu64(&vr[2], r2); + storeu64(&vr[3], r3); + storeu64(&vr[4], r4); +} + +__INLINE void x25519_scalar_mul_dual(U64 out[], U64 scalar[], U64 point[]) +{ + __ALIGN64 U64 x1[5], x2[5], x3[5]; + __ALIGN64 U64 z2[5], z3[5]; + __ALIGN64 U64 tmp0[5], tmp1[5]; + + fe52mb8_copy(x1, point); + fe52mb8_set(x2, 1); + fe52mb8_set(z2, 0); + fe52mb8_copy(x3, x1); + fe52mb8_set(z3, 1); + + /* read high and remove (zero) bit 63 */ + U64 e = loadu64(&scalar[3]); + U64 vmask = slli64( + xor64(e, srli64(e, 1)), 1); + __mb_mask swap = cmp64_mask(vmask, get_zero64(), _MM_CMPINT_LT); + + int bitpos; + for (bitpos = 254; bitpos >= 0; bitpos--) { + if (63 == (bitpos % 64)) { + U64 t = e; + e = loadu64(&scalar[bitpos/64]); + vmask = xor64(slli64(t, 63), + xor64(e, srli64(e, 1))); + swap = cmp64_mask(vmask, get_zero64(), _MM_CMPINT_LT); + } + + fe52mb8_cswap(x2, x3, swap); + fe52mb8_cswap(z2, z3, swap); + +#if (defined(linux) && ((SIMD_LEN)==512)) + // Avoid reordering optimization by compiler + U64 Z = get_zero64(); + __asm__ ("vpsllq $1, %0, %0 \n" + "vpcmpq $1, %2, %0, %1\n" + : "+x" (vmask), "=k" (swap): "x" (Z) : ); +#else + vmask = slli64(vmask, 1); + swap = cmp64_mask(vmask, get_zero64(), _MM_CMPINT_LT); +#endif + + fe52_sub(tmp0, x3, z3); + fe52_sub(tmp1, x2, z2); + fe52_add(x2, x2, z2); + fe52_add(z2, x3, z3); + + ed25519_mul_dual_wonorm(z3, z2, x2,tmp0, z2,tmp1); + + ed25519_sqr_dual(tmp0, tmp1, tmp1, x2); + + fe52_add(x3, z3, z2); + fe52_sub(z2, z3, z2); + + fe52mb8_mul_mod25519_wonorm(x2, tmp1, tmp0); + fe52_sub(tmp1, tmp1, tmp0); + + ed25519_sqr_dual(z2, x3, z2, x3); + + fe52mb8_mul121666_mod25519_wonorm(z3, tmp1); + fe52_add(tmp0, tmp0, z3); + + ed25519_mul_dual_wonorm(z3, z2, x1,z2, tmp1,tmp0); + } + + // normalize z2 and x2 before inversion + { + U64 r0 = z2[0]; + U64 r1 = z2[1]; + U64 r2 = z2[2]; + U64 r3 = z2[3]; + U64 r4 = z2[4]; + NORM_(r, 0,1) + NORM_(r, 1,2) + NORM_(r, 2,3) + NORM_(r, 3,4) + storeu64(&z2[0], r0); + storeu64(&z2[1], r1); + storeu64(&z2[2], r2); + storeu64(&z2[3], r3); + storeu64(&z2[4], r4); + + r0 = x2[0]; + r1 = x2[1]; + r2 = x2[2]; + r3 = x2[3]; + r4 = x2[4]; + NORM_(r, 0,1) + NORM_(r, 1,2) + NORM_(r, 2,3) + NORM_(r, 3,4) + storeu64(&x2[0], r0); + storeu64(&x2[1], r1); + storeu64(&x2[2], r2); + storeu64(&x2[3], r3); + storeu64(&x2[4], r4); + } + + fe52mb8_inv_mod25519(z2, z2); + fe52_mul(x2, x2, z2); + fe52mb8_red_p25519(out, x2); +} + +DLL_PUBLIC +mbx_status MB_FUNC_NAME(mbx_x25519_)(int8u* const pa_shared_key[8], + const int8u* const pa_private_key[8], + const int8u* const pa_public_key[8]) +{ + mbx_status status = 0; + int buf_no; + + /* test input pointers */ + if(NULL==pa_shared_key || NULL==pa_private_key || NULL==pa_public_key) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + for(buf_no=0; buf_no<8; buf_no++) { + int64u* shared = (int64u*) pa_shared_key[buf_no]; + const int64u* own_private = (const int64u*) pa_private_key[buf_no]; + const int64u* party_public = (const int64u*) pa_public_key[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL==shared || NULL==own_private || NULL==party_public) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + continue; + } + } + + /* continue processing if there are correct parameters */ + if( MBX_IS_ANY_OK_STS(status) ) { + __ALIGN64 U64 private64_mb8[4]; + __ALIGN64 U64 pub52_mb8[5]; + __ALIGN64 U64 shared52_mb8[5]; + + /* get scalars and convert to MB8 */ + ifma_BNU_transpose_copy((int64u (*)[8])private64_mb8, (const int64u * const*)pa_private_key, 256); + /* decode keys into scalars according to RFC7748 */ + private64_mb8[0] = and64_const(private64_mb8[0], 0xfffffffffffffff8); + private64_mb8[3] = and64_const(private64_mb8[3], 0x7fffffffffffffff); + private64_mb8[3] = or64(private64_mb8[3], set64(0x4000000000000000)); + + /* get peer's public keys and convert to MB8 */ + ifma_BNU_to_mb8((int64u (*)[8])pub52_mb8, (const int64u * const*)pa_public_key, 256); + + /* RFC7748: (x25519) ... MUST mask the most significant bit in the final byte. + This is done to preserve compatibility with point formats .. (compact format??) + */ + pub52_mb8[4] = and64(pub52_mb8[4], loadu64(VPRIME25519_HI)); + + /* point multiplication */ + x25519_scalar_mul_dual(shared52_mb8, private64_mb8, pub52_mb8); + + /* test shared secret before return; all-zero output results when the input is a point of small order. */ + __ALIGN64 U64 shared52_sum = shared52_mb8[0]; + shared52_sum = or64(shared52_sum, shared52_mb8[1]); + shared52_sum = or64(shared52_sum, shared52_mb8[2]); + shared52_sum = or64(shared52_sum, shared52_mb8[3]); + shared52_sum = or64(shared52_sum, shared52_mb8[4]); + int8u stt_mask = cmpeq64_mask(shared52_sum, get_zero64()); + status |= MBX_SET_STS_BY_MASK(status, stt_mask, MBX_STATUS_LOW_ORDER_ERR); + + /* convert result back */ + ifma_mb8_to_BNU((int64u* const *)pa_shared_key, (const int64u (*)[8])shared52_mb8, 256); + + /* clear computed shared keys and it sum */ + MB_FUNC_NAME(zero_)((int64u (*)[8])shared52_mb8, sizeof(shared52_mb8)/sizeof(U64)); + MB_FUNC_NAME(zero_)((int64u (*)[8])&shared52_sum, 1); + + /* clear copy of the secret keys */ + MB_FUNC_NAME(zero_)((int64u (*)[8])private64_mb8, sizeof(private64_mb8)/sizeof(U64)); + } + return status; +} + +////////////////////////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////////////////////////// + +/* the fe64 muTBL52[] table in radix52 */ +__ALIGN64 static int64u muTBL52[255][NUMBER_OF_DIGITS(256,DIGIT_SIZE)] = { + {0x000ffffffffffff3, 0x000fffffffffffff, 0x000fffffffffffff, 0x000fffffffffffff, 0x00005fffffffffff}, + {0x000220f416aafe96, 0x0002b4f566a346b8, 0x0005a5950f82ebeb, 0x00088f4d5a9a5b07, 0x00005142b2cf4b24}, + {0x000ebc750069680c, 0x00020a0f99c416aa, 0x000b56d0f489cf78, 0x00011a42a58d9183, 0x00004b5aca80e360}, + {0x000132348c29745d, 0x00016e1642fd7329, 0x000f67bc34f4a2e6, 0x0009b4a1e45bb03f, 0x0000306912d0f42a}, + {0x00086507e6af7154, 0x00013dfeec82fff8, 0x000abab5ce04f50e, 0x000f222aa512fe82, 0x0000174e251a68d5}, + {0x0006700d82028898, 0x000370a2c02c5cf9, 0x0004e86eaa1743e3, 0x000482e379eec98b, 0x00000c59888a51e0}, + {0x000bf1d699b5d189, 0x000d58e9fdc84fbc, 0x00031f7614acaef0, 0x000f972c1c20d062, 0x00002938218da274}, + {0x000f49beff1d7f18, 0x00022387ac9c2f6a, 0x000015c56bcc541c, 0x00013a996fcc9ef4, 0x000069c1627c6909}, + {0x0006fd2f4733db0e, 0x000f29e087de97a8, 0x000ea2a229fdb8c4, 0x0007a79095e4b1a8, 0x00001ad7a7c829b3}, + {0x000d89cad17ea0c0, 0x000a6cced2051342, 0x000bb42f7467bedd, 0x000acbb19ca31bf2, 0x00003df7b4c84980}, + {0x0006444dc80ad883, 0x0000366e3ab85a8c, 0x000164f6d8b91e44, 0x000e668c215cda00, 0x00003d867c6ef247}, + {0x000d582bcc3e658c, 0x00048ee0e5528c7d, 0x000c9f4f71fd2c47, 0x0005ddfa0fd9b95c, 0x00007529d871b067}, + {0x000568b42d3cbd78, 0x0001b91f3da82b8f, 0x000a7c3b62123301, 0x00086032dce6ccd4, 0x000075e7fc8e9e49}, + {0x000f13f1fcd0b6ec, 0x0001f29ff7a452f4, 0x000981e29bf1a8ca, 0x000b56ac249c1a72, 0x00006ebe0dbb8c83}, + {0x0004fa8d170bb222, 0x000d5bf93935f711, 0x00059c979a65a2dc, 0x0009289bdc41f68b, 0x00002f0eef79a2ce}, + {0x000cbf0c083c37ce, 0x00009ec49632242e, 0x000cfeac0d2930bc, 0x000bb80f294b0c19, 0x00003780aa4bedfa}, + {0x00017d3e7cead929, 0x000eb2e5722c556c, 0x000dbfe15ae7cb4b, 0x00052f80ce931732, 0x000041b883c76210}, + {0x00075ca0c3d25350, 0x000086eb1e351dbf, 0x0004a9b2122936be, 0x00025aac936e03cb, 0x00001d45bf823222}, + {0x000ab1036a024cc5, 0x0001c304c9a72e81, 0x000832b1fce21220, 0x0009581c5d73fba6, 0x000020ffdb5a4d83}, + {0x0003d367be5d0fad, 0x000ca8b164475a28, 0x000caaf22e6c2b25, 0x000ff499d4935467, 0x00005166408eee85}, + {0x0007baa2fab4e361, 0x000c67ef35cef3c6, 0x0001159b1cb3e433, 0x000ab33525972924, 0x00006a621892d5b0}, + {0x00074a387555cdcb, 0x0000e1208923f20b, 0x0002281dd1532aa1, 0x00044bfeaa17b776, 0x000061ab3443f05c}, + {0x000a6c422324def8, 0x0001017e3cf7f257, 0x000630a257131c6c, 0x000858023758739f, 0x0000295a407a01a7}, + {0x000443246d5da8d9, 0x000450c52fa5df8c, 0x00031bf83d19d775, 0x00047002afcfc927, 0x00007d10c8e81b2b}, + {0x0000271f70baa20b, 0x000867ca63957c8e, 0x000b7ed4bb993748, 0x00029755412efb3c, 0x00003196d36173e6}, + {0x000bcad141c7dffc, 0x000d2b395c848de5, 0x00011af3cb47cc8c, 0x000cec2a34cd942e, 0x00000256dbf2d04e}, + {0x000ab7e94b0e667f, 0x00083c0850d10875, 0x000e72c79fcad4dd, 0x000b19b47f12e8f4, 0x00005f1a87bb8c85}, + {0x0009d0b6437f51b8, 0x00055188790657ae, 0x000cf77aee12c7ce, 0x00056272ade09fe5, 0x000023a05a2f7d2c}, + {0x0008e128f17c169a, 0x000dd8ad0852d590, 0x000b102f64f77498, 0x000984574b4c4cea, 0x0000183abadd1013}, + {0x0005ba8daa92aaac, 0x0009599386705b16, 0x0008fc40d1d5c5ef, 0x0004514be2f8f0cf, 0x00002701e635ee20}, + {0x000fa80020156514, 0x0008764a8c1ce629, 0x000b3f060ef22386, 0x000a3fa5b894fff0, 0x000060d9944cf708}, + {0x000a001a1c7a201f, 0x000633ee2ce63aee, 0x000c7a07e1ebf16a, 0x00008cb6f7709594, 0x000079b958150d02}, + {0x00055e5301d410e7, 0x000dff3fdc84d24b, 0x00004032d8e3a34e, 0x000aeecd88768e49, 0x0000131384427b3a}, + {0x0005e51286234f14, 0x00039adb4c529840, 0x0000634ffd14dc47, 0x000ff93b8a2b5b25, 0x00002fe2a94ad8a7}, + {0x000c57efe843fadd, 0x00040f0bb9918ec5, 0x000f3d63052843ce, 0x000777ea4b561d6c, 0x0000743629bde8fb}, + {0x000edd46bbaf738f, 0x00028b101a651343, 0x00082c797aed9818, 0x0008730a401760b8, 0x00001fc223e28dc8}, + {0x00004e91fc0fba0e, 0x0008f052c6fa4486, 0x0009e9239cb637f7, 0x000687c91ccac3d0, 0x000023f7eed4437a}, + {0x0003b1118d9bd800, 0x000b63189d4a7517, 0x0008bbc58629d641, 0x0001df5fdbf17798, 0x00002959894fcad8}, + {0x000c8ef3b4bbc899, 0x0005ab26992b9aeb, 0x0004f92cfb414899, 0x000dee824e20b013, 0x000040d158894a05}, + {0x00000b1185af76f6, 0x0007873187a7946b, 0x000b8fff5f26bac7, 0x00024d73dc0bf95a, 0x00002a608bd89455}, + {0x00049588bd446302, 0x0001c0388439c264, 0x0003bd11b27c4bc2, 0x00076b98e98a4f38, 0x000026218d7bc9d8}, + {0x00081542997c178a, 0x000a86fb6606fe30, 0x000a2793743c2d29, 0x000b1fa5c217736f, 0x00007dde05734afe}, + {0x00010e3906d42bab, 0x0003e1980649c3bf, 0x000595bf7ae4f780, 0x0005530e6053bf89, 0x0000394faf38da24}, + {0x000efb58896928f4, 0x000e9cc6a113c7a8, 0x0000af596ffbc778, 0x0006cf772670ce33, 0x000048f222a81d3d}, + {0x000fce410d72caa7, 0x000c7213b5595f01, 0x0001fa14835a20ec, 0x000a7417bc21165c, 0x000007f89ae31da8}, + {0x0002c2b4c6830ff9, 0x0000fc631629305d, 0x0006d3a904d43e33, 0x00033b6a5a5590a9, 0x0000705edb91a653}, + {0x000ee15e0bb9a5f7, 0x000ca9e0aaf5d048, 0x000dc4a40b3240cf, 0x0004a6d8f4b71cee, 0x0000621c0da3de54}, + {0x00072836a08c4091, 0x000b010c91445928, 0x000f276394ce8375, 0x00036358a72eb524, 0x00002667fcfa7ec8}, + {0x000c173345e8752a, 0x000feee7079a57f4, 0x000f86ff34061b47, 0x000c89c25dd9afa9, 0x00003780cef5425d}, + {0x0006035a513bb4e9, 0x00079ac575ada1a4, 0x000fa24b503e1ef3, 0x0009f22c78c5f1c5, 0x0000321a967634fd}, + {0x000707b8826e27fa, 0x000d64c506fd0946, 0x0005e914363dca84, 0x0008484c18921807, 0x00006d9284169b3b}, + {0x0007e840383f2ddf, 0x000a30c4f9b753a6, 0x000783ef4733eec9, 0x000fbc43ec7c86fa, 0x000026ec449fbac9}, + {0x000f38cba09b9e7d, 0x000c762a3478c5c0, 0x0006fc121c81168c, 0x000dcdd3e23b0d30, 0x00005a238aa0a5ef}, + {0x00026121c4ea43ff, 0x0007f7c8832b51ba, 0x000adcf99a36f8c7, 0x000ebf988fbea0b0, 0x00005ca9938ec25b}, + {0x00036a5e51fccda0, 0x00097c2cd893bd54, 0x0003224a081dbc47, 0x000f46619346a65d, 0x00000f5034e49b9a}, + {0x000c3967a1e0b96e, 0x000fa867a4d88f23, 0x0007341679e58b08, 0x0006946fb2fabc6a, 0x00002a75381eb602}, + {0x000a3be4c19420ac, 0x000c681f2b6dcc80, 0x0001e9338866b1f6, 0x000a4c47cf703676, 0x000025abbbd8a660}, + {0x000a12ba14fd5198, 0x000fc4a3cffa991e, 0x0000f5ad28684950, 0x000a441f82684213, 0x00003ea988f75301}, + {0x0008109a695f8c6f, 0x0004a0530c3f3c97, 0x00044599951746eb, 0x0005cc7444d6d77b, 0x000075952b8c054e}, + {0x00003f7915f4d6aa, 0x000202f2647d8a37, 0x00011d644b66c346, 0x000d71fd01469df8, 0x000077fea47d81a5}, + {0x0009529ef57ca381, 0x000b9ce2f881ac5e, 0x0008009bd66eeeb4, 0x0003fecb6e91a28e, 0x00004b80be3e9afc}, + {0x000773c526aed2c5, 0x000b453c9a49d7e3, 0x000affb24d1b4afc, 0x000400ea920bdd7b, 0x00007c54699f122d}, + {0x0006c8e14fa94bc8, 0x000ce2952ed5eef4, 0x000bd885d5e0b074, 0x000712cbea450e1d, 0x000061b68649320f}, + {0x00085f7309ccbdd1, 0x0000d7d4d1a2d8a4, 0x00022dbef4bd0632, 0x000f770252329733, 0x0000445dc4758c17}, + {0x000434177cc8933c, 0x0002175ea059fdb0, 0x000053db34ed6fe8, 0x000af991efebefdc, 0x00004adbe867c65d}, + {0x000d71a2a90609df, 0x000856dd040503ac, 0x000157c23ce5e991, 0x000fe4d1ec69b688, 0x0000697427f6885c}, + {0x000e7b9b65e1a851, 0x000d522c536ddd7b, 0x000fd2b645a03d28, 0x00041e128399d658, 0x000049e5b7e17c26}, + {0x000c3a98700457a4, 0x000a25ebb67786f8, 0x000382960f5078f0, 0x00084b1d13c3ccbc, 0x00002e003258a7df}, + {0x0001f39be6296a1c, 0x000652a5fbfb28ad, 0x000d26f3cbc1eeaa, 0x0002ccc33ee0673f, 0x000059256173a69d}, + {0x000a07aa4e18fc41, 0x000527c87a51e41e, 0x000831ca6fd9fc19, 0x000694fbdaacb805, 0x0000445b652dc916}, + {0x0002a3a7f2172315, 0x0002de11b9964ce9, 0x00004c314a1edc28, 0x000f586a1823aafe, 0x0000790a2d94437c}, + {0x000447fb93f6e009, 0x000672284527671c, 0x00004f51698922a5, 0x00019febf70903b2, 0x00002f7a89891ba3}, + {0x00008eb577e2140c, 0x000d4427bdcf402a, 0x0004323cd1ed9a4e, 0x000355b5253ec44e, 0x00003e88363c14e9}, + {0x0006c14277110b8c, 0x0001610a23390aa6, 0x00093fc2a21ae039, 0x000c7ab2030bd12c, 0x00003ee141579555}, + {0x0004de3a6d6e7d41, 0x0008607f17efe921, 0x0008e112173ccdd8, 0x00093d0674f1288f, 0x00005682250f329f}, + {0x00000b136d2e396e, 0x0006f1014debf6cf, 0x000fcc4e836e4cf8, 0x00016b65930b1b5b, 0x0000047069b48aba}, + {0x000ce4ab69b20793, 0x0001a97d0fb9e0d4, 0x000e00d01db24db9, 0x000ddb5cdfa50f54, 0x0000221b1085368b}, + {0x00059468b1e3d8d2, 0x00063bd122f93e7e, 0x0000663f0953c565, 0x0003d42eee8a903e, 0x000061efa662cbbe}, + {0x0008ddddde6eab2a, 0x000d51435f2312cf, 0x0009f049739bf80a, 0x0009b275deadacec, 0x000029275b5d41d2}, + {0x000e0f0895ebf14f, 0x0006b054905a7cfd, 0x0001c420fdb9aab9, 0x000bbc7cae80dd9a, 0x00000a63bf2f1673}, + {0x000f6e11958fbc8c, 0x000e804822fad092, 0x0000d52517672a81, 0x00092f8cac835156, 0x00006f3f7722c8f1}, + {0x000a90ccc2e894b7, 0x000a438ff9f0df8b, 0x000ae523592c7557, 0x0003d69894d1d855, 0x000068e122157b74}, + {0x000e5570cfb919f3, 0x000cd95798db9d87, 0x0000c0a2ce3f2cde, 0x000c5b2212115471, 0x00003c66a115246d}, + {0x000dc562294ecb72, 0x000c36a280b16cbe, 0x0004078b67ba7143, 0x0004b1e9610c2efd, 0x00006144735d946a}, + {0x000f111ed75b3350, 0x0008c2041d81b536, 0x000e10413c0211db, 0x0008876f93cb1000, 0x0000149dfd3c039e}, + {0x0009dde46b63155b, 0x000e93c837976d47, 0x000f13e038b66e15, 0x0000b35dafde43b1, 0x00005fafda1a2e4b}, + {0x0000bbdf17197581, 0x0000bbe3cd2c2360, 0x000dd5be86397205, 0x000860f5938906db, 0x000034fce5e43f9b}, + {0x0008a4cd42d14d02, 0x000c53441df6575a, 0x0002e131d3828dab, 0x000d25f33dcabedd, 0x00003ebad76fb814}, + {0x00006f566f70e10f, 0x000aa51690f5ad49, 0x0006cefcf25d12f7, 0x000299945adb16e7, 0x000001f768aead23}, + {0x000cc77b6248febd, 0x00028ec3aaffd2b6, 0x0004ef486a3cd306, 0x0006c23ce1c0b80d, 0x00004c3bff2ea6f6}, + {0x000ec4094aeaeb5f, 0x000286e372ca73f2, 0x000e2a701d61b19b, 0x000e3ef5eefa966d, 0x000023b20565de55}, + {0x0001ca5279d58557, 0x000ce27c2874fe30, 0x000dcf1d6707b2d4, 0x000ff56a532cd8a9, 0x00002a52fee23f2b}, + {0x0004efb37cd8663d, 0x00020ffbd7594862, 0x0002d37445bbc7ac, 0x000ec6657b85e9c8, 0x00007b3052cb86a6}, + {0x0002f0ad2525e91e, 0x00043d28edca0348, 0x000e1b003a2cb680, 0x0001b0aaf4f6d052, 0x0000185f8c252978}, + {0x0001de5bd80ce0d6, 0x000416853e9d6aa4, 0x00057f4c3a9407b2, 0x0007bce563ec36e3, 0x00004cc4b8dd0e29}, + {0x000c1a52ffb8730e, 0x0006e67058e37a2f, 0x000ddf4ee11811f1, 0x000f09910f9a366c, 0x000072f4a0c4a0b9}, + {0x0006c06f663f4ea7, 0x000f74e970fba8c1, 0x00069ec345693b3a, 0x00080892102e7f1d, 0x00000ba53cbc968a}, + {0x000d9dc7fea15537, 0x000bb51536493ca3, 0x00044006b14c6824, 0x000cc60b98863148, 0x000040d2a72ab454}, + {0x0006a1b712570975, 0x00048debda657593, 0x00064330ea91b9d6, 0x00051d03344094bb, 0x0000006ba10d12ee}, + {0x00028468f5de5d58, 0x0004c38cc05b0192, 0x00056019900eb12f, 0x0000e0ba1039f9dd, 0x00004502d4ce4fff}, + {0x000054106837c189, 0x0004c6dd3b93ceb2, 0x000416d74fd0f654, 0x0002ef040727064c, 0x00006e15c6114b50}, + {0x0002a398cfb1a76b, 0x0007419f2f6b14df, 0x00066e604311256c, 0x0005b444a4979620, 0x0000705b3aab4135}, + {0x000ef536d797b1d8, 0x000d622ddf0db365, 0x0000575a8800076b, 0x000ca4d3bbf33b0e, 0x00003777aa05c8e4}, + {0x000745c85578db5f, 0x00049dbae5ae2392, 0x000adc98676fda41, 0x0001da3b1f0b00b8, 0x000009963437d36f}, + {0x00024e90a5dc3853, 0x000641f135cbd7e8, 0x0007ce8fccccb5f6, 0x000249f6736d86c8, 0x0000625f3ce26604}, + {0x000ac8059502f63f, 0x0000a2e351469af8, 0x00064b63050c05e7, 0x0003ac335292e9c7, 0x00001a394360c7e2}, + {0x0006d53251183264, 0x000bd43c2b74fd5c, 0x000b973f9b62065a, 0x0006e5eb5fbf5d03, 0x000013a3da366120}, + {0x000d5837725d94e5, 0x00012205016c5c6b, 0x0000033c6818e309, 0x00079872088ce157, 0x00007fba1f495c83}, + {0x000c7423f2f9079d, 0x0007b34023fc55a8, 0x0002fab351173515, 0x000e33ce4f9b49ad, 0x00006691ff72c878}, + {0x000c2adedc5eff3e, 0x000f1d8956cf4122, 0x000e9e5bdaf8dd4b, 0x000c743eb86205d9, 0x0000049b92b9d975}, + {0x00079730b0f6c05a, 0x000acc6f3a553a53, 0x00020dcd6d72a0ff, 0x000164ab0032c34b, 0x0000470e9dbc88d5}, + {0x000cf10ca237c047, 0x000711f6c81a2b19, 0x000dd80b43b65466, 0x000be8eb3321bd16, 0x000048c14f600c5f}, + {0x00051c264aa6c803, 0x00004a4fa7da6664, 0x0003128395b66e39, 0x000bc10d45f19b0b, 0x000031602627c3c9}, + {0x0000dc4832e4e10d, 0x0006756c717f7312, 0x0007280294eb20c4, 0x000c50900f52e3f6, 0x0000566d4fc14730}, + {0x000a5d40fd837206, 0x000dc7159547a7e3, 0x00068d6095c1e926, 0x000cea7216730fba, 0x000022e8c3843f69}, + {0x000074e8930e4b2b, 0x0000e84d1581633d, 0x0006ba2365b6e435, 0x000f3f35534c26ad, 0x00007773c12f89f1}, + {0x000a404da57962aa, 0x000a81999ce568cb, 0x00021692fc5b9897, 0x000c291508e862f1, 0x00003a81907fa093}, + {0x000ed0ff4725a510, 0x00010673fc5030dd, 0x000f1f4e8910d8cc, 0x000a44c5b9d151c9, 0x000032a5c1d5cb09}, + {0x000aa442b90541fb, 0x0007cc1b485db1e0, 0x000a9df2e55f85eb, 0x0002236bee595ce8, 0x000025e496c72242}, + {0x000f3c46cd0fe5b9, 0x0007ed2a433885ed, 0x000761e35234e75a, 0x000545ce488de11d, 0x00000e878a01a085}, + {0x00093c77e021bb04, 0x00043c7df899aba4, 0x000ae80d672b4d18, 0x00017949ea37a487, 0x000067a9958011e4}, + {0x0008051a6697b065, 0x0007d8d6ba6d44b5, 0x0003ca46c147e33f, 0x000db0dbb4da8d48, 0x000068becaa181c2}, + {0x000980e90b989aa5, 0x0004a2c93c99b8d8, 0x00096e73a2f95eb1, 0x000b56951c6c7c47, 0x00006e228363b5ef}, + {0x000bc0b02dd624c8, 0x0007dec8170eec6b, 0x0004cfafa9777eb4, 0x000bf9b3cde15a00, 0x00001dc6bc087160}, + {0x0007e043eec34002, 0x000677a68dc7f2e0, 0x000bd15b9a18e9fc, 0x0008253d8da03188, 0x000048fbc3bb0056}, + {0x00047d4cfb654ce1, 0x00082a058e2ad575, 0x000f154478d3565b, 0x000bb18f63eaf0bb, 0x000047531ef114df}, + {0x000c630a4278c587, 0x00046ca8e83f3e1e, 0x000adc0c2b5507d5, 0x000844e85e135c63, 0x00000aa7efa85682}, + {0x00091ba8b3e1f615, 0x000701fbe3ffa726, 0x0009bb786832b4e9, 0x00039e897b6d92e3, 0x00002cfe53dea02e}, + {0x000392cd85cd52b0, 0x000c910e29831687, 0x0009832d0627ff66, 0x000f8a097134556a, 0x0000269bb0360a84}, + {0x000e55457643f85c, 0x0008c9b597d1b706, 0x0006efa4723734a4, 0x000d9e07aee91e8c, 0x00005cd6abc198a9}, + {0x0004de06cb3ce41a, 0x000893402e1380e0, 0x00086e3772d8c6eb, 0x000a8c8904659bb6, 0x00007215c371746b}, + {0x0002a97eeae4a2d9, 0x000516394f2c5fd1, 0x000208f2949514b7, 0x00026b9266fd5809, 0x00005c847085619a}, + {0x00085410fed694ea, 0x000934a2ed254529, 0x000d3be4673c905b, 0x000e9e110bb47692, 0x0000063b3d2d69e5}, + {0x000726eedda57deb, 0x000ae10f41891472, 0x000b307614efb6c4, 0x0005b7c2b1641917, 0x0000117c554fc4f4}, + {0x000cf3118f9d8812, 0x0002050017939c07, 0x00071b282701dbd8, 0x00025ead7e803f41, 0x00001015e87487d2}, + {0x000de3fed23acc4d, 0x000c294a7be2dc58, 0x000c9cf45750db91, 0x000524a0b94d43d1, 0x00006b1640fa6e37}, + {0x000f346c5fda0d09, 0x00059fa4d3151692, 0x000777a296200b1c, 0x000fbcfb8c46f760, 0x00004b38395f3ffd}, + {0x00025e00be54d671, 0x00082bec8aba618d, 0x000b78b98260d505, 0x000043287ad8f263, 0x000050fdf64e9cda}, + {0x000567aac578dcf0, 0x0000ef2a3133b90f, 0x0002d9de71ef1e9b, 0x00001c70eebba924, 0x000015473c9bf031}, + {0x0007e8ae56b78095, 0x000666e6f078e7c7, 0x000348ba1fb678e7, 0x0003f0b2da0b9615, 0x00007cf931c1ff73}, + {0x000357f50a0a366c, 0x000f42b87d73226b, 0x00091cb2c0e9708c, 0x000bb4cc13aeea5f, 0x000035d90c991143}, + {0x0001c404a9a0d9dc, 0x000451972d25147c, 0x0003b38c31659e58, 0x0001f243875a8c47, 0x00001fbd9ed37956}, + {0x000abc6fd41ec28d, 0x000e3cd2a2dca11f, 0x000c4045957ef8df, 0x0002f2772e73b5d8, 0x00006135fa4954b7}, + {0x000c32a2de24b69c, 0x0008c1f095d88ccf, 0x000ac3f9293f5569, 0x0007eebbe3350ed5, 0x00005e9bf806ca47}, + {0x000e8fb63c309f68, 0x0003565e1f9f4e9c, 0x000a6393f15376f6, 0x0003506d1afcfb35, 0x00006632a1ede562}, + {0x000d6c390c2ded4c, 0x00081df04cb1f0b7, 0x0009ecc3c756cb32, 0x000a72a66305a124, 0x00005d588b60a38c}, + {0x000cbf78e8e5f42d, 0x0004b3c8a3eeca6e, 0x000bd2160486eeb4, 0x0006731ec219c48f, 0x00001aaf1af517c3}, + {0x0006a2836769bde7, 0x000622b1e2adbc30, 0x000bff94a6208280, 0x000f26b8027f51ff, 0x000076cfa1ce1124}, + {0x000b00562422abb6, 0x000d58f8c29c318e, 0x000531561af377c4, 0x0008a274dbbc207f, 0x00000253b7f08212}, + {0x000f091cb62c17e0, 0x000abd64628a93d1, 0x00009d42534860e1, 0x000e57652d174363, 0x0000356f97e13efa}, + {0x0001e11aa150535b, 0x000bb1dd878ccd35, 0x000ed92c983e6b45, 0x00085b80c776128b, 0x00001d34ae930328}, + {0x0000488ca85ba4c3, 0x000c33c9ce6ce4ba, 0x0007bda770985348, 0x000124a66124c6f9, 0x00000f81a0290654}, + {0x00009ca6569b86fd, 0x000fd18af9a2d9ed, 0x0003d8c20a811009, 0x000f26bff08d03f9, 0x000052a148199fae}, + {0x0003f9dc2d8d1b73, 0x0001873961a703e0, 0x0001a35970420580, 0x000d549c0d987f04, 0x000007aa1f15a1c0}, + {0x00046ce08cd27224, 0x0004f934e4239dfd, 0x0009897b596d0a02, 0x00095a2808a7a639, 0x00000a4556e9e13d}, + {0x000a991fe9c13045, 0x00048fe7751b8d21, 0x000bf300359b0e85, 0x000f7215da643cb4, 0x000077db28d63940}, + {0x000eeb614adc9011, 0x0009ae8c411ebfc5, 0x000d1dcf74522941, 0x0004cb59ec3e7787, 0x0000340d053e216e}, + {0x0007af39b48df2b4, 0x0002871a10a94cac, 0x000ca575edc0faec, 0x0003a4c140a69245, 0x00000cf1c3713427}, + {0x000e306ac224b8a5, 0x0007ccb4930b0c8e, 0x000acbe74f57eaee, 0x000657da1e806bda, 0x00007d9a62742eeb}, + {0x0006b6ef546c4830, 0x0001fddb36e2e9eb, 0x000f0d7105885cca, 0x0000412e6b9f383e, 0x000058654fef9d2e}, + {0x0005c4ffbe0e8e26, 0x000df9b31816ea90, 0x00002e88e1942de5, 0x000408d497d723f8, 0x000030684dea602f}, + {0x0005a278a3e6cb34, 0x0006f5b151dc421e, 0x000d77ca15aefb6e, 0x0008981b30b8e049, 0x000028c3c9cf53b9}, + {0x000fb721556cdd2a, 0x000a897022274287, 0x000a5432580d317c, 0x000642f7468c7423, 0x00004a7f11464eb5}, + {0x0007a4774d193aa6, 0x0006ea92129a1a23, 0x00087c1a88d86598, 0x000f5eb24c515ecf, 0x0000604003575f39}, + {0x0009f189570a9b27, 0x000de465e4b7847b, 0x000bb85c202b98ce, 0x0001901026df551d, 0x000074fcd91047e2}, + {0x0002a90a23c1bfa3, 0x0004e478519f613e, 0x000af6cf440cb007, 0x0002dbe5ff1cbbe3, 0x000067fe5438be81}, + {0x000cf64fa40f05b0, 0x0002f32283787d13, 0x000f0d2aea054dfb, 0x0000d4e4173915b7, 0x0000482f144f1f61}, + {0x00010201b47f8234, 0x000929e70b990f62, 0x000049567c5d0ae1, 0x0006f01dcd7f455b, 0x00007e93d0f1f091}, + {0x0009cbf18a7db4fa, 0x000bf6f74c62fdd7, 0x000b8291bdbe8391, 0x0001705027145d14, 0x0000585a73ea2cbf}, + {0x000ca03e928a0db2, 0x000a5742857e7485, 0x0006d551a710fc01, 0x000db8a2f482edbd, 0x00000f0433b5048f}, + {0x000a2e8dd7dc6247, 0x000d38cd4819a60d, 0x0001f6669788b4c9, 0x0007d7513033ac00, 0x0000273b24fe3b36}, + {0x0008f66a31b3b9d4, 0x000a494df49d5c6e, 0x0008b23da7281514, 0x000e548d1726fdfc, 0x00004b3ae7d103de}, + {0x00056e19ce4b9d7e, 0x000f186e3c61cc62, 0x000b8ec145ff5c5c, 0x0006574acc63ca34, 0x000074621888fee6}, + {0x000f409645290a1e, 0x000e3263a962e956, 0x000ec2647bef0bf8, 0x0007502ed6a50eb5, 0x00000694283a9dca}, + {0x000b963643a2dcd1, 0x000ea09fc5353769, 0x0003397eab42b7c8, 0x000d63a4f002aee1, 0x000063005e2c19b7}, + {0x000736da63023bea, 0x0006db12a99b7ca6, 0x000537c5e1966c7f, 0x00089eeace09390c, 0x00000b696063a1aa}, + {0x00003e97288c56e5, 0x0009f938c8be8ebb, 0x000b717f71432a9f, 0x0009d97a6a5a93d5, 0x00001a5fb4c3e18f}, + {0x0004e7ad1c60cdce, 0x00043fc02c4a01c9, 0x0007c46a20ee202a, 0x0007b588dafe4d86, 0x00000a10263c8ac2}, + {0x000ea9dfe4432a4a, 0x0007bbe9277c5d0d, 0x000212c71a856af8, 0x0001e91ce8472acc, 0x00006f151b6d9bbb}, + {0x00076c527ceed56a, 0x000b7fbf8faec267, 0x000d4609cc7d211c, 0x0000c4237ae66a6f, 0x00001f81b702d277}, + {0x0000b057eac58392, 0x000fe29744e9d2fb, 0x0007beb4f8e1dd89, 0x000d41ec964f8eb1, 0x000029571073c9a2}, + {0x0008a18981c0e254, 0x0009b65b22830a94, 0x000fcfd3c62df636, 0x000a01fa33eb2d75, 0x0000078cd6ec4199}, + {0x00084a41ad900d2f, 0x00078e2c74c524a5, 0x000431c97832142b, 0x0009fc268c4e8338, 0x00007f69ea900868}, + {0x0002c81e46a38265, 0x0002d04a832fd52f, 0x0005359e94fd7807, 0x00029d28cd7d5fa2, 0x00004de71b7454cc}, + {0x000b60ad1eda6ac9, 0x000dfdbc09c3a42e, 0x00033cc1910aad37, 0x000803c81004b71e, 0x000044e6be345122}, + {0x000e8388ba1920db, 0x00032150db00803f, 0x000af60c29f5d57c, 0x0001aee49c8c4281, 0x000021edb518de70}, + {0x00063e418f06dc99, 0x00099c166d7b87fb, 0x000e520a83a4460d, 0x000835824dd5248c, 0x00005ec3ad712b92}, + {0x00022a5fbd17930f, 0x00077d82570e3150, 0x0005783712a4f64a, 0x0000abb12bc8d691, 0x0000498194c0fc62}, + {0x0002d9d255686c82, 0x000d9193e21f038a, 0x00024a5484785c6b, 0x0000989e4d5c81ab, 0x000056307860b2e2}, + {0x000d55f78b4d74c4, 0x0004643350131429, 0x0008c71fff22f183, 0x00083ef1e60c2459, 0x000059f2f0149799}, + {0x00047d56eb494a44, 0x00054d636a18e46a, 0x0004491c3b3e22a8, 0x000cde7b346e1527, 0x00002ceafd4e5390}, + {0x000a8538be0d6675, 0x000bb50818e23ba8, 0x0005d304c34b9074, 0x00092c4cbdab8908, 0x000061a24fe0e561}, + {0x000615e6db525bcb, 0x00035a567e4cacb7, 0x000afcdd69dd7d8c, 0x0009766e6b4153ac, 0x00002d668e097f3c}, + {0x000e7e265ce55ef0, 0x000527cd4b967a57, 0x00092fd1e55d9f4e, 0x000f7aefbc836064, 0x0000090d52beb7c3}, + {0x0009515a1e7b4d7c, 0x0002599da44c009b, 0x0002c555041f266a, 0x00015cca1c49548e, 0x00007ef04287126f}, + {0x0001659dbd30ef15, 0x000eec4e0277bfed, 0x0005df32918b4ab9, 0x000f788884d6236a, 0x00001fd96ea6bf5c}, + {0x000161981f190d9a, 0x000507e6052c142a, 0x00085a2cd561d849, 0x00085d89fe113bf2, 0x00007c22d676dbad}, + {0x000770ed2bfbd27d, 0x000ece996f5a582e, 0x00009001504c05b2, 0x000bf64cd40a9c2b, 0x00005895319213d9}, + {0x000c5d703fea2e08, 0x0001258e2188ce7c, 0x0008205bf0b50c49, 0x0002d62cce30baa4, 0x0000537c659ccfa3}, + {0x0006623a98cfc088, 0x0001fa4d6aca437b, 0x0006a8d1b0fe9bed, 0x000957504d29b8e5, 0x0000725f71c40b51}, + {0x0007f89cd0339ce6, 0x0004469ddc18b28c, 0x0006a1652c8367b1, 0x0006c17883ada83a, 0x0000585f1974034d}, + {0x000fb266f1b19188, 0x00063e7c3521789c, 0x0004c0526ae63b48, 0x0004635d88c9da6b, 0x00003e035c9df095}, + {0x000d5412fb45de9d, 0x00032e4cff40ddd9, 0x00051d671cdd6845, 0x000f6904b5c999b1, 0x00002d8c2cc811e7}, + {0x0004be1d90055d40, 0x000df464aaf407f5, 0x0000e917bea464c5, 0x0006b3033979624f, 0x00002c018dc52735}, + {0x00015024e330b3d4, 0x00096691652d3a54, 0x000f9b59f173ff3d, 0x0008e5a94ec42c4e, 0x00000747201618d0}, + {0x000ca48aca411c53, 0x0002fcfa661194d6, 0x0001e227ff66415f, 0x000f7eb9c4dd4005, 0x000059810bc09a02}, + {0x000eb171b3dc101d, 0x000b99ffef68e2a7, 0x0003b359ea441c5a, 0x000112f32025c9b9, 0x00005e8ce0a71e9d}, + {0x000ccb92429503fd, 0x000752f095d55bfc, 0x00072d091ed271ba, 0x00003ba345ead5e9, 0x000018c8df11a831}, + {0x000d949a9aed0f4c, 0x000cb6660e37e90c, 0x0006c52e0bc5d1f4, 0x0008e0db8cac52d5, 0x00006e42e400c580}, + {0x00046966eeaefd23, 0x0000be39ecdcaa3b, 0x000683a51d0c4f1f, 0x000351b189dc8c9d, 0x000051f27f054c09}, + {0x00087ccd2a320682, 0x0005bb3df1c964c4, 0x00055cb8e8587ea9, 0x000d73dc8ccf79e5, 0x0000547dc829a206}, + {0x0002a6cd80c39b06, 0x000732000d4c6b82, 0x0001463b4de96d54, 0x0006e1d28535b6f9, 0x0000228f4660e248}, + {0x00099538de8d3abf, 0x0000045ebca6e987, 0x000221e7388cd833, 0x000d2bb79952a008, 0x00004322e1a7535c}, + {0x0004c11819d1801c, 0x000d84f3f5ec7b11, 0x0009260f4c2016e4, 0x0007266dd0e2df40, 0x00005ec362c0ae5f}, + {0x00062b18b8b2b4ee, 0x00050274d1afbc04, 0x00036b02d27cc8d9, 0x000ccd3f25f71054, 0x000043bbf8dcbff9}, + {0x000d1767a039e9df, 0x000a8f69d3583b6a, 0x00042931f5b0714d, 0x00009615e55fa18b, 0x00004ed5558f33c6}, + {0x00037901c647a5dd, 0x0001f8081d3571fe, 0x00013fd7a6593ddf, 0x000af610249a4fd8, 0x000069acca274e9c}, + {0x000ba3ea330721c9, 0x000c20e7e1ea0047, 0x0001314a6083423f, 0x00095271df4c0af0, 0x000009a62dab8928}, + {0x000325a49cc6cb00, 0x000c654b56cb6a5b, 0x000dc994a0e94b5d, 0x0004aad3be28779a, 0x00004296e8f8ba3a}, + {0x000689761e451eab, 0x0008bff59594a328, 0x0007a7084a2e4d59, 0x00020a849b96853d, 0x00004980a3196014}, + {0x0005b9e12f552c42, 0x000db7100fe96956, 0x0003add0d78a5318, 0x0004eda05c90b4d4, 0x0000538b4cd66a5d}, + {0x00094fc3e89f039f, 0x000f26f618045f4e, 0x000d4b9550592c9a, 0x000141908a36eb5f, 0x000025fffaf6c2ed}, + {0x00034459cc79d354, 0x000b4b1d5476b344, 0x0001615d99eeecbf, 0x000b773ddeb34a06, 0x00005129cecceb64}, + {0x0003215894993520, 0x0007cf14c0b3bee4, 0x0006bedad5772f9c, 0x0006a97d2e2fce30, 0x0000715f42b546f0}, + {0x000ecdceda5b5f1a, 0x00015a49741a9434, 0x0003edad2e0da171, 0x0009041680bd77c7, 0x0000487c02354edd}, + {0x000feff3a70ed9c4, 0x000a3e857e302b8e, 0x0008a2a5a056a32a, 0x000c444df3a68bd4, 0x000007f650b73176}, + {0x000b9b1626e0ccb1, 0x000c18b09fb36e38, 0x0009f9496479e053, 0x000f5c456d90319c, 0x00001ca941e7ac9f}, + {0x0004df29162fa0bb, 0x0003282b3330549c, 0x000abb437d8488cf, 0x000ad8695dfda14c, 0x00003391f78264d5}, + {0x000ae06ae2b5095d, 0x000d73259a946729, 0x00013921edd58a58, 0x000b592e9834262d, 0x000027fedafaa54b}, + {0x000dc5b829ad48bb, 0x00042499ee260a99, 0x000d7513fd5f0257, 0x000d938802c8ecd5, 0x000078ceb3ef3f6d}, + {0x0002f44f8a135d94, 0x00044828cdda3c34, 0x000537cfe77b9edb, 0x000b4c89436d11a0, 0x00005064b164ec1a}, + {0x0000eccfd37eb2fc, 0x0003ed90d25fc702, 0x000fa1bb341f31ea, 0x00030441b930d7bd, 0x00005344467a4811}, + {0x00073170f25e6dfb, 0x0001a50114cc8700, 0x0008fc4f00e385dc, 0x00040d82348698ac, 0x00002a77a55284dd}, + {0x0006afe0c98c6ce4, 0x00096dddfd6e4fe0, 0x0003bf1ed3c235df, 0x000bdaf1428d01e3, 0x0000785768ec9300}, + {0x0002e57a91deb63b, 0x000bfe5ce8b80970, 0x000d1d58ac61bdb8, 0x00057bc645b426f3, 0x00004804a82227a5}, + {0x0007048ab44d2601, 0x0001a4b3a69358e5, 0x0009e1c29368d650, 0x00063e2c39c9ec3f, 0x00004172f257d4de}, + {0x0008b450330c6401, 0x00017418f2391d36, 0x0000b7d90d040d30, 0x000d51f2c34bb609, 0x000016f649228fdf}, + {0x0006818e2b928ef5, 0x00091cdc11e72bea, 0x00077a36cde28ccf, 0x000fd0f594aaa68e, 0x0000313034806c7f}, + {0x000d27ac2249bd65, 0x00064018e95128a9, 0x0002b37ec719a3b4, 0x0007b21c26ccff35, 0x0000056f68341d79}, + {0x0009d6757efd2327, 0x000b6553afe155e7, 0x000eaf5a60fabdbc, 0x000743bd3e7222c6, 0x00007046c76d4dae}, + {0x000be872b18d4a55, 0x00018574e1496660, 0x00002bdcbb199925, 0x0008e8ec103053a3, 0x00003ed8e9800b21}, + {0x000b9239fa75e03e, 0x000684633c0837b0, 0x00091a7793efe9fb, 0x000fe3498a35fbe3, 0x00006065510fe2d0}, + {0x000b668548abad0c, 0x00048da87e52755c, 0x000107c1ddb45845, 0x000de352c43ecea0, 0x0000526028809372}, + {0x0005c56af9213b1f, 0x0004d017e98db341, 0x0005cf709b5bee1a, 0x0009ab613f6b105b, 0x00005ff20e3482b2}, + {0x00029c75cc2e6c90, 0x000ca3a70e2060aa, 0x0004b5c515fc7d73, 0x000c207899fc38fc, 0x0000250386b124ff}, + {0x000a28d5ae3d2b56, 0x0009dd6de60ce54e, 0x000f06d6c1991314, 0x0008fc716694fc58, 0x000046b23975eb01}, + {0x000a6a0fb4b7b4e2, 0x0005a8f7253de470, 0x000fbd3adb5d9247, 0x0006968abeee5b52, 0x00007fa20801a080}, + {0x0003faf19f7714d2, 0x000c12f4660c376f, 0x000212744eb3e840, 0x0002dd20fb4cd8df, 0x00004b065a251d3a}, + {0x000bde383d77cd4a, 0x000df882c9cb15ce, 0x00009af7596adf39, 0x0006422a2dd242eb, 0x00003147c0e50e5f}, + {0x000ca5101d1350db, 0x00079c33fc962164, 0x0003e5da08f8d134, 0x000f8bae640ce4d1, 0x00004bdee0c45061}, + {0x00046dc1a4edb1c9, 0x000b6437fd98ad7c, 0x0002a1c00b5514d7, 0x000710e58942f6bb, 0x00002dffb2ab1d70}, + {0x000fcf2fc18b6d68, 0x000a8b7806167ccd, 0x000e2937e3a8ebcb, 0x0006e8c980697f95, 0x000002fbba1cd012}, + {0x0008c360adff4565, 0x000d24bdb1519cb9, 0x0005fb15d5447101, 0x000a27d98c4c8747, 0x000005f8c76d6e22}, + {0x0009e5dffeb9f494, 0x0009d18406b75ab6, 0x0004a42d1dae09c7, 0x000fa84199b45fb9, 0x000079710aebae2a}, + {0x000deda7f334d2df, 0x00051af2a57b4a6a, 0x0006dceaa87bde9c, 0x000d07ba98fc64f8, 0x00006bbe0335c20e}, +}; + +__ALIGN64 static const int64u U2_0[8] = + {0x000b1e0137d48290, 0x000b1e0137d48290, 0x000b1e0137d48290, 0x000b1e0137d48290, + 0x000b1e0137d48290, 0x000b1e0137d48290, 0x000b1e0137d48290, 0x000b1e0137d48290}; +__ALIGN64 static const int64u U2_1[8] = + {0x00051eb4d1207816, 0x00051eb4d1207816, 0x00051eb4d1207816, 0x00051eb4d1207816, + 0x00051eb4d1207816, 0x00051eb4d1207816, 0x00051eb4d1207816, 0x00051eb4d1207816}; +__ALIGN64 static const int64u U2_2[8] = + {0x000ca2b71d440f6a, 0x000ca2b71d440f6a, 0x000ca2b71d440f6a, 0x000ca2b71d440f6a, + 0x000ca2b71d440f6a, 0x000ca2b71d440f6a, 0x000ca2b71d440f6a, 0x000ca2b71d440f6a}; +__ALIGN64 static const int64u U2_3[8] = + {0x00054cb52385f46d, 0x00054cb52385f46d, 0x00054cb52385f46d, 0x00054cb52385f46d, + 0x00054cb52385f46d, 0x00054cb52385f46d, 0x00054cb52385f46d, 0x00054cb52385f46d}; +__ALIGN64 static const int64u U2_4[8] = + {0x0000215132111d83, 0x0000215132111d83, 0x0000215132111d83, 0x0000215132111d83, + 0x0000215132111d83, 0x0000215132111d83, 0x0000215132111d83, 0x0000215132111d83}; + +DLL_PUBLIC +mbx_status MB_FUNC_NAME(mbx_x25519_public_key_)(int8u* const pa_public_key[8], + const int8u* const pa_private_key[8]) +{ + mbx_status status = 0; + + /* test input pointers */ + if(NULL==pa_private_key || NULL==pa_public_key) { + status = MBX_SET_STS_ALL(MBX_STATUS_NULL_PARAM_ERR); + return status; + } + + /* check pointers and values */ + int buf_no; + for(buf_no=0; buf_no<8; buf_no++) { + const int64u* own_private = (const int64u*) pa_private_key[buf_no]; + const int64u* party_public = (const int64u*) pa_public_key[buf_no]; + + /* if any of pointer NULL set error status */ + if(NULL==own_private || NULL==party_public) { + status = MBX_SET_STS(status, buf_no, MBX_STATUS_NULL_PARAM_ERR); + continue; + } + } + + /* continue processing if there are correct parameters */ + if( MBX_IS_ANY_OK_STS(status) ) { + + /* convert private to to MB8 and decode */ + __ALIGN64 U64 scalar_mb8[4]; + ifma_BNU_transpose_copy((int64u (*)[8])scalar_mb8, (const int64u * const*)pa_private_key, 256); + /* decode keys into scalars according to RFC4448 */ + scalar_mb8[0] = and64_const(scalar_mb8[0], 0xfffffffffffffff8); + scalar_mb8[3] = and64_const(scalar_mb8[3], 0x7fffffffffffffff); + scalar_mb8[3] = or64(scalar_mb8[3], set64(0x4000000000000000)); + + /* set up: "special" point S = {1:?:1} => {u1:z1} */ + __ALIGN64 U64 U1[5]; + __ALIGN64 U64 Z1[5]; + fe52mb8_set(U1, 1); + fe52mb8_set(Z1, 1); + + /* set up: pre-computed G-S (base and "special") */ + __ALIGN64 U64 Z2[5]; + __ALIGN64 U64 U2[5] = { loadu64(U2_0), loadu64(U2_1), loadu64(U2_2), loadu64(U2_3), loadu64(U2_4) }; + fe52mb8_set(Z2, 1); + + __ALIGN64 U64 A[5], B[5], C[5], D[5]; + + /* + scalar template is usual [01xx..xxx] [xxx..xxx] [xxx..xxx] [xxx..000] + processing scalar in L->R fashion: + - first process scalar' = scalar >>3 + - than 3 doubling + */ + __mb_mask swap = get_mask(0xff); + + /* read low and remove (zero) bits 0,1,2 */ + U64 e = loadu64(&scalar_mb8[0]); + e = srli64(e, 3); + + int bitpos; + for (bitpos=3; bitpos<255; bitpos++) { + if(0==(bitpos%64)) + e = loadu64(&scalar_mb8[bitpos/64]); + + __mb_mask b = cmp64_mask(and64_const(e, 1), get_zero64(), _MM_CMPINT_NE); + + swap = mask_xor (swap, b); + fe52mb8_cswap(U1, U2, swap); + fe52mb8_cswap(Z1, Z2, swap); + swap = b; + + /* load mu = muTBL[bitpos-3] */ + U64 mu[5]; + mu[0] = set64((long long)muTBL52[bitpos-3][0]); + mu[1] = set64((long long)muTBL52[bitpos-3][1]); + mu[2] = set64((long long)muTBL52[bitpos-3][2]); + mu[3] = set64((long long)muTBL52[bitpos-3][3]); + mu[4] = set64((long long)muTBL52[bitpos-3][4]); + + /* diff addition */ + fe52_sub(B, U1, Z1); /* B = U1-Z1 */ + fe52_add(A, U1, Z1); /* A = U1+Z1 */ + fe52_mul(C, mu, B); /* C = mu*B */ + fe52_sub(B, A, C); /* B = (U1+Z1) - mu*(U1-Z1) */ + fe52_add(A, A, C); /* A = (Ur+Z1) + mu*(U1-Z1) */ + ed25519_sqr_dual(A, B, A, B); + ed25519_mul_dual(U1, Z1, Z2, A, U2, B); + + e = srli64(e, 1); + } + + /* 3 doublings */ + for(bitpos=0; bitpos<3; bitpos++) { + fe52_add(A, U1, Z1); /* A = U1+Z1 */ + fe52_sub(B, U1, Z1); /* B = U1-Z1 */ + fe52_sqr(A, A); /* A = A^2 */ + fe52_sqr(B, B); /* B = B^*2 */ + fe52_sub(C, A, B); /* C = A-B */ + fe52_mul121666(D, C); /* D = (A+2)/4 * C*/ + fe52_add(D, D, B); /* D = D+B */ + fe52_mul(U1, A, B); /* U1 = A*B */ + fe52_mul(Z1, C, D); /* Z1 = C*D */ + } + + fe52mb8_inv_mod25519(A, Z1); + fe52_mul(U1, U1, A); + fe52mb8_red_p25519(U1, U1); + + /* convert result back */ + ifma_mb8_to_BNU((int64u * const*)pa_public_key, (const int64u (*)[8])U1, 256); + + /* clear secret */ + MB_FUNC_NAME(zero_)((int64u (*)[8])scalar_mb8, sizeof(scalar_mb8)/sizeof(U64)); + } + + return status; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_alias_avx512.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_alias_avx512.h new file mode 100644 index 0000000..b595347 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_alias_avx512.h @@ -0,0 +1,92 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#ifndef IFMA_MATH_AVX512_H +#define IFMA_MATH_AVX512_H + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include + +typedef __m512i m512; +typedef __mmask8 mask8; +typedef __mmask64 mask64; + +/* set */ +#define setzero_i64 _mm512_setzero_si512 +#define set1_i64 _mm512_set1_epi64 + +#if 0 +/* Note: intrinsics below not available in GCC 8.4 */ +#define set_i8 _mm512_set_epi8 +#define set_i16 _mm512_set_epi16 +#endif + +#define set_i64 _mm512_set_epi64 + +/* load/store */ +#define loadu_i64 _mm512_loadu_si512 +#define maskz_loadu_i64 _mm512_maskz_loadu_epi64 +#define storeu_i64 _mm512_storeu_si512 +#define mask_storeu_i64 _mm512_mask_storeu_epi64 + +/* logical shift */ +#define srli_i64 _mm512_srli_epi64 +#define srlv_i64 _mm512_srlv_epi64 +#define slli_i64 _mm512_slli_epi64 +#define sllv_i64 _mm512_sllv_epi64 + +/* arithmetic shift */ +#define srai_i64 _mm512_srai_epi64 +#define maskz_srai_i64 _mm512_maskz_srai_epi64 +#define maskz_srli_i64 _mm512_maskz_srli_epi64 + +/* logical */ +#define and_i64 _mm512_and_epi64 +#define or_i64 _mm512_or_si512 + +/* add */ +#define add_i64 _mm512_add_epi64 +#define mask_add_i64 _mm512_mask_add_epi64 + +/* sub */ +#define sub_i64 _mm512_sub_epi64 +#define mask_sub_i64 _mm512_mask_sub_epi64 + +/* cmp */ +#define cmp_i64_mask _mm512_cmp_epi64_mask +#define cmp_u64_mask _mm512_cmp_epu64_mask + +/* perm */ +#define maskz_permutexvar_i8 _mm512_maskz_permutexvar_epi8 +#define permutexvar_i8 _mm512_permutexvar_epi8 +#define permutexvar_i16 _mm512_permutexvar_epi16 + +/* move */ +#define mask_mov_i64 _mm512_mask_mov_epi64 + +/* ifma */ +#define madd52lo_i64 _mm512_madd52lo_epu64 +#define madd52hi_i64 _mm512_madd52hi_epu64 + +#define alignr_i64 _mm512_alignr_epi64 + +#endif // if (_IPP32E >= _IPP32E_K1) + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_alias_avx512vl.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_alias_avx512vl.h new file mode 100644 index 0000000..e578bf8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_alias_avx512vl.h @@ -0,0 +1,78 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#ifndef IFMA_ALIAS_AVX512VL_H +#define IFMA_ALIAS_AVX512VL_H + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include + +typedef __m256i m256i; +typedef __mmask8 mask8; +typedef __mmask32 mask32; +typedef __mmask64 mask64; + +/* load/store */ +#define m256_loadu_i64(A) _mm256_maskz_loadu_epi64(0xFF, (A)) +#define m256_maskz_loadu_i64 _mm256_maskz_loadu_epi64 +#define m256_storeu_i64(R, A) _mm256_mask_storeu_epi64((R), 0xFF, (A)) +#define m256_mask_storeu_i64 _mm256_mask_storeu_epi64 +/* cmp */ +#define m256_cmp_i64_mask _mm256_cmp_epi64_mask +/* mov */ +#define m256_mask_mov_i64 _mm256_mask_mov_epi64 +/* set */ +#define m256_setzero_i64 _mm256_setzero_si256 +#define m256_set_i8 _mm256_set_epi8 +#define m256_set_i16 _mm256_set_epi16 +#define m256_set_i64 _mm256_set_epi64x +#define m256_set1_i64 _mm256_set1_epi64x +/* shift permutexvar */ +#define m256_permutexvar_i8 _mm256_permutexvar_epi8 +#define m256_permutexvar_i16 _mm256_permutexvar_epi16 +#define m256_maskz_permutexvar_i8 _mm256_maskz_permutexvar_epi8 +/* shift logic/arithmetic */ +#define m256_srli_i64 _mm256_srli_epi64 +#define m256_slli_i64 _mm256_slli_epi64 +#define m256_srlv_i64 _mm256_srlv_epi64 +#define m256_sllv_i64 _mm256_sllv_epi64 +#define m256_srai_i64 _mm256_srai_epi64 +#define m256_maskz_srli_i64 _mm256_maskz_srli_epi64 +#define m256_maskz_srai_i64 _mm256_maskz_srai_epi64 +#define m256_maskz_slli_i64 _mm256_maskz_slli_epi64 +#define m256_alignr_i64 _mm256_alignr_epi64 +/* and/or */ +#define m256_and_i64 _mm256_and_si256 +#define m256_or_i64 _mm256_or_si256 +/* add/sub */ +#define m256_add_i64 _mm256_add_epi64 +#define m256_mask_add_i64 _mm256_mask_add_epi64 +#define m256_maskz_add_i64 _mm256_maskz_add_epi64 +#define m256_sub_i64 _mm256_sub_epi64 +#define m256_mask_sub_i64 _mm256_mask_sub_epi64 +/* ifma */ +#define m256_madd52lo_i64 _mm256_madd52lo_epu64 +#define m256_madd52hi_i64 _mm256_madd52hi_epu64 +#define m256_maskz_madd52lo_i64 _mm256_maskz_madd52lo_epu64 +#define m256_maskz_madd52hi_i64 _mm256_maskz_madd52hi_epu64 + +#endif // (_IPP32E >= _IPP32E_K1) + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method.h new file mode 100644 index 0000000..fdf4c88 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method.h @@ -0,0 +1,95 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_ARITH_METHOD_H +#define IFMA_ARITH_METHOD_H + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "owncp.h" +#include "ifma_alias_avx512.h" + +/* Modular arith methods based on 2^52 radix number representation */ +IPP_OWN_FUNPTR(m512, ifma_import, (const Ipp64u *arad64)) +IPP_OWN_FUNPTR(void, ifma_export, (Ipp64u *rrad64, const m512 arad52)) +IPP_OWN_FUNPTR(m512, ifma_encode, (const m512 a)) +IPP_OWN_FUNPTR(m512, ifma_decode, (const m512 a)) +IPP_OWN_FUNPTR(m512, ifma_mul, (const m512 a, const m512 b)) +IPP_OWN_FUNPTR(void, ifma_mul_dual, (m512 *r1, const m512 a1, const m512 b1, m512 *r2, const m512 a2, const m512 b2)) +IPP_OWN_FUNPTR(m512, ifma_sqr, (const m512 a)) +IPP_OWN_FUNPTR(void, ifma_sqr_dual, (m512 * r1, const m512 a1, m512 *r2, const m512 a2)) +IPP_OWN_FUNPTR(m512, ifma_norm, (const m512 a)) +IPP_OWN_FUNPTR(void, ifma_norm_dual, (m512 *r1, const m512 a1, m512 *r2, const m512 a2)) +IPP_OWN_FUNPTR(m512, ifma_lnorm, (const m512 a)) +IPP_OWN_FUNPTR(void, ifma_lnorm_dual, (m512 *r1, const m512 a1, m512 *r2, const m512 a2)) +IPP_OWN_FUNPTR(m512, ifma_add, (const m512 a, const m512 b)) +IPP_OWN_FUNPTR(m512, ifma_sub, (const m512 a, const m512 b)) +IPP_OWN_FUNPTR(m512, ifma_neg, (const m512 a)) +IPP_OWN_FUNPTR(m512, ifma_div2, (const m512 a)) +IPP_OWN_FUNPTR(m512, ifma_inv, (const m512 z)) +IPP_OWN_FUNPTR(m512, ifma_red, (const m512 a)) + +typedef struct _ifmaArithMethod { + ifma_import import_to52; + ifma_export export_to64; + ifma_encode encode; + ifma_decode decode; + ifma_mul mul; + ifma_mul_dual mul_dual; + ifma_sqr sqr; + ifma_sqr_dual sqr_dual; + ifma_norm norm; + ifma_norm_dual norm_dual; + ifma_lnorm lnorm; + ifma_lnorm_dual lnorm_dual; + ifma_add add; + ifma_sub sub; + ifma_neg neg; + ifma_div2 div2; + ifma_inv inv; + ifma_red red; +} ifmaArithMethod; + + +/* Pre-defined AVX512IFMA ISA based methods */ +#define gsArithGF_p256r1_avx512 OWNAPI(gsArithGF_p256r1_avx512) +IPP_OWN_DECL(ifmaArithMethod *, gsArithGF_p256r1_avx512, (void)) + +#define gsArithGF_n256r1_avx512 OWNAPI(gsArithGF_n256r1_avx512) +IPP_OWN_DECL(ifmaArithMethod *, gsArithGF_n256r1_avx512, (void)) + +#define gsArithGF_p384r1_avx512 OWNAPI(gsArithGF_p384r1_avx512) +IPP_OWN_DECL(ifmaArithMethod *, gsArithGF_p384r1_avx512, (void)) + +#define gsArithGF_n384r1_avx512 OWNAPI(gsArithGF_n384r1_avx512) +IPP_OWN_DECL(ifmaArithMethod *, gsArithGF_n384r1_avx512, (void)) + +static __NOINLINE void clear_secrets(m512 *a, m512 *b, m512 *c) +{ + if (NULL != a) + *a = setzero_i64(); + if (NULL != b) + *b = setzero_i64(); + if (NULL != c) + *c = setzero_i64(); +} + +#endif /* #if (_IPP32E >= _IPP32E_K1) */ + +#endif /* #ifndef IFMA_ARITH_METHOD_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method_n256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method_n256.c new file mode 100644 index 0000000..5cfba32 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method_n256.c @@ -0,0 +1,52 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ecnist/ifma_arith_method.h" +#include "ecnist/ifma_arith_p256.h" +#include "ecnist/ifma_arith_n256.h" + +IPP_OWN_DEFN(ifmaArithMethod *, gsArithGF_n256r1_avx512, (void)) +{ + static ifmaArithMethod m = { + /* import_to52 = */ convert_radix_to_52x5, + /* export_to64 = */ convert_radix_to_64x4, + /* encode = */ ifma_tomont52_n256, + /* decode = */ ifma_frommont52_n256, + /* mul = */ ifma_amm52_n256, + /* mul_dual = */ 0, + /* sqr = */ 0, + /* sqr_dual = */ 0, + /* norm = */ 0, + /* norm_dual = */ 0, + /* lnorm = */ 0, + /* lnorm_dual = */ 0, + /* add = */ ifma_add52_n256, + /* sub = */ 0, + /* neg = */ 0, + /* div2 = */ 0, + /* inv = */ ifma_aminv52_n256, + /* red = */ ifma_fastred52_n256 + }; + + return &m; +} + +#endif /* #if (_IPP32E >= _IPP32E_K1) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method_n384.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method_n384.c new file mode 100644 index 0000000..288b262 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method_n384.c @@ -0,0 +1,52 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ecnist/ifma_arith_method.h" +#include "ecnist/ifma_arith_p384.h" +#include "ecnist/ifma_arith_n384.h" + +IPP_OWN_DEFN(ifmaArithMethod *, gsArithGF_n384r1_avx512, (void)) +{ + static ifmaArithMethod m = { + /* import_to52 = */ convert_radix_to_52x8, + /* export_to64 = */ convert_radix_to_64x6, + /* encode = */ ifma_tomont52_n384, + /* decode = */ ifma_frommont52_n384, + /* mul = */ ifma_amm52_n384, + /* mul_dual = */ 0, + /* sqr = */ 0, + /* sqr_dual = */ 0, + /* norm = */ 0, + /* norm_dual = */ 0, + /* lnorm = */ 0, + /* lnorm_dual = */ 0, + /* add = */ ifma_add52_n384, + /* sub = */ 0, + /* neg = */ 0, + /* div2 = */ 0, + /* inv = */ ifma_aminv52_n384, + /* red = */ ifma_fastred52_n384 + }; + + return &m; +} + +#endif /* #if (_IPP32E >= _IPP32E_K1) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method_n521.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method_n521.c new file mode 100644 index 0000000..18ad7bc --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method_n521.c @@ -0,0 +1,51 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ecnist/ifma_arith_method_p521.h" +#include "ecnist/ifma_arith_p521.h" +#include "ecnist/ifma_arith_n521.h" + +IPP_OWN_DEFN(ifmaArithMethod_p521 *, gsArithGF_n521r1_avx512, (void)) +{ + static ifmaArithMethod_p521 m = { + convert_radix_to_52_p521, + convert_radix_to_64_p521, + ifma_tomont52_n521, + ifma_frommont52_n521, + ifma_amm52_n521, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + ifma_add52_n521, + 0, + 0, + ifma_aminv52_n521, + ifma_fastred52_n521 + }; + + return &m; +} + +#endif /* #if (_IPP32E >= _IPP32E_K1) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method_p256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method_p256.c new file mode 100644 index 0000000..75f3cdc --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method_p256.c @@ -0,0 +1,51 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ecnist/ifma_arith_method.h" +#include "ecnist/ifma_arith_p256.h" + +IPP_OWN_DEFN(ifmaArithMethod *, gsArithGF_p256r1_avx512, (void)) +{ + static ifmaArithMethod m = { + /* import_to52 = */ convert_radix_to_52x5, + /* export_to64 = */ convert_radix_to_64x4, + /* encode = */ ifma_tomont52_p256, + /* decode = */ ifma_frommont52_p256, + /* mul = */ ifma_amm52_p256, + /* mul_dual = */ ifma_amm52_dual_p256, + /* sqr = */ ifma_ams52_p256, + /* sqr_dual = */ ifma_ams52_dual_p256, + /* norm = */ ifma_norm52, + /* norm_dual = */ ifma_norm52_dual, + /* lnorm = */ ifma_lnorm52, + /* lnorm_dual = */ ifma_lnorm52_dual, + /* add = */ 0, + /* sub = */ 0, + /* neg = */ ifma_neg52_p256, + /* div2 = */ ifma_half52_p256, + /* inv = */ ifma_aminv52_p256, + /* red = */ 0 + }; + + return &m; +} + +#endif /* #if (_IPP32E >= _IPP32E_K1) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method_p384.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method_p384.c new file mode 100644 index 0000000..db46291 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method_p384.c @@ -0,0 +1,51 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ecnist/ifma_arith_method.h" +#include "ecnist/ifma_arith_p384.h" + +IPP_OWN_DEFN(ifmaArithMethod *, gsArithGF_p384r1_avx512, (void)) +{ + static ifmaArithMethod m = { + /* import_to52 = */ convert_radix_to_52x8, + /* export_to64 = */ convert_radix_to_64x6, + /* encode = */ ifma_tomont52_p384, + /* decode = */ ifma_frommont52_p384, + /* mul = */ ifma_amm52_p384, + /* mul_dual = */ ifma_amm52_dual_p384, + /* sqr = */ ifma_ams52_p384, + /* sqr_dual = */ ifma_ams52_dual_p384, + /* norm = */ ifma_norm52, + /* norm_dual = */ ifma_norm52_dual, + /* lnorm = */ ifma_lnorm52, + /* lnorm_dual = */ ifma_lnorm52_dual, + /* add = */ 0, + /* sub = */ 0, + /* neg = */ ifma_neg52_p384, + /* div2 = */ ifma_half52_p384, + /* inv = */ ifma_aminv52_p384, + /* red = */ 0 + }; + + return &m; +} + +#endif /* #if (_IPP32E >= _IPP32E_K1) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method_p521.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method_p521.c new file mode 100644 index 0000000..9188b9c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method_p521.c @@ -0,0 +1,50 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ecnist/ifma_arith_method_p521.h" +#include "ecnist/ifma_arith_p521.h" + +IPP_OWN_DEFN(ifmaArithMethod_p521 *, gsArithGF_p521r1_avx512, (void)) +{ + static ifmaArithMethod_p521 m = { + convert_radix_to_52_p521, + convert_radix_to_64_p521, + ifma_tomont52_p521, + ifma_frommont52_p521, + ifma_amm52_p521, + ifma_amm52_dual_p521, + ifma_ams52_p521, + ifma_ams52_dual_p521, + ifma_norm52_p521, + ifma_norm52_dual_p521, + ifma_lnorm52_p521, + ifma_lnorm52_dual_p521, + 0, + ifma_neg52_p521, + ifma_half52_p521, + ifma_aminv52_p521, + 0 + }; + + return &m; +} + +#endif /* #if (_IPP32E >= _IPP32E_K1) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method_p521.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method_p521.h new file mode 100644 index 0000000..83a39a5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_method_p521.h @@ -0,0 +1,85 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_ARITH_METHOD_P521_H +#define IFMA_ARITH_METHOD_P521_H + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ifma_defs_p521.h" + +/* Modular arith methods based on 2^52 radix number representation */ +IPP_OWN_FUNPTR(void, ifma_import, (fe521 pr[], const Ipp64u arad64[P521R1_LEN64])) +IPP_OWN_FUNPTR(void, ifma_export, (Ipp64u rrad64[P521R1_LEN64], const fe521 a)) +IPP_OWN_FUNPTR(void, ifma_encode, (fe521 pr[], const fe521 a)) +IPP_OWN_FUNPTR(void, ifma_decode, (fe521 pr[], const fe521 a)) +IPP_OWN_FUNPTR(void, ifma_mul, (fe521 pr[], const fe521 a, const fe521 b)) +IPP_OWN_FUNPTR(void, ifma_mul_dual, (fe521 pr1[], const fe521 a1, const fe521 b1, fe521 pr2[], const fe521 a2, const fe521 b2)) +IPP_OWN_FUNPTR(void, ifma_sqr, (fe521 pr[], const fe521 a)) +IPP_OWN_FUNPTR(void, ifma_sqr_dual, (fe521 pr1[], const fe521 a1, fe521 pr2[], const fe521 a2)) +IPP_OWN_FUNPTR(void, ifma_norm, (fe521 pr[], const fe521 a)) +IPP_OWN_FUNPTR(void, ifma_norm_dual, (fe521 pr1[], const fe521 a1, fe521 pr2[], const fe521 a2)) +IPP_OWN_FUNPTR(void, ifma_lnorm, (fe521 pr[], const fe521 a)) +IPP_OWN_FUNPTR(void, ifma_lnorm_dual, (fe521 pr1[], const fe521 a1, fe521 pr2[], const fe521 a2)) +IPP_OWN_FUNPTR(void, ifma_add, (fe521 pr[], const fe521 a, const fe521 b)) +IPP_OWN_FUNPTR(void, ifma_div2, (fe521 pr[], const fe521 a)) +IPP_OWN_FUNPTR(void, ifma_neg, (fe521 pr[], const fe521 a)) +IPP_OWN_FUNPTR(void, ifma_inv, (fe521 pr[], const fe521 z)) +IPP_OWN_FUNPTR(void, ifma_red, (fe521 pr[], const fe521 a)) + +typedef struct _ifmaArithMethod_p521 { + ifma_import import_to52; + ifma_export export_to64; + ifma_encode encode; + ifma_decode decode; + ifma_mul mul; + ifma_mul_dual mul_dual; + ifma_sqr sqr; + ifma_sqr_dual sqr_dual; + ifma_norm norm; + ifma_norm_dual norm_dual; + ifma_lnorm lnorm; + ifma_lnorm_dual lnorm_dual; + ifma_add add; + ifma_neg neg; + ifma_div2 div2; + ifma_inv inv; + ifma_red red; +} ifmaArithMethod_p521; + +/* Pre-defined AVX512IFMA ISA based methods */ +#define gsArithGF_p521r1_avx512 OWNAPI(gsArithGF_p521r1_avx512) +IPP_OWN_DECL(ifmaArithMethod_p521 *, gsArithGF_p521r1_avx512, (void)) + +#define gsArithGF_n521r1_avx512 OWNAPI(gsArithGF_n521r1_avx512) +IPP_OWN_DECL(ifmaArithMethod_p521 *, gsArithGF_n521r1_avx512, (void)) + +static __NOINLINE void clear_secrets(fe521 *a, fe521 *b, fe521 *c) +{ + if (NULL != a) + FE521_SET(*a) = m256_setzero_i64(); + if (NULL != b) + FE521_SET(*b) = m256_setzero_i64(); + if (NULL != c) + FE521_SET(*c) = m256_setzero_i64(); +} + +#endif /* #if (_IPP32E >= _IPP32E_K1) */ + +#endif /* #ifndef IFMA_ARITH_METHOD_P521_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_n256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_n256.c new file mode 100644 index 0000000..13e36ac --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_n256.c @@ -0,0 +1,255 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ifma_arith_n256.h" +#include "ifma_arith_p256.h" +#include "ifma_defs.h" + +/* Constants */ +#define LEN52 (5 + 3) /* 5 digits + 3 zero padding */ + +/* + * EC NIST-P256 base point order + * in 2^52 radix + */ +static const __ALIGN64 Ipp64u n256_x1[LEN52] = { + 0x0009cac2fc632551, 0x000ada7179e84f3b, 0x000fffffffbce6fa, 0x0000000fffffffff, 0x0000ffffffff0000, 0x0, 0x0, 0x0 +}; + +static const __ALIGN64 Ipp64u n256_x2[LEN52] = { + 0x00039585f8c64aa2, 0x0005b4e2f3d09e77, 0x000fffffff79cdf5, 0x0000001fffffffff, 0x0001fffffffe0000, 0x0, 0x0, 0x0 +}; + +/* k0 = (-1/n256) mod 2^52 */ +static const __ALIGN64 Ipp64u n256_k0 = 0x1c8aaee00bc4f; + +/* + * r = 2^((52*5)) mod n256 + */ +static const __ALIGN64 Ipp64u n256_r[LEN52] = { + 0x000353d039cdaaf0, 0x000258e8617b0c46, 0x0000000004319055, 0x000fff0000000000, 0x00000000000fffff, 0x0, 0x0, 0x0 +}; + +/* To Montgomery conversion constant + * r = 2^((52*5)*2) mod n256 + */ +static const __ALIGN64 Ipp64u n256_rr[LEN52] = { + 0x0005cc0dea6dc3ba, 0x000192a067d8a084, 0x000bec59615571bb, 0x0001fc245b2392b6, 0x0000e12d9559d956, 0x0, 0x0, 0x0 +}; + +static const __ALIGN64 Ipp64u ones[LEN52] = { + 1, 0, 0, 0, 0, 0, 0, 0 +}; + +#define MUL_RED_ROUND(R, A, B, IDX) \ + { \ + const m512 Bi = permutexvar_i8(idx_b##IDX, (B)); \ + m512 Rlo = madd52lo_i64(zero, (A), Bi); \ + m512 tmp = madd52hi_i64(zero, (A), Bi); \ + \ + (R) = add_i64((R), Rlo); \ + /* broadcast R[0] */ \ + const m512 R0 = permutexvar_i8(idx_b0, (R)); \ + const m512 Yi = madd52lo_i64(zero, k0, R0); \ + (R) = madd52lo_i64((R), N, Yi); \ + tmp = madd52hi_i64(tmp, N, Yi); \ + /* shift */ \ + const m512 carry = maskz_srli_i64(maskone, (R), DIGIT_SIZE); \ + tmp = add_i64(tmp, carry); \ + (R) = maskz_permutexvar_i8(mask_sr64, idx_sr64, (R)); \ + /* hi */ \ + (R) = add_i64((R), tmp); \ + } + +/* R = (A*B) */ +IPP_OWN_DEFN(m512, ifma_amm52_n256, (const m512 a, const m512 b)) +{ + const m512 N = loadu_i64(n256_x1); /* modulus */ + const m512 k0 = set1_i64(n256_k0); /* k0 */ + const m512 zero = setzero_i64(); + const mask8 maskone = 0x1; + + /* Index broadcast */ + const m512 idx_b0 = set_i64(REPL8(0x0706050403020100)); // 7, 6, 5, 4, 3, 2, 1, 0 + const m512 idx_b1 = set_i64(REPL8(0x0f0e0d0c0b0a0908)); // 15, 14, 13, 12, 11, 10, 9, 8 + const m512 idx_b2 = set_i64(REPL8(0x1716151413121110)); // 23, 22, 21, 20, 19, 18, 17, 16 + const m512 idx_b3 = set_i64(REPL8(0x1f1e1d1c1b1a1918)); // 31, 30, 29, 28, 27, 26, 25, 24 + const m512 idx_b4 = set_i64(REPL8(0x2726252423222120)); // 39, 38, 37, 36, 35, 34, 33, 32 + + /* Mask to shift zmm register right on 64-bit */ + const mask64 mask_sr64 = 0x00ffffffffffffff; + const m512 idx_sr64 = set_i64(0x0, // 0, 0, 0, 0, 0, 0, 0, 0 + 0x3f3e3d3c3b3a3938, // 63, 62, 61, 60, 59, 58, 57, 56 + 0x3736353433323130, // 55, 54, 53, 52, 51, 50, 49, 48 + 0x2f2e2d2c2b2a2928, // 47, 46, 45, 44, 43, 42, 41, 40 + 0x2726252423222120, // 39, 38, 37, 36, 35, 34, 33, 32 + 0x1f1e1d1c1b1a1918, // 31, 30, 29, 28, 27, 26, 25, 24 + 0x1716151413121110, // 23, 22, 21, 20, 19, 18, 17, 16 + 0x0f0e0d0c0b0a0908); // 15, 14, 13, 12, 11, 10, 9, 8 + + m512 r = setzero_i64(); + + MUL_RED_ROUND(r, a, b, 0) + MUL_RED_ROUND(r, a, b, 1) + MUL_RED_ROUND(r, a, b, 2) + MUL_RED_ROUND(r, a, b, 3) + MUL_RED_ROUND(r, a, b, 4) + + r = ifma_lnorm52(r); + + return r; +} + +IPP_OWN_DEFN(m512, ifma_add52_n256, (const m512 a, const m512 b)) +{ + const m512 zero = setzero_i64(); + const m512 Nx2 = loadu_i64(n256_x2); + + /* r = a + b */ + m512 r = add_i64(a, b); + r = ifma_lnorm52(r); + + /* t = r - N */ + m512 t = sub_i64(r, Nx2); + t = ifma_norm52(t); + + /* lt = t < 0 */ + const mask8 lt = cmp_i64_mask(zero, srli_i64(t, DIGIT_SIZE - 1), _MM_CMPINT_LT); + const mask8 mask = check_bit(lt, 4); // check sign in 5h digit + /* mask != 0 ? r : t */ + r = mask_mov_i64(t, mask, r); + + return r; +} + +static m512 ifma_ams52_n256(const m512 a) +{ + return ifma_amm52_n256(a, a); +} + +IPP_OWN_DEFN(m512, ifma_fastred52_n256, (const m512 a)) +{ + const m512 N = loadu_i64(n256_x1); + const m512 zero = setzero_i64(); + + /* r = a - N */ + m512 r = sub_i64(a, N); + r = ifma_norm52(r); + + /* 1 < 0 */ + const mask8 lt = cmp_i64_mask(zero, srli_i64(r, DIGIT_SIZE - 1), _MM_CMPINT_LT); + const mask8 mask = check_bit(lt, 4); // check sign in 5th digit + + /* mask != 0 ? a : r */ + r = mask_mov_i64(r, mask, a); + return r; +} + +IPP_OWN_DEFN(m512, ifma_tomont52_n256, (const m512 a)) +{ + return ifma_amm52_n256(a, loadu_i64(n256_rr)); +} + +IPP_OWN_DEFN(m512, ifma_frommont52_n256, (const m512 a)) +{ + m512 r = ifma_amm52_n256(a, loadu_i64(ones)); + return ifma_fastred52_n256(r); +} + +#define sqr(R, A) (R) = ifma_ams52_n256((A)) +#define mul(R, A, B) (R) = ifma_amm52_n256((A), (B)) + +/* + * computes r = 1/z = z^(n256-2) mod n256 + * + * note: z in in Montgomery domain + * r in Montgomery domain + */ +__INLINE m512 ifma_ams52_n256_ntimes(const m512 a, int n) +{ + m512 r = a; + for (; n > 0; --n) { + sqr(r, r); + } + return r; +} + +#define sqr_ntimes(R, A, N) (R) = ifma_ams52_n256_ntimes((A), (N)) + +/* + * computes r = 1/z = z^(n256-2) mod n256 + * + * note: z is in Montgomery domain + * (as soon mul() and sqr() below are amm-functions, + * the result is in Montgomery domain too) + */ + +IPP_OWN_DEFN(m512, ifma_aminv52_n256, (const m512 z)) +{ + const m512 r_n256 = loadu_i64(n256_r); + int i; + + // pwr_z_Tbl[i][] = z^i, i=0,..,15 + __ALIGN64 m512 pwr_z_tbl[16]; + m512 r; + + pwr_z_tbl[0] = r_n256; + pwr_z_tbl[1] = z; + for (i = 2; i < 16; i += 2) { + sqr(pwr_z_tbl[i], pwr_z_tbl[i / 2]); + mul(pwr_z_tbl[i + 1], pwr_z_tbl[i], z); + } + + // pwr = (n256-2) in big endian + Ipp8u pwr[] = "\xFF\xFF\xFF\xFF\x00\x00\x00\x00" + "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" + "\xBC\xE6\xFA\xAD\xA7\x17\x9E\x84" + "\xF3\xB9\xCA\xC2\xFC\x63\x25\x4F"; + // init r = 1 + r = r_n256; + + for (i = 0; i < 32; i++) { + int v = pwr[i]; + int hi = (v >> 4) & 0xF; + int lo = v & 0xF; + + sqr(r, r); + sqr(r, r); + sqr(r, r); + sqr(r, r); + if (hi) + mul(r, r, pwr_z_tbl[hi]); + sqr(r, r); + sqr(r, r); + sqr(r, r); + sqr(r, r); + if (lo) + mul(r, r, pwr_z_tbl[lo]); + } + + return r; +} + +#undef sqr +#undef mul +#undef sqr_ntimes + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_n256.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_n256.h new file mode 100644 index 0000000..e6d1d34 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_n256.h @@ -0,0 +1,95 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#if !defined(_IFMA_ARITH_N256R1_H_) +#define _IFMA_ARITH_N256R1_H_ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ifma_alias_avx512.h" + +/** + * \brief + * + * Montgomery multiplication + * + * a * b * R mod n, where R = 2^(6*52) mod n + * + * \param[in] a first value (in radix 2^52) + * \param[in] b second value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_amm52_n256, (const m512 a, const m512 b)) + +/** + * \brief + * + * A + B (mod n) + * + * \param[in] a first value (in radix 2^52) + * \param[in] b second value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_add52_n256, (const m512 a, const m512 b)) + +/** + * \brief + * + * Reduction modulo n (subgroup order). + * + * (a >= n256) ? a - n256 : a + * + * \param[in] a value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_fastred52_n256, (const m512 a)) + +/** + * \brief + * + * Conversion to Montgomery domain modulo n (subgroup order). + * + * a * R mod n, where R = 2^(6*52) mod n + * + * \param[in] a value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_tomont52_n256, (const m512 a)) + +/** + * \brief + * + * Conversion from Montgomery domain modulo n (subgroup order). + * + * a mod n + * + * \param[in] p value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_frommont52_n256, (const m512 a)) + +/** + * \brief + * + * Modular inverse modulo n (subgroup order). + * + * 1/z mod n + * + * \param[in] z value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_aminv52_n256, (const m512 z)) + +#endif // (_IPP32E >= _IPP32E_K1) + +#endif // _IFMA_ARITH_N256R1_H_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_n384.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_n384.c new file mode 100644 index 0000000..bc0cc03 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_n384.c @@ -0,0 +1,299 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ifma_arith_n384.h" +#include "ifma_arith_p384.h" +#include "ifma_defs.h" + +/* Constants */ +#define LEN52 (NUMBER_OF_DIGITS(384, 52)) + +/* + * EC NIST-P384 base point order + * in 2^52 radix + */ +static const __ALIGN64 Ipp64u n384_x1[LEN52] = { + 0x000c196accc52973, 0x000b248b0a77aece, 0x0004372ddf581a0d, 0x000ffffc7634d81f, 0x000fffffffffffff, 0x000fffffffffffff, 0x000fffffffffffff, 0x00000000000fffff +}; + +/* k0 = (-1/n384) mod 2^52 */ +static const __ALIGN64 Ipp64u n384_k0 = 0x00046089e88fdc45; + +/* r = 2^((52*8)) mod n384 */ +static const __ALIGN64 Ipp64u n384_r[LEN52] = { + 0x000ad68d00000000, 0x000851313e695333, 0x0007e5f24db74f58, 0x000b27e0bc8d220a, 0x000000000000389c, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000 +}; + +/* To Montgomery conversion constant + * rr = 2^((52*8)*2) mod n384 + */ +static const __ALIGN64 Ipp64u n384_rr[LEN52] = { + 0x00034124f50ddb2d, 0x000c974971bd0d8d, 0x0002118942bfd3cc, 0x0009f43be8072178, 0x0005bf030606de60, 0x0000d49174aab1cc, 0x000b7a28266895d4, 0x000000000003fb05 +}; + +static const __ALIGN64 Ipp64u ones[LEN52] = { + 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 +}; + +#define MULT_ROUND(R, A, B, IDX) \ + const m512 Bi##R##IDX = permutexvar_i8(idx_b##IDX, (B)); \ + const m512 amBiLo##R##IDX = madd52lo_i64(zero, (A), Bi##R##IDX); \ + m512 tr##R##IDX = madd52hi_i64(zero, (A), Bi##R##IDX); \ + { \ + /* low */ \ + (R) = add_i64((R), amBiLo##R##IDX); \ + const m512 R0 = permutexvar_i8(idx_b0, (R)); \ + const m512 u = madd52lo_i64(zero, K0, R0); \ + tr##R##IDX = madd52hi_i64(tr##R##IDX, N, u); \ + (R) = madd52lo_i64((R), N, u); \ + /* shift */ \ + const m512 carryone = maskz_srai_i64(maskone, (R), DIGIT_SIZE); \ + tr##R##IDX = add_i64(tr##R##IDX, carryone); \ + (R) = maskz_permutexvar_i8(mask_sr64, idx_sr64, (R)); \ + /* hi */ \ + (R) = add_i64((R), tr##R##IDX); \ + } + +/* R = (A*B) - no normalization (in radix 2^52) */ +IPP_OWN_DEFN(m512, ifma_amm52_n384, (const m512 a, const m512 b)) +{ + const m512 N = loadu_i64(n384_x1); /* n */ + const m512 K0 = set1_i64(n384_k0); /* k0 */ + const m512 zero = setzero_i64(); + const mask8 maskone = 0x1; + + /* Index broadcast */ + const m512 idx_b0 = set_i64(REPL8(0x0706050403020100)); // 7, 6, 5, 4, 3, 2, 1, 0 + const m512 idx_b1 = set_i64(REPL8(0x0f0e0d0c0b0a0908)); // 15, 14, 13, 12, 11, 10, 9, 8 + const m512 idx_b2 = set_i64(REPL8(0x1716151413121110)); // 23, 22, 21, 20, 19, 18, 17, 16 + const m512 idx_b3 = set_i64(REPL8(0x1f1e1d1c1b1a1918)); // 31, 30, 29, 28, 27, 26, 25, 24 + const m512 idx_b4 = set_i64(REPL8(0x2726252423222120)); // 39, 38, 37, 36, 35, 34, 33, 32 + const m512 idx_b5 = set_i64(REPL8(0x2f2e2d2c2b2a2928)); // 47, 46, 45, 44, 43, 42, 41, 40 + const m512 idx_b6 = set_i64(REPL8(0x3736353433323130)); // 55, 54, 53, 52, 51, 50, 49, 48 + const m512 idx_b7 = set_i64(REPL8(0x3f3e3d3c3b3a3938)); // 63, 62, 61, 60, 59, 58, 57, 56 + + /* Mask to shift zmm register right on 64-bit */ + const mask64 mask_sr64 = 0x00ffffffffffffff; + const m512 idx_sr64 = set_i64(0x0, // 0, 0, 0, 0, 0, 0, 0, 0 + 0x3f3e3d3c3b3a3938, // 63, 62, 61, 60, 59, 58, 57, 56 + 0x3736353433323130, // 55, 54, 53, 52, 51, 50, 49, 48 + 0x2f2e2d2c2b2a2928, // 47, 46, 45, 44, 43, 42, 41, 40 + 0x2726252423222120, // 39, 38, 37, 36, 35, 34, 33, 32 + 0x1f1e1d1c1b1a1918, // 31, 30, 29, 28, 27, 26, 25, 24 + 0x1716151413121110, // 23, 22, 21, 20, 19, 18, 17, 16 + 0x0f0e0d0c0b0a0908); // 15, 14, 13, 12, 11, 10, 9, 8 + + m512 r = setzero_i64(); + /* N384 + * m' mod b = 46089e88fdc45 + * + * Algorithm + * a[] b[] - input data ((in radix 2^52)) m[] - module n384 + * + * where u = R[0]*m' mod b + * 1) R = R + a[] * b[i] (lo) + * 2) R = R + m[] * u (lo) + * 3) R = R >> 64 + * 4) R = R + a[] * b[i] (hi) + * 5) R = R + m[] * u (hi) + */ + /* one round = O(32) */ + MULT_ROUND(r, a, b, 0) + MULT_ROUND(r, a, b, 1) + MULT_ROUND(r, a, b, 2) + MULT_ROUND(r, a, b, 3) + MULT_ROUND(r, a, b, 4) + MULT_ROUND(r, a, b, 5) + MULT_ROUND(r, a, b, 6) + MULT_ROUND(r, a, b, 7) + + r = ifma_lnorm52(r); + + return r; +} + +IPP_OWN_DEFN(m512, ifma_add52_n384, (const m512 a, const m512 b)) +{ + const m512 zero = setzero_i64(); + const m512 N = loadu_i64(n384_x1); + + /* r = a + b */ + m512 r = add_i64(a, b); + r = ifma_lnorm52(r); + + /* t = r - N */ + m512 t = sub_i64(r, N); + t = ifma_norm52(t); + + /* lt = t < 0 */ + const mask8 lt = cmp_i64_mask(zero, srli_i64(t, DIGIT_SIZE - 1), _MM_CMPINT_LT); + const mask8 mask = check_bit(lt, 7); + /* mask != 0 ? r : t */ + r = mask_mov_i64(t, mask, r); + + return r; +} + +static m512 ifma_ams52_n384(const m512 a) +{ + return ifma_amm52_n384(a, a); +} + +IPP_OWN_DEFN(m512, ifma_tomont52_n384, (const m512 a)) +{ + const m512 RR = loadu_i64(n384_rr); + return ifma_amm52_n384(a, RR); +} + +IPP_OWN_DEFN(m512, ifma_fastred52_n384, (const m512 a)) +{ + const m512 N = loadu_i64(n384_x1); + const m512 zero = setzero_i64(); + + /* r = a - N */ + m512 r = sub_i64(a, N); + r = ifma_norm52(r); + + /* 1 < 0 */ + const mask8 lt = cmp_i64_mask(zero, srli_i64(r, DIGIT_SIZE - 1), _MM_CMPINT_LT); + const mask8 mask = check_bit(lt, 7); + + /* maks != 0 ? a : r */ + r = mask_mov_i64(r, mask, a); + return r; +} + +IPP_OWN_DEFN(m512, ifma_frommont52_n384, (const m512 a)) +{ + const m512 ONE = loadu_i64(ones); + m512 r = ifma_amm52_n384(a, ONE); + return ifma_fastred52_n384(r); +} + +#define sqr(R, A) (R) = ifma_ams52_n384((A)) +#define mul(R, A, B) (R) = ifma_amm52_n384((A), (B)) + +/* + * computes r = 1/z = z^(n384-2) mod n384 + * + * note: z in in Montgomery domain + * r in Montgomery domain + */ +__INLINE m512 ifma_ams52_n384_ntimes(const m512 a, int n) +{ + m512 r = a; + for (; n > 0; --n) { + sqr(r, r); + } + return r; +} + +#define sqr_ntimes(R, A, N) (R) = ifma_ams52_n384_ntimes((A), (N)) + +IPP_OWN_DEFN(m512, ifma_aminv52_n384, (const m512 z)) +{ + + const m512 r_norder = loadu_i64(n384_r); + + /* table pwr_z_Tbl[i] = z^i, i = 0,..,15 */ + __ALIGN64 m512 pwr_z_tbl[16]; + m512 lexp; + lexp = setzero_i64(); + + /* fill table */ + pwr_z_tbl[0] = r_norder; + pwr_z_tbl[1] = z; + for (int i = 2; i < 16; i += 2) { + sqr(pwr_z_tbl[i], pwr_z_tbl[i / 2]); + mul(pwr_z_tbl[i + 1], pwr_z_tbl[i], z); + } + + // pwr = (n384-2) in big endian + const Ipp8u pwr[] = "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" + "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" + "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" + "\xC7\x63\x4D\x81\xF4\x37\x2D\xDF" + "\x58\x1A\x0D\xB2\x48\xB0\xA7\x7A" + "\xEC\xEC\x19\x6A\xCC\xC5\x29\x71"; + + /* + * process low part of the exponent: "0xc7634d81f4372ddf 0x581a0db248b0a77a 0xecec196accc52973" + */ + /* init result */ + lexp = r_norder; + for (Ipp32u i = 24u; i < (sizeof(pwr) - 1u); ++i) { + const int v = (int)(pwr[i]); + const int hi = (v >> 4) & 0xF; + const int lo = v & 0xF; + + sqr(lexp, lexp); + sqr(lexp, lexp); + sqr(lexp, lexp); + sqr(lexp, lexp); + if (0 != hi) + mul(lexp, lexp, pwr_z_tbl[hi]); + + sqr(lexp, lexp); + sqr(lexp, lexp); + sqr(lexp, lexp); + sqr(lexp, lexp); + if (0 != lo) + mul(lexp, lexp, pwr_z_tbl[lo]); + } + m512 u, v; + u = v = setzero_i64(); + + sqr(v, z); /* v = z^2 */ + mul(u, v, z); /* u = z^2 * z = z^3 */ + /**/ + sqr_ntimes(v, u, 2); /* v = (z^3)^(2^2) = z^12 */ + mul(u, v, u); /* u = z^12 * z^3 = z^15 = z^(0xF) */ + /**/ + sqr_ntimes(v, u, 4); /* v = (z^0xF)^(2^4) = z^(0xF0) */ + mul(u, v, u); /* u = z^0xF0 * z^(0xF) = z^(0xFF) */ + /**/ + sqr_ntimes(v, u, 8); /* v = (z^0xFF)^(2^8) = z^(0xFF00) */ + mul(u, v, u); /* u = z^0xFF00 * z^(0xFF) = z^(0xFFFF) */ + /**/ + sqr_ntimes(v, u, 16); /* v = (z^0xFFFF)^(2^16) = z^(0xFFFF0000) */ + mul(u, v, u); /* u = z^0xFFFF0000 * z^(0xFFFF) = z^(0xFFFFFFFF) */ + /**/ + sqr_ntimes(v, u, 32); /* v = (z^0xFFFFFFFF)^(2^32) = z^(0xFFFFFFFF00000000) */ + mul(u, v, u); /* u = z^0xFFFFFFFF00000000 * z^(0xFFFFFFFF) = z^(0xFFFFFFFFFFFFFFFF) */ + /**/ + sqr_ntimes(v, u, 64); /**/ + mul(v, v, u); /* v = z^(0xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF) */ + /**/ + sqr_ntimes(v, v, 64); /**/ + mul(v, v, u); /* v = z^(0xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF) */ + + /* combine low and high results */ + sqr_ntimes(v, v, 64 * 3); /* u = z^(0xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.0000000000000000.0000000000000000.0000000000000000) */ + m512 r = setzero_i64(); /**/ + mul(r, v, lexp); /* r = z^(0xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.c7634d81f4372ddf.581a0db248b0a77a.ecec196accc52973) */ + return r; +} + +#undef sqr +#undef mul +#undef sqr_ntimes + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_n384.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_n384.h new file mode 100644 index 0000000..a8cc448 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_n384.h @@ -0,0 +1,95 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#if !defined(_IFMA_ARITH_N384R1_H_) +#define _IFMA_ARITH_N384R1_H_ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ifma_alias_avx512.h" + +/** + * \brief + * + * Montgomery multiplication + * + * a * b * R mod n, where R = 2^(5*52) mod n + * + * \param[in] a first value (in radix 2^52) + * \param[in] b second value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_amm52_n384, (const m512 a, const m512 b)) + +/** + * \brief + * + * A + B (in 2^52 radix) + * + * \param[in] a first value (in radix 2^52) + * \param[in] b second value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_add52_n384, (const m512 a, const m512 b)) + +/** + * \brief + * + * Reduction modulo n (subgroup order). + * + * (a >= n384) ? a - n384 : a + * + * \param[in] a value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_fastred52_n384, (const m512 a)) + +/** + * \brief + * + * Conversion to Montgomery domain modulo n (subgroup order). + * + * a * R mod n, where R = 2^(5*52) mod n + * + * \param[in] a value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_tomont52_n384, (const m512 a)) + +/** + * \brief + * + * Conversion from Montgomery domain modulo n (subgroup order). + * + * a mod n + * + * \param[in] p value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_frommont52_n384, (const m512 a)) + +/** + * \brief + * + * Modular inverse modulo n (subgroup order). + * + * 1/z mod n + * + * \param[in] z value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_aminv52_n384, (const m512 z)) + +#endif // (_IPP32E >= _IPP32E_K1) + +#endif // _IFMA_ARITH_N384R1_H_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_n521.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_n521.c new file mode 100644 index 0000000..f059dd9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_n521.c @@ -0,0 +1,351 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ifma_arith_p521.h" +/* +// EC NIST-P521 prime base point order +// in 2^52 radix +*/ +static const __ALIGN64 Ipp64u n521_x1[P521R1_NUM_CHUNK][P521R1_LENFE521_52] = { + { 0x000fb71e91386409, + 0x000b8899c47aebb6, + 0x000709a5d03bb5c9, + 0x000966b7fcc0148f }, + { 0x000a51868783bf2f, + 0x000fffffffffffff, + 0x000fffffffffffff, + 0x000fffffffffffff }, + { 0x000fffffffffffff, + 0x000fffffffffffff, + 0x0000000000000001, + 0x0000000000000000 } +}; + +static const __ALIGN64 Ipp64u n521_k0 = 0x000f5ccd79a995c7; + +/* to Montgomery conversion constant +// rr = 2^((P521_LEN52*DIGIT_SIZE)*2) mod n521 +*/ +static const __ALIGN64 Ipp64u n521_rr[P521R1_NUM_CHUNK][P521R1_LENFE521_52] = { + { 0x0003b4d7a5b140ce, + 0x000cb0bf26c55bf9, + 0x00037e5396c67ee9, + 0x0002bd1c80cf7b13 }, + { 0x00073cbe28f15e41, + 0x000dd6e23d82e49c, + 0x0003d142b7756e3e, + 0x00061a8e567bccff }, + { 0x00092d0d455bcc6d, + 0x000383d2d8e03d14, + 0x0000000000000000, + 0x0000000000000000 } +}; + +/* ifma_tomont52_n521_(1) */ +static const __ALIGN64 Ipp64u n521_r[P521R1_NUM_CHUNK][P521R1_LENFE521_52] = { + { 0x0008000000000000, + 0x00082470b763cdfb, + 0x00023bb31dc28a24, + 0x00047b2d17e2251b }, + { 0x00034ca4019ff5b8, + 0x0002d73cbc3e2068, + 0x0000000000000000, + 0x0000000000000000 }, + { 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000 } +}; + +#define group_madd52hi_i64(R, A, B, C) \ + FE521_LO(R) = m256_madd52hi_i64(FE521_LO(A), FE521_LO(B), (C)); \ + FE521_MID(R) = m256_madd52hi_i64(FE521_MID(A), FE521_MID(B), (C)); \ + FE521_HI(R) = m256_madd52hi_i64(FE521_HI(A), FE521_HI(B), (C)) + +#define group_madd52lo_i64(R, A, B, C) \ + FE521_LO(R) = m256_madd52lo_i64(FE521_LO(A), FE521_LO(B), (C)); \ + FE521_MID(R) = m256_madd52lo_i64(FE521_MID(A), FE521_MID(B), (C)); \ + FE521_HI(R) = m256_madd52lo_i64(FE521_HI(A), FE521_HI(B), (C)) + +#define NEW_MUL_ROUND(R, A, IDX8, B) \ + const m256i Bi##R##IDX8 = m256_permutexvar_i8((IDX8), (B)); \ + fe521 amBiLo##R##IDX8, tr##R##IDX8; \ + /* (lo/hi) a[]*b[i] */ \ + FE521_LO(amBiLo##R##IDX8) = m256_madd52lo_i64(zero, FE521_LO((A)), (Bi##R##IDX8)); \ + FE521_MID(amBiLo##R##IDX8) = m256_madd52lo_i64(zero, FE521_MID((A)), (Bi##R##IDX8)); \ + FE521_HI(amBiLo##R##IDX8) = m256_madd52lo_i64(zero, FE521_HI((A)), (Bi##R##IDX8)); \ + FE521_LO(tr##R##IDX8) = m256_madd52hi_i64(zero, FE521_LO((A)), (Bi##R##IDX8)); \ + FE521_MID(tr##R##IDX8) = m256_madd52hi_i64(zero, FE521_MID((A)), (Bi##R##IDX8)); \ + FE521_HI(tr##R##IDX8) = m256_madd52hi_i64(zero, FE521_HI((A)), (Bi##R##IDX8)); \ + { \ + /* R = R + a[]*b[i](lo) */ \ + fe521_add_no_red(R, R, amBiLo##R##IDX8); \ + /* u = R[0] * 1 */ \ + const m256i R0 = m256_permutexvar_i8(idx0, FE521_LO((R))); \ + const m256i u = m256_madd52lo_i64(zero, R0, K0); \ + /* R = R + m[]*u (lo/hi) */ \ + group_madd52hi_i64(tr##R##IDX8, tr##R##IDX8, N, u); \ + group_madd52lo_i64(R, R, N, u); \ + /* get carry low + add hi compute */ \ + const m256i carryone = m256_maskz_srai_i64(0x1, FE521_LO(R), DIGIT_SIZE_52); \ + FE521_LO(tr##R##IDX8) = m256_add_i64(FE521_LO(tr##R##IDX8), carryone); \ + /* shift */ \ + FE521_LO(R) = m256_alignr_i64(FE521_MID(R), FE521_LO(R), 1); \ + FE521_MID(R) = m256_alignr_i64(FE521_HI(R), FE521_MID(R), 1); \ + FE521_HI(R) = m256_maskz_permutexvar_i8(mask_sr64, idx_sr64, FE521_HI(R)); \ + /* add hi compute */ \ + fe521_add_no_red(R, R, tr##R##IDX8); \ + } + +IPP_OWN_DEFN(void, ifma_amm52_n521, (fe521 pr[], const fe521 a, const fe521 b)) +{ + /* k0 */ + const m256i K0 = m256_set1_i64(n521_k0); + fe521 N; + FE521_LOADU(N, n521_x1); + const m256i zero = m256_setzero_i64(); + /* chunk2 shift bit >> 64 */ + const mask32 mask_sr64 = 0x00FFFFFF; + const m256i idx_sr64 = m256_set_i8(0, 0, 0, 0, 0, 0, 0, 0, + 31, 30, 29, 28, 27, 26, 25, 24, + 23, 22, 21, 20, 19, 18, 17, 16, + 15, 14, 13, 12, 11, 10, 9, 8); + + // IDX + const m256i idx0 = m256_set_i8(REPL4(7, 6, 5, 4, 3, 2, 1, 0)); + const m256i idx1 = m256_set_i8(REPL4(15, 14, 13, 12, 11, 10, 9, 8)); + const m256i idx2 = m256_set_i8(REPL4(23, 22, 21, 20, 19, 18, 17, 16)); + const m256i idx3 = m256_set_i8(REPL4(31, 30, 29, 28, 27, 26, 25, 24)); + + const m256i idx4 = m256_set_i8(REPL4(7, 6, 5, 4, 3, 2, 1, 0)); + const m256i idx5 = m256_set_i8(REPL4(15, 14, 13, 12, 11, 10, 9, 8)); + const m256i idx6 = m256_set_i8(REPL4(23, 22, 21, 20, 19, 18, 17, 16)); + const m256i idx7 = m256_set_i8(REPL4(31, 30, 29, 28, 27, 26, 25, 24)); + + const m256i idx8 = m256_set_i8(REPL4(7, 6, 5, 4, 3, 2, 1, 0)); + const m256i idx9 = m256_set_i8(REPL4(15, 14, 13, 12, 11, 10, 9, 8)); + const m256i idx10 = m256_set_i8(REPL4(23, 22, 21, 20, 19, 18, 17, 16)); + + fe521 r; + FE521_SET(r) = m256_setzero_i64(); + NEW_MUL_ROUND(r, a, idx0, FE521_LO(b)) + NEW_MUL_ROUND(r, a, idx1, FE521_LO(b)) + NEW_MUL_ROUND(r, a, idx2, FE521_LO(b)) + NEW_MUL_ROUND(r, a, idx3, FE521_LO(b)) + NEW_MUL_ROUND(r, a, idx4, FE521_MID(b)) + NEW_MUL_ROUND(r, a, idx5, FE521_MID(b)) + NEW_MUL_ROUND(r, a, idx6, FE521_MID(b)) + NEW_MUL_ROUND(r, a, idx7, FE521_MID(b)) + NEW_MUL_ROUND(r, a, idx8, FE521_HI(b)) + NEW_MUL_ROUND(r, a, idx9, FE521_HI(b)) + NEW_MUL_ROUND(r, a, idx10, FE521_HI(b)) + + ifma_lnorm52_p521(&r, r); + + FE521_COPY(*pr, r); + return; +} + +#undef group_madd52hi_i64 +#undef group_madd52lo_i64 +#undef NEW_MUL_ROUND + +IPP_OWN_DEFN(void, ifma_add52_n521, (fe521 pr[], const fe521 a, const fe521 b)) +{ + const m256i zero = m256_setzero_i64(); + fe521 N; + FE521_LOADU(N, n521_x1); + + /* r = a + b */ + fe521 r; + fe521_add_no_red(r, a, b); + ifma_lnorm52_p521(&r, r); + + /* t = r - N */ + fe521 t; + fe521_sub_no_red(t, r, N); + ifma_norm52_p521(&t, t); + + /* lt = t < 0 */ + const mask8 lt = m256_cmp_i64_mask(zero, m256_srli_i64(FE521_HI(t), DIGIT_SIZE_52 - 1), _MM_CMPINT_LT); + const mask8 mask = (mask8)((mask8)0 - ((lt >> 2) & 1)); + + /* maks != 0 ? a : r */ + FE521_MASK_MOV(r, t, mask, r); + FE521_COPY(*pr, r); + return; +} + +IPP_OWN_DEFN(void, ifma_tomont52_n521, (fe521 pr[], const fe521 a)) +{ + fe521 RR; + FE521_LOADU(RR, n521_rr); + ifma_amm52_n521(pr, a, RR); + return; +} + +IPP_OWN_DEFN(void, ifma_fastred52_n521, (fe521 pr[], const fe521 a)) +{ + fe521 N; + FE521_LOADU(N, n521_x1); + const m256i zero = m256_setzero_i64(); + + fe521 r; + fe521_sub_no_red(r, a, N); + ifma_norm52_p521(&r, r); + + const mask8 lt = m256_cmp_i64_mask(zero, m256_srli_i64(FE521_HI(r), DIGIT_SIZE_52 - 1), _MM_CMPINT_LT); + const mask8 mask = (mask8)((mask8)0 - ((lt >> 2) & 1)); + + /* maks != 0 ? a : r */ + FE521_MASK_MOV(r, r, mask, a); + FE521_COPY(*pr, r); + return; +} + +IPP_OWN_DEFN(void, ifma_frommont52_n521, (fe521 pr[], const fe521 a)) +{ + fe521 ONE; + FE521_LOADU(ONE, P521R1_ONE52); + ifma_amm52_n521(pr, a, ONE); + ifma_fastred52_n521(pr, *pr); + return; +} + +static void ifma_ams52_n521(fe521 pr[], const fe521 a) +{ + ifma_amm52_n521(pr, a, a); +} + +#define mul(R, A, B) ifma_amm52_n521(&(R), (A), (B)) +#define sqr(R, A) ifma_ams52_n521(&(R), (A)) + +/* r = base^(2^n) */ +__INLINE IPP_OWN_DEFN(void, ifma_ams52_p521_ntimes, (fe521 pr[], const fe521 a, int n)) +{ + fe521 r; + FE521_COPY(r, a); + for (; n > 0; --n) + sqr(r, r); + FE521_COPY(*pr, r); + return; +} + +#define sqr_ntimes(R, A, N) ifma_ams52_p521_ntimes(&(R), (A), (N)) + +IPP_OWN_DEFN(void, ifma_aminv52_n521, (fe521 pr[], const fe521 z)) +{ + Ipp32u i; + fe521 n521r1_r; + + FE521_LOADU(n521r1_r, n521_r); + + // pwr_z_Tbl[i][] = z^i, i=0,..,15 + __ALIGN64 fe521 pwr_z_Tbl[16]; + __ALIGN64 fe521 lexp; + + FE521_COPY(pwr_z_Tbl[0], n521r1_r); + FE521_COPY(pwr_z_Tbl[1], z); + + for (i = 2u; i < 16u; i += 2u) { + sqr(pwr_z_Tbl[i], pwr_z_Tbl[i / 2u]); + mul(pwr_z_Tbl[i + 1u], pwr_z_Tbl[i], z); + } + + // pwr = (n521-2) in big endian + const Ipp8u pwr[] = "\x1\xFF" + "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" + "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" + "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" + "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFA" + "\x51\x86\x87\x83\xBF\x2F\x96\x6B" + "\x7F\xCC\x01\x48\xF7\x09\xA5\xD0" + "\x3B\xB5\xC9\xB8\x89\x9C\x47\xAE" + "\xBB\x6F\xB7\x1E\x91\x38\x64\x07"; + + /* + // process 25 low bytes of the exponent: :FA 51 86 ... 64 07" + */ + /* init result */ + FE521_COPY(lexp, n521r1_r); + + for (i = 33u; i < sizeof(pwr) - 1u; ++i) { + const int v = pwr[i]; + const int hi = (v >> 4) & 0xF; + const int lo = v & 0xF; + + sqr(lexp, lexp); + sqr(lexp, lexp); + sqr(lexp, lexp); + sqr(lexp, lexp); + if (hi) + mul(lexp, lexp, pwr_z_Tbl[hi]); + sqr(lexp, lexp); + sqr(lexp, lexp); + sqr(lexp, lexp); + sqr(lexp, lexp); + if (lo) + mul(lexp, lexp, pwr_z_Tbl[lo]); + } + + /* + // process high part of the exponent: "0x1 0xffffffffffffffff 0xffffffffffffffff 0xffffffffffffffff 0xffffffffffffffff" + */ + fe521 u, v; + FE521_SET(u) = FE521_SET(v) = m256_setzero_i64(); + + FE521_COPY(u, pwr_z_Tbl[15]); /* u = z^0xF */ + /**/ + sqr_ntimes(v, u, 4); /* v = (z^0xF)^(2^4) = z^(0xF0) */ + mul(u, v, u); /* u = z^0xF0 * z^(0xF) = z^(0xFF) */ + /**/ + sqr_ntimes(v, u, 8); /* v = (z^0xFF)^(2^8) = z^(0xFF00) */ + mul(u, v, u); /* u = z^0xFF00 * z^(0xFF) = z^(0xFFFF) */ + /**/ + sqr_ntimes(v, u, 16); /* v = (z^0xFFFF)^(2^16) = z^(0xFFFF0000) */ + mul(u, v, u); /* u = z^0xFFFF0000 * z^(0xFFFF) = z^(0xFFFFFFFF) */ + /**/ + sqr_ntimes(v, u, 32); /* v = (z^0xFFFFFFFF)^(2^32) = z^(0xFFFFFFFF00000000) */ + mul(u, v, u); /* u = z^0xFFFFFFFF00000000 * z^(0xFFFFFFFF) = z^(0xFFFFFFFFFFFFFFFF) */ + /**/ + sqr_ntimes(v, z, 64); /**/ + mul(v, v, u); /* v = z^(0x1.FFFFFFFFFFFFFFFF) */ + /**/ + sqr_ntimes(v, v, 64); /**/ + mul(v, v, u); /* v = z^(0x1.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF) */ + /**/ + sqr_ntimes(v, v, 64); /**/ + mul(v, v, u); /* v = z^(0x1.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF) */ + /**/ + sqr_ntimes(v, v, 64); /**/ + mul(v, v, u); /* v = z^(0x1.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF) */ + + /* combine low and high results */ + sqr_ntimes(v, v, 64 * 4 + 8); /* u = z^(0x1.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.00.0000000000000000.0000000000000000.0000000000000000.0000000000000000) */ + mul(*pr, v, lexp); /* r = z^(0x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFA.51868783BF2F966B.7FCC0148F709A5D0.3BB5C9B8899C47AE.BB6FB71E91386407) */ + return; +} + +#undef mul +#undef sqr +#undef sqr_ntimes + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_n521.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_n521.h new file mode 100644 index 0000000..027658b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_n521.h @@ -0,0 +1,79 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#ifndef IFMA_ARITH_N521_H +#define IFMA_ARITH_N521_H + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ifma_defs_p521.h" + +/** + * \brief + * R = (A * B) - in domain n521r1 + * \param[out] pr value no normalization + * \param[in] a first value (in radix 2^52) + * \param[in] b second value (in radix 2^52) + */ +IPP_OWN_DECL(void, ifma_amm52_n521, (fe521 pr[], const fe521 a, const fe521 b)) + +/** + * \brief + * R = (A + B) mod n521r1 + * \param[out] pr value (in radix 2^52) + * \param[in] a first value (in radix 2^52) + * \param[in] b second value (in radix 2^52) + */ +IPP_OWN_DECL(void, ifma_add52_n521, (fe521 pr[], const fe521 a, const fe521 b)) + +/** + * \brief + * to Montgomery domain + * \param[out] pr value (in radix 2^52) + * \param[in] a value (in radix 2^52) + */ +IPP_OWN_DECL(void, ifma_tomont52_n521, (fe521 pr[], const fe521 a)) + +/** + * \brief + * fast reduction order n521r1 + * \param[out] pr value (in radix 2^52) + * \param[in] a value (in radix 2^52) + */ +IPP_OWN_DECL(void, ifma_fastred52_n521, (fe521 pr[], const fe521 a)) + +/** + * \brief + * from Montgomery domain + * \param[out] pr value (in radix 2^52) + * \param[in] a value (in radix 2^52) + */ +IPP_OWN_DECL(void, ifma_frommont52_n521, (fe521 pr[], const fe521 a)) + +/** + * \brief + * inverse R = 1/z + * \param[out] pr value (in radix 2^52) + * \param[in] z value (in radix 2^52) + */ +IPP_OWN_DECL(void, ifma_aminv52_n521, (fe521 pr[], const fe521 z)) + +#endif // (_IPP32E >= _IPP32E_K1) + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_p256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_p256.c new file mode 100644 index 0000000..a264ee4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_p256.c @@ -0,0 +1,443 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ifma_arith_p256.h" +#include "ifma_defs.h" + +#define LEN52 (5 + 3) /* 5 digits + 3 zero padding */ + +/* Modulus p256r1 */ +static const __ALIGN64 Ipp64u p256_x1[LEN52] = { + 0x000fffffffffffff, 0x00000fffffffffff, 0x0000000000000000, 0x0000001000000000, 0x0000ffffffff0000, 0x0, 0x0, 0x0 +}; + +/* Scaled modulus 4*p256r1 */ +static const __ALIGN64 Ipp64u p256_x4[LEN52] = { + 0x000ffffffffffffc, 0x00003fffffffffff, 0x0000000000000000, 0x0000004000000000, 0x0003fffffffc0000, 0x0, 0x0, 0x0 +}; + +/* To Montgomery domain conversion constant + * + * The extended Montgomery domain here with R = 2^(6*52) instead of 2^(5*62) is + * choosen to use NRMM field multiplication routine in the sequence of EC group + * operations (point multiplication on scalar / addition) without modulo + * reductions in the field addition (see Chapter 4 in "Enhanced Montgomery + * Multiplication" DOI:10.1155/2008/583926). According to the paper the chosen + * |s| in NRMM^s(a,b) shall be equal to n + 2*kmax, with kmax = 4 for EC group + * operations, so s = 264. 5 digits capacity is only 260 bits, so purely IFMA-based + * implementation shall use 6 digits. + * + * rr = 2^((52*6))*2) mod p256 + */ +static const __ALIGN64 Ipp64u p256_rr6[LEN52] = { + 0x0002fffffffdffff, 0x0000100050000000, 0x000ffd0000000500, 0x0000000fff9fffff, 0x0000fff9fffefffe, 0x0, 0x0, 0x0 +}; + +#if 0 +/* + * rr = 2^((52*5))*2) mod p256 + */ +static const __ALIGN64 Ipp64u p256_rr5[LEN52] = { + 0x0000000000000300, 0x000ffffffff00000, 0x000ffffefffffffb, 0x000fdfffffffffff, 0x0000000004ffffff, 0x0, 0x0, 0x0 +}; +#endif + +static const __ALIGN64 Ipp64u ones[LEN52] = { + 1, 0, 0, 0, 0, 0, 0, 0 +}; + +#define MUL_RED_ROUND(R, A, B, IDX) \ + { \ + const m512 Bi = permutexvar_i8(idx_b##IDX, (B)); \ + m512 Rlo = madd52lo_i64(zero, (A), Bi); \ + m512 tmp = madd52hi_i64(zero, (A), Bi); \ + \ + (R) = add_i64((R), Rlo); \ + /* broadcast R[0] */ \ + const m512 R0 = permutexvar_i8(idx_b0, (R)); \ + (R) = madd52lo_i64((R), M, R0); \ + tmp = madd52hi_i64(tmp, M, R0); \ + /* shift */ \ + const m512 carry = maskz_srli_i64(maskone, (R), DIGIT_SIZE); \ + tmp = add_i64(tmp, carry); \ + (R) = maskz_permutexvar_i8(mask_sr64, idx_sr64, (R)); \ + /* hi */ \ + (R) = add_i64((R), tmp); \ + } + +/* R = (A*B) - no normalization radix 52 */ +IPP_OWN_DEFN(m512, ifma_amm52_p256, (const m512 a, const m512 b)) +{ + const m512 M = loadu_i64(p256_x1); + const m512 zero = setzero_i64(); + const mask8 maskone = 0x1; + + /* Index broadcast */ + const m512 idx_b0 = set_i64(REPL8(0x0706050403020100)); // 7, 6, 5, 4, 3, 2, 1, 0 + const m512 idx_b1 = set_i64(REPL8(0x0f0e0d0c0b0a0908)); // 15, 14, 13, 12, 11, 10, 9, 8 + const m512 idx_b2 = set_i64(REPL8(0x1716151413121110)); // 23, 22, 21, 20, 19, 18, 17, 16 + const m512 idx_b3 = set_i64(REPL8(0x1f1e1d1c1b1a1918)); // 31, 30, 29, 28, 27, 26, 25, 24 + const m512 idx_b4 = set_i64(REPL8(0x2726252423222120)); // 39, 38, 37, 36, 35, 34, 33, 32 + const m512 idx_b5 = set_i64(REPL8(0x2f2e2d2c2b2a2928)); // 47, 46, 45, 44, 43, 42, 41, 40 + + /* Mask to shift zmm register right on 64-bit */ + const mask64 mask_sr64 = 0x00ffffffffffffff; + const m512 idx_sr64 = set_i64(0x0, // 0, 0, 0, 0, 0, 0, 0, 0 + 0x3f3e3d3c3b3a3938, // 63, 62, 61, 60, 59, 58, 57, 56 + 0x3736353433323130, // 55, 54, 53, 52, 51, 50, 49, 48 + 0x2f2e2d2c2b2a2928, // 47, 46, 45, 44, 43, 42, 41, 40 + 0x2726252423222120, // 39, 38, 37, 36, 35, 34, 33, 32 + 0x1f1e1d1c1b1a1918, // 31, 30, 29, 28, 27, 26, 25, 24 + 0x1716151413121110, // 23, 22, 21, 20, 19, 18, 17, 16 + 0x0f0e0d0c0b0a0908); // 15, 14, 13, 12, 11, 10, 9, 8 + + m512 r = setzero_i64(); + + MUL_RED_ROUND(r, a, b, 0) + MUL_RED_ROUND(r, a, b, 1) + MUL_RED_ROUND(r, a, b, 2) + MUL_RED_ROUND(r, a, b, 3) + MUL_RED_ROUND(r, a, b, 4) + MUL_RED_ROUND(r, a, b, 5) + + return r; +} + +IPP_OWN_DEFN(void, ifma_amm52_dual_p256, (m512 * r1, const m512 a1, const m512 b1, m512 *r2, const m512 a2, const m512 b2)) +{ + const m512 M = loadu_i64(p256_x1); + const m512 zero = setzero_i64(); + const mask8 maskone = 0x1; + + /* Index broadcast */ + const m512 idx_b0 = set_i64(REPL8(0x0706050403020100)); // 7, 6, 5, 4, 3, 2, 1, 0 + const m512 idx_b1 = set_i64(REPL8(0x0f0e0d0c0b0a0908)); // 15, 14, 13, 12, 11, 10, 9, 8 + const m512 idx_b2 = set_i64(REPL8(0x1716151413121110)); // 23, 22, 21, 20, 19, 18, 17, 16 + const m512 idx_b3 = set_i64(REPL8(0x1f1e1d1c1b1a1918)); // 31, 30, 29, 28, 27, 26, 25, 24 + const m512 idx_b4 = set_i64(REPL8(0x2726252423222120)); // 39, 38, 37, 36, 35, 34, 33, 32 + const m512 idx_b5 = set_i64(REPL8(0x2f2e2d2c2b2a2928)); // 47, 46, 45, 44, 43, 42, 41, 40 + + /* Mask to shift zmm register right on 64-bit */ + const mask64 mask_sr64 = 0x00ffffffffffffff; + const m512 idx_sr64 = set_i64(0x0, // 0, 0, 0, 0, 0, 0, 0, 0 + 0x3f3e3d3c3b3a3938, // 63, 62, 61, 60, 59, 58, 57, 56 + 0x3736353433323130, // 55, 54, 53, 52, 51, 50, 49, 48 + 0x2f2e2d2c2b2a2928, // 47, 46, 45, 44, 43, 42, 41, 40 + 0x2726252423222120, // 39, 38, 37, 36, 35, 34, 33, 32 + 0x1f1e1d1c1b1a1918, // 31, 30, 29, 28, 27, 26, 25, 24 + 0x1716151413121110, // 23, 22, 21, 20, 19, 18, 17, 16 + 0x0f0e0d0c0b0a0908); // 15, 14, 13, 12, 11, 10, 9, 8 + + m512 t1 = setzero_i64(); + m512 t2 = setzero_i64(); + + MUL_RED_ROUND(t1, a1, b1, 0) + MUL_RED_ROUND(t2, a2, b2, 0) + + MUL_RED_ROUND(t1, a1, b1, 1) + MUL_RED_ROUND(t2, a2, b2, 1) + + MUL_RED_ROUND(t1, a1, b1, 2) + MUL_RED_ROUND(t2, a2, b2, 2) + + MUL_RED_ROUND(t1, a1, b1, 3) + MUL_RED_ROUND(t2, a2, b2, 3) + + MUL_RED_ROUND(t1, a1, b1, 4) + MUL_RED_ROUND(t2, a2, b2, 4) + + MUL_RED_ROUND(t1, a1, b1, 5) + MUL_RED_ROUND(t2, a2, b2, 5) + + *r1 = t1; + *r2 = t2; +} + +/* R = (A*B) with norm */ +__INLINE m512 ifma_amm52_p256_norm(const m512 a, const m512 b) +{ + m512 r = ifma_amm52_p256(a, b); + return ifma_lnorm52(r); +} + +/* R = (A*A) with norm */ +__INLINE m512 ifma_ams52_p256_norm(const m512 a) +{ + return ifma_amm52_p256_norm(a, a); +} + +/* R = (A/2) */ +IPP_OWN_DEFN(m512, ifma_half52_p256, (const m512 a)) +{ + const m512 M = loadu_i64(p256_x1); + const m512 zero = setzero_i64(); + const m512 one = set1_i64(1LL); + + const mask8 is_last_one = cmp_i64_mask(and_i64(a, one), zero, _MM_CMPINT_EQ); + const mask8 mask = (mask8)((is_last_one & 1) - 1); + + const mask64 mask_shift = 0x00ffffffffffffff; + const m512 idx_shift = set_i64(0x0, // 0, 0, 0, 0, 0, 0, 0, 0 + 0x3f3e3d3c3b3a3938, // 63, 62, 61, 60, 59, 58, 57, 56 + 0x3736353433323130, // 55, 54, 53, 52, 51, 50, 49, 48 + 0x2f2e2d2c2b2a2928, // 47, 46, 45, 44, 43, 42, 41, 40 + 0x2726252423222120, // 39, 38, 37, 36, 35, 34, 33, 32 + 0x1f1e1d1c1b1a1918, // 31, 30, 29, 28, 27, 26, 25, 24 + 0x1716151413121110, // 23, 22, 21, 20, 19, 18, 17, 16 + 0x0f0e0d0c0b0a0908); // 15, 14, 13, 12, 11, 10, 9, 8 + + m512 r = mask_add_i64(a, mask, a, M); + r = ifma_lnorm52(r); + + m512 leftBit = maskz_permutexvar_i8(mask_shift, idx_shift, and_i64(r, one)); + leftBit = slli_i64(leftBit, DIGIT_SIZE - 1); + r = srli_i64(r, 1); + r = add_i64(r, leftBit); + + return r; +} + +IPP_OWN_DEFN(m512, ifma_neg52_p256, (const m512 a)) +{ + const m512 M4 = loadu_i64(p256_x4); + + /* a == 0 ? 0xFF : 0 */ + const mask8 mask_zero = is_zero_i64(a); + + /* r = 4*p - a */ + m512 r = mask_sub_i64(a, ~mask_zero, M4, a); + r = ifma_norm52(r); + return r; +} + + +static m512 mod_reduction_p256(const m512 a) +{ + const m512 M = loadu_i64(p256_x1); + const m512 zero = setzero_i64(); + + /* r = a - M */ + m512 r = sub_i64(a, M); + r = ifma_norm52(r); + + /* 1 < 0 */ + const mask8 lt = cmp_i64_mask(zero, srli_i64(r, DIGIT_SIZE - 1), _MM_CMPINT_LT); + const mask8 mask = check_bit(lt, 4); // check sign in 5th digit + + /* maks != 0 ? a : r */ + r = mask_mov_i64(r, mask, a); + return r; +} + +/* to Montgomery domain */ +IPP_OWN_DEFN(m512, ifma_tomont52_p256, (const m512 a)) +{ + return ifma_amm52_p256_norm(a, loadu_i64(p256_rr6)); +} + +/* from Montgomery domain */ +IPP_OWN_DEFN(m512, ifma_frommont52_p256, (const m512 a)) +{ + m512 r = ifma_amm52_p256_norm(a, loadu_i64(ones)); + r = mod_reduction_p256(r); + return r; +} + +#define sqr(R, A) (R) = ifma_ams52_p256_norm((A)) +#define mul(R, A, B) (R) = ifma_amm52_p256_norm((A), (B)); + +__INLINE m512 ifma_ams52_p256_ntimes(m512 a, Ipp32s n) +{ + for (; n > 0; --n) + sqr(a, a); + return a; +} + +#define sqr_ntimes(R, A, N) (R) = ifma_ams52_p256_ntimes((A), (N)) + +/* R = 1/z */ +IPP_OWN_DEFN(m512, ifma_aminv52_p256, (const m512 z)) +{ + __ALIGN64 m512 tmp1; + __ALIGN64 m512 tmp2; + + /* Each eI holds z^(2^I-1) */ + __ALIGN64 m512 e2; + __ALIGN64 m512 e4; + __ALIGN64 m512 e8; + __ALIGN64 m512 e16; + __ALIGN64 m512 e32; + __ALIGN64 m512 e64; + + /* tmp1 = z^(2^1) */ + sqr(tmp1, z); + /* e2 = tmp1 = tmp1 * z = z^(2^2 - 2^0) */ + mul(tmp1, tmp1, z); + e2 = tmp1; + + /* tmp1 = tmp1^2 = z^(2^2 - 2^0)*2 = z^(2^3 - 2^1) */ + sqr(tmp1, tmp1); + /* tmp1 = tmp1^2 = z^(2^3 - 2^1)*2 = z^(2^4 - 2^2) */ + sqr(tmp1, tmp1); + /* e4 = tmp1 = tmp1*e2 = z^(2^4 - 2^2) * z^(2^2 - 2^0) = z^(2^4 - 2^2 + 2^2 - 2^0) = z^(2^4 - 2^0)*/ + mul(tmp1, tmp1, e2); + e4 = tmp1; + + /* tmp1 = tmp1^2 = z^(2^4 - 2^0)*2 = z^(2^5 - 2^1) */ + sqr(tmp1, tmp1); + /* tmp1 = tmp1^2 = z^(2^5 - 2^1)*2 = z^(2^6 - 2^2) */ + sqr(tmp1, tmp1); + /* tmp1 = tmp1^2 = z^(2^6 - 2^2)*2 = z^(2^7 - 2^3) */ + sqr(tmp1, tmp1); + /* tmp1 = tmp1^2 = z^(2^7 - 2^3)*2 = z^(2^8 - 2^4) */ + sqr(tmp1, tmp1); + /* e8 = tmp1 = tmp1*e4 = z^(2^8 - 2^4) * z^(2^4 - 2^0) = z^(2^8 - 2^4 + 2^4 - 2^0) = z^(2^8 - 2^0)*/ + mul(tmp1, tmp1, e4); + e8 = tmp1; + + /* tmp1 = tmp1^(2^8) = z^(2^8 - 2^0)*2^8 = z^(2^16 - 2^8) */ + sqr_ntimes(tmp1, tmp1, 8); + + /* e16 = tmp1 = tmp1*e8 = z^(2^16 - 2^8) * z^(2^8 - 2^0) = z^(2^16 - 2^8 + 2^8 - 2^0) = z^(2^16 - 2^0)*/ + mul(tmp1, tmp1, e8); + e16 = tmp1; + + /* tmp1 = tmp1^(2^16) = z^(2^16 - 2^0)*2^16 = z^(2^32 - 2^16) */ + sqr_ntimes(tmp1, tmp1, 16); + + /* e32 = tmp1 = tmp1*e16 = z^(2^32 - 2^16) * z^(2^16 - 2^0) = z^(2^32 - 2^16 + 2^16 - 2^0) = z^(2^32 - 2^0)*/ + mul(tmp1, tmp1, e16); + e32 = tmp1; + + /* e64 = tmp1 = tmp1^(2^32) = z^(2^32 - 2^0)*2^32 = z^(2^64 - 2^32) */ + sqr_ntimes(tmp1, tmp1, 32); + + e64 = tmp1; + /* tmp1 = tmp1*z = z^(2^64 - 2^32) * z = z^(2^64 - 2^32 + 2^0)*/ + mul(tmp1, tmp1, z); + + /* tmp1 = tmp1^(2^192) = z^(2^64 - 2^32 + 2^0)*2^192 = z^(2^256 - 2^224 + 2^192) */ + sqr_ntimes(tmp1, tmp1, 192); + + /* tmp2 = e64*e32 = z^(2^64 - 2^32) * z^(2^32 - 2^0) = z^(2^64 - 2^32 + 2^32 - 2^0) = z^(2^64 - 2^0)*/ + mul(tmp2, e64, e32); + + /* tmp2 = tmp2^(2^16) = z^(2^64 - 2^0)*2^16 = z^(2^80 - 2^16) */ + sqr_ntimes(tmp2, tmp2, 16); + + /* tmp2 = tmp2*e16 = z^(2^80 - 2^16) * z^(2^16 - 2^0) = z^(2^80 - 2^16 + 2^16 - 2^0) = z^(2^80 - 2^0)*/ + mul(tmp2, tmp2, e16); + + /* tmp2 = tmp2^(2^8) = z^(2^80 - 2^0)*2^8 = z^(2^88 - 2^8) */ + sqr_ntimes(tmp2, tmp2, 8); + + /* tmp2 = tmp2*e8 = z^(2^88 - 2^8) * z^(2^8 - 2^0) = z^(2^88 - 2^8 + 2^8 - 2^0) = z^(2^88 - 2^0)*/ + mul(tmp2, tmp2, e8); + + /* tmp2 = tmp2^(2^4) = z^(2^88 - 2^0)*2^4 = z^(2^92 - 2^4) */ + sqr_ntimes(tmp2, tmp2, 4); + + /* tmp2 = tmp2*e4 = z^(2^92 - 2^4) * z^(2^4 - 2^0) = z^(2^92 - 2^4 + 2^4 - 2^0) = z^(2^92 - 2^0)*/ + mul(tmp2, tmp2, e4); + + /* tmp2 = tmp2^2 = z^(2^92 - 2^0)*2^1 = z^(2^93 - 2^1) */ + sqr(tmp2, tmp2); + /* tmp2 = tmp2^2 = z^(2^93 - 2^1)*2^1 = z^(2^94 - 2^2) */ + sqr(tmp2, tmp2); + /* tmp2 = tmp2*e2 = z^(2^94 - 2^2) * z^(2^2 - 2^0) = z^(2^94 - 2^2 + 2^2 - 2^0) = z^(2^94 - 2^0)*/ + mul(tmp2, tmp2, e2); + /* tmp2 = tmp2^2 = z^(2^94 - 2^0)*2^1 = z^(2^95 - 2^1) */ + sqr(tmp2, tmp2); + /* tmp2 = tmp2^2 = z^(2^95 - 2^1)*2^1 = z^(2^96 - 2^2) */ + sqr(tmp2, tmp2); + /* tmp2 = tmp2*z = z^(2^96 - 2^2) * z = z^(2^96 - 2^2 + 1) = z^(2^96 - 3) */ + mul(tmp2, tmp2, z); + + /* r = tmp1*tmp2 = z^(2^256 - 2^224 + 2^192) * z^(2^96 - 3) = z^(2^256 - 2^224 + 2^192 + 2^96 - 3) */ + m512 r = setzero_i64(); + mul(r, tmp1, tmp2); + + return r; +} + +#undef sqr +#undef mul +#undef sqr_ntimes + +IPP_OWN_DEFN(m512, convert_radix_to_52x5, (const Ipp64u *arad64)) +{ + const m512 mask_rad52 = set1_i64(DIGIT_MASK); + + const m512 idx16 = set_i64( + 0x0019001800170016, // 25, 24, 23, 22, + 0x0016001500140013, // 22, 21, 20, 19, + 0x0013001200110010, // 19, 18, 17, 16, + 0x0010000f000e000d, // 16, 15, 14, 13, + 0x000c000b000a0009, // 12, 11, 10, 9, + 0x0009000800070006, // 9, 8, 7, 6, + 0x0006000500040003, // 6, 5, 4, 3, + 0x0003000200010000); // 3, 2, 1, 0 + + const __m512i shiftR = set_i64( + 12, 8, 4, 0, + 12, 8, 4, 0); + + m512 r = maskz_loadu_i64(0xf, arad64); // load 4 digits + r = permutexvar_i16(idx16, r); + r = srlv_i64(r, shiftR); + r = and_i64(mask_rad52, r); + return r; +} + +IPP_OWN_DEFN(void, convert_radix_to_64x4, (Ipp64u * rrad64, const m512 arad52)) +{ + const m512 shiftL = set_i64( + 4, 0, 4, 0, + 4, 0, 4, 0); + const m512 perm1 = set_i64( + 0x3f3f3f3f3f3f3f3f, // {63,63,63,63,63,63,63,63} + 0x3f3f3f3f3e3d3c3b, // {63,63,63,63,62,61,60,59} + 0x3737363534333231, // {55,55,54,53,52,51,50,49} + 0x302e2d2c2b2a2928, // {48,46,45,44,43,42,41,40} + 0x1f1f1f1f1f1f1e1d, // {31,31,31,31,31,31,30,29} + 0x1717171716151413, // {23,23,23,23,22,21,20,19} + 0x0f0f0f0e0d0c0b0a, // {15,15,15,14,13,12,11,10} + 0x0706050403020100); // { 7, 6, 5, 4, 3, 2, 1, 0} + + const m512 perm2 = set_i64( + 0x3f3f3f3f3f3f3f3f, // {63,63,63,63,63,63,63,63} + 0x3f3f3f3f3f3f3f3f, // {63,63,63,63,63,63,63,63} + 0x3a39383737373737, // {58,57,56,55,55,55,55,55} + 0x2727272727272726, // {39,39,39,39,39,39,39,38} + 0x2524232221201f1f, // {37,36,35,34,33,32,31,31} + 0x1c1b1a1918171717, // {28,27,26,25,24,23,23,23} + 0x1211100f0f0f0f0f, // {18,17,16,15,15,15,15,15} + 0x0908070707070707); // { 9, 8, 7, 7, 7, 7, 7, 7} + + m512 r = arad52; + r = sllv_i64(r, shiftL); + m512 T = permutexvar_i8(perm1, r); + r = permutexvar_i8(perm2, r); + r = or_i64(r, T); + mask_storeu_i64(rrad64, 0xf, r); // store 4 digits +} + +#endif // #if (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_p256.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_p256.h new file mode 100644 index 0000000..1193ef9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_p256.h @@ -0,0 +1,163 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#if !defined(_IFMA_ARITH_P256R1_H_) +#define _IFMA_ARITH_P256R1_H_ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ifma_alias_avx512.h" +#include + +/** + * \brief + * + * Montgomery multiplication + * + * a * b * r mod n, where r = 2^(6*52) mod n + * + * Note: final normalization to 2^52 radix is not performed and shall be + * handled separately. + * + * \param[in] a first value (in radix 2^52) + * \param[in] b second value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_amm52_p256, (const m512 a, const m512 b)) + +/** + * \brief + * + * Dual variant of ifma_amm52_p256() function. + * + * \param[out] r1 a1 * b1 mod p + * \param[in] a1 first value (in radix 2^52) + * \param[in] b1 second value (in radix 2^52) + * \param[out] r2 a2 * b2 mod p + * \param[in] a2 first value (in radix 2^52) + * \param[in] b2 second value (in radix 2^52) + */ +IPP_OWN_DECL(void, ifma_amm52_dual_p256, (m512 *r1, const m512 a1, const m512 b1, m512 *r2, const m512 a2, const m512 b2)) + +/** + * \brief + * + * A * A + * + * Note: final normalization to 2^52 radix is not performed and shall be + * handled separately. + * + * \param[in] a value (in radix 2^52) + */ +__INLINE IPP_OWN_DEFN(m512, ifma_ams52_p256, (const m512 a)) +{ + return ifma_amm52_p256(a, a); +} + +/** + * \brief + * + * Dual variant of ifma_ams52_p256() function. + * + * \param[out] r1 + * \param[in] a1 value (in radix 2^52) + * \param[out] r2 + * \param[in] a2 value (in radix 2^52) + */ +__INLINE IPP_OWN_DEFN(void, ifma_ams52_dual_p256, (m512 * r1, const m512 a1, m512 *r2, const m512 a2)) +{ + ifma_amm52_dual_p256(r1, a1, a1, r2, a2, a2); + return; +} + +/** + * \brief + * + * A / 2 + * + * \param[in] a value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_half52_p256, (const m512 a)) + +/** + * \brief + * + * Modular inverse modulo p. + * + * 1/z mod p + * + * \param[in] z value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_aminv52_p256, (const m512 z)) + +/** + * \brief + * + * (-A) mod p + * + * \param[in] a input value + */ +IPP_OWN_DECL(m512, ifma_neg52_p256, (const m512 a)) + +/** + * \brief + * + * Conversion to Montgomery domain modulo p. + * + * a * r mod p, where r = 2^(6*52) mod p + * + * \param[in] a value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_tomont52_p256, (const m512 a)) + +/** + * \brief + * + * Conversion from Montgomery domain modulo p. + * + * a mod p + * + * \param[in] a value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_frommont52_p256, (const m512 a)) + +/* ===================================================================================== */ +/* ===================================================================================== */ + +/** + * \brief + * + * Convert to radix 2^52 from radix 2^64. + * + * \param[in] pa pointer to array of 4 64-bit chunks + */ +IPP_OWN_DECL(m512, convert_radix_to_52x5, (const Ipp64u *pa)) + +/** + * \brief + * + * Convert to radix 2^64 from radix 2^52. + * + * \param[out] pr pointer to array of 4 64-bit chunks + * \param[in] a array of 5 64-bit chunks + */ +IPP_OWN_DECL(void, convert_radix_to_64x4, (Ipp64u * rrad64, const m512 arad52)) + +#endif // (_IPP32E >= _IPP32E_K1) + +#endif // _IFMA_ARITH_P256R1_H_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_p384.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_p384.c new file mode 100644 index 0000000..708cf84 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_p384.c @@ -0,0 +1,432 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ifma_arith_p384.h" +#include "ifma_defs.h" + +#define LEN52 (NUMBER_OF_DIGITS(384, 52)) + +/* p384r1: p = 2^384 - 2^128 - 2^96 + 2^32 - 1 */ +static const __ALIGN64 Ipp64u p384_x1[LEN52] = { + 0x00000000FFFFFFFF, 0x000FF00000000000, 0x000FFFFFFEFFFFFF, 0x000FFFFFFFFFFFFF, 0x000FFFFFFFFFFFFF, 0x000FFFFFFFFFFFFF, 0x000FFFFFFFFFFFFF, 0x00000000000FFFFF +}; + +/* modulus 4*p */ +static const __ALIGN64 Ipp64u p384_x4[LEN52] = { + 0x00000003FFFFFFFC, 0x000FC00000000000, 0x000FFFFFFBFFFFFF, 0x000FFFFFFFFFFFFF, 0x000FFFFFFFFFFFFF, 0x000FFFFFFFFFFFFF, 0x000FFFFFFFFFFFFF, 0x00000000003FFFFF +}; + +/* to Montgomery conversion constant + * rr = 2^((P384_LEN52*DIGIT_SIZE)*2) mod p384 + */ +static const __ALIGN64 Ipp64u p384_rr[LEN52] = { + 0x0000000000000000, 0x000FE00000001000, 0x0000000000FFFFFF, 0x0000000000000020, 0x0000FFFFFFFE0000, 0x0000000020000000, 0x0000000000000100, 0x0000000000000000 +}; + +static const __ALIGN64 Ipp64u ones[LEN52] = { + 1, 0, 0, 0, 0, 0, 0, 0 +}; + +/* R = (A/2) */ +IPP_OWN_DEFN(m512, ifma_half52_p384, (const m512 a)) +{ + const m512 M = loadu_i64(p384_x1); + const m512 zero = setzero_i64(); + const m512 one = set1_i64(1LL); + + const mask8 is_last_one = cmp_i64_mask(and_i64(a, one), zero, _MM_CMPINT_EQ); + const mask8 mask = (mask8)((is_last_one & 1) - 1); + + m512 r = mask_add_i64(a, mask, a, M); + r = ifma_lnorm52(r); + + /* 1-bit shift right */ + /* extract last bit + >> 64 */ + const mask64 mask_shift = 0x00ffffffffffffff; + const m512 idx_shift = set_i64(0x0, // 0, 0, 0, 0, 0, 0, 0, 0 + 0x3f3e3d3c3b3a3938, // 63, 62, 61, 60, 59, 58, 57, 56 + 0x3736353433323130, // 55, 54, 53, 52, 51, 50, 49, 48 + 0x2f2e2d2c2b2a2928, // 47, 46, 45, 44, 43, 42, 41, 40 + 0x2726252423222120, // 39, 38, 37, 36, 35, 34, 33, 32 + 0x1f1e1d1c1b1a1918, // 31, 30, 29, 28, 27, 26, 25, 24 + 0x1716151413121110, // 23, 22, 21, 20, 19, 18, 17, 16 + 0x0f0e0d0c0b0a0908); // 15, 14, 13, 12, 11, 10, 9, 8 + + m512 shift_right = maskz_permutexvar_i8(mask_shift, idx_shift, and_i64(r, one)); + /* set last bit is first bit (52 radix) */ + shift_right = slli_i64(shift_right, DIGIT_SIZE - 1); + /* join first new bit */ + r = srli_i64(r, 1); /* create slot by first bit 1111 -> 0111 */ + r = add_i64(r, shift_right); /* join first and other bit */ + + return r; +} + +IPP_OWN_DEFN(m512, ifma_neg52_p384, (const m512 a)) +{ + const m512 M4 = loadu_i64(p384_x4); + + /* a == 0 ? 0xFF : 0 */ + const mask8 mask_zero = is_zero_i64(a); + + /* r = 4*p - a */ + m512 r = mask_sub_i64(a, ~mask_zero, M4, a); + r = ifma_norm52(r); + return r; +} + +#define MULT_ROUND(R, A, B, IDX) \ + const m512 Bi##R##IDX = permutexvar_i8(idx_b##IDX, (B)); \ + const m512 amBiLo##R##IDX = madd52lo_i64(zero, (A), Bi##R##IDX); \ + m512 tr##R##IDX = madd52hi_i64(zero, (A), Bi##R##IDX); \ + { \ + /* low */ \ + (R) = add_i64((R), amBiLo##R##IDX); \ + const m512 R0 = permutexvar_i8(idx_b0, (R)); \ + const m512 u = add_i64(slli_i64(R0, 32), R0); \ + tr##R##IDX = madd52hi_i64(tr##R##IDX, M, u); \ + (R) = madd52lo_i64((R), M, u); \ + /* shift */ \ + const m512 carryone = maskz_srai_i64(maskone, (R), DIGIT_SIZE); \ + tr##R##IDX = add_i64(tr##R##IDX, carryone); \ + (R) = maskz_permutexvar_i8(mask_sr64, idx_sr64, (R)); \ + /* hi */ \ + (R) = add_i64((R), tr##R##IDX); \ + } + +/* gueron data */ +/* R = (A*B) - no normalization radix 52 */ +IPP_OWN_DEFN(m512, ifma_amm52_p384, (const m512 a, const m512 b)) +{ + const m512 M = loadu_i64(p384_x1); /* p */ + const m512 zero = setzero_i64(); + const mask8 maskone = 0x1; + + /* Index broadcast */ + const m512 idx_b0 = set_i64(REPL8(0x0706050403020100)); // 7, 6, 5, 4, 3, 2, 1, 0 + const m512 idx_b1 = set_i64(REPL8(0x0f0e0d0c0b0a0908)); // 15, 14, 13, 12, 11, 10, 9, 8 + const m512 idx_b2 = set_i64(REPL8(0x1716151413121110)); // 23, 22, 21, 20, 19, 18, 17, 16 + const m512 idx_b3 = set_i64(REPL8(0x1f1e1d1c1b1a1918)); // 31, 30, 29, 28, 27, 26, 25, 24 + const m512 idx_b4 = set_i64(REPL8(0x2726252423222120)); // 39, 38, 37, 36, 35, 34, 33, 32 + const m512 idx_b5 = set_i64(REPL8(0x2f2e2d2c2b2a2928)); // 47, 46, 45, 44, 43, 42, 41, 40 + const m512 idx_b6 = set_i64(REPL8(0x3736353433323130)); // 55, 54, 53, 52, 51, 50, 49, 48 + const m512 idx_b7 = set_i64(REPL8(0x3f3e3d3c3b3a3938)); // 63, 62, 61, 60, 59, 58, 57, 56 + + /* Mask to shift zmm register right on 64-bit */ + const mask64 mask_sr64 = 0x00ffffffffffffff; + const m512 idx_sr64 = set_i64(0x0, // 0, 0, 0, 0, 0, 0, 0, 0 + 0x3f3e3d3c3b3a3938, // 63, 62, 61, 60, 59, 58, 57, 56 + 0x3736353433323130, // 55, 54, 53, 52, 51, 50, 49, 48 + 0x2f2e2d2c2b2a2928, // 47, 46, 45, 44, 43, 42, 41, 40 + 0x2726252423222120, // 39, 38, 37, 36, 35, 34, 33, 32 + 0x1f1e1d1c1b1a1918, // 31, 30, 29, 28, 27, 26, 25, 24 + 0x1716151413121110, // 23, 22, 21, 20, 19, 18, 17, 16 + 0x0f0e0d0c0b0a0908); // 15, 14, 13, 12, 11, 10, 9, 8 + + m512 r = setzero_i64(); + /* P384 + * m' mod b = 2^32 + 1 + * + * Algorithm + * a[] b[] - input data ((in radix 2^52)) m[] - module p384 + * + * when u = R[0]*m' mod b + * 1) R = R + a[] * b[i] (lo) + * 2) R = R + m[] * u (lo) + * 3) R = R >> 64 + * 4) R = R + a[] * b[i] (hi) + * 5) R = R + m[] * u (hi) + */ + /* one round = O(32) */ + MULT_ROUND(r, a, b, 0) + MULT_ROUND(r, a, b, 1) + MULT_ROUND(r, a, b, 2) + MULT_ROUND(r, a, b, 3) + MULT_ROUND(r, a, b, 4) + MULT_ROUND(r, a, b, 5) + MULT_ROUND(r, a, b, 6) + MULT_ROUND(r, a, b, 7) + + return r; +} + +IPP_OWN_DEFN(void, ifma_amm52_dual_p384, (m512 * pr1, const m512 a1, const m512 b1, m512 *pr2, const m512 a2, const m512 b2)) +{ + const m512 M = loadu_i64(p384_x1); /* p */ + const m512 zero = setzero_i64(); + const mask8 maskone = 0x1; + + /* Index broadcast */ + const m512 idx_b0 = set_i64(REPL8(0x0706050403020100)); // 7, 6, 5, 4, 3, 2, 1, 0 + const m512 idx_b1 = set_i64(REPL8(0x0f0e0d0c0b0a0908)); // 15, 14, 13, 12, 11, 10, 9, 8 + const m512 idx_b2 = set_i64(REPL8(0x1716151413121110)); // 23, 22, 21, 20, 19, 18, 17, 16 + const m512 idx_b3 = set_i64(REPL8(0x1f1e1d1c1b1a1918)); // 31, 30, 29, 28, 27, 26, 25, 24 + const m512 idx_b4 = set_i64(REPL8(0x2726252423222120)); // 39, 38, 37, 36, 35, 34, 33, 32 + const m512 idx_b5 = set_i64(REPL8(0x2f2e2d2c2b2a2928)); // 47, 46, 45, 44, 43, 42, 41, 40 + const m512 idx_b6 = set_i64(REPL8(0x3736353433323130)); // 55, 54, 53, 52, 51, 50, 49, 48 + const m512 idx_b7 = set_i64(REPL8(0x3f3e3d3c3b3a3938)); // 63, 62, 61, 60, 59, 58, 57, 56 + + /* Mask to shift zmm register right on 64-bit */ + const mask64 mask_sr64 = 0x00ffffffffffffff; + const m512 idx_sr64 = set_i64(0x0, // 0, 0, 0, 0, 0, 0, 0, 0 + 0x3f3e3d3c3b3a3938, // 63, 62, 61, 60, 59, 58, 57, 56 + 0x3736353433323130, // 55, 54, 53, 52, 51, 50, 49, 48 + 0x2f2e2d2c2b2a2928, // 47, 46, 45, 44, 43, 42, 41, 40 + 0x2726252423222120, // 39, 38, 37, 36, 35, 34, 33, 32 + 0x1f1e1d1c1b1a1918, // 31, 30, 29, 28, 27, 26, 25, 24 + 0x1716151413121110, // 23, 22, 21, 20, 19, 18, 17, 16 + 0x0f0e0d0c0b0a0908); // 15, 14, 13, 12, 11, 10, 9, 8 + + m512 r1 = setzero_i64(); + m512 r2 = setzero_i64(); + /* P384 + * m' mod b = 2^32 + 1 + * + * Algorithm + * a[] b[] - input data ((in radix 2^52)) m[] - module p384 + * + * when u = R[0]*m' mod b + * 1) R = R + a[] * b[i] (lo) + * 2) R = R + m[] * u (lo) + * 3) R = R >> 64 + * 4) R = R + a[] * b[i] (hi) + * 5) R = R + m[] * u (hi) + */ + /* one round = O(32) */ + /* 0-1 */ + MULT_ROUND(r1, a1, b1, 0) + MULT_ROUND(r1, a1, b1, 1) + /* 0-1 */ + MULT_ROUND(r2, a2, b2, 0) + MULT_ROUND(r2, a2, b2, 1) + /* 2-3 */ + MULT_ROUND(r1, a1, b1, 2) + MULT_ROUND(r1, a1, b1, 3) + /* 2-3 */ + MULT_ROUND(r2, a2, b2, 2) + MULT_ROUND(r2, a2, b2, 3) + /* 4-5 */ + MULT_ROUND(r1, a1, b1, 4) + MULT_ROUND(r1, a1, b1, 5) + /* 4-5 */ + MULT_ROUND(r2, a2, b2, 4) + MULT_ROUND(r2, a2, b2, 5) + /* 6-7 */ + MULT_ROUND(r1, a1, b1, 6) + MULT_ROUND(r1, a1, b1, 7) + /* 6-7 */ + MULT_ROUND(r2, a2, b2, 6) + MULT_ROUND(r2, a2, b2, 7) + + /* set out */ + *pr1 = r1; + *pr2 = r2; + return; +} + +/* R = (A*B) with norm */ +__INLINE m512 ifma_amm52_p384_norm(const m512 a, const m512 b) +{ + m512 r = ifma_amm52_p384(a, b); + /* normalization */ + return ifma_lnorm52(r); +} + +/* R = (A*A) with norm */ +__INLINE m512 m512_sqr_norm(const m512 a) +{ + return ifma_amm52_p384_norm(a, a); +} + +/* to Montgomery domain */ +IPP_OWN_DEFN(m512, ifma_tomont52_p384, (const m512 a)) +{ + const m512 RR = loadu_i64(p384_rr); + return ifma_amm52_p384_norm(a, RR); +} + +static m512 mod_reduction_p384(const m512 a) +{ + const m512 M = loadu_i64(p384_x1); + const m512 zero = setzero_i64(); + + /* r = a - M */ + m512 r = sub_i64(a, M); + r = ifma_norm52(r); + + /* 1 < 0 */ + const mask8 lt = cmp_i64_mask(zero, srli_i64(r, DIGIT_SIZE - 1), _MM_CMPINT_LT); + const mask8 mask = check_bit(lt, 7); + + /* maks != 0 ? a : r */ + r = mask_mov_i64(r, mask, a); + return r; +} + +/* from Montgomery domain */ +IPP_OWN_DEFN(m512, ifma_frommont52_p384, (const m512 a)) +{ + const m512 one = loadu_i64(ones); + + /* from mont */ + m512 r = ifma_amm52_p384_norm(a, one); + r = mod_reduction_p384(r); + return r; +} + +#define sqr(R, A) (R) = m512_sqr_norm((A)) +#define mul(R, A, B) (R) = ifma_amm52_p384_norm((A), (B)); +#define mul_dual(R1, A1, B1, R2, A2, B2) \ + ifma_amm52_dual_p384(&(R1), (A1), (B1), &(R2), (A2), (B2)); \ + ifma_lnorm52_dual(&(R1), (R1), &(R2), (R2)) + +__INLINE m512 ifma_ams52_p384_ntimes(const m512 a, Ipp32s n) +{ + m512 r = a; + for (; n > 0; --n) + sqr(r, r); + return r; +} + +#define sqr_ntimes(R, A, N) (R) = ifma_ams52_p384_ntimes((A), (N)) + +/* R = 1/z */ +IPP_OWN_DEFN(m512, ifma_aminv52_p384, (const m512 z)) +{ + m512 u, v, zD, zE, zF; + u = v = zD = zE = zF = setzero_i64(); + + sqr(u, z); /* u = z^2 */ + mul(v, u, z); /* v = z^2 * z = z^3 */ + sqr_ntimes(zF, v, 2); /* zF = (z^3)^(2^2) = z^12 */ + /**/ + mul(zD, zF, z); /* zD = z^11 * z = z^xD */ + mul_dual(zE, zF, u, /* zE = z^11 * z^2 = z^xE */ + zF, zF, v); /* zF = z^11 * z^3 = z^xF */ + /**/ + sqr_ntimes(u, zF, 4); /* u = (z^xF)^(2^4) = z^xF0 */ + mul_dual(zD, u, zD, /* zD = z^xF0 * z^xD = z^xFD */ + zE, u, zE); /* zE = z^xF0 * z^xE = z^xFE */ + mul(zF, u, zF); /* zF = z^xF0 * z^xF = z^xFF */ + /**/ + sqr_ntimes(u, zF, 8); /* u = (z^xFF)^(2^8) = z^xFF00 */ + mul_dual(zD, u, zD, /* zD = z^xFF00 * z^xFD = z^xFFFD */ + zE, u, zE); /* zE = z^xFF00 * z^xFE = z^xFFFE */ + mul(zF, u, zF); /* zF = z^xFF00 * z^xFF = z^xFFFF */ + /**/ + sqr_ntimes(u, zF, 16); /* u = (z^xFFFF)^(2^16) = z^xFFFF0000 */ + mul_dual(zD, u, zD, /* zD = z^xFFFF0000 * z^xFFFD = z^xFFFFFFFD */ + zE, u, zE); /* zE = z^xFFFF0000 * z^xFFFE = z^xFFFFFFFE */ + mul(zF, u, zF); /* zF = z^xFFFF0000 * z^xFFFF = z^xFFFFFFFF */ + /**/ + sqr_ntimes(u, zF, 32); /* u = (z^xFFFFFFFF)^(2^32) = z^xFFFFFFFF00000000 */ + mul_dual(zE, u, zE, /* zE = z^xFFFFFFFF00000000 * z^xFFFFFFFE = z^xFFFFFFFFFFFFFFFE */ + zF, u, zF); /* zF = z^xFFFFFFFF00000000 * z^xFFFFFFFF = z^xFFFFFFFFFFFFFFFF */ + /* v = z^xFFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFF + = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF */ + sqr_ntimes(v, zF, 64); + mul(v, v, zF); + /* v = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFF + = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFFF */ + sqr_ntimes(v, v, 64); + mul(v, v, zF); + /* v = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFE + = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE */ + sqr_ntimes(v, v, 64); + mul(v, v, zE); + /* v = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE.0000000000000000 * z^xFFFFFFFF00000000 + = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE.FFFFFFFF00000000 */ + sqr_ntimes(v, v, 64); + mul(v, v, u); + /* r = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE.FFFFFFFF00000000.0000000000000000 * z^xFFFFFFFD + = z^xFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE.FFFFFFFF00000000.00000000FFFFFFFD */ + sqr_ntimes(v, v, 64); + m512 r = setzero_i64(); + mul(r, v, zD); + return r; +} + +#undef sqr +#undef mul +#undef mul_dual +#undef sqr_ntimes + +IPP_OWN_DEFN(m512, convert_radix_to_52x8, (const Ipp64u *arad64)) +{ + /* load mask to register */ + const mask8 mask_load = 0x3f; + const m512 mask_rad52 = set1_i64(DIGIT_MASK); + /* set data */ + const m512 idx16 = set_i64( + 0x0019001800170016, // 25, 24, 23, 22, + 0x0016001500140013, // 22, 21, 20, 19, + 0x0013001200110010, // 19, 18, 17, 16, + 0x0010000f000e000d, // 16, 15, 14, 13, + 0x000c000b000a0009, // 12, 11, 10, 9, + 0x0009000800070006, // 9, 8, 7, 6, + 0x0006000500040003, // 6, 5, 4, 3, + 0x0003000200010000); // 3, 2, 1, 0 + + const m512 shift_right = set_i64(12LL, 8LL, 4LL, 0LL, 12LL, 8LL, 4LL, 0LL); + + m512 r = maskz_loadu_i64(mask_load, arad64); + r = permutexvar_i16(idx16, r); + r = srlv_i64(r, shift_right); + r = and_i64(mask_rad52, r); + return r; +} + +IPP_OWN_DEFN(void, convert_radix_to_64x6, (Ipp64u * rrad64, const m512 arad52)) +{ + /* mask store */ + const mask8 mask_store = 0x3f; + const m512 shift_left = set_i64(4LL, 0LL, 4LL, 0LL, 4LL, 0LL, 4LL, 0LL); + const m512 idx_up8 = set_i64( + 0x3f3f3f3f3f3f3f3f, // {63,63,63,63,63,63,63,63} + 0x3f3f3f3f3e3d3c3b, // {63,63,63,63,62,61,60,59} + 0x3737363534333231, // {55,55,54,53,52,51,50,49} + 0x302e2d2c2b2a2928, // {48,46,45,44,43,42,41,40} + 0x1f1f1f1f1f1f1e1d, // {31,31,31,31,31,31,30,29} + 0x1717171716151413, // {23,23,23,23,22,21,20,19} + 0x0f0f0f0e0d0c0b0a, // {15,15,15,14,13,12,11,10} + 0x0706050403020100); // { 7, 6, 5, 4, 3, 2, 1, 0} + + const m512 idx_down8 = set_i64( + 0x3f3f3f3f3f3f3f3f, // {63,63,63,63,63,63,63,63} + 0x3f3f3f3f3f3f3f3f, // {63,63,63,63,63,63,63,63} + 0x3a39383737373737, // {58,57,56,55,55,55,55,55} + 0x2727272727272726, // {39,39,39,39,39,39,39,38} + 0x2524232221201f1f, // {37,36,35,34,33,32,31,31} + 0x1c1b1a1918171717, // {28,27,26,25,24,23,23,23} + 0x1211100f0f0f0f0f, // {18,17,16,15,15,15,15,15} + 0x0908070707070707); // { 9, 8, 7, 7, 7, 7, 7, 7} + + m512 r = arad52; + + r = sllv_i64(r, shift_left); + m512 T = permutexvar_i8(idx_up8, r); + r = permutexvar_i8(idx_down8, r); + r = or_i64(r, T); + mask_storeu_i64(rrad64, mask_store, r); + return; +} + +#endif // #if (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_p384.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_p384.h new file mode 100644 index 0000000..849c206 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_p384.h @@ -0,0 +1,162 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#if !defined(_IFMA_ARITH_P384R1_H_) +#define _IFMA_ARITH_P384R1_H_ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ifma_alias_avx512.h" +#include + +/** + * \brief + * + * Montgomery multiplication + * + * a * b * r mod n, where r = 2^(6*52) mod n + * + * Note: final normalization to 2^52 radix is not performed and shall be + * handled separately. + * + * \param[in] a first value (in radix 2^52) + * \param[in] b second value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_amm52_p384, (const m512 a, const m512 b)) + +/** + * \brief + * + * Dual variant of ifma_amm52_p384() function. + * + * \param[out] r1 + * \param[in] a1 first value (in radix 2^52) + * \param[in] b1 second value (in radix 2^52) + * \param[out] r2 + * \param[in] a2 first value (in radix 2^52) + * \param[in] b2 second value (in radix 2^52) + */ +IPP_OWN_DECL(void, ifma_amm52_dual_p384, (m512 * r1, const m512 a1, const m512 b1, m512 *r2, const m512 a2, const m512 b2)) + +/** + * \brief + * + * A * A + * + * Note: final normalization to 2^52 radix is not performed and shall be + * handled separately. + * + * \param[in] a value (in radix 2^52) + */ +__INLINE IPP_OWN_DEFN(m512, ifma_ams52_p384, (const m512 a)) +{ + return ifma_amm52_p384(a, a); +} + +/** + * \brief + * + * Dual variant of ifma_ams52_p384() function. + * + * \param[out] r1 + * \param[in] a1 value (in radix 2^52) + * \param[out] r2 + * \param[in] a2 value (in radix 2^52) + */ +__INLINE IPP_OWN_DEFN(void, ifma_ams52_dual_p384, (m512 * r1, const m512 a1, m512 *r2, const m512 a2)) +{ + ifma_amm52_dual_p384(r1, a1, a1, r2, a2, a2); + return; +} + +/** + * \brief + * + * A / 2 + * + * \param[in] a value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_half52_p384, (const m512 a)) + +/** + * \brief + * + * Modular inverse modulo p. + * + * 1/z mod p + * + * \param[in] z value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_aminv52_p384, (const m512 z)) + +/** + * \brief + * + * (-A) mod p + * + * \param[in] a input value + */ +IPP_OWN_DECL(m512, ifma_neg52_p384, (const m512 a)) + +/** + * \brief + * + * Conversion to Montgomery domain modulo p. + * + * a * r mod p, where r = 2^(6*52) mod p + * + * \param[in] a value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_tomont52_p384, (const m512 a)) + +/** + * \brief + * + * Conversion from Montgomery domain modulo p. + * + * a mod p + * + * \param[in] a value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_frommont52_p384, (const m512 a)) + +/* ===================================================================================== */ + +/** + * \brief + * + * Convert to radix 2^52 from radix 2^64. + * + * \param[in] pa pointer to array of 4 64-bit chunks + */ +IPP_OWN_DECL(m512, convert_radix_to_52x8, (const Ipp64u *pa)) + +/** + * \brief + * + * Convert to radix 2^64 from radix 2^52. + * + * \param[out] pr pointer to array of 4 64-bit chunks + * \param[in] a array of 5 64-bit chunks + */ +IPP_OWN_DECL(void, convert_radix_to_64x6, (Ipp64u * rrad64, const m512 arad52)) + +#endif // (_IPP32E >= _IPP32E_K1) + +#endif // _IFMA_ARITH_P384R1_H_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_p521.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_p521.c new file mode 100644 index 0000000..33640e6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_p521.c @@ -0,0 +1,1175 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ifma_arith_p521.h" + +#define NORM_LO_MID_ROUND(CARRY, RLO, RHI) \ + (CARRY) = m256_srai_i64((RLO), DIGIT_SIZE_52); \ + (RLO) = m256_and_i64((RLO), filt_rad52); \ + (CARRY) = m256_permutexvar_i8(idx8_shuffle, (CARRY)); \ + (RHI) = m256_mask_add_i64((RHI), 0x1, (RHI), (CARRY)); \ + (RLO) = m256_mask_add_i64((RLO), 0xE, (RLO), (CARRY)); + +#define NORM_HI_ROUND(CARRY, RHI) \ + (CARRY) = m256_maskz_srai_i64(0x7, (RHI), DIGIT_SIZE_52); \ + (CARRY) = m256_maskz_permutexvar_i8(0xFFFF00, idx8_shuffle, (CARRY)); \ + (RHI) = m256_add_i64(m256_and_i64((RHI), filt_rad52), (CARRY)); + +IPP_OWN_DEFN(void, ifma_norm52_p521, (fe521 pr[], const fe521 a)) +{ + const m256i filt_rad52 = m256_set1_i64(DIGIT_MASK_52); + + fe521 r; + FE521_COPY(r, a); + + const m256i idx8_shuffle = m256_set_i8(23, 22, 21, 20, 19, 18, 17, 16, + 15, 14, 13, 12, 11, 10, 9, 8, + 7, 6, 5, 4, 3, 2, 1, 0, + 31, 30, 29, 28, 27, 26, 25, 24); + m256i carry; + /* shift low carry */ + NORM_LO_MID_ROUND(carry, FE521_LO(r), FE521_MID(r)) + NORM_LO_MID_ROUND(carry, FE521_LO(r), FE521_MID(r)) + NORM_LO_MID_ROUND(carry, FE521_LO(r), FE521_MID(r)) + NORM_LO_MID_ROUND(carry, FE521_LO(r), FE521_MID(r)) + /* shift mid */ + NORM_LO_MID_ROUND(carry, FE521_MID(r), FE521_HI(r)) + NORM_LO_MID_ROUND(carry, FE521_MID(r), FE521_HI(r)) + NORM_LO_MID_ROUND(carry, FE521_MID(r), FE521_HI(r)) + NORM_LO_MID_ROUND(carry, FE521_MID(r), FE521_HI(r)) + /* shift hi */ + NORM_HI_ROUND(carry, FE521_HI(r)) + NORM_HI_ROUND(carry, FE521_HI(r)) + + FE521_COPY(*pr, r); + return; +} + +IPP_OWN_DEFN(void, ifma_norm52_dual_p521, + (fe521 pr1[], const fe521 a1, + fe521 pr2[], const fe521 a2)) +{ + const m256i filt_rad52 = m256_set1_i64(DIGIT_MASK_52); + + fe521 r1, r2; + FE521_COPY(r1, a1); + FE521_COPY(r2, a2); + + const m256i idx8_shuffle = m256_set_i8(23, 22, 21, 20, 19, 18, 17, 16, + 15, 14, 13, 12, 11, 10, 9, 8, + 7, 6, 5, 4, 3, 2, 1, 0, + 31, 30, 29, 28, 27, 26, 25, 24); + m256i carry_1; + m256i carry_2; + /* shift low carry */ + NORM_LO_MID_ROUND(carry_1, FE521_LO(r1), FE521_MID(r1)) + NORM_LO_MID_ROUND(carry_2, FE521_LO(r2), FE521_MID(r2)) + NORM_LO_MID_ROUND(carry_1, FE521_LO(r1), FE521_MID(r1)) + NORM_LO_MID_ROUND(carry_2, FE521_LO(r2), FE521_MID(r2)) + NORM_LO_MID_ROUND(carry_1, FE521_LO(r1), FE521_MID(r1)) + NORM_LO_MID_ROUND(carry_2, FE521_LO(r2), FE521_MID(r2)) + NORM_LO_MID_ROUND(carry_1, FE521_LO(r1), FE521_MID(r1)) + NORM_LO_MID_ROUND(carry_2, FE521_LO(r2), FE521_MID(r2)) + /* shift mid */ + NORM_LO_MID_ROUND(carry_1, FE521_MID(r1), FE521_HI(r1)) + NORM_LO_MID_ROUND(carry_2, FE521_MID(r2), FE521_HI(r2)) + NORM_LO_MID_ROUND(carry_1, FE521_MID(r1), FE521_HI(r1)) + NORM_LO_MID_ROUND(carry_2, FE521_MID(r2), FE521_HI(r2)) + NORM_LO_MID_ROUND(carry_1, FE521_MID(r1), FE521_HI(r1)) + NORM_LO_MID_ROUND(carry_2, FE521_MID(r2), FE521_HI(r2)) + NORM_LO_MID_ROUND(carry_1, FE521_MID(r1), FE521_HI(r1)) + NORM_LO_MID_ROUND(carry_2, FE521_MID(r2), FE521_HI(r2)) + /* shift hi */ + NORM_HI_ROUND(carry_1, FE521_HI(r1)) + NORM_HI_ROUND(carry_2, FE521_HI(r2)) + NORM_HI_ROUND(carry_1, FE521_HI(r1)) + NORM_HI_ROUND(carry_2, FE521_HI(r2)) + + FE521_COPY(*pr1, r1); + FE521_COPY(*pr2, r2); + return; +} + +#define ROUND_LOW_MID_LNORM(RLO, RHI) \ + { \ + m256i carry = m256_srai_i64((RLO), DIGIT_SIZE_52); \ + carry = m256_permutexvar_i8(idx8_shuffle, carry); \ + (RLO) = m256_and_i64((RLO), filt_rad52); \ + (RLO) = m256_mask_add_i64((RLO), 0xE, (RLO), carry); \ + (RHI) = m256_mask_add_i64((RHI), 0x1, (RHI), carry); \ + } + +#define ROUND_HI_LNORM(R) \ + { \ + m256i carry = m256_srai_i64((R), DIGIT_SIZE_52); \ + carry = m256_permutexvar_i8(idx8_shuffle, carry); \ + (R) = m256_and_i64((R), filt_rad52); \ + (R) = m256_mask_add_i64((R), 0xE, (R), carry); \ + } + +#define CREATE_MASK_ONE(OUT, R, MASK_CMP) \ + { \ + const int mk_lo = (int)(m256_cmp_i64_mask(filt_rad52, FE521_LO((R)), (MASK_CMP))); \ + const int mk_mid = (int)(m256_cmp_i64_mask(filt_rad52, FE521_MID((R)), (MASK_CMP))); \ + const int mk_hi = (int)(m256_cmp_i64_mask(filt_rad52, FE521_HI((R)), (MASK_CMP))); \ + \ + (OUT) = (mk_lo & 0xF) | ((mk_mid & 0xF) << 4) | ((mk_hi & 0xF) << 8); \ + } + +#define ADD_VALUE_BY_MASK(R, MASK, VAL) \ + { \ + FE521_LO((R)) = m256_mask_add_i64(FE521_LO((R)), (mask8)((MASK)&0xF), FE521_LO((R)), (VAL)); \ + FE521_MID((R)) = m256_mask_add_i64(FE521_MID((R)), (mask8)(((MASK) >> 4) & 0xF), FE521_MID((R)), (VAL)); \ + FE521_HI((R)) = m256_mask_add_i64(FE521_HI((R)), (mask8)(((MASK) >> 8) & 0x7), FE521_HI((R)), (VAL)); \ + } + +#define FILT_52(R) \ + { \ + FE521_LO((R)) = m256_and_i64(FE521_LO((R)), filt_rad52); \ + FE521_MID((R)) = m256_and_i64(FE521_MID((R)), filt_rad52); \ + FE521_HI((R)) = m256_and_i64(FE521_HI((R)), filt_rad52); \ + } + +IPP_OWN_DEFN(void, ifma_lnorm52_p521, (fe521 pr[], const fe521 a)) +{ + const m256i filt_rad52 = m256_set1_i64(DIGIT_MASK_52); + const m256i one = m256_set1_i64(1ULL); + const m256i idx8_shuffle = m256_set_i8(23, 22, 21, 20, 19, 18, 17, 16, + 15, 14, 13, 12, 11, 10, 9, 8, + 7, 6, 5, 4, 3, 2, 1, 0, + 31, 30, 29, 28, 27, 26, 25, 24); + + fe521 r; + FE521_COPY(r, a); + + /* standart step - first round normalization */ + /* low */ + ROUND_LOW_MID_LNORM(FE521_LO(r), FE521_MID(r)) + /* mid */ + ROUND_LOW_MID_LNORM(FE521_MID(r), FE521_HI(r)) + /* hi */ + ROUND_HI_LNORM(FE521_HI(r)) + + /* create mask add ONE(1) to slot */ + int k1, k2; + /* create k2 (r) == 0xF(13) */ + CREATE_MASK_ONE(k2, r, _MM_CMPINT_EQ) + /* create k1 (r) > 0xF(13) */ + CREATE_MASK_ONE(k1, r, _MM_CMPINT_LT) + + k1 = k2 + (k1 << 1); + k1 ^= k2; + + ADD_VALUE_BY_MASK(r, k1, one) + FILT_52(r) + + FE521_COPY(*pr, r); + return; +} + +IPP_OWN_DEFN(void, ifma_lnorm52_dual_p521, (fe521 pr1[], const fe521 a1, fe521 pr2[], const fe521 a2)) +{ + const m256i filt_rad52 = m256_set1_i64(DIGIT_MASK_52); + const m256i one = m256_set1_i64(1ULL); + const m256i idx8_shuffle = m256_set_i8(23, 22, 21, 20, 19, 18, 17, 16, + 15, 14, 13, 12, 11, 10, 9, 8, + 7, 6, 5, 4, 3, 2, 1, 0, + 31, 30, 29, 28, 27, 26, 25, 24); + fe521 r1, r2; + FE521_COPY(r1, a1); + FE521_COPY(r2, a2); + + /* standart step - first round normalization */ + /* low */ + ROUND_LOW_MID_LNORM(FE521_LO(r1), FE521_MID(r1)) + ROUND_LOW_MID_LNORM(FE521_LO(r2), FE521_MID(r2)) + /* mid */ + ROUND_LOW_MID_LNORM(FE521_MID(r1), FE521_HI(r1)) + ROUND_LOW_MID_LNORM(FE521_MID(r2), FE521_HI(r2)) + /* hi */ + ROUND_HI_LNORM(FE521_HI(r1)) + ROUND_HI_LNORM(FE521_HI(r2)) + + /* create mask add ONE(1) to slot */ + int k11, k21; + int k12, k22; + /* create k2 (r) == 0xF(13) */ + CREATE_MASK_ONE(k21, r1, _MM_CMPINT_EQ) + CREATE_MASK_ONE(k22, r2, _MM_CMPINT_EQ) + /* create k1 (r) > 0xF(13) */ + CREATE_MASK_ONE(k11, r1, _MM_CMPINT_LT) + CREATE_MASK_ONE(k12, r2, _MM_CMPINT_LT) + + k11 = k21 + (k11 << 1); + k11 ^= k21; + + k12 = k22 + (k12 << 1); + k12 ^= k22; + + ADD_VALUE_BY_MASK(r1, k11, one) + ADD_VALUE_BY_MASK(r2, k12, one) + + FILT_52(r1) + FILT_52(r2) + + FE521_COPY(*pr1, r1); + FE521_COPY(*pr2, r2); + return; +} + +#define group_madd52hi_i64(R, A, B, C) \ + FE521_LO(R) = m256_madd52hi_i64(FE521_LO(A), FE521_LO(B), (C)); \ + FE521_MID(R) = m256_madd52hi_i64(FE521_MID(A), FE521_MID(B), (C)); \ + FE521_HI(R) = m256_madd52hi_i64(FE521_HI(A), FE521_HI(B), (C)) + +#define group_madd52lo_i64(R, A, B, C) \ + FE521_LO(R) = m256_madd52lo_i64(FE521_LO(A), FE521_LO(B), (C)); \ + FE521_MID(R) = m256_madd52lo_i64(FE521_MID(A), FE521_MID(B), (C)); \ + FE521_HI(R) = m256_madd52lo_i64(FE521_HI(A), FE521_HI(B), (C)) + +#define MUL_ROUND(R, A, BI, BJ) \ + FE521_SET((R)) = m256_setzero_i64(); \ + group_madd52hi_i64((R), (R), (A), (BI)); \ + group_madd52lo_i64((R), (R), (A), (BJ)); + +#define REDUCTION(R, U, REDUCT) \ + (U) = m256_permutexvar_i8(idx0, FE521_LO(R)); \ + /* shift */ \ + FE521_LO(R) = m256_alignr_i64(FE521_MID(R), FE521_LO(R), 1); \ + FE521_MID(R) = m256_alignr_i64(FE521_HI(R), FE521_MID(R), 1); \ + FE521_HI(R) = m256_maskz_permutexvar_i8(mask_sr64, idx_sr64, FE521_HI(R)); \ + /* carry chunk 1 */ \ + FE521_LO(REDUCT) = m256_maskz_srai_i64(0x1, (U), DIGIT_SIZE_52); \ + /* carry chunk 2 */ \ + (U) = m256_and_i64((U), filt_rad52); \ + FE521_HI(REDUCT) = m256_maskz_add_i64(0x2, (U), (U)); \ + FE521_LO(R) = m256_add_i64(FE521_LO(R), FE521_LO(REDUCT)); \ + FE521_HI(R) = m256_add_i64(FE521_HI(R), FE521_HI(REDUCT)); + +IPP_OWN_DEFN(void, ifma_amm52_p521, (fe521 pr[], const fe521 a, const fe521 b)) +{ + const Ipp64s *pb_lo = (const Ipp64s *)(&(FE521_LO(b))); + const Ipp64s *pb_mid = (const Ipp64s *)(&(FE521_MID(b))); + const Ipp64s *pb_hi = (const Ipp64s *)(&(FE521_HI(b))); + + const m256i b0 = m256_set1_i64(pb_lo[0]); + const m256i b1 = m256_set1_i64(pb_lo[1]); + const m256i b2 = m256_set1_i64(pb_lo[2]); + const m256i b3 = m256_set1_i64(pb_lo[3]); + const m256i b4 = m256_set1_i64(pb_mid[0]); + const m256i b5 = m256_set1_i64(pb_mid[1]); + const m256i b6 = m256_set1_i64(pb_mid[2]); + const m256i b7 = m256_set1_i64(pb_mid[3]); + const m256i b8 = m256_set1_i64(pb_hi[0]); + const m256i b9 = m256_set1_i64(pb_hi[1]); + const m256i b10 = m256_set1_i64(pb_hi[2]); + + fe521 r0; + fe521 p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10; + + /* first */ + FE521_SET(r0) = m256_setzero_i64(); + group_madd52lo_i64(r0, r0, a, b0); + /* last */ + FE521_SET(p10) = m256_setzero_i64(); + group_madd52hi_i64(p10, p10, a, b10); + /* round mul */ + MUL_ROUND(p0, a, b0, b1) + MUL_ROUND(p1, a, b1, b2) + MUL_ROUND(p2, a, b2, b3) + MUL_ROUND(p3, a, b3, b4) + MUL_ROUND(p4, a, b4, b5) + MUL_ROUND(p5, a, b5, b6) + MUL_ROUND(p6, a, b6, b7) + MUL_ROUND(p7, a, b7, b8) + MUL_ROUND(p8, a, b8, b9) + MUL_ROUND(p9, a, b9, b10) + + const m256i filt_rad52 = m256_set1_i64(DIGIT_MASK_52); + const m256i idx0 = m256_set_i8(REPL4(7, 6, 5, 4, 3, 2, 1, 0)); /* B[ 0] -> chunk1[0] */ + /* chunk2 shift bit >> 64 */ + const mask32 mask_sr64 = 0x00FFFFFF; + const m256i idx_sr64 = m256_set_i8(0, 0, 0, 0, 0, 0, 0, 0, + 31, 30, 29, 28, 27, 26, 25, 24, + 23, 22, 21, 20, 19, 18, 17, 16, + 15, 14, 13, 12, 11, 10, 9, 8); + + fe521 reduct; + FE521_SET(reduct) = m256_setzero_i64(); + m256i u; + REDUCTION(r0, u, reduct) + + fe521_add_no_red(r0, r0, p0); + REDUCTION(r0, u, reduct) + + fe521_add_no_red(r0, r0, p1); + REDUCTION(r0, u, reduct) + + fe521_add_no_red(r0, r0, p2); + REDUCTION(r0, u, reduct) + + fe521_add_no_red(r0, r0, p3); + REDUCTION(r0, u, reduct) + + fe521_add_no_red(r0, r0, p4); + REDUCTION(r0, u, reduct) + + fe521_add_no_red(r0, r0, p5); + REDUCTION(r0, u, reduct) + + fe521_add_no_red(r0, r0, p6); + REDUCTION(r0, u, reduct) + + fe521_add_no_red(r0, r0, p7); + REDUCTION(r0, u, reduct) + + fe521_add_no_red(r0, r0, p8); + REDUCTION(r0, u, reduct) + + fe521_add_no_red(r0, r0, p9); + REDUCTION(r0, u, reduct) + + fe521_add_no_red(r0, r0, p10); + + FE521_COPY(*pr, r0); + return; +} + +#define MUL_LO_ROUND(R, A, \ + AI, MMI, MAI, \ + AJ, MMJ, MAJ) \ + { \ + fe521 tmp; \ + FE521_SET(R) = FE521_SET(tmp) = m256_setzero_i64(); \ + /* high */ \ + FE521_LO(R) = m256_maskz_madd52hi_i64((MMI), FE521_LO(R), FE521_LO(A), (AI)); \ + FE521_MID(R) = m256_madd52hi_i64(FE521_MID(R), FE521_MID(A), (AI)); \ + FE521_HI(R) = m256_madd52hi_i64(FE521_HI(R), FE521_HI(A), (AI)); \ + /* lo */ \ + FE521_LO(tmp) = m256_maskz_madd52lo_i64((MMJ), FE521_LO(tmp), FE521_LO(A), (AJ)); \ + FE521_MID(tmp) = m256_madd52lo_i64(FE521_MID(tmp), FE521_MID(A), (AJ)); \ + FE521_HI(tmp) = m256_madd52lo_i64(FE521_HI(tmp), FE521_HI(A), (AJ)); \ + /* double hi */ \ + FE521_LO(R) = m256_mask_add_i64(FE521_LO(R), (MAI), FE521_LO(R), FE521_LO(R)); \ + FE521_MID(R) = m256_add_i64(FE521_MID(R), FE521_MID(R)); \ + FE521_HI(R) = m256_add_i64(FE521_HI(R), FE521_HI(R)); \ + /* double lo */ \ + FE521_LO(tmp) = m256_mask_add_i64(FE521_LO(tmp), (MAJ), FE521_LO(tmp), FE521_LO(tmp)); \ + FE521_MID(tmp) = m256_add_i64(FE521_MID(tmp), FE521_MID(tmp)); \ + FE521_HI(tmp) = m256_add_i64(FE521_HI(tmp), FE521_HI(tmp)); \ + /* add lo + hi */ \ + FE521_LO(R) = m256_add_i64(FE521_LO(R), FE521_LO(tmp)); \ + FE521_MID(R) = m256_add_i64(FE521_MID(R), FE521_MID(tmp)); \ + FE521_HI(R) = m256_add_i64(FE521_HI(R), FE521_HI(tmp)); \ + } + +#define MUL_LO_LAST_ROUND(R, A, \ + AI, MMI, MAI, \ + AJ, MMJ) \ + { \ + fe521 tmp; \ + FE521_SET(R) = FE521_SET(tmp) = m256_setzero_i64(); \ + /* high */ \ + FE521_LO(R) = m256_maskz_madd52hi_i64((MMI), FE521_LO(R), FE521_LO(A), (AI)); \ + FE521_MID(R) = m256_madd52hi_i64(FE521_MID(R), FE521_MID(A), (AI)); \ + FE521_HI(R) = m256_madd52hi_i64(FE521_HI(R), FE521_HI(A), (AI)); \ + /* lo */ \ + FE521_LO(tmp) = m256_maskz_madd52lo_i64((MMJ), FE521_LO(tmp), FE521_LO(A), (AJ)); \ + FE521_MID(tmp) = m256_madd52lo_i64(FE521_MID(tmp), FE521_MID(A), (AJ)); \ + FE521_HI(tmp) = m256_madd52lo_i64(FE521_HI(tmp), FE521_HI(A), (AJ)); \ + /* double hi */ \ + FE521_LO(R) = m256_mask_add_i64(FE521_LO(R), (MAI), FE521_LO(R), FE521_LO(R)); \ + FE521_MID(R) = m256_add_i64(FE521_MID(R), FE521_MID(R)); \ + FE521_HI(R) = m256_add_i64(FE521_HI(R), FE521_HI(R)); \ + /* double lo */ \ + FE521_MID(tmp) = m256_add_i64(FE521_MID(tmp), FE521_MID(tmp)); \ + FE521_HI(tmp) = m256_add_i64(FE521_HI(tmp), FE521_HI(tmp)); \ + /* add lo + hi */ \ + FE521_LO(R) = m256_add_i64(FE521_LO(R), FE521_LO(tmp)); \ + FE521_MID(R) = m256_add_i64(FE521_MID(R), FE521_MID(tmp)); \ + FE521_HI(R) = m256_add_i64(FE521_HI(R), FE521_HI(tmp)); \ + } + +#define MUL_MID_FIRST_ROUND(R, A, \ + AI, MMI, \ + AJ, MAJ) \ + { \ + fe521 tmp; \ + FE521_SET(R) = FE521_MID(tmp) = FE521_HI(tmp) = m256_setzero_i64(); \ + /* high */ \ + FE521_LO(R) = m256_maskz_madd52hi_i64((MMI), FE521_LO(R), FE521_LO(A), (AI)); \ + FE521_MID(R) = m256_madd52hi_i64(FE521_MID(R), FE521_MID(A), (AI)); \ + FE521_HI(R) = m256_madd52hi_i64(FE521_HI(R), FE521_HI(A), (AI)); \ + /* lo */ \ + FE521_MID(tmp) = m256_madd52lo_i64(FE521_MID(tmp), FE521_MID(A), (AJ)); \ + FE521_HI(tmp) = m256_madd52lo_i64(FE521_HI(tmp), FE521_HI(A), (AJ)); \ + /* double hi */ \ + FE521_MID(R) = m256_add_i64(FE521_MID(R), FE521_MID(R)); \ + FE521_HI(R) = m256_add_i64(FE521_HI(R), FE521_HI(R)); \ + /* double lo */ \ + FE521_MID(tmp) = m256_mask_add_i64(FE521_MID(tmp), (MAJ), FE521_MID(tmp), FE521_MID(tmp)); \ + FE521_HI(tmp) = m256_add_i64(FE521_HI(tmp), FE521_HI(tmp)); \ + /* add lo + hi */ \ + FE521_MID(R) = m256_add_i64(FE521_MID(R), FE521_MID(tmp)); \ + FE521_HI(R) = m256_add_i64(FE521_HI(R), FE521_HI(tmp)); \ + } + +#define MUL_MID_ROUND(R, A, \ + AI, MMI, MAI, \ + AJ, MMJ, MAJ) \ + { \ + fe521 tmp; \ + FE521_MID(R) = FE521_HI(R) = FE521_MID(tmp) = FE521_HI(tmp) = m256_setzero_i64(); \ + /* high */ \ + FE521_MID(R) = m256_maskz_madd52hi_i64((MMI), FE521_MID(R), FE521_MID(A), (AI)); \ + FE521_HI(R) = m256_madd52hi_i64(FE521_HI(R), FE521_HI(A), (AI)); \ + /* lo */ \ + FE521_MID(tmp) = m256_maskz_madd52lo_i64((MMJ), FE521_MID(tmp), FE521_MID(A), (AJ)); \ + FE521_HI(tmp) = m256_madd52lo_i64(FE521_HI(tmp), FE521_HI(A), (AJ)); \ + /* double hi */ \ + FE521_MID(R) = m256_mask_add_i64(FE521_MID(R), (MAI), FE521_MID(R), FE521_MID(R)); \ + FE521_HI(R) = m256_add_i64(FE521_HI(R), FE521_HI(R)); \ + /* double lo */ \ + FE521_MID(tmp) = m256_mask_add_i64(FE521_MID(tmp), (MAJ), FE521_MID(tmp), FE521_MID(tmp)); \ + FE521_HI(tmp) = m256_add_i64(FE521_HI(tmp), FE521_HI(tmp)); \ + /* add lo + hi */ \ + FE521_MID(R) = m256_add_i64(FE521_MID(R), FE521_MID(tmp)); \ + FE521_HI(R) = m256_add_i64(FE521_HI(R), FE521_HI(tmp)); \ + } + +#define MUL_MID_LAST_ROUND(R, A, \ + AI, MMI, MAI, \ + AJ, MMJ) \ + { \ + fe521 tmp; \ + FE521_MID(R) = FE521_HI(R) = FE521_MID(tmp) = FE521_HI(tmp) = m256_setzero_i64(); \ + /* high */ \ + FE521_MID(R) = m256_maskz_madd52hi_i64((MMI), FE521_MID(R), FE521_MID(A), (AI)); \ + FE521_HI(R) = m256_madd52hi_i64(FE521_HI(R), FE521_HI(A), (AI)); \ + /* lo */ \ + FE521_MID(tmp) = m256_maskz_madd52lo_i64((MMJ), FE521_MID(tmp), FE521_MID(A), (AJ)); \ + FE521_HI(tmp) = m256_madd52lo_i64(FE521_HI(tmp), FE521_HI(A), (AJ)); \ + /* double hi */ \ + FE521_MID(R) = m256_mask_add_i64(FE521_MID(R), (MAI), FE521_MID(R), FE521_MID(R)); \ + FE521_HI(R) = m256_add_i64(FE521_HI(R), FE521_HI(R)); \ + /* double lo */ \ + FE521_HI(tmp) = m256_add_i64(FE521_HI(tmp), FE521_HI(tmp)); \ + /* add lo + hi */ \ + FE521_MID(R) = m256_add_i64(FE521_MID(R), FE521_MID(tmp)); \ + FE521_HI(R) = m256_add_i64(FE521_HI(R), FE521_HI(tmp)); \ + } + +#define MUL_HI_FIRST_ROUND(R, A, \ + AI, MMI, \ + AJ, MAJ) \ + { \ + fe521 tmp; \ + FE521_MID(R) = FE521_HI(R) = FE521_HI(tmp) = m256_setzero_i64(); \ + /* high */ \ + FE521_MID(R) = m256_maskz_madd52hi_i64((MMI), FE521_MID(R), FE521_MID(A), (AI)); \ + FE521_HI(R) = m256_madd52hi_i64(FE521_HI(R), FE521_HI(A), (AI)); \ + /* lo */ \ + FE521_HI(tmp) = m256_madd52lo_i64(FE521_HI(tmp), FE521_HI(A), (AJ)); \ + /* double hi */ \ + FE521_HI(R) = m256_add_i64(FE521_HI(R), FE521_HI(R)); \ + /* double lo */ \ + FE521_HI(tmp) = m256_mask_add_i64(FE521_HI(tmp), (MAJ), FE521_HI(tmp), FE521_HI(tmp)); \ + /* add lo + hi */ \ + FE521_HI(R) = m256_add_i64(FE521_HI(R), FE521_HI(tmp)); \ + } + +#define MUL_HI_ROUND(R, A, \ + AI, MMI, MAI, \ + AJ, MMJ, MAJ) \ + { \ + fe521 tmp; \ + FE521_HI(R) = FE521_HI(tmp) = m256_setzero_i64(); \ + /* high */ \ + FE521_HI(R) = m256_maskz_madd52hi_i64((MMI), FE521_HI(R), FE521_HI(A), (AI)); \ + /* lo */ \ + FE521_HI(tmp) = m256_maskz_madd52lo_i64((MMJ), FE521_HI(tmp), FE521_HI(A), (AJ)); \ + /* double hi */ \ + FE521_HI(R) = m256_mask_add_i64(FE521_HI(R), (MAI), FE521_HI(R), FE521_HI(R)); \ + /* double lo */ \ + FE521_HI(tmp) = m256_mask_add_i64(FE521_HI(tmp), (MAJ), FE521_HI(tmp), FE521_HI(tmp)); \ + /* add lo + hi */ \ + FE521_HI(R) = m256_add_i64(FE521_HI(R), FE521_HI(tmp)); \ + } + +#define ADD_LO(R, A, B) \ + FE521_LO(R) = m256_add_i64(FE521_LO(A), FE521_LO(B)); \ + FE521_MID(R) = m256_add_i64(FE521_MID(A), FE521_MID(B)); \ + FE521_HI(R) = m256_add_i64(FE521_HI(A), FE521_HI(B)); + +#define ADD_MID(R, A, B) \ + FE521_MID(R) = m256_add_i64(FE521_MID(A), FE521_MID(B)); \ + FE521_HI(R) = m256_add_i64(FE521_HI(A), FE521_HI(B)); + +#define ADD_HI(R, A, B) \ + FE521_HI(R) = m256_add_i64(FE521_HI(A), FE521_HI(B)); + +IPP_OWN_DEFN(void, ifma_ams52_p521, (fe521 pr[], const fe521 a)) +{ + const Ipp64s *pa_lo = (const Ipp64s *)(&(FE521_LO(a))); + const Ipp64s *pa_mid = (const Ipp64s *)(&(FE521_MID(a))); + const Ipp64s *pa_hi = (const Ipp64s *)(&(FE521_HI(a))); + + const m256i a0 = m256_set1_i64(pa_lo[0]); + const m256i a1 = m256_set1_i64(pa_lo[1]); + const m256i a2 = m256_set1_i64(pa_lo[2]); + const m256i a3 = m256_set1_i64(pa_lo[3]); + + const m256i a4 = m256_set1_i64(pa_mid[0]); + const m256i a5 = m256_set1_i64(pa_mid[1]); + const m256i a6 = m256_set1_i64(pa_mid[2]); + const m256i a7 = m256_set1_i64(pa_mid[3]); + + const m256i a8 = m256_set1_i64(pa_hi[0]); + const m256i a9 = m256_set1_i64(pa_hi[1]); + const m256i a10 = m256_set1_i64(pa_hi[2]); + + fe521 r0, p0, p1, p2, p3, p4, p5, p6, p7, p8, p9; + + /* start round */ + /* r0 = r0 + a*a[0](lo) */ + FE521_SET(r0) = m256_setzero_i64(); + FE521_LO(r0) = m256_madd52lo_i64(FE521_LO(r0), FE521_LO(a), a0); + FE521_MID(r0) = m256_madd52lo_i64(FE521_MID(r0), FE521_MID(a), a0); + FE521_HI(r0) = m256_madd52lo_i64(FE521_HI(r0), FE521_HI(a), a0); + /* r0 = r0 + r0 (no update r0[0]) */ + FE521_LO(r0) = m256_mask_add_i64(FE521_LO(r0), 0xE, FE521_LO(r0), FE521_LO(r0)); + FE521_MID(r0) = m256_add_i64(FE521_MID(r0), FE521_MID(r0)); + FE521_HI(r0) = m256_add_i64(FE521_HI(r0), FE521_HI(r0)); + /* + * 1111 = 0xF + * 1110 = 0xE + * 1100 = 0xC + * 1000 = 0x8 + * 0000 = 0x0 + */ + /* lower */ + /* p0 = p0 + a*a[0](hi) + a*a[1](lo) */ + MUL_LO_ROUND(/* R = */ p0, /* A = */ a, + /* AI = */ a0, /* MMI = */ 0xF, /* MAI = */ 0xE, + /* AJ = */ a1, /* MMJ = */ 0xE, /* MAJ = */ 0xC) + /* p1 = p1 + a*a[1](hi) + a*a[2](lo) */ + MUL_LO_ROUND(/* R = */ p1, /* A = */ a, + /* AI = */ a1, /* MMI = */ 0xE, /* MAI = */ 0xC, + /* AJ = */ a2, /* MMJ = */ 0xC, /* MAJ = */ 0x8) + /* p2 = p2 + a*a[2](hi) + a*a[3](lo) */ + MUL_LO_LAST_ROUND(/* R = */ p2, /* A = */ a, + /* AI = */ a2, /* MMI = */ 0xC, /* MAI = */ 0x8, + /* AJ = */ a3, /* MMJ = */ 0x8) + /* mid */ + /* p3 = p3 + a*a[3](hi) + a*a[4](lo) */ + MUL_MID_FIRST_ROUND(/* R = */ p3, /* A = */ a, + /* AI = */ a3, /* MMI = */ 0x8, + /* AJ = */ a4, /* MAJ = */ 0xE) + /* p4 = p4 + a*a[4](hi) + a*a[5](lo) */ + MUL_MID_ROUND(/* R = */ p4, /* A = */ a, + /* AI = */ a4, /* MMI = */ 0xF, /* MAI = */ 0xE, + /* AJ = */ a5, /* MMJ = */ 0xE, /* MAJ = */ 0xC) + /* p5 = p5 + a*a[5](hi) + a*a[6](lo) */ + MUL_MID_ROUND(/* R = */ p5, /* A = */ a, + /* AI = */ a5, /* MMI = */ 0xE, /* MAI = */ 0xC, + /* AJ = */ a6, /* MMJ = */ 0xC, /* MAJ = */ 0x8) + /* p6 = p6 + a*a[6](hi) + a*a[7](lo) */ + MUL_MID_LAST_ROUND(/* R = */ p6, /* A = */ a, + /* AI = */ a6, /* MMI = */ 0xC, /* MAI = */ 0x8, + /* AJ = */ a7, /* MMJ = */ 0x8) + /* high */ + /* p7 = p7 + a*a[7](hi) + a*a[8](lo) */ + MUL_HI_FIRST_ROUND(/* R = */ p7, /* A = */ a, + /* AI = */ a7, /* MMI = */ 0x8, + /* AJ = */ a8, /* MAJ = */ 0xE) + /* p8 = p8 + a*a[8](hi) + a*a[9](lo) */ + MUL_HI_ROUND(/* R = */ p8, /* A = */ a, + /* AI = */ a8, /* MMI = */ 0xF, /* MAI = */ 0xE, + /* AJ = */ a9, /* MMJ = */ 0xE, /* MAJ = */ 0xC) + /* p9 = p9 + a*a[9](hi) + a*a[10](lo) */ + MUL_HI_ROUND(/* R = */ p9, /* A = */ a, + /* AI = */ a9, /* MMI = */ 0xE, /* MAI = */ 0xC, + /* AJ = */ a10, /* MMJ = */ 0xC, /* MAJ = */ 0x8) + + fe521 reduct; + FE521_SET(reduct) = m256_setzero_i64(); + m256i u; + const m256i filt_rad52 = m256_set1_i64(DIGIT_MASK_52); + const m256i idx0 = m256_set_i8(REPL4(7, 6, 5, 4, 3, 2, 1, 0)); /* B[ 0] -> chunk1[0] */ + /* chunk2 shift bit >> 64 */ + const mask32 mask_sr64 = 0x00FFFFFF; + const m256i idx_sr64 = m256_set_i8(0, 0, 0, 0, 0, 0, 0, 0, + 31, 30, 29, 28, 27, 26, 25, 24, + 23, 22, 21, 20, 19, 18, 17, 16, + 15, 14, 13, 12, 11, 10, 9, 8); + REDUCTION(r0, u, reduct) + + ADD_LO(r0, r0, p0); + REDUCTION(r0, u, reduct) + + ADD_LO(r0, r0, p1); + REDUCTION(r0, u, reduct) + + ADD_LO(r0, r0, p2); + REDUCTION(r0, u, reduct) + + ADD_LO(r0, r0, p3); + REDUCTION(r0, u, reduct) + + ADD_MID(r0, r0, p4); + REDUCTION(r0, u, reduct) + + ADD_MID(r0, r0, p5); + REDUCTION(r0, u, reduct) + + ADD_MID(r0, r0, p6); + REDUCTION(r0, u, reduct) + + ADD_MID(r0, r0, p7); + REDUCTION(r0, u, reduct) + + ADD_HI(r0, r0, p8); + REDUCTION(r0, u, reduct) + + ADD_HI(r0, r0, p9); + REDUCTION(r0, u, reduct) + + FE521_COPY(*pr, r0); + return; +} + +IPP_OWN_DEFN(void, ifma_ams52_dual_p521, (fe521 pr1[], const fe521 a1, fe521 pr2[], const fe521 a2)) +{ + ifma_amm52_dual_p521(pr1, a1, a1, pr2, a2, a2); +} + +IPP_OWN_DEFN(void, ifma_amm52_dual_p521, + (fe521 pr1[], const fe521 a1, const fe521 b1, + fe521 pr2[], const fe521 a2, const fe521 b2)) +{ + const Ipp64s *pb_lo_1 = (const Ipp64s *)(&(FE521_LO(b1))); + const Ipp64s *pb_mid_1 = (const Ipp64s *)(&(FE521_MID(b1))); + const Ipp64s *pb_hi_1 = (const Ipp64s *)(&(FE521_HI(b1))); + const Ipp64s *pb_lo_2 = (const Ipp64s *)(&(FE521_LO(b2))); + const Ipp64s *pb_mid_2 = (const Ipp64s *)(&(FE521_MID(b2))); + const Ipp64s *pb_hi_2 = (const Ipp64s *)(&(FE521_HI(b2))); + + const m256i b0_1 = m256_set1_i64(pb_lo_1[0]); + const m256i b1_1 = m256_set1_i64(pb_lo_1[1]); + const m256i b2_1 = m256_set1_i64(pb_lo_1[2]); + const m256i b3_1 = m256_set1_i64(pb_lo_1[3]); + const m256i b4_1 = m256_set1_i64(pb_mid_1[0]); + const m256i b5_1 = m256_set1_i64(pb_mid_1[1]); + const m256i b6_1 = m256_set1_i64(pb_mid_1[2]); + const m256i b7_1 = m256_set1_i64(pb_mid_1[3]); + const m256i b8_1 = m256_set1_i64(pb_hi_1[0]); + const m256i b9_1 = m256_set1_i64(pb_hi_1[1]); + const m256i b10_1 = m256_set1_i64(pb_hi_1[2]); + + const m256i b0_2 = m256_set1_i64(pb_lo_2[0]); + const m256i b1_2 = m256_set1_i64(pb_lo_2[1]); + const m256i b2_2 = m256_set1_i64(pb_lo_2[2]); + const m256i b3_2 = m256_set1_i64(pb_lo_2[3]); + const m256i b4_2 = m256_set1_i64(pb_mid_2[0]); + const m256i b5_2 = m256_set1_i64(pb_mid_2[1]); + const m256i b6_2 = m256_set1_i64(pb_mid_2[2]); + const m256i b7_2 = m256_set1_i64(pb_mid_2[3]); + const m256i b8_2 = m256_set1_i64(pb_hi_2[0]); + const m256i b9_2 = m256_set1_i64(pb_hi_2[1]); + const m256i b10_2 = m256_set1_i64(pb_hi_2[2]); + + fe521 r0_1; + fe521 p0_1, p1_1, p2_1, p3_1, p4_1, p5_1, p6_1, p7_1, p8_1, p9_1, p10_1; + fe521 r0_2; + fe521 p0_2, p1_2, p2_2, p3_2, p4_2, p5_2, p6_2, p7_2, p8_2, p9_2, p10_2; + + /* first */ + FE521_SET(r0_1) = m256_setzero_i64(); + group_madd52lo_i64(r0_1, r0_1, a1, b0_1); + FE521_SET(r0_2) = m256_setzero_i64(); + group_madd52lo_i64(r0_2, r0_2, a2, b0_2); + /* last */ + FE521_SET(p10_1) = m256_setzero_i64(); + group_madd52hi_i64(p10_1, p10_1, a1, b10_1); + FE521_SET(p10_2) = m256_setzero_i64(); + group_madd52hi_i64(p10_2, p10_2, a2, b10_2); + /* round mul */ + MUL_ROUND(p0_1, a1, b0_1, b1_1) + MUL_ROUND(p0_2, a2, b0_2, b1_2) + + MUL_ROUND(p1_1, a1, b1_1, b2_1) + MUL_ROUND(p1_2, a2, b1_2, b2_2) + + MUL_ROUND(p2_1, a1, b2_1, b3_1) + MUL_ROUND(p2_2, a2, b2_2, b3_2) + + MUL_ROUND(p3_1, a1, b3_1, b4_1) + MUL_ROUND(p3_2, a2, b3_2, b4_2) + + MUL_ROUND(p4_1, a1, b4_1, b5_1) + MUL_ROUND(p4_2, a2, b4_2, b5_2) + + MUL_ROUND(p5_1, a1, b5_1, b6_1) + MUL_ROUND(p5_2, a2, b5_2, b6_2) + + MUL_ROUND(p6_1, a1, b6_1, b7_1) + MUL_ROUND(p6_2, a2, b6_2, b7_2) + + MUL_ROUND(p7_1, a1, b7_1, b8_1) + MUL_ROUND(p7_2, a2, b7_2, b8_2) + + MUL_ROUND(p8_1, a1, b8_1, b9_1) + MUL_ROUND(p8_2, a2, b8_2, b9_2) + + MUL_ROUND(p9_1, a1, b9_1, b10_1) + MUL_ROUND(p9_2, a2, b9_2, b10_2) + + const m256i filt_rad52 = m256_set1_i64(DIGIT_MASK_52); + const m256i idx0 = m256_set_i8(REPL4(7, 6, 5, 4, 3, 2, 1, 0)); /* B[ 0] -> chunk1[0] */ + /* chunk2 shift bit >> 64 */ + const mask32 mask_sr64 = 0x00FFFFFF; + const m256i idx_sr64 = m256_set_i8(0, 0, 0, 0, 0, 0, 0, 0, + 31, 30, 29, 28, 27, 26, 25, 24, + 23, 22, 21, 20, 19, 18, 17, 16, + 15, 14, 13, 12, 11, 10, 9, 8); + + fe521 reduct_1; + FE521_SET(reduct_1) = m256_setzero_i64(); + m256i u_1; + fe521 reduct_2; + FE521_SET(reduct_2) = m256_setzero_i64(); + m256i u_2; + REDUCTION(r0_1, u_1, reduct_1) + REDUCTION(r0_2, u_2, reduct_2) + + fe521_add_no_red(r0_1, r0_1, p0_1); + fe521_add_no_red(r0_2, r0_2, p0_2); + REDUCTION(r0_1, u_1, reduct_1) + REDUCTION(r0_2, u_2, reduct_2) + + fe521_add_no_red(r0_1, r0_1, p1_1); + fe521_add_no_red(r0_2, r0_2, p1_2); + REDUCTION(r0_1, u_1, reduct_1) + REDUCTION(r0_2, u_2, reduct_2) + + fe521_add_no_red(r0_1, r0_1, p2_1); + fe521_add_no_red(r0_2, r0_2, p2_2); + REDUCTION(r0_1, u_1, reduct_1) + REDUCTION(r0_2, u_2, reduct_2) + + fe521_add_no_red(r0_1, r0_1, p3_1); + fe521_add_no_red(r0_2, r0_2, p3_2); + REDUCTION(r0_1, u_1, reduct_1) + REDUCTION(r0_2, u_2, reduct_2) + + fe521_add_no_red(r0_1, r0_1, p4_1); + fe521_add_no_red(r0_2, r0_2, p4_2); + REDUCTION(r0_1, u_1, reduct_1) + REDUCTION(r0_2, u_2, reduct_2) + + fe521_add_no_red(r0_1, r0_1, p5_1); + fe521_add_no_red(r0_2, r0_2, p5_2); + REDUCTION(r0_1, u_1, reduct_1) + REDUCTION(r0_2, u_2, reduct_2) + + fe521_add_no_red(r0_1, r0_1, p6_1); + fe521_add_no_red(r0_2, r0_2, p6_2); + REDUCTION(r0_1, u_1, reduct_1) + REDUCTION(r0_2, u_2, reduct_2) + + fe521_add_no_red(r0_1, r0_1, p7_1); + fe521_add_no_red(r0_2, r0_2, p7_2); + REDUCTION(r0_1, u_1, reduct_1) + REDUCTION(r0_2, u_2, reduct_2) + + fe521_add_no_red(r0_1, r0_1, p8_1); + fe521_add_no_red(r0_2, r0_2, p8_2); + REDUCTION(r0_1, u_1, reduct_1) + REDUCTION(r0_2, u_2, reduct_2) + + fe521_add_no_red(r0_1, r0_1, p9_1); + fe521_add_no_red(r0_2, r0_2, p9_2); + REDUCTION(r0_1, u_1, reduct_1) + REDUCTION(r0_2, u_2, reduct_2) + + fe521_add_no_red(r0_1, r0_1, p10_1); + fe521_add_no_red(r0_2, r0_2, p10_2); + + FE521_COPY(*pr1, r0_1); + FE521_COPY(*pr2, r0_2); + return; +} + +#undef group_madd52hi_i64 +#undef group_madd52lo_i64 + +IPP_OWN_DEFN(void, ifma_half52_p521, (fe521 pr[], const fe521 a)) +{ + fe521 M; + FE521_LOADU(M, p521_x1); + const m256i zero = m256_setzero_i64(); + const m256i one = m256_set1_i64(1LL); + + const mask8 mask_last_bit_one_line = m256_cmp_i64_mask(m256_and_i64(FE521_LO(a), one), zero, _MM_CMPINT_EQ); + const mask8 mask_is_lb_one = (mask8)((mask_last_bit_one_line & 1) - 1); + + fe521 r; + FE521_LO(r) = _mm256_mask_add_epi64(FE521_LO(a), mask_is_lb_one, FE521_LO(a), FE521_LO(M)); + FE521_MID(r) = _mm256_mask_add_epi64(FE521_MID(a), mask_is_lb_one, FE521_MID(a), FE521_MID(M)); + FE521_HI(r) = _mm256_mask_add_epi64(FE521_HI(a), mask_is_lb_one, FE521_HI(a), FE521_HI(M)); + ifma_lnorm52_p521(&r, r); + + /* 1-bit shift right */ + /* chunk2 shift bit >> 64 */ + const mask32 mask_sr64 = 0xFFFF; + const m256i idx_sr64 = m256_set_i8(24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, + 23, 22, 21, 20, 19, 18, 17, 16, + 15, 14, 13, 12, 11, 10, 9, 8); + + fe521 shift_right; + /* shift right */ + FE521_LO(shift_right) = m256_alignr_i64(FE521_MID(r), FE521_LO(r), 1); + FE521_MID(shift_right) = m256_alignr_i64(FE521_HI(r), FE521_MID(r), 1); + FE521_HI(shift_right) = m256_maskz_permutexvar_i8(mask_sr64, idx_sr64, FE521_HI(r)); + /* extract las bit */ + FE521_LO(shift_right) = m256_and_i64(FE521_LO(shift_right), one); + FE521_MID(shift_right) = m256_and_i64(FE521_MID(shift_right), one); + FE521_HI(shift_right) = m256_and_i64(FE521_HI(shift_right), one); + /* set last bit is first byte (52 radix) */ + FE521_LO(shift_right) = m256_slli_i64(FE521_LO(shift_right), DIGIT_SIZE_52 - 1); + FE521_MID(shift_right) = m256_slli_i64(FE521_MID(shift_right), DIGIT_SIZE_52 - 1); + FE521_HI(shift_right) = m256_maskz_slli_i64(0x3, FE521_HI(shift_right), DIGIT_SIZE_52 - 1); + /* join first new bite */ + FE521_LO(r) = m256_srli_i64(FE521_LO(r), 1); + FE521_MID(r) = m256_srli_i64(FE521_MID(r), 1); + FE521_HI(r) = m256_maskz_srli_i64(0x7, FE521_HI(r), 1); + /* join first and other bit */ + FE521_LO(r) = m256_add_i64(FE521_LO(r), FE521_LO(shift_right)); + FE521_MID(r) = m256_add_i64(FE521_MID(r), FE521_MID(shift_right)); + FE521_HI(r) = m256_add_i64(FE521_HI(r), FE521_HI(shift_right)); + + FE521_COPY(*pr, r); + return; +} + +IPP_OWN_DEFN(void, ifma_neg52_p521, (fe521 pr[], const fe521 a)) +{ + fe521 M4; + FE521_LOADU(M4, p521_x4); + + const mask8 mask_is_not_zero = (mask8)(~(FE521_IS_ZERO(a))); + fe521 r; + FE521_LO(r) = m256_mask_sub_i64(FE521_LO(a), mask_is_not_zero, FE521_LO(M4), FE521_LO(a)); + FE521_MID(r) = m256_mask_sub_i64(FE521_MID(a), mask_is_not_zero, FE521_MID(M4), FE521_MID(a)); + FE521_HI(r) = m256_mask_sub_i64(FE521_HI(a), mask_is_not_zero, FE521_HI(M4), FE521_HI(a)); + ifma_norm52_p521(&r, r); + + FE521_COPY(*pr, r); + return; +} + +/* to Montgomery conversion constant + * rr = 2^((P521_LEN52*DIGIT_SIZE)*2) mod p521 + */ +static const __ALIGN64 Ipp64u P521R1_RR52[P521R1_NUM_CHUNK][P521R1_LENFE521_52] = { { 0x0000000000000000, + 0x0004000000000000, + 0x0000000000000000, + 0x0000000000000000 }, + { 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000 }, + { 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000 } }; + +IPP_OWN_DEFN(void, ifma_tomont52_p521, (fe521 pr[], const fe521 a)) +{ + fe521 RR; + FE521_LOADU(RR, P521R1_RR52); + ifma_amm52_p521(pr, a, RR); + ifma_lnorm52_p521(pr, *pr); + return; +} + +static IPP_OWN_DEFN(void, ifma_fastred52_p521, (fe521 pr[], const fe521 a)) +{ + fe521 M; + FE521_LOADU(M, p521_x1); + const m256i zero = m256_setzero_i64(); + + fe521 r; + fe521_sub_no_red(r, a, M); + ifma_norm52_p521(&r, r); + + const mask8 lt = m256_cmp_i64_mask(zero, m256_srli_i64(FE521_HI(r), DIGIT_SIZE_52 - 1), _MM_CMPINT_LT); + const mask8 mask = (mask8)((mask8)0 - ((lt >> 2) & 1)); + + /* maks != 0 ? a : r */ + FE521_MASK_MOV(r, r, mask, a); + FE521_COPY(*pr, r); + return; +} + +IPP_OWN_DEFN(void, ifma_frommont52_p521, (fe521 pr[], const fe521 a)) +{ + fe521 ONE; + FE521_LOADU(ONE, P521R1_ONE52); + ifma_amm52_p521(pr, a, ONE); + ifma_lnorm52_p521(pr, *pr); + ifma_fastred52_p521(pr, *pr); + return; +} + +__INLINE IPP_OWN_DEFN(void, ifma_amm52_p521_norm, (fe521 pr[], const fe521 a, const fe521 b)) +{ + ifma_amm52_p521(pr, a, b); + ifma_lnorm52_p521(pr, *pr); + return; +} + +__INLINE IPP_OWN_DEFN(void, ifma_ams52_p521_norm, (fe521 pr[], const fe521 a)) +{ + ifma_ams52_p521(pr, a); + ifma_lnorm52_p521(pr, *pr); + return; +} + +#define mul(R, A, B) ifma_amm52_p521_norm(&(R), (A), (B)) +#define sqr(R, A) ifma_ams52_p521_norm(&(R), (A)) +#define mul_dual(R1, A1, B1, R2, A2, B2) \ + ifma_amm52_dual_p521(&(R1), (A1), (B1), &(R2), (A2), (B2)); \ + ifma_lnorm52_dual_p521(&(R1), (R1), &(R2), (R2)) + +/* r = base^(2^n) */ +__INLINE IPP_OWN_DEFN(void, ifma_ams52_p521_ntimes, (fe521 pr[], const fe521 a, int n)) +{ + fe521 r; + FE521_COPY(r, a); + for (; n > 0; --n) + sqr(r, r); + FE521_COPY(*pr, r); + return; +} + +#define sqr_ntimes(R, A, N) ifma_ams52_p521_ntimes(&(R), (A), (N)) + +IPP_OWN_DEFN(void, ifma_aminv52_p521, (fe521 pr[], const fe521 z)) +{ + fe521 u, v, zD, zF, z1FF; + FE521_SET(u) = FE521_SET(v) = FE521_SET(zD) = FE521_SET(zF) = FE521_SET(z1FF) = m256_setzero_i64(); + + sqr(u, z); /* u = z^2 */ + mul(v, u, z); /* v = z^2 * z = z^3 */ + sqr_ntimes(zF, v, 2); /* zF= (z^3)^(2^2) = z^12 */ + /**/ + mul_dual(zD, zF, z, /* zD = z^12 * z = z^xD */ + zF, zF, v); /* zF = z^12 * z^3 = z^xF */ + /**/ + sqr_ntimes(u, zF, 4); /* u = (z^xF)^(2^4) = z^xF0 */ + mul_dual(zD, u, zD, /* zD = z^xF0 * z^xD = z^xFD */ + zF, u, zF); /* zF = z^xF0 * z^xF = z^xFF */ + /**/ + sqr(z1FF, zF); /* z1FF= (zF^2) = z^x1FE */ + mul(z1FF, z1FF, z); /* z1FF *= z = z^x1FF */ + /**/ + sqr_ntimes(u, zF, 8); /* u = (z^xFF)^(2^8) = z^xFF00 */ + mul_dual(zD, u, zD, /* zD = z^xFF00 * z^xFD = z^xFFFD */ + zF, u, zF); /* zF = z^xFF00 * z^xFF = z^xFFFF */ + /**/ + sqr_ntimes(u, zF, 16); /* u = (z^xFFFF)^(2^16) = z^xFFFF0000 */ + mul_dual(zD, u, zD, /* zD = z^xFFFF0000 * z^xFFFD = z^xFFFFFFFD */ + zF, u, zF); /* zF = z^xFFFF0000 * z^xFFFF = z^xFFFFFFFF */ + /**/ + sqr_ntimes(u, zF, 32); /* u = (z^xFFFFFFFF)^(2^32) = z^xFFFFFFFF00000000 */ + mul_dual(zD, u, zD, /* zD = z^xFFFFFFFF00000000 * z^xFFFFFFFD = z^xFFFFFFFFFFFFFFFD */ + zF, u, zF); /* zF = z^xFFFFFFFF00000000 * z^xFFFFFFFF = z^xFFFFFFFFFFFFFFFF */ + + /* v = z^x1FF.0000000000000000 * z^xFFFFFFFFFFFFFFFF + = z^x1FF.FFFFFFFFFFFFFFFF */ + sqr_ntimes(v, z1FF, 64); + mul(v, v, zF); + + /* v = z^x1FF.FFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFF + = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF */ + sqr_ntimes(v, v, 64); + mul(v, v, zF); + + /* v = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFF + = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF */ + sqr_ntimes(v, v, 64); + mul(v, v, zF); + + /* v = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFF + = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF */ + sqr_ntimes(v, v, 64); + mul(v, v, zF); + + /* v = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFF + = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF */ + sqr_ntimes(v, v, 64); + mul(v, v, zF); + + /* v = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFF + = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF */ + sqr_ntimes(v, v, 64); + mul(v, v, zF); + + /* v = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFF + = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF */ + sqr_ntimes(v, v, 64); + mul(v, v, zF); + + /* v = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.0000000000000000 * z^xFFFFFFFFFFFFFFFD + = z^x1FF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFE.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFF.FFFFFFFFFFFFFFFD */ + sqr_ntimes(v, v, 64); + mul(*pr, v, zD); + return; +} + +#define ROUND_CONV_64_TO_52(R, A, MASK_LOAD, MASK_RAD52, IDX16) \ + (R) = m256_maskz_loadu_i64((MASK_LOAD), (A)); \ + (R) = m256_permutexvar_i16((IDX16), (R)); \ + (R) = m256_srlv_i64((R), shift_right); \ + (R) = m256_and_i64((MASK_RAD52), (R)); + +IPP_OWN_DEFN(void, convert_radix_to_52_p521, (fe521 pr[], const Ipp64u arad64[P521R1_LEN64])) +{ + + /* chunk 1 */ + const m256i idx16c1 = m256_set_i16(12, 11, 10, 9, + 9, 8, 7, 6, + 6, 5, 4, 3, + 3, 2, 1, 0); + const m256i idx16c2 = m256_set_i16(13, 12, 11, 10, + 10, 9, 8, 7, + 7, 6, 5, 4, + 4, 3, 2, 1); + const m256i idx16c3 = m256_set_i16(12, 12, 12, 12, + 12, 12, 12, 8, + 8, 7, 6, 5, + 5, 4, 3, 2); + + const m256i shift_right = m256_set_i64(12LL, 8LL, 4LL, 0LL); + const m256i mask_rad52c12 = m256_set1_i64(DIGIT_MASK_52); + const m256i mask_rad52c3 = m256_set_i64(0x0, 0x1, DIGIT_MASK_52, DIGIT_MASK_52); + + fe521 r; + FE521_SET(r) = m256_setzero_i64(); + + /* chunk 1 */ + ROUND_CONV_64_TO_52(FE521_LO(r), arad64, 0xF, mask_rad52c12, idx16c1) + /* chunk 2 */ + ROUND_CONV_64_TO_52(FE521_MID(r), arad64 + 3, 0xF, mask_rad52c12, idx16c2) + /* chunk 2 */ + ROUND_CONV_64_TO_52(FE521_HI(r), arad64 + 6, 0x7, mask_rad52c3, idx16c3) + + FE521_COPY(*pr, r); + return; +} + +IPP_OWN_DEFN(void, convert_radix_to_64_p521, (Ipp64u rrad64[P521R1_LEN64], const fe521 a)) +{ + /* filter chunk3 */ + const m256i mask_filtc3 = m256_set_i64(0x0, 0x1, DIGIT_MASK_52, DIGIT_MASK_52); + const m256i shift_left = m256_set_i64(4LL, 0LL, 4LL, 0LL); + /* idx create chunk1 */ + const m256i idx8_upc1c1 = m256_set_i8(31, 31, 31, 31, 31, 31, 31, 31, + 23, 23, 23, 23, 22, 21, 20, 19, + 15, 15, 15, 14, 13, 12, 11, 10, + 7, 6, 5, 4, 3, 2, 1, 0); + const m256i idx8_downc1c1 = m256_set_i8(31, 31, 31, 31, 31, 31, 30, 29, + 28, 27, 26, 25, 24, 23, 23, 23, + 18, 17, 16, 15, 15, 15, 15, 15, + 9, 8, 7, 7, 7, 7, 7, 7); + const m256i idx8_upc2c1 = m256_set_i8(5, 4, 3, 2, 1, 0, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7); + /* idx create chunk 2 */ + const m256i idx8_upc2c2 = m256_set_i8(31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, + 23, 23, 22, 21, 20, 19, 18, 17, + 16, 7, 7, 7, 7, 7, 7, 6); + const m256i idx8_downc2c2 = m256_set_i8(31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 30, 29, 28, 27, + 26, 25, 24, 23, 23, 23, 23, 23, + 15, 14, 13, 12, 11, 10, 9, 8); + const m256i idx8_upc3c2 = m256_set_i8(13, 12, 11, 10, 9, 8, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7); + const m256i idx8_downc3c2 = m256_set_i8(7, 7, 7, 7, 7, 6, 5, 4, + 3, 2, 1, 0, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7); + /* idx create chunk 3 */ + const m256i idx8_upc3c3 = m256_set_i8(15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 16, 15); + const m256i idx8_downc3c3 = m256_set_i8(15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 14); + + fe521 r; + /* shift left */ + FE521_LO(r) = m256_sllv_i64(FE521_LO(a), shift_left); + FE521_MID(r) = m256_sllv_i64(FE521_MID(a), shift_left); + FE521_HI(r) = m256_and_i64(FE521_HI(a), mask_filtc3); + FE521_HI(r) = m256_sllv_i64(FE521_HI(r), shift_left); + + m256i t1, t2, t3; + /* chunk 1 */ + t1 = m256_permutexvar_i8(idx8_upc1c1, FE521_LO(r)); + t2 = m256_permutexvar_i8(idx8_downc1c1, FE521_LO(r)); + t1 = m256_or_i64(t1, t2); + t3 = m256_permutexvar_i8(idx8_upc2c1, FE521_MID(r)); + + FE521_LO(r) = m256_or_i64(t3, t1); + + /* chunk 2 */ + t1 = m256_permutexvar_i8(idx8_upc2c2, FE521_MID(r)); + t2 = m256_permutexvar_i8(idx8_downc2c2, FE521_MID(r)); + t3 = m256_or_i64(t1, t2); + t1 = m256_permutexvar_i8(idx8_upc3c2, FE521_HI(r)); + t2 = m256_permutexvar_i8(idx8_downc3c2, FE521_HI(r)); + + FE521_MID(r) = m256_or_i64(t3, m256_or_i64(t1, t2)); + + /* chunk 3 */ + t1 = m256_permutexvar_i8(idx8_upc3c3, FE521_HI(r)); + t2 = m256_permutexvar_i8(idx8_downc3c3, FE521_HI(r)); + + FE521_HI(r) = m256_or_i64(t1, t2); + + /* store */ + m256_storeu_i64(rrad64, FE521_LO(r)); + m256_storeu_i64(rrad64 + 4, FE521_MID(r)); + m256_mask_storeu_i64(rrad64 + 8, 0x1, FE521_HI(r)); + + return; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_p521.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_p521.h new file mode 100644 index 0000000..493384b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_arith_p521.h @@ -0,0 +1,266 @@ +/****************************************************************************** + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#ifndef IFMA_ARITH_P521_H +#define IFMA_ARITH_P521_H + +#if (_IPP32E >= _IPP32E_K1) + +#include "ifma_defs_p521.h" + +/* + * p521 = 2^521 - 1 + * in 2^52 radix + */ +static const __ALIGN64 Ipp64u p521_x1[P521R1_NUM_CHUNK][P521R1_LENFE521_52] = { + { 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF }, + { 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF }, + { 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x0000000000000001, + 0x0000000000000000 } +}; +/* 2*p */ +static const __ALIGN64 Ipp64u p521_x2[P521R1_NUM_CHUNK][P521R1_LENFE521_52] = { + { 0x000FFFFFFFFFFFFE, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF }, + { 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF }, + { 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x0000000000000003, + 0x0000000000000000 } +}; +/* 4*p */ +static const __ALIGN64 Ipp64u p521_x4[P521R1_NUM_CHUNK][P521R1_LENFE521_52] = { + { 0x000FFFFFFFFFFFFC, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF }, + { 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF }, + { 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x0000000000000007, + 0x0000000000000000 } +}; +/* 6*p */ +static const __ALIGN64 Ipp64u p521_x6[P521R1_NUM_CHUNK][P521R1_LENFE521_52] = { + { 0x000FFFFFFFFFFFFA, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF }, + { 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF }, + { 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x000000000000000B, + 0x0000000000000000 } +}; +/* 8*p */ +static const __ALIGN64 Ipp64u p521_x8[P521R1_NUM_CHUNK][P521R1_LENFE521_52] = { + { 0x000FFFFFFFFFFFF8, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF }, + { 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF }, + { 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x000000000000000F, + 0x0000000000000000 } +}; +/* Montgomery(1) + * r = 2^(P521_LEN52*DIGIT_SIZE) mod p521 + */ +static const __ALIGN64 Ipp64u P521R1_R52[P521R1_NUM_CHUNK][P521R1_LENFE521_52] = { + { 0x0008000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000 }, + { 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000 }, + { 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000 } +}; + +/** + * \brief + * normalization input fe521 to (in radix 2^52) + * \param[out] pr ptr value (in radix 2^52) + * \param[in] a value (52 radix or more) + */ +IPP_OWN_DECL(void, ifma_norm52_p521, (fe521 pr[], const fe521 a)) + +/** + * \brief + * duplicate normalization input double fe521 to (in radix 2^52) + * \param[out] pr1 ptr first value (in radix 2^52) + * \param[in] a1 value first (52 radix or more) + * \param[out] pr2 ptr second value (in radix 2^52) + * \param[in] a2 value second (52 radix or more) + */ +IPP_OWN_DECL(void, ifma_norm52_dual_p521, (fe521 pr1[], const fe521 a1, fe521 pr2[], const fe521 a2)) + +/** + * \brief + * light (mul|add only) normalization input fe521 to (in radix 2^52) + * \param[out] pr ptr value (in radix 2^52) + * \param[in] a value (52 radix or more) + */ +IPP_OWN_DECL(void, ifma_lnorm52_p521, (fe521 pr[], const fe521 a)) + +/** + * \brief + * light (mul|add only) duplicate normalization input double fe521 to (in radix 2^52) + * \param[out] pr1 ptr first value (in radix 2^52) + * \param[in] a1 value first (52 radix or more) + * \param[out] pr2 ptr second value (in radix 2^52) + * \param[in] a2 value second (52 radix or more) + */ +IPP_OWN_DECL(void, ifma_lnorm52_dual_p521, (fe521 pr1[], const fe521 a1, fe521 pr2[], const fe521 a2)) + +/** + * \brief + * R = (A * B) - no normalization + * \param[out] pr ptr value + * \param[in] a first value (in radix 2^52) + * \param[in] b second value (in radix 2^52) + */ +IPP_OWN_DECL(void, ifma_amm52_p521, (fe521 pr[], const fe521 a, const fe521 b)) + +/** + * \brief + * duplicate mul (R = A * B - no normalization) input double complect + * \param[out] pr1 ptr first value no normalization + * \param[in] a1 value first (in radix 2^52) + * \param[in] b1 value second (in radix 2^52) + * \param[out] pr2 ptr second value no normalization + * \param[in] a2 value first (in radix 2^52) + * \param[in] b2 value second (in radix 2^52) + */ +IPP_OWN_DECL(void, ifma_amm52_dual_p521, (fe521 pr1[], const fe521 a1, const fe521 b1, fe521 pr2[], const fe521 a2, const fe521 b2)) + +/** + * \brief + * R = A/2 - with normalization + * \param[out] pr ptr value (in radix 2^52) + * \param[in] a value (52 radix or more) + */ +IPP_OWN_DECL(void, ifma_half52_p521, (fe521 pr[], const fe521 a)) + +/** + * \brief + * compute R = (-A) enhanced Montgomery (Gueron modification group operation) + * \param[out] pr ptr value (in radix 2^52) + * \param[in] a value (in radix 2^52) + */ +IPP_OWN_DECL(void, ifma_neg52_p521, (fe521 pr[], const fe521 a)) + +/** + * \brief + * to Montgomery domain (in radix 2^52) + * \param[out] pr ptr value (in radix 2^52) in Montgomery domain + * \param[in] a value (in radix 2^52) + */ +IPP_OWN_DECL(void, ifma_tomont52_p521, (fe521 pr[], const fe521 a)) + +/** + * \brief + * from Montgomery domain (in radix 2^52) + * \param[out] pr ptr value (in radix 2^52) from Montgomery domain + * \param[in] a value (in radix 2^52) + */ +IPP_OWN_DECL(void, ifma_frommont52_p521, (fe521 pr[], const fe521 a)) + +/** + * \brief + * R = 1/Z + * \param[out] pr ptr value (in radix 2^52) + * \param[in] z value (in radix 2^52) + */ +IPP_OWN_DECL(void, ifma_aminv52_p521, (fe521 pr[], const fe521 z)) + +/** + * \brief + * convert radix 64 to (in radix 2^52) + * \param[out] pr ptr value (in radix 2^52) + * \param[in] arad64 ptr array size (9) - 521 bit + */ +IPP_OWN_DECL(void, convert_radix_to_52_p521, (fe521 pr[], const Ipp64u arad64[P521R1_LEN64])) + +/** + * \brief + * convert (in radix 2^52) to radix 64 + * \param[out] rrad64 ptr array size (9) - 521 bit + * \param[in] a value (in radix 2^52) + */ +IPP_OWN_DECL(void, convert_radix_to_64_p521, (Ipp64u rrad64[P521R1_LEN64], const fe521 a)) + +/** + * \brief + * R = (A * A) - no normalization + * \param[out] pr ptr value + * \param[in] a first value (in radix 2^52) + */ +IPP_OWN_DECL(void, ifma_ams52_p521, (fe521 pr[], const fe521 a)) + +/** + * \brief + * duplicate sqr (R = A * A - no normalization) input double complect + * \param[out] pr1 ptr first value no normalization + * \param[in] a1 value first (in radix 2^52) + * \param[out] pr2 ptr second value no normalization + * \param[in] a2 value first (in radix 2^52) + */ +IPP_OWN_DECL(void, ifma_ams52_dual_p521, (fe521 pr1[], const fe521 a1, fe521 pr2[], const fe521 a2)) + +/* R = (A + B) */ +#define fe521_add_no_red(R, A, B) \ + FE521_LO(R) = m256_add_i64(FE521_LO(A), FE521_LO(B)); \ + FE521_MID(R) = m256_add_i64(FE521_MID(A), FE521_MID(B)); \ + FE521_HI(R) = m256_add_i64(FE521_HI(A), FE521_HI(B)) + +/* R = (A - B) */ +#define fe521_sub_no_red(R, A, B) \ + FE521_LO(R) = m256_sub_i64(FE521_LO(A), FE521_LO(B)); \ + FE521_MID(R) = m256_sub_i64(FE521_MID(A), FE521_MID(B)); \ + FE521_HI(R) = m256_sub_i64(FE521_HI(A), FE521_HI(B)) + +#endif // (_IPP32E >= _IPP32E_K1) + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_defs.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_defs.h new file mode 100644 index 0000000..ab9fe60 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_defs.h @@ -0,0 +1,71 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#ifndef IFMA_DEFS_H +#define IFMA_DEFS_H + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ifma_alias_avx512.h" + +/* Internal radix definition */ +#define DIGIT_SIZE (52) +#define DIGIT_BASE ((Ipp64u)1 << DIGIT_SIZE) +#define DIGIT_MASK ((Ipp64u)0xFFFFFFFFFFFFF) + +/* Number of digit in "digsize" representation of "bitsize" value */ +#define NUMBER_OF_DIGITS(bitsize, digsize) (((bitsize) + (digsize)-1) / (digsize)) +/* Mask of most significant digit wrt "digsize" representation */ +#define MS_DIGIT_MASK(bitsize, digsize) (((int64u)1 << ((bitsize) % digsize)) - 1) + + +#define REPL8(e) e, e, e, e, e, e, e, e + +/** + * \brief + * + * Check if bit on corresponding position is 1. + * Position counting starts from zero. + * + * \return 0xFF - if MSB = 1 + * \return 0x00 - if MSB = 0 + */ +__INLINE mask8 check_bit(const mask8 a, int bit) +{ + return (mask8)((mask8)0 - ((a >> bit) & 1u)); +} + +/** + * \brief + * + * Check if ZMM register contains all zeroes. + * + * \param[in] a value + * \return 0xFF - if input value is all zeroes + * \return 0x00 - if input value is not all zeroes + */ +__INLINE mask8 is_zero_i64(const m512 a) +{ + const mask8 mask = cmp_i64_mask(a, setzero_i64(), _MM_CMPINT_NE); + return check_bit((~mask & (mask - 1u)), 7); +} + +#endif // (_IPP32E >= _IPP32E_K1) + +#endif // IFMA_DEFS_H diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_defs_p521.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_defs_p521.h new file mode 100644 index 0000000..1d6aa55 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_defs_p521.h @@ -0,0 +1,88 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#ifndef IFMA_DEFS_P521_H +#define IFMA_DEFS_P521_H + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ifma_alias_avx512vl.h" + +#define DIGIT_MASK_52 (0xFFFFFFFFFFFFF) +#define DIGIT_SIZE_52 (52) + +#define P521R1_LEN52 (11) +#define P521R1_LEN64 (9) +#define P521R1_LENFE521_52 (4) +#define P521R1_NUM_CHUNK (3) + +typedef m256i fe521[P521R1_NUM_CHUNK]; + +#define REPL4(a1, a2, a3, a4, a5, a6, a7, a8) a1, a2, a3, a4, a5, a6, a7, a8, \ + a1, a2, a3, a4, a5, a6, a7, a8, \ + a1, a2, a3, a4, a5, a6, a7, a8, \ + a1, a2, a3, a4, a5, a6, a7, a8 + +/* one */ +static const __ALIGN64 Ipp64u P521R1_ONE52[P521R1_NUM_CHUNK][P521R1_LENFE521_52] = { + { 0x1, 0x0, 0x0, 0x0 }, + { 0x0, 0x0, 0x0, 0x0 }, + { 0x0, 0x0, 0x0, 0x0 } +}; + +#define FE521_LO(A) (A)[0] +#define FE521_MID(A) (A)[1] +#define FE521_HI(A) (A)[2] + +#define FE521_SET(A) FE521_LO(A) = FE521_MID(A) = FE521_HI(A) + +#define FE521_COPY(R, A) \ + FE521_LO(R) = FE521_LO(A); \ + FE521_MID(R) = FE521_MID(A); \ + FE521_HI(R) = FE521_HI(A) + +#define FE521_LOADU(R, A) \ + FE521_LO(R) = m256_loadu_i64(FE521_LO(A)); \ + FE521_MID(R) = m256_loadu_i64(FE521_MID(A)); \ + FE521_HI(R) = m256_loadu_i64(FE521_HI(A)) + +__INLINE mask8 is_msb_m256(const mask8 a) +{ + return ((mask8)0 - (a >> 7)); +} + +__INLINE mask8 is_zero_m256(const m256i a) +{ + const mask8 mask = _mm256_cmp_epi64_mask(a, m256_setzero_i64(), _MM_CMPINT_NE); + return is_msb_m256((~mask & (mask - 1))); +} + +#define FE521_IS_ZERO(A) (is_zero_m256(m256_or_i64(m256_or_i64(FE521_LO(A), FE521_MID(A)), FE521_HI(A)))) + +#define FE521_CMP_MASK(A, B, ENUM_CMP) \ + (m256_cmp_i64_mask(FE521_LO(A), FE521_LO(B), (ENUM_CMP)) & m256_cmp_i64_mask(FE521_MID(A), FE521_MID(B), (ENUM_CMP)) & m256_cmp_i64_mask(FE521_HI(A), FE521_HI(B), (ENUM_CMP))) + +#define FE521_MASK_MOV(R, SRC, MASK, A) \ + FE521_LO(R) = m256_mask_mov_i64(FE521_LO(SRC), (MASK), FE521_LO(A)); \ + FE521_MID(R) = m256_mask_mov_i64(FE521_MID(SRC), (MASK), FE521_MID(A)); \ + FE521_HI(R) = m256_mask_mov_i64(FE521_HI(SRC), (MASK), FE521_HI(A)) + +#endif // (_IPP32E >= _IPP32E_K1) + +#endif // _CP_DEFINE_FE521_K1 diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_addpoint_p256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_addpoint_p256.c new file mode 100644 index 0000000..f9fdc4a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_addpoint_p256.c @@ -0,0 +1,60 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpstuff.h" +#include "pcpgfpecstuff.h" + +#include "ifma_arith_method.h" +#include "ifma_ecpoint_p256.h" + +IPP_OWN_DEFN(IppsGFpECPoint*, gfec_AddPoint_nistp256_avx512, (IppsGFpECPoint* pR, + const IppsGFpECPoint* pP, + const IppsGFpECPoint* pQ, + IppsGFpECState* pEC)) +{ + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + ifmaArithMethod *pmeth = (ifmaArithMethod *)GFP_METHOD_ALT(pME); + + __ALIGN64 P256_POINT_IFMA P, R; + + BNU_CHUNK_T *pPool = cpGFpGetPool(3, pME); + + recode_point_to_mont52(&P, ECP_POINT_DATA(pP), pPool, pmeth, pME); + + if (pP == pQ) { + ifma_ec_nistp256_dbl_point(&R, &P); + } else { + __ALIGN64 P256_POINT_IFMA Q; + + recode_point_to_mont52(&Q, ECP_POINT_DATA(pQ), pPool, pmeth, pME); + + ifma_ec_nistp256_add_point(&R, &P, &Q); + } + + recode_point_to_mont64(pR, &R, pPool, pmeth, pME); + + cpGFpReleasePool(3, pME); + + ECP_POINT_FLAGS(pR) = gfec_IsPointAtInfinity(pR) ? 0 : ECP_FINITE_POINT; + return pR; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_addpoint_p384.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_addpoint_p384.c new file mode 100644 index 0000000..5be4dd3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_addpoint_p384.c @@ -0,0 +1,60 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpstuff.h" +#include "pcpgfpecstuff.h" + +#include "ifma_arith_method.h" +#include "ifma_ecpoint_p384.h" + +IPP_OWN_DEFN(IppsGFpECPoint*, gfec_AddPoint_nistp384_avx512, (IppsGFpECPoint* pR, + const IppsGFpECPoint* pP, + const IppsGFpECPoint* pQ, + IppsGFpECState* pEC)) +{ + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + ifmaArithMethod *pmeth = (ifmaArithMethod *)GFP_METHOD_ALT(pME); + + __ALIGN64 P384_POINT_IFMA P, R; + + BNU_CHUNK_T *pPool = cpGFpGetPool(3, pME); + + recode_point_to_mont52(&P, ECP_POINT_DATA(pP), pPool, pmeth, pME); + + if (pP == pQ) { + ifma_ec_nistp384_dbl_point(&R, &P); + } else { + __ALIGN64 P384_POINT_IFMA Q; + + recode_point_to_mont52(&Q, ECP_POINT_DATA(pQ), pPool, pmeth, pME); + + ifma_ec_nistp384_add_point(&R, &P, &Q); + } + + recode_point_to_mont64(pR, &R, pPool, pmeth, pME); + + cpGFpReleasePool(3, pME); + + ECP_POINT_FLAGS(pR) = gfec_IsPointAtInfinity(pR) ? 0 : ECP_FINITE_POINT; + return pR; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_addpoint_p521.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_addpoint_p521.c new file mode 100644 index 0000000..eb852f5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_addpoint_p521.c @@ -0,0 +1,60 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpstuff.h" +#include "pcpgfpecstuff.h" + +#include "ifma_arith_method_p521.h" +#include "ifma_ecpoint_p521.h" + +IPP_OWN_DEFN(IppsGFpECPoint*, gfec_AddPoint_nistp521_avx512, (IppsGFpECPoint* pR, + const IppsGFpECPoint* pP, + const IppsGFpECPoint* pQ, + IppsGFpECState* pEC)) +{ + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + ifmaArithMethod_p521 *pmeth = (ifmaArithMethod_p521 *)GFP_METHOD_ALT(pME); + + __ALIGN64 P521_POINT_IFMA P, R; + + BNU_CHUNK_T *pPool = cpGFpGetPool(3, pME); + + recode_point_to_mont52(&P, ECP_POINT_DATA(pP), pPool, pmeth, pME); + + if (pP == pQ) { + ifma_ec_nistp521_dbl_point(&R, &P); + } else { + __ALIGN64 P521_POINT_IFMA Q; + + recode_point_to_mont52(&Q, ECP_POINT_DATA(pQ), pPool, pmeth, pME); + + ifma_ec_nistp521_add_point(&R, &P, &Q); + } + + recode_point_to_mont64(pR, &R, pPool, pmeth, pME); + + cpGFpReleasePool(3, pME); + + ECP_POINT_FLAGS(pR) = gfec_IsPointAtInfinity(pR) ? 0 : ECP_FINITE_POINT; + return pR; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dh_p256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dh_p256.c new file mode 100644 index 0000000..d2d7917 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dh_p256.c @@ -0,0 +1,81 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpecstuff.h" +#include "pcpgfpstuff.h" + +#include "ifma_defs.h" +#include "ifma_ecpoint_p256.h" +#include "ifma_arith_method.h" + +IPP_OWN_DEFN(int, gfec_SharedSecretDH_nistp256_avx512, (IppsGFpECPoint * pR, + const IppsGFpECPoint *pP, + const BNU_CHUNK_T *pScalar, + int scalarLen, + IppsGFpECState *pEC, + Ipp8u *pScratchBuffer)) +{ + FIX_BNU(pScalar, scalarLen); + { + IPP_UNREFERENCED_PARAMETER(pScratchBuffer); + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + + const int orderBits = ECP_ORDBITSIZE(pEC); + const int orderLen = BITS_BNU_CHUNK(orderBits); + + ifmaArithMethod *pmeth = (ifmaArithMethod *)GFP_METHOD_ALT(pME); + + ifma_export from_radix52 = pmeth->export_to64; + + /* Mod engine (mod p) */ + ifma_decode p_from_mont = pmeth->decode; + + /* Copy scalar */ + BNU_CHUNK_T *pExtendedScalar = cpGFpGetPool(2, pME); + cpGFpElementCopyPad(pExtendedScalar, orderLen + 1, pScalar, scalarLen); + + __ALIGN64 P256_POINT_IFMA P52, R52; + BNU_CHUNK_T *pPool = cpGFpGetPool(3, pME); + + /* Convert point coordinates to a new Montgomery domain */ + recode_point_to_mont52(&P52, ECP_POINT_DATA(pP), pPool, pmeth, pME); + + ifma_ec_nistp256_mul_point(&R52, &P52, (Ipp8u *)pExtendedScalar, orderBits); + + + /* Check if the point - point to infinity */ + const mask8 is_zero_z = is_zero_i64(R52.z); + int finite_point = ((mask8)0xFF != is_zero_z); + + /* Get X affine coordinate */ + ifma_ec_nistp256_get_affine_coords(&(R52.x), NULL, &R52); + + R52.x = p_from_mont(R52.x); + from_radix52(ECP_POINT_X(pR), R52.x); + + cpGFpReleasePool(5, pME); + + return finite_point; + } +} + +#endif // IPP32E >= _IPP32E_K1 diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dh_p384.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dh_p384.c new file mode 100644 index 0000000..9fd110d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dh_p384.c @@ -0,0 +1,81 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpecstuff.h" +#include "pcpgfpstuff.h" + +#include "ifma_defs.h" +#include "ifma_ecpoint_p384.h" +#include "ifma_arith_method.h" + +IPP_OWN_DEFN(int, gfec_SharedSecretDH_nistp384_avx512, (IppsGFpECPoint * pR, + const IppsGFpECPoint *pP, + const BNU_CHUNK_T *pScalar, + int scalarLen, + IppsGFpECState *pEC, + Ipp8u *pScratchBuffer)) +{ + FIX_BNU(pScalar, scalarLen); + { + IPP_UNREFERENCED_PARAMETER(pScratchBuffer); + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + + const int orderBits = ECP_ORDBITSIZE(pEC); + const int orderLen = BITS_BNU_CHUNK(orderBits); + + ifmaArithMethod *pmeth = (ifmaArithMethod *)GFP_METHOD_ALT(pME); + + ifma_export from_radix52 = pmeth->export_to64; + + /* Mod engine (mod p) */ + ifma_decode p_from_mont = pmeth->decode; + + /* Copy scalar */ + BNU_CHUNK_T *pExtendedScalar = cpGFpGetPool(2, pME); + cpGFpElementCopyPad(pExtendedScalar, orderLen + 1, pScalar, scalarLen); + + __ALIGN64 P384_POINT_IFMA P52, R52; + BNU_CHUNK_T *pPool = cpGFpGetPool(3, pME); + + /* Convert point coordinates to a new Montgomery domain */ + recode_point_to_mont52(&P52, ECP_POINT_DATA(pP), pPool, pmeth, pME); + + ifma_ec_nistp384_mul_point(&R52, &P52, (Ipp8u *)pExtendedScalar, orderBits); + + + /* Check if the point - point to infinity */ + const mask8 is_zero_z = is_zero_i64(R52.z); + int finite_point = ((mask8)0xFF != is_zero_z); + + /* Get X affine coordinate */ + ifma_ec_nistp384_get_affine_coords(&(R52.x), NULL, &R52); + + R52.x = p_from_mont(R52.x); + from_radix52(ECP_POINT_X(pR), R52.x); + + cpGFpReleasePool(5, pME); + + return finite_point; + } +} + +#endif // IPP32E >= _IPP32E_K1 diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dh_p521.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dh_p521.c new file mode 100644 index 0000000..5d179d6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dh_p521.c @@ -0,0 +1,80 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpecstuff.h" +#include "pcpgfpstuff.h" + +#include "ifma_ecpoint_p521.h" +#include "ifma_arith_method_p521.h" + +IPP_OWN_DEFN(int, gfec_SharedSecretDH_nistp521_avx512, (IppsGFpECPoint * pR, + const IppsGFpECPoint *pP, + const BNU_CHUNK_T *pScalar, + int scalarLen, + IppsGFpECState *pEC, + Ipp8u *pScratchBuffer)) +{ + FIX_BNU(pScalar, scalarLen); + { + IPP_UNREFERENCED_PARAMETER(pScratchBuffer); + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + + const int orderBits = ECP_ORDBITSIZE(pEC); + const int orderLen = BITS_BNU_CHUNK(orderBits); + + ifmaArithMethod_p521 *pmeth = (ifmaArithMethod_p521 *)GFP_METHOD_ALT(pME); + + ifma_export from_radix52 = pmeth->export_to64; + + /* Mod engine (mod p) */ + ifma_decode p_from_mont = pmeth->decode; + + /* Copy scalar */ + BNU_CHUNK_T *pExtendedScalar = cpGFpGetPool(2, pME); + cpGFpElementCopyPad(pExtendedScalar, orderLen + 1, pScalar, scalarLen); + + __ALIGN64 P521_POINT_IFMA P52, R52; + BNU_CHUNK_T *pPool = cpGFpGetPool(3, pME); + + /* Convert point coordinates to a new Montgomery domain */ + recode_point_to_mont52(&P52, ECP_POINT_DATA(pP), pPool, pmeth, pME); + + ifma_ec_nistp521_mul_point(&R52, &P52, (Ipp8u *)pExtendedScalar, orderBits); + + + /* Check if the point - point to infinity */ + const mask8 is_zero_z = FE521_IS_ZERO(R52.z); + int finite_point = ((mask8)0xFF != is_zero_z); + + /* Get X affine coordinate */ + ifma_ec_nistp521_get_affine_coords(&(R52.x), NULL, &R52); + + p_from_mont(&(R52.x), R52.x); + from_radix52(ECP_POINT_X(pR), R52.x); + + cpGFpReleasePool(5, pME); + + return finite_point; + } +} + +#endif // IPP32E >= _IPP32E_K1 diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dsasign_p256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dsasign_p256.c new file mode 100644 index 0000000..18499c7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dsasign_p256.c @@ -0,0 +1,143 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpecstuff.h" +#include "pcpgfpstuff.h" + +#include "ifma_defs.h" +#include "ifma_ecpoint_p256.h" +#include "ifma_arith_method.h" + +IPP_OWN_DEFN(IppStatus, gfec_SignDSA_nistp256_avx512, (const IppsBigNumState *pMsgDigest, + const IppsBigNumState *pRegPrivate, + IppsBigNumState *pEphPrivate, + IppsBigNumState *pSignR, + IppsBigNumState *pSignS, + IppsGFpECState *pEC, + Ipp8u *pScratchBuffer)) +{ + IPP_UNREFERENCED_PARAMETER(pScratchBuffer); + + IppStatus sts = ippStsNoErr; + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + gsModEngine *nME = ECP_MONT_R(pEC); + + const int orderBits = ECP_ORDBITSIZE(pEC); + const int orderLen = BITS_BNU_CHUNK(orderBits); + + ifmaArithMethod *pmeth = (ifmaArithMethod *)GFP_METHOD_ALT(pME); + ifmaArithMethod *nmeth = (ifmaArithMethod *)GFP_METHOD_ALT(nME); + + ifma_import to_radix52 = pmeth->import_to52; + ifma_export from_radix52 = pmeth->export_to64; + + /* Mod engine (mod p) */ + ifma_decode p_from_mont = pmeth->decode; + + /* Mod engine (mod n - subgroup order) */ + ifma_encode n_to_mont = nmeth->encode; + ifma_decode n_from_mont = nmeth->decode; + ifma_mul n_mul = nmeth->mul; + ifma_add n_add = nmeth->add; + ifma_inv n_inv = nmeth->inv; + ifma_red n_red = nmeth->red; + + /* Copy scalar */ + BNU_CHUNK_T *pExtendedScalar = cpGFpGetPool(2, pME); + cpGFpElementCopyPad(pExtendedScalar, orderLen + 1, BN_NUMBER(pEphPrivate), BN_SIZE(pEphPrivate)); + + __ALIGN64 P256_POINT_IFMA P; + + if (ECP_PREMULBP(pEC)) { + ifma_ec_nistp256_mul_pointbase(&P, (Ipp8u *)pExtendedScalar, orderBits); + } else { + BNU_CHUNK_T* pPool = cpGFpGetPool(3, pME); + + /* Convert base point to a new Montgomery domain */ + __ALIGN64 P256_POINT_IFMA G52; + recode_point_to_mont52(&G52, ECP_G(pEC), pPool, pmeth, pME); + + ifma_ec_nistp256_mul_point(&P, &G52, (Ipp8u *)pExtendedScalar, orderBits); + + cpGFpReleasePool(3, pME); + } + + /* + // signR = int(ephPublic.x) (mod order) + */ + /* Extract affine P.x */ + ifma_ec_nistp256_get_affine_coords(&(P.x), NULL, &P); + + P.x = p_from_mont(P.x); + P.x = n_red(P.x); + + /* + // signS = (1/ephPrivate)*(pMsgDigest + private*signR) (mod order) + */ + m512 ephPrivateInv = setzero_i64(); + BNU_CHUNK_T *pTmp = cpGFpGetPool(1, pME); + ZEXPAND_COPY_BNU(pTmp, orderLen, BN_NUMBER(pEphPrivate), BN_SIZE(pEphPrivate)); + ephPrivateInv = to_radix52(pTmp); + + ephPrivateInv = n_to_mont(ephPrivateInv); + ephPrivateInv = n_inv(ephPrivateInv); + + /* Message */ + m512 msgDigest = setzero_i64(); + ZEXPAND_COPY_BNU(pTmp, orderLen, BN_NUMBER(pMsgDigest), BN_SIZE(pMsgDigest)); + msgDigest = to_radix52(pTmp); + msgDigest = n_red(msgDigest); /* reduce just in case */ + msgDigest = n_to_mont(msgDigest); + + /* Regular private key */ + m512 regPrivate = setzero_i64(); + ZEXPAND_COPY_BNU(pTmp, orderLen, BN_NUMBER(pRegPrivate), BN_SIZE(pRegPrivate)); + regPrivate = to_radix52(pTmp); + regPrivate = n_to_mont(regPrivate); + + m512 signS; + + signS = n_to_mont(P.x); + signS = n_mul(regPrivate, signS); + signS = n_add(signS, msgDigest); + signS = n_mul(signS, ephPrivateInv); + signS = n_from_mont(signS); + + const mask8 isZero = (mask8)(is_zero_i64(signS) | is_zero_i64(P.x)); + if ((mask8)0xFF == isZero) + sts = ippStsEphemeralKeyErr; + + from_radix52((Ipp64u *)pTmp, signS); + ZEXPAND_COPY_BNU(BN_NUMBER(pSignS), BN_SIZE(pSignS), pTmp, orderLen); + + from_radix52((Ipp64u *)pTmp, P.x); + ZEXPAND_COPY_BNU(BN_NUMBER(pSignR), BN_SIZE(pSignR), pTmp, orderLen); + + /* Clear secret data */ + clear_secrets(®Private, &(P.x), &signS); + + cpGFpReleasePool(3, pME); + + return sts; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dsasign_p384.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dsasign_p384.c new file mode 100644 index 0000000..d541903 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dsasign_p384.c @@ -0,0 +1,143 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpecstuff.h" +#include "pcpgfpstuff.h" + +#include "ifma_defs.h" +#include "ifma_ecpoint_p384.h" +#include "ifma_arith_method.h" + +IPP_OWN_DEFN(IppStatus, gfec_SignDSA_nistp384_avx512, (const IppsBigNumState *pMsgDigest, + const IppsBigNumState *pRegPrivate, + IppsBigNumState *pEphPrivate, + IppsBigNumState *pSignR, + IppsBigNumState *pSignS, + IppsGFpECState *pEC, + Ipp8u *pScratchBuffer)) +{ + IPP_UNREFERENCED_PARAMETER(pScratchBuffer); + + IppStatus sts = ippStsNoErr; + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + gsModEngine *nME = ECP_MONT_R(pEC); + + const int orderBits = ECP_ORDBITSIZE(pEC); + const int orderLen = BITS_BNU_CHUNK(orderBits); + + ifmaArithMethod *pmeth = (ifmaArithMethod *)GFP_METHOD_ALT(pME); + ifmaArithMethod *nmeth = (ifmaArithMethod *)GFP_METHOD_ALT(nME); + + ifma_import to_radix52 = pmeth->import_to52; + ifma_export from_radix52 = pmeth->export_to64; + + /* Mod engine (mod p) */ + ifma_decode p_from_mont = pmeth->decode; + + /* Mod engine (mod n - subgroup order) */ + ifma_encode n_to_mont = nmeth->encode; + ifma_decode n_from_mont = nmeth->decode; + ifma_mul n_mul = nmeth->mul; + ifma_add n_add = nmeth->add; + ifma_inv n_inv = nmeth->inv; + ifma_red n_red = nmeth->red; + + /* Copy scalar */ + BNU_CHUNK_T *pExtendedScalar = cpGFpGetPool(2, pME); + cpGFpElementCopyPad(pExtendedScalar, orderLen + 1, BN_NUMBER(pEphPrivate), BN_SIZE(pEphPrivate)); + + __ALIGN64 P384_POINT_IFMA P; + + if (ECP_PREMULBP(pEC)) { + ifma_ec_nistp384_mul_pointbase(&P, (Ipp8u *)pExtendedScalar, orderBits); + } else { + BNU_CHUNK_T* pPool = cpGFpGetPool(3, pME); + + /* Convert base point to a new Montgomery domain */ + __ALIGN64 P384_POINT_IFMA G52; + recode_point_to_mont52(&G52, ECP_G(pEC), pPool, pmeth, pME); + + ifma_ec_nistp384_mul_point(&P, &G52, (Ipp8u *)pExtendedScalar, orderBits); + + cpGFpReleasePool(3, pME); + } + + /* + // signR = int(ephPublic.x) (mod order) + */ + /* Extract affine P.x */ + ifma_ec_nistp384_get_affine_coords(&(P.x), NULL, &P); + + P.x = p_from_mont(P.x); + P.x = n_red(P.x); + + /* + // signS = (1/ephPrivate)*(pMsgDigest + private*signR) (mod order) + */ + m512 ephPrivateInv = setzero_i64(); + BNU_CHUNK_T *pTmp = cpGFpGetPool(1, pME); + ZEXPAND_COPY_BNU(pTmp, orderLen, BN_NUMBER(pEphPrivate), BN_SIZE(pEphPrivate)); + ephPrivateInv = to_radix52(pTmp); + + ephPrivateInv = n_to_mont(ephPrivateInv); + ephPrivateInv = n_inv(ephPrivateInv); + + /* Message */ + m512 msgDigest = setzero_i64(); + ZEXPAND_COPY_BNU(pTmp, orderLen, BN_NUMBER(pMsgDigest), BN_SIZE(pMsgDigest)); + msgDigest = to_radix52(pTmp); + msgDigest = n_red(msgDigest); /* reduce just in case */ + msgDigest = n_to_mont(msgDigest); + + /* Regular private key */ + m512 regPrivate = setzero_i64(); + ZEXPAND_COPY_BNU(pTmp, orderLen, BN_NUMBER(pRegPrivate), BN_SIZE(pRegPrivate)); + regPrivate = to_radix52(pTmp); + regPrivate = n_to_mont(regPrivate); + + m512 signS; + + signS = n_to_mont(P.x); + signS = n_mul(regPrivate, signS); + signS = n_add(signS, msgDigest); + signS = n_mul(signS, ephPrivateInv); + signS = n_from_mont(signS); + + const mask8 isZero = (mask8)(is_zero_i64(signS) | is_zero_i64(P.x)); + if ((mask8)0xFF == isZero) + sts = ippStsEphemeralKeyErr; + + from_radix52((Ipp64u *)pTmp, signS); + ZEXPAND_COPY_BNU(BN_NUMBER(pSignS), BN_SIZE(pSignS), pTmp, orderLen); + + from_radix52((Ipp64u *)pTmp, P.x); + ZEXPAND_COPY_BNU(BN_NUMBER(pSignR), BN_SIZE(pSignR), pTmp, orderLen); + + /* Clear secret data */ + clear_secrets(®Private, &(P.x), &signS); + + cpGFpReleasePool(3, pME); + + return sts; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dsasign_p521.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dsasign_p521.c new file mode 100644 index 0000000..345173c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dsasign_p521.c @@ -0,0 +1,146 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpecstuff.h" +#include "pcpgfpstuff.h" + +#include "ifma_ecpoint_p521.h" +#include "ifma_arith_method_p521.h" +#include "ifma_alias_avx512vl.h" + +IPP_OWN_DEFN(IppStatus, gfec_SignDSA_nistp521_avx512, (const IppsBigNumState *pMsgDigest, + const IppsBigNumState *pRegPrivate, + IppsBigNumState *pEphPrivate, + IppsBigNumState *pSignR, + IppsBigNumState *pSignS, + IppsGFpECState *pEC, + Ipp8u *pScratchBuffer)) +{ + IPP_UNREFERENCED_PARAMETER(pScratchBuffer); + + IppStatus sts = ippStsNoErr; + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + gsModEngine *nME = ECP_MONT_R(pEC); + + const int orderBits = ECP_ORDBITSIZE(pEC); + const int orderLen = BITS_BNU_CHUNK(orderBits); + + ifmaArithMethod_p521 *pmeth = (ifmaArithMethod_p521 *)GFP_METHOD_ALT(pME); + ifmaArithMethod_p521 *nmeth = (ifmaArithMethod_p521 *)GFP_METHOD_ALT(nME); + + ifma_import to_radix52 = pmeth->import_to52; + ifma_export from_radix52 = pmeth->export_to64; + + /* Mod engine (mod p) */ + ifma_decode p_from_mont = pmeth->decode; + + /* Mod engine (mod n - subgroup order) */ + ifma_encode n_to_mont = nmeth->encode; + ifma_decode n_from_mont = nmeth->decode; + ifma_mul n_mul = nmeth->mul; + ifma_add n_add = nmeth->add; + ifma_inv n_inv = nmeth->inv; + ifma_red n_red = nmeth->red; + + /* Copy scalar */ + BNU_CHUNK_T *pExtendedScalar = cpGFpGetPool(2, pME); + cpGFpElementCopyPad(pExtendedScalar, orderLen + 1, BN_NUMBER(pEphPrivate), BN_SIZE(pEphPrivate)); + + __ALIGN64 P521_POINT_IFMA P; + + if (ECP_PREMULBP(pEC)) { + ifma_ec_nistp521_mul_pointbase(&P, (Ipp8u *)pExtendedScalar, orderBits); + } else { + BNU_CHUNK_T *pPool = cpGFpGetPool(3, pME); + + /* Convert base point to a new Montgomery domain */ + __ALIGN64 P521_POINT_IFMA G52; + recode_point_to_mont52(&G52, ECP_G(pEC), pPool, pmeth, pME); + + ifma_ec_nistp521_mul_point(&P, &G52, (Ipp8u *)pExtendedScalar, orderBits); + + cpGFpReleasePool(3, pME); + } + + /* + // signR = int(ephPublic.x) (mod order) + */ + /* Extract affine P.x */ + ifma_ec_nistp521_get_affine_coords(&(P.x), NULL, &P); + + p_from_mont(&(P.x), P.x); + n_red(&(P.x), P.x); + + /* + // signS = (1/ephPrivate)*(pMsgDigest + private*signR) (mod order) + */ + fe521 ephPrivateInv; + FE521_SET(ephPrivateInv) = m256_setzero_i64(); + BNU_CHUNK_T *pTmp = cpGFpGetPool(1, pME); + ZEXPAND_COPY_BNU(pTmp, orderLen, BN_NUMBER(pEphPrivate), BN_SIZE(pEphPrivate)); + to_radix52(&ephPrivateInv, pTmp); + + n_to_mont(&ephPrivateInv, ephPrivateInv); + n_inv(&ephPrivateInv, ephPrivateInv); + + /* Message */ + fe521 msgDigest; + FE521_SET(msgDigest) = m256_setzero_i64(); + ZEXPAND_COPY_BNU(pTmp, orderLen, BN_NUMBER(pMsgDigest), BN_SIZE(pMsgDigest)); + to_radix52(&msgDigest, pTmp); + n_red(&msgDigest, msgDigest); /* reduce just in case */ + n_to_mont(&msgDigest, msgDigest); + + /* Regular private key */ + fe521 regPrivate; + FE521_SET(regPrivate) = m256_setzero_i64(); + ZEXPAND_COPY_BNU(pTmp, orderLen, BN_NUMBER(pRegPrivate), BN_SIZE(pRegPrivate)); + to_radix52(®Private, pTmp); + n_to_mont(®Private, regPrivate); + + fe521 signS; + + n_to_mont(&signS, P.x); + n_mul(&signS, regPrivate, signS); + n_add(&signS, signS, msgDigest); + n_mul(&signS, signS, ephPrivateInv); + n_from_mont(&signS, signS); + + const mask8 isZero = (mask8)(FE521_IS_ZERO(signS) | FE521_IS_ZERO(P.x)); + if ((mask8)0xFF == isZero) + sts = ippStsEphemeralKeyErr; + + from_radix52((Ipp64u *)pTmp, signS); + ZEXPAND_COPY_BNU(BN_NUMBER(pSignS), BN_SIZE(pSignS), pTmp, orderLen); + + from_radix52((Ipp64u *)pTmp, P.x); + ZEXPAND_COPY_BNU(BN_NUMBER(pSignR), BN_SIZE(pSignR), pTmp, orderLen); + + /* Clear secret data */ + clear_secrets(®Private, &(P.x), &signS); + + cpGFpReleasePool(3, pME); + + return sts; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dsaverify_p256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dsaverify_p256.c new file mode 100644 index 0000000..ac9f21c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dsaverify_p256.c @@ -0,0 +1,145 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpstuff.h" +#include "pcpgfpecstuff.h" + +#include "ifma_arith_method.h" +#include "ifma_ecpoint_p256.h" + +IPP_OWN_DEFN(IppECResult, gfec_VerifyDSA_nistp256_avx512, (const IppsBigNumState* pMsgDigest, + const IppsGFpECPoint* pRegPublic, + const IppsBigNumState* pSignR, + const IppsBigNumState* pSignS, + IppsGFpECState* pEC, + Ipp8u* pScratchBuffer)) +{ + IPP_UNREFERENCED_PARAMETER(pScratchBuffer); + + IppECResult verifyResult = ippECInvalidSignature; + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + gsModEngine *nME = ECP_MONT_R(pEC); + + const int orderBits = ECP_ORDBITSIZE(pEC); + const int orderLen = BITS_BNU_CHUNK(orderBits); + + ifmaArithMethod *pmeth = (ifmaArithMethod *)GFP_METHOD_ALT(pME); + ifmaArithMethod *nmeth = (ifmaArithMethod *)GFP_METHOD_ALT(nME); + + ifma_import to_radix52 = pmeth->import_to52; + ifma_export from_radix52 = pmeth->export_to64; + + /* Mod engine (mod p) */ + ifma_decode p_from_mont = pmeth->decode; + + /* Mod engine (mod n - subgroup order) */ + ifma_encode n_to_mont = nmeth->encode; + ifma_decode n_from_mont = nmeth->decode; + ifma_mul n_mul = nmeth->mul; + ifma_inv n_inv = nmeth->inv; + ifma_red n_red = nmeth->red; + + /* Convert input parameters to 2^52 radix */ + m512 msg, signR, signS; + msg = signR = signS = setzero_i64(); + + const int elemLen = GFP_FELEN(pME); + + BNU_CHUNK_T* pPool = cpGFpGetPool(3, pME); + BNU_CHUNK_T *pBufMsg = pPool; + BNU_CHUNK_T *pBufSignR = pPool + elemLen; + BNU_CHUNK_T *pBufSignS = pPool + 2 * elemLen; + + ZEXPAND_COPY_BNU(pBufMsg, orderLen, BN_NUMBER(pMsgDigest), BN_SIZE(pMsgDigest)); + ZEXPAND_COPY_BNU(pBufSignR, orderLen, BN_NUMBER(pSignR), BN_SIZE(pSignR)); + ZEXPAND_COPY_BNU(pBufSignS, orderLen, BN_NUMBER(pSignS), BN_SIZE(pSignS)); + + msg = to_radix52((Ipp64u *)pBufMsg); + msg = n_red(msg); /* reduce just in case */ + signR = to_radix52((Ipp64u *)pBufSignR); + signS = to_radix52((Ipp64u *)pBufSignS); + + /* Convert public point to proper Montgomery domain and 2^52 radix */ + __ALIGN64 P256_POINT_IFMA pubKey; + recode_point_to_mont52(&pubKey, ECP_POINT_DATA(pRegPublic), pPool /* 3 elem */, pmeth, pME); + + m512 h, h1, h2; + + /* h = (signS)^(-1) */ + h = n_to_mont(signS); + h = n_inv(h); + + /* h1 = msg*h, h2 = signR*h */ + h1 = n_to_mont(msg); + h2 = n_to_mont(signR); + + h1 = n_mul(h1, h); + h2 = n_mul(h2, h); + + h1 = n_from_mont(h1); + h2 = n_from_mont(h2); + + BNU_CHUNK_T *pExtendedH1 = cpGFpGetPool(2, pME); + BNU_CHUNK_T *pExtendedH2 = cpGFpGetPool(2, pME); + BNU_CHUNK_T *pH1 = cpGFpGetPool(1, pME); + BNU_CHUNK_T *pH2 = cpGFpGetPool(1, pME); + + from_radix52((Ipp64u *)pH1, h1); + from_radix52((Ipp64u *)pH2, h2); + cpGFpElementCopyPad(pExtendedH1, orderLen + 1, pH1, orderLen); + cpGFpElementCopyPad(pExtendedH2, orderLen + 1, pH2, orderLen); + + cpGFpReleasePool(2, pME); /* pH1, pH2 */ + + __ALIGN64 P256_POINT_IFMA P; + + /* P = h1*basePoint + h2*pubKey */ + ifma_ec_nistp256_mul_point(&pubKey, &pubKey, (Ipp8u *)pExtendedH2, orderBits); + + if (ECP_PREMULBP(pEC)) { + ifma_ec_nistp256_mul_pointbase(&P, (Ipp8u *)pExtendedH1, orderBits); + } else { + /* Convert base point to a new Montgomery domain */ + __ALIGN64 P256_POINT_IFMA G52; + recode_point_to_mont52(&G52, ECP_G(pEC), pPool /* 3 elem */, pmeth, pME); + + ifma_ec_nistp256_mul_point(&P, &G52, (Ipp8u *)pExtendedH1, orderBits); + } + + ifma_ec_nistp256_add_point(&P, &P, &pubKey); + + /* Get X in affine coordinates */ + ifma_ec_nistp256_get_affine_coords(&(P.x), NULL, &P); + + P.x = p_from_mont(P.x); + P.x = n_red(P.x); + + const mask8 mask_ok = cmp_i64_mask(P.x, signR, _MM_CMPINT_EQ); + if ((mask8)0xFF == mask_ok) + verifyResult = ippECValid; + + cpGFpReleasePool(7, pME); + + return verifyResult; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dsaverify_p384.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dsaverify_p384.c new file mode 100644 index 0000000..3ab461e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dsaverify_p384.c @@ -0,0 +1,147 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpstuff.h" +#include "pcpgfpecstuff.h" + +#include "ifma_arith_method.h" +#include "ifma_ecpoint_p384.h" + +IPP_OWN_DEFN(IppECResult, gfec_VerifyDSA_nistp384_avx512, (const IppsBigNumState* pMsgDigest, + const IppsGFpECPoint* pRegPublic, + const IppsBigNumState* pSignR, + const IppsBigNumState* pSignS, + IppsGFpECState* pEC, + Ipp8u* pScratchBuffer)) +{ + IPP_UNREFERENCED_PARAMETER(pScratchBuffer); + + IppECResult verifyResult = ippECInvalidSignature; + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + gsModEngine *nME = ECP_MONT_R(pEC); + + const int orderBits = ECP_ORDBITSIZE(pEC); + const int orderLen = BITS_BNU_CHUNK(orderBits); + + ifmaArithMethod *pmeth = (ifmaArithMethod *)GFP_METHOD_ALT(pME); + ifmaArithMethod *nmeth = (ifmaArithMethod *)GFP_METHOD_ALT(nME); + + ifma_import to_radix52 = pmeth->import_to52; + ifma_export from_radix52 = pmeth->export_to64; + + /* Mod engine (mod p) */ + ifma_decode p_from_mont = pmeth->decode; + + /* Mod engine (mod n - subgroup order) */ + ifma_encode n_to_mont = nmeth->encode; + ifma_decode n_from_mont = nmeth->decode; + ifma_mul n_mul = nmeth->mul; + ifma_inv n_inv = nmeth->inv; + ifma_red n_red = nmeth->red; + + /* Convert input parameters to 2^52 radix */ + m512 msg, signR, signS; + msg = signR = signS = setzero_i64(); + + const int elemLen = GFP_FELEN(pME); + + BNU_CHUNK_T* pPool = cpGFpGetPool(3, pME); + BNU_CHUNK_T *pBufMsg = pPool; + BNU_CHUNK_T *pBufSignR = pPool + elemLen; + BNU_CHUNK_T *pBufSignS = pPool + 2 * elemLen; + + ZEXPAND_COPY_BNU(pBufMsg, orderLen, BN_NUMBER(pMsgDigest), BN_SIZE(pMsgDigest)); + ZEXPAND_COPY_BNU(pBufSignR, orderLen, BN_NUMBER(pSignR), BN_SIZE(pSignR)); + ZEXPAND_COPY_BNU(pBufSignS, orderLen, BN_NUMBER(pSignS), BN_SIZE(pSignS)); + + msg = to_radix52((Ipp64u *)pBufMsg); + msg = n_red(msg); /* reduce just in case */ + signR = to_radix52((Ipp64u *)pBufSignR); + signS = to_radix52((Ipp64u *)pBufSignS); + + /* Convert public point to proper Montgomery domain and 2^52 radix */ + __ALIGN64 P384_POINT_IFMA pubKey; + recode_point_to_mont52(&pubKey, ECP_POINT_DATA(pRegPublic), pPool /* 3 elem */, pmeth, pME); + + m512 h, h1, h2; + h = h1 = h2 = setzero_i64(); + + /* h = (signS)^(-1) */ + h = n_to_mont(signS); + h = n_inv(h); + + /* h1 = msg*h, h2 = signR*h */ + h1 = n_to_mont(msg); + h2 = n_to_mont(signR); + + h1 = n_mul(h1, h); + h2 = n_mul(h2, h); + + h1 = n_from_mont(h1); + h2 = n_from_mont(h2); + + BNU_CHUNK_T *pExtendedH1 = cpGFpGetPool(2, pME); + BNU_CHUNK_T *pExtendedH2 = cpGFpGetPool(2, pME); + BNU_CHUNK_T *pH1 = cpGFpGetPool(1, pME); + BNU_CHUNK_T *pH2 = cpGFpGetPool(1, pME); + + from_radix52((Ipp64u *)pH1, h1); + from_radix52((Ipp64u *)pH2, h2); + cpGFpElementCopyPad(pExtendedH1, orderLen + 1, pH1, orderLen); + cpGFpElementCopyPad(pExtendedH2, orderLen + 1, pH2, orderLen); + + cpGFpReleasePool(2, pME); /* pH1, pH2 */ + + __ALIGN64 P384_POINT_IFMA P; + P.x = P.y = P.z = setzero_i64(); + + /* P = h1*basePoint + h2*pubKey */ + ifma_ec_nistp384_mul_point(&pubKey, &pubKey, (Ipp8u *)pExtendedH2, orderBits); + + if (ECP_PREMULBP(pEC)) { + ifma_ec_nistp384_mul_pointbase(&P, (Ipp8u *)pExtendedH1, orderBits); + } else { + /* Convert base point to a new Montgomery domain */ + __ALIGN64 P384_POINT_IFMA G52; + recode_point_to_mont52(&G52, ECP_G(pEC), pPool /* 3 elem */, pmeth, pME); + + ifma_ec_nistp384_mul_point(&P, &G52, (Ipp8u *)pExtendedH1, orderBits); + } + + ifma_ec_nistp384_add_point(&P, &P, &pubKey); + + /* Get X in affine coordinates */ + ifma_ec_nistp384_get_affine_coords(&(P.x), NULL, &P); + + P.x = p_from_mont(P.x); + P.x = n_red(P.x); + + const mask8 mask_ok = cmp_i64_mask(P.x, signR, _MM_CMPINT_EQ); + if ((mask8)0xFF == mask_ok) + verifyResult = ippECValid; + + cpGFpReleasePool(7, pME); + + return verifyResult; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dsaverify_p521.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dsaverify_p521.c new file mode 100644 index 0000000..b45827c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_dsaverify_p521.c @@ -0,0 +1,147 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpstuff.h" +#include "pcpgfpecstuff.h" + +#include "ifma_arith_method_p521.h" +#include "ifma_ecpoint_p521.h" + +IPP_OWN_DEFN(IppECResult, gfec_VerifyDSA_nistp521_avx512, (const IppsBigNumState* pMsgDigest, + const IppsGFpECPoint* pRegPublic, + const IppsBigNumState* pSignR, + const IppsBigNumState* pSignS, + IppsGFpECState* pEC, + Ipp8u* pScratchBuffer)) +{ + IPP_UNREFERENCED_PARAMETER(pScratchBuffer); + + IppECResult verifyResult = ippECInvalidSignature; + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + gsModEngine *nME = ECP_MONT_R(pEC); + + const int orderBits = ECP_ORDBITSIZE(pEC); + const int orderLen = BITS_BNU_CHUNK(orderBits); + + ifmaArithMethod_p521 *pmeth = (ifmaArithMethod_p521 *)GFP_METHOD_ALT(pME); + ifmaArithMethod_p521 *nmeth = (ifmaArithMethod_p521 *)GFP_METHOD_ALT(nME); + + ifma_import to_radix52 = pmeth->import_to52; + ifma_export from_radix52 = pmeth->export_to64; + + /* Mod engine (mod p) */ + ifma_decode p_from_mont = pmeth->decode; + + /* Mod engine (mod n - subgroup order) */ + ifma_encode n_to_mont = nmeth->encode; + ifma_decode n_from_mont = nmeth->decode; + ifma_mul n_mul = nmeth->mul; + ifma_inv n_inv = nmeth->inv; + ifma_red n_red = nmeth->red; + + /* Convert input parameters to 2^52 radix */ + fe521 msg, signR, signS; + FE521_SET(msg) = FE521_SET(signR) = FE521_SET(signS) = m256_setzero_i64(); + + const int elemLen = GFP_FELEN(pME); + + BNU_CHUNK_T *pPool = cpGFpGetPool(3, pME); + BNU_CHUNK_T *pBufMsg = pPool; + BNU_CHUNK_T *pBufSignR = pPool + elemLen; + BNU_CHUNK_T *pBufSignS = pPool + 2 * elemLen; + + ZEXPAND_COPY_BNU(pBufMsg, orderLen, BN_NUMBER(pMsgDigest), BN_SIZE(pMsgDigest)); + ZEXPAND_COPY_BNU(pBufSignR, orderLen, BN_NUMBER(pSignR), BN_SIZE(pSignR)); + ZEXPAND_COPY_BNU(pBufSignS, orderLen, BN_NUMBER(pSignS), BN_SIZE(pSignS)); + + to_radix52(&msg, (Ipp64u *)pBufMsg); + n_red(&msg, msg); /* reduce just in case */ + to_radix52(&signR, (Ipp64u *)pBufSignR); + to_radix52(&signS, (Ipp64u *)pBufSignS); + + /* Convert public point to proper Montgomery domain and 2^52 radix */ + __ALIGN64 P521_POINT_IFMA pubKey; + recode_point_to_mont52(&pubKey, ECP_POINT_DATA(pRegPublic), pPool /* 3 elem */, pmeth, pME); + + fe521 h, h1, h2; + FE521_SET(h) = FE521_SET(h1) = FE521_SET(h2) = m256_setzero_i64(); + + /* h = (signS)^(-1) */ + n_to_mont(&h, signS); + n_inv(&h, h); + + /* h1 = msg*h, h2 = signR*h */ + n_to_mont(&h1, msg); + n_to_mont(&h2, signR); + + n_mul(&h1, h1, h); + n_mul(&h2, h2, h); + + n_from_mont(&h1, h1); + n_from_mont(&h2, h2); + + BNU_CHUNK_T *pExtendedH1 = cpGFpGetPool(2, pME); + BNU_CHUNK_T *pExtendedH2 = cpGFpGetPool(2, pME); + BNU_CHUNK_T *pH1 = cpGFpGetPool(1, pME); + BNU_CHUNK_T *pH2 = cpGFpGetPool(1, pME); + + from_radix52((Ipp64u *)pH1, h1); + from_radix52((Ipp64u *)pH2, h2); + cpGFpElementCopyPad(pExtendedH1, orderLen + 1, pH1, orderLen); + cpGFpElementCopyPad(pExtendedH2, orderLen + 1, pH2, orderLen); + + cpGFpReleasePool(2, pME); /* pH1, pH2 */ + + __ALIGN64 P521_POINT_IFMA P; + FE521_SET(P.x) = FE521_SET(P.y) = FE521_SET(P.z) = m256_setzero_i64(); + + /* P = h1*basePoint + h2*pubKey */ + ifma_ec_nistp521_mul_point(&pubKey, &pubKey, (Ipp8u *)pExtendedH2, orderBits); + + if (ECP_PREMULBP(pEC)) { + ifma_ec_nistp521_mul_pointbase(&P, (Ipp8u *)pExtendedH1, orderBits); + } else { + /* Convert base point to a new Montgomery domain */ + __ALIGN64 P521_POINT_IFMA G52; + recode_point_to_mont52(&G52, ECP_G(pEC), pPool /* 3 elem */, pmeth, pME); + + ifma_ec_nistp521_mul_point(&P, &G52, (Ipp8u *)pExtendedH1, orderBits); + } + + ifma_ec_nistp521_add_point(&P, &P, &pubKey); + + /* Get X in affine coordinates */ + ifma_ec_nistp521_get_affine_coords(&(P.x), NULL, &P); + + p_from_mont(&(P.x), P.x); + n_red(&(P.x), P.x); + + const mask8 mask_ok = FE521_CMP_MASK(P.x, signR, _MM_CMPINT_EQ); + if ((mask8)0xF == mask_ok) + verifyResult = ippECValid; + + cpGFpReleasePool(7, pME); + + return verifyResult; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_mulpoint_p256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_mulpoint_p256.c new file mode 100644 index 0000000..e7b477a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_mulpoint_p256.c @@ -0,0 +1,65 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpstuff.h" +#include "pcpgfpecstuff.h" + +#include "ifma_arith_method.h" +#include "ifma_ecpoint_p256.h" + +IPP_OWN_DEFN (IppsGFpECPoint*, gfec_MulPoint_nistp256_avx512, (IppsGFpECPoint* pR, + const IppsGFpECPoint* pP, + const BNU_CHUNK_T* pScalar, + int scalarLen, + IppsGFpECState* pEC, + Ipp8u* pScratchBuffer)) +{ + IPP_UNREFERENCED_PARAMETER(pScratchBuffer); + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + ifmaArithMethod *pmeth = (ifmaArithMethod *)GFP_METHOD_ALT(pME); + + const int orderBits = ECP_ORDBITSIZE(pEC); + const int orderLen = BITS_BNU_CHUNK(orderBits); + const int elemLen = GFP_PELEN(pME); + + BNU_CHUNK_T *pPool = cpGFpGetPool(5, pME); + BNU_CHUNK_T *pExtendedScalar = pPool; /* 2 pool elem to hold scalar */ + BNU_CHUNK_T *pPointPool = pPool + 2 * elemLen; /* 3 pool elem to to hold 3 point coordinates */ + + /* Copy scalar */ + cpGFpElementCopyPad(pExtendedScalar, orderLen + 1, pScalar, scalarLen); + + __ALIGN64 P256_POINT_IFMA P, R; + + recode_point_to_mont52(&P, ECP_POINT_DATA(pP), pPointPool, pmeth, pME); + + ifma_ec_nistp256_mul_point(&R, &P, (Ipp8u*)pExtendedScalar, orderBits); + + recode_point_to_mont64(pR, &R, pPointPool, pmeth, pME); + + cpGFpReleasePool(5, pME); + + ECP_POINT_FLAGS(pR) = gfec_IsPointAtInfinity(pR) ? 0 : ECP_FINITE_POINT; + return pR; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_mulpoint_p384.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_mulpoint_p384.c new file mode 100644 index 0000000..568acde --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_mulpoint_p384.c @@ -0,0 +1,65 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpstuff.h" +#include "pcpgfpecstuff.h" + +#include "ifma_arith_method.h" +#include "ifma_ecpoint_p384.h" + +IPP_OWN_DEFN (IppsGFpECPoint*, gfec_MulPoint_nistp384_avx512, (IppsGFpECPoint* pR, + const IppsGFpECPoint* pP, + const BNU_CHUNK_T* pScalar, + int scalarLen, + IppsGFpECState* pEC, + Ipp8u* pScratchBuffer)) +{ + IPP_UNREFERENCED_PARAMETER(pScratchBuffer); + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + ifmaArithMethod *pmeth = (ifmaArithMethod *)GFP_METHOD_ALT(pME); + + const int orderBits = ECP_ORDBITSIZE(pEC); + const int orderLen = BITS_BNU_CHUNK(orderBits); + const int elemLen = GFP_PELEN(pME); + + BNU_CHUNK_T *pPool = cpGFpGetPool(5, pME); + BNU_CHUNK_T *pExtendedScalar = pPool; /* 2 pool elem to hold scalar */ + BNU_CHUNK_T *pPointPool = pPool + 2 * elemLen; /* 3 pool elem to to hold 3 point coordinates */ + + /* Copy scalar */ + cpGFpElementCopyPad(pExtendedScalar, orderLen + 1, pScalar, scalarLen); + + __ALIGN64 P384_POINT_IFMA P, R; + + recode_point_to_mont52(&P, ECP_POINT_DATA(pP), pPointPool, pmeth, pME); + + ifma_ec_nistp384_mul_point(&R, &P, (Ipp8u*)pExtendedScalar, orderBits); + + recode_point_to_mont64(pR, &R, pPointPool, pmeth, pME); + + cpGFpReleasePool(5, pME); + + ECP_POINT_FLAGS(pR) = gfec_IsPointAtInfinity(pR) ? 0 : ECP_FINITE_POINT; + return pR; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_mulpoint_p521.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_mulpoint_p521.c new file mode 100644 index 0000000..e094acb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_mulpoint_p521.c @@ -0,0 +1,65 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpstuff.h" +#include "pcpgfpecstuff.h" + +#include "ifma_arith_method_p521.h" +#include "ifma_ecpoint_p521.h" + +IPP_OWN_DEFN (IppsGFpECPoint*, gfec_MulPoint_nistp521_avx512, (IppsGFpECPoint* pR, + const IppsGFpECPoint* pP, + const BNU_CHUNK_T* pScalar, + int scalarLen, + IppsGFpECState* pEC, + Ipp8u* pScratchBuffer)) +{ + IPP_UNREFERENCED_PARAMETER(pScratchBuffer); + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + ifmaArithMethod_p521 *pmeth = (ifmaArithMethod_p521 *)GFP_METHOD_ALT(pME); + + const int orderBits = ECP_ORDBITSIZE(pEC); + const int orderLen = BITS_BNU_CHUNK(orderBits); + const int elemLen = GFP_PELEN(pME); + + BNU_CHUNK_T *pPool = cpGFpGetPool(5, pME); + BNU_CHUNK_T *pExtendedScalar = pPool; /* 2 pool elem to hold scalar */ + BNU_CHUNK_T *pPointPool = pPool + 2 * elemLen; /* 3 pool elem to to hold 3 point coordinates */ + + /* Copy scalar */ + cpGFpElementCopyPad(pExtendedScalar, orderLen + 1, pScalar, scalarLen); + + __ALIGN64 P521_POINT_IFMA P, R; + + recode_point_to_mont52(&P, ECP_POINT_DATA(pP), pPointPool, pmeth, pME); + + ifma_ec_nistp521_mul_point(&R, &P, (Ipp8u*)pExtendedScalar, orderBits); + + recode_point_to_mont64(pR, &R, pPointPool, pmeth, pME); + + cpGFpReleasePool(5, pME); + + ECP_POINT_FLAGS(pR) = gfec_IsPointAtInfinity(pR) ? 0 : ECP_FINITE_POINT; + return pR; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_on_curve_p256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_on_curve_p256.c new file mode 100644 index 0000000..c671edf --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_on_curve_p256.c @@ -0,0 +1,45 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpecstuff.h" +#include "pcpgfpstuff.h" + +#include +#include + +IPP_OWN_DEFN(int, gfec_point_on_curve_nistp256_avx512, (const IppsGFpECPoint *pPoint, IppsGFpECState *pEC)) +{ + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + ifmaArithMethod *pmeth = (ifmaArithMethod *)GFP_METHOD_ALT(pME); + + BNU_CHUNK_T *pPool = cpGFpGetPool(3, pME); + + __ALIGN64 P256_POINT_IFMA P; + + recode_point_to_mont52(&P, ECP_POINT_DATA(pPoint), pPool /* 3 elem */, pmeth, pME); + + const int onCurve = ifma_ec_nistp256_is_on_curve(&P, /* use_jproj_coord = */ !IS_ECP_AFFINE_POINT(pPoint)); + + cpGFpReleasePool(3, pME); + return onCurve; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_on_curve_p384.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_on_curve_p384.c new file mode 100644 index 0000000..4f53c59 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_on_curve_p384.c @@ -0,0 +1,45 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpecstuff.h" +#include "pcpgfpstuff.h" + +#include +#include + +IPP_OWN_DEFN(int, gfec_point_on_curve_nistp384_avx512, (const IppsGFpECPoint *pPoint, IppsGFpECState *pEC)) +{ + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + ifmaArithMethod *pmeth = (ifmaArithMethod *)GFP_METHOD_ALT(pME); + + BNU_CHUNK_T *pPool = cpGFpGetPool(3, pME); + + __ALIGN64 P384_POINT_IFMA P; + + recode_point_to_mont52(&P, ECP_POINT_DATA(pPoint), pPool /* 3 elem */, pmeth, pME); + + const int onCurve = ifma_ec_nistp384_is_on_curve(&P, /* use_jproj_coord = */ !IS_ECP_AFFINE_POINT(pPoint)); + + cpGFpReleasePool(3, pME); + return onCurve; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_on_curve_p521.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_on_curve_p521.c new file mode 100644 index 0000000..4e3eed8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_on_curve_p521.c @@ -0,0 +1,45 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpecstuff.h" +#include "pcpgfpstuff.h" + +#include +#include + +IPP_OWN_DEFN(int, gfec_point_on_curve_nistp521_avx512, (const IppsGFpECPoint *pPoint, IppsGFpECState *pEC)) +{ + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + ifmaArithMethod_p521 *pmeth = (ifmaArithMethod_p521 *)GFP_METHOD_ALT(pME); + + BNU_CHUNK_T *pPool = cpGFpGetPool(3, pME); + + __ALIGN64 P521_POINT_IFMA P; + + recode_point_to_mont52(&P, ECP_POINT_DATA(pPoint), pPool /* 3 elem */, pmeth, pME); + + const int onCurve = ifma_ec_nistp521_is_on_curve(&P, /* use_jproj_coord = */ !IS_ECP_AFFINE_POINT(pPoint)); + + cpGFpReleasePool(3, pME); + return onCurve; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_pubkey_p256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_pubkey_p256.c new file mode 100644 index 0000000..70a97af --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_pubkey_p256.c @@ -0,0 +1,75 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpstuff.h" +#include "pcpgfpecstuff.h" + +#include "ifma_ecpoint_p256.h" +#include "ifma_arith_p256.h" +#include "ifma_arith_method.h" + +IPP_OWN_DEFN(IppsGFpECPoint*, gfec_PubKey_nist256_avx512, (IppsGFpECPoint * pR, + const BNU_CHUNK_T* pScalar, + int scalarLen, IppsGFpECState* pEC, + Ipp8u* pScratchBuffer)) +{ + FIX_BNU(pScalar, scalarLen); + { + IPP_UNREFERENCED_PARAMETER(pScratchBuffer); + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + gsModEngine *nME = ECP_MONT_R(pEC); + + const int orderBits = ECP_ORDBITSIZE(pEC); + const int orderLen = BITS_BNU_CHUNK(orderBits); + + ifmaArithMethod *pmeth = (ifmaArithMethod *)GFP_METHOD_ALT(pME); + + BNU_CHUNK_T *pPool = cpGFpGetPool(5, nME); + BNU_CHUNK_T *pExtendedScalar = pPool; + BNU_CHUNK_T *pPointPool = pExtendedScalar + 2 * GFP_FELEN(pME); + + /* Copy scalar */ + cpGFpElementCopyPad(pExtendedScalar, orderLen + 1, pScalar, scalarLen); + + __ALIGN64 P256_POINT_IFMA R; + R.x = R.y = R.z = setzero_i64(); + + if (ECP_PREMULBP(pEC)) { + ifma_ec_nistp256_mul_pointbase(&R, (Ipp8u *)pExtendedScalar, orderBits); + } else { + /* Convert base point to a new Montgomery domain */ + __ALIGN64 P256_POINT_IFMA G52; + recode_point_to_mont52(&G52, ECP_G(pEC), pPointPool /* 3 elem */, pmeth, pME); + + ifma_ec_nistp256_mul_point(&R, &G52, (Ipp8u *)pExtendedScalar, orderBits); + } + + recode_point_to_mont64(pR, &R, pPointPool /* 3 elem */, pmeth, pME); + + cpGFpReleasePool(5, nME); + + ECP_POINT_FLAGS(pR) = gfec_IsPointAtInfinity(pR) ? 0 : ECP_FINITE_POINT; + return pR; + } +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_pubkey_p384.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_pubkey_p384.c new file mode 100644 index 0000000..7c6ff4d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_pubkey_p384.c @@ -0,0 +1,75 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpstuff.h" +#include "pcpgfpecstuff.h" + +#include "ifma_arith_p384.h" +#include "ifma_ecpoint_p384.h" +#include "ifma_arith_method.h" + +IPP_OWN_DEFN(IppsGFpECPoint*, gfec_PubKey_nist384_avx512, (IppsGFpECPoint * pR, + const BNU_CHUNK_T* pScalar, + int scalarLen, IppsGFpECState* pEC, + Ipp8u* pScratchBuffer)) +{ + FIX_BNU(pScalar, scalarLen); + { + IPP_UNREFERENCED_PARAMETER(pScratchBuffer); + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + gsModEngine *nME = ECP_MONT_R(pEC); + + const int orderBits = ECP_ORDBITSIZE(pEC); + const int orderLen = BITS_BNU_CHUNK(orderBits); + + ifmaArithMethod *pmeth = (ifmaArithMethod *)GFP_METHOD_ALT(pME); + + BNU_CHUNK_T *pPool = cpGFpGetPool(5, nME); + BNU_CHUNK_T *pExtendedScalar = pPool; + BNU_CHUNK_T *pPointPool = pExtendedScalar + 2 * GFP_FELEN(pME); + + /* Copy scalar */ + cpGFpElementCopyPad(pExtendedScalar, orderLen + 1, pScalar, scalarLen); + + __ALIGN64 P384_POINT_IFMA R; + R.x = R.y = R.z = setzero_i64(); + + if (ECP_PREMULBP(pEC)) { + ifma_ec_nistp384_mul_pointbase(&R, (Ipp8u *)pExtendedScalar, orderBits); + } else { + /* Convert base point to a new Montgomery domain */ + __ALIGN64 P384_POINT_IFMA G52; + recode_point_to_mont52(&G52, ECP_G(pEC), pPointPool /* 3 elem */, pmeth, pME); + + ifma_ec_nistp384_mul_point(&R, &G52, (Ipp8u *)pExtendedScalar, orderBits); + } + + recode_point_to_mont64(pR, &R, pPointPool /* 3 elem */, pmeth, pME); + + cpGFpReleasePool(5, nME); + + ECP_POINT_FLAGS(pR) = gfec_IsPointAtInfinity(pR) ? 0 : ECP_FINITE_POINT; + return pR; + } +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_pubkey_p521.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_pubkey_p521.c new file mode 100644 index 0000000..5f8d8d1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ec_pubkey_p521.c @@ -0,0 +1,75 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpstuff.h" +#include "pcpgfpecstuff.h" + +#include "ifma_ecpoint_p521.h" +#include "ifma_arith_p521.h" +#include "ifma_arith_method_p521.h" + +IPP_OWN_DEFN(IppsGFpECPoint*, gfec_PubKey_nist521_avx512, (IppsGFpECPoint * pR, + const BNU_CHUNK_T* pScalar, + int scalarLen, IppsGFpECState* pEC, + Ipp8u* pScratchBuffer)) +{ + FIX_BNU(pScalar, scalarLen); + { + IPP_UNREFERENCED_PARAMETER(pScratchBuffer); + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + gsModEngine *nME = ECP_MONT_R(pEC); + + const int orderBits = ECP_ORDBITSIZE(pEC); + const int orderLen = BITS_BNU_CHUNK(orderBits); + + ifmaArithMethod_p521 *pmeth = (ifmaArithMethod_p521 *)GFP_METHOD_ALT(pME); + + BNU_CHUNK_T *pPool = cpGFpGetPool(5, nME); + BNU_CHUNK_T *pExtendedScalar = pPool; + BNU_CHUNK_T *pPointPool = pExtendedScalar + 2 * GFP_FELEN(pME); + + /* Copy scalar */ + cpGFpElementCopyPad(pExtendedScalar, orderLen + 1, pScalar, scalarLen); + + __ALIGN64 P521_POINT_IFMA R; + FE521_SET(R.x) = FE521_SET(R.y) = FE521_SET(R.z) = m256_setzero_i64(); + + if (ECP_PREMULBP(pEC)) { + ifma_ec_nistp521_mul_pointbase(&R, (Ipp8u *)pExtendedScalar, orderBits); + } else { + /* Convert base point to a new Montgomery domain */ + __ALIGN64 P521_POINT_IFMA G52; + recode_point_to_mont52(&G52, ECP_G(pEC), pPointPool /* 3 elem */, pmeth, pME); + + ifma_ec_nistp521_mul_point(&R, &G52, (Ipp8u *)pExtendedScalar, orderBits); + } + + recode_point_to_mont64(pR, &R, pPointPool /* 3 elem */, pmeth, pME); + + cpGFpReleasePool(5, nME); + + ECP_POINT_FLAGS(pR) = gfec_IsPointAtInfinity(pR) ? 0 : ECP_FINITE_POINT; + return pR; + } +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecpoint_p256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecpoint_p256.c new file mode 100644 index 0000000..9dc88ea --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecpoint_p256.c @@ -0,0 +1,766 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +/* Paper referenced in this file: + * + * [1] "Enhanced Montgomery Multiplication" DOI:10.1155/2008/583926 + * + */ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpecstuff.h" + +#include "ifma_arith_p256.h" +#include "ifma_defs.h" +#include "ifma_ecpoint_p256.h" + +#define LEN64 (4) +#define LEN52 (5 + 3) /* 5 digits + 3 zero padding */ + +/* Modulus scaled: 2*p */ +static const __ALIGN64 Ipp64u p256_x2[LEN52] = { + 0x000ffffffffffffe, 0x00001fffffffffff, 0x0000000000000000, 0x0000002000000000, 0x0001fffffffe0000, 0x0, 0x0, 0x0 +}; + +/* Modulus scaled: 4*p */ +static const __ALIGN64 Ipp64u p256_x4[LEN52] = { + 0x000ffffffffffffc, 0x00003fffffffffff, 0x0000000000000000, 0x0000004000000000, 0x0003fffffffc0000, 0x0, 0x0, 0x0 +}; + +/* Modulus scaled: 6*p */ +static const __ALIGN64 Ipp64u p256_x6[LEN52] = { + 0x000ffffffffffffa, 0x00005fffffffffff, 0x0000000000000000, 0x0000006000000000, 0x0005fffffffa0000, 0x0, 0x0, 0x0 +}; + +/* Modulus scaled: 8*p */ +static const __ALIGN64 Ipp64u p256_x8[LEN52] = { + 0x000ffffffffffff8, 0x00007fffffffffff, 0x0000000000000000, 0x0000008000000000, 0x0007fffffff80000, 0x0, 0x0, 0x0 +}; + +/* Mont(a) = a*r mod p256, where r = 2^(6*52) mod p256 */ +static const __ALIGN64 Ipp64u p256_a[LEN52] = { + 0x000ffffffd000000, 0x000fffffffffffcf, 0x000300000002ffff, 0x0000000000000000, 0x0000000000000300, 0x0, 0x0, 0x0 +}; + +/* Mont(b) = b*r mod p256, where r = 2^(6*52) mod p256 */ +static const __ALIGN64 Ipp64u p256_b[LEN52] = { + 0x000c30061de0b74e, 0x000916229c4bddfd, 0x000c9c542a72f7e5, 0x00069e0d6acf005c, 0x000051ea29688e16, 0x0, 0x0, 0x0 +}; + +/* + * r = 2^(P256_LEN52*DIGIT_SIZE) mod p256 + */ +static const __ALIGN64 Ipp64u p256_r[LEN52] = { + 0x0000000000ffffff, 0x0000100000000010, 0x000effffffff0000, 0x0000000fffffffff, 0x0000fffffffeff00, 0x0, 0x0, 0x0 +}; + +/* Aliases for operations */ +#define add(R, A, B) (R) = add_i64((A), (B)) +#define sub(R, A, B) (R) = sub_i64((A), (B)) +#define mul(R, A, B) (R) = ifma_amm52_p256((A), (B)) +#define sqr(R, A) (R) = ifma_ams52_p256((A)) +#define div2(R, A) (R) = ifma_half52_p256((A)) +#define inv(R, A) (R) = ifma_aminv52_p256((A)) +#define norm(R, A) (R) = ifma_norm52((A)) +#define lnorm(R, A) (R) = ifma_lnorm52((A)) +#define from_mont(R, A) (R) = ifma_frommont52_p256((A)) + +/* Aliases for dual operations */ +#define mul_dual(R1, A1, B1, R2, A2, B2) ifma_amm52_dual_p256(&(R1), (A1), (B1), &(R2), (A2), (B2)) +#define sqr_dual(R1, A1, R2, A2) ifma_ams52_dual_p256(&(R1), (A1), &(R2), (A2)) +#define norm_dual(R1, A1, R2, A2) ifma_norm52_dual(&(R1), (A1), &(R2), (A2)) +#define lnorm_dual(R1, A1, R2, A2) ifma_lnorm52_dual(&(R1), (A1), &(R2), (A2)) + +/* to affine coordinate */ +IPP_OWN_DEFN(void, ifma_ec_nistp256_get_affine_coords, (m512 * rx, m512 *ry, const P256_POINT_IFMA *a)) +{ + m512 z1, z2, z3; + + inv(z1, a->z); /* 1/z */ + sqr(z2, z1); /* (1/z)^2 */ + lnorm(z2, z2); + + /* x = x/z^2 */ + if (NULL != rx) { + mul(*rx, a->x, z2); /* x = x/z^2 */ + lnorm(*rx, *rx); + } + /* y = y/z^3 */ + if (NULL != ry) { + mul(z3, z1, z2); /* (1/z)^3 */ + lnorm(z3, z3); + mul(*ry, a->y, z3); /* y = y/z^3 */ + lnorm(*ry, *ry); + } +} + +IPP_OWN_DEFN(void, ifma_ec_nistp256_dbl_point, (P256_POINT_IFMA * r, const P256_POINT_IFMA *p)) +{ + /* + * Enhanced Montgomery group algorithm described in [1]. + * + * l1 = 3x^2 + a*z^4 = [a = -3]= 3*(x^2 - z^4) = 3*(x - z^2)*(x + z^2) + * z2 = 2*y*z + * l2 = 4*x*y^2 + * x2 = l1^2 - 2*l2 + * l3 = 8*y^4 + * y2 = l1*(l2 - x2) - l3 + * + */ + const m512 *x1 = &p->x; + const m512 *y1 = &p->y; + const m512 *z1 = &p->z; + + m512 x2; + m512 y2; + m512 z2; + x2 = y2 = z2 = setzero_i64(); + + m512 T, U, V, A, B, H; + T = U = V = A = B = H = setzero_i64(); + + const m512 M2 = loadu_i64(p256_x2); + const m512 M4 = loadu_i64(p256_x4); + const m512 M8 = loadu_i64(p256_x8); + + add(T, *y1, *y1); /* T = 2*y1 */ + lnorm(T, T); + + sqr_dual(V, T, /* V = 4*y1^2 */ + U, *z1); /* U = z1^2 */ + + sub(B, *x1, U); /* B = x1 - z1^2 */ + add(B, B, M2); + add(U, *x1, U); /* U = x1 + z1^2 */ + + lnorm_dual(V, V, + U, U); + norm(B, B); + + mul_dual(A, V, *x1, /* A = 4*x1*y1^2 */ + B, B, U); /* B = (x1 - z1^2)*(x1 + z1^2) */ + + add(x2, A, A); /* x2 = 8*x1*y1^2 */ + add(H, B, B); + add(B, B, H); /* B(l1) = 3*(x1 - z1^2)*(x1 + z1^2) */ + + lnorm(B, B); + + sqr_dual(U, B, /* U = l1^2 */ + y2, V); /* y2 = 16*y^2 */ + + sub(x2, U, x2); /* x2 = l1^2 - 2*l2 */ + add(x2, x2, M4); + div2(y2, y2); + + sub(U, A, x2); /* U = l2 - x2 */ + add(U, U, M8); + + norm(U, U); + + mul_dual(z2, T, *z1, /* z2 = 2*y1*z1 */ + U, U, B); /* U = B(l1)*(A(l2) - x2) */ + + sub(y2, U, y2); /* y2 = B(l1)*(A(l2) - x2) - y2(l3) */ + add(y2, y2, M2); + + norm_dual(r->x, x2, + r->y, y2); + lnorm(r->z, z2); +} + +IPP_OWN_DEFN(void, ifma_ec_nistp256_add_point, (P256_POINT_IFMA * r, const P256_POINT_IFMA *p, const P256_POINT_IFMA *q)) +{ + /* + * Enhanced Montgomery group algorithm described in [1]. + * + * A = x1*z2^2 B = x2*z1^2 C = y1*z2^3 D = y2*z1^3 + * E = B - A F = D - C + * x3 = -E^3 - 2*A*E^2 + F^2 + * y3 = -C*E^3 + F*(A*E^2 - x3) + * z3 = z1*z2*E + */ + const m512 *x1 = &p->x; + const m512 *y1 = &p->y; + const m512 *z1 = &p->z; + const mask8 p_is_inf = is_zero_i64(p->z); + + const m512 *x2 = &q->x; + const m512 *y2 = &q->y; + const m512 *z2 = &q->z; + const mask8 q_is_inf = is_zero_i64(q->z); + + m512 x3; + m512 y3; + m512 z3; + x3 = y3 = z3 = setzero_i64(); + + const m512 M2 = loadu_i64(p256_x2); + const m512 M4 = loadu_i64(p256_x4); + const m512 M8 = loadu_i64(p256_x8); + + m512 U1, U2, S1, S2, H, R; + U1 = U2 = S1 = S2 = H = R = setzero_i64(); + + mul_dual(S1, *y1, *z2, /* s1 = y1*z2 */ + U1, *z2, *z2); /* u1 = z2^2 */ + + lnorm_dual(S1, S1, + U1, U1); + + mul_dual(S2, *y2, *z1, /* s2 = y2*z1 */ + U2, *z1, *z1); /* u2 = z1^2 */ + + lnorm_dual(S2, S2, + U2, U2); + + mul_dual(S1, S1, U1, /* s1 = y1*z2^3 (C) */ + S2, S2, U2); /* s2 = y2*z1^3 (D) */ + + lnorm_dual(S1, S1, + S2, S2); + + mul_dual(U1, *x1, U1, /* u1 = x1*z2^2 (A) */ + U2, *x2, U2); /* u2 = x2*z1^2 (B) */ + + lnorm_dual(U1, U1, + U2, U2); + + sub(R, S2, S1); /* r = D - C (F) */ + sub(H, U2, U1); /* h = B - A (E) */ + + /* checking the equality of X and Y coordinates (D - C == 0) and (B - A == 0) */ + const mask8 f_are_zero = is_zero_i64(R); + const mask8 e_are_zero = is_zero_i64(H); + const mask8 point_is_equal = ((e_are_zero & f_are_zero) & (~p_is_inf) & (~q_is_inf)); + + __ALIGN64 P256_POINT_IFMA r2; + r2.x = r2.y = r2.z = setzero_i64(); + if ((mask8)0xFF == point_is_equal) { + ifma_ec_nistp256_dbl_point(&r2, p); + } + + add(R, R, M2); + add(H, H, M2); + norm_dual(R, R, + H, H); + + mul_dual(z3, *z1, *z2, /* z3 = z1*z2 */ + U2, H, H); /* u2 = E^2 */ + + lnorm_dual(z3, z3, + U2, U2); + + mul_dual(z3, z3, H, /* z3 = (z1*z2)*E */ + S2, R, R); /* s2 = F^2 */ + mul(H, H, U2); /* h = E^3 */ + + lnorm(H, H); + + mul(U1, U1, U2); /* u1 = A*E^2 */ + sub(x3, S2, H); /* x3 = F^2 - E^3 */ + add(x3, x3, M2); + add(U2, U1, U1); /* u2 = 2*A*E^2 */ + mul(S1, S1, H); /* s1 = C*E^3 */ + sub(x3, x3, U2); /* x3 = (F^2 - E^3) -2*A*E^2 */ + add(x3, x3, M4); + + sub(y3, U1, x3); /* y3 = A*E^2 - x3 */ + add(y3, y3, M8); + + norm(y3, y3); + + mul(y3, y3, R); /* y3 = F*(A*E^2 - x3) */ + sub(y3, y3, S1); /* y3 = F*(A*E^2 - x3) - C*E^3 */ + add(y3, y3, M2); + + norm_dual(x3, x3, + y3, y3); + lnorm(z3, z3); + + /* T = p_is_inf ? q : T */ + x3 = mask_mov_i64(x3, p_is_inf, *x2); + y3 = mask_mov_i64(y3, p_is_inf, *y2); + z3 = mask_mov_i64(z3, p_is_inf, *z2); + + /* T = q_is_inf ? p : T */ + x3 = mask_mov_i64(x3, q_is_inf, *x1); + y3 = mask_mov_i64(y3, q_is_inf, *y1); + z3 = mask_mov_i64(z3, q_is_inf, *z1); + + /* r = point_is_equal ? r2 : T */ + x3 = mask_mov_i64(x3, point_is_equal, r2.x); + y3 = mask_mov_i64(y3, point_is_equal, r2.y); + z3 = mask_mov_i64(z3, point_is_equal, r2.z); + + r->x = x3; + r->y = y3; + r->z = z3; +} + +IPP_OWN_DEFN(void, ifma_ec_nistp256_add_point_affine, (P256_POINT_IFMA * r, const P256_POINT_IFMA *p, const P256_POINT_AFFINE_IFMA *q)) +{ + /* + * Enhanced Montgomery group algorithm described in [1]. + * + * A = x1 B = x2*z1^2 C = y1 D = y2*z1^3 + * E = B - A(x1) F = D - C(y1) + * x3 = -E^3 - 2*A(x1)*E^2 + F^2 + * y3 = -C(y1)*E^3 + F*(A(x1)*E^2 - x3) + * z3 = z1*E + */ + + /* Coordinates of p (jacobian projective) */ + const m512 *x1 = &p->x; + const m512 *y1 = &p->y; + const m512 *z1 = &p->z; + const mask8 p_is_inf = is_zero_i64(p->z); + + /* Coodinates of q (affine) */ + const m512 *x2 = &q->x; + const m512 *y2 = &q->y; + const mask8 q_is_inf = (is_zero_i64(q->x) & is_zero_i64(q->y)); + + const m512 M2 = loadu_i64(p256_x2); + const m512 M4 = loadu_i64(p256_x4); + const m512 M8 = loadu_i64(p256_x8); + + m512 x3, y3, z3; + x3 = y3 = z3 = setzero_i64(); + + m512 U2, S2, H, R; + U2 = S2 = H = R = setzero_i64(); + + mul_dual(R, *z1, *z1, /* R = z1^2 */ + S2, *y2, *z1); /* S2 = y2*z1 */ + + lnorm_dual(R, R, + S2, S2); + + mul_dual(U2, *x2, R, /* U2 = x2*z1^2 (B) */ + S2, S2, R); /* S2 = y2*z1^3 (D) */ + + sub(H, U2, *x1); /* H = B - A (E) */ + add(H, H, M8); + sub(R, S2, *y1); /* R = D - C (F) */ + add(R, R, M4); + + norm_dual(H, H, + R, R); + + mul(z3, H, *z1); /* z3 = z1*E */ + + sqr_dual(U2, H, /* U2 = E^2 */ + S2, R); /* S2 = F^2 */ + + lnorm(U2, U2); + + mul(H, H, U2); /* H = E^3 */ + + lnorm(H, H); + + mul_dual(U2, U2, *x1, /* U2 = A*E^2 */ + y3, H, *y1); /* y3 = C*E^3 */ + + add(x3, U2, U2); /* x2 = 2*A*E^2 */ + sub(x3, S2, x3); /* x3 = F^2 - 2*A*E^2 */ + add(x3, x3, M4); + sub(x3, x3, H); /* x3 = F^2 - 2*A*E^2 - E^3 */ + add(x3, x3, M2); + + sub(U2, U2, x3); /* U2 = A*E^2 - x3 */ + add(U2, U2, M8); + norm(U2, U2); + mul(U2, U2, R); /* U2 = F*(A*E^2 - x3) */ + sub(y3, U2, y3); /* y3 = F*(A*E^2 - x3) - C*E^2 */ + add(y3, y3, M2); + + norm_dual(x3, x3, + y3, y3); + lnorm(z3, z3); + + /* T = p_is_inf ? q : T */ + x3 = mask_mov_i64(x3, p_is_inf, *x2); + y3 = mask_mov_i64(y3, p_is_inf, *y2); + z3 = mask_mov_i64(z3, p_is_inf, loadu_i64(p256_r)); + + /* T = q_is_inf ? p : T */ + x3 = mask_mov_i64(x3, q_is_inf, *x1); + y3 = mask_mov_i64(y3, q_is_inf, *y1); + z3 = mask_mov_i64(z3, q_is_inf, *z1); + + r->x = x3; + r->y = y3; + r->z = z3; +} + +IPP_OWN_DEFN(int, ifma_ec_nistp256_is_on_curve, (const P256_POINT_IFMA *p, const int use_jproj_coords)) +{ + /* + * y^2 = x^3 + a*x + b (1) + * + * if input + * * Jacobian projective coordinate (x,y,z) reprepresented by (x/z^2,y/z^3,1) + * * Affine coodinate -> (x/z^2,y/z^3,z/z=1) + * + * Mult (1) by z^6 + * + * y^2 = x^3 + a*x*z^4 + b*z^6 + */ + const m512 M6 = loadu_i64(p256_x6); + const m512 a = loadu_i64(p256_a); + const m512 b = loadu_i64(p256_b); + + m512 rh, Z4, Z6, tmp; + rh = Z4 = Z6 = tmp = setzero_i64(); + + sqr(rh, p->x); /* rh = x^2 */ + + /* rh = x*(x^2 + a*z^4) + b*z^6 = x*(x^2 - 3*z^4) + b*z^6 */ + if (0 != use_jproj_coords) { + sqr(tmp, p->z); /* tmp = z^2 */ + lnorm(tmp, tmp); + + sqr(Z4, tmp); /* z4 = z^4 */ + lnorm(Z4,Z4); /* norm */ + mul(Z6, Z4, tmp); /* z6 = z^6 */ + lnorm(Z6,Z6); /* norm */ + + add(tmp, Z4, Z4); /* tmp = 2*z^4 */ + add(tmp, tmp, Z4); /* tmp = 3*z^4 */ + + sub(rh, rh, tmp); /* rh = x^2 - 3*z^4 */ + add(rh, rh, M6); + norm(rh, rh); + + mul_dual(rh, rh, p->x, /* rh = x*(x^2 - 3*z^4) */ + tmp, Z6, b); /* tmp = b*z^6 */ + + add(rh, rh, tmp); /* rh = x*(x^2 - 3*z^4) + b*z^6 */ + } + + /* rh = x*(x^2 + a) + b */ + else { + add(rh, rh, a); /* rh = x^2 + a */ + lnorm(rh, rh); + mul(rh, rh, p->x); /* rh = x*(x^2 + a) */ + add(rh, rh, b); /* rh = x*(x^2 + a) + b */ + } + + lnorm(rh, rh); + + /* rl = Y^2 */ + sqr(tmp, p->y); /* tmp = y^2 */ + lnorm(tmp, tmp); /**/ + + /* from mont */ + from_mont(tmp, tmp); + from_mont(rh, rh); + + const mask8 mask = cmp_i64_mask(rh, tmp, _MM_CMPINT_EQ); + return (mask == 0xFF) ? 1 : 0; +} + +#undef add +#undef sub +#undef mul +#undef sqr +#undef div2 +#undef norm +#undef from_mont +#undef mul_dual +#undef sqr_dual +#undef norm_dual + +static __NOINLINE void clear_secret_context(Ipp16u *wval, + Ipp32s *chunk_no, Ipp32s *chunk_shift, + Ipp8u *sign, Ipp8u *digit, + P256_POINT_IFMA *R, P256_POINT_IFMA *H, + P256_POINT_AFFINE_IFMA *A) +{ + *wval = 0; + *chunk_no = 0; + *chunk_shift = 0; + *sign = 0; + *digit = 0; + + if (NULL != R) + (*R).x = (*R).y = (*R).z = setzero_i64(); + if (NULL != H) + (*H).x = (*H).y = (*H).z = setzero_i64(); + if (NULL != A) + (*A).x = (*A).y = setzero_i64(); +} + +#define WIN_SIZE (5) + +__INLINE mask8 is_eq_mask(const Ipp32s a, const Ipp32s b) +{ + const Ipp32s eq = a ^ b; + const Ipp32s v = ~eq & (eq - 1); + const Ipp32s msb = 0 - (v >> (sizeof(a) * 8 - 1)); + return (mask8)(0 - msb); +} + +__INLINE void extract_table_point(P256_POINT_IFMA *r, const Ipp32s digit, const P256_POINT_IFMA *tbl) +{ + Ipp32s idx = digit - 1; + + __ALIGN64 P256_POINT_IFMA R; + R.x = R.y = R.z = setzero_i64(); + + for (Ipp32s n = 0; n < (1 << (WIN_SIZE - 1)); ++n) { + const mask8 mask = is_eq_mask(n, idx); + + R.x = mask_mov_i64(R.x, mask, tbl[n].x); + R.y = mask_mov_i64(R.y, mask, tbl[n].y); + R.z = mask_mov_i64(R.z, mask, tbl[n].z); + } + + r->x = R.x; + r->y = R.y; + r->z = R.z; +} + +#define dbl_point ifma_ec_nistp256_dbl_point +#define add_point ifma_ec_nistp256_add_point +#define neg_coord ifma_neg52_p256 +#define add_point_affine ifma_ec_nistp256_add_point_affine + +/* r = n*P = (P + P + ... + P) */ +IPP_OWN_DEFN(void, ifma_ec_nistp256_mul_point, (P256_POINT_IFMA * r, const P256_POINT_IFMA *p, const Ipp8u *pExtendedScalar, const int scalarBitSize)) +{ + /* Precompute table */ + __ALIGN64 P256_POINT_IFMA tbl[(1 << (WIN_SIZE - 1))]; + + __ALIGN64 P256_POINT_IFMA R; + __ALIGN64 P256_POINT_IFMA H; + + R.x = R.y = R.z = setzero_i64(); + H.x = H.y = H.z = setzero_i64(); + m512 negHy = setzero_i64(); + + /* compute tbl[] = [n]P, n = 1, ... , 2^(win_size - 1) + * tbl[2*n] = tbl[2*n - 1] + p + * tbl[2*n + 1] = [2]*tbl[n] + */ + + /* tbl[0] = p */ + tbl[0].x = p->x; + tbl[0].y = p->y; + tbl[0].z = p->z; + + /* tbl[1] = [2]*p */ + dbl_point(/* r = */ (tbl + 1), /* a = */ p); + + for (int n = 1; n < ((1 << (WIN_SIZE - 1)) / 2); ++n) { + add_point((tbl + 2 * n), (tbl + 2 * n - 1), p); + dbl_point((tbl + 2 * n + 1), (tbl + n)); + } + + Ipp16u wval; + Ipp8u digit, sign; + const Ipp32s mask = ((1 << (WIN_SIZE + 1)) - 1); /* mask 0b111111 */ + Ipp32s bit = scalarBitSize - (scalarBitSize % WIN_SIZE); + + Ipp32s chunk_no = (bit - 1) / 8; + Ipp32s chunk_shift = (bit - 1) % 8; + + if (0 != bit) { + wval = *((Ipp16u *)(pExtendedScalar + chunk_no)); + wval = (Ipp16u)((wval >> chunk_shift) & mask); + } else { + wval = 0; + } + + booth_recode(&sign, &digit, (Ipp8u)wval, WIN_SIZE); + extract_table_point(&R, (Ipp32s)digit, tbl); + + for (bit -= WIN_SIZE; bit >= WIN_SIZE; bit -= WIN_SIZE) { + dbl_point(&R, &R); + dbl_point(&R, &R); + dbl_point(&R, &R); + dbl_point(&R, &R); +#if (WIN_SIZE == 5) + dbl_point(&R, &R); +#endif + chunk_no = (bit - 1) / 8; + chunk_shift = (bit - 1) % 8; + + wval = *((Ipp16u *)(pExtendedScalar + chunk_no)); + wval = (Ipp16u)((wval >> chunk_shift) & mask); + + booth_recode(&sign, &digit, (Ipp8u)wval, WIN_SIZE); + extract_table_point(&H, (Ipp32s)digit, tbl); + + negHy = neg_coord(H.y); + + const mask8 mask_neg = (mask8)(~(sign - 1)); + H.y = mask_mov_i64(H.y, mask_neg, negHy); + + add_point(&R, &R, &H); + } + + /* last window */ + dbl_point(&R, &R); + dbl_point(&R, &R); + dbl_point(&R, &R); + dbl_point(&R, &R); +#if (WIN_SIZE == 5) + dbl_point(&R, &R); +#endif + + wval = *((Ipp16u *)(pExtendedScalar + 0)); + wval = (Ipp16u)((wval << 1) & mask); + + booth_recode(&sign, &digit, (Ipp8u)wval, WIN_SIZE); + extract_table_point(&H, (Ipp32s)digit, tbl); + + negHy = neg_coord(H.y); + + const mask8 mask_neg = (mask8)(~(sign - 1)); + H.y = mask_mov_i64(H.y, mask_neg, negHy); + + add_point(&R, &R, &H); + + r->x = R.x; + r->y = R.y; + r->z = R.z; + + /* clear secret data */ + clear_secret_context(&wval, + &chunk_no, &chunk_shift, + &sign, &digit, + &R, &H, NULL); + + return; +} + +/* #include "ifma_ecprecomp4_p256.h" */ +#include "ifma_ecprecomp7_p256.h" + +/* mul point base */ +#define BP_WIN_SIZE BASE_POINT_WIN_SIZE +#define BP_N_ENTRY BASE_POINT_N_ENTRY + +__INLINE void extract_point_affine(P256_POINT_AFFINE_IFMA *r, + const P256_POINT_AFFINE_IFMA_MEM *tbl, + const Ipp32s digit) +{ + Ipp32s idx = digit - 1; + + m512 x, y; + x = y = setzero_i64(); + + for (Ipp32s n = 0; n < (1 << ((BP_WIN_SIZE)-1)); ++n, ++tbl) { + const mask8 mask = is_eq_mask(n, idx); + + x = mask_mov_i64(x, mask, maskz_loadu_i64(0x1f, tbl->X)); + y = mask_mov_i64(y, mask, maskz_loadu_i64(0x1f, tbl->Y)); + } + + r->x = x; + r->y = y; +} + +IPP_OWN_DEFN(void, p256r1_select_ap_w7_ifma, (BNU_CHUNK_T * pAffinePoint, const BNU_CHUNK_T *pTable, int index)) +{ + __ALIGN64 P256_POINT_AFFINE_IFMA ap; + + extract_point_affine(&ap, (P256_POINT_AFFINE_IFMA_MEM *)pTable, index); + + ap.x = ifma_frommont52_p256(ap.x); + ap.y = ifma_frommont52_p256(ap.y); + + convert_radix_to_64x4(pAffinePoint, ap.x); + convert_radix_to_64x4(pAffinePoint + LEN64, ap.y); +} + +IPP_OWN_DEFN(void, ifma_ec_nistp256_mul_pointbase, (P256_POINT_IFMA * r, const Ipp8u *pExtendedScalar, int scalarBitSize)) +{ + /* precompute table */ + const P256_POINT_AFFINE_IFMA_MEM *tbl = &ifma_ec_nistp256r1_bp_precomp[0][0]; + + __ALIGN64 P256_POINT_IFMA R; + R.x = R.y = R.z = setzero_i64(); + __ALIGN64 P256_POINT_AFFINE_IFMA A; + A.x = A.y = setzero_i64(); + + m512 Ty = setzero_i64(); + + Ipp16u wval; + Ipp8u digit, sign; + const Ipp32s mask = ((1 << (BP_WIN_SIZE + 1)) - 1); /* mask 0b11111 */ + Ipp32s bit = 0; + Ipp32s chunk_no, chunk_shift; + + wval = *((Ipp16u *)(pExtendedScalar + 0)); + wval = (Ipp16u)((wval << 1) & mask); + + booth_recode(&sign, &digit, (Ipp8u)wval, BP_WIN_SIZE); + extract_point_affine(&A, tbl, (Ipp32s)digit); + tbl += BP_N_ENTRY; + + /* A = sign == 1 ? -A : A */ + Ty = neg_coord(A.y); + mask8 mask_neg = (mask8)(~(sign - 1)); + A.y = mask_mov_i64(A.y, mask_neg, Ty); + + /* R += A */ + add_point_affine(&R, &R, &A); + + for (bit += BP_WIN_SIZE; bit <= scalarBitSize; bit += BP_WIN_SIZE) { + chunk_no = (bit - 1) / 8; + chunk_shift = (bit - 1) % 8; + + wval = *((Ipp16u *)(pExtendedScalar + chunk_no)); + wval = (Ipp16u)((wval >> chunk_shift) & mask); + + booth_recode(&sign, &digit, (Ipp8u)wval, BP_WIN_SIZE); + extract_point_affine(&A, tbl, (Ipp32s)digit); + tbl += BP_N_ENTRY; + + /* A = sign == 1 ? -A : A */ + Ty = neg_coord(A.y); + mask_neg = (mask8)(~(sign - 1)); + A.y = mask_mov_i64(A.y, mask_neg, Ty); + + /* R += A */ + add_point_affine(&R, &R, &A); + } + + r->x = R.x; + r->y = R.y; + r->z = R.z; + + /* clear secret data */ + clear_secret_context(&wval, + &chunk_no, &chunk_shift, + &sign, &digit, + &R, NULL, &A); + return; +} + +#undef dbl_point +#undef add_point +#undef neg_coord +#undef add_point_affine + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecpoint_p256.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecpoint_p256.h new file mode 100644 index 0000000..21a320a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecpoint_p256.h @@ -0,0 +1,200 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +/* Paper referenced in this header: + * + * [1] "Enhanced Montgomery Multiplication" DOI:10.1155/2008/583926 + * + */ + +#ifndef IFMA_ECPOINT_P256_H +#define IFMA_ECPOINT_P256_H + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ifma_arith_p256.h" +#include "pcpbnuimpl.h" + +/* p256 point (x,y,z) */ +typedef struct { + m512 x; + m512 y; + m512 z; +} P256_POINT_IFMA; + +/* p256 affine point(x,y) */ +typedef struct { + m512 x; + m512 y; +} P256_POINT_AFFINE_IFMA; + +/** + * \brief + * + * R = [pExtendedScalar]*P = (P + P + ... + P) + * + * \param[out] r point in radix 2^52 + * \param[in] p point in radix 2^52 + * \param[in] pExtendedScalar pointer to a scalar + * \param[in] scalarBitSize scalar size in bits + */ +IPP_OWN_DECL(void, ifma_ec_nistp256_mul_point, (P256_POINT_IFMA * r, const P256_POINT_IFMA *p, const Ipp8u *pExtendedScalar, const int scalarBitSize)) + +/** + * \brief + * + * R = [pExtendedScalar]*BasePoint + * + * \param[out] r point in radix 2^52 + * \param[in] pExtendedScalar pointer to a scalar + * \param[in] scalarBitSize scalar size in bits + */ +IPP_OWN_DECL(void, ifma_ec_nistp256_mul_pointbase, (P256_POINT_IFMA * r, const Ipp8u *pExtendedScalar, int scalarBitSize)) + +/** + * \brief + * + * Convert point to affine coordinate + * + * \param[out] rx X-affine coordinate + * \param[out] ry Y-affine coordinate + * \param[in] a point in projective coordinates + */ +IPP_OWN_DECL(void, ifma_ec_nistp256_get_affine_coords, (m512 * rx, m512 *ry, const P256_POINT_IFMA *a)) + +/** + * \brief + * + * Check if the point is on curve p256r1 + * + * \param[in] p point in radix 2^52 + * \param[in] use_jproj_coords flag whether coordinates are in Projective of Affine representation + * \return int 1 - true + * 0 - false + */ +IPP_OWN_DECL(int, ifma_ec_nistp256_is_on_curve, (const P256_POINT_IFMA *p, const int use_jproj_coords)) + +/** + * \brief + * + * Point doubling on p256r1 curve (Enhanced Montgomery Algorithm, see [1]) + * + * \param[out] r point in projective coordinates + * \param[in] p point in projective coordinates + */ +IPP_OWN_DECL(void, ifma_ec_nistp256_dbl_point, (P256_POINT_IFMA * r, const P256_POINT_IFMA *p)) + +/** + * \brief + * + * Point addition on p256r1 curve (Enhanced Montgomery Algorithm, see [1]) + * + * \param[out] r result point + * \param[in] p first point + * \param[in] q second point + */ +IPP_OWN_DECL(void, ifma_ec_nistp256_add_point, (P256_POINT_IFMA * r, const P256_POINT_IFMA *p, const P256_POINT_IFMA *q)) + +/** + * \brief + * + * Point addition on p256r1 curve (Enhanced Montgomery Algorithm, see [1]) + * + * \param[out] r point in projective coordinates + * \param[in] p point in projective coordinates + * \param[in] q point in affine coordinates + */ +IPP_OWN_DECL(void, ifma_ec_nistp256_add_point_affine, (P256_POINT_IFMA * r, const P256_POINT_IFMA *p, const P256_POINT_AFFINE_IFMA *q)) + +/** + * \brief + * + * Extracts affine point from the precomputed table. + * + * \param[out] pAffinePoint array of x and y coordinates of affine point in 2^64 radix + * \param[in] pTable pointer to a precomputed table + * \param[in] index index of desired point in the table + */ +IPP_OWN_DECL(void, p256r1_select_ap_w7_ifma, (BNU_CHUNK_T * pAffinePoint, const BNU_CHUNK_T *pTable, int index)) + + +#include "ifma_arith_method.h" +#include "gsmodstuff.h" +#include "pcpgfpstuff.h" +#include "pcpgfpecstuff.h" + +__INLINE void recode_point_to_mont52(P256_POINT_IFMA *pR, + const BNU_CHUNK_T *pP, + BNU_CHUNK_T *pPool, + ifmaArithMethod *method, + gsModEngine *pME) +{ + ifma_import to_radix52 = method->import_to52; + ifma_encode p_to_mont = method->encode; + + const int elemLen = GFP_FELEN(pME); + + BNU_CHUNK_T *pX = pPool; + BNU_CHUNK_T *pY = pPool + elemLen; + BNU_CHUNK_T *pZ = pPool + 2 * elemLen; + + GFP_METHOD(pME)->decode(pX, pP, pME); + GFP_METHOD(pME)->decode(pY, pP + elemLen, pME); + GFP_METHOD(pME)->decode(pZ, pP + 2 * elemLen, pME); + + pR->x = to_radix52((Ipp64u *)pX); + pR->y = to_radix52((Ipp64u *)pY); + pR->z = to_radix52((Ipp64u *)pZ); + + pR->x = p_to_mont(pR->x); + pR->y = p_to_mont(pR->y); + pR->z = p_to_mont(pR->z); +} + +__INLINE void recode_point_to_mont64(IppsGFpECPoint *pR, + P256_POINT_IFMA *pP, + BNU_CHUNK_T *pPool, + ifmaArithMethod *method, + gsModEngine *pME) +{ + ifma_export to_radix64 = method->export_to64; + ifma_decode p_from_mont = method->decode; + + const int elemLen = GFP_PELEN(pME); + BNU_CHUNK_T *pX = pPool; + BNU_CHUNK_T *pY = pPool + elemLen; + BNU_CHUNK_T *pZ = pPool + 2 * elemLen; + + pP->x = p_from_mont(pP->x); + pP->y = p_from_mont(pP->y); + pP->z = p_from_mont(pP->z); + + to_radix64((Ipp64u *)pX, pP->x); + to_radix64((Ipp64u *)pY, pP->y); + to_radix64((Ipp64u *)pZ, pP->z); + + GFP_METHOD(pME)->encode(ECP_POINT_X(pR), pX, pME); + GFP_METHOD(pME)->encode(ECP_POINT_Y(pR), pY, pME); + GFP_METHOD(pME)->encode(ECP_POINT_Z(pR), pZ, pME); +} + + +#endif // (_IPP32E >= _IPP32E_K1) + +#endif // IFMA_ECPOINT_P256_H diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecpoint_p384.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecpoint_p384.c new file mode 100644 index 0000000..7e3b689 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecpoint_p384.c @@ -0,0 +1,798 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpecstuff.h" + +#include "ifma_arith_p384.h" +#include "ifma_defs.h" +#include "ifma_ecpoint_p384.h" + +#define LEN64 (6) +#define LEN52 (8) + +/* modulus 2*p */ +static const __ALIGN64 Ipp64u p384_x2[LEN52] = { + 0x00000001FFFFFFFE, 0x000FE00000000000, 0x000FFFFFFDFFFFFF, 0x000FFFFFFFFFFFFF, 0x000FFFFFFFFFFFFF, 0x000FFFFFFFFFFFFF, 0x000FFFFFFFFFFFFF, 0x00000000001FFFFF +}; +/* modulus 4*p */ +static const __ALIGN64 Ipp64u p384_x4[LEN52] = { + 0x00000003FFFFFFFC, 0x000FC00000000000, 0x000FFFFFFBFFFFFF, 0x000FFFFFFFFFFFFF, 0x000FFFFFFFFFFFFF, 0x000FFFFFFFFFFFFF, 0x000FFFFFFFFFFFFF, 0x00000000003FFFFF +}; +/* modulus 6*p */ +static const __ALIGN64 Ipp64u p384_x6[LEN52] = { + 0x00000005FFFFFFFA, 0x000FA00000000000, 0x000FFFFFF9FFFFFF, 0x000FFFFFFFFFFFFF, 0x000FFFFFFFFFFFFF, 0x000FFFFFFFFFFFFF, 0x000FFFFFFFFFFFFF, 0x00000000005FFFFF +}; +/* modulus 8*p */ +static const __ALIGN64 Ipp64u p384_x8[LEN52] = { + 0x00000007FFFFFFF8, 0x000F800000000000, 0x000FFFFFF7FFFFFF, 0x000FFFFFFFFFFFFF, 0x000FFFFFFFFFFFFF, 0x000FFFFFFFFFFFFF, 0x000FFFFFFFFFFFFF, 0x00000000007FFFFF +}; + +/* mont(a) */ +static const __ALIGN64 Ipp64u p384_a[LEN52] = { + 0x000FFFFDFFFFFFFF, 0x000FF00000002FFF, 0x000FFFFFFBFFFFFF, 0x000FFFFFFFFFFFCF, 0x000FFFFFFFFFFFFF, 0x000FFFFFFFFFFFFF, 0x000FFFFFFFFFFFFF, 0x00000000000FFFFF +}; +/* mont(b) */ +static const __ALIGN64 Ipp64u p384_b[LEN52] = { + 0x00091C81CD08114B, 0x0003708118870D03, 0x000431BF24475444, 0x000209B1920022FC, 0x000E94938AE277F2, 0x000022094E3374BE, 0x000FF9B62B21F41F, 0x00000000000604FB +}; + +/* Montgomery(1) + * r = 2^(P384_LEN52*DIGIT_SIZE) mod p384 + */ +static const __ALIGN64 Ipp64u p384_r[LEN52] = { + 0x0000000100000000, 0x000FFFFFFFFFF000, 0x0000000000FFFFFF, 0x0000000000000010, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000 +}; + +#define add(R, A, B) (R) = add_i64((A), (B)) +#define sub(R, A, B) (R) = sub_i64((A), (B)) +#define mul(R, A, B) (R) = ifma_amm52_p384((A), (B)) +#define sqr(R, A) (R) = ifma_ams52_p384((A)) +#define div2(R, A) (R) = ifma_half52_p384((A)) +#define inv(R, A) (R) = ifma_aminv52_p384((A)) +#define norm(R, A) (R) = ifma_norm52((A)) +#define lnorm(R, A) (R) = ifma_lnorm52((A)) +#define from_mont(R, A) (R) = ifma_frommont52_p384((A)) + +/* Dual mult/sqr/norm */ +#define mul_dual(R1, A1, B1, R2, A2, B2) ifma_amm52_dual_p384(&(R1), (A1), (B1), &(R2), (A2), (B2)) +#define sqr_dual(R1, A1, R2, A2) ifma_ams52_dual_p384(&(R1), (A1), &(R2), (A2)) +#define norm_dual(R1, A1, R2, A2) ifma_norm52_dual(&(R1), (A1), &(R2), (A2)) +#define lnorm_dual(R1, A1, R2, A2) ifma_lnorm52_dual(&(R1), (A1), &(R2), (A2)) + +/* to affine coordinate */ +IPP_OWN_DEFN(void, ifma_ec_nistp384_get_affine_coords, (m512 * prx, m512 *pry, const P384_POINT_IFMA *a)) +{ + m512 z1, z2, z3; + z1 = z2 = z3 = setzero_i64(); + + inv(z1, a->z); /* 1/z */ + sqr(z2, z1); /* (1/z)^2 */ + lnorm(z2, z2); /**/ + + /* x = x/z^2 */ + if (NULL != prx) { + mul(*prx, a->x, z2); /* x = x/z^2 */ + lnorm(*prx, *prx); /**/ + } + /* y = y/z^3 */ + if (NULL != pry) { + mul(z3, z1, z2); /* (1/z)^3 */ + lnorm(z3, z3); /**/ + mul(*pry, a->y, z3); /* y = y/z^3 */ + lnorm(*pry, *pry); /**/ + } + + return; +} + +IPP_OWN_DEFN(void, ifma_ec_nistp384_dbl_point, (P384_POINT_IFMA * r, const P384_POINT_IFMA *p)) +{ + /* + * Algorithm (Gueron - Enhanced Montgomery Multiplication) + * + * Gueron (2002). Enhanced Montgomery Multiplication. + * In Cryptographic Hardware and Embedded Systems - CHES 2002, + * 4th International Workshop, Redwood Shores, CA, USA, + * August 13-15, 2002, Revised Papers (pp. 46-56) + * [ DOI: 10.1007/3-540-36400-5_5 ] + * + * l1 = 3x^2 + a*z^4 = (if p384 a = -3) = 3*(x^2 - z^4) = 3*(x - z^2)*(x + z^2) + * z2 = 2*y*z + * l2 = 4*x*y^2 + * x2 = l1^2 - 2*l2 + * l3 = 8*y^4 + * y2 = l1*(l2 - x2) - l3 + * + * sum arithmetic: 8 mul; 9 add/sub; 1 div2. + */ + const m512 *x1 = &p->x; + const m512 *y1 = &p->y; + const m512 *z1 = &p->z; + + m512 x2; + m512 y2; + m512 z2; + x2 = y2 = z2 = setzero_i64(); + + m512 T, U, V, A, B, H; + T = U = V = A = B = H = setzero_i64(); + + const m512 M2 = loadu_i64(p384_x2); /* 2*p */ + const m512 M4 = loadu_i64(p384_x4); /* 4*p */ + const m512 M8 = loadu_i64(p384_x8); /* 8*p */ + + add(T, *y1, *y1); /* T = 2*y1 */ + lnorm(T, T); /**/ + /*=====*/ + sqr_dual(V, T, /* V = 4*y1^2 */ + U, *z1); /* U = z1^2 */ + /*=====*/ + sub(B, *x1, U); /* B = x1 - z1^2 */ + add(B, B, M2); /**/ + add(U, *x1, U); /* U = x1 + z1^2 */ + /*=====*/ + /* normalization */ + lnorm_dual(V, V, /**/ + U, U); /**/ + norm(B, B); /**/ + /*=====*/ + mul_dual(A, V, *x1, /* A = 4*x1*y1^2 */ + B, B, U); /* B = (x1 - z1^2)*(x1 + z1^2) */ + /*=====*/ + add(x2, A, A); /* x2 = 8*x1*y1^2 (4p) */ + add(H, B, B); /**/ + add(B, B, H); /* B(l1) = 3*(x1 - z1^2)*(x1 + z1^2) */ + /*=====*/ + /* normalization */ + lnorm(B, B); /**/ + /*=====*/ + sqr_dual(U, B, /* U = l1^2 */ + y2, V); /* y2 = 16*y^2 */ + /*=====*/ + /* normalization */ + /*=====*/ + sub(x2, U, x2); /* x2 = l1^2 - 2*l2 */ + add(x2, x2, M4); /**/ + div2(y2, y2); /**/ + /*=====*/ + sub(U, A, x2); /* U = l2 - x2 */ + add(U, U, M8); /**/ + /*=====*/ + /* normalization */ + norm(U, U); /**/ + /*=====*/ + mul_dual(z2, T, *z1, /* z2 = 2*y1*z1 */ + U, U, B); /* U = B(l1)*(A(l2) - x2) */ + /*=====*/ + sub(y2, U, y2); /* y2 = B(l1)*(A(l2) - x2) - y2(l3) */ + add(y2, y2, M2); /**/ + /*=====*/ + /* normalization */ + norm_dual(r->x, x2, /**/ + r->y, y2); /**/ + lnorm(r->z, z2); /**/ + return; +} + +IPP_OWN_DEFN(void, ifma_ec_nistp384_add_point, (P384_POINT_IFMA * r, const P384_POINT_IFMA *p, const P384_POINT_IFMA *q)) +{ + /* + * Algorithm (Gueron - Enhanced Montgomery Multiplication) + * + * Gueron (2002). Enhanced Montgomery Multiplication. + * In Cryptographic Hardware and Embedded Systems - CHES 2002, + * 4th International Workshop, Redwood Shores, CA, USA, + * August 13-15, 2002, Revised Papers (pp. 46-56) + * [ DOI: 10.1007/3-540-36400-5_5 ] + * + * A = x1*z2^2 B = x2*z1^2 C = y1*z2^3 D = y2*z1^3 + * E = B - A F = D - C + * x3 = -E^3 - 2*A*E^2 + F^2 + * y3 = -C*E^3 + F*(A*E^2 - x3) + * z3 = z1*z2*E + */ + const m512 *x1 = &p->x; + const m512 *y1 = &p->y; + const m512 *z1 = &p->z; + const mask8 p_is_inf = is_zero_i64(p->z); + + const m512 *x2 = &q->x; + const m512 *y2 = &q->y; + const m512 *z2 = &q->z; + const mask8 q_is_inf = is_zero_i64(q->z); + + m512 x3; + m512 y3; + m512 z3; + x3 = y3 = z3 = setzero_i64(); + + const m512 M2 = loadu_i64(p384_x2); /* 2*p */ + const m512 M4 = loadu_i64(p384_x4); /* 4*p */ + const m512 M8 = loadu_i64(p384_x8); /* 8*p */ + + m512 U1, U2, S1, S2, H, R; + U1 = U2 = S1 = S2 = H = R = setzero_i64(); + + mul_dual(S1, *y1, *z2, /* s1 = y1*z2 */ + U1, *z2, *z2); /* u1 = z2^2 */ + /*=====*/ + /* normalization */ + lnorm_dual(S1, S1, /**/ + U1, U1); /**/ + /*=====*/ + mul_dual(S2, *y2, *z1, /* s2 = y2*z1 */ + U2, *z1, *z1); /* u2 = z1^2 */ + /*=====*/ + /* normalization */ + lnorm_dual(S2, S2, /**/ + U2, U2); /**/ + /*=====*/ + mul_dual(S1, S1, U1, /* s1 = y1*z2^3 (C) */ + S2, S2, U2); /* s2 = y2*z1^3 (D) */ + /*=====*/ + /* normalization */ + lnorm_dual(S1, S1, /**/ + S2, S2); /* (need by correct compute F = D - C) */ + /*=====*/ + mul_dual(U1, *x1, U1, /* u1 = x1*z2^2 (A) */ + U2, *x2, U2); /* u2 = x2*z1^2 (B) */ + /*=====*/ + /* normalization */ + lnorm_dual(U1, U1, /**/ + U2, U2); /**/ + /*=====*/ + sub(R, S2, S1); /* r = D - C (F) */ + sub(H, U2, U1); /* h = B - A (E) */ + + /* checking the equality of X and Y coordinates (D - C == 0) and (B - A == 0) */ + const mask8 f_are_zero = is_zero_i64(R); + const mask8 e_are_zero = is_zero_i64(H); + const mask8 point_is_equal = ((e_are_zero & f_are_zero) & (~p_is_inf) & (~q_is_inf)); + + __ALIGN64 P384_POINT_IFMA r2; + r2.x = r2.y = r2.z = setzero_i64(); + if ((mask8)0xFF == point_is_equal) { + ifma_ec_nistp384_dbl_point(&r2, p); + } + + add(R, R, M2); /**/ + add(H, H, M2); /**/ + /*=====*/ + /* normalization */ + norm_dual(R, R, /**/ + H, H); /**/ + /**/ + mul_dual(z3, *z1, *z2, /* z3 = z1*z2 */ + U2, H, H); /* u2 = E^2 */ + /*=====*/ + /* normalization */ + lnorm_dual(z3, z3, /**/ + U2, U2); /**/ + /**/ + mul_dual(z3, z3, H, /* z3 = (z1*z2)*E */ + S2, R, R); /* s2 = F^2 */ + mul(H, H, U2); /* h = E^3 */ + /*=====*/ + /* normalization */ + lnorm(H, H); /**/ + /*=====*/ + mul(U1, U1, U2); /* u1 = A*E^2 */ + sub(x3, S2, H); /* x3 = F^2 - E^3 */ + add(x3, x3, M2); /**/ + add(U2, U1, U1); /* u2 = 2*A*E^2 */ + mul(S1, S1, H); /* s1 = C*E^3 */ + sub(x3, x3, U2); /* x3 = (F^2 - E^3) -2*A*E^2 */ + add(x3, x3, M4); /**/ + /*=====*/ + sub(y3, U1, x3); /* y3 = A*E^2 - x3 */ + add(y3, y3, M8); /**/ + /*=====*/ + /* normalization */ + norm(y3, y3); /**/ + /**/ + mul(y3, y3, R); /* y3 = F*(A*E^2 - x3) */ + sub(y3, y3, S1); /* y3 = F*(A*E^2 - x3) - C*E^3 */ + add(y3, y3, M2); + + /* normalization */ + norm_dual(x3, x3, /**/ + y3, y3); /**/ + lnorm(z3, z3); /**/ + + /* T = p_is_inf ? q : T */ + x3 = mask_mov_i64(x3, p_is_inf, *x2); + y3 = mask_mov_i64(y3, p_is_inf, *y2); + z3 = mask_mov_i64(z3, p_is_inf, *z2); + + /* T = q_is_inf ? p : T */ + x3 = mask_mov_i64(x3, q_is_inf, *x1); + y3 = mask_mov_i64(y3, q_is_inf, *y1); + z3 = mask_mov_i64(z3, q_is_inf, *z1); + + /* r = point_is_equal ? r2 : T */ + x3 = mask_mov_i64(x3, point_is_equal, r2.x); + y3 = mask_mov_i64(y3, point_is_equal, r2.y); + z3 = mask_mov_i64(z3, point_is_equal, r2.z); + + r->x = x3; + r->y = y3; + r->z = z3; + + return; +} + +IPP_OWN_DEFN(void, ifma_ec_nistp384_add_point_affine, (P384_POINT_IFMA * r, const P384_POINT_IFMA *p, const P384_POINT_AFFINE_IFMA *q)) +{ + /* + * Algorithm (Gueron - Enhanced Montgomery Multiplication) + * + * Gueron (2002). Enhanced Montgomery Multiplication. + * In Cryptographic Hardware and Embedded Systems - CHES 2002, + * 4th International Workshop, Redwood Shores, CA, USA, + * August 13-15, 2002, Revised Papers (pp. 46-56) + * [ DOI: 10.1007/3-540-36400-5_5 ] + * + * A = x1 B = x2*z1^2 C = y1 D = y2*z1^3 + * E = B - A(x1) F = D - C(y1) + * x3 = -E^3 - 2*A(x1)*E^2 + F^2 + * y3 = -C(y1)*E^3 + F*(A(x1)*E^2 - x3) + * z3 = z1*E + */ + /* coordinates of p (jacobian projective) */ + const m512 *x1 = &p->x; + const m512 *y1 = &p->y; + const m512 *z1 = &p->z; + const mask8 p_is_inf = is_zero_i64(p->z); + + /* coodinate of q (affine) */ + const m512 *x2 = &q->x; + const m512 *y2 = &q->y; + const mask8 q_is_inf = (is_zero_i64(q->x) & is_zero_i64(q->y)); + + const m512 M2 = loadu_i64(p384_x2); /* 2*p */ + const m512 M4 = loadu_i64(p384_x4); /* 4*p */ + const m512 M8 = loadu_i64(p384_x8); /* 8*p */ + + m512 x3, y3, z3; + x3 = y3 = z3 = setzero_i64(); + + m512 U2, S2, H, R; + U2 = S2 = H = R = setzero_i64(); + + mul_dual(R, *z1, *z1, /* R = z1^2 */ + S2, *y2, *z1); /* S2 = y2*z1 */ + /*=====*/ + lnorm_dual(R, R, /**/ + S2, S2); /**/ + /*=====*/ + mul_dual(U2, *x2, R, /* U2 = x2*z1^2 (B) */ + S2, S2, R); /* S2 = y2*z1^3 (D) */ + /**/ + sub(H, U2, *x1); /* H = B - A (E) */ + add(H, H, M8); /**/ + sub(R, S2, *y1); /* R = D - C (F) */ + add(R, R, M4); /**/ + /*=====*/ + norm_dual(H, H, /**/ + R, R); /**/ + /**/ + mul(z3, H, *z1); /* z3 = z1*E */ + /**/ + sqr_dual(U2, H, /* U2 = E^2 */ + S2, R); /* S2 = F^2 */ + /**/ + lnorm(U2, U2); /**/ + /**/ + mul(H, H, U2); /* H = E^3 */ + /**/ + lnorm(H, H); /**/ + /**/ + mul_dual(U2, U2, *x1, /* U2 = A*E^2 */ + y3, H, *y1); /* y3 = C*E^3 */ + /**/ + add(x3, U2, U2); /* x2 = 2*A*E^2 */ + sub(x3, S2, x3); /* x3 = F^2 - 2*A*E^2 */ + add(x3, x3, M4); /**/ + sub(x3, x3, H); /* x3 = F^2 - 2*A*E^2 - E^3 */ + add(x3, x3, M2); /**/ + /**/ + sub(U2, U2, x3); /* U2 = A*E^2 - x3 */ + add(U2, U2, M8); /**/ + norm(U2, U2); /**/ + mul(U2, U2, R); /* U2 = F*(A*E^2 - x3) */ + sub(y3, U2, y3); /* y3 = F*(A*E^2 - x3) - C*E^2 */ + add(y3, y3, M2); /**/ + + /* normalization */ + norm_dual(x3, x3, /**/ + y3, y3); /**/ + lnorm(z3, z3); /**/ + + /* T = p_is_inf ? q : T */ + x3 = mask_mov_i64(x3, p_is_inf, *x2); + y3 = mask_mov_i64(y3, p_is_inf, *y2); + z3 = mask_mov_i64(z3, p_is_inf, loadu_i64(p384_r)); + + /* T = q_is_inf ? p : T */ + x3 = mask_mov_i64(x3, q_is_inf, *x1); + y3 = mask_mov_i64(y3, q_is_inf, *y1); + z3 = mask_mov_i64(z3, q_is_inf, *z1); + + r->x = x3; + r->y = y3; + r->z = z3; + return; +} + +IPP_OWN_DEFN(int, ifma_ec_nistp384_is_on_curve, (const P384_POINT_IFMA *p, const int use_jproj_coords)) +{ + /* + * Algorithm + * + * Gueron (2002). Enhanced Montgomery Multiplication. + * In Cryptographic Hardware and Embedded Systems - CHES 2002, + * 4th International Workshop, Redwood Shores, CA, USA, + * August 13-15, 2002, Revised Papers (pp. 46-56) + * [ DOI: 10.1007/3-540-36400-5_5 ] + * + * y^2 = x^3 + a*x + b (1) + * + * if input + * * Jacobian projection coordinate (x,y,z) - represent by (x/z^2,y/z^3,1) + * * Affine coodinate (x/z^2,y/z^3,z/z=1) + * + * mult formala (1) by z^6 + * + * y^2 = x^3 + a*x*z^4 + b*z^6 + */ + const m512 M6 = loadu_i64(p384_x6); + const m512 a = loadu_i64(p384_a); + const m512 b = loadu_i64(p384_b); + + m512 rh, Z4, Z6, tmp; + rh = Z4 = Z6 = tmp = setzero_i64(); + + sqr(rh, p->x); /* rh = x^2 */ + /* rh = x*(x^2 + a*z^4) + b*z^6 = x*(x^2 - 3*z^4) + b*z^6 */ + if (0 != use_jproj_coords) { + sqr(tmp, p->z); /* tmp = z^2 */ + lnorm(tmp, tmp); /**/ + /**/ + sqr(Z4, tmp); /* z4 = z^4 */ + lnorm(Z4, Z4); /**/ + mul(Z6, Z4, tmp); /* z6 = z^6 */ + lnorm(Z6, Z6); /**/ + /**/ + add(tmp, Z4, Z4); /* tmp = 2*z^4 */ + add(tmp, tmp, Z4); /* tmp = 3*z^4 */ + /**/ + sub(rh, rh, tmp); /* rh = x^2 - 3*z^4 */ + add(rh, rh, M6); /**/ + norm(rh, rh); /**/ + /**/ + mul_dual(rh, rh, p->x, /* rh = x*(x^2 - 3*z^4) */ + tmp, Z6, b); /* tmp = b*z^6 */ + /**/ + add(rh, rh, tmp); /* rh = x*(x^2 - 3*z^4) + b*z^6 */ + } + /* rh = x*(x^2 + a) + b */ + else { + add(rh, rh, a); /* rh = x^2 + a */ + lnorm(rh, rh); /**/ + mul(rh, rh, p->x); /* rh = x*(x^2 + a) */ + add(rh, rh, b); /* rh = x*(x^2 + a) + b */ + } + lnorm(rh, rh); /**/ + + /* rl = Y^2 */ + sqr(tmp, p->y); /* tmp = y^2 */ + lnorm(tmp, tmp); /**/ + + /* from mont */ + from_mont(tmp, tmp); + from_mont(rh, rh); + + const mask8 mask = cmp_i64_mask(rh, tmp, _MM_CMPINT_EQ); + return (mask == 0xFF) ? 1 : 0; +} + +#undef add +#undef sub +#undef mul +#undef sqr +#undef div2 +#undef norm +#undef from_mont +#undef mul_dual +#undef sqr_dual +#undef norm_dual + +static __NOINLINE void clear_secret_context(Ipp16u *wval, + Ipp32s *chunk_no, Ipp32s *chunk_shift, + Ipp8u *sign, Ipp8u *digit, + P384_POINT_IFMA *R, P384_POINT_IFMA *H, + P384_POINT_AFFINE_IFMA *A) +{ + *wval = 0; + *chunk_no = 0; + *chunk_shift = 0; + *sign = 0; + *digit = 0; + + if (NULL != R) + (*R).x = (*R).y = (*R).z = setzero_i64(); + if (NULL != H) + (*H).x = (*H).y = (*H).z = setzero_i64(); + if (NULL != A) + (*A).x = (*A).y = setzero_i64(); +} + +#define WIN_SIZE (5) + +__INLINE mask8 is_eq_mask(const Ipp32s a, const Ipp32s b) +{ + const Ipp32s eq = a ^ b; + const Ipp32s v = ~eq & (eq - 1); + const Ipp32s msb = 0 - (v >> (sizeof(a) * 8 - 1)); + return (mask8)(0 - msb); +} + +__INLINE void extract_table_point(P384_POINT_IFMA *r, const Ipp32s digit, const P384_POINT_IFMA *tbl) +{ + Ipp32s idx = digit - 1; + + __ALIGN64 P384_POINT_IFMA R; + R.x = R.y = R.z = setzero_i64(); + + for (Ipp32s n = 0; n < (1 << (WIN_SIZE - 1)); ++n) { + const mask8 mask = is_eq_mask(n, idx); + + R.x = mask_mov_i64(R.x, mask, tbl[n].x); + R.y = mask_mov_i64(R.y, mask, tbl[n].y); + R.z = mask_mov_i64(R.z, mask, tbl[n].z); + } + + r->x = R.x; + r->y = R.y; + r->z = R.z; +} + +#define dbl_point ifma_ec_nistp384_dbl_point +#define add_point ifma_ec_nistp384_add_point +#define neg_coord ifma_neg52_p384 +#define add_point_affine ifma_ec_nistp384_add_point_affine + +/* r = n*P = (P + P + ... + P) */ +IPP_OWN_DEFN(void, ifma_ec_nistp384_mul_point, (P384_POINT_IFMA * r, const P384_POINT_IFMA *p, const Ipp8u *pExtendedScalar, const int scalarBitSize)) +{ + /* default params */ + __ALIGN64 P384_POINT_IFMA tbl[(1 << (WIN_SIZE - 1))]; + + __ALIGN64 P384_POINT_IFMA R; + __ALIGN64 P384_POINT_IFMA H; + + R.x = R.y = R.z = setzero_i64(); + H.x = H.y = H.z = setzero_i64(); + m512 negHy = setzero_i64(); + + /* compute tbl[] = [n]P, n = 1, ... , 2^(win_size - 1) + * tbl[2*n] = tbl[2*n - 1] + p + * tbl[2*n + 1] = [2]*tbl[n] + */ + /* tbl[0] = p */ + tbl[0].x = p->x; + tbl[0].y = p->y; + tbl[0].z = p->z; + /* tbl[1] = [2]*p */ + dbl_point(/* r = */ (tbl + 1), /* a = */ p); + for (int n = 1; n < ((1 << (WIN_SIZE - 1)) / 2); ++n) { + add_point(/* r = */ (tbl + 2 * n), /* a = */ (tbl + 2 * n - 1), /* b = */ p); + dbl_point(/* r = */ (tbl + 2 * n + 1), /* a = */ (tbl + n)); + } + + Ipp16u wval; + Ipp8u digit, sign; + const Ipp32s mask = ((1 << (WIN_SIZE + 1)) - 1); /* mask 0b111111 */ + Ipp32s bit = scalarBitSize - (scalarBitSize % WIN_SIZE); + + Ipp32s chunk_no = (bit - 1) / 8; + Ipp32s chunk_shift = (bit - 1) % 8; + + if (0 != bit) { + wval = *((Ipp16u *)(pExtendedScalar + chunk_no)); + wval = (Ipp16u)((wval >> chunk_shift) & mask); + } else { + wval = 0; + } + + booth_recode(/* sign = */ &sign, /* digit = */ &digit, /* in = */ (Ipp8u)wval, WIN_SIZE); + extract_table_point(/* r = */ &R, /* digit = */ (Ipp32s)digit, /* tbl = */ tbl); + + for (bit -= WIN_SIZE; bit >= WIN_SIZE; bit -= WIN_SIZE) { + dbl_point(&R, &R); + dbl_point(&R, &R); + dbl_point(&R, &R); + dbl_point(&R, &R); +#if (WIN_SIZE == 5) + dbl_point(&R, &R); +#endif + chunk_no = (bit - 1) / 8; + chunk_shift = (bit - 1) % 8; + + wval = *((Ipp16u *)(pExtendedScalar + chunk_no)); + wval = (Ipp16u)((wval >> chunk_shift) & mask); + + booth_recode(/* sign = */ &sign, /* digit = */ &digit, /* in = */ (Ipp8u)wval, WIN_SIZE); + extract_table_point(/* r = */ &H, /* idx = */ (Ipp32s)digit, /* tbl = */ tbl); + + negHy = neg_coord(H.y); + + const mask8 mask_neg = (mask8)(~(sign - 1)); + H.y = mask_mov_i64(H.y, mask_neg, negHy); + + add_point(/* r = */ &R, /* a = */ &R, /* b = */ &H); + } + + /* last window */ + dbl_point(&R, &R); + dbl_point(&R, &R); + dbl_point(&R, &R); + dbl_point(&R, &R); +#if (WIN_SIZE == 5) + dbl_point(&R, &R); +#endif + + wval = *((Ipp16u *)(pExtendedScalar + 0)); + wval = (Ipp16u)((wval << 1) & mask); + + booth_recode(/* sign = */ &sign, /* digit = */ &digit, /* in = */ (Ipp8u)wval, WIN_SIZE); + extract_table_point(/* r = */ &H, /* idx = */ (Ipp32s)digit, /* tbl = */ tbl); + + negHy = neg_coord(H.y); + + const mask8 mask_neg = (mask8)(~(sign - 1)); + H.y = mask_mov_i64(H.y, mask_neg, negHy); + + add_point(/* r = */ &R, /* a = */ &R, /* b = */ &H); + + r->x = R.x; + r->y = R.y; + r->z = R.z; + + /* clear secret data */ + clear_secret_context(&wval, + &chunk_no, &chunk_shift, + &sign, &digit, + &R, &H, NULL); + + return; +} + +#include "ifma_ecprecomp4_p384.h" + +/* mul point base */ +#define BP_WIN_SIZE BASE_POINT_WIN_SIZE +#define BP_N_ENTRY BASE_POINT_N_ENTRY + +__INLINE void extract_point_affine(P384_POINT_AFFINE_IFMA *r, + const P384_POINT_AFFINE_IFMA_MEM *tbl, + const Ipp32s digit) +{ + Ipp32s idx = digit - 1; + + m512 x, y; + x = y = setzero_i64(); + + for (Ipp32s n = 0; n < (1 << ((BP_WIN_SIZE)-1)); ++n, ++tbl) { + const mask8 mask = is_eq_mask(n, idx); + + x = mask_mov_i64(x, mask, loadu_i64(tbl->X)); + y = mask_mov_i64(y, mask, loadu_i64(tbl->Y)); + } + + r->x = x; + r->y = y; +} + +IPP_OWN_DEFN(void, p384r1_select_ap_w4_ifma, (BNU_CHUNK_T * pAffinePoint, const BNU_CHUNK_T *pTable, int index)) +{ + __ALIGN64 P384_POINT_AFFINE_IFMA ap; + + extract_point_affine(&ap, (P384_POINT_AFFINE_IFMA_MEM *)pTable, index); + + ap.x = ifma_frommont52_p384(ap.x); + ap.y = ifma_frommont52_p384(ap.y); + + convert_radix_to_64x6(pAffinePoint, ap.x); + convert_radix_to_64x6(pAffinePoint + LEN64, ap.y); +} + +IPP_OWN_DEFN(void, ifma_ec_nistp384_mul_pointbase, (P384_POINT_IFMA * r, const Ipp8u *pExtendedScalar, int scalarBitSize)) +{ + /* precompute table */ + const P384_POINT_AFFINE_IFMA_MEM *tbl = &ifma_ec_nistp384r1_bp_precomp[0][0]; + + __ALIGN64 P384_POINT_IFMA R; + R.x = R.y = R.z = setzero_i64(); + __ALIGN64 P384_POINT_AFFINE_IFMA A; + A.x = A.y = setzero_i64(); + + m512 Ty = setzero_i64(); + + Ipp16u wval; + Ipp8u digit, sign; + const Ipp32s mask = ((1 << (BP_WIN_SIZE + 1)) - 1); /* mask 0b11111 */ + Ipp32s bit = 0; + Ipp32s chunk_no, chunk_shift; + + wval = *((Ipp16u *)(pExtendedScalar + 0)); + wval = (Ipp16u)((wval << 1) & mask); + + booth_recode(&sign, &digit, (Ipp8u)wval, BP_WIN_SIZE); + extract_point_affine(&A, tbl, (Ipp32s)digit); + tbl += BP_N_ENTRY; + + /* A = sign == 1 ? -A : A */ + Ty = neg_coord(A.y); + mask8 mask_neg = (mask8)(~(sign - 1)); + A.y = mask_mov_i64(A.y, mask_neg, Ty); + + /* R += A */ + add_point_affine(&R, &R, &A); + + for (bit += BP_WIN_SIZE; bit <= scalarBitSize; bit += BP_WIN_SIZE) { + chunk_no = (bit - 1) / 8; + chunk_shift = (bit - 1) % 8; + + wval = *((Ipp16u *)(pExtendedScalar + chunk_no)); + wval = (Ipp16u)((wval >> chunk_shift) & mask); + + booth_recode(&sign, &digit, (Ipp8u)wval, BP_WIN_SIZE); + extract_point_affine(&A, tbl, (Ipp32s)digit); + tbl += BP_N_ENTRY; + + /* A = sign == 1 ? -A : A */ + Ty = neg_coord(A.y); + mask_neg = (mask8)(~(sign - 1)); + A.y = mask_mov_i64(A.y, mask_neg, Ty); + + /* R += A */ + add_point_affine(&R, &R, &A); + } + + r->x = R.x; + r->y = R.y; + r->z = R.z; + + /* clear secret data */ + clear_secret_context(&wval, + &chunk_no, &chunk_shift, + &sign, &digit, + &R, NULL, &A); + return; +} + +#undef dbl_point +#undef add_point +#undef neg_coord +#undef add_point_affine + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecpoint_p384.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecpoint_p384.h new file mode 100644 index 0000000..9dc8b0c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecpoint_p384.h @@ -0,0 +1,200 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +/* Paper referenced in this header: + * + * [1] "Enhanced Montgomery Multiplication" DOI:10.1155/2008/583926 + * + */ + +#ifndef IFMA_ECPOINT_P384_H +#define IFMA_ECPOINT_P384_H + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ifma_arith_p384.h" +#include "pcpbnuimpl.h" + +/* p384 point (x,y,z) */ +typedef struct { + m512 x; + m512 y; + m512 z; +} P384_POINT_IFMA; + +/* p384 affine point(x,y) */ +typedef struct { + m512 x; + m512 y; +} P384_POINT_AFFINE_IFMA; + +/** + * \brief + * + * R = [pExtendedScalar]*P = (P + P + ... + P) + * + * \param[out] r point in radix 2^52 + * \param[in] p point in radix 2^52 + * \param[in] pExtendedScalar pointer to a scalar + * \param[in] scalarBitSize scalar size in bits + */ +IPP_OWN_DECL(void, ifma_ec_nistp384_mul_point, (P384_POINT_IFMA * r, const P384_POINT_IFMA *p, const Ipp8u *pExtendedScalar, const int scalarBitSize)) + +/** + * \brief + * + * R = [pExtendedScalar]*BasePoint + * + * \param[out] r point in radix 2^52 + * \param[in] pExtendedScalar pointer to a scalar + * \param[in] scalarBitSize scalar size in bits + */ +IPP_OWN_DECL(void, ifma_ec_nistp384_mul_pointbase, (P384_POINT_IFMA * r, const Ipp8u *pExtendedScalar, int scalarBitSize)) + +/** + * \brief + * + * Convert point to affine coordinate + * + * \param[out] rx X-affine coordinate + * \param[out] ry Y-affine coordinate + * \param[in] a point in projective coordinates + */ +IPP_OWN_DECL(void, ifma_ec_nistp384_get_affine_coords, (m512 * rx, m512 *ry, const P384_POINT_IFMA *a)) + +/** + * \brief + * + * Check if the point is on curve p384r1 + * + * \param[in] p point in radix 2^52 + * \param[in] use_jproj_coords flag whether coordinates are in Projective of Affine representation + * \return int 1 - true + * 0 - false + */ +IPP_OWN_DECL(int, ifma_ec_nistp384_is_on_curve, (const P384_POINT_IFMA *p, const int use_jproj_coords)) + +/** + * \brief + * + * Point doubling on p384r1 curve (Enhanced Montgomery Algorithm, see [1]) + * + * \param[out] r point in projective coordinates + * \param[in] p point in projective coordinates + */ +IPP_OWN_DECL(void, ifma_ec_nistp384_dbl_point, (P384_POINT_IFMA * r, const P384_POINT_IFMA *p)) + +/** + * \brief + * + * Point addition on p384r1 curve (Enhanced Montgomery Algorithm, see [1]) + * + * \param[out] r result point + * \param[in] p first point + * \param[in] q second point + */ +IPP_OWN_DECL(void, ifma_ec_nistp384_add_point, (P384_POINT_IFMA * r, const P384_POINT_IFMA *p, const P384_POINT_IFMA *q)) + +/** + * \brief + * + * Point addition on p384r1 curve (Enhanced Montgomery Algorithm, see [1]) + * + * \param[out] r point in projective coordinates + * \param[in] p point in projective coordinates + * \param[in] q point in affine coordinates + */ +IPP_OWN_DECL(void, ifma_ec_nistp384_add_point_affine, (P384_POINT_IFMA * r, const P384_POINT_IFMA *p, const P384_POINT_AFFINE_IFMA *q)) + +/** + * \brief + * + * Extracts affine point from the precomputed table. + * + * \param[out] pAffinePoint array of x and y coordinates of affine point in 2^64 radix + * \param[in] pTable pointer to a precomputed table + * \param[in] index index of desired point in the table + */ +IPP_OWN_DECL(void, p384r1_select_ap_w4_ifma, (BNU_CHUNK_T * pAffinePoint, const BNU_CHUNK_T *pTable, int index)) + + +#include "ifma_arith_method.h" +#include "gsmodstuff.h" +#include "pcpgfpstuff.h" +#include "pcpgfpecstuff.h" + +__INLINE void recode_point_to_mont52(P384_POINT_IFMA *pR, + const BNU_CHUNK_T *pP, + BNU_CHUNK_T *pPool, + ifmaArithMethod *method, + gsModEngine *pME) +{ + ifma_import to_radix52 = method->import_to52; + ifma_encode p_to_mont = method->encode; + + const int elemLen = GFP_FELEN(pME); + + BNU_CHUNK_T *pX = pPool; + BNU_CHUNK_T *pY = pPool + elemLen; + BNU_CHUNK_T *pZ = pPool + 2 * elemLen; + + GFP_METHOD(pME)->decode(pX, pP, pME); + GFP_METHOD(pME)->decode(pY, pP + elemLen, pME); + GFP_METHOD(pME)->decode(pZ, pP + 2 * elemLen, pME); + + pR->x = to_radix52((Ipp64u *)pX); + pR->y = to_radix52((Ipp64u *)pY); + pR->z = to_radix52((Ipp64u *)pZ); + + pR->x = p_to_mont(pR->x); + pR->y = p_to_mont(pR->y); + pR->z = p_to_mont(pR->z); +} + +__INLINE void recode_point_to_mont64(const IppsGFpECPoint *pR, + P384_POINT_IFMA *pP, + BNU_CHUNK_T *pPool, + ifmaArithMethod *method, + gsModEngine *pME) +{ + ifma_export to_radix64 = method->export_to64; + ifma_decode p_from_mont = method->decode; + + const int elemLen = GFP_PELEN(pME); + BNU_CHUNK_T *pX = pPool; + BNU_CHUNK_T *pY = pPool + elemLen; + BNU_CHUNK_T *pZ = pPool + 2 * elemLen; + + pP->x = p_from_mont(pP->x); + pP->y = p_from_mont(pP->y); + pP->z = p_from_mont(pP->z); + + to_radix64((Ipp64u *)pX, pP->x); + to_radix64((Ipp64u *)pY, pP->y); + to_radix64((Ipp64u *)pZ, pP->z); + + GFP_METHOD(pME)->encode(ECP_POINT_X(pR), pX, pME); + GFP_METHOD(pME)->encode(ECP_POINT_Y(pR), pY, pME); + GFP_METHOD(pME)->encode(ECP_POINT_Z(pR), pZ, pME); +} + + +#endif // (_IPP32E >= _IPP32E_K1) + +#endif // IFMA_ECPOINT_P384_H diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecpoint_p521.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecpoint_p521.c new file mode 100644 index 0000000..473c8ad --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecpoint_p521.c @@ -0,0 +1,764 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ifma_arith_p521.h" +#include "ifma_ecpoint_p521.h" +#include "pcpgfpecstuff.h" + +#define add(R, A, B) fe521_add_no_red((R), (A), (B)) +#define sub(R, A, B) fe521_sub_no_red((R), (A), (B)) +#define mul(R, A, B) ifma_amm52_p521(&(R), (A), (B)) +#define sqr(R, A) ifma_ams52_p521(&(R), (A)) +#define div2(R, A) ifma_half52_p521(&(R), (A)) +#define norm(R, A) ifma_norm52_p521(&(R), (A)) +#define lnorm(R, A) ifma_lnorm52_p521(&(R), (A)) +#define inv(R, A) ifma_aminv52_p521(&(R), (A)) +#define from_mont(R, A) ifma_frommont52_p521(&(R), (A)) +/* duplicate mult/sqr/norm */ +#define mul_dual(R1, A1, B1, R2, A2, B2) ifma_amm52_dual_p521(&(R1), (A1), (B1), &(R2), (A2), (B2)) +#define sqr_dual(R1, A1, R2, A2) ifma_ams52_dual_p521(&(R1), (A1), &(R2), (A2)) +#define norm_dual(R1, A1, R2, A2) ifma_norm52_dual_p521(&(R1), (A1), &(R2), (A2)) +#define lnorm_dual(R1, A1, R2, A2) ifma_lnorm52_dual_p521(&(R1), (A1), &(R2), (A2)) + +IPP_OWN_DEFN(void, ifma_ec_nistp521_get_affine_coords, (fe521 prx[], fe521 pry[], const P521_POINT_IFMA *a)) +{ + + fe521 z1, z2, z3; + FE521_SET(z1) = FE521_SET(z2) = FE521_SET(z3) = m256_setzero_i64(); + inv(z1, a->z); /* 1/z */ + sqr(z2, z1); /* (1/z)^2 */ + lnorm(z2, z2); /**/ + + /* x = x/z^2 */ + if (NULL != prx) { + mul(*prx, a->x, z2); /* x = x/z^2 */ + lnorm(*prx, *prx); /**/ + } + /* y = y/z^3 */ + if (NULL != pry) { + mul(z3, z1, z2); /* (1/z)^3 */ + lnorm(z3, z3); /**/ + mul(*pry, a->y, z3); /* y = y/z^3 */ + lnorm(*pry, *pry); /**/ + } + + return; +} + +IPP_OWN_DEFN(void, ifma_ec_nistp521_dbl_point, (P521_POINT_IFMA * r, const P521_POINT_IFMA *p)) +{ + /* + * Algorithm (Gueron - Enhanced Montgomery Multiplication) + * l1 = 3x^2 + a*z^4 = (if p384 a = -3) = 3*(x^2 - z^4) = 3*(x - z^2)*(x + z^2) + * z2 = 2*y*z + * l2 = 4*x*y^2 + * x2 = l1^2 - 2*l2 + * l3 = 8*y^4 + * y2 = l1*(l2 - x2) - l3 + * + * sum aripmetic: 8 mul; 9 add/sub; 1 div2. + */ + const fe521 *x1 = &p->x; + const fe521 *y1 = &p->y; + const fe521 *z1 = &p->z; + + fe521 x2; + fe521 y2; + fe521 z2; + FE521_SET(x2) = FE521_SET(y2) = FE521_SET(z2) = m256_setzero_i64(); + + fe521 T, U, V, A, B, H; + FE521_SET(T) = FE521_SET(U) = FE521_SET(V) = m256_setzero_i64(); + FE521_SET(A) = FE521_SET(B) = FE521_SET(H) = m256_setzero_i64(); + + fe521 M2, M4, M8; + FE521_LOADU(M2, p521_x2); /* 2*p */ + FE521_LOADU(M4, p521_x4); /* 4*p */ + FE521_LOADU(M8, p521_x8); /* 8*p */ + + add(T, *y1, *y1); /* T = 2*y1 */ + lnorm(T, T); /**/ + /*=====*/ + sqr_dual(V, T, /* V = 4*y1^2 */ + U, *z1); /* U = z1^2 */ + /*=====*/ + sub(B, *x1, U); /* B = 2*p + x1 - z1^2 */ + add(B, B, M2); /**/ + add(U, *x1, U); /* U = x1 + z1^2 */ + /*=====*/ + /* normalization */ + lnorm_dual(V, V, /**/ + U, U); /**/ + norm(B, B); /**/ + /*=====*/ + mul_dual(A, V, *x1, /* A = 4*x1*y1^2 */ + B, B, U); /* B = (x1 - z1^2)*(x1 + z1^2) */ + /*=====*/ + add(x2, A, A); /* x2 = 8*x1*y1^2 (4p) */ + add(H, B, B); /**/ + add(B, B, H); /* B(l1) = 3*(x1 - z1^2)*(x1 + z1^2) */ + /*=====*/ + /* normalization */ + lnorm(B, B); /**/ + /*=====*/ + sqr_dual(U, B, /* U = l1^2 */ + y2, V); /* y2 = 16*y^2 */ + /*=====*/ + sub(x2, U, x2); /* x2 = 4*p + l1^2 - 2*l2 */ + add(x2, x2, M4); /**/ + div2(y2, y2); /**/ + /*=====*/ + sub(U, A, x2); /* U = 8*p + l2 - x2 */ + add(U, U, M8); /**/ + /*=====*/ + /* normalization */ + norm(U, U); /**/ + /*=====*/ + mul_dual(z2, T, *z1, /* z2 = 2*y1*z1 */ + U, U, B); /* U = B(l1)*(A(l2) - x2) */ + /*=====*/ + sub(y2, U, y2); /* y2 = 2*p + B(l1)*(A(l2) - x2) - y2(l3) */ + add(y2, y2, M2); /**/ + /*=====*/ + /* normalization */ + norm_dual(r->x, x2, /**/ + r->y, y2); /**/ + lnorm(r->z, z2); /**/ + return; +} + +IPP_OWN_DEFN(void, ifma_ec_nistp521_add_point, (P521_POINT_IFMA * r, const P521_POINT_IFMA *p, const P521_POINT_IFMA *q)) +{ + /* + * Algorithm (Gueron - Enhanced Montgomery Multiplication) + * + * A = x1*z2^2 B = x2*z1^2 C = y1*z2^3 D = y2*z1^3 + * E = B - A F = D - C + * x3 = -E^3 - 2*A*E^2 + F^2 + * y3 = -C*E^3 + F*(A*E^2 - x3) + * z3 = z1*z2*E + */ + const fe521 *x1 = &p->x; + const fe521 *y1 = &p->y; + const fe521 *z1 = &p->z; + const mask8 p_is_inf = FE521_IS_ZERO(p->z); + + const fe521 *x2 = &q->x; + const fe521 *y2 = &q->y; + const fe521 *z2 = &q->z; + const mask8 q_is_inf = FE521_IS_ZERO(q->z); + + fe521 x3; + fe521 y3; + fe521 z3; + FE521_SET(x3) = FE521_SET(y3) = FE521_SET(z3) = m256_setzero_i64(); + + fe521 M2, M4, M8; + FE521_LOADU(M2, p521_x2); /* 2*p */ + FE521_LOADU(M4, p521_x4); /* 4*p */ + FE521_LOADU(M8, p521_x8); /* 8*p */ + + fe521 U1, U2, S1, S2, H, R; + FE521_SET(U1) = FE521_SET(U2) = FE521_SET(S1) = m256_setzero_i64(); + FE521_SET(S2) = FE521_SET(H) = FE521_SET(R) = m256_setzero_i64(); + + mul_dual(S1, *y1, *z2, /* s1 = y1*z2 */ + U1, *z2, *z2); /* u1 = z2^2 */ + /*=====*/ + /* normalization */ + lnorm_dual(S1, S1, /**/ + U1, U1); /**/ + /*=====*/ + mul_dual(S2, *y2, *z1, /* s2 = y2*z1 */ + U2, *z1, *z1); /* u2 = z1^2 */ + /*=====*/ + /* normalization */ + lnorm_dual(S2, S2, /**/ + U2, U2); /**/ + /*=====*/ + mul_dual(S1, S1, U1, /* s1 = y1*z2^3 (C) */ + S2, S2, U2); /* s2 = y2*z1^3 (D) */ + /*=====*/ + /* normalization */ + lnorm_dual(S1, S1, /**/ + S2, S2); /* (need by correct compute F = D - C) */ + /*=====*/ + mul_dual(U1, *x1, U1, /* u1 = x1*z2^2 (A) */ + U2, *x2, U2); /* u2 = x2*z1^2 (B) */ + /*=====*/ + /* normalization */ + lnorm_dual(U1, U1, /**/ + U2, U2); /**/ + /*=====*/ + sub(R, S2, S1); /* r = D - C (F) */ + sub(H, U2, U1); /* h = B - A (E) */ + + /* checking the equality of X and Y coordinates (D - C == 0) and (B - A == 0) */ + const mask8 f_are_zero = FE521_IS_ZERO(R); + const mask8 e_are_zero = FE521_IS_ZERO(H); + const mask8 point_is_equal = ((e_are_zero & f_are_zero) & (~p_is_inf) & (~q_is_inf)); + + __ALIGN64 P521_POINT_IFMA r2; + FE521_SET(r2.x) = FE521_SET(r2.y) = FE521_SET(r2.z) = m256_setzero_i64(); + if ((mask8)0xFF == point_is_equal) { + ifma_ec_nistp521_dbl_point(&r2, p); + } + + add(R, R, M2); /**/ + add(H, H, M2); /**/ + /*=====*/ + /* normalization */ + norm_dual(R, R, /**/ + H, H); /**/ + /**/ + mul_dual(z3, *z1, *z2, /* z3 = z1*z2 */ + U2, H, H); /* u2 = E^2 */ + /*=====*/ + /* normalization */ + lnorm_dual(z3, z3, /**/ + U2, U2); /**/ + /**/ + mul_dual(z3, z3, H, /* z3 = (z1*z2)*E */ + S2, R, R); /* s2 = F^2 */ + mul(H, H, U2); /* h = E^3 */ + /*=====*/ + /* normalization */ + lnorm(H, H); /**/ + /*=====*/ + mul(U1, U1, U2); /* u1 = A*E^2 */ + sub(x3, S2, H); /* x3 = F^2 - E^3 */ + add(x3, x3, M2); /**/ + add(U2, U1, U1); /* u2 = 2*A*E^2 */ + mul(S1, S1, H); /* s1 = C*E^3 */ + sub(x3, x3, U2); /* x3 = (F^2 - E^3) -2*A*E^2 */ + add(x3, x3, M4); /**/ + /*=====*/ + sub(y3, U1, x3); /* y3 = A*E^2 - x3 */ + add(y3, y3, M8); /**/ + /*=====*/ + /* normalization */ + norm(y3, y3); /**/ + /**/ + mul(y3, y3, R); /* y3 = F*(A*E^2 - x3) */ + sub(y3, y3, S1); /* y3 = F*(A*E^2 - x3) - C*E^3 */ + add(y3, y3, M2); + + /* normalization */ + norm_dual(x3, x3, /**/ + y3, y3); /**/ + lnorm(z3, z3); /**/ + + /* T = p_is_inf ? q : T */ + FE521_MASK_MOV(x3, x3, p_is_inf, *x2); + FE521_MASK_MOV(y3, y3, p_is_inf, *y2); + FE521_MASK_MOV(z3, z3, p_is_inf, *z2); + + /* T = q_is_inf ? p : T */ + FE521_MASK_MOV(x3, x3, q_is_inf, *x1); + FE521_MASK_MOV(y3, y3, q_is_inf, *y1); + FE521_MASK_MOV(z3, z3, q_is_inf, *z1); + + /* r = point_is_equal ? r2 : T */ + FE521_MASK_MOV(x3, x3, point_is_equal, r2.x); + FE521_MASK_MOV(y3, y3, point_is_equal, r2.y); + FE521_MASK_MOV(z3, z3, point_is_equal, r2.z); + + FE521_COPY(r->x, x3); + FE521_COPY(r->y, y3); + FE521_COPY(r->z, z3); + + return; +} + +IPP_OWN_DEFN(void, ifma_ec_nistp521_add_point_affine, (P521_POINT_IFMA * r, const P521_POINT_IFMA *p, const P521_POINT_AFFINE_IFMA *q)) +{ + /* + * Algorithm (Gueron - Enhanced Montgomery Multiplication) + * + * A = x1 B = x2*z1^2 C = y1 D = y2*z1^3 + * E = B - A(x1) F = D - C(y1) + * x3 = -E^3 - 2*A(x1)*E^2 + F^2 + * y3 = -C(y1)*E^3 + F*(A(x1)*E^2 - x3) + * z3 = z1*E + */ + /* coordinates of p (jacobian projective) */ + const fe521 *x1 = &p->x; + const fe521 *y1 = &p->y; + const fe521 *z1 = &p->z; + const mask8 p_is_inf = FE521_IS_ZERO(p->z); + + /* coodinate of q (affine) */ + const fe521 *x2 = &q->x; + const fe521 *y2 = &q->y; + const mask8 q_is_inf = (FE521_IS_ZERO(q->x) & FE521_IS_ZERO(q->y)); + + fe521 x3; + fe521 y3; + fe521 z3; + FE521_SET(x3) = FE521_SET(y3) = FE521_SET(z3) = m256_setzero_i64(); + + fe521 M2, M4, M8; + FE521_LOADU(M2, p521_x2); /* 2*p */ + FE521_LOADU(M4, p521_x4); /* 4*p */ + FE521_LOADU(M8, p521_x8); /* 8*p */ + + fe521 U2, S2, H, R; + FE521_SET(U2) = FE521_SET(S2) = FE521_SET(H) = FE521_SET(R) = m256_setzero_i64(); + + mul_dual(R, *z1, *z1, /* R = z1^2 */ + S2, *y2, *z1); /* S2 = y2*z1 */ + /*=====*/ + lnorm_dual(R, R, /**/ + S2, S2); /**/ + /*=====*/ + mul_dual(U2, *x2, R, /* U2 = x2*z1^2 (B) */ + S2, S2, R); /* S2 = y2*z1^3 (D) */ + /**/ + sub(H, U2, *x1); /* H = B - A (E) */ + add(H, H, M8); /**/ + sub(R, S2, *y1); /* R = D - C (F) */ + add(R, R, M4); /**/ + /*=====*/ + norm_dual(H, H, /**/ + R, R); /**/ + /**/ + mul(z3, H, *z1); /* z3 = z1*E */ + /**/ + sqr_dual(U2, H, /* U2 = E^2 */ + S2, R); /* S2 = F^2 */ + /**/ + lnorm(U2, U2); /**/ + /**/ + mul(H, H, U2); /* H = E^3 */ + /**/ + lnorm(H, H); /**/ + /**/ + mul_dual(U2, U2, *x1, /* U2 = A*E^2 */ + y3, H, *y1); /* y3 = C*E^3 */ + /**/ + add(x3, U2, U2); /* x2 = 2*A*E^2 */ + sub(x3, S2, x3); /* x3 = F^2 - 2*A*E^2 */ + add(x3, x3, M4); /**/ + sub(x3, x3, H); /* x3 = F^2 - 2*A*E^2 - E^3 */ + add(x3, x3, M2); /**/ + /**/ + sub(U2, U2, x3); /* U2 = A*E^2 - x3 */ + add(U2, U2, M8); /**/ + norm(U2, U2); /**/ + mul(U2, U2, R); /* U2 = F*(A*E^2 - x3) */ + sub(y3, U2, y3); /* y3 = F*(A*E^2 - x3) - C*E^2 */ + add(y3, y3, M2); /**/ + + /* normalization */ + norm_dual(x3, x3, /**/ + y3, y3); /**/ + lnorm(z3, z3); /**/ + + fe521 ONE; + FE521_LOADU(ONE, P521R1_R52); + /* T = p_is_inf ? q : T */ + FE521_MASK_MOV(x3, x3, p_is_inf, *x2); + FE521_MASK_MOV(y3, y3, p_is_inf, *y2); + FE521_MASK_MOV(z3, z3, p_is_inf, ONE); + + /* T = q_is_inf ? p : T */ + FE521_MASK_MOV(x3, x3, q_is_inf, *x1); + FE521_MASK_MOV(y3, y3, q_is_inf, *y1); + FE521_MASK_MOV(z3, z3, q_is_inf, *z1); + + FE521_COPY(r->x, x3); + FE521_COPY(r->y, y3); + FE521_COPY(r->z, z3); + + return; +} + +/* P521 mont(a) */ +const static __ALIGN64 Ipp64u p512_a[P521R1_NUM_CHUNK][P521R1_LENFE521_52] = { { 0x0007FFFFFFFFFFFF, + 0x000FFFFFFFFFFFFE, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF }, + { 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF }, + { 0x000FFFFFFFFFFFFF, + 0x000FFFFFFFFFFFFF, + 0x0000000000000001, + 0x0000000000000000 } }; +/* P521R1 mont(b) */ +const static __ALIGN64 Ipp64u p512_b[P521R1_NUM_CHUNK][P521R1_LENFE521_52] = { { 0x00014654FAE58638, + 0x00028FEA35A81F80, + 0x000C41E961A78F7A, + 0x000DD8DF839AB9EF }, + { 0x00049BD8B29605E9, + 0x0000AB0C9CA8F63F, + 0x0005A44C8C77884F, + 0x00092DCCD98AF9DC }, + { 0x0005B42A077516D3, + 0x000E4D0FC94D10D0, + 0x0000000000000000, + 0x0000000000000000 } }; + +IPP_OWN_DEFN(int, ifma_ec_nistp521_is_on_curve, (const P521_POINT_IFMA *p, const int use_jproj_coords)) +{ + /* + * Algorithm + * + * y^2 = x^3 + a*x + b (1) + * + * if input + * * Jacobian projection coordinate (x,y,z) - represent by (x/z^2,y/z^3,1) + * * Affine coodinate (x/z^2,y/z^3,z/z=1) + * + * mult formala (1) by z^6 + * + * y^2 = x^3 + a*x*z^4 + b*z^6 + */ + fe521 M6; + FE521_LOADU(M6, p521_x6); + fe521 a, b; + FE521_LOADU(a, p512_a); + FE521_LOADU(b, p512_b); + + fe521 rh, Z4, Z6, tmp; + FE521_SET(rh) = FE521_SET(Z4) = FE521_SET(Z6) = FE521_SET(tmp) = m256_setzero_i64(); + + sqr(rh, p->x); /* rh = x^2 */ + /* rh = x*(x^2 + a*z^4) + b*z^6 = x*(x^2 - 3*z^4) + b*z^6 */ + if (0 != use_jproj_coords) { + sqr(tmp, p->z); /* tmp = z^2 */ + lnorm(tmp, tmp); /**/ + /**/ + sqr(Z4, tmp); /* z4 = z^4 */ + lnorm(Z4, Z4); /**/ + mul(Z6, Z4, tmp); /* z6 = z^6 */ + lnorm(Z6, Z6); /**/ + /**/ + add(tmp, Z4, Z4); /* tmp = 2*z^4 */ + add(tmp, tmp, Z4); /* tmp = 3*z^4 */ + /**/ + sub(rh, rh, tmp); /* rh = x^2 - 3*z^4 */ + add(rh, rh, M6); /**/ + norm(rh, rh); /**/ + /**/ + mul_dual(rh, rh, p->x, /* rh = x*(x^2 - 3*z^4) */ + tmp, Z6, b); /* tmp = b*z^6 */ + /**/ + add(rh, rh, tmp); /* rh = x*(x^2 - 3*z^4) + b*z^6 */ + } + /* rh = x*(x^2 + a) + b */ + else { + add(rh, rh, a); /* rh = x^2 + a */ + lnorm(rh, rh); /**/ + mul(rh, rh, p->x); /* rh = x*(x^2 + a) */ + add(rh, rh, b); /* rh = x*(x^2 + a) + b */ + } + lnorm(rh, rh); /**/ + + /* rl = Y^2 */ + sqr(tmp, p->y); /* tmp = y^2 */ + lnorm(tmp, tmp); /**/ + + /* from mont */ + from_mont(tmp, tmp); + from_mont(rh, rh); + + const mask8 mask = FE521_CMP_MASK(rh, tmp, _MM_CMPINT_EQ); + return (mask == 0xF) ? 1 : 0; +} + +#undef add +#undef sub +#undef mul +#undef sqr +#undef div2 +#undef norm +#undef from_mont +#undef mul_dual +#undef sqr_dual +#undef norm_dual + +static __NOINLINE void clear_secret_context(Ipp16u *wval, + Ipp32s *chunk_no, Ipp32s *chunk_shift, + Ipp8u *sign, Ipp8u *digit, + P521_POINT_IFMA *R, P521_POINT_IFMA *H, + P521_POINT_AFFINE_IFMA *A) +{ + *wval = 0; + *chunk_no = 0; + *chunk_shift = 0; + *sign = 0; + *digit = 0; + + if (NULL != R) + FE521_SET((*R).x) = FE521_SET((*R).y) = FE521_SET((*R).z) = m256_setzero_i64(); + if (NULL != H) + FE521_SET((*H).x) = FE521_SET((*H).y) = FE521_SET((*H).z) = m256_setzero_i64(); + if (NULL != A) + FE521_SET((*A).x) = FE521_SET((*A).y) = m256_setzero_i64(); + return; +} + +#define WIN_SIZE (5) + +__INLINE mask8 is_eq_mask(const Ipp32s a, const Ipp32s b) +{ + const Ipp32s eq = a ^ b; + const Ipp32s v = ~eq & (eq - 1); + const Ipp32s msb = 0 - (v >> (sizeof(a) * 8 - 1)); + return (mask8)(0 - msb); +} + +__INLINE void extract_table_point(P521_POINT_IFMA *r, const Ipp32s digit, const P521_POINT_IFMA tbl[]) +{ + Ipp32s idx = digit - 1; + + __ALIGN64 P521_POINT_IFMA R; + FE521_SET(R.x) = FE521_SET(R.y) = FE521_SET(R.z) = m256_setzero_i64(); + + for (Ipp32s n = 0; n < (1 << (WIN_SIZE - 1)); ++n) { + const mask8 mask = is_eq_mask(n, idx); + + FE521_MASK_MOV(R.x, R.x, mask, tbl[n].x); + FE521_MASK_MOV(R.y, R.y, mask, tbl[n].y); + FE521_MASK_MOV(R.z, R.z, mask, tbl[n].z); + } + + FE521_COPY(r->x, R.x); + FE521_COPY(r->y, R.y); + FE521_COPY(r->z, R.z); +} + +#define dbl_point ifma_ec_nistp521_dbl_point +#define add_point ifma_ec_nistp521_add_point +#define neg_coord(R, A) ifma_neg52_p521(&(R), (A)) +#define add_point_affine ifma_ec_nistp521_add_point_affine + +/* r = n*P = (P + P + ... + P) */ +IPP_OWN_DEFN(void, ifma_ec_nistp521_mul_point, (P521_POINT_IFMA * r, const P521_POINT_IFMA *p, const Ipp8u *pExtendedScalar, const int scalarBitSize)) +{ + /* default params */ + __ALIGN64 P521_POINT_IFMA tbl[(1 << (WIN_SIZE - 1))]; + + __ALIGN64 P521_POINT_IFMA R; + __ALIGN64 P521_POINT_IFMA H; + + fe521 negHy; + FE521_SET(negHy) = m256_setzero_i64(); + FE521_SET(R.x) = FE521_SET(R.y) = FE521_SET(R.z) = m256_setzero_i64(); + FE521_SET(H.x) = FE521_SET(H.y) = FE521_SET(H.z) = m256_setzero_i64(); + + /* compute tbl[] = [n]P, n = 1, ... , 2^(win_size - 1) + * tbl[2*n] = tbl[2*n - 1] + p + * tbl[2*n + 1] = [2]*tbl[n] + */ + /* tbl[0] = p */ + FE521_COPY(tbl[0].x, p->x); + FE521_COPY(tbl[0].y, p->y); + FE521_COPY(tbl[0].z, p->z); + /* tbl[1] = [2]*p */ + dbl_point(/* r = */ (tbl + 1), /* a = */ p); + for (int n = 1; n < ((1 << (WIN_SIZE - 1)) / 2); ++n) { + add_point(/* r = */ (tbl + 2 * n), /* a = */ (tbl + 2 * n - 1), /* b = */ p); + dbl_point(/* r = */ (tbl + 2 * n + 1), /* a = */ (tbl + n)); + } + + Ipp16u wval; + Ipp8u digit, sign; + const Ipp32s mask = ((1 << (WIN_SIZE + 1)) - 1); /* mask 0b111111 */ + Ipp32s bit = scalarBitSize - (scalarBitSize % WIN_SIZE); + + Ipp32s chunk_no = (bit - 1) / 8; + Ipp32s chunk_shift = (bit - 1) % 8; + + if (0 != bit) { + wval = *((Ipp16u *)(pExtendedScalar + chunk_no)); + wval = (Ipp16u)((wval >> chunk_shift) & mask); + } else { + wval = 0; + } + + booth_recode(/* sign = */ &sign, /* digit = */ &digit, /* in = */ (Ipp8u)wval, WIN_SIZE); + extract_table_point(/* r = */ &R, /* digit = */ (Ipp32s)digit, /* tbl = */ tbl); + + for (bit -= WIN_SIZE; bit >= WIN_SIZE; bit -= WIN_SIZE) { + dbl_point(&R, &R); + dbl_point(&R, &R); + dbl_point(&R, &R); + dbl_point(&R, &R); +#if (WIN_SIZE == 5) + dbl_point(&R, &R); +#endif + chunk_no = (bit - 1) / 8; + chunk_shift = (bit - 1) % 8; + + wval = *((Ipp16u *)(pExtendedScalar + chunk_no)); + wval = (Ipp16u)((wval >> chunk_shift) & mask); + + booth_recode(/* sign = */ &sign, /* digit = */ &digit, /* in = */ (Ipp8u)wval, WIN_SIZE); + extract_table_point(/* r = */ &H, /* idx = */ (Ipp32s)digit, /* tbl = */ tbl); + + neg_coord(negHy, H.y); + + const mask8 mask_neg = (mask8)(~(sign - 1)); + FE521_MASK_MOV(H.y, H.y, mask_neg, negHy); + add_point(/* r = */ &R, /* a = */ &R, /* b = */ &H); + } + + /* last window */ + dbl_point(&R, &R); + dbl_point(&R, &R); + dbl_point(&R, &R); + dbl_point(&R, &R); +#if (WIN_SIZE == 5) + dbl_point(&R, &R); +#endif + + wval = *((Ipp16u *)(pExtendedScalar + 0)); + wval = (wval << 1) & mask; + + booth_recode(/* sign = */ &sign, /* digit = */ &digit, /* in = */ (Ipp8u)wval, WIN_SIZE); + extract_table_point(/* r = */ &H, /* idx = */ (Ipp32s)digit, /* tbl = */ tbl); + + neg_coord(negHy, H.y); + + const mask8 mask_neg = (mask8)(~(sign - 1)); + FE521_MASK_MOV(H.y, H.y, mask_neg, negHy); + + add_point(/* r = */ &R, /* a = */ &R, /* b = */ &H); + + FE521_COPY(r->x, R.x); + FE521_COPY(r->y, R.y); + FE521_COPY(r->z, R.z); + + /* clear secret data */ + clear_secret_context(&wval, + &chunk_no, &chunk_shift, + &sign, &digit, + &R, &H, NULL); + + return; +} + +#include "ifma_ecprecomp4_p521.h" + +/* affine */ +#define BP_WIN_SIZE BASE_POINT_WIN_SIZE +#define BP_N_ENTRY BASE_POINT_N_ENTRY + +__INLINE void extract_point_affine(P521_POINT_AFFINE_IFMA *r, + const P521_POINT_AFFINE_IFMA_MEM *tbl, + const Ipp32s digit) +{ + Ipp32s idx = digit - 1; + + fe521 x, y; + FE521_SET(x) = FE521_SET(y) = m256_setzero_i64(); + + for (Ipp32s n = 0; n < (1 << ((BP_WIN_SIZE)-1)); ++n, ++tbl) { + const mask8 mask = is_eq_mask(n, idx); + + /* x */ + FE521_LO(x) = m256_mask_mov_i64(FE521_LO(x), mask, m256_loadu_i64(FE521_LO(tbl->x))); + FE521_MID(x) = m256_mask_mov_i64(FE521_MID(x), mask, m256_loadu_i64(FE521_MID(tbl->x))); + FE521_HI(x) = m256_mask_mov_i64(FE521_HI(x), mask, m256_loadu_i64(FE521_HI(tbl->x))); + /* y */ + FE521_LO(y) = m256_mask_mov_i64(FE521_LO(y), mask, m256_loadu_i64(FE521_LO(tbl->y))); + FE521_MID(y) = m256_mask_mov_i64(FE521_MID(y), mask, m256_loadu_i64(FE521_MID(tbl->y))); + FE521_HI(y) = m256_mask_mov_i64(FE521_HI(y), mask, m256_loadu_i64(FE521_HI(tbl->y))); + } + + FE521_COPY(r->x, x); + FE521_COPY(r->y, y); +} + +IPP_OWN_DEFN(void, ifma_ec_nistp521_mul_pointbase, (P521_POINT_IFMA * r, const Ipp8u *pExtendedScalar, int scalarBitSize)) +{ + /* precompute table */ + const P521_POINT_AFFINE_IFMA_MEM *tbl = &ifma_ec_nistp521r1_bp_precomp[0][0]; + + __ALIGN64 P521_POINT_IFMA R; + __ALIGN64 P521_POINT_AFFINE_IFMA A; + fe521 Ty; + FE521_SET(R.x) = FE521_SET(R.y) = FE521_SET(R.z) = m256_setzero_i64(); + FE521_SET(A.x) = FE521_SET(A.y) = m256_setzero_i64(); + FE521_SET(Ty) = m256_setzero_i64(); + + Ipp16u wval; + Ipp8u digit, sign; + const Ipp32s mask = ((1 << (BP_WIN_SIZE + 1)) - 1); /* mask 0b11111 */ + Ipp32s bit = 0; + Ipp32s chunk_no, chunk_shift; + + wval = *((Ipp16u *)(pExtendedScalar + 0)); + wval = (Ipp16u)((wval << 1) & mask); + + booth_recode(&sign, &digit, (Ipp8u)wval, BP_WIN_SIZE); + extract_point_affine(&A, tbl, (Ipp32s)digit); + tbl += BP_N_ENTRY; + + /* A = sign == 1 ? -A : A */ + neg_coord(Ty, A.y); + mask8 mask_neg = (mask8)(~(sign - 1)); + FE521_MASK_MOV(A.y, A.y, mask_neg, Ty); + + /* R += A */ + add_point_affine(&R, &R, &A); + + for (bit += BP_WIN_SIZE; bit <= scalarBitSize; bit += BP_WIN_SIZE) { + chunk_no = (bit - 1) / 8; + chunk_shift = (bit - 1) % 8; + + wval = *((Ipp16u *)(pExtendedScalar + chunk_no)); + wval = (Ipp16u)((wval >> chunk_shift) & mask); + + booth_recode(&sign, &digit, (Ipp8u)wval, BP_WIN_SIZE); + extract_point_affine(&A, tbl, (Ipp32s)digit); + tbl += BP_N_ENTRY; + + /* A = sign == 1 ? -A : A */ + neg_coord(Ty, A.y); + mask_neg = (mask8)(~(sign - 1)); + FE521_MASK_MOV(A.y, A.y, mask_neg, Ty); + + /* R += A */ + add_point_affine(&R, &R, &A); + } + + FE521_COPY(r->x, R.x); + FE521_COPY(r->y, R.y); + FE521_COPY(r->z, R.z); + + /* clear secret data */ + clear_secret_context(&wval, + &chunk_no, &chunk_shift, + &sign, &digit, + &R, NULL, &A); + return; +} + +#undef dbl_point +#undef add_point +#undef neg_coord +#undef add_point_affine + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecpoint_p521.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecpoint_p521.h new file mode 100644 index 0000000..6cdb335 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecpoint_p521.h @@ -0,0 +1,162 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ +#ifndef IFMA_ECPOINT_P521_H +#define IFMA_ECPOINT_P521_H + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ifma_defs_p521.h" + +typedef struct P521_POINT_IFMA { + fe521 x; + fe521 y; + fe521 z; +} P521_POINT_IFMA; + +typedef struct P521_POINT_AFFINE_IFMA { + fe521 x; + fe521 y; +} P521_POINT_AFFINE_IFMA; + +/** + * \brief + * compute R = [pExtendedScalar]*P = (P + P + ... + P) + * \param[out] r point (in radix 2^52) + * \param[in] p point (in radix 2^52) + * \param[in] pExtendedScalar ptr Extended scalar + * \param[in] scalarBitSize size bits scalar + */ +IPP_OWN_DECL(void, ifma_ec_nistp521_mul_point, (P521_POINT_IFMA * r, const P521_POINT_IFMA *p, const Ipp8u *pExtendedScalar, const int scalarBitSize)) + +/** + * \brief + * compute [pExtendedScalar]*BP + * \param[out] r point (in radix 2^52) + * \param pExtendedScalar ptr scaler (length 384 bits) + */ +IPP_OWN_DECL(void, ifma_ec_nistp521_mul_pointbase, (P521_POINT_IFMA * r, const Ipp8u *pExtendedScalar, int scalarBitSize)) + +/** + * \brief + * convert point to affine coordinate + * \param[out] prx affine X coordinate + * \param[out] pry affine Y coordinate + * \param[in] a point + */ +IPP_OWN_DECL(void, ifma_ec_nistp521_get_affine_coords, (fe521 prx[], fe521 pry[], const P521_POINT_IFMA *a)) + +/** + * \brief + * check point on curve P521R1 + * \param[in] p point (in radix 2^52) + * \param[in] use_jproj_coords is Jacobian Projection coordinate or Affine Coordinate + * \return int 1 - is true | 0 - is false + */ +IPP_OWN_DECL(int, ifma_ec_nistp521_is_on_curve, (const P521_POINT_IFMA *p, const int use_jproj_coords)) + +/** + * \brief + * compute double point P521R1 Enhanced Montgomery Algorithm + * \param[out] r point + * \param[in] p value point (in radix 2^52) + */ +IPP_OWN_DECL(void, ifma_ec_nistp521_dbl_point, (P521_POINT_IFMA * r, const P521_POINT_IFMA *p)) + +/** + * \brief + * compute add point P384R1 Enhanced Montgomery Algorithm + * \param[out] r point + * \param[in] p first point (in radix 2^52) + * \param[in] q second point (in radix 2^52) + */ +IPP_OWN_DECL(void, ifma_ec_nistp521_add_point, (P521_POINT_IFMA * r, const P521_POINT_IFMA *p, const P521_POINT_IFMA *q)) + +/** + * \brief + * compute add point affine P384R1 Enhanced Montgomery Algorithm + * \param[out] r point + * \param[in] p first point (in radix 2^52) + * \param[in] q second affine point (in radix 2^52) + */ +IPP_OWN_DECL(void, ifma_ec_nistp521_add_point_affine, (P521_POINT_IFMA * r, const P521_POINT_IFMA *p, const P521_POINT_AFFINE_IFMA *q)) + +#include "ifma_arith_method_p521.h" +#include "gsmodstuff.h" +#include "pcpgfpstuff.h" +#include "pcpgfpecstuff.h" + +__INLINE void recode_point_to_mont52(P521_POINT_IFMA *pR, + const BNU_CHUNK_T *pP, + BNU_CHUNK_T *pPool, + ifmaArithMethod_p521 *method, + gsModEngine *pME) +{ + ifma_import to_radix52 = method->import_to52; + ifma_encode p_to_mont = method->encode; + + const int elemLen = GFP_FELEN(pME); + + BNU_CHUNK_T *pX = pPool; + BNU_CHUNK_T *pY = pPool + elemLen; + BNU_CHUNK_T *pZ = pPool + 2 * elemLen; + + GFP_METHOD(pME)->decode(pX, pP, pME); + GFP_METHOD(pME)->decode(pY, pP + elemLen, pME); + GFP_METHOD(pME)->decode(pZ, pP + 2 * elemLen, pME); + + to_radix52(&(pR->x), (Ipp64u *)pX); + to_radix52(&(pR->y), (Ipp64u *)pY); + to_radix52(&(pR->z), (Ipp64u *)pZ); + + p_to_mont(&(pR->x), pR->x); + p_to_mont(&(pR->y), pR->y); + p_to_mont(&(pR->z), pR->z); +} + +__INLINE void recode_point_to_mont64(IppsGFpECPoint *pR, + P521_POINT_IFMA *pP, + BNU_CHUNK_T *pPool, + ifmaArithMethod_p521 *method, + gsModEngine *pME) +{ + ifma_export to_radix64 = method->export_to64; + ifma_decode p_from_mont = method->decode; + + const int elemLen = GFP_PELEN(pME); + BNU_CHUNK_T *pX = pPool; + BNU_CHUNK_T *pY = pPool + elemLen; + BNU_CHUNK_T *pZ = pPool + 2 * elemLen; + + p_from_mont(&(pP->x), pP->x); + p_from_mont(&(pP->y), pP->y); + p_from_mont(&(pP->z), pP->z); + + to_radix64((Ipp64u *)pX, pP->x); + to_radix64((Ipp64u *)pY, pP->y); + to_radix64((Ipp64u *)pZ, pP->z); + + GFP_METHOD(pME)->encode(ECP_POINT_X(pR), pX, pME); + GFP_METHOD(pME)->encode(ECP_POINT_Y(pR), pY, pME); + GFP_METHOD(pME)->encode(ECP_POINT_Z(pR), pZ, pME); +} + + +#endif // (_IPP32E >= _IPP32E_K1) + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecprecomp4_p256.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecprecomp4_p256.h new file mode 100644 index 0000000..6d4d9b4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecprecomp4_p256.h @@ -0,0 +1,716 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#if 0 + +#ifndef IFMA_ECPRECOMP4_P256_H +#define IFMA_ECPRECOMP4_P256_H + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ifma_defs.h" +#include "pcpgfpecstuff.h" + +#define BASE_POINT_WIN_SIZE (4) +#define BASE_POINT_N_ENTRY (1 << ((BASE_POINT_WIN_SIZE)-1)) + +#define OPERAND_BITSIZE (256) +#define LEN52_P256 (NUMBER_OF_DIGITS(OPERAND_BITSIZE, DIGIT_SIZE)) + +/* P256 affine point */ +typedef struct { + BNU_CHUNK_T X[LEN52_P256]; + BNU_CHUNK_T Y[LEN52_P256]; +} P256_POINT_AFFINE_IFMA_MEM; + +extern const __ALIGN64 P256_POINT_AFFINE_IFMA_MEM ifma_ec_nistp256r1_bp_precomp[64][BASE_POINT_N_ENTRY]; + +#if !defined(_DISABLE_ECP_256R1_HARDCODED_BP_TBL_) + +/* Montgomery coefficient R = 2^(6*52) mod p */ +const __ALIGN64 P256_POINT_AFFINE_IFMA_MEM ifma_ec_nistp256r1_bp_precomp[][BASE_POINT_N_ENTRY] = { + { /* digit=0 (1,2,..,8)*(2^{0})*G */ + {{0x0008905f76bdc7b5,0x0007bd418a9143c1,0x000c475d568abc1f,0x0009a701075ba95f,0x00003d1f32c8b4b9},{0x000571ff18aafa5c,0x000f757ce95560a8,0x000434a7e54432f7,0x0002797258b4ab8e,0x00009df870e37032}}, + {{0x0006bb32e5348a6d,0x000d9d410ddd64df,0x0000ad784f985075,0x0008a23d9aa6ae3c,0x00001fb0f13f1e58},{0x0008c57751832101,0x00047d361bee1a57,0x000b725df8a6ac15,0x000e32cbe152cd7c,0x00008c240484bd0e}}, + {{0x0006936a3fdd92de,0x00061904eebc1272,0x0009e1ea17bc2219,0x00038de98b027f84,0x0000be1daceb9daa},{0x0005f06a2abb7836,0x000261fc983a7ebd,0x000c327193eff4d4,0x000b6b38e583e47a,0x0000315e09d4065e}}, + {{0x00077362f599237a,0x0003b0d46918dcc5,0x000d6eb05e7ddb8d,0x000ec90f24650a6e,0x0000604e8ac3b74e},{0x0004b14c39d10c04,0x000ee4ce4cbaba68,0x00017625a80d5c8a,0x000d1ce2e1762847,0x00009cb7c6eea514}}, + {{0x0009079606520cb9,0x000d1aec45c61f5c,0x0009cbb1bd776c0e,0x0006a090c90ec649,0x0000ce1d21d8a47e},{0x0003a076bba1725f,0x0003c4ae7ba4f107,0x000f724b117049be,0x00007c06873c568e,0x0000d1631291cbdd}}, + {{0x0003074b59672881,0x000c6373e77664a1,0x000f2165a2e4d910,0x000ef22ad55ae744,0x0000cd292bcbc0f3},{0x0004c9916deed188,0x000da20d377e44ba,0x000309358347a501,0x000921711299c2b5,0x0000cd3e2ccadf00}}, + {{0x0003ba5119d6cc8f,0x000a64ea0173b4f1,0x0003be81afdd3079,0x000572c082bd2021,0x00001db750e89b35},{0x000aedd9165911e1,0x000ef303f5b9d4de,0x000172b8a2c6cf35,0x000b760956742f2f,0x00007d5db743c61e}}, + {{0x000763891f69c8ac,0x000a0d19499a78fb,0x0004b837ab35be28,0x00064506b462ab5c,0x00002c41f6130b86},{0x0006ec12f2cdf2f2,0x000b1a9532d49775,0x000d78b2a72327aa,0x0006dc9f021e3327,0x00006889435a91f0}}, + }, + { /* digit=1 (1,2,..,8)*(2^{4})*G */ + {{0x0004d63c80e5d9f3,0x00018650bc6fb805,0x0004eb27f1ea9ab1,0x000aa02495882e07,0x0000a466f2e5fb46},{0x0004edf7b266ddd9,0x00042d652a23f9b6,0x0008e61d6dd58c13,0x0007a359e3670c31,0x0000d257b443894a}}, + {{0x0009227077a277a3,0x00059cfc5e3a3d83,0x000c07576f48364e,0x000355e97befd904,0x00001c25c29ce15c},{0x000231d792f7c39a,0x000a46ddbef42f56,0x0003e5d8c175121e,0x000d694407e449b6,0x00003711285e87b5}}, + {{0x00084039030f437c,0x000c53077adc612c,0x0008f397ca9d5bf0,0x0004489bda749652,0x0000af611eeda45e},{0x0005efc7e9c72224,0x0007b66175adff18,0x000b31e8fdcd72d3,0x0007b51e30b26d7f,0x0000abe9a851e48b}}, + {{0x0005fb8394ec96bc,0x000daf305968b809,0x000272a3f024eed6,0x0002f6201380a091,0x00005bd44243eed6},{0x000967204437eb69,0x000fc1280edfe2f2,0x000966478f3e6250,0x00056e2f88f726bb,0x0000b48e94ed5c40}}, + {{0x0000bfffc5132382,0x0005a0752b3e584c,0x0007f5386a673608,0x0005a637335ff9aa,0x0000ee72d7a3789f},{0x000995919cce62b7,0x0000b2a1d49c6052,0x0005898b5a5e55c7,0x000be3d281b037d7,0x0000fcf783c6d6c3}}, + {{0x0001eddf7fe28eb9,0x000d42333e12b701,0x000b07ef531b9d9c,0x0002d7c4d6d490a4,0x00008ce594304c88},{0x000f2a78b36344a4,0x000df28d87fd1d83,0x000cbdd7df5fbadb,0x000e518a2fd2f4f8,0x00002427ceb81118}}, + {{0x0001704d00faa90e,0x000f7760a99cbca7,0x00059333dbf16feb,0x00089ac0c94e258f,0x000067cee41b4c7d},{0x0005000c252c9746,0x000650ef88b8b706,0x000e5b0ea5e2af00,0x0009f378056feb92,0x000066ae9a8c4350}}, + {{0x000d311f2cf406c5,0x00006817a2ad62aa,0x0008ff31e0276fbf,0x000a6d8b57ef2b6b,0x0000437e1f6177bf},{0x000a38d321e63954,0x0002c87c2ff3b6df,0x00023c3f61ddd141,0x0009d5d38b46feaf,0x0000a9e4aaa74b8f}}, + }, + { /* digit=2 (1,2,..,8)*(2^{8})*G */ + {{0x000c3da1963d1b37,0x00058fa696946fc7,0x00083d8e03d70b52,0x000d8231550fbc6d,0x0000c23f1ad7a853},{0x00099d49ebcaeaa8,0x00055b03cfd5d8b3,0x00024f00b2ba1b89,0x000385daa0704b7c,0x00003f881bb6342a}}, + {{0x00042370d35642b3,0x000ce10399492969,0x000aa1fa40e1eea0,0x0002bfcf55b63827,0x00006300726619ca},{0x000dc61f92142820,0x0008f2ee0d985a17,0x000af47364dbdea2,0x0003af2f8192cc64,0x00009a126d8fb59a}}, + {{0x0005d5bbfc230970,0x000353eed4c37174,0x000ef73916a712cd,0x0004acaa59f8240c,0x0000b069410109c2},{0x0004d5e0fcf149d6,0x000f673137fd28e3,0x000f92655f0108ba,0x000ab48f2862ac6e,0x00009efcb3eef840}}, + {{0x000da26e1aec73e6,0x0009be144cc3adda,0x0005da1c14b462fc,0x000697e72d5ffa1d,0x00007dc85a835122},{0x0001c794b1a357c1,0x000661563483caaf,0x00083d98e34b8417,0x000eead3ce6924cd,0x0000cfb7382ae666}}, + {{0x000a786d62a93d0d,0x0002b5c08369a907,0x000a9e9647958a75,0x000519f802990c59,0x000076b05f00e31d},{0x000b2ce9e430d6f9,0x000b4be20ac3a8dd,0x00007f8d434c0323,0x000cdfc6531a2178,0x00009e7647365eaa}}, + {{0x000f3a8f9d8cc3f9,0x00072f38bd7aff18,0x000a8e4721a92651,0x00064b0151b5ee4c,0x00003b792b15a449},{0x000255f53b1b9da9,0x00010c9a0748be7d,0x0004d6d86ce4578c,0x00072ff371d391c9,0x0000702e661ee631}}, + {{0x0008c889de2e32a8,0x000b0d474a861082,0x000fc583d3b1de17,0x0003aa480f8048a8,0x00003ee931e746ba},{0x0001c240230a2d47,0x0001f2eb214a040f,0x0007461ec3244db5,0x0007f2d8ae8c48e9,0x0000149fdf2f3f76}}, + {{0x000c5fe1306bd6ce,0x00056b9eb7926b83,0x0003915e73d8d154,0x00006960bbff88cb,0x000076006a867f79},{0x000daf6b21ea15f4,0x0006ef43764fb3df,0x000733a24b0b851f,0x00086f1f37b5af31,0x000030c22394a5ed}}, + }, + { /* digit=3 (1,2,..,8)*(2^{12})*G */ + {{0x000a20ed2a8c1c6f,0x00041e5ab4b35a24,0x0001abcdbda4db56,0x0003f26801d8b600,0x000004b48afeb9ed},{0x000a24142725a709,0x00053d44fec01e63,0x000c6ddb3c8679c1,0x000868efd3ed7ddc,0x0000203192c73ce6}}, + {{0x00058ae74fac8317,0x0008162d5d721d5b,0x0002a648ba32d37d,0x0001fb6a40c3357a,0x0000d41eebd7ebce},{0x0007312ec0f21311,0x00009812e6dad6be,0x00010d0c3d9a387e,0x00083b3d8de28605,0x00000e56beae8107}}, + {{0x000fb3a0d6a300fe,0x0006afeaae6ee702,0x0005264f0ff2ccae,0x00093072a78f4cc1,0x000056f1807ed1cb},{0x0002f775e441af97,0x000f83bb25a43ba1,0x000106f6b36db209,0x00000ca2a30bf803,0x0000b4aeb7054ee0}}, + {{0x000161957d933aea,0x00028e76e6485b37,0x000b05babd7b6596,0x0000f456d4d3e24d,0x0000ce6abd42c3d5},{0x000aefa7aee9d873,0x0009f83cdb12a6c3,0x0001f36139b52f8a,0x0008e5973bf3fa88,0x0000e7a50e16b447}}, + {{0x000497cd9642f15e,0x000a28037a929a9e,0x000d3566f9270405,0x000ba2d6265601a8,0x000064b1d8ef45cf},{0x000a573239e26ada,0x0005e7f9aa5b2f93,0x000f2d605dae4d62,0x000f28430e5faa3f,0x000025c3e2fb5346}}, + {{0x0009b7e2c88699d1,0x000576bd17c24093,0x0003296e6ad3635b,0x0005cb70f2c4e479,0x00000b645230e891},{0x000c2f42b8f4bf96,0x0005122d096f4a9c,0x000af2a6a3deda09,0x0006383116861aec,0x000032eb61a73777}}, + {{0x000707b18213a914,0x000b751d0a917b49,0x0002b19247af2039,0x000f9e2b6c2cfa3f,0x0000392f6ab1c9d5},{0x00021fa98a0d29bf,0x000e272d5f8b10d2,0x00052db4aaa0a8d8,0x0006faa92fd30783,0x00006bd9fc3c53d0}}, + {{0x0003f63771beff8f,0x0006f623b36f9fd2,0x0003d9eb684ebab1,0x00084f3ef26543b2,0x000096ae77bc2ed6},{0x000fb89d795ff65e,0x0007796b6a1142e5,0x0005d3a20e91b4d9,0x00011bfd55ea6aac,0x0000a386dea94493}}, + }, + { /* digit=4 (1,2,..,8)*(2^{16})*G */ + {{0x000564bade8d111c,0x00049fce3779ee34,0x000d77e4a2bf81f0,0x00017327ae00e7f9,0x0000f7675150c393},{0x0007f7ccb1668c80,0x000b12ac4708e8e4,0x000b5f26ce2e2a59,0x000dbcb9f4f5725f,0x0000c707f408fcd6}}, + {{0x000dbde4ac826d0d,0x0007c0361a341c13,0x000fcf3f7c95b3aa,0x000c6246c3604dc5,0x0000ce46d1be3f02},{0x000f0d1f30affc92,0x0006a5d1c7eef696,0x000562423070d98e,0x0003f8fca5889680,0x00003b45ef942f70}}, + {{0x000c9a9d2168c124,0x000671dbe7a2af37,0x0008d31cad5154da,0x000c67297ec51caa,0x0000f6a5ba0129fa},{0x00003937564cf036,0x000ae4f6ef5f0145,0x000298d331a9ca83,0x00051e01e8abcf41,0x0000803532cecc34}}, + {{0x000688350470c6e0,0x000970977f7195ad,0x0006d7634e8b89f8,0x00088212d8ec8616,0x000043845130a44a},{0x000f5c2ee645cfc9,0x0005bf8309004c28,0x000584a3ba78a88d,0x0005275a4a96e4b7,0x000038c353982c77}}, + {{0x0006877b288e2940,0x000f33959145a65f,0x000803b8e80eaf37,0x0006112a0cd9bc36,0x0000b81a72e0ba9f},{0x00040fa5a0fd54f6,0x0008b9bf5cbdb258,0x000859f95ca88625,0x00044185f2a4fd1d,0x0000bbdb6fce4677}}, + {{0x0000d8b2dae7ca94,0x000fcb36b0cf82eb,0x0005753d030cfdc3,0x000d1805a6438d24,0x0000a53ea3da217e},{0x000afa8c8dcb9f12,0x000c8dbd9184c4d2,0x0005ce430ce88936,0x000756b473dbb18d,0x000096aae294c905}}, + {{0x000dfe0d903a457a,0x0001207e3533d77b,0x000970360f08ec9b,0x000e84aedd7222e6,0x00008819a858ad52},{0x000eb874ec49378f,0x0000bb61d056605f,0x000587efe032d6ae,0x000c3fe8ca9ea9df,0x0000fbcf08ae31fe}}, + {{0x00022a049341c337,0x00033a9bd7638026,0x00000306390eab7d,0x00022f7c4c21486a,0x0000d141475b8b86},{0x000540cb1b1be032,0x000da9dbb7a8ee0d,0x000b96c0e3c063f7,0x000de42ecdc2ed3b,0x0000397c62c0f6ec}}, + }, + { /* digit=5 (1,2,..,8)*(2^{20})*G */ + {{0x000e6e2135aa3e34,0x000406a584c5e205,0x00015b6e4efffd6a,0x00016589b267e4ea,0x000028a50a289a0f},{0x000fef4e6229a8b1,0x0001b6a8c8f936e6,0x000445d2f37ab589,0x000b64a868af84d0,0x00002d1d526d0d0a}}, + {{0x0005c78880bdb670,0x0001f09257226088,0x000a8d556d463880,0x000fe87ac131260d,0x0000a978da1cbb3c},{0x00081d1c0fbab2fc,0x00085ffc3bdf24eb,0x000b3da6a9989162,0x0009b73405bff75c,0x0000d9387ea82fe6}}, + {{0x00091ae46f2b2572,0x000703d442a8ad11,0x000ced9edcf5c7ca,0x000e533e5c3816a7,0x0000d4bbe9bb1cb0},{0x000a4a73efeed78e,0x000f18700611c5bf,0x000704c62558948e,0x00091cb5283171ed,0x00000228300708db}}, + {{0x0002b79aff1b7640,0x0003cc8f43a730f8,0x000bb0f3be0fe168,0x0008ce40be89b6f3,0x0000da60e29a2a91},{0x000b662498ee8663,0x000c556962fe5de3,0x000b255f6d5a6808,0x000a245580c590ad,0x0000e4d2e7887b5b}}, + {{0x00031ec55d438d92,0x000bcd082dd1b6ad,0x000794983e52056e,0x000cfd8aa5783a13,0x00009f94d2983cbc},{0x00042b80509ea4d5,0x000fe3f8e9a76414,0x000cc5621e2a00e9,0x000e39f93625453e,0x0000b6fe3802944d}}, + {{0x000414582cf35827,0x000a80d6c39765a2,0x000d882871de9e8b,0x0003d1450a2db3ac,0x000046f7d2fddbd8},{0x000672303b75f511,0x0007abc60eee9a8a,0x0001cc28700717f7,0x0003de4d554c79f0,0x00009aa79fb6b0ef}}, + {{0x0006940f54c45508,0x000502d3bfab839b,0x000378f6cb89828a,0x0002513c392474e0,0x0000cd44686f1550},{0x000e788bf3109a0c,0x00082f2f2759f255,0x000948a168d333d9,0x000b6b7459dd9578,0x0000c7da7b3800f6}}, + {{0x0004ff052f5917b5,0x000faa8719c05631,0x000ca760c26fde52,0x0005d57d3cd8ad2d,0x0000bd74588b7072},{0x0006ad71d02a024a,0x0003acf3e2d65208,0x00078da16339cd4e,0x000f011987ebdf0f,0x0000079790c922a9}}, + }, + { /* digit=6 (1,2,..,8)*(2^{24})*G */ + {{0x000f5d55f460757a,0x0004ec4868af75df,0x000e466b71f6795b,0x0000c72b0d7325cf,0x0000e10c5f3da26c},{0x00094ad8a7522d79,0x000cc411977a0ee8,0x0004e8d5601f6c03,0x0001e3b955085c4c,0x000028f635d618a7}}, + {{0x00066010689e1641,0x000f3b22c11bb373,0x000db6ec643ab5eb,0x000afb15d2d0366d,0x0000ab615f997250},{0x000b5f7e8a49c22d,0x000402c68d7b6d44,0x0008c124aab7be0b,0x000a684eea8f74f0,0x00006ee0d7c27a2d}}, + {{0x00079739f7318d32,0x00027b21994ef202,0x00093bf3a921a053,0x000dc5e102a653b6,0x0000417acbe360fd},{0x000b40cff123155f,0x000073c8b799336a,0x000c74fb67fa3a8f,0x000c69c2dfdfdf14,0x00008baaad38eb7e}}, + {{0x000984d8c0756da9,0x000f60b879fbbed4,0x000691183fa97cc8,0x00008c6450ff0ae8,0x0000bc7e3aa798e1},{0x0002c0087c8490da,0x000bba6d21008f03,0x0000075a13ccfa3b,0x000db76f329bd54a,0x0000cdfd68fa494d}}, + {{0x00075553bbd18331,0x0005fabfbaf50a55,0x000b10f8b3952494,0x000f37f05df55e76,0x000083667f4aec1f},{0x0001b91149c64f10,0x0006ca3553afa736,0x000beb9cb1077179,0x000ef4d35315f3ff,0x0000724cdc42e0e8}}, + {{0x0009f8db144f77e9,0x000026662b5f3af3,0x0000d0c25c1ff887,0x0007b73b1cefab56,0x0000cd5a1cf3f9ca},{0x000d5a818694755a,0x000725b3c61bfd60,0x000d33c6e57d6b39,0x000d8c2c2a5a4d41,0x0000da06aa84668d}}, + {{0x000633d510319630,0x0000a4956f908239,0x000fdde9a26b1a49,0x000f5c760ba0f507,0x000011a47fccb3c1},{0x00074a4f6a780782,0x0003543b8c9de617,0x0003e69cf7b58be5,0x000c69ab2b475125,0x000059a05b225efc}}, + {{0x00041b32e5ee0cd0,0x000bea601799a527,0x00014451f66c2b00,0x000f25a7ca20cec4,0x00001caa549ccc91},{0x000641caaadc47ea,0x00097814eb57d471,0x0004006d25c11bc5,0x000a4cb887a2d0ed,0x000005cf797f0540}}, + }, + { /* digit=7 (1,2,..,8)*(2^{28})*G */ + {{0x000cf9a2f2b6ddb7,0x000cb4f20151427a,0x0005e5495f13c8bb,0x000336c57206828b,0x0000deb474abe52a},{0x000c630fc54290e0,0x000e9da279153564,0x0006b3bbef1e9949,0x00021a8e27e0734b,0x0000614b2885f690}}, + {{0x0004ab68d7e959ee,0x000227a1ac2703b2,0x0008471ceedd9bdd,0x00080ded1b49258d,0x0000011c8d9a06d3},{0x0000b360f8d06cf6,0x000bfd4734e59d08,0x000843e26b87ae6b,0x000b0c3b35f8740c,0x0000b6d468726158}}, + {{0x000ac1e3e5d6df58,0x000075d5fa067d1d,0x000ebba6b226d7b2,0x000af82ac4134b57,0x00001bb953caf0cd},{0x000cde538c8bc9c2,0x0003669d81fbb26c,0x000be58de047db86,0x0007841e8449ed3e,0x0000d643fef5794d}}, + {{0x0003331d3c69f2bc,0x0000a8da61a76fa3,0x000d96e2379febe3,0x000486b9c598b441,0x0000390a58c27d0d},{0x000cdb9daf4c5417,0x0006d8e298de399d,0x0006c4c6e1a6e0f8,0x000649470cfdb8e7,0x0000690440cf3b0b}}, + {{0x000cd3d0d4027b25,0x000d3c2019f2a596,0x00035e7beba07964,0x000164eddb3ce5bb,0x00000d8037d39488},{0x0000c29968554c2a,0x0009ad6efed021c1,0x000059b49a323da0,0x0003c9c4a09a9f9b,0x0000c0cc53fbad1c}}, + {{0x000c73d2287162dd,0x000c529ed6e82ef1,0x00034977dd25e6f6,0x0004a8aae16f338e,0x0000dc61c21e5eed},{0x00096fe447e4eae7,0x0006a3174a5c8c7d,0x000fa5f9c92267ca,0x0007c117e01cd296,0x000068abc8039604}}, + {{0x00041d6367599350,0x000706e88fbd3813,0x00042818412bda97,0x0005e6bcd02cd4a8,0x0000b40e707676fb},{0x0006fe0a3a3a6e7f,0x000d8fbd238202b9,0x000a1286e09fb174,0x000d4efa0600b4d1,0x00005566cd140a17}}, + {{0x000246780b6116f2,0x00015367f636a38c,0x00064c305dbfeb38,0x0009f998b9f943fb,0x00002a4f058aaf47},{0x000300fadd2ed37d,0x00097156d549fc2b,0x0000ad7686ed0d31,0x00005d713c2953f3,0x0000c01130ae8caa}}, + }, + { /* digit=8 (1,2,..,8)*(2^{32})*G */ + {{0x000953c50eb5331d,0x00069024147519ad,0x000b4d5fade16af5,0x00052cec8d0981ea,0x0000f785b3ffbbf2},{0x00063ebb7f24f144,0x0001dccfd590f8f8,0x000abea80d987872,0x000fa20cf72e9626,0x0000721e8be990fd}}, + {{0x0009fb18a0fee18b,0x000f1f5678a31b0f,0x000d3f9d9d7f8899,0x0002e2896d50301a,0x0000467828ca24c1},{0x000a2f2a98bf1696,0x00042ebe8758851b,0x0005a36a8fac4075,0x000e6fdb71e64e4c,0x00002450761a46be}}, + {{0x000b89a4e82f20ac,0x0005be0e90fb21e5,0x0000601e57b9726e,0x000acbbb500fd2b8,0x0000def3cf863f66},{0x000f548f31add03f,0x0006e3f894407ce5,0x00040ccdd200e1b3,0x00036823ba0025b9,0x000057161113964b}}, + {{0x000379ae15e41fc7,0x0000ca7c35d87949,0x00076fdd0ccf2ac7,0x000135bfd042e655,0x00007cefff827780},{0x000b09f648a49efa,0x000b462fcc9981ea,0x0000c21f3e7dc01a,0x000af6bebaed3d63,0x0000701de9b6b9b9}}, + {{0x000bdcfbd504057f,0x00054d331b85f091,0x000a8cadea75bdae,0x000e6d1fd0f45354,0x00002490dea9fe2b},{0x0000ac8047450e35,0x000f366cd5225391,0x00027062fa6c9f4b,0x000f7bd93fa9d473,0x0000df0d238b6d83}}, + {{0x00004675e9e96a79,0x0009c366d91cd2c0,0x000ada5d7afaf95d,0x000d7678340a2bee,0x000031e43a7d6f3a},{0x000939ae632995b9,0x000f1ff840e446e0,0x000fe0f6e10f4eb2,0x0000641d99f9f258,0x000059714d33363a}}, + {{0x000dac0c56b9c972,0x00003146fc627e20,0x000583ab2dfb01e8,0x000b8d1d5a0569bc,0x0000e9531ed3477b},{0x000c960233998e0f,0x00024b5da590d5fc,0x000311eb725dffaa,0x0003166d0a806780,0x00002e32acfedeb4}}, + {{0x000a823a4160c545,0x000f51222248acc7,0x000eb7e2a9255142,0x00038a9281f6ec0e,0x0000f63c9cc1c2fa},{0x000efe9ed3f2c904,0x0002604e1900a791,0x000aaaefc6a23190,0x00095a14acde09d4,0x00007b2680ab6b58}}, + }, + { /* digit=9 (1,2,..,8)*(2^{36})*G */ + {{0x000c47940a63bdc1,0x000acbe305406dd9,0x000f4115854d7ebc,0x000bf8c8f8eb7dc7,0x0000eebcc0956bd4},{0x00040deabe2974cf,0x0008c5d99b307781,0x00065a6b4bc494d8,0x0005271e325380c5,0x00006c75941c35b1}}, + {{0x0003c3a8ea6b2c0c,0x0005526c49a861ec,0x00051d8b641c7fc2,0x0006ff017024f3b6,0x00006af90c54fe5b},{0x0006848b57a6af77,0x000fe2cb103d4c8d,0x0002bb9428138d50,0x000a1d94a02bc461,0x00005196aa6193cf}}, + {{0x000f0e96b0521716,0x0006bdb1782269b6,0x0008b6894ea0436c,0x000213b53ae34bf7,0x0000f89e64dfb40d},{0x000020f0e5f7aa5e,0x000a41577884456e,0x000b89101dc1c7bf,0x0005074deb3b5688,0x0000322f52afaf0d}}, + {{0x00039b66645c58c8,0x0007a8dbbe5a1a9a,0x000e8d8b5703704b,0x000b790268187fd5,0x000003b8f3f8d5eb},{0x000b2f3a4a4db66a,0x00074ef2d65a8087,0x000f193430cf1f8c,0x000a6ec4c4969044,0x0000c1c8991dfd8f}}, + {{0x000b8888bb33c9a0,0x00038dd0f82b2148,0x0009657b430acfe7,0x0000dfe19460c34f,0x0000b39b6f7347e5},{0x000bf0221449dce2,0x00055dcc07226779,0x0008802bed13455f,0x0001abec610d21be,0x0000f48269445497}}, + {{0x000dd04b97f56424,0x0001d60b7fe7b6e0,0x000a3239f41aa1f1,0x000c4a61e7d16189,0x000064bed27452ed},{0x00079b5499dfb43f,0x000fd9b506db8cc0,0x000ec79d2cb2ca13,0x0009706e65cd47aa,0x00001dff152d73d6}}, + {{0x000aa7cb935a507d,0x000551ae0ac29416,0x0004582b4da3a965,0x0007915d0279b025,0x00008622b071bf70},{0x00035a70c90666d4,0x00051800d37c7a50,0x000730d2e35953b1,0x0006c1b9213380c3,0x00008f95b8909dcf}}, + {{0x0003fba6baaee763,0x000548571ba18615,0x000c7ccd5282a222,0x0004c589348b22d5,0x0000c343e640ecd7},{0x000d50cd542a5f8c,0x000f1f3e5eea7d82,0x00063b79a4045592,0x0000f8e05521c879,0x000038b6e3d1e7a4}}, + }, + { /* digit=10 (1,2,..,8)*(2^{40})*G */ + {{0x0005910a01f9dd2f,0x0003e65533ef2177,0x00044e3924858ec2,0x0004e12677158c7e,0x0000817fb332f9e2},{0x0009f8be4c5579ed,0x000d207cf88577d2,0x0000ba656c829dbc,0x0002850852224525,0x0000a45a5a5d4127}}, + {{0x0006aebf577abd9d,0x00015452133ffd9d,0x0000ac605ac980fc,0x000e4582a39a8b2f,0x00004eec5aa7265a},{0x000045d5cf88aa51,0x00029cb76ccdac60,0x000c15412957a97b,0x000342cd1af4d36e,0x0000e6170f039470}}, + {{0x000f32ef99cc6085,0x000bc4351facc618,0x000d57573622f8bd,0x00097ededbaf2647,0x0000f2cc80070643},{0x00034049491087f9,0x0004f8c7593cbd4c,0x000c3f77e662b98a,0x000634616ed266c5,0x00002b5777286404}}, + {{0x0008c8bf5c4dddf3,0x000e274c005979da,0x0001c17823f45915,0x00004f9f9c2072b4,0x0000aaa1baf4fa40},{0x0001cb9e28458fbd,0x0002855114df14a7,0x000d8bfa4e43521f,0x000189718d4374a4,0x0000a04910e166e7}}, + {{0x000ca7b94abcb8cf,0x0009802094704963,0x000daf2a7eca4d08,0x000247498a489a5e,0x0000e144eefa19d2},{0x0008666e16797254,0x00009682ad636a41,0x0004f075f107c2c6,0x0002fc5436702702,0x0000b46d254bc836}}, + {{0x0002b2a3e2819447,0x000b6350369c8e10,0x0002b729e058e5fe,0x0006e9bab9271aa6,0x00006605f347cecb},{0x000ced0cf5b648e5,0x000fe5f0e2b379be,0x000be9659e2ffbdd,0x0000d1c063d3de39,0x00006c7602acd385}}, + {{0x000aa397878c7fdc,0x000d8ca2a604b3b7,0x0003bfbdcfdab93e,0x0000ba46e56f6518,0x000008070b9ee90b},{0x000ff6976d6bc06b,0x000412a0e01fabc4,0x0004d387028c757f,0x00060b4fe7c6ee8a,0x00005840dd588099}}, + {{0x00032caa306d36d2,0x0007e7903605c397,0x000e2e161f3c1bd9,0x0005b6484f0843d9,0x0000c3ed7e7103a9},{0x0002e9423a811470,0x000527061160170e,0x000b6096786931d4,0x000d933acc32788c,0x000040d7f8af5425}}, + }, + { /* digit=11 (1,2,..,8)*(2^{44})*G */ + {{0x0001682c10b4b42c,0x00049455922ac1c3,0x000c73c31352d93e,0x0002a2bc4a7b3ef5,0x00009c11c3b203bd},{0x000c092172b4577e,0x000a6f04bd55d319,0x00057bacfaff1310,0x0004d8a9db6d1c08,0x0000b14a2965910e}}, + {{0x000910b95bc8d4b3,0x0009d35ea6c39976,0x000a5950529391f9,0x000ac2a3954259aa,0x0000954edcb4e373},{0x000982812f18224f,0x000a61043e09aee6,0x000f3de4d536e4c0,0x000508c61ced56c9,0x000097bf82337dc0}}, + {{0x000fc9cdf0a602a7,0x0000f90768fccfce,0x000cc86be572242c,0x000fb55cef402d37,0x00007fc53399e03c},{0x00038fa2f84181fd,0x0002e72d1669a8bc,0x000b93e529c4e96f,0x000b28a0e33c536b,0x0000c1aea47311b9}}, + {{0x000e413c023b543e,0x000ff931341ed7a4,0x000c597461477da5,0x0006486a44223272,0x0000e62ade1b4548},{0x00073540e59ce025,0x0006c27e17e44ceb,0x00047c7e6d15d0b9,0x000ca71caee86bf7,0x000062dc3b140088}}, + {{0x000bd5ca4b43dca6,0x000b875b2c69dbcc,0x000837ee1021381f,0x000b45c713aa77a2,0x0000a05614cbf186},{0x0006c5c5b213f9f3,0x00065076db476cb9,0x0001c871dd6ccdd0,0x000210116fde15d6,0x00006771cf226ee0}}, + {{0x0001dde5dfbb6e3a,0x000b56c872b4a606,0x0003e4356c3c10e2,0x000cfdbd1ab2a34a,0x0000f1935615b0dc},{0x000bfd9fd4818329,0x000cd219f2275f33,0x0006091fe9776294,0x0002d94aa0750c8b,0x000091bb35d3e4f7}}, + {{0x0009529ca84f2105,0x000db2adb22b94b8,0x0009189807b37871,0x000fd1763993d8ae,0x000032507bd52b7d},{0x0006d1b1faa44f66,0x0007daae572a2530,0x0007fbd06d00887f,0x000f39d58e02b643,0x0000ff4486cf7424}}, + {{0x000977faf2fd74af,0x00010abaf95894c5,0x0006cdf3274ead88,0x00068f58b7b9bdc0,0x0000d28ef6376f5e},{0x000d37fa9f638299,0x000a9ef052c4b5b7,0x0000893d97515b4d,0x0006fb5c79fe87f7,0x0000baaee7122abf}}, + }, + { /* digit=12 (1,2,..,8)*(2^{48})*G */ + {{0x000e4daa4cf03ebf,0x000a5880a750c0f3,0x00031006e436dc3b,0x000d2500539bacfe,0x0000d4ef32a1291b},{0x000a444f665140ba,0x0002db55ffc69ff6,0x000c27c0e1c0d5f3,0x000e2259446f147b,0x0000e26f578ece85}}, + {{0x000392f2ec04cd39,0x0000d5ac3f16ea83,0x000d7f5ede30754d,0x0009b6872115a461,0x00008418a223a967},{0x000fc5ff44240c84,0x00018cde526cdc7f,0x000b22632c3da39d,0x000c70d988e537fd,0x0000709b1f542582}}, + {{0x0007f2ec2dc98f00,0x00016f4bdaedfbdc,0x000fbe8180bfa37c,0x000bc3e1723fd325,0x0000f58d97310235},{0x00064f972387d213,0x0001ff245104b0d2,0x000b879a945ab636,0x00029e80f5f00daf,0x0000fd8b684b4312}}, + {{0x0008d2c9ae428f00,0x0004f7ea90567e6d,0x0006dddb93095522,0x00081903d513257b,0x0000d1ef01808e5c},{0x000ffa707b56bdd1,0x000c2246b29cb44f,0x000c77ce5b30e21c,0x000d3bd42b540a21,0x0000c8f28344b9aa}}, + {{0x000eec41cb82328d,0x00015f3e30bc27ff,0x0009b1972345ec7f,0x000331e0ee5f0c05,0x000087d0ba66f3c7},{0x000ae59616536a6e,0x000025f99d0b09f8,0x000669480fe3c760,0x000e84b31c5d6cc5,0x0000ee121b26c778}}, + {{0x000c41d1ca8f6911,0x000d2f1d4e353b7b,0x00043304ded57d7b,0x000e262fd062d8a1,0x0000c7373014e0c7},{0x000d825d0c68baec,0x000f5e76be77800a,0x000717e2f324cc7d,0x000e0471a71fe8b3,0x00007ed811a51502}}, + {{0x000a2679405d4a59,0x0005c7b39f8868a8,0x0003eba5f06f98ea,0x0008af06ee27f4fd,0x0000bf77a2388b73},{0x000dcc46466ccfdf,0x000bc72fa30f349b,0x0008f8ca8aee594a,0x000b0c7d194c9d9c,0x0000db282f8561eb}}, + {{0x00067b085c91370d,0x000d35b1cb76219d,0x00009adb7621c58b,0x0008dbc100ec0bf9,0x000035a1c37429d0},{0x000539991832fa6f,0x00095595e93a96a9,0x000a66a28b826cbe,0x000e29cb77526c1e,0x0000acab05a94fb9}}, + }, + { /* digit=13 (1,2,..,8)*(2^{52})*G */ + {{0x00096e6d076e5fb1,0x000e067ceca9754a,0x0005a20acc991594,0x000e01fbf426d2cf,0x0000e8ea47274e03},{0x0003439dd70f73cd,0x000dbdd87880e616,0x000581c546fe37f4,0x0004291151554381,0x000076b608169ba2}}, + {{0x000be5a560218c0f,0x0002245983c38b5e,0x000b7795172b6411,0x000736ce6b14f176,0x0000b95d3a653da9},{0x00090806472dd13c,0x000e1cd9f87dc596,0x000392d3ef194f1a,0x00019f6577c595cd,0x000044201d70daf9}}, + {{0x000a4bf3535ca72f,0x000398eadd70482e,0x000990fe5b370e05,0x000e33fef708de92,0x000095e812192018},{0x0004f0f83a164ec7,0x000582c87912868c,0x00052a18313ff9b5,0x000a4bf0ab1b1be9,0x0000846b0bf28b93}}, + {{0x0003043896a03a07,0x0004b381d531696e,0x000fa5c9a5d318fa,0x0004ca8c757201bf,0x00003e9b1ef0bafa},{0x0000bf7978610e72,0x0004fe18555970de,0x0008447619e614fa,0x00020318b8267dfa,0x00002d551502ded1}}, + {{0x00020ab732d47663,0x000a19f16a66e918,0x0000781056db02b9,0x000d5c5eae97f282,0x000094939d8bd05b},{0x000b2c87b6265c14,0x0008af9287144234,0x000a628484fc8c50,0x0002a90cc54ecfff,0x0000ff62b0d3b8d2}}, + {{0x0000f30fee8c3f0a,0x000134286d2e0f86,0x0001ad978fe6de9b,0x0004d1149e592012,0x0000ed2e1530c0e4},{0x0008c9d2519e343d,0x0007006858e6a61a,0x00069709a27ec803,0x00054a1fbce4c776,0x0000b72f3fb5dbd3}}, + {{0x000c5a4b66e99b1a,0x000a6965f5206989,0x000da89d6d7224d3,0x000bf4eb9640631f,0x0000b4e03e0395b9},{0x000436d53d463c5d,0x000f7bf80b6783d1,0x000e25fb34d34e57,0x000cddcb280e701f,0x000018bf1fde0cd3}}, + {{0x0008e7dd3da6ac7a,0x0007068c094dbb56,0x000e7080f6d2bb87,0x00006862db77d062,0x0000957c36387eb2},{0x000c48a3b41f412c,0x0002b40013a5e585,0x0006e05598676e41,0x000588aab2131174,0x0000a5cfa92af35d}}, + }, + { /* digit=14 (1,2,..,8)*(2^{56})*G */ + {{0x000b63b80c17b3a5,0x0009991991724f36,0x000bce681e7aafdd,0x00021571e5eda799,0x0000608a0582bb4c},{0x0001f37958fc7a0f,0x000ed697e5fb5166,0x000a6efe500b7226,0x0002f1da5737708b,0x0000d330af0a0615}}, + {{0x0001d5f271b2ffcd,0x000ce2caa6650728,0x00046dfd327de7f3,0x00027f03178322fa,0x0000216e3f0e2310},{0x00003bbfc59abb34,0x000d842f48027f5b,0x000bd4fb27522c72,0x0007b690faa40cdb,0x0000abded9d9b492}}, + {{0x0002c6f30c43f226,0x0009e180d738ded3,0x0000ae17641d02b6,0x000b5f0756f1a5bb,0x0000e1022d63ad47},{0x000709b3807f334a,0x000c89dd8fffb620,0x0007ad500b84f625,0x000081d766a281b4,0x0000baefeb53ba7b}}, + {{0x000e8143ae4b352e,0x0008432a3326505c,0x00021f9bfe1140f1,0x000adfcfa38d6927,0x000034810d2f90b4},{0x0000bf0d00ef8992,0x0002ffa7e6ab6665,0x000fd1695563a3a5,0x000ad66ed6cd2d22,0x0000bbe464cbfd77}}, + {{0x0000824eeec7168a,0x000ce52877a21ec5,0x0006bed12ba5446b,0x000931ca2300414a,0x0000eeb5b0f5515a},{0x000322d64e381de0,0x0001cecf83cba5d5,0x000840921d7268be,0x000d46dd7f953814,0x00002b466f4b1410}}, + {{0x000bd6a7d881d987,0x000ec843d9325f3b,0x0000379b23abd56a,0x00044afc5a9bef2d,0x00004a3dd4f7b324},{0x00032b0b1e2614cb,0x000de028f61bc0b8,0x000ceba5ab839425,0x00059b798f49085e,0x00003b2eafe5b888}}, + {{0x00020cf3f9601768,0x0005c47f1f0ced18,0x00031285e9324320,0x0008926fa800cc79,0x000017299d89245e},{0x00019c8dbe5b7b2e,0x00054dc1c0f7d133,0x00005341590ca39e,0x0007e40e3ef92196,0x00008544b679b3ea}}, + {{0x0006e673cd25c857,0x000e1717b82b99b6,0x000ce0284257ae21,0x000cd7d6675922d4,0x0000134e48cf8715},{0x00020fa3844dead5,0x00092c4b5f2b89a4,0x0005e6feb94f1d13,0x0001f55da3de9c1e,0x0000bf9802bd31e3}}, + }, + { /* digit=15 (1,2,..,8)*(2^{60})*G */ + {{0x000a6b62d93fdd41,0x00055796424c49b9,0x0007cd56def02492,0x000194a49f888dfe,0x0000f9869f29b9e5},{0x000f96a765fed464,0x0004e6d179483cb7,0x0002157b537c82e0,0x00066411f666f963,0x00007c51bf707ed6}}, + {{0x00083fd5597eef8d,0x000467c051af62b9,0x000a73b96f3d0664,0x000e60da1e9a998e,0x00006cc734074631},{0x000f6191644596a8,0x00018ca65d698049,0x000dcf446dc950da,0x00041ba3b732b22d,0x0000e67d9922451a}}, + {{0x00021487098d3893,0x00021e0bdefdd4f1,0x0005e1525cf8642f,0x000032bc06995846,0x00002ad5c774a21a},{0x00067f595255747a,0x000b83318969006d,0x000e0b7efa7ee217,0x00097c961cb4d73c,0x00007cde45c8bbbd}}, + {{0x00054660dbc8b1ab,0x000d6088cad38c0b,0x000fb6902a066a7d,0x000dc7ca3471b7e8,0x0000e86a1789d644},{0x00095173a8f8b220,0x000fe11a6fd8de44,0x000a5a2626461987,0x000b8a17c44dbffe,0x000097f4a165ced5}}, + {{0x0008066addda0cdd,0x0005a434573eab0b,0x000b19a5f5541137,0x000d0bdfd11570df,0x00000f19e0267d6a},{0x0006506ae95d526b,0x000bfd8f8a3f0b44,0x00099a5c77b8b8a6,0x0006120809e0e489,0x00003bd4cf0ac49d}}, + {{0x000426d74951de36,0x00033f78fca399da,0x00015de0e419f0bc,0x000aabf82fe9e27d,0x0000b79a1cbd8d35},{0x000f1d0d57258a2f,0x0002d02ca81c66da,0x0006f614c6cdf2df,0x0007519ba84fb8d2,0x00009db0a7aaffb9}}, + {{0x000baef6bff2b941,0x0005465352c4b5c2,0x00012d5ccd22674c,0x000b65e5f4926153,0x00005b47a9bf6203},{0x000db83c64adc010,0x000ea6509ed3561b,0x0002da92e6a83a26,0x0008ccc691137023,0x00007a1159393722}}, + {{0x00013e2c380a9bf2,0x000364d391c2a82a,0x00027444dcb0941f,0x00055fbcabcdd486,0x0000ff69ccdcb479},{0x00052b540c86a1b2,0x000a4f04a057a87a,0x000001893b8ba415,0x000c36a44a876550,0x00000e23c958c1e1}}, + }, + { /* digit=16 (1,2,..,8)*(2^{64})*G */ + {{0x000f5e69622c79cf,0x00060c516a0d2bb2,0x000beb03cb372318,0x0004c5d8b0d5cc16,0x0000e18f5d34d82b},{0x000648f91783b808,0x000bf0bf5a01797f,0x000c6a4c687ad85d,0x00067d3523d20b44,0x00009192002a1f83}}, + {{0x0003271e6c831d51,0x00077b8fac61d9a1,0x0002d09fe01d7f5f,0x000a2a4037d25e06,0x000064507e385362},{0x000ea6041214d255,0x00051c0232f76a55,0x000d1178f014410a,0x00037fc788957c32,0x000085a49edcb98c}}, + {{0x000f3338ee1c0299,0x000f84bb5def9961,0x0003040a98f9248d,0x0003ac5761cb69c8,0x000000beb921e57f},{0x00066c038dfaec8c,0x000d10d2f5378a8d,0x00065894c993b6fc,0x000735690acf4c2c,0x00003b9318cab2f0}}, + {{0x000e0341bd6ee336,0x000102bb4d8bc50a,0x0005fb1615b32bf2,0x0005df468181c0b1,0x000049acb4f25f09},{0x0002c9d27e3d254d,0x000f8cf7003e8666,0x000a275ad19d7996,0x00026d7c2f11a6de,0x0000e62e628f9c93}}, + {{0x000a5ff674823c2b,0x0000030614c09004,0x000172a0cbd7525d,0x0008894a1da98d12,0x000030b72f8253fc},{0x000c72e27f2e7888,0x0000eb81d7ea1d7e,0x00029433ef893a49,0x00053356e38cf289,0x0000353f5c728ab5}}, + {{0x000b6b6e1ca760d8,0x0007f19a8afd30bb,0x0007cf16ffb18b25,0x0004823e08679832,0x0000efcafa52751d},{0x0002beaaf58337ec,0x0006f11471f1ff0f,0x0008cb570ffa8731,0x0005ed83ed76c338,0x00004c92081ca03b}}, + {{0x0000aaee181a2a7a,0x000ff32b88756ddc,0x000230d35049ce8e,0x000b74aa4ed4e865,0x0000261c5ca08124},{0x000d55e8ca475a13,0x000dab96c168af3a,0x000f0644ffaace6a,0x00053a1366563c7b,0x00008a47ff05a4d9}}, + {{0x000f835f9d0aad8a,0x000272e523b8bf60,0x000b800dac7dffa3,0x00068eb888009eb5,0x00005af05fd9e53f},{0x00029d4c6fdd401d,0x0000c312afceb620,0x000a13ffebd0185b,0x0006be5b0c797df2,0x00003ac5e2304089}}, + }, + { /* digit=17 (1,2,..,8)*(2^{68})*G */ + {{0x0000b1d093f471bb,0x0006510f5b343bcf,0x000a023f726a643d,0x000ed90df0f2e400,0x00003c3843aa4ab3},{0x0000bb7742e67add,0x0003e95c70816030,0x00093cfe99544825,0x000be16a6e943e4c,0x0000792d7df86acd}}, + {{0x0009c15b35bafc8c,0x000bc1b7bfe71786,0x000dfb7efd32fb77,0x0008ffa7d4e1deba,0x00001f575658ec09},{0x000e07633366d529,0x000ed66368710884,0x000cc4194ecbd474,0x00068764de5e29f5,0x0000a864508101bc}}, + {{0x0002009d82ccc196,0x0002d9731815e696,0x00067adc8a7f6da2,0x000f360af6df9cbd,0x0000fab708e5c748},{0x0008c7c19580ff2a,0x00056d28b898fc75,0x0009d3500789e54e,0x000af3c0972f1eed,0x00002ac6bc3c26f9}}, + {{0x000ff71880242404,0x000f1c7a3e6fced0,0x0005e468a56ae92f,0x0000f9670e8cbbdd,0x0000020c88ed5eef},{0x000f9526f1333efc,0x0008b74bd70437a2,0x00018a76b379190f,0x0004cd66e5fb3289,0x0000132085c0e1be}}, + {{0x000708697faad111,0x000b805a8271d7e9,0x0004c1dd178bfc81,0x000c149eb4735dfa,0x0000d2f552ec6eb2},{0x00048ef153d6518a,0x0004b40bfcc9a0be,0x000c00e5e2afad26,0x0000f1ea67b27e01,0x0000122effb0b8f9}}, + {{0x000484026aa4c38f,0x0009f8c0aeaa3c09,0x000e1764589f8c76,0x000e5ee3a89f46d9,0x000024e2c82b5288},{0x000f97b2a763d952,0x0004d8a87d83b5f4,0x000662574fedee3c,0x000e6c8cfdb2220d,0x00008ad1e213c69f}}, + {{0x000601630c24de4a,0x00030c2c880e5cac,0x000dc7106144a7e1,0x000995cbf8299dbe,0x00009f1a04b80eea},{0x00081b8f5e3ca421,0x00098fb79e7f8ea9,0x00075c1ae32a9ecf,0x00091361be448bba,0x0000009a3ed31173}}, + {{0x000edc4d24ff34d2,0x0005e5192c4d6840,0x00056736a2a84199,0x00067dc208b04d72,0x000091aa355f1170},{0x0000b62c6dbea762,0x00084bf854279005,0x00058b53b37ca64b,0x000f4906c3de8129,0x000017420c7a1978}}, + }, + { /* digit=18 (1,2,..,8)*(2^{72})*G */ + {{0x0004645d0c999206,0x000f55ed005832ae,0x000aac9fd1427420,0x000def2ca5f5efd3,0x0000d8417da6ba53},{0x00015bfe3a91ab48,0x000ad8bc9001d1f8,0x000bccb573cb03eb,0x000ed588452c8f0b,0x0000190d99ea9851}}, + {{0x000f6928ec987520,0x0001acce548b37b2,0x0002556adc5cbc76,0x000befb4fb38e754,0x0000e6f549a164e2},{0x000cbaf6791e178f,0x000242386ce6c413,0x0004d3f2b753e67f,0x0009e6bf901be1c5,0x0000fea1ab006e2d}}, + {{0x000afcbcddc5b4e3,0x0003f6ad7ab9a2d8,0x000cf2fe2c7a8c98,0x0009cf5c29c00090,0x00000a465e1b5bc1},{0x00039efc04140b7d,0x000aa29e99d043b6,0x000b4b1d9836432d,0x0007aac35ef51263,0x00007f50574f8f0c}}, + {{0x0003b25760e88423,0x0009b7d341d81dcf,0x00083e622c18ffb5,0x0007e4badcddb688,0x0000becf33dd410e},{0x0009a60cc895ade5,0x000e3f6a7a73ae65,0x000e082fa7baab21,0x000f729bc83879a5,0x0000c301b48a24ab}}, + {{0x000f2386acf8a72d,0x000604ac3b819900,0x00028ee94251e622,0x0000cfab91cb8372,0x000019e78eaae4da},{0x000f8063ac90f7e0,0x0006d1eb32912137,0x000dda25c7380214,0x00046a9e3a2f82b2,0x00002b889ab4a768}}, + {{0x000a78b173431ac6,0x000037c593710000,0x000074fceaa51d30,0x000c0a565e1c48c7,0x0000087805015262},{0x00044c6adeb5098c,0x000f59dbcc8ad945,0x0006844fc921b6d7,0x000aaf80a70dc04f,0x00009ea6e278d5f1}}, + {{0x000cf86624af5d13,0x000f4cf02ff6072b,0x0008cde126b78788,0x000985309a47d2ca,0x0000c4b1abc9fa46},{0x00028fb8b07c6947,0x000bf87aa9e5cb6d,0x000144b063bc3aa2,0x0003ccd35c98124f,0x0000562b62f1af58}}, + {{0x000915ee22d46e27,0x00009b665c7322db,0x0000faabc5d86f27,0x00004cf85cf12cc0,0x00003fe2633a632d},{0x0000a1c1ed8106c2,0x0002e3ba317db248,0x000897173dd719f7,0x000cd0e5087dec65,0x0000d47fab598e63}}, + }, + { /* digit=19 (1,2,..,8)*(2^{76})*G */ + {{0x0002aabc59c94f12,0x00095b397acf4ec2,0x00003bfdf9e674d3,0x0004780850426c40,0x0000ca04d6eb4c85},{0x0006908d03fd22f0,0x000814c5f5942bc4,0x000db3a0f5e9bc06,0x0001e62ba9a6168b,0x00006bd8a9096113}}, + {{0x00009d21104a01b4,0x00024745a1251fb6,0x0008668a04b75ff4,0x000fe349e967747a,0x00006e02c0c6d4e7},{0x000084a333d89913,0x00059a82fe122271,0x0005f3a1a369f264,0x000fa053caf9b5b9,0x000022f6b350187d}}, + {{0x000f36cba7ccb6b8,0x000008210395755c,0x00038f8b13d90150,0x0003ea696117ce63,0x0000c5fc7ad16c23},{0x000e0dbb58d94e9b,0x00051a9a40ebf884,0x000f65c5c2b999f2,0x0006d5effe6ec1bc,0x0000b7559295f5ad}}, + {{0x00048ff65b341317,0x0008233aed1d1f71,0x00033bc63a356251,0x0001b22cf566eaff,0x00007f53d3789579},{0x00086fa303582c2a,0x0005fd4cf94cf0dc,0x0003899cb5afac02,0x00035800d319bec8,0x0000afb9e2fd22fb}}, + {{0x00033425fb22b5b2,0x000c99c7ea2ee345,0x000bb698147612e7,0x00068f17a3fd0d94,0x0000828ad54f2117},{0x000657e7e87abddd,0x000bcc3810156e0e,0x00034a697c600213,0x000a648420ea020e,0x0000d279bdaf20cf}}, + {{0x000f31d817bed07d,0x00055ab5c8b06d5c,0x00022f1cd42f3db3,0x0009efa79b1a785a,0x00001ca931baa4c5},{0x000b66d8a740682e,0x000d06a15d7dc85d,0x0003d89a928be329,0x000b486c72f132b0,0x0000478e55d53bd7}}, + {{0x000bb6e6b0a9a305,0x000098b593d070f7,0x0005d69e7bacb02a,0x000d6fe994375751,0x00009b830bfbf6b1},{0x000cb4666d9987a2,0x0001462334c0922f,0x000e103fb50bd914,0x0006b82023df631d,0x0000f49e7f2f57df}}, + {{0x000e23139482dcb9,0x0001ee1adc1696fc,0x00023caa5f3be88a,0x0003ebb3598ebe59,0x0000801aabeabc49},{0x000560a9c8a8fda0,0x00082d4db067df91,0x0005b59ef1377a4c,0x0000329c198dda09,0x00005daf596bd7b8}}, + }, + { /* digit=20 (1,2,..,8)*(2^{80})*G */ + {{0x00064232b86345c6,0x000491cf1c367cab,0x000b12cd89c580bf,0x0005b4a329bc85a9,0x000061507757ea7e},{0x0007fbd8ddb9611c,0x00062b0167dad297,0x00011cbbb1d53bf6,0x000c8188b1604f30,0x00001f4d0fd7d22e}}, + {{0x000eb572d99661c4,0x0008402fcf0a7fd8,0x0008c5d33d449b8f,0x000f0f09842fd078,0x0000da7983de9402},{0x00076f6bdb01f785,0x0001f4c4d194a88b,0x00090292128bdeab,0x0009129fcd2b63fd,0x0000ebc377dfedf6}}, + {{0x0009a1904af269a5,0x000ee2c68af43eec,0x000da69b7072b08d,0x00046bcf55502468,0x0000f80ea4ac443d},{0x000a17081f1c5679,0x000da8d470c56a4f,0x000e1f2684ceb758,0x000ba15f3159abc5,0x0000964c76954003}}, + {{0x000943b9b2fa6dc9,0x000e0b2b4b4b67c8,0x00005ebe4c512574,0x000128fd8c1d10df,0x00006e0e8ebb49c2},{0x000b34a5656dfc03,0x000d6143b3def048,0x0001c1b2d09a22f2,0x000c08d4085ab3aa,0x000060432732650d}}, + {{0x000d702cfbe0903e,0x0006ef88e2f7d902,0x00074a4e05d17b88,0x0005b20e9fdbf33d,0x000010bd286ddbf2},{0x000dd0fd8fb34fd8,0x00013db85c88ce89,0x000b8030d8880391,0x000235bac23a3ce3,0x0000fa45c03bfea3}}, + {{0x000be6868dc83e03,0x0005e0798fa7aaac,0x0004248d23c0ea74,0x00098ae9b41209ee,0x00003cedae7fcc2a},{0x0000a5ccc7b9ebec,0x0005fdc32c04dd3e,0x000def4916cf5e80,0x0009ad4cfa6c35fa,0x0000a98de7d24b36}}, + {{0x000fd321053b8bde,0x00015a36fa6110cc,0x00012bc0351a15e4,0x000cb742b74fb1eb,0x0000cad7c54d4248},{0x000d56251a7ec868,0x0001ca2cc80ad571,0x00056953911caea0,0x0002c80cd7c27fc3,0x00000a95d3fdd32f}}, + {{0x0003858b5c81bd3f,0x000d2a9431dd80e8,0x0009efaae17d3682,0x000f67830eb7c7cc,0x00002f1f3290cd4f},{0x000990a639034a2c,0x000a593b6251d5c7,0x000299c23319bc02,0x000b194511cca1fd,0x00006f501add6b8d}}, + }, + { /* digit=21 (1,2,..,8)*(2^{84})*G */ + {{0x0006beb87707b7a2,0x000c72a87dec0e16,0x000d90f4e489ddf2,0x00017feb5010ded8,0x00009f1c146a514d},{0x0002ae989277487f,0x00076313476cd0e8,0x00052ddea0b6d98f,0x000ff20c6a63d0e6,0x0000d40ea3d516db}}, + {{0x00019c667d0d91d1,0x000e8105ca7d8669,0x0001a93ed4b79dc6,0x00058efbc582967e,0x00007205a3aecabf},{0x000187f1f85aef05,0x00012b160f5dcd7b,0x000f2c42bbc43ffd,0x0004562f5ec697b9,0x000026b5000648eb}}, + {{0x0000d41a77c52336,0x000441d214aeb181,0x000c6340187fcbdb,0x0006e6ac41506af3,0x00003b6fa4818220},{0x0006bdb65cf1fb29,0x0003ce4a84bde96b,0x00083b4cb3bfaea2,0x0008473e742f060d,0x0000fba067aea100}}, + {{0x000b6c2d46254ea3,0x00039b6ec7fae9f1,0x0004d44a4114c60f,0x000f5ba52995f271,0x000066e8cbd34843},{0x00062c42a011d210,0x0000c318129d7161,0x0000f32c7f0a2090,0x000229f63b03909f,0x00009687ec5c5909}}, + {{0x000507db0a04df74,0x000af43753b9371c,0x00099a17c1cd2a88,0x00066679629cab45,0x0000a296edbca1ad},{0x000519b397e39c16,0x000e052af036c326,0x00079fe7dac46a92,0x000efcd5086f0cc7,0x0000bf3f8cd63cc7}}, + {{0x00042e43a80c6fad,0x000b1ef9c053df72,0x00078ed2a6c7dc5b,0x000da22fb8de25b3,0x000063c34563eabb},{0x00066648e3f185ac,0x000a5f4dd6f958ec,0x000f2dde11a9f374,0x00087dd496925a77,0x00007412068d6cf7}}, + {{0x00005e399e662c0f,0x0000a57173e460c5,0x0004e0120bf24c7f,0x000f062621bbbce7,0x0000fbd676e31f74},{0x000bef99ec94a32a,0x00023cb57797ab7b,0x0009ae3d0efbd3a0,0x000900cc160ad35b,0x00000124b141f449}}, + {{0x0007c8bdf49d7f19,0x0007df31711ebec8,0x000f46d03fcfcebc,0x00035281f2da40f1,0x0000aacf4dcfdeba},{0x0004907c5d800621,0x00068e3c2eef12d5,0x000ae7e3f5965a34,0x0000ca494de95bb9,0x0000c88b84c6fe58}}, + }, + { /* digit=22 (1,2,..,8)*(2^{88})*G */ + {{0x000cadd1a806623b,0x000e2b6f79588c00,0x0009a8a99724a1aa,0x000f2088afa52fc6,0x0000f705025b0678},{0x0009b5b7e0f923c4,0x000e2b31803dd6fc,0x00048c34f654baea,0x0000a8a16488e4fa,0x000078ce7289743f}}, + {{0x000c047df2f3ac76,0x000c2b67c4a658ad,0x000864e2a38fcd7a,0x000ec6e4fb3c4763,0x000051531fb65393},{0x0005e4fd59db390b,0x000c9c55e59d92d3,0x0005b30150334900,0x000919016cedca47,0x0000584c78dab3ac}}, + {{0x0000d9a50b845667,0x000a70683a7337bd,0x00042f134d3dc726,0x0004c501f1e3416d,0x000077d800d0f3e2},{0x0002ade2f283b9d7,0x000aa506fe28ef7e,0x00054084698d5e5f,0x000e17b633cc5ef1,0x000066c31b2862f2}}, + {{0x000fc8a61448916d,0x000c74f3a29467a3,0x0009855614595002,0x0005455f2de81e94,0x0000c7e3b2cd575d},{0x0001458e271cf38c,0x0006c8f06d0de9fa,0x00049a303fe35dec,0x000c8fbd5bbc11cf,0x0000091b6978a5de}}, + {{0x000d37663e141628,0x000a0c7dacddb7dc,0x0003b0e7766eefa0,0x000f1de3392ed500,0x00000be7c32df13f},{0x000ffd3cfb24917d,0x00007c5f365b7cc3,0x000aa79e7c6214d8,0x0002056c1294e3a2,0x0000f07f35bd7c03}}, + {{0x000ef267775da03f,0x00054e3abb830d13,0x00021d39eaf8a67c,0x000aae69d2ad2353,0x0000cecdaaf36b90},{0x00073f3903ddce42,0x000b5039b0d7d86e,0x000c943f361c4d2a,0x000faa09580f5af2,0x0000f9ac892638f0}}, + {{0x0007e7ed58b7eafb,0x000a211399f9cf33,0x000de654563b6e7c,0x000addff178e7a48,0x0000fb88753b9155},{0x000f8b801c68011b,0x000ffd43c8aae72e,0x0004bce716798b71,0x000c168527878be9,0x00007ce2d9d5e353}}, + {{0x00072002d0deafc1,0x0007568e039c2560,0x000b74f7fa8c3e04,0x000fa452b5f26fb8,0x0000d5c673e4de9a},{0x00094308345d1eb9,0x000e937e84fb7e3e,0x0000233f0b08ef7d,0x0005f8881b401d8a,0x0000861e80d65e10}}, + }, + { /* digit=23 (1,2,..,8)*(2^{92})*G */ + {{0x00076ddafe513028,0x000b231be24319a2,0x000bb927cde9a7fd,0x00047f98503f7d28,0x0000bacef2354247},{0x000e4d52a90363e2,0x00072961d7a64eb7,0x000900d06b997d69,0x000d4f0d5e436088,0x0000deb49837ce80}}, + {{0x000bb2d0d12e52d0,0x0002d9d615faa8f2,0x00006ad2841c4cc3,0x0005178828fa1eb4,0x0000037e5443ad4c},{0x00018eaef0ca0cd3,0x00057646caa6d2f2,0x000e0158862a3d51,0x0004399628eb879d,0x0000cd6553865dd0}}, + {{0x0000a94af9ea7cde,0x00069d83d30a2c5c,0x000e36e136e23569,0x0009d6c9b0857f77,0x0000ee3ba9363a28},{0x00091c4d690c1482,0x000da0bdaa6ec1a9,0x0002efb68a84b025,0x0004cc597ba9fe49,0x00009c2e3b6a4baf}}, + {{0x000f0606c39967da,0x0008ecee978a1d35,0x00031bb32640de50,0x0003dde62e3a68b3,0x0000e08b646ad0ee},{0x0000f5f9d3887578,0x00046276f9326f11,0x0004b608425a9f9f,0x00069fb512f521c1,0x0000178ec5cbaa24}}, + {{0x00019c380abd55aa,0x000a2e43d34a2e3f,0x000dc29f25566282,0x0003bdcc7ee3759c,0x00006455153981a4},{0x000fab2b4f7c81ad,0x00074aade744b1f6,0x000ff14cda1c443c,0x000bd5111a7d222a,0x0000a28194e30835}}, + {{0x000876812dab14b9,0x0007f78a4d32282f,0x000c4f8b89ba0bce,0x00094fe50e36e029,0x00004692f67fcd8c},{0x000d3eea24df7225,0x000ee8ed23a17f08,0x00006374aae9a53a,0x0006455e06d7b448,0x00001b31dbe9cf50}}, + {{0x000a821ab5d22890,0x0001522b99a72cb0,0x00077cdf65604ed9,0x000cad53e06de6e6,0x000000279f3d1814},{0x000490a31fb979cc,0x000d92b7cb0b5a2b,0x000c1473e470c4c5,0x00054393aa7fb121,0x0000cf1f5b79004d}}, + {{0x0007d51e7e42a51e,0x000f444c5f95cd80,0x0000ecd677766dc5,0x00040656dda8c8af,0x00007a567e594477},{0x0001a1b8848c495f,0x000ff6ac2ccbcda6,0x000f23870b5b7597,0x000568bd43bc1923,0x0000956dd9ebf318}}, + }, + { /* digit=24 (1,2,..,8)*(2^{96})*G */ + {{0x000589fb928c5f43,0x000ab31b0e63d348,0x0002245b5418c388,0x0008872a4f460057,0x0000c3c71e7aa249},{0x000bb0696dee0485,0x00070cc6583553ae,0x0009f6a5e077bc6c,0x000ffc1520879094,0x0000bccee4609921}}, + {{0x0005aaf668ac79d7,0x000433ea2dee7a63,0x000c3da02c1bc912,0x00031cfaddcde2f3,0x00001022732669a2},{0x000b89dea0558e1e,0x000e6a9e5aa049a6,0x000d25865fe4287c,0x00021cec3e74083a,0x00001aa8b5c32deb}}, + {{0x00040e7bf97f20a1,0x0006d0399375235e,0x0006d582fd773a7e,0x0002e1f677614c84,0x00001999db61efd3},{0x0003489b6ccfa05d,0x000a931ee4c47744,0x0009f7bbf5dfe5c6,0x000fd75266f03847,0x00006555c655bd0c}}, + {{0x000be00843e7b7e6,0x000b723fe67ba665,0x000037ef26f02671,0x0004513139145076,0x00001e80333c65b0},{0x00067e7cf2b69e56,0x000e805d53ff04d3,0x0003276aa2047eed,0x000006ba0bf7ccdc,0x00008a2d8826cd00}}, + {{0x0003d241c54ab435,0x0009d27cc2338fb4,0x00094d101f8dfc4b,0x00065cb7689ff6fa,0x000074c538fab3e8},{0x00070471a2ea65d5,0x000e47dde9b62a31,0x000489a9bf481367,0x000580a7d6a06e98,0x0000679d2079dd65}}, + {{0x0009f3c5562d73af,0x000f9ae3113655ef,0x000a5dc46c2ad1e7,0x0006d3bbefa2c6e2,0x0000ec4fc810c34a},{0x0006b238915ff4c1,0x000af22a42581cb1,0x00020244d851d41a,0x0003999a8e1de0bc,0x0000d237a6934d96}}, + {{0x000987f8338f6cc0,0x00059e455950cc3a,0x000bad2e7384a6a9,0x0003fad84c71d665,0x000016cfe8130120},{0x000a6c81f092f72a,0x000fd727a79a6a7a,0x00073800ea330d7c,0x0001f541056e5d01,0x00003e66db6185a5}}, + {{0x0009356a79debb8e,0x00065b7b0dc85957,0x000825e834b42de6,0x00021f4a727de460,0x0000c18079e2bfdc},{0x0003b0bfe5e20c23,0x0000045f5f9a0529,0x00087fe98313de54,0x000411dfc1a8b0f8,0x00004e039a515ca2}}, + }, + { /* digit=25 (1,2,..,8)*(2^{100})*G */ + {{0x0009d20f961c2e82,0x000b85dc8de610b8,0x000ade101437f35a,0x0007eebc5e8c515f,0x00002509d1321032},{0x000842e3dac8ba0c,0x000fd66098583ce0,0x00048bafc3fcb163,0x00076414aa2eedb9,0x000007db9f48bc83}}, + {{0x000595c4b3f8dbd9,0x000359d74cd06ff8,0x000f270e29f8a825,0x00087d12b9c17c7d,0x0000b2e80e811f87},{0x000db27ce43a86c1,0x0003fca990f62ccd,0x0002ec59bf016957,0x0000628f5d9bed21,0x000027d55a4ba6a5}}, + {{0x000d03f718913dd5,0x000c2befc15aa1ee,0x0007c847102cc8f2,0x0005c8a1240d1254,0x0000fdffe724edc0},{0x000f68ea1cc2db7d,0x0003e6c4a02c4997,0x000d509587b544b9,0x0005f5faff725083,0x00002c702007fc4c}}, + {{0x000d112927153319,0x0003f1d26ee83821,0x0005188c3a678749,0x000f5953dcf17dcc,0x00006ee6a46384a1},{0x000b8bdb94fd5a59,0x0002b7750491a746,0x00019255042413ef,0x00049dcab687fa76,0x0000d3bdf3fd8d81}}, + {{0x00073eb5de6f16ea,0x000838af7f91d0f2,0x000180eba14a2d69,0x000c07d543b61b0f,0x0000a48e18374f25},{0x0002413c252f7b56,0x00055f77b4e4d538,0x000f5bfedb51fa0d,0x000d51bb3b7ce66b,0x0000bd4cf4850d41}}, + {{0x0005fe87134cd0da,0x000a17e3cc8ac855,0x00078703ee282882,0x00067052f8d725f5,0x000030f3d852516a},{0x000242d3ae1ef785,0x0007fa96ab6b01c8,0x0008fa96637638d6,0x0008ee49b69a02a8,0x0000dfec375d87b0}}, + {{0x000f921d710cc9ab,0x0008a269be47be0e,0x0007db96e1305cfa,0x000a2bce5323b7dd,0x0000b7178cc6492d},{0x0006c2921097b191,0x000d0fc3c4b880ea,0x000df1d178576177,0x0003f6e393f6f914,0x0000ac9a7fe9fa67}}, + {{0x000bc95725aee3d5,0x0002ea553fb2b560,0x0007423c28530356,0x00000bbd96ce141e,0x00009b1c7fa39ddb},{0x0007326ca7661923,0x00044911220fd06a,0x000b99589f2eb8ac,0x000d2c8a9716e3a3,0x000016f0b4225082}}, + }, + { /* digit=26 (1,2,..,8)*(2^{104})*G */ + {{0x000a527e6b978cee,0x000db82cf7d62d27,0x000ca96802e48946,0x000e9519e1f36e29,0x00007265e87b84fa},{0x000802786e991660,0x0003097581a85e38,0x000f541313771b92,0x00067126e1f1a520,0x0000d10aa19b0e7d}}, + {{0x0006f0af00724aea,0x0002a9e36f09ab05,0x0009f3200c5b7d67,0x000971f9803f163c,0x0000abeb83ab4494},{0x000dc4a79b89e736,0x00099ec70880e0de,0x000ed078e9a3f10d,0x000e77ebd7332a66,0x0000efcf0956aef4}}, + {{0x000be864b03d7cc0,0x000d4d1f4c6b6ec2,0x0006eccb68ebb96b,0x00094bfbf526b082,0x0000a5858dadf7d4},{0x00072e6a44328071,0x0001da542099b646,0x0009a026229e5471,0x000dbab9c2770b28,0x000007628b15e3f6}}, + {{0x000dce3782cc4229,0x00054d0c55c44969,0x000687ed744ff73a,0x00095ca63a6a9635,0x00006189f2556fe1},{0x0005e2f1f92bf8e0,0x000c16ab2b91f71a,0x0002b5bd2443b5ec,0x000ec35ade448982,0x0000c8788b8a39ca}}, + {{0x0004a1f390200e56,0x0008cc847c643671,0x000f310d5b30aabe,0x00025ae2c6a496b9,0x00007a3574996340},{0x000eafc1730bb4b0,0x000e3633ef1f5404,0x000a23f7ac20dc45,0x000363451d323b30,0x0000fabcf3b74204}}, + {{0x000392b0b18c6d03,0x00098e7abded5516,0x00086524935ee7e4,0x000a3d6a903d358b,0x00006875db035981},{0x0006fc5d51954294,0x0000e980b75c904f,0x000db1fb9c1ad202,0x000ea6532c305b0a,0x00001aa27478cb82}}, + {{0x00014710f8720b12,0x0004851b3e59b897,0x0004cfec90d17238,0x00000dc591361395,0x0000dc88b86a7ad2},{0x0001febd7319c15d,0x000eeced6c446c96,0x00060a4e95f5a70d,0x000a79f37e0aa7fd,0x0000ca962020a7e2}}, + {{0x0008f862ec09473d,0x0000ac9d64cc78c4,0x000b52744f8f7011,0x00038e5ab6c50621,0x000008c758760cd6},{0x0006d3b2c6ac19b7,0x0007bede1603c166,0x000ef5c6e18a250d,0x0000ffdfc19a80e0,0x0000dc276b838e08}}, + }, + { /* digit=27 (1,2,..,8)*(2^{108})*G */ + {{0x00011cac13161f9c,0x0001bbc453cadd69,0x00072aef15e577c3,0x0008c37af203900a,0x00000e41db5e3490},{0x000b87d44a487f26,0x000dc42ec965469f,0x00012e582d33e4d5,0x0001872850e9f769,0x000038b03659451d}}, + {{0x000da8e4f1aff818,0x000caa2d551ee106,0x000804d524b4b06e,0x00008fc8d87a8f1d,0x0000ef431fbeb97c},{0x0000c0d9b00298e4,0x00033f0cf9a4e718,0x000fbb34f16943d4,0x000ca08c0ff50200,0x0000745ea7a228cd}}, + {{0x000f1e0fa64b9ba1,0x000a4ff1d7aadabe,0x000fe6d896d5ce68,0x00034116f65d3825,0x00007bcae08c6246},{0x000ac54f612c30f9,0x000ceba33db72cdc,0x00092fb725511bb6,0x000d9573c017cbc0,0x0000a29dfebf2fc8}}, + {{0x0000903c4cbe46bc,0x000c0eb1bf4581ca,0x00060c177519da3e,0x0005066ace635e18,0x00005345445cd5a7},{0x0007197ca469b8c0,0x000858948f214029,0x0004225140fdccb5,0x00087719031b49c1,0x0000a2053b8e6576}}, + {{0x0002dda543915e7e,0x0005f5d4b07e2b19,0x000afac0fcdde015,0x000a72d14d144c4c,0x000034e73127607f},{0x0002db773ecd3720,0x0000fa8cf03dce66,0x0004b74758be1c61,0x0007668c48b5ea96,0x00009b6250934072}}, + {{0x000d7803581ab555,0x000eab4820357c72,0x000e419c5554fe07,0x000fef5a1992039a,0x0000d9875b456d0a},{0x0000ee0ea7058063,0x0001b4dc77ad4d4a,0x000858f1411d534f,0x000f0bd5ed49a799,0x000050a8f7fd0e17}}, + {{0x0003cab91b752141,0x000a5e89c92b2350,0x000e676eda85d689,0x0001c8f31a73bbd0,0x00003841c3e8d0f4},{0x000cd61ad4ee96e2,0x000a42f1db3574b2,0x000537e4eb45e77e,0x000813c06485b018,0x00002be90410b3b1}}, + {{0x000add5f69c02408,0x000d61cf3480d4af,0x0006c1ad6fb043be,0x00005529cf0d8edc,0x0000e6b784db99a8},{0x0005a454130bbd68,0x0002fad96f8acb1a,0x0004c17786cf7d98,0x0001fdbf4bddd49d,0x00002c7d15e91516}}, + }, + { /* digit=28 (1,2,..,8)*(2^{112})*G */ + {{0x0003c0440fd7f2f9,0x000bdbfc360e25a8,0x00048399d668b4f6,0x0002439f4e642519,0x000089f1fa0b9870},{0x00097b3d282ee427,0x0008cf1d720281f6,0x000e67baa329978c,0x0002104e72205910,0x000061a43a34e6ac}}, + {{0x00071279a45648bd,0x000de9afb3ff9ed0,0x000b10e3d7b7d810,0x00004e74928c2e0a,0x0000f66bf52858f1},{0x000e9fccc7111eb0,0x0004f6601fbe8556,0x0007f13a84fd868d,0x0003acc0b3ee394e,0x000011bb82692f56}}, + {{0x000f57841f3144bf,0x0001f9314092ebb4,0x000cf3369fecda45,0x0008e3164f17256b,0x000033b176e2d462},{0x000333e4efa6df27,0x0001d05e708d8553,0x0001d0bddee0a802,0x000f8c7d5856aae7,0x0000603c90846016}}, + {{0x0007512e2eb6a7c2,0x000b8dedda492f81,0x000a60d843d6851f,0x000eb594cc11a3ae,0x0000e9fde037b87d},{0x000cf109a400a483,0x000c6f3177bf8b6c,0x00093bd6f59a5d1e,0x0006c915d232ea4b,0x000066d0031672f4}}, + {{0x00008a54ec2b1382,0x000f888a1f258972,0x000f812b7fecbc4a,0x000e06b5c7ac6961,0x0000e7ee6a3486dd},{0x0003eeb9a9acb8ed,0x000811978660710a,0x00067f8f391f11c9,0x000b760a50cf70a9,0x0000a64a54a69740}}, + {{0x000869d0d5ac68e0,0x0008233169bca968,0x0003a53cda259d70,0x0005a1a9404d286d,0x000088e4951616f4},{0x00052a733ab82011,0x00026e5d0150d651,0x00030ee1a18ef179,0x0006e1c49a92e250,0x0000660861970a58}}, + {{0x0001843074252dfe,0x0003f588434a920a,0x0002c09cd3518f04,0x0005fc19ac0af8e9,0x000065ea1ee67b6e},{0x0000581fc169a790,0x00004aa5447ab801,0x0005b7021d9fcb63,0x000998a9ee5e5a32,0x000086ce09c0bdfe}}, + {{0x000b0da855c77138,0x0004d7080eb24a90,0x000d3d8064a79971,0x000e895b87bedfb4,0x0000ff2e824b81fa},{0x000939b48c4a16ef,0x00015dbde314c709,0x000bd2f933b9e136,0x0004c0d9ecdbf1fb,0x000042e460791f67}}, + }, + { /* digit=29 (1,2,..,8)*(2^{116})*G */ + {{0x000c8ac9b3f30faa,0x000821f711b0eb9e,0x00078b7fefa0a379,0x000f7e8dbb905f2c,0x0000d9b3674355ae},{0x000e568622b92879,0x0005fb6b40a24474,0x0001518d75018f42,0x00065c23f60121fc,0x0000b6c0f8efac61}}, + {{0x0002b23a10647092,0x000b4c64a8a3d626,0x000fc1c509c1f5da,0x0008752338469c4c,0x0000592d71f92d24},{0x000e65e5e66ab21c,0x0007ab4a63d1a4c4,0x000a2eb259587d83,0x000fd941c454e7fe,0x00001e808c047aac}}, + {{0x0003840fe88767bf,0x000b8176add85450,0x00066f408b7e5e73,0x000da5e771b71cb6,0x0000d35c39b650e0},{0x000174cced9e5822,0x000b8d51ec699775,0x0008df0a9a391539,0x00071f40cae243fb,0x0000e2156e1d8e4a}}, + {{0x0003c2f435522de5,0x00099b6cbc613e57,0x0004d5bbb1e76fb3,0x00050f9d533131d5,0x0000500dd4c4695d},{0x0006451f5801b985,0x0009f93a4a375fa7,0x000c41eea66a4aeb,0x000b4eccf5f06787,0x0000be0cc26ec73c}}, + {{0x00000b6c75700e6b,0x000408495fe1347f,0x00087f18e3c6eed8,0x00078726bab0f6c3,0x00006a00813e85d4},{0x0004f3ad785966ec,0x0000da6a78629992,0x000c99e4f0adef59,0x0001d2deb85e5a06,0x00008ebb465036e4}}, + {{0x0003d92b271a34e2,0x00021fc6c68d6876,0x000cbc1727a04409,0x0009c8c7a3e732c3,0x00006da92ca67f54},{0x000e2c06175bb538,0x0004abadd44867cb,0x000e3a947ba1a418,0x000b74f85a71b4a9,0x0000e7099fa89017}}, + {{0x0003b5fb75d8727b,0x000f2282bcffbc46,0x0006a334e0047ac4,0x000c33369708817a,0x000053e4b18d8ff4},{0x0005f5e93d863d2a,0x000bf53e7fe08c4a,0x00044f8de0daeb8c,0x000aaf25cb4d8bfd,0x000075bed4b944bf}}, + {{0x000274dcf1b355e8,0x000d5ef694db7e04,0x0007778800d4cda3,0x00062f4e1edca878,0x000092c279274250},{0x000be2b2abb3de8a,0x000b9b86def6d1a2,0x0000bff865bb4348,0x0002de558d25b167,0x0000a14c44f5e051}}, + }, + { /* digit=30 (1,2,..,8)*(2^{120})*G */ + {{0x000da4a7963ed790,0x000cc9dd111f8ec7,0x000d3a203fc9ff36,0x000619a51bcc33f8,0x0000c3316282fb7e},{0x00058ffec2ca8e3f,0x000cbe30bb1151be,0x00071d53238e4f7c,0x000c05f7854febac,0x00007acf3890bf9e}}, + {{0x00066adbf6bb7535,0x000ff9d29c4120ba,0x000f1a65fd1f4042,0x0009b85519f94391,0x000007d911a8b098},{0x0008234499aed7de,0x000763c34bfca38e,0x000fd1be8863128c,0x00048439ce0f2755,0x000013608ea8ba39}}, + {{0x000b6304a752fd49,0x0007c13516e19e45,0x00076a61d0ec2826,0x000e9d5f656e2e84,0x00006c970cc1cc09},{0x000935381bb3523e,0x00069363ab7e433f,0x0000053ef767b2e6,0x0002d839f1adea47,0x0000b39a71be38a8}}, + {{0x000cb10d572b5962,0x000ed9d918e49366,0x000645a02c2b89f6,0x000f43ab965035ef,0x00005fc3fbf56b43},{0x0007c7032b9ad449,0x0000eb7a242da112,0x000d3f646f3cae05,0x0002e606b16a4d3d,0x000066b08ba950ea}}, + {{0x000fa003d16724c1,0x000aa7d5846426f7,0x0008a4dd404bee11,0x0004afe48e7d09e8,0x00004e388ffd7c0c},{0x000d40da8e2b1cbb,0x000a64f17fd95965,0x0000d88abe4cfada,0x00066ec6a49e0e0a,0x000085fa4175ed03}}, + {{0x00089bc84468e031,0x0005ab4a595939dd,0x00084fd839d2cf16,0x000cd45120355647,0x0000a31eb877381b},{0x00012d8643774d44,0x000965d85a9184d6,0x000b3e932a3180a6,0x0003d448e562563b,0x0000bf1cf2a46781}}, + {{0x000d03d7d3318572,0x00049a354530bb24,0x00077e0492176e2e,0x000bd9f63bde3ef7,0x0000086e3e2a72d9},{0x000280da9fc53e22,0x0002d9e43b6a782b,0x0006ea07cbe66e70,0x000c1cbd9216db30,0x0000dfa49ad0403b}}, + {{0x00048234a455944a,0x0003c913e538cd75,0x000c398678aeeffb,0x000c8cbffa7001f6,0x00008153cadc4269},{0x000cbd665cbd5dcd,0x000d02d3d5b40458,0x0005bdd2db3441fc,0x00002b28bb0ffeeb,0x000086864413478b}}, + }, + { /* digit=31 (1,2,..,8)*(2^{124})*G */ + {{0x000d41f01735a335,0x00073f833f6746cb,0x00054167a0b39250,0x0003a7ba921e46f6,0x00006c09c11aae95},{0x000d59e95785f38c,0x000b834426ffb589,0x00057781d8acae7b,0x000a30f7f8055943,0x0000f1dc76c3a6d4}}, + {{0x00097ba641eb7f97,0x00074a9b733aa5fc,0x000b26df4bf350af,0x0002ceba5dece47c,0x0000cc1ae7e7d3b7},{0x0007fab43b1d99a8,0x000ad0409c110608,0x0004beb49cf2a615,0x00058d94656ea2c0,0x0000cf90618174d6}}, + {{0x000a41e9fa25ab62,0x000af5c775d6ece0,0x000bde7efd93c9a3,0x000b19a8169d2e3b,0x000085cb8fc6f6ee},{0x0005f7b3cfc01d0c,0x00027be909a1f0b1,0x0001f4efb7604d75,0x00015af69150a845,0x00001a9458cd6746}}, + {{0x000b433b59bb0813,0x0000341d4c5105f3,0x000e323c820b4e81,0x000deaab01ae80f8,0x0000ba1d2bfa0603},{0x00002fefd81e661f,0x0004eb693e856387,0x000273b996572680,0x000f613ecf7b5925,0x00002889ae807f47}}, + {{0x00070814d5389736,0x00073141339609ae,0x000d572fd5e8361d,0x000f5579c2b627de,0x0000b93bcfa4fc39},{0x000b4a9a9b91c818,0x000435717398d14c,0x0004b4b95c2dbae8,0x000ba7a759d37d25,0x0000ea36d4f85bce}}, + {{0x000489527486828e,0x000dc5c97290293f,0x000bdeff5a4b77f2,0x0000edddabf04a19,0x0000608db4f55ad8},{0x0004f5d47694d29d,0x000e1c32a5923d75,0x000ba8eaeefa4a0d,0x000473bfcc7bc949,0x0000b5fd2de9fe6f}}, + {{0x000e65bcd4111964,0x000a1f7d13fb27da,0x0002be0bd8ecb0f5,0x000e2f2c5c05b30d,0x0000ff05e9764421},{0x000cdc5908086e16,0x00094b8630d99eaf,0x0005e9509364c388,0x00080bcf1164be81,0x00001478d33e9527}}, + {{0x000e89f3d33e658c,0x000bb5d0fefb0c3a,0x000a984dcf89f8ee,0x000d20ca4d48fb56,0x0000b9304c4e5ff7},{0x0003083bb07dcf66,0x00023c86363d14cb,0x0000b4a396cd193c,0x000063a218981752,0x00009e66befec3c7}}, + }, + { /* digit=32 (1,2,..,8)*(2^{128})*G */ + {{0x0001d587d483a8ac,0x0001644bfe209256,0x00032e076092df00,0x0008fc46391c19ac,0x0000ca7c69ca0159},{0x000a11fe134033ed,0x00086a2a37173eaf,0x000759658d52a842,0x00086c73e2384800,0x0000d0571a330ac5}}, + {{0x000617ff6bb8909f,0x000aafeb8a24a205,0x00019f14a5d36b68,0x0006aa95317ebfed,0x0000c8fde1bb5304},{0x000967ce0fa78f8e,0x000a6c5e3c99ebdb,0x000d5e1c475a7c8b,0x000cdd9dffc64242,0x000020302e081fc1}}, + {{0x000418871e270c0b,0x00029a3b23358342,0x000a6cd6e14299eb,0x000d6b00d9526205,0x000018ce7f6af8dd},{0x000fdb3c22d19812,0x00064814082c1576,0x000fb9e6c451057f,0x000efd528cc914ac,0x000020534397c913}}, + {{0x000e205c1544399e,0x000710441c23fa36,0x000053783e19700e,0x00068fea7b4712eb,0x0000a112e4fbfa4f},{0x000514d89ca1abfd,0x0007c57b36416860,0x000d4097b2b1ef2f,0x000a206500432691,0x0000b3a43e7e1054}}, + {{0x0008a5b251a46688,0x000c41c2cf9d7c1c,0x000a25f0335a2815,0x000478f0c1320886,0x0000b3973c66051a},{0x0004dada9450a7b7,0x0005d32c11d23031,0x00020a289c0afe30,0x000abe1287da6691,0x0000967694826933}}, + {{0x000ce5870a0c41fd,0x000faaaf0dcbc49e,0x000883d62d516f72,0x000eaa5e57de551f,0x0000f8ef2d69da92},{0x000ff5e2f5425d4e,0x000c0dd167d79ae1,0x0002da879bcbf034,0x000d36191039df8a,0x00004f16c971cb78}}, + {{0x000aee99a83de201,0x0000551cbae2f701,0x000578bc0f4313d7,0x0003bf399efc4bc0,0x00004db1c0ede19f},{0x000c5f317b20b8d4,0x000b1b9ae8274c57,0x000500d18221751b,0x000091b816c14d44,0x0000685f584b909c}}, + {{0x0003bc3770684e62,0x000ca6bc4fe3c395,0x000318bfb46f7b63,0x0001ad259ba4a815,0x00008206cce146bd},{0x000c2731bc0908f4,0x000d4ab5afc60db8,0x00049492c0e73d3c,0x00006d107aa02235,0x0000a2cf845790c5}}, + }, + { /* digit=33 (1,2,..,8)*(2^{132})*G */ + {{0x00044ed736b966ec,0x000a5c2e48fb8898,0x0002f73bc5cfb109,0x0001bb4e226882c1,0x000063d7619b2c62},{0x0006a413e95e2beb,0x0005c6fd86e27c75,0x0009ae0bdbf2ca03,0x000edf84da04edac,0x00007492ad45d302}}, + {{0x00024f4d696da3cb,0x0003b1d5a74be506,0x00047c52228c154a,0x0001bfda2e41781c,0x000096ec0545b52c},{0x00001059ae4af1e1,0x000735dbdcc9ae4f,0x0005508ac4dfef91,0x000d754392573dbb,0x000045a30ae8165b}}, + {{0x000beab27c67ef12,0x0000c274d559d96d,0x0002dcf0b984d026,0x000aa0faf4f18c0d,0x0000a4d5a80a0707},{0x000bb192b84c56cb,0x000d3aa7040bf3b5,0x0004ba631dd8d7d4,0x000f4e45d39c479e,0x0000fd24341672c1}}, + {{0x000c13673627c363,0x0003c973112795f7,0x00071c3b07ab4999,0x000f308cde824443,0x000024017b4422c4},{0x0007e6782a430857,0x000be955dbec38e2,0x000c10a45929115d,0x000d83e2f51c0782,0x0000d2b6c6a41083}}, + {{0x000a0300c013b50b,0x000b751ec5d7f652,0x000666363fc769d4,0x000b40fb30c8f561,0x0000e14a50273cdc},{0x000989050f53d764,0x000cbadb114faf4d,0x00087892f0c26b7b,0x00024d2de328fc0b,0x00006f37a0582ce8}}, + {{0x000f9b21b06e4d48,0x000483d4ed2dbc25,0x00081ca00bd2f9e2,0x000314b350eba59c,0x0000879d4fee9d2e},{0x000846aa4551a9b0,0x0002394af267bae5,0x000565e76b699192,0x000e51500aa86cc2,0x00009d486efc3818}}, + {{0x0008b841146d2064,0x0005638e7e0c2780,0x000e4fd2ed5afa66,0x0003f8a3a09bf87c,0x0000a8d6a3930dd9},{0x000281d429091c44,0x000e8e029e646294,0x000f8a25ba1a0598,0x000386591adb60e8,0x00000de777e039ad}}, + {{0x00023eed8fe44b7b,0x0001c40e2c2bf152,0x000587aaa9751a65,0x000b04a65627a220,0x000078ae9a0140de},{0x0008dfbb97c7d47d,0x000a883a8363b49a,0x000920e2f50e570e,0x000074b5c1a0b6cb,0x00004368ec853ac9}}, + }, + { /* digit=34 (1,2,..,8)*(2^{136})*G */ + {{0x000c2a360971c31d,0x000066e846e364f6,0x000c727ad6974fa9,0x000639b6d7f33527,0x000015ca19ee6b97},{0x000b4ef351021aa4,0x000ba52f024e9245,0x00044265bc79a45a,0x000687390e0fa07a,0x000026bc8f7f2c64}}, + {{0x0001a05fe05069b5,0x00097050f15dde91,0x0007fbbecba1b4b7,0x0007e7662a47a76a,0x0000cef9eea25012},{0x000d251439c9ff5b,0x000996e9d529b36f,0x0000927929bcb4b0,0x000321b9855ab130,0x0000e6913e2a37cf}}, + {{0x000a22c213a1381c,0x0005998a9d82abfd,0x000b0c9eaee2416b,0x0002e94875f188cc,0x0000ed487d7becaf},{0x000a1bfe43d8e989,0x000d43afbddb4799,0x0003d3555d553c34,0x000ae422e4b606b8,0x0000a98941400820}}, + {{0x000495775835ddea,0x00038f3dbfb894e2,0x0006a991d3c0ab53,0x0007ac0dfbe9b79f,0x0000a996ed108d94},{0x000a1485e6867237,0x000b861bb9390993,0x0009ecbcb0ad8aab,0x00089ee3b1d6a974,0x00004490d265e84d}}, + {{0x0001f9addb3fd772,0x00068b61430c9ab7,0x0002b03f3bbc1b26,0x000a8d0ae0bdd41d,0x0000a6be72385501},{0x00063accbd8fe7a9,0x0001716c52dd907b,0x000c73a737c1e0a9,0x0002868a6d9bdf44,0x000035c09edac28b}}, + {{0x0002ad31f429e92f,0x0007dab4baef62e3,0x000275ae5ff44e2e,0x000fc47109f5a220,0x0000f78da2e8f242},{0x00089cf1ff54aea4,0x0008ca425bcf4d6c,0x000288b364e70dd4,0x000847d56ea95059,0x0000d5e592ce8a8c}}, + {{0x000e70062ee0a4a2,0x000917ac0b7eff35,0x000e6774850d2a32,0x0004c985f8552225,0x0000ed692c26dd93},{0x000068c99ecd36b0,0x000a048e7073969a,0x000922d3ea339e56,0x00093f20f392d2a2,0x0000a61b1d58dd81}}, + {{0x0006e4617f3e69bc,0x000caec1ed66f181,0x000bbf3b6cc4ad74,0x000551b4d225d906,0x00003684306257aa},{0x000516497c9675a0,0x0000addb2fbc3dd3,0x000f55d795decef3,0x000598fadedb5484,0x000015e8ee776bd7}}, + }, + { /* digit=35 (1,2,..,8)*(2^{140})*G */ + {{0x00020f2beb76812c,0x00013ecfa181e695,0x0000dec76ead7889,0x000698d5d9ea02f8,0x00000cd75bcfa2f0},{0x00017a069c2d2cc5,0x0008f0a1df843618,0x0000f08066c134ef,0x000c931ff1203772,0x00008f19ef39b4e2}}, + {{0x0000aecc1c7112bc,0x00073b6b294cda6a,0x00051854d3181244,0x00049ff216ce9bf7,0x0000f5f14402d398},{0x00056e372344bdbd,0x0005f3af02909e50,0x000f436065b91304,0x0002579f8c7d59ec,0x00001766823e2146}}, + {{0x00027d200e7695ba,0x000fb3189e800ca4,0x0006d1d4e8ef137a,0x0007f003750fe0c1,0x0000dad25c5ac540},{0x000807804fa82f49,0x000994fe616e2c00,0x0008e610d4715daf,0x00004e1739c25f4c,0x0000a1ed59eb55e7}}, + {{0x000c966787f80791,0x000d43a4f0d56f34,0x00077d92507dca1a,0x0006bb24b961e404,0x0000a0d775222852},{0x000bb6d594089b2b,0x0004142864fee422,0x000a2f57f8c8c37e,0x000969659c1be93d,0x0000e98561f48eb8}}, + {{0x0003d36eea5a411b,0x00083c9b809b7ceb,0x000b2ef3bd41c883,0x0005fa4368a41486,0x00005327a94036a4},{0x0009d81a294550be,0x000028a328cc987f,0x000b405a4a382a8d,0x000c01dba0a3bcd2,0x0000ecbc7c687492}}, + {{0x000ee9ea4399d83c,0x00029ade4d559419,0x000d914e5643a410,0x000194f9bdedafa5,0x0000ab6a2f9c77b5},{0x00023fc56d6b71dc,0x000ce1637a55a4a5,0x0003af1fce4bba9d,0x0002a5998eb19a51,0x0000fb6b0a026533}}, + {{0x0005384859f3a770,0x000970cb98fe684f,0x00091d11cbfe5c75,0x00014fb3fd60fbe9,0x000024a2c6896e9f},{0x000a7731d1c175f6,0x0007bf7b59e763bd,0x000880e35c8c898c,0x0002e8c923d4606a,0x00001705d921c944}}, + {{0x000bd17983bec34f,0x000e79390d458714,0x000fba44f409e51f,0x000e5b503e976403,0x0000bb28b50bfeca},{0x000460fc77585d24,0x000439ed5d2c53a7,0x0008896e3f104db9,0x0000d454e5ec4bcd,0x00005c92699d9c5e}}, + }, + { /* digit=36 (1,2,..,8)*(2^{144})*G */ + {{0x0000693a18674889,0x00096e1c63c4962c,0x0008d7b6a10d190a,0x00003a26b50541e8,0x0000e282f1e08996},{0x0001ed4e5703fca0,0x000c04e0117f203b,0x000258ac6a79aec7,0x000248c44245f196,0x0000ff00eb7253c2}}, + {{0x000d0301187bc101,0x0000eb6700a1acd0,0x0003f09695693995,0x000f3e0bae823fd7,0x0000bc494f5d06f0},{0x00015737c0a7b0f1,0x000bfc989fca8cdb,0x000b4882d64acc9e,0x00080e66f970d01d,0x00002a01327ea3d1}}, + {{0x0002887ba15c20fd,0x0007d9c6bdf22da8,0x000f0e54073dbcd2,0x0004f3951efbc432,0x0000469ec570ed01},{0x000deaa2e18dbfd8,0x000c5dab920ec1de,0x0002fe53ea59ef2c,0x000e8af47d0d7e8c,0x000015e430f8b3be}}, + {{0x00062dbbba9d84ad,0x0008b4ece53c2d04,0x000d18184b2003a6,0x0006765b0779d897,0x000067fc9538f5d6},{0x000035f7ff931704,0x00005f2cafe37b68,0x000f6c983617d6e7,0x000df03fd273d1eb,0x0000249da2e138a9}}, + {{0x000807b1c5ae6f72,0x0005a0f296c9005d,0x0009a3033a160965,0x0009f56d44b9094f,0x0000c6b826d12c11},{0x00092c2460ebcfff,0x0002ddfbe7133835,0x000e27e80740bd71,0x0003a6cdc435e063,0x00001a2aeff08bc3}}, + {{0x000bb88baca8d454,0x000d95a2093c22a9,0x000139b7af41216f,0x0005e33b6ebd0bd3,0x0000663daaf350bc},{0x000c3aa8010516da,0x000ab3875c1e3b2d,0x000fedd69c7bf698,0x00045728dbd2ac94,0x0000085149b36be5}}, + {{0x000d63c348e3925d,0x0000c0016e23e9d2,0x000b7a687e98afd3,0x00043c75ccb220c6,0x0000b36999ba7a86},{0x000e9f6ec46bda19,0x0002519cdc0bcb5a,0x000ee0c0acd4376d,0x000092de470f18b3,0x00002848124d0362}}, + {{0x0009d1febf2de9a9,0x0008bc77e6c55202,0x000fb2e7eb995763,0x0009cbd6dc27df9e,0x0000444476cdaaa8},{0x000ba7ced0785f36,0x0004b9e93470afed,0x000906fab0ce1fe7,0x000bf2f043e6a966,0x0000d9c1876fd26a}}, + }, + { /* digit=37 (1,2,..,8)*(2^{148})*G */ + {{0x0008d94f6be7acf2,0x0000341338d6e434,0x0002fc6886610503,0x0000f96ca56f7dd7,0x00002f54a7c972af},{0x0003e19d89ed269e,0x00005bb2ef8279cf,0x000a1736ca68762e,0x000fbb351d575465,0x000061e8deb175a7}}, + {{0x0002aae2b99711c5,0x00031ce51efb3108,0x000412e3130475fc,0x00003c959cb5b2eb,0x0000efeaac806f9f},{0x0004837f98bf8cb0,0x000da39411aa636f,0x0004e03d299b3bd8,0x0006376b77152ecc,0x0000b7bc15a5f18a}}, + {{0x00083eca38629302,0x000f164b71698f58,0x0002173c96261574,0x000a82abb6ba418d,0x0000a7d774e7d73d},{0x000c1dd5576a4dbc,0x000283ff3437f24a,0x0001f86ddde594be,0x000503b65e910b43,0x00004e079387d0c8}}, + {{0x00039d3ee2963f2f,0x000233a626332d5d,0x000281c47c620310,0x000452fa27bc3883,0x00006bc300c4cda7},{0x0004c341a09ee9c3,0x0003ad3e7676b0dc,0x000f36ad1e76c678,0x0007b36a8620e35f,0x00009710f4c3af53}}, + {{0x0000e7c2becbd6ac,0x000da1cf405ff065,0x000cf786bfcc5059,0x0009287fcebac86b,0x000006628092b297},{0x0003a21f33a3f231,0x0006f95fa90d7679,0x000b52ce8c481fe2,0x00088d46dae60eb7,0x0000dd27b34c3095}}, + {{0x0007112386fc20ce,0x0005122369b87ada,0x00088beb81cf895b,0x0008f96663c00e5d,0x0000786fe6f72494},{0x0003cd4c08b1b97b,0x00066986d9bd5f51,0x000845f5fa36c27c,0x000c259b22dcc7e3,0x00003a7a6a264018}}, + {{0x00014104124194f3,0x000f16022caf46b4,0x000f5955fa3d17a1,0x000fa544e6a45dd8,0x000027f7e6432277},{0x0001bcd329f0aee1,0x000ac1241de97bff,0x0000cbc3ebca8120,0x0002c1f37b8547b6,0x000028b36f4746d4}}, + {{0x0004a07f82ebea7c,0x000b5769bb816483,0x00053090a14d150d,0x000e04062d69eb48,0x00003488c7727160},{0x00062b085ef31ed0,0x0002787bbfb55541,0x00004cb77b99391c,0x0007343a0b06ed4d,0x00001334165040ae}}, + }, + { /* digit=38 (1,2,..,8)*(2^{152})*G */ + {{0x00027d09f86b87a6,0x000672f7189e71f4,0x00081c5287eec6df,0x000edd421c643874,0x0000dd3e802a5f6f},{0x0005cba1123d99ff,0x000318f38384a7a9,0x00041aa78a8746d0,0x000ea5919aac3acb,0x00001546626df6d9}}, + {{0x000842e7e3646f13,0x000721aaaca5e9b7,0x0000de2e53b43348,0x0009b4ef6518aa52,0x00004c1f8413e3b0},{0x00027091000d9504,0x000ab656868a5489,0x00024df86b81806a,0x000fc7333d963bd8,0x000027dffecf8b57}}, + {{0x000032dd3467eff0,0x0002e48a105fc8eb,0x00097868c757cfb0,0x0002c3180769d754,0x0000bb784982ad6d},{0x0009004e124bcf40,0x000e6db020b0aa69,0x000f6d64792a4968,0x0003db95d05137e7,0x00009c7931f4b21a}}, + {{0x000de8cdd0926a17,0x000b2a7d298c241d,0x00097897af2c8ee1,0x000fad3dbe3b697b,0x000095a39dc131b2},{0x00088519ea3f0b69,0x00087a75a4d2604d,0x000e93901ec55560,0x0004fb6cc27154d7,0x00006bfc52e1469f}}, + {{0x0007c1ff4bc6220d,0x000d7f796ea6ca1f,0x000927b7292549f1,0x000438045567cdc2,0x0000e37c17362496},{0x000832367b234acc,0x000ba574b6cad39a,0x000411ee7dad7e6f,0x0009c31fb3d58d63,0x0000883b19b2637f}}, + {{0x000d28e06912ab7f,0x0000d9f0e4938f75,0x00097f5a913534b6,0x0006ee5cc58d47d6,0x0000cf8cceb8bae1},{0x000d0bd2f66354b5,0x000c029210466f66,0x000f599a7c02757e,0x00079b57a2af1152,0x0000cd4c1d2a65b6}}, + {{0x000d0b153348f81a,0x000e92af416448df,0x000fef6bc43dd5d3,0x0001354ae4a7e8c0,0x0000c6d13c9cd270},{0x00067cecd88b842b,0x000ed48d98b5ca3e,0x000c09bcc716fe72,0x000f7af2ed01fe5f,0x000018e9887155d6}}, + {{0x0002571fa1c005d3,0x0000a1a7cb1282c7,0x00064bef7798f823,0x0000303b2a08d762,0x000094b95e409f27},{0x000cc832f936b83a,0x000392ff22dfd98c,0x000633bedd944ef9,0x000cfe67a87ab01a,0x00004e149b05af22}}, + }, + { /* digit=39 (1,2,..,8)*(2^{156})*G */ + {{0x0005d70b7657b7a1,0x0007085ca8d9d1ae,0x000cf3b0d35bf0d7,0x0000cc024adf2c77,0x000069d110cf7a09},{0x0003a7564e157769,0x0001d506260e70ff,0x000703a97ab76d5e,0x000435a6439d75ea,0x000076aec3e36360}}, + {{0x0002f99d592d4607,0x0009d3e3fcd3efca,0x0004150ef1418504,0x000f73feadd26c03,0x0000c8303bb7708f},{0x00028bb4209cedef,0x000e4552f1da46cd,0x000d486f19b140bd,0x000e6167872c3f8c,0x0000b4a36e89cdf5}}, + {{0x0007709b7b97f8c5,0x000cdd2e7417ce17,0x00011f9e4c798c2e,0x0006edf1eff42bc7,0x0000d3407d05f1c4},{0x000b2930ca06e395,0x000de1dbe217f2cb,0x000ca08df0ad2e70,0x0003a3b592af2a8c,0x000040a4a94b9f52}}, + {{0x000cf60f96d10e48,0x00095daf176f2c08,0x0004556116c14d5c,0x00025e7fb01ca460,0x00006c4e588656b4},{0x0007fb754d3440e4,0x000851cc9071c4aa,0x00038c48b2b6677b,0x0005b2981fd58874,0x0000cc23558b384c}}, + {{0x000981afc00085d1,0x000d0d5af71013f7,0x0004926c19866a71,0x000ed9fa0e68216e,0x00008c7bcfc44bd1},{0x0005694612f9623f,0x0002b679449f0e1f,0x000d14dea79b08e7,0x0009c8b2286cfc4b,0x00000d346cc223b8}}, + {{0x0001287ee355832e,0x000a8715bde48f8d,0x000250613d9a672a,0x00086c08ac970387,0x0000dae19fd8bb71},{0x0005fdcbcf36d30f,0x0007974db1dbf1fd,0x000ac30ebd07464c,0x000637413ea46588,0x00007cc18fa7cb4d}}, + {{0x0008561eaab750ec,0x000fd959890272db,0x0009861b1b6617ec,0x000313bb875f3432,0x0000725d0e14f52b},{0x000a16eb56377ba8,0x000efa08332aafd7,0x0004f969b00c6f26,0x000e75b6f9b0d8bb,0x0000fe905ed9a6d4}}, + {{0x0009badc329d1f15,0x000798a2ac13e274,0x000f04f6cdc35ac5,0x000fff5624494f6d,0x0000234123bdb8fa},{0x0003d01cf2e4388b,0x0001a41004f7571b,0x00002b4c77fb6c86,0x000ef3131bac67d1,0x00001a55a5e1aed2}}, + }, + { /* digit=40 (1,2,..,8)*(2^{160})*G */ + {{0x00010ee253308b7a,0x0006b0f6d3549cf8,0x000f79be840ba3a8,0x0008682b46f33696,0x0000b318d895599d},{0x000485717b66f888,0x00087e17159bb2cd,0x000da105c3fffe4c,0x000a76272f7dfbeb,0x0000bfbd7894f96f}}, + {{0x0005fd5fa205d0f4,0x0004ee8ee36860ce,0x000c5b16628b839b,0x0003f4e13daf04df,0x00008b3aaf4c153e},{0x0004e879df3f3f34,0x00029941a4e0551e,0x000d33e8877228e4,0x0005911236772cfb,0x00007681b72c03ca}}, + {{0x000b1aad44ef0b79,0x00091a9b898fd522,0x0007938044627d81,0x000438af82fb38a5,0x0000b4a5e385644c},{0x0005a947eeafb773,0x000c02e7cf2c0846,0x000e9dbe8a7ea8fd,0x000b6d78d1c57d96,0x0000586ca615cf4a}}, + {{0x000e349a4b31aebd,0x000f2b0285b94916,0x0007b017d6137900,0x0002b0dab01a0be8,0x00008ac2977211f9},{0x0006974d05362415,0x000fd464a1c6a163,0x0005f2a0e55b98f7,0x000493f71d99e6b6,0x0000a0e9ca0b6129}}, + {{0x0008c8ce34c9e3ed,0x000c916fdd5b8549,0x0007be5fa36b2b51,0x000ae2fad592549c,0x00002d134c65513c},{0x000e5adca237cbc0,0x00077e7e9090b363,0x0008cd9cdf14f0ed,0x0000cb97467fc3ab,0x0000a00ba453f572}}, + {{0x000b8f7229523466,0x000e3a1cfe89d80f,0x000cdf0d11487037,0x0009c00d2b42c026,0x0000172110f51188},{0x0009ff21f71bf171,0x00008ecd850935b3,0x000c9c32bfbbecc4,0x00028143434c1a74,0x00001cad52349f90}}, + {{0x000ae1d3d9866434,0x00034a7b3f85ff0f,0x000c1fa6ac6b4768,0x00077b494acd21cd,0x0000ad41f3fb547f},{0x000c5c3baf25baf6,0x000e8ce8a1c5d92d,0x0003fff5cbcfc843,0x000221dedbca01a9,0x00005c09e40eecf1}}, + {{0x000d9cce837c6d61,0x0003d148f9290579,0x0006232b855bab85,0x0007c64ce7a64ae0,0x000028043d63ffdc},{0x000f181a75f69e00,0x00059796e93b7c7b,0x000172a383b1d5d0,0x0000029a0e1e4709,0x0000b9ee91160db4}}, + }, + { /* digit=41 (1,2,..,8)*(2^{164})*G */ + {{0x00026fc9c4cd4abf,0x000e23f1402b9d0c,0x00026a0bfe9f0668,0x000417f6e573441c,0x000042560b13ff8a},{0x0003ef07f65b14ba,0x00061fd7493cea35,0x000ec7090c603bd2,0x00077a68fd05d4b3,0x00006ce1efdc940f}}, + {{0x000f63cfc83f45b8,0x00011d646ac49d20,0x000b73afaae16770,0x000dca23842c77c0,0x0000cf54b1e93428},{0x0004dcfda1adde56,0x0005a1bc6441f959,0x000957b146ed74f3,0x000a15bba7d38f71,0x000080b43552bdca}}, + {{0x00014e77a0de8421,0x000cc45526f09fd2,0x000c5fd9ac6926fb,0x000a7dc8de8a4f10,0x0000d25068992420},{0x000b6cda791fe0f2,0x0007b7314faa40ec,0x000b3679170d12a8,0x000c08f3e767867e,0x00000e1e221077f7}}, + {{0x0000cd9c328d84f2,0x000ec10296c36eff,0x000191f73e449397,0x000f344da7ee8967,0x00003d52cf283e17},{0x000adab3ad961303,0x000c8f7e455fe908,0x000af39881456bea,0x0006ae1ec3fb53cb,0x00007d83567df6b8}}, + {{0x00097128795b26d4,0x000007d53b618c0e,0x000b80b150a9c145,0x0006c0564c424f46,0x00006ae4b9ab6582},{0x0000d841a1380e4e,0x0005832f815561dd,0x000502e81430573a,0x000171a4f85f48ff,0x000063896020863d}}, + {{0x000f7199dc46d818,0x000722b599ff0f94,0x00073193628eba9f,0x0009137f368a923d,0x0000ee5360c30393},{0x0004f710bd7921fc,0x00093f6e46f2a79c,0x000d25010260474b,0x00092d27c08b5dea,0x0000fab67c859c32}}, + {{0x0004828817b2b97d,0x0002c0f0e0b040d2,0x0002217d0167ff6f,0x00040e422baf02d8,0x00006eb8e36ece4e},{0x0001f86203c5e993,0x0008a31113ec3567,0x000da3f78ff4f368,0x000ea84159e48861,0x0000bb7e93050f1a}}, + {{0x000e3d7a82a07312,0x000248b4ed80940f,0x00057c32545a8fa1,0x00024459f67e6d05,0x0000781e5623c72a},{0x0009422f1dd9d9ed,0x00045027e096ae27,0x0006ab7164488446,0x00002f2fcb1f3e1e,0x0000a08771e4d556}}, + }, + { /* digit=42 (1,2,..,8)*(2^{168})*G */ + {{0x000b1adfbccd5f72,0x000bafb9db3b3818,0x000e49c42a8e58da,0x000a5741f9c3a2de,0x0000e1b4d1992caf},{0x000d2ae779d25bd8,0x00001397e053a1bd,0x000689b00f8d9c66,0x000aeefabee2be5c,0x0000ed75eb0e9aae}}, + {{0x00070ef12df3aecb,0x0000e7a205b9d8b0,0x0003fe5865a61087,0x00049560e6eb8f06,0x000018c288645dc3},{0x000c1f205200dec5,0x000d0053bcc876ae,0x0007bb212c914ca5,0x000c3165e12a7533,0x0000fee6eaee8fb7}}, + {{0x000b625175d3e131,0x000ba79b6828f364,0x0007b65b0a28d9d9,0x000a31a0c9d7a025,0x00003f761efd974a},{0x000cea06f50c8e7a,0x00025dd9669b6210,0x0006ea0e74a30782,0x0007f7cbc88a2ca5,0x0000eefdd32a930a}}, + {{0x0006927bdc72fcef,0x00092b5c4e83d33c,0x0008986accaed0f0,0x000ee5e0fd9f3587,0x00006fc2b4d5332a},{0x000bd4c284a559fb,0x00092f79f9e0f036,0x000e91031f24a068,0x000494df12868661,0x000064b67a214c5a}}, + {{0x00038062d4c1e75b,0x0004591289a8619a,0x000fc2f14e9e6431,0x000a96b32ef796e5,0x0000cf84b53f10ce},{0x000e2d93f2a93799,0x000b1200573274eb,0x0003eaf97fa1c33c,0x000a47520d07b67e,0x000099241c28bfc2}}, + {{0x000e16a8fa9459c5,0x00069533f36d1411,0x00042fe5fb485de4,0x000223d3ae84bb3d,0x0000362e47c092d7},{0x00051ac53cf453e0,0x00072adddd472e03,0x0006d8041bea2700,0x0004e95997d405ee,0x000072103589e10b}}, + {{0x000c45b260b78e4a,0x000cab84444896b8,0x000f6cfa759ac76b,0x000f5fe7d64974d2,0x0000fc1b25688826},{0x00018f2e67924f42,0x00079c2d84634875,0x0006ee2d190516ad,0x000a501c0d1b2b3f,0x0000036290195036}}, + {{0x000578dc4ce14fbe,0x000b08c06d75fc1f,0x00063cd0cc5274b3,0x000f629dd2dcf7bb,0x0000f36db3fef100},{0x000304c0d907ee38,0x0005103df2ce7a06,0x00083934eed34414,0x00075ccabaee3628,0x0000f1816df3580c}}, + }, + { /* digit=43 (1,2,..,8)*(2^{172})*G */ + {{0x00065ca49bf70465,0x0002633070d3aab6,0x000b33d03149eda8,0x000f82c732643672,0x0000dae397b7ff25},{0x0005986e7c2b0613,0x000c759b3efb9983,0x000ccf96a4c52f87,0x000f392308a5b922,0x000053a40c602f11}}, + {{0x00003172599604ef,0x000e04fda79e5acb,0x0009d5feaf05bd45,0x00080866e68b83b3,0x00000b424807d53a},{0x0005296e9538c34c,0x000381ac5ccc2c46,0x000ad873e1d42e72,0x0005408bd7d7dc96,0x00006a74e1c17bc4}}, + {{0x000cfc53b4244564,0x0004fedf0290a8fd,0x0005ee4b6fd35ed5,0x00009974fcae8196,0x0000c1f220ef19f4},{0x000bedf5f9d4ef28,0x000ca3c4cb632f9e,0x0005cd318a6d91a9,0x00094f00ac42a1ad,0x0000689a17da238d}}, + {{0x000fd495f78c275c,0x00068cdb30cfb3a6,0x0000fcc91ed14bb9,0x000a17ddf6d09b8c,0x0000645d0ce04a7b},{0x000dc229b0415b16,0x000009f275264daf,0x0005e7bb59b2b9b3,0x000525c2280c2b74,0x00002b3172744708}}, + {{0x000e98378c350e3c,0x000c25a59b0ff628,0x0002b1fc99481f98,0x00035431d25ec81a,0x000068a37387c1fe},{0x000ebf428f4945e2,0x0009bd6f48485983,0x000f243e60db09cd,0x0002efd4687f2c5b,0x0000e57b62bb98a6}}, + {{0x000fa5e9a55e99f0,0x000bc7ad0d350a3a,0x000860b4358a0879,0x0008b504684f6ebf,0x00009bc6954fb2b7},{0x000e47f1dc72cd57,0x000587c1cdd69b63,0x000020a0729886ae,0x0005a9f2a64081a3,0x0000092024e42c9f}}, + {{0x0008186df8037536,0x000287fb7f01d83d,0x000a6db487d36c22,0x000fb5d0f5f1f860,0x0000704dccf6c287},{0x0004aaea176b654e,0x0000fa01720468f0,0x0000fb9b626920a1,0x0001293ca4247726,0x0000473f7f1fa757}}, + {{0x000fc20755e3176b,0x000828452666a58c,0x000016e7bc6ccdba,0x00078f9084bcb6e0,0x0000554faca4c643},{0x00094e142cf0b0d6,0x00046505294dba30,0x0006822fbef1afda,0x0006df474a30ba28,0x0000e6be6e6ae1a8}}, + }, + { /* digit=44 (1,2,..,8)*(2^{176})*G */ + {{0x000b296904664fc3,0x000b53e979f39254,0x000a642320a351cd,0x000cc34fa1efd130,0x0000a26d827b4096},{0x000df088ada01cc1,0x000d534b9db65b69,0x0001656914dd4d71,0x00018a2f335c82e3,0x000058dcd3dda1a0}}, + {{0x00008fc59cde3468,0x000e7124237b64b8,0x0009635d376aaed3,0x0004ff5d8688ebe9,0x0000c1b55d497018},{0x000b143a98f532c8,0x000632ba3585862b,0x000532de58edb3d0,0x000ebfa9bb66825d,0x000060efc436424d}}, + {{0x000f8f66b8a7cdf7,0x000c324f111661eb,0x0003f176844bf6c5,0x00056be78edced48,0x0000217f1b2be94c},{0x000ec85fbc8973e6,0x00067f4d7ed8216f,0x00068f645f12fdb9,0x000735154bf07f37,0x0000c3cd2d5edd0f}}, + {{0x000f53916fc6beae,0x0001122765fa7d04,0x000fce16c2004cf7,0x000c22d9859805be,0x000052df10ff2d7b},{0x00099f450e1f9830,0x000c61f33ddf6269,0x000e06e68bf551b7,0x000e86ee34206238,0x00002aa249bfa9c5}}, + {{0x00047aa4c7ac2b3c,0x00082731a3a93bc2,0x00057da03752d6c3,0x000d7d2fd42bbf46,0x00008348bc06dd42},{0x000825653cf027b6,0x0001032f60c77da5,0x000a69fded019b66,0x0001dc3cf6ffbc26,0x0000333cf94fd444}}, + {{0x000ddab96e59ccf0,0x00077ba4a493b31c,0x0001ce1d36b7f226,0x000c8a14b54f20a5,0x00001a428916f43d},{0x00047a670dede59a,0x0002d0aed25ea138,0x000afd5154d9620d,0x0009faac5fa1d09e,0x000024f267bf7958}}, + {{0x00017e76ea8b4aec,0x000d995d314e7bc2,0x000b24fa2303625b,0x000d3e9d32ee7464,0x0000fd86bbb7517e},{0x000a5817e3ed6c66,0x000681fe7cda917f,0x000a903127323a60,0x0005ae96b12d8016,0x000044afeb2ca0b2}}, + {{0x000b69cbf65aaebc,0x00090c5ac751e7b7,0x000281d845c75cee,0x0002f93c693f9647,0x00005a6bb7c3dc6e},{0x000be7048732b0cf,0x0003da2f4bf94e19,0x00085541905b0af8,0x00067070b1a5325f,0x0000e3a10d49e546}}, + }, + { /* digit=45 (1,2,..,8)*(2^{180})*G */ + {{0x0002dfa557bfa495,0x0000476e1337c262,0x000db45dc38dd4f8,0x00012d3d96faadeb,0x0000c13ada75bd32},{0x000c6ca5801dad5e,0x0007c17be93c6d61,0x000188985039af29,0x000bbefa124866c6,0x0000462261edbc6f}}, + {{0x0002edf52c078d85,0x000c57dd4b79bb87,0x000adfda839ce9d2,0x000ba33e8aee806f,0x00007fc8b3dee585},{0x0003b2818874b38e,0x0001a4e2127015a3,0x000069054d6b7749,0x000ba60c89051d0c,0x0000fcc7ea0acfb4}}, + {{0x0005464d0e55495d,0x00003ed9aed9f40e,0x00006de43f8b680d,0x0007a14ea70ca6ad,0x0000ab349a9bed40},{0x000e0296a877751d,0x00012dfd9e535ef8,0x00070ac681b2a37e,0x0003f0788abf57ea,0x0000a2d6ecfa6e9b}}, + {{0x00063f4153af6dcd,0x000c683171b445fe,0x00029af26bf85199,0x000cbbae89bcf21e,0x0000e2560e6db219},{0x000e1d75ae224a6c,0x000f6a930add43df,0x0002cb637aac5049,0x000bc7d6862ebb14,0x00005e73664b3718}}, + {{0x00093a787b955171,0x000d06e1f017d5eb,0x0005bd3a711ff605,0x000a317a03753339,0x0000d94a848d4553},{0x000d57bd8e1e0fef,0x00062a2dcc99ccc0,0x00038d4cbe8fa59c,0x000cfe9d5c134b41,0x00007c0347ff2191}}, + {{0x000145c114649f08,0x000f3ccf0bcfc46a,0x000e148a3dd22dd7,0x000be895b3a822ac,0x00007ff5e819ea62},{0x0001437bd32a8c35,0x0004c538be17bcdc,0x0002e928c1bccb8e,0x0008a932f01e867c,0x000082bf7d9a8a3a}}, + {{0x000d10ec409dd077,0x0005b006c19d4c7d,0x000704a0fd36352c,0x00021c0e8a0f3c43,0x0000f1b113eab5f7},{0x000cbd604ebce52c,0x0003d6cac7cd3e1f,0x000b62675f9792a4,0x000ca4b06ed3714e,0x00007ff6667d89a2}}, + {{0x00080034798ca4d4,0x000f2c7af8685c8b,0x000d778ecd30d2f0,0x000cf952ad4c1c8c,0x0000e198f05db898},{0x00063d51806a84e5,0x000e1e52eca3cdb7,0x000b75c78219120a,0x000a6e0cfea1db16,0x000052a4a35bae24}}, + }, + { /* digit=46 (1,2,..,8)*(2^{184})*G */ + {{0x0008e13092cf573a,0x0005c3f4c8303203,0x00005750d3512460,0x000db40fb842c7aa,0x0000494598d37817},{0x00077989cb6b3d8b,0x0007062757558f1a,0x000150731ef6087b,0x000f58b273cc3e83,0x0000ffa7db45a98a}}, + {{0x000d84601a040653,0x000c4f4b0166f7a3,0x0007916814ccaa7c,0x00080f8fdfbd3e3f,0x0000bdc4370992dc},{0x000a6279f7786e45,0x000a95c7c1620b02,0x000d01014b0992a4,0x000fa801eba68b4d,0x000052fa0f983aed}}, + {{0x000a03aee51cbd7a,0x0003eb0b796d2197,0x000eba706ef4966f,0x000307cf5c3e95f4,0x00007329daec404c},{0x000aacb62b09eeb6,0x0000d76f53a89aa8,0x000109dd72b102e5,0x0003bfa4f0d8af9b,0x0000584ec8a3c986}}, + {{0x000a4c97a4abc3b1,0x000629f8f5fcda83,0x000483a1c45951ca,0x0007bee77f3e558c,0x00000e48e3d45037},{0x000f9b7025a36d96,0x000524bf6dce6a73,0x0005d44b52ac36ba,0x000ec10e9ff373d9,0x00005254e73f1733}}, + {{0x000f6e149c30d036,0x000786c252bd4796,0x0009a3476ff36be0,0x000f9406305e0f88,0x0000f83a58f41674},{0x000e1a0a8596c4ab,0x000785cde601a897,0x00087559b39a4f3d,0x000826d27d17bbe9,0x00009be39f3ad611}}, + {{0x000e3c3c1d7cda3d,0x00032ea87baa6279,0x000158ee6a629579,0x0004420c958c1fec,0x0000d64ac7b719e3},{0x000792f805f03b27,0x000eb7b2bcfe0b0f,0x0001327ffd07ff8c,0x0003e8c973510710,0x00002665759bab58}}, + {{0x0000291d43fc236f,0x000e82235e6fc06c,0x000e4b14f6918efa,0x0006704723477728,0x0000cd51067b09ae},{0x0009f8b71be82198,0x0002467cec5a196b,0x0003a235d360dea7,0x0004c8451deeb31e,0x000071e8bcc32913}}, + {{0x00005ab8b7207ede,0x000a6911a335cc88,0x0009e9fe5b2d6bb2,0x000344b19563459b,0x000063bbb632aaa7},{0x000b647545ffb149,0x000c9da8d006086c,0x0009f1914bb72b13,0x0006fefd76ca4846,0x000088e4372592ac}}, + }, + { /* digit=47 (1,2,..,8)*(2^{188})*G */ + {{0x000cfd57c06983bb,0x00098e453544774f,0x000b3dde4afbe78c,0x0005fb6f5834d0ec,0x00007bae9c3b3751},{0x000ee4d65bdcce8d,0x000ab3dd6901b1d3,0x00047673a7680203,0x00093d2623b49fbb,0x0000be9b2e57312a}}, + {{0x0009ae30f9bf470b,0x000087d0d63d1fae,0x000118fba7a5a59e,0x000fc5809658fc05,0x00005a1e22227a90},{0x00083acb655ee723,0x000edd1818baf918,0x0006032f40bacfef,0x000334844e27e9e0,0x000040f4d53495bd}}, + {{0x000363b0cc192c11,0x0001d774e6daae2d,0x0002caa668f0d453,0x00082567e7b3ae0e,0x0000eb18f19fbe78},{0x000e6d2a4ad162d6,0x00046cf309ae93b1,0x00071ab3a096ecba,0x0002c0901a3f80be,0x0000db5da2662978}}, + {{0x000ee84e265cd5da,0x000988dc2c6ff707,0x000f46f3d40f2a5f,0x0000c9df979d6122,0x0000ed01e35bdcbd},{0x0002d0793015eb1a,0x00095b063cd880a6,0x0005d2a436307bbd,0x000f365423f3ea7c,0x000081c0177d53ec}}, + {{0x0003403ae96daf9b,0x0005aaabeccefb5d,0x000862e19cddd34e,0x00005dbc1cf56ede,0x00001050a209e4e5},{0x0009710fdf7e6a41,0x000c40de8785a64c,0x0006cd11554ee79f,0x000527d0210f3d64,0x00003c2dab8fffd1}}, + {{0x00065f36c733ecb7,0x00097438a5807cef,0x0002edb163b75a2e,0x000e8e6daad28389,0x00005a009a4a047b},{0x00026231865878db,0x0006b9ea60d32b2d,0x000a6c1f604286b8,0x000e303b5ee93df4,0x0000ae2226c13edc}}, + {{0x000f246cdd4e3845,0x00068e73245980e9,0x0006e1e20523881d,0x000db26e1a6f5965,0x000056940ffbe18c},{0x000cde6b503aafdb,0x0005617f961ac0ef,0x0004eefd32ca8809,0x000b960d1c8a741b,0x00006dc0bf51f631}}, + {{0x000968734e4dd7f6,0x0008df9dec31a21e,0x0002817162256254,0x000a3ed65988c8b2,0x00009bb859066057},{0x000f94739f3adb65,0x000858f5da6309bd,0x0003e25d5a77565c,0x00068a42110f3a62,0x000053d37190038f}}, + }, + { /* digit=48 (1,2,..,8)*(2^{192})*G */ + {{0x00003f3e034d81e9,0x000580ef4f8b16a8,0x000e443be8670976,0x0001a9ec197241af,0x00001dd783168b20},{0x000097440f10fef1,0x00018a804dbec69c,0x00067b238c506e04,0x0008f287fa83b85f,0x0000d016711f649c}}, + {{0x000a1a96613ebed5,0x000aaa0b311898c7,0x000452f4160400b5,0x00055fbbce2a272e,0x000044105e96f0ba},{0x0002280dcf97d62e,0x00039cea1c8cde9a,0x0008d0499e144dcd,0x0003fe7cd9d958ba,0x00007fe9ad9ee6a5}}, + {{0x000320c5cb34fa83,0x000c1a43794f8dcb,0x0009951b966af168,0x000a28da0dcca923,0x0000bdaf4528f40a},{0x00083be9a0dbe4b2,0x00076ff47b50a951,0x0001b6b449d08629,0x0002b4f53933e3b0,0x000077b42a87e4bd}}, + {{0x0000a02b0eaceafd,0x00088c341dca5663,0x000aa7c1ff9509d6,0x0003070fb7de7fed,0x00009716331f83e3},{0x0005f449ec69c78b,0x00092aa2fd3cf36d,0x000709959bb35e00,0x00081dc33d131954,0x0000174691c29863}}, + {{0x0006166b44b3b527,0x000e8abe085116b3,0x00055111e7cb7191,0x0000aae2f04a4337,0x0000b8cbd7d43a2f},{0x000af71e91ff6ddc,0x000673cfd4b322cf,0x0006bf828cbf933f,0x0006d39eb726aa81,0x0000f4fde56a46c2}}, + {{0x00081dcee597d414,0x00087dac97e65165,0x00006a1325b93f7a,0x000a0d6498877936,0x000087e6a262fd86},{0x0009a34fc66e7cce,0x00055918f483f148,0x000c479481754c72,0x000f76c426d2dddf,0x00002d39a7634863}}, + {{0x000cca31c7a57625,0x000ade949dee1686,0x0002d160c071cd59,0x0001de16a1ae0522,0x0000da394a68f678},{0x0007da93fd13da3f,0x000971d9bc499082,0x000cce082821a131,0x0009544ac4a9b4a8,0x000025d76c70fa80}}, + {{0x000e658aedcda8bb,0x00031644cd8f64c3,0x000fe937f063f6ea,0x0001e975658a2d78,0x0000049c304fd752},{0x000473ffef5b241b,0x000792e05da59eb6,0x0002a9683732b03d,0x0000a6487ba3d60a,0x00005d72e137e219}}, + }, + { /* digit=49 (1,2,..,8)*(2^{196})*G */ + {{0x00099ee566eb922a,0x000364f692ac542f,0x000028dcac6c9771,0x00018cfbdf079ffe,0x0000610be982fc2a},{0x00013889ee80e815,0x000511fd62191997,0x000663545a2e0935,0x0008cc4048c883bc,0x00000920149ca0bb}}, + {{0x000eee0c0ea6e1a4,0x0009b8f81a1b3bed,0x0004ef395891f049,0x000e351f74f3ce14,0x000069aac489bd81},{0x00027729a694a1ff,0x000aec0d39e25c35,0x000071f076d9e952,0x0002d5b584061982,0x0000a6863991eccd}}, + {{0x0001ad919aac7231,0x000d837b1b76ba6f,0x00045b7295421df9,0x000677d0a1a2eab8,0x0000c06c35fd7a6d},{0x000314874f18d1ae,0x00031b807ea7b0ef,0x000e4bc5643a1ce2,0x0009b5aabedf5496,0x00007faef00cb014}}, + {{0x000747ea0bede08a,0x000e0753f73f449b,0x0007d5fd126e01d1,0x000e0524d1d1c94f,0x0000db80899bb94d},{0x000c10697e01e74f,0x00074fe228d291e6,0x0004d7b0c0585031,0x000c5761635b804b,0x0000e6909f3f3acf}}, + {{0x000a8f2439972c6b,0x000d88f0876fd4e6,0x00071943cf4a2c61,0x000dcc9cb45f0c30,0x0000274cda09b319},{0x0004426efde75793,0x000b01b65ccde7b6,0x0007c3a4fb720a13,0x000dcf1d741e2cd1,0x0000483cddf39166}}, + {{0x000d32fba5127c0b,0x0008344fc94dea3f,0x000edda3a50b098e,0x0009531ef34c8cdb,0x00009ea6e5479af1},{0x0003173f9743bb5a,0x00036f6795dcfb75,0x000330414c181516,0x0006467c0fef01b0,0x0000608f1137cc86}}, + {{0x000731021c804f19,0x000272aba16f73bc,0x0003c5c8b7dfaace,0x0001fdebf2fb3101,0x000060e66ffcf1c4},{0x00051f55a60c6b7d,0x000ca244fee99d47,0x000685a66c4490cf,0x0007f1df74bca48b,0x0000e008421c6eaf}}, + {{0x0008949a50ad4589,0x0009c1acb452fdbb,0x0004985e6bffc0ea,0x000efbfd931ee696,0x0000aba564d3b7e5},{0x0007a12f75fecfe0,0x000a3263c88f3bd1,0x0000c37c6a1321bb,0x00004ff03ab147c3,0x00003bd493c68746}}, + }, + { /* digit=50 (1,2,..,8)*(2^{200})*G */ + {{0x00032811a7abd874,0x000db7881fdad909,0x00057a04e50379b8,0x0002cd9e5cf638f5,0x0000f7d1bc229820},{0x000dd099becd5a2a,0x00089309bbd11ff0,0x000fa633afb561a6,0x000f7a86676108a6,0x0000803fa54a21da}}, + {{0x0002ede0e2398f58,0x000858a58d6cd461,0x00057b7853efdb37,0x0008c6b289cb633b,0x0000c7e7d1c765be},{0x0009e66e59813ff9,0x000d162065a6f4f9,0x000ccdd7da21b1ec,0x00007f41cc3a47ec,0x00006d5b783c2d8e}}, + {{0x000e4bf888b90c57,0x0004f777b0ac93d1,0x000d5aa9e84f32ca,0x000e8f22ba6e37b0,0x0000c2bfd71f0dbd},{0x00029438c4942ce8,0x00044e04df0ba2b4,0x000d0dcdfb573987,0x0007fc420935d5cf,0x0000d2dbc8e26eee}}, + {{0x000aebfa4044443e,0x000fabe67e573e06,0x0008d598ca0bb6e3,0x0007d84c505891db,0x00008b6a89573352},{0x000fd835db9f7005,0x000d7e9e56e55c19,0x00028cd210f50d2c,0x000afe3eb7148ced,0x0000282da8971416}}, + {{0x000d8c8bc0a17b96,0x000d956c17706169,0x000d6c4e0e6ccae6,0x000f778455351909,0x000039e97d726501},{0x000386493c5aa83e,0x0003ab6056becfd1,0x00038d0a6a30f997,0x0009bf4f933f1325,0x00003ec9cf3e5b1e}}, + {{0x000af546c8a478e5,0x00009fa5abea82a6,0x000929091c0b931c,0x000a611068dc611b,0x000031a3754d6e83},{0x000243fc70d1d9f1,0x00087c7c977ec238,0x000c3b180905907b,0x000f7b3c0a8de294,0x0000453273a39eaa}}, + {{0x000f989c69b6cdd9,0x0001b00b008733a5,0x000cf19582ee2e99,0x00082c3d0cba0697,0x0000f5ec6cff4889},{0x0003404faefb719d,0x0002c724c3c2b2fb,0x00084bc2cff9824d,0x00064341f90e02fa,0x0000ab3d02a9a4d4}}, + {{0x0009d7b51caacc1c,0x0008509081e9387c,0x0006c2bba30bf8a1,0x000749415fb9780d,0x000016b6886a163a},{0x0009d9971a12e8dd,0x000f041d6d9f6711,0x000897c914fde7c4,0x00037415fba6c1e2,0x000044ff79be19da}}, + }, + { /* digit=51 (1,2,..,8)*(2^{204})*G */ + {{0x000a38783edbf56a,0x0008a62060b5f619,0x0003e197df84183c,0x000ec5565a56f46b,0x00003d764abc2d6b},{0x000a0edc3fc096b3,0x00080a9da710718f,0x000633fc0eb6b9c6,0x000931e875a77998,0x000072910f080d6c}}, + {{0x000c351b873ccd50,0x000f849cb198ac73,0x000cd2f12adddfbb,0x00052b678a884a93,0x00006d2e4199685e},{0x000ba6f9e2ce488e,0x000619fe2c4b44b8,0x0006a77f7f29e16a,0x000b9984a580f6c4,0x0000c4fae9993e3c}}, + {{0x00043775095179f6,0x00074b65eb03c0ee,0x00038e84cad4f821,0x000c08ad2f19b795,0x0000a815addc931c},{0x000a6a2475d15354,0x000e250bb3ee8a3e,0x000bc6c7e2e9f012,0x00084e0f675eb14f,0x0000728fb5f890a9}}, + {{0x00079fb23be5ff7d,0x000b7abd3c095f18,0x000e5d17bb3553d9,0x0003eec8404b261b,0x00004c7b8e343501},{0x000f0ac52ff88cf7,0x0007572dfd754907,0x0004e2b22c9118c3,0x000d179073a97d08,0x0000d6bca24f52c6}}, + {{0x00062d8c5230f8f7,0x00029ee4b049136e,0x00014f3cb9c19a54,0x000e78a288b63bf1,0x0000675ced19a43f},{0x0004ac0245017d70,0x000da67379e7896c,0x000206517d607078,0x000c44a6ab25237a,0x0000c32d492b5336}}, + {{0x000e008c5bcdd3fe,0x00003fb319d76820,0x0008fc97a392e47d,0x0008db544b029312,0x00005611953b5d34},{0x000d3a1aadc08c32,0x000ab1c0278ca331,0x0006c870390417da,0x0008770cf666f76a,0x0000e48921cecb9f}}, + {{0x0000ffc1739db82e,0x000d6ff50f75f9cd,0x0002ad7569ae9e9d,0x000e1e3181d8eddf,0x0000eddba8e1699e},{0x00002aac66c37326,0x000c6e3037d90f29,0x0005d02ad905e85e,0x000d947afe3f307d,0x00000675780cc1ba}}, + {{0x000709def8c8c5c7,0x0004c1a567193ec5,0x000a8eed0812adee,0x0005924ddaf3c305,0x00002a0743a5403a},{0x000f431d23ed5fa0,0x000f9830eb2b6692,0x000b5818530569a5,0x000050c164d80ce1,0x0000cf41a7008416}}, + }, + { /* digit=52 (1,2,..,8)*(2^{208})*G */ + {{0x00037dccb38225c9,0x000f315adf7cccff,0x0005ec2414fcf3b3,0x00075f87e81a3e5d,0x0000f61ae1e090a2},{0x00009fa4f71b333a,0x0000e73907fba12d,0x00097997840dbf32,0x000506cc535daa6d,0x0000b54ff864cf47}}, + {{0x0008797e91aea7cc,0x000a480eb6b242d6,0x0002d2f863b4d485,0x00089479dd30bd02,0x0000e4b655e68a0f},{0x000d53005ec1aeb9,0x000290a6b4e185ab,0x000d8586b6a88091,0x00048c81b82f2c67,0x0000bb2a23d0098b}}, + {{0x000933d489d97ae7,0x000bb9e5cfe5c487,0x000e082b0c0ceb9f,0x000cebf8a82c020a,0x0000ded88fbff89b},{0x0000c73ff49bb729,0x000642e67c51e574,0x000a98cd0b0768e2,0x00043b1c60f64020,0x000020d51988ed0c}}, + {{0x0002f4209bf9bcf0,0x00059be9d8e68fd9,0x000288102284ec39,0x000064398db0f053,0x00004a5dc6b917e7},{0x000420b560d4b030,0x0007fcc1a739d4b6,0x000f3e6a037e1521,0x00016c1d009aea75,0x00000584c6da5516}}, + {{0x0005c25bb9b62679,0x00086c75488771a4,0x0001396cc5d8091e,0x000d1f142cb76e3f,0x000053b1fbdb29de},{0x00090a8fd649ba14,0x0008a31f88ce6dce,0x000c62ad1ca78ee8,0x000acf8c7be2becf,0x00000873b8c63e6b}}, + {{0x0003c778da9603b1,0x0004c5696f37750f,0x00069d8907179717,0x0006192ed25dda55,0x0000eacdd235716a},{0x000ba0f75d54c23c,0x000b8463b04938d5,0x0006234e9d1e300a,0x00020e9b53d9dded,0x0000bab7f97bd886}}, + {{0x000760871c2b484f,0x0004c8b4ebd3f02d,0x000caf56e9cde387,0x00059c4cea47acd9,0x0000eae7a0124e02},{0x00098b00ed0b01b8,0x000a0e536e530d74,0x0004cf87ac646f5d,0x0009e6d2a7abc52a,0x00002f3c4af4c323}}, + {{0x00082b921a859ae8,0x000619c6254dc41f,0x00038257ec641913,0x0006a3ca77e29392,0x00000c021e167183},{0x000a3d8c1544aca0,0x000f62e1402cec5e,0x00070ff1a4d2048f,0x0000bd9cb9f17ca1,0x000074174697d3a5}}, + }, + { /* digit=53 (1,2,..,8)*(2^{212})*G */ + {{0x000e0ace9790f135,0x00011e60ebcf7262,0x0001342678767f4b,0x0008aec094482b83,0x000086d2546132b7},{0x000cf8daedabfb38,0x00071d31f8d1f420,0x00014a5069e4864b,0x00083fb4b1b1e83f,0x00006eb2e2034cf9}}, + {{0x0004d5ef9bcae914,0x0000b0d1e935abb5,0x0003c077f3debaf5,0x000b3d7b5defd10b,0x00005a2eed07f66b},{0x000687601569519a,0x000a7ba74f17e266,0x0008123a7dcee411,0x000877efdc0a0e3c,0x000028a9a1af6c1e}}, + {{0x0009432af25a44dc,0x0003c0c207674f17,0x0006ba6b64e961d4,0x00069db1e112e09a,0x0000b97a68266210},{0x00016fbbff8bf955,0x000008fde2ddec64,0x000f11e7b94049f2,0x000e7c08a81392d1,0x000090f2310497ae}}, + {{0x000a995e5dfbc6e3,0x0004fc7904fc3fa4,0x00057d4a050cf2ac,0x000caf6aa14a23f4,0x00009e74b3de65f4},{0x00016c563b4a41b8,0x00083ecf3a5598a7,0x0007407e5359d663,0x000c3283c56534b1,0x0000a377a93e6974}}, + {{0x00064110e3d7be93,0x000db892f3b26e7d,0x00000446416395d6,0x000c44aa3f470986,0x0000d33c4ed3954c},{0x0002e3c7706265bd,0x000d02cfad414e41,0x00081bec63b3b22c,0x0001c5ef12b155f5,0x00004a979c09ab69}}, + {{0x000e709afd4ebe8c,0x000b8ea308780f21,0x000119d55ae78e9a,0x000664ba3dc8de76,0x00001b9c38ab8027},{0x000526074ae5ec7c,0x000c776021704560,0x000ac3d34419af49,0x000944e68bf204b3,0x0000bf2f1a243170}}, + {{0x000a45755beb5e5a,0x0004c3e7882f14f1,0x0004ed815580f741,0x0002464d0c9f6dc3,0x000088180b1213cd},{0x000ca2ca9f747a28,0x00054277c4608cf7,0x0001d9dd48232ab2,0x000f8730e7ccbdf3,0x00005ef4880c3dec}}, + {{0x0006539c7de5db9b,0x00043a7e0f222c2d,0x000b9c92b4c46a68,0x0003b3230309d42a,0x000069a4869c5fac},{0x000331fb46a1f47e,0x000277c432ac7d72,0x00000f4ddec961d1,0x000121ade72692cf,0x0000f86aeb4b1886}}, + }, + { /* digit=54 (1,2,..,8)*(2^{216})*G */ + {{0x000ffdfc7e7c2a09,0x00085eae457a4773,0x000bae6fdf8723eb,0x0009a0d71d19857d,0x0000d6ef525ea59c},{0x000d15002a515a26,0x0002c5426ee7cda3,0x0009a1edc37de8e0,0x000e199c8341b086,0x0000bbb468e51820}}, + {{0x000f6927b0d79bd1,0x000614c85edfa308,0x0003f86bcc29875b,0x000193b862597655,0x00001c7d62051005},{0x000d0721ecd294e3,0x000feba55f2f94c5,0x0008d744240dfdf9,0x00015625d6a996f9,0x0000e456c18ca0d6}}, + {{0x000072c30c9a3e3f,0x000a8e9262a35393,0x000d35fe1292598c,0x00087a2b978a49d1,0x000028a1a98b5727},{0x000912e3eefde708,0x000825277c850e61,0x000f065759f71f40,0x000459fad84f15f8,0x0000a72b7cdf565b}}, + {{0x0004cae01e64c20e,0x0009b2079e5fb67e,0x000a8bdf924006e5,0x00033ca37e1331fe,0x00003ed077fad719},{0x000c957822ca746f,0x000cfe60412a77db,0x000c22af18030eaf,0x0007aeb3106ff7ca,0x0000cacc54eeaa59}}, + {{0x000cfcde2a6c31ad,0x000d386365e668b9,0x000a0dc0c7d17066,0x0002d72beada8dcd,0x0000ed8b493a21b9},{0x000d65adf799777f,0x000c63e7e85d1e48,0x00096bbdaabe1343,0x00090a4ff9ca0967,0x000015220dfe9e8a}}, + {{0x0007e0aa7adc7a48,0x000db0cb0ce1c552,0x0004e3df0b8cb07c,0x0009bee3b5b534d0,0x0000fda2b88e9831},{0x000f7d9eeff2ac2e,0x0007e2d79362c410,0x0000823dcdc0db71,0x00077b12467920c9,0x00001801931c732f}}, + {{0x0001ff32eab51c2a,0x0002df10296f4fd3,0x0003b852814bc239,0x00078de051dca76a,0x0000dc7706867646},{0x000c2397edc895b3,0x000c8f18d2c714bf,0x0008327992dc9ba8,0x000467a2364b5c33,0x000088766fb7dea2}}, + {{0x000f5fa5904e2bd3,0x000209afa8338349,0x000206d2dfd21b18,0x0004529102295172,0x0000fd30a26b44f3},{0x000137b286ed0846,0x0004125e77d9a3b0,0x000a624d3959c964,0x0003d9c4a11235ce,0x000037f27954916b}}, + }, + { /* digit=55 (1,2,..,8)*(2^{220})*G */ + {{0x00061bbb05f9956d,0x000106ac42bd6d29,0x00082503dba8e1c1,0x000ae458e6df8646,0x00004572780d144e},{0x00004d881133b185,0x00070f070a6a26b8,0x000cb240e6288319,0x000ae43d370686c0,0x0000da343e03be55}}, + {{0x0001567f755e516f,0x0002de74462007dd,0x00047b5f76420568,0x0000d6ee7b8ab48a,0x00004f2bc1635d97},{0x000931de26c2af42,0x000d96b0887bcec5,0x000e8847159b8388,0x000b324cb694497c,0x000039c7e289bc5b}}, + {{0x000b15d6243f1091,0x0000ca11f17a34c7,0x0008a8443e31d5d4,0x0003fa53b5420ab3,0x0000927b5e2d1cf0},{0x000424051138a243,0x000fdb1e274e49c4,0x0004528d80f9684c,0x0000da2a45cf5074,0x0000abcad67dae2e}}, + {{0x00084e81ad96c3d5,0x00080021a93507a4,0x000744ed85217d67,0x000286a40b4cd118,0x0000702de63abcfb},{0x00077e27e30a727b,0x0000cb5272e9d6ec,0x0004ff812967789d,0x000a6af8eea1c93e,0x00003caac07df9b9}}, + {{0x000098baf9cb4cd3,0x000b81e48ac28403,0x0004bc21d97de9e8,0x0004798431831129,0x000013750d1196db},{0x000ae9e34b83b95b,0x00066584198da522,0x000be98219cfe30b,0x0007d4e08ab4fc17,0x00008387d9c3f13b}}, + {{0x0009f80d55b0c703,0x0003dec0ccced589,0x000b73ac42429524,0x000510fc625cd4b9,0x0000a65aaf5a02d6},{0x000f34bb38b3eac6,0x0007ac9ce6dc1532,0x0007a93199e8a328,0x0001c3b4d138d511,0x0000ca319150839a}}, + {{0x00061e15b0193f73,0x00026e84a34f239b,0x000c02e5ed252835,0x000fb353215fdb83,0x0000611a80a40f29},{0x00053277336c58d7,0x00082d4a127f89a5,0x000bb2b59fa73e65,0x000bfdf925d541e0,0x000059c1e0662639}}, + {{0x000f9b71b8c0f425,0x00077f2d0c051995,0x00029b4c129bcbe6,0x0007e7c82502fbc2,0x00002b22165d2734},{0x000b6e3e8373047f,0x000bb592b82dd077,0x000519b4c4ab6dee,0x0004bdb327630273,0x0000227777e79e51}}, + }, + { /* digit=56 (1,2,..,8)*(2^{224})*G */ + {{0x000c41114d0d0f0a,0x00071c035d0b34a2,0x000b56e6af5ad632,0x000f458d1440b386,0x00009070851ee09e},{0x0000477abf63470d,0x000c1f1ad95a0b12,0x0008478dc8a2c85e,0x0009d79c9c09b37a,0x00005669d660129e}}, + {{0x000a68df70882277,0x00065b3292a92874,0x00062d47b35717e2,0x000498f05ddc15cf,0x00002045f41bf3f8},{0x000a8b9343580755,0x000f94505bf7dd0a,0x0008e243ec49440b,0x000aea3afa4e63d3,0x00000f5462133be9}}, + {{0x0006113c503cd9d4,0x00061b51e706ad97,0x00044d98af8ed595,0x00086b990b99cebb,0x0000a86e1c215f82},{0x0003cbb144e6b9aa,0x000e4b097e2b5aa3,0x000c2fed61bf9a24,0x000125c6c7e1022c,0x000044eec8aec086}}, + {{0x0004e7b4f75c69c7,0x0009d717af715d2e,0x0003eb2b959f67ad,0x000b50256e2f7f59,0x0000faa39a85f847},{0x000657624d4d6888,0x000779788d5374a6,0x00031a2adb0e9860,0x0008607e22b915e6,0x00009ed17ced0865}}, + {{0x0007e49f538144e9,0x0000801dace5aca9,0x000179c203139436,0x000579d09c4fdbcf,0x0000b8c43e3ced43},{0x000f036040802177,0x0007090937e2ad5c,0x0007636ab7b11bc7,0x0009a49dc846e250,0x000015f05617398b}}, + {{0x000858e1e42fa26a,0x00096e07442f1d58,0x0000de8801c7a755,0x00023d647475607d,0x00002129ca073288},{0x0007cdea3e2c120d,0x000050b8231ee10a,0x000abbbc34902c47,0x0009866a41b80e7e,0x0000ea4fb6751c9a}}, + {{0x0005b9d57b4ca325,0x0009b07f8ae7c38d,0x000b67ba2c17122d,0x0004048b36db07a5,0x0000c13547ebaf13},{0x000fe5a101822457,0x000dbe78eba20f2f,0x000e71d40250d287,0x000744f58ef11ca8,0x000067b29ced3d7d}}, + {{0x000af127793627f3,0x0008811e51732d26,0x0007ba47d495211f,0x00011fbc5dfd6eb2,0x0000d0277a7c5305},{0x00059aba7caa2e14,0x000712d06c425aef,0x000b6687611ec682,0x00054599c6df92f8,0x00003f2120a92224}}, + }, + { /* digit=57 (1,2,..,8)*(2^{228})*G */ + {{0x0006605ecd65db55,0x000d72e421d3aa42,0x000cc1ef49735da2,0x000798f1cf926407,0x0000115826b66fae},{0x000b337ce7ef919a,0x000a7a6d6a5eabf5,0x0003c637e9a63491,0x0006f67021edb84f,0x0000746c950ad014}}, + {{0x000add3db7ec68b8,0x000b386d23ddc820,0x000c732406385f9e,0x000123ac503fd344,0x0000078adedd4745},{0x000755e7ed4c6729,0x000153f8260e01bc,0x00048d4be4a45000,0x0008bbb33fbeb49e,0x0000d816465d0546}}, + {{0x00074a9e86f273ac,0x0003768da9f3804c,0x000dda1996154227,0x000ef0ea5470f07f,0x00007a00585a4292},{0x000a3d7f108e6847,0x000633543471a24c,0x000deddbc6014539,0x000748d4d239446b,0x00003d82eda4eff3}}, + {{0x000157c30c88e767,0x0007447f23206d55,0x000fcbce3e45a30e,0x000a8919a2f6d341,0x0000644f481a46b6},{0x000508455987e93f,0x00086c52d4fb936f,0x000bf1494782ed2e,0x0009b3b64ef22f7a,0x0000e271957d8d37}}, + {{0x000051f896c2e8eb,0x000d7a69b583160e,0x0003d49398fdf12f,0x0009af8ffbe74fcd,0x000031d40fe11379},{0x0003fa86ded14ca8,0x000aa3806ceb84af,0x000ded17778cfed6,0x000b7ad455ace48c,0x0000241e365b6a4a}}, + {{0x000cdf92ebb9f2d2,0x00044fa908ec5b55,0x000d433b1e41dba3,0x0008fdc8a882d661,0x00004a21b29f9689},{0x000b80cdc8345f8c,0x00040425ef6c9c1e,0x00017accb726646f,0x000c62cc4b52d668,0x00004fe7071d103a}}, + {{0x000fc5b114eed902,0x00092c1ec222ba0e,0x000e6278ac333ebe,0x0000b826bc3dff42,0x0000884990265b21},{0x000e3baff2f463ac,0x000065bc6b05e85d,0x00088711208431cd,0x000750c9d0a6d302,0x000019cb19f15c3e}}, + {{0x000595cd1fa2a781,0x000d8d8df7304d44,0x000bb98b416f08ee,0x000983c60b71bcf1,0x0000fe06b3f76c34},{0x0009429622589d67,0x000cb9a4835859dc,0x000cb478d834436f,0x000f5234e4a0f0d2,0x000076555e5f3c86}}, + }, + { /* digit=58 (1,2,..,8)*(2^{232})*G */ + {{0x0004c9b20d431706,0x0002e62f23f2d925,0x000d0bef8e6b4e0a,0x00055d8206cab71b,0x0000c95a2a5e55d4},{0x00031469c615f1a7,0x00031aa9f2ac02f8,0x00069d5a83ea26e1,0x00016bd3403f8e61,0x00007f5ad3cb507d}}, + {{0x0008b75007f2d3e8,0x000b9583ae9c1bde,0x0007fac5923887d3,0x000d750b4e0af6d9,0x0000d1ef5fdad135},{0x0004069ea597b54b,0x0003bafe02358f2b,0x0006368b73835819,0x000671cfc31b8b85,0x0000fde8d8c56c72}}, + {{0x00028ad2a84e6c41,0x0004fcde36d07576,0x00017717c0d9fc24,0x000484cdff722d7b,0x0000ebe7611ea3b6},{0x000c336b09195735,0x0008ab520226040e,0x000064d14bbbb3e8,0x000ac8c886c34ecb,0x0000b85de43f45af}}, + {{0x000dcefcddd2868c,0x000b79e558df0194,0x000a5ea22e49ccca,0x00091ce24230da4b,0x00002dd640a90582},{0x00072d824d931811,0x000feb2a47d4c5ab,0x000efbe07e5114c0,0x000d6c17355ac9ab,0x0000fcd91a520be1}}, + {{0x000b8b3bb657e0d1,0x0002823cbb13d1b0,0x0002507060487a33,0x0003073a998799f4,0x0000a4673e731318},{0x00071b9484805f36,0x000b92e3ecebb211,0x000b1f5665228e4a,0x0000fe71f17cb6c4,0x0000fab132e7caf2}}, + {{0x000e8ef838c9fbc1,0x000c452de7e5c194,0x000973312d33ad9f,0x000d8b33dfab2860,0x000073e06257b5e1},{0x000ec0507e35022e,0x000df264cb1bf3d1,0x0009551e8ee3b962,0x000b4deed4c1f9d0,0x00009cc539a7598a}}, + {{0x00094abebcf27683,0x0008e09a607419d5,0x000262210bc3d71d,0x0004daef3faa71e6,0x000036cd41505615},{0x000a29a36702adec,0x000ba91eb78399f1,0x00052519283d6b50,0x000ce18f048aafb3,0x000081d651ae4804}}, + {{0x000a55b52ea9cdb8,0x000fab5ef7d92893,0x0007df294c2baacf,0x00036dbb62480d4a,0x00005fd14d5bdd1a},{0x000d5e9d7d31477a,0x0000715309eadb09,0x0009f58728a80fa4,0x00022c35adea7de5,0x0000ddf8bf0e739b}}, + }, + { /* digit=59 (1,2,..,8)*(2^{236})*G */ + {{0x0008d2fe5724a2d6,0x0008a411e84e0e5b,0x0002f5d04e3bf4c5,0x000d0d132a5db84d,0x00001765a592c24b},{0x0008b4a422ebc11c,0x0004336f3eb82fab,0x000c454ae73559b6,0x000b3a5108cb20cc,0x0000bc49662e3c97}}, + {{0x000dc9d1ec6f170f,0x000bea3af2ebc2fd,0x000ced852c3855dd,0x000b601b1a0af843,0x0000efe50594d52d},{0x000740f316de5b77,0x00051bb62771deb1,0x000a7a84b76a9d83,0x000c81aae829277a,0x0000e1420c11bfa4}}, + {{0x000d60f8e1cfb95a,0x000af10cab91f1eb,0x00038fbbd9345d96,0x00016c852bedd9e4,0x00005be88d186e1e},{0x000701924533f212,0x000417b9e05614a8,0x000660adcd80a4d5,0x00039c726fd26c76,0x00000cb351353c24}}, + {{0x000fcf57c64d0fe1,0x000089aeeaf8c495,0x000d0e557f623c19,0x0001b346cee7aa73,0x0000e3399090fc9a},{0x0006eb38aad81cf7,0x000a739b6057604e,0x0000db9b5314c754,0x000f3472d7d343c0,0x00004c1dfbe3a0a5}}, + {{0x000701a7d15ac5c0,0x00023520a981b0d0,0x0003b61899d2b0df,0x000d9eb9c1c354cb,0x0000a66e6fc26984},{0x00090a54c87214b2,0x000caefdcf2b9218,0x000333ea2b996b05,0x000111ae66ceef0b,0x0000fbf98e12d06c}}, + {{0x0002d09a5a1d9cdf,0x00039db4f6d01b11,0x00027c414e4622ba,0x000bc3b3be73c6bc,0x00008bb6de003282},{0x000e59669367bb32,0x00042eee7de13f4c,0x000535f66fef6436,0x0008d7d70177e2be,0x00000eff574037b4}}, + {{0x000291aba861ed06,0x000e76c048752a14,0x000e5d81960c63b1,0x000d9aa9241f2702,0x0000cb8d90ea933b},{0x00055408ef3f36fd,0x000ad70a653d61da,0x0003a9daac2f423c,0x00082428fd1b648d,0x0000eccb9ac2171b}}, + {{0x000848949c160154,0x0000e4f8be762b49,0x00006d459f779d3d,0x000d6b1fa2a9ee4d,0x000073144722cd51},{0x000631594e5d8939,0x0009ee5d23a84178,0x000d6db14456f57f,0x000650e1f892f3b1,0x0000d7a0b3f34fe0}}, + }, + { /* digit=60 (1,2,..,8)*(2^{240})*G */ + {{0x000585113ba8dd62,0x00080ea1f095615b,0x00065f617af767a6,0x000aab6be0a28ad7,0x0000085f54531694},{0x00033ed1deec48b1,0x000ae0b30df8aa1c,0x000b877911cd914a,0x000e64fcbaddda07,0x00004abbcd21ba68}}, + {{0x00032b33d4f558d6,0x00037483e07113c9,0x0008ea602fd9386b,0x0000f762b6e4a5d3,0x000040b7b04b7684},{0x000f178169d618d6,0x00020510dbbf08dd,0x000885aab8e91f1c,0x00043069b211d07c,0x0000c105f022a618}}, + {{0x000568d0917c4acb,0x000c55a83cdd60ed,0x0004cbae30f32f9c,0x000d8992a50602fb,0x0000910f7a813c33},{0x000ec2057b354ced,0x00035c70259217fd,0x0008e56f3c6925cc,0x000490b6e93831cd,0x0000a336551d0bd7}}, + {{0x0008835dedbc740c,0x0007d25913cc16df,0x000ce91f116c9a1b,0x000808dae18bc5b6,0x000071a394c1d139},{0x000ce433b9ac0821,0x000ead865a40ab05,0x0008bc6703c50add,0x0000029696559643,0x00009a0bfc436be6}}, + {{0x00066dd6d2079f06,0x0004c3d42e06189e,0x000067cd28417e6e,0x000130050bf07794,0x0000de2c929cf8ba},{0x000333b1e86df344,0x000b967ff99abde3,0x00009477132566ad,0x000536ed7c990355,0x0000973cee61a30e}}, + {{0x0000428dffccb096,0x000cfe784d6365d8,0x000f0f3311b8f429,0x000378330bcb7443,0x0000aa82351ab1ae},{0x00058c1fb6bdb9c6,0x000e9efa4faf8433,0x000dca1d65a933e3,0x0008101ceced8538,0x0000d4a8d9e6e600}}, + {{0x000e6c984782812a,0x000b4f7be42a5821,0x00004697d39f6d6b,0x00067b1a9d3fc608,0x0000472dc3a58087},{0x0006fec314288ecc,0x0007d25d30c31f70,0x000aa52b5e0e12c8,0x000798de7b7eb72b,0x00005698fa1698c6}}, + {{0x0009650f1aafc77c,0x00018b07e50122b3,0x000b118b6d8a8280,0x000784d7bdd744f8,0x0000b7a7039bee26},{0x0003cde18959c8fa,0x0004df71fa38477c,0x0009110e30c5b652,0x000a5bb8a3d815f1,0x00000fab64c70c58}}, + }, + { /* digit=61 (1,2,..,8)*(2^{244})*G */ + {{0x0009c02205f112dc,0x0009e952f41deff1,0x000acc7b67b111a0,0x0002d1d510e44a59,0x0000fa3e8511d623},{0x00051ff946f13b54,0x000dc69c4b7d692e,0x000317f509655586,0x000e6b4462e6392c,0x0000a96c730c5b29}}, + {{0x0000a3933301eb25,0x000a651566c5c43e,0x000854f45d136c9a,0x000a9b44acff9c91,0x0000afb49c7ad947},{0x000a2e067e61d8ad,0x0004dc10dc1eb2bf,0x000135c5137c224f,0x000bda57488cfd8c,0x0000c44a7f8d059f}}, + {{0x00052723d33fd8ae,0x000d77c5b0f7bd45,0x0008bb677dfa7a03,0x000ac19a96990c2c,0x00004b6f2495b079},{0x000fb6607adecf05,0x000bde2919ba0f93,0x00090c3da815b883,0x0000a7d12075a3bd,0x00004310fdce89c8}}, + {{0x0006c13cc1c1b2ba,0x000707f3e5f9f11f,0x0009e009feccf526,0x0004546bd9afe153,0x0000da180beb161c},{0x000f8417a1d44bc5,0x000008e325fc3ee3,0x00006399ac1041ab,0x000580ee77109540,0x0000ceab562965db}}, + {{0x0001872bafa156fb,0x0007a216ce400bb1,0x0007b96951ea5324,0x0007767e0f72919f,0x0000be25883d565f},{0x000a586c9ea652aa,0x00077eb211f7dc69,0x0005ba936a575e48,0x000dacedab8900e5,0x00003e88330c056f}}, + {{0x000fab17514db1c6,0x0008172813b230d7,0x000e97892cd69e71,0x000cdff6a634d0f5,0x0000c7df5b396089},{0x000b9815a106666f,0x000a1a74f7c4f830,0x000416725238afc5,0x000ab9aa0a7d2edc,0x000091170e9acf8b}}, + {{0x000a68529f79ae19,0x0006ae151c3ebe53,0x000797f52949338a,0x000e08fc9db14d5c,0x00004a632a0f71db},{0x0004b14f4695b847,0x0005adc149f75b88,0x000082eb2d6c5560,0x0009f624e9b8946a,0x0000a038b421d27b}}, + {{0x000db6d961011f15,0x000439b319540c33,0x000964ccfd972723,0x0008f8acb18490f5,0x00000165db5b23d9},{0x0003f6e09555115e,0x00005bb146110697,0x000e9391de70734b,0x0006302d2b19436a,0x000046716ed749df}}, + }, + { /* digit=62 (1,2,..,8)*(2^{248})*G */ + {{0x000b8a4cbe978aa8,0x000fbc8ee3c76cb3,0x0003979fd2b05b7b,0x000d941563cf1162,0x0000e542d606a5d0},{0x0005e56df6f115ed,0x0003ca59ce6bb278,0x00017eef4378ebbf,0x0000d8c8dc4afaa5,0x0000a21b92c92f88}}, + {{0x00035dc5396eb247,0x000c8ef4e4ca4631,0x00072209072a4380,0x000c5c200bcef728,0x0000b72cb6549160},{0x000b0c5738b12183,0x000822e27bf1bc68,0x000f7cd34933b26d,0x000119eda10a224e,0x0000c80f888434bc}}, + {{0x000e0efb3b8b1b5d,0x00038959884aaf7a,0x000259a44d6afe4d,0x000f91f87b1959be,0x0000337331701bb0},{0x000b01a0216bb368,0x0000c5eca8c325e7,0x000671fd9f4f814d,0x000e76c1d3c91169,0x000010645e8443c5}}, + {{0x0005b4488b3fb1e8,0x000d26b23a5d8961,0x00056a2fedc3595e,0x00081de771fe19e3,0x00005b981b48385a},{0x0002c4f79da9b17b,0x00017541a1f22bff,0x000fb8bc6c4a7592,0x00033ea8e920a8a6,0x000037f6be35ab5e}}, + {{0x0006210d9e58a61a,0x00029b5224c08dca,0x0003db93fef2cd7b,0x00002ae4f2bbb09b,0x00004673f3e36b54},{0x000acede4893a3e0,0x000167a09cb54d69,0x000fb53a3b5bd9ea,0x0006f485791eef6d,0x0000d389cb15387b}}, + {{0x00078c2ec1e3ed4f,0x00051691416a6a5e,0x00085c63595aa0e1,0x000163b5d8f860c7,0x0000283d923fc854},{0x0005beef7ad3f5b3,0x0007b815ad735813,0x000023a1511c7557,0x0006b79bab4cc9e0,0x0000199d8c4e721f}}, + {{0x000e42f66dd9a70c,0x0000846de6736294,0x000e21f936b36c3b,0x000215cc757f7aa1,0x0000e42c4db111bf},{0x000f78959874b51d,0x00067dc910778965,0x000f6350f2c2eb73,0x000e53487a0d690c,0x00008713f1619ac6}}, + {{0x000bdec2116e9b29,0x000389c76497ee80,0x0006bb3874fd1cc1,0x000343fe15d2b0ac,0x0000a3fadcb3a4e1},{0x00037248e9d64745,0x00087efa63b10110,0x0007dd6bd1db932c,0x000c6c78bf9e3fa9,0x00009e31d0655466}}, + }, + { /* digit=63 (1,2,..,8)*(2^{252})*G */ + {{0x000be5ed0e405bae,0x0007fc91ec34f9e7,0x0004b79b18f54024,0x000b106f3d8772d3,0x000037037c975e12},{0x000aec44147d71ff,0x0003d1931e82b100,0x0007bf1327384e2c,0x0002fb55ec63d285,0x0000df2ba6d3b215}}, + {{0x0007a23f356e3059,0x00041e71e29a3efb,0x000f94f0f0f98de8,0x0007a880ecbe906e,0x00002d869e92df60},{0x000cf0bbf87a34ec,0x00089c2efd2119d3,0x000ebfbd0dd06fa3,0x0006c851303198c6,0x000030a29d4bfdd3}}, + {{0x00089caef387625f,0x000fcb72b7247593,0x00017a38174726e5,0x0001b102c945353e,0x0000fcd0db4d1457},{0x0003429bba3484a4,0x0001753db65ef147,0x000e6a574289160a,0x0006d9c5f2dc2cb7,0x00005d42b1ac334f}}, + {{0x00012c9e1ee0d0a1,0x0003490b01e6e274,0x000da05414bdd548,0x000de03a9047e2cc,0x0000c371569c9623},{0x00078851bc8c9a7d,0x0000d36794075521,0x0006dbaa6726fc38,0x000ce611949c5013,0x000053af2d9b1059}}, + {{0x00035aa24062df0d,0x0002a52fb85fa4ca,0x000d94e1eaa94f02,0x0001840aed61257c,0x0000fb93c2113070},{0x0000d05aabec972c,0x000f75d4421fc835,0x00007292ec8f71dc,0x000b37558f6df8ee,0x0000d9d09ec67e4f}}, + {{0x0000235a102cb4ec,0x000fb35a64785f45,0x000b0f0672f75fa1,0x0002e6467bc56637,0x00008030444d7012},{0x000881065be741fe,0x000b8d8f2c4aa658,0x0000fb14fbdf31d8,0x000b607bac347583,0x00006adf01034a09}}, + {{0x000f12502f6f25d9,0x0000c22cc1b5e838,0x0000f6f93bcc544b,0x0008c7ddfde2d4ad,0x0000c68d49d6c0f8},{0x0003511dcdcef6f9,0x000a7dc7783920a0,0x000dbd6b689c8148,0x0000f52a6e80014a,0x0000730b2f927704}}, + {{0x000211e48a709134,0x000a06d95a7b1a29,0x0005aa515d70a8ec,0x000baface9c4e7ad,0x00001420ee199ed7},{0x0005f92e47355a0e,0x000716ec78ef1f42,0x000fe05e173edf3f,0x000e4dfe82b2c090,0x00005f26894a26e6}}, + }, + { /* digit=64 (1,2,..,8)*(2^{256})*G */ + {{0x0008910bd3c74498,0x000d8af3aeac968f,0x0008cc252692ed9e,0x00067be64e4f7fee,0x000092836060ef1c},{0x000890896b28139e,0x000470cc9c0726e3,0x0009a0eab753f427,0x00046bc1b1530956,0x000038600b6014d5}}, + {{0x000cd15a3cc6fb9b,0x000a19f82e4c6346,0x0003cd730abb95f1,0x000222f258efb831,0x000068413078deb3},{0x0001e7c1ed462bd3,0x000d5fdbfcd8fb51,0x000f5c6d0b354d1b,0x000acc02f31db2ee,0x00009cf8f2c231b2}}, + {{0x000ee3e9bd875a0b,0x00079f5903fa2711,0x00029af6a861120e,0x000561354e6da0fd,0x00000c0f6913abd6},{0x000948148819fd8a,0x0008e1ce27a94979,0x0003f4d9497c8870,0x0002f21ca36d254e,0x00009bc3a89fe40b}}, + {{0x0004af860fe1d658,0x0005c3a43228d831,0x00003626b989c96b,0x000ceba2924ae1c3,0x0000c45b79310a64},{0x0002ceb1de0d0667,0x00088613f714aa18,0x000d68a9c780c9b4,0x000a36f94f51865a,0x0000055e19d4f0d9}}, + {{0x00014fc2a4b59f10,0x0004bfd2f9fd51a3,0x000216cbd55294c7,0x00097be507f2f1fe,0x0000924f1faac4f7},{0x0003541b7a3d4f01,0x0003c8cb55fc5c6d,0x0007ec5dc6c3980a,0x000f63bcee3510ce,0x0000ab63b2a3eef9}}, + {{0x000560910cddf7bd,0x000a825eca445df4,0x0001e77f13b9bb31,0x000af16bcf1af24f,0x0000d48e4c17550d},{0x0005fc863ac98830,0x000ec32093eaa327,0x0009073fe1808aec,0x000fee1100183134,0x0000b88f64b6ab63}}, + {{0x00017535ffca9e3e,0x0004c4176a9f05d0,0x000b8e9c9e88a67d,0x000aa685e06ca4e4,0x0000941d64ce2bd7},{0x0004eb4d5b7fac46,0x0003ece406851d75,0x000f3bfe1359045f,0x0007cb734afa3acd,0x000087d6f48d53f1}}, + {{0x0004ef0fb7992ec5,0x000237355dbd4b3d,0x0007914aabfa41db,0x0003654621f87992,0x00002b7c7dfd2d83},{0x000c21100586c6db,0x000ded8f1bfb12a4,0x0008223d7b6ca10c,0x000b4b5146ab877e,0x000069c991a7978e}}, + } +}; + +#endif /* #if !defined(_DISABLE_ECP_256R1_HARDCODED_BP_TBL_) */ + +IPP_OWN_DEFN(const cpPrecompAP *, gfpec_precom_nistP256r1_radix52_fun, (void)) +{ + static cpPrecompAP t = { + /* w */ 4, + /* select function */ p256r1_select_ap_w4_ifma, + /* precomputed data */ (BNU_CHUNK_T *)ifma_ec_nistp256r1_bp_precomp + }; + return &t; +} + +#endif // (_IPP32E >= _IPP32E_K1) + +#endif /* #ifndef IFMA_ECPRECOMP4_P256_H */ + +#endif /* #if 0 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecprecomp4_p384.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecprecomp4_p384.h new file mode 100644 index 0000000..69dfb3b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecprecomp4_p384.h @@ -0,0 +1,1129 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#ifndef IFMA_ECPRECOMP4_P384_H +#define IFMA_ECPRECOMP4_P384_H + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpecstuff.h" + +#include "ifma_defs.h" +#include "ifma_ecpoint_p384.h" + +#define BASE_POINT_WIN_SIZE (4) +#define BASE_POINT_N_ENTRY (1 << ((BASE_POINT_WIN_SIZE)-1)) + +#define OPERAND_BITSIZE (384) +#define LEN52_P384 (NUMBER_OF_DIGITS(OPERAND_BITSIZE, DIGIT_SIZE)) + +/* P384 affine point */ +typedef struct { + BNU_CHUNK_T X[LEN52_P384]; + BNU_CHUNK_T Y[LEN52_P384]; +} P384_POINT_AFFINE_IFMA_MEM; + +extern const __ALIGN64 P384_POINT_AFFINE_IFMA_MEM ifma_ec_nistp384r1_bp_precomp[97][BASE_POINT_N_ENTRY]; + +#if !defined(_DISABLE_ECP_384R1_HARDCODED_BP_TBL_) + +const __ALIGN64 P384_POINT_AFFINE_IFMA_MEM ifma_ec_nistp384r1_bp_precomp[][BASE_POINT_N_ENTRY] = { + { + /* digit=0 [{1,2,3,..,}]*([2^0]*G) */ + {{0x000607664d3aadc2, 0x000fa3dd07565fc8, 0x000e1e26a4ee117b, 0x0003afc541b4d6e6, 0x000459a30eff879c, 0x0004ede2b6454868, 0x000513812ff72361, 0x00000000000299e1}, {0x000af93c2b78abc2, 0x0006e23043dad1f8, 0x000d385481a72d55, 0x000e7562e83b050c, 0x000968f4ffd98bad, 0x00069a840c6c3521, 0x0005e9dd80022639, 0x000000000005a15c}}, + {{0x000a271bdb93b776, 0x00066c8229e549ca, 0x000a0046a4ddbf0b, 0x000e6f0ff9d48a26, 0x0005f0687f503504, 0x000e4b506da82149, 0x0000c39c90a4fd2d, 0x0000000000042746}, {0x000777e3e34947f7, 0x000cf42ea84624df, 0x000c322ca0a5f414, 0x000f18bdc588259c, 0x00015172bad915e4, 0x000b0e68409f1fe4, 0x0000c2070d430900, 0x00000000000123df}}, + {{0x0008420bd283fe68, 0x0000405e4dbe5ef5, 0x0007d2a868c2d376, 0x00034e9a170ccf19, 0x0002d51c6c3e6b20, 0x0003aa4703a48d73, 0x0003ace36f7e2d26, 0x00000000000e7c1c}, {0x000a7b5b465465fc, 0x000697e28482179f, 0x00092befa3c13549, 0x00063c04ef67446d, 0x000ad2e1d0b41326, 0x00002b33968012d5, 0x0002aff6db68b151, 0x0000000000098329}}, + {{0x0007a84fcfeee6dd, 0x000c00aae84771bc, 0x00004833a9bdf308, 0x000153b0aecac470, 0x0004736400ad2e4f, 0x00085d979078358d, 0x000228fb40f647d6, 0x0000000000034179}, {0x00059b3d50946875, 0x000f354f3e8e74aa, 0x0007e02066cc9331, 0x00061a34c542ad23, 0x00030418c6938e3e, 0x00020017d147162d, 0x000319e607b9e338, 0x00000000000303df}}, + {{0x0006ca2ee1bb26b1, 0x00017bb595eb9873, 0x000340e77dae425b, 0x000b1b5293c703ca, 0x0005aacc05e67f1e, 0x000e8e4c660db2cf, 0x000ffbc676b987e2, 0x000000000001d178}, {0x0002304b4db1c9d6, 0x0003c2b694ba022c, 0x0000733804c0a50f, 0x0001b3101c35b997, 0x000f982c041180b6, 0x000de236d4b237fa, 0x0004a3e6c5944024, 0x00000000000e209e}}, + {{0x0003f94fc482e189, 0x000c37eb5c930b8d, 0x000fa7363cfe5622, 0x000930f580d57f38, 0x00061bdf6015ec52, 0x00002d33b2a33f66, 0x000c404f0f6a962b, 0x00000000000f0430}, {0x000a60b1c9962152, 0x000203f62b16dde1, 0x000d30e7f024d36f, 0x000bffcb79e33b13, 0x00061b546f058bd4, 0x00021559a93c9e5f, 0x000eba586d8ededf, 0x00000000000af2a9}}, + {{0x000c82aa932c9f81, 0x00032df13b9d0ea3, 0x000012e0e11a7414, 0x000dcf8211faa95e, 0x0001659753ed731e, 0x000b2df555f4215d, 0x00025bf893db589d, 0x000000000001c890}, {0x000c8f68d90a16b6, 0x0002f0996b21f9df, 0x000c5d608c816e33, 0x000d76f50130844e, 0x000401fff78065aa, 0x0003b07060ff37c0, 0x000b3ef57f238e69, 0x00000000000af6c9}}, + {{0x0006da365db4f184, 0x000fe23f60a057fe, 0x000be85f6a0c5049, 0x0002e7193d7e30ff, 0x00064f3ddb371c5c, 0x000b664231d9aebd, 0x0009b11c7b5fe116, 0x00000000000349cd}, {0x0008ec6d3c0c6dd7, 0x0005e0d2cfe83aa5, 0x000f7a290df3f1cc, 0x00054bf7d8686e4b, 0x0003a42dbba27017, 0x0008ecf0ee992326, 0x000f617008d943c4, 0x000000000000d27e}}, + }, + { + /* digit=1 [{1,2,3,..,}]*([2^4]*G) */ + {{0x000b56e7a10896aa, 0x00082da6e8a7edb2, 0x000a339205afd669, 0x00065517917652b3, 0x000a2887d5ff37cf, 0x000bdc3fa317b63e, 0x000aa137065f5313, 0x00000000000435ab}, {0x000e15a5659db481, 0x0008e9b21615f8a0, 0x0000a5926b88aaa4, 0x00071dc154d41105, 0x000bc88ee1489148, 0x0002d1967b333bdf, 0x00051351c305c6a7, 0x0000000000081ef2}}, + {{0x0007c0045dd2a4a0, 0x00075ad852b872e3, 0x000b4af19266ca1e, 0x000b9fcc651b1dd6, 0x000612e8dc871896, 0x00031cfb0ba8953d, 0x000d793a9865baa6, 0x00000000000626b3}, {0x000b7328c510ad93, 0x000901148bc71a36, 0x0008838d56b7b5d9, 0x0005f9e9448fd096, 0x00000a2377b67731, 0x0005b4ff04bcb06f, 0x00099f73b42725b3, 0x00000000000aabca}}, + {{0x00077d8466e4794b, 0x000763ccf806a4c7, 0x00041b06944ed785, 0x0004e06ea52bef99, 0x00053a7d2f3c4f50, 0x0003a1bc940d01ff, 0x00040062e5c5d3e7, 0x00000000000ae7d7}, {0x0000b071271e42a1, 0x000923d30625e38e, 0x000c9ea33ece8520, 0x000a10d04bab9856, 0x00009da2a2ca5c3c, 0x000c9462c2605ca0, 0x00058348eab00eb5, 0x000000000002624a}}, + {{0x000c57a24f5d5ab6, 0x000eded4de4f83f2, 0x0004d9f2c578d7f4, 0x0003d30f8a0580de, 0x000dca57b7bde04b, 0x000e44d56a309199, 0x000e5cfc8e87cf3b, 0x00000000000d1b30}, {0x000dc1d0888b708b, 0x0008cb213c69fa81, 0x00085d35b9791d2c, 0x000bbbf1090fede9, 0x000c301fe259fe51, 0x000cd3fe86d97cab, 0x000a513ee127895e, 0x000000000009404c}}, + {{0x0004d8911e8568cc, 0x000c5194924b48e8, 0x00026b2f852cc83a, 0x0006428b12136094, 0x000351fea1dc4906, 0x00015ace6dd2ec6d, 0x00024620fe8c27a7, 0x00000000000a4463}, {0x0003c328530abb42, 0x000b900c213bba9c, 0x000bf43a5f2c2e1e, 0x000903c6484641de, 0x000a1378e68fbc7a, 0x000cd8ae42413063, 0x0006f9b960b5efee, 0x000000000000614a}}, + {{0x0001dedb8bf3dccc, 0x000a0dc529384912, 0x000bc9fafda07c1d, 0x000597e52ce08f71, 0x000581998af2ee21, 0x00041eb4226de4c6, 0x000b96cb4aa43c97, 0x0000000000039c18}, {0x000a9ce2b257fb6c, 0x000a566e1d5da261, 0x0002d61f72303077, 0x000396f305ee4f10, 0x0000c831254b8545, 0x0005680b8f9d19ea, 0x0004a4cee0842f5d, 0x000000000005a443}}, + {{0x00007168b4a67147, 0x000a12e206547853, 0x0001c6a852120cb6, 0x00009d5504c8129c, 0x000c9710b70b2b56, 0x000296a52fb25e37, 0x000dce83f2fd2cd8, 0x0000000000062f45}, {0x0000b128f82bb944, 0x000a8a818b9bda93, 0x000ed2d611039805, 0x000e43a2ec76a180, 0x0007caa846883e7e, 0x000182141473e687, 0x00004db9a19eb57c, 0x0000000000045ed2}}, + {{0x0001e64ba6661cc4, 0x000cee484fd9edbf, 0x0005b5c2b4988114, 0x000449e7c1c3984b, 0x000118eb5195c0dd, 0x0007a16d2f313389, 0x00020a0336aab877, 0x00000000000c2417}, {0x0001e21e239dcab2, 0x00034678db970845, 0x000627331787ffae, 0x000a0e4a022c7a44, 0x000434a02a6b5d85, 0x000791ce3b01f1e0, 0x000c5b2657eedda5, 0x00000000000a277a}}, + }, + { + /* digit=2 [{1,2,3,..,}]*([2^8]*G) */ + {{0x000fa21fa335ab82, 0x000a49a7a5b41c7a, 0x000300862e13765b, 0x000438e3d9f0e627, 0x0009e328c2e27539, 0x000cbf891013c671, 0x000d287f4a706ccc, 0x00000000000735a2}, {0x000a7119424dd00b, 0x0004246694eeffb4, 0x000059afb703b483, 0x000ed8b423d47e45, 0x000bf44c91809d54, 0x000e9b3848075a8c, 0x000c75d4f5b184ab, 0x0000000000041abd}}, + {{0x00093e732cc6e06c, 0x000c65e2cb07faab, 0x0006c10c7767a2e6, 0x000c53fd4de1f262, 0x000c838f7169a296, 0x0008a6ce7d408060, 0x00067168e19d7b2e, 0x0000000000094b58}, {0x000136755dca2adc, 0x0000293d02a07640, 0x000ed9dfab92ca5c, 0x00069f51aa3bc4ef, 0x0000dd09b1426aa0, 0x0002e59450e44fbf, 0x0006ace264f34383, 0x000000000001fc16}}, + {{0x0001b41eba2511e6, 0x0003e9ee4f521f6d, 0x0005af7a840c9880, 0x000396db7edb07d1, 0x000c2e8290630d5f, 0x0003495da09b3457, 0x0009b8f1d28188f8, 0x00000000000cce55}, {0x000f6b035c499b66, 0x0005617cbaf577ca, 0x0007eb3582ad9848, 0x00008995145b8fd9, 0x00081a33b1a72982, 0x0005149e992cb5da, 0x0004c0ca49fe334c, 0x000000000001772b}}, + {{0x000b80038e0f9767, 0x0006756ad758212d, 0x00066af19dfc2941, 0x000c6ffe2c8b0369, 0x0007fcd7336b85f2, 0x000a46acd55c6d35, 0x000ac7b1ecc56d22, 0x0000000000036277}, {0x000330b02f145871, 0x000c1a4ed11e8d27, 0x000297add7ae640e, 0x000ba45266158ab0, 0x000d89e0dff05fda, 0x0006b02d06f0b27f, 0x0006e132ef7ae2eb, 0x00000000000cc1b4}}, + {{0x0008162061985fbc, 0x00005c112733b3ba, 0x00062ae17de90bd5, 0x0008e01810097859, 0x0002bfe16c4fbb7d, 0x000d9f8107640a3e, 0x0005d74e34813ec1, 0x000000000008d260}, {0x00078cdfc58ed763, 0x000f72a544cd81e6, 0x000e167259300b75, 0x00057bacec18a7f0, 0x000511b882d69e61, 0x000f86563a555fc9, 0x00096e4305a4dd04, 0x000000000001d0fd}}, + {{0x00008da90a96090a, 0x00032f04145e8229, 0x000a916fb6ff9132, 0x000ba4e12aa299fb, 0x000991b3b5179ffc, 0x00081c747cc5ec24, 0x0003eb9edcd4616f, 0x0000000000077a88}, {0x000a4909883002a4, 0x0008b9b0bab581a7, 0x000e659d0317cb87, 0x000d81e438a9d43f, 0x000e25ca8b3cfe8c, 0x000bc720cf40e2b5, 0x0006a34254030067, 0x000000000006b244}}, + {{0x00099b58a43c6d42, 0x0005180e1cd16205, 0x000e96620312fe6d, 0x00019d509ddce071, 0x000e70c4b03267a0, 0x0003ba57e52573e3, 0x0004f716d253e14d, 0x0000000000016250}, {0x0003e944594baca0, 0x00013a237bbf8f9b, 0x000a642b05f4171f, 0x000531a1f384daed, 0x0003981251654b13, 0x0002dccc139067f3, 0x0007b5e98fb14167, 0x00000000000a75e2}}, + {{0x0009f542630002ea, 0x00044e65245ce93e, 0x00012350ea59da7e, 0x000c121bad2c8070, 0x0002060fcf245677, 0x00078cccac52dec3, 0x0006fb78d070675d, 0x000000000001bc8b}, {0x000ac9684403d046, 0x000b5c5cb86bea72, 0x00053d522dc955a1, 0x000cdf2c92a70d83, 0x0001f53cd2a1fbb7, 0x0004f11395a9ff1f, 0x0009f1fdbe6b7a98, 0x00000000000a470a}}, + }, + { + /* digit=3 [{1,2,3,..,}]*([2^12]*G) */ + {{0x0006766eb19e084c, 0x00028eb06571b5db, 0x000430cbda13e4c9, 0x000966726eed225a, 0x00046100e387a185, 0x0006298d18d9e56e, 0x000ad0470506b9dc, 0x00000000000f3350}, {0x0009595e79f27f3a, 0x0006683eb62a798b, 0x000ae3d2069c14b3, 0x000e880e1bd4a82e, 0x000fcaf3b3fcb089, 0x000ffd65cd4d1e70, 0x000ebbd0b1ec6395, 0x000000000009b184}}, + {{0x000d72326a677bdc, 0x0001bd4277730e1a, 0x0004e8c2adc8ef98, 0x00046b099f1867d4, 0x0002602dd4cc6b07, 0x0000811201ec73d7, 0x000f2d27fae51538, 0x000000000002f8b2}, {0x000e28e4b1971c05, 0x0001bb924af64246, 0x0005d0fd898e9387, 0x000e9ae068565acc, 0x0005a9a4f1464e88, 0x00093f7348a3dbd2, 0x0003bcdb4a3e483f, 0x000000000008f1d3}}, + {{0x000da9f02128c46a, 0x00049d1de964bafd, 0x0007f571d595c1ce, 0x00055af0de9eb074, 0x0005a60289bfbc4b, 0x000392c619f11b99, 0x0004fc3e59000c52, 0x000000000005ccff}, {0x000c017748720be7, 0x00064b28b306ba1a, 0x0007e101bd3e41b0, 0x000542ce3f824faa, 0x00022f52b71c59b0, 0x000c6d26370f097b, 0x000e5b4483b72604, 0x0000000000034d93}}, + {{0x000b2e0b9f0415b8, 0x000c7721bb8359b7, 0x000a5f46c16031df, 0x000a789348538714, 0x0007af598c4f9cc9, 0x0006f27c878b604a, 0x000ba5d370375e47, 0x000000000000b15c}, {0x00021b9613cec089, 0x000662bcfd9a4c03, 0x000e3ea0c45f94ee, 0x0006464a211b19f3, 0x00019990b504b05a, 0x0004951d3ce059d4, 0x0007b0011c5f87d9, 0x0000000000000d9c}}, + {{0x0002533a1c8fbed1, 0x000ce64e84c28804, 0x000338cbe4f167c9, 0x000d9ed9fbf23cc9, 0x000f5b93118bb77e, 0x0006255cf155fd45, 0x0008941e9f6d7d9c, 0x00000000000c4f64}, {0x00008205c725e2b1, 0x000154bc3a502a87, 0x00030c3fbf39b6ac, 0x0005548d3c862428, 0x0004030f713cc7df, 0x000785cbf9dbfc08, 0x000637f3623326ad, 0x00000000000dd3ad}}, + {{0x00053a3eba12bfa4, 0x000ced8b8b37a274, 0x000ff25533a7ef36, 0x000684bd17d58a93, 0x0002032fda408ac7, 0x0004b49645f9557e, 0x0001097fe128e6ed, 0x00000000000ed02e}, {0x000f765f56c35dab, 0x0008c0052d88eb68, 0x000256931b154329, 0x00010798446a4f6a, 0x0007c99ad35fbf46, 0x0000906073bc4391, 0x0008aada18234dda, 0x000000000005164f}}, + {{0x000f715095892612, 0x0004e02c16c7865e, 0x0009e82bb73222b5, 0x000bbc0795486af0, 0x00070427332d3abe, 0x0005d3cabad858cc, 0x00019c9a1d4b6aa3, 0x00000000000e208c}, {0x000d5b54420318c5, 0x0000afcc14276eea, 0x0008e6c4a86b5358, 0x0007cb4e7706b5bf, 0x000e479e2c750027, 0x0007ad688c01ed42, 0x000626ff1759604d, 0x00000000000c045f}}, + {{0x000e0b227c3a04b7, 0x00029f365419f1ee, 0x00001db5dec2705b, 0x000c165c41880aa9, 0x0007f9712fbd8a91, 0x000c556783eb27a9, 0x0009cfa6587aec76, 0x0000000000002cd7}, {0x000e78bc85d2b5fc, 0x000fefc878f9c549, 0x000d411713959cf3, 0x00084d8caf6df5e8, 0x0002aabcde7509a1, 0x000de597ad32bf23, 0x000858f601d0de03, 0x00000000000c5da2}}, + }, + { + /* digit=4 [{1,2,3,..,}]*([2^16]*G) */ + {{0x000f068a28ea9470, 0x0008bb6029961859, 0x0007d86ade910602, 0x000693b4df3e5b1b, 0x0008a0c3e35782db, 0x000b2f577b513148, 0x000cc3bfb01ff3cc, 0x0000000000027a72}, {0x0000fdf0e7cd346e, 0x000f626170927fbd, 0x000aa1bbda6cc535, 0x0006a634c872d772, 0x0000b14d9c9f0bec, 0x0006c7778a0a7cc9, 0x000a4c8a32d2c44f, 0x000000000003b889}}, + {{0x000f462aea173d82, 0x000a4860ef793767, 0x000a7a5856850902, 0x00083662ee7f523d, 0x000f54122af0322c, 0x000bb2d8058ccd95, 0x0005777454880c2b, 0x0000000000086d8d}, {0x000038487c4c8fb9, 0x00042d5a3057c6dd, 0x000955643c37ff31, 0x000c99ec3c512f97, 0x0006556d891e26aa, 0x0009f6112c3eac03, 0x0007e9866c3aa7bf, 0x00000000000c144f}}, + {{0x00000b161de71555, 0x000aead0d24c7983, 0x0006a55d94bbb854, 0x00034ff7655aa29d, 0x00057a5e217ea551, 0x00021b295a3d1038, 0x00036dfbb9eeb53a, 0x00000000000c84a1}, {0x000aac3258d9db81, 0x000087579398db29, 0x000fa470f6fa27aa, 0x0002e1e464522581, 0x0005479d8f2c99b3, 0x0000b80ef99d5495, 0x00050bc2a8a6a193, 0x00000000000656c8}}, + {{0x000f81f2532800c4, 0x00045171898aa3c4, 0x000ea2712f9cc33b, 0x000835ffdb2c1bad, 0x0001591f5aafbc0e, 0x000272c6a4ee3028, 0x00068afd71de3bcc, 0x000000000006e93f}, {0x000145dbf5847f9b, 0x000bc35ee08038de, 0x0001b04c30c2d081, 0x0007ff5957b2ff76, 0x000e5ec029c049f3, 0x000324c12315d8e7, 0x00057833230602ef, 0x00000000000966b2}}, + {{0x0003e43e11c6c113, 0x0002e86283e21e81, 0x0009c3e50b313030, 0x0009b1bb9784a9a5, 0x000ea0c0acb57d02, 0x0007c682b7c7798b, 0x0002b041241c716d, 0x000000000001d33c}, {0x00079a15b39e351d, 0x000dc5d469ca181c, 0x000ce825406e72f2, 0x0004cc2a13cf4ce5, 0x00069e3ce2793d05, 0x0004beafc13bd216, 0x00087e01bc70e68a, 0x000000000008aba0}}, + {{0x0009cf16a3c4418b, 0x0005884aa863e012, 0x00089c47322b55de, 0x0003206b5c399b2c, 0x00073cc109bd553f, 0x000384088775b921, 0x0003cf25c01263fe, 0x00000000000d5f74}, {0x00057c2efedc75c2, 0x000933d69705ce0b, 0x000359bbe99d9a50, 0x000ab1a2626cebe5, 0x000285b1afe80198, 0x0007e6efdaf8320f, 0x000bb6b9c0968ce1, 0x0000000000090215}}, + {{0x00066543cf4fd691, 0x0000d3ee52d8e909, 0x00094816ee49cd7e, 0x00095c61881a757d, 0x0009c13e370735ce, 0x000d2d3f60a8cf9d, 0x000c0de71258d548, 0x000000000000bbe7}, {0x000476d4cb00031d, 0x000bfbd6496e1309, 0x000c1c69b8768cb6, 0x000501358cfdfb53, 0x000b59275b4acbe8, 0x000f722ba655c902, 0x000aad7e0ff05b20, 0x0000000000042b17}}, + {{0x000b76411fcb09e7, 0x00066da643272cd3, 0x0006802b1cc8eac2, 0x0005a7c35b43943a, 0x00084606bbf22386, 0x00059fb9a6ac0158, 0x0001a59660ab7215, 0x000000000003ce2f}, {0x00083d9ad8b4f172, 0x0006e62af29aaa08, 0x00060fa06813a370, 0x00029b744c110388, 0x0001d36c2571ee9f, 0x000b552b7b2a19cf, 0x0003d4b87d88e265, 0x00000000000beb25}}, + }, + { + /* digit=5 [{1,2,3,..,}]*([2^20]*G) */ + {{0x0002b4a5d3a4b643, 0x0000231bdb4829ee, 0x0006d713dc69c388, 0x00042469b6fc64eb, 0x000a15786b8c18c0, 0x00063da4a0dcc15f, 0x0000fb81acdb068e, 0x00000000000dada7}, {0x0008c1ca45ab6321, 0x0009146361afe98f, 0x0001d88fcfcf648c, 0x000b61b615694e72, 0x0001872951d2f389, 0x0003badc77036def, 0x0008d340fd3bdad9, 0x00000000000380a6}}, + {{0x000d66b2c4fdf4c0, 0x0007ac5cf8997090, 0x000a08d2a2626f49, 0x000681e307b254af, 0x000775cd94d78139, 0x000684954cc87fb5, 0x00099190e7027478, 0x0000000000095ceb}, {0x0004649153ee63bb, 0x0006891bbc0ab337, 0x0001f845e221f84c, 0x0003d704b93d45fb, 0x0004f541da1f1cb8, 0x0007ffd10e229902, 0x000b9a3eef7ce14b, 0x00000000000b3fc1}}, + {{0x00067cf1182bf349, 0x0007f23c1f744697, 0x000288faa5d1b184, 0x0002c9d5afd1bfd3, 0x0004f89d76fea4f8, 0x000702f80a3d1e9a, 0x00089d964f705075, 0x00000000000f56c0}, {0x000478c2e0092e1f, 0x00012e205fa8ede0, 0x0002998cd8ea4a27, 0x0001004356d43961, 0x0001fdbbdfde331a, 0x000a00e7158b7890, 0x000a30a52a15979b, 0x0000000000055497}}, + {{0x000d05d101bb2988, 0x00097e5004e8001a, 0x000e96c63ff7fdd5, 0x00050d1bd869f77c, 0x000de7ebea2c405f, 0x0007620baecffa54, 0x000ff43354dc22d8, 0x00000000000b1c01}, {0x000d9286dd26e577, 0x000c2d9370076043, 0x00025722a20b989f, 0x00076273291e5c62, 0x0007f0a7ca55c0de, 0x000592a305cfebd8, 0x000ce4de1162809e, 0x00000000000a78eb}}, + {{0x000153343d6d3c05, 0x000a15562a856338, 0x00041dfd1ca25266, 0x000317409c75b8cc, 0x000124923f80c19f, 0x0005b291e21c7cc3, 0x000b05e63fe47f8f, 0x000000000000dc08}, {0x000a81ce4831b7d3, 0x0001558ae455ea5d, 0x0000f05c04530b31, 0x000c97f3ee3989cc, 0x0005f38759ae42c7, 0x000f46899c1b03af, 0x000c7d1a673c75bc, 0x000000000008d508}}, + {{0x000fc69642676f12, 0x0008d1e9b23b9bca, 0x000626ac6d6d75ba, 0x00000fe59b7721d0, 0x000c9e2f4cebd4cc, 0x0000af70ed5c36f9, 0x000799deca06bac9, 0x00000000000416ee}, {0x000affe525098c8a, 0x000df0d7afe1b4a6, 0x000083fa6f5ecd29, 0x0003d6ee6eaed183, 0x0002496087e011e4, 0x000a3a66e5baf860, 0x000677f833634fb1, 0x0000000000079398}}, + {{0x000c39f67e66a95e, 0x0005b76512d1b46e, 0x0009e5603da596ca, 0x0003aa8025a5f25a, 0x00095cbd2a40d1c7, 0x0008d61c62aba192, 0x0000f3b53cadc3c8, 0x0000000000009829}, {0x000894e8b1d3bb8b, 0x0003a72800ecafd7, 0x0003c62dea4f99fb, 0x00092a7282ba9d23, 0x0002bd5f1bb214bf, 0x0007c6c969062967, 0x000601362f68eba9, 0x000000000007ea9d}}, + {{0x0004d04ff62d3721, 0x00071e141e762de4, 0x000fa0d592d3e0eb, 0x000cde496131447a, 0x0005cb6c2ef746e6, 0x0002f9fd80458a5d, 0x000457b774763453, 0x0000000000016544}, {0x000adb5cae252cf8, 0x0005abfacb4de24e, 0x000e9db72a61c26c, 0x0003220f22d92e51, 0x0006557232589b54, 0x000ddb8bcaa4590f, 0x0007bdc7b6730e01, 0x0000000000069e1e}}, + }, + { + /* digit=6 [{1,2,3,..,}]*([2^24]*G) */ + {{0x000e98d9a0583230, 0x000f77d27d71f312, 0x000823b17edc1a45, 0x000afeb6075b2d00, 0x0006d93a06f7418c, 0x000d001b9c0d691e, 0x0007b9c16a95259d, 0x00000000000026f1}, {0x000b72219cfa1dea, 0x000984c1041afdb4, 0x00056e257be49c48, 0x000efd62c1758e9c, 0x0007e6c3229a8d08, 0x0002d89249cc6f20, 0x000fe7ec69e90208, 0x00000000000f331d}}, + {{0x00061c722c01a99c, 0x000518ec4335f7fe, 0x000df3425d366c49, 0x0006de001141ab62, 0x000c2eca98a13ff9, 0x000fdf648b21acc2, 0x000e8b6154849010, 0x00000000000d1403}, {0x00097be7041b8df0, 0x0008ff1a35f306b8, 0x000b8cedd3e80c1c, 0x0006c077fc9a752f, 0x000e420d48a089c2, 0x000fe2e738d2535f, 0x000f3980ec5ddd52, 0x0000000000071704}}, + {{0x000259172095dfcf, 0x000a020aa15d95a2, 0x000d85bd292d185e, 0x00005caef579e8f8, 0x000b325981bfe2f2, 0x000438be8ad27e38, 0x0000d9087b8284c3, 0x0000000000042236}, {0x00091bc7ac277af8, 0x000bb87cdf5accaa, 0x000de0f7da8c4a28, 0x00040c1891046669, 0x000c9c1578e8a712, 0x00050ffa2eb5a175, 0x000a28bd66910ad1, 0x0000000000011459}}, + {{0x000f920077501dce, 0x00091498808ed4f0, 0x000dc6c59ac5d089, 0x00025f176c6b964a, 0x000bac474261796e, 0x000a460c11aced64, 0x000e48a62470fc29, 0x000000000005e751}, {0x000842d2c145f36f, 0x0007acc00053aac5, 0x000ca1b81e5b854d, 0x000cc2e3f9ca178e, 0x000a0b80d1b0ddac, 0x000642225ad33f34, 0x00061b6a76df9364, 0x00000000000778e7}}, + {{0x0009eb69fa5f1bc6, 0x0008ed30302342c1, 0x000e3ef7d69039f0, 0x0009b575c4630f26, 0x00008d098d745364, 0x0007f5cbc60197fc, 0x000efc9c295d5464, 0x00000000000c0813}, {0x0001c5999be2ce7e, 0x000e7f6e08007370, 0x000b6019bc473a63, 0x000e08e11d9b388e, 0x000a5db61c657af3, 0x000b4dc4d073ec38, 0x00082a9b480cb89d, 0x00000000000372fa}}, + {{0x0003179049382c1e, 0x00069dffb03ae77d, 0x0002e9528cdd6bd1, 0x0002521b19fe0db8, 0x000f3d5c7fee4c26, 0x000e68e1e0ec1e54, 0x0008a62856510b05, 0x00000000000dc80b}, {0x000ac17897e6fc5d, 0x0000680a509308c8, 0x0000b4dde3197e47, 0x00012ee28235c538, 0x0008301f9653ca61, 0x0001fcdf8d28a0eb, 0x0005f322b11e26e1, 0x00000000000e4d73}}, + {{0x0002a6a91ebadb85, 0x0006346a2a08bc3b, 0x00020e574ba891b2, 0x00056a2b3df9fbc8, 0x000eb121d51228c5, 0x0003bc86c81d3161, 0x0000d14e27ce0b12, 0x00000000000bfb24}, {0x00072db1b86f039c, 0x0005986edb71958e, 0x00011b5a99c9a865, 0x000c6d8067f5870b, 0x00033fe8e5322f6b, 0x000a7997ff558b88, 0x000b9be9433e2321, 0x0000000000087e53}}, + {{0x00084d7bdfada95f, 0x0008b9c66a32b0d1, 0x0004ace9a2f6c763, 0x000fab721e716f0c, 0x000b96ed74e68c6c, 0x000110c8332c8fcc, 0x000efe475890dd0b, 0x000000000005cb4e}, {0x0003947e05207b63, 0x000d29d7b89a68e0, 0x00001e2e33262bf2, 0x000b55bca7a7d527, 0x000655d04585c3f0, 0x00057acc5a6e56a0, 0x00039e818e221c42, 0x00000000000fcb8d}}, + }, + { + /* digit=7 [{1,2,3,..,}]*([2^28]*G) */ + {{0x0007398749666d45, 0x0009c0a74da828f4, 0x00001ff782080bc1, 0x00026bb57c2f5ad1, 0x0002845d45e4896c, 0x0009a7d36981e2f7, 0x000e8fca152b877e, 0x000000000007b582}, {0x000b1649b1810a70, 0x000c3ea3b9bd9987, 0x000d4cd2bb2df2fb, 0x000fc5d5748a6550, 0x0007622665eed346, 0x000f16e277ac2f21, 0x000dc8bb5efe7fb6, 0x00000000000644c9}}, + {{0x000db9a336c7d7d8, 0x0003598d0898164f, 0x00065860354f4784, 0x00018287cfc13dbd, 0x000c8655a658651b, 0x000c91b712d606e4, 0x000090ba64d3c563, 0x00000000000b82a5}, {0x000726397fcaaf5f, 0x0006c2d1dff024ae, 0x00092238373e43a8, 0x000ee6b0ea1fe022, 0x000cd5c1273c1ac2, 0x000e603e7c100b60, 0x000dfb4496084cea, 0x0000000000077c2f}}, + {{0x00064d07c56a20fe, 0x00000a93fec079c4, 0x000155e36a436889, 0x00045a5cce5662fd, 0x000f83a9a4a9c00b, 0x000bbeb632e8a0a7, 0x00080f6e0cdebbc0, 0x0000000000063ccd}, {0x0008f36be2f62c1e, 0x00061fc10fa07d22, 0x000b3e653f03a3be, 0x0009cc66bf53af92, 0x0000f10bb6c9fda6, 0x0007625e1474b744, 0x0003cbcda9db3b1e, 0x000000000001dc7f}}, + {{0x0006fec7b896d97b, 0x0007de8e32259b22, 0x00051ccb0af3cd54, 0x000a4219f42edba4, 0x000d0d411d4df147, 0x000014bb46d4bc00, 0x00066fa1a13a2770, 0x00000000000fa101}, {0x000b6039e0c4cc34, 0x000d8b2a1dfa62b6, 0x000ae98992614f2f, 0x000a3a2f88c7359e, 0x0008347726a08409, 0x000507bb9071f383, 0x000167d18a551c27, 0x00000000000b359b}}, + {{0x000fae4c55d4b2c3, 0x0000aeaaaf45fd46, 0x000aa7e37459675f, 0x0009b673fe123f1e, 0x000dd8fd0129989b, 0x0004982a4e2ca56d, 0x000ec777d6d0cd62, 0x0000000000071e1f}, {0x0001c6bdd9bc3a7f, 0x000043e9a049f5c5, 0x0006deb929a38a20, 0x00008e24fed8f86f, 0x000ce199e8dbac2b, 0x0009cc964a1d1357, 0x00063b7cf06ec8e9, 0x00000000000d85ec}}, + {{0x000ba68a3fc0bcb2, 0x0004e7d111c66c1b, 0x000d9aa66fbcd347, 0x000730c6db857e9e, 0x0009f4b46d124cd8, 0x0008472dc3c9c03e, 0x0001bbd42f0242a7, 0x0000000000026084}, {0x000aac1b65a94c0d, 0x000ea6332b11a21d, 0x000acbe9385d6783, 0x00028eee7e8944ac, 0x00005ab28372402f, 0x0005e1ff33d1bab5, 0x0007296944e82cad, 0x00000000000e8c75}}, + {{0x00058e168fe9a81d, 0x00043a151dcbb9f9, 0x0002eed94828803a, 0x000fc00604d46e1d, 0x000572f3e28c947a, 0x000b1cd1dc3c9d57, 0x000a45ce4c1cbd14, 0x00000000000f80de}, {0x000d8f65d998669e, 0x0003c50920f39bce, 0x000b6be78ee5193f, 0x0008ba13f798e332, 0x0006c5edde471997, 0x000714a1e1294aaa, 0x00003c280002c2be, 0x00000000000f2126}}, + {{0x000493dde1b54616, 0x0002ea44f6ef79f3, 0x000c2b67fffeca1c, 0x0004ed80eaf66728, 0x0008181514a2cb0e, 0x0002927ea2bf485f, 0x00064574670e180a, 0x0000000000012c14}, {0x000339b9a314b3a8, 0x000724068c073875, 0x0004212e0016a517, 0x000651d698b28177, 0x00096da14fa8391b, 0x000a578b1f310d16, 0x000ad7a089be6bd8, 0x0000000000044389}}, + }, + { + /* digit=8 [{1,2,3,..,}]*([2^32]*G) */ + {{0x00024e7304503422, 0x000f0ba86aec16bc, 0x0007f0cf87c57f69, 0x000ff0789df2f808, 0x000d97a773d58978, 0x0003f35f685750cf, 0x0008c9806bb730fc, 0x00000000000fed86}, {0x0006b0ff06192aaf, 0x000eadc0fcde080e, 0x00055bc2901e7a1e, 0x0007d028d3ad6cd9, 0x000997293550fefb, 0x0005cfbba5c652b5, 0x000d2232e12942ed, 0x0000000000098800}}, + {{0x000418b23a7be4e3, 0x000cb162cdf33f48, 0x000c8d04be100c6b, 0x000d114454eb977c, 0x0008dea38a674478, 0x00035728a8ce403d, 0x000504d459633b74, 0x00000000000a63b0}, {0x00010a5f9fdcafdf, 0x000c40c74066a938, 0x000e6c61b766c71e, 0x000b588a99690ede, 0x000c3ad775623398, 0x000bb60ee4949517, 0x000becf9824f09cb, 0x0000000000085660}}, + {{0x00083bb80ede991f, 0x000a02daddb11952, 0x000f09f6c4b181d6, 0x000e82721a6aa89b, 0x0007467506deb73c, 0x0008d8daa1091958, 0x000dfd0927724c42, 0x000000000007c17a}, {0x0009a9bb30e43182, 0x0003a8518dab18e9, 0x000594c3465b3913, 0x000c37f89e7a3983, 0x0008e273f6f35943, 0x000143d228e63f5e, 0x00028ec6d0352b83, 0x00000000000ebd16}}, + {{0x000731dadf48f7e1, 0x000a14074ee26b83, 0x00088243bc9a29c8, 0x000d53972cecb4c8, 0x00079a7dd9c4aa01, 0x000c787cf4b0cf12, 0x00053f3e3e3f165b, 0x00000000000942de}, {0x000bfa5d149fa2b1, 0x000010cc6971941d, 0x0007bdd5c6a1acca, 0x000c1e292314a097, 0x000614a1adcb9fed, 0x00062b86a7547d22, 0x000b7d405561a486, 0x00000000000f5480}}, + {{0x000be69f3af05d97, 0x00082c4e59f2ff48, 0x000865d4a01ec6bd, 0x0000d824464bbbbd, 0x000016f9540dfbc8, 0x000595b5d3bacfa2, 0x00080d1954efb613, 0x000000000007a5cd}, {0x0009c6dbd9d7e6b6, 0x000ac926a54cf784, 0x000f3366624e7b07, 0x000167ccb5c8d4c7, 0x000ff9a21ce20677, 0x0002df8cdc994d22, 0x0009083a25ff6b42, 0x00000000000f68e9}}, + {{0x000a093607905265, 0x000ede544b89ab7b, 0x00017731e314dedd, 0x0000da69a73104b6, 0x00067274b105a6a9, 0x000c61bb65c26021, 0x0005068f9705cf60, 0x000000000003c4d1}, {0x000ed5677f9dc5c6, 0x00020ab5a27accb8, 0x000e0bb2ed27cc25, 0x00036d15a36afae3, 0x00095c455916e68f, 0x000b5d1fa79004b1, 0x00048916ffe6249b, 0x0000000000049338}}, + {{0x000a7603914a9a59, 0x0000b941be86e102, 0x0001a6f35b551149, 0x00095b469d75ec8a, 0x0003db0d4374658f, 0x000fc77053fa79d5, 0x00012885da635c6a, 0x000000000008e7e8}, {0x000f4285c3e56baf, 0x00002558cfa8eed1, 0x0000effdf411ca89, 0x00098b96a32e8849, 0x000e3c45ce1a104f, 0x00085de0268237ef, 0x000de35c820dc22d, 0x000000000007459e}}, + {{0x000a78ec870a31bb, 0x00036923d0b44369, 0x0005db7ea085ecd1, 0x000be009cf5b87e5, 0x000d1d61103d1656, 0x00065239313a37d0, 0x000ed81d705880fb, 0x00000000000ed712}, {0x000ff1a5976c303e, 0x0006f15ad02e6160, 0x00077114865ad858, 0x000376cba2b3ffe8, 0x000f9745443c56aa, 0x000903660c3be2b2, 0x00092d47c8a870eb, 0x000000000006c6c1}}, + }, + { + /* digit=9 [{1,2,3,..,}]*([2^36]*G) */ + {{0x000b550138d02bd3, 0x00038148bd39cbc2, 0x000f6b4c6038c07a, 0x000fbe2ce5484157, 0x000c87fdde9ff397, 0x000e9c179441e5c2, 0x000c716366b49ffe, 0x000000000002938d}, {0x0008a64bcbf3adf9, 0x000d026d450f9f8a, 0x00015da756f71781, 0x000bf4d298fd8771, 0x0007544768b65f68, 0x000491267e86df04, 0x00071a40b69a32f8, 0x00000000000f917c}}, + {{0x0007f3a58f523dab, 0x000a7a66c70349a5, 0x000f8ae356d6f09f, 0x0003e96b5ab54115, 0x000c7c57d123dee3, 0x000d6ad37d068929, 0x0001780839a208f1, 0x00000000000123f8}, {0x000f3c2b5c9dfc15, 0x000f4b3e5e52449d, 0x00055ba373af8955, 0x0000ab7389f2dd3e, 0x0005890bba6f513a, 0x00066bf093197a14, 0x00072261add75b6f, 0x000000000004eef1}}, + {{0x000fbdf154b15bac, 0x00063810b6ab3193, 0x0006da8c3809a3ef, 0x00038dd898977511, 0x000e7a336c9a3cf8, 0x0006f89c03e391e4, 0x000e227014833717, 0x00000000000bc4f1}, {0x000e7d4400e0ab41, 0x0006b32b104f92b2, 0x000d1a7a3b67e3fa, 0x0000437bf178ac12, 0x0005c99370d5b831, 0x0002b93a8722299f, 0x0007190a493cf033, 0x00000000000a420f}}, + {{0x00046acf9a0ee15c, 0x0006a21feb7fb87b, 0x000579369777bef5, 0x000557624b04e704, 0x0006342cb0ad03a6, 0x0004f64262531f18, 0x0003ea088c4d54a2, 0x0000000000006a87}, {0x000f1f11e0fca837, 0x000d5dbe0253ef23, 0x000bcfdbd73eb554, 0x000368173e65902b, 0x0002ccbfa504eaaf, 0x0000e163e71f1fab, 0x000f3bb7b845224b, 0x000000000003c779}}, + {{0x000cf36036019ecd, 0x00029cd7b3c4286b, 0x0005e1ca08cbdeb9, 0x000bcbd24ef5c386, 0x000ce579c309eb66, 0x000f6c9007edcc21, 0x000c2c7b19d49116, 0x00000000000b6317}, {0x000aad793c4e52a3, 0x000e7554ba553558, 0x0002315e3b514170, 0x000e33bffda4032e, 0x00082306675c3d1f, 0x0000c91e75dfec47, 0x000879be59305e00, 0x0000000000025a6b}}, + {{0x0002c6fd041a12fc, 0x0006aa35802f5d21, 0x0000c3d459456256, 0x000991d472b9d211, 0x0006a2f8e875261a, 0x0009b6d63d81a1ed, 0x000758942f213a69, 0x000000000000ae57}, {0x00067bfe08ea2ebd, 0x0007061191c82b48, 0x0000611a48f73652, 0x0003e86525112224, 0x000d30dabb91abe2, 0x000d2742466dd967, 0x00005077650c597e, 0x00000000000ab25a}}, + {{0x000abb01ee5e0194, 0x000bca624ab366b4, 0x0009dc413b0af513, 0x0009c4273aa694c3, 0x0009779288abe822, 0x000575e0e0cc3102, 0x00003ef8eff30f57, 0x000000000007d528}, {0x00093a51fb5fbbe1, 0x0002f32d87e548f1, 0x0004001c13dfb44c, 0x000b8dd16c6e6274, 0x000c2c140452aa2c, 0x0003031b1add098a, 0x000543d25f285d2d, 0x0000000000075b59}}, + {{0x00032a5061a42b94, 0x000dc520b0bb8a42, 0x000466f1305a432b, 0x000c73a73c239760, 0x0009783aabba85c1, 0x0004631556e4dec4, 0x00017b69f0c69bb0, 0x000000000009c97b}, {0x000fbbef3e8375b4, 0x000b155af24a9074, 0x000991d9ad3481f7, 0x000283d708671c48, 0x00035fd9001a4034, 0x0002eaf3b200ddab, 0x0006c4e45f28e434, 0x000000000001ba93}}, + }, + { + /* digit=10 [{1,2,3,..,}]*([2^40]*G) */ + {{0x000e8393cd68c757, 0x000b2b083ba6a1e9, 0x0004638d474c7417, 0x0007a21fc82dc041, 0x000a9d3679d89536, 0x0009724c0227be26, 0x000c0fc70f6d6c7e, 0x00000000000f9ebe}, {0x00075bdec21bc5d4, 0x000b029dde03dcdd, 0x000a669d8fc534ff, 0x00090c90f602f4cb, 0x000849722bc4daf5, 0x0009b22b617c5288, 0x000b90a8df99f008, 0x00000000000e59b9}}, + {{0x00015e6442d15d01, 0x000dc6f5775290ef, 0x000cdd79298e58a8, 0x000842778b96c6d8, 0x00022f59350519a1, 0x0007209d6a674f99, 0x000fff5abeeec46b, 0x0000000000047cf5}, {0x0009d3497d146805, 0x000ede24509b7378, 0x000ed2fba1e0b34e, 0x000af595761e8e3f, 0x0008d420a2887f7d, 0x0000ff696ed5cfcb, 0x000c8f365b29eb7a, 0x0000000000099a1a}}, + {{0x000785db50fa1164, 0x000694936c6a0393, 0x0005ce545ed4b2d3, 0x000e8b45714f2c6a, 0x00023f5ac25a03ae, 0x000b33794139bd69, 0x000ba96a2e42bab5, 0x000000000003ff7b}, {0x00034248c56f7e51, 0x00088b61d8643327, 0x0008d647e582cbe4, 0x0000e1472eb77fae, 0x00013b99c6356211, 0x00074c9f23d5b96f, 0x000250956ecf64f6, 0x00000000000ef2ba}}, + {{0x000a8baf84131eb9, 0x000019ee1ec3a29b, 0x000d9f684960ce84, 0x000737588102ac15, 0x0009c08527f432b9, 0x000e3dfbedd296cf, 0x000c4fb74f8145fa, 0x000000000006cd7c}, {0x000debad8e4205ae, 0x00062a0f2fe7a043, 0x00094ce3fc7d23aa, 0x000f2d40eeb90a7f, 0x0000be4de6846e7d, 0x000dd06bce2f46e2, 0x0009f6cd28feba3f, 0x00000000000e6d6d}}, + {{0x0002283f4c1e03dc, 0x000bc246ffcb6b34, 0x000a382150ba305c, 0x0003ae2250e66766, 0x0000924ce5fab4b4, 0x000d8c77695c1b5f, 0x0009d02555795beb, 0x00000000000acd9d}, {0x000cf0d26acc1b8f, 0x00088e1d74aa6321, 0x00035822f91490d5, 0x000df2795af56df1, 0x000fb331b6f4df74, 0x00059e13724b10c5, 0x0007f2b0a6df9a65, 0x00000000000c0663}}, + {{0x000c55f77d493f59, 0x000089ad73168775, 0x000791ccc3015317, 0x0006c2d30b3a5f4a, 0x0007c89723d59e94, 0x00031f6077bc4ced, 0x00034179f514a1bd, 0x000000000003a274}, {0x0007950f4645c0c2, 0x000e07eb010278e1, 0x000a3d29cb5ab91d, 0x000760f35be21cba, 0x000b7c793331718d, 0x00030d29eba58160, 0x00003afbc0ce1f8d, 0x00000000000c6f4b}}, + {{0x000986e6462b5c9b, 0x0000cbd8c0867ee8, 0x000db80962770b4f, 0x00012de024593896, 0x000fdef840b687ed, 0x0000b56e13f7d98f, 0x000e8771eee0cb5f, 0x00000000000d8d9b}, {0x000f5c1c38b9eff3, 0x000c1e6b50b5a5f4, 0x000fada267894657, 0x0001bd17cb1f9925, 0x000d4ff11827418b, 0x000042c63607818e, 0x000ae3e630d93a9b, 0x000000000008c779}}, + {{0x0000de60ecec558f, 0x000bb35d474260aa, 0x0007deb342712d19, 0x00015e22e91bf5f3, 0x000cc08b6b1abd6a, 0x000b97de8e366a84, 0x000f29759c122f55, 0x0000000000008a03}, {0x0005b54173576b1f, 0x00000dcc9fca2774, 0x00073c06ae128d8b, 0x00039029b59cd052, 0x00006f5e5bd4deae, 0x000099f4df532ede, 0x00005284fbeeb936, 0x00000000000088d2}}, + }, + { + /* digit=11 [{1,2,3,..,}]*([2^44]*G) */ + {{0x000b3d633d0721cb, 0x000732ba8c78fe5a, 0x00016e2c1c57f816, 0x000f36a2fc2451f3, 0x0008bb91e1e36842, 0x000ead762fc5c955, 0x000556035d1dfcc3, 0x00000000000031e5}, {0x000b4359fe8646d5, 0x000383af0cc803c6, 0x00070f15b8bb97ea, 0x000de0a6ade1d137, 0x000d93b2dcb580c3, 0x000a2214de8c3a5b, 0x00048de3adbd7c90, 0x0000000000011929}}, + {{0x000e8783f9b6f97e, 0x0009b65026296c0d, 0x00086ba77888a60e, 0x00063a460c8bbf8b, 0x00078b2a71206237, 0x0005497e7fa8f5ad, 0x000618fd744bdf08, 0x000000000002ba35}, {0x0004df87b45c7eff, 0x000870cebfe9d444, 0x000c034f12ddb3df, 0x00017a3fcf19627f, 0x0007c2f112616558, 0x000f2c85030ab44f, 0x0000c3bb001c9ddd, 0x0000000000007326}}, + {{0x000e365e9f55b0a4, 0x000aeb08fb116bd9, 0x0002cf623c1f798f, 0x000fc7b6f9549671, 0x000f76bd243c73ae, 0x00009d5a8c0fb886, 0x000049871eacc5ce, 0x00000000000e773f}, {0x0001eb3732cb8726, 0x000aa92945c840e9, 0x000022c04533de34, 0x000bc1d0509d7400, 0x00010f1af1754762, 0x0008c160f15cf97f, 0x000f0c1f85569532, 0x000000000003b439}}, + {{0x0004f7c9bedca76f, 0x0006dfa7d1236235, 0x000a7e4930642e7b, 0x0007288beb1282c5, 0x000a07fee8a99ea2, 0x00070fee91c069ef, 0x0006fa5749c7b558, 0x00000000000afcec}, {0x00048441716f41a1, 0x00064a3f8f1b0daf, 0x000b8af2f805e4cc, 0x00029a9b59dc06f1, 0x000b98a92c387533, 0x0002b4662fa8e5f5, 0x0006c66b6f46fd3c, 0x00000000000ec04c}}, + {{0x00054b9f6efe8494, 0x0005eaa16c27a15a, 0x000106292d7b104e, 0x000d193aae87c9d3, 0x0009916d634e7ae2, 0x000a65b4b125ab45, 0x000e2202ded714cf, 0x000000000004e212}, {0x0009494225bd1826, 0x000c097c48a1862b, 0x000bf9e4c3ff8573, 0x000b77b2652f5018, 0x000d078efd386fe8, 0x000cb82991daa602, 0x00062635885364db, 0x00000000000b8240}}, + {{0x000f5a3697f1c244, 0x0000e0430af76c1b, 0x000f0e87f66ce63d, 0x000905f12e919108, 0x000012db9e14e1a7, 0x000baeeac1c689b7, 0x0003196bdd3dc90a, 0x00000000000504f0}, {0x000e18cdf6373284, 0x0003c874afd60b16, 0x000a978150da10ac, 0x000eee1ebf4aab2c, 0x000c1aa49fe60d33, 0x000217cfda3631ca, 0x000e770d8340fbf2, 0x00000000000423b7}}, + {{0x000b3813851ecc4b, 0x0001df8c07372826, 0x000ea9f99e2d35f1, 0x000faf1a6305a291, 0x0007f3e0f93d2b97, 0x000aeb8c15bc61f6, 0x00024b7238583cd7, 0x0000000000039f5f}, {0x0002b746db300ac6, 0x000a11cc8b467be6, 0x000e46954d17b55e, 0x0005f95ba2641ae4, 0x0002ce9d565b1b9a, 0x000eedc6287a0c36, 0x00003d07fb51b2e1, 0x00000000000a9739}}, + {{0x000d77fe5e566bbd, 0x0001978a53b5a370, 0x00081dca6fe505a1, 0x000f427019a6f8e7, 0x0006dc3ad0ba3520, 0x000745b7cde6fcad, 0x0002dcfec96e4f79, 0x00000000000b133f}, {0x000924a225ecf745, 0x0000c50088a2b006, 0x000c145291ebead7, 0x00032ff23ae4b9d3, 0x000e85246712f213, 0x0000b515e8cbc659, 0x0008b727fa9c8df5, 0x00000000000494ac}}, + }, + { + /* digit=12 [{1,2,3,..,}]*([2^48]*G) */ + {{0x000ff6bf222c5c9c, 0x000322986475308d, 0x000309c5ef927cbf, 0x000d6b4216ab9acb, 0x0007be12d76a038c, 0x000347bdb3df9b7b, 0x00048913f4d9785f, 0x0000000000013e94}, {0x000466717b5c75f3, 0x0000a5f8e796eab2, 0x000d6af2aad3919a, 0x0005d8ad10740b88, 0x000b5337dee6254b, 0x000f02247c38b8ef, 0x000c4cf688c2e194, 0x000000000006c25b}}, + {{0x000272cd3b35df41, 0x000d936c9dbbec27, 0x00026ae70fa619c9, 0x0008db696a8f9f19, 0x00056b01e6bc1ab3, 0x000fc4adae031d23, 0x0004e410466ae68a, 0x00000000000ed9c4}, {0x0005ea962547af52, 0x000cb61272c12a27, 0x000f929706a5a2ac, 0x0007a910ecc49eb8, 0x000ccbe84d5cf4c4, 0x000e497d7eb95dfe, 0x000ce443f3b71c8e, 0x000000000004c6fe}}, + {{0x0002d9d94d551889, 0x0006182e5d818574, 0x000101531df0c231, 0x00044261daa2e22b, 0x0000e46f32576b02, 0x00069db38b86a358, 0x00027eacf145bd76, 0x000000000004df27}, {0x000d2ba752047cd9, 0x000c203d9391e25b, 0x0007c9592434b2d4, 0x0007845ec38fa9ac, 0x000a265ad6bbefb7, 0x00054a1b2dd40660, 0x000499a22d988618, 0x00000000000737ea}}, + {{0x000ef1248ca55f15, 0x00028e323ed0c422, 0x0001736a7d35b006, 0x0002f06e8d68e4e9, 0x000ad0742e5d9c09, 0x000d3df92d8f5555, 0x000eabe2d175bf00, 0x000000000004f71a}, {0x000a6a143a42cf09, 0x000c6d1762d7229e, 0x000840a2cbe90735, 0x000cb4c6281f2a74, 0x00003603e53a2caa, 0x000fecf29635ba47, 0x00036194a9811d49, 0x00000000000466bf}}, + {{0x0009fc85048451fa, 0x000fd4737236d065, 0x0005b89cfa755eca, 0x00070306da6e06f0, 0x0006f3838f569da9, 0x00043188730279bd, 0x0005d0fb328c8b94, 0x00000000000be90e}, {0x000859016f87df1b, 0x000843334a6711d3, 0x00078d74e5890358, 0x0007b6e38904b738, 0x000296b588a53493, 0x000577ae391e227c, 0x000c7da599b21544, 0x00000000000214a0}}, + {{0x000fcc62fe159c27, 0x000c9e63fbb0fb71, 0x0007ab3cd12c8947, 0x00030677afd4bf85, 0x000dfd37120d5cea, 0x000d718a74494e39, 0x0005fb8c572c7249, 0x000000000005fa30}, {0x0005abf2e0c1181d, 0x00074751c217ee1b, 0x0005917c5a26a520, 0x0006e6efe7a64872, 0x0000f53b0e479a99, 0x0005fd5931a4f6d1, 0x0009ee651390ecd1, 0x00000000000739ee}}, + {{0x000e27c9677a2151, 0x000ef0b6d37446aa, 0x0007f13e8f2bd87e, 0x000c94fa109847d5, 0x000044944c7712bb, 0x0005a31b874c0d53, 0x0005920d280b18eb, 0x000000000007ef42}, {0x000c07ddca373d80, 0x0004ef11030c77be, 0x000075bee798eeea, 0x0002d013d22f1b04, 0x000cd93ee54dd5e0, 0x00041d4b1b6d66c9, 0x0006ed80b4154faa, 0x00000000000acf8f}}, + {{0x0004ca485f1804e7, 0x000ad0f05710ab2a, 0x00002b0d41da0420, 0x000a67a46b8d0e2d, 0x000c698b78cc137d, 0x0008a9393454b89f, 0x000e69f2a6e1de25, 0x0000000000016488}, {0x000b96b954a8287c, 0x0003d7c6c5501c10, 0x000fb63222050457, 0x000e30e92f152478, 0x000327e70a0a4b9d, 0x00014936309d4ca9, 0x0001379c8b16340d, 0x00000000000ce642}}, + }, + { + /* digit=13 [{1,2,3,..,}]*([2^52]*G) */ + {{0x0003c432161be476, 0x000c0f8a8499b505, 0x000715248b87d78c, 0x0001b1d515e1328e, 0x000ba941e788b85e, 0x0005dd8d888a2636, 0x000350a045241d2b, 0x00000000000332f0}, {0x0008eeabc026bdc0, 0x000f796c4b204e16, 0x000ce54b1f342310, 0x0003fc6d00b602a1, 0x000e89aa3b796fc3, 0x000d4dd0007a914e, 0x00095635353eb7a4, 0x00000000000673e8}}, + {{0x00027e0f6ecb7465, 0x00040f36e83987c1, 0x0002e0c806d929c2, 0x00007464efc5b0d5, 0x000ad316c43436ab, 0x000ccf839b211e59, 0x000a072515ec9f16, 0x0000000000003dc6}, {0x000ec4a69e8d5661, 0x00017842b727527c, 0x00065c4526d40261, 0x000711ccef5255e9, 0x00075108cb92967d, 0x0001b9740cd3bfb0, 0x000308e50c0d8aec, 0x000000000001a9e6}}, + {{0x0002078cea733c1a, 0x0005b283eec25eca, 0x00036d44d991d5b4, 0x00083b827ad302e8, 0x0002bde3fdd0269a, 0x00030b3a6225f2f1, 0x000043046fcf3801, 0x00000000000c3ed9}, {0x00066dedc2439ae4, 0x000eff870f14cae6, 0x000680b39cf67cb8, 0x000d5f4847be7732, 0x0003d0ed73a0f3e1, 0x000b3babba949822, 0x000f706933ccf014, 0x0000000000037f08}}, + {{0x000ba839c8cf3524, 0x000ed1afa6aa5579, 0x000f9ef0d2ddddd9, 0x000920b5d36da502, 0x0009291e774f07fb, 0x0000d87a8144d51f, 0x0001a026c2c134f4, 0x00000000000a932f}, {0x0003544f31a7b78f, 0x0009935bb2a42294, 0x000ea47969f6664d, 0x000cabfaa1838ed8, 0x000fe2855f1f5c40, 0x000525934ea0c05f, 0x000fd4931ebb02fd, 0x0000000000016246}}, + {{0x000bd623cb7fe067, 0x00038ecfabd26775, 0x0008c3832c0a3527, 0x00064ccfe2691ca7, 0x00058347566acf4e, 0x000b8c733e3889e2, 0x000f5748da354885, 0x000000000009d9c9}, {0x000592cc5e5c9fb4, 0x0000e8c26a8d609c, 0x0000459f168ec210, 0x000a70a3f8db9f92, 0x0000e758213e181c, 0x000b2653e25aa645, 0x00003aa4898f9169, 0x00000000000ccd83}}, + {{0x000fac468f3d59db, 0x00099517d13b0e90, 0x000cf2490366b8de, 0x00026866d752aaab, 0x0001cab026676e2e, 0x0003163c395da9fa, 0x0003c9be8d91ad42, 0x00000000000a6995}, {0x0007a57ea4ee9030, 0x0006ef728b1d231f, 0x00013aa7ae93d8f6, 0x00000a82e75d9c30, 0x000e3ad09def97a9, 0x0008a2be8136c6b1, 0x0004474bab14a6dd, 0x0000000000025cf1}}, + {{0x000e1d30e97eb41c, 0x000b411b59f3da92, 0x00020acfcec12d54, 0x000f0a1936595900, 0x000cca0b1b0e5cad, 0x000274a5e8fef04f, 0x00027ebcb4d9fb0c, 0x00000000000ba784}, {0x0008e71ae4477c5e, 0x000fbc49f0bc478a, 0x000ac96d890c62e2, 0x0009e583f796b820, 0x000b17964262200e, 0x000db00395bbea92, 0x0002ba86b3c15756, 0x00000000000ead48}}, + {{0x0007642e08638534, 0x000b5cd92906c650, 0x000ae6db49a06b5d, 0x00029781fdc19156, 0x000c269d611e0d69, 0x00065b00a45d01a8, 0x000388e7bd1e7096, 0x000000000009bcaa}, {0x0007591cdd6ae97d, 0x000ed8f189e87506, 0x000ccf1d10959a9b, 0x000bf16c633b1123, 0x0009d6f1dac8ca65, 0x000c3381dc6adc9c, 0x000d120df2c293a3, 0x000000000008388c}}, + }, + { + /* digit=14 [{1,2,3,..,}]*([2^56]*G) */ + {{0x000653d60a9d872c, 0x0003bffdbd0eb2dc, 0x00061fddaf39f568, 0x000cead35e7a3a29, 0x000c67833028c11a, 0x0004c79764998cf8, 0x000a1c8f3a346b84, 0x00000000000db9b1}, {0x000642a471b701b9, 0x000628735dabc47d, 0x0005300f39a216a2, 0x000dd49d267a01b0, 0x0003ffa20d117c0a, 0x000ab2d4a2b46c91, 0x000080f2acef26d8, 0x000000000003d926}}, + {{0x000ba70a22083784, 0x00084e9d2a98a2f2, 0x00091072e4da23c2, 0x000325dceaf86ae5, 0x00088f161525f399, 0x000211b9d03b17c8, 0x000a48d8ac35c984, 0x000000000009050c}, {0x000bbfa19d5ef891, 0x0006ba818c44b2c9, 0x0005e1b560830da0, 0x000af35f8715b052, 0x00099d8829a9633a, 0x000a820f15463e1b, 0x00075db18df52d84, 0x00000000000d0966}}, + {{0x000ae853a945611a, 0x000bed54e2c7c031, 0x0009b2bf77ae12b1, 0x00005e8e60f7f5a6, 0x0008427483adcb41, 0x000bff383705db30, 0x0006e9f73ba98bf2, 0x000000000000220e}, {0x000231b48f25af77, 0x000e8c01c46b84a3, 0x0004ae472f3bd7a2, 0x000dc0bfa3403e6d, 0x0007d2202896f738, 0x000882e5e098bc68, 0x000b13ec0c217a5f, 0x00000000000a1f4e}}, + {{0x0009208204f520a2, 0x000ed2615c78254f, 0x0002b7a2ee7b484b, 0x0001d771c84a04b5, 0x000fcb9f9e349c3b, 0x0004f7846b6fc203, 0x000248bed65464c6, 0x00000000000eb004}, {0x00040455a574f8cd, 0x0003f5c017726a4e, 0x000b0a7d5066828d, 0x0006666aceb0e3ac, 0x0008fc046f0ab78c, 0x000aa959518d3c18, 0x0004e87f3e2f301b, 0x000000000008a284}}, + {{0x0008a96d207a2eb2, 0x000e3c899eba3614, 0x0002ec9689146ad2, 0x0008629da55568ed, 0x00082b1dc1d9607c, 0x000c001ff6b54722, 0x000b8523232311c9, 0x00000000000488f8}, {0x0000aca655c2cd82, 0x000723867ac9a850, 0x00092fe2557b4773, 0x000c647a74488fff, 0x000fbe0876407398, 0x00019a571f6ed920, 0x000aeb72beddc763, 0x000000000006a244}}, + {{0x0006fe658431a6d6, 0x00064b7da79c5a1c, 0x000fc6b2b6576354, 0x000b7b54aa36d18e, 0x0008ed0e30913481, 0x00093074c6efeaf8, 0x000654eb017bddc9, 0x00000000000529dd}, {0x00089f1ff5fdf666, 0x000fcf5177230c70, 0x000373317732e646, 0x00082d34ca267426, 0x0005adcd1650194d, 0x0007758b7eaeffe1, 0x0008194dcec3d9af, 0x000000000004cc2c}}, + {{0x0003e55601cd21ee, 0x0004794bdc7f4a7b, 0x00080eb7c8f212fa, 0x000dab0d654cb574, 0x00037a49195627e2, 0x000d5d0991d4e1e5, 0x000de7a1ef570c31, 0x00000000000295f3}, {0x000c78902e5817a8, 0x000198681b00d89c, 0x00061b3376c1d033, 0x000e90c6a1b57484, 0x000c1222e5544324, 0x000d53dd044f9324, 0x000ef0e30ba10fff, 0x00000000000b48ee}}, + {{0x0004a14fee68295a, 0x00041a349bb65da9, 0x0006f09eba200d68, 0x000d891c18d37516, 0x000dae8d2bfba6e1, 0x000b330789985aa4, 0x000e948baec9ae31, 0x0000000000098750}, {0x000e6cd5311f8630, 0x0009c0d834bf8a5b, 0x000c536623c88198, 0x000faaa0c51d098d, 0x0002b887a10f0c22, 0x0008bed323240404, 0x00066231f6a61424, 0x000000000001ce0d}}, + }, + { + /* digit=15 [{1,2,3,..,}]*([2^60]*G) */ + {{0x000c2532e44da930, 0x0009eac6dbdf6097, 0x0009dd0474f3e9a9, 0x0004de3dc28e1b18, 0x0002a66477111669, 0x000b08b4b2d039cf, 0x0008cea13bbe6fc5, 0x0000000000010ebd}, {0x0006e345ee3db6d1, 0x000c2cd4720862b2, 0x000fcd0516567c14, 0x0006303929bfa29c, 0x0003818249bfb3f6, 0x0007f7cd97c378e4, 0x000a1676068c8084, 0x00000000000987eb}}, + {{0x000e55f593b72e2d, 0x0008c1238e0971aa, 0x00081a4c08ea4523, 0x00054e7f74c5c514, 0x00028805813b4512, 0x0005e7a6e238b16d, 0x0000bc23987b2892, 0x000000000002a1a1}, {0x000c0e8fa1a83cf0, 0x00010944c784c643, 0x000f1939fa2e366e, 0x000ab2aead457171, 0x000631c042056bf1, 0x0000b8881d5f8f0e, 0x0005a8ac062526a1, 0x00000000000fe550}}, + {{0x000d9597e54c99bb, 0x00076cbbd8575ded, 0x000019092c8277a0, 0x00056202e6b72a58, 0x000e7c024443e5cb, 0x0005368f35738ea0, 0x00044f8ed06170b5, 0x000000000001aeed}, {0x0001a7ce730f30e4, 0x0003a3d90a6b26fd, 0x000480ba5c428a59, 0x00093dcc5612aec9, 0x000f99c1bacc0890, 0x00043eaadc272ef6, 0x00089147db3b43dc, 0x000000000000832d}}, + {{0x0001abaa498687bf, 0x0001bac92ee14ee5, 0x0002ef494748554e, 0x000ca876e7a32661, 0x000a8456c9af29a7, 0x00054326f0f2b7e7, 0x000ec87471824e97, 0x00000000000364d2}, {0x0007a1e32547416e, 0x000f386d8aacd172, 0x000735a9921c3b5f, 0x000fc881d79f7eb2, 0x00040040547d9805, 0x000b4e90b377ac3a, 0x00081bd39215d461, 0x000000000004c5fd}}, + {{0x000bf8e11ac3f3fa, 0x000df9ffe5562f1b, 0x0000905bb11344f5, 0x000f981cbefa3c77, 0x0003667bfd643039, 0x00040e3df2ab0688, 0x0009ca9007a25743, 0x000000000005a3a4}, {0x000a64031de3bcfa, 0x000fd9c7be629ab9, 0x0003fbe0cdc1be9c, 0x000ff39e7380dbb1, 0x00059facc3b8ffe7, 0x000ae422c620bb9b, 0x000c432ddcb8cd31, 0x00000000000d12c3}}, + {{0x00072db65199d9d3, 0x000d703621d34a54, 0x0008bb1d8b92a619, 0x00000f66ca2933b1, 0x0002de1494a2886d, 0x000dcdf82d0a2238, 0x000832b8656c4d0b, 0x00000000000c4a58}, {0x0005f2f5aceb0154, 0x0002fa982c1aa34d, 0x0006b9fce5077d2d, 0x0008de431390c6e9, 0x00018c4fe595fc26, 0x00090d82fea4160e, 0x0009076c427a6367, 0x00000000000fc519}}, + {{0x000775c688e983e3, 0x0003e8c0749464df, 0x000f192d78daad34, 0x000e96904bb2048e, 0x0004d6dc8b606cda, 0x000438bbc6dec959, 0x0005a58d26586954, 0x000000000001b0e9}, {0x0001e04544fa17f3, 0x0005e8189f1141bd, 0x000c131e8abc17dc, 0x00075a227f8ec1ab, 0x0000607e37397757, 0x000793e4e7a12524, 0x000af4afae84e74f, 0x000000000005bf5b}}, + {{0x00039549b6d366af, 0x0000aa10292b850d, 0x000fff80cff6798a, 0x000f18f73ad7a1fd, 0x000c5897b11f7a36, 0x000664c618b2c7b0, 0x00022cf7b9a272cb, 0x000000000008f81e}, {0x000e957ffad5963b, 0x000663b99b210935, 0x000ea3abc7ab4283, 0x000175d001db74bf, 0x00067159cb8a3af1, 0x000de4601526f084, 0x000eac6a3c6e1feb, 0x000000000008c232}}, + }, + { + /* digit=16 [{1,2,3,..,}]*([2^64]*G) */ + {{0x0004c667c92f7888, 0x000aaa54768d9e88, 0x000d1397d0aa7f52, 0x000f203faef6863d, 0x0004bd7471b3774d, 0x0007de2e9f795a03, 0x000cfff0958718b4, 0x00000000000e1160}, {0x0005c7ba07a0ffd6, 0x0005186ded97af9a, 0x0008fa18cb4fab4b, 0x0008920424e84590, 0x0004eecf8b2b0558, 0x00068a6fa3745591, 0x00019fe7d3df1fb9, 0x00000000000bad07}}, + {{0x000b4e8b4136f0e4, 0x000ae2566021f579, 0x000cbf2ef6760188, 0x00047a5d9f51b6a3, 0x000df8efa64634e0, 0x0001f584f0b50d91, 0x00091cc2bbcbb297, 0x000000000000907c}, {0x000fd43610d5f812, 0x0000ca7ebeb0dd65, 0x0008a7b6eef1e9c1, 0x0009a073f5c962ab, 0x00053ff74a2fc04d, 0x000f0749d95c155e, 0x0005d0923c65a53d, 0x0000000000027ae3}}, + {{0x000cd3b33afd62e7, 0x0003c4d37c266037, 0x00042b261375e38f, 0x000a2e928ca9d674, 0x000a79beb236566d, 0x000f801e7a9771c1, 0x000358af6b97a976, 0x0000000000071259}, {0x00004ab4fe03d3c3, 0x000bcc23a5e31cf5, 0x000c506466ff69e3, 0x000233b1911ccf3a, 0x000cfa3b3ace3f3a, 0x0004c5f5c93e4664, 0x0003c04bdc14832d, 0x000000000006abf1}}, + {{0x000832aefb763bd9, 0x000f0d5469c7af17, 0x0000d7bc962f1b04, 0x000ca21a16caa7b0, 0x0002e6cc7f39f881, 0x000378723221de18, 0x00066010d61ab531, 0x00000000000520c9}, {0x000ed27c50cc42c5, 0x000ac145214ccbc9, 0x000ac441f327ce66, 0x00059e30cb1fe6a2, 0x000a3e299fe79fce, 0x000af78cf2e6e77f, 0x0001593a0cf652d5, 0x000000000003e852}}, + {{0x00087e649fa97c56, 0x00029ebbe18b74d8, 0x000417a0e476f2ee, 0x0005a2a0b24f97ba, 0x000262fc61243ddf, 0x00003b73af9a400c, 0x000fde9ad9b0bc6b, 0x00000000000153c8}, {0x0006f3a6ac904b01, 0x000f41b6477d9542, 0x0008ea414bce433e, 0x0007128953069569, 0x000db9e775794428, 0x00005e0d14b5db01, 0x0002f5237edf0bde, 0x0000000000085533}}, + {{0x000e220415482512, 0x000190791dfa1ca2, 0x0007078d88a1aeff, 0x0005b77ccedf4f34, 0x000d5d965c0549f2, 0x0009e672705170cc, 0x00017637d9521bd4, 0x0000000000086a00}, {0x000e53f005b29758, 0x000b5dab44493664, 0x0005df02ede5bef3, 0x0000c3e4c2506ba1, 0x00065ec6f1324366, 0x000ac7691b2fb261, 0x000eb5ccc4221a99, 0x00000000000a576d}}, + {{0x0006b1405a6d6a62, 0x0001e3a17f86cedc, 0x000ffc614b14d0e0, 0x000a3f0927935e2a, 0x000ac04d4fb0a86e, 0x0007694212f43742, 0x00012f32a30bce38, 0x00000000000d8d39}, {0x00091bf3c81523e6, 0x000ed34154b18f9f, 0x0001725c2ac49b1c, 0x000be0f9f7a13c1b, 0x000a531736e526cf, 0x000cfa2c250d50b3, 0x000e5f81849773b2, 0x000000000003ba8a}}, + {{0x0003089126b94126, 0x000854a87307686d, 0x000fced9f497ff98, 0x000042f427ea198a, 0x0001ed259219c1d3, 0x0004850cadc59211, 0x000e8d0abdbd1623, 0x0000000000043b6f}, {0x0004352ff19d9aba, 0x00069d3c79d16450, 0x00090120dc8bc8d6, 0x000535d5a98bf664, 0x000dab2cdb2ba736, 0x000af89eeacb3b7d, 0x00053237d3743ada, 0x000000000000b348}}, + }, + { + /* digit=17 [{1,2,3,..,}]*([2^68]*G) */ + {{0x0006e0a86f5827e3, 0x000e1a68295d9bd5, 0x00032e6b7dcbdf31, 0x000a0cffe0c3df09, 0x000b3cd00a1a8deb, 0x0000f90885b4d037, 0x000ef7e9edc429aa, 0x000000000005847d}, {0x00025d7642f87bf0, 0x000f49739d03ced7, 0x000f63949ff1cd98, 0x00034ff1759060ab, 0x0009a7e94dbbdc3d, 0x000e8b7f3029e9aa, 0x0006f42a3cdfa0f7, 0x00000000000bbd8f}}, + {{0x000178a4d4ad7d87, 0x00040cd8c2ec862e, 0x0003f7b9dc665542, 0x00086d4db83e1ac5, 0x0002bb549bb5e123, 0x00007c0f19a79203, 0x00008eaf81cba628, 0x00000000000abf20}, {0x000001ed540ad4e0, 0x0000a9d7a72302a9, 0x0007110b90df5c50, 0x00024daec9005170, 0x00037193eb3047c3, 0x00015c655408cd0d, 0x000c228f0bcce2d3, 0x00000000000869d6}}, + {{0x000e5354c88a38dc, 0x000717192a26df13, 0x00059c532d5dc6ab, 0x000195620aeab120, 0x00057c2328498938, 0x00079bc39ebe39dd, 0x0000925787970e4f, 0x00000000000143e4}, {0x0006e01b286241f3, 0x0001e5f60303562b, 0x000cf202635ec6b7, 0x000b3a2f1856e619, 0x000bbf77d65c7ec4, 0x00011058fbef7dc4, 0x000945b444d50670, 0x00000000000c33cb}}, + {{0x0000e30b66c01c80, 0x000354b5e753d742, 0x000c1ca06540fdb4, 0x000d75f42fa905c4, 0x000ac33e462b4034, 0x0008de3bb89b85ed, 0x00039131f413c305, 0x000000000006ba75}, {0x00046bc0a659bd1e, 0x000573f50020190e, 0x0006ea7059f1f5a4, 0x000aa10efa2adddf, 0x000e1a6566aa7297, 0x000b4f2143e8ccdb, 0x000ae74dfda07fd7, 0x000000000007cb1c}}, + {{0x0008cb34259001d3, 0x0001d49d1cfcc46a, 0x0008f80188e0d2b9, 0x000d285d2ab0a996, 0x00071df5b6cb8cd5, 0x00003fadd9c5cff6, 0x000c0019f4095a30, 0x00000000000aa463}, {0x0002c2d4a3652c8f, 0x00087f86673b013d, 0x000814cd905b85f4, 0x00075b44cc2cd5ed, 0x0000b0342517b376, 0x000817cd771d5262, 0x0006183af1f31657, 0x0000000000079199}}, + {{0x000066ce8afe9a22, 0x000815600fedffd0, 0x000fa276d4e1e61e, 0x00062674cb08e4e2, 0x00052e9ab52d8f12, 0x0005686adfab0c1f, 0x000e4e090b1ac94c, 0x00000000000bb10d}, {0x000c301d650cbaa8, 0x0003b3383f48546f, 0x00045006d2b6b386, 0x00092fe500058f01, 0x0007f4ce508ac5ab, 0x0007a5166f03c7c9, 0x000d66525ac2cea7, 0x00000000000996ef}}, + {{0x0002ee0f55870dbe, 0x000f522daed9eb3c, 0x0008bd7a619de4ba, 0x0007daa46ec27562, 0x0004df0f7a076593, 0x0006036c14c58eeb, 0x000aa5eb972393fa, 0x000000000004d17f}, {0x0005a44cb91701e0, 0x000a3de29c31e831, 0x0004699fe7072908, 0x000952e2c563ddbf, 0x000e5d6b0b57ac00, 0x000c4728203a767f, 0x000d56e858fdf3a6, 0x00000000000a5213}}, + {{0x000b5ed7b705b7ba, 0x0002631796c22d0c, 0x00099ed65d60e408, 0x000cb716cb0c301b, 0x0007d1702ca94ff0, 0x000a02a90dd0cdbd, 0x000625470a26c8e1, 0x00000000000d7054}, {0x000218ab60af89bd, 0x000e0fadc2e8673f, 0x0000890003c99982, 0x0005ebb2951a8fe7, 0x000364197a76042b, 0x0009c1b5de4af9d0, 0x0002b1dd2c6bb324, 0x00000000000f0193}}, + }, + { + /* digit=18 [{1,2,3,..,}]*([2^72]*G) */ + {{0x0004ea16a709d85d, 0x000329ded9b0ccd4, 0x0002e9bda4d583bd, 0x000aaf5393937290, 0x0002a438413e94f3, 0x0003c7de32213686, 0x000540449286da37, 0x0000000000029aa1}, {0x0001d592acb2cf64, 0x000ef1bd13e4055d, 0x0004681cce9d2c6b, 0x000558f6bcd0cb2e, 0x000e9e610369d43b, 0x0006e0651f5757c0, 0x0000aa15c80b23c1, 0x0000000000000182}}, + {{0x0004d40f93bd737d, 0x0008c10a8aabc8f9, 0x000d5b1177a72237, 0x000df1a076945a2e, 0x000c03861f02a009, 0x000866a3cf869152, 0x000cbbc405226e9f, 0x000000000008b41e}, {0x000cb1b314f36d3a, 0x000bab5c9ed101e9, 0x0005d83bc9ae2498, 0x000f22e57589279a, 0x000d7fe2d6aa1a9e, 0x00073dbfa83ef607, 0x000066f2da845434, 0x0000000000057879}}, + {{0x000f91047abd3166, 0x000593df98ea92d5, 0x0002ec06cefe6ff8, 0x000d1b5ea6d69771, 0x00097e25ffbcc9cb, 0x000a6280e3f9f231, 0x000713bca8e0567d, 0x0000000000035cda}, {0x000fe70b79caf906, 0x0003946ef321cfa3, 0x000b4a657c54ec43, 0x0003b4b51de40452, 0x000cccd9ba8d37b7, 0x000f7045b5bccf3c, 0x000dcca2ca080dd0, 0x0000000000068cf4}}, + {{0x000d265e2cd7626e, 0x0007e3499b44b9a9, 0x000ad7706fd7669e, 0x0000406c1f517d80, 0x000b82a174e83014, 0x0002c9b3ad4b3c8e, 0x0006f4c8835b13af, 0x0000000000044371}, {0x000fa90f05b33562, 0x000c2336c4f4b966, 0x000385e7d9f805d4, 0x000a2dc828c34d3a, 0x000c34d0d1f76ffb, 0x000a4fbce257a345, 0x00049fbc1eff056c, 0x0000000000007683}}, + {{0x000db1540add72ad, 0x0009e5ebdc79cc93, 0x000baaf845cf7653, 0x000efe79c8b7f7df, 0x000ac469aa5793f6, 0x00033c455b2f1e40, 0x00012a882f0e9434, 0x00000000000aea07}, {0x00030f4a7dec8786, 0x000e9b13578c31bc, 0x0001af154c9f03fd, 0x000a2d854e09133c, 0x000cc6ad853283ac, 0x00004f75cae03943, 0x000c37ce0a619171, 0x00000000000f9838}}, + {{0x0004bc7eb2d5ec5c, 0x0007e9db114e2be8, 0x000c421a1f0f05da, 0x000016bd2f3080f0, 0x000fb9b7203704df, 0x0004fe2ab0cb3f7c, 0x000ec1adb877c781, 0x0000000000037761}, {0x00081bf9c70c3c38, 0x0003cb5d6068d8ec, 0x0006b75a387ed168, 0x000aa5457d36d422, 0x0007423cc8cb39b4, 0x000a6566dec1de46, 0x000f55280e02dafd, 0x000000000003ca55}}, + {{0x000e1cebdad8aedb, 0x0006b340d77c218d, 0x000b7391085bcda5, 0x0005b028b74d32db, 0x000a3cc1117e3e91, 0x0006c5939ae7101c, 0x000caa3c36152b52, 0x00000000000d3ec0}, {0x0001039f162ada74, 0x0004534de058e1ef, 0x0005d23a486e5201, 0x00030efc214ac0e4, 0x000cc7100acb27c0, 0x0009097b72a216d1, 0x000aaf3b73004330, 0x000000000007afd8}}, + {{0x000af1b588df6219, 0x0003a5d7afe99881, 0x000b20bd69a8d3f2, 0x000067dc65234936, 0x000527557f5de39a, 0x0000e9f9d3458527, 0x0002d8756b8d8693, 0x00000000000040d5}, {0x0003845ac0bf2beb, 0x000a9606eb60a92a, 0x000238ad785ef768, 0x0005748e068d2949, 0x00022fce158da621, 0x000183c8f73fbdca, 0x000c989cfee07dcb, 0x00000000000a5e03}}, + }, + { + /* digit=19 [{1,2,3,..,}]*([2^76]*G) */ + {{0x0000e52cf2e9b4ad, 0x0003f86230933fe6, 0x000fe37b1c8aab75, 0x00086ba6f595ed40, 0x000de2284a5801e1, 0x0009e5e25d3291c3, 0x000f5303dee2311b, 0x0000000000015cc9}, {0x000f15883981ad15, 0x000444fba15675de, 0x0008ff144b8a465b, 0x0001eb92532d015f, 0x00084e7455de8690, 0x000f811c94396fd7, 0x000b372fbcea8fbc, 0x00000000000ae952}}, + {{0x0007195bc501819f, 0x000f06bd2c54c87b, 0x000c8e0d99fb53f8, 0x0002654e3eee748a, 0x00039598d90727f5, 0x000cb371256f2058, 0x000c5b5f91c2d027, 0x00000000000a3e33}, {0x0002b15fc69c25d9, 0x0003e248490a9884, 0x000516ef1dee14b6, 0x0008b3d74f7d38ca, 0x00002fb602142013, 0x000033d4eae791f6, 0x000f176b4fb300bd, 0x00000000000bdfd1}}, + {{0x000558e7c27dd125, 0x0004d9bd2b68cfc0, 0x0001783c86b4626f, 0x000f35989586da1d, 0x000afd0944ca67e7, 0x000569b5abc04c8a, 0x00046eb8db5424f0, 0x000000000000cc55}, {0x0008ec9be17ec8dc, 0x0009e8f0ef9decbe, 0x000cdc094579a394, 0x000eb644e7329b8f, 0x0007e6896b042ec1, 0x00018b6bf3e0e7f3, 0x000fa6ad2bff11c1, 0x0000000000040f93}}, + {{0x000fa586c442030a, 0x000a30c3d2055f9d, 0x000061ff065411bd, 0x000a98b1fbf56cae, 0x000f396e31cd1b68, 0x00082c03b56fd85f, 0x000917d2ca584443, 0x000000000007d3ef}, {0x000572f96fe58801, 0x000c74129a730d28, 0x000ce4a000ce9071, 0x0004180653f3b241, 0x00049ee066bd9e85, 0x0002ea5fee65a1a1, 0x000e2e3420084e36, 0x00000000000c7911}}, + {{0x00079bb68a21bf45, 0x000b04c3f4f487b8, 0x000f5bdc807ffd5c, 0x0009bad1844be875, 0x000e272afc140fbd, 0x00040a7386b3b80e, 0x000a37d727b0b6aa, 0x000000000004d809}, {0x0001886f07e4e9c3, 0x0004c09fb0640689, 0x000e568c8f4da447, 0x000ff8ad73a211ba, 0x0001d9a6b97da7e5, 0x0009e8f56ccdbf62, 0x000a0938448e2cc8, 0x000000000002126d}}, + {{0x000bfb3f1befeb61, 0x0004c43f0ae67505, 0x0007b68cdb6c3beb, 0x0006c25dbb4492ec, 0x000884bff97fd5be, 0x0002da2e27efc9ee, 0x000c3254fbd9d706, 0x00000000000c6d2a}, {0x000d04af3852eaac, 0x0003e14cf6dc0935, 0x0006b6f8f3aa240c, 0x0006543d9a16800c, 0x0009beb1e28dd056, 0x000e658339cc2ade, 0x000cb2cbfeb45038, 0x00000000000d0f5a}}, + {{0x00014927529fad7a, 0x000714f10299a19b, 0x0007bc14eebc4178, 0x00018bc0cbede41f, 0x000e6b03ac7d024b, 0x0005213be7883c01, 0x000540a7b99e741c, 0x0000000000085a97}, {0x0007979358dabc76, 0x000060b6fe59caa3, 0x000a32d2efcf0218, 0x000969d6ad1891a4, 0x0003ee7761dccaa0, 0x000b12539c47afe1, 0x0005f4cae681fcdf, 0x0000000000002ea6}}, + {{0x00033b53cfd1d032, 0x000865e930d64f49, 0x000e6882e13322aa, 0x0004f7efa08bd4a8, 0x0006a6e10f64cfac, 0x0007ac162277e845, 0x00068b41be306740, 0x0000000000052a9b}, {0x000cf81e872685ed, 0x000e28d8d216a150, 0x000406cac9d54222, 0x0004102c60d54c43, 0x000028815187ad9a, 0x0003b8e59b09420f, 0x000ba1881179c60b, 0x000000000005b09a}}, + }, + { + /* digit=20 [{1,2,3,..,}]*([2^80]*G) */ + {{0x0002bb0ebd1ffa8d, 0x0000c22313ded9b3, 0x000c76c37673c0ae, 0x000eeaea76ec3812, 0x00060275ad3643d6, 0x000095bdaa165513, 0x0004b0e5c1b65adf, 0x00000000000367c4}, {0x000f3876ae1976b5, 0x0007de2b419c1bd2, 0x0005d5344cebbb0b, 0x000e51fa2baff060, 0x00006269b5d0b5fb, 0x000e8d667a0594d7, 0x0006e70b07b70523, 0x0000000000063e01}}, + {{0x00082243f0134f99, 0x000d9022aa09c335, 0x0001e1b9a8966dc5, 0x00044c5b38b417ba, 0x000b436451ec3173, 0x000b1c87629a74cd, 0x000ffdd898eb6ca8, 0x00000000000b74ee}, {0x000187e3675bea14, 0x000410d9eab63bb0, 0x000dd59e68360fb6, 0x00094ce14dbff2ff, 0x00092706528037c8, 0x00029a0cd3704d34, 0x000c4f9ee435d8ba, 0x0000000000009c11}}, + {{0x000414827bc8962c, 0x000552fde4893802, 0x00000bdd0aabfbb8, 0x0008a5b09456ed5c, 0x000925797c6c157b, 0x000c14c0673606a7, 0x0001caa9d0f47c33, 0x00000000000faf97}, {0x00082453cb3879aa, 0x000ec6e763c509a5, 0x0006f45dc5dfe238, 0x0007ecbc49e9efce, 0x0007b38f2b59d95b, 0x000c397db0c31792, 0x000bc797912b53b3, 0x0000000000045c7a}}, + {{0x0002600fe807b0d9, 0x000578b3269a1894, 0x000cd6a11a8caa23, 0x000c714e428865d4, 0x000ffdaba094bc5a, 0x000d657f2531dd17, 0x000bbf86d2405718, 0x000000000002f99d}, {0x00063a9cad22b4ec, 0x000d1c428a80ebb2, 0x000111bbf67dd9f1, 0x000691922a5d2566, 0x000318a18586a752, 0x000f633b3bfe6392, 0x00029828a0c772fc, 0x000000000003f3c5}}, + {{0x000cdc4baa08bf36, 0x00052cd81bdce7a2, 0x0009dbe6198554f1, 0x000228bf8ebe39d1, 0x000eb2cdd8524cb0, 0x000e9b3af496a9dc, 0x000f8a0f115a1ad0, 0x00000000000d8eee}, {0x000de9efc567fd4a, 0x0009b5232a514417, 0x000762e720496fa2, 0x00018b08994b4e8e, 0x000809097a52289d, 0x0004621d0b4e346a, 0x000ab8c641510f32, 0x0000000000095a41}}, + {{0x000c5913c30a847d, 0x00004c8c9b0adae6, 0x000c6088d1ec2259, 0x0007946e62c508f6, 0x0006e9fe2321b03b, 0x000c1d1366042592, 0x000c9c73b10bba80, 0x000000000009d218}, {0x000e72b1483512ee, 0x0006d6ed0c8eed83, 0x0009abfdad8b62ec, 0x000096ab9e96167e, 0x000cc473e3773078, 0x0003a3bc8b28f0e6, 0x000796b44e499568, 0x00000000000d3523}}, + {{0x000ec81c53fdc5cd, 0x00095261578c7817, 0x000dba0eec8c348b, 0x00066b414c803900, 0x0005faa7cc399932, 0x0009b7e27bacad75, 0x000533418c4defd6, 0x00000000000ae751}, {0x000b26e534337f55, 0x000f36f3bd3298fb, 0x000bd59fe7197151, 0x0004f73dc4c54ecd, 0x000d44d3d74eb716, 0x0009ea511ca66290, 0x000c49f77c62424c, 0x000000000001f714}}, + {{0x000343bd88e025df, 0x000b893a62e7c716, 0x0003b75e6fa4a448, 0x0003cbb10cc95456, 0x00062f37154c265e, 0x000f2e08b28cceb0, 0x0000916b79b0713b, 0x000000000003ab39}, {0x00046ae52480b7d8, 0x000a750a8d04cede, 0x000f3c798fdbfa72, 0x000336ac21cc62f0, 0x00036fc004503db4, 0x0004b2d954747f0a, 0x0007a7067cbf1c54, 0x00000000000fd2be}}, + }, + { + /* digit=21 [{1,2,3,..,}]*([2^84]*G) */ + {{0x000f8f632d3f21bf, 0x000880233e423a7e, 0x000fc3a5113e93d2, 0x0009cef6349e35be, 0x0003ded2ec542ca0, 0x000dded3d3b70afc, 0x000ee0c813474d52, 0x0000000000012f00}, {0x0002c304534f52c2, 0x00002a0908f763fa, 0x000cccb3aba1eb57, 0x00010c134b6a8e10, 0x00056e7c163e84ad, 0x00057e0c36f37fd9, 0x0001a27caae8c8d0, 0x0000000000055372}}, + {{0x00086fa76217d20b, 0x00015a354c1de108, 0x000a55c4648be905, 0x00010999d532bbe4, 0x000f520117516766, 0x00059e35eb7c2f4c, 0x000b7b6945d34ff6, 0x00000000000a1303}, {0x000495f21c8085bb, 0x00028a200ddba08f, 0x0003caf84b1e53dc, 0x000445faa82b46fc, 0x000cfccedc1b3018, 0x00053e29fc3e6a3a, 0x000547e86ae87033, 0x00000000000fd7e8}}, + {{0x000683184a93e477, 0x000a4728c8e14128, 0x0003d8bece755c7e, 0x000c2da4796df734, 0x000e00a55efc3016, 0x0006b1cb8f6d79e4, 0x0005a94fced26952, 0x0000000000091f16}, {0x00031165825c7f6e, 0x000cf528b857275f, 0x000d8cb6dce0ced4, 0x0003520e07bb990b, 0x000f4ea8ae6b4f90, 0x000b735bb07ddb97, 0x000d4a29c01e70b3, 0x000000000005000d}}, + {{0x000f814f4e7d6929, 0x0005e306b63e3665, 0x00098d89bb61e0d2, 0x00068898bdeddea9, 0x000d67a329ac36bb, 0x00092ccce9331655, 0x0009b414c7fe26ba, 0x00000000000be051}, {0x00042824d37928be, 0x000830dc39dbc42d, 0x00083d595f84e0c5, 0x0008818fa8682bba, 0x000db60aad97c7f7, 0x0002c64cc43396ea, 0x00043e751784d7ff, 0x00000000000866af}}, + {{0x0007b94e1c1e3bf9, 0x000a53742b61de85, 0x0004d0debb63955e, 0x0006787abce34016, 0x000ea4cb41b4f52c, 0x000ca817e4749711, 0x0000487ce0dfb03f, 0x0000000000096e85}, {0x000c4c6d7efe5c7c, 0x000b6cd65250dc1f, 0x0007ec2aec96d815, 0x0003cb2b4dd561a2, 0x000abb5379f57352, 0x000a847ba15ba773, 0x0002decc5e62d6c0, 0x0000000000004d85}}, + {{0x0003450b4a8e849f, 0x00036f0a69e68100, 0x000d9356a0504fb6, 0x0003ab7cd14af1d4, 0x000d558d555d773a, 0x000e63dd6c0b88ed, 0x000db3eb12d197d2, 0x00000000000bcd9c}, {0x000ed15d1fefd8b7, 0x000fa06432985766, 0x00052d6c3d3a4a47, 0x000f6164171b9bc6, 0x0009bbfcf90abd28, 0x00036084d3778163, 0x000a751f93898f3a, 0x00000000000dd00c}}, + {{0x0000e82da9d478f7, 0x000ae3b8d8e48b47, 0x0000534e09405090, 0x000430c2aad8c684, 0x0006b82c16731d72, 0x0000fb41de8c5d50, 0x00034f989978868a, 0x000000000000f812}, {0x000d9518cbfe70fc, 0x000f1a0c941ce78c, 0x000c323a8cf0aba1, 0x000e296101f89af8, 0x0003ca6a3227aa7d, 0x00064ec0c4cadd8e, 0x000db43b4db51f27, 0x0000000000009256}}, + {{0x000e28a122d0f32b, 0x00022b9207dbc085, 0x000dce5b9cc6b932, 0x0004e5534ba150b8, 0x0001dfec99724992, 0x0006f870d2935ebf, 0x00085c34bf5e94b7, 0x000000000004c203}, {0x000bdbb9ed40b2b9, 0x000c3c78ac1719f8, 0x0008866adbfa0f7d, 0x0007ee4cfb4b8bb6, 0x0002cd8724b676c6, 0x0000bc1efecfaefb, 0x0001b5e9bc3d6734, 0x00000000000ca554}}, + }, + { + /* digit=22 [{1,2,3,..,}]*([2^88]*G) */ + {{0x0004ccdc054842bd, 0x000704f293478100, 0x0000ca74cad5c267, 0x000c94f64d55f6f2, 0x000d725c51251bf7, 0x000b6a0914489b57, 0x0000fa3aa4d43ab1, 0x00000000000cf7a6}, {0x0007fef943134dbd, 0x00038aa918a4ce7d, 0x000eee4d56ef907f, 0x00000a16812a03d0, 0x00070b0a1e4bed0e, 0x00028386bb09acf2, 0x000321c73a41f7ac, 0x00000000000f4cd1}}, + {{0x000b2804db21ede5, 0x000fd10a53b8f9a2, 0x00080087f23a6e5f, 0x000513fabef2b974, 0x000cd3f802fd740d, 0x0000627afff6cc47, 0x0002dc1be6825beb, 0x000000000005886c}, {0x0002ebc44567c313, 0x000afb6cfb2c6b07, 0x000e1dddbc404a5d, 0x00016f53a5a485ba, 0x0002e7b2223e7a1e, 0x000a13a9b16b60b9, 0x0001a2332f33d836, 0x00000000000876cd}}, + {{0x000bcac1cf1b2eda, 0x00095802f495c1f2, 0x000e55ffc912d44b, 0x000b294e9b3f0f6d, 0x000033cbdcd1eaf3, 0x00083fce582cc760, 0x0004cc1a5642278e, 0x00000000000ffa0e}, {0x000dada72c71a95f, 0x000ed7d93e9766ea, 0x00015d2d9c24e57f, 0x000dd79eafe5f361, 0x0000fcaafabbcb53, 0x000053efecbbed8f, 0x000b7a570472705f, 0x000000000001ebfe}}, + {{0x0005f3669cd44c3c, 0x00025a896d28dae1, 0x0002e7c6820a84bb, 0x000fe8c06fb15cd6, 0x00061cdffdad1bba, 0x000119b3a2daab29, 0x0005a3984defc8e1, 0x00000000000de2a2}, {0x0000f914a19c745f, 0x000b3a7f54ecdb35, 0x00041f75def22ce4, 0x0000efc6a3b94437, 0x00076785163b6913, 0x0002b1d0a5d17f12, 0x00093cc019911b17, 0x000000000007e3e0}}, + {{0x00049fc81e69d5e0, 0x00085d307d4279b4, 0x00039a33441f2443, 0x00078371531d6263, 0x0008b42639ecfff7, 0x00099fb76fdd61b0, 0x000f0eca9462e7be, 0x000000000007c3f2}, {0x0008f656f3641bfa, 0x000f85c07066bcc1, 0x0008bd883c4e6617, 0x0001387d93ff4937, 0x000b864eed19b6d3, 0x000fad6382fd7d67, 0x000fe667b707ff5a, 0x00000000000d0206}}, + {{0x000bb568a2a2f3e3, 0x000a9235455295a9, 0x0008072539522964, 0x0006d10c75720aab, 0x0009f6a0df88ac83, 0x0005a9c96ea2c538, 0x0007bedfbf31519e, 0x000000000008cad4}, {0x000d6b001091ce3f, 0x00036dbd5de982e7, 0x000e935fa43e76d4, 0x000fef6324ba9235, 0x000e8f14a2f5a2bd, 0x000880ade8b0f981, 0x000d4a590de1c45f, 0x00000000000107f3}}, + {{0x00091e8b5ebad75d, 0x00009b03d47621b9, 0x000da50c79b05bb6, 0x000ca72548723c56, 0x0007c944e3849da9, 0x000947973a12e40d, 0x0001ea403dbcf2cc, 0x00000000000c6ef4}, {0x000f1cf0966fef48, 0x000d74ccc2856b9a, 0x00011bbeae1e30d2, 0x0007f7e43ee1576e, 0x000a0a894961701a, 0x000396f6e7f061c1, 0x000ca956059fb88f, 0x000000000001a91f}}, + {{0x0000a27b9e3336b7, 0x0000be185bdc20d5, 0x0008a8bb1575f81e, 0x0003397c3fe5695a, 0x000f61f6a04f1f6f, 0x00090a92b6c9f3a5, 0x0004543c0f9d4bb3, 0x000000000004793b}, {0x00055e696b3d9202, 0x000d991ad0c33a93, 0x000c4d967cf46a4f, 0x000d90565d53c83a, 0x00034176007dfa24, 0x000d1623cd63e583, 0x000345741089fd2c, 0x000000000002685d}}, + }, + { + /* digit=23 [{1,2,3,..,}]*([2^92]*G) */ + {{0x000883b867d2fbb5, 0x000271d51015761f, 0x000afd0328ae7e6f, 0x0007e2fc6dde9e2b, 0x000937d5bb1f2ea0, 0x000cc6e28ceed9ad, 0x000fc43121994b98, 0x0000000000067ad8}, {0x000d53a60b81c5b7, 0x000ca34707fb3d11, 0x0005e1e4ff2b5765, 0x0009b8d81ef6cb57, 0x000b1223a9a1fe93, 0x00074cf375c51e8a, 0x0008e18f6d7993db, 0x0000000000097280}}, + {{0x00067952d0358e02, 0x0005b0db3322730d, 0x000098edfd5f000a, 0x00085380e8a38fae, 0x0005769360d86efd, 0x000f27ef5c1812ea, 0x0002e8505723dc76, 0x00000000000f35af}, {0x000764970d5cb557, 0x000849f7bb7fecc3, 0x00052323da5066c3, 0x0007549773eaf7f9, 0x000489d51bda0118, 0x0000ccde605bbbba, 0x00087d52cabb0664, 0x00000000000e7ff3}}, + {{0x000fa071b96c7c7e, 0x0009c70187dfb0a2, 0x0000f19decc5bd43, 0x0008b816a6c0d616, 0x000d6a27e97b3018, 0x000a389aaeaeda67, 0x0002734a5192826b, 0x00000000000a2bf1}, {0x000832bf68c7b455, 0x0000c8ff10656bdb, 0x0008bc56c7023546, 0x000a3e11dc7aed54, 0x000b38853d61e34b, 0x000cbf658c457048, 0x000edde89825db1c, 0x000000000006b255}}, + {{0x00067336ad6a8789, 0x000d0c377b3730db, 0x00063e31dca6f13f, 0x000f2adf39b68951, 0x0006bd6da22ed9c2, 0x00009678e1bbff10, 0x000e783f7e5b8e7e, 0x000000000007ed3e}, {0x000a0e3ab40df97f, 0x00071689e6b30fb8, 0x0000b670f8712d68, 0x000277edda78c55a, 0x000cefb65cb717d3, 0x000a30f0ce686cac, 0x00071e758aa1ab19, 0x00000000000b11f0}}, + {{0x0007d3d2024290f7, 0x0000c0bef2d25159, 0x000b908d7c494e4d, 0x00017cedd4af7aeb, 0x000bc8e88ca42c24, 0x00020d15452b5b3b, 0x000d4bc00328c44f, 0x0000000000030af5}, {0x00053687c2073cc9, 0x000dee8bacbeb2cf, 0x000bf613b57931f9, 0x0001da61d9e70314, 0x000c0e70a331b67d, 0x000e325772ddefcb, 0x0009797d09bc6d61, 0x000000000005b52e}}, + {{0x0000d7a5438e4dd1, 0x000b5d40f36e5e3e, 0x000702a395a7941d, 0x00094d7cc51dbaf1, 0x000f0edf8bf9e340, 0x000b5e16520f93f0, 0x0003a8534b75aa23, 0x00000000000dc1b7}, {0x000196b771cc1d7e, 0x000b09dd48c40ca1, 0x000664851f0b3766, 0x000f8a2a9ffa0db3, 0x0004d94ed9bd4212, 0x00069cb198d5236c, 0x000dfc81bec489f1, 0x000000000006104e}}, + {{0x00076550d0174653, 0x0008d9cb21b56311, 0x000fe59ad5e35eee, 0x000a67100d7b34c9, 0x0001a6c1b0c3e3f3, 0x0003c3e31d4bcb3d, 0x000be86ebd26fb9f, 0x000000000002dc79}, {0x000582f4b4dfea09, 0x0002ae7cf9a0a1f3, 0x000c37b50dcbdab8, 0x000320e69e1f08bd, 0x000b946f83d78b0e, 0x000a399ad2a8caf9, 0x0000866daff0ba98, 0x0000000000038ed6}}, + {{0x000af81ab2009a09, 0x000fc16f3708ae33, 0x000f381b02290f16, 0x00015b227bb71a8b, 0x0004162f8b5597a6, 0x0001e3fb9a8b9a86, 0x000ff680ee8362d9, 0x00000000000f83a4}, {0x0005ffc830c362b9, 0x0009507a7873a90a, 0x0008c2637b53ccc0, 0x000683d860d60ba4, 0x00071f87d32a427f, 0x00050319dea99592, 0x0009ceb2ac69faa0, 0x00000000000d2d0b}}, + }, + { + /* digit=24 [{1,2,3,..,}]*([2^96]*G) */ + {{0x000fbfb66a502f43, 0x0008324480c57881, 0x000d6a55d7a45e41, 0x00002c3273e2bc82, 0x00053ef1ed4c7350, 0x0004948e88c42e9c, 0x000632028babf67f, 0x000000000008a978}, {0x0008edce917c4930, 0x0001bf5f13a4625c, 0x000e9dd4dc6a263a, 0x0006bcc37232768a, 0x0001576e8c1830f3, 0x000bcb766c5317b3, 0x0004dcef1d57a69b, 0x00000000000b3e3d}}, + {{0x000aa97e11eb2f75, 0x0004f53ebbaaea28, 0x000f59d2921161ee, 0x000f25d340986b44, 0x000e3365312a3c15, 0x000641f96c5925d2, 0x0008e4c35d3ee251, 0x0000000000098412}, {0x00032040e2f19c59, 0x00029b41a21a75a1, 0x0005c4225a472860, 0x000f4c0faf5663dd, 0x000a0a62c9dc4ffc, 0x0002d60c5889d285, 0x000be50908665acb, 0x000000000000a131}}, + {{0x0006f87a21caa381, 0x000a6318feb4c006, 0x000ed4b6b2c3b939, 0x000786d5b2386621, 0x0005a722c5911e4a, 0x00006cb0188e0430, 0x00036438eb062af4, 0x00000000000e7216}, {0x0003e9f2fb9e0c73, 0x00093450c1d28f54, 0x00023904331c181f, 0x000a53a6f6c06813, 0x000a25eac032fe46, 0x000c497224553199, 0x000a2941fa659aff, 0x00000000000bbcb7}}, + {{0x000a3dcccc791733, 0x000a8c0de60f43c9, 0x0004454fef6e3630, 0x0002aa8e734e2e0a, 0x0003ef773db6f412, 0x0002d71c508f40f6, 0x0008250a7e9484ce, 0x0000000000078a3f}, {0x0005a99063a64539, 0x000ef0cac7a5b50d, 0x000064e61e079dfa, 0x0009a90d3e181458, 0x000a0aadf689ecf1, 0x000f0acf70d1b062, 0x0002b942299f1ff3, 0x000000000005ac25}}, + {{0x000b4c356075ce0d, 0x000a8a4bfe150fb5, 0x000907dbaf6f1c40, 0x0008260a0ee708f9, 0x0004abc9d5f541e2, 0x000aae99eff4fdff, 0x0004d96ed92241ff, 0x00000000000c04fe}, {0x000731763369357f, 0x000a8f10a173e2a4, 0x00065723921ac826, 0x000823f8ed02e85d, 0x00042e522dc1d2a0, 0x000c939fdffa78cf, 0x000bba07041e4600, 0x000000000003a9a8}}, + {{0x0008012ba5de4e7d, 0x000c7a1fc20f7144, 0x000540a292e6e434, 0x00081c28e2e8e16b, 0x000a4c4562342e55, 0x000e31ad4912c80f, 0x0008117fd61fb9e9, 0x000000000001c19e}, {0x000de815bbe3ce38, 0x0003b6a7ab2cb6d7, 0x000aeaa95e1796a8, 0x0006a85ab06af4e7, 0x00044866378d33f7, 0x000826f8cda387e5, 0x000cb6bb71deb856, 0x00000000000eb64e}}, + {{0x000d7e72d8af89da, 0x000ba0e93593424a, 0x000584230569d130, 0x0003dba8e15d8645, 0x0001cab64644ab77, 0x000fff0619cea4a5, 0x0006a4516754d72e, 0x00000000000cd3a3}, {0x000ee6af7eb6909f, 0x000c6c7d352abc75, 0x00007221eaa0649d, 0x000ed979199938f5, 0x000acd6f34958a8a, 0x0005f93eeeb06789, 0x000851c51d776677, 0x000000000000a8af}}, + {{0x000b089742823cef, 0x000c544bfd166199, 0x0008c58b08cf9fc3, 0x000cf81c39366838, 0x00068b08680fe50f, 0x00032aab0c9b4eb0, 0x000ddaf90882c528, 0x00000000000ecbf5}, {0x0004bc1447794cdc, 0x000f8fd4d51c6fd3, 0x000cc355ce034b68, 0x00053dbfb187c34f, 0x0006f8ed037fb729, 0x0007bad45166f7f2, 0x000808d2b1077a09, 0x00000000000790dd}}, + }, + { + /* digit=25 [{1,2,3,..,}]*([2^100]*G) */ + {{0x000273ed4a6f3c1a, 0x00092a05c751c876, 0x000bdb5d554c8320, 0x000e26cf4c98fee4, 0x0009e7b3a7c79c56, 0x000cc16466084f8b, 0x0000bca0b042f6de, 0x00000000000fbf3a}, {0x000078eb34f9801e, 0x0006694524c2c1cd, 0x0008ad2d5e6f66b7, 0x0002d6e7ce1f9f62, 0x0007fc0c34fa1879, 0x0000a16938ebc2fd, 0x000019032a9f4178, 0x0000000000056a60}}, + {{0x00009be19d72cca6, 0x0009ea4b957d1c55, 0x000c88712c99979c, 0x0005e80a271d9e14, 0x00044b9367e51157, 0x00079779f0fff68d, 0x000f68f55ba67322, 0x00000000000d4d68}, {0x000ebae305d14369, 0x000b401474ab1533, 0x0007e0884a0d1ea6, 0x0003a8336111d25d, 0x000fab531cef6390, 0x0002ed86737342a5, 0x0007e1d3c93fe770, 0x000000000006279f}}, + {{0x000ed609fa3c6148, 0x000beb398290bf69, 0x000b5366c9b47f2a, 0x000aa2956560b89f, 0x000c50e6647b3e71, 0x000be42da80817aa, 0x000a4e35c833ada0, 0x00000000000f1bab}, {0x000a4dc6e615148d, 0x0009ac57be644e77, 0x000de3f8f9ac6cc1, 0x00077310defd9095, 0x000e899cba568300, 0x000effb5a92fe9bc, 0x0004bf48450ec4bc, 0x00000000000f2f5f}}, + {{0x000e1a6a67c12c1b, 0x000e7dccf68bc63e, 0x000af5107f8ecb57, 0x000e199254dd3e41, 0x0006ddce5c84e43a, 0x0000b4258e8526e2, 0x0004c452bad815ea, 0x0000000000009457}, {0x000fc838b5157d7f, 0x000e1a5362fcad0f, 0x0003a5c9b577c868, 0x00012b5eeffc2cdf, 0x000b14de4e5b0f93, 0x000233fa55475657, 0x000ec2746e67d4f0, 0x0000000000035471}}, + {{0x000afd6c233d63fa, 0x0009cf55edabb6d7, 0x000b353b7e2a9194, 0x000822b9bf171614, 0x0003afe808dcf53e, 0x000272f554a5b3b9, 0x0009a1590bbbded7, 0x00000000000eaea7}, {0x000101d093edb837, 0x000b5d7042bea6a3, 0x0006a766d7a462c5, 0x000e69031078aa66, 0x00087e37bc8bd8d1, 0x0000089759e837f1, 0x0008679558ff4f85, 0x00000000000421fe}}, + {{0x00076617121c3ac1, 0x000a13503d9370d0, 0x0006e1d369d53e18, 0x00001ef5b52e0f07, 0x0002f955b5fa67f1, 0x0000e0569b6b5b4c, 0x0003efb5f03d5388, 0x00000000000c9939}, {0x000c81fd72768469, 0x000a690755bd5e4b, 0x000324dbfd474da1, 0x000fd519e9ce792e, 0x000396ccfa14326f, 0x0004a22de1b772d7, 0x000342652710f487, 0x00000000000db210}}, + {{0x0008597293bd01e8, 0x000dfaa6489def01, 0x000dc630321a7c29, 0x00043a95fa2efed7, 0x0003720e1a8c4d89, 0x000a4f8a3c9baae5, 0x00006bc055444b95, 0x00000000000a7c12}, {0x00017016dee8b7c9, 0x000dfd97765b5926, 0x00050b911df827d2, 0x000c35af1503b16b, 0x000fa9fa21f8115c, 0x000fed1f11dd5359, 0x000679197c32996d, 0x00000000000f77f2}}, + {{0x0004d7575fc73ce2, 0x000ce58d6998db75, 0x000d11d5d661c62a, 0x0002a0165739fcf5, 0x000822f7a073420b, 0x000042f005c5db30, 0x0003d9016c547805, 0x00000000000a1241}, {0x0008aac8eaa9ff4f, 0x000fe8ceb2d6ffe8, 0x000f806ab6efe1ae, 0x000d012339c146c9, 0x000628ac05552638, 0x000d46e6302fbb7b, 0x00088e7fcb0c8162, 0x00000000000066d0}}, + }, + { + /* digit=26 [{1,2,3,..,}]*([2^104]*G) */ + {{0x00029831f2b8ed10, 0x0004917e3d0b77e8, 0x00062b9d59b96ed5, 0x0000261d0bb1849e, 0x000ec71a3bbef8ef, 0x0002a88ae9b804ce, 0x000b9e00b7d17154, 0x00000000000e9097}, {0x0001f74d24b8094d, 0x0000cdbad9f12075, 0x000917cb364517bb, 0x000834fb83debf54, 0x000d637e449ba8a4, 0x000b82664c3ee226, 0x0000e8e3070d93ca, 0x00000000000b3732}}, + {{0x000cd133b578c566, 0x0004bb506b7b86f8, 0x000a189da5b723c8, 0x00029ad3789281ee, 0x0001f9af456164b7, 0x000f9186069ae8f8, 0x0000fb007befe156, 0x00000000000edc25}, {0x0003b5e8dc3a9219, 0x000ab1c16d7598b7, 0x0000fdec987e0e87, 0x0007da936881f89e, 0x000103a5b1ea5dc9, 0x000153a0faddb603, 0x000ee27eb515b13e, 0x000000000002b4a1}}, + {{0x000bcceb06f924be, 0x000d4e6b739c7071, 0x0004a4fc7ff778b8, 0x0003acc1b4b45493, 0x0008f6bad44a14e3, 0x000020972d6704b2, 0x00033543cd7e5ddf, 0x00000000000e0d6c}, {0x0008ad0a64850139, 0x000b67d395c3ef6a, 0x00040e40560ce9c8, 0x0009f21efe4a5757, 0x0004f55ea727b626, 0x000450c823f97b37, 0x0005d5b689b4de42, 0x000000000004f5ae}}, + {{0x000bd2a72ae0f683, 0x000d7e7365f9e264, 0x0009dec60e7c539c, 0x00089ac8e611fc3c, 0x0005699c227dbfb3, 0x000f2ef17ce778fd, 0x000ed4dc1f47b63f, 0x0000000000002672}, {0x00013b4caed73e7f, 0x000a551c63805fa1, 0x0002b72b6fe7e133, 0x0003d13e00e12a38, 0x0000bf8df325375b, 0x000f5f21a91f9f75, 0x000584f1ea0e421d, 0x0000000000032c60}}, + {{0x000c6a3d385d54a6, 0x000a26d720050d7c, 0x00008151ff431078, 0x000aee6f1c89ce3d, 0x000b57b2b79d5d1d, 0x000c7ce823f7644b, 0x000b0bb1063f92fc, 0x00000000000c15a9}, {0x0004b892e4ed0e8c, 0x0007865661a5290d, 0x000a266171fca5c3, 0x000dc5d42ecede66, 0x000c8eb0689e0661, 0x000d7ff77be586ca, 0x000c3bc86641a02d, 0x00000000000b9ef4}}, + {{0x0006b862a25bd60b, 0x0005903b07fb53e1, 0x0000603f2bb26df9, 0x0006449356376454, 0x0002a55bcadf272e, 0x00071c6af2b6ad1c, 0x00080686c527765c, 0x00000000000c1678}, {0x000fb64422cb7cb4, 0x000660cae829453e, 0x000b68619a56abb9, 0x000f8337513dfb65, 0x000e1b351dda504d, 0x000af7cd04260ee8, 0x0000fda473c5d9db, 0x0000000000071e39}}, + {{0x000c98e3f8d5e928, 0x000c217544e9789d, 0x00047053e368c072, 0x0000e3937af8cf8c, 0x0009a2f308e2e146, 0x0005060e98b4aa2f, 0x000a7dafa2bdbc3a, 0x00000000000f84dd}, {0x0004961384232a8d, 0x000d53b256b55aeb, 0x00063be03fa9b44c, 0x00010d13c8e52f06, 0x000b38865772c405, 0x000416f1062c1542, 0x000eb54755de7ebf, 0x0000000000097aa2}}, + {{0x0009115b2961da7d, 0x000f01d9aa9f9fd6, 0x0004c642e6501980, 0x000d6a17f167b479, 0x00085bd9153d7ebe, 0x000682ba6324c13f, 0x0006e2e9ea5b734c, 0x000000000007e3ff}, {0x000ac4814edd792c, 0x000361ed2b04fb2c, 0x0001100e4a13806f, 0x000a5d3d68b8ec70, 0x000d1c6ce0d2afce, 0x00019410c21dc058, 0x0005340043de75e7, 0x00000000000e15cf}}, + }, + { + /* digit=27 [{1,2,3,..,}]*([2^108]*G) */ + {{0x000e0e09fdbcad95, 0x0004ef7aaa73197d, 0x000fdf5aa02c7c2e, 0x00017372e0c286f0, 0x00025da3472da1a4, 0x00076f2a23b66850, 0x0007d4bc0d116b75, 0x00000000000a36a2}, {0x000b11735059e67b, 0x0001ecdb3f4744cf, 0x00033adc74dab02a, 0x00011b969d4f1723, 0x00066cca4ef7c70d, 0x000c6afdfdf96d13, 0x0004da570693db2f, 0x0000000000056750}}, + {{0x00080567673bd755, 0x000df02c6b607d3c, 0x00025bebf3a2d95d, 0x000b01d4f57eb0c9, 0x0004d750e515af5a, 0x00089ad6859935b9, 0x000eff32721b408f, 0x00000000000a7e3c}, {0x0009a7a317218bf0, 0x00010bd462fd8d81, 0x000035ae54257b2d, 0x000da619263fa61b, 0x000eb6488eb34162, 0x0005680c42ed1f2d, 0x0007b0b0bd37a872, 0x0000000000029ec2}}, + {{0x000d73368a1e2f2d, 0x00047170f8a745d8, 0x00056c4ca9228944, 0x000bd224b08921af, 0x000de6caca3a5f65, 0x000cb9d5b1017ae3, 0x00099b613f36b626, 0x00000000000ed19d}, {0x00092b03ebebeebf, 0x000c752c915a4d7d, 0x00088621f044eee8, 0x000f7c206a51a132, 0x00066e0197ce2cd7, 0x00071b04cfed60db, 0x000b134e5b2bb7d9, 0x000000000002f45a}}, + {{0x0009fb6e89b64981, 0x00034fb09e755c21, 0x0005804661454c99, 0x0005b01cdb1f4f66, 0x0008f1dd1cf0451c, 0x000687e41b02f906, 0x000bd3d4765e7c0c, 0x00000000000d1967}, {0x000f135de5248a08, 0x000f406a0e4ec0b1, 0x000b70c2868ad046, 0x000d01ac651d8073, 0x00030618a96cc1e9, 0x0004ad8ce5e6ebd8, 0x00090f9ce1aacec5, 0x000000000009953f}}, + {{0x000f6e431dc5b814, 0x00044fa4b66978a6, 0x00032f1c66e89ca4, 0x000ba60986c85001, 0x0004dae014e110aa, 0x000bc0e4649bdfff, 0x000d5fe2a810bd0a, 0x0000000000062d1d}, {0x0007dc411c936920, 0x0002c15d2f2c7072, 0x00075b1f6234ac4c, 0x000b6f94c545f5b7, 0x0002f99241e840c6, 0x0007082935875b6b, 0x000344fa3e88a9d6, 0x00000000000ad6d9}}, + {{0x000edb2b4857e570, 0x000fba8d7901c33b, 0x000ae7d92e8b3979, 0x000d7c504e4bce1a, 0x000e5ab0696fb905, 0x000db9cd5dfbf9c4, 0x0000fe0f4e6e2ab6, 0x00000000000482e4}, {0x000aa1f98e714cdb, 0x000225ccd9b418dc, 0x00058c553b3cc2f6, 0x0000dbe59065754a, 0x000a461c1620cebe, 0x000e541a74d052d3, 0x0002d8396bac422f, 0x0000000000070bf3}}, + {{0x000a0c1fea68e228, 0x0005003c88a847c6, 0x0008f73e5d2fc7c6, 0x0003d393870d90fd, 0x0004f2fdb976d90a, 0x000312a302fd78d1, 0x000f2f9472a6066a, 0x000000000003484a}, {0x000f5efbe46efdf7, 0x0008c7cf25950c82, 0x0006be05e61c118c, 0x0009379e563a2bde, 0x0004b46e93d85d47, 0x00037285ce87b50a, 0x0007e69128fc11f1, 0x000000000007c67d}}, + {{0x0006a0e99182cb3e, 0x00059da2b072529a, 0x0000a5cae364c3e2, 0x000e35a5b132756b, 0x0004fadb6907c721, 0x00083f546e5695e8, 0x0008aebc5a3bf4c2, 0x00000000000dde12}, {0x000842d7630a9d87, 0x0000d179c7fa6028, 0x00016ae518569923, 0x00083bea16bad558, 0x000e9137ecd8af7e, 0x00030d1ab6f41231, 0x000ee8ac87543d8f, 0x00000000000b1ee0}}, + }, + { + /* digit=28 [{1,2,3,..,}]*([2^112]*G) */ + {{0x00021880433d0094, 0x000fe359cbfa01b1, 0x000ccfcddd1c5f17, 0x000e90f630cbcb5a, 0x00006ddbf2382fd5, 0x000f753e62b0f613, 0x0000165070970a3a, 0x0000000000041727}, {0x000724192d1373c9, 0x0003e2eb15b3b70b, 0x000786ed962decdc, 0x00074aee930a75c6, 0x0006849e77270d43, 0x0006a19ff3cd3604, 0x000c049ac3117e33, 0x00000000000433dd}}, + {{0x000b49aee05ae3b9, 0x0005d985dcb4303d, 0x000d13447c8994a2, 0x0007f3055fbf1d6c, 0x000f43fcd98a4059, 0x000ec99cab6d166b, 0x0003705932570915, 0x00000000000c5bdd}, {0x00050b2df8b6890e, 0x000dac18f782613c, 0x000d7472d5468bfb, 0x000a89df478e56c7, 0x000864d290535d50, 0x000cf7e0e242c2f4, 0x000d40cda12c6161, 0x00000000000ac8d1}}, + {{0x0001c7858f5f5373, 0x000027a9a9dd7275, 0x000baba450a60383, 0x000267b6726c3253, 0x000d7f841afb1550, 0x00004643012cfb2f, 0x0006cba41b19052f, 0x00000000000b7d78}, {0x0007d7e12176321a, 0x000667b61b39c512, 0x00091b7379479d0c, 0x000efb84a3489052, 0x000cf2e864b965d6, 0x00096cdb2f377862, 0x0000f63c94000dab, 0x000000000008efef}}, + {{0x0001ea0f696fd259, 0x000ff8c558000585, 0x0007b878d3ed6d3e, 0x000d75ceb7a35cc0, 0x000cf0a93110acf0, 0x000c50c850fbdce4, 0x000d6f82b2d13a9c, 0x00000000000cef70}, {0x000ce46086dea462, 0x0001e1cc9be2dbe9, 0x00043757ada5e365, 0x000479f195a532b9, 0x000a1ab3605c52bd, 0x00024c1e0769fe6c, 0x000d7aba6a63e48a, 0x0000000000099da5}}, + {{0x000ad5fa3c1cbaf9, 0x00066df2a6cec962, 0x0003dc76a0f4995b, 0x0007e4c0de8a6aa7, 0x0004be7b8f5d7c8a, 0x000da3e7e5c806b9, 0x0008e5ccca1c714f, 0x00000000000cff78}, {0x00056778e2279cec, 0x0006e17a081a8743, 0x0007133643f614b8, 0x00090549e4cfac8a, 0x00029d53a2000d7c, 0x000977a8b6fb74c6, 0x0007c465b9ed6d5f, 0x000000000000ba2e}}, + {{0x000a6aa57649a837, 0x0000f8435e295b6f, 0x0009b668270d6cf9, 0x0006e6fece5cb161, 0x00095336f7ffe2fe, 0x0008ee2676f249b0, 0x00017b5801feb990, 0x00000000000fc8f2}, {0x00006b2a4f1b5cde, 0x00053fc1adc3e51d, 0x000e0bc8cae40bdd, 0x0005404cc1d9b726, 0x0001a1f2c5a44afb, 0x00008f19575cabd6, 0x000f492bd797e1cd, 0x00000000000cbea0}}, + {{0x0008ce5f4471b983, 0x000d8e9759aa9a33, 0x0001c65f7f2a0258, 0x00059aa2c80bb617, 0x000230c9b328e49a, 0x0004ebffcdb89e21, 0x0008e4f42b9adb00, 0x0000000000047967}, {0x000c42c02e645b6d, 0x000f20dde5b0e690, 0x000d9a21c0cf9036, 0x0007a5cd8c8285c5, 0x00071d1562069f8d, 0x000e126bcd7cde59, 0x00060bdbe765d7a8, 0x00000000000e3f4e}}, + {{0x000bcd16afac5b0a, 0x00041509abccab5b, 0x000ba67813ffa4bc, 0x00071bd640bc2f7a, 0x00078f546c681ae3, 0x0008b8cbfa155c29, 0x00048b8858cadcc0, 0x000000000001d969}, {0x000da5c95714aa4d, 0x00010919cb2262bc, 0x00085c89980779aa, 0x0000f1ab4abe5612, 0x00069cb75bd125dd, 0x000a778e4c6bbe63, 0x0004b3bb367cf947, 0x00000000000bde52}}, + }, + { + /* digit=29 [{1,2,3,..,}]*([2^116]*G) */ + {{0x000545d771f3d008, 0x00076c6307398a47, 0x00054915c4cf9c72, 0x00026aae3a993222, 0x0006da9308d18e01, 0x00068e5050fbfdf7, 0x0008bd2163ed6b61, 0x000000000007500d}, {0x000f06ed5b023878, 0x000d3f571595fffb, 0x000ccfd2dfb01965, 0x0005f7e53d84c661, 0x0009025029da6b22, 0x000d89aed9c03126, 0x00015054c1edfa17, 0x000000000006b435}}, + {{0x0007c3fb4d5bcba6, 0x0005b009ef80a60c, 0x00017966ca95c6c2, 0x00055ed685add960, 0x000672c7dac8ab44, 0x00072f16818f323f, 0x000a99e7009790e3, 0x00000000000067be}, {0x000c5bcf19664dda, 0x00043f15bdbcc383, 0x0007e49d4e4912df, 0x0002c304bdbec36f, 0x000d1b3ac6c72fd6, 0x0002938898b3ea93, 0x000f8bce1c8e5c9c, 0x0000000000011564}}, + {{0x00098a86761cd6cf, 0x0008477faac22471, 0x0009e917079e4be8, 0x0001dd45fbd35ab8, 0x000a45d6d40e31ad, 0x000749ef67e0de9d, 0x000cf2a16c5f1939, 0x0000000000085691}, {0x00046f38fcf548ca, 0x000e7156536ac506, 0x000b8e250c84b9f9, 0x0001eacd35088fde, 0x000538fb995c165a, 0x00093b3a5ce8a733, 0x000e7934d0d33132, 0x000000000008b570}}, + {{0x0002b3fc481df360, 0x0006d6a3a232752d, 0x0000509b75e73d18, 0x0000c3322a82d404, 0x000a14cbc8351703, 0x000da272724bf18e, 0x000d4a319dc4cab2, 0x000000000006d020}, {0x000398eafd6e92bb, 0x00066aeebdd89825, 0x000a73a882a68cbb, 0x0002978595f361eb, 0x000c1ae8fa3cfc8a, 0x000341575e60dd96, 0x000e4819c2109aa5, 0x0000000000006a0e}}, + {{0x000065dd44a81443, 0x00083443deaaabf2, 0x000c83e66800298b, 0x00018ded51d6acd9, 0x00017d4ec9bef889, 0x000c48a6948c9b2f, 0x000f5478b41104df, 0x0000000000078827}, {0x0003f535562e2f60, 0x0004674ed7948c07, 0x0001ccc08940c103, 0x000449b0d9fe252c, 0x0006c14e825a6fe6, 0x000690f0031743d0, 0x000748d9d4ad8c08, 0x000000000000e65f}}, + {{0x000eaf196980dd99, 0x0009692501ff821f, 0x00072b4a2cc10b2a, 0x0009a05e53038473, 0x000c2543d1a995b9, 0x00080877130a72fd, 0x0001456648126b11, 0x00000000000eefb8}, {0x00041f6a86207b43, 0x000d8d9a87f836b7, 0x00016f9bd17dd926, 0x000e118c40122e4c, 0x000f853b291e3341, 0x000565bade513567, 0x000f77d4f1bbb73b, 0x00000000000a7658}}, + {{0x0007b4a08d152bb9, 0x0007cbab37a630ed, 0x00077618410cb7bb, 0x000086c1d4a8f5be, 0x00046581428269e1, 0x00076cc7d3a87384, 0x0003a0b642dc1626, 0x0000000000097e42}, {0x0008ca2aba5729e1, 0x0005b66ac1b365bb, 0x000e67e449b0ae7e, 0x000debbadcc68242, 0x0004b1536fa18f47, 0x0009e546144e9bf6, 0x000b901cfdd50493, 0x0000000000043bb3}}, + {{0x0005a0a5974046e0, 0x000112e32f89636a, 0x000495f55b511bd6, 0x000b1ad8f8328866, 0x0008adcbd933278d, 0x00064f863271d907, 0x0003652208fae34a, 0x0000000000039c89}, {0x0000e167567c2b6c, 0x00048cb46f2725d5, 0x00086f7986cb61ca, 0x000d316edd504188, 0x0004cabe36ed3421, 0x0002efcdab1d8a06, 0x0003eedb13e56036, 0x00000000000c71eb}}, + }, + { + /* digit=30 [{1,2,3,..,}]*([2^120]*G) */ + {{0x0000e8a3d453ef19, 0x000752af8ed809c8, 0x000017d0798a8632, 0x000726f782193578, 0x0001b87254c44c0e, 0x000e7691a8c1962a, 0x0002ee30796a71c9, 0x00000000000a75a1}, {0x000c339094f215d4, 0x0003e535f42c17eb, 0x000b753210a29b6e, 0x000ef35e39a7a591, 0x0002d459b91ab7f1, 0x000fd429da2789ae, 0x000f57eadbca7f02, 0x0000000000065290}}, + {{0x000ffd02d3a50b87, 0x00027c085500177e, 0x00023ee786608759, 0x0001d964318e861c, 0x000604fe9b85dda7, 0x0005e7e2001d3d39, 0x00081cda4bc065e2, 0x00000000000e076c}, {0x000171ac92e482e8, 0x000095b9f82189f0, 0x000cf8881039863b, 0x00083e4d8dd159bf, 0x000720b18043f526, 0x000a0d8f5ca9c888, 0x0005c473c040fa08, 0x0000000000017952}}, + {{0x00023e634e793b49, 0x0000c37ed2be4ce8, 0x000e92823d3628c0, 0x000ad8ae77c2f00c, 0x000a44de16a8b061, 0x000e490ffe87e1a9, 0x0003eddf4f57c87e, 0x0000000000000599}, {0x00036b9f1b67eda0, 0x00020e1036387a16, 0x000cdc81b1b14889, 0x00020d15ab42f920, 0x000dac0ff03359cb, 0x000c1e7f4a738a18, 0x0006e0da501a2e2a, 0x0000000000084d8a}}, + {{0x0000efaf35b1418f, 0x000173a8289046f2, 0x0004fa8840c897aa, 0x000898ae5fa19d2f, 0x00065e1c5fa4c574, 0x000390fc20b13dbb, 0x000187f11343ba7c, 0x00000000000d7d32}, {0x000b8b2ed2cc734d, 0x00011c92cb1bab11, 0x000e6307a3aa4fd9, 0x000dfe5672f2af7d, 0x000430932441da69, 0x000af02d69c62d7b, 0x000c22165672ad94, 0x00000000000cde81}}, + {{0x0006bff8295a2913, 0x000d91d27efcde72, 0x000bdb7b59692a46, 0x000fb83caa26d18c, 0x0003b82babbe99a3, 0x00083dd60d27e613, 0x000cb861030dfdd7, 0x0000000000073c78}, {0x000a3caf05842de3, 0x0002d8707a2cf633, 0x000f47297bf43775, 0x000d412a2b176b8c, 0x000fc72021017c3f, 0x000a6d536d5b52e2, 0x000ec024a63030f0, 0x0000000000004648}}, + {{0x00024dd131928645, 0x00063d45e3501026, 0x000cd2a97f7d8b99, 0x000a088e302483ae, 0x00082c2485c01532, 0x0009cdbe63475e8b, 0x00073808f9ea5696, 0x00000000000253cd}, {0x0002a544f2d5917a, 0x000581cf323a3b9c, 0x00023a0e49f58c76, 0x000e84fc06f3d0ad, 0x000e39efe6d99a31, 0x000ac4decd326b86, 0x00058277e3e1df14, 0x00000000000f3e0c}}, + {{0x000611e8121168e1, 0x0008967477cdc1e4, 0x0003ef00563660fb, 0x00060b9917eec666, 0x0004d66c5c1d31fd, 0x0009508baacd95bd, 0x0002432e0551f3bf, 0x00000000000688cb}, {0x000033e51fe09e4b, 0x000448c039740256, 0x0004cddb41207f6c, 0x00025e144db62afe, 0x000bc4a030da1918, 0x0000815043ee8aca, 0x000cd08ab154a894, 0x00000000000486c9}}, + {{0x0009bbd8fc757c25, 0x00003e92b56bf065, 0x0003d138d6f390b1, 0x000d0ef50f5c483a, 0x000611fa89fe7754, 0x0004ea7d8850a9ef, 0x000a2e97d74b1bba, 0x00000000000aab7b}, {0x00042e268ab251b7, 0x000af06f30ab067f, 0x000bbd0bcb9d997c, 0x0000bca7a2e053e3, 0x0008dcf0e14c4758, 0x000f553108579559, 0x000e18c3596781d6, 0x000000000004c9b7}}, + }, + { + /* digit=31 [{1,2,3,..,}]*([2^124]*G) */ + {{0x00056db726934b05, 0x000d28db8b78ca20, 0x000efbe1df76bc8b, 0x0000f022dde2e3c8, 0x00038cbf406c67e5, 0x000f7ff602e2461f, 0x00068832182781e1, 0x00000000000e30e2}, {0x00008c8748c4086f, 0x0000895adc204b38, 0x00054339345edf7d, 0x000f060b73417379, 0x000fd128d46cd5ca, 0x000fc1e04ed93187, 0x0000a13f2819cb20, 0x000000000002b7f7}}, + {{0x000424a4cdc9ef0a, 0x0009fd74d09c0410, 0x000c23c8eb2570ac, 0x000cfdce132b9412, 0x000c843d3c66db1c, 0x000cb3ef4a0e5309, 0x0001771fbd03a5f9, 0x0000000000000ea5}, {0x0001bb0b6df68f60, 0x000fa1ebf5a155d8, 0x00046a120ff8039f, 0x000c37aa0d34d161, 0x00050fca43af3256, 0x000841bdeee40efa, 0x000a0bc299bbd4b9, 0x000000000003bef4}}, + {{0x0001fdb0e87d4c69, 0x0008f5bbf8bfdcec, 0x0002c8b1b68f641d, 0x000718bfd3fb74be, 0x00015a085196abd2, 0x000dba2ea03c0150, 0x0008bc147474dc4d, 0x00000000000eaa2c}, {0x000a67e6cbd2f408, 0x000e9bc3dc5c2f94, 0x00045e933f00eed5, 0x000e98325d341410, 0x000406fcbc8e0276, 0x00024ebab9372694, 0x00050b8c4b7f2ca3, 0x000000000007dc3a}}, + {{0x000547a274927d68, 0x00075e01295fd654, 0x0003c43252dc8b6c, 0x000689f9eaf8eb82, 0x0005c622acc52ec9, 0x000757f2a327b96c, 0x00006eae918f81b2, 0x000000000004fd66}, {0x000f52e65907ef2f, 0x000a109bb7fcdbf4, 0x0007ab72ec41ee7a, 0x0006f125fb475125, 0x00018e399520df2a, 0x000b1c3a2be88faa, 0x00081f0166d57e3f, 0x00000000000e525f}}, + {{0x0005132af5b85725, 0x0006e0c64cec623a, 0x000d73b377e49971, 0x00043ae5a8027b17, 0x0007d0a7406ac27a, 0x0002bfeed1c5729c, 0x000357445fc7d34e, 0x00000000000809bc}, {0x000fe2c3577b04e1, 0x000a30598626b6a7, 0x00059236250da683, 0x0009b314294839a7, 0x00093ab9bc662151, 0x000ab597466282da, 0x00012e84c422c25c, 0x0000000000071c0f}}, + {{0x00044dd5af953087, 0x000b3f24fab9aa7e, 0x0007e53a0eea7298, 0x00070e733ac4fb68, 0x0002c8a6976575e3, 0x00030aa0ed8d6164, 0x000637f057f10a5e, 0x000000000004f1e8}, {0x000514c4bb67bcdc, 0x0005e887f06b2b4b, 0x00032e66a34587bd, 0x000f7eb6e6483a19, 0x000f475d4bfd4319, 0x000811fc744f1aaf, 0x000f8ba8ee73b50f, 0x0000000000082916}}, + {{0x000b0ecbb3371c13, 0x0009596495c1e57a, 0x0002216cb93b15bb, 0x000c858fe6e84ffb, 0x000b0b189be26806, 0x00058a7685c215e8, 0x000202ac61577e80, 0x0000000000088e0d}, {0x0004f2a99a53c49a, 0x000de70322e164ce, 0x000973bc42c42efc, 0x0001f014d701ab8e, 0x0004a61df25f8863, 0x000f6dbeb889f10d, 0x000168affec5a9bb, 0x00000000000934ee}}, + {{0x0006f789ccd0fed7, 0x0003cdefe3a7abd6, 0x000569da64526056, 0x000c26a73d1f9b54, 0x0008237ae8b77366, 0x00047825935d5d71, 0x000ee33efb20feb4, 0x00000000000aa545}, {0x000d70972e2560bb, 0x00081750edd05bef, 0x000da581b51c4635, 0x0000f370a9e29dc7, 0x0006bfbec7f616e6, 0x00047e1439cf0a13, 0x0001a9b430a34b2e, 0x00000000000c6cdd}}, + }, + { + /* digit=32 [{1,2,3,..,}]*([2^128]*G) */ + {{0x00056f0266d7e788, 0x00065f186d6bc61b, 0x000e03ac4fb95d1d, 0x000432edfe64dc3e, 0x0002d795ea57c9e7, 0x00045af2bd9fc483, 0x000517f4ffdb81c8, 0x000000000002b670}, {0x0004d8950582c931, 0x0007aa7c1be04d23, 0x00045b4daadb356b, 0x0008aba03f2ef6dc, 0x000420dc701d62ff, 0x000f1011960ecaac, 0x000fd09f4b559f4f, 0x000000000003cd54}}, + {{0x000973c51356e0d2, 0x000aa31954a56667, 0x00047c018911cdb5, 0x0003ec5e37ef1d2d, 0x000575bc3b668c43, 0x0009cd98bcd0f203, 0x0002a33723f14524, 0x00000000000ea3b4}, {0x000998df93db2ed6, 0x0000ff607fee05d9, 0x0005662b349502ed, 0x0005d74cea5417fc, 0x000a10fca22bd47b, 0x000635e8b6891940, 0x000fef5cea41332b, 0x00000000000b5934}}, + {{0x0000bd5aa7cff3d9, 0x0003c365a4a426e6, 0x00069eea0aae5d6c, 0x000c8b4b7205a704, 0x000357df815ca330, 0x000a926744858eab, 0x0005d76e522afaf1, 0x00000000000f4136}, {0x0001a59021cca096, 0x0002a00dd3461384, 0x0008629f956ac671, 0x000103f7c082301c, 0x00076b092b71f427, 0x000037a3314a2a18, 0x000e14210f632130, 0x0000000000039340}}, + {{0x00025fc00f86f9f5, 0x000b5b1f4fead323, 0x000e624d9ef69f8a, 0x0007efe2e28aa2a7, 0x0009b3151d6748b4, 0x0006a346070dad2e, 0x000af787a8178b43, 0x00000000000801f7}, {0x000701583b4bfabc, 0x00034fab462e36e3, 0x000ddd2177064fe4, 0x0006b3fe66812831, 0x000a6a3fdc1db469, 0x000e37dac3bd9910, 0x000fb34409128449, 0x00000000000cf605}}, + {{0x00081f556f82acd9, 0x0006f59470e4953f, 0x000963a40f813b02, 0x0003c1e0fb30ecd3, 0x0003b62892f14761, 0x000eefa161fbeffa, 0x000719ddfeef49f8, 0x00000000000daeff}, {0x0004929822ef7d6f, 0x000dcff1872cd89c, 0x0009edd2dba5c5c9, 0x0003a387695218a2, 0x000720802b8852a6, 0x000f4a473a0d413a, 0x000a4a233f1f3118, 0x00000000000c9d7d}}, + {{0x0006ae71b9b2b784, 0x000ca86737912907, 0x000e250552a02dec, 0x000185ee92c712de, 0x0006c201e3272efe, 0x000f6b0788b908f8, 0x000bf33ab5528894, 0x00000000000ce0a5}, {0x000abf1c0844ab25, 0x000185eab37ac7f5, 0x000c6bca8c37680c, 0x000edff304f1cf97, 0x0008f6fd3e90f3a3, 0x00016e5ed8992ecd, 0x0005c4ff24d7c69f, 0x00000000000def9a}}, + {{0x00054a31e739d50e, 0x0001e2edaaaa9c57, 0x000574cc3b6825a1, 0x000fc42bcb161908, 0x00046709d8513542, 0x000af4ce06c04ec2, 0x000e2e5fe9768ea9, 0x00000000000eb6eb}, {0x0005126ebc807472, 0x000c87b7da4abcae, 0x000d07139021766d, 0x0001c51fd563c816, 0x000926c775e17f69, 0x000b58889ea990d6, 0x000c2c2cd96f1321, 0x000000000000f1aa}}, + {{0x000e697116bc35a7, 0x0002057edf71ec8d, 0x00063f77e7793641, 0x000ba1aa6934160d, 0x000c5e639a54b37a, 0x000bce957d45b3d2, 0x0007f362c6b9ad70, 0x000000000005d7e8}, {0x000a8127f71a285b, 0x00017b02169816f9, 0x000c7a7939056bf2, 0x000692b478e4b92a, 0x0009be3fe25a15aa, 0x000bdd4a8dd67cf2, 0x00027af1ef75b006, 0x0000000000041df6}}, + }, + { + /* digit=33 [{1,2,3,..,}]*([2^132]*G) */ + {{0x000455d88f7fc60e, 0x00091a58a27c4fad, 0x000ee65aa7e47d3c, 0x000826600079a263, 0x000f2606caf52431, 0x000cb28c8113f6fc, 0x000aa6f6ff2be316, 0x000000000003c17c}, {0x000c05f668a9dc76, 0x000798ea577e0148, 0x000c912137590c65, 0x000eff4592a57ef2, 0x000c5e3f67b24b28, 0x00000890276e1f87, 0x000b7e40d7a676fb, 0x000000000004b6e6}}, + {{0x0007f95d338f5fb6, 0x0000076e27519dec, 0x000fefb9477deeec, 0x0002dfb71fba6625, 0x00085e0886af5832, 0x000ee544c228aec0, 0x0006398d83b6276a, 0x00000000000e29f9}, {0x0004808391599977, 0x0006d1ba4cfe02e0, 0x0000441e3d4b3e92, 0x000ed58cee95d92c, 0x000abe5355407e0e, 0x000a9a1f42d29282, 0x000c82866638b607, 0x000000000006ab8d}}, + {{0x000c43ee7094e067, 0x0000b599ece9688c, 0x00025bcbe937baac, 0x0005ba86373ccef5, 0x0003c64f72612afc, 0x0004ce1abe455683, 0x000a77f29ad54041, 0x000000000003ee82}, {0x000e0b1548890be5, 0x00094fa26eaa0940, 0x000c96b0a3e57442, 0x000ade54e1c94bd7, 0x000e20b0ddbe9570, 0x0006eff6ff4a86d8, 0x0007a4967ab653dc, 0x00000000000e8bab}}, + {{0x000de87dcf2f3d8e, 0x00056a16c79cee01, 0x000db47cfbc9e301, 0x0006aa0c2f47e69c, 0x000f2c9b4914b8bd, 0x0007242a8277d590, 0x000512b6d1c81009, 0x0000000000045f75}, {0x00044d25d22dbf16, 0x000f02176162aa27, 0x0001167bcefd598c, 0x000c0d1a503aee8c, 0x00057af4dd78fcf8, 0x000b43be45d1f94a, 0x00071b8f0bc1a62a, 0x00000000000ba9e0}}, + {{0x000adf4d6bef5a3d, 0x000a19a2dc376622, 0x000b08bb2e394780, 0x000e5cf7c0765207, 0x000c4063c3538e70, 0x000baec46c0dbb4a, 0x0001ad3f4550566a, 0x00000000000f9356}, {0x000e15a92e3f32ae, 0x00055d0bc6d91fc6, 0x00092d1d7bd54520, 0x00033424b80c021f, 0x0007af5e458c62a7, 0x000bfde586e1d546, 0x00052c6922574f70, 0x00000000000d19f0}}, + {{0x00077ca19b6937db, 0x000c3bf87f30d9c2, 0x000f3d29bdf61e62, 0x00032bd50ec0ba46, 0x000ae60d1e72d5fd, 0x0004ff2d37de8fe0, 0x0007e452d77fe0e1, 0x0000000000055ca4}, {0x000fd784e6c9781f, 0x0000339e3d19e6d1, 0x00045fa26fa441a0, 0x0000c7afe92c6d07, 0x000869a829043bd7, 0x0006d1d9ea3fbc06, 0x00024be263f00ed9, 0x0000000000075c0c}}, + {{0x0002619f4e7dcc16, 0x00090a1d1a8f0893, 0x000f4151f0e91898, 0x000c39b84ee41fc5, 0x0002e4a019099db2, 0x000a66c12ad292a0, 0x0000f6ee6b26ddfc, 0x00000000000f04f7}, {0x000e18e3a8161ac5, 0x0007f01e66788b92, 0x00099db080ad62ab, 0x000e4542595d5a6c, 0x000bdce965cc3e7c, 0x000ec7931f894a64, 0x0005be12e46952d9, 0x00000000000fb65b}}, + {{0x000d91fc53fa82b7, 0x0007d705bfe3760e, 0x0006eb6118c41a96, 0x00077a9bc4567977, 0x000617081a2f811e, 0x000e1640e4f52c4e, 0x0001d787405d819f, 0x0000000000070771}, {0x00085581b6dc5925, 0x00062dc6fff82580, 0x000af03b55b43c90, 0x00066bea9cf3eae3, 0x000e87139b8b0ecb, 0x00042b4d77418372, 0x00032e6aaccf295e, 0x0000000000031fc9}}, + }, + { + /* digit=34 [{1,2,3,..,}]*([2^136]*G) */ + {{0x000f65dda78f831c, 0x00073fa3b4c8e10f, 0x0006b8117d5cb12e, 0x00039562e8c4d8cc, 0x0005cf89935b06aa, 0x000fa071e4981c3e, 0x0003bebdbd0c4745, 0x0000000000049607}, {0x00004bb91c157448, 0x0007009a7298688d, 0x000867eb798cb22b, 0x000b5b3988f2781c, 0x000d73b1719d9a64, 0x000d2e8076ac9440, 0x00042b58ad54c7e3, 0x000000000008f067}}, + {{0x000489635dd868b5, 0x000ef339521774bb, 0x0001f606d4b5774d, 0x000d61e0a285bd5e, 0x000e71171b1c1084, 0x0009b29f93935a84, 0x0009bd8ac2433cf2, 0x000000000006dd1e}, {0x000f576ac5f0cc26, 0x000f788da0477c71, 0x0007313f812b64cc, 0x000a25f9d5b19e1d, 0x000b6a27fa79a792, 0x000a16a8e9ee015c, 0x000e67ea3bf8b57b, 0x00000000000c15fd}}, + {{0x000f3b091fcd53e3, 0x000c537f50e43695, 0x00003782e79d52fb, 0x000af85e1d111511, 0x000f6785ae1c3916, 0x0006ada8cf56e852, 0x0005b2bf8c72adae, 0x00000000000e1328}, {0x000cbe0d0bda153b, 0x000e35327920cb17, 0x000e7daa3650306d, 0x00028573caf37928, 0x000b550ffe1ca713, 0x0005d4e4fb15ab34, 0x000f9d818980666a, 0x000000000002f854}}, + {{0x0005d9265089916f, 0x000013b623150146, 0x0007ede4f0fb2f49, 0x000c31c56471100e, 0x000b1bae8d2a4bc2, 0x000f1f76a4a0e73b, 0x0006cabfc0770a8d, 0x00000000000a7bb1}, {0x00039cb13d9c4b7d, 0x000f42afe5b1cb58, 0x0008a2ce9b2dade4, 0x000a333a1af2a824, 0x0005dafed6cd97ca, 0x000c3b2e393cb92c, 0x00095609553e7e92, 0x0000000000061af2}}, + {{0x000ea55edca2a058, 0x0003f5559dd3109c, 0x000f7ba60d37c6c8, 0x0000fabcaf57d0dc, 0x000578742ab60d84, 0x0009c1c8e9625866, 0x000167c85482ad40, 0x00000000000adaa6}, {0x000208d39ae67d2a, 0x000dcec26ad9971a, 0x00067a35ccc54689, 0x000a7fc2539986bb, 0x0006a23dee41340a, 0x0001a9837d767487, 0x000da9a948a9292e, 0x0000000000071438}}, + {{0x000778a60c6bd7ad, 0x0007f39038863eb8, 0x000e85efa03ef35f, 0x000682ef3d8310f4, 0x0008f412315338aa, 0x00026310bb52d41e, 0x0006924fbef3dd70, 0x00000000000f6ff5}, {0x000ba50eddf2b7fe, 0x000b63831c6b1cbc, 0x0003bc6684c6c5e0, 0x0006abc6516bba59, 0x0005446d35a876d2, 0x00012a8d0c237f9a, 0x00015bb2b16c0ff0, 0x000000000000ee03}}, + {{0x00014182dba86bd2, 0x0009c817d77b02b0, 0x0006420950654aca, 0x0004107da68b7691, 0x000dbebc4c4dd3ac, 0x0003d39e96904bcd, 0x000950b0d2103ca5, 0x0000000000030a5e}, {0x00028ff31a9f909b, 0x000c7b092568034b, 0x000262a60542e8eb, 0x000ab34c15855ae5, 0x00063017194f2389, 0x00046b838c14dfd9, 0x0006fc420e071911, 0x000000000008fb4b}}, + {{0x0009c78614d38ab4, 0x000d813722ab0651, 0x00088626a03970e0, 0x000f467f76fdaf74, 0x000912ddfd9ad3d0, 0x000bdefd37ce072f, 0x000315ce918a5747, 0x00000000000750e5}, {0x000fa00e65975639, 0x000cd08bbb20dcda, 0x000822e7b86b49be, 0x0005c21ca865ba6a, 0x00002e6e8e6fc8fb, 0x000608b60ee6e41e, 0x0005cdd00ae6214c, 0x000000000006ff68}}, + }, + { + /* digit=35 [{1,2,3,..,}]*([2^140]*G) */ + {{0x000cd00e20fc1fe8, 0x0002bbbccce39826, 0x0009c6cbc7ade77c, 0x00035a5d2252fdaf, 0x000954e7dd499eae, 0x0005100fda8f4f20, 0x000727a56d72a629, 0x0000000000056767}, {0x000898f026420cbb, 0x000adbf60b247e57, 0x000b35db15577b1e, 0x0007ad4b93dab9cc, 0x000022d71f39c2a6, 0x000304db218cd0ed, 0x00082104380c425f, 0x000000000006729c}}, + {{0x0000345995afd46f, 0x0009dca07923b790, 0x000129149d2f3565, 0x00079e83cb025114, 0x0005beb383ef41e3, 0x00076dd0dfabac00, 0x00012724d12d9a10, 0x00000000000b208f}, {0x000d58cd6475c579, 0x000359cc38a604c8, 0x000857e410d47fbc, 0x00060d98ac219eff, 0x000faf284c806f63, 0x0009e366a1edaab3, 0x0007269c3b528101, 0x00000000000bc9e9}}, + {{0x000a386526319283, 0x0008d26bd07d697c, 0x0006e6a9c305333a, 0x000c5466798b96c3, 0x00081d3f3859d831, 0x0001b9ec71d6b410, 0x0001a9501d38ec99, 0x00000000000d7843}, {0x0008f62f7d623e0d, 0x000dd8b85baf6942, 0x0003f90bead64135, 0x000c1107acdcf58c, 0x000c848d5842efc7, 0x000c7fdacd9af415, 0x00019b6b5a06bc0a, 0x00000000000a3443}}, + {{0x000517cc17c08a8e, 0x000a28410d82975c, 0x0000ae8bc362b8a4, 0x0000d0d18c253486, 0x00007eb035a3ae46, 0x0001144145d0279a, 0x000f7987e7c1289a, 0x00000000000c0744}, {0x000ccad1ad801112, 0x00072b7b2b4f054d, 0x0007f502703051c0, 0x0007395b51ee6864, 0x000bf6122422124a, 0x000f1a7fff2937c4, 0x00032eb4ec133207, 0x00000000000f8860}}, + {{0x000daf15d19f8632, 0x000794c0d78053af, 0x000ade8ac0a0ca73, 0x000c453e4b57e236, 0x0005f4172b285217, 0x000f999f8b4669e3, 0x000d41509a3cd049, 0x0000000000087c69}, {0x000ec5ba18211916, 0x00032e14d01e8346, 0x000c499a14031eb7, 0x000bff270dc2bf04, 0x000bc01864000e17, 0x000a4dd3446560b7, 0x000d06e28c7b9c49, 0x000000000000f325}}, + {{0x000161555383d319, 0x000d0dd4eb3d0283, 0x000dca801a8bb250, 0x000285ddc1973c7e, 0x000aa0046b981200, 0x00019a7fcfd342be, 0x000fe5d191734912, 0x00000000000b7caf}, {0x000a5ee9f805422f, 0x00057ea5a0c4d360, 0x0004c8206c9d4abd, 0x00016dc6046eeeb1, 0x000ecc8e64b15b2a, 0x0007fadda1755e1f, 0x000ec251e4946e9e, 0x00000000000fcbf4}}, + {{0x00024bd4d62de244, 0x0001d46d7088801b, 0x0002ae4b99b01f02, 0x0008fca4fbea2725, 0x0005bd80a9dfe494, 0x000871c717d6c776, 0x0005701b470ea6f3, 0x000000000008330a}, {0x0004db08d904e89f, 0x0009bdaecddc0ed5, 0x000f26cdb2e08463, 0x000cd84caeeef145, 0x000fc685a35656c7, 0x000d93ddcc60c910, 0x000895e68bc5c59d, 0x00000000000feb64}}, + {{0x0001bb5474d7445b, 0x0008af877e8c6483, 0x000948d44b23fa45, 0x000269f2b004cfe9, 0x000795450f8b19ff, 0x000a7d4588adc5d7, 0x000fdc688ce8bce2, 0x0000000000097bd7}, {0x00043a2684d27187, 0x0007c1b9f4ad5bd1, 0x000b255e8d74e0bd, 0x0001f7e71d83d86e, 0x000d25ffc219abee, 0x00073f553e693c76, 0x000a551c9a84afc8, 0x0000000000066d77}}, + }, + { + /* digit=36 [{1,2,3,..,}]*([2^144]*G) */ + {{0x000e4a9a609d4f93, 0x000f05584cbb3289, 0x0002a9b59e61225b, 0x000d8267df2d43de, 0x00000109e8014126, 0x000172f1cdd5bbbf, 0x0000d985b92ee338, 0x00000000000f3143}, {0x000ac2d50dd03701, 0x000b11e059a07dd0, 0x000eb68a6d1ce296, 0x0000751560f20e77, 0x000e7aaf3a9ad622, 0x000bae14ea59489a, 0x0002497b70e2f664, 0x0000000000076d08}}, + {{0x0001e7720f69ad57, 0x0004baff47822226, 0x00011f4e0e8fa5c8, 0x0005f8d2ef696aa0, 0x000b4fa94fcabeb8, 0x000d41a438ce5baa, 0x0007720a32f96200, 0x0000000000074f6e}, {0x0005b3ae79f59da9, 0x000a8be0221aef2d, 0x0005793443a4452f, 0x00085d6a7e49f3a4, 0x0009cb5d3c6378ff, 0x000eb65f56300658, 0x000bdfe8e4383596, 0x00000000000ff8ad}}, + {{0x000ec83ab52a5d83, 0x0006cf6fd26cb934, 0x000ee1af72ab19a2, 0x000788be372817f8, 0x000ae8c90c31694a, 0x000aa0ce585f3dc1, 0x0002238c90c6bf15, 0x00000000000f01bb}, {0x00043ef688d7f41f, 0x00075bdae01dd56c, 0x000cf79ad5ecb5f0, 0x000ff4ec2548f251, 0x0008d12802a750dd, 0x0004d924054a4986, 0x000d1acd922fb640, 0x00000000000957f6}}, + {{0x0006a17488184bb0, 0x0004d082ea61c88d, 0x0004b5a68e9c5821, 0x000df962e54f5d00, 0x0006e39dc6ab7d33, 0x000179b13340b0d3, 0x00088cd97a8b848d, 0x00000000000288d3}, {0x000bcf17110adf56, 0x000462237e507d00, 0x000fa28fb932260c, 0x0008bcb42be74594, 0x0004de2176b6645f, 0x000e2f09cce2b0f5, 0x000001af570a09d1, 0x0000000000057fdc}}, + {{0x00009414087acee6, 0x000fc7dd81467f3f, 0x0007997c546fe735, 0x0004e832d2e9fbfb, 0x00061c9d9c7ca2ae, 0x000f27f140f127f6, 0x00029ef06c5a57e6, 0x000000000001f303}, {0x000b431a1206d910, 0x00037a1b2b82f82b, 0x0007312610c2e220, 0x000cb8b039f12bbb, 0x000902b92d6b1cf0, 0x000f649e606ce433, 0x0006eccb5e826869, 0x00000000000e9309}}, + {{0x000f672480aedb22, 0x000c79d21c740037, 0x0001a34fe6cb5f73, 0x000b1a3aae5d701b, 0x000891978d1557f8, 0x000b85b1e477e4f8, 0x000fb42913ffb40f, 0x0000000000058489}, {0x00047efe2cde596c, 0x000a60e1ab266dc3, 0x00073cc3adaf7198, 0x000ccb657ab8d871, 0x0006547f64bdf578, 0x0003e497e339fd79, 0x000706904693943f, 0x000000000001d864}}, + {{0x0009da8454e1979d, 0x00030da92e482d7a, 0x000b666249139a91, 0x000ebb3639c78714, 0x0009d77aac3c8763, 0x0002769a43bf7177, 0x000eda92a5420d3f, 0x00000000000d092e}, {0x000352bd3a0254a9, 0x000b5d7bd8d8f8e8, 0x0009446f6266c164, 0x000fa3ed6b0cf872, 0x00040490a771a5ea, 0x000ff37225255afc, 0x000cdbd40bb0fd15, 0x0000000000079294}}, + {{0x0005908307814aa6, 0x00009158bfe27b53, 0x000df35ce707d624, 0x000153b8e71aca19, 0x000171b11643f07b, 0x0004e2f905e67698, 0x00010977b7dd7dd0, 0x00000000000f0dcf}, {0x000bd23edf0138e0, 0x00071d3bab2a41f1, 0x000facf3129d6b1f, 0x00099afc6f53cda2, 0x000f628039e454fe, 0x00083aa16071f2a4, 0x0006f9f1f44181b1, 0x000000000005010f}}, + }, + { + /* digit=37 [{1,2,3,..,}]*([2^148]*G) */ + {{0x000b28d4d03ca18c, 0x000a0b1e36500cc3, 0x000e44ebafd13e2f, 0x0001d2ca4bfdcedc, 0x000fb29cdcfebf45, 0x000514871d210892, 0x0001e35b12bcd894, 0x000000000001809b}, {0x000c86d0fbcc5e1a, 0x000ce09b243105c8, 0x00037f6b6be14182, 0x000a16b5de334b63, 0x00020b116076cef0, 0x000ae2bf4fe0bd1e, 0x00093754e4b48289, 0x0000000000068c8a}}, + {{0x0007e2c503fdc14e, 0x00049309b1eb33af, 0x000b5e3acfac1d05, 0x000b81ebf8a894ca, 0x000e0d29329387c2, 0x000bf371427a40bc, 0x0008957f4c315be4, 0x0000000000001236}, {0x0002d22bfd1d81d3, 0x000f4319c88f7e2d, 0x000189361ce75d22, 0x000a05df0dd13811, 0x00024acba9fafcc2, 0x00027313ec8d55b0, 0x00094a871358de59, 0x0000000000017ce2}}, + {{0x0002c99cf8ff48b8, 0x0002410777188c8a, 0x00068c821fc35883, 0x000b515120380e77, 0x000fe13577d1261a, 0x000862db1453c858, 0x000e1cb1f6bb58f3, 0x00000000000b9529}, {0x000338bb0bed7b45, 0x0008003f63a416f7, 0x00081208dbc793e6, 0x000048b756e5af2e, 0x00031c984bef8423, 0x000a00f3e4d978ed, 0x0009e7a06242995b, 0x000000000004f4d1}}, + {{0x000e08308825a639, 0x000e8d3797831773, 0x000f79843c567224, 0x000a6a611a4e33a2, 0x00005328043a2ff7, 0x000e9f7dd904f86f, 0x000904e29d31c012, 0x00000000000c0a51}, {0x0002dae695c951e1, 0x00024070c2696563, 0x00060638255bc0fb, 0x0000d0c12c8a1f65, 0x0001ab352a835b97, 0x0005eb0c7572aaaa, 0x00059963df45a90c, 0x00000000000d4977}}, + {{0x000fe53cf0bc7d5d, 0x00073e35daffd3d8, 0x00005ab0f149d24a, 0x0005a40f009a48b1, 0x000d308a81c693f0, 0x000885426a4a801f, 0x000f0575e5dc467e, 0x000000000004629f}, {0x000c6c457ceba67d, 0x000c6356b879b5ee, 0x00084a5c7d7e4e3e, 0x000856eec2dd55b5, 0x000880f0c80411c9, 0x00001b21720f0443, 0x0007b8c0b5e3e218, 0x000000000006ea09}}, + {{0x00042a359a9c02d9, 0x0006601c2df8ca11, 0x0001ea46897d0b3b, 0x000341c8360fa6b2, 0x0002c52bb2d6e198, 0x0002efba5e67f809, 0x00032af944dc63a0, 0x000000000002c123}, {0x000d5d58228e0e7d, 0x000b239684f6c863, 0x000f4f910b494aa3, 0x0008eb646594725d, 0x0005793c32ddb7fb, 0x000f94b55bb4f5f0, 0x0002773ef3c33845, 0x0000000000009eac}}, + {{0x000bc0df5aa0ebff, 0x00046efd26dca17d, 0x000d31eadaba6e9d, 0x0008a89e1830a96b, 0x00013e039a029f10, 0x0006fbb3b7e8e368, 0x0002f11747b3e925, 0x00000000000abb3b}, {0x00023cb577b95e94, 0x000e2cac5818c280, 0x000b36c4a5c24e15, 0x000fd4d7a5485367, 0x0009aa3645074081, 0x0001a5f81fe2d8e7, 0x000db4e86ce00ea8, 0x0000000000077a9b}}, + {{0x000a3bf39d563e4f, 0x000f02c5b0e421f4, 0x000bae31a917643d, 0x00085959aa907285, 0x000af658699bace4, 0x0003b18e632be886, 0x000667ce75d6c6da, 0x0000000000069caf}, {0x000af371b713c401, 0x000f0c17c66ce4f4, 0x0002f4e783050dba, 0x0000041623db4f0b, 0x0002c74762e1ceb8, 0x00071c52fe75615b, 0x0002ecade8a54386, 0x00000000000cacaf}}, + }, + { + /* digit=38 [{1,2,3,..,}]*([2^152]*G) */ + {{0x000ea5d38989c046, 0x000c5a933aaf71af, 0x00084b51a5d47afc, 0x000dff8854de4972, 0x000b247bec1525a9, 0x00061e58da8b31d9, 0x000707468a25c846, 0x00000000000786a0}, {0x000d126f8d197fbf, 0x000be282db8ca2e4, 0x0000d8a3ccd2e3a9, 0x0000faaeaeda06f0, 0x0009add94b47a2c4, 0x0000690766963292, 0x00092cc72354f6b0, 0x000000000007878e}}, + {{0x000456ff7fbc201b, 0x00083854b0583e19, 0x0005b9f9d05986b8, 0x00093894c32fc71b, 0x000c9ec8f90dec82, 0x000c7b9c0882fad4, 0x0005e52d39990dc6, 0x00000000000d71a2}, {0x0003262dabc3b450, 0x000866b852e64a5a, 0x000281968ae95022, 0x00011545857f0497, 0x000c1ccb9bc83700, 0x0002bb853746621f, 0x0004ed97e44e6361, 0x00000000000758da}}, + {{0x000e66dca895aedb, 0x0007e58775856e71, 0x0003edbdf1471e8b, 0x0002e3da62265d35, 0x000672d98b0886a1, 0x0001e9c858ec4278, 0x0004ceb9da8016f6, 0x0000000000080099}, {0x0007a34c46f751da, 0x0004f63d0878c9c9, 0x00077928ee65f2b7, 0x0009e126a1c1efae, 0x000eebd0497a780b, 0x00065231eeed68d0, 0x000127bfeee3d292, 0x0000000000080e03}}, + {{0x000369b381ff008e, 0x00068d25f6507829, 0x000435b503d33f46, 0x000031108b9b08bf, 0x000f3fe92e910b36, 0x000189ddc16477af, 0x00038f6e5f6cb103, 0x00000000000c698a}, {0x000953f049518733, 0x000e102a092187fe, 0x000b4e74068daf16, 0x0005eac8fd6b76cb, 0x000611ef96b455a1, 0x000e433b51e37ec3, 0x0004c3d1b3b3fc30, 0x0000000000051d17}}, + {{0x000c3efd55e9f108, 0x0004ee67813dd55d, 0x000e8b95d557829a, 0x0007b634c8cf1647, 0x000fa3556b4674ea, 0x000db03dff1bde0b, 0x0006b45343f260c1, 0x00000000000e0a1d}, {0x000557bdaa85b25c, 0x000b5af56bed0543, 0x0009694c640e2d2a, 0x000c5892c72fa801, 0x0006a49486e504e0, 0x000f78943812e259, 0x000431d5e0bddb2e, 0x00000000000acdba}}, + {{0x0005b2c1b41a7e30, 0x000e8b6e95494a98, 0x000e156b8fa7f1c3, 0x00012ee2183ce113, 0x0000cc01d1434741, 0x000d0d25f4180ec4, 0x00016ddc5f8f7b8d, 0x00000000000974a6}, {0x00054a8b6ee62d01, 0x00072b9a7f0a96a9, 0x0007a0f1f81abc8f, 0x0000b82bc5671a8c, 0x00000466ffaf50eb, 0x000fdc348fa58667, 0x000299a75ff5aab9, 0x000000000007f784}}, + {{0x00064348b9a55592, 0x0008f24b18ccf351, 0x00046d73b67eefc8, 0x000977c532d340c4, 0x000191002448043a, 0x000960397a2de526, 0x00034e1e11027870, 0x000000000000164e}, {0x00056330f30da4d6, 0x00014cc5f57288f2, 0x000974f0c19f8ace, 0x00020963266aaedd, 0x0002c3ccd59f3b15, 0x000a6dd5dfaea30f, 0x00035a1da2e1fbc9, 0x00000000000ceda8}}, + {{0x0007996f48325ebf, 0x0004f7914213d709, 0x000270acf84435ed, 0x000e5a126a34238c, 0x0000701ce8c76eda, 0x000656e02e566bf5, 0x000fbf3562e87555, 0x00000000000b4e8e}, {0x0008d6657de7885e, 0x0004a5f10a503e86, 0x0006a10baa0b8f13, 0x000fc25cc2f2e415, 0x000caf5718c149d1, 0x000d6b890e973bb1, 0x000f129b8825dad2, 0x00000000000e2f00}}, + }, + { + /* digit=39 [{1,2,3,..,}]*([2^156]*G) */ + {{0x0009cd8cd3edcb0c, 0x00022e37211bdab0, 0x000bfe0383f52218, 0x0007e26a1b9f8b57, 0x000d7d7f72d5fdcd, 0x00049c9205641e45, 0x0002a15377c1bec4, 0x00000000000efc7b}, {0x0007344da1d40eaf, 0x000b6c657a9ff3f7, 0x000acc6777a25729, 0x000d1bcd020eaa96, 0x000a3f6860c76bfc, 0x000c7c80617534b0, 0x00056b8ce5722284, 0x000000000002bd74}}, + {{0x00070f4fca1b2907, 0x000d2aed02b6a844, 0x00048854f708389d, 0x0006ec4654e7c314, 0x0003e7034bfd8222, 0x000d6b555008ac00, 0x000d44e343c5407f, 0x000000000001b429}, {0x0003fda90bb8f0f9, 0x000bce0702a33908, 0x0003c08ba27edf85, 0x00072f6d46524015, 0x000d35f600437e6d, 0x0003e8cc4d92655e, 0x000b40a0dbaac627, 0x000000000003c3ff}}, + {{0x00098d832fcb2cab, 0x00078bc06ecab0e1, 0x0004e7cd0ece1448, 0x0006fa0453c94bf2, 0x0003ed1a6731a6fc, 0x000c3f1fb5460f94, 0x000d4eeb11656a4e, 0x00000000000ff1e7}, {0x000efd2eb43b2558, 0x0009059526466dba, 0x0007bc3cfc024713, 0x000588824fd2ce63, 0x00039f2c29257bdf, 0x000e97f3013df0c8, 0x0006411bf621e6de, 0x00000000000ebea6}}, + {{0x0003b015d907e90a, 0x0004e5906a35b43f, 0x000f32388b4d1260, 0x000ac9136f847648, 0x000bbe0f1dd365c8, 0x000d26c21f73b3de, 0x000ae740358868a2, 0x0000000000076792}, {0x000fc16ec80c7bf1, 0x0008d3ecea1669e7, 0x000b1ce4e42f6130, 0x00091090e0062443, 0x0006dd94681b6db8, 0x0005a3de2d29106e, 0x0006ccb40b8694a8, 0x00000000000936b8}}, + {{0x000fa93eb46c6ed1, 0x0009f28d33e792c1, 0x000af8e666ab1b38, 0x0009f3bce683c5c2, 0x00098371fe755a74, 0x000712c1d717629d, 0x0001aa5e828fc057, 0x000000000007e4c6}, {0x00082c4505e4cd17, 0x000035d927bad55e, 0x000bbbc997dd1436, 0x00099a398591dc25, 0x000a4836664c560a, 0x000d79298c885fe8, 0x0001d7d18acd4226, 0x00000000000185df}}, + {{0x000755a507c76d63, 0x000c69d8a925b591, 0x000ef5ac5d730610, 0x000ca6ddfb534b8b, 0x000c6dd78a324f53, 0x000a146d54e64874, 0x000201336e5b46c2, 0x0000000000098395}, {0x0001a5ef82624226, 0x0004ca4c095220d8, 0x000ae0a7c3b4840e, 0x0002f64c36286ed0, 0x0003c3ebb08c0ff9, 0x000f00c3057b1b90, 0x00036ec6bdc9b665, 0x000000000009a46d}}, + {{0x000c3639d049078f, 0x00048d67dc92ab51, 0x000e52783edc1242, 0x0009baa8b87c0b05, 0x000052760ef4b6f0, 0x00047bf855a8a903, 0x000e742e2ae75610, 0x0000000000085bd4}, {0x00097e1bd11078ed, 0x00093ced11ff7661, 0x000e3244d9aa20e1, 0x00088ff24b3e912a, 0x000a1afd219683d4, 0x0000d21286e166fd, 0x00039c66a912114f, 0x0000000000005c9f}}, + {{0x00070dd4c4f3a35e, 0x0006d913f92069a9, 0x000aaa1c8b2107ab, 0x0004e2c787c35959, 0x000fe7b7b7ddefab, 0x000e28e6aa55d465, 0x0007bc921aaad834, 0x0000000000012d6a}, {0x000d81dd493823d1, 0x00072109803c417d, 0x00095b2d5e30421d, 0x0008bdb99c5bb670, 0x0006bc7c2da71a8c, 0x000927eef1cd1c2b, 0x00041050189c975f, 0x00000000000229f9}}, + }, + { + /* digit=40 [{1,2,3,..,}]*([2^160]*G) */ + {{0x00086c2a1c05c1ca, 0x000a911a8fde5d4c, 0x0008768c091692e8, 0x000c275c74dfe82d, 0x000c38373a506818, 0x000e5f88f2b0294a, 0x00083a584c4061e8, 0x0000000000073423}, {0x0003f61270e03ada, 0x0002d263895b3203, 0x00007b3b0d15f74f, 0x00059da84da7f0d6, 0x000a924a09f21443, 0x0009ad83576e3095, 0x0002af612986e3d6, 0x0000000000039212}}, + {{0x000c342a3198c068, 0x0005d3ee0a31c35b, 0x0003fc2cd3c80f60, 0x0004f4ff0fbfa963, 0x000efafea65a90a6, 0x000f4b9f9513f054, 0x000590796ba7479c, 0x00000000000932a9}, {0x00037547e06b6b4e, 0x0005d93af8a64743, 0x000922627b827de1, 0x00071e9c1a46c909, 0x000f4c9bca7223bc, 0x0008cea30000643a, 0x000ec2b6d967c784, 0x0000000000073312}}, + {{0x000ecb2626cc0655, 0x000eafeb707ee40d, 0x0001edb8dca9d02f, 0x00016459a0d32bae, 0x000a1ffb926626de, 0x000578b4a9a2ff38, 0x0004434e5ad96d8f, 0x00000000000883e7}, {0x00041045bc252635, 0x0002d1d8eecaa72c, 0x000edd696d444309, 0x0002faed758ae41f, 0x00004a85d867e49b, 0x0008cba866a9a229, 0x0009822fb89dcee6, 0x000000000007f09a}}, + {{0x0009f15eb1bbb5c7, 0x000d952ec99556bc, 0x00015795c8a4dc35, 0x000337c6dcc7816c, 0x0009791f7cf1e881, 0x000885a42e4e7b6d, 0x000e41faa717aa59, 0x00000000000f9c01}, {0x000171c4ffe4a6bc, 0x0000ccf208d57a05, 0x00042714fd20944a, 0x000871b264ce04b2, 0x0006cd42026a7261, 0x0009f99fe66be4a8, 0x000e2bc5397b7782, 0x0000000000024578}}, + {{0x000b87e03bf622cc, 0x0002fadc79436506, 0x0000be1fdf9a8888, 0x000aaa40406c1296, 0x000c8b658d3cef9f, 0x000435baff4e6388, 0x000c997262beb41e, 0x00000000000fdaea}, {0x000ba84e0b8f458e, 0x00045d359f7d8428, 0x00016b951ae31b5d, 0x000f03229498ba51, 0x000a85bf35adb18c, 0x000aa4bc8c67388c, 0x00027a8a7a6aa262, 0x0000000000072f46}}, + {{0x0001e4a772f0f152, 0x000520733667a853, 0x000ae90cf08b2488, 0x0009b27a7bc26604, 0x0008eb4f0c7a7ca2, 0x000f276309fefa69, 0x0001337b6f351301, 0x00000000000fcfb1}, {0x00022648d86fc4cc, 0x000979928d9cf841, 0x0009da8838c5ffbb, 0x0001acbe041bef47, 0x000af8d998547fbd, 0x0005a0dc57d25159, 0x0009332ec3a7d8db, 0x0000000000087e3e}}, + {{0x00088fc716a5b4c1, 0x000423fb812eeafe, 0x000a6b9f65779527, 0x00041838fbefbe46, 0x0004a4b24a05e572, 0x0000c0a432b7d49f, 0x0001db23b138d071, 0x00000000000a85ec}, {0x00095de9d8768d19, 0x0006d7cc1f3d657e, 0x000b6e219733e2a8, 0x00023128f56981d2, 0x000eb4080a011bda, 0x000999b7f68663ea, 0x00080c8dd7ba7e9f, 0x00000000000b6865}}, + {{0x000b2ea58bc5b2ba, 0x00008ba8418f2c05, 0x000ab2d74f8719f3, 0x00038d7bce1e82e0, 0x000eb397915b4e64, 0x0004cf6599e489b3, 0x000affa9baea9ffb, 0x0000000000042ef4}, {0x0001f2cfd470aeb9, 0x000eeae3fb532a9f, 0x0001d5334d1ddf3c, 0x0008e28defe047ca, 0x0009d24e60fe8972, 0x0000f710c63a8c67, 0x000d3cadacbf9247, 0x000000000005e198}}, + }, + { + /* digit=41 [{1,2,3,..,}]*([2^164]*G) */ + {{0x0008597cdef10944, 0x000f4294d29542ec, 0x00058a58394c4343, 0x0007f9158f95038b, 0x000164420c94fa1a, 0x0003caea8b137981, 0x0003402b686e1e09, 0x00000000000f9e1c}, {0x00006da6276d2e4f, 0x000cacd1beecf39b, 0x000bd69a9fc92254, 0x000c7192dfc2b165, 0x000dcee7e854cecb, 0x000635cc82cdb955, 0x0007d39fefc321f2, 0x000000000002936f}}, + {{0x000f165eb19500c6, 0x000f54962c020b57, 0x000116eb855d7c76, 0x0009a79dd189c401, 0x000d6ce517a77854, 0x000d6bb9747675cd, 0x0000295102294e32, 0x000000000006ed1a}, {0x000926ef8530f97d, 0x000353efb54e82c3, 0x0002eec5a292f22f, 0x000c2948967fb050, 0x0009be04e909955b, 0x0001efad6373615c, 0x00097af1fcf82011, 0x000000000006fd2e}}, + {{0x000445389cf3da63, 0x0002b85e1e4ab383, 0x0004d8f24163478b, 0x0004da1a717e0356, 0x00080f1bf3dfc0b8, 0x000a6fd41f1a63b2, 0x000cab47b74201ab, 0x000000000003518f}, {0x000a17bf41f156b8, 0x00031e12511fae1c, 0x0005a421fc7a58f4, 0x000fb10db5665c05, 0x0003b99f4d5a20e0, 0x000b7dbceb5448a3, 0x000b87cb1847ad46, 0x00000000000fdcad}}, + {{0x00040c75d27c479b, 0x000823f2b5bd3e20, 0x000298ef45cbe8a6, 0x000f2db94256fe1f, 0x0009318ddb03532e, 0x00024c8e1acbfc45, 0x0003db2375f9fd5f, 0x0000000000037078}, {0x0002335b7531e78b, 0x000251bd461e7e12, 0x000e223c32f4b08c, 0x000777d8845f315e, 0x000697fc92c7c9f8, 0x00092954081aed29, 0x000fe09f2f8d7949, 0x00000000000ff5eb}}, + {{0x0004fbb34790cc07, 0x000a4397049f47ff, 0x00000e7e84e96f9b, 0x0005ec1e7862c0cf, 0x000834350bf1ce6d, 0x000ec8d7417a98db, 0x000aa86ceccd8030, 0x00000000000bcab4}, {0x00032ba3b5c44605, 0x000c04378b1fdff3, 0x0002189e90f61242, 0x0000faa8d60f86df, 0x000a3573271abddf, 0x0008f8032987583d, 0x0003f6b0afe4ec4b, 0x00000000000b69d0}}, + {{0x000f8df0532bb051, 0x000c5bd9a66d6010, 0x000c7470f77e12e7, 0x00051dfef38e9b37, 0x0005754fb3fba751, 0x000069b72a3348ae, 0x000faf1218fa8f13, 0x000000000000835d}, {0x0007a4c220543f04, 0x000c02121a98ebed, 0x0005ec4ceabea5d8, 0x000a4be1a1eb6eea, 0x0002adeb3ae51b23, 0x000ea2b45b5c48b9, 0x0008eaeebd305ded, 0x00000000000c3719}}, + {{0x00055d2a529f419b, 0x00062f73471b59c9, 0x0003ecf010a8f9e8, 0x000497dae082cef5, 0x0005c3e563bc57cb, 0x000125f95dfcbf2b, 0x00001922149c0fd1, 0x00000000000425bc}, {0x000c9ac134356e8b, 0x0000e049476de7aa, 0x0008c62ed440f124, 0x000c62d1256424f6, 0x000541bc66fa56c5, 0x0006ada140a118ee, 0x0008c50d8e9ff829, 0x000000000000134f}}, + {{0x000b2c8bea306a01, 0x000b1d0960bd7257, 0x0002ca4efc9832f7, 0x000cc77a98509175, 0x0001e5f81554975c, 0x000cff2c8b41bd53, 0x000be3bdf8ab57c8, 0x00000000000f5822}, {0x000ad597c5610627, 0x000ae1f0ac0e6f99, 0x000855b71125b2a5, 0x0008b82e3cc86ac3, 0x000d2b2beaee22bd, 0x0002617ffc43bde8, 0x0000968168781e41, 0x00000000000b7ee7}}, + }, + { + /* digit=42 [{1,2,3,..,}]*([2^168]*G) */ + {{0x000956a7f54f949d, 0x00074086945790c5, 0x000771ce236fcb82, 0x000b0c6000575064, 0x00080b09adeb2c04, 0x00081474f468be6e, 0x000cce2f3bf32b6f, 0x00000000000a712c}, {0x000c24bf8b416a6b, 0x0004c292cee41c19, 0x0008e149d7276386, 0x000a66156f47b2ac, 0x0005840b5d1be54d, 0x0006e8cf62ca7683, 0x00053eb52adb6a8a, 0x00000000000dade1}}, + {{0x00026c98ae834331, 0x000b365f7d2c0d6e, 0x0004cfcca31f7dbe, 0x000634c986436f32, 0x000133356165e268, 0x000c7957d6334d8d, 0x000983f17164269e, 0x00000000000b8093}, {0x0008676a1037657d, 0x000d9edb1fe5cc2a, 0x0001cd1de0787da3, 0x0005f4da691b6657, 0x0001e1e2727e746e, 0x00027b0296117129, 0x000197bb4aa8f16f, 0x000000000007f42c}}, + {{0x000a6e4b3ca52862, 0x000cf2af8bdfa5a6, 0x000675c2d00d6d96, 0x000d5ced7046d2e5, 0x0007d545fd33d57f, 0x00061ffd75ea025f, 0x000242e2ccb6f431, 0x0000000000009406}, {0x000194e235777423, 0x00019f3536d60805, 0x0005fe57dd0b2a05, 0x000b06a5cc554450, 0x000f9a9e2a66fd15, 0x0003dfd0261b0feb, 0x00051fdc3c057665, 0x000000000000a8ab}}, + {{0x00014e15511a3745, 0x00067fe19901abc4, 0x0000c6f09a808e87, 0x000012556c4ce5cb, 0x0005938c89ab92fd, 0x000a587b123172d6, 0x000c50a71f8a33aa, 0x00000000000b55c9}, {0x0009dec34d6b29ab, 0x00056ec005f6a241, 0x000b67510d45fff0, 0x000d67f9e26361fb, 0x000321389c2598a4, 0x000ffbcee7f0a2b2, 0x000888d158820795, 0x000000000009f36a}}, + {{0x00065f3227de5d3c, 0x0001e5ffec1d7642, 0x000dceb00d947f3b, 0x000844c649c850b5, 0x000610fa337dfbe3, 0x00080773d450263e, 0x00066c44f7c8f402, 0x00000000000e8969}, {0x0009e0325576575b, 0x000a42587f475435, 0x0002c020daa3c5c5, 0x0002f667071543b1, 0x000e06577b749e90, 0x0000bee1303398aa, 0x000cec030926d691, 0x00000000000ffa92}}, + {{0x000298023770357c, 0x000dae4ee3345cdf, 0x000943bb20278bd9, 0x000ca667ce118490, 0x000dcb69c7ead817, 0x000680afeda222cd, 0x000ca874ac74709d, 0x00000000000d9596}, {0x0002187a0dd3835c, 0x00082dff57da7e9a, 0x0006d7aa1a2ea94a, 0x000ba28b7fbc1b01, 0x000c13d4ed0f71c0, 0x0003a2ef260faab1, 0x000dff6fbe3567ea, 0x000000000004e577}}, + {{0x000b9f236346df15, 0x000f89b758b50c99, 0x00075b638b06df34, 0x000b3c39ee88a784, 0x0001105e1ec04669, 0x00088ea133f36a67, 0x000aadd0f30e5d9e, 0x00000000000baecb}, {0x000d89dcc4fedb34, 0x000d3ec65be650a4, 0x00093dd7d659e073, 0x0005d29942ce52c4, 0x0007d3ce28d4f719, 0x0007fc9041220187, 0x00055e9c962aa1a9, 0x0000000000065c5a}}, + {{0x00095e9750ada30e, 0x00094eccb421e7ff, 0x000dfa5cb406ba75, 0x000da9e05a53972f, 0x00007bc99fead344, 0x000f77bf53a8035e, 0x00078ae0214485c0, 0x00000000000e54df}, {0x00053ac771ed9aaf, 0x00027def45af5dcd, 0x00024afc33d18821, 0x0007db1c337e0181, 0x0006b84671d5a9b1, 0x0007d696d026a4f6, 0x00022a606142343b, 0x0000000000081cfd}}, + }, + { + /* digit=43 [{1,2,3,..,}]*([2^172]*G) */ + {{0x000d2a53b2756567, 0x000f3feb984b33c0, 0x0004e95d9895327d, 0x000f97e3ca0a9a03, 0x000dacc000ac177e, 0x00040f51a76d4796, 0x000810bad0fa6eb1, 0x0000000000006ebc}, {0x000c76db6b103c54, 0x00004f89eb78f367, 0x000ad3bb031162c2, 0x0009cfbb25e9d1c3, 0x000fb4e7aef3e2b2, 0x00067e8459388ea8, 0x0006d7ee606c12e2, 0x000000000009f580}}, + {{0x0008f59540514451, 0x00028cf947c06046, 0x00062665be2e4fbd, 0x00015c05f1835ca3, 0x0002d8c79d90d001, 0x000b5f791df08415, 0x0008cdc0c3a6846d, 0x00000000000749b1}, {0x0000ce2ed12d25c3, 0x000d8d314b7d4a22, 0x000dfaf9b4508139, 0x0005af06855a9438, 0x000ee9d02ab997ed, 0x00038a5d0ea84ae9, 0x000eabb87e903432, 0x00000000000650a0}}, + {{0x000c8f034ea36274, 0x00088357540ab419, 0x0000f5e32c760e57, 0x00073ac14d7fed37, 0x000fd186ee3d33d2, 0x000a7b1d9fef6b9f, 0x000bc2a94c207101, 0x0000000000050bc8}, {0x00013f9560f76983, 0x000e5c68959c999a, 0x000056367228b52b, 0x000a4fce6861310f, 0x0000c828330f7a73, 0x000f74fdb19bcac9, 0x0002f473e3b66f7e, 0x0000000000052d8f}}, + {{0x0008b2d06a3248b0, 0x0008e58225208828, 0x00095368e614a61f, 0x000e45562fe86dbb, 0x00082dc5b2f11224, 0x000096555a8c8a47, 0x0004528a957b8dc3, 0x00000000000b1591}, {0x00063a60dfca11b8, 0x000161e563c57eb9, 0x00024a9e6e7bbb16, 0x0002de99462e7a31, 0x0001c2489214f8f6, 0x00074639c6b3dfe9, 0x0004b56c1d8fc421, 0x00000000000ef88d}}, + {{0x00071f1bee09bff3, 0x0004a6eda384edeb, 0x0001d457a27a9c42, 0x000e6af10e01b58c, 0x000ae1175185c441, 0x0000dfcc6ac962bc, 0x0001b472d34d676f, 0x00000000000a9412}, {0x000f38c043747e7d, 0x000ad77dea14649d, 0x00065b52576e79ef, 0x000c09dd5d17db20, 0x000db45a6e94625a, 0x0001f8caf579edea, 0x000b8ea659282a84, 0x00000000000c11a7}}, + {{0x000f9bf364d3e9eb, 0x000407d0850b3dae, 0x000e31cf311bd53b, 0x000f1f891cb75575, 0x00089d3b6afe68ce, 0x0003b46e427a6ba7, 0x000059f220f9c1f8, 0x00000000000f526e}, {0x0007512e159b9fa5, 0x00094edf3cccac66, 0x0003e3258b2c0e34, 0x000d9e98cc78d3e7, 0x000b5fa48469cc09, 0x000686001391bc40, 0x00073a22bcdcd522, 0x000000000007c599}}, + {{0x0002ae0971dbb084, 0x000bd9de8555bad1, 0x000c9834d42cb891, 0x0000285de654b80c, 0x0008802be80bf17f, 0x000d823bf33998d8, 0x0006b64095923d5a, 0x00000000000e848f}, {0x000fcca2a1ce91f8, 0x00015c6e95017db4, 0x00048a3538b7e8ae, 0x0001be7a70558759, 0x0008441da4770c3f, 0x00048d4c2b652671, 0x0008deaba3b06f9c, 0x00000000000086f1}}, + {{0x000d473962fbc397, 0x000b15a4a5ce3b56, 0x0003f34c786123b8, 0x000388e4efe9a313, 0x000aef9d074bd459, 0x000ab9cb03a45c11, 0x000566f68ab50b93, 0x00000000000ecd9a}, {0x0008303c4d104a42, 0x000f0bfb79b7f7f4, 0x000a0520b1df9755, 0x000aa21877390f14, 0x0005e314cfb54f31, 0x0008eab122de0c64, 0x0004f656d904f623, 0x00000000000ccb4d}}, + }, + { + /* digit=44 [{1,2,3,..,}]*([2^176]*G) */ + {{0x0007aa3a213de299, 0x000c93eb83a8707f, 0x000eb1f52edb04b6, 0x000c77a53abe4e9c, 0x000ebb9feb8257b8, 0x0006f922e0a14673, 0x00026cc0a6cbf7f2, 0x00000000000a32e5}, {0x000ec20f9c4de649, 0x00049ca417e66df1, 0x0009b8c741987bef, 0x0000e7a62e135de2, 0x000ea7ee82c72bae, 0x000776c74962bdca, 0x000feb573d7f6ae5, 0x000000000006ffbe}}, + {{0x000a70e677b6f831, 0x000b9cbc0bb18f0d, 0x00037297c3884bb7, 0x000b350d38064428, 0x000c397a7602f62c, 0x000987d82e4d2f1c, 0x0009faa54bd48e43, 0x000000000004f0a1}, {0x000a1b4997cc48fa, 0x00005fa0c9a44d6a, 0x00012729fa8ce7f0, 0x0002c5b9cbd8d343, 0x000813d979568e24, 0x000da55fa1671eb0, 0x000ce5ccae388f40, 0x00000000000f376d}}, + {{0x000f917fe9a26ae0, 0x0003ae54d92cd055, 0x000e0e9599c7454d, 0x0007fd2849f6143a, 0x000dacd0202d7c90, 0x00092d19abb11dd2, 0x000677a4913a701d, 0x00000000000cf610}, {0x000f3626f0aa0d9f, 0x000a3fff1e1d462d, 0x0009edea6495252f, 0x000f33a92cabf724, 0x000ce329d1adcafc, 0x0009a0e4cd571e13, 0x000f867626ad237a, 0x00000000000130d7}}, + {{0x000f8196d845d3e6, 0x0002f8a89daef137, 0x0004ece7e9ffa3ea, 0x00023775b80bb4b5, 0x0002a45b362648d9, 0x0005bfe910a587c5, 0x0007d7daff503cc7, 0x00000000000116d0}, {0x000600a28d48ec00, 0x000916b5a471517d, 0x0001f4eebe019105, 0x0006cfc595abf8dc, 0x0009ed0391ce8f07, 0x000fcabd3c9de4ce, 0x0009edaaaad03ae1, 0x0000000000087b19}}, + {{0x000b51557a9fd4ff, 0x00034b0641d0941f, 0x00049c97e60548cf, 0x0005b4b00d5ec6f7, 0x00015569d89ad12a, 0x000ac72089be1a11, 0x000df19c0566deff, 0x000000000007034e}, {0x00000817b5f1bb5e, 0x000e5636aeb6adf8, 0x0006c2164b0bbfac, 0x000f898734e9d301, 0x00095ccdc6bcf4e2, 0x0008a4f28daf7421, 0x0002c39d39249f60, 0x000000000008820e}}, + {{0x000d9387d2303055, 0x000ac620248118e0, 0x00080c838dc206ab, 0x000eb38d7f033fdf, 0x000232646d6f86b3, 0x000dae596ee3226b, 0x0008e58c4825f6f1, 0x00000000000a5bcb}, {0x0000edaef1eabcc8, 0x0006f904a53484a2, 0x0007fa1deed27103, 0x000329f21d45f8ad, 0x000a06605546af6a, 0x000a93d14f20ad88, 0x000174cf7a0e9619, 0x0000000000091c97}}, + {{0x000ae272638371ad, 0x0005559edd263abb, 0x000509e662e63add, 0x0000d304e7f07169, 0x000119b88200bf4a, 0x000da801e36fc0e4, 0x0002c791db560240, 0x000000000002c72c}, {0x0006387b3ebbf52c, 0x0000dfe960c2596c, 0x00093166fd89300b, 0x00080febccc1576b, 0x0005c3c88f475a64, 0x0007bd8ec72f4e5e, 0x00055e224e7e749a, 0x0000000000076314}}, + {{0x00073d61157ae785, 0x000ab42c9fad4fc2, 0x0000e5d3107f2d93, 0x000c7fdb149854b8, 0x00002fc359eb0cf4, 0x000ec86034a7d900, 0x000e149ff0c3ea29, 0x000000000009b24e}, {0x000512a0eb94d71b, 0x000e0638c80999de, 0x000012d24a63feaf, 0x000e0f8a07ea5482, 0x0002aecdec3f2fb1, 0x00074025b1e580d3, 0x000164bbf895730f, 0x00000000000dd529}}, + }, + { + /* digit=45 [{1,2,3,..,}]*([2^180]*G) */ + {{0x00051b4cfaf372e5, 0x000b9fa2db519927, 0x000d4edc529b8ffb, 0x000af1d605917201, 0x000e782a09939f88, 0x000c710d31c5fea5, 0x000365e0fb15884e, 0x00000000000d32ce}, {0x00054592dfab29b3, 0x000d7d9f896aa46a, 0x000ce02cec631ef0, 0x000992dca6e73436, 0x000a6fd0c08b1b76, 0x0000f5c2376338fc, 0x00035bc3ea4c65a0, 0x00000000000b316b}}, + {{0x0001e6b975664a84, 0x000aa14f962e38f4, 0x000c4a3248d485a4, 0x00011c079e777bf3, 0x000b4657c31b2c0e, 0x000494de3a1705ef, 0x00071802688fd23c, 0x00000000000412a8}, {0x000b58909626d6fa, 0x00004878fc7ace41, 0x000bfc58da0a4094, 0x0003c4c704b70c17, 0x00087323c0087c81, 0x000089f1a98553da, 0x0002914f63cec663, 0x000000000009655d}}, + {{0x000f1be887dbd011, 0x0008384e5541ec05, 0x0002bbc7edd0f227, 0x0003ad69beaad3d1, 0x0000f51c2714e0a4, 0x000ea5b0c97dd182, 0x0007caac2b4cb457, 0x00000000000d1176}, {0x000327be0b446dbc, 0x000e318ccf36cb48, 0x000fe4f5fd270bff, 0x000ce43d8d292a8d, 0x0008d58c4ee79811, 0x0005c65a772c5fc6, 0x0006695bc0f0bed3, 0x00000000000cbbf9}}, + {{0x00002ac6c24f6061, 0x0003810586ea68e3, 0x0007603f1b64b158, 0x000c4d7dfbfa352d, 0x00021e1c2291a42f, 0x00011760abf38c38, 0x0004d746e40ef60e, 0x00000000000dcb97}, {0x000c4f8f01f54098, 0x000f1755b105ba56, 0x0001f9ffa1baf4e7, 0x000d0b00945db608, 0x000cf2809e1ca630, 0x0006c95c5a160ac9, 0x00074f38fc1113dc, 0x000000000005d525}}, + {{0x0001da84872cc14a, 0x0009a7eba3da615c, 0x000558d3935c6438, 0x0007d982b93d8b2c, 0x000c7cad1f758c91, 0x000ff150aca8fc6a, 0x0007fa5f581f19a0, 0x00000000000e0832}, {0x0005d538d28d5c50, 0x00038c774f18716a, 0x00051c30fd1c0854, 0x0006e9b8ad72b112, 0x000b917986cfce03, 0x00025cf9b463f9eb, 0x000feabe51632813, 0x00000000000dd7e5}}, + {{0x000710a35bbc6ad8, 0x00005a4e9f29eaf6, 0x000a92c5e19e2d59, 0x00084e42993359de, 0x000f224d5aa30e21, 0x000132c484f96ce7, 0x000f5f0862e2003a, 0x000000000000f015}, {0x00066db2ab4fc1aa, 0x0007cab9d51492bc, 0x0009b538d3bdb7a7, 0x000e671c1fd96e3a, 0x000e71a703b24865, 0x0002add107baf4db, 0x000d3083dd6cf914, 0x0000000000098461}}, + {{0x000b2da1393d8e42, 0x00051714c1d1ba41, 0x000ef78ff03cd88b, 0x000ea3a6951ac80b, 0x0000ac00c8377f23, 0x00024cc1b5c59929, 0x00062bf6efa2b3bf, 0x000000000001e844}, {0x0006a668e721edeb, 0x000069bda627d119, 0x000d91ed1a995ffb, 0x0007089c3b94ec3c, 0x000e3031699ad1ee, 0x0002b2453f75dba6, 0x000ed48ff75f7924, 0x00000000000289bf}}, + {{0x000ac44be8741dd1, 0x0002fcda68a7d811, 0x000fb56aeb52d290, 0x000e2dadce20b92a, 0x000cfc69dca6483b, 0x0004f98917de1601, 0x000a564bec17aaac, 0x000000000008d479}, {0x0009255137ea7d35, 0x00025c623cb8d743, 0x0009f513ea4e4bb1, 0x000b7c030dcde621, 0x00073a1733fdda9b, 0x0002fac31f84ea32, 0x000449d7afb6c3e8, 0x00000000000d3897}}, + }, + { + /* digit=46 [{1,2,3,..,}]*([2^184]*G) */ + {{0x0006c80cb0000ec5, 0x0007da189f30f16a, 0x000d675bfc196669, 0x0009ec37d8da76e8, 0x0006c1ea7c10307c, 0x0008c62d4b3e1d00, 0x000b3ac15e20b3b8, 0x000000000000bff3}, {0x000b01f1748ea86b, 0x000e29e330eb12b6, 0x000af2a26953e630, 0x0003cb002e1eb2af, 0x000bf525e4d4157d, 0x000a3dff1638f297, 0x000051a20f833234, 0x0000000000045a9c}}, + {{0x000b35a3033b3940, 0x00007fe9fdde8b8e, 0x0004a1bfc8bd5420, 0x00049acddde6f6e9, 0x00097e54356ca653, 0x00009f73cc53c29c, 0x000277ee15ad9457, 0x00000000000e5429}, {0x000bdd741f2769e6, 0x000d6f52035cdb19, 0x000b835933b3195b, 0x000b0ceca319bd4b, 0x000b6951fd8d26e0, 0x000c34d6f4e7eb67, 0x000b59ac3a6f4395, 0x0000000000000f60}}, + {{0x000d7763c7959780, 0x000c02c47010c514, 0x000f6a495cc56b87, 0x000be509d930f6e7, 0x000d5f56cf045c8a, 0x0002f1fc16bcf875, 0x0007101f6456c006, 0x000000000005304d}, {0x0001668b6ef47661, 0x0006dd76452e46c9, 0x0004266da10fc06f, 0x0009fc89021bde74, 0x000f4babfae0b5ae, 0x000a61fd6505c6b9, 0x00025a99d943c17e, 0x0000000000059bf1}}, + {{0x0003de8a392493e6, 0x0009cbd10b8bfc25, 0x00082fa94d1d5f3e, 0x000be5ec0c907818, 0x00071ad167ce9a18, 0x0008c1c563677f4c, 0x000275ccb254e2a6, 0x00000000000e2c4d}, {0x000250541623e5c3, 0x0003f44958cb1bf9, 0x000db2f9dd62ce34, 0x000a767a2ffdbd52, 0x0009b0b6c22d7445, 0x000c92b2e0e789c9, 0x0007823ff8b6565d, 0x00000000000eca98}}, + {{0x0000ad4fec1eb621, 0x000c36618fcad673, 0x0002540e2f8dc71a, 0x00039947d7ce5530, 0x00095257f24b90ad, 0x00098768cbf8c458, 0x0008305a94992020, 0x00000000000bb6d8}, {0x0007503283325be4, 0x0007dfae8f1616ec, 0x000fc3e0aeb8a2a8, 0x0001ea9139ea0507, 0x00012e39e1d8a72b, 0x0009c7bc282229a9, 0x0008254153bf3e47, 0x00000000000c5541}}, + {{0x000605d76cba7718, 0x000a6fadf9be90b6, 0x000490316f096fe8, 0x00024ec2f9953940, 0x000861203303cbad, 0x00089a4be6236a26, 0x000a82e4bafc3365, 0x000000000005e23f}, {0x000642137e1da447, 0x0005c8ccbc576c76, 0x0003f63011c9e098, 0x0002d5841df8dd26, 0x000914d31fcde6bc, 0x00026010bec24e1d, 0x0002f3acaaf13efc, 0x00000000000e01b9}}, + {{0x0005639fc34e5a27, 0x000c7c52789bb2f8, 0x0006d4ce23fe7231, 0x000f95aacafbbfb9, 0x000dc7f6eb6d8b6d, 0x000b4c9d737afdcb, 0x00045357775bdd6c, 0x0000000000011007}, {0x000bddf07c5f1b9e, 0x000e3903e5ba1399, 0x000d7d2fc919a9a5, 0x00018932d7ac9e4f, 0x000543ce66a8046d, 0x000956410e2fe9d1, 0x000f7244b5beb4d4, 0x00000000000bb147}}, + {{0x0006e03c443812f7, 0x0005c6f6a6104456, 0x00098182647d3e84, 0x0009a6ab51989e5d, 0x000f68f5b16842fd, 0x000f7fe671b60ce0, 0x0005a897324d8756, 0x0000000000067681}, {0x00061f269664533a, 0x0002a265ee993d1f, 0x000a90ce6a02c969, 0x000232334b4adb0c, 0x000e1e5a8d18e909, 0x0004f6456ddcd233, 0x000d9b3dc5b27c5f, 0x000000000007f421}}, + }, + { + /* digit=47 [{1,2,3,..,}]*([2^188]*G) */ + {{0x000b170252588bc8, 0x00099bfc40c30a63, 0x0001ef23d6587c46, 0x000b54dc027511d3, 0x0004cf3484ce4fd7, 0x0009beea5f479928, 0x000280655ab81106, 0x000000000007392e}, {0x000148f913baced2, 0x00084522e7b403b5, 0x000493599cdac0b8, 0x000b95877f3913fd, 0x00067dd525149cf1, 0x00008cbca3e06b92, 0x000529992e920e29, 0x000000000006d6ed}}, + {{0x00097dff92ad4838, 0x0001e8a46c9112d1, 0x0000711ed277a798, 0x00064f15cf8e4ca2, 0x000a01cb3488d4ba, 0x000bd7ded01b3908, 0x000ae169b1fa5d38, 0x0000000000018b2e}, {0x000f3694f52a1f22, 0x00019619324bdbe0, 0x000d1d925851b48d, 0x000f42e925b3f6ab, 0x00095f7e4d11a397, 0x0000d9f0132169a4, 0x0002e5c0fa9a548d, 0x0000000000089d8f}}, + {{0x0000115baf0f7c2d, 0x00040240239ae483, 0x000c7482d351827f, 0x0007151f58ec53e4, 0x000cc080d59ff9ab, 0x000f782f4d397862, 0x000d5873eee88536, 0x000000000006c1d8}, {0x000c16042c611b2f, 0x000f258bed3e5b15, 0x00036e097964eba0, 0x000c8ab8a482af89, 0x0009e746a8549044, 0x000a0548e0065858, 0x000a492538d1b926, 0x0000000000007cdd}}, + {{0x000178ddffec9175, 0x0004c9c0d5230baa, 0x0004bce21493d0f7, 0x00090dd985154559, 0x000c8dbfb46a67df, 0x000757ce3223e8b9, 0x000296f39529a36d, 0x00000000000b4648}, {0x00000562605aa919, 0x000dc330749e8973, 0x000a3ffdafa653e5, 0x0000b3494083aa87, 0x000321e321c68c32, 0x0003e78921161f5a, 0x000ccc0980deedde, 0x000000000006ad76}}, + {{0x0003725fda5a777d, 0x00086af9a69e965a, 0x000a3534516a8b8a, 0x000a77f3e52375ce, 0x0009a019a5932dff, 0x000238091ac65569, 0x000085d402f5c4df, 0x00000000000d3518}, {0x0004a37271e8fafd, 0x00007dcee2db4b54, 0x0006c1d813edf12e, 0x0005b6121bc49990, 0x000e68b9808d9cb1, 0x000d6ac5b40b34f9, 0x000b8a98de63590b, 0x0000000000039766}}, + {{0x00069c8c3d7c7657, 0x0005171191261c8a, 0x000244a0eba69bbd, 0x000344bdda57f44c, 0x000ac4f0cfd2ad4b, 0x000543efd674b758, 0x000d063bc058a077, 0x0000000000056618}, {0x000a82ca14a01b7a, 0x0009d95107c74391, 0x000a3c4cfae47f34, 0x000af35e3f1d63cf, 0x000643ab87265dbe, 0x00056c6fd012029c, 0x000e304f588a4ea2, 0x000000000003e5d2}}, + {{0x00081d2046b48f0f, 0x00043847622b5217, 0x000c2a7014a5d0be, 0x0009da7b82c435cd, 0x00025b73e01114da, 0x0008b37b399c8c43, 0x000ddab978fe55ec, 0x00000000000337d6}, {0x00034bf111412925, 0x00071e0d4ffda16d, 0x0003fc3275d2e3f4, 0x00062872913cddbb, 0x0004f67405be2a7d, 0x000a31060229afd0, 0x0009d6e372202e49, 0x000000000009fc21}}, + {{0x0001a0d511e4c022, 0x0009173fa3508062, 0x000e92c1603f0953, 0x000549f58493d985, 0x000adc79f602f64a, 0x000512b84d9ceae0, 0x0001516569e37bd1, 0x00000000000151c9}, {0x000c6addaeefed36, 0x0004c075678c2066, 0x00015cc88eb8c3b8, 0x000dca3a57fb96a6, 0x0000223dc3ce6334, 0x00011e2770ed9082, 0x0008e274f9c3aebd, 0x0000000000079c0b}}, + }, + { + /* digit=48 [{1,2,3,..,}]*([2^192]*G) */ + {{0x000688b6fc0935b1, 0x000f5378205dd339, 0x0000b901357b7bc3, 0x000c06c682e00f2c, 0x0003114d5423dbce, 0x00052463ef2a145c, 0x000b0aa01d98747a, 0x000000000007d717}, {0x000d2bf78a72f39e, 0x000d29653bc4f4a5, 0x00051b32471fd3a0, 0x00043dcaf8e3f402, 0x0000e86fe16ef779, 0x0009ffdcf70774a1, 0x0005c96b62e6f1bf, 0x0000000000058874}}, + {{0x000b3ac410563249, 0x000bc2a5f8ecef60, 0x000af14f01d834e5, 0x0001cedc59c4301d, 0x00010111d9989de3, 0x0000d5b951e0f40b, 0x000ab8d29d229f96, 0x00000000000d1dab}, {0x00033bacd39b8f1d, 0x000bd7b225cc8ccc, 0x0003c9f7b44c8f47, 0x00052a1f5fb06b38, 0x000f842b9081009b, 0x0002725128a575d3, 0x000cb7fddb48afe9, 0x000000000000b452}}, + {{0x000dcfd459bff4dd, 0x00050ae10069e26c, 0x000e9f25bee973af, 0x000caf27ebaad0bb, 0x00073dd6119cbbe4, 0x000fceefe5907bf3, 0x000c7e0a723dff9d, 0x00000000000f7cff}, {0x0002a3a44c0ca01e, 0x000d17bc95fa21e6, 0x000c0e71f388ad82, 0x0007ecd27b3335bd, 0x00027b8d7d49316a, 0x00019058fbf08e67, 0x0009ea4b209f93c6, 0x0000000000059d8f}}, + {{0x000cfbdc0726f5f2, 0x000ba167ec88a4a8, 0x00009c64d249271a, 0x000e2443877e6342, 0x000603462cb310f2, 0x0003afcee6321bf2, 0x000dcd1dbd10ee9d, 0x000000000002ca17}, {0x000667ac9826886c, 0x000d509465265738, 0x000151279a7a2541, 0x000b0f95e1c59136, 0x0001757d3a630043, 0x000def1e0a09b94d, 0x000d41533956529f, 0x00000000000d4fed}}, + {{0x000c29ae93761a8a, 0x0008a3459097559c, 0x000f79e8fee087bc, 0x0009a286ec406ef0, 0x0006fee5454dcc93, 0x000257f708d21427, 0x00085e66a0e1a56b, 0x00000000000006fb}, {0x0006c4387ea9f222, 0x0000ac44f9df22bc, 0x000c5644721083d5, 0x000a8224fca1a819, 0x0008f3ed85bf5894, 0x000899b5b8586e41, 0x000371d494dfb202, 0x00000000000ecb8e}}, + {{0x00094e8a6ed08358, 0x000cf690c0cfe457, 0x00092e98638a5e98, 0x0008042204d98a6b, 0x0005bad2eb082250, 0x0005823eec87a97f, 0x0003f6d307c59ed2, 0x000000000000df8b}, {0x0002a9b6fd1bc660, 0x000e9280ae343343, 0x00077184e86c10ae, 0x0000e5a24d04e396, 0x0007309830fcea93, 0x000afeebc0269d9a, 0x00002d41dc8f0ae4, 0x00000000000f14ee}}, + {{0x000ef795d8c64486, 0x00026efca7f7acdf, 0x000411f7c32b0a9c, 0x000b87fe57d08e0a, 0x000a7d9a1967c9ea, 0x000c2749248c01c2, 0x00063911c97ed97d, 0x000000000001cf1f}, {0x000b6334379af438, 0x000d1541f8c0d49b, 0x0006f782375b1fd6, 0x000935ba6be190e1, 0x000764494b4e9806, 0x0003c00b6ec6c5de, 0x000f15f04e2d4cb8, 0x00000000000c0b84}}, + {{0x000c9d81a70917bd, 0x000702e75a26e455, 0x0005bf4870175e47, 0x000d057ee7d8e4ba, 0x000d8994f44953df, 0x000538367b959110, 0x00029cb4a16596bb, 0x00000000000ef82f}, {0x000ea0c85d7d05e6, 0x000305a7642ffe63, 0x000b9d7a5d2d391f, 0x000b23803a4184c4, 0x00020f7fcd62c7ea, 0x000a8a0c660c67ef, 0x0002b041e05799df, 0x0000000000004d35}}, + }, + { + /* digit=49 [{1,2,3,..,}]*([2^196]*G) */ + {{0x000ffb708a3b5e85, 0x00030c97c01eab92, 0x0007510b2d7953a7, 0x000ce807fa7d3c2a, 0x000e81060874dba1, 0x0007eeead69e6f96, 0x000f3d6e3e0df74d, 0x00000000000b0c87}, {0x000317c5146f214f, 0x00028c55ae3dbb43, 0x00014b4be1d3dc49, 0x0008c591de7860a7, 0x00066e546731a600, 0x0001e45d48202f8b, 0x00015a652f2d07aa, 0x000000000006df54}}, + {{0x0007d6371007dea2, 0x00049041c706cbe5, 0x000dcf6b55c23258, 0x000cd27839e9d5ae, 0x0003bf3c4c067dc4, 0x000b7bd22dfc9db8, 0x0002da85b8094138, 0x00000000000d0f4c}, {0x0000b16d9a334a33, 0x00092d7b340062c0, 0x0002bb5502deaa2f, 0x000b2c2752366864, 0x00010113a85fa340, 0x000b327045ddd055, 0x0002ff7dfc7ab29c, 0x00000000000dabf2}}, + {{0x000a8373e5c690f8, 0x000ca2fbce9bdf20, 0x00049076d1995e9c, 0x00045939f4cbaf1b, 0x00092574a3bd48ea, 0x00092c39a56c5400, 0x00034384a39630e7, 0x00000000000eef81}, {0x000361503c11fa79, 0x00095f996760edb5, 0x000c1bbc8ea81e13, 0x00012e6966d70279, 0x00052e6f7c63a0ca, 0x000d13ead92a6d5d, 0x00068146809d269b, 0x0000000000067aac}}, + {{0x000cf4e4cd35d7a3, 0x000a13fc9b3cedea, 0x000ac33c871e844f, 0x000a58afe1ad536c, 0x0008cb39149f2003, 0x000edf470cec4be3, 0x0005194d578c99bd, 0x000000000003a356}, {0x0001980c5865f55f, 0x000607a762f2732a, 0x0003ce874c8a141c, 0x000817f270c508e9, 0x00049fdb29c8dc0e, 0x0002711d35a7be20, 0x0000c2fa1a0be3cb, 0x000000000003786a}}, + {{0x00065cdb1cddc024, 0x000c41d0af6b5128, 0x0006106e0f532684, 0x0001951b1ea8fc4c, 0x0004b1fae4826764, 0x00023477bc0b9006, 0x0009ce7012642f66, 0x000000000005bf01}, {0x00001d44438309f3, 0x0007fca1f46757f4, 0x0004d56451db59bb, 0x0004ef2d3868de95, 0x0001044e0c189c03, 0x0005e38c30533d92, 0x000053ba6cf14ecb, 0x00000000000509d7}}, + {{0x000af6aee4d4a85d, 0x000c0e164268de02, 0x0001633ba7cb9816, 0x000979478ab17f45, 0x000e0179ed0e734f, 0x000a2686746d468d, 0x00085d7e68f006df, 0x00000000000e3d04}, {0x00073699ad94d8f6, 0x000c30913a1d74ab, 0x000e2aa1b6d33ea2, 0x000a79e49eadd08e, 0x00017dd8f954eeb1, 0x000bb26d0433f5e4, 0x000e2970a6281430, 0x00000000000ff5e8}}, + {{0x000235f9cfa08aab, 0x000eb6a352b56ce9, 0x0002152b2e478d04, 0x0007c7a240e6dc62, 0x0002d313b4a9ee1f, 0x00001a40585d5be6, 0x000d5a1522c5d25c, 0x00000000000960af}, {0x000459bf66d63a1d, 0x000e4a3cb77f327b, 0x0001a15093d3f2d2, 0x0000c7d3b93fa9e1, 0x00013c0383ad8409, 0x000f7a220c77f1ee, 0x000bfc461c93b776, 0x0000000000004ac0}}, + {{0x000fd3f75cd14c88, 0x0002a3c7d6b63ea7, 0x0002b345f341120e, 0x0005d20aa0eaa1ec, 0x000fc0eab4908ed1, 0x000d9f260e944ad2, 0x000ba371525aa1f6, 0x0000000000016146}, {0x000bd29ab6e83fdb, 0x00068f94019075db, 0x0002a02a4fd970a1, 0x0001c37cab1060af, 0x0000c8cac96f6a4e, 0x0002466ec357fe4d, 0x000e7097a8b8ab6a, 0x000000000009c01b}}, + }, + { + /* digit=50 [{1,2,3,..,}]*([2^200]*G) */ + {{0x0007eae876f30205, 0x000645b0b5d68b38, 0x0002f6471178cf56, 0x0000a4a404a3458c, 0x00059f467b6072ad, 0x0006348091de8e25, 0x000178a4b3570590, 0x00000000000706f0}, {0x0007cba07f8d2545, 0x0006d588d21aac4c, 0x0001bb1a8ee3a06e, 0x000e73d241bcd915, 0x00022facc7ccf4e7, 0x00025d2a0b8d8a1d, 0x000608483c35a71d, 0x000000000001ef9f}}, + {{0x0009cd91f152b14c, 0x00034a704015f319, 0x000a64fabfbdef40, 0x000301f2ccb94180, 0x00046f00d8aa697f, 0x00038a0173ee8776, 0x0005432b5afaf881, 0x00000000000832d7}, {0x0002183eafee3abb, 0x000d627c27ce1884, 0x000735007191c91b, 0x0005ac75b752008f, 0x0001e84fe5f192dc, 0x0005929cecf382e0, 0x000ffa90e034197d, 0x0000000000015ca3}}, + {{0x000596896506329d, 0x00058cb51f038efe, 0x00073c05f41ddada, 0x000fafab41fe1a74, 0x000da719f25493c8, 0x0004f5cde6297701, 0x0005426e9165bc64, 0x000000000000c11c}, {0x000368f61fe7d95a, 0x00098a2564809894, 0x000e829acda88407, 0x000592622b1d1be2, 0x00026ecdd041286a, 0x0009f952486a3d75, 0x000b0f4b867e0a64, 0x000000000000629c}}, + {{0x000259f3facaa9bd, 0x0001d11dd860d21a, 0x000b8c19c604b970, 0x000aff635c019302, 0x0001e3a4a900d4f8, 0x00078c8ba96a727b, 0x0007c41426daffde, 0x000000000008d152}, {0x0001e6f4fd354295, 0x0004a0c0d5233cfd, 0x00066c04a38eba93, 0x000bee43d914fb41, 0x0008f3ba26a64828, 0x0004eb26f8324ea3, 0x0007bf027590f3a9, 0x00000000000acd95}}, + {{0x000a71b96f713d9b, 0x00013f4f668435ae, 0x0008fef0f35f5919, 0x000e86e7365712f9, 0x00088a822bc0f607, 0x0001299b3d588229, 0x000b1a2cfbd63ac6, 0x0000000000067167}, {0x0006f5a47be411d6, 0x000b0750f673f622, 0x00032c38df6a058a, 0x0005bd169123c758, 0x0006eab99b375e6d, 0x000aec6a36a93d1b, 0x0008186ef4f7e68c, 0x00000000000cf3ed}}, + {{0x0006410726f50135, 0x000fd959353be170, 0x000b4de98d5dc91d, 0x00026f799d7a4f4a, 0x000e52fe4b656a48, 0x000038573ab146ec, 0x000e8494fc21d735, 0x00000000000f4d56}, {0x0006901ebf8c490f, 0x00093e15ca04c71d, 0x000ef178dcf47997, 0x00079244f21a9114, 0x0009dcc76132ef7e, 0x000e890482eecb7e, 0x0002c55b484745db, 0x000000000006e43a}}, + {{0x000b8d876ab51a4b, 0x0001af92b3072f8e, 0x000d8f5d67f2d2e3, 0x000d5edc578e3a39, 0x00029587fa22e51b, 0x0002eba85efda70d, 0x000530cfec17089b, 0x00000000000af7ba}, {0x0004893a5eb2bed8, 0x000bb5ac155ae396, 0x0009a3394a2b6335, 0x00086c2c38718a82, 0x0003d63745b7280e, 0x0008a79aa9d12de7, 0x000bf70e8ea855bf, 0x00000000000bd705}}, + {{0x000260c123f30563, 0x000c53ede2484b68, 0x000620a80e97a435, 0x0009e93962a667bd, 0x000b130f2cea5606, 0x000366a66c931266, 0x0003b14bd6a6fca7, 0x00000000000aa5ac}, {0x0004e3f2adddce7d, 0x00044a025d0ef29e, 0x00075ab6560ff06a, 0x000927f2b3057f30, 0x000a1499f8844809, 0x000b9a653b001c10, 0x000d05309d141c30, 0x00000000000bf659}}, + }, + { + /* digit=51 [{1,2,3,..,}]*([2^204]*G) */ + {{0x000d68f9d41abcc8, 0x00016a6c328ffdb0, 0x000797038aa63e5d, 0x0007d39063de7eb8, 0x000710daf9bd691e, 0x0008b5d7a998df4e, 0x0004b8c7085b9e71, 0x0000000000016b3d}, {0x00019da01ecaa2a1, 0x000494dfce693daf, 0x0007011a8e84696a, 0x00004bf4491fb345, 0x00014552451c2c19, 0x0005e5e407c1bf11, 0x0003726562cc2c3c, 0x00000000000fe0e4}}, + {{0x00073ecab0b13cfe, 0x0002484c3630b425, 0x0005d7cee5256fab, 0x000125ff61af001c, 0x000fbfea35255abd, 0x000e0cb6e69bca56, 0x0005a6384af19900, 0x00000000000d0047}, {0x0001438f80a7fcba, 0x00090edafde48dd9, 0x000a30b2135b9aaa, 0x000f97a6c8ffcca5, 0x0003e5a9cc5cf14c, 0x000104e054d6cec2, 0x0007c1b0d678f88a, 0x000000000009fb52}}, + {{0x000cf7bdfab400e3, 0x0009e8618ffa6a37, 0x00041539f0cbda9a, 0x0000d744f61edff9, 0x0009eb7a476f5b1b, 0x0002ee99b33df67d, 0x0002cc7f1a767ea9, 0x000000000002223a}, {0x0004324c9cd0a9f1, 0x000f616f376f2586, 0x000c0794e16b9222, 0x000516ca58765df0, 0x00062260ed6b8dc1, 0x0004b29ba8934082, 0x000bba060eb0afcf, 0x000000000001d252}}, + {{0x000bb25431b5857e, 0x000160d6cadbf906, 0x000790df51943fb8, 0x0003c734ab19507e, 0x0005660086b33b43, 0x000eb0f434fc5340, 0x00058d770ae903a1, 0x00000000000f6b5b}, {0x000dc8bc8bb4bdd2, 0x000d6e206e14147d, 0x00079b5341d6e69b, 0x000ca2f47449e081, 0x000639fef8e1cfbf, 0x0006fc80cebaef25, 0x00061b959e37b8c1, 0x00000000000e911d}}, + {{0x0006b0541df5b61f, 0x000b0014a0907c72, 0x00060742ec5c6420, 0x0007c4dc999acc04, 0x00075e7ab1e3dbbb, 0x0008fb11e01b7710, 0x0005c2a33fefbfcd, 0x00000000000c0b8a}, {0x0004467ba7747f4b, 0x000ec774dc2669d0, 0x0007562d1fca7010, 0x000dc694d9c84626, 0x000a1e772c5f5ac4, 0x00083fe91bf6e002, 0x000dce4922120e0b, 0x000000000003efba}}, + {{0x000263fbc0d157fd, 0x0005bc483d4b1827, 0x00037dfadf4ae121, 0x000df6fcbab1fb10, 0x0000f6cdfc0c5165, 0x0004320fceb28437, 0x000e80ab565c0099, 0x0000000000062133}, {0x0007401414422436, 0x000ee7850cda5472, 0x0007bd0ba094b0ec, 0x000805cc2c82eddf, 0x0006cf244539d14b, 0x000dbe92dcb5468b, 0x000f1e97d43ee825, 0x0000000000089fb8}}, + {{0x000a494639f26e1a, 0x000b5421afbe0092, 0x0004dc6db28bf654, 0x000cf4db1a2705ad, 0x0006d128bcd556ca, 0x000191ad86a3a413, 0x000d242411c4b866, 0x0000000000015b45}, {0x000845fec1268b55, 0x000a74cc82459052, 0x000da0992b42bc3f, 0x000bb1c69f6298e1, 0x00031933fdb59c88, 0x0001308c3fe567b0, 0x000dfa8a6aea7188, 0x00000000000a2f0a}}, + {{0x0006a4d1fcc1ad65, 0x000d2d7bf2ac5a3e, 0x0005e362c26e5671, 0x000abaa9afa97632, 0x0004b62b36ab162f, 0x0006a84e97b7f166, 0x00043d77b9f79729, 0x000000000007dbd8}, {0x000519c3add29e33, 0x000fcc6e9c1e11fa, 0x000ac380a63f4305, 0x000a93d3bfa90c04, 0x0009d050e46afa7f, 0x000c5625655846fe, 0x000a65473b9a0d35, 0x000000000002b656}}, + }, + { + /* digit=52 [{1,2,3,..,}]*([2^208]*G) */ + {{0x000f0e1b25d4e4dc, 0x000a6372798f002b, 0x000537a488c42515, 0x000f9a98a1e25677, 0x000ce70391c85e64, 0x000f024585870254, 0x000bed2def81a341, 0x000000000001d087}, {0x0002a1629bafa8b1, 0x0006d3557d07cb43, 0x0009543a3877e0bb, 0x000c5675f73ba510, 0x000b9c7c670608c8, 0x0009850725309050, 0x000e962ab67da3bd, 0x00000000000e5df4}}, + {{0x00057ab93a62b1b3, 0x0004b7be81fb0ec2, 0x000d385405273506, 0x00040e27a8d16791, 0x000ad520811ebb3d, 0x000f65d231806c67, 0x0003d7add4bb6686, 0x00000000000e20e2}, {0x0008a96c64700a7f, 0x00088208b470000d, 0x000907fb1a1c5c32, 0x00064f8121c37e26, 0x0001a598efbbbd39, 0x000eef966d35ef30, 0x000e46bd76a276c5, 0x000000000000af64}}, + {{0x0007b8ba2901e630, 0x000573f40494a69d, 0x0001d7e86c246f17, 0x0003360c9e634b1f, 0x00096ab166bbacc3, 0x000fdb67e6cf72ff, 0x000736477d8f2db9, 0x00000000000cb644}, {0x0000821b2e82caf8, 0x0007549454e1ad4f, 0x000486f6c48cff7d, 0x0005f0be8e7b06ec, 0x00047dc40b498042, 0x000ed620b862df52, 0x000a648ca7d7c812, 0x000000000003c45b}}, + {{0x000d7620f273aa67, 0x000e1169474a1e10, 0x000056ab42590c74, 0x000de922ede425c9, 0x000a6df8a8908589, 0x000a8b8e350e03fe, 0x00091a0d8a5c1c4b, 0x000000000003fffb}, {0x00026981fafa18b5, 0x000f721cf05437b0, 0x0007e513859293da, 0x0007eaed0962c826, 0x0004213f6004c323, 0x000148b6b43d6ac3, 0x00080a45e619b2d4, 0x00000000000ea5fb}}, + {{0x00014768f5f99aa5, 0x00067314ea217285, 0x000017c8fd29a716, 0x000fb46a63ea8fc5, 0x000890d84e5b0902, 0x000e49b8a925a6dc, 0x000be5e2e74f9c14, 0x0000000000007d45}, {0x000cae18673b6270, 0x000667b768d075ba, 0x00089b2a5deeff6b, 0x000223360d5b216a, 0x00080a7386f475db, 0x000c47746b132b67, 0x00031d7f933fd580, 0x00000000000fbaa6}}, + {{0x000ee5b308cc45e3, 0x0002faba967ac481, 0x000d29b2fe96bd68, 0x000c601ef5f681db, 0x000ffddc580fe033, 0x0007572d85c34f77, 0x0004d0d30f5b66f3, 0x00000000000da20d}, {0x000030e8bcc549d9, 0x0002cdb2310273e6, 0x000ec784d2efa81a, 0x000a33f7899cd7f5, 0x000fda29c3821cce, 0x000e14ecf0f4e0a7, 0x000839c6d7c5f32b, 0x00000000000d9caa}}, + {{0x0003fe55b28c2fec, 0x0007ba884edf1601, 0x000775572e4af6c1, 0x00004152d7852a27, 0x0007f26efb4c66d0, 0x00022f8cb34732d7, 0x000ff518b3ef8e29, 0x0000000000018bcc}, {0x000ec4cab3e21461, 0x00004a219cb1deb6, 0x000868a49e96a154, 0x00099e1d90760ec0, 0x00078a94df2ef0af, 0x000058f89e6fe194, 0x000d9764b5dfcd04, 0x0000000000023d21}}, + {{0x000d7944d758c20d, 0x000209a8580a957a, 0x000f955204f37a28, 0x0000970be07f7827, 0x0000712f7b7cb4da, 0x0006a7b970ac2a26, 0x000f62c9b8ee8443, 0x0000000000011fb2}, {0x000ff5f68230c1a3, 0x0002a5daabed96f3, 0x0003bdf181469c85, 0x000b5b7d96cfb8e8, 0x000344d9e84382d6, 0x000e7da3c4d7d0d9, 0x000253fa2a9ea991, 0x00000000000d531b}}, + }, + { + /* digit=53 [{1,2,3,..,}]*([2^212]*G) */ + {{0x0003b2a8a65e5b7e, 0x000424cc41f278dc, 0x000bf1d7ec4af5a5, 0x00066640fa1ca255, 0x000b91e5edaf7053, 0x000d3de14eeb40f3, 0x000c43cdf98235f1, 0x00000000000ff018}, {0x000927ebce051283, 0x00074aa3228e6dee, 0x00043f750dae9462, 0x0000425650b2dab8, 0x00026d875f1790e9, 0x000e8a46ee4a8cad, 0x000fb5c212029c9c, 0x000000000005ed7c}}, + {{0x0007539a8f390740, 0x0008eadb5966f40b, 0x000e0b7342eb902f, 0x00073e244693a961, 0x00055982bbd3a76e, 0x0002ca13214da743, 0x000e7646e982cd5d, 0x0000000000024938}, {0x000cf856b36cb844, 0x00029749206b2571, 0x0006030c0c47215a, 0x000d1567025bb7cf, 0x000e19555c9ebee3, 0x0001639bae23f0e1, 0x000bd00dec383775, 0x0000000000005d43}}, + {{0x0002c235491635a5, 0x000e4e4e52e86121, 0x000459dc25e36e9d, 0x00051bfb49f2b393, 0x000b3f8097cf73b9, 0x0008fbf057b6cb7d, 0x0002119dfb8d0b32, 0x00000000000ebce6}, {0x000c890c36814c6b, 0x00007a31a15235a7, 0x0009c26d4a535440, 0x000834e6b5638766, 0x000d10ee5a281d22, 0x000aee4eafd91b30, 0x000a763d7a282d59, 0x0000000000073300}}, + {{0x000f6efbfb5bea3a, 0x000f878b5c14b0f6, 0x0005485b973e6dbd, 0x0002ab209e1759fd, 0x000db4b2f68cffa2, 0x0005be7f45a86263, 0x000f6d71e77c516b, 0x0000000000019844}, {0x000dc7fe7c7337ad, 0x0009d2519d0058c1, 0x000edd3b9e6ca5d6, 0x00074a685b3c2a9b, 0x000fc294f4492c6d, 0x00069fb469306f68, 0x000886552e77c22c, 0x0000000000010bb9}}, + {{0x0001ae09a4f32c66, 0x0007beba7daac862, 0x000767fe0f73dc31, 0x00018f885bdbc832, 0x000094d43909985c, 0x0009e108b86555ff, 0x000313b0b1b2b653, 0x00000000000c0bf1}, {0x0003b62754d457e4, 0x00021bd4777c10d3, 0x0007d7f58d2fb40a, 0x00057374a27f1ddc, 0x000eeaaa58ab85bc, 0x00076fac29a8ae24, 0x000377161cb2f5e8, 0x000000000006636e}}, + {{0x0001f89428b5c457, 0x0007f1674b959a73, 0x000b96ebf7106c2e, 0x000a32dc67c36488, 0x000368d720a63962, 0x00057a5b24949617, 0x000c0f4e81df85a9, 0x0000000000053123}, {0x0003624d70103a1a, 0x0003f5091dd340e1, 0x0000c9fe10861f33, 0x00020f52c119dbe8, 0x0006c94d609a5e77, 0x000dccd1fd584ae7, 0x000c6e476c63ba86, 0x0000000000032508}}, + {{0x000df9bca60288d5, 0x000016bbf77cab44, 0x0000fa9d18796041, 0x000eb1a2b9febf8d, 0x0001a25330ce357f, 0x00091799874240a8, 0x000f5c7a9ab575b4, 0x00000000000eda3e}, {0x000c7149276e2420, 0x00036360410d2e37, 0x0006d4d0d5e12db0, 0x000b466cc381b581, 0x0008247a49047bae, 0x000c58130024a98b, 0x0006d26e70b4c3e3, 0x00000000000ae8a5}}, + {{0x000d9a7dd453995c, 0x000393313a9d4705, 0x000fd95bba01fcaa, 0x000ef915e4dd5cea, 0x0003c565dd67c0fd, 0x000ed05ac902a2a9, 0x000ae9d8eba4dc7f, 0x00000000000e157d}, {0x00019071237f3ae4, 0x00006d655d0b3ced, 0x000513db82a990cd, 0x000525a0652872b6, 0x000fe68c0ddb5b7e, 0x0001cb31caf7968e, 0x00071e2ec02930f5, 0x00000000000f2be0}}, + }, + { + /* digit=54 [{1,2,3,..,}]*([2^216]*G) */ + {{0x0003b3ac56ccd2a3, 0x000649b23ab4e3e0, 0x000d023509576972, 0x0009e51e798edf99, 0x0009307675c7dbe9, 0x0008c0fb63854744, 0x00037223ffaf5562, 0x000000000001698c}, {0x000420dd9073adb8, 0x000d039f45a56f2d, 0x00011e9a2cdfa00e, 0x000079e4af138fd7, 0x000a2ee4ecc02a89, 0x000bbf92fb86371e, 0x000c51076d256a06, 0x00000000000ae3c4}}, + {{0x000340cb6908d50e, 0x00092ba2e95430b3, 0x000660e7e985a29e, 0x000b95145bdc19ee, 0x000e382e77bdf94d, 0x00020b29a951d00a, 0x0001f19940a5fbb2, 0x0000000000058fc9}, {0x000d804932dbc0b5, 0x000be682e42eaaa2, 0x000400a2efd4aee0, 0x000810016294d055, 0x00032e326d68be15, 0x000e64fceaea13fe, 0x000a8ac0dfe1ef15, 0x00000000000b8237}}, + {{0x0004480f8fce3f16, 0x000a7e59b80017bf, 0x0006c7396aa46dc8, 0x000172a5af5b47f5, 0x0000160d7e8d8799, 0x000f9a549f72c978, 0x00044a1d1ce972b2, 0x000000000004857b}, {0x000d15fb2758caea, 0x000542545bdd6f77, 0x000984fe91e9b1e2, 0x000343a4e23c0644, 0x00091d1fd9cd5a60, 0x00070b5b3986779f, 0x0005a35bd5611b35, 0x00000000000f9d76}}, + {{0x000b72123cb1cd13, 0x000e76ee65a0886b, 0x00081e4e332045b1, 0x000cc382876e523b, 0x0006d3bf53aac4a2, 0x0007f290cab7aba2, 0x0005bd5bf00871db, 0x000000000001ee6d}, {0x000bcea869ddbc32, 0x000334cafb21874b, 0x000bcaaf9d600f4d, 0x000785520b281cd0, 0x000e64d9c65ea1fe, 0x000a0e67be457198, 0x00068aa3d1a6d0c5, 0x0000000000090cc4}}, + {{0x000450a44e08b4d9, 0x00014cb0a365753f, 0x000b82633a02b2b6, 0x000210997ed887af, 0x000f30d9b2970b85, 0x000fb9c745fec3e1, 0x0007854ce4149f10, 0x000000000008cbff}, {0x000dc4bd785a06f1, 0x0009f81b0d7b3b6a, 0x000116390fc1ac37, 0x00021de2b841eb88, 0x000ad83c22e6aec2, 0x000affe9162c7d86, 0x00081d5504dcf885, 0x00000000000f4454}}, + {{0x000578651af84c0a, 0x000e0d4ee3f7f52b, 0x000cec289c787837, 0x000ee1363ebab5bb, 0x0007005ec2374c0b, 0x0002fb00670e32d7, 0x000899302fc73dc5, 0x000000000008f159}, {0x000ba114d96a8216, 0x0009d42a5478e2d1, 0x000e66d84b639b08, 0x0004970c8378f0e8, 0x00058e2c86c5042c, 0x000c7c76770c1957, 0x0003a861a95e6884, 0x00000000000d6fb4}}, + {{0x0000a2299e18ff96, 0x0001ceaf237a8503, 0x0006d80455ecbada, 0x000fa473f251ad61, 0x0006d828578e5fbf, 0x000e118adc40570f, 0x0005485cc65c0dd4, 0x000000000005da48}, {0x00073e60bf0732eb, 0x0000fe27fc2e7307, 0x0009067267d2e6a8, 0x0002fa55e27fb12d, 0x000810003fae35a3, 0x0009800c17fcfd72, 0x0002e6c74b50a3f4, 0x00000000000dbafb}}, + {{0x0002a6bfc8996b96, 0x000bd0c62fd2c8ba, 0x000a840806b7cf85, 0x000933dcef3f9e43, 0x000d9889ffa276b0, 0x0003c88d251b1ec2, 0x00052f9e84b2ba9a, 0x000000000001913e}, {0x000a4507b899f92f, 0x000e6bafc5e94164, 0x0002238654296051, 0x000cc41bed171099, 0x00036c7a41c84e9b, 0x0005369cd0db5b73, 0x000934d4be07a779, 0x000000000007bd3a}}, + }, + { + /* digit=55 [{1,2,3,..,}]*([2^220]*G) */ + {{0x0006d08f59277dc6, 0x0008a3f2eff5384f, 0x00049c170a3dfb6a, 0x000b18a0dde190dc, 0x000da26b0409af10, 0x000b1d944f491b98, 0x00054166080782a2, 0x0000000000097e8c}, {0x000baebab71369f0, 0x000d1fdbfc5f5495, 0x000a70804cb1f0f5, 0x000263857645ef4f, 0x0006a02583638e5d, 0x0005d250331bcfda, 0x000285f5330ab8d3, 0x00000000000c7cab}}, + {{0x0004a7ee3780eead, 0x0001ef16938f4dd4, 0x0005af2b9dcbcc11, 0x00095530b6490d71, 0x0001a28a296a2d50, 0x00026415c8432fef, 0x0008656f254dd08d, 0x00000000000d50c2}, {0x0005457026e64224, 0x0003c4f5bc4553f7, 0x0006183dc27db1b2, 0x000dd6e65e593411, 0x0002a56dc2eabab8, 0x000a90e05676baca, 0x000da038eea06bea, 0x00000000000174ba}}, + {{0x00021d43da6aa865, 0x0005de6a19dcb664, 0x0006a4c857b63184, 0x0009b9fc6455613f, 0x0004a7390d0eb4d8, 0x000ea135a6cb0fe4, 0x000982ade197a459, 0x0000000000020680}, {0x000776554c3cb5c6, 0x000b803db9be90e0, 0x000e56e339783849, 0x000e8d4753c196c6, 0x00000b7c6ff544de, 0x000a1b14259adcc7, 0x0004f2c6260ec24c, 0x0000000000046cbd}}, + {{0x000c69e90d279f7b, 0x00051a35e411c1f8, 0x000aa4eec7d05943, 0x000859e89a66f2be, 0x000e0def8ecd7c7c, 0x0004947b79908c37, 0x000ce88274124e34, 0x00000000000568b0}, {0x000eb0436a41e1a9, 0x00070e52919611c1, 0x000a98c568a44a8e, 0x00039e156bd3a7e1, 0x0006268f856260fb, 0x000fd8293e56a34d, 0x000fcbb3a1fe1613, 0x0000000000067537}}, + {{0x000d9811d879790e, 0x000da8a15d6fdcb9, 0x0006c38fcc4a52b8, 0x000e38c55bbe733d, 0x00051ff94a9d7a7e, 0x000585ab5eff146a, 0x000a5de572d13edd, 0x0000000000006491}, {0x000a7bbc541e4f72, 0x000a29c84e1c4d63, 0x000bbb62e6c0b1d5, 0x000cd9b385f06c82, 0x00077a7759c3db12, 0x0003bd7060c93eb9, 0x000d50f1f5b0fe68, 0x0000000000085ec3}}, + {{0x000e87011f7cd75e, 0x0006e89e48d8ba73, 0x0007fdc53e3e2631, 0x00033d7302c0daa2, 0x000a048eefe360f0, 0x000a7224415e4578, 0x0009cdfd6dec89b0, 0x0000000000030948}, {0x0003345fd128739e, 0x000ad7cdcc2a0188, 0x0002b63966c3b413, 0x0000a455812b560a, 0x00052ca31d8ca630, 0x0003a5b5a8fa5c41, 0x0004e036aa3c234f, 0x00000000000c86cf}}, + {{0x0004ff5664ce36b2, 0x0005e9a0e15351cb, 0x00019cbdd0d2f66a, 0x00059eafb29777cc, 0x000ae354cafdc170, 0x00007c3717e40e5f, 0x0009459cf594054d, 0x00000000000a71c3}, {0x0007429ea783b1e9, 0x0003469309e95af4, 0x0004f55088c266f7, 0x0004070e25823b6e, 0x000d0bc27359f216, 0x000925094ead851b, 0x000f4e3d21bfe8b0, 0x0000000000034a97}}, + {{0x000a4c18541d03ec, 0x0004ad927282fbf3, 0x0005c034c274cf2b, 0x000207f450db7135, 0x000423e16d9558b9, 0x0000e349cae95338, 0x0002bc4f10c6d4e6, 0x00000000000feb12}, {0x000eced76985b33a, 0x0002f22548cd1c2d, 0x0005b37b87399908, 0x000f912b6167b3cc, 0x00027902d2baa1c6, 0x000de34ba6967bab, 0x00025eebbe0b0836, 0x000000000004b796}}, + }, + { + /* digit=56 [{1,2,3,..,}]*([2^224]*G) */ + {{0x000e99ecf706c6bf, 0x0005c9e857f32800, 0x0001e880e21c15d7, 0x0008d68ff4f65674, 0x0005ac339148f853, 0x000dfc12f35380f1, 0x00093efef0bfdd5d, 0x000000000001387d}, {0x0009274bbe5eb9e6, 0x000aa618ce77c94f, 0x000ef0d12ae1c332, 0x000f06e00dc0da6a, 0x000e07603cc724ea, 0x0006963c7049113b, 0x0003005cf489088f, 0x00000000000ede4a}}, + {{0x000abae3c29bb132, 0x000af77e486f79a6, 0x000ea167f51170e1, 0x00028ab7df36628c, 0x00016704dcd6322b, 0x0009a35672d14d13, 0x0003b6d359977af2, 0x00000000000ec96d}, {0x00053212afaa74ef, 0x000f0fd6e400a371, 0x0003860e13fc28c5, 0x0001c7d9b8533afb, 0x00028de66eb862d8, 0x0006784eeefa638c, 0x0002237405a9d7e8, 0x00000000000a6c22}}, + {{0x000fc1e6b9032350, 0x000a46909994e4c6, 0x0006261c6638f0ac, 0x000ca05884aacaa5, 0x000996995a981505, 0x0002c000ee4b6530, 0x000b0930e00a5ed0, 0x00000000000236e7}, {0x0005ec99c1d0db26, 0x0002ce696f09d532, 0x0002f7914e3f9268, 0x000a7b401e1e2a4e, 0x00069d2d025aa9ad, 0x0004ffeb19630acb, 0x0004f69fab2c6ed8, 0x00000000000ab758}}, + {{0x000c87e27d06e6af, 0x00073f9b2dba43cc, 0x000cbbdd7e7ab099, 0x000a4f33b8104eed, 0x0000e4e1896e7692, 0x000d2aa365da885b, 0x000bcac2a30fec73, 0x0000000000086f60}, {0x000adfed330d989a, 0x00086bc8bf16d541, 0x000f4b7104707db4, 0x000e2f37e35610a9, 0x000482f9d71c8e79, 0x000e62733981139f, 0x000061f8997ec424, 0x00000000000a3518}}, + {{0x000efb4736bf4182, 0x0003f6cf0e6ef64b, 0x000b24ffed39dfee, 0x0007783856111dce, 0x0000c0e9f2b00277, 0x0007fe5073a1d36f, 0x0008f1fcf4f6365e, 0x000000000007fa7c}, {0x000ce2543c17ec02, 0x0005509a02de874d, 0x000cd3e25ee5e59f, 0x000a9654b7f4e35d, 0x0009805b58bd7211, 0x00057860ca6b2ba7, 0x000d58418302c209, 0x00000000000f99f9}}, + {{0x000634f7fb73c6b6, 0x0002e4e6ef40fcf1, 0x000c701a714f0702, 0x0003403fd41d144d, 0x0007e0774c37a4f0, 0x000c7484a3a50717, 0x00066b078e8c568e, 0x00000000000fbb3f}, {0x0000fb0d6daae4e9, 0x0002c169c9474ce5, 0x00027d6aef77ce07, 0x000968508303114b, 0x0008fad0def23e8e, 0x000c1c8da7a9797b, 0x0007210ad14404ef, 0x0000000000021ced}}, + {{0x000169a6e51baf05, 0x00088fde0d1b3e6e, 0x0008e5407b7aa0be, 0x000ad79c9eb9de48, 0x000b0ffbcdac16d3, 0x00020287c2707ec8, 0x00055ad7e6750fa2, 0x000000000009c2e1}, {0x000dcbd856a04522, 0x000e43018c309307, 0x000def4e0648d266, 0x00023aecf15a4af5, 0x000cc1b8cca01aa3, 0x00043a969f085d69, 0x00043047a3eaccb1, 0x00000000000f3a98}}, + {{0x000270a279eabd20, 0x000d5c7e9ddef0ef, 0x00060c66b8938b7d, 0x000746db239bb82a, 0x000b28ea13416bc0, 0x0001309b0c811a8e, 0x000b345f714ca71d, 0x00000000000d4eb9}, {0x000f50441ed062cb, 0x00091e0e5afdcc03, 0x0009d20438aad877, 0x0001e7b843343663, 0x000a0eed1116670e, 0x0009bbd50c8c38f9, 0x00095af914fae261, 0x0000000000051c19}}, + }, + { + /* digit=57 [{1,2,3,..,}]*([2^228]*G) */ + {{0x0001b38ab493e121, 0x0005bde849cd1240, 0x000576b3d2c358dc, 0x0009e3dabe92fbab, 0x00043324900a3fbd, 0x00020904e785414d, 0x000ba8daead1abde, 0x00000000000aa5f1}, {0x0002d0438c4bd099, 0x0002fd60a4f2ce26, 0x000593174efc1656, 0x000c78934efa243c, 0x000f216a8d8c163d, 0x00001617b3067dcc, 0x00051e116b6534a9, 0x00000000000bbabe}}, + {{0x0004b6e85f0076cc, 0x000c1929454f6549, 0x000021b9b8ac3fe0, 0x000a7c5ee25c0b0a, 0x000f2e752295f5b0, 0x000acac687d3372f, 0x000e3cd6dadc7d6e, 0x00000000000a96a8}, {0x0006465e062c14dc, 0x00030ea831db66b4, 0x0000548165c8c6c9, 0x00017e572e3c00c5, 0x000d2a5fb6ba5ff8, 0x000392476b022e25, 0x0005a0b611c5bcbb, 0x0000000000019048}}, + {{0x0000dddcc280d252, 0x000d5efda99d90b9, 0x0002988f9d0202d2, 0x0006cd1ad14ac705, 0x00031f4138808b9e, 0x000dd7fb91239ee3, 0x000b12d98e93d993, 0x00000000000894fc}, {0x000d9440883321ae, 0x000d433a92019c9f, 0x000ee20fd3f674ff, 0x00051280d0a320b4, 0x000b4b607b538450, 0x000228c2ec20551d, 0x00025c6e63c766ea, 0x00000000000ac48f}}, + {{0x000ea4ad3f5b0bfd, 0x000140372678d84b, 0x0008ab3dd6009aeb, 0x000bca4b79594c43, 0x000baf3b75cfebae, 0x0001e09c6e587850, 0x0004cd534183ac2c, 0x00000000000c0820}, {0x00012c542116a023, 0x000a7dac2cf06c18, 0x000f5e79e9f15f10, 0x00009f490b0f6c27, 0x0006c2c62207f6f5, 0x000ff18873ffc3cd, 0x000c6fbb21eb1132, 0x00000000000e62cc}}, + {{0x000a11bec64a35cd, 0x0004f1ca74a30c77, 0x000de3a654c55d5a, 0x00049038d6b4b005, 0x00002f7906ee3709, 0x000452bbb12ba86e, 0x000039aa76f77adc, 0x00000000000f9837}, {0x0000782bfd430ed7, 0x0001841fe306f87d, 0x000ce68ff3cdd73a, 0x00024ccaa7d44b2c, 0x0005d86900f9cffa, 0x0003ecfa022bae39, 0x000980e7a138782e, 0x0000000000086d28}}, + {{0x000489915b0c1e42, 0x0000991b8b685879, 0x0005ba3b38e17597, 0x000d69ea7d9931a2, 0x000c7632a26bcdb0, 0x000ba170f3c8441d, 0x0008adf11a365c62, 0x0000000000034e74}, {0x000d3f6d2f87f536, 0x0009d523ca2d7db2, 0x000b5fe1b40ff204, 0x000c7771d07308bd, 0x0007c291c2ef71af, 0x000b40d1773588d7, 0x00015629baa3b0aa, 0x00000000000eea9f}}, + {{0x000a95a8a249bd36, 0x000f90ae9143a26d, 0x000709bd167b8510, 0x000f7a7f3b4b882d, 0x000de22d9b9923df, 0x0004f02b1b178e73, 0x000c2fa83861afaa, 0x00000000000b9064}, {0x000de7d4573c6d34, 0x00022142eb294574, 0x0006f55c30205aad, 0x000717fe649a8b70, 0x000cad53c9bbd589, 0x000ecd8f7a925c47, 0x000142c339b11a09, 0x000000000001bbf1}}, + {{0x000abd60f6eb49c8, 0x0008c406a7e201fa, 0x0007246dc14c8322, 0x0002eff020748efc, 0x000af39b58dbd440, 0x0008cfb047827442, 0x00078f77e3f2768d, 0x00000000000e45a9}, {0x000ab42dc779cb3d, 0x000229db0829881a, 0x0005eb7284cde06b, 0x000ce47b82775f69, 0x000f63910016e434, 0x00047792fe84995e, 0x000eb9a35e8b971e, 0x000000000007c6aa}}, + }, + { + /* digit=58 [{1,2,3,..,}]*([2^232]*G) */ + {{0x00097b7667d86ea7, 0x0001b1fa064cf475, 0x00026db64fb0c148, 0x00002a1fa9d94539, 0x000bdc6bd7eada81, 0x0002f6044786aeca, 0x000208caf91e3bca, 0x000000000008573f}, {0x00036746e95246de, 0x0006cd309fce8dbb, 0x0001300c9068d932, 0x0001ae0f3d530575, 0x00000d1fd61e5779, 0x0005ebfa626b053f, 0x00097991c962c004, 0x00000000000076ed}}, + {{0x000013c3d6e02921, 0x000c449f2499410e, 0x000e2ab53501cdc4, 0x0009103d5e91bc0f, 0x000bcb404f68897b, 0x000f7fc0263db1ef, 0x0000af70efd9d842, 0x00000000000c6a23}, {0x000a0390300406c6, 0x000308d5b199ee1a, 0x0009868ea4fe37e1, 0x0003d34504dd889f, 0x0001f823686fde58, 0x000fc73dc0375600, 0x000f42f19ab86f95, 0x0000000000093f12}}, + {{0x000b8a18e1e24b49, 0x000143f896ca9186, 0x000d7ce3f5b4c07a, 0x0001632600e4e2b2, 0x000f69e702d5d074, 0x000e0df2b87c7db5, 0x000fba1f5a3b8053, 0x00000000000f443d}, {0x0005af3bc7c98ae2, 0x000deeb90e22f972, 0x00070953899b58fe, 0x0008299a5aa335b5, 0x000d4197b32afb1e, 0x00055918ed78504c, 0x000c720e7a79cc67, 0x00000000000883b1}}, + {{0x0004af5925d66db3, 0x0008dbf66baa58e7, 0x0001c386a0ca25fc, 0x00032abeaaa466ce, 0x0007e5f2733b80bc, 0x000531afa605b789, 0x000369e9e7e3a1a2, 0x000000000003deb8}, {0x000174c1d570e84f, 0x0008d36212ea2dd8, 0x0008479c4475fe18, 0x0004b31444e9ea02, 0x000169530c1befa5, 0x00079bdd19c2229a, 0x000cc368feb9854b, 0x000000000008b984}}, + {{0x000682b315cae64a, 0x000981df8d98c41e, 0x0009fabd7bca288f, 0x00064f91703e1431, 0x000e9de0ab4ca54d, 0x000dc4e0fa12198b, 0x00091f160e06241d, 0x0000000000066958}, {0x000b5c1e9cafc463, 0x0006f808565e66f7, 0x000a9467f76914c8, 0x000cd934bac17690, 0x000ca5be965f6682, 0x000a38e9062e7ac2, 0x00016ed33f6f8ad7, 0x000000000005b8b5}}, + {{0x000509d8741fea0a, 0x000c1f041e421362, 0x000c0d8899228fbc, 0x00093751128ed62b, 0x000acec6eae64fea, 0x000a65dae041a1c1, 0x0008e81f32359415, 0x000000000000154e}, {0x000a9eb331e84371, 0x000ec309e9f286a4, 0x000765936bc21528, 0x0004dd5692420c27, 0x000ec4fa1a6a7bb5, 0x000779e77193a41a, 0x000b8c35f5f3b43d, 0x00000000000046eb}}, + {{0x000b5769d7b9ec0d, 0x000aa3a5a366dc99, 0x0005d073ce2a60d9, 0x000e4aafe5355bee, 0x0003ced676663e16, 0x0003440036d8bac0, 0x00020c403eb33ed9, 0x00000000000333ab}, {0x00094bfc36e2db30, 0x000739fce19869f6, 0x000435b17be8c513, 0x0009611921a58e5d, 0x000620d5c61a8e68, 0x000c81b4e8f5f115, 0x000779b17f612fad, 0x00000000000e562f}}, + {{0x000d75f385d1b0f1, 0x000f0d6a4d25bfe8, 0x0007b705bfa0d54e, 0x00059731cdedfc0f, 0x000603d6502f9420, 0x0005ce8c80c4e385, 0x0000b7981a4fc5e1, 0x000000000007aca3}, {0x000ea22cf2bcfc17, 0x0009ef2037ef684f, 0x0007e5010cdb37dd, 0x000a23c71f3e4e4b, 0x0009f47b504b9c98, 0x0003233aaa73c8b8, 0x000f68b9e33f5402, 0x00000000000f92c9}}, + }, + { + /* digit=59 [{1,2,3,..,}]*([2^236]*G) */ + {{0x000a9e3a73909533, 0x00090ba03ba3b07a, 0x00090d7a3c9c5a5a, 0x0008cfe4f0f60b35, 0x000e6fcccd96f96b, 0x0009dd17ab908d77, 0x0000487208ef7de7, 0x000000000003ec3d}, {0x0006af6a704d4f0e, 0x000d09c5ad2d9a11, 0x000e77d5943d9764, 0x0009449470eb938b, 0x000bee7d772fac99, 0x000b7ad09faaf27f, 0x0000a9fbe402abd0, 0x0000000000057db0}}, + {{0x000ca62ea2a4a457, 0x0001b10c082a59d1, 0x000bdd4313beafb9, 0x000935b4cb291a7f, 0x000313e9ce08785d, 0x000f6f1c4fc2ae15, 0x00024c3146fabf4d, 0x00000000000c87dd}, {0x000f74ecc24bd4cf, 0x0000385fdd8765b9, 0x000dc418405d512e, 0x00005013e7e0297e, 0x000fb92df904c81d, 0x0005ddccbb56ddd1, 0x000f1d4612df9f29, 0x000000000000e27c}}, + {{0x0002069dae7548ac, 0x0002986d9b05f69f, 0x000025ad33463063, 0x0008e7c27d9d64d3, 0x000aba04e6ad9b6e, 0x0008abbbe79bd66b, 0x000d0477e4d0b082, 0x00000000000cc540}, {0x000bdbdb8d90e2e4, 0x00067f5d8a46ef90, 0x00078bfd47c637af, 0x000b852563fe6b52, 0x000997cdd04fb93b, 0x0007de47d06bb3ae, 0x00061c07f011d48b, 0x00000000000de78f}}, + {{0x000eb7904785958b, 0x0007d7830460f8a0, 0x000cf49e72cbbaa0, 0x000e2aa307d9c790, 0x000b5aad8b6c73be, 0x000848db51b02af1, 0x0004765e31882703, 0x00000000000f230f}, {0x000b79128735694b, 0x000bd4ea6535cc84, 0x00008e33135971e8, 0x0007b332fd33cac2, 0x0001c566914f4c19, 0x000952d3b24c7b0b, 0x0005f2956ce3c371, 0x00000000000fabae}}, + {{0x0004121555388dc8, 0x0008b8a95e5c1a8a, 0x000cd5dbd7b0133e, 0x000df6acda40bc0b, 0x00058234471d1859, 0x0003c0100f9097aa, 0x00067caddc0c9b78, 0x00000000000feb70}, {0x000ce59ade26052d, 0x000a6e3805fd0a27, 0x000626ac8acaae6d, 0x00099921943a0a1d, 0x00091dfe627ea459, 0x00006515fb47f061, 0x00073e313d4f09da, 0x000000000001f54a}}, + {{0x000f6dd7137bd615, 0x0007e0db87bc3c53, 0x0002eda89127238c, 0x000e9c4a3fd2a60d, 0x0001c9f017e5ea81, 0x000c75768190c9c7, 0x000de621864180bc, 0x0000000000043dbc}, {0x000fb5dd8276da18, 0x000de5b090c70dac, 0x000cd9057894e345, 0x000d268a918bf24b, 0x0008ae204f49fef3, 0x000d7c356e10c52c, 0x000a17f4be898d86, 0x000000000002fdb5}}, + {{0x0009bfc4aad5cc44, 0x000a20079c69c96d, 0x0008a5713957941d, 0x00086660139685a2, 0x0004946fbeddb8d1, 0x0001aace408590ca, 0x0001b67fecd9b964, 0x000000000002936a}, {0x000267114a49f247, 0x00065925b6235aee, 0x00055e3538cd2015, 0x000663d8c6fb34ac, 0x000c2fc1c10d971e, 0x00042b822543146d, 0x00024d6e4053c706, 0x00000000000492e5}}, + {{0x000d356fb82e9b9f, 0x000beca9872e0a14, 0x000f54683a031c30, 0x0007cb84e05b2811, 0x0006fa234d4596b1, 0x00035a7d89798714, 0x000384ba78949ebb, 0x000000000006fc59}, {0x0001361f78e02fa8, 0x00081a303e549f81, 0x00064be08532a2fa, 0x0002de8ee7220467, 0x000563e27035e57e, 0x000d2fe6fa05c106, 0x000aadaa38e86602, 0x00000000000ee2f6}}, + }, + { + /* digit=60 [{1,2,3,..,}]*([2^240]*G) */ + {{0x00044971cf281b0a, 0x00052c0426b768f1, 0x000ef3f4445c186e, 0x00012e3172c0d3e8, 0x0006ee75473731d3, 0x000a7ee615f49fde, 0x0005fb895530f06d, 0x00000000000e8b3a}, {0x0006345afdc270e9, 0x00019fd14973443f, 0x000f6896912e434e, 0x000ae07653908d03, 0x0006ba02a278e2ba, 0x00021b8f8c3d0143, 0x000297a0d0222e7b, 0x00000000000d7ec1}}, + {{0x000653659b1a252c, 0x000514f120aa7478, 0x000c72dfe03d7757, 0x000ac5ecfe5f7a92, 0x0009bbf3cec6c96b, 0x000361cd5d4e73d5, 0x00044ced8d233560, 0x00000000000562f4}, {0x00045d3e2b7ac684, 0x00022bd37d3cf9b9, 0x000cb601f2d0a968, 0x000535a3d2f41ee1, 0x000ee8b1743e7e35, 0x0005a27650353b52, 0x0008b831d89dfd7b, 0x000000000008d9ea}}, + {{0x000718fb55d90569, 0x000b306a67bd2493, 0x0001471031374c3f, 0x0005d5197bc62d32, 0x000924c51874ee0b, 0x000a1a0d552b1703, 0x0000acfed1f42382, 0x00000000000db627}, {0x0006189cf7edbc97, 0x0006c36be4a9b658, 0x000680236e8f5c91, 0x00036d3b8f8074cb, 0x0009718545c6c174, 0x000757d213bb4d39, 0x0003668e1ea3555c, 0x000000000008c474}}, + {{0x00063be615177c6f, 0x0002773457010af5, 0x0001ce08b2f26f1c, 0x0000e8c9c25fe5be, 0x000182dd0485705b, 0x000ac280540f36ea, 0x000b923d55bc8527, 0x00000000000ad921}, {0x000b6da293461f09, 0x0009551586cd4c76, 0x00086171a05efa67, 0x000605e84f0abcb8, 0x0003772dd0dabb4e, 0x0004e1d41354ef8e, 0x0004917f1a8f795e, 0x00000000000de5d8}}, + {{0x000beb4ebddc46f4, 0x00073ec72c64fb0c, 0x0005bac2d9d9096a, 0x00022001819efb1b, 0x00068cdde8703c5e, 0x000a87aeedf5ab6d, 0x000f1975a44e9d92, 0x00000000000bcf77}, {0x0009407ed3c226cf, 0x0005e8191efbc92d, 0x00064a74c9c1339d, 0x000e58265cf242d2, 0x000180b1d17be62b, 0x000de59a9ae99a3b, 0x000ce248cbb44692, 0x000000000002dcb3}}, + {{0x000a48783de6cfb4, 0x0006bf3899558552, 0x0009d51bfff43e77, 0x0004fd32df8d1a75, 0x000376d3fbbf0b1c, 0x000fd52bcf16bcc2, 0x0001f0d5888916f4, 0x00000000000d5cde}, {0x000f03d1ac917a2c, 0x000ae764ffffd280, 0x000af8be538ef59b, 0x0004762ccd57b860, 0x00032935106234f6, 0x000c642f32233a4c, 0x000f34df076095d9, 0x0000000000059f0d}}, + {{0x00010c66eff8425e, 0x000379580cdfaafe, 0x000d1f7ccb185b5d, 0x0005f77c327f3e8d, 0x000c35353c5f5d3d, 0x000258eb105d5339, 0x000f79c56fb5fe5c, 0x00000000000edce1}, {0x0005bd6f7b6e122d, 0x0007cab7aa141541, 0x0008987b379beb7f, 0x0001491458d9e533, 0x000caa7f0f31e124, 0x000fda7abdd2448c, 0x000a4dec58d3c7f0, 0x00000000000c91bb}}, + {{0x0002f037fabc6138, 0x000b73bd258d77ca, 0x0006aa4d0ec1d1f3, 0x0002512e3f966a14, 0x0007709d0c2d5b43, 0x000658259338bfca, 0x000023d142cc1049, 0x00000000000636b8}, {0x0007458ca547abc1, 0x000cda9ef9400a80, 0x000ad926836a9402, 0x00063c55cb644887, 0x00011cea475bfd2f, 0x00067a25fbae949b, 0x000a6aa45446031e, 0x00000000000dc6a7}}, + }, + { + /* digit=61 [{1,2,3,..,}]*([2^244]*G) */ + {{0x00085fa16820f665, 0x0009fd699ea2d24e, 0x0004f1772a862ed3, 0x0004bad66a8b35ba, 0x00024233fccb4660, 0x000d3cf0c0c779b6, 0x0007af578458acbf, 0x0000000000096bf5}, {0x0008a325d9d68d07, 0x00045a9724244e54, 0x0005b4b1e20150f9, 0x000a5b8c6be8c159, 0x000c774d62c40980, 0x000bde24b6230e3e, 0x000204da1467d84f, 0x00000000000cc862}}, + {{0x000b4d1a75edabf3, 0x0007567c51633fd8, 0x00020dc66cdc521c, 0x000c8dc9ee450d03, 0x0008b41a3e2f77fc, 0x000bf06898dd2b31, 0x000464df6a935e93, 0x00000000000a92e5}, {0x0001e3ee6beb3c9c, 0x000e449afcd9ef46, 0x00031a4b44405106, 0x0008ad2c7ea7810a, 0x000e550822b2cdbd, 0x000606adcfe61571, 0x000110744a4f9386, 0x00000000000d9d4e}}, + {{0x00006ff4ac15d783, 0x0007ef1084276ccf, 0x0009d3b1212d957e, 0x000dcf5bfb4283a4, 0x000db74017eb3752, 0x00078fcf8f6b2214, 0x00039afc1cdf7245, 0x0000000000012265}, {0x000c5dc1b7858cd2, 0x000cdcc0796680d4, 0x000e05b222bc5975, 0x0003a9a504cf7d65, 0x0003c93ed5932027, 0x000303f1b0c7b7e5, 0x0006a4aaf9c36866, 0x00000000000cb013}}, + {{0x000bfa5cdf24bf96, 0x000411fef389c07d, 0x00022753fd218088, 0x0000a1437f04a344, 0x0009d0169369bd77, 0x000377cc3c7438e2, 0x000a4f6b265742e2, 0x00000000000c369f}, {0x000bb384dc3d9a84, 0x00060dfdbcf462e9, 0x000d3f52e65bb342, 0x000a0b82a9c483da, 0x00042de432285574, 0x000d1fabe0563fe9, 0x00096658ca0e8aea, 0x0000000000066023}}, + {{0x000afbbacd3ede36, 0x00007746325d090f, 0x00094f8b4a38ccef, 0x0001c2866c3931a7, 0x000a783a73e7d9f2, 0x000d82d13c12880e, 0x00010e382e1ce28b, 0x00000000000ac023}, {0x000fda6b09a40144, 0x000b69802d06233d, 0x000de8140422422b, 0x000367efd4cf75a0, 0x0000e8f2f6ed38b4, 0x000e72ff4765cdee, 0x00070ae0b4d72b35, 0x00000000000947d4}}, + {{0x000a35bb9d72eb2a, 0x000c5383bda07268, 0x00038c9d09f99c2f, 0x000717d369f39c03, 0x00011a5a39006f3c, 0x000ec6c2b1bb593d, 0x000202d0f07ecc2a, 0x000000000004240b}, {0x00083c5449860db8, 0x0001935342f6c7b8, 0x0009a1541ab519cd, 0x000eb09ccb6a888b, 0x000785aa42c5fcd6, 0x0004e5895abb7a6f, 0x000582952f8824aa, 0x000000000005c406}}, + {{0x0001d7b0f8433a5d, 0x0004359d6e052cda, 0x000fea341e325461, 0x000d07d7907cc890, 0x000dc4ce5d800459, 0x00004f40267d720d, 0x0002a83262028eed, 0x00000000000d7881}, {0x00055f59d844fe29, 0x000fcf735fd6cf7f, 0x0001c0c0179cc733, 0x00006e8e19a43f29, 0x000f19592b76328c, 0x000b836f7b97ef65, 0x000a9981325f3db8, 0x00000000000d6e6c}}, + {{0x000a67318a4b19fa, 0x000a63667a71faf0, 0x0007be6235b29837, 0x000535efc62f7919, 0x000b389faf7fe084, 0x00071b7f65bc1652, 0x00070340cf51683a, 0x00000000000d4f39}, {0x0002b576e30c499d, 0x00099823e7549478, 0x00032769bfa306a4, 0x000ee027225b31ad, 0x0006fc282f165639, 0x0009f61ae7533bc8, 0x000803710009d2c6, 0x00000000000bf65e}}, + }, + { + /* digit=62 [{1,2,3,..,}]*([2^248]*G) */ + {{0x0006c5f4c042c4cd, 0x0001ceb29e44bf59, 0x0004c11cc5ce653f, 0x00004943d2bf689a, 0x000a47428dd2d09c, 0x000aafac83ab7799, 0x0006e0dc558d6be9, 0x0000000000087f9f}, {0x00056bc34f65dad0, 0x000c793842bcd3a9, 0x000241a2ffbfced6, 0x00052687e6d47b5b, 0x000a4c37eeee1645, 0x000c412ceab304b7, 0x0002c8dbb3d4e13f, 0x00000000000a726a}}, + {{0x00059e97675084b3, 0x0008d79d88dffddc, 0x00002dc16c994a53, 0x000000fce7d606f0, 0x000e2fa27fd3b528, 0x0009436afc773557, 0x0004c755b53dd3e1, 0x00000000000e10b6}, {0x00077a15a41de95f, 0x000bfe5832664b2b, 0x000d15e689d49c17, 0x000bf537af3e3dd9, 0x000cf47d298c7b93, 0x00012136994fafa2, 0x000ff694c2ff9a21, 0x0000000000033468}}, + {{0x0001b740495480b0, 0x0003f91e8c991baf, 0x000b4c043871430f, 0x0003f6dd095f5b94, 0x000cddf3cad27c5e, 0x00057aacfed7522f, 0x000f5180bb87056c, 0x000000000000cc5f}, {0x0006cac5a2f35aa0, 0x000e0964def7e61e, 0x000e006a84529a7b, 0x000192584de66a22, 0x000e075f07c5cc75, 0x000eade939acaf7f, 0x00058f6505c2f81e, 0x000000000000e3cf}}, + {{0x0001fb7f00850a1a, 0x0001dbe45d81a1aa, 0x0000ee7897985bd0, 0x000a516b078ea895, 0x0001b446476463c5, 0x0001e52329f3efe4, 0x00022084580e2410, 0x00000000000c8ae8}, {0x000539dde4d08a27, 0x00077bff4077d088, 0x000c1e715cd7b849, 0x000eef207e1c03eb, 0x00096e654d584df4, 0x0006f15964ab3d03, 0x000cd5b20056cd08, 0x000000000008acd7}}, + {{0x0008f4a5a8f26e12, 0x000538caa623c979, 0x000e7d0ec3691999, 0x000f856ee285bfe4, 0x0006d0674ecd089a, 0x0006f661277b461f, 0x0009f8baa9d9b38d, 0x0000000000059066}, {0x000460fe25d6fd2a, 0x000d2b164340c38e, 0x0003981c9e27c186, 0x0003b9176e4346f2, 0x00017caad5fc73c0, 0x00087803d6a678eb, 0x000a94103ff0790a, 0x00000000000f7430}}, + {{0x000dd3174fcc39a0, 0x00094517075af0fd, 0x000fd65c623f7ba2, 0x000e9493b86f6398, 0x00017f296bf4320c, 0x0001082765115657, 0x0009000919607a96, 0x000000000002f035}, {0x0004b29394d28cac, 0x0008f5d13de7c5ae, 0x000ea9b2719ceec5, 0x0008089c0f58697a, 0x00030ca7ca20f1b2, 0x0001e1be52adcd1a, 0x000ba096c07f42c2, 0x00000000000baa0a}}, + {{0x000dc0265a01c54c, 0x000233027c169336, 0x0003342d9c6b202a, 0x0006cd31b8b0179e, 0x000a6dd5a4a7e6eb, 0x000c82f110d2d27e, 0x00002241682c0007, 0x0000000000081075}, {0x000f1169b3430f67, 0x00019f3903f514b2, 0x00091dfa21d2d176, 0x0005eed470bd3b32, 0x0007e85a62931b62, 0x0005ad640a46398a, 0x0003a58ff6ef8c80, 0x0000000000095bfa}}, + {{0x0006e424a5742fd1, 0x0003a8cd6c9f7239, 0x000180484ef81e04, 0x000b0f589b8ad2c9, 0x00070d9c999d9c0d, 0x000f84ab4692a8db, 0x00006ca407fd03c9, 0x00000000000cc9a9}, {0x000b5d757bfbcaf2, 0x000014813a7654ce, 0x0008615f305ee56c, 0x000f2934fa0a23f1, 0x0007cb1f8d7aca6b, 0x000d10c531dfa3af, 0x000c1328036f5498, 0x000000000000e012}}, + }, + { + /* digit=63 [{1,2,3,..,}]*([2^252]*G) */ + {{0x00034e0d2aae29e9, 0x00091a53f1a10241, 0x0004dd23936a5886, 0x000ed8532976d137, 0x00065ee692d130ef, 0x0009e7cc1cf5ada1, 0x0004723ceda69d25, 0x000000000009baab}, {0x00095b8627836d36, 0x000237ee5baef4bf, 0x000e9b1caaa769fb, 0x0008ddacfdaff633, 0x0009cec076939e5c, 0x000fb509d575c8db, 0x0006979afc8ae0fd, 0x0000000000085651}}, + {{0x0000da11d13ab853, 0x00042a7342f9446b, 0x00039f5ef3dbc527, 0x0008bf07471c3856, 0x000130827f3450e4, 0x000779c23729e5ab, 0x000d5817199191ea, 0x000000000009fa9f}, {0x00077a9b9dc5fca3, 0x000827d5799369d5, 0x0003bee46a6f5cc0, 0x0007d51e85417260, 0x000dec720355253f, 0x000a16268107a793, 0x00031b1c14d0566a, 0x000000000008bbb2}}, + {{0x00016132445f3e21, 0x0009ef5689c5dce5, 0x0006397102e2e73d, 0x00079d012eaa5340, 0x0006d9271e941af8, 0x000e63c34dba775e, 0x00024b60f8507310, 0x00000000000a686a}, {0x0006624c82f5c532, 0x000dded5ca8f992c, 0x000b2edebcb9407c, 0x00039426d90a7d51, 0x000bccd37e76e2ee, 0x000412e0fe5b3e4c, 0x0000489013bd08f1, 0x00000000000ce999}}, + {{0x00009cf56d1d974c, 0x0005561d0e01b86e, 0x00046be97f98120d, 0x000e224b362902dd, 0x00029e0a5d16d4a4, 0x000e580e945923c1, 0x0009544fc2bdd495, 0x00000000000a8c3d}, {0x0004daa7a4671003, 0x000a1aeff11352c5, 0x0005a1c5b6c0120c, 0x0007d3eab8f9f31a, 0x00066d566c158090, 0x0002e6fa7af2c121, 0x00003a0082daba95, 0x000000000002df9e}}, + {{0x000fc79dfc540dc4, 0x0004091b379c535c, 0x00090bc691de574f, 0x0004208176b3b828, 0x0008aaba5cd3d0f4, 0x000794f9fd18c211, 0x0008944aed5c9770, 0x00000000000f4c33}, {0x000946afeb1a61f9, 0x000b9b8e67989163, 0x0000523ed92edf12, 0x000ef39ab5573985, 0x0004b71031051b7e, 0x000ac89e4175e393, 0x000db8943abf4a82, 0x0000000000057ffc}}, + {{0x00066f02c7c0f325, 0x000f899c2b4cc9e8, 0x000f2b3e2c2a45ff, 0x000b66b5a352b7b1, 0x00043ec757ed067b, 0x000d36adc3c7d6fc, 0x0003a42f69291cb9, 0x000000000007c914}, {0x0004d853fbc3268f, 0x00006768c3c3f0a6, 0x000dd7feada4c10c, 0x0004b025299ea1bb, 0x00091a2d4005d86e, 0x00017fa60be46b7e, 0x000fbeaf865a1693, 0x00000000000cdce2}}, + {{0x000297083b5951f8, 0x0002b66fcba529da, 0x000a6f9544c17490, 0x000b97241b4305fd, 0x000e25b6af89a371, 0x0005f9e8cb2d858c, 0x000ef8bab07d5327, 0x00000000000d525b}, {0x00073848e93e22c2, 0x00084ebf79e3bb83, 0x0007a079aa9da5d4, 0x0008b66cba1a65a3, 0x00038b1eb8b67dbd, 0x000a6436a88ea401, 0x000e33210ac1b38f, 0x000000000003df40}}, + {{0x0000b48bcae58bd8, 0x000c45f79f0de54f, 0x000db929ac4cc6a4, 0x000a68e11571c5c9, 0x000e2474faf7d631, 0x0002a924a6d072b4, 0x000d7a5e043a6d86, 0x00000000000b0fc8}, {0x000fb0fbc0056e2d, 0x0008bf54bb7f2f12, 0x00058e2455c41895, 0x0009f4c5909ec050, 0x0005abd164608145, 0x000b1c69ed28cda2, 0x0006f4bb676d018d, 0x00000000000ac503}}, + }, + { + /* digit=64 [{1,2,3,..,}]*([2^256]*G) */ + {{0x0005d534448865a4, 0x00079dbe04c30004, 0x000de820bf917110, 0x000b54670911457d, 0x000b3f7a1757dbf8, 0x0007bd62c9e683b5, 0x00097e7b89a08a9c, 0x000000000000b3fc}, {0x000ba720a0047774, 0x000980ac9abfb9ba, 0x00007bd7b6be79d8, 0x000c335d9deed984, 0x000f72603e080aa6, 0x000783bb2e270580, 0x0009ae70857a946c, 0x00000000000caa92}}, + {{0x00008763d38fe0f6, 0x00001f071c986bc1, 0x000413c627ede21a, 0x000b12a61483bc2d, 0x00001caf6dd6845e, 0x000c21b9217471d8, 0x00036d7b603616dd, 0x00000000000b9925}, {0x000a996617818c3e, 0x0007f433f065211f, 0x000b56653ac9464b, 0x0007386a9adf49b1, 0x00092cb2944fdf61, 0x000d9c0764bfeb39, 0x000576bf288427b3, 0x00000000000965b4}}, + {{0x000b333ccbe322d0, 0x000fbce23a199d59, 0x000074dc302e3830, 0x00000121d15a0063, 0x0005f4355a1fc720, 0x0007f780715fde73, 0x0006913849761f60, 0x0000000000018204}, {0x00074bfa03274297, 0x0002575756e4ea63, 0x0009809c9584a1de, 0x00032763d4ce3dc3, 0x000ef1d66e006312, 0x0008829e6a769d2f, 0x0006ef8624ddbac2, 0x00000000000d2df0}}, + {{0x000f606b2db32f32, 0x000b59ec11596f9e, 0x0004a5f37de5fafc, 0x00094e131fca111c, 0x000896076f45acf7, 0x000732588438b917, 0x0000daad71035b51, 0x00000000000a5d91}, {0x000d85cad07f3bba, 0x000a7efaad0e82d9, 0x000f829c6ffe7ff7, 0x0002426474ce3273, 0x000a2aa270da9940, 0x0008fa6ebd53b687, 0x00041196c8bb78d7, 0x000000000006e699}}, + {{0x00044c83de3d28f8, 0x0008b2dfc39e611a, 0x00099024ec940ff4, 0x0002123313aa4788, 0x000b9c6acbd11cf1, 0x000d54177785b025, 0x0001eab4aeb5b8e4, 0x00000000000d943c}, {0x00041945f056ffdc, 0x000097892db846ab, 0x0000f6f8e04921ff, 0x00013c61cbd3232b, 0x000700b6c34ebdee, 0x0003de32b873e0a7, 0x000302b279505ae2, 0x00000000000c5a03}}, + {{0x000395d621a49e0a, 0x000c7430b669fc05, 0x00093c4a448f25c5, 0x000730c4c33493e3, 0x000fca00ed3901ad, 0x0004a55168c882c1, 0x00027dab2e9c89d2, 0x00000000000f5d03}, {0x00020231a1107ea0, 0x000f946488bd7b79, 0x000d5a9d2b183f55, 0x0000e90aecfac7d9, 0x000fac6a9e8cf9f6, 0x0008bcda84afd1f5, 0x00099857a20bf8c9, 0x000000000008b46b}}, + {{0x0005f4b3119ad50d, 0x0006c1f2f6c1efc1, 0x000be8ee71be11e0, 0x000e4a6569c090bd, 0x000adb3986d020dc, 0x000f553eae4f7401, 0x000bff1389799485, 0x0000000000009026}, {0x0003665df6023a4c, 0x000b388067aa6ec6, 0x00069e9d017576cb, 0x0005019459bf26a1, 0x00058b6e76f68416, 0x0000d2434ec125bd, 0x0000b0d636f9321f, 0x000000000008e05e}}, + {{0x00021074c1c07346, 0x00068fe1f11ac76d, 0x000a3e93fcce998e, 0x000015a3c5babf98, 0x000d5929ea0a99d0, 0x000fb7b7e7e795b4, 0x0002e8c9cf68331d, 0x00000000000a6499}, {0x000716aaedb25dc3, 0x0000d0b7e0dd7aad, 0x00064fd90a09f648, 0x0003cf034f02b979, 0x000a5662f0c86402, 0x000058ccaf7dd410, 0x0001eda3bb6088a1, 0x000000000004e780}}, + }, + { + /* digit=65 [{1,2,3,..,}]*([2^260]*G) */ + {{0x0006b5199cfd1ff1, 0x0009d140a0f9f409, 0x0004971fd020b4cb, 0x0007c2304a1f742e, 0x0001195ba34e78ff, 0x0007f78b4c0568f5, 0x000a53e97183603b, 0x00000000000f9efa}, {0x0009e6e1cb2c1b4d, 0x000bae924d2c4efd, 0x0004b415d4c5ceca, 0x000e73f6ee37e106, 0x000a5e5a1dde4b12, 0x000cd64161836fdf, 0x0004d87f1b92259d, 0x0000000000067754}}, + {{0x000befa999003a02, 0x0000e279fd119411, 0x000606c204c4310e, 0x0007da4da44105e5, 0x000a28223fe1d8f5, 0x0006f2d2eb1814b7, 0x000e06cf2fd241e0, 0x0000000000001dfd}, {0x0001563b8cdc2810, 0x000a4876a31af711, 0x000bd72037bc4e78, 0x000d608493a6a0aa, 0x0008a88c03e75117, 0x000916897dcec808, 0x000c57eae1d352ea, 0x000000000006e8b2}}, + {{0x0001a9c4fdcf93d1, 0x000650254486c9e5, 0x0006796f28c8ff02, 0x00058ba000f54926, 0x000934009fb6fb58, 0x0002bde301315bdd, 0x000358b18a8e0ac4, 0x00000000000f1070}, {0x000b8de4d767059a, 0x0004ebeb1db7458b, 0x000305d015a22913, 0x00014d4e4122b217, 0x0002aabccc7b1522, 0x000d07571d3e673f, 0x000f1794c50f64ae, 0x00000000000e66b4}}, + {{0x0004bef0d3847d2e, 0x000aa09f8bb05816, 0x000388d5b381065f, 0x000c7b6076e13ec8, 0x00035d5ac07f26eb, 0x000ab69e6bda0b55, 0x0001fabcb8132248, 0x000000000001c0f2}, {0x0006ceb771ee0889, 0x0007b7a466528564, 0x000e55b024527048, 0x00011864c1d7cb8d, 0x000f2d08130185dd, 0x0005ea0f0096f849, 0x0009b2b4f503dd8a, 0x00000000000d1bf6}}, + {{0x000f0c6218b7c4e9, 0x0008540336b12c41, 0x000f74c446fc6c56, 0x000048d9c841f0d0, 0x00051d617f50c337, 0x00017d3794ce6d02, 0x00024300fef21981, 0x00000000000f95be}, {0x000f7d33fe9f8bcc, 0x000ec98de119d3a3, 0x000e778f8a8c16b8, 0x0005b720f9678bed, 0x000d334ace309412, 0x000e86e04fc5b57c, 0x00003909486527b7, 0x000000000006e552}}, + {{0x0001aa0f2e0127d7, 0x0007b4ab7a22bd4a, 0x00047f417a4172fd, 0x0009f95078de336c, 0x0006f786924520f5, 0x000f1d96ffd28fa3, 0x000f0f1de42581cb, 0x00000000000366e1}, {0x000e183b180aaf06, 0x000febc65fa9d0a4, 0x000739e25cf57814, 0x0004f3a4a0822a93, 0x000bfd05a0aa0638, 0x0005ee3ce1c81332, 0x000d00bfb12361d9, 0x0000000000042016}}, + {{0x000f0b32c63695dc, 0x000143b75c45b10b, 0x00037d16d187f9f5, 0x000227df8e0a114c, 0x000e73ddba245450, 0x000f246455e69f1a, 0x0007412007b80e57, 0x000000000003d159}, {0x0007528fbc79a394, 0x000ffcbe54d2888b, 0x0004e47907e4e2fb, 0x00070594cc39f745, 0x00032b1b8da9e19f, 0x000f68d2680f00ab, 0x000829b765aaf973, 0x00000000000e993f}}, + {{0x0000b5faa7527283, 0x000530a159f61ee6, 0x0008e1f9dcf0213d, 0x000b8ee1661a2405, 0x000ec95c41b36d1a, 0x00051eb56cae7f40, 0x000250bc3d20407c, 0x00000000000e8754}, {0x000ae69a0a837ff0, 0x0000281561056151, 0x00032df2869a92d9, 0x000c1bf6af2a00cc, 0x00042d0a56c0abdb, 0x000c8959ee9a9425, 0x0002f34774a7b77b, 0x0000000000019cef}}, + }, + { + /* digit=66 [{1,2,3,..,}]*([2^264]*G) */ + {{0x00035e2ab65ec55c, 0x000b3114a25c78e3, 0x00036712a123ad50, 0x00068411e7bd7df9, 0x000ad6da54dbc49b, 0x0000da3215b0359f, 0x00087ea6e6e5f93f, 0x00000000000d640a}, {0x0003244eb630ddc0, 0x000f7c1a4f6cdf83, 0x0008137a92bebef0, 0x000eb8e3c0a631d4, 0x000db756445ff44c, 0x000c8880e0205b11, 0x0000d304844e845b, 0x00000000000b85e0}}, + {{0x0005c0fc3837219a, 0x000cee76894c3764, 0x000faaa07cdf021d, 0x000fcfbe55cd1e6b, 0x00023a7ad5b9566d, 0x00087a4ef5421a9f, 0x000423a005838a46, 0x0000000000023a2c}, {0x000326b6922fa665, 0x000ed4b780011f85, 0x00024bae2459585b, 0x00069e937ced8f3d, 0x000d8c1e1eaa2b5f, 0x00089cc4d306b621, 0x0007748acbbe71d3, 0x00000000000f4ab7}}, + {{0x000e8b5d217d418f, 0x0002ee557e7c707f, 0x000dffc8ed3b58d7, 0x000efa4b42b3ab4b, 0x000d8e53b32db9b0, 0x000b1bfbd9d71dc4, 0x000d3fe3a93e4e11, 0x000000000009901b}, {0x0009bf805b79f71c, 0x000b35f3a5304e02, 0x0007d4e3dff75554, 0x0007a7bef469e919, 0x000278f160dcf415, 0x0009189a9f03282f, 0x000f2197e4b7f4c8, 0x00000000000a967e}}, + {{0x00083fb3cef3edcd, 0x000642df6f242046, 0x00054e87770387bc, 0x000232b372bcc88c, 0x00086e428cc59e80, 0x000a1b76326ba13b, 0x0005f32526ef1f13, 0x00000000000dc97c}, {0x0007320a0cda3969, 0x000344954867fb10, 0x0004f1baa6436a30, 0x000fa69be143027a, 0x0003cf54f28b7e39, 0x00097c2da1232946, 0x000099cf0991dc75, 0x0000000000052e07}}, + {{0x0004ba256e3a80b8, 0x0005fbdfea74874f, 0x00090a65af244c4b, 0x0005675aaba39901, 0x0006a12be348d5a6, 0x00018ac5d2250648, 0x0006f9766dd428e8, 0x00000000000fff9b}, {0x000ff89f1b07248b, 0x000174b3b10f3ab0, 0x000ac70bef37c1ca, 0x00011ed5c9e36bef, 0x000364cdfcdd2a61, 0x000462f54a99a302, 0x000b5fdbfbbe7a59, 0x00000000000cc266}}, + {{0x00081f8725c117c2, 0x00037dc9daeefc8d, 0x0009cda216a5b4ef, 0x0002271ca53d53df, 0x00054d1aa50b206d, 0x000001e99b054633, 0x000aa452bca91088, 0x0000000000017fb7}, {0x0007b7c6cf80a17f, 0x0007afff3af9472e, 0x0007c0765e5ecd82, 0x00004be24753989d, 0x0006e90950c6aac0, 0x0008efe5b4a5de2b, 0x0009f9b46af0ab73, 0x00000000000db445}}, + {{0x000c5196b2aa222f, 0x000d15bdb6d3f8f0, 0x000389884e011601, 0x00008a96ddd9e23e, 0x0008467e7577ab50, 0x0007555edac1e974, 0x000d57e74e35b601, 0x000000000009d04f}, {0x00080785cb147efa, 0x000908585c6fc59b, 0x00035ae9e9b63afd, 0x000d1e80b684dd21, 0x0009edbaddcc2739, 0x000c57660e1ba788, 0x00089a464f42059d, 0x00000000000a7cf2}}, + {{0x0002dd131270b84a, 0x000dd412f64a3e09, 0x000b9c94f3cfd9dd, 0x000cdc2c2e964e0f, 0x000d0011cb01fbe6, 0x000228f23e9a3b1f, 0x0009a16c30762dcf, 0x00000000000be919}, {0x000e7d10046b4ea1, 0x000b4732711dfdf5, 0x0008160cd8dd88e3, 0x0008ba0c6eceba7a, 0x0001f3c3d31d8ee0, 0x000716948b171153, 0x000add65060b633c, 0x000000000002ff2c}}, + }, + { + /* digit=67 [{1,2,3,..,}]*([2^268]*G) */ + {{0x00000725d23401df, 0x000096f178dbeb92, 0x000498bead595449, 0x000b4b459b46611d, 0x0007c72b4a58a6e8, 0x0005985adafce826, 0x00073321142175a4, 0x00000000000e649d}, {0x000052db6dbc2445, 0x0003a6bf42fe0182, 0x000384cfd9aea017, 0x000a58acd0291983, 0x0009215492e1b0c8, 0x000b5a5f0a73ff32, 0x00049b895c545eb1, 0x000000000006fcaf}}, + {{0x000a5a0bc1e856f0, 0x000f28baef5de481, 0x000a8b075c84a181, 0x00058d754c8267b1, 0x0005ccf703932685, 0x0008ee021f924f79, 0x000a4be3763f30f6, 0x000000000005c01b}, {0x0006ee7749647d88, 0x0008acc01a3e928c, 0x000c22c3ac36bfb8, 0x000512e6c45e3401, 0x00084ef433f61ab3, 0x00021f5afa978fe4, 0x000a4023e2ea018e, 0x0000000000011524}}, + {{0x000ecc04426e192d, 0x000c692a7fbcd69c, 0x000df4111d9bb7a6, 0x000db2d02a8feb62, 0x0002728cecbe8e45, 0x000837662176c0cb, 0x00082c33fcfbb0d1, 0x00000000000d6f28}, {0x000d967a59df021f, 0x000bc75a0f344b04, 0x0001f4de7fbff391, 0x00090cefa0453b03, 0x000ff54c96fbf4ea, 0x000f77afc5108858, 0x0002fc86b46b5731, 0x000000000006bf7e}}, + {{0x000215c3f055a1c5, 0x0002fe1d42d93e03, 0x000c50c3e0aa63b9, 0x00098e783686967e, 0x000474a30dbf5f66, 0x000d20230bb5e159, 0x000f8bef86bb5bcf, 0x00000000000403b8}, {0x0001c2443b169c63, 0x000fbf249aac89ec, 0x000e2f941f78369d, 0x000960cefca9a90a, 0x00088f449c4b3b80, 0x000cfe09ef28e338, 0x000d260cdfc61bf0, 0x000000000008b22c}}, + {{0x0001733d4c0dd30d, 0x000af7013d596371, 0x00017f590b1fdd3f, 0x000898e35915f523, 0x00039fd967d4bba7, 0x00073558f9fbe5f1, 0x000bea0aba8344d1, 0x00000000000ffeb6}, {0x0009792191976aee, 0x0004e928f68cbbf7, 0x0007210e163722d8, 0x000c4e06abb0ac9a, 0x000a6895762709b5, 0x0004045a401ef3f8, 0x000ac355dbe9f79f, 0x00000000000ef178}}, + {{0x00094c375a95d7b3, 0x000dd7f89c2c3b20, 0x000a45fd88318202, 0x000e48b12d9f811e, 0x000dc7a43dd0be70, 0x00018ca3a703ac1b, 0x000012a4309a2c21, 0x0000000000001943}, {0x0009ce1f49259e54, 0x000e6d9597d8737f, 0x0001541e69df2d37, 0x000760df1561aaa4, 0x000e2d91ee39fcf8, 0x000bbcb6de1e0306, 0x00074699979031e0, 0x00000000000cfcc8}}, + {{0x0005490cc3cc0b27, 0x0002a4073f24615c, 0x0003e0a87e36ac6d, 0x00006c18fcec6a28, 0x0005d75a73f10abb, 0x0008c94883a7d129, 0x00074165fd8700f1, 0x00000000000732f7}, {0x00049117079dd0b3, 0x00098d15d8c801e0, 0x0009200404155fcf, 0x00007c7120e12665, 0x0001f42cc9fd9816, 0x000a888cf0e486b1, 0x000249e606c16c01, 0x000000000001ea23}}, + {{0x00014f7d981a63eb, 0x000cb923e948b882, 0x0000fdde16bb34e4, 0x000011a6df27debf, 0x0001d57f4ca2345a, 0x0004196e9ba6784c, 0x00026df01b370311, 0x000000000001aab4}, {0x000a2240acace9dc, 0x000d82ffdc977a4c, 0x000a7e87839c540c, 0x000c01216f09c1f8, 0x000e5b9cc0b65ca1, 0x000150569612021d, 0x0003ea910e10e95e, 0x00000000000e2ab9}}, + }, + { + /* digit=68 [{1,2,3,..,}]*([2^272]*G) */ + {{0x0008093fcd51c0da, 0x000aaac58c9dfd06, 0x00005aab38065215, 0x0004afc2cc4252aa, 0x000397c2bdf932ef, 0x00049e19a08bd48b, 0x000496ac8a7803a8, 0x0000000000075c31}, {0x000fef0d3072e592, 0x00057733dc7dec06, 0x000d0f9063e4be72, 0x0005b0847be21a8a, 0x00055ebf426f6d9c, 0x0004e9e5bfab0fdc, 0x000089d60748caf9, 0x0000000000069366}}, + {{0x0000e5d6509e91ef, 0x00048b06c17a3a05, 0x0001e37973be9551, 0x000ae990038aeb31, 0x000b58b402ca2440, 0x00077732a83bf711, 0x000932b8763e00b5, 0x00000000000f651a}, {0x000cf91f09ef177f, 0x00070be02ab9cf6b, 0x0005ba59eb97ec90, 0x00072f64283f8100, 0x00072365da5e7ed2, 0x000dc6beb098cf05, 0x000d72d90e6f1805, 0x00000000000cf7b9}}, + {{0x000bed14c2e5c1d5, 0x0009f89b2edba20f, 0x000638f30da8440c, 0x000d9ce8943fce4f, 0x00045a2ff9961dde, 0x0006e87feaaf07ff, 0x00008c69e60f92c9, 0x00000000000b9ee8}, {0x000d99ec41a8cc2a, 0x0005279d6d8c67c9, 0x000b28385a21a71a, 0x0005267350b7fc9c, 0x000d8a2abab0c8a6, 0x0004faf7ce9fcb46, 0x000c8a57c3bfc62c, 0x00000000000a8207}}, + {{0x000f000e371891b6, 0x000b56f101762b79, 0x0004acb33b163eb0, 0x000e2aaac1dc8274, 0x0009f835a1b62c58, 0x0009915a451410e8, 0x00048d981333a762, 0x000000000000c141}, {0x0009810640a6c340, 0x000184d20b3d37fe, 0x000e4eb008d53e3a, 0x000b68c2a2645f81, 0x0007add082eaa664, 0x0005ff2067a6bc85, 0x000fcce7467dc63e, 0x0000000000004e2d}}, + {{0x0000cfcf5d6b5960, 0x000cc57fe04dac14, 0x000998e82d998b8f, 0x000bd40e0a341ea4, 0x000b9c24fdba5128, 0x000c1dea81adf3cb, 0x000ef2b1ce8520f4, 0x00000000000db22e}, {0x000911256ee6f617, 0x000b2e9f05f28713, 0x0000043d1376f349, 0x00052d3f63bfade4, 0x00006cbcc395a2ad, 0x000714bd81b43575, 0x000ca6fd1d7e0666, 0x00000000000a40ea}}, + {{0x0004ca398c06d768, 0x000c23d0efae2846, 0x000878540a1ede95, 0x0005a253f00eeef2, 0x000cd86161cea5cb, 0x0004a389ac2f9258, 0x000159d2b0865f5c, 0x000000000009b1f2}, {0x0008e174cdba2f0b, 0x000245a758a60599, 0x00087073a8d80dcb, 0x0005a4949d301c92, 0x00061c4bb89b8865, 0x00060259c129647e, 0x0007d106f0665ec3, 0x0000000000006619}}, + {{0x000cae6e407127c8, 0x000070262c90bff7, 0x000ca8e9829543e7, 0x000f8fb1577634a8, 0x0004597668ad6514, 0x000ff6ecf4678d63, 0x000dfc90213b637c, 0x000000000007518c}, {0x0005a2999ad1c0e5, 0x00071d006db8f897, 0x0004562511312669, 0x0003aeadf6356731, 0x000f1a366456acc7, 0x000e73aeaaeaa4fa, 0x000a480df51aea1c, 0x000000000009a8e4}}, + {{0x0001074c67a33fda, 0x00086a23545689bd, 0x0009b9de484075b5, 0x000247b132c568a3, 0x000dcc4760bbab0c, 0x000ae821b129a5c9, 0x000f8303eba46726, 0x000000000003df1c}, {0x000f0c5101b30f29, 0x0009f010813cfb68, 0x000cc25f999a807c, 0x00020b31fe4b6007, 0x000398dd32399073, 0x000abb34cda3bfeb, 0x0005f123f1ed1641, 0x00000000000946f8}}, + }, + { + /* digit=69 [{1,2,3,..,}]*([2^276]*G) */ + {{0x000261be9524972c, 0x0005728524830014, 0x000a9e4b0c851ae7, 0x000fa9900e4f78b6, 0x000b54a3f7a33a66, 0x00065a79afd8a65b, 0x00069a505d3f6319, 0x0000000000018914}, {0x000675265cae6514, 0x0008278367cbbd6a, 0x000c906352414281, 0x000b3f6f5af173f1, 0x000a5ca36597e41b, 0x00099f79557cb03c, 0x000ef127f86cb87b, 0x00000000000ad4dc}}, + {{0x00092527cce78878, 0x000b138a49bf4890, 0x00076c4f31c0490e, 0x0009eca01b2f7c2e, 0x00005be8359f5f70, 0x000646b3fc479a65, 0x00011b6b6a22bfd6, 0x00000000000cc5b7}, {0x000c9aa1253a31c2, 0x000d18e7dbc638f0, 0x00085601e946c9b1, 0x0007e04030271eb4, 0x0008e22fb3c72d3a, 0x000e0d46cb08b597, 0x000915860d62789b, 0x000000000001d49a}}, + {{0x000e171e53bdf97e, 0x0008556d65155b11, 0x00013206465d7fcc, 0x000049f10dfaeae2, 0x000ae30b70b8ef9e, 0x000c6a56219750b3, 0x00002611ed860015, 0x0000000000012097}, {0x000714d4a4467bbb, 0x000e279559d04c7c, 0x000363f176594d50, 0x0007323ae8fb3c53, 0x0009a4111f88d6fd, 0x00010a683834639c, 0x000e9afc69a029a3, 0x00000000000255f2}}, + {{0x0006e6e51e1dd6f5, 0x0009766797f569dc, 0x0007480d93f86f88, 0x000f4a531adb034d, 0x000e5d185cfc18ee, 0x00053c27bcf1cb5e, 0x000569f596a59bbd, 0x0000000000069002}, {0x000a4105949da385, 0x00053d9433e78e1e, 0x00022ac847a15a5d, 0x000d7ed65d68ece1, 0x0009fb2aded8233c, 0x0009b750e201bbe0, 0x000f25987f4975ac, 0x00000000000337f7}}, + {{0x0009f8de8126d12d, 0x000626fb6b7bb82e, 0x000f3dc00034e80a, 0x00089d8316ff5318, 0x00079894d65f98ad, 0x0002801704800ee1, 0x000537034ea3c448, 0x00000000000c30be}, {0x00066423e58a8102, 0x0009243c2d27d6b9, 0x000c30559bd3f060, 0x00040105eaef29b7, 0x00017b678fc76545, 0x0003012701d8f07a, 0x00036554bc6f737b, 0x00000000000e9473}}, + {{0x0005ee6b65e3f046, 0x0002a3561b3fe3d4, 0x0003adfe7b57a7b4, 0x000831347fb79d76, 0x0001c48002ed4374, 0x00044dcb0a7f497e, 0x000e0686221cce2c, 0x0000000000043785}, {0x000bdab4ad1119ad, 0x000279ee9061f323, 0x000458b83a9d33ec, 0x000e85b76f7f52c7, 0x00090fd6c65f1d8d, 0x0001f8a3939a713c, 0x00040af74ca06771, 0x000000000008ebc6}}, + {{0x000c37bd5d97c8e0, 0x000841942c2ff6fd, 0x000f5c9ed6475a4f, 0x00052b5771a5972c, 0x0004363c2048f3c1, 0x000db8da24cfb1e7, 0x00043dcd2ce8249d, 0x00000000000a5ee4}, {0x00044b387eb3689d, 0x000606fa84b3cea2, 0x00069e100b1b94b7, 0x00071368e13869e1, 0x000c96f0f8a12e45, 0x000d93dc039ac6a0, 0x000ec9e24458ac42, 0x000000000005f60b}}, + {{0x0001937d698b9991, 0x000c7514cc1dc470, 0x0007f6d12f9a74ce, 0x00022ac2fc9230fb, 0x000d92e23e21f0e4, 0x0006d1ac0aa50a1a, 0x00049370ac46d867, 0x00000000000f9f54}, {0x0006e32302426e7c, 0x0002059a6b86a840, 0x0004338952e5938c, 0x000cff13192968cc, 0x00040b342dd8d3d9, 0x0001da44113e700b, 0x000760fd5dc7bb3b, 0x000000000002c883}}, + }, + { + /* digit=70 [{1,2,3,..,}]*([2^280]*G) */ + {{0x00067944f366244c, 0x000d59e9af31428e, 0x0009cbe9f6c4942e, 0x000bd92582864947, 0x0009b2e09205204a, 0x00056b1017995988, 0x000f260c727a3dfe, 0x000000000008b657}, {0x000fac647b56af7e, 0x0004cde35d954514, 0x000e44c1af2c53f1, 0x000261f36019feec, 0x000966511d8ffa03, 0x0001148b9afda42d, 0x0003e1f63211dd82, 0x000000000006f13a}}, + {{0x000fe40d0b19b4c9, 0x0006a3997900ec16, 0x000abf1980c5bfc2, 0x0005b0a661f3c579, 0x0008aab1292b71bf, 0x000d993f244b1338, 0x000966cbe80cb9d4, 0x000000000002203f}, {0x000fa9a95e9148b0, 0x00053bc82a651a1e, 0x0005a004b90c2827, 0x000c7e7502f0003e, 0x000056a38f73a388, 0x00087d9b09c704ef, 0x0001f393cde8a734, 0x00000000000ec11a}}, + {{0x0004e6a4ec3efae0, 0x000e047fe47986cd, 0x0007fd532d6e5353, 0x000fa8b9a8671744, 0x0000514bf471b76c, 0x000dcf1fc9bc87bf, 0x000fa837392dce71, 0x000000000003ad1e}, {0x0001e59290757ebf, 0x000cebbea5a33841, 0x00075b7402f28c87, 0x0004ecd9b1665aa2, 0x0006e329df225b3d, 0x000aa808de7b0df2, 0x000faf18fb438e0b, 0x00000000000dd516}}, + {{0x000d8d8f4a6b2b66, 0x0002a63f27a255cd, 0x0003341f52773d72, 0x0007cd965f9ce38b, 0x0000635ba3005d31, 0x000343662162c92e, 0x000ac85259f64ffe, 0x000000000008e614}, {0x000a1c59580d5fe2, 0x000e397fb55baa90, 0x000365cc03ff132a, 0x0007325780618255, 0x00086c1e6306a57e, 0x00067b14c892f6b3, 0x0008c5f12e4c0723, 0x00000000000c83a4}}, + {{0x00040eada4657ebb, 0x000e31e6f9a95314, 0x000a04269b290326, 0x00056256e8b41991, 0x000b7a4a53f9365d, 0x0002b9b16e1b9c53, 0x00070155dc1d50e3, 0x00000000000bb6d5}, {0x000b5b1121311bd1, 0x000984c270249c99, 0x000f7b0846375c38, 0x0005b610689902bd, 0x000fa4cfc5a819a9, 0x000e5b424dd5706b, 0x000a87d33bdb7314, 0x00000000000e69eb}}, + {{0x000fd0482abbdf6b, 0x0004f6897401cbf6, 0x000652cc2d40814f, 0x000b32939e2b43ba, 0x0001bf809331e511, 0x00058f8f43952286, 0x000727815f6c1dc9, 0x00000000000c213e}, {0x0009a038bd421fe7, 0x0008ceb09ae59065, 0x000d25d81924ad76, 0x00015a2a20f86c7b, 0x00069aa308078f48, 0x000758878748a176, 0x0003d7f1f46a211b, 0x000000000006f6fc}}, + {{0x0005626ca6bcb860, 0x000ac75ba2f9acbe, 0x000c234135494b93, 0x0009e0b610d080fd, 0x000333a99c2f3c33, 0x000580aff614ac29, 0x000a3aa5e23af2aa, 0x00000000000e1fdb}, {0x000ca5f5b322ee30, 0x00065aa603365011, 0x0001b05f57ab1134, 0x000524456330a336, 0x000b9e025aa3a2b2, 0x000a0d5cfc396b5b, 0x00083babbf77faf8, 0x00000000000ea31c}}, + {{0x00047a49edb97340, 0x0002d300b04831fb, 0x0008ca060cb164c9, 0x000c5420cc10a74f, 0x0004eff9d01017d9, 0x000aa12857f637e8, 0x000114d732aa1e27, 0x00000000000e2a77}, {0x0008fc5f031be351, 0x000d262acf1585bd, 0x0001b0dfbfbd395e, 0x000c5d22e6eb2db2, 0x00028d18263d88be, 0x000bbd6acaec2720, 0x0004c04b687353e4, 0x000000000000ff7e}}, + }, + { + /* digit=71 [{1,2,3,..,}]*([2^284]*G) */ + {{0x0006ecc79d0fef75, 0x000dfa6e19d09c24, 0x000097d1dea89821, 0x00080950ee527a58, 0x000b7c4278a6ef73, 0x0000798ff78b7174, 0x0000cac133d867e7, 0x000000000009e923}, {0x000f7954ffb5e165, 0x000877431eeef1ad, 0x000728c869881eba, 0x000306a8d6af3f83, 0x0000bbec076af7a4, 0x000257ca3633bb5a, 0x0006636d07623e7a, 0x0000000000021c00}}, + {{0x000f8db3d9e25bfd, 0x000ed35ae891ae18, 0x00094f42c82c0dd3, 0x000b3cae63745c4f, 0x0005f218d139c2da, 0x000f3b0e255b8c18, 0x0003fa6ab039c889, 0x00000000000c644c}, {0x0008fe1e4d75d65d, 0x000d9c8496f1ff0b, 0x00010bc25fdaddb7, 0x0008392921149191, 0x000fef73f3455c67, 0x000fcfb22e962e52, 0x0009113818baf354, 0x000000000009d4bc}}, + {{0x000af695e08e48a8, 0x000683a35938698b, 0x0003114a51ff94f2, 0x0000f82899d204cf, 0x0009158780b5fcce, 0x0004edd4f4ca4158, 0x000c34c28f1bb73c, 0x000000000007a6ae}, {0x000b281e555c9d40, 0x000676d4783df7ee, 0x000d1f8a08acdccd, 0x000c2bcdf6d7f923, 0x00000f356ca3b7b8, 0x000b8e964f0a8d47, 0x0000e1229894bfde, 0x00000000000d3aa7}}, + {{0x000d000279a1a838, 0x000b77f1f985b888, 0x000a43ce9b20f9f6, 0x000930a5188fcf70, 0x000ba3a82904b40a, 0x0001510273fad79a, 0x00017b7bee8d22f3, 0x00000000000c2c3e}, {0x0003fdcf0848373a, 0x000041fbe06e92b8, 0x000ae48de4a30594, 0x00090ec2d9cf7a56, 0x000a9bd062c5fff2, 0x000d353815d0deda, 0x000eda080344abd7, 0x00000000000f5cf9}}, + {{0x00012d81fb28e752, 0x00018093526223ac, 0x000d23e13369699e, 0x000d26ee94c9abed, 0x0003a20b52679a62, 0x0003d5fa4425b86b, 0x000fc68164f9c8e7, 0x000000000003b12c}, {0x0006c5fe565db2c2, 0x000be6cc34bad21b, 0x0002901d6245306f, 0x000f50970e4963cb, 0x000cf0813b31f80f, 0x000f1ab8ee403ccc, 0x000b92a72ab5f584, 0x0000000000070f15}}, + {{0x000e213ed814f6f4, 0x00034170098ddc99, 0x000e9c62ee04edbc, 0x000b04b8660d7749, 0x000b5152ad1e35b1, 0x0008e92aa8b4b6c4, 0x000f4e0b0bd1c24a, 0x0000000000088172}, {0x0007445379f53f7e, 0x000d513a76898c7b, 0x000f4e73336bcd42, 0x0004f973dc17e72a, 0x00060a7124b24b39, 0x0008a372c1b0cd16, 0x00003b7549880cce, 0x00000000000e7126}}, + {{0x00055aa45f74a004, 0x0008dc14a3abb979, 0x0004a411c66e9f88, 0x000f1828a4be08cc, 0x0009deca5a89e49b, 0x000fbb36be6e7bf8, 0x000e57a7326cf88e, 0x00000000000c7f6a}, {0x000da2212d19f21a, 0x0005dceced37b724, 0x00062d313930a591, 0x0005b3fb62a62e4c, 0x000793e5c0e1a7e6, 0x000b8d77382cb79a, 0x0002f014c9a99461, 0x000000000000b05f}}, + {{0x000a2dd7deb1dde9, 0x000a831d31f799b5, 0x0008e986349103b0, 0x000362e52f04d685, 0x000921c7b6511793, 0x0006f47f65808e70, 0x00040533fe9123ed, 0x0000000000058f71}, {0x000d5dbe8f1956b4, 0x00061821d30451f3, 0x000e7b4e166f399a, 0x000db145b2048aa2, 0x000ed7811330932d, 0x0008fd85f17d989f, 0x000434032ae4cb12, 0x000000000007d1ef}}, + }, + { + /* digit=72 [{1,2,3,..,}]*([2^288]*G) */ + {{0x000acb4203cc71b2, 0x000a9070d34e112c, 0x00047b523d821ba6, 0x000f9b95eb8ad292, 0x0006a6a45dda269a, 0x000df41e0dde0a03, 0x000f2aa18b528e63, 0x00000000000a260d}, {0x000c22d6a6aefb01, 0x000d424a67709f9b, 0x0002a9971343d570, 0x000e2a1968444330, 0x000d131a6cf073c1, 0x000e794543660558, 0x000dcdb0289c832e, 0x00000000000c6d8d}}, + {{0x000d86ee6f475d5b, 0x000d30ed1082f1a9, 0x000a6711ccf28680, 0x000ef0eece773534, 0x00064bf0c426a35f, 0x00011b2fb76adaa4, 0x0004d3fbab929aa0, 0x000000000009d8cc}, {0x000bb954fd395b2c, 0x000dabe6d7f2076f, 0x000a4cbdfe911e02, 0x000cba5106ceaac2, 0x000a45258697c7e2, 0x0005945b0fe94528, 0x000f7a7109b17d07, 0x00000000000cae17}}, + {{0x000ce6be22a12c0b, 0x0008a6855029961b, 0x000a3c801158be2e, 0x000d2b4a8e92e70b, 0x000f0cc9fca9f601, 0x000bc2e96fd02799, 0x0009b889f99ca013, 0x00000000000ff9db}, {0x0006b1bd37f91ceb, 0x000616ae70849faf, 0x0005d65bd4da44b0, 0x00092a4e0945e8f9, 0x0002ecea36f6394d, 0x00018a069ddab675, 0x000860650b74d60d, 0x00000000000ad650}}, + {{0x000bf5b4a4380ef4, 0x0001cfc1faedb8cc, 0x000e6ba03d77f4c0, 0x00085456416defe8, 0x0000c58653e7f004, 0x000f6f99e75de777, 0x000c07dd2dcbebe2, 0x00000000000d159a}, {0x000c9af1998a9b17, 0x000a045dd713c4a8, 0x000e1c3678ddf502, 0x000756b6b962b38d, 0x000bbffd3f0c3f73, 0x00008ebae3cb9f53, 0x000dbad723c40b7f, 0x000000000000c3cd}}, + {{0x00003107748ef382, 0x00058a804b2f0156, 0x000b319b886fb1eb, 0x000f4fd353970be7, 0x000712854a9131ba, 0x0009713983481c8b, 0x00094ab0424eecf3, 0x000000000000f4ed}, {0x000648ea129bd87a, 0x000a992b74ad013d, 0x000692f5d2444d08, 0x000b28070a5f0c93, 0x000b9f5101ca6741, 0x000d0f81e46c12cf, 0x000288e7014d7901, 0x00000000000d758c}}, + {{0x0000c39f81600214, 0x000686a565269228, 0x000c17ee858626b0, 0x00011a7d8c6ddb92, 0x000b2332347b0bda, 0x0009dff48b7ebfaf, 0x000c83cf791881e9, 0x00000000000dc357}, {0x00082704ffef2257, 0x000fe2a264f8831e, 0x000c9352584eb7a7, 0x0008260bb47d4188, 0x000c3d8170326d47, 0x00005f901544de4e, 0x00063cac7150a9dc, 0x0000000000075c09}}, + {{0x000bcfd07a0f22c4, 0x0007c58805ca858d, 0x000b1c2b2351e4ce, 0x000c9a1349cfed00, 0x00017800a980f9cb, 0x000847e9d67bc7b4, 0x000c48e7e6dbc0f6, 0x00000000000af379}, {0x00047d63263e04cc, 0x0003c80b6fc048b9, 0x000808eb22fe8f7f, 0x0001ed14e9dc24bc, 0x00073fab87e6d2d1, 0x000c630ae48d74f1, 0x00020a1feca5af50, 0x0000000000062d06}}, + {{0x000e56ad2bfbbdbc, 0x0008ff6377e3bcab, 0x000f30b92835c3f0, 0x000ac42c21c0cf1e, 0x0009857d8edb7f85, 0x000b77ed67a3e31c, 0x00010b2eb1076327, 0x0000000000038dae}, {0x00026ce2d0dbce6d, 0x000dced7c6fb0aad, 0x00066cec0d07ac1c, 0x00035504569e7309, 0x0007bedae3ab4dc4, 0x00027e463aa82fb9, 0x000a1106d37bef4f, 0x00000000000f0c35}}, + }, + { + /* digit=73 [{1,2,3,..,}]*([2^292]*G) */ + {{0x00011f784b3a0046, 0x000066edf5561b45, 0x000c8578d54224c1, 0x000ae56d8d66d29a, 0x000ae9d58eb27699, 0x000f9176d5f81602, 0x0003b1d0c04874fa, 0x0000000000052315}, {0x00067980492adf03, 0x000309690930fad0, 0x000146ea47ef3788, 0x000725873bcba90f, 0x000de902e3292d0e, 0x000bc27b957efe72, 0x000c093fcd598676, 0x0000000000093940}}, + {{0x000cdd973668c530, 0x00077c68b641e912, 0x00064e1ee190b62b, 0x0008cbad89ce6688, 0x00006c269d8eeda9, 0x000f3402315f84dc, 0x00034af0eb685ecb, 0x00000000000c0326}, {0x000bc909db075650, 0x000562e3d7fff094, 0x00012e59576050a3, 0x000abcfa3e050e0c, 0x0007c001dcd9f02c, 0x0006d2d52ccd2da6, 0x00041f50aa372306, 0x000000000007224a}}, + {{0x000ec1f1185b9762, 0x0007e4b1f8698c94, 0x000d0af36ed765ea, 0x00049f56536e156a, 0x000109c95a75ba1c, 0x0006d9af131c3163, 0x0008b7268a308f0a, 0x00000000000fa9ca}, {0x000f13b9ac944487, 0x000b1c3b1cbccb86, 0x000446bb4e60be0c, 0x000c7b334396850c, 0x000a048a69431b4e, 0x0005820bf42f21d5, 0x000a580bf948f55b, 0x00000000000b48dc}}, + {{0x000b86ea9a29c966, 0x000cd04418b58cb8, 0x0002d1612a6d4e73, 0x000e22c7c65a6da4, 0x00051dbeffe45f87, 0x0003cc05caac7106, 0x0002c4ff619d64bd, 0x00000000000e65c9}, {0x0002126bd4b5ea52, 0x000d123af2b21062, 0x000e0691b9e9f16b, 0x0007efac6478c561, 0x000af6cd140ec34a, 0x000a0db2e57cde95, 0x000b7664b74575f5, 0x0000000000075d7f}}, + {{0x00078c972345257c, 0x00058a9532cb266c, 0x0002b89ba64976de, 0x000067545b1733a4, 0x0004cc6e65dd74ea, 0x0006ff2aa7f37f02, 0x00079e6612b1425c, 0x0000000000030374}, {0x000f1ac7f7aa2f0c, 0x00097e512bca4fe9, 0x0003049963042b95, 0x000bd29a259d6515, 0x00077373034f5b97, 0x0005aa640215eb65, 0x0008b779da0fc488, 0x000000000007e435}}, + {{0x00041294c6ef7a53, 0x00023ce3cc51372d, 0x0009af7f675a386a, 0x000d981efc9f71b0, 0x000a20bea9baa9e6, 0x000ef7c1fc01ded6, 0x000a5cc4b7e6424f, 0x00000000000f9dec}, {0x000954a122b474b3, 0x0002aa0a43d0a3de, 0x000a0d55c109a592, 0x000a173b5d57bbf3, 0x00056299978197ce, 0x000ca2bb13c3f30f, 0x0002e069a7820a8e, 0x000000000000ff69}}, + {{0x000deeb8201c384c, 0x00029aca3f998176, 0x0001642e9c9fe922, 0x000dbd63d82750be, 0x0002faa3400be031, 0x0005fd54bfb2552a, 0x00030374e9b5d365, 0x0000000000035894}, {0x000839a12661f3f2, 0x000a15a575a4221a, 0x00056e3aabcc078b, 0x000ada596e1e1175, 0x000d0f305dc3ca6f, 0x0008d7af5126e9af, 0x00069391da8d7305, 0x00000000000d8353}}, + {{0x000cb9cfccb5f12c, 0x0002c28405061c1b, 0x000211e4805b5eb0, 0x000944405547f7c3, 0x000d3441e472789d, 0x00044ce877fa445e, 0x000d14feb198254a, 0x00000000000129ae}, {0x00073247d16dc8ed, 0x0008622b05de2de2, 0x000590692ea55dfa, 0x00052caa83f7f094, 0x0006cefaa757fba2, 0x00017d8060f6ad6e, 0x0007ade1ad187165, 0x000000000008e9d9}}, + }, + { + /* digit=74 [{1,2,3,..,}]*([2^296]*G) */ + {{0x000c5ac4350713f2, 0x00062cbb991946e6, 0x00025875bf6be408, 0x00024aab6ab6c041, 0x0000bc41f4b7d50b, 0x0005496cf419d281, 0x000ac0e2e88d7a29, 0x00000000000f2457}, {0x000a357ad05dad34, 0x0008e838e4a3533c, 0x00068ef45f1df4fb, 0x000ab1c7a5c5fdd4, 0x000dde34cb8d9fa3, 0x0009105fe324798c, 0x000249bfa5a9d088, 0x00000000000fd0df}}, + {{0x000b286ea52474ce, 0x000419974759bb2f, 0x00039b1e3f8aef8e, 0x000c58ec64941cb4, 0x000a01d055758afe, 0x000448db52d7c57f, 0x000440c07c74cc5c, 0x0000000000001bb1}, {0x0007977c86a9c43f, 0x0003893162d96796, 0x000ee52587841e45, 0x000c57e6af18dedc, 0x000f174a30894f28, 0x000dd626536faed8, 0x000d7b702dbd64a3, 0x000000000006adc0}}, + {{0x0002a70e7d991630, 0x000854981be688ec, 0x000de0f53a95f8f1, 0x000e05074cff00d2, 0x0001b52c7f9c3168, 0x0001c7986111150a, 0x0002ad0db2ed84a5, 0x00000000000e6127}, {0x000c16b3e4d1fe2a, 0x000897443d935440, 0x0002c51d6c8aaceb, 0x000ef4e866b5e41e, 0x0005b965a67071e0, 0x000e1aa7705c57a4, 0x000e3285bdb58c70, 0x000000000009df3c}}, + {{0x0009a11de31a0e3c, 0x000c6c41e4aed12a, 0x000342590d5103b5, 0x000ab7b8ed812d38, 0x00081afb58b8f5ea, 0x000efcc3a7801fbb, 0x000b91be5cdba671, 0x00000000000cd10c}, {0x000761b96524aa0d, 0x00013882e821e20b, 0x0008d1b11d20578f, 0x000de2bd8a0f0393, 0x000c857fd3a03afe, 0x00060f767e20a117, 0x000da1ebaa2aa430, 0x000000000002b9d1}}, + {{0x000d9fea4e493f3b, 0x000e070e478fb44f, 0x0000254b6a02edec, 0x0002b005be36ea75, 0x000f6717e06aaec8, 0x0008f197782618b8, 0x0000d12db1f6d400, 0x0000000000016b39}, {0x000ddfa794ec9644, 0x000bafe86fd4d041, 0x0006bc15de0d2bf6, 0x00023e70c0491593, 0x0005ed4562356cbb, 0x000d836efd05be7e, 0x000604601374069b, 0x0000000000032e5f}}, + {{0x0001ebab46405b61, 0x000f009db138cd2b, 0x0009fd7df23c53c0, 0x00047794feebab9a, 0x00017fdb1e710bb6, 0x000f1f65dfce4aee, 0x0006cd7d5c0c61a7, 0x000000000008d02d}, {0x000621234d968fad, 0x0005c7cac0485224, 0x0002b8dbf87d2555, 0x0009a59db0aa864d, 0x0008ff94f8b5b587, 0x000c97f8e75f391b, 0x000232a6c994ae49, 0x00000000000d690b}}, + {{0x000e6486f79bfa96, 0x000e507c60c742b4, 0x000d5aa8cb9b07e8, 0x0000a09dcba89982, 0x000a802ce50ca8b0, 0x0002144b0a77f1f3, 0x0000db6c4050df2a, 0x00000000000ab1b1}, {0x000a93fc6ce00720, 0x000e89025d46fd65, 0x000c1e4037fdb9f5, 0x0002d629ae2d5361, 0x0009eff0f6a9bcad, 0x0005fa5a07a15282, 0x00039ce87345cd92, 0x000000000000e840}}, + {{0x000cab548e246919, 0x000d15017e1454d5, 0x0005f67186d3e9b6, 0x000059dfa98234b5, 0x00099985af982541, 0x0005b7b1c4bf344d, 0x0001ad39737ed67b, 0x0000000000087c41}, {0x00069e3ff4c10920, 0x000022fad8cecc3b, 0x0008e45c000ca718, 0x000d856404043f8d, 0x0005863927a11a53, 0x0008921216c1f07a, 0x000c8f38d3a4f4b6, 0x0000000000001976}}, + }, + { + /* digit=75 [{1,2,3,..,}]*([2^300]*G) */ + {{0x000dd4906efe62a1, 0x00063bcc8842253e, 0x0007a247757fdab1, 0x000a1ef2571d6b1e, 0x0001739a956e745d, 0x000bffc16f01c292, 0x00072ceffd8065e4, 0x00000000000f36fe}, {0x000478592361b14b, 0x0003ff49e90d1ebe, 0x0001338e12b047f8, 0x000b583b7240d91d, 0x000c03f8387fcb78, 0x00000f2d92e1a7c3, 0x000a8416566c2232, 0x00000000000aaf80}}, + {{0x000f32d335de93bd, 0x0002d7a1a522aee8, 0x000b425f9e81403f, 0x0004abe3d4ac4abe, 0x000337f996e841bb, 0x000727761c52950b, 0x0007d201d991e679, 0x00000000000978fd}, {0x0006e2e2907ff054, 0x0007486bad5e5c68, 0x00010431aa78725c, 0x000eb7705757453a, 0x00093939df7758ff, 0x00078ea0fbb18612, 0x0001d3bfdd394a6a, 0x000000000006e33e}}, + {{0x000c3ccc7d56f63d, 0x00076db1cffd2381, 0x0002131b488cf4cf, 0x0001e5aa3c1db85a, 0x0001ac55d391a418, 0x000d270d0e5183e5, 0x0005462793d5efef, 0x00000000000c0631}, {0x000c80c1d49264dc, 0x00008ecb40a58378, 0x000aa60ec18fee5f, 0x00014ab6c5830fab, 0x000ca03f201ea89b, 0x0008c54f64dc477d, 0x00066e5604f5dac8, 0x000000000002acfc}}, + {{0x0003e9c742a5516d, 0x0009da7f29531cb5, 0x0007fe865f9eba30, 0x0004ce9efae348cc, 0x000cac52758dd2f5, 0x0002961a45acb931, 0x000fe1287b3e18d2, 0x00000000000748f8}, {0x000f0554bb131f76, 0x0001d1b62b340cc5, 0x000b08ad2a8e1d6a, 0x0008dadbe8da8486, 0x000954f18276bad7, 0x0004bfdad85fa710, 0x000237e2cc9d287c, 0x000000000002c75f}}, + {{0x0004d2c2f1f927d3, 0x00033283dd7ef2a5, 0x000a91f40054b9d9, 0x000c095870282aba, 0x0008f4e36324ba08, 0x0004cecf2b02b00d, 0x00024e53aaddc060, 0x0000000000084ddd}, {0x000a889fa3214142, 0x000ca34bc00a03f8, 0x0000c263c00349e0, 0x000eaaacef68f74d, 0x00023c0293515228, 0x00042e3b740a790d, 0x0001e0e0e9af5c84, 0x00000000000a9f14}}, + {{0x00087ba6916bcaca, 0x0005bcdcdd7d6721, 0x000b58247a460e0a, 0x000c569c00ce5c40, 0x000fc589d0c75923, 0x000fa6fe6099c17f, 0x0008c40335eeea89, 0x00000000000a4e86}, {0x000a7415516e07c0, 0x00024b7037325a9f, 0x000944c467e9c1bd, 0x00032efd5e26d28a, 0x0008de919cbedf31, 0x000bbb18d7ed994b, 0x00050812df67cd6f, 0x000000000006baff}}, + {{0x000c3f58ee594ee5, 0x0008b72f4949e0b4, 0x0001e5bef192dfaf, 0x0004726092a58066, 0x00010f5cdb7271b0, 0x0004bfc49ad5c3c1, 0x000be7d4951f9e55, 0x000000000006131e}, {0x0009f77a3840f92f, 0x0009237da59f303d, 0x000ff0c06b034503, 0x000962a4d89fa660, 0x000e71d824f5d020, 0x000050e312610c61, 0x000bfb9c917da73c, 0x00000000000e6e7e}}, + {{0x0003a9eb85c489f9, 0x0003af2ec9ef5122, 0x000d827990bab108, 0x000b2387d2ac6e41, 0x00020318a3b790e2, 0x00084a7a0cf682b1, 0x000d057fe76089ea, 0x0000000000016546}, {0x000bc6c0a0421345, 0x00041f987118eca7, 0x00079f0cde4a8da9, 0x0007774888a7e8d0, 0x000a63ec5831544c, 0x0002fe985bd2647b, 0x0003e8b479cea3fd, 0x0000000000028d16}}, + }, + { + /* digit=76 [{1,2,3,..,}]*([2^304]*G) */ + {{0x0005b08a8a2a82b4, 0x000fa04baa17598e, 0x0006a0661405c62f, 0x000031dc6cd61096, 0x000030bf87a5c4a6, 0x000e497ccba2534d, 0x0006657ebc6e2131, 0x00000000000851fd}, {0x00018d257f3e221b, 0x000e69ecae010ee2, 0x0009788f9b15d390, 0x000a9c22fa527837, 0x0001236d442e39e8, 0x000c289ce74c9380, 0x0009cf8b21ba23b1, 0x0000000000025c76}}, + {{0x000aafc618669c84, 0x00037761e10e2b1a, 0x000c03aea1a1a99e, 0x0000ff43dbc6fcaf, 0x0006cbe0ba8aa087, 0x000f1ff6a9765128, 0x0005b647d46075c0, 0x000000000003abeb}, {0x000f7f59840e13b0, 0x000af1234c80e297, 0x0000d9f885a37dab, 0x0002d41d82ab780a, 0x000fde426e50399a, 0x0006dde778a03afa, 0x0005eca2bcb109ca, 0x000000000000618f}}, + {{0x000adbd12143606f, 0x0003e0d27500f7d7, 0x000cde21714cb370, 0x00078f8610763e40, 0x000f47ef08feee75, 0x0007d04c06e56078, 0x000aa3a8d03a7f04, 0x0000000000027cc8}, {0x000677d27af9cf4f, 0x000056b08eb37c00, 0x000a4a2b5d3fdd3b, 0x000869ae3f10ba99, 0x0003e500e8466f9b, 0x000171b1a3bfb983, 0x0000456d975557b9, 0x0000000000018fa0}}, + {{0x000e348941e625d9, 0x000fc15d47e52aa7, 0x000dc7038ccf5369, 0x0006a3070cddb11e, 0x00032b5a8db75122, 0x000f37d6563a253d, 0x000b91e8be4d1fbe, 0x00000000000c0920}, {0x00045423a83becc9, 0x0008308b713a8319, 0x000ebb9d576c8cba, 0x0005877917ee393e, 0x00091528cd12a897, 0x000daf6aa294792e, 0x0009b94512dc8c37, 0x00000000000197a9}}, + {{0x000e5d8cfc9c3b49, 0x000e0cfebe026cd6, 0x00010748e4a52d81, 0x0009d16725ebd099, 0x00007a584e8d00f8, 0x0001cab5894c6f2b, 0x0003c8095ac9cc91, 0x000000000009c407}, {0x000462695d0ed83f, 0x00056265919b06a1, 0x0003cfa033d1cd2a, 0x00097f1815a70aa4, 0x00001c5fd35a6db5, 0x000d52a8bdd9e4a1, 0x000fab38b8e35112, 0x000000000002ef20}}, + {{0x000003aa77fd8f2e, 0x000b43b21dc1edb9, 0x000008b1e2380339, 0x000e5d30da87d664, 0x000f0cd999a0f536, 0x000e08f5d4ce10ed, 0x0007e68949181450, 0x0000000000052664}, {0x000bcdd23ee4cc2b, 0x000a3cb207ee8e61, 0x0002ac9dd73c4f6c, 0x00040d225e0cf154, 0x0001067fa0527d49, 0x0008b2e21b688b31, 0x000eab89308326a9, 0x0000000000072311}}, + {{0x000c6c4de58ef59c, 0x000db2a6280408d6, 0x000f4e9cd796cbfa, 0x000daa3c647fcba8, 0x000f4952d3a44a2c, 0x000e2e2c3e1a9e91, 0x00040ebc1b3d8011, 0x0000000000018e43}, {0x00017ee00926dad8, 0x000cceb8696ff13d, 0x000436379cf7b867, 0x000d6ce6c1e905c6, 0x00091ed2e8cba471, 0x0007c8c914e13d60, 0x00084e52951509c7, 0x00000000000e2348}}, + {{0x000a3b1514eda4ba, 0x000ac37be5d3f53e, 0x000cab6203ce338b, 0x00024b59d1569e7a, 0x0009b4e293ab165a, 0x000b7c0a4254aaf5, 0x000b183c751fbd6f, 0x000000000005018c}, {0x0000fe7bbd1e6c72, 0x000eaacb3b8965e4, 0x000a2f2ab6db69d7, 0x0007258621df6d82, 0x0003185f0e2a245d, 0x000ddbd812af0e28, 0x00002a1e7edc818f, 0x00000000000c538d}}, + }, + { + /* digit=77 [{1,2,3,..,}]*([2^308]*G) */ + {{0x00042da90721f8f9, 0x0009f6858b6747d2, 0x00082ecd0813dd9f, 0x000674e29d9194d6, 0x0006dcb30a51324e, 0x0008aa5492dc6fdf, 0x000a923f3ecb7752, 0x0000000000023ffa}, {0x000c7344981b01f4, 0x00090d8efcbd58f8, 0x000d0144b46e312e, 0x000b0b74cf4a1708, 0x000fbbf335fd04f2, 0x000928c36bc0bb49, 0x000d45c6b6e5bd5a, 0x00000000000d893d}}, + {{0x00016a2799bedc34, 0x00022fb083c2a5e5, 0x0000296a1d9c8cea, 0x00030e747a81b965, 0x000658a987376e41, 0x0002829c9cd704dc, 0x000d63842ae32959, 0x00000000000be20d}, {0x000f531daa9bdad7, 0x00053abee8e552a9, 0x00078232300aac8e, 0x00039b2467112ba1, 0x000fbe341038781f, 0x000a2f8be6f06058, 0x00066e5effdca512, 0x00000000000af344}}, + {{0x000afa3ee32c886c, 0x0006d3933b4c0742, 0x000439c40f3f936d, 0x0003c0f0b7e019e4, 0x000fa48715d652fd, 0x00039be537f8e5dd, 0x000cdc0879c5defa, 0x0000000000056f01}, {0x0005bfa2d160b7fa, 0x0004671b2dcba45c, 0x000e0bb6fffdcb60, 0x0004846640b2d753, 0x0000049a2cbe6af9, 0x0002ab0f178a1071, 0x00043841a1ce67ac, 0x00000000000a76d8}}, + {{0x000a317691072fc1, 0x00025ab86639e0b1, 0x0006a85f7a13b8bd, 0x0004418d763f0bbd, 0x000211a9802cb573, 0x000269b8ad2cc332, 0x000355e41d7db881, 0x000000000002ecfa}, {0x0004e57bc1e94af4, 0x0006159d6912442e, 0x000356e842631b03, 0x0008b6b8a7397395, 0x00091cc83ae38447, 0x0007ccd97e86ad7d, 0x0004c6bc9b208479, 0x0000000000069494}}, + {{0x0006d3d1d84bf486, 0x000a4bbb6725a843, 0x000b1260c5ec574a, 0x000dd0664bcb2d64, 0x000a780d5c1bab65, 0x000a61e6e755e848, 0x000b4b84e6f686a3, 0x00000000000bd100}, {0x000cfe4a11d7ea4f, 0x000a3b34fdf84517, 0x000d9afd2967c440, 0x000c881f9a125f0d, 0x00064aeadf8e3c2c, 0x0006bf0c2b1fcce5, 0x0001bebdfc54c0c4, 0x0000000000009d28}}, + {{0x000af84d84069bc1, 0x00010e1aa844a91f, 0x000628bd439b9e84, 0x000aba65ddcb5d25, 0x000a38009c6a3025, 0x000261ee5ac8cdab, 0x000430b9ee07b46e, 0x0000000000006ab8}, {0x0003196ba81697a7, 0x000207a36b461d0a, 0x0005a884d3b0fe10, 0x000f469341275a1d, 0x000f1c7cf858ae4e, 0x00092af3df7cb393, 0x00017683c00eafd1, 0x00000000000a1e15}}, + {{0x000475d9329b61bc, 0x00078687d0980c72, 0x000d109af3ad73e4, 0x00007fabbcde5b3c, 0x00016ed103a56414, 0x000a6ee2212ec9d2, 0x000c11f858c7851f, 0x00000000000818f2}, {0x000247f6c002433a, 0x000a04720d42a19d, 0x000f64099b42a8aa, 0x000f25533723e4ef, 0x0003d3260b6299af, 0x000391c33d6c67ce, 0x00038163a6fc9d86, 0x0000000000031de2}}, + {{0x00092324c0c2dff4, 0x0004f585edee2c89, 0x000820f42406ab00, 0x000924fd85ad3203, 0x00019124bffa3bd7, 0x000c628682e7d8f9, 0x000f5d885397c044, 0x000000000007fda9}, {0x00035c11900b0d2c, 0x000e6d302582e314, 0x0003e1f066fcf73e, 0x0006b5b63653c06e, 0x000d9c70dad4b021, 0x000bba93d8f08a1a, 0x0002b7ccf014aaff, 0x00000000000a33f1}}, + }, + { + /* digit=78 [{1,2,3,..,}]*([2^312]*G) */ + {{0x000cb198f774dfa5, 0x000dd1e347819af2, 0x00020411ce747fbb, 0x0005ccc28af85bb9, 0x000b3ae9ba11fc18, 0x000bdb313c923d78, 0x000b877d494d7ea7, 0x00000000000af8f8}, {0x0004fe8c8323652c, 0x000bdc178ccc093c, 0x00006a94572bacfb, 0x00047cc80a50372b, 0x0000e6f901976087, 0x000c2f2168147aa9, 0x000b514683d4c978, 0x00000000000552f3}}, + {{0x000f8017d10c6b96, 0x0002229d81537326, 0x0007f1980aaaed23, 0x000ce6e8783d1fe4, 0x000df0e3c126aa4a, 0x000ca4b719c14ebd, 0x000f350eb1c08d6e, 0x000000000001c91d}, {0x00013bd5d85a7c75, 0x00064db8119bb0fc, 0x000fdf5036f03e4b, 0x000916835436f82c, 0x000b9b18d212053c, 0x000bc3497eb41ccb, 0x000d8193b3fb43b1, 0x000000000008c1ce}}, + {{0x0007ad7a8ff88fe5, 0x00031521f4af6aed, 0x00033eaf4c7af196, 0x0009ab7ae0377807, 0x000db65f73ffad09, 0x0006059b9541cadc, 0x000f8253430463a8, 0x00000000000043e9}, {0x00061f938516beb2, 0x0001bd515f4c75f1, 0x000d0a9d767932f1, 0x00053567ef1b9007, 0x000f1295b5fe7bed, 0x0009176278782b45, 0x000d9fab54ebaa03, 0x000000000008787e}}, + {{0x0006f626b572390e, 0x000905255508b5c5, 0x000fa6a60258070c, 0x0005eea3095205c4, 0x00002c470ef7d976, 0x00040bf0afea2ed7, 0x000a16f5e8a0fc9c, 0x00000000000e4137}, {0x00055209e40da33c, 0x00003b917537047e, 0x00003ab2893cdb40, 0x000bce9023854871, 0x0008a8bc3a94bea6, 0x000c450444620840, 0x00015164a87a2a3a, 0x000000000007df70}}, + {{0x000cf2f10b0520cd, 0x00004c38d07270c5, 0x000e4dc64fc6cd1a, 0x0001d8f300e953df, 0x00040a7dce63abde, 0x000425847b98b23d, 0x00061b63e51543fc, 0x0000000000043a21}, {0x000d54e8ecf42bd9, 0x000ee2650a1d5920, 0x00086c043a07e5a3, 0x000b7048bd9bd78b, 0x000df89e86bdc15d, 0x000579426bad1dba, 0x0005f8d7d4845024, 0x00000000000287bf}}, + {{0x0006ca98107f85ef, 0x000615fb05303f18, 0x00091aa7ab0bf32f, 0x0000fa47d92132d0, 0x0008994564bf7a0a, 0x00029e066e993faf, 0x000574bf30252517, 0x00000000000d4260}, {0x000990cf5caa4f19, 0x00009f1eecc3d948, 0x000f8dbc962efd6e, 0x0001cf1d1dd88736, 0x0003b6f1aae115fd, 0x0009344b97ce882a, 0x0009ca7efb7ec6da, 0x00000000000cc5b5}}, + {{0x0003c28364de2038, 0x000c48bcbf6cdce6, 0x000e355114722287, 0x000febf92ebe1bcf, 0x00019af24c2e4ee8, 0x000a861d57847829, 0x0007ec618760b094, 0x000000000006bae2}, {0x000de113e64c1ae9, 0x000fc77c174b9e18, 0x00055d1ba670f4b5, 0x0004f6a58a0062c2, 0x00047781acd1db9e, 0x0006d86b03480c9b, 0x000ef52d7c628ef5, 0x00000000000b3e2e}}, + {{0x000ade0c1eab3d7e, 0x0000fee9e25d91a6, 0x00060f30210199eb, 0x0001157d2d91300a, 0x000fffcbe50fd2fc, 0x0001d376ee7597e2, 0x0005624be3814fe1, 0x00000000000dbf14}, {0x000860a341c4b797, 0x0005638a107c07ee, 0x00045c1dbdbe39f3, 0x0004e09caebd481c, 0x000bf9ace60d1801, 0x0001cb8e7b28c331, 0x00065ee8fba04f21, 0x00000000000e42dc}}, + }, + { + /* digit=79 [{1,2,3,..,}]*([2^316]*G) */ + {{0x00060f98795275e8, 0x0009d009dc2f44a9, 0x000392d1d1132d95, 0x000faa2ec5d91fec, 0x0000f160afcbc89b, 0x0007566b5461be9a, 0x0009a2fdd084bb7d, 0x00000000000e4809}, {0x000f69da99c8ac01, 0x000bd1b461fc964c, 0x00012ac0cf0d2b27, 0x000332d9f09a206f, 0x00036941d6ef3246, 0x00087f5f6bde4c0c, 0x00040044ef03fdf3, 0x0000000000057b63}}, + {{0x000b480bf51dac8f, 0x000ea7e485158a07, 0x0002c9e9ee8b46dc, 0x0000dafa6fbabfcb, 0x00038f09fff720de, 0x00006994f04f0158, 0x000d335a0562976a, 0x000000000006e3cc}, {0x000d89d62ca77de2, 0x00045c087ef9c0e4, 0x000f0235509703e4, 0x0004814d916ee937, 0x0008d23d140b4fc0, 0x00062028f3369c97, 0x000adbb0fcd70f16, 0x00000000000e1e28}}, + {{0x0003a6684cfed7ee, 0x00094e3ccfe8dea6, 0x000d721274c869f9, 0x000e2cbb605c4274, 0x0005b0d809923f13, 0x0009913815ee1239, 0x000b6e4dcaea94b9, 0x000000000008c4c4}, {0x000ec88cc6307d53, 0x0004bd7aad54ab94, 0x000859bdd379b119, 0x0001eee68048fd9e, 0x000fd57225fc6eb3, 0x0008dfd3f693842d, 0x000cf13e62cd85a8, 0x00000000000d5462}}, + {{0x000df1c3e7de2ba1, 0x000ff99b5f14f615, 0x000c00e1f8e35232, 0x0003900ff19bf251, 0x0000260925ede749, 0x000ea1eafc82e5b8, 0x00021e3398d340c0, 0x0000000000012871}, {0x000896ddafdbd954, 0x00046ea6dfb0ad7a, 0x00039aa9f60646cc, 0x00078bf1201299a2, 0x000d32e5cebce266, 0x000181e32b5369e4, 0x000add2304eac86d, 0x000000000003d364}}, + {{0x000f357cfbf02e91, 0x000a2f55f868d743, 0x000049d92d19de08, 0x000ff9bbb5ba194e, 0x000957724703bec0, 0x000e0e4fbf7325c4, 0x000d7c5ab8f06323, 0x00000000000ecb0f}, {0x000a0f97049bea9a, 0x00069cc72e8da9e4, 0x000c7a0fc237673e, 0x00046e64252763c4, 0x000eb8d81de3bfa6, 0x000772f007a769ef, 0x000a5ef5ec70031e, 0x000000000002729a}}, + {{0x000cc742f063ab99, 0x0003b51f19bb30d3, 0x000878f139d45b8c, 0x000de2f987112cb9, 0x0008759de2cb5f7c, 0x00049bd98dc51d94, 0x0001fd9d05c410ed, 0x0000000000036434}, {0x00020d69ae77fca8, 0x000a9d7869d68233, 0x00007a8daf0bfe29, 0x0008180c25910747, 0x000a6ebc4ebc7f25, 0x000da1d687b90fc7, 0x0004935565390f3d, 0x00000000000a44e4}}, + {{0x000116432be8131b, 0x000ebb4a2742da3f, 0x0008bb7a8fda7aaf, 0x000adccdd7fcf4fc, 0x000337d4fcc6b15f, 0x0001c7ab70b1fa2b, 0x000cc9242c7b26f0, 0x0000000000050574}, {0x000069eaa2593652, 0x000b76ee50f421b3, 0x000a5035e1a2b7ad, 0x0003af18420501c6, 0x000b1b6e7d2a04d1, 0x000516f80c22e092, 0x0005b5a393be7ee9, 0x00000000000b80bb}}, + {{0x0003f2ad59508de2, 0x000bd9e41b9b6b62, 0x0006d3eb906556a9, 0x0000c8cf272346c8, 0x0005def5749367e5, 0x0002676fcc98a5ef, 0x000659c2dea8afa0, 0x000000000006ace4}, {0x000f1f5cd12db7d7, 0x00090c2b707aa093, 0x000d4cda421d1577, 0x0000fa2a778c20c3, 0x000d57af166b5b58, 0x00078ce62272c3a1, 0x000087e47a64a9ca, 0x0000000000071d35}}, + }, + { + /* digit=80 [{1,2,3,..,}]*([2^320]*G) */ + {{0x000e3ee1ae74a706, 0x00069c20fb9106bc, 0x0005cf0e923045cf, 0x000bc6a5cc0aa89b, 0x000b4fb9f01e12bc, 0x000ae1b895279c6a, 0x00003cbc8e178911, 0x00000000000c2900}, {0x0006736dff18cbd8, 0x000579949d6eb7ac, 0x000931ae3b95e19f, 0x0006ee2dd2275c0b, 0x0008c419f1d568de, 0x0002b5a40a9e4fff, 0x00096dbc759ca4a5, 0x0000000000063b09}}, + {{0x000eafd50b503ff0, 0x00058f682715078f, 0x000b7ec2b450662f, 0x0008f55c51916c40, 0x000b61ee081fa46c, 0x00064872a98c3524, 0x000faa5ab7f2c2ad, 0x000000000007e555}, {0x000425ebf2c1673e, 0x0003c802e0d22c20, 0x0004e9f77f229a7f, 0x00019087af20d270, 0x00016be50784b56d, 0x00072b8518d619e2, 0x00076e10fdbb7213, 0x0000000000049355}}, + {{0x000df2627e9f5583, 0x000e073c57fde593, 0x000f9da9fd5339d4, 0x00071f1cfb9821f2, 0x0009cf6a1d68d25c, 0x000d24e649427e20, 0x0008b5bf3c3916ac, 0x00000000000bda7b}, {0x0001a88dfb057584, 0x000e2e7c5f7c7d49, 0x00071c4b5c378478, 0x000c472e66b277de, 0x0007b2816f1e818d, 0x000383c6d4413fd4, 0x00053740f262af48, 0x000000000004f190}}, + {{0x000aa901e2868cf5, 0x00009d97a9b136f1, 0x000f9d37eb7a3b6c, 0x00060c2548188194, 0x00061d59be44c571, 0x000f2b2474dbdeab, 0x000b33b93a9dad81, 0x00000000000ecbca}, {0x000a766b5fedf0a6, 0x000ce5e1ce82823b, 0x000997bad589b344, 0x000273a2d82b0df8, 0x0006ac8ff52fe716, 0x000ea7ccdb017f5f, 0x00074b7563e79970, 0x000000000003f0e6}}, + {{0x000fda8d824654de, 0x00033487edc06864, 0x00024946c59b50d3, 0x0002bea10fdba0ce, 0x00052c3dbf337fc6, 0x0007d2a46c836bbc, 0x000f714fdfb5c34f, 0x00000000000dca0d}, {0x00024bebc7614440, 0x0007730a0767544c, 0x00067fffd43f5d03, 0x000d49dfacc13dc4, 0x000aae1a261ad307, 0x000d98650148d673, 0x000fdee008f95b52, 0x000000000009f558}}, + {{0x000015df21e067c3, 0x0003dd084564b8db, 0x000d0b4c4a73b501, 0x0004bf63bc7206d7, 0x000ffc3d577224bd, 0x000c4924fdd37b4e, 0x00096e8156d6f635, 0x000000000001d1d3}, {0x000ca9d7511fd3e1, 0x0009d977b3b3982e, 0x000fc4c09646cd37, 0x000f9c4ae811d70c, 0x000ca06c7f0de581, 0x00049d87fe5441d5, 0x0009bb0275c92b19, 0x0000000000078046}}, + {{0x000cdb7d87948ad3, 0x0009117cd6af6151, 0x000e60b294f18979, 0x0005b1d788224330, 0x00031b9745257e49, 0x000f3ce21fed338e, 0x000896527743d386, 0x00000000000b515c}, {0x000e2d7dc921023b, 0x000d19fde7038e85, 0x000c1b9c9ba2d99f, 0x000600b36ff74dd6, 0x00048302c3c95d46, 0x00044b5eb8ea74b0, 0x000402d560f570f7, 0x00000000000fe762}}, + {{0x000cf97b63daba80, 0x000eea7f8501424f, 0x000f397f945052f2, 0x0006b8a2ad5053d9, 0x0003fbcebe43c658, 0x000054f1688c09a4, 0x00047fde7f2a4a20, 0x00000000000bbbb1}, {0x00041659e06114fc, 0x00075087e48f29cd, 0x0009df479d6378c0, 0x00002ff7f27cac87, 0x000c37b1f6e18ae8, 0x0006d1deab01a131, 0x000f803f0d4c2e9a, 0x00000000000e7cee}}, + }, + { + /* digit=81 [{1,2,3,..,}]*([2^324]*G) */ + {{0x000643c3f46ade04, 0x0007aaa03b473d97, 0x00091126d04a500a, 0x000ef86f37b57bd6, 0x000b2a6371d2b286, 0x00077ccbd95b797e, 0x000a6da489ef8940, 0x000000000008e99c}, {0x0007980fdcec94f4, 0x000845cf9e236463, 0x0000b4681fa85d18, 0x00023b6ba7ba4781, 0x000757de30bf3295, 0x00036ca01d406ab8, 0x000ac5aa10e4a215, 0x00000000000dfa7a}}, + {{0x000146df48d94f2a, 0x000a9762a5eebd68, 0x000728ae94bf3482, 0x00034aeaa4f964b4, 0x000ed6526f010fd5, 0x000d71d72cf46648, 0x0008eabc62a54a18, 0x000000000004f848}, {0x000d9dd392598cb3, 0x000a262c40de70e6, 0x00099451ce04b75f, 0x000d7466ab190976, 0x000b2dd1ad3d3a51, 0x000f88078f07bea4, 0x00035d2a0ec4672e, 0x00000000000664e4}}, + {{0x0002f7cfce391389, 0x000d017c563bd4e7, 0x0006d61076868226, 0x00049ac4cd1bfc15, 0x0000d4e53d3b591d, 0x000079a6ed446551, 0x000eb4ca95ccf6ad, 0x00000000000b5ea6}, {0x00099522fba1cf25, 0x000cd600ccc9ba8e, 0x000c729701bdaf25, 0x000edc569c6c6e81, 0x000a586abcdc5f27, 0x000dbc5b6b3fc0f4, 0x000fe0dbfdf3f985, 0x000000000004ea80}}, + {{0x000e7847d76787ae, 0x00083ee6e7005dda, 0x000646f5e7ea8d16, 0x000df7dfc45111c8, 0x0001875c23c8373a, 0x000039d490c7535b, 0x000a062a7e04f897, 0x000000000009afd1}, {0x000cbb06099a2183, 0x000fb049dd385881, 0x0008a6ac539cc7cc, 0x00057b9925498c29, 0x0002da25daa3dce0, 0x000decb7d7b9c9bd, 0x0006ef6d0b70a3d4, 0x0000000000003df7}}, + {{0x000405f3ffe95388, 0x0004eff54e8bb8ae, 0x000ad0d572466c0c, 0x0005bc4ed1fae2ed, 0x0009ea983db1b226, 0x000851454a5e23c2, 0x0005064af5f55909, 0x0000000000036eb4}, {0x00062710e23e547f, 0x000844c72b2d0dfe, 0x0003b7abb313e4c9, 0x0008ad4e19c9c269, 0x00053e8b8004b1aa, 0x000c747fdb9cfea9, 0x000e4107782d788a, 0x00000000000ce9d9}}, + {{0x000aeca3f9d9a2f2, 0x000f8779905b5deb, 0x00064a28a88ba6a4, 0x000f0aa679096dc3, 0x0000f558913cce9a, 0x00024a2e2fdf04f1, 0x000b3505300cb06f, 0x000000000005d581}, {0x000679a2a74abdaf, 0x000aa855c8de8c32, 0x0000f6e26bb556cb, 0x0002a4197740d667, 0x000744c9360d6761, 0x0005cc8807588a52, 0x000188928ac91032, 0x00000000000acdd3}}, + {{0x000ac6fa45a00390, 0x000933774af3674c, 0x000bcfd34179ae6d, 0x000205ec11875399, 0x00018799c22ede45, 0x000b0842ef3b1144, 0x000d64fd53f681e8, 0x0000000000066d36}, {0x0007de25fd86d3c2, 0x0009ca2c3391c37b, 0x000760d08855c7c1, 0x000fa4a6ab8ab4bf, 0x000bd841a6234df1, 0x0000b189a2b0cd34, 0x000da752441d79b1, 0x00000000000fa025}}, + {{0x0009e3af5a1f4313, 0x0000c3c6c56185fb, 0x00083406c1a4afdd, 0x0009c90ae2fb830d, 0x0003ec31a504b325, 0x0006230bdf7cc6e4, 0x000e3a83bb13c612, 0x00000000000ff1da}, {0x000c261cfaa4903f, 0x000010cc6c1a44f0, 0x00029f13aa21ef15, 0x000c75fe604a58e6, 0x00012491f872d454, 0x000e116a5ceadfd9, 0x0004816c5575da9c, 0x00000000000b24a4}}, + }, + { + /* digit=82 [{1,2,3,..,}]*([2^328]*G) */ + {{0x00087efd2e620105, 0x00045c5ad791e85f, 0x000885fe6de88f7b, 0x0009e4998544bc47, 0x0005d273c360673b, 0x000a75a8c934d8f5, 0x000c977fb48d0768, 0x000000000005e0fa}, {0x000ee7923fcfedee, 0x0000dbe01655fb13, 0x0001ed409cde6661, 0x00030de533251391, 0x0001de9f3fc574f8, 0x000e01fa5a724033, 0x0004b496b25206a7, 0x0000000000007eda}}, + {{0x000a5e409b41a55f, 0x000283e6e9380f21, 0x000bd6398225ced2, 0x000c8b4b843ec071, 0x0002af8a23f1f2d7, 0x0007923350055d88, 0x000e74e848010ed3, 0x0000000000055e08}, {0x000839aa213ff6af, 0x0003d956ea8e93ca, 0x000299a04653a35e, 0x0009271cca1b5491, 0x0001f1d0b7f15f7b, 0x0009963fbd41d228, 0x000cd8b187047b5c, 0x0000000000002536}}, + {{0x000a16fe10008718, 0x000fd7138a03f4b5, 0x0008b9afc3d67514, 0x00041fd2a7cbcc17, 0x0005f6073ecba29c, 0x00004fa086014349, 0x00053298b46d3805, 0x00000000000401f9}, {0x000b314f178aa1e6, 0x0009808cc4abc63d, 0x0005190f1545e910, 0x000c8faa75d9af1b, 0x000fc2e5592d6eb2, 0x000468e666f50198, 0x000fc3773b9d55c0, 0x000000000001a2a9}}, + {{0x0004cdf21adad97a, 0x00008e51a956ff23, 0x0002a26288680976, 0x0000fa2f974d2cb8, 0x00074c6a78dec616, 0x000c89d11e00474f, 0x000133ee916e510a, 0x00000000000f826f}, {0x0002a0e3be0ad710, 0x000103fc65d3ecf2, 0x00089ef785c9b19a, 0x000a82e0b9196b29, 0x0000a572b6f55237, 0x000bed2bc4975e68, 0x00025e41ea8b92b9, 0x0000000000098268}}, + {{0x000bc901bf39d0d3, 0x000fae352fb9c164, 0x0000b33959aa5988, 0x000ce6fafcb7de1c, 0x0004f694cad0f9ff, 0x00061d6110e574cf, 0x0005d8adb4671cb7, 0x00000000000c8aa0}, {0x000b2a78a9127846, 0x000a987cd7872df6, 0x00010a813ed5d7a6, 0x000531ec2cb43e23, 0x0004b9e6b89f1c41, 0x000e5d4d262fc310, 0x0001dd7f406a0dc4, 0x00000000000df3f6}}, + {{0x0005300bc3a99814, 0x0008c06f721d246f, 0x0003df2a36c6f8d6, 0x0006b7a5a44ffdab, 0x000915764390c5ca, 0x000747d7dc53e328, 0x0001227f16917459, 0x0000000000061f79}, {0x000965ef8eb0e108, 0x000ae099f4e3a08f, 0x0009b651d265b539, 0x00043576b72f3285, 0x000cc33695bd270a, 0x0002249029d6bb26, 0x00051f60f9202126, 0x00000000000e1b0a}}, + {{0x000bf8353cd33783, 0x00086265d4eff002, 0x00010092c6e845e0, 0x0005c996be6ce254, 0x00096b6056eec399, 0x000a007266fd43e5, 0x000fb1d328c28f28, 0x00000000000f719a}, {0x000f1e2a3462b4c4, 0x000fb75dcabd2f96, 0x0006c939c482a7c5, 0x000a522d764ab7e8, 0x000dcf8a487267c6, 0x000810a449458c96, 0x000786af1af61b78, 0x0000000000073902}}, + {{0x000b7c441c7f3df5, 0x000a88a390dca834, 0x0004f92ea9440c60, 0x000ea69bc5e82f15, 0x000d4628d6f177bb, 0x00068e24071f02e2, 0x0000ee6260790cf9, 0x0000000000066527}, {0x0001df8e5c25d23a, 0x000f233639544be6, 0x000c37cf5bac2371, 0x0000f867807ed557, 0x00019dfa452c6906, 0x000d27c0d1b0ac01, 0x00090cbbdb25fe06, 0x00000000000a60ef}}, + }, + { + /* digit=83 [{1,2,3,..,}]*([2^332]*G) */ + {{0x000957058557b81a, 0x000dc1877dfd042b, 0x000fe0adb79e32df, 0x0009536c78e295f6, 0x000374a04cc59d08, 0x0007f75edcbb5d55, 0x00078ce366600682, 0x0000000000061e73}, {0x000dd46741146ae1, 0x0000a74170170ac0, 0x00052de550198353, 0x00030f8ac1c4a0fc, 0x000d3cd92824d75a, 0x0005e86d4d7e7be0, 0x0005ef5ea0abdb1b, 0x00000000000b3b61}}, + {{0x0003f4e13d621f8b, 0x0005a618ca06b4bf, 0x0002813dfa928efe, 0x00072cca3b6ce7e1, 0x000e5f14bb579813, 0x00073abbbb44a68e, 0x000c0afc1167e963, 0x00000000000767d4}, {0x000277deb88a4f9a, 0x00077c6dcdfeb3e6, 0x000664e1fd3b5c4b, 0x00001aabfcbf2f89, 0x000a67aa82774775, 0x000aab9cd0be7228, 0x000b4a5de7b8331d, 0x00000000000262fe}}, + {{0x000cce85ebd04b7f, 0x0007f7a9623b5682, 0x000f8d813f260bd0, 0x000dec9e0f8e530f, 0x00091d65dcd4f8a5, 0x00039541c88bcbe2, 0x00097a09fe380367, 0x00000000000e716a}, {0x000184fc8c1d72cf, 0x0003bbd8e34de924, 0x000a19b51363e270, 0x000d905351941206, 0x0006f7a3c4830183, 0x0003557396345099, 0x0002f718283cea01, 0x000000000009aaa7}}, + {{0x000a56eb624d0cf2, 0x0006a7b7da0287f4, 0x00087c61d1ede60c, 0x000d83c51f3dbd9c, 0x000fa91d085ac056, 0x00035262de2f4d46, 0x00029e31759c9ceb, 0x000000000000c9dd}, {0x0009a41b82bb0c12, 0x000e8108cf9ba936, 0x0004fc5060b7103d, 0x00016b273d372b32, 0x0007d5415e53f698, 0x0006b8b0e126575a, 0x000a5e546bb4c130, 0x000000000004d54e}}, + {{0x0002c932a7c5abec, 0x000a881dd074d9bd, 0x000f76c734b204ea, 0x000e1f0352b58f62, 0x00029c390eefb5af, 0x0006a8d236d2d948, 0x000be5d073f9a0f3, 0x00000000000466eb}, {0x000118cb3dd8d87a, 0x0004d968c04a5c76, 0x00007c8586d1cdac, 0x00096bb15c0970b9, 0x000370ab8dda98e1, 0x000039fa92af28ea, 0x000ff6b912fbda7a, 0x00000000000cbd02}}, + {{0x000e45ea08b7c4d8, 0x0001d5bcf28785c2, 0x000fc63d7c5e8cf8, 0x0005de6707a7138f, 0x000cfb2041a48868, 0x000db3060a77646d, 0x0006908608695289, 0x00000000000e27a1}, {0x0000f0b051870cd1, 0x000794148d936eb2, 0x000a4e50dd50fda5, 0x0002d4db0391d14a, 0x000462b0373f9d72, 0x000a9992c5683112, 0x000b8487b0363be9, 0x000000000008853d}}, + {{0x0002c578f8f683b4, 0x000576c8e5a2524a, 0x0007522f4370ef9a, 0x0004aad50ad6f3a6, 0x00051f107e269b01, 0x00051bf732a3933c, 0x0006df4b04395cc9, 0x00000000000b0ef5}, {0x00017d2e75e44b21, 0x000cd28c9fcdca32, 0x00061a5012b72649, 0x000f58e53ace23e2, 0x000efe688f02b546, 0x00091406dee5fec4, 0x0004d9d8b43f6d2e, 0x0000000000026176}}, + {{0x0002d2ad73a4790a, 0x00034daa7fcc985a, 0x0000276056234a54, 0x00018c59894edceb, 0x000da9ac29a7fbf9, 0x000372f2c470c45e, 0x000d3fdfd44f6fbc, 0x00000000000a1e38}, {0x00009dd8fe96aa5f, 0x0003923d2400aaad, 0x000c441c1c12013d, 0x000488148baa4ac3, 0x000ebc26e6b0c76a, 0x000ef566273227e4, 0x0002bbe732edcfe8, 0x000000000000c566}}, + }, + { + /* digit=84 [{1,2,3,..,}]*([2^336]*G) */ + {{0x000d5d97020dd4a2, 0x0005087c7dd7d118, 0x000d6556f04f9110, 0x00048789fff00158, 0x000b909eaf1abcbc, 0x000443a788ffcef8, 0x000fed9905f4eef1, 0x00000000000e15cb}, {0x000831894b4ea50c, 0x0006cca2969ec57b, 0x000ae2b1ba048b54, 0x000fab787ea6e3a2, 0x000ff98c4fe567f7, 0x0009cb91e0d0ee10, 0x000ab4c646879ac1, 0x000000000007d1d7}}, + {{0x00092725c84d0435, 0x00013cfbcbc49b56, 0x000166c387130162, 0x000e05622322f91f, 0x000a7662ae23735d, 0x0002be7a6d2590bd, 0x00017263d468fed8, 0x00000000000695ea}, {0x0005986e2611645f, 0x00050c50f4940fa9, 0x0003729def8820a9, 0x000cd041fd257785, 0x00044a3d0808680c, 0x000684cba25ddac4, 0x00088841d8b738c4, 0x0000000000053963}}, + {{0x0003a8028963f3e3, 0x00076b05cb83426c, 0x00074bc4cf79f053, 0x000b5aaefa9e4f0b, 0x000d9bd075ecf023, 0x000a41bfbb8061e5, 0x000d967d2ba50f1a, 0x0000000000090865}, {0x0004ae64b7cf3e16, 0x000d97f221a788f4, 0x00000b032e3ccff2, 0x000eb6af6ab15418, 0x0004dc87cd93085f, 0x00039dbf6fd14102, 0x0003643e196fdb32, 0x00000000000dbcd5}}, + {{0x000d4ecde2fce0e7, 0x000891466c9f4fd3, 0x000b63a0ee692739, 0x0008c75b58655519, 0x0004e65862cc291e, 0x0009971671ddb715, 0x000c19285153bc2b, 0x00000000000954b6}, {0x00039e1aad688b1d, 0x000e0985d4505697, 0x000559cd349baa64, 0x00098dafa5fe5e6a, 0x0006f90f231d39e0, 0x000afd0b53fc2604, 0x000c18de5d5ced32, 0x0000000000060c09}}, + {{0x000b1d0586854e54, 0x000cf3720b17188f, 0x0005eebb9c78a9a7, 0x0006e315b54eefca, 0x000472b0edaa03c4, 0x00077476e387e466, 0x000259fc59b03dee, 0x00000000000607a7}, {0x000e6b356394b5a7, 0x0008b1478bceddaf, 0x000e3cdb4ff0323d, 0x0002f72cf272ee0a, 0x00038bd885cd7127, 0x0006ba31c19e3a4a, 0x000b369af6415b37, 0x00000000000807b2}}, + {{0x0008b39051ba2cdd, 0x00030dbfcfa7593b, 0x00015a46addd3783, 0x000851d67a19b610, 0x000431ae1a67cc6c, 0x000d1e135ace88bf, 0x00090a74554a6193, 0x0000000000042118}, {0x000bd87496a1468e, 0x000277cb3268907e, 0x000047feb42207f2, 0x0002c21766e72f3c, 0x000a880642e30a67, 0x0004d2b3624bb718, 0x0004ab425dc41d18, 0x0000000000089102}}, + {{0x000caa1ac9c2c636, 0x000673180789b5d1, 0x000d386a756090c3, 0x000b895e99e922ea, 0x00003d0bbe807951, 0x000fe79c0987ea25, 0x000d2a6d2f49f0e2, 0x00000000000c2b18}, {0x000ed8226be989ea, 0x000d8707798f36cd, 0x000ebf8dc541b02a, 0x0001e32bc4479412, 0x000938a845f346cf, 0x0005e5d924dfc145, 0x000c3f210083fe45, 0x00000000000a1fed}}, + {{0x000faea0d2bc54ec, 0x000a972fc8198080, 0x00029736c6c3c3d0, 0x00098bbd99e30373, 0x000a0efddfca3691, 0x0009147416b68390, 0x0008078c35f3e4f0, 0x00000000000ca7d7}, {0x0005a76575b0ee6c, 0x000e9564d991dc4d, 0x0003cf033490be81, 0x0000dab5d635893d, 0x0006f944e49a51f3, 0x000233bc00427e34, 0x00093e1e0bf1b56a, 0x00000000000617bf}}, + }, + { + /* digit=85 [{1,2,3,..,}]*([2^340]*G) */ + {{0x000129009060318c, 0x00056d23658c8428, 0x0004596f83a71a5a, 0x00010ec082210f1b, 0x0001bfc364906dab, 0x0004f2d14fb1add9, 0x000443eda8b02f9a, 0x0000000000056560}, {0x0004169ffc0d0413, 0x000986c01479d686, 0x000d5173dbdf44ef, 0x0003c1d718883983, 0x000e13c34fd24dcf, 0x0000c15ac87a9c04, 0x0005d2fe0e08ec51, 0x00000000000c0f49}}, + {{0x00099e97b82e73d1, 0x0005db097b2c50cb, 0x00071fe923d57f3e, 0x00066e9420996453, 0x000eff7290736382, 0x0008c394043d059d, 0x000925eebb5fe114, 0x00000000000daa8e}, {0x000995963e49eb67, 0x00007f43c78d7e6f, 0x000a34a7c03b2d6b, 0x000a7d66b6363576, 0x000be096a954cdd5, 0x000afa2fc8b70a2e, 0x00095a011d5419d0, 0x0000000000004fb0}}, + {{0x00033bb61172fb0c, 0x00008eb05c51603f, 0x0000d435c61a818f, 0x00073cd489576f54, 0x0000535a8d57cfd5, 0x0001436c4e653538, 0x00022d1394731467, 0x000000000005f0a1}, {0x0007992fb047bfc7, 0x00099aecc7ec110e, 0x000e8ab91f3e896f, 0x000f6523d4221aba, 0x000f2851996bdf96, 0x000cfb67efef5649, 0x00005246fb9f26d5, 0x00000000000ef5c4}}, + {{0x00016ebee78befa2, 0x0003188e7ac8d2f0, 0x000b37c50c491499, 0x000d52918be419b3, 0x0004c057621c3b96, 0x000e57e597e75e35, 0x000463cb1b709f5c, 0x00000000000844f2}, {0x00053157dde1a349, 0x0006d53608198549, 0x0000450e27f360c3, 0x000d348e04114157, 0x00004d73eeae6dd0, 0x0000ed85e5b28e95, 0x000b580843923269, 0x000000000006da14}}, + {{0x000ce0f0a89b6284, 0x000735cbdc4424ba, 0x0003b0cbdca1bfb5, 0x0006886c118d02b1, 0x000675aaf27658a2, 0x0005f6ec187ff74e, 0x000d32b133be95f6, 0x000000000004b1b8}, {0x000fda7aaf38358a, 0x0001bdd8a8ef25a8, 0x000eba5b65bc2a4f, 0x000a6db26623fb02, 0x0000bae15453d525, 0x000f89fc9f2368d4, 0x000a3e55d6a84d84, 0x0000000000086021}}, + {{0x0009f5565cd739fd, 0x000f210520b26d6e, 0x0004ac2940ef5ddb, 0x0008ef88948f78bf, 0x000550ad452e1246, 0x000c4847040a9d27, 0x000ace5e382347c1, 0x0000000000033e73}, {0x000c8b4278956db1, 0x000437d83d02e97a, 0x000af99c625baa1b, 0x000d448885679e6a, 0x0005fc95919c6716, 0x000c2194a0e12bdd, 0x000d2d7da4420492, 0x0000000000015ffc}}, + {{0x000c11f07976d090, 0x00025bd048bdc85d, 0x0003c8a142cbe0e8, 0x000a758a985ac100, 0x000cf2c7ace940aa, 0x000ec347d6039bcd, 0x000ab7712092cc6a, 0x000000000006b5ac}, {0x0003db66ec59bab1, 0x000551ebcf80c829, 0x000fffebfb9d4dbf, 0x0003d1ad0b610f09, 0x000498c28ac73fdd, 0x00059750fdd3f6f9, 0x0001ac650b77943a, 0x000000000002d399}}, + {{0x0001cfefd0dbceda, 0x000826cbd2756691, 0x000e925943cf3919, 0x000aa0be4c58c7c2, 0x000f0e488784177e, 0x000916b0f603551e, 0x0002e2c8eba131df, 0x0000000000015973}, {0x00008eec1292bc0e, 0x0000355acca849df, 0x000e7c404ec832b8, 0x0002c703bd3b202e, 0x00056ddd8eba162a, 0x000b1d93d4c5e5d2, 0x000dcf66e7844a77, 0x00000000000110b9}}, + }, + { + /* digit=86 [{1,2,3,..,}]*([2^344]*G) */ + {{0x0003ab751954f075, 0x000f91b66faabe09, 0x0001714e51539902, 0x000f3a0a675f7c8e, 0x000f30313a711a82, 0x000aea9e682884a2, 0x00005d7ac5d7b058, 0x00000000000cd5ff}, {0x000d5d715b7b74ff, 0x000287c29638d05f, 0x0006736db974b38e, 0x0003c47a17ae3a7c, 0x00077009e38ae85e, 0x000f9c52e91b107c, 0x0008a0f3b777d8f1, 0x0000000000011b68}}, + {{0x00072d048b012b70, 0x0009a4ae3d232353, 0x000ebed55756fa98, 0x000c769ec62fd6b0, 0x000f62a4720cba73, 0x000c1f491d586ba7, 0x0005716497cd140c, 0x000000000007b2ac}, {0x0007af894008277c, 0x000b9a65eabb5e68, 0x000cb737865439ae, 0x0001b84231457d7c, 0x0005901645c525b3, 0x000f7b656cab62f7, 0x00095c2377d74db2, 0x000000000002d33c}}, + {{0x000e54e4ebfecf4f, 0x000b310105dfc241, 0x0000ded90576b5a5, 0x00068324e80fc085, 0x000287dc7da6122e, 0x0004728cf3b26e76, 0x0001297bc183de6d, 0x000000000008bcdc}, {0x000bafda0190b71b, 0x0005d8b995cec24c, 0x00099629f2c641a0, 0x0009e6b4da5b77d0, 0x00096caf9161612e, 0x000f45eb68ec048a, 0x0000e9e3628ee2c7, 0x00000000000d8565}}, + {{0x000dc37cf3c258c7, 0x0005aaae2f447f93, 0x000c6f7663c30e3d, 0x000b2f482c1f372f, 0x000d351a5f7f3262, 0x0006c75a85521feb, 0x000fb8f8ec919091, 0x000000000002728d}, {0x000f483180d24d43, 0x0005f5dd4ff4f0a1, 0x0000b042bcddd9b8, 0x0005b98ba8b777b7, 0x0006fb7f8409318b, 0x000fd31dbd971d42, 0x000347ed1465e8b0, 0x0000000000000f66}}, + {{0x000f5311360afd40, 0x000bacea0374a33b, 0x000feb5ded889fab, 0x00002361bb01f474, 0x000a8c328b8bf6ce, 0x00053d6302a5b28b, 0x0005d86991b1d8b2, 0x0000000000085945}, {0x0008eeb3e3866a93, 0x00024f5c6e141989, 0x000231ed9c304f2e, 0x000032f67e76ece0, 0x0002338980594eb4, 0x000001b765bf14aa, 0x0001340804a7c00e, 0x00000000000734cd}}, + {{0x00036d45a9f2195d, 0x0007e5ba5288b70f, 0x000e413923c56371, 0x000a997602f3c65a, 0x000fe08c0223c1a9, 0x0007e6f1dc5c7512, 0x00059748a19c3c36, 0x00000000000fb241}, {0x0001d161a9f86145, 0x000879f674a5f0bc, 0x0001526754b42988, 0x0008c4303d6f13ed, 0x000d917433fb5aeb, 0x000f534900fed575, 0x000616e4a5ef9a59, 0x00000000000f315e}}, + {{0x0008596b7b07e015, 0x0009c7059c585ee9, 0x000cfabffa6395f2, 0x000d9318d9633cd0, 0x000f37bda14896dc, 0x0008964dcb2abc44, 0x00076ce31adb3feb, 0x00000000000e3168}, {0x000ed4ab82ecfd95, 0x000363173288028b, 0x000f24a3fd6c4552, 0x00034c91d6f69add, 0x00022c34e118b5b3, 0x0005984ede613e56, 0x0003a18bbdbf5c5f, 0x00000000000cf4f6}}, + {{0x0008f4401a1165b4, 0x000b4315ad7a4644, 0x000672c1b06e4df0, 0x00097a8baa564733, 0x000446edcfcbe12f, 0x0001a968ef263db4, 0x00087547cf91d53c, 0x00000000000c15db}, {0x0007f5c515f16ba5, 0x000345d35e53a1e8, 0x000a90f359724b01, 0x00011ea246da3d37, 0x000653f068205d3d, 0x00010c00fc1ddfcc, 0x0008c78169d71166, 0x000000000004bddc}}, + }, + { + /* digit=87 [{1,2,3,..,}]*([2^348]*G) */ + {{0x0005418902e018d8, 0x000328002f4583b6, 0x00029b160f5eca39, 0x000d112fb93f735b, 0x0002196e3084a8ce, 0x000e8c74427cb629, 0x0005ff72395bdd77, 0x00000000000ee71f}, {0x00030cc165e06d5c, 0x000ca7cfc14d95b5, 0x000ac1b9673d9545, 0x0000129d213738ef, 0x0001bc0b5ea366e5, 0x000a067007a905d9, 0x00082192cb630afc, 0x00000000000bf3a0}}, + {{0x0009c76f27cbedd6, 0x00086e96c4aebbe2, 0x00087447d6551831, 0x000d2f632f9151d3, 0x0004302b99e2f86a, 0x000fd317105daf87, 0x000c624299dbfa14, 0x00000000000812cf}, {0x000ccd383b8a542c, 0x0001b42e615367ce, 0x000e792323a5de78, 0x0006c70548fffa38, 0x00077b6db825c34c, 0x000f2989e1fbed77, 0x0003ee850bded44e, 0x0000000000081546}}, + {{0x0008bd3d7d2dac19, 0x000ce352e14371a1, 0x000574a96d5757aa, 0x000d9395a7b7719e, 0x000b8544328b64d5, 0x000f9c5934d5197b, 0x00045e5220626522, 0x0000000000074d6a}, {0x000ed277c567a2e7, 0x00003f52c9eeb86b, 0x00037cef0ab9cbea, 0x000b3bfc9ed39349, 0x0007a14c3d70e606, 0x000db5876fc5046d, 0x0007a181cd053e5e, 0x00000000000a3034}}, + {{0x00026a12e217941d, 0x0008a9decf2164e6, 0x000546598e5e9913, 0x0009ce8aeb36b93a, 0x000158fb4dc8d564, 0x0002d60cfab9f77d, 0x00061966b11fb6b5, 0x00000000000eaaa4}, {0x00098700891e3d12, 0x000dbf3522a998ce, 0x00034cf7624bc215, 0x00097c625e387237, 0x00072d4595ee2679, 0x000e5456ced5047b, 0x0006feaaa41e5f55, 0x0000000000078cfc}}, + {{0x000abf51b7538110, 0x000353fd75579f7b, 0x000019e5c13ce4a6, 0x00076854d208bb77, 0x000ac1c9512f4c82, 0x00081e9f3941aeb9, 0x0003396ce0bed5a1, 0x000000000003744a}, {0x000bd7c923f4230a, 0x0006180eaff7a041, 0x000dbb73984381c7, 0x0007c1b0f8e7fd8b, 0x000aaf499630f16e, 0x00080a16856bdddd, 0x00012be8c112df11, 0x000000000002987f}}, + {{0x000b02027989cfc0, 0x000a60ce9ab12f61, 0x000973d1a5ee6cd8, 0x000b3b69a1753a9f, 0x00002bae5685f031, 0x000d3c06632160ba, 0x000f5cdfde9ae80c, 0x00000000000a180c}, {0x000f0a3aff152330, 0x00023e2c194158b5, 0x000e1481b10c0c49, 0x000e7d12ea20322e, 0x00007968c7ff67ea, 0x00027bab93eb507b, 0x0006eaeb300ff9ce, 0x0000000000097575}}, + {{0x000b71b2ec924484, 0x00007325de655ef5, 0x00027d8f4ae5da5e, 0x00026ad89e4c34bb, 0x0000ec5a615fa909, 0x000c770ac8e61adb, 0x000f8a5a2233d43f, 0x00000000000cbb23}, {0x0003b01804f61225, 0x000a4a7e6c5861cf, 0x0004fcae81492249, 0x00040be697e7dc09, 0x000eb6f29135b0f3, 0x000e89899783a7d0, 0x000f55412469b007, 0x000000000001bfaa}}, + {{0x0003c3de6d71b673, 0x000cca438634e69c, 0x000f00cb40863203, 0x000dedfac40dd56f, 0x000e4956a5de2ab4, 0x00013f84d758e95e, 0x00016cfc11e39451, 0x000000000006059f}, {0x0008f87a862c83c5, 0x00038fd8e2236750, 0x000df4ebd7b9092c, 0x000b1538ea13b455, 0x0003013d382702ae, 0x0009ca201de1275d, 0x000351470a7b7e65, 0x000000000005c77b}}, + }, + { + /* digit=88 [{1,2,3,..,}]*([2^352]*G) */ + {{0x000bd58c2fd385c4, 0x000d8281f6e58982, 0x000c3afee2ff7056, 0x000a41afd89abd8e, 0x00007984feefe29f, 0x000d20a64fcb0b0b, 0x000cd50b0928a6d9, 0x000000000006979c}, {0x0008ab27cda5c7bd, 0x0001cce9c34c7521, 0x0006dc0b027875db, 0x000635250946a5c3, 0x0006f39f53b9d464, 0x000bc8b64b09a97c, 0x0000d61d47bc20dc, 0x00000000000d458b}}, + {{0x0004b16a9cc79c48, 0x00075763e36638c9, 0x000ff772bf788245, 0x00011a8c66b40e9f, 0x0008f384b70862ab, 0x0001978760624469, 0x000837e7cdf3bd69, 0x00000000000064f8}, {0x000fbc5f9be69c3f, 0x0007895900a21f89, 0x00053a9326cbd6b2, 0x000fbbddd3e6e471, 0x0006baa2e2a03f65, 0x00085282484f52e0, 0x00032c1dc462a8e3, 0x000000000007e4c0}}, + {{0x0008c150b5ec2626, 0x00094cc580ea6853, 0x000526b13f535e43, 0x000604fb23480cfc, 0x000a344146898665, 0x00010a94595787cb, 0x000c78425d7c6f4d, 0x00000000000b2f1c}, {0x000940b59f5f9db6, 0x000f455da8884e6f, 0x000468b788890b3d, 0x00081d7d99e417f5, 0x000abf28fba2c648, 0x0004eff801eeba5a, 0x000fb720feb7b350, 0x0000000000050deb}}, + {{0x0002dafb4e000df5, 0x0009720ebf79c9aa, 0x000a041b02faa426, 0x00007e78d630d2b3, 0x0007fa605dcb016c, 0x000d0470520021d5, 0x0007e66190f3e942, 0x000000000008974b}, {0x000bb7ed0e0135bf, 0x0003b6710da6c4cb, 0x00011bda556d9709, 0x0004b8ba583089a0, 0x0004ea7dbcdd192c, 0x000df1097171ab8c, 0x0000ed715f60818c, 0x00000000000205d1}}, + {{0x000c0f09863d151a, 0x0004b4a6226f970c, 0x0004a88f8872d167, 0x0002e60a1193cac8, 0x000543dda270b44e, 0x000d647382ce6393, 0x0006751a9e8a2138, 0x000000000009d843}, {0x000abb28a3b6891b, 0x0008a98a1222e3ef, 0x000341bb8ccbdd0d, 0x0005be5555088026, 0x00017b38f0648047, 0x000e249f5c39ccd9, 0x000b74ea31304de8, 0x000000000004d42d}}, + {{0x000330dc4e7217de, 0x0009c39a689bbd9a, 0x00001ce7a86200c3, 0x000108a8d29457b6, 0x000014ed4b4dcf33, 0x00015625f312612e, 0x00063bcbb21f3451, 0x00000000000303a3}, {0x0009f7756d9dff06, 0x00004aeb0c8c0639, 0x000d964999e9958d, 0x0002604683b37dcb, 0x0007862b08477a02, 0x000d69390fa4ced7, 0x0003145a49bdc136, 0x00000000000c0215}}, + {{0x000155abc265f191, 0x000f0544874b6521, 0x0007a036ebfb6d29, 0x000afd631411ba1e, 0x0000466d6415b303, 0x000d4c2de30047aa, 0x0007b13a1b676594, 0x000000000002860c}, {0x0008a20d9745d768, 0x00046fe140a72d14, 0x0007d03c948ac2bf, 0x0006df9f46f144f8, 0x000bc0defbc46c9f, 0x00024c075f7e7b95, 0x000c39fbc9a96978, 0x00000000000d7773}}, + {{0x000f0ec4d363b0b7, 0x000bdd2db4d34f56, 0x0005740dc154bacd, 0x00065cc723a57c02, 0x0007d8ded62c4475, 0x0002f1a9ed98c359, 0x0003b20aeac6d9de, 0x000000000000a98d}, {0x0004a99c9e88ef97, 0x00034d9708f06642, 0x0001bd570d037e70, 0x00032cf49cda0113, 0x000a858467e24993, 0x000748e8e546df74, 0x0000738b84a55093, 0x0000000000006f09}}, + }, + { + /* digit=89 [{1,2,3,..,}]*([2^356]*G) */ + {{0x000f37fe6b03f897, 0x00052c0a40cabe70, 0x00091f1b94fec55b, 0x000898c340187426, 0x000636d9e57a8625, 0x0008aa21169c9f3f, 0x00016869d9d7f337, 0x00000000000e7dca}, {0x000214154225907a, 0x000ba24d49aebb77, 0x000fbddce2036f3c, 0x0001d01576f533f5, 0x00099ef82ece667c, 0x00007c14d372eee1, 0x0007f6577723c0c2, 0x000000000000eed3}}, + {{0x000850ebff25b818, 0x00013c61db976bce, 0x000a1c9cf36381eb, 0x000b8ca060b1adcb, 0x000188e2c178ce89, 0x0003bdd3c41db448, 0x00042cba6339f392, 0x00000000000d29db}, {0x000bbd1437baf649, 0x000f53521116eaf1, 0x0005d6c8cce9ea0c, 0x000c984bd79fe5d8, 0x000e9d45eee49357, 0x00088e01f2eda73b, 0x0008731a50c59f62, 0x0000000000075018}}, + {{0x00038b1bc1ced6c1, 0x0004df323953ab33, 0x000a2512b2fa2401, 0x000dfd9bd32cb8d7, 0x00037ecf35382937, 0x00039941507c4e56, 0x000a06b573960eb6, 0x00000000000dd1a4}, {0x000a33b92253abc1, 0x00050c625f400562, 0x000bec6e4c3f3a23, 0x0001e5b6220f24df, 0x000827d01c6f66c5, 0x00012372d9f97d75, 0x00004860b1572404, 0x0000000000004d55}}, + {{0x000a3b570044f6d8, 0x0005fc552c6c6093, 0x000f9e99da8c0559, 0x000375c19610b0cd, 0x000bb051dad5c9ed, 0x000556643f0e7b4e, 0x0001d87267a304c6, 0x00000000000c96dc}, {0x000cd649696f60dc, 0x000dbf0ed5f9a8b8, 0x00051f0009075842, 0x00066f4af5a4b4c2, 0x000d20644cf2ef5d, 0x000ae23c00b5bed3, 0x000b66f7e4543a75, 0x0000000000041325}}, + {{0x0006798502ee1353, 0x0006c04ef7ad5a7b, 0x000c6878548d78d6, 0x0008a6a47591d88b, 0x00046902edb49ae3, 0x0002b143a27d9125, 0x000df65dae3d8381, 0x0000000000093a2f}, {0x00091ef1ecb7e486, 0x0000807a00388858, 0x000fb3b7ca4398f1, 0x000c17172b1bcba8, 0x00032d0033c73fd6, 0x00016c28aa8d70f9, 0x000ad79ea9eea329, 0x00000000000423ed}}, + {{0x000a6b031af68d65, 0x000b59949b33d112, 0x00063134f5d066ef, 0x00071e788e17f2bf, 0x0002da9c088188ce, 0x000d8c9e9d851baf, 0x0005b6e869c5ba86, 0x00000000000a0142}, {0x0009ab9cc7c38dc7, 0x000eceae8b1c5d3f, 0x000b0c8e79b3cc01, 0x0002910c374bb97b, 0x0007054874831494, 0x000bc6ee13e13c45, 0x00047be0e6fad435, 0x00000000000b54d2}}, + {{0x000e68e47f9ea217, 0x0003c9c85e50a3a1, 0x000b0543d8520966, 0x000ce81807f33dba, 0x000a3db81e06cb78, 0x000638d709d337d8, 0x000babe223eae472, 0x000000000006bf2e}, {0x0006763908bcca20, 0x000662804b59c92a, 0x000900c614fc9957, 0x000ca1e7bc6949c1, 0x00008f051155321e, 0x000539f40bf2906c, 0x000808b802a3289c, 0x00000000000073cd}}, + {{0x000fc3dc90267bca, 0x000956d34bdf61b1, 0x000bd7a38090ba35, 0x00051135a3bfdd10, 0x0005f1296bdd61d0, 0x0002e9a7c4abab57, 0x00003d72da68494d, 0x000000000008d11f}, {0x000319b6e5281755, 0x0002247811121597, 0x000526a929004138, 0x0003ec67521898f7, 0x000996c1da75ec29, 0x000f2c7b3f0cf026, 0x000af8b0dbd4c380, 0x0000000000034e4b}}, + }, + { + /* digit=90 [{1,2,3,..,}]*([2^360]*G) */ + {{0x000c4e9ab57885e4, 0x000f3c32730e8279, 0x00033a83277668a3, 0x000fdbf3f5cd8478, 0x00067272d4dd2bac, 0x000b7577645f3641, 0x000be2dd813a795e, 0x0000000000050997}, {0x0005df7d4526b8ac, 0x00040da054e2b966, 0x00068890ccefdc5e, 0x0008f31026116a38, 0x0000dcd85e914c42, 0x0003adc2d372af9a, 0x0009e6fda6e90367, 0x00000000000a8db5}}, + {{0x00029349f348fe26, 0x0004251a033a4db8, 0x000e0f2d8a80c6e4, 0x000ce49a42a266f1, 0x000b82c4d11e92ee, 0x000bae08a87e037d, 0x00078c875be1416f, 0x00000000000c5394}, {0x000064a55c345b79, 0x0001651f8b907a36, 0x0000a11c59759db6, 0x000bd666c51b9c32, 0x0004ee565de82c74, 0x00082c8c3635d3d2, 0x00093c6bd6566389, 0x00000000000daf6a}}, + {{0x0007ac96ab52d7dc, 0x000dd68fe359d36e, 0x000f3dea99f83698, 0x00052c3fc704935e, 0x000952e4e22d0ec6, 0x0003e3ada1eeff1f, 0x000871ba6777cb08, 0x000000000008befc}, {0x0007324fc4458b0a, 0x000e34ede689e853, 0x000cf3cdc3eb9d08, 0x00080517ab83c288, 0x00052c8c1f48e0f0, 0x000241aac1f3d5f2, 0x0007e057991607af, 0x00000000000b8a33}}, + {{0x000ca2047d1f8f40, 0x00004179c59cea3e, 0x000880cb97b10e4b, 0x0005cfdba0cf3857, 0x000b3a8fbafadda7, 0x000384e0b94081a5, 0x000a45b91de90a33, 0x0000000000027aa2}, {0x00058a31cd3d8717, 0x000220747bcc094c, 0x0000191551c0619d, 0x000e3e9722484f1a, 0x000da62d1dda9c1c, 0x000963bce13a7ee5, 0x00021b77fd79343a, 0x0000000000083d2f}}, + {{0x0009d92935034280, 0x0008b3dbb3fa61cd, 0x000a3f5ecd8961d8, 0x0006574793ce8041, 0x000783dbce4f35ac, 0x0003cf6b0b2697dc, 0x000164e35c5bf2e1, 0x00000000000b0c4a}, {0x0008df1996e7f4a8, 0x00038aa49090842d, 0x0003b655f6623523, 0x000e96c54d504e4c, 0x0001b73310a3f646, 0x000bde5dad74e754, 0x000ef7ead7159618, 0x00000000000aa09a}}, + {{0x0005ac9c05f620d2, 0x000df609deb16279, 0x000a25447b2b8795, 0x0003132b378305ce, 0x000869275f5e4a9f, 0x000b089b5f101799, 0x0000c514be746761, 0x000000000000c81b}, {0x000e2dd3d7fa135b, 0x00030ca90a9bf94b, 0x0006097de4edabc4, 0x000ac9620c71989d, 0x000a390aedd01b25, 0x000d8cd39b971b61, 0x00011a995214d779, 0x0000000000065c6e}}, + {{0x0003855cea5ad9ec, 0x000d4a8393a23731, 0x0006e8588fe37236, 0x0004e0255a202691, 0x0002a446bcc6d882, 0x00051499b9dd9a27, 0x000535c929cded53, 0x00000000000cfe76}, {0x000bc0f428a70f0d, 0x0009626f3d7d9b38, 0x000a6acd906c92a5, 0x000cc06759e6ddb7, 0x0004e302b89191fd, 0x00007e1f7ac2e190, 0x000b4aad25c645c2, 0x00000000000ab3de}}, + {{0x000338669b2cb8c7, 0x00071a13f19b3741, 0x00031ac1789cc21d, 0x0000c997044a6f4f, 0x0008dca1075c00ec, 0x00020ab0caf5665f, 0x000effded5ca3f06, 0x00000000000a896d}, {0x000378285debab1d, 0x000a5032ab2b2a9f, 0x000438bbbc4fc5ea, 0x000b6f725133304f, 0x0001977a45672286, 0x000abae4f0d16d53, 0x00003b00a66036f1, 0x0000000000095f01}}, + }, + { + /* digit=91 [{1,2,3,..,}]*([2^364]*G) */ + {{0x000925dc8a89e47c, 0x00066f2cdae310b9, 0x0002591b317d8ec6, 0x000f25ad750b29bd, 0x0005dba15f3e9d14, 0x0009fe2dd931b9c6, 0x00043334db169ebf, 0x0000000000052663}, {0x000c577396a26bac, 0x000968859a4f76bc, 0x0005596f973fb63a, 0x0008d6a67ac4db88, 0x0002be25d7557463, 0x000db10ee5c8ea7b, 0x000cd38a0f0a8738, 0x00000000000e890b}}, + {{0x000bf56a7de9f2ff, 0x00022883533aee6b, 0x0008916bc1e75ddf, 0x0005fb546d0fcf42, 0x000a4ab05da78f0b, 0x0007b66147e6b0a0, 0x000cf2cd91a869c6, 0x000000000006c6f7}, {0x000593da575680e9, 0x000d7c072a1067a9, 0x0006a216ef99f7a9, 0x000192c8f0ce9469, 0x000fd11494417378, 0x000ccf45adf2e7b9, 0x0005684fa17cb614, 0x0000000000045f03}}, + {{0x000725d8721dd338, 0x000930f4ca77fc15, 0x000b6681d170fa2e, 0x000625b4ef805afc, 0x000fc015c5864873, 0x0001a8c93bddba22, 0x000f427091b50aa4, 0x000000000004c005}, {0x00044f31df11a4ec, 0x0009ff43d3745415, 0x00007d532af636b7, 0x000a10c82770690b, 0x000b77513f003efc, 0x000c3af9bd0c25d7, 0x000adac0015cc12d, 0x0000000000094c6c}}, + {{0x000e8ca221c6ffc6, 0x0008d8b70e94f6d7, 0x000aee261a6327ec, 0x00019ba4d3ef22f4, 0x000a1999f948e222, 0x000fe73027f8a712, 0x000af025575e96ab, 0x00000000000564a1}, {0x000b25cff3d9db0b, 0x000b1045e9c658aa, 0x0003561802f8c969, 0x000b825c3db161be, 0x000aba33a906dd23, 0x0000e41f205fb173, 0x000d1d9a8b5ecb87, 0x00000000000ccc30}}, + {{0x0002aea737bc7de5, 0x000c2bed7d94c6a0, 0x00033bc161f07397, 0x0006af6106834bff, 0x00044c4f74486541, 0x0001dfc06dade8a1, 0x0007bc62227a1d88, 0x000000000008dc2b}, {0x000d41c50320c146, 0x0001c4f1154170fa, 0x0005645ed0c5caa2, 0x00053aba0f1b9617, 0x000d48b0ec8e98cc, 0x00072d0342f68011, 0x000e384bbc34679c, 0x000000000009c576}}, + {{0x000919b377dae55c, 0x000a5d9243926709, 0x000605401369e5f3, 0x000feee0e35f201a, 0x0003196f23f3a1bc, 0x000c1d068d25be5f, 0x00050b298c67f2ff, 0x000000000000c3d9}, {0x000d303412559962, 0x000fa5e15ab9975c, 0x000533c7cf964ba3, 0x0002ec947a7e2696, 0x00022ac22e49a9f8, 0x00006f090f02af9a, 0x000cb0dfb3103fa7, 0x000000000003cf8d}}, + {{0x000298e2791b8ead, 0x000259f5f44f9ed5, 0x000ce416b16e73bb, 0x000f258f9627f9cc, 0x000938fd51bdd7e8, 0x0006a4a7922499dc, 0x000250a9d7497c84, 0x00000000000026ae}, {0x000e3b88b9c7c3db, 0x0009084d47d19214, 0x000f52469be04308, 0x000138806a52e316, 0x00027f08953f9b2f, 0x000fcac083fc8da0, 0x0003fc22d074d292, 0x00000000000701d7}}, + {{0x000b913b1d418177, 0x000cbb7b856256b6, 0x000f3717023b8633, 0x000ddd5c91b8ffd9, 0x000155d38511b4eb, 0x0000da525ef815ae, 0x000bbd98587d551b, 0x0000000000034a9e}, {0x0005ef8ce15a684b, 0x000b8844811fa3d0, 0x000d70ffbd583fff, 0x0008bb4f623789be, 0x000e404980584b26, 0x0002b435ab26ae5b, 0x000eb3b47a5ded3c, 0x000000000008fcec}}, + }, + { + /* digit=92 [{1,2,3,..,}]*([2^368]*G) */ + {{0x00069e40601a7dfa, 0x000cb70765682efa, 0x0003c3be2bc6c542, 0x0005ef0a6db6169b, 0x00012032d5992a93, 0x0006f13160029276, 0x000f51b5babb2d3a, 0x00000000000db26a}, {0x00039e460e4f3b3d, 0x0000200c3401304d, 0x0003ff9e293a0443, 0x000e2ed0f9b36565, 0x000934031d18a1bb, 0x000fe1224e17a5d4, 0x000cf5e3661047f8, 0x00000000000623c6}}, + {{0x000ad46943acea12, 0x000a859367582797, 0x00066c45ce5e5faf, 0x000351af6a27741d, 0x00087f929e0d5d96, 0x0003f1afbeab94db, 0x0001dd865ba01104, 0x000000000005fb63}, {0x00040d1f1e090717, 0x00038192e065294f, 0x0002fb37089941d8, 0x000228db7cae5f66, 0x000d6a828b037bcc, 0x000f301aa02eaecc, 0x000a077c48a2ea91, 0x00000000000f5eb1}}, + {{0x0000bb651b864105, 0x000154dd3bd8408c, 0x0001daf9340de0f6, 0x0006998ba0668966, 0x000cad713bbd1ad4, 0x000ca4fa5b7c679a, 0x000a1cef9389aff7, 0x00000000000d68b6}, {0x0009051030865994, 0x000093acfff601ae, 0x000275b28bea6bfe, 0x0006441c66ad734b, 0x00042fd3eb165e2f, 0x0004e211749f144c, 0x0001a3bd7f22082e, 0x0000000000041791}}, + {{0x0002b76266c69181, 0x0004edf0ae8df2d5, 0x000906801183c79d, 0x00017ca1dd286917, 0x0009709b678ede37, 0x0000acdc60c1db87, 0x00003a0288959a40, 0x000000000005ca2d}, {0x000fd72975fb9eb5, 0x00084e52534b365b, 0x000f103241b3e4c6, 0x00034f873eece315, 0x000d642a67991a9a, 0x000a7e8a80fb0c7b, 0x000238375cc0cfea, 0x00000000000d00ec}}, + {{0x0005a38c81bac32b, 0x000bdb67d88fb498, 0x000506df3f19aea7, 0x000433ede7910791, 0x000a20ec085cc26e, 0x0000eefb30ed8fc0, 0x0008c22684a901b3, 0x00000000000e855e}, {0x0003801582a535c0, 0x000a23d31d6c91c3, 0x00014e8637446682, 0x000380e755f1af17, 0x000a6c985ecfd938, 0x00065084e6a8e434, 0x000ec653bec9c1cb, 0x0000000000070309}}, + {{0x000a18c1c3a6f1f6, 0x000f49a2461c2364, 0x0005b210149e5bc7, 0x000bfea0ae15fd0a, 0x00027ac4d98f2265, 0x000831902d817b35, 0x0001fe9788aa4511, 0x00000000000d4616}, {0x0004b65395df8642, 0x0003d2a7cac2f471, 0x0003af0e90c12844, 0x00048d211070e2f3, 0x000ab769926571a6, 0x000aea1556050d8b, 0x000411cf8f24a040, 0x000000000003522b}}, + {{0x000a6d728de1a709, 0x0002e45d4ed63b83, 0x000803bc165cd24c, 0x00029022b267f598, 0x000fc8446afe76be, 0x0004d6f50791c22a, 0x00077ed6ce8b8859, 0x00000000000e3f1f}, {0x0000a258fb32c514, 0x0006d1a72b16f0c7, 0x0009dc6ac4083dba, 0x000743e1cad2e785, 0x00041e0e640d6a1c, 0x00074648529ff0a5, 0x0001fb4cd519be4b, 0x000000000004584f}}, + {{0x00051eecfb6439ff, 0x000619ca8cc9cbbc, 0x0007f10adb0b792f, 0x0005d53059b2bbb3, 0x0000730e5dc37211, 0x0001d89988fe7ac3, 0x0007a54e67ef6984, 0x00000000000c8ed3}, {0x00004a045edf3ac0, 0x0005e48164b3dc80, 0x0006b3cd6d953b04, 0x0004643beed38ce9, 0x000fa3e30b3db7fd, 0x000ddd484993cfa9, 0x00075ddcc5e7af01, 0x0000000000068401}}, + }, + { + /* digit=93 [{1,2,3,..,}]*([2^372]*G) */ + {{0x0007c9efed4b7222, 0x000ab79768e9e648, 0x000c0eb72891fdba, 0x000821bbe0c67d8d, 0x000a6909fdfcc194, 0x000ccdda7537a6a4, 0x00026cae6d705195, 0x0000000000092b39}, {0x0004e2be194fc116, 0x000ab2c5dd6af51d, 0x000bd2ad4ffd8821, 0x0007388397a3bd3a, 0x000a8baf59c2dd3c, 0x0000ec9589d176c6, 0x000315d5c6219e38, 0x0000000000054e8e}}, + {{0x00045f27ccc6232a, 0x000560683349d797, 0x0004a272721dca0a, 0x000181050663b9f6, 0x00099733dac647c7, 0x0002137c5f327d14, 0x0001c703f55e4a29, 0x00000000000da59e}, {0x000b58d5c8ca43e3, 0x000c7baa8b4dce4f, 0x000e015119c73eca, 0x0008c1db0322dfa0, 0x0007fd69954f99a8, 0x000a29889394440a, 0x00064a9060473da5, 0x000000000002f048}}, + {{0x000df3b38c2a0a79, 0x0000190762c1ecbf, 0x000c4019f47729e5, 0x000849f5c47c645a, 0x00056d72e7487420, 0x000e11773b082319, 0x00050eaeac683b8e, 0x00000000000fb5c5}, {0x000722990680a985, 0x0003e6d986d69658, 0x000da73ac5c10442, 0x000136fd94d52235, 0x0009ed5b01c6d13e, 0x0008fcdcc1aa7541, 0x0005ee7c2014b140, 0x00000000000cffde}}, + {{0x0009c28bf944531a, 0x000bbc7d0abb1f4a, 0x000c2054201bd117, 0x000defca80767df6, 0x000871820908eb67, 0x00074d19a6fa174d, 0x00081a3215df4856, 0x00000000000960a0}, {0x000602066fc6cc5c, 0x0006993ed71807f6, 0x0002dfba1c3883c8, 0x000d56efe787e592, 0x00098fe8b96f0909, 0x000ab68115ae74fc, 0x000e0c8fc342c435, 0x000000000000b991}}, + {{0x00047f2399bf97b6, 0x00021c600dd569b8, 0x000917d84fb94235, 0x0000a27de43c1d70, 0x000e616bdb162d7b, 0x00029795321bf3a9, 0x0008b233ce767c96, 0x000000000009e439}, {0x000800714c3c1216, 0x000934069cced410, 0x000f1cffed0b3d13, 0x000bf89c53942369, 0x0009acb24e389981, 0x000da473507d9c07, 0x000cd9d2e5e904a2, 0x000000000002f1b6}}, + {{0x000ee38595e6d6e3, 0x0003bfeecc6e7a5f, 0x00087ab2bdd62b28, 0x0004e20e81879cac, 0x00089c4a27124845, 0x0000749c3d0ea9cb, 0x000f93ba8b6f3354, 0x00000000000e971d}, {0x0002b66fa9a77846, 0x000b64c2a1b129de, 0x0005071339da6099, 0x0000b53435739873, 0x000a57040c00e6c6, 0x00091a7b4b9f56a6, 0x0002466de6489f62, 0x0000000000053bc3}}, + {{0x000554330abff5cf, 0x000ae828a9dbe5c5, 0x000263a5672326a0, 0x000428089faa5816, 0x00002dba2a1d1558, 0x000197ae62da0922, 0x000abba225f631df, 0x000000000008f4eb}, {0x0005dcc2ee38818c, 0x0001bbbadc9b2cb5, 0x000ab544fb33427b, 0x0006cc2759413f76, 0x000121dd1222072a, 0x000ac2c737b39ea4, 0x000296d6baf9e196, 0x0000000000075d46}}, + {{0x000b2448cce42a48, 0x0001555671128874, 0x0000a30717d307ff, 0x000207ce99ca16c1, 0x00057c7a26b97e21, 0x000c3ab00785fa10, 0x00059f33c28658c2, 0x00000000000d79cd}, {0x0004e804722ed471, 0x0006f9a674db4469, 0x000c9ef186db33b8, 0x000722166bf2c6d5, 0x000abd4c3791990e, 0x000696ee1ba29aff, 0x0006a5d98510cf20, 0x00000000000f93d2}}, + }, + { + /* digit=94 [{1,2,3,..,}]*([2^376]*G) */ + {{0x000bbb4af21bdc49, 0x00065e7cf5babca3, 0x0003f75a29c95b64, 0x000531d04fa84740, 0x00045780efda9c1a, 0x00062cb5a399e848, 0x000f3b6e9c12be4f, 0x0000000000094a5d}, {0x000aeeb34b3f3b9c, 0x0006d3c15fe11c69, 0x0009e05f576f256d, 0x0006da4be294c912, 0x000aa99b83983ef7, 0x0001b9f6a5cf21d5, 0x0007bf50679cf875, 0x0000000000054d0b}}, + {{0x0009b2dd4ee443f9, 0x000fcc84f9171b4d, 0x000ea8b580b8da51, 0x000753ba05068eb1, 0x0004077777efcdc4, 0x0004cf44aa177ad1, 0x00044e0b7f54eb7e, 0x0000000000018601}, {0x000becbcecff7dd7, 0x0002b1279ed4c55b, 0x000e10b5af306b99, 0x00058fab8cdb6cf3, 0x0004a0c7614aebc9, 0x0002c6d8ebc4f93f, 0x00038ac5bd6cde7b, 0x000000000008d65f}}, + {{0x0000f0922779f009, 0x000bb4da7d63a943, 0x00025a4f402efdea, 0x000001668841d1ef, 0x0005b10366f61e8c, 0x000b121e90688f48, 0x000c1e38ee34b005, 0x0000000000014e0d}, {0x000a8eaa6c54d224, 0x000e683c1d44e6fa, 0x000050a1a6ad5a9a, 0x000274453715349d, 0x000dc5ca18f66b73, 0x000ef86e35065bdc, 0x000f5d677313cf2d, 0x0000000000033ebf}}, + {{0x000a3669bab3aa8c, 0x0001fffec66743ba, 0x000af6b2298cee7c, 0x0001365ee61bc95f, 0x000e8948b6e23181, 0x0001644be358bbd6, 0x0007a71cd9c342e3, 0x0000000000060a8a}, {0x000569b8813054fe, 0x0000ae065519f224, 0x000901053fc569ee, 0x000c03b8517ae8b7, 0x000173750d6e8957, 0x000aee6d7c96a040, 0x0001144eb2e364b2, 0x00000000000ed099}}, + {{0x0006635997173f0f, 0x0003b71714896339, 0x00025444ec94256f, 0x0005f33fe56d0905, 0x000638a1b2efb078, 0x0000fc15add43e0f, 0x000d362df35cfb36, 0x0000000000090b3e}, {0x000659873829fac7, 0x0000bd4970bdd7ef, 0x000e315a3dccca14, 0x0008a3e4a43c4f3e, 0x000fd9cb4808c99e, 0x000c3d0948013609, 0x000223f3fef0eaec, 0x000000000008642d}}, + {{0x0005a811304eaea0, 0x00018ffe3104b37f, 0x000808bc57214341, 0x000ed42cca2f1506, 0x0002c421c69415aa, 0x000b87b368618aa0, 0x000d5f4d6a091dfa, 0x000000000000e786}, {0x0009341357580c61, 0x000ee0be97746d4b, 0x000cd6467b049086, 0x000823b7a70d4224, 0x000809de860f6094, 0x000cf60205928791, 0x00086e7f7fca496d, 0x000000000005f460}}, + {{0x0007768861c39ce8, 0x00007b509f7028f0, 0x000fc09189281be1, 0x000832a7f07f3291, 0x000d8fbdcb83853d, 0x000a4650d074fdec, 0x00072080ed8de9d6, 0x00000000000a5d68}, {0x00079ea50cb0600f, 0x0009ae66666d224c, 0x0008a45c66bc5d53, 0x00099510da620fbe, 0x000fdf866f777e2d, 0x000045beb901145f, 0x000008442a37e5db, 0x00000000000beb1b}}, + {{0x00048a2dbd606668, 0x000c9bb39a179bed, 0x0009439710e05c24, 0x000abe3b91f14833, 0x000fbd39a2bc6be6, 0x0001d813ef972804, 0x000d4a06737e54f9, 0x000000000001a87c}, {0x000d0c4b5d0bc923, 0x000b2bf88b2e5982, 0x00052c33a491bb9f, 0x000aedfa58af0431, 0x0006ac0511b07a93, 0x0007e2c198853cdf, 0x000d8a9d7f416d06, 0x00000000000f9671}}, + }, + { + /* digit=95 [{1,2,3,..,}]*([2^380]*G) */ + {{0x000f6957a24c76d1, 0x000222e7d0a157df, 0x0001697f0073308f, 0x000ddc63eb317f9f, 0x00015adf71d715f1, 0x000858bc3f027507, 0x00071a2ce33c2eee, 0x00000000000da73b}, {0x00035cb76cad67cc, 0x000de9ef6a709ffc, 0x000c7c7ed1727cfd, 0x000f5a67502de655, 0x0001a47019739f5b, 0x000ea7b24d11122a, 0x0002e182cfaac2a2, 0x000000000000a458}}, + {{0x0007ab134ebd334b, 0x0005196a1e74f032, 0x00014e69e2323c83, 0x0009c4fbe35109cc, 0x00004521d7e61244, 0x000d002931fad8b3, 0x000229c85eb23d57, 0x0000000000035f68}, {0x0000d113aed4dbde, 0x0000c7c5b86677f2, 0x000189b64d0338ba, 0x0000456757cd5717, 0x000e3a1c866b067b, 0x000ba88c0eaab81b, 0x0007dc72a6af75bb, 0x000000000000ef56}}, + {{0x000ac045af51c5bb, 0x00010597a26ff058, 0x00059bcfafa87a4b, 0x000aff65c27f5f6d, 0x0005d8d544e60b06, 0x000275c32ce5ab66, 0x000e7c92f031be99, 0x00000000000f42e0}, {0x000c8a905adb28dd, 0x000175bb28b05c35, 0x00031ae347df4e7d, 0x000d299f93fb7dcb, 0x00086e2ccad9e73f, 0x0004d4f57fcd999a, 0x000c9bb8c8a6df33, 0x000000000009a2ac}}, + {{0x00030bea7fcf0ecc, 0x0006a3afdee26fd5, 0x000cc01988c78a70, 0x0008ee3ba4a33026, 0x000a2a883b7f340c, 0x000d7412b1a6ea51, 0x000b6e64f27976d5, 0x0000000000091251}, {0x0008bb3d89325d7a, 0x000d25f3f8f40ad2, 0x000216e9116c139b, 0x00073d6ad61b2cbc, 0x0005d542676dde3b, 0x0007d712bf08398d, 0x000023d373931e8e, 0x000000000007f96e}}, + {{0x0004e4e1b942add8, 0x000e15adf3d9957d, 0x0005ba50de0860ba, 0x000ce33a82d3736d, 0x000e718c8aa3f9fb, 0x0006ccf69f307823, 0x00065b860578cf41, 0x000000000001ef84}, {0x0009d5f6cc7a9ceb, 0x000939ee0cf97011, 0x0002f3cbdb7c8fc5, 0x0009a8dc09da90cb, 0x000dcbf3ccb8f99b, 0x000a626321f521c2, 0x0009da6bf6694891, 0x00000000000854fe}}, + {{0x0000917c5016c853, 0x00049e44e3f85fb2, 0x0000e1793eed8bac, 0x000254501c4e171e, 0x000cce52defb1004, 0x00063100ac12df68, 0x00035fb1fae2fbf3, 0x0000000000035732}, {0x0007edc8c1da40d3, 0x000c58d4deb655bb, 0x000bfb14f6d9d28f, 0x00085835a4e118f4, 0x0001308772e93249, 0x0004e48765abfa96, 0x0007b241c7611ff5, 0x000000000001f586}}, + {{0x000782cfada548c3, 0x0001e463031709b7, 0x0005afd4d0d5166c, 0x00097fff172c7d05, 0x000735b193cfd8e4, 0x0009df4bc7f7009d, 0x000e376b9fd3f7f1, 0x00000000000b46f1}, {0x0004c800fedd3a70, 0x0000987c6eaa8c8c, 0x000dff8047ad562a, 0x0001042539eab96e, 0x000779b8f5cc2a12, 0x0007840dc29a6d5d, 0x00030f33803a10b7, 0x0000000000011a6a}}, + {{0x0000457a60ad991e, 0x0005e3c90a59f250, 0x0005eeda9345d42a, 0x000885c1a0d58e29, 0x000d6816b0099aca, 0x000e4718f77b9b6d, 0x000c458e5c12139e, 0x000000000004e4ea}, {0x0005f9c85e9bfc5c, 0x00086c00c3568eed, 0x000b5b4b0eba61f9, 0x000ffc75eaf3eab0, 0x000d42a87e32cba1, 0x0007a882f5f99092, 0x0005122e7b49c76f, 0x0000000000029040}}, + }, + { + /* digit=96 [{1,2,3,..,}]*([2^384]*G) */ + {{0x000dbfc10083fe9c, 0x000bfcf17f9dc084, 0x0005e1e364b063e9, 0x00054ffe437b29cb, 0x0009e27ee9e2a694, 0x0003af03b6628d78, 0x000d6256e3b975ee, 0x000000000002f532}, {0x000e1d6aa4b057a7, 0x00043cae15213418, 0x000003dc9b2b9ddc, 0x000a959fa4d82608, 0x00055ae17566902b, 0x000c82eb4bad700d, 0x000ea716b2c5dc14, 0x0000000000036708}}, + {{0x000e41b4a9c3409a, 0x000849cbd62cd8bb, 0x00047e8c2d4de38c, 0x000bbe98f886eca3, 0x00049b432990b7f3, 0x00030035684860e8, 0x00029ed19a3bafa4, 0x000000000000e209}, {0x000ed539231869f6, 0x00098f15ff660294, 0x00050e66a84ebdc7, 0x000de8d2955612f1, 0x0006ec97a4a53ec6, 0x0000d15fe2d95b4e, 0x00024d868731f0f9, 0x00000000000aefae}}, + {{0x000253f48e5fab58, 0x0006889f97859df6, 0x000dee3f7bba228a, 0x0001d4ea62a5c827, 0x000ea3045da6826e, 0x000d3d237ecba3e4, 0x000358beed1da058, 0x00000000000b8b41}, {0x000293271a9d4293, 0x000d2730e7ac94c3, 0x000f438703e7096b, 0x0004cace8c1c6462, 0x0001055d39de9ba9, 0x000b71db3ec7c382, 0x000a11c89705a82f, 0x00000000000781b6}}, + {{0x000606caa5686427, 0x000cdc1951fb0886, 0x0000daff401d6319, 0x00009bb2754f1d38, 0x000e0e0e2bf19831, 0x0005a3da40f7db33, 0x000197348c4e1937, 0x00000000000840a6}, {0x0008d7f45b2f9769, 0x000d11687a735d22, 0x000434d1e1dee1ff, 0x000773c0aa9e79d1, 0x0009f56de9654a25, 0x000dda7af656f6e2, 0x00063fdf84666df0, 0x000000000003f9e4}}, + {{0x000a9c9951735edd, 0x0006afe9f90d3ad9, 0x000f97a0b45a07d7, 0x0007d6ec36e3b706, 0x000f580c8e6dd513, 0x000bab45c3ea55f4, 0x000d5819398b33e1, 0x00000000000aee76}, {0x000d9c115c469d59, 0x00071e72c8214ce4, 0x0006b3ef7ac6abe3, 0x0001ac198f9553cb, 0x000a1c98ad467fa5, 0x0000ebbb2019ac4e, 0x0000a68942e16e01, 0x00000000000bed3c}}, + {{0x000f13cdd62e69a0, 0x000461a814bf695a, 0x000e4323f4986584, 0x0001bfa2a4dd16f8, 0x000a76fb9f2b33c9, 0x000b1f1114af4d9c, 0x000b45a23635ef75, 0x00000000000a0891}, {0x000bef0dcdd6b903, 0x0005f5d1b87ba5f6, 0x0005d7ac1d6bdae8, 0x000cb4543d4ef435, 0x00064aa784ea70bd, 0x00070e4220ed12d3, 0x000483009cd34901, 0x00000000000280cf}}, + {{0x000bddf9aa596a1d, 0x0008952c04b79c0e, 0x00099a2770f3fa4a, 0x00044bf1911b3184, 0x0006bb897d318407, 0x000e9de5dd13f080, 0x00052b376dc8b81d, 0x00000000000c996b}, {0x00047f465159b51f, 0x00041d91e47b224a, 0x0009b71ad19e642b, 0x000c167eface7572, 0x0001d4805ed6a441, 0x0003fd6654eb9588, 0x0005778fc93daf3f, 0x00000000000cc570}}, + {{0x0000b16f46a35f8c, 0x00065a630c20c4e5, 0x0001f4362772ed03, 0x000aca10c0dec6cd, 0x000ba9e2f55428c8, 0x000bbb1705d34bb5, 0x000f6e8e81b4f732, 0x000000000008363b}, {0x000ca7950547e910, 0x000969603fe028be, 0x00047954fea1ddef, 0x000bb8efc191d12e, 0x0005dba97347c0da, 0x000656aaaf0e463b, 0x000cf0b7f7c207a8, 0x000000000003f08d}}, + }}; + +#endif /* #if !defined(_DISABLE_ECP_384R1_HARDCODED_BP_TBL_) */ + +IPP_OWN_DEFN(const cpPrecompAP *, gfpec_precom_nistP384r1_radix52_fun, (void)) +{ + static cpPrecompAP t = { + /* w */ 4, + /* select function */ p384r1_select_ap_w4_ifma, + /* precomputed data */ (BNU_CHUNK_T *)ifma_ec_nistp384r1_bp_precomp + }; + return &t; +} + +#endif // (_IPP32E >= _IPP32E_K1) + +#endif /* #ifndef IFMA_ECPRECOMP4_P384_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecprecomp4_p521.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecprecomp4_p521.h new file mode 100644 index 0000000..bc6582f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecprecomp4_p521.h @@ -0,0 +1,1373 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#ifndef IFMA_ECPRECOMP4_P521_H +#define IFMA_ECPRECOMP4_P521_H + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpecstuff.h" + +#include "ifma_defs_p521.h" +#include "ifma_ecpoint_p521.h" + +#define BASE_POINT_WIN_SIZE (4) +#define BASE_POINT_N_ENTRY (1 << ((BASE_POINT_WIN_SIZE)-1)) + +#define OPERAND_BITSIZE (521) +#define LEN52_P521 (NUMBER_OF_DIGITS(OPERAND_BITSIZE, DIGIT_SIZE)) + +/* P521 affine point */ +typedef struct { + BNU_CHUNK_T x[P521R1_NUM_CHUNK][P521R1_LENFE521_52]; + BNU_CHUNK_T y[P521R1_NUM_CHUNK][P521R1_LENFE521_52]; +} P521_POINT_AFFINE_IFMA_MEM; + +extern const __ALIGN64 P521_POINT_AFFINE_IFMA_MEM ifma_ec_nistp521r1_bp_precomp[131][BASE_POINT_N_ENTRY]; + +#if !defined(_DISABLE_ECP_521R1_HARDCODED_BP_TBL_) + +const __ALIGN64 P521_POINT_AFFINE_IFMA_MEM ifma_ec_nistp521r1_bp_precomp[][BASE_POINT_N_ENTRY] = { + {/* digit=0 [{1,2,3,..,}]*([2^0]*G) */ + {{{0x00031a16381adc10,0x000f3f18e172deb3,0x000e0c2b5214dfcb,0x00017fd46f19a459}, {0x000ac947f0ee093d,0x000d50a5af3bf7f3,0x0001457b035a69ed,0x00009c829fda90fc}, {0x00011cada214e324,0x000274e6cf1f65b3,0x0000000000000000,0x0000000000000000}}, {{0x000460e4a5a9e268,0x000f4a3b4fe8b328,0x0004351396120445,0x000fd683b09a9e38}, {0x000132062a85c809,0x00064bf7394caf7a,0x000d7de8b939f331,0x000224abcda2340b}, {0x000163e8deccc7aa,0x000de0022e452fda,0x0000000000000001,0x0000000000000000}}}, + {{{0x00090cf08640909d,0x000f1c99dd36bc1e,0x000b26b07ecb3fa1,0x000ae2d7a0e797d1}, {0x000aa83d508251d1,0x000bd9d9026d377a,0x000372a82ebb4df4,0x0003cd8e66031a96}, {0x000a461413a3a019,0x000f3f3417e59440,0x0000000000000001,0x0000000000000000}}, {{0x0003d2ee331fe1b6,0x0001ab6b30fa0d81,0x0004af6e07a7b8df,0x000d19247a757e5f}, {0x0008fb5c9c9bfd4c,0x000dd9f1bbef4f92,0x00090d14c836216d,0x00083e26d4bba055}, {0x0007769f85ae35a8,0x0009338053f9f677,0x0000000000000001,0x0000000000000000}}}, + {{{0x000e9cf4d4910f78,0x0008ce976f1bd6be,0x0004316197502d2c,0x000acff10dd75a48}, {0x00019028ed35e8b5,0x0008d69f8b251d24,0x000d6bd0896bd46e,0x00072d891ecd5cf2}, {0x0005acaca3cda953,0x00048caeec8eb532,0x0000000000000001,0x0000000000000000}}, {{0x000cfa6c0ee5f7e9,0x000c4650f7436072,0x000de49d2c8212fa,0x000f61e3867882e4}, {0x000bad816ad6768e,0x00061716ea67c6e2,0x0007c558fd1aae77,0x000fd4154e818be9}, {0x000655c0a7978aab,0x00016eeccbcfc363,0x0000000000000001,0x0000000000000000}}}, + {{{0x0008d6d77d92b8ab,0x00043a09438c8179,0x0002d8472d2f17de,0x0003c5783350ea81}, {0x00083a8745c474f8,0x0006432cf1257f1e,0x00062eaaaa0e9e75,0x00048e2ff9cd7e03}, {0x0003e483866e30e4,0x00010261aa5a41a4,0x0000000000000000,0x0000000000000000}}, {{0x000a0825be109849,0x000fa3fe1a372686,0x00078234ce8ecf10,0x0004adc2f75dbfd7}, {0x000c2a029127ba85,0x00093cf941f2a5d1,0x000731ff178cc83f,0x00077b7371970dad}, {0x000585a55db2a90d,0x000ce95b39f00bc7,0x0000000000000001,0x0000000000000000}}}, + {{{0x000194afcf14a49e,0x0005a84b7647983c,0x000f36c498b9c6ad,0x0009bf3cd194ebf0}, {0x0000a11b8897f578,0x00021c1e0636af18,0x00081ed5c78bbd67,0x00077eda9f869267}, {0x000e027585fbd2cb,0x000219639ede19c8,0x0000000000000001,0x0000000000000000}}, {{0x000d6f9bbc6f7598,0x000e61f46f584865,0x00092b9aa7bfc0b9,0x000e7affbce8f803}, {0x00079ba188aa0108,0x0003ddb44be48396,0x000ec0be4d01a384,0x000f4743970028f6}, {0x000a54089488e6c7,0x0000eb764515b988,0x0000000000000000,0x0000000000000000}}}, + {{{0x000fb915a75b36d6,0x00098df6fbc9035c,0x000ab2bf9c05711b,0x000a98df4617b374}, {0x000b9ca70393d11c,0x00092fde650b0a9f,0x0000a8356f25580a,0x00084bbfeb8e79cc}, {0x000a24068cab11e9,0x00090ca9977f9a7c,0x0000000000000000,0x0000000000000000}}, {{0x000f780956b43319,0x000666bc2c6a278b,0x0005aae506d6f0f5,0x00013a79101ee3dc}, {0x000464efcb64c26f,0x000b655b96872b32,0x000205493100d454,0x000db9ed2d404739}, {0x000a371d8889555d,0x0007ac35716e9382,0x0000000000000000,0x0000000000000000}}}, + {{{0x00015b574766756d,0x000756c4140b766a,0x000a87ee130cd00e,0x000e71dde237ca9f}, {0x0004c6c64d36f986,0x000ec61846855fe3,0x0000c69617b88a62,0x0000747aa4191478}, {0x00005839d062f917,0x000fb1a3775b2fed,0x0000000000000001,0x0000000000000000}}, {{0x0008f4b46df66eaa,0x0005c5e48292928d,0x000952eef7e3dae3,0x0008e70d2fcf3b38}, {0x0004f15ca91d1a2c,0x000ab5e87949e6f6,0x000edecc51365ef2,0x0001681412786eb8}, {0x0001ceb423c5ae2c,0x0001508868ec18bd,0x0000000000000001,0x0000000000000000}}}, + {{{0x0000208b103ed8c0,0x000cf52a553c6734,0x000dad37a0202c37,0x00046bcf0d5ab144}, {0x000a4f845acc60de,0x0007adff52dc2bcf,0x000c51d82fc1314c,0x000ec54d801f0545}, {0x0005808712dea714,0x000fb931541a41cb,0x0000000000000000,0x0000000000000000}}, {{0x00058cc64475550b,0x000b21788f8bc00e,0x000a004a389a9c56,0x0002e2bc34cf9dd4}, {0x000da5ff85d06f83,0x0008c4f4e0552c88,0x00041ef30833bd47,0x0006f4f16038ada8}, {0x0003c429dcd227c7,0x000e2410247ed5b7,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=1 [{1,2,3,..,}]*([2^4]*G) */ + {{{0x000f45f44362a272,0x00062847c0d42017,0x0004003ee6e299d3,0x000c2b186f86dfae}, {0x000005072cb2ed3c,0x00094a1ff8c430e5,0x000197c69058c452,0x000b30c97e9f9eeb}, {0x000563a1d859543f,0x0005682eed4bed13,0x0000000000000000,0x0000000000000000}}, {{0x0001e8dcd0f160b5,0x0001397c4de39d5f,0x00069468c3199a97,0x000eccc4294f1802}, {0x000a3d505983ad64,0x0000e0c1709cc6da,0x000e51864a5b5c16,0x000261dc006a8763}, {0x0009e9f34b9099af,0x000b800fe38a58c6,0x0000000000000001,0x0000000000000000}}}, + {{{0x00059ae8d65b4828,0x0008ad5c3b72b54f,0x000ae6b62e3c123c,0x000d2e8fca4e7ba3}, {0x000cb7633eb4deb5,0x000b750251aee39a,0x000d3f677dcf2f5e,0x000b32a703401c6d}, {0x000a5d0ad9a1f1f0,0x000d234a21b83d7c,0x0000000000000001,0x0000000000000000}}, {{0x00011dec3fd34650,0x00054d4c8a50ab55,0x000b2d5cb0b1b4ae,0x0005079b6fe6cc64}, {0x000eb5cdba6f0e3f,0x000ef10266dc0c66,0x000ef80e32e16ebb,0x000c5faff80cb3e0}, {0x000f9f041347fbde,0x00055c088b1af3af,0x0000000000000000,0x0000000000000000}}}, + {{{0x000fecd778ec8555,0x00013eb0956c3cd6,0x0005e90a57516438,0x0007f84773320ba9}, {0x000dfd66f6e10d88,0x000a16d04c079fb7,0x0009e6452c01f397,0x0006b339081619a5}, {0x00076316466573eb,0x000a315640ac72e5,0x0000000000000001,0x0000000000000000}}, {{0x000e34d296b0561e,0x000ac0c9e92b6a35,0x000402420e573d8c,0x000d871e142cb792}, {0x000bf1bdbe0a1707,0x000d7310a6b250ec,0x000894a7b366170d,0x000fe4a684f16643}, {0x00025d9fae0bb793,0x000c96f6d42adcf6,0x0000000000000000,0x0000000000000000}}}, + {{{0x0006ae6a85f73844,0x000ee1c671907ba1,0x000014d027ea0da9,0x0001f3800efad55c}, {0x0001bb9d3e0160b0,0x00038df5e9e2f7ed,0x0000a5ce67c43969,0x000dd40c305dd65b}, {0x000c97f61533f5ed,0x00097668a79ebe01,0x0000000000000000,0x0000000000000000}}, {{0x0009d235c3ae123a,0x0008b82c52fc6fad,0x000edd0329eea2d7,0x0007cac021e0e0d2}, {0x000a5887a53dcf1d,0x0005d60b2dd0a846,0x00031dd1b4f43d5c,0x00064597d8ee0e86}, {0x0002bdcac36cdd8e,0x000902b9d50810be,0x0000000000000000,0x0000000000000000}}}, + {{{0x0002ff15cc9efbda,0x000d0adea1bf15e4,0x000712003c28f70c,0x0009acbb183fe29d}, {0x000e35ff46058b74,0x0000e6fbe9505c1f,0x000949d4ed7c586f,0x0006bc96db22c518}, {0x000e9fa6b4caa767,0x0006b3b723ba4dae,0x0000000000000000,0x0000000000000000}}, {{0x000d4a1dbb01ce30,0x0002733373d1589b,0x000695b9bd79abea,0x00024c417444789b}, {0x000a3e366b3687f5,0x00018eccb8e87edb,0x0007fa4355949d4f,0x0003b1b5225855c5}, {0x00040d600e111ea7,0x0002ca9912224327,0x0000000000000000,0x0000000000000000}}}, + {{{0x0001ccaa77955b13,0x000980cc45066e57,0x00005f6f02b3f9b1,0x000f381f6757639e}, {0x000d14e03775ce84,0x000770c3535ef348,0x0000c573845ff7f4,0x0007edbd50365fb5}, {0x000ae96135b16a31,0x000b1cbda8a1cae0,0x0000000000000000,0x0000000000000000}}, {{0x000fb5af8ac6debc,0x000ccace499e422e,0x000f85f9a34dea5d,0x00099ffe8fc76f8d}, {0x000700f62f621078,0x00048a20afbdb94b,0x000353ffe99ecf56,0x0009d5421253436c}, {0x000d53ffd3fcac92,0x000092a9413337e3,0x0000000000000000,0x0000000000000000}}}, + {{{0x000cc6739bc22034,0x0002d46c578a3cfb,0x00074dd1918f2158,0x000820e4ee32d9c8}, {0x000ee4769c451119,0x000a52dc0b788cd9,0x000ea08ad1dce239,0x000e6c51ce30fbf6}, {0x000c6808b739646f,0x00014612aa8ed807,0x0000000000000000,0x0000000000000000}}, {{0x00084f57ce386096,0x00001bd8dd33a19d,0x0006d6cf7616b3bf,0x000aa0ec8a31fd3f}, {0x0000f33cd7d20cf3,0x000d8c7b490a7ae4,0x0003afd25187398f,0x0001d4b82d520c24}, {0x00008bc63a1c99f6,0x000da20702f937af,0x0000000000000000,0x0000000000000000}}}, + {{{0x0005f3504967c9f8,0x00081372b9b7d4f0,0x00090711a09dabb5,0x00052fc0a79713f3}, {0x0008ebea8efb840b,0x000ba724a4b43b13,0x00089257bae703ec,0x000c5de16b6e9669}, {0x000a9811fd41e4b4,0x000c0fc1b18e0380,0x0000000000000001,0x0000000000000000}}, {{0x0009bdb977d41aa0,0x00084a0964efd898,0x000049a3954725e1,0x0002567309871a1a}, {0x000d6462734e1923,0x0004d24ffa586e4c,0x000a7d5e1d8d7ce5,0x000f69f3efbcb30d}, {0x0007de3d3416e700,0x00041ee729987fef,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=2 [{1,2,3,..,}]*([2^8]*G) */ + {{{0x000757d0283c0ad6,0x000745c834df0056,0x000e5caf285c0d9c,0x000faea391f23599}, {0x000232d4e48a9620,0x0009bdc7a7b74615,0x0002fd4f47934e59,0x000c4f65ada3d4dc}, {0x0000798974c81e39,0x00061064a2c57e3c,0x0000000000000001,0x0000000000000000}}, {{0x000a38d40ea9cd04,0x00060922a435ef3f,0x000a826a53bc247d,0x000e94d33d8a1866}, {0x000de75ac695cced,0x000bcb2e7b2b9c71,0x00016e1d52b9aa78,0x000540f2da2b1a83}, {0x000881db4e2a0769,0x000e217e4c0ddd49,0x0000000000000001,0x0000000000000000}}}, + {{{0x0008396a04a1a9b5,0x00071e1d4dea3fc9,0x000f1b1b60428524,0x000875279e270a42}, {0x00031e46c1327bff,0x000c05c823fe0222,0x0001988e4c1b07ef,0x000346e86dbfa458}, {0x000ea14d7c3803e0,0x0002698c2f4163f3,0x0000000000000001,0x0000000000000000}}, {{0x0004df73f1c64b3b,0x000bae3f77cba047,0x0009fac52f482f0e,0x00046303eabe2a5c}, {0x000605a86c7774d0,0x0006157561d8716f,0x0006dae76cfe4cf1,0x0006f10528e0564b}, {0x0008d8ad69113bb2,0x000c2f933ccc8b87,0x0000000000000000,0x0000000000000000}}}, + {{{0x000963cd234bdeab,0x0005b961fb152043,0x000192d80595fa4f,0x00021e095276439a}, {0x000ab3cdd8200acb,0x000b977cb97a7564,0x000f1ac5e95df534,0x000082ed91f8da63}, {0x0005eeb0ff149253,0x000e11676ac3e35f,0x0000000000000001,0x0000000000000000}}, {{0x0001cccbb8a0782c,0x0008ef764987a4d5,0x000621e7649cd3df,0x0006ca705ffe9912}, {0x000d887d63f6769b,0x000a2f7ad40e5963,0x00074e3b9fc2e426,0x000bd597aa3dacd3}, {0x0000fea916df09cd,0x000c4db4a1b5ffeb,0x0000000000000000,0x0000000000000000}}}, + {{{0x0006312ffff6fab5,0x0006b5b394c36d7c,0x000ae9f8123d8c52,0x0007a4616b7fb3e1}, {0x000a92d9f22f9728,0x00095d4a0fd21b31,0x0002d23d7cbfded8,0x000b5c10574881ff}, {0x000e2bd04e830bd0,0x000fd69dfeb7774f,0x0000000000000000,0x0000000000000000}}, {{0x000b243fcf68f023,0x00066b7e7441cd83,0x0005c91b009a23e1,0x000f85c785f70865}, {0x0002122e7768c122,0x000fb751856db403,0x0001836d6df94b82,0x000098df3edc80b3}, {0x000298e9aeea7ce8,0x0006e6048ecb9605,0x0000000000000001,0x0000000000000000}}}, + {{{0x000cad0b80b053c4,0x000883158e365644,0x0001acd6f66175a4,0x00078eba00f9062c}, {0x0000f302446bdc8b,0x0004ffcc5c874a3b,0x0001cc9e542baabd,0x000ca8d66f56fe73}, {0x000fec6e656e27fc,0x0000e12576ad29e3,0x0000000000000000,0x0000000000000000}}, {{0x000f2114ad6a5008,0x000ff37b5e9ff077,0x000fe609c9e85ddc,0x000612f68cb97937}, {0x00035cc97418d8cb,0x000e6da8506bf5e6,0x0001561e4122e23c,0x00026ba1b08bd010}, {0x0007eb4a708cbd94,0x0000c9d309deaf41,0x0000000000000001,0x0000000000000000}}}, + {{{0x0001bba765fa9bcf,0x0003babdc21b3ff6,0x00025b55117dbb2e,0x000cd85081bffb4d}, {0x00081cb8cbfa61b2,0x000244b920db891a,0x000db06e819df932,0x000532ca257bcc77}, {0x000ade96bb81f698,0x000552846b3d2e5e,0x0000000000000000,0x0000000000000000}}, {{0x000cc4965acbbc52,0x00019fc3317e41d4,0x000a394d289bf4a3,0x000dfcc926b6b451}, {0x00099fdc4795a3bb,0x000fbdeb87e6005a,0x000e9db4adbdc0b3,0x00050e680b14c0a6}, {0x000a1f2811642efe,0x000396ef97cb540f,0x0000000000000000,0x0000000000000000}}}, + {{{0x0005f3ef0a7bd1f0,0x000acae8e898fb4a,0x000ba953fee10eac,0x000ac34b13e74541}, {0x000d80a4f317458b,0x000699b6bc22b700,0x0009d659b3752d47,0x000972beec5644da}, {0x00015e14a99a228f,0x0008731fa64678ce,0x0000000000000000,0x0000000000000000}}, {{0x00076f9ee24a5ff4,0x0001761a3d49abfd,0x000099dc50257cde,0x000182905bef244d}, {0x000f586d9a90e6d1,0x0007ebb4cb4857b6,0x00070d79d0a32ad0,0x0004bfe6c4dca5d6}, {0x0009e03cd5b7b143,0x00028c04a0538f51,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c6fc14a74584c,0x000402292021e48e,0x0009500ecd0a9680,0x00002339ed719b16}, {0x000ebb81e8a19412,0x00040e8e4db85440,0x0002a313f6a53c2d,0x00082796c5c684a1}, {0x000636765497c008,0x00020c751837b791,0x0000000000000000,0x0000000000000000}}, {{0x000740897b89a933,0x000f39feb6c7cfd4,0x0006675504305fd0,0x000708d724da0165}, {0x0003fcde5846c915,0x000cbcc847c7bb1c,0x00035875d5c58a40,0x000f531dd999d009}, {0x000ff3f98178ab52,0x000a7c4485d31888,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=3 [{1,2,3,..,}]*([2^12]*G) */ + {{{0x000bc092fe2354e8,0x000850a2f27fd64d,0x0002ad51407fff03,0x000808402ffc14aa}, {0x0007fbe516b67c4b,0x000f027098449910,0x0009af3715688b40,0x000dbddce7795e2c}, {0x0008a5dc626ec8f7,0x00032acc9e1305cc,0x0000000000000001,0x0000000000000000}}, {{0x00014a5956e30ed0,0x000921ce664e13cd,0x000b7485d5a678ff,0x0001d65fed6fe685}, {0x000152b7d0453dd6,0x0001e48dc7a066d9,0x00012560c3395f08,0x000d6053e587c1cb}, {0x00076afca630f2cd,0x000814f0d70553c7,0x0000000000000001,0x0000000000000000}}}, + {{{0x000a19c3686b1f78,0x0009545540d3da61,0x000fed5fc9dde90b,0x0009be8908cbe546}, {0x0002b931292ec657,0x000e0b221531c8bf,0x0003dcf64709233d,0x0006a91e2913f0e3}, {0x0003880d89929920,0x000d07ab37b02493,0x0000000000000000,0x0000000000000000}}, {{0x000b1d587081c0df,0x00062b5f29f3ee6e,0x00013755e246f468,0x000c2e51e2652ae3}, {0x00046ba6a65e2952,0x000fd1b792013b94,0x00049175e7bffb43,0x000166af7dd896a1}, {0x0003d0d5f68a4101,0x0003a34ff29cf955,0x0000000000000000,0x0000000000000000}}}, + {{{0x000b9e5f2e5de279,0x0006be9e4c557ee8,0x000c510883da6331,0x00016eedfba63955}, {0x000b55eba66cb586,0x000d93dd071f901b,0x0000d11e4c33f467,0x00029c2288bdd752}, {0x000f22d4f3c9b728,0x000416f979cce9a3,0x0000000000000000,0x0000000000000000}}, {{0x000f91fa66b3408c,0x0009041780ab3969,0x0001e17f9e99f2b3,0x0002825a0408a22e}, {0x00013e814b39af10,0x00017c70c14077db,0x000fd91116e8d047,0x00025157bba11642}, {0x0003d53fd072760c,0x000130f596860d22,0x0000000000000001,0x0000000000000000}}}, + {{{0x000ddddf5d1e5c64,0x000b39631d236577,0x0005fc5e812c4b6d,0x000ec807d1cccab0}, {0x0002c8729f1a1c38,0x000999e4061629e9,0x00088f56b4c00d1c,0x000c3cac8f29781d}, {0x000b02141cce3380,0x000920c7e0e0cc16,0x0000000000000001,0x0000000000000000}}, {{0x000234580d88382b,0x000b0ad02da7d076,0x000cc82cf5ae2d27,0x0008a15c3adad7f2}, {0x0004d7009305d2c0,0x000e9e632a55fa7b,0x00011560b55b693d,0x0008b565732e2a82}, {0x000f0adb63788cf9,0x00038e2d1f605489,0x0000000000000001,0x0000000000000000}}}, + {{{0x00007e33611831c8,0x00062ce163c826e6,0x0001c1873d3e07b5,0x000df7813a79a532}, {0x0007451a6bebf65d,0x000789f014abc3f1,0x000cc2aa01512853,0x000bc25e7cee2985}, {0x000dd32f61a13543,0x000f9b061a57a2a5,0x0000000000000000,0x0000000000000000}}, {{0x0003734e520ae3c4,0x00041544fe045295,0x000321b2f9da98d0,0x0004ac066f5c4118}, {0x0004f9ed8d4d338c,0x000c2ea3577c3f4a,0x000a775eee973782,0x000efa3a3176188c}, {0x000aa0ec53f7823f,0x0009b227eec20996,0x0000000000000000,0x0000000000000000}}}, + {{{0x000cab486125bb6b,0x000ce02028378ff0,0x0005e625f7d1c380,0x000dc7cec93bc7c0}, {0x000d0c3b6e1a5a67,0x0001da033baa95ac,0x000e0b126ba3c688,0x0005b900f71e6919}, {0x000cfb69d4bcab68,0x00090d9f859cecf3,0x0000000000000001,0x0000000000000000}}, {{0x00030307c7f8906c,0x0006bf1f14afae02,0x00001d0c8e70d7c3,0x00031fcdd396a945}, {0x00035972de152221,0x0001149c59aa6c9c,0x000e743c53f3251e,0x00072a17186bca64}, {0x000bae52281f6178,0x000c98982b2d35fe,0x0000000000000001,0x0000000000000000}}}, + {{{0x000abaada33bf5eb,0x000334cbe9475074,0x000503921efaf87b,0x0001a3119b05ca55}, {0x000a4fa919940136,0x000a8bd8f158f3c3,0x0002e855587cab4c,0x000fd133003309bc}, {0x000a468f055eab49,0x0008de65f435935f,0x0000000000000001,0x0000000000000000}}, {{0x0005a479ed62ca37,0x00046fb6c0237009,0x000c6558be89daa3,0x000e86cd9c606b6e}, {0x000dcb5426f2b48f,0x0002744bfa702fd9,0x000e9ceab7372571,0x000cd8bef4768a91}, {0x000d361ea2d4f5d1,0x0000ca847b5f94d5,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c6a7b65b21bde,0x000fc723a29c73b0,0x000392643c39c3ea,0x0000b213f81be3c4}, {0x000e3ec734fa388c,0x000b26d37a33b98a,0x000332e230742689,0x000e28354ec1687a}, {0x0000d4b7e6935b64,0x000ba79d55aecff6,0x0000000000000000,0x0000000000000000}}, {{0x000073362910afa1,0x0007fb8bcd336bd6,0x000b6c7a7845b5f6,0x000017305633e845}, {0x00076a907be72df6,0x000e65734d2814a5,0x000f113c7084b86f,0x000cd7bad9f20758}, {0x000f6af2a5030c22,0x0001647ff1cabc3e,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=4 [{1,2,3,..,}]*([2^16]*G) */ + {{{0x00084cce9eb37269,0x000406ac65525f61,0x000c9acc4f25051a,0x0007bdd2651c4a44}, {0x00059571fa6bdb63,0x000cf1489d2ae9ce,0x000a821f56bdf324,0x0000e5fa827f61b0}, {0x00046a2449dcea62,0x000c947027c9ed4b,0x0000000000000001,0x0000000000000000}}, {{0x00095f1c50d4d450,0x0002c227a410cd04,0x000bc9ba135ee643,0x0004257073536858}, {0x0000b7e39c350531,0x00016eeb65d0616e,0x000e949a694a0693,0x00069aba0dc455bb}, {0x000d36d721f9d7b7,0x000a041dcb7a1d32,0x0000000000000000,0x0000000000000000}}}, + {{{0x00030d330fc15e51,0x0006a88312448f9b,0x00027c12fd1499ca,0x000b765eaf5a132e}, {0x0008d01b2d2a5c3f,0x000e3517c807951a,0x000936a97c68ed6c,0x000f8cdd161ce67d}, {0x0005d9876ad5eb28,0x0009976496ac4a79,0x0000000000000001,0x0000000000000000}}, {{0x000d912524de7c0e,0x0001e66e4dff627f,0x000a96a9194e4460,0x000ccae884a673b1}, {0x0005d06054966f81,0x00032269452eba8c,0x000ba7677e70b535,0x000298891e5c17de}, {0x000f9a70e2fe55a9,0x0004d48b39032dcc,0x0000000000000001,0x0000000000000000}}}, + {{{0x0009dc9a72f4ff50,0x000df54e86b3f74b,0x000b7fc672cd56a4,0x000ac313c91daa4c}, {0x00053d8b04fac047,0x000047ffb771df8b,0x000a8ad48cf7c44d,0x0008bf663542e196}, {0x000aa68b0ea4fed6,0x000483dbd49e0b45,0x0000000000000001,0x0000000000000000}}, {{0x0007d603e389e5cb,0x000ee233664de2d7,0x000994f96855ef7d,0x000c5bf8c8ab10b1}, {0x000c2f5ab3d235e3,0x000bff37afff2ae5,0x00050de9d0fd0f4d,0x000ca6d91d5250db}, {0x00042da0be2c950f,0x0001c70ec3836fa7,0x0000000000000000,0x0000000000000000}}}, + {{{0x0009b222a53c1578,0x000733b1bb114a22,0x000887f6c13ff59f,0x000d5dfb2679cded}, {0x0001fd35dec8bbba,0x0000930770ea94d4,0x0007da8d4f0a6019,0x000d2142901c2ad0}, {0x0002aaa8648f142e,0x000f42252e455969,0x0000000000000001,0x0000000000000000}}, {{0x0004f335e4753950,0x00010578c42f0d9b,0x000fda89975c2716,0x000761372c49b195}, {0x000583ac76051357,0x000cd0c4d54de0d0,0x000c35f47ffa549f,0x000731f21817e11b}, {0x000ac2b103f57a56,0x000284cde0cd7146,0x0000000000000001,0x0000000000000000}}}, + {{{0x000bb2a3bcba0504,0x00052359d22ba169,0x000ee4d727c18bc1,0x000338aababfd9ca}, {0x000cae35505124c8,0x000599b6e8a9cc3c,0x0003c6415386807e,0x00043919da2fc5ab}, {0x0001a4c6fd2ee43d,0x0007be38ead93480,0x0000000000000000,0x0000000000000000}}, {{0x0008c66b564a97d4,0x0002177834d44e8b,0x000690ef30774807,0x0007151d926feb1c}, {0x0003fbe2f1f3454c,0x00048ce8e6456bd0,0x000270c04a6964dc,0x000fe8febbc7afec}, {0x0000f159a483b3a5,0x000aca96cb139ad3,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c0e9763d8a013,0x000baa65d7b1d37f,0x000608a4b87c8c06,0x0008c2592e527b8c}, {0x000aacc19bb3aa2d,0x000ce5b0adb09308,0x000e0f42458761d4,0x0002d73d4f707a6e}, {0x0003867f8d791c44,0x000c943ba7a1a60d,0x0000000000000001,0x0000000000000000}}, {{0x0007ffca3e51b076,0x000d23467af3d90e,0x0009427b9fa60c44,0x00054ce0e4a16358}, {0x0001655e4129aaff,0x000befd5ea275c28,0x0000ce27c03c7fcc,0x0000c97ca421b716}, {0x00069ee6f84bb35f,0x0007ec35e0436eea,0x0000000000000000,0x0000000000000000}}}, + {{{0x00033b8c99430421,0x000e3aa65723117d,0x0001482e2ca3fcee,0x0006dfdb52560262}, {0x00036a105a9eb6d9,0x000c0fd8b7bdc41e,0x000c58ba2f2edd58,0x000c050043d8b271}, {0x0009966a34a51907,0x000aee0fa52e13a7,0x0000000000000000,0x0000000000000000}}, {{0x000c2d7066d5fc91,0x0000d462accbe2da,0x0008397028d0b78e,0x000b525e2c9d107f}, {0x0003dfedd5666711,0x00083957250c9620,0x0006d0f2be094638,0x00026dd96c8ff985}, {0x00098fe829c7a670,0x000dacfc430b6d43,0x0000000000000000,0x0000000000000000}}}, + {{{0x000aba8bae3f0048,0x0006fc5f421abd9f,0x00094ac402ce8227,0x0005bead91f2efc8}, {0x000d28241f32e7d5,0x0008bce170cc1090,0x00050cb19f59df3e,0x00034ac35c2de273}, {0x0003cf90c3e6cfc4,0x0002a784bc2847d1,0x0000000000000001,0x0000000000000000}}, {{0x0003f87f754f1aa3,0x0003382713cbe9fd,0x00034163c334fd8d,0x0004cbe346cada61}, {0x000dd6aa94a54721,0x0007b9235830a042,0x000500be120acf2f,0x000d30c3e8d009be}, {0x000225e2751dc7f0,0x000714b7edd06e6f,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=5 [{1,2,3,..,}]*([2^20]*G) */ + {{{0x000c1256fe47d13c,0x0007012fd1181020,0x000c4a469315aa78,0x000b1163ea26a86c}, {0x000e4be00b905056,0x0002f1dad4a0ac68,0x000e2d8c19c57695,0x0000bbc11daec6fd}, {0x0003baf9c6293f81,0x000d375056fba03a,0x0000000000000000,0x0000000000000000}}, {{0x00073f08bbfc9af7,0x0006c14cc716b559,0x000b5b613b18efce,0x000f005d64d3ad94}, {0x00034ba83b800248,0x0009ee4cf2a375eb,0x0007d29413af2a4c,0x000525ea872268a2}, {0x000c082bd8d12fde,0x0006fa2d233189c9,0x0000000000000000,0x0000000000000000}}}, + {{{0x000f1bef2c1b123b,0x000c9ca73fb5cd85,0x0001a80d76a111a8,0x00025f888d3b7461}, {0x0003f7765b87f2e3,0x0002e36012c8ad9e,0x0009dc42c7cf6c49,0x000d7d5db366bf5a}, {0x0005359f96228a81,0x000772725123cd91,0x0000000000000001,0x0000000000000000}}, {{0x0006c7a0e2cfcba5,0x00097fa38cc5da8b,0x000b43bb38eee14f,0x000f15c0770c4afd}, {0x00093138850f3aa0,0x000658cf7e3953b9,0x0007c8bb70f07792,0x0000d78fd38c1d44}, {0x00023ebe4681177a,0x000c9d704ca7518e,0x0000000000000001,0x0000000000000000}}}, + {{{0x0008fa7e4527a9d3,0x0004db4e9fda74ba,0x000404855f433494,0x000f130f65201753}, {0x000d719a9846d31d,0x000c651ab9661cb9,0x000b653c04c2995b,0x00031b2c3fb591c2}, {0x0000b91d21b65fb3,0x000c1f9233b624c9,0x0000000000000001,0x0000000000000000}}, {{0x000eac108061f9eb,0x000a2e86d9cc5afc,0x0000e2ec8f94cdd0,0x00040675309b7d38}, {0x000320d2223f6f2c,0x0003be480dc1e34e,0x0007b72b364f62ba,0x00063595753dec52}, {0x000283d90f6639f0,0x0001d567ed0c354e,0x0000000000000001,0x0000000000000000}}}, + {{{0x00067e2e3d478324,0x000b9d2b33e93756,0x0005cc9c7d0711cf,0x000aa7c2edf0adb9}, {0x000b6610b704fc5a,0x000107368e770150,0x000cc4ef9af2a471,0x000afe1e566d06e6}, {0x000a67146814dd0c,0x000fd36c67f6637c,0x0000000000000000,0x0000000000000000}}, {{0x000b744b33ca6a46,0x000a2ad960d19dec,0x00099ff41dbc0bcf,0x000977ca933b28a6}, {0x000a7951faf63b97,0x0005168f23ca3752,0x00097d901e0f16b1,0x000105f55f964ea3}, {0x0003c0d403b374a5,0x000d43e408ed3a81,0x0000000000000001,0x0000000000000000}}}, + {{{0x0007586c50a55f86,0x00026583c230d093,0x0009c8f1eaf61062,0x0009876910419f67}, {0x0006e8d67dbad2c6,0x000c3c184d440783,0x000753899fd2f814,0x00037825fefa52a3}, {0x000f0758545a721e,0x000498a4b823d5ff,0x0000000000000000,0x0000000000000000}}, {{0x000e376ebd4ed258,0x00004d6a23fbe496,0x000b69ec3505f765,0x0008fe6b545afc26}, {0x0000e87ed2073fb2,0x0006145047af95f2,0x00053f84d27cd1ba,0x000fa35d865dc4cc}, {0x0007b711a9ee96b7,0x0008ec430aefdeb0,0x0000000000000000,0x0000000000000000}}}, + {{{0x000354ba169146af,0x0008c79fdb88cac7,0x0009f85e2efdc64a,0x0001012d7f3a69d0}, {0x00017d2bed232563,0x0004dfd89cfd4d1f,0x0008288e64d46be0,0x00089f8bf20fd559}, {0x0001d08641f269d1,0x0009fc333e29ffc1,0x0000000000000001,0x0000000000000000}}, {{0x000c7dab1b832059,0x000223bbef8e949c,0x0007f10fed75c714,0x000647b0bb61d266}, {0x000b8e823dbf309c,0x000601c5a1f58db2,0x0009c023a71fa3e4,0x000a0b5cbdd6344f}, {0x000df6a6577b11f1,0x00027e6eb12db5f8,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b94e9f2c64c2a,0x000dff9c4cc3460d,0x000339e03c0646b9,0x000ca76c7ae26f18}, {0x000612ba1712f64d,0x0006950e5f2c8040,0x000569eb5bf0fae1,0x0006185858b613d1}, {0x00080b1245b35ba8,0x000d9a3c93740668,0x0000000000000000,0x0000000000000000}}, {{0x0005f1c44e1c9646,0x00061096f83044ee,0x000e69176fe10924,0x000a78875cfb2614}, {0x00007825516a8324,0x000065d69cfbad90,0x0001f873d71727bc,0x000185c81c530662}, {0x000c1471ae21856f,0x00047e68582e4e3a,0x0000000000000001,0x0000000000000000}}}, + {{{0x000ce1c67b68f07c,0x000dfa5469124c9a,0x000ccd6db2024f3d,0x0002fc6faadd52b4}, {0x0005124af076574a,0x000510591517b271,0x0000081118a106bb,0x0007cffda2d67e24}, {0x000b3b39fc6925ec,0x00012037b374e288,0x0000000000000000,0x0000000000000000}}, {{0x000d91b81541ec51,0x000c413a683e17ef,0x000952a60eda72b7,0x000b61d80495c130}, {0x000f6bf06a5749c4,0x000c7cbd39872c4b,0x0001a82e01cb7ce0,0x0001726d7547989f}, {0x000742de944906b4,0x00009f2e02ff3752,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=6 [{1,2,3,..,}]*([2^24]*G) */ + {{{0x0000872f1cba7983,0x000a9c28a369cd44,0x00007ce63b6f5daf,0x0009c12dc12c54cd}, {0x000a00a67eec84f7,0x000bc7a4edee7a34,0x000ea82ff4e7f63d,0x000950ab492940f7}, {0x000953027dc3353f,0x000be21b37a3c6cd,0x0000000000000001,0x0000000000000000}}, {{0x0005f758ab77d6e7,0x000bab416ce18f4c,0x000b9e45e70a1adf,0x00038074a53a6ae6}, {0x0005919bbc5eaf8e,0x0006580a40639d33,0x00033f83c3446f86,0x0009f20b5de7abe3}, {0x000a42c48703c063,0x000ed659dbbfadd4,0x0000000000000000,0x0000000000000000}}}, + {{{0x00025b49cc7d0744,0x000d5ca66a4107d0,0x00009a278704a0b9,0x00027e26383562d7}, {0x000c28bfb3e1fb8b,0x0006132452cd156b,0x000d9a249ca82ac0,0x0005a22d3a0e92d3}, {0x000b19aafd34655b,0x00048d36ff20dea2,0x0000000000000000,0x0000000000000000}}, {{0x000f6b3025835a5b,0x0008531143727c86,0x000c5c003858192a,0x0001da9352b2fd4b}, {0x000d25e79d62bdaf,0x0003c74903de7c27,0x000ad4718fc47302,0x00063e48286ed6b3}, {0x0005e2dbee1fb246,0x000857558feb16de,0x0000000000000001,0x0000000000000000}}}, + {{{0x0009f56e96a543bc,0x000d31c6bb399ae7,0x00031b99e11a37bd,0x0006c0e42fdb7dca}, {0x00074aad0723ad0f,0x000aae97e4f8e5ac,0x0000fdee33ad0efe,0x000675078a78e14a}, {0x000909b38b03996b,0x0007469ca609f2ac,0x0000000000000000,0x0000000000000000}}, {{0x000466dcc50132b4,0x000eddd95df5a1fb,0x000a8d9a82cb91aa,0x000178685d056d78}, {0x000f44580cbb5024,0x000b8404125b7ea5,0x000c959397a21279,0x000b3c397b77ec41}, {0x00050358651ee34e,0x000febe48525c5d2,0x0000000000000001,0x0000000000000000}}}, + {{{0x0003aa997f7f37af,0x000bd0399f83a0f3,0x000f152344eac6ee,0x0003cc5d1e36ba58}, {0x00058bcb7b01bb23,0x000e6e01c3b95ccb,0x0003f3cbd70b3470,0x0005471f142928b7}, {0x000f33fe8192a8fa,0x00067751b82a4272,0x0000000000000000,0x0000000000000000}}, {{0x000ea1f801a1be23,0x0000e029d67e2091,0x000355df0fe9219d,0x000069ec86128187}, {0x000e9196ceb57b88,0x000b831fc0c5d6f3,0x000b18561a90e72c,0x0006875cf51476cc}, {0x0009d0fdc775daca,0x000556f11fb0d65e,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d755a99415514,0x00085952e4dc0b72,0x000ddaa98f7198b9,0x0002cf0597f96150}, {0x000e7060ed10994b,0x0005df6128c96445,0x000ea5cf35637a95,0x000b47cf25cfbeee}, {0x0003e924da7f8ade,0x000d5d59754f0973,0x0000000000000000,0x0000000000000000}}, {{0x0007794536974a33,0x000bee39625b4887,0x000c916eff917626,0x000c143d7b2fe99a}, {0x0002b42a1f214f15,0x000a92c52323d0ac,0x000fb97a09db1674,0x00097d2bc99da798}, {0x000062e5a6c7b18b,0x000498e32a3a4e9d,0x0000000000000000,0x0000000000000000}}}, + {{{0x0006f0528c6a99de,0x0005f78365a5404f,0x000a6130da7d607f,0x000ce8f385d457a1}, {0x000127a71db8c282,0x00075b1780f22e8e,0x0005f271ed981d16,0x0000879b71615445}, {0x0003f2395d4c8d2d,0x00002ffbb885eae5,0x0000000000000001,0x0000000000000000}}, {{0x0004e8a699830814,0x000f60c1a29f03dd,0x000620f1843d15d6,0x0005370351aa6497}, {0x0003f3bdc3d85b92,0x0005713630b15aa8,0x000b25038bafd76d,0x000aa324c45046f5}, {0x0008ff854c7ef0e3,0x000f873074cd9a42,0x0000000000000001,0x0000000000000000}}}, + {{{0x000a5c1655f0c965,0x000ad8760b75d0c0,0x0002d3b7a7dc8ea4,0x000d9d864941a528}, {0x000eb47e8ca867c6,0x0001a8847066dfb5,0x000249722cfbbbd6,0x000d270c27ad4dec}, {0x000e74307c896550,0x000812fcfddc8f52,0x0000000000000001,0x0000000000000000}}, {{0x0009df50d9240b18,0x000a5431583e94ff,0x00044e5e649470bb,0x000156c0cde5a76d}, {0x000ad9f50c93b508,0x00099e86567b7ad7,0x000e7618f9dede32,0x000b84978d6f1f46}, {0x000aba70858d56a1,0x000626f5aa274be1,0x0000000000000001,0x0000000000000000}}}, + {{{0x00038150eda6a189,0x000d5498edde21db,0x00010e8c4a802c3e,0x000d570892441512}, {0x000a6f9b009f95d3,0x000c80e954785d69,0x0008dbfa197859ea,0x000f9a954ad6166d}, {0x0005360545cac81c,0x00042d01aaa83bd5,0x0000000000000001,0x0000000000000000}}, {{0x0008a0342c362ba4,0x0008202cf6246957,0x0008c4987d0b8dd8,0x000becfec2b48e0f}, {0x000a0d5c621a63b1,0x000668f2b6a90343,0x0008bd4fe098700f,0x0008bd8fa8829c08}, {0x0007948513f0936a,0x000733ae92e72067,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=7 [{1,2,3,..,}]*([2^28]*G) */ + {{{0x000f8517a4c96e5e,0x000b0e039e54c124,0x0001ec83d522549e,0x0008ead8eb02627a}, {0x000ef95771e9618a,0x0004f3a6b4497c2c,0x00055a41c4917c7d,0x000be8babd80080b}, {0x000548c7990585b7,0x000c64dabe54a06c,0x0000000000000001,0x0000000000000000}}, {{0x000269bac978d382,0x000c6bdf9b98b02b,0x0000247e075c4ed8,0x00030716d8e1a299}, {0x0003a019c94edbc8,0x000c1665fb3458f3,0x000e700cd4d82fa9,0x000a2326e507ff63}, {0x000452b8341c9bbf,0x0000dfc9214a3f5b,0x0000000000000001,0x0000000000000000}}}, + {{{0x0001109faf34910d,0x00006e16fb098733,0x000f23db2247dc5b,0x000f53ef62c76a0d}, {0x00018f68431633f2,0x0003cbe6a9413972,0x000253b6d05ffc26,0x00047ed103a8b14a}, {0x00089d6938ee0b45,0x000f4f3310d7c6f3,0x0000000000000001,0x0000000000000000}}, {{0x000ae5ea5e61f0fe,0x000803eba63d9ee3,0x000ff5d7d44281cc,0x0006da525a289454}, {0x000688b5bf3e7fa2,0x0006c91ab88b9a08,0x0009907565a2877f,0x0001737c021210a3}, {0x00031db551223d6a,0x00041419f277aae1,0x0000000000000000,0x0000000000000000}}}, + {{{0x000ab41a098c92a7,0x00072fa6811333e6,0x000851f03fd8fe3b,0x0008e34493163c43}, {0x00028174f0009cb6,0x000db83d4e177695,0x0004d1bfc734499f,0x000c56272647cc55}, {0x000d38b2b770d722,0x000fcea5c023f6ae,0x0000000000000000,0x0000000000000000}}, {{0x000d52df57e5ca52,0x00065ec98ce0bb77,0x0002140495f37162,0x000d03a5ffc6d632}, {0x000f0d0061a4f743,0x000af61ada3e0094,0x0004b70091a92992,0x0003fe83591d84a0}, {0x0008b7e8b2e59f37,0x0006892e459b7561,0x0000000000000001,0x0000000000000000}}}, + {{{0x0005e28d077485e2,0x00020e97a1a0b220,0x0006679b5d8662c1,0x0000da8bd256806d}, {0x000925177882d56f,0x000fd447d9b3a0c2,0x000077e15eb4fc81,0x0000f15d98b7d44b}, {0x00021ac9944ac5bb,0x00096438b1a53927,0x0000000000000001,0x0000000000000000}}, {{0x00015ec8db3161c2,0x000f397c19309926,0x000de0771a46014e,0x00075e027929bb3e}, {0x0009dcd3fad002d9,0x0008c634c3d387bd,0x0009c3f4ebf24793,0x000a2c2da2b50807}, {0x000e85151e2b2209,0x0005f4bb80ad2ed2,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d84a6577d1d88,0x000209b195d9875f,0x0007dc25315ab09e,0x00041e59650e28f1}, {0x000f81b07fd77f04,0x000191d3288a8ccb,0x0001838964e3273f,0x00014aa2bd5786f8}, {0x000d924c29c83190,0x000d240eca2dec17,0x0000000000000001,0x0000000000000000}}, {{0x000c94cf500620a8,0x00017a01ff5bfc59,0x0001d4396ff8b7cb,0x0004fb6ee69226c5}, {0x000565facfed55e4,0x000fa0b30de43edc,0x000b20ea8612b643,0x00056d93e66dab8d}, {0x00098f9a9e46bfc5,0x000537cf26d7b09b,0x0000000000000001,0x0000000000000000}}}, + {{{0x000ab3bbacf312ed,0x00003114377687ca,0x000ecbe360e1fdea,0x0000336e644349d4}, {0x000ce9701c97498e,0x000b970f76e36640,0x000d18aeb034f110,0x000d8c04737b96e3}, {0x000b828ff89d3a4a,0x000ceb091b279c08,0x0000000000000000,0x0000000000000000}}, {{0x00002832632c0b79,0x000f06083c30b6b6,0x000d6d46a5784fec,0x00018a1a822c2ef2}, {0x000ffd16846c810e,0x000859c7f32abfb3,0x000410206687e5a6,0x0001a4731948bcab}, {0x0009310e425eab64,0x0006640ebb114d39,0x0000000000000000,0x0000000000000000}}}, + {{{0x000759039c7a9d17,0x000c2d4dd47b8758,0x000c1f2cd883e5fd,0x0007f9cf12cfd22e}, {0x000b330c8103c8b6,0x000fef902907b0c0,0x0008540db26c476e,0x000e726a64472792}, {0x000b090ef761ac1d,0x000f96107e57dd18,0x0000000000000001,0x0000000000000000}}, {{0x0004650aeff40f19,0x000f9022a822976b,0x000a5007e542efc8,0x0007e6336d263df3}, {0x000d3e6b563a86e0,0x000fc14e677c3dd9,0x0001920cf4b81cd1,0x0001cedaf8056af9}, {0x0006ebfcbc28b238,0x000af30ce3975f89,0x0000000000000001,0x0000000000000000}}}, + {{{0x00090c95c3d0b9c7,0x000844b8f686b48f,0x000a746779417d91,0x000bd2f6b498e1e2}, {0x0002adf2e61e9354,0x0000db08d1aea1ed,0x000a497f347f08db,0x000c7871378402cb}, {0x000753c735de0850,0x0001f9ed5b707951,0x0000000000000000,0x0000000000000000}}, {{0x000d045e45f819b7,0x00024591e83186d4,0x0002f07b1add4180,0x00067074059bab68}, {0x000ea2b0f371c1d4,0x0000e5f219b0e391,0x000f844f22e68c28,0x000679b7f960b058}, {0x0002502036ca9899,0x00028a1f63acdbd1,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=8 [{1,2,3,..,}]*([2^32]*G) */ + {{{0x00023b6569acb75b,0x0008bff580463d22,0x000bd3d896f7a55f,0x000bc35819d6f3c8}, {0x000fcf8dfdd32dc1,0x0007477a57a2786c,0x000a3b0ae5589b65,0x000806de5eae9f47}, {0x000782fcd93f2178,0x00096dd9479ee32c,0x0000000000000001,0x0000000000000000}}, {{0x00029435cbeebc5d,0x000d8352a35de209,0x0004ce809af5b11d,0x000a76cf080ab8c3}, {0x0002250478b1ce25,0x00003b3ff7216bfb,0x0004133d87f47621,0x000a4132748099b3}, {0x00029b68995f627a,0x0006fb62b3cf30a7,0x0000000000000000,0x0000000000000000}}}, + {{{0x0009267d43d108b4,0x000b5b0396f4d947,0x000b1e5f7f088692,0x0009c3c1fd9437e3}, {0x0006cb9b9b7fa950,0x000a37b6c115a75e,0x00062d05b42f650b,0x000d5b3cf510dbbc}, {0x000a49bf06d5fe62,0x000049824e6593c6,0x0000000000000000,0x0000000000000000}}, {{0x0007f70f36b1fb53,0x0004a3eb94fc53cc,0x00082c0e60f6a2b3,0x0002888aea4bc9d4}, {0x000d37ab9ed318f4,0x000a505ae0c0fe3e,0x00082c9e94fad3b1,0x000efa0f31285453}, {0x000bd4be69ae4fb8,0x000003a516fe8844,0x0000000000000001,0x0000000000000000}}}, + {{{0x0005e8088591d8c2,0x000c6e24bb4a7ab3,0x000968fd3709e47e,0x000d55e3e7d95582}, {0x000f8783f2538474,0x0003b454e19d35c0,0x000724c4578fbe58,0x000b98f51326114b}, {0x00090d99b97ee546,0x000691801229b8f1,0x0000000000000000,0x0000000000000000}}, {{0x000e4a0a2aa86765,0x000aa08b19b4b67e,0x000c220cb11d4f5f,0x000c7d426ad0862b}, {0x0002dd7a63508ce1,0x000aae47773236a7,0x000cce936cda4ace,0x0000808ba7a0be3f}, {0x000c14a5b972c384,0x000887cda4655c79,0x0000000000000001,0x0000000000000000}}}, + {{{0x0007786aa39096d4,0x000a4070017bccec,0x000926b1ae026576,0x000026855e810d17}, {0x00039def24d306c0,0x0008ac55c7d826c4,0x000b521e40f66cc7,0x000ae26697c9fbf9}, {0x000a75caf01aaaf8,0x0002a9d094aec346,0x0000000000000000,0x0000000000000000}}, {{0x000a161f4c1b67b7,0x000e72cc3c7e5f99,0x000fefa749b88977,0x00063737d6e24cf4}, {0x000ed4607a910568,0x00017686854dcfe4,0x00011acac7bc09c0,0x000cc731394d9ec2}, {0x0009e4dd8ac76909,0x000bc7246fb612d5,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d5c2ad970ea88,0x0000eb7dbc22b4be,0x00070d534e1a9e33,0x0004b0eab21921c0}, {0x00071b3a90f4b6c3,0x00015ce8387fd7ac,0x000ec28efad61dad,0x000f792985d577d5}, {0x0009ac313d175e05,0x0002d86582a5f995,0x0000000000000000,0x0000000000000000}}, {{0x000258f797f0af53,0x0006198f53a67607,0x0008e97661c802c3,0x000001f410140442}, {0x0003c270534a9c94,0x000a5333d3d840fe,0x00076ed16f99ddeb,0x00021b94ff80e532}, {0x0002761460189f70,0x00007bce48c909c6,0x0000000000000001,0x0000000000000000}}}, + {{{0x00018e1640843ac0,0x000d435c11720f14,0x00057cebd3881c75,0x00023104876137be}, {0x000da25ae87bc97f,0x000052fc640f68c5,0x00027372b9f14882,0x000b4e854fcb3017}, {0x0009d3406b0f1a39,0x000e653db1df6886,0x0000000000000001,0x0000000000000000}}, {{0x000c416fe50a1e7a,0x000c29b7a99148a5,0x000d660bcfe2e7c6,0x000630309213716a}, {0x0004b63e38c0d1fe,0x00033bb86d3923a4,0x00056213e592fda9,0x00080b360f884315}, {0x000db3b21dcd9fe7,0x000617bc0a7836eb,0x0000000000000000,0x0000000000000000}}}, + {{{0x00035cd63c7c9de3,0x000a91ffca828c02,0x000b3e377bfdd504,0x000496682e625999}, {0x000b8af1c462d519,0x0009570d96676ec9,0x000bf815f3869cb1,0x000b476a8b4fba76}, {0x00057597ce654eda,0x000e2235375a2127,0x0000000000000001,0x0000000000000000}}, {{0x000e9252adfeeb9b,0x000707b233b4a650,0x000ab158faa771d2,0x000df78fbc7a8536}, {0x000be88e5009a864,0x000abbc52635e311,0x0008769bd53cfa60,0x000305e36a566f93}, {0x0000e758e66271c0,0x000b10e12e9cdd0e,0x0000000000000000,0x0000000000000000}}}, + {{{0x00092f38031c9da3,0x000f6f0dd54d0455,0x000298fe24144d05,0x0006274ae2d191b2}, {0x000909113fc1ead4,0x000dea702ccc54c5,0x00008c5763ea8c10,0x000635441e7274f2}, {0x00006b2f2578c573,0x000c3b8134369537,0x0000000000000000,0x0000000000000000}}, {{0x000ada464a85cfcd,0x0006064c2605a588,0x000004bbf31d5464,0x00010805ca04e18d}, {0x00067998cccf9032,0x0008398a6f896278,0x000dacf8d2faed14,0x0002de01ea7eca85}, {0x000eb7d829278474,0x000eb89fe5859b0f,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=9 [{1,2,3,..,}]*([2^36]*G) */ + {{{0x000a73aaf5f01aa9,0x0009abe5e08d9505,0x0009fd8bcd318e91,0x000915895b386f06}, {0x0004a0b5abd07d68,0x00039901b007bc24,0x000ff05c36936913,0x0003179dcef7684c}, {0x000125196e69b799,0x000a2823db7308b0,0x0000000000000001,0x0000000000000000}}, {{0x0008e83616d0d063,0x0003bf0c66e267f8,0x0006d375b822f9f9,0x000dc02109656c0d}, {0x000530c1fc16b6d5,0x0006bf6541ba81db,0x000443cb4caf94b8,0x000a87f59d3c1a6d}, {0x0002afcd60e10fe1,0x000a088d7e1995b7,0x0000000000000000,0x0000000000000000}}}, + {{{0x0005b8daa50517e7,0x000dbed46676160a,0x00016e7a7e78dcdb,0x0008e1fc1b0f5e9c}, {0x000933cb9503e917,0x0006f78b36f2294c,0x0006ec3420442aae,0x000c01e17c7ffb2d}, {0x0007293b87d30fdd,0x00092836bed4739c,0x0000000000000001,0x0000000000000000}}, {{0x00075943e2ebfd04,0x0007b784cf37b849,0x000645eaa02ab285,0x0000b0ace12a0d65}, {0x000ff267c93aa3d3,0x000fa6954230be3e,0x00001e1903aa7ed6,0x000c311ae0c82cdb}, {0x0005f0bd3c5056d4,0x00022094d54a9827,0x0000000000000001,0x0000000000000000}}}, + {{{0x0001c0660a4f2bbd,0x000c3d2fe061f455,0x000a2b6dd7091d40,0x000bf7437cc4883d}, {0x000f258011ddabd2,0x0001bdaba7aee852,0x000195b39f463ce7,0x000efb33b881bd7c}, {0x0007d695aede085f,0x000380f8fa2471bf,0x0000000000000001,0x0000000000000000}}, {{0x0006eeab983554b4,0x000f138a6f5a6dc9,0x000b29f154f07f71,0x0007d1c7242d7a9e}, {0x00042f3c7f1afb70,0x000af2896b44aee8,0x000b55601f569990,0x00061e36ed07c5c5}, {0x0005430facca2678,0x00016ea78eb21424,0x0000000000000000,0x0000000000000000}}}, + {{{0x000aeaaabe68e207,0x00098b6eaee3177d,0x000f5c1267a6e353,0x00092ae2d01bbbbb}, {0x000bf3af1a4f4a2b,0x000532088e5cdfa4,0x000c8ee638a58f55,0x000bbce5e7aa2a0e}, {0x000a1644879c3f50,0x00025e378387c246,0x0000000000000001,0x0000000000000000}}, {{0x00069ee8884e88e1,0x00090f8ec2984b67,0x0001f9e47bd62fea,0x000f9867e7522665}, {0x000cb916cb0493d9,0x000106987f5be2cc,0x000869c71156e1c1,0x0005011576bd7bad}, {0x000b76a6e945007a,0x0000aec18de41f38,0x0000000000000001,0x0000000000000000}}}, + {{{0x0007c59949c6f67a,0x000c897a229d2368,0x000118e3e788aa53,0x000794e3080e973b}, {0x000815a91046f15c,0x000ac04ed530c0f2,0x00078a4fe35eb925,0x0001fe18e7070c54}, {0x000f409ce7d9c57a,0x0008aafcd17edd4b,0x0000000000000001,0x0000000000000000}}, {{0x0002850fe8174557,0x00037e234694753d,0x000c528ec7d2c45c,0x000bdd41f346792a}, {0x00032ca964be2a36,0x000fef3e7911d679,0x0000101c72673a03,0x000653d6358ef0bc}, {0x000f874e2b9caaba,0x00045ebbcc31882b,0x0000000000000001,0x0000000000000000}}}, + {{{0x00090baa505384e0,0x000993148d7d8454,0x000d948464cae379,0x000f46feac4f8ace}, {0x0007268d1b7c8dda,0x0000d4770e6c39b6,0x000c1955967ac571,0x00052098b74830a2}, {0x0008d17fe1904773,0x000191e5071ced85,0x0000000000000000,0x0000000000000000}}, {{0x000b4f003b56551f,0x0000872a7e419344,0x0001262c90a767f8,0x00015a9791770fa4}, {0x0001a4bc5606e253,0x000a665041696dd3,0x000da6d7ec137466,0x000cac07c562bd01}, {0x000c4a92e59de1e2,0x0007068f7d29994b,0x0000000000000000,0x0000000000000000}}}, + {{{0x000172698b05a9bf,0x0000570dde3c4b9d,0x0004dad7e2ab8a86,0x000cc4fe376d08bb}, {0x000df2d396e50709,0x0008138716b38f91,0x0004e9a6a45d9577,0x000feda477817a06}, {0x0007062a4f79bf83,0x000ca12aff8e0103,0x0000000000000000,0x0000000000000000}}, {{0x000748577bd0ea3b,0x00068ca23c2d1e49,0x00079fc968bdb9b4,0x000359089d5d5807}, {0x00037ae7478afb4f,0x000f6ac1a17ffb5a,0x00076fb0ad6d0956,0x0003cc73da4935d0}, {0x000b38ddd4e896c8,0x00091fcd942654f0,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d06bf88e947b1,0x000a0ca717a0accc,0x000b428946c6ceb8,0x000992652bd2594e}, {0x00041806882365c1,0x0004f2f3523a8241,0x0002e27b634a60c7,0x0006f4680f2b83ce}, {0x0005e159f3342680,0x00053718c76ca148,0x0000000000000001,0x0000000000000000}}, {{0x00063dec4e0c9dd7,0x000cb7d74bb74e1f,0x0004a5f09446f95c,0x000f4555c75d0662}, {0x00028d6e7583fd36,0x00083fa30323cc2b,0x00010012c0c49bb7,0x0009907cd3d64f77}, {0x000c79e69ac90f89,0x0006c12cb352bde2,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=10 [{1,2,3,..,}]*([2^40]*G) */ + {{{0x000bc2a7fe05ab80,0x0001a246f49f7a93,0x00091072db7cc0b4,0x00011f97d30ac7ec}, {0x000169de685e7537,0x0008a2ae9aef0acc,0x000e3c46bded4c6b,0x000604789cdc4497}, {0x000a082feb2b2ea2,0x00078fe890cba4fe,0x0000000000000001,0x0000000000000000}}, {{0x000edbc5c5076824,0x000298a472aca466,0x000a80660505ad15,0x0000c1cc1b16ca97}, {0x000dda2937bbf38b,0x0002545fac4d56c3,0x00090f07e35494b7,0x0000903e9ca74b57}, {0x000b43111b0c8bc4,0x0005465923f9e9c4,0x0000000000000000,0x0000000000000000}}}, + {{{0x0008b52bfb73ff4e,0x000a4d6410d489c1,0x000ae318bb528530,0x0004b71f3f7d7028}, {0x000970b21b686b53,0x00001ea2751f5995,0x0001322663379ddf,0x000e77c11915800a}, {0x000261c476db4bea,0x000ae8e89c22a1e3,0x0000000000000000,0x0000000000000000}}, {{0x000e16965a7b1dfa,0x0009afd6426762fc,0x000532ec13f2e53d,0x000fe4131e801ed5}, {0x000c86963e2f9e9e,0x000f46e5098aecc3,0x000faa2b47801a35,0x00043b6f3406c3d8}, {0x000b57971349f37b,0x0007d749ef39f4ad,0x0000000000000000,0x0000000000000000}}}, + {{{0x000dcaceb126ca58,0x00046c5a73dec3f9,0x0000e0ffb30d8879,0x00026935e7e62831}, {0x000f5e074c83b841,0x0008b04291158a7d,0x000f72a08eaada21,0x000a40d05438fa20}, {0x00005e6ac9aa8c4a,0x000b92b755928484,0x0000000000000000,0x0000000000000000}}, {{0x00081326c74e90da,0x000cdbf1a037a879,0x0009887e29013ab2,0x000ffd6598bd3d86}, {0x0003b3c803a95fec,0x000e4c50722fd839,0x0004e70129b699c3,0x0000fa72cdd65e3c}, {0x0000ccbba65f24da,0x0008975325682f6c,0x0000000000000001,0x0000000000000000}}}, + {{{0x0008d8231d21b98b,0x000f46eede07ff72,0x000c57dc8bc52ba3,0x00096a93bbd28782}, {0x000609e0a7a0e49d,0x0009fbe4aa495765,0x0004c5f79dbfb8ae,0x0005e1789960cccb}, {0x000b74da3d25ebfd,0x000c2e56df642b09,0x0000000000000001,0x0000000000000000}}, {{0x0007057a2be83a30,0x00026ba759ce4dd1,0x00057744ef0ab9d2,0x000c9ee2b7115a63}, {0x00000d77f24c2ddd,0x000142ea1adfee89,0x0000f3fa9d5346a2,0x0007a84ecd7e6d50}, {0x00035caeb7a1527e,0x000c8110e6262dae,0x0000000000000001,0x0000000000000000}}}, + {{{0x000457989aea7c3e,0x000a16bd6196a6cd,0x0006f76c2cbdd85c,0x0005840dcfd847e9}, {0x000ec8ea001b3aa2,0x000898be24444e27,0x000397d8a0c53dda,0x0003fa5f98a5b3e4}, {0x0008e197364ea986,0x000bbd922c6bbfbe,0x0000000000000000,0x0000000000000000}}, {{0x00064b7db5084e37,0x0008c954735a1906,0x0007371d65a99056,0x000bdaf052b4c902}, {0x0009ec2cc9668600,0x000adac668469729,0x0002f7ceb4949cfb,0x0001eda14ca03327}, {0x0009270923989fbe,0x0002a80f1714c9d9,0x0000000000000001,0x0000000000000000}}}, + {{{0x000da9fadd2d2e35,0x000ebe39d9c06e00,0x0007878b1c269b9b,0x00045c0350e16aa4}, {0x000d604f7fb31a05,0x000233dc435d57a4,0x0004a59629977c5d,0x000caa747e5387e5}, {0x0006980680cca577,0x000c7f3aa2473480,0x0000000000000000,0x0000000000000000}}, {{0x0006ecf72ff177c1,0x000ea087d84398f4,0x000f8e3dbad5b5e3,0x000793b7c29bdf29}, {0x000d48b4ad4a2c86,0x0004cf9d25337e0d,0x000be01c858ea723,0x000fe90a676bb282}, {0x000306f507590c7b,0x000915155053c47d,0x0000000000000001,0x0000000000000000}}}, + {{{0x000311b42bb970cd,0x00056fa08e6727fa,0x0006285b2f7609fc,0x000807c307ce1a3e}, {0x000d8c9c1df6de94,0x00070b979619a317,0x000fdde052a3379d,0x000baa7d20b3870e}, {0x0006e443d8f7406d,0x000355511beafebb,0x0000000000000000,0x0000000000000000}}, {{0x0002e82c90634dd6,0x0002249d51e499d6,0x000615ca9d89995a,0x00097ac7d99162d9}, {0x0009551034000ab8,0x00070ca9d924f35e,0x000b526853be7dfb,0x000638dc8c8aff11}, {0x00031fb01463b8a6,0x0004e7b55e740433,0x0000000000000000,0x0000000000000000}}}, + {{{0x0008d4f13a03485a,0x0004c5dd3ccf1850,0x000d8ab1776f2552,0x000a54a5e0bf0c9e}, {0x000e81226e24a017,0x0000e5b1ecb9626b,0x0006acb8b7b3bc5b,0x000414da0130f24c}, {0x000b2f1d94673605,0x0003ae73af8b9c7d,0x0000000000000001,0x0000000000000000}}, {{0x0001a4baf806025a,0x00068cf7b99ef6b1,0x000b70549901e9bb,0x000cc5d4a5ca0071}, {0x000b555009f7be57,0x0008dbaab6501aba,0x0005c582d17b21b0,0x000d9921c7bacde3}, {0x00013fb4b9991c48,0x000d72c6f664c93f,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=11 [{1,2,3,..,}]*([2^44]*G) */ + {{{0x00000e09c15f3e5c,0x000f97f45b409fb9,0x000ff4a01087837b,0x000d204b2bcafcbc}, {0x0000a0da165ec6b8,0x00011716978423a6,0x000f2f7f8295351b,0x00081f58e2d13eb1}, {0x000ed84592b66922,0x00025f5f9819aebb,0x0000000000000001,0x0000000000000000}}, {{0x0007ea077d668278,0x000b9653ee2ff77b,0x00098e8334b5b359,0x0005210487baabe1}, {0x0001a95a5886c85a,0x00009649f4c23788,0x00056127f95c6f68,0x000aed6c6419d339}, {0x000be49aa657d29f,0x000da67ae0b376a5,0x0000000000000001,0x0000000000000000}}}, + {{{0x000fb32bacbbde73,0x0004f08a721e3545,0x000584020b45c467,0x000fd3a284a774fc}, {0x00004477afffeada,0x0006a2c4ec266e10,0x00066dc326c6652d,0x00070b3a65b94280}, {0x00055b8104c7d5c7,0x000d8c4b6af237e3,0x0000000000000000,0x0000000000000000}}, {{0x0000b97b7ca1cffd,0x000435de1358221c,0x000876cab38cc7ac,0x00054cdc8f30b09e}, {0x0000ccb3a4f5aec6,0x0002ac30ca26a9da,0x00011038e2a6fa3b,0x0004545c20a577ee}, {0x000bff8e2f50fb14,0x00093158359a6d97,0x0000000000000000,0x0000000000000000}}}, + {{{0x000b8fa7bdc8384d,0x000f2859e636e718,0x000ed301cc9c465c,0x000006a5b105b2b7}, {0x0007075698827d1b,0x000fbe7e34565b30,0x0006110c56ecfa0a,0x000be3136324ecec}, {0x000a684fbfe59f06,0x00056bc39bf313cf,0x0000000000000001,0x0000000000000000}}, {{0x0008543e894e8a16,0x0006d339681c386b,0x0001fc0aac97005d,0x000c6eaa38aa9f79}, {0x000a90d6089956c5,0x0005ec33374fdc05,0x0007a4eaa1631484,0x0008df64f8ad9e1c}, {0x0007f44336b984af,0x000859e7ad53b8dd,0x0000000000000000,0x0000000000000000}}}, + {{{0x000c8ba9ccf89d24,0x000874ddf8d1b9b2,0x00027291ffd7f24e,0x0008bd9d563287c7}, {0x00065d01bdb48d02,0x0001b99b973b0c12,0x000050d618319b97,0x00038420d531f686}, {0x000d7c201c411c3a,0x000d97468eb84cae,0x0000000000000000,0x0000000000000000}}, {{0x000eb2fc05bf609a,0x00073e1dab9da1f6,0x00049847c3ac275b,0x0007880554d322f9}, {0x000fedd0cd2b7f05,0x00085bb3e74958ea,0x000fcd8d9061a481,0x0006f9ac370d5c6d}, {0x0004cb188a021786,0x00057aa132c3b5f5,0x0000000000000000,0x0000000000000000}}}, + {{{0x000a082d83368da6,0x0009c04ba3a92f9d,0x00018e366fb4a189,0x000dca28f8a4aa8f}, {0x0008c98efdf41b12,0x000e7f6ce11df724,0x0003850179926b22,0x000c0fa0bae8aa29}, {0x0007fa3acc87191b,0x0008222f18a88f29,0x0000000000000001,0x0000000000000000}}, {{0x0009dda00370c8d5,0x000720c0497d2a02,0x000cdb5a6b3dc411,0x00032f0daa73c7c9}, {0x000505d3784e14b1,0x000d7b77b06edb5d,0x000c1f94b45f2c9d,0x0003193f38ea4c8e}, {0x0001706f6af3bbf0,0x000f0df22ed99a0b,0x0000000000000001,0x0000000000000000}}}, + {{{0x00097825d9e0b2bb,0x00096d1340276af1,0x000d82fe6324bbcc,0x0000475ecad6233b}, {0x0009a0cd8d04ac29,0x000e8e067d738cce,0x0004317aa038ad08,0x0009a7ce55aad83e}, {0x0006a1887d5e91f4,0x000a135efeae9285,0x0000000000000001,0x0000000000000000}}, {{0x000fa0b6a035b085,0x0005853d153ead9b,0x000ca7f6fb4ef7bc,0x000bfbb30b798e2a}, {0x0006653595cf1f8a,0x000774e7d1791820,0x00048df862d39281,0x0002db1e40868b45}, {0x000153b336e38fc5,0x00052ce2e4b80e72,0x0000000000000001,0x0000000000000000}}}, + {{{0x000306c918831eac,0x000de7b8556b1913,0x000c579621fb2237,0x0001fa1088e011f5}, {0x0007511558db4265,0x000c63131310e896,0x00029cec356a91fc,0x0004b85a2f85b7d1}, {0x0006bc4404e3b5cc,0x000ac5c4fcc3a6fc,0x0000000000000001,0x0000000000000000}}, {{0x000498b1e054105b,0x000818b7c971dcbb,0x0006cf66f8ee2eea,0x000fa05b49db517f}, {0x00012073a35daeb3,0x000d5bb674f202b9,0x00032d411842792d,0x00076cdcc487632e}, {0x0006568c96ca8dd3,0x000ecdfe1c986f51,0x0000000000000000,0x0000000000000000}}}, + {{{0x000595043dc23233,0x0008d9e1b752f3f5,0x000f6e2b3641b931,0x00064e0f5c02bb70}, {0x000169d8f287038d,0x00062f3a1b075424,0x00001bf3b8c6755a,0x000a2b642127d597}, {0x000c17f0c20fbe8b,0x0005263410177dfa,0x0000000000000000,0x0000000000000000}}, {{0x00008cc560c65efb,0x000ef6706807502b,0x0007a1e8980e532c,0x0007963729a4a8b8}, {0x0007ccb3a4f193b6,0x0002e07ae80043db,0x0005be0346fea839,0x0009ef33f7a00da3}, {0x0001ea778cb41f4e,0x00096ebb760e7727,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=12 [{1,2,3,..,}]*([2^48]*G) */ + {{{0x0009ba658c1f9460,0x000b18b48e1df3f7,0x0005fc03a103eb15,0x0001ad263bed592a}, {0x0000a127b78a3359,0x0000337c7b07e9d8,0x000d2a0349dd74fc,0x0003b1a807c5364e}, {0x000d92cca588d420,0x000d9e772a1716ec,0x0000000000000001,0x0000000000000000}}, {{0x000fc1df3f66f295,0x00015742d25980f6,0x00036f0fdb08922f,0x0001fe47a583206a}, {0x00001c73f88168cc,0x0001b777671d2798,0x00068317ac8979ce,0x0002a98b48363dba}, {0x000f36b7460d4015,0x000da1c3d46c62c7,0x0000000000000001,0x0000000000000000}}}, + {{{0x0001fb046e7fab2a,0x0000e190213473d6,0x000cbb6e9b84f9db,0x0008fb8a36fcff78}, {0x000c47cd5e9d16aa,0x000c2601e9337a00,0x000713efe8445d72,0x00030681fd15bbab}, {0x00051cff90b2dd23,0x00024900ab444b21,0x0000000000000000,0x0000000000000000}}, {{0x000de9a880ca8289,0x000e3bcb8ede5206,0x000e1369e32209ab,0x00036516b711e224}, {0x00025533569db531,0x000419656e59d965,0x000ee21f2d680255,0x0001d59bb004326e}, {0x000bb722c073cca7,0x000866aa784f931c,0x0000000000000001,0x0000000000000000}}}, + {{{0x000d262bb3ca8da3,0x00045288c7563397,0x000d10c59451797d,0x0004e594575ac7f4}, {0x000311006db0bd6b,0x0004d6485bec56e6,0x00089e23442953a9,0x000610efbe7ae598}, {0x000f1a0f166d5639,0x0004e5f91e0ed7d5,0x0000000000000001,0x0000000000000000}}, {{0x000aca95d51d383f,0x0000d23ae1d67e17,0x0001b867b62738a3,0x000907e6a529f72f}, {0x00059335e44e6986,0x0001d5fbd59c1e58,0x00018393e6789776,0x000e5bc878acd1b0}, {0x00059b26328cd9f3,0x000f3d83306eef44,0x0000000000000000,0x0000000000000000}}}, + {{{0x0009313593b1d240,0x0003b3672b4b0a51,0x00078ea42e614acc,0x000784cd22fe0a9a}, {0x000a6c20faf43e72,0x000e49f3038f9c3e,0x000fb914c50987c5,0x00000c76e9b912d1}, {0x000dc2b7b96a89b9,0x0008de238b29a074,0x0000000000000000,0x0000000000000000}}, {{0x00068ea377031f72,0x000e9606adb168ca,0x000e58dde885ecba,0x000177424d422e92}, {0x0008aa609937ceba,0x000f30fc81145199,0x000c9f99eba807b8,0x000e200db6e7a724}, {0x000db2dc7651c126,0x00009cb58e38f0c9,0x0000000000000001,0x0000000000000000}}}, + {{{0x0002dda36b654cc9,0x0008acdfcec83248,0x000164e7c3c6f06e,0x000ff27341f98b3f}, {0x000b145595e7208c,0x000f8fbf6e4d7416,0x000a519d6e691f26,0x00082c10e5fd7be7}, {0x000b1af42c760524,0x000f36238c97dbcb,0x0000000000000000,0x0000000000000000}}, {{0x000a3b8e5e8a7251,0x0006fabc0a713664,0x0004fe41fe20cb31,0x0004067dc2b00d32}, {0x0004fa30d7f12a4e,0x0003f79c759905ca,0x000ef9b323da1724,0x000357ea84efa0af}, {0x00083350488b839e,0x000408f03e5cf22c,0x0000000000000001,0x0000000000000000}}}, + {{{0x000e616991835801,0x00028f75d4e1a857,0x00031b04bb8869ad,0x000fb7737091410d}, {0x000cdd4fad2e164a,0x0004c7cf900934ac,0x0009b433f8948c75,0x00028687541974f6}, {0x0008f0af013f8d2a,0x0000b59eac62d398,0x0000000000000000,0x0000000000000000}}, {{0x0000c23b25d034e1,0x0007d927e7c611fb,0x0007596a05e54d05,0x000f8995596eb4b3}, {0x0005b26be4690555,0x000ae1277dbebe99,0x0006bc99c064aa3f,0x0001128fda3fad13}, {0x000cd244392bf3eb,0x000a481bf8fac406,0x0000000000000001,0x0000000000000000}}}, + {{{0x000d8f6c1e9aa431,0x000e3d32dbe1679d,0x0006391e1828c205,0x00026f450a2821e3}, {0x000a00cf3fc9e32a,0x000a2fc5a7a744f6,0x0006b70facb792c6,0x000e8edf92ab0b29}, {0x000d53faf8f11ba4,0x000b1e87f01ff5b4,0x0000000000000000,0x0000000000000000}}, {{0x000e6fdf30e29338,0x00014840cefab311,0x000c4e092d034ea0,0x000be36bee8c5b8d}, {0x0001f39e2788cf85,0x0008f32c467a702e,0x0005e353cc3b7917,0x000f137377576e06}, {0x0000db55d1c5fcce,0x000d16e9cd523a0e,0x0000000000000001,0x0000000000000000}}}, + {{{0x000e8870b81dfb8c,0x0006908ea654a671,0x000c3eb3660a23dd,0x000daf70673dbdf6}, {0x000c19bbf5d38a5d,0x000fe1371d1e7af5,0x000e30bcc1eff610,0x000f1308bdd31572}, {0x0000db70b20ce33c,0x000e036ab6b3edc6,0x0000000000000000,0x0000000000000000}}, {{0x000357b86d4f22a6,0x000b893ce6e16bae,0x000a3849b8d94e06,0x000e1675b6058ad8}, {0x000ed6add0f99ace,0x0003cd380c39df12,0x000e2335c645ff14,0x000994a0f6180481}, {0x0005a52a4c84b4bf,0x000e9849a710f480,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=13 [{1,2,3,..,}]*([2^52]*G) */ + {{{0x0002ea2996fba378,0x000cf933ae1a32c4,0x000b43b79b9d0dda,0x0002636c30561bd9}, {0x000f9122413700ad,0x0005a779a0d830de,0x000580fda5f65618,0x00067e785d8628b8}, {0x000ce8b188bafa8c,0x0006d2c75df63d48,0x0000000000000001,0x0000000000000000}}, {{0x00090afd2d7e01b7,0x000e57c72b304c5a,0x00040d7dec21b4b2,0x00094cfde0f45d07}, {0x00010aabbfa713eb,0x0007fa8b4fae1b3f,0x00047d2b080d24d3,0x0001142abdb36f64}, {0x000470df72045350,0x00047e76e433f8fd,0x0000000000000001,0x0000000000000000}}}, + {{{0x000081cbd682dd91,0x000d2c1433b543ad,0x00094641d2488d8c,0x00036e702da0394f}, {0x0008248288ca4d8e,0x000112c8a6461fe7,0x0004a486f063613a,0x000f77efb66bb862}, {0x0006e8d41511d90f,0x0009a1ce80969401,0x0000000000000000,0x0000000000000000}}, {{0x000feced91fc3935,0x0002e83ecdac7136,0x000ee8e2857921f4,0x000ef9bbe82b293d}, {0x000bd182b25ab2c3,0x00097ad819ac32f4,0x000916b74b598de2,0x0004d5e666a5dd15}, {0x0007be0b151456a2,0x000b794dc25c5c44,0x0000000000000000,0x0000000000000000}}}, + {{{0x00087c377c23c5aa,0x000d33ca314119f6,0x0002512d0943eacf,0x000f9fd69c0e1850}, {0x000b7c3c6ea7ee55,0x0006291556c20685,0x00053374868b07c6,0x0009f9f339d7b589}, {0x000d6855b9238a10,0x000491ac6af37f75,0x0000000000000000,0x0000000000000000}}, {{0x000b5d5b2f49812c,0x000855e7603bff6e,0x0008f73b087f7552,0x0005c0adc19b7320}, {0x000d355df5442e8f,0x000a4b8876b6aeab,0x000a7378dc2b22b3,0x000c26f89265f8bc}, {0x0006645f23dbb040,0x000f38b09ab1bbfb,0x0000000000000001,0x0000000000000000}}}, + {{{0x00011ca0d8fcc245,0x0008ea92267e1494,0x000bbfcc2abb385c,0x00029656bfd56d29}, {0x000075f2180a734b,0x000dc3400006f728,0x000f754023104376,0x00021bae73e6854a}, {0x000a8d2ddbc75324,0x0003d711770a3406,0x0000000000000001,0x0000000000000000}}, {{0x000476594b8d6365,0x000aedeb8cb49714,0x000c86324ad6ba99,0x00016428c49863ca}, {0x0002a2e5cfc3d8a3,0x0009adc3e0cb62d8,0x000eff79e5f3fda7,0x000eb4f990b6cadd}, {0x000b0e410ae15a98,0x00000aedf394c7b9,0x0000000000000001,0x0000000000000000}}}, + {{{0x00021de1984e2e65,0x000603254887ed1d,0x00087e7535512fa1,0x0006f4ddde7dd7d2}, {0x000fd9cd7f2faac0,0x00009274ea570744,0x000b8cb9c92e3b10,0x000294aa3ad7752d}, {0x0001e25f444d43f5,0x000d6a378f2a4af0,0x0000000000000000,0x0000000000000000}}, {{0x00007294901179cf,0x000fc01f2357da9c,0x0003c6543ad3aafc,0x0004db8aa97c6078}, {0x000dd86b1f882854,0x000e842c5d55c6a1,0x000e30c9aff1c67b,0x000d71107c0af546}, {0x00097038444a87de,0x000d8c2bb6354272,0x0000000000000001,0x0000000000000000}}}, + {{{0x00086f90f323a729,0x00082963dacbf625,0x0004056ab9d5b07c,0x0001d30212e37a0e}, {0x00065da9a3d1070f,0x000445a33677e429,0x000c39a6aee3c700,0x0006b968d6cd0279}, {0x00098776415f9a3e,0x0007bd59dc1684b7,0x0000000000000000,0x0000000000000000}}, {{0x0005cafc1fb833a8,0x000b18657b2cb0ad,0x000af58b0bf123c6,0x000acc1e7ceb4334}, {0x000e5575f0e10ca6,0x000d22916c7bc194,0x00097f735880fd80,0x000604a86980d5b3}, {0x0002f218f0f8d8fa,0x0000806dbeec3ed6,0x0000000000000000,0x0000000000000000}}}, + {{{0x000b6c1669c31b94,0x000b37bb5eb33b20,0x000f2f86abcd3194,0x000093b3893ef395}, {0x000d0b48537fd7dc,0x0006863b5109af48,0x000cb076c2fca9e7,0x000fc785f088e6f5}, {0x000c88facfd552fb,0x0001054316685a3b,0x0000000000000001,0x0000000000000000}}, {{0x000b6af5d1972479,0x000ccf45d30e213b,0x000db2b3881288b7,0x0008f7bff9008ba2}, {0x000607009e5be8cf,0x000e10b176da0a78,0x000522cf433a33d0,0x000090924737e4d8}, {0x000cc0a11c296771,0x0001fd3715d11b75,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c391c2f2938b1,0x0006b396d1c5f420,0x0006bb17f5eeaef7,0x0006a57b7feb16a1}, {0x00026cc8015523f1,0x000ded6e6d4aadf1,0x000f602e23393c9c,0x000e2c8dbcb36848}, {0x00011e23c49f3a9a,0x000730c0c1ebfaf8,0x0000000000000000,0x0000000000000000}}, {{0x0001b88cdd5da561,0x0002fcb4c22029af,0x0009624d6d5aa7f2,0x000db935bb120735}, {0x00019a8308449416,0x000467f9f185fd32,0x00057d8b4d3e00fc,0x000e187052a8a69d}, {0x0009e66380528c91,0x0007a42a603bc9b7,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=14 [{1,2,3,..,}]*([2^56]*G) */ + {{{0x00026efe1474fe09,0x00029da3ad38a0ca,0x000ec34abeaf5cd5,0x000847ac94808b1e}, {0x0001587ade96127c,0x000a43fa8cfa6df2,0x000bb39bcfdb5ad6,0x0005dd4d0c9f947f}, {0x000772a4ebca687c,0x0009227d79e215e8,0x0000000000000000,0x0000000000000000}}, {{0x000926e1c81cb032,0x000ffdb04fbc5abf,0x00034707ba5b9c12,0x000a347c4ee8c89b}, {0x00072367a152d81a,0x0004511a3a4cd565,0x000b8f1a66429397,0x000260ea13e9d0e3}, {0x000a19a28ee14ab4,0x000d5dea76ba4c81,0x0000000000000001,0x0000000000000000}}}, + {{{0x000f1e1b3fe2f1a3,0x0006da87421ab9af,0x000a3305ebdbbfcc,0x000cb7784b75a8e1}, {0x0006a4410056f8f4,0x000ff65612b5abdc,0x0004b1cd83f32f54,0x000fb989d25121c4}, {0x00014abed80a7bb1,0x0005ba8f200e1152,0x0000000000000001,0x0000000000000000}}, {{0x000fb8525d63a07f,0x0002a4f5f23c02f4,0x000405911d9aa8e0,0x000dae0345abb8b1}, {0x0007b4834d14e7a6,0x000b31fdc5462195,0x0005dca7cbf9b75e,0x0001ee84304e26ee}, {0x0006a2c7d37349cc,0x000fc85a34c3afcc,0x0000000000000001,0x0000000000000000}}}, + {{{0x000cefac8c58a5d9,0x000939b188506808,0x0005985dd6ee8422,0x00094a64ab14cf0e}, {0x0001ac27af983cda,0x00024f6eafd12785,0x00025d8bab20f8ff,0x0004a549d9c6da3b}, {0x000f18f37ed810bd,0x000655f630e4c95b,0x0000000000000000,0x0000000000000000}}, {{0x00018594e51ad76d,0x000d8952697460ae,0x000aec56660f8de9,0x00093a39294777cd}, {0x000bdf7dc98fde3a,0x0000c53dc363fcc0,0x00091985d2c2708c,0x00093692d05055da}, {0x000c4d312ebcde24,0x0000c18d0017f5cd,0x0000000000000000,0x0000000000000000}}}, + {{{0x000a3f86e5f8496f,0x0007f32df7ec8ea1,0x000d8551991f1f8f,0x0000f4eeb16ec397}, {0x000f5ebe5be1abc8,0x000f8233b8a1e6cb,0x00037675c403702a,0x000cbf97ecb04148}, {0x0006555682899a5c,0x0000280720d39958,0x0000000000000000,0x0000000000000000}}, {{0x000312054dc27af9,0x00074dd550df7288,0x000193eb1e2f87e2,0x00073656a715c43f}, {0x0006ecb67dce2977,0x000aacb5db8a585c,0x000d92a6332fcd10,0x000cdeebccba4f16}, {0x00036c8da2b8001a,0x0007817b62765789,0x0000000000000000,0x0000000000000000}}}, + {{{0x0002d6d084e03d59,0x000c244d84ecf179,0x000c93fa7f1e0110,0x000f695cc72b1bb4}, {0x000921730f1b2908,0x000b0b36b38d0bc6,0x00029c0e4bf469cd,0x000c1d41428da1db}, {0x000d1253d7a577f2,0x00006a23b655222c,0x0000000000000000,0x0000000000000000}}, {{0x000ba5fbd7ebe31e,0x0005209808ec8aa4,0x00049718327a5383,0x000bb2492c210a5f}, {0x0002eef53e1dbdc5,0x0009d3c1717e38e0,0x000d4877b41e983c,0x00012d8aedea3a07}, {0x0007e058b6c0e3ba,0x000fd022c8be6da1,0x0000000000000001,0x0000000000000000}}}, + {{{0x000870ab98aeb123,0x000044a353002cf5,0x0006150f34fa12da,0x0006eea20086b83e}, {0x000a0a2cdf13169e,0x00028616b25e80e0,0x000c5982d13e0cc5,0x00019702e01a4a67}, {0x000b60ef183d6e66,0x000a1f6f9172f815,0x0000000000000001,0x0000000000000000}}, {{0x0002b57766386476,0x0001a0e6acc5477b,0x000ba422b2405581,0x00090991a9873020}, {0x00045310acf2c8c9,0x0008701ea796459d,0x0008c83917c30ec7,0x0009db51be44d168}, {0x000514c3bb42ce9e,0x00048d0b03fd870b,0x0000000000000001,0x0000000000000000}}}, + {{{0x0007edbd640c9bb6,0x0001c6f483d72a01,0x00058a225c5b0849,0x000e8697568a7e71}, {0x00022821bf73d7fd,0x000c765e3aef4bc0,0x0001d2e8d1daf2fe,0x000772d486e7b59a}, {0x000595f951edfc03,0x000ffff1683f882a,0x0000000000000000,0x0000000000000000}}, {{0x000fc53814c4cc13,0x00014196f30cc555,0x00076a3af64c6ce2,0x0009bfff339f5668}, {0x000ffe438adb5544,0x000aa59ae8f3c48d,0x0006c57ce59b5441,0x000eb7bdc7b7c0fd}, {0x0003b8e1d8e51d10,0x000d6a6427d57897,0x0000000000000001,0x0000000000000000}}}, + {{{0x00005519264c8b35,0x00066ff272a08c1d,0x000142d545c343f7,0x000ef117b8bd86e9}, {0x0001c60c69c66860,0x000b54e53cb6de93,0x0000c9b9924f2f51,0x00030b949095878c}, {0x00016f5f1fba7e2a,0x0005817da79c3a69,0x0000000000000000,0x0000000000000000}}, {{0x0006ad6babd55997,0x000be6d551de11e0,0x0006c45d4c33b3cb,0x0009e3dfcc4aa553}, {0x000821bb5c238e3c,0x000dfc012d05a1e3,0x000650684d8d4638,0x000805b7e2413b85}, {0x000718949cdcfd8e,0x0002efc1a85e6627,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=15 [{1,2,3,..,}]*([2^60]*G) */ + {{{0x000f6a9bd09d8e58,0x00004bdca0a6cc0a,0x000d9e6d336fe5f9,0x000c9d8bd87d0339}, {0x0003f4d463bab3b8,0x000203e46dfb629c,0x0000ef34ea62ed4c,0x000564035458998a}, {0x00069592c6278328,0x00081b3c56ebb377,0x0000000000000001,0x0000000000000000}}, {{0x0009a17aab96cc87,0x000f8ed51ce44125,0x000c62b1c658666d,0x000b6999437c7966}, {0x0008f0fecb36474d,0x000f725b1f7c6099,0x000396c71fdafc21,0x0006a547fb5a5b56}, {0x000566ae79d88868,0x0000f4130033ff0f,0x0000000000000000,0x0000000000000000}}}, + {{{0x000a6c73f41a4fd1,0x000bf46616431912,0x0009c6ffbd2fe4c8,0x0009e4fd42f313ec}, {0x000b9f8b100ba286,0x000e18229bbae712,0x000550161a1f1da0,0x0000032c80f2ffe5}, {0x000f0b1d53bfaa0e,0x00035fb83c760748,0x0000000000000000,0x0000000000000000}}, {{0x000ed33353cf7a1f,0x00075b8b3c031fa0,0x00053c30e33c1415,0x000945a8fa62217c}, {0x000aa8b667de4f9f,0x000c4952fb889399,0x000b6e3b711abc77,0x000959e7e12fabed}, {0x00057ebfe5a1b2cb,0x0005344206e24318,0x0000000000000001,0x0000000000000000}}}, + {{{0x0005f9c5a301de7d,0x000b800d937115a9,0x0004b1412c82ee0e,0x00039cf3df5a5904}, {0x00016cd50327ee6f,0x000841dfd19a796b,0x0005d79c493ac5c0,0x00097275eb2319d1}, {0x0003b6feb4b9d447,0x000542e1eb10df1a,0x0000000000000000,0x0000000000000000}}, {{0x00051bac56d17f38,0x0007837a907c7875,0x00082e7d67e232dc,0x000c3c225acaf222}, {0x00056e17100c95eb,0x000198b234622502,0x0006b8a4beb3ba23,0x0005492d30351698}, {0x000c0dd28973e413,0x00031b2e1155d6fc,0x0000000000000001,0x0000000000000000}}}, + {{{0x00014d7bc4df6981,0x000c45e951da151f,0x0003964143f3d397,0x00056c9c24be6549}, {0x0000ae1293e252e5,0x000bfda40e3aed33,0x000e72cdf82159a4,0x000f514f7b173b13}, {0x0000684bfa5b859f,0x000fce90812f67e2,0x0000000000000001,0x0000000000000000}}, {{0x000a9abf7e9a0c25,0x000a823b0b3a0fbc,0x00011d2709072194,0x000b7a7f17f5564f}, {0x0007987f0af999bb,0x0009d6201796c014,0x000d35c45cce25a6,0x0009265843370c43}, {0x000a55401cbff6e8,0x0000eab503e2ea19,0x0000000000000001,0x0000000000000000}}}, + {{{0x000caabc4b5cd512,0x00034cdeccde50ae,0x000395d24049ffdf,0x0005918925068e1b}, {0x0003f93fb9ea4405,0x000a60ba95d141ad,0x0005981c42f76f02,0x0005946bf800414a}, {0x000435023138c47b,0x000f3cf314147e38,0x0000000000000001,0x0000000000000000}}, {{0x0008bdcc69ae19e3,0x000ac73cebd917e3,0x000c35337880966b,0x000e6ede2718c3e8}, {0x000fd10236ae833e,0x0004797bb14f5b88,0x0001296485e76bd4,0x000a68194c12b2b3}, {0x000b75dc1e45112b,0x000dd88574000b0c,0x0000000000000001,0x0000000000000000}}}, + {{{0x00037d315eea24bc,0x000160f77a65b38f,0x000de27962237731,0x000bd3346f06ae65}, {0x0007a25b38b1587e,0x00055c6b9f2a1d2c,0x0002f34b1687394a,0x00054f27c66a0ccf}, {0x000866c84ecf3de7,0x000b4da4a0f4aaa9,0x0000000000000001,0x0000000000000000}}, {{0x00066dd8b8dffeb8,0x000121bfeaeff003,0x000a80b5c3bfe941,0x0005b6a4c3fed2fa}, {0x000c623dfdf4718a,0x000b0791d22ef007,0x000949ccec61c6bd,0x0006e328d9cc6d79}, {0x00014a1530d03e69,0x000d45fdb36710aa,0x0000000000000000,0x0000000000000000}}}, + {{{0x0006dcfa61a8d3cd,0x000ff977e2649373,0x000089ee4ac6af49,0x000b86d61720bd71}, {0x0007848d2c5df2f3,0x00078e07afbcc66a,0x0007ceb1f230a9ed,0x0000d2f61bf5077a}, {0x000770c3ffbf99e7,0x0001487ae5f08492,0x0000000000000000,0x0000000000000000}}, {{0x000ee44c464c2996,0x000d8f990f4f03a6,0x000453774274aacb,0x0005c8730ef447b6}, {0x000e5e02e661f55b,0x0009f13f1011e65a,0x000f4c8fe17d3ed9,0x0000dbeb35dd393c}, {0x000a7d1cd2327711,0x0003fb444802cd65,0x0000000000000000,0x0000000000000000}}}, + {{{0x00071a842f621fd7,0x0008594c057a1dea,0x000e1689c80fc8fb,0x00022f52adc9a8e1}, {0x00099c47b816309c,0x0000c495f00a960c,0x0002e200a0f356d9,0x000a87494b798824}, {0x000dcd587b7f9ca6,0x0009c4d76d2c396f,0x0000000000000000,0x0000000000000000}}, {{0x00035970dd6ecf15,0x000449aee47a261e,0x000adfd394c8df13,0x000dfbec67553f2c}, {0x000aca43c615471c,0x000606556ef09db2,0x000a225f2e040114,0x00099dfb28da12ec}, {0x000812bc587a4c83,0x0001ab8cba898428,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=16 [{1,2,3,..,}]*([2^64]*G) */ + {{{0x0006146954276a1b,0x0000c171344edf1e,0x000b30130812b4a5,0x000314a14896c770}, {0x000e796a686592cf,0x00079226d890053f,0x000869a5847ac79d,0x000cf60993a83ada}, {0x000e4b5fe7d156a5,0x000add7850cdf667,0x0000000000000001,0x0000000000000000}}, {{0x0005bcbb35fb3dea,0x0000a34e2d6021f3,0x00090be93989877f,0x000303404d6435bb}, {0x00007e5919257861,0x000c99d1992710c0,0x0001c7987d3586cc,0x0008e8681c58c145}, {0x00059a487fa896da,0x0009a8b1a9e54366,0x0000000000000000,0x0000000000000000}}}, + {{{0x00029533273f3ddc,0x0009580b259ba7fa,0x000a4092fea94f8c,0x000efd38be9d56f6}, {0x000720f2ba425622,0x0007c0adb2a4d25a,0x00018752498a9ea5,0x000d893bbb4d11f1}, {0x00056b02f195ec41,0x000bca2ad72c4b2f,0x0000000000000000,0x0000000000000000}}, {{0x000a4013f1ab7060,0x000f17521f983a0f,0x0005292b2f1ebae7,0x00075002debce289}, {0x00003b6cd203ad6d,0x000c3592c993bfe5,0x0005400a40b351b3,0x000e9b6bafed180f}, {0x000d6a9f0291283a,0x000563036cf95dd4,0x0000000000000001,0x0000000000000000}}}, + {{{0x000efa3e474a5b75,0x000e5a6d55141813,0x00008a31bd3435d9,0x0006a68b3c599425}, {0x000252817b3af9bc,0x0006a695a37f35e7,0x000f06836ac1d6b0,0x000b19a92f525bb2}, {0x000215d9b8e544be,0x00080723f1554fe6,0x0000000000000000,0x0000000000000000}}, {{0x000e709e6109d280,0x000e367fd7bcb2f8,0x000be531b1c6fe21,0x0002d1fc5388b64a}, {0x000c169c0609b0df,0x000cd0fc3d5f664a,0x0003de00815f78ff,0x00049af9ff38956a}, {0x000e62c250aedb30,0x000af977c662b6e7,0x0000000000000001,0x0000000000000000}}}, + {{{0x000cfa5a95db7680,0x000cc333878665a8,0x000809b2a4ba5401,0x0009594f6cdc3f0e}, {0x000e99bbfac6790d,0x000d836074d551d6,0x0009d9ae874e847f,0x000a264b3b0b13f8}, {0x0003ac51f7a6ec5f,0x00021d6dd250c60a,0x0000000000000000,0x0000000000000000}}, {{0x000e14abaa7747bd,0x000f127c3196cad1,0x00078a629241495e,0x000ded5d0cbcf8af}, {0x0005b83d56ec31f4,0x000c6ef029fa54b1,0x000cc516f0a12c6c,0x0000ce830e11ae62}, {0x000747fe9964fd2d,0x000c6756076a3288,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b9a69b333564c,0x000cec5ad994fbe9,0x00091ab4f5d4d95d,0x0004dbde503f2b0d}, {0x0003c8b32fed7498,0x0009fa10339bf799,0x000b1e6b4c7bd908,0x0001c54e64a99794}, {0x000598ef7979d915,0x000f29a322ae1bc8,0x0000000000000000,0x0000000000000000}}, {{0x00042520c237fcf4,0x000303a14c0e24d4,0x000f7ed62c67df4c,0x000ee5dfd3635dff}, {0x00097bcf94654c63,0x000eb529e2ea1f28,0x000969c7cff702bb,0x0003bf0591306903}, {0x000474be25c3afe3,0x000d8f2570e7350c,0x0000000000000001,0x0000000000000000}}}, + {{{0x0003d3d928f89c37,0x0004d9c668cfa4b7,0x00097ee2907da69c,0x000fb743bf4c3402}, {0x000cd4034c59cbf5,0x0009bc4b73d60ae9,0x0007664da82be729,0x0007e3800a84da1f}, {0x000700f12fb007b6,0x000882b546161eb7,0x0000000000000000,0x0000000000000000}}, {{0x000e150bbd0f66b9,0x000122fc5d0def4b,0x0001ba0f43d660c9,0x0004e9263a5b4550}, {0x000ef33c24e5b722,0x000249e1b7ba92b4,0x000aa152b1856c8d,0x00095fe68108b2c9}, {0x000e766ae6e54017,0x000903a379f58c2f,0x0000000000000000,0x0000000000000000}}}, + {{{0x000980153e2aed11,0x000ca0053853a90a,0x00003bf5f23c9661,0x0009061283e91bda}, {0x0006236d967ae1de,0x0004f3d80708ed52,0x000bb014e9c6763d,0x0002f3d82f09bad2}, {0x000f8d828de34c80,0x000187805c46ac1b,0x0000000000000001,0x0000000000000000}}, {{0x00086a0c8fd9feb0,0x000d468bed15d861,0x00036573dedab729,0x000d88e4dce7ee93}, {0x000d9e86caa8e75e,0x000c0d08d6110fe2,0x00082b709f0b0d08,0x000ab3fbb5af39b1}, {0x000c1b7910befecc,0x00062bf43fafb941,0x0000000000000000,0x0000000000000000}}}, + {{{0x0005341f232f3278,0x000c66dff5ad0b4d,0x0006270a82ebb141,0x0002897d7912e413}, {0x0006b6b16ad87fc6,0x000fe7c18f348f2e,0x000a03bae57af6d0,0x000a6d2d6ab02f22}, {0x00017c3e7efa7a28,0x0009c673423958d7,0x0000000000000001,0x0000000000000000}}, {{0x0004f0f2ce49ed5e,0x00055b8c6c92190b,0x000aff1be7fa884b,0x0002c375de74b331}, {0x000f37a676c7d888,0x0001190b6b57c355,0x0009c95180dbbfa7,0x0001d7dc77b1599b}, {0x0007eba118f76648,0x0004aa840229ee22,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=17 [{1,2,3,..,}]*([2^68]*G) */ + {{{0x000507b4b3de6fe0,0x0009064574b533f1,0x00076eaa707b56c3,0x0005e98aa9532376}, {0x00034611c9b6716e,0x000eb6a26112d9a9,0x0006e068430b4789,0x000e50fd96103fab}, {0x000509b62d215cdd,0x0000b7c4da786d1d,0x0000000000000001,0x0000000000000000}}, {{0x0002a0af86e4be0a,0x0008383ebf635e75,0x0009175f3f7680f5,0x000b999d9f1a0d87}, {0x0001f04cce1e2861,0x00086e6afd75ef23,0x0004476af7240e6e,0x0005e887f56c0473}, {0x00094ba352837e09,0x000f469e3dc524c1,0x0000000000000000,0x0000000000000000}}}, + {{{0x000726e7d4a98265,0x000a397d1e874b3c,0x0008c78c755a513d,0x000fef1392677915}, {0x000e1e9e24f3ae62,0x00096cf6213d1cf9,0x00004b1f503d4fcd,0x0000f07e39bb0e12}, {0x000406c607195818,0x00046d3b7b9a617a,0x0000000000000000,0x0000000000000000}}, {{0x000b43a8c35ac9c8,0x000a92f37f3857cd,0x0008ae49b6bed377,0x0000e3380827d789}, {0x000e2deff6865bd5,0x000758e466fdb287,0x0001f3ba0c560a0e,0x000db418a2645432}, {0x000aa26f5d44767f,0x000d36cc7b7f8bfa,0x0000000000000000,0x0000000000000000}}}, + {{{0x000020f87c5aeb2e,0x00035693c551e0d7,0x0004a7f0938f3f98,0x0008d4d829ae419e}, {0x000d8e42f19ee358,0x0004d94560e54e60,0x000ddb20bc96b546,0x00014bfcf64c2c92}, {0x000518fec9517e43,0x000a92773a95b0b8,0x0000000000000001,0x0000000000000000}}, {{0x000c159b5e61b250,0x00070c6538519060,0x000d0372c4dd1c2d,0x000fd8998866b5fd}, {0x000ceccabb79115b,0x0000eee1fb689260,0x000b839fc8378f05,0x000677dee33a7134}, {0x000b9326b080c62a,0x0003ae07e602129a,0x0000000000000001,0x0000000000000000}}}, + {{{0x0000dde02f16a4fb,0x0001ca814d149b58,0x0003cc8c599eaa3b,0x000d833d43a45440}, {0x000afdba29de3c98,0x000b2ff2056e31f2,0x000ab3bf81e95cba,0x000649419f19b530}, {0x000585b648a6e1bb,0x000ff11dfbba1ed0,0x0000000000000001,0x0000000000000000}}, {{0x0004734696bc60cf,0x000199c0250d4a2b,0x000d4759758b9f4e,0x000f68bc326d4e2a}, {0x000cea78113abc32,0x000d248f92840b01,0x000d61ebd87644ac,0x000ae9a32d38a8d8}, {0x0000c706b58a69c2,0x000561f4b942e16a,0x0000000000000001,0x0000000000000000}}}, + {{{0x0002fe7e20769488,0x000a1c068b87a199,0x00032a3459c496fa,0x000e96c391b8839f}, {0x0007f4914cec7211,0x00039756176c58a1,0x000aa63441905f9c,0x0008ea16377fb867}, {0x00042578ce2a2865,0x000cd140c73dd697,0x0000000000000001,0x0000000000000000}}, {{0x00074ef7ab059a14,0x00031e0f7d0f636a,0x000fea67e10b4430,0x0006ef5711a15a06}, {0x000f717856862153,0x0002513381cfd3a9,0x000a8b9906086974,0x000a08521bae079c}, {0x00052ef6a1d4b90c,0x0004f5ad24ad39a3,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c2d28a3a30e7c,0x000933949c694069,0x000d1f2a5ed527fa,0x000e2b5573d97d5c}, {0x0006aa892b500798,0x0008ac4418f1419e,0x0007bf2d1e227f12,0x000bc2143f4c8a62}, {0x000ae6ef99da3403,0x000679827f46143c,0x0000000000000001,0x0000000000000000}}, {{0x000a8cba49268935,0x00038ccee58ce9de,0x00061df0ac493218,0x00075eb49225f314}, {0x000073000d3eb206,0x000145d79e65edfa,0x00007b1990aba7cc,0x000c916353a37685}, {0x000854a40d94fba3,0x000cb2e109e1b50c,0x0000000000000001,0x0000000000000000}}}, + {{{0x0000c263771f69c1,0x000d1595543d358f,0x000ef2e3d8a56184,0x0005866bc8c3965a}, {0x00021c100a7748c1,0x00029b46541a4ddf,0x000c5cb49ea744d6,0x0007a3a96b8b1fb0}, {0x0005aa7e85061bd2,0x000163e83c339e28,0x0000000000000000,0x0000000000000000}}, {{0x000102ac6923a672,0x0004153c3e1bd59a,0x0003473bcdaf9384,0x000f589da2a6c958}, {0x0003af0eb1a0311a,0x00041f50baac8d3b,0x000bd2c8728d9afa,0x000fc24b4e16e953}, {0x0003e6d4570ad37f,0x000aac57efaaa8de,0x0000000000000000,0x0000000000000000}}}, + {{{0x0008c7fccb0a85bf,0x0003bdff198eec53,0x00029ee8af84ec04,0x000572ea125b846f}, {0x00025280cfc9ed01,0x000f73f2654ba803,0x000ffbf57e3b7be3,0x0004f83701a26bca}, {0x000d20a251a2d372,0x0003ec410f80b319,0x0000000000000001,0x0000000000000000}}, {{0x000961197dc0e519,0x0003d8c136f93b3b,0x000000ba8d6c2646,0x00084f848d99824e}, {0x0003fcfbb42b20e0,0x00038715f781fef3,0x000bdd048ed10781,0x000042869724ca7d}, {0x000de2c203c66b90,0x00090489fab2c4cf,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=18 [{1,2,3,..,}]*([2^72]*G) */ + {{{0x00038bb0c7aa46ca,0x000d2cad2e37a158,0x00054d33ad65a28c,0x00004b20d4f1caa9}, {0x000b180e4c9d244b,0x00070a13f58c65ce,0x0008816ecff016c3,0x00016ad260aeee75}, {0x000dbb7b595c36fe,0x0002944d06dfe8bb,0x0000000000000000,0x0000000000000000}}, {{0x000c325d67839921,0x0006a5955c2a22f5,0x0001664092579a37,0x0003aac4f8e9390b}, {0x000722a8dbf2236b,0x0007b02d94034f2b,0x000fcd8d5de86b97,0x000fa8bc9f80729c}, {0x000bcbc03be296bc,0x000364ec1469f11d,0x0000000000000001,0x0000000000000000}}}, + {{{0x00048e2fcaa83f58,0x000495fafcc6105f,0x000d34073fc2c5d9,0x00042510321d1a08}, {0x000d83427742e304,0x000ec5f97b8068ff,0x000530da7faa8a52,0x00005d010e52ac14}, {0x000df5701f277a14,0x0000eacdd532283e,0x0000000000000000,0x0000000000000000}}, {{0x000f566cbb172daf,0x00050c51771845f8,0x00066aafeeea7b0e,0x0005258081cf4ee6}, {0x000c71bc2c6ec8fd,0x000790d250232a19,0x00011bed4c06ab26,0x0000acd06e0bdc44}, {0x000734273e0fd2a2,0x0005c1c9fb738a19,0x0000000000000000,0x0000000000000000}}}, + {{{0x000b6ef22ab96e69,0x00026b8cd18dc5dc,0x000563a07bc4111e,0x000f73913482455e}, {0x000edcb5ec4ad0e2,0x0000caacaf21483b,0x0000f16a5a48441c,0x0005bffbf280c9e8}, {0x000f37a7690242b8,0x0002eed954418691,0x0000000000000000,0x0000000000000000}}, {{0x00066d6e31428479,0x000f47fc4b8794a7,0x000a81360ec38293,0x0004d77d31e9f867}, {0x00042db92af31be3,0x000d799976882df8,0x00005ddd34a906cc,0x000b961ddfb3abb5}, {0x000bbb326a3a37b0,0x0005d4f7af85a74f,0x0000000000000001,0x0000000000000000}}}, + {{{0x0006d183b4b1bfc3,0x000143cdd1f50d0b,0x000721cf9d088770,0x000338fad247118a}, {0x0004efa498ee55a8,0x0008d9808733ff45,0x000faa72107a954a,0x000a392986064eae}, {0x0000c503bc385af5,0x00095cfc7e0cec3e,0x0000000000000001,0x0000000000000000}}, {{0x000f6c1133ea5097,0x000685bf161ebfa2,0x000bb087e9c48b5f,0x000987c958eb481e}, {0x0001ea465a54c3b7,0x00081943446e92e0,0x000a8941e66d88ba,0x000d40dc6c71b0c7}, {0x0007f59a3690cafa,0x000130f02679ac05,0x0000000000000000,0x0000000000000000}}}, + {{{0x000c5b4b93b19e8d,0x0001b316df16919e,0x000549bdce1a1a0a,0x0001cc00d06d528f}, {0x000cf17722a1d1b7,0x0009c7821f1ae098,0x00007c6fc2825a3d,0x000f796a6b06d5bd}, {0x0004ca4366a06040,0x00072103b88040d4,0x0000000000000001,0x0000000000000000}}, {{0x00038b6ea0266cec,0x0003d4ff03eab713,0x0008f81bfb033aae,0x0004b58cdd700e03}, {0x0002486b62595d6a,0x00027288e5a75ffb,0x000043649c2c3428,0x00056516d9dc3a87}, {0x000145473b4564bc,0x000f9a0961272b27,0x0000000000000000,0x0000000000000000}}}, + {{{0x0006ad885aea13aa,0x00022fbb0e237052,0x000115fb0a1aa034,0x000ccb8ce398278d}, {0x0004178f8f78a3bc,0x000403be5854a510,0x0006bcde0e2e720a,0x000edf5f632b7fc4}, {0x00038eed71d606d0,0x000d21f92dbc6d75,0x0000000000000000,0x0000000000000000}}, {{0x000631ff7cd8f78c,0x000698475f524849,0x0005eafa796bd6f5,0x000fa7a4ee42bd53}, {0x000b8f8540687776,0x000540f9fdea3095,0x000f5bf17004e5b9,0x000834c440599d47}, {0x00063874f7c8a4ef,0x0006bec9a4f28185,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c12a0297bd390,0x00020c5821857743,0x000e07f2b528d6e4,0x000318df51deafc4}, {0x0009c03086d94e26,0x0004d01fde5e064b,0x00071c1d5855c1d9,0x000cdd2d762cca56}, {0x00022e583500cf8e,0x0003f52279b68b61,0x0000000000000001,0x0000000000000000}}, {{0x000b572837258eb6,0x000b88af33e25ea0,0x000ba976e7082c63,0x0000d0abef5e06ce}, {0x0005c8fda6ac69af,0x000f82b26435f42d,0x0005ef9369e629b7,0x00029491bb2be768}, {0x000ae13ca4da59b7,0x0000f00ce8846bfb,0x0000000000000000,0x0000000000000000}}}, + {{{0x0000079959682d5c,0x00076719e3233b3a,0x000c78c2194cd545,0x000e51d3c744ff86}, {0x00053afacd6dd789,0x00098cb1ba7a5cd2,0x0004fb918b560853,0x0009ff1bce38273c}, {0x000a7efa90ba240c,0x000ba73bb2e372bc,0x0000000000000000,0x0000000000000000}}, {{0x0003a114b353d398,0x000d2df4adbd1d56,0x000e9ad940e90284,0x000fca7fe3af63ef}, {0x000de96feaa4e61f,0x000bf94ff4ba0669,0x0005279d7b8471ad,0x00071dda976e696c}, {0x00066b8800a22911,0x000f5fba44b58815,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=19 [{1,2,3,..,}]*([2^76]*G) */ + {{{0x000b896b5317311c,0x000ccb025efc3b29,0x000bcd9f85c60e34,0x0005f821ae29c1d9}, {0x000293dcc63561e8,0x000f95824c27219e,0x000843c9e01039f3,0x00048ef0f79fd3a9}, {0x000ddb5a3bba44b8,0x000011f0a7f5379c,0x0000000000000001,0x0000000000000000}}, {{0x0000be31597dedd3,0x000aaa0d73669dd4,0x00090c605d60706d,0x0007d62f262a826a}, {0x0005e90997b0e2e6,0x0004dc73225ac29c,0x0008be39728fe4ca,0x00038656b7a746c2}, {0x0008bd5a3cf46a3d,0x00080a0c58ac7031,0x0000000000000001,0x0000000000000000}}}, + {{{0x00070e2b4e96cd1b,0x00023ea39d68ac53,0x000f98a99040b26e,0x000362a9be557ba1}, {0x0003c202765cccef,0x000726d7b5a7731e,0x000faf8cd815e2ba,0x000ba6579cd91c25}, {0x000ec8fb3dafb2e8,0x000af4648049fac5,0x0000000000000000,0x0000000000000000}}, {{0x0004b6251452046e,0x000cb296110b89a0,0x000551f88c9d1ddc,0x000bbb0b0f26d015}, {0x000c2b5bd39d39b8,0x000dc18ef79d52ff,0x000b527ab6d006e2,0x000a804f61d0142f}, {0x000be5992391511f,0x000815a3e717ea9d,0x0000000000000001,0x0000000000000000}}}, + {{{0x00029eaddc553333,0x000212246b16c20f,0x0009da31a639b833,0x0005a63d297cf150}, {0x0003190a2a3a8499,0x0002ca8af6260545,0x00018490cdf918d2,0x0005a5ed4b646253}, {0x0004fec387ca9de6,0x00010b72b35acffa,0x0000000000000000,0x0000000000000000}}, {{0x0006d539b23d78ec,0x000a4b221e3646f9,0x000b6bf83af256f3,0x000d62f0c408a90f}, {0x000fdaefff14a7ab,0x000e41ce0c4069cd,0x0001cba29824953a,0x0007a382ab7eb47d}, {0x0007f64599eb440b,0x00074a4c148b6095,0x0000000000000001,0x0000000000000000}}}, + {{{0x0005a160c85caa71,0x000f0e79edff6be6,0x0007704970c1ae3a,0x000a395f8b957c42}, {0x000c85f0f181eb8a,0x0007d8f529bdf3d6,0x000d3534e626c58e,0x000c770dabfad83f}, {0x0003e65d7e5ada98,0x0005676430730bac,0x0000000000000000,0x0000000000000000}}, {{0x000d0476d73fcec9,0x000d714dcf97c309,0x000a56a3252ab9b5,0x00097fc79648c08c}, {0x0006b897ba609ff2,0x000c446a06ff8430,0x0002d01ddb643744,0x000f97ee1218bc20}, {0x00048db33f9b0f80,0x0005e8b5f54bb8f0,0x0000000000000001,0x0000000000000000}}}, + {{{0x000851bfcfbb80d1,0x00082b51c40077e9,0x00087cd565dbfe09,0x000954bdd372a1cb}, {0x000f6bbff7b4dccd,0x000237c51d294b36,0x00003d64ce0f8798,0x0000569d6e3c2614}, {0x000a6224fb79e0e6,0x0004d7c33dd3b5eb,0x0000000000000001,0x0000000000000000}}, {{0x000f054ad5a9cfa1,0x000bee5f93daceaa,0x0008aa260aa160bb,0x0005025da9f4b722}, {0x00004817d1e67b1b,0x000e00279781308a,0x000c2084afd2f00f,0x000c154f68e6680c}, {0x000c6b0f1d4b7ecc,0x000fe2b89761184c,0x0000000000000000,0x0000000000000000}}}, + {{{0x0007fbeff2b9ea68,0x00035c954c6cefab,0x000062277d67291b,0x000820637553137c}, {0x000e8b75730d8af4,0x00068d2250710c68,0x0007a2fbae3e7c1b,0x000f6b643e1aff63}, {0x000dc46991ef002b,0x00096e38ab4582dc,0x0000000000000000,0x0000000000000000}}, {{0x000bea0d80f3758d,0x0001d6899ee62692,0x000cdd2a79a4d763,0x0002f50f2ee3aea9}, {0x000fb1476eea0816,0x0007c81475c4d433,0x0009d9fe82142372,0x000756c76934dc96}, {0x00033eb086d918c8,0x000e3056d2a89175,0x0000000000000001,0x0000000000000000}}}, + {{{0x000ef0de9496fb55,0x00076806fa762c50,0x000770bcaf6f226f,0x000a0d3e47cff6b0}, {0x0004eb780ef8cec1,0x00034df87449872a,0x000896382a505c84,0x00046b56a94dfc29}, {0x00021c7a5c037f2d,0x000659ef98ff9417,0x0000000000000000,0x0000000000000000}}, {{0x000001c935289072,0x000f4a229e4010ce,0x000b1be023ab7710,0x00073fb44f780b68}, {0x0002944ddc611373,0x000ba09ab8b61290,0x000be003d4bb157e,0x000f7557730f52d1}, {0x000506c275d184bf,0x000c57abea8b4979,0x0000000000000001,0x0000000000000000}}}, + {{{0x000004498fadaa48,0x0009d0bea3c3894e,0x000a8f46458aa39f,0x0000008b0b3654a0}, {0x000ca4cd7392bf83,0x00012eb97aa46a22,0x000b9cb80e1d7afb,0x000cf74c8adcd888}, {0x000b51d04bb6e179,0x000b50968eb22473,0x0000000000000000,0x0000000000000000}}, {{0x000f96d03d831756,0x0007499de9e10051,0x000ecddcd4ffade3,0x0009dbdffc72771e}, {0x0006b5e1bb9647aa,0x000addb508dc2415,0x000ddf40de78eeab,0x000ab1c488946fac}, {0x000c823824a964d6,0x0003b16f258c8749,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=20 [{1,2,3,..,}]*([2^80]*G) */ + {{{0x000cc508d53fa611,0x00057fcd40894532,0x000f3eb54e960b10,0x000ea40877d231ff}, {0x00043e5110313bd6,0x0005209f6eb9ee83,0x000589764924e778,0x000332e258b2e7fa}, {0x000e2e038618a6eb,0x000aaf96067c3511,0x0000000000000001,0x0000000000000000}}, {{0x00022156015c8ff4,0x0003a0ef974e440f,0x0008ea1f931a1b7a,0x000e417472932b48}, {0x00003bb75d745720,0x00096758e51bf9c8,0x000f97c7f0b39099,0x000b39d56a488d83}, {0x000fe1ded1fac932,0x000399aaf43ccf55,0x0000000000000000,0x0000000000000000}}}, + {{{0x000fb78344f73774,0x000a49ad3e73876f,0x0008771e37ad3158,0x00003f2cb98ec469}, {0x000f31bd531106f1,0x000434959ff8325a,0x00064eee47f875ba,0x0001cf224bc0a102}, {0x0007d33a19ccf2f6,0x000f2186ce603133,0x0000000000000001,0x0000000000000000}}, {{0x00006e91f2869773,0x0000239179c5ea67,0x0009aa4ed3879ba3,0x0003eb977e239f26}, {0x00090ef091443aa8,0x00036fc4d282853a,0x000a0bb2b260d343,0x0008119fbd0756b3}, {0x00053a3a6e0f1619,0x000016a2af080234,0x0000000000000000,0x0000000000000000}}}, + {{{0x00000185bd1fcc92,0x000a0002ebe1f280,0x00030d3e5f23ebcd,0x0009d40f75cccaba}, {0x00063108edd488ea,0x000028024e152a65,0x0008296732e422c6,0x0006142e6cc11761}, {0x0004e44889dea726,0x0004d1b05325e95d,0x0000000000000001,0x0000000000000000}}, {{0x000270a2e4063870,0x0004d9c29b5dcaf3,0x000d2f759d7bad98,0x000ff7c2ad7bc046}, {0x0000fa4e4f59d347,0x000a06be29c16d4c,0x000bb31872d14ff0,0x0002b7a5b6ec2390}, {0x0008ae4cc66be2ce,0x0007006b9b1fe040,0x0000000000000000,0x0000000000000000}}}, + {{{0x000c4cee52cfafd0,0x000916a99628d59c,0x0004417813a4764e,0x0003f0c49a05da16}, {0x000992babb644e42,0x000179a66e24dca8,0x000cbef894f6883d,0x0001ed7756c7c157}, {0x000ff08e144c3013,0x0008ac78b0a3e9cd,0x0000000000000000,0x0000000000000000}}, {{0x000dabd753963ba6,0x000426be7ba3ec43,0x000d17b8f8b93626,0x000d7ac0bfcd2a78}, {0x000c2aeda53c9486,0x000c99eeaefc3c49,0x0003d0949fb4a9cb,0x0005db07562812ab}, {0x0005da4c6c0f863b,0x0009e08ec31fe43d,0x0000000000000001,0x0000000000000000}}}, + {{{0x0008d00b0f45825d,0x00075f4acb7a9109,0x000fe317cf8f4f81,0x0009a77cf8155d16}, {0x000d7ac3ddeef2bc,0x000aeae3c417520b,0x000e6ff44ee6fbc0,0x0008d8c238521aaa}, {0x0003f42c9f47bc82,0x000d2fc09b26d055,0x0000000000000000,0x0000000000000000}}, {{0x00032ac7c6897ed6,0x000498c1e669bb9a,0x000697322f4c8aca,0x000625a543042d46}, {0x0003cdf16aa69334,0x000b4b67c267bda0,0x0005d6f205d341fa,0x000005daa2bd83a5}, {0x000c9573fcdfd94e,0x0004e81cb76afe9a,0x0000000000000001,0x0000000000000000}}}, + {{{0x00074fcede51930f,0x0001c997863b91f0,0x00092d449a3c4328,0x000c91197a68c2d7}, {0x0006a3b2de0b3063,0x0003e7d82555e166,0x000f427f70b4227f,0x00066c04e18d6aac}, {0x0004c82c2c2b9b61,0x00095a376fa210aa,0x0000000000000001,0x0000000000000000}}, {{0x000de0f4a3a29f55,0x00023263844f1727,0x000fd0bec7770941,0x000e79f43b5f4e85}, {0x000035cbc9a5768f,0x0005bb2328826a73,0x0008a77da7d22096,0x00096978fe424078}, {0x000ae1a0514c7cf9,0x00085e77943ce3c2,0x0000000000000001,0x0000000000000000}}}, + {{{0x0000d1b4594afc85,0x0006a925e9937fdb,0x000d1cf398814c56,0x00000694cd250848}, {0x0005fbfd82b6ccc3,0x00047db4ae135bc7,0x000c1f18639e63fa,0x000730a5e5b32295}, {0x00041bb1c61f91b2,0x0009451335383b28,0x0000000000000000,0x0000000000000000}}, {{0x000df27e3f2dca32,0x000ee40fb695c7e1,0x000c8c313d1721a9,0x0008bc93267e9801}, {0x0006a9aafbe12b28,0x000e34c2b180d7a3,0x000e6b65e8b79ae5,0x00047da7f03b22ef}, {0x00094e563552e913,0x000f4aab16538cee,0x0000000000000001,0x0000000000000000}}}, + {{{0x0006db42e9e50fad,0x00042149f7546b33,0x00057093c06f6900,0x000dbce88e00d7d3}, {0x0004ad9ede7428d2,0x0001940521d004dc,0x000e4970d3be2ce0,0x00031bb5cf60dc4a}, {0x0003df5670a6ccb0,0x000b64df04605d80,0x0000000000000001,0x0000000000000000}}, {{0x0005f0fb0c7d8a77,0x0006951f8ad28aa0,0x000e5b908dd39d0a,0x0005f76fd67e92ff}, {0x00080f281077f416,0x0009ee2db2c8d529,0x000e9a09ff0b841c,0x000f9a5850f2e792}, {0x000e9887cd74d1ff,0x0009aa468c4978db,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=21 [{1,2,3,..,}]*([2^84]*G) */ + {{{0x00045bc39568d9b4,0x000deebca99c74cf,0x000ae132b3816aa0,0x0003fb57b713a9d8}, {0x000e4139b0131f8d,0x00042164bbc38156,0x0005a533293d5ffc,0x000f28a54d0e74da}, {0x000246758970fcef,0x0005cb3ef8fd2b56,0x0000000000000000,0x0000000000000000}}, {{0x000e59886b3941bc,0x0009ade1a4b217a0,0x00023117719aa3f6,0x000c88c1b7b4e45e}, {0x000c3f1294233118,0x0001ed8c9dd7dfa5,0x0003ffafa2104f8f,0x000e89ed71382221}, {0x000d0f8573ea0a97,0x000a81f09db9f82c,0x0000000000000000,0x0000000000000000}}}, + {{{0x00077d542b1d8c85,0x0009eb6c76648a7d,0x000ae9936fca1675,0x000c8db3d9556eeb}, {0x0008f1fc23af7239,0x000956a6643df02c,0x0005264fec894e1d,0x0000aaa92f802a0d}, {0x000d756f014a90b9,0x000e0753d197ff93,0x0000000000000001,0x0000000000000000}}, {{0x000dc80b7f62d4df,0x0005de026f897406,0x000bf46ad6add1ea,0x000f1c5d1c416858}, {0x0004d6b3d82ce8f0,0x000459159df587ce,0x000ca473b92c19aa,0x0006da5bec10b6ba}, {0x000e0b4be600af3c,0x000cf6c81e2b9b40,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c67c15f2b99a5,0x0002e78f41b12d02,0x000f7e85b9e3bb96,0x000e6c293dd82110}, {0x000a0b7296f35eee,0x0008b9f01a446a48,0x000f7d8d89873daa,0x000d90409d0bfd67}, {0x000b42d1c91c3962,0x00005e6c3afa1fdc,0x0000000000000001,0x0000000000000000}}, {{0x000607a511880114,0x00046b3caa5f65bf,0x000ed1d7edcb6427,0x000223b76ca3eaf2}, {0x000c254785aeacd8,0x0003d96bed772614,0x000fe17074c3138f,0x000fc4599803c303}, {0x00029d4441854c4a,0x00068d2f47c7e6ad,0x0000000000000001,0x0000000000000000}}}, + {{{0x00027dadf66942d9,0x0002d934d4f0887d,0x00024ab4a3b4ff58,0x0008094d151ee4b7}, {0x00059df116aee58c,0x000c8ad8141ceee5,0x000c277f6cc0cd16,0x000b9d41dd2c3d13}, {0x00018d63c75e0cd1,0x000c06bb0767f3f3,0x0000000000000000,0x0000000000000000}}, {{0x0003f3f369ef6b69,0x0007808f1170f91f,0x0006338ef6344906,0x000cb4597495e6b8}, {0x000283f524ab766d,0x000d7731127ec634,0x00075be86373e0af,0x0008d2af0e3a5495}, {0x00014c6819dfc2be,0x000f7affa7af5c63,0x0000000000000001,0x0000000000000000}}}, + {{{0x000892c7124cd33e,0x000b92182e71043c,0x000db014cae4df90,0x0009ceb0a4ea4119}, {0x000ef942ed19b706,0x0000628b68098d1b,0x00084c6cb159ff0e,0x0004d4b5184cdc53}, {0x000190191d032e77,0x0006b2ec68efce97,0x0000000000000001,0x0000000000000000}}, {{0x0003436236c90d20,0x000e12c74816ecfc,0x0003e074e74849dd,0x0007be255b09c30d}, {0x000526507824bfa8,0x000ef7cb512aeee2,0x0002afeb20a50c2e,0x0008aa489a2c4123}, {0x000334adba554c1c,0x000ed1b94b601c93,0x0000000000000000,0x0000000000000000}}}, + {{{0x0001c752e894f83b,0x0001102db47294b0,0x000efd031528755f,0x0008e7a5233b9d7e}, {0x000dc69c62c7500f,0x00051f44711ee3b5,0x0009d94a5e280f05,0x000b11042cd8c7dd}, {0x0008b14370c2167e,0x0007807f4636e4af,0x0000000000000000,0x0000000000000000}}, {{0x000598691d482817,0x000b8412f599a077,0x0000459d6b4cf61c,0x000405e26f27cc0b}, {0x000ddbc7fdaf5126,0x000cdbba7c4a3026,0x000b262658e4a3b0,0x0000f2e795bb25d0}, {0x000766509eec95e9,0x0005a452259c52c8,0x0000000000000000,0x0000000000000000}}}, + {{{0x0005f6dc2c523bd7,0x000003d082f18f83,0x000ef7ee57fec42f,0x000f2558c3c5a1e3}, {0x00038f0a9e89f93b,0x000efbe9151ca6a7,0x000ed767c4685458,0x00079d486204e6f8}, {0x000a36a1a4b728f1,0x00043b885c25b705,0x0000000000000001,0x0000000000000000}}, {{0x000eabc2b52d4646,0x0006c95bf22f084b,0x000314aed2f78fee,0x000dd61877362fd0}, {0x0009044697d005db,0x0001d56d41727a4e,0x000627e07ab413dd,0x0000b5903e3cd67c}, {0x0009c224e9a02779,0x000806739d1d3428,0x0000000000000000,0x0000000000000000}}}, + {{{0x0004109673b0becb,0x00029788f9eb9435,0x000ae5dfb3d20da6,0x00057d88527623e5}, {0x00015c844e99c175,0x0006a57ec3b40311,0x0000b594aff5aa0b,0x000cea2e84adbe7d}, {0x0007fcbaf1e84a37,0x0008e3048c293594,0x0000000000000001,0x0000000000000000}}, {{0x000f58bede275de5,0x000b21503171b093,0x0007b8e1c737aaa2,0x000dfceb6261f263}, {0x000d21e8e8701620,0x000e453d37db241d,0x000825774e79c85c,0x00066f92bc717db8}, {0x000b9d9ff5a2566e,0x000bfd4ae0bd7b6f,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=22 [{1,2,3,..,}]*([2^88]*G) */ + {{{0x000cd0f9ef7a9efa,0x0001757bbdb01042,0x0001996a380a3fbb,0x0009e651f39db731}, {0x000cdf3f08146bb6,0x000679b0b6ec6679,0x000ae26608474788,0x000d883e5a5990d1}, {0x00061924fa5e209e,0x0001de3c755c0bba,0x0000000000000000,0x0000000000000000}}, {{0x0007c1f82ebae92f,0x000ccf8ace9c6a84,0x000857d9026a1434,0x0005b0b7ad864d4c}, {0x00018f613e2210ee,0x0001165b2c86a357,0x00019fb55984d679,0x000f15401901080d}, {0x0009a0e8b3389ecc,0x000e608b509b98d9,0x0000000000000001,0x0000000000000000}}}, + {{{0x000a526cf604c662,0x0003afa9ed0f7a0c,0x0001dd685ac125b3,0x0007ff8f516f5290}, {0x000ad927c416e17e,0x000fc77cc9720475,0x000071767e1e9190,0x000f6652fcb33aec}, {0x0000f0d48cb2653d,0x00086c8bf16720d8,0x0000000000000000,0x0000000000000000}}, {{0x000404c440590fcd,0x0003377f43e4e408,0x000defb22729c42e,0x000241aec3b37e10}, {0x00092c795c866daf,0x000f4d30790a07c8,0x000075bb2425f5fb,0x0001b7cb5830a5db}, {0x000c950890875f16,0x000e6591cad66493,0x0000000000000000,0x0000000000000000}}}, + {{{0x0003de8c91c31e47,0x000a2de46bc7eb7b,0x000f3de301a0b479,0x00094aa7479aa235}, {0x000be5191502fe1c,0x000266a8b4adef28,0x0002961e0e185a17,0x000f211c2c23ddea}, {0x0007d3314a5b8f85,0x0007d3d59f2b026d,0x0000000000000001,0x0000000000000000}}, {{0x00032b1fd4f159bf,0x0003721f923e006e,0x00024a7d38b715f2,0x0000994620d23277}, {0x000ab6678b2917d4,0x000fa4a8245b0c3a,0x000912038e0b4172,0x000f6fdcf8c4b1b8}, {0x000ccd3b3eaaf19e,0x000d40f97e7249e4,0x0000000000000001,0x0000000000000000}}}, + {{{0x0007975597dddacd,0x0009d9266f6975c3,0x000599f544c22dfb,0x000c2be6db081480}, {0x0002aeb8ec462839,0x0009d49cd3b5cdf1,0x0006a8ba917fb29d,0x0008233b216f9614}, {0x000abf1a9d936c0b,0x0007878c8a45a2f0,0x0000000000000000,0x0000000000000000}}, {{0x0005dd64a0356029,0x0008c2d1625aef0f,0x0005ff56fc705652,0x000323cb9b293d67}, {0x000bd02b295cca5c,0x000c7129104d697c,0x000d83fe4eb4b02b,0x0004a4e9327cc1e4}, {0x000f46cdc9c23cdd,0x00053f946406995a,0x0000000000000001,0x0000000000000000}}}, + {{{0x000d1e43a1e16a36,0x000514cd5be788b6,0x000f3827d2f55b7d,0x0000a3f8f594d05c}, {0x0006accb980d1448,0x0004ad1e4f1f6b59,0x000e7dd3016801fc,0x0007c435098ccf31}, {0x000c6bc2e61b9059,0x000588b12be241e3,0x0000000000000001,0x0000000000000000}}, {{0x0000b8e2e65481b3,0x000aef410135fbc4,0x000c7a2e23221960,0x000fd4a513faa141}, {0x000a0357c6f569e3,0x000f4ac3eadff143,0x000f6174146f64bf,0x000c13f9a5506c59}, {0x0005ed7b5be845e0,0x000d0deb4721eef8,0x0000000000000001,0x0000000000000000}}}, + {{{0x000e8dfde28d4a2d,0x0001761a2d5208a2,0x00005b6ff5ebb541,0x000b58d09bebbafe}, {0x000a9eccec1be769,0x0004b3933ce41d83,0x000e22e31e1a1a39,0x00081430e109c5df}, {0x0005c36dffc66dc3,0x0006a93ad68549c2,0x0000000000000001,0x0000000000000000}}, {{0x000135d5a197f614,0x000b4beb4875f558,0x000374737f60f213,0x000b3391150b17e2}, {0x000fd51cce3f02fd,0x00090f492191217f,0x000bccbadc4f8a8a,0x0005d89eef2479ce}, {0x000b7145781a1c26,0x000e4d791a7d8953,0x0000000000000001,0x0000000000000000}}}, + {{{0x0006a3d38b806bee,0x0002ffcbc7fbb018,0x0004904b1192a4cb,0x00039b7f571c01df}, {0x00040717c8035f6f,0x000f75f4a725c0f7,0x0003575b1f8427ca,0x000cefd7922924c3}, {0x000c3f91c6705a27,0x000f14804d6b5694,0x0000000000000001,0x0000000000000000}}, {{0x000d6688c387623f,0x000533d1b2b7249a,0x0006a96c61952611,0x0009fa8adb7cd547}, {0x0006614cc5f19687,0x0000dc38a5b55739,0x000b43a1185de8e8,0x000489b0890de38d}, {0x000cfe16e6aaab3a,0x0009a02c0ab01066,0x0000000000000000,0x0000000000000000}}}, + {{{0x00047414fc9de104,0x000b82aed9435d61,0x00062ff16a9bd16d,0x000caf4a3b07e71a}, {0x0003ff9456ee752d,0x000d78dd65ea0d3e,0x0005bf864901fef1,0x0007bc9f42253114}, {0x000cb13ee366fd36,0x000fe6290083f481,0x0000000000000001,0x0000000000000000}}, {{0x00032088e6e77ebe,0x000f38c5e887c852,0x000f005e149cc7b3,0x00089874e1bede78}, {0x000c72dfeaf32e8c,0x000cb0a4d9cb4e28,0x000aba5da48c7113,0x000b1fe289a0af7d}, {0x0005d0dc00d3633a,0x000801c0b05c86bd,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=23 [{1,2,3,..,}]*([2^92]*G) */ + {{{0x000567a7a06de008,0x0007cf045155d807,0x000ce8bb24635169,0x000c2900cba64633}, {0x000c724297174dd5,0x000c3a3851e7f044,0x000f59548c830bf0,0x0003817a26f0d35f}, {0x0003d8b027d923f5,0x00062c2b3dd7cad9,0x0000000000000000,0x0000000000000000}}, {{0x00094cbf6924bf9f,0x000c09986d299bcc,0x000f89ccb5adf6f5,0x00099f825aee26f4}, {0x00056c1b545bb186,0x0000d22aa56595e6,0x000ba5ea3953faeb,0x0003a9580b4b6abc}, {0x000465246d4e240d,0x000d8613b6fdf7ef,0x0000000000000000,0x0000000000000000}}}, + {{{0x0008a64cd725af51,0x000d6dda66823389,0x0001f40d7d1c8516,0x000115e05fb1f564}, {0x000ad7906c2d8d5a,0x000f4efe00496ac4,0x0002d973643f7076,0x0004414f58386c89}, {0x0002d83c2e34b14c,0x000437c00d08bc7c,0x0000000000000000,0x0000000000000000}}, {{0x0004451656bebe71,0x0003f2219e2e5bca,0x000118227eacbf3a,0x0007e2cef1a84019}, {0x0002d58a5f9de601,0x0001ecfa6e192212,0x0006fb198696eb0f,0x000854826be2d3df}, {0x000dea0068fefc08,0x00074e77c2979102,0x0000000000000001,0x0000000000000000}}}, + {{{0x00048eb205a5b22a,0x0007d680f21ad148,0x0008e50bcbdde0fb,0x0006f6074c1119fd}, {0x00079f9f2e43583d,0x00065361f1d9961c,0x000463e625f26bb7,0x000f2b47c8db008a}, {0x0008c397787cd134,0x00029b36eea7ef32,0x0000000000000001,0x0000000000000000}}, {{0x0006647223894ce2,0x00087adfe036fb3d,0x00067daf1eb206e8,0x000b19b372f017c4}, {0x000a8ad33a99ef7d,0x00055c0da806ea7b,0x000c02414bd637ef,0x000b598649739b12}, {0x000feed3dd282f3b,0x0003ecce69b37255,0x0000000000000001,0x0000000000000000}}}, + {{{0x000e5f81ac137092,0x0000518f81e9a3bd,0x0009720014a7f4da,0x0001f457a02bf073}, {0x0006be074553e9ff,0x0004351eaa3a46ea,0x00022b27e32f0dd6,0x0003b488462fd22b}, {0x0006ddddacafc2c2,0x00035bfb75908f56,0x0000000000000001,0x0000000000000000}}, {{0x000987332b5b9a11,0x00044ffe94dfd9e8,0x000f9b91bc64f63b,0x00077f430dbd772b}, {0x000dfd580392aecb,0x000ddc69fb2fb67d,0x000314d2fdb69c91,0x000e754b9b9f9ea2}, {0x000f2e9c2e624f23,0x0000a3c6e677e1f3,0x0000000000000001,0x0000000000000000}}}, + {{{0x000e37a930c3bc57,0x00006f969b4cf1af,0x000df3ff74237c4c,0x000d1ea35c488b6e}, {0x000e084f47372a77,0x00065b61ccb75b2d,0x000a4e47577fd773,0x000dbd98a086f5f2}, {0x000533002b74aa9d,0x000bedfb9c8068e6,0x0000000000000001,0x0000000000000000}}, {{0x000b7aec02da823b,0x0009baf08dcbb7e2,0x000400bf0e5f485f,0x0003a3c7f9dfd4e3}, {0x0007472ece43050d,0x00091a092b1134f2,0x0004c1a0e70ff8a8,0x00037f3b75a31807}, {0x0000f51f254e9906,0x000c2183d0367614,0x0000000000000001,0x0000000000000000}}}, + {{{0x000a39c0dd80f6fe,0x000a13cb07adfd25,0x000ee3ca4d8c1a3a,0x0007a28212986a18}, {0x000c5167c4c47943,0x000beb7b7306e9bb,0x00038cfc9b26c1f3,0x000b68e5bc206604}, {0x0003bd9665b63ba8,0x000661b9d7de6c40,0x0000000000000001,0x0000000000000000}}, {{0x000c9e6e7c3229ea,0x000fb93cb721672d,0x0002429ae120fbe8,0x0007d3de1f64e3d5}, {0x000492888bee099a,0x0009c2f4894dc1d6,0x00011bda9ef3c7dd,0x000289559ae47758}, {0x0008fcbb34df4ae3,0x0005c440c5d0ce22,0x0000000000000001,0x0000000000000000}}}, + {{{0x000fc1fd2377e231,0x000257cc9235e4c6,0x000a8f0482692e0c,0x00027d24355728b1}, {0x00087856d3663645,0x000dfe4f7c7f19ee,0x000c0f9ccd7eb847,0x0001019511788244}, {0x000c4838c54b2a98,0x0008b89d2d46944a,0x0000000000000001,0x0000000000000000}}, {{0x000e02a1a1d2a70d,0x000b7d63b4d27dff,0x000ab61d5fa9a970,0x000d7730420f7a7f}, {0x00079f4479996041,0x0001189500cc1597,0x000981b342dd1a18,0x000b0c87b154435a}, {0x000fe5a645fb9438,0x000a1ab34fb7fc43,0x0000000000000001,0x0000000000000000}}}, + {{{0x000f438f4b100160,0x000f22d13ff0c314,0x000d55796ecb8e45,0x0000ab873dd2e2bb}, {0x000789eb71d33f83,0x0003167e0b14a364,0x00002c246513aa48,0x000b03e86d3a7935}, {0x000b2db2bb0fe98c,0x0000d70404a0ec4e,0x0000000000000001,0x0000000000000000}}, {{0x0004384c5c6b60bc,0x0002570cd19a5c8f,0x0001c33b468c19b3,0x000cbac39210942f}, {0x0007d36048a2a29c,0x000f69ef5fd4ffa9,0x000ece5cd6b0a674,0x000b1322973982d0}, {0x0001493bd4bce1b8,0x00083d4d6596bf49,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=24 [{1,2,3,..,}]*([2^96]*G) */ + {{{0x0000db5e813acae0,0x0005831117f6d456,0x0001106059c8e19b,0x000f908ce8232c57}, {0x00092d0f09782c78,0x000bd0fcb64a24aa,0x00077e3d766becf8,0x000f155f53d2f599}, {0x000389ae2fa9a727,0x000920ff877e9249,0x0000000000000001,0x0000000000000000}}, {{0x00085d510d2d4458,0x000dc73b4e520499,0x000aa68342be4788,0x0004f89c8a0ca8e6}, {0x000e8668748927b1,0x00017375ddf19eb3,0x00041d15e5f8b7ce,0x00052912af54652a}, {0x000b9a777a86a727,0x00003bbaf706d85a,0x0000000000000001,0x0000000000000000}}}, + {{{0x00017d39542a7b35,0x000d6953d2cbb145,0x00044a3ef5b5b733,0x00076565472126ff}, {0x000b2a4a1334dee0,0x0002573d17b26c37,0x0002c7ab5b295171,0x0008d328148c129c}, {0x000907f5aa2c72b0,0x00018b1d10e10308,0x0000000000000001,0x0000000000000000}}, {{0x000159666154b57d,0x0005dd9359d8885e,0x0000281b6f14827d,0x0009bc4ba475f3a4}, {0x000c32eef44696b1,0x00082b50dbdc6dfb,0x000236a9ef4383e7,0x000fd73208450583}, {0x000d190b07767db3,0x0003153c0278a00d,0x0000000000000000,0x0000000000000000}}}, + {{{0x000ba905f5456251,0x0008ee5b3d8d39c0,0x000ea2a0a4462a26,0x000032f3094457cb}, {0x0009bab36ceff80f,0x0001b0fdf3879073,0x0009dc840209bce2,0x000df0c1c8e03824}, {0x000c51d81213ecb4,0x00063c2b025e0d70,0x0000000000000001,0x0000000000000000}}, {{0x0003bb32c4b899f8,0x000ccb798bfbf249,0x00028838277f622f,0x000e5b67c2594827}, {0x0003c2c07c4dd5cb,0x000c19526a2c4c70,0x0000177dcd0df4c1,0x000457a743a1ed39}, {0x00032bea63a4c527,0x00075d1c302e78ac,0x0000000000000000,0x0000000000000000}}}, + {{{0x000fc5003828ff0b,0x000436a9841f4323,0x000c6f35f8a62407,0x0009286efc260a1f}, {0x000fbe74c4b2df5e,0x000cb3568b504bfa,0x000dbcf3548e5047,0x0005d92aaad71af9}, {0x00018241085e423c,0x0004f894d1d8842d,0x0000000000000001,0x0000000000000000}}, {{0x00075b2a3f29b75d,0x000ec555f7834899,0x00092b31a410939e,0x000b7bc223255263}, {0x000db65a25c264a1,0x0008fc1aed283464,0x0005c70ecd1a9b70,0x000d90a7a2a0ea33}, {0x000d21f2e9f14ffd,0x000fb49566dadd7f,0x0000000000000000,0x0000000000000000}}}, + {{{0x000fa5757545376f,0x000ae164cbfd55ff,0x0008a8545451f1c3,0x0002e007d0be9705}, {0x000ed2a8f4c49727,0x00097ed736254138,0x000516215e864c7c,0x000bb624fc1b83df}, {0x0000313aaf4114fd,0x000a7a8c7f0423cc,0x0000000000000000,0x0000000000000000}}, {{0x000ed76abc8d276d,0x000bfe3e74f599a4,0x00025d1f92d8b381,0x0005a3599e406956}, {0x00071869bdf5e06a,0x000ec86f625afaf6,0x000d724bbcc12cda,0x0003da7516890dd1}, {0x0009b69252163060,0x0002541f15a18b40,0x0000000000000001,0x0000000000000000}}}, + {{{0x000e1c4e632e5d61,0x0002987ad659b7d5,0x0003f7f338de2f2a,0x000b55a5aaeb06f1}, {0x000b9a60e84f26d9,0x000d10563130c6f8,0x000e760d017d58e9,0x00039e20b973fa41}, {0x0000eaafdb2f4acf,0x000501ec9c6ab584,0x0000000000000000,0x0000000000000000}}, {{0x000f4549ba5a6302,0x000a98b140b89722,0x0003e099225c2510,0x000f31b19117bbe6}, {0x00046ba7147bd18a,0x0000f540e368bb5c,0x000eacf29d33114f,0x0007e59588a01c9a}, {0x000ef0e25eb2d0e6,0x00058e4bb1b8d029,0x0000000000000001,0x0000000000000000}}}, + {{{0x00057915cb5440b5,0x00088e89a3e1eb04,0x000ed12c670e08cc,0x000eab1d89133ab9}, {0x000f615d9bc0c1fa,0x00081504d63c4250,0x00062cd084c8e8f8,0x000aaf76dbe53ead}, {0x000bf1dcc49cfac6,0x000fc7007ea0b885,0x0000000000000001,0x0000000000000000}}, {{0x000472352fc50515,0x000fa2123835c747,0x00067bab29e80692,0x000cca008379c2a8}, {0x000799065aafbc2e,0x000a605d2e32da97,0x0002283421bbbfbd,0x000dbdc2e1151243}, {0x0007a9d899c126b9,0x0007467ce3f8d643,0x0000000000000000,0x0000000000000000}}}, + {{{0x00000c97f4a1ee70,0x0000a55fa6cdb3e0,0x000fd5fcd60595ed,0x000522bda02a23c6}, {0x0000361844a1d76e,0x000c6c179ebaf8c0,0x000a6ccd0a47af40,0x000071e2a1156aa1}, {0x000a1b0fc4eb0062,0x000bb4c1c5314a2c,0x0000000000000000,0x0000000000000000}}, {{0x000c048376702b16,0x0002ef5b4e8123cd,0x000a7d67834242a3,0x0003bc3accb0fead}, {0x00007e65ed32fc2a,0x000b8b44e6e71194,0x00077e9aeb1712aa,0x000039ce4f895a59}, {0x0009d43ed708cfeb,0x0004254957cd1ca1,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=25 [{1,2,3,..,}]*([2^100]*G) */ + {{{0x0009a3fb62f03e91,0x000fc3cfe3b9a1c2,0x0008c5a528bca033,0x00096cd7b4bc3e3f}, {0x0008c4bd134e2233,0x00065224c739c3eb,0x00071ec25548c0a5,0x0007b0fb17f6f014}, {0x000aee1015fc6579,0x000448c4d69b6d18,0x0000000000000001,0x0000000000000000}}, {{0x00074be708f600f9,0x000042a14b550a00,0x000f8e6b95a52425,0x0004e9813f438c42}, {0x0005181004aa1017,0x00010cd9a834ae43,0x0002105b1b67e295,0x000141438bad8cdf}, {0x0004d11308ec5ba9,0x00034300e8c08dc6,0x0000000000000000,0x0000000000000000}}}, + {{{0x000625d111480c24,0x00034cdcf3505fb2,0x000c306874b99629,0x00004410981e8fcd}, {0x000492bd0a650027,0x000a534a84249eb3,0x000e1326b6bb40b6,0x000ebe5d29140c32}, {0x0009956b2cb2ca52,0x000b8c77c7225102,0x0000000000000001,0x0000000000000000}}, {{0x0002b4e077c5c4dd,0x0008846314442efe,0x00066618e794431d,0x000a933fcd3eeea2}, {0x00006644159656a5,0x00022dc52fbda24f,0x000542f82f45dda5,0x000e0e5075c9d412}, {0x0002aba0fff34a66,0x000d69512c4a1d9a,0x0000000000000001,0x0000000000000000}}}, + {{{0x000dc5b949f6aa55,0x000cb79872016ba3,0x0001df5e18d2889c,0x000aebf5e0129254}, {0x0003a4cd20b4cdbc,0x000f30108963d6c3,0x000c0dbc46a1dad1,0x00042c0e39b6755f}, {0x00007fa126ef9e69,0x000805d500d36fac,0x0000000000000000,0x0000000000000000}}, {{0x0000b5e7bd19e5fb,0x000b3765e8dbbff9,0x000e491cc2deb8ec,0x000ab995d314c068}, {0x000b4e810513ad31,0x000b50dc0fcca181,0x00029580c1e05269,0x0006b6453c858930}, {0x0009a98b2de5a7d2,0x000b386f7a77183c,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d861fc542368a,0x000a737b3c184cd7,0x00014a6e3b95c425,0x000f514e85d4a651}, {0x00098b665bb45532,0x00066a39b08b87e5,0x00008dbdbcbbabba,0x000ba64b561fa462}, {0x0005692509520864,0x000281de8b31e205,0x0000000000000001,0x0000000000000000}}, {{0x0001bb6a74473c21,0x000932e76a8c5ddd,0x000c6ee633cc0f66,0x000f68d0c546bb80}, {0x00006828f4e0c911,0x000b2a4276c213a2,0x0008cb204a16b2ce,0x000d38c09aa1cbe9}, {0x000f3ebeebcc1671,0x00032c7a684ba9a6,0x0000000000000000,0x0000000000000000}}}, + {{{0x000a3463989cd762,0x00035114b160b22c,0x00057f2d520e3cc4,0x0000ff788707011b}, {0x00068b1a346a61d3,0x00084618b8d69eda,0x00020c04008115fa,0x000dfeeecaa806f5}, {0x0007e08436a14e30,0x00005f68bc839ccc,0x0000000000000000,0x0000000000000000}}, {{0x000ae58e3c998f3f,0x000951d35d5af6b3,0x00038625415f29bb,0x000fd087552cd755}, {0x0002087ef7e8ab49,0x0006b067b5de9ebd,0x0001e74110309c17,0x0007b224505a1ece}, {0x000ba962991a5a2d,0x000b8879263dad03,0x0000000000000000,0x0000000000000000}}}, + {{{0x0001b7e0189fcda6,0x0008775ba885f2a7,0x000b98305b9915b6,0x00019b2753769a90}, {0x000638d87ac0d10c,0x00083c77c18f7acf,0x000d23964d02af25,0x000de5be92026e04}, {0x0005a30998f85294,0x0002d2bb22f8803a,0x0000000000000001,0x0000000000000000}}, {{0x0000daae09876e93,0x0007b9f1b9b10415,0x000e48eb13c50096,0x000cf9ccec3e5c4d}, {0x000f7b6158629895,0x000aa201ea7d90f3,0x0006e88c0cda29f8,0x000f4c0d70150c9a}, {0x000ee70bc97d1c62,0x000b8e4fd0f68d56,0x0000000000000000,0x0000000000000000}}}, + {{{0x0003edbb844d6c8a,0x00076a792ccd3b41,0x00072527a7c1564f,0x00055b682778d6f2}, {0x0002167ba3cee45b,0x000d96d43a6e138f,0x000806538c932f15,0x000d4892afee6363}, {0x0002b82f06ed7c45,0x000fee287b4614b8,0x0000000000000000,0x0000000000000000}}, {{0x000953f4fc1bb9d4,0x000e995150d18cb0,0x00067e23c2e107a5,0x000bfba071a733f6}, {0x00008ca46066c2e8,0x000cfb49871d6c61,0x0004ece39bb5a648,0x00040cf34f514816}, {0x0009b9250336996f,0x000c69d6e08146e9,0x0000000000000001,0x0000000000000000}}}, + {{{0x00008e517921a752,0x000ab87a6c13d140,0x000c4597b07c5d69,0x00074a68c66db12e}, {0x00019ca40dec9dbd,0x000a617fff4579d7,0x0005876131725395,0x000d09e3b946e383}, {0x000d20c852478942,0x00087982ecbef742,0x0000000000000001,0x0000000000000000}}, {{0x000589886da1602b,0x000c3fc9ae2bbd5f,0x0002126ee978ba22,0x00075595e212b5ab}, {0x000a2389b739eff8,0x00063595af9d6707,0x0000f9787d12dd72,0x0003b014c3309267}, {0x0002f506a0067880,0x000f67060764da69,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=26 [{1,2,3,..,}]*([2^104]*G) */ + {{{0x000365108b90f9d0,0x000f80765f67fb1e,0x000b1b38d141aae9,0x00024d697a9407e4}, {0x0003f9693e7ccc84,0x000a50e7d291d93e,0x000cd34385c13c5b,0x00066fcf73c9d94e}, {0x000598f4a80eb0bb,0x000b721c4d4c290d,0x0000000000000001,0x0000000000000000}}, {{0x0000fb9a3bbeb3c7,0x0007ba326d546e3b,0x000a848cf094c6d2,0x0000e41609d2dc18}, {0x000266f0069ca46c,0x000c4aef799231b9,0x000abacbdbead081,0x000b272ba1959d4d}, {0x00049b7208e216ce,0x0002ba83cc03eccc,0x0000000000000001,0x0000000000000000}}}, + {{{0x0004c0998b5250d8,0x000c867c43b599d6,0x0004c9f6ac785a2e,0x0004ec8b59f29f0d}, {0x0004df16ae8c0faa,0x000b8d8f78201760,0x000bc38bb59089da,0x0007384039822772}, {0x000d1571c6e88e81,0x000a7b7d4e8e0cf3,0x0000000000000001,0x0000000000000000}}, {{0x0007bc572ea0f919,0x00064539b5eb1047,0x00077d71bc88d22a,0x0004dc62d769223e}, {0x000adfe2b562c973,0x000173fab241cdb0,0x000603370ddf3ff3,0x000f70dbbbbd997d}, {0x0008a88a56d59561,0x0004be64aafc3299,0x0000000000000000,0x0000000000000000}}}, + {{{0x00045d30a00d827f,0x0006bbe20955e997,0x000cead0ee1643eb,0x000fce2db7c5ac6f}, {0x000ad1d2efb30bc2,0x000d9fbc667affb8,0x000bb6fb3983d3ef,0x000f70a834538091}, {0x0008384fd172dade,0x0003f32390fa3030,0x0000000000000000,0x0000000000000000}}, {{0x00022aae53d2000c,0x000caf0273baad84,0x0004a6a878c712a8,0x00002615beecdb4d}, {0x000b0e868dba85ad,0x000964d03b5c6537,0x000cbcf8b6d0fd98,0x0006a16ca9f74a32}, {0x000552ffb8c5dbeb,0x000bf0f20d682f2c,0x0000000000000000,0x0000000000000000}}}, + {{{0x0009a64c8deb9f4b,0x000532674c0fe944,0x00001e88fe681603,0x000b8697595c6e13}, {0x0008cf6f513d4913,0x0008c1e3203b6d47,0x000b68db28573518,0x000bdfb9fd4390cf}, {0x0006601496c4bb93,0x0000633f388af7cc,0x0000000000000001,0x0000000000000000}}, {{0x0005258fb2317523,0x00040dacae0a8b9a,0x000ba0560abb741a,0x0008bc6a795d005e}, {0x00096caa47999397,0x000ff04fef1c0b24,0x000b0926ddcefe71,0x0008f281ff3947c3}, {0x000027cc7cc93f3d,0x000e78773c9a3f23,0x0000000000000000,0x0000000000000000}}}, + {{{0x0001c60cc85e7257,0x00011fa52f42f768,0x0002f380ec8fe41b,0x0004893bc07fb9d3}, {0x00034451f49fbf7a,0x000388023d1b4705,0x00079c42053de8b9,0x000cf909c04d42cb}, {0x000eca57f6797b3a,0x000cf34d35027c43,0x0000000000000001,0x0000000000000000}}, {{0x00086008351763ff,0x000f9e1103300427,0x000991707dcb32e9,0x000efb21a41fbb4f}, {0x000b5649d55978db,0x00004e4d7b3fa6d3,0x00012196968dee3a,0x0003561c60989b74}, {0x0009c91c53ee540c,0x000b9823a2ee0db7,0x0000000000000000,0x0000000000000000}}}, + {{{0x000f6a15601d1f8d,0x000406c4591dc921,0x000b36c8aaaf7c15,0x000834fd3b0d0813}, {0x000e544ef9e76287,0x0002fb609294a18c,0x00008d9bd0198775,0x000cd4816092b24d}, {0x0008b097d39d2d32,0x000b3a5b9f00f218,0x0000000000000000,0x0000000000000000}}, {{0x000da9d6f0979e9d,0x00080741dad104cc,0x0004ee619b7637d2,0x000d71560f5a9cc8}, {0x000b897bb554b4f3,0x000890a210367054,0x000aff63f1f61c3e,0x000cb92963c20784}, {0x0009317afc9acc43,0x0005c1dadb0d3e30,0x0000000000000001,0x0000000000000000}}}, + {{{0x0004debc7af95096,0x0002bf8bc26e7cbc,0x000070bd1c43e146,0x00017181a62e8acb}, {0x0001c46b5339f3b4,0x000a2c14c9b646ef,0x0005de576b9f4c81,0x0004f9a155e97645}, {0x0003cfbba219f7e3,0x0009fb1fd4f92031,0x0000000000000001,0x0000000000000000}}, {{0x000bcf11449769d9,0x000ea5acc7cb996d,0x0006bdb653213f43,0x0003a2cf9c396c5f}, {0x000777fb3075bc28,0x0009fab46fa97f51,0x000d80cd600dc1e5,0x0008ba6a57a9d490}, {0x000c9975b8111175,0x00067a2ad8270658,0x0000000000000000,0x0000000000000000}}}, + {{{0x0005e67b8d52ab83,0x000eb2049665d86d,0x000b56e1ced19993,0x0009c1fc7a62ba87}, {0x000276fc5cf75dfb,0x00054f5dad4712b6,0x00089fbe0548bd15,0x000d1ee24125ecba}, {0x000176a53fa18f5a,0x0004e18796b5267e,0x0000000000000000,0x0000000000000000}}, {{0x000a0f1a17a9eb45,0x000584e4e5f968ad,0x0008e12a3e089107,0x0009c73cd6a2ba69}, {0x00002e23b2a1f1ee,0x00028e9adc43a76e,0x00062c6e3d7526f4,0x000d0557ab8af09d}, {0x00058b1d337cd537,0x000000e54434b827,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=27 [{1,2,3,..,}]*([2^108]*G) */ + {{{0x00036c3a6abc7042,0x00004a11953f80fe,0x0000b4cc57cd15f7,0x000df44d3d3a8bb5}, {0x00015b5099398347,0x00081f3a553789e2,0x000d0115f2bce30d,0x00090b7f91f0853e}, {0x0007ec29320d53ac,0x00085b63e7bfbe8d,0x0000000000000000,0x0000000000000000}}, {{0x000cdcd80232c6da,0x000d8fc636cf5e56,0x0006e4c3d9621241,0x000b84a86812f9d5}, {0x000287741d3de81f,0x000ab3d77eb50a79,0x00048627cc80386b,0x000a1901afee8f37}, {0x00095591fbf5ceb2,0x00080aed0c8140dd,0x0000000000000000,0x0000000000000000}}}, + {{{0x0002fcdd07efc38c,0x000c9ad8b1dbd166,0x000af6b6e1566c06,0x0005c4ad28998a9b}, {0x000b12d2005dbca4,0x00009acb17fcd947,0x000af2e6bf7b35f6,0x0000b8a8aba325eb}, {0x000302e3f599df52,0x00080d2bf9b973e4,0x0000000000000001,0x0000000000000000}}, {{0x000aebd112a3c0c1,0x000c408868630c25,0x000af7c4f6ba5529,0x000d49e0f5657b1a}, {0x000da3fa70b84c0f,0x0009f530044d86ec,0x0003f7ec59dce6d3,0x0004bdaf73433951}, {0x00022dd61822c292,0x000466acb0786ece,0x0000000000000000,0x0000000000000000}}}, + {{{0x000bab3d6168fc90,0x0007cbb6b2be9857,0x0001cf2ccb017656,0x0001630304c6ccfa}, {0x000ab9344dbdd28a,0x00075f0fec3093b4,0x000ff2494e7a4029,0x000ca3bad1c9c568}, {0x000b17aa4b7bc2e4,0x0001be8844e06c12,0x0000000000000001,0x0000000000000000}}, {{0x000748f359f51efa,0x0006b016fa4643bf,0x000de784a5c3f1e9,0x0006412db1ab3996}, {0x0002a042b01eeae2,0x0005a0d3a2a424ea,0x0009b9a7aeefe348,0x000b209614c9d52b}, {0x000556ff1e6afd00,0x0001e2290dec8fe5,0x0000000000000000,0x0000000000000000}}}, + {{{0x000dedb27f20e8c4,0x000f535a0fc33855,0x000788ccd8803e8a,0x0001f7d6e10cabd0}, {0x000355f889d7fa1f,0x000583e3030487ee,0x000f3dd1885d800a,0x000f09ae9a4a2fc9}, {0x00054fc302887b5b,0x000678d91181d3a5,0x0000000000000001,0x0000000000000000}}, {{0x000b146d6cdca631,0x00046652f280d553,0x000e0b73d63dfaac,0x000399cd0d77869d}, {0x00057ba5ffe6aa8a,0x000ffc1da65c61b7,0x000738771cf6c9ea,0x000620ae124834d2}, {0x0006504def184b95,0x000d761c974cb47f,0x0000000000000000,0x0000000000000000}}}, + {{{0x000b12824a32cfa1,0x0009c552049f6e5b,0x0002cf47aab27c7f,0x000c49affe61e35a}, {0x0002247309f28abd,0x000d8f5d3d157087,0x000c6b864fe18c22,0x000addbbfad216e7}, {0x0006e83c0b6f0f26,0x000fd5df730de2cf,0x0000000000000000,0x0000000000000000}}, {{0x000d5ed49aa6d720,0x0003b8bf56601c9e,0x000932a05f265897,0x000b6e7997e501b5}, {0x000a12ea7ac82871,0x0009ee973ffc91a5,0x0008bc4a89b5c6c1,0x0003ff7cc4ea3cde}, {0x000dd04f7b689969,0x000d1dbe61024f53,0x0000000000000001,0x0000000000000000}}}, + {{{0x0009c8cbe80fbbcf,0x0001820704f59919,0x000a2837d2432345,0x0001d7c57c052221}, {0x0004dd7abcf6f7fc,0x000ceb0c9ac384fa,0x000cbcbae19fcf70,0x00044465c2e5cd22}, {0x000a11ffc8ca98c5,0x0007b72231b3018f,0x0000000000000001,0x0000000000000000}}, {{0x0005516f6500002e,0x00015b8dc7f05ee2,0x000111229aa6356e,0x0006bd0e54525c30}, {0x000dcfc5ecb4c0df,0x000355785f844693,0x000ab74792af79c9,0x0007357c34213765}, {0x00008a7b5867734f,0x00069a6d820a3083,0x0000000000000000,0x0000000000000000}}}, + {{{0x000b3a34b0a583e8,0x0004109d1c2a05b3,0x00056d185c8227f4,0x000c30eb0f8b309e}, {0x000e5f0d836e7213,0x0002fdae4ffef15c,0x000257bcdd2d7309,0x000dad1ff9a8d757}, {0x00098efcc192b734,0x000fe0e5b46f1d9f,0x0000000000000001,0x0000000000000000}}, {{0x000ec3057c0b1268,0x000e1eea4fea07c8,0x000ac5ab201c4cbc,0x0001b75086654bcd}, {0x000e612c2aee9474,0x00065fa077070bf8,0x00015dd97afeaab6,0x00088802f9979ef0}, {0x000c9b3109eb556d,0x000e513d135fae08,0x0000000000000000,0x0000000000000000}}}, + {{{0x0009b9b53c1efab4,0x000babd37156ff65,0x000a115d2c7f8338,0x0007371c9d1175b5}, {0x000a353c22d6aa92,0x00079ee37be5b07d,0x00020293585421cb,0x000abe2b0a938ac9}, {0x0003622f3d489e47,0x000ac9ccd5811b36,0x0000000000000000,0x0000000000000000}}, {{0x000cb54f0f506ac3,0x000feebf83fb7441,0x0007d9fa2d5527b4,0x000b40376d4a3597}, {0x00045e4619c87f8a,0x000b913b27d590e9,0x0001da0e8861075a,0x000dd8fb707f389b}, {0x000140b6fe0beb49,0x000bf7392dd17235,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=28 [{1,2,3,..,}]*([2^112]*G) */ + {{{0x00082a01cdf12457,0x000bf550e3442670,0x00027cfd7b191616,0x0009bf54426bd9ae}, {0x000375f468d0ec29,0x00095e63540487ca,0x000f558b93aa7dc6,0x00028f48edec9322}, {0x0007ee742818f059,0x000d23aca5b08895,0x0000000000000001,0x0000000000000000}}, {{0x00018972085008e4,0x0009e445a0130711,0x0005bf246e5348cb,0x0008ccf1f5c183c6}, {0x000f2e9a40aeb3fd,0x00087abdef0fbda6,0x00050f5daf09cee0,0x0003e33344ee90c4}, {0x0004044243abe107,0x000b8f02a065d1a3,0x0000000000000000,0x0000000000000000}}}, + {{{0x000b66d418145236,0x0007896ea70c3fff,0x000cb17d54fc5491,0x00042a641eaf6e4d}, {0x00096b15be10c7c6,0x00011efe5f993282,0x000c04930829e9c6,0x000a5f18e8613cde}, {0x0007985a51a7c38d,0x000b8f3536d908ab,0x0000000000000001,0x0000000000000000}}, {{0x000ce50b447f989e,0x0006725435f6e48e,0x00060505d7413d04,0x00051fa907efc4e5}, {0x00091cc601ad28a5,0x000eeaf4b18fed33,0x0002e1a4338a8549,0x000b61868d3372c5}, {0x0003a511bce70bb6,0x000e1f5c8d75eb9c,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d68d2a3e7e05d,0x0009bf6f65a63322,0x000368db479d7794,0x000e22f5738f46ed}, {0x000947212d465e52,0x000bb783e24758d1,0x0009d33d677a59c8,0x0005609046041b23}, {0x000f6497a9c2f277,0x000e7a9be5339a8d,0x0000000000000001,0x0000000000000000}}, {{0x000804d780847503,0x000fb6bd5cd190b5,0x000d58769b6bfbeb,0x000a5b2366d25685}, {0x00084206ac283f9e,0x00045e93a909d14a,0x0007818e03b612f8,0x0005061fa312c680}, {0x000501efdeb98070,0x00043cfa3670b66b,0x0000000000000000,0x0000000000000000}}}, + {{{0x0008296f30c9bed5,0x00099a4bb11c1f32,0x00015b4084960501,0x000c50ce53a7ca7c}, {0x000f50a2c1da281b,0x0002c0e34f682873,0x000f21f441021705,0x000e9f354fbc9c5e}, {0x000d7990a0bba954,0x000ca402432a326c,0x0000000000000001,0x0000000000000000}}, {{0x000e6dddd976d76d,0x000a57e55cac7b2b,0x000da37392c8a3b8,0x000fecd4ec1dc93e}, {0x00009cf4f78c92e3,0x000ff689fefedf3f,0x000abd5033740521,0x0000df4087ca092d}, {0x00002763eb9e4e11,0x000689f3f329b79d,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d09236f94e981,0x000f3b87c2492089,0x000090559806590f,0x0006b638cb962e83}, {0x00043caf0b4cf40f,0x00066f61c7a2faca,0x000edaa5c1d246d3,0x000b19d9f9d7b0bc}, {0x000d3b97c04cafce,0x000f23bf9f27399e,0x0000000000000000,0x0000000000000000}}, {{0x000b2a804eb31689,0x0009a2a7437938a7,0x000f3c9cd990c2d5,0x000b68b7f843ec1e}, {0x0005141dcc07de04,0x000fdd1e7d56eecb,0x000a099220b61db4,0x0004184b463b9f76}, {0x00076a6a6fd2bcb4,0x00067d87f98ad9c6,0x0000000000000000,0x0000000000000000}}}, + {{{0x00025736286ff497,0x0002401f1271e328,0x000981dcd7607683,0x000cfb7ab6548084}, {0x000df4da03d9c990,0x000d822ebf4ae8a1,0x00099db5f23d4a99,0x000f6315e3fb9894}, {0x000021f262e80d80,0x00027457b91d5ecd,0x0000000000000000,0x0000000000000000}}, {{0x0007a0ed90d98205,0x000cf6f6b6c517c8,0x00057c5ae11bc6b8,0x00000efaef845be3}, {0x0005e74813e365be,0x000c8e0106700aae,0x000d776c99ef5ca6,0x0001e3d37bff71e1}, {0x0003a89779b96572,0x000c15ea2a31bd25,0x0000000000000000,0x0000000000000000}}}, + {{{0x0000092959940ede,0x000bb4c7bf2ceef4,0x000d604555c30636,0x0003fe5635d8d5c1}, {0x0008cdef0cb902ce,0x000b6f22235732b0,0x000fddf6f7cfc6cb,0x000369004df1a446}, {0x0004ba65bcff1dc7,0x00020adfd4756b77,0x0000000000000001,0x0000000000000000}}, {{0x000afeaa9dfdfa89,0x0000b80d92606c1c,0x0006888e19c65eb3,0x000bf9bb4db51900}, {0x0008e84280b5d191,0x00077a1cb4c3d007,0x00055fc83156ea4d,0x0009f40279edea0b}, {0x000aa5223480bee2,0x000f264b4f70afa9,0x0000000000000001,0x0000000000000000}}}, + {{{0x00054382d016c8d9,0x000ec7826f7b17bd,0x000dce64f2832c36,0x000193ae22a16680}, {0x0000aaf6a85c2ab2,0x000f20270252cc0a,0x000f317cc1335b32,0x00003743776e2afb}, {0x000a199000deb474,0x0006bc61591f25f9,0x0000000000000001,0x0000000000000000}}, {{0x00084eebf2800729,0x000608b06a4eb61d,0x000b23e73968bb72,0x000a3ae82e886104}, {0x000d27c8605d2992,0x00033bec6e418a91,0x00029d45f2b49e6e,0x000bd1f4a3f4a9d8}, {0x000bc4ceaeb2f044,0x000c63b1ef09fa28,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=29 [{1,2,3,..,}]*([2^116]*G) */ + {{{0x000826845611f97e,0x00015b6b1ee54a04,0x000608b1dc092400,0x0009050925698b8a}, {0x00031b5e532ada13,0x0000c41c46df4acb,0x00090c116e05bee3,0x0009642c127307d1}, {0x000365a48b566eca,0x00046d5c3cffa21b,0x0000000000000000,0x0000000000000000}}, {{0x000b8836b9754189,0x00079ea005768621,0x0007bf51510520f5,0x000bbc0ca43d38cb}, {0x000c9fe21891a0a4,0x000242b093687446,0x0006d618feab8811,0x00047a921f31cacb}, {0x000cb09d3cf611aa,0x0007d8fef9a8efc5,0x0000000000000001,0x0000000000000000}}}, + {{{0x00042c81af50749b,0x00081540e019366d,0x000d6072e7b7487e,0x0004156c32da913c}, {0x0003df1e87478e7b,0x000880f5ccb21742,0x0002347ca344dd54,0x000d15da2c269018}, {0x000993e2887d2337,0x000379604cc23f8d,0x0000000000000001,0x0000000000000000}}, {{0x000778d40c2ec9c0,0x00027ec9dd1808f9,0x000dcd7b63f43450,0x000cf65f198a63ab}, {0x000d3a7a4c38803b,0x000476f99f1130c2,0x000d6b9c1ea5019b,0x00034f67377e991a}, {0x0009047dfa9f5ad1,0x0005dc80641e2fd9,0x0000000000000001,0x0000000000000000}}}, + {{{0x000de0d9dfa7af75,0x000eba7ea5710283,0x000dd5435234652c,0x0006f821b8a36856}, {0x000c319e00261b58,0x000ed079e56ce309,0x0001099e0f75ac31,0x000e2442020d51ff}, {0x0008b83fa0c077ae,0x000e6fc85e1f8724,0x0000000000000001,0x0000000000000000}}, {{0x000872b798445d10,0x00032b311d3108af,0x0005040c97d2ca2a,0x0005703d4fa4c2f0}, {0x0006980d5eb27761,0x0005f074a536c8c1,0x000181395daa1e3b,0x000b672dad89bda9}, {0x0001f3d94395bd4f,0x0004b4c4a2c81ef6,0x0000000000000001,0x0000000000000000}}}, + {{{0x000924e7856ff846,0x000ee4f61f664ccb,0x0005ac67cb02e404,0x00050da76b002de5}, {0x000c4537e3c3c875,0x000c36c052b6b43f,0x000b204b2d5ce01c,0x00088e7f6d0e0c5b}, {0x000c188bbf930fde,0x000168056f87d909,0x0000000000000001,0x0000000000000000}}, {{0x0001106b668bd3a0,0x0008dce76203aabd,0x00002fff3110182e,0x000f7d1e1307d3fa}, {0x000347101339296b,0x0004a22e456ed2ca,0x0002d011b668eed2,0x000b79cf95e5e410}, {0x0006693b0681d10c,0x000355f94e08ac6c,0x0000000000000000,0x0000000000000000}}}, + {{{0x000323242b9413a5,0x000d8925b57cdbcc,0x0004d31e6960afac,0x000cc1c8075e88b1}, {0x0003a4d853d5880e,0x000c2d17b4e21339,0x000c35a1d02b3405,0x000f7f4eb22a29f6}, {0x0001b6570763f945,0x00008a38d9e91699,0x0000000000000000,0x0000000000000000}}, {{0x0009e262a8faf74b,0x000d89cdb707d091,0x000c28362e27b3cc,0x0000a8d2e31adec3}, {0x0004f2e5340b0d97,0x00076d44ac11f1ff,0x000ddee42bd388ab,0x00052165e718528c}, {0x000c2384a7cb055f,0x000e3bd81cae87a8,0x0000000000000000,0x0000000000000000}}}, + {{{0x000e0ef29067ec5b,0x000b3efb1759268e,0x00093d33d241c82d,0x0005ebc6b912da50}, {0x00004cea7d557bb1,0x000a95c0c2531329,0x000338d1728bce52,0x00023e934774d703}, {0x000bdaa179ff6232,0x0009c05a25267ea4,0x0000000000000000,0x0000000000000000}}, {{0x000b3f1bf490cbf7,0x000ec049cf21d24d,0x0001567c730a18c0,0x0008c3e0f359d391}, {0x0004ea1bf7eca8f7,0x000252d4d89f9aa6,0x0007a2e5b2ffd6d4,0x000e70d5197d3cf7}, {0x000ac046e420f1fd,0x000f82fbaabfd6c4,0x0000000000000000,0x0000000000000000}}}, + {{{0x00023d833bf81b28,0x000064787960d20f,0x0001e23da2c1a82d,0x000fca0df31fd1ab}, {0x0000d67beaa32632,0x0009e45d2648f548,0x000d563bb162f9bb,0x000110e02089d434}, {0x0007082d3a10eef0,0x000cb7b7735d1d64,0x0000000000000000,0x0000000000000000}}, {{0x000d95b89701e6ec,0x0003bbe61d29d940,0x0001c7d5b4e68b4d,0x00012a5ad78df4bc}, {0x00047d83302cabd6,0x00011140b2809827,0x000211a754f62795,0x00041d43610e16e7}, {0x00099e665f0dec95,0x000ee6baca9f0f39,0x0000000000000001,0x0000000000000000}}}, + {{{0x000657257b2d9252,0x000915208861aaee,0x0008adfc02b5d4bf,0x000f78398b2a8792}, {0x0002cd1929e3951b,0x0001878fc66ac2d8,0x000a1972453f26a5,0x000c0ebd963c67c6}, {0x0006feb8829e6f9c,0x000c986a8aecc7ab,0x0000000000000000,0x0000000000000000}}, {{0x00030636d8df74f1,0x00011de6a5beb09f,0x000247b37675f6af,0x0003d122a04301fc}, {0x0003f577167d7789,0x000a69addd4d974f,0x000f8be983fc60de,0x0003627055a8d35b}, {0x000c83aaf95c80a8,0x000a9aa21f06b151,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=30 [{1,2,3,..,}]*([2^120]*G) */ + {{{0x000c1e136664d27c,0x000e853cf04eac1d,0x000599f98901c4f5,0x000f0e3ecbc44867}, {0x000ee5a12a7f834f,0x000066c152851c12,0x0002df97ca61be6f,0x00027153da2c7383}, {0x0003e882e14acdbe,0x00030b87567338b7,0x0000000000000001,0x0000000000000000}}, {{0x000fe8148de5b00a,0x0003a405fd56d3d1,0x000e986a7db49ee5,0x000cf7bc11101981}, {0x000a9750760e2695,0x000815cb90b6aca2,0x0009f299f5ace2a4,0x000d6b06b61bc3dc}, {0x0002e5c223b28698,0x000c0b5687880a6b,0x0000000000000001,0x0000000000000000}}}, + {{{0x000f552c0e5d59cd,0x00029aaaadcddf1a,0x000f071e91a160c3,0x000bbaf777f33e93}, {0x000696e836178f9c,0x00030ecc6d74f3bc,0x0002571349ec6474,0x000dbbec63ff9e68}, {0x000eff8b43f624e0,0x0008000d19e23a64,0x0000000000000000,0x0000000000000000}}, {{0x00060d53484cb54f,0x0000d83eff3832ce,0x00012f600dae89d0,0x00089d2df8745dbd}, {0x0008a48217cd83eb,0x0005ce0f8ae79b86,0x0004021c2c4ae44c,0x000ea980ca2b0fe9}, {0x0004146745ab9482,0x0001c2cffa33fcf0,0x0000000000000001,0x0000000000000000}}}, + {{{0x00076fd51fd99bf9,0x0007e3a2b01fa7b1,0x0001a17875cbebf2,0x0008df20ca98073a}, {0x0001c738732531a0,0x000c360b05cea958,0x0008986bad316bfd,0x00009591db5fb8a6}, {0x0008ce8516941db2,0x000cd50df495ad83,0x0000000000000001,0x0000000000000000}}, {{0x000d46b24a5b2933,0x000a4af0d09b27b5,0x000e34ef392f2b04,0x00028d0cc4e0cb50}, {0x0005bbe1270619c0,0x00002d927660b899,0x000c444a9beaf922,0x0003686effea3a61}, {0x000321e427cc238c,0x000fe609075147ce,0x0000000000000000,0x0000000000000000}}}, + {{{0x000e9dd164c62b53,0x0001878a3599a216,0x0000821091d05317,0x0002cda324ef2697}, {0x000e94950f2f16ed,0x000815b553eaefd2,0x000f8019f00612dc,0x0001930eacc547c1}, {0x0006fc4a1fd1730a,0x000db88252d52d13,0x0000000000000001,0x0000000000000000}}, {{0x00077522a6c4bee6,0x0006b12deb38426b,0x000ca869197aea9f,0x000c43193823d16a}, {0x00028f12c9d38187,0x00031f43dad5cc98,0x000728a436529c3e,0x000863d40c6f0781}, {0x000da1798bfbb097,0x0001e17a19693394,0x0000000000000001,0x0000000000000000}}}, + {{{0x000a20633820f8b6,0x0008884ce6057395,0x000b9e9ac4298b05,0x000f80c79f28e7bc}, {0x00012abb15751770,0x000ce75763d01472,0x000afbe67296f82c,0x0002a2950d9f8034}, {0x00031ca6f1179141,0x0008bb87c616f997,0x0000000000000001,0x0000000000000000}}, {{0x000f27dc8004bd5d,0x0004fc5fa5d017c7,0x0009fdb4deb95bcc,0x00051c1e39917e40}, {0x000cfbefa777d300,0x0006ebd51f3f36df,0x000089ed9696a852,0x000c58a6c0bc16cc}, {0x00093efb56723f03,0x000f77e4f7a67531,0x0000000000000001,0x0000000000000000}}}, + {{{0x00082edbf63cd0fa,0x000fb67ff0d41a00,0x00076aa53cf1522f,0x00099dda453dcda7}, {0x000bf634bcd8a3ac,0x000f09af12ca31a6,0x00045d3da6aee65d,0x0000b2e1c131b960}, {0x0001888166f3c7e7,0x000d21cb58f8b972,0x0000000000000001,0x0000000000000000}}, {{0x000f3e0321dcdf91,0x0009a8d4da7b1151,0x000e3a95788cafbe,0x0007071e39c010af}, {0x0004b05cb3faf8c8,0x0008a702fbafcfc0,0x000618742c775b70,0x00068aab53d65b3b}, {0x000b27ffbb84f938,0x000a7508491b708b,0x0000000000000001,0x0000000000000000}}}, + {{{0x00020328d4b15dd1,0x000274b581eaa62f,0x0008fb2a285d269e,0x0006ea89604b1779}, {0x000933aa53ad75b2,0x000fa62691d5119e,0x00067e03e002a949,0x0000629215018ba1}, {0x000ae2796195dffb,0x000282dc1f93eae4,0x0000000000000001,0x0000000000000000}}, {{0x0000977c61f7743f,0x0008f7654950f798,0x0009f0fcf77422ba,0x00070562b7dc1d4c}, {0x0008f0b2f76176b9,0x00094ad6c12de606,0x000d53dd34579508,0x0001fc63f78fe569}, {0x000a90b5214981ae,0x0005ab902dadf9f2,0x0000000000000000,0x0000000000000000}}}, + {{{0x00006fc86d7474a9,0x000491c759885f54,0x0002d4cddc55bd2a,0x00061045c35aa122}, {0x0007a2154985eb54,0x000f0dcbe4188b45,0x0006a7e235148dff,0x0006a2535a30d70c}, {0x000e2be337d8e901,0x000cf899a19ee96b,0x0000000000000001,0x0000000000000000}}, {{0x000dc1860747030f,0x000a1d519771baa1,0x000e6bf7f8dea4c9,0x000b88d5c44825c6}, {0x0001648270d80fd4,0x000d7c088d619d7b,0x000e67f50ac4887c,0x0009d1ac72f9cc2c}, {0x000dce091aafa6b8,0x000ac9b9365de8af,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=31 [{1,2,3,..,}]*([2^124]*G) */ + {{{0x000c9884018b1bd5,0x00017e335350476d,0x000f240ea34a105e,0x0007225c0ca7c1ed}, {0x0002e60ee9bcde0a,0x0001b7a04f8d5abc,0x000d636ed201196d,0x000fee08dcde421f}, {0x000648f1c3a41da5,0x00014b37a2b18a4d,0x0000000000000001,0x0000000000000000}}, {{0x000574ca3d13216a,0x00000c8f4aa46ce2,0x0005e6cb8b142b50,0x000aeecc2cc007b3}, {0x0008b139d4602d18,0x000857b6e6fad62b,0x000703a0b8035154,0x00047dfe5be4aaf5}, {0x000e255f15b88d9b,0x000ceeb42f23b0c7,0x0000000000000001,0x0000000000000000}}}, + {{{0x00027bf41035c3be,0x000003d6c228d698,0x000ac8482db53bd6,0x000f6c6cedd6d84e}, {0x00055554b59c1199,0x000b3dd0d5c80a25,0x00098fd9a255d70b,0x00088ce8ece5b616}, {0x00010e4ff0160238,0x000e8b21f2b5b409,0x0000000000000001,0x0000000000000000}}, {{0x0009be6e93956f12,0x00028be014bad7ba,0x0007941a6f1d6c8e,0x000374aa983d3be4}, {0x0001ab03efe8a93e,0x000ecc1517778750,0x000a07f3863f0102,0x00022339ade08ce1}, {0x0002e138fb118165,0x00037ded66083914,0x0000000000000000,0x0000000000000000}}}, + {{{0x000be450955d0f49,0x000350db3612b4e4,0x0008b05c6937d0f5,0x00091d0bc00589e9}, {0x000f08134602fbb3,0x00072898cb46a43b,0x0000c3f425983612,0x00092a9319d102b7}, {0x0002adb9889640c9,0x000f69984da61b9f,0x0000000000000000,0x0000000000000000}}, {{0x000925b413e1ae3e,0x00046d808d55308e,0x0001e8e95f85495a,0x0001aa328926b87f}, {0x000e6f7c0b2cf48a,0x00097c234327b453,0x000bc1b584c452ac,0x000ac04a1db1b2a7}, {0x0002c2df96a4716f,0x0008b35f6c998afc,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c246de542c405,0x00006abed2f33bb7,0x000d46decdec7b50,0x000afeed50c509c6}, {0x0007109502cf683e,0x000fa7b0916c8d21,0x000971ce284eb826,0x000b5478a9a06ef3}, {0x000dbb05d7e812b4,0x000fa9bdf3afd0be,0x0000000000000001,0x0000000000000000}}, {{0x000c0e4a6519aab5,0x000d79de9fb97617,0x0002d46f889510f0,0x00025cb75085caf9}, {0x000f963379f4c576,0x00002dc4877679ee,0x000748161e8da062,0x0007933c7094d95a}, {0x000527ab96f198e7,0x0000a23cef9bb67e,0x0000000000000000,0x0000000000000000}}}, + {{{0x000254c4bef57a31,0x000f8fe05ebbca4c,0x000036d19011dab0,0x00092f09f97449f1}, {0x000feb784d8ec601,0x000551e5955591a4,0x00088c74b266204b,0x000fefa58c8cd4d4}, {0x000954c3d29f9c68,0x000262f04bf8acfc,0x0000000000000000,0x0000000000000000}}, {{0x0005e5c9d99f906b,0x0005ebe8f2ee55b8,0x000f127eee6b4d59,0x0005b10ba97a057a}, {0x000c7680e692309f,0x0007c47d151a8543,0x000975bcc38cb298,0x00056b40037e7ed1}, {0x00069095953732ea,0x000e079710f9d766,0x0000000000000000,0x0000000000000000}}}, + {{{0x0007ef5561843b50,0x000725adb4b17e58,0x000223554b9e6db7,0x00040d6a298840a9}, {0x00089b9987d3e8ea,0x000c544359088f19,0x000712498c4e6798,0x0009d49555742687}, {0x000531911aeb4757,0x000c25edd6bd8c42,0x0000000000000000,0x0000000000000000}}, {{0x000da2be384ee90b,0x000ed1578452ef17,0x00026ec7e64f3506,0x000d93fd400c530b}, {0x0006442c14bcb0a9,0x000bc44330eec280,0x000b7a321d894abd,0x000383284ca21784}, {0x000aabf2cbd2fe67,0x000a0b333a314bbd,0x0000000000000001,0x0000000000000000}}}, + {{{0x000deccb4cc0fcd3,0x0002b5d6f0d87e14,0x00022447f7757c4f,0x000f03b5e4d05426}, {0x0003dcf9d56f98d5,0x0001c6d571746b68,0x000648164a40c197,0x00086a775d32054f}, {0x000bbccdf9c7e93f,0x000ab95b370c616c,0x0000000000000000,0x0000000000000000}}, {{0x000ad22f9be04193,0x00049e68cea75e3b,0x0004683245da929e,0x000316e38a93792f}, {0x000965e00dbe19e7,0x000e9c59fb61ef64,0x00094c5036fbab0d,0x0009e4c2c8f5993c}, {0x000c76d4a049b3bd,0x000c781dc2d523d7,0x0000000000000000,0x0000000000000000}}}, + {{{0x00010cba8003a62b,0x0002963dead37561,0x00024e572ee261b1,0x000924c14f710c53}, {0x0003a3234879da4d,0x0000242c6b2bb72d,0x00025965319d73bf,0x000c5d438ac356b7}, {0x000eb1ea69c1467e,0x0006ea40556d55e4,0x0000000000000000,0x0000000000000000}}, {{0x0003bb0cfbfbdc6b,0x000292f755482f11,0x000b750229b1fdd8,0x0006dd9d36eb56b3}, {0x0009fd65055f0875,0x00005fbea1ad24bc,0x000b5ba29626eb13,0x0004c9855409fcec}, {0x000000d0af627326,0x000b249d561b2281,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=32 [{1,2,3,..,}]*([2^128]*G) */ + {{{0x00097f8c2da756c3,0x00065716b51e78af,0x0004d4e4ac9bb4d7,0x000be63f12ece85a}, {0x0007f2c2556ca2a2,0x0002341b0c191c3b,0x00063796c15ecee1,0x00092e302dd7df66}, {0x000d162a4ce9cb82,0x0003dfa7f8ba9276,0x0000000000000000,0x0000000000000000}}, {{0x000403973587aa55,0x000a9956dae839d8,0x000d9da7dcbd9d38,0x000d0fffb69b8acf}, {0x000544e0adb2ad93,0x000b2ad644f74f04,0x000e7d5b5de013bb,0x000f944ef674d489}, {0x000e01d0ea2d2bd3,0x0008aedd32d1ec0a,0x0000000000000000,0x0000000000000000}}}, + {{{0x0003c3743a6b3bf8,0x00083b2cea274d8e,0x000f6accb4a79b20,0x000ac9cff7eff159}, {0x000c5bd1a458b1a2,0x000af5afd8c30597,0x000e95f67ad0a34d,0x000ffcb5f547ad0c}, {0x0002c927ef492633,0x000118d70d201bd4,0x0000000000000000,0x0000000000000000}}, {{0x00025271d14dfd7c,0x000f83511be77473,0x000e33f2540532d9,0x0002d9c50e1e6624}, {0x000b9f8f4394e620,0x00085289919c8fa1,0x000d6412359d3b9f,0x000a4c00c9ead88e}, {0x000626daa054c125,0x000853e0db1f33bd,0x0000000000000000,0x0000000000000000}}}, + {{{0x000ad013227cbee4,0x0005d963a674d5e9,0x0002422839e1c90e,0x0006f11c073f8abd}, {0x00051cbde443fb90,0x000f94add68e37cb,0x000d9cdf247fb6d7,0x0003b3ec024c71cd}, {0x000015c9b5032da9,0x0007607e0d83a94a,0x0000000000000000,0x0000000000000000}}, {{0x000d4287c9083b11,0x000b167379dfa59e,0x00092871f05ea23f,0x0000a530c1fbcc90}, {0x000c9b670aaadfc5,0x000840b4f012b3f9,0x0006d74574236725,0x0005827f0582ac31}, {0x00047f13870e7720,0x0006f314a9061f10,0x0000000000000000,0x0000000000000000}}}, + {{{0x0003be466658f617,0x0009fd565e43add7,0x0004a046e438ce3b,0x0007e9edef2d69e6}, {0x0006c7f11d4e7b33,0x0009fce23db4d264,0x0007ee69cfe36cf0,0x0009d497797ff857}, {0x0000fa9f71e1b23f,0x0002d2813fdfceba,0x0000000000000001,0x0000000000000000}}, {{0x0005801d34f0db76,0x0008b9ba1d6ad8bc,0x00038f8437efa8c8,0x000755dc58d2c493}, {0x0008ea5d4147adf5,0x000454e0d19f3138,0x0005174d880f0ef2,0x0006f4ab4400ed7c}, {0x0002f97c02972f59,0x000bb7fd1f05bd42,0x0000000000000001,0x0000000000000000}}}, + {{{0x000f26521378e02f,0x0002d730a6852468,0x000763f810a221e5,0x000f48636e9cb246}, {0x0004c1b9e4959290,0x00093bb4ca5e4a52,0x000f2a6103eedfad,0x0009e3ea2bb0a0a9}, {0x00013663d597d7ac,0x000a98c50bf00061,0x0000000000000001,0x0000000000000000}}, {{0x00050b19a3ef90bc,0x0000ea69cbaa303c,0x00074f8f2fe993d5,0x000223c8be835b07}, {0x000f455e7085ef84,0x0002207ec6b0871e,0x000e48bd79a714e6,0x000daa1613ef3b51}, {0x00098f36229d3dd8,0x000c2d9134c94bcc,0x0000000000000000,0x0000000000000000}}}, + {{{0x00050a19f5342c54,0x000dc653d78301f8,0x0006bbf10f389ca9,0x0004326e54a5af67}, {0x000fdd8a13b29709,0x0008aa3fafda8b65,0x0006c2e3f7baf8bf,0x000bca4111083f35}, {0x000cdf78842a34d7,0x0002a973106a6473,0x0000000000000001,0x0000000000000000}}, {{0x000155f05f4ec88f,0x000ffd8679f932ba,0x000e00ae69c605da,0x000b7c72ba823e0d}, {0x0009e0bbb62edf8b,0x000f7bf5bd871069,0x0004e610f36c3cf4,0x000e41a06519118f}, {0x00014868b2ff5976,0x0000f7b8d8554854,0x0000000000000000,0x0000000000000000}}}, + {{{0x0003e5d502754293,0x000e9ee9e0f2543e,0x000a3731f423dc07,0x0003b8bc5bf91184}, {0x000726f4ea8f1d1a,0x0008035844d23059,0x000c7266e29e66af,0x00082574f0c5767c}, {0x0001a6dd0e57d5d2,0x0004e537115bccdf,0x0000000000000000,0x0000000000000000}}, {{0x0009f7ce1bfb6728,0x0000ccb130bc8170,0x00039a62c614cf63,0x00068e187476935f}, {0x00034de841f4a723,0x00011647b88ac5f1,0x000e09f54f07f6bb,0x000f505a8bf2adfb}, {0x000d75a5f605b64d,0x0000df9bbeb2b499,0x0000000000000000,0x0000000000000000}}}, + {{{0x000308733efc5f8c,0x000b75cdb37e83e5,0x00060b5bfda48081,0x0009f06138365296}, {0x0009688a8974b9f6,0x0005444cc05fb9ec,0x0005a67f252002f7,0x0009664675a1899c}, {0x000b6d7be11db7cc,0x000549e5e85617c6,0x0000000000000001,0x0000000000000000}}, {{0x0000536e04ec0d89,0x000ceb7897a84665,0x000b8acad3957bde,0x000ba89439f416b8}, {0x000cfde12e814bb4,0x000a77e0ef45c679,0x000f35bbfcd091bf,0x0009f3ea6cc5ae92}, {0x000f66583ff4f9db,0x0009a867f0fed315,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=33 [{1,2,3,..,}]*([2^132]*G) */ + {{{0x000487d30649cced,0x000796a5efc98a70,0x00086f3d00556482,0x000c177d81ed5742}, {0x000f3693c918841a,0x00044078e141f63f,0x000ad9ccb0cceba5,0x0005cd9ca803f396}, {0x000a3b9f81f2f890,0x000c5b4318691bb9,0x0000000000000001,0x0000000000000000}}, {{0x00076e3095e41a52,0x0001ffb6fd45a8f8,0x000a8a0715ef8788,0x000192a0b8d73d7d}, {0x00086ca88981c074,0x0000f41a80dc66d0,0x0002bbb8f279d460,0x000cb55640383488}, {0x00052b11c10c7a90,0x000053f89b04d855,0x0000000000000001,0x0000000000000000}}}, + {{{0x0002e1e8d88792dc,0x00022e3ae581427b,0x00090f869db2e712,0x000e06689d393376}, {0x000702d537bfdb1c,0x0007346bbf1a9bff,0x00090f54aeeb8464,0x000273c9dd468a0e}, {0x000c871a654e3afa,0x0007465945d8c3b6,0x0000000000000000,0x0000000000000000}}, {{0x0000e770af4a5960,0x000be4ac70e87a10,0x000797d6d911c87d,0x000533fb961a5c5e}, {0x000b8548c0001c5b,0x0009d47191b560cf,0x0009eeca65c8463a,0x000ecad37d2137d3}, {0x0000514ad716bab4,0x000f8789ad5bc27b,0x0000000000000001,0x0000000000000000}}}, + {{{0x000dbc583693bf30,0x000ef8d24b6766c6,0x00095890706d31b0,0x000c51cce3a35296}, {0x00080b8ed7618c90,0x000973ebf17cff3a,0x0009c68d473b1c44,0x0001098525e43a12}, {0x00074031f5036c9f,0x0001d33955ea92c3,0x0000000000000001,0x0000000000000000}}, {{0x0006f1c314ce3a37,0x000a4064ddf24cf4,0x00070db52569e1fd,0x000405305ea2c55e}, {0x000d5f14297acf89,0x00046ea96e034f59,0x000622a42888331a,0x000a102ad1347dc4}, {0x00088a514e007741,0x0000461db8ec7cfe,0x0000000000000000,0x0000000000000000}}}, + {{{0x00023847d8cf0718,0x00021ccb2c301d0f,0x00024b1883c46a31,0x00063cce64fb5faa}, {0x0003cc10bc090432,0x000510506a731fce,0x0009a05134986c0e,0x0003aa30a907d289}, {0x00071f165d859243,0x000eeaa5074a4066,0x0000000000000000,0x0000000000000000}}, {{0x000b1d8c9f3b369e,0x0008874f03f7bd39,0x0004a870054ed9a2,0x000756adbd121753}, {0x0001a9a0d0a37510,0x000529605385faa5,0x000c2eddf5c089f3,0x000a130a237e15a5}, {0x00074ff2cbd316fb,0x0007ee2c9d3ce137,0x0000000000000001,0x0000000000000000}}}, + {{{0x0009dbdc68f3280d,0x000657e090c25dbe,0x00024c6421981e70,0x00037a5a5d0a99ac}, {0x00037fc8c760c5df,0x00044d1a53e37845,0x000ff43dddfe06f5,0x000e4ab44983d93f}, {0x000c0915a82bb80c,0x000646c74033c3a1,0x0000000000000000,0x0000000000000000}}, {{0x00004b5d5c7ce211,0x0001372b933ba5de,0x000ea64d116a103a,0x0006df2d3d823414}, {0x000619b7a1bf8333,0x0006eac1655d2034,0x0009aeedd521f9b1,0x0006d98d98725948}, {0x000610e8ae934636,0x000efd7b215cbea4,0x0000000000000000,0x0000000000000000}}}, + {{{0x0005d0308e1127f5,0x000cf03c8e2019c4,0x000aa0237cf6545f,0x000764372510cd18}, {0x00097139d47739de,0x000a85188927140f,0x00037dc9b8b5e192,0x000b2545731c10b5}, {0x000991ea59d03831,0x00067dea66743632,0x0000000000000001,0x0000000000000000}}, {{0x000cc6866cf98c39,0x0009016f3fd54524,0x000592bcebafd818,0x0002f06cc34bdbb8}, {0x000b6cb2e7a209f5,0x0005749dab7ee649,0x00076406958e2abb,0x0009516d049dfaf2}, {0x000de5f7e8adfc4b,0x0007ce97062d4c5e,0x0000000000000001,0x0000000000000000}}}, + {{{0x0007da121df0699c,0x000ad3597a95de87,0x000181d213d52a92,0x00002da71c92de4d}, {0x000dd3803396793e,0x000c8761f332f0e0,0x0005c1e0d83b70eb,0x00048cf70527f5a7}, {0x000ceb82fe3e31cf,0x00029a56f1047f6a,0x0000000000000001,0x0000000000000000}}, {{0x000ae721531eb7f3,0x000c70c79169f267,0x000d345b58d29bb0,0x0006c1daec3533a3}, {0x000115913b369665,0x00099820ca585f3e,0x000623473863b5d7,0x0003e2b952a9549d}, {0x00011f22279d7812,0x0007e6cad8cfd481,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c384b610f9960,0x0009e42d4100e245,0x000c5264e577187b,0x0000ec202477a817}, {0x0009dc146fbb4cb2,0x000c49fc51a5dd07,0x000dd34b66b540f6,0x000418cb3114a207}, {0x000042a4afc85f36,0x000592a886f4d479,0x0000000000000000,0x0000000000000000}}, {{0x00062b5959d642be,0x000c107df28ef33c,0x000c98bc18d09a83,0x000908cb61720266}, {0x00034bfa40c64e8b,0x0005f7d00d3266ed,0x0006699785d5c5ac,0x00070fda50cded6e}, {0x000a7129a0528d63,0x000df6226a01349f,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=34 [{1,2,3,..,}]*([2^136]*G) */ + {{{0x0004a83b5020b6b5,0x00064ea6b7250085,0x000f5cc5dee82b8a,0x0007e307a44f4310}, {0x00061a979f99982f,0x0006271c95260383,0x00087bd9d4a6e7e3,0x000183121a682c2e}, {0x000a0c42c801461a,0x0009efc46dd1bbdd,0x0000000000000001,0x0000000000000000}}, {{0x000ff9d53c8adce8,0x0004cbac7e6d6ff5,0x0008a2a18c9ba604,0x000457234e0b1c61}, {0x0001b538c1881476,0x000d20849fff1d07,0x000e3333d9430380,0x0001d1326f05033a}, {0x0004a49c4e89c642,0x000e640c63716450,0x0000000000000000,0x0000000000000000}}}, + {{{0x000a7c6657e0c514,0x00010e4ba5060489,0x00003183bcaf92c4,0x00051063325bb838}, {0x000624a227afade7,0x000d611fad61ce2f,0x0001f27e1c057fe8,0x000426a808156374}, {0x00051e188cc3f494,0x000e601fb19202dc,0x0000000000000001,0x0000000000000000}}, {{0x000f5c4ba35ecd6e,0x000c838b90f28423,0x000ecc8f9f7eac00,0x0004ae3bc63ca5b1}, {0x0000a61f4eb49abd,0x000e5e94c7586825,0x00050828aa62e59d,0x000ca27ce17d2e20}, {0x000f7dcd24d94b7e,0x0004ed84ff72ff3c,0x0000000000000001,0x0000000000000000}}}, + {{{0x000dae22413fe9d4,0x000881c93ceb2c9e,0x000376b68f1c40b8,0x0004d107493ec443}, {0x000de2613f0552fe,0x000264177a2adbc0,0x00044456850f4d4c,0x000c024b1759999b}, {0x00032c490b5528e8,0x0004e4fe9cb25fa5,0x0000000000000001,0x0000000000000000}}, {{0x0002401de7dabddf,0x00056529f2c840ea,0x0006004e218ae4f0,0x00026d7d9745c833}, {0x000bc1aa8e8c745a,0x000254366c2e1e3a,0x0009a65d176c592f,0x000575f2ce2f5dba}, {0x000390121cb70eda,0x000ad4df3bd7c9ef,0x0000000000000001,0x0000000000000000}}}, + {{{0x00052f406338228b,0x00000044f05c5be3,0x0003a7061d336069,0x000371f2e7fd3e15}, {0x000ddb123a32ed82,0x000a15e8eec0c29b,0x00046b80938b2d11,0x000ba2ae38c19bba}, {0x0000c4e7466a69b9,0x000ba8e7a0607a47,0x0000000000000001,0x0000000000000000}}, {{0x000e250e34d513c2,0x0009900d3d611604,0x0002850e69a99aa8,0x000ea018e87aacf0}, {0x0008ea9b70f5d0f5,0x0009dfec50e62995,0x000ef7267ad0ad8c,0x0009fbbc4dd8a19f}, {0x000ef73af4e91334,0x000d0636506a6e44,0x0000000000000000,0x0000000000000000}}}, + {{{0x000533f2ba0eb14a,0x000b9fc6b2073504,0x00059889c71f37ca,0x000d3e3b2957243a}, {0x00033cd4ef031ee6,0x000e1fa792c82e2f,0x000936a9431aaa2b,0x00075897dee2d5df}, {0x0005c1a2769038db,0x0004c149337ba93c,0x0000000000000000,0x0000000000000000}}, {{0x000ff077c9595fa2,0x000b4e92632965da,0x00073090129d489d,0x00024c2f940397cb}, {0x0000f08747c463ab,0x000063f57ea7844d,0x000a687de4ab15b4,0x000d7bdc8db9dfb6}, {0x0000393c5c4b7272,0x0002b3cc129fac67,0x0000000000000001,0x0000000000000000}}}, + {{{0x000d85352986f86b,0x000b3bb5e1a9d134,0x00096ed674c04b6f,0x0001eaebc5869197}, {0x0001ec13b24f0220,0x0005acb88043fe14,0x0006b2f77717702d,0x0001f913c28eb4c3}, {0x0008bc0cbbd9e8fe,0x000014871dc376bb,0x0000000000000000,0x0000000000000000}}, {{0x000b182392919d22,0x0005619062a004b3,0x00084b9c0aa0d96f,0x000e6a14d38134a4}, {0x000b962e9b9dd384,0x000d2a3f87434945,0x000e17c26111d5b0,0x000cca088afb0558}, {0x0004109b67e83601,0x000ebef3372d865f,0x0000000000000000,0x0000000000000000}}}, + {{{0x000c5a0356b17ac7,0x000f616fb80668c9,0x000f7001431d3037,0x0006786061783bd7}, {0x000e2a8db044a7eb,0x000d63e80e687c5b,0x0006dba72619e19b,0x000b3f54433d79bd}, {0x000179eabd3da5ab,0x000fcebeded88553,0x0000000000000001,0x0000000000000000}}, {{0x0001156c4d223604,0x0006fa0e48c3398c,0x000d70c895f6a870,0x0000aa32def1e5d8}, {0x000a3628036e774e,0x0006fa3b42b31a93,0x0003f15e7bb3f2aa,0x000fd667e0a491ab}, {0x0002f04b61d5276e,0x0001fdac2e330e17,0x0000000000000001,0x0000000000000000}}}, + {{{0x000760d231c36820,0x0008bd2a50426c8d,0x000d4451d722fd74,0x0001877484a5084a}, {0x00019395bd1ac7d5,0x000dc03d6541ad77,0x00068a254b40eaa5,0x000dc699a962f42c}, {0x000f2ecdd481b2b4,0x000345d9badbf178,0x0000000000000000,0x0000000000000000}}, {{0x00090c94035684fe,0x000e517a9849bb68,0x0005822be91e8615,0x00067ca7e3c3e516}, {0x0004c5ebee67a9ed,0x000f03236f5438f4,0x000029ef9e45ec0b,0x0003412d001129c5}, {0x000bad0b64fd4f4e,0x0000e15f591e3c09,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=35 [{1,2,3,..,}]*([2^140]*G) */ + {{{0x00067ca62dd9afd4,0x000678b2e8cc8368,0x000d7d6c96a2abfd,0x00075f62bb6c702c}, {0x000988eb9ab34b7b,0x0007b382272a8eb6,0x0005e40ee1d17286,0x000b6f600751bff1}, {0x000ff996b4ec3001,0x00015d7fb8efdf30,0x0000000000000000,0x0000000000000000}}, {{0x00062d76a29a2746,0x000f091c80dd81fc,0x000c1a9825d4a2f2,0x000a4fb54ae9b61a}, {0x000db71a812fcb05,0x000bb96eaaa7baf2,0x000dfd9cc434e4e8,0x000b8fce567253c2}, {0x000b948eeceeb8e7,0x000abac787b7e9d6,0x0000000000000001,0x0000000000000000}}}, + {{{0x000566d2087a8f7e,0x000f8d816dab3c44,0x00068ad0a5ea555e,0x000ab76093fa3eae}, {0x000bcad51a41fb45,0x000c784a1114a732,0x0002d99cd96f3573,0x000f7808bc957e91}, {0x00022a461547dff3,0x000e9dd3f93d98d0,0x0000000000000001,0x0000000000000000}}, {{0x000f5792b3bed20d,0x0003199e50e443dc,0x000ab35921f1c5d0,0x000cb763ce7e3777}, {0x0009ec69a2c8061a,0x0004921d8bd5a1f1,0x000d3f186d49b86d,0x0003d287849a3eff}, {0x00019a1d3969ee2c,0x00097e7987e8d923,0x0000000000000001,0x0000000000000000}}}, + {{{0x000e6b3554a3f3c4,0x000c8b48d7c64666,0x0004319bb26494ce,0x00023bd53c15f132}, {0x000a4b25b7340a49,0x0002c82187e36296,0x00076cb62a70b23d,0x00013ce0a44b3a26}, {0x000e1376215ada95,0x0004bdcdd5bfa093,0x0000000000000000,0x0000000000000000}}, {{0x0006f0577c34a522,0x0002d6eb1d23f2e1,0x00074b1ae5a563bc,0x00076c1922ce417d}, {0x0008d8b56e586f06,0x0003d2111864665c,0x0007a1f4a9d1f08d,0x00009ad18a2eb5b5}, {0x000f16f69121b144,0x00055ad3dba51f31,0x0000000000000001,0x0000000000000000}}}, + {{{0x000a0990f6c14c34,0x0002ae1571f4bd14,0x000a7e981428a12a,0x0008a57064ea4bd5}, {0x0005ec2f56d89f54,0x0004fcfb513a99f0,0x00081deb029c28b2,0x000c16eb364a4688}, {0x000f6df6754a22d8,0x00039a8e7ba7c29d,0x0000000000000001,0x0000000000000000}}, {{0x000585b840875f9d,0x0006688b87eab66d,0x00061b8a4aef8f2e,0x000968d01b210ab1}, {0x0007a38c32d9fcd5,0x000170203f9469f2,0x00027ba7e65bf262,0x0003268e8f3ddf53}, {0x000d5d6a50d743f2,0x0006f76866dcf3bb,0x0000000000000001,0x0000000000000000}}}, + {{{0x00075ceb39ee406f,0x000fddc2dbf93cfe,0x00005aa3d0f7d044,0x000c04043459ab15}, {0x000bfbea051fd1e4,0x0005c86723eeca2c,0x000dd90428637a5a,0x0006d3aca9d581d9}, {0x000277709f646127,0x000d9e5fdc588878,0x0000000000000000,0x0000000000000000}}, {{0x0005fdeadee7c5a7,0x000b59b799ae3c10,0x0005e3595acc919d,0x000b6f6b2aa1f7f7}, {0x000cc519dab324e9,0x00070aa0c81054ee,0x0006840dab1fa02d,0x000ce8162c464504}, {0x0007fc117382d8fa,0x00079cc63a2e343f,0x0000000000000001,0x0000000000000000}}}, + {{{0x000f45643ca65cbd,0x0007305e42072e40,0x0006980bc47b22b4,0x00091f480c0959ae}, {0x000df17382117d00,0x000fb6755fe76ce6,0x0008195083b13716,0x00053ce928778e33}, {0x000eadd235784446,0x0005f288650fd122,0x0000000000000001,0x0000000000000000}}, {{0x00032d4f966b7e9c,0x0006ec40795011b8,0x00056106a162f5eb,0x00060472439d72fa}, {0x000ed9a6959807a3,0x000d3315f177c4b5,0x000b196cd83808fb,0x000c21f3f41dc773}, {0x000518607dcca40d,0x000920d975bf1042,0x0000000000000000,0x0000000000000000}}}, + {{{0x00043d0a4a0b7f26,0x000c9bca61488d76,0x00078d40864c9a4e,0x0009191208ac32aa}, {0x00065e2c33dbbd1f,0x0006b041d84ce172,0x00022f6c73e5e84a,0x0000caf07f551302}, {0x000e0bc76bc20bdd,0x000a43482195b22f,0x0000000000000000,0x0000000000000000}}, {{0x000f04c8745c6a12,0x0002b2bdd6ee1437,0x000b9431fd260182,0x000e54b7f10879b1}, {0x000a6b8d5c027ebe,0x0002358509530c61,0x00071ee3b953e075,0x0001d055e247c05d}, {0x000f78c21fc120f3,0x000c40b71a77f551,0x0000000000000000,0x0000000000000000}}}, + {{{0x000ca1655dd01fc4,0x00023cfcdcd83fdc,0x0006fe01dad6f137,0x000a92f448fc724e}, {0x00071e9506b3510e,0x0002c50316bacd31,0x000812e5b9c38263,0x0005b06a2041b525}, {0x000d1e51d6095bd3,0x00018f8c9f2aff29,0x0000000000000000,0x0000000000000000}}, {{0x000e8440d9f6b1c5,0x000d8f5815a76ff5,0x0000ba6e7eb4652d,0x0000cfa7a2d772d1}, {0x000e12c2c10a367d,0x000122408a9134fb,0x0006be74d3fc999e,0x0007f158ed729839}, {0x000445a86f173644,0x0008103589b3e72e,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=36 [{1,2,3,..,}]*([2^144]*G) */ + {{{0x00002ef5664a50e9,0x000569e6626b5584,0x000bb9dc4c85c34e,0x0006cac4009d6dab}, {0x00047cf68656c674,0x000e65ab973336b9,0x000ecf3e266a898f,0x000b5830a2ee0371}, {0x0007109821d57e75,0x0002643e097669c9,0x0000000000000001,0x0000000000000000}}, {{0x000e2ad77fec8187,0x0001deddfb754e78,0x0004aaa3d5328431,0x000f5938ac9d56ca}, {0x0000419e9ec29fe5,0x00089e92d324185a,0x00068c4746f628de,0x00086959a461fd09}, {0x00039e1752cc1b19,0x000f685c4efa867f,0x0000000000000000,0x0000000000000000}}}, + {{{0x000578941d3daa6e,0x0001e81a86314a15,0x000e2ec49066a742,0x00085f37e975bc97}, {0x000abd59fd20aa74,0x000b001318e5e712,0x000bdca951133a15,0x0006057f57ee1259}, {0x000dad04acbd3b2c,0x0009e7ef3153ef33,0x0000000000000001,0x0000000000000000}}, {{0x000d37d508c6263d,0x000d87a4e81e7b2e,0x0005a01a3eff8f36,0x000726730288c3e4}, {0x0009b846f52088b3,0x000f560651a99118,0x000aeef71db52e56,0x000e58e36c06431c}, {0x000d03f83a3f98d5,0x000adc020099b8d8,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d0b7b6a8a3320,0x0007c1820c541666,0x000008d89959635c,0x0006f6b2d261e2b0}, {0x000b20857dc9286c,0x0008490da31aca2d,0x00001835ecf8b430,0x000769b376d5c408}, {0x0006c3593326d702,0x000ae359276d5c27,0x0000000000000000,0x0000000000000000}}, {{0x000052b955ccca9f,0x000132f8bdb8b0e1,0x0001fc788aac66dc,0x0007aff377f81134}, {0x0005f3f5c42cf8fe,0x00063a0cd5ec56c8,0x00044e19f92551b9,0x0005a03e4e9df2c1}, {0x0005981a18a9c38e,0x00038410aaa01483,0x0000000000000001,0x0000000000000000}}}, + {{{0x000ab1b79d73f8b8,0x0002c67e2040bd52,0x00089ab066095a12,0x00020058f1cb78af}, {0x00035c77cb75101a,0x000e13361531375e,0x00075eaea159ba65,0x000a7ecbfca3524c}, {0x00019d039ab8ae0f,0x00099c623ac91c57,0x0000000000000001,0x0000000000000000}}, {{0x0001430a249d36df,0x000efe8450eb5d6b,0x000afb92b30c47b9,0x00024beea9991147}, {0x00039e1752c3ff68,0x000fd6a6252b160b,0x00046e76256f4b47,0x0009076f7bff5746}, {0x0003f350ce5bbdfa,0x000da84642b5dbcc,0x0000000000000001,0x0000000000000000}}}, + {{{0x000f59f034423df4,0x0000c74c6502a253,0x00051f6b04936dd5,0x000ac1dfa20fd420}, {0x000a26ee480942df,0x000b8f32c2aa1793,0x0008f9fd5819aba5,0x000593ae9a68c3be}, {0x000809f95c514dfc,0x0005832a0100c51d,0x0000000000000000,0x0000000000000000}}, {{0x00084df91d0b9d7b,0x00019af96dda8b28,0x0009e06515fe82b7,0x0004e7882cd87d52}, {0x000d3f4a8cd84bcd,0x0005a839e5e4c173,0x000eace8b4e2b402,0x00009c378fed4c2c}, {0x000b8a183c245522,0x00026032daca8b53,0x0000000000000001,0x0000000000000000}}}, + {{{0x00021d74b7f15174,0x000b1737719209fa,0x00000c8bba28cfe5,0x0000523f1c2878b2}, {0x000c0170331c9a62,0x000cd83b50a5843a,0x000131d0381135b8,0x0003a643b75eb047}, {0x000ef1464d2ab54c,0x0007f362ed0e42c5,0x0000000000000000,0x0000000000000000}}, {{0x000bb20fbad15614,0x00040a78f8613291,0x000895f7e0d7805c,0x0004b54ca2a8624a}, {0x0000e6579a8713ce,0x000626e2cc1b0cde,0x00093c66377df41d,0x0009cd6454de0451}, {0x00009db1f1c3ca34,0x000d91b047b0a149,0x0000000000000001,0x0000000000000000}}}, + {{{0x0006148c3607c309,0x00072dd5c6a1cd15,0x0004ea2e6d51f4ab,0x000bb9012a38398f}, {0x00025cc1f09df84a,0x0009c3bace064bf4,0x000f3b1a1aeaac49,0x0008ba0e470586b9}, {0x00026aca2a7b1cc0,0x00037064de7e58e0,0x0000000000000001,0x0000000000000000}}, {{0x000f70f6864c071b,0x000cebbcde808997,0x00042d9268c9d10f,0x0001f3646bf97f61}, {0x00099c6124289f61,0x000f65308f5fa877,0x0003b1cbe10f2164,0x000db1cfb717f6c1}, {0x0002178fd0705446,0x00053784aa2a3cca,0x0000000000000000,0x0000000000000000}}}, + {{{0x00032b93eb6bf0f8,0x0008d44a6f35d7f4,0x00062f74f5a61124,0x0008d968ff45509d}, {0x00090f78b11dcef9,0x000e0fdb4e540d2d,0x000178df19486918,0x000b775c9c48f839}, {0x000a4516e1546952,0x000548b05a9a422d,0x0000000000000001,0x0000000000000000}}, {{0x0000e6542e705240,0x000ea85c40801a5a,0x0008cf4381fc9bfc,0x00026551ecff5ed1}, {0x00006e3765708042,0x000f10bb393addaf,0x0004c0be6d6327db,0x000ade98dcbda7a9}, {0x00045d1d2c9cc265,0x0001d23919800694,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=37 [{1,2,3,..,}]*([2^148]*G) */ + {{{0x000136d3adf7c0d8,0x000a615150ed1e22,0x00048b602d1f12f4,0x000a438f58c86ca8}, {0x0006c2ad94dbc8f3,0x0001741520fd2861,0x0006926fc8f344fa,0x000aa2867b7697e9}, {0x00063769f3f74f49,0x0003389eafe4ecc9,0x0000000000000000,0x0000000000000000}}, {{0x0003271ab880c04c,0x0007eceb904c8b8d,0x000cf0e8b6b36124,0x000b8dfe9dc846a9}, {0x000c71bd5a3dcf58,0x000bb872ef46766e,0x000ea257028f76aa,0x00037d56cad75976}, {0x000e6e410a7a4c1e,0x000aa4d9ef6dff50,0x0000000000000000,0x0000000000000000}}}, + {{{0x0003f21a068b1990,0x00028b83926d837e,0x00046424f5058ff9,0x000540b150a21088}, {0x0007bd69839e2656,0x000836bb43217215,0x0008f5d34535e3bc,0x000a61ec6b271f81}, {0x00014bd57f4cd40a,0x0009c8fdb8302a87,0x0000000000000000,0x0000000000000000}}, {{0x0006b22f2553a3a3,0x0003b58b7033af0a,0x000213a07cddbf4f,0x000434d1d71e271e}, {0x0003ac069f3affa9,0x0004ccd448d5d23e,0x0005a3de785990cc,0x0009500536e9dd21}, {0x0005a1f484316890,0x0008d39f92d8e2e4,0x0000000000000000,0x0000000000000000}}}, + {{{0x000e463b75bb6ea5,0x000ed0a11789a549,0x000dd32eea4152e4,0x000b779f3244c612}, {0x000a765d467497f7,0x0001f2317d762ed2,0x000fb479fdc0f1f6,0x0005b625c778a26f}, {0x00086e4279e293c7,0x000778007cc51b6a,0x0000000000000000,0x0000000000000000}}, {{0x0008d10c9ca3a103,0x000303626747aa01,0x0009d0d1059a4b9c,0x00047b992888178d}, {0x0005dee73f8df999,0x0003009b79f58ee6,0x00095bbb19efb6cd,0x000ee364f704fdad}, {0x00075210ba101581,0x0009e1f5ffb0008b,0x0000000000000000,0x0000000000000000}}}, + {{{0x00061dbbf0d6ad65,0x00037943121ba6f6,0x000176edfca2325e,0x00061e28f0cef68c}, {0x00084617ac6eda38,0x0007535e8ca77e7f,0x00023d1d31f498d5,0x0000546d78b2f36e}, {0x000c7d55e2c3f881,0x000891156a1cb9cf,0x0000000000000000,0x0000000000000000}}, {{0x0004ce76bf8c0b46,0x000f73894a4c0a97,0x000e4d65f8fc178a,0x000cb9405d4f42d7}, {0x00089f73dac29f71,0x00028141921d35c6,0x00055dee3cb66f43,0x0003af9effca7532}, {0x0004e3d9ea981425,0x00042022e23b71d3,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d0b7e99b465d6,0x000dcdbbc6a7f631,0x000fdb4862b8d0eb,0x0002d00e72a3f599}, {0x0003fff9e95d96f0,0x00098b66f4cdbc62,0x000ac921634cb2b5,0x000ec0deea0b5c81}, {0x0005b0a27a76f063,0x0001e02d30431834,0x0000000000000000,0x0000000000000000}}, {{0x000c98aa1c9ac7a1,0x0007c5a7466568e8,0x000fb66755607c49,0x000ef99d83842e7b}, {0x0003f63d03b04212,0x000eea39bde1d43a,0x000c0a3d0947a656,0x000da642406f1e3f}, {0x000c396f50ca4ece,0x00002261add500ea,0x0000000000000000,0x0000000000000000}}}, + {{{0x000faccc08dd24e3,0x0002a75b2039c36b,0x0007b00e10c370cd,0x000decd44a6f8b5d}, {0x0005623fd6ab9f44,0x00019a1ddd85e65f,0x0007f5f2f27bfaa2,0x000934e1d6c8baa2}, {0x0002398cf5bc3e40,0x000936e76ebca08a,0x0000000000000001,0x0000000000000000}}, {{0x000b72cc3defb1bf,0x0007c48e0f8a00ef,0x000b50a281abffca,0x000290112a957074}, {0x000c6cc0bcd2913c,0x00080828c73f9cbe,0x0002b7332c142bad,0x000b82fd75dcc17d}, {0x00071d9dc563b115,0x000fa46eb1fc833d,0x0000000000000000,0x0000000000000000}}}, + {{{0x000293af2196db74,0x000aa8f516dffa78,0x000ab476917c3f2d,0x000a17a44ba1c0ca}, {0x000771d0d213b4e6,0x000872d37a8c50d3,0x000a9d4f31d594dc,0x00020ae17df8f34d}, {0x00012ccecab30c4d,0x000fa5e5b7722b87,0x0000000000000000,0x0000000000000000}}, {{0x000e6e4b1df4a900,0x0005d66c421f23c8,0x000473d466e42335,0x000dad17c61ddae2}, {0x000ed279fa7168cb,0x000addca2ea2657f,0x0004d479a5bdfda6,0x0009ec1a80eb84e8}, {0x0001a95ca1a89847,0x000f2ca605fdb8f2,0x0000000000000001,0x0000000000000000}}}, + {{{0x0000bda5161e9684,0x00055c62c59939aa,0x0001e39fae89d4f2,0x00072aef74c49bbe}, {0x00060180fc9e6093,0x00063da12ade7248,0x00028defa823f501,0x000a965a30e8a72a}, {0x0005cf1083c600ec,0x0004af9f8b968790,0x0000000000000001,0x0000000000000000}}, {{0x000afd7d7d936a7a,0x00003b13810cfd26,0x00037d1ddbf986aa,0x0005d035eede05c2}, {0x00071b7ae0b88271,0x000812487895ef9e,0x000f170e50423460,0x00003054f1639f87}, {0x0002e674eebc0936,0x000654593b42f2ce,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=38 [{1,2,3,..,}]*([2^152]*G) */ + {{{0x000a2f3a1cbc282c,0x000e19a09feeb1c6,0x000c54628c6180b5,0x000cefbae8c61be2}, {0x00031054eeedc773,0x000005e41190648d,0x0004925364893510,0x000a54e9064644b7}, {0x00026639e573a22c,0x0003b5a6074dacd6,0x0000000000000001,0x0000000000000000}}, {{0x0002e1f28cb4398c,0x000fba11161ac99e,0x000aaf012b04f328,0x00060a6cb74a91c1}, {0x000690cf3c48dadb,0x0007c4e07d1b8182,0x000bed19eb0dacbf,0x0001aba090482e6f}, {0x000b8c6fb9ea1ef8,0x000d6d4b567809aa,0x0000000000000000,0x0000000000000000}}}, + {{{0x0006de757b8ee9dd,0x0004f4a9eb572b8b,0x000650813f9e5a92,0x000081024cddfbbc}, {0x0003e750529ae0f8,0x000c407a678dbdc2,0x0005c643db36c6df,0x0009adee9ab1549e}, {0x000add1f855d46bd,0x000aedf68182d8ac,0x0000000000000001,0x0000000000000000}}, {{0x000e2fb66eef3f12,0x00004b24a7282866,0x00050b3c877e75f1,0x000590fae38bb301}, {0x00008b7b5535d2f8,0x0001b50eaef87c62,0x0000c4541ba355de,0x00098bfe96023f0d}, {0x000dcf2eadc15969,0x000f41ab8c033f3c,0x0000000000000001,0x0000000000000000}}}, + {{{0x0001bfbe835763b7,0x000da08736745919,0x000d000c52158859,0x000dbafd4373d9cc}, {0x0003efeee235e560,0x000fe980f98d303a,0x0004f012a6082ad1,0x000e567ed43eb524}, {0x000eddca68306748,0x000f954e531e38a7,0x0000000000000000,0x0000000000000000}}, {{0x0000101b465ee778,0x000f8f4e95956310,0x000bcb6c6057ab1d,0x00052b140218cd6f}, {0x0000a217b7b093a1,0x000924c99c9b3267,0x0000b388550cfd67,0x000eda396f8cf9af}, {0x00035154327557bf,0x00098cf74a0d9f01,0x0000000000000000,0x0000000000000000}}}, + {{{0x000cd9335dea0190,0x00015fcbf855836a,0x0007808f96352fb5,0x0005c5cb374fc6d2}, {0x0001eebbbb50f586,0x0007f3a5b9a4d0c8,0x000bc6329ed702e4,0x000264f0fada97b4}, {0x0005e3bbcf73be9b,0x000dd442f9f14de6,0x0000000000000000,0x0000000000000000}}, {{0x00029cdbcd6414f5,0x000db247ce590b47,0x0005be836ddf363c,0x00032e4b6a8da968}, {0x000c049bdb9815fd,0x000d8f7528076e41,0x000d50b097db4cb8,0x0007f829470b7fc1}, {0x000b6caef75b1cc6,0x000ed3d55324b1d2,0x0000000000000001,0x0000000000000000}}}, + {{{0x0002ed4b0886efde,0x0002e69216f70caa,0x000a19fc084c43fa,0x000bf1fcd447360e}, {0x00088b44bef17255,0x000c941e542e92c6,0x000c9462ab05687b,0x000f4a55ed27b06d}, {0x00006c4879438508,0x000b3f5a16cdbf59,0x0000000000000001,0x0000000000000000}}, {{0x00090da308f335bb,0x0002c62f75607156,0x000ac3878c204f5d,0x00019a70e1d9ebee}, {0x00065ce18d8dd345,0x00053d4a7a6a59b3,0x000ab840d7a66249,0x000e9f8efc3edfda}, {0x0006a32d022f5317,0x00072a29be1120dd,0x0000000000000001,0x0000000000000000}}}, + {{{0x000bdbfdee3c7792,0x000b747684a7e37c,0x0006fd64e032b7ae,0x00031287b0371790}, {0x0003245ed96aa7b2,0x000bcb2d9689dd51,0x0009eceb1d9918cd,0x00053e6e8dbeb445}, {0x0003cd8435c2226c,0x000a212888b11938,0x0000000000000000,0x0000000000000000}}, {{0x0001020457ce98be,0x0009fbe80ddb3555,0x000e385ff7a0d009,0x000821584b1de3c2}, {0x000f36cd64d47bdf,0x000d2c6d2f0d97c5,0x000126f061319f64,0x00050f118e4e9aeb}, {0x000dd0fd66a1918b,0x0008c9bb60247ff9,0x0000000000000000,0x0000000000000000}}}, + {{{0x00006452a69b40ef,0x000a58916404d8d3,0x000489351fe54188,0x00083228fc171eb7}, {0x000410e07508f15f,0x000b2d0bdc317f64,0x00075755181b0c85,0x000c4ca35cc09ad3}, {0x000bdec4e4688b7b,0x000ec2f6e18ec10c,0x0000000000000000,0x0000000000000000}}, {{0x0000b91ff7109c43,0x0001353357108dc1,0x0006db397b6c17eb,0x00038d3f418ed53c}, {0x000a1a7d0d6f5104,0x000558963fe8aa2f,0x000629ed52f70d0f,0x00070987dde60b3f}, {0x000b95a72bb4ccee,0x000aae64e168b593,0x0000000000000000,0x0000000000000000}}}, + {{{0x0009ff50fcb47b0c,0x000001fc1e2456f6,0x0001c124be702b84,0x0007f671a6c9b545}, {0x0000e07337c72285,0x000d3661d0b0a89f,0x0007db2af0223087,0x0001d9b173b261f1}, {0x000c65404d0457b5,0x00086eefc1cd30f8,0x0000000000000001,0x0000000000000000}}, {{0x000fafb34fbb3972,0x000db4c5bd6770ff,0x0000de59815fc7a7,0x000b5602342e8ca8}, {0x000220e1c9e4f843,0x000b0b7c5b3bfe91,0x000b313a1e2826c8,0x000988ce465ce442}, {0x000217ce5f2ef9e9,0x000682a10ff59077,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=39 [{1,2,3,..,}]*([2^156]*G) */ + {{{0x00041390c3361b6e,0x000b58054f802294,0x0009b74e1597143a,0x000652a48a901ba0}, {0x0004b9b4f3635116,0x0005e2ee300afb31,0x00079f7d46228864,0x0001b66e61674d2f}, {0x00005aad2298ff3c,0x000bd327d6400925,0x0000000000000001,0x0000000000000000}}, {{0x000b20dd543f093d,0x000dac4b51c2ba0e,0x000bf1d364874c9c,0x0002601310d4063e}, {0x000c6c8c6fbaa6b7,0x000ce6639ff8b94a,0x000066c91f488ec6,0x000524c600b8f454}, {0x000ff656ef37706e,0x0008a0434286c21c,0x0000000000000001,0x0000000000000000}}}, + {{{0x00077e284b1fd44f,0x0001dc37f60445d8,0x00069f0b4dfca419,0x000aa285358c7759}, {0x0003172cf55e112a,0x000ffea4f47f71ae,0x000412afc352eb30,0x000bc7ffc3d95b8d}, {0x0009ccbacabdbf74,0x00030dd4b6acd123,0x0000000000000001,0x0000000000000000}}, {{0x000870d6326f819d,0x0001f9d1598751a6,0x000c925f0b6c6b0d,0x000c309ba890fd44}, {0x000f1cedd20fe106,0x000408588dc46673,0x00053cbebfbcf6f2,0x000da52fed53b541}, {0x0002bbf3f7b4aace,0x0008d484a22a2167,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d633fc6b2523f,0x0003c8573eaf11ae,0x0005f254d2bc0511,0x0001c7fd283764aa}, {0x000f770135776ee7,0x0003df5ba988759f,0x00065da842051883,0x000b809c0705d522}, {0x0001067f4912507f,0x000fb628d91a9464,0x0000000000000000,0x0000000000000000}}, {{0x000e6e2ac33e3aac,0x00065c0000ebfac5,0x000ced796bda6c05,0x000a32836c90c0d4}, {0x000d2ee187fc8100,0x000d7848e982bcb3,0x000be08290e6b628,0x00085ab586db4a59}, {0x000b07e2fb9a0080,0x000d56210d8de2f4,0x0000000000000001,0x0000000000000000}}}, + {{{0x0000ac7a2fd05258,0x00034c744e57f4bf,0x0002edb448a88343,0x0004d56a4c1f9523}, {0x0003b85d4cde6c8e,0x0003063d710bd23e,0x000833d45b52f378,0x000d2012d08a14ca}, {0x000ccbe55ff85aae,0x000e919fa9b95c02,0x0000000000000001,0x0000000000000000}}, {{0x000999b76646e255,0x0001f5f355b09a04,0x00000d64b669309a,0x00089605b2bd55ad}, {0x000656b121bac578,0x000d693b7220d91b,0x00053ea1faab888e,0x000745d07a303444}, {0x000e7a52e75e36d6,0x000a7986433618f7,0x0000000000000000,0x0000000000000000}}}, + {{{0x0002484fcef15b60,0x000485fc2dc91c4b,0x000d7e9f8403b5be,0x0005ec8542217cb5}, {0x0003f3deede9d858,0x0008c56ddda1f005,0x0009028845902ce4,0x000cdbb111feb2e9}, {0x000537c2b8f6659b,0x00075d89960f5bf1,0x0000000000000000,0x0000000000000000}}, {{0x000a9e85b9799e89,0x0001de39c6986f88,0x0000fa555ee69af1,0x000f3b270c555b9d}, {0x0004c62266b30411,0x00084a11940b0e86,0x0005b26112da8247,0x000fc56950bfb7ec}, {0x00066d81f8a57ba0,0x0003b772e0aa0038,0x0000000000000000,0x0000000000000000}}}, + {{{0x000710f55ddf9e13,0x00018f67ff0e8dc1,0x000601481b67ef67,0x00009ffab39b462d}, {0x000b5ad90ba1057d,0x00018d94f2f83bbb,0x0008d2eed4c7a169,0x00019ddb61a12bee}, {0x00096ab74dadd029,0x0003902e5753e9de,0x0000000000000001,0x0000000000000000}}, {{0x000926dcd7de034d,0x000eab5af3e375a3,0x000eb250dce827a6,0x00008bd108623cdc}, {0x000d49a7d0e9c524,0x00066e3019236fda,0x00040ab55ed033af,0x000667077bc755cf}, {0x0006972e633b49be,0x00095334396ea43d,0x0000000000000001,0x0000000000000000}}}, + {{{0x000671c0c20009e5,0x000956db94fffef7,0x000bca8fdc30361d,0x000ebfa5860aa7a6}, {0x000feca2b724bff4,0x000572f34fd506fb,0x00048ff2e88a7d1e,0x000874822e19430c}, {0x0003c0129eb20b17,0x0003db07cc6f0162,0x0000000000000000,0x0000000000000000}}, {{0x000244f5da60b490,0x000fbd8954a885e9,0x000f39699542bf3b,0x000c93a6a7e331fd}, {0x0009816b29c5180d,0x000ad960c8e85d80,0x0003a3a7931b35df,0x00092e570f2974ab}, {0x000904daaf442234,0x000cf25e1f700754,0x0000000000000000,0x0000000000000000}}}, + {{{0x00056e1b7ae8ee13,0x0009b65f8128c4ea,0x000e5e0d92d02840,0x00074f688ed0e1c4}, {0x000f9c55f66d6f3b,0x000eb2ab8035d3f9,0x0006b643bde4296d,0x000f25e29f7ea7cb}, {0x0007f5f239b9d057,0x0001af17e3fac208,0x0000000000000001,0x0000000000000000}}, {{0x00063cbb323c7d21,0x0002b6d926fd3ed4,0x000ab9ee679014b1,0x0007e6093a1bcb9a}, {0x000dc171705931f4,0x000b0a4387f44f73,0x000c7cdd2a12e513,0x000a473ec3b73ce5}, {0x000ef17967f341e3,0x000a3809a474c86a,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=40 [{1,2,3,..,}]*([2^160]*G) */ + {{{0x000dc98081f0b504,0x0002377f1bc655c2,0x00067de245fb688a,0x000260cdd7a61e34}, {0x000d89aaf28a330b,0x0004e078039aeae5,0x000a42253d349d8d,0x000438cabcfed7ae}, {0x000a9960f3728bd2,0x000af658af568325,0x0000000000000001,0x0000000000000000}}, {{0x0006e52ab17d640b,0x00019d1bc21ee481,0x00026613d4c31a58,0x000c14072a5969b3}, {0x0005babfa75ee1a8,0x000c563bc4d35701,0x0000425d2086ecb7,0x000c9b8fafb1a4a8}, {0x0007ef737c2661a2,0x000c20e7afb2d654,0x0000000000000000,0x0000000000000000}}}, + {{{0x000726f329838a5e,0x000204b7a9942b65,0x0004a26b80fa33e2,0x0006f40abbf82a56}, {0x00026970dfcc973c,0x0001c38e96f95485,0x00019abd2bbae55f,0x000c1edd71d62ecb}, {0x00020adf26d97496,0x0000d917e1cf322e,0x0000000000000000,0x0000000000000000}}, {{0x000aaf44ac399116,0x00067bb67e29ba76,0x0003d1213c21031c,0x0003b345e37fdfde}, {0x000dab46f2bbeb4f,0x0003442227ef5d5b,0x0005c11bdace9105,0x00060e12dac175a6}, {0x000cdb1cf99010c3,0x00087106f2502658,0x0000000000000000,0x0000000000000000}}}, + {{{0x00044188249a4961,0x000eb8deb1c61cee,0x0006080b71cf8ff5,0x000b75b57b2ccc29}, {0x000b9ffb3c6aa214,0x0000a50e70e80f53,0x000fd2ffeb156be9,0x0005a94620e80211}, {0x0005db41e15422e5,0x00055d2030526508,0x0000000000000001,0x0000000000000000}}, {{0x0009e1933e619307,0x00086b5084131313,0x000b6d55898976e9,0x0003f79536a0866b}, {0x000b8e06bc0b2a44,0x00034e542863ba00,0x00040e4dd7a73a37,0x0008b2efa3822134}, {0x0005312ecb0905af,0x0008efb084f884e9,0x0000000000000000,0x0000000000000000}}}, + {{{0x000172c5ec6e6f32,0x000784d8ddaafa3f,0x000785f2ae4eb6e8,0x000db162f77d65ef}, {0x00085dec5c58d4e5,0x000a30bffa2375c7,0x0000bb7c92e0f7f4,0x0002294b17a00c92}, {0x0009107e026f93d7,0x000911ce9dc0950a,0x0000000000000000,0x0000000000000000}}, {{0x000841c6766f1f49,0x000079724523292b,0x0000e7ddb4cb0490,0x000d47f955646515}, {0x000b44b2a0877c3a,0x0004c3de4bd8708d,0x0009d24b4a9131fa,0x000585e650ae938e}, {0x000bb2e4980176c4,0x0001820248559a60,0x0000000000000001,0x0000000000000000}}}, + {{{0x000cef71c9a9281b,0x00078b2e3e260928,0x000b15a4e8453115,0x000c76cc66031c77}, {0x000e2f2c06ffcc30,0x000a471db8c352a0,0x000184b9a687b94e,0x0009b19798642e1e}, {0x0001d84cf08e1a1c,0x000462a36c823a7d,0x0000000000000000,0x0000000000000000}}, {{0x000b775551fedfb4,0x000c921b0d298e47,0x000071e1319e7833,0x000e6f4ce5e5ae43}, {0x0001348ff7cbdbf6,0x000c042f31447260,0x00061f1861a992ae,0x00020e5f80d48204}, {0x0008846b75b72853,0x0005ef4edf14c058,0x0000000000000001,0x0000000000000000}}}, + {{{0x00051608c9277436,0x00036641c6cf4e0f,0x000263e7b7515b1b,0x000a50636eb6d459}, {0x000df53679a56041,0x000b4abcaa6ef1d0,0x0005077b47a03019,0x0009d2d427efae97}, {0x0003dfa9162f30c4,0x000f8bc801e5655a,0x0000000000000001,0x0000000000000000}}, {{0x000202783ac347e0,0x00001a26d59f4868,0x0003895e6664e175,0x000031f4202e3866}, {0x00069d2af7613aa8,0x00021cc1e58ddf28,0x000159ee13d84ffa,0x000c8f6eb59a5da3}, {0x0005df9b7b87bbc9,0x000771b8b6006cdc,0x0000000000000001,0x0000000000000000}}}, + {{{0x0006aa5bb86ea29f,0x0000e29e7a21c03e,0x0009e430844ee3c4,0x000584b091ca8307}, {0x0006afb05a033420,0x00015d7ef65dc354,0x000acb0dfae44d05,0x000bad35608c8e97}, {0x000a78e5d1c181a0,0x0007cad8ba90d885,0x0000000000000000,0x0000000000000000}}, {{0x0001cba5026e7f38,0x0009593d89eff94b,0x000b88834191828d,0x000881379cd1acbe}, {0x000c4d9c16250e77,0x000f4d66dbf51b1f,0x000703cbf985d682,0x000998e4fae0e78a}, {0x000e124668125e5c,0x00095c7096d1799f,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c267dbe90f79a,0x000a682de8af3a96,0x000fc2373c7d7a0d,0x00046c045ae71058}, {0x00067b05a94e6008,0x0009ec9a78879108,0x000973f0df20f654,0x0003d4a6c168aecf}, {0x00050f6bc30604ed,0x000f342722d4210b,0x0000000000000001,0x0000000000000000}}, {{0x00089badc8348ffd,0x0005ea32767a9d3c,0x000dc1a4baa76ac9,0x000219cd3eced60d}, {0x0007d2d3cddf3114,0x000c14e1ea557cfa,0x000c466d40b6e234,0x000224ae183077a3}, {0x000e59e159bfca75,0x0004c30d62fa0c8e,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=41 [{1,2,3,..,}]*([2^164]*G) */ + {{{0x000acef3fd9ab352,0x000a04dda16fb097,0x000e90de3351fbdc,0x0001f9baff197a3e}, {0x0003610909fc0701,0x0003d538e2bf8355,0x000fbd9d3c214c4a,0x00064b2db047d1ad}, {0x0000b9e3fa7800e1,0x0001033ba0bb0ce3,0x0000000000000001,0x0000000000000000}}, {{0x000cb2552f015a84,0x000cdab20301de3c,0x000af7c3af2c8c0d,0x000fe99606c79c8a}, {0x000d638fb52847ee,0x0009bf56737cb586,0x00000b1ec4f260f5,0x000362ff887d36f2}, {0x000ed3b693913291,0x0006e3c40f0d7ae5,0x0000000000000000,0x0000000000000000}}}, + {{{0x0008a516e5648dad,0x00047202a3fb8a9d,0x000ecb3edff5fedd,0x000220d17fb9838f}, {0x000dc8b9ac40f762,0x0009a8311e23ad98,0x00084edfb615d6c5,0x0004e6c85dc486c7}, {0x000bf81a7ee5f8f8,0x000ecc58d5bb866b,0x0000000000000001,0x0000000000000000}}, {{0x000d41ef176fcfa3,0x00078f007acce1a1,0x000d8b8126b20e97,0x000a71b3438944de}, {0x000134e76c73c437,0x0004a56abd9a1b4b,0x000de8db7385f9b1,0x00043115d58229b4}, {0x00034725891b4078,0x000c55ba8c32f815,0x0000000000000000,0x0000000000000000}}}, + {{{0x0008e051939a3b7d,0x00010361ba38d482,0x0009c9a686091017,0x00050f6456db6700}, {0x00031ba66d450ac6,0x000e0bdd4225b759,0x0006e52d315738bc,0x000d21403d74797c}, {0x000a041eb53ed58e,0x0002f948ee60c564,0x0000000000000001,0x0000000000000000}}, {{0x00014e408d1cd601,0x000cfebde9152461,0x0004123bd5f4532c,0x00083ab7b9fe9b7a}, {0x0009704a01b31e49,0x00040d9c79a4402d,0x0009ff5b8e0a168a,0x00048d442aea1790}, {0x0001410d782cc3c0,0x0002bf2d98ba10cb,0x0000000000000000,0x0000000000000000}}}, + {{{0x000943818d1af858,0x0005d42684f68399,0x00068a5f9139d27b,0x000d03a1a3ed9c84}, {0x00024e699de7f9f9,0x000ddd7e41e31174,0x00089cc967769d86,0x000a0e9e00b5f6fd}, {0x0007b63934a6926e,0x00011c5b068a8b3d,0x0000000000000001,0x0000000000000000}}, {{0x0007d21cfa86aa41,0x000f529a2aecb429,0x000251f8677cf147,0x000bad3bd2a35774}, {0x00090bedc57bbf0b,0x000a31f1dbfe5b37,0x0001e75b3cb7422a,0x0006476bcd9901bb}, {0x000278bd8b31cdbf,0x00082a6fb171258c,0x0000000000000000,0x0000000000000000}}}, + {{{0x0009d9b88bd44811,0x00086c7bafe985de,0x000eb018e7fc2f20,0x00037a5b53cb3738}, {0x0000a097f28e364d,0x0000b5541e546ab4,0x000530e972bd2ec0,0x000b65a95e020994}, {0x000221ddc10db4cf,0x000295000b94fc68,0x0000000000000000,0x0000000000000000}}, {{0x0006ace5e2ed6000,0x0008ffd606613047,0x0002a24af3b853da,0x000aba583e1b87cd}, {0x0004618719533717,0x000d61f56ae2be40,0x00025ef5e9069ea8,0x000f94027fe98e78}, {0x000db6fc7d9c1583,0x00075271696c0d71,0x0000000000000001,0x0000000000000000}}}, + {{{0x000f12734c3ec92d,0x0002bb3d48fbed19,0x00049bdd26ff69ad,0x000fbf26985b989b}, {0x000ad451c21eb61b,0x000237a30e35f12c,0x000a3b3680a082df,0x000188ebe4c92751}, {0x00087a8fbc731694,0x000b03a8bdfe9408,0x0000000000000001,0x0000000000000000}}, {{0x0006f89f4e0d5883,0x000d80de19c8b935,0x00077afef27eab9c,0x000538f8f941390d}, {0x0002b8c79f62a16f,0x0004a907ee9a2c1a,0x000951eb7aa5d968,0x0001fe7d75aa9877}, {0x0007b983b59fbafe,0x0002bab437db42c1,0x0000000000000001,0x0000000000000000}}}, + {{{0x000bf512d363aeb8,0x000e0db50c6d9411,0x0000e1101f753fc2,0x0006d62a0ded1b7d}, {0x0004a1ec0fcb3b8e,0x0000a9719f9cb02e,0x000fa60331be1189,0x000525c569643656}, {0x00081aa5691f4f2d,0x000a963a9ea530b1,0x0000000000000000,0x0000000000000000}}, {{0x000098b88fd83f01,0x0009e9aed5969329,0x000769c1597dbeae,0x0003f34dc1aadb7c}, {0x0006d041d0f773cd,0x000dcc7a18555ae3,0x00057b66cab6672b,0x000c3dbd797513fa}, {0x000ec420f27eb3f3,0x0003c62ce13b7853,0x0000000000000000,0x0000000000000000}}}, + {{{0x00057f7ec577ceaf,0x0009584ce56b583a,0x000ce15377e1306d,0x0005b26b1e23a49b}, {0x000f42d98c317bad,0x000c523283ae8b11,0x00081ddf50073f0d,0x0004ab516099e7af}, {0x00029299e519277c,0x000c0d8cb7cfdd6a,0x0000000000000001,0x0000000000000000}}, {{0x00029a85f4a1c822,0x000b7e9213cb42ad,0x000364e5e4a37030,0x000a3941f8a54d03}, {0x00050b7d507ec771,0x000db1def4d6f8ad,0x000eab3bd493bf4d,0x0009716822a9c65e}, {0x0005d463b7e2f601,0x000728062fa75d1d,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=42 [{1,2,3,..,}]*([2^168]*G) */ + {{{0x0002ee214ad0e3d6,0x000d51de6a66c4a4,0x000c1ce94446c6c7,0x000c0d5dd2eee21b}, {0x000e88f8f4a8deaa,0x00055296fd5914a3,0x000dd876c3945207,0x00083798ebb4e647}, {0x000fd6484696a7a6,0x0001d866ec9d8ec9,0x0000000000000001,0x0000000000000000}}, {{0x000ca34e120495f9,0x00050701e46446f4,0x0004431e6d90fc27,0x0006b7610b310d40}, {0x00086e5199614976,0x000e7e80f704e266,0x000aa764f7efe74a,0x0003d9535c6d9829}, {0x0005a23c25702e18,0x0000a0457bd92a75,0x0000000000000001,0x0000000000000000}}}, + {{{0x0009cf085024c2f1,0x000c0aff178cc9fb,0x000cd1f6670717cd,0x000588548870fa8c}, {0x0001a99c44c6bc4d,0x0007a4c31ed62743,0x000f88c552f232dd,0x0008940140f085da}, {0x000a8211a1d88681,0x00041216e4c1b09a,0x0000000000000001,0x0000000000000000}}, {{0x0006cac59e6c3159,0x0000ba3374279c4b,0x0008991eda2c878a,0x0003b4cf84ea0b3f}, {0x000025e729a3932e,0x00047222c0cc5f31,0x000ba94b4346c5bd,0x000e2995032ec5c9}, {0x000db493f41a4bab,0x00024e7b6e042b7d,0x0000000000000000,0x0000000000000000}}}, + {{{0x000aefc5789d3eda,0x000070117b5af24b,0x000a5c6b9c3050d4,0x000dbfc9621085b7}, {0x0006f4c0b7973deb,0x00006f6cf4b4e834,0x000082f092a35673,0x0002d877db7b37b4}, {0x000c2eac8682b506,0x000eac10f86afedc,0x0000000000000000,0x0000000000000000}}, {{0x0002caf651b8b0a4,0x0008310eef2a1934,0x00026025b8808ec6,0x000dcff1e64f055a}, {0x000a67192e09ae5e,0x000d785482258125,0x0007daa7d24e92c4,0x000a9c45e5162876}, {0x000fc7c72fb7aba4,0x000522976bb5f88c,0x0000000000000001,0x0000000000000000}}}, + {{{0x000921c0f982798b,0x0001a2079475b7fe,0x000ea0fd52e410ea,0x000e44af77d4bbcb}, {0x0001f260a54b0212,0x000269af2ec66a7d,0x00034794993bda84,0x00050b15e358d04f}, {0x00067a4d30bdfadc,0x000e912250ea3d1c,0x0000000000000001,0x0000000000000000}}, {{0x00083de4fe7bebfa,0x0004fdfb63579e27,0x0001abe0cbad5ac8,0x000820014b8a145c}, {0x000a85d987c51840,0x0009eba9aacfadab,0x000291af5fccfd5d,0x000785e551a982de}, {0x0002bcee4372c455,0x00049c9d89842d5e,0x0000000000000001,0x0000000000000000}}}, + {{{0x0000817678d00826,0x0003072fc12b3906,0x000fabe24fd4868f,0x000b2a4f9e0b8813}, {0x000ccd87b27441cf,0x000e7fd0a48234db,0x000ea747d9a2a9fe,0x000c91ba0d4add56}, {0x000c9d0dd2d3e7c8,0x00005660e4fd17f4,0x0000000000000000,0x0000000000000000}}, {{0x000a904f88c1be1b,0x000e5cce4c6964ff,0x0005fb6194a74952,0x0000f033d222444b}, {0x0008c26fbb11965c,0x00055ed1ac1d1bab,0x00020c09d970630d,0x00075b608324cefd}, {0x00050cf259835d15,0x000462cd49bc1143,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b46239d54de36,0x00013af2871bf6df,0x000bb6d9f31a1b7c,0x000528b0f5b2569d}, {0x0006b9497778a81b,0x000c963043af6788,0x0003bf9954a12672,0x00059feeec8df36a}, {0x000d60c22b5fdead,0x00060d265f0f8b6f,0x0000000000000001,0x0000000000000000}}, {{0x00050d0b6534b1ea,0x000703b71e08797c,0x000d2bd35e284a7c,0x000105aa68827a45}, {0x000902245c12e4ca,0x000f1afdb8eeeba0,0x000caa7693b8db6c,0x000824a39f45018e}, {0x000945d0c9d12756,0x00055f86289ff82e,0x0000000000000000,0x0000000000000000}}}, + {{{0x000f246813a5e2f3,0x0004d2dd1bc96870,0x0005743352958a8b,0x00007a9386d4a79c}, {0x0008b4a29091d043,0x00059ba9e47bd2da,0x000478de8a606e11,0x0008c9f2a8c27ae1}, {0x0002946224c1c93c,0x0003ef3adcff2ded,0x0000000000000001,0x0000000000000000}}, {{0x0005100b79546483,0x000ac268f6c48348,0x000e4b2ec17a54ad,0x000f85818d6815bf}, {0x0004f425318546e1,0x00013cbacecaadf3,0x0006d908fa2a9c92,0x000e8808196d6c46}, {0x000ad801f4a291fd,0x000b7ecba0623fa6,0x0000000000000000,0x0000000000000000}}}, + {{{0x0007b0f39088ef39,0x000a435ae74e03d1,0x000fbdcdaf3b17b1,0x00090868e5084910}, {0x000019102285b63c,0x0005454d88d8e63c,0x0009e2380d185fed,0x000af9e19dfe50f9}, {0x0008e09d7ce8d3eb,0x0000155127749872,0x0000000000000000,0x0000000000000000}}, {{0x0007fda1b031ef4b,0x000dfd7188feeb77,0x0006801e0f6f597f,0x000e9d1729652f82}, {0x0009d58dec034252,0x0003cc68d0c6aa0c,0x000a4e76779b37e4,0x00008d509f569c62}, {0x0000c41330558ca7,0x00056956b5657bd9,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=43 [{1,2,3,..,}]*([2^172]*G) */ + {{{0x0009008a556937c2,0x000f76241f10378e,0x0004e7ecf0092193,0x00097f48f8905d70}, {0x000c86b4870ad280,0x000f86eb6f389aca,0x0003ffbc3b9a3132,0x000a9c6b9598fe5a}, {0x000429f1014fb463,0x000e06408908552f,0x0000000000000000,0x0000000000000000}}, {{0x0000c94ae41024de,0x000a6dd0399afa53,0x0007da5ef17ac70c,0x00080b49854eb299}, {0x000104afd62b1e2c,0x0000b1375777d7cf,0x000794db8dbecfef,0x000ff21b1b05dfbd}, {0x000f1e68e47404db,0x00080928abdaf296,0x0000000000000001,0x0000000000000000}}}, + {{{0x0001636898bf4d11,0x000fb75ab01bffaa,0x000ba4b1f3e58bf7,0x00059cab50bc67e3}, {0x0008acea4689ce8b,0x000f1932a30d30cf,0x0006e5cb3d1d8eda,0x0005fa7949e492c0}, {0x00041db2db16d8b2,0x000b0610d851b96f,0x0000000000000001,0x0000000000000000}}, {{0x00054a2b36667691,0x000ca196d36fe2b1,0x000766e2a6109d47,0x000f9263f1863dba}, {0x0003be92b5a5ba8e,0x000aad9918a5da16,0x0009189520c8298c,0x00010a27963af5e7}, {0x000b8c3b84ab05f9,0x0007bafd0103c420,0x0000000000000000,0x0000000000000000}}}, + {{{0x000e3e45f8d73683,0x000eb8ebb6d11caa,0x0000f274ee508fde,0x00020c83562c576b}, {0x0008510e47baeee6,0x00079b810588c571,0x000894a919ff42e2,0x00007edf259b927c}, {0x0009d16a223100a0,0x000a5b2acb9ccb16,0x0000000000000001,0x0000000000000000}}, {{0x000e8415a9179d06,0x0004b594d74f07fe,0x0004fb6e0e5cacca,0x000788b708cc549b}, {0x000c0c62edae508e,0x000b9ef886c2847d,0x000e40664c8eee69,0x0003ed24b57a9dee}, {0x0002b9d44a547432,0x000a6f16f12261ca,0x0000000000000001,0x0000000000000000}}}, + {{{0x0005496bc42e6e51,0x000d33d320585033,0x000cf402bd388067,0x0003e6730074be0b}, {0x00043e8db4e94291,0x000f7beef462a0cb,0x000e58a8c2ead81c,0x000b97eccd5df06d}, {0x0005954e3501f23b,0x0004b4a8b8e4e11d,0x0000000000000000,0x0000000000000000}}, {{0x000598c1e025da1b,0x000b09bf9648fc1d,0x000d224f8ad9987c,0x00065a60d88fba1f}, {0x00054d86a1d9f606,0x0006c4ad1df1e7f7,0x000e1da4acf77f72,0x0000938971a27713}, {0x0007fc94e0f78da1,0x00083992811d7d3b,0x0000000000000000,0x0000000000000000}}}, + {{{0x000c9cc6dd4a5914,0x000862d80be96443,0x00008c7a249fd0f7,0x0001ea54a0d2c9f8}, {0x00048d9e55013f6b,0x0007ab76e8d002ac,0x000cbf4462d73cac,0x000faf5cdb58c492}, {0x000ca322b819e5c1,0x000b840745406425,0x0000000000000001,0x0000000000000000}}, {{0x0005f739a14940b4,0x00097d20ee2a886c,0x00035ab04c341a53,0x000f7a9d2904ba7b}, {0x000d9cae762f47f9,0x00007d2eaeeeb5be,0x00070ab079042e0b,0x00060a339ed63e5f}, {0x000c7b9658ba8e43,0x0002d18d85499745,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b401e3813737f,0x000ff61118cf0239,0x000909b65cbad1c8,0x000da081fe099573}, {0x00028cfc70caa9fc,0x0002bfc31062db69,0x000e85af4aac9c83,0x000ba3d1d4e51a7e}, {0x000363dd1405fca7,0x0009117097ec2370,0x0000000000000001,0x0000000000000000}}, {{0x00065c78edf4f376,0x000f36321b45daed,0x0003283085bc71ee,0x00018a85f5b5aa08}, {0x00055758f2c2f181,0x0001623497212c90,0x0006ecdfee014f92,0x0000a48aadb790d1}, {0x000881e12f4528d4,0x0004482acaa286a1,0x0000000000000000,0x0000000000000000}}}, + {{{0x00050bbeb7940a25,0x000eb62b0dec1d2e,0x000f7146dd0fa42c,0x00051beef911c829}, {0x0009947cccb28e02,0x0009505362c5e903,0x000767ef06d55451,0x0003dcda2ee6b16e}, {0x000b2373652d7be8,0x000cb5af116c86d2,0x0000000000000001,0x0000000000000000}}, {{0x000d36b874b6449e,0x0002d7bb963c1def,0x000de8609229e57d,0x000375fd0ce127b2}, {0x000b5890dc213b70,0x000a1d88db3c82d4,0x0007b583b09ebdc9,0x000cc6b137b10089}, {0x000cf29a5fc13efd,0x0001907605ca17a3,0x0000000000000001,0x0000000000000000}}}, + {{{0x000665e2bbfdb04a,0x00017e4232c5cb5c,0x00026232f5f9a245,0x0007a0275981cd79}, {0x0005ae253d4d80a2,0x0006c00bb7783b1d,0x0004c2589b5ab0bd,0x000f6c48caf740ea}, {0x00082e177fc5351c,0x0000fd2b0e714ba4,0x0000000000000000,0x0000000000000000}}, {{0x00052b5ad6ac73dd,0x0007a311881ba785,0x0004ccac10cfb206,0x0003dcbe5d449097}, {0x0008b8873accd901,0x00080d70e5b2cf2a,0x000440b2c2817333,0x00067c3b711d4631}, {0x000b996623747bc6,0x000383423c70b2d6,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=44 [{1,2,3,..,}]*([2^176]*G) */ + {{{0x000b11d1789dc869,0x00016c2eed227fa8,0x000916842cb7fd9a,0x0008564ce12a5d02}, {0x000bee59ed474675,0x000e675f354b48f9,0x0005d69ece126be8,0x00018ce3aca2f7c7}, {0x000768d6000f88d2,0x00090f26ea6ff29a,0x0000000000000000,0x0000000000000000}}, {{0x00096ef4ce69e270,0x000f2da0efb2f05d,0x000a99dc276ac3a2,0x000e0342757c443d}, {0x0003b390d2a5e23c,0x000e7ea78e9b674e,0x00085e132e72b987,0x0006b6c21856dca4}, {0x0005bed8cda17d0d,0x000237220788bdee,0x0000000000000001,0x0000000000000000}}}, + {{{0x0002364996ca25a6,0x0007ec8d70cd440e,0x000d8467c5161afc,0x000408724c9aa882}, {0x000b962a215cbbc1,0x000c4986c1ad6d3c,0x0001332912aaf7a6,0x000d6db2702d8369}, {0x000a17e017d4a1ec,0x000f2dbf2405e93f,0x0000000000000000,0x0000000000000000}}, {{0x0000641168090c5e,0x0006fce42ae3e68a,0x00039938713395b0,0x000394a15bb1098f}, {0x000db97734c1bca3,0x000edfc62ae8c0be,0x000bc2f9b0452cb1,0x000304c79c90c661}, {0x0000dada4e625332,0x0006fc2e4ae4342a,0x0000000000000000,0x0000000000000000}}}, + {{{0x0001bd7fc156252b,0x0000ddb25b337fb0,0x000ac5d025ae1e66,0x000c26056a73c379}, {0x00095dede6af2b69,0x0001ae9121b7e81b,0x000754f6cd030d2a,0x0008b47d1e9a5f7f}, {0x00025d238c9b7c0f,0x000d32d6fa902ce9,0x0000000000000000,0x0000000000000000}}, {{0x0005ace423d94184,0x00056f6ab6a655f9,0x0008fa78d47709a3,0x0003f5d39d32f258}, {0x000beab0a90b8c58,0x000bc517995d68b6,0x000ea4acae9d65d8,0x000fd569fb104a80}, {0x000b09db02cb3b12,0x000c624b3e1e5f67,0x0000000000000000,0x0000000000000000}}}, + {{{0x00093cbca6dca0b6,0x0004a2146559221e,0x0006c357ebc20032,0x000e73dafbe29569}, {0x00073f1c77b70537,0x0002b0a8e959d415,0x00055d9c50a71dc1,0x00037c9b3656d184}, {0x000283b617fcbc17,0x0000976acf8093d9,0x0000000000000000,0x0000000000000000}}, {{0x0008b573715b4734,0x000173f0027024fa,0x000386bfccf3b38a,0x00095480bbc99c54}, {0x000668bfbf241bdb,0x000353dffbcc88d5,0x000216b7968e8858,0x000de22f661faa2a}, {0x000189437f0cc373,0x000c1f5601679c0c,0x0000000000000001,0x0000000000000000}}}, + {{{0x000a86967182c501,0x0001f634e40148fc,0x0001c864ffbfa398,0x0009d6d142879632}, {0x000443e4b6047507,0x000e1e5a879eef57,0x000d2b8fd7f7f136,0x000d19b6378838d5}, {0x0007815ed1c2726d,0x00042ef17abcb4c1,0x0000000000000000,0x0000000000000000}}, {{0x000b9a5999895b25,0x000be140e558227b,0x0007f28ae923d146,0x000d00a58852f582}, {0x0000e60ada16c8cd,0x000158a85a7def11,0x000e5c61d1152d28,0x000d4be61bf1a55a}, {0x000cf413c0a31606,0x000b3cd625cdfd8f,0x0000000000000001,0x0000000000000000}}}, + {{{0x0003f2f8ccce2027,0x000fd5cd45c4a564,0x000a6b2411224a0a,0x0007a5ca2258c4c8}, {0x000678f855fedfa8,0x00055199f43975cb,0x000e9a39edc6298c,0x00007312684e5a48}, {0x000adaeb9f55daba,0x000b39c9f5f377bf,0x0000000000000001,0x0000000000000000}}, {{0x0003e0968382a7ce,0x000869c70ffd115a,0x000ba001f2afcccb,0x000107bdfe8068fd}, {0x0000206868f7c124,0x000821a90928b9fe,0x000afc533728dac3,0x000b3e9edff0ac94}, {0x000d10c697f67565,0x000bea250773ba0b,0x0000000000000001,0x0000000000000000}}}, + {{{0x000275a2c8f91400,0x000bb4c241f78224,0x000c4fd93b4ba60f,0x0002941b616268c1}, {0x00020107f7964087,0x00031b438825e04f,0x00019247786625f8,0x00028de20083c5f6}, {0x000abde39791c6d5,0x000b3b75c25ecfb0,0x0000000000000001,0x0000000000000000}}, {{0x0008e09f47b9d8c3,0x0009374c6bc5ceb5,0x00038e27941b3112,0x000e3235cee2666e}, {0x000ea8dbee896ca0,0x00030660009b498d,0x0005f8f0f2897645,0x0000fb5ee44458ff}, {0x000fb559aa7b5e14,0x000272ac85e138f9,0x0000000000000000,0x0000000000000000}}}, + {{{0x0009f0c6193905a5,0x00013e99256667bb,0x00027fdbbfc34892,0x0000d2c71218ca33}, {0x000915a83f00e563,0x000d628331bdc8df,0x0003e8128ee96b80,0x00016a5f7e06bfe7}, {0x00016364a2a7cd33,0x000c748cd2a08bdd,0x0000000000000001,0x0000000000000000}}, {{0x0001d90fa51d3800,0x00020c814ecb8822,0x00000fc79208b5df,0x000f252076343a10}, {0x0008a14b68032c99,0x00054fe0dc71413b,0x000d97c9a173cb46,0x000c85a386e9a9ac}, {0x000bf160a14a40bf,0x000032849e997087,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=45 [{1,2,3,..,}]*([2^180]*G) */ + {{{0x000384b0dc2ffbb2,0x000e0c16c289b477,0x0009eabe48cf9601,0x000199d671ddca51}, {0x0006f3fce7863b3f,0x000e01be3ea3ecba,0x000b70167c58c7d2,0x000f4893679afbf4}, {0x00019a4362cb78d1,0x00061515a3d7fee0,0x0000000000000000,0x0000000000000000}}, {{0x000f2840f746e722,0x0002ef160c51fc25,0x00097156a16516e7,0x000e8398d9625db3}, {0x000d63f5b2c0ebf6,0x000c5b6523651404,0x000476dd10c4d87f,0x0001f40ffa318eef}, {0x000788025e5d3977,0x0003c298fa2547e3,0x0000000000000000,0x0000000000000000}}}, + {{{0x000523e81658a625,0x000aef8e050759b2,0x000b0377d5042659,0x000b9ae72b36823c}, {0x0007eff957169419,0x0009705cebf46fc1,0x000bd18b61ce7ad5,0x0007a7135b602fff}, {0x000f2e092fe9192a,0x00074d30a3a8e596,0x0000000000000000,0x0000000000000000}}, {{0x0007c895ead96751,0x0006523da4889766,0x000467afe86eb732,0x000a5ee25b7a7cf8}, {0x00000f2568e46393,0x00079a3304b15dd0,0x00036bd203f1569b,0x0009a5e938c0d91a}, {0x0001da1271a34645,0x0004c688c575bf52,0x0000000000000000,0x0000000000000000}}}, + {{{0x000c62a6b633bf04,0x0003c0eaef0121c8,0x00058d7354098cc5,0x000448cc925273a9}, {0x0006f73c56bf4c04,0x00042b800bc52be4,0x0008d6b39147d475,0x000444cb5cfe3029}, {0x000d4247fb2312e0,0x0007054c4d89dd9e,0x0000000000000001,0x0000000000000000}}, {{0x0000edd6a97a9163,0x000582ed4f4d5b46,0x000b9ca61309206a,0x000fafa93e18c6dd}, {0x000bda68f9bb8a3e,0x00070a52c8b2d783,0x0007728c0dda564b,0x000c0dc789e7dbe4}, {0x000119aa3e8a6481,0x000bed27f421a4e4,0x0000000000000001,0x0000000000000000}}}, + {{{0x00001ee133405081,0x000b94055dadf3f3,0x0008803374bd3d6a,0x0000e431a078817f}, {0x0000ae1298465c73,0x000a08da98aae817,0x00076bc8b779119b,0x000c1b8f7410f128}, {0x000bc98dcbe46247,0x0001761805980867,0x0000000000000001,0x0000000000000000}}, {{0x0009de67dab5cae2,0x0003d2d0125b70f5,0x0008c5ad3a01682d,0x000cf59a9c7c1b26}, {0x000ada095cf6362f,0x000b79b1ed6482c8,0x0002b3bc253c84e5,0x000756917d1dd695}, {0x0008f439fdfad9c3,0x000651a63232aa5c,0x0000000000000000,0x0000000000000000}}}, + {{{0x0003a055275e1f13,0x000bce620ca4b51f,0x000765c9fcc48133,0x000387e5710e23a7}, {0x00041d9c294797a6,0x000fe4eedda6621b,0x0009f733bf9d9ac9,0x000e4cb8a3045df1}, {0x000d5c96c4f51d70,0x00041a25c50ad245,0x0000000000000000,0x0000000000000000}}, {{0x000acd86687a04f6,0x0009bc4b6a5c45b3,0x0003f85a2b09f7d1,0x000f69420758494b}, {0x0007e554c9337d50,0x000ccb9c2f40c240,0x000e482c5dfc1a60,0x00016ad44e8b11e7}, {0x000d080e60fea531,0x0001889fd549f4ed,0x0000000000000001,0x0000000000000000}}}, + {{{0x0007e29c5ef5cdd9,0x00046b2b2e558b7e,0x0004702314f3e6bc,0x00026fae56eeaa30}, {0x000145ca44a1b067,0x000ea8da792ee6f2,0x0007e4c829cf9680,0x000d723cb279141e}, {0x000c514c645b326c,0x000b3d5e8e8931da,0x0000000000000001,0x0000000000000000}}, {{0x000e5ed0862bd48f,0x0009404a34e74e61,0x000e1d4a98483644,0x000f45001f65c56b}, {0x0008e062ee7183e5,0x000a39ef75aa764b,0x000f4509012ed646,0x000742837f0ebdde}, {0x0009ab588faa786a,0x0008d7474accf0f8,0x0000000000000001,0x0000000000000000}}}, + {{{0x000f31aa7add26e0,0x000b5f70683b341e,0x0002190eb5f5ed33,0x000e3b2bf3278604}, {0x0002cdb29e4008f6,0x00042f0700c911a6,0x000f5e3688f5189d,0x0007c2de5c257eff}, {0x00089c193e2d4667,0x000cca5de47c9861,0x0000000000000001,0x0000000000000000}}, {{0x000dddac10383cca,0x000803caddccaca7,0x0000778df17cf555,0x0009278c5faf93e7}, {0x0000e7cfbb523b02,0x0003ef004ba7546c,0x0007290d52d052d3,0x000f54a34c36c895}, {0x000e1b89dcc555fa,0x00058777136cbca3,0x0000000000000001,0x0000000000000000}}}, + {{{0x00060b5ef6c20e82,0x000bf430fe1ead47,0x0003a480e70d1479,0x00097c0aba684ec7}, {0x000549990971954c,0x000a1c5645d306cb,0x000cc85cc5c264ce,0x000739efac323d9e}, {0x0003b20c4465cbfa,0x000b4ee9cad749e6,0x0000000000000000,0x0000000000000000}}, {{0x000242934808827e,0x0008a9860bc18213,0x0007a452bdb41b29,0x0006c3f651ceda44}, {0x000f153ca2965078,0x000e0cd8cc7845a5,0x000c9cd5913baf87,0x00050312de2e060b}, {0x000a1444279bfb31,0x000ff8a16f8265f7,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=46 [{1,2,3,..,}]*([2^184]*G) */ + {{{0x00033bcc8924eb55,0x0002e9d518ffb740,0x0001ae0cd732da2c,0x000cfbc19a4290d6}, {0x000d8784c1f06357,0x000fe209893ca1ae,0x000969a85a8dedb3,0x0009c8eb2e932f87}, {0x000bcc740550ff52,0x0004f554bf85aafb,0x0000000000000000,0x0000000000000000}}, {{0x000bc6372d7f8438,0x000dc0557f4f2ed5,0x0009d0c30f3c2efb,0x000ddabb262ac2fc}, {0x000f7a05b87d4d5d,0x000c91e745769d1c,0x0008c994d0a4907f,0x000889250072dcd3}, {0x000ffae1ac453a28,0x0002a8e72458000d,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d9177fbc76a5c,0x0003975e3cbf7406,0x0002f09def039ed5,0x00034da80caf736f}, {0x0008efcc790febc5,0x0000ad47e746448c,0x0001ca336b92fa7b,0x000b90e92c64c767}, {0x000dfd8d6637080a,0x00032f5711517b52,0x0000000000000001,0x0000000000000000}}, {{0x000a0257ff4a1581,0x000bc02441238656,0x000364971ed77234,0x0008b1d09b2d316f}, {0x000bfdf4ae5e00cc,0x000468fa8d307856,0x000805be3791c041,0x0003fa589236fc69}, {0x000a337620c1fe73,0x0001e737b5760989,0x0000000000000001,0x0000000000000000}}}, + {{{0x000e1990ae2c44bd,0x00083ee2175b6a29,0x00005d1fb5989698,0x0001375eaf1c4bd3}, {0x0009f54aec5725cd,0x00017c1f0f7d222f,0x000f9f96b74e2d73,0x00053fff253f88e6}, {0x000b31a11aea1b12,0x000aa32748b4a307,0x0000000000000001,0x0000000000000000}}, {{0x00007b074c7b461c,0x000258e6e4224b52,0x000162fc7af983c9,0x0004966825052f5a}, {0x000f9138a0346a4a,0x000a7041242b7952,0x0000a366f5699476,0x000460c88c7d5eb7}, {0x0003f2b3125e31d3,0x0002a892d14ada09,0x0000000000000000,0x0000000000000000}}}, + {{{0x00001e6a21a7b432,0x000c3971b4886b8a,0x000dae7cb7883120,0x00059d28f3efe6ce}, {0x00028e1699713fd6,0x000252af65756250,0x0002a3acd7c4a210,0x000f7efc9c5a81fe}, {0x000e2a5f82fab4ec,0x0005c0924441558a,0x0000000000000001,0x0000000000000000}}, {{0x000495dd493563c9,0x000ccadf9b0e7295,0x000ab5a8f70bb0fa,0x0005b32501ed29d2}, {0x00036f439adfe6b6,0x0000a6c7202e9c24,0x0006531bcd403e24,0x00064526a2b69777}, {0x0001dc2d590cd125,0x000b64170acdcfa6,0x0000000000000000,0x0000000000000000}}}, + {{{0x000b0bb5e03ac48f,0x0009bb5837030273,0x000a05cded5e6ec1,0x0008034dce79ed12}, {0x0000fc54532094d5,0x0008fcc6c534a769,0x000bca87d8be0fee,0x00041150d981f9e6}, {0x000222345254c456,0x000b2aa496b6e112,0x0000000000000001,0x0000000000000000}}, {{0x000d2c8de5bb7e5b,0x000278f0be2794e5,0x0001b31bbb57b1c7,0x00001958330187ab}, {0x000f5dd751abf9bc,0x0002de4b57d090a0,0x0009cb74fbe565f2,0x000f83e310b95170}, {0x000d1301cd0ee2f4,0x00049fd2006501c7,0x0000000000000000,0x0000000000000000}}}, + {{{0x000e8d883bcef9f8,0x000be3770de7cc8b,0x00007c65e3e95107,0x000ac96f780e3eca}, {0x000413d615089cf6,0x000585b5b22549b6,0x0008b5facd5da79f,0x000f3c8b5c5a4c0d}, {0x000d6dfaa970b49f,0x00065cc025c0e7ad,0x0000000000000000,0x0000000000000000}}, {{0x0003c64dd34154da,0x000343c797b7cd0b,0x0001f367813bc308,0x000fbf3f138ae118}, {0x0006f1f8c6302e7b,0x000f35ea2ee3cc54,0x0003a0b904ac34ee,0x00052596f106852c}, {0x0006e533ab1310ec,0x000abf763b19381e,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d9a73e24c887c,0x000707461d095f01,0x0005d3ad552ce968,0x000f402b6c527f5f}, {0x000818672d6016b3,0x000279bc4633dd66,0x0000c571c90fed28,0x000cee78b5512020}, {0x00048d6ae97b4812,0x00055292fa8b91d9,0x0000000000000000,0x0000000000000000}}, {{0x0004608b8c1577cc,0x00058615049c4716,0x00077fa05c3b187a,0x0000d33110dc1846}, {0x000554a923122c03,0x00015b3d3cf40b2d,0x0005e05a3843c4de,0x0006438408a6964a}, {0x0005f646af7c591a,0x000ae89a0f7132ab,0x0000000000000000,0x0000000000000000}}}, + {{{0x000b5e462dd556b6,0x000f3aab5e9c2f29,0x00040c3ae00c87a1,0x000aade98fdfc7cb}, {0x000f2f671ec86f72,0x00069dd7b2aa376f,0x000b6f90c4b07483,0x000ae5c39e831a9e}, {0x000ef6929b8bdb31,0x0007125a4c5224c9,0x0000000000000000,0x0000000000000000}}, {{0x00080ab908d10b8d,0x000b7c8a32a9943d,0x00051b7d4fd0edbb,0x0008eaa89eb83ad0}, {0x000fb343de0ebbe0,0x000d3c4d0cc33cc9,0x000b15124b0953fa,0x0007582773fc9c30}, {0x000ab2c193a021a4,0x000b73ddfb881675,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=47 [{1,2,3,..,}]*([2^188]*G) */ + {{{0x000f014ad29ca649,0x00075a10e7e9c3d6,0x000042dda6a91edf,0x00069276fbe9047f}, {0x0005a497f91416df,0x000b982ab7fce403,0x000b8b61e6adadfa,0x000d218a9fd9d973}, {0x0001c8c04e2c23f1,0x000cb12274d47d9e,0x0000000000000000,0x0000000000000000}}, {{0x0000ec3de397b98a,0x000d9a272cecd709,0x00050e492db6d724,0x00082a50e32d2f19}, {0x000db6bf40e9c68f,0x000b25727f0678af,0x0007a36e6ae78194,0x000cbb096d1806b7}, {0x0001afd3feedfa35,0x000e57c17d9b9ff4,0x0000000000000001,0x0000000000000000}}}, + {{{0x00078e57ab05c549,0x00081a123d2b219f,0x000ecb0183ca3cf9,0x0008ed9f1eddfd07}, {0x0002f8f90e3c6699,0x000ad41bb20e0515,0x00019c77dab5c5ef,0x0002ca7830069394}, {0x000ae5cd1de605b3,0x0003933d6039cc98,0x0000000000000001,0x0000000000000000}}, {{0x0000ae5b05bb2b74,0x00071168c4bf8259,0x00001a66f3efdf4f,0x000e1da4a65b0015}, {0x000ba0665dbdf241,0x00015f360d4c3387,0x0004e85c88fe301f,0x000c061a8e048acf}, {0x000bcc0119ca9957,0x0009ea8585dfcf51,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c2a4869f8ca68,0x0005748af64adfdd,0x00044c661ce61bf4,0x000fa33532bda5e6}, {0x0004409ebd1c4df9,0x00063b107af0f45d,0x00013ef2dc8804b9,0x000a186c6c3f3d75}, {0x0005a7ed73f6b67c,0x0000b525bc5b72fc,0x0000000000000001,0x0000000000000000}}, {{0x00008fa0f93430b8,0x000bf3cdf63616c2,0x000a2ed8b79415ed,0x00008df169f3be1c}, {0x00006a9b7669442a,0x000be581a2acda6a,0x00093d2dae365779,0x0006ea987608134d}, {0x00059b31b3052589,0x000d8078ebfe0716,0x0000000000000000,0x0000000000000000}}}, + {{{0x000a0e4faa59f069,0x000d33d96e52e437,0x00029f2632f3aebd,0x0000c85e4fd15682}, {0x000de4f3be7892c7,0x0007d9fb180a1634,0x000b175638b44c2c,0x0002e33499b53e6c}, {0x000cdad290b60dc3,0x0009d1cf1fcbab2e,0x0000000000000001,0x0000000000000000}}, {{0x0001854cefb1da9f,0x000257b5b7539f5d,0x00096df1240b9d47,0x000f9e9a561ffc72}, {0x0000d6d9452715e9,0x0005aea9109f0df3,0x0005d521e814b452,0x000c7037d6e74c47}, {0x00060afbb2239aca,0x000dfe3a178a1e6b,0x0000000000000001,0x0000000000000000}}}, + {{{0x00085fac7aac5d24,0x000e6efcd816f4de,0x00001aa65b85ec90,0x00042ef288a94272}, {0x000527aa3e5baff2,0x0002c225dbf973aa,0x00073237d80e262e,0x000ac84d268b446c}, {0x0000056980d17882,0x000089041c6047c7,0x0000000000000001,0x0000000000000000}}, {{0x000f5b36106ae92d,0x000b658d82ab92e5,0x00036754a0792779,0x00032fae5cee6721}, {0x000bd0394c90fe6b,0x0001825f86d43336,0x000fa2e7d57eb880,0x00022294f3c7cd21}, {0x00070bb4a5b81f76,0x000e5c797152fbf7,0x0000000000000001,0x0000000000000000}}}, + {{{0x000d4560b8c46ee0,0x000b73ca40a188f7,0x000b593661397fde,0x00026cffd18ba8cc}, {0x000c47a735170abd,0x0002887e49a94d3e,0x0002a2ec01caeb9b,0x0007426e36269901}, {0x000522aa9c52b5bf,0x00066af9e1ef518f,0x0000000000000001,0x0000000000000000}}, {{0x000d742afa309fe0,0x0006f810d9df60b6,0x000084e739c300f5,0x00056da63d1a036a}, {0x000cd4f33c2df42f,0x0000b456fee1f0f2,0x0008f569e5589fab,0x00062492c9b09b89}, {0x000079adb7885960,0x000cf412b618c75e,0x0000000000000000,0x0000000000000000}}}, + {{{0x000e60d6e9faeb2f,0x000298054778495c,0x0000f99d00d5919a,0x00052dad049c1204}, {0x000e8b4f342e902f,0x000c225bd4b3fa2a,0x0004f48aaba21576,0x0009d33ad4b48b96}, {0x000a24fa9d5b26df,0x000f2799a4fd763d,0x0000000000000001,0x0000000000000000}}, {{0x00017650a566263d,0x000e25ee8cfe1893,0x000763623662e408,0x000a4360ad7e6d05}, {0x0000ba65a8e233ee,0x000338261baaa5ea,0x000af51b8dde194d,0x000003aaa23ac37d}, {0x000ab55391298a43,0x0008f52d2a4172a8,0x0000000000000000,0x0000000000000000}}}, + {{{0x000760e887bee904,0x0009f91c8cff613a,0x0003af1d33766225,0x0002f55a798ee44c}, {0x0002c7171b76304c,0x00051b89de6b4202,0x000754ce995dc454,0x000a5d7e90f40166}, {0x00037fe2c45f5e9e,0x000d835f81a1be14,0x0000000000000001,0x0000000000000000}}, {{0x000c04a7dafbcd8e,0x0007d22e0b1aaf34,0x000ad92815662ecd,0x000b88ed3ef4d947}, {0x0000190778cccdc8,0x000ea32bf7d0a755,0x00094ed615d6df41,0x00066cdce7de3703}, {0x0005d94b6a5a2d85,0x000869b1500a755b,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=48 [{1,2,3,..,}]*([2^192]*G) */ + {{{0x000556d5fcb0ca3e,0x000bb40eb6f5de7b,0x00092b00751e1fef,0x000d985badf10d77}, {0x000bda78c0fd8245,0x00097cec621ec6c5,0x0009de36f6534761,0x000929578b20f59e}, {0x00008129148b6a34,0x000260858df1e4dd,0x0000000000000001,0x0000000000000000}}, {{0x0002df7b80140bb6,0x000f0872cf54b64a,0x0005ba02c9e702cc,0x0006a4694fa2136f}, {0x000ae62ca46c9431,0x000a69d6c72a601f,0x000ff0af210ce686,0x000a108647e23ca9}, {0x00072d54b7301dc8,0x000b4fc0d011e4bc,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c2c9272139ecb,0x0008248890056b04,0x000319a82e4c5944,0x000bd6a55d37d95b}, {0x00074d80dfb735a8,0x000b368732a7edec,0x000dbb960fac47dd,0x0009b7d149244f46}, {0x0005c8153e4ae15b,0x000dd7d6f5637025,0x0000000000000000,0x0000000000000000}}, {{0x00023077c37f59fa,0x000e01c814ef1183,0x000d2dfe1b52b965,0x000d66c5cad600e8}, {0x00064cd44f8d02cd,0x000b170f04ad1f49,0x000b95d6b03da74b,0x0009721ac42809f8}, {0x0003fd08dc3ee705,0x0005bd69cd062aab,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b883d845a0ae0,0x000afbae353f2a2f,0x000473d0adeb61aa,0x00037ae3890f51bd}, {0x000480f0c4103d1b,0x000087e22deca493,0x000ae96c669a58f0,0x0000d7ec27b93462}, {0x0005f63a771fe3af,0x0004035d6f692734,0x0000000000000001,0x0000000000000000}}, {{0x00047ce82b6063a3,0x000a032d78ca1a20,0x000ffe80d92bf2a2,0x000357128144148d}, {0x00065f437565141e,0x000044794e70453a,0x000ed9d74d6e72f4,0x0002c9dec888c3b6}, {0x000d35b1703c9efa,0x0005564a8b5ee101,0x0000000000000001,0x0000000000000000}}}, + {{{0x000408c0ef420ccb,0x0003751466bdddb5,0x000686318317d090,0x000f5c351d77faab}, {0x000e6e1c56990fd6,0x00044f54fded7bc7,0x0007a03658746405,0x000acb87ac9e9b1f}, {0x000e1a951d060b45,0x0002d4d46b22b133,0x0000000000000000,0x0000000000000000}}, {{0x000b308639e1f9dd,0x00088fb9f340687c,0x000545e0d0da3dfc,0x0007c0127b5897e6}, {0x000708cdc1322bbf,0x0003ce8bdc5bfb35,0x0009aefe13ad9991,0x0006b6cb7333e158}, {0x0005d1b9d92265f8,0x00065e9dfffba15f,0x0000000000000001,0x0000000000000000}}}, + {{{0x000dfc8b990a47ce,0x00010afcf7dc8ef3,0x000e84517ba6c8fc,0x0002906aab79d6ea}, {0x000563aab2198045,0x00005b6fc1e2c314,0x000e1c75139c8775,0x000f87b03c5339f9}, {0x00013db5c7dd8d56,0x000be729d4f04194,0x0000000000000001,0x0000000000000000}}, {{0x0003ca2969a49a45,0x0003d7263734a805,0x000a96220f01659f,0x000a81ea503a05cb}, {0x0009a4b1e826b6e7,0x000b9124852cc317,0x000a53155592390b,0x000adcbcb9064287}, {0x000f6cda7ecb3423,0x000745fa7b75f585,0x0000000000000000,0x0000000000000000}}}, + {{{0x000020f9870c5652,0x0008ed2cf4bf5440,0x00043a293130a211,0x000b5775772ea602}, {0x00033659b8c321b9,0x000b8b914e8eddd5,0x00016d7acd2ea084,0x000600a84078289a}, {0x000073293df2fe16,0x000f1b7632be02d1,0x0000000000000000,0x0000000000000000}}, {{0x000ae60c00fc41df,0x000dee1aec5298cf,0x000dc3a384668cb0,0x00039330262dc7a6}, {0x0008200480e942b2,0x0007f1ad6908e660,0x0004d8902ca00250,0x0009dfae47555ead}, {0x000dd7fb7e96dfa4,0x0004d09336664145,0x0000000000000001,0x0000000000000000}}}, + {{{0x000d1eaa79614564,0x0008f53026152677,0x0004e1ef41c569ec,0x0009c61bebc6f47b}, {0x0005a764d392b91a,0x00081c91fd03bca4,0x000d12e91e91f33f,0x000ceb9cb7de2868}, {0x0000e9b7cc6516bd,0x000c8bcc47c28272,0x0000000000000000,0x0000000000000000}}, {{0x0007539aee683a33,0x000df86171edb94b,0x000cb40fe4798476,0x000b93f35533bd14}, {0x0009da702f13dc6d,0x000ba7ad4d348188,0x000a392741f6a108,0x000c52b97e5c2cce}, {0x000568205c383605,0x000d4f58766e7a6c,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d3d9203521aef,0x0003dde5091b5f60,0x000ec304735ae314,0x000afe69e360b755}, {0x000f3119298c9f78,0x000c6a7738e3ed2f,0x000298a24d640365,0x00078b486bf006b1}, {0x000e9050b3448a96,0x000ac46d50f02b81,0x0000000000000001,0x0000000000000000}}, {{0x000b1ce68a1d8699,0x000b8559ff13a9f1,0x00023011f5efc1bf,0x0004e57b1d2b17a5}, {0x0008efdcb9ac6bec,0x00009c3a1153d5a5,0x0006b2a4b16461a1,0x0005a5edc709e6c0}, {0x000d62c80c93e99c,0x000b0218529aa94e,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=49 [{1,2,3,..,}]*([2^196]*G) */ + {{{0x000e7cc655ddb9f3,0x00013549c78f7abc,0x0006489c7f6e90f1,0x000e52627145775f}, {0x00027c353f1cebe8,0x000a2f29fc36a4b9,0x0000acc3ef5baced,0x000fb8074e6a3d4c}, {0x0008d3fd643a9c64,0x0004c070fffe4c63,0x0000000000000001,0x0000000000000000}}, {{0x000fbd2cdf57826f,0x0005bd9bf19e6e5e,0x0007942ab0ce8665,0x0000c790e82b0e8c}, {0x0001e2f2b552cb1c,0x00090a098c9dab8e,0x0009810a67eba463,0x000c6a4756fc9b4d}, {0x000e8cb25c97785e,0x000215f5f5b6c18f,0x0000000000000000,0x0000000000000000}}}, + {{{0x0002bf06c6b067f9,0x000a2b3dcbcaa8f7,0x000559f9fa4cedc2,0x00001ff4707cbdc6}, {0x0005eb59f1a1d653,0x000b9620b3fc409d,0x00091f76c53a5feb,0x000e766a3eea48b5}, {0x00013597ec3fc458,0x000a5eb4cf309e19,0x0000000000000000,0x0000000000000000}}, {{0x000b24162ae5ce89,0x000e57dda1da6f8b,0x00092393366cd895,0x000d0cbc02de8414}, {0x0007365ce8f0759a,0x000cfa2564893b65,0x000a74873186b40b,0x0004d0156cb04fbe}, {0x000490f66512a03d,0x00036b328165e70e,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d63dbde7982b9,0x000c29e4cd39f5d1,0x000aa87372309f98,0x000951ea561fec44}, {0x0001afd07b42ddc9,0x00045755866c4665,0x000b2e07bf78c6a3,0x000e9284f87ca447}, {0x00035d4199cdea2e,0x00049a677e175372,0x0000000000000001,0x0000000000000000}}, {{0x000229ff95010f5f,0x0003cbc8f306c814,0x0001a7861e7a79e9,0x0002d63c05616521}, {0x000d995f90f64784,0x0000e16cd8cf737e,0x0001408ff0413e3c,0x0009c3a4f30dedcf}, {0x000a0a6b443a170a,0x0006e0cc49b5c711,0x0000000000000001,0x0000000000000000}}}, + {{{0x000ac308c74d834c,0x000a8dd825f7406b,0x0000fbb496f34b0d,0x000075e1de870fb5}, {0x000227841bcf2365,0x000b8e05ffd3c983,0x000e33dc39c86d83,0x0009f0fd6d0e74cb}, {0x000d62a5a8904ae1,0x0008c32b1e28056e,0x0000000000000000,0x0000000000000000}}, {{0x000d2a2671b67c79,0x0004494c1cde5597,0x000c0326e9105031,0x000b1ee150606033}, {0x00061f18317b0423,0x000cc474ed398c9c,0x000df4796a972375,0x000696b52ef07eb4}, {0x000ed96071372ae4,0x0005ffabf9d1feb1,0x0000000000000000,0x0000000000000000}}}, + {{{0x00045b8e28f19059,0x000286054435bd90,0x00012ddcca1e377c,0x0009f510d747b1a8}, {0x000d3775c0ea63eb,0x000865c7834fcce9,0x0007bbad37d19f75,0x0007bbc7cb402eb6}, {0x000530a0f5327111,0x000908600a1a8bd5,0x0000000000000001,0x0000000000000000}}, {{0x000abbe5e02132c6,0x000b17b10fe3c6dd,0x00011b655993587d,0x0008aa4f1c163208}, {0x00092e7539751ad7,0x000229bfb751c187,0x00003ce5719f77dc,0x0009dd5c3eedf84f}, {0x000b8c257bb9c60b,0x000345e60da1b9c4,0x0000000000000000,0x0000000000000000}}}, + {{{0x000e7d40935779ea,0x0002b583ea2a70e6,0x000137328e54c9ba,0x000501ecd4654390}, {0x000e5d733683ab93,0x0009f374dd118e98,0x000b90700d407bd8,0x000c13b0afb65295}, {0x00048095857db6bf,0x00070895fc47c66b,0x0000000000000000,0x0000000000000000}}, {{0x00037df304273762,0x00053684543a49aa,0x000af1483095c127,0x000176dbbf08a1c2}, {0x000fbbab267dccb7,0x000bd6efdfa7bbd2,0x000abfc8aeeb27ea,0x0008c902ad03e86e}, {0x0009e682a4e44e71,0x000fe0064991f1f0,0x0000000000000000,0x0000000000000000}}}, + {{{0x000aed77f4d8e151,0x000d64f6fa111299,0x000d4feb2e79c04e,0x0007b4f97a120999}, {0x000d370550af65d5,0x0001340660d07357,0x000084ce4afb7c64,0x00040826e57205ac}, {0x000a7fc0bae197ca,0x0008238f07d6803e,0x0000000000000000,0x0000000000000000}}, {{0x000454a02cb353a2,0x0000deb5cdf6d6af,0x000f3bb89c8b32bd,0x000b355a1bd8c3f6}, {0x000e63db355ab5de,0x0005c6b3982f043a,0x000910f0e90987dd,0x0001380521adabe8}, {0x000fa044a4bf6a24,0x000fd8fb752ed23d,0x0000000000000001,0x0000000000000000}}}, + {{{0x000fdd1be70b4926,0x0009a826f6dba658,0x000cb4e95121bc79,0x000f676af00f6eae}, {0x000cee75a521d56d,0x000eca7d7729d333,0x0002e5027fb68ac4,0x0004a49aec5f206d}, {0x0006a988faa272aa,0x00047f341efc691b,0x0000000000000000,0x0000000000000000}}, {{0x000df0f078415461,0x0006ca3afd9193e6,0x000e7785c7d78268,0x00030f1e3c2a9148}, {0x000aaa49f1fa54c3,0x000e96229782ded4,0x00093b6b845da08d,0x00020729c99179a9}, {0x000904b0df8fef02,0x0009cee6016c6aad,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=50 [{1,2,3,..,}]*([2^200]*G) */ + {{{0x0008c22f80cbfbe4,0x000130943a347193,0x000e2773aac837e8,0x000010c64a3c4f46}, {0x000be2b750229f24,0x00007131ff138446,0x0000ce7731813b90,0x0000e94672d6c2c9}, {0x000dcb075dd149a0,0x0006d07531381b69,0x0000000000000001,0x0000000000000000}}, {{0x000b38c7be8e6de0,0x000a9739ced7c6b6,0x000a61fbc4fb63d5,0x000fe4d18f6b6bae}, {0x000bd6ae1dbab075,0x0002c3dbf8c1ebed,0x000b0516dce109a1,0x000e4a2962c4c087}, {0x0000db685a1e1733,0x0001ad9f800e79f4,0x0000000000000001,0x0000000000000000}}}, + {{{0x0003feea78bb9ff0,0x00046e4fde5cbe66,0x0003f440437dd027,0x0002a08933232942}, {0x000e421f2f6038cb,0x0008d7b95a50b4f0,0x00073ae18c0b0f4a,0x000bc9451cc035a4}, {0x0005154ba8955b22,0x000d349d1fd0859c,0x0000000000000001,0x0000000000000000}}, {{0x000f04652c1a8bcf,0x000b73e19db868af,0x000eea574f2961dc,0x00027664f8c3e1f9}, {0x000839b512b73f43,0x00002a0ec5b683e4,0x000b38a0fb615a0b,0x0001b1e55bb87991}, {0x0003094173f71955,0x000d3c0ba8f16419,0x0000000000000001,0x0000000000000000}}}, + {{{0x000ed0fcb63247bd,0x000c61e26950f920,0x0008ac76960a5916,0x00049cfc2ae5b02a}, {0x0008cda5eb1a5171,0x000118595b5c8f4a,0x000c88e0004518e9,0x00089dcbce699e0c}, {0x000dca7cdb0b0583,0x0009678f7a0d455e,0x0000000000000001,0x0000000000000000}}, {{0x0003e3080d452c74,0x00067cd9ebc5ab77,0x00092748ad132f85,0x0000eb812f890896}, {0x0009083d0c649d6e,0x000d13cd26dc1732,0x000539d6815ffdae,0x0007727168b4775e}, {0x000ad256509166ff,0x0001747a36c1d3bf,0x0000000000000001,0x0000000000000000}}}, + {{{0x000e6e936a559258,0x000787f262712646,0x0007ee8f55296d6c,0x000b44326540e78b}, {0x000ef2fb8850453e,0x000e4739b6073a9c,0x00032c19bbfb39a0,0x0005bec805ba5b65}, {0x000b74df44331c49,0x0009d3002e8ec8ed,0x0000000000000001,0x0000000000000000}}, {{0x000d7ba6c48685b3,0x00073d4bae18cecb,0x000a9e818b43a66d,0x000e109d5a439da7}, {0x00084e2bd60c3422,0x00082785ad715748,0x0009a5bf6bd330b4,0x00066c8383da8c0c}, {0x0009a00bf0007cc5,0x000256a489783e2a,0x0000000000000001,0x0000000000000000}}}, + {{{0x0004a8e407a2ccd9,0x00086ad221fba29d,0x000fddda1e9d46ad,0x0008643114fcb5bf}, {0x0005a60db0e24f96,0x000659be98b0468c,0x0000c785c91bca8a,0x000b1e072204cabd}, {0x000ebbe04d9453df,0x0007688aef77cf50,0x0000000000000000,0x0000000000000000}}, {{0x000b62e349b426c5,0x0004467872d194b8,0x000ddbd4e1c43334,0x00024117aad0f260}, {0x000ea7d8cfb9c423,0x00083e18f4bd6c92,0x0008dd5687682258,0x000359ac483a7289}, {0x000ec708225923bd,0x00062148de2e57ff,0x0000000000000000,0x0000000000000000}}}, + {{{0x0004d07f78796d38,0x000a0307a33d42f6,0x0008948a2a44d434,0x000f90db03ccc6f0}, {0x000696ff7592ebb8,0x0007ff2ae969af49,0x00014fcec7fca3c8,0x0000d6cec6f56874}, {0x0009ae9c6b325541,0x0003ea961c98239a,0x0000000000000001,0x0000000000000000}}, {{0x00084bbf91f7e4e9,0x0009ac472d023742,0x000d63ca5a686ea8,0x000614346cd552ac}, {0x000072fb24ab8e61,0x00080d3677dc07d1,0x000c8c0d1833f7c8,0x000717c50635d225}, {0x000f8a2192bf84ae,0x00027f2e83c678c1,0x0000000000000001,0x0000000000000000}}}, + {{{0x0007ea965c0d1be3,0x000dffe0762dd1bf,0x000d60aa7917e003,0x0002fc7262c54da8}, {0x0004421eaa7edfa9,0x0002c86ea7ff6dc2,0x000473729f82d5e6,0x000d535c6df46821}, {0x0004bda6bb69bc00,0x000b33f34e260149,0x0000000000000001,0x0000000000000000}}, {{0x00096ecec3563dba,0x0004fd12da169210,0x0002f945903cc5db,0x00014cb4c4f95586}, {0x0007af70f6fbb150,0x000a6967e23d80e4,0x0003ebeadb489f20,0x0006a009490665a1}, {0x000afaa96a28958e,0x000a31da82221f9f,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b5dfd8f4cc713,0x0009de59f2c453ba,0x0000c9b2cc1e3fa6,0x00033c17b6318b0b}, {0x000e92d5d399a56b,0x00008f8a6f6dc3c1,0x000b7f39e28633ff,0x000ff1fbcd4351ff}, {0x000013e8c77388ee,0x00066a953e5ebf9d,0x0000000000000001,0x0000000000000000}}, {{0x000bfd2f419a3879,0x0001d195e5a481bd,0x0001ef3e1ae017a3,0x0008b706b5b37267}, {0x0003f748e8ba9898,0x000e9de7d3391698,0x0007cde6e3e3c930,0x0009ee4ca324e7e3}, {0x000b6a772ae3cdd9,0x0004235fda48d82f,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=51 [{1,2,3,..,}]*([2^204]*G) */ + {{{0x000ce56e11c7765f,0x000fe4cfdef6377d,0x00035b399363df3d,0x000ca630a715e9e9}, {0x000a21011f820ffa,0x00022d3bc633f64d,0x000dab0c59875522,0x000dc95736a8523a}, {0x0008fef5b787c715,0x00032a66393c6305,0x0000000000000000,0x0000000000000000}}, {{0x00034ecf897f6f48,0x000d40891f4ace54,0x00051c5f6bf7708a,0x0007ca62fe89ee25}, {0x000eae6011a07c37,0x000028c949d24cd2,0x000ab99c094a1a4d,0x00031fed19d9cf84}, {0x00051c154036f7f0,0x0003d437b50c3205,0x0000000000000001,0x0000000000000000}}}, + {{{0x00036aa0bf5fef4c,0x000b4bb069d26c89,0x0002e3dd1d0d3718,0x000bf3daeaaab400}, {0x00076315a34c426e,0x000eb5f38604c676,0x000805197e2eb1b1,0x00037226db1abc31}, {0x0009ad73df5e17eb,0x000d3797d098f510,0x0000000000000000,0x0000000000000000}}, {{0x00082cf0882acea0,0x00006b48806f5a59,0x000abd275055f094,0x00049a504db94328}, {0x000ec38e43c40b3c,0x00082b99e608d386,0x000cf443b07fe475,0x000b7186cac29089}, {0x000d982cfac474a1,0x0009aa7b0368d422,0x0000000000000000,0x0000000000000000}}}, + {{{0x0006f1287e600c1a,0x000cffcc5624ecb0,0x0002d1ca07fb78a4,0x000a1f9666cc7bc9}, {0x0008539fb634b6eb,0x000c73e361397798,0x00014a496c8d68c2,0x000d7ca4181be620}, {0x00069a299a451732,0x0001c06004061fe8,0x0000000000000000,0x0000000000000000}}, {{0x00029e4b308242d0,0x000a13eeecee128a,0x000f659548ea451c,0x000490cf14707b99}, {0x000362f80a26ba79,0x000292eee64971fe,0x000871b89c8fc38f,0x000d6dc0d122e55c}, {0x000979f11919fbb1,0x000f85c98fe350a4,0x0000000000000000,0x0000000000000000}}}, + {{{0x0006cb60e51b31a9,0x0009c2f6d82cb3ba,0x000f86e04fd98949,0x00087bb6cb66fb0c}, {0x000cb7e6257cf354,0x000caa5a38dbe642,0x0007ff70132dd977,0x000a9fe7cec8cf0c}, {0x000f2a9f98b24a15,0x000c7552eb7ce954,0x0000000000000000,0x0000000000000000}}, {{0x000aec9e842f8ae5,0x00096d766fc55447,0x00099065768ced0e,0x000adad9493166bd}, {0x000c328be045e2b9,0x000e70d305222a08,0x000ec1d4f554727a,0x0002fed1873d61d8}, {0x0001c46d541e23d8,0x00042ef348b3f19f,0x0000000000000001,0x0000000000000000}}}, + {{{0x000ccf976377d19f,0x000f1cf68ff2fc1a,0x000d98b274ba0faf,0x000788c074f0bd37}, {0x000a798f5cd04250,0x0003cafaaeab508b,0x0009fd1e881c2856,0x000dd63706ee8361}, {0x0001b79794f0ad14,0x0006f33d5c0505bc,0x0000000000000000,0x0000000000000000}}, {{0x000090b50612a9e3,0x0003e3a662f706ea,0x000cdebf25eed8bb,0x000d3a7d888eb094}, {0x000fdc87a6c4d17f,0x000574531e25001b,0x0007e6d823c00762,0x00014c6a486f9d98}, {0x0006d2d6573a4d0f,0x00074c191e9a8851,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d58220482e6f9,0x000c77488e8bdb82,0x000ba1084c7aabd7,0x000ae76567c7272c}, {0x00079ab82a151ca6,0x000826c75c1ca84a,0x000434806316ad0a,0x00040dd7f329f3c6}, {0x00033bd2dde3d6a0,0x000c43689c3e453f,0x0000000000000001,0x0000000000000000}}, {{0x00061be6ee98af37,0x000291b8361cc48c,0x000b8a7b622624c3,0x0000547d4c3e4e24}, {0x000c937a21d5f3c1,0x00007a153ffa09bf,0x000a63d54fa325a4,0x00048d13bea75c7f}, {0x000e1e55bfb7c45a,0x00055a1a8b85312f,0x0000000000000000,0x0000000000000000}}}, + {{{0x000094bb55dffec6,0x000b1ee483a0b1a7,0x000e63531718eed4,0x0009354c0be15b59}, {0x00029a3b937fed84,0x000295e4506353d0,0x000205f7b6d68fbd,0x000e59b3375fdbda}, {0x000b4aaddd0cb573,0x000348c879da0f51,0x0000000000000000,0x0000000000000000}}, {{0x000afd066ea2106e,0x000ee293d8c45af2,0x000bc67374a1f66a,0x00016d7c0fd1fd2b}, {0x0002493231dcd541,0x00019502a277efda,0x0003c7872e2fab64,0x000a95c3d06567be}, {0x000fa48d7ff41d2a,0x0001cc69c52d2e6d,0x0000000000000000,0x0000000000000000}}}, + {{{0x00046d25ca0b12ed,0x0003e7af11f36102,0x000d246f5c00ca56,0x0005f2713dbda22c}, {0x000571517fe92c1f,0x000e558aff5b82f3,0x000852ec7e4811aa,0x000a0a7d7a0cfe93}, {0x0009a571e80c69ef,0x000d5ecf0c8dc85a,0x0000000000000000,0x0000000000000000}}, {{0x000629e0362ea61b,0x00019570463933fa,0x0004d8bdb2997e91,0x0008d16c1670ff63}, {0x00042ac0e352c375,0x000bf5c218c31489,0x0009269789d4077e,0x000eedd9111468df}, {0x0009e59bdc949bb1,0x000694a97ced001c,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=52 [{1,2,3,..,}]*([2^208]*G) */ + {{{0x0000e658f63653aa,0x000362a4e263b15e,0x000f1a72f5cb787e,0x0008dd85ade21c8a}, {0x0009351d6c477346,0x000ea4254fd69f8f,0x000c982ae15e0af4,0x0005dd836935db86}, {0x00023278398d3a2d,0x00082c5ffb0769dc,0x0000000000000000,0x0000000000000000}}, {{0x0002ae6ecd27779d,0x000456043db3d94c,0x00073642e7c09230,0x000692df6f9dd795}, {0x000422cd985762a8,0x0006ac0a49a83e72,0x00059cfa2e9e20f1,0x0000d0093708d3fe}, {0x000c84d0b10a4692,0x00035fa5bda12a10,0x0000000000000000,0x0000000000000000}}}, + {{{0x0009924edfb9aeab,0x00028a46d2968a10,0x000fe84ed7a9147a,0x000478aa49744c91}, {0x00030fd88965188e,0x000dc8d99e65a34f,0x00006f221fd955c8,0x00027ea7cd997402}, {0x000f83ab9dedce89,0x0006e8a7c26d23d4,0x0000000000000000,0x0000000000000000}}, {{0x000728e182c8cb8b,0x00078b0fa5f32091,0x000760a4e2a3ad9d,0x0002b50f65aca369}, {0x0003d46ee027e681,0x0005a7e2b8db993f,0x00003752acac076e,0x0003a179054a6029}, {0x0007bff0fddbfa0d,0x0008feee0dfeeff8,0x0000000000000001,0x0000000000000000}}}, + {{{0x000e1e66266af2df,0x0002e36c081c96d8,0x0002c3c896d714ef,0x0002acfe977ac59e}, {0x0001a95d1b3f90be,0x00003c79555da335,0x000ac68d1f4138b3,0x000d2dc5e8301aff}, {0x0007f5144ad06837,0x0000d0ff81349a59,0x0000000000000001,0x0000000000000000}}, {{0x0003ebad0af48caf,0x0005e5d9aadd9976,0x00026bfaeb96c7e8,0x000c9ceb03564b50}, {0x000254586ab1371e,0x00038a2ddf7b0228,0x000ca35a77209961,0x0003fd9f60e7f5a6}, {0x000f0eb7073273c4,0x000a1bce76f5e62d,0x0000000000000001,0x0000000000000000}}}, + {{{0x000a60a63bd16196,0x00077f090aae19bd,0x000e7638c32cb3f5,0x0002a6ddf59abf93}, {0x0001d3548613634b,0x0001b5e6513c50db,0x000b5bd49476ec89,0x000a83b636d2f4bb}, {0x000071e3a3dd95f7,0x000ef977c02f69d5,0x0000000000000000,0x0000000000000000}}, {{0x0008603799531d83,0x000ec49c9ad3cfc6,0x0004e50cb9635f1e,0x0003ca9d26b39588}, {0x0006f3bd6a0e5d70,0x0008ef03a93fe903,0x000aad2605b0ecc0,0x000b6abd3a9b070f}, {0x000a81977f3494ea,0x000831164f95f67f,0x0000000000000001,0x0000000000000000}}}, + {{{0x000d576c129bba19,0x000dc781b1958f67,0x0004c725830444b8,0x000d772989a02957}, {0x0003160191e1f14a,0x000b3aba62ef31af,0x000a9c026782cae3,0x0005e2df9de1d797}, {0x0007d5d393897669,0x000e09de2e3f7f60,0x0000000000000001,0x0000000000000000}}, {{0x0002f590d9230f6d,0x0008e59ce71ff8f2,0x00065bfab0b97ecb,0x0005b773840ae894}, {0x0005df550efa55da,0x000a03525e361e4f,0x00032a275d9fd4c6,0x000f30d16226c4bb}, {0x000512eb88ebc1cc,0x000a048dd37790a6,0x0000000000000000,0x0000000000000000}}}, + {{{0x0005c82e7c82fe0b,0x000b6b153f48688a,0x00079cad28ce32c1,0x0004dd65d7330f19}, {0x0005fd6ce0841643,0x0004ad25ddac7886,0x0007705833e0df48,0x0008294753bb4502}, {0x000ad8dbbbf63937,0x00019b436c10a463,0x0000000000000001,0x0000000000000000}}, {{0x0005b7516a304d40,0x0008185865f25816,0x000cfc1c163a4965,0x000ed7eed35d77f6}, {0x0002d2246fce8e80,0x00064fa59660ff02,0x000081304ce77dc8,0x000c6fc813dd85ee}, {0x000db13057fabb84,0x0008d66a509ef64a,0x0000000000000000,0x0000000000000000}}}, + {{{0x000ae85bb3c1e90e,0x0001923787b4a031,0x0009e575e9892a6e,0x00074c2ebb769eea}, {0x00063064d9df4bf8,0x000e1bf584f54d2f,0x0007ecc49f863e83,0x0003213fe023848a}, {0x000c07e3d527042f,0x000c51626ab6da53,0x0000000000000000,0x0000000000000000}}, {{0x000783a377f5ac26,0x00010cfafa402a74,0x000f762b70abfb39,0x000eac42e208b3a1}, {0x000497274db0b567,0x000b54ede1f18ddf,0x0001de4808fda6ea,0x0002ceac81237a87}, {0x0007edd6e7428e9d,0x0000cb9fb5801abb,0x0000000000000001,0x0000000000000000}}}, + {{{0x000205d81676b493,0x0001cfe8f546e257,0x00087afe8b644287,0x000676bad5e346c5}, {0x000f4a964afa3748,0x0001422f71ca39ba,0x000328b0e9e0a58e,0x0001d31cca18d62c}, {0x000787f6507714d7,0x0009ad810168e375,0x0000000000000001,0x0000000000000000}}, {{0x00030f78aa1440c8,0x000f7e509d6354b7,0x000beae80e0ec14f,0x000f7cc09793053f}, {0x00025b6b1fd1b019,0x0004558d48e4fca0,0x0002aae7ed4a0374,0x00070e2db1c48699}, {0x0007f4b02f033375,0x000af43011764955,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=53 [{1,2,3,..,}]*([2^212]*G) */ + {{{0x00015e37a2d438c7,0x0006bb436c808cd7,0x000782325918615d,0x000d68ce58c6e6b2}, {0x00075a40e8f75ca6,0x00001da381c4c378,0x00055d9be962879c,0x00075dd3d4cf58a1}, {0x00099fd85847d5de,0x000e158f7f76b4ee,0x0000000000000000,0x0000000000000000}}, {{0x000c9f66a4cec18d,0x0006e45302a76bd4,0x0001b679cdf64708,0x0002c24293b84a7e}, {0x000a092243bc4d41,0x0005c3c375519ccb,0x000d06b585371f2d,0x000590f4c0f28ba3}, {0x00073b4091daa768,0x00073c6342e78bf5,0x0000000000000001,0x0000000000000000}}}, + {{{0x000dcb7fa9fca3c3,0x00047144679a2a9d,0x0003be369b38069c,0x000a82620de84dbc}, {0x00098d5e1f28d82c,0x000205de8989e877,0x000abac84051f10a,0x000ac26a5b9c5c22}, {0x00074ecbb99bba5f,0x000253359fa6c2dd,0x0000000000000001,0x0000000000000000}}, {{0x0002ac09a82c210a,0x000b422ac08c572e,0x000c5c720e071535,0x00095966a7bd1c3d}, {0x0003a0a4b0c9e18f,0x000faa6c6449a62e,0x000bdd44f85595c2,0x00047bb9f75f529c}, {0x000f46a6493955cf,0x000622defa5af650,0x0000000000000001,0x0000000000000000}}}, + {{{0x0006df335e8acd01,0x0009b6996f91a727,0x0008be8a5f6bba5b,0x0006ef24a13311be}, {0x000b7fe2a95d51f2,0x00084a38b4ee11ba,0x000caf93233a4f76,0x00073771e4762ff2}, {0x00024adb050176c1,0x000a9e7e6a96ea5e,0x0000000000000000,0x0000000000000000}}, {{0x000540fbecee1541,0x0003f39f444fded1,0x0006ce01534c5cef,0x000a886125f5b460}, {0x000b437e2199b1d8,0x000a994fb5a77157,0x000fcae599e6b65e,0x0002599cf24da91d}, {0x0009f0d7248964c9,0x0007ac171d21915d,0x0000000000000000,0x0000000000000000}}}, + {{{0x000ca4b8955dde0b,0x000f850c9b5cc251,0x000b99572d6cb500,0x000f8952b82356c8}, {0x00088575ebe209ee,0x00089253a4607df2,0x000ff9ba421a6b33,0x000799f0eb4dd513}, {0x0003ed789c84b56b,0x000293e6a5c4d425,0x0000000000000001,0x0000000000000000}}, {{0x000ff217c0f18530,0x0008986930ae0198,0x00012fd1e778abc5,0x000f509594fad4d8}, {0x000656733dccef2f,0x0006afad83f10fab,0x00078f717d75c593,0x00018a1a313cd995}, {0x000cd73871602741,0x000f5446078ec40b,0x0000000000000001,0x0000000000000000}}}, + {{{0x00034871165b0407,0x00041aa1bdd6d509,0x000fcc9e894c7b1e,0x000369e1389494fd}, {0x00087f5116c5472d,0x0000a1b0d04b3663,0x0000b4c49058303b,0x0002bdbdc1d8cd26}, {0x000ae8f01c121d9d,0x000dc1d8c7270b76,0x0000000000000001,0x0000000000000000}}, {{0x000d922f0e06b7cb,0x000f07bd16843c5d,0x000c01dd32245ab8,0x000fa9f6081477df}, {0x000c2db624f56ea9,0x000b2fb3e2bccdb5,0x000cb3b768793cb3,0x000c8d894f04384c}, {0x00010a3af20b15d0,0x00045ab5af8411db,0x0000000000000000,0x0000000000000000}}}, + {{{0x0006b9ed8590e4a5,0x0008a58a2974d283,0x0001c7668bc64c35,0x000acc81cd837cfc}, {0x000fc7b9eb02f729,0x000fee9d6ad1171e,0x000a8d7ca7eeb433,0x000a79ea2e79ed5e}, {0x0001fb8522e38381,0x000cc8162a76e7a5,0x0000000000000001,0x0000000000000000}}, {{0x000c67543c98a4ea,0x0001f713941c4de6,0x00081dc75d38a3eb,0x000374ce9c459cdd}, {0x000bdd7ff2d1d555,0x0000d7b03a560a54,0x000ad6193d7f976c,0x000099ff27e977d0}, {0x000749edcba5d394,0x0002c6804ed826ed,0x0000000000000000,0x0000000000000000}}}, + {{{0x0003e3bd3291cd15,0x0006a26271a72635,0x0001ef4581082f50,0x000cb6109637a812}, {0x00043e376f158b63,0x000e1fb8b4f92150,0x000fd4238523b166,0x000edb09ffe09019}, {0x000a7d8e27b82fb2,0x0001989a4df94f9f,0x0000000000000000,0x0000000000000000}}, {{0x000410feb007ca74,0x000ed314198cb45e,0x000e234a35b4b72c,0x000c1779579fdfb5}, {0x000da54f63558ebf,0x000b68211f279305,0x000b7f7a88ea265f,0x0000914728cae4ca}, {0x0002ed65205137c3,0x00045ed6df45a086,0x0000000000000000,0x0000000000000000}}}, + {{{0x0001cfb71cb23bf8,0x00067b25479ac001,0x0002f48bcdb785ff,0x000da267c454fd25}, {0x000154e460f65f62,0x0006fa21e0ac759a,0x000770bdb56d239e,0x000db91d7ccefa07}, {0x000f07f46fe275a8,0x000527a152927368,0x0000000000000001,0x0000000000000000}}, {{0x000969a0d74d64a1,0x00059c36f066b2d7,0x0003a664061d81a7,0x0008c41d929c3e1c}, {0x000a4033ad63b5ba,0x000387b665fdb5a8,0x0002e2c2b433c84f,0x000e4f2d4d931e74}, {0x000631141b030d2d,0x00086de3845c248d,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=54 [{1,2,3,..,}]*([2^216]*G) */ + {{{0x0000c039d020c1dd,0x000d44ab7b690765,0x00026b42700b4f64,0x0002c1e20576d051}, {0x0009e70d2ad712c7,0x00015f46314322ae,0x000dfb189904b573,0x000124905a45ef02}, {0x00026cce68a7b470,0x0004ac5f0db2cade,0x0000000000000000,0x0000000000000000}}, {{0x00042cd205524778,0x0000d86fcfeb993a,0x0001cc7f2d2ee992,0x000c209546bdb299}, {0x000531516a6eab71,0x0005f1492f99e62c,0x0002cf5197ae7709,0x0002013e12e8e95c}, {0x000b78cb72aad7be,0x0000320e70b96760,0x0000000000000000,0x0000000000000000}}}, + {{{0x00011fae297851e8,0x0004f69afd113575,0x000ab322fa242843,0x000f361ecbe5e3de}, {0x000b7c1b08880d89,0x000a50aa80c1fbd2,0x000104d9e40537e6,0x000ed4f51df57fdf}, {0x0008a6cfbe707164,0x000f36d887e3d0b7,0x0000000000000000,0x0000000000000000}}, {{0x000365e7f9a5983b,0x0003c6129d87d996,0x0002952186a64aad,0x00009284f8224d3c}, {0x0007bc689c1d4452,0x0003f44aec2c194d,0x00057e00e7c6b2d0,0x000de28c9eb3f18a}, {0x0008b7de6fac4981,0x000009552159064b,0x0000000000000001,0x0000000000000000}}}, + {{{0x000147285c5f1e84,0x00054bbe273f4c79,0x000ec66bd4d71c06,0x000771b4dd0505e9}, {0x0002d4c891619ed4,0x00021a0542316ff1,0x000ffae0c65ede48,0x000db678c0c5a23c}, {0x000bbf48abf05e02,0x00032691ff4f9e32,0x0000000000000000,0x0000000000000000}}, {{0x0004ec586f495f39,0x0007665c351b2d94,0x0003939c0c800e74,0x0000a9b2fe310474}, {0x000c05e63ba55b78,0x00011119c911cfce,0x0000043322972c9c,0x000235f5ba3b0c8e}, {0x0002b78daa0946bc,0x000a82622257dc22,0x0000000000000001,0x0000000000000000}}}, + {{{0x00053147b3b22c3b,0x00004897a83e73cf,0x000d11d0f510fba4,0x000de4abfcf9128e}, {0x00047095c92d56df,0x000f3cd334bcfa2a,0x0000038a5b6f4d4f,0x00095df73169a249}, {0x000aba80e663bcbc,0x000c64a47769e841,0x0000000000000001,0x0000000000000000}}, {{0x00004365a62c4cf6,0x000531a582938f21,0x00043321e6dc439a,0x00061253f7387146}, {0x0008352424fe3ee8,0x000a676302135902,0x000702fe71ab2ae0,0x0005f460396ca837}, {0x000882e04f7bce51,0x000f93d936239901,0x0000000000000001,0x0000000000000000}}}, + {{{0x000dc17de84a8cec,0x0000802c374c6c6c,0x000bd6a5b20e3454,0x00046c40337b0f58}, {0x0003431d7aff9402,0x0001dba6a849292d,0x000f90b37486a260,0x000c12d61cbe41a0}, {0x00033f828782067d,0x000486ebc39fcd30,0x0000000000000001,0x0000000000000000}}, {{0x0006818da93cb68a,0x00039ef42673be02,0x00040e906d3f2e27,0x0000e1f1d8ecf6f0}, {0x000655c024d3837b,0x000417cf596708c0,0x0000946c57882083,0x00058942a1031702}, {0x0009752bc7086a40,0x000474bd8d65b027,0x0000000000000001,0x0000000000000000}}}, + {{{0x00057bccc2137aff,0x0003cdce9d3ef437,0x000021aa7d7b10a8,0x00014071804bf03e}, {0x0005009822fa479a,0x000c4d2fe91adecf,0x000bad18fc3b061e,0x0002ffe82ea4479f}, {0x000b0bed7b70c576,0x000d5c8a667da425,0x0000000000000001,0x0000000000000000}}, {{0x00036c22b574750b,0x0001a79beeade4d8,0x000d7c41634d28e0,0x00030d7e124715f6}, {0x000b28e33c4f983d,0x000442a0682ee2bd,0x0000a48a18cfc1f5,0x0009a6cb16376839}, {0x000ecc58844abd78,0x000383bd6682cba8,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c36944a224339,0x0002221a86a3568f,0x000a933230388a55,0x0002ca0cfc1f186c}, {0x000051f24d4d5f8c,0x00053ac024a82a69,0x000e4e0b6761f2a7,0x0006b4a03fa98a0f}, {0x000862d5fd1d2058,0x0006922bcb9949cb,0x0000000000000001,0x0000000000000000}}, {{0x00085574f50da47c,0x000f8cb1192295a2,0x0002436f1d423eb5,0x000f6ae23f4febd2}, {0x000640f1f266f842,0x000e94d3498091b2,0x00016f1f4561f25e,0x000c6e303b526f4a}, {0x000bbc14e80ed7ff,0x0009ac6f957c1908,0x0000000000000001,0x0000000000000000}}}, + {{{0x00089a398d206da8,0x000e6a92326c61c3,0x0000b658149c80b1,0x000f4f6423067f4a}, {0x00022c96735aed5c,0x0003cb53c31cf4dd,0x000c7214b478ba14,0x000d3b1bbb1eb353}, {0x0005aa79b46f2e84,0x0007d44a4fa3d67b,0x0000000000000000,0x0000000000000000}}, {{0x000d31f07cf711e2,0x00031e5402e45d5a,0x0005565057819f0b,0x00050ebe214cdf81}, {0x000e63efd8e063fd,0x000c82d63ad808de,0x000d0ee39ccec300,0x00085249be7d39df}, {0x000e9271ab6c788e,0x000a727ffa3cadeb,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=55 [{1,2,3,..,}]*([2^220]*G) */ + {{{0x00061361be3ee97b,0x0006e1f3125658fd,0x00096b636a1d69b6,0x000b9e470c7ac9e9}, {0x0005bf9bb3617e69,0x0002050a8c1b1e89,0x0002213fa5a11a51,0x000bc2919affa249}, {0x000be1b1008d55c3,0x0003926dcf2c08c4,0x0000000000000001,0x0000000000000000}}, {{0x000fb57fda93efc6,0x0008276ce4aac2b4,0x0009277cab16292f,0x000e677dc90518a9}, {0x000ab0432d015144,0x0005d9214eea4408,0x000b8a649b20eb23,0x000b48a45d8a2560}, {0x000cf7d1d37dd269,0x00049d71a47616ce,0x0000000000000001,0x0000000000000000}}}, + {{{0x00034d86d189072b,0x0000b9475ac257a7,0x0003a9d12f132433,0x000adf08cecaa5dc}, {0x000dc33641cc3048,0x00040352a6bfcc4f,0x000dac876f01bade,0x00035bd7dfb06e93}, {0x0001d4494c0e1ca5,0x000219b51965b8b2,0x0000000000000001,0x0000000000000000}}, {{0x000f90e9be7f6998,0x00040776cb857a46,0x00031907caf1b517,0x0007040038843e17}, {0x00014b3c14ab5377,0x0008c99d12b47cfe,0x0001590d18daa185,0x000f84db2817d455}, {0x0008544bbc4d8f7e,0x000ee1752b595c5a,0x0000000000000001,0x0000000000000000}}}, + {{{0x000718c9c3382876,0x000798d016412ea8,0x000c7059eeb3d459,0x000e910707afd251}, {0x000366ae603f57f6,0x000f1d424b7c29b3,0x000f4591f08b6d11,0x0004fa0b188406f8}, {0x000a09ce46dfa46f,0x000f3e1ee6193a22,0x0000000000000001,0x0000000000000000}}, {{0x000ef37aacfd9628,0x00044abc282a7b59,0x00016eca5fb04908,0x0002a4a0fa414af6}, {0x00010748c915edd4,0x0003d4af5a6f1ab8,0x000aff331ef86bba,0x0006d8ffc35ce768}, {0x000278d36b6c4e53,0x0006238f61ddd56e,0x0000000000000000,0x0000000000000000}}}, + {{{0x0008204ec1fe994f,0x000a401febacf810,0x000a1fbe66e0a0c3,0x000654b72570b727}, {0x0007b5299d8ae343,0x00058b323a83e064,0x00034d13d86ee0d1,0x000be946224b7585}, {0x000a38321cb77d0e,0x000ea9b845ec39e9,0x0000000000000000,0x0000000000000000}}, {{0x00021327ccd07b3e,0x000b9c13edfc5044,0x000efee69b80c4b9,0x000b9736ce07a452}, {0x000de2779de28110,0x0009cb506b810433,0x00009831468d2371,0x000fb54615eed293}, {0x000a72f30895d360,0x0000822a939f9fa6,0x0000000000000000,0x0000000000000000}}}, + {{{0x0003613a0bad2c78,0x000a2841479bab61,0x000ae853dfe64b3b,0x000c5d69d7d5f8f3}, {0x000913c023c98edd,0x000e51c064c2af1c,0x000caf23e811beb5,0x000a297f73a13e28}, {0x0002c2db1b2c63f7,0x000869272783c6e9,0x0000000000000000,0x0000000000000000}}, {{0x0009e33e7c2b0e8b,0x000a6e2930859a4d,0x00021c82319f96d2,0x00062855234b3a37}, {0x000cfe1e952c7999,0x0009fff526834c3f,0x00082932d66290aa,0x000ed0618b6fc1aa}, {0x000f51b1765d795b,0x0000fa7ad3a784d8,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b99ec5345605e,0x000d705b03ab3b6f,0x000d514df02df673,0x000d979497926d51}, {0x000327f3cad9968b,0x00007b8edfa7bbd7,0x000fc5c1dee65278,0x00036db8f1709072}, {0x000d430f38a088d0,0x000f9df3373c9bfa,0x0000000000000001,0x0000000000000000}}, {{0x0002932b26523f7a,0x000a4698ce826d56,0x000d64992b915d43,0x000f137e0e1471fe}, {0x0006811a1c25612c,0x0007e5e74619907c,0x0000f455dcff6a59,0x0006fe503afcb4d1}, {0x000af47382daf8e9,0x00085e51a1e9e2bf,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c899ed52c7ebf,0x000805309ddc9f57,0x000f9ec0561000b8,0x0005599061be65ad}, {0x000c7c6ac2e8bdb7,0x000546a9f7f8f392,0x000b38f709f90fbc,0x0005b81ee256aa4e}, {0x0000cd9ffe3bb73d,0x000ad1e54f791392,0x0000000000000000,0x0000000000000000}}, {{0x000e432eaecfeb4f,0x000efd7d99d4376f,0x000ecbf257042314,0x0006524d0d11bf19}, {0x00070c070e8811a2,0x0009bb46ac80db71,0x0005deced6976256,0x0001f5d4f1996e7f}, {0x000c6d1bc35c855b,0x000b7fcfaf131b63,0x0000000000000001,0x0000000000000000}}}, + {{{0x0007faf89dff5e6d,0x000701e289504c4c,0x000c21c143d7c67c,0x000b8b1051104808}, {0x0001a8547ea3f429,0x00042d1597643f8b,0x000c20d8e30463a4,0x00019700b9ca1322}, {0x000c7c74112313e3,0x00021d429e582d53,0x0000000000000001,0x0000000000000000}}, {{0x000df174dc25e320,0x000421f30a9c65a6,0x000cca7bd70734a8,0x000970e912f441c1}, {0x000b0da35c85642e,0x000fbc6108990f29,0x00004f9201a5ca87,0x0000b4ba5b8a0067}, {0x00032f15dd79c420,0x0001dcaa2c572053,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=56 [{1,2,3,..,}]*([2^224]*G) */ + {{{0x0009f88e8ef42daf,0x000fa3828b99d9e1,0x00051fa512ec2bee,0x0004d684d33e3c3c}, {0x000ae34a6c37abe1,0x0009d4bea55b5936,0x000e6492802583c8,0x000098da605bd938}, {0x000288cfc1f04542,0x000ed0659c47e455,0x0000000000000001,0x0000000000000000}}, {{0x000ddaea5046a68e,0x0003e422472a49b9,0x000c2da95690aefe,0x000dcef36e21cee3}, {0x000eab14f0abf0c7,0x00064941e198c3c9,0x000fcf64819eee0b,0x000dbfe77fa8433c}, {0x000c3b28a2f7686d,0x0009c3dbfd233403,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b7107c522fbbd,0x000c42e8887082c3,0x000cd304c29d218c,0x0004d8477a96d44b}, {0x000ecee7f483ff1c,0x000951d19c530d4b,0x000d68d4d6bf1fdc,0x000fe03d009b71d2}, {0x0005537694d3bd1d,0x0003c3db4cb1a2c4,0x0000000000000000,0x0000000000000000}}, {{0x00086f0e3fa15f33,0x000388caf5fb4c5e,0x000b2f14ba7715c3,0x0002610381191db4}, {0x000e0f68a08e3384,0x000342059cb75d25,0x0002bd292f767fcd,0x000a7a696b414dec}, {0x000dfb67016057d6,0x00035999c277b18a,0x0000000000000000,0x0000000000000000}}}, + {{{0x0008ba73e1f475e0,0x000dd3f1b783f77c,0x0007375b491b116c,0x0007d166e4a377a6}, {0x00011bc5434f7131,0x000a2dbe9d40b215,0x0001220856a6b5e6,0x0007b890e757f7e0}, {0x000f02578a944660,0x0004883ff77008f5,0x0000000000000001,0x0000000000000000}}, {{0x000a3e2632a212b7,0x000302b8f038a3d8,0x000a306506181a49,0x000301702b3192c1}, {0x0002c30f8e6d2d73,0x0002a0c5b0c340d2,0x000f8bfe6e8c90fd,0x000e8d703d0478e5}, {0x0005135b5979aaf2,0x00034da81fa7add8,0x0000000000000001,0x0000000000000000}}}, + {{{0x000f16b10733860a,0x00007e088e20bfd6,0x000859fdd39b30d0,0x00096072d4c40b6e}, {0x0007d0a59d2a4c91,0x000f5b531a3c4f60,0x00005885c546c30a,0x000ddc1d5df2fdc4}, {0x00026f4df2971b1a,0x000bb67cb15104fb,0x0000000000000001,0x0000000000000000}}, {{0x000f74646d17eec9,0x000e3de3bee8f39f,0x000560fc63e96143,0x000d7aab2e0395d9}, {0x0003f099cc808fd9,0x000e3c3dca422f15,0x0002c61efabb0d74,0x0008d736943aed2c}, {0x000d74e4178f87c1,0x0003d1b76afadf96,0x0000000000000000,0x0000000000000000}}}, + {{{0x0009583eea912fe1,0x0006a5587f3b9010,0x0002a020829ea8e9,0x000b82ec346b73a8}, {0x0001f44eda406972,0x00067b4504fbb553,0x000e7a6189bcd268,0x00039bdb40c39e05}, {0x0005c436a03e5e06,0x0007019eedb498d3,0x0000000000000000,0x0000000000000000}}, {{0x000662e3f77764f2,0x000cda79a7c8507e,0x00040c75e0594299,0x000bed938a93a1fb}, {0x000829d1bab3f2df,0x000df99ce48d52d8,0x00032fb8c4092f5f,0x000036b5fa768aef}, {0x000201a9ee7997be,0x000ca0344038e8d5,0x0000000000000000,0x0000000000000000}}}, + {{{0x0004ae17c6c4ef83,0x000409ab5b4b2b20,0x000c4c863d0780e0,0x0003a003f73d00d9}, {0x000f7d17b97edd24,0x000335ec2eca6e6e,0x0007def3f246d97d,0x000f0e2518ced6c0}, {0x000f728fdb8b0595,0x0006dcc1ccb10bfd,0x0000000000000001,0x0000000000000000}}, {{0x0009f012dd02f504,0x0009d50a767d8e86,0x0001bbb0510be6fa,0x000cdab7deeebbfe}, {0x000f08332cdf98f4,0x000468782175d651,0x000610add0fc83f5,0x0004965277afa428}, {0x000ff5c3438635dc,0x0001fc0961df5b9d,0x0000000000000001,0x0000000000000000}}}, + {{{0x00069bd3ef1fabd9,0x00064fe7b00b747c,0x000195c6097cd6a8,0x000470709e13f896}, {0x0002cf79709fe958,0x000b72450de744f3,0x000df3f80fb1524e,0x000527ea0bf24b8d}, {0x0001b2774e7c3f06,0x00031ad38e7aad24,0x0000000000000001,0x0000000000000000}}, {{0x000b784c1f4489c7,0x000f2a889f589505,0x000a5a3175ee6df1,0x00047a68c157fe56}, {0x000497f7b784bd67,0x0005ed8ff0792025,0x0001528aca7e0427,0x000d473d7c41594f}, {0x000cb017e35b5669,0x000f1ce78bb2a52b,0x0000000000000000,0x0000000000000000}}}, + {{{0x000290556eb6cb2d,0x0005feea12a072e4,0x000d082bac976238,0x0007c4331be16424}, {0x00021f06c7a59869,0x000de72b68a812dd,0x000652502f900697,0x000d9acaec02a2e5}, {0x0009120c4a89c7ef,0x000a1bdf713a324f,0x0000000000000001,0x0000000000000000}}, {{0x00067220e1027a34,0x000cabc964145333,0x00029a9fac99b048,0x0001c2850e9e757a}, {0x00057de9cc170cb2,0x000d017c03bef969,0x0008cbf0f3534a37,0x00085a627fd6a9c1}, {0x000c29da8ed78ac5,0x000ae4ef092acab9,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=57 [{1,2,3,..,}]*([2^228]*G) */ + {{{0x00060edf11bb374f,0x00007883a0baf3f0,0x000eb8cb2a326de7,0x00095554f5a8d631}, {0x000236ba14fb5cfa,0x00053769e29d950c,0x00043eac3b6e6d4c,0x000400e396b6857d}, {0x000b3b40a28cc2c6,0x000f0142ca52de79,0x0000000000000000,0x0000000000000000}}, {{0x000536d43744d95a,0x00080dbb696fec4e,0x00086879c7360fe9,0x0006d4736a564e15}, {0x0001262e7ce9d679,0x0004cced89ee75c8,0x0005fe2dd4f732cd,0x00004a4d97d2a3a7}, {0x000e046cc62def6c,0x000c590bfe781afc,0x0000000000000001,0x0000000000000000}}}, + {{{0x0001e39da3d53729,0x000c68258dcf68b5,0x000b1289a021c50c,0x00028ef654112229}, {0x000f4c73b83c7489,0x000a0ebdbdf0df33,0x00033245f1663936,0x0006bdfac3bf0988}, {0x0001bcc9c21bceec,0x000bcb64a15de987,0x0000000000000001,0x0000000000000000}}, {{0x000fb2f27cba002c,0x0002f0ce8d66f97e,0x000c4f49e106a48e,0x000720d60d05177f}, {0x0001b9e35427304e,0x00008c355b746f84,0x00061577b7b7cd5e,0x0008586eb9a88122}, {0x00003b21c38ce383,0x000aec02872c5a8c,0x0000000000000000,0x0000000000000000}}}, + {{{0x0003ee736c032ac5,0x000efdfdb4c3d4de,0x0006ff01d94dbf41,0x000a0bb5ee1889b9}, {0x0009e7a8dc456a87,0x000c2bb8ca379bb9,0x0000100b2eae0668,0x000d2063b0184325}, {0x0006daafabf5c71a,0x000da2051e835c68,0x0000000000000001,0x0000000000000000}}, {{0x000eaacab83acf78,0x000761a09a5ec30c,0x000838055c361c20,0x0002e34e4fa05860}, {0x0001b9a7a3c102c4,0x000f91746c97086b,0x000c731ba10529e6,0x000e8a7f6450292d}, {0x00076f3144af4061,0x00077cacdce041a7,0x0000000000000001,0x0000000000000000}}}, + {{{0x0004b67237f032bd,0x0003fda08c68528a,0x000c719571479a5b,0x00011c75d8054fed}, {0x000bf503c35807c6,0x00037b9ea9e88e5d,0x000c9422b4c4521b,0x0002950eb17e75f1}, {0x0008eca427950847,0x000c0f845c91f923,0x0000000000000001,0x0000000000000000}}, {{0x000c1e80ea8c1010,0x0002e1978ae396a0,0x0002c0d00f914b24,0x0004e43fb47bf7bf}, {0x00034b416e50ae94,0x0007c8d114906c36,0x0000ad0347f03a3e,0x0006a6eba25165b0}, {0x00021e9dc53a14e2,0x00076911eb83f4f5,0x0000000000000000,0x0000000000000000}}}, + {{{0x000bb95adc5ef776,0x0008da0ab8c5f233,0x000a4ab915fb3034,0x000423a63a5dffb8}, {0x00063715aa2304a4,0x000018304840df01,0x00037b80028296bc,0x000854f7402fb23a}, {0x0006ebdeaf3e5fac,0x00005463cd3d8406,0x0000000000000000,0x0000000000000000}}, {{0x000c9468fa481453,0x0003a0abd61a0348,0x00047c025f468b21,0x00090d7078cd984d}, {0x0005284e72a10788,0x000156fa2b542f64,0x0003b3c956017512,0x000991f2c57b982f}, {0x000989b3c05c1c45,0x0001ce6eb96dccdb,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c0c76c91a3d86,0x00001a3cab8eb5fd,0x000752ec9cff796c,0x0003f81d37bdda1a}, {0x000e12c746df5d95,0x00027c38d512fd50,0x0004e23ce093d983,0x0004adeb3545f200}, {0x0008acce0db1764a,0x0004097d3ea0eee0,0x0000000000000000,0x0000000000000000}}, {{0x00057c8acd0df62f,0x000a235a3f7e354d,0x00043547a0eeee98,0x000831405ee1adbc}, {0x0006a916c043f6f0,0x000e4844ab1da766,0x0000728cc7b74a71,0x0007192544b4fdc6}, {0x0009d41decd4aad8,0x000497605d2bc303,0x0000000000000000,0x0000000000000000}}}, + {{{0x000c16253894ed63,0x00072cd0f6a2350d,0x000412983f7baee8,0x00009f78d054975c}, {0x000d7bad42e9614f,0x0005f2ac60d67eaa,0x000622d671f099d8,0x00021f07bed1e830}, {0x00081424aeea7845,0x000f7592014c5f2e,0x0000000000000001,0x0000000000000000}}, {{0x000619d3fbb4d7a6,0x000a9269d1beee9e,0x0002e4ef27debfd4,0x00069156fc40ac5c}, {0x000b31c86af27730,0x000f4aa3fc30b8ff,0x000bfd86f500b895,0x000c9dccb88b68a2}, {0x0003b7832b9f66f8,0x000853497ba1895e,0x0000000000000000,0x0000000000000000}}}, + {{{0x0000b0ab29d3bde4,0x0009c71ea0d8e5cf,0x0004b6204df30d49,0x000d3f548f748b9e}, {0x0006eba95c754c4d,0x0007ad6cc9c44387,0x0009e2d08e10896d,0x000ea9428b4a544e}, {0x000fe71891a2d3e1,0x0006b905c7660eb6,0x0000000000000000,0x0000000000000000}}, {{0x00066e6414125179,0x000e2877106f8fb3,0x0002d9d82f542b36,0x000b4eca63743c2e}, {0x000cfc887146aaf2,0x000ed669e0b28d08,0x000c885c73913714,0x000914ee1f56da28}, {0x000d4479d84424da,0x00015e3364622742,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=58 [{1,2,3,..,}]*([2^232]*G) */ + {{{0x000bd25b5edea690,0x000347c3abc5bd1f,0x00078a11d29a3138,0x000c6d62f2283223}, {0x0007b4af4ece3ada,0x0001c75e4372b0dc,0x0009560a7308e28f,0x000d0ea7127d9913}, {0x0000172da9cb3c31,0x000c36c69386a7ee,0x0000000000000000,0x0000000000000000}}, {{0x0003be2a2cae0b56,0x0000778293313929,0x00041cdee07af8d3,0x0000895f4118b415}, {0x000b9b3a9502ad6d,0x00004e1d44242767,0x0001a843924f3834,0x000a3c5c40dce7a9}, {0x000443e9e0f30db5,0x000c5338df60b62d,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d88264070b39a,0x00060f5a265bdb95,0x000717063ef771ed,0x0004ca000dd813de}, {0x0005a4d348196d97,0x000def16129d2691,0x000bcbbfcdfb3523,0x00057290d6981eca}, {0x0000c5d2444e08c7,0x00091c65a063c65c,0x0000000000000000,0x0000000000000000}}, {{0x0001941d0c9b7cd3,0x000cc0a5845e0e75,0x000c51c09cc90c04,0x000ce8aff11f3a2b}, {0x000303944ca09536,0x0005cb9d10d614a2,0x00062674cf73a00e,0x000639b5262911f3}, {0x000379f49e681d43,0x000ea07634bb0927,0x0000000000000000,0x0000000000000000}}}, + {{{0x000aea3b47740815,0x000ede22e76e20e2,0x0007e03afde6eca1,0x000f70d608d1c44e}, {0x0000b88d585444ed,0x00074d298cd8ec39,0x0001146c4c3f6754,0x00073274110e9ce2}, {0x0009cf1f4b12ae7e,0x0004fa67e9f4b4ce,0x0000000000000000,0x0000000000000000}}, {{0x0002ef9318bf7948,0x0004cb3efdaff610,0x00059bd31047a8b9,0x000fad7c4b8b1235}, {0x000a02488482725a,0x0003b462badd939d,0x0009f8945cb99ab4,0x000c0f6b65c66a4b}, {0x000e97dd0bddb4b1,0x0009d75a1f797639,0x0000000000000001,0x0000000000000000}}}, + {{{0x0000ec0aca9d0f94,0x0002fb13ee984570,0x000e5ad047be55b4,0x00073a0845bc6993}, {0x000ffee41f2aee8f,0x0003c832041ac680,0x000bb4a740b12fcd,0x000521ebc1645e36}, {0x000dfcc7735a45c9,0x0008ef92b0fbdab8,0x0000000000000000,0x0000000000000000}}, {{0x000805ed4e45e0eb,0x000584829d754db5,0x00036cc488a6c810,0x00060f1ec9632468}, {0x000c0f2ebc30d39f,0x000e960758390502,0x000f4626d1feec9d,0x0004f944bca363ad}, {0x000375dbfdeea282,0x00050ba887d0959c,0x0000000000000000,0x0000000000000000}}}, + {{{0x000857ce82a61386,0x000fab480b18fa40,0x000223002b9297b0,0x000f9b41f698e6a5}, {0x0005eb688e33b253,0x000cc09f30bec2ad,0x000eeb6333a5ffe4,0x000bc81e94524303}, {0x000091b660b277fb,0x0008fd03ed404dad,0x0000000000000001,0x0000000000000000}}, {{0x0005dd59faaf5be9,0x000abe017edd5410,0x000aaa4856720e0a,0x000d987f4b1178d8}, {0x000fe5e556ef405d,0x00081f659d8f4002,0x000f7836cb450ed9,0x000350d35c2f69df}, {0x00095c15741d4705,0x0004c982d30172da,0x0000000000000001,0x0000000000000000}}}, + {{{0x000f854cfa0f1b9c,0x00036c27f5709e7d,0x0002fcca4c8efcec,0x000a965fa53691ad}, {0x000f70705ad9f4c9,0x0006ca89080d5043,0x0005cff60c807ea8,0x000ff2e47ec8fa95}, {0x000846288bda4094,0x00003dd7b4f68fa6,0x0000000000000000,0x0000000000000000}}, {{0x000c953f0c60b7aa,0x000060469787d616,0x0009579f0ed52085,0x000faf3c529ad3be}, {0x0006f18223d47482,0x000c07ac1ad869d6,0x000437b5cb0dcd56,0x0004dc765feb592a}, {0x00004a294944df05,0x000184cb4ad2de82,0x0000000000000000,0x0000000000000000}}}, + {{{0x00003d153000960d,0x0008f85dc5ffb696,0x00009b3d9302c4f3,0x000fd9668bf9ae36}, {0x000a25bdc3c8ecf4,0x0005c1a7bbfdb2a9,0x000393a4a4463083,0x000a4d41df85c97b}, {0x000d92cc6610ea8e,0x000bee7a14446540,0x0000000000000001,0x0000000000000000}}, {{0x000fe7092ab307cd,0x00055b75f48236e0,0x0009f206af987311,0x000191b95f1aca72}, {0x000b01a57a7e9b13,0x000784300caa5513,0x0001b4e8ad08534f,0x000e7a92df980dd1}, {0x000edd78871ce4ff,0x000f2fbe3e6a8b6a,0x0000000000000000,0x0000000000000000}}}, + {{{0x000cf8b0ae3366de,0x000c06074154422f,0x00007a17d21e5dad,0x00001e78c237ee85}, {0x000d108f5fd2183c,0x00098eec2dada49a,0x0001c303e35f7659,0x00091f2075445a12}, {0x0007426d8f5fdddb,0x0006af1dd7a92c53,0x0000000000000001,0x0000000000000000}}, {{0x000c31da2f658936,0x000e72228504949a,0x0007646877ed3189,0x000cbac9e04fe426}, {0x00093f8802494ee0,0x000975ea85d9fdf7,0x00038fb0c9c6045a,0x000802b88cb118cf}, {0x00057e39ea12c877,0x0002548e571e0697,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=59 [{1,2,3,..,}]*([2^236]*G) */ + {{{0x0001f89eed993384,0x000860d77e7f7ed9,0x000e9a249862cf0f,0x00026963705ade19}, {0x0005d0f929eafa4b,0x000a12f7eef6f18b,0x0009d590ff1861d1,0x000d6b67d2954ef5}, {0x000540dd418bcbd0,0x00096e83b51e5170,0x0000000000000000,0x0000000000000000}}, {{0x0000a67de6f33ef5,0x000d0f090cb0d50b,0x000b8b3eeba89c8f,0x00026dc5289c0d96}, {0x000e0e6dd7431a8a,0x0000a660c8afde18,0x000cfa02cc76374f,0x000b7654494d397b}, {0x000476b8fd958a15,0x00097d53a314e324,0x0000000000000000,0x0000000000000000}}}, + {{{0x0005d2d913938d7d,0x0006d9d471ad5a92,0x00018a88bdce6ad3,0x000b90be599734aa}, {0x00099aa6c0fb7b20,0x000eda14b233c887,0x00024ee7dad41f1c,0x000cb7c643d13bb9}, {0x000e7d84ea63db6f,0x0003a846de4666cf,0x0000000000000001,0x0000000000000000}}, {{0x000e6128e81e8c45,0x0001464a9235f4eb,0x0002db34ea2780d1,0x000ad3e8b3ecdbc0}, {0x000f8cdbe120731f,0x000f4318097ab5eb,0x00079ebbf1d4990d,0x0007e93c15830f8e}, {0x000c40a1cfba03ee,0x00068df76de664ef,0x0000000000000001,0x0000000000000000}}}, + {{{0x000267eeb458a4eb,0x000e492a1279f17f,0x000c76f729af11c0,0x0007339e92b8f2a3}, {0x00014e00c8ca3543,0x000dbcc01641f96b,0x000755e449dde571,0x000a0a0df11e5fa3}, {0x00031770742dc646,0x0006f53610d1c65d,0x0000000000000001,0x0000000000000000}}, {{0x000e5b808553c438,0x000499c99fe83176,0x00019eef1a061f75,0x0007deb8d4c21a60}, {0x00033192fdd7ecc6,0x0003250ef2ec37ce,0x000aff8f6ed93441,0x00058d1e9777ac4b}, {0x00083407f0d67d5c,0x00079fd1b5287171,0x0000000000000000,0x0000000000000000}}}, + {{{0x0009d6152b9c90b3,0x000aff28f438100d,0x0000a2146e64ecd5,0x0009ad5b7e0c0df1}, {0x0003f6775181b0d1,0x0003c7a1b8151d7c,0x00035c338a3ef9f4,0x000e1fdd81712dd2}, {0x00034726a1f2597e,0x000cb3083971de47,0x0000000000000001,0x0000000000000000}}, {{0x000c041200d9bef5,0x00067c85ff590dfa,0x0008b72aaea952b7,0x000265e7843ae9c1}, {0x00032307d7542004,0x0002dea503395a89,0x000e1eec7f73a8fc,0x0004b86c7eb53399}, {0x00021cf862926b2c,0x00019347a4f0820d,0x0000000000000001,0x0000000000000000}}}, + {{{0x0004f7b384cd7297,0x000acc1a4787e2cb,0x000a5590739ea069,0x000be73fa99d3f7a}, {0x000e7cf6cca23601,0x000960cc53aa83e6,0x000b86b5920b1173,0x000fa7ac9dbe20e0}, {0x00068bb6f6b3ef99,0x00064060ac6c56f4,0x0000000000000001,0x0000000000000000}}, {{0x000626c189f97eeb,0x00092eb7dd82a1dc,0x000142f8425008f8,0x0004b241705cbf37}, {0x000c04c8483d101a,0x0002075cfd275951,0x000f282a7dae45a8,0x000263c98e81a9d9}, {0x000c331b130bf290,0x000c95c5f55addef,0x0000000000000000,0x0000000000000000}}}, + {{{0x000ac9a1c36e3c24,0x0008e09fb461cd04,0x0007bca251b2abd0,0x00098ad8fa6e8384}, {0x000087e953f04f5d,0x000d9f6c57ca3dcc,0x000f179679c5992f,0x000cc9cf93cdfac8}, {0x0000d1a320e29622,0x000580703c650a05,0x0000000000000000,0x0000000000000000}}, {{0x000188e58d397fdc,0x00021860a160b5c1,0x000be9a3f426d729,0x000f8747311911d3}, {0x000b78a79eb1a864,0x0006b881dee2ff5e,0x000fc87ef83107b6,0x000ed139997a8784}, {0x00080e84fd894e5e,0x0003aa1a2f41979a,0x0000000000000000,0x0000000000000000}}}, + {{{0x00031707560bd53c,0x000c9501712fc58e,0x00010e30ced00607,0x000465e677b023a2}, {0x00067620a31c653b,0x0008ea62e5b10af4,0x000d99048d08ca5a,0x000d65af877831dd}, {0x00029f5ab5be8899,0x00028321d38a082b,0x0000000000000001,0x0000000000000000}}, {{0x0006bb42378c6e38,0x000fb8ba4bc4d5d5,0x000a4ef2e4b28b7a,0x000b41f7c314822c}, {0x000a2882d9a51ae9,0x000e6c2280f2b327,0x00019697ce965ab1,0x00075708dae4a49d}, {0x000f1175eea346d9,0x0000e9572d2fbccf,0x0000000000000000,0x0000000000000000}}}, + {{{0x0007d4bad9839247,0x0002744af0bab5bd,0x000cd1e04b0ade61,0x00080a7fe21c3a53}, {0x000864b9871244ec,0x0003f6fd1428f483,0x000889d63b180bf2,0x0006acf748b0e2b4}, {0x000a061072996dea,0x0006607cdec9d5f3,0x0000000000000000,0x0000000000000000}}, {{0x000a9858e242fc0a,0x0001fa2d38ce4fc2,0x00087521e69c709b,0x0005bae8f5996fd9}, {0x0005e131ac99f4d0,0x0002578fe3c3bad7,0x0002279c9d139206,0x0004b6fbac903ada}, {0x000292885696da36,0x000ddbd02ce1356c,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=60 [{1,2,3,..,}]*([2^240]*G) */ + {{{0x0001074e0ee1b4c2,0x0001a0ff10eeb73d,0x000c77549f2e087c,0x0006808a5e2e0837}, {0x000e948c7156c74d,0x000c13bf7c11f82c,0x000a51472ee287eb,0x000b28c4e6f906f6}, {0x0000fef0b165038c,0x0002d8b6f1c9d932,0x0000000000000001,0x0000000000000000}}, {{0x000d310ce2cf5a19,0x000c08add6b6df57,0x000dcd4cb28cd825,0x000e1cbe48bebf85}, {0x00051f1d5aa6a644,0x00008ba85cbebbd3,0x0001bc84d8a2aa19,0x000d343d2d77518b}, {0x000a9098229b988e,0x000a9f940fc8d07e,0x0000000000000001,0x0000000000000000}}}, + {{{0x00096ebc173a5d8b,0x0000ed54a67c958f,0x000ef67809a984cd,0x000abb728dd8453d}, {0x0009f4fe5f363de0,0x000e4fc4614b7360,0x000fee4aab1b83c8,0x000606d2158cb989}, {0x000096597fe56f7d,0x00057270734a0c52,0x0000000000000001,0x0000000000000000}}, {{0x0004ef503a18dccd,0x0009b732b2f44d09,0x0002c29898debd6d,0x0005ffd4e0ef3ff7}, {0x0003830b99ae2e5d,0x000dd5fca5e1c94a,0x000e97015b084de2,0x0000e94504be6d08}, {0x00079eaed90fe0fe,0x00051eafa2897ddb,0x0000000000000001,0x0000000000000000}}}, + {{{0x000dd470cee4255c,0x0008ea907208e445,0x0001157ba3e551b3,0x000f94c51b72b693}, {0x000b183c616c9cf9,0x000fe84fcaaf1c59,0x00077c97ed67f1d2,0x000a1d1e1a09f1bd}, {0x000c2f47751550da,0x00038d58d345e7ba,0x0000000000000001,0x0000000000000000}}, {{0x000d95b5f854d3f8,0x0004daa404c99ba5,0x0005a1bac7d29f41,0x000da46981c9d673}, {0x000652c1bc49956f,0x000e505f2a66bdcd,0x00081069783eab5f,0x00036f9996ab9237}, {0x000238170a7330e6,0x000670fa70e33da6,0x0000000000000000,0x0000000000000000}}}, + {{{0x000e86a58386cf1e,0x000572f15be04bfe,0x000e35f663e32b87,0x000f5c5294b48632}, {0x0001362fb4267165,0x00007dadeb7d3b94,0x000012e189d86c34,0x000d63e5f7805689}, {0x000b7e33561c7907,0x0003c0c0dc085fa2,0x0000000000000000,0x0000000000000000}}, {{0x000cbe4f2b4f6916,0x0002f30cfb081da9,0x00011fe0a525ac2d,0x000bbd4632662679}, {0x000223e344c2d8fc,0x0000a08757b1ff41,0x00001c0a48229e9a,0x000456b8c92c6f71}, {0x0009a086c4af0e80,0x000d74c21360d8bb,0x0000000000000001,0x0000000000000000}}}, + {{{0x00000ce7164d30ca,0x000fedb34ef3a45a,0x000800620f8ac5c1,0x000c7c79ba6237c0}, {0x00088ff56f44997d,0x0008ab94745563d5,0x000df21b2b00a9ae,0x00086a286295b8d1}, {0x000426dbb4cf9b37,0x000ec830b7004300,0x0000000000000001,0x0000000000000000}}, {{0x000a97f98f792db1,0x000f9d5a4e4b251f,0x00061b4d385d3a62,0x000b4cdaad987701}, {0x0002341727a73b90,0x000c4abf7e84f908,0x000f0ded814c4a3e,0x0002c453671bed3d}, {0x000a888690c8945f,0x00019a94087855dd,0x0000000000000000,0x0000000000000000}}}, + {{{0x000e5f28e7fb0752,0x0005ca705e5e780c,0x00093ca952d3af74,0x000eccdc7351cb28}, {0x000bfc12e9837fed,0x00014b93567e7bfb,0x000546247999f34d,0x000a3f729887c629}, {0x0006edd285692b0d,0x000fc9812e383cc9,0x0000000000000000,0x0000000000000000}}, {{0x0001bc941d72f110,0x0009ba70199860ef,0x0002d090c33493b9,0x000503ff279e0c37}, {0x0005c3fbe286bbe1,0x0006d81c3e80f646,0x0008d3a0a9257bf0,0x000e402ee72a2a44}, {0x00092b91c6a5669f,0x000ad95315497ce5,0x0000000000000000,0x0000000000000000}}}, + {{{0x000b397c9c0b3f5e,0x0008573318572b93,0x0000e667f17a3327,0x0000b9137af5bff7}, {0x00017fb65a96b2e6,0x000188dfee9e25ef,0x0002c9edd117ab29,0x00015472d03d830a}, {0x000512ca1063aa4e,0x000adf9593f42bd8,0x0000000000000000,0x0000000000000000}}, {{0x00043be475ba796e,0x00023dbbccf7b7a1,0x00080d4a4b140a81,0x000247748562ea6a}, {0x0002f53e144c7e84,0x0009ac8b6fa39d88,0x000b1eab451fe154,0x000ec4538f34ffe4}, {0x00076cc290527aed,0x000d83bcd8efdc10,0x0000000000000001,0x0000000000000000}}}, + {{{0x0000c5c5bcdaef57,0x000720046d980167,0x0000601ff33f7b5e,0x000e6b0aa07084cc}, {0x00007791af83b900,0x000dd856fdd2391e,0x0008d880430654d2,0x000f826068896340}, {0x000a4b443c176e8b,0x000c4e18c663e62f,0x0000000000000001,0x0000000000000000}}, {{0x000d8ae0441a598d,0x00065c950e8cec48,0x0001316d53d11b80,0x000cd6863a5ccc8c}, {0x0008ced41a668fee,0x00076ba771bb4164,0x000b49a9a4bb4b6f,0x000e370974d5a9b2}, {0x0000485def9f130a,0x0001b84c25f49d20,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=61 [{1,2,3,..,}]*([2^244]*G) */ + {{{0x000f43bc2dfb404c,0x0008b314169ebb20,0x0004d9ce016776ec,0x000b6f3c40cc814f}, {0x00075de6701fd64d,0x000be16687bc27b3,0x000c8ca41b2641f1,0x000e5c7ebdd8aaf1}, {0x00021918a667e429,0x000d9384d2d4cd92,0x0000000000000001,0x0000000000000000}}, {{0x0009667398bc8b35,0x00026c502a6f1f65,0x000612fc65ae7726,0x00047588d1717c45}, {0x000b0cd0bd27306b,0x0007b68702fd9dbc,0x000a43a5beaed3af,0x0008d9e3bfb560d2}, {0x000c14b9b618c615,0x0002421dad1537ee,0x0000000000000000,0x0000000000000000}}}, + {{{0x0009e7889bf8b6ea,0x000292e957ebe27b,0x0002bd715ca3ae1f,0x000ea4756ae08fea}, {0x000f5cab67aaf3b2,0x00047caa37f247c9,0x0001b8953af0925d,0x000557e7bd1f409f}, {0x000979eb14ee5b8e,0x0008bb0e2890300b,0x0000000000000001,0x0000000000000000}}, {{0x00078d6b71ea9467,0x0000c63dfcb1d01d,0x0008e6aaf05595db,0x00006ff49217ec90}, {0x000b8ab12df36451,0x0004207aff8489ad,0x000b85d257b835bc,0x0003706e08a1abbb}, {0x0002a5b7d71ad10a,0x00056ae224792f6d,0x0000000000000001,0x0000000000000000}}}, + {{{0x00028daca765356a,0x0000ee74680323bf,0x00005f51f95b6f29,0x000c6656c3eab37e}, {0x00038ce239752348,0x000cf8aff4ec8ff2,0x0003adc6745b80af,0x000e56c344a76be9}, {0x0006011d48a9f827,0x000ed4a3e016c1c3,0x0000000000000001,0x0000000000000000}}, {{0x00096e22c7a80fb0,0x000891624b09ab2c,0x00049f6d4616a383,0x000b275d1cd7006e}, {0x000adb9075332618,0x000586e58c4079b5,0x000645e2b19bf36f,0x000f76906dc5820f}, {0x000f6bdd7a51a11b,0x000b67dd8137d034,0x0000000000000000,0x0000000000000000}}}, + {{{0x00095b8b650fd71f,0x00014965b39aaa82,0x0001270c9510690f,0x000b3a9f763e6fce}, {0x000be3f839143baf,0x000866c1890dc990,0x00074c8a0d6a0e73,0x0004b520d476087b}, {0x00081006ed8910a1,0x00052b16e6fb91bd,0x0000000000000000,0x0000000000000000}}, {{0x00057756a63f7354,0x000a1ebe4b13d097,0x0003f1884cec54fb,0x000503924519ff97}, {0x00059fb9e4f42689,0x0002ce5e202d309b,0x00098e0004b85f0f,0x000b05c20b1735d8}, {0x000add1fbd4f54e0,0x000dcb178e2a7f34,0x0000000000000000,0x0000000000000000}}}, + {{{0x000b4ec871f1c2e0,0x0003b5de22756058,0x0000837f536100e3,0x0000c16f1ccaf424}, {0x00056dcf857da01f,0x00038c07ebb5fb6d,0x00096a096d6242a9,0x0003055c2b2c30cf}, {0x000a782e55f66ac8,0x0004f012cc9c2c3a,0x0000000000000001,0x0000000000000000}}, {{0x000506352d9bc635,0x000b06aeaaa56b9a,0x000095186c35174d,0x0002167f537385d7}, {0x000cf421a089f0ea,0x00022c37dc99678c,0x000356bf33a69466,0x0003e97eafc66d63}, {0x000e3c0197b16669,0x000d912a0fa3fee5,0x0000000000000000,0x0000000000000000}}}, + {{{0x000a52c892d50f2a,0x000dfd8fc48ea89f,0x0004b09bae86a680,0x000a1e1278d67917}, {0x000370f37ae3b7ce,0x000f51107a8383d3,0x0005c8e157913dc9,0x0004c347e479bbd0}, {0x0007dfb63e7f7f02,0x000ac4a32c2410c2,0x0000000000000001,0x0000000000000000}}, {{0x000c5983d275b47a,0x000f3ebc8a42a9a0,0x000db0533ac31f03,0x000cb9b707b4440f}, {0x00064522041e91b5,0x000076367218816d,0x00023be78c44489a,0x00060289668fa7d8}, {0x000b7bda9a033e06,0x000c041bf9880e14,0x0000000000000000,0x0000000000000000}}}, + {{{0x000df487e3ac46f1,0x00026f6aeda132f0,0x000f9138020c745f,0x000a8841334fb2d5}, {0x000c66722bd6a712,0x0005a1ea4533d309,0x000636323c57e66d,0x00008eb26f26bbe3}, {0x000fa479b5dc3fac,0x0008f02e50177e02,0x0000000000000000,0x0000000000000000}}, {{0x000c92b56635aa25,0x0007f4d610ada23b,0x00010104e5566f0f,0x0004f4b30ea26b57}, {0x0004efd9675fd322,0x0001af8f50ca2f0b,0x000382d177aefb1d,0x000538151171ef73}, {0x0003347891a319cf,0x000711dcdab779a2,0x0000000000000001,0x0000000000000000}}}, + {{{0x0002b024db8d3d78,0x000aa5cc47fd4499,0x0004ca3c2ae301e6,0x000f6635a239d460}, {0x0001e72a93968b59,0x000f3e7cb493da74,0x00057a0e451c8476,0x00090539a4ae9584}, {0x000df123a0ccc6f4,0x000f3e4b36ee4a70,0x0000000000000001,0x0000000000000000}}, {{0x000bf5964aec0226,0x0008d0d54e934ea5,0x000838881f3a4f9d,0x0001904c5759057f}, {0x00054f74d21e3d23,0x0009110e0965fa28,0x00025473e3fbb9d0,0x00066659568773f8}, {0x000e05953c3213d4,0x000de0c6c9fbf74e,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=62 [{1,2,3,..,}]*([2^248]*G) */ + {{{0x00023dd4a823eac8,0x000e2984db5faf62,0x000d39d495d0b0f5,0x000841ce0a14e52d}, {0x000cb365ed8de723,0x000bcb1fe9b4ef0f,0x00047ee3aa7d5d2d,0x00032d33c7e0b190}, {0x000d3b9dde2978f5,0x0005329dbc97d12a,0x0000000000000001,0x0000000000000000}}, {{0x000d165556c00ac2,0x00057fb3172c7a02,0x000beeb32c48804b,0x00099dadad774958}, {0x0007934b2bc9659e,0x0003fd281f90ab3c,0x00013abe477effe3,0x000378b329a3faa7}, {0x00036cbb9f3df235,0x00014562e824d0ac,0x0000000000000001,0x0000000000000000}}}, + {{{0x00078de5aeb19662,0x000d1f458b071bd2,0x0000313ed9b5b584,0x000ce5fe5f476cd6}, {0x00077007678c1c54,0x000258964a6d145a,0x000420c0c62f8fee,0x0008595f7056fa68}, {0x00089119263621f2,0x00019c5d8cf9f82d,0x0000000000000000,0x0000000000000000}}, {{0x0000243d2e6cb7a7,0x0008d78d5a87bb13,0x000c4e517b3bcf90,0x0000f65dd0ef67bc}, {0x000d930e5dd35ba7,0x000eb9fb0741af2f,0x00075eb448314eea,0x00003702fdd3f71f}, {0x0002c0a91bac4835,0x0008e91ebd11ec1d,0x0000000000000001,0x0000000000000000}}}, + {{{0x000160f761d5cda1,0x000d737ebbdcd5d3,0x000d7809f98ac1a9,0x000dcdbd555d558c}, {0x000fa398a682b263,0x0004f2c8dcb516cf,0x0003f1fdc39fa308,0x000c8f35f0892119}, {0x000b1a2e7b6b0e00,0x00060305cb4b78de,0x0000000000000000,0x0000000000000000}}, {{0x0005dc3804225ef5,0x000a4dd0142d36ef,0x0006183af50e0ed5,0x0006c670fcbeb371}, {0x0008629b90cb9a81,0x0002af3261b35739,0x0008aa6b6e9823af,0x000fbd19a0ec458b}, {0x000311a98270a0a9,0x000299fe388efd5b,0x0000000000000000,0x0000000000000000}}}, + {{{0x000bf04df8057734,0x000b1031cb2cfcf4,0x00013e25b6e0c3ad,0x000fa4cf3dd2ab40}, {0x00007758e2edfff8,0x00031ad907feffd3,0x000b4564baf111bf,0x00073c4f6a12eba1}, {0x000fad75513a8160,0x00074de7fc43bd94,0x0000000000000001,0x0000000000000000}}, {{0x000d44eee593810b,0x000f1e369ffff5b8,0x00064f19da65334d,0x00021d0f5c2b9ceb}, {0x00091c5ca3390013,0x0005689acf02b87e,0x000bf7c3a49c8b54,0x000093f7ed7c049d}, {0x000a0a1d58d27784,0x0003d7726f20ba73,0x0000000000000001,0x0000000000000000}}}, + {{{0x000ec366cfe8b8dc,0x0005109510bbe0e0,0x000aea6451541d52,0x0007f0a613f8478f}, {0x0000f85f758544a0,0x000f2b250e479f86,0x000b98246494bdaa,0x000483bd40872db5}, {0x0001ef43cbf9eee7,0x000bc8abec7ccf7e,0x0000000000000001,0x0000000000000000}}, {{0x0005c21b7bfcf30c,0x0003ee117f493af1,0x000fdb4b0534832d,0x0001fa1bf55f45c9}, {0x00078f7e3c1efc43,0x0008a7de78a32d11,0x000dc3559e7ee791,0x0004e7a865c863d0}, {0x0007e1e671e05898,0x0005634bd3bf85ef,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d81691c2ab48f,0x0002541bcc0607cd,0x000da791a7f87cac,0x0008347771261610}, {0x0008e3bac250c120,0x000659e04001e2db,0x00052cf1b9282899,0x0009b392289990cc}, {0x000164d7d417f75a,0x000945df41fb11a9,0x0000000000000000,0x0000000000000000}}, {{0x0009c07103c6ae84,0x000480fee72710d4,0x000b8360a2b9d155,0x0000e0d3d7e07da2}, {0x000d149f0388daf1,0x0002f71dfdfd1523,0x000418130ff86f14,0x0009f7346c3e8b09}, {0x000b2e35c5e699a7,0x00052b1c1fdc6ba3,0x0000000000000001,0x0000000000000000}}}, + {{{0x0003345adc3b52d1,0x00068ceb5ac071d1,0x000e11041a7e60d2,0x00074a2a70677323}, {0x00040734a03353e1,0x00041105a4c9d8da,0x00044e1f19873918,0x00091f314a743285}, {0x000322e1b26f9179,0x000d3d1613ed69ba,0x0000000000000001,0x0000000000000000}}, {{0x0004543cf00ed5fb,0x00053fab663cf4cd,0x000be8cc8843f104,0x000f74fe45dc2276}, {0x0004319b78ce0e03,0x000389520a05dc90,0x000cbb592960d2ca,0x00026ed0dfe3e1c7}, {0x0009036fca8841aa,0x000549d9ea193025,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d95d4b57fe6e7,0x000237e45eed99e3,0x000ddc0cb978fe19,0x00073f687eb46e14}, {0x0006f57bdaf6e4df,0x00047741a18670ac,0x0004925a46fbe2b8,0x000282f9632d0245}, {0x0002e144fc15a10d,0x000815c55aed10af,0x0000000000000000,0x0000000000000000}}, {{0x0004dce0659c9bac,0x000b25a506cebbc4,0x00048b6559b03aaa,0x0008048a933863a2}, {0x00020d37a9de4c34,0x00020f440226cd5e,0x00074e9ee95db69c,0x00082f425e1eff1c}, {0x00033a6f8d820bb8,0x000ef06f95cad219,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=63 [{1,2,3,..,}]*([2^252]*G) */ + {{{0x000e4e5c561d5097,0x000b584aedde8b8f,0x000aba27830c0d36,0x000c7d59c0f869dc}, {0x000b714a35cbc32b,0x0004bed4bc22d71a,0x000061f00680d9e0,0x00033bf0836adf25}, {0x0000d7fedb3d768c,0x00095c616b984e3b,0x0000000000000001,0x0000000000000000}}, {{0x0008c5d5c16b3659,0x0009b2bd923c283c,0x000fbaf03219a32e,0x0005bdb0f0d8e95a}, {0x00027d3039b5fed1,0x000c59ce26d89427,0x000676fa1dec9a4c,0x000121992696f0b3}, {0x000260c98b8c7cfb,0x000541c1c9792927,0x0000000000000000,0x0000000000000000}}}, + {{{0x0002f821dc52854f,0x00009f01e839c81b,0x000f9520b0acb7e7,0x0005ef6ec4857ab9}, {0x0007209f9eda63a8,0x0006daca6651f475,0x000d6976e7173379,0x000240b37bfaccec}, {0x000b4437229f4ce0,0x0001910fe8a0ffcb,0x0000000000000001,0x0000000000000000}}, {{0x0007819702adce4e,0x000559cf2ed75ea3,0x000524a7d80f948f,0x000a15733e1fceef}, {0x0008e25fd5510058,0x0004c29ed3586531,0x0004e51cb7d9fa11,0x0006f171b487caa6}, {0x0006163b82558054,0x000686de74000000,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b150deff72861,0x0006369a87c08222,0x000222ff210ad76f,0x000b4d66f2177234}, {0x0006ffb6d673f874,0x00059b847a7a63aa,0x00088181fb601b85,0x00061d5a56e17f52}, {0x0009cad3b2dceae5,0x000b7064799ea515,0x0000000000000000,0x0000000000000000}}, {{0x00071777678fbf96,0x000ee3fb840953c6,0x0006c82ee1fe6943,0x0000476345986586}, {0x00027b2c01a1da88,0x0001dd9ed1d2e620,0x00093b57ac9ecf98,0x000aea3ed52ca86a}, {0x00083c732ab42d43,0x0005274badd57297,0x0000000000000000,0x0000000000000000}}}, + {{{0x000860f86c3463c3,0x000fa451b92975d3,0x000e9c89aced751f,0x00082df00fff3b8f}, {0x0004d44ceed1d22e,0x00022e7d389ef4bd,0x000e91dec43e5b63,0x00003ac6cd725daf}, {0x0003c7139385f22e,0x000deeecc87ca1c2,0x0000000000000000,0x0000000000000000}}, {{0x00051580221e0fc8,0x000941fbf97dbfd6,0x000689ac9e872372,0x000740f84611974e}, {0x00046e04ba0c6ce7,0x000419caa07a8f97,0x000565905b0cdaf7,0x0003cd2570033075}, {0x0003b2c015af7e40,0x0003287d54b47d8e,0x0000000000000000,0x0000000000000000}}}, + {{{0x000087653c1971e3,0x00077f0a153e5f28,0x0003ed97c2828991,0x000095935157a12d}, {0x0003722663e4ec8c,0x0008ec4fe824b403,0x000dd44aff641d97,0x0006da087485323c}, {0x00044e2cb2c2b861,0x0006eba170078d74,0x0000000000000001,0x0000000000000000}}, {{0x000b5b83303bc746,0x00013dec62dad85e,0x000eb52890ef39fd,0x0007414a67a192cc}, {0x000ca32294492ed1,0x000e1460a8a16787,0x00024c190537d3f9,0x0004d8dd49fa5c1c}, {0x00025dcc6564966f,0x000872ce77f122af,0x0000000000000001,0x0000000000000000}}}, + {{{0x000f907cb543fab3,0x000c69a83a50077e,0x000cb4bde1d63e29,0x000c33fc44a3bb56}, {0x000f82a91020da98,0x00021551aca18ce5,0x0000524e8061c783,0x00030bbde84c8ff8}, {0x00080aaffd7e7a07,0x00025503d2cefb4d,0x0000000000000001,0x0000000000000000}}, {{0x000d489c09cd7406,0x0003bbf72e670d86,0x000584e3d6cd578d,0x000c5357fb7dc505}, {0x00078eb6069dbc02,0x0003d4fa59d19b85,0x0000398d29e9ce8a,0x000efb99649fd29d}, {0x000ce49c616fbd54,0x000860686cd02ccf,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b5280bf38c012,0x000b16e5d0e927a5,0x000e0fabbb77cb27,0x0009a5fc1a89a6d3}, {0x000a92cdb34cf1fa,0x000464dc9a9cf793,0x000eb464387be7aa,0x0007fbfde96cd4d4}, {0x000232f3e162772a,0x000656aee57c0a02,0x0000000000000000,0x0000000000000000}}, {{0x000a4f88de057838,0x00017126373b6048,0x000e4a0f6263ace7,0x0001d00ce841a9eb}, {0x0004964f3fadfdda,0x00053a6795df2847,0x0007970db46fe5d6,0x000fc5cc5ee32eac}, {0x000634cedbf5f3ce,0x000fca10e755fd0e,0x0000000000000001,0x0000000000000000}}}, + {{{0x000aeb844e0e5a47,0x000278171b725206,0x0003ef95a8f2f67f,0x0001fdfb411d7c3d}, {0x0002cbc9db5d5134,0x0004cd3d499c831f,0x00090a75fa0db406,0x00072c8d72cfb8bb}, {0x00070a986050fdef,0x000f2a584d26e897,0x0000000000000001,0x0000000000000000}}, {{0x000357b6cd8cc5f7,0x0006b375fe114c8a,0x000aa2296d0e1fc2,0x0007cba1e2fe623c}, {0x000b8ca73315ce03,0x0007e86db236843e,0x0005e04b5b70ddfb,0x000420198f9d4b15}, {0x000535cfa0692139,0x0005a2aa06d43751,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=64 [{1,2,3,..,}]*([2^256]*G) */ + {{{0x000cc45663bc9f5b,0x000b2868f57feb89,0x0004bd3cbd6a2543,0x000a5e560bf63c0e}, {0x0000e648f4a5666d,0x000591427cb7d9cc,0x0005977ab848b1a7,0x000af4656829e85c}, {0x000ae8f7a4025667,0x0003b4ab876527cd,0x0000000000000001,0x0000000000000000}}, {{0x0004ed81840ffbcd,0x000e4830db96c420,0x00026c352dd1b3e5,0x00003369497308c9}, {0x000023370174e547,0x000c6d8497a95345,0x000ecbfae86058c7,0x0008a32e4cdcae7a}, {0x0004e9eb567daf0b,0x00085eaf8dd7df3a,0x0000000000000001,0x0000000000000000}}}, + {{{0x000deae24e7511a4,0x000762cb23734ed7,0x00066bcd84d23939,0x000c037c989a46bd}, {0x000c06543988385e,0x0003f08c8acc808e,0x00000e7680dc66ca,0x000e4c3c5332a768}, {0x00063204acc98ee9,0x000db0a0ef46de86,0x0000000000000000,0x0000000000000000}}, {{0x000b4a4e27fff289,0x0007eeb14e22b305,0x000e9d3141e930a3,0x0004f15435f5cd09}, {0x00052f55ccf3f336,0x0006c9377055f313,0x0000f610c74549ef,0x000c8d0207da4bf8}, {0x000b6ee97b1c2b15,0x0000920992fd2caf,0x0000000000000001,0x0000000000000000}}}, + {{{0x00043f2525d2ebb4,0x000811edb2cca106,0x000c996f27900e32,0x00092edb6f6af92c}, {0x0002dbdef8275bf9,0x0004dd3d26338446,0x0004401818a7ff9a,0x000d60e7694d8e21}, {0x00054e87fa7aec62,0x000f988bdd22449d,0x0000000000000001,0x0000000000000000}}, {{0x00063c9fbe4e6775,0x00026d7e7ff11afb,0x000c6b3e18f7eec0,0x0005c983e08b80f1}, {0x000b75d6b5a4c84b,0x0005f99e3a4b0fd4,0x0005a7cfc4904ce8,0x0006c336a99a7afd}, {0x000e4a736ba1e62f,0x000595be20ba2924,0x0000000000000000,0x0000000000000000}}}, + {{{0x0008c92f010fc781,0x0002463423f6e871,0x0005f129e35dae8d,0x000d59f4aeff7db0}, {0x0000c963932f5dba,0x0005e468db3cf82c,0x000e23c6b7d10e1f,0x00006e08595910e6}, {0x0008880e8c76fb1b,0x000134e8c1259453,0x0000000000000000,0x0000000000000000}}, {{0x000506649d87b671,0x00014c272ea4f089,0x000aa2740669dd1a,0x000622f8a6cc0d62}, {0x000e392244f6f191,0x0003dcbd9dd28338,0x0000c61a8dd7166c,0x000e4930a90ca39c}, {0x000d41296b979b8c,0x000037aa5c88b76c,0x0000000000000000,0x0000000000000000}}}, + {{{0x00051d16f31cd955,0x0003fd720fff5e54,0x0006c62e42ceacd9,0x000b72856f74fc83}, {0x0002b8a51db93ff9,0x0006ca983e7b6bb4,0x000e06f8fd893a36,0x0002491c6c8908ee}, {0x0008e9f64e123094,0x0005764984e58063,0x0000000000000001,0x0000000000000000}}, {{0x000ad9aba979f347,0x0005557b9d835c0b,0x00089b7877984846,0x000ce8c3c6bb325d}, {0x0002e0fb571c388f,0x0007185f17237c5f,0x000ac5737bcf4832,0x0005f037df6f53b0}, {0x000b6f7ae34a972e,0x000821f685c7b273,0x0000000000000001,0x0000000000000000}}}, + {{{0x000e5575bc0dd4ad,0x00028eabf66053ac,0x00054861cbd6dc53,0x0005b123ea9fdaff}, {0x000c00ecf823c855,0x0005d8934d09e411,0x000eb090ae97a01a,0x000591dabc9c170c}, {0x000f751f273c40a7,0x000f0f52861011d8,0x0000000000000000,0x0000000000000000}}, {{0x0002bc9a33075cd8,0x000bb779de4fde35,0x0002eb1b199f0130,0x000e29003c4457b6}, {0x0009ff04878d3a95,0x00004ebfeec1a9dc,0x0007d0d097a6545e,0x0008673c7b41f5aa}, {0x0007894e63c5c4ce,0x00005b385d1700a6,0x0000000000000001,0x0000000000000000}}}, + {{{0x00046d7efcbdeae0,0x000fbb2f349cfbe6,0x00022f14a9cfd187,0x000ef46f7fb5a2ff}, {0x000d8084df701781,0x000b2e7da6ada115,0x000273537b36285a,0x000d779e5cbe2143}, {0x0007b1bb342159b5,0x000321182d17ef98,0x0000000000000000,0x0000000000000000}}, {{0x000974b9395d5c1b,0x000a203e9046670c,0x000c9fa51be4f31c,0x0000167fed87df23}, {0x0006d7ab1aee3553,0x0006c8a7b334d671,0x00037b8b3f821601,0x000677ee013df3eb}, {0x0007a3a1013ff132,0x000a7ed7d1a2e9a5,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b3fd002806466,0x000fd4605cfac818,0x00085d2f0b811efd,0x000167145bb41efb}, {0x0006e3c03cac7ee2,0x00085c4b2dade36a,0x000220dcd3725a14,0x00032cf525a550bc}, {0x00014db66b11c84f,0x000013664e47ace3,0x0000000000000001,0x0000000000000000}}, {{0x000a48858e7d464c,0x0002276f7bbfd1a7,0x000e24ada567d04c,0x0006a941adced466}, {0x000c270addbb103a,0x000761ca82f14e02,0x0004d0794b62798c,0x0007a0bec3f90326}, {0x000caf618966e8d4,0x00021a1f211c02e6,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=65 [{1,2,3,..,}]*([2^260]*G) */ + {{{0x000c24408b5b4bc1,0x000d861e48c2aa26,0x0008746f93b2fb6c,0x0005f018515690c4}, {0x0008d76a3c1b771e,0x00093035c899fbb2,0x000d18ac338e0049,0x0001b4e7f02fa3d8}, {0x000fabf9b804c035,0x0002f83e6175e309,0x0000000000000001,0x0000000000000000}}, {{0x000830680485a054,0x000962c1a8a622f6,0x000f94a3f3450d94,0x0007a83e0a44d62d}, {0x0000105319e21805,0x000a4a1ebdd2bed2,0x000f4863d6076c13,0x00034672ca137368}, {0x0006135e4ef4b1a4,0x00020b6692537ff9,0x0000000000000001,0x0000000000000000}}}, + {{{0x0001b7a5e08fe30e,0x00029eb048815ab8,0x0002e0a161f11e12,0x0001801becb84207}, {0x000ad5b394a58f8e,0x0007512807890edf,0x000f675b3e4e4773,0x0008c99841055d81}, {0x0003ed35c1050ce1,0x0006c217bd56acf4,0x0000000000000000,0x0000000000000000}}, {{0x0000d6c8c5a1e005,0x000ddbaf53db5dcf,0x000c6ab4e4f6ee72,0x000e686032c8481a}, {0x000415c545af61b4,0x0003595ad60e7c0e,0x000f59b261ffe75c,0x000cf66fa7cfbf47}, {0x0002e7097fa1aaf6,0x000a8386b7977f21,0x0000000000000000,0x0000000000000000}}}, + {{{0x0000029a7619e46f,0x000f29f5c333074b,0x0009e45f3bb1eec5,0x00035aadf8396133}, {0x0000825d3e2a3176,0x00034ef799bdaba5,0x0007f38574f4d09c,0x0009085e808678d4}, {0x0004a57483db0387,0x00054465ae9f6d1c,0x0000000000000001,0x0000000000000000}}, {{0x0002fb74fcaebc4b,0x000ef4b901e46eb5,0x00068ec4a861868e,0x000f51a07bab1199}, {0x000748f19df109f2,0x000d75da4f6a75a0,0x000f255613859652,0x000e60c8067759f7}, {0x000b663826b7b569,0x000f450533f4d440,0x0000000000000001,0x0000000000000000}}}, + {{{0x0008e66704c4911e,0x000b5ad20de07d3b,0x000dab27e9b7aafa,0x00052dbe3fb66eb5}, {0x000cf7ce856345ac,0x000025496cdf84c8,0x00082f095b8e1e29,0x000e8e6db4cd7761}, {0x000b0faa321aaa54,0x00088ae73fef003b,0x0000000000000000,0x0000000000000000}}, {{0x000d643fd4fac454,0x000c4870e138d616,0x00069ca59b85612f,0x000a26a00889a9e6}, {0x000c093e3dad6fb3,0x0006ce66bb43df1b,0x000244dae036271e,0x000c05182a82fcd4}, {0x0002559d8958ca2a,0x000a7526838c8510,0x0000000000000001,0x0000000000000000}}}, + {{{0x00084b954cc2f383,0x000a776cd88b3801,0x00070c99422dcf3b,0x000bd45086f66f43}, {0x0005f7b81c0e8bc4,0x00004cad2493575c,0x00070ce4091825d7,0x0004f1ff4bbf4b9f}, {0x000d28bd9ac2a0a2,0x000c23e5ebf7a3b5,0x0000000000000000,0x0000000000000000}}, {{0x000270e7ebdf9c15,0x000050eb783548eb,0x00081562bc5fd0b1,0x0005f68896b8a59a}, {0x00073bc130375edb,0x0001c5bd938ab1fc,0x000c19c89b28feaa,0x000f8d6e4b1c7f18}, {0x0009f7384e98a494,0x000a3f55131bf640,0x0000000000000000,0x0000000000000000}}}, + {{{0x000a27923d9331dc,0x000f04ffb6351b25,0x0007f29f1e1c9bc2,0x00069b7d91e80528}, {0x000d48a56cb26370,0x000d9a9a20ff75d6,0x0000b393f52dd397,0x0003703dee3c5227}, {0x000f9c1c267288a6,0x00083849651d4713,0x0000000000000001,0x0000000000000000}}, {{0x000d56c853c2dd2e,0x000a93bc1a8d521c,0x0008735173646598,0x000967ee4685de4b}, {0x0004cf35701ef418,0x00080b116b6dbbce,0x0006b03c5acf7cf3,0x000fe839b4243541}, {0x000841fbd8d1a9cf,0x0003a6e1730d1f15,0x0000000000000000,0x0000000000000000}}}, + {{{0x000fdbea99958b96,0x000e01f76bf65ac0,0x0000c6778adf573b,0x000d0f51ffd85a6f}, {0x0004afe98c72d927,0x00087e8ec8173887,0x000d76d032ae57d1,0x0004df95e88800c6}, {0x000ec4042dee55d1,0x0008e8cd5760c30d,0x0000000000000000,0x0000000000000000}}, {{0x000eac1084c10f00,0x0001c37bdb463a14,0x00076281603cbf77,0x00034037d48543b5}, {0x000f3b965ac3efc6,0x0009a7be5b6e5426,0x0003d0f87fba3664,0x0004ddb5ca9f2e20}, {0x000052649abbc317,0x0004b72eb6083655,0x0000000000000000,0x0000000000000000}}}, + {{{0x0001bc4f61438294,0x00002964a43b5d5c,0x0005d7c26171c634,0x000967cc93c0fb82}, {0x0004b96145dca1b7,0x000b5c4ddfd06836,0x0007f0eec5bd3c73,0x00082d7bf8f0d500}, {0x0005b93c7771d6fd,0x000475f222990f21,0x0000000000000001,0x0000000000000000}}, {{0x00026e01b4722367,0x000926340c9a0a1d,0x0007edb2bec04b5b,0x0005d17d0417ca25}, {0x00072b41c7280efd,0x0003c942f670df33,0x0007910fbfcef999,0x000437ef3a577e3d}, {0x0004d4c9039005c5,0x000edddb0ceb3a8c,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=66 [{1,2,3,..,}]*([2^264]*G) */ + {{{0x000a3a0634a3ce07,0x0008f38d14324956,0x0002232fc3562771,0x000d389e5a0479ef}, {0x000b882744b806a6,0x0000bd687af4d435,0x000d3172792b960b,0x0001f792e60e4cec}, {0x0009dcb17063be91,0x000c5902f6ffb4b0,0x0000000000000000,0x0000000000000000}}, {{0x0004fbf6f215ba3d,0x000918e7c66f8fb0,0x00095b38bbb07b90,0x000022d201c5b207}, {0x000344fdf3937c67,0x0005a11142ee01b8,0x000cb5fc7b97506e,0x000c2ae4043311b8}, {0x000e1937f2450b7b,0x00045fa26a70cfe3,0x0000000000000001,0x0000000000000000}}}, + {{{0x000fdc4396bfa539,0x00092b7edbcb88f2,0x00019d35421db912,0x0000a538d5dee79a}, {0x0003b035e9ea2a42,0x00021709fe9cf14f,0x000a5b749703f94e,0x0002495b47e8690c}, {0x0002ef0574deb7af,0x000dd4b09d6324cc,0x0000000000000000,0x0000000000000000}}, {{0x000f7df3b9fe6e0b,0x000f9e25c764e67f,0x000b9153d851593e,0x000822d7ef6d9489}, {0x000c9238e5449117,0x000bd3333b1e34e4,0x0006cfb58cc8198b,0x000b7b487650416c}, {0x000068c07a8085b4,0x000b7b5e20cc8eb3,0x0000000000000000,0x0000000000000000}}}, + {{{0x000af6a4691425a8,0x000eb7a958bb9efe,0x00085a7002151b0a,0x0006a8775208123e}, {0x000055575f5446b3,0x000528fdeea0c896,0x000a43b1b4d824bd,0x00096f550c5c7315}, {0x00050cdbb6610168,0x000b9c3638cd4a93,0x0000000000000001,0x0000000000000000}}, {{0x00039bfed8d0ec30,0x000230c0cb3e1745,0x0004b3cb7f69dddc,0x000a60a97d0d2a3a}, {0x000c6d61d74827e9,0x00048c2e237c10e2,0x000c78dae64ccf95,0x00031e036445ee96}, {0x000bee13f244e4cb,0x00012ec220b81c78,0x0000000000000000,0x0000000000000000}}}, + {{{0x0008b837d4bef687,0x000919e15922b2f9,0x000c8afde9c62c29,0x0009534c95a1a3c5}, {0x0008f604b1043ffe,0x0007a01a13fa2f63,0x000393b04cd8a8d2,0x0006e26fd0c22660}, {0x0000808a072545d9,0x0003871dd10699cf,0x0000000000000000,0x0000000000000000}}, {{0x0007dfe3a4ea56b7,0x000094c38223ef03,0x000e8b66c87d36e2,0x0002766ee28405ae}, {0x000f0a065b535e3e,0x00084a317ded4b87,0x000866d3f53ac0b0,0x0007a0ee55860ca5}, {0x000a7080382b21bd,0x0002f3ff1d58cf1f,0x0000000000000000,0x0000000000000000}}}, + {{{0x000760a0d077a674,0x000256d1eb02b933,0x000a86e63e76464f,0x000e22b473632441}, {0x000db481034e4b15,0x000914ef870834a6,0x0004c2dffa8bbf34,0x000d9d0466943feb}, {0x00031a2dbf893cc1,0x000615fa26911a87,0x0000000000000000,0x0000000000000000}}, {{0x0002f76d13211f86,0x000b53e5f9871239,0x000bd8e3f6da4b87,0x000367de3555d6f5}, {0x00095ebc54c2b323,0x000413dbb2bd8e48,0x000ddef4e974c105,0x000de08da1d15f48}, {0x000eb47f901deb65,0x0004f245e4c32880,0x0000000000000001,0x0000000000000000}}}, + {{{0x0005b2c93a39a68f,0x00012def13c9b690,0x000a5eaf60ed34ff,0x000eed4546115d13}, {0x000acc0704820ad4,0x0000c499c60761b0,0x00013af5dd51e45f,0x0007a978e5524abd}, {0x0009f811db1ec09b,0x0005900dab7d3aa7,0x0000000000000001,0x0000000000000000}}, {{0x0005f21e6490481e,0x0009c0ea3d3b19a7,0x00068df5edf0364a,0x000c93c95e1d6b4a}, {0x0008333e2dcc0b44,0x000a8fc7be078322,0x000350437cb9512e,0x0005c965c20fe9e1}, {0x000d3176a887068c,0x000004e870a54162,0x0000000000000001,0x0000000000000000}}}, + {{{0x00005c3b211947b8,0x00027fda98024305,0x000832a6aa27a459,0x00057be3feea006c}, {0x000ba0bb599ef407,0x0009c187eff74059,0x00032aa4ef6180b7,0x00072b398f55afe2}, {0x000cc5ddc46af4a2,0x000e611d5ea71271,0x0000000000000001,0x0000000000000000}}, {{0x00085981da5484c4,0x0002661fa0fa34a4,0x00025e8fb282c8f6,0x00009d9542344d00}, {0x0005e720cc73e773,0x00081ad0388c6b65,0x00097a5b5f85f411,0x0006173a273c5fde}, {0x0000126c0a036ea1,0x00030085c778a65a,0x0000000000000001,0x0000000000000000}}}, + {{{0x000dd3a67c2984a5,0x00072c7824703af1,0x000bf0c69c5b39c7,0x0000901a46f942bc}, {0x000f89e0174be31d,0x000b6326f7d38bdf,0x000eadb91bfcc1ea,0x0002541868bb8787}, {0x000a48a8ba038566,0x0004606d8787616b,0x0000000000000001,0x0000000000000000}}, {{0x000e290f5d5f8e88,0x000eb41d33d54569,0x000662b634f1ba52,0x000fc5049dfd1d1b}, {0x00059be51c909876,0x000b7406ba93e420,0x000475a49355b9dc,0x00048963ea182651}, {0x000985ceebcf7670,0x00062eaa85c80508,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=67 [{1,2,3,..,}]*([2^268]*G) */ + {{{0x000b5648e1eb2291,0x000ba734c2f6e413,0x0008535398b202b7,0x000a8b23ebd7f177}, {0x0009fd3b0fc5e7ee,0x0003df55ddc7a0f1,0x000261d577641e2a,0x0008f496646ecb9f}, {0x000f2be9c112454e,0x000e02c23da6a9da,0x0000000000000000,0x0000000000000000}}, {{0x0009d35c52fed679,0x0007671efd66d8e5,0x000904f29b91b401,0x000c352b084f27ae}, {0x0000123e88566129,0x000229faaea32636,0x00087f4c04620cfd,0x0005535f695ec91c}, {0x000fc2a2127c5854,0x000fa1c94873fa3c,0x0000000000000001,0x0000000000000000}}}, + {{{0x0004b028b8f44cc5,0x000fa18c73e470d1,0x00046af781c684d5,0x000aadf0eb44feae}, {0x000604610320e3e3,0x0008fffa44fd9c59,0x000908270d9d9d3d,0x00088a6283f3ebbc}, {0x00060d649fcec534,0x000e1d44a603fadd,0x0000000000000000,0x0000000000000000}}, {{0x000740ae33023df3,0x0000135f6a91ebce,0x0003780772b2000b,0x0004747ae7ea71ec}, {0x0007f6f03b13d0e5,0x0006603e33fc299d,0x0000d28b6e9df68d,0x000a2043747f8604}, {0x0006089683aeee37,0x00024e9926fb8de4,0x0000000000000000,0x0000000000000000}}}, + {{{0x0006c9f15018542e,0x00079fb1f6fe5f6f,0x0005889ab7c13ba5,0x000129b7c5358418}, {0x000a9ac0306d8ad0,0x00008b37c84cfb84,0x000acf7da701dcb0,0x00014bb42427b3eb}, {0x0006e318d3d02d27,0x000958db234da419,0x0000000000000001,0x0000000000000000}}, {{0x00031a9f2e6d1d68,0x0000ba7009240e02,0x00063eb8c05422c8,0x00081b3a6ec6a77b}, {0x000bc58689a5aa19,0x0006306dfb930061,0x00007013549ec203,0x0005410416293e63}, {0x0001a4d303fabd7a,0x000ac172f1e81c7f,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b9ad3970b9e18,0x0000bf6af8e430ab,0x000f59d55e652348,0x0002bd614bc56b8b}, {0x000183df0a6ecc07,0x000ee1786d25c98f,0x0003feabcc84059b,0x0007b20a09a6f26f}, {0x000d600ce79a2dfb,0x000f39cf2e6f0393,0x0000000000000001,0x0000000000000000}}, {{0x00072a39f3507cb8,0x0007042b1470af9b,0x0007da313b04804a,0x0000e590e67c9622}, {0x000cabec90cccc29,0x0005e76e6a796f29,0x0001637cadb620bf,0x000d15b03af38ec0}, {0x000dcf7634087520,0x0001906c0ca6b7d8,0x0000000000000000,0x0000000000000000}}}, + {{{0x0007eccab473c8b5,0x000354c80ff211e0,0x00045d2bf3c3aaa3,0x0005bdea352cfb52}, {0x00016f804d4fc256,0x000e5ad706ca34f0,0x0006619c96cff608,0x000490525e689857}, {0x0004163c78de4a6f,0x0007e4b87dad9b7c,0x0000000000000000,0x0000000000000000}}, {{0x00035a48ce31e1e3,0x000c4a6aef85129a,0x0002e4b9afe02466,0x000a8b4fb7e5c1b3}, {0x000e4ea65ec8abb1,0x0009979db25393ba,0x00064b047eeb2086,0x000d6e5e56906ac0}, {0x0008d958b203281d,0x000de4155ed9842b,0x0000000000000001,0x0000000000000000}}}, + {{{0x00019bc174c6d988,0x000cf4d9f702e572,0x000883fd073431fc,0x000f7c52e0996638}, {0x000331a7f7f18050,0x000e9196a2374342,0x00087a7400e71f3a,0x0003791b5d724c12}, {0x000499ca89fe518a,0x000e565820d945a9,0x0000000000000001,0x0000000000000000}}, {{0x00035b7cb9fac00e,0x000a632b3417a9e4,0x000077b9768bc095,0x000afb46dde432fd}, {0x00032553ea0ffde2,0x0000725529dbe056,0x00071cbdddb8ca1c,0x000edb41a198fcb3}, {0x000f5060041dd314,0x0009a1fe368a743c,0x0000000000000000,0x0000000000000000}}}, + {{{0x00059195e6de2d3f,0x00030ea9f4518076,0x000e3f066c30b9f7,0x0003bd981db2f294}, {0x00046e95c601e580,0x000782f5d49b6ae4,0x000f2a24bea15da5,0x000d57d82c482be0}, {0x000747b852b55a3b,0x000400c12ad66ee4,0x0000000000000000,0x0000000000000000}}, {{0x000106fc9ab22be0,0x000f3ff85e4be397,0x00064c04ee049a9a,0x00068c1771b2f4aa}, {0x0009c56a4f6e25c6,0x0006c0243a7cc3be,0x000082558b6f8b1c,0x000d25ba47737910}, {0x0009a82551abc22c,0x00044f195c4a902e,0x0000000000000000,0x0000000000000000}}}, + {{{0x00037e57ac34630d,0x000fc2c030f2d54f,0x000b84aa880649ef,0x000b55a13ad19d77}, {0x0002091bd296d1ca,0x0008f7f0b28c0816,0x000d7580c469726c,0x00021840b8cfb847}, {0x000fd3c1cc59a8b1,0x0008b90e2778fdc8,0x0000000000000000,0x0000000000000000}}, {{0x0006cc1f42f98900,0x00026c3bf2f82644,0x0008d5f8bb74fc86,0x00077c747df08423}, {0x0005d41c77ea13a8,0x000556d8fe471d93,0x0008287e98cde5b5,0x000c068f4d40279a}, {0x000305a88e400538,0x000e77c091d74bdb,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=68 [{1,2,3,..,}]*([2^272]*G) */ + {{{0x000190a6f12cf864,0x0009e0eee766f86e,0x0005b775cd536070,0x00057c69d4566a98}, {0x0005745df1e07e40,0x00047733f9c06722,0x0006e2b1b1c2a5a9,0x000fc80987a24bcd}, {0x000f4061ae7293fb,0x0007d711f7042b89,0x0000000000000000,0x0000000000000000}}, {{0x0003c1b0341e791c,0x000537daedd9c1c5,0x000495a12d7c48bf,0x0002d4a32c8c9765}, {0x0005a662fe9dfe7c,0x0007c6bad9faed52,0x000660c5c4df70a2,0x000bba7fb07624dd}, {0x00091b1d621abac8,0x000771b618ce5d4a,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c915030a7a399,0x00068b673999f751,0x00008c22b2afe146,0x000bf6a5fc300d5b}, {0x000b3e178c0bceca,0x0009d38258020b90,0x000176381c171fe7,0x000e86f32623a2f1}, {0x000f64b9bf3a66cd,0x0002c55668f5ac6f,0x0000000000000001,0x0000000000000000}}, {{0x0001812516e735d9,0x000aea3ea58c5dac,0x000356a5f10de279,0x000295b07e9f7153}, {0x000f586ce9eb4d08,0x000daab1a3f7c783,0x00000a0030b2d7c4,0x000c2198f3166033}, {0x000184aa9d0c0475,0x000b11a6fe88caba,0x0000000000000000,0x0000000000000000}}}, + {{{0x000ba5c8d0193b1f,0x0007644c52e9d2ec,0x000084d971b1a2c0,0x0000ba2904071452}, {0x00016420810c95b0,0x000726c12ee37ace,0x0005cdbfb3b34658,0x0001ddb8f12176e9}, {0x0002665465a782ea,0x0005989e91fb9ee1,0x0000000000000000,0x0000000000000000}}, {{0x0000c16d55245f9b,0x000a5ab01d1b1ade,0x000186cc016cdfa5,0x000f20c1907f643d}, {0x000b819ce2692951,0x0001e463db499758,0x000551bae173a15a,0x000c9960164a1a60}, {0x0005b509da7db4de,0x000254d9cec8875c,0x0000000000000000,0x0000000000000000}}}, + {{{0x000b44e11c323d06,0x000fd5d6d986113f,0x000894e506cf902d,0x00052247fd0b1d00}, {0x000a2782247b185a,0x0003bd1827ee7a96,0x00075cc817a81ab7,0x000d58e21da1b5a6}, {0x0006b1f8c96f3b0a,0x0005eb0b4feab1ba,0x0000000000000000,0x0000000000000000}}, {{0x000e1e70f2721b75,0x000160a2caaa6a94,0x000c595ff3de4a5a,0x000a75c84e2aab67}, {0x000e555f145b7c4c,0x000c6003a47731be,0x0008f07e7fe03b5f,0x0007c95ac06b0bfb}, {0x0000ec8f9062bb21,0x000d58a73aafef97,0x0000000000000000,0x0000000000000000}}}, + {{{0x000123c6709431e0,0x00010ae6e4e2793d,0x00084b28d12b01b7,0x0007bca51962e973}, {0x000f98e6a52dea7f,0x0002d8bb8dd35519,0x00063c8d202bed1f,0x00009ec87640cd7a}, {0x000e8802cf11aa3f,0x0002789d1734b8f7,0x0000000000000000,0x0000000000000000}}, {{0x00015a8a3e8d859e,0x000534ae889a29d5,0x000da86637742cb4,0x0005ee22c15c8252}, {0x000ad4f9397a87f5,0x00093d8a684e2ab5,0x0006af7ba63ff2ec,0x0004f0fe6a52e297}, {0x000effff8321e195,0x000d2957562f8dbd,0x0000000000000001,0x0000000000000000}}}, + {{{0x00023e7069e42e0e,0x000d2239c3ca8089,0x00005746ae0d546f,0x0009bcbb8ee8194a}, {0x0005279f5c84f339,0x000373c06a06d19b,0x000c70d34e701108,0x0008ba438171e418}, {0x0007c7306769b5b5,0x000064bf193f46da,0x0000000000000000,0x0000000000000000}}, {{0x0005544b033a42d2,0x00061e556329a54d,0x0006aab67ee7a477,0x000c61bcc63287d0}, {0x00034397f22f1672,0x00021907d612d307,0x000ccf64c77cc188,0x000f80df2be0b759}, {0x0001db9d853899e4,0x0007cb0c4b9b73aa,0x0000000000000000,0x0000000000000000}}}, + {{{0x000ff9f36c3349d4,0x000c3db537864c16,0x000d990835ee2990,0x0003c5fb1d408a12}, {0x0004b5ff931b6504,0x000452158a1b1e0b,0x000db8a2c29f72a6,0x0009d61be31c5985}, {0x0007a2320489621d,0x000480174d202617,0x0000000000000000,0x0000000000000000}}, {{0x000294c195148880,0x0009ed88751bd34b,0x000f46ac8a70ae99,0x000917e5c0560f97}, {0x0002525c8ba24922,0x00048c3502a88bf3,0x000ee6a458ba7134,0x0003f607a14e42a8}, {0x000dd29054842573,0x0009c1acd66ae0e2,0x0000000000000000,0x0000000000000000}}}, + {{{0x000edee700e2b9a1,0x000ca05fd3e47e10,0x000775354363597f,0x000b8ab9d14d9e5f}, {0x0009809ae6cb63e8,0x0008a4dd84740965,0x000dd249f1a5c96c,0x0004d2f79af0cb6e}, {0x000166e5361d2b7a,0x000692fe3d22a60e,0x0000000000000000,0x0000000000000000}}, {{0x0002fa6f8995a329,0x0006e396d7a363f7,0x000d92f57cc488ad,0x0009d1958510a286}, {0x000c0b888aa8bf0a,0x00042decec317136,0x0008b9bdf9fc71bf,0x000ac0298d41b6cc}, {0x0009ecbb249e5d99,0x0004db314b57f810,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=69 [{1,2,3,..,}]*([2^276]*G) */ + {{{0x000d708c668b7357,0x000d96ce839038b2,0x00020e1ea62ce28e,0x000713c58763eab5}, {0x000e2c4523fd6ed2,0x000eae1cb271027f,0x0005e4f9c4b8cf67,0x0009601ad02024a9}, {0x0007d73ac0716494,0x00064337442ffcdd,0x0000000000000000,0x0000000000000000}}, {{0x0007851b23ec84bf,0x000beedb8574d7b7,0x000286ebfe9b645b,0x000e45ce0c8710d8}, {0x00056a79aecb4766,0x000f379f83c2d312,0x000bbc5340ea164b,0x000df851521a164b}, {0x0009d5d5e1ac3081,0x000081b205779b7e,0x0000000000000001,0x0000000000000000}}}, + {{{0x0003e4b56c489ffe,0x0000450823173431,0x000479ae5276717f,0x000cb05ce3d436d8}, {0x000ddf2257834d02,0x0007cf804300a63f,0x00048d555acde6aa,0x000b3233d0de457f}, {0x000aa55b4de5db66,0x0008a60379d9ac81,0x0000000000000000,0x0000000000000000}}, {{0x000e90717067058f,0x000132c47bead613,0x00090e8a449f2111,0x0001c278b92dfa6c}, {0x000a20e5052e4286,0x000d62ef7fbb21a8,0x0005d03da29cea2d,0x0002b105405706ce}, {0x000dda27b321921a,0x000d13a8070a212b,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d23cf55c45926,0x0005b98778f5f299,0x0001705a5c8c8b02,0x000b88f43919b71b}, {0x0001fcb92372e0d5,0x00043296e160fa37,0x000a4970a89cc719,0x000334a3ae695fe1}, {0x00051e4b95dec2f9,0x0004d6275a594212,0x0000000000000001,0x0000000000000000}}, {{0x00047e08dd859c11,0x000f2faa12f1b2ff,0x000ffb55ea0ad152,0x0005927a2d49016e}, {0x00063e898a743956,0x0007e768ee6cdbde,0x0009ce79201bbe74,0x000b64e8832a0a06}, {0x000cff0774d3af5c,0x00081d58cc25a522,0x0000000000000001,0x0000000000000000}}}, + {{{0x00027f0e1b46fd00,0x0003f1c1b9a1af5b,0x0003c2c754fc491a,0x0007d3165cbaab1f}, {0x000360310665c2a7,0x000a6d64bd760e64,0x0004ce1abfa1968d,0x000d2b0e1701fb5f}, {0x000d3c4d7b466c4e,0x000f912ebf21257e,0x0000000000000000,0x0000000000000000}}, {{0x000f6e44d2ef47a9,0x000cdeddd0d09650,0x000acd3234c8ca37,0x000cba5fb7244def}, {0x000f3acca56c2d39,0x0004d3ff0e42e4fe,0x0003498d03959e10,0x000b101ed923e651}, {0x0000842e240deada,0x00047140cf65c53a,0x0000000000000000,0x0000000000000000}}}, + {{{0x000420de7a3cca92,0x00078c40b5d961dc,0x0000669c0650c56a,0x0006512b20ea2fcd}, {0x000c80cfdc1ced7d,0x0002dc4c42793f28,0x00014af2a2c66b61,0x00038712d0f465ef}, {0x000a3e10f498de28,0x000eefd43378b1ab,0x0000000000000001,0x0000000000000000}}, {{0x000182339af2de22,0x0000f52743e62529,0x0007cb967f975c8a,0x000a495f3b942e5d}, {0x000442c93c4c7a6f,0x0005ea4e81af911e,0x0001c3361393032e,0x00046ad975cf453b}, {0x0008fb85fbd84437,0x000e9f19bef58359,0x0000000000000000,0x0000000000000000}}}, + {{{0x0001c8a03a758373,0x0002ff5d3c5d987b,0x0008d28755f9b053,0x000696c9a6811aec}, {0x000671a05c762526,0x000b50917885c6fe,0x000e435b1cd97329,0x0007d6424b129ce1}, {0x000fb69b5d25c001,0x0005bf8fd3449e7d,0x0000000000000000,0x0000000000000000}}, {{0x0003b3b4745aa0fc,0x000a304f13caa8db,0x000c42581d38c65d,0x000a26e3f1637449}, {0x00075086fb17831b,0x00070bccf51f20a3,0x000ec67794f66bbe,0x00008759114e8fb9}, {0x000e55fefba15c3e,0x0006b06274e840ba,0x0000000000000000,0x0000000000000000}}}, + {{{0x0001392a85eee65b,0x00037ebdba8a73a9,0x00057c70feb31623,0x000f9841fe6064d0}, {0x000abb8ea58571cb,0x000f4d78e10f9265,0x00010194ea34ee68,0x000ee932bccc86c6}, {0x00018b05ba4d88af,0x000c7f1b666c9e95,0x0000000000000001,0x0000000000000000}}, {{0x000bfc49e59e8fd1,0x000ae405208c75f8,0x000d373c5a99df54,0x000a977279933e71}, {0x000da64aa0b3f5b8,0x00036d2c0da96b6a,0x0002eed5227ca046,0x000b6e0ffb64c414}, {0x000baa05111d0f26,0x0006496cce8a3203,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c440343aef952,0x00019eda15115cbf,0x0004c0ca17c7be65,0x000b899e1e404697}, {0x0006bd56cb968d39,0x00077f25afafaffc,0x000e8460c48baa89,0x000cb8ddcae733c5}, {0x0004e7b4d8db3dd6,0x000d4730c42ef0ea,0x0000000000000001,0x0000000000000000}}, {{0x000ded4336e0e996,0x0007761c36485b3b,0x0003a9ef9b460a85,0x000ea119c163b2e2}, {0x000f8699c32d39ca,0x0009afc21d6fbf2a,0x0006105b2f30acbc,0x000d2782e1795cd4}, {0x000bd0296c5de1eb,0x0003f540db331e94,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=70 [{1,2,3,..,}]*([2^280]*G) */ + {{{0x000566c4dd75c1b4,0x000a0570856265b3,0x000cbace31affa63,0x0002b4ed64645336}, {0x0006de49945b2d79,0x000ffedb2ccdc41c,0x0001239fc3fec1e4,0x00056c094341fb38}, {0x00028185bb5868f9,0x000adaf680572da8,0x0000000000000000,0x0000000000000000}}, {{0x000e0585aa2d876a,0x000b95480f8f0fbf,0x0005be334d530bd3,0x0002f278c2d3c86e}, {0x0006b676d6c82d76,0x00039dec8e1488b5,0x0003e4b756194ec5,0x00094e5ad8a2c0fc}, {0x0001d4129e01cce4,0x000c459cb7e94c1e,0x0000000000000000,0x0000000000000000}}}, + {{{0x000e172f71314c57,0x00073776d707122e,0x00007937b43cc86b,0x0005775ac42e1bf4}, {0x00055e0abab130a1,0x000a4930e58a6f41,0x0005a237af5f75c8,0x0004f7ffdb8ee4d2}, {0x0008fe7af47745ba,0x000467da1f09c11c,0x0000000000000000,0x0000000000000000}}, {{0x000ab833ab90a6b8,0x000669bed4019358,0x00007b7fe4d74f45,0x000d834f00a5516c}, {0x000f035a36d318fe,0x000f8a31bb46f3d5,0x000972946e1df3bc,0x0001ef24a7e886c9}, {0x00085864c383ab5e,0x000a5b85a50c0d32,0x0000000000000001,0x0000000000000000}}}, + {{{0x000d65b66d82688c,0x0003872c2925e382,0x00080fde81ac79d0,0x000a2b38a2432027}, {0x0005fa34422b6bc3,0x000948d55e24a659,0x0004efa7ed8f149b,0x00071011867cfac1}, {0x000bd1e94e578bbe,0x000803efa0276501,0x0000000000000000,0x0000000000000000}}, {{0x000f7389f88ed003,0x00099b4f58dfa2c9,0x00027af2024d3bff,0x000a0ca5e0ca7fb1}, {0x00083782dbf7ba09,0x0009fa5d6101098d,0x0005231abdbaa4b1,0x0009c03b0d70ddb8}, {0x000cba60cbb85949,0x0003ba8d596ce326,0x0000000000000001,0x0000000000000000}}}, + {{{0x000ec042211830fd,0x000f78b67d56f78a,0x000d0dd5f0085f0a,0x00077ef6d1879493}, {0x00009d412929cede,0x000f8f47fc6254c3,0x00038136cfbb6a12,0x00090a561c304af6}, {0x00064012770283e0,0x0006e8c77dd363b1,0x0000000000000000,0x0000000000000000}}, {{0x0009730231689c1a,0x00078a2910104065,0x0006adb22ec70366,0x00007eaa5919dc9a}, {0x0000eff75ea6bb96,0x0001b154022f103f,0x0002e9dc4300dcfa,0x00023ebbf1abfdcf}, {0x000c7610a1be47bf,0x000ee8e48e43b23b,0x0000000000000000,0x0000000000000000}}}, + {{{0x000366222079ff20,0x000dd6c255282fb1,0x000e5f65eafcf45b,0x0009b30515c02959}, {0x000efd5074c2f0d5,0x000460bf3169d34d,0x000d88198c4daf0f,0x00042d35aae9cd4d}, {0x0006f8fb3a3e2b92,0x000e5b7ee19179cd,0x0000000000000000,0x0000000000000000}}, {{0x00015c0c8bdd26b4,0x0009fc2499885330,0x000c27ee4ed9b18e,0x000e30d901ee8c44}, {0x000a438c4d057961,0x0007a847d74e4722,0x0005e9ebd34c3ce2,0x00049ec1f371fc17}, {0x00062cfa628ad626,0x0000258cb8ba2199,0x0000000000000001,0x0000000000000000}}}, + {{{0x000414eef61746f8,0x0002f44b16d63548,0x00091c53690466a2,0x000700b4b9b4826e}, {0x00069ba41d6fddfa,0x000f48b184a9d1c2,0x000af1a9a1ae5623,0x000688445e2f1d66}, {0x00028a0ef16e7621,0x000ea0a509e87455,0x0000000000000000,0x0000000000000000}}, {{0x00016c4ebf9d1f30,0x000002405c88d64d,0x00021995ea2cb159,0x000476f59206340d}, {0x0007fbdb47138c1b,0x0001fc51a673c4a8,0x000c132d81d7d81f,0x00033035e2c568d2}, {0x0001981e0c2e86c3,0x000c9f25fcaa15dd,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b2a49ca5dfb81,0x0008d370aadb7c4a,0x0009f4ebf1398343,0x0004d610d25c9ac8}, {0x000b49c7f0f75d8d,0x000cfed5c3ca14e0,0x000cd7f4d6f7590d,0x0007d93cbfaa36f5}, {0x000dce79a65cb3d1,0x0004b4bd97f1011d,0x0000000000000000,0x0000000000000000}}, {{0x00001207087dd6cd,0x00049477a85a518b,0x0007671964d279a4,0x0001d11aff88af2c}, {0x000a1dbb6c2c5f27,0x0005ba326b82395c,0x0003c2898f431018,0x000463dc513c0cb7}, {0x00069f2786b20305,0x00061fc5c18db944,0x0000000000000001,0x0000000000000000}}}, + {{{0x00080adc636061c1,0x0006244e403a263d,0x000ad04e1deab320,0x00059720dcbb6130}, {0x000d66e8505322a2,0x00060fb3f5231b1a,0x00049e1cd79b6e2a,0x0006179c366e663d}, {0x000c6ea0d01277eb,0x000473883c4ffde6,0x0000000000000001,0x0000000000000000}}, {{0x00040167fca5210d,0x000d95aec71f687e,0x0001c63bde59a231,0x00074dfa4af79a44}, {0x00060fd79e68ddec,0x000b613ae2a19527,0x000036b08e61ca70,0x0003e30d2b549e73}, {0x0008fb383d922e0f,0x000aa028c146216d,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=71 [{1,2,3,..,}]*([2^284]*G) */ + {{{0x000090e2290855e3,0x000c63856faf70c3,0x000a537fff7cf9c3,0x00002007d0317a7a}, {0x00010dc853b3261c,0x0006ccdf2c616875,0x000385cd6f188d53,0x000a2955fa1d26fd}, {0x00087bdaef2d7d6e,0x000b0f3173148e30,0x0000000000000000,0x0000000000000000}}, {{0x00084c9a254216ed,0x00047cc0770de6d7,0x00035e4c8fa4bf8c,0x0000f637aece660d}, {0x000adedb7b99e891,0x00082ce72b5ced4f,0x0001d4d0fa07446a,0x000194600c851570}, {0x0004ffcea4152f30,0x000c96f31c15edf3,0x0000000000000001,0x0000000000000000}}}, + {{{0x0005b1c7831bdcc8,0x000d027bafc7d23f,0x00050c19efdbe5d1,0x0007a7533cdce225}, {0x000573af279d2535,0x0006015a5e144120,0x000803b571412097,0x000eb203384ac91c}, {0x000edd68012ba72d,0x00091d825c3d8d2b,0x0000000000000000,0x0000000000000000}}, {{0x00023553a39f8385,0x0005b8eaf27fe164,0x000c4539fb7ef933,0x000adc9ffa5830e5}, {0x000950a5e503466b,0x0003a2a96a1dcbb8,0x0001a89a62dca0dd,0x0001d5f98db48a88}, {0x000554b9506e0a31,0x000dcd69efeec8e2,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b8cbe4b97fc6c,0x000e5a71915ba245,0x00044263935f5290,0x000ce189775a8de5}, {0x000cd0549c8def51,0x00027c89477b9645,0x00047cfbc54728e8,0x0000c7cb6412bf4c}, {0x00016bf97806ea22,0x000feea61b447d92,0x0000000000000000,0x0000000000000000}}, {{0x000a3fafee06d0a1,0x00032a2cf4c624fa,0x000baceff4ee9dd4,0x0007411d429aa2e7}, {0x000d154759f7bffe,0x00091cde409d4d9c,0x000a9a31ac9508e0,0x000deb0f736891db}, {0x000c17f78c058b52,0x000b43e218c9736e,0x0000000000000001,0x0000000000000000}}}, + {{{0x0000f4f238352945,0x000f07d445a023b0,0x000551441addd54c,0x00008f85e62fb5bf}, {0x000ffd275f3aa334,0x00001eb4ed1f4e87,0x0000af1b08c2f8d7,0x00093b5987facbec}, {0x000542336f0bc311,0x000587219a8c1237,0x0000000000000001,0x0000000000000000}}, {{0x00052f04dae724bc,0x000e221a68b0a55d,0x0005bdff6f9d5fa5,0x0007e3f2da24e831}, {0x000628b43c649948,0x000a393f548cc549,0x000acee934d621cc,0x000e0ce12d1b52e8}, {0x0000b93c4f0e6025,0x00077116663ffdcb,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b53f717a44956,0x0002c65fee76e1bb,0x000ae70d2da363ec,0x000cd6887a61f9c3}, {0x0002283a4e23325b,0x000ae6d909c60aab,0x000702a1dcca3dd1,0x000938d6a31c5517}, {0x0003e01f167bc2d3,0x00067596be65d549,0x0000000000000000,0x0000000000000000}}, {{0x0007978a4e132cb0,0x0005df10adf1702e,0x000364f7e05324c1,0x000693e3866569f8}, {0x00082e303b6c9df3,0x000298933d134854,0x0003d7696e64c215,0x0000f465322046cb}, {0x000e839e686c1c09,0x000b7f8ac2e6f682,0x0000000000000001,0x0000000000000000}}}, + {{{0x000e71627b708629,0x00012bf54385d85d,0x0007bb4ab478b057,0x000a3b2ccc6b5489}, {0x000f67fa6f6a05a9,0x00073cd5233bfef8,0x000c4c4f505a80c7,0x0006a3fe8c185c9c}, {0x00059e110f37ae33,0x00054cb440dc7a62,0x0000000000000000,0x0000000000000000}}, {{0x00050c98bc7b0e1e,0x0005a04b3793331e,0x0003b8d31034a8aa,0x000c95dac900df01}, {0x000fd52027ee1c99,0x000b5be3aeb891a1,0x00042f3b68574224,0x000ae00bb37d5ba8}, {0x000a7f31f3f36375,0x0008c304743a2813,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c7e03b7aa645a,0x00091f7e39a28977,0x000995d7d443d68e,0x0009f39216c442e1}, {0x000b376cc21c4025,0x000705644f75967c,0x000592e3b90bca20,0x00056be0ef613507}, {0x00097f2f7456e836,0x0004b46438409baf,0x0000000000000001,0x0000000000000000}}, {{0x0005a21df85e1c6c,0x000b0ab454500393,0x0004c5912b476c6d,0x00065a0d1aa49d94}, {0x00059cdd7e7069d5,0x00090a402b7eeef8,0x000283db40b228bb,0x0005dcdf2c268733}, {0x0005a42d37d44810,0x000b80e5d1af0360,0x0000000000000000,0x0000000000000000}}}, + {{{0x000c14cb537e819c,0x0005f2b3bf6b7569,0x000cc275187a0bac,0x000925ef6c2d559b}, {0x00025a54533380eb,0x00019f0d6cd0382a,0x0001433bbf74a021,0x000bbd994d4caf68}, {0x00027772c53999a9,0x0003af249226a38d,0x0000000000000001,0x0000000000000000}}, {{0x0006a46b5b493127,0x000bee887c8f7779,0x00045dd063eabbf7,0x00029199a0a8e117}, {0x000918d22e28aea4,0x0000ecebe77e69a9,0x0007cecb0ed2deab,0x000637aa98d20edf}, {0x000a194790e52888,0x00060c73078fbd0c,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=72 [{1,2,3,..,}]*([2^288]*G) */ + {{{0x0009af391a8697ed,0x0007fe5bb7320546,0x00080d05ad03f5f9,0x000f9b7973b1a3ca}, {0x00033b52add9800a,0x0003cc487ccc82c5,0x000f71a0da2ae069,0x000c060c7047e46c}, {0x000a21503aeb64ab,0x0004ed0075b1d33a,0x0000000000000001,0x0000000000000000}}, {{0x000a2bed91298551,0x0000c49a79f6b129,0x00022374a19da463,0x0001305962f001a1}, {0x000a3a3cc4dcc90a,0x000188b4cc026cef,0x0002ff30fbb1d3fb,0x000c36e3761cad09}, {0x000dbdbdc6354b93,0x000b2ab73317cff3,0x0000000000000000,0x0000000000000000}}}, + {{{0x00003f3665635e4e,0x0004e17fd85da26c,0x00052fd006ed5f69,0x00032d252f043a61}, {0x0005c8bc9cc74510,0x000f5370ca9348d5,0x000540b56333c4c6,0x000bc9a5ca533610}, {0x000d8071b716d25c,0x0002367337f70a39,0x0000000000000001,0x0000000000000000}}, {{0x000db6fc5387c11b,0x000cfd3251b14397,0x000d84aa2bfdb755,0x000e38100cc3e62a}, {0x000046071f1f89e9,0x000e7012d9e47fb1,0x000e6ad97ec5c7c3,0x000698bc4de4f6c7}, {0x0000c6a07a4e7cef,0x000198a03a3a1224,0x0000000000000000,0x0000000000000000}}}, + {{{0x0003b8713123ec24,0x0000419a51d2d151,0x000c8ee7e2c9c855,0x00039f110d3d8f17}, {0x0008ce870e320ac3,0x0000d4e71030e599,0x000ace1a5cbbd4da,0x0007c15bfcfe41d9}, {0x000408b36096093d,0x0006b26a0c56d39a,0x0000000000000001,0x0000000000000000}}, {{0x00035ac9181317f5,0x0008d46d26280ae2,0x0003c29370c4bff6,0x000addee48f7eaa2}, {0x00030f605627b203,0x0003e03ba6674883,0x000cdd81ae186660,0x000d8bc04a9667b4}, {0x000cbc45ee0c1b5b,0x0003d34e81dc5ff7,0x0000000000000000,0x0000000000000000}}}, + {{{0x00063b161dddd94b,0x0003d7c93f0cc576,0x00022d6ac11071af,0x00012d84b9149bdc}, {0x00088e44e4632e63,0x000448cc8ec50d4c,0x000f89a6c85277ac,0x000e128700eabfe4}, {0x00042928ea38e5f2,0x000e293e261880b7,0x0000000000000000,0x0000000000000000}}, {{0x00051028cc113b68,0x0001919b6a14e2fa,0x00082dfd5da09549,0x000ca662e13022a3}, {0x0002996fafc24233,0x00018dea4f505fe4,0x0005a2d96182166a,0x0000199ba55815ea}, {0x00077232622a4ac8,0x000311b13c3b8133,0x0000000000000001,0x0000000000000000}}}, + {{{0x0008673adf41d204,0x000e2a0a390575d9,0x000dcb844f35fe0b,0x000b6f4f917550d3}, {0x000fc6ecf7285d09,0x000925f580342d9d,0x0008746022d9fb4c,0x0001b6b73d4bc619}, {0x0004aa1c30d7a3bb,0x00003bf32bccb3c5,0x0000000000000000,0x0000000000000000}}, {{0x000783e73d6e9052,0x00006db6e57f7b8e,0x0005d1afd274f869,0x000318770758e016}, {0x000290175b314342,0x000e6d25df464413,0x000da9ee1f0d5093,0x000297855e553138}, {0x0007c7b64937a717,0x00057ccbf262664b,0x0000000000000001,0x0000000000000000}}}, + {{{0x000d9056eeba30b5,0x000355a8c404c7b9,0x00039ffec5e11fca,0x000c3ac7b4f4c637}, {0x0001ace1869851e4,0x000e1816afc7de47,0x000d517fb7e6978f,0x00040e256f62bd56}, {0x0004bb4c53034a6f,0x000aa98eb4b9ad2d,0x0000000000000000,0x0000000000000000}}, {{0x000fd2909662ab1d,0x000c55f8463596cd,0x000074f62f74b11c,0x0004dfc43f12c5ee}, {0x000ad69405653ae5,0x00089f1d6e4668a4,0x000187ca431697ab,0x000831cc4eed080d}, {0x0003a69f7d775edf,0x0004e434435eef6f,0x0000000000000000,0x0000000000000000}}}, + {{{0x000a1403851eb93a,0x00085c9ae49b53f2,0x000926a512455b57,0x000ca729dbef4a6b}, {0x000b8440248e399a,0x000191c77a9a0e66,0x000873c8f27a8296,0x0005408861a08ed7}, {0x0002bbe9c5e3865f,0x0003549235be42ba,0x0000000000000001,0x0000000000000000}}, {{0x000a46fbe3334020,0x000abf1524bdbc9e,0x000b376fa382ce55,0x00075b35cc1fb214}, {0x0000910d13b19369,0x000cf5d3212ad3b7,0x000c7e93b691b3bd,0x0001ba22d0312eb4}, {0x000285b3975e5da3,0x000c999e2ec96eee,0x0000000000000001,0x0000000000000000}}}, + {{{0x0005bf15f46be575,0x000a1061cb09c821,0x000ae2de789e5912,0x000cecccd84851c6}, {0x00001b95ccd21d74,0x00032dddf26a2851,0x00049210122d3f6d,0x000f02c5d952db55}, {0x0004be99796a4aa1,0x0004078cde88aab2,0x0000000000000001,0x0000000000000000}}, {{0x000d753b80855f9f,0x00078288aff9b92c,0x000f7cdce61cc49d,0x00048cc3dac4e445}, {0x000cb0ac2a937fad,0x0008c5bdda956fdf,0x000bb3e81841ce29,0x0005170e6c819f12}, {0x000efd73ecab58ad,0x0000f476a3a48130,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=73 [{1,2,3,..,}]*([2^292]*G) */ + {{{0x0004f332dc67a3c5,0x0003213d82a4383a,0x0000fc0621356af6,0x0004ca32ca05acfd}, {0x000c1e80103ea886,0x0009dd0ff4c2e9c7,0x0009d764c64b7589,0x000680c128a8c988}, {0x000fdb7cd881a125,0x000ec5f09f58ad77,0x0000000000000000,0x0000000000000000}}, {{0x000ecd4f9cda86d6,0x000c4c41a633a8c6,0x000847c2f58ecb1f,0x0007b29592dae51a}, {0x0006e268f50e3a7d,0x000a27de2a3b5eef,0x0006d540d7a599e3,0x0000d01f9b571491}, {0x0005e52e5204fbca,0x0003d9eb48615c67,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d9eaa867420c4,0x0002f7f5caa2ddb0,0x000c31038c0ddc32,0x0001596a08fb4b57}, {0x000a6d9ca6980a65,0x00095c78a8ab32e2,0x000ba78e5808eeac,0x0004f5f9923d5a32}, {0x000ad1c8d3bbece3,0x0007098f8b845926,0x0000000000000001,0x0000000000000000}}, {{0x000843645beef787,0x000fa28875d75316,0x000e13608c5a90e9,0x0005556eaf90c364}, {0x000dacc40e05857e,0x000c5012b59e332d,0x000230b2b76768f9,0x00032932d53c8a76}, {0x000999fbd573bdff,0x000840eee9300114,0x0000000000000001,0x0000000000000000}}}, + {{{0x0002362363f6901a,0x000be66748446167,0x0005f5c47c0db36f,0x0002fbb39fb54024}, {0x000e344525a7871f,0x00071375bdbedf63,0x0000bd89f085fb8e,0x00001c59b6e647d5}, {0x0008031fa2f2ea43,0x000d9b58012b6657,0x0000000000000000,0x0000000000000000}}, {{0x0003015fd48eca8d,0x00082cfde2151d47,0x000e4d908c99616e,0x00004977d4ec3b2f}, {0x000f513df9ad204a,0x000b66641e3ee923,0x0009df2175bb5d92,0x0004cdb5c3c90fcd}, {0x000de8809fac5725,0x000985c6981a627f,0x0000000000000001,0x0000000000000000}}}, + {{{0x0001a0d912586411,0x0001b0d49fbe5702,0x000bb57b277de5b2,0x000a4b2d7291e7e7}, {0x0008c16da29ce1e6,0x0008f8b71f4426f8,0x000fbf76a6ebaff6,0x000cab510adb9995}, {0x0004b996a6ec18d2,0x0003bc3ce11f1f8d,0x0000000000000001,0x0000000000000000}}, {{0x00004c4051321f3c,0x0003af34703d798e,0x000dd55e68b6b0a3,0x0007f09cb14161e8}, {0x0005357558d9c473,0x000d9a485a90c00b,0x000dac2508e73fc9,0x000ed252e5f5bb09}, {0x000b1efcb4ba2132,0x000593c58bf23933,0x0000000000000001,0x0000000000000000}}}, + {{{0x000d1217cba9c6d8,0x00087cb7a562f7c7,0x00004fa0d0ea0e23,0x000c05e3acd8379e}, {0x0006d159e80cdee0,0x000029ffd63834d6,0x000500f3ee777b61,0x0007b75be130d659}, {0x0003756de768a261,0x000a541b809584c6,0x0000000000000001,0x0000000000000000}}, {{0x000af54a67204972,0x00085857b547d484,0x000954a25746036a,0x000bc1881e0295bb}, {0x000d3d231831b4de,0x00002708fd517190,0x000b1e9571812770,0x000c3e25dfa6c88c}, {0x000fb3c57a9f5467,0x0001d404949d8103,0x0000000000000001,0x0000000000000000}}}, + {{{0x000a34fad1eabbbf,0x000de71fa7ad8210,0x000d67acfecf1047,0x0005ec7ec527279f}, {0x0007906ba2abc3a7,0x00066a25ce54f7d9,0x0001e7d558ddfb44,0x000ba77bccd91efa}, {0x0005405027796dc3,0x00072f9ae8e511f6,0x0000000000000000,0x0000000000000000}}, {{0x000feaa4cfcc2a79,0x000d60cf749b854d,0x000ac1632a7218b2,0x000a7ce21f4e0ce3}, {0x000550325628caf5,0x0006b7d84c8e6b8a,0x00081d8e5bd9a461,0x000bb7affd9d5730}, {0x000dca775afff520,0x0006a52e629d73fb,0x0000000000000000,0x0000000000000000}}}, + {{{0x0008f6b8297fa26e,0x000d76f386ac99ac,0x000ce235f3a2acb0,0x000620801a076df3}, {0x000c62031f455ef8,0x000d6e488b1627c3,0x0008ebc5bf3c9afb,0x000c4512c254ea59}, {0x000abb87bcef237d,0x000cb4dcacb5a5ed,0x0000000000000001,0x0000000000000000}}, {{0x0008e7718ef189e7,0x0004207cf70cc015,0x000e9a66810fd747,0x000d69bb7c99d1b5}, {0x000454409a3962cd,0x00014e1d3b541825,0x0008bec33aca5afe,0x000a9c5e4a92268e}, {0x000bf2e95060a307,0x00093a289c336375,0x0000000000000001,0x0000000000000000}}}, + {{{0x000781a84aba36d2,0x000ad7584291d55b,0x000992c0a266ea73,0x000e02af20e9954c}, {0x000ca4d73d175169,0x0001612ee12718e0,0x000cf1ded50926de,0x0008c1060d91c638}, {0x0007dc332b5998df,0x0002624eb7dc884b,0x0000000000000000,0x0000000000000000}}, {{0x0008eae21aadf4bc,0x0005c2a9f4bf2cd7,0x00086c74c6b37272,0x000a4de4b5b5158f}, {0x00093ba4800d6736,0x000138590e451f46,0x000263ed2239cb95,0x000545bdc4c56f5d}, {0x0006676d4c0f8acf,0x0004038bbd0743d0,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=74 [{1,2,3,..,}]*([2^296]*G) */ + {{{0x0004fe80d04c169d,0x0001452da244cb71,0x000cda8b72206f04,0x0005887084ee9fa6}, {0x000920e111da7c3d,0x0003bb35ef1c2673,0x0008e61906e57c45,0x00073eebaa206a85}, {0x000dd3b5bf458238,0x00015dcf71c4a720,0x0000000000000001,0x0000000000000000}}, {{0x000605cdd81e2955,0x000ac0e98756fb91,0x000f0286c4ccda7a,0x00017819b4372718}, {0x00031dae0a5d8a40,0x000720f8cb219351,0x0003217261dafa40,0x00006fb18c8c40e0}, {0x0003c7d483485194,0x000d46e02770cacd,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b16ebe5676f0c,0x0006e7f47e5b4652,0x00071c81701e6ae3,0x0007f8d6e3a4cc33}, {0x00067b29431c31ae,0x000869125faeb29c,0x000fd62e16be2afb,0x0004934ce6d0052c}, {0x000cd74901063882,0x0002e62b021800e1,0x0000000000000001,0x0000000000000000}}, {{0x000a39e3523a9de5,0x00076121f34b3253,0x000d9c36b4aaa3b5,0x000b23992e0442ed}, {0x00023d2725144419,0x000ced8d6e4d3747,0x00011186d58a7080,0x00050cd63ed11b1f}, {0x000c6faa6d7a4d0b,0x0002559c897561f4,0x0000000000000001,0x0000000000000000}}}, + {{{0x000657d806465325,0x00095fd29c362706,0x000e70b9e028365a,0x000fa40d084d6b18}, {0x0009d80bbbcfec68,0x000a06728aae56df,0x000361eb62533738,0x000dc77e1c92bd4a}, {0x0004fbeefd3c11b1,0x000cfba52dffafa9,0x0000000000000000,0x0000000000000000}}, {{0x0009acb1f00a47f4,0x000024e668d3f2f1,0x000095d2d5d23cbf,0x00076b697f105b84}, {0x000da5b550d7489f,0x000c9d3a15e73345,0x00093e0d2b26a8fb,0x0001e4494adfd6a2}, {0x0008e6417e038745,0x000fb0b051833181,0x0000000000000001,0x0000000000000000}}}, + {{{0x00013685fee3f167,0x00059193a123f397,0x000f28bae616ef5a,0x000d2c6567b33050}, {0x0004dae5b6596da7,0x000481adbd59fc9e,0x0002c1618dde4a40,0x0001f2a194682fe9}, {0x000c05bc15adc043,0x00013edb30fcceea,0x0000000000000001,0x0000000000000000}}, {{0x00047d7a2900c6ea,0x000e954cf35b8dee,0x00074908606c425e,0x00094a88cdac359e}, {0x000fbef2c858622c,0x000d0f745810d20b,0x00041d71ca77f658,0x000782f59f1e4267}, {0x0004af36c640314c,0x000e35ec0709c829,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c5c739f094c18,0x000b75fcd89a8d55,0x00071a4d047c0dcd,0x000ef9a3d498ccfb}, {0x0008f669e7edb8ba,0x0008d6b13c20d8ae,0x000a564d16a48c79,0x000250d241703f4c}, {0x000680ef9509f9dc,0x000970b2c17a388c,0x0000000000000000,0x0000000000000000}}, {{0x000778808942861e,0x0009abe8d8d33c23,0x0009bd2fb189dd26,0x000112581e8769b1}, {0x0009d657bacd662b,0x00084fbcaa8521c1,0x00091c546adc05d5,0x000c68fad3f4e938}, {0x00057bce78617aeb,0x000701e39a422608,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d79f624f8eb1a,0x0008e9c5d103b9cf,0x00071b65a5829554,0x00034991b8462f3d}, {0x000b51453c2b2409,0x000cf62fb341c087,0x0006cdb37eea50d1,0x00027ada571b02ab}, {0x0009e677d1b49b69,0x000004b52dc4d6da,0x0000000000000001,0x0000000000000000}}, {{0x00042f171b839d5a,0x000765b6cceda32e,0x00009960889dbcc6,0x00040e8e2e336d9f}, {0x0001f34cefd7de54,0x0008bc6b5ab410a8,0x0000cbb91782d60d,0x000a95e22f3046d8}, {0x000775cf7c99291e,0x000282ca473be09e,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b6a03ba3e1ec6,0x0006b7fefa851ec2,0x0003a92f6680a7bf,0x00075deabf4905c0}, {0x00096ff483a6ae8a,0x000c163b2b784673,0x000be098999b6fbd,0x0009c4a535388968}, {0x000b8e919419a12c,0x000b0bd87c889640,0x0000000000000000,0x0000000000000000}}, {{0x000f69e3681b9e47,0x0008fc5971b3c868,0x000bd601db453ea0,0x000aee960ff01a96}, {0x000a0347158b6a72,0x0008994151ef1dc3,0x0007de4b70d9ea4c,0x0005a29068423993}, {0x000e4276c73a1788,0x0008a834f8bee71a,0x0000000000000001,0x0000000000000000}}}, + {{{0x00059366b70a0b8b,0x000d60abadb72457,0x00087caf5b975b84,0x000cc3f1983f793d}, {0x0002e94d8de541ce,0x00090b687cd8885c,0x0006b6cd952f2acf,0x000016cc05b6d504}, {0x000f97cdb266e5bb,0x0007c95959c78452,0x0000000000000000,0x0000000000000000}}, {{0x0001e70e0efd1f63,0x000528c8989bfc93,0x000c244410e36663,0x000645d5f14f2667}, {0x0008cf8059cc8bca,0x000d2134f71c94e0,0x000ba7b1d1a84e91,0x000e21c35b983489}, {0x00003bfc5dddbab1,0x00099886156a8fa3,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=75 [{1,2,3,..,}]*([2^300]*G) */ + {{{0x00083fd796b1a72b,0x000c549640b910ec,0x0001f3c076c5166d,0x000b7e4e553ff932}, {0x000e97e2f7e676d2,0x000a2cdad5097c11,0x0004bc0c5cedff5a,0x000757d09eef397f}, {0x0007434958d95f66,0x0004eee32eada9e7,0x0000000000000001,0x0000000000000000}}, {{0x00041f85b3f06836,0x0003aba0e3496ce6,0x0007931548ea6b77,0x000149273aecbf5b}, {0x0001858c42c5d2ef,0x000ddc70d91528b8,0x0008d341c157c1ef,0x000a690e10df0a32}, {0x00064f5d30760fd8,0x0000d14ec3b44ea3,0x0000000000000001,0x0000000000000000}}}, + {{{0x00087f011572bb6c,0x0000d126f1e48db1,0x0000d05c4b7b6201,0x000a2ba1e0fbe5e8}, {0x0000224d802f13f0,0x000e0b8698189e96,0x0001b4bc0b8af43d,0x000c706b742b2859}, {0x000d9196336bf7ae,0x00007fec0d2fd363,0x0000000000000001,0x0000000000000000}}, {{0x0000b0b6ce7179b1,0x0007cde982e3e244,0x000d814a6fad99d6,0x0007d3497edfbb4d}, {0x0007a46c6afc84ac,0x0004cd907a63bbd7,0x000909a18bcc3d68,0x000d756b5193f098}, {0x000f37ab5d6e0581,0x000ffb06adf4d102,0x0000000000000001,0x0000000000000000}}}, + {{{0x0000415e4f4b1ce2,0x0002f16bd8544ce9,0x0003e7600d6c664d,0x000d9aa401912f05}, {0x000dd3e0c268e1e8,0x000f14013443dcda,0x000f59cbf936e212,0x0004b8aaec39252c}, {0x000d652c200b8cef,0x0000aac64c11e47e,0x0000000000000001,0x0000000000000000}}, {{0x000c08e2e7a2dfe0,0x00048459176893a7,0x000c7c5a8f0afe8a,0x000e999b6f043d0d}, {0x00000b3c3cfd6e33,0x000b4fbc5e2a496c,0x000f5eee2690de5c,0x00073d2db4511dfa}, {0x000fecdd2743e927,0x0004f4ade6a1948b,0x0000000000000001,0x0000000000000000}}}, + {{{0x000039609cadf211,0x000339488fdf258d,0x0000e39945a1e037,0x0009a8444d16fa60}, {0x000c454408c23f53,0x000ef729fbf7f8cc,0x0001fe369b8abc6b,0x00053dbd87a47666}, {0x0000f6266a890341,0x000de3c7b4bcd279,0x0000000000000000,0x0000000000000000}}, {{0x00006e205ec9aa47,0x00063cce1af477a4,0x000f7fcf64572e77,0x000c931743d00999}, {0x0003902adccdc1f7,0x00041be26f8b87a1,0x00006779ffcb96f5,0x0001c0636264bb59}, {0x000484331c0db1d6,0x00076e5585c8ae19,0x0000000000000001,0x0000000000000000}}}, + {{{0x0005899b377cc15c,0x000f315fff92e2e8,0x000a8cf599eb4a44,0x0002e4cfca1a3e87}, {0x0003f205f34e217a,0x0009eb1362e4f28f,0x00061c84aa7c205e,0x0006fa76515b6ace}, {0x00083aca37e55058,0x00030435e870f8ec,0x0000000000000000,0x0000000000000000}}, {{0x00002203ed113de7,0x000655318327d42e,0x0005d903904004d9,0x00094adb7cd1d2f1}, {0x000ebca1242c89d2,0x000af2a1423b5bb4,0x00066f393818e824,0x00079a30444115e3}, {0x000183b2cd6a53de,0x00047bfd324d8249,0x0000000000000001,0x0000000000000000}}}, + {{{0x0001ee196db2cf29,0x000718a1d049034d,0x000acaf386ae53d6,0x00006d746605e4e6}, {0x000c4458e136fcc3,0x000b7a1ac677dc40,0x0006ee4bb331955a,0x0007b95c386ed47f}, {0x0006640fb4384103,0x00081000a37bb752,0x0000000000000001,0x0000000000000000}}, {{0x0008cc55b8f1bcdb,0x00025171d84cd78b,0x0001b8eb12ee7e03,0x000bd8c564c45f59}, {0x00076c2283df12d7,0x0008a36461b03bfc,0x0000c8cafd6fc812,0x000ae72b6275a058}, {0x000438282511c376,0x0008780ca3213fce,0x0000000000000000,0x0000000000000000}}}, + {{{0x0002c0f6f353cadb,0x00056c17179cb81e,0x000af140db008101,0x0001bb5e63cd1fa8}, {0x000b0729533d8217,0x0006676827544c77,0x000fe1b0143af56f,0x00045a759df2599e}, {0x000962b593accffd,0x0007a3c61321e555,0x0000000000000001,0x0000000000000000}}, {{0x000e58a6aa5358dc,0x000739bb4d42d566,0x000e6ef3760e0cfd,0x0004fb370620ef46}, {0x000c2253e0f9e680,0x000b5c8c64f4e9cd,0x00008daec6f6658c,0x000504153037a80d}, {0x000215b47cc959be,0x000e4e7aaaa8653d,0x0000000000000000,0x0000000000000000}}}, + {{{0x0002bc233166381e,0x0000199700029af5,0x0003201dddf2a837,0x00085cdad02e2c74}, {0x0006daea6cd36d88,0x000c3f784e7e3512,0x000e882fa40cea6b,0x0001eec16a3a5130}, {0x000c43706ace2f12,0x0009ccf3c3a16a13,0x0000000000000000,0x0000000000000000}}, {{0x00064823f9e7a6c2,0x0007bae8e4729ac9,0x0003fe54edfeed8d,0x000a4e7bcfba42bd}, {0x00039e917bf88ec6,0x00004163c606be20,0x00009b017d5a63a6,0x000c276869bd6bdf}, {0x00094144ff021410,0x000a13f038cdd94b,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=76 [{1,2,3,..,}]*([2^304]*G) */ + {{{0x0002f8bd1741a941,0x000d4f3177637189,0x00060a2c8b16c0b1,0x00012ea49a688a19}, {0x0004556eef3eca0d,0x0000c833814746c3,0x0008d8b842db71e9,0x000b1bf4ae9e3fdb}, {0x0000ee706bc576b9,0x00054e85a8de649e,0x0000000000000000,0x0000000000000000}}, {{0x00032799b9f19edf,0x000975259805923c,0x00076f95241c4760,0x000d3b18d6c8d637}, {0x000fa677dee0ddd3,0x0002ad9334e5bcea,0x000ceb27ca464781,0x000594d56dac3990}, {0x000338deb39d6e55,0x00096c7a9d83d78e,0x0000000000000001,0x0000000000000000}}}, + {{{0x000d7b49014b2147,0x000f134215e0ae52,0x00084df8cb33a781,0x000b10bfaf858985}, {0x00065449e13a9c3a,0x000859368bea1a4d,0x000d78c2c8134dfb,0x000ba7d94439b51f}, {0x0003f1c754b1e6f5,0x000d464cf5d47b00,0x0000000000000000,0x0000000000000000}}, {{0x000c286403cdb3ba,0x000c15d6597a8e6d,0x000e5e853fbe3d2a,0x0004c9898bd192da}, {0x0003c76032d13f80,0x00038d5e8bdb66ee,0x0007ea1349bcb167,0x0001e4001679e16c}, {0x000f1f584d22ec20,0x0007b7d9f317a982,0x0000000000000001,0x0000000000000000}}}, + {{{0x000196d862941c7f,0x00015fd4c9380d4d,0x0000a23a9cf4968a,0x00039223ad85ace8}, {0x000b20fbe77f6374,0x0003ae3be886de7e,0x000a36dd5eea175a,0x00061d0a78e19da4}, {0x000a64f9638a5733,0x00007bdb82dd1f3a,0x0000000000000000,0x0000000000000000}}, {{0x0007fe74ebe49201,0x000a7702be0b4d0b,0x0000cad19bf2f6ec,0x000d6c0b5b514f13}, {0x000c08518708375c,0x00035c8f337b8d0b,0x00055fc06adb3974,0x00083bfc4b137fa3}, {0x00045b3d91207a07,0x0009d53cd23a6fdf,0x0000000000000000,0x0000000000000000}}}, + {{{0x0000a67f092fe026,0x000e142efbca10ec,0x000cc433a343767e,0x000f6b9e68131944}, {0x000e836fcc884370,0x0005328231b4d1b5,0x000895d85b956e68,0x000117afd7ce3e4b}, {0x00028a48e23cd96a,0x0007756cbf1cc4fc,0x0000000000000001,0x0000000000000000}}, {{0x000c45bb98d0ddee,0x0002774201b8565c,0x000b4f52020bec2c,0x000bda65cd76ab62}, {0x000ae2fb221ac3aa,0x0007ba962fd348e9,0x000e7c381d5e875b,0x00068ae119edacdd}, {0x000b495d57186eb9,0x00056fdf795bd0d5,0x0000000000000001,0x0000000000000000}}}, + {{{0x0003ca5ab59ddb41,0x000f7f1ebc9c6cc7,0x000f57e85f634cbe,0x0005a16eddf122c7}, {0x000c8a63efc695ac,0x0003541555a07f9c,0x0002a4d80e98783f,0x0006230998aabf3c}, {0x0008cb24667129fc,0x0002de1b3b8ba86c,0x0000000000000000,0x0000000000000000}}, {{0x00062aec0af56692,0x0007719a7045c1fc,0x0005b9718436f251,0x000c9b5d581f4e26}, {0x00069b5489f5725c,0x00051e7c1aa8e91e,0x00028ec256fe93ea,0x000487a680376ebe}, {0x00030e63df392cad,0x0009887cd96ae342,0x0000000000000001,0x0000000000000000}}}, + {{{0x000e42367a9d725f,0x00039383477d80ab,0x0009bf84781ae655,0x000527eaf1389d3f}, {0x0004102ffac6350d,0x000a3b0583300052,0x0001cf3332af83e5,0x0006d633aac030a5}, {0x00033508c7e87b5f,0x00094e54cf554461,0x0000000000000000,0x0000000000000000}}, {{0x000f352ebddfa61b,0x00025e3d5e304a79,0x000673478bec8a85,0x00072acfcd082890}, {0x000446528a7ef652,0x000120a7a607746b,0x000c8aaaa126f2d6,0x000e0714c411ed8d}, {0x000219322242ecc1,0x00076f4dd29b9909,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c19050d8003bd,0x0000966c18887790,0x000653ed598ed239,0x000bbd18179a89ab}, {0x00050e32e18d6ebb,0x0003a242275a6e33,0x000cf9fda141c240,0x000cba7fbc6f732a}, {0x0007d9042bdcaed5,0x0004cddd17f09749,0x0000000000000000,0x0000000000000000}}, {{0x000dc494bed8e1aa,0x00072f9b4efbdcb4,0x000846e40ceacaf0,0x0008e7a08c964b79}, {0x000b697992c6dba0,0x000a31236d2a3b33,0x000f13c67e503cfa,0x000758371b2297ad}, {0x000a30fe4d1b0d5b,0x000937f2f3ba13a0,0x0000000000000000,0x0000000000000000}}}, + {{{0x0001e3e792f54f5e,0x0002d66e1c349e9d,0x000411c782a7cb86,0x0004b067774f6f73}, {0x000f3d88b7029f91,0x0008ac9342be145f,0x000fba10730a2fc6,0x00017ace014c77dc}, {0x0006ebecfe34f062,0x000f04b7a85b9087,0x0000000000000001,0x0000000000000000}}, {{0x000e45d39d99da4a,0x0006122af68cfe2b,0x000beab553aeda14,0x000338ecc8cf47bb}, {0x0007a9f26575d185,0x000dafc93ebcf570,0x0006b569b4f26152,0x000a1c5170968500}, {0x00059575a58e4408,0x000d43a451b6b327,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=77 [{1,2,3,..,}]*([2^308]*G) */ + {{{0x00031eea8652aa77,0x00075a9b33eda2b8,0x000ccb42fd2dd1ac,0x0005ab8fe55769fc}, {0x000ee2ea2c9e4a86,0x0008effb93c60208,0x0001522321dff3f2,0x000e5124fb782d6b}, {0x000f961e96e29dc3,0x000a6d2f39193c4c,0x0000000000000001,0x0000000000000000}}, {{0x0005e63b7e317479,0x0009d667da851f03,0x0007183aa797e9f3,0x00059f9fe886c75f}, {0x000d96d9e68574aa,0x000ee25277706045,0x000aa94b18ceb8f2,0x000d5bc3971ae4bc}, {0x0007b608157fc8f0,0x000150dd6428487b,0x0000000000000000,0x0000000000000000}}}, + {{{0x0003f45b89aa7776,0x00091f2af99e2d25,0x00076414d1e112b4,0x000800014a8e2b12}, {0x000ffa17691a71c3,0x000bd4233ab0f6f9,0x000bd49cf4e764a4,0x0004012735f0ccb8}, {0x000037d3dc7fc071,0x000ae03da811ddf6,0x0000000000000000,0x0000000000000000}}, {{0x0004a8fb960d3228,0x00048eb39e1a42f2,0x00007ea05b5a3c40,0x000b7bd9d581bb93}, {0x0008feeb6ce36fa0,0x0000329bb28b7b8d,0x000e608fcc8cab6a,0x000504c03b085c44}, {0x000593cce3f2c4c9,0x00093bb2ef5a3d51,0x0000000000000001,0x0000000000000000}}}, + {{{0x0005eee4a29b9e00,0x000dc20740482d3d,0x000e9ca5a465e210,0x000dbcf9b04be3f8}, {0x000cf79a10d11c21,0x00098890d16c757a,0x000df4eccacfdb2e,0x000f11b8ece9836e}, {0x000bc4d7a9a35695,0x0009ee2c9b7d003b,0x0000000000000000,0x0000000000000000}}, {{0x000e2dd26ff13203,0x000133cc51c6bc72,0x0009ca679c2a2da6,0x000951011dc4426e}, {0x00093bedba6ea308,0x00088d9217c2a099,0x0000e9979694eb83,0x000d521e97150620}, {0x000035d356c66be4,0x000161e2cef90abf,0x0000000000000001,0x0000000000000000}}}, + {{{0x0002a5d7a2fe5424,0x000c5abd77fdcc05,0x0007ae8084591e67,0x0006481030c17511}, {0x0001f92e55b534fb,0x0006bd1eba6c21b3,0x00096514d056aef9,0x000df597cfd93bd8}, {0x000c4de2939ce9f5,0x0004ef6737cbaf10,0x0000000000000001,0x0000000000000000}}, {{0x0006ccf17f5b4aa6,0x00049ba070d7b272,0x00033084cb0ef8a2,0x00018fa7fece71d7}, {0x000e7f11b328ddbf,0x000b8c5b898e7f8f,0x000699bc5842d33d,0x000944b7141938ef}, {0x00039a13d619477d,0x000d5228db36f5cd,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b8747f420dd4b,0x000fb52bb177eca6,0x000f43dcaa9b6760,0x0007a3f15a0ec05a}, {0x000736800fc5d305,0x000bb74572a151d8,0x00065a96cf22ff7d,0x00019519394cd42f}, {0x0008227699f90840,0x00068419e24fe2e2,0x0000000000000000,0x0000000000000000}}, {{0x00025578363927ed,0x000574db66d688d6,0x000d409ddc1459fb,0x0007f82b70d69f87}, {0x000cb67c0931340b,0x000a17e9ba495398,0x00095f5bb6fc86af,0x000372330a201b0b}, {0x0002bfb684331f96,0x000759b31f0fa4ba,0x0000000000000001,0x0000000000000000}}}, + {{{0x0008a9686e50ea9a,0x000fa53caf16bb29,0x0001e6775d854b08,0x00037450aecb3742}, {0x000a7e3971ac67a8,0x000916720ee3a053,0x0007ef527af5a2ad,0x0008c78907d26a51}, {0x0000187ff88dddeb,0x000580501085ffc1,0x0000000000000000,0x0000000000000000}}, {{0x000bd54c631b5ef7,0x0000a533ff81bdf1,0x0000234fa962cf5c,0x000a7f99121d8411}, {0x0004d769fb90a1f3,0x0005f3f42e621b7c,0x0007e5f02655798a,0x000d995fc9fe9a95}, {0x000560ed712fc645,0x00059000f20fc194,0x0000000000000001,0x0000000000000000}}}, + {{{0x000ca8aff309b100,0x0000c85dcacc4c44,0x000282732cee9318,0x000efc73a9f020ed}, {0x00070ac2ec46ccda,0x0004edd612a22d61,0x000fb7673b4cf6d2,0x0000510828deb00f}, {0x000a061d4c49e843,0x00016c8aea6c342b,0x0000000000000000,0x0000000000000000}}, {{0x000ffaa28b87e79b,0x0001bb9617a5663f,0x00097f1ba9e99062,0x0002ee20742a4b1f}, {0x0000500e34cdf43b,0x0002b6de4ad6c0ab,0x00002eba08ef6f81,0x0004e0bbfd3a6364}, {0x00094460899a3582,0x0008335d3e07b17d,0x0000000000000001,0x0000000000000000}}}, + {{{0x0005a71e395d239e,0x00026a160db974ae,0x0004df55ba5b2671,0x00091fe0f2bfd214}, {0x00027e4215d39cf2,0x000849498b3dc0a6,0x0000c7dfec311ed8,0x00029fbb50995b22}, {0x0005f9ca4a3d83cc,0x00085332f62dd6c5,0x0000000000000001,0x0000000000000000}}, {{0x0003278db69ec48f,0x000c56caebf5d4e7,0x000e4ab979b38b01,0x00005e1e7e210f66}, {0x000e800e35bf7fe3,0x00041625bdbbd247,0x000140764eabbcaf,0x000ae49d3fb6b3c0}, {0x000bed09dc31a840,0x00044a6c67185e6e,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=78 [{1,2,3,..,}]*([2^312]*G) */ + {{{0x000976185cedb78e,0x00000d5ac29ab973,0x0005376ef5010492,0x000fcfdeb7cabe96}, {0x000e82ebeaa6eb29,0x000856863e849702,0x0002b7dd5820c1d9,0x00080b85b8b6adb3}, {0x000cc2bebcb2a1da,0x0001cb911240a24e,0x0000000000000001,0x0000000000000000}}, {{0x0009339d66471f42,0x00086865738f288e,0x000ec9ab31ac9389,0x0006dfc0b78b477c}, {0x000d22531d4c9b75,0x000957b1f72bea7b,0x000f90819668750a,0x000023544082b7ac}, {0x00010dd35fa97aa9,0x0000657c9376d4d3,0x0000000000000001,0x0000000000000000}}}, + {{{0x000d393a7e12c9a1,0x00087315af1e7695,0x0005dad48c909f6c,0x00065a7e168b010a}, {0x00026d86fdc5603c,0x0008f52d5373f51c,0x000a497697c8b7d8,0x0006670982debc64}, {0x0009f942aaf7a067,0x0000beb15cc57a80,0x0000000000000000,0x0000000000000000}}, {{0x000728397fd456a4,0x000c8ff5563ef971,0x0004e73a2dd305f3,0x000cc516a80ae4a1}, {0x000352258160b828,0x00008533e6e74db7,0x000028314ad68011,0x0007541598a03b32}, {0x000c3d815763ab10,0x000089f632644f56,0x0000000000000001,0x0000000000000000}}}, + {{{0x00096953520ee092,0x000306d200f67521,0x0008a20dfb41aa95,0x000ecbdc450070d5}, {0x00044a73c2aa2e56,0x000cf15e09936979,0x00039822bf1cc5f8,0x000ba98dee98b81e}, {0x00049763f39d2614,0x000dce88bf80d042,0x0000000000000001,0x0000000000000000}}, {{0x00090be494194f3d,0x00009eeb5f7526ea,0x00042892f629d76e,0x000de6b9665e7661}, {0x000e4db45bef0dd2,0x000f0c29ede66edd,0x000cdcbd947a3fe0,0x0009dc0bb66739bc}, {0x000aa185f9760092,0x000d98f355b62fee,0x0000000000000000,0x0000000000000000}}}, + {{{0x00095d178d8fc8db,0x00088ab909e6143a,0x00089b7600c18603,0x0000f4aa942112c1}, {0x0008b5a1967f7a08,0x0003543a0e5057c0,0x000cafbf9ac78fd1,0x00049408a20a1598}, {0x000b58bcdfa7974b,0x00036217ad4e198f,0x0000000000000000,0x0000000000000000}}, {{0x000138c5bd603cb6,0x00072af896026457,0x000ea9d78b2185f1,0x000482318652917b}, {0x0008b9b757159621,0x000f2c7ae3b7470a,0x000d10f532d77474,0x000d6b40b8bfc96f}, {0x0004da23277081db,0x0005192cd44f13a5,0x0000000000000001,0x0000000000000000}}}, + {{{0x000210b018203491,0x00081f7f56d37ef1,0x0003499eb28c2bde,0x000d16d800ddf3bd}, {0x0001a8ad11d2638e,0x00081f6ac5a19953,0x00073cbdac06d819,0x00089e5df343701d}, {0x00080e56132b9d88,0x0008f82b847a3187,0x0000000000000001,0x0000000000000000}}, {{0x000ae2fb5a3b782f,0x000dfec66b9766a4,0x0003802b7f84c2ba,0x0002adc75633f423}, {0x0009ba300fd1c729,0x000a85bdc16e7491,0x000bea7ecdc049f9,0x0008cba7fd0e8182}, {0x000a5a9aa8bce3aa,0x00008a0977d90d48,0x0000000000000000,0x0000000000000000}}}, + {{{0x000ee51072d71e54,0x000e9e2a68ca9bef,0x000a9707bfc11ef4,0x000ae91635e6d8f0}, {0x000dd87435cc0e00,0x0002d6a89b217b9e,0x000c5bada1452b8b,0x000d602de29b175c}, {0x0003f43b9bd73fb3,0x0000724373642133,0x0000000000000000,0x0000000000000000}}, {{0x0002c3dc04031002,0x000c73ba54808162,0x00084fe7d46fa018,0x000e61eb44ff91a6}, {0x000448f89778fb3a,0x000d0dc5c96379eb,0x000f26ee0588d9eb,0x000aafd623f750bb}, {0x000b6e0aafb4a855,0x000ab06c0e1eab53,0x0000000000000001,0x0000000000000000}}}, + {{{0x0001292993c3f0c9,0x0009243ce8e0477f,0x00002cd6d125a6a7,0x000424b8f8d2f4dd}, {0x0000e036966f98b0,0x000b28d9609c6ed4,0x0006f54dde908ae8,0x0001261395876628}, {0x0001a89f477ea392,0x000b89a6253dab22,0x0000000000000001,0x0000000000000000}}, {{0x0000776cda9e2906,0x0004dd51a83a13fd,0x00024cb69c0833d7,0x000afa9f2d0d7515}, {0x000d5068104d274c,0x000b1536c256cfc3,0x00005c2c5a3bb4ea,0x000e695b252297e0}, {0x000674ad5032178b,0x0007afaaac5118a0,0x0000000000000000,0x0000000000000000}}}, + {{{0x000b5f87444140a9,0x00070937761e89c1,0x000052402d8c768e,0x000e0d8fa7063fcc}, {0x000065032ca28437,0x0000560b81d70497,0x000a63bfcc5af72e,0x0007abb68cfddac1}, {0x0007b3b85e89f391,0x000e3880d7454a25,0x0000000000000001,0x0000000000000000}}, {{0x0001d4cbebc9cca5,0x00014fe965181800,0x00064d65a9766b10,0x000646ec511b3639}, {0x000cc26e7c4e0d56,0x00094ae11adfae8d,0x0001a6886e8d406a,0x000547bbf4ad6e9a}, {0x000e8901b3004a68,0x000c8d5981c48013,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=79 [{1,2,3,..,}]*([2^316]*G) */ + {{{0x0003223ffb7f0034,0x00056eeed01f7ec6,0x000a95759d3b10c6,0x0008fe9fc8ceacdd}, {0x00036b6ab8ec35f6,0x0008c0b21550e979,0x0008e1de4ccc3b92,0x0007fc6be17e92be}, {0x0001bdb6320828c7,0x0009d213352a78b8,0x0000000000000001,0x0000000000000000}}, {{0x0009b3c1ecfa9e13,0x000b3a0b5daa423d,0x0006bf95aa594567,0x000fbfb5a3d149d4}, {0x0003d4e9979588d0,0x0001e08ca45e636b,0x0005c358fa3b11bc,0x0003552e11b17364}, {0x000b67bc8931ab99,0x0002d3ce614c5f0d,0x0000000000000000,0x0000000000000000}}}, + {{{0x000ed714844f3544,0x000cc4bb9010843a,0x000996ac0026321b,0x0003453553c74ad5}, {0x00041741982c7cb6,0x000196d8fedc48a0,0x00020d244c9f0921,0x00070a8fce97fabc}, {0x000a6d44732828d2,0x00060bcab0c77562,0x0000000000000001,0x0000000000000000}}, {{0x00061da3943969d2,0x000d674b749d7b3a,0x0009df6dae1fb5b6,0x00005c30ec275083}, {0x000ae7da1a9284fd,0x000c82a28eb7bd6d,0x000a71d66cd19bae,0x0002d599b2c6a08c}, {0x000faa1546c312c5,0x0007f06795e3065b,0x0000000000000001,0x0000000000000000}}}, + {{{0x000fff9188aa7d8c,0x000deb80ad064a0a,0x0007114ab9689af5,0x000fad0b4dbde778}, {0x00055fd29cd3c099,0x000d379d42525d60,0x00016d04df50e85b,0x00053602e006dfc1}, {0x000e6c63f374d96b,0x0000f26509a7f32e,0x0000000000000000,0x0000000000000000}}, {{0x000822c176aa9790,0x000f58fc039cb1ee,0x0005872cad56c2fd,0x000e8ae0ea665324}, {0x0004daf2e64bf3b9,0x0008f96bb4b8314c,0x00090e063c57f41a,0x000d5149d3063df9}, {0x000ba61281d5f9b0,0x0004ba3d6cc9c608,0x0000000000000000,0x0000000000000000}}}, + {{{0x0005d4a76da49005,0x000c3b224e91125c,0x00062ab184a74621,0x000b682a17406495}, {0x00053c3f7c8cdedf,0x0008e38d4416ae20,0x0009c28df044060d,0x000f86143e5739ed}, {0x00095f9f7f327b97,0x000872538531478b,0x0000000000000000,0x0000000000000000}}, {{0x000b98e468155010,0x0001fc05661b3943,0x0009ee23198c1bcc,0x000744fc64ff1647}, {0x000560f20d871115,0x0002c9feeacdf5ac,0x00070b263cba9c39,0x000badbac8fda72f}, {0x0001aad35365c71d,0x0002f99d51687d17,0x0000000000000000,0x0000000000000000}}}, + {{{0x000acd4ec9302acf,0x00046876812a6969,0x00062f921abd47ef,0x00030834c8ee3434}, {0x00087c08c033bb79,0x000e51d0a2369c3e,0x000c1fbd98cac8fe,0x0006a309b704c675}, {0x000a173a43fcbb3c,0x00009432c4949569,0x0000000000000000,0x0000000000000000}}, {{0x0005e781f2ef36de,0x000e3479bd3a702b,0x000d74e86eb68837,0x0003849622881aa5}, {0x000ba91b89a84ecb,0x000caeee87dd2964,0x0000f40b0230b757,0x000e7853cadc83a1}, {0x0005f5ad1465657a,0x000e75100e5033e4,0x0000000000000000,0x0000000000000000}}}, + {{{0x000e44f3911651e4,0x00088f7f22b492d2,0x00072f850dbf1662,0x0003ab2a45a14853}, {0x000e1480b82ee674,0x000ca609c8235a84,0x00085d8422668b9b,0x000a5d6f0bf02f4e}, {0x000afb880792321d,0x000cb82c095f0261,0x0000000000000000,0x0000000000000000}}, {{0x000bc2f5725cea9c,0x000e1f43f99381e6,0x000e6089c844a832,0x00000aa951ad7011}, {0x000282a695207656,0x00007e689c114477,0x0000e537bb9a4c6a,0x0009df06eaf0cd7b}, {0x0007c474d7895232,0x000f0710d2b00b77,0x0000000000000001,0x0000000000000000}}}, + {{{0x00093c8e0aebae92,0x0001a4248ef98180,0x0005353d71384b97,0x000564228ab8dd10}, {0x000ee9e95615cf3f,0x000dbed91f163427,0x000ff8c7cd8d83e1,0x00002a117b26a05b}, {0x000d651309094b7c,0x0005fa8d73f3a728,0x0000000000000001,0x0000000000000000}}, {{0x00045cf4f3fc6c29,0x00069a5dd01b3bfb,0x000e3b24278a983b,0x000ba6e8dda15e64}, {0x0007ceabdafeb0be,0x00028dd1f4ce3cbe,0x00083c003c3a01ee,0x0004286b68f03154}, {0x0003661bd44cc13c,0x0006d7a5a2b18a65,0x0000000000000001,0x0000000000000000}}}, + {{{0x000f4f2d2f859a83,0x000584cf466ff03e,0x00078dc82b847044,0x000110dcb52d320e}, {0x0007adfe140d78b5,0x000b45fd46e07b11,0x000943939af65810,0x000f26b6d5c5bda1}, {0x00091095a8309f53,0x0000d4aad23c7d80,0x0000000000000000,0x0000000000000000}}, {{0x000dd82bca9e7ca1,0x0001551ee78b6090,0x000453fc776839c5,0x0001d0262966b875}, {0x000729e2a29966bf,0x0008c49cfc825d3c,0x00009961345ab1d4,0x00007e6049f3ad60}, {0x000007da4f1b3985,0x000c382c8f36cb0b,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=80 [{1,2,3,..,}]*([2^320]*G) */ + {{{0x000a284c690646dd,0x0003eedcd5cc91eb,0x000ea471fd6292fa,0x00023a125841cc32}, {0x0002e35810a749fb,0x00061336484e18eb,0x000b6f65228a2bfd,0x000207542e7452ca}, {0x000526cd940c7469,0x000d2936b9b32962,0x0000000000000001,0x0000000000000000}}, {{0x000573e4f063ef2a,0x000418ca996c2a17,0x00033e1f9c1c3d42,0x000b3cbf970fbd47}, {0x000088c0b1561246,0x0006e93234f08535,0x0009d7f8ff901881,0x0000aa556f8574d9}, {0x000a5d989d3b1d29,0x000ed0ab78218edd,0x0000000000000000,0x0000000000000000}}}, + {{{0x000233d6557077d6,0x000f2b32cea9ff87,0x0006963e65db1454,0x000b05b7d5f2627e}, {0x0005a68cad15bd15,0x0009679cfef2c921,0x000cc1f982da4ecf,0x000673910763dd21}, {0x000110fdc3925aff,0x000a3cad0858b1ce,0x0000000000000000,0x0000000000000000}}, {{0x0000c51bfdb7b566,0x000e8b88f58f7516,0x000713a7cf39e39e,0x0004ac365af813ed}, {0x00040a788e43e8ac,0x00010b5e01e789c0,0x000ddf33d0cb49ec,0x000a3f2d9bd726fd}, {0x0006e3c30504e525,0x000e51a456acdf77,0x0000000000000000,0x0000000000000000}}}, + {{{0x000707d7da6499d4,0x000c004b6d85bc23,0x0004b483dd72372e,0x000b15c9838f63c9}, {0x000dd40b6584e869,0x0005bb5ad6291644,0x00069be693ec1c90,0x0007b5c6018dc109}, {0x000c9c113b81150c,0x00040bbd460de804,0x0000000000000001,0x0000000000000000}}, {{0x000d558a1b81757d,0x00046da356589b5a,0x000f093ea9cf88e0,0x000cd54eede9de0f}, {0x0001f19ec3f8839a,0x0004ec243ffcbf45,0x000d0d1f8b02c0e4,0x000a42d2cc07981f}, {0x000363b43f4701bc,0x0007f930f2e9e43f,0x0000000000000001,0x0000000000000000}}}, + {{{0x0006b0772d9f5845,0x000f28a8c7c3d700,0x000c1d96b231ba3f,0x0000f432c17a4f5f}, {0x00014ca88f653da5,0x0001ac5da9fce5ef,0x000105dd10257bb3,0x000206b910de18d3}, {0x000d121d64f95008,0x000e83748b9a298e,0x0000000000000001,0x0000000000000000}}, {{0x000dc94560ad3e4a,0x000b34cda193affe,0x000f39dff5030add,0x000d72c1a3a58a0d}, {0x0001ad7c02e84586,0x0006dddddd7190f7,0x000431dd7f7815ab,0x0001b059af2893fd}, {0x0006d66ebdb90a30,0x000ac5b55b254562,0x0000000000000001,0x0000000000000000}}}, + {{{0x000a7032755a2cc5,0x00087b60173b4c02,0x000853a0c8b700e1,0x000d3fcbebfa5d41}, {0x000106636a248a74,0x000d439df17f1529,0x0001a48433bf866c,0x000ed52b92a93d36}, {0x000de5dbf96508fc,0x00064c08fb48dbcd,0x0000000000000001,0x0000000000000000}}, {{0x000e6d70757d607d,0x00099aa287bad741,0x000aca83d8bc1d01,0x000a6deb3248272c}, {0x0006281490886dee,0x00003b3e7ed5830e,0x000b8f50b5515018,0x000ee61ae410329b}, {0x000dd209b1b1ec67,0x0009f79d8f057d2a,0x0000000000000001,0x0000000000000000}}}, + {{{0x0009240ac4aeeb4f,0x0009de701cba0d15,0x0001e2030d5be49e,0x000f5b9fa8d80ea6}, {0x0009e389aa0a7891,0x000f08f46a281d5c,0x0003d8942c2a6a9b,0x000dcae5c6263013}, {0x0006ed6f226d80fb,0x000e7cd744527397,0x0000000000000000,0x0000000000000000}}, {{0x00064112eebe5a16,0x0005561ba10f054b,0x000076de3983c715,0x000338a0051c721c}, {0x00017ed93ec2b1b6,0x00040d08e30b18e6,0x00086d02546a805e,0x000b289546bf39d9}, {0x00029a40d87fe36c,0x00003828ca6d96cb,0x0000000000000000,0x0000000000000000}}}, + {{{0x000888aaaf08b61b,0x0004baeb89b8a3b8,0x00013c31ce0504b2,0x000084891577d88f}, {0x0001501541da01d3,0x000be18906c31edb,0x000cf8acb88a0c0f,0x000de0a54814b123}, {0x000d30b10ce17eb8,0x0003c65435ad1112,0x0000000000000001,0x0000000000000000}}, {{0x0003c3081d6e0b2e,0x000bd1198cbd6e7a,0x0009feff60218481,0x0009559a8a4e33b7}, {0x000ae242155d34dc,0x000458dbdb49b265,0x0003688660033750,0x000853753ede19c3}, {0x000b6969aac09e0c,0x000a5225b275670e,0x0000000000000001,0x0000000000000000}}}, + {{{0x0004c030e13db910,0x000b8e4bb182d8fd,0x00024d5733c45ba7,0x000e09929bbae322}, {0x000c5e18395c5857,0x000beb34317a4b7e,0x00065979d2ffacfb,0x00000dac7ff47099}, {0x0002ac181634b33c,0x000b325a113dabd2,0x0000000000000000,0x0000000000000000}}, {{0x000f6d0b44a18451,0x000ebd4b60b5e31a,0x0007c6c236a60067,0x000b1be8ccf47b3d}, {0x000a21dbd1cc7199,0x0004932466e888eb,0x0001dee034c21f8b,0x000ff9da169619ff}, {0x0007e95c7e040c95,0x000b5a9dbe56efee,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=81 [{1,2,3,..,}]*([2^324]*G) */ + {{{0x0008f08161486b22,0x000be755ab4efe68,0x00048d9609c012b1,0x00098843db068e78}, {0x0002d958488ad0fb,0x0003f6d23cff5eda,0x000c41d3ec7372a8,0x00035d185ec0b176}, {0x0006314a5925e490,0x000a3de4ff957947,0x0000000000000000,0x0000000000000000}}, {{0x00029cc94c1b43aa,0x000d43d2ad417dd5,0x000360ab4fa2dfe7,0x000e9eb552cc454d}, {0x00035a4732c24fb2,0x0008d1e843cf82a2,0x000ab8e67bb7a406,0x000191877eafbca2}, {0x000574ab099566cd,0x0009f922f9872f62,0x0000000000000000,0x0000000000000000}}}, + {{{0x00085f8208bbe4fe,0x000ecc287db7bb2d,0x000e568667e702fb,0x000cbeb7a157f36d}, {0x000bf484f3352f4e,0x00058da014941bbf,0x000586c3d5fe38d5,0x000a8a8ef1b37b22}, {0x0005949627a9e7fe,0x0004740c422ebeba,0x0000000000000000,0x0000000000000000}}, {{0x000c2ec0fe63724c,0x000be0eb88269034,0x00016f607ed8c7ff,0x000b235b0a729f09}, {0x000b2fb783d219ca,0x000a1f91a0e85a3b,0x000f36eaf1659ef7,0x00023c3d4be9067e}, {0x0005e5bd92b43e99,0x0001e3e81391aa3b,0x0000000000000000,0x0000000000000000}}}, + {{{0x000cdab06c0f497a,0x000ba682d62fd58a,0x000624d8816a960e,0x0006c48669b75099}, {0x0001fb169d6d2670,0x0009d5b924784941,0x000361b9811c3888,0x00005ccc21278392}, {0x000173173a0f5de5,0x000862da0030f596,0x0000000000000000,0x0000000000000000}}, {{0x000580503b5bd1a6,0x000460a53f77d734,0x0009ac0fc516703a,0x000cc1b1f0c1292d}, {0x000599d98d3204b9,0x000365fabf012bdb,0x0001f82c240390bf,0x0007f3c05cb807a9}, {0x0001aacb4486b03f,0x0002d4bf3cd7e64a,0x0000000000000000,0x0000000000000000}}}, + {{{0x000cf4fb66902a59,0x0006522d970cf058,0x0008db985646108b,0x0005bd091c524ec7}, {0x0001c8ded01bac37,0x000a7571d5eaf41a,0x0003ae41513bf75c,0x0000831a58ce8343}, {0x000b5c1d0b1cbad6,0x000a127e3558b4d4,0x0000000000000001,0x0000000000000000}}, {{0x000c5bcaa2423577,0x000e95cd90416c5f,0x000f9cd3e851fd11,0x00043cec77429d71}, {0x00033818263e74b1,0x000b0bed2ab694e3,0x0009d3b078fef207,0x000322c62d90900d}, {0x00013057fb2dcc39,0x000b0aee2cd8c7f7,0x0000000000000001,0x0000000000000000}}}, + {{{0x000ca837cf1af373,0x0003ae0bc7638546,0x0005fbd88b9c057e,0x0000f14623993437}, {0x000d8418b02866d1,0x000938602b2a7398,0x000d172bab8e8fb6,0x00001960cc17eb57}, {0x0000f437b4d86f8d,0x0000c3266073d8a3,0x0000000000000001,0x0000000000000000}}, {{0x000b4a64de3a3170,0x000d07d300586858,0x000590b48d2c8237,0x00064608c48f8521}, {0x00055a4fad372745,0x000ad1664c3cb9d0,0x0000e5148678bfbc,0x000ce1f67995bdb9}, {0x0000ce6b82ba0137,0x000865e65d5e9060,0x0000000000000001,0x0000000000000000}}}, + {{{0x000a3697b62e274a,0x0004f036666b6cc6,0x0004615de0a77111,0x000d1a544656bc00}, {0x0004cfe1b0ffd27e,0x0007b011fe7a367a,0x000028f9395287ba,0x0000e474d2cd1539}, {0x000df81c967ab663,0x000809d416f7d850,0x0000000000000000,0x0000000000000000}}, {{0x0004ff0171522411,0x000cb614e9eb99cd,0x000efe5013168ce9,0x00000068878690dc}, {0x0003ea58b25b4f05,0x000697bfbefe708a,0x00061484cbc18871,0x000fd61b572d109e}, {0x000507a67ea6e538,0x000c108cf0642bcd,0x0000000000000000,0x0000000000000000}}}, + {{{0x00073712d28f1037,0x000dcbb85bee07b2,0x0002d427b383b0b0,0x000886a4c921569c}, {0x000a22bb54a08b63,0x00016efd18019f62,0x000be30e896a901a,0x000b1c79b63790d0}, {0x0007550d4ad3f339,0x000c014ebb2a7324,0x0000000000000000,0x0000000000000000}}, {{0x0000eb80b7360646,0x0004292a8064819e,0x0001aa8d66966eba,0x000cb5433c6786ce}, {0x000c176010116f6a,0x00042b7f40b26c85,0x000bb51bb431c00e,0x00079fd699bb4a71}, {0x00083711f2c2acd8,0x0008a64b1e3905a7,0x0000000000000000,0x0000000000000000}}}, + {{{0x0004f281744c4f31,0x000fb974f0427525,0x00022ee390de534c,0x000caed9f368e25e}, {0x000470d3f5a1ebc4,0x0006bb7427d70104,0x000830891f024042,0x0006993cd5f37f0d}, {0x000a9d89f4ebe578,0x000c7b2549a02c1b,0x0000000000000000,0x0000000000000000}}, {{0x0005527d2ef3c160,0x000aa733afd2d857,0x000200435a36240e,0x000dbed50df72a8e}, {0x00064e9f3dcc7104,0x00084041401f5c34,0x000b281791b398c9,0x000ad77cd49e1581}, {0x00029530ca203aa2,0x00024a7004073823,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=82 [{1,2,3,..,}]*([2^328]*G) */ + {{{0x0004b896de83f7b2,0x000dd1b2e000727a,0x000d43d965547a68,0x000c1f1e1ce79a50}, {0x000e5ec87252c6ab,0x00060d6a5ae7160f,0x000985fdcf45caab,0x00091b3180d9d235}, {0x000646ab8f898256,0x0009f0446d6798dc,0x0000000000000000,0x0000000000000000}}, {{0x000ec719b8387586,0x0001de5f9db6636e,0x00038f4a187cc943,0x000ac366ef383b03}, {0x000022fa366743db,0x000760fac1cc0b0d,0x000265067948b1b5,0x000693934495e8d4}, {0x000c64b8f4f88921,0x0005281cd8ec2ed3,0x0000000000000001,0x0000000000000000}}}, + {{{0x0008479d81fdc38a,0x00096a893e0d1c8a,0x000972cecee045ea,0x0000ed0b37445b26}, {0x000bf1c0a16a9b25,0x000509c76808e477,0x00070c4c826b6837,0x0008008a3f9fb748}, {0x00088d0f74d58040,0x00063d18d474fc4b,0x0000000000000000,0x0000000000000000}}, {{0x000a143fb2178ecd,0x000ee2726dcabd2f,0x000e017d3cb36d39,0x000d77e2d8d9e011}, {0x00069332c865043b,0x000231a13fc89650,0x000e078f7a775c43,0x0002e93c91cf1e3d}, {0x000d48604e0f7c89,0x00031d2709749250,0x0000000000000001,0x0000000000000000}}}, + {{{0x0006fca2f24da625,0x000adeacc535625c,0x00020bfcd99308f2,0x000732872a2697fb}, {0x0007578b9abdd39d,0x000d7e3c3e30d29a,0x000dc63c314955de,0x0002de0cf2a14e6d}, {0x0002e4f4d6b0bbff,0x0008423dc75d4cf8,0x0000000000000001,0x0000000000000000}}, {{0x000dc6563086abd4,0x0007f2e300951d8c,0x000a989054197751,0x000a88edbb68daee}, {0x000ee17db6bcc0fa,0x000d996d71fcc36f,0x00038a1dafea4a84,0x000fc8d72ef68c32}, {0x000c6b07cf974baf,0x000b2a140ef1e1e5,0x0000000000000001,0x0000000000000000}}}, + {{{0x0006034713a3e42b,0x00008cfb50e4479e,0x000f617634a25363,0x000ba17f9549272f}, {0x000163e264556302,0x000db056ef0f6ed9,0x0009449c67d2e92d,0x00027bf608bef04b}, {0x000a41494e0fd2d6,0x000ecafaa0f9ad5b,0x0000000000000001,0x0000000000000000}}, {{0x0000394a908a0374,0x0002739ed3e6c1ac,0x0002bf950e49017e,0x0006ca69441c9b0d}, {0x0003c717b7978985,0x0006f1bf123315b5,0x000d5ffc5bce1316,0x0000f7b0dacbdc85}, {0x0005306a73236570,0x0000fc22ce0f19f8,0x0000000000000000,0x0000000000000000}}}, + {{{0x000a6e6af09b8eae,0x000343b43c513568,0x0003425270500040,0x000fb4508580e8a3}, {0x000b307fa3d1f99e,0x000113638a6f65cd,0x000be36db7f5ca63,0x000ede447eb7ec8b}, {0x00022e6ed48cec2c,0x0007bb1afd7c927f,0x0000000000000000,0x0000000000000000}}, {{0x000c11f9a334fdc7,0x00026ae8011e9b06,0x000ba4fc7ee152d9,0x0005ebe2d92b45f4}, {0x000643466d3d6908,0x000a09e637a743ca,0x000aba719664e4ce,0x0003162cc6d5e41c}, {0x00060cdf202e7dd3,0x000c307d09d0c5c7,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b621fceac0c99,0x000cd9f0c6398393,0x000246af19db94f8,0x000def03f4511189}, {0x0007e278282d9eea,0x0005d51872d67451,0x00069251c97cb275,0x000152306520f8d9}, {0x000f0746e9696ce9,0x0009ec26638c1b12,0x0000000000000000,0x0000000000000000}}, {{0x0006fbb28d6db318,0x000282f1e3fd8169,0x0001638aa745ba97,0x000741cbd6f6afe6}, {0x000674be06f9a37c,0x000f0e4d5ef37aa1,0x000c66c847151102,0x00086ff305cd0c0a}, {0x000dec0bec87dfaa,0x0006e29a33525521,0x0000000000000000,0x0000000000000000}}}, + {{{0x0009e94294764d17,0x0002085b396e5ed1,0x000a114f1c4a72c2,0x0007528c8545f538}, {0x000963b0d0aaa085,0x0000ee6c82fe930c,0x0009cc321991f876,0x000bfd79b22e04e6}, {0x000e404e5103af68,0x0009a8126bf818d5,0x0000000000000000,0x0000000000000000}}, {{0x0006f18c56b9245c,0x000be8cf2d749759,0x000d0534792607af,0x000737082c9ac67b}, {0x000b81d246c242f3,0x0004a1675a873285,0x000b8be65c0f0c83,0x00031f4367aaf8a7}, {0x000eeb7fdd2c4760,0x000b71b7bc940b35,0x0000000000000000,0x0000000000000000}}}, + {{{0x000e474b29bda866,0x000e2b3a2640ed4a,0x00021f91da93684d,0x0005df67424bab62}, {0x000b550d60209995,0x00096c99d55193b4,0x0003c748e9f27481,0x000631e6b3fe7e7f}, {0x000f47f4e257248f,0x000f8f56db9ba373,0x0000000000000000,0x0000000000000000}}, {{0x000bc357ebc4dd12,0x0003a33556892c48,0x000b72124cffa435,0x000e35056ea40bee}, {0x000a1b37aa3c71db,0x000ddc96c72e951c,0x000291c71de6fcaa,0x000ff88244eb58ae}, {0x00089a7bf96fd42a,0x000eb41c5d8ae97e,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=83 [{1,2,3,..,}]*([2^332]*G) */ + {{{0x0008ac7511fc1235,0x0005af51e9a589c7,0x000c09d63e56a5f8,0x00055b518ce24a89}, {0x000a75441652a9b6,0x0004ffab489b445d,0x00071289523b0e9f,0x000290aaf7cb23e7}, {0x000c9bc7899234af,0x0001dda65dc198b5,0x0000000000000000,0x0000000000000000}}, {{0x000eeaccabd5a6f7,0x000d5900e72c44fa,0x00047a63782a2bbb,0x000393bbf531aecb}, {0x00009c7dda45067c,0x000719fa6f31630b,0x000f3a0c95e46b2e,0x0004deaf5d70f849}, {0x0007dd5d46829965,0x00096c286bc1f082,0x0000000000000001,0x0000000000000000}}}, + {{{0x00012d5838ca6af1,0x000964e9241a8a04,0x00024077ab2ac6aa,0x00089da3e1536c1d}, {0x00001df56af4aad1,0x0006ef9e57fbfb6e,0x000b17f8ca8e6244,0x0009cbcb351a7c9e}, {0x00085fb54f3eda4f,0x000c296970873909,0x0000000000000000,0x0000000000000000}}, {{0x00056d8d32e5fdd1,0x0006814d7980be46,0x00065dbc6a68d7ed,0x000cebebe9b6528d}, {0x000269dfcc27d433,0x00073aec8225c88d,0x000d90643f7caaf2,0x00041c78327e8662}, {0x000763fdae4a09eb,0x000b0ead9bd2f604,0x0000000000000000,0x0000000000000000}}}, + {{{0x0008f42a8c29bd6f,0x0001879cf21db610,0x000555cfe2bddcfd,0x0004d452af269c11}, {0x000fcc4011856e7b,0x000e6e45593cb7c3,0x000b215415957c0f,0x000d3b983b2c8996}, {0x00012f953be5cf31,0x000f7078abc3a003,0x0000000000000001,0x0000000000000000}}, {{0x0002c0d6b39b80ab,0x0007b2847e724ed5,0x000cfdfc83942bcb,0x000c1c0a2ddac314}, {0x0001da690e67e2aa,0x000310507c60736a,0x0008b8515f2f407e,0x000203447dd4a30b}, {0x000208fe5f3ddc7c,0x000470482e113587,0x0000000000000001,0x0000000000000000}}}, + {{{0x000d7682eb5538d3,0x00058bd8734ac8b7,0x00075fcdec6d3223,0x000ffc3556d9d86e}, {0x0003a6c363b61e72,0x000d03c2ead1066f,0x00074fd7095dbd73,0x0002b42f9972cd16}, {0x00070acc2acc3e68,0x0006fd80c7114993,0x0000000000000001,0x0000000000000000}}, {{0x00049c09589c235f,0x000294e10709485a,0x000e55bcaedd0d2a,0x00084da3a073e38c}, {0x0004725346561d28,0x000ed8195e95d347,0x00001ec990b2c19b,0x000817664aecefe7}, {0x0005f12464c59ce8,0x000f85a23cc1c6cb,0x0000000000000000,0x0000000000000000}}}, + {{{0x000fb48cd5b9dd85,0x000ea5640b16ec8e,0x0004991733d00d1c,0x000e2230142b823b}, {0x000d1646ff2bfccb,0x000e87a4efbbe898,0x0007933417bfbbcd,0x0002ac0c744278e4}, {0x00079c8483d8b4b3,0x000fafae62ecea8b,0x0000000000000001,0x0000000000000000}}, {{0x00063d47c065003b,0x0004185cb3cef55d,0x000bfcfe8d3872f3,0x0009112c490c5e61}, {0x0003b1e7d4274313,0x000517366c9c6308,0x000bc9bb53389bc7,0x00025afa81750fe0}, {0x000afda6e1b71dc2,0x000c6a9aba51c85b,0x0000000000000000,0x0000000000000000}}}, + {{{0x00000e070af57b1c,0x000441246548fb7a,0x000c563ead66464d,0x0007508f0851b47e}, {0x000f31999ecfeae6,0x000f85797a3840b6,0x000c74080d60c378,0x000f3eaf1e242bf3}, {0x00003ca35d886cbe,0x00022754c9a357ba,0x0000000000000000,0x0000000000000000}}, {{0x000982d0aaae9f75,0x000176d1b4b6ca3d,0x0007980f7d421826,0x000979133a61fcc4}, {0x000cb63f41cac3bc,0x00079bb4b9516419,0x00033cd18a279569,0x00075e18895b57d5}, {0x00063599127b0d23,0x00063dbbded01670,0x0000000000000000,0x0000000000000000}}}, + {{{0x000342ad817f361c,0x000cb1bd9eb81b06,0x000c7b97a0843c43,0x00083f804f029e13}, {0x000b3486644af0ff,0x0002a64dc632346d,0x0002a39cc0dcbed6,0x00056457b2ff9c5a}, {0x0000fe8536001cce,0x000def7d14af048a,0x0000000000000000,0x0000000000000000}}, {{0x0006c4115861eef1,0x0006a14bad6b1ac1,0x0005dee3e9d201f9,0x0002b396cf147ed1}, {0x000f4643f8a21b3b,0x0004c9915584149b,0x000a893cfd7ba449,0x00027a8a4eb7aef9}, {0x000cc4fa992b1d31,0x000eec6c99a8fa7f,0x0000000000000001,0x0000000000000000}}}, + {{{0x000a8d985f7ab9a0,0x000b7d39a70c2f72,0x000f6f8acabea7d5,0x00095d0273f642dc}, {0x0007d8a54bc56deb,0x000a63f1883f3cc3,0x0004b831ef1bba05,0x0008e112f14c8c07}, {0x00089737917f937a,0x0004874d335ca229,0x0000000000000000,0x0000000000000000}}, {{0x0008374f770af11f,0x000cad8ee96d7e99,0x000bff9e11a7d432,0x00056384e6665366}, {0x0004a9b692423f6d,0x00075f044efc7e34,0x000ee60ee1b3dddf,0x00039cd00df7a827}, {0x000529eb5b2612c9,0x000cd7c4ffa6a13d,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=84 [{1,2,3,..,}]*([2^336]*G) */ + {{{0x000c16671a6774f1,0x000af03753ce5839,0x00054c5f8c07356f,0x0001afc71165a356}, {0x000f9d6adf86cf5c,0x000a6b4966903b89,0x0009f4ebff86c3fb,0x0004a87b0151b151}, {0x000efd27bbe4f95b,0x000b040513d26385,0x0000000000000000,0x0000000000000000}}, {{0x000622a63fa5d90d,0x0008d92aca99c7d4,0x0001d6acf3aa6efd,0x0001b7387e55d6dd}, {0x00010db119c2295a,0x00011a67dad9703d,0x000eedb427c0f52d,0x0001e055192fe412}, {0x0004a5758174c7a3,0x00055cfd4b1dde40,0x0000000000000000,0x0000000000000000}}}, + {{{0x0000d119f7fe9853,0x000fe49990254b37,0x0008a86cb40764d3,0x000820af39be0e2c}, {0x00027458321b04b9,0x000c2ba583294752,0x000ae89a07a5c7f2,0x0008fa6d520652e9}, {0x0005604c9bdc0eee,0x0000c06f2e484243,0x0000000000000000,0x0000000000000000}}, {{0x00014a30a2bfa81c,0x000cbd6640003017,0x000ab87a938a3f37,0x000e1c91f3132874}, {0x000e57e9d7ac6ecb,0x000e33fb881734fa,0x000073b600765b07,0x0009428cbfb5edfc}, {0x00029028585e9a20,0x0001e077ef7692bd,0x0000000000000000,0x0000000000000000}}}, + {{{0x000cf34a0d6f6ffe,0x000370c22b280291,0x000e87a26b1fd975,0x0008088a662b3666}, {0x000eea746601046d,0x0008edbdfb0988f2,0x000f2131f7fc1ebd,0x0009266b6d41f4b2}, {0x0003c1c020089694,0x000be27c849de8c8,0x0000000000000000,0x0000000000000000}}, {{0x000bcda37f3a594e,0x000726480ec74a90,0x00026216e2ddde9d,0x00064b02bef16495}, {0x000aafba3c749a5c,0x000a872930c3f630,0x000654a8695df3be,0x000cb5372491b21d}, {0x00017f3b3a2f3f6f,0x00094613fe01cfe9,0x0000000000000000,0x0000000000000000}}}, + {{{0x0008937b47b39090,0x000415bb7112fdec,0x00066e9e19d3ed5a,0x0006597801eab0fa}, {0x0005e740c409bbbe,0x000050b19bba9267,0x000df2c3e8b56daa,0x00012fbcf099e6ee}, {0x000195262a55e069,0x000ee2f2c7d1e980,0x0000000000000001,0x0000000000000000}}, {{0x000c1384c013e53c,0x00074951ffea5bee,0x000d0ad477deaca6,0x0004ee3245756473}, {0x00030808642161fd,0x00050c8b97a30694,0x000340f405f653b8,0x000d5a543cae9de4}, {0x00031ca24347d550,0x00092a75f4312ea3,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c8162c746ef6f,0x000f6715dbf9ea58,0x0005523d8217dd87,0x0000b2c52bc5b0b4}, {0x0005db8903ecd878,0x0004296f75f92e78,0x0003c6e6397e4045,0x000e84bad1cccce3}, {0x000b82162d3c5f54,0x00009333f935ae95,0x0000000000000001,0x0000000000000000}}, {{0x0004ff1c33e26a2e,0x000785b4ec10e1f5,0x000a1634274c2886,0x000b5d5ee5822d49}, {0x0002fbe9122e0bfa,0x00003c2cc2955a06,0x000e08e579ad9b7f,0x0001dd6ee255f2e5}, {0x0004f08f71b65e70,0x000c9dcd7d23cb93,0x0000000000000000,0x0000000000000000}}}, + {{{0x000927819d85c389,0x00088dd986cf5001,0x000eb6ebdddab174,0x000a1e388628281f}, {0x000014511bc0392f,0x000b79c2a5e1691b,0x000d866b842f8440,0x0000343c71a48805}, {0x0002ac5c5d5d795d,0x0004aaedd8558804,0x0000000000000000,0x0000000000000000}}, {{0x000896067875f110,0x00080b0d43dab8fa,0x0009a3104ecd6f15,0x00017c31840b3b59}, {0x0007841201091767,0x0008de871b2243eb,0x0003f7be2323a388,0x000f764799c353ac}, {0x0009d244edaf476f,0x000513c595b87c99,0x0000000000000001,0x0000000000000000}}}, + {{{0x000800832e6fdb7c,0x000df2ceb7712003,0x000625cbf7ec3398,0x0007b9eb4c74b442}, {0x00090424f2515df6,0x000abc10516b9778,0x0005df82462b4902,0x000a6d60d9807c8f}, {0x000606aee1bb838c,0x00030e1e7a2ff1ff,0x0000000000000000,0x0000000000000000}}, {{0x0008fce9b8ce853f,0x000b7049a4c20923,0x000ae2b39f773f7b,0x0002f55bcfbf4b1e}, {0x000b07309ae9653e,0x000cf869d6026775,0x00099fe1b0e83daf,0x000202f8a21d72ed}, {0x00037619de81bf7a,0x000b7b2bf238b7d7,0x0000000000000000,0x0000000000000000}}}, + {{{0x0008ca9dd54298a1,0x0000e1308270c47f,0x0002be3ff9743378,0x000471b0b186cdf8}, {0x000b1747ce696eb6,0x00015fe31005f60a,0x000ac1b5a457da88,0x000f4901af0f6bb9}, {0x0002f972c925bd14,0x000186acd8b58a65,0x0000000000000001,0x0000000000000000}}, {{0x0000f355372184c6,0x0005e1f7ba0c693e,0x0005b11db3d6dbda,0x00051b89e46fae1c}, {0x000c97c0b46b0f1f,0x000cc037caa48d5e,0x0000355bdcc75991,0x000f28784dc0e5f4}, {0x000837eadd1a3fa4,0x0009eeb5a1926d0b,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=85 [{1,2,3,..,}]*([2^340]*G) */ + {{{0x000847fc397e26dd,0x000de02cc0ff17de,0x00063e6fe388ee8b,0x0001e73a774123d0}, {0x000daf8cb9f5597b,0x000b938535ee20c8,0x0000d7da8b1bfac8,0x0008e2a819363df1}, {0x00079861ae7d4273,0x00003eaf0a677999,0x0000000000000000,0x0000000000000000}}, {{0x000611141de00c8a,0x0001d2aefc5b58bb,0x00033633ce29b3d9,0x0004bfef0d5e3306}, {0x000d78956c10a254,0x000fa84101beaa2d,0x000f9588ba22402c,0x000e0df8f46296a5}, {0x000c7018734ace12,0x000ca8e0e00d25c6,0x0000000000000000,0x0000000000000000}}}, + {{{0x000c26d9a28cda66,0x000ac34788e2808c,0x000a2e3c895ddb52,0x00092cc3305bc55d}, {0x00086ee8076376e7,0x000da2d9cb5f9d99,0x000f0d8aea185a30,0x00068ef462b956f8}, {0x0000f61c2096bebd,0x00044e7be11b5930,0x0000000000000000,0x0000000000000000}}, {{0x000ea58bb15fdd13,0x000d5b2585fb779d,0x000d75c3d978271c,0x000e827a5ac1a4b5}, {0x0006c6fe4d4804ee,0x000f66c09f0147df,0x000b3203f6a4c217,0x00081eeb950292c3}, {0x000da3f441776841,0x00071712f688beeb,0x0000000000000000,0x0000000000000000}}}, + {{{0x0004586de8027c8d,0x0003b7dc25b073ea,0x00049b36fccf2477,0x000032458466e794}, {0x0006f36f854043cc,0x00051e24c902d71d,0x000ec681a81ea4c2,0x00060e710d119e39}, {0x000dfa8e50e27e69,0x000bc96885ae0f44,0x0000000000000001,0x0000000000000000}}, {{0x00072cc7ee3e54e0,0x000e54b8224b0f78,0x000e5d4bd3db5696,0x000d27cc64ead37d}, {0x0009b5b2f36d2cba,0x00021210e2a45e52,0x0002c8d788fbf745,0x000440c5440be1bd}, {0x000157b392b99018,0x00045deeecc510bc,0x0000000000000001,0x0000000000000000}}}, + {{{0x0001f34441941c1e,0x000786af242ed224,0x00082ffb7bd73f71,0x00040040ee8f684b}, {0x0000b0e0a77660b9,0x00055090773cb918,0x0003b4341ff934d0,0x000efe154397d7e6}, {0x000566086597b1f2,0x00038e115dad8d73,0x0000000000000001,0x0000000000000000}}, {{0x000320279e97eed7,0x00092b44ad3c59a0,0x0009546c02d95d0b,0x000317d617644016}, {0x000b3b1278de80cf,0x000150eec20cc035,0x0004047a454911ca,0x000eb15350f140f3}, {0x000297dd664a854c,0x000c1545fd389a24,0x0000000000000000,0x0000000000000000}}}, + {{{0x00054c6d42f4ddeb,0x0001a5d46442b31b,0x0008eb97dd3c9497,0x000f35259e3e1ff6}, {0x00049058db0b2e2d,0x0006968077e0b694,0x000456e4ea6ec9f4,0x00098457796aba76}, {0x0005412cc7718336,0x000c5eb4e4306f25,0x0000000000000000,0x0000000000000000}}, {{0x000f4a91bf2060dc,0x000e9a57dadc33ce,0x00051f56adeb934d,0x000b29d22e8fe341}, {0x0000f85723b5e49a,0x000ee66b41fabf52,0x0005253bffe67611,0x000c50202f550a60}, {0x0006d250b9e49468,0x0001a2956ea13fef,0x0000000000000000,0x0000000000000000}}}, + {{{0x0000dc097ea2b524,0x000c5eb5323240e3,0x000082d33c53dd49,0x000c5d6917c692b6}, {0x000337e9d695a12b,0x00078372d602c7fd,0x000ef2985e92117b,0x0008ceacebcbefa0}, {0x00068cc3e9b4e8e1,0x000db3e3e50ee13d,0x0000000000000000,0x0000000000000000}}, {{0x000980bec0d2f5e3,0x000eaa9b062f585f,0x00083ee3b5103eda,0x000605534f1f8028}, {0x000add292d29ee4a,0x000f9ba28df95f8d,0x000d134fb5785f57,0x000f0fe162fe54b8}, {0x00047f0902bd8287,0x000de3769ce1a122,0x0000000000000000,0x0000000000000000}}}, + {{{0x0006e3d538c32c91,0x000eb774341e7141,0x0009e6b225ba2b4e,0x000824dcff742236}, {0x0004d5e3b9c67d3c,0x0006e4276dd54722,0x0001d2dcc105d46c,0x000392da4b3a8a00}, {0x000e2f3953b25248,0x0002fecbb5174b67,0x0000000000000000,0x0000000000000000}}, {{0x00008d720508826b,0x000b9b4e2e807123,0x0001bf6b1169562d,0x0006d2acb14e6841}, {0x0008cbfd3257245f,0x000189ac1c8cdf45,0x000ee493f894fd3e,0x000fa59cbf0ab5d6}, {0x000476d8e672a0c8,0x000ed9fbb78753d8,0x0000000000000000,0x0000000000000000}}}, + {{{0x00039334c1cd9788,0x0008ab0560e3e74e,0x000fa2ff6e2c62b5,0x000acd7d25b0cfee}, {0x000c456f469aee3d,0x000a862fad6476d0,0x000688f0d8d2340a,0x000b648a9494468a}, {0x000f4ed209d4d2fe,0x0008a93e7c5890e9,0x0000000000000001,0x0000000000000000}}, {{0x000cdd07e3f60721,0x0002466078437612,0x000ea1835868d6d1,0x0005ba6a85400753}, {0x000f7f252808d5c3,0x0007b45d6857ba4d,0x000d683048ddde70,0x000c759393e38c60}, {0x0000c630e919b183,0x0007209017172576,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=86 [{1,2,3,..,}]*([2^344]*G) */ + {{{0x00060ead50b81885,0x000bcef73cb6f0c1,0x00053fb7eb3e2cd2,0x00062b319bb7bc05}, {0x000a38a471706b6b,0x00046c6b42daf298,0x00005e59d404cf98,0x00048fac2e73085e}, {0x000ff0af6c53893b,0x000de9e3d8eea7a8,0x0000000000000001,0x0000000000000000}}, {{0x00034470acd5b055,0x0000c4c1af94ede9,0x0004fba6b3889b3f,0x00023ee49af80496}, {0x000f0d89fd53a3e4,0x00053cc302793fad,0x000b36dcd463b613,0x0003782e102e51fb}, {0x000c63732d6d1c6c,0x00059dc97f604bbb,0x0000000000000001,0x0000000000000000}}}, + {{{0x000db31484a7e177,0x0001740a1bc3a05f,0x000c265e95ebda07,0x000546d643b1d3c4}, {0x000c2611b9709edc,0x00015784fc807b04,0x000e5bd473ceec4e,0x0003c97fb33e58af}, {0x000d6d5327b94dc5,0x00062a914fc6dc39,0x0000000000000000,0x0000000000000000}}, {{0x000cbcf73d880ddb,0x000029df80627a67,0x00091ccf95a67d3e,0x000ec7aefd91b52d}, {0x0003aa855273ca53,0x0007213a95113157,0x000b98a49db550c0,0x000b470643affa5f}, {0x000f0628a0fa67f5,0x000d2cf906186e6b,0x0000000000000000,0x0000000000000000}}}, + {{{0x00039cff07a9f5bc,0x000b27814ba6cdbc,0x0008da0e67469efd,0x0004ccf0a198fcd4}, {0x000cbf71a6e5b71a,0x000500eabe51b9c8,0x0007908463c5cf80,0x0004962539aa0260}, {0x0001de61db956e33,0x0004f9e2c1ac338f,0x0000000000000000,0x0000000000000000}}, {{0x0005f37dc080da53,0x000c1c2a74369386,0x000025722593f4e4,0x000a4e09b626a8d5}, {0x0004b1e7f8c96db6,0x0002c5a75c187079,0x000cb91e644a3045,0x00024e7eb18af641}, {0x000fb48086d9cefa,0x000b4dd6532fa3f7,0x0000000000000001,0x0000000000000000}}}, + {{{0x0001be067e3e2f37,0x0007737f152d2c7f,0x000242f8dec8659f,0x000c7d958df47d63}, {0x0001b91c63b0acba,0x000c2c6ad3f62088,0x00099adda54c0028,0x000a3012f6937019}, {0x00014f4c499516f6,0x000da068d44cb73c,0x0000000000000001,0x0000000000000000}}, {{0x000209ec58d1b414,0x0008e876dcc7401d,0x0009323106751dc5,0x000f75f24e14fe98}, {0x000ac88f5086e5a4,0x000294dbdb4ccd6a,0x000be99edf86543f,0x0003767f48ab30e3}, {0x0007667c622dcd0b,0x000d7fd6615681d8,0x0000000000000001,0x0000000000000000}}}, + {{{0x0008c453ac10be58,0x0006d75a48dfcf2c,0x000c4944bc5042f3,0x000e7ad5c9cee1eb}, {0x000996c45d109bc9,0x000689f02d424fbf,0x00023528e926c326,0x000e84ac793d58e5}, {0x0008eeaddb5a4ed0,0x0005d31f9ea2560c,0x0000000000000000,0x0000000000000000}}, {{0x0007ba171a6ebc35,0x000d39e242ba28da,0x0000694f0cc97464,0x000d7fb496b0d2fb}, {0x000d7e1b5b66b3d8,0x00001db81788dc1a,0x0003854dd5fe4e9e,0x000f9965063e6021}, {0x0003751c74ab4631,0x00006945420e7941,0x0000000000000001,0x0000000000000000}}}, + {{{0x0005b110bb6dcc9c,0x000e9d1e6c13e60d,0x00058585159a0842,0x000b46fe443356f4}, {0x000853b25c086265,0x000ebbff20877291,0x000136ec71c4d1d0,0x00003ebeca5e79c2}, {0x000a0af3b3a96ed2,0x000f93012e330a32,0x0000000000000001,0x0000000000000000}}, {{0x00017ed67dacd4b6,0x000ceade583a567d,0x000316840b4e5703,0x000c414303d396ce}, {0x000ed6970ea480cf,0x00063761b9b1974c,0x000788f3de4383da,0x000ad07d6726e400}, {0x0003056bda545993,0x000385d3fe822ea3,0x0000000000000000,0x0000000000000000}}}, + {{{0x000285c58225166f,0x00072d2451ec99f6,0x0005efbddf5101b0,0x000dbc066e055890}, {0x0002c29985ac7cd6,0x000f1839b6caca94,0x00093cafd8d9c1ce,0x000b177ba7911d6a}, {0x00098fc762e30d5d,0x00067686b89a78b5,0x0000000000000001,0x0000000000000000}}, {{0x000dff4557a32b93,0x00037e16c6af191e,0x0007ad7362550f1a,0x000983cb772b5537}, {0x000b0746f50f2068,0x000dbb42f7ee6b8c,0x000f0cdda882070d,0x000a732384a13e83}, {0x00055b3be67dc4f2,0x0003cf20d84f4cbc,0x0000000000000001,0x0000000000000000}}}, + {{{0x00007348004e40c0,0x000448f78fb0602f,0x000c2ac8aebb2604,0x00064b78277ca03f}, {0x000eb2c6f473d278,0x000cb793a9eb1664,0x000e2b358eee9a37,0x000194f18cbc9c2c}, {0x000f6078bc87a3dc,0x0006220e93cd112d,0x0000000000000001,0x0000000000000000}}, {{0x000d1ed5d96d6d2f,0x000b72be10752f3f,0x000d1e476660c38f,0x000b1d6d9b093c35}, {0x00026d898dff773a,0x0004b445df00e4cf,0x000d1ce422c1136c,0x000db6e821b59ee0}, {0x000de6252e82511e,0x0002f481c804e41a,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=87 [{1,2,3,..,}]*([2^348]*G) */ + {{{0x000d7989af2b44ed,0x0006f91b1a9086f8,0x0002b4d5672b30f1,0x00072009919c3dae}, {0x000003b6ec0a964f,0x00012b7f4f64ce56,0x00076bfe0f0d4fbc,0x00043eb40f821444}, {0x000cbb4480332a8a,0x000f3ff375566080,0x0000000000000001,0x0000000000000000}}, {{0x00018caf35e5d712,0x0009aba53d6591e5,0x0004f1b1b50e170d,0x000f9eca3e56ed3e}, {0x000288dfe4cc67c4,0x00059c7726fa0dd0,0x000a0660a01f234a,0x000a704007db6c8f}, {0x00070b32e8366767,0x00096994810fb845,0x0000000000000000,0x0000000000000000}}}, + {{{0x00092a1897b4c0b5,0x0009027fe35612df,0x000a5105c7f9f97d,0x00021853ba021326}, {0x0005dd2cadb219c5,0x0003ab9d259b3ed4,0x000857fddadc1ebb,0x0004addab0607b4a}, {0x000ff916fbbc92a3,0x000721ee7e6c527d,0x0000000000000001,0x0000000000000000}}, {{0x000ba1dabc6f5958,0x0000087b0e564aa3,0x000b9c963dd27f6c,0x00028eca3c030970}, {0x000c23cd7a457768,0x00017b833d0834d3,0x000be25f44c50d54,0x000a153d4a6bf0d8}, {0x000e9c71da49590e,0x000d43f3a30dc247,0x0000000000000001,0x0000000000000000}}}, + {{{0x0009bd585a57c01e,0x000ee844d7078c7b,0x0006c16cb258be3e,0x000a3282f9caf55b}, {0x000e08d004fc2ebd,0x0009db3ca2d054d0,0x000ff89010ddde2b,0x0006615b9b156d6c}, {0x00000ee87cbb5e96,0x0000965826673f04,0x0000000000000000,0x0000000000000000}}, {{0x0005c973c58e4e2f,0x000bfed17990af45,0x000982806a235d03,0x0003a7e6203578b6}, {0x0002679d20d4837e,0x000fa09a67212915,0x0001e993e548aecf,0x000034e7752d33de}, {0x0008c0133b8d99be,0x0005d644443ed88b,0x0000000000000001,0x0000000000000000}}}, + {{{0x0005d09795891669,0x00055ce0c6b8f799,0x000cdfa7a67baad2,0x0009a5462a84102f}, {0x000ca5fff322b2ed,0x00016895f0238b4c,0x000e1fc27a1fc8f3,0x0007399300db4369}, {0x00016f718708ed71,0x000aa4931503fe5f,0x0000000000000001,0x0000000000000000}}, {{0x00040da9e1ff0c6b,0x00022af9967269a0,0x0005871908b86944,0x000801c88350fa73}, {0x000f680cd1b5d61c,0x000f0b415826cc63,0x0008b363474f5f7a,0x00027800e5401e93}, {0x0002305262f20f7f,0x0005e6d27bb44c56,0x0000000000000000,0x0000000000000000}}}, + {{{0x000dab714dbca0be,0x000450d634bc786d,0x000d2802c42f3afb,0x000ef6e542d9994d}, {0x000946669336b0ea,0x0006a8fe65059b68,0x000f702cce1812e5,0x00030e3c70e359c5}, {0x0009fd5b9f2069d5,0x000093f3f0186d75,0x0000000000000001,0x0000000000000000}}, {{0x0006d10e2b40858b,0x000a417e22a4fbc2,0x00013ba0de831401,0x000d41a31a86b763}, {0x000893ad0b78b7c2,0x000aacf20f1564cd,0x000bac0041297947,0x0004ac69dc2da24d}, {0x000ba071987933c2,0x0007176068dfbd75,0x0000000000000000,0x0000000000000000}}}, + {{{0x000c8b4eb4601064,0x00099a991f16e053,0x0005bcf58e425bfa,0x000b1e6fefc21bff}, {0x0005da04d7e97cf6,0x0003f93bddaedacc,0x000336130ff2741f,0x00028f428d033ae2}, {0x000b98d58f0cc054,0x000281be6796c6ea,0x0000000000000001,0x0000000000000000}}, {{0x000562d3cd261bc6,0x000c4e652831be2e,0x0007f84fc06f2ac2,0x000b078d9ca13774}, {0x000d2b248e882f5c,0x0006981dfa0231e8,0x0005f7dcfabfd673,0x000f3658f51759de}, {0x000d6a68de41452d,0x00038358c049f993,0x0000000000000000,0x0000000000000000}}}, + {{{0x000dba36a11f468b,0x000ddefecb4596c7,0x00004044ba328343,0x000e3d89658e943c}, {0x000955f5e1aeb372,0x00008650d658c0f4,0x0004309aed6ff5c1,0x0000ad875f7bb480}, {0x000c35156e670707,0x00066875cf1c6033,0x0000000000000000,0x0000000000000000}}, {{0x000bceb289713705,0x000a8a9a03fa8061,0x000cac052e91978c,0x000b61eb6bb99c20}, {0x000e50fb8f33460f,0x000ec61a887398ae,0x0007437f45e3c633,0x000fcf4c74c22971}, {0x0004691a2d9b4866,0x0009004647f4a64b,0x0000000000000000,0x0000000000000000}}}, + {{{0x0004f3cfa7e54e73,0x00076a7075c33047,0x000b446bdd4b3ee5,0x0003371a1b7efe90}, {0x00013826a3c98a14,0x000412cdba45fcf1,0x00048a44b5601caa,0x000206ebe3f76143}, {0x00050e4439f111b6,0x0008451f6bb4a9d7,0x0000000000000000,0x0000000000000000}}, {{0x000c0d59b5f2d48b,0x00029bcb29ae2863,0x0003a9a3c78e216b,0x0007856c2465c5b5}, {0x0004736d155fd956,0x0001ce6b07dfbfe0,0x0008361c3a4fa43a,0x000d0e9f03c0f19b}, {0x000e803f9b21f548,0x0002f885460ccb9d,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=88 [{1,2,3,..,}]*([2^352]*G) */ + {{{0x0006b287cbed671f,0x00071c978130d6d2,0x0007aadd881d433a,0x0004f45fb4ad7bb1}, {0x000d7b1940d6b52e,0x0002d44569722e2b,0x000de70f91dc84e7,0x000ed42546436d3f}, {0x00047e41abd1bc41,0x00010f544a7be2b8,0x0000000000000000,0x0000000000000000}}, {{0x000e82545325818a,0x0003cf3d8e5d2be2,0x0005f30317d1c986,0x00015ce098fb8ec2}, {0x000158947db8581f,0x00055d8793f3e6c3,0x000f50843a7feb50,0x0008ac153d3d8417}, {0x0004329e7248bbc3,0x000d2ffcfbcb0366,0x0000000000000000,0x0000000000000000}}}, + {{{0x000b91e88872c802,0x0009859329a6f390,0x000332091e85e0fd,0x000d0a1fc7233994}, {0x000c07172741e069,0x000870fafc953488,0x000d8073b040fb91,0x00089e841e1bbb2f}, {0x000f582161687272,0x0007bfa72dc0f548,0x0000000000000001,0x0000000000000000}}, {{0x000c2f4044695d52,0x000e9fc898f3ae4e,0x000d6d16346893df,0x000cc356cfc2a2d6}, {0x0008f9780e14adcf,0x00040c34a952a0f5,0x000bf1f1f74017fe,0x000ae85cc7e49637}, {0x0002400547db8273,0x000eafd4e119d7f7,0x0000000000000000,0x0000000000000000}}}, + {{{0x000a5355afe08a2e,0x000687a2f29baf29,0x000653058a23e11c,0x000110b2ab5abb63}, {0x000e4ead1d1b9533,0x0005d1b7b6254324,0x00074059ad5a8616,0x00090712ab62d100}, {0x000e9d5016f88f2a,0x000afeefd62c6b78,0x0000000000000000,0x0000000000000000}}, {{0x000d2d42ce0d173b,0x0009198d15289e62,0x0004baf7b535d68b,0x0008566e4a9af773}, {0x000402c278158bfd,0x000603f6310f0f5b,0x000331a366d639ea,0x0007457655beed79}, {0x0004b46175b5f4bc,0x00048f6ced012274,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b20efdb706d0d,0x000c40117c40b081,0x0000a6d9c2aec008,0x0004d3e0693270e3}, {0x000674266a5ea611,0x0001ebf62144a6af,0x0003d45ee3917e38,0x0008c35ff5d67fca}, {0x0006e79dba352604,0x00081a7e7bfed40f,0x0000000000000000,0x0000000000000000}}, {{0x00006eb8dc692380,0x000fe33343c5a20c,0x0003e67d0a53418c,0x0008959e15eb001a}, {0x000ac0ead5e7c7e4,0x0002e4162f0962e6,0x00017bb3e28513c3,0x0004317568fafb81}, {0x000912ceb3a2e303,0x000102559381740c,0x0000000000000001,0x0000000000000000}}}, + {{{0x0005f372c77f1047,0x000b3d958eb7b744,0x00069ac2a8ab1157,0x00057d2ec5015809}, {0x0007f4db5c158ff6,0x000e3fc15a71737f,0x00048e949735c5de,0x0003845758233ed6}, {0x000e91137608f198,0x000720cc72b9199c,0x0000000000000000,0x0000000000000000}}, {{0x000b050528cb006d,0x000ac29d53a71cfd,0x0009a1f2ad6eb262,0x0006c829676f56dc}, {0x0003dd6ddbfb591a,0x000c61cac801979f,0x00031b13cd6cc83a,0x0003cf1a5e5c85bc}, {0x0007cff1f95623b3,0x0004f7b2cd595d6e,0x0000000000000001,0x0000000000000000}}}, + {{{0x000699fc8b3dab7b,0x000229f393f97b3f,0x000012376dbad08a,0x00038a797638cccb}, {0x0006110a40e7e328,0x000e3d1acf08dea4,0x00003f85adfc07de,0x000fffa8d9d37eb4}, {0x000db3ca114eff2b,0x000b41dd72c79e23,0x0000000000000001,0x0000000000000000}}, {{0x000822fbbe3ad3af,0x000ddd71461cfcd7,0x000d3d8f03aedaa8,0x000f069c35282be1}, {0x0009283f776eb004,0x000746b05b9838ad,0x0002fcc85c205f31,0x000bd9143e61b0eb}, {0x0004e7f4435b3321,0x000a673088e100ce,0x0000000000000001,0x0000000000000000}}}, + {{{0x00052e132b686982,0x000419eaf166734f,0x000edb4ffc59ba11,0x0004fda13f9e0e45}, {0x0000c12226a0efde,0x00070adf716ee2a5,0x000402012f467257,0x000f2ecf2b32d94e}, {0x000205b1386ade63,0x000b3bc779c31b50,0x0000000000000001,0x0000000000000000}}, {{0x000cbbdbe0875f47,0x000a208dcda3e65d,0x00048b61b40cb945,0x0002b16480725e0f}, {0x000aa186079b7359,0x0007e306ab144462,0x0004b09effdc0e10,0x000a76ab4395ae17}, {0x0008c3ddeeefeec9,0x000216cdb5d669d6,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c70f6d4b2fef7,0x000c0c92ef06acfe,0x000790f3344c38aa,0x0000fed753c30edf}, {0x0003dfc8006501b4,0x000df722f2f6da80,0x000c340284a42e2f,0x0002a0f154005cec}, {0x00082f0dfb36ac65,0x0007bed1506b21ef,0x0000000000000000,0x0000000000000000}}, {{0x000d76b785906061,0x000edca1c3d7b884,0x000307e9a89c6050,0x000e0ccc1519baa6}, {0x000663495eff88c7,0x000a17475b22e916,0x000c39e69639f1ce,0x000b1f0e827f8c53}, {0x000066355ede8121,0x000d5b91249281eb,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=89 [{1,2,3,..,}]*([2^356]*G) */ + {{{0x000ccccf35a37229,0x000465167517203a,0x0001bf938eaa2ac7,0x000d73d683a983dd}, {0x0007f598a6f1db73,0x000235f9ed630b4f,0x000f332db784cb56,0x0003b330540f52bd}, {0x0005843b0221e5e8,0x00044a09499b4ec2,0x0000000000000000,0x0000000000000000}}, {{0x0000fb3cab0a1b02,0x0008968b6e52dc01,0x00022c046dc60a24,0x000695beae1e187a}, {0x000d3006acf49482,0x000960f10535934b,0x000df011e1d0143b,0x00085de371d84cfd}, {0x00082841456c439e,0x000585582ff3b564,0x0000000000000001,0x0000000000000000}}}, + {{{0x000e94ac6fbf17fb,0x00044289803b61bc,0x0002e798f31e8afe,0x0005e43d9a42f37b}, {0x000f377aef7a7947,0x00003a6947a8f685,0x000c1b4969c3b8c2,0x000b9c542cdbdf0d}, {0x000501682c76bc8f,0x0006972a768660ff,0x0000000000000000,0x0000000000000000}}, {{0x000f9daec5f3009b,0x000325c4a46652c8,0x000b09499ac89b1c,0x0003ccd5721c0cae}, {0x00098da46e3445e6,0x0002db691caca0b9,0x000845a793a1fc73,0x000ad927f614049e}, {0x000024bf07aea310,0x0007245359be8b80,0x0000000000000001,0x0000000000000000}}}, + {{{0x0000db1596cc9e80,0x0009a231d0f49f13,0x000f1d499d6bd9c7,0x000757ac7ea9b7cf}, {0x0005305a4d545367,0x000f2c85480a42a4,0x00029efd461a5b51,0x000fe691c9e6b8ed}, {0x00090ea1ca549541,0x00058f09c0153e64,0x0000000000000000,0x0000000000000000}}, {{0x000525f593f9a0ed,0x000edbc140a1f67f,0x000f5bef166a98aa,0x0007a559750be5c2}, {0x000fb8cba58b2d45,0x00014d93d0c5d96c,0x000723470bfa2f95,0x0004f6b79058b86d}, {0x000d58f11a8a7858,0x000fe32b2d0ad418,0x0000000000000001,0x0000000000000000}}}, + {{{0x00016f6d1a0d42ca,0x0000a7c2c062afb4,0x0001630676c3dacb,0x000c297ad74ee6b3}, {0x000d18f736e4995f,0x00064edc2548a7a2,0x00031596b5d5f53d,0x00040e945b2c8330}, {0x000587c06dfaa52c,0x00037c462a8f05b0,0x0000000000000000,0x0000000000000000}}, {{0x000cd636b4f0d870,0x000b2b0835ddc02f,0x000d233347086482,0x000bf92bc7f1c7b2}, {0x00050d5f30c92b32,0x000ce136c0491539,0x000254d29288cec9,0x000c34eb38494ac8}, {0x000ba2a1b0b3117a,0x000c473a85376a14,0x0000000000000001,0x0000000000000000}}}, + {{{0x0002a61629b67537,0x000edcfef26d3aba,0x0004bac42f2af22b,0x0004da8b32bd0514}, {0x000be474d59e6af5,0x000c190f17846ca3,0x000f3e17e7c79bfa,0x000a13543ecbaea0}, {0x000e74acd0ff996b,0x000cbde27a5f5aab,0x0000000000000000,0x0000000000000000}}, {{0x000ccc73ffeccff0,0x00082b1e746179a8,0x000b19b717d62af8,0x0005045a4e0895be}, {0x0006b8f194a8bb25,0x00089f1cd50b3736,0x000f2a57b3da3e10,0x000691e4f67468b1}, {0x0008976ca9c4602f,0x0005e53ed98ad969,0x0000000000000000,0x0000000000000000}}}, + {{{0x000f79bf523ecdd9,0x00097ebf36d74486,0x0000fe48147bf45f,0x000868d46235b3ae}, {0x00073a9b13d93d40,0x0004e9264c45fa91,0x0008c79b5705f4c3,0x0000fd4b166fd0d5}, {0x000aca2edaf4ff87,0x000955b68a488f7a,0x0000000000000000,0x0000000000000000}}, {{0x000f3e0b19951fb2,0x000c26747dd972f5,0x00092d84bc8f6fc0,0x000255f7088102b5}, {0x000c984970893201,0x000a6791707f6288,0x00072769309b54e6,0x0006389f4da5d532}, {0x000c1eb23c48b5de,0x000d1bac794b858f,0x0000000000000000,0x0000000000000000}}}, + {{{0x000e2d8dfd2d5a87,0x000e5a708c918328,0x0009b9fb00f1dbe8,0x000fe9c7a3695cb8}, {0x000749205b4caaea,0x00056f204bd6ec0b,0x0001a73a9e0254c7,0x0004152441cfd51d}, {0x0000d2b8b0ca9156,0x0004dee3cdd9e937,0x0000000000000001,0x0000000000000000}}, {{0x000ab13754dec146,0x000b5d5322d78e9d,0x0000adbfc5578b8a,0x0000ce27a2b97f9c}, {0x000b49cd573f2d5d,0x0006ee23d2e94e39,0x00061ea213bd15a0,0x000e561b9d34708d}, {0x000fb576c6271f59,0x0001669ae9450741,0x0000000000000001,0x0000000000000000}}}, + {{{0x0001a66376c54f37,0x000a06b888102e23,0x000430b0efaaecdf,0x0003b1e3d888793c}, {0x0004f8beed2dbb12,0x000ea5e72a8887df,0x000d4aa2425e9853,0x0009d98e93f3ccc7}, {0x000cba97fe918171,0x0006b08ea6eef307,0x0000000000000001,0x0000000000000000}}, {{0x00019b171f51c344,0x000d5e5d9f40be57,0x000fea96313e16ec,0x0001461efe1e359f}, {0x00043de9904f3d9f,0x00081bb7a038f6d9,0x000ed552c5787d58,0x000a67fc2cd9a74e}, {0x000643f377ccb483,0x0009db7070b5762c,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=90 [{1,2,3,..,}]*([2^360]*G) */ + {{{0x000188556b53942b,0x000736bd7c7672ca,0x000a466705820ced,0x00039ba4b83d6897}, {0x000af174ecbf7e3f,0x00003dc58b34188f,0x000b453db5dba0b2,0x000ef54df32d5206}, {0x000d08e3c52fcf51,0x0003732f551f3408,0x0000000000000000,0x0000000000000000}}, {{0x000937ade92603b2,0x000b6a7f7f5dfd6b,0x0003151876b632c9,0x0009040d3ee4a789}, {0x0007441b009fd7a5,0x0008b427fedfc2d2,0x0007f921c0ceded6,0x0002220fc8f207d5}, {0x0000f675383c79a4,0x000d6f410a2e837b,0x0000000000000001,0x0000000000000000}}}, + {{{0x000fb8b792ff9c0f,0x00062d82addc4301,0x000b9cdf1d9fdb00,0x00042255b1cf25ad}, {0x0009daaea42ebb5a,0x000dffd105199066,0x00001d688f207641,0x0001da7769bd6130}, {0x0004ea507a275aa1,0x000073ea612e43e0,0x0000000000000000,0x0000000000000000}}, {{0x000f18b4b24386fb,0x000f72268a5e0821,0x000ad126436a7554,0x000ba02f714fe1c3}, {0x00019b7c7cde45b5,0x000c576f09f2da35,0x000aef34fb328e0f,0x0001a0386e0f185f}, {0x0007a6bb32adc73d,0x000733da21be9ac9,0x0000000000000001,0x0000000000000000}}}, + {{{0x00023d540d542b80,0x0004cc500040b26a,0x000e6a09fa7f8755,0x00027fbb548aea96}, {0x00065fa1d8c060cf,0x000943cfee1a6187,0x00061bce6a8c7ea1,0x000d99730b0b20bf}, {0x000eac170744528d,0x000423d049742c42,0x0000000000000000,0x0000000000000000}}, {{0x0002bba636da345a,0x000a62e601cd801e,0x000c9e240a6cbeef,0x000103af8106469f}, {0x00007c7109e54da8,0x000b9a3ec3dcc449,0x0007788e44b6df8d,0x000d0e67c93ee34b}, {0x000e8347b4a58495,0x00037223b5096e63,0x0000000000000001,0x0000000000000000}}}, + {{{0x000417e035b970b5,0x000ca1b60364c1bf,0x000ea847f52dda37,0x000517fb28527f5c}, {0x00007a1e399f798d,0x000452c79fff102f,0x000688a87dfab3cc,0x0006490b0295c5aa}, {0x000d17acd0dbc605,0x0005acb4f6972c3d,0x0000000000000001,0x0000000000000000}}, {{0x000fa35559042635,0x0004e33a903ffa23,0x000c46f6e3526281,0x0007bc6b1cec4214}, {0x0004bca2e1dc8726,0x000b50045720b747,0x000c697f394811de,0x000ba5001f3d4304}, {0x0009ea7fd0f7a5e2,0x00090dead3d0124c,0x0000000000000000,0x0000000000000000}}}, + {{{0x000e47c23d19de00,0x0009afd475bc3cb7,0x0001acc6490ff459,0x0009f5dc39b1950f}, {0x00064d14540f1ee0,0x0001b75050e51c95,0x0005647bebd088ff,0x000f240dba4c1789}, {0x0009b95e8097400c,0x00085b5d4b842055,0x0000000000000001,0x0000000000000000}}, {{0x000986a76d06fbfb,0x000d7fc2ffb65385,0x00018e264c5a478e,0x0005a2784841d184}, {0x000fe21d9e8a017a,0x000bf52154297fe2,0x0007dad072d6d911,0x000eae77c8ea8832}, {0x000786b6a02d1fcb,0x000e682555450013,0x0000000000000000,0x0000000000000000}}}, + {{{0x000de731f9f48a0a,0x000a357753cff617,0x00073655403972b7,0x0000484d28d73a10}, {0x000c846d46c140c7,0x00055b7ef1516a9d,0x00014890b5525944,0x0005f418ade1b816}, {0x000a465f264a9164,0x000ed37693e9a176,0x0000000000000001,0x0000000000000000}}, {{0x000e5c3bfdcbca2f,0x000121dc135bc4e5,0x0003d39b5c7ca946,0x000be46855877498}, {0x000879fb5d801318,0x000afd92b1e5cb62,0x00024aecd7f80343,0x0004a3835c8434ed}, {0x00025764c6aa7d95,0x000a0241780668d3,0x0000000000000000,0x0000000000000000}}}, + {{{0x000c0928cf2280c1,0x000f2c37933b1734,0x0006bae2a2974a56,0x0001e8bdb1d26ac8}, {0x0009c84c336cb6bd,0x000ca41014cdaa1b,0x00024b87838c44fa,0x000f525239cae2ce}, {0x000cb0507515f204,0x000993dbd0e0a58b,0x0000000000000001,0x0000000000000000}}, {{0x0001411bdc3926ce,0x00087f3e15aa5363,0x000ade47bf68672c,0x000028e493da50d5}, {0x000120048f8cd148,0x0005ecfaeb03c756,0x000e1347b7867aab,0x000ba0208953afcd}, {0x000be9b23e2411e3,0x000a2e848d40b424,0x0000000000000001,0x0000000000000000}}}, + {{{0x000583ec08d4ad28,0x0007687b7ba7d916,0x0002b3f0b4e2bbb4,0x0002caace0e4b3fb}, {0x000b0e6fb63a6917,0x00000520c822aab4,0x000f41f7930e37aa,0x00046bfa91da4dcb}, {0x0008bd604f521a69,0x00040fa707c1f0b8,0x0000000000000001,0x0000000000000000}}, {{0x000520b880d23952,0x000bb822333018d8,0x000aa6a00bca6bc2,0x000f3469011553af}, {0x000c20ed5fc0a5de,0x000ee0e8c5bcfec7,0x000476e2f464224d,0x0002d844542e8adb}, {0x00009924fd3c1bdb,0x000f2fac98d161a7,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=91 [{1,2,3,..,}]*([2^364]*G) */ + {{{0x000c9a655367407c,0x00001acf48b04d30,0x00004344830b68ea,0x00058a53174d6fa7}, {0x000f59044eeb31ac,0x00087d51a60524d6,0x000a344fb882d4df,0x000d1ed41d08aa0b}, {0x00086b6aea85fb93,0x0007f27fa57f4860,0x0000000000000001,0x0000000000000000}}, {{0x0006f6fa7b7febd7,0x000ef92aae956259,0x000abc183c404813,0x00011be4ded30d2a}, {0x000e220b7ae966a0,0x000c3e6cfc88e77b,0x000a92b77d5e0cab,0x0003d7f99c6dac06}, {0x0000a6be4c76c302,0x00032d1d55150da8,0x0000000000000001,0x0000000000000000}}}, + {{{0x0000d37f0300cf42,0x000564d1a1ebfaea,0x000bce4cf04b07ea,0x00084f2b4677d784}, {0x000db14a4f867741,0x0000b95ce93b8741,0x000a31735b5960b0,0x0003d2c80a76fae4}, {0x00022c4d123107ec,0x0006cd8678a9d705,0x0000000000000001,0x0000000000000000}}, {{0x0004b58dcaec13d9,0x00067d88c3d5f230,0x000f847248f45f52,0x0003da2628ef4e85}, {0x000e37945a7b9c0f,0x000a2da387ea2c17,0x0008e98e84de9888,0x00038290c88f211a}, {0x0004ce3667557434,0x000040ca4612f56b,0x0000000000000000,0x0000000000000000}}}, + {{{0x0009e872c584974d,0x0006ca75132450c3,0x0006c420ca2dafbb,0x00069da0e632e68f}, {0x00018e1d45c5a963,0x0000394fa7a8601f,0x00098adcea6c9852,0x0001b23e3c6dad95}, {0x000f0655b2b99628,0x0001992529d81db4,0x0000000000000000,0x0000000000000000}}, {{0x00000b6625e14e8c,0x0006611065dd0a46,0x0000a833140f2868,0x000735d82490f09d}, {0x000326d182482735,0x00074f69a678ac02,0x000e336a3425366d,0x0000093a8f215358}, {0x000644f6ddf3569b,0x000d19beff776a6e,0x0000000000000000,0x0000000000000000}}}, + {{{0x0007a73bd7975b73,0x0004ba1e3ef4b56c,0x000835871e0104fc,0x000ed4624759b57a}, {0x000eed3c95d4d9b4,0x00029d8353648a71,0x0006bedece81ad28,0x0001452c12f2b2a5}, {0x000ab19b8b67ec3e,0x000ccd3f8f88bf35,0x0000000000000000,0x0000000000000000}}, {{0x000062e0d5c7f0b6,0x000dd34abff69676,0x0009b89962a6641c,0x0002be1c0add12e1}, {0x00014a078191a9f4,0x000c488cf972d9da,0x00090e9607f65fc7,0x000d5cdadd7da7e7}, {0x0001ca37f83b3584,0x000dd43c6df02d38,0x0000000000000000,0x0000000000000000}}}, + {{{0x0003f8ca58799292,0x000601d1cbef0700,0x000f41308b0cefb1,0x00090de4387a468e}, {0x00035f90ae8d18bb,0x000b356306c0a768,0x0005e167044866de,0x000114237a5a47dc}, {0x000143e3b4bbc084,0x000c1c6d0277c186,0x0000000000000000,0x0000000000000000}}, {{0x000c81887bc12544,0x0006af4b6c2a8e8c,0x000ba5958fe82cbe,0x00067cd479340299}, {0x000d360f1809e7e6,0x000de77ee94fcc0f,0x0009d25d201341ff,0x000fac2af6e20977}, {0x000dfea19f8974a2,0x000fba7a5252c712,0x0000000000000001,0x0000000000000000}}}, + {{{0x0001e6367ded4905,0x000fd6fd8b41173a,0x0000c717ef3cdcf0,0x000608fdb3300d01}, {0x0000d527d7c8e07e,0x00039cd9ece69c0a,0x000677211bdaf48d,0x000b5d520c7fa557}, {0x000bf842692f3c61,0x000055814bffe31f,0x0000000000000001,0x0000000000000000}}, {{0x000c94502a0e0f49,0x0000528193bb953d,0x000d7bdda5ab1a23,0x0007c4a219650b25}, {0x0000f78abb7ba4c6,0x000eb157bdb9dbe1,0x000ace6b3d0ff943,0x00048180c4dc1a32}, {0x0000124b69e9b36a,0x000da7fee72796eb,0x0000000000000001,0x0000000000000000}}}, + {{{0x000530dbdddc111e,0x0004131a0423e417,0x0002a2fcd4c890b5,0x000685a6ffdb8021}, {0x0001c68bcc7cf314,0x000fb29a153281e3,0x00090775f2e2a729,0x000fe4fc20716627}, {0x0005fa8915460a5d,0x000b7744dbded762,0x0000000000000001,0x0000000000000000}}, {{0x00058330ded93a3e,0x0002ad9ca2943c52,0x00031d21ab8030d0,0x000fe30e76f64897}, {0x0002b30fd418c647,0x000815868a20f82e,0x00098d35bf8cbd5b,0x000facccde412b85}, {0x000358ff02bb4ec8,0x000fa54fb1673bc1,0x0000000000000001,0x0000000000000000}}}, + {{{0x000aed08c7bd3b0c,0x0004e546195fa3ed,0x000c31c13efbcb9e,0x000f2eaeb2cc8a6a}, {0x000a1912ca200483,0x000f0ff27a5ee60f,0x0000a7b9e9c56cff,0x00044977503cdac7}, {0x00064deabbda5a3e,0x00038efe3a9fcba5,0x0000000000000001,0x0000000000000000}}, {{0x000821113784eeb7,0x000a12560a5e577e,0x000ae4b9aaf4ec38,0x0005c9a38358d926}, {0x000497b69c24b1cb,0x00007485410e5464,0x00026fb660a2d50c,0x000987263ee5a4c4}, {0x000b3ba20c286e0b,0x000ae54bed6c50f7,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=92 [{1,2,3,..,}]*([2^368]*G) */ + {{{0x000e2418ebe8f2ee,0x000f4560bac026d4,0x00008c6a85ee3153,0x000c7d7d8e05a0fb}, {0x000d35867d053abe,0x000ebaaa06ca6918,0x00022207d8627f01,0x000fdfe74b9c6ea9}, {0x000478deb27dc332,0x0006b633ddba7b54,0x0000000000000000,0x0000000000000000}}, {{0x000eb3b84da8ae44,0x000dced254321c2f,0x000ae0be12cbd92c,0x000b5fae91edd7e2}, {0x000adacef448565a,0x0003f288c1607c22,0x000ea8d01e22b70e,0x0004e3598c73a3ba}, {0x0009cd9f6c24e3c9,0x0007a5595791d3f8,0x0000000000000000,0x0000000000000000}}}, + {{{0x000cd049d3a9026e,0x000dbe859af0b3a1,0x000d9aa6b9632b70,0x00029dc483656cba}, {0x0007d02bc7ba1a52,0x000fc68a06574b48,0x000470a9518ff35f,0x000aaf20c720ad36}, {0x0003bb49ecf8b908,0x000f66f8b9d88aee,0x0000000000000001,0x0000000000000000}}, {{0x0004ae92aaca41ff,0x0009e2ff799aa5c0,0x00048de6d0a352ca,0x0008f5f2eb0f3051}, {0x000fea98f1062e2b,0x000285eca4dfe726,0x00044d322419400c,0x0001441ba1f95272}, {0x000c0f6113ec0c84,0x000b67f7b093769a,0x0000000000000001,0x0000000000000000}}}, + {{{0x00010c16516e4d3c,0x000611e277c39f9f,0x00040673dd719aec,0x000e007e471514eb}, {0x0006cb0a884f04c5,0x0007cdfc977e1e7d,0x00096fd19b101b5e,0x0008b661589b4413}, {0x0006ee58455ad5da,0x00030384dfd6a666,0x0000000000000001,0x0000000000000000}}, {{0x000dcee7cc09a195,0x000f049d8452fc3f,0x000453637f0d8b3f,0x0001dc43d12fc712}, {0x000f4b97aa7b885d,0x00030970db43c87b,0x000015eb8214e6b6,0x0006a5744b5ac5b8}, {0x0008d3fab8987808,0x0008438a227d82a5,0x0000000000000000,0x0000000000000000}}}, + {{{0x0002d58e6de7c70e,0x000a184fd2b399ce,0x000d46ffafde56a0,0x0003266443a772e3}, {0x0009e5e99ec73618,0x00068acc975a652a,0x000b99dda22ced10,0x000f17534159829e}, {0x000ab0176c94c616,0x000fbda334609df6,0x0000000000000001,0x0000000000000000}}, {{0x000e586ebac6018e,0x000f2f03144a03f0,0x00070d82d13df49e,0x000fad35f054795a}, {0x000bfca4e83c93d4,0x000ccd2e817178dc,0x00006d906f96d5dd,0x000999860a4c0599}, {0x0007c4473b0cc898,0x0002a9c7a2422f0b,0x0000000000000001,0x0000000000000000}}}, + {{{0x0002b09542f251bc,0x000e9c6a4a88693c,0x000ca160d5142fa0,0x000a8912377d1615}, {0x000e8a6efbddaae6,0x000a4c058235658a,0x000f27a6a6dd7c56,0x00070769ea7bd61f}, {0x0006f5dde4e365fe,0x000d56d5c0ff87a3,0x0000000000000000,0x0000000000000000}}, {{0x000ceae2e0074d51,0x0000c081ba44424f,0x000d9e0eec434166,0x000bbc85793c6ecf}, {0x0000cfdda19dc769,0x0009ff9b44e244b1,0x00055190f00a8e3e,0x000a30e6f1e94105}, {0x0002df24f630f9cd,0x000164028859bc1d,0x0000000000000001,0x0000000000000000}}}, + {{{0x00091d8b229586b3,0x00062d4212a9d0d7,0x000a44dcf82dea37,0x000886a8066a0e31}, {0x000c8d5c7f9428cd,0x000e7fa7c681f5e4,0x000db54aa6ccbecd,0x000e8442c468c6ab}, {0x000eb0d35eb4fd66,0x000ecc62bfda1f45,0x0000000000000000,0x0000000000000000}}, {{0x00043fbb4e2cb5f4,0x000b78f0a96488d5,0x0000f4585ef033e7,0x000ccf9f3f7c9386}, {0x0002c56f736843a4,0x0003c2a98300d2f8,0x000d0e9651445c74,0x0006ad9774234178}, {0x000b0d7d2ff89fee,0x0002dc8f018f3a2c,0x0000000000000001,0x0000000000000000}}}, + {{{0x000a0b1e42fb1d75,0x0005875ba8ec633c,0x0009652adf59ccbb,0x000c9a8c4da98461}, {0x000058421fa35715,0x000861494cf70c21,0x0001a367a4a6e22b,0x000f1f29e364cf7f}, {0x0002eb8412063103,0x000046241f101761,0x0000000000000001,0x0000000000000000}}, {{0x00035295fd113dfe,0x00087ca26d83dc56,0x00009b60485e1379,0x0006992ee53da791}, {0x0003f63e81d304b9,0x0005e83c82169ed5,0x000cdfc2181cd77e,0x0002864256eb4e4a}, {0x000dd24e836cab6d,0x0000560017a3ed5b,0x0000000000000000,0x0000000000000000}}}, + {{{0x0000d40ea7dbd218,0x0000bffd292d5f99,0x0000b3c033efe2aa,0x0003caf5350ffa07}, {0x00062cba18d05709,0x000de1ef348e77aa,0x00050628dcafce95,0x000654c13b978d30}, {0x0004b7581a218420,0x000aebc1eed7a302,0x0000000000000000,0x0000000000000000}}, {{0x000467c3cff7787c,0x000cc65919f6e7d2,0x000e4ef4ee66f3a2,0x0005339dd95dc335}, {0x00083538624188b1,0x000c9f6ee9c47f71,0x000c2d00164075ad,0x0000fb8c9b9b8fc3}, {0x000ab425082f15ec,0x0003706da80b242c,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=93 [{1,2,3,..,}]*([2^372]*G) */ + {{{0x000981c54aaf54da,0x000345cebb5e6933,0x000544b1b1632546,0x0001b01f84cafbc6}, {0x000115cddc181798,0x000378ad86aa1393,0x00075fe68cbb4941,0x0009588ce3ac7e26}, {0x000708d627f2694e,0x000a9deda381ddab,0x0000000000000001,0x0000000000000000}}, {{0x000d5f56fe3c020f,0x00064c1a13df6f31,0x00002f2a54ccdb45,0x00018f47586ae362}, {0x000f6db9ebb1f191,0x000b71b6517fa3e3,0x000f8c282c695b00,0x000758306aa882ec}, {0x000bb71fac8a72bb,0x0005351671f4f924,0x0000000000000001,0x0000000000000000}}}, + {{{0x0008318350691a0c,0x000b269d0103c9cb,0x000bca486ca47a18,0x0001af062d151fe9}, {0x000e7c2bcd6c38a3,0x000a2dd4944c38da,0x000d5e2d0cab5f56,0x0001aaffb6b80e8d}, {0x000b4ab87f1aa045,0x000baad7ec926bd1,0x0000000000000000,0x0000000000000000}}, {{0x0001499620b24214,0x0007f0d276179361,0x0005f42f5e14dbe6,0x000a3745bdaa4d19}, {0x000b02770c1ff9ae,0x00030746fe336a10,0x0005a19965986ef7,0x000598b61ae7500d}, {0x000c2b7c92c56a5b,0x0007c0f20ece26fb,0x0000000000000000,0x0000000000000000}}}, + {{{0x00087e69d72ded5b,0x000cbed0493c7849,0x00059557a83ba0a5,0x000cabc475bbdb17}, {0x0003d65e4a623f2a,0x00071fb7df8bf3c5,0x000f576c8eb2466d,0x0003c6d8140f7545}, {0x00002bd4e620a012,0x000cb967837a469f,0x0000000000000001,0x0000000000000000}}, {{0x000871f9216cfc43,0x000a4dc25382488b,0x000ef93af7d4b3a3,0x0003ed77dbf730c3}, {0x000a6f764526ebfd,0x000b6152b407f935,0x00025711c016476c,0x0003dee3ad5a2bf3}, {0x00009ed5688aaec7,0x0007cb5a614fc9fb,0x0000000000000001,0x0000000000000000}}}, + {{{0x000d2983eb57fb01,0x0006f8090fc8d49a,0x00055cd8ed43313e,0x000063128651d168}, {0x0009d55315e356a9,0x0003731bdb591a91,0x00090f35172884d8,0x000ce2ac6b9b209c}, {0x000e9deccfbd125c,0x0004c67d19839d92,0x0000000000000000,0x0000000000000000}}, {{0x0007835fb081749a,0x00038c29318405ed,0x000c88969fe7858c,0x000d9e6b39c13839}, {0x0001a193b85889e3,0x0000167b73a0e542,0x000e3a6c6ebad78e,0x000fea5061213cbb}, {0x000f99af7a8cbcf0,0x000934dbdf82d320,0x0000000000000001,0x0000000000000000}}}, + {{{0x000dd97cee516977,0x000fc49266fa93ba,0x000b3371de037896,0x000f3467c9ec4ba4}, {0x000bff13f9988671,0x000c0500963fcb88,0x0009447add0b56dd,0x0005dc495e382a32}, {0x000c531e8e3d8f13,0x0003f8ab53419d64,0x0000000000000001,0x0000000000000000}}, {{0x0005999b06e01bce,0x0004ac4c86e9d1b3,0x0001acac55379f5c,0x000ed5f60cf793db}, {0x000deecc778efaf8,0x0005438ff64587a9,0x000da09df4f7f063,0x00095b550b56ea7d}, {0x000d4a2f2118371b,0x000897f5ee846337,0x0000000000000001,0x0000000000000000}}}, + {{{0x000e73c876ead4f8,0x000074f77e1ef006,0x00068e9d113a8d10,0x00068c4c79701512}, {0x000c8af63c9e4975,0x0001a355ba1ddb87,0x000a7f69e6cd38b6,0x0001a9ae068bd113}, {0x000bc3a633214952,0x000e40b4f3b0cfc9,0x0000000000000001,0x0000000000000000}}, {{0x000dc3573f4bb5e4,0x000ce2ef087b7068,0x000ffeeffd386368,0x000053bd77e46931}, {0x0008c8fd63c43732,0x0009915ece7b6a54,0x0009d1980b122996,0x0000807d3a509f62}, {0x000abb80ee8c76fd,0x000ea6d20af82699,0x0000000000000001,0x0000000000000000}}}, + {{{0x00023cc70a1b7646,0x000fb2fea8f69076,0x00051b6f61d27313,0x00053b59960e18ea}, {0x0003b52c79117607,0x0003c4870ae1dead,0x000b0ba34bd1f16b,0x000eb6ec0bc7a291}, {0x000e48716fbdf694,0x000f3b9ceb0fbb83,0x0000000000000001,0x0000000000000000}}, {{0x000f4df150c14682,0x0004d73e71ecfad2,0x0001f4eed11e2bf5,0x000e8f7ba0e6130a}, {0x000d0ee611881177,0x0004b3748fb1ee0f,0x00018bc0f1b2681f,0x000d95b14d7b81f4}, {0x0005715bb9df9123,0x00010458047bf5c1,0x0000000000000001,0x0000000000000000}}}, + {{{0x0007c0378bc40ed7,0x000325851811caa9,0x0000425fd5a042ad,0x00083181ae223e37}, {0x000dcf721e5a193b,0x000e3457f090e949,0x000bab83df6a8520,0x00045f22447cb654}, {0x0000dfbcc720ad35,0x00030064eddb51ec,0x0000000000000001,0x0000000000000000}}, {{0x0007e2d48a4ebf78,0x00086d5a22dda9a9,0x0004ad7ed63a6603,0x000bdfd6f4eac86c}, {0x0006ea0b3cfe90e9,0x0009746e43f3d057,0x000247c38598edeb,0x00066c63f53c5a4f}, {0x0007de7ce9110d7d,0x000580edc5628f93,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=94 [{1,2,3,..,}]*([2^376]*G) */ + {{{0x000ceb75a39240f6,0x000cd422d17ed307,0x000816d46db1d003,0x0000a452acb6fef8}, {0x000cb9bbe93acb00,0x000e7044e1a33425,0x000fc32d94105ed8,0x00077b448d72bf04}, {0x000527b278a8006d,0x0003f3ce1c27af77,0x0000000000000001,0x0000000000000000}}, {{0x000e21a0404fea41,0x000ba2bea8a562ed,0x0007dcaa390fe905,0x000e3be58bd01814}, {0x00024ad37906a8c2,0x0004147c934bd6ba,0x0008700ab35993ee,0x0003eb32c19654b1}, {0x000e9fb6dc4b6283,0x000fffefce4982fd,0x0000000000000000,0x0000000000000000}}}, + {{{0x0008ddce89d919a0,0x000c8653953842cb,0x000510be22ec94ea,0x0004fc6818af52b2}, {0x000d36cd384c6520,0x0000918e38b08bd4,0x0008cd3bbca8f664,0x000f9b3d5866c2ac}, {0x00015b5a22e4fdae,0x00028fcebfa696bd,0x0000000000000001,0x0000000000000000}}, {{0x00086becff5dbeff,0x0002b0a9f4f0a089,0x0002781857ab5bde,0x000f7d34f623a384}, {0x000ab2fc32d5df48,0x000357b29a5aed2e,0x0002a4f85d8000c3,0x000a47c091a8d7a0}, {0x0008c875883e2289,0x00004c78f3991d74,0x0000000000000000,0x0000000000000000}}}, + {{{0x0006929bd70c6394,0x00085f2018c22473,0x0004be72094183d0,0x00086c43b8504d5a}, {0x00086b6f18e21a1b,0x000ad297dd4ff469,0x0000f368a9142ac6,0x000e09fe86beaf09}, {0x0008c552b3b6921a,0x000bce953006e93c,0x0000000000000000,0x0000000000000000}}, {{0x00072ac6e0006309,0x00015780956c6baf,0x000c4c2c5f7fa741,0x0001eba55e9870c9}, {0x00060f57cfca17ec,0x0006e490ccc10b4b,0x0008a9db0618cc6a,0x000131fe5f0039e9}, {0x000970dbcf0f5361,0x000700a442baa6b1,0x0000000000000001,0x0000000000000000}}}, + {{{0x0003ec842981eb23,0x0008fb16678799e5,0x000db8c26f2eb3f7,0x000307e41091298b}, {0x000c2265d3b22e09,0x000829161a79682b,0x00094108a62536ff,0x0002002fb6dedea9}, {0x00079fdbc3d369ec,0x0002baf58b2f20ca,0x0000000000000000,0x0000000000000000}}, {{0x00056d9fea698757,0x00041a61419646f7,0x000308b017c99346,0x0002b76a5de627fc}, {0x000cb23ee7d29bea,0x000ab47900ba8603,0x00096bb85c794766,0x000df41684cc004b}, {0x0007656ed94d547e,0x000e302003142b8b,0x0000000000000000,0x0000000000000000}}}, + {{{0x0005cdf0222a69bc,0x0005e1e346fabbc8,0x000e07629de2b094,0x0005724f76a1e2b5}, {0x0006b43bc885c45e,0x000d1f53506f8c50,0x000458ec4a247aeb,0x000ee8c49a8e9759}, {0x000961f24ad9f81f,0x000c780789ce81ef,0x0000000000000000,0x0000000000000000}}, {{0x000ac3a5a536c8ac,0x000fe30d120ebbbd,0x00029a29c912f38c,0x000d27e5470f8673}, {0x000f785f54b6aa93,0x00069bc2c6347ce7,0x00091c1681c6e838,0x00015f8951322402}, {0x00054c132d778a68,0x0009133565718293,0x0000000000000001,0x0000000000000000}}}, + {{{0x000f8e71a3d24acd,0x000de7f8ccaa4039,0x0004a565eb389f9f,0x0001445b3a88c7c8}, {0x000b1b88e20b6224,0x00022d8db00479b1,0x000369096695cdce,0x000d48a7013202fe}, {0x000e3713baba6a66,0x0009b65be868e0af,0x0000000000000000,0x0000000000000000}}, {{0x00018718a6375d71,0x0003fe3e38b8c6c5,0x000ee16d3bd00a61,0x0001a73a8bab2dac}, {0x0001decd0dde75f5,0x0009a19d5d5d598b,0x0002f5dcc2ed8e1f,0x0007b66c768674ed}, {0x000717b781a03645,0x000cd5eb14d9fd45,0x0000000000000001,0x0000000000000000}}}, + {{{0x0009a4c40a33f00c,0x000d2d3bef2c63e1,0x000922215ae57c68,0x000763ee03e85348}, {0x000337a4a0d2cb54,0x00047d23204381fb,0x000be443d9712227,0x0007d96627f6c828}, {0x000fb6b5f6d1199b,0x000a433d2170b3ab,0x0000000000000001,0x0000000000000000}}, {{0x0001d3366971bb69,0x0007a38c946a2ebb,0x000b29fb103a111d,0x00047d3675997def}, {0x000fd82824e10a96,0x00029d6d05a45fde,0x000500fa9b94f37f,0x0002317e08c1e35c}, {0x0006ed9214c60102,0x000ee0a2afcd4a2a,0x0000000000000001,0x0000000000000000}}}, + {{{0x0003291f02ce62ff,0x0005b8a731176ae1,0x00053e5b4c8c2f09,0x000d5a2322d8f01b}, {0x00092f09c90539b8,0x000c4c22646cfad1,0x0008df5016016f3a,0x0002500c56f4c2d4}, {0x0006618c9bed5731,0x000f52249d380720,0x0000000000000000,0x0000000000000000}}, {{0x00052bb2164b93c6,0x00009ba854f0dba7,0x0002bd9fbffad821,0x0002cee339d0e928}, {0x000515cfc63fee61,0x0008541bf33aca9e,0x0001f0f0fd5f8231,0x0008dccc44f51df1}, {0x000e26d92e5d7f0f,0x000334dc204c43c8,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=95 [{1,2,3,..,}]*([2^380]*G) */ + {{{0x000df775a4cea698,0x00057fa877dfb590,0x000f628f95d0e8c6,0x0004b622c58775b2}, {0x000e755966c521f9,0x000b826bcad94ba3,0x000585e536e18362,0x00056a8bf64e1429}, {0x000c0d065ab9cff4,0x000c6b7ad254f1fd,0x0000000000000001,0x0000000000000000}}, {{0x000be22413e6824f,0x0009f5a869cd6010,0x000399cde94b5cc4,0x000c96f6029dfb84}, {0x00068c7d0822053b,0x0006cb5f4bf3d33d,0x00090bad70bd72fb,0x000c5f85b78281e7}, {0x000fbd3aad6b87dd,0x00067e9bddab2fd6,0x0000000000000000,0x0000000000000000}}}, + {{{0x000f0963f658551e,0x0002ea1215b91acf,0x000276c4b8c7ce3e,0x0002c599ae4d76fd}, {0x00006cf3a3b9f27c,0x00006667b3985a81,0x00095ab3dd5545f7,0x000e9ae8ea63e5bf}, {0x0008eb3015a494d9,0x00014b36df8e2aac,0x0000000000000000,0x0000000000000000}}, {{0x0001e0605f96eb43,0x000d54ec384ae024,0x000bfb3e2ee19fc3,0x000c041bce0a2d7c}, {0x000b57e0aa0d1a5a,0x000f6adf1022b978,0x000452550508726c,0x000b77d6d81e1380}, {0x000536c9802fbac9,0x0007d16666d2c234,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d75964c31ca58,0x0004e22c167fba0f,0x0006865896879fb3,0x00085f113d2ac14b}, {0x000fcf92650326bb,0x0009815c6ae567c4,0x000703330478f2d0,0x000c2d4e045bb2da}, {0x0004917027450186,0x0001cb3d6ff5bbeb,0x0000000000000000,0x0000000000000000}}, {{0x0006aee5ed6230d6,0x0006f57fa7f974a9,0x000d445b19954b86,0x000cc41b7edd540d}, {0x0002f6672b9eada3,0x0005adb45c3c302e,0x00085355ea3de1bf,0x000a70efd3fa201f}, {0x0002e28049bc11d2,0x000a9e4d97e0553d,0x0000000000000000,0x0000000000000000}}}, + {{{0x0005aaa83e0b7193,0x000ef4020dd38cdb,0x0000a2db89cf16c4,0x0008b727a5cd426b}, {0x000baf5617c8e43e,0x00043d6e5823ddc0,0x000180e259e17f2b,0x000506737413c826}, {0x0004e741255f63ef,0x00009623e6163c43,0x0000000000000001,0x0000000000000000}}, {{0x00095e5ae0c64c88,0x0006e547505a1996,0x00074ec16e26e1e3,0x0001814a43d8b0e2}, {0x0004c037ed439483,0x00075672e85b1a10,0x00064a05bc4b4563,0x0004f4e8604b9fdc}, {0x000d8d54cdb5b099,0x0008d7035e5850a1,0x0000000000000000,0x0000000000000000}}}, + {{{0x0006358cac1e1dd9,0x000326ae97ec9d6b,0x00096931bf3f89cd,0x0008a8a20db33ff8}, {0x000f15df6988b172,0x0009cd5efc8b413b,0x000bb187876052fc,0x0004662d8014980d}, {0x000d9235f7d44d41,0x000edff0c8921456,0x0000000000000000,0x0000000000000000}}, {{0x0002553d46a6bdb6,0x000a25d43349dd7d,0x00098c5095dc275f,0x000e81697e9d6a23}, {0x00021486070955da,0x000de66e5e004d62,0x00061fc887538530,0x000b0cbeddeb407e}, {0x000968acbb1e576a,0x000ee36df6504685,0x0000000000000000,0x0000000000000000}}}, + {{{0x0001376b4df2834d,0x0009122c927fc01d,0x000cb3e3200f4b1b,0x00077db8d6a633a1}, {0x000324c991410544,0x0009a4d4ddbf0c1c,0x0004f89da008df0a,0x000df68e550730c0}, {0x000e5c51e1a10f51,0x0008da410315475c,0x0000000000000001,0x0000000000000000}}, {{0x000c76b031581137,0x000f3f4ca12b9bde,0x0004e3a329753a8b,0x000499c86ef86a89}, {0x000c838a372fcc5a,0x000ec44e4a97d666,0x0000ea241b991236,0x0001650c8dbf9560}, {0x0007627fd4eb71cc,0x00078654f79c844c,0x0000000000000000,0x0000000000000000}}}, + {{{0x000ce7225c38fca0,0x000cf6f1a6e96969,0x0009cc6268e77785,0x000308efc7d8303c}, {0x0008a6b0e52762b8,0x0004bf9968cba9dc,0x000db7c416fd26fa,0x0000e7d932fa8c4c}, {0x0002063b5de7df0b,0x000a1bd8e36d9447,0x0000000000000000,0x0000000000000000}}, {{0x0002f11b9d88f945,0x000b6a528d0c6d85,0x0000491c222e34eb,0x0005246f572cf3b4}, {0x000826a507a97323,0x00051b49545419c4,0x000f246a45d14681,0x0008555cac591e9f}, {0x00067c66c74cf909,0x000c814d10852a3f,0x0000000000000001,0x0000000000000000}}}, + {{{0x0004d6495109aae5,0x000c4b86a81e7f4d,0x0004316eb1054df2,0x0006877c19b90005}, {0x0007963ac12d941b,0x000bdc46a9dbe383,0x000befbe68280f87,0x00071d97d1dc04af}, {0x000aabbcc64f8fe9,0x0008543ef9d7ecfd,0x0000000000000001,0x0000000000000000}}, {{0x00056ebb21998e32,0x000ab05f744a3f42,0x000c6587a4d462bb,0x000c14a18de305cc}, {0x00056d2c0e8c5f7c,0x0007f552fd1c2fa4,0x000165b93096db0d,0x0003eef9e935df5d}, {0x00013440e0a7d4ef,0x00020c2ecbc3b683,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=96 [{1,2,3,..,}]*([2^384]*G) */ + {{{0x0003ac5243ed86b5,0x00075f9805e79dc4,0x0002bee2dfe95a21,0x000284b06125c31c}, {0x000a0103195082b6,0x000cedfa4a2264eb,0x000afa325bc143e8,0x000ae3ae24853199}, {0x000ebe96963067c6,0x00096c54a7cecdeb,0x0000000000000000,0x0000000000000000}}, {{0x000e3a5229434e36,0x00055f3a1a50446d,0x000644f2db4f7215,0x000ad43f6dc38924}, {0x0002239beb126b72,0x000840de05e7dd77,0x0002862d6caacd0d,0x000bce6fa639c67a}, {0x000087602ba53021,0x000679ec9b598271,0x0000000000000000,0x0000000000000000}}}, + {{{0x0005ed4dab9a7e9f,0x0006797aaab2f5b7,0x000301593051ee37,0x000962a30b02f44a}, {0x000a1d622cf13021,0x000a7dfa0555a3ee,0x0006aca4fcd685c9,0x000ad2e750777a2a}, {0x0006aa905bd4c914,0x0008ffeec52d7b1e,0x0000000000000001,0x0000000000000000}}, {{0x00095204a3f6fa1e,0x000c34539f85e4fa,0x000e8ddc16e36eee,0x00044a9e74599d1c}, {0x0007ab343c6c5502,0x00007951ae714c01,0x0005c8c4503f92db,0x0006830499e544d1}, {0x000188a7b94680ff,0x000147d6c4809fe7,0x0000000000000000,0x0000000000000000}}}, + {{{0x000e6ff25ab97af1,0x000ec754ac7f57b5,0x000607a92403e802,0x0000eca9aece9592}, {0x00080834b57b8bd9,0x000e2fcd37a127de,0x000ed63155d3ec1e,0x000f99b11a3b9b54}, {0x000db302a147172f,0x0005d9290b10d36a,0x0000000000000000,0x0000000000000000}}, {{0x000eee6089f16e1a,0x000772f210e83cea,0x0002496230bbfc3c,0x000c1fbbcea01caa}, {0x00032cc99a5937d0,0x00074603f91be511,0x0003cfecb00c641e,0x000ef833255bc0f0}, {0x00085cab2f6311c8,0x000c7fcc61e9c871,0x0000000000000000,0x0000000000000000}}}, + {{{0x000f147189a2546b,0x00070bf89fcfd41b,0x000a403ed89c0790,0x0003f861a107b324}, {0x0009b4dade6e318c,0x00032b6327665e9f,0x00016ecb408e3b33,0x000c11ee2181f62f}, {0x000ba590467bbd1b,0x000a0f4b5440b9d0,0x0000000000000001,0x0000000000000000}}, {{0x000b1aa7ade86660,0x000d1a8a32a33d9c,0x0009ae722d5edb96,0x0004c7770654bd1b}, {0x0001d03d0e5a5166,0x000b01ee816a4a63,0x00015843d1d9344f,0x000c1e1821b3c769}, {0x000c22520be2d285,0x000fe7e0834d6faf,0x0000000000000001,0x0000000000000000}}}, + {{{0x000a30d8168852db,0x0007853d06621fd0,0x0000e561a31ba03d,0x000e2351f7abbee6}, {0x000bfa802185d2d6,0x000a8cb11d19a8eb,0x000b4f2b7623f602,0x000f149b3db4a55d}, {0x000399a66e1d6eee,0x000dded7de613b1a,0x0000000000000000,0x0000000000000000}}, {{0x000694da4aed0861,0x00013409999d11ea,0x00021cefe8e175cf,0x000ca47b4e5a0420}, {0x0008274d934ac552,0x000e270720e741b5,0x0003c0a8bd69787c,0x000e75f9cc756f74}, {0x0003e654e1ea2208,0x00040979701ca5a9,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d12f4d256a679,0x000fb6f240706408,0x0001c6799f2d7255,0x0002b1c7ad7d86d9}, {0x000c6259fb2898c3,0x000dc9f2eb172083,0x000f61ec85a5a26c,0x0002303eee79dca9}, {0x0003cc24582cff2b,0x000018aea38a1c28,0x0000000000000001,0x0000000000000000}}, {{0x000fee514ac3447b,0x00004db0b385f6f7,0x000785b0f880a482,0x0006256aaff858dd}, {0x0005224f69e65ae6,0x00099c8d9092f5e3,0x000a4d5b2e1cceee,0x00065201f6ee40cf}, {0x000abc908fefbb83,0x000f4461f21689bc,0x0000000000000000,0x0000000000000000}}}, + {{{0x0006ba04a3b21865,0x000b257a4daf424e,0x000a3bec5e4ad4b6,0x0001bf9b577cca5f}, {0x000e764bad272e02,0x0003f73ae3883e3d,0x00078df5e21be44c,0x00046e6d9107b020}, {0x00009a9e8ecf5414,0x00062dbf7ce0cbdd,0x0000000000000001,0x0000000000000000}}, {{0x000ed268a93dd876,0x0001345efa0e6741,0x000a124edec5cb04,0x0002301ceb5830ec}, {0x000c074b8d138854,0x0003717a195a4b92,0x00078d388cdf3e4c,0x000aa6e0172f6f1a}, {0x000474fe593756f7,0x00063e14f10ad2f4,0x0000000000000000,0x0000000000000000}}}, + {{{0x00056d8ef5183d33,0x0006d4aef07505d7,0x000fd1d5c09dd4c2,0x00026645b43bb4e7}, {0x000876a817eb253c,0x000209c32af92f54,0x00083a63e205a086,0x000d802502f8b9b8}, {0x00069b5a7a6f9cb8,0x000e8d87089ec121,0x0000000000000000,0x0000000000000000}}, {{0x000c2981d5c3f78b,0x000fd7c0b6dcff4c,0x000e2051e1d39135,0x00056f990f800ca1}, {0x000bbae12c766764,0x000fc5fcbf987a86,0x000db02cbf344cfd,0x000965a4f55ea853}, {0x000d8cf4b24b115f,0x000ddb28cffa2bf4,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=97 [{1,2,3,..,}]*([2^388]*G) */ + {{{0x0004c9f1b01bbcb1,0x0008c564ba5d43cd,0x000b6d345ac26289,0x000156220f11b91d}, {0x000dcfa53c395f80,0x000e8f0647eb32e3,0x00071de125c49a24,0x00035c7837015374}, {0x000bc6a877a9741e,0x000a3e1c5d3aa600,0x0000000000000000,0x0000000000000000}}, {{0x000eae31d5cbf2fb,0x000ff21336f73218,0x00063097ec47555d,0x000e41ac8f16a8d5}, {0x00010c063790a928,0x000842e2a872cf72,0x000d214bed4668e2,0x000e7b5aab91e7f0}, {0x0002089cbbd94783,0x000d7755df6f3c47,0x0000000000000000,0x0000000000000000}}}, + {{{0x000c45be80c5ceff,0x0008b5f3a736f81d,0x000f684853db7b94,0x0007af37a1d8a120}, {0x000c1e37602997de,0x0001ba866cadb401,0x000496a24e3e6caa,0x0002c3eae9d6cd94}, {0x00074e14e06f82a8,0x000343069d99834a,0x0000000000000001,0x0000000000000000}}, {{0x00060baf525e3e5c,0x000adc3855bae42a,0x00020017824fbb6a,0x000d2439950ab7cd}, {0x0001396b971049df,0x000925d16b65cd1e,0x0000815f2ec4dd14,0x00099059e2e1d82c}, {0x0007231633ba03a7,0x000c17f35a3c602f,0x0000000000000000,0x0000000000000000}}}, + {{{0x0005cd392892c72b,0x000067c79588a466,0x0003e61043554996,0x000e4fea3dc40ec6}, {0x0006245b592ed7ab,0x000d5fddb91366af,0x0005bcef5cf9183d,0x0006f2afe7d93c79}, {0x00038f789bd81e98,0x000a24ae25969d23,0x0000000000000000,0x0000000000000000}}, {{0x00015df628bff28b,0x0008b0238f67e086,0x000debae13d569b1,0x000537b081c52561}, {0x000e3e637e8d5f7e,0x0000b558a9b4dba0,0x00092e9e6c338cb1,0x00054c7742924ded}, {0x0001b877e357069a,0x00077dfb12df7675,0x0000000000000000,0x0000000000000000}}}, + {{{0x000eac10daa6fc06,0x000a451d0d93fa05,0x00030dbe3663fb80,0x0003df5569a67246}, {0x000583a7b7740137,0x000a1bf5763b688d,0x00007976f4ce0d19,0x000972348a80c203}, {0x000296b9dd874e74,0x000ad726fc872424,0x0000000000000001,0x0000000000000000}}, {{0x000626a33acca649,0x00019c0b206a34a4,0x000597e1cfe661fc,0x000915e61f98c11b}, {0x0006400c41adb010,0x000ed21859bdcb2d,0x000247507de82c2f,0x0007ba1295daeffa}, {0x0000842d673a4f29,0x0001721fc083b457,0x0000000000000000,0x0000000000000000}}}, + {{{0x00034490e59e8714,0x0004db94a827c594,0x000be5c43f7b8f49,0x0005c1ed99f38332}, {0x000c89e5a5eae6d0,0x0007a328c3f873b0,0x000b195eb0205cc5,0x000957c8e1b9bcde}, {0x000a9c05764ec15f,0x0003785ee6082c58,0x0000000000000000,0x0000000000000000}}, {{0x0006efc5f7d22a96,0x000c221343eb9e1a,0x000752365f225594,0x00006ed92aaffd4a}, {0x00047b4ac0a51a79,0x00034752a1b444db,0x0000b01d86935b36,0x000ff91905b43171}, {0x000a16a9f33a34ae,0x00098febd2ea9993,0x0000000000000000,0x0000000000000000}}}, + {{{0x0009425bd3be24b1,0x000ddf604bb70186,0x00004732993f7d31,0x0003e481243d40e7}, {0x000cdd085631cbfd,0x00020a779443a447,0x0005a3d17a4bd6e9,0x000c2ba013a6f159}, {0x000091ba40454351,0x000bd45892d66dc6,0x0000000000000000,0x0000000000000000}}, {{0x000413e025588ac6,0x0006b5fdc28cccb3,0x0001b7df9aa499d4,0x000dec43b5bc5fae}, {0x00066d0478edbc5a,0x000c5c7523db595f,0x000f66c3689cc921,0x000369286e3460ae}, {0x0009f5650aae055f,0x000ecf05c2d82ed3,0x0000000000000001,0x0000000000000000}}}, + {{{0x0002dc4c335595b5,0x00087d3146806d12,0x000280502c85e0da,0x0006080ca34c63d6}, {0x00057bee19e4fbfb,0x000b329c24618627,0x00059c63cee44932,0x000729a1018b673d}, {0x00038096ab196705,0x0008deabcefcd004,0x0000000000000000,0x0000000000000000}}, {{0x0008d702c45db29c,0x000164a83c7ff2f7,0x00004bdcc8d9c10d,0x0006a51badde1985}, {0x0007b3d628a8f0d9,0x000238514e884060,0x0003ad2af9123d94,0x0001c02c79997897}, {0x000547f0e64e4ad3,0x0009546706bd2921,0x0000000000000000,0x0000000000000000}}}, + {{{0x000474c793f4e626,0x0001b9f27eccc505,0x0004061bf3c9bbd8,0x000776661385eed6}, {0x000ca2d5cbcfee54,0x000b0a8618a0f415,0x000c45418675f88a,0x000e0d5df7ae47e7}, {0x000f8808275324e3,0x0005bfe818ccf0e2,0x0000000000000000,0x0000000000000000}}, {{0x000d8f452acbfbda,0x000031ef224de6c2,0x00070dcb7f79ce12,0x00097b18d958660b}, {0x000224dfd1c89c45,0x000d69ce10da0fa5,0x0001b65989afa822,0x00094e1b3fa146e8}, {0x0002ae217dcac503,0x0006327d3eef4183,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=98 [{1,2,3,..,}]*([2^392]*G) */ + {{{0x000f424e1d9c6487,0x0001fab2d1847091,0x000c18abe11482b3,0x00002a204548d244}, {0x000d2901117bfc9e,0x00086c34a4b1dc1c,0x00025a48ee98a6d2,0x000cb76b3ae604c3}, {0x0001c8298db518ae,0x00008fb7194b2498,0x0000000000000001,0x0000000000000000}}, {{0x000055d976a5cca1,0x000a2cc7061a3bfd,0x0005a667bf88a107,0x00028dc3af0a63ac}, {0x000873e8641fb210,0x00040a558da80d94,0x000fc0963f8d8c7d,0x0009ccc465473acc}, {0x0005ce362fbd5307,0x00081d15006f15e8,0x0000000000000001,0x0000000000000000}}}, + {{{0x00022141cc3287fc,0x00038266149a76c1,0x000fdcc4f6ec602d,0x000673e316b23844}, {0x0002f7ff0591cea0,0x000226449753082d,0x0008c9cafeaf6b58,0x000801a84f6bca52}, {0x000738672e240363,0x0000fb54c4174c43,0x0000000000000000,0x0000000000000000}}, {{0x00005f340ab8f558,0x0003c50b6de62a42,0x000c67ea2a5a8485,0x000584f292e0d583}, {0x000976bb7a441296,0x000e6eb31fa9f0cd,0x0005886d0d33dfac,0x0005ca9b5dbb850b}, {0x0003bbd95e75a3fb,0x000aa9fc35ee4214,0x0000000000000000,0x0000000000000000}}}, + {{{0x00048296df946f82,0x000d1ae0f7178948,0x0009fea01e8f3835,0x00032c83af761a04}, {0x000c81060dc76c7d,0x00037519156c5ff4,0x000bd4aec6a8a45c,0x000cdf6166cd6f1d}, {0x00042edd4a64bcec,0x00087353c4d352a9,0x0000000000000000,0x0000000000000000}}, {{0x0007c5d100fac674,0x000db9e757c12a95,0x0005aae530260819,0x00099d6efbc64c70}, {0x00069b00bbd4972a,0x0000b755c13bf68d,0x000c9bf125fdc71f,0x0002e7e830894502}, {0x000fe09937e20c9f,0x00002b5fa7bc0dae,0x0000000000000000,0x0000000000000000}}}, + {{{0x0006f3e4a3b653b6,0x000d1a688f6e0a4c,0x0009e0c6a622a9b6,0x00092759febdd4b3}, {0x0007de14d66ca7fb,0x000a2edb8e3d698e,0x0001ab808ea4d110,0x00080f85570610b2}, {0x0004c29e8f8405b0,0x00027dadf7ff6310,0x0000000000000001,0x0000000000000000}}, {{0x00057c2f52f741db,0x0008e2e671ed8847,0x000d5288875b3801,0x000d96a8629331d9}, {0x0005a7e602196279,0x0001f2dc09559eca,0x0008fe5f274c1468,0x0006483d04249681}, {0x000f5286b1c8246f,0x00077944ba105276,0x0000000000000001,0x0000000000000000}}}, + {{{0x00065afeabf73b11,0x0008c74def9ab376,0x00022e8ebbf2f73b,0x000cbc3cc29566a6}, {0x000e491e3c34a56e,0x0005e3e30062fb22,0x0008f34bd61c9faa,0x0006ad9002a6b766}, {0x0001e57f55fbc947,0x00052212e9e41cf0,0x0000000000000001,0x0000000000000000}}, {{0x0000895f7a4eff76,0x00077650f0c0269a,0x000b638552f2435d,0x000cabbe1734cb6d}, {0x000ac1912708cc1d,0x00051cb37cd08219,0x0002a65eea8444e1,0x000bbf996f86f52b}, {0x000fc6bb767f7430,0x000d61b13f4e2c8e,0x0000000000000001,0x0000000000000000}}}, + {{{0x000d5ccc9d50240d,0x00084d2e5496b2b0,0x000f884405e473c3,0x00057903c51b5ae8}, {0x000d55afcc0675e0,0x000125af4b250a76,0x000937bc2b1d7e99,0x000a633470b9206a}, {0x0000cfe28137ad4e,0x0002cd8906210714,0x0000000000000000,0x0000000000000000}}, {{0x000877153680ec3c,0x00019913626b5dae,0x000bd89f532b2d72,0x000d578470af1c82}, {0x000b3f975bb94d7b,0x000f53a6076b8092,0x000db22bb5b744d5,0x000f43f924c34cb2}, {0x0006114e078b5812,0x000fb3e3de49501e,0x0000000000000000,0x0000000000000000}}}, + {{{0x00028536905d5e34,0x00092fdd83e93bc7,0x0001a64b28a1f75a,0x000a170fe5a66cb3}, {0x0001e6a59754016b,0x000c5178f6263ade,0x000bc78f0892f5d7,0x0002723285c0f5f3}, {0x000ce420f8de3807,0x0004c264b36ac9d2,0x0000000000000001,0x0000000000000000}}, {{0x000334fb73061393,0x000f6bbeabea50b7,0x000a60e0b4b57374,0x000565e15377dc7c}, {0x000ef0ff94b0e964,0x000e47e49a96f0ea,0x00030ac68d3dc2bc,0x0009347ee5d6453c}, {0x000eac1f6901d599,0x000086abf91bda19,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d441660f7e89b,0x0004d20a8f4b5442,0x000f7bf31e592476,0x0002835026e4af60}, {0x000359fc53bdf7be,0x00012726ea080568,0x00075b1726e147ea,0x0001ca2a0207d5e1}, {0x000322cae833e591,0x000e936cba51a14d,0x0000000000000001,0x0000000000000000}}, {{0x000a1d6534e7b937,0x00065ce0948dd2da,0x0002f88e2acca2b5,0x00091e6c94ccb3a4}, {0x000c7d8fe477585d,0x000807f46db59ceb,0x000b940f42378210,0x000e402ecff0fc9e}, {0x0007bc598d173f94,0x000f16af975145bf,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=99 [{1,2,3,..,}]*([2^396]*G) */ + {{{0x000942a4c75e9fe7,0x000c673dad29e115,0x000e6be55a2350de,0x00045659ca3c399f}, {0x0003c87e22652712,0x000bd4c4458f51c6,0x00057db758ae1a11,0x000c547db810319b}, {0x000c8ba847d5c89d,0x000239959a5bbe62,0x0000000000000001,0x0000000000000000}}, {{0x0003d490f3876f02,0x000521482b690e8c,0x00083aada0850d48,0x0004a53582c13331}, {0x0007bae5a342545c,0x0001c50d6f31c146,0x000d97c2d093b815,0x000e82d6fbdb84a7}, {0x00053468edb41ffb,0x0009e6ae0e9fad49,0x0000000000000000,0x0000000000000000}}}, + {{{0x00060773f6c223a7,0x000ee1e8255739aa,0x000672547b158459,0x0000639d3f4e65fb}, {0x000635059b89e003,0x000ffb7b9ac29a4f,0x000cad3267e18233,0x000a7f7d6bbb6854}, {0x000d91b6e79c3b99,0x00065ebec22c4720,0x0000000000000000,0x0000000000000000}}, {{0x0002fbb0f6be8071,0x0006dc307d9b73d7,0x000c6fa2f527f8c4,0x000e537cfbf3d06f}, {0x00018d7888122d62,0x0001ea61a84ebd38,0x0005532479a6f831,0x0004a4bf3325eafa}, {0x000717809c7a3155,0x0000399520a8095a,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b807898f2ae67,0x000d7e4f60fc8927,0x000bd2ecac1c97bb,0x00075b365b008578}, {0x000f53b54d53eaaf,0x000e5a0ab86540b6,0x00009c6ac54c1d9c,0x000267e6b65d52d9}, {0x000a061126607ea0,0x00011c0a94e0f97e,0x0000000000000000,0x0000000000000000}}, {{0x000dd52c7077ac64,0x000e6849bb24e97f,0x000e60101c7fc9b8,0x000e52f0dcbe3ce9}, {0x000023c779930c2a,0x00050df58185f016,0x000a6864a7c2cfea,0x000d52f720dc8c89}, {0x0006aecf1e2d3b3f,0x000fe51ac6cabd56,0x0000000000000000,0x0000000000000000}}}, + {{{0x0002f47341f417ba,0x000cd8bb740324e4,0x000e45bfed89909b,0x00089a7fe9550616}, {0x000b7861f9828778,0x0008946a71446a53,0x0006502fbe6cdbfc,0x000f70373dcf7291}, {0x000b59a0e59c8ba5,0x0004ed07606ee31a,0x0000000000000001,0x0000000000000000}}, {{0x0003f237a7cf7e71,0x000f61b5ddd50efa,0x00056f3a63a0f211,0x0004dae68319e664}, {0x00090e5acc6a3312,0x000d6c5fe3f2c7d0,0x0005f435cc8df625,0x000420f4229fa016}, {0x0009c92c95adf8ff,0x000c03d951affc2e,0x0000000000000000,0x0000000000000000}}}, + {{{0x0008e856f1a63237,0x000c38cfb4864e09,0x000a9b6c8ef2a48d,0x000565ee070c9954}, {0x000b247d25c31c3e,0x000d383653eec5d7,0x00033815ac0ce45d,0x0001d40035f31f5f}, {0x00016fd77486c728,0x0002145bbe546eb5,0x0000000000000000,0x0000000000000000}}, {{0x000f8a00f9535f58,0x0006793f432f2f0d,0x000a22b5ed0740ff,0x00046e71ff907935}, {0x0006ed013a6683d4,0x0006c4dc5feeee1f,0x000e49d371167831,0x000c07fabe848ed6}, {0x000dbccd8d710493,0x0003939263e99e29,0x0000000000000001,0x0000000000000000}}}, + {{{0x0009123d0b7caf2a,0x00010c9813e058eb,0x000b14ecd8d170c4,0x000aafb543a445ef}, {0x00066b13251ba12c,0x000a58ee44692c76,0x0009f4193734f736,0x000a0ff39b6d1ecf}, {0x00049d16c582e2c3,0x000a54bc874e11dd,0x0000000000000000,0x0000000000000000}}, {{0x00086c1422f1debe,0x000be6be161bed99,0x00094c11992adb85,0x0007d02a974392b1}, {0x000dc1355dd0b472,0x00001779ff9ef84f,0x0008dc1dbbcd1c30,0x0007615360f70921}, {0x000cd90cece2bff5,0x000a085b2b4772a1,0x0000000000000000,0x0000000000000000}}}, + {{{0x000bac8ffc9b1cbb,0x00041fbeb1c7a5f9,0x00032bb744d70f6e,0x0006377a1f50a7aa}, {0x00008611c61b42a6,0x000ce936f3324a60,0x000a084c3dc201c1,0x0002b70db700c5bc}, {0x000f0caf347988c8,0x00025e2dfe3675fe,0x0000000000000001,0x0000000000000000}}, {{0x000fee427e9f9424,0x00069a165d8d1c76,0x000dc1fbe1e937f2,0x000db180dfee35cc}, {0x000b55475d4c9745,0x00036d0a2fefc6cd,0x000476726ccd4e8b,0x000f3783e580f7ab}, {0x0005d079b31ff3c1,0x00009ced18ab19ce,0x0000000000000001,0x0000000000000000}}}, + {{{0x0000806a0ffa1183,0x000b368c8ad93735,0x00045322006bc207,0x000356cef3feb2a7}, {0x0000d9772041b29b,0x00015326534db436,0x000c3a4ffd400337,0x0004525bb0cb62b4}, {0x0003dfc1aef8cba9,0x00023c5f95e7e7b2,0x0000000000000001,0x0000000000000000}}, {{0x0009f0d5faafe7a7,0x00079f056236883a,0x000e0f2e02fe7ca4,0x00020a75821e669e}, {0x000fc3208dc12d93,0x000c95bd4d2a90f1,0x000127ab836d5d0a,0x0006a56f1f5b0f15}, {0x00053533b35a8c80,0x000e1db5d6ee484b,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=100 [{1,2,3,..,}]*([2^400]*G) */ + {{{0x000b291af321cc48,0x0000d263480c29b1,0x000ec9060276afae,0x000489d0d90afac7}, {0x000ec62d3da37067,0x00088f38b3e31b78,0x0007f35ff5fafe7d,0x0007885361013dfa}, {0x000270f897a6237d,0x000c2a42a2eac980,0x0000000000000000,0x0000000000000000}}, {{0x000b6527c69198d0,0x0008c038b2196095,0x00056c8573aab3e2,0x000c299349dbf002}, {0x0009c016fc238b06,0x000c26c63dc550c5,0x000712822cb43957,0x00044a5765b7d6c8}, {0x0000be87f42f34ea,0x00061a9436ed5036,0x0000000000000000,0x0000000000000000}}}, + {{{0x0006c0f3c9200410,0x000c1523b6d0c7cf,0x000b2a524b701323,0x00080e1d445c4f05}, {0x000cfb721bd24fb2,0x0004d74f5043a450,0x000c3ca459a36903,0x0001f8a997769ed3}, {0x0004569349bd35cd,0x000fa7a1b94559af,0x0000000000000000,0x0000000000000000}}, {{0x000479e9fda50d86,0x00039193e5dd5ac7,0x000fcfc382ffdaf1,0x0006e937727251e2}, {0x000819f976e0e477,0x000e0dea37faf936,0x0002b3218ec38c97,0x00020308bb264566}, {0x000441534d581e3f,0x000046c275dc071a,0x0000000000000001,0x0000000000000000}}}, + {{{0x000cb5968e20ed2c,0x000a52702c5bb89b,0x00033f897addfb47,0x000f030ce16c51a7}, {0x000945e8bc092078,0x0000a224a9b9a4c1,0x00074fb0d2a2dbea,0x000a00b01506244c}, {0x000403180ef3b3ed,0x0000d544f09c72f4,0x0000000000000001,0x0000000000000000}}, {{0x0007f0289c261b7e,0x000d803c0211ff4a,0x0001ffe93b188323,0x000b503181a127a4}, {0x000960bd651117de,0x000f238b1565ffd2,0x000706280cef133f,0x0004ddb33d18643d}, {0x00057a46393ccc6e,0x00011d1fcc4678c9,0x0000000000000001,0x0000000000000000}}}, + {{{0x000cb18067eb6c9c,0x000904e0e232321c,0x000c2c362eb5eae4,0x0004f20ada675b34}, {0x0009513d2fa912c1,0x000c8c7ff960f4ae,0x000ea3278df20646,0x0004b702cc146790}, {0x000b87bb57608da3,0x000f1fadae0fb9e2,0x0000000000000001,0x0000000000000000}}, {{0x0006a843b0ca7a84,0x000ffac89f77d5a2,0x000265d14c35a368,0x000486e7957c89a9}, {0x000f9514b7e05bd1,0x00037cf3d5a9030e,0x0002008b9ea39985,0x00050c45cfba4fb3}, {0x0008d5a1561ff956,0x000bc01cc6a56407,0x0000000000000001,0x0000000000000000}}}, + {{{0x0005bb52ea9ac7ec,0x000e28f7ce0ec366,0x0004c059fd569d2f,0x0000e89276b354dc}, {0x00013674b639e129,0x00051c92206d8283,0x0005fd8d62852509,0x00083a0ba16eee81}, {0x00023ff408ee3851,0x000e736678fced53,0x0000000000000001,0x0000000000000000}}, {{0x000a8b28dba67d24,0x000216ba84ecb673,0x00034998afbbd048,0x0004e06cfb264967}, {0x00064c024c958fcd,0x000c3e07fdd668c7,0x000e902ee5004559,0x000c7be484240ea9}, {0x00085d1cbdf78504,0x00001fc315ffe9b1,0x0000000000000000,0x0000000000000000}}}, + {{{0x0003f67e6d554604,0x000a7edbdf25d6db,0x0004a86faf823503,0x000561d458cbd82a}, {0x000fd50ad1fb2727,0x000d57e2f00994b8,0x0006a571b2b47251,0x00025dcba3cd1573}, {0x0003351634df5819,0x000f90f716e579ec,0x0000000000000001,0x0000000000000000}}, {{0x00015c741fd15e62,0x000782e4509eb436,0x000bb1dede8ef754,0x000e32d87a793f6d}, {0x000cb27d972fea02,0x0000af4ace00b65e,0x000665980ede0c9d,0x000f6c809dcb9681}, {0x0003f6f1f979a653,0x000fa70638c8e694,0x0000000000000001,0x0000000000000000}}}, + {{{0x00069573569a82f9,0x00074c79907894c3,0x000923a4f546a942,0x0003c148a698895f}, {0x000c357afe3d1621,0x0000597be114eca3,0x0008bde57638ac14,0x000ab1f30c4d2325}, {0x000c1e648b9d09ce,0x000b5b544e3974e2,0x0000000000000000,0x0000000000000000}}, {{0x000a45f392d296b8,0x0008740111ad4028,0x000c3e262fdbe28f,0x000c3453830a9ee4}, {0x000ae0fdaa4f4e3c,0x000db8b9c8044def,0x000827b06665f64c,0x0004b0bfa5e94a06}, {0x000288ab3926f336,0x00095cd9d3c3ecaf,0x0000000000000000,0x0000000000000000}}}, + {{{0x000b181f2f210670,0x0009ac047db30c5e,0x000f0b9977abb73f,0x000685fec5355db3}, {0x000cba356655da26,0x00050bb7fd48c1a2,0x000094ac2c2dc459,0x0002437ff8a8e766}, {0x00026425d80146ca,0x000d215b7aafe50b,0x0000000000000001,0x0000000000000000}}, {{0x0008a54b77260e44,0x000200683c3c461c,0x000806c758fee244,0x000ee02fb90af5d8}, {0x0005167ac6f6571d,0x000f1dfed253aeab,0x00051fc762d73380,0x000f059d556559bf}, {0x0001430491432973,0x00077ebf133e93a9,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=101 [{1,2,3,..,}]*([2^404]*G) */ + {{{0x0002ff226442343f,0x000cd833f176501e,0x00095759918597f6,0x0008de415f1a7fb7}, {0x000e4316d41dcb51,0x00029d89c6966644,0x000513d7775eaca8,0x000a9d45da4eed7a}, {0x0002852aadd14ef8,0x00062ec016fff623,0x0000000000000001,0x0000000000000000}}, {{0x00091f95c7b230dd,0x000a732cc46d0d07,0x00004775f23d4641,0x000d9d15b9dcdfdd}, {0x0007a727ace99c9b,0x00077fe3e5aec6f6,0x00023beeac1ee2fa,0x00008f21d63275b9}, {0x00065885355852cf,0x000ce1be6d4550b6,0x0000000000000000,0x0000000000000000}}}, + {{{0x0006e321251c61dc,0x0002560a1fff242c,0x000a45d55894b5a6,0x000bc8f1bc1ece0d}, {0x00099655945af852,0x00081d51a6f4152e,0x00048184573b7d9f,0x000f69e42e8015c7}, {0x000b2c20669dba53,0x000e9f9624512355,0x0000000000000000,0x0000000000000000}}, {{0x000c96019553b866,0x000abe8a5146d011,0x0006c8e83c7c8d9f,0x000d33f93fede45c}, {0x0006fb87d2fad3d5,0x0007a48456e1fe30,0x000e6e9f030c2436,0x000ab8f59e2c2aa7}, {0x000e2ecee8097392,0x000ce7dbed7e8f7a,0x0000000000000001,0x0000000000000000}}}, + {{{0x0009869c0eb3066b,0x000607798fe984da,0x000507796822415b,0x000b35e4cf4e7bb0}, {0x00043514e2eac3dd,0x0006f5f2ec73d5e0,0x00073c55dac4fc5a,0x00009ae9d9b66b29}, {0x000e7849157dbfd1,0x000a5482a9437d4b,0x0000000000000001,0x0000000000000000}}, {{0x000c4e943ae95512,0x0006cf652253cdb5,0x000d6ec12d70a741,0x000569eacfb8355d}, {0x000eedc557ace0f2,0x00079ed65a726737,0x000cbaa427929f79,0x000f9f7b12d69985}, {0x000f61768135719d,0x000ae5a402492b60,0x0000000000000001,0x0000000000000000}}}, + {{{0x000feaf2d1e3892c,0x000ab68c7ed96e8e,0x0003cb959b9f3e2f,0x000c2e34fe65989c}, {0x00089b397dfd24ce,0x0005a4f7a72a8210,0x0003d4d194fcfc29,0x0004009eac36cd18}, {0x0004df54ac2005f3,0x00037ac4355ce338,0x0000000000000000,0x0000000000000000}}, {{0x00018f15a5288002,0x00084aab158f0c62,0x000919d3c1cc9db6,0x000c654f22157c5c}, {0x00061a5d7e7c5733,0x000dc89cd41bb67f,0x00046690cac1f786,0x000f2b55f183f6e7}, {0x000f41e4cb445fa4,0x0005a969c4dd429d,0x0000000000000000,0x0000000000000000}}}, + {{{0x000f3df64800b6f3,0x000d7480b73133bc,0x000b600425660d57,0x000b837e3aef4df6}, {0x0005fb5d2c2dd02d,0x00011fe018b03d5a,0x000c71322317085a,0x000d374ad3a452d3}, {0x000a4d81aa830346,0x0004a3299709ce7b,0x0000000000000001,0x0000000000000000}}, {{0x00016776ed8701be,0x0009a6af0a211f00,0x000ee5799e2d0b99,0x0004ffaa89dc73ac}, {0x0005974371ca9e1e,0x000e0cc77bcb98c1,0x000c6b4ceea4b327,0x000a8308bd6b7617}, {0x000c8a4f2fabfb14,0x0006930491f45b7a,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d9bdcbcf79471,0x00074d3dd4ca53d8,0x0001af7d8d51306c,0x000b82f03e680d58}, {0x000a7884ca0be9c6,0x000c62e3720aacdb,0x000ad9ad633f5955,0x0000c84d067a1c4c}, {0x000e24eee54e3c55,0x00019dbe3f67b54f,0x0000000000000001,0x0000000000000000}}, {{0x00099b839c026b9e,0x000bc7d75cb7b6b9,0x0005e6b4aa8a5275,0x000156cdfaa9f40a}, {0x0004a1992d1c2e6b,0x000b18092816e51f,0x00027a900c94afd0,0x00010f9d0fb16b34}, {0x0008f98b409eefa5,0x000aaed3cae46309,0x0000000000000001,0x0000000000000000}}}, + {{{0x00060d5e996dc11b,0x000619082845b56f,0x0006ef36f7ab0ecb,0x000d028f81fdfa6c}, {0x000060da295844af,0x0009596e31cc9ebb,0x000308c32eb79211,0x0001ea951754105b}, {0x00008750f063690e,0x00083b8a021ee964,0x0000000000000001,0x0000000000000000}}, {{0x000d813e43ebc1f3,0x000af78ecca7ac06,0x000d8eb52a4c2d59,0x00067d3099d75877}, {0x0005b00d48668cb3,0x0002755481aaf570,0x0000b8ddd277a751,0x000dd4573c6cfa3c}, {0x00075e83624bf68a,0x000ee89ee03e9ce2,0x0000000000000001,0x0000000000000000}}}, + {{{0x0001adc31d17fe65,0x000ba5bb3a93b688,0x000b603dd9e8ce1c,0x0008b0cf4f5b70c1}, {0x000175f958dd3aed,0x00070f44e15eae25,0x000177aa5b942f5b,0x000302efb949c526}, {0x0002ba3a2c8dd115,0x000b89f9288a9513,0x0000000000000001,0x0000000000000000}}, {{0x0005972ebede20db,0x0005b1bc841aedc4,0x000933a99b87853d,0x000597276e1536aa}, {0x000a0238abf3c852,0x000485ab1105488f,0x0006d521debe07d8,0x0002f1ad18f16d6f}, {0x000d3c55a54637f9,0x000804a2b58773df,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=102 [{1,2,3,..,}]*([2^408]*G) */ + {{{0x00085142795b0fc9,0x00032b75a8973a1f,0x000125d27c2f65e5,0x000245efb7e6ca7e}, {0x000a3d37a1c1d680,0x0008ed8871c0ac3f,0x000f92273ed1f61c,0x000f1c0619b125a3}, {0x000c151e1baaf99b,0x000b5fb9c92ca12f,0x0000000000000001,0x0000000000000000}}, {{0x000d45f1302c2800,0x00028a46eca65c4c,0x000181d940c2f16b,0x00008156dae561c3}, {0x000ffdb51b5dbdef,0x0007d0f3f05aff9f,0x000216756731470d,0x000d3e4323b09f64}, {0x000c256f1c5c736f,0x000c46af757ebac2,0x0000000000000000,0x0000000000000000}}}, + {{{0x0008b99d3a5cf086,0x000a6f7dad9f189d,0x0009613956296f2e,0x00029ebaf5d410b4}, {0x000d4c6b1f46d86c,0x0001709ad92dbba6,0x0000cc09de07504b,0x0006c7c9ec95eea8}, {0x00007648647d01eb,0x000b3919b1d6cd99,0x0000000000000000,0x0000000000000000}}, {{0x0005f9f34e61ba7e,0x000eff53cc24a00a,0x000672781ea5e367,0x0005266f275cfce0}, {0x0009992d98139edc,0x0002c0efd50d9e20,0x000de18f3d9cb26c,0x000b647d23fe687b}, {0x00054a71ad97b9cc,0x00027a258eaff20c,0x0000000000000001,0x0000000000000000}}}, + {{{0x0008f39dfcb8ed4b,0x000811922f1fcd37,0x000a2846f7edab95,0x000566857a64449b}, {0x0005d6755b91a9b1,0x00033e748b542e81,0x00055f4eecb18a57,0x0006c4663a3a4f59}, {0x00052acaf4bb5b32,0x00079a0d2cfa0bdf,0x0000000000000001,0x0000000000000000}}, {{0x000a0b89e0ca131f,0x000be75df1d52db6,0x000534480e6cb8fa,0x000be7af1d438291}, {0x000ad1a493673eca,0x000a16c7e34998f0,0x000d3ab56804db13,0x000479fc142bd461}, {0x000daa988aca4c3a,0x0001acd3bd93e84a,0x0000000000000001,0x0000000000000000}}}, + {{{0x000cbad8f99ea394,0x000cf3fa23022a30,0x0009f3a186b0f3c6,0x000e922b33420e3c}, {0x000c1bffbbdb1dc6,0x000aa59cdeeac227,0x0003dd943d5b8787,0x0005513a5be5e367}, {0x0004c0fefae77a5b,0x0003518e4c10fcbc,0x0000000000000000,0x0000000000000000}}, {{0x00024505728229a8,0x00014b020fe0ed2a,0x00039e8625b5e8e9,0x000ac893dbd2dbf4}, {0x0002a5bf5b95c3df,0x0009c6d879c2cfde,0x000a75fca30a3152,0x000f3ac05ce905a9}, {0x000445553894b84c,0x000dc2eb87696cb5,0x0000000000000000,0x0000000000000000}}}, + {{{0x000c1e6489e4e3ff,0x000e40fadb280ee1,0x0009bea8383ad81c,0x000e1b3d235f576b}, {0x000a3198405d1a52,0x00058352c1e2f66a,0x0004288cd9077fa6,0x000dbd50c9d00a90}, {0x0009bd9a7d893793,0x000b363fd16ecce4,0x0000000000000001,0x0000000000000000}}, {{0x00020eac52973702,0x0003e0c70e76207c,0x00006b2a996d0f74,0x000f4f6a44bb0744}, {0x000df28bc6807b4d,0x00017b088a4a4664,0x000ec1355d57eb8f,0x000182cf9ce6cfbc}, {0x000610a314e418de,0x00010173ffff5e3f,0x0000000000000001,0x0000000000000000}}}, + {{{0x000165051d386e1e,0x000bd47b826f3a9b,0x000a1c2b86d2360d,0x0009b8835ea5cf34}, {0x0009783bb9260611,0x0002b789b3e982a5,0x000e987100210011,0x00097fc2b9ab3e6b}, {0x000ba0e81d76b916,0x0007fbcd1d8d4518,0x0000000000000000,0x0000000000000000}}, {{0x00091e776c559b38,0x0004c1e4fe7c517d,0x0006b3ca068cf1e7,0x0001a6fbc371e38f}, {0x0003b1539a3d7c5a,0x000cd0c3d50e8468,0x000c7e40ea4660b7,0x00012b9d873faeb0}, {0x0007f3ea3f2ff663,0x000c8c52e58fcf13,0x0000000000000001,0x0000000000000000}}}, + {{{0x000e0d1c0e7b0a6e,0x000182d5a79223c1,0x0002d07160d41c08,0x000de95d46cca38e}, {0x0004d00dd9df759c,0x0001f3ff3dc94e0c,0x000ad18e4b33fc34,0x000d2cfdbdfdd807}, {0x000c70a120714770,0x0005cb86265c3704,0x0000000000000001,0x0000000000000000}}, {{0x0002e095e7588e93,0x000bb0fc7a7538c8,0x00049b1bf8ce184d,0x0006eac46bad884f}, {0x000da577205521af,0x00073592b9bc40d1,0x000dae6302c8fae5,0x000e9a408dc07f7a}, {0x000d86c203a973c3,0x000db4c1145f0edb,0x0000000000000000,0x0000000000000000}}}, + {{{0x000b134fd6bc3c71,0x00030305a92256f9,0x0007ccfce0b23245,0x000ca36a6d8cb621}, {0x000236d0ef54fd61,0x000a182b1b210c1e,0x000e2c48ae4f2531,0x000aa16671b7b1f2}, {0x0001cf556d29f38d,0x0001c83fa6c8eaae,0x0000000000000000,0x0000000000000000}}, {{0x000a18df67396e49,0x000978a098406db9,0x000d15a5ed3d588a,0x0008786d781ea818}, {0x000c4ad06fce15e6,0x0006d7a550f98680,0x0004140981589bd6,0x000f7ff83976d3ff}, {0x00088eabc6ffe6df,0x00031647479f189c,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=103 [{1,2,3,..,}]*([2^412]*G) */ + {{{0x0000d7e3378fb507,0x000c9bb0731c42eb,0x0005dc372a3657ba,0x00074ab3d967282a}, {0x00057f9ac8856a93,0x000b740967bdf210,0x0003024fec8274b3,0x00015596459a5693}, {0x000d21c1794a1687,0x000a98ef7bcfc7c8,0x0000000000000000,0x0000000000000000}}, {{0x000af7b9ab0a89f0,0x000a24bfd8b660f9,0x0009cb13ed97b728,0x0000fd15ee5e0227}, {0x000febd3b7d28a45,0x000367bf5b972ff1,0x00091b608f71ea2f,0x000f496276edaa41}, {0x000a6a4c152d016b,0x0000bf52e7daddc4,0x0000000000000001,0x0000000000000000}}}, + {{{0x000ed176dc4c781a,0x000f2149979e6d8c,0x000b3c390a71d8f2,0x0009ec42d1cc9018}, {0x00013805d407dff4,0x00092c79f656592d,0x000250a69b4fae8b,0x0009ea40b75d7816}, {0x000e49fbb9c23c18,0x000630012080c698,0x0000000000000001,0x0000000000000000}}, {{0x000297ec2f3d27ef,0x0009b4394adc76de,0x000084a2dca39d2e,0x0004162fa4a3c98c}, {0x000de4df52d9fff5,0x0006af6c27847a48,0x00049729a4d9dcca,0x000b96ed3609128a}, {0x0002001168323c41,0x00051e81fed2290a,0x0000000000000000,0x0000000000000000}}}, + {{{0x00023ceeaf6fda5e,0x000fb751aba8d27a,0x00058878762241c7,0x0009d28160069d96}, {0x0001830ade2dd51c,0x0001c3eb509b6317,0x000cb86f909879a9,0x000dee7eb48aca8d}, {0x00028bf799244bc3,0x0009202a06470538,0x0000000000000000,0x0000000000000000}}, {{0x00015f7fc2843df9,0x0002f53ba48f85a0,0x00095ae129c2b6a1,0x000ddb2a444e10a6}, {0x0006aecfba54d8ae,0x0007c39b4f0e8bdf,0x000e6010a72c4d1c,0x000d5cdfd0f373b3}, {0x00068c9e099b50a0,0x0001c8bed1929d51,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c2c46683ba5b5,0x00057c57128a2090,0x000dbf610a813452,0x000009fdd16c4a33}, {0x000c78f1d5b65bff,0x000560ad02b49af8,0x000ea450eb8499df,0x0003a52dc630fb45}, {0x000da8ac57e35202,0x000fb72fd6cb3d8a,0x0000000000000001,0x0000000000000000}}, {{0x0006d77839d10202,0x000f1f4a52a42a8f,0x0004175b2fd3f44a,0x000ac14905fe7f14}, {0x000701757d0c0079,0x0008ae6d1c475fac,0x000d56b7bfd93878,0x0001b7dbf13b33f3}, {0x000d1df20cacad13,0x0006098aef62c8eb,0x0000000000000000,0x0000000000000000}}}, + {{{0x000224c02776a096,0x000fdf0428bd2668,0x000824486a9c4320,0x0009f3de8bb8cb98}, {0x00008a99c85515ad,0x00087cd9c5ea0fc3,0x00030bd0c213cb33,0x00075f7ddf36e0e9}, {0x0008b09a16a3e9e2,0x000ff92ba0c69ed0,0x0000000000000001,0x0000000000000000}}, {{0x000dd8b8dacd787a,0x00044526b1b4ed23,0x000c4fcde2872824,0x000a0aecc5884baf}, {0x0000edf7e8ded34b,0x000a4bc010f4ef7d,0x0007a006485cbfe6,0x000ba70311a2f225}, {0x00032199cd733758,0x000ee0992d474968,0x0000000000000001,0x0000000000000000}}}, + {{{0x000d4dc4b29d1e7e,0x000ced039d3b8a01,0x000815590ff2c2bd,0x00021d5655ef3773}, {0x0006fddb06996f34,0x0006f2915cab4832,0x0005a14903f506e5,0x0004bd51fa3522da}, {0x0009df3d4ada1113,0x00041c2c8aabe985,0x0000000000000001,0x0000000000000000}}, {{0x000708b80fba1a02,0x00028b6a0afd8d60,0x00058773df43d0b1,0x000ecec358a258ce}, {0x0008b0ae7440af59,0x0000e74789fcab99,0x0007429cf7b8fd56,0x000584c72e545e3c}, {0x0001897264b148aa,0x000a07b9a2d9b131,0x0000000000000000,0x0000000000000000}}}, + {{{0x000edce9ca1286ed,0x000ff3a46b7fad16,0x000535c9d7d2b840,0x0003708d5ca958c2}, {0x0005420c1e063332,0x000713a6fa0fa9a8,0x000918405a51ff70,0x000da85f855da776}, {0x000d1bea73da5e0f,0x00064183a6508fa4,0x0000000000000001,0x0000000000000000}}, {{0x00083d4f23ca730b,0x000725501529af4b,0x0008000d1b6d21a0,0x000a7b3eee3507cb}, {0x00022a88a07a33b4,0x000540f9d09c28e9,0x0004983b28faa40e,0x000f54edb432d0fc}, {0x000ce23569311803,0x000d266759b9bb64,0x0000000000000001,0x0000000000000000}}}, + {{{0x000fd03e2a0c4500,0x000d5cf3f782f796,0x0003ffeae620bf4a,0x000ca0158514f603}, {0x000545633e085e39,0x0004884fbea88f4e,0x000882a85fc77f29,0x00012678c646f605}, {0x0009ba323a505f9b,0x00029223217b4379,0x0000000000000000,0x0000000000000000}}, {{0x0002e8744f4170bf,0x00081a29194f6c03,0x000a932a7916cab1,0x000fb0f9f60ec063}, {0x000ff217a0ff0b94,0x00003ea56b8f066a,0x000ed4a56ee8b26d,0x0005efbf8ce2c1ff}, {0x0002eb114ed13051,0x00043d7447433992,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=104 [{1,2,3,..,}]*([2^416]*G) */ + {{{0x000affb50d183763,0x000c1c5fc7f37a88,0x000a1f73a2f170a0,0x0001192983474ff9}, {0x0006b4738ed4fea8,0x0004d293dcdd7868,0x0000cd916188a232,0x000bc585fd523667}, {0x000b67188a2e54db,0x0008ed10b37344d3,0x0000000000000000,0x0000000000000000}}, {{0x000548b086336003,0x000991fbe0b348b4,0x000ef3cdca5ad120,0x000cfcd2034c9a59}, {0x0004f56699960d16,0x0006df1f5f10f252,0x00000324733c5f1f,0x000a757f84ed98a5}, {0x0002f7eec2ce4fa9,0x000ae3d296f3ba03,0x0000000000000001,0x0000000000000000}}}, + {{{0x0004e9382b32007c,0x000d81cd77eae7c3,0x00013604a2dad7f0,0x00043d13d0e3fde5}, {0x0004c6cb59871704,0x000a8441d1c5f3e6,0x0002361adfd909c7,0x0004efba78610053}, {0x0001559070eb9abb,0x000708ee4fe6fb05,0x0000000000000001,0x0000000000000000}}, {{0x00079200ca4f6cc8,0x000579128ad5ec75,0x000c265973749006,0x000f0a7f8cf2fa39}, {0x0002ddb548c37c9d,0x000da31069648b65,0x0006a7e075eeef13,0x000504d0e4093491}, {0x000fd613bea6b782,0x00055bba92eb2c08,0x0000000000000001,0x0000000000000000}}}, + {{{0x0005ab4da4a4107d,0x000f0dbf85411a8b,0x000a933992a7b991,0x0000accdce47a748}, {0x000c5662f2eb8c82,0x00064b5fdd12508c,0x000db73adddfe6b9,0x000af97a44a31358}, {0x00044c5ddfefeaca,0x0003403a084f6ff5,0x0000000000000000,0x0000000000000000}}, {{0x000ad406fec21428,0x000e8954cddbeba4,0x00092a7fe19ec844,0x00084bffa4c49f5f}, {0x000ec8eb76b96304,0x00014948f0f75a70,0x000dff2c139503c4,0x00032fdf031b7606}, {0x000fa11a5ead6208,0x00047ef7a1eba7c5,0x0000000000000001,0x0000000000000000}}}, + {{{0x00073a3dabdb3534,0x0006dd5d8f611c69,0x000799bf33f3f4e8,0x00026f63749fb625}, {0x00092667bd3584a3,0x00060fa9fea81613,0x000999ea3a8de550,0x0004dd75d71ac4af}, {0x00078319418b1e64,0x000c67e995c857c8,0x0000000000000000,0x0000000000000000}}, {{0x000c7afa552eb541,0x000a7222bb4a0732,0x000c7e0e1a608c59,0x00009057e1132506}, {0x0009c5c6a19981fa,0x00019205096b7bf3,0x000a7d38ab7490c4,0x00099e016ba7462b}, {0x00007f474dc3595d,0x0006636a3d8c9f12,0x0000000000000001,0x0000000000000000}}}, + {{{0x000058d88f35f241,0x000a59b2e8d2532f,0x00015502597aca02,0x000fb1e8bd7d1caf}, {0x00055680e361da0d,0x000ed31cfd9355f1,0x00047c0b0064d2b2,0x0004f348830f3080}, {0x000b7440fbffaf7d,0x000a5a553b98e17b,0x0000000000000001,0x0000000000000000}}, {{0x0000f6eacb637570,0x000d4925881bc39f,0x000d655e7e9a7105,0x0002f09a033db883}, {0x000e97d5a4975dc8,0x000036e619a17847,0x00079d0e9b209304,0x000434fdadf80484}, {0x000412216e6c7daa,0x0000eb152f330b19,0x0000000000000001,0x0000000000000000}}}, + {{{0x000362cca8e2db27,0x000b58fb4260cc2d,0x000e2d527b899614,0x0001f0b467cb8aa3}, {0x000d1ef71b82f08c,0x000dc68072c4649a,0x00098aa11a9313c4,0x000165002fbe1ac2}, {0x0000bf5399f23796,0x00024b537dfdd6e9,0x0000000000000001,0x0000000000000000}}, {{0x000f6c830ade2b53,0x00074af2e76469fc,0x00051f1bc5f4ed41,0x0000406c3a450f7e}, {0x00002b53708a683c,0x000428a6e3aa7dce,0x000c0df44b377b62,0x000e9c1a58f5f1ab}, {0x000c5458b0f02c35,0x000b16ea8718da27,0x0000000000000000,0x0000000000000000}}}, + {{{0x000134d37c05c59c,0x0007233d57586878,0x0006cf5af7409f53,0x0008ae6dfc4fd018}, {0x000e8b54df4cc39a,0x0005f3046db1d402,0x000312aeece717b2,0x000da13a0c5d98af}, {0x00073d6305f96c47,0x000587c80a3e3a7f,0x0000000000000001,0x0000000000000000}}, {{0x00027d5a2516f5d0,0x000f9338bbf8fa7f,0x0002109c7d0c4360,0x0006004be57b26a1}, {0x000da32aad5aeeea,0x00041aa5daf9dede,0x000b0a16abc83073,0x000088080bdafdd6}, {0x0004fa8814cecd6e,0x000f5f24b2b7fe1d,0x0000000000000000,0x0000000000000000}}}, + {{{0x000cc19d6d74494a,0x0009296d31ebaa05,0x0003edd43b02f30e,0x000c79aac72cbbb4}, {0x000d57829df3e827,0x0008bb62624e4cf8,0x0004f05ffe745fb9,0x000950b350aaad89}, {0x000ea5e2db566ef1,0x000e1e37f6dcf4f2,0x0000000000000001,0x0000000000000000}}, {{0x00034202ee7f3c59,0x000ed5d748da48fa,0x000e1cf505b68fd9,0x00031b86c7778cb3}, {0x000dbdadb45073af,0x0004b6e80bfe717f,0x0001ee413036b30b,0x0003482b138b2c3f}, {0x000e1ed1e4fc0159,0x000e84d788bd2771,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=105 [{1,2,3,..,}]*([2^420]*G) */ + {{{0x0009afb73836ce2c,0x00035d1c0854627d,0x000acf6649f2eb9d,0x000acf4c38a8a9ec}, {0x0002178be52c0095,0x0008f6e06df7d7ea,0x0008285115ce7bb4,0x000b7f232680bedc}, {0x0004103a6e51e8f4,0x000aaa09aa0bc0d2,0x0000000000000001,0x0000000000000000}}, {{0x0001c4bc539f42b1,0x0009f1f757159ce1,0x0000e9e10c0bc8d0,0x0007345be3621884}, {0x000d1822e5e0a60d,0x000ae792acddc802,0x0006be0f49763d76,0x000dff0f1717868a}, {0x0004437867cae1bc,0x00070e8bfe19f269,0x0000000000000000,0x0000000000000000}}}, + {{{0x0001b8994b02326b,0x00031ce496416ae0,0x000dc0825cea213f,0x00050bdf0281aa93}, {0x00059236853f9e44,0x00041e294dc8c09e,0x000b03a595c72a58,0x000c2bc6e5381e14}, {0x00020b03546b3008,0x0008eca57d1874ef,0x0000000000000000,0x0000000000000000}}, {{0x00051a61ce5948af,0x000925d36a169359,0x0001712f7655b84b,0x0002f3fdc1e05016}, {0x0009aa758020ba42,0x0006927405b02281,0x0009822fced2aa8b,0x00019ae63d9313a7}, {0x000d9c07887cbebb,0x00065e13e45febcd,0x0000000000000001,0x0000000000000000}}}, + {{{0x00034f1f7e499842,0x000a48878049fec4,0x000692a3fe1f0c9c,0x00048261277fbbb0}, {0x00032263dc0fe7ad,0x000e09052ac6fec0,0x000683804d38aeb7,0x000d6c55fb1237fd}, {0x000bce4b453925e9,0x00056ac33e2d8263,0x0000000000000000,0x0000000000000000}}, {{0x000a764c5c6d3730,0x000ed9705b2adb70,0x0003bab4631b9f1b,0x00014535b4850149}, {0x0009c2385e82937a,0x0007aa5ebdcd9ea7,0x000656517e5b5eb0,0x00078fb6885fd321}, {0x00087f791c6b2bd8,0x000bcad44bfcfcce,0x0000000000000001,0x0000000000000000}}}, + {{{0x000a7c1bbda99502,0x000d13ad86ee95d4,0x0001edd6edef741f,0x0001221485b8fada}, {0x00062b65b3c0a089,0x000583aac0300922,0x000ec2e6716727e3,0x0006d6729d8a817e}, {0x0002ad34233bfe29,0x000806779921dca9,0x0000000000000000,0x0000000000000000}}, {{0x000373f0b0bc37c3,0x0006835632a2bfc1,0x000a8cd4f472c2b1,0x0004b6c9a4cedeb1}, {0x00076180690c132a,0x000eca05f9fa510c,0x0004e0551eb02c41,0x000e63213fc52e0f}, {0x0001b429bb6165cd,0x0002bc1fdd188cd9,0x0000000000000000,0x0000000000000000}}}, + {{{0x000629f7658d599b,0x000b1e87d39d0d52,0x000caaa997607dbd,0x000fcf23eb7d6dd2}, {0x000857e0cd30a02f,0x0003ecd22778d510,0x0005c5e961e1f158,0x00068aa70a145465}, {0x000c8fb1ff96ec7c,0x0009b464d2f55e62,0x0000000000000001,0x0000000000000000}}, {{0x000e904d20ac7941,0x000e4f0bec2602b6,0x00080f6effaef59a,0x00060688330793e6}, {0x000db2442ae08549,0x0005e3d773ff5a5f,0x000cecbc6ac0199c,0x00012fa7a795cacf}, {0x000d6f9bfc57e52d,0x000ce32e4eaeafdf,0x0000000000000000,0x0000000000000000}}}, + {{{0x000eeeccbba2a7d7,0x000ad1d77ed0ffef,0x0005e752d769db74,0x00015b240b6200a7}, {0x000597b48ab8cfc3,0x000f97504538ef48,0x0005a2e980da41b6,0x000c0010c2010969}, {0x0007fe53f0d2b23a,0x000a1efe8b48887b,0x0000000000000001,0x0000000000000000}}, {{0x0007b952e8dd021c,0x00026e6b8cb163c9,0x000d62feeed16bdd,0x0007e3db6129cfd5}, {0x0009dcfa4489e9be,0x000f551707d804ec,0x00012a2adb1fbcf8,0x000c05be22830553}, {0x000c9f74b7d87937,0x0008d7e5edd3be33,0x0000000000000000,0x0000000000000000}}}, + {{{0x0004b2f4546a8f7e,0x000d2a223a807885,0x000155358a6954ae,0x00086e4b30349ae5}, {0x000a5982c7a2aae1,0x0006ca4f6415564a,0x000013abb73fd2e2,0x00092d10cb063fca}, {0x0006104963b01aea,0x000bbcd0f1f68f33,0x0000000000000000,0x0000000000000000}}, {{0x00069aedbf43f1a2,0x000ebe1ef2c9b47f,0x0003246ee727e6d3,0x0003f5bd1a5a6120}, {0x000987e829b05fc2,0x000b70444c023806,0x000f0d6e903fa23f,0x000baba52743b14c}, {0x0009994d97e9872d,0x000a52cbf74f834a,0x0000000000000000,0x0000000000000000}}}, + {{{0x0007ea3240feb8cc,0x000792ae77094407,0x0001201418e13db8,0x0003920cbc07e7f8}, {0x000aed56c93837f4,0x00013ae3d2a87da3,0x00044b8ddc44f1df,0x00069328fcaa2209}, {0x000b928d8e381964,0x0007313e55f26bee,0x0000000000000001,0x0000000000000000}}, {{0x000ec4f451bd8008,0x0007bdfe195b7e42,0x000e12b70e68b680,0x000159222b99f5fa}, {0x0000c4e0659127ea,0x000efe86909b5076,0x0008c58b9b667112,0x00047b7026cf16a3}, {0x0007373740ea01a9,0x0000153963989580,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=106 [{1,2,3,..,}]*([2^424]*G) */ + {{{0x000546c7f70bcb2f,0x0007212027084052,0x00013f0d6d83c3eb,0x000eaca7142e7b62}, {0x00029f973a763019,0x000be00c2a025efc,0x0003eca6f6199ab8,0x0002b5618bcc0394}, {0x000b02749fbfb9ab,0x00013c0ae9ab795f,0x0000000000000001,0x0000000000000000}}, {{0x000f295b93c3d712,0x000bf1ab6d0e9f50,0x000bf53d7cb083d8,0x000d744e07076abe}, {0x0005a51a53561f29,0x000d647b915c2fc1,0x0009ead253f84287,0x000e91bd9d6251a2}, {0x00006dd74006b7bc,0x000c92c770e4efe1,0x0000000000000001,0x0000000000000000}}}, + {{{0x0002703ae4d5c812,0x000427e95afaa341,0x000271961d1eb61e,0x0000fb71509a4412}, {0x0007bef04644ecf6,0x00044d4556d64abf,0x0002a7bfd03e651f,0x00094add4bdd9b05}, {0x000438d74bb2156b,0x000fa36c6c16572a,0x0000000000000001,0x0000000000000000}}, {{0x000ae16f96be2911,0x000d52afa2d73a0e,0x0005bc81a5a44951,0x0000fc89324d90b4}, {0x00018cab36337849,0x000cb411d87db838,0x00021f7d6cb710ad,0x0006ad26521480af}, {0x00094a666f370ca0,0x000375d8bc966e31,0x0000000000000000,0x0000000000000000}}}, + {{{0x00082baa7d62ed15,0x000a2a0c35293a3e,0x00068a91eba6b63e,0x000715c2cd81a65b}, {0x000ed39839d20270,0x0001cd1c736c49a2,0x00023b8eae3fee67,0x000006012e0a11a6}, {0x0006b901f0657746,0x0003944e0864a739,0x0000000000000000,0x0000000000000000}}, {{0x00089e4689d04d8a,0x00060efa98dca953,0x000708ea3f1bf018,0x000f379e0740bbd3}, {0x000ab07cf1111586,0x00040f6bae4389cc,0x000c0c62ec14d78f,0x000a8ca2edc050cd}, {0x000eb9e22353ae3b,0x000221497d609a22,0x0000000000000000,0x0000000000000000}}}, + {{{0x000dc15d0ed0cc63,0x0001ce35b540e1bb,0x000973de7aa979e9,0x0005e965745fa684}, {0x0008a999e957c84d,0x00071ead702675f7,0x0002beeec63eb2b6,0x00062262a9349f4e}, {0x0005b027fa151a2d,0x0005b45a63374388,0x0000000000000000,0x0000000000000000}}, {{0x000ade9fdcbfe121,0x00019964a13f4063,0x000ea6e576b23957,0x0009fa2021e1e294}, {0x000f93e8e8d0c1a3,0x000627a8aaeb1db8,0x0001c83a239f73f2,0x0009dc2704f34ba1}, {0x00053062be1591a1,0x0004c975cd21326d,0x0000000000000001,0x0000000000000000}}}, + {{{0x0006b8fb25e6414b,0x0003f3c35210ef6c,0x000fcf4305218c66,0x00044c1aba52d092}, {0x000ad9ddbd46b892,0x000a630015bf7459,0x000b32b246737751,0x000ced081a65d447}, {0x0002e2e5f643a94b,0x00044927f131866d,0x0000000000000001,0x0000000000000000}}, {{0x0004645014a53592,0x0009ee4e5661d1ca,0x0009024d9b573293,0x000159f5f55c84df}, {0x000a0100b8c24f2c,0x0002de6a0aef0178,0x0009e7eedb7e96ff,0x0002067fc1f195df}, {0x000897eb377d8a42,0x000f1229a3daed81,0x0000000000000000,0x0000000000000000}}}, + {{{0x0005287cf2f5961b,0x000cef9beb6f8461,0x000a7eaedc75201f,0x000d39dfabd8969f}, {0x0004252116284485,0x0002945b419a298f,0x0000f03b98d985bc,0x0007be292f990254}, {0x0007d1f59fb5c829,0x00002d379bfc2592,0x0000000000000000,0x0000000000000000}}, {{0x0001179af2527211,0x000f2724128932bd,0x000920719e949c7f,0x000f184c1ce441c4}, {0x00091d4d77a7c7bd,0x00086d6826d00ec0,0x000125b3901be8d2,0x000c6d7da5fd8de5}, {0x00084f0587b71c09,0x000ecbd13d8a200a,0x0000000000000001,0x0000000000000000}}}, + {{{0x00034258888b309d,0x000da5700df117ec,0x000eecc1e03d3856,0x000d69afef5df311}, {0x000ced3bd1e65823,0x0009ca00a3d94409,0x00053de41e7eb569,0x00002f1291722236}, {0x000dbd2ee0ded7b4,0x000a909f40f6e980,0x0000000000000001,0x0000000000000000}}, {{0x00082d95c76e2f3a,0x0006d18779f8f974,0x000021eeb4e7e5b3,0x00070089298e7654}, {0x00003aa90b7f11d4,0x00020fd2c120d0de,0x000ddfdcf3b2e6f9,0x000134293b957e6e}, {0x000d5361a94456e0,0x0004bc62d97be2a1,0x0000000000000000,0x0000000000000000}}}, + {{{0x00047691dfe1ac32,0x00095bb25141cf40,0x000cb8b02c59109a,0x000eb57046b0d286}, {0x000a8b7de816343c,0x000109a4f745b33c,0x000bf34e4de2617d,0x000950869270fe1e}, {0x000a3a1b01ca6170,0x00012e38634a9ad0,0x0000000000000001,0x0000000000000000}}, {{0x000f079109fbe1e8,0x00071bce15260c38,0x00059aca1efc1bab,0x0006c7ad5da68584}, {0x000030bec54d9816,0x0009a9c32e8f4612,0x0001154d88a51704,0x000ebe8180a7efea}, {0x00008abc6ec5ede1,0x000d7de0d26459ad,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=107 [{1,2,3,..,}]*([2^428]*G) */ + {{{0x000019394140932c,0x0009a362bec3a200,0x0005ab884c7a2c57,0x000cab2ba06cf4c6}, {0x0003ee7eda004083,0x000107bb12f5f6cd,0x0008a0ae8649e92a,0x000a5a344683492e}, {0x000cfb24291fc3fb,0x000c3894cfd1718d,0x0000000000000001,0x0000000000000000}}, {{0x000751b1b1d1f30f,0x00043fce22373903,0x00023da45dac2f10,0x0000105df9307b4a}, {0x000892b6d1b2c311,0x000f645131b39e6a,0x0006331d8e317be2,0x0000b2f395de9046}, {0x000d56345f1e436a,0x0001b0547fe7481c,0x0000000000000000,0x0000000000000000}}}, + {{{0x00078f4b7084325f,0x000727f8bc0fa450,0x0001eb36b7bf5948,0x0004268cea0106cf}, {0x000e8c80e06d4c70,0x000c216323300186,0x000b3016e94d5075,0x000d882e86c372cc}, {0x00048a28a053b8cf,0x000343d171001931,0x0000000000000001,0x0000000000000000}}, {{0x000c0287664ae003,0x0002766d8c63b224,0x000b8220989c0738,0x000b967cd418521b}, {0x0004152d761e7a59,0x000fa019e4601efc,0x000f11b41cb4d7c0,0x000dcac0131d9d52}, {0x000c991e4e27a162,0x000789975ffa76aa,0x0000000000000001,0x0000000000000000}}}, + {{{0x000adcdaf64c0019,0x0002a5ef0c2175ba,0x0001cfec89a01d6c,0x000b25fa94296a25}, {0x00089c7ef16afeec,0x000c007e24585f16,0x000bbbc143cb4742,0x000531bcdafbbc24}, {0x000013773a99937c,0x000e19b7d849db6c,0x0000000000000001,0x0000000000000000}}, {{0x000e83417610eec6,0x000e934e9e9d81b9,0x000af4480e73d7b1,0x00086003aa3fea2a}, {0x00079c57f5c52d5f,0x000495c2aa9268f2,0x000922e4d082c9f5,0x000448a981d714f5}, {0x000bdf27622c6821,0x0001ae01e6436230,0x0000000000000001,0x0000000000000000}}}, + {{{0x0007602120347ad3,0x000dd3e9ad407753,0x0002f89c76c52251,0x00044aaf5957d5e2}, {0x0006157c28051b58,0x00069eb597630bd3,0x000a387acac6af33,0x0002aad0ef875611}, {0x000ccba64de2edc3,0x000ff1731d170b0b,0x0000000000000000,0x0000000000000000}}, {{0x000557f665f21957,0x00064548e1ee52c2,0x0000cf955615f44f,0x0002bf2c64b6036a}, {0x0008d89213e751e5,0x0005aa570d34bd85,0x0005ae3bf1cb2e71,0x0003f29e703acc20}, {0x000a34a47fb1911e,0x00089578616fc498,0x0000000000000000,0x0000000000000000}}}, + {{{0x000dc828f86252e4,0x000d09ec8dfb2555,0x0005077092f2430a,0x00078d5b0c56862b}, {0x0002f9b2b116f869,0x0007b998e4dcbf23,0x000fbcf91af3491c,0x00056f1110038af4}, {0x000e0888ba8f38ec,0x00043daf07963a9c,0x0000000000000001,0x0000000000000000}}, {{0x0008da925a008487,0x0004f369188e0311,0x000ec8e641278107,0x000a270db9969754}, {0x00072aaa78c429c9,0x0001a344c1da5196,0x0003087bd58c2f7e,0x00088aeffcfb4dad}, {0x000fbcec5f2532f6,0x000a809230f905b4,0x0000000000000000,0x0000000000000000}}}, + {{{0x0000b4a0ddb8dfab,0x0009deda0e27481d,0x000949ab745b2f9c,0x000dc2842dbe3df2}, {0x00059b98be5c35da,0x0002ed3249c0c912,0x000dc475c0841958,0x0005a3cc60128529}, {0x000078110f67e016,0x00017fe4fa3624ca,0x0000000000000000,0x0000000000000000}}, {{0x0006abd2bfebf71d,0x000617fcd7e4413e,0x000ac8bf7c39197b,0x00022ac8be897f87}, {0x000878bd60d51691,0x000ed69bb2587c48,0x000a51aae1defc49,0x0003d1f77705c7ae}, {0x00021ab01c739c3e,0x000801dacd9f17c4,0x0000000000000000,0x0000000000000000}}}, + {{{0x0002a912967cbb3e,0x000f46687e4df221,0x00007e4eef350ebe,0x000e38a4de052972}, {0x0009d5282d27fb20,0x0006e64b3686af97,0x00067350cb71692e,0x0008437b220d3baf}, {0x00042c826bce6850,0x000642d60e139b00,0x0000000000000000,0x0000000000000000}}, {{0x0004eb43ead3cd1c,0x0007d1cbedb5fe6f,0x0001b45fb1ca7933,0x000a503cea7b46bd}, {0x000fef3868c8b110,0x000f9fe2cde99fc9,0x0006dd89f348e20b,0x00022d25b6cd54cf}, {0x00051adcb2128528,0x000ea79e7afb9c6c,0x0000000000000000,0x0000000000000000}}}, + {{{0x0001fc12e7ca5055,0x00025e91d6adbf30,0x000587a0d31dc42e,0x000cc8a9cecd10d8}, {0x0009d8d1b41ccf88,0x0003b3f96490bd95,0x000e028999c5a513,0x0004c3a0882547be}, {0x000fc3438b39f5f7,0x000fa75af39ed4d4,0x0000000000000000,0x0000000000000000}}, {{0x00002b54c1f9ad9d,0x000dbd0a29ee5fbf,0x00057045bbc4f0d6,0x0000fce2e6fde293}, {0x00033040027dfc7d,0x0003be51ffcda45a,0x00054869841c14d2,0x000156ff1e336d04}, {0x000a634d9fdc8522,0x0003ae619b696b5e,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=108 [{1,2,3,..,}]*([2^432]*G) */ + {{{0x000019f2fc6beb7f,0x000666646b70deb9,0x00090a5717d2e533,0x0006810fd38f5274}, {0x000233bb33bad791,0x0009b6b88efd9526,0x000d5745dad71c05,0x0004ca300788fe6e}, {0x000ed4a6a21a98ac,0x000e10eeff58916d,0x0000000000000001,0x0000000000000000}}, {{0x000da52e803a67cf,0x0007d398c1a5fc48,0x00042b185c01901e,0x0004eda6cb2700b0}, {0x0004eeec867cdaca,0x000dcb7e930a19ae,0x00010a288471dcc7,0x0003198ed17517e0}, {0x0004e8756bb36f68,0x00027df915ed36bb,0x0000000000000001,0x0000000000000000}}}, + {{{0x0009d95ae2bd1168,0x000fb9d6f73873c0,0x00027b3bc9e7cafd,0x00005fa81f14d40f}, {0x000ed4eebf4e66fc,0x000bb6036a4e6b76,0x0009e3f736cb7387,0x000b2277b08a58e1}, {0x00055c0d7a4986be,0x0004d18f830adabb,0x0000000000000000,0x0000000000000000}}, {{0x0005e59f982356bc,0x0000599a3ae563b2,0x000d06e16bf18997,0x00088dac7505891e}, {0x0004c76b3eb973d6,0x0006498b2d10cb08,0x00074af2d6e8e95c,0x000a503f71b8afa5}, {0x00066999bc16133c,0x000bd76f3443fa99,0x0000000000000001,0x0000000000000000}}}, + {{{0x0009cc5a1fdf300d,0x0000ede5aa101a8c,0x00053342a7d32907,0x000bb4fe852398fa}, {0x00036673ceb9c007,0x000c9247b3a94312,0x0001ee51cb682683,0x00075c07f2ef115b}, {0x00083b0143f6486b,0x000d101585ec3771,0x0000000000000000,0x0000000000000000}}, {{0x00080da54d84cedc,0x000e569a2e8c0006,0x000e94c0472e41a7,0x000a41c1b7c713ec}, {0x000e5742d18ccf2a,0x000df3c3763e3162,0x0006b9977417a84e,0x000f165b44c75791}, {0x00045d988246f2e2,0x00095c8287828d1c,0x0000000000000001,0x0000000000000000}}}, + {{{0x000f9c015fdb1c06,0x00099f25767d4418,0x00077c536b749d61,0x00054bfdb54e847b}, {0x000237979776c1af,0x0001eefa220b8386,0x00018acdf9bb4a75,0x0008ce45beb5028f}, {0x00030f98a7dd8621,0x0003023f055e3ab9,0x0000000000000001,0x0000000000000000}}, {{0x000f321b72c7f6a2,0x000659eeb57c141d,0x0000b2255cf53902,0x0009dbecf2a776fd}, {0x000e6453cf8ab4cc,0x0002d5647863e94e,0x00049fe93a4007af,0x00039cf116d00271}, {0x000dc8184a637605,0x00061de7465f7317,0x0000000000000000,0x0000000000000000}}}, + {{{0x000dae8508371ab1,0x0009f5759cf42671,0x0001705cf63d991b,0x000e90ca5edd0265}, {0x000af5c715a86e95,0x00096331dd94a89f,0x0005808565c73b46,0x0004c9ca8a595723}, {0x000561170be646eb,0x000a57b5fc879ce5,0x0000000000000000,0x0000000000000000}}, {{0x00080540adb44437,0x000abb97c57c16dd,0x000be7baf4488307,0x00019e753dbf21d2}, {0x000ac5be29dc51e1,0x000df21c9ad3cc57,0x000a9aa74fbb9193,0x000a11ec7b8e665f}, {0x000c01bfce350115,0x0009909f2c73b53a,0x0000000000000000,0x0000000000000000}}}, + {{{0x000744ab95b71183,0x0005aa6c81566c3d,0x00033310fbbacd74,0x00006a2dd55a7d6d}, {0x0002db05de914221,0x0001ffd4282301fd,0x0008cf7489b2de85,0x000c6f0a02baf309}, {0x0002ce9af0a85604,0x000261197b0884c2,0x0000000000000000,0x0000000000000000}}, {{0x000e2c60da7deb89,0x0008d4b2f2e54489,0x0002e5dee75d5785,0x0005e0e953899934}, {0x000ecd968f2eb742,0x000229cbd8cbbbd2,0x0008dcc6b871d5e3,0x000490c7582e68a9}, {0x000cde3999720429,0x0005eaec58ffb639,0x0000000000000001,0x0000000000000000}}}, + {{{0x00028ab59549216b,0x00005d61d82596b8,0x0009fa679fcfc930,0x000ccf438ed2cad9}, {0x000a40190b3069f6,0x0007f5af4bb6d49c,0x00002f9e7c50f6a7,0x00019940efdd8671}, {0x00002add96c554a6,0x000e007d80786684,0x0000000000000001,0x0000000000000000}}, {{0x000ad0fb54ef0a37,0x00046af5d15fc57d,0x000b825452e8ccd9,0x00044232c710cb6b}, {0x000cf88855ecdc6f,0x0001d6d2b8f40b3c,0x000436e2bc3e510d,0x00007b1ff42aeddf}, {0x0000ed88484552b4,0x00075c1505c18430,0x0000000000000000,0x0000000000000000}}}, + {{{0x000a55f734e8c16e,0x00026ef042fa2f6e,0x000b24c1848ab1d2,0x000acbe86862a1dd}, {0x000651f4168e7413,0x000d596e07914083,0x00079ca23961d189,0x000e6d53679701b3}, {0x000cf35fa05ec7b7,0x000d20d7f6b70713,0x0000000000000000,0x0000000000000000}}, {{0x00018785b4c707b1,0x00038676095f2dbc,0x000e28a0370a0054,0x000af09e50c89610}, {0x0000f144bba0bfee,0x0004cf6dd7455cf1,0x000e722f509d9783,0x000b05c279e5f94f}, {0x0001244fe8092deb,0x000153b314f061e7,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=109 [{1,2,3,..,}]*([2^436]*G) */ + {{{0x000803868bfafe94,0x0002ddeb7719717f,0x000911e1ad005b4c,0x00076f1e0df34f87}, {0x000b29958d5da570,0x0005d1ebf66f49ec,0x000f5712ca7b49e5,0x0005b2ff1b32fcb4}, {0x000f97d3d42a971c,0x000804838bb32749,0x0000000000000001,0x0000000000000000}}, {{0x0002e7908789eb65,0x000af786529c3ba5,0x000e71594737ddd9,0x0005400f6dd64d51}, {0x00039922bf016830,0x000db4bbaa21ca42,0x000935d8c94ee851,0x000c80623440afda}, {0x000110efc0a576c9,0x000396d79f58dcd9,0x0000000000000000,0x0000000000000000}}}, + {{{0x000b38ccc9df95f7,0x000d1849ee6062ae,0x000164333dec1411,0x000e81b4a4a4727a}, {0x0009c74b241dc566,0x0005069fb7290aa5,0x0009d322865cb6d8,0x000ecbd648392838}, {0x000909864ce3c8f7,0x000ac9fef248d28c,0x0000000000000001,0x0000000000000000}}, {{0x00061d435126259d,0x0005a6b96bec8542,0x0001f509bbc62a6f,0x000ddbc8b43f9c90}, {0x0005e94118466c53,0x0002a386779ad388,0x000db58d109dd2e2,0x000aeef4f2af60b6}, {0x000bc0cd7adf1adc,0x0008220fb47811d5,0x0000000000000001,0x0000000000000000}}}, + {{{0x000f1b4d9b2b9d58,0x000635d4601e4549,0x0005712ad28d37e9,0x000e19b42c3143dd}, {0x000fdc6366f04af7,0x0004b34637aa565a,0x0005b2ac12b452bc,0x000277fe7f5bdd13}, {0x000c6ff31feea8b4,0x0000c45cdaec8e9e,0x0000000000000000,0x0000000000000000}}, {{0x0007813178366af1,0x000e83664c221b3f,0x000afac4ecd652c3,0x0007c4666da93d03}, {0x000652ceac0d6cfa,0x000b1a4cf4039d2b,0x0005c58a6eb1946c,0x0007422c9b53a228}, {0x0007349ed1b4d836,0x000afa4c55a379cc,0x0000000000000000,0x0000000000000000}}}, + {{{0x0009ee173ac4010e,0x000fb942f467a4b5,0x0005770a30141b66,0x0004198802513df6}, {0x0002c7e0148f2f1d,0x0009a1b6c6f54abf,0x00056f2bb47b51bd,0x0004f5846505fba9}, {0x0002a3ddcb02618f,0x000a7b69ec8c6450,0x0000000000000000,0x0000000000000000}}, {{0x0004b69533aa9231,0x000b0804316d8f19,0x000006107c58f7bc,0x000310f29f43afd0}, {0x000e7a15ea5dc32f,0x000849a363e2b91a,0x00074452b4966c63,0x00071d63455b6a45}, {0x00039b535b8835c1,0x0005091ae86f54cd,0x0000000000000001,0x0000000000000000}}}, + {{{0x000de1ec8614daa5,0x000f0834c4db4a7b,0x00048a29f6ba70a6,0x000be231587d1015}, {0x000bb998c9a2049d,0x0005eedcf88aceaf,0x0008878e6b738f7e,0x00019b693ecfcab0}, {0x0008dd1ddb374ede,0x0003be7a6cd94f00,0x0000000000000001,0x0000000000000000}}, {{0x000bd130c16a2f12,0x000343c20757ed83,0x000228e06b217bff,0x000046a91bef19ad}, {0x000400e88da5cc51,0x0000a9f961011b58,0x0003f9049ae6f047,0x000e9d079a03c6f8}, {0x000401435912f072,0x0007490b78fe3f9a,0x0000000000000000,0x0000000000000000}}}, + {{{0x00081fdfff4245fc,0x00050168c14a6614,0x000764d4d2b84edc,0x000aaa60be356501}, {0x00030e6771588736,0x000714d50c0c40e3,0x00029157b8fe8875,0x000a9215aac4fbdf}, {0x000b4091b549e25d,0x0001ad442d2b0058,0x0000000000000001,0x0000000000000000}}, {{0x000ee6a91c584138,0x000d84934568ba88,0x00010dd1585307ef,0x0009b046cb644b24}, {0x000dc2bd376e1321,0x000cecc49bef0c68,0x000f76556c2d2d1d,0x000d810f8810c907}, {0x000a20da5052b3dd,0x00079ec3448a3c1b,0x0000000000000000,0x0000000000000000}}}, + {{{0x000084366f4dadcf,0x00087f5cc0a55e0e,0x000a139c3fe7ef01,0x0003f2e749d53f7b}, {0x000ffd809a727542,0x0002e74f9e5a94a4,0x0001929541f08d0e,0x000114dd07932254}, {0x000f53ad14915984,0x00078a408f5bb7db,0x0000000000000000,0x0000000000000000}}, {{0x000b74adedde4d64,0x000eeb46e287111d,0x0001ad3605f3b22d,0x000070fc8863541b}, {0x00093fdd530f51fe,0x000f3d69c04af47e,0x000f551d93cb6477,0x0002bc684cde6d16}, {0x000154a9f50cd685,0x000e9f16ac0cc2b5,0x0000000000000001,0x0000000000000000}}}, + {{{0x0006190bccf09089,0x000de2a4a386bf1b,0x00095703cbb17c47,0x00013d223bf84891}, {0x0007a124742673f0,0x000290f2b86fdb82,0x000f44e50e9b7e10,0x000ecc65826079a8}, {0x0004d12b589a9228,0x0001b683a119d1ab,0x0000000000000001,0x0000000000000000}}, {{0x00025950f93cc637,0x000b7a6b02229a2d,0x000fb0617d538c46,0x0001dd7b6bc581dc}, {0x0005b6b522d590ce,0x000133e3f5d0dcdf,0x0005bdf5cce47e79,0x000e21b8ecd0d71f}, {0x000ac21b717d9aef,0x000b887b6090257a,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=110 [{1,2,3,..,}]*([2^440]*G) */ + {{{0x0009fcba235d8a1e,0x000ca359a63f98f8,0x000f60025c086d07,0x00018d4e590915cb}, {0x000c915cc7c3b68c,0x000933480185575e,0x000511ae8d10d820,0x000082704b904789}, {0x000a4e997db2e76c,0x0008c3f5824d99f6,0x0000000000000001,0x0000000000000000}}, {{0x00053628d8f32dc9,0x00000ea757555069,0x0008537e14185044,0x0007f7a0609d8295}, {0x000c55da70118c7b,0x0009ad1223c50379,0x0008629c936f6ea7,0x00054f7f839cbde4}, {0x000f8def61ba0172,0x000f5c1bef09ebdf,0x0000000000000001,0x0000000000000000}}}, + {{{0x0002eeedc5fe3f41,0x0004f9330d665ae8,0x0003f5e64a30753a,0x000e92f39e477096}, {0x000aa07f9d297ef9,0x00048c3ddf388062,0x000e61e60ab0df5c,0x00095a47567e55e6}, {0x00066d0129872a6f,0x000953425f368c3a,0x0000000000000001,0x0000000000000000}}, {{0x000b7cc7bf66ffa4,0x000f16b2825eba03,0x00090e67535ba3ce,0x0004aef14aec5704}, {0x00001511ac67bcc3,0x0001002739d95c0e,0x00029220f4f36575,0x0001465557ab45e9}, {0x0009abecf1baabf9,0x000fe98337c9760e,0x0000000000000000,0x0000000000000000}}}, + {{{0x000025751d2b325a,0x000cde6a01039da1,0x0005ba8462228499,0x0003490747232500}, {0x0001a523417ab4da,0x0003451baf54b07c,0x000516f3fa7e4ffd,0x00042fbff2147ce5}, {0x0003f1b0afc522cc,0x000cb395c7010ca3,0x0000000000000001,0x0000000000000000}}, {{0x0005ed5f55af51c6,0x00015b980e568466,0x000a5a1b30bd5964,0x000bb04e8834a37b}, {0x000becf282494fee,0x00040dc6ceb29d17,0x0004a868d5399a53,0x000276012bcea50f}, {0x000c769aa83faa31,0x000d8e6550a0654b,0x0000000000000000,0x0000000000000000}}}, + {{{0x00066fa0ef4dbc14,0x00071d134a53f7ec,0x0001ee39cdaa7b28,0x0009b3f183070c04}, {0x000c76da77991974,0x000916f1eb867841,0x000b27421e5438ae,0x00051b0e12d8e409}, {0x0005e08b83842a6e,0x00094374b9e008de,0x0000000000000000,0x0000000000000000}}, {{0x000a4cba763a6340,0x00013308e07acdfe,0x000db2143a906789,0x000fe6dd815c887a}, {0x000e2a9d2043c85f,0x0003ceab79a68d05,0x000986393674d33d,0x000d12ee73ca7a8a}, {0x00003b9bbd54b7af,0x000b702eead11264,0x0000000000000000,0x0000000000000000}}}, + {{{0x0000b987f57f80d5,0x000d4367c06145fa,0x00034438e79fd55d,0x0000f9f8e8ca9c52}, {0x00036810f12c2fad,0x0003ec5af1a97a71,0x00085613d0eababb,0x000aecb3da01b7b5}, {0x0000150798aadf26,0x0008029cce9cadbb,0x0000000000000000,0x0000000000000000}}, {{0x0005a72e54383960,0x000decd025e95126,0x0005b2c914390e3d,0x000864784955e972}, {0x0008cdae63ed0053,0x000aa5ded860c28f,0x0001dd80fb99e774,0x00020f07854bb74c}, {0x00091581c2caae0f,0x000d48069f6ba766,0x0000000000000000,0x0000000000000000}}}, + {{{0x000f135836126647,0x0004208b738df6f5,0x000786c7341e91f4,0x00084ed91ae2188a}, {0x0000b08e3293bda3,0x000e09af3119b1a0,0x0003662652676669,0x000fd07b9f37322f}, {0x000ae129d764ea40,0x000b85c16a911b11,0x0000000000000001,0x0000000000000000}}, {{0x000021aec95ad18b,0x00080eeb3197c759,0x000dfd4a4334daed,0x000ff78d606234ad}, {0x0003ff98a1d73ab3,0x000c9cac669f90a4,0x00063cf99bed1762,0x00028f03fcd45e80}, {0x000b17bf750672a2,0x000879027e080bfd,0x0000000000000001,0x0000000000000000}}}, + {{{0x0006a647cfcf8e23,0x000d745dafe04723,0x000c3212b4d30081,0x000945780f548f13}, {0x000a0d885e14f51f,0x0006ed3092941059,0x000651e189c478f0,0x00007a26e8c75042}, {0x000a14b52f36b6ee,0x0001a032dfdec009,0x0000000000000000,0x0000000000000000}}, {{0x0008ca67379eb582,0x000b6f527f0a50be,0x000adaba76e4c6be,0x0009b987fc7fd1fd}, {0x000a047c90091990,0x0006d6f45b992155,0x000e37e2da697e00,0x00011a38bcd1740d}, {0x0009b93e8ce3867f,0x000b60503be8b250,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c16ca71ab9de2,0x000d4b4d3bbd16e8,0x00053785c45519f4,0x000fab776454947e}, {0x00019d9b9416f1aa,0x000337e34e6883b4,0x00041570208ba8ab,0x000cab67774a7e58}, {0x0008ac5165a84d18,0x0003a977b69d3110,0x0000000000000000,0x0000000000000000}}, {{0x000e5bcfd652943f,0x000743cd5e892a91,0x000502744c85aa27,0x000bb91ba0414bf5}, {0x000f8bc4ef773e26,0x000f9e301b8bcd45,0x00028983589038c8,0x00019a5f5e5a30d4}, {0x0005c6671a609f77,0x000a3aade09eb4af,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=111 [{1,2,3,..,}]*([2^444]*G) */ + {{{0x0008fa162e0dbf60,0x0003f486c08a442e,0x000d94f9cd2b15f8,0x000f2a2350bb6a89}, {0x0005b606cc5727bb,0x00003f198a74b132,0x0001a6f73b79d3ab,0x0001c95046a9cf73}, {0x0005ed71e298efd1,0x000274d622bb2409,0x0000000000000001,0x0000000000000000}}, {{0x000c383b3b59eae1,0x000e81b2f1927508,0x0000d888be6e14de,0x000fb612ce4b12e8}, {0x0001378248f53213,0x000330dbca43092c,0x0009ef5e40c52b24,0x000e9d869889952b}, {0x000c05f4131f1126,0x000503fd03ae1dfb,0x0000000000000000,0x0000000000000000}}}, + {{{0x000868b8bbdfd1e3,0x00069e244e266c6e,0x0007c7bf40b05a43,0x0003b7e1e296776b}, {0x000e22cfd9c18aed,0x000ea90d63ddbc31,0x000929198abb7bc1,0x0005791f36a5e50b}, {0x000737c71c2f87e5,0x00024f75c6d8e7fe,0x0000000000000000,0x0000000000000000}}, {{0x0004c9eb7d596ad7,0x0001e1a0fb486ad5,0x000d820f02c91d1e,0x000160179160a67f}, {0x00057f1163f25e5d,0x000cbc9a928b61c5,0x000df9b84ed79f2f,0x000854ba69556a33}, {0x000c5cb8ac8febe1,0x0009a7ec5a344343,0x0000000000000001,0x0000000000000000}}}, + {{{0x000a1f2d5d90dce2,0x00048086d7fdecb8,0x00005e3d618ccc17,0x00081c1e93d5b7f5}, {0x000a9cc469c44a02,0x000561bd59d9df3b,0x000788578aa5fba5,0x000f6a295757cdc7}, {0x000387a16bd50e1c,0x00029d1a6695b773,0x0000000000000001,0x0000000000000000}}, {{0x000b40770bcc13d9,0x000a0b4cb7c0701f,0x00058a0433c71ad0,0x000e93cd5406bd3d}, {0x00098c999eda26c3,0x000092f416481951,0x00047a1b9bfd4b10,0x000f450e0ae46263}, {0x000aa14d201a717d,0x000a1f0cadae7402,0x0000000000000000,0x0000000000000000}}}, + {{{0x00025a8ecef36f15,0x0000495828c615a7,0x000ac113424f603a,0x00042c1687a77e81}, {0x00098761c2762346,0x0009b1a4749d0db2,0x00097828ac3391f2,0x0005050c5b69143a}, {0x00078b0a25ae0092,0x000d6c144730edc5,0x0000000000000001,0x0000000000000000}}, {{0x000bfa3844e6437c,0x000f55606aeb933b,0x00097e413566e3da,0x0001c1ae4263527e}, {0x000823a0378934ac,0x0002143f588c3363,0x00027262bb7d997b,0x00082419935b6941}, {0x0004eeef99cb555c,0x0009b82ef7f7cb72,0x0000000000000000,0x0000000000000000}}}, + {{{0x000f0062dfac9c48,0x00002d81a18ce131,0x000ddcf23a0713ab,0x000a1fcdc4c43fef}, {0x0007bce6f244518e,0x000ad87348b34742,0x0006967b2b850180,0x00045248a8e5aac5}, {0x000029c18af90606,0x000027d44e26ac08,0x0000000000000001,0x0000000000000000}}, {{0x000cb47b7d6fa01c,0x0000fa06669eb12a,0x000ba3426d87d938,0x000cc362d25b1811}, {0x0005c938c9a5e9c2,0x000513be5311d61f,0x000c741f99e49306,0x000858e6ea44630f}, {0x00001034df68ea15,0x0009fda601eea831,0x0000000000000000,0x0000000000000000}}}, + {{{0x0001335ca463c947,0x000056d6526151f8,0x0003c494f0f999ff,0x00082dc10a7433ca}, {0x000067fa3bcdc41b,0x000c803bb39af11e,0x00031fc0ac7bb35e,0x0005b5e185aa457b}, {0x0006ab2cbed55591,0x000c973304481958,0x0000000000000000,0x0000000000000000}}, {{0x000c07b3b2112108,0x00057bae813666bf,0x0008eee1f424f0a9,0x0002122582cf0958}, {0x000314daeb7bdc33,0x0004de4e23458ec0,0x000a768d0f97884e,0x000a1655c201fc50}, {0x000f1a537b424f36,0x000a6c0cdff48114,0x0000000000000001,0x0000000000000000}}}, + {{{0x0007e2caf6bd29ac,0x0003376357d313a7,0x0007372bec99c462,0x000dae50c449ad64}, {0x000a9851fff00473,0x00047fec5d20efdd,0x000826de033e859f,0x000f46311785804a}, {0x000104e3184caf5e,0x0001ecedcc25e31e,0x0000000000000001,0x0000000000000000}}, {{0x000e47c0beec6da0,0x0001b1920715db74,0x000ae32e6188b124,0x000b5dfc6ee77c87}, {0x0006b132638e4040,0x000b86b665e1c6f4,0x00028fe14b060006,0x000b71efffb869b7}, {0x0006eab86370f188,0x0003e80379ec88c1,0x0000000000000001,0x0000000000000000}}}, + {{{0x000f8cb24f3b637b,0x00076af131c20322,0x000815ccfff0c2b0,0x000fbdbff056364e}, {0x00060c8028853dbd,0x000af0ee0841ab57,0x000da56ca93ac088,0x000d301350923094}, {0x000228a255054010,0x000a058e7dde6774,0x0000000000000000,0x0000000000000000}}, {{0x0005176aa5c512c3,0x00020d3779fd868a,0x0007658fb3dbe164,0x000cf13041f45c5c}, {0x0005049dea64d110,0x0003f6746e19e0e3,0x000a39087ca45757,0x0008108ab4e2fe7d}, {0x0009cce4e874c545,0x000b791d64965ce3,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=112 [{1,2,3,..,}]*([2^448]*G) */ + {{{0x0001f7c72b336855,0x000f76e247b483cf,0x000202781dc97b6b,0x0005bb58c0f81747}, {0x0000c92efba888b6,0x0009612af59611a6,0x000ccbeaf54a57cd,0x000ef8689ba520d7}, {0x00091cc366d3cbf9,0x000056dc1abfe905,0x0000000000000000,0x0000000000000000}}, {{0x000a04beccd53894,0x00021b1e600b02d4,0x0006c3ebe8f2a150,0x00007cfe9586be60}, {0x00062f4028af5b85,0x000d392e8954dda7,0x000dadc519d37584,0x000b58c3813ebde8}, {0x000557ce681db641,0x00006323fa3b9991,0x0000000000000000,0x0000000000000000}}}, + {{{0x000327209a17e11d,0x000d294da0ba85ba,0x0002e3b7145fac1e,0x000fef1248cf218d}, {0x00034de112f175cb,0x00094a8f1676f3e2,0x000f9c2657870861,0x000518958d56de1a}, {0x0003dbcba495c76a,0x000cbfa5e9c9c9dd,0x0000000000000000,0x0000000000000000}}, {{0x000fcebaff9f1e95,0x00070930a1b712b1,0x0008296f1f273d82,0x00071eddfa6e1f41}, {0x000af7dd190815ef,0x000f6fda9bc4a2f8,0x000482585b1234b2,0x000e23556036541a}, {0x0001ac1cc79e6b22,0x00079988ea71f991,0x0000000000000000,0x0000000000000000}}}, + {{{0x000a9c0f0a888100,0x0007d24e7957d723,0x000347cf9592bff9,0x000365bb77516430}, {0x000532639eeab522,0x00075ffdcf80bee0,0x000715b06a25a527,0x000f841be2cacf87}, {0x000bc301ef1a4c9c,0x000be09159817e68,0x0000000000000000,0x0000000000000000}}, {{0x000ae077a00f18c6,0x00066d4baba09883,0x00026a600e57c245,0x000d614baadec7ac}, {0x000df9f1c7a24538,0x0006e61c4241b694,0x000e25d7cba200ff,0x0002e6c811de028a}, {0x000a7a2ca09bc137,0x0004b214e3d38684,0x0000000000000000,0x0000000000000000}}}, + {{{0x0002cc5d8d86efa4,0x00086c8ee779a0a2,0x000fd2159545dd5d,0x000c7262fd5e2c81}, {0x0008275f13cf7ab1,0x000759a0b74f36ad,0x0003c0c8c3ddc91e,0x000d10948a51d222}, {0x0007160cf9b2c7f7,0x00060f285822b597,0x0000000000000001,0x0000000000000000}}, {{0x0001e962392851c3,0x000b50d7c127ef1a,0x000d984c5287e5e2,0x0005ce7d3999dfdf}, {0x0006fd1373907aad,0x0007f8f0825c8472,0x000b5c55ebbc32d9,0x0002bd51b3a068dc}, {0x000935287a1b4f59,0x0006c8f3eb9dca36,0x0000000000000000,0x0000000000000000}}}, + {{{0x000e20e45266c880,0x000d20d089ee45a2,0x000924531cdc6e8e,0x000f7ce2e1fa4436}, {0x0002517f9a18e676,0x000d570398bec610,0x000dea4d0f9eb1c1,0x000c9914ddf30f65}, {0x000cb21d0d421e9a,0x0002d38fb5fd0304,0x0000000000000000,0x0000000000000000}}, {{0x000db46cd0b507ab,0x000ba53f3ce83b8d,0x000a8aca81c50a03,0x000245a83198cc38}, {0x0008bb4ffb6faabe,0x000c93601934e1ba,0x000061b676968674,0x000710e9ebcf5033}, {0x000185f4475389bc,0x000687b9bd9763f5,0x0000000000000001,0x0000000000000000}}}, + {{{0x00029514447f1de3,0x000d950771604321,0x0003a8cfdfffb078,0x00022ba412e08811}, {0x00097ef9f248a108,0x000b82c1196accaa,0x0001f8d1787041bc,0x000ec8430565e6d4}, {0x000eacea7ff3e1c3,0x000896b9bec2206b,0x0000000000000001,0x0000000000000000}}, {{0x000508946b504e8a,0x00067cc71474da28,0x0009ad4094d373bd,0x00068f87798411db}, {0x000b0b90d5d507cc,0x0000617d88846f61,0x000a437ea42aac2f,0x00012a4ea205a926}, {0x000adcb25d613e80,0x000fb7ef615aab09,0x0000000000000000,0x0000000000000000}}}, + {{{0x000127afd82f8260,0x000f4ea3f09c69bf,0x000b49f3f0e43a3e,0x00052457b8b9c3f9}, {0x0004fe6ce00409f0,0x00044efa3e31f441,0x000e3e0240cf1253,0x0001f0b4a170983e}, {0x000d166b0cc9e8c1,0x000b455417a2207f,0x0000000000000000,0x0000000000000000}}, {{0x0000fd522cdf167d,0x000b48039daf9a36,0x0007cdbcd7341cc4,0x000c7db3b7805076}, {0x000641b0db786edf,0x0006ea9b69b8bccc,0x000eded2a4e25e3e,0x000651b8f0911cf0}, {0x000614b601e95ba9,0x0005000aeb19e877,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c0c51d56f967b,0x0008dddbb7684495,0x00035dbc45f7bb76,0x000f9e6deda49098}, {0x000e73639006a39d,0x000878e5a247f77e,0x000cd83d141b2c8d,0x000804a47e332c8c}, {0x0009dc7a02d4027f,0x000b1b9934bb002c,0x0000000000000001,0x0000000000000000}}, {{0x000777a838efe004,0x000368d9919c1d8d,0x0009dd721650f685,0x0002b1de892863f1}, {0x000d78f2b25a32a9,0x0002a4320690ff3d,0x00005a4af7bb8bc1,0x000cd763efcfe035}, {0x000701c70cf4f256,0x000443ef26775353,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=113 [{1,2,3,..,}]*([2^452]*G) */ + {{{0x000e0800cd422057,0x000077d1b7504d49,0x00004fd80e159ebe,0x000a18a8714afb4f}, {0x000de28810d8b90e,0x00019cff83f02c3c,0x000eb9e19367a867,0x0009952bac438786}, {0x0000e0748ecceb4e,0x000faa55aefa6646,0x0000000000000000,0x0000000000000000}}, {{0x000aa315b1f2623e,0x0002b544f96e097a,0x0005dae237e7a5db,0x000873d0a9362519}, {0x0005569799223163,0x0001a58ea84d0fbf,0x000d43fd3bb728ac,0x000f100cfe43661e}, {0x0004f55c6f1cd21a,0x00091825dcbe9fa2,0x0000000000000000,0x0000000000000000}}}, + {{{0x000fc794a5858d75,0x000a4affcd84d641,0x0005082ece4f5985,0x000b4853cf3bb3f4}, {0x0000bb1d8af65850,0x000953dc3e670d98,0x000424ef579458a6,0x0008ac2f2e4a7963}, {0x0000d771e540b685,0x0009be1f5fed2292,0x0000000000000001,0x0000000000000000}}, {{0x0001be223c4864af,0x000fbb662c4dc573,0x000a57017521419c,0x000c0240d65099ca}, {0x000b13af88f3bfbc,0x000bc4861e1643ac,0x00067ed67405bcdf,0x000d9351f1c835f0}, {0x000d0e188cb8018e,0x0001f8d276f971ee,0x0000000000000000,0x0000000000000000}}}, + {{{0x0006a9e449621f3d,0x00052e91ee418d60,0x0007e5ea0e7d2dd0,0x0006f73c92e787f3}, {0x000cc4508ea4869d,0x0002d461acfe248e,0x000bd24529ffcc1a,0x0002f90dc5dc22dc}, {0x0002f8abba336456,0x00056cd254e4f954,0x0000000000000001,0x0000000000000000}}, {{0x0000aa036c0262fa,0x0002edc4f4234a99,0x0001031cef9b59ee,0x000145d5d5d4b081}, {0x00087df2c037b984,0x000a2d6af2c7b077,0x0006ff431d568532,0x000e309a7c9b6d5e}, {0x0006a3ee9bb40c66,0x000c81af9db41cda,0x0000000000000000,0x0000000000000000}}}, + {{{0x0000f6f067a194c8,0x00066d7ad7abcf3a,0x00041cf832b531ca,0x000f470a4ac4965a}, {0x000cbe00766a00f9,0x000432af80a92657,0x0008968ac40c892b,0x000cbd44ededa94b}, {0x0008d46204be8b74,0x0005dd98f760bd2a,0x0000000000000001,0x0000000000000000}}, {{0x0007f464f2479db1,0x000c0fdb3e7dcb2a,0x0001a96b28900b58,0x0000a2993c1d7ee5}, {0x00057bf0ce9357a3,0x0001f5d39eb49f5f,0x000fb8b8f0c1970f,0x00060718d4faac9c}, {0x000ec4ed9e1c25a3,0x00004bb0d7504b66,0x0000000000000000,0x0000000000000000}}}, + {{{0x000c9e1b2fe9b1f5,0x0008d34221feb754,0x00042a589958e706,0x00008c19119e3c31}, {0x00021ac738645a87,0x0004d7be72a33383,0x000a72a39907dc71,0x000beb5759bb91f8}, {0x0009abd4b13e9374,0x000b96398db67d76,0x0000000000000000,0x0000000000000000}}, {{0x0000945a75d3799d,0x00031dd530ad32ad,0x0000c605c645aa0c,0x000c247141d0f230}, {0x000876cc8fca8350,0x000943188b899279,0x0001484afb9ffa8c,0x000536d0aa571f07}, {0x0002b7a0c7fbe8ba,0x000c736f36c17e51,0x0000000000000000,0x0000000000000000}}}, + {{{0x000618a7bb29e0e4,0x000c09c2bff10aed,0x000781dbe57f3993,0x000965b0e2b9baeb}, {0x00005cf5dc5bd4b3,0x000ce5975c6e0f92,0x000471ab0fcf2bf8,0x0002a2d43bae9f5d}, {0x000bcafa18f22cc7,0x0005911e3b357902,0x0000000000000001,0x0000000000000000}}, {{0x00057df200a9fe3a,0x000b50d956c845fd,0x0007f211bf361887,0x0009b4007c382ac0}, {0x000ee4f517e7f4f0,0x0008256df5e905bd,0x000fa86a9368d1c3,0x000a7df56d37c426}, {0x000efab6ec26a5b8,0x0005cdfe9cfac9c1,0x0000000000000001,0x0000000000000000}}}, + {{{0x00082202f99826cc,0x0004867eca10dd40,0x00071f5c4e281cfc,0x0007b5da880b8221}, {0x000325e5cd3e67f8,0x000671c0e0906564,0x000df4bd6ef65d7c,0x000c8a693695da96}, {0x00076e79dcdb1aea,0x0005bd96f080eaaf,0x0000000000000000,0x0000000000000000}}, {{0x000240c6e69e5a49,0x00080c138884ae0a,0x0002f7e55e87844e,0x000a57d17ee0d45e}, {0x00033a1990475a82,0x0004fc5139e3efb1,0x000093a8d1fcb820,0x0007e2fdf23f8eaf}, {0x000aec2eac127538,0x000a0742581e77ed,0x0000000000000001,0x0000000000000000}}}, + {{{0x000de604b9db6d8d,0x000262da62b65572,0x0009db8d0d3fb7d9,0x00067b7f8e9c2aa3}, {0x00074f2912d3c9b8,0x00079e6d831a5ad6,0x0006f3cc6935b1c2,0x00009a75e08b8223}, {0x000aaa28cfcf8f6f,0x0004f75ff407271b,0x0000000000000000,0x0000000000000000}}, {{0x00024706c120a90c,0x000b4f991d9aa4ab,0x000767e06952ed85,0x00006ffc0793e3c1}, {0x0003d6115d975f7d,0x000c57472c131644,0x00024430b8651df9,0x000bda1a64f28b97}, {0x0006db846f1bbd8d,0x0002f562aea1650c,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=114 [{1,2,3,..,}]*([2^456]*G) */ + {{{0x000d810c86eab937,0x0008e6e6eb51bdd5,0x0009bc2e3a0e4219,0x0007e391c4b3dc48}, {0x000736ed77fe5eda,0x0005e60972cd0e3d,0x0001aaa05f70f41c,0x000db07669f426a5}, {0x0005914839830a47,0x00029045b98cf434,0x0000000000000001,0x0000000000000000}}, {{0x0009d9c576932524,0x000409f4c3b8bddc,0x000d467dc9295086,0x000bdef23e6cf0fa}, {0x000729684c1e0fbe,0x000c3a23011daa3a,0x0008c4ef40ca0da8,0x00034dda12c0850f}, {0x0002c5e51990ccbe,0x000494c2f0adaf8f,0x0000000000000000,0x0000000000000000}}}, + {{{0x00087b36a390065e,0x000c93a9bb0064f8,0x0004572329d651f2,0x00010e01d988aed3}, {0x000be48541e9e6f5,0x0000ac10a8abc023,0x000b700a8621efc9,0x0003eb208400f943}, {0x000d85b1c768bb3b,0x000f42634af0db64,0x0000000000000000,0x0000000000000000}}, {{0x000e0a10a250b4be,0x0000634e42e593a7,0x000adef0026acaa8,0x000002da6f2f96cb}, {0x0008dca66aa2e955,0x0005a69e8157271d,0x000f32666dc76291,0x000c378977ddcf29}, {0x0000e5eaa07d6619,0x0002a548e47a94e1,0x0000000000000000,0x0000000000000000}}}, + {{{0x000cd510985c6ead,0x000d1cc399a1876d,0x000fc77243f5c966,0x000c3b2c4abf82c9}, {0x0001efaf22c713ae,0x000a22e1704988df,0x0002a2d8f28a287a,0x0000f724ea967d19}, {0x000ed48e76179ade,0x000ff318acdc5b8c,0x0000000000000001,0x0000000000000000}}, {{0x00063196ac8ad685,0x0002708e70052b0c,0x000d8ff45f4a08be,0x00064862e9bd37fa}, {0x000efcc39748e461,0x000dcfa2843cc067,0x0006ae8688367317,0x000b8aabfd38c458}, {0x0005642ea85ecef5,0x000873af78e84b81,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b763fd4a23b33,0x00004ce905f61100,0x000ad5b3ffa31c96,0x000817bf059b27d0}, {0x0004f957d997fe85,0x0008adabfcc9cea6,0x0002fa1bf24cb58a,0x000afe218a3174dd}, {0x000ee69ca08cb0de,0x000fdf310fed00cf,0x0000000000000000,0x0000000000000000}}, {{0x0006e131eb000160,0x000a4d18c779a4d6,0x00013180dd747a1f,0x0009340c23f27ad3}, {0x00004df4a2f35316,0x000ec77b35a8c2be,0x0007fb23a1f8aa1a,0x0009e69edc272eed}, {0x000d58ddb6110abc,0x00042ae7590226a5,0x0000000000000000,0x0000000000000000}}}, + {{{0x000f3cc8ae818a77,0x000f54ede150dbf6,0x000c2f06bc371295,0x0006ab2d173c2266}, {0x0007fda0b8b46bd2,0x00070909e3958aad,0x0005242beb035184,0x00006b5aad800c13}, {0x0003a70dce6b782b,0x0007761ebe42a4f4,0x0000000000000001,0x0000000000000000}}, {{0x000ee87f40792877,0x00080b5c7acfe8e9,0x0007c5775efcb053,0x0007bf0a2d540e71}, {0x0008c839d644d450,0x000d1ff451c6b81a,0x0001b7ea45f8834b,0x000306bfc9c3c653}, {0x000cabe92f1a607c,0x000c13152a3731fa,0x0000000000000001,0x0000000000000000}}}, + {{{0x000ad1ff805361f9,0x0003b3536327ce4d,0x0005b0b62672d1bf,0x000ddf3847367af0}, {0x000278798d158f13,0x0004fbc252a948f1,0x0006fc03d0fe92b8,0x000c0138676b978e}, {0x000ea4ba8ef9334e,0x000f98b13cf22496,0x0000000000000001,0x0000000000000000}}, {{0x000ac5c627d693fd,0x00015f19d6d21fa8,0x000b4a2fb70b6705,0x0002c39928de441a}, {0x000db0fd9e912476,0x00096888c13371fc,0x00013f185fb68ee8,0x00092d86c189d0e2}, {0x0001cf6facea1f84,0x0005b949b94eff15,0x0000000000000000,0x0000000000000000}}}, + {{{0x0003e85508c930d5,0x00050167c798885c,0x00018ef2850f4f27,0x000547f00d7f6b01}, {0x000899cb15ed2f40,0x000bc417a989d6f1,0x00008165c9378941,0x000bb0e0c28f2218}, {0x000839d9572ddfe6,0x000d6ab0b70e2e95,0x0000000000000001,0x0000000000000000}}, {{0x00092c31b5a755fc,0x0005623261be8d00,0x0002ed776ec81547,0x000ed7b92bc72da3}, {0x0006d943fbdb47af,0x00096c45167b5a5d,0x0006fef44f208158,0x000d5bd5c28e23f0}, {0x000da432e8f4f6c6,0x0007b3355da25eae,0x0000000000000001,0x0000000000000000}}}, + {{{0x00076c8ab8c349a0,0x000850693d15749f,0x000d18a6991254fd,0x000f60f54944e4ef}, {0x000febc73879cf78,0x0004c63e00125696,0x00042f68e1e2dbe0,0x000b688fd93a88e1}, {0x000f1e83abbe7321,0x0003aaa7b2fa13d6,0x0000000000000001,0x0000000000000000}}, {{0x000caa293a9d97c5,0x00059519ff3b97cb,0x000cedb5893b39ad,0x00064ed07e59369f}, {0x0003315852af3473,0x000d5a40d47c32e9,0x000386582768fe34,0x000453f8e4c60653}, {0x0004a96dcf43bf2b,0x000e332542e1828b,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=115 [{1,2,3,..,}]*([2^460]*G) */ + {{{0x00011a7aae3fec0f,0x00081883626ec806,0x000aebe504491b56,0x00031dd2ecc113fd}, {0x0001068171e0f3fc,0x000d2fbb6b5fdadf,0x0000ee0e94a492b4,0x00090723f06e06c2}, {0x00016b906e2fed2d,0x00033b2f32d5d083,0x0000000000000001,0x0000000000000000}}, {{0x000731c43f27e685,0x000c912924bed063,0x00058df5bd8f8996,0x00080e3a04d16a64}, {0x000303ff6f14cb47,0x000f56c8175aed03,0x0003011b62e0f3ec,0x000eee8bd1d8f816}, {0x000a055be5c28fa8,0x0005635edb8d9c9f,0x0000000000000001,0x0000000000000000}}}, + {{{0x000eb4c2e9d6a36d,0x000f1c0e66e6ba0d,0x000058a747cb2451,0x0004b10a20962d66}, {0x000e1da104e82021,0x0004693d32e594ca,0x0003bb7f837609cb,0x000e53eda7c5059d}, {0x000b602751dd16ca,0x00007ac67ede2a3e,0x0000000000000000,0x0000000000000000}}, {{0x00038202cc49b145,0x00003535c208aec8,0x00056079145b2fb2,0x000814d455be9713}, {0x00082fd8f0bb395c,0x000c755426c09f67,0x0000b748edafadbc,0x000deaf4bc3a4ecf}, {0x000bff049553943e,0x00057ee542c407a2,0x0000000000000000,0x0000000000000000}}}, + {{{0x00058193376e77cb,0x000e72c0bba4380d,0x000bc54c3a89ddc3,0x000dd63a1bbc6d0f}, {0x000bf1518f660f2c,0x0007e5bf5faf2f62,0x0004682bfe7727a2,0x000da33defeb7b16}, {0x00032b5ef40ec257,0x000a0e902b2e8961,0x0000000000000000,0x0000000000000000}}, {{0x000ab468387524e3,0x00024e69a88271e3,0x0004545479e82998,0x000121a7373761e0}, {0x00093e3b1f753397,0x000463acb40aaebe,0x000cb721af707dad,0x000c14e152331d6d}, {0x00065553048e5280,0x000612aaad009d91,0x0000000000000001,0x0000000000000000}}}, + {{{0x00092b39f964e28f,0x000fa7cfd7976897,0x000f279c07ec556d,0x0001cf40d7670e8c}, {0x0007b5e04abdedad,0x0006fcc19990b137,0x0008572f5067ad94,0x000a28101c966a08}, {0x000b5e33c2ad58ac,0x000eb0e333e24c43,0x0000000000000001,0x0000000000000000}}, {{0x000f0361a475ac89,0x0002dcf79463ef54,0x000dd053538c22ce,0x0001013b4e6817ca}, {0x000b44b01a6e12bc,0x000d109d85844a6e,0x000b985bfdaef54a,0x000f6830be544481}, {0x000fd8dd0297f121,0x0007cb6d0b67a68b,0x0000000000000001,0x0000000000000000}}}, + {{{0x000ba65ede23b338,0x00026845dcbcdc45,0x0000a0b1cdddd83e,0x000092ea63968b4f}, {0x0004b71e6e72d35e,0x00046c5ed03f1ddc,0x0006efc5a166bbfa,0x00090de0b5f6b7c7}, {0x000445136c1387b7,0x000ba78923450ab6,0x0000000000000000,0x0000000000000000}}, {{0x000a850194005e74,0x0009c00cae44ea99,0x000e5e17b631e4af,0x0001c80d51c0dae8}, {0x000120c3cbe08ee9,0x00023c8041a40936,0x000ada73446b0eda,0x000643d14026f215}, {0x0006fbac37813fb6,0x000fc70031b68bb2,0x0000000000000000,0x0000000000000000}}}, + {{{0x00075ab901660b40,0x0001e645ee7e0ee2,0x00037b06a37399f2,0x00080496e9bd12b1}, {0x00026a5d50d58960,0x000c1e3f3705ba3f,0x000274a1d4a00817,0x000d2d00866a4d39}, {0x000f146bf0317c40,0x00024e2ec71ea064,0x0000000000000001,0x0000000000000000}}, {{0x0003c199f5c7f563,0x00062e4f78f16893,0x000a194ad2a3fb13,0x000080225259655c}, {0x000bc5898f9da5f4,0x000553edffd8a6c1,0x000e20b2793c4797,0x00041ea73083e26d}, {0x000a2971a533d937,0x0000e54fd22035b2,0x0000000000000000,0x0000000000000000}}}, + {{{0x000f9d4de9a27585,0x000400debb5987f8,0x000d526ec09e3156,0x0008bd03e0023f66}, {0x000aed7d715cc557,0x00058cc9c03099f2,0x000506b4417ca0c8,0x000d23b4df572ea5}, {0x000c5dc147420ffb,0x00032899652bdfe0,0x0000000000000000,0x0000000000000000}}, {{0x00031987cf8e9148,0x000b259461ad7d0d,0x000c859a4a7e6bba,0x00030e2b3d2a289c}, {0x000e9be629139087,0x0003904cf1cf6e14,0x000cca9dab045a7c,0x00082a43de3eacb2}, {0x000d82c3afa439f6,0x000decd187ae70d4,0x0000000000000001,0x0000000000000000}}}, + {{{0x00087b4497808581,0x00012be293d33455,0x0004ce4906c6f8ad,0x000a66938d521bfa}, {0x00046a914bdb3b26,0x000ae5f6e99dc3e7,0x00059dbb0881c2e3,0x00009191a1b5ac25}, {0x000c6a97a72e5343,0x000ddd07226ad4a6,0x0000000000000000,0x0000000000000000}}, {{0x00039e249234483b,0x000669419af2063a,0x00042122752c72bc,0x000aa7e19a44c7a3}, {0x00011188ac573c28,0x0009a3360e14ec6b,0x000bc0bc86245880,0x000741e192993af7}, {0x000d8d75742bd481,0x0006cd8768555ba8,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=116 [{1,2,3,..,}]*([2^464]*G) */ + {{{0x000ebaeb9266fcb3,0x0006c49166afc8c7,0x000b1a4fb9f8df09,0x0003a2759ef63e0b}, {0x000ded0e62d1d0a6,0x000215cb79a13c16,0x000942482d5b46ef,0x000a5cf390334543}, {0x00039ce21c9b239a,0x0000a2fcf03ed34a,0x0000000000000000,0x0000000000000000}}, {{0x000466a8adf517f0,0x000775523be0b6d8,0x00074759167493a7,0x000284c64894bb12}, {0x000e2864e9ca25e2,0x0008b7f98fd07d26,0x0003fdb6d6620610,0x000ef64b5a668e1e}, {0x000d31c44a0ba6ca,0x000891dac14a11ed,0x0000000000000001,0x0000000000000000}}}, + {{{0x0007e005c5b2f805,0x00010b6d99c24dd3,0x000813da1409fa02,0x0008bf13d53cbcd5}, {0x0005bb6d8655f948,0x00021f224e5b2d05,0x00077dd3ba305b4b,0x00005337f568059a}, {0x0008b4b1e783aa9f,0x000d6ce8c56442b8,0x0000000000000001,0x0000000000000000}}, {{0x0007e0acb71f23b1,0x0007f2e0e90fde9b,0x000336f8ff1da186,0x0004751614e3d072}, {0x0003187e51c7e8e6,0x0007ef17101ca72a,0x0008a9761c42d892,0x000ebb69cc0c641d}, {0x0002903e96138250,0x0006b8d2873a54c1,0x0000000000000000,0x0000000000000000}}}, + {{{0x0007b97b6b68155f,0x000799e78b707b09,0x000e1f75bd19e80f,0x00035096285e6628}, {0x000d9b8661ae0f1a,0x000fe90a8911dc8a,0x00018180a1d7fb12,0x000733dbb76ae258}, {0x000085cc47bdbf1a,0x0008c417eada4711,0x0000000000000001,0x0000000000000000}}, {{0x0006498715c0f18b,0x0008cd3093549e44,0x0007888a48d3a384,0x000a0d971c9394dd}, {0x000f79cc4aa385aa,0x000d44729501e42f,0x0002ea4042d9ad6a,0x000c1e43ab081b30}, {0x0002a374ef11901b,0x0009de3ad60e42b4,0x0000000000000000,0x0000000000000000}}}, + {{{0x00023f47f06415e1,0x000ec51fe7219c05,0x000bd8a88f411a49,0x000308976713e8b2}, {0x0006ee0f84892d3f,0x000957e9fa410c61,0x000903d60b015584,0x000f41fc07f1fce0}, {0x000fa3ce182117ef,0x000654b039b5693f,0x0000000000000000,0x0000000000000000}}, {{0x000f700d59c0d688,0x000bcc693fd9aa04,0x000b8b0e7fea0743,0x000182c181c35812}, {0x0008864896cc8fcb,0x000c77cf499f019f,0x00010bba6594c508,0x000e88406e142c41}, {0x000b45fd50fcaee7,0x0009894dc1ba3eb8,0x0000000000000000,0x0000000000000000}}}, + {{{0x0007488ba3fe2367,0x000752106ad62ea4,0x000980bc62d3b8ad,0x000328ef733708df}, {0x000d47b00f88c069,0x000f6bb0176a2cf3,0x0009f6480ef1ba36,0x000ac0712001822e}, {0x00032f8aac418a68,0x000ff326047c12c9,0x0000000000000001,0x0000000000000000}}, {{0x00039bbf3cc8d8c5,0x00091d5e4cf0789b,0x00042078d73c679f,0x000b71a650f7aeca}, {0x00082a530b74abff,0x000a71711bd8405b,0x000c3bb0ac95d510,0x00094cee94894b3c}, {0x000906c5e7b66990,0x00030a027cdba56d,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c446c47eef0c8,0x00044a0878421c07,0x0008722c55cf2755,0x000ec763424a48fb}, {0x000f4f6b5b3b9028,0x00078d4fe3ca8f7b,0x0003f4277d82e20f,0x000fbc6300a704e2}, {0x000a908b8f5f71bb,0x000b090bc8e8a53a,0x0000000000000001,0x0000000000000000}}, {{0x000dcad6549fc8da,0x0004e635d31de3d0,0x0009ac9c9dae5fbc,0x0005d812525deba7}, {0x00028465a1ffb0b8,0x000039c002085422,0x000d1431962a343c,0x0001729577d460c9}, {0x000befcdb0fe4b63,0x000f982552806705,0x0000000000000000,0x0000000000000000}}}, + {{{0x000c16470afeefa0,0x000c0df482969752,0x000988294a6b7345,0x000192c323f56fda}, {0x000ece866a0fbb9d,0x0003fb58e433eb87,0x00044e600bdf291f,0x000bc9a43f5fab4d}, {0x00060683951124ce,0x00065a7c641677f4,0x0000000000000001,0x0000000000000000}}, {{0x000ce490f8bb04ea,0x0002e7ad3382ee72,0x000272231533e6b8,0x000186b13ffe5f10}, {0x000cc8faf0cc5e1a,0x000c709ed2b173b2,0x00067e514381962f,0x000cd58bc198455b}, {0x0006402864604346,0x0009d46f62db1463,0x0000000000000000,0x0000000000000000}}}, + {{{0x0007bb607e896c28,0x000e0a894887a4bc,0x0006eb1e97614230,0x00003e71e2c653f8}, {0x00096dded494be93,0x0008ac95d09fc0dd,0x000bea563fba0619,0x0008f3c1624d738a}, {0x0009df64d4a0ea98,0x0001d5c6ae182338,0x0000000000000001,0x0000000000000000}}, {{0x000eeeb900454516,0x0006377d7a8b0a7f,0x000a9c345a7de36c,0x000100d5611067e9}, {0x000806bcdcedd0a9,0x0002b5dec6f6c68c,0x00051f38d7d4a349,0x00022d5061b9ad36}, {0x0005c9ea7f739c0f,0x000ee734e6cedbd1,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=117 [{1,2,3,..,}]*([2^468]*G) */ + {{{0x000eb05013cb5aab,0x0009afca55bd420f,0x0009c3c71fd8695f,0x00000cafcc6e5ed1}, {0x000728edc89ca3d5,0x00076471854e21b8,0x000201f0ff872ac7,0x0003e23036d8ee45}, {0x0002c13f7c8bee8b,0x000707a1d51a1e5f,0x0000000000000000,0x0000000000000000}}, {{0x00022b011bf68532,0x0000cf529ed8e280,0x000b8a477a52b6ff,0x000cd2f63b6e8238}, {0x0007c291c55c9cf5,0x000f4796ab42ab24,0x000c0989f93937b3,0x000bca7b47aa5dbf}, {0x0006f0a6b7620e79,0x000f58dd4ea00729,0x0000000000000000,0x0000000000000000}}}, + {{{0x0004f87aac9cea1d,0x000fabffc3ba2342,0x000a3b2b167162b1,0x00073da5b86c7978}, {0x000769b5f991d83e,0x000cb3d9088d484c,0x0004542e085b4392,0x0004eba2ea8f2806}, {0x000c46cb82b91d2b,0x0008c0e83321f8cd,0x0000000000000000,0x0000000000000000}}, {{0x00097601a9d31426,0x0009ef4a0c50dffd,0x000fc4b8056305e9,0x0000c1c8a29f6e86}, {0x000d4be1babedf5b,0x0005e98d4d558d2c,0x0007fd7d17d7bc87,0x00009a0a33b745e5}, {0x0008c2a2bc3cf9b6,0x0006235277b76d2d,0x0000000000000001,0x0000000000000000}}}, + {{{0x000ad9eb395de05c,0x00050bddffa75224,0x0008d48f88a50e57,0x0005bffe3c2175d5}, {0x000aded74a44ab8f,0x000f36097483dc36,0x00041e5290fdaaf9,0x00051f0f28ee6aea}, {0x00006d6a082ff0ae,0x000068e3568c35a6,0x0000000000000000,0x0000000000000000}}, {{0x00052b137174e56f,0x000a242808c3c521,0x000cee713ccf3a0d,0x0007f4523126e210}, {0x0009790a26049fad,0x000293cb237c8c65,0x000391754b82c700,0x0005060129b75ceb}, {0x000ad78c9cbc1be8,0x000452fab6d7a60a,0x0000000000000000,0x0000000000000000}}}, + {{{0x000614e7a62b479f,0x00091b1806f1503e,0x000c937295d07735,0x000468f3b432690c}, {0x000027af2bc376d3,0x0001568b1fc765b5,0x00004c84508081cf,0x00093b08d2fa4f2d}, {0x000aa23530d43841,0x000eddd4118eb0fd,0x0000000000000000,0x0000000000000000}}, {{0x00095d916a74bf7e,0x00052f879c30fb73,0x00033e906c3732a6,0x0009ecd6d707078b}, {0x000764fe7914feb0,0x000164429576e244,0x00070601ef70830f,0x00038d1a94c290ff}, {0x0007e067e8e38b39,0x00004d7b7a7e7934,0x0000000000000000,0x0000000000000000}}}, + {{{0x00092dd8b3c6e87d,0x0006c49275954a31,0x00016b291ceab935,0x0002ae4fca80283b}, {0x0002ae3f769e974a,0x00047d8e3a1a96a2,0x000875bc17f080fc,0x000a5eabb84f45c1}, {0x000e2f2661df31c8,0x000663147f43e69b,0x0000000000000000,0x0000000000000000}}, {{0x0002716571c12164,0x000b5580b0b11754,0x000bedf056c51a6e,0x00019d3572bd29fd}, {0x0006ce46ac6d5756,0x000d711d909ced7c,0x00056c0ec115b51f,0x0005ebe8c1ea4d63}, {0x0005d77cf7a0032e,0x000ddbec9919b1e6,0x0000000000000000,0x0000000000000000}}}, + {{{0x0002437a972f2c78,0x00015b5e5010e549,0x000fb968e48079c6,0x000ee7041338af95}, {0x0007cad3cea2147a,0x00063a330c01aad8,0x000ecdd3002cfb58,0x000bf58e568343ac}, {0x0007567e97251d0a,0x000b568be5a71fbd,0x0000000000000001,0x0000000000000000}}, {{0x0003982ec0a47e47,0x000862df20207b07,0x0002c656e08dda4d,0x0005ad59afe9aa6b}, {0x000ee04d8bf8d524,0x000d84cab42cc2b5,0x000ee837ff8c6d4a,0x0008b691af448525}, {0x0003b100c345fb36,0x0000d061abec1e80,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c9cc4477a0393,0x0002f0a9ecb165af,0x0008a80444bd4fb6,0x000a9d4dd001619b}, {0x0000b10c19a21051,0x000801b4ef3f9384,0x00076219bc4d71f3,0x000ea7190314526d}, {0x000da93e188e8697,0x0000967b67776acf,0x0000000000000000,0x0000000000000000}}, {{0x0006ab2717f542c5,0x000ae82426abc0aa,0x000641af35023777,0x0005dcf6c7153a96}, {0x000977e0ee60ca55,0x000c371732ddcc64,0x000063579dd744f3,0x0004f34c496af4bc}, {0x00000f9df0536028,0x000b1c2a26167cf5,0x0000000000000000,0x0000000000000000}}}, + {{{0x00061fdc0b1fc134,0x00059fe59da03f7e,0x00051831e7698bd3,0x0009f81df982fb68}, {0x0001c64253ce4407,0x00084a0c0fffbd0a,0x000fd2724ab08376,0x000aaefd7b900fa3}, {0x000017be08cd54b9,0x000359398932034f,0x0000000000000000,0x0000000000000000}}, {{0x0008bfed571b358d,0x0006e5bd73f12c5f,0x0007574722cbd9b4,0x000789de672fc532}, {0x000cf9d4c5de795b,0x0008e00647313e84,0x00001e2002f19344,0x0007649a15d6dda4}, {0x000a4f04c96114ef,0x000ac34a9dc08537,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=118 [{1,2,3,..,}]*([2^472]*G) */ + {{{0x0005e5ed4ea72f88,0x000dc27eafbd5d75,0x0008274c8f222817,0x000e4b956ed11c56}, {0x0002ac506cd96a0b,0x000c56121cfca3c6,0x000e3c5160f64376,0x0001cd969d9794e1}, {0x0002818ba2a1b9ac,0x000bbd5d12cb946a,0x0000000000000001,0x0000000000000000}}, {{0x000065269838f81f,0x000a256eaf242367,0x000743908912f4af,0x00048d332342e954}, {0x000f75565f855ec0,0x000e4ec59a3a8816,0x000505255b015d48,0x0006bf898ef06a71}, {0x000af90ae385313b,0x0001d5415dc8688b,0x0000000000000001,0x0000000000000000}}}, + {{{0x000115486df6f2ba,0x000803b08c738eec,0x0002302443593e59,0x000e60eb4f00e934}, {0x000e57c91438ebdb,0x00080e89c523e859,0x000d75cba0053e05,0x0000d11317c9b329}, {0x00035e5703d38955,0x0003845aac6426d2,0x0000000000000000,0x0000000000000000}}, {{0x000349105fa06b94,0x000e9b21c69e6c42,0x000f70eb1519a81d,0x000827ae2990539b}, {0x00048fe9bf0bbafc,0x0008dbb3c1ee9dd5,0x0002a908c4d4274b,0x000cb7224476f3be}, {0x00028346b6e6842b,0x0003ad39da0c7384,0x0000000000000001,0x0000000000000000}}}, + {{{0x000a9a342fa007b1,0x00065415bb7a90ea,0x000dcc6aa0c26771,0x000a1f0bc720264a}, {0x000b5e93f7bc1986,0x000bf6b3fca182f9,0x0009a7b5c22f84c1,0x00087bdec7ad14eb}, {0x0008902c942c3b07,0x000a69d7973e7810,0x0000000000000000,0x0000000000000000}}, {{0x000230ff44676602,0x000d3064b8821220,0x000d730a5228928d,0x000ea087c0f54e1a}, {0x0000abe8035de528,0x000d9e98d62188b8,0x000d85f12fe3f391,0x00095d1c13b3c4b4}, {0x000c9a4940436d0a,0x0007d2cc5f243602,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c484e2708d61a,0x000e14446b62d85c,0x0009c0fff451d84c,0x000a49ca08f1ae70}, {0x0003d71899e1ae53,0x000709a90d9917f9,0x000f39b12fbb0503,0x000038af72a2045e}, {0x000817cd90b8cc9a,0x0002f352b4ed83f0,0x0000000000000001,0x0000000000000000}}, {{0x000a71d48ae023ef,0x000d8744e50d53a7,0x000192ec22667e7a,0x0002e19437867d3b}, {0x0007124825f0cce3,0x000aa41f07fce902,0x000fc674b8262125,0x000d1f602e03c9ce}, {0x000f93cc0c071ae6,0x0003fae4c52caef9,0x0000000000000001,0x0000000000000000}}}, + {{{0x000d102ba58b5201,0x000cb021f49742da,0x00078984de6430a2,0x0005002098f1e552}, {0x000c9b804a37766b,0x00095479cd052eaa,0x00083f87dcee9cc2,0x000ce159b125b614}, {0x00001a384dc69331,0x0002e4ec5a2f8699,0x0000000000000001,0x0000000000000000}}, {{0x00049fffc681e26f,0x00001b52ea90f687,0x00080117445f69ef,0x00093fb5f32e704c}, {0x000d36110b81abea,0x000a3e64f4ffada3,0x0000ff99dcba475c,0x0000eb1b968c2040}, {0x000ded29d4be59cb,0x000ba78061ed859d,0x0000000000000000,0x0000000000000000}}}, + {{{0x0004949755c59603,0x000c796691a84904,0x000a8445d0aabdba,0x0002396cff6a7bd7}, {0x000562547f935def,0x0009038b91c344b0,0x0001f20274812ef2,0x000e1fe565e5087b}, {0x00074866fd902829,0x00032844aad6b8f8,0x0000000000000001,0x0000000000000000}}, {{0x00092e9251d05a0d,0x000328420cd4a328,0x000e00dfce4ccaef,0x000beca3a94d516d}, {0x0000fce64bd11bb9,0x00054cb1e99e5269,0x000cfc1bfec9624a,0x0004e634fe2023b3}, {0x000e53481128c3b5,0x000cd2258147276f,0x0000000000000000,0x0000000000000000}}}, + {{{0x000e9ad8d783865a,0x0008e5a58d54402a,0x000e236d64e3be62,0x000daf46f4aae16f}, {0x000049ba7000cd7f,0x000deb437c36e86b,0x00025e5a9179d63e,0x000fb30e5ec5adff}, {0x000fc4d691408156,0x000d8b4dff418608,0x0000000000000000,0x0000000000000000}}, {{0x000c4eed479219ce,0x0008e4d5ead8e114,0x0005f4f71316535a,0x0001c636b70e9489}, {0x0008aa2310479eec,0x0004e1dcd4f2eb70,0x000474c99b5134b7,0x000ce4754e99035c}, {0x000218919b4c0dad,0x000e1d1dd3f97981,0x0000000000000001,0x0000000000000000}}}, + {{{0x000cc18c358773b9,0x00036bf6385f12eb,0x0008a0c24ba8caf5,0x0003093cd83891ca}, {0x000f5b8c3762108b,0x00041e3399b26a0e,0x000fadfca8c426db,0x0004173bf6760263}, {0x000a6677ad40bf58,0x0004eca4760acdd8,0x0000000000000001,0x0000000000000000}}, {{0x000c42b8b4207fa1,0x0006dc5a60d34efb,0x000a367e08d67868,0x0008cead3e942c85}, {0x000409d289bdc2dd,0x0004b034c3a6d1bb,0x000889304955940a,0x00080034f3684e43}, {0x000808a7cddeee0c,0x0001148f3d9aa263,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=119 [{1,2,3,..,}]*([2^476]*G) */ + {{{0x000df4a3fd3e1bab,0x0006287d84daca3d,0x000d7eaf57017e3e,0x000d1a4e4870b354}, {0x0004c26a3e3ca9fe,0x000ce1ea5e5710e0,0x000709e17a2ff920,0x0000f8a3bc06ee67}, {0x000788ab8c019a66,0x0006f43d909c0fdb,0x0000000000000001,0x0000000000000000}}, {{0x000c0c61eebda5c3,0x000395c130704b51,0x000762ffbcb5d086,0x000f660bf6639983}, {0x0001646d9fb03337,0x00065cf06a8fa37c,0x0005f2e3f14b6d28,0x00088227d360e736}, {0x000a48fcdc5c3e58,0x00059e8c2eaf07b6,0x0000000000000001,0x0000000000000000}}}, + {{{0x0001ce2b5f2dc50d,0x0007e39f8c4d01b0,0x00078d34284f417d,0x000d157acbf04214}, {0x000c0c4238071f59,0x000b0a1e05f8a594,0x000d81bbaf85cdc7,0x000d1d1329e8c9cf}, {0x00068fc55c9be4f2,0x00029b5c20884e31,0x0000000000000000,0x0000000000000000}}, {{0x0009fd4109cb4727,0x00021d96d542276e,0x000d61e57db16c16,0x00052da58656adf3}, {0x00089d546ecce2da,0x00041508ee2098e0,0x00011997499c874b,0x0006f525839d9cf3}, {0x000de08e59654896,0x000e511cdd85c0a0,0x0000000000000000,0x0000000000000000}}}, + {{{0x000db13610c4d993,0x000192018344e51f,0x000cb8a7e81016f0,0x000425ff1ca2c27e}, {0x00047a8df5318c36,0x0004872bcd56d5d2,0x000d142a2e0d2618,0x000a83feb22e4866}, {0x00013dac70999b14,0x000ed007863be6ab,0x0000000000000000,0x0000000000000000}}, {{0x000023bbbd62b467,0x000e6ef8f48d21ce,0x000ea9c5f9c35940,0x0009af532bd76e0a}, {0x0000f8ff97911a1f,0x000efcff41750c50,0x0007cfa3985ad13d,0x000136812ef99c02}, {0x000319ee534694b3,0x000b9d9722dca85d,0x0000000000000001,0x0000000000000000}}}, + {{{0x00040d25bac4c923,0x0005026132b9fa82,0x000ddc2ef3e74ec9,0x000151b0a9db4e16}, {0x000ff5ad95c1429d,0x0008144cde9bb57b,0x0000c02f2a19e480,0x0005655b0b6aef98}, {0x00038725b1f2df6c,0x000672346457ed21,0x0000000000000000,0x0000000000000000}}, {{0x0003077ffe12bd18,0x000e682804b9bb8e,0x000b8a3a7328db75,0x0007b6f50cb1bbec}, {0x0007a823e8549b58,0x000d7be7a7e70575,0x0007103b60b8617d,0x0004131d7bc32367}, {0x000713f91128ac22,0x0000deadb3b9bf03,0x0000000000000000,0x0000000000000000}}}, + {{{0x000b46eea58e4f6c,0x000e8500cef9c4a5,0x000e36179b50381a,0x000498cb317e7dbb}, {0x000df2d824ab9ac6,0x000aa97d96328707,0x00068fd80e79f5f6,0x000ee03799c5c193}, {0x000688d4b109d20b,0x000fae5dfd91a5a4,0x0000000000000001,0x0000000000000000}}, {{0x000e4aa8649aaa1b,0x000caf8a4a894ffc,0x000f0a6af855f3c5,0x000fa6ef0a082826}, {0x000564cf46392869,0x000d9060255a7500,0x0007688b437590a1,0x0005fa2a21425afd}, {0x000dd69d65b91f19,0x0008045302895146,0x0000000000000000,0x0000000000000000}}}, + {{{0x0009a872aecfc094,0x000c8cde3af050fb,0x000ebe6b500bec0f,0x0004d4b7e7c4ef2e}, {0x00094b38a6c4228e,0x0004f9fb0e82362d,0x0000dbf4e229d20c,0x0003a6e45bdfa369}, {0x000b1c90f730c74e,0x000306f2fc481fa7,0x0000000000000000,0x0000000000000000}}, {{0x000e496c4b887a36,0x000e6ae46148f8e5,0x0004268188f16f8a,0x000cf1b360936452}, {0x000ea828f2ec9dce,0x000a581be5eec097,0x0000a093e062b3a8,0x000e4da12b498543}, {0x000b50541562092d,0x000eae33c27b17cb,0x0000000000000001,0x0000000000000000}}}, + {{{0x000bed4ffad0684c,0x000bb264e57bfffb,0x0009eb6b035825f2,0x00013466fd8b6643}, {0x000ab9c3537903c2,0x000b0366be7313de,0x00096ae2121723c9,0x0005953e87c3ac29}, {0x000b6974bbd38278,0x0003e43a30236cf9,0x0000000000000001,0x0000000000000000}}, {{0x0003b2707ffdea7f,0x0002da68809f795f,0x000374c5228ca4a1,0x000ef9a132cc5a86}, {0x0002bae5f8c0d15c,0x00061ce20672616f,0x000abed75c41da6e,0x0006a5fc5af7de33}, {0x00076a4d15065912,0x000ed44c16e78857,0x0000000000000001,0x0000000000000000}}}, + {{{0x0003498018e534a0,0x00031b029f064c8f,0x000b893aedc07be9,0x000b0eea14f71f6a}, {0x000eee179067b242,0x0009f6bf528af895,0x00020985e852a279,0x000b94bc19691d5c}, {0x00005deba296ab7d,0x0000b231b9475f76,0x0000000000000001,0x0000000000000000}}, {{0x000c45d63f8d3bc5,0x000aab0d9145a0f8,0x000bc0cd8bbb3a1d,0x0001299d614875d3}, {0x0008bad650d624f5,0x000b91d8407baf74,0x00054a383b9d385d,0x0006840ae765f5cc}, {0x0005a54bdbe2653a,0x0002865728a0edab,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=120 [{1,2,3,..,}]*([2^480]*G) */ + {{{0x000e419064732d33,0x0000fb9f1fdd6e2b,0x000b458dd169ab15,0x000b79def3f55fa5}, {0x0001fd9b88ebfb1b,0x000b8b17a8c1d98e,0x000e6b37f6beb8b7,0x000dbc72340b6c86}, {0x0007c19d37bb70ed,0x000f6867a99418dc,0x0000000000000001,0x0000000000000000}}, {{0x000a4a09f22c0fb3,0x000bb19cb6bc1256,0x00077d8b51c8ded9,0x000574809f35ca45}, {0x000bef1168ba7eb2,0x0002cdae11770b52,0x000ff68ed4f42bd1,0x0003d326b225de9d}, {0x00037f1445631a8c,0x00012cb14a3c371d,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b0b95f0603f03,0x000fa7f969adad66,0x0001acf774657813,0x000a2615220707f6}, {0x0002f71d4cd53712,0x000f82a44a2fd4ef,0x000773fd9e262931,0x0002f763ad200681}, {0x0009b206fe31fd70,0x000765c3a8767fa9,0x0000000000000000,0x0000000000000000}}, {{0x000a7f8ce84569e5,0x0009a821c3dd4741,0x000e90e3290cc915,0x000cf99306b623fa}, {0x000d7531760ae9e8,0x0009e7cf282874af,0x0006e1ae6527ae8c,0x0008f99eef73293d}, {0x00037109e03d3d87,0x00036ee1efdba892,0x0000000000000000,0x0000000000000000}}}, + {{{0x00051928ed074ce9,0x000b292af7a58dcc,0x0005ec5d4bdfb374,0x000cdd85d01fb1db}, {0x0006e626365656d8,0x00074fc478641e47,0x00016a5e28d244d6,0x000adbaa94ddb39d}, {0x0007fdde95fd5183,0x000b47ea66d8626b,0x0000000000000000,0x0000000000000000}}, {{0x0004c9d4962ab02a,0x00021388f7fd2b57,0x0006c23d66031232,0x000a1a6ab2ca0c2c}, {0x00017664a406bce3,0x000f5497442ca199,0x000866b6f2fc1498,0x000a41cbc3b0ab32}, {0x000557ca37a277ae,0x000af01602653825,0x0000000000000000,0x0000000000000000}}}, + {{{0x000db75622a8dfc2,0x0005479be9e5c74f,0x000548d39ec29bd5,0x0007942d29c79da4}, {0x00079c4bc1f5df3f,0x0004a7cecb948e1f,0x0008793b63229ed3,0x000939c1a7d67689}, {0x00057ad78be3b341,0x00052f2801351b91,0x0000000000000000,0x0000000000000000}}, {{0x000cbeae6fece889,0x000b3085ddee3b59,0x000eeab1d348140d,0x0006bba941a033c2}, {0x000b685703aafb67,0x0005046b6423a9d8,0x00075dab832a7c83,0x00015b8c259b9e24}, {0x00018a6bbb51f863,0x0003a253eb5dc8db,0x0000000000000000,0x0000000000000000}}}, + {{{0x000cca37a85cafbb,0x000b3657f26e3623,0x000787ec793c4d2a,0x000337f7520b9137}, {0x0000dbcfb7906436,0x00018cfaf22caa7a,0x00044625a502d754,0x0000066c6a130ba1}, {0x0001212f51d083e4,0x0004ebb9541e99d2,0x0000000000000000,0x0000000000000000}}, {{0x0009384f4e2ab22a,0x000ff707cf7953a3,0x000aa5f9b05bdfba,0x000626e81b083e95}, {0x000defb350599782,0x00092399d206f421,0x0008bd9415729d3e,0x0008cf10387904ad}, {0x0006e0bc19370ad7,0x000b48f2c002a076,0x0000000000000000,0x0000000000000000}}}, + {{{0x000b8bb85d8adb3e,0x00067b9a142f9bd6,0x000fc51be0f979dc,0x000cb118f84e32d8}, {0x000a7f5b6ca36f9c,0x000a900f565e79ab,0x0003143fcfd2df63,0x000122db9b751516}, {0x00086015e5f85f9f,0x000bf0e7c48af6d8,0x0000000000000001,0x0000000000000000}}, {{0x000cbc466d0dec7f,0x000fcfc13f4daf5d,0x000613ac2b0043ae,0x0007d2ec60909041}, {0x000eff4b79cb6956,0x000e04188e57b5e5,0x00045aa9dd05dcf8,0x0007cd8106c6759c}, {0x0004b84b0c6c633a,0x00041ee796334569,0x0000000000000001,0x0000000000000000}}}, + {{{0x0008ed21f68b4a3f,0x0009e0f39b982afe,0x000ef033664df945,0x0006109c1245ad2f}, {0x0004d6578f9c34c2,0x0008e9fc097b7383,0x0007b3121a085c72,0x000365666df584bd}, {0x0007af58ed558596,0x0007e9fd1e18ec9d,0x0000000000000000,0x0000000000000000}}, {{0x00017df29af6bc16,0x000dbbf468848de7,0x000d747cd3b7c888,0x000801a051097e9d}, {0x000f68bb9b824e70,0x00027a8a5f172bbf,0x00074f9f45d5351c,0x00080ba6fcc24020}, {0x000d4e050d7e5a57,0x000cebcb9d2f1cc1,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b990fe3b9d773,0x000eb81096bf3df2,0x000eb580e653b2d5,0x000cfd31a2ad7396}, {0x00065cddd150bca4,0x000cde916b4cdae8,0x00019b56ffe74e35,0x00021e7dc0b21b6f}, {0x00099d8bf333016e,0x000eb146cec318c7,0x0000000000000000,0x0000000000000000}}, {{0x00030acdbab36d51,0x00089ddd1e911c98,0x000891db5801a0df,0x000f1a5d646bbddd}, {0x0000ac4d27510e25,0x00044af2f910d55b,0x00024a75bcea08e1,0x00037ae5f37d50da}, {0x000d372739ad211e,0x0002a2d9d5c41773,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=121 [{1,2,3,..,}]*([2^484]*G) */ + {{{0x000ce74763660052,0x000da3e409da1731,0x000098b5f715b328,0x0003538d607382a2}, {0x0007bc3ee7b0651c,0x0006d5eed9abf1dd,0x000eb18e8c0d16d9,0x000e3fe464dc1a4c}, {0x00030d6fa6b9f8f1,0x000cfa359d987d0c,0x0000000000000001,0x0000000000000000}}, {{0x00047d09810803ed,0x0007b5b97b578929,0x000cc27fc5005d73,0x00040feb2087e2c1}, {0x000b7dd0d960662d,0x00025ee555f37345,0x000d7c17f3858a72,0x000a0cf2ae739ae8}, {0x00000ee77dcf4e1a,0x000c12649e41ecee,0x0000000000000000,0x0000000000000000}}}, + {{{0x0008de672d619b61,0x0000ea8326922a80,0x0001a0841b015626,0x0000f8a963e3e317}, {0x00037806aeb44acb,0x000d9d8a14334837,0x00026bd761a3419a,0x000d2e7a343fbffd}, {0x00086e32c6d361b6,0x00023ef433219c4d,0x0000000000000000,0x0000000000000000}}, {{0x00025620f22d4f25,0x00067dd5c03d381f,0x00080f734643a87f,0x00006c5ee876505e}, {0x0002b491baac4e49,0x0003e07deb178a01,0x000ad060f735b869,0x000576ce5dd8d75f}, {0x000dd4cd9c97cb18,0x000cbc634bbb55f5,0x0000000000000001,0x0000000000000000}}}, + {{{0x000b9733710e8e01,0x000e73a5711788b8,0x000bf8afcacf73a9,0x000d6725ee57149b}, {0x000e7fe486c64e2e,0x000322f9087bd5a6,0x00009af08709418b,0x000084990390cb99}, {0x000a6bb3ab911d03,0x000d2868a69e665b,0x0000000000000001,0x0000000000000000}}, {{0x0005e749b382f6d2,0x000a9a1034406b89,0x000826ec06265b52,0x000e64e9aec95b07}, {0x0003982f9a9c5d16,0x0000698b37d7e83c,0x00050d8bbdbeb42b,0x0007c82f4fbc8ae9}, {0x000adc1e63d423d5,0x000b249310802372,0x0000000000000000,0x0000000000000000}}}, + {{{0x0007033614a6d5dd,0x000fddc5f2fac137,0x000e014aa4b4dee5,0x0004a9b72218fde8}, {0x000c10e229612a68,0x000cb5b99f1d9b66,0x000eff01796c1307,0x000ec087152271c7}, {0x0009f171d27930b1,0x000dd53091f21ad1,0x0000000000000001,0x0000000000000000}}, {{0x0004c873e4172f54,0x0006ecbd512368a7,0x000d3ea21d4bc31a,0x0003a95f62eff689}, {0x0007c73a33474bd4,0x00088fa97a141350,0x000b4d3b01846eff,0x0005fbac8f6a8f06}, {0x0009ddd58dc2a301,0x00001f7b911f1a15,0x0000000000000000,0x0000000000000000}}}, + {{{0x0002e681058bcd0b,0x000ecb766f6bc98d,0x000866fedccaaef5,0x000b2e2473204d11}, {0x000f6e18757016ad,0x00011d59effc1a8a,0x0002050629e88cfc,0x00093c7bdc024782}, {0x000a9b2aeb9bb00e,0x000336991f06d2c0,0x0000000000000000,0x0000000000000000}}, {{0x0001568955531744,0x0008281170681859,0x00050d7be99cf6e1,0x0008cb9d185c0963}, {0x000f49cfc22a2afc,0x000f9d20626a2a56,0x000ad87b48f04b95,0x000bd1441cc30d3e}, {0x0003e9b72d43f56f,0x000e3b3843d17383,0x0000000000000001,0x0000000000000000}}}, + {{{0x000e873b05f77e97,0x00071d5ebf3c8d2f,0x0005b9ca7cc32fe3,0x0008798cc245b054}, {0x000e6eaf83f8b265,0x00061d87bdf09afd,0x00048a529e1b9707,0x00001501c97ba4fd}, {0x000ca96655ab0a10,0x00042f0ec7beee1d,0x0000000000000001,0x0000000000000000}}, {{0x000296b82c7a9289,0x00070c171dfdb228,0x0001dac3a3a171bb,0x000b7ea6ad9a13af}, {0x000251fe361dde21,0x000cea9acd2f8b81,0x0008480e8df3c1ec,0x00038a5f495ca4b4}, {0x000fd225cb8ecc78,0x000454bc6bffc707,0x0000000000000000,0x0000000000000000}}}, + {{{0x000af33412f12687,0x00015e41163cf0f3,0x000967fdc5a6a476,0x0004235cf9f62e34}, {0x000b314d06a6a848,0x000820f5665619a2,0x000f11a14ea427a8,0x000ce9a80c44b6a1}, {0x000f92bed7985fca,0x000dc713540bdff6,0x0000000000000001,0x0000000000000000}}, {{0x00065826c0cb51e6,0x00030220ec95e76a,0x00064aea77a786d0,0x000cb5ba9e93c602}, {0x000f020c5b781189,0x000e7b4655282299,0x000e97af8ea95e4d,0x000a8a80a0f194a4}, {0x0006433581c41d62,0x00066e34a29ca8e3,0x0000000000000001,0x0000000000000000}}}, + {{{0x000cef36ab807b63,0x000040cdf4c99984,0x000c211953a5f8d7,0x000ab4c0faefc5ed}, {0x0005ca17066a1563,0x000fb2c0940c339a,0x000b1e8517a5667a,0x000c3d2a94a0b135}, {0x000185e4d4526e2e,0x0001b53c05d493d9,0x0000000000000001,0x0000000000000000}}, {{0x000c5ced3676f843,0x000195ff470fab2f,0x000ed29f4a221ddb,0x0000868b2d94f5fe}, {0x0003caf8fcc5069f,0x000dcfc1418631be,0x000998943a070623,0x0009bafa5f731c9d}, {0x000c5c56c1cc4a06,0x000a82f502e626e1,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=122 [{1,2,3,..,}]*([2^488]*G) */ + {{{0x0009edf282d019ec,0x00099e8e335e18d2,0x0004ace8ce0e046e,0x00001d0f72c0503a}, {0x0007e9c6d09e242f,0x000998b6c2fcb456,0x0000be40686ceb13,0x000db8fca6af9143}, {0x000c77e852236ef5,0x000ba3718e1a2901,0x0000000000000001,0x0000000000000000}}, {{0x0005ae430ab427d9,0x0003d8a843a1b6ab,0x000c9500fb6025f6,0x000b9cb8d803e788}, {0x000fcea023d9bfb7,0x00003f3ec5cdad70,0x000188da7e50d4c8,0x000f9eb540fd9c07}, {0x00014ab57822ee2a,0x000574aff12ba00d,0x0000000000000001,0x0000000000000000}}}, + {{{0x0003c20dbe0952a3,0x000480b6013f7fd5,0x000447348d109d4a,0x000fe6fecb6a7da1}, {0x0006564e8c529d8b,0x000034045fa60672,0x0003ee2a8df68fa9,0x00021796dbc7ff3f}, {0x000a130fededc279,0x000fe24c3f368ae9,0x0000000000000001,0x0000000000000000}}, {{0x0002961eb9eed66d,0x000919ed55f27279,0x0000068193a9b014,0x000f317444cb0bf8}, {0x00096e22227ee32e,0x00047c8b854cb4a8,0x000bf6dc6a73b281,0x000d6804296e2ed1}, {0x0003e6f8a77be001,0x00084c89b143ab22,0x0000000000000000,0x0000000000000000}}}, + {{{0x0008d791f2d40215,0x00003b05d1fd525c,0x00037b16b3ca30ae,0x00070792a856131f}, {0x000b7639faf0f678,0x00006b7cf12eff42,0x00098ab7a44f2173,0x0006714e846ec06d}, {0x000eac350874a266,0x000b56e5920dc3ae,0x0000000000000000,0x0000000000000000}}, {{0x000d703e34853d1e,0x0002cd53a7cce717,0x000410bff4f394e1,0x00074ecb0bba0cc1}, {0x000de8fa9da2c436,0x000f5e3f74e2caa0,0x000cc28b148d1eb1,0x0009fc1ac5bad585}, {0x0001220666fb73af,0x0000c3241a57ee07,0x0000000000000001,0x0000000000000000}}}, + {{{0x0000e99218ea1f1f,0x000ccf21044500ec,0x0000c873630cba88,0x00064f806fd4e4b8}, {0x000a7056645dd457,0x0002ed87394551a9,0x0008987025ba6b17,0x0005dd01b45fa9a7}, {0x000ccea3a1f9f135,0x000592807cbab8d2,0x0000000000000001,0x0000000000000000}}, {{0x0006c96e8e24e119,0x000d921a51e8134c,0x0002ab9759957065,0x0004035ca89e1baa}, {0x000df057c2aafabc,0x000c0890aa1a6716,0x0006bd3f802387d9,0x0006a39383e5c778}, {0x000601e4e1f62705,0x000096f226577900,0x0000000000000001,0x0000000000000000}}}, + {{{0x0001d3b5076ee66b,0x000068e996c31106,0x00063f8bf5d922ca,0x00008ed44203a2fa}, {0x0001df0821d991eb,0x00019d54e602f04f,0x000bf35cd4ee7bb8,0x00015f2609e3729a}, {0x0009e8e65b2fcd60,0x0001df9e9c109298,0x0000000000000001,0x0000000000000000}}, {{0x00058eae5edc3042,0x000fe31ba09cdc97,0x0006f52853b56fdb,0x0003df0f5ca36adc}, {0x0000a54940d61878,0x000902cef58665ba,0x0004efbb68e67641,0x00036806e0aaf0f5}, {0x000194a89e785e8c,0x0008883379ceb241,0x0000000000000000,0x0000000000000000}}}, + {{{0x0001a19f7b341ee3,0x000076e0e5354bd1,0x0001806a485286dc,0x000d2bf681431840}, {0x0002c82334b1343e,0x000b6908add258b2,0x000737bf47a5dbe4,0x0007303c531a0b11}, {0x0003a29d6501615d,0x0009313aaea01e10,0x0000000000000000,0x0000000000000000}}, {{0x0007b22b906a725c,0x00024ad4ce011033,0x000bf242639d28fc,0x000fa39cd38de6c9}, {0x000a72beeb0f8fd8,0x0009a0ad9b93380e,0x000c8221fc799b7c,0x000be51116d9d8de}, {0x000a10526402ca53,0x000777aa2c9ac3b1,0x0000000000000001,0x0000000000000000}}}, + {{{0x0001a723455b8da2,0x0001d0c7c777796b,0x000fb41cc9b1644a,0x0000ef5ce0972939}, {0x000637a26ddcdf7a,0x0008cf1a639f0844,0x000023a3ce642ca3,0x00098f7db827cdb3}, {0x0005279eee7f6f0d,0x000565523e47e762,0x0000000000000001,0x0000000000000000}}, {{0x000fd1b034828d4a,0x0000721a61eafcaf,0x0001ce95f4ec0ae3,0x0003ba2db4a66fba}, {0x0006525e6f06fee9,0x000817b26fcb0ef3,0x0000265db68ac06f,0x000f74e811cb24b3}, {0x0006550f9c2bf885,0x0003940f2d2fcd83,0x0000000000000000,0x0000000000000000}}}, + {{{0x000e8955d1b109b2,0x000e1de1d0f381d4,0x0006407a6cf45a79,0x000f2393e689a76d}, {0x000d3d92aed2a407,0x0005547cc6ac261b,0x0005e0b9e62fcac9,0x00081e2910774983}, {0x0003d6780dde8f90,0x0003c5c4cab77f7b,0x0000000000000000,0x0000000000000000}}, {{0x000e052d3f3dc82c,0x00039caa1aeecdbd,0x00024153092958c9,0x000c11b7ca5c0f7b}, {0x00027c92847965c0,0x000732af643698d8,0x0000367351c0ba1d,0x000f1b1bf491a3ee}, {0x000df3514ec2302c,0x000b4c4436d640af,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=123 [{1,2,3,..,}]*([2^492]*G) */ + {{{0x0004265bd7179d88,0x00032014b97128c5,0x000fd3dafdfe0b08,0x000b1956b3fd6699}, {0x00091416a87bbb8b,0x0001dd4344038f86,0x000566c88826c840,0x000f07a8a4b77456}, {0x0007671e1b2fca59,0x000200797dc52a03,0x0000000000000000,0x0000000000000000}}, {{0x0007843bbe8d7f70,0x0004f9ee9b4c465d,0x000303b1652fa39c,0x000ae7c4c4a55ae2}, {0x000263ccdcb67c15,0x000a17fd06da8ac5,0x000c10d8d1d1e927,0x000e5bfc6232685a}, {0x0003162cd048bbb8,0x000b11c2cffebb23,0x0000000000000000,0x0000000000000000}}}, + {{{0x0002cb202ec3c178,0x000285de81ad92d1,0x000b71b77497dfd0,0x0007a8a10c150a03}, {0x000bbe99f4ad3f59,0x000f4533b0aef51d,0x0003b27838ed4931,0x0008ffc95a8ebcf1}, {0x0002cefcf5623ddf,0x00010737c166832b,0x0000000000000000,0x0000000000000000}}, {{0x000e740d8da2ff7e,0x000624f3d3ab048c,0x000376415ced03ed,0x000fbe5676391c7d}, {0x00081671ffe7b22f,0x0000390438cc5f49,0x00084a5ae289dd49,0x0008f9a1f5bbef09}, {0x000b05c4941c6652,0x00083aef77ff073e,0x0000000000000000,0x0000000000000000}}}, + {{{0x000604a9bc6900b6,0x000878e5f51ce9df,0x000b763f98cad97f,0x000f5a1389d3ab54}, {0x000ab3d0efdef6fe,0x0009be5cf0df2543,0x0002a0d518696763,0x000134850193d832}, {0x000860abf9047761,0x0004b4f04b3d8de0,0x0000000000000000,0x0000000000000000}}, {{0x000e57c44a551894,0x000a3fa66238e065,0x000d140af7878a12,0x000de14d55dcc858}, {0x00061c7b391cc65f,0x000d7dbf324d8825,0x0005d09aa74e78f2,0x0008ed166a4503f8}, {0x000da0c2a7ad860c,0x0002048d8fe387e6,0x0000000000000000,0x0000000000000000}}}, + {{{0x000505ddc7162a8e,0x0001fb5ed0deab66,0x0005e972cc689dc5,0x000e495fb69dc78b}, {0x000ca3f1826690c6,0x0005f6186896d605,0x000fd32f66789288,0x0008863f96f8edfb}, {0x000e644e5dece22e,0x0009b7f857a4d564,0x0000000000000000,0x0000000000000000}}, {{0x000691ddbeebfc5d,0x0005901566a70055,0x000e1f4e6067fa43,0x00086e62796a672c}, {0x000b4e5ee14cf308,0x000b327c1a40aaca,0x0002fed3294bd689,0x0009103e56992c00}, {0x000b1323df8494f5,0x000d7b51fec2bb9e,0x0000000000000001,0x0000000000000000}}}, + {{{0x0009f1b91641749a,0x000415c17bca5ff6,0x000c6d3a845d2248,0x0005f9b6f404856a}, {0x000cbf400b63630d,0x0003b4ba273b60be,0x0005d238fa843e67,0x00015df9fe916d32}, {0x000747b338f22dd6,0x000213df0dde0478,0x0000000000000000,0x0000000000000000}}, {{0x0000235c1fe8e923,0x0003196a6c0855b8,0x0009e7caa33347ee,0x000bc0a45596b47b}, {0x00011e3fa8377c58,0x000acaedbc6f8b16,0x000fc365c99edd72,0x0002dbbec0f6e0a6}, {0x0002b70a4b7723c1,0x000a4bf65f3c20c8,0x0000000000000001,0x0000000000000000}}}, + {{{0x0004bf14c1a4d603,0x00062d773ba946d2,0x000a858973ed2e17,0x000af47e821e4637}, {0x000c4e2f1b685a3b,0x000a24d7fe743f00,0x00011c916f0b3711,0x00019d3f29631796}, {0x000070eeb3ea27a8,0x000fcf9d0e9d8d24,0x0000000000000001,0x0000000000000000}}, {{0x000a9e0cc6d6de0a,0x0007d8c5fca34ad4,0x000c81d46494c7d6,0x0008c618856d1751}, {0x000ea22fc514e835,0x000e085f741c8235,0x000c321d004a049d,0x000bcb516087e553}, {0x0002d6363ccbbe68,0x00083e572fe7f6b5,0x0000000000000000,0x0000000000000000}}}, + {{{0x000e20ba96faf5c5,0x000e8d8c8a0cc4ab,0x0009b5c593a344d4,0x000c6c34af049395}, {0x0005aa8d456d94a3,0x000ed953bf7c9473,0x000962cd0b8cc1dd,0x0000c01bb3088b5c}, {0x000c82c42c7d7139,0x0005bd26c576d9ee,0x0000000000000001,0x0000000000000000}}, {{0x000ee2f364e79144,0x000e681f5a9a561f,0x00014c6812d4021d,0x000e552f50051d32}, {0x000f6a99f35033a6,0x000e505a2349153d,0x0001622a1be8e97e,0x00067971f625164b}, {0x000d441d9fc2328e,0x000d2eb0550478b4,0x0000000000000001,0x0000000000000000}}}, + {{{0x000769c73aea3c08,0x000df9a9593240cb,0x0004e8217f3b057c,0x000ceca2220054ab}, {0x000c95ba1c2a734d,0x0006500d1322b719,0x000ec571b4360381,0x0006f76e87cb0ba1}, {0x000c5938559db2c7,0x000397be033b5877,0x0000000000000000,0x0000000000000000}}, {{0x000b6a77feb075ca,0x000d9cc6a6cde3dd,0x000e49872538f578,0x000e469feaf37819}, {0x0002ddc9c48cda10,0x0001a6e5450f9883,0x0002d31bf05ea5f5,0x000375cd216a195d}, {0x0008007e689987c4,0x000cbc358f3e07d1,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=124 [{1,2,3,..,}]*([2^496]*G) */ + {{{0x0006a73f6f2770c4,0x0000f968ca281cd6,0x000827efca6a0867,0x0003a96b180e8f32}, {0x000809979b757eac,0x000d9223bfbff7df,0x00047dd166015fc2,0x00065475a88730d7}, {0x000ce16229ea9d12,0x00076d23756de3fb,0x0000000000000000,0x0000000000000000}}, {{0x000ed537bee27c6e,0x000943e46c7c15a8,0x0004b3f87656d7df,0x000a9213335be530}, {0x00076cb0ee208db8,0x0004f5fc16b61ee3,0x000c1114ee85495a,0x000253ced62c2d47}, {0x000641c92453ad35,0x0003e4e1a21d73af,0x0000000000000001,0x0000000000000000}}}, + {{{0x000483ff2c9de102,0x00017f0cb9492bab,0x0001999673c19107,0x0005c7a75ba40ad7}, {0x00022ec8c1ec861f,0x00078704457a9540,0x0001194ab6d023c8,0x0000daf5008c607c}, {0x000d394925361233,0x0005ab5bf20a934a,0x0000000000000001,0x0000000000000000}}, {{0x0008ed3301b16277,0x000b31045574d7ab,0x000ed11cb44f38e3,0x0001af67ab10bb4e}, {0x000d033cee10ca51,0x000549874c9fe7c1,0x000392d6999489f1,0x000ffcfe4a15a85b}, {0x000b006a13684dec,0x0007b8baefda3eb3,0x0000000000000001,0x0000000000000000}}}, + {{{0x000e82cbfe7306a8,0x00002e52832eb494,0x000b381c8b461b41,0x000a6e877f0afbca}, {0x0006b8482ae88f1b,0x000709eb28c8cc07,0x0000cd45fc8e5ced,0x000b1363d1cf0c64}, {0x00093a63d6be8f78,0x0001407a8e7f6a49,0x0000000000000000,0x0000000000000000}}, {{0x000fd703c088bf64,0x000708b3415df01b,0x000f82eeb2c8b57f,0x0003ed35407aa69d}, {0x000449767c6b4a72,0x0002e1a8184dbc3f,0x000d25edffc3e965,0x0003e8855e29ad89}, {0x000c695a889f2e49,0x00025dae2a995ab2,0x0000000000000000,0x0000000000000000}}}, + {{{0x000c300d63dacf22,0x000f84149cc93249,0x0004f71e87a984e1,0x000ebeada635f884}, {0x000cac51f48942eb,0x00076c6b878d815d,0x000587460dede95b,0x000883c91cf8c3f2}, {0x00013e9be5375387,0x00076d4ce987a56d,0x0000000000000000,0x0000000000000000}}, {{0x000fb20151675f42,0x000bf54bf1c2d622,0x00022da7f9e8bc4a,0x000051b0e7b83f3a}, {0x00073eda536a6e42,0x000ce8431a9d89d5,0x0002b7a64d23c5a3,0x000007a6be7b3eec}, {0x000672919b5fb43a,0x000c454a7d18005b,0x0000000000000001,0x0000000000000000}}}, + {{{0x0009f484a29e97a0,0x0005e90aa411ba92,0x00016fc135a72a99,0x0006cfa3c8dd8a3a}, {0x00066efeed6df222,0x000e66eb40ebb1a2,0x0000f8ad15ad7b0b,0x000e0c3a19929e39}, {0x00051e3404d13a05,0x000175cf393bacd1,0x0000000000000000,0x0000000000000000}}, {{0x0006cdbeb4d89814,0x000b884f6ce10295,0x0003138d1321a20d,0x0006528c4bd065b9}, {0x0003d082878ee395,0x00029465651ab383,0x000cb4e49e6b0dcc,0x000710248d30e955}, {0x0006e01a51ae9cc3,0x0002fc5567eab89e,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c59218072f54a,0x000195bad5f014fe,0x000deabd55429cb3,0x000b2ab5fb9c1406}, {0x0006cf39524ff8ca,0x000fbb57c01480bd,0x00018cbc932f5376,0x000c9e4e5da034f1}, {0x000fb16c36eb8d83,0x000048c80fc4eaa6,0x0000000000000000,0x0000000000000000}}, {{0x000668aa11e1cfe6,0x000d4f614afa98be,0x0004cadab479a412,0x000864b0b94d7822}, {0x0002651053a74933,0x000dd43fe6424e5f,0x0004f2c600bdaac3,0x000ec0b432ccf8a0}, {0x0004d82574257110,0x00024e58edc3e12f,0x0000000000000001,0x0000000000000000}}}, + {{{0x000d5163c2f29845,0x000271b8856ff717,0x000a79d0557c5c08,0x0001484032810d21}, {0x000a5a6e95174165,0x00093b8782ce8f06,0x000a6f7f14b15d03,0x000ca398eeacdf3d}, {0x000eb31c7c040c1a,0x0003ace9ed34f4d2,0x0000000000000000,0x0000000000000000}}, {{0x00086bac2cc4ff50,0x0008d5294f7bd063,0x000ec0b7a55f986d,0x0006868155592285}, {0x000e215833824965,0x0003366d9a307162,0x0001de9196efa150,0x00076afbb75f7833}, {0x00046ce65ce11aa8,0x0002f7a207e31942,0x0000000000000001,0x0000000000000000}}}, + {{{0x000d6129f7d0fa54,0x000150bddf5a7cf8,0x000b4988625b2f43,0x0009bbfb3c2f3809}, {0x000f5b080f7b3129,0x0000ab0abb84ed45,0x000510d824f7bed2,0x0006d6447243533e}, {0x000c576b7b64fbbb,0x000e16caa9ee8267,0x0000000000000001,0x0000000000000000}}, {{0x00053a269ea0b07f,0x000e06f68fe62242,0x000a777b6874572d,0x000d5f86cf599bf5}, {0x000fe2a811045a16,0x000873264294a33d,0x0007a04ac970a0c0,0x000ceb2b7d05d686}, {0x00029a28a0e51a57,0x000bacbf79a38ead,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=125 [{1,2,3,..,}]*([2^500]*G) */ + {{{0x0003efe866dc2f62,0x000853cb9e407c10,0x0000e6c71edaaa13,0x00018f751b70a2af}, {0x000b0cf7e3e825ae,0x0005c36a5a1ec11b,0x000487f56ab4b564,0x000a86df052ea4e5}, {0x000d750313868ef9,0x000e60ee422740c7,0x0000000000000000,0x0000000000000000}}, {{0x000ee652bd47edd5,0x000397faa97f40b7,0x000294d2d1ba3dd7,0x000344d3453daecf}, {0x000d324bd3f56e65,0x00078c6f611c9985,0x000e24f2675985ec,0x00038b4060d38ad7}, {0x000dfb7496c92821,0x000be627a6ad57ff,0x0000000000000001,0x0000000000000000}}}, + {{{0x0009be6a7e5a166a,0x000d06313031bd58,0x000704962d984289,0x0002ec5b522512df}, {0x000386a669eef493,0x000fe747674db075,0x0004dbaabd7aebbc,0x0004d27fb22ce794}, {0x000e70494458ef81,0x000fadc96805636a,0x0000000000000000,0x0000000000000000}}, {{0x0005b60c511b3ff8,0x000584915adb1e6c,0x000f8937e8e108c6,0x0008406d64ea3a9f}, {0x0004461d268f9ab8,0x000e3ff279d6126f,0x000d3b3ed1f3032a,0x00023a1b63af22a5}, {0x000caf9282fd7a53,0x000d99f7a42a7590,0x0000000000000001,0x0000000000000000}}}, + {{{0x000dfb005014a6bc,0x000d36179f05f79f,0x0001f0a00c591c70,0x00009f861bdb8aa0}, {0x000851877e4cc13b,0x00004921bdab098b,0x000265f47ca34718,0x000478a5d59cb874}, {0x0008aac74eb734d8,0x0002f6a87e5bc7bb,0x0000000000000001,0x0000000000000000}}, {{0x0005dd559082ec4f,0x000fd3340b409a63,0x000e395e6174cff9,0x00035fdf83237476}, {0x0006e995df5d90e4,0x000535beb0acf902,0x000ddc2f60fe3f20,0x000821d68a60c3ba}, {0x0008005435d079f0,0x00084ef7a20c388b,0x0000000000000000,0x0000000000000000}}}, + {{{0x0001ce9624902c85,0x000dacee54d7fc06,0x0008e982883e676d,0x000cea68fc5997cf}, {0x000b94f3d06f1e8c,0x0007b80d8242831b,0x000ec2e625c36045,0x00015466a1d87389}, {0x000c5009313ff25b,0x00045efb5d45d1f9,0x0000000000000001,0x0000000000000000}}, {{0x0004e32c6f246301,0x00064608ec78b5ab,0x00053a9c0b324014,0x0004ddead5ffb795}, {0x0008bf933bdcc559,0x0005249289dc110f,0x00047d25d52f652d,0x000d95ab06c0cb41}, {0x000bbff17968adb2,0x0002664039be6fa1,0x0000000000000000,0x0000000000000000}}}, + {{{0x0006c159dee64527,0x000d486c1da85118,0x0002ffecac887f87,0x000401c5326e8ac4}, {0x000fe68a082d270a,0x000b17b4ec0bd703,0x0009b1a9fe82427b,0x0008f75c6b25d502}, {0x000ac56e28859df5,0x000bc70f97f9bffa,0x0000000000000001,0x0000000000000000}}, {{0x00073b932e4e3e21,0x0005f8e721dc13af,0x000825bde1498f11,0x00080fd3102f60c7}, {0x000f292e5a9e9f07,0x0006a4edbc9fcac3,0x0008f2651ae44279,0x0002622ca1bb123f}, {0x000a4ca103d2d6a7,0x000ca577d0f1994f,0x0000000000000000,0x0000000000000000}}}, + {{{0x000e577af10e1302,0x000156bf557eaa33,0x000ca7cba0f98005,0x0001e9d6e3d41486}, {0x000ad3a30b9f973d,0x0009856b55aa3443,0x000724f819219409,0x0008250cf13f0ca4}, {0x0006f0f69ba1696b,0x00063bba1deb1e9b,0x0000000000000001,0x0000000000000000}}, {{0x00037cb6672f9435,0x000d7e7437eb08a0,0x000579149c6faf55,0x000e4c6943f7c61c}, {0x00026eba03e36921,0x000a3ade34c342ab,0x000052386264eb60,0x0000ffc57653cf12}, {0x00070eec7914e6e2,0x0004912d67845657,0x0000000000000001,0x0000000000000000}}}, + {{{0x00006fc3e072e1bd,0x00066e9739a62b31,0x0006ea97e6ad1669,0x0002aa2bbc843284}, {0x000871768bc4f0fb,0x000a51f2d1bd1f5f,0x000245a8890f99b0,0x000b6a0000aa5f53}, {0x000547c1538e0dc0,0x0001c33dbece2149,0x0000000000000000,0x0000000000000000}}, {{0x000b56de17b97c51,0x0000fccb15a8e3ea,0x000c00352a78e0ff,0x000e7d480dae3bf3}, {0x00092273cee30716,0x0000962a4283dde8,0x000e674e18ae53b0,0x0002b8c78835cb2c}, {0x000faee271217641,0x00046294e0f5e7c6,0x0000000000000000,0x0000000000000000}}}, + {{{0x0005697612d854d6,0x000590266d871a78,0x000015aa94be7df9,0x000482ac4e8bbf72}, {0x0007c12880439150,0x0003495f23aa4b2f,0x00074815ef777bb5,0x000838a798c004a6}, {0x0008a425cc7aadc8,0x000c432ccb5f5730,0x0000000000000001,0x0000000000000000}}, {{0x00006af85640f288,0x0003a6718ebc6cb5,0x0002c50bba9dd21e,0x000e3c4d56098fde}, {0x000a4a8a721857b6,0x000218f9c402f4d7,0x000a6f255530e5d9,0x0000bf7b3c63a541}, {0x000f0181b97421bb,0x00023de7a08f2804,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=126 [{1,2,3,..,}]*([2^504]*G) */ + {{{0x00061ebad9ee2c24,0x000dd46aced3ddaa,0x0000bd3e3fde5fe2,0x00020569fe14f9f4}, {0x00088d818d1a2095,0x0002f0bdc9b4968b,0x000e3de0b8b77328,0x0007fe9e8edc6520}, {0x000017cf0272ff76,0x000eda0f65dc99bc,0x0000000000000000,0x0000000000000000}}, {{0x0009b50b03dc034f,0x000ff04ea634ab0a,0x0007b191db6e6308,0x000a9de7ee04399a}, {0x000e6da7bdea8dde,0x00054c55ae492d45,0x000f4e939e666b7b,0x00090c925a51f573}, {0x000f916220292c15,0x0002d380fc7f5071,0x0000000000000001,0x0000000000000000}}}, + {{{0x000639a92b83d191,0x000b3a1ce7b1b453,0x0000d260e431474f,0x00032954aefab808}, {0x0006dfaf9e670c4e,0x000e42d0d7b5bae7,0x000bfa89eb4687fd,0x000c7d89b1ca5f45}, {0x000ecce4fba638bb,0x0008a21de873fcc0,0x0000000000000001,0x0000000000000000}}, {{0x000c9c2b49165fd5,0x0005fb318f9f9636,0x0006f676d6c2cb81,0x000c633a7560919e}, {0x00011e2d4752541c,0x000199c5999a79e2,0x000515dfbee081ee,0x00053107dec5265f}, {0x0002bdc9ed0ea4e2,0x00041c5a539ab36f,0x0000000000000001,0x0000000000000000}}}, + {{{0x00009de7ecb2ac56,0x0002d837bb7a345c,0x0004863cfd4369c7,0x00077c66a5755a3b}, {0x000682ccc872cef3,0x000bc1363c743442,0x0008b997f1a0d907,0x000d72224eed734a}, {0x000d1850457f924f,0x000f3bbd996258f2,0x0000000000000001,0x0000000000000000}}, {{0x0004953714391350,0x0002a08de1fb04ea,0x0009bb0ca7d3f0e3,0x00020e2fc4a54760}, {0x000d525812eece55,0x000dc3c7c680f4ea,0x00064f097079b269,0x0001e81a267b891a}, {0x0002df53061afa20,0x0007339d7335dbc1,0x0000000000000000,0x0000000000000000}}}, + {{{0x000fa11c22d57017,0x0007061bfc65866f,0x000e25c9a781f882,0x00052d54eb5d1a25}, {0x00034e5fcb1fe128,0x0008c5dfb74f3317,0x000cca2e48b7e54d,0x0005e9b41639cfad}, {0x000e1f8c2193402d,0x000348c49e8f71b8,0x0000000000000001,0x0000000000000000}}, {{0x00033ca43943f50d,0x00016ce98a1f644e,0x000bd595ac8a7cb4,0x000d4eb1e328cd45}, {0x000e8ec3fd8cbf8f,0x000eb626b0f768cf,0x0001524476b1bbc9,0x000d8d0ffe31069d}, {0x00025aa89220edd8,0x00063660b3755829,0x0000000000000000,0x0000000000000000}}}, + {{{0x0003a78191abe914,0x000aed083110e37c,0x00072de9a657b228,0x0003d434d0dfafec}, {0x0009778c8ba568ce,0x00021f193e09ab8a,0x00032b59891165df,0x00090a0e20c8f7ac}, {0x0005b8a1f64088c4,0x000b717ed2f2d69f,0x0000000000000001,0x0000000000000000}}, {{0x000ac4849a39a0ab,0x000e9224368b5d72,0x0009a57abd9c0589,0x000f04bfeb21218d}, {0x00049bfe57f2080a,0x00082fbca66b72d8,0x000f0c09eaa93852,0x000e04db305e15a6}, {0x000bb2bcc4365052,0x000d0e18047907b5,0x0000000000000000,0x0000000000000000}}}, + {{{0x0003c1031f4f778c,0x0009bbc5cc621ce8,0x000ac957c67434bc,0x000a368627bcbc47}, {0x000adace0a905430,0x0008aa0831ed2cb5,0x000c11d0f4f5d323,0x0001c48e91c91fa3}, {0x000229765edbfb35,0x00032bdf2591e498,0x0000000000000000,0x0000000000000000}}, {{0x0008e455572bcfd0,0x000ecba53bea9ae3,0x000196518c997db1,0x0005c33258970b56}, {0x000a3c6b46d1e689,0x0008e44ad30fe772,0x0005dd6482160561,0x000b86d1933faf1a}, {0x0005a53d718ae6de,0x000e4e4345a6badf,0x0000000000000001,0x0000000000000000}}}, + {{{0x0009f15950170034,0x0008a7d9e0681bfb,0x0002dc602830c283,0x000e8af178838dfd}, {0x000987d3f8bcc134,0x0008904081cb94f3,0x000a460c0da2bca6,0x000f718b8913d63c}, {0x00022ed274e647de,0x000ff3a58fd52338,0x0000000000000001,0x0000000000000000}}, {{0x0002950ff4836dad,0x000fe4a51fa111b5,0x000e13de206b61a9,0x000941373ab42508}, {0x000c7bf8c1651d7a,0x000dbd9276666a02,0x00056dcdd411c37b,0x000f5a9ab3e87536}, {0x0000f464671775e6,0x000bc59bc827eaae,0x0000000000000000,0x0000000000000000}}}, + {{{0x000eafce57fbc903,0x00072ba11885f40d,0x000d640aa98f9421,0x000bfcf817ce6b52}, {0x000389d8e40bcdc1,0x000c804e7d14a7d6,0x0008fa880e955163,0x00034af7c92b6304}, {0x000fd685115381b0,0x000faf73ec6a9688,0x0000000000000001,0x0000000000000000}}, {{0x000f85d1af3848b2,0x0008716ba36666f4,0x000fbfc6c17f47de,0x00003b35f9474540}, {0x0004b72b1ddc670e,0x000f48bdd3ad6387,0x000d7cfd249ea687,0x0009c16141926a15}, {0x000d8d963e9d101f,0x0000771b9b1c2fac,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=127 [{1,2,3,..,}]*([2^508]*G) */ + {{{0x0006bccc65f30461,0x0006e44b51d15f3b,0x00061b0ed084d989,0x000f84c9df4f3be0}, {0x000626ed8e29bced,0x000a49395cf45bde,0x0001bfb128499bea,0x000d30e7b9a9812a}, {0x0000016d9442e44b,0x000577251b4710e5,0x0000000000000000,0x0000000000000000}}, {{0x000c58ea8faa5f8e,0x0006398972e1afb4,0x0002d603c3a6a3b7,0x000090f464b41953}, {0x0003e1c1cc6d5ad6,0x0009fc4551644bda,0x0003bf0e003b3c67,0x0004879a2d4dc4fe}, {0x0002492059f3993d,0x00089490933a0bd0,0x0000000000000000,0x0000000000000000}}}, + {{{0x0003d39ba21abe88,0x000b8dfe497366b8,0x0003fb2e32d1a08b,0x00040dcc68fadea4}, {0x0009bc78d9e5b900,0x000999242f620135,0x000e38d4ca94e098,0x000f3e9a937783f1}, {0x000c271edf5e42de,0x000cdad3d42e09c5,0x0000000000000001,0x0000000000000000}}, {{0x00014caf1b7e4897,0x00092052b710207f,0x000840b578b42fb2,0x000c859d4e0acb78}, {0x000efbd1059c69f5,0x000fc0d187f5ce1f,0x000018dd99b98e1d,0x000fd9d695b6b702}, {0x00087f7037056ff1,0x00080b73121d9b11,0x0000000000000000,0x0000000000000000}}}, + {{{0x00078d674ce9ba28,0x00098667edd1931d,0x0007a2c798ce814e,0x00054444ef1eef63}, {0x000bbc4342cdf2cd,0x0008690dad1f9bf2,0x000e17a85c72b72f,0x00027215c615b685}, {0x000ab98eb37dbb69,0x00072850ebda62ab,0x0000000000000000,0x0000000000000000}}, {{0x000b543dc482d1d8,0x00066dc72bee2f05,0x000576c719cbc26f,0x000bf9361970fe88}, {0x00012aede3868858,0x000d0a81339054e9,0x00050f3227db12ff,0x00054f85925da234}, {0x0007c4995e06e9ee,0x000343899d0c96c8,0x0000000000000001,0x0000000000000000}}}, + {{{0x000622bfd7a6e63b,0x000ca212bebe9c89,0x000487abee2cafd2,0x000143f4d290457b}, {0x0007d05d13bf4c04,0x00067f0ae3716aab,0x000a3097740d4130,0x000debe6d02d5925}, {0x000ef2c2714370b8,0x000fcffae20be9e8,0x0000000000000001,0x0000000000000000}}, {{0x00009dc727eac0b8,0x000802b463618a8d,0x000e00f824949d83,0x00021e8850666aae}, {0x000a354be3730d5a,0x0006ce164b258522,0x000386cbf3fd223a,0x0000b7ba5ba4fefa}, {0x0008ecff5479bc6a,0x000ec6ece410bb37,0x0000000000000001,0x0000000000000000}}}, + {{{0x000fb0fd2b03ceeb,0x000c34d346742d91,0x0007fb1da808f925,0x00014c3e50e576d5}, {0x00045dde45aec274,0x0007634b06ff5225,0x000e8f635cfee4ae,0x0006af16b33722c1}, {0x000b6c9512df9861,0x00087905f5ba1ed3,0x0000000000000001,0x0000000000000000}}, {{0x0006849e44fd7308,0x000ca8c188190c88,0x000568eece8b82ee,0x000afcede37dce03}, {0x000956f0105616d6,0x000fcd4f42159b96,0x00016a63204d42e3,0x000f90e66b95446b}, {0x0003ff7137d7895f,0x00015f08ed8c3b6a,0x0000000000000000,0x0000000000000000}}}, + {{{0x000080c877cdc551,0x00063fd88170e394,0x000069a5394fa226,0x000fe13aafd4abd3}, {0x000aac77685a1a56,0x0002270caf504acd,0x000b15ab3b28bcb6,0x000ff30df2535d28}, {0x000dad1836a25af3,0x000fa6c25940501a,0x0000000000000001,0x0000000000000000}}, {{0x00072e0f5dfd0431,0x000680aa3b20ecdb,0x000be825e1b5fe43,0x0005a5c2422130db}, {0x000f0dc02a5e8b8e,0x00053d6d0a6ed0ac,0x0007b39bd52bad22,0x000defc7beec9197}, {0x000d80e249e46059,0x0009869fd32b5153,0x0000000000000000,0x0000000000000000}}}, + {{{0x000d5488a9c159ea,0x000c7c99f362daa9,0x000520e4056562a1,0x000f58587c3780d4}, {0x000cf122d12decdc,0x00066fa3df2e094a,0x0006c99c7f2dc2a5,0x0005a3f3e1f8fe88}, {0x000ead106c542a2f,0x0001bd548c70f9bd,0x0000000000000000,0x0000000000000000}}, {{0x00025e9e39905a6d,0x000d9b4651431540,0x0007d3343ac64109,0x00071e3dd6b3d33e}, {0x000a550593155d88,0x0005e988444776c0,0x00092c204b5ef211,0x000132ec6af46f8f}, {0x0000975bc42ba82e,0x0000ec605e3e60a7,0x0000000000000000,0x0000000000000000}}}, + {{{0x000ebdae497fc314,0x000068fcc42342b8,0x0002e8a76fa0addd,0x000d98aa1ba58a99}, {0x0006f585d2056972,0x0001a667125a290a,0x000e664a47990be5,0x0003e44696beab19}, {0x0000ab4a22f64d1c,0x00054a0b8ce48449,0x0000000000000000,0x0000000000000000}}, {{0x000cf9a538895145,0x000e7e3b3fd1990a,0x0001c1f1592daaaf,0x000016843015bdb3}, {0x00095b2dbb2d0adb,0x000f77ef5d670d12,0x0008bf1b3f98aca3,0x0003b5280fd35140}, {0x0007a58660b5ee9d,0x000f86e58791223a,0x0000000000000000,0x0000000000000000}}} + }, + {/* digit=128 [{1,2,3,..,}]*([2^512]*G) */ + {{{0x00031c51f5b59b03,0x000c7665ab584951,0x0008739b754d5115,0x000b253e840523eb}, {0x000078a1f77e3b96,0x000742a046765d97,0x000823d7e942e5b8,0x00080b3194bd1539}, {0x0002679bf560b997,0x00061abda6ff32b5,0x0000000000000001,0x0000000000000000}}, {{0x000820e93e66dad8,0x000f2c881e08a892,0x0007e5fd839208f9,0x000d25804e86968c}, {0x000fc76aeb554305,0x0004c686c9b44037,0x0002e51b80d02e02,0x000e5774d5a620e6}, {0x000000eae653df90,0x00072ac9b31961f0,0x0000000000000000,0x0000000000000000}}}, + {{{0x000f8d7860917b4f,0x000bb3357359429f,0x000995f4b0a05d76,0x000e0f1c58d0fe01}, {0x0002921dccd2ee40,0x0000ab0ca33af9c6,0x000637c074069c34,0x0002098a102fa30b}, {0x00037701844888bc,0x000d6cb2e96e33de,0x0000000000000000,0x0000000000000000}}, {{0x00070d506a96d190,0x000f2ce57ed3ba0f,0x0002492cf26e59c4,0x000305995879a0eb}, {0x0000675760ae93d1,0x0009f9d0d04103b2,0x000d618a2b740898,0x000e723e7b444b0a}, {0x000b80451ab5c813,0x000616305a1f27eb,0x0000000000000000,0x0000000000000000}}}, + {{{0x000b480b60680f46,0x00097a71a65ccb4a,0x0002360920f061e1,0x000428aeb306dab3}, {0x0005aee5267509d9,0x00058e47b1cbaf9d,0x0003350d9a6f7e9f,0x00035af36c30696a}, {0x000ff438c1f66ddb,0x000119d4937e17ea,0x0000000000000000,0x0000000000000000}}, {{0x0009df61e7821be6,0x000b8322655044ac,0x0001ae7bb1e106e2,0x00039508343bc8e6}, {0x000a2bc1e06e0991,0x00066bd6b8166453,0x00044e23756d0eb9,0x00003795c5a1b4bb}, {0x000605deb625fe17,0x000248426f42f1b7,0x0000000000000000,0x0000000000000000}}}, + {{{0x000ebb49b8b6d8d8,0x000c9576edd0b2c0,0x000089746d87ef78,0x000ee54686ff89a1}, {0x000cd51992a8e30d,0x000fcb70ff8362ad,0x00008d8883f2631a,0x0002a13e25b5a551}, {0x000d32baa9313847,0x00049764387fbe1f,0x0000000000000000,0x0000000000000000}}, {{0x000652373d0f2fcd,0x0007e9e1299928e8,0x00016c54d21ce8fd,0x000e62d7938b0123}, {0x000ce4d602bacad0,0x00055138cbf9df41,0x000e0e625dfe098c,0x000dbf9a6851bc01}, {0x0000b0da12bdbc63,0x000aec8b07cebaa7,0x0000000000000000,0x0000000000000000}}}, + {{{0x0006a52b356ba3e0,0x000fa486ee1abbf6,0x00074bf145f1f28e,0x000ef4af39b6c538}, {0x00084376c3d3f698,0x00007cbe4b46d363,0x0003db26e35a57f0,0x000ecc45a1933b62}, {0x0007b9610a9a44f8,0x000057b84f95bccb,0x0000000000000001,0x0000000000000000}}, {{0x000bdacb48b9bb6b,0x000642a5eb931fe7,0x00097c327f5fbd2e,0x000a3062ea3deca5}, {0x000a3dad9ef787fc,0x00092f83b35a8216,0x00042427945728d9,0x0007537c6394f3cf}, {0x000b615698a93762,0x0001f468793ceb99,0x0000000000000000,0x0000000000000000}}}, + {{{0x000217c6562eea14,0x000a6a042cb3da1e,0x00060f9ac91e9595,0x00013fef00e33063}, {0x0001954b4b6c8a05,0x00007e6bb37592ca,0x00018fef0e569527,0x000f4449441c7534}, {0x0003c8e93a050abd,0x00050bb02598e0eb,0x0000000000000000,0x0000000000000000}}, {{0x00044683d956af18,0x000340076bbbf06c,0x00006b8e5826911d,0x000bd47b4442693c}, {0x000bdbfb4883b7be,0x000a67f8d229f111,0x000eceec785481be,0x000d18ec8ccc0a58}, {0x000ed2b1cc9fd671,0x000ee9b77fbdfe40,0x0000000000000000,0x0000000000000000}}}, + {{{0x00033f2848525772,0x0002def8aeb9f6cd,0x0009c738e373fa1e,0x000ca3f54bf1bb33}, {0x0000dd3b12e5c621,0x000856fbe6604112,0x0004fa2d98399489,0x0000ba4686140d31}, {0x000be8a65034e7d6,0x000fc1d2c9034f2b,0x0000000000000000,0x0000000000000000}}, {{0x0002e0e02cbac5de,0x00036d1bbef4bfe6,0x000db812e9ceffa7,0x000c4f097569d138}, {0x000c2795f975ff41,0x000cfa1a794c9978,0x000e1e65c9b377a9,0x000e6d66f7a547fd}, {0x00058785dc7270ac,0x000b34d6e188dec1,0x0000000000000000,0x0000000000000000}}}, + {{{0x000a511d09de2eac,0x000031698e16faac,0x0002e96a74ccb4d0,0x000b85017b609854}, {0x0005887d91373679,0x00039fd4a56f39c3,0x000f60f3aea2bb2a,0x00084e8edd3fc7ee}, {0x000e1d001d481288,0x0003732f4d1fa989,0x0000000000000001,0x0000000000000000}}, {{0x000629c27855b7b6,0x000652d6fdccbf0a,0x0005f32800f6bc14,0x0007f62ad29c1358}, {0x000dbf3a9fdce69e,0x000aa9f4b69418d0,0x000e5fef492796fd,0x000332f4a27a525a}, {0x0001a7293d91d135,0x0006fadc6b1bb1cc,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=129 [{1,2,3,..,}]*([2^516]*G) */ + {{{0x00076fe9e4b410aa,0x0002e2e46e8257e7,0x000285dece1c90ea,0x0004f07e02e70a0d}, {0x0007d9e22652fd2e,0x000325ca4b7fb066,0x0001305e36daaa95,0x000feebe9d3d04df}, {0x0000e3be0fcd0755,0x0005f74f74e60357,0x0000000000000001,0x0000000000000000}}, {{0x000a7ff33f158714,0x00012737b93b28b8,0x000a408dc4f02791,0x0009c78d219fcf52}, {0x0008fbf0f03e3758,0x000d6cdb0c0bb10f,0x0007428d517b4d78,0x000d37e06b1f9904}, {0x000f8786ba69f3ae,0x000b508624d39698,0x0000000000000000,0x0000000000000000}}}, + {{{0x000314e66fead340,0x0009f5c35e0de977,0x00086281b424220d,0x00095c9d69b8f421}, {0x000075fd90a74e5f,0x000c09fc499a89c7,0x0004b64e12f34808,0x00068c161166a676}, {0x000f40cf1886f3c3,0x000c4968ad8aaa55,0x0000000000000000,0x0000000000000000}}, {{0x0001f4bec36790bc,0x000d77489002d4f9,0x0001759ca38e8177,0x000e759ae14e5a1f}, {0x000289005868dab2,0x000a1dff8ca02b41,0x000a4d82b9cd06ea,0x0004578741ea12d6}, {0x000343e8d641aef6,0x0002d4c6a85c8b1a,0x0000000000000001,0x0000000000000000}}}, + {{{0x0007c33639a0b3f1,0x000010f791b828fa,0x00069b98b785836e,0x000f9367ba2d3b6c}, {0x0005ecea4290d504,0x0003dd0621586083,0x000de6622ac6245f,0x0002d8153e71208b}, {0x00091cd77d2a1a55,0x0007ebc7df7c52cd,0x0000000000000001,0x0000000000000000}}, {{0x000fe4e1268ac84e,0x000d05fdf620f30b,0x00078c9c26de9aa6,0x000fef3a7d0e875e}, {0x0001f45acf57d581,0x0009769d1b4f2f3f,0x000f8a2b516d4fb8,0x000ca50e2afa7161}, {0x00069ea98a831731,0x000a4069e2a679a5,0x0000000000000001,0x0000000000000000}}}, + {{{0x000573b5ab69b37b,0x00099e464e098a37,0x0004b6ae9bd2eac1,0x0007e83941109e44}, {0x000b3638c710996c,0x0005b0374099e6be,0x000e0cd0943a1c3a,0x0008fab6ecb18490}, {0x0009f0eb14e71bae,0x00011229f06246b6,0x0000000000000001,0x0000000000000000}}, {{0x0006e2d865b8fa8c,0x000e4ba2a3d8ca53,0x0004c428120f4f50,0x00003e5bcb6eaa4c}, {0x0006c69871129b42,0x00018bdacf6da13d,0x0006a314f621f852,0x00031ea900e85204}, {0x000d280193a13fa0,0x000437167b70a8c7,0x0000000000000001,0x0000000000000000}}}, + {{{0x000e47d41aa029d1,0x00083c1c1e4e8c82,0x000819d110bf6923,0x00080e340caaa47b}, {0x000e4f6e315e8cb2,0x000865960449f5a7,0x0002a736db3e3bf6,0x0008b38233a8c18d}, {0x0006ab95cab54c2d,0x0007b4670af6e721,0x0000000000000000,0x0000000000000000}}, {{0x0009cd8d15df2f21,0x000c9e95f873ccf7,0x000d4ae259b8946e,0x0003026a352c8cec}, {0x000c84b6773b4638,0x000327edc3574f14,0x000980243f9e1167,0x000c58a2e8d3fc2d}, {0x000e789b426360b9,0x00084beac487c72a,0x0000000000000001,0x0000000000000000}}}, + {{{0x000a504d7a128f19,0x000bec706161d077,0x0008d3f449e0d717,0x00027553cd6aa437}, {0x00023bfa097584c3,0x00031032e9ecfed0,0x000de734abfe6661,0x0005972524c62301}, {0x00068a20ad67cd7b,0x0001d7d8e4bd981b,0x0000000000000001,0x0000000000000000}}, {{0x000dd411d75c8560,0x00020a136bd0f9a0,0x000e9c06f6a8521a,0x0001770134117a07}, {0x000c6625cc2c0b14,0x0006f01c93534fec,0x000bcd3698e9742e,0x000543a9724acea5}, {0x000820ced54b554d,0x00005a7954cbcc77,0x0000000000000001,0x0000000000000000}}}, + {{{0x00060a7aed1a8afb,0x00037a6526a7b404,0x000f752335e80ab0,0x000a8f65f14c43bd}, {0x0004f9b989eabd11,0x000976a80f23fa92,0x0000171f16dad870,0x000d3baaac454e44}, {0x000f34704b9635c0,0x00051b0e1ca863f0,0x0000000000000000,0x0000000000000000}}, {{0x000fcc07647c957a,0x0007e8420a4e5f40,0x000cec847e324cf5,0x000db2fe1f189869}, {0x0009827f88deeb20,0x00023c17a4487124,0x0009568284e63444,0x000010b9bfa3fe02}, {0x000626ee2d426c18,0x0006c496cbf2d807,0x0000000000000001,0x0000000000000000}}}, + {{{0x000a48e1016cc257,0x000e7a08472093bf,0x000b7492e4ec8ac9,0x0001bd1c471d73ed}, {0x000a7243428ce53d,0x0007d49d1740adec,0x000c66dd60077c0b,0x0005737593cb7a95}, {0x0007c6f6ef237e1e,0x000660a7929ff06a,0x0000000000000001,0x0000000000000000}}, {{0x0004bcc2e6c3da59,0x0003e87b3c416f89,0x00021231992a493d,0x0006d0824c0a5993}, {0x000e60d5c6c61f37,0x000a3430f29e1550,0x0005e81f1d1beb92,0x00027b6bc5ab11f9}, {0x000858c60a78bae0,0x000f95bdd9ea9017,0x0000000000000001,0x0000000000000000}}} + }, + {/* digit=130 [{1,2,3,..,}]*([2^520]*G) */ + {{{0x0008b69f6c0a103c,0x00014cb96747cd0b,0x0009e707ed23f30c,0x00092336029690c8}, {0x0002b884265ced12,0x00087e627555cfec,0x0008f62a097ab7dc,0x000df635118ea555}, {0x00070b69e23dda1a,0x00011d1b70ffff57,0x0000000000000000,0x0000000000000000}}, {{0x0002606bce985157,0x000cf38c9c6bbd4f,0x00058d1b308ccb92,0x000fb6fe2ce6913c}, {0x0005b1967bfefbbd,0x00053132fce5cb51,0x000b2f22527584e4,0x00082a1591dfc389}, {0x0002d57610460d61,0x000dc57498b0d160,0x0000000000000000,0x0000000000000000}}}, + {{{0x0008cffee5938175,0x000028a135a61fb3,0x0002c9fe1a18ac17,0x000b968e30418249}, {0x0000b4c958b813ec,0x00023285e86f834a,0x000b05e54df836f5,0x0008f77b5fb229e4}, {0x000371505e864d89,0x000fb3cce3c32e48,0x0000000000000000,0x0000000000000000}}, {{0x000a0e5e3ce8bcd8,0x000dc2fb74fd48c4,0x000875aaa2018996,0x000827185e9f86f5}, {0x000196642422b5eb,0x0006d6f0dd874310,0x000f5719d7e7982d,0x0005346044e6cb46}, {0x00002d250a348c67,0x0009ec517508afdc,0x0000000000000001,0x0000000000000000}}}, + {{{0x000145140a7a865f,0x000095ded9f8c118,0x00040df2b19ba498,0x0007dec03834dace}, {0x0009e3687e664a0c,0x0009d971f2feee9a,0x0008796a032ee720,0x0003a0b2cc310714}, {0x000067d48b65bcec,0x0007f5d8c2af6546,0x0000000000000001,0x0000000000000000}}, {{0x00015c856b094864,0x0004f779c821c5d7,0x00062cf6310bebe0,0x00086c833844c220}, {0x000322644f899899,0x00080ac439439357,0x0009f2dfd741ec2f,0x000c3022b589299a}, {0x000663af1bcf4790,0x000a46767587f6f5,0x0000000000000001,0x0000000000000000}}}, + {{{0x000cdfa0667f8929,0x00055eda7b7826f1,0x000dd93845997fea,0x00073f54c39699cc}, {0x000d85971fbd417d,0x000631f8049f08e1,0x0003b8e65a870ee3,0x000aa3cdc8e351a7}, {0x00099901454f69b2,0x0002d659cbe0fe85,0x0000000000000001,0x0000000000000000}}, {{0x0005bfaeb9c77879,0x00078817f6a7aa08,0x0001e5f6c61e07f9,0x000a5dffb271762a}, {0x000df5d3e3cd3cc6,0x000eadd52e895eb0,0x0003b342cd665b46,0x000393078c5b9252}, {0x0006202122ebcff3,0x0005f4777dd50c48,0x0000000000000000,0x0000000000000000}}}, + {{{0x0003fcdbd57bfa90,0x000d1b118dee98fe,0x0000a524ceccfc4a,0x0007420c6d1ffa5c}, {0x00053919d859de0d,0x00081fb74544af55,0x0007c3f0981d6def,0x00076680c297a17a}, {0x000d22135e0cc4fe,0x000d15de36e57af6,0x0000000000000001,0x0000000000000000}}, {{0x0008ae03fd78b0a1,0x000c06ac0c41950b,0x000c994e2c7c2638,0x00075a944523ebb6}, {0x0002bdb72a76549b,0x0007a73d0573310e,0x000ae4c8ce6d6b85,0x000e4309eb9e1e23}, {0x0008aedc0842f06a,0x0006ededf71264f9,0x0000000000000001,0x0000000000000000}}}, + {{{0x00047c62f7f81bbb,0x000568d4c38f8cbe,0x000085a52c049173,0x000151d87d057b19}, {0x000462de5c929612,0x0004036de18a4ea9,0x000290b0a8300ddc,0x000e195518bc5a6e}, {0x000e21097894ed83,0x000a74d8dd898c35,0x0000000000000001,0x0000000000000000}}, {{0x000eee2e29a47676,0x000325c0b89ddfd0,0x0000c5c995a8a7bf,0x0003e514a5131a24}, {0x00078e5998b6e957,0x000ba59336939798,0x00032157906f3c87,0x0001093cead5ff51}, {0x0003bc2e1b76588a,0x000a055abc03c824,0x0000000000000001,0x0000000000000000}}}, + {{{0x000c9d2081074b7d,0x000b4b234d223107,0x000c6fe7d17524b8,0x000b3e26469a9182}, {0x0001d35461f18688,0x0008f9ef1eb0f49d,0x00072164bb2130a7,0x00069240661ee72b}, {0x0004674d4ec1e4e4,0x000df195346b1529,0x0000000000000001,0x0000000000000000}}, {{0x00094d4cd2beae09,0x00021cf533e3fea5,0x00099051c26db75a,0x0000237aea1a808b}, {0x00066230cfcd19c1,0x0000aedfcfe63965,0x000965518cdd9349,0x0000446502d88c39}, {0x0006de0650de6be1,0x000c487b3640aa4a,0x0000000000000000,0x0000000000000000}}}, + {{{0x000315f2c4f3b202,0x000844f2e07c55f6,0x0009040fc8f62ee6,0x0006bb02af168881}, {0x000eea1922677aec,0x0003b90132968e4d,0x000b75f92b21f9e5,0x000b4ae4c1d62b9c}, {0x0000f373265408d1,0x00013d70be8f94d2,0x0000000000000000,0x0000000000000000}}, {{0x000c00995b2d1ac5,0x000acf4e16ae2b05,0x000d04ee882f20b5,0x000b5e061f8c5834}, {0x00040c9dbfe4135d,0x000f7b55b63a42f5,0x0003b47500e266e3,0x0004da6911e652ce}, {0x00043c4db0c53d0a,0x0009946052a6d7ee,0x0000000000000001,0x0000000000000000}}} + } +}; + +#endif /* #if !defined(_DISABLE_ECP_521R1_HARDCODED_BP_TBL_) */ + +IPP_OWN_DEFN(const cpPrecompAP *, gfpec_precom_nistP521r1_radix52_fun, (void)) +{ + static cpPrecompAP t = { + /* w */ 4, + /* select function */ NULL, + /* precomputed data */ (BNU_CHUNK_T *)ifma_ec_nistp521r1_bp_precomp + }; + return &t; +} + +#endif // (_IPP32E >= _IPP32E_K1) + +#endif /* #ifndef IFMA_ECPRECOMP4_P521_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecprecomp7_p256.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecprecomp7_p256.h new file mode 100644 index 0000000..5bab5b8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_ecprecomp7_p256.h @@ -0,0 +1,2544 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#ifndef IFMA_ECPRECOMP7_P256_H +#define IFMA_ECPRECOMP7_P256_H + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpecstuff.h" + +#include "ifma_defs.h" +#include "ifma_ecpoint_p256.h" + + +#define BASE_POINT_WIN_SIZE (7) +#define BASE_POINT_N_ENTRY (1 << ((BASE_POINT_WIN_SIZE)-1)) + +#define OPERAND_BITSIZE (256) +#define LEN52_P256 (NUMBER_OF_DIGITS(OPERAND_BITSIZE, DIGIT_SIZE)) + +/* P256 affine point */ +typedef struct { + BNU_CHUNK_T X[LEN52_P256]; + BNU_CHUNK_T Y[LEN52_P256]; +} P256_POINT_AFFINE_IFMA_MEM; + +extern const __ALIGN64 P256_POINT_AFFINE_IFMA_MEM ifma_ec_nistp256r1_bp_precomp[37][BASE_POINT_N_ENTRY]; + +#if !defined(_DISABLE_ECP_256R1_HARDCODED_BP_TBL_) + +/* Montgomery coefficient R = 2^(6*52) mod p */ +const __ALIGN64 P256_POINT_AFFINE_IFMA_MEM ifma_ec_nistp256r1_bp_precomp[][BASE_POINT_N_ENTRY] = { + { + /* digit=0 (1,2,..,64)*(2^{0})*G */ + {{ 0x0008905f76bdc7b5, 0x0007bd418a9143c1, 0x000c475d568abc1f, 0x0009a701075ba95f, 0x00003d1f32c8b4b9 }, { 0x000571ff18aafa5c, 0x000f757ce95560a8, 0x000434a7e54432f7, 0x0002797258b4ab8e, 0x00009df870e37032 }}, + {{ 0x0006bb32e5348a6d, 0x000d9d410ddd64df, 0x0000ad784f985075, 0x0008a23d9aa6ae3c, 0x00001fb0f13f1e58 }, { 0x0008c57751832101, 0x00047d361bee1a57, 0x000b725df8a6ac15, 0x000e32cbe152cd7c, 0x00008c240484bd0e }}, + {{ 0x0006936a3fdd92de, 0x00061904eebc1272, 0x0009e1ea17bc2219, 0x00038de98b027f84, 0x0000be1daceb9daa }, { 0x0005f06a2abb7836, 0x000261fc983a7ebd, 0x000c327193eff4d4, 0x000b6b38e583e47a, 0x0000315e09d4065e }}, + {{ 0x00077362f599237a, 0x0003b0d46918dcc5, 0x000d6eb05e7ddb8d, 0x000ec90f24650a6e, 0x0000604e8ac3b74e }, { 0x0004b14c39d10c04, 0x000ee4ce4cbaba68, 0x00017625a80d5c8a, 0x000d1ce2e1762847, 0x00009cb7c6eea514 }}, + {{ 0x0009079606520cb9, 0x000d1aec45c61f5c, 0x0009cbb1bd776c0e, 0x0006a090c90ec649, 0x0000ce1d21d8a47e }, { 0x0003a076bba1725f, 0x0003c4ae7ba4f107, 0x000f724b117049be, 0x00007c06873c568e, 0x0000d1631291cbdd }}, + {{ 0x0003074b59672881, 0x000c6373e77664a1, 0x000f2165a2e4d910, 0x000ef22ad55ae744, 0x0000cd292bcbc0f3 }, { 0x0004c9916deed188, 0x000da20d377e44ba, 0x000309358347a501, 0x000921711299c2b5, 0x0000cd3e2ccadf00 }}, + {{ 0x0003ba5119d6cc8f, 0x000a64ea0173b4f1, 0x0003be81afdd3079, 0x000572c082bd2021, 0x00001db750e89b35 }, { 0x000aedd9165911e1, 0x000ef303f5b9d4de, 0x000172b8a2c6cf35, 0x000b760956742f2f, 0x00007d5db743c61e }}, + {{ 0x000763891f69c8ac, 0x000a0d19499a78fb, 0x0004b837ab35be28, 0x00064506b462ab5c, 0x00002c41f6130b86 }, { 0x0006ec12f2cdf2f2, 0x000b1a9532d49775, 0x000d78b2a72327aa, 0x0006dc9f021e3327, 0x00006889435a91f0 }}, + {{ 0x00005b3081d12011, 0x0005d8f264e20e8e, 0x000c794c77bfa4a9, 0x000a8da00abe6bfe, 0x0000ec1d857c8273 }, { 0x00086659ce05e9b9, 0x000be7aa45f33140, 0x000dc5f6ec1518cd, 0x000c7761a56af7be, 0x00006928f160cc82 }}, + {{ 0x000c61c947f1150b, 0x000d0d19dc21ec8c, 0x0009436db7f13b03, 0x00010004998f9868, 0x0000d698306e8e57 }, { 0x0008719a5563ace5, 0x0005b7880dd9e767, 0x0000bfce23b2a87e, 0x000ef00d2c43a899, 0x00000a4ab8781013 }}, + {{ 0x000e7090f1a30d21, 0x000050a6245e4043, 0x0005300f4eef4770, 0x0009ef9b59de4079, 0x00006484903423ea }, { 0x000893002449b8b8, 0x000cd612b944e886, 0x000e7cec61a3d0bd, 0x000f4ac3d250f939, 0x0000b1ed336264d4 }}, + {{ 0x000ee1df781a65c0, 0x0005aa7d26977684, 0x00057b527abaea51, 0x000d1785eabdedef, 0x0000cdef8db325d3 }, { 0x000ff57c95d31907, 0x0000bd91cbb5b437, 0x00064a93d9a13993, 0x000c56666170ed2f, 0x00005f7b461764b7 }}, + {{ 0x00038477acc71a2f, 0x000f6634b2ed7097, 0x000911eb5b6105a9, 0x0008469110e35676, 0x00000610528b928e }, { 0x000bc0876ac5a83f, 0x000ec90c00ee17ff, 0x000f786e4b786fcc, 0x000771cc16874838, 0x00000146b81bb7f7 }}, + {{ 0x000aae7333f7c13a, 0x000695bb0cf664af, 0x000684e894afaa81, 0x000c1ec60126e48f, 0x000061142a8620ce }, { 0x00052706dc43d122, 0x0000bb7995d586b1, 0x000e6dfb30e5e3e4, 0x00094c5dbbe29569, 0x000098178e3461e6 }}, + {{ 0x0002aa0e43ed7de8, 0x000f0b7bc60055be, 0x000d74387007853e, 0x00003ab7003cc23e, 0x00004cf9074de0f7 }, { 0x000042170a90bbc4, 0x0008e4f6383c45d2, 0x00042262041aaffc, 0x000ce8397d766355, 0x0000fed56953d3e6 }}, + {{ 0x0004d63c80e5d9f3, 0x00018650bc6fb805, 0x0004eb27f1ea9ab1, 0x000aa02495882e07, 0x0000a466f2e5fb46 }, { 0x0004edf7b266ddd9, 0x00042d652a23f9b6, 0x0008e61d6dd58c13, 0x0007a359e3670c31, 0x0000d257b443894a }}, + {{ 0x0001a35c0b2e1db3, 0x0006acbd53c5c9dc, 0x0005ea66bb7068eb, 0x000e07afff17624b, 0x0000591d6b9069fe }, { 0x000bd512271e49d5, 0x0005c4693e79987e, 0x000ffcc7a5e70433, 0x000ceaebb0575bf2, 0x0000f661c2e757ee }}, + {{ 0x00075abd1bd6dba8, 0x000a9c856ada97a6, 0x000d283083228862, 0x0005281794afc964, 0x0000f61673328f6a }, { 0x000c6fc7d9717201, 0x000bc725e485a1d6, 0x000f22df4362f530, 0x000127aba4b2a5ca, 0x0000262b28cba5aa }}, + {{ 0x000e545c2926ec20, 0x000f2baa5659ae8f, 0x0009278d4545d7bc, 0x0004021ac68363ab, 0x000024e11be64e4e }, { 0x000720ee26643347, 0x000a2e7dc4c696bf, 0x000a0488899ec903, 0x00061c1c15806244, 0x0000a0f278b7757f }}, + {{ 0x0002d1eeb7e12dea, 0x000a729d2337a31e, 0x000e2cb43d24a969, 0x000a499ba5916868, 0x00000502dc95217c }, { 0x000ee104e224dfe5, 0x00037417884005d5, 0x0007a11ec6cc5e0f, 0x000c97966a2d4ec1, 0x0000b25760a55d65 }}, + {{ 0x000cead9f5ae523c, 0x00010ecc135b208e, 0x00045a991a87a7dc, 0x000ecff3074e1b26, 0x0000f12e91b276ab }, { 0x000086d5f289ce55, 0x0007fb2e286e5b9c, 0x000fc334666f68cf, 0x000ddd4a41b0fadb, 0x00009543bc814bf5 }}, + {{ 0x000dc777e9e6fad0, 0x00041352f6f3076f, 0x0002e5c9b2b028cb, 0x000fdea3dc99ffa2, 0x00008d533e5e9beb }, { 0x00093e8d07d69b5d, 0x0002abb35415f047, 0x0009d3e5719cbe0c, 0x0009ed8b5640c2d6, 0x00007bfb45e7d23e }}, + {{ 0x000f2cea7cc654a9, 0x000cd532de45068a, 0x0007ef0134f22ea3, 0x00003976937c7a7e, 0x0000bee7aa1e25dd }, { 0x0003d0687c1447ce, 0x000d6fb9e4785a98, 0x000ba35936cdef18, 0x000646171e5fda49, 0x0000ead87df3975d }}, + {{ 0x00097b2330287754, 0x0002fb215d02819f, 0x000f4762770544c1, 0x000b3f29f6d0d754, 0x00004490d3fcc8f4 }, { 0x00095dc55f23d140, 0x0009750b24bcbc75, 0x00092f441df78180, 0x000fa350111ea494, 0x0000e940dd6ad26f }}, + {{ 0x00066742ebfbaa6d, 0x000acacd837879f9, 0x000e570424c4a8f8, 0x0004786af6fc1b49, 0x00001a48d8c34737 }, { 0x000b948dc3d28c27, 0x000a2feb49662287, 0x0001c85fab8c7901, 0x0002b0aee96cc631, 0x00002b68fe2195f7 }}, + {{ 0x00094baaa8ef3ea5, 0x0000832ed1f80089, 0x000a3f65ca8f72a6, 0x0004151fbe6c9267, 0x0000b870bac04296 }, { 0x000d178031be45da, 0x0003ae11d248018e, 0x000a757344d6c689, 0x000636af55a39898, 0x0000f7e62c4349a1 }}, + {{ 0x000f8aa54bce826c, 0x000bef497e2feb49, 0x00070f34cdc735a9, 0x000cd91d7d36a42d, 0x0000b578ca909f39 }, { 0x000d431069058eee, 0x000c9ba09895e702, 0x000fb074eaef8c71, 0x0005c5ebb3bd0c66, 0x00009a05e3ae9f73 }}, + {{ 0x00097f6b6bdc2a47, 0x000f7b7172ccd1fe, 0x000747273d266f27, 0x000b70ceb7323cb2, 0x00000289b30a54f3 }, { 0x0009b4cc5fea0d1e, 0x0001b3ed1a83da1a, 0x0006d6965e083330, 0x000ff991ea6a7f9c, 0x0000d78ac0dce842 }}, + {{ 0x000266f95a2ac736, 0x0007e6acb66e132e, 0x00092b2b5f263396, 0x0005083e2f1be963, 0x0000a1dcaaacccd8 }, { 0x000798142a61d3f6, 0x00070735c2088993, 0x00076b6304793707, 0x000a86b9f9047244, 0x000045b4d24b0f15 }}, + {{ 0x0000288cb622df88, 0x00094b13298b343d, 0x0008e426d8eb9144, 0x000e7f2cda3a14e4, 0x0000f218abd21717 }, { 0x00060d45ce976d65, 0x000272f1c040abc4, 0x000dc0b2557c3e5f, 0x000cd3f41b675511, 0x00004127c7f96903 }}, + {{ 0x00048dc4f9468a4c, 0x000f0696755ff899, 0x0006b2a252ed9c6c, 0x0006d140ddd3cf7e, 0x0000ad40f1efa3f6 }, { 0x00038ae89241062f, 0x000cf814ea53299f, 0x0001a560779698e3, 0x00091d4fc2d921ca, 0x0000d74868346acb }}, + {{ 0x0009227077a277a3, 0x00059cfc5e3a3d83, 0x000c07576f48364e, 0x000355e97befd904, 0x00001c25c29ce15c }, { 0x000231d792f7c39a, 0x000a46ddbef42f56, 0x0003e5d8c175121e, 0x000d694407e449b6, 0x00003711285e87b5 }}, + {{ 0x0003ec6869243119, 0x000fb560f6645343, 0x000c177c889a62e3, 0x0002671b985ceae7, 0x0000759c4e754541 }, { 0x00016c4304749ba6, 0x000dd2b3a8ec1f1b, 0x0007a6db92eaf628, 0x000a36f745509d12, 0x0000c8bc38c34b01 }}, + {{ 0x000982ddd5a6b31c, 0x00004147ca23cd3c, 0x0005da0ea6c80ff8, 0x000b47194caa7a5c, 0x0000393836b1cdce }, { 0x000159100cf86685, 0x0007b0b5db6f971a, 0x000ece2e5f860514, 0x000f00f26bba2cb1, 0x0000051e80a79643 }}, + {{ 0x000a640b870d07f0, 0x000947fdec67ef5c, 0x000358da6b30f00e, 0x000bac4c8742ee46, 0x0000a74e3225ade2 }, { 0x000ba90162b75d7c, 0x00085908ceb6aa92, 0x000c28cf5172addc, 0x000c5fef7313c300, 0x00005ab09925afb3 }}, + {{ 0x000569656d0ab3a3, 0x000b5d6a8219bb73, 0x000ce59d601232aa, 0x000a3b46ec691d0b, 0x00002fe02f08a3c7 }, { 0x00080b9e12257943, 0x0006bbd94cda03a4, 0x0002ce206001c072, 0x000d85dc6934e82d, 0x000038227865a8bc }}, + {{ 0x000763a388070738, 0x00043c58cce08b52, 0x000c57b3e9ccf93f, 0x00070a006b636458, 0x000073a51820269f }, { 0x0008b36143bab3e9, 0x0005b8a7d3cebb65, 0x000e166d770ce677, 0x000a45d2ce7adda3, 0x0000ec583f44a15a }}, + {{ 0x000df0c7dd2a5105, 0x000beca4d63c0eea, 0x000661f880f15fdb, 0x000ddee3b5123311, 0x000048f0c43aa7f7 }, { 0x0009a4c281762585, 0x000a83bfe7765e5d, 0x0008e5934187989d, 0x00033ac9cccaab35, 0x0000ce5a2b475518 }}, + {{ 0x000853e4c48e856b, 0x0002988a059c1425, 0x0004a6b7ae821b53, 0x00075cb646f5ae71, 0x0000703cc0a9b888 }, { 0x000c0bc0e57524e4, 0x000992363c52f980, 0x00067576c7916db3, 0x00029971c2ec4a76, 0x0000edb0d188156c }}, + {{ 0x00062d7b00bc24ad, 0x000bb1db82e9f3e4, 0x000b98f8663c28b2, 0x000cdc58083200f0, 0x000027ff00ca4baa }, { 0x0003466612d4f0d6, 0x000cb88f26517980, 0x000c3364b622041a, 0x000046d5974c6e18, 0x0000427ccfde908a }}, + {{ 0x0008100a5073116b, 0x000f5492ed22e919, 0x0004d6f71dd1ef3b, 0x000bcb7716fdfe0b, 0x00009fc9cb71c4c6 }, { 0x00070bfeecfa9449, 0x00091a2a35c628ff, 0x000dd1729b5abc1c, 0x000c390ceb6f94d2, 0x000067c9ff7c6c22 }}, + {{ 0x00049c89cec09788, 0x000d4cfed2bad016, 0x0002b90a1bf89f1b, 0x00011a78fa155cbe, 0x0000121edd577d70 }, { 0x0003a8cfe47d654d, 0x000197de9ff257af, 0x0000ff8380ca53f3, 0x000fac7cf9b35961, 0x000020d4bd2f2712 }}, + {{ 0x00079c85db554c9f, 0x000a0fd248a7d061, 0x00046febee1efaa5, 0x00027ba8180c5bfb, 0x00006ff589c6b4b0 }, { 0x0007795f4fdd2c3c, 0x000746261a6966c4, 0x0002a3638658dd15, 0x000ba67184d82d05, 0x0000f03444a9ebc8 }}, + {{ 0x000f3425f708c1f8, 0x000841fd55a897c5, 0x000b535ceb04e43b, 0x000bfdbb0009194a, 0x000088089bf24fda }, { 0x000b629f9123de43, 0x0002387730d50dae, 0x000b5fa59bee9bc2, 0x000bef1170423446, 0x0000ba71f5c7ecb2 }}, + {{ 0x00094f7d3556d1af, 0x00077ccd5cd79bfe, 0x000aab1cf0e2c812, 0x00075d0580032940, 0x0000ac88a0bcab00 }, { 0x00073acbff0414b2, 0x0001996782ba21ad, 0x00091ff067a6c333, 0x000e35a81c525446, 0x0000b3a408dcdfc3 }}, + {{ 0x000ec684c472efb6, 0x000ffb149ee90d9c, 0x0009d1a819f5cd85, 0x000086fe0785c339, 0x0000a0452c3e9739 }, { 0x00033da338c4f59b, 0x000c7c9c1586e630, 0x000657769bf27db5, 0x00001e85d45431d6, 0x00009e1980aa8791 }}, + {{ 0x000b969975373e25, 0x0005013caa760974, 0x0008eb319f90f0fe, 0x0001ccdc70a624fa, 0x00007256c83b773a }, { 0x000bbff86f099df5, 0x00060ccd921d60e9, 0x000fdfeed1a41869, 0x0004f9ed49b944e0, 0x000095725f99e254 }}, + {{ 0x00084039030f437c, 0x000c53077adc612c, 0x0008f397ca9d5bf0, 0x0004489bda749652, 0x0000af611eeda45e }, { 0x0005efc7e9c72224, 0x0007b66175adff18, 0x000b31e8fdcd72d3, 0x0007b51e30b26d7f, 0x0000abe9a851e48b }}, + {{ 0x000bdbe608fb93c1, 0x0004a421ff6acd3a, 0x0003a80083002601, 0x000e2a027ffe7048, 0x0000344f04aba92c }, { 0x000394077fc5dc01, 0x00091285e1109e8d, 0x000720d053dabda0, 0x000a58dd826147d2, 0x0000b53fc0ef1331 }}, + {{ 0x000dceb8db394d64, 0x0005ed0488c171a0, 0x000a0599999cd66c, 0x00064966aa78bfba, 0x000083197a8d8d9c }, { 0x0007d341959b35cd, 0x000b3899ae2b710d, 0x0005cc715b012468, 0x000de6d46efde7ae, 0x0000a80719fcd630 }}, + {{ 0x00020ebe0e2e4e62, 0x0008a1c3b3f64c9d, 0x00094c502131223e, 0x000f4b6d4e20bc0b, 0x000051365d8f7700 }, { 0x000731e3847a272f, 0x0001fa773241ea3d, 0x000b86e87be0bce4, 0x00040ed6261f1511, 0x0000b13c9b2db908 }}, + {{ 0x0008f77f3559dd64, 0x0008f6e834459048, 0x0009264e7a8accb5, 0x00097c4b72e0ec46, 0x000077614016cc6c }, { 0x000ef2f2271b0201, 0x0008c295864687c7, 0x000da3ec433bdb78, 0x000bcc29e23b92ea, 0x0000b3f836102480 }}, + {{ 0x000c3a318e1ca066, 0x0003038546c4d8de, 0x0003a6b814eb7a95, 0x0002846d95f9c3fc, 0x00003dc194c05f8c }, { 0x000ba1da5a3ee82c, 0x0005341c31cb264f, 0x000e567f3998412e, 0x000c7b5973db82f6, 0x0000dcf49c0e789d }}, + {{ 0x000b8a0feba9b0d8, 0x000838eb212cf53b, 0x000e3ccb6c73f691, 0x00013fd6c4f2e512, 0x00003caf4f380d7c }, { 0x000a556e4004eb04, 0x0006cd2e7d6107e1, 0x0004cd0cf76abda4, 0x0002e32b17ee0c44, 0x00002b28880d6418 }}, + {{ 0x000cee844a04a5b5, 0x00006f9e73056835, 0x000eab00d79c20e6, 0x000225fc131eea5b, 0x000026b5a23f0042 }, { 0x0005e801401734f5, 0x000a0c5a0182bde2, 0x0007783fa574d37f, 0x000b023caee759f8, 0x000034b41c5e2869 }}, + {{ 0x000a3f0c4b2e90ba, 0x00087407e8d7a14b, 0x0009db17e71f6133, 0x000cf4644bb1ff3c, 0x0000added5f139e5 }, { 0x000392768185fea2, 0x000205b1f9af32a8, 0x00062e5faaca59cc, 0x000b6f4ac01e0db8, 0x000089510687411e }}, + {{ 0x00013298312c0224, 0x0000ee77acaca28c, 0x00070605c2f812c6, 0x0001c8b381bfeea5, 0x0000fc71519eb2d1 }, { 0x000a44c6c6da463c, 0x000b6db2aac9e59d, 0x000cdc1cb163dec3, 0x000553e015748060, 0x0000ecc5ac9b6bed }}, + {{ 0x00005f725f94ec75, 0x0008be8a7ea82f03, 0x000db3ca28505d7f, 0x000eb16e199cac80, 0x00002d3afce656e9 }, { 0x00063749945c9e90, 0x0008bc4623fb21b0, 0x000e742bd018d74e, 0x000c16b3efa70533, 0x0000d56e129c9961 }}, + {{ 0x000c2bab1bd708fe, 0x00050d65c46aa8e4, 0x00035ca4b8b592cb, 0x000c87e362100d5d, 0x0000d356747acecb }, { 0x0000624998e75dc4, 0x000250c15426704a, 0x000307d5a251b927, 0x00086b3f8778afcd, 0x000023ad4d716895 }}, + {{ 0x00040c4cce907be5, 0x0008328ed8a6e19d, 0x00056d7d8cf94ba2, 0x000a054bc33fc233, 0x0000e2bedf2813d6 }, { 0x0004820109c066e9, 0x0004cbbf80e75ca0, 0x00001e2bfe22ef3a, 0x000eac49612c651a, 0x00003c7feeecb38e }}, + {{ 0x00094a44a734cd7a, 0x000711a7f4c04cc9, 0x00010ff60add019a, 0x000013b0a5955662, 0x0000a10d512bac4a }, { 0x000d1bc7809447a3, 0x0005519dbeb9b690, 0x0000e1b86d8d43b3, 0x000fd18a2473c568, 0x0000743d915fe645 }}, + {{ 0x000dbcb03af2f664, 0x0008d75a1962f910, 0x0001df9c456cf23f, 0x000933e336ed7e06, 0x00004dfcb458852d }, { 0x000b60cfffa10db0, 0x000924ebb32e38e6, 0x000efca5a01f42ce, 0x0003e39ee6472e5e, 0x00000912f4df98e0 }}, + {{ 0x000763a4348a5ffa, 0x000df319ff91fe50, 0x0000e8edeab92e73, 0x00022d126039c480, 0x0000c218bfcbfe07 }, { 0x0000876a01d11d05, 0x000ffd5383e76ba9, 0x000ff1c717959f5e, 0x0000608e0ac98b92, 0x000071d355178dc0 }}, + {{ 0x0005fb8394ec96bc, 0x000daf305968b809, 0x000272a3f024eed6, 0x0002f6201380a091, 0x00005bd44243eed6 }, { 0x000967204437eb69, 0x000fc1280edfe2f2, 0x000966478f3e6250, 0x00056e2f88f726bb, 0x0000b48e94ed5c40 }}, + }, + { + /* digit=1 (1,2,..,64)*(2^{7})*G */ + {{ 0x000d311f2cf406c5, 0x00006817a2ad62aa, 0x0008ff31e0276fbf, 0x000a6d8b57ef2b6b, 0x0000437e1f6177bf }, { 0x000a38d321e63954, 0x0002c87c2ff3b6df, 0x00023c3f61ddd141, 0x0009d5d38b46feaf, 0x0000a9e4aaa74b8f }}, + {{ 0x000c3da1963d1b37, 0x00058fa696946fc7, 0x00083d8e03d70b52, 0x000d8231550fbc6d, 0x0000c23f1ad7a853 }, { 0x00099d49ebcaeaa8, 0x00055b03cfd5d8b3, 0x00024f00b2ba1b89, 0x000385daa0704b7c, 0x00003f881bb6342a }}, + {{ 0x000e4b32b147920e, 0x0003b6d6ec293cde, 0x0006a0251f43f078, 0x000b7060a733dbda, 0x0000a65cc513dc70 }, { 0x000af675f284aa5c, 0x000a1a9cae368d14, 0x0000b2ba3d012d1a, 0x00090686e5001a7b, 0x000048a7a97981a9 }}, + {{ 0x00042370d35642b3, 0x000ce10399492969, 0x000aa1fa40e1eea0, 0x0002bfcf55b63827, 0x00006300726619ca }, { 0x000dc61f92142820, 0x0008f2ee0d985a17, 0x000af47364dbdea2, 0x0003af2f8192cc64, 0x00009a126d8fb59a }}, + {{ 0x000e8aa3ab0b8178, 0x000835bc2da7de94, 0x000ec16e3013acaf, 0x000091e41d0e643d, 0x0000aceee54c0f0d }, { 0x000d296a148a8106, 0x0005f329a49110f0, 0x000620ad480c6f7a, 0x00042e45ad18317f, 0x00005efb94892cc8 }}, + {{ 0x0005d5bbfc230970, 0x000353eed4c37174, 0x000ef73916a712cd, 0x0004acaa59f8240c, 0x0000b069410109c2 }, { 0x0004d5e0fcf149d6, 0x000f673137fd28e3, 0x000f92655f0108ba, 0x000ab48f2862ac6e, 0x00009efcb3eef840 }}, + {{ 0x000662dcbd70a604, 0x000642008913f4f8, 0x0001646660d881ac, 0x00000ae6cea20ed6, 0x00006d3c305ff6fc }, { 0x00037f7913789dfb, 0x0002e5c725d2aaaa, 0x000eead363c6f511, 0x00079dfccbad2752, 0x000044f72f690d92 }}, + {{ 0x000da26e1aec73e6, 0x0009be144cc3adda, 0x0005da1c14b462fc, 0x000697e72d5ffa1d, 0x00007dc85a835122 }, { 0x0001c794b1a357c1, 0x000661563483caaf, 0x00083d98e34b8417, 0x000eead3ce6924cd, 0x0000cfb7382ae666 }}, + {{ 0x000adbb3f3f634c4, 0x000728882a8042c0, 0x000fc24bcea4eeec, 0x00062e340d931cfa, 0x00006ff1f31aecff }, { 0x000577c11e875a33, 0x000621e5eaa8eb87, 0x00049c3ccd5a10e4, 0x000a9bfc5e663a8b, 0x00000028143def02 }}, + {{ 0x000a786d62a93d0d, 0x0002b5c08369a907, 0x000a9e9647958a75, 0x000519f802990c59, 0x000076b05f00e31d }, { 0x000b2ce9e430d6f9, 0x000b4be20ac3a8dd, 0x00007f8d434c0323, 0x000cdfc6531a2178, 0x00009e7647365eaa }}, + {{ 0x000c56feb051eaca, 0x000694b168984df5, 0x0006d73097883b04, 0x0004c56f0e81dce1, 0x00009cec7a5febd4 }, { 0x000233e35ef6807b, 0x000db01fde58c84a, 0x00068c98cfb5957e, 0x0001a0f7874e2411, 0x0000c7113cb30256 }}, + {{ 0x000f3a8f9d8cc3f9, 0x00072f38bd7aff18, 0x000a8e4721a92651, 0x00064b0151b5ee4c, 0x00003b792b15a449 }, { 0x000255f53b1b9da9, 0x00010c9a0748be7d, 0x0004d6d86ce4578c, 0x00072ff371d391c9, 0x0000702e661ee631 }}, + {{ 0x000f627439d898dc, 0x000c204f1788ee3f, 0x000ea906c4682ecd, 0x000a125c0c14f27a, 0x000041647e0afb10 }, { 0x000c1c33bbabfba6, 0x000dfd8ca2a59655, 0x00027acd85f7ce0d, 0x000fdcce83d9a554, 0x0000b9121e2a4fa0 }}, + {{ 0x0008c889de2e32a8, 0x000b0d474a861082, 0x000fc583d3b1de17, 0x0003aa480f8048a8, 0x00003ee931e746ba }, { 0x0001c240230a2d47, 0x0001f2eb214a040f, 0x0007461ec3244db5, 0x0007f2d8ae8c48e9, 0x0000149fdf2f3f76 }}, + {{ 0x000359ed623dd564, 0x00070c35f74040ab, 0x00014990dbf56e58, 0x000ef8823409aeb7, 0x00004dda364e2017 }, { 0x0002f1d4df601f90, 0x000b926ed6f4a607, 0x000ed89f5903b254, 0x00050ca59e21e8d7, 0x0000edcf85157a9c }}, + {{ 0x000c5fe1306bd6ce, 0x00056b9eb7926b83, 0x0003915e73d8d154, 0x00006960bbff88cb, 0x000076006a867f79 }, { 0x000daf6b21ea15f4, 0x0006ef43764fb3df, 0x000733a24b0b851f, 0x00086f1f37b5af31, 0x000030c22394a5ed }}, + {{ 0x0008cc13d69c73d0, 0x00097ec6063540cf, 0x0001ed2db7b8d6e8, 0x00025f6ab50b259c, 0x00005b5c1b0bc3a3 }, { 0x000f8efeb1d7be88, 0x0002b05c52176597, 0x0003cf322ba2a903, 0x00054bdfefe5364d, 0x00003115b68dac90 }}, + {{ 0x000e583c57029879, 0x00078e5d59233595, 0x000cf5436359e633, 0x000ed2298db0cea9, 0x00001cb09b717846 }, { 0x000b4d4d5956cfae, 0x0006c2dd48185b77, 0x000df304ca2dbb2c, 0x00040ad8bc93fbc7, 0x000021236a00feae }}, + {{ 0x0003768f8316e330, 0x00000b046eb842ab, 0x0009c86455eacc56, 0x00014cdfc8e844a9, 0x0000f8985bdb1afd }, { 0x00051b6eefc870ce, 0x00026e04e09e61c1, 0x00018bac5eec5fb1, 0x0002391cd414dc3a, 0x0000e1888cf3478c }}, + {{ 0x000e19084bc4ca0c, 0x000ca136339c0831, 0x0005c19d3eb5ee51, 0x000cbf8ab39ff815, 0x00008e22e48c871d }, { 0x000222840c0fb45d, 0x000f7c0eccd47ef9, 0x0004b83b7724081a, 0x000269cf68969338, 0x000064e761fcef38 }}, + {{ 0x000c5d75f6acf986, 0x00082f5803bac625, 0x000cdb3e5e689697, 0x0006a3f810b7dd91, 0x00004626edf8311d }, { 0x000036b9b2545d4c, 0x0007b381f2dc308f, 0x000ee94d9b0c2902, 0x000ba643120faa7b, 0x00005df95dde3c50 }}, + {{ 0x00004884ebe91706, 0x0005b0939842194e, 0x000d6987cda9dbf9, 0x0009d28e0b7e2353, 0x000008055a9e7106 }, { 0x00013520c2fe68f0, 0x0006d817419f40e7, 0x00023860a17ef38e, 0x0005c66465b0ac16, 0x0000599f3cf0f8c6 }}, + {{ 0x00024a8434f084ae, 0x000ac5ce98c51002, 0x000d0b928b25c421, 0x0001e398b6cec871, 0x000093f359354aab }, { 0x0006936830d34d21, 0x0005e6125f5c46fd, 0x000b71c0f68f5335, 0x0006e3bdc6a22bed, 0x0000bf2268186fde }}, + {{ 0x0007c2eec1224f3e, 0x00041e891e5d7d3f, 0x00067bdd87b70fb3, 0x000909f61f14b7d1, 0x0000092310e6f29e }, { 0x0000472beb05816d, 0x00006f43756e621a, 0x000f77de8d5da4d4, 0x000061872c013a23, 0x00006e813df45a5e }}, + {{ 0x000e87acdcb83dff, 0x0002f270ac69a804, 0x0005a696a81a3be4, 0x000c1a69134096d2, 0x00005b6a5c9aeed7 }, { 0x0004a56cd8946d50, 0x0009037e285ccb47, 0x0003dc3862797108, 0x000a2a719188089e, 0x000041ca5eaccff2 }}, + {{ 0x000421cc52cbf744, 0x0000ae21726931ae, 0x0007523f20aa75d9, 0x0008a4a180bbbb2c, 0x0000d98ec4e59258 }, { 0x000359b3dc444ac7, 0x0000b08bea87be65, 0x00035e7422a6acdd, 0x000509cf716a475d, 0x0000928ea2c9a7ff }}, + {{ 0x000fc3a5983894c4, 0x0002dde8bf06e31d, 0x0003fd749369a738, 0x0008323d21fdf8f4, 0x00008669487b1517 }, { 0x0001b4f11e41bf9b, 0x0002da0b1ecff106, 0x000bc68b142b2d0a, 0x0008ff273ee6ed6c, 0x0000d795b8258fd9 }}, + {{ 0x000f3d47897647f6, 0x000e09bd4eff2d71, 0x0001234b6e8604ea, 0x00059f12388ae677, 0x0000156c9f102ac1 }, { 0x000794b13b286f95, 0x00083c07f9871da6, 0x00088ea1313d5e21, 0x000e488ac35fbda7, 0x00009a346a1a06b8 }}, + {{ 0x00023a9f8173b2f9, 0x0006bac09cc0917d, 0x0008f44aa17b929d, 0x0006f7d7127c5372, 0x000084aea5a60d7b }, { 0x00040f1c67e5aa9c, 0x000f92852650359a, 0x0000d0929045b31a, 0x000ab235cfa09ad0, 0x0000eb49c8d9c125 }}, + {{ 0x0002383e48552e85, 0x000f02edcc18770a, 0x000ee5ed855ce52d, 0x00082ceed4baf1f2, 0x00006b9ecc862bcc }, { 0x000b2dfd7c4b99b3, 0x000af0a2646d19a8, 0x000b789a42819c25, 0x000d236f156cb44e, 0x0000656fde1d4d03 }}, + {{ 0x0007bdb99d7bc622, 0x00050f2a672c5c73, 0x0007d5960c453caa, 0x0007d3732eacb11c, 0x0000d966e32b0083 }, { 0x00026693bd8b6c09, 0x00026b62899c20f7, 0x0001d2183a676b7a, 0x000eb73e33535d51, 0x0000b014e1e0cb73 }}, + {{ 0x000a20ed2a8c1c6f, 0x00041e5ab4b35a24, 0x0001abcdbda4db56, 0x0003f26801d8b600, 0x000004b48afeb9ed }, { 0x000a24142725a709, 0x00053d44fec01e63, 0x000c6ddb3c8679c1, 0x000868efd3ed7ddc, 0x0000203192c73ce6 }}, + {{ 0x0007fd26ae8cf314, 0x000cb2c6a8ea8208, 0x000b6a2f9ba58ade, 0x000360026b2b50d3, 0x0000e7087b5a0c2a }, { 0x00066f7ffa7c7f1a, 0x0007ccc02bee4ba6, 0x0003a0128fdc2037, 0x00093cf855312180, 0x000003bbef5c6e85 }}, + {{ 0x0006c3200fa35ff3, 0x000da33bccd9617a, 0x000413b4109421a3, 0x0007b1486365dede, 0x0000cb76293645ba }, { 0x000a57354a206551, 0x000aa2ce5e35efd0, 0x000b4afe6c76e5aa, 0x000a5e65199a4e25, 0x00004027e76f2bd6 }}, + {{ 0x0006a6f79278950c, 0x000a24bf8e4b0651, 0x000417ac0a99741e, 0x000fde19607a834c, 0x00007e7ec115210d }, { 0x000ea49b3d3fd55e, 0x00005c66a8649edc, 0x000db7faa2d74f3b, 0x000d255ecfbcdfcf, 0x00002d9e2c5a83c0 }}, + {{ 0x0000ab7b49321f95, 0x0006f4fe9d964888, 0x000e00d72578140d, 0x0007254e94b53d52, 0x000052362af789d1 }, { 0x000995d53e000a78, 0x0009dd427d3317fc, 0x000a50ce91e0ec03, 0x00091e3928dfc1dd, 0x00006eca410021d4 }}, + {{ 0x000c32fbd2c44064, 0x000ddec090090ae0, 0x000ce17d6c5d6bed, 0x0008932902278a0c, 0x0000acd50fc16037 }, { 0x000cf307bda4feac, 0x0009adaf341a6c11, 0x0008fefc60e33292, 0x000789f3aae0ba13, 0x0000954ec449a5e8 }}, + {{ 0x0003b88a9ddcf1a4, 0x00019c719803511c, 0x000ac4d03920e342, 0x000d52c5ea9f97b3, 0x00008856a1c81ea8 }, { 0x000bd715882bc1be, 0x0004948c48b0ee35, 0x000babd03139016e, 0x000ab99f37506bc7, 0x0000a2cc32ebf442 }}, + {{ 0x0005c8485a9d19bd, 0x0004708c1bc52257, 0x000ab150da1307d3, 0x000737043d0b946d, 0x0000f537f04207d8 }, { 0x000df3874cc30296, 0x000c9499ce4177a4, 0x000e2bcd84f0edb4, 0x0006566d74fa61a1, 0x0000f53571e1b3de }}, + {{ 0x0008d158672d3eea, 0x00042f983dfedc91, 0x000474b60ab2fc6f, 0x000eaded2f843713, 0x0000d9c3c4a8aa46 }, { 0x000a5956bae7a26b, 0x000d3cf69769bb7a, 0x00061a2054c79b62, 0x0000e89b2c55eb85, 0x0000df9c408d613a }}, + {{ 0x00045c51f0556ac4, 0x000d2c29fe20ebae, 0x000b00ce08e2eb0f, 0x000b2d989f27168b, 0x000069b4c47cc369 }, { 0x000a9db957a6f395, 0x000bd76099b42210, 0x000c1ab9e80238d6, 0x0000fc3958dc6407, 0x000089ab035eddea }}, + {{ 0x0006a52433a64b6b, 0x000cbd3df61bc767, 0x000fc8d268a83dbb, 0x00023378f13a619b, 0x0000e79d7f68711f }, { 0x000ac045e020fb6c, 0x000ac91ac5bdc5d3, 0x000280079621025d, 0x000e41853ff4a1a7, 0x00003f7f41088141 }}, + {{ 0x000fb4f9a3e59e5a, 0x000da1bad53dba78, 0x000bf3c668b6a4a0, 0x00044b7898ec269c, 0x0000c42d0387de55 }, { 0x0001da22fc32c200, 0x000f823ff9875cf9, 0x0006cb6a079f4d88, 0x00004267923224f5, 0x0000acbd3518a2ba }}, + {{ 0x000364ebbf27875b, 0x000d74d6c04a661c, 0x00059ad281587be8, 0x000bd88579710d3b, 0x0000db51af083108 }, { 0x000aa7641d8ee984, 0x000d9517733d61c0, 0x0009c400c2a5a595, 0x000b8d148d55644b, 0x0000c50509008fc6 }}, + {{ 0x000b6066dd443936, 0x000b681318106601, 0x0007c2ead965cfff, 0x00004cc739dfa650, 0x00008f398de4fb88 }, { 0x000bf773f8103f84, 0x000c1919c9efebdb, 0x000ee624abfcc370, 0x0003683d3425d4bd, 0x00007bb42dbf21c5 }}, + {{ 0x000ed26d818f0304, 0x000b96994b03ed1e, 0x0002747a57c4f532, 0x000e11ccad9ad1de, 0x00005dcd24ba7a71 }, { 0x000ff2db2716bd02, 0x00009e7a4db09d2e, 0x000ca61347d0c139, 0x000d9c529f47f569, 0x00003c0c5c86b05b }}, + {{ 0x000187d6657e670b, 0x000c514b913e759b, 0x0005de6d082e2895, 0x000b95de153da478, 0x000092e140a21444 }, { 0x000e8d345fd30ff2, 0x000c4501dc90b596, 0x00030f1ce6c7249f, 0x000fc18848170e94, 0x00005a2d667b789c }}, + {{ 0x0004b7839d2bce29, 0x00096a1c5c5ea50e, 0x000b82de82e45901, 0x00006e042d3baf14, 0x0000ff6d502965e2 }, { 0x000d93355322d13b, 0x000bc94e7de6dc08, 0x0003d497727bb320, 0x000dbc3ec5cd0f4d, 0x000059e7d3f99e36 }}, + {{ 0x000be924c158e0e0, 0x000aec5776c57224, 0x0001df76044b6efa, 0x0001345a9dc467e6, 0x000033e12139aef7 }, { 0x0000c9ca4e9b5641, 0x00084aea18921310, 0x0003a63bff437207, 0x000da19c58bf8a8e, 0x0000f70e2d2715de }}, + {{ 0x000bc2ffb0c0a600, 0x000e80f83774bdde, 0x000199b044d49d58, 0x000a469a96313160, 0x000087769824fdd0 }, { 0x00009015dea24c28, 0x00015b77cfa46a5c, 0x000e60d52571b07d, 0x00070300a71e6161, 0x00003117310e7902 }}, + {{ 0x000f2b9ffa5ce94f, 0x000f8b2d174d7aa6, 0x00088475bd09ee91, 0x00010b8ed4072d8e, 0x0000abdc7583857a }, { 0x0001914e9bad73f4, 0x00022153760fe8aa, 0x000f4edb1d776b51, 0x0002edc717a96e6b, 0x000069595ca97461 }}, + {{ 0x0005b20d1b136d4c, 0x0009757d8a3c5cf1, 0x000315091df843b9, 0x000413cdd1197ecc, 0x00001969c8be641e }, { 0x00099ea238db2755, 0x0005526705840398, 0x00091e3df50ef1df, 0x0002e83152277c96, 0x0000630e2f484e9d }}, + {{ 0x000875384b505bc8, 0x0005a8b760c1c9d4, 0x000152e6e08ee77d, 0x0009b13c5c7efbb1, 0x0000a686f3ab7cc9 }, { 0x000912a263785680, 0x0008baa3ba8cd86a, 0x000b197dbb7d4f6e, 0x000cc9952b0ce40f, 0x0000893fc7e41001 }}, + {{ 0x0000049c55f94a22, 0x000bbcceb5b76c16, 0x0004c6f76e5ea2ee, 0x000e2049f746f528, 0x000076ebebbffadc }, { 0x000d3fe2ee578482, 0x0000d823c2f7c9eb, 0x0002d43d7a059070, 0x0007d72696bd9904, 0x0000ce8557b6ba2f }}, + {{ 0x000ff4f96876ad1b, 0x0005eca12ef3d36a, 0x000b8e89ac75cb71, 0x000521bf79ee3c36, 0x000090ab39ba4d57 }, { 0x000a347a48af4239, 0x00064d0c47adb7e5, 0x0002a8fa03b1669f, 0x0006bc96a45699b2, 0x0000959575cb4270 }}, + {{ 0x000e6e24892d0ac0, 0x000933b3cef0d7d8, 0x00079d4d1ef8b947, 0x000763500d94f7b1, 0x0000842223ee34e1 }, { 0x000d016ec18d3932, 0x000e49716caca6a2, 0x00080b51ac183c3b, 0x000187be1866c49d, 0x0000c9294fa577e4 }}, + {{ 0x000e969937eb4c0c, 0x000c0c900eac3f90, 0x0000d456ae10b8bb, 0x0007d820d835f628, 0x000023a90a360f31 }, { 0x00089014af462bca, 0x000fee88f30c9cbf, 0x00043eac31b1d4ea, 0x0002462aba606ae7, 0x0000fd95461debef }}, + {{ 0x000b9912384ae29c, 0x000ce9c31c71f7be, 0x00023dd6ec23d54c, 0x000ec27a301f95aa, 0x00007cbf09e9dc08 }, { 0x000c8b46801a5c63, 0x000305e5d91ba877, 0x00008ecb0399943e, 0x00077eb69c6ac628, 0x0000c52861049c21 }}, + {{ 0x000e5d49ffd84deb, 0x0003a1fea5b8e69b, 0x000d075a15bd1f24, 0x0003205e5aeecb3b, 0x00003034bad431b0 }, { 0x000f13f5a83d106c, 0x000a7aae5c8646f1, 0x000801cda3d81b1b, 0x0009b5194dbaa12e, 0x00000bad3d6972ba }}, + {{ 0x000608b69829f0bc, 0x00035eeaceb4fc02, 0x0002386658aa3391, 0x00022b4c83625600, 0x0000214f4fdc3f4b }, { 0x000209f113e9fbcd, 0x00005fc05268301b, 0x00088e26a0fee5c5, 0x000d514c2a6943d3, 0x000089511c6cae35 }}, + {{ 0x0003ffdde3be7cc0, 0x0009e0697bf1298a, 0x000f7d9d85bb314f, 0x000a1cc6fcbdb672, 0x0000c8ea86a332f9 }, { 0x000a3d7005cb8287, 0x00033aaa914d3ba5, 0x000f2ea215e8f9bd, 0x0006b8051836d500, 0x0000034ad2de159b }}, + {{ 0x000da3ce9ebe72c9, 0x0009a4d39eedbbab, 0x000746b1d5ce6360, 0x00065220c35cd2e6, 0x0000dcc999f8c73b }, { 0x00032fd30a1b75d3, 0x000a7602c9cf7a47, 0x0004089bb469c24b, 0x000ff75df01572ee, 0x0000ed51712af770 }}, + {{ 0x000a4544aa046b21, 0x000e8c716543a402, 0x0006d498f7c2ddb5, 0x000b9c5149acafd3, 0x00004ad54644a863 }, { 0x0003a83affc7b669, 0x000e96291d60b063, 0x0008f4ccfa455de1, 0x0003ad227281b7be, 0x0000b48074a91ac9 }}, + {{ 0x00058ae74fac8317, 0x0008162d5d721d5b, 0x0002a648ba32d37d, 0x0001fb6a40c3357a, 0x0000d41eebd7ebce }, { 0x0007312ec0f21311, 0x00009812e6dad6be, 0x00010d0c3d9a387e, 0x00083b3d8de28605, 0x00000e56beae8107 }}, + }, + { + /* digit=2 (1,2,..,64)*(2^{14})*G */ + {{ 0x000161957d933aea, 0x00028e76e6485b37, 0x000b05babd7b6596, 0x0000f456d4d3e24d, 0x0000ce6abd42c3d5 }, { 0x000aefa7aee9d873, 0x0009f83cdb12a6c3, 0x0001f36139b52f8a, 0x0008e5973bf3fa88, 0x0000e7a50e16b447 }}, + {{ 0x0003f63771beff8f, 0x0006f623b36f9fd2, 0x0003d9eb684ebab1, 0x00084f3ef26543b2, 0x000096ae77bc2ed6 }, { 0x000fb89d795ff65e, 0x0007796b6a1142e5, 0x0005d3a20e91b4d9, 0x00011bfd55ea6aac, 0x0000a386dea94493 }}, + {{ 0x00086584c9b213cb, 0x000f83af72e34d46, 0x000fa8a9676c4401, 0x000cad84e0bd9ea3, 0x00006265c102920f }, { 0x00057594c72c920d, 0x00088d38c97b942f, 0x000958e6e09ae71b, 0x000e172894609561, 0x00007cd5b11ce7c5 }}, + {{ 0x000564bade8d111c, 0x00049fce3779ee34, 0x000d77e4a2bf81f0, 0x00017327ae00e7f9, 0x0000f7675150c393 }, { 0x0007f7ccb1668c80, 0x000b12ac4708e8e4, 0x000b5f26ce2e2a59, 0x000dbcb9f4f5725f, 0x0000c707f408fcd6 }}, + {{ 0x0003a7c17140472d, 0x000581b8d9465810, 0x0003ba4cdea132db, 0x0003dade0f0d1313, 0x0000dedff07af825 }, { 0x00058b1494211ca0, 0x000df38e475381a1, 0x00026ff71ea03acc, 0x000c4dddaca1ba42, 0x0000a403ce5132ea }}, + {{ 0x0007c64e36e21202, 0x00075e5522e6b693, 0x000384071de1f828, 0x00012fb5669c908c, 0x00002dc9675211f7 }, { 0x000fb08f0866ab60, 0x0007bdfee478d7d1, 0x000be1e36897f3a4, 0x00021b6dfba75c8b, 0x000059cbb6ef7bf0 }}, + {{ 0x0007560abcab01ab, 0x00000f9842e9f302, 0x000f375578f2df7b, 0x0007d7f6aa331d4b, 0x0000a0561e6ffd56 }, { 0x0003b77509e9eaaa, 0x00089870e3a6b750, 0x000267e7dbebaacd, 0x000134cd025c6aea, 0x00001e66709e13cc }}, + {{ 0x000dbde4ac826d0d, 0x0007c0361a341c13, 0x000fcf3f7c95b3aa, 0x000c6246c3604dc5, 0x0000ce46d1be3f02 }, { 0x000f0d1f30affc92, 0x0006a5d1c7eef696, 0x000562423070d98e, 0x0003f8fca5889680, 0x00003b45ef942f70 }}, + {{ 0x0006f04d3172e6a8, 0x0009cd4196224f85, 0x0004e05cba6c1ba5, 0x00062492375a4ab9, 0x00000994feb99b0a }, { 0x000000efa0311470, 0x0004f1cbda602b24, 0x0002b9603cc9b10c, 0x0003b606b73ee174, 0x00007167141027d7 }}, + {{ 0x00099b055d1c6101, 0x0001a4b12cfe2978, 0x000b9f946f882bbf, 0x0008773e83eee129, 0x00009e74c40cf1ad }, { 0x0005132b2b61a7c7, 0x000b3643a39c8cf8, 0x000df383119dec9f, 0x0008c316382f09ef, 0x00008e4c68996471 }}, + {{ 0x000b4afef949e9b3, 0x000f0ab516eb17bb, 0x000f177c3832341c, 0x0009fbaa673bec06, 0x0000087396035c70 }, { 0x000f54befe9fd403, 0x000bbc8e61d66b0f, 0x0003efd4710c8da1, 0x00097a75a02bda4b, 0x00003e2c39ecdf62 }}, + {{ 0x000c9a9d2168c124, 0x000671dbe7a2af37, 0x0008d31cad5154da, 0x000c67297ec51caa, 0x0000f6a5ba0129fa }, { 0x00003937564cf036, 0x000ae4f6ef5f0145, 0x000298d331a9ca83, 0x00051e01e8abcf41, 0x0000803532cecc34 }}, + {{ 0x0001e274de55c8d4, 0x0002956ed6def63f, 0x00018b709b8e424b, 0x00056a6c153cf648, 0x00008cfd0094deed }, { 0x0009fdddc13851d6, 0x000243c44cc78808, 0x000af2278bc0bdee, 0x000524ae0411a426, 0x0000bf64ecb23c5b }}, + {{ 0x00088c68b8e8aff1, 0x000162232398baaa, 0x0001b47037788dbe, 0x00052abf2205fee4, 0x0000c5e4c2db8891 }, { 0x000e9e19413381a3, 0x000ad417337aaa82, 0x000ce0156bb388a5, 0x0002a062f9ed364a, 0x0000c9e564f943f6 }}, + {{ 0x0009d84aaee0a3f0, 0x000044a3dafd2d50, 0x00058e154e29774b, 0x0006409b135d1fae, 0x000075272e315a42 }, { 0x000286418cf25deb, 0x00030c788aaa6917, 0x00093d60897674a9, 0x0001ac1012dea57a, 0x0000de5fb8c5274b }}, + {{ 0x000688350470c6e0, 0x000970977f7195ad, 0x0006d7634e8b89f8, 0x00088212d8ec8616, 0x000043845130a44a }, { 0x000f5c2ee645cfc9, 0x0005bf8309004c28, 0x000584a3ba78a88d, 0x0005275a4a96e4b7, 0x000038c353982c77 }}, + {{ 0x000270f0d3fad15e, 0x0004ca5dd968198b, 0x0001bfd0499861e3, 0x000657b03fa131c5, 0x0000619c34d7bc9a }, { 0x000c59900ea25f24, 0x0005164a053a3bc5, 0x0006be2d3ae0af2d, 0x00056cbd0088254b, 0x00006bf9c020a568 }}, + {{ 0x00058215b141a06f, 0x00072dda1887395c, 0x0005cda8502dab0c, 0x00008e62940960f3, 0x0000e7123058aaf9 }, { 0x000dd036eeb10f06, 0x0009558f0e1a5240, 0x000be7c0a8ca237d, 0x000ca5062b406856, 0x0000716a9f2d15eb }}, + {{ 0x00060caadd4449ef, 0x00082acc33156b3e, 0x000964736c5160b7, 0x000929feff09d24e, 0x0000caac5cb6e932 }, { 0x00018427c2846cf1, 0x00007a6979b1d8f6, 0x000c35f465637c02, 0x00002fdc060908a1, 0x00001090f1c298a2 }}, + {{ 0x0006877b288e2940, 0x000f33959145a65f, 0x000803b8e80eaf37, 0x0006112a0cd9bc36, 0x0000b81a72e0ba9f }, { 0x00040fa5a0fd54f6, 0x0008b9bf5cbdb258, 0x000859f95ca88625, 0x00044185f2a4fd1d, 0x0000bbdb6fce4677 }}, + {{ 0x000251fe80573ec6, 0x000aebe6558842d7, 0x0003a5a1d225d97a, 0x000041e6d70df8c6, 0x0000a3616cfca01e }, { 0x000d97a9062076a5, 0x0008fcb5e9aac9a8, 0x0004626cb46bc51a, 0x00054784e11bb093, 0x000096e6c4d5afbb }}, + {{ 0x00077708c667d059, 0x0003a277946d3f97, 0x00038fd019ad9658, 0x000e65586e132bd2, 0x00002ddf44a145b6 }, { 0x000ac06c3764df11, 0x000b7cd1297029e8, 0x000c3138c426b403, 0x000a6c5d92c61095, 0x000067be85672106 }}, + {{ 0x000e2347ffc2edcb, 0x0001139a13046683, 0x0004c9d54e06e078, 0x000d9578de4a9c88, 0x000002ea6f0c8fe9 }, { 0x000ff875671915b2, 0x0004977ea7d23a38, 0x000cd731fc668ce1, 0x0005492d32fac257, 0x0000e0f225fb9165 }}, + {{ 0x0000d8b2dae7ca94, 0x000fcb36b0cf82eb, 0x0005753d030cfdc3, 0x000d1805a6438d24, 0x0000a53ea3da217e }, { 0x000afa8c8dcb9f12, 0x000c8dbd9184c4d2, 0x0005ce430ce88936, 0x000756b473dbb18d, 0x000096aae294c905 }}, + {{ 0x000ad8848abb5fc6, 0x0008de8bc36742c8, 0x000e63c869056bf3, 0x000e7b500898f427, 0x00006d577bd5f382 }, { 0x00074a2d9a7f8c04, 0x0003838d74a2a464, 0x000954574e07ece4, 0x0002da5e45e3e05a, 0x000086fa3c45cbcb }}, + {{ 0x000df865364f0739, 0x00000ad653ae3263, 0x000a6954560d380b, 0x000cd878314bcf72, 0x0000e5984638d7d1 }, { 0x000ccfe3cd6ce14b, 0x00024e7b53ca5557, 0x000d2f05f16b65f7, 0x000f23968688cb00, 0x000083bf9324d5ad }}, + {{ 0x00074a1d2d22f9b4, 0x00093e06c4c1fe61, 0x000a12419e4cac55, 0x000daa614557e1f1, 0x0000f204eb2e220d }, { 0x00075797ca396209, 0x0003230c51fbf539, 0x000157111a5b409e, 0x000ad3873d894099, 0x00001cfd1518ece2 }}, + {{ 0x000dfe0d903a457a, 0x0001207e3533d77b, 0x000970360f08ec9b, 0x000e84aedd7222e6, 0x00008819a858ad52 }, { 0x000eb874ec49378f, 0x0000bb61d056605f, 0x000587efe032d6ae, 0x000c3fe8ca9ea9df, 0x0000fbcf08ae31fe }}, + {{ 0x0005f49faf4f53a7, 0x00043867ee0d98fd, 0x000b060178b5af0c, 0x0006c8ad4201ad34, 0x0000d0bf380597e8 }, { 0x000a24911e00876c, 0x0004691c7f4a40c8, 0x0000e8e94c74da88, 0x0004ffd872e69d9c, 0x0000a4ebb0eb3649 }}, + {{ 0x000d6de7bd491998, 0x000801060a716760, 0x0008e8fca31418e2, 0x000fd8b1ee852d1a, 0x0000dc52d76da81a }, { 0x0007f0f335115f76, 0x000f988c8e347f89, 0x000703e8a98e5e7e, 0x000a7f3501c58754, 0x0000eef253d19b22 }}, + {{ 0x000b3fe0f9da5724, 0x000bdce4924867ad, 0x000f8578681e0a84, 0x000597679bd5f51a, 0x0000d1366a434662 }, { 0x000ca8b8c079486a, 0x0002ad8556fa9db6, 0x0008b69876ff7430, 0x000332891fa26216, 0x0000f462db921b41 }}, + {{ 0x00022a049341c337, 0x00033a9bd7638026, 0x00000306390eab7d, 0x00022f7c4c21486a, 0x0000d141475b8b86 }, { 0x000540cb1b1be032, 0x000da9dbb7a8ee0d, 0x000b96c0e3c063f7, 0x000de42ecdc2ed3b, 0x0000397c62c0f6ec }}, + {{ 0x000736f9291ef406, 0x000e74ecf84f6c7a, 0x000b85d2161ab278, 0x0002f97b48561fb9, 0x0000bcf6264a5050 }, { 0x000c70cb296a5d7b, 0x0005e292c1950ee1, 0x00033a6b46899d57, 0x0000d0c8eda17754, 0x0000b3f3cc6b8110 }}, + {{ 0x00070c1c8efd9357, 0x000358b6f0f7c500, 0x00087e41afdefe0f, 0x000242f42a8eba2b, 0x00006e5680230b39 }, { 0x00079d74c341868c, 0x00037716484d2e49, 0x000e75a3cc667a4d, 0x00052af0bfb78318, 0x0000b9d00abad929 }}, + {{ 0x000667ac0d2f8437, 0x0001f4761fb04286, 0x000a587e78c7cc95, 0x0002a4ffd4d78954, 0x000020826521d389 }, { 0x000ac4bacee14c89, 0x0007c1243bcf832f, 0x0006a53c4668bdcb, 0x0006e01a7fbadf8b, 0x000053ad230ebad2 }}, + {{ 0x00058566c410a298, 0x000708d977eab403, 0x0005f1db4ccc80c1, 0x0006b3d5351f4c5b, 0x00008dad5554cb23 }, { 0x0009e5f0b7cfc603, 0x000d39e5068f5ff5, 0x000c81197a4e6a94, 0x00018bcf0f31435f, 0x0000e92434aca407 }}, + {{ 0x000943b3af624454, 0x000dbb898559acf1, 0x000e3473e7a0fb04, 0x0005b12aa96018c2, 0x0000fe9ef544dffe }, { 0x000ec75726be2472, 0x00024df15a40d39b, 0x00038d59a09f0400, 0x00073ce4cb2a4468, 0x00009d760075d37d }}, + {{ 0x000213754eefea23, 0x0007e63bea064441, 0x0002e51141a3c690, 0x0004ea5ab27e95bb, 0x00006619f7919ae2 }, { 0x000cc970d32e84eb, 0x0002520716e0f742, 0x000ed2f724ef5dbe, 0x000273e326679c82, 0x000049306f2821a2 }}, + {{ 0x000f8396342a360d, 0x000122f740f0e66f, 0x000a2be78d984691, 0x000d47ed7545c616, 0x0000556fdb469789 }, { 0x000fcb8983c3880e, 0x00019d9025ac99a0, 0x00001b9859b74109, 0x000503719d314d4a, 0x00003e7c7f956365 }}, + {{ 0x000fd58ff1b72f01, 0x000216138634818c, 0x0003e4769e197596, 0x000b0daa3501814f, 0x00007ddf3af2b89d }, { 0x000db6eef931c2b7, 0x0002414d3b507a84, 0x000e7754ebdfc74a, 0x0002342aa57bd478, 0x00004e0dcd5fe91f }}, + {{ 0x000de92572a83fdd, 0x000151ad1fc0654d, 0x0002a33d78008489, 0x0003f6cf9c71cc96, 0x000005b75269915f }, { 0x0005f44d1e992bfc, 0x000daa3f97ad8f78, 0x000f0c59969f0192, 0x00014ac00e6c19d3, 0x000025a10641d608 }}, + {{ 0x000332b6dc4e101e, 0x000a28b09d64c52b, 0x000f42ace06d0b43, 0x00063417da1b5e49, 0x000046ce47f6dc04 }, { 0x000f0bc5a141ae2f, 0x00059a0b4511aa47, 0x00025cca06b413dd, 0x0001f23bd0978875, 0x00000bd956a4fb97 }}, + {{ 0x0009ff29ad65488e, 0x00084d4c11f61ef0, 0x00043084bdf0af25, 0x000773c159ce1075, 0x0000ec6435d82c07 }, { 0x0008b5fef46ca97d, 0x000f5bde7da946e1, 0x000bd5caa5fe7671, 0x000b00fe4ebf9eb3, 0x000075259e3b13aa }}, + {{ 0x00038c534e92ad2a, 0x000a3d35958cd79a, 0x000479aaddc64328, 0x000e62d263580a1b, 0x0000ae928f234367 }, { 0x0003cc3e51cd352e, 0x000b60a2ff5845be, 0x000cd9abd92da353, 0x0004a6b20e5bb40b, 0x0000ae386f9756b8 }}, + {{ 0x0008ec52c22da1f9, 0x000befee5682d265, 0x000f55dba5b9efc6, 0x0007758ab5206f76, 0x0000dc15c6ab6dd6 }, { 0x000ae060d5586b0b, 0x00084990f75cf9ae, 0x000ecfa270009add, 0x0001d8aabf411951, 0x0000e895791fe909 }}, + {{ 0x000bd450efe0661a, 0x000c5737fb54dc25, 0x000ac9c1e559ba7b, 0x0009f36811f3e391, 0x0000aca8c08c3b1d }, { 0x000890e30907b5a8, 0x0005467981eb389e, 0x000751746dcc2ec9, 0x000873f5ced7e192, 0x00002b55533c62a3 }}, + {{ 0x000d7f7a52f077e7, 0x000cf82feccef760, 0x00057e3af33fc8e9, 0x000ab8ec48b8e11f, 0x00004d722538f93a }, { 0x000e2b5b0ebc17e7, 0x000ec0a9efe5633d, 0x000a76269e26417c, 0x0003683002d8d246, 0x00005e002152d468 }}, + {{ 0x0007d2501ccbde9a, 0x000a25abf776f5b8, 0x0000d8632ae86106, 0x000e0a57a6f72407, 0x0000bbf66580faa5 }, { 0x000a4228cbe2da60, 0x000f2e187b053405, 0x0007c7202366612a, 0x000206ddbb7bf7cc, 0x0000bac9edde0240 }}, + {{ 0x000632d18502246d, 0x00069d6fd4fd6717, 0x000e4ca7cf89ed63, 0x0009c7e96a233687, 0x00006b0153bdbf01 }, { 0x000381366bc36648, 0x000b0e940735ff4f, 0x0002cc87fb85426a, 0x00097f7183a3e6e1, 0x00007201b1efff17 }}, + {{ 0x00034d89dd22ce0a, 0x00018a960167d923, 0x000841dc67af01f4, 0x00085a71462f9d6f, 0x00002030a417e2e0 }, { 0x0007d6912f7833d4, 0x0008fba2a4383ce6, 0x0006f5229a7161b8, 0x0008aa0c0cec8e93, 0x0000c10be8855950 }}, + {{ 0x00084ec70d77159c, 0x000d0461b9131492, 0x000f11ff3bcd0483, 0x00018499436aae2e, 0x000053055174f620 }, { 0x0001596fa6d718a3, 0x0000fd061391d54e, 0x00061cb7deec1cba, 0x0002f8aff69c5d5d, 0x00006bb3ce340edd }}, + {{ 0x0008ec05f3a623c5, 0x000965696a71cba9, 0x000264ded8c36a35, 0x000a9bf71944938b, 0x0000d4afba995956 }, { 0x00012ecd8b87d02c, 0x000a21b0698304a7, 0x0003b08f2a1059a1, 0x00051b2c12d69144, 0x0000fce76e99e893 }}, + {{ 0x000176fbaba509a9, 0x00057a700fd56e19, 0x000267db806ce842, 0x0003499e102ec969, 0x0000b53556f68ba5 }, { 0x000e116c316d7216, 0x000dbba44b7b6187, 0x000973ccdceb19b4, 0x000436cc0b4d7aae, 0x00006be87ea1879a }}, + {{ 0x0008553aba1f2c4a, 0x000c581466265d74, 0x000205a2733679fa, 0x00047457fb15b6fe, 0x00004fd50ac219da }, { 0x0001bd339486e911, 0x00019c8c21e94db8, 0x000d3c038859a5de, 0x00057368f795ac38, 0x0000e90f47c1b4a7 }}, + {{ 0x000cf9b00c68264b, 0x00090e956b3c4f2a, 0x000ee347c7bffba2, 0x000dac4c1017a99c, 0x00002d4fa858cee4 }, { 0x0003e8b689fdab85, 0x000378d47173611a, 0x00080894bee52bc5, 0x00029bca21a4cbe0, 0x0000cd609cc3206a }}, + {{ 0x00010dfae15b10c8, 0x0009eda4d88f1dda, 0x0004d9c563094fbf, 0x000680267c604f16, 0x0000e738ded43ee9 }, { 0x0004f544eac66991, 0x0008b76af36f4e47, 0x00023a4a4a5e2681, 0x0005f158c994b229, 0x00008811132390b1 }}, + {{ 0x0002b76b184bc485, 0x000d6b98113a7572, 0x000467e11acc370b, 0x000e3015f4ef2d2b, 0x00006a474045ccce }, { 0x000def0c94233a80, 0x000a310443516830, 0x000696031bac0d4f, 0x000e459479d86189, 0x0000536ea5513960 }}, + {{ 0x000f5796a5a19f3a, 0x0001a174c7c15e09, 0x00019acfe58d4e71, 0x0003d955f539bfb0, 0x000062e8e0c9e958 }, { 0x0004a8ce6d3e344e, 0x0004fe84f157269f, 0x000485874afe4d4f, 0x000560c91c825c1e, 0x000052b87c2a4f9e }}, + {{ 0x000a6c9d078887c3, 0x00072a770cbac78d, 0x000b1c459c509446, 0x000e4ed1183488e9, 0x00005eb15cc6694e }, { 0x000d2b810054b0ec, 0x0000f30dd648c523, 0x000c15d01ed16251, 0x000a54486994ca02, 0x0000ab1e189c171a }}, + {{ 0x0006c4fcdd32e0c3, 0x00085c85963a46e3, 0x000572d964eab5be, 0x000a454b4658ab87, 0x0000336a4535aa18 }, { 0x000adbe47f352065, 0x000c2a1a3906def5, 0x00070765a4c6a931, 0x000b907de9fe95f5, 0x00005cfb0d9f02c8 }}, + {{ 0x0003d6b0785941c9, 0x0006e0f396de2b60, 0x000bed16174546c7, 0x000db3d7298583d4, 0x0000fd49ba83df8b }, { 0x000c93db80ebbbf3, 0x000ea93b0b64912e, 0x000328d49f883c1f, 0x000302da99b3a343, 0x0000fc0ce27be066 }}, + {{ 0x000a22cd41ae626e, 0x0001e2466e48bdd8, 0x000887aa0a4c529d, 0x000c298032514f2f, 0x0000980fe0c00e32 }, { 0x00016284b5cf64ac, 0x000e1f4b283e4c6c, 0x0008c9d6f389d580, 0x000c0ec0978c2985, 0x0000926037d62cd5 }}, + {{ 0x0001d36874bc33f5, 0x00084161338830dc, 0x000b103ed755e9c1, 0x0009151f86d4b8a6, 0x0000aa1dcf33163d }, { 0x000f1ebe4e82f431, 0x000b3958d436d13c, 0x0000c091c2614a82, 0x000b90eb5d4d9c22, 0x00002e1a3189da65 }}, + {{ 0x000e6e2135aa3e34, 0x000406a584c5e205, 0x00015b6e4efffd6a, 0x00016589b267e4ea, 0x000028a50a289a0f }, { 0x000fef4e6229a8b1, 0x0001b6a8c8f936e6, 0x000445d2f37ab589, 0x000b64a868af84d0, 0x00002d1d526d0d0a }}, + }, + { + /* digit=3 (1,2,..,64)*(2^{21})*G */ + {{ 0x0005c78880bdb670, 0x0001f09257226088, 0x000a8d556d463880, 0x000fe87ac131260d, 0x0000a978da1cbb3c }, { 0x00081d1c0fbab2fc, 0x00085ffc3bdf24eb, 0x000b3da6a9989162, 0x0009b73405bff75c, 0x0000d9387ea82fe6 }}, + {{ 0x0002b79aff1b7640, 0x0003cc8f43a730f8, 0x000bb0f3be0fe168, 0x0008ce40be89b6f3, 0x0000da60e29a2a91 }, { 0x000b662498ee8663, 0x000c556962fe5de3, 0x000b255f6d5a6808, 0x000a245580c590ad, 0x0000e4d2e7887b5b }}, + {{ 0x000414582cf35827, 0x000a80d6c39765a2, 0x000d882871de9e8b, 0x0003d1450a2db3ac, 0x000046f7d2fddbd8 }, { 0x000672303b75f511, 0x0007abc60eee9a8a, 0x0001cc28700717f7, 0x0003de4d554c79f0, 0x00009aa79fb6b0ef }}, + {{ 0x0004ff052f5917b5, 0x000faa8719c05631, 0x000ca760c26fde52, 0x0005d57d3cd8ad2d, 0x0000bd74588b7072 }, { 0x0006ad71d02a024a, 0x0003acf3e2d65208, 0x00078da16339cd4e, 0x000f011987ebdf0f, 0x0000079790c922a9 }}, + {{ 0x0005c0e37aec2320, 0x000de9226f435728, 0x000ad7c163ddcc04, 0x0006d74c1dfd3ab5, 0x0000e297c00ebf67 }, { 0x00035b9485776668, 0x00011727be19ae06, 0x00067bb1a22fdff6, 0x0004b977e616a339, 0x0000d8b5f68742af }}, + {{ 0x0005de4f4a416a75, 0x000c7dada430c0b9, 0x000da4b88b7c5154, 0x00041bd6a4702850, 0x00002e74a9165ad4 }, { 0x00017cb319502a4a, 0x0006309e265c17ce, 0x000326fe85685ff1, 0x000d07bca24e4546, 0x0000bf42ebef2391 }}, + {{ 0x000819dd9659911d, 0x000754374dcec468, 0x000ccca8d9a91332, 0x000dbbc9c33cfc02, 0x00001f5408b7ae98 }, { 0x000eb357eb3611e6, 0x00044a60871f4274, 0x0005a83cc448d345, 0x000e00faf8e0cf36, 0x00003b6703b79d89 }}, + {{ 0x000f5d55f460757a, 0x0004ec4868af75df, 0x000e466b71f6795b, 0x0000c72b0d7325cf, 0x0000e10c5f3da26c }, { 0x00094ad8a7522d79, 0x000cc411977a0ee8, 0x0004e8d5601f6c03, 0x0001e3b955085c4c, 0x000028f635d618a7 }}, + {{ 0x000f311e0eabc31c, 0x0000741ada354383, 0x0008c56a6e3fb07b, 0x00054b334f4a9fc8, 0x00001f2b7d334bf3 }, { 0x00027ce89b6036fc, 0x000043892ab98919, 0x0000ac10010e629b, 0x0008e3fee257bdcc, 0x0000d4e8e7aaee28 }}, + {{ 0x000cdb6e7895d161, 0x000cbca523db2808, 0x0009c3f8c90b1ad1, 0x000c6706f5c889f8, 0x0000605933f73ab2 }, { 0x000076d41f277058, 0x000c4f0179c8e744, 0x000207aa49414348, 0x00014ba00d8874e5, 0x000029e04258b6c3 }}, + {{ 0x0008a790b23045f3, 0x000333c47b517ea8, 0x000ca7c0cdac91c8, 0x000c03ea079a448f, 0x0000f88cfdd1c6c8 }, { 0x00044cc5afb2f094, 0x000787d80969fe8f, 0x00034cfc5609af25, 0x000a4e12654f92fd, 0x00002a65082a73a9 }}, + {{ 0x0006c0d50cd87d6c, 0x00017278b1e68b79, 0x0009997e54c71ee5, 0x00052d2cbc731ad7, 0x0000b44017339c7f }, { 0x000bf64f7d3a4aac, 0x000f3e79b4f7f248, 0x0007b1a318728a1b, 0x000e1748dfdd781d, 0x00003026e7244622 }}, + {{ 0x00007a3c1ceeb879, 0x0000d2c467be912a, 0x000926c2f4afd167, 0x000ffc5c7f4c85ee, 0x0000156de531a17b }, { 0x0002c133aac4e622, 0x000cd450c00beb33, 0x0002cd4c1893c312, 0x000229d5b30e2c2b, 0x0000fcacc4f9ea39 }}, + {{ 0x000894a646301fb9, 0x0005dbb32e5685df, 0x000860524efe08bf, 0x000a632c668a9e06, 0x0000a480e5c2b19d }, { 0x0007ec2caf1e7fea, 0x0004b0a09e27ecf4, 0x000e110aeaeed496, 0x000f877baeced201, 0x0000dd057fe48dca }}, + {{ 0x0007eca65a996482, 0x00015c4f68367ced, 0x000ae66db0fb60e8, 0x000342779e4f47d0, 0x0000d6d9565d0319 }, { 0x000b6873efddb317, 0x000a47149ded0a33, 0x0004bfcc5fd5434a, 0x000d273334cb2aad, 0x0000fecf3dc03882 }}, + {{ 0x00066010689e1641, 0x000f3b22c11bb373, 0x000db6ec643ab5eb, 0x000afb15d2d0366d, 0x0000ab615f997250 }, { 0x000b5f7e8a49c22d, 0x000402c68d7b6d44, 0x0008c124aab7be0b, 0x000a684eea8f74f0, 0x00006ee0d7c27a2d }}, + {{ 0x0008fc242b74d79d, 0x000d47789070d267, 0x000e6420679c51cf, 0x000c68217a1f28e4, 0x00000be63ac7dad1 }, { 0x00042c485ec72b60, 0x00065928b0588f1b, 0x000c610949c0029c, 0x000ba4e35b6b7a0f, 0x0000c2df5af332f6 }}, + {{ 0x000752e537b3aeec, 0x0005213dbab98aa3, 0x000b6a282aeced62, 0x00097413ade9d469, 0x0000c4a05092000c }, { 0x000617ba15d6404b, 0x0001fdd15544eb9e, 0x00069ae3e8640d68, 0x000dfb6b5f013aea, 0x0000c07b8448571b }}, + {{ 0x000d7158f2903d4a, 0x00038a6936219d36, 0x000440e072274354, 0x000ca5d4c5ce1f19, 0x0000aeea5b76a62e }, { 0x0008beb35eb7c48c, 0x000df21e300ce18d, 0x000f545273fe83bf, 0x0005f0f9b35fba62, 0x00009e3ee834d7e8 }}, + {{ 0x0002699749223725, 0x00006b01f7e320d9, 0x0003db3989a12b79, 0x000dc64a30641c36, 0x000085fd6388145f }, { 0x000b92552feb649d, 0x0005217ceb6c1434, 0x00018da120fb460b, 0x000267af867fcb84, 0x0000515b55c60526 }}, + {{ 0x00099bdc6fc777ca, 0x000fd0e46aca7930, 0x0004dbdd54abe6f3, 0x0008b2d161cf0b0e, 0x0000bd4ba21f4e90 }, { 0x000ce18473bb041f, 0x0000cb20caf8b051, 0x000a55f4b97f1131, 0x000c0d508f74d505, 0x0000a157aa07d7d6 }}, + {{ 0x0004d831d9980ef8, 0x0007d9783abdb4a0, 0x0006da10193eb141, 0x0008f6fdc850fbcb, 0x0000b1f65be994c8 }, { 0x000ca16d905a0431, 0x0007d3bfa12d45da, 0x000d9aa574b633e6, 0x000f1c21ee425f8c, 0x0000472d6784e4ca }}, + {{ 0x0006be873329a887, 0x000d36a9152f9347, 0x000d764f8b8de418, 0x0009ef6acf1235e5, 0x000039762606b9b0 }, { 0x0003ee10d17b708a, 0x000ee310c0cd3130, 0x0003a226153c53ce, 0x00041a3153c52455, 0x0000f1a929d0274f }}, + {{ 0x00079739f7318d32, 0x00027b21994ef202, 0x00093bf3a921a053, 0x000dc5e102a653b6, 0x0000417acbe360fd }, { 0x000b40cff123155f, 0x000073c8b799336a, 0x000c74fb67fa3a8f, 0x000c69c2dfdfdf14, 0x00008baaad38eb7e }}, + {{ 0x00090bdcfccc29da, 0x000d983d20b42d5b, 0x000ee66ca762f3ee, 0x00090c3f0f9a810e, 0x0000f553cd3456f1 }, { 0x000e248f78d9528b, 0x0007a91f01ab782d, 0x00008ccc9b0f3f1f, 0x0006eb901c823f21, 0x00005ba0497c7a34 }}, + {{ 0x0005b53add7650bf, 0x00085e50cafe751d, 0x000bff2797447909, 0x00034812473997c9, 0x000097d3990af468 }, { 0x000e25f2ce2b08a5, 0x000d511dcedd8d12, 0x0008468d39b5d584, 0x000d543b1406bd1c, 0x0000fd5a272530da }}, + {{ 0x000a70e6bf33183a, 0x000a35e29dfc254e, 0x00054127cba88ba8, 0x00015b8ad4455fcf, 0x00005553f496e694 }, { 0x00036bb6459a57f5, 0x0006c695820f3bf6, 0x000d3acc40499055, 0x0008db783f410d2d, 0x000031a7e7bc74e1 }}, + {{ 0x000fa724e36fba72, 0x000dca39754e21c0, 0x00067dbc9e8fb755, 0x0004f699abc87a3e, 0x00008aa8469ef3d1 }, { 0x0006c5db934e2c84, 0x000c829b0720b169, 0x00026fe7131a51c7, 0x0003ca5a62dd0cf1, 0x0000be10a91df994 }}, + {{ 0x00074764ddb736f7, 0x000ddc05f18395e0, 0x000418396b551790, 0x0004f105bd3f2274, 0x0000b92a58459d14 }, { 0x000f0b4e7fc1d0d6, 0x00013507bc7f3893, 0x00007a6a15e11d01, 0x000d3ab57c580c85, 0x00003331b44a24bb }}, + {{ 0x000f11236279ebc4, 0x000a9e8a9f19c539, 0x0002ca63bfeba930, 0x00064cba95717bd7, 0x000086b4bc72b477 }, { 0x00076fcfeec45e44, 0x0001a31823b73482, 0x000584f3707abbb6, 0x000397e584dd8f0d, 0x00002080c6a3d85e }}, + {{ 0x0001374518550834, 0x000e75e2bc0aea9e, 0x0004fcfb4f1939ad, 0x000dc847e4f668fd, 0x00002277741998e3 }, { 0x00010649ba9269ce, 0x00029e6853ac983a, 0x000dc7e1409c87dd, 0x0003f9fd1c3bdf42, 0x00001a9e558bca9c }}, + {{ 0x000984d8c0756da9, 0x000f60b879fbbed4, 0x000691183fa97cc8, 0x00008c6450ff0ae8, 0x0000bc7e3aa798e1 }, { 0x0002c0087c8490da, 0x000bba6d21008f03, 0x0000075a13ccfa3b, 0x000db76f329bd54a, 0x0000cdfd68fa494d }}, + {{ 0x000eda0108b2b2b2, 0x0008ae60c1cdb260, 0x000746a2a0fce8f9, 0x000ff4bcdcd94d94, 0x000051bef4868fea }, { 0x000105f6c2d3fcbc, 0x000aa1056152fcc1, 0x0006d6133470c109, 0x0003149a4c2f037e, 0x0000f1a1f67957f5 }}, + {{ 0x00013c1b9dfbee2e, 0x00007008733913cd, 0x00026dfec0a26cf9, 0x000b44ae9cce8616, 0x0000301eb9bba43c }, { 0x00058c5a3c8e1fd6, 0x000a8c71c74cf5c3, 0x0008bc18677805cd, 0x0001a3bf18887dc4, 0x0000f290c47af3bd }}, + {{ 0x000c23f6bd8f7001, 0x000c304890228299, 0x00069f5598bb730d, 0x0004e6dad40e9397, 0x00001a83f93d79a9 }, { 0x0004233cffc4cc06, 0x0001435fbffd8bb2, 0x0008f52696de0ca5, 0x00059d33a60fcfb4, 0x000067aa71b84bb3 }}, + {{ 0x0006fe382d94c44a, 0x0009a6ce6d982af4, 0x000ea056154ed2e9, 0x000c78687ebb6bf3, 0x0000ae247fcc48a8 }, { 0x000a247e575d5c4d, 0x0005ae8b5bdbd756, 0x00014ecdaaa75fdd, 0x000bff600ab38133, 0x000002fa41762570 }}, + {{ 0x000df2688e5c0bfa, 0x000f81b636288bec, 0x0003fab55a773328, 0x00085a4049dfdca7, 0x000085fc068e8d8e }, { 0x000976ef578a35f1, 0x000ee44d0f8a67ee, 0x000a21e1e0b95bd1, 0x000ef07f4d985df4, 0x000004f8b7366d81 }}, + {{ 0x000b539e244d918a, 0x0000baca1c323732, 0x000a47f86bef05d2, 0x000da4bbc351027a, 0x000011f52fa1b210 }, { 0x000bea3fa5e2c7e5, 0x00067ec0eaa1d710, 0x000d5a338fbff1d1, 0x0008ce2402a83381, 0x00009435f8928998 }}, + {{ 0x000aad621cd624b4, 0x000028436cc61347, 0x000345f6e1c0c72b, 0x00016c92a8eb9bbf, 0x00007457c7601162 }, { 0x0003e95ca5e2ca28, 0x00025740a8f7cc07, 0x000d936cd0bf865e, 0x000cd30eee820c2c, 0x0000d32e75fd1366 }}, + {{ 0x00075553bbd18331, 0x0005fabfbaf50a55, 0x000b10f8b3952494, 0x000f37f05df55e76, 0x000083667f4aec1f }, { 0x0001b91149c64f10, 0x0006ca3553afa736, 0x000beb9cb1077179, 0x000ef4d35315f3ff, 0x0000724cdc42e0e8 }}, + {{ 0x0006710053e87e6e, 0x00027d3fa326dc3b, 0x000e628b2c963b4e, 0x000250f591df4da1, 0x00007156cd0888e8 }, { 0x000ea109e476ac64, 0x000027e618344d14, 0x000251c33bcb204d, 0x000e1583acc7ce04, 0x00003fc838df305f }}, + {{ 0x00061e97bf3ccbad, 0x000faacb40961bf5, 0x00071486d43b1a75, 0x000ebb8d14ada593, 0x0000416698a685c5 }, { 0x00083dfe218457e7, 0x000beb6da3982f5d, 0x000cdd08f647376c, 0x0000946200c2659e, 0x0000016501cf88d5 }}, + {{ 0x000970713804550a, 0x000ba5fb835398c3, 0x000f7a19d08a80db, 0x000a0d2db6106a8b, 0x000024eccd926e31 }, { 0x000c5926d6e5fabb, 0x0007d34d8249f00d, 0x000ececc10c98ac7, 0x0006705c29f14b58, 0x00007dd9e5578e79 }}, + {{ 0x00059b045ae4cdd6, 0x0002d398e77738ae, 0x000f16095fcbf91e, 0x000b19886d07a63e, 0x00009b2d8bb94209 }, { 0x000115fba320b394, 0x000f0e520fa03389, 0x000859e95f2ba538, 0x000efef1397939ac, 0x00002792ae3d4093 }}, + {{ 0x00097bb53067a096, 0x0007ac25fadf8c35, 0x0002749e0bb22636, 0x0005dba2266be38a, 0x0000e8e24f086be5 }, { 0x0008e7ffb7148254, 0x00032b7c153ca7f9, 0x0002d7956406285c, 0x000de7e1ea8d30fb, 0x000062282ab3dba8 }}, + {{ 0x000b613b880d4a8f, 0x000eef8e9726a306, 0x000f446cf9abc9c9, 0x000a75d40b5e216f, 0x000085573a0b1122 }, { 0x00016530e4f15d5b, 0x00032ba792d56100, 0x000aa02bcafc965b, 0x0009826da2ee1270, 0x000079f2ed0cf73d }}, + {{ 0x000b9754a0266bcf, 0x0006efd8fd6ee891, 0x00062aed3b5fa54d, 0x00038abdf16d3d98, 0x0000116ae37983f7 }, { 0x000a47cbfca3dc3a, 0x0002cdf70c33b988, 0x000e54ec4c71ebb6, 0x000f60732aadc87a, 0x0000b626c156e4a6 }}, + {{ 0x0009f8db144f77e9, 0x000026662b5f3af3, 0x0000d0c25c1ff887, 0x0007b73b1cefab56, 0x0000cd5a1cf3f9ca }, { 0x000d5a818694755a, 0x000725b3c61bfd60, 0x000d33c6e57d6b39, 0x000d8c2c2a5a4d41, 0x0000da06aa84668d }}, + {{ 0x000f71b5f5f2e3b8, 0x000bcf7d22da9a9b, 0x00080d18e517fe23, 0x000df74ff45b8a67, 0x00009b0d34e92cca }, { 0x0001751253d7af6a, 0x000152f69fc4db96, 0x000c14e7ef8dbe16, 0x000e0dc35b6dad34, 0x0000dff44628ebf3 }}, + {{ 0x0005c4cb0846e24e, 0x000db9781af2c2d1, 0x00046bf50cd29179, 0x00070f08dc0f7d9c, 0x000019b4a7b7a5c1 }, { 0x000c9beacfa5ec60, 0x000d2858e121e14e, 0x000d98641f8ff20b, 0x000390274eea7dc1, 0x00008ec5b0cee6b7 }}, + {{ 0x000bad30cf8c7404, 0x000a6d71df978283, 0x00056ce647a78ac9, 0x000f06b3cbf851cb, 0x000022014a98c474 }, { 0x0009c81e9e8866f3, 0x000bf02496bbcea3, 0x000eded9ca1c764b, 0x0009f6d293cf9fd4, 0x000097210cbc6093 }}, + {{ 0x00067291d1c7e163, 0x000a9b97b390d355, 0x000d3fd8195610dc, 0x0000e01f801df2bb, 0x0000aeaffaeb93fc }, { 0x000e7ba92c88e286, 0x000166c5f5406d3b, 0x00089d2d39130c67, 0x00026e8ce71fdda3, 0x0000429a520ee24c }}, + {{ 0x00059814cd121bf2, 0x000e16bc6fb5a7df, 0x00097795bc1333ec, 0x0007bb16ae3eea15, 0x0000ea9234811e7d }, { 0x000b866e7b5afbdb, 0x000eab01c33c23f3, 0x00043423d2bafbcd, 0x000358a51a96c1da, 0x00000a66348cf983 }}, + {{ 0x000243ce8547c141, 0x000e6d81495ed6bc, 0x0002f43f0e622ab5, 0x000c4cff3b239424, 0x0000291035b7c92d }, { 0x000997f8cbbf2476, 0x0009771dbbf370fb, 0x0006bf48fefebc0d, 0x00056dc3fff9afa3, 0x00000d06a64f152b }}, + {{ 0x00080c94a72ec503, 0x000ca2a37bbb1840, 0x00099c508afef996, 0x000d131cb51dcec4, 0x0000989196a9bdb3 }, { 0x0008f4ead02f1a6d, 0x000d0b12283fbe37, 0x0007e19e2b1d8b3d, 0x0007bdeec902bddc, 0x0000d0bde74ee357 }}, + {{ 0x000633d510319630, 0x0000a4956f908239, 0x000fdde9a26b1a49, 0x000f5c760ba0f507, 0x000011a47fccb3c1 }, { 0x00074a4f6a780782, 0x0003543b8c9de617, 0x0003e69cf7b58be5, 0x000c69ab2b475125, 0x000059a05b225efc }}, + {{ 0x000530e06a0e952c, 0x00059f773848f22f, 0x0005fec775249e0e, 0x0002f45f953dad71, 0x000030c4df804685 }, { 0x000aa392d634bc5b, 0x00056ffdd59d31ae, 0x000e1e3714bf08e3, 0x000102dd74382e0d, 0x0000d560753775a1 }}, + {{ 0x00032c96e155ca24, 0x0001867648024ee6, 0x0002290314ec3e56, 0x000e4a435188763f, 0x0000359aad14cb65 }, { 0x000aec2c62bf9c7a, 0x0002c0e4c00a95ea, 0x00003525bd976a14, 0x0001af8232ef17f4, 0x0000950b712f6e56 }}, + {{ 0x000cb27078304cb0, 0x000d8d83d84e58c4, 0x000c48d30163a309, 0x000b9d2a62af5094, 0x0000f43399bda145 }, { 0x00025423b825ac58, 0x0006d474ff0e321a, 0x000578e0a546e9ad, 0x0000fc3a7169f34a, 0x0000fbaf5744de2d }}, + {{ 0x0001d6998c9bae7d, 0x00058d50ac8b7823, 0x000bfbdd2f05c1a7, 0x000dd3942ff6622e, 0x000068b2d4cab4cf }, { 0x00020e9934c184c5, 0x000bb04cadcaed01, 0x0002c60e6c081a2a, 0x000de49c6801b014, 0x0000c34427b5c797 }}, + {{ 0x000321a50d8134d2, 0x000c8a4fdc6a4043, 0x000e6e6ef43ad26d, 0x000b3c48119d8e01, 0x0000448b0fd8e456 }, { 0x000066350f1fe586, 0x000e263e9a861112, 0x0000e63d8756b063, 0x000ba08ef8c0cde6, 0x000050af05bd3adc }}, + {{ 0x00058deddc932110, 0x0003689e56ddfbd6, 0x000798a4aa5dbc69, 0x0001ba8cbea1b0c0, 0x00007b58c49685f4 }, { 0x0002efa53a6c1ccc, 0x000e34d0f64ef589, 0x000ce01d226df9d2, 0x000e7b170fe43287, 0x00000387e93337d0 }}, + {{ 0x000bbb69838b4dc3, 0x000c15794b565268, 0x000e7da596a97b99, 0x0008eee0f415cb80, 0x00008189781ee1eb }, { 0x0008da59e33a17c1, 0x000b85b9abc26689, 0x0003858d35ce15d0, 0x00065e1b9b5d4f53, 0x0000e8073f5adb41 }}, + {{ 0x00041b32e5ee0cd0, 0x000bea601799a527, 0x00014451f66c2b00, 0x000f25a7ca20cec4, 0x00001caa549ccc91 }, { 0x000641caaadc47ea, 0x00097814eb57d471, 0x0004006d25c11bc5, 0x000a4cb887a2d0ed, 0x000005cf797f0540 }}, + }, + { + /* digit=4 (1,2,..,64)*(2^{28})*G */ + {{ 0x000cf9a2f2b6ddb7, 0x000cb4f20151427a, 0x0005e5495f13c8bb, 0x000336c57206828b, 0x0000deb474abe52a }, { 0x000c630fc54290e0, 0x000e9da279153564, 0x0006b3bbef1e9949, 0x00021a8e27e0734b, 0x0000614b2885f690 }}, + {{ 0x0004ab68d7e959ee, 0x000227a1ac2703b2, 0x0008471ceedd9bdd, 0x00080ded1b49258d, 0x0000011c8d9a06d3 }, { 0x0000b360f8d06cf6, 0x000bfd4734e59d08, 0x000843e26b87ae6b, 0x000b0c3b35f8740c, 0x0000b6d468726158 }}, + {{ 0x000ac1e3e5d6df58, 0x000075d5fa067d1d, 0x000ebba6b226d7b2, 0x000af82ac4134b57, 0x00001bb953caf0cd }, { 0x000cde538c8bc9c2, 0x0003669d81fbb26c, 0x000be58de047db86, 0x0007841e8449ed3e, 0x0000d643fef5794d }}, + {{ 0x0003331d3c69f2bc, 0x0000a8da61a76fa3, 0x000d96e2379febe3, 0x000486b9c598b441, 0x0000390a58c27d0d }, { 0x000cdb9daf4c5417, 0x0006d8e298de399d, 0x0006c4c6e1a6e0f8, 0x000649470cfdb8e7, 0x0000690440cf3b0b }}, + {{ 0x000cd3d0d4027b25, 0x000d3c2019f2a596, 0x00035e7beba07964, 0x000164eddb3ce5bb, 0x00000d8037d39488 }, { 0x0000c29968554c2a, 0x0009ad6efed021c1, 0x000059b49a323da0, 0x0003c9c4a09a9f9b, 0x0000c0cc53fbad1c }}, + {{ 0x000c73d2287162dd, 0x000c529ed6e82ef1, 0x00034977dd25e6f6, 0x0004a8aae16f338e, 0x0000dc61c21e5eed }, { 0x00096fe447e4eae7, 0x0006a3174a5c8c7d, 0x000fa5f9c92267ca, 0x0007c117e01cd296, 0x000068abc8039604 }}, + {{ 0x00041d6367599350, 0x000706e88fbd3813, 0x00042818412bda97, 0x0005e6bcd02cd4a8, 0x0000b40e707676fb }, { 0x0006fe0a3a3a6e7f, 0x000d8fbd238202b9, 0x000a1286e09fb174, 0x000d4efa0600b4d1, 0x00005566cd140a17 }}, + {{ 0x000246780b6116f2, 0x00015367f636a38c, 0x00064c305dbfeb38, 0x0009f998b9f943fb, 0x00002a4f058aaf47 }, { 0x000300fadd2ed37d, 0x00097156d549fc2b, 0x0000ad7686ed0d31, 0x00005d713c2953f3, 0x0000c01130ae8caa }}, + {{ 0x0000e57c5b8cc8c5, 0x000b65087e9189fa, 0x000c53485dcbf535, 0x000419404db17375, 0x0000410edb213b42 }, { 0x000cf14a112a5ea6, 0x0006bd3f40e3c80d, 0x0000b0c5bbb47bb1, 0x000a038c3b118932, 0x00002d9c468c8d26 }}, + {{ 0x0006c8b3e970a581, 0x000a55b556913429, 0x0003fa3be9dd94c6, 0x00015610ae8e0850, 0x00009ceceff70c59 }, { 0x000953c3d70998e4, 0x0001438fb4ee6b40, 0x000b3895091982c7, 0x000476cc6fc5a9ae, 0x000057745cd78df1 }}, + {{ 0x00071e305a112a00, 0x000268238dfe7e82, 0x000504996cf1100d, 0x000bdcf2dea417e1, 0x000026436feec265 }, { 0x000396770a6a23bd, 0x000c0fd7253ecbd7, 0x0004dc33ea7bf828, 0x00062b19c2f552e2, 0x00001ac28594c2e3 }}, + {{ 0x000b1dac7015ef7f, 0x0009263c27901b4b, 0x000fce7ddf838b5b, 0x000d75b0afd9236d, 0x0000069d82647cb5 }, { 0x000f374bb4fe7c59, 0x000c453a91cf76e1, 0x0002b3e7d3328269, 0x0007376774110a41, 0x00002b0cccd10515 }}, + {{ 0x0001616338327868, 0x0007af20d188dab3, 0x0000c35722b72c00, 0x0009bef0fe3968ed, 0x00005947d75b5640 }, { 0x0006a44d86a5f4a0, 0x00047e24f0c8afe3, 0x000fa8462b5372ed, 0x00068ea514caa75f, 0x0000f6f22bba6bd6 }}, + {{ 0x000ff17a8fcc8db3, 0x000a8da0333974fb, 0x000d67b586b70a9e, 0x0008146215db516a, 0x000074410c8af154 }, { 0x000c274e88147e8f, 0x000f24e12476da14, 0x0002e79a1af86475, 0x000cebccac1e1661, 0x00006666b278e72d }}, + {{ 0x0001da368c16a10b, 0x000ad8d6c2f5226c, 0x00039333461ca008, 0x000230c0914d1b94, 0x0000a4acd35a7009 }, { 0x000b480cfa6c0f39, 0x0004cced5113cf71, 0x00033eec6394bef8, 0x000bd06880670f63, 0x00008997a9636ca9 }}, + {{ 0x000953c50eb5331d, 0x00069024147519ad, 0x000b4d5fade16af5, 0x00052cec8d0981ea, 0x0000f785b3ffbbf2 }, { 0x00063ebb7f24f144, 0x0001dccfd590f8f8, 0x000abea80d987872, 0x000fa20cf72e9626, 0x0000721e8be990fd }}, + {{ 0x00083e50e54aa3f3, 0x000c4616aca8ee7c, 0x00090f75b27323c6, 0x000bbf80d97d10b3, 0x00005e98ed7591d3 }, { 0x0005f9680c64a8ca, 0x000dffec954b3132, 0x00070b13b7552acc, 0x000af06e0cc2e8f4, 0x0000143469e3b900 }}, + {{ 0x000561bf1c36fc6e, 0x000f8094354080b3, 0x000546913f00c25f, 0x000812afa5225bfa, 0x00007cc75ab3c625 }, { 0x000332d9e420107e, 0x0002117ba1514741, 0x00049d1c94e63e5f, 0x0008a4d1edec2f93, 0x00007771c10cbd06 }}, + {{ 0x000cf057fdaa6cc2, 0x0000d69df23de05b, 0x000143a2e1a2b66a, 0x000fc772366d17da, 0x00006a8d4733875f }, { 0x000c23b3ee9a963f, 0x00089c5f0b5662c7, 0x000860969b36292f, 0x000952aff25318dd, 0x0000ef7f5b56866d }}, + {{ 0x00039af66ebf381e, 0x000ec0a097e4403b, 0x000565ea4ff656a8, 0x000452a26cb3d0a8, 0x0000a7947609a317 }, { 0x000290d33bd0a0a9, 0x0009c88bd475ca8d, 0x0005ede6e4cfd558, 0x00087196ce06b796, 0x00001bfceaea4a09 }}, + {{ 0x0003e4fe080af01f, 0x000ad47f34fb0fa1, 0x00080bccad01991b, 0x00083a0e3b4760cc, 0x00004486d52a8b97 }, { 0x00069059367f304a, 0x0006f92fdb05163c, 0x0000459ac07fbc9d, 0x000c299e28c484b1, 0x0000e9d36b72292b }}, + {{ 0x000ab74e6a2a524b, 0x000249d586e74e18, 0x000cdd00b61a885b, 0x00049f2db488883a, 0x00009b15259f5d0c }, { 0x000de0781ad6fc39, 0x000f4285e21beb19, 0x000993abcade39ff, 0x000d307c33484518, 0x0000f3708fe8c17c }}, + {{ 0x000604437bc3417c, 0x0006c58693807e1b, 0x0006987d894bcc6d, 0x00041df84a386532, 0x00004f10004876ac }, { 0x000a01df0ef92b9e, 0x000ec985ca1823d5, 0x0008a9bc2415bad0, 0x0003cac33b82f7ec, 0x00008c8d609652d6 }}, + {{ 0x0005e94efbb5789d, 0x00065f31306583d2, 0x00001d931332b2e9, 0x000f1dec805b10da, 0x00003f4f29f81351 }, { 0x000ede6596524043, 0x00086b7fa3dc26dc, 0x0002c8209e2f37f5, 0x0008351cee0e5fb9, 0x0000f62ae015ad99 }}, + {{ 0x00025ef904c10f7c, 0x000a6f6a19935459, 0x0005927778b6116c, 0x0005a468eab01cc3, 0x00007a92aaab72e0 }, { 0x00060897892570ee, 0x000b5cf53aa0b32d, 0x000a9bbc05f115be, 0x00094f47fb27beb5, 0x00009605766eabab }}, + {{ 0x000cf6c431cbebca, 0x00082e0d6fbc2acd, 0x0000a469311f571d, 0x00003479c738037a, 0x0000ebc968b789a0 }, { 0x0002ad6eced662ff, 0x000ff7be3ef104a2, 0x000be758e7592a81, 0x0002f053eebdd9a2, 0x0000d738f011f4b1 }}, + {{ 0x000c11a2e84862b7, 0x0005cc039cd297dc, 0x000d39ac02f15755, 0x000ad94968ec7e16, 0x0000f7a4858d93c4 }, { 0x0007c422e7b70ed4, 0x000865396ee63809, 0x000536df645adfbb, 0x0003f267d054c865, 0x0000eed4df196b1e }}, + {{ 0x000ba62aa6a9ec8b, 0x000ea090c22b5403, 0x000a7c1c3d2e15a3, 0x000c44b932cde42a, 0x000080285f5a46b0 }, { 0x00066c5b9ee54102, 0x000f77bab9ea96a6, 0x00027d49b293cd0a, 0x0000c99d55d03964, 0x000042cd3eeb6797 }}, + {{ 0x000a56a7143dc2a2, 0x0003ad98edbd7cc8, 0x00073c76303fbc7d, 0x00024144f4660bb8, 0x0000534e55030be4 }, { 0x000b2524cccf0e8b, 0x00055965726d47f7, 0x00018ff575b2dcee, 0x000a8bf379eed01b, 0x00003984d331077f }}, + {{ 0x000539a4fc4c4c2e, 0x0003a3b8f4b002f6, 0x0009dcbb57241ddd, 0x000e75cccc2200d7, 0x000071b64d7d7ded }, { 0x000b7a26a5d05150, 0x000cc14c33c7fcf7, 0x000b4c440b85fe71, 0x000f5a6237eb6799, 0x00004066761e4395 }}, + {{ 0x0003ed2c34dbfa56, 0x0009a8d403f46f2f, 0x000226219492ef3d, 0x00037be2f94b8fc4, 0x0000aea3923eaee7 }, { 0x00047ff958e890e9, 0x00097afb639126c8, 0x00005ed54185a48f, 0x000ed4209f7b6be6, 0x0000839d619a4633 }}, + {{ 0x0009fb18a0fee18b, 0x000f1f5678a31b0f, 0x000d3f9d9d7f8899, 0x0002e2896d50301a, 0x0000467828ca24c1 }, { 0x000a2f2a98bf1696, 0x00042ebe8758851b, 0x0005a36a8fac4075, 0x000e6fdb71e64e4c, 0x00002450761a46be }}, + {{ 0x000106b953c0b0b7, 0x00055bae8053433d, 0x000f6c25dc31bcc8, 0x000283eedc8e7259, 0x000070486fc2230d }, { 0x0009b714490d84cc, 0x000ab5b0eff13a96, 0x000a76a0a4e72ed7, 0x0003240bd242442b, 0x0000e38dc2108bbc }}, + {{ 0x000ad0436ab80ab8, 0x0008ef60e2ed7422, 0x00051536312889e3, 0x0004a0b04573f104, 0x00006d3f1bc994e5 }, { 0x0002b773beca9fd2, 0x000d82d431a8121d, 0x000ab5b79e2f23aa, 0x000aa3cb7cd38b3a, 0x0000164fbcfce738 }}, + {{ 0x000d927bed2c4843, 0x00057c49b5c1f14c, 0x000970c2633e1e0d, 0x0005bd591c444be1, 0x00007472609e17c1 }, { 0x0007c6a79d00e920, 0x0006516e48c8a348, 0x0001c38dbeccea53, 0x00064d8ec1b7ac81, 0x000001913870f958 }}, + {{ 0x0000b9e865afa38c, 0x00016f09b56ddbdb, 0x000bd975348dfe5f, 0x000fa730d1339b5a, 0x0000a2f19f0d61d3 }, { 0x000bafec279d834b, 0x0007f4177bb064e9, 0x00026174b33bb275, 0x000ec1d9a145a281, 0x0000853c8b1d855a }}, + {{ 0x0004b8f6ae320ba5, 0x000a7654265aa91f, 0x00030ea60793f5f9, 0x0002e39c7a4b1830, 0x000052a5da5f3864 }, { 0x0008eb65e67a4037, 0x0005ccc971a8a130, 0x000d43a7e7218152, 0x0006c85293f159e5, 0x00000f9da5baca8f }}, + {{ 0x0007b514916e3af9, 0x000fc8930b75c3ed, 0x0001bc940801e1b4, 0x0007469bb0bb82fe, 0x0000306f321be91b }, { 0x000a91fcbaa89e73, 0x000277b04497f19d, 0x00093fe8d485032c, 0x00029fbd61b35ed0, 0x0000ab396c946696 }}, + {{ 0x00043b4fbd7ee0ab, 0x000a57bd7ab27e14, 0x00051f590bbe1f47, 0x0001bed277c19a55, 0x00004ea5e94be42c }, { 0x00079cdff78fa878, 0x000b6656962c88f7, 0x00045670d0d30682, 0x000e4f44f139da8d, 0x00001bd14a9de58b }}, + {{ 0x00003907bb055ec3, 0x0007cddae57c7b7a, 0x00017b5a1d67b643, 0x0000e87a3345342f, 0x00001c62c23c4350 }, { 0x0006a27dca73d570, 0x000a0e1ff2189336, 0x000443af93b0c4da, 0x0001d76eac066b65, 0x00007c9b470b5621 }}, + {{ 0x0007ec6e15fb0c4c, 0x000ad93048b37172, 0x000a898c40066ebb, 0x00042d16bbf5a989, 0x0000641d1a6e425b }, { 0x00009c3566d8138b, 0x00078737276e7fc9, 0x0008f93a899b74ba, 0x000ba4e7964086d7, 0x000097a1ea45df74 }}, + {{ 0x000ca56513af6eb9, 0x000036b75dd7125f, 0x00068afadd796c99, 0x0007cb29db4c6bc9, 0x0000450348223f4e }, { 0x000cf70def0b74d0, 0x000a181d0e8bd06b, 0x000709c438fb0a64, 0x000dde624384dd31, 0x0000e9a2bfac312d }}, + {{ 0x000ec09a48204c79, 0x000886e6628e8c30, 0x0007a749dada16de, 0x000e7d5e207a0d66, 0x000034e09153d2e9 }, { 0x000c7f44552abda4, 0x000994ed83dce467, 0x000d1d53b819438d, 0x00011604c43a6316, 0x00000c57dbe9342c }}, + {{ 0x0009b1ddeddfb155, 0x0005c21fbabbe926, 0x000b77812288d335, 0x000d7ab15281850f, 0x0000e27f34f8feb7 }, { 0x000c9f1b87c58c64, 0x0004d8fab1051053, 0x00016bfdb467b764, 0x00004dc09953ced3, 0x000034ba3f40d826 }}, + {{ 0x00067e10d16361cc, 0x000d5a14a7eadcb4, 0x00010bb060fe6649, 0x000cc8383928e750, 0x0000a507d1a255d6 }, { 0x000d744bd0c7e26b, 0x000bfc3b955dcf05, 0x000d766199572b61, 0x000958974ba96307, 0x0000c161d1c03e35 }}, + {{ 0x000b7e2a947ab5c8, 0x000602aa777899e0, 0x000fc4c1124a57d1, 0x000fab531e20eda8, 0x000078d9b61326d5 }, { 0x00039bad6e2b1553, 0x00011ad19f593cf4, 0x000edadb41af06b6, 0x000180d9c7b1a9e4, 0x0000bec906422f8b }}, + {{ 0x0006fb4c2d81211e, 0x0007266e032f1445, 0x000891b7a03ca07c, 0x0008011781db632b, 0x0000932932e1a77c }, { 0x00027d6babb8c904, 0x0002ae1255a03f19, 0x000256ed85caa685, 0x0009fb89761cd6af, 0x00003a067d13fd17 }}, + {{ 0x000b89a4e82f20ac, 0x0005be0e90fb21e5, 0x0000601e57b9726e, 0x000acbbb500fd2b8, 0x0000def3cf863f66 }, { 0x000f548f31add03f, 0x0006e3f894407ce5, 0x00040ccdd200e1b3, 0x00036823ba0025b9, 0x000057161113964b }}, + {{ 0x00088658197f5e5d, 0x0000e8feac6d1135, 0x000bb5fae45cabee, 0x0008397b9a67e3f9, 0x000027605a7662c0 }, { 0x0005660e1a5eff95, 0x000cf0cf7f06f200, 0x00062500dda83af3, 0x0007a8908dc61021, 0x000035ac8c485b17 }}, + {{ 0x0000a886286039e7, 0x00030626d3c46934, 0x0006f744f5c20ad5, 0x000213b6df0e7d62, 0x000063e4a8b99f57 }, { 0x000bf9a358ea18f6, 0x000c4a7bbb38ce0f, 0x0004c1e30a289e84, 0x000b0711fe0404b6, 0x0000cdecdc1fe1a1 }}, + {{ 0x000970195f38876c, 0x000fdb028f33398d, 0x000c9978f6062539, 0x00023b2b1b07ec11, 0x0000e90d968debc2 }, { 0x0003e03b335e7e4e, 0x0000ebe0181e6402, 0x00020262f609f432, 0x00074d182f5debd6, 0x0000f8b0b29420ff }}, + {{ 0x000447ad7afa3513, 0x00076494fde0c1fc, 0x000e216f3ea60540, 0x000acc86cbf8a1ab, 0x000009a60fde808d }, { 0x0006736b8b4a3189, 0x000213803d0ccf24, 0x00083b271077fea3, 0x000e645cc80acb33, 0x00000d9f85e61323 }}, + {{ 0x0007a56d943a749e, 0x000dd60c0432f963, 0x0003a70fd69b9be8, 0x0008acba2ddebe7a, 0x0000b67c71837343 }, { 0x0008b4408a4b9b5b, 0x0002fedf2350d0a8, 0x00057ac6cd104010, 0x0002b01ae13c3f27, 0x000012894baff1f9 }}, + {{ 0x000b605ef4a68b33, 0x000b2a35cb026649, 0x0000eb26683a34fd, 0x000f7a23e5d4c0bf, 0x000025f5686f6dff }, { 0x000715b3a144285b, 0x0006702c69645d0f, 0x00097305b3462d92, 0x00066f2c4a115f03, 0x00002d4b3efd98d1 }}, + {{ 0x000d8c3079ecf96a, 0x000a048d413213a4, 0x000d72bfbc61a285, 0x0008c1df52035806, 0x0000eb8c40841f70 }, { 0x00042be3c0b0a8c3, 0x00067678d95a8f6a, 0x000d4de5294c6a81, 0x000e6f73fc58c9d7, 0x000028eb6b61151e }}, + {{ 0x0001f8fbd7da4886, 0x0007142d43b5eaaa, 0x00063dac9fad46de, 0x000237745054a076, 0x000066d55ce85e38 }, { 0x0000aad9183a3f7e, 0x00002f5d563fccc1, 0x0005e1d5d00ff737, 0x00075facf48ca505, 0x00002d09a02699ed }}, + {{ 0x0005475f79a136dd, 0x0009e62b7bdc9534, 0x00021679276438a9, 0x0002811779f0f6ff, 0x00006a65c14a85c4 }, { 0x000d71419bada82b, 0x000fdfa86f702ccb, 0x00045190dd6b9201, 0x000e77dc692e6090, 0x0000c7da1d12ccd1 }}, + {{ 0x000831b6ca7da654, 0x0008dcc6028da9ad, 0x0009ea203ea89978, 0x0000343f45352fe1, 0x0000299e73c0a3c2 }, { 0x000d50ba88136cf1, 0x0003d1de350e2c27, 0x0008a0bc297b691a, 0x0000804cbc56240d, 0x0000b08ffae6ef56 }}, + {{ 0x000b049ff6e562d0, 0x000c4265a3a2518a, 0x00091b2b9d4cf195, 0x0003b8696cf81779, 0x0000718b68347a7c }, { 0x0000d0bf5e9df0d2, 0x00027b0feb13ec70, 0x00079cbf39ae46dd, 0x00009c4afd5e9711, 0x00000194165fa7fd }}, + {{ 0x000d542a16dde633, 0x000fd1f358bcdc04, 0x0008bd491de20858, 0x0005da13a07689e9, 0x00001aaffaa71871 }, { 0x0009749558e9ff03, 0x00090d59ca0a7072, 0x00056151eb0cc4a3, 0x00090fd8cd061fe4, 0x00000ed0be2cc216 }}, + {{ 0x000229acb440cbd2, 0x0002a59c1d5d0340, 0x000665b8692c33ea, 0x000362ee6f712e9f, 0x00003d3bf4320945 }, { 0x000d5dbb02d12e1f, 0x000c24cfff0c82e2, 0x00022c68e370cba3, 0x0009e9a8987684b6, 0x0000967e564a19c5 }}, + {{ 0x0001f390883520b9, 0x0001a3ae86b173c3, 0x000575ac605bbd0a, 0x000ec5310e8e41ea, 0x00008104ffff9da7 }, { 0x0005cd6ee65978c1, 0x000b0e17829839ea, 0x00039f353f7221c6, 0x00083a0150cf691b, 0x0000faa310dfd765 }}, + {{ 0x000fa20deeef2680, 0x0000d7b5f1d23474, 0x0009fd20174b3146, 0x000121e08794645b, 0x000077749c4b1069 }, { 0x000a1c9150ecf76e, 0x00004d8d797770a6, 0x000d55260cd5fea9, 0x000b814797360c91, 0x000033f4db806293 }}, + {{ 0x000379ae15e41fc7, 0x0000ca7c35d87949, 0x00076fdd0ccf2ac7, 0x000135bfd042e655, 0x00007cefff827780 }, { 0x000b09f648a49efa, 0x000b462fcc9981ea, 0x0000c21f3e7dc01a, 0x000af6bebaed3d63, 0x0000701de9b6b9b9 }}, + }, + { + /* digit=5 (1,2,..,64)*(2^{35})*G */ + {{ 0x000a823a4160c545, 0x000f51222248acc7, 0x000eb7e2a9255142, 0x00038a9281f6ec0e, 0x0000f63c9cc1c2fa }, { 0x000efe9ed3f2c904, 0x0002604e1900a791, 0x000aaaefc6a23190, 0x00095a14acde09d4, 0x00007b2680ab6b58 }}, + {{ 0x000c47940a63bdc1, 0x000acbe305406dd9, 0x000f4115854d7ebc, 0x000bf8c8f8eb7dc7, 0x0000eebcc0956bd4 }, { 0x00040deabe2974cf, 0x0008c5d99b307781, 0x00065a6b4bc494d8, 0x0005271e325380c5, 0x00006c75941c35b1 }}, + {{ 0x000049d7316af76b, 0x0006567afd32acfd, 0x000a11ad459dd98a, 0x0009f87dad8f0fcc, 0x00002a6cb670db4a }, { 0x00027074910360a8, 0x0007e3d42ab580e7, 0x0000b1869323c43e, 0x000fd66da09411bb, 0x0000931dee156e7f }}, + {{ 0x0003c3a8ea6b2c0c, 0x0005526c49a861ec, 0x00051d8b641c7fc2, 0x0006ff017024f3b6, 0x00006af90c54fe5b }, { 0x0006848b57a6af77, 0x000fe2cb103d4c8d, 0x0002bb9428138d50, 0x000a1d94a02bc461, 0x00005196aa6193cf }}, + {{ 0x000a9423bc0124a5, 0x0005de6c30fa92bf, 0x000c9fc103c8bc73, 0x0005e719b5a70d96, 0x0000e39716462e3e }, { 0x000d96bc718ee9dd, 0x000e112fe97a5b9f, 0x00087422d4a6a114, 0x0002dab4685da604, 0x0000ff2e0f500dd4 }}, + {{ 0x000f0e96b0521716, 0x0006bdb1782269b6, 0x0008b6894ea0436c, 0x000213b53ae34bf7, 0x0000f89e64dfb40d }, { 0x000020f0e5f7aa5e, 0x000a41577884456e, 0x000b89101dc1c7bf, 0x0005074deb3b5688, 0x0000322f52afaf0d }}, + {{ 0x0002cfa977b4a1d5, 0x000cbab0976505f7, 0x0002968e854dbe69, 0x0008fd671567f681, 0x00001688d84b98ae }, { 0x000f34f1cd8b8aab, 0x000b3ad8a2373da8, 0x0003ec6f689aa85a, 0x00077f215a8d0d5f, 0x0000c0b67348ce77 }}, + {{ 0x00039b66645c58c8, 0x0007a8dbbe5a1a9a, 0x000e8d8b5703704b, 0x000b790268187fd5, 0x000003b8f3f8d5eb }, { 0x000b2f3a4a4db66a, 0x00074ef2d65a8087, 0x000f193430cf1f8c, 0x000a6ec4c4969044, 0x0000c1c8991dfd8f }}, + {{ 0x0002864e5048aa8c, 0x000f145300eb294a, 0x000d7415c6444b6a, 0x000f9eea92b2a656, 0x0000f466251b268a }, { 0x0006be862b1eb6e3, 0x0008947db89842e4, 0x00071a89198c63ba, 0x000fc13dc12dfd7d, 0x0000921392892bc1 }}, + {{ 0x000b8888bb33c9a0, 0x00038dd0f82b2148, 0x0009657b430acfe7, 0x0000dfe19460c34f, 0x0000b39b6f7347e5 }, { 0x000bf0221449dce2, 0x00055dcc07226779, 0x0008802bed13455f, 0x0001abec610d21be, 0x0000f48269445497 }}, + {{ 0x0007c2ae5722977f, 0x000fe1242b48b998, 0x000cc07f923ee912, 0x00063ddfb1bcb665, 0x00006426594d7fc9 }, { 0x0001f5e5985dc788, 0x00029220de367261, 0x00082de9d7899404, 0x000b24c60e7043c6, 0x00006397a224d1f0 }}, + {{ 0x000dd04b97f56424, 0x0001d60b7fe7b6e0, 0x000a3239f41aa1f1, 0x000c4a61e7d16189, 0x000064bed27452ed }, { 0x00079b5499dfb43f, 0x000fd9b506db8cc0, 0x000ec79d2cb2ca13, 0x0009706e65cd47aa, 0x00001dff152d73d6 }}, + {{ 0x000ad76ee910eedb, 0x0003602c1c24a3b9, 0x0007813ead31dac1, 0x000012f4aca24e56, 0x0000638656568026 }, { 0x0007b4a7bd68d6f7, 0x00028c2538e0ff72, 0x00093e500bf32e6e, 0x00072d8d794f8980, 0x000026a43392b0c9 }}, + {{ 0x000aa7cb935a507d, 0x000551ae0ac29416, 0x0004582b4da3a965, 0x0007915d0279b025, 0x00008622b071bf70 }, { 0x00035a70c90666d4, 0x00051800d37c7a50, 0x000730d2e35953b1, 0x0006c1b9213380c3, 0x00008f95b8909dcf }}, + {{ 0x000c43c6cd7fe37b, 0x000aadd4ce4f152f, 0x0000fd9bb539417c, 0x0000140141f419a7, 0x00009af0fad64712 }, { 0x0001a9c2a0337ed0, 0x0009591925d6b2de, 0x0002806ea2f81671, 0x0009c46294b37d9d, 0x0000a4de29b6d19c }}, + {{ 0x0003fba6baaee763, 0x000548571ba18615, 0x000c7ccd5282a222, 0x0004c589348b22d5, 0x0000c343e640ecd7 }, { 0x000d50cd542a5f8c, 0x000f1f3e5eea7d82, 0x00063b79a4045592, 0x0000f8e05521c879, 0x000038b6e3d1e7a4 }}, + {{ 0x000a89dbdc1abdd9, 0x0003cd6c5a3ef169, 0x000d1f7db645ad63, 0x0009fcf1c993eff9, 0x000061650e298230 }, { 0x000c6ab8862e32eb, 0x000b3bf2b892891c, 0x000090a9ec4f324e, 0x000cab92153902b2, 0x000009cc8b82cba9 }}, + {{ 0x00084b1fe4ac70cd, 0x000d433f74cff170, 0x0003e46cf40d687a, 0x000f58eee240aaa0, 0x00005873e1125bc2 }, { 0x000ada9f07b6f730, 0x0002d16d081f6a60, 0x0008e41d8d8849a6, 0x000a77a2235304fe, 0x0000eb9e1fd7cdf3 }}, + {{ 0x0001bd2ee144dc4b, 0x000cb78968a61446, 0x000ba5f5eb3d6205, 0x000a5b0e154c1f77, 0x00002bb9da949d12 }, { 0x0001db4ebe3697d8, 0x000c9e9781105fc0, 0x000f79543fc2bbc2, 0x00067b99b9cf2971, 0x00002c5b8ddd5497 }}, + {{ 0x00065daedf9277bf, 0x0007f23b40df1cf8, 0x00030e8bc2d4eb23, 0x000253a215933737, 0x000043cb71cd34d4 }, { 0x0009699f551d880e, 0x000fdd3832791254, 0x000eeb833b56212e, 0x000ed4eafbc3d5b9, 0x00005766025c2941 }}, + {{ 0x00074ce7196c4a18, 0x00058e1d4a7a15b6, 0x000a835fa0cdac0d, 0x000e71bea08f3587, 0x0000153a1d49c1da }, { 0x0001ea48a86150df, 0x000b935088b3e0e6, 0x0008fb22fef91a3c, 0x00023c4c47a0356a, 0x0000149d050cb103 }}, + {{ 0x000a6b2421ccb0e5, 0x00092580f3e48342, 0x0001ed3e8ba96390, 0x000e982b3fdbb21c, 0x0000ca17b5625d62 }, { 0x0004efa84d6f03e8, 0x0003f944af02291c, 0x000565685cb0f5ba, 0x000eca7d7ade465c, 0x0000883813fcec37 }}, + {{ 0x00007ae2719b9b5b, 0x000c02a442b0f5c9, 0x00073f1e94c5061d, 0x0004db316b748e31, 0x000031983bdb4e33 }, { 0x0001a9c0d712eeb4, 0x0009f4b0c9bcddd6, 0x0003d8040daccc42, 0x00037d06a043fb05, 0x00006a4a48036cc5 }}, + {{ 0x000ac71e23fc88f2, 0x0008e46477184276, 0x000b03bbf167de2b, 0x000112147df17ff9, 0x0000d1d24f52c4e8 }, { 0x0008aa8296dcbae1, 0x000d8fd1defd73c7, 0x000743117c2eec10, 0x000c417ea1984065, 0x0000c69deb7b1ec1 }}, + {{ 0x000974f73d3cbc56, 0x00061f0a3738c290, 0x000ab2bd5966671d, 0x000a1f6af0a2f235, 0x0000b7892c65b907 }, { 0x0006060d7d7242b0, 0x0000ba35ba2f5a80, 0x000364edf2e9613b, 0x000948e2c542c6e6, 0x0000bb93163dd653 }}, + {{ 0x00001b3ad0550c5c, 0x00095c2065fef4a2, 0x000ddd40581378b2, 0x00064275077e60f7, 0x0000900cb66b1fd1 }, { 0x000fc0738fd99566, 0x0005ca10334fc145, 0x0008d484df668635, 0x000cfca828a9a9e0, 0x000018f7ad9443bd }}, + {{ 0x000efb14b1a6205e, 0x000c64d17d2d8c38, 0x0004e187f1d2d25c, 0x000d6dce5effc634, 0x0000567500e02c3e }, { 0x000a688b252296a0, 0x0009eee2245ae7aa, 0x000e3fa966194c20, 0x000197f5dbca4061, 0x0000512da556f80e }}, + {{ 0x000e48482682519c, 0x000f271bd3502bad, 0x0003d6962d79e50e, 0x0000f89e54ef41f2, 0x000065549732387b }, { 0x0001a43c79f96f00, 0x0008d9865cc22950, 0x00016c6aa32825aa, 0x000845fc7428a377, 0x00004147615b28b4 }}, + {{ 0x0002125ed3349bd9, 0x00097590fd3659a3, 0x000c896ccec77bc1, 0x0005dabd38927f30, 0x00001842ad4bd958 }, { 0x000b61fc67c66d39, 0x000f4ad3d4142055, 0x00028898864234e9, 0x0009d1c8173000ef, 0x00000cb713a5822a }}, + {{ 0x000568faf8627d03, 0x0008a5b7cae440c3, 0x00010ac27a395ea8, 0x00016202121c08b4, 0x0000a8232e915d7c }, { 0x00081b20b280f35d, 0x00001999ecf965b6, 0x000880fc06bcf0be, 0x000199ce2f14ed24, 0x000030dfcd06ef01 }}, + {{ 0x000461905346ffbc, 0x000b78dc6c011203, 0x00037a0f508825de, 0x000b0258c71db081, 0x00003aa89a31f12d }, { 0x00074bf462da8bad, 0x000afeadb2a3b15d, 0x000e04007c10a5d5, 0x0004726670a49a19, 0x00000c115b3d65b5 }}, + {{ 0x0005910a01f9dd2f, 0x0003e65533ef2177, 0x00044e3924858ec2, 0x0004e12677158c7e, 0x0000817fb332f9e2 }, { 0x0009f8be4c5579ed, 0x000d207cf88577d2, 0x0000ba656c829dbc, 0x0002850852224525, 0x0000a45a5a5d4127 }}, + {{ 0x0000f984312881bf, 0x000fba7076fe12bc, 0x000aed1bb3c8b51c, 0x0003c3c463f2400c, 0x00007182316ff6ef }, { 0x00040b796b1fe386, 0x00067e10170911db, 0x000a80c9381435ad, 0x0004332d27562f5b, 0x0000c44dd22a86f8 }}, + {{ 0x0008f3a69b2e986f, 0x00082973467ba92a, 0x000b4e15ae15a7c3, 0x000e0825d8c9b46d, 0x0000fc1d6d6957df }, { 0x000e56dec238f6a0, 0x000c60f61e4ce9b9, 0x0002d04f9a4a094b, 0x000a1c416bf06aab, 0x0000e7611030ccb3 }}, + {{ 0x000bb61ceb16d2d4, 0x00072f49a9898d9a, 0x000aaa7db1043ba9, 0x000cb6e97d799e77, 0x00001e9007832b66 }, { 0x0003918e6dab135c, 0x0008ca7f778443cd, 0x0000564ca17913cf, 0x000f05f6f3bb42bf, 0x000063a36f543384 }}, + {{ 0x000b7faa84dc37b4, 0x0007e719a5103f11, 0x0006d98f01816fb6, 0x000a50d195243efc, 0x00008d79992c468e }, { 0x000fc83cb87ebe8b, 0x00046bd6dfcd08ca, 0x000d5c1beb25b405, 0x00067b183129dec4, 0x0000b767a1b5e39e }}, + {{ 0x00044f601bff0a52, 0x0000e882fa6be76c, 0x000a4ed96be286e1, 0x000573bda892585f, 0x000089fc0fc5d1d0 }, { 0x00076f26dd1cd1c7, 0x000b60b2be1fdfdc, 0x000c59a459115f23, 0x000e3caa8a833bd2, 0x000032e8b64f5910 }}, + {{ 0x000c72c01889f6f7, 0x000ed7d3fdf5a517, 0x0002d9570a979390, 0x000e9616e7afb5f9, 0x0000219f402c76ff }, { 0x00053985e9b35d2b, 0x0007ffb99b5c3dc4, 0x000cf30b28c6f648, 0x0000ab982a48a0f1, 0x000083df9c5cf403 }}, + {{ 0x0008892c68df5eaf, 0x000196001b25dd8c, 0x0008330050216af6, 0x0004a5dd90dd37f4, 0x000075766028a72f }, { 0x000015de22cdfad4, 0x0001bc0ef514ca5f, 0x0008f716c0c41d51, 0x000c9395f478eb67, 0x00006c79af16b80d }}, + {{ 0x0009f0694bcc6e29, 0x00024e0eaf4b8a5d, 0x000971d5a4e6ddfc, 0x00063357068cfd9c, 0x0000e859080ab5b3 }, { 0x0003e274e3cfa9ed, 0x000ddd5ba708bcd2, 0x000937d6e7472719, 0x00016b88f5cd2ba6, 0x00005455f039ef64 }}, + {{ 0x000072e8b9f6e62c, 0x0003d2ddfd20a4a9, 0x0002f1337d994ddd, 0x000ba497d091c5fd, 0x0000a3d27ef9d712 }, { 0x000fccd127ddda11, 0x000fdb080848beca, 0x0009ca3065a453f9, 0x0001332f595bd480, 0x0000e62fb4bc3988 }}, + {{ 0x000f46369ed3f8aa, 0x0001e2d1cc1884bc, 0x00008e011d1618d7, 0x000955f9fc85ac20, 0x00002ecc593e0d91 }, { 0x0008bfe3c7e56fef, 0x000e149506467ea2, 0x00020f0e0904fd77, 0x00010253a481b63a, 0x00004ffcd9b71108 }}, + {{ 0x0002f7b360a494a2, 0x0006e090643d84a3, 0x000878d95ce09757, 0x000a592d6885f366, 0x00001f4c9f31cc53 }, { 0x000d2bd553f44765, 0x0001337dad1fe7ac, 0x00088715cb5cfc25, 0x0008d105060a157a, 0x00007397650095ee }}, + {{ 0x00076feec510b8c5, 0x000332f33e24a0b2, 0x000fcae661b4ee9f, 0x000a693259cbee23, 0x000026ff7e0a62da }, { 0x0006368fffd036a1, 0x0008ac6da10032b9, 0x000d5ab09a54017f, 0x000ea46ba718351d, 0x00008ccd45317653 }}, + {{ 0x000c8455fd5923e1, 0x0004c26150a49e46, 0x0005d2801a6da7aa, 0x0008db5d00c28b63, 0x000079a775e548ff }, { 0x00039639e7b06e8c, 0x000fd5f2d71867e2, 0x000bca63b9a5389d, 0x0008860045c0be31, 0x000038b70ff37728 }}, + {{ 0x0008d12af80f55bd, 0x00096159c7c4c6b6, 0x00006e6f6fffc941, 0x0001b75fb603aa3c, 0x0000305f59cf13f1 }, { 0x00010c71d2070171, 0x00096e7bf5d92796, 0x000549ce02fcb704, 0x0007d1bcb9aa5005, 0x0000308e0578f9b3 }}, + {{ 0x000938f757b46a0c, 0x000d5a75aa07093f, 0x000aae24ad24e9c4, 0x0007d4801ca84004, 0x0000e8c27e4df989 }, { 0x00028446eb3f9ed5, 0x000a11800a43421b, 0x0001553831dd567a, 0x000783d7639a2536, 0x0000f2b4a9ff1eca }}, + {{ 0x00086d90b771416f, 0x00012bfe3b6a6611, 0x000dce64e93fa706, 0x0005d47627751cb1, 0x00005f51d3741f54 }, { 0x0004fb8c61a5d90b, 0x000abc1dfb4f754b, 0x000a762c757a6872, 0x0007b4b4d5c5cf56, 0x000074b63f62f079 }}, + {{ 0x0004d9a14957069f, 0x000f73033ff98c71, 0x000aba3dd553a17d, 0x000f7390a9619fff, 0x0000ad3a3609acc4 }, { 0x00013dbda876d927, 0x0009c606f6df9ea2, 0x000d00515a423c7b, 0x000120d80dbcf770, 0x0000ce7619212003 }}, + {{ 0x00051b92e197fca7, 0x000790b29fc81b30, 0x000f1201f4e0144f, 0x00085039eebc3e09, 0x0000dc245aa94e7c }, { 0x00064b685717ec74, 0x0001ee9ecb5537f9, 0x000b92c10c2cc7a3, 0x000f23c1644b1b2c, 0x00008328b2510312 }}, + {{ 0x0007a54f50c83a40, 0x0005d99cec036235, 0x0005978d229986db, 0x000b408490553222, 0x00005422c6af52fd }, { 0x000d092808697dfe, 0x000441d9598b3525, 0x000c0a3802abd4af, 0x00045ab12e865a49, 0x000065df0e601cb3 }}, + {{ 0x0009d129392f3324, 0x00035084088567a3, 0x000de550212844c1, 0x000bd9d3deb6b280, 0x0000b981fcd6fb9b }, { 0x0001e32d5d00ce68, 0x000d25ebc6edfb49, 0x00016c69f3b1b20d, 0x000b846727485b1b, 0x0000e475193ea3e0 }}, + {{ 0x0006e21e7dd420a9, 0x000cbd74be59224c, 0x000baa8b377599d4, 0x000b64433ebc88cc, 0x00008d34017152f6 }, { 0x000ec23629ccb29a, 0x000ea0e5bcc36bb3, 0x000e3ee5193f6012, 0x0009ef9a89292445, 0x00003d19d936f781 }}, + {{ 0x000e29493df08894, 0x0001a85b134defa9, 0x000f7d898b37d761, 0x000db2c0d6073b1c, 0x0000020c0cac2037 }, { 0x00014b0d0b49b08d, 0x00090ea4e9acf4f7, 0x000833c2ab82797a, 0x0002a046c3e1f01c, 0x00004988c2f0debb }}, + {{ 0x000ee06e882c17f3, 0x000a0fc9920cb5e5, 0x000733e735241fbc, 0x000d49f355b44b1f, 0x0000db6fdef70b90 }, { 0x00050566058761ad, 0x000d473da621607b, 0x000be4f06c8b479d, 0x0009587ac2bb03e4, 0x000036fba157651f }}, + {{ 0x0008a673a256cc00, 0x000920ce079afee3, 0x00002ee30f7d0279, 0x0007e5ae5686e17a, 0x0000e8778c4b4660 }, { 0x00015ec10c65e1bb, 0x00006f283141c954, 0x000ecd38237a573b, 0x0000441783b2ecf4, 0x000086f1bda28a0f }}, + {{ 0x0007bf22ad35869a, 0x000aa19805033800, 0x000acb5590e4df9c, 0x0004452ae513d917, 0x0000b8de7ca6a12e }, { 0x0001fa0bdb4ffa7c, 0x0006ca1e4306839f, 0x00016bddf2cf9767, 0x000ccfcf11b3be96, 0x0000d0ade7990324 }}, + {{ 0x000b1c1f39afd405, 0x00055316ec3f510d, 0x0003f77be53bd54c, 0x0005ce6fd2c7e4a1, 0x00007db2f3ff86bf }, { 0x000930967a574854, 0x000c6755dce364a2, 0x000bdc5ec569a6e0, 0x000b64f8cb7b22a3, 0x0000915aea8087e0 }}, + {{ 0x0001bd8b5ca9eac1, 0x00048de0c061c658, 0x000c6c0961e7de05, 0x000c4f0ebcb014aa, 0x0000e471ea525791 }, { 0x0004753e714b1b3f, 0x000797ef01a17aff, 0x0003f29ac7c9d1bf, 0x000e1429321f37ac, 0x0000ccb1125704f7 }}, + {{ 0x0004fb0c0cc4be26, 0x000535ea15b0a44b, 0x0006d45d9d4b3d51, 0x000f84fb58f37c8d, 0x0000f6737b028edf }, { 0x000bc5118d042dee, 0x000f839a7ff1f2e5, 0x000305d1036ee967, 0x0004363ab9880f5a, 0x000068871fdb7590 }}, + {{ 0x0004ddb6540d8771, 0x000af554c20cea5a, 0x000d6fe72046cb52, 0x000a5e89bcac2776, 0x0000a43574dd6d54 }, { 0x0005542e3d637e0e, 0x000e3511b4941b44, 0x000429a7774ac8d1, 0x0000537e0e1ff19b, 0x000001532d232661 }}, + {{ 0x0004dcf4218a8c26, 0x000a591349f7aed5, 0x00003aed8fff670b, 0x0001ed0d83b2b5a0, 0x0000d7b1c690c699 }, { 0x0007a54456147aa0, 0x000a30727d6561e7, 0x000459d06d8bf726, 0x00005ddb5879d5ee, 0x0000236499921d85 }}, + {{ 0x0006a53ca0e071a9, 0x00056130105c0720, 0x000f873541040651, 0x000bb8cc118f7a99, 0x00002325fc847f2d }, { 0x0006bd173faf618c, 0x000401ef1aa1d9e1, 0x000433b89aa8fac0, 0x0005d13dc3241433, 0x000099c31187e85f }}, + {{ 0x0006aebf577abd9d, 0x00015452133ffd9d, 0x0000ac605ac980fc, 0x000e4582a39a8b2f, 0x00004eec5aa7265a }, { 0x000045d5cf88aa51, 0x00029cb76ccdac60, 0x000c15412957a97b, 0x000342cd1af4d36e, 0x0000e6170f039470 }}, + }, + { + /* digit=6 (1,2,..,64)*(2^{42})*G */ + {{ 0x0008c8bf5c4dddf3, 0x000e274c005979da, 0x0001c17823f45915, 0x00004f9f9c2072b4, 0x0000aaa1baf4fa40 }, { 0x0001cb9e28458fbd, 0x0002855114df14a7, 0x000d8bfa4e43521f, 0x000189718d4374a4, 0x0000a04910e166e7 }}, + {{ 0x00032caa306d36d2, 0x0007e7903605c397, 0x000e2e161f3c1bd9, 0x0005b6484f0843d9, 0x0000c3ed7e7103a9 }, { 0x0002e9423a811470, 0x000527061160170e, 0x000b6096786931d4, 0x000d933acc32788c, 0x000040d7f8af5425 }}, + {{ 0x000fa20877d18f60, 0x000739b0260e52a4, 0x000c40c32aad97d8, 0x0003b1954d42585c, 0x00002a8f892d8d50 }, { 0x0002278164a77a97, 0x00058f8e34a0bbeb, 0x000bd2cf6dffb947, 0x00034ed8356b0040, 0x0000781e1140c00a }}, + {{ 0x0001682c10b4b42c, 0x00049455922ac1c3, 0x000c73c31352d93e, 0x0002a2bc4a7b3ef5, 0x00009c11c3b203bd }, { 0x000c092172b4577e, 0x000a6f04bd55d319, 0x00057bacfaff1310, 0x0004d8a9db6d1c08, 0x0000b14a2965910e }}, + {{ 0x0004577870abf0ac, 0x000415bc00cc638c, 0x000a9cd01752318c, 0x000474cd61a2015e, 0x00001d16278617be }, { 0x00098cd105a45700, 0x0004cce2c51211a6, 0x0001bb3533930fa7, 0x00036b5e5ad9b10b, 0x0000c8cba2a94a2e }}, + {{ 0x00023c2d425da318, 0x0001507e91b536de, 0x0001bc7ddc93f16e, 0x000d9216fceb982f, 0x000065b77c65e9bb }, { 0x000b019211c4463a, 0x0001dd93f330d370, 0x00016e0efbd983c6, 0x000d362e5b3a4c8a, 0x0000e6296830cd06 }}, + {{ 0x00025fbe30039a0d, 0x0009992840bb3366, 0x000c440a3c4b2158, 0x0002fb6007c353bd, 0x0000477b5b25ccdb }, { 0x00082dd27d9f45bb, 0x000fb1adbc173281, 0x000f912383a4036b, 0x000db4446008965b, 0x000099d080f1bc26 }}, + {{ 0x000910b95bc8d4b3, 0x0009d35ea6c39976, 0x000a5950529391f9, 0x000ac2a3954259aa, 0x0000954edcb4e373 }, { 0x000982812f18224f, 0x000a61043e09aee6, 0x000f3de4d536e4c0, 0x000508c61ced56c9, 0x000097bf82337dc0 }}, + {{ 0x000ba2f548cb64d1, 0x00040c6ce8009982, 0x00028d606b905e59, 0x000e6049072bb175, 0x000008cbdb0d4e8e }, { 0x000f7d4aa9ad99b8, 0x00036bbe490ebe0c, 0x000cb030c716d24b, 0x0006a023712a0a4c, 0x0000814fb43cd83b }}, + {{ 0x00007b3c44361615, 0x00056f43fd5684c4, 0x000f3fbace5d5c99, 0x0003e55822b26eec, 0x000042b330e7ebca }, { 0x00033f9344702a99, 0x0004cfe249c82ba0, 0x00072123e10816a0, 0x00066c9729e0faec, 0x000059f4e057a424 }}, + {{ 0x000fee4cf4b36856, 0x0005ad18be82e84d, 0x000d9a69e2fa6af6, 0x00080cbfa89967f0, 0x0000ae907090e0f3 }, { 0x000caf68098cfd0f, 0x0005a65b493c465c, 0x000ba986582707f3, 0x000f34f9f638b9a1, 0x00002e36b41f5ad9 }}, + {{ 0x000fc9cdf0a602a7, 0x0000f90768fccfce, 0x000cc86be572242c, 0x000fb55cef402d37, 0x00007fc53399e03c }, { 0x00038fa2f84181fd, 0x0002e72d1669a8bc, 0x000b93e529c4e96f, 0x000b28a0e33c536b, 0x0000c1aea47311b9 }}, + {{ 0x000d9775dc6e01cc, 0x000242bbf81f3f5e, 0x000dc1e6cfa506c4, 0x0003a88ae0525a5a, 0x0000a1093d00ddd5 }, { 0x00031ec1f450f0af, 0x000c48ac0edb37d9, 0x000dd3ad5d5d822f, 0x0006f5e7625fdb6e, 0x00006a6a380a22bf }}, + {{ 0x00008d930a7b99e6, 0x000376219ee43f1f, 0x000d6ff50a8cea27, 0x000b200e85cd57c3, 0x0000c75d78e4991c }, { 0x000037762700e77d, 0x000215bc016e4eae, 0x0008319cc70493d5, 0x000889455d391683, 0x00003ba91c61b072 }}, + {{ 0x00025cc3dbf43b85, 0x00044ecaa80ba596, 0x000a05f6ca758ac6, 0x00045a3af3083411, 0x00002728e23215f4 }, { 0x000579c8e5f5af02, 0x000646c01ac3107f, 0x0006df8bb1de094a, 0x0000f5bf7b2a8d57, 0x000009b1279bb39d }}, + {{ 0x000e413c023b543e, 0x000ff931341ed7a4, 0x000c597461477da5, 0x0006486a44223272, 0x0000e62ade1b4548 }, { 0x00073540e59ce025, 0x0006c27e17e44ceb, 0x00047c7e6d15d0b9, 0x000ca71caee86bf7, 0x000062dc3b140088 }}, + {{ 0x000308c289f47083, 0x0009a7b1b4a158cc, 0x000d64d9e0df3e0f, 0x000696e97e571c99, 0x000037723337361c }, { 0x000efbce6ccabf4d, 0x00050d3eb7958f24, 0x000be10cab44dc30, 0x0001fb936a7226df, 0x000059e89431f5b4 }}, + {{ 0x00050d86fc79732f, 0x000fb058d6990216, 0x00091217dfe6fc85, 0x000a7a33a586d4c7, 0x0000a84bf24df4fc }, { 0x0004cefe9970cdc2, 0x0001257842ba251a, 0x00059d544a9a0981, 0x000dd14576b65e3e, 0x0000027b14071dc0 }}, + {{ 0x000c30477ad22c94, 0x00046909f47befbc, 0x000a7500e598e35b, 0x000c53afe6562e9f, 0x000070ac8e36df1d }, { 0x000c834e7cd5fa4f, 0x000d5610b0ffd3dd, 0x000a7eb20ad9cb68, 0x0002a9d24a1f16f3, 0x0000a6548b8dd66c }}, + {{ 0x000bd5ca4b43dca6, 0x000b875b2c69dbcc, 0x000837ee1021381f, 0x000b45c713aa77a2, 0x0000a05614cbf186 }, { 0x0006c5c5b213f9f3, 0x00065076db476cb9, 0x0001c871dd6ccdd0, 0x000210116fde15d6, 0x00006771cf226ee0 }}, + {{ 0x000f2bc1c174c5bf, 0x00084b7eb4ceb9be, 0x000f5f4b7558a096, 0x0009784af2e99300, 0x000032a2c38518f2 }, { 0x0000f8a74351233e, 0x0000d7b9342f6b2e, 0x00019b95caad4566, 0x000fc00fc201eadf, 0x0000476372345bf8 }}, + {{ 0x00015f9c975edbc2, 0x00082b6b3eba40cc, 0x0009006244398c21, 0x000c18e5ef9739f2, 0x0000d49e0d6c8302 }, { 0x000ec5b26dc91097, 0x000db8583afbe332, 0x000ed62c2e2492d3, 0x0000a0611a3b6d6a, 0x00004081b00b4bac }}, + {{ 0x00088d1c341332d0, 0x00024d902f6fb8d2, 0x00016e040530d834, 0x000bc60004063c56, 0x000024d616048d6d }, { 0x0008547775093d38, 0x00030d607806fd8f, 0x000453677d9adf3f, 0x000502e27b2f7f9d, 0x0000af367fefc5a3 }}, + {{ 0x0001dde5dfbb6e3a, 0x000b56c872b4a606, 0x0003e4356c3c10e2, 0x000cfdbd1ab2a34a, 0x0000f1935615b0dc }, { 0x000bfd9fd4818329, 0x000cd219f2275f33, 0x0006091fe9776294, 0x0002d94aa0750c8b, 0x000091bb35d3e4f7 }}, + {{ 0x0001dc0dbedb662a, 0x0001430ee9120b67, 0x0002b1c86680159b, 0x0006b239e2b51af9, 0x0000f154211bef6d }, { 0x000650c83c8fe6f4, 0x0006ed20180cca4c, 0x0006fd05c1270162, 0x00094d0441ae6067, 0x0000b656885374b3 }}, + {{ 0x000c2c8d4a51bcd4, 0x000f3ddfe7ccfc4a, 0x000c6f660ffdd205, 0x000bb6833f925c89, 0x0000a3e9f13e00b4 }, { 0x0007ece5fa9b1c9a, 0x0001a07b807bba06, 0x000d0212a9406024, 0x000d2caa49d1e362, 0x0000acd66e4a8730 }}, + {{ 0x0005a83685cf0044, 0x000659b739f10e3a, 0x000959aa229f9a00, 0x000e28c41c3341ca, 0x0000631770b3a0da }, { 0x000031413543d5f2, 0x0001a819d7de3cd3, 0x0005f0318e67af71, 0x0003afbb0edf0258, 0x0000e407d9d3c57c }}, + {{ 0x0009529ca84f2105, 0x000db2adb22b94b8, 0x0009189807b37871, 0x000fd1763993d8ae, 0x000032507bd52b7d }, { 0x0006d1b1faa44f66, 0x0007daae572a2530, 0x0007fbd06d00887f, 0x000f39d58e02b643, 0x0000ff4486cf7424 }}, + {{ 0x00026e4489446194, 0x000d932c47447335, 0x0000e53159ab1422, 0x000ad62ca40517c7, 0x0000c82158b4d5ce }, { 0x000eb5da0a8b9559, 0x00064177ee4a9765, 0x0003335d89c6f8d2, 0x000d487411d9dc91, 0x00005c05b1cb2924 }}, + {{ 0x000bea4094530df8, 0x00044b1197b994f7, 0x0004ac59aa54c901, 0x000fb86a53c95a6c, 0x0000fb4e8ab73552 }, { 0x00098a15acd05a20, 0x000ff734db378f9b, 0x000029f161ffe433, 0x000ca8d4670025b0, 0x000031ab0dd33ade }}, + {{ 0x0002618905cfd82c, 0x0000cba2743b0043, 0x000700e28ce86b04, 0x0008aa45ab1c7f4f, 0x0000bc87ba82894e }, { 0x00094df65f20b8d8, 0x00033b5a23a0f46b, 0x000ab0b39b2d0f04, 0x000bb67945bc971d, 0x000089df6cbaeb93 }}, + {{ 0x000977faf2fd74af, 0x00010abaf95894c5, 0x0006cdf3274ead88, 0x00068f58b7b9bdc0, 0x0000d28ef6376f5e }, { 0x000d37fa9f638299, 0x000a9ef052c4b5b7, 0x0000893d97515b4d, 0x0006fb5c79fe87f7, 0x0000baaee7122abf }}, + {{ 0x0002ca17164dce09, 0x0007b1a15562627c, 0x0002ff79991dd942, 0x000f87d092c01a61, 0x00000ba8174e3fdd }, { 0x000ff6218fe602a4, 0x000c82d3eb81d82a, 0x0009df77cfaee393, 0x00070103bc02abf4, 0x00003fadbfce47a9 }}, + {{ 0x0006755712b47088, 0x000ea6c8d55b9d21, 0x0009837468e85634, 0x000495f2adc0adbe, 0x0000242e92858a3b }, { 0x00077f3fc7d01a76, 0x0008384f26847f99, 0x00074833ccf00d71, 0x00057799cbcb8e38, 0x000006fa9868df81 }}, + {{ 0x000ac30885fb7702, 0x000ad2bf4babda01, 0x000ddc4e340815f7, 0x00067f326feab68b, 0x000088b297e9debc }, { 0x00080e110807e7ff, 0x0006f7b40142d9d3, 0x0002f8840b031472, 0x000b917a3839b560, 0x00007518cceff22e }}, + {{ 0x000356d404f08cea, 0x00000d858854be16, 0x000e687e009888b9, 0x0009cd8ef4111c12, 0x0000fa7a19c7e749 }, { 0x000abe5cc63d28f8, 0x000728f594d320e9, 0x000d5b93700c61e4, 0x00066e5e40989316, 0x0000820e9cacb3d2 }}, + {{ 0x00086e2220dbc255, 0x0000bfb9723f6711, 0x000d66cf371bf433, 0x000f7b19621b2a12, 0x0000a5a81f4bb948 }, { 0x000435c651e5363c, 0x00028e088312c29f, 0x000823a6b1008cf9, 0x0004fb28994282a9, 0x0000f15346f16d75 }}, + {{ 0x0000179a7dfec12f, 0x000e63943b3a50bb, 0x000b6a3c553ecb3d, 0x000ef96b77132130, 0x000049806974e5f8 }, { 0x0002dd6225d0701e, 0x00046b3c85f455b2, 0x00086c909e3c4260, 0x00005b8d94725932, 0x0000eda3e114e5e8 }}, + {{ 0x0008515b8d05849c, 0x000678eb9c3bd6dd, 0x00062368cc3bb0d3, 0x000de285d00cdb0d, 0x000009399b53aa0a }, { 0x0000a8dafe0d5d78, 0x000f19e8cbb5ecf2, 0x0008a55ffcdabe31, 0x000e088db8c4347f, 0x00001718c6220cad }}, + {{ 0x00040e94e3de442a, 0x000ea8045ee2f40e, 0x000e5d5ccbebdf33, 0x000057ec475e354a, 0x000019d840edbdaf }, { 0x000a9e01bfa34f38, 0x0006d9f62accd765, 0x000a31d826781a00, 0x0003cef7b05cf466, 0x0000fffe5d10d9a1 }}, + {{ 0x000800bf0a44ff12, 0x0006598326aad8d2, 0x00049b9638dd5846, 0x00054d092bdef495, 0x0000f18cb0c7987d }, { 0x000e5e7f141dfe5e, 0x0000f3144438fd49, 0x0002bfc720860444, 0x000f80464ef23392, 0x0000bec1282f27e5 }}, + {{ 0x0002b639eaff99d6, 0x000e16c5fcc1aba9, 0x0007c6ff860d56c4, 0x0001cfdfaea63254, 0x000005c26b5c9cab }, { 0x0003b90f531f7c6d, 0x0007a82953b135c8, 0x000dbe99599be78a, 0x0009207697308912, 0x0000c7f561f073e9 }}, + {{ 0x000f18cdffd1566d, 0x00046d0244975b3f, 0x0007127d8f7275fa, 0x000375a51f3a4435, 0x0000eee9da0674cb }, { 0x00063c9f3d509320, 0x00018332028e2472, 0x0002cd831d3c736e, 0x000f4ea8496e280e, 0x00008f205a98807f }}, + {{ 0x0005a0beee1cf41a, 0x000fbb32d2ce8114, 0x0002b2885e3319c9, 0x000abb5db846dc0c, 0x000088344d69dd5b }, { 0x000cf49ef219e334, 0x0006b0796d69af13, 0x00081ca617ead2ab, 0x000ffac9f31a7c5b, 0x00003db05bdafddf }}, + {{ 0x0001e5f7c08d641d, 0x0002ff3086cff9bc, 0x000077bd3e4ede7f, 0x0005ba01d88dbd49, 0x000011ea14de0b80 }, { 0x000c2b9818dff733, 0x0007f1428819d98c, 0x000baebf05b22550, 0x0002635c8c770dd9, 0x0000c0c2bc6fae24 }}, + {{ 0x00049dd198ff7e19, 0x000a376fe86c6072, 0x00072b8cc8fc8049, 0x000cf92636b7b933, 0x000056f429238ed1 }, { 0x0003d2fe1f767829, 0x0003c84a981a2029, 0x0007a6b6356871bd, 0x000e449093531dd1, 0x000078fe864a7151 }}, + {{ 0x000649cddbdcf87f, 0x00040e2ec6d6b975, 0x000466c0d802a52d, 0x0007f205f9cf6d6f, 0x0000cf17c30aef77 }, { 0x000497381a72d1fc, 0x00039793f26ce5a8, 0x0001d0780b03bf41, 0x000e6506fc289a10, 0x0000f143d893c000 }}, + {{ 0x00064e359de41d5c, 0x000f9e6f2606a828, 0x00010aa4c5b9d100, 0x00087877ce25f706, 0x0000a40abe928ccd }, { 0x000056538d0f8dac, 0x000bf187ce573162, 0x00001a405eb03810, 0x00063c1f72cab250, 0x000042dc7ff0d57f }}, + {{ 0x000620742019e130, 0x000565ef15d31016, 0x0009fb94dfb78dfe, 0x000f0129480c5533, 0x0000f5ea2ed57734 }, { 0x0005d06cf408388d, 0x0004cc4edc454181, 0x000eabfbdda91999, 0x0008bd5f0005b859, 0x0000bbd16a163c99 }}, + {{ 0x000378dfaf3e1b18, 0x0000cee3484be472, 0x00062693b0806164, 0x000f0ac532ff33e9, 0x00008a1b602ca81b }, { 0x000e287f8b8af0ea, 0x00036ec856720f22, 0x000b6850f3059c33, 0x000992b582ad9d95, 0x0000342dec973aeb }}, + {{ 0x000a1efa102f0077, 0x000fcd0990cecaa7, 0x0000ed3e46704a67, 0x000f05684ade55f8, 0x000011db276b92f2 }, { 0x0005cbba4dc5488d, 0x000f720037cc0618, 0x0000500883c5313c, 0x000d8fa10682e05f, 0x00004999403c4661 }}, + {{ 0x000f5f5dc0118781, 0x00016297a3cd22af, 0x000b4b0325d760e8, 0x000a9e4ed9f8cd5d, 0x00009329e15df9b0 }, { 0x0003608bc8757f53, 0x00091b5654a446d2, 0x000f4411cbf24c65, 0x000f71756d1a9496, 0x000012b5a681cc68 }}, + {{ 0x00073e00f8daf702, 0x00020d76688c4d4f, 0x00010ff939c43bae, 0x00037f8a39d6b428, 0x00007889e808b3d9 }, { 0x00072755fb51d00c, 0x00053ec49421c3e1, 0x000b26a7d07ce447, 0x00050e49fc4e44f9, 0x0000cab187f97020 }}, + {{ 0x0003ecaa12dc8429, 0x0007457446139cc3, 0x00022bf3767cde34, 0x00061aa990a0a6e0, 0x00001e7652e2cf35 }, { 0x000dcf01768fd984, 0x0000721f7c426641, 0x00028d96cf13c980, 0x00076a126b3e91b3, 0x00001f7562a57864 }}, + {{ 0x0000c3aa50085a1b, 0x0003e8f3d589e023, 0x000f811292dc489e, 0x00017b207df0478f, 0x00001d6316d96d0d }, { 0x00082736a4a1651d, 0x000e04b941ade7f2, 0x000c0a51c96c8483, 0x000da82f03d1debd, 0x000040def4c31898 }}, + {{ 0x000c4a826abdd1fd, 0x000a3c3d41d3bd3d, 0x00082b57e9743197, 0x0005ce29fb533b8c, 0x00005ac407982d1d }, { 0x00046833e45024d6, 0x00082513d590dbeb, 0x000b39e65f731bf7, 0x0008e5d7484632f6, 0x0000ca6f29d8d329 }}, + {{ 0x0001416d4f55d1cf, 0x0006a13f7fc0586d, 0x0006957dd9c9e1b7, 0x0004686dc385315f, 0x00004cf755c4445d }, { 0x00028a41409d5b93, 0x000187851047213e, 0x00010db688562158, 0x0002f8a0c447f6e6, 0x00006e8b3e82ccc6 }}, + {{ 0x000df39b884c94aa, 0x000cccb82003f867, 0x00068dda211b1bc5, 0x000101a55610d07f, 0x0000743e976e9950 }, { 0x000fa15eb2e071d1, 0x0008e96e22db2188, 0x000a7901ed91d8b3, 0x0002dc1768cc6d9e, 0x0000e81e206dd3a5 }}, + {{ 0x000ffef809be123e, 0x000fa45cf2c24b5a, 0x000ea2facbb12142, 0x0009590b389f66a9, 0x00008f9d35fbb026 }, { 0x000dbfa947e096f3, 0x000eec68eb8fac22, 0x000afaef20465dbc, 0x0000d48cb93c1e4e, 0x0000996990687d52 }}, + {{ 0x00088969f06f59c6, 0x0000b90ca84bad76, 0x000f44cf7c6c710c, 0x00071a831fb13919, 0x00002f4311cd9b82 }, { 0x00029f789c4e2b27, 0x00025e721264d452, 0x000950c7da8f86ce, 0x0001a7c9dfdfb65e, 0x0000d786a79714bb }}, + {{ 0x00006451aee0821e, 0x000013e6f49f15d1, 0x0008801ab5e3f872, 0x00024ca67678ce82, 0x00003896b89ad41c }, { 0x0009a3bf5194d5f6, 0x00001f019fc32d29, 0x000c1668938086df, 0x000c8ae4564633df, 0x0000568a458b4918 }}, + {{ 0x000bac9c2aea0873, 0x000b2a2534d4dbc4, 0x000176f66278228d, 0x00028891a45b8f1d, 0x00005f1547f876bb }, { 0x0006452e97fd7988, 0x0004f6a7b4e35876, 0x000dab3dab4b5c1d, 0x0008e693d0045cde, 0x0000d8b2227e2114 }}, + {{ 0x000e4caa4ece9077, 0x00039549f950cd0d, 0x0000f9b126264855, 0x000941f639cc72fb, 0x0000195f212e9897 }, { 0x000d4a60338b6b56, 0x0009f7a55c7ac17f, 0x00031d3b82198646, 0x000f52be7779cbd4, 0x0000760624055358 }}, + {{ 0x000e4daa4cf03ebf, 0x000a5880a750c0f3, 0x00031006e436dc3b, 0x000d2500539bacfe, 0x0000d4ef32a1291b }, { 0x000a444f665140ba, 0x0002db55ffc69ff6, 0x000c27c0e1c0d5f3, 0x000e2259446f147b, 0x0000e26f578ece85 }}, + }, + { + /* digit=7 (1,2,..,64)*(2^{49})*G */ + {{ 0x000392f2ec04cd39, 0x0000d5ac3f16ea83, 0x000d7f5ede30754d, 0x0009b6872115a461, 0x00008418a223a967 }, { 0x000fc5ff44240c84, 0x00018cde526cdc7f, 0x000b22632c3da39d, 0x000c70d988e537fd, 0x0000709b1f542582 }}, + {{ 0x0008d2c9ae428f00, 0x0004f7ea90567e6d, 0x0006dddb93095522, 0x00081903d513257b, 0x0000d1ef01808e5c }, { 0x000ffa707b56bdd1, 0x000c2246b29cb44f, 0x000c77ce5b30e21c, 0x000d3bd42b540a21, 0x0000c8f28344b9aa }}, + {{ 0x000c41d1ca8f6911, 0x000d2f1d4e353b7b, 0x00043304ded57d7b, 0x000e262fd062d8a1, 0x0000c7373014e0c7 }, { 0x000d825d0c68baec, 0x000f5e76be77800a, 0x000717e2f324cc7d, 0x000e0471a71fe8b3, 0x00007ed811a51502 }}, + {{ 0x00067b085c91370d, 0x000d35b1cb76219d, 0x00009adb7621c58b, 0x0008dbc100ec0bf9, 0x000035a1c37429d0 }, { 0x000539991832fa6f, 0x00095595e93a96a9, 0x000a66a28b826cbe, 0x000e29cb77526c1e, 0x0000acab05a94fb9 }}, + {{ 0x000fa297b54139e5, 0x00029e99e240d181, 0x000c9516acb11ffc, 0x000ab4d5c057cdca, 0x0000f108fdca1bee }, { 0x0001468d72cb1fb0, 0x000d1e7db083d762, 0x0006d0c02fa32f20, 0x000a1ef8c31993be, 0x0000ca3322492849 }}, + {{ 0x000f2a6cb5724fa2, 0x0002383a4c506ece, 0x0007e28291e1ba7c, 0x000a20ca2957ed18, 0x0000e20ed40d7da0 }, { 0x00052999a29a324e, 0x00090428e195c43b, 0x00095936b72378db, 0x000361c5dc842025, 0x00004a12b227046e }}, + {{ 0x000cae6fbba2a2e6, 0x000aab910a5a8bda, 0x00085cb9f51f374d, 0x000c4f5c1ca91d24, 0x000007845f0a029c }, { 0x000257ba0e5ed119, 0x000ade0d21411a01, 0x000fc7b290c9b692, 0x000bb673028fa787, 0x000016219da80e31 }}, + {{ 0x00096e6d076e5fb1, 0x000e067ceca9754a, 0x0005a20acc991594, 0x000e01fbf426d2cf, 0x0000e8ea47274e03 }, { 0x0003439dd70f73cd, 0x000dbdd87880e616, 0x000581c546fe37f4, 0x0004291151554381, 0x000076b608169ba2 }}, + {{ 0x000f4d069bfe9043, 0x0003d2fc14e35c23, 0x000c03f42ca0baaf, 0x00044b06271735b7, 0x000029c2e58d6818 }, { 0x000d0809671ae302, 0x000ac01f78f5a7c7, 0x000dff46e39ff2a0, 0x000fe0d8350a8ffb, 0x000048652a819f5b }}, + {{ 0x000dbc9d0aa61099, 0x0005fe1698e04cc2, 0x0002e8d368be0b0d, 0x0002bc879877be20, 0x00000faac89a81a9 }, { 0x000fe29735ce620f, 0x00040939454d9370, 0x000da49d76e54a1b, 0x000fc59e8e682ce9, 0x000091679212a247 }}, + {{ 0x0005ef72b2ec9bc7, 0x0002d0c5b9e02b75, 0x0001dffe92356668, 0x0003a9c2ba37c969, 0x00000df5f3206d48 }, { 0x000de1f2900b0c39, 0x000231f7b91b0620, 0x0007ad51f591a0e0, 0x0009308170a79e1c, 0x0000eab82efa982f }}, + {{ 0x000043d055253a89, 0x00013cf3a4434b4b, 0x00006a3584870bc7, 0x0000ee9a7a345811, 0x0000321ed180b8be }, { 0x00093e67f479fe7d, 0x000b49e11137b1a0, 0x000ee41e584a881d, 0x000ebddd400a754c, 0x0000aa7b62dca051 }}, + {{ 0x0005185e8a26f6cd, 0x00031e9c97e3f6b8, 0x000431c215d74bd7, 0x000a139b9c10bcab, 0x0000d0dce141728e }, { 0x000380e52200f33e, 0x000551f0eb6c4d46, 0x000d717975e238f3, 0x000cb40b84d4440b, 0x0000ddf18ad14c33 }}, + {{ 0x00046e79f34d63c5, 0x0003e51d598d7107, 0x000656cbd3b0d348, 0x0000e191f272c416, 0x000002b47849277d }, { 0x00013ba3f17082d1, 0x000f6364b573e9b5, 0x0006ac0fac4efdfc, 0x00016077b75d03f4, 0x0000927b63f07b9b }}, + {{ 0x000fbb3bb17bdaa1, 0x0000e26abb177dde, 0x000788d699b2fdf6, 0x000488073f82ab56, 0x0000c9b786cdf7f6 }, { 0x00048e86e3fd3f51, 0x000c0dba6d183586, 0x00074c02e656b55c, 0x000499190519e279, 0x0000636d8ce30c94 }}, + {{ 0x000be5a560218c0f, 0x0002245983c38b5e, 0x000b7795172b6411, 0x000736ce6b14f176, 0x0000b95d3a653da9 }, { 0x00090806472dd13c, 0x000e1cd9f87dc596, 0x000392d3ef194f1a, 0x00019f6577c595cd, 0x000044201d70daf9 }}, + {{ 0x000d06a5428770ef, 0x000550381e9ede3c, 0x0003fdf039c7f7bd, 0x000964c11243c389, 0x0000a436caa56e94 }, { 0x0004e52114b12630, 0x000594949d4e2e5b, 0x00069611476b77f0, 0x00082203533649d0, 0x00005fbc01d77138 }}, + {{ 0x000b45d0b53b1b49, 0x0006e0678a6513b9, 0x000a0395488ea188, 0x00052e4d892ea4a2, 0x000048a2399a61a0 }, { 0x0005c5b4d8712eaa, 0x000c6dbc69d11ae9, 0x0008b691a486a01f, 0x0004918721477dd1, 0x000074978afddc7b }}, + {{ 0x000353dc8bc4376c, 0x000906342e532b72, 0x0002778f7604f48d, 0x000ca80bc386ba42, 0x0000629c4169b70d }, { 0x0003fe03ce10af95, 0x000f0ea68f7e978d, 0x000156d8cd51895b, 0x00046f36d96ec568, 0x00001a9ad92e9ed5 }}, + {{ 0x0003a1c621b2b2f9, 0x00026a47ea67c77a, 0x000f4048baeb4ba1, 0x0004eda74952bd2a, 0x00008f88312234da }, { 0x0005059bc76aee10, 0x000c5a9c7ec6d792, 0x000f0d20277aefe2, 0x0007b501bff7038a, 0x0000444624dac251 }}, + {{ 0x000600da329b3360, 0x00005b2f2086fbfe, 0x0005a09086465f16, 0x000d4bd6cf6840ea, 0x0000778cdefbedd4 }, { 0x000017637cd7cb9e, 0x000d163529b8a80f, 0x00073d660bc129c5, 0x000992c7ce80e485, 0x000088119f4a6bfe }}, + {{ 0x0009ef732984387b, 0x0004d0676cb25661, 0x000198591fb0163d, 0x000d083458f76acb, 0x000001e530c1a29a }, { 0x000baf76b4f735a9, 0x00002715f228bf03, 0x000945c0bc22367f, 0x000763ac0f3cdea3, 0x000010c6a57a79e5 }}, + {{ 0x000ef5d295913cb7, 0x000aee010089465d, 0x0008af81f3014353, 0x0000106953bab5d2, 0x00002e287da81088 }, { 0x000de01f9c58ac00, 0x00058b9286b24756, 0x00095fa0668f5980, 0x000ed2388a0b9393, 0x0000e81acea89e97 }}, + {{ 0x000a4bf3535ca72f, 0x000398eadd70482e, 0x000990fe5b370e05, 0x000e33fef708de92, 0x000095e812192018 }, { 0x0004f0f83a164ec7, 0x000582c87912868c, 0x00052a18313ff9b5, 0x000a4bf0ab1b1be9, 0x0000846b0bf28b93 }}, + {{ 0x000cad0a1ecf03bf, 0x0000da397f6306a7, 0x000debf69a916d7b, 0x0009e19a2744c44a, 0x000096bc23c881e4 }, { 0x000db7091f94675d, 0x000e2cd33791d9ed, 0x000c96fbd74055fd, 0x00013a2a29cc2a05, 0x0000612a63e5bd69 }}, + {{ 0x000cbdfda897260d, 0x0001abf8538a5c69, 0x000cecee016f2ff8, 0x0004bd6da195c63d, 0x000048f79961dc84 }, { 0x000c5bd833e2b6f3, 0x00083ca849d7eab8, 0x000aafcb5a3dbb01, 0x00051e9eaebe2764, 0x0000a3bc8e72b00f }}, + {{ 0x0003a594c485ad1b, 0x00099f59837fc0d8, 0x000823f75b4b4716, 0x000321e909b641ba, 0x000075762faa795a }, { 0x0000344e3d60af43, 0x000021a526c45028, 0x000deaa6f1565663, 0x0004fc9cd9d39438, 0x000000de2d56a689 }}, + {{ 0x00082cc2c2ba07f1, 0x0001c7c2fa4f1264, 0x0005146cdf6fbb88, 0x0001defc308a3b86, 0x0000a19653e4fd21 }, { 0x000d3322183959bf, 0x000bf645f321bb80, 0x000b8d4ce39bfe85, 0x000325a588a93821, 0x0000226bc98df493 }}, + {{ 0x000b779c98067abe, 0x000c319e246b0e64, 0x000487d2607f1a30, 0x0009fe300b39781e, 0x0000008d964486ac }, { 0x000b59cb15ee57a6, 0x00073e1c84705003, 0x0008d0f2bb875ce7, 0x0004f15cc45b7efe, 0x0000d78e82ce74b3 }}, + {{ 0x00020a4bb98bb7ca, 0x0004adf3fd165e81, 0x0008f4153c582e9d, 0x00037d500499fd6a, 0x000043ec7a9e2734 }, { 0x0002fa487b4aec69, 0x00067955f4a5ac5d, 0x0000c2ad5a75a813, 0x0002b51f1cfd174f, 0x0000044c15223f52 }}, + {{ 0x000d3785755ef9c9, 0x000d5feb1dc77e1f, 0x00082492110d3db6, 0x000669ebbe956693, 0x0000caee16ac8c0d }, { 0x000b93cfea66af2d, 0x0003fbe81796b334, 0x00060d54afa0c4bb, 0x0006bd4b462d550f, 0x0000fc36919b3218 }}, + {{ 0x0003043896a03a07, 0x0004b381d531696e, 0x000fa5c9a5d318fa, 0x0004ca8c757201bf, 0x00003e9b1ef0bafa }, { 0x0000bf7978610e72, 0x0004fe18555970de, 0x0008447619e614fa, 0x00020318b8267dfa, 0x00002d551502ded1 }}, + {{ 0x000b228da47438a6, 0x0000528c06fece4c, 0x000dc9c526f8a5d5, 0x000350528d405991, 0x00009f6e58cf2dfc }, { 0x00076a72c60289e4, 0x0005d64107d1cea6, 0x0003698b662b1fc4, 0x000e45dcdeb9fdab, 0x00000c5b4b14c00a }}, + {{ 0x000b95b2310a2529, 0x000b4984d6cb00b1, 0x000ac298db4bf5b1, 0x000f75036cef9c5c, 0x00005f904b6ca659 }, { 0x000bb3c9580effd9, 0x000d02ee8ac25b1c, 0x000b6e64ee6ee865, 0x000ccb066ceb04cf, 0x00005b6ae05c96fb }}, + {{ 0x0000ebfbfb92eddc, 0x00095a1a43ff93c7, 0x000730f75cae09ab, 0x000ebe0825bc2db1, 0x00004d67b6663315 }, { 0x000d6c3f4e74d6e2, 0x00082c792871591c, 0x0005e6c0c2cb3b08, 0x000020e3cea5bcae, 0x00008a85be0f06b6 }}, + {{ 0x000d7b2902aa47f5, 0x00001230bca6de32, 0x0009846bddfb146e, 0x0004864d4dd20dd9, 0x0000510d80cb9b02 }, { 0x000dbc84fa3046b6, 0x0005459ce98840f7, 0x00000653e31059c2, 0x000ecc3a5a5d947e, 0x00004ffc38657061 }}, + {{ 0x000fc1dd78654cb5, 0x000241654a1aff29, 0x00099ab71e2397f4, 0x000dd445eabf318b, 0x00007215a0065fca }, { 0x000e995e35c91e4f, 0x0002a1513595c172, 0x000860c955739687, 0x0001206b0b950466, 0x00005b2b004bef36 }}, + {{ 0x000c3c9001d90f0f, 0x000c0697e9bbcfb2, 0x0006bc3106e1aecb, 0x00086985c95d0c1d, 0x0000231244ed3a54 }, { 0x000d62b88afe3771, 0x00044616c973f57b, 0x000be7bf5db81fb3, 0x000b808d4108b7e2, 0x000050a0c350cec6 }}, + {{ 0x000320c24f7e2d8c, 0x0009dc4197c75d61, 0x0002a3e6ad9c36af, 0x000c55311266a6df, 0x00003c001f9a9acd }, { 0x00013439c1f03bd2, 0x0000ee101a595966, 0x000b8ec570ea7b76, 0x0001ed4c98317c5b, 0x000050f4c2d89c48 }}, + {{ 0x00020ab732d47663, 0x000a19f16a66e918, 0x0000781056db02b9, 0x000d5c5eae97f282, 0x000094939d8bd05b }, { 0x000b2c87b6265c14, 0x0008af9287144234, 0x000a628484fc8c50, 0x0002a90cc54ecfff, 0x0000ff62b0d3b8d2 }}, + {{ 0x0007448f03eef746, 0x00087200a44295d8, 0x0000d1822140f960, 0x0009f2f7c707d7d2, 0x0000181637d8f606 }, { 0x000cf227783ae69d, 0x0001f455ebcd58d0, 0x00098430ac48d8c3, 0x00020c94279bcced, 0x0000f56a58b7cf63 }}, + {{ 0x0008df2c35144158, 0x00056e471d444b66, 0x000f5c291de84a20, 0x0009292f6211236b, 0x00002a2977e8653b }, { 0x00023b925d834202, 0x0004d0f59bf4f363, 0x000ba5b7bb161b25, 0x00007791df883669, 0x0000991591315aae }}, + {{ 0x000aa252908f15e7, 0x000884274082008c, 0x0009fda5d93a92ff, 0x000014d79effc521, 0x00002020dbc62e83 }, { 0x000f621ec39ad5ed, 0x000232a8f1b3c4d1, 0x00010e39cd49e9fc, 0x000cf59efcf64dfc, 0x0000b4beb3b9e9dd }}, + {{ 0x000023a25232bd58, 0x0007e27373386156, 0x000aa1b54ba4382f, 0x0002b1366162082a, 0x00002d55d436c6e7 }, { 0x000df2593f2a2fa6, 0x000663abbf04b89f, 0x0008b0824105cb8b, 0x000e4383b6735eb6, 0x00000bfd1da5861c }}, + {{ 0x000ec9322dec7ec3, 0x0007e4efdd236675, 0x000a9b56e3c02f10, 0x0001762ffdd52aac, 0x0000b9cbe745e9d1 }, { 0x00021cdfd65b39cd, 0x000df85a96f5294d, 0x00028f8386946139, 0x000cedf4282aa5bf, 0x0000d1d292453a63 }}, + {{ 0x0000fc5e10e7712b, 0x0008a7b67f8be10c, 0x000b96034915aa1b, 0x000c1b393365d1a7, 0x0000124aaa91f37d }, { 0x000148c2fe2e9e96, 0x00066f89cbbb1423, 0x000a7e7c43943fb3, 0x0001e18221d436e5, 0x0000b673c1d30111 }}, + {{ 0x000e85a1614b76ce, 0x0002fc9233222fa2, 0x0002b5945a0a0751, 0x000ae7b7789ff109, 0x00000859c80837d9 }, { 0x000d07e6b6010c68, 0x000ec7cdc810bc9e, 0x000eb01d956cacc5, 0x000ca499bea3f458, 0x00005b7830a10826 }}, + {{ 0x0000f30fee8c3f0a, 0x000134286d2e0f86, 0x0001ad978fe6de9b, 0x0004d1149e592012, 0x0000ed2e1530c0e4 }, { 0x0008c9d2519e343d, 0x0007006858e6a61a, 0x00069709a27ec803, 0x00054a1fbce4c776, 0x0000b72f3fb5dbd3 }}, + {{ 0x000e5b62c68d8a29, 0x000e65eb24fe1dcd, 0x000d8d188f85cb71, 0x000dade059eaf9de, 0x0000ef583c1ec4a5 }, { 0x000a60c4ef22069a, 0x00035aed3c2b36ea, 0x000458e370eb3a6c, 0x000784745868c19d, 0x0000a0000634a4bf }}, + {{ 0x00029b382fff57d0, 0x000efd23b60c472f, 0x000bf881f819bd9d, 0x000e0c5c9af4ef13, 0x00005143fbdccce1 }, { 0x0008035f45578ded, 0x000cb5373efaef69, 0x000c10c60ffac46e, 0x0008b3124cf56ac9, 0x00000797387ec422 }}, + {{ 0x000b0f588a0f9e16, 0x0005bb91e750c846, 0x000bf2b76bdfe46c, 0x0001e03c1c57f7b0, 0x000041ee2fa51260 }, { 0x0004a5ad25356120, 0x000b084a07c4e9ff, 0x0000b68230ef27de, 0x0007f6dccc383011, 0x0000c74f049cd8fe }}, + {{ 0x000840fa43472eaa, 0x00018bf490f97ca7, 0x0005ef60d3d59758, 0x000159d8d288f09c, 0x00008305396842d1 }, { 0x0005e16ea2c38bce, 0x0002fd25a631b378, 0x000e1e9b1cda4ede, 0x00090946694761f1, 0x00004a39b68d7484 }}, + {{ 0x000a2a463cbfea70, 0x000f033924910480, 0x0003eaa9bc9b0ced, 0x00080165c0c2f696, 0x0000c418f721a31f }, { 0x0001669c5553f128, 0x00073e7bf2f12ee2, 0x000245543d335cf1, 0x0003299d7f0a2200, 0x0000ed5a8309ae3d }}, + {{ 0x0003e34ebe7538d4, 0x000c40392f145296, 0x000bffb1ff34a332, 0x0002172f6d3d056c, 0x00001640598299a2 }, { 0x000c8da4b79d1b0b, 0x000e2ee5b80cfa5b, 0x00001774371a4424, 0x0008ecb67e872a12, 0x00004a32f2dc5f4b }}, + {{ 0x000ac787ee441dff, 0x000babb1b703e75a, 0x0006296bfb74f2f1, 0x000d17cff6773b18, 0x0000c7b2314a8aa8 }, { 0x000884ecd8191468, 0x000828b0437164b0, 0x00006b798c862eed, 0x000d18373f430ad2, 0x0000e6c1f3907575 }}, + {{ 0x000c5a4b66e99b1a, 0x000a6965f5206989, 0x000da89d6d7224d3, 0x000bf4eb9640631f, 0x0000b4e03e0395b9 }, { 0x000436d53d463c5d, 0x000f7bf80b6783d1, 0x000e25fb34d34e57, 0x000cddcb280e701f, 0x000018bf1fde0cd3 }}, + {{ 0x000ba1cfa6be5052, 0x0001c6d6556c3629, 0x000046a38dd836b9, 0x00021def9bc23a3c, 0x0000a35d2970adc2 }, { 0x000fc77d5dfd93f1, 0x000147a99c5d7062, 0x000b545b986b85a6, 0x000133e128be85f4, 0x00002b9e4f8b3c69 }}, + {{ 0x0001f6c57acb1b9f, 0x000eb805616ee306, 0x000f85b44b08579c, 0x000f6d1cdfb09548, 0x000025f3fb849285 }, { 0x000bf9f676805df1, 0x000ce7290aecb08b, 0x000851fd8e2e2099, 0x000512c03849f87c, 0x000043ec2b969f4f }}, + {{ 0x0003e4e910ccd797, 0x0009565765ba5436, 0x000bdf0577ccc478, 0x0006c55b318ce3ca, 0x0000fc4ec1949ba1 }, { 0x000fc62b2afeca4c, 0x00003a3c85932eeb, 0x000a8cca99222759, 0x000c004f696883e8, 0x00000968be080da2 }}, + {{ 0x000739fcc7a7b873, 0x0000a906d3e1fa9c, 0x000836cbbbdc2a72, 0x0007329011711561, 0x0000690a8b0f5f13 }, { 0x00013b9b26e41c59, 0x0003857165e439a0, 0x00026954c8b16ff2, 0x000cccea306353a6, 0x0000a667ba136d7d }}, + {{ 0x000c2af206d24a25, 0x000c73b973497f10, 0x000dfd13ca267da0, 0x00064e08315c0f94, 0x0000473115252061 }, { 0x000aa99338a37060, 0x0005df14457397cb, 0x0007101327a82eed, 0x00010f3062e8ed01, 0x0000a0cbc7d6001f }}, + {{ 0x0008fe977ed7b8cc, 0x00065823b0ec7de9, 0x000abfe06e603098, 0x0000aea831e1b822, 0x00004c9aceb8b0ce }, { 0x0004ee56e7d95e11, 0x00005c20d5e88732, 0x000a981eeb9623c5, 0x000e1b861e48d839, 0x000045fdcadc3088 }}, + {{ 0x0002d762b3177523, 0x000825b91e4de589, 0x00054d073fd57a0f, 0x00067a643a7488ca, 0x0000af2bc1467b91 }, { 0x000e0fcd863d7b0f, 0x0003b12091ff4a95, 0x0009ae9dd4edfec9, 0x0005ddbe2581113f, 0x00007099494bbef3 }}, + {{ 0x0008e7dd3da6ac7a, 0x0007068c094dbb56, 0x000e7080f6d2bb87, 0x00006862db77d062, 0x0000957c36387eb2 }, { 0x000c48a3b41f412c, 0x0002b40013a5e585, 0x0006e05598676e41, 0x000588aab2131174, 0x0000a5cfa92af35d }}, + }, + { + /* digit=8 (1,2,..,64)*(2^{56})*G */ + {{ 0x000b63b80c17b3a5, 0x0009991991724f36, 0x000bce681e7aafdd, 0x00021571e5eda799, 0x0000608a0582bb4c }, { 0x0001f37958fc7a0f, 0x000ed697e5fb5166, 0x000a6efe500b7226, 0x0002f1da5737708b, 0x0000d330af0a0615 }}, + {{ 0x0001d5f271b2ffcd, 0x000ce2caa6650728, 0x00046dfd327de7f3, 0x00027f03178322fa, 0x0000216e3f0e2310 }, { 0x00003bbfc59abb34, 0x000d842f48027f5b, 0x000bd4fb27522c72, 0x0007b690faa40cdb, 0x0000abded9d9b492 }}, + {{ 0x0002c6f30c43f226, 0x0009e180d738ded3, 0x0000ae17641d02b6, 0x000b5f0756f1a5bb, 0x0000e1022d63ad47 }, { 0x000709b3807f334a, 0x000c89dd8fffb620, 0x0007ad500b84f625, 0x000081d766a281b4, 0x0000baefeb53ba7b }}, + {{ 0x000e8143ae4b352e, 0x0008432a3326505c, 0x00021f9bfe1140f1, 0x000adfcfa38d6927, 0x000034810d2f90b4 }, { 0x0000bf0d00ef8992, 0x0002ffa7e6ab6665, 0x000fd1695563a3a5, 0x000ad66ed6cd2d22, 0x0000bbe464cbfd77 }}, + {{ 0x0000824eeec7168a, 0x000ce52877a21ec5, 0x0006bed12ba5446b, 0x000931ca2300414a, 0x0000eeb5b0f5515a }, { 0x000322d64e381de0, 0x0001cecf83cba5d5, 0x000840921d7268be, 0x000d46dd7f953814, 0x00002b466f4b1410 }}, + {{ 0x000bd6a7d881d987, 0x000ec843d9325f3b, 0x0000379b23abd56a, 0x00044afc5a9bef2d, 0x00004a3dd4f7b324 }, { 0x00032b0b1e2614cb, 0x000de028f61bc0b8, 0x000ceba5ab839425, 0x00059b798f49085e, 0x00003b2eafe5b888 }}, + {{ 0x00020cf3f9601768, 0x0005c47f1f0ced18, 0x00031285e9324320, 0x0008926fa800cc79, 0x000017299d89245e }, { 0x00019c8dbe5b7b2e, 0x00054dc1c0f7d133, 0x00005341590ca39e, 0x0007e40e3ef92196, 0x00008544b679b3ea }}, + {{ 0x0006e673cd25c857, 0x000e1717b82b99b6, 0x000ce0284257ae21, 0x000cd7d6675922d4, 0x0000134e48cf8715 }, { 0x00020fa3844dead5, 0x00092c4b5f2b89a4, 0x0005e6feb94f1d13, 0x0001f55da3de9c1e, 0x0000bf9802bd31e3 }}, + {{ 0x00024c930f786d38, 0x0009e3b0eb919b03, 0x000d74ff03109274, 0x00006f10cd74ee51, 0x00001f7f74fdc12e }, { 0x000970167dca0115, 0x0003819fbac27d45, 0x000a427dd82e631e, 0x0006685b84086e8c, 0x0000417d6e1f9dd1 }}, + {{ 0x000343e877e929af, 0x00043c9faf1fc639, 0x00099f48a33c4af7, 0x000c84f4a616d333, 0x0000b1d34afcd46e }, { 0x00039a2aff6798ee, 0x000296be970e78cb, 0x000eacb90b27effc, 0x000350e1ba36cceb, 0x000025b07b3680a2 }}, + {{ 0x000613ab7466a572, 0x000a9a16224408a4, 0x0001de6950ea5159, 0x000e65293cc856e9, 0x00009b2282c9735a }, { 0x000316fb85a186df, 0x000099cc420bf2b3, 0x00065d1c833d089f, 0x0007b80c700a5317, 0x000062412e82bddb }}, + {{ 0x0007022f7ed617cd, 0x0003df59ac50814f, 0x0007a38c12a4513a, 0x000a6f8a0fdf95e7, 0x0000db96fd7cab80 }, { 0x000a0e79e834cb0c, 0x000deb1cac1fe8ff, 0x000b3d3697805f23, 0x0009cd0e68040188, 0x000080b9a53ca2a4 }}, + {{ 0x000e74c82c8bdfad, 0x0007ffc2064cfd1e, 0x000dc1f05107db53, 0x0005c1eea339c31d, 0x0000f20f060da89e }, { 0x000e4bb978fc26e7, 0x000e95ebaf28ee6f, 0x000b9f25d6af0d00, 0x0008c1b8982cecf2, 0x0000a108e862c5e1 }}, + {{ 0x000756758aa3f827, 0x0009290640bfd9e7, 0x00070b9fab7e1763, 0x00007b656d226e27, 0x00003f7891bafa5f }, { 0x000ca7e720c3e4d0, 0x00092a35285fe91b, 0x0009cff4ea1fff46, 0x00077e008edbc546, 0x000012258488f2eb }}, + {{ 0x000c20ce094f2cf7, 0x00033b817a91cae1, 0x0004ecc1b15a91de, 0x00076e6f9b89aab6, 0x00007bae3c9d9e8c }, { 0x000d1adb3d74632f, 0x000e8b45fcf7e33a, 0x0002e7421004f735, 0x000000e19e679374, 0x0000b0e04475d7e8 }}, + {{ 0x000a6b62d93fdd41, 0x00055796424c49b9, 0x0007cd56def02492, 0x000194a49f888dfe, 0x0000f9869f29b9e5 }, { 0x000f96a765fed464, 0x0004e6d179483cb7, 0x0002157b537c82e0, 0x00066411f666f963, 0x00007c51bf707ed6 }}, + {{ 0x000f21da47afb8d4, 0x000e1bf074a30ce2, 0x0006d138589ef83f, 0x0006aaad4d7f5210, 0x000072f03502bfa0 }, { 0x000998db4eb5079d, 0x000ffb4eec863a09, 0x000cfb8e86bc3ed9, 0x000747ff8d18f77f, 0x0000d2b07489cd8b }}, + {{ 0x00012874a0a79f95, 0x000f4e5c247d44d3, 0x00010e26c8dfd693, 0x000d911c71250714, 0x00005f4b31216fa2 }, { 0x000bb7940ea2bdb7, 0x000a2a6759443709, 0x000f9f7477cf3c12, 0x0000351e3d6123d8, 0x0000f7ac71946caf }}, + {{ 0x0000e639fd11e75f, 0x000ce5984ac066c6, 0x000285a62fa36766, 0x000f23eccf5954a9, 0x000013d9c6495b3d }, { 0x00065ad32f4dc3be, 0x00043257de792488, 0x00046ad4f3a87522, 0x000779f32e9ef640, 0x0000b913b812d8dc }}, + {{ 0x0002e257466c6324, 0x0000f1b13fb70b6a, 0x0000f27743e13f9f, 0x00059e42c037b44c, 0x000046e406dc7df4 }, { 0x000140e65ebb8a31, 0x00056a81da0263ef, 0x00043bd00d7832a9, 0x000681932931bfb9, 0x0000e425b1bd11e6 }}, + {{ 0x00090be4402602f7, 0x000e891b2f1ac7a0, 0x000b6d0409e7294a, 0x000bf31e5f99eaab, 0x0000fb74eb134a23 }, { 0x000a063ac1c5c13d, 0x000bee45004f022a, 0x0001ac97e034e6d0, 0x0005c6da5d838c2c, 0x000081f282d44949 }}, + {{ 0x000cc1e4250112ec, 0x000b82259558a787, 0x00090773f60a0825, 0x0003124dc5343c66, 0x0000291b4782f302 }, { 0x000581ddda3dc5d2, 0x000823f16dfbb7d8, 0x0001d5d9f7b848db, 0x00077fe3c921fa94, 0x0000f85610bf9475 }}, + {{ 0x00047ab1f32b86a0, 0x000bfcb81d73c9e1, 0x0008f5e3e5b8290c, 0x000b3cbab6d07e97, 0x00000464d3a117a3 }, { 0x0001f4e9cbfa8244, 0x0004a3d9fafdee4f, 0x000a6195a1c9102d, 0x0006ef471fab3dbc, 0x00001448eb4ee774 }}, + {{ 0x0009c341cf277327, 0x000fdd366b33f1dd, 0x000f832103f17459, 0x00020d8613b97b81, 0x0000e530424a7a93 }, { 0x000bac5ff89d496b, 0x0003c6e16efb6f3b, 0x0009a4af35445de7, 0x000b6909fdf24b8c, 0x00008e6d72360625 }}, + {{ 0x0001088fdb965d4a, 0x000d187459afccd8, 0x0003363a9676ad62, 0x000588d8e6bd4514, 0x00005899c7ff54bf }, { 0x0000a630bc8a6a3d, 0x000682c1e269375c, 0x00036d1fbb542336, 0x000557f11f9f037c, 0x0000eb8a2a124b0a }}, + {{ 0x0009609d82cffc8f, 0x000ea7eab64c31ee, 0x000a8aefd3c18eff, 0x000a9ea0cffdb1da, 0x000041dcae34044c }, { 0x000ce67dc8a145d6, 0x000af9500d24fc90, 0x0003d839364f3a43, 0x000b1e5de9c75033, 0x000024e5e262c95d }}, + {{ 0x000982daa186bf3d, 0x000d1d44763eb501, 0x0001a82832c513b3, 0x000c80bf324a0e22, 0x000032d177c5f1dd }, { 0x0001ab688e43f0b8, 0x000f1f78bbc55499, 0x0000ed8acfc02b0f, 0x000a6c7507a1783e, 0x0000b89a79b2959a }}, + {{ 0x0007e04350f65b7d, 0x00039b56d0bdd662, 0x000e3cd6ced83cbf, 0x00020b82f0a6ef32, 0x000023dea34ebe08 }, { 0x000df290bff0bdd6, 0x000b4cdc1070c828, 0x0002f2d76e2f397e, 0x000e16129fb2034d, 0x000008903ddb617f }}, + {{ 0x00014fb49f1ee184, 0x0001b33fc2e43263, 0x000d2d8e3b92e418, 0x00083abb53b30076, 0x00001aa8b3c82fed }, { 0x000ada351697521c, 0x000d0a52f7007501, 0x00035650105f08c1, 0x000dedfb02b9e3c2, 0x00008861ad3c4117 }}, + {{ 0x000ebf2c0b49920f, 0x000ddb27b1356705, 0x000f1e3cf422fc64, 0x0003faf9fe85d19d, 0x00004ef9ee33bc0f }, { 0x000f87fda813afff, 0x000ff4eabc199c90, 0x00024732b3060181, 0x0006fed54c703259, 0x0000301ad32fa058 }}, + {{ 0x00001e4bd167db75, 0x000d1fc609e4a74a, 0x00031a4833c074c3, 0x000fcbf362a44a14, 0x00002983fab4c44c }, { 0x000f18181f25f8ed, 0x0003c3273483c908, 0x000118af9d79551f, 0x000b1d37ea79c6aa, 0x000005561a58a7c4 }}, + {{ 0x00083fd5597eef8d, 0x000467c051af62b9, 0x000a73b96f3d0664, 0x000e60da1e9a998e, 0x00006cc734074631 }, { 0x000f6191644596a8, 0x00018ca65d698049, 0x000dcf446dc950da, 0x00041ba3b732b22d, 0x0000e67d9922451a }}, + {{ 0x0001f01cf87103c7, 0x00060eeacee50437, 0x0002bcc0f2099af8, 0x000403405ae419e7, 0x00002218d3d2d918 }, { 0x000ecb8ef6d99c8f, 0x000e47e8887a132f, 0x00086765f5199df3, 0x0003eea5e6751330, 0x0000a872eae8aca4 }}, + {{ 0x0002a910af444e76, 0x000d4223021926a4, 0x00092cf23b6d15a8, 0x000dc5aabb7c28a4, 0x00004beebfd8904c }, { 0x0002c1c510c876a0, 0x00026e07bd68cab6, 0x0008141545b76fcc, 0x000dd0aeb2b57fa3, 0x000028f01be29773 }}, + {{ 0x000fb2f0f13fe534, 0x0008dfbb2d09dc72, 0x0001f6b8e14a4dfd, 0x000cd49b6c3dfed1, 0x0000cba2af46aa95 }, { 0x000b67125e538c1f, 0x000817b80be73992, 0x000df438154aa4c9, 0x0000fbff7ed9327c, 0x0000e6c8e2a6691f }}, + {{ 0x00002e29099045b3, 0x00073e88670ced73, 0x000f92edaaaae805, 0x00082a9bce3dfe41, 0x0000a6fe4cc3fd2c }, { 0x0005bdd6a63370ab, 0x0001411a6eac16e3, 0x0000746d6406529a, 0x0009a52ab7644700, 0x00006744b2717351 }}, + {{ 0x00010c6b129788b3, 0x000e1b07e5c9ce97, 0x0002afa2c95c509c, 0x000dcfa4a0f379e5, 0x000038870779d124 }, { 0x000e2ff99da77528, 0x0003ef971954cf1a, 0x00025edaa877aa54, 0x00047fba8b1cec79, 0x000023185a87ebdf }}, + {{ 0x0006bf94a9ce8a49, 0x0004c3c340ceaa2d, 0x0004576efe8a0a53, 0x000fca76d2635527, 0x0000ef0d9f9a4fb6 }, { 0x000c6f98643cdd39, 0x000610ab76a9af0c, 0x0002ed99c292f6e8, 0x0005b63a052740ab, 0x00002f34b27f25e7 }}, + {{ 0x0001d6152495a6cb, 0x00076a76b2515cf3, 0x000163afb2252000, 0x000dd139771c4160, 0x000019dc05c32b50 }, { 0x000c06a0b6aebb7e, 0x00047e1fbce090b8, 0x000d6fc489a4cb51, 0x0001cce35aac927e, 0x0000dd0bd5bb9272 }}, + {{ 0x00040304b2f32575, 0x000608bafe7ddf39, 0x000d2de3e0a8e028, 0x000e121e755851fe, 0x000012305bc663a2 }, { 0x0009215b9ac0aa93, 0x0006561b0a2a8600, 0x0002055b6de6ae99, 0x0000bb796cf12128, 0x0000d6c9a199774a }}, + {{ 0x00001d3c3571f271, 0x000420537387c09e, 0x000e1021ab2abf2c, 0x000057919c5832fc, 0x000085c20ac422c1 }, { 0x0003afcdec0503c2, 0x0002c41da48ea80e, 0x000deb9fd6d517a2, 0x000ee1fc734e71e8, 0x000090527eb13975 }}, + {{ 0x00010dd399eaf33a, 0x0004ad700235e9a6, 0x00042b3eafd5b269, 0x0003ff3e50308d3f, 0x0000894a8bd16a9e }, { 0x0009738198fc9a32, 0x0007260d1ac16352, 0x000af72f3f46fc54, 0x000b016c584cbb3f, 0x0000e913d664efea }}, + {{ 0x000dd645686c5987, 0x000ed7fa6d74081e, 0x000467cc1cabbdbd, 0x0002f3efe60ea4c0, 0x0000d3d1d48d4bb3 }, { 0x00069876a93ccb94, 0x00037ec96460b259, 0x000dd2439ab5990b, 0x000acaceaa12fd0c, 0x0000da6bef5c292c }}, + {{ 0x00001c7318116168, 0x000fa17472ff5805, 0x0004fab7a07b4f61, 0x000da2392f422970, 0x00000e0d1e93d88e }, { 0x0001e184b20a7a54, 0x000b5b318464945d, 0x000ba2e4420c0b15, 0x0009fe2606d5e594, 0x0000d365f13f5f27 }}, + {{ 0x0006362953e25a5b, 0x000ec524c6a7e046, 0x000adf1ee2117945, 0x000664ea547fa1e2, 0x00008a103e0c9c66 }, { 0x000e3f0859cdc2f8, 0x0004cb15cb12a88b, 0x00096d25592eff44, 0x000ffc8c0561b6f9, 0x0000f36918054b98 }}, + {{ 0x000ab5b320711223, 0x000a2beb92041b8f, 0x00062b91c9ed202f, 0x0001a579401ae38c, 0x0000fb40e7e1d360 }, { 0x000f7ad0829dc074, 0x0000092487443e97, 0x0000199242c6d8ee, 0x0008596378595a31, 0x00001217219125df }}, + {{ 0x000ab2609e227e6c, 0x000e1b61059705ab, 0x000a2c0765f9ca9e, 0x000351fd5a3dd492, 0x000054b35ef99c4c }, { 0x000517ab66adb516, 0x000e8896e70ee822, 0x000deecf0c7cd20f, 0x0001d59caeff2c56, 0x0000ac23b6003787 }}, + {{ 0x00021487098d3893, 0x00021e0bdefdd4f1, 0x0005e1525cf8642f, 0x000032bc06995846, 0x00002ad5c774a21a }, { 0x00067f595255747a, 0x000b83318969006d, 0x000e0b7efa7ee217, 0x00097c961cb4d73c, 0x00007cde45c8bbbd }}, + {{ 0x000e56937bdc883b, 0x000a71e9e70c72e5, 0x0008f7103f8014bd, 0x0009b5cf5b2e52e2, 0x000059c31c61ce26 }, { 0x0003442dae76f13a, 0x000bb17456c61e10, 0x000aa56c75d9f1eb, 0x000b77d55bc2e954, 0x00008d57023bf8e9 }}, + {{ 0x000621dbccc0ef22, 0x000c547ceb26210c, 0x0007b259d32c7710, 0x000b4bc31983545e, 0x0000a78659cb68c5 }, { 0x0000cb3b3dddc4f9, 0x00049e959266fbbf, 0x000642d20bdea51f, 0x00008b496a3514c3, 0x00002d270b753385 }}, + {{ 0x0008c2bf83f14439, 0x000a2ce478e2135c, 0x000ef4e093be4e87, 0x0002341f67547b5c, 0x00000ad1fffb6d99 }, { 0x000c1d8afc50c539, 0x000b1af455809854, 0x00057d1dbacf8e42, 0x000b613acc68d1f0, 0x0000821512510276 }}, + {{ 0x0007e0606f082f40, 0x00083fd9eb45ab26, 0x000b6cd1ce05d312, 0x000e6558a5b234b5, 0x00000f7a741d7de9 }, { 0x0003ddf9a8ed377e, 0x0007270ef751b115, 0x0005a3127425be84, 0x0000278b4f352f17, 0x000037791e3b067b }}, + {{ 0x00051e4e6b8cb501, 0x000c279c21520b0e, 0x000ff6b7834b4d94, 0x00094c6f790864fe, 0x000042d69c0fb7d2 }, { 0x000d60d912391757, 0x0004f7f5c9dc340a, 0x00070f46ed0e8a4a, 0x000afffd05ad53c3, 0x00002565d3e375f9 }}, + {{ 0x000effdf0c11cd5f, 0x0001cdf0eeec8c68, 0x0000788a3915c4f7, 0x0003c36244a27fec, 0x00006b8aecf409b1 }, { 0x000bb395927f21d8, 0x000141c65842df85, 0x00087766136f5c4f, 0x00020dd298821b35, 0x000018965c995c5f }}, + {{ 0x000dee4dc0194e60, 0x000d316fd6f7140f, 0x000abfa03437b7c9, 0x000b47962f9fadb5, 0x0000e38b7d704c40 }, { 0x00062cdef4e7d515, 0x0005f0735d7620e5, 0x00039e5f211dd6fc, 0x000802f9a04e3c2c, 0x0000b9308ad3b24c }}, + {{ 0x00048abb8f5d2a2e, 0x000fc5f477010973, 0x0008cafe1533323d, 0x0009158dd617125d, 0x000015bd2f4c94fa }, { 0x0006588b45797dda, 0x00010b599d94bbd6, 0x000de02c461b7193, 0x0004b1a1ad51ba28, 0x00008fbf0633d6b7 }}, + {{ 0x000970de6f2beb58, 0x0006701fad097a5a, 0x0007b79da2ede893, 0x00026865649883ea, 0x0000dc66c7da4b5e }, { 0x000820bb894ec5ea, 0x0002a245e17fc196, 0x0006990084cddd1e, 0x00078883fd833c65, 0x000023fa1aafb1e7 }}, + {{ 0x000baeb23e56969d, 0x000cc70d875d56ae, 0x000b664d448127e8, 0x000719f12d6a0a9a, 0x0000a1bfa272524c }, { 0x000a8a927cea79d8, 0x000843ab99a907a9, 0x000f67e1618ebdee, 0x00088f59ef1e7454, 0x0000b1cb62e5aadd }}, + {{ 0x000e3f57dc2d6e51, 0x000a50c08191224d, 0x000b000f68b5b448, 0x000753c84c4126eb, 0x000061310eee001d }, { 0x000304ef0417f39d, 0x000a637d446a1dda, 0x000fe29988f21ca2, 0x0002976f57bf2179, 0x00002b04dfb32376 }}, + {{ 0x00051e67299da7b9, 0x000a547d19dfa5a2, 0x00035a8f31025ec7, 0x0008f5cf5db007de, 0x0000c7a0a9aca7c3 }, { 0x0004743570466a59, 0x000f8841ae98e78c, 0x000c62a876c22c05, 0x00089b7f5f818537, 0x0000034a977f0e71 }}, + {{ 0x0003e514e5a4fbe0, 0x000fc5ac242316b4, 0x000c980f9b0feddf, 0x0009eaec6abe060a, 0x000079cfa30418eb }, { 0x000756c3e6d3c154, 0x000a63807d784f90, 0x0000ed5eef704cb7, 0x000ec963d1161a88, 0x0000f21eee5e923f }}, + {{ 0x000fac0dd10881c6, 0x0003801aaa79697b, 0x0008968d0fe0583b, 0x000197fa004a73b2, 0x0000aa2663657610 }, { 0x00019b3e879170f7, 0x00043f57d3d2ec27, 0x000caee09527a33a, 0x000e416ed6fa2fc2, 0x00009abfc9b90ad4 }}, + {{ 0x0007050b07a66baa, 0x000fa966d17fbc7e, 0x00098ccda746ff07, 0x000fbbf370cd1a70, 0x000027f44964c30b }, { 0x000e2646e5bb924d, 0x000fbb61ede9046d, 0x0000b93fdf744056, 0x0006ed7c5a053005, 0x0000ad3637db0835 }}, + {{ 0x00054660dbc8b1ab, 0x000d6088cad38c0b, 0x000fb6902a066a7d, 0x000dc7ca3471b7e8, 0x0000e86a1789d644 }, { 0x00095173a8f8b220, 0x000fe11a6fd8de44, 0x000a5a2626461987, 0x000b8a17c44dbffe, 0x000097f4a165ced5 }}, + }, + { + /* digit=9 (1,2,..,64)*(2^{63})*G */ + {{ 0x00013e2c380a9bf2, 0x000364d391c2a82a, 0x00027444dcb0941f, 0x00055fbcabcdd486, 0x0000ff69ccdcb479 }, { 0x00052b540c86a1b2, 0x000a4f04a057a87a, 0x000001893b8ba415, 0x000c36a44a876550, 0x00000e23c958c1e1 }}, + {{ 0x000f5e69622c79cf, 0x00060c516a0d2bb2, 0x000beb03cb372318, 0x0004c5d8b0d5cc16, 0x0000e18f5d34d82b }, { 0x000648f91783b808, 0x000bf0bf5a01797f, 0x000c6a4c687ad85d, 0x00067d3523d20b44, 0x00009192002a1f83 }}, + {{ 0x00028c5bd0e3d52e, 0x0007eafe60b7cf78, 0x0000021d7c9889ec, 0x000e55cc425860a5, 0x00006eb845413903 }, { 0x000c4c0a359d0286, 0x000bd5edc55ae5f2, 0x000c93c569dc0f68, 0x00021ae2ca527f56, 0x000043058b802960 }}, + {{ 0x0003271e6c831d51, 0x00077b8fac61d9a1, 0x0002d09fe01d7f5f, 0x000a2a4037d25e06, 0x000064507e385362 }, { 0x000ea6041214d255, 0x00051c0232f76a55, 0x000d1178f014410a, 0x00037fc788957c32, 0x000085a49edcb98c }}, + {{ 0x00060cf82fa14b89, 0x0000d4ed0b8892be, 0x0002acf4c3455b03, 0x00078fd269ea1768, 0x0000311da4567f5b }, { 0x00005c27534d6570, 0x0008797d4df55311, 0x00017d92c88f0a48, 0x000cafeb1235b59a, 0x0000809db71804f1 }}, + {{ 0x000f3338ee1c0299, 0x000f84bb5def9961, 0x0003040a98f9248d, 0x0003ac5761cb69c8, 0x000000beb921e57f }, { 0x00066c038dfaec8c, 0x000d10d2f5378a8d, 0x00065894c993b6fc, 0x000735690acf4c2c, 0x00003b9318cab2f0 }}, + {{ 0x00040461e09c5a97, 0x0008f4af94d70cf7, 0x000eff105d9de3f2, 0x000d95602bdb802e, 0x0000b9b80b8301a6 }, { 0x0003d814bf093a24, 0x0008ec8365e9383a, 0x00029c06e19337d9, 0x0007f32c1ea762c8, 0x000062b44e7d6f3e }}, + {{ 0x000e0341bd6ee336, 0x000102bb4d8bc50a, 0x0005fb1615b32bf2, 0x0005df468181c0b1, 0x000049acb4f25f09 }, { 0x0002c9d27e3d254d, 0x000f8cf7003e8666, 0x000a275ad19d7996, 0x00026d7c2f11a6de, 0x0000e62e628f9c93 }}, + {{ 0x000b4e10567aa138, 0x0000a3588cceff67, 0x000f0fb9dd12b3f5, 0x000abd50d8594c54, 0x0000be14cc0ee64d }, { 0x00086ed7a832f320, 0x000a0d5b5ec995a1, 0x0004fbb9c2ac7055, 0x00008b777bf4b9d5, 0x0000c5b6259dfe26 }}, + {{ 0x000a5ff674823c2b, 0x0000030614c09004, 0x000172a0cbd7525d, 0x0008894a1da98d12, 0x000030b72f8253fc }, { 0x000c72e27f2e7888, 0x0000eb81d7ea1d7e, 0x00029433ef893a49, 0x00053356e38cf289, 0x0000353f5c728ab5 }}, + {{ 0x0004cde963d2fb2c, 0x0003cf5a44f77f7a, 0x000d0f6e695842ef, 0x000f5ba097aa5f9f, 0x00008eb52caa0bc1 }, { 0x000239c14011b93e, 0x00062c0eee470afa, 0x000de8059d887af6, 0x000e9d43184137d0, 0x00000f5b3380d1e8 }}, + {{ 0x000b6b6e1ca760d8, 0x0007f19a8afd30bb, 0x0007cf16ffb18b25, 0x0004823e08679832, 0x0000efcafa52751d }, { 0x0002beaaf58337ec, 0x0006f11471f1ff0f, 0x0008cb570ffa8731, 0x0005ed83ed76c338, 0x00004c92081ca03b }}, + {{ 0x000c9c96430e9a0b, 0x000a59ec3076a27d, 0x000e0779be1a0693, 0x000d3642e5e82908, 0x000023d4c1872f26 }, { 0x000b6eb671cf9704, 0x000b48d519bf6156, 0x00059a3370dc9737, 0x000c72ba0e29ecd7, 0x0000d89e2cc152ef }}, + {{ 0x0000aaee181a2a7a, 0x000ff32b88756ddc, 0x000230d35049ce8e, 0x000b74aa4ed4e865, 0x0000261c5ca08124 }, { 0x000d55e8ca475a13, 0x000dab96c168af3a, 0x000f0644ffaace6a, 0x00053a1366563c7b, 0x00008a47ff05a4d9 }}, + {{ 0x000813e66abf3deb, 0x00033d8b76da1f56, 0x00063d76e529d81c, 0x000c8178210dfa46, 0x000088e501fc78ec }, { 0x000d7ee53868e5f5, 0x000b91e350cb901c, 0x0005834c92ea4be6, 0x000c54687b653d65, 0x000007ddaf56d876 }}, + {{ 0x000f835f9d0aad8a, 0x000272e523b8bf60, 0x000b800dac7dffa3, 0x00068eb888009eb5, 0x00005af05fd9e53f }, { 0x00029d4c6fdd401d, 0x0000c312afceb620, 0x000a13ffebd0185b, 0x0006be5b0c797df2, 0x00003ac5e2304089 }}, + {{ 0x00033878925685fc, 0x0009fc1afab4bc58, 0x000ed33fb9b57d7f, 0x000a73a05833f5c6, 0x0000c58881e99e36 }, { 0x0001b7f21a55c701, 0x000b20fadd9419a0, 0x0009ae2c0ace83ac, 0x00042a9c6bcca109, 0x00002b5c2691fab2 }}, + {{ 0x000ce9141bcb3848, 0x0004a19909523c8b, 0x000ee6b8b3259a8a, 0x000919d5aa62f648, 0x00009596a8c13395 }, { 0x000520dbd9ba1d54, 0x000298435f9b9887, 0x0001bce399dce0dc, 0x00073f36c0210da6, 0x00000d8103e648d7 }}, + {{ 0x000a389397c74a91, 0x0009bfec1ab1bbdd, 0x000e022c75a0a320, 0x00012eac4ef8a65b, 0x0000945f9752a8b9 }, { 0x000b8d4c407c4937, 0x000863674660876b, 0x0003f8d0b2f52d8c, 0x000aca39411d67c5, 0x0000712bb3bc0828 }}, + {{ 0x000ecf1276d09b96, 0x00055d1e0a1b12ac, 0x000ca83c96e8b70c, 0x0008f53cb1e4ef88, 0x0000798dffb7e60b }, { 0x00068fc88f16a70b, 0x000f62e1b39b80f3, 0x000cff33731551d4, 0x000ef56c2fd90d0a, 0x00002d84069cd845 }}, + {{ 0x000203d2c908ee78, 0x000ef61196a2fa2a, 0x00071b575e65ca1d, 0x00006d231931b981, 0x00000a20eba5436a }, { 0x000781b3c4ff64b0, 0x0002ee098939db32, 0x000c16deb83df347, 0x000bb4643e37d6c2, 0x000068ca7511af52 }}, + {{ 0x00077057e537e323, 0x000ddb07850ec06e, 0x000895c9b8ea2e5f, 0x00067909dac5a38b, 0x00009de2a5e3a8ab }, { 0x000620a4c251a5ac, 0x0001442b59e78df5, 0x0007c77739906b4b, 0x0003b1e4a4cfc91e, 0x00007e41e6a9a9ed }}, + {{ 0x000e5ae52c223602, 0x000d2aa049f842cb, 0x000456e602ff290e, 0x00090ef56ceabc5d, 0x00009d94422dd0f3 }, { 0x000f78ebd3835c75, 0x00072e6334957f10, 0x0000219b5e9c8395, 0x00039cf7b57e388f, 0x0000e66f4d445c5c }}, + {{ 0x00080ff42e6bc91f, 0x00007052b7ce5421, 0x00095c62c9b34c59, 0x00009d5dae6d4ce9, 0x00005ca79a998f9e }, { 0x0005aebf80664f4a, 0x00082022c34a1c65, 0x0008ec7d87425f4c, 0x000c55ea23803d6f, 0x0000a8295081cef5 }}, + {{ 0x000d185891700399, 0x000e4587135593fc, 0x00065ecd183bf605, 0x00088ed0d32e6eff, 0x0000f3fb39aa7eba }, { 0x0003c612df40b960, 0x000899d6d97e1340, 0x0000d9be34ef3c31, 0x0000ba273d42c6b7, 0x0000fd96d1086b78 }}, + {{ 0x000bf546da5d876f, 0x0005b1504b6c5a08, 0x00088cfb9cc92732, 0x00010fd0035216f3, 0x0000d81054fae260 }, { 0x00001d5cefaf619b, 0x000baa44cd623c3e, 0x0007700e316b2264, 0x0003f24e7366ce71, 0x00007cfc40c2c53f }}, + {{ 0x0005eccae559fb71, 0x000d4d9036520f83, 0x00015ce743b334c4, 0x0005a2904dfb3c3d, 0x0000962fbaf2e972 }, { 0x0007e09d0efe288f, 0x000a3dbf8a5a0ac5, 0x00059388f9f119e2, 0x000fe2da0547972a, 0x00006949be49f2cb }}, + {{ 0x000564c83f9203e1, 0x000b60e2fdd23cc0, 0x000c511e21322752, 0x000e48dbcb80288b, 0x0000797385f1df08 }, { 0x00012d656be5c3a7, 0x0008ba239560368e, 0x000a236ec3766f55, 0x000ddb1c3e893752, 0x0000ce0f9a1079c0 }}, + {{ 0x0003f375576cb563, 0x000f24e63e3bc1da, 0x0001a110da9fde91, 0x000d84707c719b6a, 0x00001a17e6c0b377 }, { 0x0002b1aa240c1a70, 0x0004f760700d3607, 0x00040f793abe8e2d, 0x000d78b4cbb1a182, 0x000018bc4b3b7f7d }}, + {{ 0x000b0c591bd83326, 0x000c4cabe063f644, 0x000371d820edcf2c, 0x0009d54ca9b392f4, 0x00002b614ae1ed42 }, { 0x000be79af8c567bc, 0x00021fd92f1169ae, 0x000b60d7889967db, 0x00029e6af63aeb1a, 0x0000d1f76a097e41 }}, + {{ 0x000fe8bd2ff0d26a, 0x000c04e40e50acf6, 0x00073f191935df0c, 0x0005f541ff0a98ad, 0x0000d0709fed8f27 }, { 0x00066466bcac1668, 0x0009ba16cafbc916, 0x00091ca479cfe8aa, 0x0006a974f170f875, 0x0000aea394d3eeeb }}, + {{ 0x0000b1d093f471bb, 0x0006510f5b343bcf, 0x000a023f726a643d, 0x000ed90df0f2e400, 0x00003c3843aa4ab3 }, { 0x0000bb7742e67add, 0x0003e95c70816030, 0x00093cfe99544825, 0x000be16a6e943e4c, 0x0000792d7df86acd }}, + {{ 0x0003a9e4817dcf52, 0x0001e83824297b45, 0x000001da60dd959c, 0x000f8849e9d0e558, 0x0000f7d56e3c5b73 }, { 0x0001d44293968f25, 0x0008be065eb24132, 0x00046defc5136d73, 0x000f02f92dadbbb8, 0x0000281b17b810f6 }}, + {{ 0x0002077086307267, 0x00015e63b5f1cc41, 0x000abc7fbddc361f, 0x000938b4437ae52e, 0x0000d5d97371ebf8 }, { 0x000fc8b47bfe2c04, 0x0004a491058edeaf, 0x0002b67015a14d38, 0x0000c551c827510e, 0x0000de8b56068862 }}, + {{ 0x000270352a539749, 0x000d7e61b4d5e634, 0x00025b25e620e74a, 0x0003a22edbd86474, 0x0000a57233245448 }, { 0x000a08d537795ccb, 0x00034e664cfc50e9, 0x0009842df60b03bb, 0x0007f949a50dee01, 0x000062e5de6beabc }}, + {{ 0x000ff7a5b46ad8be, 0x00056576be5f7de2, 0x0008922e96e9e3fe, 0x000f20d36d93006f, 0x0000601b5481985f }, { 0x000c615b703d9998, 0x000bbf1cd871236b, 0x000d064911e1ff34, 0x000ff3f6df156a39, 0x0000a91dfe2f0173 }}, + {{ 0x000cde4adff641f7, 0x00041e8de3eee6b3, 0x00086eb2c2b1af03, 0x0009081bd58c7753, 0x00008deb4969dc62 }, { 0x0001141f046d8047, 0x000d8d25302169cc, 0x000fe875ce874407, 0x0001659ee8ca60f9, 0x0000414e05bc5492 }}, + {{ 0x00090cfd9375a854, 0x000ec8d6dfafed30, 0x0003305917f500ce, 0x000ca1ba033a0165, 0x00006da9376c5895 }, { 0x000e293b24e6645a, 0x000f1aea3c40af51, 0x000bb3e7eeccfcea, 0x000bc22e83469ead, 0x00001e948c862537 }}, + {{ 0x00073554cc7c12ee, 0x0004f131e6c0bb48, 0x0005094218689b04, 0x000d40b221900469, 0x0000eedc75a767f1 }, { 0x000c08d7b45cd697, 0x0002f27cbf18a512, 0x000b0429e8cca640, 0x0006fec4d4d93651, 0x000034514ce6bcdc }}, + {{ 0x0007f5e2fb1758d8, 0x00077b486df2a61b, 0x00073501d1a6fc21, 0x000776affa1ed9c0, 0x0000e7124755e764 }, { 0x0003bc919d78f96e, 0x0007e6bcc2423b79, 0x000f26a75d0a01ad, 0x0005738e175ce0a7, 0x00003c4ce13c1479 }}, + {{ 0x0003e291023c1b3b, 0x000ba6f871942dfe, 0x000e94771c645656, 0x000afdf782372ff6, 0x000084b784d593f7 }, { 0x00061a89634cbefb, 0x0001b11e4e8110ce, 0x00044ce70f6b5b2e, 0x00061f4e02a2d70d, 0x0000222495a4851a }}, + {{ 0x0006cd7685d97af3, 0x0004785825808bd3, 0x00070909727c9ff2, 0x000e726d151dc3cb, 0x0000b806e9f304b7 }, { 0x000deb57a092fbd7, 0x000c605b22922a44, 0x000e8db73ec050df, 0x000d714c76494c87, 0x00002e35bd113f29 }}, + {{ 0x0009ed0eda00c05d, 0x000c259c3d12a732, 0x000c7bd5b6434166, 0x000b91ecb5f71687, 0x00008aa1f29eec04 }, { 0x0003fa2fed9642f5, 0x00066474683c2eb9, 0x0005034a4c594fbb, 0x000dd7c674956eeb, 0x0000dc5d5eff55da }}, + {{ 0x0001e6e6273a9971, 0x00099f231f63950e, 0x000032e3bca22cf9, 0x000130d62a77797c, 0x000012ab210bc6ed }, { 0x000864d0edf2e53f, 0x00039b0b3c9fef02, 0x000044fabfd0707c, 0x000e912cfaf7752e, 0x00000d5f6d1d2afb }}, + {{ 0x00024de3470dc37d, 0x000668397c69134a, 0x000648db2f4c601f, 0x0009d3de05a42c31, 0x00001832ed2bd25c }, { 0x000033860bde0c47, 0x0002daab75d49411, 0x00061d1c99e46bc7, 0x0006d2dcf9890058, 0x0000216cef95030d }}, + {{ 0x0006acbb473c0f8d, 0x0000dc82094cec3b, 0x0007898ebbc57ab7, 0x00049cc0d9976fb8, 0x000043a34cd4250c }, { 0x000d91996754db45, 0x0007f6144569d857, 0x00061a5eefb627fb, 0x000fa5c7dc3190a3, 0x00002f9e3edd2df3 }}, + {{ 0x00005c3197c1a20c, 0x0003c37dd1c8a206, 0x0007cf0c0281a36e, 0x000f1f62e7e5410c, 0x000059f6d21710fc }, { 0x00077db2a94dd7cd, 0x000a80101683d546, 0x0004f1dd5f8af1e8, 0x000a95a7f1d7127c, 0x000083f71312d346 }}, + {{ 0x00054d652ad70ca6, 0x00026aaddce33450, 0x0006fbdcde261523, 0x0004ce5dc2a6811b, 0x000053683550c6d5 }, { 0x000c91c825e881fb, 0x00056d492a210056, 0x0000c67f47b9218f, 0x000de150fad7f397, 0x0000880885c56630 }}, + {{ 0x00093e4b2fa0fdfa, 0x000b8a2ce309f06b, 0x000910e8d51c27ca, 0x0004ae92efdb27b5, 0x00004db9e3997567 }, { 0x000141c1e6fe9e0b, 0x0007fe644aa3dedf, 0x0009ced4138cc692, 0x000f98c07317a666, 0x00003ed5e5173c1b }}, + {{ 0x00060318b442f800, 0x0007de5619063736, 0x000e85967ce17243, 0x000a65c36858dbad, 0x0000f669b02efe62 }, { 0x000465311b7e9e0e, 0x0000ecd12ba4b7aa, 0x000b5462f70f3e78, 0x00014050df399daa, 0x000097efc0401f70 }}, + {{ 0x000096a8921179a4, 0x0004ac864d3779b8, 0x00062cce18df4dfc, 0x00017a0283c5d047, 0x0000bb1823fedf19 }, { 0x00068eded5c79181, 0x0002d9d23b3ebc2e, 0x000abef1f1010c41, 0x0001e9e6c90bdd6d, 0x0000b133de5a1dfe }}, + {{ 0x000990b4e4f167a4, 0x000b77799eb6df0e, 0x000089f602942610, 0x000b0008e26c3cc5, 0x0000153c35c8a372 }, { 0x000a4c3313da4eec, 0x0007f7e2586abba3, 0x00051f7d3b8634f6, 0x0009c5606239ca6a, 0x00001e8329fe8908 }}, + {{ 0x000c7bfad0211361, 0x000e706f6a3f6fbe, 0x000e9946570d08fd, 0x0009423d3ad5cad2, 0x0000a4c7595c43d4 }, { 0x00060c43d2476796, 0x00037333e23cb3ba, 0x00041f9a075abd4c, 0x000f0fb063979fe8, 0x000050ba89794e36 }}, + {{ 0x00004ca6d9af362f, 0x0006182be7cf3a64, 0x00027c49cb9a7e78, 0x0006a79a9cc86ba9, 0x00008d6173b50eff }, { 0x0000b23616d2a4f4, 0x000a85d005921b83, 0x0001143565e32ad9, 0x000d0e9c2f56297f, 0x00009e435672493a }}, + {{ 0x000a3e7d01e42a4d, 0x000c021bfe50e2b2, 0x0000b78170df66dc, 0x00081fde07ef8cfd, 0x00003567a5d00f46 }, { 0x00087981154ff114, 0x00063ece38d9d1cc, 0x000dd3e3acce4961, 0x0004b94c9b500249, 0x0000c4108a4b0ba1 }}, + {{ 0x0005187a5a610468, 0x000aa5dec3f1dec5, 0x00003bf8fe7fb8e1, 0x00045ff3807b1f04, 0x000027e52acc802f }, { 0x0000c3db2b856872, 0x000d522eb329d415, 0x000ea0acd8667267, 0x000318628f449099, 0x00009931d6113e45 }}, + {{ 0x000842a93785dba6, 0x000d1843757b3927, 0x000d31f9f72363de, 0x00009edd4326caef, 0x00004a867e26046c }, { 0x000d01979080ff2e, 0x00091965f96b10d1, 0x0005e9a735756595, 0x000f199fc373a9df, 0x0000befd7ab4c167 }}, + {{ 0x000fc047a72b6bf5, 0x000e3585499fb325, 0x000917e8670fd4f8, 0x0009cf9c57b67bad, 0x0000c3e8ea66d001 }, { 0x000b687b7d967ef6, 0x000db59e54a64bb8, 0x000b38cc1c666ef8, 0x000616a8d8862201, 0x000029b881c8aa96 }}, + {{ 0x000ca73dcaf21a68, 0x0006b678b2ecfa91, 0x000d8c64fe6d7cba, 0x000cb5c2b24dfe62, 0x00006a76f958454d }, { 0x000efc172f16fa07, 0x00031b776bb87ad6, 0x000b8346a0f01ef6, 0x0000389cd674976a, 0x0000c0f729ce4f9a }}, + {{ 0x000f232b5f42ffb1, 0x000059b92222f1f5, 0x000a0dd48310f507, 0x0005266ee5a2459c, 0x00000b4b667c3d65 }, { 0x000bd63ba4fde95d, 0x000e1c548e9e5167, 0x0006b5c1af8c6824, 0x000136d23124b4e4, 0x0000f392b3ec6ead }}, + {{ 0x0003122eee899cef, 0x000e8cc290a7f4fc, 0x000d3f748573265d, 0x0000718af6b409c9, 0x0000611f3640b176 }, { 0x0006bba5dab0844e, 0x00050504ef24d7e7, 0x0003c76eb236673b, 0x0000be0015d92979, 0x00002b8b64708c52 }}, + {{ 0x00078e66aa656cc6, 0x000a09cda94951ec, 0x000c9f2a6d6e848c, 0x0006760a44b6af58, 0x0000d198106fc9b2 }, { 0x00085099b31bdc5f, 0x00061834a53b9649, 0x0008670f897ce93a, 0x000e96997fc659d3, 0x0000918e4123a93c }}, + {{ 0x000954ac9501ff4f, 0x0004ffcc52fc2384, 0x0008216d2daa041e, 0x00084af08712b272, 0x00008cb41023b490 }, { 0x0004ccdbf9cf67dd, 0x000d0a47fb2e74f1, 0x000ea3cc4ab6b658, 0x00069d25bee8ba98, 0x0000589f94d48354 }}, + {{ 0x0009c15b35bafc8c, 0x000bc1b7bfe71786, 0x000dfb7efd32fb77, 0x0008ffa7d4e1deba, 0x00001f575658ec09 }, { 0x000e07633366d529, 0x000ed66368710884, 0x000cc4194ecbd474, 0x00068764de5e29f5, 0x0000a864508101bc }}, + }, + { + /* digit=10 (1,2,..,64)*(2^{70})*G */ + {{ 0x000ff71880242404, 0x000f1c7a3e6fced0, 0x0005e468a56ae92f, 0x0000f9670e8cbbdd, 0x0000020c88ed5eef }, { 0x000f9526f1333efc, 0x0008b74bd70437a2, 0x00018a76b379190f, 0x0004cd66e5fb3289, 0x0000132085c0e1be }}, + {{ 0x000edc4d24ff34d2, 0x0005e5192c4d6840, 0x00056736a2a84199, 0x00067dc208b04d72, 0x000091aa355f1170 }, { 0x0000b62c6dbea762, 0x00084bf854279005, 0x00058b53b37ca64b, 0x000f4906c3de8129, 0x000017420c7a1978 }}, + {{ 0x000c11118ac1c9dd, 0x00006b4c72dfe67a, 0x0005e4d08e55132f, 0x00049e035c416b0f, 0x0000282e71d5028a }, { 0x0004097747951cdc, 0x000c718f00628413, 0x000f55e9dfad607b, 0x000fa9e0cbfe0602, 0x00007edbf6c8369a }}, + {{ 0x0004645d0c999206, 0x000f55ed005832ae, 0x000aac9fd1427420, 0x000def2ca5f5efd3, 0x0000d8417da6ba53 }, { 0x00015bfe3a91ab48, 0x000ad8bc9001d1f8, 0x000bccb573cb03eb, 0x000ed588452c8f0b, 0x0000190d99ea9851 }}, + {{ 0x0000b35e6bb694d2, 0x000723f23885e5fd, 0x0009178fdc3fe150, 0x000401404f72f8f9, 0x00006aa2e224e92e }, { 0x000c8b1a1eb06618, 0x00088489164722d3, 0x000b4beb04a9a74c, 0x0004220323e3c665, 0x000059dfb811788d }}, + {{ 0x00086a67d771ac98, 0x00014e654317cac3, 0x000919ad09dbd2eb, 0x0003350b8be600ab, 0x0000e6ed7070c73f }, { 0x0003ebd5c9545d9c, 0x000f18d4431cc288, 0x00021471ef5ad0f6, 0x000672232a7cff14, 0x0000aa1dd2bbd83f }}, + {{ 0x000fd169fdcd85cc, 0x000b4a7b1fd2b0bc, 0x00058c1ac9eaecb4, 0x000ee73b39b3ad39, 0x0000f7b13aef63ab }, { 0x0007e6961db7503a, 0x0005fd79ce0a6af4, 0x00019c46a9db1c64, 0x00073a7b255dc1cf, 0x0000917465424f0f }}, + {{ 0x000f6928ec987520, 0x0001acce548b37b2, 0x0002556adc5cbc76, 0x000befb4fb38e754, 0x0000e6f549a164e2 }, { 0x000cbaf6791e178f, 0x000242386ce6c413, 0x0004d3f2b753e67f, 0x0009e6bf901be1c5, 0x0000fea1ab006e2d }}, + {{ 0x0003452936f20eda, 0x000474550ca42470, 0x000ae49807739ac2, 0x000811424ba1aa47, 0x00001f1e4b8ebaf3 }, { 0x000ac86d5bc8d43a, 0x0000181243aea965, 0x000ebafb6389c308, 0x000c049619a2919a, 0x00002b0cb20d9047 }}, + {{ 0x000f1bcbbb3db35c, 0x0007d18c36cf4406, 0x00075a1da5e1badf, 0x0006e2f64aed3e83, 0x0000987f12991782 }, { 0x00022fe40b1a2d46, 0x00090ed60919b8eb, 0x0000d89215e03057, 0x000a894efd890079, 0x0000d06046a39ef3 }}, + {{ 0x000d57eb5760d1c2, 0x00099a2157f2db33, 0x000f50d0df45370f, 0x000e83254bda2fc8, 0x000085807ee44cd6 }, { 0x00038f14b932ce60, 0x0008ba2fb5ee9973, 0x0002ad862411b6e7, 0x00064197401648ca, 0x00007031fac23dc3 }}, + {{ 0x000afcbcddc5b4e3, 0x0003f6ad7ab9a2d8, 0x000cf2fe2c7a8c98, 0x0009cf5c29c00090, 0x00000a465e1b5bc1 }, { 0x00039efc04140b7d, 0x000aa29e99d043b6, 0x000b4b1d9836432d, 0x0007aac35ef51263, 0x00007f50574f8f0c }}, + {{ 0x0007b58f4a281850, 0x00004b366b4be7ad, 0x000dacedead3f7c8, 0x000bf0cd93f25a9d, 0x0000ff4401b26350 }, { 0x0006a8e2b1dfd7fe, 0x000fba9ff563436b, 0x000932066f358f9c, 0x0001181ecf718ff2, 0x0000bb7b63ee6136 }}, + {{ 0x0001fe1782b8ceeb, 0x0004fa8327720c15, 0x0002680b1545ef26, 0x000a5814d956ca32, 0x000014f5d4a9c76f }, { 0x0007ab1de209b63f, 0x000f5db3c832c80a, 0x000bc5f7c5d2c7e4, 0x0005a0dcc60dccc5, 0x0000de8668f74c00 }}, + {{ 0x000686123db19be9, 0x000c50b3893d123d, 0x0002dad8d4df00f7, 0x0000d5d8bf7b7578, 0x00008135b99cd691 }, { 0x000b6a20e86bb9c5, 0x000a391a7ab4116f, 0x0007a97c837120a7, 0x0002c35d76fcd72c, 0x0000531c7135280c }}, + {{ 0x0003b25760e88423, 0x0009b7d341d81dcf, 0x00083e622c18ffb5, 0x0007e4badcddb688, 0x0000becf33dd410e }, { 0x0009a60cc895ade5, 0x000e3f6a7a73ae65, 0x000e082fa7baab21, 0x000f729bc83879a5, 0x0000c301b48a24ab }}, + {{ 0x0007d6af9dfc33c4, 0x000f9ceb18762621, 0x000ffad89f4b1f3f, 0x0003a59072b0d79e, 0x0000cd702c4f6dd8 }, { 0x00002614f15b98f8, 0x0000c616d7364517, 0x0006e68aaa6917b9, 0x00000a39d43cbbd9, 0x0000899bf61d6a7a }}, + {{ 0x0006dedaf4a7ec4d, 0x000a31a59c3e9f87, 0x0007578c622162c1, 0x000f162ad40200e7, 0x0000d60b302fdf36 }, { 0x0007b6a2e807e456, 0x0002a7c4bc2e88f8, 0x00008cb732bd420b, 0x000a903e96ba8129, 0x00003121d1ba9275 }}, + {{ 0x000582b557d8da3c, 0x0005d971b4e598eb, 0x000e517b95c5fc8d, 0x000d82bc5f499ef1, 0x0000275cd05a5723 }, { 0x000af53901d82522, 0x00090ca1715cb07e, 0x0001998aeb1b5a8e, 0x00031fccd4c3de6a, 0x0000aa50e5e8ebf3 }}, + {{ 0x000f2386acf8a72d, 0x000604ac3b819900, 0x00028ee94251e622, 0x0000cfab91cb8372, 0x000019e78eaae4da }, { 0x000f8063ac90f7e0, 0x0006d1eb32912137, 0x000dda25c7380214, 0x00046a9e3a2f82b2, 0x00002b889ab4a768 }}, + {{ 0x000586ef16cc6afe, 0x000b76759ad2877c, 0x0001d2ded83e32a3, 0x000039c846f45464, 0x0000d9ede27f0357 }, { 0x0000d0e5fc379cde, 0x000d8e024bafad93, 0x00023409aa6e07c5, 0x0009598a0c8b42bd, 0x00004ed05a16b8a2 }}, + {{ 0x00086b599a8c25a4, 0x00086e1e8b4dfd4f, 0x00068a9672118f45, 0x0000495acd754d5c, 0x000052f9c3c08e85 }, { 0x00027d16d7f92149, 0x00095a799f9d5812, 0x0008a07de7d22da8, 0x00003bca50473b1a, 0x0000faeb0de17d42 }}, + {{ 0x000721dfee849503, 0x000f43cad3d97f9a, 0x000477f19466ae81, 0x000004b7c52bf3bb, 0x0000ff72b1d2fc85 }, { 0x0004e7ef4acdcc1e, 0x0007a21141d4579b, 0x000bf553e4eae6c6, 0x0002e42153411321, 0x000058c897845f4a }}, + {{ 0x000a78b173431ac6, 0x000037c593710000, 0x000074fceaa51d30, 0x000c0a565e1c48c7, 0x0000087805015262 }, { 0x00044c6adeb5098c, 0x000f59dbcc8ad945, 0x0006844fc921b6d7, 0x000aaf80a70dc04f, 0x00009ea6e278d5f1 }}, + {{ 0x00018b6ba74db2c9, 0x000294e907a544b9, 0x000a9fb236691c4e, 0x00031d04ba7520dc, 0x000017babaf8cdc4 }, { 0x000818ebab294990, 0x00050fc644be8924, 0x0006b5968097aa9c, 0x0001283fe707a981, 0x0000f54144f9bcc8 }}, + {{ 0x000a2e6ab0d7e737, 0x0009cad58d2f767a, 0x000d2db95cc250c3, 0x000c89598dc5267f, 0x0000fb76d298d7eb }, { 0x000b5ae1ce39226a, 0x0008204d0c63eaaf, 0x00041ea4c6649b4d, 0x00017f91fa1cb92c, 0x000080a55559aaab }}, + {{ 0x000d96332d46ecf1, 0x0000a906e77de048, 0x000062d7886a21c8, 0x0002acf1d7992bcf, 0x00002e42c527e67d }, { 0x000072690f62575b, 0x0006e3a413aa2b94, 0x0005bc265f962dd8, 0x00007b63799a4d91, 0x00009074a785f7f4 }}, + {{ 0x000cf86624af5d13, 0x000f4cf02ff6072b, 0x0008cde126b78788, 0x000985309a47d2ca, 0x0000c4b1abc9fa46 }, { 0x00028fb8b07c6947, 0x000bf87aa9e5cb6d, 0x000144b063bc3aa2, 0x0003ccd35c98124f, 0x0000562b62f1af58 }}, + {{ 0x00050467db792ce5, 0x000972a40c251f81, 0x00078dd5dce34079, 0x0006e6f2388cd5d8, 0x00009ca7a668bb67 }, { 0x000b3e67f86537e5, 0x000f9bbcdabb8397, 0x0002764532467c29, 0x000f53b2ba79e974, 0x00008ea58750c9ff }}, + {{ 0x00068bd1f9103bc2, 0x000158fb1cb6ac9d, 0x000df3913b4c4a6a, 0x00069fd95ffa13e8, 0x000056704285e41b }, { 0x000990a0579bee8e, 0x000f8f926d783f2c, 0x000332aa0f6a131e, 0x0008f1b7c543a59b, 0x00009d122d6f7adf }}, + {{ 0x0006ba4245768c48, 0x0002db8519cce2c4, 0x000f6759961dc7db, 0x00035185cfc9af71, 0x00002da71c5029d1 }, { 0x0009e5918edc1491, 0x0005da4d3fd524d4, 0x000123fc1b33c45b, 0x000399a421e72464, 0x0000b38767166257 }}, + {{ 0x000915ee22d46e27, 0x00009b665c7322db, 0x0000faabc5d86f27, 0x00004cf85cf12cc0, 0x00003fe2633a632d }, { 0x0000a1c1ed8106c2, 0x0002e3ba317db248, 0x000897173dd719f7, 0x000cd0e5087dec65, 0x0000d47fab598e63 }}, + {{ 0x000caeecf58691f9, 0x000319678bf030ed, 0x000973a71452fdfe, 0x000a42b26fb5e9c9, 0x0000351791169cdc }, { 0x0008168ffbb6faf1, 0x00096bb518d0c6b1, 0x0002b381cba209dd, 0x00080d2819918115, 0x0000fd4bfcc99d38 }}, + {{ 0x00083365828a1975, 0x00058aa2502753c1, 0x0006390d16bf0feb, 0x0001befe5bb279e2, 0x0000995cb1132868 }, { 0x000733edbe3a75bd, 0x000fbc4eb8098772, 0x000dc2da79704c0e, 0x000789b67b336e18, 0x00009002f7d131fa }}, + {{ 0x000791f21653a4b0, 0x000d09a26025c3c4, 0x00064bbeed725db9, 0x0005e4d8ee6e981a, 0x0000b19e4a202131 }, { 0x000266ff7fb4c307, 0x000608925f02425c, 0x0003b83044c40262, 0x0007ec552ec86319, 0x0000716e68f00346 }}, + {{ 0x000c27ebc50428fd, 0x000abc3017025f39, 0x000c5f6d8def3924, 0x0000e19ecefcf628, 0x000032c44e11511b }, { 0x0005c3dffed38017, 0x000f7a11da1767e7, 0x0001d13c6c588b39, 0x000a35cb7fe15145, 0x0000aa988aa4f7b8 }}, + {{ 0x0009889769d0f731, 0x000f973ddc925fcd, 0x000055dd56a16ccb, 0x000258eebb679c84, 0x00009268c8cc2b74 }, { 0x00097b2662417e26, 0x000a87fb28ad2470, 0x0003f70e2717aaa2, 0x000e018a7e99146c, 0x000093a9487d1a5d }}, + {{ 0x0003e3172fc44d4d, 0x0001de6cf9f82a81, 0x00092fac300b1732, 0x000eb09c3319decb, 0x00004b56a4e95663 }, { 0x0006f61f75789888, 0x00070dc7d5a006c0, 0x00066f0587dcb0c1, 0x00029a49e2dbc27c, 0x00002832c4e33d0f }}, + {{ 0x0005c14ee801bced, 0x000cc7104779ce2a, 0x000164ec7c3519ae, 0x000edc5d5aca8381, 0x0000fa85767ed4ac }, { 0x000ab6c37ae9b82f, 0x0009683f17a62c70, 0x000c0c80967bc048, 0x00011798f39eb962, 0x0000d8fded0d599d }}, + {{ 0x0001e7018d266862, 0x000e5dba28a0749b, 0x00063c6a8fd87c38, 0x00018a515246c20d, 0x000045c4ea3224ad }, { 0x0003ddb70877d76c, 0x00045faf380ff628, 0x000d6fed75158c1e, 0x000ba3bdaef7fb1d, 0x000069deaeeea084 }}, + {{ 0x0005f83f869b4f81, 0x0006111fe7df7a4b, 0x000150472fba2a01, 0x00027069ca7cd07f, 0x000060963577ae8e }, { 0x000e155d967e0f8b, 0x000475ef3d8ac3d1, 0x000f446c416bd09a, 0x000e51a646fe2066, 0x000079aaace72c50 }}, + {{ 0x0005c34d6e4e9113, 0x0008e9bc3a72d004, 0x000b3f5075acc01f, 0x0003533374037665, 0x0000a738807c50b6 }, { 0x000a94ea00049bb0, 0x000310ff01fd7967, 0x000183a17db85256, 0x0008620c04dfb810, 0x0000ab6af468538b }}, + {{ 0x000fa6b43cf5d930, 0x00080d9bb9aa8822, 0x0005bc5f48d33762, 0x00043764eea20e4e, 0x0000fab3b1672224 }, { 0x00016dd2c48181ee, 0x000f5cf3aa91121f, 0x000948c634d733f5, 0x0005e41c08c6bd5e, 0x0000bc67f9d574df }}, + {{ 0x0003b606436178b1, 0x000c5d980101b980, 0x000db479092353f9, 0x0004b3ef3760883f, 0x000083e79a8931ea }, { 0x0007ca7221dd1d3e, 0x00015dfe05beeac9, 0x00022468b3abf051, 0x00031c5c3f2f1e86, 0x00003d226244ff9a }}, + {{ 0x0002738eddf0b5e3, 0x00087fe1cd19f723, 0x000cbf106d0d6957, 0x000765d3ca20915d, 0x0000205e1c9f6292 }, { 0x00018be236919468, 0x000b5f6b5eccf1aa, 0x0001872fffb9b505, 0x0007994934b5aff4, 0x00005e213e2348c5 }}, + {{ 0x000cecf93e151784, 0x0008bbab00ed3a95, 0x000416e0ff80909b, 0x000ade39734c6137, 0x00001fe486ea0896 }, { 0x0002d754720a1bc6, 0x000f841a968fbf0c, 0x000257852acb7321, 0x0006be3c7d23d458, 0x00003c39d3db2eb6 }}, + {{ 0x000b84a8eb836754, 0x0004b4a4eb732ec8, 0x000d2645c301b385, 0x000de24926c943bb, 0x0000f1ce27e0f66e }, { 0x0004a527b9540d18, 0x00054d32499dd5f3, 0x000dda2b269e8c86, 0x00023459d12053d7, 0x00005b6b44f0bd90 }}, + {{ 0x000c957990ece73d, 0x0000df09cec46eaa, 0x0000b8f62236fab3, 0x000c7f275b15347a, 0x0000c5ac243cde48 }, { 0x000f0546734a7a95, 0x00048a3c33a655d3, 0x000e494c042fdbef, 0x000d948415d503c2, 0x0000ebeb1661c3ec }}, + {{ 0x000222b6eb506da1, 0x000b577496125bd4, 0x0007f55249816eaa, 0x00095d799fb0023c, 0x0000430859f143e9 }, { 0x000dfb4f419b49b7, 0x000a7767866d25a6, 0x0007da3c5b2da294, 0x000a816b8b6eb04f, 0x00007d8063c0d891 }}, + {{ 0x000b45f1f50f92af, 0x0008fa448ee1f3a5, 0x0006aa5cbde06eff, 0x000579a489604d9f, 0x000024726a073e31 }, { 0x00065f33c2e91ce3, 0x000a0764acc757ed, 0x00078d775c347692, 0x000f15d4ba7cf9ab, 0x0000a19d93dea6f5 }}, + {{ 0x00095867f0c050ab, 0x0003315a377b14ed, 0x000ab4e18e1c0315, 0x00095d6486bf5463, 0x00009848fb503c6c }, { 0x000d4e25b124b432, 0x000d044ef5ab0172, 0x00011d673761aee5, 0x000af5f690bd2d8c, 0x00003bc0b64a021c }}, + {{ 0x000ab18bf1911b77, 0x0003cb5e224c5d71, 0x000c4dafdd27e4f1, 0x000f8ae52fa3baf8, 0x0000e716268c21fb }, { 0x000e9459a90edef4, 0x000013e9bdf25ddf, 0x0002ccd877ab4708, 0x000099c8004cc0bf, 0x00006969a689a087 }}, + {{ 0x000150ff364d5f18, 0x00071cacce9bb324, 0x0001b68d834f5fce, 0x000f2a54bddea65e, 0x0000dd6efba4ff65 }, { 0x000d6aaf6a6490d4, 0x000f401345f4e475, 0x0000c2cf77022252, 0x000ca4b133bf21f7, 0x0000818207356a20 }}, + {{ 0x000e990b48335965, 0x000ade1813d28f11, 0x000d5c3e99daeab5, 0x000eb3c7b6000e11, 0x000048dc9938a793 }, { 0x000bccfa65f60763, 0x00077ee93e976d58, 0x0000ba9412fd72f6, 0x000367788696241d, 0x0000ac1c3f505d93 }}, + {{ 0x0007de5f15b7c4ec, 0x00039b4c5cd14112, 0x0002d757f99bfdcf, 0x000bfe393701b404, 0x00007673cbf8b48f }, { 0x000d5487494f9df8, 0x000d7cef8d53f19f, 0x0000a6b5ebc3fb1b, 0x00050128addb1031, 0x0000e5c1f0ad87ff }}, + {{ 0x00098a932a972998, 0x000cb16a8123ef0f, 0x0003ee6ec10db2ac, 0x00074e05473c32db, 0x000030cbcdd6de76 }, { 0x0004ed4bc54db179, 0x000c3f60c6aa20a6, 0x0004103a4fed7821, 0x00098fa079a32ba1, 0x00008266f6300629 }}, + {{ 0x0002377007b82c41, 0x000d906d4caed0d9, 0x000684e5b2ac09e2, 0x0000627d7b802140, 0x00002dfa1b9afd19 }, { 0x000deb5a5140a60f, 0x000bcc371f21a855, 0x000a9a380f69015e, 0x0002249f918461b7, 0x0000baa37b31dfff }}, + {{ 0x00024176970787f9, 0x000a446d5eea89e7, 0x0003960208b4ce0d, 0x000762c4ffdff842, 0x0000b88756f44d35 }, { 0x000fc0da3195c361, 0x00027b9102cae3e6, 0x000d81d583cdc7bf, 0x000b577472d143e3, 0x00006226cc2a450a }}, + {{ 0x00019f54522deaac, 0x0003edfe45083a2c, 0x00079cbbf77b74d0, 0x00060e27f66bc721, 0x0000bc98cdbbf0a2 }, { 0x000fb17de31b650e, 0x0001e495b7eeed57, 0x000cfaf861be44b3, 0x000d376add9954ec, 0x0000243fb73909ab }}, + {{ 0x0007b8aeb5cd1941, 0x00023c5abb020e8c, 0x0003520c40119c32, 0x0002e52e53d18c18, 0x00009ba1a706394f }, { 0x000fe115bb0b6f18, 0x000b243f0f8c16bb, 0x000c16989d3b2f32, 0x00032edc9224ed5e, 0x00004558e4f0f44a }}, + {{ 0x0001277b7939f433, 0x000d5493927f4fec, 0x0003b4732c4c5df1, 0x000f4d751f91fbf9, 0x00009c1127aef031 }, { 0x000ec971f4c96c45, 0x000206ee7a759524, 0x000043c434006fef, 0x00038004f00df7de, 0x0000b58465048083 }}, + {{ 0x0000d60da1b917d4, 0x000f107e75d9ccc5, 0x0009ecf2f5648cab, 0x0009c77dc63b7be8, 0x000043a02262c4a4 }, { 0x00017f5de960c25a, 0x0000bee08df1b120, 0x00097e712bf60ed9, 0x00025a73d3873487, 0x0000b1fb76167cdf }}, + {{ 0x0005d84aa3afdbbf, 0x000b966e8e575672, 0x000ed4249074932b, 0x000372274c9f781c, 0x0000cdc2169236eb }, { 0x0007c4b46d4fac70, 0x0002e094d0c0ce2b, 0x000adafa0737f634, 0x000c614c71564008, 0x0000178fd068c458 }}, + {{ 0x0002aabc59c94f12, 0x00095b397acf4ec2, 0x00003bfdf9e674d3, 0x0004780850426c40, 0x0000ca04d6eb4c85 }, { 0x0006908d03fd22f0, 0x000814c5f5942bc4, 0x000db3a0f5e9bc06, 0x0001e62ba9a6168b, 0x00006bd8a9096113 }}, + }, + { + /* digit=11 (1,2,..,64)*(2^{77})*G */ + {{ 0x00009d21104a01b4, 0x00024745a1251fb6, 0x0008668a04b75ff4, 0x000fe349e967747a, 0x00006e02c0c6d4e7 }, { 0x000084a333d89913, 0x00059a82fe122271, 0x0005f3a1a369f264, 0x000fa053caf9b5b9, 0x000022f6b350187d }}, + {{ 0x00048ff65b341317, 0x0008233aed1d1f71, 0x00033bc63a356251, 0x0001b22cf566eaff, 0x00007f53d3789579 }, { 0x00086fa303582c2a, 0x0005fd4cf94cf0dc, 0x0003899cb5afac02, 0x00035800d319bec8, 0x0000afb9e2fd22fb }}, + {{ 0x000f31d817bed07d, 0x00055ab5c8b06d5c, 0x00022f1cd42f3db3, 0x0009efa79b1a785a, 0x00001ca931baa4c5 }, { 0x000b66d8a740682e, 0x000d06a15d7dc85d, 0x0003d89a928be329, 0x000b486c72f132b0, 0x0000478e55d53bd7 }}, + {{ 0x000e23139482dcb9, 0x0001ee1adc1696fc, 0x00023caa5f3be88a, 0x0003ebb3598ebe59, 0x0000801aabeabc49 }, { 0x000560a9c8a8fda0, 0x00082d4db067df91, 0x0005b59ef1377a4c, 0x0000329c198dda09, 0x00005daf596bd7b8 }}, + {{ 0x000029268430e2db, 0x000923e99e0925f5, 0x000612f910d69e1d, 0x000881aba2dd74a9, 0x000075a644f8e684 }, { 0x000b66a21626e638, 0x00014b41aeaffa32, 0x000f702ee5abbbdd, 0x000eafd0c8b17203, 0x00001a68d527f1c3 }}, + {{ 0x00089de4e0f889d4, 0x0005e5efe51bf748, 0x000dd5a81bb30fc6, 0x000505b259c4f579, 0x000013cc73a8497e }, { 0x00057568b82d7835, 0x000b706646083ce7, 0x00054a6bbebb3b88, 0x000bc0c10be2a033, 0x0000d865d4a67d97 }}, + {{ 0x000e11a1d8d399bf, 0x000d87900f7e6c85, 0x000eb19b3117a1bb, 0x00064247538e8b94, 0x0000d5742a10d630 }, { 0x0007b6a724c5c32f, 0x000812ec1e2f2ef2, 0x000b9e4effa0ede3, 0x000d6dc2d193d2bb, 0x00006ad06c86c96d }}, + {{ 0x00064232b86345c6, 0x000491cf1c367cab, 0x000b12cd89c580bf, 0x0005b4a329bc85a9, 0x000061507757ea7e }, { 0x0007fbd8ddb9611c, 0x00062b0167dad297, 0x00011cbbb1d53bf6, 0x000c8188b1604f30, 0x00001f4d0fd7d22e }}, + {{ 0x000307a8c0590774, 0x0009aa5b964e39a7, 0x000c5dcc1eae81ad, 0x000c9a564d4c29e3, 0x0000ef66dab1bbfd }, { 0x000139e05cf3cde4, 0x000ddaa91708c3b5, 0x000bd7852e8e71ed, 0x000d569b7a151e62, 0x0000050f4937247b }}, + {{ 0x0001e893a8e9fd20, 0x00031b494a7cd2e1, 0x0000f4b09fc385f9, 0x0009d696a5671ffd, 0x00009847745f23ad }, { 0x00023ac856040d5f, 0x000a7a165cf7b104, 0x000882d2bc2f947b, 0x0002854de81b6717, 0x0000ef3de081df86 }}, + {{ 0x000b2c9190e12088, 0x0008587dac50b74b, 0x000e9d3ce1bbce37, 0x0002da0fb28b2b89, 0x0000cabf97afa5a6 }, { 0x000f9f3f579f2fc2, 0x000a71884a9d5b7c, 0x0004aed7c41c7a6f, 0x000092de27ebe232, 0x000029c3d363af68 }}, + {{ 0x000aec5d6d717771, 0x000f8e1096187086, 0x000456979c440027, 0x000f2b0afcc9e836, 0x00008cf2ab4e27c4 }, { 0x00007dfe122baae8, 0x0000f8a994f04e95, 0x0007d694479ec283, 0x00039574fb39ccae, 0x0000aa93a28e976c }}, + {{ 0x000a7d33d6e39217, 0x0006acb489b03e9c, 0x000aa446e1236c2d, 0x0008923e5db86ec5, 0x0000d513e9282311 }, { 0x000832e75dca4e42, 0x000ed5c0ae6c4d0e, 0x000d8b63ae0adf74, 0x00059047ea9ca1d1, 0x0000237a1f96d350 }}, + {{ 0x000170458c664431, 0x00049ea5232123db, 0x000bb43e414ecb07, 0x0000237ed08307c8, 0x0000c4bbbb01d044 }, { 0x000c7f7317b92a8f, 0x000afc6c19bb9866, 0x0006418391d59403, 0x000750822c5f3484, 0x000021de21b188b3 }}, + {{ 0x00068d834bef31c4, 0x000d00e9d6d2e8b4, 0x00010e11f94b7363, 0x0003f7485d318b8c, 0x000049cae972b8a8 }, { 0x000b12c61b315467, 0x0007eb2bfba796e0, 0x000b625f2afba6cb, 0x000620753152364d, 0x0000ade1ce1d3e84 }}, + {{ 0x000eb572d99661c4, 0x0008402fcf0a7fd8, 0x0008c5d33d449b8f, 0x000f0f09842fd078, 0x0000da7983de9402 }, { 0x00076f6bdb01f785, 0x0001f4c4d194a88b, 0x00090292128bdeab, 0x0009129fcd2b63fd, 0x0000ebc377dfedf6 }}, + {{ 0x000ba4521833ac99, 0x000984bb46c2397b, 0x0005a248d600e4d2, 0x000c9b434004cf56, 0x00002121f29dec04 }, { 0x00078cd8c6df3a54, 0x000f1711171121e1, 0x0001d03cddaa270c, 0x000d52107ad7b7eb, 0x000098a27d377993 }}, + {{ 0x0008ca59bbd2a43f, 0x000a201b2a672385, 0x000756d1bb36ec67, 0x000c1e1753556d36, 0x0000c58a255d75d2 }, { 0x000aed0cd5454528, 0x0002b92a569a73b1, 0x00083babe9f4067f, 0x00068af55517a52e, 0x000059afbc58d069 }}, + {{ 0x000c3323e86fa8e0, 0x0001f8480af13722, 0x0003c58739374a0c, 0x00050ad31244c311, 0x0000126dd368d686 }, { 0x000979d546fc2f7c, 0x0002f4a5594b1dd8, 0x000b453aa4371b74, 0x00058ee19a1928bf, 0x0000915af098fee8 }}, + {{ 0x00033084d9c7276b, 0x000bbbc19f9e9676, 0x000285cb3a74d9cf, 0x000491240d9d0315, 0x0000ba33cddb43f0 }, { 0x000d408f6f73b28c, 0x00010dc55667eaf7, 0x000ab10446007a1b, 0x000e39f1473b7f92, 0x00002ef1a03646da }}, + {{ 0x000103f6bba1307c, 0x000d1dc164a89bb6, 0x0001ddefc64a2b32, 0x0005d70bb74a42bb, 0x00005152ad60f6a1 }, { 0x000e4781a0271b2b, 0x000ca81355a5752f, 0x0007024ce4d4d310, 0x00084f73b562f96a, 0x0000c477204acd46 }}, + {{ 0x0000757b5f4712e4, 0x0004c75f8dfcf8a2, 0x000a74f11cc657d7, 0x0006a5bd8a284f0a, 0x00007dda9a0af13f }, { 0x0002a3e7718f7a6e, 0x000d22093b8a5ded, 0x0009191c86356347, 0x000f92e33d3f9335, 0x000055ce4eafcd22 }}, + {{ 0x000c17e23aa997b3, 0x00096f4cd941534e, 0x000af7af39d3fdd7, 0x0000044e30d37406, 0x000017d4ad645c92 }, { 0x00052737e78e08ec, 0x00019516f496ba27, 0x0001c18619005520, 0x000df03f56a69317, 0x0000b6ddc6edbaf5 }}, + {{ 0x0009a1904af269a5, 0x000ee2c68af43eec, 0x000da69b7072b08d, 0x00046bcf55502468, 0x0000f80ea4ac443d }, { 0x000a17081f1c5679, 0x000da8d470c56a4f, 0x000e1f2684ceb758, 0x000ba15f3159abc5, 0x0000964c76954003 }}, + {{ 0x0008102033e8adcd, 0x0002e2b10c8c0f59, 0x0009a854e5133f03, 0x00015f2a72102c3e, 0x0000c9a9e311aba2 }, { 0x0007a89342969c71, 0x000f33ff1d18a13b, 0x00012053c7add2ec, 0x0004e3ed835eebd9, 0x0000d9e680e72864 }}, + {{ 0x000ce7c67c39f8eb, 0x000838decf2a73a0, 0x0007816d5317f027, 0x0009468a2066a639, 0x000026677b2d0e7f }, { 0x0005a428ef86e036, 0x0009071759a113ce, 0x000269cb3f0ac1ee, 0x000563c7be3b373d, 0x00006ff591e18a76 }}, + {{ 0x000b6b40b404a661, 0x0007ac81c86682bd, 0x0006dd64ea76b3ad, 0x00015d19ab78d272, 0x0000fbd431bc398a }, { 0x000802739262b87e, 0x000297db8c4961f6, 0x000bb8e36d198176, 0x00030325eb1a12f7, 0x0000129ba25ce217 }}, + {{ 0x000c873312835882, 0x000685f16b1bd5e5, 0x000ff15bd7c0e8a6, 0x000dcb99ef957e41, 0x0000fdf280b53be6 }, { 0x000093ef1a573f41, 0x000b88cb65f57cb1, 0x00014d78ee26f748, 0x00048d340db60a62, 0x0000c4ae6cba4b4f }}, + {{ 0x0005f9a99adf6ad0, 0x000dfb53f4ece64e, 0x000d900aab7fe12a, 0x000bbe8cd6a6bb02, 0x0000244f4571b32b }, { 0x000cb9c4a0dadc5c, 0x000682511a7de841, 0x0005e0eaa53ce5b2, 0x0009034a28356809, 0x0000d2319e930091 }}, + {{ 0x00070b941f431f96, 0x00035babb2eec644, 0x000492f8199f509d, 0x000c8d4c4a0c23b6, 0x0000055b8ea1eb9d }, { 0x00073291adbc3b63, 0x00073799df435742, 0x0008e83346629e9d, 0x000445a814ccfb8a, 0x0000b7b0d75235ee }}, + {{ 0x0005efcdfd123b2e, 0x000866b42a92806b, 0x000bf27ae64de389, 0x000d09bf8810684e, 0x000050a1dcc40eb7 }, { 0x00050352364d1ab7, 0x000d2076e9edd128, 0x00004fd440c2c9e1, 0x000e85a6de29d0b5, 0x0000da6c7734e64d }}, + {{ 0x000943b9b2fa6dc9, 0x000e0b2b4b4b67c8, 0x00005ebe4c512574, 0x000128fd8c1d10df, 0x00006e0e8ebb49c2 }, { 0x000b34a5656dfc03, 0x000d6143b3def048, 0x0001c1b2d09a22f2, 0x000c08d4085ab3aa, 0x000060432732650d }}, + {{ 0x0009efab06795cba, 0x0000f4a10358560e, 0x0001b46c7da3c8a3, 0x0006b31152d8a507, 0x0000637882aff5d3 }, { 0x00020630df6bfbd5, 0x00072a9d19e9b915, 0x0004b06fe46d6faf, 0x00003090d8cfed74, 0x0000ce2f38fbcec1 }}, + {{ 0x0004f8e4921219b0, 0x000ef7ce382360f1, 0x000973c673c5135b, 0x0003f246cb6db05c, 0x00000c2638a18b5a }, { 0x000d76ff4b38f5e9, 0x000897a11c203407, 0x000f3205cc6b7d94, 0x000eab1200386f86, 0x000080180ab081c5 }}, + {{ 0x00082444682aa7c3, 0x00028e7db15be7ac, 0x0004107abe9a7d16, 0x00071706467a0805, 0x000025b9428d4196 }, { 0x0007714718a7a5d9, 0x000b58b20d841b88, 0x000854475e368b17, 0x0000c2936127a054, 0x000012a411de8dbc }}, + {{ 0x000cc20bdfcef420, 0x000aaaaaa8a5288b, 0x0009f26195e97eb7, 0x0002dee0c91cc0c8, 0x00002946f9408a70 }, { 0x0007409503d2975e, 0x000e4daa20ede1b5, 0x000c1d5f70119b97, 0x000b42e4bc441f00, 0x0000a83a9d6eb2a0 }}, + {{ 0x0005f1068db22608, 0x0004a61e4585d453, 0x000d3143dfb4ed4d, 0x0004ae66c9231fae, 0x00004c8d8da2e90c }, { 0x00009002a8263ee7, 0x0001200a2d261155, 0x00029e7afc3b45b0, 0x000e539a1649406f, 0x00005414f28a6295 }}, + {{ 0x00027abb0cc2f99c, 0x0009aa53ff3571bf, 0x000790fe6cb4f16e, 0x0003a2b1624f98b7, 0x0000a232d3065d82 }, { 0x00072b0da96f4899, 0x000cbfc5c6d5bfd7, 0x000179a26cce6886, 0x000505c346915c75, 0x000008d2131b6918 }}, + {{ 0x00071a4c560e39dd, 0x000490c2e03fa69a, 0x000994c7571e040a, 0x000ccdc6036cf0e3, 0x00004bd59ba6dde0 }, { 0x0005e78c885a065c, 0x000b65133c673db2, 0x00004e0134a9208d, 0x000f06b34c0bea51, 0x00008995c776117a }}, + {{ 0x000d702cfbe0903e, 0x0006ef88e2f7d902, 0x00074a4e05d17b88, 0x0005b20e9fdbf33d, 0x000010bd286ddbf2 }, { 0x000dd0fd8fb34fd8, 0x00013db85c88ce89, 0x000b8030d8880391, 0x000235bac23a3ce3, 0x0000fa45c03bfea3 }}, + {{ 0x00047e2ad1ea2431, 0x0001602b57872b88, 0x000ee0ff350f3dd3, 0x000ab0c402ef26b4, 0x00004bfae875386d }, { 0x000c3fc966771b4f, 0x0008aa009996a74c, 0x000c5de647f65d59, 0x00053fe4316a56ac, 0x00001cad819b5d5a }}, + {{ 0x000c2f5e8ebb9875, 0x0006573ce07d1b0c, 0x00018c7de2e97cd3, 0x0008da2f4d82910c, 0x0000fe01b8994111 }, { 0x00085928bcf47e0c, 0x0001950b8c30ccc0, 0x000a12352e9d8df2, 0x0002d88597b856ae, 0x00004ba7a71332bd }}, + {{ 0x0001d44d0c939385, 0x0007decba8f4b4d6, 0x00099ee0887ec970, 0x0008ef97ac07cd4b, 0x00007b8f4d37c895 }, { 0x000aa760774d89e6, 0x000f62b135e6f4d6, 0x000b7cecb07896fc, 0x00076a2b5680962e, 0x0000fd88fe58cca3 }}, + {{ 0x0008eb579c89ba85, 0x000b8118fb98871c, 0x0002f2dc1362f68b, 0x00025b114cb26f5c, 0x00008a957abc4172 }, { 0x000dfba9354cbe2e, 0x0001e15c93bae41a, 0x000c8cf0fa7d7f42, 0x000606cb146fbae5, 0x0000f605d4ba99c2 }}, + {{ 0x000697b7814fba90, 0x0004013f216c980b, 0x0008410654403924, 0x000dd1237c8ac4fb, 0x0000683bbe6245ab }, { 0x00033aad090c72fd, 0x000c939ddb945b56, 0x00077d5a16a024cf, 0x000b36f90447b06c, 0x00004f7703863965 }}, + {{ 0x000d40e0825f1a07, 0x0008c6bb57c64779, 0x00044c0cdc444452, 0x000779f7b871f8b6, 0x000076eaaa21e046 }, { 0x00087b30ac366f12, 0x000d8f6f2d84b5af, 0x0008acea8597faa2, 0x00014068b228993f, 0x0000424abfd6e5f6 }}, + {{ 0x000ba6b8ef40499b, 0x000f0317642bca8e, 0x000e66d95628781b, 0x00075f9451a032d7, 0x00004159c0acf639 }, { 0x000ec80051301541, 0x0005badd44856928, 0x000dfabee96d2c76, 0x000f8734884bda40, 0x0000d972ba5752fe }}, + {{ 0x000be6868dc83e03, 0x0005e0798fa7aaac, 0x0004248d23c0ea74, 0x00098ae9b41209ee, 0x00003cedae7fcc2a }, { 0x0000a5ccc7b9ebec, 0x0005fdc32c04dd3e, 0x000def4916cf5e80, 0x0009ad4cfa6c35fa, 0x0000a98de7d24b36 }}, + {{ 0x000b2d7405bd794a, 0x0002eaa1536189f8, 0x000f2f40816c05a9, 0x0001b0c0e1126c55, 0x0000e641bd7bfcf4 }, { 0x0001863ad3cc6a83, 0x000109e3738be712, 0x0009f8c75cd5b817, 0x000f12615b5090b1, 0x00007dbc50add023 }}, + {{ 0x0003c1abe2e2289b, 0x0008b79a7aae5d37, 0x000c9a2415e1f025, 0x000dd1b297076013, 0x00009d9bb7e7d7d8 }, { 0x00022100092b6697, 0x0001d22f2488af76, 0x0002041aa56cc46e, 0x0003ffd57e4105d0, 0x00004447fba18228 }}, + {{ 0x0000d8ea16b68aca, 0x0008a7877a50ec69, 0x0006795ebd16449a, 0x00056c47abf0392f, 0x0000c5c659818595 }, { 0x00060dbb7c6ed91f, 0x0008393ef953f575, 0x000567b78ade06a9, 0x00077e81790a64d0, 0x00009be345281b73 }}, + {{ 0x000d3b3730c3da90, 0x000a2864b66abfb6, 0x0004ebc7d1cf59b3, 0x000aa041ad26f52e, 0x00008875834f1291 }, { 0x0008f625c182ae12, 0x000942f4789dba54, 0x000529bfa3f61d44, 0x000e1880e91fcb81, 0x0000772dfeb7bbe9 }}, + {{ 0x0006ee1cad7e490c, 0x000dc011c6219b80, 0x000921be12763c71, 0x00098f988e7a562d, 0x0000ff590f7684a8 }, { 0x0009c6ab4f30f5e0, 0x000ce43faff79f72, 0x00023b683e91a933, 0x000696bd0c141a41, 0x0000472541b41fe8 }}, + {{ 0x0009a4a020384d2c, 0x000e87730448112c, 0x00011afee636432e, 0x000e4241082b51af, 0x0000f39a9a8e69f6 }, { 0x000d49d0784959b5, 0x000e0d8257e5818e, 0x000fc707ca9eace4, 0x0004f5bbfdd8efa0, 0x00002149995867c4 }}, + {{ 0x0002daa6890484d1, 0x000db5eb7caee1e4, 0x000d651a33cea7b5, 0x000e198b91033898, 0x00002903be868557 }, { 0x000107f8ce029e4d, 0x000745c1a1d885a8, 0x000073af6f495ffe, 0x000f1422825572ec, 0x00005e04a9386799 }}, + {{ 0x000fd321053b8bde, 0x00015a36fa6110cc, 0x00012bc0351a15e4, 0x000cb742b74fb1eb, 0x0000cad7c54d4248 }, { 0x000d56251a7ec868, 0x0001ca2cc80ad571, 0x00056953911caea0, 0x0002c80cd7c27fc3, 0x00000a95d3fdd32f }}, + {{ 0x0002503375e361ed, 0x000e509c955bef4d, 0x0001345d7cd1281e, 0x0006562eedf02cad, 0x0000ba01e9851b7b }, { 0x000be946c753683b, 0x0003a5edf53cb366, 0x0001c77f0e544309, 0x00037eaa454dcb61, 0x00000c4e558ee575 }}, + {{ 0x000541c7c1655fac, 0x0009ad1a995e0941, 0x0006cfdc3d170c39, 0x000be851365e848f, 0x0000c5364d173f04 }, { 0x0000fc20ed612d22, 0x000a7faf4acfade6, 0x0006e3d0ea83ec7d, 0x00024ab770e58589, 0x00005a5480d97ba9 }}, + {{ 0x000018f5e5fc5e81, 0x00083eca42768678, 0x000bb4da53ad5c55, 0x000d864bd46f6c3c, 0x00002cc3c22b21c9 }, { 0x0005acc0361b36ae, 0x0007470e47b668f5, 0x000d986831cd85e9, 0x000c8ed4dcef48cc, 0x000012e983797847 }}, + {{ 0x000dbf24eccee72c, 0x000e6af64db44442, 0x0004910eaf9370ff, 0x000496b8a19e9d63, 0x00002081b3c75ada }, { 0x00086167eacf78d7, 0x0000c8c2cd706b27, 0x000abd2eafce6c92, 0x000681eae5b012a5, 0x00008d905b32a1a3 }}, + {{ 0x0009ad7d324559d4, 0x000b57d594881dc3, 0x0008960aa38fa09f, 0x000f644de6b5efad, 0x000000bdc3e7618f }, { 0x000a309b4ea8cce9, 0x00039e02cfc0e816, 0x00018ca8025c0c84, 0x000981109117c466, 0x00002b74c94f9322 }}, + {{ 0x000b6614f982ece8, 0x000b7b7abea49b1e, 0x000c257cb1cc77aa, 0x000c2f67ab4b1d27, 0x00005154e895caa0 }, { 0x000ea5d68b9993ba, 0x000c76b7f6f7459c, 0x000e3a9037efb0c9, 0x00075d74ce431a44, 0x000063571583a729 }}, + {{ 0x000102c8a0fb0bfe, 0x000f8653a0a3f957, 0x00060be8d9da3b5c, 0x0006691e98936941, 0x00000313a1550ce9 }, { 0x00067e106b240330, 0x000010e235d21c8d, 0x0000e87d6f10977d, 0x000925a67505e55d, 0x000059ee2e27a5ee }}, + {{ 0x0003858b5c81bd3f, 0x000d2a9431dd80e8, 0x0009efaae17d3682, 0x000f67830eb7c7cc, 0x00002f1f3290cd4f }, { 0x000990a639034a2c, 0x000a593b6251d5c7, 0x000299c23319bc02, 0x000b194511cca1fd, 0x00006f501add6b8d }}, + }, + { + /* digit=12 (1,2,..,64)*(2^{84})*G */ + {{ 0x0006beb87707b7a2, 0x000c72a87dec0e16, 0x000d90f4e489ddf2, 0x00017feb5010ded8, 0x00009f1c146a514d }, { 0x0002ae989277487f, 0x00076313476cd0e8, 0x00052ddea0b6d98f, 0x000ff20c6a63d0e6, 0x0000d40ea3d516db }}, + {{ 0x00019c667d0d91d1, 0x000e8105ca7d8669, 0x0001a93ed4b79dc6, 0x00058efbc582967e, 0x00007205a3aecabf }, { 0x000187f1f85aef05, 0x00012b160f5dcd7b, 0x000f2c42bbc43ffd, 0x0004562f5ec697b9, 0x000026b5000648eb }}, + {{ 0x0000d41a77c52336, 0x000441d214aeb181, 0x000c6340187fcbdb, 0x0006e6ac41506af3, 0x00003b6fa4818220 }, { 0x0006bdb65cf1fb29, 0x0003ce4a84bde96b, 0x00083b4cb3bfaea2, 0x0008473e742f060d, 0x0000fba067aea100 }}, + {{ 0x000b6c2d46254ea3, 0x00039b6ec7fae9f1, 0x0004d44a4114c60f, 0x000f5ba52995f271, 0x000066e8cbd34843 }, { 0x00062c42a011d210, 0x0000c318129d7161, 0x0000f32c7f0a2090, 0x000229f63b03909f, 0x00009687ec5c5909 }}, + {{ 0x000507db0a04df74, 0x000af43753b9371c, 0x00099a17c1cd2a88, 0x00066679629cab45, 0x0000a296edbca1ad }, { 0x000519b397e39c16, 0x000e052af036c326, 0x00079fe7dac46a92, 0x000efcd5086f0cc7, 0x0000bf3f8cd63cc7 }}, + {{ 0x00042e43a80c6fad, 0x000b1ef9c053df72, 0x00078ed2a6c7dc5b, 0x000da22fb8de25b3, 0x000063c34563eabb }, { 0x00066648e3f185ac, 0x000a5f4dd6f958ec, 0x000f2dde11a9f374, 0x00087dd496925a77, 0x00007412068d6cf7 }}, + {{ 0x00005e399e662c0f, 0x0000a57173e460c5, 0x0004e0120bf24c7f, 0x000f062621bbbce7, 0x0000fbd676e31f74 }, { 0x000bef99ec94a32a, 0x00023cb57797ab7b, 0x0009ae3d0efbd3a0, 0x000900cc160ad35b, 0x00000124b141f449 }}, + {{ 0x0007c8bdf49d7f19, 0x0007df31711ebec8, 0x000f46d03fcfcebc, 0x00035281f2da40f1, 0x0000aacf4dcfdeba }, { 0x0004907c5d800621, 0x00068e3c2eef12d5, 0x000ae7e3f5965a34, 0x0000ca494de95bb9, 0x0000c88b84c6fe58 }}, + {{ 0x0000c1e7da0cf5f8, 0x000cdf08db60e357, 0x000689c14b3f9c46, 0x000208bc9743e3cd, 0x00005819e16c67eb }, { 0x000b4a07ea1a52ab, 0x0009f39a6be09070, 0x00057b86afe8f489, 0x000d94f6036d4703, 0x0000fe9b7f7d190c }}, + {{ 0x0003057bc03a43d5, 0x000782945c1dd539, 0x000790186ff52f5a, 0x0003b2be0cadc589, 0x0000ce868856076d }, { 0x0009d9be5f0800fd, 0x00081cd027a4cd18, 0x00033c3a42bb278a, 0x000667f68114734b, 0x0000ae65c75c585f }}, + {{ 0x000361bc67e9e34d, 0x0007e2e089808ef0, 0x000dd23c849fe842, 0x0002e6cc9f1f7a27, 0x0000c13a3157f998 }, { 0x0001b02bc1e235bd, 0x000a00b7f071426a, 0x00004556f9a54850, 0x000360dbd6a3c181, 0x00007d7ec15b5c2b }}, + {{ 0x000f7d41e8e79665, 0x000895931fba2393, 0x000359404fe8f948, 0x00083d71df47424d, 0x000023b7a1aa0b1f }, { 0x000ef6eea564a053, 0x000d0c2cf6eb9980, 0x000f51a339da0be5, 0x00028b46286bb49a, 0x0000f0e9d516a3ff }}, + {{ 0x000b44589e394e0d, 0x000207236f5defe7, 0x000bdbb538d8261f, 0x0003ed3c7fa9922d, 0x0000fc2a961330be }, { 0x0009acb43064282b, 0x00010ca9ea837700, 0x000f210a61177cf3, 0x000f5b1c0d7fa2c7, 0x00003784b94dc9c5 }}, + {{ 0x000028746afacb37, 0x000132e649d4e574, 0x000fd8eede035012, 0x000e913c4cd53a2a, 0x0000887fc8e5c525 }, { 0x000c3ac1b0c48f95, 0x000ee9064291d4c7, 0x0001722017588ca1, 0x00088c181bf48f15, 0x0000f3153a52a8b6 }}, + {{ 0x0008e6567629384f, 0x0002ec3c093c171c, 0x00035c0dd4ef3c20, 0x0002147cbae7acb2, 0x000090ba8583656c }, { 0x0009e2f0802b74dd, 0x00042ba28b8d0b1c, 0x0006ecde1a103afc, 0x000ae4d8015d7413, 0x0000a3d25fc46b0e }}, + {{ 0x000cadd1a806623b, 0x000e2b6f79588c00, 0x0009a8a99724a1aa, 0x000f2088afa52fc6, 0x0000f705025b0678 }, { 0x0009b5b7e0f923c4, 0x000e2b31803dd6fc, 0x00048c34f654baea, 0x0000a8a16488e4fa, 0x000078ce7289743f }}, + {{ 0x000bf0e7f5c451a2, 0x00030d3957626499, 0x00072bbdbc00a6ca, 0x000755696a9604ee, 0x0000c65ea7b931a9 }, { 0x00082e474cddae64, 0x00064217cb44b924, 0x0003667b7a354fe4, 0x000a641fda2f5ce6, 0x0000f36622cc64b3 }}, + {{ 0x000c9dd0ef759995, 0x000ddd0b611c24be, 0x0004740b22624bff, 0x00035cf211d468e6, 0x0000807471447a06 }, { 0x0007b4f8f0064eff, 0x000d3bf956c2e32a, 0x00043529005e550a, 0x0009f776573dc686, 0x0000b45bf82931b6 }}, + {{ 0x000d2dba29a420a1, 0x00077dd7280f8553, 0x00004e810c5f5788, 0x0001f6e42bbaac26, 0x00000e0aa6d6e58f }, { 0x000163cfe5ae3f65, 0x000b594985093755, 0x000f7d4053e708b7, 0x000c881fb8f33031, 0x0000315d7722ea99 }}, + {{ 0x00023d24bafda93d, 0x0005217cf41c6e88, 0x0009813e6da10623, 0x00029057cf1f03c2, 0x00009f3f08321169 }, { 0x000891e4e39dfe78, 0x0001813901f0b4f7, 0x00018002374a94f1, 0x00029912e684360f, 0x0000123427926e8e }}, + {{ 0x0007ced2848ab4fb, 0x0004819ac445e3d2, 0x000766d870fcc671, 0x00034ca0a553432e, 0x00004c49d4d2e4dd }, { 0x000df18da1030901, 0x000d2757a4afa571, 0x0003c69333af3c8d, 0x000b49823de0a14c, 0x0000c31b61984a35 }}, + {{ 0x00069041a0c93e91, 0x000fe82d8d38a9b9, 0x0000514dd1f04e0f, 0x00082e4dd2e97c60, 0x0000d5f81ee7afd8 }, { 0x000a8b342d40d5ba, 0x000993b76a2086b3, 0x000b9d37b4d18b27, 0x000c84167d6bba71, 0x000006b92014b8c8 }}, + {{ 0x000a7dbf4441e65d, 0x0001949522f9be32, 0x000bb4a18a63f9da, 0x0006e8f42690c075, 0x00000b984a4ca3f6 }, { 0x0000aca623b9e392, 0x0006a0e3faf40783, 0x000eae64e0090621, 0x000be25b3b2f02e9, 0x0000a47a4149ebe6 }}, + {{ 0x000e79d046d6c0d9, 0x000e60502f40d9ab, 0x00076db7ba07217b, 0x000db4d744681c46, 0x000074b09e5ed137 }, { 0x0005cee5b6dd4217, 0x000310a78bd01e0c, 0x0003acd63802d1dc, 0x0006da3b66dd359e, 0x00004f7a41696b06 }}, + {{ 0x000a2e4e3632e762, 0x000ced46ffd9fbbc, 0x00086c97c281ada3, 0x000fb00cf6e5bc69, 0x0000f75bfce3bc69 }, { 0x00075117090a5ff5, 0x0006fc016de36c5e, 0x000c5b91a11ff6bd, 0x00025b5f8e0b10fd, 0x00009f9ba307785c }}, + {{ 0x000ff3974d378756, 0x00082fc3b922bf8a, 0x000f9c3619e62f97, 0x000e743c4e4d8c09, 0x0000070f7ed0ec03 }, { 0x0002c29a21b92463, 0x00031052c18df9b0, 0x00038e3dd7174a50, 0x00055a895c5c3a29, 0x00008d20d8ec2e2a }}, + {{ 0x000b0838226e8425, 0x000ccd6141ecf611, 0x000e20a9f7315526, 0x00050ee579195509, 0x0000827cd560e1b4 }, { 0x000ee0b7e6ba0f71, 0x000fd35e07289f09, 0x000bcef7b719136d, 0x000002bfa1f94c48, 0x000063e3e9d89c31 }}, + {{ 0x000fbe8ff0b58802, 0x000fc2e34d74e31b, 0x000f7c390ac5a4ca, 0x00076e98fa352c30, 0x0000f58dc8de69f6 }, { 0x0001022c67c9035b, 0x000c1e6a79046089, 0x000a33fa75e20d04, 0x000e287001f812f3, 0x00004fd325eb32f0 }}, + {{ 0x000f7a6bb4ac623e, 0x0005e5d30a713a1e, 0x000f9574a3df3bdb, 0x00099bb284876e3e, 0x000058865f427515 }, { 0x00067d2e36052baa, 0x000c828dfa2a659d, 0x000801fa7659b2da, 0x00031653fd5cd339, 0x000060ed537958fd }}, + {{ 0x0004a96feb09a9c1, 0x00026d7cc5f43941, 0x000c308d71570ea4, 0x000a0e50ab5551b5, 0x0000c0238a0327c9 }, { 0x0009f80382b52a70, 0x0007f740d12bb0b3, 0x000026c163fb98f7, 0x000cd031dc522f02, 0x0000b4064550e478 }}, + {{ 0x000c58e2b0841de2, 0x000810c39024a940, 0x00088ad8b1a25b9a, 0x00085a47caacb96a, 0x00007b4e4f38b169 }, { 0x0008ab30080ea35c, 0x00001634da919fac, 0x000a28b2f2812eb1, 0x000e84f10ef79b69, 0x00003e9ab4693571 }}, + {{ 0x000c047df2f3ac76, 0x000c2b67c4a658ad, 0x000864e2a38fcd7a, 0x000ec6e4fb3c4763, 0x000051531fb65393 }, { 0x0005e4fd59db390b, 0x000c9c55e59d92d3, 0x0005b30150334900, 0x000919016cedca47, 0x0000584c78dab3ac }}, + {{ 0x0006a76dd7489981, 0x0007e18e4bcf6b1a, 0x000482a8ab23b028, 0x000ca4849856d628, 0x0000dfe5317984e9 }, { 0x0000d6fb0302b210, 0x0001334cc7b4f79f, 0x0001159b13471521, 0x00027aed5a1d0312, 0x0000d3bf917b1a7f }}, + {{ 0x000739327907285e, 0x000bcd51fb747d2c, 0x000868fa43b278a8, 0x000c7515aab50f95, 0x0000376a6d9ca2f4 }, { 0x00038542d43e54fa, 0x000a8440df6f9be3, 0x000b3e6a8f41a44a, 0x000e875d99092757, 0x00005ff9309a3d10 }}, + {{ 0x0008059420aa911f, 0x000200838961a0f4, 0x000e5063f3a99a11, 0x000f51dc61426ead, 0x0000999972790d9f }, { 0x000bd6e76938fb53, 0x000e7d87396e1dea, 0x00078ceec367f734, 0x00093fd665c8bdc4, 0x000025cb654cb623 }}, + {{ 0x0008dd4397eab620, 0x00020bb23330a011, 0x0004aa716c087df4, 0x0002a0c21d23f5ee, 0x00009e0d97bf7d73 }, { 0x0006bba1a6ca60ae, 0x000e59a122d7b44f, 0x0009ac7ccf2f7140, 0x000ef35f8e6d3b40, 0x00006562071bc951 }}, + {{ 0x000eb4a728dfb0d1, 0x000ece5cfc0aee0c, 0x0005f1d7a9946ac7, 0x000b2f8b2847edc4, 0x0000f3cb9e6d73cd }, { 0x000fd7e0f19cebb2, 0x000320e26da033d0, 0x000cf4084592072e, 0x000e145256cc3889, 0x0000f9a9e6566860 }}, + {{ 0x000e87665c992681, 0x000f23b1f0df9d48, 0x0003142e7e966892, 0x00009ced48cc2c5f, 0x0000034da4569da1 }, { 0x0009723443b0241c, 0x0000decd73dc965f, 0x0004451def97585b, 0x000aa0cb415ace45, 0x00008cda22cac57b }}, + {{ 0x00036cf77e985285, 0x000e12eb72d9f264, 0x000ead77798b269b, 0x000e004d0b19396d, 0x0000b0da9787f548 }, { 0x000ed049715c4ef1, 0x00042b3e990ef77c, 0x000c4c41b2b3b279, 0x0007441ce7373e3e, 0x0000c74b8d8215b3 }}, + {{ 0x00087b9090eee8d9, 0x0001e857e977ca05, 0x000a8761cc9b0977, 0x000721b61b66ee8d, 0x00000d8c50c1bfa1 }, { 0x000f5f64b5366b2f, 0x0005b1b32ba0127e, 0x000624b6b0f733ae, 0x00003c3f2a7720e0, 0x0000f1e94afee463 }}, + {{ 0x0007d52107341dcf, 0x000da6292b8c5599, 0x000fd19ee9087108, 0x0009ff1a50c14aab, 0x0000faa98a57dc72 }, { 0x00064ed85eec4469, 0x00054303e7c253b2, 0x000a21af1562e2d2, 0x000f6422ecc834b1, 0x00005e395857864a }}, + {{ 0x0003b3e080e596b9, 0x00020c066d41870b, 0x000093d3d98894b0, 0x0003d8d35a50c20b, 0x00005c4a4156d0fe }, { 0x0006900c54967769, 0x0007c58a179e9d94, 0x00092e175b2d285a, 0x000c91d4f2daa3f7, 0x000009935a179da3 }}, + {{ 0x000e2ddd1a219abc, 0x0005b1ef76765d05, 0x0002379a97e897c4, 0x000e1ddbd4ad726d, 0x0000c1f23448c02b }, { 0x000809d5edd717e7, 0x00030fca93e6dea4, 0x000b4b6d3b35eb09, 0x000f0bca2e8a2583, 0x0000004d3d719c22 }}, + {{ 0x00066b19648d9dd3, 0x00077a3da7a095ed, 0x0000508a9178ada3, 0x0004ae54a9049acf, 0x000017300723e8de }, { 0x0006ccb81ee1f40b, 0x000b7a0de6e95555, 0x00085ef0cf517a36, 0x000de7e21cf52d09, 0x00008ba0c8b2872d }}, + {{ 0x00094348a4e6b6a4, 0x000e6b89f4ccbb8c, 0x000211dece842856, 0x000082d0ced5f44d, 0x00002f608774e31a }, { 0x000c51779a9eec3d, 0x000acc429ef8479b, 0x0002a4faf1a4db07, 0x00027fd144c13a4e, 0x00005e156474bbef }}, + {{ 0x0000f4728b20513d, 0x0003c922774856f7, 0x0004fb2782d3e590, 0x000a061166e52fb0, 0x0000b15e370c9ac4 }, { 0x0005ee9fb249de3d, 0x000dbd1b2947f224, 0x0005e22ed0071342, 0x00012adbdc827ea1, 0x0000818801cfdb43 }}, + {{ 0x000537c1f02c6851, 0x000b321bb937d636, 0x0005671375b66207, 0x000852a68df8ff2d, 0x0000496bf164c230 }, { 0x00045b0e55c95c63, 0x00007263b37f8e84, 0x000b9a90b8785c72, 0x000cc742c170e9a9, 0x0000126204cdfcbd }}, + {{ 0x0000d9a50b845667, 0x000a70683a7337bd, 0x00042f134d3dc726, 0x0004c501f1e3416d, 0x000077d800d0f3e2 }, { 0x0002ade2f283b9d7, 0x000aa506fe28ef7e, 0x00054084698d5e5f, 0x000e17b633cc5ef1, 0x000066c31b2862f2 }}, + {{ 0x000e6fbcea358249, 0x000d7fa458d36710, 0x000c1f54c790f34a, 0x00096f8e31549722, 0x0000b594e2aa6b23 }, { 0x0007b2736a31e704, 0x0001a6b3bef0262d, 0x000bb7a26d997c33, 0x000d94eecfa8c472, 0x000052cf6e513e40 }}, + {{ 0x00075c9c88b57072, 0x0006ff1716843491, 0x000e1b84ff0e5ecf, 0x0002798ff36b4722, 0x0000fd7a19679edb }, { 0x000cf60d2f9f4605, 0x00056226e6bfdb1f, 0x0001c1e53e170898, 0x0000469445ea5b7b, 0x00001577a6f96234 }}, + {{ 0x000ca2e2a9f36d95, 0x000a42d677819e1a, 0x000dfdb7473e6ddc, 0x0005aca3f7be74c7, 0x0000e0f21bb12470 }, { 0x000f16207d6ca7e8, 0x0001b88122a6fe3d, 0x0002a913e42b12c8, 0x000d30557db69f70, 0x0000ec28095ae062 }}, + {{ 0x000023dda39ee7d8, 0x00071ab63ed9998a, 0x0004075a01f290e2, 0x0004c6872a3125c4, 0x0000f904a184e285 }, { 0x0002370cab7cf923, 0x0001efc0c743032b, 0x0000ac0c798dc522, 0x00042f8074f2120c, 0x0000c5c63ec4a64b }}, + {{ 0x000b922caee9c36d, 0x0006c626cc820fb6, 0x000c4cc8181044ef, 0x000a22bf059feee5, 0x0000001293501b71 }, { 0x000509216ce29c86, 0x000bfad106bed4e9, 0x00054a98c83f1cea, 0x000260e87546e71f, 0x00009ce0ef82db3c }}, + {{ 0x0009536e2465169d, 0x000936785c9a2dbc, 0x0001a294f24b62b3, 0x00035c81dd6ae515, 0x0000b9a59ef9c27c }, { 0x0000a8075f1088c7, 0x0006ea934521b032, 0x000c66df4fa0987c, 0x00065293cf39c526, 0x0000932812328d7a }}, + {{ 0x0005b9502ede5c00, 0x0009c62640264225, 0x000fcf00804813d3, 0x0009329508dd1077, 0x00003a1cef23fb5d }, { 0x0007fcafd4d1e2ef, 0x00025b486a098170, 0x000c3fbe909cd3bb, 0x000fd6a2fa39dd3a, 0x0000301ef9be9169 }}, + {{ 0x0000d74541238bb1, 0x000a1ee346eb2263, 0x0005bb58677672aa, 0x0008f2c267dfab08, 0x00008519f05e7745 }, { 0x000b740ffbc988fa, 0x000ed3ef2c2d065e, 0x000fe8badacd3c5f, 0x00097a322ff9b89f, 0x0000270de0bc09cd }}, + {{ 0x00021b2cceb112c7, 0x0003213a8aef5181, 0x000bedd9a2d4160d, 0x000a099487bfc74b, 0x0000bfb60e023e05 }, { 0x0007a59316c50a89, 0x0003f36a01eb3d34, 0x000a17d37464fae9, 0x000fa32cf50eb1c6, 0x00003421f128bfda }}, + {{ 0x000c2da0c5ea518b, 0x000897f931020167, 0x000418a384b07616, 0x0006ae09112c5f65, 0x00005b5dd45be1df }, { 0x000c2e94a72326bd, 0x00088b6d4a1dd5d1, 0x000674f41c95d1d6, 0x000cebd483e7ad9b, 0x0000363b24c971c3 }}, + {{ 0x000e5d95c2d82265, 0x0002a075a97d231b, 0x00068f25f195c1c4, 0x00064c2277c80de9, 0x0000d8d6a5c50d2d }, { 0x000e26d6207fa264, 0x000f8b19c2492eed, 0x000062269993127a, 0x0005724d342192dc, 0x0000a4ee006ab0b3 }}, + {{ 0x0005cdcbbff59de0, 0x000fcb6137c7408e, 0x000550d49b1acec8, 0x00020df1b9a38d7b, 0x0000a28ec5df468e }, { 0x000827dcdbc769cd, 0x0006a6df65965a08, 0x0002638c0c826940, 0x00084a22060fe88c, 0x00007b733efb859f }}, + {{ 0x00066d1dfef7b4aa, 0x000f3240c56c6904, 0x000d974a461ec7d5, 0x000b6bc2b40265da, 0x00009a037291c042 }, { 0x000effe346803700, 0x00069621484469b5, 0x0008a3dfac592a2a, 0x00081bf510db6054, 0x00002ee4b2e6bbbc }}, + {{ 0x000c3f2c5b8478f1, 0x0006164ff112c32f, 0x000c7f57f357647d, 0x000fc170d8a9c736, 0x0000f62d2b5914f6 }, { 0x000853ac70c175fe, 0x000797e5aeb49c20, 0x000dd1c7318d5bf6, 0x000bfb1fcf3db034, 0x00006e90923fe993 }}, + {{ 0x0008ef1d90bd82e4, 0x000f1948b2a29d5d, 0x000674f0f04a7a60, 0x000a94e8b28f1f45, 0x0000cb6517882536 }, { 0x000ed48d66270e5a, 0x000086041dc61bde, 0x000c3919a2246fed, 0x00086105df74e8a9, 0x000062b49a271a05 }}, + {{ 0x000fc8a61448916d, 0x000c74f3a29467a3, 0x0009855614595002, 0x0005455f2de81e94, 0x0000c7e3b2cd575d }, { 0x0001458e271cf38c, 0x0006c8f06d0de9fa, 0x00049a303fe35dec, 0x000c8fbd5bbc11cf, 0x0000091b6978a5de }}, + }, + { + /* digit=13 (1,2,..,64)*(2^{91})*G */ + {{ 0x00072002d0deafc1, 0x0007568e039c2560, 0x000b74f7fa8c3e04, 0x000fa452b5f26fb8, 0x0000d5c673e4de9a }, { 0x00094308345d1eb9, 0x000e937e84fb7e3e, 0x0000233f0b08ef7d, 0x0005f8881b401d8a, 0x0000861e80d65e10 }}, + {{ 0x00076ddafe513028, 0x000b231be24319a2, 0x000bb927cde9a7fd, 0x00047f98503f7d28, 0x0000bacef2354247 }, { 0x000e4d52a90363e2, 0x00072961d7a64eb7, 0x000900d06b997d69, 0x000d4f0d5e436088, 0x0000deb49837ce80 }}, + {{ 0x000bab8085f68f7e, 0x00057b11d59715df, 0x0001dbdd17b82354, 0x000627448c7eaa76, 0x000008e08b2e2ab5 }, { 0x0009f8090d0bada8, 0x000652153765b2f5, 0x0003a01a76305a37, 0x000cd7310bdd4e08, 0x0000e04ea95f672d }}, + {{ 0x000bb2d0d12e52d0, 0x0002d9d615faa8f2, 0x00006ad2841c4cc3, 0x0005178828fa1eb4, 0x0000037e5443ad4c }, { 0x00018eaef0ca0cd3, 0x00057646caa6d2f2, 0x000e0158862a3d51, 0x0004399628eb879d, 0x0000cd6553865dd0 }}, + {{ 0x000c0571a1474d7d, 0x000eb4744ddfa350, 0x0005ceae85a5f097, 0x0009beff49ccfc5c, 0x000030d609ea9dd7 }, { 0x00057759b5198b47, 0x0006cd54b3a7b3cd, 0x0008885661e41633, 0x000b6143a35c0366, 0x00004e96b4e32798 }}, + {{ 0x0000a94af9ea7cde, 0x00069d83d30a2c5c, 0x000e36e136e23569, 0x0009d6c9b0857f77, 0x0000ee3ba9363a28 }, { 0x00091c4d690c1482, 0x000da0bdaa6ec1a9, 0x0002efb68a84b025, 0x0004cc597ba9fe49, 0x00009c2e3b6a4baf }}, + {{ 0x0001e48f047ae313, 0x0007928d2f01cb3a, 0x0003357921acd842, 0x00099fab493d6eaa, 0x0000971a98af096d }, { 0x0007186cef6f15a9, 0x00051f6c508b23ac, 0x00096ebcc85a8832, 0x000e40f38f137571, 0x0000e4475b94f38d }}, + {{ 0x000f0606c39967da, 0x0008ecee978a1d35, 0x00031bb32640de50, 0x0003dde62e3a68b3, 0x0000e08b646ad0ee }, { 0x0000f5f9d3887578, 0x00046276f9326f11, 0x0004b608425a9f9f, 0x00069fb512f521c1, 0x0000178ec5cbaa24 }}, + {{ 0x00015759ba88c80e, 0x000536b269fb37d5, 0x00042c8ed42b45c6, 0x000153353aaf7380, 0x000055e311c5bac7 }, { 0x00087e91e457b502, 0x000e9f8aa423a61a, 0x000124c369ae3354, 0x000978736592245a, 0x0000f1cbf0fedd43 }}, + {{ 0x00019c380abd55aa, 0x000a2e43d34a2e3f, 0x000dc29f25566282, 0x0003bdcc7ee3759c, 0x00006455153981a4 }, { 0x000fab2b4f7c81ad, 0x00074aade744b1f6, 0x000ff14cda1c443c, 0x000bd5111a7d222a, 0x0000a28194e30835 }}, + {{ 0x0004f489bbe9959d, 0x0004d5f4bf1d7255, 0x0001fa780b53c39e, 0x000e212069c1d8ee, 0x00009ce8effaceb9 }, { 0x0005945723762269, 0x0006d34992fb7e89, 0x000a0d0fec2418ef, 0x000dfa5000169a7a, 0x00002dfdd7e68c1e }}, + {{ 0x000876812dab14b9, 0x0007f78a4d32282f, 0x000c4f8b89ba0bce, 0x00094fe50e36e029, 0x00004692f67fcd8c }, { 0x000d3eea24df7225, 0x000ee8ed23a17f08, 0x00006374aae9a53a, 0x0006455e06d7b448, 0x00001b31dbe9cf50 }}, + {{ 0x00008ccbe2c9375c, 0x000c475b15d5acc9, 0x000f2cf48fe143cd, 0x0006c10f2c6d9c79, 0x000052c6758da616 }, { 0x0004e7f0deea926a, 0x000b8c4ba35afce8, 0x00007882ce2c78f4, 0x00030cbc15a3da8b, 0x00007993789e4ddf }}, + {{ 0x000a821ab5d22890, 0x0001522b99a72cb0, 0x00077cdf65604ed9, 0x000cad53e06de6e6, 0x000000279f3d1814 }, { 0x000490a31fb979cc, 0x000d92b7cb0b5a2b, 0x000c1473e470c4c5, 0x00054393aa7fb121, 0x0000cf1f5b79004d }}, + {{ 0x0005647406b4d83d, 0x000a77d62dd61ad3, 0x000c367b9f6b2ca3, 0x0000b8eba781a961, 0x0000c08fd788cd26 }, { 0x000f2bdb2ab6d0a3, 0x000f27b2736a129c, 0x00021de8172c3e3d, 0x0004959c5a631370, 0x0000e07bdbb63a83 }}, + {{ 0x0007d51e7e42a51e, 0x000f444c5f95cd80, 0x0000ecd677766dc5, 0x00040656dda8c8af, 0x00007a567e594477 }, { 0x0001a1b8848c495f, 0x000ff6ac2ccbcda6, 0x000f23870b5b7597, 0x000568bd43bc1923, 0x0000956dd9ebf318 }}, + {{ 0x000e676223b52ac3, 0x000ab08700230ef6, 0x00014eb24bd5447d, 0x0003015f20af585a, 0x00006fbd09543e40 }, { 0x000716998cd613d8, 0x0004d7c4e774d3fe, 0x0002c4f9938e4db7, 0x000fe92a582a3851, 0x0000708acb1bfba4 }}, + {{ 0x000ce0a3b86fb6ef, 0x000e671fb1675839, 0x0008348bebd1d282, 0x000eb8619dd011c7, 0x000021f9f273dcf9 }, { 0x000c00295b68f0cc, 0x000a689d5ec67385, 0x000a2c40c61023a6, 0x000546819a3b49f9, 0x0000308d7cb76e31 }}, + {{ 0x0004ada5e11c599f, 0x00075fb1d4bcc766, 0x0001e0ece31035ea, 0x000b3720fb07691d, 0x0000d42fa02b82da }, { 0x00097f2f7329b17d, 0x0007c4e5b544cf50, 0x0005f3a7e48b861f, 0x0005af7f4b4a13a1, 0x0000b8d9cdcafafb }}, + {{ 0x0008e1b8c2d74b0e, 0x00029d8affa8208d, 0x000b0347f239c200, 0x000ef527fc3c31b0, 0x0000983c109b8ea2 }, { 0x00039cab167742fb, 0x0001d15c8418682f, 0x000e8a2c7280fa2b, 0x000566e82541d487, 0x000043cc4696534f }}, + {{ 0x000f6985d44acee8, 0x00019a0e68db0554, 0x00086ac9b300b789, 0x0005c1284f356975, 0x0000a5ac2e22d6a3 }, { 0x000c9d95d6ef7c99, 0x0008615132ed17ab, 0x000403a5d02745a2, 0x000355766510ed0b, 0x0000b2d6d88461f6 }}, + {{ 0x000ddcc3fd82271c, 0x000b36e19abd09e2, 0x00012214bade2615, 0x00022eace0b4097c, 0x0000faacf01c7b54 }, { 0x000b860163208096, 0x000ba3d9e8153b8e, 0x00018bee1638b3c8, 0x000587c0c3324fd0, 0x00003de457dc65bb }}, + {{ 0x00039ed2562fdbc2, 0x000fbd1972f07f4b, 0x000e1edcea95283b, 0x0003623a6e5579e2, 0x00009e02c169c92f }, { 0x0006d17da40df3f7, 0x0006d17a2f62e55f, 0x000e24fe04fed313, 0x000f732f96b9ab38, 0x0000e962da3c756c }}, + {{ 0x00012c0401631864, 0x00079bd0f968c006, 0x000b9ee0840a483b, 0x000826e4da03da7e, 0x0000d17d0ff62307 }, { 0x000c74ec710cf280, 0x0008493109f5df10, 0x0000743a88df63de, 0x000c729b3fbda403, 0x0000893136d324b9 }}, + {{ 0x000c593df45e820c, 0x00049646edb951fb, 0x0007c2c9848e3595, 0x0000b05965f4a9d7, 0x00007059afd6a155 }, { 0x00047ed87de67e90, 0x000cbd46af3641a8, 0x000ab91c654ee10d, 0x00064a3047802200, 0x0000176fc6f6b4a5 }}, + {{ 0x0008923bbab39469, 0x0002e4e6d6711921, 0x000ac584ba5544c8, 0x0002f1e971e14100, 0x00005ee0cd3a9fe2 }, { 0x000af777f11421a3, 0x0004599bbe7e86e1, 0x000e0668efcedb0a, 0x000b42b3e4a1510e, 0x000081d2b6ac6370 }}, + {{ 0x000f3a93c5729e80, 0x00034004ef657241, 0x00091cd213392d1e, 0x00061a9943e04a4c, 0x0000714ea218efe2 }, { 0x000dbd736683a3d4, 0x0003afe3923555b8, 0x00089059de839c3c, 0x00012bf57744bedd, 0x0000f135993bf2e7 }}, + {{ 0x0002c8361b7a2aca, 0x0006699df1ea40e2, 0x000b350de8491faa, 0x00043ee12b3f24a0, 0x00002727eb357cdf }, { 0x00035a753655680b, 0x0001497f9c0d95c1, 0x00027ae958781188, 0x000c30ac38846117, 0x0000025367304961 }}, + {{ 0x0004c10cf8896213, 0x000ddf75f7ae2f68, 0x000d533960a9a46b, 0x000722eb615fc6e1, 0x0000ae866b88a248 }, { 0x0008a74fcafc168d, 0x000df95a73fbbd0f, 0x000a54e81f1dad9b, 0x0000f13437f24e0c, 0x0000c84c23966c22 }}, + {{ 0x00040a120307c01c, 0x0004985c9f842969, 0x000aa7b866ad1714, 0x0008fafd07fbaa8f, 0x0000621c9f76d164 }, { 0x00092f8868ee11e7, 0x000fa2055a4aec84, 0x000492a7ba874698, 0x0007a81ab550e9a7, 0x00002f1e26b56e34 }}, + {{ 0x000b6155609a0534, 0x000e1adb2d8f3984, 0x000c102373f2ce23, 0x000b3e3d6f13b51c, 0x0000421c4ce957c1 }, { 0x00020f448ada30a5, 0x000a9850a52c1c82, 0x0004fea81b397d5a, 0x000db0ec6eab1a69, 0x00001f18e2afb1cf }}, + {{ 0x000589fb928c5f43, 0x000ab31b0e63d348, 0x0002245b5418c388, 0x0008872a4f460057, 0x0000c3c71e7aa249 }, { 0x000bb0696dee0485, 0x00070cc6583553ae, 0x0009f6a5e077bc6c, 0x000ffc1520879094, 0x0000bccee4609921 }}, + {{ 0x000c457b0b8cd23b, 0x000e4deb9718710f, 0x0007f72f2ee9cd44, 0x0006782c138f1bed, 0x0000d9595242d62d }, { 0x00095c766178c8de, 0x000a1d4883841e20, 0x00092f15dd20214f, 0x0002f65958ec25b3, 0x0000f894c054a41e }}, + {{ 0x000a9fc894303e0f, 0x000b95c4bdc56104, 0x0003ccfeea6d613c, 0x0003aaeccb2885fe, 0x00002d45649bb35b }, { 0x0005fcc99d08171b, 0x000d5cd3ce299af5, 0x000357f6c183b368, 0x00076c08f195be9b, 0x0000561cacedeacb }}, + {{ 0x00098ba31d719d2c, 0x0002f4c721a4593f, 0x00047d0307a4b9cc, 0x000eccd14ed1e9a1, 0x00002d73755fff02 }, { 0x000ffb3c0ec394be, 0x00055739b128eb4e, 0x000de44d733bbc94, 0x000accfd37801214, 0x000050af6c641c91 }}, + {{ 0x0005f825c4f07a21, 0x000e67cc7bf3827e, 0x000a61fffdcaef2b, 0x000eab454f2165bc, 0x00005d08b9caa394 }, { 0x000659f053bb7164, 0x000eb67ffed4d6f5, 0x00085eb0445c885a, 0x000d096f1bc60957, 0x000009efc070a594 }}, + {{ 0x000c64a71a41fd04, 0x0007ec36044d63bf, 0x000810766586a678, 0x000b5b8f6a608940, 0x000015d2319af57d }, { 0x0002002ddd63570e, 0x000cc8083dbedba1, 0x000219e65019b26d, 0x000e6e1de804ae11, 0x0000b12ba3767b3e }}, + {{ 0x000d0a9304441d8a, 0x00024895dd21b96d, 0x000af239cf49a271, 0x000aaff7a8b99f28, 0x00007171f2ff03da }, { 0x0001f37bf44ca0c6, 0x000af224e233de42, 0x000109432cb14982, 0x00098cdd9192445b, 0x000083d6a7497fb5 }}, + {{ 0x000a5ab0888caa4f, 0x000f449f8bd2bbd7, 0x0008ce0d68d4a75f, 0x0005367f11d902cd, 0x0000a44f775a6959 }, { 0x000b840d135a5237, 0x0007406f7a09e03c, 0x0006666eb4106a14, 0x000c92f572f8a197, 0x00008f690387b3a9 }}, + {{ 0x0007753ef58b6176, 0x000379f7db326750, 0x0003002b30262a9f, 0x000bdea9478fea13, 0x00009fe0ef597427 }, { 0x000109beb4404306, 0x00054ed9c0bea786, 0x00095b35866fc305, 0x0003c6c6d41bb398, 0x0000d3e8797ead29 }}, + {{ 0x00025549467a0bf6, 0x0001bea3b6d1145a, 0x00090602dd0e0635, 0x000da6f62b6271ea, 0x0000e2394cc29faa }, { 0x000777d45fafe2fb, 0x0009d5e99bfe35f8, 0x000ff3d9f99729e1, 0x000a51d329770ccb, 0x000030a49d96b3e2 }}, + {{ 0x000b3786e5ed1d59, 0x0008122d847999db, 0x0006081ad64caea1, 0x0008c00a103aa33b, 0x0000197d6168ee3b }, { 0x000f6688f1378163, 0x00070ce6961f247b, 0x000e4392b44e7734, 0x000264b4f20aa85a, 0x00009914ec874ca0 }}, + {{ 0x000e24bf90175fbf, 0x0004f867469ceadb, 0x0003727b0ab89a54, 0x000f4fcc71904c52, 0x000031f4ae121f33 }, { 0x000f170db279ffb1, 0x0005690bd98764be, 0x0006b3571bec445e, 0x0009033c7a0f45f1, 0x000097d5d9760ff1 }}, + {{ 0x000ee1767e237834, 0x0002da029e0ccc17, 0x000de1cbc04beb15, 0x0009f0422cd53e87, 0x0000d2d72323a5a4 }, { 0x0003f6bc86ce1e10, 0x0002259648945af4, 0x000ce1d14415bbd3, 0x000439b8c9e45a5f, 0x00000c39573f5251 }}, + {{ 0x000f428fce95e48b, 0x0009848f2142e791, 0x0009a7a393bbf194, 0x0009f129b17d8925, 0x0000aaebd8ce27b4 }, { 0x000b5da0d65b712d, 0x00069010db06dfe5, 0x0000f54602f5c09f, 0x000bfc71bbfc16bc, 0x0000a4f9df66163f }}, + {{ 0x000e43c42cf85546, 0x000c44856ef96a74, 0x000849e83415f3c7, 0x000ea2a0db47eb84, 0x00000d10f03005f3 }, { 0x00020e52e37d20d6, 0x0005431d1a4aa2a8, 0x00083df6a0eb24b6, 0x000ddb47a440d466, 0x000048087cb057c8 }}, + {{ 0x00097c4c88758f6e, 0x000f1cf25e51b09f, 0x000e26a784d0b41c, 0x0005c8b0e180cd2b, 0x000009259c42e92c }, { 0x0008e2f4d6cd7faf, 0x0001824c03dbb7ea, 0x000458a9c70dd19e, 0x000e82253ae04376, 0x0000c21fa7e45f39 }}, + {{ 0x00099cfe2f0814de, 0x0007447d95ea1682, 0x0009ffd3781b0f15, 0x00082a3691758fad, 0x00004a8af81a2215 }, { 0x0002e22a81992dca, 0x0005417b0a98927c, 0x00005d1f1a13cbfc, 0x00047eb6a2e26e1d, 0x000016512c4f1dd8 }}, + {{ 0x0008cdcef72b2cef, 0x0000afbd90a060ab, 0x000b0225b18e21e3, 0x0004a7ccf496a27d, 0x000094a48f740aaf }, { 0x0002d4d14b785966, 0x000a937436a0bbaa, 0x0007d3b12efde960, 0x000dbdccf263fa10, 0x00008368434e4361 }}, + {{ 0x000e4e4a56705d8e, 0x0000f6812e9a1bcd, 0x000b811fe812283e, 0x0004f583861d9075, 0x000054c73bae5329 }, { 0x000eb36201e51c99, 0x0007b4ad673c0175, 0x000e10baa3e900c7, 0x000b52456667417a, 0x000012e79d4dacf7 }}, + {{ 0x00069ef68c67b87f, 0x00046043a1aa9145, 0x0009203d62e8c5c3, 0x0000f935c175a129, 0x000001fa1a6965d8 }, { 0x000d9e2a0a581188, 0x000536363a113b4a, 0x0005eae52d5c3801, 0x0002fa5eae3bd3f5, 0x000066503ca8481e }}, + {{ 0x0009192ffa7d59dd, 0x000bb4c508b65e9e, 0x000084d870d6c300, 0x00060a3dabde4a1d, 0x000027aed4e0a762 }, { 0x000479a99a505811, 0x00085965099fe925, 0x000bb6adfbb04f0c, 0x00038c96125ddb65, 0x0000b41360e14375 }}, + {{ 0x0004fb15b0510f55, 0x000f905013fe1620, 0x00025e32316b7fd8, 0x000feb39fbe11dc9, 0x0000d62b1f4b95aa }, { 0x000a88e26620cd37, 0x00028c213c999279, 0x000fa35a08cf8550, 0x0000d94b4a033e9d, 0x0000037e5cb1f698 }}, + {{ 0x0001d933d32c017b, 0x000346254671ff6c, 0x000a9dcc5230cb16, 0x00075f8c04bd4241, 0x0000bb7123eb38ba }, { 0x000c6641dc5f7a5b, 0x00051f3689e2c70c, 0x00067a54bb55957b, 0x000f99be592aab0e, 0x0000100ca18fcdd4 }}, + {{ 0x0002150673a6e180, 0x000f757d31e535ec, 0x0005c5ade79e2400, 0x000211ffa04cc43b, 0x00009f71ec100187 }, { 0x0005a52faef35750, 0x00071e08d68b0139, 0x000f584d694447ac, 0x00028f464a9d6816, 0x0000d379add57aaf }}, + {{ 0x0006293a2dec1ae3, 0x0003e600e1cb64eb, 0x0005c8568fd06998, 0x000961893004828f, 0x00004d492af4542f }, { 0x00028e52e6b40b4b, 0x000a421d145d2d9a, 0x00031901531d2a2a, 0x000c3da2dbe6225b, 0x0000a8127987847e }}, + {{ 0x0008303593dd6e66, 0x000629b4c371e6da, 0x00048ebed994bfaf, 0x000bac981cebe067, 0x000016931e8d75c6 }, { 0x000130cefca57500, 0x0005b0a7e37a9bb0, 0x000e2a5edfc6fbfc, 0x00054b1028c56f61, 0x0000ecf694a5348e }}, + {{ 0x000752496e016bd2, 0x000cda0bd865cfbc, 0x0007f50cd4695497, 0x00052a0244b03e57, 0x0000637b077b76b9 }, { 0x0005ae9668f90345, 0x000e83b564beb6b0, 0x000033f32f9fd853, 0x000f2bc260db1d11, 0x00009c49e1c7cb1d }}, + {{ 0x000633e728e450f7, 0x0007637c92544f2b, 0x0002ed2450ac358d, 0x0000057cecc08435, 0x00008ef9d3967336 }, { 0x0002ca7d5cb05be9, 0x000ed9f936c581cb, 0x000dc16f479d4101, 0x000e8a15b96e7b8c, 0x0000175bf8daeb7d }}, + {{ 0x0003f3086f736b1f, 0x0000641f50240258, 0x000960cfd0c861ef, 0x000f6f99f47bc576, 0x0000686ca9cfba8f }, { 0x000b7e65f9f1ce62, 0x0000b94b587579ab, 0x0007756906660826, 0x000329dcfec2d22d, 0x0000255e257067be }}, + {{ 0x0002c8ff8f6efa40, 0x00054776652a220f, 0x0008e3cdc9f20d82, 0x000f5bff361618f8, 0x0000f1cc52b08e1a }, { 0x00062b9b103f16d5, 0x0003ead8efb4d3e2, 0x000fa7ebd284ee8d, 0x00051001c937b5ad, 0x0000d93b71b723eb }}, + {{ 0x000588993b473591, 0x000bee4c43ff28b8, 0x0006e4ddb53f06fb, 0x0006fd4c27647662, 0x00006c003ddbd070 }, { 0x000832749a9bb5b3, 0x0004ed68f938829a, 0x000d96ab08b23bff, 0x0001781c7996627e, 0x0000272b5788c7f5 }}, + {{ 0x000ddbacedcf37d2, 0x000bf7d4bf503533, 0x000b178bb90d5ee0, 0x0009dd96685dd726, 0x0000ebe65a8ae8c4 }, { 0x000da2828c5f8f65, 0x00074a412f22e85f, 0x00066f647430480a, 0x0003ab137e2a2486, 0x00009656ded0c9e4 }}, + {{ 0x0005aaf668ac79d7, 0x000433ea2dee7a63, 0x000c3da02c1bc912, 0x00031cfaddcde2f3, 0x00001022732669a2 }, { 0x000b89dea0558e1e, 0x000e6a9e5aa049a6, 0x000d25865fe4287c, 0x00021cec3e74083a, 0x00001aa8b5c32deb }}, + }, + { + /* digit=14 (1,2,..,64)*(2^{98})*G */ + {{ 0x000be00843e7b7e6, 0x000b723fe67ba665, 0x000037ef26f02671, 0x0004513139145076, 0x00001e80333c65b0 }, { 0x00067e7cf2b69e56, 0x000e805d53ff04d3, 0x0003276aa2047eed, 0x000006ba0bf7ccdc, 0x00008a2d8826cd00 }}, + {{ 0x0009356a79debb8e, 0x00065b7b0dc85957, 0x000825e834b42de6, 0x00021f4a727de460, 0x0000c18079e2bfdc }, { 0x0003b0bfe5e20c23, 0x0000045f5f9a0529, 0x00087fe98313de54, 0x000411dfc1a8b0f8, 0x00004e039a515ca2 }}, + {{ 0x000607c8357f525c, 0x0008102c024789cf, 0x00094badb3c6d4e2, 0x000e13cdf90dca9e, 0x0000f013f32caca4 }, { 0x000259ced8a8fa9f, 0x0001f97c2c99b76f, 0x00062668c42a120d, 0x00009098c6576ba6, 0x0000ddf41abff43e }}, + {{ 0x0009d20f961c2e82, 0x000b85dc8de610b8, 0x000ade101437f35a, 0x0007eebc5e8c515f, 0x00002509d1321032 }, { 0x000842e3dac8ba0c, 0x000fd66098583ce0, 0x00048bafc3fcb163, 0x00076414aa2eedb9, 0x000007db9f48bc83 }}, + {{ 0x000c04346f8b4a53, 0x0001fa33b9e2d7b5, 0x0007687de490598f, 0x000ca4a917cc60b2, 0x00003bbf0d26c8f8 }, { 0x0008cace3b4eac97, 0x00066f2614650a98, 0x0007392b98719d3d, 0x0000e25835e35b53, 0x0000792a37dc963a }}, + {{ 0x000a9c2a39c1acbe, 0x000e5753de92e23f, 0x0007651f8ca95eef, 0x000048a7eccb03cc, 0x00005893c9a2ca16 }, { 0x000cba2866a3fc9d, 0x0009b236724ceedd, 0x00073c72dabcd16f, 0x00061668e3a90225, 0x0000f7789359b346 }}, + {{ 0x000a35b6a1ba7e68, 0x000bb7f788907328, 0x0008c9c6fe382aca, 0x000b618d6c443bef, 0x000027c8cd8a0530 }, { 0x00044a35a0bbe471, 0x000d61665e9a90a2, 0x000d40e5d11ff8ff, 0x000053d3b45bf380, 0x00001226d9bd6c94 }}, + {{ 0x000595c4b3f8dbd9, 0x000359d74cd06ff8, 0x000f270e29f8a825, 0x00087d12b9c17c7d, 0x0000b2e80e811f87 }, { 0x000db27ce43a86c1, 0x0003fca990f62ccd, 0x0002ec59bf016957, 0x0000628f5d9bed21, 0x000027d55a4ba6a5 }}, + {{ 0x00088b9aa14b8da6, 0x000a46658926ddd8, 0x000907f47b17a1a9, 0x00091d4b8138b2d4, 0x00001b4b6e70d892 }, { 0x000f508ea42a5543, 0x000c04e3a96e3551, 0x0008d3bf2ff6a996, 0x000cbf37aba65b0b, 0x000012115142acd4 }}, + {{ 0x000f55cded9f6e0e, 0x0000c49747c866c7, 0x000d85c2d274f8e8, 0x0006e2808bbb1e5f, 0x00001441178e9e4d }, { 0x0005bd1a0a03f0a8, 0x0004495f38c85f0f, 0x000e96069eb98983, 0x000220e6f5b589fd, 0x00000accc96b8e0b }}, + {{ 0x000a8761f7872bd9, 0x00061e55e915918c, 0x0008bc714273ddea, 0x000c0209c6158d6d, 0x0000e6410b2c4167 }, { 0x00091666024d7e90, 0x000b9c29ebbc601e, 0x0000f7ad482d61b5, 0x00067a752ef3b0f4, 0x0000530ed4817b10 }}, + {{ 0x000d03f718913dd5, 0x000c2befc15aa1ee, 0x0007c847102cc8f2, 0x0005c8a1240d1254, 0x0000fdffe724edc0 }, { 0x000f68ea1cc2db7d, 0x0003e6c4a02c4997, 0x000d509587b544b9, 0x0005f5faff725083, 0x00002c702007fc4c }}, + {{ 0x000b3261f5dfe89f, 0x00029c7b7ffd977f, 0x00048543b82d075f, 0x000575c83ec104c3, 0x0000ac684c28b97a }, { 0x0003997db2fbb677, 0x000d260d883e5448, 0x000009297105579d, 0x0001fe89913520d7, 0x00000d81132cc8d8 }}, + {{ 0x000236a324c0abe2, 0x0000f060dbd2c016, 0x000abd6646100dc4, 0x000ed57214f8eea6, 0x0000ebb2e4bb264d }, { 0x00004281091086cd, 0x0005284a16022e9a, 0x0007a38f5aa9551d, 0x000c6229c0c18d87, 0x00007ce409a2371c }}, + {{ 0x000ceb5971a6fc75, 0x0007c5e9afdfb3cb, 0x00029ef45d6dc1e8, 0x00085c89749a5614, 0x00009b93cd3762b8 }, { 0x0004b4099139dd59, 0x000ec5fb269bbc4a, 0x000e450853d0fc8d, 0x0003b57867cd85e1, 0x00004a5c6d138b15 }}, + {{ 0x000d112927153319, 0x0003f1d26ee83821, 0x0005188c3a678749, 0x000f5953dcf17dcc, 0x00006ee6a46384a1 }, { 0x000b8bdb94fd5a59, 0x0002b7750491a746, 0x00019255042413ef, 0x00049dcab687fa76, 0x0000d3bdf3fd8d81 }}, + {{ 0x000d8811182f36a0, 0x000bdc93bf834aa0, 0x000941e46d331d24, 0x000001cad0437181, 0x0000a5e4336ef208 }, { 0x000560a8b653f914, 0x000fb82f48c8e33c, 0x0004dc5e9825a5d5, 0x0002e1fe5a11fd07, 0x00008d6b831c2888 }}, + {{ 0x000054303f46b6d4, 0x0004be23929d0979, 0x0000029d58b20226, 0x000b8f9ac3885b29, 0x0000aac58d89d493 }, { 0x000f3d6452261dfd, 0x00021440738042ae, 0x0006ce71fff79069, 0x000abe21b98e6a97, 0x00004b0f9c9efeab }}, + {{ 0x00018d761d16f241, 0x0006d82b6d72d285, 0x0001a44aca62ed09, 0x0009cbb1c77ca9db, 0x0000674f75f7e28e }, { 0x000e54ba56fa1c83, 0x000f10ef0d1b5ce8, 0x000c5b965d2e6f98, 0x000d8b082717039c, 0x0000b2abaa683928 }}, + {{ 0x00073eb5de6f16ea, 0x000838af7f91d0f2, 0x000180eba14a2d69, 0x000c07d543b61b0f, 0x0000a48e18374f25 }, { 0x0002413c252f7b56, 0x00055f77b4e4d538, 0x000f5bfedb51fa0d, 0x000d51bb3b7ce66b, 0x0000bd4cf4850d41 }}, + {{ 0x00058897dc9c0e4c, 0x0007cc4e0e8ad934, 0x000e1a22eeb0bf71, 0x0002afaf3b5679ae, 0x0000e7087f871d8f }, { 0x000917072350b58a, 0x0003f3089d0caf3f, 0x000e7f4d276ecde3, 0x00008251839564c5, 0x0000087315f576bd }}, + {{ 0x000446bca9aa0ee7, 0x0001c17e4515fa57, 0x000486c5bc65bdfd, 0x0001f2065f85cff8, 0x0000dbfd705501c3 }, { 0x0002113ffe594d91, 0x00076aa6d63184f9, 0x00079416c3a80498, 0x00070ef357c1a46a, 0x0000f69954823f55 }}, + {{ 0x0000c22f68309ca1, 0x0003f8312e7e46ca, 0x0009b5503c6d1a9c, 0x00023dd0469d0a37, 0x0000cf8b19a3e343 }, { 0x000c392312daf2bd, 0x00043514d2c7d8fb, 0x00030fa9a7d2b439, 0x000fe3577bc45ced, 0x0000ab008068f4cd }}, + {{ 0x0005fe87134cd0da, 0x000a17e3cc8ac855, 0x00078703ee282882, 0x00067052f8d725f5, 0x000030f3d852516a }, { 0x000242d3ae1ef785, 0x0007fa96ab6b01c8, 0x0008fa96637638d6, 0x0008ee49b69a02a8, 0x0000dfec375d87b0 }}, + {{ 0x000c24d8d4d1fb51, 0x000be65d62105e5f, 0x00093104e58c9f41, 0x000009e09bb222bf, 0x000003c8d01fc99b }, { 0x000f5d057a938685, 0x000eb85d41299673, 0x000c9cdf24c8ca30, 0x0005254a3db81c3c, 0x0000a0f18edf4572 }}, + {{ 0x0000a412c93e0e06, 0x000ef6d88c546946, 0x00053a654474d349, 0x0009ff9d1b26baba, 0x00004c0a14f6e6e4 }, { 0x000e419776983108, 0x000e18b54b25039c, 0x00029a3cd9d75545, 0x000cfad94d4ee67d, 0x000011516306d1e2 }}, + {{ 0x000a4bb3d86217f3, 0x0007c851c203a40e, 0x000600b1dbb8dedf, 0x0003f9ad230d352d, 0x00001526e5ac6533 }, { 0x0006ec4cd010437b, 0x0004203b98e782b8, 0x000f7f38453d3262, 0x0001381c8ac958c3, 0x00002e418f417cec }}, + {{ 0x000f921d710cc9ab, 0x0008a269be47be0e, 0x0007db96e1305cfa, 0x000a2bce5323b7dd, 0x0000b7178cc6492d }, { 0x0006c2921097b191, 0x000d0fc3c4b880ea, 0x000df1d178576177, 0x0003f6e393f6f914, 0x0000ac9a7fe9fa67 }}, + {{ 0x000e1999144d70e0, 0x000616651b8d9a36, 0x0008a1e1749d7d0f, 0x0001207ce42531bc, 0x00005552258e045d }, { 0x000453c20f47dfdb, 0x000d63dd5d369463, 0x0005c23b3f858e11, 0x00030f7308255dc6, 0x000097f02a0dbe3f }}, + {{ 0x0000448da6108dbd, 0x000d0c089a6ef014, 0x0000581351df8a6b, 0x0008d1c244dbcc3f, 0x0000ec04c9855b4b }, { 0x000882a424d7120b, 0x000876d4ee343e2f, 0x000c9865fedd1f17, 0x0006f503f6343f1b, 0x00001fce589b6c69 }}, + {{ 0x000b6c5db412e00f, 0x000c6f4e7db9f57e, 0x000950cbee732732, 0x000e651b17dfba38, 0x0000cfa203fd495c }, { 0x00067ce8c320f904, 0x000ea661cdfe0491, 0x000e2dd72666a257, 0x000f366ad9baac0c, 0x00006dc6d3d1e884 }}, + {{ 0x000bc95725aee3d5, 0x0002ea553fb2b560, 0x0007423c28530356, 0x00000bbd96ce141e, 0x00009b1c7fa39ddb }, { 0x0007326ca7661923, 0x00044911220fd06a, 0x000b99589f2eb8ac, 0x000d2c8a9716e3a3, 0x000016f0b4225082 }}, + {{ 0x0001690ce1657980, 0x000051ccd090c436, 0x00038fc32ca6d826, 0x000d1fbe4561e8f1, 0x00005c54597e892a }, { 0x000bc9fb209cabb7, 0x000e3370f009052d, 0x0000145c0e0d8cf2, 0x00051250f258758f, 0x0000f2608411d361 }}, + {{ 0x000532da0afeae19, 0x000c46e246518404, 0x00046092c0b1374b, 0x0001c3038dca45a8, 0x000033c9b87864ee }, { 0x0007d1a0f94114b6, 0x000a8a51999a6bf9, 0x0009c69a51f582fe, 0x000472a933aa9505, 0x000027a0c0040cfe }}, + {{ 0x000422b0ea14fa4b, 0x000a0ba2e1c21dd4, 0x000be844e6da17b6, 0x00005c51ff41b29f, 0x00007397b90d379e }, { 0x000a349d39ffa32b, 0x000651c92f1bfc4c, 0x000f17ac719f4a41, 0x00085fbb806ab1f8, 0x0000fbba41e33ddb }}, + {{ 0x000e959ff7b2ee8f, 0x000ae32343bf1a9e, 0x0007493c06b9bdc7, 0x00079d6b48fd3bd2, 0x000098c8ad76a47e }, { 0x0004498bbccb8f72, 0x0006af89706dc3f5, 0x0001c204bf7138d2, 0x000f8d1c8737db0e, 0x0000b2806c0cc363 }}, + {{ 0x00065b514edf386a, 0x00009faa29e24afb, 0x0003ba4f3652565c, 0x0001ed0cefcc3c45, 0x0000b4260964a621 }, { 0x0003d84d2f4de65c, 0x0008bcf2193e3934, 0x0001b16c00689684, 0x00018dced60e9a4e, 0x000031f1675d66b3 }}, + {{ 0x0000da1a5636e5cb, 0x000e641ee398a212, 0x000ec2e2ab685487, 0x0004e95eab88a56a, 0x0000f1202655d900 }, { 0x00084fcb18da2d59, 0x000409d65bf0384d, 0x0003850a4cf62127, 0x00046220471a14d2, 0x0000aabe22696247 }}, + {{ 0x000c58b582f24ca4, 0x000ef7bfa44b4794, 0x0009104047cc8e91, 0x000a3e3275350549, 0x0000d5c4cf012836 }, { 0x000c293a547b773e, 0x000f8d88327f246f, 0x00095154b8cc443a, 0x000e8e6f2c3a3bfa, 0x0000876c2770053d }}, + {{ 0x0001a4c61890bb48, 0x000ed2b5acf6e10e, 0x000cce418caac285, 0x0009c327fc50343f, 0x000054fa09fb1487 }, { 0x0001219614c932a6, 0x0002acd33425d0a3, 0x000d6a5a2e6ad1f4, 0x000390c0b306399e, 0x000069c28a3f1ea7 }}, + {{ 0x00080094dc4fe82e, 0x000926487f5d28b0, 0x00018e21e3213666, 0x00082bf5f11392d9, 0x0000b5fd9212666b }, { 0x000eebe7f030377c, 0x000dc8c208ba19bd, 0x000ea6270a941ac2, 0x000bdfd9ad3e7570, 0x0000821cf9f8f5d2 }}, + {{ 0x000233c3bcb79698, 0x0007798e5c298ff6, 0x000dd44472627899, 0x00035021617f561b, 0x00004d4a92c755e1 }, { 0x00039704dff5bf49, 0x000cca192e3a2cb7, 0x0001729f4c39863a, 0x000cd0ab1961bcfd, 0x000099be056ddf36 }}, + {{ 0x000983e996aeb001, 0x000fa587e7b754b2, 0x00017e82e022cefc, 0x000a043522336079, 0x00001a6d17a03274 }, { 0x0000980f9163017c, 0x0006929958d881dd, 0x000255946bab7a49, 0x00009d6d1026bae0, 0x00008bccd8fe0233 }}, + {{ 0x000d2d5a30705fba, 0x000888cd5f30851d, 0x00064d220bffdbbf, 0x000fd8ea494dfed2, 0x0000fde12797428b }, { 0x00094c486a8e2a7a, 0x0009bbdd44cc7a57, 0x000d43c60acfc916, 0x00005ddc94ba6b20, 0x00007845c76e4d96 }}, + {{ 0x0000a4725a093707, 0x000cdcef21496e4d, 0x00081bc4f0d7dfee, 0x000567a2ff951d19, 0x000004d0a3778435 }, { 0x000aeb8750fba3d2, 0x000d0b50e3bab661, 0x000590c1a91e5597, 0x00060f29e462caff, 0x00005cc394b808b3 }}, + {{ 0x0001170c04952602, 0x000ef2f2f7d0ca2a, 0x00083f53c32b2b0e, 0x000d4ca2f6d2e812, 0x00000b742e457e8c }, { 0x000c212c2c471b8d, 0x000904f7805a99ca, 0x000620057f79b6ee, 0x000379240bde56a3, 0x000033f6108aa3ee }}, + {{ 0x00052e379c819e4d, 0x0001ef315697be5d, 0x000b57359024af02, 0x0002f60af6f0545d, 0x00005709fccda5fb }, { 0x0001cedd8c6a019a, 0x000c747f462c23e6, 0x0007e4ed47245af3, 0x00083d34fd44a429, 0x00006bc4a68eae78 }}, + {{ 0x000f0076f4926fec, 0x000495699f241d77, 0x00068119bda8ce25, 0x00019f891ee4adbd, 0x00003a1ea5347618 }, { 0x000550518bb90e9b, 0x0007dba9264da3d1, 0x000d4d575100188b, 0x000532e716eb2d2c, 0x0000e97178dd93ce }}, + {{ 0x00024db6c372c507, 0x000dbf057d0b70c4, 0x000e8ac584e05e23, 0x0007c63eceea8612, 0x000024b1bdc12cd1 }, { 0x000dd6f3a8ca9d0a, 0x0002fd116213fd1b, 0x00045acc53e70ec3, 0x0006f4621c61e7fa, 0x000085b4cbd42400 }}, + {{ 0x00099a3c67370e5c, 0x0003a675d97f7e2b, 0x0000f571b84ecef2, 0x00083067a31db0fc, 0x000017a0dc2e6d1d }, { 0x00085346d4a4a2c2, 0x0009d5536b1ff923, 0x000718d7f1b94518, 0x0005a30632973857, 0x00001c47d9a812ac }}, + {{ 0x000f7d00d2868ab4, 0x00011ef31631f9e0, 0x0000f428795737fc, 0x000994421bf91da2, 0x0000273abfaac3f4 }, { 0x0004d97e8ccbeaa3, 0x000c93bd8fe79da6, 0x0000704c4b09c411, 0x00014c02acc5e8c4, 0x0000b4ad51bd9104 }}, + {{ 0x0002ef029032eee0, 0x000f97302f1cd1e1, 0x000b0ae310148f43, 0x000a922f7ce87eac, 0x00004d8e4c69e3b3 }, { 0x000bf1615a2e8a8c, 0x0003b4e1bcea6e23, 0x000400c7cb616594, 0x000051105d0b5402, 0x0000087338e47a50 }}, + {{ 0x000f0469962cc718, 0x000da79f8a7a838e, 0x0000165cecca8079, 0x0005857a111aea63, 0x00005efd15955b73 }, { 0x00066e898fcc53b0, 0x0000ec9d8080711c, 0x0009d97055e6ac0a, 0x000ce1126e1aec60, 0x0000c1f2ccbe262d }}, + {{ 0x000ba8ad1ed135cc, 0x0009dc408811fdbf, 0x0007341f43636885, 0x0001975388bfe1ef, 0x0000b69e74c27446 }, { 0x000526bed9600a55, 0x0007bcebfd2fd5bb, 0x0001b9bf4e885c17, 0x000c7cd530b839a8, 0x00008c55b30c7f4f }}, + {{ 0x00038b8ee0eac787, 0x00034c2385644641, 0x000778ba2d92e1fa, 0x0005103789e3ff94, 0x0000d8a525b4c26d }, { 0x0005fab8823fdd61, 0x000c24c4123c4899, 0x000864697a151ca0, 0x000c3fea366e6936, 0x0000a8b00c976bf9 }}, + {{ 0x0002f26699c531ca, 0x000a607eafbb1e1a, 0x0006446913f9ed17, 0x00084c378d75c9ce, 0x00002f579798942a }, { 0x000be4b0873a045a, 0x000c4cfe469b17ae, 0x0008fb488e3bb18c, 0x000c6b73f6254577, 0x000013dcc4276048 }}, + {{ 0x00017c8bfcafa626, 0x00038d78cdfedb1f, 0x00070ca48e783c87, 0x0002c1ec5a535c07, 0x00007254a1aa9e62 }, { 0x000e719e57fc7500, 0x000412efb11fbc42, 0x00026639a10db9dc, 0x000828243ea0b548, 0x00002ca6e35f3d10 }}, + {{ 0x0005b95cd967e18c, 0x0006485304ddc5b8, 0x000cc7f246112826, 0x000bb9ca9095e8c6, 0x0000dc3c5bfcdad7 }, { 0x0008349cc1a3f2ac, 0x0001c19b97447336, 0x0008944abac218e4, 0x0009d8ce429c5f53, 0x0000d9a807c23931 }}, + {{ 0x000adb13a8f96a40, 0x000cc1d21830ee5a, 0x000e3ee48eeed2de, 0x00073da9436f9c4e, 0x000084a78c9eeb4f }, { 0x0004d8c4d2962386, 0x00024a081313dbae, 0x0007dd3c6905bb3b, 0x0003055f8c76bb46, 0x000014e6ca4ea310 }}, + {{ 0x0006544814c3dd38, 0x000ae75b055cb405, 0x000b41270952f9db, 0x000528063898f8e7, 0x0000345a4f463cf4 }, { 0x000d3837fbd3e57e, 0x0001095263a75a91, 0x000efceac4dedcad, 0x00056557ccfb6836, 0x0000601bca5adf6b }}, + {{ 0x0008cfb7c7648894, 0x0002138ff4f94ab1, 0x000254b0012b17d7, 0x000d3be877243c71, 0x0000e33accb24543 }, { 0x000bac55e7f80d33, 0x0008ce6729082196, 0x000b31a4eec38aec, 0x000e7812f35c4592, 0x0000abded42c4fa4 }}, + {{ 0x000c639ea67a665e, 0x000231e94dea0f66, 0x0004752922fd3b67, 0x0008002f8259ecae, 0x000098781142c79b }, { 0x000839e6a69aa2dc, 0x000dccb7e1026fd1, 0x000b4aec329bec7e, 0x0004d41a39e73b50, 0x0000263b2ffecddd }}, + {{ 0x000014c5132e700e, 0x00086155150b8057, 0x000e8917ab8428ab, 0x000983d951892389, 0x00006f4bc50a7b1b }, { 0x000b4f4f21091832, 0x000a9361b07523ca, 0x000d80cd15d93e68, 0x00013de5a2f6219c, 0x0000812154648f57 }}, + {{ 0x000a527e6b978cee, 0x000db82cf7d62d27, 0x000ca96802e48946, 0x000e9519e1f36e29, 0x00007265e87b84fa }, { 0x000802786e991660, 0x0003097581a85e38, 0x000f541313771b92, 0x00067126e1f1a520, 0x0000d10aa19b0e7d }}, + }, + { + /* digit=15 (1,2,..,64)*(2^{105})*G */ + {{ 0x0006f0af00724aea, 0x0002a9e36f09ab05, 0x0009f3200c5b7d67, 0x000971f9803f163c, 0x0000abeb83ab4494 }, { 0x000dc4a79b89e736, 0x00099ec70880e0de, 0x000ed078e9a3f10d, 0x000e77ebd7332a66, 0x0000efcf0956aef4 }}, + {{ 0x000dce3782cc4229, 0x00054d0c55c44969, 0x000687ed744ff73a, 0x00095ca63a6a9635, 0x00006189f2556fe1 }, { 0x0005e2f1f92bf8e0, 0x000c16ab2b91f71a, 0x0002b5bd2443b5ec, 0x000ec35ade448982, 0x0000c8788b8a39ca }}, + {{ 0x000392b0b18c6d03, 0x00098e7abded5516, 0x00086524935ee7e4, 0x000a3d6a903d358b, 0x00006875db035981 }, { 0x0006fc5d51954294, 0x0000e980b75c904f, 0x000db1fb9c1ad202, 0x000ea6532c305b0a, 0x00001aa27478cb82 }}, + {{ 0x0008f862ec09473d, 0x0000ac9d64cc78c4, 0x000b52744f8f7011, 0x00038e5ab6c50621, 0x000008c758760cd6 }, { 0x0006d3b2c6ac19b7, 0x0007bede1603c166, 0x000ef5c6e18a250d, 0x0000ffdfc19a80e0, 0x0000dc276b838e08 }}, + {{ 0x00006fa37070bc95, 0x000637e822e37bef, 0x000334cc36ddfbe8, 0x0005c75eb73f237b, 0x0000f47794e531d0 }, { 0x000fa0c354bdcde3, 0x000d86efc621c127, 0x0003c1174b714c9d, 0x000d773f35d624b6, 0x0000fc893443d0bd }}, + {{ 0x0003d69f745fb237, 0x000c331e3229d41f, 0x0001a07c7c60896f, 0x000214b09a929c65, 0x000030eff0f7b347 }, { 0x0009ce6bfe978f9e, 0x000339a168336319, 0x000d6a06afe4f177, 0x000c17ed868a34c2, 0x0000a6c91a8a3c76 }}, + {{ 0x000567bcf9802012, 0x000c8aaffcdb4631, 0x000a1f3bce0ebf8c, 0x00088f58d658bbc1, 0x0000d77d6056ff71 }, { 0x000c08728ddda15b, 0x0003de03572359bb, 0x0004ca5173fe02e3, 0x00082257ccf0353e, 0x00002fdd1fd23a9f }}, + {{ 0x00011cac13161f9c, 0x0001bbc453cadd69, 0x00072aef15e577c3, 0x0008c37af203900a, 0x00000e41db5e3490 }, { 0x000b87d44a487f26, 0x000dc42ec965469f, 0x00012e582d33e4d5, 0x0001872850e9f769, 0x000038b03659451d }}, + {{ 0x000cc69f23d9740e, 0x000004bfc25419a6, 0x0002df389dda0805, 0x00042da3f3646720, 0x0000f63588d2624c }, { 0x000b9eea3275aa7e, 0x000665a6bc68bb94, 0x0005dde089018c6f, 0x000b34b7ce43d79b, 0x0000cca74fe735d0 }}, + {{ 0x00075c28e4f6afb1, 0x00015ba6102d0212, 0x000807e8390524e7, 0x000ef68a8eaafac7, 0x00005aa40be45ccd }, { 0x00072a4f8c8e1dc1, 0x000599ddd2e54e03, 0x0004cedb11fea727, 0x000190c246991adb, 0x0000a2b27ebd40ed }}, + {{ 0x0009ee1edba4daa1, 0x0001c73a6a5da60f, 0x000fe25e811523b6, 0x00078cd576f083d9, 0x00001e9fcdcfb94b }, { 0x000aaf21de46d633, 0x0000eec8bdcf8e7c, 0x00004054fc48103c, 0x00029bd40a5db827, 0x0000e01632abbb9d }}, + {{ 0x000759b8001f79c6, 0x000ece14f0dd2e8f, 0x000e12ddc2c79568, 0x000005b98e77e7c4, 0x0000b791403a3eee }, { 0x000aa7ff8c7ea4a3, 0x00083eddcfe314e5, 0x0003f92dd00d49d0, 0x000ac95b6eb84c52, 0x000080d5b1ce41d2 }}, + {{ 0x000baaaaa42eba90, 0x00083c3b9791a263, 0x000a922ae072d050, 0x000f042de960022d, 0x0000bdcee541ee28 }, { 0x0003bb68bfa63380, 0x000a3460d881efdb, 0x000f9276f56af417, 0x0002898d4506416b, 0x00006f83b035b65d }}, + {{ 0x000fd6b0c5230ca5, 0x00013bf5116f9379, 0x0006214fd87df2b3, 0x0000cfdc87c64a58, 0x000027f51850fe61 }, { 0x00092d51304b9476, 0x000443ce8ee0e0e2, 0x0000005ed0ca005c, 0x000d248e06f6fba6, 0x0000747a3ca00962 }}, + {{ 0x00048c919c326cd0, 0x000405dbab093b30, 0x0008fdf707eac1e4, 0x00007d89c012f59b, 0x000033b871966602 }, { 0x000eaf517a2fb4d1, 0x000d1b591071cd58, 0x0009749a1f2b94c1, 0x00002c84b791ed89, 0x0000772ac30fc8b2 }}, + {{ 0x000da8e4f1aff818, 0x000caa2d551ee106, 0x000804d524b4b06e, 0x00008fc8d87a8f1d, 0x0000ef431fbeb97c }, { 0x0000c0d9b00298e4, 0x00033f0cf9a4e718, 0x000fbb34f16943d4, 0x000ca08c0ff50200, 0x0000745ea7a228cd }}, + {{ 0x000d1395ccbc828b, 0x000e6d8033f28764, 0x000630492e0f2bf4, 0x00047311db0186ec, 0x0000dac857b0b37f }, { 0x000038d71b0c1915, 0x0004dd6186244a03, 0x0002e0d184389b23, 0x0000269ab918e5f2, 0x00006cc928b6af68 }}, + {{ 0x000de6ad0f11bc0e, 0x000f6b7a93c32461, 0x000982d307be7245, 0x0001a341821ab606, 0x0000a0e9cfcf1ca0 }, { 0x000b972fd6728dfb, 0x000af88ff1929c78, 0x000ad87757dee47c, 0x000d111ccc6df4c6, 0x0000d7c581401c53 }}, + {{ 0x0001f7f91b4ffc66, 0x000d0b60b846531f, 0x00051eadeb59f002, 0x0004950c5ba7e45e, 0x0000b7e2a63df2f1 }, { 0x0004c828296f804a, 0x0005c9244de90d77, 0x000a7cd199349397, 0x00056f3eb951a1d4, 0x0000788cafa21d5a }}, + {{ 0x000ce2e57adaafd4, 0x000a4fce7560b907, 0x00012b053f53cfb7, 0x0009317a8b4073e6, 0x0000b35cda57c1fb }, { 0x000b6dd71e0124a6, 0x00021891df1ad467, 0x00018e156fdf889e, 0x0000b1b7783a0692, 0x00003ab3282c54d9 }}, + {{ 0x000182d411742393, 0x000da0a73fa9cb0d, 0x000d6109219851e6, 0x00008f982edd3992, 0x00006166219cbf0f }, { 0x000cb8df03c5c3e4, 0x0009038d9a528dba, 0x00091e3c5a90b7d9, 0x00055b47ce8b1a0e, 0x0000a0f6e1eafa48 }}, + {{ 0x00012fcc62fd2575, 0x000c5328300129f8, 0x0001e7946f9a564a, 0x000bfcb511f63766, 0x00002838b2c1b003 }, { 0x00017bbc879d576e, 0x0002f5f29fb85da6, 0x000a07cb9d60a23f, 0x000afb8b772f4e00, 0x0000907e471fcadb }}, + {{ 0x000682238fc10ebe, 0x000cdb997aee3170, 0x0006e19313189010, 0x000b1dd9b812a409, 0x000009ce0a98c979 }, { 0x000b75351318c260, 0x0008ad6aeccb4bdf, 0x00096db6712057b0, 0x0003d3d336a34e9a, 0x000074e57bd9af57 }}, + {{ 0x000f1e0fa64b9ba1, 0x000a4ff1d7aadabe, 0x000fe6d896d5ce68, 0x00034116f65d3825, 0x00007bcae08c6246 }, { 0x000ac54f612c30f9, 0x000ceba33db72cdc, 0x00092fb725511bb6, 0x000d9573c017cbc0, 0x0000a29dfebf2fc8 }}, + {{ 0x00044c4ab2167d7a, 0x0005980b2369f0f0, 0x000d1f7b169f84ef, 0x000e52fc697d3a70, 0x00000371cf064a81 }, { 0x0002f2497b167080, 0x00020fd834a2ddc6, 0x0002698938f1ecee, 0x0008c649049e6b86, 0x00006463bb9f084c }}, + {{ 0x00070d2d35f711e1, 0x00024b6c64582934, 0x0006a1c3ddc10d9e, 0x000d436b836bb527, 0x00000a5cf78fdd97 }, { 0x000e797ecd201480, 0x0008832feeb1b77f, 0x000886ec00cee664, 0x000e180042bb75c3, 0x000057f8a6d620a0 }}, + {{ 0x0008abdd527350b0, 0x0003c2a753aebcc9, 0x000c308e0f53280b, 0x00044fd09daf9f3d, 0x00004eb1613ae15d }, { 0x000462e7e6da22c6, 0x00044318d168be52, 0x000ffec2ba55ff61, 0x0004c09afcf7c10d, 0x0000ff5b8008b9a4 }}, + {{ 0x000d3eea67fe0e27, 0x000638ba0cc3f125, 0x000d5f283fc1ad1f, 0x0006105d368dd485, 0x0000f512d2d3ea5b }, { 0x0004dc21e92938f6, 0x0002481e6602671c, 0x000f4f264a2357d1, 0x0007c19858f144a3, 0x00008d20d20e844d }}, + {{ 0x0001db64b60dcc0f, 0x0001eea521d03782, 0x000de325b7b9b75a, 0x000800aea802b8e0, 0x0000a207924704fd }, { 0x000e7c824e8b91d0, 0x0007ee970342f9d6, 0x000dad47c7c593a7, 0x0006b3baa93a10ae, 0x0000b3c50420ccb3 }}, + {{ 0x00044dd196d3754c, 0x000ae819966e7eef, 0x00071169c0771aae, 0x0001b5ed464ec4aa, 0x00006b0c6bc0965a }, { 0x000ca9d721b441b9, 0x000b8f1bb46b6a1a, 0x000350de796ed922, 0x00067beb41e65d31, 0x0000103b3fe2d35f }}, + {{ 0x0000c504bf8e9753, 0x00032a0fcff6a17e, 0x00023e05782f0bc3, 0x000b8f8e39970a3f, 0x0000a169715bac3a }, { 0x0006fd0c5dcd460e, 0x000cb5677fcc2f46, 0x0006febe5366b0c3, 0x0004c78aaef1a522, 0x0000e87aa65153be }}, + {{ 0x0000903c4cbe46bc, 0x000c0eb1bf4581ca, 0x00060c177519da3e, 0x0005066ace635e18, 0x00005345445cd5a7 }, { 0x0007197ca469b8c0, 0x000858948f214029, 0x0004225140fdccb5, 0x00087719031b49c1, 0x0000a2053b8e6576 }}, + {{ 0x000332d0815fb0b2, 0x0002b1e274983d81, 0x000e423eac0e20c3, 0x000b147d1da1a3bd, 0x00005049bb2f20a3 }, { 0x000aa2f407901242, 0x000b5a3aa6d0e10a, 0x00034aac28430b6c, 0x000c1136717db2e7, 0x00007894ad4da6bc }}, + {{ 0x000fe4ca540470ce, 0x000a79e7ace3fc7a, 0x000e95c40e7284f5, 0x00012800e9594eac, 0x000074e26fc1378d }, { 0x000ab186fc36c333, 0x000b556e6ee81c69, 0x000ad0d7933ec714, 0x0002305298f671be, 0x00009b93b73b2fc4 }}, + {{ 0x000df0fefced17bd, 0x000f66910b5be761, 0x0009dd130cd0a056, 0x0005c2188046b773, 0x0000009bdb10a2de }, { 0x0009ee86e04a5ad7, 0x00080a6bcfcd0279, 0x00052142deea631c, 0x0003ecdd9a8786c7, 0x00002b9a476e5434 }}, + {{ 0x000074d3de5dd49a, 0x00093ddc15c9f0ab, 0x00039220c4bffda7, 0x000fbf5dadf87a73, 0x0000e0074caa9da5 }, { 0x0006df62f75d4dc7, 0x000b20351a03776e, 0x000a43811d0fc2f2, 0x000c76a852bb1f77, 0x00003d9c33f35883 }}, + {{ 0x0005c11927d7db11, 0x000c0beed51275ad, 0x0006594e2f1817d8, 0x000e7addf5de47dc, 0x0000c87b4cd166bf }, { 0x000890cd3245ae9a, 0x0007800a73a294ea, 0x0009f531344057c6, 0x000c0b66a5f88434, 0x00007a6fab6ca15c }}, + {{ 0x0005929d26ae4d60, 0x0004ec170f4d3b44, 0x0008ca69efd95c9e, 0x000881838fd1a136, 0x00008822948519a9 }, { 0x000418f5d76d3725, 0x000ab6025b15ca6a, 0x0000ca2e4e6eecf5, 0x000e72c344b3c83e, 0x000017e3caa4a180 }}, + {{ 0x00041b28983194f2, 0x0000a8a213ed68b6, 0x000cecef298c37a2, 0x0005ba953860ae7e, 0x00002530dfcc9dad }, { 0x00085a2310dabfe5, 0x00053e101110f35d, 0x00018c30f6264efd, 0x000d17ff479c26f4, 0x0000836740a708c2 }}, + {{ 0x0002dda543915e7e, 0x0005f5d4b07e2b19, 0x000afac0fcdde015, 0x000a72d14d144c4c, 0x000034e73127607f }, { 0x0002db773ecd3720, 0x0000fa8cf03dce66, 0x0004b74758be1c61, 0x0007668c48b5ea96, 0x00009b6250934072 }}, + {{ 0x0006f31c01a3dfbe, 0x00044fb93f245674, 0x0009ea5f3bc92e2f, 0x000eafb87fd46c69, 0x0000f667b3eec5e7 }, { 0x00083cc1ece5f671, 0x0006a59888b219e5, 0x000789c00b155ab8, 0x00029106ce50ecc3, 0x0000f8a9874cc0ab }}, + {{ 0x0009a7ffecf51501, 0x0002d66a483bf119, 0x0005180ac17cfeb8, 0x000d4f928fa08a6f, 0x00000c4ee2e35856 }, { 0x000cba1424521b06, 0x000b7dbc4ae62da7, 0x000dba90db391823, 0x00031ffbc6cea00a, 0x00007680c0ab694b }}, + {{ 0x000ff63e0965f49b, 0x0003121208490def, 0x0002bd04ea6fe3be, 0x0007b166f1ca66ec, 0x0000bbe50fc3adb5 }, { 0x00010d8a6a559d7c, 0x000debe8b2610bed, 0x000f68d0b38a6296, 0x0007eb6b4b732ed7, 0x0000e692b4dee5cb }}, + {{ 0x00046bae92221adf, 0x0006498ed4de248b, 0x000d5bdcda959eec, 0x0004c5f13d7cedac, 0x00004112599c6663 }, { 0x000b84c4c8c12811, 0x0007372fcef7ad8e, 0x0007e92e99ce45e0, 0x000ba88866115160, 0x00004192a1db5e1c }}, + {{ 0x000f01e9b964c398, 0x0002d5c164e1214b, 0x000ce245d14a5d2b, 0x000e5d035891be86, 0x000058456cfb56a6 }, { 0x000ed62a7c96800a, 0x000d739b186ebc0e, 0x000596e2f14dd4cd, 0x000592e54d5c74c2, 0x000037f83089202c }}, + {{ 0x00069dfbf102d869, 0x000846f76b7618a3, 0x0004cd18647184a0, 0x000791ddbbff750a, 0x00000418f22b31b3 }, { 0x0008af82f7454899, 0x000196c7191a321c, 0x0008783dee978292, 0x0008738839edac3f, 0x00008a38ad413f8b }}, + {{ 0x00018d12afd54f4c, 0x00098d897ce0b2a0, 0x000fc2184a2fde82, 0x00068cefee6d7c87, 0x0000880946a4e593 }, { 0x0008157478782d1a, 0x000480801206e150, 0x00039ba30e4db2ff, 0x000d448d5e0e4245, 0x0000a2f55c162b2c }}, + {{ 0x000d7803581ab555, 0x000eab4820357c72, 0x000e419c5554fe07, 0x000fef5a1992039a, 0x0000d9875b456d0a }, { 0x0000ee0ea7058063, 0x0001b4dc77ad4d4a, 0x000858f1411d534f, 0x000f0bd5ed49a799, 0x000050a8f7fd0e17 }}, + {{ 0x00033d6d545551ce, 0x00020cfab7bd7712, 0x00031284f564f60d, 0x0004b93e76c87307, 0x0000270e7036f3e7 }, { 0x0004b690d19b6f88, 0x000ce26fc195bcc5, 0x0007b020fa92369c, 0x0007d06f32544595, 0x00005326f8c26b7c }}, + {{ 0x000a5acf73de976d, 0x000bf8a212306466, 0x00089df47218a462, 0x000994d43f51aabb, 0x00006b738c5c4d3b }, { 0x000ea5938bb50b53, 0x0006057d635b4d5d, 0x000d0e302e4e6897, 0x0000f9dddc3de68d, 0x0000247ceec78021 }}, + {{ 0x000446bc5330e6c4, 0x000131afe637294c, 0x0005e1616084d14b, 0x000a79fb501968aa, 0x00003b7dc534fcd3 }, { 0x000178d549b09def, 0x00061a696d6134bb, 0x0004ec90cb133eee, 0x000053a1a9361fbf, 0x00006af0a49f5d2c }}, + {{ 0x000b745f8fc43a35, 0x000024ba4df39156, 0x000ec4617678c3bc, 0x000a91c02566ce1b, 0x0000719b228fbdb9 }, { 0x000f385adab6fcc4, 0x00066f663122edfe, 0x000c4c603faed143, 0x000bd6d563190f9e, 0x00007f12983cad34 }}, + {{ 0x000f1627dec28dca, 0x000b7de3f642c293, 0x000851f9d3a9edf4, 0x0009b5b4620be088, 0x00004176cb7f6c12 }, { 0x000787c0dcf777ac, 0x000f413eefcd6385, 0x00061d44b6e6ae82, 0x000d1495e8f3bf62, 0x00007ad2517a88ab }}, + {{ 0x000193cecacc8855, 0x000e5a2d9a23e449, 0x0002bc7c9a4324de, 0x00070bc0888dfa99, 0x00001357e1a16b5f }, { 0x000cdf57355cc9f8, 0x0003a7d2d738fc52, 0x00076a8097379e6a, 0x0007681a56697b03, 0x0000e15f32fe6b63 }}, + {{ 0x0000441aedd1db0b, 0x0003920bd55659e1, 0x000b3331ee1ea4b9, 0x000604e1a7973e32, 0x0000b848523ff344 }, { 0x0006429320a3e8ed, 0x000324a045ad915b, 0x0006ee6d1990975a, 0x00014ced86c11b43, 0x0000ed1e5721bace }}, + {{ 0x0003cab91b752141, 0x000a5e89c92b2350, 0x000e676eda85d689, 0x0001c8f31a73bbd0, 0x00003841c3e8d0f4 }, { 0x000cd61ad4ee96e2, 0x000a42f1db3574b2, 0x000537e4eb45e77e, 0x000813c06485b018, 0x00002be90410b3b1 }}, + {{ 0x000fbec0cb6f0fc0, 0x0002d5a2ff9f403a, 0x000ab264cf5dcff0, 0x00007f70d8dc98d8, 0x0000dfe1570152b0 }, { 0x0008d1e1d5d329b1, 0x000c06480130dfa7, 0x00051130fb1fccc3, 0x000875ec0306dc2b, 0x000026a348354b2b }}, + {{ 0x000e31318e23bd1d, 0x0006f5a192ef7106, 0x000acd43006b8a32, 0x00067ef9e88afbd4, 0x00009533ab5f7940 }, { 0x0002a21cf2ee4b63, 0x00061b824f89b4e2, 0x000826214d380f04, 0x000d957a765a5dd8, 0x0000431ac8130d01 }}, + {{ 0x0009ea88981c66a0, 0x000fdee42a554f77, 0x000486443c22796b, 0x00047173d828983a, 0x0000d1238eb95f6f }, { 0x0000e913e1673541, 0x00064a7d065fd831, 0x0001a1798cdb2234, 0x000f61027f49af79, 0x00009e532ad5d547 }}, + {{ 0x0006b46e1f443280, 0x0001d14bc21ef518, 0x000ac63104730cf1, 0x000894c3be51c3ce, 0x00000536de024903 }, { 0x000be017a2ce58da, 0x000e02380041ef4b, 0x0004a6821c8b404d, 0x00079bad4d72fe50, 0x00006b9d35305a63 }}, + {{ 0x000b652edcec4a5b, 0x0009f09b389328ad, 0x0001f4b242d8c5b2, 0x00065eabf322fbc6, 0x0000dd15a5fc8674 }, { 0x000ea672143089ac, 0x000e72c03ec6627f, 0x000cc347decf2065, 0x000adfd0dde1b3b2, 0x0000a98c9af7b1ab }}, + {{ 0x0005cdafe7faf119, 0x0009ccb4f2999aa1, 0x0006a2fe40f56991, 0x00042ecbb38476cc, 0x0000d4790f63e31c }, { 0x000bf2c38ae9c943, 0x00018a4d935b7db4, 0x0003d0a6e142ea64, 0x000ec115b648b600, 0x00003a16278132b9 }}, + {{ 0x000ee635fb2543bc, 0x00020634442449cd, 0x000954c719bd8c67, 0x00024b182e0e9921, 0x000066a2b2a91885 }, { 0x00074ea6ed7740f1, 0x000314a18ee6e8a7, 0x000675f4394244fb, 0x00033dc08eed748a, 0x0000cce701dd1d5d }}, + {{ 0x000add5f69c02408, 0x000d61cf3480d4af, 0x0006c1ad6fb043be, 0x00005529cf0d8edc, 0x0000e6b784db99a8 }, { 0x0005a454130bbd68, 0x0002fad96f8acb1a, 0x0004c17786cf7d98, 0x0001fdbf4bddd49d, 0x00002c7d15e91516 }}, + }, + { + /* digit=16 (1,2,..,64)*(2^{112})*G */ + {{ 0x0003c0440fd7f2f9, 0x000bdbfc360e25a8, 0x00048399d668b4f6, 0x0002439f4e642519, 0x000089f1fa0b9870 }, { 0x00097b3d282ee427, 0x0008cf1d720281f6, 0x000e67baa329978c, 0x0002104e72205910, 0x000061a43a34e6ac }}, + {{ 0x00071279a45648bd, 0x000de9afb3ff9ed0, 0x000b10e3d7b7d810, 0x00004e74928c2e0a, 0x0000f66bf52858f1 }, { 0x000e9fccc7111eb0, 0x0004f6601fbe8556, 0x0007f13a84fd868d, 0x0003acc0b3ee394e, 0x000011bb82692f56 }}, + {{ 0x000f57841f3144bf, 0x0001f9314092ebb4, 0x000cf3369fecda45, 0x0008e3164f17256b, 0x000033b176e2d462 }, { 0x000333e4efa6df27, 0x0001d05e708d8553, 0x0001d0bddee0a802, 0x000f8c7d5856aae7, 0x0000603c90846016 }}, + {{ 0x0007512e2eb6a7c2, 0x000b8dedda492f81, 0x000a60d843d6851f, 0x000eb594cc11a3ae, 0x0000e9fde037b87d }, { 0x000cf109a400a483, 0x000c6f3177bf8b6c, 0x00093bd6f59a5d1e, 0x0006c915d232ea4b, 0x000066d0031672f4 }}, + {{ 0x00008a54ec2b1382, 0x000f888a1f258972, 0x000f812b7fecbc4a, 0x000e06b5c7ac6961, 0x0000e7ee6a3486dd }, { 0x0003eeb9a9acb8ed, 0x000811978660710a, 0x00067f8f391f11c9, 0x000b760a50cf70a9, 0x0000a64a54a69740 }}, + {{ 0x000869d0d5ac68e0, 0x0008233169bca968, 0x0003a53cda259d70, 0x0005a1a9404d286d, 0x000088e4951616f4 }, { 0x00052a733ab82011, 0x00026e5d0150d651, 0x00030ee1a18ef179, 0x0006e1c49a92e250, 0x0000660861970a58 }}, + {{ 0x0001843074252dfe, 0x0003f588434a920a, 0x0002c09cd3518f04, 0x0005fc19ac0af8e9, 0x000065ea1ee67b6e }, { 0x0000581fc169a790, 0x00004aa5447ab801, 0x0005b7021d9fcb63, 0x000998a9ee5e5a32, 0x000086ce09c0bdfe }}, + {{ 0x000b0da855c77138, 0x0004d7080eb24a90, 0x000d3d8064a79971, 0x000e895b87bedfb4, 0x0000ff2e824b81fa }, { 0x000939b48c4a16ef, 0x00015dbde314c709, 0x000bd2f933b9e136, 0x0004c0d9ecdbf1fb, 0x000042e460791f67 }}, + {{ 0x00031e066075581e, 0x0007aa7cb0c9c8cb, 0x000fcea630fbe552, 0x00099ab0f75105f2, 0x00001f912867f6e9 }, { 0x0004564766700a62, 0x0007075d4b6d45a8, 0x000cde96422fe35a, 0x00087645bd363f3a, 0x0000ada1862cd2d3 }}, + {{ 0x000f0a0c47a9219b, 0x000f3cc37107c78e, 0x000a68021e1e77af, 0x000af3d45acff3b6, 0x0000e6dd110491f2 }, { 0x0006145e628202b8, 0x000de5176b6028e0, 0x00041e71b471983e, 0x00035f755c49ec67, 0x000098d7591e8ff1 }}, + {{ 0x000bedcf6b007f50, 0x0000d07e75746b54, 0x000d781ea953336a, 0x000b0a8e21c1e1f6, 0x0000184b09b48421 }, { 0x00054ff226110b80, 0x00069c13d7a4c8a9, 0x000bd88ff53a5a2c, 0x0002edbbc2748887, 0x00009a72c0defd99 }}, + {{ 0x000775780f973b43, 0x00047b197c3dfb92, 0x0000a7a109452cf0, 0x00079c39255a549b, 0x0000db1d2d13d82a }, { 0x00002f1085a734db, 0x000a5b8e1c9d7c80, 0x00013facbcd0a776, 0x000350fce8c69dae, 0x0000dfeeec7f75f2 }}, + {{ 0x0000deb4ca3a9977, 0x00053446d3fea556, 0x000693a2509e138a, 0x000c97b7db4ae9c8, 0x000056217a7b50ef }, { 0x000b113eaae38142, 0x00096947de6c34cb, 0x0008898319d2c511, 0x000be485b4ee6825, 0x00002fb9bee4a336 }}, + {{ 0x00009ed751690d07, 0x000fcb92115b4c9e, 0x0007a869f38018bc, 0x0000b109a7c163d3, 0x0000d25180d0c741 }, { 0x000626f43012e59c, 0x00047fa26f516cac, 0x0006c84c3b619391, 0x000580cb4753cacf, 0x0000d99e17d2ba70 }}, + {{ 0x000d31e20e626aa0, 0x0002899a7b56eaf9, 0x000b97bd91088d33, 0x00016ea8600c0e52, 0x00005520570c169c }, { 0x000e6b9cffa1bae0, 0x000b32d3ed689748, 0x0007ede2eac48130, 0x00023baefe44bbc0, 0x00004454704b6e6c }}, + {{ 0x000c8ac9b3f30faa, 0x000821f711b0eb9e, 0x00078b7fefa0a379, 0x000f7e8dbb905f2c, 0x0000d9b3674355ae }, { 0x000e568622b92879, 0x0005fb6b40a24474, 0x0001518d75018f42, 0x00065c23f60121fc, 0x0000b6c0f8efac61 }}, + {{ 0x00029b6e563d4ca5, 0x000090d196aa5a1c, 0x00064da647a59d2e, 0x0006673b7e0df8c7, 0x00003ebfc59c82e9 }, { 0x00077c02963ba3d8, 0x000dc8b711484633, 0x00032cc947dff086, 0x0005fd3e4b5738de, 0x00002458f4cf0a1e }}, + {{ 0x000f18fb2162c8d8, 0x00018b92bb39c1fd, 0x000c06efdda4112d, 0x00032e00514373f2, 0x00001c606faff29e }, { 0x000b75dfb7307746, 0x000394242b3523f1, 0x000d6c807144512e, 0x00093377ce958532, 0x000075bf8f2eff5a }}, + {{ 0x000f8237e184aa56, 0x000a1c92d7d1413a, 0x0004e8b2bdb60c3a, 0x0005e6de867e2d6b, 0x0000a224fd131823 }, { 0x0003b73e5c38837f, 0x000e0dc84687cee2, 0x000181fb2daced27, 0x00066f750c5ded6b, 0x0000dba3169b9090 }}, + {{ 0x000e01470209ad8f, 0x000b4183037bf522, 0x00054ac5ded5224d, 0x0000656d7b61e6db, 0x0000906aa9885eb2 }, { 0x0002c37583833f85, 0x0004a9a360aa449b, 0x0004c69fd7852936, 0x000bb90efb70cfd0, 0x0000ef760d62ea7c }}, + {{ 0x00045a8ff05fe573, 0x000ccdcf50225f97, 0x000c3c869b135443, 0x000c34c64c52e175, 0x0000b6e1d4a6ca70 }, { 0x0007a71cdf35f6f4, 0x000f1a358321bc3f, 0x0006141dc268fae0, 0x000f35c58b1732be, 0x0000235d7150b4b3 }}, + {{ 0x000eeee592f8a112, 0x000413e894c5bbba, 0x0008bae94aef8e2b, 0x00074b0446915c7d, 0x000077f05c59725f }, { 0x000df69b21ac3da5, 0x000f813781354864, 0x000ec8c0d40ddda5, 0x000f943f53a275c1, 0x00001739b2d5fa33 }}, + {{ 0x00040e9f29ec6f84, 0x0002e502d0f39aaa, 0x000a7199ba1db519, 0x0007170fa622118b, 0x000062772036e611 }, { 0x000754b54fc3aaa5, 0x000aceb22b57f0fd, 0x0008a72baefab7bf, 0x00045f0fe849a33b, 0x0000513d3bc11f59 }}, + {{ 0x000b33f80501f2c1, 0x000fa22236f4a982, 0x0008db3408b65b08, 0x000a7b6740b0c59e, 0x00004d2185638a58 }, { 0x0004d80f0e06a637, 0x0005b9532e6c4662, 0x000f29818d65accc, 0x000b5facc68956d0, 0x00005c86484c910e }}, + {{ 0x00093cca39225886, 0x00097fb988baf015, 0x000d82749c0eab29, 0x000f43360e6331a7, 0x0000dde7d468f9e6 }, { 0x0008a7318da83916, 0x000968d265674566, 0x000957a433029011, 0x0007db2f640fa030, 0x0000ecd715314932 }}, + {{ 0x00048c5860d99716, 0x000ba4efc4d31579, 0x000197676589c678, 0x0000e5b72b56f8ab, 0x000013465a13e776 }, { 0x000bbbc6da512eeb, 0x00072f00ebd4f086, 0x000ca2c6ccb3e2fc, 0x000b072d03e5168c, 0x0000f06060e5f45f }}, + {{ 0x0005959cbe186e9c, 0x0003886605daaa67, 0x00074604cfe0af51, 0x000b65f8946fd72b, 0x00007a875d7ef388 }, { 0x000a73bb803de3a1, 0x000fe65c8a588eea, 0x00060d978e1d0473, 0x000b6bcf1368c92e, 0x0000847dbedc4d5b }}, + {{ 0x000eb1b6c88eb6f6, 0x000b10d6457d188b, 0x000accc881c36490, 0x000166a1386eda86, 0x00007a87d33e3d7e }, { 0x000eb5009735b4ad, 0x000768799240a9f2, 0x000f48ac891ad54d, 0x000b16c3a1bbb384, 0x0000158d9593875a }}, + {{ 0x00040b8b0c182e20, 0x00089c04a7d2caa5, 0x0001519f1783ee26, 0x000194e8f6f3fb3d, 0x0000f047804535e7 }, { 0x0007a5b479c3334f, 0x000ebc905e214f55, 0x0009fefb01bee293, 0x0006b190b9a74075, 0x00007399d8ab170b }}, + {{ 0x000505be1efb2b17, 0x00031e726bfe65cf, 0x000b7d8b04e8129f, 0x0003ed02e097e391, 0x0000ca6e619f0f6b }, { 0x000d838a95ee8071, 0x000cda1e3a3035a8, 0x00060f518312713d, 0x000e396f26431ebf, 0x00004e5e1383c630 }}, + {{ 0x00083bf494f1922a, 0x000d07f650006f01, 0x00033885071d3899, 0x000678396dfd7dad, 0x00008e6b9b5a00db }, { 0x00061e1a6d262a4f, 0x00090434a7bd989d, 0x000bcf24f627dcab, 0x00019a63e505385c, 0x0000162da51b10d7 }}, + {{ 0x0002b23a10647092, 0x000b4c64a8a3d626, 0x000fc1c509c1f5da, 0x0008752338469c4c, 0x0000592d71f92d24 }, { 0x000e65e5e66ab21c, 0x0007ab4a63d1a4c4, 0x000a2eb259587d83, 0x000fd941c454e7fe, 0x00001e808c047aac }}, + {{ 0x000985fe38c5fb2d, 0x00031a1ef74c45bd, 0x0005d4ac86cd7bdd, 0x000df50a92cfbfa6, 0x0000f38525a7bc74 }, { 0x00027e0dfe332909, 0x0005ffe5a0148b4a, 0x00068afa056fbba5, 0x0008e20ce6f9a55c, 0x0000c1b3d8c7c69f }}, + {{ 0x0002ab75b80ff311, 0x000335212b4603ee, 0x000088c4f75c8654, 0x000b6c312a876c55, 0x0000ec0c5bfb1759 }, { 0x0009a79cb22aa461, 0x00039761a23ff976, 0x0001f38e005de83b, 0x000defe22c613899, 0x00008b2f4e8bfccf }}, + {{ 0x0005afec02a30992, 0x000a30d594cc7e13, 0x0002fda0bf775275, 0x000f5c74a3079ca6, 0x00002d8fc95c11f7 }, { 0x0008853e7d3553e5, 0x00061f406ed2c6ed, 0x0004a50e5bda669a, 0x0007d947e58da173, 0x000046a193aa855a }}, + {{ 0x000e471b95b01638, 0x000188da35fcd5fe, 0x000b5b15af55904f, 0x000cb71628965df8, 0x000058d0fcc4ba62 }, { 0x000d74938578142d, 0x0006ec80a08fb758, 0x0004f469a20c56dd, 0x000db2288704958f, 0x0000e41307922cc2 }}, + {{ 0x0007a222bccd2e68, 0x000ed378c9049f43, 0x000343f0b5f94e09, 0x000b4556b5be948f, 0x00000a6efbb7a0e9 }, { 0x0004a461c9039832, 0x000e8bb8809bf618, 0x000b8ea28fc4e42d, 0x000e3db1c46f07fb, 0x0000437a4a4ab1e4 }}, + {{ 0x000e03fdde9732a4, 0x00038e6d9278d982, 0x00036ff876027328, 0x000107dc324d9481, 0x0000e4b863ced76e }, { 0x0002045a740a4e7f, 0x000871947b533c0b, 0x000e7a9f65e10697, 0x00067ac8f1a8bc73, 0x00005706a2f93e11 }}, + {{ 0x000fb693dc0269f5, 0x000e0afbd76e195d, 0x000c898214343849, 0x00079e0cb478dd1a, 0x0000bf92ae14b76c }, { 0x00050e61bd7654af, 0x000959651dbf1a78, 0x000e839485a58314, 0x000470b9acab4b4e, 0x000053da0f7c685d }}, + {{ 0x0008bb92b4197f71, 0x000bfe3dc09508b4, 0x000f2b75e3a19e68, 0x000e99255faf6d2c, 0x00008a43244b37c5 }, { 0x00012686517eb89e, 0x000b6845c7cf8929, 0x000944af69cde6fe, 0x000877f69f6cd7ac, 0x0000c9b055220817 }}, + {{ 0x0008da07e4a739f9, 0x0000013b78c558ff, 0x000acfd5d2c490d2, 0x000b770d343c0bae, 0x0000eee5df52f0ad }, { 0x000001122fea8b0b, 0x000b99da8a90ea83, 0x000b53a188f1cbeb, 0x0004dc77681c1ff2, 0x0000b1f2bc907c40 }}, + {{ 0x0000f5091f08fba5, 0x000efbe6718e4488, 0x0001fb4964f4d8db, 0x0002b10ef246c148, 0x0000fbde6b0ad954 }, { 0x000f886488030d36, 0x000f82f724bdd38b, 0x0003c4e9843fc28b, 0x00002bca902e915b, 0x000027da8403820a }}, + {{ 0x0002c8cdd5a84a71, 0x00075a6c93f72d63, 0x0002980f33151397, 0x000628017d5f75d1, 0x00002b6fc243ca3f }, { 0x0001bcaa034d4c38, 0x000e1927023c4994, 0x0006f72f8efe26d5, 0x000c24bf0a8afe5d, 0x0000af62213dc85b }}, + {{ 0x000bbee6e35723eb, 0x00074f98d2668d5c, 0x0009af1932828c67, 0x000a2e3110715281, 0x000018a72e0098ac }, { 0x000b8bfd572fd9c8, 0x00047e80cdb9808a, 0x000652b2d5bdd6d6, 0x00031881817d6e06, 0x000045b2eb036f1d }}, + {{ 0x00010e2fa9c3473f, 0x0000b59513aed8bf, 0x000c403da0d07f42, 0x000fb7682f77f3b6, 0x00008f8e17dae5ca }, { 0x000f98a2d0c72d35, 0x0001b2262a2f201c, 0x000f81558fe0288c, 0x0007a54a84d59ea4, 0x000094d0140957a6 }}, + {{ 0x00034de6ed85c687, 0x000eb39720943c28, 0x0004b642d2e37390, 0x0002c014aba044cf, 0x00009c5d21883610 }, { 0x000c0c6eeb0043f2, 0x000fc9a5f25bd125, 0x000a136e4386c89d, 0x0008c19769259cea, 0x00001522a51ab777 }}, + {{ 0x0008f8514711e5e4, 0x000de0f5606b81d7, 0x000494a0cd9e404c, 0x0004cf551abd7b37, 0x0000aae61990ee74 }, { 0x00069b8f8064759f, 0x000d4f0f30c3282a, 0x000000ed938b30a2, 0x00022f77f0da8ac8, 0x000034f8a1881bef }}, + {{ 0x0003840fe88767bf, 0x000b8176add85450, 0x00066f408b7e5e73, 0x000da5e771b71cb6, 0x0000d35c39b650e0 }, { 0x000174cced9e5822, 0x000b8d51ec699775, 0x0008df0a9a391539, 0x00071f40cae243fb, 0x0000e2156e1d8e4a }}, + {{ 0x0008e419d542ad83, 0x000d666e58ba37d6, 0x00020f33198855f5, 0x0004e44bffdcc8da, 0x000059353e915b41 }, { 0x000e22cd180b3421, 0x00018f402a403801, 0x000f01c6aaa43566, 0x0005abda688940fa, 0x000095b351fd3384 }}, + {{ 0x0006fb438a17556f, 0x000f9ce59ffc3e20, 0x0005bae3a15dcb2f, 0x00027446360aa1b5, 0x00007cd6cec33e6c }, { 0x000544607e7d51cd, 0x000b1c4500d8e20f, 0x000c112c7a91b2e3, 0x000e9affd1ad9095, 0x0000a60766068749 }}, + {{ 0x0007d49c6cff6daa, 0x000e9b1a20ef103d, 0x000f93d1bb0e184c, 0x000346aa0ad85912, 0x0000118d622a8970 }, { 0x000916a1f7d47a14, 0x000d88d20bb569d7, 0x000275e91a3c185f, 0x00072f067bd4b250, 0x0000837bba3c6297 }}, + {{ 0x0002546a940f5586, 0x000c37d53a4f5668, 0x00041560fd7cc5a3, 0x0003f20218973466, 0x0000fc85e2f5dff5 }, { 0x000a16ad6208772e, 0x0009d19c61ac1625, 0x000c51965fdc16c3, 0x00040bc4352f8fa9, 0x000086c395e5c1ac }}, + {{ 0x000644324f2ef32f, 0x000897f4faed184c, 0x000c89e5c21081c9, 0x000d382fc5f236b1, 0x0000323cdb72b7dc }, { 0x0000ed6d6bc3b8ef, 0x0002e595c5d84462, 0x0006f2c0ac0dca60, 0x000ecadc7c020807, 0x00002650966fe977 }}, + {{ 0x00067238b80e6e38, 0x0008219b6efcffc7, 0x0008214e7fc3fae4, 0x0003bf8bc60f12d6, 0x000005e6dec70c8e }, { 0x000a94dc7004d5b7, 0x0008221501b5d92f, 0x000ec80e9ac7e6f1, 0x00043be433279e3d, 0x0000bdaa02065868 }}, + {{ 0x000ae57d7389dc14, 0x000a01a530ccbbd2, 0x0006bf9cb7b1e9b6, 0x000fd790c87c8214, 0x0000c004262ed87b }, { 0x000d81a25c341115, 0x000e2bf3042a6dd3, 0x000fdbe2d8085f66, 0x000b98267b1a007b, 0x000031f1387a3dd8 }}, + {{ 0x000030f0a67c1e0d, 0x0005af8ff81578e1, 0x0000801453d6f902, 0x000e79d6f7862215, 0x00001e2caf4c2888 }, { 0x000a817bd311699b, 0x00004151f0bd7390, 0x0007dc5d42b22089, 0x000fde3982d7989c, 0x0000a862ef2d9f7c }}, + {{ 0x000a570c2107ac75, 0x0006330b0f2ac952, 0x000a80c9da2db964, 0x0007d589a48d0995, 0x000066ec9369c06f }, { 0x0005135ae5c22331, 0x0008bfd4e86214dc, 0x00020a4df9b2cf0a, 0x0000da66e2d014ee, 0x0000b652f2880073 }}, + {{ 0x000ba378872d98f6, 0x000313ad4f3049fd, 0x00030b64fbefd2cd, 0x0002deffdbb8e989, 0x0000f27faf9b53e7 }, { 0x000726e0841b2012, 0x000cea65a72f2cfd, 0x000ea4c375be3f8e, 0x000671a371208bfb, 0x0000a839bfdbb74c }}, + {{ 0x0003b681d32f73b1, 0x00029a5061f16587, 0x00016b5650dfe8e1, 0x000c99fdec0016df, 0x0000ea7e61216b53 }, { 0x000fc9eebda36ba8, 0x000c5c43ac343c03, 0x000d45a775638b98, 0x000ac46d49c847e7, 0x0000844582f5d365 }}, + {{ 0x000647b7837876bd, 0x000ad727fe02299e, 0x000143077b95970a, 0x000cf08be6335ccc, 0x0000f0ca4f1d1bce }, { 0x000627278ca4b059, 0x000f0de02cafb8aa, 0x000f569bc5401d7d, 0x0005e2982299bc2f, 0x00009199192015c9 }}, + {{ 0x00058f5c2f9e22e4, 0x000b3337933e47b6, 0x0002c8e707d309be, 0x00021299ff4ff6b1, 0x000037dd6c45dafa }, { 0x000ca95dd4570da3, 0x000b377c95b3b3ec, 0x0008648e592b39a9, 0x000b6d929fa04426, 0x00008591df48c6ec }}, + {{ 0x0004d3be78cdc4c0, 0x000cba42333fc4c2, 0x0008d262e3383d84, 0x000588b2e274f8e6, 0x00001215944a8491 }, { 0x0006a7c58ef9ec49, 0x000af58ca14f530f, 0x000264f068c28b49, 0x000f57662a6e7f16, 0x0000a687c108833c }}, + {{ 0x000f323c80b194e1, 0x0005603f9374ab69, 0x0004cf2404c4a6aa, 0x000bd8c50864f919, 0x000016461ee3c770 }, { 0x000be0fe51bd269a, 0x0000dae3f046a9a9, 0x000e43a90548ed71, 0x0008371a670da183, 0x00007e7cae92ca95 }}, + {{ 0x0003c2f435522de5, 0x00099b6cbc613e57, 0x0004d5bbb1e76fb3, 0x00050f9d533131d5, 0x0000500dd4c4695d }, { 0x0006451f5801b985, 0x0009f93a4a375fa7, 0x000c41eea66a4aeb, 0x000b4eccf5f06787, 0x0000be0cc26ec73c }}, + }, + { + /* digit=17 (1,2,..,64)*(2^{119})*G */ + {{ 0x000274dcf1b355e8, 0x000d5ef694db7e04, 0x0007778800d4cda3, 0x00062f4e1edca878, 0x000092c279274250 }, { 0x000be2b2abb3de8a, 0x000b9b86def6d1a2, 0x0000bff865bb4348, 0x0002de558d25b167, 0x0000a14c44f5e051 }}, + {{ 0x000da4a7963ed790, 0x000cc9dd111f8ec7, 0x000d3a203fc9ff36, 0x000619a51bcc33f8, 0x0000c3316282fb7e }, { 0x00058ffec2ca8e3f, 0x000cbe30bb1151be, 0x00071d53238e4f7c, 0x000c05f7854febac, 0x00007acf3890bf9e }}, + {{ 0x00074fc72746a6d0, 0x000c2fcd1e0cf9e1, 0x000da997ebc1b1ae, 0x0008e884edea75f0, 0x000051ef991ea87a }, { 0x0007fb62f575580c, 0x0006c77eb930beae, 0x000a76b4c76571cf, 0x000cc0f9b504cacc, 0x000049875ab2027f }}, + {{ 0x00066adbf6bb7535, 0x000ff9d29c4120ba, 0x000f1a65fd1f4042, 0x0009b85519f94391, 0x000007d911a8b098 }, { 0x0008234499aed7de, 0x000763c34bfca38e, 0x000fd1be8863128c, 0x00048439ce0f2755, 0x000013608ea8ba39 }}, + {{ 0x0000cbaad6fa0860, 0x0002b769d4c54230, 0x0000af867172d5b0, 0x0007ea0a19e56eb1, 0x0000613acac087c4 }, { 0x000d6254b0899ec5, 0x00036ded92a5e67d, 0x000b41d22a615f85, 0x00021935bca4979a, 0x00009335529dd4e2 }}, + {{ 0x000b6304a752fd49, 0x0007c13516e19e45, 0x00076a61d0ec2826, 0x000e9d5f656e2e84, 0x00006c970cc1cc09 }, { 0x000935381bb3523e, 0x00069363ab7e433f, 0x0000053ef767b2e6, 0x0002d839f1adea47, 0x0000b39a71be38a8 }}, + {{ 0x00035142c413720c, 0x0009f2d799e9a748, 0x000e25f8b46540fc, 0x0001df95a96239e0, 0x0000745a4815a75d }, { 0x000856e0e8e441ac, 0x000ba7aa2e8807fe, 0x0004324cf2b5bfc3, 0x000bac6f529f2c08, 0x000024c840287b5e }}, + {{ 0x000cb10d572b5962, 0x000ed9d918e49366, 0x000645a02c2b89f6, 0x000f43ab965035ef, 0x00005fc3fbf56b43 }, { 0x0007c7032b9ad449, 0x0000eb7a242da112, 0x000d3f646f3cae05, 0x0002e606b16a4d3d, 0x000066b08ba950ea }}, + {{ 0x000b92420a9f510e, 0x0001d18e4136a147, 0x000cebfbf0b51e6f, 0x0004b2e03bacf969, 0x000059ed4c55993e }, { 0x0008945e1eb1c203, 0x0001c78c64d565c0, 0x0000085e247722f2, 0x0007b021a9f969d0, 0x0000a495ad2457ff }}, + {{ 0x000fa003d16724c1, 0x000aa7d5846426f7, 0x0008a4dd404bee11, 0x0004afe48e7d09e8, 0x00004e388ffd7c0c }, { 0x000d40da8e2b1cbb, 0x000a64f17fd95965, 0x0000d88abe4cfada, 0x00066ec6a49e0e0a, 0x000085fa4175ed03 }}, + {{ 0x0002031717ee94cd, 0x000f705050bad247, 0x000d50a7310c0423, 0x000ee98a78918426, 0x0000a7a77c69c5b4 }, { 0x000fd7cb763dc643, 0x000939960a94120d, 0x000d09c6e3764d3a, 0x0009a67fd478a255, 0x0000291d53e1e931 }}, + {{ 0x00089bc84468e031, 0x0005ab4a595939dd, 0x00084fd839d2cf16, 0x000cd45120355647, 0x0000a31eb877381b }, { 0x00012d8643774d44, 0x000965d85a9184d6, 0x000b3e932a3180a6, 0x0003d448e562563b, 0x0000bf1cf2a46781 }}, + {{ 0x0000885735dbeebd, 0x0001ea7f64085e79, 0x000399222ddb6a70, 0x000b41488b230f30, 0x00007e9e9ee9f0f1 }, { 0x000cdbdf44d9a14c, 0x000b39a652f54f6d, 0x000382d24cab9561, 0x00054a2e365c8229, 0x0000b474dc1a9b89 }}, + {{ 0x000d03d7d3318572, 0x00049a354530bb24, 0x00077e0492176e2e, 0x000bd9f63bde3ef7, 0x0000086e3e2a72d9 }, { 0x000280da9fc53e22, 0x0002d9e43b6a782b, 0x0006ea07cbe66e70, 0x000c1cbd9216db30, 0x0000dfa49ad0403b }}, + {{ 0x000fd508e7dab215, 0x000b68a1635e7417, 0x0009f8e8f9bf57f5, 0x000c8cb30fe14008, 0x0000d09a8cbf4c40 }, { 0x0000c96743cb7c12, 0x0002019d607ad51e, 0x0009ac772f06a74c, 0x0000b7ba20f229c0, 0x0000c906956df251 }}, + {{ 0x00048234a455944a, 0x0003c913e538cd75, 0x000c398678aeeffb, 0x000c8cbffa7001f6, 0x00008153cadc4269 }, { 0x000cbd665cbd5dcd, 0x000d02d3d5b40458, 0x0005bdd2db3441fc, 0x00002b28bb0ffeeb, 0x000086864413478b }}, + {{ 0x000230378fd7d8e6, 0x0004ee0425830687, 0x000b81c6aef3b01f, 0x000e83c4802da2ae, 0x00008bd11eeccd2d }, { 0x00084d402dbd9f8e, 0x000f2b571e8e5cae, 0x000bcb68e4f6d61d, 0x000fcc72e53ab041, 0x000092f0d852e5ae }}, + {{ 0x0007fbf56c3f426c, 0x000c973f37f59371, 0x000db9e90739643c, 0x0004844fceb0f6c7, 0x00001850ac1be86d }, { 0x000ae327a65859cf, 0x0009cd9acf7d720d, 0x0007353f1fbfd5d5, 0x0007b6cef41fc8f8, 0x000076cef0bc0d2d }}, + {{ 0x0001699a1c4e7ed0, 0x000d065c48182672, 0x000b165821a51c51, 0x000ba5a9233aa189, 0x00000691ac4135d8 }, { 0x00081497d823b2a6, 0x000d19cc2d7a7fda, 0x0008f1b3a51aa96d, 0x0000e12c7e318605, 0x000025170752334f }}, + {{ 0x00044990c52f624e, 0x0000e891f5f82dce, 0x00067f1776d921fb, 0x0004105309a7f67d, 0x0000dff3f33f1de2 }, { 0x000830f09a7edeef, 0x000fe95af103e596, 0x0007257ba1c748f6, 0x000b5c9d3ece8aba, 0x00001dd530aeff14 }}, + {{ 0x000520304e191d19, 0x00010b488172d014, 0x000585df6db40bed, 0x00089199a8b20bd6, 0x00006e8ca44ed0cf }, { 0x000244c2cdc93ee2, 0x000f81f2b47f7ef2, 0x000839bb0272e506, 0x000261674e07f528, 0x0000069e938f3b6b }}, + {{ 0x0004b043a81d5e7b, 0x0009fc194d7d9b17, 0x0002537e3b6f2665, 0x0007b94ca5bafdd8, 0x00005587b44c1942 }, { 0x000143bdc27d5d11, 0x0009ea19e71734c5, 0x000e4b01516bbe49, 0x0008531b115d5437, 0x0000847671d73bea }}, + {{ 0x000037543bac95c5, 0x000c653fc5e89202, 0x00087a6599cca795, 0x000b2a908363bf9a, 0x00000d8e95cfd516 }, { 0x000badceac56e4f2, 0x000cacc625640b46, 0x000e20185d8f1a24, 0x0005a34eacbc1051, 0x0000d0e37c834129 }}, + {{ 0x0003505bb870eafa, 0x000d47c6b53f5f95, 0x0004c7c711e131be, 0x000710e1b6424659, 0x00009ecec42b7e48 }, { 0x000febceda1c9c8b, 0x0001361f2ba70b0d, 0x0008ab0f057922d3, 0x0004ebf6ad07e16c, 0x000067dbc750e1cb }}, + {{ 0x0000e4587f5ed835, 0x000a7dc53868c8b3, 0x0002077caf2733fb, 0x000c31964174311a, 0x0000356efdbcfd09 }, { 0x00003d37200e52c0, 0x0002a310811b3b2d, 0x000c3c910cca75c8, 0x00047b8036144d41, 0x00004f8461ce611c }}, + {{ 0x000602500ca3dbe7, 0x000f0f2b1f3390b4, 0x000c5fb3c266229a, 0x0000540a99f7a1b8, 0x0000cbac9856bd1f }, { 0x000289eb1fd8ddb6, 0x000c11c4711230f9, 0x000b6f851ea1a62b, 0x000082d8b058eb37, 0x0000dff1bd082c26 }}, + {{ 0x00009e4ea0d957a8, 0x0009eabd51b9cf59, 0x00020e62d30ee577, 0x000b61a74d2aa9c0, 0x0000dc7f6451320f }, { 0x0007e947df046bb8, 0x0004c966f32dbdb8, 0x00000319c35fe24b, 0x0004a5fe2672188a, 0x00007d6b8255d4cb }}, + {{ 0x000f11c4d76b7c46, 0x000e5391770f5a7b, 0x00004f3326ab38d0, 0x000ed2b2ae4d4d79, 0x0000e4ef511e46d3 }, { 0x000a61e28b58878e, 0x00065e542c4224c6, 0x000bda3bb1915d60, 0x000c95a78d6b4e81, 0x0000df4b98688afb }}, + {{ 0x000f0956569ac4ed, 0x0003394e6f1cd952, 0x000cde0f10e89862, 0x0003727647512f30, 0x0000d19eb33b73c0 }, { 0x0003d7563e475d5b, 0x000eccaa82262004, 0x000cd6e91d2b1111, 0x000b651be02c868a, 0x00008a5278bd077e }}, + {{ 0x000c20578305b14e, 0x0004fc7ab952578a, 0x00017ac7f5884f29, 0x000d5d242b5423df, 0x0000620247456895 }, { 0x000f807ef9d65a21, 0x00075eb21f4c77af, 0x0003ab31f4b5799c, 0x000dc92f11a79c3e, 0x00007300963434bd }}, + {{ 0x00011afff4ca371a, 0x000d03d778f22a00, 0x00085a437474b875, 0x000e1d47dfb10b2e, 0x00002489cd838b5d }, { 0x00011554104be448, 0x0008926ad124418d, 0x00051618d54f49c2, 0x00010937531c081a, 0x000071b18a1c2aa7 }}, + {{ 0x000d41f01735a335, 0x00073f833f6746cb, 0x00054167a0b39250, 0x0003a7ba921e46f6, 0x00006c09c11aae95 }, { 0x000d59e95785f38c, 0x000b834426ffb589, 0x00057781d8acae7b, 0x000a30f7f8055943, 0x0000f1dc76c3a6d4 }}, + {{ 0x00008d0596f30cdc, 0x000bb1c8ad4cf4b4, 0x000fef6e8a21fea3, 0x000272fa40344885, 0x0000efa2f182a572 }, { 0x000acd518b235195, 0x000f9a498c364b09, 0x00046a51392ac475, 0x0000bfb02a4aac4a, 0x0000e0426618d525 }}, + {{ 0x0009dd82ce983ea9, 0x000bf8fd06f56c0a, 0x000d3524f6a44c8b, 0x0005bb79da48af70, 0x0000dc57803535c6 }, { 0x000cdada016942b8, 0x0001263a0ef18d34, 0x0004f091c9bc1fc1, 0x000b1c5039fb7ee3, 0x00008fc41c7e9b42 }}, + {{ 0x0005a0cad09fcdc8, 0x0006f499355a244c, 0x0005898086d8b232, 0x000e727cbe77fd2b, 0x0000f5438a6a25c6 }, { 0x000338ecde1a98b7, 0x000475cef8118650, 0x00089b6cdf5ef5b6, 0x0001cd6bcf52813e, 0x0000ef9cb64dfe26 }}, + {{ 0x000e182e702cbf4d, 0x000093774921592b, 0x000cfba36ec1cce1, 0x000dd0759b4f1261, 0x0000c2c3bb484721 }, { 0x000f11d5488868ed, 0x000c37cdfc50573d, 0x000983acb7fc4b3a, 0x0004278d39ba1afd, 0x00004a4526ea82ca }}, + {{ 0x000db4207965c6e3, 0x00020ee02a2404ab, 0x0003635469f7a3cd, 0x0003dcb71497406f, 0x0000db415a568fea }, { 0x000e3be64146ee7c, 0x000270fd88284423, 0x0007be7a06dae484, 0x000b58415283add2, 0x000031007377bf1f }}, + {{ 0x000b74b0ed03932d, 0x00050c1e038a6759, 0x0002f0e7b233a87f, 0x00079ff53729deff, 0x0000c01ffe0be7a5 }, { 0x000184c1d9aaf43a, 0x0006c23dab407e5c, 0x0004b52e7eecb1c1, 0x00033ff2ccdbd361, 0x0000098af3b980d2 }}, + {{ 0x000904bdbb6109f8, 0x000571981d3a80f5, 0x00016fdb66776216, 0x0003fe8c1fde0c84, 0x00002e8025758320 }, { 0x00011469a106add9, 0x00044d1101b68053, 0x000c189603a82a4e, 0x00069678743c223b, 0x0000a9746fe0a073 }}, + {{ 0x00021ed0b2fc0d23, 0x00093845e43fc616, 0x00057eb4daa4b57c, 0x0000afaa8f328237, 0x00006ae9919db4b5 }, { 0x0003856ec86c297e, 0x0000fe1ad9906763, 0x000b234fd19d2eee, 0x0000f4401d51de67, 0x00002462d036fc09 }}, + {{ 0x00033a00db254686, 0x0007615be6cdcde0, 0x00039bd26eadffdc, 0x000f7aaa61e603f3, 0x0000199ba172e7ea }, { 0x000eef4c32c845b6, 0x000ee09f7bb7fd76, 0x000dd73dccc40b89, 0x0003a2c9570f8781, 0x000026152e934da0 }}, + {{ 0x0000ea7d6756508f, 0x000a13b2f7e85c33, 0x000140ae12343058, 0x000e81db55fce462, 0x00007980f96d8f6a }, { 0x000a553d4d7f0b2d, 0x00088f4e51ab9a22, 0x000a83b13f2b7013, 0x000bf6fa60587c98, 0x0000a508840aeef8 }}, + {{ 0x0000b460d07f52d4, 0x000ae364da29ec28, 0x0003391112464836, 0x000853947dbdd5d1, 0x000061c41be19aad }, { 0x0005f292a360f3f1, 0x0009babf963c31e5, 0x0003765aa05a3770, 0x00003d58767f9f63, 0x00004a15f588fa9d }}, + {{ 0x0003283f79a5eaca, 0x0009eb2cd21ab3de, 0x0001e06b9418e3f9, 0x0001670f9e504f02, 0x0000446de46c5aff }, { 0x000101dc7671519a, 0x0009bf6226bf99fc, 0x00034d2741f38239, 0x000ba7c2de835427, 0x00003411b6f79f40 }}, + {{ 0x000e01db0d46ca69, 0x0004bc2255bd6178, 0x0002de28b6473c59, 0x0009d2a876c96969, 0x0000307faee6531b }, { 0x00082ca080d00c87, 0x000a173893a55595, 0x000aef7740c863b7, 0x000c2aef87630f04, 0x0000b32db7ee850d }}, + {{ 0x00037f9c22a738a4, 0x000b2e10b1894a02, 0x0001b49ccc9d8538, 0x0007871b59203400, 0x00001570db64de21 }, { 0x0009ed518775904d, 0x000a62f4470e2c05, 0x0000fed0fd0d9912, 0x0001e0763b725f7c, 0x00008b2ac0c6733c }}, + {{ 0x000b0b081e23ebf4, 0x000ff16d0115d4d7, 0x000f585e0d7bd2f4, 0x0009316415180b12, 0x0000ddbe5f1eab2e }, { 0x000974e2b1e03fb6, 0x000f48abf4f0dd06, 0x0006e3b043e48d7a, 0x000b8aefe99b289c, 0x00005cf08248acfb }}, + {{ 0x000d737e72363c9d, 0x00021e206ddd6572, 0x000102802afaff9f, 0x0002f4a58e7cbdd1, 0x0000cf910137f5d1 }, { 0x0006f52b1e708142, 0x000696286ec37766, 0x000d2d6a7c82ef02, 0x0006dc73398aa649, 0x0000cffe04031902 }}, + {{ 0x00032721221eeb7a, 0x000defbd803738b6, 0x000781c149828000, 0x000c99a9291aaade, 0x00009626df3ef586 }, { 0x000571b8055894f6, 0x00032f0862c51290, 0x00092dafec9b2579, 0x000acd28f0a8f79a, 0x0000403590545ef7 }}, + {{ 0x000f5f638ebe18a2, 0x0002adec95207114, 0x000ced2c21311398, 0x000e6511455a9e4e, 0x0000ba02dffbaae3 }, { 0x00017596079bc6ae, 0x00070d32bf9341bd, 0x00007bcd3fa21e63, 0x000a719c5d63c132, 0x0000589ce7628195 }}, + {{ 0x000076bc4095656c, 0x000b41ff9e6ed353, 0x000536bfd61a4343, 0x000f2aef7b7b5ee4, 0x0000916f72ab7c03 }, { 0x00070cb65edfe066, 0x000638c02dec312a, 0x00077c8115197531, 0x000bba2c9270de89, 0x0000cc65c87f3d73 }}, + {{ 0x00032e89869168bc, 0x000050ecfe57bbde, 0x00055272392f1fc6, 0x000b3abbd82c7b65, 0x0000528c02084684 }, { 0x000152ce2cb419fd, 0x0009744d11a5529a, 0x000182d52bedc6a0, 0x000877a0ae964ed0, 0x0000f927ef680070 }}, + {{ 0x000871bfa5e6e0c7, 0x0008ca48ec07649b, 0x000f8d2c14ad8a64, 0x00083ea1018b4c28, 0x00004818a1be6e94 }, { 0x000b0cdf9b0c7f86, 0x0008ce59e2e3c9b3, 0x0005099db15ba9f1, 0x000d40a06fbf250e, 0x00000ac8ed941db0 }}, + {{ 0x0006facb0812925d, 0x00042eaf8c511875, 0x000c1d23829fdb64, 0x0007ee64505bb67e, 0x0000b90376a129f5 }, { 0x000841357c4ddd34, 0x000d3e98f6229e49, 0x0003ba2e8493a602, 0x00056697962d103f, 0x00003d04cfbd7b62 }}, + {{ 0x000750cc3c1a7b0a, 0x000b159c34e67058, 0x000af889dc10e012, 0x000957b37c3c7180, 0x0000968c82b0c089 }, { 0x000d86dbd34a175e, 0x000689db7275d7d1, 0x000b10d2d7166c05, 0x000b45693ffa0168, 0x0000abf693ace924 }}, + {{ 0x00071b360f0679b7, 0x0005467e28ef5ba5, 0x000973fdab5aa7a4, 0x000f206202c9a469, 0x0000e0be2b8f2417 }, { 0x0004bed023283151, 0x0009958101c1d454, 0x000dd3c9342fa56e, 0x0005c83466651788, 0x0000f38d09a711f7 }}, + {{ 0x000343b5fb200f96, 0x000952d33da525d4, 0x000dcfd099dce93b, 0x000b1701df193678, 0x0000ef95773ba9d4 }, { 0x000484d4e14aaebc, 0x0006771638127137, 0x000d82dc48b25dc1, 0x0007325d747bf6d2, 0x00005e1c9c82ab72 }}, + {{ 0x000c8d3645c866b0, 0x000beea60dbac1fd, 0x0003fd796cb1baa7, 0x00063a35b23d8c48, 0x0000e075408e6a88 }, { 0x0007c5425467d1b6, 0x00043fab12177b4c, 0x00038b881b47a89d, 0x000d9800bf79bf46, 0x0000a2c54416f53e }}, + {{ 0x000a0d41b10e0cdc, 0x000c447e5911a766, 0x0007fa71dd36a5ba, 0x00049cf4261ffa5c, 0x000079b9ae6dbe81 }, { 0x000e89e390495427, 0x00042f5cc30d3577, 0x0000ece8db061fa0, 0x000e6ad61be9adf8, 0x0000aae7a36a6e46 }}, + {{ 0x000c6cb4fe4e3e7e, 0x000afe5facf69d43, 0x00060b087f4ea2b9, 0x000a4ef04363b7e7, 0x0000a37a66bc1464 }, { 0x00008b945060128f, 0x00034394008e1f2c, 0x000964139e1c25c7, 0x00079cf065e9a85e, 0x0000f97bbcc0ba07 }}, + {{ 0x0003879cbfbabf2e, 0x000922c89f71f0f1, 0x0001ebb607012381, 0x000532fedad8f3e3, 0x000042e5dc9aab87 }, { 0x0007a589356da4cc, 0x00048a56b306a0b0, 0x00065fbee32901a2, 0x000cd9768359c2ea, 0x00005460cecc0d99 }}, + {{ 0x000dc35c554b2226, 0x0003bd168754ab0e, 0x000c817f3e20d9b4, 0x0006ac169801fce1, 0x0000801dc7720a5a }, { 0x00096d0960303067, 0x000c06fa3f660d1d, 0x000ddf8f6807ba00, 0x0005d49aea0184eb, 0x00007165e44fb9ed }}, + {{ 0x0003109d80d2b121, 0x000e1b5f8c2a25b9, 0x0008728167361b42, 0x00003c24c1bb772b, 0x0000da906c6e6147 }, { 0x000cc31b47879cf5, 0x000d7f78ef59fb59, 0x0001eb43ee1bcd5a, 0x00056c59b3b438fe, 0x0000d98902a011aa }}, + {{ 0x00097ba641eb7f97, 0x00074a9b733aa5fc, 0x000b26df4bf350af, 0x0002ceba5dece47c, 0x0000cc1ae7e7d3b7 }, { 0x0007fab43b1d99a8, 0x000ad0409c110608, 0x0004beb49cf2a615, 0x00058d94656ea2c0, 0x0000cf90618174d6 }}, + }, + { + /* digit=18 (1,2,..,64)*(2^{126})*G */ + {{ 0x000b433b59bb0813, 0x0000341d4c5105f3, 0x000e323c820b4e81, 0x000deaab01ae80f8, 0x0000ba1d2bfa0603 }, { 0x00002fefd81e661f, 0x0004eb693e856387, 0x000273b996572680, 0x000f613ecf7b5925, 0x00002889ae807f47 }}, + {{ 0x000e89f3d33e658c, 0x000bb5d0fefb0c3a, 0x000a984dcf89f8ee, 0x000d20ca4d48fb56, 0x0000b9304c4e5ff7 }, { 0x0003083bb07dcf66, 0x00023c86363d14cb, 0x0000b4a396cd193c, 0x000063a218981752, 0x00009e66befec3c7 }}, + {{ 0x000047f6491b324e, 0x000f3debe949a447, 0x0007e422595c1d64, 0x000d22ddc638e938, 0x000012ead0995384 }, { 0x0007d3336dfbf8ab, 0x000df9041fbab170, 0x00084f98a83458eb, 0x0003252dcda8e0b2, 0x00002ddc2d664ecb }}, + {{ 0x0001d587d483a8ac, 0x0001644bfe209256, 0x00032e076092df00, 0x0008fc46391c19ac, 0x0000ca7c69ca0159 }, { 0x000a11fe134033ed, 0x00086a2a37173eaf, 0x000759658d52a842, 0x00086c73e2384800, 0x0000d0571a330ac5 }}, + {{ 0x000b0b43fae02d19, 0x0005e2e81685d7b3, 0x00018429657b244e, 0x000322fef23f27d8, 0x0000be6797dde7b1 }, { 0x0000b732a6bc6506, 0x000f52663ca72840, 0x00041c968590b9ae, 0x000d64c6b7f5729b, 0x000078435f1bf19b }}, + {{ 0x000b01ea6b5b07b3, 0x00041262048e3968, 0x0005ad3bb37932fb, 0x00016d2ad436b50b, 0x000041c792fd15ec }, { 0x00054b52698a8d32, 0x0007dee97015c07d, 0x000878f05045493a, 0x000466fd246cdf1a, 0x00005d2f84c497f9 }}, + {{ 0x000f1bc8463a493f, 0x00090304e23e9bca, 0x000563526dde04db, 0x000c22e614387f81, 0x0000c1b1d1e0bd00 }, { 0x00058493c7b7d8e9, 0x00018ba527de79c6, 0x0003dcdd27ee6deb, 0x000a262c70e1346d, 0x0000f69deb198370 }}, + {{ 0x000617ff6bb8909f, 0x000aafeb8a24a205, 0x00019f14a5d36b68, 0x0006aa95317ebfed, 0x0000c8fde1bb5304 }, { 0x000967ce0fa78f8e, 0x000a6c5e3c99ebdb, 0x000d5e1c475a7c8b, 0x000cdd9dffc64242, 0x000020302e081fc1 }}, + {{ 0x000d6c2966703b01, 0x000365121c80b5d3, 0x000c65b2a339f5a0, 0x000fd9f7e2e7a563, 0x0000bb183ed3e7e1 }, { 0x000a43951972d8ef, 0x0008e1aaf70b8855, 0x0008d324cf4076c4, 0x000efcb5bafc3bad, 0x0000bcb50e192c99 }}, + {{ 0x00087e191f93cace, 0x00049a85fe4e3142, 0x0000d43a775064ea, 0x0003b4d23e60234d, 0x0000bc6b31f78d5e }, { 0x0002da4b5432efe0, 0x0009ddc09d877d88, 0x000b918d3decee95, 0x000f07ea41c23478, 0x0000526c06550283 }}, + {{ 0x0003639eb906c44f, 0x0001891fadb86087, 0x000d08112f3df08e, 0x000f4bb34049e832, 0x0000c537963213bb }, { 0x00072064455862d7, 0x0005c1fe1e4a71b6, 0x0003195294d3f757, 0x00004b9326ffd6b9, 0x0000135f9599d70d }}, + {{ 0x000418871e270c0b, 0x00029a3b23358342, 0x000a6cd6e14299eb, 0x000d6b00d9526205, 0x000018ce7f6af8dd }, { 0x000fdb3c22d19812, 0x00064814082c1576, 0x000fb9e6c451057f, 0x000efd528cc914ac, 0x000020534397c913 }}, + {{ 0x0009874445537fd6, 0x0003ea6f2f9f8e93, 0x00031665058cdd50, 0x00085d2d9963ece2, 0x0000dfa018b78042 }, { 0x0000899785dc3ccd, 0x000326512bf476a1, 0x0007f945e60b257a, 0x0008189ca598a64d, 0x00009ba39facc3b7 }}, + {{ 0x000b8bc785ae777b, 0x00017fd02eee3ad4, 0x0004e4888aeb0a29, 0x0009b3b75a86b3d3, 0x00009495294c40d6 }, { 0x000cd576bcde4c88, 0x0009754413f9cdef, 0x000e2ed5dfa79ee3, 0x000736423a5e4fb4, 0x0000c8fea236703e }}, + {{ 0x00088fe5f08c69ad, 0x000d03e83731a34b, 0x000e34f3889d77d8, 0x000a91d7c2bb9028, 0x00002a8744970829 }, { 0x00069e9172abed8b, 0x000e0c6edea4eb34, 0x000301f05b05123c, 0x000837be143b9313, 0x0000a5596eeef405 }}, + {{ 0x000e205c1544399e, 0x000710441c23fa36, 0x000053783e19700e, 0x00068fea7b4712eb, 0x0000a112e4fbfa4f }, { 0x000514d89ca1abfd, 0x0007c57b36416860, 0x000d4097b2b1ef2f, 0x000a206500432691, 0x0000b3a43e7e1054 }}, + {{ 0x00090d4325fd9679, 0x0009b7fa8e67fc38, 0x000b22134298196a, 0x00091d5579e2e0b8, 0x00008a49ee4dfeae }, { 0x000a54ce4da649a5, 0x000b96ec55e68a39, 0x000aa8e4fa0ce432, 0x000d91155bf12e90, 0x00004d3d79423852 }}, + {{ 0x0008a43035135501, 0x000e73af1a142952, 0x000c89e1a4c4c49f, 0x000aad9ba916f955, 0x000087d02c81ab86 }, { 0x00053db5fc807a2d, 0x000d2a2b8a9cef88, 0x000e34ae4f1d89bc, 0x000eeb9725517407, 0x0000f797b5d79a69 }}, + {{ 0x00061e9a4969a240, 0x0000ba9e864a51ae, 0x0008886b8141b7a5, 0x0006fca286c26769, 0x0000240005dbdd2a }, { 0x000449c2ec4a9e15, 0x0003b039f84216d6, 0x000748fa0a993339, 0x00074b159ab020d0, 0x0000997a8c005c8f }}, + {{ 0x0008a5b251a46688, 0x000c41c2cf9d7c1c, 0x000a25f0335a2815, 0x000478f0c1320886, 0x0000b3973c66051a }, { 0x0004dada9450a7b7, 0x0005d32c11d23031, 0x00020a289c0afe30, 0x000abe1287da6691, 0x0000967694826933 }}, + {{ 0x000c3bd859c92095, 0x0002e217ef6b0d15, 0x000a205766c8c264, 0x00023768252a9477, 0x0000b7cbdb2f3134 }, { 0x0000ac583b4bc642, 0x0003ef5f22c2d3e6, 0x0000570f148a45c5, 0x000cf5f1c368d504, 0x0000aba17b2c65da }}, + {{ 0x0009f147f44133d1, 0x000ea980eab2437e, 0x000474350cd3f048, 0x000a45f7f8cc08c5, 0x000087556fee2ca5 }, { 0x00043f23a09ea234, 0x0004914d2fd6cf16, 0x0003c910a80fc0fa, 0x000d10df7a3ecd06, 0x0000fce7e70f4fef }}, + {{ 0x00085c4a77db6510, 0x00068abe65ab743f, 0x00003d917e24f825, 0x000a8e568bf7c75b, 0x0000f920e13ab677 }, { 0x000d8f276467f157, 0x000feb2e68acf374, 0x000d5da38820e4a7, 0x0006b9d93a544df3, 0x00004f65f0f3913a }}, + {{ 0x000ce5870a0c41fd, 0x000faaaf0dcbc49e, 0x000883d62d516f72, 0x000eaa5e57de551f, 0x0000f8ef2d69da92 }, { 0x000ff5e2f5425d4e, 0x000c0dd167d79ae1, 0x0002da879bcbf034, 0x000d36191039df8a, 0x00004f16c971cb78 }}, + {{ 0x000fc8cc32aee2fd, 0x000a3396ea51d668, 0x0004f0b4adf36311, 0x000779e24506c144, 0x0000d07ff487d2c6 }, { 0x0003173d2554ae54, 0x000d94d9433d67d1, 0x000f576695712eb5, 0x000cd8a077fa5cb8, 0x00002098fca8b646 }}, + {{ 0x000723c7f687405a, 0x000ff9dd5b340fff, 0x0006b9daaecf62ef, 0x0004f00dcfc6b529, 0x000082ac85fad224 }, { 0x0007febee863592d, 0x00096219773d4025, 0x0002c3531e5edeaa, 0x000d2b53cdf7c6aa, 0x00006f48a45191f1 }}, + {{ 0x0005d48ee54627c0, 0x000fee47bd8e7397, 0x0004388f30907fc4, 0x000d3c92379d4499, 0x000014e6df0129f2 }, { 0x000bb8f6ef5589da, 0x0009daadb5cdf33b, 0x000580b07079ffc5, 0x000738d4e3396e89, 0x0000d3c9aebf037e }}, + {{ 0x000aee99a83de201, 0x0000551cbae2f701, 0x000578bc0f4313d7, 0x0003bf399efc4bc0, 0x00004db1c0ede19f }, { 0x000c5f317b20b8d4, 0x000b1b9ae8274c57, 0x000500d18221751b, 0x000091b816c14d44, 0x0000685f584b909c }}, + {{ 0x00037932990a975f, 0x000b1754ef442271, 0x000ec8a7c1fcf689, 0x00000f4cda17bed6, 0x00008003fe108b77 }, { 0x000ac3793bd9befd, 0x0003378dbbe307b4, 0x000e2c2b1ffa6260, 0x0007dad206dd1c20, 0x000049b2e9cf1b68 }}, + {{ 0x00013eddb15b9981, 0x000b7ebed1f8ca0c, 0x00078d7c47f137aa, 0x000d0086f3e54665, 0x0000ee5e70525d64 }, { 0x000020737111ef2e, 0x000f24fd49eb8f61, 0x000cc72ea2ff6814, 0x0000cd758d0d5bdf, 0x00004f38669d1545 }}, + {{ 0x000ed13c9ad8f9ee, 0x000f8900647a82b8, 0x0004006ec8b4868d, 0x0008b4a03908e0ed, 0x0000ecf41ee539a0 }, { 0x0004380e0ee30563, 0x00093761f460f648, 0x000ab29855275b29, 0x000bf349b53930b9, 0x0000a5933dc89dac }}, + {{ 0x0003bc3770684e62, 0x000ca6bc4fe3c395, 0x000318bfb46f7b63, 0x0001ad259ba4a815, 0x00008206cce146bd }, { 0x000c2731bc0908f4, 0x000d4ab5afc60db8, 0x00049492c0e73d3c, 0x00006d107aa02235, 0x0000a2cf845790c5 }}, + {{ 0x000e83daed4291aa, 0x0005eab18a0339d4, 0x000ce8b1687fb104, 0x00007814fac7a658, 0x0000c1a639fa1a8b }, { 0x000721d740a55e87, 0x00040be2f39ea3e5, 0x000f598395ede4d6, 0x0007ef938ef0c005, 0x0000b5b0afcf29c9 }}, + {{ 0x000530a00b6f6334, 0x000e111fbeeee1be, 0x000309ec6bf9af40, 0x0005d4935d1ef5e7, 0x00009dcd3183c604 }, { 0x000632a3a250e986, 0x00031fef15ebfb03, 0x000c9cefa5780c6c, 0x00071ac37549de03, 0x000059126cd86023 }}, + {{ 0x000f033918a2c159, 0x00061890ef59f78e, 0x0002dae1f2816a7a, 0x000a1bb470dfc644, 0x0000c8d185b680cf }, { 0x0003bfaab3be88e6, 0x0008d5db13839481, 0x0002ad1dd2c94e09, 0x000b5ea68c1fc29a, 0x0000900f4ef1e381 }}, + {{ 0x0007640487bdae6e, 0x00058a90deeaf523, 0x00021500178d835f, 0x0001696c4b003fb0, 0x0000fe99cf01049e }, { 0x0002f3acc9ce3d00, 0x0001bc6222a77da2, 0x00073b543e47375c, 0x000de432c62260a5, 0x0000b372f8b94a23 }}, + {{ 0x000e8055d0741a6d, 0x000dc228e41d155c, 0x00076c40041ede78, 0x0006dd7114d02456, 0x00005f55ba2e84d9 }, { 0x00009bcf6bd46086, 0x0004baff62045490, 0x000ec66a97edf917, 0x000123c29a0a00a3, 0x00001c1ad528832e }}, + {{ 0x00017eca6dedef2e, 0x0000aa822305177a, 0x000e9fe591513c37, 0x000b875c0d51d59d, 0x00005bf481a68fd2 }, { 0x00042c3114902bd5, 0x000b2ed60d9dcec0, 0x00011c80f3407b8c, 0x000452dbad6d28e5, 0x00001d77ed06f416 }}, + {{ 0x0004dacf96735388, 0x00012c58aa68e305, 0x000bbfa881a889b6, 0x000da070c7a6a3d7, 0x0000eb1e4523df46 }, { 0x000d0b1e0fc6e9e4, 0x000856fa4a9af892, 0x000fd9a17add2f5b, 0x000bc96203a41193, 0x000035db9779e65b }}, + {{ 0x000a132f7ea9a6a6, 0x000eb9240be34e8c, 0x0006c77d4fb67e21, 0x000afca2bc7162b3, 0x00001a6fac0542ff }, { 0x000f9b75c44f951a, 0x0002af90e2025b4c, 0x00083cc77b87ef39, 0x0004eb21167aaec6, 0x0000d22d6db853d3 }}, + {{ 0x000c7cce51163e7a, 0x000b1164d348272c, 0x0009ccdd071becbb, 0x00037af16bd99d61, 0x0000d10610c58a02 }, { 0x0007eb5875e27497, 0x00054aad846bd832, 0x000c8134cfd36ae5, 0x000aab5a98775a9d, 0x000086c275896b8b }}, + {{ 0x0005528b5f9ab927, 0x0002651662cc565f, 0x000a1861882d656f, 0x000797e9482353a6, 0x0000a67f2e464b0f }, { 0x000d6414aca8eb30, 0x000d2312f23c5ff3, 0x000a2d9b8ef54a36, 0x00042ada0e3e8147, 0x00008166c958af77 }}, + {{ 0x000188223a7a093e, 0x00044e1a51a168a0, 0x00014726320dde66, 0x0001a00b8b712c67, 0x0000e412af6796fc }, { 0x000487211ce123d4, 0x0008a21f7c18de1f, 0x0000bdf491697418, 0x0002e15297682e45, 0x00006b2b6ce9962d }}, + {{ 0x0009c135ff5a9bcb, 0x00077770c24efc82, 0x0004953601a4b26d, 0x000b112280349fd0, 0x0000209df732d2f6 }, { 0x0007f7525d0fd68b, 0x000ba08f78b0b6f2, 0x00023f94c1dfda97, 0x0000b9365176f5dd, 0x000020eff3bcbb3c }}, + {{ 0x000a7c2c5789636a, 0x000a409748ab1a42, 0x0000348217db9653, 0x00093f0f29cba50a, 0x0000ffd72e054e8c }, { 0x000e9d1cf178ebdd, 0x00089a43f0a449a3, 0x000b6ef3df4bc9f9, 0x00075f35745474c3, 0x000083ae967e4c12 }}, + {{ 0x000fa729a87fe481, 0x0006353f380a6e68, 0x00010b6d0e19961f, 0x00044e00a0b86e43, 0x00009f6a7ecb4a6f }, { 0x0002ed3e3509796d, 0x000026e6c9c217e3, 0x00095ea456ef349f, 0x000c3e2b1be1d307, 0x0000a66932593b0b }}, + {{ 0x00008a89af9a9081, 0x000817f9eb1a8b70, 0x0007c527a6393cf4, 0x000462091bc37eb7, 0x0000c5e0712c411f }, { 0x0003154f5208b269, 0x000bffb7d42c10ab, 0x000e969ea76610fd, 0x000ee44e37fe8c92, 0x0000cdae2d966e9c }}, + {{ 0x000c5def04862376, 0x00079b6d9790ed61, 0x000c8a4c8a36ca43, 0x0001a8e2ea77a0cb, 0x00004f8456e6379c }, { 0x0007052acbcdeaab, 0x000557873b6c1da8, 0x000275d32aae4098, 0x0007648b155cf85d, 0x00002e5661fcaa36 }}, + {{ 0x000e8d5c0dec3769, 0x000c280907eae66d, 0x000f6ee4c07b1ce9, 0x0000e431c3cb068d, 0x0000c8b6234ea0f1 }, { 0x000a212bebfadf0c, 0x00013dce3502e6ed, 0x000073687498df63, 0x000e33ffbc894420, 0x0000da93c59a72f5 }}, + {{ 0x000d7b05dd7857f5, 0x000ad4bf82ce682c, 0x00053a7c1a0771d5, 0x000a2e3de058d381, 0x000087e07d1384e7 }, { 0x0002699a5f9d773f, 0x000b9ed1d9f54ec8, 0x0008306a1b6b36cb, 0x000f3ede28be3d61, 0x00001ffd2fb37142 }}, + {{ 0x000e5782cd47a8b9, 0x0004df571c0c3a75, 0x0009aedc953ef848, 0x0002f1a758f580f5, 0x000042380c4840db }, { 0x000acf2240badb82, 0x000d60c99c82a70a, 0x000e6b1b6a0b6799, 0x00065dc70e8359fb, 0x00005463c0ed77c0 }}, + {{ 0x00044e55ad6bba22, 0x000aaf5e4652f1d2, 0x000ab113020533f0, 0x000a7ca50bd6fdd2, 0x0000ffec60c771f6 }, { 0x000d2dbc0fe69c05, 0x0008e05047d320ba, 0x0004b4536a2fa180, 0x000f3b3ee1ca983d, 0x0000736ebce33440 }}, + {{ 0x000b59f3e30d097b, 0x000bfa29268b3de0, 0x000adb866cb76535, 0x000eac7520d1ca29, 0x0000d07af7edc75f }, { 0x0007f84282de8bcf, 0x00016ff7f4a6c51d, 0x000189e3ecfc03a9, 0x00009091fa8768fd, 0x000063e18fffaf2c }}, + {{ 0x0005978d0a53ff5e, 0x000de0e0546063ea, 0x0001b7ca1ebc5a0b, 0x0009b66007fbadcb, 0x000016843f73bee2 }, { 0x0007fb68ea74f94b, 0x000d61e4f85eaaca, 0x000152cf5d96b415, 0x0008a39cde61e2ad, 0x0000f97b10343ac3 }}, + {{ 0x00048364d3e5d0bd, 0x000fc363daed4c29, 0x000f918392b59488, 0x000d61c0a80a9aa3, 0x000059055976d210 }, { 0x0005f25bd356dc7f, 0x0003d6070985182c, 0x000d6d3eacc1af52, 0x0004d43f5a6db5b0, 0x000050f58b44a28c }}, + {{ 0x000c527f92db6aac, 0x00019242d0a4c230, 0x00045f614cbf9dab, 0x000338092eb5d26e, 0x00008b05b2bd83e5 }, { 0x000c81836dfe11e3, 0x00036382aede0ac3, 0x00083041c5cba047, 0x000916061b292220, 0x000053fe2de8f49b }}, + {{ 0x000a0369abbd0203, 0x000b27fa001ec5a8, 0x000b16d8f40e8efc, 0x000481080e993f5b, 0x0000af7a86da5db5 }, { 0x000551c295b0441c, 0x00071608c927db84, 0x000f4f1394ac39c2, 0x00072abeabf5f37c, 0x000060b2c65c0c54 }}, + {{ 0x000b04f7d8f72e56, 0x0001be7ed77ee8ba, 0x00004804ddee725d, 0x000b92acabddf7bd, 0x0000ccae2b20533d }, { 0x000c088c89918989, 0x000e9944313a8cee, 0x000ba6c1c0748b0a, 0x0001e207ae532e4a, 0x0000e42ef48bf73f }}, + {{ 0x00046b3d3b414b91, 0x0008dcc42ce9e515, 0x000d5df49fef1fdf, 0x0009c67398f9840f, 0x0000bb29bacc84c0 }, { 0x0008c7d4845045fe, 0x0001ba980d309162, 0x00072f844c3fd6d1, 0x0008188084549710, 0x0000d0d70dbdf40b }}, + {{ 0x000ed2436122cc51, 0x000c66788a2ffe41, 0x00050a0f2644da13, 0x0002d783adc506a3, 0x000069129b160058 }, { 0x000218e8eafe18f8, 0x000b292a3d694e78, 0x000d6d37d38998d1, 0x000f855b4c0f43b4, 0x0000bf4a6ab4a165 }}, + {{ 0x0000f3271f894fd3, 0x0008afb185f88448, 0x000a946484dc2331, 0x0007019dfcd7e90c, 0x0000692c2e123626 }, { 0x000981d9ea441cc2, 0x000d8a7d0e62f477, 0x0000cc886efb3ba6, 0x000724e1e88d519d, 0x00000023f082da8e }}, + {{ 0x000afff39d12e58b, 0x000da2c9ac382032, 0x000f2c658c484952, 0x00039c91686eaf87, 0x0000523e755bd5cf }, { 0x0005df02e43ae54f, 0x0006c07256d4eab9, 0x0001eca6a8a4fff6, 0x000375ffe2259869, 0x00006edafe5d45c9 }}, + {{ 0x000d1c4cc4d8b709, 0x0002c6ec7004bf30, 0x0004ce677b1974ac, 0x000529eb7230a08f, 0x0000f295c26eabae }, { 0x000fed0da0e7efbd, 0x000096e7550fee88, 0x00070d0a16dda21c, 0x000c5fae7369cd4c, 0x00001bcd58dde3aa }}, + {{ 0x00044ed736b966ec, 0x000a5c2e48fb8898, 0x0002f73bc5cfb109, 0x0001bb4e226882c1, 0x000063d7619b2c62 }, { 0x0006a413e95e2beb, 0x0005c6fd86e27c75, 0x0009ae0bdbf2ca03, 0x000edf84da04edac, 0x00007492ad45d302 }}, + }, + { + /* digit=19 (1,2,..,64)*(2^{133})*G */ + {{ 0x00024f4d696da3cb, 0x0003b1d5a74be506, 0x00047c52228c154a, 0x0001bfda2e41781c, 0x000096ec0545b52c }, { 0x00001059ae4af1e1, 0x000735dbdcc9ae4f, 0x0005508ac4dfef91, 0x000d754392573dbb, 0x000045a30ae8165b }}, + {{ 0x000c13673627c363, 0x0003c973112795f7, 0x00071c3b07ab4999, 0x000f308cde824443, 0x000024017b4422c4 }, { 0x0007e6782a430857, 0x000be955dbec38e2, 0x000c10a45929115d, 0x000d83e2f51c0782, 0x0000d2b6c6a41083 }}, + {{ 0x000f9b21b06e4d48, 0x000483d4ed2dbc25, 0x00081ca00bd2f9e2, 0x000314b350eba59c, 0x0000879d4fee9d2e }, { 0x000846aa4551a9b0, 0x0002394af267bae5, 0x000565e76b699192, 0x000e51500aa86cc2, 0x00009d486efc3818 }}, + {{ 0x00023eed8fe44b7b, 0x0001c40e2c2bf152, 0x000587aaa9751a65, 0x000b04a65627a220, 0x000078ae9a0140de }, { 0x0008dfbb97c7d47d, 0x000a883a8363b49a, 0x000920e2f50e570e, 0x000074b5c1a0b6cb, 0x00004368ec853ac9 }}, + {{ 0x0003cdd7ea38297d, 0x000d04c65c7c2aba, 0x00073ef8129a7789, 0x00012cdb51d4610e, 0x0000ae1d9b12315f }, { 0x000a8296356d58cb, 0x0007e83064417ee3, 0x000c72e948f5f2f8, 0x0009e5bc31459b23, 0x00007f5688f80bef }}, + {{ 0x000e7c88b34096a2, 0x000ee1fab192c181, 0x000e51b38d9be8b8, 0x0008dc9a6ec5fcbf, 0x0000c004fd1604a1 }, { 0x000d9b52b3513161, 0x0007c93c92d88c5e, 0x000d236ba8a62196, 0x000c4ae1441c2148, 0x0000e3b813e3424f }}, + {{ 0x0002d6dbe520bd47, 0x00002ebb0a5b358f, 0x0001aba752f14bc6, 0x000e71bdbb154c5c, 0x00004df8bc57eacc }, { 0x0007ad0af3ff3ee3, 0x0000af38665e5b22, 0x000353d24c2f432c, 0x0008cd086488a851, 0x00006ced5e1bbf8c }}, + {{ 0x000c2a360971c31d, 0x000066e846e364f6, 0x000c727ad6974fa9, 0x000639b6d7f33527, 0x000015ca19ee6b97 }, { 0x000b4ef351021aa4, 0x000ba52f024e9245, 0x00044265bc79a45a, 0x000687390e0fa07a, 0x000026bc8f7f2c64 }}, + {{ 0x000eb356d99847ee, 0x000c1c4b6205969f, 0x0001f8142e759402, 0x000504d4842563f0, 0x00002d9aff65fb85 }, { 0x000340185f8291c1, 0x0006b81d1a39bd77, 0x000f7353328a0998, 0x000e743da8f44340, 0x00008523dea1bcc1 }}, + {{ 0x000c87111ada74f5, 0x0000bfb7409ab463, 0x0006445b7897551a, 0x00013f0d1057e78e, 0x0000fca1d2e611ce }, { 0x000f201ffa6c5d1a, 0x000f41c7809460b4, 0x000222a98bccef3d, 0x0006f03c9e751c85, 0x000002a8d1482e5b }}, + {{ 0x000984b27405ee90, 0x000005c3e6721cea, 0x000a272eb4d7fd0e, 0x000927da4d52d70d, 0x00000cba2cc35612 }, { 0x0009d70b01b1eae7, 0x0003e2784de5ca41, 0x0002ae7bd3c2b1ce, 0x000f8f7092f1c987, 0x000030b0c4119210 }}, + {{ 0x0005583b9f7c63fb, 0x000702da6dc1d29b, 0x00094319a5a4c61c, 0x000909776d303000, 0x0000ea690a72942e }, { 0x0000d34175165b72, 0x0004c7895695f204, 0x000148fcd1400755, 0x0009aa443fc84181, 0x00009d399222fcef }}, + {{ 0x000758ee5709e62d, 0x00069f5674198335, 0x00022ee28cb1cdab, 0x00070430c6059e25, 0x00001b376756c012 }, { 0x0008f5c1bf938d37, 0x000401d013b0ea65, 0x000afbd027f9ea15, 0x000a32680fddf524, 0x0000feb88c703890 }}, + {{ 0x000d105f424816f7, 0x00040f17c696042c, 0x000b7d14d99c5d98, 0x0008d2f00d4cba22, 0x0000ca9c23515356 }, { 0x00065cd5704583a6, 0x0002919ddb81e743, 0x00091afae35a6c5e, 0x0001a1285472f2d8, 0x0000a117968098a9 }}, + {{ 0x00061f2dfaaea695, 0x0006dbba2be70539, 0x0005e49a78db79a4, 0x000511030dcbbf7c, 0x0000c25cacbca172 }, { 0x0006bc1247ed44bf, 0x000d53570dc80c4f, 0x0002f9bc2ed8f5b8, 0x00011b222104d6b6, 0x0000be75e1d87966 }}, + {{ 0x0001a05fe05069b5, 0x00097050f15dde91, 0x0007fbbecba1b4b7, 0x0007e7662a47a76a, 0x0000cef9eea25012 }, { 0x000d251439c9ff5b, 0x000996e9d529b36f, 0x0000927929bcb4b0, 0x000321b9855ab130, 0x0000e6913e2a37cf }}, + {{ 0x00002e1795824e90, 0x000a54723d695ead, 0x000d46b77220484d, 0x00039408648ce626, 0x0000dd1d7bdd322f }, { 0x0000941af25d109b, 0x0004e8f7198111d6, 0x00057c65a8da2e48, 0x0002feeb090ca630, 0x0000f1530e7e8096 }}, + {{ 0x000b5c989f6938ee, 0x000d2a222eb51e58, 0x00041f4057fb3762, 0x000dcb106c0bb724, 0x0000ad14845ce016 }, { 0x0007d1e1160ea6c5, 0x0002e16512e1f433, 0x000475371e4c0b2c, 0x0002b464e02eac55, 0x0000f6d7ff9c35f9 }}, + {{ 0x000f15a10f16a85a, 0x000e41aab7b19a8c, 0x000e0fe32c369f9b, 0x000b5a9daf08d067, 0x000024ab40acda51 }, { 0x0009684eaabd75c3, 0x00009e5b738a4250, 0x00058cce645336d6, 0x000db1305ebe131d, 0x0000484813fc96ba }}, + {{ 0x000d8bd56620b954, 0x000129c72ba075b6, 0x00054cbabaaf6b83, 0x000e637a889f78b5, 0x00003d0a3429781d }, { 0x0004afac04fc7b8c, 0x0005a3b805f08d67, 0x000d4f0d15b02979, 0x000aa4c477f48200, 0x0000acd64bf963b3 }}, + {{ 0x00065ecce6ddbcf2, 0x0003a2b7e63d6900, 0x000ffbc500d16630, 0x000c7e1feefb6bbe, 0x00008b274a61c74a }, { 0x000c29f8ff0209d3, 0x000c1ce69f790713, 0x000ec8218d676a18, 0x000acca8f0d9a8e5, 0x0000f615e9cc3915 }}, + {{ 0x000f37825c9ef1ce, 0x000848febae68c4c, 0x0001618e37c4f175, 0x000eb4cec5b38563, 0x0000d423c83cbc80 }, { 0x0009c52aea308776, 0x0008380beb11454b, 0x0000a6fc857c4768, 0x000022bc740a1a99, 0x0000ca5ec3d496c8 }}, + {{ 0x000cd4fde296905a, 0x00062ce45650ef47, 0x00000dcd9ce46f6b, 0x000746a70ae000f1, 0x000007a206cd8fb8 }, { 0x0006cd56404f4acd, 0x000e52ae88408697, 0x0008e8efb046152c, 0x000cc12baa886885, 0x0000b6caba36edc1 }}, + {{ 0x000a22c213a1381c, 0x0005998a9d82abfd, 0x000b0c9eaee2416b, 0x0002e94875f188cc, 0x0000ed487d7becaf }, { 0x000a1bfe43d8e989, 0x000d43afbddb4799, 0x0003d3555d553c34, 0x000ae422e4b606b8, 0x0000a98941400820 }}, + {{ 0x000d93563973829e, 0x0008a092a67d48a1, 0x000de58c53a8a8ae, 0x000c1054acd6a671, 0x00000d14b278f09a }, { 0x000286662fa4b2d7, 0x0004d898fea73ea3, 0x00046daf4982b9ed, 0x000b2de4c8247b36, 0x00007827af7176ad }}, + {{ 0x0004d969cb439a74, 0x000636bb4ec4c20e, 0x00062a667a3273e5, 0x0003e53e8f0a12fa, 0x0000735706c629fe }, { 0x000d4b3454ad59df, 0x00074b743e232a3f, 0x000b3aad70059ea6, 0x0007e3d8fdc7a3fa, 0x000053ec510c7073 }}, + {{ 0x000267aa86866f79, 0x0001686d36e30622, 0x00077edc547bf065, 0x000077b2e626c527, 0x000032b789435829 }, { 0x00088821cbb8b0bc, 0x000a75172e572d59, 0x000721eb0f204226, 0x000dcca57ab861af, 0x00001dc0ca62fb88 }}, + {{ 0x000ce1680d34fa1e, 0x0000722cccc18ad5, 0x000d94c5781a4479, 0x000c8174daadf3f8, 0x000096cf50e285e2 }, { 0x000921dfeb77456e, 0x0004dfcd9ecbee97, 0x000ba38c82c0be59, 0x000d5558a4330689, 0x0000d3b38073c94d }}, + {{ 0x000da886aed8ecd4, 0x000eaf8271d1ca55, 0x00078b0236696085, 0x000b711e33e423bc, 0x000014d2f4ec906b }, { 0x00036e1f5d068ff6, 0x0007034a99643755, 0x0003162eb1a26190, 0x000151fe9356a2fa, 0x0000f7623180c7d5 }}, + {{ 0x000f35afef5374dc, 0x000b53a5c04e4ea9, 0x000e37bb65248044, 0x000884194d549dcf, 0x000056a261a80626 }, { 0x0003eb6a05b04ad5, 0x000b824f1314f53b, 0x000816d16e1d2218, 0x000a9b8b062baf94, 0x0000b751a249239e }}, + {{ 0x0008e248259fd6b1, 0x0005fae1208e16aa, 0x00054f53f5fe83ea, 0x000752f0b1a4d15b, 0x0000322765d11e76 }, { 0x000a06028f32c493, 0x000e0a290b170cfe, 0x000058eb8f6521c2, 0x000831797a1bef33, 0x0000d5b9b2a9415f }}, + {{ 0x000495775835ddea, 0x00038f3dbfb894e2, 0x0006a991d3c0ab53, 0x0007ac0dfbe9b79f, 0x0000a996ed108d94 }, { 0x000a1485e6867237, 0x000b861bb9390993, 0x0009ecbcb0ad8aab, 0x00089ee3b1d6a974, 0x00004490d265e84d }}, + {{ 0x0005d8bafdc81765, 0x00002b4147c15b45, 0x000b25ed58041f7f, 0x000f1de81e34f553, 0x00001e756a2afa4e }, { 0x0008b5d2446a18ad, 0x000cc95f1c0e74e9, 0x000e53045657f07f, 0x00033d5515a9b292, 0x00004e129faf6e81 }}, + {{ 0x00054ce98632c8d9, 0x000e666683425204, 0x0009755b794b887e, 0x0003dd2cd08d6894, 0x000093ff6f3941b6 }, { 0x00011538eb926db1, 0x000ab9eb55b803b2, 0x000a10ce88821a3d, 0x000060e12b03468e, 0x000072cd7c995240 }}, + {{ 0x00032cd90a50d0eb, 0x0005f2f731dea2d9, 0x0001e1c0a21fc8bd, 0x000d579fed5856cf, 0x00009b444a44e0c2 }, { 0x0009e7c0c1c75493, 0x00002d4c0c4ddc0f, 0x00073a980207bdba, 0x000b2d8cbc0f422f, 0x000047b3f99102be }}, + {{ 0x000665427bc303fe, 0x0004fdb0f5f27cac, 0x000ae1a81783a295, 0x0005c1223e85461f, 0x000076fba4039ea3 }, { 0x00022639d9c5cdc0, 0x0005a697587fd52b, 0x0007db23b5f4ea89, 0x000744688935289f, 0x0000887566528724 }}, + {{ 0x000bc9cdaec3de6b, 0x000f720361ecf90e, 0x000f398696b456af, 0x0007dc079001f23d, 0x000063bcc338ae46 }, { 0x0007015cc0864754, 0x00010a1934aa7289, 0x000e179a04332cea, 0x00055a7fd750eb00, 0x0000b649d7589b15 }}, + {{ 0x000c21aa55e58d8c, 0x00038a5bf5151dff, 0x0000b44f6e9cced5, 0x000b2182c21adcc4, 0x000000bf7836051e }, { 0x000405a74bccb408, 0x000bb2f4e51ea80e, 0x000f4195b6421b97, 0x000db135e3dae45e, 0x0000c8a422289ff6 }}, + {{ 0x0005afe3df554744, 0x000a5c69f1c56bdb, 0x00079441b2c565d3, 0x0002dc52b8c176b9, 0x00001263e0533213 }, { 0x00004e3fc1406d5b, 0x000a249145813e50, 0x0009e2650265686d, 0x000129a7f687fc4d, 0x00000832fee46c0c }}, + {{ 0x0001f9addb3fd772, 0x00068b61430c9ab7, 0x0002b03f3bbc1b26, 0x000a8d0ae0bdd41d, 0x0000a6be72385501 }, { 0x00063accbd8fe7a9, 0x0001716c52dd907b, 0x000c73a737c1e0a9, 0x0002868a6d9bdf44, 0x000035c09edac28b }}, + {{ 0x000add2b06f54586, 0x0005c351272a95b5, 0x00074b4a3fc1da82, 0x0009268f70c66771, 0x000044d6759202ed }, { 0x000d2a2bf9fa5f60, 0x000732d14ea5d65e, 0x0005a053a90c3bdb, 0x000554ac6f8e01f0, 0x00005fd71f56197b }}, + {{ 0x000d3d1774087028, 0x00051b5e9d3a7c31, 0x0006647adbd73b2d, 0x0005492758e0ee5a, 0x00003bba70bd8115 }, { 0x0007246f125d77c6, 0x000ab127747ae836, 0x000fb258fc2e4b6e, 0x0001e4e0031f4315, 0x000079fb7523dc70 }}, + {{ 0x00022cef7e044521, 0x000ab5aef0c31335, 0x000d2be979bd6bea, 0x000e49649247cc45, 0x00008daf607fcbc2 }, { 0x0004be69d5e2999c, 0x000580559ab93b3c, 0x0000da59a69c02d5, 0x0009972d0225fba1, 0x0000cc519897bf32 }}, + {{ 0x00000ed311acec40, 0x00036680448087c4, 0x000f31345a9c7960, 0x0000d7f47ac30903, 0x000014a52f1f5454 }, { 0x0006f2e733d21f38, 0x00041fef8fd24247, 0x000a837ac257323f, 0x000ff2f3afdfd974, 0x00009ba0f6470d39 }}, + {{ 0x00050c547e9953dd, 0x0000eb52a6017539, 0x000c306fd8ca8187, 0x000b82024b286514, 0x0000a0dcbe1ef184 }, { 0x000f64a7dd340b95, 0x0008469e5d32bfe8, 0x0001e5a8c4619dc8, 0x000d6ea3a30bc147, 0x0000e933dd6721ac }}, + {{ 0x0007ccc59280d519, 0x0008037633dfb1fc, 0x0004cfb25fb4af10, 0x000498d46ea82c39, 0x0000e384e83287e5 }, { 0x000e6e0a4144dfdd, 0x0009c6281761e131, 0x000f73287e1b3d37, 0x0001fe070da57596, 0x0000b0e289bb07e4 }}, + {{ 0x00066b4e44ff11ff, 0x000882c4152fcf57, 0x0000280c990f629c, 0x0003c0009441c87b, 0x0000b659204e90a4 }, { 0x000df4368ea4d935, 0x000701211f1c7924, 0x0002fdef25b0372d, 0x00033119eea02ae9, 0x0000eb3caf4bdf44 }}, + {{ 0x0002ad31f429e92f, 0x0007dab4baef62e3, 0x000275ae5ff44e2e, 0x000fc47109f5a220, 0x0000f78da2e8f242 }, { 0x00089cf1ff54aea4, 0x0008ca425bcf4d6c, 0x000288b364e70dd4, 0x000847d56ea95059, 0x0000d5e592ce8a8c }}, + {{ 0x000bb30b4c5c4041, 0x000f81304e60cc0f, 0x000c4a72c8291975, 0x000e6a462ce811e8, 0x0000eb639b79afd0 }, { 0x000e60a95e94bf15, 0x000e32c485281182, 0x00059ec7afb2ce90, 0x000767a287c6fe08, 0x000067e5869c6204 }}, + {{ 0x000b6d146193e6d2, 0x000ab3d9b785dbf2, 0x000b7beca8857459, 0x0000f4039530889a, 0x00009e5aa29e84bf }, { 0x0004f877f4c051e2, 0x0004a0451a7bbf7f, 0x000f81f9d270263e, 0x0006a8ba030ad99e, 0x0000b432329b85d8 }}, + {{ 0x000cd2b7ced02b6a, 0x0008c2d4c44f1190, 0x0006dce4bc47e034, 0x000836c3b4555f53, 0x0000788b19828abc }, { 0x0004d4598c6d92d3, 0x000557b46945fa35, 0x000081a82e04d1a0, 0x000a4ea08f8785b2, 0x000063de08f3d528 }}, + {{ 0x000d3e738b6fd0d3, 0x00087cbc46d7ce17, 0x00070dc9e4ec18cd, 0x000a97d509a515bb, 0x0000c0ed53ffb19b }, { 0x0009fd2c8134d2bd, 0x00005ce2a6cb0ebd, 0x0004b57daff8514a, 0x0002ea6766f2e295, 0x000010bc9a64cffe }}, + {{ 0x000465c200623f5c, 0x000db38b2b836a16, 0x000017a7a4228368, 0x000823ef1855b41a, 0x0000cb16497d171a }, { 0x00094b4447686be0, 0x000e23f6974b99e0, 0x000b99263876de79, 0x000c0065875a7cf8, 0x0000dfac4f57f79d }}, + {{ 0x000a8e6f779b6900, 0x00085edb940cb5af, 0x00018239bf98af89, 0x0009c26fa6706d79, 0x0000b769a307f467 }, { 0x0007c08b475deaab, 0x000f234591ee4f0d, 0x00066866b61f19ff, 0x0007c69575f46e33, 0x000028a209b8748c }}, + {{ 0x000b82c392e80537, 0x000f93c1c08ad635, 0x000ff1019ae85ec6, 0x000ee34f79954343, 0x000039a812312260 }, { 0x000b0e72020ed2dd, 0x00035c4928435d5c, 0x0000ae24917dfa24, 0x00007284a071cf0f, 0x00005e0b1098229a }}, + {{ 0x000e70062ee0a4a2, 0x000917ac0b7eff35, 0x000e6774850d2a32, 0x0004c985f8552225, 0x0000ed692c26dd93 }, { 0x000068c99ecd36b0, 0x000a048e7073969a, 0x000922d3ea339e56, 0x00093f20f392d2a2, 0x0000a61b1d58dd81 }}, + {{ 0x000a4141443fe874, 0x000f11fb87a29137, 0x0004c28111acbcec, 0x000845fe4ea3e3c1, 0x00002b3d24c59afb }, { 0x00010dc2aa355a06, 0x0008f1aa71ff4939, 0x00022ac5ba4c733b, 0x00075e8fffe980c9, 0x000092ea09a1baa9 }}, + {{ 0x000bd61a53336f7c, 0x000c326cca9f54d3, 0x000b572ddc8897d0, 0x0007f6a7c928bbdf, 0x00001633d0f6cb76 }, { 0x000e19742c3c3494, 0x000a4e6395c9a791, 0x000d43d1c100cd6d, 0x00052c2316940ca2, 0x0000fd9b397c3285 }}, + {{ 0x000baa11cf6966bf, 0x0001a0823fc2e6e0, 0x0008fceb7f79dccd, 0x00086bea5a76e29a, 0x000020bec34acd1b }, { 0x000a26322d5b0637, 0x000b564961849da5, 0x000f551759fb7518, 0x0009cb74493b75f1, 0x000085589c5471c1 }}, + {{ 0x000097196463dab3, 0x000235861a023efa, 0x0004194e0b09b4ae, 0x0007b8bc01d72aab, 0x0000a76ce5fce481 }, { 0x00008748d29fbf98, 0x0003d4badb9b5b76, 0x000163d30978b6be, 0x000794e0873fac1a, 0x00001acf21ce255e }}, + {{ 0x0009ab9f40344f97, 0x0008094c486094f9, 0x0002f24809967c07, 0x000bb7725869254a, 0x000017af2e8590fc }, { 0x0003a576905d94dc, 0x000ea8a6706c02e5, 0x000c15d397dbab92, 0x000dc5b6c22b5e76, 0x0000e77621b7d00e }}, + {{ 0x000d38e9c47b4340, 0x000b852edffcf3a5, 0x0003ded185911220, 0x0000e5b9eb4d2ed0, 0x00005ef944ff1997 }, { 0x000cbf19807bc19d, 0x0008bbf303747e2b, 0x000c38f6a80d0b53, 0x000aa5c23a208e77, 0x00001f5c1520084e }}, + {{ 0x0009789f1f62f6fe, 0x000f230273231941, 0x000b1a24067e38a4, 0x0006f97203b055a8, 0x000043fde981f2ac }, { 0x000ee11ececcedef, 0x000a568aaf018013, 0x00004c88b6cdcf5b, 0x00006d17972ee119, 0x00001719e065c6e0 }}, + {{ 0x0006e4617f3e69bc, 0x000caec1ed66f181, 0x000bbf3b6cc4ad74, 0x000551b4d225d906, 0x00003684306257aa }, { 0x000516497c9675a0, 0x0000addb2fbc3dd3, 0x000f55d795decef3, 0x000598fadedb5484, 0x000015e8ee776bd7 }}, + }, + { + /* digit=20 (1,2,..,64)*(2^{140})*G */ + {{ 0x00020f2beb76812c, 0x00013ecfa181e695, 0x0000dec76ead7889, 0x000698d5d9ea02f8, 0x00000cd75bcfa2f0 }, { 0x00017a069c2d2cc5, 0x0008f0a1df843618, 0x0000f08066c134ef, 0x000c931ff1203772, 0x00008f19ef39b4e2 }}, + {{ 0x0000aecc1c7112bc, 0x00073b6b294cda6a, 0x00051854d3181244, 0x00049ff216ce9bf7, 0x0000f5f14402d398 }, { 0x00056e372344bdbd, 0x0005f3af02909e50, 0x000f436065b91304, 0x0002579f8c7d59ec, 0x00001766823e2146 }}, + {{ 0x00027d200e7695ba, 0x000fb3189e800ca4, 0x0006d1d4e8ef137a, 0x0007f003750fe0c1, 0x0000dad25c5ac540 }, { 0x000807804fa82f49, 0x000994fe616e2c00, 0x0008e610d4715daf, 0x00004e1739c25f4c, 0x0000a1ed59eb55e7 }}, + {{ 0x000c966787f80791, 0x000d43a4f0d56f34, 0x00077d92507dca1a, 0x0006bb24b961e404, 0x0000a0d775222852 }, { 0x000bb6d594089b2b, 0x0004142864fee422, 0x000a2f57f8c8c37e, 0x000969659c1be93d, 0x0000e98561f48eb8 }}, + {{ 0x0003d36eea5a411b, 0x00083c9b809b7ceb, 0x000b2ef3bd41c883, 0x0005fa4368a41486, 0x00005327a94036a4 }, { 0x0009d81a294550be, 0x000028a328cc987f, 0x000b405a4a382a8d, 0x000c01dba0a3bcd2, 0x0000ecbc7c687492 }}, + {{ 0x000ee9ea4399d83c, 0x00029ade4d559419, 0x000d914e5643a410, 0x000194f9bdedafa5, 0x0000ab6a2f9c77b5 }, { 0x00023fc56d6b71dc, 0x000ce1637a55a4a5, 0x0003af1fce4bba9d, 0x0002a5998eb19a51, 0x0000fb6b0a026533 }}, + {{ 0x0005384859f3a770, 0x000970cb98fe684f, 0x00091d11cbfe5c75, 0x00014fb3fd60fbe9, 0x000024a2c6896e9f }, { 0x000a7731d1c175f6, 0x0007bf7b59e763bd, 0x000880e35c8c898c, 0x0002e8c923d4606a, 0x00001705d921c944 }}, + {{ 0x000bd17983bec34f, 0x000e79390d458714, 0x000fba44f409e51f, 0x000e5b503e976403, 0x0000bb28b50bfeca }, { 0x000460fc77585d24, 0x000439ed5d2c53a7, 0x0008896e3f104db9, 0x0000d454e5ec4bcd, 0x00005c92699d9c5e }}, + {{ 0x000cb1236b66c01c, 0x0007f8295caabee0, 0x00057be3de780986, 0x000aeb665de024ca, 0x00001dc5e9d49aa0 }, { 0x0001359afecfc3d8, 0x00005bf0972ebf9d, 0x00045177093370a2, 0x000872776f1dd387, 0x000022a719d4d0a4 }}, + {{ 0x000802eabd5b22c0, 0x000fa2b91502cf37, 0x0007904acc9ff780, 0x0000df1c92c3832a, 0x0000c8e7c1032083 }, { 0x0006aeb0c9d5f503, 0x0007027a37fddab7, 0x000b8400f8a95d7b, 0x000f1f61a65d6f2a, 0x0000d4946c17ee3b }}, + {{ 0x000af61472ec944c, 0x00013f62f93a6750, 0x0003fafe7206fbed, 0x000f69c84a6ec968, 0x0000e64f69ece348 }, { 0x00036a8a8778d2a5, 0x0009e72a5f696be1, 0x000a5bb870e8f960, 0x0009a4590c65e57e, 0x0000b18bf5bd851b }}, + {{ 0x000b4b36c8c2e8ff, 0x00082dc0f9f44d43, 0x000ff0fb2191ea75, 0x0005ed2b1f09a695, 0x00007a5902599c01 }, { 0x000a904b3c1f4b94, 0x000837b3b6b234cb, 0x00045ca08e940188, 0x00034bf1d096a250, 0x0000986acd027cd0 }}, + {{ 0x0009958546371d81, 0x00058d493a3147a2, 0x000c70c0cffeda75, 0x000d2f1129f30a5d, 0x0000599e39f58537 }, { 0x0008d658f17504df, 0x0005fffc17db9ba2, 0x000ca23febb75e2d, 0x0004520bccb18548, 0x00007564d18ce2da }}, + {{ 0x000e430a33a255cb, 0x00095cda99f8c71a, 0x000a9db438cba264, 0x000f5749967b7abd, 0x0000945a43d068ee }, { 0x000cb8f38cf63ea6, 0x000d0d388ace52f4, 0x0004061e12f02828, 0x000c9271b4bc0fa2, 0x00004dad48391d98 }}, + {{ 0x000a1c5643447901, 0x000bd808c3bff364, 0x000208478f323f53, 0x00022dbd1005a0bd, 0x0000ae406a2896e1 }, { 0x000137007e0dbeb6, 0x00025b99fe4fc88b, 0x00081b60bb371dbd, 0x000646bae09418e2, 0x0000cc34af053693 }}, + {{ 0x0000693a18674889, 0x00096e1c63c4962c, 0x0008d7b6a10d190a, 0x00003a26b50541e8, 0x0000e282f1e08996 }, { 0x0001ed4e5703fca0, 0x000c04e0117f203b, 0x000258ac6a79aec7, 0x000248c44245f196, 0x0000ff00eb7253c2 }}, + {{ 0x0002a6cd6b58c40f, 0x000e675f8547be3b, 0x000cf73d922da584, 0x00000f2000a7033c, 0x000044f163153dc0 }, { 0x000273a484b87506, 0x000aef557933c6b8, 0x000ca899db77a846, 0x0006120b6a7538eb, 0x0000ded4a50d08d4 }}, + {{ 0x0003995b1a10381b, 0x0007d450168e508a, 0x0001f60fefa3784d, 0x000788b718cbc9bb, 0x000015ddd98778ea }, { 0x000b59fe5b0f3810, 0x000568e1239e1525, 0x0008605ee199d7d9, 0x000b969635689255, 0x0000ae3fda8f50cc }}, + {{ 0x0000f551a177b5e0, 0x00040eb34a9f7aec, 0x0002165c9e43067d, 0x000084f37e5e8cf7, 0x0000983560921b62 }, { 0x000e549e04b370fb, 0x0006768038f67254, 0x00051f3e35b1a2ef, 0x0005adb861dd38e3, 0x0000ac6cfc2b7810 }}, + {{ 0x000ddba4b3d8e704, 0x000f53b08b193400, 0x0004bceb33e96771, 0x0002e98c7c2fae6f, 0x0000d016b8e9d8b6 }, { 0x0009d9e008a1dcc9, 0x000feb51dd95f8c7, 0x000754cbe24d3872, 0x0006dc34c1d163cd, 0x0000d6587365fd85 }}, + {{ 0x000846f5f2308016, 0x00088bfe6ddd5053, 0x00047cc4f727d222, 0x000b2675737064e7, 0x0000ae37a4ce7376 }, { 0x00090c603077e9a7, 0x000cd9160df1b9b3, 0x0004f411426f0fd9, 0x000e7cd3c4cfb289, 0x0000f64f81bf99d6 }}, + {{ 0x000762921d03717d, 0x000d6128c0a78de0, 0x000a17234d538bcc, 0x000a9e882ccd02bd, 0x0000eed2d646e6a2 }, { 0x000287c7f8080a37, 0x000d123d85010c02, 0x000addbd0727cb42, 0x000abb08ad73aaac, 0x000037e0c3c8c39b }}, + {{ 0x0009ea4996a3c06e, 0x0008942580b1a014, 0x000d1657f1e2a72a, 0x0001e3344f080415, 0x0000093afc473704 }, { 0x00090e3df700aea3, 0x000eeef571f3913f, 0x0003fca41c93b37a, 0x00008fb8f0610f21, 0x0000984f4e2a8e30 }}, + {{ 0x000c69c29d48bff8, 0x00065eb43e2e0349, 0x0003a7d6620d144a, 0x0004ca77f53fb5a3, 0x0000f354b8c1cb22 }, { 0x0006812c52e71fa4, 0x0007bebbace47b69, 0x0009082476997303, 0x00043e13ebdce028, 0x0000fc56789759ad }}, + {{ 0x000a5a5512b3d42d, 0x0002a7157151692d, 0x0007ff33c7322027, 0x000656ee1eb2721f, 0x0000ca99ca38747e }, { 0x0001ef52167f204a, 0x0008b5dc6f567809, 0x00037bd855ddf45e, 0x00073d003e20d300, 0x00004fefff6aeb76 }}, + {{ 0x000c08620a2c393a, 0x00045c14092d85f7, 0x0005e041820ffc39, 0x00031138172d223c, 0x000087580008988e }, { 0x000769b5baf39b4b, 0x000017dfc588b090, 0x0009510b05f510e9, 0x000b0b995728cd49, 0x00004ab8441733f4 }}, + {{ 0x00082ded8c15f913, 0x0000f8bf99c24714, 0x000ad0f0c3845063, 0x000745d24f2d8a11, 0x00002d13601a08bd }, { 0x000fc739d67f549d, 0x0000cd8dadd48854, 0x000ea20b7b714a04, 0x000b658507004477, 0x00002739d35a18aa }}, + {{ 0x00010188c19608f8, 0x000b727786f08a98, 0x0005c1c0eabddfc4, 0x000452e708700bb2, 0x0000b09a61e07f86 }, { 0x0009d4f07bc35bad, 0x00023c9abcd3297f, 0x0006faac186ffef6, 0x000734030ae4c896, 0x000068869b2a6b55 }}, + {{ 0x000a11de9f5e8da7, 0x000e7d583fa1e08b, 0x000e0499f35bfc4b, 0x000ad6a967780d33, 0x00006387ac199d31 }, { 0x00028fa43af1c617, 0x0001dda02c2d064f, 0x000cb1fe3e00767d, 0x000ca72d87ecf360, 0x00008656d8273918 }}, + {{ 0x0003edd0e98b762e, 0x00054cbea4570b35, 0x000b51fdfe00672e, 0x000f31ed5ee65d68, 0x00001bd0155d4d2d }, { 0x000500888c6c33f6, 0x0003d07beb9bc6d2, 0x00077a5bdedc774a, 0x000be200fe9abb90, 0x0000328aca3a48e5 }}, + {{ 0x000d700c7f811577, 0x00027d9926fce430, 0x0006f76a1cb0b72a, 0x000373b27809dd1c, 0x0000b7a49c601709 }, { 0x000574395a4d1adf, 0x0000b33ab64a0652, 0x0004edc4a625cb0b, 0x00049167faab9b73, 0x00008e3cbcb5b34e }}, + {{ 0x000d0301187bc101, 0x0000eb6700a1acd0, 0x0003f09695693995, 0x000f3e0bae823fd7, 0x0000bc494f5d06f0 }, { 0x00015737c0a7b0f1, 0x000bfc989fca8cdb, 0x000b4882d64acc9e, 0x00080e66f970d01d, 0x00002a01327ea3d1 }}, + {{ 0x000860368ee95e14, 0x0001fe8a8b484e77, 0x000901c6a838f33b, 0x000c0ed36b2fdbdf, 0x0000699447c715b5 }, { 0x000e12a5ec3c2973, 0x000b91cf9ad3e183, 0x00072a8ca88d9d13, 0x000267eb66d6c5f1, 0x000078786b1aa576 }}, + {{ 0x000c54b9cb562934, 0x00040e9a56b872f8, 0x000c70a1d4d6fe0c, 0x0002cdaf7a1ad550, 0x0000563272b206e9 }, { 0x000acf260393473d, 0x000cd4a088ac342d, 0x000b3aa7a0ccb38c, 0x0004f4ce9ec450c7, 0x00001b6e01fa188c }}, + {{ 0x000cba2eeb210411, 0x00026c6c8cb6b42a, 0x00030cb4c496878e, 0x000ec67251326fc9, 0x000017ca1f4e8294 }, { 0x0007a18a6d684fc4, 0x000c47a633c30000, 0x0002d02a45040c90, 0x0001d2dbecb6cd17, 0x000008914cbb715b }}, + {{ 0x000b3294258afecb, 0x0005b799edce57e9, 0x000443cd891c286d, 0x000a327c652b892f, 0x00002fc850e5378b }, { 0x000c76a331bc95f9, 0x000e1f2bd2e21df5, 0x000ca5d06767748a, 0x000c7ba2e55298d2, 0x0000676b98053ae3 }}, + {{ 0x000d2c801bc4bf64, 0x000fdfff8a4f29c3, 0x000099630d2de628, 0x000f1ce1c70dc924, 0x0000ea1da14883bc }, { 0x0001e8e630540696, 0x0006172425218760, 0x000c3f839381a073, 0x00037c5c90bed93b, 0x000079f89d6bd4d2 }}, + {{ 0x000f0e75ce977889, 0x000de38c6f3431e1, 0x0002c417ff72df79, 0x00013fe7ce609cb0, 0x00009a2977125eea }, { 0x000b453193606cdc, 0x000ee641158544d7, 0x0008f171ed3d9490, 0x0003d6b555d777e8, 0x0000d3214ae1a449 }}, + {{ 0x000a448a130c115a, 0x0009c8839f05ffea, 0x000fc33dcd3bcf48, 0x00017d31b8f4f4bd, 0x0000992eb014f9e2 }, { 0x000af5cebde70108, 0x00089b3f7914967f, 0x000a1e12d72fc4cd, 0x0000f3ae74537f09, 0x0000f1941d7b610e }}, + {{ 0x00011f21588397cb, 0x00032874ca6b39ac, 0x000c41d0b1e9ef78, 0x000c4c3db70fb72e, 0x0000a9adbfdfa153 }, { 0x000e9af3e27ac409, 0x000ed35bf3bfebf3, 0x00042d3e6c3d4995, 0x0005cc2733a1ff0b, 0x00008605fa3d4149 }}, + {{ 0x00028d0a1e02ab66, 0x000c672b41c34403, 0x0005b0769041de1c, 0x000dc2fa3175239e, 0x00007eb126a76c52 }, { 0x00054fa107e49200, 0x0005f2ed3788a049, 0x0005d171799725b7, 0x0004492d625d8ff1, 0x0000ec4457bdfdba }}, + {{ 0x0004ee0734a79d5e, 0x0004df2e98aaa995, 0x000041c2f15629cd, 0x000b6a46a90cd8ba, 0x000006e82e9e4e19 }, { 0x0002e25421fbf8ee, 0x000b05445b4147a9, 0x000107702d4ac0ff, 0x000bd40773d102f2, 0x0000f9d11eeb5c3d }}, + {{ 0x000a2e45bf1c1c78, 0x0006071bb608558c, 0x000e2ca6b9dcbb05, 0x000d5fe3e60e4eba, 0x0000055d8f3a5a25 }, { 0x000d5d7316bcf10b, 0x00067ec2bfd7a5f1, 0x000f57733e16a9e0, 0x000cd65dc528a8f2, 0x0000538112b6221d }}, + {{ 0x0004396860fdc301, 0x000174adf423e315, 0x00015aa1b1af53ef, 0x00098c9b9cb41729, 0x000050c9de633264 }, { 0x00088ae797a2c6e3, 0x00013c4a54ae57ef, 0x000ff5d9a705ab8d, 0x0003413bc0ffeb58, 0x000021a5f9a644c8 }}, + {{ 0x0001abd8610e73d8, 0x0004544d2359ed9e, 0x000311a87481122b, 0x000de0d69ac68dd0, 0x0000f82040d9e6ee }, { 0x00048969704b7037, 0x000e1d9343b6e3a4, 0x00086f80b23dd859, 0x000cf0767480797e, 0x00006cd8d794a707 }}, + {{ 0x000fc09d2dcb11e8, 0x000285552a82e8de, 0x000488856faec190, 0x000f18978b2caf78, 0x000085276cc6351d }, { 0x0006b1386691ac89, 0x00070ee9113be5c0, 0x000fad47eeb5cd81, 0x000d773542fbda78, 0x0000a86b95d248fd }}, + {{ 0x0006d449d8e3dcd4, 0x000740f43e3865d4, 0x0007d70afe005519, 0x000438bf772dd77a, 0x0000b18fb1739868 }, { 0x00018bef015b1939, 0x000e0ea185d706fc, 0x000b06519061374c, 0x000a2e731e47e02e, 0x0000ca817e6aa631 }}, + {{ 0x0002887ba15c20fd, 0x0007d9c6bdf22da8, 0x000f0e54073dbcd2, 0x0004f3951efbc432, 0x0000469ec570ed01 }, { 0x000deaa2e18dbfd8, 0x000c5dab920ec1de, 0x0002fe53ea59ef2c, 0x000e8af47d0d7e8c, 0x000015e430f8b3be }}, + {{ 0x000111a7619e2658, 0x000c5936cc874756, 0x00020c11dc5ba7f4, 0x00069dd46d92a52e, 0x00007437d17bf839 }, { 0x00049cab0510e4cd, 0x000b7feb3d5f6126, 0x000cfc4b7de6fa00, 0x00023a3305ea69b8, 0x000003184fab8043 }}, + {{ 0x0000efb2747d10ac, 0x00041cf8bdce214d, 0x0001b5bfc0eca8dc, 0x000280de2b5622f7, 0x0000eaba332135f3 }, { 0x00040a52c6704d02, 0x000f68b79df975b7, 0x0005a58efd781132, 0x000ebea46856bf28, 0x000034b117e458df }}, + {{ 0x00080784d755ed6d, 0x0004e152cbb1a3f8, 0x000d8fa0af51d84b, 0x000794e74c3eb99f, 0x0000d39835b41a87 }, { 0x00041e50a81e7f57, 0x000f51cb4e7a0be1, 0x000da8f75687bf48, 0x0006cd9e18780510, 0x0000e64f1df80763 }}, + {{ 0x00043afaf8568dee, 0x000691847be87ae0, 0x0007f32f4879b707, 0x00079d5d368a6141, 0x00002b47d1718f6f }, { 0x0004001ae4f33965, 0x0007c211552a4d28, 0x0002374d2fcfd8db, 0x000feb1a4b6dee69, 0x00000e25e4bee3ac }}, + {{ 0x0003473caf8cdb36, 0x0004d44d795fb455, 0x000d645818f5cf12, 0x000f1a9fc57326e7, 0x0000b402e73630bd }, { 0x0005614508afbc82, 0x0006993a13df4c82, 0x0006726a9f66c94a, 0x000284af8a73e51f, 0x0000c55c0677d85b }}, + {{ 0x0000b9c41619d1d1, 0x000286211a71b27b, 0x000abafe3623481e, 0x0000622efdf71412, 0x00003f1c33b3a212 }, { 0x000e3bcc6e9333c9, 0x0007c52014cdde5e, 0x000a343f7f4d5b8d, 0x000b5bff3702c624, 0x000071d00de55440 }}, + {{ 0x00046ed52661de90, 0x000f22a42ac6c853, 0x0005e98b1d913b21, 0x000ff8e582db0cea, 0x000088053a8d84e0 }, { 0x0005662908845062, 0x000c7951a2faf097, 0x000006bf38262f45, 0x000a502902fa8a58, 0x0000693f983dd1f8 }}, + {{ 0x00020c5984ecc7c0, 0x00035c53d080847c, 0x000890357b714942, 0x000ce5a63cb081d3, 0x000080dfb69e8b8a }, { 0x000fbd9c6a0c8dae, 0x000fc7cf902f245a, 0x00059e95ceb430b2, 0x00025654f9cb1273, 0x000018bbaa36708d }}, + {{ 0x000322f9d772ff68, 0x0007b0ac7088c7d2, 0x00000ee43ec82717, 0x0004112535ab6571, 0x0000826ae5c2a874 }, { 0x0003b5b6363c24f8, 0x000fa2d7430de4de, 0x000602d6f37384cd, 0x0006960681938269, 0x000039706131bb55 }}, + {{ 0x000a9ac3ed8e4ce8, 0x00011803fbc43419, 0x000fc0c429608622, 0x00035fc1d359f2c7, 0x0000af92e79860e6 }, { 0x000cd6f4e3313ee1, 0x000c5fe91fc50ae7, 0x000b493d05996022, 0x000f4105a89ccc66, 0x0000e63d8e773195 }}, + {{ 0x0000f3ffc1435cfd, 0x0009b69ce56b40e9, 0x000c2f39ded7b89a, 0x000912fd7f93e094, 0x000003a249c0e7aa }, { 0x0003ffdeb1ddd9da, 0x000235608241aedd, 0x00072cb1d136a3ce, 0x00057335295ab7ad, 0x0000f449ec6ba460 }}, + {{ 0x000389cfbf4c9e0c, 0x0005091d11a8594e, 0x00074e459df97c5a, 0x00017d3d02e74d25, 0x000044ebb0b8e3b6 }, { 0x000ea3c28478a740, 0x0002405941768d8b, 0x0004451bbfd95774, 0x00025d7e24510399, 0x0000fd86525ee1de }}, + {{ 0x000301e47f2cd335, 0x0008b6ed9a475207, 0x000a3c833f852147, 0x000b42f8afbfe18a, 0x0000580fe7496353 }, { 0x000afffe040e720e, 0x000ebddb299b9777, 0x000b2ab785106b87, 0x0000a82e776697a7, 0x0000b8a44884646f }}, + {{ 0x000cfb5fa19ed6c2, 0x0007c4b28c22cee9, 0x00076d01f60cbc18, 0x000539b6aefb0ecd, 0x0000f84ec5d3906e }, { 0x0001527a37e73f9e, 0x0006d57f36e1ac55, 0x000ce5788145521e, 0x000c926db008fa9a, 0x0000af255a03386b }}, + {{ 0x000db8eaa84aa125, 0x000c9305bd213110, 0x000282e3a62f6f6c, 0x000e571d36ed41b2, 0x00002902def1b031 }, { 0x000bc226cb0016cb, 0x0003337b24b4b6f6, 0x0007e66af842d281, 0x0008210d9fe58afe, 0x0000f22b8a2b6b8c }}, + {{ 0x00062dbbba9d84ad, 0x0008b4ece53c2d04, 0x000d18184b2003a6, 0x0006765b0779d897, 0x000067fc9538f5d6 }, { 0x000035f7ff931704, 0x00005f2cafe37b68, 0x000f6c983617d6e7, 0x000df03fd273d1eb, 0x0000249da2e138a9 }}, + }, + { + /* digit=21 (1,2,..,64)*(2^{147})*G */ + {{ 0x0009d1febf2de9a9, 0x0008bc77e6c55202, 0x000fb2e7eb995763, 0x0009cbd6dc27df9e, 0x0000444476cdaaa8 }, { 0x000ba7ced0785f36, 0x0004b9e93470afed, 0x000906fab0ce1fe7, 0x000bf2f043e6a966, 0x0000d9c1876fd26a }}, + {{ 0x0008d94f6be7acf2, 0x0000341338d6e434, 0x0002fc6886610503, 0x0000f96ca56f7dd7, 0x00002f54a7c972af }, { 0x0003e19d89ed269e, 0x00005bb2ef8279cf, 0x000a1736ca68762e, 0x000fbb351d575465, 0x000061e8deb175a7 }}, + {{ 0x000f96f33f9e0680, 0x000b268e32dd620a, 0x000eb2e15662f4a1, 0x000310882913ec99, 0x00003ddc488aba8e }, { 0x0003f9b5a7ed9adf, 0x00058e5dc3fa8a51, 0x00019cb68475d2cc, 0x0005558b0236bb53, 0x0000fb0d86c8819c }}, + {{ 0x0002aae2b99711c5, 0x00031ce51efb3108, 0x000412e3130475fc, 0x00003c959cb5b2eb, 0x0000efeaac806f9f }, { 0x0004837f98bf8cb0, 0x000da39411aa636f, 0x0004e03d299b3bd8, 0x0006376b77152ecc, 0x0000b7bc15a5f18a }}, + {{ 0x0001162ade988968, 0x0009ac4c753c45fb, 0x000648b216204571, 0x0006f3be618ca81b, 0x00009a6be30dbc18 }, { 0x00042cd4528f9326, 0x0008720236e3ab64, 0x00056e788ed3e0dd, 0x000e9967af88cdaf, 0x0000c45162d8c709 }}, + {{ 0x00083eca38629302, 0x000f164b71698f58, 0x0002173c96261574, 0x000a82abb6ba418d, 0x0000a7d774e7d73d }, { 0x000c1dd5576a4dbc, 0x000283ff3437f24a, 0x0001f86ddde594be, 0x000503b65e910b43, 0x00004e079387d0c8 }}, + {{ 0x0004a222c42e90db, 0x00035f924a0a3fcd, 0x000413dfa13526e5, 0x0007cd19938e17bf, 0x0000dbc59707ebbe }, { 0x00038d8fd101963f, 0x000debe13f542b66, 0x000d80984486c605, 0x0005578791fc65e0, 0x0000318e04247e1c }}, + {{ 0x00039d3ee2963f2f, 0x000233a626332d5d, 0x000281c47c620310, 0x000452fa27bc3883, 0x00006bc300c4cda7 }, { 0x0004c341a09ee9c3, 0x0003ad3e7676b0dc, 0x000f36ad1e76c678, 0x0007b36a8620e35f, 0x00009710f4c3af53 }}, + {{ 0x00073207beab3a63, 0x0009a2f69beb6e85, 0x0003163e05455030, 0x000493e4f3a50b99, 0x0000ae1c490dee61 }, { 0x000c1b26b4aa0c62, 0x00020e309fe7e5b9, 0x00039f25812cb0ec, 0x00023f4e2957678a, 0x0000be251e8728aa }}, + {{ 0x0000e7c2becbd6ac, 0x000da1cf405ff065, 0x000cf786bfcc5059, 0x0009287fcebac86b, 0x000006628092b297 }, { 0x0003a21f33a3f231, 0x0006f95fa90d7679, 0x000b52ce8c481fe2, 0x00088d46dae60eb7, 0x0000dd27b34c3095 }}, + {{ 0x00081eea1daaae26, 0x000110babb886433, 0x0006eac32b7bbb8e, 0x000c7f07191df36d, 0x00000e8acae417e0 }, { 0x0009aba1faa58bab, 0x00046ba39966e6ca, 0x00083d7db2981427, 0x00026c07f7c464a2, 0x0000d90bb6000f64 }}, + {{ 0x0007112386fc20ce, 0x0005122369b87ada, 0x00088beb81cf895b, 0x0008f96663c00e5d, 0x0000786fe6f72494 }, { 0x0003cd4c08b1b97b, 0x00066986d9bd5f51, 0x000845f5fa36c27c, 0x000c259b22dcc7e3, 0x00003a7a6a264018 }}, + {{ 0x00066ec8de9b7d5f, 0x000fef8e32377106, 0x000e65642ed171bd, 0x0002ca92b0dadb26, 0x00004cd467fccd49 }, { 0x000b7d34d2311a46, 0x000d8d5ee5242161, 0x000b265d7be3eae2, 0x0006c90bacd93c59, 0x0000677b279563a0 }}, + {{ 0x00014104124194f3, 0x000f16022caf46b4, 0x000f5955fa3d17a1, 0x000fa544e6a45dd8, 0x000027f7e6432277 }, { 0x0001bcd329f0aee1, 0x000ac1241de97bff, 0x0000cbc3ebca8120, 0x0002c1f37b8547b6, 0x000028b36f4746d4 }}, + {{ 0x0003e297caf789c4, 0x0006a0910caad676, 0x000ff14f0a163a1f, 0x000c604401f59195, 0x000036cf5f7f1567 }, { 0x000cd19846f4da2f, 0x000dc2e49abad67c, 0x000c206c18924750, 0x00014ee406ec405f, 0x000043041d3be98b }}, + {{ 0x0004a07f82ebea7c, 0x000b5769bb816483, 0x00053090a14d150d, 0x000e04062d69eb48, 0x00003488c7727160 }, { 0x00062b085ef31ed0, 0x0002787bbfb55541, 0x00004cb77b99391c, 0x0007343a0b06ed4d, 0x00001334165040ae }}, + {{ 0x000ae1cac5e3f9c0, 0x0000b31e0dd3eca2, 0x000cba4d48dfd8b5, 0x000c10297c6237fb, 0x0000108543603d43 }, { 0x0006f85dc5404165, 0x000a2d47845ae4f6, 0x00026761151b0706, 0x00037b0bbfec7dd9, 0x00008f9d062c7bdd }}, + {{ 0x000c6ff6e5be4b0a, 0x000ae88bdbcf0ca3, 0x0006a2b7a7ba1b36, 0x00009df34d75f5da, 0x0000309466d23eb2 }, { 0x000f30b5ece886f8, 0x00084f4047d26e4d, 0x00077119364a90d4, 0x000524daff370f7b, 0x0000a8cfc76f60e6 }}, + {{ 0x000e32f8475f689e, 0x000ae624896246e5, 0x0006905dc38a3b9a, 0x000963d735652c01, 0x00005aec6e092828 }, { 0x00069e5622c8ef2d, 0x0008612cf933c835, 0x000dc3e0f1c33e14, 0x000acf715d05bb76, 0x00001af4af25e9cb }}, + {{ 0x000f6d12f8d61cbc, 0x0002f3a6568013e8, 0x000e93853103e5cd, 0x00025b74a8dbd203, 0x00007ce46d1f9bdd }, { 0x0000b6a7c6e4974b, 0x000c5a2c9eade291, 0x000337266751d4ee, 0x00084c246bab4f32, 0x0000f12b5a16b482 }}, + {{ 0x0004d506bf73235e, 0x0007ff33035b41cc, 0x000ed4ef51d687f6, 0x000a4f9e5d336343, 0x000094104ca078e1 }, { 0x000895041f284f32, 0x00096cbbbc33f758, 0x000a09cdc35e1489, 0x000caf6237281f08, 0x000051a7f97c87c8 }}, + {{ 0x00013888f004cd15, 0x000d5499c438edfc, 0x0002d0b53154f812, 0x000ed3bf07678dcc, 0x00000b340fc797dd }, { 0x000ce0637b7a5424, 0x00036d41140af420, 0x0003858e6e2c47bf, 0x000a2455f8e5104f, 0x00009c5fb7f1500e }}, + {{ 0x00000c81b681942b, 0x000d6e6fe631e8f0, 0x0007d7b150953146, 0x00006a94f1c5563d, 0x0000b00f364d03b5 }, { 0x0006d17634ee0c1b, 0x0008e51aae1fd9a7, 0x000716897755713c, 0x0008cab22e35626c, 0x00003897cfcc2e16 }}, + {{ 0x000127ee79d5b629, 0x000e4daeb300f7aa, 0x000c4c8df9876122, 0x0001efb4c9dcf7df, 0x00001b98eaecee9f }, { 0x000be67d99c079ff, 0x000cd53575f1dbba, 0x0004d4ff98848810, 0x000ac51d095037e8, 0x0000af6c7913e968 }}, + {{ 0x000227b7d5fa2779, 0x0001b4b3f8e10652, 0x000610e6d14e4258, 0x0002b8836ed908cb, 0x00003ee0ee50ff75 }, { 0x000381ae0cc87753, 0x0007593b5cd6dd54, 0x000162a5e8db86de, 0x000a18e2c82225e1, 0x000026e1628fc502 }}, + {{ 0x000ea9dd3d739ece, 0x00045d37fafa4c8b, 0x000145e7cd82c0ba, 0x000ca584c3d91808, 0x00000a483c77e1ca }, { 0x000b89ee5c5235ad, 0x00023673eadd3e7c, 0x000739e9ce7c6b70, 0x0006c9b6c0033c1b, 0x000042ac04521d0b }}, + {{ 0x000ec25669d8ed84, 0x00024f322532e5d5, 0x000013bba5e2b6f3, 0x00079a2dbb6410ff, 0x000030dbc6caee49 }, { 0x0005a7ea04cc0c9d, 0x00091064c3beb017, 0x00000526bb51e340, 0x00076ab7a852123d, 0x0000ef64f923f62b }}, + {{ 0x0004049e0b602df5, 0x0009a356798e4ba6, 0x000563cc4285eaa9, 0x000fb0f039214e6e, 0x000054c04a0b7d50 }, { 0x0008dabf1a099126, 0x00034f13ae9841ff, 0x0008b7dba38bfa5e, 0x000b40e58dbe4ca1, 0x00003b573ac345f0 }}, + {{ 0x0000c60f36acc716, 0x000daaacce2793bb, 0x000e1526388b0989, 0x0005c39a971db851, 0x0000043ab896078e }, { 0x000ff069e32c40e4, 0x0008d21774eadbd4, 0x000f7e41ce7fd901, 0x000c7857625367fa, 0x0000b451922fe6a0 }}, + {{ 0x0000e7363ef92c3e, 0x00014568776e6676, 0x000c515535767ae8, 0x0000e4c876e76142, 0x00007574380ed322 }, { 0x0004548dca9348c5, 0x000800eb7366d800, 0x000a43e34e14c3f8, 0x0008b6c6fe4ee14c, 0x0000f0365a22d0d6 }}, + {{ 0x000b8dd3d3c731b7, 0x000974c5ae888eb9, 0x0000bb5bd8c5d56d, 0x000390929733abb5, 0x00006206511f4cf3 }, { 0x0006c1ff0991bcef, 0x0005d920bc5d11a2, 0x0007759d899a17fe, 0x000604db8f20127a, 0x0000afba4da06484 }}, + {{ 0x00027d09f86b87a6, 0x000672f7189e71f4, 0x00081c5287eec6df, 0x000edd421c643874, 0x0000dd3e802a5f6f }, { 0x0005cba1123d99ff, 0x000318f38384a7a9, 0x00041aa78a8746d0, 0x000ea5919aac3acb, 0x00001546626df6d9 }}, + {{ 0x000d7ba8b08355a7, 0x000a87fb00009c49, 0x0006f276e2aeebe7, 0x00041f05db8396c2, 0x0000859783ac9299 }, { 0x000c3a6250aec83b, 0x000b59f34f485b64, 0x000c17d7c2c584c4, 0x0000c36febc0ddac, 0x00001a06deb2c157 }}, + {{ 0x000630745c2443f6, 0x000db03d1295837c, 0x000f71d776d34583, 0x000701e91a2893e4, 0x00009134f5638299 }, { 0x000d8586465dff85, 0x0008b9e48661ae16, 0x000b7ca5a02e6a92, 0x0002951b58d17e7e, 0x0000e3f1660e4165 }}, + {{ 0x0009be4d8ef6f6d7, 0x0004d3e19115ead5, 0x000df1ee024075e8, 0x000331b5b1ce1393, 0x000093c9ce9841a5 }, { 0x00058519cc609023, 0x000bd6cd0def72b5, 0x000d8aaf0d7bacb6, 0x000dd273db923db5, 0x00006b02deb40132 }}, + {{ 0x000c548b69ea5df5, 0x000f6f8827097ef5, 0x000b740069f3544a, 0x000d0db89259353d, 0x0000e612d2045282 }, { 0x0008b57bb32e6372, 0x0008416994b995dc, 0x0005dcceda4ea724, 0x000bc67f04d1531f, 0x00007cded8a96b16 }}, + {{ 0x000c6b9000bbb209, 0x00006cb1cd5ad73c, 0x000594d94a4eac92, 0x000b0b5b8b9c860e, 0x00007f19f8ffa9c8 }, { 0x000d03e590e97791, 0x0007ca420c8a471b, 0x0000bafb9a1ec30a, 0x0003e497769ae05a, 0x00000ae589eaa8d6 }}, + {{ 0x00098231b2fdae8d, 0x0009e5eb2940d9e2, 0x00062218c3b180d7, 0x00034763d3c663d8, 0x000079ac5ceb7ec0 }, { 0x0008b408269ba673, 0x0009a2c552ad093f, 0x000c06d13e6ea662, 0x000789a00a4799d1, 0x00006bad7b20b037 }}, + {{ 0x000f3764f7b38e54, 0x0005dedf1bd72186, 0x000adbed6377389b, 0x000079665d491c53, 0x00000d1755479780 }, { 0x00048c7a526618d7, 0x0003a6dcdc348009, 0x00062d56bf03752a, 0x00092516bcdbcd45, 0x0000cec167b1ee0e }}, + {{ 0x000439be9201d0e9, 0x000c1377378058e6, 0x0000a00752c2e9b0, 0x000b1a1c741c746a, 0x0000bb3ea39e2af9 }, { 0x000f7de5688e9cd2, 0x000e2b043a2fbad0, 0x000f733802d7bd56, 0x000faeb4c39e6dad, 0x000033ce90509fd4 }}, + {{ 0x000b539dc2f62b1e, 0x000a7a1d2ffbfc15, 0x0009f177c68e623a, 0x000880758f74211f, 0x0000f1010e0ac5ca }, { 0x000a989eca797cca, 0x000b9d02f3af6f6c, 0x0007cefab003de17, 0x0004d1f835213207, 0x000053ff9d904563 }}, + {{ 0x000c547b711b42fe, 0x000ca29d6f62f995, 0x0008efd6009b4f01, 0x0005dcd0a8f06a30, 0x0000d50d91362f7d }, { 0x000916e5205bf484, 0x000bce1682afe112, 0x000a522adb87ea28, 0x00018d9369e0fc77, 0x000083acc24ddd86 }}, + {{ 0x000fc96822eaed69, 0x000e9e83f883faf0, 0x00091bbd25130ab0, 0x000b42f81396f963, 0x0000c9a426aabc61 }, { 0x000ba8446d25cd38, 0x0003d8766f1c68d2, 0x0003cf9f27a01b56, 0x00052259010fc6de, 0x0000b7f2dba90c5e }}, + {{ 0x0002ac509835efa4, 0x0001520ef4a9975b, 0x00037083f2c76c48, 0x000c843b560bb600, 0x0000295a37fec299 }, { 0x000d83ca9a68bcb1, 0x0003d2fb0d1ea67c, 0x0009a7419796877a, 0x000c6d9e6ae779a6, 0x000089f4f8df2e42 }}, + {{ 0x000ecbc81ce29d74, 0x0004af5e33deed3a, 0x000e514cca8bd88a, 0x00042493ee6356f6, 0x0000b9f30d92af98 }, { 0x000e29242f1443f0, 0x000c909e6e861dcc, 0x000385ccd3d2d0ba, 0x000f07e5790a247a, 0x00004947dfd69c12 }}, + {{ 0x0001d1645c39f241, 0x000e04a2f998a912, 0x0001b2e98e1ecaea, 0x000d02622271ee9b, 0x00004196510475ae }, { 0x0005bbdedb159a7a, 0x000aa2c7501779c0, 0x0007f452a12df2d4, 0x0007df729f006140, 0x0000ac021b7f9c4a }}, + {{ 0x000c314056ab1eb9, 0x0008f3225d984730, 0x0005503443d0bd10, 0x0004846a1f207fe8, 0x0000f455eb506396 }, { 0x000e1c005b309371, 0x000938f4107e28f1, 0x0001230567e4dd24, 0x000104f6e74ed387, 0x00008f86c0103426 }}, + {{ 0x000a793b2657b1ce, 0x0005c73f37ec3c39, 0x000d42e0b07a8381, 0x0009b6457c65259b, 0x0000b5dfca037683 }, { 0x000a1d4307f2b6b5, 0x0002e4be43756f0e, 0x000db0de13c200e9, 0x00069d6272f76bdc, 0x0000caa7d0f8726e }}, + {{ 0x000ad40c5a7ab2a8, 0x000ab308bdf623a6, 0x000a196def2311a6, 0x0002b55955f5accd, 0x0000420ad92fbdb8 }, { 0x000ee1c791d19a99, 0x0002189aeca8709a, 0x00007d42c30b7260, 0x000c8e176c64a7cf, 0x00004c391ad3c0a0 }}, + {{ 0x00097b04cf4dcd27, 0x00071fd21a843b21, 0x000d738557e2a32a, 0x0004063be56e4ed4, 0x0000f569b199b0d1 }, { 0x00027ecdb28f119d, 0x000161ffd42c13f0, 0x0002b190c58ab2e8, 0x0004f1dd45ef7feb, 0x00003bb3c53e8fce }}, + {{ 0x000985b93206998e, 0x00056d14e3352feb, 0x000eb9d3625dfe11, 0x00040ef6008414d2, 0x00001dabe9e71cb1 }, { 0x00010ec86988d8f2, 0x000e1ab96bf36f90, 0x0005bad089761cb9, 0x00093257d012dbed, 0x0000392d438d5756 }}, + {{ 0x0006074b7b7f3ff9, 0x00007f6bf926dffa, 0x00014eb820473ad7, 0x0009a18f57c9fdbd, 0x0000f0b5518a13e5 }, { 0x000bb55bbfa1756b, 0x0009566e84f44a48, 0x000846c6398e72b1, 0x0007e239a99998d3, 0x0000d5d77e99f313 }}, + {{ 0x0002f49e3766fab5, 0x000bdb40ea2ab303, 0x0000c448caa69039, 0x000c82cebd365a34, 0x000050f4208830b2 }, { 0x000004213cabc507, 0x00050f101d8b4d66, 0x000bc6b055c40b77, 0x000137b6a03c8423, 0x0000747dddf18bf9 }}, + {{ 0x0008f4ffcfd9c86f, 0x000f5a1c1c06681f, 0x000ef60c8e811dc3, 0x0007a43b316e0a16, 0x000027129f4ec201 }, { 0x000aba00062f76e3, 0x0008ecd4cc5e3e06, 0x000a366fe3da9e55, 0x0003be8f1d55c7cf, 0x0000d191984e99e0 }}, + {{ 0x000c5d7ec808f4e1, 0x000fa30d232cfc4d, 0x000df92cbe274727, 0x000ebb0499ddcf12, 0x00001e2f6a31d47e }, { 0x000abfb950d7fce1, 0x000b76e0b50d7dd6, 0x000715bf57d98c4a, 0x0000e08f688fdbaf, 0x000030c606a9d9d8 }}, + {{ 0x0002335df2a1fb08, 0x0003c878a429f4f8, 0x000b591e08172eac, 0x0006eaddf0649712, 0x0000e4d7ee646545 }, { 0x00038956fb888a73, 0x0007d675c93d9950, 0x000a64de24ea9aff, 0x0004f221b59ac367, 0x0000ca2c7aa20399 }}, + {{ 0x000ac7d898fb100b, 0x0003dcbbb734e37e, 0x0001411dd9d6657e, 0x000a195d9c08c0bf, 0x0000b128040fa2fe }, { 0x0004303f220f9f80, 0x000cbafbc1a6dea1, 0x000f02fa70b9b9bc, 0x00096441afdd0e2b, 0x0000f73ba67d9a72 }}, + {{ 0x000a9157dd651c99, 0x0009d210408f237b, 0x000a32417a1054e9, 0x0008f1da2cad3b09, 0x000057f0f2c67ced }, { 0x00053cba344ed082, 0x000b3074f013efbe, 0x000ac4fc0a6eafae, 0x000d5c1451b112c4, 0x00003f0ad4322c4c }}, + {{ 0x00074ea0b52dc146, 0x0001a63ded40d232, 0x000d2d41ed701785, 0x0007af6313d5f1f4, 0x0000d9a531a3ad12 }, { 0x000744ef50a57f11, 0x0007c99451d1b71a, 0x0002e834e0295a02, 0x0004f26f580e8264, 0x00004d9fa000c2ce }}, + {{ 0x000fe5e5592df016, 0x000d1f5cf551396c, 0x000db6a682124601, 0x000541636c616898, 0x0000d5c526596e5e }, { 0x000fa2cd6fae11ac, 0x0002c187051f476d, 0x0005354d66d26431, 0x0003324f656fad2a, 0x0000bcf48f474c5c }}, + {{ 0x0002136d7196d889, 0x000f83ad15d15174, 0x0001f569d783786a, 0x000612a5f61214cb, 0x00008b285ec5b82f }, { 0x000ec5c4199a8b19, 0x000581b084172219, 0x0002b38e963dfce9, 0x000c53f2d0a5546b, 0x0000d57c27a12776 }}, + {{ 0x000378c25b25433e, 0x0009ead771849f1d, 0x00042e5efd140744, 0x000ce30b2b0c932c, 0x00002dbc7048e363 }, { 0x000fbfe01a53e807, 0x00099da9e6e3e319, 0x000d16caf5b6722a, 0x000dee6b80417a54, 0x0000e7b46d452fce }}, + {{ 0x000e093761507e7d, 0x000a805ba9384a2d, 0x000d86a4325fdf2a, 0x000bcb8a621b8596, 0x0000d301c511e847 }, { 0x0004f1d7d5d3607e, 0x0004ec8f7bffe6d9, 0x000ec8a24e675bb4, 0x000936e3027566bf, 0x00008ff4238ff05a }}, + {{ 0x000842e7e3646f13, 0x000721aaaca5e9b7, 0x0000de2e53b43348, 0x0009b4ef6518aa52, 0x00004c1f8413e3b0 }, { 0x00027091000d9504, 0x000ab656868a5489, 0x00024df86b81806a, 0x000fc7333d963bd8, 0x000027dffecf8b57 }}, + }, + { + /* digit=22 (1,2,..,64)*(2^{154})*G */ + {{ 0x000de8cdd0926a17, 0x000b2a7d298c241d, 0x00097897af2c8ee1, 0x000fad3dbe3b697b, 0x000095a39dc131b2 }, { 0x00088519ea3f0b69, 0x00087a75a4d2604d, 0x000e93901ec55560, 0x0004fb6cc27154d7, 0x00006bfc52e1469f }}, + {{ 0x0002571fa1c005d3, 0x0000a1a7cb1282c7, 0x00064bef7798f823, 0x0000303b2a08d762, 0x000094b95e409f27 }, { 0x000cc832f936b83a, 0x000392ff22dfd98c, 0x000633bedd944ef9, 0x000cfe67a87ab01a, 0x00004e149b05af22 }}, + {{ 0x000f839761bec299, 0x000fc0e36ff3bf0c, 0x000f212faed6d78a, 0x00014bf5f8ad20f6, 0x0000003cd07951f7 }, { 0x000f568724f96e64, 0x00061b38288c545c, 0x000a43bae4ac5ec1, 0x000b2bcef2f8e4e9, 0x0000482335f80a39 }}, + {{ 0x0005d70b7657b7a1, 0x0007085ca8d9d1ae, 0x000cf3b0d35bf0d7, 0x0000cc024adf2c77, 0x000069d110cf7a09 }, { 0x0003a7564e157769, 0x0001d506260e70ff, 0x000703a97ab76d5e, 0x000435a6439d75ea, 0x000076aec3e36360 }}, + {{ 0x000b491d7bce26e6, 0x0001ecd2f52d2a7a, 0x000a379638de4a08, 0x00078ccf74fde149, 0x000012daea2770d9 }, { 0x00082aa266a6dcf5, 0x0007718c6312c7fe, 0x000c827977f0f2fa, 0x000766ea0ca0d5f3, 0x0000282fd2c5bf2a }}, + {{ 0x000b4394784f1e40, 0x0000700ec3128c2f, 0x000192f4aad1458d, 0x00011e4eb6c76d86, 0x0000e130115cd871 }, { 0x00052289429acd15, 0x000614a8dc796230, 0x000daf1e7270619b, 0x000f8d90d5b86995, 0x0000166c13057d31 }}, + {{ 0x0006fdbec46ef830, 0x00010a51e6a75c1d, 0x000537aa19534622, 0x0007f159724327c7, 0x0000279e5d72bd98 }, { 0x000d87b2825485ae, 0x000effb13c90f48c, 0x00029ae73fe0cc43, 0x00070a0ebd6ac527, 0x0000b44da3075aaa }}, + {{ 0x0002f99d592d4607, 0x0009d3e3fcd3efca, 0x0004150ef1418504, 0x000f73feadd26c03, 0x0000c8303bb7708f }, { 0x00028bb4209cedef, 0x000e4552f1da46cd, 0x000d486f19b140bd, 0x000e6167872c3f8c, 0x0000b4a36e89cdf5 }}, + {{ 0x000c77a0a3360371, 0x0009611745edc116, 0x000cd150d4b51af8, 0x00099ad649dd9ad7, 0x00009c97f502587e }, { 0x0009ae48860379bb, 0x000eb296baf7cb10, 0x0001c348e49c5f3f, 0x000e9d571cf065f9, 0x0000b56bcf3f6375 }}, + {{ 0x0003b56be8e6e5b5, 0x00086ecee8314f30, 0x000dd834bda67796, 0x0009af7ec1c068ae, 0x000043ec03cb0895 }, { 0x000635be527dea03, 0x000b5f29222882d6, 0x00065e79cda53115, 0x000db7a8c95ff38f, 0x000042689771e2a1 }}, + {{ 0x0006fa038810126b, 0x000e776a5177900a, 0x00074f8b83c9f4be, 0x000030e6c4e1dbb4, 0x000046c45fbd9ab6 }, { 0x000f312eb8992e97, 0x000c6562844c6d0a, 0x000a547118c55450, 0x000088ad6c633cf9, 0x00009ee0bbc48794 }}, + {{ 0x0007709b7b97f8c5, 0x000cdd2e7417ce17, 0x00011f9e4c798c2e, 0x0006edf1eff42bc7, 0x0000d3407d05f1c4 }, { 0x000b2930ca06e395, 0x000de1dbe217f2cb, 0x000ca08df0ad2e70, 0x0003a3b592af2a8c, 0x000040a4a94b9f52 }}, + {{ 0x000519834b80ba88, 0x00058477d930cfc2, 0x000114564ea90584, 0x000e06fe14c262ad, 0x0000425017607c82 }, { 0x0004968949e7bfda, 0x000414cd105f961c, 0x000511a5410a9516, 0x000e2177ca5415da, 0x0000759aabd156a0 }}, + {{ 0x0008eb87587c0983, 0x0000e26cecdaa7af, 0x000d12279d229b41, 0x0007b3ec4fc8c7e0, 0x000092acdf0c79b8 }, { 0x0009f00adefde818, 0x000ea12c00674527, 0x000e01ab878d0446, 0x0005ac2f43f6f69e, 0x0000c28cfb3ac1f1 }}, + {{ 0x000aae38b1dc4f8d, 0x0004e7181236c972, 0x000b14323f6d85c1, 0x00094867274f7685, 0x0000efff992ec9c7 }, { 0x0002818fa5d10725, 0x000bd22f6dd9d1bb, 0x0007a909d4e14581, 0x00027b9aa5769544, 0x00000f6612b6817a }}, + {{ 0x000cf60f96d10e48, 0x00095daf176f2c08, 0x0004556116c14d5c, 0x00025e7fb01ca460, 0x00006c4e588656b4 }, { 0x0007fb754d3440e4, 0x000851cc9071c4aa, 0x00038c48b2b6677b, 0x0005b2981fd58874, 0x0000cc23558b384c }}, + {{ 0x000a3165c2d1e96e, 0x000ce5cd51805e1f, 0x0008b86876e5f775, 0x000018700ab4ccd3, 0x00005515bbda4876 }, { 0x00052c258fdcb035, 0x0005654e160e552a, 0x0009eaedab8f23ae, 0x000ea157adc4972b, 0x000066592aa69e2f }}, + {{ 0x000ffedd92412715, 0x0008af4c1e99d816, 0x000633c73a2eb72e, 0x0006556a708e1a7d, 0x00002684c134b0e3 }, { 0x0009bf250d515cb9, 0x000a0d2d629d2080, 0x00052ad12e01390d, 0x000f3a089a9d141d, 0x0000fb8a94900ae6 }}, + {{ 0x0005d66a6d649943, 0x00090b1b38227904, 0x00004de22caebda1, 0x0005eb7626c5926c, 0x0000f67f61d51111 }, { 0x0009445c606af316, 0x000c063987d2792d, 0x000c656ec0cad92f, 0x00039a3e64a73caa, 0x0000e4821ae46138 }}, + {{ 0x000981afc00085d1, 0x000d0d5af71013f7, 0x0004926c19866a71, 0x000ed9fa0e68216e, 0x00008c7bcfc44bd1 }, { 0x0005694612f9623f, 0x0002b679449f0e1f, 0x000d14dea79b08e7, 0x0009c8b2286cfc4b, 0x00000d346cc223b8 }}, + {{ 0x00096cf17200d729, 0x0008e23901515ea5, 0x00051b2ffec6cf8c, 0x000c727594c6c77a, 0x0000320acc1aeeaa }, { 0x000269e868e20629, 0x0002afa6530ae7aa, 0x000d6528bf3d9e09, 0x000d558c90f5e631, 0x0000229c5edcbc0d }}, + {{ 0x0004cc22b4444cf0, 0x00009e12cb7191e8, 0x000860e46bf2a7c8, 0x000e63bb978ea1bd, 0x0000d8fbe677e701 }, { 0x00039a0269232967, 0x00082e9d49d5bf18, 0x0005956e7b3add6f, 0x000cd7c57cd4ec1c, 0x000020dc2c6c5c8b }}, + {{ 0x00021474c07616b6, 0x0003562c6d8607f9, 0x0004d89521244fd3, 0x0009dcd17759689f, 0x00003d544887a945 }, { 0x000f77453607db96, 0x0007d5b2a354c8c4, 0x000231b85f7fc2e2, 0x0005589e8d51e52d, 0x000081a4d861604b }}, + {{ 0x0001287ee355832e, 0x000a8715bde48f8d, 0x000250613d9a672a, 0x00086c08ac970387, 0x0000dae19fd8bb71 }, { 0x0005fdcbcf36d30f, 0x0007974db1dbf1fd, 0x000ac30ebd07464c, 0x000637413ea46588, 0x00007cc18fa7cb4d }}, + {{ 0x0000a61162698c8d, 0x000e043810d5ab00, 0x00090431d46a2727, 0x000a6ef336739d8f, 0x0000a26b0295f5a3 }, { 0x0005c0032a22784c, 0x000f53b379f0a1ff, 0x0004f422a86b7376, 0x000e6cd9cfe12660, 0x00003c940fd45b0c }}, + {{ 0x00089fa086f4a3d4, 0x000903037577dd85, 0x0003d1265e0202f1, 0x00019ed3283b82af, 0x00002538201d9fd9 }, { 0x00032bd123d0e234, 0x0006138b13799dff, 0x0000df69ff10325c, 0x000bba8ca85fa8b2, 0x000057bcce5a1bec }}, + {{ 0x000774df733efabd, 0x000cba7858b044cc, 0x00019335042416f4, 0x000dd98591f0d69c, 0x000024cd837a5001 }, { 0x000d42a35cffec65, 0x00084f6d3bd2e1c7, 0x00002b1b5b735de0, 0x00000a4c4eb30da2, 0x000019e181b75ed4 }}, + {{ 0x0008561eaab750ec, 0x000fd959890272db, 0x0009861b1b6617ec, 0x000313bb875f3432, 0x0000725d0e14f52b }, { 0x000a16eb56377ba8, 0x000efa08332aafd7, 0x0004f969b00c6f26, 0x000e75b6f9b0d8bb, 0x0000fe905ed9a6d4 }}, + {{ 0x0005c59502191a75, 0x00080f678e62ddc9, 0x00037237e0292642, 0x0002906be2267c45, 0x000076e18afb6945 }, { 0x000b67703ed66163, 0x000fc85f2b254d9a, 0x000c814dbb101610, 0x000e0fcb5844b617, 0x000051187f47b6f8 }}, + {{ 0x000df6ef65292f82, 0x000ccc13cf482834, 0x00054e9380f92393, 0x0004dcd84b09daa2, 0x000002ad22636b32 }, { 0x0004d671be580fab, 0x0005c63aa21ba469, 0x00080dbd873d895b, 0x00010ffbfc6b1d17, 0x000037d32eef447d }}, + {{ 0x0008511a63429641, 0x00057cf4f5792fac, 0x000648d037fe39e7, 0x0002f65d47c5e0d6, 0x0000b0155eccce65 }, { 0x0003bd08cf6b4388, 0x000da1ca5a60d998, 0x00084ba13ffba0d2, 0x0009a6608f7fbab8, 0x00000b92a4ebd38c }}, + {{ 0x0009badc329d1f15, 0x000798a2ac13e274, 0x000f04f6cdc35ac5, 0x000fff5624494f6d, 0x0000234123bdb8fa }, { 0x0003d01cf2e4388b, 0x0001a41004f7571b, 0x00002b4c77fb6c86, 0x000ef3131bac67d1, 0x00001a55a5e1aed2 }}, + {{ 0x000dc88b3456a9ee, 0x0008d1b55fd0b8ba, 0x00035123a63efb0d, 0x0006484b7a099d18, 0x000092c90c828310 }, { 0x0005f1462aa51ab5, 0x0006c6af989d9051, 0x000013c446d68fcd, 0x000caa593c2e68d2, 0x00006aa1218b67ca }}, + {{ 0x0000bd29c9340da1, 0x0003275ea3d62f2e, 0x0002210b6db3e508, 0x0006d85f6a17765a, 0x00006dc8a9e841b0 }, { 0x000bb9f1ed8d6637, 0x000606d08d84858f, 0x00046e14e3d3d3f9, 0x000cec356370dfe8, 0x0000614d73954f19 }}, + {{ 0x000e324aa43d7d34, 0x000b853060dd7ef1, 0x000d50332ed5585a, 0x000234e35eef2dac, 0x0000e1739c044fa5 }, { 0x000d402c7e387317, 0x0006771e6e676710, 0x0000b212ce791c9b, 0x000c9af2abccd519, 0x0000dc23b3bd97e9 }}, + {{ 0x000480be272bb944, 0x0007c39306a5a3b6, 0x000d31f77bf47852, 0x0009a89d44e0a41f, 0x00005f69215972d6 }, { 0x000a8f43e1bf1716, 0x000481d2285f3821, 0x000fd17c18552f1f, 0x000f83782188d8d8, 0x00007bd976ec665f }}, + {{ 0x00075ffd7e18ac01, 0x000216d77aefb3a4, 0x000dd712c4d568bc, 0x000ad4cf1f69a751, 0x0000e55f50391cd0 }, { 0x000dd2e229415715, 0x000adb13844e95c1, 0x0007bf400d6b744b, 0x000af280085caf64, 0x00003a42515d76af }}, + {{ 0x0000db2ffa6b60c9, 0x000179d73d172440, 0x00068eeb23f6cd90, 0x000c0191c3ede774, 0x0000e518c9f2fb70 }, { 0x00079a440a790688, 0x000a43930cfd9057, 0x00042e26ce9b9861, 0x0008576954934d07, 0x000024bf5ca7badd }}, + {{ 0x000b28c70e222015, 0x000dd1897d6fbdcf, 0x0006e54ea39dea03, 0x000f2d3dbd3a5cd8, 0x00009af5dacd64b4 }, { 0x00051b3088c8cb5e, 0x0006928b61301a0c, 0x00080e5712cc3715, 0x000c4a0dca628216, 0x00007183a4f877ad }}, + {{ 0x00033f48d51a3fa9, 0x0006b7c3902dda59, 0x00066b2a2adf8b68, 0x000179c7e35d2f70, 0x0000efe1c34ffaa8 }, { 0x00032c6af415357b, 0x00034884912beb26, 0x0005b30274a0ae30, 0x000fd784b7f5a4de, 0x000068784fca87f6 }}, + {{ 0x000f4a94d8f05720, 0x000d2e3753e43a97, 0x0006a58a4d66687d, 0x00096c6a670e1d21, 0x00003923588250bf }, { 0x000b64615e973715, 0x000f5968d4faa6a0, 0x0002a502553213a1, 0x0007fc6fdd9ed0b3, 0x00008fdecf871ce1 }}, + {{ 0x000e553fc34528cf, 0x0005148cfb9fad53, 0x000ce5b636f9dd65, 0x000b208bcbeaa386, 0x000021343a11054b }, { 0x00047831ec12559a, 0x000419b9d5f97505, 0x000a7e142922e027, 0x000f2299b3e85c52, 0x000013d6e4fdc33f }}, + {{ 0x000101b9ab1418b8, 0x000ea274a82f424f, 0x00078d463ef2df0e, 0x000ae812c36919c7, 0x0000a9078f376e2f }, { 0x000e6b4528b15e10, 0x0008e9c1c5009d2d, 0x0002be663e1d0b7e, 0x0007067c7fb55ea1, 0x000065c8a8ad08f2 }}, + {{ 0x0005e18cc10dacdb, 0x000c8339bb26f5f6, 0x000d9e6bf8f50349, 0x0006ca3541e85db8, 0x0000696b34c84786 }, { 0x0008da3ee5c6c70a, 0x0004550a471fe6ea, 0x0009863904b6dcc5, 0x00040278bf031747, 0x00002271b171d4f9 }}, + {{ 0x0000398ec951344f, 0x000641c558216b17, 0x000bbf4057f8b421, 0x0005a9e3c90c7810, 0x0000ef7c615f84ad }, { 0x000d37bb5223d947, 0x000fd9ef701b2350, 0x0003de5d38438590, 0x000f22d310ceecdb, 0x0000b63f014f16ac }}, + {{ 0x0007f5e17c602df8, 0x0001538b880d2dd4, 0x000b1d9d2bdaa4e3, 0x000343df5a60f177, 0x00003595db60eb93 }, { 0x00042647efbb60cc, 0x0007f57db394af4c, 0x000cac10af9a0c05, 0x000c6e87239ba215, 0x0000a63e0d4e1e73 }}, + {{ 0x00049c0c51c25079, 0x000fc4564488f1d6, 0x000890337983826b, 0x000946663aa92270, 0x00003ecfce6e8391 }, { 0x0006ae337281abf8, 0x000de4e3cffcb6c1, 0x000e3a48ea23ea44, 0x000159ca0e3bf93f, 0x0000c6e00209e63b }}, + {{ 0x000bda54e9b5f5de, 0x0003cdf5ca48f3fb, 0x00062650469b09ca, 0x00013c04a64352f0, 0x0000a9c900d6b80b }, { 0x00034c2438695df8, 0x00045f56f6e55fbe, 0x00033ccb4b4fad0c, 0x00059bdff1ca44a2, 0x0000d8197cd15076 }}, + {{ 0x0006b3a610c31e52, 0x0006975861421033, 0x000ae881563cbbe1, 0x000a35f226704ba1, 0x0000fa0894f27e4d }, { 0x0009ee25bb322589, 0x00037a775a0d0508, 0x0006b2f5187779d7, 0x000d880be066f919, 0x00000c76c7c238f7 }}, + {{ 0x0003422f090ad74e, 0x0009022fd29dc766, 0x0008d3af43571f67, 0x00073f3267fd32ed, 0x0000d0b442ee5f1d }, { 0x000b361400cf8a31, 0x000f3f40807e199e, 0x00003777308f86a0, 0x0002880b4fe9ce5d, 0x00000da749380f3e }}, + {{ 0x0003eea570082bb1, 0x00099052680813be, 0x000c722c6366b500, 0x000f4a458527aa55, 0x00005bce35af9e9e }, { 0x0003afeb82f8170f, 0x0002917a0ff58b3a, 0x000476b9faa4836e, 0x00052a6d6b67d3f6, 0x00006f5fdddf36b0 }}, + {{ 0x0009432bf6f80693, 0x000ae6d3e5548920, 0x000d39cafe4e671d, 0x0002c0fa08413b53, 0x0000a88888d1f798 }, { 0x000b84f7978aae7d, 0x000d3c6daf0ecc11, 0x0005404428b0ca79, 0x000d2960749ebce5, 0x00002f4edff10ec9 }}, + {{ 0x000b1771c2c13be7, 0x0001d818d1d7a0dc, 0x00026f583393b81d, 0x000627635d9fabe0, 0x0000399a341196f9 }, { 0x00046124aa70c355, 0x000f2cae9a6312bd, 0x000f80ccb41ae366, 0x000b5f216a34b355, 0x00003e9a5b7e359c }}, + {{ 0x000bc3dd47503d26, 0x0001d5d68afe8b4d, 0x000f9049b15a067a, 0x000db429827f2053, 0x0000558730c6cd22 }, { 0x0007e4665dbf09b3, 0x000e542c03b0c057, 0x0003013d5638d610, 0x000167b0a8b1b3bb, 0x0000b3ec98ffe86c }}, + {{ 0x000ac7168a05d60b, 0x00002fcd23e39845, 0x000c910a2bac02e1, 0x0003b40a7ab86e8b, 0x00007c554cfc6e51 }, { 0x0003931577b8f581, 0x00022840139d9af1, 0x000862d13578d007, 0x000e1ce4a72733f8, 0x0000f2968562b2ac }}, + {{ 0x0001f568c4e93444, 0x00095eb70f8d5a4d, 0x000705c64963ec81, 0x000e7c732375adde, 0x00006e492766c122 }, { 0x0003224c6ff30b48, 0x000b4835459df02e, 0x000a2cab4e0e2287, 0x0003bf30a2beac0f, 0x0000b7c08ff94828 }}, + {{ 0x0002d191d34652ff, 0x000d10ee87edf5b7, 0x0001ae15eb2221b8, 0x000efd2b230385f0, 0x00004abd2d05ed06 }, { 0x000564f3ba13cfbd, 0x0001dca0917d9e55, 0x0008ca2af9c50195, 0x000eb7b96394eab5, 0x000048607473d36b }}, + {{ 0x0003da3500cac108, 0x0001505d73654ef8, 0x00038ffd58742011, 0x000886ba068d1a9c, 0x00002aa8c6ea07a2 }, { 0x0008af07c42cfaab, 0x00051a468ae01189, 0x0009377e98a53d43, 0x000b0d802a1b9a4c, 0x0000f99f7b9d03be }}, + {{ 0x000dd2d01e319688, 0x000eb26995f0f9ff, 0x000f73e2bba85402, 0x000cbd22b467fadd, 0x0000ea7ba8013c8b }, { 0x0005ee2fa76c4cbc, 0x000f228c693f9fa1, 0x00091ea149f1bfeb, 0x0001539834862232, 0x000070ee1dffe6b5 }}, + {{ 0x00052e2214ff073f, 0x00089455777e1891, 0x000030410614a29e, 0x0006eecd5cc10bee, 0x0000bd74b27941db }, { 0x000b6dff74a64cd2, 0x0003c727cb23c74a, 0x000ffd9e676af10e, 0x000132a53fd5d766, 0x000022b0948b387c }}, + {{ 0x00015648065531e4, 0x00083816f4cbe9d9, 0x0008d90103d196a3, 0x000769dfa0ed8b24, 0x000076059de2f82c }, { 0x000b69dcba54041f, 0x0000c085b8628093, 0x00011a1bb0c183bd, 0x00005ca2a1facd1f, 0x0000198f2db6c25b }}, + {{ 0x0002cf1f8a6fd4d2, 0x000bc9f967365efa, 0x00024532d63e9c0d, 0x000ae632f57300eb, 0x00004160be5ee065 }, { 0x00025c377f9c4ca1, 0x000fa744dcc40cc3, 0x00057b48ec7bdfbf, 0x0004d2b4dfee38c4, 0x0000461e8ab76c74 }}, + {{ 0x000377f264a991d9, 0x0007fce476cc9ff0, 0x000cc35cd8ff3792, 0x000e04f9b580e0b6, 0x00003f7b7baf52e6 }, { 0x0008e50f30877c3e, 0x000156288e591a5c, 0x000cac3ebafa9eec, 0x000630733a453a7b, 0x0000a9492c65a23a }}, + {{ 0x00010ee253308b7a, 0x0006b0f6d3549cf8, 0x000f79be840ba3a8, 0x0008682b46f33696, 0x0000b318d895599d }, { 0x000485717b66f888, 0x00087e17159bb2cd, 0x000da105c3fffe4c, 0x000a76272f7dfbeb, 0x0000bfbd7894f96f }}, + }, + { + /* digit=23 (1,2,..,64)*(2^{161})*G */ + {{ 0x0005fd5fa205d0f4, 0x0004ee8ee36860ce, 0x000c5b16628b839b, 0x0003f4e13daf04df, 0x00008b3aaf4c153e }, { 0x0004e879df3f3f34, 0x00029941a4e0551e, 0x000d33e8877228e4, 0x0005911236772cfb, 0x00007681b72c03ca }}, + {{ 0x000e349a4b31aebd, 0x000f2b0285b94916, 0x0007b017d6137900, 0x0002b0dab01a0be8, 0x00008ac2977211f9 }, { 0x0006974d05362415, 0x000fd464a1c6a163, 0x0005f2a0e55b98f7, 0x000493f71d99e6b6, 0x0000a0e9ca0b6129 }}, + {{ 0x000b8f7229523466, 0x000e3a1cfe89d80f, 0x000cdf0d11487037, 0x0009c00d2b42c026, 0x0000172110f51188 }, { 0x0009ff21f71bf171, 0x00008ecd850935b3, 0x000c9c32bfbbecc4, 0x00028143434c1a74, 0x00001cad52349f90 }}, + {{ 0x000d9cce837c6d61, 0x0003d148f9290579, 0x0006232b855bab85, 0x0007c64ce7a64ae0, 0x000028043d63ffdc }, { 0x000f181a75f69e00, 0x00059796e93b7c7b, 0x000172a383b1d5d0, 0x0000029a0e1e4709, 0x0000b9ee91160db4 }}, + {{ 0x000b670a385747ca, 0x00062658ead09f79, 0x0004fc19c7159df7, 0x000d39cbc1335e1d, 0x00000025ace0a875 }, { 0x000ab38fcace8fb2, 0x0002a8128efbeedb, 0x000bc873171affc4, 0x000fee8630c74e77, 0x00001249e10307e9 }}, + {{ 0x000ba8e478b7779e, 0x0004ac85c0a3f1ee, 0x000190dedf06d75f, 0x0006c198dbcad249, 0x0000bec9ca1c42e7 }, { 0x000b123f368231d5, 0x000b7b0eeaede4b8, 0x0007ee649f7fb5a1, 0x0002f72fb0bc498e, 0x00003fa9b3e1cde9 }}, + {{ 0x0005c8295bc7ccf1, 0x000ebc7de79dc241, 0x00023fc74071d988, 0x0004b5fdff1168a3, 0x0000a46cc6dd3945 }, { 0x0003a26e3c4f8d24, 0x00086367d8184ffe, 0x000d4ca20d9fa6d4, 0x00057819ec396228, 0x0000217db807076e }}, + {{ 0x00026fc9c4cd4abf, 0x000e23f1402b9d0c, 0x00026a0bfe9f0668, 0x000417f6e573441c, 0x000042560b13ff8a }, { 0x0003ef07f65b14ba, 0x00061fd7493cea35, 0x000ec7090c603bd2, 0x00077a68fd05d4b3, 0x00006ce1efdc940f }}, + {{ 0x000613ddd45cbe56, 0x000a5d27824fc56a, 0x00076d14b2a2aae7, 0x000062d493467521, 0x00004912a1180184 }, { 0x00016c4168b43a86, 0x0003b9c5450d8660, 0x000c8e186318cb3e, 0x000c25c3946f4409, 0x00002251eb5dbfdc }}, + {{ 0x00035d95c1f7dfd4, 0x0004ff0c74359787, 0x0007ad857857c300, 0x0001a9bafb152c88, 0x00002220a65483ec }, { 0x0002c5cc53ddbb41, 0x0003473039553173, 0x0009bd757d41255e, 0x0006c05000b4b02a, 0x00005bb2616a0592 }}, + {{ 0x0006f267f3450bcf, 0x0009e1f606241297, 0x0004236bc36ba3a3, 0x000fc5e72b7bc5d6, 0x00006d062e175519 }, { 0x000d792d60b7e583, 0x000e00fb36eed930, 0x00072e7c3aa2aa86, 0x00041b3a055230cd, 0x00005222efcee916 }}, + {{ 0x00025b58889424a0, 0x0003bfdd5c7c5d2f, 0x000614cfdba46ef4, 0x00069438ab38f8da, 0x0000c9d7330553e7 }, { 0x00099f94c0e0b82c, 0x00086842946ad608, 0x000d6031dbbdd21d, 0x000b21909854f29a, 0x0000f80169f11b4b }}, + {{ 0x000d22a1149324b5, 0x0002751f4a55b03c, 0x000d68c111c13c46, 0x0005dc651261762d, 0x00005612c643fd71 }, { 0x00026455eddae243, 0x000931a247c9569c, 0x00030eee68dcf34d, 0x000c4d3d459a5097, 0x0000bc8501472388 }}, + {{ 0x000cd3142c078a4f, 0x000e7e049ce89e7e, 0x0008dfb5f889206a, 0x000b1dba11890853, 0x0000fe4b40447fd0 }, { 0x000af34eb1a2b760, 0x0005f59b3f5ab84e, 0x000807cb5614c7ce, 0x000ecf41f3c320a6, 0x0000376551d686ca }}, + {{ 0x000d8eec2b43abc0, 0x0004129977c97c4f, 0x000ac7efcdced8cc, 0x0007704785ff9bee, 0x00003fe869d8f984 }, { 0x0005c75015e7e220, 0x000f94ef027458c3, 0x0008e754910013fb, 0x00084b6f4b4ff531, 0x0000df161344c360 }}, + {{ 0x000f63cfc83f45b8, 0x00011d646ac49d20, 0x000b73afaae16770, 0x000dca23842c77c0, 0x0000cf54b1e93428 }, { 0x0004dcfda1adde56, 0x0005a1bc6441f959, 0x000957b146ed74f3, 0x000a15bba7d38f71, 0x000080b43552bdca }}, + {{ 0x0004512fad93b7bf, 0x000f4395f6d4a09f, 0x000bfa01887930c4, 0x00063d40a6b4f355, 0x000015dc769eb061 }, { 0x0001dbaca543e3fd, 0x00084d5206c4c74f, 0x0003d97181a31083, 0x000245f2dd021d4d, 0x00004c9b52a51d86 }}, + {{ 0x000f5829708831a1, 0x000f226490a1aca3, 0x000c01fa4191db95, 0x00065f158cb64dd9, 0x0000b704509d211a }, { 0x000d54e389cc560a, 0x00066d17c213ba70, 0x0005db4beb4b0a15, 0x000ad2ec1c28febb, 0x00007d2a923ea7f4 }}, + {{ 0x00069bcfe21bda90, 0x00044604af206ee4, 0x0006314dacd7546e, 0x00089f02e786c88f, 0x0000fea9761aff90 }, { 0x0009a4ef238392e0, 0x000540d470622302, 0x00079c0fb4897908, 0x0005c65ace743bb5, 0x0000ac57eea586d6 }}, + {{ 0x0009fcce8362831a, 0x00014feb483689bc, 0x0005873df3dedda1, 0x00023c1722575d3f, 0x0000658b868344aa }, { 0x000525a7345027f1, 0x00021ff0149362d4, 0x0001d725423d3479, 0x00035076c95e5758, 0x0000632aa6fb7444 }}, + {{ 0x000caab33910184d, 0x0003e628f3383609, 0x00028c433ff2d754, 0x00072cdec483ff59, 0x000059db5682d119 }, { 0x0008c75049787047, 0x0008bfd668927090, 0x000cac92ccf7e399, 0x000d9fc942496b4d, 0x00000b039ef1d169 }}, + {{ 0x000389359d183e8e, 0x00043c2c905d85f7, 0x000f93aa78bab2ac, 0x0003e4f94e9d7f78, 0x0000cdc97efebfd8 }, { 0x0007303b35c1dcbb, 0x0004eb7367e45f70, 0x0007c44e3fa733aa, 0x000da6d181f20306, 0x0000693cf0cbca05 }}, + {{ 0x00016554135f365c, 0x00088e4d13b7ea1a, 0x00043f81ed6cbee8, 0x0003a9370e6489b2, 0x0000a763c3cdfd1f }, { 0x0005fb3cd26c7a98, 0x00026fb7af4194e1, 0x000c788e80c9e8e4, 0x0000ec056533c1c3, 0x0000e7c9daf6851d }}, + {{ 0x00014e77a0de8421, 0x000cc45526f09fd2, 0x000c5fd9ac6926fb, 0x000a7dc8de8a4f10, 0x0000d25068992420 }, { 0x000b6cda791fe0f2, 0x0007b7314faa40ec, 0x000b3679170d12a8, 0x000c08f3e767867e, 0x00000e1e221077f7 }}, + {{ 0x00083667c4ab99b7, 0x000646d349d51aa0, 0x000edb9151250bb8, 0x00006dfdff56b5a9, 0x0000e96a55350487 }, { 0x00075a324beb86c4, 0x000dd3518087f2fd, 0x000bf9dcb2114ad7, 0x000280589f1b8eaa, 0x0000a4dccd763888 }}, + {{ 0x000cc4569fc6534b, 0x000850101dae185d, 0x000abeb63e1cf161, 0x00085800b45434e0, 0x00003dded22e1035 }, { 0x000e50258feb27e0, 0x000518a09a512993, 0x0006746000488c61, 0x000c0779281b4d20, 0x000087a3225e890f }}, + {{ 0x0001a7ec6bfaf389, 0x000c3db2996864b3, 0x0008eda07338e1f4, 0x00058e15aa2e6708, 0x0000a57bd6c84a58 }, { 0x000d24ee7c6e4db1, 0x000c666d6bc20da4, 0x000c39512313aed3, 0x000225f634acdb5e, 0x000017899b25ac05 }}, + {{ 0x0000ef95acc5b65a, 0x00091d3a19fd8680, 0x000cda9e478efabc, 0x000ae7e52612e481, 0x00005879a89ba754 }, { 0x000ff9d98e9d1fb0, 0x00022edaa0a6c469, 0x0005a37d6e017817, 0x000202a106129408, 0x00008051d4fc5997 }}, + {{ 0x000a9868340ef12b, 0x000847c468b8835a, 0x000ac8766672f855, 0x000de1f81977a31c, 0x0000ae09622e4cb3 }, { 0x0004b60159822469, 0x00032f3a6bd9d340, 0x000b2f0fd6fae8ce, 0x000eed25e6a62fe9, 0x0000a1ce552aab34 }}, + {{ 0x0005f9ee8b1f3b1b, 0x0007bdf5579bea49, 0x0007c6f4ec662d97, 0x00081aab110e35ac, 0x0000838e05f272c4 }, { 0x0008c93722dc320b, 0x0004eee82b6eb0fd, 0x0004bcf973ba5062, 0x000f5f1fe2e84576, 0x0000987ab3588bf6 }}, + {{ 0x0007afbfe9bfa97c, 0x000db48f83e8a3b0, 0x00068574d6094e13, 0x0009ff29ba579aa9, 0x0000ac0b5352b6ee }, { 0x0008e78e9f640109, 0x000d36d5b7bd2931, 0x000f573aa39dd6c0, 0x0000935763b592cf, 0x00006b052765f938 }}, + {{ 0x0000cd9c328d84f2, 0x000ec10296c36eff, 0x000191f73e449397, 0x000f344da7ee8967, 0x00003d52cf283e17 }, { 0x000adab3ad961303, 0x000c8f7e455fe908, 0x000af39881456bea, 0x0006ae1ec3fb53cb, 0x00007d83567df6b8 }}, + {{ 0x000d359f6fd4efcb, 0x000232f53d648293, 0x0007652ba9bb3daf, 0x000db7f2c7a471e1, 0x00009d7cb3d94a0f }, { 0x0006782372900602, 0x00065860aacd6975, 0x00031f4ad535a466, 0x00057f24e92b444c, 0x0000a4012d7fa548 }}, + {{ 0x00018c74a1ce7f88, 0x00008f2bbbab3b85, 0x000e39780cc99267, 0x000135cd24851f8a, 0x00000d44135a09ce }, { 0x0003ccc547f7ecdc, 0x000d90af9877d4ca, 0x0000054d05740761, 0x00071d6b614578d9, 0x00002bd6390f7cb4 }}, + {{ 0x0007167876861ca8, 0x0004c8e327f83495, 0x00069d263dca7c3f, 0x000f26885f3e9734, 0x0000cc0b31df68c9 }, { 0x00011b5e5c6b6cf5, 0x0001fb9411925a6b, 0x0007eff6df88f06c, 0x000be215d7c078d4, 0x000083a8bd8ed8ac }}, + {{ 0x000abc1295834d88, 0x0001f6c1eea7b85f, 0x00009818e5c95ff1, 0x000938dbb2352b46, 0x0000387993489f2c }, { 0x000c5e12df15edd0, 0x000fdc05233f9251, 0x000bcb21d976697d, 0x0007be659e0a802d, 0x000053b74438966a }}, + {{ 0x000b70d5488e28d0, 0x00088c64b8457ee6, 0x0008e40dd4aef873, 0x00037e38fa5360f6, 0x00007f082ebb5766 }, { 0x0006f388f3012c36, 0x00001f13bfae3154, 0x000284ec6a763ea6, 0x00088e0f93379785, 0x0000205217d30224 }}, + {{ 0x00034f30d0a7f21e, 0x000285889d8ddbbc, 0x000dada45e17bd7a, 0x0001ff4b08830b26, 0x00008227c3ea8c36 }, { 0x0009e76d098d7344, 0x000eda356ab64b81, 0x000cbfb4f01d1e3d, 0x00024d9947f77f0c, 0x000077da11412e32 }}, + {{ 0x000ecd83bcdac8cc, 0x000bec396238f398, 0x000e19632faa0ca3, 0x00085e899c0a482b, 0x00002203b4334230 }, { 0x000b32d5095835f0, 0x0003cb7a2f3776f8, 0x00046985b5d63ee5, 0x000e9b446b42176a, 0x0000db866498ec65 }}, + {{ 0x00097128795b26d4, 0x000007d53b618c0e, 0x000b80b150a9c145, 0x0006c0564c424f46, 0x00006ae4b9ab6582 }, { 0x0000d841a1380e4e, 0x0005832f815561dd, 0x000502e81430573a, 0x000171a4f85f48ff, 0x000063896020863d }}, + {{ 0x00042ad522e18a65, 0x00044260b9267c6a, 0x00012add234060bb, 0x000833924c78913f, 0x000039e7e3dc6b6b }, { 0x000136fd8d64e5e2, 0x0003549a2c989d55, 0x000f90beea015851, 0x00038c4da928292d, 0x0000b65cea1cb0a5 }}, + {{ 0x0005f43682904822, 0x00022b1f2c34a996, 0x000459641f29f5c4, 0x0006715be3b00aca, 0x00008ad64a961e6a }, { 0x0000473f5f609763, 0x000f501a11b12e13, 0x000cb1c79515bd02, 0x000aa7dd578a7ab2, 0x0000493e312a1f39 }}, + {{ 0x000c13966b2406fb, 0x000e59d169c7d97d, 0x0003f39b6362b933, 0x0009d4d035cd5baa, 0x000048a4caa8a041 }, { 0x000310fa4ebd46a4, 0x00087e82b8357c1f, 0x0004f6c481db4a34, 0x000b8923e0cb1b4c, 0x00003d448294e348 }}, + {{ 0x000fb357b1ee936f, 0x0004452f97f68c6f, 0x000f0b4f77a6ed24, 0x0008c27d90c773b4, 0x0000c6d90e5f6f4d }, { 0x000ae575c3e40e20, 0x00004cde0f28039c, 0x00097bbe8620d5bb, 0x00038bb6c0b36c95, 0x0000c238d358e07a }}, + {{ 0x00073da0ba4fff56, 0x000bd02a70dab865, 0x00091a87ec422f67, 0x000e17592fcbd12a, 0x00003dc4b150156e }, { 0x00022cb8d1d292ad, 0x0004ffe0d41d5505, 0x0002cf29172c74c0, 0x000c83548e7eec0e, 0x00001f902993f981 }}, + {{ 0x0009a38d4ab89a62, 0x000342a70cbb96cb, 0x000a032d975e6da2, 0x00090838de034362, 0x000051f017afdd83 }, { 0x000aebcb837a69e7, 0x00043821ff140fd2, 0x000ff5c31c5d3793, 0x000bb3249bd162f2, 0x0000c09d95c15150 }}, + {{ 0x000f921c720023d1, 0x000e15f937b05271, 0x0007cbeb7d25606a, 0x000816929f42e1e4, 0x0000a8cc384e52b8 }, { 0x0006ea82172f000f, 0x0008f7b5b26bbd51, 0x000d247724699de2, 0x0001f0d7e7a8ef5e, 0x0000f1fe050d2c8b }}, + {{ 0x000f7199dc46d818, 0x000722b599ff0f94, 0x00073193628eba9f, 0x0009137f368a923d, 0x0000ee5360c30393 }, { 0x0004f710bd7921fc, 0x00093f6e46f2a79c, 0x000d25010260474b, 0x00092d27c08b5dea, 0x0000fab67c859c32 }}, + {{ 0x00064519deca0cdb, 0x000d55778bde41ac, 0x000fedd3517b736a, 0x000b97c416474bf7, 0x00003208530088be }, { 0x000025813a05a641, 0x000f7460edfcc4f9, 0x0006b8cc2931d960, 0x000ea5e77319aa73, 0x000029012049c8c4 }}, + {{ 0x000de5048b827884, 0x0004fcac0c227196, 0x00045be510935647, 0x00029229cc60209e, 0x00001a8f07ca2a57 }, { 0x000130388eeb5d29, 0x0000fd8e115ac04c, 0x000f5b3ade432bbc, 0x000316be77df7093, 0x000022d0dba6d494 }}, + {{ 0x00039aaf90e9d532, 0x0008c8477f54e6e0, 0x0002623b60542e7b, 0x000bc14d69f258a7, 0x00009ce5ba732f82 }, { 0x0000931ba5ed41d1, 0x000a9739460342f0, 0x000c3ee643e44bb0, 0x0003fd87e4703148, 0x00003a0863fbd5a9 }}, + {{ 0x00004525097c5017, 0x0005a9d786f337c0, 0x0002e6bf14745166, 0x0000c18efd9c01cd, 0x00002c8b553116d9 }, { 0x0000641053c931ba, 0x000c64784db8e765, 0x000a436499b28b73, 0x0005fda223bacf1a, 0x0000938a983f292e }}, + {{ 0x0007bde5834630c1, 0x00077749f7144aea, 0x000e9c16c77a33ff, 0x000ad12ef170c963, 0x00002afc875cc612 }, { 0x0007882dab2f3b77, 0x000638b6fa687508, 0x0001cfa5d49dd82e, 0x000de3631fa64e53, 0x000078a45e28f1f6 }}, + {{ 0x0009c5714d71771f, 0x00098316c67a7751, 0x000ad6005f642df7, 0x00040e7b2cb10471, 0x0000cc0be522bf70 }, { 0x000bc82ce45c7f2a, 0x000617bed03fd3f8, 0x000ea5f3ec6a645d, 0x000e3f6b5dd03344, 0x00007777234fc32f }}, + {{ 0x00081d837a740b62, 0x000d301b35f1341b, 0x000c6d86bac884d5, 0x000a3565ce53156d, 0x00002f8f9bd103d9 }, { 0x000c596b31a8eb5d, 0x000f76e2e4358674, 0x00049f4c722e721f, 0x000256e9d9020eb9, 0x000000946b032fd9 }}, + {{ 0x0004828817b2b97d, 0x0002c0f0e0b040d2, 0x0002217d0167ff6f, 0x00040e422baf02d8, 0x00006eb8e36ece4e }, { 0x0001f86203c5e993, 0x0008a31113ec3567, 0x000da3f78ff4f368, 0x000ea84159e48861, 0x0000bb7e93050f1a }}, + {{ 0x000560b0e9aafe66, 0x00042da3fd19408b, 0x000b722cecb04b0f, 0x000c6b6904aa716d, 0x0000052ee2e70dfd }, { 0x00015ca19443e93c, 0x0008d92fa34bdad6, 0x00043b7857295ea6, 0x000c094d3f024fa1, 0x0000ea3f81cb62df }}, + {{ 0x000d3a74d028189e, 0x000aa3d6f73c51e3, 0x0002acd34e0d4511, 0x000a079ffa4cb241, 0x00006e3b446570f6 }, { 0x0008027c148bf575, 0x000240f4a7a92024, 0x00095faee0289dbe, 0x000ea1d08a9bccf5, 0x00003daa96c5c819 }}, + {{ 0x00079738d8eab83c, 0x000fb7c40b557f08, 0x000645d10d87e8cc, 0x0005ffe27e001c36, 0x0000017e39f092f7 }, { 0x000943d1b495d0bb, 0x0000c4bbda948826, 0x00083d40e0e510b7, 0x0006a7c24d1bbcc4, 0x0000acdbfbfa5df0 }}, + {{ 0x000bf3dd51c8d19c, 0x00007bb098cafb4c, 0x000f94e0af5e1773, 0x000601c4d27ed230, 0x0000285109152371 }, { 0x00024c7557bb1196, 0x000891641b1d830b, 0x000d18bfa8cf69f1, 0x0009cdfdba7ec851, 0x000077b4e1b088fa }}, + {{ 0x00040f8ac3df8d0b, 0x00006dc1d4636c38, 0x0005e40b0e75786b, 0x000f4a2b3f843652, 0x0000977ae94715bc }, { 0x000123d8f150f27a, 0x0001e11f4387f1af, 0x0002c3f18f8f2228, 0x000f40e677c501cd, 0x0000110598eb168e }}, + {{ 0x000796ff08cfa1b0, 0x0005e71d607039eb, 0x000515a3463d5019, 0x0009090912b70e21, 0x0000a34adda11483 }, { 0x00038e8cdf8fccad, 0x000cd863c57c4aa3, 0x000f84360d47a30b, 0x0009660982aed9ca, 0x000082fbf17ecca1 }}, + {{ 0x0005213908309c96, 0x000cb8266cde8dbf, 0x000f3851c42e5054, 0x000d1ba724e15997, 0x0000c913344e336a }, { 0x000a1c0c559056b7, 0x000681d8ce4396ea, 0x000ffdd114ecbcec, 0x000c3f86b37a1a67, 0x0000bba57201413f }}, + {{ 0x000e3d7a82a07312, 0x000248b4ed80940f, 0x00057c32545a8fa1, 0x00024459f67e6d05, 0x0000781e5623c72a }, { 0x0009422f1dd9d9ed, 0x00045027e096ae27, 0x0006ab7164488446, 0x00002f2fcb1f3e1e, 0x0000a08771e4d556 }}, + }, + { + /* digit=24 (1,2,..,64)*(2^{168})*G */ + {{ 0x000b1adfbccd5f72, 0x000bafb9db3b3818, 0x000e49c42a8e58da, 0x000a5741f9c3a2de, 0x0000e1b4d1992caf }, { 0x000d2ae779d25bd8, 0x00001397e053a1bd, 0x000689b00f8d9c66, 0x000aeefabee2be5c, 0x0000ed75eb0e9aae }}, + {{ 0x00070ef12df3aecb, 0x0000e7a205b9d8b0, 0x0003fe5865a61087, 0x00049560e6eb8f06, 0x000018c288645dc3 }, { 0x000c1f205200dec5, 0x000d0053bcc876ae, 0x0007bb212c914ca5, 0x000c3165e12a7533, 0x0000fee6eaee8fb7 }}, + {{ 0x000b625175d3e131, 0x000ba79b6828f364, 0x0007b65b0a28d9d9, 0x000a31a0c9d7a025, 0x00003f761efd974a }, { 0x000cea06f50c8e7a, 0x00025dd9669b6210, 0x0006ea0e74a30782, 0x0007f7cbc88a2ca5, 0x0000eefdd32a930a }}, + {{ 0x0006927bdc72fcef, 0x00092b5c4e83d33c, 0x0008986accaed0f0, 0x000ee5e0fd9f3587, 0x00006fc2b4d5332a }, { 0x000bd4c284a559fb, 0x00092f79f9e0f036, 0x000e91031f24a068, 0x000494df12868661, 0x000064b67a214c5a }}, + {{ 0x00038062d4c1e75b, 0x0004591289a8619a, 0x000fc2f14e9e6431, 0x000a96b32ef796e5, 0x0000cf84b53f10ce }, { 0x000e2d93f2a93799, 0x000b1200573274eb, 0x0003eaf97fa1c33c, 0x000a47520d07b67e, 0x000099241c28bfc2 }}, + {{ 0x000e16a8fa9459c5, 0x00069533f36d1411, 0x00042fe5fb485de4, 0x000223d3ae84bb3d, 0x0000362e47c092d7 }, { 0x00051ac53cf453e0, 0x00072adddd472e03, 0x0006d8041bea2700, 0x0004e95997d405ee, 0x000072103589e10b }}, + {{ 0x000c45b260b78e4a, 0x000cab84444896b8, 0x000f6cfa759ac76b, 0x000f5fe7d64974d2, 0x0000fc1b25688826 }, { 0x00018f2e67924f42, 0x00079c2d84634875, 0x0006ee2d190516ad, 0x000a501c0d1b2b3f, 0x0000036290195036 }}, + {{ 0x000578dc4ce14fbe, 0x000b08c06d75fc1f, 0x00063cd0cc5274b3, 0x000f629dd2dcf7bb, 0x0000f36db3fef100 }, { 0x000304c0d907ee38, 0x0005103df2ce7a06, 0x00083934eed34414, 0x00075ccabaee3628, 0x0000f1816df3580c }}, + {{ 0x00082f4b52e4cec2, 0x0001aa2a91f6791e, 0x000fb37b53a9c983, 0x000cdb8c12abe418, 0x00008cd67259a170 }, { 0x0009b47fddf3ac31, 0x000ff274073e81c2, 0x000cd201171eb3c4, 0x000d8c60cc0276fa, 0x0000bde77950ebb2 }}, + {{ 0x000db1d445e12d1b, 0x000c603459b19402, 0x00092d4f6a4d5460, 0x000260fd3ec881c5, 0x00002404d2934d68 }, { 0x0000c0e5d7b17ee8, 0x0007387df239fd84, 0x0004d45c3f1714c5, 0x000bd7a59c718af7, 0x0000cf1f5fc6cd42 }}, + {{ 0x0007e104334dd14b, 0x00045f4721b33f2e, 0x000b16fe00b71c74, 0x00074a4e72124f1f, 0x00000c5ed583ab49 }, { 0x000b6a0a0c1b3b39, 0x000698ee4d0c7e6c, 0x0009bbbb310fa8c1, 0x00008ad43847e339, 0x0000d0eb2a823c1c }}, + {{ 0x000c83d2104da3af, 0x00087c327d02dccc, 0x0001ade4eab2adc4, 0x000bae22d5ad0098, 0x00000d9c44913bb2 }, { 0x0005934268273a75, 0x00009c1666fba0ca, 0x00066ee203cd0f18, 0x0003334c26994819, 0x00001a7e83e4402e }}, + {{ 0x0001240501d5f60c, 0x00008eda714181da, 0x000abf43ae40161c, 0x0004fd31f609ac13, 0x00006d341d5395cf }, { 0x0002c2adc36656d7, 0x00035231409ca976, 0x0007bda0de8008a3, 0x000a46fcd254cc1a, 0x0000ead8778cec76 }}, + {{ 0x00082b49f9e98b30, 0x0006df056e464833, 0x00002b48195cdcf0, 0x000c68c52a55abb2, 0x0000455cf3a69edb }, { 0x0004c589d9fde490, 0x00091ad4dff8d47e, 0x00045eb86afca0dd, 0x00060d50979caf61, 0x0000ed7832e28abd }}, + {{ 0x000716dd735d4299, 0x0000fab3fd40e095, 0x000e201a6135ca74, 0x00012ff8be455842, 0x00006f917d8d6ec2 }, { 0x0007e76496178dbd, 0x00054e54e8bf3de2, 0x0005f39e59a54bae, 0x000840fa69a0e77b, 0x0000a545f75f92f1 }}, + {{ 0x00065ca49bf70465, 0x0002633070d3aab6, 0x000b33d03149eda8, 0x000f82c732643672, 0x0000dae397b7ff25 }, { 0x0005986e7c2b0613, 0x000c759b3efb9983, 0x000ccf96a4c52f87, 0x000f392308a5b922, 0x000053a40c602f11 }}, + {{ 0x00060575d954191d, 0x0004e5cff3513cc7, 0x0009bb938203e64a, 0x000d85286bb0cf8b, 0x00009896e7ac48ed }, { 0x00058c95fcf57592, 0x000bc169f7aa1811, 0x000fdf7d571f4181, 0x000cb1291163a3ec, 0x0000a74ad3d22246 }}, + {{ 0x000d52e6bccfbd9d, 0x0007e6e13cda46f1, 0x000516dcc813d3db, 0x000adbc707948241, 0x00009aa625122196 }, { 0x000fc7bf7e178d47, 0x0005ae78608dd3d1, 0x000aca3fa6085efe, 0x0004ed3fa4930db2, 0x000095d11cde96d0 }}, + {{ 0x000d1764efa19423, 0x00089cdd9eee96cd, 0x000b08758747c9b3, 0x0006ea12a0ca277f, 0x00003fd5bca22445 }, { 0x000cff95c47d1a9d, 0x000dfde7b3ed7397, 0x0008dfea4bfb8703, 0x0000b151250cb745, 0x0000e035718c4eac }}, + {{ 0x0003e982abb759a4, 0x000a8bd8088b454e, 0x0006b6b489deac94, 0x000caa3cbdb7f32f, 0x00004686e7f56cb3 }, { 0x000bbf1d3400c329, 0x00098e4c2a2a5938, 0x00047bb4d51b0609, 0x000c26b2372c3686, 0x0000c3163f597394 }}, + {{ 0x0009e8e77eb6f0ef, 0x0001a73da50c991b, 0x000ac8448763bb7e, 0x0000c887148afcd5, 0x00001fe304790c6a }, { 0x0005b82406b0e023, 0x000ec4499a7703de, 0x000d7bd612ecce7d, 0x000a058be6972930, 0x000055f6a476b172 }}, + {{ 0x000a9e252eded6d3, 0x000b8e3bd3790620, 0x000d23302563dbb4, 0x000be5b37d64b7a1, 0x0000f8d8432196d3 }, { 0x00008e4790ff828a, 0x0000b6bc39d3bcbd, 0x000fe29ce5d2ddc1, 0x000bef0181a31c9f, 0x000059056576275c }}, + {{ 0x000e5d55febb0e31, 0x000e9e9255c09801, 0x00010ced1da0ddbc, 0x0003c14a14bd8638, 0x00002df70d0ce263 }, { 0x000e5bf8ecb0386b, 0x000b0068ff5cc292, 0x000ee05cac07c42d, 0x0004c2124026b389, 0x00009cd54793b7a8 }}, + {{ 0x000c6360bfa4642b, 0x0005cd0ad886aac2, 0x000a8b0e55855f5d, 0x000dc1aab5c16878, 0x0000ad287ec6b022 }, { 0x000e29d50b1fec3f, 0x0001cb5b9c972a3d, 0x000d9e7098af043d, 0x000bb5ba6c9e6f88, 0x0000ccd2c2291972 }}, + {{ 0x000d8f2c9dc84b95, 0x0005f83d309cbe6a, 0x000a332dc0b1d6bf, 0x000348ee35b020d8, 0x0000d07b2b00cfeb }, { 0x000c5bec9951a99c, 0x000921de7400e938, 0x0008c66271b3c77f, 0x0005ce4bffe3e18a, 0x0000fcae372683dc }}, + {{ 0x0009779f4565f3cd, 0x000992a74a55d1f0, 0x000efaf7c792c505, 0x000e87a94e3fbf28, 0x00001e0023586f6e }, { 0x000da67cfc184107, 0x000b6afc3d19d8d3, 0x00041a3029e3c052, 0x000ed5b55d5a7509, 0x0000fb679388fd7c }}, + {{ 0x0007c48aae979814, 0x000e22a2c160dcd0, 0x000387cc6040b48b, 0x0000e495a34e6c5e, 0x0000ebc559de6b12 }, { 0x0000871f6c9b95e8, 0x000c6da50068527a, 0x00027c0973fe1c51, 0x000e34a4c2c09ef2, 0x0000a42cfbc74a4c }}, + {{ 0x0006897ae28b8aa6, 0x0005568d84d835d2, 0x00095f9447042666, 0x0000a52d7c90caf3, 0x00007477e6db63ea }, { 0x0002c7980e3a62d5, 0x000b508a4d6755dd, 0x000b36ca63cc8293, 0x0005c42e8403ee41, 0x000072dad9713001 }}, + {{ 0x000f22110209bd6a, 0x000f832d2e11305f, 0x0001f49e696d2947, 0x00057e69d966be49, 0x0000a57523d1fdb1 }, { 0x000e0267ea28eb43, 0x000134aabe30129b, 0x000f756bfce543b4, 0x000213d22c6f2c93, 0x0000707ad02d7862 }}, + {{ 0x000a76e9e4386d11, 0x00081f193bc042b7, 0x000437c0da73ea74, 0x000bf3068f085b53, 0x0000015ec7b9d94a }, { 0x0001889c8d4274a0, 0x000e2fab88911b55, 0x0004635272281033, 0x0003323ffc85345e, 0x0000f694aa06193f }}, + {{ 0x000043f2c0486dd3, 0x0004069a5e829e5c, 0x000b55c3b7815495, 0x000afb1c1cbb4c6f, 0x0000adfdd639e5db }, { 0x000d515f74e0a7f0, 0x0009f758ea5e1853, 0x00099b2b5e2aa1fb, 0x000e7d6018bda40b, 0x00002841bc77e94c }}, + {{ 0x00003172599604ef, 0x000e04fda79e5acb, 0x0009d5feaf05bd45, 0x00080866e68b83b3, 0x00000b424807d53a }, { 0x0005296e9538c34c, 0x000381ac5ccc2c46, 0x000ad873e1d42e72, 0x0005408bd7d7dc96, 0x00006a74e1c17bc4 }}, + {{ 0x0001fdfedce79aef, 0x000c3d0bc8fa5bc5, 0x0006fe0e289f0f1a, 0x000059b5d8a5ead6, 0x000023b6e31609f8 }, { 0x000d66af2147f6ba, 0x000890b289fca32e, 0x000eb0352d1e23d2, 0x000f354dee36ff0c, 0x00005f8a7192aa10 }}, + {{ 0x00010989c077ae7b, 0x0001b8b5fa9f4d67, 0x000dc699d9268932, 0x000596c722d4066c, 0x000090b03f888845 }, { 0x0004d874f8f53831, 0x000ae35ce40cbd89, 0x000f826ab444e0a6, 0x0006a1d2cab55c5e, 0x0000acd6cecd4b95 }}, + {{ 0x0004c0e1b8f9d216, 0x000dae8a518001b4, 0x000658440d56a993, 0x000bb3725ee605eb, 0x00002b0f4a4a14c0 }, { 0x000bf395cc3b7ed0, 0x00054e88b6740dc8, 0x00036229b9409031, 0x00046137289087a5, 0x0000be581ada8fe4 }}, + {{ 0x000d51afe901e219, 0x000b0a1deb8568b6, 0x000013a98d491c8f, 0x00016e281a35daea, 0x0000537d475b7d2b }, { 0x000c6865cafe90b4, 0x000f86e51803a198, 0x000ef9b92ab5832d, 0x000de923ce3d24b7, 0x0000b2e4a54bd2e0 }}, + {{ 0x000c70ca45125561, 0x0002a7db743f4ef2, 0x000bbb6053892073, 0x0009d8ae83793909, 0x0000dbb97d8fdd1d }, { 0x0000384cb6274347, 0x0005229cb1131b10, 0x0008255041c3b80a, 0x000952d0a7826f29, 0x00004174450b76ca }}, + {{ 0x0009570c16f8f650, 0x000c25b3ab480953, 0x0006297944efd26d, 0x0009f08453451c25, 0x00004a96dc9b66ab }, { 0x000ad4b67319678a, 0x00020d3adecf263d, 0x000101a22a8fbe78, 0x0005f1e36f1c3d8d, 0x0000ed9d0a1de53b }}, + {{ 0x000d2db93bcd3273, 0x000260ff0d796a03, 0x0008a1e0d1c6f14c, 0x000d98dad5fc6247, 0x00004b1ab345b835 }, { 0x000ea6e007bc5637, 0x0005379d85d5ddc0, 0x00075d099f2d2836, 0x0004877d5e907caa, 0x00002c068a8b32e7 }}, + {{ 0x000651f5d8d828b1, 0x0004069d23233f3a, 0x0005d1796da707e5, 0x000781e6c3e32279, 0x0000b55160373545 }, { 0x0008bdd37cafc9ac, 0x00003191a230c767, 0x000254ec8ace2d98, 0x000e2217ebd52727, 0x00005901810eba20 }}, + {{ 0x000491ccaacfb314, 0x00002c217cd93fe1, 0x000dc64b1a37f286, 0x0009b2ef865b97a4, 0x0000b59b701c4047 }, { 0x000da2b7ef8c43e7, 0x0009b80231963324, 0x00070ad88faa8816, 0x00098e9ed3c3862d, 0x00000eddebb7b375 }}, + {{ 0x000ee5cf3c1ad26f, 0x0003f33b126e48e4, 0x0002a5af12fbde15, 0x000885bdb404a0b3, 0x00001567d76438aa }, { 0x00055c5bfa632ed7, 0x0005cb97df3c8c32, 0x0003920bad02d03c, 0x00016ea6abd905fe, 0x00005ccd15e48747 }}, + {{ 0x0004610714435ee0, 0x0007cbfc3dd9b4d7, 0x000f87a583732e37, 0x000fc229bce23edb, 0x0000d1868b201130 }, { 0x000f1bc7d1d5a193, 0x000645f1266bf52e, 0x000129c8d3e49b75, 0x000cd86e30204672, 0x000000c5c9aacd67 }}, + {{ 0x000c3fefca460bc8, 0x0005fc17078c4322, 0x000c8b3ffded87cf, 0x0001d3d70e1961b9, 0x00005038a807927c }, { 0x000eb77640143e4f, 0x000bb36c2af389a8, 0x000d2eefa6aabb49, 0x0006c356396c610f, 0x00004a75c050691e }}, + {{ 0x0004778ffba62fa6, 0x0007e542a296e775, 0x0002ee0bea399904, 0x000da2131c871868, 0x0000d6cf7f2694c3 }, { 0x000c13e651883a51, 0x0006eecaf06defd3, 0x000f497ed0b9e2f9, 0x000a01014fe3c105, 0x000029455e6852d9 }}, + {{ 0x000dffbb5f63e5d5, 0x000d329855938e83, 0x000e5fe590137f7d, 0x000e725b92eeb5d9, 0x000084e49f54aebc }, { 0x0005d787c5f79f6e, 0x000f0431acc093d5, 0x000ce38d469b87c5, 0x0004a46600cb6cc3, 0x0000f30c7a69dbee }}, + {{ 0x000a17ef8e44cede, 0x0001f47deadbf02f, 0x0008b22da0da0860, 0x0002433f011e8021, 0x0000e0cbc364982d }, { 0x000764196f27ef16, 0x000e72593e43bff7, 0x000e947fecc38758, 0x000f707a35cb5ff3, 0x00007e04f36d9766 }}, + {{ 0x000cfc53b4244564, 0x0004fedf0290a8fd, 0x0005ee4b6fd35ed5, 0x00009974fcae8196, 0x0000c1f220ef19f4 }, { 0x000bedf5f9d4ef28, 0x000ca3c4cb632f9e, 0x0005cd318a6d91a9, 0x00094f00ac42a1ad, 0x0000689a17da238d }}, + {{ 0x000ea61e2a63d186, 0x000c69f97e3e0442, 0x000fe32a819f8e55, 0x000bce8234201851, 0x0000668bd1c6ed6d }, { 0x000868c740040cde, 0x000b3bc575eaa945, 0x000160968aed23c5, 0x000e36f6f9e7bc72, 0x0000880b167f7fa3 }}, + {{ 0x000c19e283331498, 0x0002adb7be0eead0, 0x0002f357aeb26c71, 0x000d5f8d7511e784, 0x0000b48101676692 }, { 0x000621b037cb2a49, 0x000a7cfe9206c550, 0x00042f321032bfce, 0x00035370489389cb, 0x000054fee486602b }}, + {{ 0x00049dfb6757e9ab, 0x0003b3596e78cc4c, 0x00014768b940c469, 0x000fed2035385c8b, 0x0000146b998f7458 }, { 0x000d882de450b14e, 0x000937059f707727, 0x0003bc9587871fc0, 0x000d9a3903a073a8, 0x0000d3d672691eea }}, + {{ 0x000882984a6412d2, 0x0006d0ab2cf89408, 0x0007e711f7072d50, 0x0001381c872b0b17, 0x00004f08b915e614 }, { 0x00012be0ff5c5e45, 0x000180c6a19d2bbd, 0x000bb5cba1cc5bab, 0x000322b644b9e7f0, 0x00003b06299d88d1 }}, + {{ 0x000246e8af14e4f9, 0x000d93d0e165dc33, 0x00021bdc901f20a9, 0x000d3a7873d47ce6, 0x00006860c9c10ae9 }, { 0x000c598bba00d6aa, 0x000a4c703ce2b4df, 0x000abf66ec1544c8, 0x000c296c1e9b4ca1, 0x00006a2a90d3380d }}, + {{ 0x000a1105556dd7cb, 0x000fb2b45036099d, 0x00013c6b6083296c, 0x000e64e43a9dee74, 0x000078a017dad06c }, { 0x0003baaf9a80f826, 0x0007f2c651a3be27, 0x000d14899a0c2d08, 0x0003b98d1cda5111, 0x00004e13797d5a7e }}, + {{ 0x00000cab02e61927, 0x000e59d7fc63668b, 0x000c5bf6f1db6ce1, 0x00098fc516886c9d, 0x000015963851966f }, { 0x000d956d9d123a74, 0x000a2bc81b69442a, 0x000007c6cf517a7b, 0x000612e784148670, 0x0000acd5ae38e4e2 }}, + {{ 0x000d774e4ee4ff1f, 0x000d38b638a7e819, 0x000ebc098f74b062, 0x00070057a54155cd, 0x00004efa631b47ed }, { 0x0006843cb3f5f8f6, 0x000782e078637d27, 0x000ce786a3b624a5, 0x000d6ad541c363e2, 0x000040f408018af3 }}, + {{ 0x0004197ed9721891, 0x000d491d4b66947f, 0x0007102c8bb348b1, 0x0005dfe55b452ce9, 0x0000b7e62fe46cd1 }, { 0x000ada31f6ad76d5, 0x000df72400dd0f70, 0x000f40443e9cb7aa, 0x0008c63ae59f5ca8, 0x0000bd128c94245a }}, + {{ 0x000c2a3869098900, 0x0004eb694a2c2152, 0x000ad97b424b1af0, 0x000e689fed83a43a, 0x00002b03a91bd120 }, { 0x0001ce6e973b5ec6, 0x00073408439d9b28, 0x000188feb07a97d8, 0x000880619715ea67, 0x0000f2fc52299eac }}, + {{ 0x000af556ee5923e0, 0x000ffe0d809c7bd0, 0x000c841af9bcabc1, 0x00080376fd7b2580, 0x00006abbc0368a7f }, { 0x000e61844f2bc503, 0x0008873921de0acb, 0x0001c045ffb0ea65, 0x000091e7452ac9c2, 0x0000c85b8dcd9bc4 }}, + {{ 0x000b192421f0e56b, 0x00086bc789a283b1, 0x000d6cef12d213ac, 0x000ffd52d72d3ac3, 0x00005f7fe8a58197 }, { 0x000a3bc86647bf41, 0x00058989dc84cacd, 0x000cc232ad52144c, 0x00059dbe2b0a8482, 0x000095d11cda69f2 }}, + {{ 0x0007b4d51ba3adb8, 0x0004e53056e6f8fb, 0x0000ad1c5e8c26eb, 0x000bd064084861c4, 0x00001e7739155ad0 }, { 0x0000518677787161, 0x000d111220a255d5, 0x000bb1f017f68d42, 0x0009aceeac997152, 0x0000559f1f223eea }}, + {{ 0x00043d74bc914746, 0x00072bacf902b0bb, 0x0004839dbb7ea13c, 0x00008cbbc8d4b4eb, 0x000057a993c570e4 }, { 0x000004512228d85d, 0x0000ec98adba3503, 0x000c8517c8053c9c, 0x000d515ec364eaf8, 0x0000420c9b7173e7 }}, + {{ 0x0008a61cf4dc1641, 0x00058b7ea7b979b7, 0x00046e551ed89117, 0x000b9bafecd78cd7, 0x0000ea75547ea9aa }, { 0x00033effd03f2de0, 0x000e8723d502295f, 0x0003524db2b8913d, 0x00001a586f137685, 0x00006e05ca06d0f5 }}, + {{ 0x000fd495f78c275c, 0x00068cdb30cfb3a6, 0x0000fcc91ed14bb9, 0x000a17ddf6d09b8c, 0x0000645d0ce04a7b }, { 0x000dc229b0415b16, 0x000009f275264daf, 0x0005e7bb59b2b9b3, 0x000525c2280c2b74, 0x00002b3172744708 }}, + }, + { + /* digit=25 (1,2,..,64)*(2^{175})*G */ + {{ 0x000fc20755e3176b, 0x000828452666a58c, 0x000016e7bc6ccdba, 0x00078f9084bcb6e0, 0x0000554faca4c643 }, { 0x00094e142cf0b0d6, 0x00046505294dba30, 0x0006822fbef1afda, 0x0006df474a30ba28, 0x0000e6be6e6ae1a8 }}, + {{ 0x000b296904664fc3, 0x000b53e979f39254, 0x000a642320a351cd, 0x000cc34fa1efd130, 0x0000a26d827b4096 }, { 0x000df088ada01cc1, 0x000d534b9db65b69, 0x0001656914dd4d71, 0x00018a2f335c82e3, 0x000058dcd3dda1a0 }}, + {{ 0x00036f4fb314c50d, 0x000a366bb3fd30a1, 0x000adc80e59bd10a, 0x0005fd66c429169d, 0x0000d081f2b59f20 }, { 0x0002bf6af341698f, 0x00020011fbba712e, 0x00036dac36ae06bd, 0x00045183e6ef0f0b, 0x000041383caf3e36 }}, + {{ 0x00008fc59cde3468, 0x000e7124237b64b8, 0x0009635d376aaed3, 0x0004ff5d8688ebe9, 0x0000c1b55d497018 }, { 0x000b143a98f532c8, 0x000632ba3585862b, 0x000532de58edb3d0, 0x000ebfa9bb66825d, 0x000060efc436424d }}, + {{ 0x000ae5a380deb2a8, 0x000ecce12ae381b9, 0x000bebe7370f573b, 0x0005442ca3a7f176, 0x0000600a0769fd67 }, { 0x000d65f035454aa2, 0x000eb15ed251b464, 0x00007d646e2f56a1, 0x000a76d071f5d6d3, 0x0000bb2c5fed005a }}, + {{ 0x000f8f66b8a7cdf7, 0x000c324f111661eb, 0x0003f176844bf6c5, 0x00056be78edced48, 0x0000217f1b2be94c }, { 0x000ec85fbc8973e6, 0x00067f4d7ed8216f, 0x00068f645f12fdb9, 0x000735154bf07f37, 0x0000c3cd2d5edd0f }}, + {{ 0x0002b052011404ae, 0x0005feea5ba5b0b7, 0x000dd02000988637, 0x000ab7f42e6a116c, 0x0000b74febd1fdd5 }, { 0x000b000c06b573c2, 0x000ad7d31a3b4ea6, 0x000820bd8f2c06dc, 0x0003d6897e589307, 0x000078133015312f }}, + {{ 0x000f53916fc6beae, 0x0001122765fa7d04, 0x000fce16c2004cf7, 0x000c22d9859805be, 0x000052df10ff2d7b }, { 0x00099f450e1f9830, 0x000c61f33ddf6269, 0x000e06e68bf551b7, 0x000e86ee34206238, 0x00002aa249bfa9c5 }}, + {{ 0x0004c88f374f6f88, 0x000d851e4c79e9bf, 0x0002d1a210b3161a, 0x0004ef3231394cb7, 0x000034829a179e77 }, { 0x00060e7f8aef7dc8, 0x000aa1f4aaa499e0, 0x0005ddecaae02032, 0x00014bd1d68b3a7d, 0x00009e90f751f24b }}, + {{ 0x00047aa4c7ac2b3c, 0x00082731a3a93bc2, 0x00057da03752d6c3, 0x000d7d2fd42bbf46, 0x00008348bc06dd42 }, { 0x000825653cf027b6, 0x0001032f60c77da5, 0x000a69fded019b66, 0x0001dc3cf6ffbc26, 0x0000333cf94fd444 }}, + {{ 0x000648487f9f745e, 0x000ab7ebc1aa2b92, 0x00031157eabb1368, 0x0002c7d380788939, 0x000092962173f198 }, { 0x00006a7a345f44e5, 0x0002e363fcef2614, 0x000cce4176f57b8b, 0x000e286e26239c81, 0x000007adc1ccaa3d }}, + {{ 0x000ddab96e59ccf0, 0x00077ba4a493b31c, 0x0001ce1d36b7f226, 0x000c8a14b54f20a5, 0x00001a428916f43d }, { 0x00047a670dede59a, 0x0002d0aed25ea138, 0x000afd5154d9620d, 0x0009faac5fa1d09e, 0x000024f267bf7958 }}, + {{ 0x00006f4098b6440f, 0x000331e64bcb626e, 0x0007466fa01d3aef, 0x000cc256caea638c, 0x00006bacbc3672fc }, { 0x000063d8a3c3a04d, 0x000c7527512a30b9, 0x000d5525ccf645f5, 0x000307480f3d867f, 0x00004f61af99ab8f }}, + {{ 0x00017e76ea8b4aec, 0x000d995d314e7bc2, 0x000b24fa2303625b, 0x000d3e9d32ee7464, 0x0000fd86bbb7517e }, { 0x000a5817e3ed6c66, 0x000681fe7cda917f, 0x000a903127323a60, 0x0005ae96b12d8016, 0x000044afeb2ca0b2 }}, + {{ 0x00098ad9f490f5c1, 0x00080640821ee4c9, 0x0003723b3f9d8e3f, 0x0007cc69f1583eab, 0x0000a66bfbb2018c }, { 0x000ee4ed6e631317, 0x0000250a41157f70, 0x0006d9f54d0fbb05, 0x00013a25fa9d7e1e, 0x00007eea65ae7d21 }}, + {{ 0x000b69cbf65aaebc, 0x00090c5ac751e7b7, 0x000281d845c75cee, 0x0002f93c693f9647, 0x00005a6bb7c3dc6e }, { 0x000be7048732b0cf, 0x0003da2f4bf94e19, 0x00085541905b0af8, 0x00067070b1a5325f, 0x0000e3a10d49e546 }}, + {{ 0x000c686e55c3531d, 0x00063bfe1dc5c055, 0x000a92567d01f185, 0x000bcc9a3d738add, 0x0000afd5ebc87e1b }, { 0x0008a947fad2853c, 0x000cf50dfd0b7ec5, 0x000b11a03426ca2b, 0x000cf3096bdd0264, 0x00002575e52919b9 }}, + {{ 0x000fab1ac5f1c3a5, 0x000e84e49ccd6d74, 0x000c39123abacef0, 0x00010ceb2ff8fb02, 0x0000b9696ac02316 }, { 0x000d8260aa363274, 0x0002fbc97576ec06, 0x00079dfc1c84ccdd, 0x000fde200e8c946e, 0x00005e0955dd2239 }}, + {{ 0x000fba35ff441c20, 0x000104f70bba85cb, 0x0000e4e69d11bf4c, 0x000f9cec1ad090da, 0x000045f5d2d270df }, { 0x0002160efb05643b, 0x000c69ac1e919380, 0x000318e632645d14, 0x00055d30fa47638f, 0x0000433a1488c0b7 }}, + {{ 0x000cd99e61699f04, 0x0000c73615cd9e45, 0x0007944a9da57d26, 0x000dc398c498ec04, 0x0000524b1feda847 }, { 0x000ec9746fc9036f, 0x000a3df7851dafeb, 0x0009e8c6ed72b8df, 0x00008ea0ef156f5b, 0x0000108b102601bf }}, + {{ 0x00012050d9a73af9, 0x000b0c1c8ed1f7a4, 0x000566fd0f18765d, 0x00065804a09e43bb, 0x0000103c0ab78e66 }, { 0x0003868fbbf33bcb, 0x000b14e90747e405, 0x000b62a3a6cab563, 0x0005268536dc05eb, 0x0000757f03238b3d }}, + {{ 0x00080f9fb1b49db5, 0x0001dc3cfdcf7dd0, 0x00036a302b02766c, 0x00057c7a341a9fce, 0x00000720671b4553 }, { 0x000320fd327ff00b, 0x00083849ba7a5317, 0x00067a79174cfc18, 0x00058b83607d3558, 0x0000ab6cf754792f }}, + {{ 0x000b96d16da0f72c, 0x000db98b18bd1ff8, 0x0000244657980980, 0x0004b3bcd52c2e30, 0x0000cd07defe68a2 }, { 0x00019448d72e9986, 0x0008a494b45706bd, 0x000da2a8dd214d19, 0x000597a76e7f58b8, 0x00003146bb218459 }}, + {{ 0x000769788c7ccd96, 0x000f6b9d3210a041, 0x0003c391d9dca1fe, 0x000f5b29f6bc7d46, 0x0000a7a3bdce7dea }, { 0x000971da0e302072, 0x000462dbd6cb838c, 0x000559fb85145a33, 0x000e7e16e7066d6e, 0x0000166a58b45338 }}, + {{ 0x000941b2a22935bd, 0x000eb6bc49a2fac6, 0x000fc75134d7d87d, 0x00039f8563b8420b, 0x000028c1f97d5e54 }, { 0x00075bcd770bf80b, 0x00058080658ff2ac, 0x00026cc8ddbf0813, 0x00059d7998780436, 0x0000b046effada4b }}, + {{ 0x0002a6b802d70649, 0x000f7907f8f875d6, 0x000df41bb3bcbbe4, 0x0000a72e79c9d754, 0x0000ec89ec7b5eaa }, { 0x00019fe96bba031c, 0x00025aac6e7394b3, 0x00037bb805a4ed99, 0x000a3d80c445b6a8, 0x0000d094b5aa7fb4 }}, + {{ 0x0000920d0920b3ba, 0x0000c1f385f63cb6, 0x0003c22bc41dea5c, 0x0003588c241250c8, 0x00006627f300e7e9 }, { 0x000a9bfe7766cbd7, 0x0008046724d81a52, 0x0002311e3421bbd3, 0x000755932b7ffb89, 0x0000c38dbc2a2a77 }}, + {{ 0x000c94a8b3fecb0b, 0x000d7393191f4fd2, 0x0006a74204edcd02, 0x000ab8b0643093e1, 0x00000fea7bdc3f59 }, { 0x000ac3f5f4222e4b, 0x0006ec05c437d8e4, 0x00038e15fdd446f4, 0x0004245a23d4ae42, 0x0000b1ce9686cd0c }}, + {{ 0x00048ef841ec905b, 0x000adfc6b6af9917, 0x000fa6cdf37b4794, 0x000b3b2d70d5270f, 0x00003f3048a38ef0 }, { 0x0002c56f83b0eb6a, 0x000940edfc6fafb1, 0x0005e46a9fa3f7f0, 0x000cee4c9eadfdf0, 0x00001b1b2c4fdc4d }}, + {{ 0x00000f6ee71a6cd6, 0x0006c03d39b8c34b, 0x000581167ef45d8b, 0x00076c7c5ed8b1be, 0x000070c61c8b5adc }, { 0x000c324341fd93fc, 0x00072cf57d0ea992, 0x0007ebae2901bcb1, 0x000c74faadcae0f5, 0x0000ffd2c5f6a211 }}, + {{ 0x000d8607e66825bc, 0x000c355e126d4682, 0x000f48ee862317f1, 0x0008401c2ed325f1, 0x00004e9bb5b46467 }, { 0x0006b0cdc1954e42, 0x000fc276009d660b, 0x000dd65ff7df7908, 0x00041c7a03a525d2, 0x000009b10a289aa6 }}, + {{ 0x0002dfa557bfa495, 0x0000476e1337c262, 0x000db45dc38dd4f8, 0x00012d3d96faadeb, 0x0000c13ada75bd32 }, { 0x000c6ca5801dad5e, 0x0007c17be93c6d61, 0x000188985039af29, 0x000bbefa124866c6, 0x0000462261edbc6f }}, + {{ 0x000e8d3ac716b9cf, 0x000de8ce964021e7, 0x0009849e47ecab8b, 0x00035c998b83bffb, 0x00005192ceb4ff31 }, { 0x000b2b0564f91674, 0x000ac91bc2d124af, 0x000c0a865a8f2693, 0x00065fbb7bd54c3d, 0x0000ca36576246cf }}, + {{ 0x000570dfff87d7fe, 0x000c7575b45afb42, 0x0009ca47b12dc69f, 0x0001ee757e9ded64, 0x0000ffe130d9f5bb }, { 0x00098ca255a80b64, 0x000977888e084bd4, 0x0008a9f3435e8330, 0x0005936eea024b23, 0x00002da3669daeed }}, + {{ 0x0005af3c6e99cba1, 0x000c5eaaaa07e869, 0x000b5db14917a1fb, 0x000f6b8c80db6fac, 0x00000dbef989db23 }, { 0x0006542e486c2d07, 0x0008b8c02e30afbc, 0x000c3a365a70473d, 0x000f0979d9b5356a, 0x000015f1407ea962 }}, + {{ 0x000afe3247ac4682, 0x0009de87adfb6cc6, 0x00032d694ef9c6bf, 0x00039cf5acc826fd, 0x0000ca422fa86990 }, { 0x000a6778b884ff75, 0x00093b9b6ae8a751, 0x000d3067d9288acb, 0x0006244d6b8cf388, 0x0000c49ee251a7e4 }}, + {{ 0x0004f1a6a51a864f, 0x000aec1ecc4c7e38, 0x000c91e9c227bdba, 0x0005bc7aade9fe47, 0x0000637ef4323989 }, { 0x0003365b1415cdea, 0x00093725a7b162b9, 0x00091a91bc8f680e, 0x000e7a346c5cfda1, 0x00002d033d623c62 }}, + {{ 0x000f201022c136d0, 0x000022526a1fc900, 0x00023e50603a6d0e, 0x000d575f40d6d10d, 0x00009c8b622aee15 }, { 0x000394ff6b33c8ff, 0x00067c5658fde413, 0x000d4a9260b29b00, 0x0005a0ce7a8d2bc7, 0x000020c000e376cb }}, + {{ 0x000550bd2e68ff28, 0x0009d098132c9af4, 0x000af0cdbb3a6f91, 0x0002dedaaaac4b0e, 0x0000ab754349a57f }, { 0x000fb1b899561971, 0x000c08eea5399d41, 0x000a71b1d2e6e9c0, 0x00031302dd989bff, 0x0000dd9508dd374b }}, + {{ 0x000f01732de8a9ab, 0x000ce6944c44d6a8, 0x0009c8e6ee09ad42, 0x00004014491df144, 0x00002eeb080df28b }, { 0x0009476f970fa389, 0x00014253aa0a6332, 0x000ca08f6605dcb4, 0x0004bf71006de5e7, 0x000099a78f12c1a2 }}, + {{ 0x000aff2f226112af, 0x000c8c0d34c35659, 0x0001809fef7bd257, 0x000cdd102ef1105b, 0x0000c029fa506998 }, { 0x000b332e089dab46, 0x00009853a80e75d8, 0x000de08a787917ca, 0x000ecb77ade71853, 0x000002996859d6e3 }}, + {{ 0x0002ca07544326df, 0x000bce8b9c20cda9, 0x000543030543c2b2, 0x00092628389f7aad, 0x0000462f560eaa18 }, { 0x0004fdfd9c7882f6, 0x000e51174a1e0359, 0x00077eececf624e2, 0x0006205f202628cc, 0x000053d65edaa5d1 }}, + {{ 0x00013045bf671e99, 0x000c75463428c6ba, 0x0001ef83c3e5fed6, 0x0002715112f7205b, 0x000071bd670eb998 }, { 0x00099f3ec14c07f5, 0x0001fff03ef77fee, 0x000f16504d2e9402, 0x00092e922689cd59, 0x00006897e1a72bd2 }}, + {{ 0x000d2466e3ef3d22, 0x0005b91ea1b3c3ec, 0x0002eb5dd366b2c8, 0x0008d231e2d1a705, 0x000040cdd3ee863d }, { 0x00007c4b82673e6c, 0x000882fc8cb762d2, 0x000e5613d59cd383, 0x0009208507e312aa, 0x0000c2327bddf0de }}, + {{ 0x0005670fbf65ef80, 0x000a5c1375913eca, 0x0005c229726133e4, 0x000785bf694e45e9, 0x0000158117a0c867 }, { 0x00077fefaeecfab4, 0x000024a66f5b8fca, 0x0009d0effc797a1d, 0x000049552d5a56ae, 0x000049ee3eb9b0d4 }}, + {{ 0x00004a61164aac1e, 0x0000f7c98379d445, 0x000b00547b73dedc, 0x000c3c664d000bdf, 0x0000f6c15b16639b }, { 0x00067ecf4a4fa56c, 0x0004c71effea31ae, 0x000d0a434b0a070e, 0x000a1e7bf2d3de26, 0x0000cf8b029be733 }}, + {{ 0x000004ceed9ae923, 0x0004652151fe690f, 0x00038f260c71df2e, 0x000cd078d0351580, 0x0000d3995d9f2836 }, { 0x000008e04b1b6128, 0x000c27cf0386fac2, 0x0002b1f3c705ed36, 0x00018dcec3b62ff1, 0x0000371099c99400 }}, + {{ 0x0002a2c2241af6c3, 0x000518e41dbd7725, 0x0002cff16430b650, 0x000ce81f72b419f7, 0x00006d2ef331afce }, { 0x0006510652363465, 0x0008914634dfb02e, 0x00000e2763a12775, 0x0005ed5cc5f074ff, 0x000022c4d1e63b6d }}, + {{ 0x000fde089e1449b9, 0x000db82f1012e7a7, 0x000028c75e109fa1, 0x00076543b51013cc, 0x00008371ca42a533 }, { 0x000f1f5764038bdc, 0x00078fec731b4b3b, 0x0003dade48fdff7b, 0x0008e59bc12a136a, 0x000052c07ce95456 }}, + {{ 0x0000061987d3a8bf, 0x000e66da16d4b346, 0x00097c9c02c81980, 0x000e525867fba9d0, 0x000068da7a517899 }, { 0x000d7e28755b891e, 0x000f1869da28ff0c, 0x0007764631272fb6, 0x0009fad0d3133f70, 0x000092271cbf4392 }}, + {{ 0x0008dc98e16ac555, 0x0009bf272ed91fcd, 0x0000c3f80a91efc0, 0x0005fd03c85214f3, 0x0000571a08a1dcae }, { 0x000cbff4f341eeda, 0x000d5fe585cc3a25, 0x000d169c04872f31, 0x000c46b81724952e, 0x0000703ab0cce8fc }}, + {{ 0x000126332eeca533, 0x000d70e87d8cc7b9, 0x000360ac4da1a08c, 0x0004994049beaa7f, 0x000085d58a9542af }, { 0x0006092eed4373ee, 0x000d3b48ed3bd6db, 0x0002233cbc36be87, 0x00016f45535bb2c1, 0x00009ff16f286157 }}, + {{ 0x00011e4bcd460f2c, 0x000e83fcc058865e, 0x00018d064d550766, 0x00041a7537b0a5af, 0x000046efe0747324 }, { 0x0002573bfa73d9cf, 0x000ca3aa14a90fa5, 0x000c67b515d9d098, 0x00058982d7706e20, 0x00007d1585bc34a1 }}, + {{ 0x000717596c322d2f, 0x00087a93b5bdb831, 0x00008d8fc9ac384e, 0x000bdd75f08da65c, 0x0000f1e0a2f83e31 }, { 0x0002f3b50d9aca4d, 0x000a3d0b266d88b4, 0x0007164ddf4c7663, 0x0005edfacf688ae9, 0x000069a04a1ed5e1 }}, + {{ 0x0001bc81d1567fa5, 0x00054986a90fc34e, 0x0002840e08bc1725, 0x000d25262c8f257d, 0x00007d37c1cef912 }, { 0x0000aa9ebd5aee4b, 0x0004467251a2a12f, 0x0006cddfd21ec0d8, 0x0000a022e10d8658, 0x00004f408c5bc296 }}, + {{ 0x000652b5c82520ed, 0x000d104275968936, 0x0005df6c8f6344c2, 0x000a5abc8b6bb02a, 0x00005bc743f852cf }, { 0x000c773c3afe64cb, 0x0008afd4f5c04999, 0x000818dbeefd7cd2, 0x00084a3f76ea1655, 0x00008333b4274ea4 }}, + {{ 0x000f9a78b85b575e, 0x000dc409b323abc8, 0x000161475ed8d00d, 0x0002fb77ae26605e, 0x0000ce1190f5abef }, { 0x000c94bddefb0e25, 0x0001faac407eddd6, 0x000836c6d870d410, 0x000fa149981ddd1d, 0x0000e0d5b79ac001 }}, + {{ 0x0003624f3e74bdef, 0x0005bda8545d185d, 0x0009a029698ce572, 0x000c9f20e82ae44e, 0x00002103e5d5a56a }, { 0x0001f363f57898e9, 0x000379b14be42542, 0x000c9bc8cab053ab, 0x0002a92be7d0efcb, 0x0000217605b86c8c }}, + {{ 0x000f7246a4a10356, 0x000e71ca85015501, 0x0001b938ca17004c, 0x0005e6f1c2251e0e, 0x00000d5fa3abf1c8 }, { 0x000120a64c2cae9e, 0x000a6ee5f2591f26, 0x000efd41cb0b0208, 0x000451f683c47d33, 0x0000a218fb5a7cf6 }}, + {{ 0x0001c18807f2e6b7, 0x0000fc07b5d07df4, 0x00083d320f7b47b4, 0x00053e44fa92b955, 0x00000d924dd02cb8 }, { 0x00001b784f6b6a60, 0x000708980ba9b1c1, 0x000eb517318e9114, 0x000ba0809ac8dc6d, 0x00005f1ad1849fe9 }}, + {{ 0x000fd8ae60415444, 0x00001bbb6f116037, 0x000dc1f9d1be7f45, 0x00020f0a857b09db, 0x000074bd20b5cf68 }, { 0x0007958e09141208, 0x0009e6678308435e, 0x000bebf0fe4490f9, 0x000c5a2b09572f77, 0x000007396bf5f419 }}, + {{ 0x000fd7296995f44b, 0x000e342253466896, 0x000040a5469aeb31, 0x000a0c3e3716da29, 0x00001e5c16338121 }, { 0x000ef6d5416eee55, 0x000430308b640d30, 0x000e79883d5fe34a, 0x000403e115b0026e, 0x00003880d9961f05 }}, + {{ 0x0001e0f84f93b4ba, 0x0006df0f9c41135f, 0x000876c5514402dc, 0x0008ccd5cf21d60c, 0x00008207421ebd51 }, { 0x000f04b824048527, 0x0007c5906002d609, 0x000b67145d2e9747, 0x00092f7c3be2da60, 0x00003cc3292a8a5c }}, + {{ 0x0002edf52c078d85, 0x000c57dd4b79bb87, 0x000adfda839ce9d2, 0x000ba33e8aee806f, 0x00007fc8b3dee585 }, { 0x0003b2818874b38e, 0x0001a4e2127015a3, 0x000069054d6b7749, 0x000ba60c89051d0c, 0x0000fcc7ea0acfb4 }}, + }, + { + /* digit=26 (1,2,..,64)*(2^{182})*G */ + {{ 0x00063f4153af6dcd, 0x000c683171b445fe, 0x00029af26bf85199, 0x000cbbae89bcf21e, 0x0000e2560e6db219 }, { 0x000e1d75ae224a6c, 0x000f6a930add43df, 0x0002cb637aac5049, 0x000bc7d6862ebb14, 0x00005e73664b3718 }}, + {{ 0x00080034798ca4d4, 0x000f2c7af8685c8b, 0x000d778ecd30d2f0, 0x000cf952ad4c1c8c, 0x0000e198f05db898 }, { 0x00063d51806a84e5, 0x000e1e52eca3cdb7, 0x000b75c78219120a, 0x000a6e0cfea1db16, 0x000052a4a35bae24 }}, + {{ 0x0006c8bebe50eb45, 0x0007cb808bd98d05, 0x00040ead3381f366, 0x0008a8d70644b1cd, 0x00007b632a717522 }, { 0x0008239899b73b3c, 0x00096acc4632b46f, 0x0009b7b269a90fd8, 0x00021394617da427, 0x000076f048091e12 }}, + {{ 0x0008e13092cf573a, 0x0005c3f4c8303203, 0x00005750d3512460, 0x000db40fb842c7aa, 0x0000494598d37817 }, { 0x00077989cb6b3d8b, 0x0007062757558f1a, 0x000150731ef6087b, 0x000f58b273cc3e83, 0x0000ffa7db45a98a }}, + {{ 0x000e632a990cd2b6, 0x00061a144a081e00, 0x000da8aafed1d92a, 0x0004cbb9cd797fb7, 0x0000eb29b6b6973b }, { 0x00010f9eff77898f, 0x0004216c5f0c2848, 0x000eac88dd631243, 0x000a457cff51088a, 0x0000f820a5bcdad4 }}, + {{ 0x0002830fdbcd7a9a, 0x000fab97ffe4b3ed, 0x000113774d45e378, 0x0008122750b691d2, 0x0000dbabb5016bd0 }, { 0x0004aef03d79fd03, 0x000219c52434f573, 0x0005635a7d6e7d8f, 0x000fbbdb34b1f754, 0x000089b90d825715 }}, + {{ 0x0005157b6a28f2a3, 0x000aef4b7ac7ec57, 0x000a6ff82c6bcfde, 0x000476bb8d6ceb95, 0x0000cd5591ba46b3 }, { 0x000117f5ae655dc1, 0x000064ec5413d62f, 0x0006cb47cf197d67, 0x0009cdcdabe329ff, 0x000034f79ff63bfb }}, + {{ 0x000d84601a040653, 0x000c4f4b0166f7a3, 0x0007916814ccaa7c, 0x00080f8fdfbd3e3f, 0x0000bdc4370992dc }, { 0x000a6279f7786e45, 0x000a95c7c1620b02, 0x000d01014b0992a4, 0x000fa801eba68b4d, 0x000052fa0f983aed }}, + {{ 0x0001c6a659019d30, 0x00039b06ae85c104, 0x0008dc372d4d1620, 0x0006a5bec7294697, 0x0000ab9976bc0f2a }, { 0x000333af21cebd7b, 0x00095240e0239001, 0x00019dd9652bff2c, 0x000cd3ddcbaeec12, 0x0000ff15e9da5236 }}, + {{ 0x0009088b4d137854, 0x00096e03aaba1f19, 0x000da268b982cbad, 0x0003344bc2cada16, 0x0000badfeeab305e }, { 0x0002071cf6a8f781, 0x0003fa03549dbd42, 0x000fcf51a6b55cde, 0x000f6558542cbdf8, 0x0000c6c6a717b9de }}, + {{ 0x000cdbcd7623a9b8, 0x000ba1fff2811e50, 0x0003f1a3e116af2c, 0x000483fbbdb81b70, 0x0000d353c95f79b0 }, { 0x0004c3074db8bf13, 0x000512a566a808ce, 0x00034f3d5d88967a, 0x00069e76f02b7445, 0x000003ceb7b97c4c }}, + {{ 0x000a03aee51cbd7a, 0x0003eb0b796d2197, 0x000eba706ef4966f, 0x000307cf5c3e95f4, 0x00007329daec404c }, { 0x000aacb62b09eeb6, 0x0000d76f53a89aa8, 0x000109dd72b102e5, 0x0003bfa4f0d8af9b, 0x0000584ec8a3c986 }}, + {{ 0x0002c97a20562f30, 0x00055ebf62ffd521, 0x000e44ce44efba3a, 0x000b3daefe670b54, 0x0000098ea607e96f }, { 0x0001985e228d3952, 0x000f4ce1fe32093e, 0x0005576f4aa6830c, 0x00035206b13cb4ff, 0x0000bb374024af47 }}, + {{ 0x000f278cb2b3179b, 0x00043a910a230cd3, 0x00038e096c95125c, 0x000ca660a24f46a9, 0x00000266524e1cf8 }, { 0x000207cf075b1512, 0x000b47d50b920eb4, 0x000d4c6e366a9240, 0x00080a509f1cb9a2, 0x000070955e1401ad }}, + {{ 0x000747d648fddd2d, 0x0005181c4127fe1a, 0x000c93a1ef1d3d25, 0x000e28ea5a9f8b9a, 0x00003db1465f08ae }, { 0x000a7234813df238, 0x000f35c2ba97bf75, 0x000d6287fea38e24, 0x00093ef9d15e0f77, 0x00005486c5bb1638 }}, + {{ 0x000a4c97a4abc3b1, 0x000629f8f5fcda83, 0x000483a1c45951ca, 0x0007bee77f3e558c, 0x00000e48e3d45037 }, { 0x000f9b7025a36d96, 0x000524bf6dce6a73, 0x0005d44b52ac36ba, 0x000ec10e9ff373d9, 0x00005254e73f1733 }}, + {{ 0x000917ab9ff1c65a, 0x000f031317cd0627, 0x000da087d792e7af, 0x000eac10b30779d8, 0x00007850b5ef6624 }, { 0x0004d90fa2ac1df6, 0x000a6be28fb1d8e4, 0x0002ce95ca910a5c, 0x00053b26c2e31356, 0x000059685056c6f7 }}, + {{ 0x000012062fd2dce6, 0x0002267d6b3e243a, 0x000cb8f28618953c, 0x000962f389fe6cd9, 0x00007eebcb8ff8c2 }, { 0x0002260beaa35c3a, 0x0008ebc9399d0822, 0x000a1fb906f7ddab, 0x00093057347e8c54, 0x0000f595f07f2d13 }}, + {{ 0x0007401e4d1294c0, 0x000661e1a18cc20d, 0x000e4be1b7e97e55, 0x0007cf6a02192999, 0x0000422ab974c7d0 }, { 0x000b5cbc27a1ccf6, 0x000a3836fbca2bae, 0x0002a46340237734, 0x000b9c94046ad329, 0x00002357b30dce95 }}, + {{ 0x000f6e149c30d036, 0x000786c252bd4796, 0x0009a3476ff36be0, 0x000f9406305e0f88, 0x0000f83a58f41674 }, { 0x000e1a0a8596c4ab, 0x000785cde601a897, 0x00087559b39a4f3d, 0x000826d27d17bbe9, 0x00009be39f3ad611 }}, + {{ 0x0007ac8746207af9, 0x0003896270580c34, 0x0009aa1d02ce4c08, 0x000b9e7a51e23383, 0x0000cf4081696e8d }, { 0x000d9da6c4963c2a, 0x0004d9a100dd467c, 0x0006814766449538, 0x0002461c9274a433, 0x00004c28dc524eed }}, + {{ 0x000774457fbdd20a, 0x0000f9f355116ac9, 0x00091f5d496d688e, 0x00079b671cc38bb5, 0x0000fd56d3b79f71 }, { 0x0000f140b199642b, 0x0007ec8c6db88bb2, 0x000f2dd84d526d42, 0x000887f8b6429d07, 0x00002d0578cfc96c }}, + {{ 0x000e6fa37e3db750, 0x000bf09ca4946937, 0x000073e3458ba1d4, 0x000f9fb1248d3a02, 0x00009a0ba3d63119 }, { 0x000d35c7755bbb5b, 0x0005a0649a0c938c, 0x00079eb02caa0b2c, 0x0007a3912b72c0d4, 0x0000f4f8e7d64356 }}, + {{ 0x000e3c3c1d7cda3d, 0x00032ea87baa6279, 0x000158ee6a629579, 0x0004420c958c1fec, 0x0000d64ac7b719e3 }, { 0x000792f805f03b27, 0x000eb7b2bcfe0b0f, 0x0001327ffd07ff8c, 0x0003e8c973510710, 0x00002665759bab58 }}, + {{ 0x0006229c472f32af, 0x0009904a9b46402f, 0x000ccaec6c09216a, 0x0002f59aaedddf85, 0x000022cb9423951c }, { 0x0004f9e47fc8c2c3, 0x000011e7727b9b64, 0x000b654a63c2fce5, 0x000e684ea289e4e4, 0x00003b8864805a20 }}, + {{ 0x000e57c4ea5be12e, 0x00073fe611f8b8ef, 0x0009f6c12f3d9e96, 0x0006d8cc3fd2e416, 0x00006523896385b5 }, { 0x000a38efd1795651, 0x00002acb1aaf6131, 0x000fd01f6a74847a, 0x00041ff3431df210, 0x00000190aa223cab }}, + {{ 0x000936a01ee6f056, 0x0009c370b1e9e212, 0x000cf487e88bd896, 0x000d164f9b258514, 0x0000b0dbf25b69a9 }, { 0x000717e61d824da3, 0x0003b986dd5b20c8, 0x0000e9e1a8b25329, 0x000df8cd7b586bf7, 0x000098c804cbf541 }}, + {{ 0x0000291d43fc236f, 0x000e82235e6fc06c, 0x000e4b14f6918efa, 0x0006704723477728, 0x0000cd51067b09ae }, { 0x0009f8b71be82198, 0x0002467cec5a196b, 0x0003a235d360dea7, 0x0004c8451deeb31e, 0x000071e8bcc32913 }}, + {{ 0x0006bbee27162ea6, 0x000fc87968a2ab91, 0x00092fffdcf8b836, 0x0007f9930e3c5ce6, 0x000052f83aed92d1 }, { 0x000aee94a5c4b253, 0x000cd17fd42cd227, 0x00007dc33dfaa1ab, 0x0006df6321e13997, 0x00008a5c209883b4 }}, + {{ 0x00046af04a0b75ce, 0x000336089821a663, 0x00085b2d261f5d51, 0x000566845f7c3767, 0x000046b01307b23d }, { 0x000e266e7ea58076, 0x0003c32370bf29c0, 0x000e34bfe26f7ad7, 0x000a63590f90c73c, 0x0000fa778d8e6bfb }}, + {{ 0x0008c7ca3988b700, 0x0008eb90fca65d93, 0x0009dde77b7ef7a0, 0x000105a4e7e2989e, 0x0000f1c42756f442 }, { 0x0008f99897246c90, 0x0003494d402676af, 0x0005fe32e3b161e4, 0x000cdb64e26ab5a6, 0x0000d779dfab8fa9 }}, + {{ 0x00005ab8b7207ede, 0x000a6911a335cc88, 0x0009e9fe5b2d6bb2, 0x000344b19563459b, 0x000063bbb632aaa7 }, { 0x000b647545ffb149, 0x000c9da8d006086c, 0x0009f1914bb72b13, 0x0006fefd76ca4846, 0x000088e4372592ac }}, + {{ 0x000ffea6429715c7, 0x000278fbd7d90402, 0x000e519cf6536d14, 0x000921044ee66a2d, 0x0000fdbc110ce347 }, { 0x0005fc22ff710224, 0x000b7d1114344eab, 0x00064711998bde17, 0x000ea5b910430453, 0x00004bbc5151aab8 }}, + {{ 0x000e608d3b84a63c, 0x00027a91ee682e0d, 0x0000202539ebdc33, 0x000ca402ddf48abc, 0x000039b2f368d7a9 }, { 0x000a5e736f80cbe2, 0x000e08bedafd8830, 0x000eb3366aaf8525, 0x0008d5c413c362ed, 0x00001f043dd12d92 }}, + {{ 0x00051c374c6eccaa, 0x0007911f68cf9d16, 0x00075c5d3e0a50dc, 0x000351809a4f9ef8, 0x0000ad1a87ccd2e7 }, { 0x000f8b5ca64e7fbd, 0x0004e5888b5cead1, 0x0008501e7f03fb15, 0x00096363a0ef0005, 0x0000e724123c7fff }}, + {{ 0x00090f925a13bf98, 0x00028a3ee9523f07, 0x00021e4f171e207b, 0x000b3eff4bb75eab, 0x000087b67cf8d7fc }, { 0x000a80ca34fe2f62, 0x00001ad1e4f6a5f6, 0x0003f7d0d01bf374, 0x000368fee06e08b8, 0x000092f43fe95274 }}, + {{ 0x000932ad964afa6f, 0x0002868aaba4864c, 0x0001c41ffa4036b8, 0x000b294f1b13cabf, 0x000056378b24b243 }, { 0x0006cecce0d322ce, 0x000a28ecae1e27be, 0x0008343f2967812f, 0x000aaeabc9dac426, 0x0000540f92402c0c }}, + {{ 0x000d95bc6cfae79d, 0x000e75e95081181f, 0x000f6bfd9968c225, 0x000cbd8a8cc8a791, 0x0000480e6df000c8 }, { 0x000bdbfb94f56234, 0x00036266888523a0, 0x0006bf25a4d854ae, 0x000577f974d142bd, 0x0000a963460af4b8 }}, + {{ 0x00001be6298ca10a, 0x00098291660f6a6a, 0x0005b5001e039d37, 0x00022f5e87f6abcd, 0x000023ff88c1c17e }, { 0x000064091feffe54, 0x000ed030ca1c3287, 0x000d489f896e7ea4, 0x00092f0b0c74114b, 0x0000370a3f4a2eae }}, + {{ 0x0004f21fd5b99340, 0x000b521e447f2c48, 0x0009db50721aea61, 0x000c5e6de81b8da7, 0x0000050b21dc70c3 }, { 0x0009f72f7ea729c2, 0x0006f814e9df3d85, 0x0009dfb45db575f4, 0x000efe84a1ca4861, 0x0000e4cdd0716db3 }}, + {{ 0x00033c65bf36ee47, 0x000f841386d5087c, 0x000c1a2b70906c0a, 0x00081da5840bf739, 0x00003727fb23462f }, { 0x0008de9cfbb71ad6, 0x0008f7c38ca61533, 0x000aa39a7c7b16b2, 0x00053493997f4519, 0x00002e4ab8b82e16 }}, + {{ 0x0009ed75a8075803, 0x000e8a37b793f853, 0x0009b67a736cdcf0, 0x000bcab87e9f8dbf, 0x0000d28da38fbee9 }, { 0x000320fecff5c637, 0x00011487ffba390a, 0x000ac7d8bd39b7eb, 0x000ae3fdb67f8cb8, 0x0000a584d17078a6 }}, + {{ 0x000c04d32f4cd7b7, 0x0007902b2aced2bb, 0x0008a56701019371, 0x0005a0770dfba180, 0x00007ee3ef2a1576 }, { 0x000b1949fd771f7e, 0x00022d10f9f142d6, 0x0004dcd51025a0e6, 0x0006a36e32c44a0c, 0x00007e3d8d8e45c2 }}, + {{ 0x000b87bcd0217256, 0x0004808be0f44920, 0x0003de4e28470c89, 0x0002210a83ff0da0, 0x00000219c682d1d6 }, { 0x00045c0f29f6bf67, 0x0009583ab1f58ab6, 0x000a1d22cd60f4a5, 0x0007c00e56d9598d, 0x00007cfc00573c26 }}, + {{ 0x000e4c720d49953f, 0x0009e82eabaf21cb, 0x000963f4f390dba7, 0x0000a319182b9602, 0x0000a7b7f6d4dee4 }, { 0x000b26d3ec37e7f8, 0x000c4036f08d6c97, 0x000733e9eb5434fd, 0x00029071236e8a99, 0x0000804003513506 }}, + {{ 0x0008486600572aec, 0x000e6d0c9469ad61, 0x000d18673a5b5b15, 0x00040b686c4a11b4, 0x00004c8a42d0ce86 }, { 0x0001f6653acf2e26, 0x000b0ba8d9ce5be6, 0x00059d5595d2893b, 0x000d2a9730d8f68c, 0x00007023d6e1d087 }}, + {{ 0x000ac7e2a0e832ad, 0x000962b5aff5a48b, 0x0008b7f257c9170a, 0x000e5ecef0d81c4b, 0x00009e1bf2654911 }, { 0x000e23feb138dff4, 0x0004b6c1c664399f, 0x00005da714ac9df9, 0x000cb88bf4c288de, 0x0000a0bdbf12fa64 }}, + {{ 0x000d63ff677c67f5, 0x0002f66cdedca850, 0x00000ca5542522b3, 0x000a91c5d140bb71, 0x0000061b8edde9e1 }, { 0x000dd6d4fcf285d8, 0x0007d517d234afbb, 0x000dc049bac482d8, 0x00004735d61c2db4, 0x00000bdf475cd200 }}, + {{ 0x000379bc0760a44f, 0x000536d3be01cb1e, 0x000b40c0e4757120, 0x000abe6029ba14ff, 0x0000b2768766a251 }, { 0x000499f0b7403ff6, 0x00022e9d4ca8d68a, 0x000bdc215072febc, 0x00005b368083558e, 0x00000674a9be8956 }}, + {{ 0x000524194f020327, 0x000303cb8349abe3, 0x0004fb90e2cd1e88, 0x0004f22903baab3e, 0x0000cf54c8866bc9 }, { 0x00078e9bd6163cb1, 0x00066257d4bcc42b, 0x000b01726de44c34, 0x000e730e0e90a3d9, 0x00006e99c9f280fc }}, + {{ 0x0001b393ef075bff, 0x000b8469e03d2781, 0x0005c55acea80038, 0x0006d79558017860, 0x00004b0cbffe0992 }, { 0x000975eb80eb4487, 0x000de82d8884f278, 0x000a35a3cede5daa, 0x000dbdbc6bd44737, 0x0000d4e86779cbb8 }}, + {{ 0x000a12aeaeda2167, 0x000f0c7d7fe71f3b, 0x000002851f89faec, 0x000cf14631534219, 0x00002b71cc5b2f73 }, { 0x000a778c8bea0c7d, 0x000b8d2a9b4f8f67, 0x0005805f4d10c1d4, 0x00088167c04619d6, 0x000025b9fb741b8f }}, + {{ 0x000fbfee7fb522a9, 0x000dd2320a71b899, 0x000c7c62a2e5d72d, 0x000d59199241a2ae, 0x0000474a8ea015f5 }, { 0x00083dd738132807, 0x00051b95c4017e32, 0x000fecd4344ed7ce, 0x000db466f1dc7f9d, 0x000070c33943a538 }}, + {{ 0x000e55d0c2e5f7e9, 0x0003d09cd85d6a2d, 0x00051c9030b43387, 0x0001b15de6ebbc34, 0x00002dbfb11b5868 }, { 0x0000178c71b5b3ad, 0x000367f24dcdd5a7, 0x00050eba05896a00, 0x0004e910d8d602da, 0x000031562bd0e4c0 }}, + {{ 0x000d3b63b60559ae, 0x000f5863fe54cf05, 0x000ea58050bff57b, 0x00043e6d7a4a3ec4, 0x000068c2c4366a8c }, { 0x000e0ffffce55a24, 0x0000fd85ad3d2aff, 0x000cd39c7a3a9a4d, 0x0001d7214fbc586d, 0x0000260f7349e15b }}, + {{ 0x000c326d123c14a3, 0x000437f906151e50, 0x000f49ae99370325, 0x0006854b7cea27f5, 0x0000400c840acf85 }, { 0x000dea4b70fa8920, 0x0008c337dd35df39, 0x000925ac906e0b6e, 0x0002ebb1c0c3b763, 0x00001965b1ca3b09 }}, + {{ 0x000ebf9747546784, 0x000ea53aae3f639a, 0x000c4d67f464f2f6, 0x00028b35ffaba0e7, 0x000050bca1ceda40 }, { 0x0004311819988796, 0x000193ff41ad5d66, 0x000c4eb87a0aad72, 0x00047c0aa03c4623, 0x0000ecb71c39431b }}, + {{ 0x000e526f3fe97340, 0x000b2118be489ac8, 0x000d2b4ea10c02e4, 0x000f1a0072821895, 0x00003dbc2e993576 }, { 0x000c5a3e30980bb1, 0x00019775b645aee2, 0x000f53b3d2ced666, 0x0009145de3d1dcb9, 0x000083a266843180 }}, + {{ 0x000c4e57b437a58e, 0x0003f3e2a6c6fbf0, 0x000a3fab215289e1, 0x000f7db01a74516f, 0x0000f786a2c596ef }, { 0x000965f3c5b529a6, 0x0000c29b6e386b6e, 0x000cb078b861c0d4, 0x000478029b46793f, 0x0000a2baaffa251a }}, + {{ 0x000f1ef3355edcbb, 0x0004083425c65590, 0x0005fbd17ca52ef5, 0x000fc9bdf7fc00ee, 0x0000d8394a59efd4 }, { 0x0004329d7084b6fe, 0x0005345b79b8ea2c, 0x0004f98d5517a604, 0x000615fb740fd294, 0x00000a2b05ab5452 }}, + {{ 0x0006c177171f353e, 0x0003d52ed1be45d6, 0x000d7e0bb55daa85, 0x0000bfbb1891dd20, 0x00002712c1116863 }, { 0x00091b3fa0fa9f46, 0x0007b412843c4e0c, 0x0006f063761e63bc, 0x0006d4fc7d5ac481, 0x000065f27cb5b34a }}, + {{ 0x000c1fcd2acc0b57, 0x000dd0aab9d97f89, 0x0006f48308e66a00, 0x0009ba0e5fb57bd0, 0x00003a73253811da }, { 0x000f78f55b916830, 0x0009ca9c7ea74322, 0x000c0107819b05c3, 0x000f31ca51b0b924, 0x000084e2a1bbee98 }}, + {{ 0x0006f2e3dafbe296, 0x000de4ac18a414fe, 0x000da94a26abfc55, 0x00062a955c611eaa, 0x000053a9ca6e3b2e }, { 0x00087883af2cb656, 0x0000a179320dc3cc, 0x000ec57e31f52ec4, 0x0008aa490692e382, 0x0000c6a39be83100 }}, + {{ 0x000cfd57c06983bb, 0x00098e453544774f, 0x000b3dde4afbe78c, 0x0005fb6f5834d0ec, 0x00007bae9c3b3751 }, { 0x000ee4d65bdcce8d, 0x000ab3dd6901b1d3, 0x00047673a7680203, 0x00093d2623b49fbb, 0x0000be9b2e57312a }}, + }, + { + /* digit=27 (1,2,..,64)*(2^{189})*G */ + {{ 0x0009ae30f9bf470b, 0x000087d0d63d1fae, 0x000118fba7a5a59e, 0x000fc5809658fc05, 0x00005a1e22227a90 }, { 0x00083acb655ee723, 0x000edd1818baf918, 0x0006032f40bacfef, 0x000334844e27e9e0, 0x000040f4d53495bd }}, + {{ 0x000ee84e265cd5da, 0x000988dc2c6ff707, 0x000f46f3d40f2a5f, 0x0000c9df979d6122, 0x0000ed01e35bdcbd }, { 0x0002d0793015eb1a, 0x00095b063cd880a6, 0x0005d2a436307bbd, 0x000f365423f3ea7c, 0x000081c0177d53ec }}, + {{ 0x00065f36c733ecb7, 0x00097438a5807cef, 0x0002edb163b75a2e, 0x000e8e6daad28389, 0x00005a009a4a047b }, { 0x00026231865878db, 0x0006b9ea60d32b2d, 0x000a6c1f604286b8, 0x000e303b5ee93df4, 0x0000ae2226c13edc }}, + {{ 0x000968734e4dd7f6, 0x0008df9dec31a21e, 0x0002817162256254, 0x000a3ed65988c8b2, 0x00009bb859066057 }, { 0x000f94739f3adb65, 0x000858f5da6309bd, 0x0003e25d5a77565c, 0x00068a42110f3a62, 0x000053d37190038f }}, + {{ 0x0004cf63c0f10c8f, 0x0005106cce85c9b6, 0x00063ff85f12ae8f, 0x000ed8f60360e70d, 0x00002db912ea29bd }, { 0x000f8b36c18dac42, 0x0003da3e1cf6f904, 0x000686550e256d64, 0x00088999fb7e937c, 0x000091b309d5b740 }}, + {{ 0x0002ceef4d55dd08, 0x0005da434a35ea84, 0x00070a7523a8c814, 0x000ab2ec33418e0f, 0x0000b26e001161fb }, { 0x000772f4c008038e, 0x0003c0b543a1957f, 0x000490f9bc634d5d, 0x000955f30c33761e, 0x00003c76f2e45ff6 }}, + {{ 0x0009ff51a206fb0a, 0x00036bb4885d4104, 0x000051a6a735edea, 0x0009dd5b17287f81, 0x000044dfcc90c3d7 }, { 0x0009e739a001c317, 0x000e3536bc3012e7, 0x00019d261a7db4f3, 0x0004c32663d31fd7, 0x0000a707f44faf49 }}, + {{ 0x00003f3e034d81e9, 0x000580ef4f8b16a8, 0x000e443be8670976, 0x0001a9ec197241af, 0x00001dd783168b20 }, { 0x000097440f10fef1, 0x00018a804dbec69c, 0x00067b238c506e04, 0x0008f287fa83b85f, 0x0000d016711f649c }}, + {{ 0x0000c59ad98e2c8c, 0x00023b63524ff164, 0x000e417a2bf4e4e2, 0x00011bec0d7f9b51, 0x0000f07523fe58b6 }, { 0x00038fdb3274f644, 0x000cf6f17c387a48, 0x000567f6a51af73c, 0x000642fc95043b8d, 0x0000e865fd6bdff2 }}, + {{ 0x0007b53ed41360f7, 0x000bfdd418e7ddd1, 0x00032f79db02b1a6, 0x000c6f44d39888d9, 0x0000587a59dee9f8 }, { 0x000b2fea9bbd72cd, 0x000351b2b01ae4fe, 0x0006e4189be73eb4, 0x000eb608788462e8, 0x0000369123795063 }}, + {{ 0x0003fd41b6333ef3, 0x000685fb821fc28e, 0x0006f820e91c2a59, 0x000f1c76289d2e19, 0x00007ba4f718025c }, { 0x000b586236bb3b57, 0x000552eb9a96d612, 0x000a0b3be1bdfbfd, 0x000c84d407f8567c, 0x000013167b56f73c }}, + {{ 0x0003132b63c9bbbe, 0x00042c756b95bce4, 0x00061e5a7b1cf210, 0x0006584525ec0390, 0x000035ffbec182a8 }, { 0x00063e8924f392fe, 0x00041f22f3263bf6, 0x000c3692181d584a, 0x000ae5ecbd5b3733, 0x00000ee6831f5d50 }}, + {{ 0x0000089cda3b5352, 0x000505e5236344e0, 0x000dbd9ed20c4505, 0x00078eb2282e8df9, 0x0000041841a4fdd6 }, { 0x000a6297cdfe9726, 0x0007a71bd38a3db5, 0x000095102157933c, 0x000a7b9754ea97fc, 0x0000a0b0d44f8deb }}, + {{ 0x000a11dae24e01a0, 0x0000b35ce7c42d4d, 0x000ea8ecd02dbf37, 0x0000f7b459fd493e, 0x0000a4b9f5308b34 }, { 0x00014369a85af1f5, 0x0000d785545a7fb2, 0x000c1f919e3e8182, 0x000750558bdb2601, 0x0000d24bcc8a4315 }}, + {{ 0x00019d679200106c, 0x0009dcb06e0956ce, 0x000e1f6f6bb91946, 0x0006139324c9d254, 0x0000df1391186504 }, { 0x000a6bd1e8bf9fff, 0x000a728db5c7be33, 0x000877e20c1d3762, 0x000fb2f2ec714121, 0x0000a3614c505432 }}, + {{ 0x000a1a96613ebed5, 0x000aaa0b311898c7, 0x000452f4160400b5, 0x00055fbbce2a272e, 0x000044105e96f0ba }, { 0x0002280dcf97d62e, 0x00039cea1c8cde9a, 0x0008d0499e144dcd, 0x0003fe7cd9d958ba, 0x00007fe9ad9ee6a5 }}, + {{ 0x000c0f4672046418, 0x0008dc1cf6401471, 0x000d42748aa65277, 0x000c0ad11ea5a2e3, 0x000085ed223cfbd5 }, { 0x000a05f641ba27d2, 0x000e5a9695653812, 0x0000b4c92919d77e, 0x000cdfe09dc96624, 0x000016ab44addb42 }}, + {{ 0x000306333a079f4d, 0x0008866d4312483b, 0x000cb137ee35846f, 0x00064f7935179a95, 0x00005b20b179b20f }, { 0x000dc92787c3fd1b, 0x000d1db0081a726e, 0x000561df74341404, 0x000949bb90149b0c, 0x00002a861cf9d27f }}, + {{ 0x000e08e3479258df, 0x000e34aae6a8e13d, 0x000dc12fb46451af, 0x000c1281585833fd, 0x0000a5f7d00a3745 }, { 0x000c0dcfd1df99fd, 0x0001b4ccdeff2e08, 0x000a8a50deff90fa, 0x000e2498a49f9387, 0x0000a9b6e98e5797 }}, + {{ 0x00094d3f1ba6043c, 0x0009de37a6a308bd, 0x000f72da28b830d5, 0x0008882b66237582, 0x0000b9540188ab4d }, { 0x0007f04703513909, 0x0006a5bba21f248e, 0x000484e1a8ad4170, 0x000772984272ad0e, 0x000027dd807898b4 }}, + {{ 0x00077c871440e1a8, 0x000184acba2ececf, 0x000fec2ab72d4975, 0x000078f250c18126, 0x0000bcb6186f88c3 }, { 0x000109fe96d12373, 0x0001e83fa7f9f909, 0x000bfb8d0916c333, 0x0009e56a3f2e7aa4, 0x00009e0eec3e2355 }}, + {{ 0x0003f901e0ee3da8, 0x000c0a9e9adfe1c3, 0x000cfd5537daacaa, 0x000c08f8f3984403, 0x0000096fcbfd4004 }, { 0x000dbd3ddcb09925, 0x00000fe3652438c6, 0x00063014b42f3978, 0x000265abdae9ec4e, 0x000078614ff4c20e }}, + {{ 0x000bd4d8f33bce8b, 0x0009fecedd36776e, 0x0007fdb0b7462639, 0x00060e565f612c47, 0x000003f48d640dc7 }, { 0x0006125c1b173232, 0x0006eaaaac4cb325, 0x0006bfdae3ea27f1, 0x000704c63ea3aadb, 0x0000876e11548998 }}, + {{ 0x000320c5cb34fa83, 0x000c1a43794f8dcb, 0x0009951b966af168, 0x000a28da0dcca923, 0x0000bdaf4528f40a }, { 0x00083be9a0dbe4b2, 0x00076ff47b50a951, 0x0001b6b449d08629, 0x0002b4f53933e3b0, 0x000077b42a87e4bd }}, + {{ 0x0008df26cd861bb0, 0x0002c5ad6bbe24db, 0x000f9aa510c7c822, 0x000d07d4cc6c7462, 0x00007f4555c709df }, { 0x000cd8924978fe2f, 0x000b4995a483f8df, 0x000680fa9ef11fdd, 0x0000da47572d8a95, 0x00005ee12e0f4b32 }}, + {{ 0x00058280d4ddee6f, 0x000d4d77613aa81f, 0x000f907be4afaca5, 0x000c29388801007f, 0x0000bc59683be7a1 }, { 0x00091554cb1a8b9b, 0x0003d2f7c978c385, 0x0000b4cef8affa72, 0x000d48a2fdf1204c, 0x000070620062b926 }}, + {{ 0x0000f9e4bb715381, 0x000b6d6a05bd5254, 0x000ac69b9680aff6, 0x000bd1ac948d5f09, 0x0000a232a16a1065 }, { 0x0008d30a61a7b345, 0x000b503f46ad2ec6, 0x000f5b10ef8d1b72, 0x00087007700f766c, 0x0000f4ea35a1072c }}, + {{ 0x000b867e6badbcba, 0x000c7ea77b46a081, 0x0007abc1882dd01f, 0x000773c83cf5a647, 0x0000cbd5df06c8b8 }, { 0x000a7c66d6817ff2, 0x0001f141f60558ee, 0x000244c4de37c212, 0x000e31c8bcdbcdd6, 0x00001916c3d12b4a }}, + {{ 0x000dfcf4d119ad85, 0x00033a504e99fd8c, 0x000179885fc958ac, 0x00057cde198a8dd1, 0x00006e59f62aab6a }, { 0x000f97af8853b43a, 0x00033b253758da79, 0x00034b31e858e0f4, 0x00058217ee643bb8, 0x0000da4a5c964eff }}, + {{ 0x0001f37864c9d751, 0x0003720fbeee3f97, 0x000fd715a2986b25, 0x0005aa25c93e5348, 0x000088ce2f7480d0 }, { 0x000c9682db64cb9b, 0x00099e0943aa75e5, 0x00084ef974089e2f, 0x000037219a18c9c5, 0x000082104449d5cf }}, + {{ 0x0007678cab68e47e, 0x000cea6b709c3de9, 0x00019e0e2c451e58, 0x0001cd6bb64a8426, 0x00005633f2cb7399 }, { 0x000c5ab10a85e889, 0x000f05f951a95c33, 0x000314a21afe0190, 0x000f06731dd35b16, 0x0000667bfe6e1cad }}, + {{ 0x0000a02b0eaceafd, 0x00088c341dca5663, 0x000aa7c1ff9509d6, 0x0003070fb7de7fed, 0x00009716331f83e3 }, { 0x0005f449ec69c78b, 0x00092aa2fd3cf36d, 0x000709959bb35e00, 0x00081dc33d131954, 0x0000174691c29863 }}, + {{ 0x00091481c1059a6a, 0x000b11b25fe531df, 0x00017108d4857909, 0x00043e61af979743, 0x00005c928c67b126 }, { 0x000f9f87695e4b85, 0x0005f65f8b05bc60, 0x000ef348b5716c43, 0x000905e3b1844e1c, 0x00004c192e0ee156 }}, + {{ 0x000c992a8a0219ee, 0x000c3c038b27fe29, 0x000b0f0ad855471c, 0x00051edab63b6359, 0x000025a641ada563 }, { 0x00048d3f95b0f902, 0x0008d152a60aed16, 0x00047c320a49af15, 0x00049d79aeb089ca, 0x00008fe88effe77c }}, + {{ 0x000de501a3cb5123, 0x00060a8e1add3f34, 0x000a017fa1a51172, 0x000dfb8dbf42c0c6, 0x000001a82938cb34 }, { 0x000025a2cad7b920, 0x0005c1f4b9a64bc7, 0x0002b4712a8e7257, 0x000cfbffba77f0ad, 0x000059630c29979e }}, + {{ 0x000c477cc78dbf35, 0x0007dd1d1a8f4e7a, 0x0006c70d15b57029, 0x000612235f5b3da4, 0x0000dec5fdbedc4b }, { 0x0009b795f963ab82, 0x000f1d2a883239a8, 0x000642d3ed5e9773, 0x00073cd86e12572f, 0x00006777294aa397 }}, + {{ 0x000ef0e314d85ff1, 0x00065f1b54398dc7, 0x0006c0fd0bc04f5e, 0x0000d10401890efd, 0x0000f8bc668d0ac4 }, { 0x0002c591e55172f8, 0x0006e31cc61dab32, 0x000dd1c0dc69ff57, 0x000c343394a39801, 0x00008f9576364d5c }}, + {{ 0x000e02fc57543709, 0x0009a08de02a53e8, 0x000f4586f7004114, 0x000bd02662a6c060, 0x000038ad03da91a7 }, { 0x0003660ea088e386, 0x000f19573adb4ba8, 0x000c244a29655bb1, 0x000aae9a67b86122, 0x0000a46975ac7183 }}, + {{ 0x00097b8de565ca26, 0x0006b0521ba473cd, 0x000e0559aab8550d, 0x000b275c0b6c50be, 0x00008c7a7b476844 }, { 0x000df7725c5e3459, 0x000e570ab3bbddbe, 0x000e37a00efe4bc1, 0x000e15ad93898aab, 0x0000dfe696962d4d }}, + {{ 0x0006166b44b3b527, 0x000e8abe085116b3, 0x00055111e7cb7191, 0x0000aae2f04a4337, 0x0000b8cbd7d43a2f }, { 0x000af71e91ff6ddc, 0x000673cfd4b322cf, 0x0006bf828cbf933f, 0x0006d39eb726aa81, 0x0000f4fde56a46c2 }}, + {{ 0x0002cadc98401f3b, 0x0006b40641fad989, 0x0007969748c19d4a, 0x000225edcb799591, 0x00004bb11b390d9e }, { 0x00095f7b90617d6a, 0x000b98b256534e4c, 0x0005bc35c509aeaa, 0x000e05b9b73e80ce, 0x000010012b3ad66e }}, + {{ 0x0009cf84aa0fcd26, 0x0004a14d856ac253, 0x000d7282c920e466, 0x0003773f5441bd9d, 0x00002c3a08dedb4f }, { 0x00017edeb92c2909, 0x000a755b2aa95a02, 0x0009ec6039a6d8be, 0x00016624329e33f0, 0x0000c812d89efc16 }}, + {{ 0x0005b9b9cd84c5c7, 0x000783583cbea55b, 0x000d065ea3a2d3f2, 0x00099a292c485ee4, 0x0000a2524bfd6d99 }, { 0x0008ca325a560d39, 0x000b58e4e9e8a506, 0x000af99fa8a3269d, 0x000da58c6462e907, 0x0000ea5c18d58fa5 }}, + {{ 0x00060fff17c23588, 0x00035f213b5ba885, 0x0009dfdf0bac693d, 0x00084130860294c8, 0x0000c20857b9d557 }, { 0x000c2f31600cb009, 0x0007b87d67703743, 0x000d5e27c525487b, 0x000416cd9a373202, 0x000070a468d0a739 }}, + {{ 0x000f684368d3db41, 0x000a8dd7b9520e80, 0x0001fb87f44c3d55, 0x000c0cf4cc078f9e, 0x0000fafd402c5e4a }, { 0x0009cd516968dccf, 0x000e6ed6247e7ef6, 0x0008cfa0acb86b14, 0x000bdd3c2b32d33a, 0x0000aa46051a8ae1 }}, + {{ 0x000f0837e6eccb46, 0x000940f4b3234da4, 0x0005654f66d7528e, 0x0005930c7cf0b023, 0x0000ef0adfc93e80 }, { 0x00089a4060c4f953, 0x00022bb906f6e98b, 0x00040ec7a770d547, 0x0007a5c10253bb43, 0x00007cea8d76fea9 }}, + {{ 0x000f5d57fd84b794, 0x000acc5b8cc037fb, 0x000ab4a8634a4c70, 0x000abb4b83d093a5, 0x0000a4275cfe559b }, { 0x000ef22458521286, 0x000430c8c3bef483, 0x00031b1871b237e4, 0x00018bcab23ac762, 0x00004ef2dd9f1303 }}, + {{ 0x00081dcee597d414, 0x00087dac97e65165, 0x00006a1325b93f7a, 0x000a0d6498877936, 0x000087e6a262fd86 }, { 0x0009a34fc66e7cce, 0x00055918f483f148, 0x000c479481754c72, 0x000f76c426d2dddf, 0x00002d39a7634863 }}, + {{ 0x0008653ffddba12a, 0x000cbb312e08ce5b, 0x0007ef8b72cc3850, 0x000f6fe05a80540b, 0x000001a887b97925 }, { 0x00059b30e1948bdc, 0x000ab8702b0b4c9b, 0x000ee52fe628f9ef, 0x0008038e52675261, 0x0000cae50b48aefe }}, + {{ 0x000aee5ef0da9c40, 0x0005ea60200ec7db, 0x000850dfb43a4325, 0x000a1e7e0b6f4a3f, 0x00005fd883bfa016 }, { 0x000d4f59c8134ed7, 0x00093aae4c6cf389, 0x000633e73e4d5bf1, 0x0008679c6ab4cfe0, 0x00000083cf7bbfdc }}, + {{ 0x00032363a7734ed2, 0x00081b151deb6adc, 0x0003ee6413a23cd6, 0x0001134e1812399a, 0x00001e7ebc657857 }, { 0x000b6f7d030cfaa8, 0x000a7bedc36c64d8, 0x00011c72315df52d, 0x0000f314a86c81e3, 0x0000b2a701f2e2ac }}, + {{ 0x000d56d36a2b2bd9, 0x000140756faa38ab, 0x000bd5aee7eac178, 0x0009b5761a3790e6, 0x00007e4559f47050 }, { 0x0008c39768edb7d6, 0x0008349229a8aa9c, 0x00067cd254bfe89f, 0x000f379e6dcca8f4, 0x00007ea637445818 }}, + {{ 0x0009ebbecd961518, 0x0004736523f2b367, 0x00031821d4ee4c8f, 0x000dc4b2b995c649, 0x00002214d933e9fa }, { 0x0007d928a6a9b4ac, 0x00091db79592f48f, 0x000a67afc0f21e06, 0x0004c6d3e67216a7, 0x0000a28afa8ce512 }}, + {{ 0x000011b6acb767b5, 0x000cf17e991f600b, 0x000d796ff2879e7b, 0x0004986de2a91113, 0x0000f0d31fd49b4f }, { 0x00076430677f20c6, 0x000ef18045ec6135, 0x00016b9162a2f02b, 0x0000d9fde522d2d3, 0x0000d682877747b9 }}, + {{ 0x00075b4a21968cc3, 0x000ad5d1c7193f0d, 0x0006648d3b6cc785, 0x0003a1cc7f0f32f4, 0x00007e378db65c00 }, { 0x0007387e1e912b73, 0x0007e7b342659d41, 0x00042db7b9710081, 0x000269f9de596ea3, 0x0000a3bf0865713d }}, + {{ 0x000cca31c7a57625, 0x000ade949dee1686, 0x0002d160c071cd59, 0x0001de16a1ae0522, 0x0000da394a68f678 }, { 0x0007da93fd13da3f, 0x000971d9bc499082, 0x000cce082821a131, 0x0009544ac4a9b4a8, 0x000025d76c70fa80 }}, + {{ 0x0005f1bfec0f543d, 0x0000f964821c8c54, 0x000d624a5608dadd, 0x0009ef3048de49de, 0x0000fc43c74c346b }, { 0x000087cdaa1179d9, 0x0002d2654b9df609, 0x000c18d43d897611, 0x00089848e5e4fdc3, 0x000036663c4320b0 }}, + {{ 0x0009a7b853608cac, 0x00070238319ade44, 0x000a94e545a4d669, 0x000e28b05766f287, 0x0000f55f670cf4d3 }, { 0x0002971e277b213e, 0x000baa75cedadfd1, 0x00057b1b89ee3e92, 0x000d551958f56281, 0x0000b62941f52c86 }}, + {{ 0x0002ad741cbec2f6, 0x000c096d0ff34fce, 0x00012f74fd700b35, 0x000d2324d20824de, 0x00005c4e2951824d }, { 0x00078a169ffa86e9, 0x000bddf54043ccfa, 0x000b30b9fc13145c, 0x00042a1f016ff479, 0x00009b49c42140e5 }}, + {{ 0x000e5064503a3e24, 0x000634699c930e9d, 0x00076b90b3559e70, 0x000e6519f1d33e85, 0x00006cd16fbc642b }, { 0x000249ef84233f4a, 0x0001300caf3155ed, 0x000bea29f7db9bad, 0x000cbb7db672d66a, 0x0000fcd7af334203 }}, + {{ 0x0004b29cdde6a208, 0x000ced473978fe39, 0x0001e09863c37a8f, 0x0003dc7e9cc4e454, 0x0000cd9ff54a2771 }, { 0x000d2de14cb33c91, 0x0007bff9bc7edf24, 0x000d86ac84a73116, 0x000ae7a488ac236f, 0x00008575be2d1315 }}, + {{ 0x0004d1f3d3a5e8cd, 0x000eea760430e540, 0x0006d56095d81a1d, 0x000aecf5910a2d0d, 0x0000183d1a538a31 }, { 0x000c80b505a3cfa0, 0x00084738a0983cd2, 0x00089c0315b5fc22, 0x000d5e99528e25b3, 0x0000e560b7c69fa3 }}, + {{ 0x000d37c5b1792718, 0x00023714a953beb9, 0x000e790b2245754f, 0x000aecf22502e223, 0x0000c74bd4088fe8 }, { 0x000016a8607ff0a0, 0x0004d70f8330bc73, 0x000645907f288125, 0x000de2cc2139850e, 0x00005677b1f28c7f }}, + {{ 0x000e658aedcda8bb, 0x00031644cd8f64c3, 0x000fe937f063f6ea, 0x0001e975658a2d78, 0x0000049c304fd752 }, { 0x000473ffef5b241b, 0x000792e05da59eb6, 0x0002a9683732b03d, 0x0000a6487ba3d60a, 0x00005d72e137e219 }}, + }, + { + /* digit=28 (1,2,..,64)*(2^{196})*G */ + {{ 0x00099ee566eb922a, 0x000364f692ac542f, 0x000028dcac6c9771, 0x00018cfbdf079ffe, 0x0000610be982fc2a }, { 0x00013889ee80e815, 0x000511fd62191997, 0x000663545a2e0935, 0x0008cc4048c883bc, 0x00000920149ca0bb }}, + {{ 0x000eee0c0ea6e1a4, 0x0009b8f81a1b3bed, 0x0004ef395891f049, 0x000e351f74f3ce14, 0x000069aac489bd81 }, { 0x00027729a694a1ff, 0x000aec0d39e25c35, 0x000071f076d9e952, 0x0002d5b584061982, 0x0000a6863991eccd }}, + {{ 0x0001ad919aac7231, 0x000d837b1b76ba6f, 0x00045b7295421df9, 0x000677d0a1a2eab8, 0x0000c06c35fd7a6d }, { 0x000314874f18d1ae, 0x00031b807ea7b0ef, 0x000e4bc5643a1ce2, 0x0009b5aabedf5496, 0x00007faef00cb014 }}, + {{ 0x000747ea0bede08a, 0x000e0753f73f449b, 0x0007d5fd126e01d1, 0x000e0524d1d1c94f, 0x0000db80899bb94d }, { 0x000c10697e01e74f, 0x00074fe228d291e6, 0x0004d7b0c0585031, 0x000c5761635b804b, 0x0000e6909f3f3acf }}, + {{ 0x000a8f2439972c6b, 0x000d88f0876fd4e6, 0x00071943cf4a2c61, 0x000dcc9cb45f0c30, 0x0000274cda09b319 }, { 0x0004426efde75793, 0x000b01b65ccde7b6, 0x0007c3a4fb720a13, 0x000dcf1d741e2cd1, 0x0000483cddf39166 }}, + {{ 0x000d32fba5127c0b, 0x0008344fc94dea3f, 0x000edda3a50b098e, 0x0009531ef34c8cdb, 0x00009ea6e5479af1 }, { 0x0003173f9743bb5a, 0x00036f6795dcfb75, 0x000330414c181516, 0x0006467c0fef01b0, 0x0000608f1137cc86 }}, + {{ 0x000731021c804f19, 0x000272aba16f73bc, 0x0003c5c8b7dfaace, 0x0001fdebf2fb3101, 0x000060e66ffcf1c4 }, { 0x00051f55a60c6b7d, 0x000ca244fee99d47, 0x000685a66c4490cf, 0x0007f1df74bca48b, 0x0000e008421c6eaf }}, + {{ 0x0008949a50ad4589, 0x0009c1acb452fdbb, 0x0004985e6bffc0ea, 0x000efbfd931ee696, 0x0000aba564d3b7e5 }, { 0x0007a12f75fecfe0, 0x000a3263c88f3bd1, 0x0000c37c6a1321bb, 0x00004ff03ab147c3, 0x00003bd493c68746 }}, + {{ 0x000b70130c3e980c, 0x0006d6d36cf69db9, 0x0008baf6f90bfcc1, 0x0006f018e1219eee, 0x000018a15b3aeb3b }, { 0x0007bfbcdfd435df, 0x00045620ae18e0e8, 0x0002bf00e958626d, 0x000f8d1d42021a62, 0x00007271917a37f1 }}, + {{ 0x000c923aa6870f8d, 0x0002805cdd61d649, 0x000c2c2f4db1707b, 0x00004e8158c7a53f, 0x00007b90d120c88f }, { 0x000555917e796337, 0x000d2cb5c18871f5, 0x00081e5e3af648e3, 0x0006e9dc47d53bec, 0x0000007d33f08184 }}, + {{ 0x0009ea61f1bde96d, 0x0004eda548f5a0e4, 0x0009f1d559f00cff, 0x0008e42c31910eab, 0x00004e2256a193b8 }, { 0x0004b491b0ca83fb, 0x00059784f7c39221, 0x000d0c11fd3eade1, 0x000f39fb995d337c, 0x0000591ec14b3514 }}, + {{ 0x000f450138039993, 0x00028e028e3fe894, 0x00047665bfb376d3, 0x000e4be40e7e1fee, 0x000057339c7f6d65 }, { 0x000251437e2a615a, 0x000058495cd81854, 0x00018b61a205b101, 0x000fd13e474be0ba, 0x0000b796f513feb2 }}, + {{ 0x0004e39da416895e, 0x00014a0ac6203668, 0x000c31f1f5a051e4, 0x0006217b94be3fb9, 0x0000b0003105438b }, { 0x0005627d144873ff, 0x0001590b38f04092, 0x000306c8616def58, 0x0001ca3d80295194, 0x0000cebe7a096ec9 }}, + {{ 0x000f637e4fd3515f, 0x000f1aa658a6d877, 0x00075038f4d59b63, 0x00035e9c91405aaa, 0x0000302534f4de89 }, { 0x000f15b373895ff9, 0x0002e7614ffad2c7, 0x000d3be5b41e6f6a, 0x0002f9afce58fb1b, 0x0000c8215fd1b036 }}, + {{ 0x0008d69d5541959a, 0x00013d2d842ca726, 0x0000d602c5aff83f, 0x0006b2414a71e439, 0x00004b346f996925 }, { 0x0007ac7bdebe7e5f, 0x000f46737183bcfe, 0x000a4bc46418e5e0, 0x00093f35b7152b7b, 0x0000aba65a5ecc00 }}, + {{ 0x00032811a7abd874, 0x000db7881fdad909, 0x00057a04e50379b8, 0x0002cd9e5cf638f5, 0x0000f7d1bc229820 }, { 0x000dd099becd5a2a, 0x00089309bbd11ff0, 0x000fa633afb561a6, 0x000f7a86676108a6, 0x0000803fa54a21da }}, + {{ 0x000c23f829f27693, 0x00017aaabe329866, 0x0009e4b1a3fd664f, 0x000b9e0050fe9dd2, 0x0000a20ae7baca9a }, { 0x0009a25a18798042, 0x000e0180b136a798, 0x000dedffaf7dd73a, 0x0003b0ba67e2174d, 0x0000f25ede8ca147 }}, + {{ 0x000ca5bdc1609c2b, 0x000777ec11ee01d3, 0x000aa94b503a968c, 0x000505e3cf900553, 0x0000383c90f106a2 }, { 0x000b715a6d47b75e, 0x000b73797eada9f1, 0x0002f4b407c350c0, 0x000850bb359c50ab, 0x0000b660a6466bea }}, + {{ 0x000adfe0af71e0c3, 0x00011766ae06e0be, 0x000db26301b0323d, 0x000d19720cdd7888, 0x00002c963a41c4e4 }, { 0x0007e4b0dd7b445f, 0x000f3a9699500937, 0x000aba9adc633558, 0x0009b296a3525d9c, 0x00009fbb8308b6ec }}, + {{ 0x0001610588bce8e4, 0x000ce0a517d7061b, 0x0002baccf0fa5475, 0x000de186777fe343, 0x00000b73f8bc3f10 }, { 0x0006327dec759e0e, 0x00076cad9c4ddbe2, 0x000c1742cdf5be97, 0x00048ba94b3164f1, 0x000038569d7d4e4c }}, + {{ 0x000aeebb5d055643, 0x00083de97b5cd328, 0x000e2afe337097f3, 0x0008894fe40835da, 0x00006140cad890d4 }, { 0x00048990ad4be3f7, 0x00038b75b8ee9ecc, 0x000cc9b3106b05f0, 0x00010490ed7a17bd, 0x0000a0c831b76ce3 }}, + {{ 0x000b19131435469c, 0x000e12a2199e347e, 0x00053f20cd3d5899, 0x00027b5dcbee7555, 0x0000ffa0bf10725c }, { 0x000a33a79b0335bc, 0x000a50d37d341eda, 0x0000aca174906343, 0x0001371ba434fbb4, 0x00002e3e4d934cd4 }}, + {{ 0x00031af02f504322, 0x0002e54e4bd6d3d2, 0x0004379a61a0f9b9, 0x000ffb1a609540f0, 0x00006ab7f4c255f8 }, { 0x000c8be793b491d3, 0x000b9ac0992c163c, 0x0000bc025420deb3, 0x000fa538c1fef8e7, 0x0000930f5d791209 }}, + {{ 0x000a5e00b2f8398a, 0x000f1aec61c3855c, 0x000c8e63fc8830ec, 0x000d209b8ebff429, 0x00000550b4100d93 }, { 0x000174306911bdd3, 0x0002e17c3fed8b79, 0x0009f027d18dbd75, 0x00051ea54d49ac6f, 0x000037ee2449542a }}, + {{ 0x000db067bf6ab9fa, 0x0009e1eaff2ef812, 0x0004fa6846c16bc5, 0x00070ae92c3654d3, 0x0000e7bed9e284b0 }, { 0x0007c60de3cf3529, 0x0007586703dcc974, 0x0007996dd4321892, 0x00065515ce66f9b3, 0x000045556034c314 }}, + {{ 0x000ee790cfd235d5, 0x0008439d860f1b25, 0x000c9705bc7a9c0a, 0x00080e7bd3e9a1db, 0x000066b271a5cef7 }, { 0x0005613245b1f9b0, 0x00084b15bdaf9bbe, 0x0006674163ab3eae, 0x00094878c02cedda, 0x0000e912b227e619 }}, + {{ 0x0009176f1bf4b9dc, 0x00046cc76b23a502, 0x000ce154b2b21706, 0x00010e7f54a2793a, 0x00007203d4f46495 }, { 0x000438b216a5a2bb, 0x000a6874f6f59eb5, 0x000937a3b3776086, 0x00040490286e9bca, 0x0000d17e13cdbcb2 }}, + {{ 0x0005ab800dbb99e9, 0x0001d3c7672765ab, 0x00034f9c39b14ed0, 0x00017bd3337a3ce6, 0x00006e48af02badd }, { 0x000ed04363ca7756, 0x000e95911a5acd6a, 0x0006974fcd00271e, 0x0001ecb0abaee615, 0x0000ef5280b8e5db }}, + {{ 0x0002ace7a063fccf, 0x000ae1ca4a82b765, 0x000ea0d384496b84, 0x000df9cb55586960, 0x00008b8c0663e8a5 }, { 0x000afe4322af8875, 0x0000118cbbc9b701, 0x000a454e62bf1e10, 0x000356c334f06fd5, 0x000087253946a7be }}, + {{ 0x0009b42e68963555, 0x000d7a6c44b2c6c4, 0x0008c5444f9610d7, 0x000340e77ab72679, 0x0000087d908b7398 }, { 0x000d2878a8e58f15, 0x0002fc3cea0b1760, 0x0007bdfe18d92915, 0x0002b88cb53a8ddf, 0x0000dccc4d1142d6 }}, + {{ 0x000803fc21b1eff9, 0x0003cd5d019b6a39, 0x000da86b222452c9, 0x00067c811bb5cf88, 0x000099bb7d29f4a9 }, { 0x000469591564f145, 0x000d79ec07764a90, 0x000eb2f2773fea31, 0x000f584ded93286e, 0x0000a5e431316829 }}, + {{ 0x0002ede0e2398f58, 0x000858a58d6cd461, 0x00057b7853efdb37, 0x0008c6b289cb633b, 0x0000c7e7d1c765be }, { 0x0009e66e59813ff9, 0x000d162065a6f4f9, 0x000ccdd7da21b1ec, 0x00007f41cc3a47ec, 0x00006d5b783c2d8e }}, + {{ 0x000ff6ea2132c1aa, 0x0009aca63d0ff123, 0x00047619b63e3c43, 0x000ebd147e5efc78, 0x0000e1e9f3d79049 }, { 0x000b88bb8cfbac0c, 0x0006d1bf21db8b7e, 0x0000ec5079d8d077, 0x0009fcead7dac70b, 0x0000f978502a906d }}, + {{ 0x000d9e8a3ff2c9a2, 0x00092b4dfdbeb1be, 0x0001c74fc3aad1fc, 0x00088a62122f5ca7, 0x0000f11570bb28f3 }, { 0x00061bd711a3d136, 0x000f6b1b961f49a5, 0x000124e4bf5ef7ad, 0x0007f446d7fee2ec, 0x00002e82c2e6427e }}, + {{ 0x0000ba9d0379c1f5, 0x000a7cc320bbcaff, 0x000eee634788d256, 0x000e6fc28568434c, 0x00003ef3f9080a95 }, { 0x0008117623d9511e, 0x000220432ff9f60d, 0x0005ea4c590f9a09, 0x000762eef3dd8e4b, 0x0000d0f362b6f269 }}, + {{ 0x0007df2ca0800f08, 0x0002076f51fe76eb, 0x000b59f6f8e8b673, 0x000134f188a6811c, 0x00006d8c2c59c4c6 }, { 0x00047ed447b1ad6a, 0x000e452dd7b205bb, 0x000793240fcc896f, 0x000fba748b695947, 0x0000bc03136d4c3d }}, + {{ 0x000c521308e5a687, 0x00011b904c9d9bfb, 0x00005f271d3f34ca, 0x000785d61661e288, 0x000077daf5dd5dca }, { 0x000d465b18f0dbea, 0x000b6fc2477de506, 0x000c1076f6d6d82d, 0x0004d5f05d80bb41, 0x0000b02bf176d362 }}, + {{ 0x000288573948f643, 0x000b030fb6840fd2, 0x000411798f8ece00, 0x000bf71f8faaeb21, 0x000084d5f1bed683 }, { 0x000b394dafddac9a, 0x0009c7c7bc467dfe, 0x0002ed0984ca7407, 0x0003d7aa6926562e, 0x000088d23eb26432 }}, + {{ 0x0004b74cc6bc1151, 0x00094d211c0be981, 0x00016c97349e82d9, 0x0001a7d1cb1e6ed1, 0x0000a0c06a8af629 }, { 0x000cbac448c5b98a, 0x00087868c060985e, 0x000d4102bb339e15, 0x0004fb8ca071047d, 0x0000652742014308 }}, + {{ 0x0003ca782157f884, 0x0004ce2ebf3232f7, 0x000e515fb6bc3716, 0x000bcdbdbfff80f9, 0x00003291c8c0121d }, { 0x000f24a939bc6d67, 0x00079a0e6251b462, 0x000c1ce8e9f3b13f, 0x000f03a9a89bc6b5, 0x000034c1818b3d4c }}, + {{ 0x00027c1c2e5d8f7c, 0x0000246766561b79, 0x00057c5415fc3c25, 0x000fe9c65736600e, 0x00002dd1b7dc8305 }, { 0x000cdd998788c6f5, 0x0002e370cc5df696, 0x00019cf8b021a08d, 0x00028e62799b37c0, 0x0000aee1d95fb4e2 }}, + {{ 0x0004274408c7cb95, 0x0007431390420d28, 0x000b855cab9ba1b9, 0x0006e19ad299c40a, 0x00009b7daa386fe7 }, { 0x00051fddc7f82d4e, 0x000c1b991a711a0c, 0x0007da32cd4f739f, 0x0000f20d6461592c, 0x00002ee7a072ece2 }}, + {{ 0x000bcde201ec9e56, 0x00068b6da50d5311, 0x000d8ad9d7d8a81c, 0x000ad8689521b840, 0x00004b23d5891c53 }, { 0x000e61d34d4d78fc, 0x000ab8b3b76a59a8, 0x000d791746cea6ae, 0x00018f11cf84841c, 0x000090aaea3d7172 }}, + {{ 0x000e7ba65e352870, 0x00054515541b8923, 0x0001be21f7f5e835, 0x00096f10db186ee4, 0x0000fd944bfe00a0 }, { 0x0006f4ce31269aef, 0x000897aa83694867, 0x000717c0ec22773f, 0x000ad2cc309f9dab, 0x0000350aa14102af }}, + {{ 0x000ff862f1ec19ab, 0x00055cff4f066b52, 0x000316254cd0bce0, 0x0001eee9889ab514, 0x0000f50054e6e5b7 }, { 0x0001743d0a1819e9, 0x0001085b05556e3e, 0x000d52f2cb14146f, 0x000ac4eeacd96058, 0x0000e5b40fed6cd5 }}, + {{ 0x000d1e0148639a3f, 0x00033e27eb37726e, 0x000d27639eef0363, 0x0005a6b79f7fa264, 0x00006a16397995e2 }, { 0x0000ad13cee1fbfd, 0x000fc7982e7abe2a, 0x0001a54920299425, 0x0003bc23a19eedc7, 0x00001d27e9fe8f7f }}, + {{ 0x00045446dee19896, 0x0003f241c0e651a2, 0x0009de79db9fb24b, 0x00019e8caab1a6e2, 0x000018a4f366b869 }, { 0x00075841e5ee98b0, 0x0007a2156c07613a, 0x000e4ccf12b26b49, 0x00064594643deafc, 0x0000c457c3bae579 }}, + {{ 0x000e4bf888b90c57, 0x0004f777b0ac93d1, 0x000d5aa9e84f32ca, 0x000e8f22ba6e37b0, 0x0000c2bfd71f0dbd }, { 0x00029438c4942ce8, 0x00044e04df0ba2b4, 0x000d0dcdfb573987, 0x0007fc420935d5cf, 0x0000d2dbc8e26eee }}, + {{ 0x0002e6bead7a7f69, 0x000d2c9049d33fa6, 0x000a3188a951a288, 0x000226da358b82dd, 0x000075c3d6d80aed }, { 0x0006a933d8e7223f, 0x000590fde46c58c0, 0x0009631fa3e61563, 0x00058f6eca7f6937, 0x0000e503ce844e9c }}, + {{ 0x000ca3f44a175048, 0x000c69b077ff55dc, 0x0007ee82f2366b2b, 0x0004c6d4f5397779, 0x000038a3ff5c2a06 }, { 0x00093eed830e7f4f, 0x0000d87a07eda0fc, 0x000f34251345035d, 0x000668b72c138bcc, 0x00000099d6e25005 }}, + {{ 0x0009915109fb7643, 0x0006e574d0b75c00, 0x000259f4b65201c7, 0x00004e9169716eb4, 0x00000af81e525286 }, { 0x0003c7b8c6ede208, 0x000a1285cca6c3bc, 0x0009713e103947e7, 0x0002278c4b25f7d1, 0x0000e315b9cc5975 }}, + {{ 0x0008f732f82265f4, 0x000243c5a1f8e24e, 0x0009b4db22f81d33, 0x000196ea6fdb118f, 0x000088f3204c98a0 }, { 0x00032a76c6083b31, 0x000fc910be786ab0, 0x000f079b2a2f4a0e, 0x000eac5945d32ade, 0x00004fa0648f108e }}, + {{ 0x000ceb151b607fe0, 0x000b1a3ebeb15447, 0x000bc1ec25a82383, 0x000ecd8b344b7b88, 0x00002db5e0795ec4 }, { 0x000e9e1a3e4dbbe7, 0x0007fd05bc1e6a8e, 0x000a1a70ed816faf, 0x000d78e89ec70cec, 0x00003b862411af2f }}, + {{ 0x000a22d40d38f45d, 0x000ad84745b98d25, 0x0000cb34193377df, 0x000202fb9da429a2, 0x000043d54875e04c }, { 0x0009c0bb328b4510, 0x000a9e570e8a4cea, 0x000af4ed4846b07f, 0x000c6c47ebfd1445, 0x000063d3c380d98e }}, + {{ 0x0008653ca70dc05b, 0x000f2600a4c3f6b0, 0x000923b0b29d901b, 0x00087efccbdfaad7, 0x0000bcd3b0c08f71 }, { 0x000be1fdf72ebf6f, 0x0001824538873a39, 0x000d10ffc9f0003b, 0x00023c3ba09299e5, 0x0000ad9a8d1c6f7f }}, + {{ 0x0004957d29e4f3e6, 0x00077595758b11cd, 0x0002f3cfd216bfa7, 0x00043ae470b85289, 0x000035c69d5fca6d }, { 0x000e1c2610168c00, 0x0004363646559b01, 0x000c2b5c647e66bc, 0x0004254a9f4a8273, 0x0000abe6b046a81e }}, + {{ 0x0006133b9250e9e5, 0x000fd28cf6a4a81c, 0x000255354be09066, 0x000bfdc5b1fa3b6a, 0x0000d8a9469a7e74 }, { 0x000ceecdb6a205f7, 0x000de5c9dffa978f, 0x000c624a983be8bf, 0x0002cbe2fba1d1c1, 0x0000bd9e811e29d5 }}, + {{ 0x0009e1cce6add388, 0x000825f10f2b8f5d, 0x000e826a627d023e, 0x000263a13c83a6cd, 0x0000bb21395fab4b }, { 0x000121696926c8fb, 0x000c17caba7e3721, 0x0008ceb03971e6c8, 0x0007d1de5fcdc74c, 0x0000c71b97399710 }}, + {{ 0x0001acfdfce2e995, 0x0000323f6021c5dd, 0x00041e444626f1df, 0x000e37e96880d5e8, 0x0000877660535d8a }, { 0x0005ab89623c630f, 0x00055e8bbb66840f, 0x000d7d09faf7745e, 0x000d5d9ecf7f4301, 0x0000fb09525a4469 }}, + {{ 0x0005b6c3c5a8fbdb, 0x0009e740e9547505, 0x00080f430f00f4c1, 0x00089482e0121de8, 0x00003c819e927d30 }, { 0x000f3e226365392e, 0x000b747519cb9fb2, 0x000f54ea3daa561a, 0x000484b1af134019, 0x00006b90423d137b }}, + {{ 0x00062246dec7aa4b, 0x000e51773d273b06, 0x000610ffa8bbeae1, 0x000a69139ccd2107, 0x00003c66d4c6b3ee }, { 0x000b65168903af15, 0x0003dfa6b83c0d12, 0x00076c84b6bd8c9c, 0x000e810e4a72df26, 0x0000ab60b5424485 }}, + {{ 0x00088043d808618d, 0x0000e7902f345331, 0x0003f3f77d7057e6, 0x000f1ef785e354c1, 0x000037a49cbea99f }, { 0x00040c9ad6afca4a, 0x000e5428c1d50261, 0x00067fc80fd4ebd6, 0x0008d7a502e0c8a2, 0x0000c659782ab4fa }}, + {{ 0x000ce3f19260a150, 0x0005a4225ac0f182, 0x000ad7ee0a60bfc6, 0x0000ef824f7b1295, 0x000023483fef7f44 }, { 0x0008e821b9993c9c, 0x00081558f2e1b72b, 0x00057742808a033d, 0x00081f954fe93228, 0x00002729d121fb3b }}, + {{ 0x000aebfa4044443e, 0x000fabe67e573e06, 0x0008d598ca0bb6e3, 0x0007d84c505891db, 0x00008b6a89573352 }, { 0x000fd835db9f7005, 0x000d7e9e56e55c19, 0x00028cd210f50d2c, 0x000afe3eb7148ced, 0x0000282da8971416 }}, + }, + { + /* digit=29 (1,2,..,64)*(2^{203})*G */ + {{ 0x0009d7b51caacc1c, 0x0008509081e9387c, 0x0006c2bba30bf8a1, 0x000749415fb9780d, 0x000016b6886a163a }, { 0x0009d9971a12e8dd, 0x000f041d6d9f6711, 0x000897c914fde7c4, 0x00037415fba6c1e2, 0x000044ff79be19da }}, + {{ 0x000a38783edbf56a, 0x0008a62060b5f619, 0x0003e197df84183c, 0x000ec5565a56f46b, 0x00003d764abc2d6b }, { 0x000a0edc3fc096b3, 0x00080a9da710718f, 0x000633fc0eb6b9c6, 0x000931e875a77998, 0x000072910f080d6c }}, + {{ 0x000f379672eac03b, 0x000fb216a31e09bd, 0x000ce1fd77c2b332, 0x000e09452de89e44, 0x0000aa34a9ecfe61 }, { 0x000697c8bf759643, 0x0000e0a2591d61b1, 0x000741c37bbc5394, 0x000923c7779aa87d, 0x000040557c30456f }}, + {{ 0x000c351b873ccd50, 0x000f849cb198ac73, 0x000cd2f12adddfbb, 0x00052b678a884a93, 0x00006d2e4199685e }, { 0x000ba6f9e2ce488e, 0x000619fe2c4b44b8, 0x0006a77f7f29e16a, 0x000b9984a580f6c4, 0x0000c4fae9993e3c }}, + {{ 0x0003c99a73f9f984, 0x000135f8fb547385, 0x0007d3fa5060028e, 0x00050845a3516078, 0x00000a2971005ebd }, { 0x0009a4598291f660, 0x000fbe05b81c51b3, 0x00088463f934f561, 0x00056b9a7925bafa, 0x000056cef7017b16 }}, + {{ 0x00043775095179f6, 0x00074b65eb03c0ee, 0x00038e84cad4f821, 0x000c08ad2f19b795, 0x0000a815addc931c }, { 0x000a6a2475d15354, 0x000e250bb3ee8a3e, 0x000bc6c7e2e9f012, 0x00084e0f675eb14f, 0x0000728fb5f890a9 }}, + {{ 0x00058b5fe31ed9d4, 0x00029950f8d67583, 0x00010b00a7fa7e99, 0x000085e21cb84e10, 0x00000fa25fc01a87 }, { 0x0003e86e791375d2, 0x000a5ec95f40e756, 0x0000d98b855d088f, 0x000a5c966f015545, 0x0000404ce330eea9 }}, + {{ 0x00079fb23be5ff7d, 0x000b7abd3c095f18, 0x000e5d17bb3553d9, 0x0003eec8404b261b, 0x00004c7b8e343501 }, { 0x000f0ac52ff88cf7, 0x0007572dfd754907, 0x0004e2b22c9118c3, 0x000d179073a97d08, 0x0000d6bca24f52c6 }}, + {{ 0x0003c66b47990520, 0x000abc46eb564d4c, 0x000bda90a2e18e10, 0x000f0dd5d8d6c752, 0x00005bcd40fc8105 }, { 0x0008ac0b779ee6a7, 0x0000d010f87de563, 0x000f8e19f83df780, 0x000692242ce62c06, 0x00009114b5e7e8df }}, + {{ 0x00062d8c5230f8f7, 0x00029ee4b049136e, 0x00014f3cb9c19a54, 0x000e78a288b63bf1, 0x0000675ced19a43f }, { 0x0004ac0245017d70, 0x000da67379e7896c, 0x000206517d607078, 0x000c44a6ab25237a, 0x0000c32d492b5336 }}, + {{ 0x000eff66bf542bac, 0x000fc6921fea4ca8, 0x000764e07402005f, 0x0003cb3afd36d081, 0x000058cf051c60a0 }, { 0x00069fb9dd0bb00e, 0x000a0e70b57235e6, 0x00009a66b74e0c7f, 0x0004cceb0ee3c39b, 0x000096f6178742b5 }}, + {{ 0x000e008c5bcdd3fe, 0x00003fb319d76820, 0x0008fc97a392e47d, 0x0008db544b029312, 0x00005611953b5d34 }, { 0x000d3a1aadc08c32, 0x000ab1c0278ca331, 0x0006c870390417da, 0x0008770cf666f76a, 0x0000e48921cecb9f }}, + {{ 0x000a418ea93670f8, 0x0002d3173faa4851, 0x000fa665d9cf986c, 0x000a06e7dc8ee6c4, 0x000003f9ec391ff3 }, { 0x0001e3de1002180f, 0x000ae9251f73971d, 0x0002084afa8fd8f9, 0x00025f4193e4b3c8, 0x0000beca7924e4e5 }}, + {{ 0x0000ffc1739db82e, 0x000d6ff50f75f9cd, 0x0002ad7569ae9e9d, 0x000e1e3181d8eddf, 0x0000eddba8e1699e }, { 0x00002aac66c37326, 0x000c6e3037d90f29, 0x0005d02ad905e85e, 0x000d947afe3f307d, 0x00000675780cc1ba }}, + {{ 0x00081be1ba9dae65, 0x00082bf94cdfadef, 0x000c87aaf47a18f3, 0x0004cfef86c97e1e, 0x0000bf0bbd559305 }, { 0x00049a602bc81752, 0x00013e4d14b4206e, 0x000dab067b39bec0, 0x00068756cbecc2e0, 0x00007986052ab4e9 }}, + {{ 0x000709def8c8c5c7, 0x0004c1a567193ec5, 0x000a8eed0812adee, 0x0005924ddaf3c305, 0x00002a0743a5403a }, { 0x000f431d23ed5fa0, 0x000f9830eb2b6692, 0x000b5818530569a5, 0x000050c164d80ce1, 0x0000cf41a7008416 }}, + {{ 0x000bead645c0c39c, 0x00076a6f9294dd32, 0x000f6f60373224a4, 0x00024945cb448d01, 0x0000848887261b44 }, { 0x000041af6242087e, 0x000c54d84fd8b072, 0x000fd7682f82c87c, 0x0004d3f85537b983, 0x0000ee0f932db6ad }}, + {{ 0x0003d2646d5d67ea, 0x0009d4cb940c71e3, 0x0009a78ce6cd30ea, 0x000b333c3211935a, 0x00008dbdca8a24d2 }, { 0x0003929f2c8f6353, 0x000834bef9114044, 0x000a56a9bfc7874d, 0x000d1d9789d1f25e, 0x0000546cc2b4beba }}, + {{ 0x0001b0310c8f78fa, 0x0006c67c7196e295, 0x000041e18d3c15ec, 0x00067d36e7992c2f, 0x0000cbe87245b10f }, { 0x000d6e5a5a506ca1, 0x000e9cfdd9f22cb0, 0x0001a8177b29d312, 0x000d4c288bb1d81a, 0x0000a9d05bcc9b3b }}, + {{ 0x000d04e18964b4be, 0x000696638235d4e4, 0x0006fc69510f82cc, 0x000629dc81c62bd6, 0x0000d69b284282da }, { 0x000f0e6f4ef7aeaf, 0x000670185a8068c5, 0x000133a42b363af9, 0x0005edb3bdb58e4e, 0x00008a81469b37cf }}, + {{ 0x000d487b80c0f188, 0x0000e8469ed23707, 0x000c8b2972a54721, 0x000aad30534dc30b, 0x0000077e05f5c7bb }, { 0x0008c4e13b06dfc6, 0x000f437dd0e71799, 0x00011b3f36d639a9, 0x00094b5a545f8019, 0x0000cc8a58f51d91 }}, + {{ 0x0003ff120d6b43b6, 0x00080e6e82c9f9e4, 0x000c7f3afe366902, 0x000473605711db87, 0x0000b79687b6563f }, { 0x000c63a68310f640, 0x000492fae77779bc, 0x0008c078f8ffc2c7, 0x000cf71bcf0fe0cc, 0x0000f50ef21963e0 }}, + {{ 0x0007fcd27a7b8803, 0x000b70c09dbe19a1, 0x000d535fa1484c62, 0x0005d09302f3585a, 0x000034ee43c2d08f }, { 0x000ec3df2b85cbb5, 0x000983410b3d5a5a, 0x00017bd6edc62975, 0x0009ead4209c107d, 0x00006cece29a740e }}, + {{ 0x000eb585fc03a598, 0x000fab73f894ae03, 0x000578a992d2287e, 0x000ae32d0f59df3e, 0x00009041a752d68b }, { 0x000f18757657a10e, 0x000fe8632c7f717e, 0x000ac07bc9d5c038, 0x000db43d2eb8c795, 0x0000beb76ad83f9e }}, + {{ 0x000f678f4d369698, 0x00025b20389168b6, 0x00087aefedb6701e, 0x000dd9882c4ecd25, 0x00001a019280de1e }, { 0x0006b6c517470dd7, 0x0003acf65e58dd88, 0x000e4d4bfbb60e27, 0x00029ea4cd53b4e3, 0x0000c0f835501259 }}, + {{ 0x000e54523f9dd595, 0x000453eb38dff6f0, 0x0002dc0f19a88a5f, 0x00007a2f8be012c5, 0x00001ab8ae664cc4 }, { 0x0008e19b0ec8804e, 0x0006636036a05d8c, 0x000e55b225f11a9e, 0x00001cfdf83e3cdf, 0x0000493f71b10088 }}, + {{ 0x000a39687fe60d79, 0x0001b3fb569a5fec, 0x000409fac4f5e2bf, 0x000df2700ad0090d, 0x00006a9ddd1bcbe4 }, { 0x0005efb775a4c718, 0x000f5f765959d77e, 0x0007b6a6922442f3, 0x0007a3c5139d7821, 0x00007d9756c3730d }}, + {{ 0x00063dec3e3e2558, 0x000b9a4e822f0d08, 0x0000d44918ba007a, 0x000a12a0fbc647ad, 0x00002a7318c6f712 }, { 0x00063dde65759c1b, 0x000423af017bfc7f, 0x0008a12da3c38c63, 0x00020f23b2005443, 0x00007385de06d196 }}, + {{ 0x000223802a112a4d, 0x000a2b32a4c94e9b, 0x000de4d4a76011b1, 0x000d92c3dafbff0f, 0x000052dfb56d8723 }, { 0x0002e8a4369887b3, 0x0008b578ec3fecad, 0x00036655cc1ccce8, 0x000df81c406c3831, 0x000054665660b4e3 }}, + {{ 0x000668270095818c, 0x00004525e9ca895c, 0x000dd0b84bde7efc, 0x00044e95c2f4dd31, 0x00005870893f8e51 }, { 0x00050bf75458d456, 0x00020a30cf43895a, 0x000fdeb71226c646, 0x00064b80ea9d604d, 0x0000539558e1b50d }}, + {{ 0x00090b3defe902aa, 0x00013def6a702510, 0x000b335f737a1d94, 0x000cddd62548b801, 0x00003e17f46314bb }, { 0x0009b0f6c59961be, 0x0000d8969cb91584, 0x00044fadd77dee87, 0x000783df544a9074, 0x0000f348045d2e71 }}, + {{ 0x00037dccb38225c9, 0x000f315adf7cccff, 0x0005ec2414fcf3b3, 0x00075f87e81a3e5d, 0x0000f61ae1e090a2 }, { 0x00009fa4f71b333a, 0x0000e73907fba12d, 0x00097997840dbf32, 0x000506cc535daa6d, 0x0000b54ff864cf47 }}, + {{ 0x000f979bec99a237, 0x00067c9aeb19a362, 0x000e982823c7ceec, 0x00020c3a6d913f1c, 0x0000fee386354fc8 }, { 0x000f2edcdad2a280, 0x0001d0f359679ec3, 0x000cc86c2785e2f4, 0x000583870ebcf523, 0x00007790dea32d3e }}, + {{ 0x0001f3e3242b67c2, 0x00037a76f568431c, 0x000b4104a667e1fd, 0x000a399d5f848c27, 0x00000c5d8c8cbcfa }, { 0x000519b637651d15, 0x0001c9c26c185ddb, 0x00079f55548ce1cc, 0x0006288576d85b7d, 0x00007ef27cc4babf }}, + {{ 0x000f6557c2e6a3cf, 0x00094a9ab87d59c5, 0x0002da65524e84bd, 0x000d3697bff9f58c, 0x0000bbd202141b4a }, { 0x000ea096448fc959, 0x000e70636a830417, 0x0009f74e8327d786, 0x000c320409b1c96c, 0x00005b046b469a20 }}, + {{ 0x000e20be26cf50fa, 0x000b9b56aa39dff0, 0x0002916d8f65e5bf, 0x000fb519f59b43da, 0x00002869fe6dd6ce }, { 0x00029808bacacede, 0x0009382e8ee47c94, 0x000b3b2178464d49, 0x0001271f6d9687cd, 0x00002d0288e392d7 }}, + {{ 0x000aa219cfc47b4c, 0x0005a0d4944b6639, 0x000909570115df01, 0x000bb97b1f901f7a, 0x0000200ebd199ed9 }, { 0x000598dee385254a, 0x0009875056a2512b, 0x000928ccfa78af5a, 0x0007a79eda4d643d, 0x0000a165f63eb2e4 }}, + {{ 0x000ef8428d09f84c, 0x00075e5dda9e5c65, 0x00026bb1ba085e32, 0x000339e444e0d3c7, 0x000054c2fab4516d }, { 0x000e342afaa45999, 0x0004b7a460ff0166, 0x000f041e56d127f3, 0x000ca11d2da6d12b, 0x0000227aebe64d5d }}, + {{ 0x000d33b92ef918a8, 0x000130b93a37c047, 0x000b5fdd04671ae8, 0x000939eb762b1bc9, 0x00007fee369cbc14 }, { 0x000ef80e1f1184ae, 0x000fb4be61fa29a5, 0x000ea74a1414172d, 0x00088df374b2be13, 0x000023e963704321 }}, + {{ 0x000d96dd292ac476, 0x000d0996d16768e9, 0x0009c4f514edec88, 0x00089d30d9fc4ff6, 0x0000a3049df36aea }, { 0x000386b3c21696a6, 0x00033d8ce5d72c41, 0x0004ad95bd8d614c, 0x0004dfa3e9d06c5a, 0x00004d05ba437463 }}, + {{ 0x0007d17dc0d747a5, 0x00043a6fb283f61f, 0x0004fbda463b273b, 0x0000591227df203e, 0x000021ce3dea9063 }, { 0x00056d9dd063bbcd, 0x000a144522055f0a, 0x000aea39ab0b5fd9, 0x00061ff14de3012d, 0x00000a88edfa935b }}, + {{ 0x000ff42bc607a539, 0x0004db68bcec9c2f, 0x000868177340b0bc, 0x0002d59817cf24df, 0x0000d7cc1c5324b9 }, { 0x0004c5f529e4d5e0, 0x0005c404d289427b, 0x0008ffbe999a2d50, 0x0006aa00c3d5f189, 0x00004942095ae6eb }}, + {{ 0x0009c75b689ed4a6, 0x000998fde94fdcb0, 0x0009bdfb94f62d7f, 0x00051ece1e96af73, 0x0000523f6b4acde9 }, { 0x00018c7501c3e7db, 0x00062f2b5932a7a7, 0x000e6c7d04ab9f0e, 0x000febb312658252, 0x00006c7c749a3df1 }}, + {{ 0x0006ba6d4797b4e0, 0x0004d69ea40dc3a4, 0x0002681194478eef, 0x000bc104adc84ad2, 0x00003476bddf339a }, { 0x000c417d369d684d, 0x00084c593e58a8e2, 0x0000e5ca8b55cecf, 0x0000639daa239473, 0x00006f34e5122d2e }}, + {{ 0x00056f1bda88811f, 0x000dac58b6f8efae, 0x00095202695bd2ef, 0x000b8aa2b671a2fa, 0x00009c76e0d6ec2c }, { 0x000d267e667d7ee6, 0x00088c49ff3c8eb5, 0x0001e6229e840d8a, 0x0001551942deae43, 0x0000a944be2e849a }}, + {{ 0x0000009e494c202c, 0x000e56db07159d03, 0x000f9188c3c7511c, 0x00075e979ae07a67, 0x00001fabe7db3e6f }, { 0x0007b796c365a4fe, 0x000b122e31328195, 0x000c9a831e3c24e7, 0x000b8d887cffa197, 0x00006565fda180ba }}, + {{ 0x000195969620544b, 0x000ef6ca1779ad79, 0x000da3e6732eeb47, 0x000605a7ea33cfe2, 0x0000c3bd61477f1d }, { 0x0004f291ca3b4ea1, 0x00018a6e546c8792, 0x00043da3370bd63f, 0x0009ddf6b8e996df, 0x0000fff3f513ba1a }}, + {{ 0x000136d7f23d9040, 0x000e3bb440e22296, 0x000212b875128f14, 0x00081ac39324673a, 0x0000c58b94474200 }, { 0x00061d7940cb50ff, 0x0002999b053a9270, 0x0001a7e9058caf2a, 0x000269dd93eaa266, 0x00002b4a7d3f8737 }}, + {{ 0x0003f67568134616, 0x0009283f06d18bd9, 0x00031e7992db2ac5, 0x000d4884d4ba6de5, 0x00005538b6570045 }, { 0x0002d4d4f88e4a39, 0x000381b4347237bd, 0x000f14bf8b95cbb8, 0x000ad048cbc041e2, 0x0000ef112474fce2 }}, + {{ 0x000533837f402aea, 0x0009089b3e93ce7b, 0x0004a806a89c97a7, 0x00030378af7b5a87, 0x0000d33d95cf20e5 }, { 0x000972a7c51e92cf, 0x0008b73579fab2e2, 0x0001c3649bc0b31d, 0x000cbc215b41055a, 0x000013a756b9af25 }}, + {{ 0x000fd1823a519a79, 0x000ba4ef678e68a2, 0x000069ded5b31a6d, 0x000323d9a4e4160f, 0x0000ac5ec7087e01 }, { 0x000c3930bb0e4209, 0x000fe2f3298f0557, 0x0009828e19b3d13c, 0x00072c7418c0566f, 0x00000b7752b85e4d }}, + {{ 0x00058d3c48fce8ff, 0x00042fcaaa2902bc, 0x00091f695e8ae388, 0x00056a0fd539ad79, 0x00003fdda1024b84 }, { 0x000ceb97675efd50, 0x0001fad5dc64e964, 0x000965554bed5435, 0x000a497a93aafcaa, 0x0000072f2cc19c4d }}, + {{ 0x000fde3ee29e89ec, 0x0005249e32a858e7, 0x000e2eb23ccb4e1e, 0x0002fc8944c32892, 0x0000fdf45fd7d390 }, { 0x0000793aacf7c0a2, 0x0000909caf549579, 0x000e557c9b281bf5, 0x0004340c0bfb028b, 0x0000eff9d16da328 }}, + {{ 0x0009ee5f4330c617, 0x0001779c81710a00, 0x00061dde4e912dce, 0x00009560c557e4a3, 0x00004f997a74131b }, { 0x000606bdf720e7be, 0x0001de9d4292f19b, 0x0005adeaad214a89, 0x00013f62a56f74c2, 0x0000372d1eb10015 }}, + {{ 0x000b1ef85cb235c6, 0x000974d044573ac0, 0x00064a4c13476c7b, 0x000d830f77dc3cf8, 0x0000e93ec667d713 }, { 0x000aaf7c181ff6b2, 0x0004684c96958bee, 0x00034ab98d4b092c, 0x000a94e617c32fa9, 0x00007362c69c64ca }}, + {{ 0x0005faa9c8bd37eb, 0x000f34687032d585, 0x00030831d017617c, 0x000cd0423c54f3d8, 0x00007c6ebecf2b9f }, { 0x000b53e0cd41d986, 0x000c96f89b95355d, 0x0009dbed7bb78cfc, 0x0003534d34860d2e, 0x00008bc6614104c3 }}, + {{ 0x0004603695bcdea6, 0x0001a319feb65939, 0x000fd66cc881fb52, 0x00076b610dd87f30, 0x0000fa4b825b15e2 }, { 0x0002aac5a362f271, 0x000a0c32e83358c6, 0x00048c503bd59590, 0x000c81552508dd9b, 0x0000887ce9e262ca }}, + {{ 0x00078ebe5bca260d, 0x000c5a08b1ea7b31, 0x000396fc5629d03b, 0x000df2299d495ab6, 0x000045eb0b5e28bc }, { 0x0004348796ae1fc6, 0x000f6ca5b36d17fa, 0x000306c3e75354eb, 0x000d923e0b5e4cce, 0x00000bbf054089ba }}, + {{ 0x00055e14b2edfcf1, 0x0003209eef1a7522, 0x000749cc8d3ad05c, 0x000038cd25d7162d, 0x0000d5775e923a21 }, { 0x000c80242e03d505, 0x000db720303b6972, 0x0005c3d6fcd1b6b0, 0x00053e6b6c5d17e2, 0x000040da0a6e404d }}, + {{ 0x00094cc7f33d38db, 0x000ef5fabfae1ca1, 0x000c00d47da85466, 0x0002dc9d2937afaa, 0x000024c1f8bc28e2 }, { 0x000c53b0093312f3, 0x000213a8aa116906, 0x000c9d61edff7a12, 0x0003d930abe40ad8, 0x000054fe955092b4 }}, + {{ 0x000b70ce22f66999, 0x000a85d1356eb809, 0x0002fa0810b5bf08, 0x000261a8bb8bf9da, 0x0000447d5e591368 }, { 0x0008f8655fa050ca, 0x000378a829a81803, 0x00037f266d35dbaf, 0x000ec208384329f9, 0x000095e38ca342be }}, + {{ 0x0009bb9dc9c786dd, 0x000c3d01d3ec517a, 0x0001a76783e4deda, 0x0002248989874465, 0x0000b42617e4c825 }, { 0x00071bb70b07ff38, 0x0000b5578b4c2405, 0x000b09f0f8d10cc6, 0x000394c6b5ed62a3, 0x00006bb834b02c13 }}, + {{ 0x000e530573a2840b, 0x000ae5b2da705d20, 0x0006a34d8a2512d2, 0x00068f0e53ef8ada, 0x000041196132343b }, { 0x0007f458eb02c886, 0x000c75521a9255d9, 0x00015aff273f5d8c, 0x0007ef27563a32fa, 0x00008ba09cf2c5ca }}, + {{ 0x0008797e91aea7cc, 0x000a480eb6b242d6, 0x0002d2f863b4d485, 0x00089479dd30bd02, 0x0000e4b655e68a0f }, { 0x000d53005ec1aeb9, 0x000290a6b4e185ab, 0x000d8586b6a88091, 0x00048c81b82f2c67, 0x0000bb2a23d0098b }}, + }, + { + /* digit=30 (1,2,..,64)*(2^{210})*G */ + {{ 0x0002f4209bf9bcf0, 0x00059be9d8e68fd9, 0x000288102284ec39, 0x000064398db0f053, 0x00004a5dc6b917e7 }, { 0x000420b560d4b030, 0x0007fcc1a739d4b6, 0x000f3e6a037e1521, 0x00016c1d009aea75, 0x00000584c6da5516 }}, + {{ 0x00082b921a859ae8, 0x000619c6254dc41f, 0x00038257ec641913, 0x0006a3ca77e29392, 0x00000c021e167183 }, { 0x000a3d8c1544aca0, 0x000f62e1402cec5e, 0x00070ff1a4d2048f, 0x0000bd9cb9f17ca1, 0x000074174697d3a5 }}, + {{ 0x0006d4822ab85c35, 0x000e0e75e0dee6e5, 0x0002858e08782681, 0x00005869374f2487, 0x00000b357177d3c0 }, { 0x0000031f8adc8033, 0x000e4cfc59826f90, 0x00092404f713d9e3, 0x0001ec585ca26096, 0x00001aca73b07beb }}, + {{ 0x000e0ace9790f135, 0x00011e60ebcf7262, 0x0001342678767f4b, 0x0008aec094482b83, 0x000086d2546132b7 }, { 0x000cf8daedabfb38, 0x00071d31f8d1f420, 0x00014a5069e4864b, 0x00083fb4b1b1e83f, 0x00006eb2e2034cf9 }}, + {{ 0x0003ad49be80c260, 0x0002ea800ee11c1c, 0x0003f23693b6b015, 0x000c9e3c5eb053cd, 0x000021587ac78054 }, { 0x000143e74d589844, 0x0004a41b64279905, 0x000d1f3ac6d14165, 0x00013731e12cf15b, 0x00005b3101abcdc2 }}, + {{ 0x00088690b3b5dbba, 0x000a936c88bdee1c, 0x00058fefe5c87a4a, 0x000975f9625f2930, 0x000004895caebe10 }, { 0x00007be238c00228, 0x000042d852705b44, 0x00040d8d72f4674d, 0x000634dd1b0bf8d4, 0x000089f5af04b2d8 }}, + {{ 0x000bcaced2f7f0f6, 0x0003a49a639fc6b8, 0x0008fa356752166d, 0x000db2046e75c35d, 0x00000c7919830565 }, { 0x000757ac98dd7581, 0x0004e1db55367425, 0x0006c34659f304b9, 0x0000388c5013dd89, 0x00009e9e11e632ad }}, + {{ 0x0004d5ef9bcae914, 0x0000b0d1e935abb5, 0x0003c077f3debaf5, 0x000b3d7b5defd10b, 0x00005a2eed07f66b }, { 0x000687601569519a, 0x000a7ba74f17e266, 0x0008123a7dcee411, 0x000877efdc0a0e3c, 0x000028a9a1af6c1e }}, + {{ 0x0008f0539945768c, 0x000b29cf36658f0c, 0x000ba907b1e581ba, 0x000f3f75ac433ef1, 0x0000e501b89aacd5 }, { 0x000e4ebf5dc2f5a3, 0x000e9dfe983c4a11, 0x0001e568f9d43c43, 0x000c3615bbf5e816, 0x0000c937c01fe8f4 }}, + {{ 0x000792a7ad2ea66a, 0x000c06043a24fe7c, 0x000e3820a17ecaed, 0x00089740529c1191, 0x0000da0679e69c61 }, { 0x000df141eef101e6, 0x000a4d8b737982c6, 0x0009342d1ef43ed6, 0x00081731a7e3a031, 0x0000448cc1825582 }}, + {{ 0x00001f483720d8ad, 0x000f339e6d6f4714, 0x000ed17988f78de8, 0x0001638ac14b2ba0, 0x000005cc7b06291a }, { 0x000e8e9670cc34e3, 0x00054964fd03f663, 0x00099f00a8a162af, 0x000032c183fdb4e4, 0x000086fc98b4855d }}, + {{ 0x0009432af25a44dc, 0x0003c0c207674f17, 0x0006ba6b64e961d4, 0x00069db1e112e09a, 0x0000b97a68266210 }, { 0x00016fbbff8bf955, 0x000008fde2ddec64, 0x000f11e7b94049f2, 0x000e7c08a81392d1, 0x000090f2310497ae }}, + {{ 0x00073c690595d548, 0x000b5fd0d4a9dcf2, 0x00086d11760d0890, 0x000b020667bc29e4, 0x0000e82ab371d3a6 }, { 0x000ed3d505fc5b0f, 0x000d518cdfaa6b88, 0x0002c18df1996da1, 0x0009c4f27606bd82, 0x000024b05bcf9a6e }}, + {{ 0x00060e86489642ef, 0x00039416ab25b6a0, 0x0007250bbedb1912, 0x00086af03c6c0ffc, 0x0000643b4ceb328b }, { 0x0004f435d3812e8a, 0x0006f0f93c7f3988, 0x000afe049b37d0d9, 0x000a26fe39f4a96c, 0x0000c8d9ac21a5b9 }}, + {{ 0x0008aca1dc8cc929, 0x0009c5b8e33787f1, 0x00058d8ba496be3a, 0x0008890f0ef42f97, 0x00009f738f8c1ac3 }, { 0x000f6c9c74fac735, 0x000eba72d4bfe80d, 0x000c25d1d6b75c29, 0x000709516fd823b3, 0x0000df2d17686f7a }}, + {{ 0x000a995e5dfbc6e3, 0x0004fc7904fc3fa4, 0x00057d4a050cf2ac, 0x000caf6aa14a23f4, 0x00009e74b3de65f4 }, { 0x00016c563b4a41b8, 0x00083ecf3a5598a7, 0x0007407e5359d663, 0x000c3283c56534b1, 0x0000a377a93e6974 }}, + {{ 0x000bf3352222107c, 0x000fba41d6178e77, 0x000fecaeca6c05a9, 0x000037a3de4c80c3, 0x0000151b1e5776e8 }, { 0x000829e76053679c, 0x000858e10b365bbb, 0x000357ae0314b797, 0x0004c0d258102180, 0x000045a0e1b22a51 }}, + {{ 0x000867e8fbae98cc, 0x000a9a54d53e5fb8, 0x00024cbf2ee7bd43, 0x0000b62c204a5dc0, 0x00007dace033343b }, { 0x0007f6fd799a066e, 0x000ee6ebec9889bd, 0x0001f350cc14257d, 0x00005bc1254f491f, 0x00000465189040df }}, + {{ 0x000fa37d174d524d, 0x0005f30acde5e173, 0x000028af37c05817, 0x000b76154d0f996d, 0x000078f2cc1b9df8 }, { 0x000a99951b3d4f35, 0x000518434b8fb41f, 0x000ca461cfd62559, 0x000f9acf299a2acb, 0x000083fa11ba4326 }}, + {{ 0x00064110e3d7be93, 0x000db892f3b26e7d, 0x00000446416395d6, 0x000c44aa3f470986, 0x0000d33c4ed3954c }, { 0x0002e3c7706265bd, 0x000d02cfad414e41, 0x00081bec63b3b22c, 0x0001c5ef12b155f5, 0x00004a979c09ab69 }}, + {{ 0x000f6db89c6efc0b, 0x00052e59fff83814, 0x00020c510ff875a0, 0x000517dc41105323, 0x0000b7c05ea42fd6 }, { 0x0007d34d61e350d7, 0x00068d6ca563a441, 0x000541073d5751ad, 0x0002419a17f5c221, 0x00006c4fc28b0bb1 }}, + {{ 0x0004b95e3dbdc078, 0x000c1101c474019d, 0x0009007390bdd0dc, 0x000a1b42bcaff262, 0x000042e2d657f503 }, { 0x00021d322b01c90c, 0x0009a1f1594438c2, 0x0005e28dacb4f9f3, 0x000183d401eb6e6a, 0x0000c764dc01c967 }}, + {{ 0x000629a6c01aa769, 0x00040f6b7cb179a6, 0x000d5169a70cd2e7, 0x00061af0ddc7b764, 0x00009db12efb8043 }, { 0x0007d3a142e389a6, 0x000b84e9f5a915e1, 0x00074a30a2dbdc3b, 0x00082b953b1db9d3, 0x0000f4c3e36d8caf }}, + {{ 0x000e709afd4ebe8c, 0x000b8ea308780f21, 0x000119d55ae78e9a, 0x000664ba3dc8de76, 0x00001b9c38ab8027 }, { 0x000526074ae5ec7c, 0x000c776021704560, 0x000ac3d34419af49, 0x000944e68bf204b3, 0x0000bf2f1a243170 }}, + {{ 0x000f57691dba40a3, 0x0002bf8ecd92af61, 0x000a4d7a0b3cdb98, 0x000e520e4e6b9fa7, 0x0000cbd83da71fe8 }, { 0x000c2134f4315826, 0x000cdfe9d2e1a33d, 0x000ecfe50841b739, 0x000bb7d27d430093, 0x00008342dc8c1e23 }}, + {{ 0x000a626f3a19601b, 0x000e32c5c2102869, 0x000d8248f65290f1, 0x000d557923f68aae, 0x000026d9214bd1df }, { 0x00023bbc1636c361, 0x000daf3199d78e38, 0x0006ef80072fc767, 0x000961f2cb7a2af7, 0x000031f1a12763e7 }}, + {{ 0x00049f321e4bf811, 0x00028f299d0a4221, 0x0005dfb794dd3969, 0x000277ad69ac3df6, 0x0000a5e53844a824 }, { 0x000d0299708cd13b, 0x000bc14721b7ebaf, 0x000734a097ba251b, 0x000f39a81e17df09, 0x0000236afe787c48 }}, + {{ 0x000a45755beb5e5a, 0x0004c3e7882f14f1, 0x0004ed815580f741, 0x0002464d0c9f6dc3, 0x000088180b1213cd }, { 0x000ca2ca9f747a28, 0x00054277c4608cf7, 0x0001d9dd48232ab2, 0x000f8730e7ccbdf3, 0x00005ef4880c3dec }}, + {{ 0x000a91f32af5551d, 0x000cfb16f640f620, 0x000b2f63261c003a, 0x000339244e274b92, 0x0000106a1125d96d }, { 0x000ec1cc3386a669, 0x000be4bd6aaba315, 0x000fb0a6b8c5bd06, 0x00052b520719a163, 0x0000859041c6c354 }}, + {{ 0x0008724aa3301862, 0x0007c963b54a0599, 0x0005574ab31008dc, 0x000a32ba60e0015e, 0x00004faf233b361d }, { 0x000a0642cb6dca07, 0x000815bfdb6558ef, 0x0009cd8357cac326, 0x00063ba12f9f7a28, 0x00002446724efd19 }}, + {{ 0x0007e5eeb46274cc, 0x0009d81e7dfc8d6c, 0x000f57cfca95b2b4, 0x00069313767cd444, 0x00007aa955fa3cac }, { 0x0002cf71b93c6df1, 0x0009d3d81d950286, 0x00020de171a3c63e, 0x0001a2ee30088f17, 0x000063c5a29ef2a5 }}, + {{ 0x0006539c7de5db9b, 0x00043a7e0f222c2d, 0x000b9c92b4c46a68, 0x0003b3230309d42a, 0x000069a4869c5fac }, { 0x000331fb46a1f47e, 0x000277c432ac7d72, 0x00000f4ddec961d1, 0x000121ade72692cf, 0x0000f86aeb4b1886 }}, + {{ 0x0005cc064fc464c6, 0x00034322867e6332, 0x0001e4d0c1c60ee7, 0x000f969a578709a8, 0x00000556972914c2 }, { 0x0008ecec23a18976, 0x0009f1c493a65c5a, 0x0000aa2b3840fd26, 0x000bca2e25fb5d94, 0x0000d962837b0d4e }}, + {{ 0x0003895ce010bd56, 0x000728bb96209bd8, 0x000b30586d00bcad, 0x000d038ac65fa8cd, 0x00004331b11c1539 }, { 0x000e7f5c64f975b7, 0x0001359b56cd3d1b, 0x000b1c7425cdaf38, 0x00092539f10c8350, 0x00009e660c7a4eab }}, + {{ 0x000780dd3a8a4d1b, 0x000253997403a113, 0x0006da9ab974a228, 0x0004b23c294626cf, 0x00003195ea2589e0 }, { 0x0004c61ed6713b28, 0x000e6462baf8e3b1, 0x000d3d0c7ad8714e, 0x0003efc08abe68aa, 0x000057bcb3c222f3 }}, + {{ 0x000a3f84425ebffe, 0x0004614c53352e78, 0x000b88f8502c29f3, 0x0009f2dc85a34889, 0x00007c35ac8f1e9a }, { 0x000c80f9252c564d, 0x00051975224da68d, 0x0007e76a07c43bb7, 0x00015767560cd6e8, 0x0000e6884afd696e }}, + {{ 0x000dbc7e7af4947a, 0x000ba156ac774075, 0x000ec785880677b6, 0x000732a58a1e5ea8, 0x000073eaf3f59951 }, { 0x0004bc8be8ae5699, 0x000ec1b31590a478, 0x00099131e20fb613, 0x0005da6c0039e85b, 0x0000ecfb39af4929 }}, + {{ 0x00021c3831e88d1f, 0x0000a36f20df03ab, 0x0001d3ec9c404310, 0x000bc012a2405438, 0x0000e8ad27a0d2a8 }, { 0x0002abde10a71547, 0x000e959d6fdee104, 0x0003359cb06136c3, 0x000e7b4259da47ad, 0x000064fe3586ecf9 }}, + {{ 0x0008660001e36d74, 0x000a1fc17eab9fe8, 0x00044b810970315e, 0x000c49d2cd6e863e, 0x0000d984aeea5a80 }, { 0x000086431bc9586b, 0x000ac5a2c088eaef, 0x000d91aa75b86dd0, 0x00006d87913afbca, 0x00002d1d48a935c8 }}, + {{ 0x00092167c2a88bb1, 0x0006855c298b9743, 0x0000bf4fb6418eba, 0x0000aec18905fb5f, 0x000010e7e5087856 }, { 0x0000016589c91554, 0x000b92d28c511da9, 0x000c2668c0a43364, 0x0004ea9f0f127c7d, 0x0000d032d8aa0c34 }}, + {{ 0x0008dd39e5a927e3, 0x000c43c11aa600d6, 0x000d36d8716ccbaa, 0x0008f1d682f5379b, 0x00008d24e8fcbed6 }, { 0x0005ff84acc088eb, 0x000e4572ecaa9c3c, 0x000b2224fdbef927, 0x000a8b8c4ba0e103, 0x00004e5b4b7fde85 }}, + {{ 0x000b9042d1372832, 0x0008c22ac95aff87, 0x000b3a142a37f646, 0x000c61d8d1c9bb6d, 0x0000bffd2852091c }, { 0x0000111675a26c12, 0x000de80830cf6bdb, 0x0009beb790506d1b, 0x000c2efaa4861d19, 0x00004ac5fbd62b7d }}, + {{ 0x000f2bed06be1b4e, 0x00088cf1afb20d91, 0x0000213b2ebeaee5, 0x0006a9c9b1369500, 0x0000b3302a11d566 }, { 0x00088a52e02d7437, 0x0004b176d799e206, 0x0006c9e76ffe5798, 0x000db221093415f3, 0x0000011fe4eb093e }}, + {{ 0x000ea4725092fb05, 0x00056a3e2a9a6db4, 0x000d0a54b43a9d4e, 0x000b5eb4f4601303, 0x0000be0291a05ceb }, { 0x000e240a57294dcf, 0x000b898d33fd3e7b, 0x00092d1f2cb16361, 0x000af81ad1ccd8a8, 0x0000360a29a8915e }}, + {{ 0x000acada29bfbe26, 0x000d798ca0e7ebd0, 0x000adc7d87452c91, 0x000bd8499f586783, 0x0000c65aa3596607 }, { 0x0004fd5e3650c743, 0x000937d0fe723ac6, 0x0004d4c60de7e613, 0x00084a7411d8e53a, 0x000016d0a718f6ce }}, + {{ 0x000d2fe3065bcddd, 0x000429057908805e, 0x000c71a49dd73b22, 0x0002f50d0b5b9ea3, 0x00002ca50a57bbe3 }, { 0x00041b9241ca1156, 0x00092926e9197e27, 0x00093734981b7b64, 0x0000dcb3f11719c0, 0x0000987c423fab1e }}, + {{ 0x0001f06bcf00e99c, 0x000df6849a5f4f41, 0x00028dd3efae77c2, 0x0000c5755f91d70a, 0x0000a828a98575b0 }, { 0x000341205886affa, 0x000d85d038d05e1c, 0x0004f78afcf954d4, 0x000dee60b04838ee, 0x0000776112a82e17 }}, + {{ 0x000a0f5ab82fff45, 0x000741364d9c2f41, 0x000386014e477e81, 0x0003658417468bac, 0x000084f5c1ea7f07 }, { 0x000549d8e1b91689, 0x000242f659d0ce0a, 0x000ac23bf70fc30f, 0x0000b09c74011bcb, 0x0000c926584930c5 }}, + {{ 0x00079ff428e80f59, 0x0002fd8c85438b17, 0x000d4485d6fe9846, 0x0003ca4f45680332, 0x00008f4d41216d32 }, { 0x0001d426d9f3da01, 0x000e898f059987ad, 0x0001fa94470d97e4, 0x00062fcb2f6ce8cf, 0x0000101ab31670a1 }}, + {{ 0x000bd0c9b88b7971, 0x000803fc781a2419, 0x000e3b8298f02847, 0x0002fa0843e89360, 0x0000acbc78ffab3d }, { 0x0000be4b4b26462d, 0x00020538a912a4b1, 0x0008ac36cdb271a8, 0x00021b4e0de5e15f, 0x0000a17f8d3dad9e }}, + {{ 0x000be980de325ae2, 0x00032f2668621c9c, 0x0000dfb311e3a6b1, 0x00079993cd5518f5, 0x000052f4db6222a4 }, { 0x000f159768ae1fb0, 0x0008f10ddd83683c, 0x000ccab52fd3a1ed, 0x0006425503b6cb35, 0x0000ef327d2522b2 }}, + {{ 0x0006283e8437a770, 0x0005f5343ad73b3b, 0x000f98db1c104d18, 0x000473c691b528c1, 0x0000c3c4141f72fc }, { 0x0003347161a895a5, 0x00024590a5f25b1e, 0x000475107cc11b68, 0x000483fdeefaf8aa, 0x000020a387cf4433 }}, + {{ 0x000de20ec9734397, 0x000ab198d9ea9f87, 0x000b761a0bec4dd4, 0x000ae95da090de5d, 0x00005d35adbfe798 }, { 0x000182de8a62cc1b, 0x0003d8eecdaecabd, 0x000cb6b148b91280, 0x000dd1dec6ca4b0e, 0x0000550839c85391 }}, + {{ 0x000cd9daab10e14f, 0x000763ec8a4186a5, 0x000b3793b388c571, 0x00056173cb3e878d, 0x000035bc8bc8154e }, { 0x0009c5d26f779d48, 0x000d17f586d5e720, 0x0008ba7df7ea2a57, 0x000c7dc2447500be, 0x0000ece6c6ab436a }}, + {{ 0x000636393a1d534b, 0x0005c9aa74d3f7b3, 0x000d99286b1f5422, 0x0007e296d0428fd8, 0x0000b23839cb4288 }, { 0x0008021fc06fa7bf, 0x0003a5510350bf49, 0x0008c8a8c7c25f86, 0x000c0c909b381743, 0x0000c67eee6023ce }}, + {{ 0x000caffa79164dbb, 0x000ec0c0e367a98a, 0x000462b2bd497a28, 0x000b04843bea1bc1, 0x0000c78fe367131a }, { 0x000b512e742ec763, 0x0003664457bf9c43, 0x000ad1dd7f0fc57f, 0x000ff90eb0db6407, 0x0000e05c4ba13983 }}, + {{ 0x0007f983c7b84b15, 0x000f91ffce104080, 0x00035264a1b62bef, 0x00017d0b389ddc00, 0x00002960ff9e4a4b }, { 0x000967d1accb38ae, 0x0009f155d5231e7b, 0x000c51e7c366f7b0, 0x0009802b5cff22ae, 0x000081c1f665309f }}, + {{ 0x0007012c7d47081d, 0x0004ec6cf777b6c0, 0x000d456becdc46a4, 0x000edaf850062bdb, 0x0000cc93e16237c7 }, { 0x00095cb26b4e0be4, 0x0001562ac47800d1, 0x000ba7a63b09f80c, 0x000ace5d353365f2, 0x000034e8d2215ffc }}, + {{ 0x000f81609ee40822, 0x0004c2905f2c4ce9, 0x000af8d535cd568a, 0x0004a4e15bd2bdec, 0x0000cfa19d6c9ba1 }, { 0x000241b6f9fad805, 0x000662cda350002f, 0x000007d9c63cdde1, 0x0004574bcbd05469, 0x000047e52739fbec }}, + {{ 0x0005ce69cde21889, 0x000fd432fe3cd4c0, 0x000baeedf96e9a6a, 0x000fb161fe0f33ac, 0x00001790f40c1e5a }, { 0x000b8e29796a6e6a, 0x000b116f5f98aaf9, 0x0005d395b776372e, 0x0005f93d345bb71d, 0x0000430258e053dc }}, + {{ 0x000fb4879b3fe9bf, 0x000f346ae0a6828c, 0x0003a0c44e0f5c7f, 0x000cb3cdbba533d2, 0x0000f6ea637a253b }, { 0x000dbf6a4042ef68, 0x0006b0b3281705b5, 0x000be2907641494e, 0x00070e6df7e361c6, 0x0000876051625bfc }}, + {{ 0x0000e78507af85ef, 0x0008d234b54d9bf9, 0x0000a42a3ed21495, 0x0002cc8581374e68, 0x00009d3702290f12 }, { 0x0000946dbd4e2626, 0x000d38d5d86f1749, 0x0002f55502275d25, 0x0009fad44d74e0bd, 0x000004488f9be27c }}, + {{ 0x000f710571dd1d02, 0x0002158fe1a838c8, 0x0004d299c4671897, 0x000b1580b05aae6c, 0x0000e3dbc53866c4 }, { 0x00091241f3b00662, 0x0007b48182caaac9, 0x00096af503dbdd89, 0x000b2dc6d8c4aeef, 0x0000048abae31743 }}, + {{ 0x000ffdfc7e7c2a09, 0x00085eae457a4773, 0x000bae6fdf8723eb, 0x0009a0d71d19857d, 0x0000d6ef525ea59c }, { 0x000d15002a515a26, 0x0002c5426ee7cda3, 0x0009a1edc37de8e0, 0x000e199c8341b086, 0x0000bbb468e51820 }}, + }, + { + /* digit=31 (1,2,..,64)*(2^{217})*G */ + {{ 0x000f6927b0d79bd1, 0x000614c85edfa308, 0x0003f86bcc29875b, 0x000193b862597655, 0x00001c7d62051005 }, { 0x000d0721ecd294e3, 0x000feba55f2f94c5, 0x0008d744240dfdf9, 0x00015625d6a996f9, 0x0000e456c18ca0d6 }}, + {{ 0x0004cae01e64c20e, 0x0009b2079e5fb67e, 0x000a8bdf924006e5, 0x00033ca37e1331fe, 0x00003ed077fad719 }, { 0x000c957822ca746f, 0x000cfe60412a77db, 0x000c22af18030eaf, 0x0007aeb3106ff7ca, 0x0000cacc54eeaa59 }}, + {{ 0x0007e0aa7adc7a48, 0x000db0cb0ce1c552, 0x0004e3df0b8cb07c, 0x0009bee3b5b534d0, 0x0000fda2b88e9831 }, { 0x000f7d9eeff2ac2e, 0x0007e2d79362c410, 0x0000823dcdc0db71, 0x00077b12467920c9, 0x00001801931c732f }}, + {{ 0x000f5fa5904e2bd3, 0x000209afa8338349, 0x000206d2dfd21b18, 0x0004529102295172, 0x0000fd30a26b44f3 }, { 0x000137b286ed0846, 0x0004125e77d9a3b0, 0x000a624d3959c964, 0x0003d9c4a11235ce, 0x000037f27954916b }}, + {{ 0x0000d262f88de12c, 0x000b5ce44ace1509, 0x0008b2af868437a8, 0x000e478b8e0f8d3d, 0x0000f156f86418d4 }, { 0x000dbdf5922834ef, 0x00065c9283e13c98, 0x0007211e7af3d393, 0x000a7cfcb2d04fde, 0x0000f09ff470d31f }}, + {{ 0x00073c8530e34004, 0x00002950de1e5786, 0x000782baa695ecd5, 0x000f9e8a52e1463e, 0x0000607baa031037 }, { 0x000c1d160b022745, 0x0006100747f91da5, 0x00016dae62deceaf, 0x0008e89f5b08d43e, 0x0000e0b5caf53925 }}, + {{ 0x0004669f46350981, 0x0002d5328c8e13bb, 0x0003fe873d8bc483, 0x000c485ad375f10c, 0x000035558e6d3927 }, { 0x0009293790f894ed, 0x0000cd05bbd8699b, 0x00092e6e3582dcff, 0x000e93bd35528a4c, 0x000007788f960423 }}, + {{ 0x00061bbb05f9956d, 0x000106ac42bd6d29, 0x00082503dba8e1c1, 0x000ae458e6df8646, 0x00004572780d144e }, { 0x00004d881133b185, 0x00070f070a6a26b8, 0x000cb240e6288319, 0x000ae43d370686c0, 0x0000da343e03be55 }}, + {{ 0x0008584ef74bf3d5, 0x000b27b207f16700, 0x00084609039a722e, 0x000afedbf9b0dd18, 0x0000dfa5f66ee54b }, { 0x000c334771dc5db5, 0x000dba5dcd898ffc, 0x000f5425a6cc0213, 0x00029fcc126286c3, 0x0000264ddb1fa583 }}, + {{ 0x000725af16318964, 0x00085542f770fb1d, 0x000f75b1a448a814, 0x000806afc97c1c61, 0x00000e2e1a79320d }, { 0x000e079d3c015b41, 0x000914fc01e20ecf, 0x00039cdf74e30b3a, 0x0001c76ecdfd3749, 0x0000e980c17f36eb }}, + {{ 0x0000d6a5c42da56f, 0x000780a2e6ac8d23, 0x0002df97b64102c1, 0x000402c8c9c6b5a4, 0x0000724fa29031c0 }, { 0x000d131ab892404e, 0x0002e224fc0a9ef0, 0x000388ae52355bac, 0x0009dc1de190ff08, 0x0000ec43401fd702 }}, + {{ 0x000eec726a98db71, 0x0006f7e857012919, 0x00087a53ef3da9d1, 0x00019338594793ed, 0x0000b72ad73d6c3f }, { 0x00032b2c8ec764fc, 0x000be590c9d8005a, 0x00077b4e8d42c947, 0x000eb6e1e5feca45, 0x000035b3f8239855 }}, + {{ 0x0006c04225a0b97d, 0x000447373dd0b4ea, 0x00039de012a075b8, 0x000d1d6f137a2821, 0x0000c37b6f44f5ad }, { 0x000f54feb8e2353a, 0x0001134cf671e3ce, 0x0001de54badb3643, 0x00075fe4881bbecd, 0x0000e52b6354c0d0 }}, + {{ 0x0003469898a0538f, 0x000eda7fb8088fa5, 0x0006fa95fe2d2ec5, 0x00023e2760142724, 0x0000b5efdf4b7190 }, { 0x000e706aca43c3d6, 0x00061b83a2aaac68, 0x0002cc8322d12925, 0x000fb5299e9f91dc, 0x0000dfdaec18b984 }}, + {{ 0x00093ea6c3ee3c56, 0x0001e95f688dc983, 0x0006d90da43852a0, 0x000dc215c26490cd, 0x00000e9b4f9bc030 }, { 0x00059cacd2538ba3, 0x00090437f338e0d8, 0x000adab73ae2ca93, 0x0001271d539fb832, 0x0000b214d5f9080e }}, + {{ 0x0001567f755e516f, 0x0002de74462007dd, 0x00047b5f76420568, 0x0000d6ee7b8ab48a, 0x00004f2bc1635d97 }, { 0x000931de26c2af42, 0x000d96b0887bcec5, 0x000e8847159b8388, 0x000b324cb694497c, 0x000039c7e289bc5b }}, + {{ 0x000bd0a121216b32, 0x000ad7c0e58d4939, 0x000fbce69df1668d, 0x00035914db1ae5ec, 0x0000c149038863bd }, { 0x0007f13e98b5b5ed, 0x0009bb7cc863c683, 0x000e052e0eccfa57, 0x000f268933a1cc11, 0x00008167fd8f4e54 }}, + {{ 0x000ee1b6170663fd, 0x0008967bc947badd, 0x000d4b5538170a6f, 0x00013ffac738e07c, 0x000023f0b6b7b6bf }, { 0x0007cdc47eb25821, 0x0003d672e793d12f, 0x000de923c25b844a, 0x0008859f7d6aa6ca, 0x00005201d76c5ace }}, + {{ 0x000236ac0915006e, 0x000ff9dab15247f0, 0x00004703a77613ae, 0x000a2c6e77c789c1, 0x0000808d9373070e }, { 0x00042da81b066709, 0x000b4251d7a51446, 0x000a8d111cd1a17e, 0x00089a2ca098b9c2, 0x000013d8733bafcd }}, + {{ 0x0005441757848811, 0x0004b054f7269b13, 0x0007ad43ac2df8a6, 0x0001c5e21fcf3077, 0x00008d6031c1f86e }, { 0x000728edac03ee8f, 0x000e999fd703431a, 0x00056fb46aceed43, 0x000d9a62da438d7a, 0x0000f9c288edabe5 }}, + {{ 0x000356c23a712a04, 0x00086bc42531689f, 0x00070db2a1b53dad, 0x000edd5b30a51a0e, 0x0000c4d54a3bd065 }, { 0x00050964756fe418, 0x00027ff8d3a3dace, 0x0009588ae4a9b9ac, 0x000663ea4409597f, 0x0000d6d7b2598a4f }}, + {{ 0x0001c02c179a9537, 0x0001a265c3427b0b, 0x000c20c29da60ee6, 0x0009725456401405, 0x0000041eff03bf87 }, { 0x000a43463e984012, 0x000db75d9635bc92, 0x000743e10ef12662, 0x000b1be66226790c, 0x00006caa30be64cd }}, + {{ 0x0009f2f368831642, 0x000033adb4837611, 0x00054be273c309ad, 0x00028ba87e7cc608, 0x00001790a225635b }, { 0x000590dd601ac1f6, 0x0004d175c0b800bd, 0x0008b5dcc9827fed, 0x0008cd15e1f1e7c8, 0x000021b22c5fb59e }}, + {{ 0x000b15d6243f1091, 0x0000ca11f17a34c7, 0x0008a8443e31d5d4, 0x0003fa53b5420ab3, 0x0000927b5e2d1cf0 }, { 0x000424051138a243, 0x000fdb1e274e49c4, 0x0004528d80f9684c, 0x0000da2a45cf5074, 0x0000abcad67dae2e }}, + {{ 0x000248c0344ff4d0, 0x000734311b2d595d, 0x00091c801f234930, 0x0007f0d05f136749, 0x00005dba5d3c2cee }, { 0x000e8c07a9b72e10, 0x0003f13a77739f5e, 0x000a3546a79917e7, 0x000b691cfbf4288c, 0x00009c7306d20c7b }}, + {{ 0x00089e1462801321, 0x00028996fe8393f2, 0x000fc08125d060fa, 0x0001c42c80f809a3, 0x00009fb87e79ba00 }, { 0x0001826047e09dee, 0x000287e9889feea9, 0x00024f16d8c20c68, 0x000b29cc6d5c9c0e, 0x0000ca96182e9075 }}, + {{ 0x000a074b4cbdc635, 0x000527a9a4a27515, 0x000270dfb3609166, 0x000e3e44171bd007, 0x0000f49c9e777f48 }, { 0x000fb0efe038c226, 0x0003334c1cec8edc, 0x000ddb0b13dd9e14, 0x000c85e15a6ecb49, 0x0000c013efc7e2a5 }}, + {{ 0x0003ceb3f5d13058, 0x000edee23ab3495c, 0x000747a89247407e, 0x0002caef4a132df8, 0x00002b96856566b0 }, { 0x000c0899c9fa2622, 0x000071de88fb1da6, 0x0003b19887afb52c, 0x000d8d520ea05797, 0x00000f3ef4a9c2d8 }}, + {{ 0x000f5d86d67ee298, 0x000dd04752ddad71, 0x000f824903c11932, 0x000766399d60bd74, 0x0000bacfccb4d377 }, { 0x000571960bc9fc41, 0x0007b424b5a6d88f, 0x0002254d6471eea8, 0x000cd841069eb2c3, 0x00004e6a49e4dc49 }}, + {{ 0x0009b988353bd1dd, 0x0001ecf704e23c66, 0x0001bef11e2642bb, 0x0008c5f83c71b7d2, 0x000034725a0787e8 }, { 0x000a08f0037397b3, 0x00074a3963bfeec0, 0x000ad474b7c8163a, 0x000d0f6b199947aa, 0x00006b872dacd87a }}, + {{ 0x000ed489e3948b33, 0x0000d9478ef24870, 0x0004f3fb6243555f, 0x00051c028c8d2d41, 0x00001b4ae6438b23 }, { 0x000b9784bc1dfe6e, 0x0002a6bbef3366e9, 0x0000524658fc7d96, 0x000df6de20de59c7, 0x0000fc82425dd0cc }}, + {{ 0x00084e81ad96c3d5, 0x00080021a93507a4, 0x000744ed85217d67, 0x000286a40b4cd118, 0x0000702de63abcfb }, { 0x00077e27e30a727b, 0x0000cb5272e9d6ec, 0x0004ff812967789d, 0x000a6af8eea1c93e, 0x00003caac07df9b9 }}, + {{ 0x000fcc837acc3f62, 0x0004936d2b713c5b, 0x00000fae8958d09c, 0x00070206b86274ea, 0x00001d84b95f783d }, { 0x000f1a38bcdc1a88, 0x000769dd6529b73d, 0x0007848e4745cca3, 0x00002bdb9708aa22, 0x000012697ae997ed }}, + {{ 0x0003317d153dbf41, 0x0005926b8bad853a, 0x000293f26dd1f0cd, 0x0000ae7302d52cea, 0x0000259fdb316c19 }, { 0x0006bc5834b6d375, 0x000edfefd4ddedab, 0x0009fea120f6e9b2, 0x000f37444ed2a153, 0x000027704cde3608 }}, + {{ 0x0004269d64c13d9e, 0x000b4b383d102b6b, 0x000652201ae42133, 0x00030cf37fe695a4, 0x0000c2b2d20a73d0 }, { 0x000282e5be5c5683, 0x000d3a10a1d22850, 0x0001444f87b4312a, 0x000fdb1a7feef6c5, 0x0000eff77ff509ef }}, + {{ 0x0005aaf78c7288d5, 0x0001c81721c486d5, 0x000066ad8ac27b77, 0x000ce2b31301baf1, 0x0000f49e14841d8d }, { 0x000dd8c0882b51a7, 0x000ec7159c639522, 0x00045de99b26fb30, 0x0009386c33a3bd27, 0x000050ea586ccf8c }}, + {{ 0x0001dbe32c1fbaaa, 0x0008874f5531461d, 0x00008bb9666f009a, 0x000c64fd28b4a128, 0x00008430a73d0dfa }, { 0x000431531ac44d42, 0x000df0d30c34169a, 0x0002d4895218eac4, 0x000ebb054ba74a95, 0x0000755d2f2b53df }}, + {{ 0x00006ae448efaa11, 0x0004d22f669d7ec1, 0x00042514d0cc257f, 0x000f9ecb3ca374f6, 0x0000f169c333b6fa }, { 0x00010a7d3f6b9eaa, 0x000609033c8e9a1f, 0x000180a2db4623fc, 0x0006c4314234645e, 0x000084b79c5ee4b0 }}, + {{ 0x000d8727967cf447, 0x00097ee40d5c9be3, 0x000f7cfdc601ff1d, 0x000e146ddb2bae7f, 0x000039d35690a32d }, { 0x000213071385c0a3, 0x00027818569ff132, 0x000b784d085ec27a, 0x000ae88868b89a5f, 0x00001eb069c3b03a }}, + {{ 0x000098baf9cb4cd3, 0x000b81e48ac28403, 0x0004bc21d97de9e8, 0x0004798431831129, 0x000013750d1196db }, { 0x000ae9e34b83b95b, 0x00066584198da522, 0x000be98219cfe30b, 0x0007d4e08ab4fc17, 0x00008387d9c3f13b }}, + {{ 0x00010ea88587f6e9, 0x000699e3fa11b1f7, 0x0007263c4a2f6c1a, 0x00039903b21e4367, 0x0000789308d6093a }, { 0x000722a5d5edffef, 0x00032fcc5dc67cf9, 0x0006f11db1430e0b, 0x000e4afb708d5164, 0x0000f530330b49e1 }}, + {{ 0x0006370339f57e7c, 0x000d4e2c861baa5c, 0x000ccecea904d470, 0x000fd8a3fc0c21a5, 0x0000fadeb312f995 }, { 0x000ef031de2f466c, 0x0002dc7638167c3f, 0x00057a6d812df399, 0x0003edd89fe6ffe7, 0x0000825bd8675575 }}, + {{ 0x0002fcb419b70685, 0x0001a82d57c667f8, 0x000e490ec2d75faa, 0x0000b05ecc70312c, 0x00004cf30e7acd12 }, { 0x000d8db5533f5c10, 0x000a385ab45d44d1, 0x000243edce4fdb2e, 0x0008c0b52fba0722, 0x000052d22dab8a19 }}, + {{ 0x000dc5e37371a728, 0x00090d4e344a8732, 0x000dd070f1cabef0, 0x000cebf507d8d116, 0x00005db19373e506 }, { 0x00072b8ab82adaeb, 0x00047e6e799eb95b, 0x000eb16e5c349584, 0x000d038d5e9e5f4d, 0x00009e31731daff4 }}, + {{ 0x00089197287c3914, 0x00066e02cecc84ab, 0x0007c349eff41179, 0x0008b5b58600016d, 0x000061064bc7eb5b }, { 0x000c91e7660a8876, 0x00045bddafd60884, 0x00061f4025d3c53e, 0x0003b9a721a74014, 0x0000074a6b09578e }}, + {{ 0x000f2874056008c5, 0x00020ca08819a786, 0x00095a30051c1c8d, 0x000b3077dcf48b72, 0x00004df9fccd5a7e }, { 0x0007e550b59c1a48, 0x00012c226cd44ecc, 0x00060804d4595c21, 0x000eca3c588ae32a, 0x0000363ef9fbc480 }}, + {{ 0x00061b1ca7d7673a, 0x000e2f25c727bd2e, 0x000036ddf9088a43, 0x00035ff202e4badb, 0x0000a50c99cb451f }, { 0x000f2e941699137f, 0x000faccbd6bd51f4, 0x000131175819f7a0, 0x000db8f628024e40, 0x0000d71497878c8c }}, + {{ 0x0009f80d55b0c703, 0x0003dec0ccced589, 0x000b73ac42429524, 0x000510fc625cd4b9, 0x0000a65aaf5a02d6 }, { 0x000f34bb38b3eac6, 0x0007ac9ce6dc1532, 0x0007a93199e8a328, 0x0001c3b4d138d511, 0x0000ca319150839a }}, + {{ 0x000b70ab66820487, 0x000c8d22ae8921e6, 0x000e1dbaf67c7245, 0x000bcf9b2b039288, 0x0000edb438dd0e2f }, { 0x000b5629d34a9d60, 0x0004bc9e70574810, 0x000f912ab26e33fe, 0x0008be6824581dde, 0x0000b84bd043d606 }}, + {{ 0x000fbfb416c23b23, 0x000a49eb06f5b41e, 0x000d5e6480526105, 0x0001f4ea9a42e84c, 0x0000cccc1d701e42 }, { 0x000897f9fd8a78ab, 0x0003f239d33cd6f3, 0x000871d410fa9b73, 0x00055c4969caedb8, 0x000057a5bd2f9d4f }}, + {{ 0x000f306e8d3ef41b, 0x00096f88e5d788e9, 0x000842a42261b628, 0x00064b03c1aec7ba, 0x0000bdc61d3e538d }, { 0x0002865801e16113, 0x000ed6e5040a0acd, 0x000328c1172c7e8b, 0x000896723ca9d504, 0x000064b076c1e6dd }}, + {{ 0x000671c6f1b51569, 0x000c03f6967469a8, 0x00090718c7e7267c, 0x000b208222894d83, 0x00005bb66d861506 }, { 0x000fc913dfbae46d, 0x0008bd6d7de9853b, 0x0005f229a8e82949, 0x000b8e3cf2e3ce34, 0x00006ed57b771cce }}, + {{ 0x000dbe9e4865fdfb, 0x000959fe61f7908e, 0x000eadfddce11bc6, 0x00006b838192e304, 0x0000a93c2ce3f05b }, { 0x000ebd91e9d1f474, 0x00075be2db30660f, 0x000ec22deba43a12, 0x00039956eda3e613, 0x000030ac3120ad60 }}, + {{ 0x0003a058ce6c6132, 0x0009ac4df655a499, 0x000bcb11a94222b2, 0x000f584eaa9e00df, 0x0000155d23f64159 }, { 0x0003455bbbdf94ba, 0x000d64a67bd3448e, 0x000a104fa1a088b1, 0x0008da5524a3d4e1, 0x000055bd521fe9fb }}, + {{ 0x000381dd5350039c, 0x000fc5b4b9ad1ce1, 0x000648361a401b32, 0x000ae0816a981853, 0x00003aa8a7a07cf8 }, { 0x00041e541e27a4b9, 0x00054617a4a76209, 0x0005a59a402eaafe, 0x000f30a3ac841292, 0x000004cfa2d4a052 }}, + {{ 0x00061e15b0193f73, 0x00026e84a34f239b, 0x000c02e5ed252835, 0x000fb353215fdb83, 0x0000611a80a40f29 }, { 0x00053277336c58d7, 0x00082d4a127f89a5, 0x000bb2b59fa73e65, 0x000bfdf925d541e0, 0x000059c1e0662639 }}, + {{ 0x0000309532509fa6, 0x000fafc95f57552b, 0x000fbc7c1b975eb3, 0x0008218013294764, 0x000077c5af065b77 }, { 0x000e03f65c2dc5db, 0x000e2f505c2e54da, 0x000c78e4e6c62bf5, 0x000b84353158f27a, 0x0000694c1d649f0b }}, + {{ 0x0007332f3b061b87, 0x000c7bd9c95f0f98, 0x00019d30d836a35c, 0x00039dac45ed9deb, 0x000018b00ca2bbfd }, { 0x0005cb917ba93167, 0x000079e2b9e0d888, 0x0003d0d4cecd60b0, 0x00027c84852d910f, 0x0000f373e4a9fc00 }}, + {{ 0x000137a6d994065e, 0x000be934c8998d19, 0x0000e1d6feb6644a, 0x000fc3fb02186a3f, 0x00003352e74d4d1d }, { 0x000ca1b89b828df6, 0x0008937748bc82c0, 0x000173a4e6f0ad53, 0x000fb24a287c2e93, 0x000028fafbaf323c }}, + {{ 0x000737d6fb05a03f, 0x00004aa5e1cd3cd7, 0x00082350a52d4bcc, 0x00049e09f8939780, 0x0000f18ec22e9ed8 }, { 0x000429759f59ae9f, 0x000d56a60f1524b3, 0x000585dbb50813e4, 0x000e39fa536985e5, 0x0000b06736fc8393 }}, + {{ 0x000ac8aaf9b029b8, 0x000b6c3092e9f414, 0x0006175cc0c40ea0, 0x00083907d3a238c6, 0x00008d3d8a65f0c0 }, { 0x00090005fc438ce7, 0x0001ab15e4ccab05, 0x000244e145fefc14, 0x000a4a29d9bf430f, 0x0000798b9c3dca21 }}, + {{ 0x0000075aacd2e5f0, 0x000a12f24544cb60, 0x000ad7936836f151, 0x00072b2129987b71, 0x0000b2e6c26f30ed }, { 0x0007813de7e02a59, 0x000f78b6cac83696, 0x00078dce2ea1a080, 0x0009ad9a487842be, 0x0000b0205681558b }}, + {{ 0x00040896be942344, 0x0001dcec3427239e, 0x000fe1609e1ac9a8, 0x000f5d1915f3c7ce, 0x00006570948e3d4e }, { 0x0003d4708d9dd274, 0x000f3910492d25f2, 0x0009c36c11e8ef73, 0x000b536cc8aeb30c, 0x00006263417807f4 }}, + {{ 0x000f9b71b8c0f425, 0x00077f2d0c051995, 0x00029b4c129bcbe6, 0x0007e7c82502fbc2, 0x00002b22165d2734 }, { 0x000b6e3e8373047f, 0x000bb592b82dd077, 0x000519b4c4ab6dee, 0x0004bdb327630273, 0x0000227777e79e51 }}, + }, + { + /* digit=32 (1,2,..,64)*(2^{224})*G */ + {{ 0x000c41114d0d0f0a, 0x00071c035d0b34a2, 0x000b56e6af5ad632, 0x000f458d1440b386, 0x00009070851ee09e }, { 0x0000477abf63470d, 0x000c1f1ad95a0b12, 0x0008478dc8a2c85e, 0x0009d79c9c09b37a, 0x00005669d660129e }}, + {{ 0x000a68df70882277, 0x00065b3292a92874, 0x00062d47b35717e2, 0x000498f05ddc15cf, 0x00002045f41bf3f8 }, { 0x000a8b9343580755, 0x000f94505bf7dd0a, 0x0008e243ec49440b, 0x000aea3afa4e63d3, 0x00000f5462133be9 }}, + {{ 0x0006113c503cd9d4, 0x00061b51e706ad97, 0x00044d98af8ed595, 0x00086b990b99cebb, 0x0000a86e1c215f82 }, { 0x0003cbb144e6b9aa, 0x000e4b097e2b5aa3, 0x000c2fed61bf9a24, 0x000125c6c7e1022c, 0x000044eec8aec086 }}, + {{ 0x0004e7b4f75c69c7, 0x0009d717af715d2e, 0x0003eb2b959f67ad, 0x000b50256e2f7f59, 0x0000faa39a85f847 }, { 0x000657624d4d6888, 0x000779788d5374a6, 0x00031a2adb0e9860, 0x0008607e22b915e6, 0x00009ed17ced0865 }}, + {{ 0x0007e49f538144e9, 0x0000801dace5aca9, 0x000179c203139436, 0x000579d09c4fdbcf, 0x0000b8c43e3ced43 }, { 0x000f036040802177, 0x0007090937e2ad5c, 0x0007636ab7b11bc7, 0x0009a49dc846e250, 0x000015f05617398b }}, + {{ 0x000858e1e42fa26a, 0x00096e07442f1d58, 0x0000de8801c7a755, 0x00023d647475607d, 0x00002129ca073288 }, { 0x0007cdea3e2c120d, 0x000050b8231ee10a, 0x000abbbc34902c47, 0x0009866a41b80e7e, 0x0000ea4fb6751c9a }}, + {{ 0x0005b9d57b4ca325, 0x0009b07f8ae7c38d, 0x000b67ba2c17122d, 0x0004048b36db07a5, 0x0000c13547ebaf13 }, { 0x000fe5a101822457, 0x000dbe78eba20f2f, 0x000e71d40250d287, 0x000744f58ef11ca8, 0x000067b29ced3d7d }}, + {{ 0x000af127793627f3, 0x0008811e51732d26, 0x0007ba47d495211f, 0x00011fbc5dfd6eb2, 0x0000d0277a7c5305 }, { 0x00059aba7caa2e14, 0x000712d06c425aef, 0x000b6687611ec682, 0x00054599c6df92f8, 0x00003f2120a92224 }}, + {{ 0x000eee4194a7b79a, 0x0003ed5da64309d0, 0x000682c4c56fb4d7, 0x000e3be0861a6de5, 0x00003c6f7e87d587 }, { 0x0000847a2186a0fc, 0x0009eab771e4caa3, 0x000d4837a79d473c, 0x0001408f583965df, 0x0000b258cc7ec22d }}, + {{ 0x000c06085b99b726, 0x00006f152bfda056, 0x0008f591b1313934, 0x000ea4c6fe0e4c4e, 0x0000984ae625115e }, { 0x000eea86f0bd9963, 0x00061d74463ba14f, 0x000201459c9b5827, 0x0008df06d9d292f8, 0x00000dd1c0ca9a79 }}, + {{ 0x000c48ec0b853213, 0x00079d366131e2e7, 0x000f04b53a77349f, 0x000e3ef89f31d974, 0x0000a4bfcdc5045e }, { 0x00080a6ebc348716, 0x000fbd8d1989aa7f, 0x0009f3eeb83d4cf9, 0x000be314ab34fa0a, 0x00001217dc8a396d }}, + {{ 0x000f4dca515118cd, 0x0009354afa05dd8e, 0x00010d61474d56e0, 0x0004bee5926dfcf2, 0x00000943d9f591b4 }, { 0x0009b587257c123f, 0x000448a65d306723, 0x0004c3aff7615195, 0x000c356c88b08d53, 0x00005b8eb7fb6fca }}, + {{ 0x0008c318b548154d, 0x000c3593de9abe37, 0x00012418eb094690, 0x000d8818c4043488, 0x0000566a78469087 }, { 0x00030ce97683fa8d, 0x0004216fde24c99d, 0x0004cfb615e6cd11, 0x0009b82b42a77cb7, 0x0000a14d76ffb4a3 }}, + {{ 0x000ac78848467605, 0x0004e931a62cc262, 0x000625f9580ab732, 0x000502ef323c69b9, 0x000090bdd65bcd79 }, { 0x0008b8974d570b21, 0x00057f9187d073d8, 0x00090dc6e92ff16d, 0x000634c8f6c2be91, 0x00009044aaeb8adc }}, + {{ 0x000c764803a21e34, 0x000dd77ff57d0517, 0x00077ef3d15d6733, 0x000b936d42ff3803, 0x00001b21b2a67101 }, { 0x000a30e0b2c6a5f0, 0x000e75f48879bc89, 0x000fbe9d8b0eae8c, 0x000e37c16ea7eacb, 0x00009ff4968601c7 }}, + {{ 0x0006605ecd65db55, 0x000d72e421d3aa42, 0x000cc1ef49735da2, 0x000798f1cf926407, 0x0000115826b66fae }, { 0x000b337ce7ef919a, 0x000a7a6d6a5eabf5, 0x0003c637e9a63491, 0x0006f67021edb84f, 0x0000746c950ad014 }}, + {{ 0x00034288799ea710, 0x0004c77c055dc14a, 0x0004de9614663360, 0x00056bbaf2a0ffa2, 0x0000e0884c9d70d1 }, { 0x000fd61e66f6e09b, 0x000fecf884655fb3, 0x0005f6771fda3095, 0x0003d81b7fa5b5bd, 0x0000cdb900fd5f4d }}, + {{ 0x00059455d0433f8f, 0x00013dccc78cf662, 0x00044a47a9a7232c, 0x0004db768b301817, 0x0000faaaf90a0ad3 }, { 0x0005deb6ca3cd246, 0x0001ec5cde3acec5, 0x0004bdcefbacb8b3, 0x0005df9657714192, 0x000045397101934d }}, + {{ 0x00068f895e9004fd, 0x000b957b1fa70fbf, 0x000ee177b0a26d56, 0x0009b2800fa07f50, 0x00006b90a3ae492c }, { 0x00066663c21efba8, 0x00078605cae2a6aa, 0x00075c20e4801115, 0x000e8d3d301bd721, 0x0000b7b5d77ccf5d }}, + {{ 0x00030d7189e49226, 0x000dfd73272d8383, 0x0009c755243ba1ba, 0x0009db432e22924f, 0x0000ce330e0db587 }, { 0x0000a894fbc9777d, 0x000292cbde1f7140, 0x0004e5228567b078, 0x000a5063235771c9, 0x00004ff8170ebc9a }}, + {{ 0x0002b9e66ddcbd1a, 0x000899b36194d408, 0x0004f4a73f93d7fc, 0x000121558e857a7c, 0x0000f54e23d16472 }, { 0x000c57761e509ec5, 0x000a434e4e8b9dd4, 0x000d57cc01b97144, 0x0009c1f3bd2372c9, 0x00008d5623e3a8bd }}, + {{ 0x000653a576829f13, 0x000ba0a1058a3184, 0x000531b195245a3b, 0x000277ae2369cf3f, 0x000016030f4bd9af }, { 0x0007f95f135e7fcf, 0x0000e5aaa4e5c975, 0x00015fb1dd60b809, 0x000e5f3835be80aa, 0x0000a477556e695d }}, + {{ 0x000d4690afc07e00, 0x00014e24415503b6, 0x0005df748739e313, 0x00060c966c08ff7c, 0x0000127236d69e17 }, { 0x000d769c04659257, 0x0008c2eea74237b0, 0x000ebfe0b48a99dd, 0x00039e9fec436d17, 0x0000486dc5e3dd88 }}, + {{ 0x0001787c9e286fcf, 0x000e7de34eaacda9, 0x00079a28754099d0, 0x000d3bf55d9e116e, 0x00007562dbb12950 }, { 0x00014f95d0e6cf70, 0x000e32bf129ac2fd, 0x000788e74c5ba2af, 0x000d048f3d307b7c, 0x0000f8a713e0a48d }}, + {{ 0x000d063b675b0d68, 0x00068296ae550430, 0x000f37a2427aa344, 0x000d796e28931e98, 0x0000fa3386495eae }, { 0x00040f65cbe226f3, 0x000a5b9ed0e4316a, 0x000a8c5b647a8e82, 0x000b3a4c7e74a736, 0x00005f2b0889266d }}, + {{ 0x000497e5c3ee0d06, 0x000300416df4285e, 0x000aec2d851e2818, 0x000b8c9c3b0c9bab, 0x0000197c784b55e1 }, { 0x000c8f0069e898e6, 0x00085b4164bda960, 0x000994284d372970, 0x0009f40ba1ee4241, 0x0000ca6bcd47d957 }}, + {{ 0x000394f375958bf5, 0x000f481c27253c98, 0x000b0f1e76cff0db, 0x0002bc64605a6aef, 0x0000f0b8234cc107 }, { 0x000753d452dd6652, 0x000759c5f109d0fe, 0x00057438f1b79715, 0x0008a4c68cad88a7, 0x0000fe76d824187b }}, + {{ 0x000c362a70e2529b, 0x000d9a15b0ec6f5e, 0x000366538847e1f8, 0x00084617347989fe, 0x00001b6a7bf51376 }, { 0x000a56149e2522b2, 0x000e6a7b3a8ad418, 0x000ff10dc6b2e1e9, 0x0002983f473a20fb, 0x00006887f8bed5cf }}, + {{ 0x0001f4d76b4d26fa, 0x0007e819ac6e0f4f, 0x000eb21ede824bca, 0x000d9b6fd360fdea, 0x00003d6157db8955 }, { 0x0005570e9aed5fe8, 0x0006909587f76b89, 0x000c8dc0ef1ab00f, 0x00008c8958b4ee08, 0x000053e011751c9f }}, + {{ 0x00043ef52d2d581c, 0x000a76f35d33ae72, 0x00032c7c6567763d, 0x00057cc4b200ea12, 0x00009729e0b08fde }, { 0x0008914cb19c5b91, 0x0003477e23ae233d, 0x000ecbcf853a9cd9, 0x0009ee5a4b93807a, 0x0000427af498005c }}, + {{ 0x0008f30be32aa9eb, 0x00099943fb42622f, 0x000b5715fbf20368, 0x000ec01633b2700a, 0x0000a02527da25ef }, { 0x0004d487b147ea98, 0x00060d269e55f156, 0x0001673e9c38aac8, 0x000fb56f61fead85, 0x00002fdd4c705f91 }}, + {{ 0x000add3db7ec68b8, 0x000b386d23ddc820, 0x000c732406385f9e, 0x000123ac503fd344, 0x0000078adedd4745 }, { 0x000755e7ed4c6729, 0x000153f8260e01bc, 0x00048d4be4a45000, 0x0008bbb33fbeb49e, 0x0000d816465d0546 }}, + {{ 0x00045bbb58e226bb, 0x0009639296077453, 0x00006e136fcfe782, 0x000a03edd0ca0742, 0x00001978f2908e69 }, { 0x0003f5cdb0aae739, 0x0006a8cbe0f27a29, 0x000bd029fda61b77, 0x000fb8e5767c24d7, 0x0000f943175bf6fe }}, + {{ 0x000c4647529cc278, 0x00031d7c5ee30a1e, 0x0004ccc36fd68fe3, 0x0001c8748d159363, 0x00001530c91aa13e }, { 0x0001789e131a19dd, 0x00082f39e49af6a5, 0x000a1dc463a96fbc, 0x000b4fb43f2d6238, 0x0000ead936950af1 }}, + {{ 0x000ce1ad11ac24e3, 0x0001629763231dff, 0x000ceaea30de2a0f, 0x000017aa846df9f7, 0x0000e2b10bbfaf1f }, { 0x0004deb125091869, 0x000a4f6fc6a0a7e8, 0x0007e8c574b783f8, 0x0001c41bedd431b9, 0x00002151fe66860f }}, + {{ 0x000e9dd70df9b33d, 0x000b4693c2a59987, 0x0003faa941ff8de3, 0x000a633149b9247b, 0x0000fbd0cf3f9717 }, { 0x0006a6e285611fca, 0x000aef7cbcd5e64a, 0x0006f52dddad338d, 0x000a258ae0359ac1, 0x000096f58aa22a1b }}, + {{ 0x00073f78f0342793, 0x000723f24cea1a04, 0x0009ec229ba14f6a, 0x000120be9c97bce4, 0x0000c16b99f54401 }, { 0x0001a21357d12b82, 0x000ade0f63d3d0d4, 0x00006e094fa470ab, 0x000ea0183e620b88, 0x0000c42192da71c1 }}, + {{ 0x000a9e41a1d48ae1, 0x00001fac7dcfcabd, 0x000dbaa89e6546d4, 0x000254d07650acfd, 0x0000deba6e56a7dd }, { 0x000c591b9fe8b554, 0x0005327175a54ab2, 0x0000be0471701886, 0x0004842a9ca0cd19, 0x0000e042468b20ad }}, + {{ 0x000a78a81b42cff9, 0x000dbe42bad4d5f9, 0x000064775d802ce4, 0x000422994fa4c359, 0x0000ec1267726fb2 }, { 0x000cc8b1f090da45, 0x000c079a98e84e7f, 0x000f4ccf33c47417, 0x000479ce4fe1242b, 0x00008d7b8bba4b39 }}, + {{ 0x000700ef747f8ab2, 0x000f10292d0487a1, 0x000c124aa28a6283, 0x000e05c8f1442dbe, 0x00008c88040d8a3b }, { 0x0000b08ab557d66b, 0x00094c370f1fc62b, 0x0005b91712159de5, 0x0001fc90d3b31d4b, 0x0000bf8fe20af650 }}, + {{ 0x000b9ed428d2bc49, 0x0000e01ae3400501, 0x00060151f1407234, 0x000255151ba4b301, 0x00005711c0145b29 }, { 0x000bf2831a40ba61, 0x000844eeb4819f5f, 0x000ca2263440d5ac, 0x000281a0b616cdfe, 0x00006655f6b3a3fe }}, + {{ 0x0003e7363ac7c9d0, 0x0001c85e0e60d843, 0x000fe9d0b83e0ebc, 0x000727d6e68037e2, 0x0000a4ef48579238 }, { 0x0009b9aa509df363, 0x000d76108d0756c9, 0x000b95f568fafb67, 0x000401523d727f85, 0x000004ff00a6a67f }}, + {{ 0x0000cb3198286b38, 0x0007904e2b4e075d, 0x000472b1178e49c3, 0x0003e802d4998729, 0x00008d0984c74c1e }, { 0x000afaaaa6c45a17, 0x000f87dd7808c569, 0x000eb6fdeccc2f83, 0x000631aaab5c54d8, 0x00005179697da3e9 }}, + {{ 0x00019ed11ae91fb6, 0x000a82104cf667af, 0x000aab9a64bcab50, 0x000e9707cfc2a811, 0x0000554d8be76583 }, { 0x000d6d34f69204ef, 0x0008d09067b6269c, 0x00028f41c0a35e1d, 0x000eaffbc967b853, 0x0000dbd9f34c4cd6 }}, + {{ 0x0000c8e44f888e23, 0x000b2d36cceb3707, 0x000936c497a9954a, 0x0002857622aeac57, 0x0000b6752a402de8 }, { 0x000c4acd0404bf66, 0x000a35086a34d092, 0x0006bac87c03cd3c, 0x0003e6d15e0fca25, 0x0000b1bac43b987c }}, + {{ 0x0004309c3d5be708, 0x0006d173b5a9ba6e, 0x00052eb47e145ec8, 0x0002e4097b9cbe1f, 0x0000dd504546a602 }, { 0x000270b2a44f2d47, 0x0001105f67f37466, 0x000a40f34f5ba628, 0x0001c535385fc37b, 0x0000ffcc4377268a }}, + {{ 0x000b87878a9289fb, 0x00069f5974ad33b0, 0x0007f2f3a6671d24, 0x00026447304d8597, 0x000094b1402774b8 }, { 0x000a86f6939e30ae, 0x00091015a0617323, 0x000e68a3226dcef5, 0x000e4a218c25430c, 0x0000225b9562aa3c }}, + {{ 0x00074a9e86f273ac, 0x0003768da9f3804c, 0x000dda1996154227, 0x000ef0ea5470f07f, 0x00007a00585a4292 }, { 0x000a3d7f108e6847, 0x000633543471a24c, 0x000deddbc6014539, 0x000748d4d239446b, 0x00003d82eda4eff3 }}, + {{ 0x000052b13e4c4fc6, 0x000ea85fdaf45207, 0x000fd4f729b3d2ed, 0x0005d9642bb6d3e7, 0x00004fd193b24bcd }, { 0x00032412778a49c4, 0x000491b682b90604, 0x0003691032b4873f, 0x000ee41b4a71d388, 0x0000b5eb5db6e3ed }}, + {{ 0x00089111e4596bb4, 0x000456e4901bbed5, 0x00009ba190d798a1, 0x0003e197d6419c71, 0x00002d587b9dd586 }, { 0x00051bb0af6d78db, 0x000bf337b60554e1, 0x0007d8c6d2f4ac07, 0x000c32235ea5f888, 0x00000285ce6252e2 }}, + {{ 0x000dd50e6646b5c5, 0x000ca7443b30ca81, 0x000abbbe371cf9fa, 0x0005b4bade10b5bb, 0x00002c1185fd706e }, { 0x0002c679e115a447, 0x0005654cff2b821f, 0x000c54619d7f138a, 0x000808fe33882555, 0x0000813626061721 }}, + {{ 0x0006e2ff3e3cf7e4, 0x000a4156d2a57687, 0x0001a53b7eef03f6, 0x0005810437f227bd, 0x0000890e9085f693 }, { 0x000ab8ff4c5d4c80, 0x0000d6fa3177ec75, 0x0000c87000a8d96c, 0x0006d958ebf731d5, 0x0000d478202f55ec }}, + {{ 0x0000a38f5769375d, 0x000035635edc56d2, 0x00095df07e21caaa, 0x0004f10cb5a69d34, 0x000046b4e39da99a }, { 0x0002061ea52a1372, 0x0007590fac3a15b2, 0x000e58496fc6d1e0, 0x0002758866ed7ea2, 0x00009cae713eaa92 }}, + {{ 0x0009654db53619a3, 0x0003ce6f546783f8, 0x0009d0a51668ee67, 0x000c4e665d38bc6d, 0x000084a7bd921bf8 }, { 0x0005f504d34681c8, 0x000f79a26e1adee2, 0x0004113d9000faba, 0x000febbfe291f376, 0x000014210635841d }}, + {{ 0x000e987967ce4bc6, 0x000c85ec3a0ee43c, 0x000999001f97950e, 0x0007a775670aaebc, 0x0000679e9d41df8f }, { 0x0004eac8b0e95f7e, 0x000ed195e26310aa, 0x0003933fc3f81098, 0x000d5b1e49e43548, 0x00002169b080d0d8 }}, + {{ 0x000fb265b41cf750, 0x0002512c1ad8cf84, 0x000e4a2e50e3a4c2, 0x000f10df0193b4e9, 0x0000b160e21161e4 }, { 0x0001099ad8ca10c5, 0x000b8cde51e1ae29, 0x000652cf15bd680d, 0x0004f84988e3a8ca, 0x00007fc6eb96d788 }}, + {{ 0x0001c9c40bf7f829, 0x000d01c93aa96b8d, 0x000aae006a072e1f, 0x0003a5e230fc8716, 0x00002dcd3ed4529d }, { 0x000363fc5ef58287, 0x000778642b7c094a, 0x000c71dfc3c2c438, 0x0003c765d1873439, 0x00005e33d9307f67 }}, + {{ 0x000b6c8350158412, 0x000200df862eaacc, 0x00025759c48cda62, 0x000c2c22814419c6, 0x0000de581dff2c38 }, { 0x000c9b41c4227517, 0x0002345d30f917c7, 0x00091bde6e29ed24, 0x000dae963d4c4b04, 0x0000c95bedff5a4d }}, + {{ 0x000bfb7557c62b30, 0x000cc2f2f8ba2c73, 0x000f406c34d0f5d5, 0x0004d39720966eb2, 0x0000f79e276fbe94 }, { 0x00021e887875bdea, 0x00065235a2b96204, 0x00072d90be459d5e, 0x000593923bff3ed2, 0x0000becee3d55908 }}, + {{ 0x00064e33579d3be5, 0x000737a25a416322, 0x00047d0d393b726c, 0x000afd3dbc081412, 0x0000f3a2065b9dbc }, { 0x000961bb956822d9, 0x00083cdc87c51e26, 0x000ae6bca67f4701, 0x00015a9ce16b3015, 0x000099b9f2e2caa2 }}, + {{ 0x000154c9356b909d, 0x000be8deff7a3a0d, 0x0007e08aa9f0cff1, 0x000cfad1fbec3373, 0x0000ce02e948472d }, { 0x00011828d5dd509d, 0x000b36f1cc7a906c, 0x0001ab3de5b9cee2, 0x000ed9c4473bb539, 0x000009806170e848 }}, + {{ 0x00085a892ec2931e, 0x0002de13e4b12ff9, 0x0008952ddd4d7a3d, 0x000ee02cfc3aad9f, 0x000060ef6671c961 }, { 0x000785426e98218f, 0x000231b3bb85fa06, 0x000026771ffb6393, 0x000c11deaf95375e, 0x00008e2bebfcfde6 }}, + {{ 0x00099fe2c7f70e55, 0x000c2c8d8ceb147d, 0x000ddd6a728c6e8c, 0x0005d71349b649ee, 0x0000f688ee12edb5 }, { 0x0003baeb490b7ac9, 0x0001e628d3abba2e, 0x0006c3dccc50256c, 0x0001b7d4d18fee6e, 0x00005a845d13c4bb }}, + {{ 0x000157c30c88e767, 0x0007447f23206d55, 0x000fcbce3e45a30e, 0x000a8919a2f6d341, 0x0000644f481a46b6 }, { 0x000508455987e93f, 0x00086c52d4fb936f, 0x000bf1494782ed2e, 0x0009b3b64ef22f7a, 0x0000e271957d8d37 }}, + }, + { + /* digit=33 (1,2,..,64)*(2^{231})*G */ + {{ 0x000595cd1fa2a781, 0x000d8d8df7304d44, 0x000bb98b416f08ee, 0x000983c60b71bcf1, 0x0000fe06b3f76c34 }, { 0x0009429622589d67, 0x000cb9a4835859dc, 0x000cb478d834436f, 0x000f5234e4a0f0d2, 0x000076555e5f3c86 }}, + {{ 0x0004c9b20d431706, 0x0002e62f23f2d925, 0x000d0bef8e6b4e0a, 0x00055d8206cab71b, 0x0000c95a2a5e55d4 }, { 0x00031469c615f1a7, 0x00031aa9f2ac02f8, 0x00069d5a83ea26e1, 0x00016bd3403f8e61, 0x00007f5ad3cb507d }}, + {{ 0x0006930a553f291e, 0x000c1248f9813540, 0x000c2ef5980445f8, 0x000777b2c587e23e, 0x00005d0484062c98 }, { 0x000c4f3993ac460c, 0x00001ebbe5bbe21a, 0x0004784247eb5cec, 0x000c04560a7ffa5b, 0x00003ce6bf87455c }}, + {{ 0x0008b75007f2d3e8, 0x000b9583ae9c1bde, 0x0007fac5923887d3, 0x000d750b4e0af6d9, 0x0000d1ef5fdad135 }, { 0x0004069ea597b54b, 0x0003bafe02358f2b, 0x0006368b73835819, 0x000671cfc31b8b85, 0x0000fde8d8c56c72 }}, + {{ 0x0002f45c13822eb4, 0x00022e27fa03cb3d, 0x000a3ce92174b004, 0x0009d485ddb938e5, 0x000021b3fc2ab9aa }, { 0x000cc71d2d719fc7, 0x00059b57883e6142, 0x000891b541f8e786, 0x000c81ae35fd27e6, 0x0000dc5876ef7ba3 }}, + {{ 0x00028ad2a84e6c41, 0x0004fcde36d07576, 0x00017717c0d9fc24, 0x000484cdff722d7b, 0x0000ebe7611ea3b6 }, { 0x000c336b09195735, 0x0008ab520226040e, 0x000064d14bbbb3e8, 0x000ac8c886c34ecb, 0x0000b85de43f45af }}, + {{ 0x0001830b48c43912, 0x00050a21e206ee56, 0x00002ac69aea8304, 0x000bda0c3f1e2748, 0x000032ea619c2425 }, { 0x0004c11642e0594c, 0x0001c46e78e815f7, 0x000ed740f1e78774, 0x000d57765fe40139, 0x0000edbe53f588b4 }}, + {{ 0x000dcefcddd2868c, 0x000b79e558df0194, 0x000a5ea22e49ccca, 0x00091ce24230da4b, 0x00002dd640a90582 }, { 0x00072d824d931811, 0x000feb2a47d4c5ab, 0x000efbe07e5114c0, 0x000d6c17355ac9ab, 0x0000fcd91a520be1 }}, + {{ 0x000b1757ee1625e4, 0x00036744513e2cae, 0x00014d0cb33eb84c, 0x00094c0500b41861, 0x0000123028257abf }, { 0x000558a2a835c536, 0x00008283dfac6d55, 0x000df349954d38f7, 0x00030de99b2431e2, 0x0000130b37f17b1d }}, + {{ 0x000b8b3bb657e0d1, 0x0002823cbb13d1b0, 0x0002507060487a33, 0x0003073a998799f4, 0x0000a4673e731318 }, { 0x00071b9484805f36, 0x000b92e3ecebb211, 0x000b1f5665228e4a, 0x0000fe71f17cb6c4, 0x0000fab132e7caf2 }}, + {{ 0x0008f096fe69263f, 0x0006f2d7ef0329c3, 0x000258d3a92be87e, 0x000ae1cbc0850922, 0x00004f94dbc53746 }, { 0x000b4c0dfa96c3eb, 0x000a074a1dec0270, 0x0007447bcec03082, 0x000dc7853e8cf10e, 0x00006175addc465d }}, + {{ 0x000e8ef838c9fbc1, 0x000c452de7e5c194, 0x000973312d33ad9f, 0x000d8b33dfab2860, 0x000073e06257b5e1 }, { 0x000ec0507e35022e, 0x000df264cb1bf3d1, 0x0009551e8ee3b962, 0x000b4deed4c1f9d0, 0x00009cc539a7598a }}, + {{ 0x00038fd905d568c2, 0x0009ac7cde450794, 0x000a0ddaf0def815, 0x000076f5cde173c9, 0x000000e3536d8eec }, { 0x00053add2c142e87, 0x0003e06c7d9630b4, 0x000baf4270493d17, 0x0006bf9e86457a87, 0x0000eaa209f092f9 }}, + {{ 0x00094abebcf27683, 0x0008e09a607419d5, 0x000262210bc3d71d, 0x0004daef3faa71e6, 0x000036cd41505615 }, { 0x000a29a36702adec, 0x000ba91eb78399f1, 0x00052519283d6b50, 0x000ce18f048aafb3, 0x000081d651ae4804 }}, + {{ 0x000568d2e1c64bfe, 0x00042e51c21f2bf8, 0x000c1f9ca80c3bee, 0x0005f092ef99a513, 0x0000b0e4e1a943f6 }, { 0x0007a6dafc4f015a, 0x0003aebd3f69d9f8, 0x000280091fb54144, 0x000e1d6f4181a973, 0x0000bdac310c7a67 }}, + {{ 0x000a55b52ea9cdb8, 0x000fab5ef7d92893, 0x0007df294c2baacf, 0x00036dbb62480d4a, 0x00005fd14d5bdd1a }, { 0x000d5e9d7d31477a, 0x0000715309eadb09, 0x0009f58728a80fa4, 0x00022c35adea7de5, 0x0000ddf8bf0e739b }}, + {{ 0x0004d8163beea5c0, 0x000fef5206d3ffce, 0x000298a8759903f1, 0x000a735ba23d5e03, 0x0000c5541d12d6e3 }, { 0x000219da8243f413, 0x000d6efd3082e8e5, 0x000baf7918ddca19, 0x0009adb0a4fe1246, 0x00009d116a39ce07 }}, + {{ 0x0003fcac89cfa8ee, 0x000b7c1f7ea25aaa, 0x000f57b3d0d5ac4d, 0x0003e9871d165e6b, 0x0000a30effd4d7ba }, { 0x000e263feae7d9d1, 0x0009ed4f0baa8abc, 0x0001149b82faf7d1, 0x000fe87495a4adac, 0x00006b868aef4658 }}, + {{ 0x0005547c4f5be3f1, 0x00012d307032c722, 0x000e53b4ec796f84, 0x0000587f07f40d5c, 0x00006bdd6dfb6e81 }, { 0x0007df465d70f0e6, 0x0009ffd65fb2a9eb, 0x000e66d9d13f40f5, 0x0005f5e42ce69336, 0x0000c86a6ebbc9a1 }}, + {{ 0x00058de498e2b259, 0x000aba3315980cd1, 0x0002e4ae075ed625, 0x000a16c04693bc49, 0x00003b029d23553f }, { 0x000836ce7e2b454a, 0x000abc7cfda83685, 0x000320a048ffe7ec, 0x0007c9a6ed2d7bbb, 0x000066b26f8ebcc8 }}, + {{ 0x0002b52f65c725d1, 0x0007e1994f834cbc, 0x00037fe9a81cbbe2, 0x000bfb143d35653d, 0x000092a9b3930992 }, { 0x000f186cdd7031a0, 0x000821b18f23d962, 0x000c1039e5d77384, 0x000a47402cb05aba, 0x0000c20b19c6ae17 }}, + {{ 0x0005624623a128a9, 0x00043e59a29a5c5b, 0x0006860a3ddf3f1a, 0x000b07d84b19b3c0, 0x000095b53ccaa378 }, { 0x0005a9d0868addaf, 0x000bf653c04fa024, 0x000c4900692381db, 0x000c3b43f83a7176, 0x0000f9656734975f }}, + {{ 0x000b179393f74d6f, 0x000bd1d3dc090bc7, 0x000f2f422dd3e906, 0x0009a8d4d82c996e, 0x00000d602d115417 }, { 0x0007c5dff255af16, 0x0008825f8c5dbfdc, 0x000d576c9b126092, 0x000dee6a84d9d7a1, 0x00005c2588a5a8d3 }}, + {{ 0x000ea886daec0451, 0x000078d04c82bdd5, 0x0004998917222f76, 0x000e744806800553, 0x00005c6be61fdaec }, { 0x000de96ef613e33c, 0x000ce01252a0b4d7, 0x000db80131abd46f, 0x0000659631bffb4f, 0x00007cc0d1f5bbc5 }}, + {{ 0x000fe64809d55519, 0x000140cd77d970c2, 0x0003a358b1c17566, 0x000cc06e903cfb93, 0x0000d055c34cf7b5 }, { 0x0003d31b05a74dde, 0x0007824c2ac98ccf, 0x000f2666232e242f, 0x0001827212a1372b, 0x0000ee7c4caf1682 }}, + {{ 0x00061891a0a38e82, 0x00067df5ece6e7ce, 0x000e299469b4ff0c, 0x00013b1a50603ac8, 0x0000588d5e4bda3a }, { 0x0003b117efd74152, 0x00051d299bf1603d, 0x00091dd30d6b1e14, 0x00058ec021f43afc, 0x000028e06b2d515a }}, + {{ 0x000103fa05977e5a, 0x000801365d1d131c, 0x00070276d6394917, 0x000ae7edd86f1636, 0x0000eb35b19898ae }, { 0x0001ae47f2c3b604, 0x0003f44f4b478f01, 0x0008cf69819a940d, 0x00069bd2ece91edd, 0x0000bdb21f33bd5f }}, + {{ 0x000d53ae910e4572, 0x0005d8d20f8a2425, 0x000d94da6c47cb3e, 0x0007f8cb37aff5b1, 0x00007219eed53f71 }, { 0x000e86515094eea4, 0x000d505ec0418032, 0x0006e5b9f737bc0c, 0x00073e98f85e3c97, 0x00007c74d04d9d17 }}, + {{ 0x000a04ca6726822e, 0x0007ca46a67a6b02, 0x0005885821d9ef2a, 0x00044421ef4cdee2, 0x00003f466bb9f3be }, { 0x0004a874138b9a64, 0x000ec18ef0543fbf, 0x0004dd08640a5f75, 0x0007b6f33f709a4f, 0x0000bdf2825ed29d }}, + {{ 0x0007b80581df3f89, 0x000a6a74d82f4c2b, 0x000fb04cea31d778, 0x00083b95e90725a5, 0x0000bfe40a6fc4ce }, { 0x00087f6f736e6376, 0x000631cdd1fc74cd, 0x000b0f04375170e3, 0x000a155badc1c62c, 0x0000d6efb5c07fab }}, + {{ 0x0007be9ed2271299, 0x00036c1317f4a765, 0x000b3944ccfbdba2, 0x000d491568d2703e, 0x0000e1677466f47e }, { 0x0000d9e943522c82, 0x000c371ef0b17acc, 0x000e183e31f4b70a, 0x00066db0e74b2655, 0x0000795a4aa76537 }}, + {{ 0x0008d2fe5724a2d6, 0x0008a411e84e0e5b, 0x0002f5d04e3bf4c5, 0x000d0d132a5db84d, 0x00001765a592c24b }, { 0x0008b4a422ebc11c, 0x0004336f3eb82fab, 0x000c454ae73559b6, 0x000b3a5108cb20cc, 0x0000bc49662e3c97 }}, + {{ 0x000ed06e285bdac8, 0x000fbea6fe8208b7, 0x0003a99e0ad61a08, 0x000fe721abd74b9e, 0x0000d5e8ed41f37b }, { 0x0003186b23df1bb9, 0x000a1586be8b800c, 0x0001152e70049303, 0x000ccd5d3428299d, 0x000034b1cedc477c }}, + {{ 0x00017aae8c429f29, 0x000039e8977d99b6, 0x0001227a835b6392, 0x0003488d4be94433, 0x0000ff63573b35d6 }, { 0x0001efae75205711, 0x0002ec4282aec3b3, 0x000fc5c949ae1fbe, 0x000ade84cc6063d2, 0x0000bb575870e312 }}, + {{ 0x000184a526e63b6f, 0x000fd6d40a9b97c5, 0x0002fde1bd2fd49f, 0x0004943e314702c6, 0x00006773701eca6b }, { 0x00059cd28024561a, 0x0003937003c32ea4, 0x000cdabca247db9a, 0x000b7bd1a5aa374d, 0x00006c792f7fd426 }}, + {{ 0x0003e1589a804b9d, 0x000e461459c2b92c, 0x0007b20d365b2696, 0x00064c6102f020fa, 0x0000ca6d9d3092b0 }, { 0x0008466ee83f88e8, 0x0006bfeaa3f451a6, 0x00035a933dc49cd6, 0x0009efd8b3a3c7f2, 0x000087060571aa2b }}, + {{ 0x00013994bea33721, 0x0002cd41fa266138, 0x000b3c8908a4f4fb, 0x000dbb557a2dc6da, 0x0000d41b64ebec33 }, { 0x0000b65bedddf81b, 0x0009c22fd4f75534, 0x000bb2ed0f327ed3, 0x0005d82f55135ff8, 0x00001de1ffe7c18b }}, + {{ 0x000331c0941a20d5, 0x0001da43734e5205, 0x0002cd6a1c26bf6b, 0x0007386355e3abbe, 0x00003e3eb679bfa2 }, { 0x0008f2c0da9416e2, 0x0001c2976b60c80e, 0x000a91db561dbd79, 0x00014b5842285593, 0x000090564385f22a }}, + {{ 0x0005c558090f5634, 0x0003d74c1df065c4, 0x000b19a020f15227, 0x000e520a8afeeb5a, 0x0000ebe08fef2016 }, { 0x000ab226cebd5622, 0x00094c0ecc8c1778, 0x0004ced0c5db8594, 0x000b1cd8e799ea9b, 0x0000fb29019d75f8 }}, + {{ 0x000a741ef1df7c17, 0x0002ae57ea973dc8, 0x000f9614703c94db, 0x0008185f15c00ca6, 0x000063d1e4dbdd66 }, { 0x0003b6c6786317cb, 0x000512f196b4cec3, 0x000395c7df9fe4ad, 0x00099bf2b70d08e4, 0x00008cc26ea5e3b2 }}, + {{ 0x000ef2c5fb93c237, 0x000adf807ac97946, 0x00004b14057e32a0, 0x000171de51eccf05, 0x0000be44bff72854 }, { 0x00006e0b10d8d552, 0x00011d2679784532, 0x0005fcf73fe7d9eb, 0x000e85837a0cf379, 0x0000f1351989d9a5 }}, + {{ 0x000c6469812142a9, 0x000055fd5ecfefc4, 0x0008765b8f7b26db, 0x000ef9c9607084fd, 0x0000b7551edf99c8 }, { 0x000ae28483d757ee, 0x000bd4ed65abfc39, 0x000e78719a43c760, 0x000116212e01cb91, 0x00006fcb78045826 }}, + {{ 0x00091c285c2a4c9a, 0x000b29ea66d80a18, 0x0003ecd9d0b57b14, 0x0006db5c1680a3e4, 0x00001cad3822f1a4 }, { 0x0006f8b13781c29d, 0x000dfeb799ece3c0, 0x00093ae7ccf04903, 0x00021942329b910a, 0x0000f5e1879fd271 }}, + {{ 0x00043be0aaf3f89b, 0x0005ffd385470908, 0x000f01a760b26589, 0x000ea802719ab8b9, 0x0000ed6be57be9d3 }, { 0x00085f39d0ed1528, 0x000c543a6db836ca, 0x000359452235069c, 0x000ac7b772a1330e, 0x00000393cfe1f4a8 }}, + {{ 0x000ad599c449623a, 0x000546d4ae528342, 0x0009494e42ebf43f, 0x000d510ef5a403b3, 0x0000d5d9e1fe46a8 }, { 0x000827cf81a805c4, 0x0006bb9014b3bf81, 0x0004de4580857334, 0x0002e3d027309268, 0x00005f977d08a51c }}, + {{ 0x000b5212970e3e2d, 0x000d8f6f10ff9276, 0x0000b8218d2fb9f7, 0x0008395ccd387145, 0x00005c10024eb9bc }, { 0x0003ea9e97f9fabf, 0x00005f96e3cb4a55, 0x000123a0215f0c0c, 0x000937b45e47d3f4, 0x0000f7e86aadc6c4 }}, + {{ 0x000a9b83894c7656, 0x000ba6e39cbb688c, 0x0002289ce9a8fb03, 0x000435e21532401d, 0x0000c8103121db65 }, { 0x0002767c4b7d6814, 0x000174e2093913eb, 0x0001b74604c733da, 0x0007f9ec7533d2f9, 0x0000110a91af736e }}, + {{ 0x000581cf9346f15c, 0x000a387aedbae9f1, 0x000921063ac18234, 0x0005bb2717daf0eb, 0x000088c3f46a6255 }, { 0x0003d318ac75f3ac, 0x000fc873ac1f4527, 0x00098797d751c8f8, 0x00007e1c3b417fce, 0x0000aca7b2139a79 }}, + {{ 0x000f8a35b721d031, 0x0000f0f728693e5f, 0x000a340ef236c630, 0x0001a6810e87d814, 0x00001fb62f488436 }, { 0x00014052c2a4d35e, 0x000f5d77f200d34a, 0x00076e24ea1e5f64, 0x000cd0579470f1e6, 0x0000eb853f10fab4 }}, + {{ 0x00078fe4024f3dda, 0x00020b526bc27979, 0x00060261836f1c35, 0x000006ed113670d1, 0x000079b7c4f64fcf }, { 0x000768d7e586bd52, 0x000a992d41c5dd45, 0x00077049a5bfcf2f, 0x0008264065ff8e24, 0x00002216a5fc8c7a }}, + {{ 0x000c3672af71c96a, 0x00080a360017a5ff, 0x0003a02e5bec495b, 0x0007474076b18e6e, 0x00009937b8bda59e }, { 0x000c948f1455702e, 0x0007f77d06283a1a, 0x000038410e854096, 0x000d0b9c2542fc65, 0x00006954911b34d4 }}, + {{ 0x000fc87df6be7f7a, 0x0001c82b04465c31, 0x000a44a03f1eaecd, 0x000b2137ee3d062f, 0x00008c43ce25b5a0 }, { 0x000f663391d373da, 0x0006ca80591f6528, 0x0006c92fee18905d, 0x000bd5a8ca861b8f, 0x00006b9e0397c8fa }}, + {{ 0x000aab4e6a3238d0, 0x000a8acb591ffa38, 0x0004c2349d649d6e, 0x00015d6b3027ca9c, 0x0000be661b873b12 }, { 0x00068aae83dc529a, 0x000564fb474d6b85, 0x0003add089d147e5, 0x000f552aa6597ffd, 0x00008040d2d5caed }}, + {{ 0x00043d8baa45a8fd, 0x0007164c73b23834, 0x00078d3ffbb7c8ed, 0x000f760ab49eb344, 0x0000e0df7be97747 }, { 0x000bf53de2fa0299, 0x000caa5ea7610d6e, 0x0004e2276b961c38, 0x000b80832e47e418, 0x000046bc71e68286 }}, + {{ 0x000de1666fe499b7, 0x00045b4d3c8c1cab, 0x00029f234c99a8a2, 0x0004e1c950d914ef, 0x0000577ca7b4a559 }, { 0x000e9faf6f83c4d3, 0x000f76ed449ab198, 0x000259fa7703581c, 0x00006a7538902b32, 0x00006ea68c6f0f57 }}, + {{ 0x000da8564dcf371f, 0x00016da9a85788bc, 0x000def5dd0869c6e, 0x0003cbd64d21f03a, 0x0000cb0b2e54fa19 }, { 0x0004ee3dffcc2d2b, 0x0004f5cde92c68ca, 0x0000ce43fec47260, 0x00053d47c78e035a, 0x0000a966314f762c }}, + {{ 0x000e2ab3decb0797, 0x0009e07af4e18f85, 0x0004a8483f498d7a, 0x000959a86967ec9b, 0x00007b0ac9fcfa8a }, { 0x000c0c1a5b8ce6b8, 0x000b64d11c226dff, 0x00028b6615b91ec6, 0x000a19f19f9783f4, 0x00001a2b243e738d }}, + {{ 0x0002f2a62f45d86b, 0x0008149571d92ac3, 0x000e139fab5045e2, 0x0007639af569e85f, 0x0000dc961974e55b }, { 0x0006e5ba0c5b997b, 0x000533a06d89b85d, 0x00095911bffc3d29, 0x0005e1aa2b90741a, 0x0000a4a9e49fefa6 }}, + {{ 0x000b842ac7fd440d, 0x000775112e4c9366, 0x000d5187b6b3804a, 0x000e21ec9ae419f7, 0x0000853743dec424 }, { 0x0006ae3fc4509621, 0x000d3e9440d68277, 0x000e78d3bb4fe338, 0x000b99a925f69f4d, 0x00001ccb54d831d2 }}, + {{ 0x000f9a722461f223, 0x000d3c2e9740b335, 0x000ce9c8647cec97, 0x000deea15677bc85, 0x0000f16ec53b5b06 }, { 0x000eb6094dff8468, 0x00051164ad859617, 0x000767a1a1fc7112, 0x000c1b57e5f8c47f, 0x0000305f263682f7 }}, + {{ 0x0008ca06c000315d, 0x000c0fd341905477, 0x00037793ca900aec, 0x000ade7e68a394f4, 0x0000d89206088d97 }, { 0x0001ba76c1885187, 0x000ad626ffeec228, 0x0009b5136239e6b9, 0x000b437479d91bce, 0x0000a4b31a467b18 }}, + {{ 0x000b0c03e9bb6ab5, 0x0006fcb28c682c67, 0x00080b06537181b0, 0x0007bb99827f2522, 0x0000e7f5e678902d }, { 0x000287223f5705ed, 0x000e33015b548e76, 0x000fd62c48b8a6ea, 0x0002b1f40a8be76d, 0x00002b41597b0af8 }}, + {{ 0x000fcf45b93bfe6a, 0x000f6558b399320d, 0x0005e04f008d2b8f, 0x00027b05e61fe3fa, 0x00000320b1aab1c7 }, { 0x000c421d977c1dfe, 0x0009c512f684b432, 0x00049adfeccf138d, 0x00091a78ff796c65, 0x0000756d6c50e779 }}, + {{ 0x000dc9d1ec6f170f, 0x000bea3af2ebc2fd, 0x000ced852c3855dd, 0x000b601b1a0af843, 0x0000efe50594d52d }, { 0x000740f316de5b77, 0x00051bb62771deb1, 0x000a7a84b76a9d83, 0x000c81aae829277a, 0x0000e1420c11bfa4 }}, + }, + { + /* digit=34 (1,2,..,64)*(2^{238})*G */ + {{ 0x000fcf57c64d0fe1, 0x000089aeeaf8c495, 0x000d0e557f623c19, 0x0001b346cee7aa73, 0x0000e3399090fc9a }, { 0x0006eb38aad81cf7, 0x000a739b6057604e, 0x0000db9b5314c754, 0x000f3472d7d343c0, 0x00004c1dfbe3a0a5 }}, + {{ 0x000848949c160154, 0x0000e4f8be762b49, 0x00006d459f779d3d, 0x000d6b1fa2a9ee4d, 0x000073144722cd51 }, { 0x000631594e5d8939, 0x0009ee5d23a84178, 0x000d6db14456f57f, 0x000650e1f892f3b1, 0x0000d7a0b3f34fe0 }}, + {{ 0x000a613a9e60b8e8, 0x000423cde918dcce, 0x00033c9c94ad800f, 0x00012332d2e7b599, 0x0000a4f91cafaad2 }, { 0x00000da7133e2037, 0x000286cf1fdc9f70, 0x000887923f6f84cb, 0x000fc857fb6eae8b, 0x0000f963508ecd1a }}, + {{ 0x000585113ba8dd62, 0x00080ea1f095615b, 0x00065f617af767a6, 0x000aab6be0a28ad7, 0x0000085f54531694 }, { 0x00033ed1deec48b1, 0x000ae0b30df8aa1c, 0x000b877911cd914a, 0x000e64fcbaddda07, 0x00004abbcd21ba68 }}, + {{ 0x000610ffa32530f1, 0x000285d80e877662, 0x000877ce4b1cfd91, 0x000b658349e99e6d, 0x0000e408841fab40 }, { 0x00090cb5663d4ea5, 0x000a2d4751c8159a, 0x0005062dddfd0fce, 0x000f8d733697b498, 0x000058975ab9e049 }}, + {{ 0x00091be071c8396f, 0x000c94836f024cb1, 0x00071ce580f69081, 0x0004498f185be1f7, 0x0000741b76e70b14 }, { 0x00048a876e5adf25, 0x0007aa5cca5e55cb, 0x000c4345c73e6437, 0x0009de0db3864ba3, 0x0000cb37b2d41132 }}, + {{ 0x00053fc7d8535a0e, 0x000b856580e40a2f, 0x000b7d40a0e49c2f, 0x0001f82c9f194ed8, 0x000020d26cc66d51 }, { 0x0003b7d63457b0ca, 0x000e49b78593e7d5, 0x0002ee238fa36714, 0x000f3099b2cac4ee, 0x000023acbadeefac }}, + {{ 0x00032b33d4f558d6, 0x00037483e07113c9, 0x0008ea602fd9386b, 0x0000f762b6e4a5d3, 0x000040b7b04b7684 }, { 0x000f178169d618d6, 0x00020510dbbf08dd, 0x000885aab8e91f1c, 0x00043069b211d07c, 0x0000c105f022a618 }}, + {{ 0x000b741973430cb0, 0x00096350a712229b, 0x00074db32619911c, 0x000070c8393cbe44, 0x0000fe88492c7c4c }, { 0x000da7715e131810, 0x000f9a256dbe6e72, 0x00062cb5c88e8a5a, 0x000078a31e578ec5, 0x0000121ed1b3bd72 }}, + {{ 0x00084867a70ea8a3, 0x0008d1753dfabf0f, 0x0007c09b2ab277d6, 0x000b4f658728d207, 0x00008772d5dc2672 }, { 0x000c9017adeb92a2, 0x0000f3d70e35b1c7, 0x00079d20267edcd3, 0x0008c8f210241446, 0x00002a5362cc2cc0 }}, + {{ 0x000461d767714a05, 0x00068e90a482873d, 0x0000e31c7d6d3a7f, 0x0003383e74265d6b, 0x00004a05abbc1234 }, { 0x000626833fe129d4, 0x000cac564a7e489c, 0x000cdab872b2a050, 0x00033754ecc5675c, 0x000089b5043845b7 }}, + {{ 0x000568d0917c4acb, 0x000c55a83cdd60ed, 0x0004cbae30f32f9c, 0x000d8992a50602fb, 0x0000910f7a813c33 }, { 0x000ec2057b354ced, 0x00035c70259217fd, 0x0008e56f3c6925cc, 0x000490b6e93831cd, 0x0000a336551d0bd7 }}, + {{ 0x000563cce247e851, 0x0005499f98a759a3, 0x0002c25e6f3b7c66, 0x000dca58755a0a7a, 0x00005ae8843b044c }, { 0x000eea4056f758e4, 0x0005117e7b1fbe12, 0x00092088f88989d2, 0x0007764db8a7dc97, 0x00000ee98d72925b }}, + {{ 0x000ee376f827fa0f, 0x00059a7ba772b34e, 0x0000706aae49b0d1, 0x000f205dbe16ed34, 0x0000b5d4498eade7 }, { 0x0009a59c3b7533ce, 0x000dc1783674c027, 0x0006de9d65ef7473, 0x00070cb0f65832f9, 0x0000512581da9ee6 }}, + {{ 0x000fa352b73388b3, 0x0009a2ebd19bb161, 0x00009382dacf6953, 0x000135847c6c7cfd, 0x0000dfe8b3751963 }, { 0x00059df31f52f920, 0x00035b8973db6d32, 0x0000c347c49fe544, 0x000c2371f7183685, 0x0000f1f4cc727793 }}, + {{ 0x0008835dedbc740c, 0x0007d25913cc16df, 0x000ce91f116c9a1b, 0x000808dae18bc5b6, 0x000071a394c1d139 }, { 0x000ce433b9ac0821, 0x000ead865a40ab05, 0x0008bc6703c50add, 0x0000029696559643, 0x00009a0bfc436be6 }}, + {{ 0x0001e29ee4d40158, 0x000984f6ad651437, 0x000764feac1214e7, 0x0009468c71151182, 0x0000fa9f57c32bae }, { 0x000b4d4bd6ce4114, 0x000e80963717b46b, 0x000e251d88cbd747, 0x00081172202be02f, 0x00001fa60920ef36 }}, + {{ 0x0001cc10cacd0147, 0x000ad9620efd6628, 0x000043c91c4987a3, 0x000963dc292ba6d2, 0x00000e78ee2053ff }, { 0x0005c6097c119aa3, 0x000cdad4b4d7f6ce, 0x00059edbd3562f1a, 0x000ad44975c9f1d9, 0x0000702d0c39051f }}, + {{ 0x000ea0e79fe0f90d, 0x000a2a6328e5e200, 0x000a45e06db05fe3, 0x000c6bc78f7b5244, 0x0000518bfd167866 }, { 0x000e3a0690854dfd, 0x00099eb646c7ecfb, 0x00028633e40d1fc1, 0x0005dd269eb81149, 0x000077f2043ce75a }}, + {{ 0x00066dd6d2079f06, 0x0004c3d42e06189e, 0x000067cd28417e6e, 0x000130050bf07794, 0x0000de2c929cf8ba }, { 0x000333b1e86df344, 0x000b967ff99abde3, 0x00009477132566ad, 0x000536ed7c990355, 0x0000973cee61a30e }}, + {{ 0x000982aad98c4636, 0x000db3c6c491c14e, 0x0007975a94ae8585, 0x00092fced0597668, 0x0000469878650d9d }, { 0x00092d6943d57f62, 0x000f1478be0dcf0e, 0x000d7c9ceaae9a12, 0x000bd457f700080a, 0x0000c7c4650a126e }}, + {{ 0x00038b135b3697ef, 0x0009bf885dfe9aef, 0x0004599f736f9c08, 0x000480120d825d9a, 0x00009a6f60743203 }, { 0x000847fb2ab7025d, 0x0002de062a7150b5, 0x000db3ec1370558f, 0x000b9932ef31fd8d, 0x0000640e5b498396 }}, + {{ 0x000ce5e866272fbc, 0x0008bdcdccbac8b2, 0x0007da0e9d6243e6, 0x0000930dfb642aa7, 0x00006481890aff49 }, { 0x0009424317123bb7, 0x0008088c98c44004, 0x000da8b80f62d7e6, 0x0008318d78104a8c, 0x0000c585f14062f4 }}, + {{ 0x0000428dffccb096, 0x000cfe784d6365d8, 0x000f0f3311b8f429, 0x000378330bcb7443, 0x0000aa82351ab1ae }, { 0x00058c1fb6bdb9c6, 0x000e9efa4faf8433, 0x000dca1d65a933e3, 0x0008101ceced8538, 0x0000d4a8d9e6e600 }}, + {{ 0x0008aba53a05bf33, 0x0008de5fd349961a, 0x0000db761dc2a9ea, 0x000e35a382b2cfa5, 0x0000b7879468dded }, { 0x000eebef3fda587c, 0x000b11d6e96eb8db, 0x00000cc3a6e277a8, 0x0000c796c65d8dea, 0x0000e172dfe107d8 }}, + {{ 0x0005a760658ad3df, 0x0008e98ce7852541, 0x0002a5bd40fc7160, 0x000a56bddd50e8d7, 0x00005c3561e661be }, { 0x000e509af3870a6a, 0x0003822961f23d6a, 0x0007c59c25bcd2e9, 0x0007893e4378e443, 0x000027a005b340d3 }}, + {{ 0x0005626678970f60, 0x0008e46577f44a13, 0x000c80af7872b352, 0x000e85884e09b748, 0x0000e421f514b45e }, { 0x000ceafe50229ea3, 0x0002218bf8d21e6b, 0x0000f51aa91d1556, 0x000a1d0c022d9653, 0x0000b69daf2a9172 }}, + {{ 0x000e6c984782812a, 0x000b4f7be42a5821, 0x00004697d39f6d6b, 0x00067b1a9d3fc608, 0x0000472dc3a58087 }, { 0x0006fec314288ecc, 0x0007d25d30c31f70, 0x000aa52b5e0e12c8, 0x000798de7b7eb72b, 0x00005698fa1698c6 }}, + {{ 0x000a05261d35c7d3, 0x000e910f8c22049d, 0x0006ad59994bee9c, 0x0006e5f6b9bfbcce, 0x000017336b3b0d0d }, { 0x000c744e8ee14501, 0x0005adceba7efc8d, 0x00094967709fe15a, 0x00030f027471237c, 0x0000b4abf8266348 }}, + {{ 0x0007b411642b925d, 0x0009016a56edb730, 0x0007244c6d0c0d15, 0x000ee570064357e3, 0x000045c5461cf724 }, { 0x0008434e8e56e691, 0x0008d3274c898834, 0x0000e03f995f5bf2, 0x0001d53823c16282, 0x0000bfbc0e792c1e }}, + {{ 0x00017b66ac0eda90, 0x000624f2c7ae0b94, 0x0000d7be4d9e6b1a, 0x0007eceb1c46fbf8, 0x000073445e243c47 }, { 0x000d06252a7f7cde, 0x00054b2c5ceec209, 0x000f449ba5b7dfa6, 0x00085a8e6d69661f, 0x0000bd415537281b }}, + {{ 0x0009650f1aafc77c, 0x00018b07e50122b3, 0x000b118b6d8a8280, 0x000784d7bdd744f8, 0x0000b7a7039bee26 }, { 0x0003cde18959c8fa, 0x0004df71fa38477c, 0x0009110e30c5b652, 0x000a5bb8a3d815f1, 0x00000fab64c70c58 }}, + {{ 0x000af52d3611da39, 0x000ba7bf9539a488, 0x0005e913946f430f, 0x0007dd3bbb0100e0, 0x00008c0902a3dbd8 }, { 0x0008a53a687694aa, 0x000f347dbbf698b0, 0x0009fb36e20b6928, 0x000d5b3f82961a1e, 0x0000e28e2ca89070 }}, + {{ 0x0009d84d2e9b0fce, 0x0002d481bee45d41, 0x0006229033078bdf, 0x0001f5669b3cee34, 0x000090c773d3e729 }, { 0x000fb1fb1bd23754, 0x0006fc51c4d349f6, 0x0003a68b5ea950a0, 0x000747aae6596584, 0x0000f1319ee53ae6 }}, + {{ 0x000f0bece2f566fc, 0x000010607bf8fe31, 0x0008155b86689659, 0x00090f4d346259c5, 0x0000d4aef04c0740 }, { 0x000849f911d26148, 0x0006cb3f3c72914e, 0x0004546b3a782574, 0x0000a254629e8ea9, 0x0000ef344da514b1 }}, + {{ 0x00007d28f8169af0, 0x0003fbbf4737f217, 0x000af18cccb4682b, 0x0005ebdbed8dba5a, 0x00005fcad39eb18f }, { 0x0008f0c981a47c9b, 0x000ab0bcaa999eab, 0x000c63a21b4ad294, 0x00058242dba7b174, 0x0000fa27d77ce474 }}, + {{ 0x0009619367bed35c, 0x0008e9a324b47379, 0x00029946a8624378, 0x0005f80edfba9e42, 0x00002193d51edbe3 }, { 0x000898978136c517, 0x000dc5754b7032b4, 0x0009dbaa3dc9b204, 0x000cbaf5af936b3b, 0x00005c1090706f4b }}, + {{ 0x000ed617a5a061dc, 0x000f95a13d4f95eb, 0x0004eec56d5b0d63, 0x00012183d737cff8, 0x00001b08a819d5ea }, { 0x000791e248259760, 0x000f4017dfd8ab26, 0x000dd2968e62408c, 0x000108545ae7b89a, 0x000093d95add0afe }}, + {{ 0x000d5c3a7fff1f96, 0x000a0ad324fa34d3, 0x00070f6273a886e1, 0x000bd9c99ea11144, 0x0000b3c3fd9df0bf }, { 0x000a251c3731c8ea, 0x000f1678d0412d2e, 0x000212b74f637f2d, 0x0005fbf82d92625e, 0x0000b4da522a951c }}, + {{ 0x000cf06fd6232db8, 0x000d132790691bf5, 0x0008a9c6c6d87a37, 0x000a8836eed61058, 0x0000653769772e9d }, { 0x000cf4a1aa26fac0, 0x0007facd9588e411, 0x00020e9e83d4703b, 0x000800ace1f9bf76, 0x000079b3d81e1a3f }}, + {{ 0x000fc099ea249902, 0x0008849dc1818d01, 0x000983a180c61c4b, 0x00005f4390643ff3, 0x00003e5c40cc22bf }, { 0x00058d9a82959817, 0x000a2c3eccd28fd0, 0x0002795eba28fb98, 0x0001c5eb6d743cbd, 0x0000a1953590ed50 }}, + {{ 0x000603700554b317, 0x0000b9291262b72f, 0x0008c50b6efe112d, 0x0006d514296f924f, 0x000024bbaa00c3a9 }, { 0x00040606f5dab0ef, 0x0007a9e18d7dd96b, 0x000217a51d9f847f, 0x000dafa49381e878, 0x0000e4b811835102 }}, + {{ 0x00020c00eed74984, 0x0003439fc6751a47, 0x0004899c000050c9, 0x000d0a4751619621, 0x000037a8d658db74 }, { 0x0007d16ea9dbaf18, 0x00086b0d820007cf, 0x000fabc9e97be6ba, 0x000705184880ee65, 0x0000435446efea6b }}, + {{ 0x0005512b4e835c1f, 0x00056b2c0519a238, 0x000dca182794a57a, 0x000cd2b8a9ebf94f, 0x000043f096ab191e }, { 0x0003ebbb9869ab72, 0x00019a899b16a0de, 0x000bd6898d0eaeea, 0x000c0741595c2430, 0x0000e438ccd17837 }}, + {{ 0x0003ec66f6fddbd1, 0x0006bfe18549fdb7, 0x000edbe1509ba54b, 0x000cfd46cf6e200b, 0x000098de96d82185 }, { 0x000c5616c5a42a40, 0x0006dd7d1b9a305c, 0x0008eb5804d7e8ed, 0x000a1c94aa959c5b, 0x00007e60385e5cc7 }}, + {{ 0x000a06b329f8877b, 0x0005f01e6a33f7ce, 0x0006b347bd83ae53, 0x00039e07cc6217a0, 0x00007de49a6fee11 }, { 0x00012c4acbdef85a, 0x000b7cea5f5e7b9c, 0x000fb7501b28384f, 0x0001e515466dfc30, 0x00003b6c73cc9d54 }}, + {{ 0x0005ad49e51ce957, 0x000361a1e407dd9b, 0x000bdc025fb2c747, 0x0008e60a5e8a3587, 0x0000375165792d8e }, { 0x0003f99fc369de26, 0x0008cb27006085ab, 0x000fd3c5cefed71b, 0x0005240a6a88ff33, 0x0000a54d6a89ede4 }}, + {{ 0x00017969cd6413a4, 0x0009b5eb2e636456, 0x0007234ed864a3a1, 0x0000131cfbe57e54, 0x000097da13fc490e }, { 0x0008c4f4f07e5b7f, 0x000e9cd8a7017a0a, 0x000a15d689f8531a, 0x000ba987677b4d19, 0x0000622e9398a31d }}, + {{ 0x000b8c3c1ca6d7d7, 0x0009acc67b3de77b, 0x000da7c33ce88288, 0x0009929c9ae3c0b8, 0x00005e532fdba5e3 }, { 0x00018487aab79e34, 0x000b4eac51b35836, 0x000f33a15e97b5d3, 0x00056d5f37591f5a, 0x0000464d53dbd05c }}, + {{ 0x0007623bd0306c82, 0x00063f79d61718a7, 0x000bfe3096bc0a06, 0x000cfb3cf00413bc, 0x0000bd36a5580365 }, { 0x000b0231a0568154, 0x000d33bdcdd2a50c, 0x000a3aff52ce886d, 0x0004595294a741f9, 0x00006cbf37c0b0b8 }}, + {{ 0x000a0c030ad04bbc, 0x000940d084bae24a, 0x00068a312388eb95, 0x000933c112f0ae8d, 0x000059d33f6ee7ac }, { 0x00089e38dd68ef7a, 0x0009361a0f442dea, 0x00067f1300f605ff, 0x000e94228f92e181, 0x00000cdb381fd9a1 }}, + {{ 0x000ff5b8f078049a, 0x0001e65d2fdca23e, 0x0007dc993fa509ea, 0x0008bbc145a15ee0, 0x00008afe9e821989 }, { 0x0006f2789726e911, 0x000b9bbcb1c8a0ef, 0x000cc1d5c062c9d7, 0x0006103c02e1ee9c, 0x00007b357e712163 }}, + {{ 0x00078ec0a2491df2, 0x000fd1c89ee6256c, 0x000da3a67aa3a41a, 0x000cb11b3f44ee1f, 0x00008112a4f22527 }, { 0x00023aae4f91b961, 0x000d224b7dccbc61, 0x000304010dd5da46, 0x000347705d96dd1f, 0x000040b329358dcb }}, + {{ 0x000ccb9b14c45c47, 0x000b5f5a26262cd2, 0x000cd39550c052c7, 0x0008e5caf1f7f4d5, 0x0000e4892d07ecee }, { 0x0002fe771f54a03a, 0x000e2ad52324aa65, 0x00061f496834cafb, 0x0002935826320052, 0x0000a358d4c12366 }}, + {{ 0x000d971c910c2659, 0x000483db1001684e, 0x000db3d982e44846, 0x000b52d72b56b4ee, 0x00008b44f48218c9 }, { 0x0008b44cb030e094, 0x000f291e3a07b296, 0x000c7c024a091d9c, 0x000e755ae7570d9e, 0x0000f4a48be7b74a }}, + {{ 0x000e62960e80d5a9, 0x0008bb3bf44da801, 0x00093c509857d0c7, 0x000a2f700283834f, 0x000004d0d39a88e1 }, { 0x0006394239b24d7d, 0x000357be6901c550, 0x00081ec479e5e93a, 0x0005f457d80e7853, 0x00009467d044177d }}, + {{ 0x0006f9d9edf5287c, 0x000ce40376c1944a, 0x0003bce03b234092, 0x000ec0e3b7cb6269, 0x0000dbfe013373c1 }, { 0x0006ce2871c6ea89, 0x000a3a2880dd2123, 0x00036f4311374ef1, 0x00024fe85903c381, 0x000070bac414fba5 }}, + {{ 0x000d3ed11bf23e7f, 0x0003128e10d9c12f, 0x000105f062b7d445, 0x000649c587573baa, 0x00002d7ca95fda75 }, { 0x000beec455ef108f, 0x0001b9a0cb2748e2, 0x000831811ac094cc, 0x00064aa418239f01, 0x00008c7d897f4683 }}, + {{ 0x000939e11117c855, 0x00001063f8b84cec, 0x0002c3fee75e7039, 0x000c0fedd7641708, 0x00003953dd909641 }, { 0x000585638f431edf, 0x000e3b35a914f2f1, 0x0003d1c62c0d57fb, 0x000fc50b039e35ed, 0x000031fe78392334 }}, + {{ 0x000e8d6d63ec1c25, 0x0003f6e9e2fce99d, 0x00003ff3181bb02f, 0x00027902a68a2108, 0x0000a6146405e04a }, { 0x0003b6c03c53dedd, 0x000386342146a0a4, 0x00021af375e61c53, 0x00034aa878ba826f, 0x00007332a2758132 }}, + {{ 0x00093c8fa198c421, 0x0008ebbb54dde39a, 0x000a32329dbdca62, 0x0002d37e1744e5e8, 0x000030383a3af937 }, { 0x00096449c0bb9f0d, 0x000ec56de058c5cc, 0x0001ca0b54b965e3, 0x000441167216235d, 0x0000a79f1b841b17 }}, + {{ 0x00022e45572f2733, 0x000f574170693bd8, 0x000d22301dde4f72, 0x0001b4044a28e14f, 0x0000500450e45c27 }, { 0x00086c3726a97a95, 0x000d9e2e6a285cd9, 0x0001117303c981ff, 0x000cb23445866f21, 0x00006f51ec40b74e }}, + {{ 0x000ad1ce128a1832, 0x000da2af7172277b, 0x000e51fae431e94f, 0x0002b5aa757b94f0, 0x00000f8d45d654a3 }, { 0x000ebc4f922ef269, 0x000849dd8f71caef, 0x000597638b146921, 0x0007aad37f43272a, 0x000009ef351a878e }}, + {{ 0x0009c02205f112dc, 0x0009e952f41deff1, 0x000acc7b67b111a0, 0x0002d1d510e44a59, 0x0000fa3e8511d623 }, { 0x00051ff946f13b54, 0x000dc69c4b7d692e, 0x000317f509655586, 0x000e6b4462e6392c, 0x0000a96c730c5b29 }}, + }, + { + /* digit=35 (1,2,..,64)*(2^{245})*G */ + {{ 0x0000a3933301eb25, 0x000a651566c5c43e, 0x000854f45d136c9a, 0x000a9b44acff9c91, 0x0000afb49c7ad947 }, { 0x000a2e067e61d8ad, 0x0004dc10dc1eb2bf, 0x000135c5137c224f, 0x000bda57488cfd8c, 0x0000c44a7f8d059f }}, + {{ 0x0006c13cc1c1b2ba, 0x000707f3e5f9f11f, 0x0009e009feccf526, 0x0004546bd9afe153, 0x0000da180beb161c }, { 0x000f8417a1d44bc5, 0x000008e325fc3ee3, 0x00006399ac1041ab, 0x000580ee77109540, 0x0000ceab562965db }}, + {{ 0x000fab17514db1c6, 0x0008172813b230d7, 0x000e97892cd69e71, 0x000cdff6a634d0f5, 0x0000c7df5b396089 }, { 0x000b9815a106666f, 0x000a1a74f7c4f830, 0x000416725238afc5, 0x000ab9aa0a7d2edc, 0x000091170e9acf8b }}, + {{ 0x000db6d961011f15, 0x000439b319540c33, 0x000964ccfd972723, 0x0008f8acb18490f5, 0x00000165db5b23d9 }, { 0x0003f6e09555115e, 0x00005bb146110697, 0x000e9391de70734b, 0x0006302d2b19436a, 0x000046716ed749df }}, + {{ 0x0007e83eda22ea79, 0x0001a8eb6357f373, 0x0008f3f06b88b995, 0x000bf00fcc5d00f7, 0x000073b37b16df09 }, { 0x000b956bc10c800a, 0x000d8731560bf3d1, 0x0009017a42a05ea3, 0x000140abd1086e45, 0x000062690f21558e }}, + {{ 0x000610ad741347e3, 0x000d891c31b9c38e, 0x000a62deaa41683d, 0x000f3c86f0c5aa90, 0x0000f90c15e6f59b }, { 0x000921d5778fdb23, 0x0004ddf02ba8e15d, 0x0005b38621fa615d, 0x00050e06c0337edb, 0x0000f89b6644d026 }}, + {{ 0x000357c9e3ee8927, 0x0003d392c422f7ac, 0x000229dcbeb4ed3f, 0x000d2d5d1fb63536, 0x0000f4ae39a384d5 }, { 0x000c225a84c23dc0, 0x000f5405b2e5b8ce, 0x0000f4e05f1fef35, 0x000d1f1997e94693, 0x0000aeede3f666b8 }}, + {{ 0x000b8a4cbe978aa8, 0x000fbc8ee3c76cb3, 0x0003979fd2b05b7b, 0x000d941563cf1162, 0x0000e542d606a5d0 }, { 0x0005e56df6f115ed, 0x0003ca59ce6bb278, 0x00017eef4378ebbf, 0x0000d8c8dc4afaa5, 0x0000a21b92c92f88 }}, + {{ 0x0000ceb3fa8e1f73, 0x000163c2bf296984, 0x000496ffc2087462, 0x0007ae28f91fc19a, 0x000071c2f69ba91f }, { 0x0004cf10c29819fa, 0x0006660bc9ba369d, 0x000684a977ff8395, 0x000895207927e103, 0x0000429c81a856ee }}, + {{ 0x0009c0d0a7824c13, 0x0004034859dd614c, 0x000a4f0ff846b7f2, 0x000813ed59c475b5, 0x0000cd1b934f7785 }, { 0x000230a6e8f478a9, 0x000dd91c234296f5, 0x00044257d8cd2650, 0x00071f3f22a0a83a, 0x0000ddf899163e18 }}, + {{ 0x0000030a795ce50d, 0x000b20f72e3d5c1e, 0x00056f1e39259abd, 0x000fdf6343771744, 0x0000727bbec1ac3f }, { 0x000c8d087f282848, 0x00056bf1202e9ca6, 0x000c0959dd124705, 0x000b27cf8e79dde6, 0x0000ccb0bbb230c9 }}, + {{ 0x00025cd048f7a136, 0x000913dbce913af4, 0x000b032d3782b56e, 0x0004ed68a909e587, 0x0000f4c845403bdd }, { 0x0001af92d3eba46f, 0x0004abe90e9d8bf9, 0x000aac93883ac9f8, 0x000767dff991616d, 0x00000f2d9eeda667 }}, + {{ 0x000653fe47f4d959, 0x0006992e93fda293, 0x0007b3299374a645, 0x000d388e676bb6c1, 0x00002b338f3417ba }, { 0x000d70a99798c8cb, 0x00040280f680e752, 0x000bc14618d4da4f, 0x000a4773675eefd1, 0x000070102614081d }}, + {{ 0x00005cf3d8688b24, 0x000be0720445c36d, 0x0001892a7d9c11b3, 0x000144fe971d1ac8, 0x000058a7f36a1f2c }, { 0x000e16f5c25e1654, 0x0008e0b7b1c7129b, 0x000f98876c302942, 0x0003df82dffadcc1, 0x000015505c986cfb }}, + {{ 0x000e029db51b3c16, 0x0004fb8aad581350, 0x0008f57808095893, 0x00033c0a0622c211, 0x0000848670b38a49 }, { 0x000627cfbfce8544, 0x000803794fba0377, 0x0006ebdfc7e6bd5a, 0x000ab8f8eaefbd68, 0x0000239802ac6bee }}, + {{ 0x00035dc5396eb247, 0x000c8ef4e4ca4631, 0x00072209072a4380, 0x000c5c200bcef728, 0x0000b72cb6549160 }, { 0x000b0c5738b12183, 0x000822e27bf1bc68, 0x000f7cd34933b26d, 0x000119eda10a224e, 0x0000c80f888434bc }}, + {{ 0x0002cf43d8bba995, 0x000ba2a346060743, 0x00086e27c0a14c45, 0x000f74e0680fe7fe, 0x0000fac5f4d27a25 }, { 0x000dcd1a75979538, 0x000715f5f02ffe6f, 0x0003829db97527f4, 0x00033ad5e3d9eb9a, 0x0000209af43545e2 }}, + {{ 0x000e5c0304618223, 0x000f48bb95560983, 0x000f41c035820ec0, 0x000e596124090a1d, 0x0000cc98be20fb39 }, { 0x000626289bc31852, 0x000ea4b70dceecab, 0x000461f93bf28884, 0x0001ae25cbb2d3c1, 0x0000160674019bab }}, + {{ 0x000865a96da8e141, 0x00085598baebdcf4, 0x00092d795877cd89, 0x0007735d81a28ae0, 0x00006e178e1b6c8e }, { 0x00097f5dfc26037f, 0x0009ccd237936b24, 0x000b16c21697ffd9, 0x00006a247332f4f4, 0x0000ffad165e6c11 }}, + {{ 0x000ec6787acea84e, 0x000febd32be5e0fc, 0x000648d99afacf1e, 0x0003024b17475bcb, 0x0000b743664d4e82 }, { 0x000fe814369d2cde, 0x0008214d8578c70c, 0x0007a8d35754fdd3, 0x000ce5a06c9ad99a, 0x0000e934ddca5774 }}, + {{ 0x0008474a9996f7eb, 0x0002a822f4fb344d, 0x000f6ff85c520bf6, 0x000348c49b882344, 0x0000f420ddd2a3ce }, { 0x0007a033ec204ecb, 0x0004a4d5d1c00d06, 0x0008a69e46c0c3ed, 0x000d712f87fd581d, 0x000024e3f35d2d04 }}, + {{ 0x00023c422eac602c, 0x000d2c19ef2d2e4a, 0x000e3364a85064df, 0x00011200a1bf6767, 0x00001c6223e12f57 }, { 0x00013599f87f98e9, 0x000822d540f83afd, 0x000d84b4acaf5dc6, 0x000912908c2c2c27, 0x00007299ce914de7 }}, + {{ 0x000f627fd7a63a1a, 0x000315cd7b32c6ee, 0x000ffb93983d2957, 0x0005001bb61a5a63, 0x00007c449b79b37a }, { 0x00080cf93461f5c1, 0x0006efcc4c86bc81, 0x000aee0840c9e22d, 0x00095d433e6f9231, 0x0000b966edba2633 }}, + {{ 0x000e0efb3b8b1b5d, 0x00038959884aaf7a, 0x000259a44d6afe4d, 0x000f91f87b1959be, 0x0000337331701bb0 }, { 0x000b01a0216bb368, 0x0000c5eca8c325e7, 0x000671fd9f4f814d, 0x000e76c1d3c91169, 0x000010645e8443c5 }}, + {{ 0x0003a375e672dfca, 0x000e89363a1483f9, 0x0000e68119b07752, 0x000518de589eabe7, 0x0000537b9e85ecc1 }, { 0x000e0eb3eb3def9e, 0x000f5436a2afe69a, 0x0000335814a3a97e, 0x000978293c4f8910, 0x00004bb46fe72211 }}, + {{ 0x0006fcd2c98f37de, 0x0009b0d9d79046e6, 0x000d05f136ad2051, 0x000cf40ca4d650f2, 0x0000c7cbce68d0b3 }, { 0x0006ddebdb5a0c55, 0x00009b5696605f18, 0x00071e6ee7b1b45a, 0x000ccfbcfe77e5d0, 0x000090641c6e2ee9 }}, + {{ 0x000de285025363f9, 0x000513dfddf74157, 0x0008c18c08581775, 0x000318675f7fa130, 0x0000499a04911b96 }, { 0x00079deed354d5f0, 0x00080789e2aeb6b4, 0x0004aedc64ef1a47, 0x000c7ffc3ba2b9ff, 0x0000ff43ce38948a }}, + {{ 0x000eaefbd10b4a81, 0x000379b3a8631e8d, 0x000d9b723b75c7a3, 0x000c1de118de855d, 0x0000b4dea0f1e6a4 }, { 0x00072cd4b1febb5e, 0x0003c8dd150892d4, 0x0005da5ea4d30538, 0x0007c011a37b8468, 0x00003fedb7f726b8 }}, + {{ 0x000910e7019eb79c, 0x0005a51e9f05de9b, 0x000873d8a4a77728, 0x00080664cd404506, 0x0000f06a0d5cacfc }, { 0x000d896e2f49d140, 0x00016eab3aa8f0df, 0x000681650677644b, 0x0007b0620bab2a50, 0x0000225a11e3b301 }}, + {{ 0x000d55e62f7b147f, 0x0002f53308fbd93d, 0x000d256d89ab7031, 0x00090048546cd5a2, 0x00008f07fbe7613f }, { 0x000f410b11223a4e, 0x0001ca02a81ede7f, 0x0003f9e643967f06, 0x0000701866860dd1, 0x000037f2d5f3b15e }}, + {{ 0x00004aa9d3156f13, 0x000b93849ad467b6, 0x000baa288c1e44c4, 0x000e49060efb48b6, 0x00009eca1fbdd19b }, { 0x0003345e17323e5d, 0x0006786754e48e96, 0x0004d412c0772c5f, 0x000fcdab6693cb45, 0x00001eca39d6b65a }}, + {{ 0x0005b4488b3fb1e8, 0x000d26b23a5d8961, 0x00056a2fedc3595e, 0x00081de771fe19e3, 0x00005b981b48385a }, { 0x0002c4f79da9b17b, 0x00017541a1f22bff, 0x000fb8bc6c4a7592, 0x00033ea8e920a8a6, 0x000037f6be35ab5e }}, + {{ 0x0003272fd83d30cb, 0x000f8d04aa38d1b5, 0x000c62b22135dd8a, 0x0003013c03b0db35, 0x00000b015e47a001 }, { 0x0008a2e909f55c72, 0x000d74f07a8e1078, 0x0006e1dea8f06a53, 0x000ee63444a89e13, 0x0000850dcd40ec46 }}, + {{ 0x0002e8a6205f5be2, 0x0007be3b44a3f84f, 0x000324de206fb879, 0x000c74190acb2f34, 0x0000169e7bbd47b0 }, { 0x000baae31cbf3f3b, 0x0006fb69c2d76521, 0x0003d1eae29fe835, 0x00042b610fb5307e, 0x0000e643f0cba58a }}, + {{ 0x000959c3f21417f3, 0x000255bcbea62e7d, 0x000e119c6c6709d7, 0x00009921ce8ac9ea, 0x00004c4448bc0b85 }, { 0x0008f6c667720f2c, 0x000692520e40ae55, 0x0002aeab843a32e2, 0x0008386bc320789e, 0x0000f23eb6cec001 }}, + {{ 0x00005e16bb852160, 0x00052bbc55d2d8b9, 0x00076498b39b48fc, 0x00066161441d58bb, 0x0000a86b13ffc50a }, { 0x00053f7146de7703, 0x0004e40f661acacd, 0x000ff5e2ff690eca, 0x000c929ba3b8784e, 0x00006044fa1265ab }}, + {{ 0x000d45e2d4b575cf, 0x000bfe5e5681e833, 0x0005b966bb760ac6, 0x000f54b386212610, 0x0000920d33916add }, { 0x0000ca34a27332f1, 0x000d33976e8ae983, 0x000b8eea7b6bf672, 0x000057b89b53b2ee, 0x00008733d0036fff }}, + {{ 0x0008a92a0eca49dd, 0x000fb6c783417303, 0x00087c7cb839b4d4, 0x00005e2388ef5beb, 0x0000914d653ceeed }, { 0x00093b5505c32a66, 0x000543f22433ccf6, 0x00063963b0a3b74d, 0x000ee34fcde8362d, 0x000054d95941e49d }}, + {{ 0x000cbb9da03036b6, 0x000b5c0cd1797a18, 0x000e67fc118d1fef, 0x000c2923eea17b47, 0x0000697a3f1abc0d }, { 0x000986f3fa1f44ea, 0x000139c4d8424cff, 0x0000b195410c7287, 0x0003d1ca3db7048f, 0x0000c7c46d5a24e3 }}, + {{ 0x0006210d9e58a61a, 0x00029b5224c08dca, 0x0003db93fef2cd7b, 0x00002ae4f2bbb09b, 0x00004673f3e36b54 }, { 0x000acede4893a3e0, 0x000167a09cb54d69, 0x000fb53a3b5bd9ea, 0x0006f485791eef6d, 0x0000d389cb15387b }}, + {{ 0x000410cbabdf6c15, 0x000067190b771c33, 0x000486535b12a4e1, 0x0004bedff563b71e, 0x0000ecda5577d7de }, { 0x0003d7d5dee03f61, 0x000cc2630da84dd0, 0x0009ea44ecbb751b, 0x000151692d0711ae, 0x000014aba617814f }}, + {{ 0x0004912ebf56ac82, 0x000ada8ced806b84, 0x000424ede5b8b06d, 0x0005059bf222e613, 0x000077ee32509e55 }, { 0x00024cc23645bf6a, 0x000cc742496c27c5, 0x000bcdaf647a66d4, 0x0001f552c8ea3176, 0x000068aa54cc75ce }}, + {{ 0x0005468b2dae9bab, 0x00036d89f6f1b18b, 0x00048f86582e286b, 0x000f286fc2552f00, 0x00007485d84694f9 }, { 0x000ee3525470e487, 0x000b7883ff90f2ba, 0x000e802701cf5d83, 0x000725c2877947bd, 0x00000092a2e7377c }}, + {{ 0x000c1f75e1fc6e4d, 0x000e97d3567962f6, 0x000f15694a198229, 0x00082e3a1379ed61, 0x000016040a7a66fe }, { 0x0005b5c318e65050, 0x00016423b4a79fce, 0x000c6f39817a3245, 0x000c19ca1f20bc83, 0x0000e78407cf44aa }}, + {{ 0x000450a8760aef29, 0x000895d13221bc57, 0x0001f6ee42812871, 0x0004b6ca1707baad, 0x0000156dc61f6269 }, { 0x0002697b59a57f39, 0x0008453e5d0338fe, 0x000e491ec0628d37, 0x000e4b3d01e77f7a, 0x0000a053efd0726a }}, + {{ 0x000d3b14f7467390, 0x00015bdda85d5347, 0x000f825cf9f2e0fe, 0x0008ff5e753e78c1, 0x000059772a4a8d49 }, { 0x0005e906b119a241, 0x00033b0cb8f2bd25, 0x00062c96a2e1fd6c, 0x00050eaab2d50b48, 0x0000699177c6e9dd }}, + {{ 0x000dc63a9d42470f, 0x00013fc8fe280d19, 0x0005151ada0e3238, 0x000c459b0cd8e0de, 0x00004eff38b04e9e }, { 0x000f4187bc92bdf7, 0x000c53b4be1495fb, 0x0007d4178a70e616, 0x0004bd4dc25eb3db, 0x0000eb13072f9e75 }}, + {{ 0x00078c2ec1e3ed4f, 0x00051691416a6a5e, 0x00085c63595aa0e1, 0x000163b5d8f860c7, 0x0000283d923fc854 }, { 0x0005beef7ad3f5b3, 0x0007b815ad735813, 0x000023a1511c7557, 0x0006b79bab4cc9e0, 0x0000199d8c4e721f }}, + {{ 0x0000a1b1af1e78d3, 0x000f4cfdbcf41b9d, 0x0009de07e610f9cc, 0x0008a23d85703662, 0x000010d9697c4fc5 }, { 0x00023c2688aafc37, 0x0007c7503c41c4b1, 0x000af155a9b9f44b, 0x0003f540aa023829, 0x00006df810563aa5 }}, + {{ 0x000f8aff74187332, 0x0008ecc36ba06eee, 0x00044e47cc9819db, 0x000c0b71fface311, 0x000023dc4d264ada }, { 0x00009a815983bd68, 0x00045042d05841bb, 0x000e84cfa0889ade, 0x0001ffa7e42f0e3d, 0x00006a7649f271c7 }}, + {{ 0x000cee582b9889e1, 0x00045a10fee104a6, 0x0007ca159e31ee59, 0x000326733ad7788f, 0x000065aa11dd04af }, { 0x00044cc9c627cd7f, 0x0002267e8f55a7b8, 0x000655974f477505, 0x0003eaf9defbee3c, 0x00000dd76f26bfac }}, + {{ 0x00065e1d58e34a25, 0x00069cd66926e04d, 0x000eaab2a4614ef5, 0x000205a070001e38, 0x00007f9aa8b3ba19 }, { 0x00091abd3f792d82, 0x000bf41be76ca2da, 0x000c3f7835fb599f, 0x00043b7d27895cf5, 0x000000657fe7cdbd }}, + {{ 0x000e6ea8cc26f72a, 0x0001e8387eb36bff, 0x000192dce6e80e03, 0x000c29f5fc5f0710, 0x000060b97966ad86 }, { 0x000db3af99846589, 0x000c765edfa9284e, 0x00013dd8ddcbf735, 0x0008321fd6c510bd, 0x0000a06e6ac55647 }}, + {{ 0x000b32a66e2632f7, 0x000809d6295d4269, 0x0006a471070d02cc, 0x0005b2f040c4f6ac, 0x00004f774520e665 }, { 0x000688122ab3d6ee, 0x0004566f0476ce2b, 0x0002ef7ab5896bbd, 0x000783a097554d95, 0x000014bb84887b9e }}, + {{ 0x0009e6c0473150eb, 0x000218f508efb637, 0x0007f0d62f80b77c, 0x000f0c04f9eb5b5c, 0x000069accc4a1882 }, { 0x000dea6143dd9627, 0x000522ce37a1f698, 0x000a7569ca1c970e, 0x0002be4d3cbd252c, 0x00004fc80b2b91d6 }}, + {{ 0x000e42f66dd9a70c, 0x0000846de6736294, 0x000e21f936b36c3b, 0x000215cc757f7aa1, 0x0000e42c4db111bf }, { 0x000f78959874b51d, 0x00067dc910778965, 0x000f6350f2c2eb73, 0x000e53487a0d690c, 0x00008713f1619ac6 }}, + {{ 0x0009401711af5ba4, 0x000394752b16dba7, 0x0005bbe4aa9f7237, 0x000dc9293deefc0e, 0x0000a97c7a345d34 }, { 0x000c122b07c3fd19, 0x00093e9f21a15cb6, 0x000383dc6fe3e1fc, 0x00079e86c73d2616, 0x00000e249332ac22 }}, + {{ 0x000e79dab8354f95, 0x00047e715f096905, 0x0001cf8235118233, 0x000890ac6508b9b1, 0x0000c766eb659330 }, { 0x000d4ca47748ce93, 0x0008ef06c775aa2e, 0x000b4a7b482a1c21, 0x000611206df09d8c, 0x00003cf5f801f0a6 }}, + {{ 0x00087fc1cec8e5a8, 0x000363610bbb8b54, 0x0002bdc1ebd45d49, 0x0009d74cc0737ce8, 0x000003ccfa0ebc20 }, { 0x000a1b8ea3c1eb6f, 0x0000bb3b8bb36591, 0x000e6b42b27cef52, 0x000ff9344dea5df3, 0x000047617165cfc3 }}, + {{ 0x000fbbcec8aedc8a, 0x000e71967b0867ce, 0x0007ad7a76578ca6, 0x0005466e38d5e0d0, 0x00007711fd87562a }, { 0x000079e8e108ea31, 0x000cbab0263ad100, 0x000dfc57a6a990e5, 0x00006d517d8199e6, 0x0000296d52bf5ee1 }}, + {{ 0x0003b3afef9ade77, 0x000219f8a3422839, 0x000c147c62f6768b, 0x0008c19018097799, 0x0000ebf4142ddbef }, { 0x000ac70895b5ba6b, 0x000cf2a2db3d5ba7, 0x00020ef8474730ce, 0x0007efb73263dc46, 0x00004bc34642342a }}, + {{ 0x00025b0834bbbd7c, 0x000524d5c2f342bb, 0x000a02d27279ffdf, 0x000a440d0bf80907, 0x000081a156479865 }, { 0x000ccd9a92ebac1a, 0x00074a56c3a1ddad, 0x000a29fb8884da05, 0x0007030fefe4ec7e, 0x0000acc66637fdc1 }}, + {{ 0x000105a92d7a6a29, 0x000b4dadb58da1f8, 0x000f37c53dc1205c, 0x000f2214322bbc12, 0x0000f4c05d3cc006 }, { 0x00023213222f51a0, 0x0003bf37a859d51f, 0x00053e0f0648de72, 0x000bec7c9e3ec7ce, 0x0000fa1715d802dd }}, + {{ 0x000bdec2116e9b29, 0x000389c76497ee80, 0x0006bb3874fd1cc1, 0x000343fe15d2b0ac, 0x0000a3fadcb3a4e1 }, { 0x00037248e9d64745, 0x00087efa63b10110, 0x0007dd6bd1db932c, 0x000c6c78bf9e3fa9, 0x00009e31d0655466 }}, + }, + { + /* digit=36 (1,2,..,64)*(2^{252})*G */ + {{ 0x000be5ed0e405bae, 0x0007fc91ec34f9e7, 0x0004b79b18f54024, 0x000b106f3d8772d3, 0x000037037c975e12 }, { 0x000aec44147d71ff, 0x0003d1931e82b100, 0x0007bf1327384e2c, 0x0002fb55ec63d285, 0x0000df2ba6d3b215 }}, + {{ 0x0007a23f356e3059, 0x00041e71e29a3efb, 0x000f94f0f0f98de8, 0x0007a880ecbe906e, 0x00002d869e92df60 }, { 0x000cf0bbf87a34ec, 0x00089c2efd2119d3, 0x000ebfbd0dd06fa3, 0x0006c851303198c6, 0x000030a29d4bfdd3 }}, + {{ 0x00089caef387625f, 0x000fcb72b7247593, 0x00017a38174726e5, 0x0001b102c945353e, 0x0000fcd0db4d1457 }, { 0x0003429bba3484a4, 0x0001753db65ef147, 0x000e6a574289160a, 0x0006d9c5f2dc2cb7, 0x00005d42b1ac334f }}, + {{ 0x00012c9e1ee0d0a1, 0x0003490b01e6e274, 0x000da05414bdd548, 0x000de03a9047e2cc, 0x0000c371569c9623 }, { 0x00078851bc8c9a7d, 0x0000d36794075521, 0x0006dbaa6726fc38, 0x000ce611949c5013, 0x000053af2d9b1059 }}, + {{ 0x00035aa24062df0d, 0x0002a52fb85fa4ca, 0x000d94e1eaa94f02, 0x0001840aed61257c, 0x0000fb93c2113070 }, { 0x0000d05aabec972c, 0x000f75d4421fc835, 0x00007292ec8f71dc, 0x000b37558f6df8ee, 0x0000d9d09ec67e4f }}, + {{ 0x0000235a102cb4ec, 0x000fb35a64785f45, 0x000b0f0672f75fa1, 0x0002e6467bc56637, 0x00008030444d7012 }, { 0x000881065be741fe, 0x000b8d8f2c4aa658, 0x0000fb14fbdf31d8, 0x000b607bac347583, 0x00006adf01034a09 }}, + {{ 0x000f12502f6f25d9, 0x0000c22cc1b5e838, 0x0000f6f93bcc544b, 0x0008c7ddfde2d4ad, 0x0000c68d49d6c0f8 }, { 0x0003511dcdcef6f9, 0x000a7dc7783920a0, 0x000dbd6b689c8148, 0x0000f52a6e80014a, 0x0000730b2f927704 }}, + {{ 0x000211e48a709134, 0x000a06d95a7b1a29, 0x0005aa515d70a8ec, 0x000baface9c4e7ad, 0x00001420ee199ed7 }, { 0x0005f92e47355a0e, 0x000716ec78ef1f42, 0x000fe05e173edf3f, 0x000e4dfe82b2c090, 0x00005f26894a26e6 }}, + {{ 0x0000e5daec35c312, 0x000444631890a36e, 0x000f77190f6f4a99, 0x000791e30703ef5f, 0x0000248d7cab9079 }, { 0x000a1bd7b6437663, 0x00041c3c9c9f1554, 0x00066b1d61acbaae, 0x000aa5c297e4bf17, 0x0000f13072c92c6b }}, + {{ 0x000372def7fe9e8e, 0x000089946191d3d6, 0x000d1815135fa99d, 0x00071e788281fc8d, 0x0000e04cb9db1b55 }, { 0x0003bcefb0c7e3f0, 0x000cb480d8972cae, 0x000672598a8f1310, 0x0004d1cf43f2dd4b, 0x00005a02d3e37cbc }}, + {{ 0x0005e8048f16243a, 0x0002f1ed785e73d3, 0x0005f8bdffd849c1, 0x000a2174aa5db5ab, 0x0000f3e31e72b2c9 }, { 0x000343d185ef14b5, 0x0005c8f1adaa0f81, 0x0005718a48d45076, 0x0005191b828ec3b4, 0x000095c49bafe83d }}, + {{ 0x00037b515fb7aa40, 0x000b844caaa358cb, 0x0003e5a8c6cb2e2d, 0x000dd1d9a1a1db7e, 0x000027da7bc2cde4 }, { 0x0005a481c0d7db4b, 0x00089097b165fbe5, 0x00098c626e327572, 0x000dc967efc322ed, 0x0000c100250a4181 }}, + {{ 0x00056b31e0992f90, 0x000c312c23fc9759, 0x0005706af79af358, 0x0005f4d391571580, 0x000032a43ce4ae5e }, { 0x000b37b634d74c5f, 0x00016d5ff7b33a68, 0x00098146b10a6fb6, 0x000e2c698c8d5dc3, 0x0000a98da6d07fa6 }}, + {{ 0x000f40ac3c4dd2af, 0x000cfd68b11888c9, 0x000de0fd79a08293, 0x000b4f34a5e0e8c3, 0x00005ee377d3a06b }, { 0x000ef1ec28436dca, 0x000149534e0f63a7, 0x0001ec713cd110f6, 0x000f727b7a128bf9, 0x00005d0d98ab5496 }}, + {{ 0x0000f1a43a4a1ad5, 0x0004f0fa36a27378, 0x0005e27077875e3c, 0x000e0939735dc136, 0x0000d7436395198a }, { 0x0000d7c67d880277, 0x000b04bf8cb0ba05, 0x00062fa0ff6e7415, 0x000ff59efb0b5376, 0x00001a3237f9a695 }}, + {{ 0x0008910bd3c74498, 0x000d8af3aeac968f, 0x0008cc252692ed9e, 0x00067be64e4f7fee, 0x000092836060ef1c }, { 0x000890896b28139e, 0x000470cc9c0726e3, 0x0009a0eab753f427, 0x00046bc1b1530956, 0x000038600b6014d5 }}, + {{ 0x00053794e8af97d5, 0x0007c5ca5618c937, 0x000b02a1ade8a949, 0x0005f92a8e665432, 0x0000f8ed81c1d707 }, { 0x000c21fb8d99ba56, 0x000957ce8c0dbccf, 0x00025d6478156ff5, 0x0008c850031e0261, 0x0000b4fb313fc9e3 }}, + {{ 0x000b3bcc8886aff4, 0x0000d16b0843e0bb, 0x0007ae7478a94254, 0x00027e43053b1b3a, 0x000070a88a7167ff }, { 0x00047e1384f31100, 0x0001100c93dce831, 0x00018d1416a58415, 0x000039ce87e79da6, 0x000008370d177e00 }}, + {{ 0x00034714e4870f2c, 0x00057d3af077f30f, 0x00031311a055c13d, 0x0006bc053ea78f84, 0x00006124e88ddd7b }, { 0x000ba0c58a21a931, 0x0002c15c2376c840, 0x00093179e1f91646, 0x000dbf9908315b65, 0x00005e507e2ffacd }}, + {{ 0x000b7587e7d8f6a9, 0x000e47a529a6d482, 0x000d977c0a1b5071, 0x0003784f99c7f250, 0x0000a3bfb164138f }, { 0x00043e21ae8a2808, 0x000fa33e0182a650, 0x0007dfa2f0dba6d3, 0x000d3018fb82ecf8, 0x000055f7520eb309 }}, + {{ 0x000e6559e9dabe39, 0x00010d666c834ea0, 0x000838dbe89dd09d, 0x000dbf1473be43e1, 0x000026c068976f46 }, { 0x000e63ef6977674b, 0x000b38e0c6615b48, 0x000cc7e1ae0786b6, 0x00076b88614abc8f, 0x0000e564e456248e }}, + {{ 0x000faca8d0cebb8d, 0x00074b453b31e91a, 0x0003948a0c0742a8, 0x000bf70660cba7ad, 0x000094b060ddcf41 }, { 0x0008fb9a616bc52b, 0x000311d9ee9761ca, 0x000b47c4be2e0a39, 0x0003f554c3497a16, 0x00001a01ac4f5b97 }}, + {{ 0x00024d301b78fa64, 0x000cefe392511143, 0x000c987a034b5852, 0x000ace55c36bf25b, 0x0000a71003604b62 }, { 0x000b211a0e5ee431, 0x000310cd670407e5, 0x000188521ff34062, 0x000def3e00ea8d4f, 0x0000dbd759ec2451 }}, + {{ 0x000e220933648ca0, 0x000dd205ffe7b376, 0x000aa925c9a6a480, 0x000c325193b7f3ef, 0x0000519935e92c50 }, { 0x0006fa48a2afcd4b, 0x00015dfb486d440d, 0x000568e6ebc32df4, 0x0001776517fcfec4, 0x0000575cad80abc7 }}, + {{ 0x000f59e604baf773, 0x000dc265872cd88f, 0x000b2831aa9d0b38, 0x000f9491c806c1d3, 0x00005ff0f174210f }, { 0x000212f245cd494b, 0x000c4327a0b85dd7, 0x000a21d9cfd1d70b, 0x0008cd713dec5720, 0x00009a66bf0ffb9c }}, + {{ 0x000b75466ceb202d, 0x0001411b5cae787e, 0x00073287eab73556, 0x0005581fb13cb7f5, 0x0000e12d755e6d26 }, { 0x0000fbf8f7c29358, 0x000173bdb6cff322, 0x000a4100e1a2b244, 0x00091aa8b7440f37, 0x000020bd4b0b9af2 }}, + {{ 0x000913b93cb994fa, 0x00098e1df83097f7, 0x000bdc374ec3de91, 0x00039352f28738f6, 0x0000d11cf177dae9 }, { 0x0000deb384609912, 0x00033931fe896366, 0x000f96a98121a3f9, 0x0007b93a17b98443, 0x0000d01181097cf0 }}, + {{ 0x000ab007761cf94e, 0x00006c601a982e6c, 0x000069e1c5b02fb1, 0x000c41d051dd35e1, 0x000099e98577e95f }, { 0x0009c7a367e425f4, 0x000a71f6565e15ae, 0x0008a2c80cbe203f, 0x0007aea37bf30f52, 0x00007bf67fb4b3c4 }}, + {{ 0x000f733e7e5b9a30, 0x0006ca552f03ad8b, 0x000842daf7427fea, 0x000299789c7e8dbe, 0x0000e03dc9ad2e74 }, { 0x000abfec927f95e5, 0x00016f5b8444823a, 0x0007c1385ba9089a, 0x000291140b7224e8, 0x00005f27d833f8e6 }}, + {{ 0x000cff1ae2810e9b, 0x0007a10079ea1bd1, 0x000cb35e424422ff, 0x000d7f24f1ad9add, 0x0000223f4eaec892 }, { 0x0002b37f69fa04b0, 0x0000749d4cee1172, 0x000b1d83599fe6e3, 0x000489c388e9f275, 0x000066a58613b395 }}, + {{ 0x000f59bf5ce72e10, 0x000e1fdb40146044, 0x000939e7911b9cb2, 0x0004b1efea8ce991, 0x0000b81a0bac185a }, { 0x0002588530df2495, 0x0009d4f923a277d8, 0x000ce2b6aed5cd2a, 0x000152751d9bc9b7, 0x0000c023c697a0ab }}, + {{ 0x000cd15a3cc6fb9b, 0x000a19f82e4c6346, 0x0003cd730abb95f1, 0x000222f258efb831, 0x000068413078deb3 }, { 0x0001e7c1ed462bd3, 0x000d5fdbfcd8fb51, 0x000f5c6d0b354d1b, 0x000acc02f31db2ee, 0x00009cf8f2c231b2 }}, + {{ 0x000c1b23430b5424, 0x000e10cdc9a151e9, 0x00079f9585161afe, 0x000026de79edbfc0, 0x0000dd01a58284f2 }, { 0x000676c6277d628a, 0x000928b7f8474b9c, 0x00063aff51f1e745, 0x0007dad5a90657c0, 0x0000a3eff6f709d8 }}, + {{ 0x0006cc32b494aa16, 0x0001c47b7d6dee80, 0x00077299605ebcb8, 0x000011fed0948d92, 0x0000bd274de4ad55 }, { 0x0004428c5ee2c46b, 0x0006b64061ef338c, 0x0001fdc05d8e8c6b, 0x000d73e9b1606fa0, 0x0000c773ba6a62af }}, + {{ 0x000205fb46da5307, 0x00021aef91b0b2ab, 0x00033e3801f8f3cd, 0x000ace1edce870a2, 0x0000468f39384030 }, { 0x0007205ff5fc9697, 0x00068e744cefa48d, 0x0007ca7816e1c2c7, 0x000209163b6fc37a, 0x000044e445df560c }}, + {{ 0x000b1f606a304a7a, 0x00093328887de41e, 0x000b9b7f44ca4fe8, 0x0000c70175de0df0, 0x0000e1591f40e90c }, { 0x000901a9494fafda, 0x0006996e30ea15cb, 0x0001084797dc0097, 0x000bbd98f575816a, 0x00004054cb1612ae }}, + {{ 0x0009f2630bb6056d, 0x000a208d8fc35489, 0x00003d4999fd2fa9, 0x000f8f85a5010b5d, 0x000008b55032749d }, { 0x00083d306cbd6562, 0x00019afb4e3b76fc, 0x000bf00347df1b02, 0x0009b30ce14e51dd, 0x00007b2794a6a001 }}, + {{ 0x0007c824b04ea420, 0x000471577d4715c5, 0x000913a80d8575a1, 0x000c5c986d30019f, 0x0000d81758c67e87 }, { 0x000c66382097faea, 0x000692fd8f92a230, 0x00048f5357b258f0, 0x000eb2f9a4a557fd, 0x00000e310f808729 }}, + {{ 0x000f50250aaa7e2c, 0x000928ff83a92661, 0x0006e1b1983af0c4, 0x0003f70ebc7741c9, 0x00004f20a513065d }, { 0x000e8025b4418b73, 0x0007bc81477b6547, 0x000d976fbfe6b5d7, 0x000bb3597c65b900, 0x0000304bd4ae9453 }}, + {{ 0x000683125ec1f5d5, 0x00090ef12045cf9f, 0x000ea649d65eec4a, 0x000d990256fcb2ca, 0x00006feb137cf030 }, { 0x0005f11e11f4e767, 0x0003deae312410e0, 0x000766e7ca1eab36, 0x00013d80767e2867, 0x0000b79acfa040a4 }}, + {{ 0x0004f1f61b1ed88a, 0x00070ef9d3472c26, 0x0007628b1d2a9b6f, 0x0003d25ee3adff69, 0x00007d01488de9be }, { 0x000bb88d5112808b, 0x00094ab9bfb9093f, 0x000ea55ab0ade7e8, 0x0004be60d3df8ae8, 0x0000c9e427871646 }}, + {{ 0x000b283e83436d45, 0x000d64ce88c50d10, 0x00079d4eddcc4a93, 0x000ba49e47532e80, 0x0000ad9d0c90c59e }, { 0x0000afcff7b09b52, 0x000a09149e692749, 0x000787b8e224e80b, 0x0000d3e8eb96e966, 0x00009458dc2fcb01 }}, + {{ 0x0001382d0c39d19c, 0x0000783210b08568, 0x00025db81b7d5f44, 0x000933909a5a8060, 0x00007b38a809ed42 }, { 0x0004fa1871dda89f, 0x0001b5381927e27e, 0x000165fae9106b66, 0x00004c3ce46c63b3, 0x0000a770fae667f6 }}, + {{ 0x00070bd00e57f777, 0x00059bcd7466b255, 0x000a84178c56f6d2, 0x000449f8884ddbe1, 0x0000555f11ecdfbe }, { 0x000c7d249df1def7, 0x000bf29128bfa0f9, 0x000ccfa016cb0872, 0x0008881b192fce1b, 0x00005893d5343ec4 }}, + {{ 0x000ac38b3da9a9f7, 0x00031bf0bbc6229b, 0x000e25158959fa61, 0x000d9e8051c83c05, 0x0000da4e85bd1746 }, { 0x000896ef10770631, 0x00086e3ec0dfcfd2, 0x000e627395068017, 0x0000c7966f2ecda8, 0x000082a16d1f7601 }}, + {{ 0x0004a0885be18687, 0x000916a0fe9a745f, 0x000c5529e8801da0, 0x00096254908249fc, 0x0000b2610ba18a62 }, { 0x0009895a50a60550, 0x00067317d7ead588, 0x00058f0f76b3c827, 0x000274a89f96cf7d, 0x00000ccc62724ca9 }}, + {{ 0x00023b9b36df10cd, 0x000ebe5573cecfa3, 0x000e9f2368affd6b, 0x000f7772066497ea, 0x0000e7189610325f }, { 0x0009369a44ac6ad3, 0x000660a0115a3c12, 0x000b042f2fca2382, 0x000e839789414c40, 0x000036e7483f7882 }}, + {{ 0x000ee3e9bd875a0b, 0x00079f5903fa2711, 0x00029af6a861120e, 0x000561354e6da0fd, 0x00000c0f6913abd6 }, { 0x000948148819fd8a, 0x0008e1ce27a94979, 0x0003f4d9497c8870, 0x0002f21ca36d254e, 0x00009bc3a89fe40b }}, + {{ 0x0007862d5db75ad2, 0x0002c6940d9cae17, 0x000f9a1ae1ab94f3, 0x00036e44b0586957, 0x00009909fd1ff97d }, { 0x00039531fedb222b, 0x0005a6e503491266, 0x00010e2ff490e472, 0x0009425a62be30bd, 0x0000fb87c6381af0 }}, + {{ 0x0007bbabb687d4ad, 0x000bd11b29353593, 0x000677d066b77a80, 0x0008b697fda38aa9, 0x00008397f4e80b81 }, { 0x000dd9c0966b3966, 0x00013e02c9477784, 0x00052c405d304bb0, 0x00066490cd69a517, 0x000086e14cf5ff32 }}, + {{ 0x00059cf5ae217581, 0x000af4d64f16ea85, 0x000628d490757f4e, 0x000f26f3e8b1cac1, 0x0000aed72ff0985a }, { 0x000c6e2c5d3b9f54, 0x00023bb0e2af19ae, 0x0000b4b46034cba9, 0x000cf35201f717ec, 0x000060bfeddfa2f6 }}, + {{ 0x00077eaf4cee5651, 0x000997d3fca4752b, 0x00086a9346dd5414, 0x0009165b5f12c094, 0x0000c95b968e8b8d }, { 0x000b6d88c7322484, 0x000a381fda78fd37, 0x000faae161f6e327, 0x000d9a7c4848a83a, 0x0000f137df7a291d }}, + {{ 0x000cc1bade2b6b43, 0x0007647ce417d99b, 0x0002cecfb1e654f4, 0x000bbf0234b93dcc, 0x00004be18fe3d4f5 }, { 0x0000e6a5a4e609ba, 0x00076114bc5ee855, 0x000470a83c3f2818, 0x000b337f1a561be7, 0x00002e0b52e72fc0 }}, + {{ 0x000ae9bcf25208f9, 0x000071fa00b500bb, 0x000ba2941103788d, 0x0003b71855c098cf, 0x0000543d6d693c3a }, { 0x000d4f3fa4da32f1, 0x00035838435a2c5b, 0x000b8e5e069e748f, 0x0009a03e38ceaed6, 0x0000c306dcaca2f2 }}, + {{ 0x0000ccfe8e6e9bc7, 0x000ea10a3db32927, 0x000a4469c7dc6903, 0x000ae1fa9e28a37d, 0x000071256bb0ed1f }, { 0x0008ff50ee6ae731, 0x000b3e03ba464b2d, 0x000dd3aa52a9b6ac, 0x00044e1ab0f39d60, 0x0000e50f6112c18d }}, + {{ 0x000bd9abbfb15903, 0x0000d58fa28a7e00, 0x0005001c40b5393d, 0x000c99366c726cf2, 0x0000598cccad0929 }, { 0x00032a8b5f2075fd, 0x000f2074dfc47b3a, 0x000269286dbe5403, 0x00032c0e2d9c65fc, 0x0000df0b4df59f16 }}, + {{ 0x000db8b22b2b7d9a, 0x000e26112e832330, 0x0002cf110cfd3ee9, 0x000fa75610fbb351, 0x0000c773f37679e1 }, { 0x000b47b17faba833, 0x0009efb89be01445, 0x00066f0f5fbb5ec7, 0x00061961c8c77d24, 0x0000cb969988de53 }}, + {{ 0x000d2b61710647ca, 0x000d8b016287b52d, 0x000ff3a22d3f5266, 0x0008dbb73a134351, 0x00001e8a651d5fbf }, { 0x000061da3d7648b8, 0x00081291da3b3b75, 0x0008b862d1d32fd2, 0x000cf1d835f946d7, 0x00009b66ecb267fb }}, + {{ 0x0005057a8f03a393, 0x000010dec1b49919, 0x0000a1ef0b1f2d70, 0x000769be0f12195e, 0x0000fd15ee98236f }, { 0x0007f25d916cad26, 0x000a3112e1301367, 0x0005bbd3af843715, 0x00053c82b97dfbb3, 0x000049bfae8df046 }}, + {{ 0x0001f37cbd3c62c7, 0x00088986d840cd64, 0x0000a440abcf9eb9, 0x00065da61471d62c, 0x00009f893242a192 }, { 0x000904261943916d, 0x000350f4b7099851, 0x000bd6a472422402, 0x000fcf7467a7bd33, 0x0000da56f786a44f }}, + {{ 0x00056fbb9354d802, 0x0008467cc36975fb, 0x0007e7ef6267c1ac, 0x00063a89402acf16, 0x000003023c61410c }, { 0x0001b7d9bedec678, 0x000950de9e405698, 0x000eb390c630ef9a, 0x0002b13977b99c65, 0x00009969974b5c5f }}, + {{ 0x0007c7289562f9a3, 0x00075c0c716cc0a1, 0x00006da1f3b47558, 0x0003bc7c6f4b5ff9, 0x00000ff02fe0d66b }, { 0x000283d331c15563, 0x000ed85da6be7de9, 0x000da07e001d37e7, 0x0009ee5e8ff71530, 0x00002b7baaf41117 }}, + {{ 0x000b9d6b1bdf4ea1, 0x0003c992a8498705, 0x000ae5b9f8196de5, 0x0004d3fe5a716964, 0x00004e830f707f38 }, { 0x000e1481a495c70b, 0x000ccd4a52f313bc, 0x000f9565c4bba5c4, 0x00077168fc9dd959, 0x0000dff7b5e2bd75 }}, + {{ 0x0004af860fe1d658, 0x0005c3a43228d831, 0x00003626b989c96b, 0x000ceba2924ae1c3, 0x0000c45b79310a64 }, { 0x0002ceb1de0d0667, 0x00088613f714aa18, 0x000d68a9c780c9b4, 0x000a36f94f51865a, 0x0000055e19d4f0d9 }}, + }, +}; + +#endif /* #if !defined(_DISABLE_ECP_256R1_HARDCODED_BP_TBL_) */ + +IPP_OWN_DEFN(const cpPrecompAP *, gfpec_precom_nistP256r1_radix52_fun, (void)) +{ + static cpPrecompAP t = { + /* w */ 7, + /* select function */ p256r1_select_ap_w7_ifma, + /* precomputed data */ (BNU_CHUNK_T *)ifma_ec_nistp256r1_bp_precomp + }; + return &t; +} + +#endif // (_IPP32E >= _IPP32E_K1) + +#endif /* #ifndef IFMA_ECPRECOMP7_P256_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_norm.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_norm.c new file mode 100644 index 0000000..1eb0568 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_norm.c @@ -0,0 +1,269 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include +#include +#include + +IPP_OWN_DEFN(m512, ifma_lnorm52, (const m512 a)) +{ + const m512 mask52 = set1_i64(DIGIT_MASK); + const m512 one = set1_i64(1ULL); + const mask64 mask_shift_carry = 0xffffffffffffff00; + + const m512 idxi8 = set_i64(0x3736353433323130, // 55, 54, 53, 52, 51, 50, 49, 48 + 0x2f2e2d2c2b2a2928, // 47, 46, 45, 44, 43, 42, 41, 40 + 0x2726252423222120, // 39, 38, 37, 36, 35, 34, 33, 32 + 0x1f1e1d1c1b1a1918, // 31, 30, 29, 28, 27, 26, 25, 24 + 0x1716151413121110, // 23, 22, 21, 20, 19, 18, 17, 16 + 0x0f0e0d0c0b0a0908, // 15, 14, 13, 12, 11, 10, 9, 8 + 0x0706050403020100, // 7, 6, 5, 4, 3, 2, 1, 0 + 0x0); // 0, 0, 0, 0, 0, 0, 0, 0 + + m512 r = a; + int k1, k2; + m512 carry = srai_i64(r, DIGIT_SIZE); + carry = maskz_permutexvar_i8(mask_shift_carry, idxi8, carry); + + r = and_i64(r, mask52); + r = add_i64(r, carry); + + k2 = (int)(cmp_i64_mask(mask52, r, _MM_CMPINT_EQ)); + k1 = (int)(cmp_i64_mask(mask52, r, _MM_CMPINT_LT)); + + k1 = k2 + (k1 << 1); + k1 ^= k2; + + r = mask_add_i64(r, (mask8)(k1), r, one); + r = and_i64(r, mask52); + + return r; +} + +IPP_OWN_DEFN(void, ifma_lnorm52_dual, (m512 pr1[], const m512 a1, m512 pr2[], const m512 a2)) +{ + const m512 mask52 = set1_i64(DIGIT_MASK); + const m512 one = set1_i64(1UL); + const mask64 mask_shift_carry = 0xffffffffffffff00; + + const m512 idxi8 = set_i64(0x3736353433323130, // 55, 54, 53, 52, 51, 50, 49, 48 + 0x2f2e2d2c2b2a2928, // 47, 46, 45, 44, 43, 42, 41, 40 + 0x2726252423222120, // 39, 38, 37, 36, 35, 34, 33, 32 + 0x1f1e1d1c1b1a1918, // 31, 30, 29, 28, 27, 26, 25, 24 + 0x1716151413121110, // 23, 22, 21, 20, 19, 18, 17, 16 + 0x0f0e0d0c0b0a0908, // 15, 14, 13, 12, 11, 10, 9, 8 + 0x0706050403020100, // 7, 6, 5, 4, 3, 2, 1, 0 + 0x0); // 0, 0, 0, 0, 0, 0, 0, 0 + + m512 r1 = a1; + m512 r2 = a2; + int k1_1, k1_2, k2_1, k2_2; + + m512 carry1 = srai_i64(r1, DIGIT_SIZE); + m512 carry2 = srai_i64(r2, DIGIT_SIZE); + carry1 = maskz_permutexvar_i8(mask_shift_carry, idxi8, carry1); + carry2 = maskz_permutexvar_i8(mask_shift_carry, idxi8, carry2); + + r1 = and_i64(r1, mask52); + r1 = add_i64(r1, carry1); + r2 = and_i64(r2, mask52); + r2 = add_i64(r2, carry2); + + k2_1 = (int)(cmp_i64_mask(mask52, r1, _MM_CMPINT_EQ)); + k1_1 = (int)(cmp_i64_mask(mask52, r1, _MM_CMPINT_LT)); + k2_2 = (int)(cmp_i64_mask(mask52, r2, _MM_CMPINT_EQ)); + k1_2 = (int)(cmp_i64_mask(mask52, r2, _MM_CMPINT_LT)); + + k1_1 = k2_1 + (k1_1 << 1); + k1_1 ^= k2_1; + k1_2 = k2_2 + (k1_2 << 1); + k1_2 ^= k2_2; + + r1 = mask_add_i64(r1, (mask8)(k1_1), r1, one); + r1 = and_i64(r1, mask52); + r2 = mask_add_i64(r2, (mask8)(k1_2), r2, one); + r2 = and_i64(r2, mask52); + + *pr1 = r1; + *pr2 = r2; + return; +} + + +IPP_OWN_DEFN(m512, ifma_norm52, (const m512 a)) +{ + const m512 mask52 = set1_i64(DIGIT_MASK); + const m512 one = set1_i64(1LL); + const m512 mone = set1_i64(-1LL); + const mask64 mask_shift_carry = 0xFFFFFFFFFFFFFF00; + + const m512 idxi8 = set_i64(0x3736353433323130, // 55, 54, 53, 52, 51, 50, 49, 48 + 0x2f2e2d2c2b2a2928, // 47, 46, 45, 44, 43, 42, 41, 40 + 0x2726252423222120, // 39, 38, 37, 36, 35, 34, 33, 32 + 0x1f1e1d1c1b1a1918, // 31, 30, 29, 28, 27, 26, 25, 24 + 0x1716151413121110, // 23, 22, 21, 20, 19, 18, 17, 16 + 0x0f0e0d0c0b0a0908, // 15, 14, 13, 12, 11, 10, 9, 8 + 0x0706050403020100, // 7, 6, 5, 4, 3, 2, 1, 0 + 0x0); // 0, 0, 0, 0, 0, 0, 0, 0 + + m512 r = a; + + /* standart step - first round normalization */ + m512 carry = srai_i64(r, DIGIT_SIZE); + carry = maskz_permutexvar_i8(mask_shift_carry, idxi8, carry); + + r = and_i64(r, mask52); + r = add_i64(r, carry); + + /* create mask add ONE(1) to slot */ + int k2 = (int)(cmp_i64_mask(mask52, r, _MM_CMPINT_EQ)); /* (r) == 0xF(13) */ + int k1 = (int)(cmp_i64_mask(mask52, r, _MM_CMPINT_LT)); /* (r) > 0xF(13) */ + + k1 = k2 + (k1 << 1); + k1 ^= k2; + + r = mask_add_i64(r, (mask8)(k1), r, one); + + /* create mask add MINUS ONE(-1) to slot */ + const int cadd = 0x100; + + int ma = (int)(cmp_i64_mask(r, one, _MM_CMPINT_GE)); /* (r) >= 1 */ + int mb = (int)(cmp_i64_mask(r, setzero_i64(), _MM_CMPINT_LT)); /* (r) < 0 */ + + /* add 0x100 - we make a number from which we subtract intentionally more */ + ma += cadd; + /* we emulate a shift to a neighboring slot, + * as well as perform masking with 0xFF, + * so as NOT to get a number greater than from which we will subtract. + */ + mb = (mb << 1) & 0xFF; + + /* calculate loans */ + const int mc = ma - mb; + /* after the step above, the problem arises that after subtracting from (ma), + * we see dips of units (below in response) in those slots that do not need to add -1 + * and these slots can be determined if 3 conditions are met: + * 1) (ma) == 1 + * 2) (mb) == 0 + * 3) (mc) == 1 + */ + const int mx = ~(ma & (~mb) & mc); + /* add mask mx AND if there was a suppression of 1 in the slot - add minus 1 */ + const int mask_mone = (mx & mc) | (ma & ~mc); + + r = mask_add_i64(r, (mask8)mask_mone, r, mone); + r = and_i64(r, mask52); + + return r; +} + +IPP_OWN_DEFN(void, ifma_norm52_dual, (m512 pr1[], const m512 a1, m512 pr2[], const m512 a2)) +{ + const m512 mask52 = set1_i64(DIGIT_MASK); + const m512 one = set1_i64(1ULL); + const m512 mone = set1_i64(-1); + const mask64 mask_shift_carry = 0xFFFFFFFFFFFFFF00; + + const m512 idxi8 = set_i64(0x3736353433323130, // 55, 54, 53, 52, 51, 50, 49, 48 + 0x2f2e2d2c2b2a2928, // 47, 46, 45, 44, 43, 42, 41, 40 + 0x2726252423222120, // 39, 38, 37, 36, 35, 34, 33, 32 + 0x1f1e1d1c1b1a1918, // 31, 30, 29, 28, 27, 26, 25, 24 + 0x1716151413121110, // 23, 22, 21, 20, 19, 18, 17, 16 + 0x0f0e0d0c0b0a0908, // 15, 14, 13, 12, 11, 10, 9, 8 + 0x0706050403020100, // 7, 6, 5, 4, 3, 2, 1, 0 + 0x0); // 0, 0, 0, 0, 0, 0, 0, 0 + + m512 r1 = a1; + m512 r2 = a2; + /* one */ + /* 1 */ + m512 carry1 = srai_i64(r1, DIGIT_SIZE); + carry1 = maskz_permutexvar_i8(mask_shift_carry, idxi8, carry1); + /* 2 */ + m512 carry2 = srai_i64(r2, DIGIT_SIZE); + carry2 = maskz_permutexvar_i8(mask_shift_carry, idxi8, carry2); + + /* 1 */ + r1 = and_i64(r1, mask52); + r1 = add_i64(r1, carry1); + + /* 2 */ + r2 = and_i64(r2, mask52); + r2 = add_i64(r2, carry2); + + /* add one slots */ + /* 1 */ + int k21 = (int)(cmp_i64_mask(mask52, r1, _MM_CMPINT_EQ)); + int k11 = (int)(cmp_i64_mask(mask52, r1, _MM_CMPINT_LT)); + /* 2 */ + int k22 = (int)(cmp_i64_mask(mask52, r2, _MM_CMPINT_EQ)); + int k12 = (int)(cmp_i64_mask(mask52, r2, _MM_CMPINT_LT)); + + /* 1 */ + k11 = k21 + (k11 << 1); + k11 ^= k21; + /* 2 */ + k12 = k22 + (k12 << 1); + k12 ^= k22; + + r1 = mask_add_i64(r1, (mask8)(k11), r1, one); + r2 = mask_add_i64(r2, (mask8)(k12), r2, one); + + /* add minus one in slots */ + const int cadd = 0x100; + + /* 1 */ + int ma1 = (int)(cmp_i64_mask(r1, one, _MM_CMPINT_GE)); + int mb1 = (int)(cmp_i64_mask(r1, setzero_i64(), _MM_CMPINT_LT)); + /* 2 */ + int ma2 = (int)(cmp_i64_mask(r2, one, _MM_CMPINT_GE)); + int mb2 = (int)(cmp_i64_mask(r2, setzero_i64(), _MM_CMPINT_LT)); + + /* 1 */ + ma1 += cadd; + mb1 = (mb1 << 1) & 0xFF; + /* 2 */ + ma2 += cadd; + mb2 = (mb2 << 1) & 0xFF; + + /* 1 */ + const int mc1 = ma1 - mb1; + const int mx1 = ~(ma1 & (~mb1) & mc1); + /* 2 */ + const int mc2 = ma2 - mb2; + const int mx2 = ~(ma2 & (~mb2) & mc2); + + const int mask_mone1 = (mx1 & mc1) | (ma1 & ~mc1); + const int mask_mone2 = (mx2 & mc2) | (ma2 & ~mc2); + + /* 1 */ + r1 = mask_add_i64(r1, (mask8)mask_mone1, r1, mone); + r1 = and_i64(r1, mask52); + /* 2 */ + r2 = mask_add_i64(r2, (mask8)mask_mone2, r2, mone); + r2 = and_i64(r2, mask52); + + *pr1 = r1; + *pr2 = r2; + return; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_norm.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_norm.h new file mode 100644 index 0000000..a4bcf4b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ecnist/ifma_norm.h @@ -0,0 +1,73 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ +#ifndef _IFMA_NORM_H_ +#define _IFMA_NORM_H_ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ifma_alias_avx512.h" + +/** + * \brief + * + * Lightweight single-round normalization. Can be used after multiplication or + * addition only. + * + * \param[in] a ptr value (52 radix and more) + * \return m512 value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_lnorm52, (const m512 a)) + +/** + * \brief + * + * Dual variant of ifma_lnorm52() function. + * + * \param[out] pr1 ptr first value (in radix 2^52) + * \param[in] a1 value first (52 radix and more) + * \param[out] pr2 ptr second value (in radix 2^52) + * \param[in] a2 ptr value second (52 radix and more) + */ +IPP_OWN_DECL(void, ifma_lnorm52_dual, (m512 pr1[], const m512 a1, m512 pr2[], const m512 a2)) + +/** + * \brief + * + * Subtraction single-round normalization. Can be used after subtraction or other. + * + * \param[in] a ptr value (52 radix and more) + * \return m512 value (in radix 2^52) + */ +IPP_OWN_DECL(m512, ifma_norm52, (const m512 a)) + +/** + * \brief + * + * Dual variant of ifma_norm52() function. + * + * \param[out] pr1 ptr first value (in radix 2^52) + * \param[in] a1 value first (52 radix and more) + * \param[out] pr2 ptr second value (in radix 2^52) + * \param[in] a2 ptr value second (52 radix and more) + */ +IPP_OWN_DECL(void, ifma_norm52_dual, (m512 pr1[], const m512 a1, m512 pr2[], const m512 a2)) + +#endif // (_IPP32E >= _IPP32E_K1) + +#endif // _IFMA_NORM_H_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/emptyfile.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/emptyfile.c new file mode 100644 index 0000000..4cd910b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/emptyfile.c @@ -0,0 +1,20 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +typedef int to_avoid_translation_unit_is_empty_warning; + +/* The empty file to build a dynamic library in Visual Studio. The IDE does not produce a dll without source files */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/exports.linux.lib-export b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/exports.linux.lib-export new file mode 100644 index 0000000..0ac719a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/exports.linux.lib-export @@ -0,0 +1,1123 @@ +EXTERN (ippcpInit) +EXTERN (cpGetReg) +EXTERN (cpStartTscp) +EXTERN (cpStopTscp) +EXTERN (cpStartTsc) +EXTERN (cpStopTsc) +EXTERN (cpGetCacheSize) +EXTERN (cpGetFeature) +EXTERN (ippcpSetCpuFeatures) +EXTERN (ippcpGetCpuFeatures) +EXTERN (ippcpGetCpuClocks) +EXTERN (ippcpSetNumThreads) +EXTERN (ippcpGetNumThreads) +EXTERN (ippcpGetEnabledCpuFeatures) +EXTERN (ippcpGetEnabledNumThreads) +EXTERN (ippcpGetStatusString) + +EXTERN (ippcpGetLibVersion) +EXTERN (ippsDESGetSize) +EXTERN (ippsDESInit) +EXTERN (ippsDESPack) +EXTERN (ippsDESUnpack) +EXTERN (ippsTDESEncryptECB) +EXTERN (ippsTDESDecryptECB) +EXTERN (ippsTDESEncryptCBC) +EXTERN (ippsTDESDecryptCBC) +EXTERN (ippsTDESEncryptCFB) +EXTERN (ippsTDESDecryptCFB) +EXTERN (ippsTDESEncryptOFB) +EXTERN (ippsTDESDecryptOFB) +EXTERN (ippsTDESEncryptCTR) +EXTERN (ippsTDESDecryptCTR) +EXTERN (ippsAESGetSize) +EXTERN (ippsAESInit) +EXTERN (ippsAESSetKey) +EXTERN (ippsAESPack) +EXTERN (ippsAESUnpack) +EXTERN (ippsAESSetupNoise) +EXTERN (ippsAES_GCMSetupNoise) +EXTERN (ippsAES_CMACSetupNoise) +EXTERN (ippsAESEncryptECB) +EXTERN (ippsAESDecryptECB) +EXTERN (ippsAESEncryptCBC) +EXTERN (ippsAESEncryptCBC_CS1) +EXTERN (ippsAESEncryptCBC_CS2) +EXTERN (ippsAESEncryptCBC_CS3) +EXTERN (ippsAESDecryptCBC) +EXTERN (ippsAESDecryptCBC_CS1) +EXTERN (ippsAESDecryptCBC_CS2) +EXTERN (ippsAESDecryptCBC_CS3) +EXTERN (ippsAESEncryptCFB) +EXTERN (ippsAESDecryptCFB) +EXTERN (ippsAESEncryptOFB) +EXTERN (ippsAESDecryptOFB) +EXTERN (ippsAESEncryptCTR) +EXTERN (ippsAESDecryptCTR) +EXTERN (ippsAESEncryptXTS_Direct) +EXTERN (ippsAESDecryptXTS_Direct) +EXTERN (ippsAES_EncryptCFB16_MB) +EXTERN (ippsSMS4GetSize) +EXTERN (ippsSMS4Init) +EXTERN (ippsSMS4SetKey) +EXTERN (ippsSMS4EncryptECB) +EXTERN (ippsSMS4DecryptECB) +EXTERN (ippsSMS4EncryptCBC) +EXTERN (ippsSMS4EncryptCBC_CS1) +EXTERN (ippsSMS4EncryptCBC_CS2) +EXTERN (ippsSMS4EncryptCBC_CS3) +EXTERN (ippsSMS4DecryptCBC) +EXTERN (ippsSMS4DecryptCBC_CS1) +EXTERN (ippsSMS4DecryptCBC_CS2) +EXTERN (ippsSMS4DecryptCBC_CS3) +EXTERN (ippsSMS4EncryptCFB) +EXTERN (ippsSMS4DecryptCFB) +EXTERN (ippsSMS4EncryptOFB) +EXTERN (ippsSMS4DecryptOFB) +EXTERN (ippsSMS4EncryptCTR) +EXTERN (ippsSMS4DecryptCTR) +EXTERN (ippsSMS4_CCMGetSize) +EXTERN (ippsSMS4_CCMInit) +EXTERN (ippsSMS4_CCMMessageLen) +EXTERN (ippsSMS4_CCMTagLen) +EXTERN (ippsSMS4_CCMStart) +EXTERN (ippsSMS4_CCMEncrypt) +EXTERN (ippsSMS4_CCMDecrypt) +EXTERN (ippsSMS4_CCMGetTag) +EXTERN (ippsAES_CCMGetSize) +EXTERN (ippsAES_CCMInit) +EXTERN (ippsAES_CCMMessageLen) +EXTERN (ippsAES_CCMTagLen) +EXTERN (ippsAES_CCMStart) +EXTERN (ippsAES_CCMEncrypt) +EXTERN (ippsAES_CCMDecrypt) +EXTERN (ippsAES_CCMGetTag) +EXTERN (ippsAES_GCMGetSize) +EXTERN (ippsAES_GCMInit) +EXTERN (ippsAES_GCMReinit) +EXTERN (ippsAES_GCMReset) +EXTERN (ippsAES_GCMProcessIV) +EXTERN (ippsAES_GCMProcessAAD) +EXTERN (ippsAES_GCMStart) +EXTERN (ippsAES_GCMEncrypt) +EXTERN (ippsAES_GCMDecrypt) +EXTERN (ippsAES_GCMGetTag) +EXTERN (ippsAES_XTSGetSize) +EXTERN (ippsAES_XTSInit) +EXTERN (ippsAES_XTSEncrypt) +EXTERN (ippsAES_XTSDecrypt) +EXTERN (ippsAES_S2V_CMAC) +EXTERN (ippsAES_SIVEncrypt) +EXTERN (ippsAES_SIVDecrypt) +EXTERN (ippsAES_CMACGetSize) +EXTERN (ippsAES_CMACInit) +EXTERN (ippsAES_CMACUpdate) +EXTERN (ippsAES_CMACFinal) +EXTERN (ippsAES_CMACGetTag) +EXTERN (ippsARCFourCheckKey) +EXTERN (ippsARCFourGetSize) +EXTERN (ippsARCFourInit) +EXTERN (ippsARCFourReset) +EXTERN (ippsARCFourPack) +EXTERN (ippsARCFourUnpack) +EXTERN (ippsARCFourEncrypt) +EXTERN (ippsARCFourDecrypt) +EXTERN (ippsSHA1GetSize) +EXTERN (ippsSHA1Init) +EXTERN (ippsSHA1Duplicate) +EXTERN (ippsSHA1Pack) +EXTERN (ippsSHA1Unpack) +EXTERN (ippsSHA1Update) +EXTERN (ippsSHA1GetTag) +EXTERN (ippsSHA1Final) +EXTERN (ippsSHA1MessageDigest) +EXTERN (ippsSHA224GetSize) +EXTERN (ippsSHA224Init) +EXTERN (ippsSHA224Duplicate) +EXTERN (ippsSHA224Pack) +EXTERN (ippsSHA224Unpack) +EXTERN (ippsSHA224Update) +EXTERN (ippsSHA224GetTag) +EXTERN (ippsSHA224Final) +EXTERN (ippsSHA224MessageDigest) +EXTERN (ippsSHA256GetSize) +EXTERN (ippsSHA256Init) +EXTERN (ippsSHA256Duplicate) +EXTERN (ippsSHA256Pack) +EXTERN (ippsSHA256Unpack) +EXTERN (ippsSHA256Update) +EXTERN (ippsSHA256GetTag) +EXTERN (ippsSHA256Final) +EXTERN (ippsSHA256MessageDigest) +EXTERN (ippsSHA384GetSize) +EXTERN (ippsSHA384Init) +EXTERN (ippsSHA384Duplicate) +EXTERN (ippsSHA384Pack) +EXTERN (ippsSHA384Unpack) +EXTERN (ippsSHA384Update) +EXTERN (ippsSHA384GetTag) +EXTERN (ippsSHA384Final) +EXTERN (ippsSHA384MessageDigest) +EXTERN (ippsSHA512GetSize) +EXTERN (ippsSHA512Init) +EXTERN (ippsSHA512Duplicate) +EXTERN (ippsSHA512Pack) +EXTERN (ippsSHA512Unpack) +EXTERN (ippsSHA512Update) +EXTERN (ippsSHA512GetTag) +EXTERN (ippsSHA512Final) +EXTERN (ippsSHA512MessageDigest) +EXTERN (ippsMD5GetSize) +EXTERN (ippsMD5Init) +EXTERN (ippsMD5Duplicate) +EXTERN (ippsMD5Pack) +EXTERN (ippsMD5Unpack) +EXTERN (ippsMD5Update) +EXTERN (ippsMD5GetTag) +EXTERN (ippsMD5Final) +EXTERN (ippsMD5MessageDigest) +EXTERN (ippsSM3GetSize) +EXTERN (ippsSM3Init) +EXTERN (ippsSM3Duplicate) +EXTERN (ippsSM3Pack) +EXTERN (ippsSM3Unpack) +EXTERN (ippsSM3Update) +EXTERN (ippsSM3GetTag) +EXTERN (ippsSM3Final) +EXTERN (ippsSM3MessageDigest) +EXTERN (ippsHashGetSize) +EXTERN (ippsHashInit) +EXTERN (ippsHashPack) +EXTERN (ippsHashUnpack) +EXTERN (ippsHashDuplicate) +EXTERN (ippsHashUpdate) +EXTERN (ippsHashGetTag) +EXTERN (ippsHashFinal) +EXTERN (ippsHashMessage) +EXTERN (ippsHashMethodGetSize) +EXTERN (ippsHashMethodSet_MD5) +EXTERN (ippsHashMethodSet_SM3) +EXTERN (ippsHashMethodSet_SHA1) +EXTERN (ippsHashMethodSet_SHA1_NI) +EXTERN (ippsHashMethodSet_SHA1_TT) +EXTERN (ippsHashMethodSet_SHA256) +EXTERN (ippsHashMethodSet_SHA256_NI) +EXTERN (ippsHashMethodSet_SHA256_TT) +EXTERN (ippsHashMethodSet_SHA224) +EXTERN (ippsHashMethodSet_SHA224_NI) +EXTERN (ippsHashMethodSet_SHA224_TT) +EXTERN (ippsHashMethodSet_SHA512) +EXTERN (ippsHashMethodSet_SHA384) +EXTERN (ippsHashMethodSet_SHA512_256) +EXTERN (ippsHashMethodSet_SHA512_224) +EXTERN (ippsHashStateMethodSet_SM3) +EXTERN (ippsHashStateMethodSet_SHA256) +EXTERN (ippsHashStateMethodSet_SHA256_NI) +EXTERN (ippsHashStateMethodSet_SHA256_TT) +EXTERN (ippsHashStateMethodSet_SHA224) +EXTERN (ippsHashStateMethodSet_SHA224_NI) +EXTERN (ippsHashStateMethodSet_SHA224_TT) +EXTERN (ippsHashStateMethodSet_SHA512) +EXTERN (ippsHashStateMethodSet_SHA384) +EXTERN (ippsHashStateMethodSet_SHA512_256) +EXTERN (ippsHashStateMethodSet_SHA512_224) +EXTERN (ippsHashMethod_MD5) +EXTERN (ippsHashMethod_SM3) +EXTERN (ippsHashMethod_SHA1) +EXTERN (ippsHashMethod_SHA1_NI) +EXTERN (ippsHashMethod_SHA1_TT) +EXTERN (ippsHashMethod_SHA256) +EXTERN (ippsHashMethod_SHA256_NI) +EXTERN (ippsHashMethod_SHA256_TT) +EXTERN (ippsHashMethod_SHA224) +EXTERN (ippsHashMethod_SHA224_NI) +EXTERN (ippsHashMethod_SHA224_TT) +EXTERN (ippsHashMethod_SHA512) +EXTERN (ippsHashMethod_SHA384) +EXTERN (ippsHashMethod_SHA512_256) +EXTERN (ippsHashMethod_SHA512_224) +EXTERN (ippsHashMethodGetInfo) +EXTERN (ippsHashGetSize_rmf) +EXTERN (ippsHashInit_rmf) +EXTERN (ippsHashPack_rmf) +EXTERN (ippsHashUnpack_rmf) +EXTERN (ippsHashDuplicate_rmf) +EXTERN (ippsHashUpdate_rmf) +EXTERN (ippsHashGetTag_rmf) +EXTERN (ippsHashFinal_rmf) +EXTERN (ippsHashMessage_rmf) +EXTERN (ippsHashGetInfo_rmf) +EXTERN (ippsMGF) +EXTERN (ippsMGF1_rmf) +EXTERN (ippsMGF2_rmf) +EXTERN (ippsHMAC_GetSize) +EXTERN (ippsHMAC_Init) +EXTERN (ippsHMAC_Pack) +EXTERN (ippsHMAC_Unpack) +EXTERN (ippsHMAC_Duplicate) +EXTERN (ippsHMAC_Update) +EXTERN (ippsHMAC_Final) +EXTERN (ippsHMAC_GetTag) +EXTERN (ippsHMAC_Message) +EXTERN (ippsHMACGetSize_rmf) +EXTERN (ippsHMACInit_rmf) +EXTERN (ippsHMACPack_rmf) +EXTERN (ippsHMACUnpack_rmf) +EXTERN (ippsHMACDuplicate_rmf) +EXTERN (ippsHMACUpdate_rmf) +EXTERN (ippsHMACFinal_rmf) +EXTERN (ippsHMACGetTag_rmf) +EXTERN (ippsHMACMessage_rmf) +EXTERN (ippsBigNumGetSize) +EXTERN (ippsBigNumInit) +EXTERN (ippsCmpZero_BN) +EXTERN (ippsCmp_BN) +EXTERN (ippsGetSize_BN) +EXTERN (ippsSet_BN) +EXTERN (ippsGet_BN) +EXTERN (ippsRef_BN) +EXTERN (ippsExtGet_BN) +EXTERN (ippsAdd_BN) +EXTERN (ippsSub_BN) +EXTERN (ippsMul_BN) +EXTERN (ippsMAC_BN_I) +EXTERN (ippsDiv_BN) +EXTERN (ippsMod_BN) +EXTERN (ippsGcd_BN) +EXTERN (ippsModInv_BN) +EXTERN (ippsSetOctString_BN) +EXTERN (ippsGetOctString_BN) +EXTERN (ippsMontGetSize) +EXTERN (ippsMontInit) +EXTERN (ippsMontSet) +EXTERN (ippsMontGet) +EXTERN (ippsMontForm) +EXTERN (ippsMontMul) +EXTERN (ippsMontExp) +EXTERN (ippsPRNGGetSize) +EXTERN (ippsPRNGInit) +EXTERN (ippsPRNGSetModulus) +EXTERN (ippsPRNGSetH0) +EXTERN (ippsPRNGSetAugment) +EXTERN (ippsPRNGSetSeed) +EXTERN (ippsPRNGGetSeed) +EXTERN (ippsPRNGen) +EXTERN (ippsPRNGen_BN) +EXTERN (ippsPRNGenRDRAND) +EXTERN (ippsPRNGenRDRAND_BN) +EXTERN (ippsTRNGenRDSEED) +EXTERN (ippsTRNGenRDSEED_BN) +EXTERN (ippsPrimeGetSize) +EXTERN (ippsPrimeInit) +EXTERN (ippsPrimeGen) +EXTERN (ippsPrimeTest) +EXTERN (ippsPrimeGen_BN) +EXTERN (ippsPrimeTest_BN) +EXTERN (ippsPrimeGet) +EXTERN (ippsPrimeGet_BN) +EXTERN (ippsPrimeSet) +EXTERN (ippsPrimeSet_BN) +EXTERN (ippsRSA_GetSizePublicKey) +EXTERN (ippsRSA_InitPublicKey) +EXTERN (ippsRSA_SetPublicKey) +EXTERN (ippsRSA_GetPublicKey) +EXTERN (ippsRSA_GetSizePrivateKeyType1) +EXTERN (ippsRSA_InitPrivateKeyType1) +EXTERN (ippsRSA_SetPrivateKeyType1) +EXTERN (ippsRSA_GetPrivateKeyType1) +EXTERN (ippsRSA_GetSizePrivateKeyType2) +EXTERN (ippsRSA_InitPrivateKeyType2) +EXTERN (ippsRSA_SetPrivateKeyType2) +EXTERN (ippsRSA_GetPrivateKeyType2) +EXTERN (ippsRSA_GetBufferSizePublicKey) +EXTERN (ippsRSA_GetBufferSizePrivateKey) +EXTERN (ippsRSA_Encrypt) +EXTERN (ippsRSA_Decrypt) +EXTERN (ippsRSA_GenerateKeys) +EXTERN (ippsRSA_ValidateKeys) +EXTERN (ippsRSAEncrypt_OAEP) +EXTERN (ippsRSADecrypt_OAEP) +EXTERN (ippsRSAEncrypt_OAEP_rmf) +EXTERN (ippsRSADecrypt_OAEP_rmf) +EXTERN (ippsRSAEncrypt_PKCSv15) +EXTERN (ippsRSADecrypt_PKCSv15) +EXTERN (ippsRSASign_PSS) +EXTERN (ippsRSAVerify_PSS) +EXTERN (ippsRSASign_PSS_rmf) +EXTERN (ippsRSAVerify_PSS_rmf) +EXTERN (ippsRSASign_PKCS1v15) +EXTERN (ippsRSAVerify_PKCS1v15) +EXTERN (ippsRSASign_PKCS1v15_rmf) +EXTERN (ippsRSAVerify_PKCS1v15_rmf) +EXTERN (ippsDLGetResultString) +EXTERN (ippsDLPGetSize) +EXTERN (ippsDLPInit) +EXTERN (ippsDLPPack) +EXTERN (ippsDLPUnpack) +EXTERN (ippsDLPSet) +EXTERN (ippsDLPGet) +EXTERN (ippsDLPSetDP) +EXTERN (ippsDLPGetDP) +EXTERN (ippsDLPGenKeyPair) +EXTERN (ippsDLPPublicKey) +EXTERN (ippsDLPValidateKeyPair) +EXTERN (ippsDLPSetKeyPair) +EXTERN (ippsDLPSignDSA) +EXTERN (ippsDLPVerifyDSA) +EXTERN (ippsDLPSharedSecretDH) +EXTERN (ippsDLPGenerateDSA) +EXTERN (ippsDLPValidateDSA) +EXTERN (ippsDLPGenerateDH) +EXTERN (ippsDLPValidateDH) +EXTERN (ippsECCGetResultString) +EXTERN (ippsECCPGetSize) +EXTERN (ippsECCPGetSizeStd128r1) +EXTERN (ippsECCPGetSizeStd128r2) +EXTERN (ippsECCPGetSizeStd192r1) +EXTERN (ippsECCPGetSizeStd224r1) +EXTERN (ippsECCPGetSizeStd256r1) +EXTERN (ippsECCPGetSizeStd384r1) +EXTERN (ippsECCPGetSizeStd521r1) +EXTERN (ippsECCPGetSizeStdSM2) +EXTERN (ippsECCPInit) +EXTERN (ippsECCPInitStd128r1) +EXTERN (ippsECCPInitStd128r2) +EXTERN (ippsECCPInitStd192r1) +EXTERN (ippsECCPInitStd224r1) +EXTERN (ippsECCPInitStd256r1) +EXTERN (ippsECCPInitStd384r1) +EXTERN (ippsECCPInitStd521r1) +EXTERN (ippsECCPInitStdSM2) +EXTERN (ippsECCPSet) +EXTERN (ippsECCPSetStd) +EXTERN (ippsECCPSetStd128r1) +EXTERN (ippsECCPSetStd128r2) +EXTERN (ippsECCPSetStd192r1) +EXTERN (ippsECCPSetStd224r1) +EXTERN (ippsECCPSetStd256r1) +EXTERN (ippsECCPSetStd384r1) +EXTERN (ippsECCPSetStd521r1) +EXTERN (ippsECCPSetStdSM2) +EXTERN (ippsECCPBindGxyTblStd192r1) +EXTERN (ippsECCPBindGxyTblStd224r1) +EXTERN (ippsECCPBindGxyTblStd256r1) +EXTERN (ippsECCPBindGxyTblStd384r1) +EXTERN (ippsECCPBindGxyTblStd521r1) +EXTERN (ippsECCPBindGxyTblStdSM2) +EXTERN (ippsECCPGet) +EXTERN (ippsECCPGetOrderBitSize) +EXTERN (ippsECCPValidate) +EXTERN (ippsECCPPointGetSize) +EXTERN (ippsECCPPointInit) +EXTERN (ippsECCPSetPoint) +EXTERN (ippsECCPSetPointAtInfinity) +EXTERN (ippsECCPGetPoint) +EXTERN (ippsECCPCheckPoint) +EXTERN (ippsECCPComparePoint) +EXTERN (ippsECCPNegativePoint) +EXTERN (ippsECCPAddPoint) +EXTERN (ippsECCPMulPointScalar) +EXTERN (ippsECCPGenKeyPair) +EXTERN (ippsECCPPublicKey) +EXTERN (ippsECCPValidateKeyPair) +EXTERN (ippsECCPSetKeyPair) +EXTERN (ippsECCPSharedSecretDH) +EXTERN (ippsECCPSharedSecretDHC) +EXTERN (ippsECCPSignDSA) +EXTERN (ippsECCPVerifyDSA) +EXTERN (ippsECCPSignNR) +EXTERN (ippsECCPVerifyNR) +EXTERN (ippsECCPSignSM2) +EXTERN (ippsECCPVerifySM2) +EXTERN (ippsGFpGetSize) +EXTERN (ippsGFpInitArbitrary) +EXTERN (ippsGFpInitFixed) +EXTERN (ippsGFpInit) +EXTERN (ippsGFpMethod_p192r1) +EXTERN (ippsGFpMethod_p224r1) +EXTERN (ippsGFpMethod_p256r1) +EXTERN (ippsGFpMethod_p384r1) +EXTERN (ippsGFpMethod_p521r1) +EXTERN (ippsGFpMethod_p256sm2) +EXTERN (ippsGFpMethod_p256bn) +EXTERN (ippsGFpMethod_p256) +EXTERN (ippsGFpMethod_pArb) +EXTERN (ippsGFpxGetSize) +EXTERN (ippsGFpxInit) +EXTERN (ippsGFpxInitBinomial) +EXTERN (ippsGFpxMethod_binom2_epid2) +EXTERN (ippsGFpxMethod_binom3_epid2) +EXTERN (ippsGFpxMethod_binom2) +EXTERN (ippsGFpxMethod_binom3) +EXTERN (ippsGFpxMethod_binom) +EXTERN (ippsGFpxMethod_com) +EXTERN (ippsGFpScratchBufferSize) +EXTERN (ippsGFpElementGetSize) +EXTERN (ippsGFpElementInit) +EXTERN (ippsGFpSetElement) +EXTERN (ippsGFpSetElementRegular) +EXTERN (ippsGFpSetElementOctString) +EXTERN (ippsGFpSetElementRandom) +EXTERN (ippsGFpSetElementHash) +EXTERN (ippsGFpSetElementHash_rmf) +EXTERN (ippsGFpCpyElement) +EXTERN (ippsGFpGetElement) +EXTERN (ippsGFpGetElementOctString) +EXTERN (ippsGFpCmpElement) +EXTERN (ippsGFpIsZeroElement) +EXTERN (ippsGFpIsUnityElement) +EXTERN (ippsGFpConj) +EXTERN (ippsGFpNeg) +EXTERN (ippsGFpInv) +EXTERN (ippsGFpSqrt) +EXTERN (ippsGFpSqr) +EXTERN (ippsGFpAdd) +EXTERN (ippsGFpSub) +EXTERN (ippsGFpMul) +EXTERN (ippsGFpExp) +EXTERN (ippsGFpMultiExp) +EXTERN (ippsGFpAdd_PE) +EXTERN (ippsGFpSub_PE) +EXTERN (ippsGFpMul_PE) +EXTERN (ippsGFpGetInfo) +EXTERN (ippsGFpECGetSize) +EXTERN (ippsGFpECInit) +EXTERN (ippsGFpECSet) +EXTERN (ippsGFpECSetSubgroup) +EXTERN (ippsGFpECInitStd128r1) +EXTERN (ippsGFpECInitStd128r2) +EXTERN (ippsGFpECInitStd192r1) +EXTERN (ippsGFpECInitStd224r1) +EXTERN (ippsGFpECInitStd256r1) +EXTERN (ippsGFpECInitStd384r1) +EXTERN (ippsGFpECInitStd521r1) +EXTERN (ippsGFpECInitStdSM2) +EXTERN (ippsGFpECInitStdBN256) +EXTERN (ippsGFpECBindGxyTblStd192r1) +EXTERN (ippsGFpECBindGxyTblStd224r1) +EXTERN (ippsGFpECBindGxyTblStd256r1) +EXTERN (ippsGFpECBindGxyTblStd384r1) +EXTERN (ippsGFpECBindGxyTblStd521r1) +EXTERN (ippsGFpECBindGxyTblStdSM2) +EXTERN (ippsGFpECGet) +EXTERN (ippsGFpECGetInfo_GF) +EXTERN (ippsGFpECGetSubgroup) +EXTERN (ippsGFpECScratchBufferSize) +EXTERN (ippsGFpECVerify) +EXTERN (ippsGFpECPointGetSize) +EXTERN (ippsGFpECPointInit) +EXTERN (ippsGFpECSetPointAtInfinity) +EXTERN (ippsGFpECSetPoint) +EXTERN (ippsGFpECSetPointRegular) +EXTERN (ippsGFpECSetPointRandom) +EXTERN (ippsGFpECMakePoint) +EXTERN (ippsGFpECSetPointHash) +EXTERN (ippsGFpECSetPointHash_rmf) +EXTERN (ippsGFpECSetPointHashBackCompatible) +EXTERN (ippsGFpECSetPointHashBackCompatible_rmf) +EXTERN (ippsGFpECGetPoint) +EXTERN (ippsGFpECGetPointRegular) +EXTERN (ippsGFpECTstPoint) +EXTERN (ippsGFpECTstPointInSubgroup) +EXTERN (ippsGFpECCpyPoint) +EXTERN (ippsGFpECCmpPoint) +EXTERN (ippsGFpECNegPoint) +EXTERN (ippsGFpECAddPoint) +EXTERN (ippsGFpECMulPoint) +EXTERN (ippsGFpECPrivateKey) +EXTERN (ippsGFpECPublicKey) +EXTERN (ippsGFpECTstKeyPair) +EXTERN (ippsGFpECSharedSecretDH) +EXTERN (ippsGFpECSharedSecretDHC) +EXTERN (ippsGFpECSignDSA) +EXTERN (ippsGFpECVerifyDSA) +EXTERN (ippsGFpECSignNR) +EXTERN (ippsGFpECVerifyNR) +EXTERN (ippsGFpECSignSM2) +EXTERN (ippsGFpECVerifySM2) +EXTERN (ippsGFpECUserIDHashSM2) +EXTERN (ippsGFpECMessageRepresentationSM2) +EXTERN (ippsGFpECKeyExchangeSM2_GetSize) +EXTERN (ippsGFpECKeyExchangeSM2_Init) +EXTERN (ippsGFpECKeyExchangeSM2_Setup) +EXTERN (ippsGFpECKeyExchangeSM2_SharedKey) +EXTERN (ippsGFpECKeyExchangeSM2_Confirm) +EXTERN (ippsGFpECEncryptSM2_Ext_EncMsgSize) +EXTERN (ippsGFpECEncryptSM2_Ext) +EXTERN (ippsGFpECDecryptSM2_Ext_DecMsgSize) +EXTERN (ippsGFpECDecryptSM2_Ext) +EXTERN (ippsGFpECESGetSize_SM2) +EXTERN (ippsGFpECESInit_SM2) +EXTERN (ippsGFpECESSetKey_SM2) +EXTERN (ippsGFpECESStart_SM2) +EXTERN (ippsGFpECESEncrypt_SM2) +EXTERN (ippsGFpECESDecrypt_SM2) +EXTERN (ippsGFpECESFinal_SM2) +EXTERN (ippsGFpECESGetBuffersSize_SM2) +EXTERN (ippsGFpECSetPointOctString) +EXTERN (ippsGFpECGetPointOctString) + +VERSION { + { + global: + ippcpInit; + cpGetReg; + cpStartTscp; + cpStopTscp; + cpStartTsc; + cpStopTsc; + cpGetCacheSize; + cpGetFeature; + ippcpSetCpuFeatures; + ippcpGetCpuFeatures; + ippcpGetCpuClocks; + ippcpSetNumThreads; + ippcpGetNumThreads; + ippcpGetEnabledCpuFeatures; + ippcpGetEnabledNumThreads; + ippcpGetStatusString; + + ippcpGetLibVersion; + ippsDESGetSize; + ippsDESInit; + ippsDESPack; + ippsDESUnpack; + ippsTDESEncryptECB; + ippsTDESDecryptECB; + ippsTDESEncryptCBC; + ippsTDESDecryptCBC; + ippsTDESEncryptCFB; + ippsTDESDecryptCFB; + ippsTDESEncryptOFB; + ippsTDESDecryptOFB; + ippsTDESEncryptCTR; + ippsTDESDecryptCTR; + ippsAESGetSize; + ippsAESInit; + ippsAESSetKey; + ippsAESPack; + ippsAESUnpack; + ippsAESSetupNoise; + ippsAES_GCMSetupNoise; + ippsAES_CMACSetupNoise; + ippsAESEncryptECB; + ippsAESDecryptECB; + ippsAESEncryptCBC; + ippsAESEncryptCBC_CS1; + ippsAESEncryptCBC_CS2; + ippsAESEncryptCBC_CS3; + ippsAESDecryptCBC; + ippsAESDecryptCBC_CS1; + ippsAESDecryptCBC_CS2; + ippsAESDecryptCBC_CS3; + ippsAESEncryptCFB; + ippsAESDecryptCFB; + ippsAESEncryptOFB; + ippsAESDecryptOFB; + ippsAESEncryptCTR; + ippsAESDecryptCTR; + ippsAESEncryptXTS_Direct; + ippsAESDecryptXTS_Direct; + ippsAES_EncryptCFB16_MB; + ippsSMS4GetSize; + ippsSMS4Init; + ippsSMS4SetKey; + ippsSMS4EncryptECB; + ippsSMS4DecryptECB; + ippsSMS4EncryptCBC; + ippsSMS4EncryptCBC_CS1; + ippsSMS4EncryptCBC_CS2; + ippsSMS4EncryptCBC_CS3; + ippsSMS4DecryptCBC; + ippsSMS4DecryptCBC_CS1; + ippsSMS4DecryptCBC_CS2; + ippsSMS4DecryptCBC_CS3; + ippsSMS4EncryptCFB; + ippsSMS4DecryptCFB; + ippsSMS4EncryptOFB; + ippsSMS4DecryptOFB; + ippsSMS4EncryptCTR; + ippsSMS4DecryptCTR; + ippsSMS4_CCMGetSize; + ippsSMS4_CCMInit; + ippsSMS4_CCMMessageLen; + ippsSMS4_CCMTagLen; + ippsSMS4_CCMStart; + ippsSMS4_CCMEncrypt; + ippsSMS4_CCMDecrypt; + ippsSMS4_CCMGetTag; + ippsAES_CCMGetSize; + ippsAES_CCMInit; + ippsAES_CCMMessageLen; + ippsAES_CCMTagLen; + ippsAES_CCMStart; + ippsAES_CCMEncrypt; + ippsAES_CCMDecrypt; + ippsAES_CCMGetTag; + ippsAES_GCMGetSize; + ippsAES_GCMInit; + ippsAES_GCMReinit; + ippsAES_GCMReset; + ippsAES_GCMProcessIV; + ippsAES_GCMProcessAAD; + ippsAES_GCMStart; + ippsAES_GCMEncrypt; + ippsAES_GCMDecrypt; + ippsAES_GCMGetTag; + ippsAES_XTSGetSize; + ippsAES_XTSInit; + ippsAES_XTSEncrypt; + ippsAES_XTSDecrypt; + ippsAES_S2V_CMAC; + ippsAES_SIVEncrypt; + ippsAES_SIVDecrypt; + ippsAES_CMACGetSize; + ippsAES_CMACInit; + ippsAES_CMACUpdate; + ippsAES_CMACFinal; + ippsAES_CMACGetTag; + ippsARCFourCheckKey; + ippsARCFourGetSize; + ippsARCFourInit; + ippsARCFourReset; + ippsARCFourPack; + ippsARCFourUnpack; + ippsARCFourEncrypt; + ippsARCFourDecrypt; + ippsSHA1GetSize; + ippsSHA1Init; + ippsSHA1Duplicate; + ippsSHA1Pack; + ippsSHA1Unpack; + ippsSHA1Update; + ippsSHA1GetTag; + ippsSHA1Final; + ippsSHA1MessageDigest; + ippsSHA224GetSize; + ippsSHA224Init; + ippsSHA224Duplicate; + ippsSHA224Pack; + ippsSHA224Unpack; + ippsSHA224Update; + ippsSHA224GetTag; + ippsSHA224Final; + ippsSHA224MessageDigest; + ippsSHA256GetSize; + ippsSHA256Init; + ippsSHA256Duplicate; + ippsSHA256Pack; + ippsSHA256Unpack; + ippsSHA256Update; + ippsSHA256GetTag; + ippsSHA256Final; + ippsSHA256MessageDigest; + ippsSHA384GetSize; + ippsSHA384Init; + ippsSHA384Duplicate; + ippsSHA384Pack; + ippsSHA384Unpack; + ippsSHA384Update; + ippsSHA384GetTag; + ippsSHA384Final; + ippsSHA384MessageDigest; + ippsSHA512GetSize; + ippsSHA512Init; + ippsSHA512Duplicate; + ippsSHA512Pack; + ippsSHA512Unpack; + ippsSHA512Update; + ippsSHA512GetTag; + ippsSHA512Final; + ippsSHA512MessageDigest; + ippsMD5GetSize; + ippsMD5Init; + ippsMD5Duplicate; + ippsMD5Pack; + ippsMD5Unpack; + ippsMD5Update; + ippsMD5GetTag; + ippsMD5Final; + ippsMD5MessageDigest; + ippsSM3GetSize; + ippsSM3Init; + ippsSM3Duplicate; + ippsSM3Pack; + ippsSM3Unpack; + ippsSM3Update; + ippsSM3GetTag; + ippsSM3Final; + ippsSM3MessageDigest; + ippsHashGetSize; + ippsHashInit; + ippsHashPack; + ippsHashUnpack; + ippsHashDuplicate; + ippsHashUpdate; + ippsHashGetTag; + ippsHashFinal; + ippsHashMessage; + ippsHashMethodGetSize; + ippsHashMethodSet_MD5; + ippsHashMethodSet_SM3; + ippsHashMethodSet_SHA1; + ippsHashMethodSet_SHA1_NI; + ippsHashMethodSet_SHA1_TT; + ippsHashMethodSet_SHA256; + ippsHashMethodSet_SHA256_NI; + ippsHashMethodSet_SHA256_TT; + ippsHashMethodSet_SHA224; + ippsHashMethodSet_SHA224_NI; + ippsHashMethodSet_SHA224_TT; + ippsHashMethodSet_SHA512; + ippsHashMethodSet_SHA384; + ippsHashMethodSet_SHA512_256; + ippsHashMethodSet_SHA512_224; + ippsHashStateMethodSet_SM3; + ippsHashStateMethodSet_SHA256; + ippsHashStateMethodSet_SHA256_NI; + ippsHashStateMethodSet_SHA256_TT; + ippsHashStateMethodSet_SHA224; + ippsHashStateMethodSet_SHA224_NI; + ippsHashStateMethodSet_SHA224_TT; + ippsHashStateMethodSet_SHA512; + ippsHashStateMethodSet_SHA384; + ippsHashStateMethodSet_SHA512_256; + ippsHashStateMethodSet_SHA512_224; + ippsHashMethod_MD5; + ippsHashMethod_SM3; + ippsHashMethod_SHA1; + ippsHashMethod_SHA1_NI; + ippsHashMethod_SHA1_TT; + ippsHashMethod_SHA256; + ippsHashMethod_SHA256_NI; + ippsHashMethod_SHA256_TT; + ippsHashMethod_SHA224; + ippsHashMethod_SHA224_NI; + ippsHashMethod_SHA224_TT; + ippsHashMethod_SHA512; + ippsHashMethod_SHA384; + ippsHashMethod_SHA512_256; + ippsHashMethod_SHA512_224; + ippsHashMethodGetInfo; + ippsHashGetSize_rmf; + ippsHashInit_rmf; + ippsHashPack_rmf; + ippsHashUnpack_rmf; + ippsHashDuplicate_rmf; + ippsHashUpdate_rmf; + ippsHashGetTag_rmf; + ippsHashFinal_rmf; + ippsHashMessage_rmf; + ippsHashGetInfo_rmf; + ippsMGF; + ippsMGF1_rmf; + ippsMGF2_rmf; + ippsHMAC_GetSize; + ippsHMAC_Init; + ippsHMAC_Pack; + ippsHMAC_Unpack; + ippsHMAC_Duplicate; + ippsHMAC_Update; + ippsHMAC_Final; + ippsHMAC_GetTag; + ippsHMAC_Message; + ippsHMACGetSize_rmf; + ippsHMACInit_rmf; + ippsHMACPack_rmf; + ippsHMACUnpack_rmf; + ippsHMACDuplicate_rmf; + ippsHMACUpdate_rmf; + ippsHMACFinal_rmf; + ippsHMACGetTag_rmf; + ippsHMACMessage_rmf; + ippsBigNumGetSize; + ippsBigNumInit; + ippsCmpZero_BN; + ippsCmp_BN; + ippsGetSize_BN; + ippsSet_BN; + ippsGet_BN; + ippsRef_BN; + ippsExtGet_BN; + ippsAdd_BN; + ippsSub_BN; + ippsMul_BN; + ippsMAC_BN_I; + ippsDiv_BN; + ippsMod_BN; + ippsGcd_BN; + ippsModInv_BN; + ippsSetOctString_BN; + ippsGetOctString_BN; + ippsMontGetSize; + ippsMontInit; + ippsMontSet; + ippsMontGet; + ippsMontForm; + ippsMontMul; + ippsMontExp; + ippsPRNGGetSize; + ippsPRNGInit; + ippsPRNGSetModulus; + ippsPRNGSetH0; + ippsPRNGSetAugment; + ippsPRNGSetSeed; + ippsPRNGGetSeed; + ippsPRNGen; + ippsPRNGen_BN; + ippsPRNGenRDRAND; + ippsPRNGenRDRAND_BN; + ippsTRNGenRDSEED; + ippsTRNGenRDSEED_BN; + ippsPrimeGetSize; + ippsPrimeInit; + ippsPrimeGen; + ippsPrimeTest; + ippsPrimeGen_BN; + ippsPrimeTest_BN; + ippsPrimeGet; + ippsPrimeGet_BN; + ippsPrimeSet; + ippsPrimeSet_BN; + ippsRSA_GetSizePublicKey; + ippsRSA_InitPublicKey; + ippsRSA_SetPublicKey; + ippsRSA_GetPublicKey; + ippsRSA_GetSizePrivateKeyType1; + ippsRSA_InitPrivateKeyType1; + ippsRSA_SetPrivateKeyType1; + ippsRSA_GetPrivateKeyType1; + ippsRSA_GetSizePrivateKeyType2; + ippsRSA_InitPrivateKeyType2; + ippsRSA_SetPrivateKeyType2; + ippsRSA_GetPrivateKeyType2; + ippsRSA_GetBufferSizePublicKey; + ippsRSA_GetBufferSizePrivateKey; + ippsRSA_Encrypt; + ippsRSA_Decrypt; + ippsRSA_GenerateKeys; + ippsRSA_ValidateKeys; + ippsRSAEncrypt_OAEP; + ippsRSADecrypt_OAEP; + ippsRSAEncrypt_OAEP_rmf; + ippsRSADecrypt_OAEP_rmf; + ippsRSAEncrypt_PKCSv15; + ippsRSADecrypt_PKCSv15; + ippsRSASign_PSS; + ippsRSAVerify_PSS; + ippsRSASign_PSS_rmf; + ippsRSAVerify_PSS_rmf; + ippsRSASign_PKCS1v15; + ippsRSAVerify_PKCS1v15; + ippsRSASign_PKCS1v15_rmf; + ippsRSAVerify_PKCS1v15_rmf; + ippsDLGetResultString; + ippsDLPGetSize; + ippsDLPInit; + ippsDLPPack; + ippsDLPUnpack; + ippsDLPSet; + ippsDLPGet; + ippsDLPSetDP; + ippsDLPGetDP; + ippsDLPGenKeyPair; + ippsDLPPublicKey; + ippsDLPValidateKeyPair; + ippsDLPSetKeyPair; + ippsDLPSignDSA; + ippsDLPVerifyDSA; + ippsDLPSharedSecretDH; + ippsDLPGenerateDSA; + ippsDLPValidateDSA; + ippsDLPGenerateDH; + ippsDLPValidateDH; + ippsECCGetResultString; + ippsECCPGetSize; + ippsECCPGetSizeStd128r1; + ippsECCPGetSizeStd128r2; + ippsECCPGetSizeStd192r1; + ippsECCPGetSizeStd224r1; + ippsECCPGetSizeStd256r1; + ippsECCPGetSizeStd384r1; + ippsECCPGetSizeStd521r1; + ippsECCPGetSizeStdSM2; + ippsECCPInit; + ippsECCPInitStd128r1; + ippsECCPInitStd128r2; + ippsECCPInitStd192r1; + ippsECCPInitStd224r1; + ippsECCPInitStd256r1; + ippsECCPInitStd384r1; + ippsECCPInitStd521r1; + ippsECCPInitStdSM2; + ippsECCPSet; + ippsECCPSetStd; + ippsECCPSetStd128r1; + ippsECCPSetStd128r2; + ippsECCPSetStd192r1; + ippsECCPSetStd224r1; + ippsECCPSetStd256r1; + ippsECCPSetStd384r1; + ippsECCPSetStd521r1; + ippsECCPSetStdSM2; + ippsECCPBindGxyTblStd192r1; + ippsECCPBindGxyTblStd224r1; + ippsECCPBindGxyTblStd256r1; + ippsECCPBindGxyTblStd384r1; + ippsECCPBindGxyTblStd521r1; + ippsECCPBindGxyTblStdSM2; + ippsECCPGet; + ippsECCPGetOrderBitSize; + ippsECCPValidate; + ippsECCPPointGetSize; + ippsECCPPointInit; + ippsECCPSetPoint; + ippsECCPSetPointAtInfinity; + ippsECCPGetPoint; + ippsECCPCheckPoint; + ippsECCPComparePoint; + ippsECCPNegativePoint; + ippsECCPAddPoint; + ippsECCPMulPointScalar; + ippsECCPGenKeyPair; + ippsECCPPublicKey; + ippsECCPValidateKeyPair; + ippsECCPSetKeyPair; + ippsECCPSharedSecretDH; + ippsECCPSharedSecretDHC; + ippsECCPSignDSA; + ippsECCPVerifyDSA; + ippsECCPSignNR; + ippsECCPVerifyNR; + ippsECCPSignSM2; + ippsECCPVerifySM2; + ippsGFpGetSize; + ippsGFpInitArbitrary; + ippsGFpInitFixed; + ippsGFpInit; + ippsGFpMethod_p192r1; + ippsGFpMethod_p224r1; + ippsGFpMethod_p256r1; + ippsGFpMethod_p384r1; + ippsGFpMethod_p521r1; + ippsGFpMethod_p256sm2; + ippsGFpMethod_p256bn; + ippsGFpMethod_p256; + ippsGFpMethod_pArb; + ippsGFpxGetSize; + ippsGFpxInit; + ippsGFpxInitBinomial; + ippsGFpxMethod_binom2_epid2; + ippsGFpxMethod_binom3_epid2; + ippsGFpxMethod_binom2; + ippsGFpxMethod_binom3; + ippsGFpxMethod_binom; + ippsGFpxMethod_com; + ippsGFpScratchBufferSize; + ippsGFpElementGetSize; + ippsGFpElementInit; + ippsGFpSetElement; + ippsGFpSetElementRegular; + ippsGFpSetElementOctString; + ippsGFpSetElementRandom; + ippsGFpSetElementHash; + ippsGFpSetElementHash_rmf; + ippsGFpCpyElement; + ippsGFpGetElement; + ippsGFpGetElementOctString; + ippsGFpCmpElement; + ippsGFpIsZeroElement; + ippsGFpIsUnityElement; + ippsGFpConj; + ippsGFpNeg; + ippsGFpInv; + ippsGFpSqrt; + ippsGFpSqr; + ippsGFpAdd; + ippsGFpSub; + ippsGFpMul; + ippsGFpExp; + ippsGFpMultiExp; + ippsGFpAdd_PE; + ippsGFpSub_PE; + ippsGFpMul_PE; + ippsGFpGetInfo; + ippsGFpECGetSize; + ippsGFpECInit; + ippsGFpECSet; + ippsGFpECSetSubgroup; + ippsGFpECInitStd128r1; + ippsGFpECInitStd128r2; + ippsGFpECInitStd192r1; + ippsGFpECInitStd224r1; + ippsGFpECInitStd256r1; + ippsGFpECInitStd384r1; + ippsGFpECInitStd521r1; + ippsGFpECInitStdSM2; + ippsGFpECInitStdBN256; + ippsGFpECBindGxyTblStd192r1; + ippsGFpECBindGxyTblStd224r1; + ippsGFpECBindGxyTblStd256r1; + ippsGFpECBindGxyTblStd384r1; + ippsGFpECBindGxyTblStd521r1; + ippsGFpECBindGxyTblStdSM2; + ippsGFpECGet; + ippsGFpECGetInfo_GF; + ippsGFpECGetSubgroup; + ippsGFpECScratchBufferSize; + ippsGFpECVerify; + ippsGFpECPointGetSize; + ippsGFpECPointInit; + ippsGFpECSetPointAtInfinity; + ippsGFpECSetPoint; + ippsGFpECSetPointRegular; + ippsGFpECSetPointRandom; + ippsGFpECMakePoint; + ippsGFpECSetPointHash; + ippsGFpECSetPointHash_rmf; + ippsGFpECSetPointHashBackCompatible; + ippsGFpECSetPointHashBackCompatible_rmf; + ippsGFpECGetPoint; + ippsGFpECGetPointRegular; + ippsGFpECTstPoint; + ippsGFpECTstPointInSubgroup; + ippsGFpECCpyPoint; + ippsGFpECCmpPoint; + ippsGFpECNegPoint; + ippsGFpECAddPoint; + ippsGFpECMulPoint; + ippsGFpECPrivateKey; + ippsGFpECPublicKey; + ippsGFpECTstKeyPair; + ippsGFpECSharedSecretDH; + ippsGFpECSharedSecretDHC; + ippsGFpECSignDSA; + ippsGFpECVerifyDSA; + ippsGFpECSignNR; + ippsGFpECVerifyNR; + ippsGFpECSignSM2; + ippsGFpECVerifySM2; + ippsGFpECUserIDHashSM2; + ippsGFpECMessageRepresentationSM2; + ippsGFpECKeyExchangeSM2_GetSize; + ippsGFpECKeyExchangeSM2_Init; + ippsGFpECKeyExchangeSM2_Setup; + ippsGFpECKeyExchangeSM2_SharedKey; + ippsGFpECKeyExchangeSM2_Confirm; + ippsGFpECEncryptSM2_Ext_EncMsgSize; + ippsGFpECEncryptSM2_Ext; + ippsGFpECDecryptSM2_Ext_DecMsgSize; + ippsGFpECDecryptSM2_Ext; + ippsGFpECESGetSize_SM2; + ippsGFpECESInit_SM2; + ippsGFpECESSetKey_SM2; + ippsGFpECESStart_SM2; + ippsGFpECESEncrypt_SM2; + ippsGFpECESDecrypt_SM2; + ippsGFpECESFinal_SM2; + ippsGFpECESGetBuffersSize_SM2; + ippsGFpECSetPointOctString; + ippsGFpECGetPointOctString; + local: *; + }; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/exports.macosx.lib-export b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/exports.macosx.lib-export new file mode 100644 index 0000000..6d4fbf3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/exports.macosx.lib-export @@ -0,0 +1,558 @@ +_ippcpInit +_cpGetReg +_cpStartTscp +_cpStopTscp +_cpStartTsc +_cpStopTsc +_cpGetCacheSize +_cpGetFeature +_ippcpSetCpuFeatures +_ippcpGetCpuFeatures +_ippcpGetCpuClocks +_ippcpSetNumThreads +_ippcpGetNumThreads +_ippcpGetEnabledCpuFeatures +_ippcpGetEnabledNumThreads +_ippcpGetStatusString + +_ippcpGetLibVersion +_ippsDESGetSize +_ippsDESInit +_ippsDESPack +_ippsDESUnpack +_ippsTDESEncryptECB +_ippsTDESDecryptECB +_ippsTDESEncryptCBC +_ippsTDESDecryptCBC +_ippsTDESEncryptCFB +_ippsTDESDecryptCFB +_ippsTDESEncryptOFB +_ippsTDESDecryptOFB +_ippsTDESEncryptCTR +_ippsTDESDecryptCTR +_ippsAESGetSize +_ippsAESInit +_ippsAESSetKey +_ippsAESPack +_ippsAESUnpack +_ippsAESSetupNoise +_ippsAES_GCMSetupNoise +_ippsAES_CMACSetupNoise +_ippsAESEncryptECB +_ippsAESDecryptECB +_ippsAESEncryptCBC +_ippsAESEncryptCBC_CS1 +_ippsAESEncryptCBC_CS2 +_ippsAESEncryptCBC_CS3 +_ippsAESDecryptCBC +_ippsAESDecryptCBC_CS1 +_ippsAESDecryptCBC_CS2 +_ippsAESDecryptCBC_CS3 +_ippsAESEncryptCFB +_ippsAESDecryptCFB +_ippsAESEncryptOFB +_ippsAESDecryptOFB +_ippsAESEncryptCTR +_ippsAESDecryptCTR +_ippsAESEncryptXTS_Direct +_ippsAESDecryptXTS_Direct +_ippsAES_EncryptCFB16_MB +_ippsSMS4GetSize +_ippsSMS4Init +_ippsSMS4SetKey +_ippsSMS4EncryptECB +_ippsSMS4DecryptECB +_ippsSMS4EncryptCBC +_ippsSMS4EncryptCBC_CS1 +_ippsSMS4EncryptCBC_CS2 +_ippsSMS4EncryptCBC_CS3 +_ippsSMS4DecryptCBC +_ippsSMS4DecryptCBC_CS1 +_ippsSMS4DecryptCBC_CS2 +_ippsSMS4DecryptCBC_CS3 +_ippsSMS4EncryptCFB +_ippsSMS4DecryptCFB +_ippsSMS4EncryptOFB +_ippsSMS4DecryptOFB +_ippsSMS4EncryptCTR +_ippsSMS4DecryptCTR +_ippsSMS4_CCMGetSize +_ippsSMS4_CCMInit +_ippsSMS4_CCMMessageLen +_ippsSMS4_CCMTagLen +_ippsSMS4_CCMStart +_ippsSMS4_CCMEncrypt +_ippsSMS4_CCMDecrypt +_ippsSMS4_CCMGetTag +_ippsAES_CCMGetSize +_ippsAES_CCMInit +_ippsAES_CCMMessageLen +_ippsAES_CCMTagLen +_ippsAES_CCMStart +_ippsAES_CCMEncrypt +_ippsAES_CCMDecrypt +_ippsAES_CCMGetTag +_ippsAES_GCMGetSize +_ippsAES_GCMInit +_ippsAES_GCMReinit +_ippsAES_GCMReset +_ippsAES_GCMProcessIV +_ippsAES_GCMProcessAAD +_ippsAES_GCMStart +_ippsAES_GCMEncrypt +_ippsAES_GCMDecrypt +_ippsAES_GCMGetTag +_ippsAES_XTSGetSize +_ippsAES_XTSInit +_ippsAES_XTSEncrypt +_ippsAES_XTSDecrypt +_ippsAES_S2V_CMAC +_ippsAES_SIVEncrypt +_ippsAES_SIVDecrypt +_ippsAES_CMACGetSize +_ippsAES_CMACInit +_ippsAES_CMACUpdate +_ippsAES_CMACFinal +_ippsAES_CMACGetTag +_ippsARCFourCheckKey +_ippsARCFourGetSize +_ippsARCFourInit +_ippsARCFourReset +_ippsARCFourPack +_ippsARCFourUnpack +_ippsARCFourEncrypt +_ippsARCFourDecrypt +_ippsSHA1GetSize +_ippsSHA1Init +_ippsSHA1Duplicate +_ippsSHA1Pack +_ippsSHA1Unpack +_ippsSHA1Update +_ippsSHA1GetTag +_ippsSHA1Final +_ippsSHA1MessageDigest +_ippsSHA224GetSize +_ippsSHA224Init +_ippsSHA224Duplicate +_ippsSHA224Pack +_ippsSHA224Unpack +_ippsSHA224Update +_ippsSHA224GetTag +_ippsSHA224Final +_ippsSHA224MessageDigest +_ippsSHA256GetSize +_ippsSHA256Init +_ippsSHA256Duplicate +_ippsSHA256Pack +_ippsSHA256Unpack +_ippsSHA256Update +_ippsSHA256GetTag +_ippsSHA256Final +_ippsSHA256MessageDigest +_ippsSHA384GetSize +_ippsSHA384Init +_ippsSHA384Duplicate +_ippsSHA384Pack +_ippsSHA384Unpack +_ippsSHA384Update +_ippsSHA384GetTag +_ippsSHA384Final +_ippsSHA384MessageDigest +_ippsSHA512GetSize +_ippsSHA512Init +_ippsSHA512Duplicate +_ippsSHA512Pack +_ippsSHA512Unpack +_ippsSHA512Update +_ippsSHA512GetTag +_ippsSHA512Final +_ippsSHA512MessageDigest +_ippsMD5GetSize +_ippsMD5Init +_ippsMD5Duplicate +_ippsMD5Pack +_ippsMD5Unpack +_ippsMD5Update +_ippsMD5GetTag +_ippsMD5Final +_ippsMD5MessageDigest +_ippsSM3GetSize +_ippsSM3Init +_ippsSM3Duplicate +_ippsSM3Pack +_ippsSM3Unpack +_ippsSM3Update +_ippsSM3GetTag +_ippsSM3Final +_ippsSM3MessageDigest +_ippsHashGetSize +_ippsHashInit +_ippsHashPack +_ippsHashUnpack +_ippsHashDuplicate +_ippsHashUpdate +_ippsHashGetTag +_ippsHashFinal +_ippsHashMessage +_ippsHashMethodGetSize +_ippsHashMethodSet_MD5 +_ippsHashMethodSet_SM3 +_ippsHashMethodSet_SHA1 +_ippsHashMethodSet_SHA1_NI +_ippsHashMethodSet_SHA1_TT +_ippsHashMethodSet_SHA256 +_ippsHashMethodSet_SHA256_NI +_ippsHashMethodSet_SHA256_TT +_ippsHashMethodSet_SHA224 +_ippsHashMethodSet_SHA224_NI +_ippsHashMethodSet_SHA224_TT +_ippsHashMethodSet_SHA512 +_ippsHashMethodSet_SHA384 +_ippsHashMethodSet_SHA512_256 +_ippsHashMethodSet_SHA512_224 +_ippsHashStateMethodSet_SM3 +_ippsHashStateMethodSet_SHA256 +_ippsHashStateMethodSet_SHA256_NI +_ippsHashStateMethodSet_SHA256_TT +_ippsHashStateMethodSet_SHA224 +_ippsHashStateMethodSet_SHA224_NI +_ippsHashStateMethodSet_SHA224_TT +_ippsHashStateMethodSet_SHA512 +_ippsHashStateMethodSet_SHA384 +_ippsHashStateMethodSet_SHA512_256 +_ippsHashStateMethodSet_SHA512_224 +_ippsHashMethod_MD5 +_ippsHashMethod_SM3 +_ippsHashMethod_SHA1 +_ippsHashMethod_SHA1_NI +_ippsHashMethod_SHA1_TT +_ippsHashMethod_SHA256 +_ippsHashMethod_SHA256_NI +_ippsHashMethod_SHA256_TT +_ippsHashMethod_SHA224 +_ippsHashMethod_SHA224_NI +_ippsHashMethod_SHA224_TT +_ippsHashMethod_SHA512 +_ippsHashMethod_SHA384 +_ippsHashMethod_SHA512_256 +_ippsHashMethod_SHA512_224 +_ippsHashMethodGetInfo +_ippsHashGetSize_rmf +_ippsHashInit_rmf +_ippsHashPack_rmf +_ippsHashUnpack_rmf +_ippsHashDuplicate_rmf +_ippsHashUpdate_rmf +_ippsHashGetTag_rmf +_ippsHashFinal_rmf +_ippsHashMessage_rmf +_ippsHashGetInfo_rmf +_ippsMGF +_ippsMGF1_rmf +_ippsMGF2_rmf +_ippsHMAC_GetSize +_ippsHMAC_Init +_ippsHMAC_Pack +_ippsHMAC_Unpack +_ippsHMAC_Duplicate +_ippsHMAC_Update +_ippsHMAC_Final +_ippsHMAC_GetTag +_ippsHMAC_Message +_ippsHMACGetSize_rmf +_ippsHMACInit_rmf +_ippsHMACPack_rmf +_ippsHMACUnpack_rmf +_ippsHMACDuplicate_rmf +_ippsHMACUpdate_rmf +_ippsHMACFinal_rmf +_ippsHMACGetTag_rmf +_ippsHMACMessage_rmf +_ippsBigNumGetSize +_ippsBigNumInit +_ippsCmpZero_BN +_ippsCmp_BN +_ippsGetSize_BN +_ippsSet_BN +_ippsGet_BN +_ippsRef_BN +_ippsExtGet_BN +_ippsAdd_BN +_ippsSub_BN +_ippsMul_BN +_ippsMAC_BN_I +_ippsDiv_BN +_ippsMod_BN +_ippsGcd_BN +_ippsModInv_BN +_ippsSetOctString_BN +_ippsGetOctString_BN +_ippsMontGetSize +_ippsMontInit +_ippsMontSet +_ippsMontGet +_ippsMontForm +_ippsMontMul +_ippsMontExp +_ippsPRNGGetSize +_ippsPRNGInit +_ippsPRNGSetModulus +_ippsPRNGSetH0 +_ippsPRNGSetAugment +_ippsPRNGSetSeed +_ippsPRNGGetSeed +_ippsPRNGen +_ippsPRNGen_BN +_ippsPRNGenRDRAND +_ippsPRNGenRDRAND_BN +_ippsTRNGenRDSEED +_ippsTRNGenRDSEED_BN +_ippsPrimeGetSize +_ippsPrimeInit +_ippsPrimeGen +_ippsPrimeTest +_ippsPrimeGen_BN +_ippsPrimeTest_BN +_ippsPrimeGet +_ippsPrimeGet_BN +_ippsPrimeSet +_ippsPrimeSet_BN +_ippsRSA_GetSizePublicKey +_ippsRSA_InitPublicKey +_ippsRSA_SetPublicKey +_ippsRSA_GetPublicKey +_ippsRSA_GetSizePrivateKeyType1 +_ippsRSA_InitPrivateKeyType1 +_ippsRSA_SetPrivateKeyType1 +_ippsRSA_GetPrivateKeyType1 +_ippsRSA_GetSizePrivateKeyType2 +_ippsRSA_InitPrivateKeyType2 +_ippsRSA_SetPrivateKeyType2 +_ippsRSA_GetPrivateKeyType2 +_ippsRSA_GetBufferSizePublicKey +_ippsRSA_GetBufferSizePrivateKey +_ippsRSA_Encrypt +_ippsRSA_Decrypt +_ippsRSA_GenerateKeys +_ippsRSA_ValidateKeys +_ippsRSAEncrypt_OAEP +_ippsRSADecrypt_OAEP +_ippsRSAEncrypt_OAEP_rmf +_ippsRSADecrypt_OAEP_rmf +_ippsRSAEncrypt_PKCSv15 +_ippsRSADecrypt_PKCSv15 +_ippsRSASign_PSS +_ippsRSAVerify_PSS +_ippsRSASign_PSS_rmf +_ippsRSAVerify_PSS_rmf +_ippsRSASign_PKCS1v15 +_ippsRSAVerify_PKCS1v15 +_ippsRSASign_PKCS1v15_rmf +_ippsRSAVerify_PKCS1v15_rmf +_ippsDLGetResultString +_ippsDLPGetSize +_ippsDLPInit +_ippsDLPPack +_ippsDLPUnpack +_ippsDLPSet +_ippsDLPGet +_ippsDLPSetDP +_ippsDLPGetDP +_ippsDLPGenKeyPair +_ippsDLPPublicKey +_ippsDLPValidateKeyPair +_ippsDLPSetKeyPair +_ippsDLPSignDSA +_ippsDLPVerifyDSA +_ippsDLPSharedSecretDH +_ippsDLPGenerateDSA +_ippsDLPValidateDSA +_ippsDLPGenerateDH +_ippsDLPValidateDH +_ippsECCGetResultString +_ippsECCPGetSize +_ippsECCPGetSizeStd128r1 +_ippsECCPGetSizeStd128r2 +_ippsECCPGetSizeStd192r1 +_ippsECCPGetSizeStd224r1 +_ippsECCPGetSizeStd256r1 +_ippsECCPGetSizeStd384r1 +_ippsECCPGetSizeStd521r1 +_ippsECCPGetSizeStdSM2 +_ippsECCPInit +_ippsECCPInitStd128r1 +_ippsECCPInitStd128r2 +_ippsECCPInitStd192r1 +_ippsECCPInitStd224r1 +_ippsECCPInitStd256r1 +_ippsECCPInitStd384r1 +_ippsECCPInitStd521r1 +_ippsECCPInitStdSM2 +_ippsECCPSet +_ippsECCPSetStd +_ippsECCPSetStd128r1 +_ippsECCPSetStd128r2 +_ippsECCPSetStd192r1 +_ippsECCPSetStd224r1 +_ippsECCPSetStd256r1 +_ippsECCPSetStd384r1 +_ippsECCPSetStd521r1 +_ippsECCPSetStdSM2 +_ippsECCPBindGxyTblStd192r1 +_ippsECCPBindGxyTblStd224r1 +_ippsECCPBindGxyTblStd256r1 +_ippsECCPBindGxyTblStd384r1 +_ippsECCPBindGxyTblStd521r1 +_ippsECCPBindGxyTblStdSM2 +_ippsECCPGet +_ippsECCPGetOrderBitSize +_ippsECCPValidate +_ippsECCPPointGetSize +_ippsECCPPointInit +_ippsECCPSetPoint +_ippsECCPSetPointAtInfinity +_ippsECCPGetPoint +_ippsECCPCheckPoint +_ippsECCPComparePoint +_ippsECCPNegativePoint +_ippsECCPAddPoint +_ippsECCPMulPointScalar +_ippsECCPGenKeyPair +_ippsECCPPublicKey +_ippsECCPValidateKeyPair +_ippsECCPSetKeyPair +_ippsECCPSharedSecretDH +_ippsECCPSharedSecretDHC +_ippsECCPSignDSA +_ippsECCPVerifyDSA +_ippsECCPSignNR +_ippsECCPVerifyNR +_ippsECCPSignSM2 +_ippsECCPVerifySM2 +_ippsGFpGetSize +_ippsGFpInitArbitrary +_ippsGFpInitFixed +_ippsGFpInit +_ippsGFpMethod_p192r1 +_ippsGFpMethod_p224r1 +_ippsGFpMethod_p256r1 +_ippsGFpMethod_p384r1 +_ippsGFpMethod_p521r1 +_ippsGFpMethod_p256sm2 +_ippsGFpMethod_p256bn +_ippsGFpMethod_p256 +_ippsGFpMethod_pArb +_ippsGFpxGetSize +_ippsGFpxInit +_ippsGFpxInitBinomial +_ippsGFpxMethod_binom2_epid2 +_ippsGFpxMethod_binom3_epid2 +_ippsGFpxMethod_binom2 +_ippsGFpxMethod_binom3 +_ippsGFpxMethod_binom +_ippsGFpxMethod_com +_ippsGFpScratchBufferSize +_ippsGFpElementGetSize +_ippsGFpElementInit +_ippsGFpSetElement +_ippsGFpSetElementRegular +_ippsGFpSetElementOctString +_ippsGFpSetElementRandom +_ippsGFpSetElementHash +_ippsGFpSetElementHash_rmf +_ippsGFpCpyElement +_ippsGFpGetElement +_ippsGFpGetElementOctString +_ippsGFpCmpElement +_ippsGFpIsZeroElement +_ippsGFpIsUnityElement +_ippsGFpConj +_ippsGFpNeg +_ippsGFpInv +_ippsGFpSqrt +_ippsGFpSqr +_ippsGFpAdd +_ippsGFpSub +_ippsGFpMul +_ippsGFpExp +_ippsGFpMultiExp +_ippsGFpAdd_PE +_ippsGFpSub_PE +_ippsGFpMul_PE +_ippsGFpGetInfo +_ippsGFpECGetSize +_ippsGFpECInit +_ippsGFpECSet +_ippsGFpECSetSubgroup +_ippsGFpECInitStd128r1 +_ippsGFpECInitStd128r2 +_ippsGFpECInitStd192r1 +_ippsGFpECInitStd224r1 +_ippsGFpECInitStd256r1 +_ippsGFpECInitStd384r1 +_ippsGFpECInitStd521r1 +_ippsGFpECInitStdSM2 +_ippsGFpECInitStdBN256 +_ippsGFpECBindGxyTblStd192r1 +_ippsGFpECBindGxyTblStd224r1 +_ippsGFpECBindGxyTblStd256r1 +_ippsGFpECBindGxyTblStd384r1 +_ippsGFpECBindGxyTblStd521r1 +_ippsGFpECBindGxyTblStdSM2 +_ippsGFpECGet +_ippsGFpECGetInfo_GF +_ippsGFpECGetSubgroup +_ippsGFpECScratchBufferSize +_ippsGFpECVerify +_ippsGFpECPointGetSize +_ippsGFpECPointInit +_ippsGFpECSetPointAtInfinity +_ippsGFpECSetPoint +_ippsGFpECSetPointRegular +_ippsGFpECSetPointRandom +_ippsGFpECMakePoint +_ippsGFpECSetPointHash +_ippsGFpECSetPointHash_rmf +_ippsGFpECSetPointHashBackCompatible +_ippsGFpECSetPointHashBackCompatible_rmf +_ippsGFpECGetPoint +_ippsGFpECGetPointRegular +_ippsGFpECTstPoint +_ippsGFpECTstPointInSubgroup +_ippsGFpECCpyPoint +_ippsGFpECCmpPoint +_ippsGFpECNegPoint +_ippsGFpECAddPoint +_ippsGFpECMulPoint +_ippsGFpECPrivateKey +_ippsGFpECPublicKey +_ippsGFpECTstKeyPair +_ippsGFpECSharedSecretDH +_ippsGFpECSharedSecretDHC +_ippsGFpECSignDSA +_ippsGFpECVerifyDSA +_ippsGFpECSignNR +_ippsGFpECVerifyNR +_ippsGFpECSignSM2 +_ippsGFpECVerifySM2 +_ippsGFpECUserIDHashSM2 +_ippsGFpECMessageRepresentationSM2 +_ippsGFpECKeyExchangeSM2_GetSize +_ippsGFpECKeyExchangeSM2_Init +_ippsGFpECKeyExchangeSM2_Setup +_ippsGFpECKeyExchangeSM2_SharedKey +_ippsGFpECKeyExchangeSM2_Confirm +_ippsGFpECEncryptSM2_Ext_EncMsgSize +_ippsGFpECEncryptSM2_Ext +_ippsGFpECDecryptSM2_Ext_DecMsgSize +_ippsGFpECDecryptSM2_Ext +_ippsGFpECESGetSize_SM2 +_ippsGFpECESInit_SM2 +_ippsGFpECESSetKey_SM2 +_ippsGFpECESStart_SM2 +_ippsGFpECESEncrypt_SM2 +_ippsGFpECESDecrypt_SM2 +_ippsGFpECESFinal_SM2 +_ippsGFpECESGetBuffersSize_SM2 +_ippsGFpECSetPointOctString +_ippsGFpECGetPointOctString diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_almmontinv.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_almmontinv.c new file mode 100644 index 0000000..a02b3af --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_almmontinv.c @@ -0,0 +1,105 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. Modular Arithmetic Engine. General Functionality +// +// Contents: +// alm_mont_inv() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbnumisc.h" +#include "pcpbnuarith.h" +#include "gsmodstuff.h" +#include "pcpmask_ct.h" + +/* +// almost Montgomery Inverse +// +// returns (k,r), r = (1/a)*(2^k) mod m +// +*/ +IPP_OWN_DEFN (int, alm_mont_inv, (BNU_CHUNK_T* pr, const BNU_CHUNK_T* pa, gsModEngine* pME)) +{ + const BNU_CHUNK_T* pm = MOD_MODULUS(pME); + int mLen = MOD_LEN(pME); + + const int polLength = 4; + BNU_CHUNK_T* pBuffer = gsModPoolAlloc(pME, polLength); + if(NULL == pBuffer) + return 0; // zero return value is handled by the calling function + + BNU_CHUNK_T* pu = pBuffer; + BNU_CHUNK_T* ps = pu+mLen; + BNU_CHUNK_T* pv = ps+mLen; + BNU_CHUNK_T* pt = pv+mLen; + + int k = 0; + BNU_CHUNK_T ext = 0; + + // u=modulus, v=a, t=0, s=1 + COPY_BNU(pu, pm, mLen); + ZEXPAND_BNU(ps, 0, mLen); ps[0] = 1; + COPY_BNU(pv, pa, mLen); + ZEXPAND_BNU(pt, 0, mLen); + + while(!cpEqu_BNU_CHUNK(pv, mLen, 0)) { // while(v>0) { + if(0==(pu[0]&1)) { // if(isEven(u)) { + cpLSR_BNU(pu, pu, mLen, 1); // u = u/2; + cpAdd_BNU(ps, ps, ps, mLen); // s = 2*s; + } // } + else if(0==(pv[0]&1)) { // else if(isEven(v)) { + cpLSR_BNU(pv, pv, mLen, 1); // v = v/2; + /*ext +=*/ cpAdd_BNU(pt, pt, pt, mLen); // t = 2*t; + } // } + else { + int cmpRes = cpCmp_BNU(pu, mLen, pv, mLen); + if(cmpRes>0) { // else if (u>v) { + cpSub_BNU(pu, pu, pv, mLen); // u = (u-v); + cpLSR_BNU(pu, pu, mLen, 1); // u = u/2; + /*ext +=*/ cpAdd_BNU(pt, pt, ps, mLen); // t = t+s; + cpAdd_BNU(ps, ps, ps, mLen); // s = 2*s; + } // } + else { // else if(v>=u) { + cpSub_BNU(pv, pv, pu, mLen); // v = (v-u); + cpLSR_BNU(pv, pv, mLen, 1); // v = v/2; + cpAdd_BNU(ps, ps, pt, mLen); // s = s+t; + ext += cpAdd_BNU(pt, pt, pt, mLen); // t = 2*t; + } // } + } + k++; // k += 1; + } // } + + // test + if(1!=cpEqu_BNU_CHUNK(pu, mLen, 1)) { + k = 0; /* inversion not found */ + } + + else { + ext -= cpSub_BNU(pr, pt, pm, mLen); // if(t>mod) r = t-mod; + cpMaskedReplace_ct(pr, pt, mLen, ~cpIsZero_ct(ext)); // else r = t; + cpSub_BNU(pr, pm, pr, mLen); // return r= (mod - r) and k + } + + gsModPoolFree(pME, polLength); + return k; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_almmontinv_ct.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_almmontinv_ct.c new file mode 100644 index 0000000..4a1f03c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_almmontinv_ct.c @@ -0,0 +1,136 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. Modular Arithmetic Engine. General Functionality +// +// Contents: +// alm_mont_inv_ct() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbnumisc.h" +#include "pcpbnuarith.h" +#include "gsmodstuff.h" +#include "pcpmask_ct.h" + +/* +// almost Montgomery Inverse +// +// returns (k,r), r = (1/a)*(2^k) mod m +// +// (constant-execution-time version) +*/ +IPP_OWN_DEFN (int, alm_mont_inv_ct, (BNU_CHUNK_T* pr, const BNU_CHUNK_T* pa, gsModEngine* pME)) +{ + const BNU_CHUNK_T* pm = MOD_MODULUS(pME); + int mLen = MOD_LEN(pME); + int modulusBitSize = MOD_BITSIZE(pME); + + const int polLength = 6; + BNU_CHUNK_T* pBuffer = gsModPoolAlloc(pME, polLength); + if(NULL == pBuffer) + return 0; // zero return value is handled by the calling function + + BNU_CHUNK_T* pu = pBuffer; + BNU_CHUNK_T* ps = pu+mLen; + BNU_CHUNK_T* pv = ps+mLen; + BNU_CHUNK_T* pt = pv+mLen; + + BNU_CHUNK_T* px = pt+mLen; + BNU_CHUNK_T* py = px+mLen; + + int k = 0; + int i; + BNU_CHUNK_T ext = 0; + + // u=modulus, s=1 and v=a, t=0, + COPY_BNU(pu, pm, mLen); + ZEXPAND_BNU(ps, 0, mLen); ps[0] = 1; + COPY_BNU(pv, pa, mLen); + ZEXPAND_BNU(pt, 0, mLen); + + for(i=0; i<2*modulusBitSize; i++) { + /* update mask - update = (v==0)? 0xFF : 0 */ + BNU_CHUNK_T update = ~cpIsGFpElemEquChunk_ct(pv, mLen, 0); + /* temporary masks */ + BNU_CHUNK_T m, mm; + + /* compute in advance r = s+t */ + cpAdd_BNU(pr, ps, pt, mLen); + + /* + // update or keep current u, s, v, t + */ + + /* if(isEven(u)) { u=u/2; s=2*s; } 1-st branch */ + m = update & cpIsEven_ct(pu[0]); + cpLSR_BNU(px, pu, mLen, 1); + cpAdd_BNU(py, ps, ps, mLen); + cpMaskedReplace_ct(pu, px, mLen, m); + cpMaskedReplace_ct(ps, py, mLen, m); + + /* else if(isEven(v)) { v=v/2; t=2*t; } 2-nd branch */ + mm = update & ~m & cpIsEven_ct(pv[0]); + cpLSR_BNU(px, pv, mLen, 1); + cpAdd_BNU(py, pt, pt, mLen); + cpMaskedReplace_ct(pv, px, mLen, mm); + cpMaskedReplace_ct(pt, py, mLen, mm); + + m |= mm; /* if fall in the 1-st of 2-nf branches m=FF.. else m=0 */ + + /* else if(v>=u) { v=(v-u)/2; s=s+t; t=2*t;} 3-st branch */ + mm = cpSub_BNU(px, pv, pu, mLen); + mm = cpIsZero_ct(mm); + mm = update & ~m & mm; + cpLSR_BNU(px, px, mLen, 1); + ext += cpAdd_BNU(py, pt, pt, mLen) & mm; + cpMaskedReplace_ct(pv, px, mLen, mm); + cpMaskedReplace_ct(ps, pr, mLen, mm); + cpMaskedReplace_ct(pt, py, mLen, mm); + + /* else { u=(u-v)/2; t= t+s; s=2*s; } 4-rd branch*/ + cpSub_BNU(px, pu, pv, mLen); + mm = update & ~m & ~mm; + cpLSR_BNU(px, px, mLen, 1); + cpAdd_BNU(py, ps, ps, mLen); + cpMaskedReplace_ct(pu, px, mLen, mm); + cpMaskedReplace_ct(pt, pr, mLen, mm); + cpMaskedReplace_ct(ps, py, mLen, mm); + + /* update or keep current k */ + k = ((k+1) & (Ipp32s)update) | (k & (Ipp32s)~update); + } + + /* + // r = (t>mod)? t-mod : t; + // r = mod - t; + */ + ext -= cpSub_BNU(pr, pt, pm, mLen); + cpMaskedReplace_ct(pr, pt, mLen, ~cpIsZero_ct(ext)); + cpSub_BNU(pr, pm, pr, mLen); + + /* test if inversion not found (k=0) */ + k &= cpIsGFpElemEquChunk_ct(pu, mLen, 1); + + gsModPoolFree(pME, polLength); + return k; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_enginegetsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_enginegetsize.c new file mode 100644 index 0000000..613e7c5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_enginegetsize.c @@ -0,0 +1,68 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. Modular Arithmetic Engine. General Functionality +// +// Contents: +// gsModEngineGetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbnumisc.h" +#include "pcpbnuarith.h" +#include "gsmodstuff.h" +#include "pcptool.h" + +/*F* +// Name: gsModEngineGetSize +// +// Purpose: Specifies size of size of ModEngine context (Montgomery). +// +// Returns: Reason: +// ippStsLengthErr modulusBitSize < 1 +// numpe < MOD_ENGINE_MIN_POOL_SIZE +// ippStsNoErr no errors +// +// Parameters: +// numpe length of pool +// modulusBitSize max modulus length (in bits) +// pSize pointer to size +// +*F*/ + +IPP_OWN_DEFN (IppStatus, gsModEngineGetSize, (int modulusBitSize, int numpe, int* pSize)) +{ + int modLen = BITS_BNU_CHUNK(modulusBitSize); + int pelmLen = BITS_BNU_CHUNK(modulusBitSize); + + IPP_BADARG_RET(modulusBitSize<1, ippStsLengthErr); + IPP_BADARG_RET(numpepoolLenUsed >= pME->poolLen)? NULL : MOD_BUFFER(pME, pME->poolLenUsed); + return pPool; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_inv.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_inv.c new file mode 100644 index 0000000..afc6f8a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_inv.c @@ -0,0 +1,69 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. Modular Arithmetic Engine. General Functionality +// +// Contents: +// gs_inv() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbnumisc.h" +#include "pcpbnuarith.h" +#include "gsmodstuff.h" + +/* +// returns r =1/a +// a in desidue domain +// r in desidue domain +*/ +IPP_OWN_DEFN (BNU_CHUNK_T*, gs_inv, (BNU_CHUNK_T* pr, const BNU_CHUNK_T* pa, gsModEngine* pME, alm_inv alm_inversion)) +{ + int k = alm_inversion(pr, pa, pME); + + if(0==k) + return NULL; + + { + int mLen = MOD_LEN(pME); + int m = mLen*BNU_CHUNK_BITS; + mod_mul mon_mul = MOD_METHOD(pME)->mul; + + BNU_CHUNK_T* t = gsModPoolAlloc(pME, 1); + if(NULL == t) + return NULL; + + if(k>m) { + ZEXPAND_BNU(t, 0, mLen); + t[0] = 1; + mon_mul(pr, pr, t, pME); + k -= m; + } + ZEXPAND_BNU(t, 0, mLen); + SET_BIT(t, m-k); /* t = 2^(m-k) */ + mon_mul(pr, pr, t, pME); + + gsModPoolFree(pME, 1); + + return pr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_montfactor.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_montfactor.c new file mode 100644 index 0000000..47b30a3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_montfactor.c @@ -0,0 +1,53 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. Modular Arithmetic Engine. General Functionality +// +// Contents: +// gsMontFactor() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbnumisc.h" +#include "pcpbnuarith.h" +#include "gsmodstuff.h" +#include "pcptool.h" + +/* +// montfomery factor k0 = -((modulus^-1 mod B) %B) +*/ +IPP_OWN_DEFN (BNU_CHUNK_T, gsMontFactor, (BNU_CHUNK_T m0)) +{ + BNU_CHUNK_T y = 1; + BNU_CHUNK_T x = 2; + BNU_CHUNK_T mask = 2*x-1; + + int i; + for(i=2; i<=BNU_CHUNK_BITS; i++, x<<=1) { + BNU_CHUNK_T rH, rL; + MUL_AB(rH, rL, m0, y); + if( x < (rL & mask) ) /* x < ((m0*y) mod (2*x)) */ + y+=x; + mask += mask + 1; + } + return 0-y; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_montinv.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_montinv.c new file mode 100644 index 0000000..b0a21c7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_montinv.c @@ -0,0 +1,85 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. Modular Arithmetic Engine. General Functionality +// +// Contents: +// gs_mont_inv() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbnumisc.h" +#include "pcpbnuarith.h" +#include "gsmodstuff.h" +#include "pcpmask_ct.h" + +__INLINE BNU_CHUNK_T* cpPow2_ct(int bit, BNU_CHUNK_T* dst, int len) +{ + int slot = bit/BNU_CHUNK_BITS; + BNU_CHUNK_T value = (BNU_CHUNK_T)1 << (bit%BNU_CHUNK_BITS); + + int i; + len -= (int)( cpIsEqu_ct((BNU_CHUNK_T)slot, (BNU_CHUNK_T)len) ); + for(i=0; imul; + + BNU_CHUNK_T* t = gsModPoolAlloc(pME, 1); + if(NULL == t) + return NULL; + + if(k <= m) { + mon_mul(pr, pr, MOD_MNT_R2(pME), pME); + k += m; + } + + //ZEXPAND_BNU(t, 0, mLen); + //SET_BIT(t, 2*m-k); /* t = 2^(2*m-k) */ + cpPow2_ct(2*m-k, t, mLen); + mon_mul(pr, pr, t, pME); + + gsModPoolFree(pME, 1); + + return pr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_packctx.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_packctx.c new file mode 100644 index 0000000..8465562 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_packctx.c @@ -0,0 +1,52 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. Modular Arithmetic Engine. General Functionality +// +// Contents: +// gsPackModEngineCtx() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbnumisc.h" +#include "pcpbnuarith.h" +#include "gsmodstuff.h" +#include "pcptool.h" + +/* +// Pack/Unpack methods +*/ +IPP_OWN_DEFN (void, gsPackModEngineCtx, (const gsModEngine* pCtx, Ipp8u* pBuffer)) +{ + gsModEngine* pB = (gsModEngine*)pBuffer; + + /* max modulus length */ + int modSize = MOD_LEN(pCtx); + /* size of context (bytes) without cube and pool buffers */ + int ctxSize = (Ipp32s)sizeof(gsModEngine) + +(Ipp32s)sizeof(BNU_CHUNK_T)*(modSize*3); + + CopyBlock(pCtx, pB, ctxSize); + MOD_MODULUS(pB) = (BNU_CHUNK_T*)((Ipp8u*)NULL + IPP_UINT_PTR(MOD_MODULUS(pCtx))-IPP_UINT_PTR(pCtx)); + MOD_MNT_R(pB) = (BNU_CHUNK_T*)((Ipp8u*)NULL + IPP_UINT_PTR(MOD_MNT_R(pCtx))-IPP_UINT_PTR(pCtx)); + MOD_MNT_R2(pB) = (BNU_CHUNK_T*)((Ipp8u*)NULL + IPP_UINT_PTR(MOD_MNT_R2(pCtx))-IPP_UINT_PTR(pCtx)); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_unpackctx.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_unpackctx.c new file mode 100644 index 0000000..bc0d9ff --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmod_unpackctx.c @@ -0,0 +1,50 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. Modular Arithmetic Engine. General Functionality +// +// Contents: +// gsUnpackModEngineCtx() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbnumisc.h" +#include "pcpbnuarith.h" +#include "gsmodstuff.h" +#include "pcptool.h" + +IPP_OWN_DEFN (void, gsUnpackModEngineCtx, (const Ipp8u* pBuffer, gsModEngine* pCtx)) +{ + gsModEngine* pAlignedBuffer = (gsModEngine*)pBuffer; + + /* max modulus length */ + int modSize = MOD_LEN(pAlignedBuffer); + /* size of context (bytes) without cube and pool buffers */ + int ctxSize = (Ipp32s)sizeof(gsModEngine) + +(Ipp32s)sizeof(BNU_CHUNK_T)*(modSize*3); + + CopyBlock(pAlignedBuffer, pCtx, ctxSize); + MOD_MODULUS(pCtx) = (BNU_CHUNK_T*)((Ipp8u*)pCtx + IPP_UINT_PTR(MOD_MODULUS(pAlignedBuffer))); + MOD_MNT_R(pCtx) = (BNU_CHUNK_T*)((Ipp8u*)pCtx + IPP_UINT_PTR(MOD_MNT_R(pAlignedBuffer))); + MOD_MNT_R2(pCtx) = (BNU_CHUNK_T*)((Ipp8u*)pCtx + IPP_UINT_PTR(MOD_MNT_R2(pAlignedBuffer))); + MOD_POOL_BUF(pCtx) = MOD_MNT_R2(pCtx) + modSize; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmodmethod.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmodmethod.c new file mode 100644 index 0000000..4bc74de --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmodmethod.c @@ -0,0 +1,70 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "gsmodmethodstuff.h" + +/* + methods +*/ + +static gsModMethod* gsModArith_C(void) +{ + static gsModMethod m = { + gs_mont_encode, + gs_mont_decode, + gs_mont_mul, + gs_mont_sqr, + gs_mont_red, + gs_mont_add, + gs_mont_sub, + gs_mont_neg, + gs_mont_div2, + gs_mont_mul2, + gs_mont_mul3, + }; + return &m; +} + +#if (_IPP32E>=_IPP32E_L9) +static gsModMethod* gsModArith_X(void) +{ + static gsModMethod m = { + gs_mont_encodeX, + gs_mont_decodeX, + gs_mont_mulX, + gs_mont_sqrX, + gs_mont_redX, + gs_mont_add, + gs_mont_sub, + gs_mont_neg, + gs_mont_div2, + gs_mont_mul2, + gs_mont_mul3, + }; + return &m; +} +#endif + +IPP_OWN_DEFN (gsModMethod*, gsModArith, (void)) +{ + #if (_IPP32E>=_IPP32E_L9) + if(IsFeatureEnabled(ippCPUID_ADCOX)) + return gsModArith_X(); + else + #endif + return gsModArith_C(); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmodmethod.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmodmethod.h new file mode 100644 index 0000000..16a449c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmodmethod.h @@ -0,0 +1,101 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#if !defined(_GS_MOD_METHOD_H) +#define _GS_MOD_METHOD_H + +//#include "owndefs.h" +#include "owncp.h" + +#include "pcpbnuimpl.h" +//#include "gsmodstuff.h" + +typedef struct _gsModEngine gsEngine; + +/* modular arith methods */ +IPP_OWN_FUNPTR (BNU_CHUNK_T*, mod_encode, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pMA)) +IPP_OWN_FUNPTR (BNU_CHUNK_T*, mod_decode, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pMA)) +IPP_OWN_FUNPTR (BNU_CHUNK_T*, mod_red, (BNU_CHUNK_T* pR, BNU_CHUNK_T* pA, gsEngine* pMA)) +IPP_OWN_FUNPTR (BNU_CHUNK_T*, mod_sqr, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pMA)) +IPP_OWN_FUNPTR (BNU_CHUNK_T*, mod_mul, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pMA)) +IPP_OWN_FUNPTR (BNU_CHUNK_T*, mod_add, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pMA)) +IPP_OWN_FUNPTR (BNU_CHUNK_T*, mod_sub, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pMA)) +IPP_OWN_FUNPTR (BNU_CHUNK_T*, mod_neg, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pMA)) +IPP_OWN_FUNPTR (BNU_CHUNK_T*, mod_div2, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pMA)) +IPP_OWN_FUNPTR (BNU_CHUNK_T*, mod_mul2, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pMA)) +IPP_OWN_FUNPTR (BNU_CHUNK_T*, mod_mul3, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pMA)) + +typedef struct _gsModMethod { + mod_encode encode; + mod_decode decode; + mod_mul mul; + mod_sqr sqr; + mod_red red; + mod_add add; + mod_sub sub; + mod_neg neg; + mod_div2 div2; + mod_mul2 mul2; + mod_mul3 mul3; +} gsModMethod; + +/* These functions should not be used, because they have non-constant execution time, see their safe analogues in pcpmask_ct.h */ +#if 0 +__INLINE BNU_CHUNK_T cpIsZero(BNU_CHUNK_T x) +{ return x==0; } +__INLINE BNU_CHUNK_T cpIsNonZero(BNU_CHUNK_T x) +{ return x!=0; } +__INLINE BNU_CHUNK_T cpIsOdd(BNU_CHUNK_T x) +{ return x&1; } +__INLINE BNU_CHUNK_T cpIsEven(BNU_CHUNK_T x) +{ return 1-cpIsOdd(x); } + +/* dst[] = (flag)? src[] : dst[] */ +__INLINE void cpMaskMove_gs(BNU_CHUNK_T* dst, const BNU_CHUNK_T* src, int len, BNU_CHUNK_T moveFlag) +{ + BNU_CHUNK_T srcMask = 0-cpIsNonZero(moveFlag); + BNU_CHUNK_T dstMask = ~srcMask; + int n; + for(n=0; n=_IPP32E_L9) +static gsModMethod* gsModArithDLP_X(void) +{ + static gsModMethod m = { + gs_mont_encodeX, + gs_mont_decodeX, + gs_mont_mulX, + gs_mont_sqrX, + gs_mont_redX, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + }; + return &m; +} +#endif + +IPP_OWN_DEFN (gsModMethod*, gsModArithDLP, (void)) +{ + #if (_IPP32E>=_IPP32E_L9) + if(IsFeatureEnabled(ippCPUID_ADCOX)) + return gsModArithDLP_X(); + else + #endif + return gsModArithDLP_C(); +} +/* ******************************************** */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmodmethod_gfp.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmodmethod_gfp.c new file mode 100644 index 0000000..7b22ab6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmodmethod_gfp.c @@ -0,0 +1,71 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "gsmodmethodstuff.h" + +/* + methods +*/ + +/* ******************************************** */ + +static gsModMethod* gsArithGFp_C(void) +{ + static gsModMethod m = { + gs_mont_encode, + gs_mont_decode, + gs_mont_mul, + gs_mont_sqr, + NULL, + gs_mont_add, + gs_mont_sub, + gs_mont_neg, + gs_mont_div2, + gs_mont_mul2, + gs_mont_mul3, + }; + return &m; +} +#if (_IPP32E>=_IPP32E_L9) +static gsModMethod* gsArithGFp_X(void) +{ + static gsModMethod m = { + gs_mont_encode, + gs_mont_decode, + gs_mont_mul, + gs_mont_sqr, + NULL, + gs_mont_add, + gs_mont_sub, + gs_mont_neg, + gs_mont_div2, + gs_mont_mul2, + gs_mont_mul3, + }; + return &m; +} +#endif + +IPP_OWN_DEFN (gsModMethod*, gsArithGFp, (void)) +{ + #if (_IPP32E>=_IPP32E_L9) + if(IsFeatureEnabled(ippCPUID_ADCOX)) + return gsArithGFp_X(); + else + #endif + return gsArithGFp_C(); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmodmethod_mont.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmodmethod_mont.c new file mode 100644 index 0000000..4f10933 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmodmethod_mont.c @@ -0,0 +1,72 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "gsmodmethodstuff.h" + +/* + methods +*/ + +/* ******************************************** */ + +static gsModMethod* gsModArithMont_C(void) +{ + static gsModMethod m = { + gs_mont_encode, + gs_mont_decode, + gs_mont_mul, + gs_mont_sqr, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + }; + return &m; +} +#if (_IPP32E>=_IPP32E_L9) +static gsModMethod* gsModArithMont_X(void) +{ + static gsModMethod m = { + gs_mont_encodeX, + gs_mont_decodeX, + gs_mont_mulX, + gs_mont_sqrX, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + }; + return &m; +} +#endif + +IPP_OWN_DEFN (gsModMethod*, gsModArithMont, (void)) +{ + #if (_IPP32E>=_IPP32E_L9) + if(IsFeatureEnabled(ippCPUID_ADCOX)) + return gsModArithMont_X(); + else + #endif + return gsModArithMont_C(); +} +/* ******************************************** */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmodmethod_rsa.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmodmethod_rsa.c new file mode 100644 index 0000000..ba8ac21 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmodmethod_rsa.c @@ -0,0 +1,72 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "gsmodmethodstuff.h" + +/* + methods +*/ + +/* ******************************************** */ + +static gsModMethod* gsModArithRSA_C(void) +{ + static gsModMethod m = { + gs_mont_encode, + gs_mont_decode, + gs_mont_mul, + gs_mont_sqr, + gs_mont_red, + NULL, + gs_mont_sub, + NULL, + NULL, + NULL, + NULL, + }; + return &m; +} +#if (_IPP32E>=_IPP32E_L9) +static gsModMethod* gsModArithRSA_X(void) +{ + static gsModMethod m = { + gs_mont_encodeX, + gs_mont_decodeX, + gs_mont_mulX, + gs_mont_sqrX, + gs_mont_redX, + NULL, + gs_mont_sub, + NULL, + NULL, + NULL, + NULL, + }; + return &m; +} +#endif + +IPP_OWN_DEFN (gsModMethod*, gsModArithRSA, (void)) +{ + #if (_IPP32E>=_IPP32E_L9) + if(IsFeatureEnabled(ippCPUID_ADCOX)) + return gsModArithRSA_X(); + else + #endif + return gsModArithRSA_C(); +} +/* ******************************************** */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmodmethodstuff.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmodmethodstuff.h new file mode 100644 index 0000000..6a37c03 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmodmethodstuff.h @@ -0,0 +1,500 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#if !defined(_GS_MOD_METHOD_STUFF_H) +#define _GS_MOD_METHOD_STUFF_H + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbnumisc.h" +#include "pcpbnuarith.h" + +#include "gsmodstuff.h" +#include "gsmodmethod.h" +#include "pcpmontred.h" +#include "pcpmask_ct.h" + +/* r = (a+m) mod m */ +/* + * Requirements: + * Length of pr data buffer: modLen + * Length of pa data buffer: modLen + * Length of pb data buffer: modLen + * Memory size from the pool: modLen * sizeof(BNU_CHUNK_T) + */ +IPP_OWN_DEFN (static BNU_CHUNK_T*, gs_mont_add, (BNU_CHUNK_T* pr, const BNU_CHUNK_T* pa, const BNU_CHUNK_T* pb, gsModEngine* pME)) +{ + const BNU_CHUNK_T* pm = MOD_MODULUS(pME); + int mLen = MOD_LEN(pME); + + const int polLength = 1; + BNU_CHUNK_T* pBuffer = gsModPoolAlloc(pME, polLength); + if(NULL == pBuffer) + return NULL; + + { + BNU_CHUNK_T extension = cpAdd_BNU(pr, pa, pb, mLen); + extension -= cpSub_BNU(pBuffer, pr, pm, mLen); + cpMaskedReplace_ct(pr, pBuffer, mLen, cpIsZero_ct(extension)); + } + gsModPoolFree(pME, polLength); + return pr; +} + +/* r = (a-b) mod m */ +/* + * Requirements: + * Length of pr data buffer: modLen + * Length of pa data buffer: modLen + * Length of pb data buffer: modLen + * Memory size from the pool: modLen * sizeof(BNU_CHUNK_T) + */ +IPP_OWN_DEFN (static BNU_CHUNK_T*, gs_mont_sub, (BNU_CHUNK_T* pr, const BNU_CHUNK_T* pa, const BNU_CHUNK_T* pb, gsModEngine* pME)) +{ + const BNU_CHUNK_T* pm = MOD_MODULUS(pME); + int mLen = MOD_LEN(pME); + + const int polLength = 1; + BNU_CHUNK_T* pBuffer = gsModPoolAlloc(pME, polLength); + if(NULL == pBuffer) + return NULL; + + { + BNU_CHUNK_T extension = cpSub_BNU(pr, pa, pb, mLen); + cpAdd_BNU(pBuffer, pr, pm, mLen); + cpMaskedReplace_ct(pr, pBuffer, mLen, ~cpIsZero_ct(extension)); + } + gsModPoolFree(pME, polLength); + return pr; +} + +/* r = (m-a) mod m */ +/* + * Requirements: + * Length of pr data buffer: modLen + * Length of pa data buffer: modLen + * Memory size from the pool: modLen * sizeof(BNU_CHUNK_T) + */ +IPP_OWN_DEFN (static BNU_CHUNK_T*, gs_mont_neg, (BNU_CHUNK_T* pr, const BNU_CHUNK_T* pa, gsModEngine* pME)) +{ + const BNU_CHUNK_T* pm = MOD_MODULUS(pME); + int mLen = MOD_LEN(pME); + + const int polLength = 1; + BNU_CHUNK_T* pBuffer = gsModPoolAlloc(pME, polLength); + if(NULL == pBuffer) + return NULL; + + { + BNU_CHUNK_T extension = cpSub_BNU(pr, pm, pa, mLen); + extension -= cpSub_BNU(pBuffer, pr, pm, mLen); + cpMaskedReplace_ct(pr, pBuffer, mLen, cpIsZero_ct(extension)); + } + gsModPoolFree(pME, polLength); + return pr; +} + +/* r = (a/2) mod m */ +/* + * Requirements: + * Length of pr data buffer: modLen + * Length of pa data buffer: modLen + * Memory size from the pool: modLen * sizeof(BNU_CHUNK_T) + */ +IPP_OWN_DEFN (static BNU_CHUNK_T*, gs_mont_div2, (BNU_CHUNK_T* pr, const BNU_CHUNK_T* pa, gsModEngine* pME)) +{ + const BNU_CHUNK_T* pm = MOD_MODULUS(pME); + int mLen = MOD_LEN(pME); + + const int polLength = 1; + BNU_CHUNK_T* pBuffer = gsModPoolAlloc(pME, polLength); + if(NULL == pBuffer) + return NULL; + + { + cpSize i; + BNU_CHUNK_T mask = 0 - (pa[0]&1); + for(i=0; i=_IPP32E_L9) +IPP_OWN_DEFN (static BNU_CHUNK_T*, gs_mont_redX, (BNU_CHUNK_T* pr, BNU_CHUNK_T* prod, gsModEngine* pME)) +{ + const BNU_CHUNK_T* pm = MOD_MODULUS(pME); + BNU_CHUNK_T k0 = MOD_MNT_FACTOR(pME); + int mLen = MOD_LEN(pME); + + cpMontRedAdx_BNU(pr, prod, pm, mLen, k0); + + return pr; +} +#endif + +#endif + + +/* r = (a*b) mod m */ +/* + * Requirements: + * Length of pr data buffer: modLen + * Length of pa data buffer: modLen + * Length of pb data buffer: modLen + * Memory size from the pool: modLen * sizeof(BNU_CHUNK_T) * 2 + */ +#if ((_IPP <_IPP_W7) && (_IPP32E <_IPP32E_M7)) +IPP_OWN_DEFN (static BNU_CHUNK_T*, gs_mont_mul, (BNU_CHUNK_T* pr, const BNU_CHUNK_T* pa, const BNU_CHUNK_T* pb, gsModEngine* pME)) +{ + const BNU_CHUNK_T* pm = MOD_MODULUS(pME); + BNU_CHUNK_T m0 = MOD_MNT_FACTOR(pME); + int mLen = MOD_LEN(pME); + + const int polLength = 1; + BNU_CHUNK_T* pBuffer = gsModPoolAlloc(pME, polLength); + if(NULL == pBuffer) + return NULL; + + { + BNU_CHUNK_T carry = 0; + int i, j; + + /* clear buffer */ + for(i=0; i=_IPP32E_L9) +IPP_OWN_DEFN (static BNU_CHUNK_T*, gs_mont_mulX, (BNU_CHUNK_T* pr, const BNU_CHUNK_T* pa, const BNU_CHUNK_T* pb, gsModEngine* pME)) +{ + const BNU_CHUNK_T* pm = MOD_MODULUS(pME); + BNU_CHUNK_T m0 = MOD_MNT_FACTOR(pME); + int mLen = MOD_LEN(pME); + + const int polLength = 2; + BNU_CHUNK_T* pProduct = gsModPoolAlloc(pME, polLength); + if(NULL == pProduct) + return NULL; + + cpMulAdx_BNU_school(pProduct, pa,mLen, pb,mLen); + cpMontRedAdx_BNU(pr, pProduct, pm, mLen, m0); + + gsModPoolFree(pME, polLength); + return pr; +} +#endif +#endif + +/* r = (a^2) mod m */ +/* + * Requirements: + * Length of pr data buffer: modLen + * Length of pa data buffer: modLen + * Memory size from the pool: modLen * sizeof(BNU_CHUNK_T) + */ +IPP_OWN_DEFN (static BNU_CHUNK_T*, gs_mont_sqr, (BNU_CHUNK_T* pr, const BNU_CHUNK_T* pa, gsModEngine* pME)) +{ + //return gs_mont_mul(pr, pa, pa, pME); + const BNU_CHUNK_T* pm = MOD_MODULUS(pME); + BNU_CHUNK_T m0 = MOD_MNT_FACTOR(pME); + int mLen = MOD_LEN(pME); + + const int polLength = 2; + BNU_CHUNK_T* pProduct = gsModPoolAlloc(pME, polLength); + if(NULL == pProduct) + return NULL; + + cpSqrAdc_BNU_school(pProduct, pa,mLen); + cpMontRedAdc_BNU(pr, pProduct, pm, mLen, m0); + + gsModPoolFree(pME, polLength); + return pr; +} + +#if (_IPP32E>=_IPP32E_L9) +IPP_OWN_DEFN (static BNU_CHUNK_T*, gs_mont_sqrX, (BNU_CHUNK_T* pr, const BNU_CHUNK_T* pa, gsModEngine* pME)) +{ + const BNU_CHUNK_T* pm = MOD_MODULUS(pME); + BNU_CHUNK_T m0 = MOD_MNT_FACTOR(pME); + int mLen = MOD_LEN(pME); + + const int polLength = 2; + BNU_CHUNK_T* pProduct = gsModPoolAlloc(pME, polLength); + if(NULL == pProduct) + return NULL; + + cpSqrAdx_BNU_school(pProduct, pa,mLen); + cpMontRedAdx_BNU(pr, pProduct, pm, mLen, m0); + + gsModPoolFree(pME, polLength); + return pr; +} +#endif + +/* r = to_mont(a) */ +/* + * Requirements: + * Length of pr data buffer: modLen + * Length of pa data buffer: modLen + * Memory size from the pool: modLen * sizeof(BNU_CHUNK_T) + */ +IPP_OWN_DEFN (static BNU_CHUNK_T*, gs_mont_encode, (BNU_CHUNK_T* pr, const BNU_CHUNK_T* pa, gsModEngine* pME)) +{ + //return gs_mont_mul(pr, pa, MOD_MNT_R2(pME), pME); + const BNU_CHUNK_T* pm = MOD_MODULUS(pME); + BNU_CHUNK_T m0 = MOD_MNT_FACTOR(pME); + int mLen = MOD_LEN(pME); + + const int polLength = 2; + BNU_CHUNK_T* pProduct = gsModPoolAlloc(pME, polLength); + if(NULL == pProduct) + return NULL; + + cpMulAdc_BNU_school(pProduct, pa,mLen, MOD_MNT_R2(pME),mLen); + cpMontRedAdc_BNU(pr, pProduct, pm, mLen, m0); + + gsModPoolFree(pME, polLength); + return pr; +} + +#if (_IPP32E>=_IPP32E_L9) +IPP_OWN_DEFN (static BNU_CHUNK_T*, gs_mont_encodeX, (BNU_CHUNK_T* pr, const BNU_CHUNK_T* pa, gsModEngine* pME)) +{ + //return gs_mont_mul(pr, pa, MOD_MNT_R2(pME), pME); + const BNU_CHUNK_T* pm = MOD_MODULUS(pME); + BNU_CHUNK_T m0 = MOD_MNT_FACTOR(pME); + int mLen = MOD_LEN(pME); + + const int polLength = 2; + BNU_CHUNK_T* pProduct = gsModPoolAlloc(pME, polLength); + if(NULL == pProduct) + return NULL; + + cpMulAdx_BNU_school(pProduct, pa,mLen, MOD_MNT_R2(pME),mLen); + cpMontRedAdx_BNU(pr, pProduct, pm, mLen, m0); + + gsModPoolFree(pME, polLength); + return pr; +} +#endif + +/* r = from_momt(a) */ +/* + * Requirements: + * Length of pr data buffer: modLen + * Length of pa data buffer: modLen + * Memory size from the pool: modLen * sizeof(BNU_CHUNK_T) + */ +IPP_OWN_DEFN (static BNU_CHUNK_T*, gs_mont_decode, (BNU_CHUNK_T* pr, const BNU_CHUNK_T* pa, gsModEngine* pME)) +{ + int mLen = MOD_LEN(pME); + + const int polLength = 2; + BNU_CHUNK_T* pProduct = gsModPoolAlloc(pME, polLength); + if(NULL == pProduct) + return NULL; + + ZEXPAND_COPY_BNU(pProduct, 2*mLen, pa, mLen); + cpMontRedAdc_BNU(pr, pProduct, MOD_MODULUS(pME), mLen, MOD_MNT_FACTOR(pME)); + + gsModPoolFree(pME, polLength); + return pr; +} + +#if (_IPP32E>=_IPP32E_L9) +IPP_OWN_DEFN (static BNU_CHUNK_T*, gs_mont_decodeX, (BNU_CHUNK_T* pr, const BNU_CHUNK_T* pa, gsModEngine* pME)) +{ + int mLen = MOD_LEN(pME); + + const int polLength = 2; + BNU_CHUNK_T* pProduct = gsModPoolAlloc(pME, polLength); + if(NULL == pProduct) + return NULL; + + ZEXPAND_COPY_BNU(pProduct, 2*mLen, pa, mLen); + cpMontRedAdx_BNU(pr, pProduct, MOD_MODULUS(pME), mLen, MOD_MNT_FACTOR(pME)); + + gsModPoolFree(pME, polLength); + return pr; +} +#endif + +#endif /* _GS_MOD_METHOD_STUFF_H */ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmodstuff.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmodstuff.h new file mode 100644 index 0000000..1ed3f2c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsmodstuff.h @@ -0,0 +1,162 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#if !defined(_GS_MOD_STUFF_H) +#define _GS_MOD_STUFF_H + +//#define MONTMUL_ONESTAGE + +#include "owncp.h" + +#include "pcpbnuimpl.h" +#include "gsmodmethod.h" + +#define MOD_ENGINE_MIN_POOL_SIZE 1 + +typedef struct _gsModEngine gsModEngine_T; + +typedef struct _gsModEngine +{ + gsModEngine_T* pParentME; /* pointer to parent stuff */ + int extdegree; /* parent modulus extension (deg) */ + int modBitLen; /* length of modulus in bits */ + int modLen; /* length of modulus (BNU_CHUNK_T) */ + int modLen32; /* length of modulus (Ipp32u) */ + int peLen; /* length of pool element (BNU_CHUNK_T) */ + const gsModMethod* method; /* modular arithmetic methods - regular radix */ + const void* method_alt; /* modular arithmetic methods - alternative radix implementation */ + BNU_CHUNK_T* pModulus; /* modulus */ + BNU_CHUNK_T k0; /* low word of (1/modulus) mod R */ + BNU_CHUNK_T* pMontR; /* mont_enc(1) */ + BNU_CHUNK_T* pMontR2; /* mont_enc(1)^2 */ + BNU_CHUNK_T* pHalfModulus; /* modulus/2 */ + BNU_CHUNK_T* pQnr; /* quadratic non-residue */ + int poolLenUsed; /* number of reserved temporary BNU */ + int poolLen; /* max number of temporary BNU */ + BNU_CHUNK_T* pBuffer; /* buffer of modLen*nBuffers length */ +} gsModEngine; + +/* accessory macros */ +#define MOD_PARENT(eng) ((eng)->pParentME) +#define MOD_EXTDEG(eng) ((eng)->extdegree) +#define MOD_BITSIZE(eng) ((eng)->modBitLen) +#define MOD_LEN(eng) ((eng)->modLen) +#define MOD_LEN32(eng) ((eng)->modLen32) +#define MOD_PELEN(eng) ((eng)->peLen) +#define MOD_METHOD(eng) ((eng)->method) +#define MOD_METHOD_ALT(eng) ((eng)->method_alt) +#define MOD_MODULUS(eng) ((eng)->pModulus) +#define MOD_MNT_FACTOR(eng) ((eng)->k0) +#define MOD_MNT_R(eng) ((eng)->pMontR) +#define MOD_MNT_R2(eng) ((eng)->pMontR2) +#define MOD_HMODULUS(eng) ((eng)->pHalfModulus) +#define MOD_QNR(eng) ((eng)->pQnr) +#define MOD_POOL_BUF(eng) ((eng)->pBuffer) +#define MOD_MAXPOOL(eng) ((eng)->poolLen) +#define MOD_USEDPOOL(eng) ((eng)->poolLenUsed) + +#define MOD_BUFFER(eng,n) ((eng)->pBuffer+(MOD_PELEN(eng))*(n)) + +#define MOD_ENGINE_ALIGNMENT ((int)sizeof(void*)) + +/* +// size of context and it initialization +*/ +#define gsModEngineInit OWNAPI(gsModEngineInit) + IPP_OWN_DECL (IppStatus, gsModEngineInit, (gsModEngine* pME, const Ipp32u* pModulus, int modulusBitSize, int numpe, const gsModMethod* method)) +#define gsModEngineGetSize OWNAPI(gsModEngineGetSize) + IPP_OWN_DECL (IppStatus, gsModEngineGetSize, (int modulusBitSIze, int numpe, int* pSize)) +#define gsMontFactor OWNAPI(gsMontFactor) + IPP_OWN_DECL (BNU_CHUNK_T, gsMontFactor, (BNU_CHUNK_T m0)) + + +/* +// pool management methods +*/ + +/*F* +// Name: gsModPoolAlloc +// +// Purpose: Allocation pool. +// +// Returns: Reason: +// pointer to allocate Pool enough of pool +// NULL required pool more than pME have +// +// Parameters: +// pME ModEngine +// poolReq Required pool +*F*/ + +__INLINE BNU_CHUNK_T* gsModPoolAlloc(gsModEngine* pME, int poolReq) +{ + BNU_CHUNK_T* pPool = MOD_BUFFER(pME, pME->poolLenUsed); + + if(pME->poolLenUsed + poolReq > pME->poolLen) + pPool = NULL; + else + pME->poolLenUsed += poolReq; + + return pPool; +} + +/*F* +// Name: gsModPoolFree +// +// Purpose: Delete pool. +// +// Returns: +// nothing +// +// Parameters: +// pME ModEngine +// poolReq Required pool +*F*/ + +__INLINE void gsModPoolFree(gsModEngine* pME, int poolReq) +{ + if(pME->poolLenUsed < poolReq) + poolReq = pME->poolLenUsed; + pME->poolLenUsed -= poolReq; +} + +/* return pointer to the top pool buffer */ +#define gsModGetPool OWNAPI(gsModGetPool) + IPP_OWN_DECL (BNU_CHUNK_T*, gsModGetPool, (gsModEngine* pME)) +/* +// advanced operations +*/ +IPP_OWN_FUNPTR (int, alm_inv, (BNU_CHUNK_T* pr, const BNU_CHUNK_T* pa, gsModEngine* pMA)) + +#define alm_mont_inv OWNAPI(alm_mont_inv) + IPP_OWN_DECL (int, alm_mont_inv, (BNU_CHUNK_T* pr, const BNU_CHUNK_T* pa, gsModEngine* pMA)) +#define alm_mont_inv_ct OWNAPI(alm_mont_inv_ct) + IPP_OWN_DECL (int, alm_mont_inv_ct, (BNU_CHUNK_T* pr, const BNU_CHUNK_T* pa, gsModEngine* pMA)) +#define gs_mont_inv OWNAPI(gs_mont_inv) + IPP_OWN_DECL (BNU_CHUNK_T*, gs_mont_inv, (BNU_CHUNK_T* pr, const BNU_CHUNK_T* pa, gsModEngine* pMA, alm_inv invf)) +#define gs_inv OWNAPI(gs_inv) + IPP_OWN_DECL (BNU_CHUNK_T*, gs_inv, (BNU_CHUNK_T* pr, const BNU_CHUNK_T* pa, gsModEngine* pMA, alm_inv invf)) + +/* +// Pack/Unpack methods +*/ +#define gsPackModEngineCtx OWNAPI(gsPackModEngineCtx) + IPP_OWN_DECL (void, gsPackModEngineCtx, (const gsModEngine* pCtx, Ipp8u* pBuffer)) +#define gsUnpackModEngineCtx OWNAPI(gsUnpackModEngineCtx) + IPP_OWN_DECL (void, gsUnpackModEngineCtx, (const Ipp8u* pBuffer, gsModEngine* pCtx)) + +#endif /* _GS_MOD_STUFF_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsscramble.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsscramble.c new file mode 100644 index 0000000..5f6a8fc --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/gsscramble.c @@ -0,0 +1,58 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Fixed window exponentiation scramble/unscramble +// +// Contents: +// gsScramblePut() +// gsScrambleGet() +// +*/ + +#include "owncp.h" +#include "gsscramble.h" + +IPP_OWN_DEFN (int, gsGetScrambleBufferSize, (int modulusLen, int w)) +{ + /* size of resource to store 2^w values of modulusLen*sizeof(BNU_CHUNK_T) each */ + int size = (1<=_IPP32E_K1) + +#include "pcptool.h" + +#include "pcpngmontexpstuff_avx512.h" + +#include "ifma_math_avx512vl.h" +#include "ifma_norm52x.h" +#include "ifma_rsa_arith.h" + +#define ADD104(rh, rl, ih, il) { \ + rl += il; \ + rh += ih; \ + rh += (rl>52U) | rh<<(64U-52U)) + +/* + * Almost Montgomery Multiplication in 2^52-radix + * + * Data represented as 20-qwords arrays in 2^52-radix. + * + */ +IPP_OWN_DEFN(void, ifma256_amm52x20, (Ipp64u out[20], + const Ipp64u a [20], + const Ipp64u b [20], + const Ipp64u m [20], + Ipp64u k0)) +{ + /* R0, R1, R2 holds result */ + U64 R0 = get_zero64(); + U64 R1 = get_zero64(); + U64 R2 = get_zero64(); + + /* High part of 512bit result in 256bit mode */ + U64 R0h = get_zero64(); + U64 R1h = get_zero64(); + U64 R2h = get_zero64(); + + U64 Bi, Yi; + + Ipp64u m0 = m[0]; + Ipp64u a0 = a[0]; + Ipp64u acc0 = 0; + + int i; + for (i=0; i<20; i++) { + Ipp64u t0, t1, t2, yi; + + Bi = set64((long long)b[i]); /* broadcast(b[i]) */ + /* compute yi */ + t0 = _mulx_u64(a0, b[i], &t2); /* (t2:t0) = acc0 + a[0]*b[i] */ + ADD104(t2, acc0, 0, t0) + yi = (acc0 * k0) & EXP_DIGIT_MASK_AVX512; /* yi = acc0*k0 */ + Yi = set64((long long)yi); + + t0 = _mulx_u64(m0, yi, &t1); /* (t1:t0) = m0*yi */ + ADD104(t2, acc0, t1, t0) /* (t2:acc0) += (t1:t0) */ + acc0 = SHRD52(t2, acc0); + + fma52x8lo_mem(R0, R0, Bi, a, 64*0) + fma52x8lo_mem(R1, R1, Bi, a, 64*1) + fma52x8lo_mem_len(R2, R2, Bi, a, 64*2, 4) + fma52x8lo_mem(R0, R0, Yi, m, 64*0) + fma52x8lo_mem(R1, R1, Yi, m, 64*1) + fma52x8lo_mem_len(R2, R2, Yi, m, 64*2, 4) + + shift64_imm(R0, R0h, 1) + shift64_imm(R0h, R1, 1) + shift64_imm(R1, R1h, 1) + shift64_imm(R1h, R2, 1) + shift64_imm(R2, get_zero64(), 1) + + /* "shift" R */ + t0 = get64(R0, 0); + acc0 += t0; + + /* U = A*Bi (hi) */ + fma52x8hi_mem(R0, R0, Bi, a, 64*0) + fma52x8hi_mem(R1, R1, Bi, a, 64*1) + fma52x8hi_mem_len(R2, R2, Bi, a, 64*2, 4) + /* R += M*Yi (hi) */ + fma52x8hi_mem(R0, R0, Yi, m, 64*0) + fma52x8hi_mem(R1, R1, Yi, m, 64*1) + fma52x8hi_mem_len(R2, R2, Yi, m, 64*2, 4) + } + + /* Set R0[0] == acc0 */ + Bi = set64((long long)acc0); + R0 = blend64(R0, Bi, 1); + + NORMALIZE_52x20(R0, R1, R2) + + storeu64(out + 0*4, R0); + storeu64(out + 1*4, R0h); + storeu64(out + 2*4, R1); + storeu64(out + 3*4, R1h); + storeu64(out + 4*4, R2); +} + +#if 0 +/* + * Almost Montgomery Multiplication in 2^52-radix + * + * Data represented as 20-qwords arrays in 2^52-radix. + * + * Note: |a| and |m| shall be zero-padded on 32 bytes (see R2h usage). + * this allows to save on alignr's. + * + */ +IPP_OWN_DEFN(void, ifma256_amm52x20, (Ipp64u out[20], + const Ipp64u a [24], /* 32-byte zero-padded */ + const Ipp64u b [20], + const Ipp64u m [24], /* 32-byte zero-padded */ + Ipp64u k0)) +{ + /* R0, R1, R2 holds result */ + U64 R0 = get_zero64(); + U64 R1 = get_zero64(); + U64 R2 = get_zero64(); + + /* High part of 512bit result in 256bit mode */ + U64 R0h = get_zero64(); + U64 R1h = get_zero64(); + U64 R2h = get_zero64(); + + U64 Bi, Yi; + + Ipp64u m0 = m[0]; + Ipp64u a0 = a[0]; + Ipp64u acc0 = 0; + + int i; + for(i=0; i<20; i+=4) { + Ipp64u t0, t1, t2, yi; + + /* =================================== */ + Bi = set64((long long)b[i]); /* broadcast(b[i]) */ + /* compute yi */ + t0 = _mulx_u64(a0, b[i], &t2); /* (t2:t0) = acc0 + a[0]*b[i] */ + ADD104(t2,acc0, 0,t0) + yi = (acc0 * k0) & EXP_DIGIT_MASK_AVX512; /* yi = acc0*k0 */ + Yi = set64((long long)yi); + + t0 = _mulx_u64(m0, yi, &t1); /* (t1:t0) = m0*yi */ + ADD104(t2,acc0, t1,t0) /* (t2:acc0) += (t1:t0) */ + acc0 = SHRD52(t2, acc0); + + fma52x8lo_mem(R0, R0, Bi, a, 64*0-8*0) + fma52x8lo_mem(R1, R1, Bi, a, 64*1-8*0) + fma52x8lo_mem_len(R2, R2, Bi, a, 64*2-8*0, 4) + fma52x8lo_mem(R0, R0, Yi, m, 64*0-8*0) + fma52x8lo_mem(R1, R1, Yi, m, 64*1-8*0) + fma52x8lo_mem_len(R2, R2, Yi, m, 64*2-8*0, 4) + + /* "shift" R */ + t0 = get64(R0, 1); + acc0 += t0; + + fma52x8hi_mem(R0, R0, Bi, a,64*0-8*1) /* U = A*Bi (hi) */ + fma52x8hi_mem(R1, R1, Bi, a,64*1-8*1) + fma52x8hi_mem(R2, R2, Bi, a,64*2-8*1) + fma52x8hi_mem(R0, R0, Yi, m,64*0-8*1) /* R += M*Yi (hi) */ + fma52x8hi_mem(R1, R1, Yi, m,64*1-8*1) + fma52x8hi_mem(R2, R2, Yi, m,64*2-8*1) + + /* =================================== */ + Bi = set64((long long)b[i+1]); /* broadcast(b[i+1]) */ + /* compute yi */ + t0 = _mulx_u64(a0, b[i+1], &t2); /* (t2:t0) = acc0 + a[0]*b[i+1] */ + ADD104(t2,acc0, 0,t0) + yi = (acc0 * k0) & EXP_DIGIT_MASK_AVX512; /* yi = acc0*k0 */ + Yi = set64((long long)yi); + + t0 = _mulx_u64(m0, yi, &t1); /* (t1:t0) = m0*yi */ + ADD104(t2,acc0, t1,t0) /* (t2:acc0) += (t1:t0) */ + acc0 = SHRD52(t2, acc0); + + fma52x8lo_mem(R0, R0, Bi, a,64*0-8*1) + fma52x8lo_mem(R1, R1, Bi, a,64*1-8*1) + fma52x8lo_mem(R2, R2, Bi, a,64*2-8*1) + fma52x8lo_mem(R0, R0, Yi, m,64*0-8*1) + fma52x8lo_mem(R1, R1, Yi, m,64*1-8*1) + fma52x8lo_mem(R2, R2, Yi, m,64*2-8*1) + + /* "shift" R */ + t0 = get64(R0, 2); + acc0 += t0; + + fma52x8hi_mem(R0, R0, Bi, a,64*0-8*2) /* U = A*Bi (hi) */ + fma52x8hi_mem(R1, R1, Bi, a,64*1-8*2) + fma52x8hi_mem(R2, R2, Bi, a,64*2-8*2) + fma52x8hi_mem(R0, R0, Yi, m,64*0-8*2) /* R += M*Yi (hi) */ + fma52x8hi_mem(R1, R1, Yi, m,64*1-8*2) + fma52x8hi_mem(R2, R2, Yi, m,64*2-8*2) + + /* =================================== */ + Bi = set64((long long)b[i+2]); /* broadcast(b[i+2]) */ + /* compute yi */ + t0 = _mulx_u64(a0, b[i+2], &t2); /* (t2:t0) = acc0 + a[0]*b[i+2] */ + ADD104(t2,acc0, 0,t0) + yi = (acc0 * k0) & EXP_DIGIT_MASK_AVX512; /* yi = acc0*k0 */ + Yi = set64((long long)yi); + + t0 = _mulx_u64(m0, yi, &t1); /* (t1:t0) = m0*yi */ + ADD104(t2,acc0, t1,t0) /* (t2:acc0) += (t1:t0) */ + acc0 = SHRD52(t2, acc0); + + fma52x8lo_mem(R0, R0, Bi, a,64*0-8*2) + fma52x8lo_mem(R1, R1, Bi, a,64*1-8*2) + fma52x8lo_mem(R2, R2, Bi, a,64*2-8*2) + fma52x8lo_mem(R0, R0, Yi, m,64*0-8*2) + fma52x8lo_mem(R1, R1, Yi, m,64*1-8*2) + fma52x8lo_mem(R2, R2, Yi, m,64*2-8*2) + + /* "shift" R */ + t0 = get64(R0, 3); + acc0 += t0; + + fma52x8hi_mem(R0, R0, Bi, a,64*0-8*3) /* U = A*Bi (hi) */ + fma52x8hi_mem(R1, R1, Bi, a,64*1-8*3) + fma52x8hi_mem(R2, R2, Bi, a,64*2-8*3) + fma52x8hi_mem(R0, R0, Yi, m,64*0-8*3) /* R += M*Yi (hi) */ + fma52x8hi_mem(R1, R1, Yi, m,64*1-8*3) + fma52x8hi_mem(R2, R2, Yi, m,64*2-8*3) + + /* =================================== */ + Bi = set64((long long)b[i+3]); /* broadcast(b[i+3]) */ + /* compute yi */ + t0 = _mulx_u64(a0, b[i+3], &t2); /* (t2:t0) = acc0 + a[0]*b[i+3] */ + ADD104(t2,acc0, 0,t0) + yi = (acc0 * k0) & EXP_DIGIT_MASK_AVX512; /* yi = acc0*k0 */ + Yi = set64((long long)yi); + + t0 = _mulx_u64(m0, yi, &t1); /* (t1:t0) = m0*yi */ + ADD104(t2,acc0, t1,t0) /* (t2:acc0) += (t1:t0) */ + acc0 = SHRD52(t2, acc0); + + fma52x8lo_mem(R0, R0, Bi, a,64*0-8*3) + fma52x8lo_mem(R1, R1, Bi, a,64*1-8*3) + fma52x8lo_mem(R2, R2, Bi, a,64*2-8*3) + fma52x8lo_mem(R0, R0, Yi, m,64*0-8*3) + fma52x8lo_mem(R1, R1, Yi, m,64*1-8*3) + fma52x8lo_mem(R2, R2, Yi, m,64*2-8*3) + + /* shift R */ + shift64(R0, R1) + shift64(R1, R2) + shift64(R2, get_zero64()) + + t0 = get64(R0, 0); + acc0 += t0; + + fma52x8hi_mem(R0, R0, Bi, a, 64*0-8*0) /* U = A*Bi (hi) */ + fma52x8hi_mem(R1, R1, Bi, a, 64*1-8*0) + fma52x8hi_mem_len(R2, R2, Bi, a, 64*2-8*0, 4) + fma52x8hi_mem(R0, R0, Yi, m, 64*0-8*0) /* R += M*Yi (hi) */ + fma52x8hi_mem(R1, R1, Yi, m, 64*1-8*0) + fma52x8hi_mem_len(R2, R2, Yi, m, 64*2-8*0, 4) + } + + /* set up R0.0 == acc0 */ + Bi = set64((long long)acc0); + R0 = blend64(R0, Bi, 1); + + NORMALIZE_52x20(R0, R1, R2) + + storeu64(out + 0*4, R0); + storeu64(out + 1*4, R0h); + storeu64(out + 2*4, R1); + storeu64(out + 3*4, R1h); + storeu64(out + 4*4, R2); +} +#endif + +IPP_OWN_DEFN(void, ifma256_ams52x20, (Ipp64u out[20], + const Ipp64u a [20], + const Ipp64u m [20], + Ipp64u k0)) +{ + ifma256_amm52x20(out, a, a, m, k0); +} + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_amm52x20_dual.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_amm52x20_dual.c new file mode 100644 index 0000000..a620771 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_amm52x20_dual.c @@ -0,0 +1,182 @@ + /******************************************************************************* + * Copyright (C) 2019 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owncp.h" + +#if(_IPP32E>=_IPP32E_K1) + +#include "pcpngmontexpstuff_avx512.h" + +#include "ifma_norm52x.h" +#include "ifma_math_avx512vl.h" +#include "ifma_rsa_arith.h" + +#define ADD104(rh, rl, ih, il) { \ + rl += il; \ + rh += ih; \ + rh += (rl>52U) | rh<<(64U-52U)) + +/* + * Almost Montgomery Multiplication in 2^52-radix + * + * Implements two independent interleaved multiplications. + * Data represented as 2 20-qwords arrays in 2^52-radix. + * + */ +IPP_OWN_DEFN(void, ifma256_amm52x20_dual, (Ipp64u out[2][20], + const Ipp64u a [2][20], + const Ipp64u b [2][20], + const Ipp64u m [2][20], + const Ipp64u k0 [2])) +{ + const Ipp64u *a_0 = a[0]; + const Ipp64u *b_0 = b[0]; + const Ipp64u *m_0 = m[0]; + Ipp64u m0_0 = m_0[0]; + Ipp64u a0_0 = a_0[0]; + Ipp64u acc0_0 = 0; + U64 R0_0, R1_0, R2_0, R0_0h, R1_0h, R2_0h; + R0_0 = R1_0 = R2_0 = R0_0h = R1_0h = R2_0h = get_zero64(); + + const Ipp64u *a_1 = a[1]; + const Ipp64u *b_1 = b[1]; + const Ipp64u *m_1 = m[1]; + Ipp64u m0_1 = m_1[0]; + Ipp64u a0_1 = a_1[0]; + Ipp64u acc0_1 = 0; + U64 R0_1, R1_1, R2_1, R0_1h, R1_1h, R2_1h; + R0_1 = R1_1 = R2_1 = R0_1h = R1_1h = R2_1h = get_zero64(); + + int i; + for (i=0; i<20; i++) { + { + Ipp64u t0, t1, t2, yi; + U64 Bi = set64((long long)b_0[i]); /* broadcast(b[i]) */ + /* compute yi */ + t0 = _mulx_u64(a0_0, b_0[i], &t2); /* (t2:t0) = acc0 + a[0]*b[i] */ + ADD104(t2, acc0_0, 0, t0) + yi = (acc0_0 * k0[0]) & EXP_DIGIT_MASK_AVX512; /* yi = acc0*k0 */ + U64 Yi = set64((long long)yi); + + t0 = _mulx_u64(m0_0, yi, &t1); /* (t1:t0) = m0*yi */ + ADD104(t2, acc0_0, t1, t0) /* (t2:acc0) += (t1:t0) */ + acc0_0 = SHRD52(t2, acc0_0); + + fma52x8lo_mem(R0_0, R0_0, Bi, a_0, 64*0) + fma52x8lo_mem(R1_0, R1_0, Bi, a_0, 64*1) + fma52x8lo_mem_len(R2_0, R2_0, Bi, a_0, 64*2, 4) + fma52x8lo_mem(R0_0, R0_0, Yi, m_0, 64*0) + fma52x8lo_mem(R1_0, R1_0, Yi, m_0, 64*1) + fma52x8lo_mem_len(R2_0, R2_0, Yi, m_0, 64*2, 4) + + shift64_imm(R0_0, R0_0h, 1) + shift64_imm(R0_0h, R1_0, 1) + shift64_imm(R1_0, R1_0h, 1) + shift64_imm(R1_0h, R2_0, 1) + shift64_imm(R2_0, get_zero64(), 1) + + /* "shift" R */ + t0 = get64(R0_0, 0); + acc0_0 += t0; + + /* U = A*Bi (hi) */ + fma52x8hi_mem(R0_0, R0_0, Bi, a_0, 64*0) + fma52x8hi_mem(R1_0, R1_0, Bi, a_0, 64*1) + fma52x8hi_mem_len(R2_0, R2_0, Bi, a_0, 64*2, 4) + /* R += M*Yi (hi) */ + fma52x8hi_mem(R0_0, R0_0, Yi, m_0, 64*0) + fma52x8hi_mem(R1_0, R1_0, Yi, m_0, 64*1) + fma52x8hi_mem_len(R2_0, R2_0, Yi, m_0, 64*2, 4) + } + { + Ipp64u t0, t1, t2, yi; + U64 Bi = set64((long long)b_1[i]); /* broadcast(b[i]) */ + /* compute yi */ + t0 = _mulx_u64(a0_1, b_1[i], &t2); /* (t2:t0) = acc0 + a[0]*b[i] */ + ADD104(t2, acc0_1, 0, t0) + yi = (acc0_1 * k0[1]) & EXP_DIGIT_MASK_AVX512; /* yi = acc0*k0 */ + U64 Yi = set64((long long)yi); + + t0 = _mulx_u64(m0_1, yi, &t1); /* (t1:t0) = m0*yi */ + ADD104(t2, acc0_1, t1, t0) /* (t2:acc0) += (t1:t0) */ + acc0_1 = SHRD52(t2, acc0_1); + + fma52x8lo_mem(R0_1, R0_1, Bi, a_1, 64*0) + fma52x8lo_mem(R1_1, R1_1, Bi, a_1, 64*1) + fma52x8lo_mem_len(R2_1, R2_1, Bi, a_1, 64*2, 4) + fma52x8lo_mem(R0_1, R0_1, Yi, m_1, 64*0) + fma52x8lo_mem(R1_1, R1_1, Yi, m_1, 64*1) + fma52x8lo_mem_len(R2_1, R2_1, Yi, m_1, 64*2, 4) + + shift64_imm(R0_1, R0_1h, 1) + shift64_imm(R0_1h, R1_1, 1) + shift64_imm(R1_1, R1_1h, 1) + shift64_imm(R1_1h, R2_1, 1) + shift64_imm(R2_1, get_zero64(), 1) + + /* "shift" R */ + t0 = get64(R0_1, 0); + acc0_1 += t0; + + /* U = A*Bi (hi) */ + fma52x8hi_mem(R0_1, R0_1, Bi, a_1, 64*0) + fma52x8hi_mem(R1_1, R1_1, Bi, a_1, 64*1) + fma52x8hi_mem_len(R2_1, R2_1, Bi, a_1, 64*2, 4) + /* R += M*Yi (hi) */ + fma52x8hi_mem(R0_1, R0_1, Yi, m_1, 64*0) + fma52x8hi_mem(R1_1, R1_1, Yi, m_1, 64*1) + fma52x8hi_mem_len(R2_1, R2_1, Yi, m_1, 64*2, 4) + } + } + { + /* Normalize and store idx=0 */ + /* Set R0.0 == acc0 */ + U64 Bi = set64((long long)acc0_0); + R0_0 = blend64(R0_0, Bi, 1); + NORMALIZE_52x20(R0_0, R1_0, R2_0) + storeu64(out[0] + 0*4, R0_0); + storeu64(out[0] + 1*4, R0_0h); + storeu64(out[0] + 2*4, R1_0); + storeu64(out[0] + 3*4, R1_0h); + storeu64(out[0] + 4*4, R2_0); + } + { + /* Normalize and store idx=1 */ + /* Set R0.1 == acc1 */ + U64 Bi = set64((long long)acc0_1); + R0_1 = blend64(R0_1, Bi, 1); + NORMALIZE_52x20(R0_1, R1_1, R2_1) + storeu64(out[1] + 0*4, R0_1); + storeu64(out[1] + 1*4, R0_1h); + storeu64(out[1] + 2*4, R1_1); + storeu64(out[1] + 3*4, R1_1h); + storeu64(out[1] + 4*4, R2_1); + } +} + +IPP_OWN_DEFN(void, ifma256_ams52x20_dual, (Ipp64u out[2][20], + const Ipp64u a [2][20], + const Ipp64u m [2][20], + const Ipp64u k0 [2])) +{ + ifma256_amm52x20_dual(out, a, a, m, k0); +} + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_amm52x30.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_amm52x30.c new file mode 100644 index 0000000..57c57f9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_amm52x30.c @@ -0,0 +1,142 @@ + /******************************************************************************* + * Copyright (C) 2019 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owncp.h" + +#if(_IPP32E>=_IPP32E_K1) + +#include "pcptool.h" + +#include "pcpngmontexpstuff_avx512.h" + +#include "ifma_norm52x.h" +#include "ifma_math_avx512vl.h" +#include "ifma_rsa_arith.h" + +#define ADD104(rh, rl, ih, il) { \ + rl += il; \ + rh += ih; \ + rh += (rl>52U) | rh<<(64U-52U)) + +/* + * Almost Montgomery Multiplication in 2^52-radix + * + * Data represented as (30+2)-qwords arrays in 2^52-radix. + * + * Note: 2 high qwords - zero padding. + * + */ +IPP_OWN_DEFN(void, ifma256_amm52x30, (Ipp64u out[32], + const Ipp64u a [32], + const Ipp64u b [32], + const Ipp64u m [32], + Ipp64u k0)) +{ + U64 R0 = get_zero64(); + U64 R1 = get_zero64(); + U64 R2 = get_zero64(); + U64 R3 = get_zero64(); + + U64 R0h = get_zero64(); + U64 R1h = get_zero64(); + U64 R2h = get_zero64(); + U64 R3h = get_zero64(); + + U64 Bi, Yi; + + Ipp64u m0 = m[0]; + Ipp64u a0 = a[0]; + Ipp64u acc0 = 0; + + int i; + for (i=0; i<30; i++) { + Ipp64u t0, t1, t2, yi; + + Bi = set64((long long)b[i]); /* broadcast(b[i]) */ + /* compute yi */ + t0 = _mulx_u64(a0, b[i], &t2); /* (t2:t0) = acc0 + a[0]*b[i] */ + ADD104(t2, acc0, 0, t0) + yi = (acc0 * k0) & EXP_DIGIT_MASK_AVX512; /* yi = acc0*k0 */ + Yi = set64((long long)yi); + + t0 = _mulx_u64(m0, yi, &t1); /* (t1:t0) = m0*yi */ + ADD104(t2, acc0, t1, t0) /* (t2:acc0) += (t1:t0) */ + acc0 = SHRD52(t2, acc0); + + fma52x8lo_mem(R0, R0, Bi, a, 64*0) + fma52x8lo_mem(R1, R1, Bi, a, 64*1) + fma52x8lo_mem(R2, R2, Bi, a, 64*2) + fma52x8lo_mem(R3, R3, Bi, a, 64*3) + + fma52x8lo_mem(R0, R0, Yi, m, 64*0) + fma52x8lo_mem(R1, R1, Yi, m, 64*1) + fma52x8lo_mem(R2, R2, Yi, m, 64*2) + fma52x8lo_mem(R3, R3, Yi, m, 64*3) + + shift64_imm(R0, R0h, 1) + shift64_imm(R0h, R1, 1) + shift64_imm(R1, R1h, 1) + shift64_imm(R1h, R2, 1) + shift64_imm(R2, R2h, 1) + shift64_imm(R2h, R3, 1) + shift64_imm(R3, R3h, 1) + shift64_imm(R3h, get_zero64(), 1) + + /* "shift" R */ + t0 = get64(R0, 0); + acc0 += t0; + + /* U = A*Bi (hi) */ + fma52x8hi_mem(R0, R0, Bi, a, 64*0) + fma52x8hi_mem(R1, R1, Bi, a, 64*1) + fma52x8hi_mem(R2, R2, Bi, a, 64*2) + fma52x8hi_mem(R3, R3, Bi, a, 64*3) + /* R += M*Yi (hi) */ + fma52x8hi_mem(R0, R0, Yi, m, 64*0) + fma52x8hi_mem(R1, R1, Yi, m, 64*1) + fma52x8hi_mem(R2, R2, Yi, m, 64*2) + fma52x8hi_mem(R3, R3, Yi, m, 64*3) + } + + /* Set R0[0] == acc0 */ + Bi = set64((long long)acc0); + R0 = blend64(R0, Bi, 1); + + NORMALIZE_52x30(R0, R1, R2, R3) + + storeu64(out + 0*4, R0); + storeu64(out + 1*4, R0h); + storeu64(out + 2*4, R1); + storeu64(out + 3*4, R1h); + storeu64(out + 4*4, R2); + storeu64(out + 5*4, R2h); + storeu64(out + 6*4, R3); + storeu64(out + 7*4, R3h); +} + +IPP_OWN_DEFN(void, ifma256_ams52x30, (Ipp64u out[32], + const Ipp64u a [32], + const Ipp64u m [32], + Ipp64u k0)) +{ + ifma256_amm52x30(out, a, a, m, k0); +} + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_amm52x30_dual.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_amm52x30_dual.c new file mode 100644 index 0000000..338387f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_amm52x30_dual.c @@ -0,0 +1,210 @@ + /******************************************************************************* + * Copyright (C) 2019 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owncp.h" + +#if(_IPP32E>=_IPP32E_K1) + +#include "pcpngmontexpstuff_avx512.h" + +#include "ifma_math_avx512vl.h" +#include "ifma_norm52x.h" +#include "ifma_rsa_arith.h" + +#define ADD104(rh, rl, ih, il) { \ + rl += il; \ + rh += ih; \ + rh += (rl>52U) | rh<<(64U-52U)) + +/* + * Almost Montgomery Multiplication in 2^52-radix + * + * Implements two independent interleaved multiplications. + * Data represented as 2 (30+2)-qwords arrays in 2^52-radix. + * + * Note: 2 high qwords - zero padding. + * + */ +IPP_OWN_DEFN(void, ifma256_amm52x30_dual, (Ipp64u out[2][32], + const Ipp64u a [2][32], + const Ipp64u b [2][32], + const Ipp64u m [2][32], + const Ipp64u k0 [2])) +{ + const Ipp64u *a_0 = a[0]; + const Ipp64u *b_0 = b[0]; + const Ipp64u *m_0 = m[0]; + Ipp64u m0_0 = m_0[0]; + Ipp64u a0_0 = a_0[0]; + Ipp64u acc0_0 = 0; + U64 R0_0, R1_0, R2_0, R3_0, R0_0h, R1_0h, R2_0h, R3_0h; + R0_0 = R1_0 = R2_0 = R3_0 = R0_0h = R1_0h = R2_0h = R3_0h = get_zero64(); + + const Ipp64u *a_1 = a[1]; + const Ipp64u *b_1 = b[1]; + const Ipp64u *m_1 = m[1]; + Ipp64u m0_1 = m_1[0]; + Ipp64u a0_1 = a_1[0]; + Ipp64u acc0_1 = 0; + U64 R0_1, R1_1, R2_1, R3_1, R0_1h, R1_1h, R2_1h, R3_1h; + R0_1 = R1_1 = R2_1 = R3_1 = R0_1h = R1_1h = R2_1h = R3_1h = get_zero64(); + + int i; + for (i=0; i<30; i++) { + { + Ipp64u t0, t1, t2, yi; + U64 Bi = set64((long long)b_0[i]); /* broadcast(b[i]) */ + /* compute yi */ + t0 = _mulx_u64(a0_0, b_0[i], &t2); /* (t2:t0) = acc0 + a[0]*b[i] */ + ADD104(t2, acc0_0, 0, t0) + yi = (acc0_0 * k0[0]) & EXP_DIGIT_MASK_AVX512; /* yi = acc0*k0 */ + U64 Yi = set64((long long)yi); + + t0 = _mulx_u64(m0_0, yi, &t1); /* (t1:t0) = m0*yi */ + ADD104(t2, acc0_0, t1, t0) /* (t2:acc0) += (t1:t0) */ + acc0_0 = SHRD52(t2, acc0_0); + + fma52x8lo_mem(R0_0, R0_0, Bi, a_0, 64*0) + fma52x8lo_mem(R1_0, R1_0, Bi, a_0, 64*1) + fma52x8lo_mem(R2_0, R2_0, Bi, a_0, 64*2) + fma52x8lo_mem(R3_0, R3_0, Bi, a_0, 64*3) + + fma52x8lo_mem(R0_0, R0_0, Yi, m_0, 64*0) + fma52x8lo_mem(R1_0, R1_0, Yi, m_0, 64*1) + fma52x8lo_mem(R2_0, R2_0, Yi, m_0, 64*2) + fma52x8lo_mem(R3_0, R3_0, Yi, m_0, 64*3) + + shift64_imm(R0_0, R0_0h, 1) + shift64_imm(R0_0h, R1_0, 1) + shift64_imm(R1_0, R1_0h, 1) + shift64_imm(R1_0h, R2_0, 1) + shift64_imm(R2_0, R2_0h, 1) + shift64_imm(R2_0h, R3_0, 1) + shift64_imm(R3_0, R3_0h, 1) + shift64_imm(R3_0h, get_zero64(), 1) + + /* "shift" R */ + t0 = get64(R0_0, 0); + acc0_0 += t0; + + /* U = A*Bi (hi) */ + fma52x8hi_mem(R0_0, R0_0, Bi, a_0, 64*0) + fma52x8hi_mem(R1_0, R1_0, Bi, a_0, 64*1) + fma52x8hi_mem(R2_0, R2_0, Bi, a_0, 64*2) + fma52x8hi_mem(R3_0, R3_0, Bi, a_0, 64*3) + /* R += M*Yi (hi) */ + fma52x8hi_mem(R0_0, R0_0, Yi, m_0, 64*0) + fma52x8hi_mem(R1_0, R1_0, Yi, m_0, 64*1) + fma52x8hi_mem(R2_0, R2_0, Yi, m_0, 64*2) + fma52x8hi_mem(R3_0, R3_0, Yi, m_0, 64*3) + } + { + Ipp64u t0, t1, t2, yi; + U64 Bi = set64((long long)b_1[i]); /* broadcast(b[i]) */ + /* compute yi */ + t0 = _mulx_u64(a0_1, b_1[i], &t2); /* (t2:t0) = acc0 + a[0]*b[i] */ + ADD104(t2, acc0_1, 0, t0) + yi = (acc0_1 * k0[1]) & EXP_DIGIT_MASK_AVX512; /* yi = acc0*k0 */ + U64 Yi = set64((long long)yi); + + t0 = _mulx_u64(m0_1, yi, &t1); /* (t1:t0) = m0*yi */ + ADD104(t2, acc0_1, t1, t0) /* (t2:acc0) += (t1:t0) */ + acc0_1 = SHRD52(t2, acc0_1); + + fma52x8lo_mem(R0_1, R0_1, Bi, a_1, 64*0) + fma52x8lo_mem(R1_1, R1_1, Bi, a_1, 64*1) + fma52x8lo_mem(R2_1, R2_1, Bi, a_1, 64*2) + fma52x8lo_mem(R3_1, R3_1, Bi, a_1, 64*3) + + fma52x8lo_mem(R0_1, R0_1, Yi, m_1, 64*0) + fma52x8lo_mem(R1_1, R1_1, Yi, m_1, 64*1) + fma52x8lo_mem(R2_1, R2_1, Yi, m_1, 64*2) + fma52x8lo_mem(R3_1, R3_1, Yi, m_1, 64*3) + + shift64_imm(R0_1, R0_1h, 1) + shift64_imm(R0_1h, R1_1, 1) + shift64_imm(R1_1, R1_1h, 1) + shift64_imm(R1_1h, R2_1, 1) + shift64_imm(R2_1, R2_1h, 1) + shift64_imm(R2_1h, R3_1, 1) + shift64_imm(R3_1, R3_1h, 1) + shift64_imm(R3_1h, get_zero64(), 1) + + /* "shift" R */ + t0 = get64(R0_1, 0); + acc0_1 += t0; + + /* U = A*Bi (hi) */ + fma52x8hi_mem(R0_1, R0_1, Bi, a_1, 64*0) + fma52x8hi_mem(R1_1, R1_1, Bi, a_1, 64*1) + fma52x8hi_mem(R2_1, R2_1, Bi, a_1, 64*2) + fma52x8hi_mem(R3_1, R3_1, Bi, a_1, 64*3) + /* R += M*Yi (hi) */ + fma52x8hi_mem(R0_1, R0_1, Yi, m_1, 64*0) + fma52x8hi_mem(R1_1, R1_1, Yi, m_1, 64*1) + fma52x8hi_mem(R2_1, R2_1, Yi, m_1, 64*2) + fma52x8hi_mem(R3_1, R3_1, Yi, m_1, 64*3) + } + } + { + /* Normalize and store idx=0 */ + /* Set R0.0 == acc0 */ + U64 Bi = set64((long long)acc0_0); + R0_0 = blend64(R0_0, Bi, 1); + + NORMALIZE_52x30(R0_0, R1_0, R2_0, R3_0) + + storeu64(out[0] + 0*4, R0_0); + storeu64(out[0] + 1*4, R0_0h); + storeu64(out[0] + 2*4, R1_0); + storeu64(out[0] + 3*4, R1_0h); + storeu64(out[0] + 4*4, R2_0); + storeu64(out[0] + 5*4, R2_0h); + storeu64(out[0] + 6*4, R3_0); + storeu64(out[0] + 7*4, R3_0h); + } + { + /* Normalize and store idx=1 */ + /* Set R0.1 == acc1 */ + U64 Bi = set64((long long)acc0_1); + R0_1 = blend64(R0_1, Bi, 1); + + NORMALIZE_52x30(R0_1, R1_1, R2_1, R3_1) + + storeu64(out[1] + 0*4, R0_1); + storeu64(out[1] + 1*4, R0_1h); + storeu64(out[1] + 2*4, R1_1); + storeu64(out[1] + 3*4, R1_1h); + storeu64(out[1] + 4*4, R2_1); + storeu64(out[1] + 5*4, R2_1h); + storeu64(out[1] + 6*4, R3_1); + storeu64(out[1] + 7*4, R3_1h); + } +} + +IPP_OWN_DEFN(void, ifma256_ams52x30_dual, (Ipp64u out[2][32], + const Ipp64u a [2][32], + const Ipp64u m [2][32], + const Ipp64u k0 [2])) +{ + ifma256_amm52x30_dual(out, a, a, m, k0); +} + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_amm52x40.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_amm52x40.c new file mode 100644 index 0000000..a6ca19d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_amm52x40.c @@ -0,0 +1,150 @@ + /******************************************************************************* + * Copyright (C) 2019 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owncp.h" + +#if(_IPP32E>=_IPP32E_K1) + +#include "pcptool.h" + +#include "pcpngmontexpstuff_avx512.h" + +#include "ifma_norm52x.h" +#include "ifma_math_avx512vl.h" +#include "ifma_rsa_arith.h" + +#define ADD104(rh, rl, ih, il) { \ + rl += il; \ + rh += ih; \ + rh += (rl>52U) | rh<<(64U-52U)) + +/* + * Almost Montgomery Multiplication in 2^52-radix + * + * Data represented as 40-qwords arrays in 2^52-radix. + * + */ +IPP_OWN_DEFN(void, ifma256_amm52x40, (Ipp64u out[40], + const Ipp64u a [40], + const Ipp64u b [40], + const Ipp64u m [40], + Ipp64u k0)) +{ + U64 R0 = get_zero64(); + U64 R1 = get_zero64(); + U64 R2 = get_zero64(); + U64 R3 = get_zero64(); + U64 R4 = get_zero64(); + + U64 R0h = get_zero64(); + U64 R1h = get_zero64(); + U64 R2h = get_zero64(); + U64 R3h = get_zero64(); + U64 R4h = get_zero64(); + + U64 Bi, Yi; + + Ipp64u m0 = m[0]; + Ipp64u a0 = a[0]; + Ipp64u acc0 = 0; + + int i; + for (i=0; i<40; i++) { + Ipp64u t0, t1, t2, yi; + + Bi = set64((long long)b[i]); /* broadcast(b[i]) */ + /* compute yi */ + t0 = _mulx_u64(a0, b[i], &t2); /* (t2:t0) = acc0 + a[0]*b[i] */ + ADD104(t2, acc0, 0, t0) + yi = (acc0 * k0) & EXP_DIGIT_MASK_AVX512; /* yi = acc0*k0 */ + Yi = set64((long long)yi); + + t0 = _mulx_u64(m0, yi, &t1); /* (t1:t0) = m0*yi */ + ADD104(t2, acc0, t1, t0) /* (t2:acc0) += (t1:t0) */ + acc0 = SHRD52(t2, acc0); + + fma52x8lo_mem(R0, R0, Bi, a, 64*0) + fma52x8lo_mem(R1, R1, Bi, a, 64*1) + fma52x8lo_mem(R2, R2, Bi, a, 64*2) + fma52x8lo_mem(R3, R3, Bi, a, 64*3) + fma52x8lo_mem(R4, R4, Bi, a, 64*4) + + fma52x8lo_mem(R0, R0, Yi, m, 64*0) + fma52x8lo_mem(R1, R1, Yi, m, 64*1) + fma52x8lo_mem(R2, R2, Yi, m, 64*2) + fma52x8lo_mem(R3, R3, Yi, m, 64*3) + fma52x8lo_mem(R4, R4, Yi, m, 64*4) + + shift64_imm(R0, R0h, 1) + shift64_imm(R0h, R1, 1) + shift64_imm(R1, R1h, 1) + shift64_imm(R1h, R2, 1) + shift64_imm(R2, R2h, 1) + shift64_imm(R2h, R3, 1) + shift64_imm(R3, R3h, 1) + shift64_imm(R3h, R4, 1) + shift64_imm(R4, R4h, 1) + shift64_imm(R4h, get_zero64(), 1) + + /* "shift" R */ + t0 = get64(R0, 0); + acc0 += t0; + + /* U = A*Bi (hi) */ + fma52x8hi_mem(R0, R0, Bi, a, 64*0) + fma52x8hi_mem(R1, R1, Bi, a, 64*1) + fma52x8hi_mem(R2, R2, Bi, a, 64*2) + fma52x8hi_mem(R3, R3, Bi, a, 64*3) + fma52x8hi_mem(R4, R4, Bi, a, 64*4) + /* R += M*Yi (hi) */ + fma52x8hi_mem(R0, R0, Yi, m, 64*0) + fma52x8hi_mem(R1, R1, Yi, m, 64*1) + fma52x8hi_mem(R2, R2, Yi, m, 64*2) + fma52x8hi_mem(R3, R3, Yi, m, 64*3) + fma52x8hi_mem(R4, R4, Yi, m, 64*4) + } + + /* Set R0[0] == acc0 */ + Bi = set64((long long)acc0); + R0 = blend64(R0, Bi, 1); + + NORMALIZE_52x40(R0, R1, R2, R3, R4) + + storeu64(out + 0*4, R0); + storeu64(out + 1*4, R0h); + storeu64(out + 2*4, R1); + storeu64(out + 3*4, R1h); + storeu64(out + 4*4, R2); + storeu64(out + 5*4, R2h); + storeu64(out + 6*4, R3); + storeu64(out + 7*4, R3h); + storeu64(out + 8*4, R4); + storeu64(out + 9*4, R4h); +} + +IPP_OWN_DEFN(void, ifma256_ams52x40, (Ipp64u out[40], + const Ipp64u a [40], + const Ipp64u m [40], + Ipp64u k0)) +{ + ifma256_amm52x40(out, a, a, m, k0); +} + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_amm52x40_dual.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_amm52x40_dual.c new file mode 100644 index 0000000..f45c54d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_amm52x40_dual.c @@ -0,0 +1,224 @@ + /******************************************************************************* + * Copyright (C) 2019 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owncp.h" + +#if(_IPP32E>=_IPP32E_K1) + +#include "pcpngmontexpstuff_avx512.h" + +#include "ifma_math_avx512vl.h" +#include "ifma_norm52x.h" +#include "ifma_rsa_arith.h" + +#define ADD104(rh, rl, ih, il) { \ + rl += il; \ + rh += ih; \ + rh += (rl>52U) | rh<<(64U-52U)) + +/* + * Almost Montgomery Multiplication in 2^52-radix + * + * Implements two independent interleaved multiplications. + * Data represented as 2 40-qwords arrays in 2^52-radix. + * + */ +IPP_OWN_DEFN(void, ifma256_amm52x40_dual, (Ipp64u out[2][40], + const Ipp64u a [2][40], + const Ipp64u b [2][40], + const Ipp64u m [2][40], + const Ipp64u k0 [2])) +{ + const Ipp64u *a_0 = a[0]; + const Ipp64u *b_0 = b[0]; + const Ipp64u *m_0 = m[0]; + Ipp64u m0_0 = m_0[0]; + Ipp64u a0_0 = a_0[0]; + Ipp64u acc0_0 = 0; + U64 R0_0, R1_0, R2_0, R3_0, R4_0, R0_0h, R1_0h, R2_0h, R3_0h, R4_0h; + R0_0 = R1_0 = R2_0 = R3_0 = R4_0 = R0_0h = R1_0h = R2_0h = R3_0h = R4_0h = get_zero64(); + + const Ipp64u *a_1 = a[1]; + const Ipp64u *b_1 = b[1]; + const Ipp64u *m_1 = m[1]; + Ipp64u m0_1 = m_1[0]; + Ipp64u a0_1 = a_1[0]; + Ipp64u acc0_1 = 0; + U64 R0_1, R1_1, R2_1, R3_1, R4_1, R0_1h, R1_1h, R2_1h, R3_1h, R4_1h; + R0_1 = R1_1 = R2_1 = R3_1 = R4_1 = R0_1h = R1_1h = R2_1h = R3_1h = R4_1h = get_zero64(); + + int i; + for (i=0; i<40; i++) { + { + Ipp64u t0, t1, t2, yi; + U64 Bi = set64((long long)b_0[i]); /* broadcast(b[i]) */ + /* compute yi */ + t0 = _mulx_u64(a0_0, b_0[i], &t2); /* (t2:t0) = acc0 + a[0]*b[i] */ + ADD104(t2, acc0_0, 0, t0) + yi = (acc0_0 * k0[0]) & EXP_DIGIT_MASK_AVX512; /* yi = acc0*k0 */ + U64 Yi = set64((long long)yi); + + t0 = _mulx_u64(m0_0, yi, &t1); /* (t1:t0) = m0*yi */ + ADD104(t2, acc0_0, t1, t0) /* (t2:acc0) += (t1:t0) */ + acc0_0 = SHRD52(t2, acc0_0); + + fma52x8lo_mem(R0_0, R0_0, Bi, a_0, 64*0) + fma52x8lo_mem(R1_0, R1_0, Bi, a_0, 64*1) + fma52x8lo_mem(R2_0, R2_0, Bi, a_0, 64*2) + fma52x8lo_mem(R3_0, R3_0, Bi, a_0, 64*3) + fma52x8lo_mem(R4_0, R4_0, Bi, a_0, 64*4) + + fma52x8lo_mem(R0_0, R0_0, Yi, m_0, 64*0) + fma52x8lo_mem(R1_0, R1_0, Yi, m_0, 64*1) + fma52x8lo_mem(R2_0, R2_0, Yi, m_0, 64*2) + fma52x8lo_mem(R3_0, R3_0, Yi, m_0, 64*3) + fma52x8lo_mem(R4_0, R4_0, Yi, m_0, 64*4) + + shift64_imm(R0_0, R0_0h, 1) + shift64_imm(R0_0h, R1_0, 1) + shift64_imm(R1_0, R1_0h, 1) + shift64_imm(R1_0h, R2_0, 1) + shift64_imm(R2_0, R2_0h, 1) + shift64_imm(R2_0h, R3_0, 1) + shift64_imm(R3_0, R3_0h, 1) + shift64_imm(R3_0h, R4_0, 1) + shift64_imm(R4_0, R4_0h, 1) + shift64_imm(R4_0h, get_zero64(), 1) + + /* "shift" R */ + t0 = get64(R0_0, 0); + acc0_0 += t0; + + /* U = A*Bi (hi) */ + fma52x8hi_mem(R0_0, R0_0, Bi, a_0, 64*0) + fma52x8hi_mem(R1_0, R1_0, Bi, a_0, 64*1) + fma52x8hi_mem(R2_0, R2_0, Bi, a_0, 64*2) + fma52x8hi_mem(R3_0, R3_0, Bi, a_0, 64*3) + fma52x8hi_mem(R4_0, R4_0, Bi, a_0, 64*4) + /* R += M*Yi (hi) */ + fma52x8hi_mem(R0_0, R0_0, Yi, m_0, 64*0) + fma52x8hi_mem(R1_0, R1_0, Yi, m_0, 64*1) + fma52x8hi_mem(R2_0, R2_0, Yi, m_0, 64*2) + fma52x8hi_mem(R3_0, R3_0, Yi, m_0, 64*3) + fma52x8hi_mem(R4_0, R4_0, Yi, m_0, 64*4) + } + { + Ipp64u t0, t1, t2, yi; + U64 Bi = set64((long long)b_1[i]); /* broadcast(b[i]) */ + /* compute yi */ + t0 = _mulx_u64(a0_1, b_1[i], &t2); /* (t2:t0) = acc0 + a[0]*b[i] */ + ADD104(t2, acc0_1, 0, t0) + yi = (acc0_1 * k0[1]) & EXP_DIGIT_MASK_AVX512; /* yi = acc0*k0 */ + U64 Yi = set64((long long)yi); + + t0 = _mulx_u64(m0_1, yi, &t1); /* (t1:t0) = m0*yi */ + ADD104(t2, acc0_1, t1, t0) /* (t2:acc0) += (t1:t0) */ + acc0_1 = SHRD52(t2, acc0_1); + + fma52x8lo_mem(R0_1, R0_1, Bi, a_1, 64*0) + fma52x8lo_mem(R1_1, R1_1, Bi, a_1, 64*1) + fma52x8lo_mem(R2_1, R2_1, Bi, a_1, 64*2) + fma52x8lo_mem(R3_1, R3_1, Bi, a_1, 64*3) + fma52x8lo_mem(R4_1, R4_1, Bi, a_1, 64*4) + + fma52x8lo_mem(R0_1, R0_1, Yi, m_1, 64*0) + fma52x8lo_mem(R1_1, R1_1, Yi, m_1, 64*1) + fma52x8lo_mem(R2_1, R2_1, Yi, m_1, 64*2) + fma52x8lo_mem(R3_1, R3_1, Yi, m_1, 64*3) + fma52x8lo_mem(R4_1, R4_1, Yi, m_1, 64*4) + + shift64_imm(R0_1, R0_1h, 1) + shift64_imm(R0_1h, R1_1, 1) + shift64_imm(R1_1, R1_1h, 1) + shift64_imm(R1_1h, R2_1, 1) + shift64_imm(R2_1, R2_1h, 1) + shift64_imm(R2_1h, R3_1, 1) + shift64_imm(R3_1, R3_1h, 1) + shift64_imm(R3_1h, R4_1, 1) + shift64_imm(R4_1, R4_1h, 1) + shift64_imm(R4_1h, get_zero64(), 1) + + /* "shift" R */ + t0 = get64(R0_1, 0); + acc0_1 += t0; + + /* U = A*Bi (hi) */ + fma52x8hi_mem(R0_1, R0_1, Bi, a_1, 64*0) + fma52x8hi_mem(R1_1, R1_1, Bi, a_1, 64*1) + fma52x8hi_mem(R2_1, R2_1, Bi, a_1, 64*2) + fma52x8hi_mem(R3_1, R3_1, Bi, a_1, 64*3) + fma52x8hi_mem(R4_1, R4_1, Bi, a_1, 64*4) + /* R += M*Yi (hi) */ + fma52x8hi_mem(R0_1, R0_1, Yi, m_1, 64*0) + fma52x8hi_mem(R1_1, R1_1, Yi, m_1, 64*1) + fma52x8hi_mem(R2_1, R2_1, Yi, m_1, 64*2) + fma52x8hi_mem(R3_1, R3_1, Yi, m_1, 64*3) + fma52x8hi_mem(R4_1, R4_1, Yi, m_1, 64*4) + } + } + { + /* Normalize and store idx=0 */ + /* Set R0.0 == acc0 */ + U64 Bi = set64((long long)acc0_0); + R0_0 = blend64(R0_0, Bi, 1); + + NORMALIZE_52x40(R0_0, R1_0, R2_0, R3_0, R4_0) + + storeu64(out[0] + 0*4, R0_0); + storeu64(out[0] + 1*4, R0_0h); + storeu64(out[0] + 2*4, R1_0); + storeu64(out[0] + 3*4, R1_0h); + storeu64(out[0] + 4*4, R2_0); + storeu64(out[0] + 5*4, R2_0h); + storeu64(out[0] + 6*4, R3_0); + storeu64(out[0] + 7*4, R3_0h); + storeu64(out[0] + 8*4, R4_0); + storeu64(out[0] + 9*4, R4_0h); + } + { + /* Normalize and store idx=1 */ + /* Set R0.1 == acc1 */ + U64 Bi = set64((long long)acc0_1); + R0_1 = blend64(R0_1, Bi, 1); + + NORMALIZE_52x40(R0_1, R1_1, R2_1, R3_1, R4_1) + + storeu64(out[1] + 0*4, R0_1); + storeu64(out[1] + 1*4, R0_1h); + storeu64(out[1] + 2*4, R1_1); + storeu64(out[1] + 3*4, R1_1h); + storeu64(out[1] + 4*4, R2_1); + storeu64(out[1] + 5*4, R2_1h); + storeu64(out[1] + 6*4, R3_1); + storeu64(out[1] + 7*4, R3_1h); + storeu64(out[1] + 8*4, R4_1); + storeu64(out[1] + 9*4, R4_1h); + } +} + +IPP_OWN_DEFN(void, ifma256_ams52x40_dual, (Ipp64u out[2][40], + const Ipp64u a [2][40], + const Ipp64u m [2][40], + const Ipp64u k0 [2])) +{ + ifma256_amm52x40_dual(out, a, a, m, k0); +} + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_exp52x20.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_exp52x20.c new file mode 100644 index 0000000..61c5394 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_exp52x20.c @@ -0,0 +1,161 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "owncp.h" + +#if(_IPP32E>=_IPP32E_K1) + +#include "pcptool.h" + +#include "pcpngmontexpstuff_avx512.h" +#include "ifma_math_avx512vl.h" +#include "ifma_rsa_arith.h" + +#define BITSIZE_MODULUS (1024) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,52)) // 20 +#define LEN64 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,64)) // 16 + +#define EXP_WIN_SIZE (5U) +#define EXP_WIN_MASK ((1U<> exp_chunk_shift; + + extract_multiplier(red_Y, (const Ipp64u(*)[LEN52])red_table, (int)red_table_idx); + + /* process other exp windows */ + for (exp_bit_no -= EXP_WIN_SIZE; exp_bit_no >= 0; exp_bit_no -= EXP_WIN_SIZE) { + /* series of squaring */ + AMS(red_Y, red_Y, modulus, k0); + AMS(red_Y, red_Y, modulus, k0); + AMS(red_Y, red_Y, modulus, k0); + AMS(red_Y, red_Y, modulus, k0); + AMS(red_Y, red_Y, modulus, k0); + + /* extract pre-computed multiplier from the table */ + { + Ipp64u T; + exp_chunk_no = exp_bit_no / 64; + exp_chunk_shift = exp_bit_no % 64; + + red_table_idx = expz[exp_chunk_no]; + T = expz[exp_chunk_no+1]; + + red_table_idx = red_table_idx >> exp_chunk_shift; + T = exp_chunk_shift == 0 ? 0 : T << (64 - exp_chunk_shift); + red_table_idx = (red_table_idx ^ T) & table_idx_mask; + + extract_multiplier(red_X, (const Ipp64u(*)[LEN52])red_table, (int)red_table_idx); + AMM(red_Y, red_Y, red_X, modulus, k0); + } + } + } + + /* clear exponents */ + PurgeBlock((Ipp64u*)expz, (LEN64+1)*(int)sizeof(Ipp64u)); + + /* convert result back in regular 2^52 domain */ + ZEXPAND_BNU(red_X, 0, LEN52); + red_X[0] = 1ULL; + AMM(out, red_Y, red_X, modulus, k0); +} + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_exp52x20_dual.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_exp52x20_dual.c new file mode 100644 index 0000000..19599e7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_exp52x20_dual.c @@ -0,0 +1,185 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "owncp.h" + +#if(_IPP32E>=_IPP32E_K1) + +#include "pcptool.h" + +#include "pcpngmontexpstuff_avx512.h" +#include "ifma_math_avx512vl.h" +#include "ifma_rsa_arith.h" + +#define BITSIZE_MODULUS (1024) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,52)) // 20 +#define LEN64 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,64)) // 16 + +#define EXP_WIN_SIZE (5U) +#define EXP_WIN_MASK ((1U<> exp_chunk_shift; + red_table_idx_1 = red_table_idx_1 >> exp_chunk_shift; + + extract_multiplier_n(red_Y[0], (const Ipp64u(*)[2][LEN52])red_table, (int)red_table_idx_0, 0); + extract_multiplier_n(red_Y[1], (const Ipp64u(*)[2][LEN52])red_table, (int)red_table_idx_1, 1); + + /* process other exp windows */ + for (exp_bit_no -= EXP_WIN_SIZE; exp_bit_no >= 0; exp_bit_no -= EXP_WIN_SIZE) { + /* extract pre-computed multiplier from the table */ + { + Ipp64u T; + exp_chunk_no = exp_bit_no / 64; + exp_chunk_shift = exp_bit_no % 64; + { + red_table_idx_0 = expz[0][exp_chunk_no]; + T = expz[0][exp_chunk_no + 1]; + + red_table_idx_0 = red_table_idx_0 >> exp_chunk_shift; + T = exp_chunk_shift == 0 ? 0 : T << (64 - exp_chunk_shift); + red_table_idx_0 = (red_table_idx_0 ^ T) & table_idx_mask; + + extract_multiplier_n(red_X[0], (const Ipp64u(*)[2][LEN52])red_table, (int)red_table_idx_0, 0); + } + { + red_table_idx_1 = expz[1][exp_chunk_no]; + T = expz[1][exp_chunk_no + 1]; + + red_table_idx_1 = red_table_idx_1 >> exp_chunk_shift; + T = exp_chunk_shift == 0 ? 0 : T << (64 - exp_chunk_shift); + red_table_idx_1 = (red_table_idx_1 ^ T) & table_idx_mask; + + extract_multiplier_n(red_X[1], (const Ipp64u(*)[2][LEN52])red_table, (int)red_table_idx_1, 1); + } + + } + + /* series of squaring */ + DAMS(red_Y, (const Ipp64u(*)[LEN52])red_Y, modulus, k0); + DAMS(red_Y, (const Ipp64u(*)[LEN52])red_Y, modulus, k0); + DAMS(red_Y, (const Ipp64u(*)[LEN52])red_Y, modulus, k0); + DAMS(red_Y, (const Ipp64u(*)[LEN52])red_Y, modulus, k0); + DAMS(red_Y, (const Ipp64u(*)[LEN52])red_Y, modulus, k0); + + DAMM(red_Y, (const Ipp64u(*)[LEN52])red_Y, (const Ipp64u(*)[LEN52])red_X, modulus, k0); + } + } + + /* clear exponents */ + PurgeBlock((Ipp64u*)expz, 2*(LEN64+1)*(int)sizeof(Ipp64u)); + + /* convert result back in regular 2^52 domain */ + ZEXPAND_BNU((Ipp64u*)red_X, 0, 2*LEN52); + storeu64(&red_X[0][0], _mm256_setr_epi64x(1,0,0,0)); + storeu64(&red_X[1][0], _mm256_setr_epi64x(1,0,0,0)); + DAMM(out, (const Ipp64u(*)[LEN52])red_Y, (const Ipp64u(*)[LEN52])red_X, modulus, k0); + + PurgeBlock((Ipp64u*)red_Y, 2*LEN52*(int)sizeof(Ipp64u)); +} + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_exp52x30_dual.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_exp52x30_dual.c new file mode 100644 index 0000000..22e9951 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_exp52x30_dual.c @@ -0,0 +1,190 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "owncp.h" + +#if(_IPP32E>=_IPP32E_K1) + +#include "pcptool.h" + +#include "pcpngmontexpstuff_avx512.h" +#include "ifma_math_avx512vl.h" +#include "ifma_rsa_arith.h" + +#define BITSIZE_MODULUS (1536) +#define LEN52 ((NUMBER_OF_DIGITS(BITSIZE_MODULUS,52)) + 2) // 30 + 2 (with zero-padding) +#define LEN64 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,64)) // 24 + +#define EXP_WIN_SIZE (5U) +#define EXP_WIN_MASK ((1U<> exp_chunk_shift; + red_table_idx_1 = red_table_idx_1 >> exp_chunk_shift; + + extract_multiplier_n(red_Y[0], (const Ipp64u(*)[2][LEN52])red_table, (int)red_table_idx_0, 0); + extract_multiplier_n(red_Y[1], (const Ipp64u(*)[2][LEN52])red_table, (int)red_table_idx_1, 1); + + /* process other exp windows */ + for (exp_bit_no -= EXP_WIN_SIZE; exp_bit_no >= 0; exp_bit_no -= EXP_WIN_SIZE) { + /* extract pre-computed multiplier from the table */ + { + Ipp64u T; + exp_chunk_no = exp_bit_no / 64; + exp_chunk_shift = exp_bit_no % 64; + { + red_table_idx_0 = expz[0][exp_chunk_no]; + T = expz[0][exp_chunk_no + 1]; + + red_table_idx_0 = red_table_idx_0 >> exp_chunk_shift; + T = exp_chunk_shift == 0 ? 0 : T << (64 - exp_chunk_shift); + red_table_idx_0 = (red_table_idx_0 ^ T) & table_idx_mask; + + extract_multiplier_n(red_X[0], (const Ipp64u(*)[2][LEN52])red_table, (int)red_table_idx_0, 0); + } + { + red_table_idx_1 = expz[1][exp_chunk_no]; + T = expz[1][exp_chunk_no + 1]; + + red_table_idx_1 = red_table_idx_1 >> exp_chunk_shift; + T = exp_chunk_shift == 0 ? 0 : T << (64 - exp_chunk_shift); + red_table_idx_1 = (red_table_idx_1 ^ T) & table_idx_mask; + + extract_multiplier_n(red_X[1], (const Ipp64u(*)[2][LEN52])red_table, (int)red_table_idx_1, 1); + } + + } + + /* series of squaring */ + DAMS(red_Y, (const Ipp64u(*)[LEN52])red_Y, modulus, k0); + DAMS(red_Y, (const Ipp64u(*)[LEN52])red_Y, modulus, k0); + DAMS(red_Y, (const Ipp64u(*)[LEN52])red_Y, modulus, k0); + DAMS(red_Y, (const Ipp64u(*)[LEN52])red_Y, modulus, k0); + DAMS(red_Y, (const Ipp64u(*)[LEN52])red_Y, modulus, k0); + + DAMM(red_Y, (const Ipp64u(*)[LEN52])red_Y, (const Ipp64u(*)[LEN52])red_X, modulus, k0); + } + } + + /* clear exponents */ + PurgeBlock((Ipp64u*)expz, 2*(LEN64+1)*(int)sizeof(Ipp64u)); + + /* convert result back in regular 2^52 domain */ + ZEXPAND_BNU((Ipp64u*)red_X, 0, 2*LEN52); + storeu64(&red_X[0][0], _mm256_setr_epi64x(1,0,0,0)); + storeu64(&red_X[1][0], _mm256_setr_epi64x(1,0,0,0)); + DAMM(out, (const Ipp64u(*)[LEN52])red_Y, (const Ipp64u(*)[LEN52])red_X, modulus, k0); + + PurgeBlock((Ipp64u*)red_Y, 2*LEN52*(int)sizeof(Ipp64u)); +} + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_exp52x40_dual.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_exp52x40_dual.c new file mode 100644 index 0000000..5b20e5d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_exp52x40_dual.c @@ -0,0 +1,195 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "owncp.h" + +#if(_IPP32E>=_IPP32E_K1) + +#include "pcptool.h" + +#include "pcpngmontexpstuff_avx512.h" +#include "ifma_math_avx512vl.h" +#include "ifma_rsa_arith.h" + +#define BITSIZE_MODULUS (2048) +#define LEN52 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,52)) // 40 +#define LEN64 (NUMBER_OF_DIGITS(BITSIZE_MODULUS,64)) // 32 + +#define EXP_WIN_SIZE (5U) +#define EXP_WIN_MASK ((1U<> exp_chunk_shift; + red_table_idx_1 = red_table_idx_1 >> exp_chunk_shift; + + extract_multiplier_n(red_Y[0], (const Ipp64u(*)[2][LEN52])red_table, (int)red_table_idx_0, 0); + extract_multiplier_n(red_Y[1], (const Ipp64u(*)[2][LEN52])red_table, (int)red_table_idx_1, 1); + + /* process other exp windows */ + for (exp_bit_no -= EXP_WIN_SIZE; exp_bit_no >= 0; exp_bit_no -= EXP_WIN_SIZE) { + /* extract pre-computed multiplier from the table */ + { + Ipp64u T; + exp_chunk_no = exp_bit_no / 64; + exp_chunk_shift = exp_bit_no % 64; + { + red_table_idx_0 = expz[0][exp_chunk_no]; + T = expz[0][exp_chunk_no + 1]; + + red_table_idx_0 = red_table_idx_0 >> exp_chunk_shift; + T = exp_chunk_shift == 0 ? 0 : T << (64 - exp_chunk_shift); + red_table_idx_0 = (red_table_idx_0 ^ T) & table_idx_mask; + + extract_multiplier_n(red_X[0], (const Ipp64u(*)[2][LEN52])red_table, (int)red_table_idx_0, 0); + } + { + red_table_idx_1 = expz[1][exp_chunk_no]; + T = expz[1][exp_chunk_no + 1]; + + red_table_idx_1 = red_table_idx_1 >> exp_chunk_shift; + T = exp_chunk_shift == 0 ? 0 : T << (64 - exp_chunk_shift); + red_table_idx_1 = (red_table_idx_1 ^ T) & table_idx_mask; + + extract_multiplier_n(red_X[1], (const Ipp64u(*)[2][LEN52])red_table, (int)red_table_idx_1, 1); + } + + } + + /* series of squaring */ + DAMS(red_Y, (const Ipp64u(*)[LEN52])red_Y, modulus, k0); + DAMS(red_Y, (const Ipp64u(*)[LEN52])red_Y, modulus, k0); + DAMS(red_Y, (const Ipp64u(*)[LEN52])red_Y, modulus, k0); + DAMS(red_Y, (const Ipp64u(*)[LEN52])red_Y, modulus, k0); + DAMS(red_Y, (const Ipp64u(*)[LEN52])red_Y, modulus, k0); + + DAMM(red_Y, (const Ipp64u(*)[LEN52])red_Y, (const Ipp64u(*)[LEN52])red_X, modulus, k0); + } + } + + /* clear exponents */ + PurgeBlock((Ipp64u*)expz, 2*(LEN64+1)*(int)sizeof(Ipp64u)); + + /* convert result back in regular 2^52 domain */ + ZEXPAND_BNU((Ipp64u*)red_X, 0, 2*LEN52); + storeu64(&red_X[0][0], _mm256_setr_epi64x(1,0,0,0)); + storeu64(&red_X[1][0], _mm256_setr_epi64x(1,0,0,0)); + DAMM(out, (const Ipp64u(*)[LEN52])red_Y, (const Ipp64u(*)[LEN52])red_X, modulus, k0); + + PurgeBlock((Ipp64u*)red_Y, 2*LEN52*(int)sizeof(Ipp64u)); +} + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_exp52x_dual.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_exp52x_dual.c new file mode 100644 index 0000000..6e33fa2 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_exp52x_dual.c @@ -0,0 +1,136 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "owncp.h" + +#if(_IPP32E>=_IPP32E_K1) + +#include "pcptool.h" + +#include "pcpngmontexpstuff_avx512.h" +#include "ifma_rsa_arith.h" + +IPP_OWN_DEFN (cpSize, gsMontDualExpWinBuffer_avx512, (int modulusBits)) +{ + cpSize redNum = numofVariable_avx512(modulusBits); + /* Reg (ymm) capacity = 4 qwords */ + cpSize redBufferNum = numofVariableBuff_avx512(redNum, 4); + return redBufferNum * 2 * 8; +} + +IPP_OWN_FUNPTR (void, AMM52, (Ipp64u *out, const Ipp64u *a, const Ipp64u *b, const Ipp64u *m, Ipp64u k0)) +IPP_OWN_FUNPTR (void, DEXP52, (Ipp64u* out, const Ipp64u* base, const Ipp64u* exp[2], const Ipp64u* modulus, const Ipp64u* pMont, const Ipp64u k0[2])) + +IPP_OWN_DEFN (cpSize, gsMontDualExpWin_BNU_sscm_avx512, (BNU_CHUNK_T* dataY[2], + const BNU_CHUNK_T* dataX[2], + cpSize nsX[2], + const BNU_CHUNK_T* dataE[2], + gsModEngine* pMont[2], + BNU_CHUNK_T* pBuffer)) +{ + const BNU_CHUNK_T* modulus[2] = {0}; + const BNU_CHUNK_T* rr[2] = {0}; + cpSize modulusSize[2] = {0}; + BNU_CHUNK_T k0[2] = {0}; + + modulus[0] = MOD_MODULUS(pMont[0]); + modulus[1] = MOD_MODULUS(pMont[1]); + modulusSize[0] = MOD_LEN(pMont[0]); + modulusSize[1] = MOD_LEN(pMont[1]); + rr[0] = MOD_MNT_R2(pMont[0]); + rr[1] = MOD_MNT_R2(pMont[1]); + k0[0] = MOD_MNT_FACTOR(pMont[0]); + k0[1] = MOD_MNT_FACTOR(pMont[1]); + + /* + * Dual expo implemenentation assumes that bit sizes of P and Q are + * the same, so query size from the first. + */ + int modulusBitSize = BITSIZE_BNU(modulus[0], modulusSize[0]); + int redLen = NUMBER_OF_DIGITS(modulusBitSize + 2, EXP_DIGIT_SIZE_AVX512); + /* For ymm-based implementation reg capacity = 4 qwords */ + int redBufferLen = numofVariableBuff_avx512(redLen, 4); + + /* Allocate buffers */ + BNU_CHUNK_T* redX = pBuffer; + BNU_CHUNK_T* redM = redX + 2*redBufferLen; + BNU_CHUNK_T* redRR = redM + 2*redBufferLen; + BNU_CHUNK_T* redCoeff = redRR + 2*redBufferLen; + BNU_CHUNK_T* redBuffer = redCoeff + redBufferLen; + + AMM52 ammFunc = NULL; + DEXP52 dexpFunc = NULL; + switch (modulusBitSize) { + case 1024: + ammFunc = ifma256_amm52x20; + dexpFunc = (DEXP52)ifma256_exp52x20_dual; + break; + case 1536: + ammFunc = ifma256_amm52x30; + dexpFunc = (DEXP52)ifma256_exp52x30_dual; + break; + case 2048: + ammFunc = ifma256_amm52x40; + dexpFunc = (DEXP52)ifma256_exp52x40_dual; + break; + default: + /* Other modulus sizes not supported. This function shall not be called for them. */ + return 0; + } + + ZEXPAND_BNU(redCoeff, 0, redBufferLen); + int conv_coeff = 4 * (EXP_DIGIT_SIZE_AVX512 * redLen - modulusBitSize); + /* Set corresponding bit in reduced domain */ + SET_BIT(redCoeff, 64 * (int)(conv_coeff / 52) + conv_coeff % 52); + + for (int i = 0; i < 2; i++) { + /* Convert base into redundant domain */ + ZEXPAND_COPY_BNU(redBuffer, redBufferLen, dataX[i], nsX[i]); + regular_dig52(redX + i*redBufferLen, redBufferLen, redBuffer, modulusBitSize); + + /* Convert modulus into redundant domain */ + ZEXPAND_COPY_BNU(redBuffer, redBufferLen, modulus[i], modulusSize[i]); + regular_dig52(redM + i*redBufferLen, redBufferLen, redBuffer, modulusBitSize); + + /* + * Compute target domain Montgomery converter RR' based on original domain RR. + * + * Example: modlen = 1024: RR = 2^2048 mod m, RR' = 2^2080 mod m + * conv_coeff = 2^64 + * (1st amm): 2^2048*2^2048/2^1040= 2^3056 mod m + * (2nd amm): 2^3056*2^64/2^1040 = 2^2080 mod m + * + */ + ZEXPAND_COPY_BNU(redBuffer, redBufferLen, rr[i], modulusSize[i]); + regular_dig52(redRR + i*redBufferLen, redBufferLen, redBuffer, modulusBitSize); + ammFunc(redRR + i*redBufferLen, redRR + i*redBufferLen, redRR + i*redBufferLen, redM + i*redBufferLen, k0[i]); + ammFunc(redRR + i*redBufferLen, redRR + i*redBufferLen, redCoeff, redM + i*redBufferLen, k0[i]); + } + + dexpFunc(redRR, redX, dataE, redM, redRR, k0); + + /* Convert result back to regular domain */ + for (int i = 0; i < 2; i++) + dig52_regular(dataY[i], redRR + i*redBufferLen, modulusBitSize); + + /* Clear redundant exponents buffer */ + PurgeBlock(redRR, 2*redBufferLen*(int)sizeof(BNU_CHUNK_T)); + + return (modulusSize[0] + modulusSize[1]); +} + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_math_avx512vl.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_math_avx512vl.h new file mode 100644 index 0000000..f7a950c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_math_avx512vl.h @@ -0,0 +1,208 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "owncp.h" + +#ifndef IFMA_MATH_AVX512VL_H +#define IFMA_MATH_AVX512VL_H + +/* + * This header provides low-level abstraction for 256-bit AVX512VL + * ISA instructions. + * + */ + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) // for MSVC + #pragma warning(disable:4101) + #pragma warning(disable:4127) // warning: conditional expression is constant (see fma52x8lo_mem_len()) +#elif defined (__INTEL_COMPILER) + #pragma warning(disable:177) +#endif + +#include + +#ifndef SIMD_LEN +#define SIMD_LEN 256 +#endif + +#define SIMD_TYPE(LEN) typedef __m ## LEN ## i U64; + +#if (SIMD_LEN == 256) + SIMD_TYPE(256) + #define SIMD_BYTES (SIMD_LEN/8) + #define SIMD_QWORDS (SIMD_LEN/64) + + __INLINE U64 loadu64(const void *p) { + return _mm256_loadu_si256((U64*)p); + } + + __INLINE void storeu64(const void *p, U64 v) { + _mm256_storeu_si256((U64*)p, v); + } + + #define set64 _mm256_set1_epi64x + + #ifdef __GNUC__ + static U64 fma52lo(U64 a, U64 b, U64 c) + { + __asm__ ( "vpmadd52luq %2, %1, %0" : "+x" (a): "x" (b), "x" (c) ); + return a; + } + + static U64 fma52hi(U64 a, U64 b, U64 c) + { + __asm__ ( "vpmadd52huq %2, %1, %0" : "+x" (a): "x" (b), "x" (c) ); + return a; + } + + #define _mm_madd52lo_epu64_(r, a, b, c, o) \ + { \ + r=a; \ + __asm__ ( "vpmadd52luq " #o "(%2), %1, %0" : "+x" (r): "x" (b), "r" (c) ); \ + } + + #define _mm_madd52hi_epu64_(r, a, b, c, o) \ + { \ + r=a; \ + __asm__ ( "vpmadd52huq " #o "(%2), %1, %0" : "+x" (r): "x" (b), "r" (c) ); \ + } + #else + /* Use IFMA instrinsics for all other compilers */ + static U64 fma52lo(U64 a, U64 b, U64 c) + { + return _mm256_madd52lo_epu64(a, b, c); + } + + static U64 fma52hi(U64 a, U64 b, U64 c) + { + return _mm256_madd52hi_epu64(a, b, c); + } + + #define _mm_madd52lo_epu64_(r, a, b, c, o) \ + { \ + r=fma52lo(a, b, loadu64((U64*)(((char*)c)+o))); \ + } + + #define _mm_madd52hi_epu64_(r, a, b, c, o) \ + { \ + r=fma52hi(a, b, loadu64((U64*)(((char*)c)+o))); \ + } + #endif + + __INLINE U64 mul52lo(U64 b, U64 c) + { + return fma52lo(_mm256_setzero_si256(), b, c); + } + + #define fma52lo_mem(r, a, b, c, o) _mm_madd52lo_epu64_(r, a, b, c, o) + #define fma52hi_mem(r, a, b, c, o) _mm_madd52hi_epu64_(r, a, b, c, o) + + __INLINE U64 add64(U64 a, U64 b) + { + return _mm256_add_epi64(a, b); + } + + __INLINE U64 sub64(U64 a, U64 b) + { + return _mm256_sub_epi64(a, b); + } + + __INLINE U64 get_zero64() + { + return _mm256_setzero_si256(); + } + + __INLINE void set_zero64(U64 *a) + { + *a = _mm256_xor_si256(*a, *a); + } + + __INLINE U64 set1(unsigned long long a) + { + return _mm256_set1_epi64x((long long)a); + } + + __INLINE U64 srli64(U64 a, int s) + { + return _mm256_srli_epi64(a, s); + } + + #define slli64 _mm256_slli_epi64 + + __INLINE U64 and64_const(U64 a, unsigned long long mask) + { + return _mm256_and_si256(a, _mm256_set1_epi64x((long long)mask)); + } + + __INLINE U64 and64(U64 a, U64 mask) + { + return _mm256_and_si256(a, mask); + } + + #define or64 _mm256_or_si256 + #define xor64 _mm256_xor_si256 + + static Ipp64u get64(U64 v, int idx) { + long long int res; + switch (idx) { + case 1: res = _mm256_extract_epi64(v, 1); break; + case 2: res = _mm256_extract_epi64(v, 2); break; + case 3: res = _mm256_extract_epi64(v, 3); break; + default: res = _mm256_extract_epi64(v, 0); + } + return (Ipp64u)res; + } + + #define fma52x8lo_mem(r, a, b, c, o) \ + fma52lo_mem(r, a, b, c, o); \ + fma52lo_mem(r ## h, a ## h, b, c, (o) + 32); + + #define fma52x8hi_mem(r, a, b, c, o) \ + fma52hi_mem(r, a, b, c, o); \ + fma52hi_mem(r ## h, a ## h, b, c, (o) + 32); + + #define fma52x8lo_mem_len(r, a, b, c, o, l) \ + fma52lo_mem(r, a, b, c, o); \ + if (l > 4) { fma52lo_mem(r ## h, a ## h, b, c, (o) + 32); } + + #define fma52x8hi_mem_len(r, a, b, c, o, l) \ + fma52hi_mem(r, a, b, c, o); \ + if (l > 4) { fma52hi_mem(r ## h, a ## h, b, c, (o) + 32); } + + #define fma52x8lo_mask_mem(r, m, a, b, c, o) \ + fma52lo_mem(r, a, b, c, o); \ + fma52lo_mem(r ## h, a ## h, b, c, (o) + 32); + + #define fma52x8hi_mask_mem(r, m, a, b, c, o) \ + fma52hi_mem(r, a, b, c, o); \ + fma52hi_mem(r ## h, a ## h, b, c, (o) + 32); + + #define shift64(R0, R1) { \ + R0 = R0 ## h; \ + R0 ## h = R1; } + + #define shift64_imm(R0, R1, imm) \ + R0 = _mm256_alignr_epi64(R1, R0, imm); + + #define blend64(a, b, m) \ + _mm256_blend_epi32(a, b, (int)(0x3<<((m-1)<<1))); + +#else + #error "Incorrect SIMD length" +#endif // SIMD_LEN + +#endif // IFMA_MATH_AVX512VL_H diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_norm52x.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_norm52x.h new file mode 100644 index 0000000..704aa46 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_norm52x.h @@ -0,0 +1,315 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "owncp.h" + +#ifndef IFMA_NORM_52X_H +#define IFMA_NORM_52X_H + +#define NORMALIZE_52x20(R0,R1,R2) { \ + __m256i T0 = _mm256_srli_epi64(R0, EXP_DIGIT_SIZE_AVX512); \ + __m256i T0h = _mm256_srli_epi64(R0 ## h, EXP_DIGIT_SIZE_AVX512); \ + __m256i T1 = _mm256_srli_epi64(R1, EXP_DIGIT_SIZE_AVX512); \ + __m256i T1h = _mm256_srli_epi64(R1 ## h, EXP_DIGIT_SIZE_AVX512); \ + __m256i T2 = _mm256_srli_epi64(R2, EXP_DIGIT_SIZE_AVX512); \ + __m256i MASK = _mm256_set1_epi64x(EXP_DIGIT_MASK_AVX512); \ + \ + Ipp32u kk0, kk1, kk2; \ + Ipp32u kk0h, kk1h; \ + kk0 = kk1 = kk2 = kk0h = kk1h = 0; \ + \ + R0 = _mm256_and_si256(R0, MASK); \ + R1 = _mm256_and_si256(R1, MASK); \ + R2 = _mm256_and_si256(R2, MASK); \ + R0 ## h = _mm256_and_si256(R0 ## h, MASK); \ + R1 ## h = _mm256_and_si256(R1 ## h, MASK); \ + \ + T2 = _mm256_alignr_epi64(T2, T1h, 3); \ + T1h = _mm256_alignr_epi64(T1h, T1, 3); \ + T1 = _mm256_alignr_epi64(T1, T0h, 3); \ + T0h = _mm256_alignr_epi64(T0h, T0, 3); \ + T0 = _mm256_alignr_epi64(T0, _mm256_setzero_si256(), 3); \ + \ + R0 = _mm256_add_epi64(R0, T0); \ + R1 = _mm256_add_epi64(R1, T1); \ + R2 = _mm256_add_epi64(R2, T2); \ + R0 ## h = _mm256_add_epi64(R0 ## h, T0h); \ + R1 ## h = _mm256_add_epi64(R1 ## h, T1h); \ + { \ + Ipp32u k,l; \ + k = l = 0; \ + \ + kk0 = _mm256_cmp_epu64_mask(MASK, R0, _MM_CMPINT_LT); \ + kk1 = _mm256_cmp_epu64_mask(MASK, R1, _MM_CMPINT_LT); \ + kk2 = _mm256_cmp_epu64_mask(MASK, R2, _MM_CMPINT_LT); \ + kk0h = _mm256_cmp_epu64_mask(MASK, R0 ## h, _MM_CMPINT_LT); \ + kk1h = _mm256_cmp_epu64_mask(MASK, R1 ## h, _MM_CMPINT_LT); \ + \ + k = (kk2<<16)|(kk1h<<12)|(kk1<<8)|(kk0h<<4)|kk0; \ + \ + kk0 = _mm256_cmp_epu64_mask(MASK, R0, _MM_CMPINT_EQ); \ + kk1 = _mm256_cmp_epu64_mask(MASK, R1, _MM_CMPINT_EQ); \ + kk2 = _mm256_cmp_epu64_mask(MASK, R2, _MM_CMPINT_EQ); \ + kk0h = _mm256_cmp_epu64_mask(MASK, R0 ## h, _MM_CMPINT_EQ); \ + kk1h = _mm256_cmp_epu64_mask(MASK, R1 ## h, _MM_CMPINT_EQ); \ + \ + l = (kk2<<16)|(kk1h<<12)|(kk1<<8)|(kk0h<<4)|kk0; \ + \ + k = l + 2*k; \ + k ^= l; \ + \ + kk0 = k; \ + kk0h = (k>>4); \ + kk1 = (k>>8); \ + kk1h = (k>>12); \ + kk2 = (k>>16); \ + } \ + \ + R0 = _mm256_mask_sub_epi64(R0, (__mmask8)kk0, R0, MASK); \ + R1 = _mm256_mask_sub_epi64(R1, (__mmask8)kk1, R1, MASK); \ + R2 = _mm256_mask_sub_epi64(R2, (__mmask8)kk2, R2, MASK); \ + R0 ## h = _mm256_mask_sub_epi64(R0 ## h, (__mmask8)kk0h, R0 ## h, MASK); \ + R1 ## h = _mm256_mask_sub_epi64(R1 ## h, (__mmask8)kk1h, R1 ## h, MASK); \ + \ + R0 = _mm256_and_si256(R0, MASK); \ + R1 = _mm256_and_si256(R1, MASK); \ + R2 = _mm256_and_si256(R2, MASK); \ + R0 ## h = _mm256_and_si256(R0 ## h, MASK); \ + R1 ## h = _mm256_and_si256(R1 ## h, MASK); \ +} + +#define NORMALIZE_52x30(R0,R1,R2,R3) { \ + __m256i T0 = _mm256_srli_epi64(R0, EXP_DIGIT_SIZE_AVX512); \ + __m256i T0h = _mm256_srli_epi64(R0 ## h, EXP_DIGIT_SIZE_AVX512); \ + __m256i T1 = _mm256_srli_epi64(R1, EXP_DIGIT_SIZE_AVX512); \ + __m256i T1h = _mm256_srli_epi64(R1 ## h, EXP_DIGIT_SIZE_AVX512); \ + __m256i T2 = _mm256_srli_epi64(R2, EXP_DIGIT_SIZE_AVX512); \ + __m256i T2h = _mm256_srli_epi64(R2 ## h, EXP_DIGIT_SIZE_AVX512); \ + __m256i T3 = _mm256_srli_epi64(R3, EXP_DIGIT_SIZE_AVX512); \ + __m256i T3h = _mm256_srli_epi64(R3 ## h, EXP_DIGIT_SIZE_AVX512); \ + __m256i MASK = _mm256_set1_epi64x(EXP_DIGIT_MASK_AVX512); \ + \ + Ipp32u kk0, kk1, kk2, kk3; \ + Ipp32u kk0h, kk1h, kk2h, kk3h; \ + kk0 = kk1 = kk2 = kk3 = kk0h = kk1h = kk2h = kk3h = 0; \ + \ + R0 = _mm256_and_si256(R0, MASK); \ + R1 = _mm256_and_si256(R1, MASK); \ + R2 = _mm256_and_si256(R2, MASK); \ + R3 = _mm256_and_si256(R3, MASK); \ + R0 ## h = _mm256_and_si256(R0 ## h, MASK); \ + R1 ## h = _mm256_and_si256(R1 ## h, MASK); \ + R2 ## h = _mm256_and_si256(R2 ## h, MASK); \ + R3 ## h = _mm256_and_si256(R3 ## h, MASK); \ + \ + T3h = _mm256_alignr_epi64(T3h, T3, 3); \ + T3 = _mm256_alignr_epi64(T3 , T2h, 3); \ + T2h = _mm256_alignr_epi64(T2h, T2, 3); \ + T2 = _mm256_alignr_epi64(T2 , T1h, 3); \ + T1h = _mm256_alignr_epi64(T1h, T1, 3); \ + T1 = _mm256_alignr_epi64(T1 , T0h, 3); \ + T0h = _mm256_alignr_epi64(T0h, T0, 3); \ + T0 = _mm256_alignr_epi64(T0, _mm256_setzero_si256(), 3); \ + \ + R0 = _mm256_add_epi64(R0, T0); \ + R1 = _mm256_add_epi64(R1, T1); \ + R2 = _mm256_add_epi64(R2, T2); \ + R3 = _mm256_add_epi64(R3, T3); \ + R0 ## h = _mm256_add_epi64(R0 ## h, T0h); \ + R1 ## h = _mm256_add_epi64(R1 ## h, T1h); \ + R2 ## h = _mm256_add_epi64(R2 ## h, T2h); \ + R3 ## h = _mm256_add_epi64(R3 ## h, T3h); \ + \ + { \ + Ipp32u k, l; \ + k = l = 0; \ + \ + kk0 = _mm256_cmp_epu64_mask(MASK, R0, _MM_CMPINT_LT); \ + kk1 = _mm256_cmp_epu64_mask(MASK, R1, _MM_CMPINT_LT); \ + kk2 = _mm256_cmp_epu64_mask(MASK, R2, _MM_CMPINT_LT); \ + kk3 = _mm256_cmp_epu64_mask(MASK, R3, _MM_CMPINT_LT); \ + kk0h = _mm256_cmp_epu64_mask(MASK, R0 ## h, _MM_CMPINT_LT); \ + kk1h = _mm256_cmp_epu64_mask(MASK, R1 ## h, _MM_CMPINT_LT); \ + kk2h = _mm256_cmp_epu64_mask(MASK, R2 ## h, _MM_CMPINT_LT); \ + kk3h = _mm256_cmp_epu64_mask(MASK, R3 ## h, _MM_CMPINT_LT); \ + \ + k = (kk3h<<28)|(kk3<<24)|(kk2h<<20)|(kk2<<16)|(kk1h<<12)|(kk1<<8)|(kk0h<<4)|kk0; \ + \ + kk0 = _mm256_cmp_epu64_mask(MASK, R0, _MM_CMPINT_EQ); \ + kk1 = _mm256_cmp_epu64_mask(MASK, R1, _MM_CMPINT_EQ); \ + kk2 = _mm256_cmp_epu64_mask(MASK, R2, _MM_CMPINT_EQ); \ + kk3 = _mm256_cmp_epu64_mask(MASK, R3, _MM_CMPINT_EQ); \ + kk0h = _mm256_cmp_epu64_mask(MASK, R0 ## h, _MM_CMPINT_EQ); \ + kk1h = _mm256_cmp_epu64_mask(MASK, R1 ## h, _MM_CMPINT_EQ); \ + kk2h = _mm256_cmp_epu64_mask(MASK, R2 ## h, _MM_CMPINT_EQ); \ + kk3h = _mm256_cmp_epu64_mask(MASK, R3 ## h, _MM_CMPINT_EQ); \ + \ + l = (kk3h<<28)|(kk3<<24)|(kk2h<<20)|(kk2<<16)|(kk1h<<12)|(kk1<<8)|(kk0h<<4)|kk0; \ + \ + k = l + 2*k; \ + k ^= l; \ + \ + kk0 = k; \ + kk0h = (k>>4); \ + kk1 = (k>>8); \ + kk1h = (k>>12); \ + kk2 = (k>>16); \ + kk2h = (k>>20); \ + kk3 = (k>>24); \ + kk3h = (k>>28); \ + } \ + \ + R0 = _mm256_mask_sub_epi64(R0, (__mmask8)kk0, R0, MASK); \ + R1 = _mm256_mask_sub_epi64(R1, (__mmask8)kk1, R1, MASK); \ + R2 = _mm256_mask_sub_epi64(R2, (__mmask8)kk2, R2, MASK); \ + R3 = _mm256_mask_sub_epi64(R3, (__mmask8)kk3, R3, MASK); \ + R0 ## h = _mm256_mask_sub_epi64(R0 ## h, (__mmask8)kk0h, R0 ## h, MASK); \ + R1 ## h = _mm256_mask_sub_epi64(R1 ## h, (__mmask8)kk1h, R1 ## h, MASK); \ + R2 ## h = _mm256_mask_sub_epi64(R2 ## h, (__mmask8)kk2h, R2 ## h, MASK); \ + R3 ## h = _mm256_mask_sub_epi64(R3 ## h, (__mmask8)kk3h, R3 ## h, MASK); \ + \ + R0 = _mm256_and_si256(R0, MASK); \ + R1 = _mm256_and_si256(R1, MASK); \ + R2 = _mm256_and_si256(R2, MASK); \ + R3 = _mm256_and_si256(R3, MASK); \ + R0 ## h = _mm256_and_si256(R0 ## h, MASK); \ + R1 ## h = _mm256_and_si256(R1 ## h, MASK); \ + R2 ## h = _mm256_and_si256(R2 ## h, MASK); \ + R3 ## h = _mm256_and_si256(R3 ## h, MASK); \ +} + +#define NORMALIZE_52x40(R0,R1,R2,R3,R4) { \ + __m256i T0 = _mm256_srli_epi64(R0, EXP_DIGIT_SIZE_AVX512); \ + __m256i T0h = _mm256_srli_epi64(R0 ## h, EXP_DIGIT_SIZE_AVX512); \ + __m256i T1 = _mm256_srli_epi64(R1, EXP_DIGIT_SIZE_AVX512); \ + __m256i T1h = _mm256_srli_epi64(R1 ## h, EXP_DIGIT_SIZE_AVX512); \ + __m256i T2 = _mm256_srli_epi64(R2, EXP_DIGIT_SIZE_AVX512); \ + __m256i T2h = _mm256_srli_epi64(R2 ## h, EXP_DIGIT_SIZE_AVX512); \ + __m256i T3 = _mm256_srli_epi64(R3, EXP_DIGIT_SIZE_AVX512); \ + __m256i T3h = _mm256_srli_epi64(R3 ## h, EXP_DIGIT_SIZE_AVX512); \ + __m256i T4 = _mm256_srli_epi64(R4, EXP_DIGIT_SIZE_AVX512); \ + __m256i T4h = _mm256_srli_epi64(R4 ## h, EXP_DIGIT_SIZE_AVX512); \ + __m256i MASK = _mm256_set1_epi64x(EXP_DIGIT_MASK_AVX512); \ + \ + Ipp64u kk0, kk1, kk2, kk3, kk4; \ + Ipp64u kk0h, kk1h, kk2h, kk3h, kk4h; \ + kk0 = kk1 = kk2 = kk3 = kk4 = kk0h = kk1h = kk2h = kk3h = kk4h = 0; \ + \ + R0 = _mm256_and_si256(R0, MASK); \ + R1 = _mm256_and_si256(R1, MASK); \ + R2 = _mm256_and_si256(R2, MASK); \ + R3 = _mm256_and_si256(R3, MASK); \ + R4 = _mm256_and_si256(R4, MASK); \ + R0 ## h = _mm256_and_si256(R0 ## h, MASK); \ + R1 ## h = _mm256_and_si256(R1 ## h, MASK); \ + R2 ## h = _mm256_and_si256(R2 ## h, MASK); \ + R3 ## h = _mm256_and_si256(R3 ## h, MASK); \ + R4 ## h = _mm256_and_si256(R4 ## h, MASK); \ + \ + T4h = _mm256_alignr_epi64(T4h, T4, 3); \ + T4 = _mm256_alignr_epi64(T4 , T3h, 3); \ + T3h = _mm256_alignr_epi64(T3h, T3, 3); \ + T3 = _mm256_alignr_epi64(T3 , T2h, 3); \ + T2h = _mm256_alignr_epi64(T2h, T2, 3); \ + T2 = _mm256_alignr_epi64(T2 , T1h, 3); \ + T1h = _mm256_alignr_epi64(T1h, T1, 3); \ + T1 = _mm256_alignr_epi64(T1 , T0h, 3); \ + T0h = _mm256_alignr_epi64(T0h, T0, 3); \ + T0 = _mm256_alignr_epi64(T0, _mm256_setzero_si256(), 3); \ + \ + R0 = _mm256_add_epi64(R0, T0); \ + R1 = _mm256_add_epi64(R1, T1); \ + R2 = _mm256_add_epi64(R2, T2); \ + R3 = _mm256_add_epi64(R3, T3); \ + R4 = _mm256_add_epi64(R4, T4); \ + R0 ## h = _mm256_add_epi64(R0 ## h, T0h); \ + R1 ## h = _mm256_add_epi64(R1 ## h, T1h); \ + R2 ## h = _mm256_add_epi64(R2 ## h, T2h); \ + R3 ## h = _mm256_add_epi64(R3 ## h, T3h); \ + R4 ## h = _mm256_add_epi64(R4 ## h, T4h); \ + \ + { \ + Ipp64u k, l; \ + k = l = 0; \ + \ + kk0 = _mm256_cmp_epu64_mask(MASK, R0, _MM_CMPINT_LT); \ + kk1 = _mm256_cmp_epu64_mask(MASK, R1, _MM_CMPINT_LT); \ + kk2 = _mm256_cmp_epu64_mask(MASK, R2, _MM_CMPINT_LT); \ + kk3 = _mm256_cmp_epu64_mask(MASK, R3, _MM_CMPINT_LT); \ + kk4 = _mm256_cmp_epu64_mask(MASK, R4, _MM_CMPINT_LT); \ + kk0h = _mm256_cmp_epu64_mask(MASK, R0 ## h, _MM_CMPINT_LT); \ + kk1h = _mm256_cmp_epu64_mask(MASK, R1 ## h, _MM_CMPINT_LT); \ + kk2h = _mm256_cmp_epu64_mask(MASK, R2 ## h, _MM_CMPINT_LT); \ + kk3h = _mm256_cmp_epu64_mask(MASK, R3 ## h, _MM_CMPINT_LT); \ + kk4h = _mm256_cmp_epu64_mask(MASK, R4 ## h, _MM_CMPINT_LT); \ + \ + k = (kk4h<<36)|(kk4<<32)|(kk3h<<28)|(kk3<<24)|(kk2h<<20)| \ + (kk2<<16)|(kk1h<<12)|(kk1<<8)|(kk0h<<4)|kk0; \ + \ + kk0 = _mm256_cmp_epu64_mask(MASK, R0, _MM_CMPINT_EQ); \ + kk1 = _mm256_cmp_epu64_mask(MASK, R1, _MM_CMPINT_EQ); \ + kk2 = _mm256_cmp_epu64_mask(MASK, R2, _MM_CMPINT_EQ); \ + kk3 = _mm256_cmp_epu64_mask(MASK, R3, _MM_CMPINT_EQ); \ + kk4 = _mm256_cmp_epu64_mask(MASK, R4, _MM_CMPINT_EQ); \ + kk0h = _mm256_cmp_epu64_mask(MASK, R0 ## h, _MM_CMPINT_EQ); \ + kk1h = _mm256_cmp_epu64_mask(MASK, R1 ## h, _MM_CMPINT_EQ); \ + kk2h = _mm256_cmp_epu64_mask(MASK, R2 ## h, _MM_CMPINT_EQ); \ + kk3h = _mm256_cmp_epu64_mask(MASK, R3 ## h, _MM_CMPINT_EQ); \ + kk4h = _mm256_cmp_epu64_mask(MASK, R4 ## h, _MM_CMPINT_EQ); \ + \ + l = (kk4h<<36)|(kk4<<32)|(kk3h<<28)|(kk3<<24)|(kk2h<<20)| \ + (kk2<<16)|(kk1h<<12)|(kk1<<8)|(kk0h<<4)|kk0; \ + \ + k = l + 2*k; \ + k ^= l; \ + \ + kk0 = k; \ + kk0h = (k>>4); \ + kk1 = (k>>8); \ + kk1h = (k>>12); \ + kk2 = (k>>16); \ + kk2h = (k>>20); \ + kk3 = (k>>24); \ + kk3h = (k>>28); \ + kk4 = (k>>32); \ + kk4h = (k>>36); \ + } \ + \ + R0 = _mm256_mask_sub_epi64(R0, (__mmask8)kk0, R0, MASK); \ + R1 = _mm256_mask_sub_epi64(R1, (__mmask8)kk1, R1, MASK); \ + R2 = _mm256_mask_sub_epi64(R2, (__mmask8)kk2, R2, MASK); \ + R3 = _mm256_mask_sub_epi64(R3, (__mmask8)kk3, R3, MASK); \ + R4 = _mm256_mask_sub_epi64(R4, (__mmask8)kk4, R4, MASK); \ + R0 ## h = _mm256_mask_sub_epi64(R0 ## h, (__mmask8)kk0h, R0 ## h, MASK); \ + R1 ## h = _mm256_mask_sub_epi64(R1 ## h, (__mmask8)kk1h, R1 ## h, MASK); \ + R2 ## h = _mm256_mask_sub_epi64(R2 ## h, (__mmask8)kk2h, R2 ## h, MASK); \ + R3 ## h = _mm256_mask_sub_epi64(R3 ## h, (__mmask8)kk3h, R3 ## h, MASK); \ + R4 ## h = _mm256_mask_sub_epi64(R4 ## h, (__mmask8)kk4h, R4 ## h, MASK); \ + \ + R0 = _mm256_and_si256(R0, MASK); \ + R1 = _mm256_and_si256(R1, MASK); \ + R2 = _mm256_and_si256(R2, MASK); \ + R3 = _mm256_and_si256(R3, MASK); \ + R4 = _mm256_and_si256(R4, MASK); \ + R0 ## h = _mm256_and_si256(R0 ## h, MASK); \ + R1 ## h = _mm256_and_si256(R1 ## h, MASK); \ + R2 ## h = _mm256_and_si256(R2 ## h, MASK); \ + R3 ## h = _mm256_and_si256(R3 ## h, MASK); \ + R4 ## h = _mm256_and_si256(R4 ## h, MASK); \ +} + +#endif // IFMA_NORM_52X_H diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_rsa_arith.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_rsa_arith.h new file mode 100644 index 0000000..cdec434 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ifma_rsa_arith.h @@ -0,0 +1,97 @@ + /******************************************************************************* + * Copyright (C) 2019 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#ifndef _IFMA_RSA_ARITH_H_ +#define _IFMA_RSA_ARITH_H_ + +/* + * RSA kernels implemented with AVX512_IFMA256 ISA + */ + +#include "owncp.h" + +#if(_IPP32E>=_IPP32E_K1) + +/* Almost Montgomery Multiplication / Squaring */ +#define ifma256_amm52x20 OWNAPI(ifma256_amm52x20) + IPP_OWN_DECL(void, ifma256_amm52x20, (Ipp64u out[20], const Ipp64u a[20], const Ipp64u b[20], const Ipp64u m[20], Ipp64u k0)) +#define ifma256_ams52x20 OWNAPI(ifma256_ams52x20) + IPP_OWN_DECL(void, ifma256_ams52x20, (Ipp64u out[20], const Ipp64u a[20], const Ipp64u m[20], Ipp64u k0)) +#define ifma256_amm52x30 OWNAPI(ifma256_amm52x30) + IPP_OWN_DECL(void, ifma256_amm52x30, (Ipp64u out[32], const Ipp64u a[32], const Ipp64u b[32], const Ipp64u m[32], Ipp64u k0)) +#define ifma256_ams52x30 OWNAPI(ifma256_ams52x30) + IPP_OWN_DECL(void, ifma256_ams52x30, (Ipp64u out[32], const Ipp64u a[32], const Ipp64u m[32], Ipp64u k0)) +#define ifma256_amm52x40 OWNAPI(ifma256_amm52x40) + IPP_OWN_DECL(void, ifma256_amm52x40, (Ipp64u out[40], const Ipp64u a[40], const Ipp64u b[40], const Ipp64u m[40], Ipp64u k0)) +#define ifma256_ams52x40 OWNAPI(ifma256_ams52x40) + IPP_OWN_DECL(void, ifma256_ams52x40, (Ipp64u out[40], const Ipp64u a[40], const Ipp64u m[40], Ipp64u k0)) + + +/* + * Dual Almost Montgomery Multiplication / Squaring + * (two independent operations and data arrays) + */ +#define ifma256_amm52x20_dual OWNAPI(ifma256_amm52x20_dual) + IPP_OWN_DECL(void, ifma256_amm52x20_dual, (Ipp64u out[2][20], const Ipp64u a[2][20], const Ipp64u b[2][20], const Ipp64u m[2][20], const Ipp64u k0[2])) +#define ifma256_ams52x20_dual OWNAPI(ifma256_ams52x20_dual) + IPP_OWN_DECL(void, ifma256_ams52x20_dual, (Ipp64u out[2][20], const Ipp64u a[2][20], const Ipp64u m[2][20], const Ipp64u k0[2])) +#define ifma256_amm52x30_dual OWNAPI(ifma256_amm52x30_dual) + IPP_OWN_DECL(void, ifma256_amm52x30_dual, (Ipp64u out[2][32], const Ipp64u a[2][32], const Ipp64u b[2][32], const Ipp64u m[2][32], const Ipp64u k0[2])) +#define ifma256_ams52x30_dual OWNAPI(ifma256_ams52x30_dual) + IPP_OWN_DECL(void, ifma256_ams52x30_dual, (Ipp64u out[2][32], const Ipp64u a[2][32], const Ipp64u m[2][32], const Ipp64u k0[2])) +#define ifma256_amm52x40_dual OWNAPI(ifma256_amm52x40_dual) + IPP_OWN_DECL(void, ifma256_amm52x40_dual, (Ipp64u out[2][40], const Ipp64u a[2][40], const Ipp64u b[2][40], const Ipp64u m[2][40], const Ipp64u k0[2])) +#define ifma256_ams52x40_dual OWNAPI(ifma256_ams52x40_dual) + IPP_OWN_DECL(void, ifma256_ams52x40_dual, (Ipp64u out[2][40], const Ipp64u a[2][40], const Ipp64u m[2][40], const Ipp64u k0[2])) + + +/* Exponentiation */ +#define ifma256_exp52x20 OWNAPI(ifma256_exp52x20) + IPP_OWN_DECL (void, ifma256_exp52x20, (Ipp64u *out, + const Ipp64u *base, + const Ipp64u *exp, + const Ipp64u *modulus, + const Ipp64u *toMont, + const Ipp64u k0)) + +/* Dual exponentiation */ +#define ifma256_exp52x20_dual OWNAPI(ifma256_exp52x20_dual) + IPP_OWN_DECL (void, ifma256_exp52x20_dual, (Ipp64u out [2][20], + const Ipp64u base [2][20], + const Ipp64u *exp [2], // 2x16 + const Ipp64u modulus[2][20], + const Ipp64u toMont [2][20], + const Ipp64u k0 [2])) + +#define ifma256_exp52x30_dual OWNAPI(ifma256_exp52x30_dual) + IPP_OWN_DECL (void, ifma256_exp52x30_dual, (Ipp64u out [2][32], + const Ipp64u base [2][32], + const Ipp64u *exp [2], // 2x24 + const Ipp64u modulus[2][32], + const Ipp64u toMont [2][32], + const Ipp64u k0 [2])) + +#define ifma256_exp52x40_dual OWNAPI(ifma256_exp52x40_dual) + IPP_OWN_DECL (void, ifma256_exp52x40_dual, (Ipp64u out [2][40], + const Ipp64u base [2][40], + const Ipp64u *exp [2], // 2x32 + const Ipp64u modulus[2][40], + const Ipp64u toMont [2][40], + const Ipp64u k0 [2])) + +#endif // #if(_IPP32E>=_IPP32E_K1) +#endif // #ifndef _IFMA_RSA_ARITH_H_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/internal_hashmessage_mb.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/internal_hashmessage_mb.c new file mode 100644 index 0000000..8daf08f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/internal_hashmessage_mb.c @@ -0,0 +1,36 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "internal_hashmessage_mb.h" + +IPP_OWN_DEFN (IppStatus, cpHashMessage_MB8_rmf, (const Ipp8u* const pSrc[8], int const lens[8], Ipp8u* const pDst[8], const IppsHashMethod* pMethod)) +{ + int i; + for (i = 0; i < 8; ++i) { // 8 buffers, for RSA + ippsHashMessage_rmf(pSrc[i], lens[i], pDst[i], pMethod); + } + return ippStsNoErr; +} + +IPP_OWN_DEFN (IppStatus, cpMGF1_MB8_rmf, (const Ipp8u* const pSeeds[8], int const seedLens[8], Ipp8u* const pMasks[8], int const maskLens[8], const IppsHashMethod* pMethod)) +{ + int i; + for (i = 0; i < 8; ++i) { // 8 buffers, for RSA + ippsMGF1_rmf(pSeeds[i], seedLens[i], pMasks[i], maskLens[i], pMethod); + } + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/internal_hashmessage_mb.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/internal_hashmessage_mb.h new file mode 100644 index 0000000..fcff212 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/internal_hashmessage_mb.h @@ -0,0 +1,23 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "owncp.h" + +#define cpHashMessage_MB8_rmf OWNAPI(cpHashMessage_MB8_rmf) + IPP_OWN_DECL (IppStatus, cpHashMessage_MB8_rmf, (const Ipp8u* const pSrc[8], int const lens[8], Ipp8u* const pDst[8], const IppsHashMethod* pMethod)) +#define cpMGF1_MB8_rmf OWNAPI(cpMGF1_MB8_rmf) + IPP_OWN_DECL (IppStatus, cpMGF1_MB8_rmf, (const Ipp8u* const pSeeds[8], int const seedLens[8], Ipp8u* const pMasks[8], int const maskLens[8], const IppsHashMethod* pMethod)) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ippcp.def b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ippcp.def new file mode 100644 index 0000000..4a3f557 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/ippcp.def @@ -0,0 +1,560 @@ +EXPORTS + +ippcpInit +cpGetReg +cpStartTscp +cpStopTscp +cpStartTsc +cpStopTsc +cpGetCacheSize +cpGetFeature +ippcpSetCpuFeatures +ippcpGetCpuFeatures +ippcpGetCpuClocks +ippcpSetNumThreads +ippcpGetNumThreads +ippcpGetEnabledCpuFeatures +ippcpGetEnabledNumThreads +ippcpGetStatusString + +ippcpGetLibVersion +ippsDESGetSize +ippsDESInit +ippsDESPack +ippsDESUnpack +ippsTDESEncryptECB +ippsTDESDecryptECB +ippsTDESEncryptCBC +ippsTDESDecryptCBC +ippsTDESEncryptCFB +ippsTDESDecryptCFB +ippsTDESEncryptOFB +ippsTDESDecryptOFB +ippsTDESEncryptCTR +ippsTDESDecryptCTR +ippsAESGetSize +ippsAESInit +ippsAESSetKey +ippsAESPack +ippsAESUnpack +ippsAESSetupNoise +ippsAES_GCMSetupNoise +ippsAES_CMACSetupNoise +ippsAESEncryptECB +ippsAESDecryptECB +ippsAESEncryptCBC +ippsAESEncryptCBC_CS1 +ippsAESEncryptCBC_CS2 +ippsAESEncryptCBC_CS3 +ippsAESDecryptCBC +ippsAESDecryptCBC_CS1 +ippsAESDecryptCBC_CS2 +ippsAESDecryptCBC_CS3 +ippsAESEncryptCFB +ippsAESDecryptCFB +ippsAESEncryptOFB +ippsAESDecryptOFB +ippsAESEncryptCTR +ippsAESDecryptCTR +ippsAESEncryptXTS_Direct +ippsAESDecryptXTS_Direct +ippsAES_EncryptCFB16_MB +ippsSMS4GetSize +ippsSMS4Init +ippsSMS4SetKey +ippsSMS4EncryptECB +ippsSMS4DecryptECB +ippsSMS4EncryptCBC +ippsSMS4EncryptCBC_CS1 +ippsSMS4EncryptCBC_CS2 +ippsSMS4EncryptCBC_CS3 +ippsSMS4DecryptCBC +ippsSMS4DecryptCBC_CS1 +ippsSMS4DecryptCBC_CS2 +ippsSMS4DecryptCBC_CS3 +ippsSMS4EncryptCFB +ippsSMS4DecryptCFB +ippsSMS4EncryptOFB +ippsSMS4DecryptOFB +ippsSMS4EncryptCTR +ippsSMS4DecryptCTR +ippsSMS4_CCMGetSize +ippsSMS4_CCMInit +ippsSMS4_CCMMessageLen +ippsSMS4_CCMTagLen +ippsSMS4_CCMStart +ippsSMS4_CCMEncrypt +ippsSMS4_CCMDecrypt +ippsSMS4_CCMGetTag +ippsAES_CCMGetSize +ippsAES_CCMInit +ippsAES_CCMMessageLen +ippsAES_CCMTagLen +ippsAES_CCMStart +ippsAES_CCMEncrypt +ippsAES_CCMDecrypt +ippsAES_CCMGetTag +ippsAES_GCMGetSize +ippsAES_GCMInit +ippsAES_GCMReinit +ippsAES_GCMReset +ippsAES_GCMProcessIV +ippsAES_GCMProcessAAD +ippsAES_GCMStart +ippsAES_GCMEncrypt +ippsAES_GCMDecrypt +ippsAES_GCMGetTag +ippsAES_XTSGetSize +ippsAES_XTSInit +ippsAES_XTSEncrypt +ippsAES_XTSDecrypt +ippsAES_S2V_CMAC +ippsAES_SIVEncrypt +ippsAES_SIVDecrypt +ippsAES_CMACGetSize +ippsAES_CMACInit +ippsAES_CMACUpdate +ippsAES_CMACFinal +ippsAES_CMACGetTag +ippsARCFourCheckKey +ippsARCFourGetSize +ippsARCFourInit +ippsARCFourReset +ippsARCFourPack +ippsARCFourUnpack +ippsARCFourEncrypt +ippsARCFourDecrypt +ippsSHA1GetSize +ippsSHA1Init +ippsSHA1Duplicate +ippsSHA1Pack +ippsSHA1Unpack +ippsSHA1Update +ippsSHA1GetTag +ippsSHA1Final +ippsSHA1MessageDigest +ippsSHA224GetSize +ippsSHA224Init +ippsSHA224Duplicate +ippsSHA224Pack +ippsSHA224Unpack +ippsSHA224Update +ippsSHA224GetTag +ippsSHA224Final +ippsSHA224MessageDigest +ippsSHA256GetSize +ippsSHA256Init +ippsSHA256Duplicate +ippsSHA256Pack +ippsSHA256Unpack +ippsSHA256Update +ippsSHA256GetTag +ippsSHA256Final +ippsSHA256MessageDigest +ippsSHA384GetSize +ippsSHA384Init +ippsSHA384Duplicate +ippsSHA384Pack +ippsSHA384Unpack +ippsSHA384Update +ippsSHA384GetTag +ippsSHA384Final +ippsSHA384MessageDigest +ippsSHA512GetSize +ippsSHA512Init +ippsSHA512Duplicate +ippsSHA512Pack +ippsSHA512Unpack +ippsSHA512Update +ippsSHA512GetTag +ippsSHA512Final +ippsSHA512MessageDigest +ippsMD5GetSize +ippsMD5Init +ippsMD5Duplicate +ippsMD5Pack +ippsMD5Unpack +ippsMD5Update +ippsMD5GetTag +ippsMD5Final +ippsMD5MessageDigest +ippsSM3GetSize +ippsSM3Init +ippsSM3Duplicate +ippsSM3Pack +ippsSM3Unpack +ippsSM3Update +ippsSM3GetTag +ippsSM3Final +ippsSM3MessageDigest +ippsHashGetSize +ippsHashInit +ippsHashPack +ippsHashUnpack +ippsHashDuplicate +ippsHashUpdate +ippsHashGetTag +ippsHashFinal +ippsHashMessage +ippsHashMethodGetSize +ippsHashMethodSet_MD5 +ippsHashMethodSet_SM3 +ippsHashMethodSet_SHA1 +ippsHashMethodSet_SHA1_NI +ippsHashMethodSet_SHA1_TT +ippsHashMethodSet_SHA256 +ippsHashMethodSet_SHA256_NI +ippsHashMethodSet_SHA256_TT +ippsHashMethodSet_SHA224 +ippsHashMethodSet_SHA224_NI +ippsHashMethodSet_SHA224_TT +ippsHashMethodSet_SHA512 +ippsHashMethodSet_SHA384 +ippsHashMethodSet_SHA512_256 +ippsHashMethodSet_SHA512_224 +ippsHashStateMethodSet_SM3 +ippsHashStateMethodSet_SHA256 +ippsHashStateMethodSet_SHA256_NI +ippsHashStateMethodSet_SHA256_TT +ippsHashStateMethodSet_SHA224 +ippsHashStateMethodSet_SHA224_NI +ippsHashStateMethodSet_SHA224_TT +ippsHashStateMethodSet_SHA512 +ippsHashStateMethodSet_SHA384 +ippsHashStateMethodSet_SHA512_256 +ippsHashStateMethodSet_SHA512_224 +ippsHashMethod_MD5 +ippsHashMethod_SM3 +ippsHashMethod_SHA1 +ippsHashMethod_SHA1_NI +ippsHashMethod_SHA1_TT +ippsHashMethod_SHA256 +ippsHashMethod_SHA256_NI +ippsHashMethod_SHA256_TT +ippsHashMethod_SHA224 +ippsHashMethod_SHA224_NI +ippsHashMethod_SHA224_TT +ippsHashMethod_SHA512 +ippsHashMethod_SHA384 +ippsHashMethod_SHA512_256 +ippsHashMethod_SHA512_224 +ippsHashMethodGetInfo +ippsHashGetSize_rmf +ippsHashInit_rmf +ippsHashPack_rmf +ippsHashUnpack_rmf +ippsHashDuplicate_rmf +ippsHashUpdate_rmf +ippsHashGetTag_rmf +ippsHashFinal_rmf +ippsHashMessage_rmf +ippsHashGetInfo_rmf +ippsMGF +ippsMGF1_rmf +ippsMGF2_rmf +ippsHMAC_GetSize +ippsHMAC_Init +ippsHMAC_Pack +ippsHMAC_Unpack +ippsHMAC_Duplicate +ippsHMAC_Update +ippsHMAC_Final +ippsHMAC_GetTag +ippsHMAC_Message +ippsHMACGetSize_rmf +ippsHMACInit_rmf +ippsHMACPack_rmf +ippsHMACUnpack_rmf +ippsHMACDuplicate_rmf +ippsHMACUpdate_rmf +ippsHMACFinal_rmf +ippsHMACGetTag_rmf +ippsHMACMessage_rmf +ippsBigNumGetSize +ippsBigNumInit +ippsCmpZero_BN +ippsCmp_BN +ippsGetSize_BN +ippsSet_BN +ippsGet_BN +ippsRef_BN +ippsExtGet_BN +ippsAdd_BN +ippsSub_BN +ippsMul_BN +ippsMAC_BN_I +ippsDiv_BN +ippsMod_BN +ippsGcd_BN +ippsModInv_BN +ippsSetOctString_BN +ippsGetOctString_BN +ippsMontGetSize +ippsMontInit +ippsMontSet +ippsMontGet +ippsMontForm +ippsMontMul +ippsMontExp +ippsPRNGGetSize +ippsPRNGInit +ippsPRNGSetModulus +ippsPRNGSetH0 +ippsPRNGSetAugment +ippsPRNGSetSeed +ippsPRNGGetSeed +ippsPRNGen +ippsPRNGen_BN +ippsPRNGenRDRAND +ippsPRNGenRDRAND_BN +ippsTRNGenRDSEED +ippsTRNGenRDSEED_BN +ippsPrimeGetSize +ippsPrimeInit +ippsPrimeGen +ippsPrimeTest +ippsPrimeGen_BN +ippsPrimeTest_BN +ippsPrimeGet +ippsPrimeGet_BN +ippsPrimeSet +ippsPrimeSet_BN +ippsRSA_GetSizePublicKey +ippsRSA_InitPublicKey +ippsRSA_SetPublicKey +ippsRSA_GetPublicKey +ippsRSA_GetSizePrivateKeyType1 +ippsRSA_InitPrivateKeyType1 +ippsRSA_SetPrivateKeyType1 +ippsRSA_GetPrivateKeyType1 +ippsRSA_GetSizePrivateKeyType2 +ippsRSA_InitPrivateKeyType2 +ippsRSA_SetPrivateKeyType2 +ippsRSA_GetPrivateKeyType2 +ippsRSA_GetBufferSizePublicKey +ippsRSA_GetBufferSizePrivateKey +ippsRSA_Encrypt +ippsRSA_Decrypt +ippsRSA_GenerateKeys +ippsRSA_ValidateKeys +ippsRSAEncrypt_OAEP +ippsRSADecrypt_OAEP +ippsRSAEncrypt_OAEP_rmf +ippsRSADecrypt_OAEP_rmf +ippsRSAEncrypt_PKCSv15 +ippsRSADecrypt_PKCSv15 +ippsRSASign_PSS +ippsRSAVerify_PSS +ippsRSASign_PSS_rmf +ippsRSAVerify_PSS_rmf +ippsRSASign_PKCS1v15 +ippsRSAVerify_PKCS1v15 +ippsRSASign_PKCS1v15_rmf +ippsRSAVerify_PKCS1v15_rmf +ippsDLGetResultString +ippsDLPGetSize +ippsDLPInit +ippsDLPPack +ippsDLPUnpack +ippsDLPSet +ippsDLPGet +ippsDLPSetDP +ippsDLPGetDP +ippsDLPGenKeyPair +ippsDLPPublicKey +ippsDLPValidateKeyPair +ippsDLPSetKeyPair +ippsDLPSignDSA +ippsDLPVerifyDSA +ippsDLPSharedSecretDH +ippsDLPGenerateDSA +ippsDLPValidateDSA +ippsDLPGenerateDH +ippsDLPValidateDH +ippsECCGetResultString +ippsECCPGetSize +ippsECCPGetSizeStd128r1 +ippsECCPGetSizeStd128r2 +ippsECCPGetSizeStd192r1 +ippsECCPGetSizeStd224r1 +ippsECCPGetSizeStd256r1 +ippsECCPGetSizeStd384r1 +ippsECCPGetSizeStd521r1 +ippsECCPGetSizeStdSM2 +ippsECCPInit +ippsECCPInitStd128r1 +ippsECCPInitStd128r2 +ippsECCPInitStd192r1 +ippsECCPInitStd224r1 +ippsECCPInitStd256r1 +ippsECCPInitStd384r1 +ippsECCPInitStd521r1 +ippsECCPInitStdSM2 +ippsECCPSet +ippsECCPSetStd +ippsECCPSetStd128r1 +ippsECCPSetStd128r2 +ippsECCPSetStd192r1 +ippsECCPSetStd224r1 +ippsECCPSetStd256r1 +ippsECCPSetStd384r1 +ippsECCPSetStd521r1 +ippsECCPSetStdSM2 +ippsECCPBindGxyTblStd192r1 +ippsECCPBindGxyTblStd224r1 +ippsECCPBindGxyTblStd256r1 +ippsECCPBindGxyTblStd384r1 +ippsECCPBindGxyTblStd521r1 +ippsECCPBindGxyTblStdSM2 +ippsECCPGet +ippsECCPGetOrderBitSize +ippsECCPValidate +ippsECCPPointGetSize +ippsECCPPointInit +ippsECCPSetPoint +ippsECCPSetPointAtInfinity +ippsECCPGetPoint +ippsECCPCheckPoint +ippsECCPComparePoint +ippsECCPNegativePoint +ippsECCPAddPoint +ippsECCPMulPointScalar +ippsECCPGenKeyPair +ippsECCPPublicKey +ippsECCPValidateKeyPair +ippsECCPSetKeyPair +ippsECCPSharedSecretDH +ippsECCPSharedSecretDHC +ippsECCPSignDSA +ippsECCPVerifyDSA +ippsECCPSignNR +ippsECCPVerifyNR +ippsECCPSignSM2 +ippsECCPVerifySM2 +ippsGFpGetSize +ippsGFpInitArbitrary +ippsGFpInitFixed +ippsGFpInit +ippsGFpMethod_p192r1 +ippsGFpMethod_p224r1 +ippsGFpMethod_p256r1 +ippsGFpMethod_p384r1 +ippsGFpMethod_p521r1 +ippsGFpMethod_p256sm2 +ippsGFpMethod_p256bn +ippsGFpMethod_p256 +ippsGFpMethod_pArb +ippsGFpxGetSize +ippsGFpxInit +ippsGFpxInitBinomial +ippsGFpxMethod_binom2_epid2 +ippsGFpxMethod_binom3_epid2 +ippsGFpxMethod_binom2 +ippsGFpxMethod_binom3 +ippsGFpxMethod_binom +ippsGFpxMethod_com +ippsGFpScratchBufferSize +ippsGFpElementGetSize +ippsGFpElementInit +ippsGFpSetElement +ippsGFpSetElementRegular +ippsGFpSetElementOctString +ippsGFpSetElementRandom +ippsGFpSetElementHash +ippsGFpSetElementHash_rmf +ippsGFpCpyElement +ippsGFpGetElement +ippsGFpGetElementOctString +ippsGFpCmpElement +ippsGFpIsZeroElement +ippsGFpIsUnityElement +ippsGFpConj +ippsGFpNeg +ippsGFpInv +ippsGFpSqrt +ippsGFpSqr +ippsGFpAdd +ippsGFpSub +ippsGFpMul +ippsGFpExp +ippsGFpMultiExp +ippsGFpAdd_PE +ippsGFpSub_PE +ippsGFpMul_PE +ippsGFpGetInfo +ippsGFpECGetSize +ippsGFpECInit +ippsGFpECSet +ippsGFpECSetSubgroup +ippsGFpECInitStd128r1 +ippsGFpECInitStd128r2 +ippsGFpECInitStd192r1 +ippsGFpECInitStd224r1 +ippsGFpECInitStd256r1 +ippsGFpECInitStd384r1 +ippsGFpECInitStd521r1 +ippsGFpECInitStdSM2 +ippsGFpECInitStdBN256 +ippsGFpECBindGxyTblStd192r1 +ippsGFpECBindGxyTblStd224r1 +ippsGFpECBindGxyTblStd256r1 +ippsGFpECBindGxyTblStd384r1 +ippsGFpECBindGxyTblStd521r1 +ippsGFpECBindGxyTblStdSM2 +ippsGFpECGet +ippsGFpECGetSubgroup +ippsGFpECGetInfo_GF +ippsGFpECScratchBufferSize +ippsGFpECVerify +ippsGFpECPointGetSize +ippsGFpECPointInit +ippsGFpECSetPointAtInfinity +ippsGFpECSetPoint +ippsGFpECSetPointRegular +ippsGFpECSetPointRandom +ippsGFpECMakePoint +ippsGFpECSetPointHash +ippsGFpECSetPointHash_rmf +ippsGFpECSetPointHashBackCompatible +ippsGFpECSetPointHashBackCompatible_rmf +ippsGFpECGetPoint +ippsGFpECGetPointRegular +ippsGFpECSetPointOctString +ippsGFpECGetPointOctString +ippsGFpECTstPoint +ippsGFpECTstPointInSubgroup +ippsGFpECCpyPoint +ippsGFpECCmpPoint +ippsGFpECNegPoint +ippsGFpECAddPoint +ippsGFpECMulPoint +ippsGFpECPrivateKey +ippsGFpECPublicKey +ippsGFpECTstKeyPair +ippsGFpECSharedSecretDH +ippsGFpECSharedSecretDHC +ippsGFpECSignDSA +ippsGFpECVerifyDSA +ippsGFpECUserIDHashSM2 +ippsGFpECMessageRepresentationSM2 +ippsGFpECSignNR +ippsGFpECVerifyNR +ippsGFpECSignSM2 +ippsGFpECVerifySM2 +ippsGFpECKeyExchangeSM2_GetSize +ippsGFpECKeyExchangeSM2_Init +ippsGFpECKeyExchangeSM2_Setup +ippsGFpECKeyExchangeSM2_SharedKey +ippsGFpECKeyExchangeSM2_Confirm +ippsGFpECEncryptSM2_Ext_EncMsgSize +ippsGFpECEncryptSM2_Ext +ippsGFpECDecryptSM2_Ext_DecMsgSize +ippsGFpECDecryptSM2_Ext +ippsGFpECESGetSize_SM2 +ippsGFpECESInit_SM2 +ippsGFpECESSetKey_SM2 +ippsGFpECESStart_SM2 +ippsGFpECESEncrypt_SM2 +ippsGFpECESDecrypt_SM2 +ippsGFpECESFinal_SM2 +ippsGFpECESGetBuffersSize_SM2 diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/owncp.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/owncp.h new file mode 100644 index 0000000..bb9604e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/owncp.h @@ -0,0 +1,207 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +*/ + +#ifndef __OWNCP_H__ +#define __OWNCP_H__ + +#ifndef __OWNDEFS_H__ + #include "owndefs.h" +#endif + +#ifndef IPPCP_H__ + #if defined _MERGED_BLD + #include "ippcp_cpuspc.h" + #endif + + #include "ippcp.h" +#endif + +/* +// modes of the CPU feature +*/ +#define _FEATURE_OFF_ (0) /* feature is OFF */ +#define _FEATURE_ON_ (1) /* feature is ON */ +#define _FEATURE_TICKTOCK_ (2) /* dectect if feature is OFF/ON */ + +#include "pcpvariant.h" + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) // for MSVC +#pragma warning(disable : 4324) // structures padding warning +#endif + +/* ippCP length */ +typedef int cpSize; + +/* +// Common ippCP Macros +*/ + +/* size of cache line (bytes) */ +#if (_IPP==_IPP_M5) +#define CACHE_LINE_SIZE (16) +#define LOG_CACHE_LINE_SIZE (4) +#else +#define CACHE_LINE_SIZE (64) +#define LOG_CACHE_LINE_SIZE (6) +#endif + +/* swap data & pointers */ +#define SWAP_PTR(ATYPE, pX,pY) { ATYPE* aPtr=(pX); (pX)=(pY); (pY)=aPtr; } +#define SWAP(x,y) {(x)^=(y); (y)^=(x); (x)^=(y);} + +/* alignment value */ +#define ALIGN_VAL ((int)sizeof(void*)) + +/* bitsize */ +#define BYTESIZE (8) +#define BITSIZE(x) ((int)(sizeof(x)*BYTESIZE)) + +/* bit length -> byte/word length conversion */ +#define BITS2WORD8_SIZE(x) (((x)+ 7)>>3) +#define BITS2WORD16_SIZE(x) (((x)+15)>>4) +#define BITS2WORD32_SIZE(x) (((x)+31)>>5) +#define BITS2WORD64_SIZE(x) (((x)+63)>>6) + +/* WORD and DWORD manipulators */ +#define IPP_LODWORD(x) ((Ipp32u)(x)) +#define IPP_HIDWORD(x) ((Ipp32u)(((Ipp64u)(x) >>32) & 0xFFFFFFFF)) + +#define IPP_MAKEHWORD(bLo,bHi) ((Ipp16u)(((Ipp8u)(bLo)) | ((Ipp16u)((Ipp8u)(bHi))) << 8)) +#define IPP_MAKEWORD(hLo,hHi) ((Ipp32u)(((Ipp16u)(hLo)) | ((Ipp32u)((Ipp16u)(hHi))) << 16)) +#define IPP_MAKEDWORD(wLo,wHi) ((Ipp64u)(((Ipp32u)(wLo)) | ((Ipp64u)((Ipp32u)(wHi))) << 32)) + +/* extract byte */ +#define EBYTE(w,n) ((Ipp8u)((w) >> (8 * (n)))) + +/* hexString <-> Ipp32u conversion */ +#define HSTRING_TO_U32(ptrByte) \ + (Ipp32u)(((ptrByte)[0]) <<24) \ + +(Ipp32u)(((ptrByte)[1]) <<16) \ + +(Ipp32u)(((ptrByte)[2]) <<8) \ + +(Ipp32u)((ptrByte)[3]) +#define U32_TO_HSTRING(ptrByte, x) \ + (ptrByte)[0] = (Ipp8u)((x)>>24); \ + (ptrByte)[1] = (Ipp8u)((x)>>16); \ + (ptrByte)[2] = (Ipp8u)((x)>>8); \ + (ptrByte)[3] = (Ipp8u)(x) + +/* 32- and 64-bit masks for MSB of nbits-sequence */ +#define MAKEMASK32(nbits) (0xFFFFFFFF >>((32 - ((nbits)&0x1F)) &0x1F)) +#define MAKEMASK64(nbits) (0xFFFFFFFFFFFFFFFF >>((64 - ((nbits)&0x3F)) &0x3F)) + +/* Logical Shifts (right and left) of WORD */ +#define LSR32(x,nBits) ((x)>>(nBits)) +#define LSL32(x,nBits) ((x)<<(nBits)) + +/* Rorate (right and left) of WORD */ +#if defined(_MSC_VER) && !defined( __ICL ) +# include +# define ROR32(x, nBits) _lrotr((x),(nBits)) +# define ROL32(x, nBits) _lrotl((x),(nBits)) +#else +# define ROR32(x, nBits) (LSR32((x),(nBits)) | LSL32((x),32-(nBits))) +# define ROL32(x, nBits) ROR32((x),(32-(nBits))) +#endif + +/* Logical Shifts (right and left) of DWORD */ +#define LSR64(x,nBits) ((x)>>(nBits)) +#define LSL64(x,nBits) ((x)<<(nBits)) + +/* Rorate (right and left) of DWORD */ +#define ROR64(x, nBits) (LSR64((x),(nBits)) | LSL64((x),64-(nBits))) +#define ROL64(x, nBits) ROR64((x),(64-(nBits))) + +/* change endian */ +#if defined(_MSC_VER) +# define ENDIANNESS(x) _byteswap_ulong((x)) +# define ENDIANNESS32(x) ENDIANNESS((x)) +# define ENDIANNESS64(x) _byteswap_uint64((x)) +#elif defined(__ICL) +# define ENDIANNESS(x) _bswap((x)) +# define ENDIANNESS32(x) ENDIANNESS((x)) +# define ENDIANNESS64(x) _bswap64((x)) +#else +# define ENDIANNESS(x) ((ROR32((x), 24) & 0x00ff00ff) | (ROR32((x), 8) & 0xff00ff00)) +# define ENDIANNESS32(x) ENDIANNESS((x)) +# define ENDIANNESS64(x) IPP_MAKEDWORD(ENDIANNESS(IPP_HIDWORD((x))), ENDIANNESS(IPP_LODWORD((x)))) +#endif + +#define IPP_MAKE_MULTIPLE_OF_8(x) ((x) = ((x)+7)&(~7)) +#define IPP_MAKE_MULTIPLE_OF_16(x) ((x) = ((x)+15)&(~15)) + +/* define 64-bit constant */ +#if !defined(__GNUC__) + #define CONST_64(x) (x) /*(x##i64)*/ +#else + #define CONST_64(x) (x##LL) +#endif + +/* define 64-bit constant or pair of 32-bit dependding on architecture */ +#if ((_IPP_ARCH == _IPP_ARCH_EM64T) || (_IPP_ARCH == _IPP_ARCH_LP64) || (_IPP_ARCH == _IPP_ARCH_LRB) || (_IPP_ARCH == _IPP_ARCH_LRB2)) +#define LL(lo,hi) (((Ipp64u)(lo)) | ((Ipp64u)(hi) << 32)) +#define L_(lo) ((Ipp64u)(lo)) +#else +#define LL(lo,hi) (lo),(hi) +#define L_(lo) (lo) +#endif + + +/* test if library's feature is ON */ +int cpGetFeature( Ipp64u Feature ); +/* test CPU crypto features */ +__INLINE Ipp32u IsFeatureEnabled(Ipp64u niMmask) +{ + return (Ipp32u)cpGetFeature(niMmask); +} + +#define IPPCP_GET_NUM_THREADS() ( ippcpGetEnabledNumThreads() ) +#define IPPCP_OMP_NUM_THREADS() num_threads( IPPCP_GET_NUM_THREADS() ) +#define IPPCP_OMP_LIMIT_MAX_NUM_THREADS(n) num_threads( IPP_MIN(IPPCP_GET_NUM_THREADS(),(n))) + +/* copy under mask */ +#define MASKED_COPY_BNU(dst, mask, src1, src2, len) { \ + cpSize i; \ + for(i=0; i<(len); i++) (dst)[i] = ((mask) & (src1)[i]) | (~(mask) & (src2)[i]); \ +} + +#if (_IPP > _IPP_PX || _IPP32E > _IPP32E_PX) && !defined(__INTEL_COMPILER) && !defined(__INTEL_LLVM_COMPILER) +#if !defined( _M_X64 ) && defined ( _MSC_VER ) +__inline __m128i +_mm_cvtsi64_si128(__int64 a) +{ + __m128i x; + x.m128i_i64[0] = a; + x.m128i_i64[1] = 0; + return x; +} +#endif + +#if !defined( __x86_64__ ) && defined(__GNUC__) +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi64_si128 (long long __A) +{ + return _mm_set_epi64x (0, __A); +} +#endif +#endif /* (_IPP > _IPP_PX || _IPP32E > _IPP32E_PX) && !defined(__INTEL_COMPILER) */ + +#endif /* __OWNCP_H__ */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_128keyexpansion_ni.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_128keyexpansion_ni.c new file mode 100644 index 0000000..1888ce5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_128keyexpansion_ni.c @@ -0,0 +1,106 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. AES keys expansion +// +// Contents: +// aes128_KeyExpansion_NI() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" + + + +#if (_AES_NI_ENABLING_==_FEATURE_ON_) || (_AES_NI_ENABLING_==_FEATURE_TICKTOCK_) + +/* +// AES-128 key expansion +*/ +static __m128i aes128_assist(__m128i temp1, __m128i temp2) +{ + __m128i temp3; + temp2 = _mm_shuffle_epi32 (temp2 ,0xff); + temp3 = _mm_slli_si128 (temp1, 0x4); + temp1 = _mm_xor_si128 (temp1, temp3); + temp3 = _mm_slli_si128 (temp3, 0x4); + temp1 = _mm_xor_si128 (temp1, temp3); + temp3 = _mm_slli_si128 (temp3, 0x4); + temp1 = _mm_xor_si128 (temp1, temp3); + temp1 = _mm_xor_si128 (temp1, temp2); + return temp1; +} + +#define aes128_KeyExpansion_NI OWNAPI(aes128_KeyExpansion_NI) + IPP_OWN_DECL (void, aes128_KeyExpansion_NI, (Ipp8u* keyExp, const Ipp8u* userkey)) + +IPP_OWN_DEFN (void, aes128_KeyExpansion_NI, (Ipp8u* keyExp, const Ipp8u* userkey)) +{ + __m128i *pKeySchedule = (__m128i*)keyExp; + + __m128i temp[2]; + /* + temp[0] = temp1 + temp[1] = temp2 + */ + + temp[0] = _mm_loadu_si128((__m128i*)userkey); + pKeySchedule[0] = temp[0]; + temp[1] = _mm_aeskeygenassist_si128 (temp[0] ,0x1); + temp[0] = aes128_assist(temp[0], temp[1]); + pKeySchedule[1] = temp[0]; + temp[1] = _mm_aeskeygenassist_si128 (temp[0],0x2); + temp[0] = aes128_assist(temp[0], temp[1]); + pKeySchedule[2] = temp[0]; + temp[1] = _mm_aeskeygenassist_si128 (temp[0],0x4); + temp[0] = aes128_assist(temp[0], temp[1]); + pKeySchedule[3] = temp[0]; + temp[1] = _mm_aeskeygenassist_si128 (temp[0],0x8); + temp[0] = aes128_assist(temp[0], temp[1]); + pKeySchedule[4] = temp[0]; + temp[1] = _mm_aeskeygenassist_si128 (temp[0],0x10); + temp[0] = aes128_assist(temp[0], temp[1]); + pKeySchedule[5] = temp[0]; + temp[1] = _mm_aeskeygenassist_si128 (temp[0],0x20); + temp[0] = aes128_assist(temp[0], temp[1]); + pKeySchedule[6] = temp[0]; + temp[1] = _mm_aeskeygenassist_si128 (temp[0],0x40); + temp[0] = aes128_assist(temp[0], temp[1]); + pKeySchedule[7] = temp[0]; + temp[1] = _mm_aeskeygenassist_si128 (temp[0],0x80); + temp[0] = aes128_assist(temp[0], temp[1]); + pKeySchedule[8] = temp[0]; + temp[1] = _mm_aeskeygenassist_si128 (temp[0],0x1b); + temp[0] = aes128_assist(temp[0], temp[1]); + pKeySchedule[9] = temp[0]; + temp[1] = _mm_aeskeygenassist_si128 (temp[0],0x36); + temp[0] = aes128_assist(temp[0], temp[1]); + pKeySchedule[10] = temp[0]; + + /* clear secret data */ + for(Ipp32u i = 0; i < sizeof(temp)/sizeof(temp[0]); i++){ + temp[i] = _mm_xor_si128(temp[i],temp[i]); + } +} + +#endif /* #if (_AES_NI_ENABLING_==_FEATURE_ON_) || (_AES_NI_ENABLING_==_FEATURE_TICKTOCK_) */ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_192keyexpansion_ni.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_192keyexpansion_ni.c new file mode 100644 index 0000000..016c1e2 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_192keyexpansion_ni.c @@ -0,0 +1,115 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. AES keys expansion +// +// Contents: +// aes192_KeyExpansion_NI() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcpaes_keys_ni.h" + +#if (_AES_NI_ENABLING_==_FEATURE_ON_) || (_AES_NI_ENABLING_==_FEATURE_TICKTOCK_) + +////////////////////////////////////////////////////////////////////// +/* +// AES-192 key expansion +*/ +static void aes192_assist(__m128i* temp1, __m128i * temp2, __m128i * temp3) +{ + __m128i temp4; + *temp2 = _mm_shuffle_epi32 (*temp2, 0x55); + temp4 = _mm_slli_si128 (*temp1, 0x4); + *temp1 = _mm_xor_si128 (*temp1, temp4); + temp4 = _mm_slli_si128 (temp4, 0x4); + *temp1 = _mm_xor_si128 (*temp1, temp4); + temp4 = _mm_slli_si128 (temp4, 0x4); + *temp1 = _mm_xor_si128 (*temp1, temp4); + *temp1 = _mm_xor_si128 (*temp1, *temp2); + *temp2 = _mm_shuffle_epi32(*temp1, 0xff); + temp4 = _mm_slli_si128 (*temp3, 0x4); + *temp3 = _mm_xor_si128 (*temp3, temp4); + *temp3 = _mm_xor_si128 (*temp3, *temp2); +} + +IPP_OWN_DEFN (void, aes192_KeyExpansion_NI, (Ipp8u* keyExp, const Ipp8u* userkey)) +{ + __m128i *pKeySchedule = (__m128i*)keyExp; + + __m128i temp[3]; + /* + temp[0] = temp1 + temp[1] = temp2 + temp[2] = temp3 + */ + + temp[0] = _mm_loadu_si128((__m128i*)userkey); + temp[2] = _mm_cvtsi64_si128(((Ipp64s*)(userkey+16))[0]);//_mm_loadu_si128((__m128i*)(userkey+16)); // read 8 bytes only other are zero + + pKeySchedule[0]=temp[0]; + pKeySchedule[1]=temp[2]; + temp[1]=_mm_aeskeygenassist_si128 (temp[2],0x1); + aes192_assist(&temp[0], &temp[1], &temp[2]); + pKeySchedule[1] = _mm_unpacklo_epi64 (pKeySchedule[1], temp[0]); + pKeySchedule[2] = _mm_alignr_epi8 (temp[2], temp[0], 8); + + temp[1]=_mm_aeskeygenassist_si128 (temp[2],0x2); + aes192_assist(&temp[0], &temp[1], &temp[2]); + pKeySchedule[3]=temp[0]; + pKeySchedule[4]=temp[2]; + temp[1]=_mm_aeskeygenassist_si128 (temp[2],0x4); + aes192_assist(&temp[0], &temp[1], &temp[2]); + pKeySchedule[4] = _mm_unpacklo_epi64(pKeySchedule[4], temp[0]); + pKeySchedule[5] = _mm_alignr_epi8(temp[2], temp[0], 8); + + temp[1]=_mm_aeskeygenassist_si128 (temp[2],0x8); + aes192_assist(&temp[0], &temp[1], &temp[2]); + pKeySchedule[6]=temp[0]; + pKeySchedule[7]=temp[2]; + temp[1]=_mm_aeskeygenassist_si128 (temp[2],0x10); + aes192_assist(&temp[0], &temp[1], &temp[2]); + pKeySchedule[7] = _mm_unpacklo_epi64(pKeySchedule[7], temp[0]); + pKeySchedule[8] = _mm_alignr_epi8(temp[2], temp[0],8); + + temp[1]=_mm_aeskeygenassist_si128 (temp[2],0x20); + aes192_assist(&temp[0], &temp[1], &temp[2]); + pKeySchedule[9]=temp[0]; + pKeySchedule[10]=temp[2]; + temp[1]=_mm_aeskeygenassist_si128 (temp[2],0x40); + aes192_assist(&temp[0], &temp[1], &temp[2]); + pKeySchedule[10] = _mm_unpacklo_epi64(pKeySchedule[10], temp[0]); + pKeySchedule[11] = _mm_alignr_epi8(temp[2], temp[0], 8); + + temp[1]=_mm_aeskeygenassist_si128 (temp[2],0x80); + aes192_assist(&temp[0], &temp[1], &temp[2]); + pKeySchedule[12]=temp[0]; + + /* clear secret data */ + for(Ipp32u i = 0; i < sizeof(temp)/sizeof(temp[0]); i++){ + temp[i] = _mm_xor_si128(temp[i],temp[i]); + } +} + +#endif /* #if (_AES_NI_ENABLING_==_FEATURE_ON_) || (_AES_NI_ENABLING_==_FEATURE_TICKTOCK_) */ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_256keyexpansion_ni.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_256keyexpansion_ni.c new file mode 100644 index 0000000..8f3b047 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_256keyexpansion_ni.c @@ -0,0 +1,129 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. AES keys expansion +// +// Contents: +// aes256_KeyExpansion_NI() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcpaes_keys_ni.h" + + +#if (_AES_NI_ENABLING_==_FEATURE_ON_) || (_AES_NI_ENABLING_==_FEATURE_TICKTOCK_) + +////////////////////////////////////////////////////////////////////// +/* +// AES-256 key expansion +*/ +static void aes256_assist_1(__m128i* temp1, __m128i * temp2) +{ + __m128i temp4; + *temp2 = _mm_shuffle_epi32(*temp2, 0xff); + temp4 = _mm_slli_si128 (*temp1, 0x4); + *temp1 = _mm_xor_si128 (*temp1, temp4); + temp4 = _mm_slli_si128 (temp4, 0x4); + *temp1 = _mm_xor_si128 (*temp1, temp4); + temp4 = _mm_slli_si128 (temp4, 0x4); + *temp1 = _mm_xor_si128 (*temp1, temp4); + *temp1 = _mm_xor_si128 (*temp1, *temp2); +} + +static void aes256_assist_2(__m128i* temp1, __m128i * temp3) +{ + __m128i temp2,temp4; + temp4 = _mm_aeskeygenassist_si128 (*temp1, 0x0); + temp2 = _mm_shuffle_epi32(temp4, 0xaa); + temp4 = _mm_slli_si128 (*temp3, 0x4); + *temp3 = _mm_xor_si128 (*temp3, temp4); + temp4 = _mm_slli_si128 (temp4, 0x4); + *temp3 = _mm_xor_si128 (*temp3, temp4); + temp4 = _mm_slli_si128 (temp4, 0x4); + *temp3 = _mm_xor_si128 (*temp3, temp4); + *temp3 = _mm_xor_si128 (*temp3, temp2); +} + +IPP_OWN_DEFN (void, aes256_KeyExpansion_NI, (Ipp8u* keyExp, const Ipp8u* userkey)) +{ + __m128i *pKeySchedule = (__m128i*)keyExp; + + __m128i temp[3]; + /* + temp[0] = temp1 + temp[1] = temp2 + temp[2] = temp3 + */ + + temp[0] = _mm_loadu_si128((__m128i*)userkey); + temp[2] = _mm_loadu_si128((__m128i*)(userkey+16)); + pKeySchedule[0] = temp[0]; + pKeySchedule[1] = temp[2]; + + temp[1] = _mm_aeskeygenassist_si128 (temp[2],0x01); + aes256_assist_1(&temp[0], &temp[1]); + pKeySchedule[2]=temp[0]; + aes256_assist_2(&temp[0], &temp[2]); + pKeySchedule[3]=temp[2]; + + temp[1] = _mm_aeskeygenassist_si128 (temp[2],0x02); + aes256_assist_1(&temp[0], &temp[1]); + pKeySchedule[4]=temp[0]; + aes256_assist_2(&temp[0], &temp[2]); + pKeySchedule[5]=temp[2]; + + temp[1] = _mm_aeskeygenassist_si128 (temp[2],0x04); + aes256_assist_1(&temp[0], &temp[1]); + pKeySchedule[6]=temp[0]; + aes256_assist_2(&temp[0], &temp[2]); + pKeySchedule[7]=temp[2]; + + temp[1] = _mm_aeskeygenassist_si128 (temp[2],0x08); + aes256_assist_1(&temp[0], &temp[1]); + pKeySchedule[8]=temp[0]; + aes256_assist_2(&temp[0], &temp[2]); + pKeySchedule[9]=temp[2]; + + temp[1] = _mm_aeskeygenassist_si128 (temp[2],0x10); + aes256_assist_1(&temp[0], &temp[1]); + pKeySchedule[10]=temp[0]; + aes256_assist_2(&temp[0], &temp[2]); + pKeySchedule[11]=temp[2]; + + temp[1] = _mm_aeskeygenassist_si128 (temp[2],0x20); + aes256_assist_1(&temp[0], &temp[1]); + pKeySchedule[12]=temp[0]; + aes256_assist_2(&temp[0], &temp[2]); + pKeySchedule[13]=temp[2]; + + temp[1] = _mm_aeskeygenassist_si128 (temp[2],0x40); + aes256_assist_1(&temp[0], &temp[1]); + pKeySchedule[14]=temp[0]; + + /* clear secret data */ + for(Ipp32u i = 0; i < sizeof(temp)/sizeof(temp[0]); i++){ + temp[i] = _mm_xor_si128(temp[i],temp[i]); + } +} + +#endif /* #if (_AES_NI_ENABLING_==_FEATURE_ON_) || (_AES_NI_ENABLING_==_FEATURE_TICKTOCK_) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_avx2_vaes.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_avx2_vaes.h new file mode 100644 index 0000000..284a75c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_avx2_vaes.h @@ -0,0 +1,199 @@ +/******************************************************************************* +* Copyright (C) 2023 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES GCM AVX2 +// Internal Functions Implementations +// +*/ + +#ifndef __AES_GCM_AVX2_H_ +#define __AES_GCM_AVX2_H_ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesauthgcm.h" +#include "pcptool.h" + +#if (_IPP==_IPP_H9) || (_IPP32E==_IPP32E_L9) + +#define MAX_NK 15 //the largest possible number of keys + +#define SHUFD_MASK 78 // 01001110b +#define STEP_SIZE 64 // 4*BLOCK_SIZE, due to block size is always 16 for AES +#define HALF_STEP_SIZE 32 // 2*BLOCK_SIZE, due to block size is always 16 for AES + +//is used to increment two 128-bit words in a 256-bit register +#define IncrementRegister256(t_block, t_incr, t_shuffle_mask) \ + t_block = _mm256_shuffle_epi8(t_block, t_shuffle_mask); \ + t_block = _mm256_add_epi32(t_block, t_incr); \ + t_block = _mm256_shuffle_epi8(t_block, t_shuffle_mask) + +// these constants are used to increment two 128-bit words in a 256-bit register +__ALIGN32 static const Ipp32u _increment2[] = {0, 0, 0, 2, 0, 0, 0, 2}; +__ALIGN32 static const Ipp32u _increment4[] = {0, 0, 0, 4, 0, 0, 0, 4}; +__ALIGN32 static const Ipp8u swapBytes256[] = { + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 31, 30, 29, 28, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 15, 14, 13, 12 +}; + +// shuffle masks +__ALIGN32 static const Ipp8u _shuff_mask_128[] = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; +__ALIGN32 static const Ipp8u _shuff_mask_256[] = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; + +// masks for operations with intrinsics +__ALIGN32 static const Ipp8u _mask_lo_256[] = {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 0, 0, 0, 0, 0, 0, 0}; +__ALIGN32 static const Ipp8u _mask_hi_256[] = {0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}; + +/* +// sse_clmul_gcm performs clmul with 128-bit registers; is used in the combine hash step +// input: +// const __m128i *HK - containts hashed keys +// input/output: +// __m128i *GH - contains GHASH. Will be overwritten in this function +*/ +__INLINE void sse_clmul_gcm(__m128i *GH, const __m128i *HK) { + __m128i tmpX0, tmpX1, tmpX2, tmpX3; + tmpX2 = _mm_shuffle_epi32 (*GH, SHUFD_MASK); //tmpX2 = {GH0:GH1} + tmpX0 = _mm_shuffle_epi32 (*HK, SHUFD_MASK); //tmpX0 = {HK0:HK1} + tmpX2 = _mm_xor_si128(tmpX2, *GH); //tmpX2 = {GH0+GH1:GH1+GH0} + tmpX0 = _mm_xor_si128(tmpX0, *HK); //tmpX0 = {HK0+HK1:HK1+HK0} + tmpX2 = _mm_clmulepi64_si128 (tmpX2, tmpX0, 0x00); //tmpX2 = (a1+a0)*(b1+b0); tmpX2 = (GH1+GH0)*(HK1+HK0) + tmpX1 = *GH; + *GH = _mm_clmulepi64_si128 (*GH, *HK, 0x00); //GH = a0*b0; GH = GH0*HK0 + tmpX0 = _mm_xor_si128(tmpX0, tmpX0); + tmpX1 = _mm_clmulepi64_si128 (tmpX1, *HK, 0x11); //tmpX1 = a1*b1; tmpX1 = GH1*HK1 + tmpX2 = _mm_xor_si128(tmpX2, *GH); //tmpX2 = (GH1+GH0)*(HK1+HK0) + GH0*HK0 + tmpX2 = _mm_xor_si128(tmpX2, tmpX1); //tmpX2 = a0*b1+a1*b0; tmpX2 = (GH1+GH0)*(HK1+HK0) + GH0*HK0 + GH1*HK1 = GH0*HK1+GH1*HK0 + tmpX0 = _mm_alignr_epi8 (tmpX0, tmpX2, 8); //tmpX0 = {Zeros : HI(a0*b1+a1*b0)} + tmpX2 = _mm_slli_si128 (tmpX2, 8); //tmpX2 = {LO(HI(a0*b1+a1*b0)) : Zeros} + tmpX1 = _mm_xor_si128(tmpX1, tmpX0); // holds the result of the carry-less multiplication of GH by HK + *GH = _mm_xor_si128(*GH, tmpX2); + + //first phase of the reduction + tmpX0 = *GH; //copy GH into tmpX0, tmpX2, tmpX3 + tmpX2 = *GH; + tmpX3 = *GH; + tmpX0 = _mm_slli_epi64 (tmpX0, 63); //packed left shifting << 63 + tmpX2 = _mm_slli_epi64 (tmpX2, 62); //packed left shifting shift << 62 + tmpX3 = _mm_slli_epi64 (tmpX3, 57); //packed left shifting shift << 57 + tmpX0 = _mm_xor_si128(tmpX0, tmpX2); //xor the shifted versions + tmpX0 = _mm_xor_si128(tmpX0, tmpX3); + tmpX2 = tmpX0; + tmpX2 = _mm_slli_si128 (tmpX2, 8); //shift-L tmpX2 2 DWs + tmpX0 = _mm_srli_si128 (tmpX0, 8); //shift-R xmm2 2 DWs + *GH = _mm_xor_si128(*GH, tmpX2); //first phase of the reduction complete + tmpX1 = _mm_xor_si128(tmpX1, tmpX0); //save the lost MS 1-2-7 bits from first phase + + //second phase of the reduction + tmpX2 = *GH; + tmpX2 = _mm_srli_epi64(tmpX2, 5); //packed right shifting >> 5 + tmpX2 = _mm_xor_si128(tmpX2, *GH); //xor shifted versions + tmpX2 = _mm_srli_epi64(tmpX2, 1); //packed right shifting >> 1 + tmpX2 = _mm_xor_si128(tmpX2, *GH); //xor shifted versions + tmpX2 = _mm_srli_epi64(tmpX2, 1); //packed right shifting >> 1 + *GH = _mm_xor_si128(*GH, tmpX2); //second phase of the reduction complete + *GH = _mm_xor_si128(*GH, tmpX1); //the result is in GH +} + +/* +// avx2_clmul_gcm performs clmul with 256-bit registers; is used in the hash calculation step +// input: +// const __m128i *HK - containts hashed keys +// const __m256i *HKeyKaratsuba - countains temporary data for Karatsuba method +// const __m256i *mask_lo - contains mask for taking lower bits +// const __m256i *mask_hi - contains mask for taking higher bits +// input/output: +// __m128i *GH - contains GHASH. Will be overwritten in this function +*/ +__INLINE void avx2_clmul_gcm(__m256i *GH, const __m256i *HK, const __m256i *HKeyKaratsuba, const __m256i *mask_lo, const __m256i *mask_hi) { + __m256i tmpX0, tmpX1, tmpX2; + + tmpX2 = _mm256_shuffle_epi32 (*GH, SHUFD_MASK); + // Karatsuba Method + tmpX1 = *GH; + tmpX2 = _mm256_xor_si256(tmpX2, *GH); + *GH = _mm256_clmulepi64_epi128(*GH, *HK, 0x00); + // Karatsuba Method + + tmpX1 = _mm256_clmulepi64_epi128(tmpX1, *HK, 0x11); + tmpX2 = _mm256_clmulepi64_epi128(tmpX2, *HKeyKaratsuba, 0x00); + tmpX2 = _mm256_xor_si256(tmpX2, *GH); + tmpX2 = _mm256_xor_si256(tmpX2, tmpX1); + tmpX0 = _mm256_shuffle_epi32 (tmpX2, SHUFD_MASK); + tmpX2 = tmpX0; + tmpX0 = _mm256_and_si256(tmpX0, *mask_hi); + tmpX2 = _mm256_and_si256(tmpX2, *mask_lo); + *GH = _mm256_xor_si256(*GH, tmpX0); + tmpX1 = _mm256_xor_si256(tmpX1, tmpX2); + + // first phase of the reduction + tmpX0 = *GH; + *GH = _mm256_slli_epi64 (*GH, 1); + *GH = _mm256_xor_si256(*GH, tmpX0); + *GH = _mm256_slli_epi64 (*GH, 5); + *GH = _mm256_xor_si256(*GH, tmpX0); + *GH = _mm256_slli_epi64 (*GH, 57); + tmpX2 = _mm256_shuffle_epi32(*GH, SHUFD_MASK); + *GH = tmpX2; + tmpX2 = _mm256_and_si256(tmpX2, *mask_lo); + *GH = _mm256_and_si256(*GH, *mask_hi); + *GH = _mm256_xor_si256(*GH, tmpX0); + tmpX1 = _mm256_xor_si256(tmpX1, tmpX2); + + // second phase of the reduction + tmpX2 = *GH; + *GH = _mm256_srli_epi64(*GH, 5); + *GH = _mm256_xor_si256(*GH, tmpX2); + *GH = _mm256_srli_epi64(*GH, 1); + *GH = _mm256_xor_si256(*GH, tmpX2); + *GH = _mm256_srli_epi64(*GH, 1); + *GH = _mm256_xor_si256(*GH, tmpX2); + *GH = _mm256_xor_si256(*GH, tmpX1); +} + +/* +// aes_encoder_avx2vaes_sb is used for single block encryption +// input: +// const Ipp8u *in - contains data for encryprion +// const int Nr - contains number of the rounds +// const __m256i* keys - contains keys +// output: +// Ipp8u *out - stores encrypted data. +*/ +__INLINE void aes_encoder_avx2vaes_sb(const Ipp8u *in, Ipp8u *out, const int Nr, const __m256i* keys) { + __m128i lo = _mm_loadu_si128((void*)in); + __m128i hi = _mm_setzero_si128(); + __m256i block = _mm256_setr_m128i(lo, hi); + block = _mm256_xor_si256(block, *keys); + for(int round = 1; round < Nr; round++) { + keys++; + block = _mm256_aesenc_epi128(block, *keys); + } + keys++; + block = _mm256_aesenclast_epi128(block, *keys); + _mm_storeu_si128((void*)out, _mm256_castsi256_si128(block)); +} + +#endif /* #if(_IPP==_IPP_H9) || (_IPP32E==_IPP32E_L9) */ + +#endif /* __AES_GCM_AVX2_H_ */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_avx2_vaes_decrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_avx2_vaes_decrypt.c new file mode 100644 index 0000000..884bf37 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_avx2_vaes_decrypt.c @@ -0,0 +1,188 @@ +/******************************************************************************* +* Copyright (C) 2023 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES decryption (GCM mode) +// +*/ + +#include "pcpaes_avx2_vaes.h" + +#if (_IPP==_IPP_H9) || (_IPP32E==_IPP32E_L9) + +IPP_OWN_DEFN (void, AesGcmDec_vaes_avx2, (Ipp8u* pDst, const Ipp8u* pSrc, int len, IppsAES_GCMState* pState)) +{ + const int nloop = len / STEP_SIZE; + IppsRijndael128Spec* pAES = AESGCM_CIPHER(pState); + Ipp8u* pCounter = AESGCM_COUNTER(pState); + Ipp8u* pECounter = AESGCM_ECOUNTER(pState); + __m256i pCounter256, pCounter256_1, pECounter256, pECounter256_1; + __m256i block, block1, cipherText, cipherText_1, plainText, plainText_1; + + // setting temporary data for incremention + const __m256i increment2 = _mm256_loadu_si256((void*)_increment2); // increment by 2 + const __m256i increment4 = _mm256_loadu_si256((void*)_increment4); // increment by 4 + const __m256i shuffle_mask = _mm256_loadu_si256((void*)swapBytes256); + + // loading keys from memory + __m256i rkeys[MAX_NK]; + __m128i tmp_keys_128; + for (int i = 0; i < RIJ_NR(pAES) + 1; i++) { + tmp_keys_128 = _mm_loadu_si128((void*)(RIJ_EKEYS(pAES)+i*16)); + rkeys[i] = _mm256_setr_m128i(tmp_keys_128, tmp_keys_128); + } + + // skip extra calculations if plaintext less than 4 blocks + if (nloop) { + // loading counters from memory + __m128i lo, hi; + lo = _mm_loadu_si128((void*)pCounter); + IncrementCounter32(pCounter); + hi = _mm_loadu_si128((void*)pCounter); + pCounter256_1 = _mm256_setr_m128i(lo, hi); + pCounter256 = pCounter256_1; + IncrementRegister256(pCounter256_1, increment2, shuffle_mask); + + // setting some masks + const __m128i shuff_mask_128 = _mm_loadu_si128((void*)_shuff_mask_128); + const __m256i shuff_mask_256 = _mm256_loadu_si256((void*)_shuff_mask_256); + const __m256i mask_lo_256 = _mm256_loadu_si256((void*)_mask_lo_256); + const __m256i mask_hi_256 = _mm256_loadu_si256((void*)_mask_hi_256); + + lo = _mm_loadu_si128((void*)AESGCM_GHASH(pState)); + hi = _mm_setzero_si128(); + __m256i rpHash0 = _mm256_setr_m128i(_mm_shuffle_epi8(lo, shuff_mask_128), hi); + __m256i rpHash1 = _mm256_setzero_si256(); + + // setting pre-calculated data for hash combining + Ipp8u *pkeys = AESGCM_HKEY(pState); + __m128i HashKey0 = _mm_loadu_si128((void*)pkeys); + pkeys += 16; + __m128i HashKey2 = _mm_loadu_si128((void*)pkeys); + pkeys += 16; + __m128i HashKey4 = _mm_loadu_si128((void*)pkeys); + + // setting pre-calculated data in correct order for Karatsuba method + __m256i HKey = _mm256_setr_m128i(HashKey4, HashKey4); + __m256i HKeyKaratsuba = _mm256_shuffle_epi32(HKey, SHUFD_MASK); + HKeyKaratsuba = _mm256_xor_si256(HKey, HKeyKaratsuba); + do { + // decrypt stage + block = _mm256_xor_si256(pCounter256, *rkeys); + block1 = _mm256_xor_si256(pCounter256_1, *rkeys); + block = _mm256_aesenc_epi128(block, *(rkeys+1)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+1)); + block = _mm256_aesenc_epi128(block, *(rkeys+2)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+2)); + block = _mm256_aesenc_epi128(block, *(rkeys+3)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+3)); + IncrementRegister256(pCounter256, increment4, shuffle_mask); + block = _mm256_aesenc_epi128(block, *(rkeys+4)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+4)); + block = _mm256_aesenc_epi128(block, *(rkeys+5)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+5)); + block = _mm256_aesenc_epi128(block, *(rkeys+6)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+6)); + block = _mm256_aesenc_epi128(block, *(rkeys+7)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+7)); + block = _mm256_aesenc_epi128(block, *(rkeys+8)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+8)); + block = _mm256_aesenc_epi128(block, *(rkeys+9)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+9)); + IncrementRegister256(pCounter256_1, increment4, shuffle_mask); + if (RIJ_NR(pAES) >= 12) { + block = _mm256_aesenc_epi128(block, *(rkeys+10)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+10)); + block = _mm256_aesenc_epi128(block, *(rkeys+11)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+11)); + if (RIJ_NR(pAES) >= 14) { + block = _mm256_aesenc_epi128(block, *(rkeys+12)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+12)); + block = _mm256_aesenc_epi128(block, *(rkeys+13)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+13)); + } + } + pECounter256 = _mm256_aesenclast_epi128(block, *(rkeys+RIJ_NR(pAES))); + pECounter256_1 = _mm256_aesenclast_epi128(block1, *(rkeys+RIJ_NR(pAES))); + + // set ciphertext + plainText = _mm256_loadu_si256((void*)pSrc); + cipherText = _mm256_xor_si256(plainText, pECounter256); + pSrc += HALF_STEP_SIZE; + plainText_1 = _mm256_loadu_si256((void*)pSrc); + cipherText_1 = _mm256_xor_si256(plainText_1, pECounter256_1); + pSrc += HALF_STEP_SIZE; + + // hash calculation stage + rpHash0 = _mm256_xor_si256(rpHash0, _mm256_shuffle_epi8(plainText, shuff_mask_256)); + _mm256_storeu_si256((void*)pDst, cipherText); + pDst += HALF_STEP_SIZE; + _mm256_storeu_si256((void*)pDst, cipherText_1); + pDst += HALF_STEP_SIZE; + rpHash1 = _mm256_xor_si256(rpHash1, _mm256_shuffle_epi8(plainText_1, shuff_mask_256)); + len -= STEP_SIZE; + if (len >= STEP_SIZE) { + avx2_clmul_gcm(&rpHash0, &HKey, &HKeyKaratsuba, &mask_lo_256, &mask_hi_256); + avx2_clmul_gcm(&rpHash1, &HKey, &HKeyKaratsuba, &mask_lo_256, &mask_hi_256); + } + } while(len >= STEP_SIZE); + + // loading temporary data to memory + _mm_storeu_si128((void*)pECounter, _mm256_extractf128_si256(pECounter256, 1)); + _mm_storeu_si128((void*)pCounter, _mm256_castsi256_si128(pCounter256)); + + // combine hash + __m128i GHash0 = _mm256_extractf128_si256(rpHash0, 0); + __m128i GHash1 = _mm256_extractf128_si256(rpHash0, 1); + __m128i GHash2 = _mm256_extractf128_si256(rpHash1, 0); + __m128i GHash3 = _mm256_extractf128_si256(rpHash1, 1); + + sse_clmul_gcm(&GHash0, &HashKey4); //GHash0 = GHash0 * (HashKey^4)<<1 mod poly + sse_clmul_gcm(&GHash1, &HashKey2); //GHash1 = GHash1 * (HashKey^2)<<1 mod poly + sse_clmul_gcm(&GHash2, &HashKey0); //GHash2 = GHash2 * (HashKey^1)<<1 mod poly + GHash3 = _mm_xor_si128(GHash3, GHash1); + GHash3 = _mm_xor_si128(GHash3, GHash2); + + sse_clmul_gcm(&GHash3, &HashKey0); //GHash3 = GHash3 * (HashKey)<<1 mod poly + GHash3 = _mm_xor_si128(GHash3, GHash0); + GHash3 = _mm_shuffle_epi8(GHash3, shuff_mask_128); + _mm_storeu_si128((void*)(AESGCM_GHASH(pState)), GHash3); + } + + const Ipp8u* pHashedData = pSrc; + int hashedDataLen = len; + + // decryption for the tail (1-3 blocks) + while(len >= BLOCK_SIZE) { + aes_encoder_avx2vaes_sb(pCounter, pECounter, RIJ_NR(pAES), rkeys); + XorBlock16(pSrc, pECounter, pDst); + pSrc += BLOCK_SIZE; + pDst += BLOCK_SIZE; + len -= BLOCK_SIZE; + IncrementCounter32(pCounter); + } + aes_encoder_avx2vaes_sb(pCounter, pECounter, RIJ_NR(pAES), rkeys); + + // hash calculation for the tail (1-3 blocks) + if (hashedDataLen >= BLOCK_SIZE) + AesGcmAuth_avx(AESGCM_GHASH(pState), pHashedData, hashedDataLen, AESGCM_HKEY(pState), AesGcmConst_table); +} + +#endif /* #if (_IPP==_IPP_H9) || (_IPP32E==_IPP32E_L9) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_avx2_vaes_encrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_avx2_vaes_encrypt.c new file mode 100644 index 0000000..bae8a2d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_avx2_vaes_encrypt.c @@ -0,0 +1,186 @@ +/******************************************************************************* +* Copyright (C) 2023 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption (GCM mode) +// +*/ + +#include "pcpaes_avx2_vaes.h" + +#if (_IPP==_IPP_H9) || (_IPP32E==_IPP32E_L9) + +IPP_OWN_DEFN (void, AesGcmEnc_vaes_avx2, (Ipp8u* pDst, const Ipp8u* pSrc, int len, IppsAES_GCMState* pState)) +{ + const int nloop = len / STEP_SIZE; + IppsRijndael128Spec* pAES = AESGCM_CIPHER(pState); + Ipp8u* pCounter = AESGCM_COUNTER(pState); + Ipp8u* pECounter = AESGCM_ECOUNTER(pState); + __m256i pCounter256, pCounter256_1, pECounter256, pECounter256_1; + __m256i block, block1, cipherText, cipherText_1; + + // setting temporary data for incremention + const __m256i increment2 = _mm256_loadu_si256((void*)_increment2); // increment by 2 + const __m256i increment4 = _mm256_loadu_si256((void*)_increment4); // increment by 4 + const __m256i shuffle_mask = _mm256_loadu_si256((void*)swapBytes256); + + // loading keys from memory + __m256i rkeys[MAX_NK]; + __m128i tmp_keys_128; + for (int i = 0; i < RIJ_NR(pAES) + 1; i++) { + tmp_keys_128 = _mm_loadu_si128((void*)(RIJ_EKEYS(pAES)+i*16)); + rkeys[i] = _mm256_setr_m128i(tmp_keys_128, tmp_keys_128); + } + + // skip extra calculations if plaintext less than 4 blocks + if (nloop) { + // loading counters from memory + __m128i lo, hi; + lo = _mm_loadu_si128((void*)pCounter); + IncrementCounter32(pCounter); + hi = _mm_loadu_si128((void*)pCounter); + pCounter256_1 = _mm256_setr_m128i(lo, hi); + pCounter256 = pCounter256_1; + IncrementRegister256(pCounter256_1, increment2, shuffle_mask); + + // setting some masks + const __m128i shuff_mask_128 = _mm_loadu_si128((void*)_shuff_mask_128); + const __m256i shuff_mask_256 = _mm256_loadu_si256((void*)_shuff_mask_256); + const __m256i mask_lo_256 = _mm256_loadu_si256((void*)_mask_lo_256); + const __m256i mask_hi_256 = _mm256_loadu_si256((void*)_mask_hi_256); + + lo = _mm_loadu_si128((__m128i*)AESGCM_GHASH(pState)); + hi = _mm_setzero_si128(); + __m256i rpHash0 = _mm256_setr_m128i(_mm_shuffle_epi8(lo, shuff_mask_128), hi); + __m256i rpHash1 = _mm256_setzero_si256(); + + // setting pre-calculated data for hash combining + Ipp8u *pkeys = AESGCM_HKEY(pState); + __m128i HashKey0 = _mm_loadu_si128((void*)pkeys); + pkeys += 16; + __m128i HashKey2 = _mm_loadu_si128((void*)pkeys); + pkeys += 16; + __m128i HashKey4 = _mm_loadu_si128((void*)pkeys); + + // setting pre-calculated data in correct order for Karatsuba method + __m256i HKey = _mm256_setr_m128i(HashKey4, HashKey4); + __m256i HKeyKaratsuba = _mm256_shuffle_epi32(HKey, SHUFD_MASK); + HKeyKaratsuba = _mm256_xor_si256(HKey, HKeyKaratsuba); + do { + // encrypt stage + block = _mm256_xor_si256(pCounter256, *rkeys); + block1 = _mm256_xor_si256(pCounter256_1, *rkeys); + block = _mm256_aesenc_epi128(block, *(rkeys+1)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+1)); + block = _mm256_aesenc_epi128(block, *(rkeys+2)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+2)); + block = _mm256_aesenc_epi128(block, *(rkeys+3)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+3)); + IncrementRegister256(pCounter256, increment4, shuffle_mask); + block = _mm256_aesenc_epi128(block, *(rkeys+4)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+4)); + block = _mm256_aesenc_epi128(block, *(rkeys+5)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+5)); + block = _mm256_aesenc_epi128(block, *(rkeys+6)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+6)); + block = _mm256_aesenc_epi128(block, *(rkeys+7)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+7)); + block = _mm256_aesenc_epi128(block, *(rkeys+8)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+8)); + block = _mm256_aesenc_epi128(block, *(rkeys+9)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+9)); + IncrementRegister256(pCounter256_1, increment4, shuffle_mask); + if (RIJ_NR(pAES) >= 12) { + block = _mm256_aesenc_epi128(block, *(rkeys+10)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+10)); + block = _mm256_aesenc_epi128(block, *(rkeys+11)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+11)); + if (RIJ_NR(pAES) >= 14) { + block = _mm256_aesenc_epi128(block, *(rkeys+12)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+12)); + block = _mm256_aesenc_epi128(block, *(rkeys+13)); + block1 = _mm256_aesenc_epi128(block1, *(rkeys+13)); + } + } + pECounter256 = _mm256_aesenclast_epi128(block, *(rkeys+RIJ_NR(pAES))); + pECounter256_1 = _mm256_aesenclast_epi128(block1, *(rkeys+RIJ_NR(pAES))); + + // set ciphertext + cipherText = _mm256_xor_si256( _mm256_loadu_si256((void*)pSrc), pECounter256); + pSrc += HALF_STEP_SIZE; + cipherText_1 = _mm256_xor_si256( _mm256_loadu_si256((void*)pSrc), pECounter256_1); + pSrc += HALF_STEP_SIZE; + + // hash calculation stage + rpHash0 = _mm256_xor_si256(rpHash0, _mm256_shuffle_epi8(cipherText, shuff_mask_256)); + _mm256_storeu_si256((void*)pDst, cipherText); + pDst += HALF_STEP_SIZE; + _mm256_storeu_si256((void*)pDst, cipherText_1); + pDst += HALF_STEP_SIZE; + rpHash1 = _mm256_xor_si256(rpHash1, _mm256_shuffle_epi8(cipherText_1, shuff_mask_256)); + len -= STEP_SIZE; + if (len >= STEP_SIZE) { + avx2_clmul_gcm(&rpHash0, &HKey, &HKeyKaratsuba, &mask_lo_256, &mask_hi_256); + avx2_clmul_gcm(&rpHash1, &HKey, &HKeyKaratsuba, &mask_lo_256, &mask_hi_256); + } + } while(len >= STEP_SIZE); + + // loading temporary data to memory + _mm_storeu_si128((void*)pECounter, _mm256_extractf128_si256(pECounter256, 1)); + _mm_storeu_si128((void*)pCounter, _mm256_castsi256_si128(pCounter256)); + + // combine hash + __m128i GHash0 = _mm256_extractf128_si256(rpHash0, 0); + __m128i GHash1 = _mm256_extractf128_si256(rpHash0, 1); + __m128i GHash2 = _mm256_extractf128_si256(rpHash1, 0); + __m128i GHash3 = _mm256_extractf128_si256(rpHash1, 1); + + sse_clmul_gcm(&GHash0, &HashKey4); //GHash0 = GHash0 * (HashKey^4)<<1 mod poly + sse_clmul_gcm(&GHash1, &HashKey2); //GHash1 = GHash1 * (HashKey^2)<<1 mod poly + sse_clmul_gcm(&GHash2, &HashKey0); //GHash2 = GHash2 * (HashKey^1)<<1 mod poly + GHash3 = _mm_xor_si128(GHash3, GHash1); + GHash3 = _mm_xor_si128(GHash3, GHash2); + + sse_clmul_gcm(&GHash3, &HashKey0); //GHash3 = GHash3 * (HashKey)<<1 mod poly + GHash3 = _mm_xor_si128(GHash3, GHash0); + GHash3 = _mm_shuffle_epi8(GHash3, shuff_mask_128); + _mm_storeu_si128((void*)(AESGCM_GHASH(pState)), GHash3); + } + + Ipp8u* pHashedData = pDst; + int hashedDataLen = len; + + // encryption for the tail (1-3 blocks) + while(len >= BLOCK_SIZE) { + aes_encoder_avx2vaes_sb(pCounter, pECounter, RIJ_NR(pAES), rkeys); + XorBlock16(pSrc, pECounter, pDst); + pSrc += BLOCK_SIZE; + pDst += BLOCK_SIZE; + len -= BLOCK_SIZE; + IncrementCounter32(pCounter); + } + aes_encoder_avx2vaes_sb(pCounter, pECounter, RIJ_NR(pAES), rkeys); + + // hash calculation for the tail (1-3 blocks) + if (hashedDataLen >= BLOCK_SIZE) + AesGcmAuth_avx(AESGCM_GHASH(pState), pHashedData, hashedDataLen, AESGCM_HKEY(pState), AesGcmConst_table); +} + +#endif /* #if (_IPP==_IPP_H9) || (_IPP32E==_IPP32E_L9) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cbc_decrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cbc_decrypt.c new file mode 100644 index 0000000..1abde00 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cbc_decrypt.c @@ -0,0 +1,131 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES decryption (CBC mode) +// AES decryption (CBC-CS mode) +// +// Contents: +// cpDecryptAES_cbc() +// +*/ + +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" +#include "pcpaes_cbc_decrypt.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + +/* +// AES-CBC decryption +// +// Parameters: +// pIV pointer to the initialization vector +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// nBlocks number of decrypted data blocks +// pCtx pointer to the AES context +*/ +IPP_OWN_DEFN (void, cpDecryptAES_cbc, (const Ipp8u* pIV, const Ipp8u* pSrc, Ipp8u* pDst, int nBlocks, const IppsAESSpec* pCtx)) +{ +#if(_IPP32E>=_IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512VAES)) { + DecryptCBC_RIJ128pipe_VAES_NI(pSrc, pDst, nBlocks*MBS_RIJ128, pCtx, pIV); + } + else +#endif +#if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + /* use pipelined version is possible */ + if(AES_NI_ENABLED==RIJ_AESNI(pCtx)) { + DecryptCBC_RIJ128pipe_AES_NI(pSrc, pDst, RIJ_NR(pCtx), RIJ_DKEYS(pCtx), nBlocks*MBS_RIJ128, pIV); + } + else +#endif + { + /* setup decoder method */ + RijnCipher decoder = RIJ_DECODER(pCtx); + + Ipp32u iv[NB(128)]; + + /* copy IV */ + CopyBlock16(pIV, iv); + + /* not inplace block-by-block decryption */ + if(pSrc != pDst) { + while(nBlocks) { + //decoder((const Ipp32u*)pSrc, (Ipp32u*)pDst, RIJ_NR(pCtx), RIJ_DKEYS(pCtx), (const Ipp32u (*)[256])RIJ_DEC_SBOX(pCtx)); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + decoder(pSrc, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), RijDecSbox/*NULL*/); + #else + decoder(pSrc, pDst, RIJ_NR(pCtx), RIJ_DKEYS(pCtx), NULL); + #endif + + ((Ipp32u*)pDst)[0] ^= iv[0]; + ((Ipp32u*)pDst)[1] ^= iv[1]; + ((Ipp32u*)pDst)[2] ^= iv[2]; + ((Ipp32u*)pDst)[3] ^= iv[3]; + + iv[0] = ((Ipp32u*)pSrc)[0]; + iv[1] = ((Ipp32u*)pSrc)[1]; + iv[2] = ((Ipp32u*)pSrc)[2]; + iv[3] = ((Ipp32u*)pSrc)[3]; + + pSrc += MBS_RIJ128; + pDst += MBS_RIJ128; + nBlocks--; + } + } + /* inplace block-by-block decryption */ + else { + Ipp32u tmpOut[NB(128)]; + + while(nBlocks) { + //decoder(pSrc, tmpOut, RIJ_NR(pCtx), RIJ_DKEYS(pCtx), (const Ipp32u (*)[256])RIJ_DEC_SBOX(pCtx)); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + decoder(pSrc, (Ipp8u*)tmpOut, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), RijDecSbox/*NULL*/); + #else + decoder(pSrc, (Ipp8u*)tmpOut, RIJ_NR(pCtx), RIJ_DKEYS(pCtx), NULL); + #endif + + tmpOut[0] ^= iv[0]; + tmpOut[1] ^= iv[1]; + tmpOut[2] ^= iv[2]; + tmpOut[3] ^= iv[3]; + + iv[0] = ((Ipp32u*)pSrc)[0]; + iv[1] = ((Ipp32u*)pSrc)[1]; + iv[2] = ((Ipp32u*)pSrc)[2]; + iv[3] = ((Ipp32u*)pSrc)[3]; + + CopyBlock16(tmpOut, pDst); + + pSrc += MBS_RIJ128; + pDst += MBS_RIJ128; + nBlocks--; + } + + /* clear secret data */ + PurgeBlock(tmpOut, sizeof(tmpOut)); + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cbc_decrypt.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cbc_decrypt.h new file mode 100644 index 0000000..d48ee31 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cbc_decrypt.h @@ -0,0 +1,39 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES decryption (CBC mode) +// AES decryption (CBC-CS mode) +// +// Contents: +// cpDecryptAES_cbc() +// +*/ + +#include "owndefs.h" +#include "owncp.h" + +#if !defined(_PCP_AES_CBC_DECRYPT_H_) +#define _PCP_AES_CBC_DECRYPT_H_ + +#define cpDecryptAES_cbc OWNAPI(cpDecryptAES_cbc) + IPP_OWN_DECL (void, cpDecryptAES_cbc, (const Ipp8u* pIV, const Ipp8u* pSrc, Ipp8u* pDst, int nBlocks, const IppsAESSpec* pCtx)) + +#endif /* #if !defined(_PCP_AES_CBC_DECRYPT_H_) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cbc_encrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cbc_encrypt.c new file mode 100644 index 0000000..293c464 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cbc_encrypt.c @@ -0,0 +1,89 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption (CBC mode) +// AES encryption (CBC-CS mode) +// +// Contents: +// cpEncryptAES_cbc() +// +*/ + +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" +#include "pcpaes_cbc_encrypt.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + +/* +// AES-CBC ecnryption +// +// Parameters: +// pIV pointer to the initialization vector +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// nBlocks number of ecnrypted data blocks +// pCtx pointer to the AES context +*/ +IPP_OWN_DEFN (void, cpEncryptAES_cbc, (const Ipp8u* pIV, const Ipp8u* pSrc, Ipp8u* pDst, int nBlocks, const IppsAESSpec* pCtx)) +{ +#if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + if(AES_NI_ENABLED==RIJ_AESNI(pCtx)) { + EncryptCBC_RIJ128_AES_NI(pSrc, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), nBlocks*MBS_RIJ128, pIV); + } + else +#endif + { + /* setup encoder method */ + RijnCipher encoder = RIJ_ENCODER(pCtx); + + /* read IV */ + Ipp32u iv[NB(128)]; + CopyBlock16(pIV, iv); + + /* block-by-block encryption */ + while(nBlocks) { + iv[0] ^= ((Ipp32u*)pSrc)[0]; + iv[1] ^= ((Ipp32u*)pSrc)[1]; + iv[2] ^= ((Ipp32u*)pSrc)[2]; + iv[3] ^= ((Ipp32u*)pSrc)[3]; + + //encoder(iv, (Ipp32u*)pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), (const Ipp32u (*)[256])RIJ_ENC_SBOX(pCtx)); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder((Ipp8u*)iv, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), RijEncSbox/*NULL*/); + #else + encoder((Ipp8u*)iv, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), NULL); + #endif + + iv[0] = ((Ipp32u*)pDst)[0]; + iv[1] = ((Ipp32u*)pDst)[1]; + iv[2] = ((Ipp32u*)pDst)[2]; + iv[3] = ((Ipp32u*)pDst)[3]; + + pSrc += MBS_RIJ128; + pDst += MBS_RIJ128; + nBlocks--; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cbc_encrypt.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cbc_encrypt.h new file mode 100644 index 0000000..40d6127 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cbc_encrypt.h @@ -0,0 +1,39 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption (CBC mode) +// AES encryption (CBC-CS mode) +// +// Contents: +// cpEncryptAES_cbc() +// +*/ + +#include "owndefs.h" +#include "owncp.h" + +#if !defined(_PCP_AES_CBC_ENCRYPT_H_) +#define _PCP_AES_CBC_ENCRYPT_H_ + +#define cpEncryptAES_cbc OWNAPI(cpEncryptAES_cbc) + IPP_OWN_DECL (void, cpEncryptAES_cbc, (const Ipp8u* pIV, const Ipp8u* pSrc, Ipp8u* pDst, int nBlocks, const IppsAESSpec* pCtx)) + +#endif /* #if !defined(_PCP_AES_CBC_ENCRYPT_H_) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cbc_vaes512.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cbc_vaes512.c new file mode 100644 index 0000000..2e9accc --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cbc_vaes512.c @@ -0,0 +1,170 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption/decryption (CBC mode) +// +// Contents: +// DecryptCBC_RIJ128pipe_VAES_NI() +// +// +*/ + +#include "owncp.h" +#include "pcpaesm.h" +#include "pcpaes_decrypt_vaes512.h" + +#if(_IPP32E>=_IPP32E_K1) +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(disable: 4310) // zmmintrin.h bug: truncation of constant value +#endif +//////////////////////////////////////////////////////////////////////////////// + +IPP_OWN_DEFN (void, DecryptCBC_RIJ128pipe_VAES_NI, (const Ipp8u* pSrc, // pointer to the ciphertext + Ipp8u* pDst, // pointer to the plaintext + int len, // message length + const IppsAESSpec* pCtx, // pointer to context + const Ipp8u* pIV)) // pointer to the Initialization Vector +{ + int cipherRounds = RIJ_NR(pCtx) - 1; + + __m128i* pRkey = (__m128i*)RIJ_DKEYS(pCtx) + cipherRounds + 1; + __m512i* pSrc512 = (__m512i*)pSrc; + __m512i* pDst512 = (__m512i*)pDst; + __m512i* pIV512 = (__m512i*)pIV; + + // load IV + __m512i IV = _mm512_maskz_expandloadu_epi64(0xC0, pIV512); + + int blocks; + // 4 blocks of 128-bit can be loaded into one zmm register + // assuming that vaesdec latency is 4, we need 4 zmm registers to make effective pipeline + for (blocks = len / MBS_RIJ128; blocks >= (4 * 4); blocks -= (4 * 4)) { + __m512i blk0 = _mm512_loadu_si512(pSrc512); + __m512i blk1 = _mm512_loadu_si512(pSrc512 + 1); + __m512i blk2 = _mm512_loadu_si512(pSrc512 + 2); + __m512i blk3 = _mm512_loadu_si512(pSrc512 + 3); + + // prepare blocks for the last xor + __m512i z0 = _mm512_alignr_epi64(blk0, IV, 6); + __m512i z1 = _mm512_alignr_epi64(blk1, blk0, 6); + __m512i z2 = _mm512_alignr_epi64(blk2, blk1, 6); + __m512i z3 = _mm512_alignr_epi64(blk3, blk2, 6); + + // update IV + IV = blk3; + + cpAESDecrypt4_VAES_NI(&blk0, &blk1, &blk2, &blk3, pRkey, cipherRounds); + + // the last xor + blk0 = _mm512_xor_si512(blk0, z0); + blk1 = _mm512_xor_si512(blk1, z1); + blk2 = _mm512_xor_si512(blk2, z2); + blk3 = _mm512_xor_si512(blk3, z3); + + _mm512_storeu_si512(pDst512, blk0); + _mm512_storeu_si512(pDst512 + 1, blk1); + _mm512_storeu_si512(pDst512 + 2, blk2); + _mm512_storeu_si512(pDst512 + 3, blk3); + + pSrc512 += 4; + pDst512 += 4; + } + + if ((3 * 4) <= blocks) { + __m512i blk0 = _mm512_loadu_si512(pSrc512); + __m512i blk1 = _mm512_loadu_si512(pSrc512 + 1); + __m512i blk2 = _mm512_loadu_si512(pSrc512 + 2); + + __m512i z0 = _mm512_alignr_epi64(blk0, IV, 6); + __m512i z1 = _mm512_alignr_epi64(blk1, blk0, 6); + __m512i z2 = _mm512_alignr_epi64(blk2, blk1, 6); + + // update IV + IV = blk2; + + cpAESDecrypt3_VAES_NI(&blk0, &blk1, &blk2, pRkey, cipherRounds); + + blk0 = _mm512_xor_si512(blk0, z0); + blk1 = _mm512_xor_si512(blk1, z1); + blk2 = _mm512_xor_si512(blk2, z2); + + _mm512_storeu_si512(pDst512, blk0); + _mm512_storeu_si512(pDst512 + 1, blk1); + _mm512_storeu_si512(pDst512 + 2, blk2); + + pSrc512 += 3; + pDst512 += 3; + blocks -= (3 * 4); + } + if ((4 * 2) <= blocks) { + __m512i blk0 = _mm512_loadu_si512(pSrc512); + __m512i blk1 = _mm512_loadu_si512(pSrc512 + 1); + + __m512i z0 = _mm512_alignr_epi64(blk0, IV, 6); + __m512i z1 = _mm512_alignr_epi64(blk1, blk0, 6); + + // update IV + IV = blk1; + + cpAESDecrypt2_VAES_NI(&blk0, &blk1, pRkey, cipherRounds); + + blk0 = _mm512_xor_si512(blk0, z0); + blk1 = _mm512_xor_si512(blk1, z1); + + _mm512_storeu_si512(pDst512, blk0); + _mm512_storeu_si512(pDst512 + 1, blk1); + + pSrc512 += 2; + pDst512 += 2; + blocks -= (2 * 4); + } + for (; blocks >= 4; blocks -= 4) { + __m512i blk0 = _mm512_loadu_si512(pSrc512); + + __m512i z0 = _mm512_alignr_epi64(blk0, IV, 6); + + // update IV + IV = blk0; + + cpAESDecrypt1_VAES_NI(&blk0, pRkey, cipherRounds); + + blk0 = _mm512_xor_si512(blk0, z0); + + _mm512_storeu_si512(pDst512, blk0); + + pSrc512 += 1; + pDst512 += 1; + } + if (blocks) { + __mmask8 k = (__mmask8)((1 << (blocks + blocks)) - 1); + __m512i blk0 = _mm512_maskz_loadu_epi64(k, pSrc512); + + __m512i z0 = _mm512_maskz_alignr_epi64(k, blk0, IV, 6); + + cpAESDecrypt1_VAES_NI(&blk0, pRkey, cipherRounds); + + blk0 = _mm512_maskz_xor_epi64(k, blk0, z0); + + _mm512_mask_storeu_epi64(pDst512, k, blk0); + } +} + +#endif /* _IPP32E>=_IPP32E_K1 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cbcdecrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cbcdecrypt.c new file mode 100644 index 0000000..976d050 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cbcdecrypt.c @@ -0,0 +1,83 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption/decryption (CBC mode) +// AES encryption/decryption (CBC-CS mode) +// +// Contents: +// ippsAESDecryptCBC() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" +#include "pcpaes_cbc_decrypt.h" + +/*F* +// Name: ippsAESDecryptCBC +// +// Purpose: AES-CBC decryption. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// pSrc == NULL +// pDst == NULL +// pIV == NULL +// ippStsContextMatchErr !VALID_AES_ID() +// ippStsLengthErr len <1 +// ippStsUnderRunErr 0!=(dataLen%MBS_RIJ128) +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// len input/output buffer length (in bytes) +// pCtx pointer to the AES context +// pIV pointer to the initialization vector +// +*F*/ +IPPFUN(IppStatus, ippsAESDecryptCBC,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + const Ipp8u* pIV)) +{ + /* test context */ + IPP_BAD_PTR1_RET(pCtx); + /* test the context ID */ + IPP_BADARG_RET(!VALID_AES_ID(pCtx), ippStsContextMatchErr); + + /* test source, target buffers and initialization pointers */ + IPP_BAD_PTR3_RET(pSrc, pIV, pDst); + /* test stream length */ + IPP_BADARG_RET((len<1), ippStsLengthErr); + /* test stream integrity */ + IPP_BADARG_RET((len&(MBS_RIJ128-1)), ippStsUnderRunErr); + + /* do encryption */ + { + int nBlocks = len / MBS_RIJ128; + + cpDecryptAES_cbc(pIV, pSrc, pDst, nBlocks, pCtx); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cbcdecrypt_cs1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cbcdecrypt_cs1.c new file mode 100644 index 0000000..924a90d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cbcdecrypt_cs1.c @@ -0,0 +1,132 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption/decryption (CBC mode) +// AES encryption/decryption (CBC-CS mode) +// +// Contents: +// ippsAESDecryptCBC_CS1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" +#include "pcpaes_cbc_decrypt.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + +/*F* +// Name: ippsAESDecryptCBC_CS1 +// +// Purpose: AES-CBC_CS1 decryption. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// pSrc == NULL +// pDst == NULL +// pIV == NULL +// ippStsContextMatchErr !VALID_AES_ID() +// ippStsLengthErr len AESCCM_MSGLEN(pState), ippStsLengthErr); + + /* + // enctypt payload and update MAC + */ + if(len) { + /* setup encoder method */ + IppsAESSpec* pAES = AESCCM_CIPHER(pState); + RijnCipher encoder = RIJ_ENCODER(pAES); + + Ipp32u flag = (Ipp32u)( AESCCM_LENPRO(pState) &(MBS_RIJ128-1) ); + + Ipp32u qLen; + Ipp32u counterVal; + + Ipp32u MAC[NB(128)]; + Ipp32u CTR[NB(128)]; + Ipp32u S[NB(128)]; + /* extract from the state */ + CopyBlock16(AESCCM_MAC(pState), MAC); + CopyBlock16(AESCCM_CTR0(pState), CTR); + CopyBlock16(AESCCM_Si(pState), S); + counterVal = AESCCM_COUNTER(pState); + + /* extract qLen */ + qLen = (((Ipp8u*)CTR)[0] &0x7) +1; /* &0x7 just to fix KW issue */ + + if(flag) { + Ipp32u tmpLen = (Ipp32u)( IPP_MIN(len, MBS_RIJ128-1) ); + XorBlock(pSrc, (Ipp8u*)S+flag, pDst, (Ipp32s)tmpLen); + + /* copy as much input as possible into the internal buffer*/ + CopyBlock(pDst, AESCCM_BLK(pState)+flag, (Ipp32s)tmpLen); + + /* update MAC */ + if(flag+tmpLen == MBS_RIJ128) { + XorBlock16(MAC, AESCCM_BLK(pState), MAC); + //encoder(MAC, MAC, RIJ_NR(pAES), RIJ_EKEYS(pAES), (const Ipp32u (*)[256])RIJ_ENC_SBOX(pAES)); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder((Ipp8u*)MAC, (Ipp8u*)MAC, RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder((Ipp8u*)MAC, (Ipp8u*)MAC, RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif + } + + AESCCM_LENPRO(pState) += tmpLen; + pSrc += tmpLen; + pDst += tmpLen; + len -= tmpLen; + } + + #if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + if(AES_NI_ENABLED==RIJ_AESNI(pAES)) { + Ipp32u processedLen = (Ipp32u)(len & -MBS_RIJ128); + if(processedLen) { + /* local state: MAC, counter block, counter bits mask */ + __ALIGN16 Ipp8u localState[3*MBS_RIJ128]; + + /* format counter block and fill local state */ + Ipp32u n; + for(n=0; n= MBS_RIJ128) { + Ipp32u counterEnc[2]; + /* increment counter and format counter block */ + counterVal++; + CopyBlock(CounterEnc(counterEnc, (Ipp32s)qLen, counterVal), ((Ipp8u*)CTR)+MBS_RIJ128-qLen, (Ipp32s)qLen); + /* encode counter block */ + //encoder(CTR, S, RIJ_NR(pAES), RIJ_EKEYS(pAES), (const Ipp32u (*)[256])RIJ_ENC_SBOX(pAES)); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder((Ipp8u*)CTR, (Ipp8u*)S, RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder((Ipp8u*)CTR, (Ipp8u*)S, RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif + + /* store cipher text */ + XorBlock16(pSrc, S, pDst); + + /* update MAC */ + XorBlock16(MAC, pDst, MAC); + //encoder(MAC, MAC, RIJ_NR(pAES), RIJ_EKEYS(pAES), (const Ipp32u (*)[256])RIJ_ENC_SBOX(pAES)); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder((Ipp8u*)MAC, (Ipp8u*)MAC, RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder((Ipp8u*)MAC, (Ipp8u*)MAC, RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif + + AESCCM_LENPRO(pState) += MBS_RIJ128; + pSrc += MBS_RIJ128; + pDst += MBS_RIJ128; + len -= MBS_RIJ128; + } + + if(len) { + Ipp32u counterEnc[2]; + /* increment counter and format counter block */ + counterVal++; + CopyBlock(CounterEnc(counterEnc, (Ipp32s)qLen, counterVal), ((Ipp8u*)CTR)+MBS_RIJ128-qLen, (Ipp32s)qLen); + /* encode counter block */ + //encoder(CTR, S, RIJ_NR(pAES), RIJ_EKEYS(pAES), (const Ipp32u (*)[256])RIJ_ENC_SBOX(pAES)); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder((Ipp8u*)CTR, (Ipp8u*)S, RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder((Ipp8u*)CTR, (Ipp8u*)S, RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif + + /* store cipher text */ + XorBlock(pSrc, S, pDst, len); + + /* workaround to avoid false positive stringop-overflow error on gcc10.1 and gcc11.1 */ + len = ( IPP_MIN(len, MBS_RIJ128-1) ); + + /* store partial data block */ + CopyBlock(pDst, AESCCM_BLK(pState), len); + + AESCCM_LENPRO(pState) += (Ipp64u)len; + } + + /* update state */ + CopyBlock16(MAC, AESCCM_MAC(pState)); + CopyBlock16(S, AESCCM_Si(pState)); + AESCCM_COUNTER(pState) = counterVal; + + /* clear secret data */ + PurgeBlock(S, sizeof(S)); + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ccmencrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ccmencrypt.c new file mode 100644 index 0000000..d43d927 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ccmencrypt.c @@ -0,0 +1,211 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// ippsAES_CCMEncrypt() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesauthccm.h" +#include "pcptool.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + +/*F* +// Name: ippsAES_CCMEncrypt +// +// Purpose: Encrypts data and updates authentication tag. +// +// Returns: Reason: +// ippStsNullPtrErr pState== NULL +// pSrc == NULL +// pDst == NULL +// ippStsContextMatchErr !VALID_AESCCM_ID() +// ippStsLengthErr if exceed overall length of message is being processed +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the plane text beffer +// pDst pointer to the cipher text bubber +// len length of the buffer +// pState pointer to the CCM context +// +*F*/ +IPPFUN(IppStatus, ippsAES_CCMEncrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int len, IppsAES_CCMState* pState)) +{ + /* test pState pointer */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!VALID_AESCCM_ID(pState), ippStsContextMatchErr); + + /* test source/destination data */ + IPP_BAD_PTR2_RET(pSrc, pDst); + + /* test message length */ + IPP_BADARG_RET(len<0 || AESCCM_LENPRO(pState)+(Ipp64u)len >AESCCM_MSGLEN(pState), ippStsLengthErr); + + /* + // enctypt payload and update MAC + */ + if(len) { + /* setup encoder method */ + IppsAESSpec* pAES = AESCCM_CIPHER(pState); + RijnCipher encoder = RIJ_ENCODER(pAES); + + Ipp32u flag = (Ipp32u)( AESCCM_LENPRO(pState) &(MBS_RIJ128-1) ); + + Ipp32u qLen; + Ipp32u counterVal; + + Ipp32u MAC[NB(128)]; + Ipp32u CTR[NB(128)]; + Ipp32u S[NB(128)]; + /* extract from the state */ + CopyBlock16(AESCCM_MAC(pState), MAC); + CopyBlock16(AESCCM_CTR0(pState), CTR); + CopyBlock16(AESCCM_Si(pState), S); + counterVal = AESCCM_COUNTER(pState); + + /* extract qLen */ + qLen = (((Ipp8u*)CTR)[0] &0x7) +1; /* &0x7 just to fix KW issue */ + + if(flag) { + Ipp32u tmpLen = (Ipp32u)IPP_MIN(len, MBS_RIJ128-1); + + /* copy as much input as possible into the internal buffer*/ + CopyBlock(pSrc, AESCCM_BLK(pState)+flag, (Ipp32s)tmpLen); + + XorBlock(pSrc, (Ipp8u*)S+flag, pDst, (Ipp32s)tmpLen); + + /* update MAC */ + if(flag+(Ipp32u)tmpLen == MBS_RIJ128) { + XorBlock16(MAC, AESCCM_BLK(pState), MAC); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder((Ipp8u*)MAC, (Ipp8u*)MAC, RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder((Ipp8u*)MAC, (Ipp8u*)MAC, RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif + } + + AESCCM_LENPRO(pState) += (Ipp32u)tmpLen; + pSrc += tmpLen; + pDst += tmpLen; + len -= tmpLen; + } + + #if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + if(AES_NI_ENABLED==RIJ_AESNI(pAES)) { + Ipp32u processedLen = (Ipp32u)(len & -MBS_RIJ128); + if(processedLen) { + /* local state: MAC, counter block, counter bits mask */ + __ALIGN16 Ipp8u localState[3*MBS_RIJ128]; + + /* format counter block and fill local state: */ + Ipp32u n; + for(n=0; n= MBS_RIJ128) { + Ipp32u counterEnc[2]; + + /* update MAC */ + XorBlock16(MAC, pSrc, MAC); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder((Ipp8u*)MAC, (Ipp8u*)MAC, RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder((Ipp8u*)MAC, (Ipp8u*)MAC, RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif + + /* increment counter and format counter block */ + counterVal++; + CopyBlock(CounterEnc(counterEnc, (Ipp32s)qLen, counterVal), ((Ipp8u*)CTR)+MBS_RIJ128-qLen, (Ipp32s)qLen); + /* encode counter block */ + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder((Ipp8u*)CTR, (Ipp8u*)S, RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder((Ipp8u*)CTR, (Ipp8u*)S, RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif + + /* store cipher text */ + XorBlock16(pSrc, S, pDst); + + AESCCM_LENPRO(pState) += MBS_RIJ128; + pSrc += MBS_RIJ128; + pDst += MBS_RIJ128; + len -= MBS_RIJ128; + } + + if(len) { + Ipp32u counterEnc[2]; + + /* workaround to avoid false positive stringop-overflow error on gcc10.1 and gcc11.1 */ + len = ( IPP_MIN(len, MBS_RIJ128-1) ); + + /* store partial data block */ + CopyBlock(pSrc, AESCCM_BLK(pState), len); + + /* increment counter and format counter block */ + counterVal++; + CopyBlock(CounterEnc(counterEnc, (Ipp32s)qLen, counterVal), ((Ipp8u*)CTR)+MBS_RIJ128-qLen, (Ipp32s)qLen); + /* encode counter block */ + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder((Ipp8u*)CTR, (Ipp8u*)S, RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder((Ipp8u*)CTR, (Ipp8u*)S, RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif + + /* store cipher text */ + XorBlock(pSrc, S, pDst, len); + + AESCCM_LENPRO(pState) += (Ipp64u)len; + } + + /* update state */ + CopyBlock16(MAC, AESCCM_MAC(pState)); + CopyBlock16(S, AESCCM_Si(pState)); + AESCCM_COUNTER(pState) = counterVal; + + /* clear secret data */ + PurgeBlock(S, sizeof(S)); + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ccmgetsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ccmgetsize.c new file mode 100644 index 0000000..f101403 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ccmgetsize.c @@ -0,0 +1,56 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// ippsAES_CCMGetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesauthccm.h" +#include "pcptool.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + +/*F* +// Name: ippsAES_CCMGetSize +// +// Purpose: Returns size of AES-CCM state (bytes). +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to the size of CCM (in bytes) +// +*F*/ +IPPFUN(IppStatus, ippsAES_CCMGetSize,(int* pSize)) +{ + /* test size's pointer */ + IPP_BAD_PTR1_RET(pSize); + + *pSize = cpSizeofCtx_AESCCM(); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ccmgettag.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ccmgettag.c new file mode 100644 index 0000000..3c0fadd --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ccmgettag.c @@ -0,0 +1,92 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// ippsAES_CCMGetTag() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesauthccm.h" +#include "pcptool.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + +/*F* +// Name: ippsAES_CCMGetTag +// +// Purpose: Compute message auth tag and return one. +// Note, that futher encryption/decryption and auth tag update is possible +// +// Returns: Reason: +// ippStsNullPtrErr pTag == NULL +// pState == NULL +// ippStsContextMatchErr !VALID_AESCCM_ID() +// ippStsLengthErr MBS_RIJ128 < tagLen +// 1 > tagLen +// ippStsNoErr no errors +// +// Parameters: +// pTag pointer to the output authenticated tag +// tagLen requested length of the tag +// pState pointer to the CCM context +// +*F*/ +IPPFUN(IppStatus, ippsAES_CCMGetTag,(Ipp8u* pTag, int tagLen, const IppsAES_CCMState* pState)) +{ + /* test pState pointer */ + IPP_BAD_PTR1_RET(pState); + + /* test state ID */ + IPP_BADARG_RET(!VALID_AESCCM_ID(pState), ippStsContextMatchErr); + + /* test tag (pointer and length) */ + IPP_BAD_PTR1_RET(pTag); + IPP_BADARG_RET((Ipp32u)tagLen>AESCCM_TAGLEN(pState) || tagLen<1, ippStsLengthErr); + + { + Ipp32u flag = (Ipp32u)( AESCCM_LENPRO(pState) &(MBS_RIJ128-1) ); + + Ipp32u MAC[NB(128)]; + CopyBlock16(AESCCM_MAC(pState), MAC); + + if(flag) { + IppsAESSpec* pAES = AESCCM_CIPHER(pState); + RijnCipher encoder = RIJ_ENCODER(pAES); + + Ipp8u BLK[MBS_RIJ128]; + FillBlock16(0, NULL,BLK, 0); + CopyBlock(AESCCM_BLK(pState), BLK, (cpSize)flag); + + XorBlock16(MAC, BLK, MAC); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder((Ipp8u*)MAC, (Ipp8u*)MAC, RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder((Ipp8u*)MAC, (Ipp8u*)MAC, RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif + } + + XorBlock(MAC, AESCCM_S0(pState), pTag, (cpSize)tagLen); + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ccminit.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ccminit.c new file mode 100644 index 0000000..aaa4191 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ccminit.c @@ -0,0 +1,75 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// ippsAES_CCMInit() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesauthccm.h" +#include "pcptool.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + +/*F* +// Name: ippsAES_CCMInit +// +// Purpose: Init AES-CCM state. +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL +// ippStsMemAllocErr size of buffer is not match fro operation +// ippStsLengthErr keyLen != 16 && +// != 24 && +// != 32 +// ippStsNoErr no errors +// +// Parameters: +// pKey pointer to the secret key +// keyLen length of secret key (in bytes) +// pState pointer to initialized as CCM context +// ctxSize available size (in bytes) of buffer above +// +*F*/ +IPPFUN(IppStatus, ippsAES_CCMInit,(const Ipp8u* pKey, int keyLen, + IppsAES_CCMState* pState, int ctxSize)) +{ + /* test pState pointer */ + IPP_BAD_PTR1_RET(pState); + + /* test available size of context buffer */ + IPP_BADARG_RET(ctxSize13), ippStsLengthErr); + + /* test AAD pointer if defined */ + IPP_BADARG_RET(adLen<0, ippStsLengthErr); + if(adLen) + IPP_BAD_PTR1_RET(pAD); + + /* init for new message */ + AESCCM_LENPRO(pState) = 0; + AESCCM_COUNTER(pState) = 0; + + { + /* setup encoder method */ + IppsAESSpec* pAES = AESCCM_CIPHER(pState); + RijnCipher encoder = RIJ_ENCODER(pAES); + + Ipp32u MAC[NB(128)]; + Ipp32u CTR[NB(128)]; + Ipp32u block[2*NB(128)]; + + /* + // prepare the 1-st input block B0 and encode + */ + Ipp32u qLen = (Ipp32u)( (MBS_RIJ128-1) - ivLen); + Ipp32u qLenEnc = qLen-1; + + Ipp32u tagLenEnc = (AESCCM_TAGLEN(pState)-2)>>1; + + Ipp64u payloadLen = AESCCM_MSGLEN(pState); + + ((Ipp8u*)MAC)[0] = (Ipp8u)( (Ipp32u)((adLen!=0) <<6) + (tagLenEnc<<3) + qLenEnc); /* flags */ + #if (IPP_ENDIAN == IPP_LITTLE_ENDIAN) + MAC[2] = ENDIANNESS(IPP_HIDWORD(payloadLen)); + MAC[3] = ENDIANNESS(IPP_LODWORD(payloadLen)); + #else + MAC[2] = IPP_HIDWORD(payloadLen); + MAC[3] = IPP_LODWORD(payloadLen); + #endif + CopyBlock(pIV, ((Ipp8u*)MAC)+1, ivLen); + + /* setup CTR0 */ + FillBlock16(0, NULL,CTR, 0); + ((Ipp8u*)CTR)[0] = (Ipp8u)qLenEnc; /* flags */ + CopyBlock(pIV, ((Ipp8u*)CTR)+1, ivLen); + CopyBlock16(CTR, AESCCM_CTR0(pState)); + + /* compute and store S0=ENC(CTR0) */ + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder((Ipp8u*)CTR, AESCCM_S0(pState), RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder((Ipp8u*)CTR, AESCCM_S0(pState), RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif + + /* init MAC value MAC = ENC(MAC) */ + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder((Ipp8u*)MAC, (Ipp8u*)MAC, RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder((Ipp8u*)MAC, (Ipp8u*)MAC, RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif + + + /* + // update MAC by the AD + */ + if(adLen) { + /* encode length of associated data */ + Ipp32u adLenEnc[3]; + Ipp8u* adLenEncPtr; + int adLenEncSize; + + #if (IPP_ENDIAN == IPP_LITTLE_ENDIAN) + adLenEnc[1] = ENDIANNESS(IPP_HIDWORD(adLen)); + adLenEnc[2] = ENDIANNESS(IPP_LODWORD(adLen)); + #else + adLenEnc[1] = IPP_HIDWORD(adLen); + adLenEnc[2] = IPP_LODWORD(adLen); + #endif + + if(adLen >= 0xFF00) { + adLenEncSize = 6; + #if (IPP_ENDIAN == IPP_LITTLE_ENDIAN) + adLenEnc[1] = 0xFEFFFFFF; + #else + adLenEnc[1] = 0xFFFFFFFE; + #endif + } + else { + adLenEncSize= 2; + } + adLenEncPtr = (Ipp8u*)adLenEnc+3*sizeof(Ipp32u)-adLenEncSize; + + /* prepare first formatted block of Header */ + CopyBlock(adLenEncPtr, block, adLenEncSize); + FillBlock16(0,pAD, (Ipp8u*)block+adLenEncSize, IPP_MIN((MBS_RIJ128-adLenEncSize), adLen)); + + /* and update MAC */ + MAC[0] ^= block[0]; + MAC[1] ^= block[1]; + MAC[2] ^= block[2]; + MAC[3] ^= block[3]; + //encoder(MAC, MAC, RIJ_NR(pAES), RIJ_EKEYS(pAES), (const Ipp32u (*)[256])RIJ_ENC_SBOX(pAES)); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder((Ipp8u*)MAC, (Ipp8u*)MAC, RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder((Ipp8u*)MAC, (Ipp8u*)MAC, RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif + + /* update MAC the by rest of addition data */ + if( (adLen+adLenEncSize) > MBS_RIJ128 ) { + pAD += (MBS_RIJ128-adLenEncSize); + adLen -= (MBS_RIJ128-adLenEncSize); + while(adLen >= MBS_RIJ128) { + CopyBlock16(pAD, block); + MAC[0] ^= block[0]; + MAC[1] ^= block[1]; + MAC[2] ^= block[2]; + MAC[3] ^= block[3]; + //encoder(MAC, MAC, RIJ_NR(pAES), RIJ_EKEYS(pAES), (const Ipp32u (*)[256])RIJ_ENC_SBOX(pAES)); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder((Ipp8u*)MAC, (Ipp8u*)MAC, RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder((Ipp8u*)MAC, (Ipp8u*)MAC, RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif + + pAD += MBS_RIJ128; + adLen -= MBS_RIJ128; + } + + if(adLen) { + FillBlock16(0, pAD, block, (int)adLen); + MAC[0] ^= block[0]; + MAC[1] ^= block[1]; + MAC[2] ^= block[2]; + MAC[3] ^= block[3]; + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder((Ipp8u*)MAC, (Ipp8u*)MAC, RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder((Ipp8u*)MAC, (Ipp8u*)MAC, RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif + } + } + } + + AESCCM_COUNTER(pState) = 0; + CopyBlock16(MAC, AESCCM_MAC(pState)); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ccmtaglen.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ccmtaglen.c new file mode 100644 index 0000000..d5bffc0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ccmtaglen.c @@ -0,0 +1,65 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// ippsAES_CCMTagLen() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesauthccm.h" +#include "pcptool.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + +/*F* +// Name: ippsAES_CCMTagLen +// +// Purpose: Setup length of the tag. +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL +// ippStsContextMatchErr !VALID_AESCCM_ID() +// ippStsLengthErr MBS_RIJ128 < tagLen || tagLen < 4 +// or odd value of tagLen +// ippStsNoErr no errors +// +// Parameters: +// tagLen length in bytes of the requested tag +// pState pointer to the AES-CCM state +// +*F*/ +IPPFUN(IppStatus, ippsAES_CCMTagLen,(int tagLen, IppsAES_CCMState* pState)) +{ + /* test context */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!VALID_AESCCM_ID(pState), ippStsContextMatchErr); + + /* test tag length */ + IPP_BADARG_RET(tagLen>MBS_RIJ128 || tagLen<4 || tagLen&1, ippStsLengthErr); + + /* init for new message */ + AESCCM_TAGLEN(pState) = (Ipp32u)tagLen; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cfb128decrypt_vaes512.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cfb128decrypt_vaes512.c new file mode 100644 index 0000000..901665f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cfb128decrypt_vaes512.c @@ -0,0 +1,173 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption/decryption (CFB mode) +// +// Contents: +// DecryptCFB128_RIJ128pipe_VAES_NI() +// +// +*/ + +#include "owncp.h" +#include "pcpaesm.h" +#include "pcpaes_encrypt_vaes512.h" + +#if(_IPP32E>=_IPP32E_K1) +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(disable: 4310) // zmmintrin.h bug: truncation of constant value +#endif + +IPP_OWN_DEFN (void, DecryptCFB128_RIJ128pipe_VAES_NI, (const Ipp8u* pSrc, // pointer to the ciphertext + Ipp8u* pDst, // pointer to the plaintext + int len, // message length + const IppsAESSpec* pCtx, // pointer to context + const Ipp8u* pIV)) // pointer to the Initialization Vector +{ + int cipherRounds = RIJ_NR(pCtx) - 1; + + __m128i* pRkey = (__m128i*)RIJ_EKEYS(pCtx); + __m512i* pSrc512 = (__m512i*)pSrc; + __m512i* pDst512 = (__m512i*)pDst; + + // load IV (128-bit) + __m512i IV = _mm512_maskz_expandloadu_epi64(0xC0, pIV); // IV 0 0 0 + + int blocks; + for (blocks = len / MBS_RIJ128; blocks >= (4 * 4); blocks -= (4 * 4)) { + // load ciphertext + __m512i blk0 = _mm512_loadu_si512(pSrc512); + __m512i blk1 = _mm512_loadu_si512(pSrc512 + 1); + __m512i blk2 = _mm512_loadu_si512(pSrc512 + 2); + __m512i blk3 = _mm512_loadu_si512(pSrc512 + 3); + + // prepare vectors for decryption + __m512i z0 = _mm512_alignr_epi64(blk0, IV, 6); // C3 C2 C1 IV + __m512i z1 = _mm512_alignr_epi64(blk1, blk0, 6); // C7 C6 C5 C4 + __m512i z2 = _mm512_alignr_epi64(blk2, blk1, 6); // C11 C10 C9 C8 + __m512i z3 = _mm512_alignr_epi64(blk3, blk2, 6); // C15 C14 C13 C12 + + // update IV + IV = blk3; + + cpAESEncrypt4_VAES_NI(&z0, &z1, &z2, &z3, pRkey, cipherRounds); + + blk0 = _mm512_xor_si512(blk0, z0); + blk1 = _mm512_xor_si512(blk1, z1); + blk2 = _mm512_xor_si512(blk2, z2); + blk3 = _mm512_xor_si512(blk3, z3); + + _mm512_storeu_si512(pDst512, blk0); + _mm512_storeu_si512(pDst512 + 1, blk1); + _mm512_storeu_si512(pDst512 + 2, blk2); + _mm512_storeu_si512(pDst512 + 3, blk3); + + pSrc512 += 4; + pDst512 += 4; + } + + if ((3 * 4) <= blocks) { + // load ciphertext + __m512i blk0 = _mm512_loadu_si512(pSrc512); + __m512i blk1 = _mm512_loadu_si512(pSrc512 + 1); + __m512i blk2 = _mm512_loadu_si512(pSrc512 + 2); + + __m512i z0 = _mm512_alignr_epi64(blk0, IV, 6); + __m512i z1 = _mm512_alignr_epi64(blk1, blk0, 6); + __m512i z2 = _mm512_alignr_epi64(blk2, blk1, 6); + + // update IV + IV = blk2; + + cpAESEncrypt3_VAES_NI(&z0, &z1, &z2, pRkey, cipherRounds); + + blk0 = _mm512_xor_si512(blk0, z0); + blk1 = _mm512_xor_si512(blk1, z1); + blk2 = _mm512_xor_si512(blk2, z2); + + _mm512_storeu_si512(pDst512, blk0); + _mm512_storeu_si512(pDst512 + 1, blk1); + _mm512_storeu_si512(pDst512 + 2, blk2); + + pSrc512 += 3; + pDst512 += 3; + blocks -= (3 * 4); + } + + if ((4 * 2) <= blocks) { + // load ciphertext + __m512i blk0 = _mm512_loadu_si512(pSrc512); + __m512i blk1 = _mm512_loadu_si512(pSrc512 + 1); + + __m512i z0 = _mm512_alignr_epi64(blk0, IV, 6); + __m512i z1 = _mm512_alignr_epi64(blk1, blk0, 6); + + // update IV + IV = blk1; + + cpAESEncrypt2_VAES_NI(&z0, &z1, pRkey, cipherRounds); + + blk0 = _mm512_xor_si512(blk0, z0); + blk1 = _mm512_xor_si512(blk1, z1); + + _mm512_storeu_si512(pDst512, blk0); + _mm512_storeu_si512(pDst512 + 1, blk1); + + pSrc512 += 2; + pDst512 += 2; + blocks -= (2 * 4); + } + + for (; blocks >= 4; blocks -= 4) { + // load ciphertext + __m512i blk0 = _mm512_loadu_si512(pSrc512); + + __m512i z0 = _mm512_alignr_epi64(blk0, IV, 6); + + // update IV + IV = blk0; + + cpAESEncrypt1_VAES_NI(&z0, pRkey, cipherRounds); + + blk0 = _mm512_xor_si512(blk0, z0); + + _mm512_storeu_si512(pDst512, blk0); + + pSrc512 += 1; + pDst512 += 1; + } + + if (blocks) { + __mmask8 k = (__mmask8)((1 << (blocks + blocks)) - 1); + + __m512i blk0 = _mm512_maskz_loadu_epi64(k, pSrc512); + + __m512i z0 = _mm512_maskz_alignr_epi64(k, blk0, IV, 6); + + cpAESEncrypt1_VAES_NI(&z0, pRkey, cipherRounds); + + blk0 = _mm512_maskz_xor_epi64(k, blk0, z0); + + _mm512_mask_storeu_epi64(pDst512, k, blk0); + } +} + +#endif /* _IPP32E>=_IPP32E_K1 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cfb64decrypt_vaes512.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cfb64decrypt_vaes512.c new file mode 100644 index 0000000..b710234 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cfb64decrypt_vaes512.c @@ -0,0 +1,236 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption/decryption (CFB mode) +// +// Contents: +// DecryptCFB64_RIJ128pipe_VAES_NI() +// +// +*/ + +#include "owncp.h" +#include "pcpaesm.h" +#include "pcpaes_encrypt_vaes512.h" + +#if(_IPP32E>=_IPP32E_K1) +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(disable: 4310) // zmmintrin.h bug: truncation of constant value +#endif + +IPP_OWN_DEFN (void, DecryptCFB64_RIJ128pipe_VAES_NI, (const Ipp8u* pSrc, // pointer to the ciphertext + Ipp8u* pDst, // pointer to the plaintext + int len, // message length + const IppsAESSpec* pCtx, // pointer to context + const Ipp8u* pIV)) // pointer to the Initialization Vector +{ + int cipherRounds = RIJ_NR(pCtx) - 1; + + __m128i* pRkey = (__m128i*)RIJ_EKEYS(pCtx); + __m256i* pSrc256 = (__m256i*)pSrc; + __m256i* pDst256 = (__m256i*)pDst; + + // load/store masks + __mmask8 k8_load = 0xAA; // 1010 1010 - load to high addresses of 128-bit lanes + __mmask8 k8_store = 0x55; // 0101 0101 - store low addresses of 128-bit lanes (MSB of OutputBlocks) + + // load IV and init IV vector in appropriate for further batch processing manner + __m128i IV128 = _mm_maskz_loadu_epi64(0x03, pIV); + // 64-bit values: IV_l IV_h IV_l IV_h 0 0 0 0 + __m512i IV = _mm512_maskz_broadcast_i64x2(0xF0, IV128); + // 64-bit values: IV_l IV_h IV_h IV_h 0 0 0 0 + IV = _mm512_maskz_permutex_epi64(0xF0, IV, 0xE1); // 0xE1 = 1110 0001 + + int blocks = len / (MBS_RIJ128 >> 1); + + for (; blocks >= (4 * 4); blocks -= (4 * 4)) { + // load 64-bit blocks of ciphertext to high part of 128-bit lanes + __m512i blk0 = _mm512_maskz_expandloadu_epi64(k8_load, pSrc256); // C3 0 C2 0 C1 0 C0 0 + __m512i blk1 = _mm512_maskz_expandloadu_epi64(k8_load, pSrc256 + 1); // C7 0 C6 0 C5 0 C4 0 + __m512i blk2 = _mm512_maskz_expandloadu_epi64(k8_load, pSrc256 + 2); + __m512i blk3 = _mm512_maskz_expandloadu_epi64(k8_load, pSrc256 + 3); + + // prepare vectors for decryption + __m512i z00 = _mm512_alignr_epi64(blk0, IV, 4); // C2 0 | C1 0 | IV_l IV_h | IV_h IV_l + __m512i z01 = _mm512_alignr_epi64(blk0, IV, 6); // C3 0 | C2 0 | C1 0 | IV_l IV_h + z00 = _mm512_unpackhi_epi64(z00, z01); // C3 C2 | C2 C1 | C1 IV_l | IV_l IV_h + + __m512i z10 = _mm512_alignr_epi64(blk1, blk0, 4); + __m512i z11 = _mm512_alignr_epi64(blk1, blk0, 6); + z10 = _mm512_unpackhi_epi64(z10, z11); + + __m512i z20 = _mm512_alignr_epi64(blk2, blk1, 4); + __m512i z21 = _mm512_alignr_epi64(blk2, blk1, 6); + z20 = _mm512_unpackhi_epi64(z20, z21); + + __m512i z30 = _mm512_alignr_epi64(blk3, blk2, 4); + __m512i z31 = _mm512_alignr_epi64(blk3, blk2, 6); + z30 = _mm512_unpackhi_epi64(z30, z31); + + // update IV + IV = blk3; + + cpAESEncrypt4_VAES_NI(&z00, &z10, &z20, &z30, pRkey, cipherRounds); + + // move cipher blocks to MSB part of 128-bit lanes + blk0 = _mm512_bsrli_epi128(blk0, 8); + blk1 = _mm512_bsrli_epi128(blk1, 8); + blk2 = _mm512_bsrli_epi128(blk2, 8); + blk3 = _mm512_bsrli_epi128(blk3, 8); + + blk0 = _mm512_xor_si512(blk0, z00); + blk1 = _mm512_xor_si512(blk1, z10); + blk2 = _mm512_xor_si512(blk2, z20); + blk3 = _mm512_xor_si512(blk3, z30); + + _mm512_mask_compressstoreu_epi64(pDst256, k8_store, blk0); + _mm512_mask_compressstoreu_epi64(pDst256 + 1, k8_store, blk1); + _mm512_mask_compressstoreu_epi64(pDst256 + 2, k8_store, blk2); + _mm512_mask_compressstoreu_epi64(pDst256 + 3, k8_store, blk3); + + pSrc256 += 4; + pDst256 += 4; + } + + if ((3 * 4) <= blocks) { + // load 64-bit blocks of ciphertext to high part of 128-bit lanes + __m512i blk0 = _mm512_maskz_expandloadu_epi64(k8_load, pSrc256); + __m512i blk1 = _mm512_maskz_expandloadu_epi64(k8_load, pSrc256 + 1); + __m512i blk2 = _mm512_maskz_expandloadu_epi64(k8_load, pSrc256 + 2); + + // prepare vectors for decryption + __m512i z00 = _mm512_alignr_epi64(blk0, IV, 4); + __m512i z01 = _mm512_alignr_epi64(blk0, IV, 6); + z00 = _mm512_unpackhi_epi64(z00, z01); + + __m512i z10 = _mm512_alignr_epi64(blk1, blk0, 4); + __m512i z11 = _mm512_alignr_epi64(blk1, blk0, 6); + z10 = _mm512_unpackhi_epi64(z10, z11); + + __m512i z20 = _mm512_alignr_epi64(blk2, blk1, 4); + __m512i z21 = _mm512_alignr_epi64(blk2, blk1, 6); + z20 = _mm512_unpackhi_epi64(z20, z21); + + // update IV + IV = blk2; + + cpAESEncrypt3_VAES_NI(&z00, &z10, &z20, pRkey, cipherRounds); + + // move cipher blocks to MSB part of 128-bit lanes + blk0 = _mm512_bsrli_epi128(blk0, 8); + blk1 = _mm512_bsrli_epi128(blk1, 8); + blk2 = _mm512_bsrli_epi128(blk2, 8); + + blk0 = _mm512_xor_si512(blk0, z00); + blk1 = _mm512_xor_si512(blk1, z10); + blk2 = _mm512_xor_si512(blk2, z20); + + _mm512_mask_compressstoreu_epi64(pDst256, k8_store, blk0); + _mm512_mask_compressstoreu_epi64(pDst256 + 1, k8_store, blk1); + _mm512_mask_compressstoreu_epi64(pDst256 + 2, k8_store, blk2); + + pSrc256 += 3; + pDst256 += 3; + blocks -= (3 * 4); + } + + if ((4 * 2) <= blocks) { + // load 64-bit blocks of ciphertext to high part of 128-bit lanes + __m512i blk0 = _mm512_maskz_expandloadu_epi64(k8_load, pSrc256); + __m512i blk1 = _mm512_maskz_expandloadu_epi64(k8_load, pSrc256 + 1); + + // prepare vectors for decryption + __m512i z00 = _mm512_alignr_epi64(blk0, IV, 4); + __m512i z01 = _mm512_alignr_epi64(blk0, IV, 6); + z00 = _mm512_unpackhi_epi64(z00, z01); + + __m512i z10 = _mm512_alignr_epi64(blk1, blk0, 4); + __m512i z11 = _mm512_alignr_epi64(blk1, blk0, 6); + z10 = _mm512_unpackhi_epi64(z10, z11); + + // update IV + IV = blk1; + + cpAESEncrypt2_VAES_NI(&z00, &z10, pRkey, cipherRounds); + + // move cipher blocks to MSB part of 128-bit lanes + blk0 = _mm512_bsrli_epi128(blk0, 8); + blk1 = _mm512_bsrli_epi128(blk1, 8); + + blk0 = _mm512_xor_si512(blk0, z00); + blk1 = _mm512_xor_si512(blk1, z10); + + _mm512_mask_compressstoreu_epi64(pDst256, k8_store, blk0); + _mm512_mask_compressstoreu_epi64(pDst256 + 1, k8_store, blk1); + + pSrc256 += 2; + pDst256 += 2; + blocks -= (2 * 4); + } + + for (; blocks >= 4; blocks -= 4) { + // load 64-bit blocks of ciphertext to high part of 128-bit lanes + __m512i blk0 = _mm512_maskz_expandloadu_epi64(k8_load, pSrc256); + + // prepare vectors for decryption + __m512i z00 = _mm512_alignr_epi64(blk0, IV, 4); + __m512i z01 = _mm512_alignr_epi64(blk0, IV, 6); + z00 = _mm512_unpackhi_epi64(z00, z01); + + // update IV + IV = blk0; + + cpAESEncrypt1_VAES_NI(&z00, pRkey, cipherRounds); + + // move cipher blocks to MSB part of 128-bit lanes + blk0 = _mm512_bsrli_epi128(blk0, 8); + + blk0 = _mm512_xor_si512(blk0, z00); + + _mm512_mask_compressstoreu_epi64(pDst256, k8_store, blk0); + + pSrc256 += 1; + pDst256 += 1; + } + + if (blocks) { + __mmask8 k = (__mmask8)((1 << (blocks + blocks)) - 1); // 64-bit chunks + + // load 64-bit blocks of ciphertext to hi part of 128-bit lanes + __m512i blk0 = _mm512_maskz_expandloadu_epi64(k & k8_load, pSrc256); + + // prepare vectors for decryption + __m512i z00 = _mm512_alignr_epi64(blk0, IV, 4); + __m512i z01 = _mm512_alignr_epi64(blk0, IV, 6); + z00 = _mm512_maskz_unpackhi_epi64(k, z00, z01); + + cpAESEncrypt1_VAES_NI(&z00, pRkey, cipherRounds); + + // move cipher blocks to MSB part of 128-bit lanes + blk0 = _mm512_bsrli_epi128(blk0, 8); + + blk0 = _mm512_xor_epi64(blk0, z00); + + _mm512_mask_compressstoreu_epi64(pDst256, k & k8_store, blk0); + } +} + +#endif /* _IPP32E>=_IPP32E_K1 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cfb_mb_encrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cfb_mb_encrypt.c new file mode 100644 index 0000000..346e44f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cfb_mb_encrypt.c @@ -0,0 +1,234 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES Multi Buffer Encryption (CFB mode) +// +// Contents: +// ippsAES_EncryptCFB16_MB() +// +*/ + +#include "owncp.h" +#include "pcpaesm.h" +#include "aes_cfb_vaes_mb.h" +#include "aes_cfb_aesni_mb.h" + + +/*! + * \brief ippsAES_EncryptCFB16_MB + * + * Name: ippsAES_EncryptCFB16_MB + * + * Purpose: AES-CFB16 Multi Buffer Encryption + * + * Parameters: + * \param[in] pSrc Pointer to the array of source data + * \param[out] pDst Pointer to the array of target data + * \param[in] len Pointer to the array of input buffer lengths (in bytes) + * \param[in] pCtx Pointer to the array of AES contexts + * \param[in] pIV Pointer to the array of initialization vectors (IV) + * \param[out] status Pointer to the IppStatus array that contains status + * for each processed buffer in encryption operation + * \param[in] numBuffers Number of buffers to be processed + * + * Returns: Reason: + * \return ippStsNullPtrErr Indicates an error condition if any of the specified pointers is NULL: + * NULL == pSrc + * NULL == pDst + * NULL == len + * NULL == pCtx + * NULL == pIV + * NULL == status + * \return ippStsContextMatchErr Indicates an error condition if input buffers have different key sizes + * \return ippStsLengthErr Indicates an error condition if numBuffers < 1 + * \return ippStsErr One or more of performed operation executed with error + * Check status array for details + * \return ippStsNoErr No error + */ + +/* Work Load Size from Buffers */ +#define WORKLOAD_LINES_16 (AES_MB_MAX_KERNEL_SIZE) /* size 16 */ +#define WORKLOAD_LINES_8 (AES_MB_MAX_KERNEL_SIZE / 2) /* size 8 */ +#define WORKLOAD_LINES_4 (AES_MB_MAX_KERNEL_SIZE / 4) /* size 4 */ + + +IPPFUN(IppStatus, ippsAES_EncryptCFB16_MB, (const Ipp8u* pSrc[], Ipp8u* pDst[], int len[], const IppsAESSpec* pCtx[], + const Ipp8u* pIV[], IppStatus status[], int numBuffers)) +{ + int i; + + // Check input pointers + IPP_BAD_PTR2_RET(pCtx, pIV); + IPP_BAD_PTR4_RET(pSrc, pDst, len, status); + + // Check number of buffers to be processed + IPP_BADARG_RET((numBuffers < 1), ippStsLengthErr); + + // Sequential check of all input buffers + int isAllBuffersValid = 1; + for (i = 0; i < numBuffers; i++) { + // Test source, target buffers and initialization pointers + if (pSrc[i] == NULL || pDst[i] == NULL || pIV[i] == NULL || pCtx[i] == NULL) { + status[i] = ippStsNullPtrErr; + isAllBuffersValid = 0; + continue; + } + + // Test the context ID + if(!VALID_AES_ID(pCtx[i])) { + status[i] = ippStsContextMatchErr; + isAllBuffersValid = 0; + continue; + } + + // Test stream length + if (len[i] < 1) { + status[i] = ippStsLengthErr; + isAllBuffersValid = 0; + continue; + } + + // Test stream integrity + if ((len[i] % CFB16_BLOCK_SIZE)) { + status[i] = ippStsUnderRunErr; + isAllBuffersValid = 0; + continue; + } + + status[i] = ippStsNoErr; + } + + // If any of the input buffer is not valid stop the processig + IPP_BADARG_RET(!isAllBuffersValid, ippStsErr) + + // Check compatibility of the keys + int referenceKeySize = RIJ_NK(pCtx[0]); + for (i = 0; i < numBuffers; i++) { + IPP_BADARG_RET((RIJ_NK(pCtx[i]) != referenceKeySize), ippStsContextMatchErr); + } + + #if (_IPP32E>=_IPP32E_Y8) + Ipp32u const* loc_enc_keys[AES_MB_MAX_KERNEL_SIZE]; + Ipp8u const* loc_src[AES_MB_MAX_KERNEL_SIZE]; + Ipp8u* loc_dst[AES_MB_MAX_KERNEL_SIZE]; + Ipp8u const* loc_iv[AES_MB_MAX_KERNEL_SIZE]; + int loc_len[AES_MB_MAX_KERNEL_SIZE]; + int buffersProcessed = 0; + int numRounds = 0; + #endif + + #if(_IPP32E>=_IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512VAES)) { + int workLoadSize = 0; + while(numBuffers > 0) { + /* init work load size */ + if (numBuffers > WORKLOAD_LINES_8) { /* size 16 */ + workLoadSize = WORKLOAD_LINES_16; + } else if (numBuffers > WORKLOAD_LINES_4 && numBuffers <= WORKLOAD_LINES_8) { /* size 8 */ + workLoadSize = WORKLOAD_LINES_8; + } else if (numBuffers > 0 && numBuffers <= WORKLOAD_LINES_4) { /* size 4 */ + workLoadSize = WORKLOAD_LINES_4; + } else { + break; + } + + /* fill buffers */ + for (i = 0; i < workLoadSize; i++) { + if (i >= numBuffers) { + loc_len[i] = 0; + continue; + } + + loc_src[i] = pSrc[i + buffersProcessed]; + loc_dst[i] = pDst[i + buffersProcessed]; + loc_iv[i] = pIV[i + buffersProcessed]; + loc_enc_keys[i] = (Ipp32u*)RIJ_EKEYS(pCtx[i + buffersProcessed]); + loc_len[i] = len[i + buffersProcessed]; + /* As numRounds is the same for all buffers, get it from the last one */ + numRounds = RIJ_NR(pCtx[i + buffersProcessed]); + } + + /* choosing a core for filled buffers */ + switch (workLoadSize) { + case WORKLOAD_LINES_16: { + aes_cfb16_enc_vaes_mb16(loc_src, loc_dst, loc_len, numRounds, loc_enc_keys, loc_iv); + break; + } + case WORKLOAD_LINES_8: { + aes_cfb16_enc_vaes_mb8(loc_src, loc_dst, loc_len, numRounds, loc_enc_keys, loc_iv); + break; + } + case WORKLOAD_LINES_4: { + aes_cfb16_enc_vaes_mb4(loc_src, loc_dst, loc_len, numRounds, loc_enc_keys, loc_iv); + break; + } + default: + break; + } + + /* changing the remaining buffers for processing */ + numBuffers -= workLoadSize; + buffersProcessed += workLoadSize; + } + } + #endif // if(_IPP32E>=_IPP32E_K1) + + #if (_IPP32E>=_IPP32E_Y8) + if( IsFeatureEnabled(ippCPUID_AES) ) { + while(numBuffers > 0) { + for (i = 0; i < WORKLOAD_LINES_4; i++) { + if (i >= numBuffers) { + loc_len[i] = 0; + continue; + } + + loc_src[i] = pSrc[i + buffersProcessed]; + loc_dst[i] = pDst[i + buffersProcessed]; + loc_iv[i] = pIV[i + buffersProcessed]; + loc_enc_keys[i] = (Ipp32u*)RIJ_EKEYS(pCtx[i + buffersProcessed]); + loc_len[i] = len[i + buffersProcessed]; + /* As numRounds is the same for all buffers, get it from the last one */ + numRounds = RIJ_NR(pCtx[i + buffersProcessed]); + } + + aes_cfb16_enc_aesni_mb4(loc_src, loc_dst, loc_len, numRounds, loc_enc_keys, loc_iv); + numBuffers -= WORKLOAD_LINES_4; + buffersProcessed += WORKLOAD_LINES_4; + } + } + #endif // (_IPP32E>=_IPP32E_Y8) + + for (i = 0; i < numBuffers; i++) { + status[i] = ippsAESEncryptCFB(pSrc[i], pDst[i], len[i], CFB16_BLOCK_SIZE, pCtx[i], pIV[i]); + } + + for (i = 0; i < numBuffers; i++) { + if (status[i] != ippStsNoErr) { + return ippStsErr; + } + } + + return ippStsNoErr; +} + +#undef WORKLOAD_LINES_16 +#undef WORKLOAD_LINES_8 +#undef WORKLOAD_LINES_4 diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cfbdecrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cfbdecrypt.c new file mode 100644 index 0000000..ba7fcd4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cfbdecrypt.c @@ -0,0 +1,180 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption/decryption (CFB mode) +// +// Contents: +// ippsAESDecryptCFB() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + + + +/*F* +// Name: ippsAESDecryptCFB +// +// Purpose: AES-CFB decryption. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// pSrc == NULL +// pDst == NULL +// pIV == NULL +// ippStsContextMatchErr !VALID_AES_ID() +// ippStsLengthErr len <1 +// ippStsCFBSizeErr (1>cfbBlkSize || cfbBlkSize>MBS_RIJ128) +// ippStsUnderRunErr 0!=(dataLen%cfbBlkSize) +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// len output buffer length (in bytes) +// cfbBlkSize CFB block size (in bytes) +// pCtx pointer to the AES context +// pIV pointer to the initialization vector +// +*F*/ +static +void cpDecryptAES_cfb(const Ipp8u* pIV, + const Ipp8u* pSrc, Ipp8u* pDst, int nBlocks, int cfbBlkSize, + const IppsAESSpec* pCtx) +{ +#if(_IPP32E>=_IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512VAES)) { + if(cfbBlkSize==MBS_RIJ128) + DecryptCFB128_RIJ128pipe_VAES_NI(pSrc, pDst, nBlocks*cfbBlkSize, pCtx, pIV); + else if (8 == cfbBlkSize) + DecryptCFB64_RIJ128pipe_VAES_NI(pSrc, pDst, nBlocks*cfbBlkSize, pCtx, pIV); + else + #if !defined (__INTEL_COMPILER) && defined (_MSC_VER) && (_MSC_VER < 1920) + goto msvc_fallback; + #else + DecryptCFB_RIJ128pipe_VAES_NI(pSrc, pDst, nBlocks*cfbBlkSize, cfbBlkSize, pCtx, pIV); + #endif + } + else +#endif +#if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + /* use pipelined version is possible */ + if(AES_NI_ENABLED==RIJ_AESNI(pCtx)) { + #if !defined (__INTEL_COMPILER) && defined (_MSC_VER) && (_MSC_VER < 1920) && (_IPP32E>=_IPP32E_K1) +msvc_fallback: + #endif + if(cfbBlkSize==MBS_RIJ128) + DecryptCFB128_RIJ128pipe_AES_NI(pSrc, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), nBlocks*cfbBlkSize, pIV); + else if(0==(cfbBlkSize&3)) + DecryptCFB32_RIJ128pipe_AES_NI(pSrc, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), nBlocks, cfbBlkSize, pIV); + else + DecryptCFB_RIJ128pipe_AES_NI(pSrc, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), nBlocks, cfbBlkSize, pIV); + } + else +#endif + { + Ipp32u tmpInp[2*NB(128)]; + Ipp32u tmpOut[ NB(128)]; + + /* setup encoder method */ + RijnCipher encoder = RIJ_ENCODER(pCtx); + + /* read IV */ + CopyBlock16(pIV, tmpInp); + + /* decrypt data block-by-block of cfbLen each */ + while(nBlocks) { + /* decryption */ + //encoder(tmpInp, tmpOut, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), (const Ipp32u (*)[256])RIJ_ENC_SBOX(pCtx)); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder((Ipp8u*)tmpInp, (Ipp8u*)tmpOut, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), RijEncSbox/*NULL*/); + #else + encoder((Ipp8u*)tmpInp, (Ipp8u*)tmpOut, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), NULL); + #endif + + /* store output and put feedback into the input buffer (tmpInp) */ + if( cfbBlkSize==MBS_RIJ128 && pSrc!=pDst) { + ((Ipp32u*)pDst)[0] = tmpOut[0]^((Ipp32u*)pSrc)[0]; + ((Ipp32u*)pDst)[1] = tmpOut[1]^((Ipp32u*)pSrc)[1]; + ((Ipp32u*)pDst)[2] = tmpOut[2]^((Ipp32u*)pSrc)[2]; + ((Ipp32u*)pDst)[3] = tmpOut[3]^((Ipp32u*)pSrc)[3]; + + tmpInp[0] = ((Ipp32u*)pSrc)[0]; + tmpInp[1] = ((Ipp32u*)pSrc)[1]; + tmpInp[2] = ((Ipp32u*)pSrc)[2]; + tmpInp[3] = ((Ipp32u*)pSrc)[3]; + } + else { + int n; + for(n=0; ncfbBlkSize) || (MBS_RIJ128= 1920) + +#include "owncp.h" +#include "pcpaesm.h" +#include "pcpaes_encrypt_vaes512.h" + +#if(_IPP32E>=_IPP32E_K1) +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(disable: 4310) // zmmintrin.h bug: truncation of constant value +#endif + +__INLINE Ipp64u broadcast_16to64(Ipp16u mask16) +{ + Ipp64u mask64 = (Ipp64u)mask16; + mask64 = (mask64 << 48) | (mask64 << 32) | (mask64 << 16) | mask64; + return mask64; +} + +__INLINE __m512i getInputBlocks(__m128i * const currentState, const __m512i * const pCipherBlocks, __mmask16 blocksCompressMask) +{ + // extract 128-bit cipher blocks + __m128i c0 = _mm512_extracti64x2_epi64(*pCipherBlocks, 0); + __m128i c1 = _mm512_extracti64x2_epi64(*pCipherBlocks, 1); + __m128i c2 = _mm512_extracti64x2_epi64(*pCipherBlocks, 2); + __m128i c3 = _mm512_extracti64x2_epi64(*pCipherBlocks, 3); + + // InpBlk_j = LSB_(b-s)(InpBlk_(j-1)) | C_(j-1); b = 128 bit, s = 8*cfbBlkSize bit + __m128i inpBlk0 = *currentState; + // drop MSB bits (size = cfbBlkSize) from the inpBlk_i (by mask) and append c_(i-1) to LSB bits + __m128i inpBlk1 = _mm_mask_compress_epi8(c0, blocksCompressMask, inpBlk0); + __m128i inpBlk2 = _mm_mask_compress_epi8(c1, blocksCompressMask, inpBlk1); + __m128i inpBlk3 = _mm_mask_compress_epi8(c2, blocksCompressMask, inpBlk2); + + // next InputBlock ready + *currentState = _mm_mask_compress_epi8(c3, blocksCompressMask, inpBlk3); + + // inserts + __m512i inpBlk512 = _mm512_setzero_si512(); + inpBlk512 = _mm512_mask_broadcast_i64x2(inpBlk512, 0x03, inpBlk0); // 0000 0011 + inpBlk512 = _mm512_mask_broadcast_i64x2(inpBlk512, 0x0C, inpBlk1); // 0000 1100 + inpBlk512 = _mm512_mask_broadcast_i64x2(inpBlk512, 0x30, inpBlk2); // 0011 0000 + inpBlk512 = _mm512_mask_broadcast_i64x2(inpBlk512, 0xC0, inpBlk3); // 1100 0000 + + return inpBlk512; +} + +//////////////////////////////////////////////////////////////////////////////// +IPP_OWN_DEFN (void, DecryptCFB_RIJ128pipe_VAES_NI, (const Ipp8u* pSrc, // pointer to the ciphertext + Ipp8u* pDst, // pointer to the plaintext + int len, // message length + int cfbBlkSize, // CFB block size in bytes (1 <= cfbBlkSize <= 16)); s = 8*cfbBlkSize + const IppsAESSpec* pCtx, // pointer to context + const Ipp8u* pIV)) // pointer to the Initialization Vector +{ + const int cipherRounds = RIJ_NR(pCtx) - 1; + + __m128i* pRkey = (__m128i*)RIJ_EKEYS(pCtx); + Ipp8u* pSrc8 = (Ipp8u*)pSrc; + Ipp8u* pDst8 = (Ipp8u*)pDst; + + const int bytesPerLoad512 = 4 * cfbBlkSize; + const Ipp16u blocksCompressMask = (Ipp16u)(0xFFFF << cfbBlkSize); + + // load masks; allows to load 4 source blocks of cfbBlkSize each (in bytes) to LSB parts of 128-bit lanes in 512-bit register + __mmask64 kLsbMask64 = (__mmask64)broadcast_16to64((Ipp16u)(0xFFFF << (16-cfbBlkSize))); + // same mask to load in MSB parts + __mmask64 kMsbMask64 = (__mmask64)broadcast_16to64(~blocksCompressMask); + + // load IV + __m128i IV128 = _mm_maskz_loadu_epi64(0x03 /* load 128-bit */, pIV); + + __m128i currentState = IV128; + + int blocks; + for (blocks = len / cfbBlkSize; blocks >= (4 * 4); blocks -= (4 * 4)) { + // load cipher blocks to LSB parts of registers + __m512i ciphLsb0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc8); + __m512i ciphLsb1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc8 + 1 * bytesPerLoad512); + __m512i ciphLsb2 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc8 + 2 * bytesPerLoad512); + __m512i ciphLsb3 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc8 + 3 * bytesPerLoad512); + + // load same cipher blocks to MSB parts of registers (shall be taken from cache) + __m512i ciphMsb0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc8); + __m512i ciphMsb1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc8 + 1 * bytesPerLoad512); + __m512i ciphMsb2 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc8 + 2 * bytesPerLoad512); + __m512i ciphMsb3 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc8 + 3 * bytesPerLoad512); + + // prepare InputBlocks for decryption + __m512i inpBlk0 = getInputBlocks(¤tState, &ciphLsb0, blocksCompressMask); + __m512i inpBlk1 = getInputBlocks(¤tState, &ciphLsb1, blocksCompressMask); + __m512i inpBlk2 = getInputBlocks(¤tState, &ciphLsb2, blocksCompressMask); + __m512i inpBlk3 = getInputBlocks(¤tState, &ciphLsb3, blocksCompressMask); + + cpAESEncrypt4_VAES_NI(&inpBlk0, &inpBlk1, &inpBlk2, &inpBlk3, pRkey, cipherRounds); + + ciphLsb0 = _mm512_xor_si512(ciphMsb0, inpBlk0); + ciphLsb1 = _mm512_xor_si512(ciphMsb1, inpBlk1); + ciphLsb2 = _mm512_xor_si512(ciphMsb2, inpBlk2); + ciphLsb3 = _mm512_xor_si512(ciphMsb3, inpBlk3); + + _mm512_mask_compressstoreu_epi8(pDst8, kMsbMask64, ciphLsb0); + _mm512_mask_compressstoreu_epi8(pDst8 + 1 * bytesPerLoad512, kMsbMask64, ciphLsb1); + _mm512_mask_compressstoreu_epi8(pDst8 + 2 * bytesPerLoad512, kMsbMask64, ciphLsb2); + _mm512_mask_compressstoreu_epi8(pDst8 + 3 * bytesPerLoad512, kMsbMask64, ciphLsb3); + + pSrc8 += 4 * bytesPerLoad512; + pDst8 += 4 * bytesPerLoad512; + } + + if ((3 * 4) <= blocks) { + // load ciphertext + __m512i ciphLsb0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc8); + __m512i ciphLsb1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc8 + 1 * bytesPerLoad512); + __m512i ciphLsb2 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc8 + 2 * bytesPerLoad512); + + __m512i ciphMsb0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc8); + __m512i ciphMsb1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc8 + 1 * bytesPerLoad512); + __m512i ciphMsb2 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc8 + 2 * bytesPerLoad512); + + // prepare InputBlocks for decryption + __m512i inpBlk0 = getInputBlocks(¤tState, &ciphLsb0, blocksCompressMask); + __m512i inpBlk1 = getInputBlocks(¤tState, &ciphLsb1, blocksCompressMask); + __m512i inpBlk2 = getInputBlocks(¤tState, &ciphLsb2, blocksCompressMask); + + cpAESEncrypt3_VAES_NI(&inpBlk0, &inpBlk1, &inpBlk2, pRkey, cipherRounds); + + ciphLsb0 = _mm512_xor_si512(ciphMsb0, inpBlk0); + ciphLsb1 = _mm512_xor_si512(ciphMsb1, inpBlk1); + ciphLsb2 = _mm512_xor_si512(ciphMsb2, inpBlk2); + + _mm512_mask_compressstoreu_epi8(pDst8, kMsbMask64, ciphLsb0); + _mm512_mask_compressstoreu_epi8(pDst8 + 1 * bytesPerLoad512, kMsbMask64, ciphLsb1); + _mm512_mask_compressstoreu_epi8(pDst8 + 2 * bytesPerLoad512, kMsbMask64, ciphLsb2); + + pSrc8 += 3 * bytesPerLoad512; + pDst8 += 3 * bytesPerLoad512; + blocks -= (3 * 4); + } + + if ((4 * 2) <= blocks) { + // load ciphertext + __m512i ciphLsb0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc8); + __m512i ciphLsb1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc8 + 1 * bytesPerLoad512); + + __m512i ciphMsb0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc8); + __m512i ciphMsb1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc8 + 1 * bytesPerLoad512); + + // prepare InputBlocks for decryption + __m512i inpBlk0 = getInputBlocks(¤tState, &ciphLsb0, blocksCompressMask); + __m512i inpBlk1 = getInputBlocks(¤tState, &ciphLsb1, blocksCompressMask); + + cpAESEncrypt2_VAES_NI(&inpBlk0, &inpBlk1, pRkey, cipherRounds); + + ciphLsb0 = _mm512_xor_si512(ciphMsb0, inpBlk0); + ciphLsb1 = _mm512_xor_si512(ciphMsb1, inpBlk1); + + _mm512_mask_compressstoreu_epi8(pDst8, kMsbMask64, ciphLsb0); + _mm512_mask_compressstoreu_epi8(pDst8 + 1 * bytesPerLoad512, kMsbMask64, ciphLsb1); + + pSrc8 += 2 * bytesPerLoad512; + pDst8 += 2 * bytesPerLoad512; + blocks -= (2 * 4); + } + + for (; blocks >= 4; blocks -= 4) { + // load ciphertext + __m512i ciphLsb0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc8); + __m512i ciphMsb0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc8); + + // prepare InputBlocks for decryption + __m512i inpBlk0 = getInputBlocks(¤tState, &ciphLsb0, blocksCompressMask); + + cpAESEncrypt1_VAES_NI(&inpBlk0, pRkey, cipherRounds); + + ciphLsb0 = _mm512_xor_si512(ciphMsb0, inpBlk0); + + _mm512_mask_compressstoreu_epi8(pDst8, kMsbMask64, ciphLsb0); + + pSrc8 += bytesPerLoad512; + pDst8 += bytesPerLoad512; + } + + // at least one block left (max 3 blocks) + if (blocks) { + __mmask64 k64 = (1LL << (blocks << 4)) - 1; + + // load ciphertext + __m512i ciphLsb0 = _mm512_maskz_expandloadu_epi8(k64 & kLsbMask64, pSrc8); + __m512i ciphMsb0 = _mm512_maskz_expandloadu_epi8(k64 & kMsbMask64, pSrc8); + + // prepare InputBlocks for decryption + __m512i inpBlk0 = _mm512_setzero_si512(); + inpBlk0 = _mm512_mask_broadcast_i64x2(inpBlk0, 0x03, currentState); + __m128i c; + for (int i = 0; i < blocks; i++) { + // NB: we cannot provide non-immediate parameter to extract function + switch (i) { + case 0: c = _mm512_extracti64x2_epi64(ciphLsb0, 0); break; + case 1: c = _mm512_extracti64x2_epi64(ciphLsb0, 1); break; + default: c = _mm512_extracti64x2_epi64(ciphLsb0, 2); + } + currentState = _mm_mask_compress_epi8(c, blocksCompressMask, currentState); + inpBlk0 = _mm512_mask_broadcast_i64x2(inpBlk0, (__mmask8)(0x03 << ((i + 1) << 1)), currentState); + } + + cpAESEncrypt1_VAES_NI(&inpBlk0, pRkey, cipherRounds); + + ciphLsb0 = _mm512_xor_si512(ciphMsb0, inpBlk0); + + _mm512_mask_compressstoreu_epi8(pDst8, k64 & kMsbMask64, ciphLsb0); + } +} + +#endif /* _IPP32E>=_IPP32E_K1 */ +#else +typedef int to_avoid_translation_unit_is_empty_warning; +#endif /* #if defined (__INTEL_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cfbencrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cfbencrypt.c new file mode 100644 index 0000000..e36fe7d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cfbencrypt.c @@ -0,0 +1,144 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption/decryption (CFB mode) +// +// Contents: +// ippsAESEncryptCFB() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + + +/*F* +// Name: ippsAESEncryptCFB +// +// Purpose: AES-CFB encryption. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// pSrc == NULL +// pDst == NULL +// pIV == NULL +// ippStsContextMatchErr !VALID_AES_ID() +// ippStsLengthErr len <1 +// ippStsCFBSizeErr (1>cfbBlkSize || cfbBlkSize>MBS_RIJ128) +// ippStsUnderRunErr 0!=(dataLen%cfbBlkSize) +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// len input buffer length (in bytes) +// cfbBlkSize CFB block size (in bytes) +// pCtx pointer to the AES context +// pIV pointer to the initialization vector +// +*F*/ +IPPFUN(IppStatus, ippsAESEncryptCFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int cfbBlkSize, + const IppsAESSpec* pCtx, + const Ipp8u* pIV)) +{ + /* test context */ + IPP_BAD_PTR1_RET(pCtx); + /* test the context ID */ + IPP_BADARG_RET(!VALID_AES_ID(pCtx), ippStsContextMatchErr); + + /* test source, target buffers and initialization pointers */ + IPP_BAD_PTR3_RET(pSrc, pIV, pDst); + /* test stream length */ + IPP_BADARG_RET((len<1), ippStsLengthErr); + /* test CFB value */ + IPP_BADARG_RET(((1>cfbBlkSize) || (MBS_RIJ128=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + /* use pipelined version is possible */ + if(AES_NI_ENABLED==RIJ_AESNI(pCtx)) { + if(cfbBlkSize==MBS_RIJ128) + EncryptCFB128_RIJ128_AES_NI(pSrc, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), len, pIV); + else if(0==(cfbBlkSize&3)) + EncryptCFB32_RIJ128_AES_NI(pSrc, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), len, cfbBlkSize, pIV); + else + EncryptCFB_RIJ128_AES_NI(pSrc, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), len, cfbBlkSize, pIV); + return ippStsNoErr; + } + else +#endif + + { + Ipp32u tmpInp[2*NB(128)]; + Ipp32u tmpOut[ NB(128)]; + + /* setup encoder method */ + RijnCipher encoder = RIJ_ENCODER(pCtx); + + /* read IV */ + CopyBlock16(pIV, tmpInp); + + /* encrypt data block-by-block of cfbLen each */ + while(len>=cfbBlkSize) { + int n; + + /* encryption */ + //encoder(tmpInp, tmpOut, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), (const Ipp32u (*)[256])RIJ_ENC_SBOX(pCtx)); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder((Ipp8u*)tmpInp, (Ipp8u*)tmpOut, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), RijEncSbox/*NULL*/); + #else + encoder((Ipp8u*)tmpInp, (Ipp8u*)tmpOut, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), NULL); + #endif + + /* store output and put feedback into the input buffer (tmpInp) */ + if( cfbBlkSize==MBS_RIJ128 && pSrc!=pDst) { + tmpInp[0] = ((Ipp32u*)pDst)[0] = tmpOut[0]^((Ipp32u*)pSrc)[0]; + tmpInp[1] = ((Ipp32u*)pDst)[1] = tmpOut[1]^((Ipp32u*)pSrc)[1]; + tmpInp[2] = ((Ipp32u*)pDst)[2] = tmpOut[2]^((Ipp32u*)pSrc)[2]; + tmpInp[3] = ((Ipp32u*)pDst)[3] = tmpOut[3]^((Ipp32u*)pSrc)[3]; + } + else { + for(n=0; n>8) & 0xFF; + } +} + + +/*F* +// Name: ippsAES_CMACInit +// +// Purpose: Init AES-CMAC context. +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL +// ippStsMemAllocErr size of buffer is not match fro operation +// ippStsLengthErr keyLen != 16 +// keyLen != 24 +// keyLen != 32 +// ippStsNoErr no errors +// +// Parameters: +// pKey pointer to the secret key +// keyLen length of secret key +// pState pointer to the CMAC context +// ctxSize available size (in bytes) of buffer above +// +*F*/ +IPPFUN(IppStatus, ippsAES_CMACInit,(const Ipp8u* pKey, int keyLen, IppsAES_CMACState* pState, int ctxSize)) +{ + /* test pState pointer */ + IPP_BAD_PTR1_RET(pState); + + /* test available size of context buffer */ + IPP_BADARG_RET(ctxSize>7)) & 0x87); /* ^ Rb changed for constant time execution */ + /* precompute k2 subkey */ + msb = (CMAC_K1(pState))[0]; + LogicalLeftSift16(CMAC_K1(pState),CMAC_K2(pState)); + (CMAC_K2(pState))[MBS_RIJ128-1] ^= (Ipp8u)((0-(msb>>7)) & 0x87); /* ^ Rb changed for constant time execution */ + } + + return sts; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cmacupdate.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cmacupdate.c new file mode 100644 index 0000000..549f7b1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_cmacupdate.c @@ -0,0 +1,198 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-CMAC Functions +// +// Contents: +// ippsAES_CMACUpdate() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpcmac.h" +#include "pcpaesm.h" +#include "pcptool.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + + +/*F* +// Name: ippsAES_CMACUpdate +// +// Purpose: Updates intermadiate digest based on input stream. +// +// Returns: Reason: +// ippStsNullPtrErr pSrc == NULL +// pState == NULL +// ippStsContextMatchErr !VALID_AESCMAC_ID() +// ippStsLengthErr len <0 +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the input stream +// len input stream length +// pState pointer to the CMAC context +// +*F*/ +static void AES_CMAC_processing(Ipp8u* pDigest, const Ipp8u* pSrc, int processedLen, const IppsAESSpec* pAES) +{ +#if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + if(AES_NI_ENABLED==RIJ_AESNI(pAES)) { + cpAESCMAC_Update_AES_NI(pDigest, pSrc, processedLen, RIJ_NR(pAES), RIJ_EKEYS(pAES)); + } + else +#endif + { + /* setup encoder method */ + RijnCipher encoder = RIJ_ENCODER(pAES); + + while(processedLen) { + ((Ipp32u*)pDigest)[0] ^= ((Ipp32u*)pSrc)[0]; + ((Ipp32u*)pDigest)[1] ^= ((Ipp32u*)pSrc)[1]; + ((Ipp32u*)pDigest)[2] ^= ((Ipp32u*)pSrc)[2]; + ((Ipp32u*)pDigest)[3] ^= ((Ipp32u*)pSrc)[3]; + + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder(pDigest, pDigest, RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder(pDigest, pDigest, RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif + + pSrc += MBS_RIJ128; + processedLen -= MBS_RIJ128; + } + } +} + +IPPFUN(IppStatus, ippsAES_CMACUpdate,(const Ipp8u* pSrc, int len, IppsAES_CMACState* pState)) +{ + int processedLen; + + /* test context pointer */ + IPP_BAD_PTR1_RET(pState); + /* test ID */ + IPP_BADARG_RET(!VALID_AESCMAC_ID(pState), ippStsContextMatchErr); + /* test input message and it's length */ + IPP_BADARG_RET((len<0 && pSrc), ippStsLengthErr); + /* test source pointer */ + IPP_BADARG_RET((len && !pSrc), ippStsNullPtrErr); + + if(!len) + return ippStsNoErr; + + { + /* + // test internal buffer filling + */ + if(CMAC_INDX(pState)) { + /* copy from input stream to the internal buffer as match as possible */ + processedLen = IPP_MIN(len, (MBS_RIJ128 - CMAC_INDX(pState))); + CopyBlock(pSrc, CMAC_BUFF(pState)+CMAC_INDX(pState), processedLen); + + /* internal buffer filling */ + CMAC_INDX(pState) += processedLen; + + /* update message pointer and length */ + pSrc += processedLen; + len -= processedLen; + + if(!len) + return ippStsNoErr; + + /* update CMAC if buffer full but not the last */ + if(MBS_RIJ128==CMAC_INDX(pState) ) { + const IppsAESSpec* pAES = &CMAC_CIPHER(pState); + /* setup encoder method */ + RijnCipher encoder = RIJ_ENCODER(pAES); + XorBlock16(CMAC_BUFF(pState), CMAC_MAC(pState), CMAC_MAC(pState)); + + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder(CMAC_MAC(pState), CMAC_MAC(pState), RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder(CMAC_MAC(pState), CMAC_MAC(pState), RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif + + CMAC_INDX(pState) = 0; + } + } + + /* + // main part + */ + + processedLen = len & ~(MBS_RIJ128-1); + if(!(len & (MBS_RIJ128-1))) + processedLen -= MBS_RIJ128; + if (processedLen) { + const IppsAESSpec *pAES = &CMAC_CIPHER(pState); + +#if (_AES_PROB_NOISE == _FEATURE_ON_) + /* Mistletoe3 mitigation */ + cpAESNoiseParams *params = (cpAESNoiseParams *)&AESCMAC_NOISE_PARAMS(pState); + if (AES_NOISE_LEVEL(params) > 0) { + /* Number of bytes allowed for operation without adding noise */ + int chunk_size; + /* Number of bytes remaining for operation */ + int remaining_size = processedLen; + + while (remaining_size > 0) { + /* How many bytes to encrypt in this operation */ + chunk_size = (remaining_size >= MISTLETOE3_MAX_CHUNK_SIZE) ? MISTLETOE3_MAX_CHUNK_SIZE : remaining_size; + + AES_CMAC_processing(CMAC_MAC(pState), pSrc, chunk_size, pAES); + + cpAESRandomNoise(NULL, + MISTLETOE3_BASE_NOISE_LEVEL + AES_NOISE_LEVEL(params), + MISTLETOE3_NOISE_RATE, + &AES_NOISE_RAND(params)); + + pSrc += chunk_size; + remaining_size -= chunk_size; + } + } else +#endif + { + AES_CMAC_processing(CMAC_MAC(pState), pSrc, processedLen, pAES); + /* update message pointer and length */ + pSrc += processedLen; + } + + len -= processedLen; + } + + /* + // remaind + */ + if(len) { + /* workaround to avoid false positive stringop-overflow error on gcc10.1 and gcc11.1 */ + len = ( IPP_MIN(len, MBS_RIJ128) ); + + CopyBlock(pSrc, (Ipp8u*)(&CMAC_BUFF(pState)), len); + /* update internal buffer filling */ + CMAC_INDX(pState) += len; + } + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ctr_process.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ctr_process.h new file mode 100644 index 0000000..62aa1f6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ctr_process.h @@ -0,0 +1,255 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption/decryption (CTR mode) +// +// Contents: +// cpProcessAES_ctr() +// cpProcessAES_ctr128() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + +/* +// AES-CRT processing. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// pSrc == NULL +// pDst == NULL +// pCtrValue ==NULL +// ippStsContextMatchErr !VALID_AES_ID() +// ippStsLengthErr len <1 +// ippStsCTRSizeErr 128 < ctrNumBitSize < 1 +// ippStsCTRSizeErr data blocks number > 2^ctrNumBitSize +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// dataLen input/output buffer length (in bytes) +// pCtx pointer to rge AES context +// pCtrValue pointer to the counter block +// ctrNumBitSize counter block size (bits) +// +// Note: +// counter will updated on return +// +*/ +__INLINE void MaskCounter128(Ipp8u* pMaskIV, int ctrBtSize) +{ + /* construct ctr mask */ + int maskPosition = (MBS_RIJ128*8-ctrBtSize)/8; + Ipp8u maskValue = (Ipp8u)(0xFF >> (MBS_RIJ128*8-ctrBtSize)%8 ); + + //Ipp8u maskIV[MBS_RIJ128]; + int n; + for(n=0; n= 8 * sizeof(int) - 5 + // function can process data with any possible + // passed dataLen without counter overflow + */ + + int dataBlocksNum = dataLen >> 4; + if(dataLen & 15){ + dataBlocksNum++; + } + + IPP_BADARG_RET(dataBlocksNum > (1 << ctrNumBitSize), ippStsCTRSizeErr); + } + + #if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + /* use pipelined version if possible */ + if(AES_NI_ENABLED==RIJ_AESNI(pCtx)) { + /* construct ctr mask */ + Ipp8u maskIV[MBS_RIJ128]; + MaskCounter128(maskIV, ctrNumBitSize); /* const-exe-time version */ + +#if(_IPP32E>=_IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512VAES)) { + EncryptCTR_RIJ128pipe_VAES_NI(pSrc, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), dataLen, pCtrValue, maskIV); + } + else +#endif + { + EncryptCTR_RIJ128pipe_AES_NI(pSrc, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), dataLen, pCtrValue, maskIV); + } + return ippStsNoErr; + } + else + #endif + { + Ipp32u counter[NB(128)]; + Ipp32u output[NB(128)]; + + /* setup encoder method */ + RijnCipher encoder = RIJ_ENCODER(pCtx); + + /* copy counter */ + CopyBlock16(pCtrValue, counter); + + /* + // encrypt block-by-block aligned streams + */ + while(dataLen>= MBS_RIJ128) { + /* encrypt counter block */ + //encoder(counter, output, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), (const Ipp32u (*)[256])RIJ_ENC_SBOX(pCtx)); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder((Ipp8u*)counter, (Ipp8u*)output, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), RijEncSbox/*NULL*/); + #else + encoder((Ipp8u*)counter, (Ipp8u*)output, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), NULL); + #endif + + /* compute ciphertext block */ + if( !(IPP_UINT_PTR(pSrc) & 0x3) && !(IPP_UINT_PTR(pDst) & 0x3)) { + ((Ipp32u*)pDst)[0] = output[0]^((Ipp32u*)pSrc)[0]; + ((Ipp32u*)pDst)[1] = output[1]^((Ipp32u*)pSrc)[1]; + ((Ipp32u*)pDst)[2] = output[2]^((Ipp32u*)pSrc)[2]; + ((Ipp32u*)pDst)[3] = output[3]^((Ipp32u*)pSrc)[3]; + } + else + XorBlock16(pSrc, output, pDst); + /* encrement counter block */ + StdIncrement((Ipp8u*)counter,MBS_RIJ128*8, ctrNumBitSize); + + pSrc += MBS_RIJ128; + pDst += MBS_RIJ128; + dataLen -= MBS_RIJ128; + } + /* + // encrypt last data block + */ + if(dataLen) { + /* encrypt counter block */ + //encoder(counter, output, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), (const Ipp32u (*)[256])RIJ_ENC_SBOX(pCtx)); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder((Ipp8u*)counter, (Ipp8u*)output, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), RijEncSbox/*NULL*/); + #else + encoder((Ipp8u*)counter, (Ipp8u*)output, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), NULL); + #endif + + /* compute ciphertext block */ + XorBlock(pSrc, output, pDst,dataLen); + /* encrement counter block */ + StdIncrement((Ipp8u*)counter,MBS_RIJ128*8, ctrNumBitSize); + } + + /* update counter */ + CopyBlock16(counter, pCtrValue); + + return ippStsNoErr; + } +} + +#if (_IPP32E>=_IPP32E_Y8) + +/* +// special version: 128-bit counter +*/ +static +IppStatus cpProcessAES_ctr128(const Ipp8u* pSrc, Ipp8u* pDst, int dataLen, const IppsAESSpec* pCtx, Ipp8u* pCtrValue) +{ + /* test context */ + IPP_BAD_PTR1_RET(pCtx); + /* test the context ID */ + IPP_BADARG_RET(!VALID_AES_ID(pCtx), ippStsContextMatchErr); + + /* test source, target and counter block pointers */ + IPP_BAD_PTR3_RET(pSrc, pDst, pCtrValue); + /* test stream length */ + IPP_BADARG_RET((dataLen<1), ippStsLengthErr); + + { + while(dataLen>=MBS_RIJ128) { + Ipp32u blocks = (Ipp32u)(dataLen>>4); /* number of blocks per loop processing */ + + /* low LE 32 bit of counter */ + Ipp32u ctr32 = ((Ipp32u*)(pCtrValue))[3]; + ctr32 = ENDIANNESS32(ctr32); + + /* compute number of locks being processed without ctr32 overflow */ + ctr32 += blocks; + if(ctr32 < blocks) + blocks -= ctr32; + +#if(_IPP32E>=_IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512VAES)) { + EncryptStreamCTR32_VAES_NI(pSrc, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), (Ipp32s)blocks*MBS_RIJ128, pCtrValue); + } + else +#endif + EncryptStreamCTR32_AES_NI(pSrc, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), (Ipp32s)blocks*MBS_RIJ128, pCtrValue); + + pSrc += blocks*MBS_RIJ128; + pDst += blocks*MBS_RIJ128; + dataLen -= blocks*MBS_RIJ128; + } + + if(dataLen) { + EncryptStreamCTR32_AES_NI(pSrc, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), dataLen, pCtrValue); + } + + return ippStsNoErr; + } +} + +#endif /* #if (_IPP32E>=_IPP32E_Y8) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ctrdecrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ctrdecrypt.c new file mode 100644 index 0000000..d92828d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ctrdecrypt.c @@ -0,0 +1,83 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption/decryption (CTR mode) +// +// Contents: +// ippsAESDecryptCTR() +// +*/ +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(disable: 4206) // empty unit +#endif + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" +#include "pcpaes_ctr_process.h" + +/* +// Name: ippsAESDecryptCTR +// +// Purpose: +// AES-CFB encryption. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// pSrc == NULL +// pDst == NULL +// pCtrValue ==NULL +// ippStsContextMatchErr !VALID_AES_ID() +// ippStsLengthErr len <1 +// ippStsCTRSizeErr 128 < ctrNumBitSize < 1 +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// len output buffer length (in bytes) +// pCtx pointer to rge AES context +// pCtrValue pointer to the counter block +// ctrNumBitSize counter block size (bits) +// +// Note: +// counter will updated on return +// +*/ + +IPPFUN(IppStatus, ippsAESDecryptCTR,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + Ipp8u* pCtrValue, int ctrNumBitSize)) +{ + /* test context */ + IPP_BAD_PTR1_RET(pCtx); + + #if(_IPP32E>=_IPP32E_Y8) + if(AES_NI_ENABLED==RIJ_AESNI(pCtx)) + return ctrNumBitSize==128? cpProcessAES_ctr128(pSrc, pDst, len, pCtx, pCtrValue) : + cpProcessAES_ctr(pSrc, pDst, len, pCtx, pCtrValue, ctrNumBitSize); + else + return cpProcessAES_ctr(pSrc, pDst, len, pCtx, pCtrValue, ctrNumBitSize); + #else + return cpProcessAES_ctr(pSrc, pDst, len, pCtx, pCtrValue, ctrNumBitSize); + #endif +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ctrencrypt_rij128pipe_vaes512.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ctrencrypt_rij128pipe_vaes512.c new file mode 100644 index 0000000..5ffd378 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ctrencrypt_rij128pipe_vaes512.c @@ -0,0 +1,265 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption/decryption (CTR mode) +// +// Contents: +// EncryptCTR_RIJ128pipe_VAES_NI +// +*/ + +#include "owncp.h" +#include "pcpaes_encrypt_vaes512.h" +#include "pcpaesm.h" + +#if (_IPP32E >= _IPP32E_K1) +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(disable : 4310) // cast truncates constant value in MSVC +#endif + +#define M512(mem) (*((__m512i *)((Ipp8u *)(mem)))) + +/* Mask to convert 64-bit parts of four 128-bit numbers stored in one 512-bit register + * from Little-Endian to Big-Endian */ + +/* clang-format off */ +static __ALIGN32 Ipp8u swapBytes[] = { + 7, 6, 5, 4, 3, 2, 1, 0, 15,14,13,12, 11,10, 9, 8, + 23,22,21,20, 19,18,17,16, 31,30,29,28, 27,26,25,24, + 39,38,37,36, 35,34,33,32, 47,46,45,44, 43,42,41,40, + 55,54,53,52, 51,50,49,48, 63,62,61,60, 59,58,57,56 +}; +/* clang-format on */ + +/* Increment masks for Hi-Lo 64-bit parts of 128-bit numbers in 512-bit register */ +static __ALIGN64 Ipp64u startIncLoMask[] = { 0x0, 0x0, 0x0, 0x1, 0x0, 0x2, 0x0, 0x3 }; +static __ALIGN64 Ipp64u nextIncLoMask[] = { 0x0, 0x4, 0x0, 0x4, 0x0, 0x4, 0x0, 0x4 }; +static __ALIGN64 Ipp64u incLoByOneMask[] = { 0x0, 0x1, 0x0, 0x1, 0x0, 0x1, 0x0, 0x1 }; +static __ALIGN64 Ipp64u incHiByOneMask[] = { 0x1, 0x0, 0x1, 0x0, 0x1, 0x0, 0x1, 0x0 }; + +__INLINE __m512i adcLo_epi64(__m512i a, __m512i b) +{ + a = _mm512_add_epi64(a, b); + // check overflow in each low 64-bit of 128-bit numbers + __mmask8 overMsk = _mm512_cmplt_epu64_mask(a, b); + // get mask of each high 64-bit that need to be increased + overMsk <<= 1; + a = _mm512_mask_add_epi64(a, overMsk, a, M512(incHiByOneMask)); + return a; +} + +__INLINE __m512i applyNonce(__m512i a, __m512i ctrBitMask, __m512i templateCtr) +{ + a = _mm512_shuffle_epi8(a, M512(swapBytes)); + a = _mm512_and_epi64(a, ctrBitMask); + a = _mm512_or_epi64(a, templateCtr); + + return a; +} + +//////////////////////////////////////////////////////////////////////////////// +/* clang-format off */ +IPP_OWN_DEFN (void, EncryptCTR_RIJ128pipe_VAES_NI, (const Ipp8u* pSrc, + Ipp8u* pDst, + int nr, + const Ipp8u* pRKey, + int length, /* message length in bytes */ + Ipp8u* pCtrValue, + const Ipp8u* pCtrBitMask)) +/* clang-format on */ +{ + int cipherRounds = nr - 1; + + __m128i *pKeys = (__m128i *)pRKey; + __m512i *pSrc512 = (__m512i *)pSrc; + __m512i *pDst512 = (__m512i *)pDst; + + Ipp64u *pCtr64 = (Ipp64u *)pCtrValue; + + // Initial counter + __m128i initialCtr128 = _mm_maskz_loadu_epi64(0x03, pCtrValue); + __m128i ctrBitMask128 = _mm_maskz_loadu_epi64(0x03, pCtrBitMask); + // Unchanged counter part + __m128i templateCtr128 = _mm_maskz_andnot_epi64(0x03 /* all 128-bits */, ctrBitMask128, initialCtr128); + + __m512i ctrBitMask512 = _mm512_broadcast_i64x2(ctrBitMask128); + __m512i templateCtr512 = _mm512_broadcast_i64x2(templateCtr128); + + Ipp64u ctr64_h = ENDIANNESS64(pCtr64[0]); // high 64-bit of BE counter converted to LE + Ipp64u ctr64_l = ENDIANNESS64(pCtr64[1]); // low 64-bit of BE counter converted to LE + + __m512i ctr512 = _mm512_set4_epi64((Ipp64s)ctr64_l, (Ipp64s)ctr64_h, (Ipp64s)ctr64_l, (Ipp64s)ctr64_h); + + // int blocks; + int remainded_length; + __m512i incMsk = M512(startIncLoMask); + for (remainded_length = length; remainded_length >= (4 * 4 * MBS_RIJ128); remainded_length -= (4 * 4 * MBS_RIJ128)) { + __m512i counter0 = adcLo_epi64(incMsk, ctr512); + __m512i counter1 = adcLo_epi64(M512(nextIncLoMask), counter0); + __m512i counter2 = adcLo_epi64(M512(nextIncLoMask), counter1); + __m512i counter3 = adcLo_epi64(M512(nextIncLoMask), counter2); + + incMsk = M512(nextIncLoMask); + ctr512 = counter3; + + // convert back to BE and add nonce + counter0 = applyNonce(counter0, ctrBitMask512, templateCtr512); + counter1 = applyNonce(counter1, ctrBitMask512, templateCtr512); + counter2 = applyNonce(counter2, ctrBitMask512, templateCtr512); + counter3 = applyNonce(counter3, ctrBitMask512, templateCtr512); + + cpAESEncrypt4_VAES_NI(&counter0, &counter1, &counter2, &counter3, pKeys, cipherRounds); + + __m512i blk0 = _mm512_loadu_si512(pSrc512); + __m512i blk1 = _mm512_loadu_si512(pSrc512 + 1); + __m512i blk2 = _mm512_loadu_si512(pSrc512 + 2); + __m512i blk3 = _mm512_loadu_si512(pSrc512 + 3); + + blk0 = _mm512_xor_si512(blk0, counter0); + blk1 = _mm512_xor_si512(blk1, counter1); + blk2 = _mm512_xor_si512(blk2, counter2); + blk3 = _mm512_xor_si512(blk3, counter3); + + _mm512_storeu_si512(pDst512, blk0); + _mm512_storeu_si512(pDst512 + 1, blk1); + _mm512_storeu_si512(pDst512 + 2, blk2); + _mm512_storeu_si512(pDst512 + 3, blk3); + + pSrc512 += 4; + pDst512 += 4; + } + + if ((3 * 4 * MBS_RIJ128) <= remainded_length) { + __m512i counter0 = adcLo_epi64(incMsk, ctr512); + __m512i counter1 = adcLo_epi64(M512(nextIncLoMask), counter0); + __m512i counter2 = adcLo_epi64(M512(nextIncLoMask), counter1); + + incMsk = M512(nextIncLoMask); + ctr512 = counter2; + + // convert back to BE and add nonce + counter0 = applyNonce(counter0, ctrBitMask512, templateCtr512); + counter1 = applyNonce(counter1, ctrBitMask512, templateCtr512); + counter2 = applyNonce(counter2, ctrBitMask512, templateCtr512); + + cpAESEncrypt3_VAES_NI(&counter0, &counter1, &counter2, pKeys, cipherRounds); + + __m512i blk0 = _mm512_loadu_si512(pSrc512); + __m512i blk1 = _mm512_loadu_si512(pSrc512 + 1); + __m512i blk2 = _mm512_loadu_si512(pSrc512 + 2); + + blk0 = _mm512_xor_si512(blk0, counter0); + blk1 = _mm512_xor_si512(blk1, counter1); + blk2 = _mm512_xor_si512(blk2, counter2); + + _mm512_storeu_si512(pDst512, blk0); + _mm512_storeu_si512(pDst512 + 1, blk1); + _mm512_storeu_si512(pDst512 + 2, blk2); + + pSrc512 += 3; + pDst512 += 3; + remainded_length -= (3 * 4 * MBS_RIJ128); + } + + if ((4 * 2 * MBS_RIJ128) <= remainded_length) { + __m512i counter0 = adcLo_epi64(incMsk, ctr512); + __m512i counter1 = adcLo_epi64(M512(nextIncLoMask), counter0); + + incMsk = M512(nextIncLoMask); + ctr512 = counter1; + + // convert back to BE and add nonce + counter0 = applyNonce(counter0, ctrBitMask512, templateCtr512); + counter1 = applyNonce(counter1, ctrBitMask512, templateCtr512); + + cpAESEncrypt2_VAES_NI(&counter0, &counter1, pKeys, cipherRounds); + + __m512i blk0 = _mm512_loadu_si512(pSrc512); + __m512i blk1 = _mm512_loadu_si512(pSrc512 + 1); + + blk0 = _mm512_xor_si512(blk0, counter0); + blk1 = _mm512_xor_si512(blk1, counter1); + + _mm512_storeu_si512(pDst512, blk0); + _mm512_storeu_si512(pDst512 + 1, blk1); + + pSrc512 += 2; + pDst512 += 2; + remainded_length -= (2 * 4 * MBS_RIJ128); + } + + for (; remainded_length >= 4 * MBS_RIJ128; remainded_length -= 4 * MBS_RIJ128) { + __m512i counter0 = adcLo_epi64(incMsk, ctr512); + + incMsk = M512(nextIncLoMask); + ctr512 = counter0; + + // convert back to BE and add nonce + counter0 = applyNonce(counter0, ctrBitMask512, templateCtr512); + + cpAESEncrypt1_VAES_NI(&counter0, pKeys, cipherRounds); + + __m512i blk0 = _mm512_loadu_si512(pSrc512); + blk0 = _mm512_xor_si512(blk0, counter0); + _mm512_storeu_si512(pDst512, blk0); + + pSrc512 += 1; + pDst512 += 1; + } + + if (remainded_length) { + __m512i counter0 = _mm512_add_epi64(incMsk, ctr512); + __mmask8 overMsk = _mm512_cmplt_epu64_mask(counter0, ctr512); + overMsk <<= 1; + counter0 = _mm512_mask_add_epi64(counter0, overMsk, counter0, M512(incHiByOneMask)); + + incMsk = M512(nextIncLoMask); + ctr512 = counter0; + + // convert back to BE and add nonce + counter0 = _mm512_shuffle_epi8(counter0, M512(swapBytes)); + counter0 = _mm512_and_epi64(counter0, ctrBitMask512); + counter0 = _mm512_or_epi64(counter0, templateCtr512); + + cpAESEncrypt1_VAES_NI(&counter0, pKeys, cipherRounds); + + __mmask64 rw_mask = (__mmask64)((1LL << remainded_length) - 1); + + __m512i blk0 = _mm512_maskz_loadu_epi8(rw_mask, pSrc512); + blk0 = _mm512_xor_si512(blk0, counter0); + _mm512_mask_storeu_epi8(pDst512, rw_mask, blk0); + } + + // return last counter + int blocks = remainded_length % MBS_RIJ128 == 0 ? remainded_length / MBS_RIJ128 : (remainded_length / MBS_RIJ128) + 1; + __mmask8 lastCtrK8 = blocks == 0 ? 0xC0 : (__mmask8)((Ipp8u)0x03 << ((blocks - 1) << 1)); + __mmask64 lastCtrK64 = blocks == 0 ? 0xFFFF000000000000 : (__mmask64)((Ipp64u)0xFFFF << ((blocks - 1) << 4)); + + ctr512 = adcLo_epi64(M512(incLoByOneMask), ctr512); + + ctr512 = _mm512_maskz_shuffle_epi8(lastCtrK64, ctr512, M512(swapBytes)); + ctr512 = _mm512_maskz_and_epi64(lastCtrK8, ctr512, ctrBitMask512); + ctr512 = _mm512_maskz_or_epi64(lastCtrK8, ctr512, templateCtr512); + + _mm512_mask_compressstoreu_epi64(pCtrValue, lastCtrK8, ctr512); +} + +#endif /* #if (_IPP32E>=_IPP32E_K1) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ctrencrypt_stream_vaes512.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ctrencrypt_stream_vaes512.c new file mode 100755 index 0000000..d7c35cb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ctrencrypt_stream_vaes512.c @@ -0,0 +1,238 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption/decryption (CTR mode) +// +// Contents: +// EncryptStreamCTR32_VAES_NI +// +*/ + +#include "owncp.h" +#include "pcpaesm.h" +#include "pcpaes_encrypt_vaes512.h" + +#if (_IPP32E>=_IPP32E_K1) + +/* Mask to convert low 32-bit parts of four 128-bit Big-Endian numbers + * stored in 512-bit register. The low 32-bit of each number converted + * from LE to BE */ +static __ALIGN32 Ipp8u swapBytes[] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11, 15,14,13,12, + 16,17,18,19, 20,21,22,23, 24,25,26,27, 31,30,29,28, + 32,33,34,35, 36,37,38,39, 40,41,42,43, 47,46,45,44, + 48,49,50,51, 52,53,54,55, 56,57,58,59, 63,62,61,60 +}; + +//////////////////////////////////////////////////////////////////////////////// + +IPP_OWN_DEFN (void, EncryptStreamCTR32_VAES_NI, (const Ipp8u* pSrc, + Ipp8u* pDst, + int nr, + const Ipp8u* pRKey, + int length, /* message length in bytes */ + Ipp8u* pIV)) /* BE counter representation */ +{ + int cipherRounds = nr - 1; + + __m128i* pKeys = (__m128i*)pRKey; + __m512i* pSrc512 = (__m512i*)pSrc; + __m512i* pDst512 = (__m512i*)pDst; + + Ipp32u* pIV32 = (Ipp32u*)pIV; + Ipp64u* pIV64 = (Ipp64u*)pIV; + + // start increment mask for IV + __m512i startIncMask = _mm512_set_epi32(0x3, 0x0, 0x0, 0x0, + 0x2, 0x0, 0x0, 0x0, + 0x1, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0); + + // continuous increment mask for IV + __m512i incMask = _mm512_set_epi32(0x4, 0x0, 0x0, 0x0, + 0x4, 0x0, 0x0, 0x0, + 0x4, 0x0, 0x0, 0x0, + 0x4, 0x0, 0x0, 0x0); + + // initial BE counter with 32-bit low part converted to LE: + // pIV: f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb | fc fd fe ff + // IV: f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb | ff fe fd fc + __m512i IV = _mm512_set4_epi32((Ipp32s)ENDIANNESS32(pIV32[3]), (Ipp32s)pIV32[2], (Ipp32s)pIV32[1], (Ipp32s)pIV32[0]); + + + // Update IV counter for next function calls + int blocks = length / MBS_RIJ128; + + Ipp64u ctr64_h = ENDIANNESS64(pIV64[0]); // high 64-bit of BE IV converted to LE + Ipp64u ctr64_l = ENDIANNESS64(pIV64[1]); // low 32-bit of BE pIV converted to LE + + ctr64_l += (Ipp64u)blocks; + if (ctr64_l < (Ipp64u)blocks) { // overflow of low part + ctr64_h += 1; + } + + // update IV + pIV64[0] = ENDIANNESS64(ctr64_h); + pIV64[1] = ENDIANNESS64(ctr64_l); + + //-------------------------------------- + + for (blocks = length / MBS_RIJ128; blocks >= (4 * 4); blocks -= (4 * 4)) { + // counter0: f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb | ff fe fd fc + // counter1: f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb | 00 ff fd fc + // counter2: f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb | 01 ff fd fc + // counter3: f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb | 02 ff fd fc + __m512i counter0 = _mm512_add_epi32(startIncMask, IV); + __m512i counter1 = _mm512_add_epi32(incMask, counter0); + __m512i counter2 = _mm512_add_epi32(incMask, counter1); + __m512i counter3 = _mm512_add_epi32(incMask, counter2); + + startIncMask = incMask; + IV = counter3; + + // convert last 32-bit (LE->BE): + // counter0: f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb | fc fd fe ff + // counter1: f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb | fc fd ff 00 + // counter2: f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb | fc fd ff 01 + // counter3: f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb | fc fd ff 02 + counter0 = _mm512_shuffle_epi8(counter0, *((__m512i*)swapBytes)); + counter1 = _mm512_shuffle_epi8(counter1, *((__m512i*)swapBytes)); + counter2 = _mm512_shuffle_epi8(counter2, *((__m512i*)swapBytes)); + counter3 = _mm512_shuffle_epi8(counter3, *((__m512i*)swapBytes)); + + cpAESEncrypt4_VAES_NI(&counter0, &counter1, &counter2, &counter3, pKeys, cipherRounds); + + __m512i blk0 = _mm512_loadu_si512(pSrc512); + __m512i blk1 = _mm512_loadu_si512(pSrc512 + 1); + __m512i blk2 = _mm512_loadu_si512(pSrc512 + 2); + __m512i blk3 = _mm512_loadu_si512(pSrc512 + 3); + + blk0 = _mm512_xor_si512(blk0, counter0); + blk1 = _mm512_xor_si512(blk1, counter1); + blk2 = _mm512_xor_si512(blk2, counter2); + blk3 = _mm512_xor_si512(blk3, counter3); + + _mm512_storeu_si512(pDst512, blk0); + _mm512_storeu_si512(pDst512 + 1, blk1); + _mm512_storeu_si512(pDst512 + 2, blk2); + _mm512_storeu_si512(pDst512 + 3, blk3); + + pSrc512 += 4; + pDst512 += 4; + } + + if ((3 * 4) <= blocks) { + __m512i counter0 = _mm512_add_epi32(startIncMask, IV); + __m512i counter1 = _mm512_add_epi32(incMask, counter0); + __m512i counter2 = _mm512_add_epi32(incMask, counter1); + + startIncMask = incMask; + IV = counter2; + + // convert last 32-bit (LE->BE) + counter0 = _mm512_shuffle_epi8(counter0, *((__m512i*)swapBytes)); + counter1 = _mm512_shuffle_epi8(counter1, *((__m512i*)swapBytes)); + counter2 = _mm512_shuffle_epi8(counter2, *((__m512i*)swapBytes)); + + cpAESEncrypt3_VAES_NI(&counter0, &counter1, &counter2, pKeys, cipherRounds); + + __m512i blk0 = _mm512_loadu_si512(pSrc512); + __m512i blk1 = _mm512_loadu_si512(pSrc512 + 1); + __m512i blk2 = _mm512_loadu_si512(pSrc512 + 2); + + blk0 = _mm512_xor_si512(blk0, counter0); + blk1 = _mm512_xor_si512(blk1, counter1); + blk2 = _mm512_xor_si512(blk2, counter2); + + _mm512_storeu_si512(pDst512, blk0); + _mm512_storeu_si512(pDst512 + 1, blk1); + _mm512_storeu_si512(pDst512 + 2, blk2); + + pSrc512 += 3; + pDst512 += 3; + blocks -= (3 * 4); + } + + if ((4 * 2) <= blocks) { + __m512i counter0 = _mm512_add_epi32(startIncMask, IV); + __m512i counter1 = _mm512_add_epi32(incMask, counter0); + + startIncMask = incMask; + IV = counter1; + + // convert last 32-bit (LE->BE) + counter0 = _mm512_shuffle_epi8(counter0, *((__m512i*)swapBytes)); + counter1 = _mm512_shuffle_epi8(counter1, *((__m512i*)swapBytes)); + + cpAESEncrypt2_VAES_NI(&counter0, &counter1, pKeys, cipherRounds); + + __m512i blk0 = _mm512_loadu_si512(pSrc512); + __m512i blk1 = _mm512_loadu_si512(pSrc512 + 1); + + blk0 = _mm512_xor_si512(blk0, counter0); + blk1 = _mm512_xor_si512(blk1, counter1); + + _mm512_storeu_si512(pDst512, blk0); + _mm512_storeu_si512(pDst512 + 1, blk1); + + pSrc512 += 2; + pDst512 += 2; + blocks -= (2 * 4); + } + + for (; blocks >= 4; blocks -= 4) { + __m512i counter0 = _mm512_add_epi32(startIncMask, IV); + + startIncMask = incMask; + IV = counter0; + + // convert last 32-bit (LE->BE) + counter0 = _mm512_shuffle_epi8(counter0, *((__m512i*)swapBytes)); + + cpAESEncrypt1_VAES_NI(&counter0, pKeys, cipherRounds); + + __m512i blk0 = _mm512_loadu_si512(pSrc512); + blk0 = _mm512_xor_si512(blk0, counter0); + _mm512_storeu_si512(pDst512, blk0); + + pSrc512 += 1; + pDst512 += 1; + } + + if (blocks) { + __mmask8 k8 = (__mmask8)((1 << (blocks + blocks)) - 1); // 64-bit chunks + __mmask16 k16 = (__mmask16)((1 << (blocks << 2)) - 1); // 32-bit chunks + __mmask64 k64 = (__mmask64)((1LL << (blocks << 4)) - 1); // 8-bit chunks + + __m512i counter0 = _mm512_maskz_add_epi32(k16, startIncMask, IV); + + // swap last 32-bit (LE->BE) + counter0 = _mm512_maskz_shuffle_epi8(k64, counter0, *((__m512i*)swapBytes)); + + cpAESEncrypt1_VAES_NI(&counter0, pKeys, cipherRounds); + + __m512i blk0 = _mm512_maskz_loadu_epi64(k8, pSrc512); + blk0 = _mm512_maskz_xor_epi64(k8, blk0, counter0); + _mm512_mask_storeu_epi64(pDst512, k8, blk0); + } +} + +#endif /* #if (_IPP32E>=_IPP32E_K1) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ctrencryptr.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ctrencryptr.c new file mode 100644 index 0000000..149e001 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ctrencryptr.c @@ -0,0 +1,84 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption/decryption (CTR mode) +// +// Contents: +// ippsAESEncryptCTR() +// +*/ +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(disable: 4206) // empty unit +#endif + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" +#include "pcpaes_ctr_process.h" + + +/* +// Name: ippsAESEncryptCTR +// +// Purpose: +// AES-CFB encryption. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// pSrc == NULL +// pDst == NULL +// pCtrValue ==NULL +// ippStsContextMatchErr !VALID_AES_ID() +// ippStsLengthErr len <1 +// ippStsCTRSizeErr 128 < ctrNumBitSize < 1 +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// len input buffer length (in bytes) +// pCtx pointer to rge AES context +// pCtrValue pointer to the counter block +// ctrNumBitSize counter block size (bits) +// +// Note: +// counter will updated on return +// +*/ + +IPPFUN(IppStatus, ippsAESEncryptCTR,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx, + Ipp8u* pCtrValue, int ctrNumBitSize)) +{ + /* test context */ + IPP_BAD_PTR1_RET(pCtx); + + #if(_IPP32E>=_IPP32E_Y8) + if(AES_NI_ENABLED==RIJ_AESNI(pCtx)) + return ctrNumBitSize==128? cpProcessAES_ctr128(pSrc, pDst, len, pCtx, pCtrValue) : + cpProcessAES_ctr(pSrc, pDst, len, pCtx, pCtrValue, ctrNumBitSize); + else + return cpProcessAES_ctr(pSrc, pDst, len, pCtx, pCtrValue, ctrNumBitSize); + #else + return cpProcessAES_ctr(pSrc, pDst, len, pCtx, pCtrValue, ctrNumBitSize); + #endif +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_deckeyexpansion_ni.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_deckeyexpansion_ni.c new file mode 100644 index 0000000..ef6f9c3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_deckeyexpansion_ni.c @@ -0,0 +1,52 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. AES keys expansion +// +// Contents: +// aes_DecKeyExpansion_NI() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcpaes_keys_ni.h" + +#if (_AES_NI_ENABLING_==_FEATURE_ON_) || (_AES_NI_ENABLING_==_FEATURE_TICKTOCK_) + +////////////////////////////////////////////////////////////////////// +/* +// AES decryption key schelule +*/ +IPP_OWN_DEFN (void, aes_DecKeyExpansion_NI, (Ipp8u* decKeys, const Ipp8u* encKeys, int nr)) +{ + __m128i* encKeys16 = (__m128i*)encKeys; + __m128i* decKeys16 = (__m128i*)decKeys; + + decKeys16[nr] = encKeys16[nr]; + for(nr-=1; nr > 0; nr--) { + decKeys16[nr] = _mm_aesimc_si128(encKeys16[nr]); + } + decKeys16[0] = encKeys16[0]; +} + +#endif /* #if (_AES_NI_ENABLING_==_FEATURE_ON_) || (_AES_NI_ENABLING_==_FEATURE_TICKTOCK_) */ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_decrypt_vaes512.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_decrypt_vaes512.h new file mode 100644 index 0000000..0811d69 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_decrypt_vaes512.h @@ -0,0 +1,192 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES decryption (VAES-512 kernels) +// +// Contents: +// cpAESDecrypt1_VAES_NI +// cpAESDecrypt2_VAES_NI +// cpAESDecrypt3_VAES_NI +// cpAESDecrypt4_VAES_NI +// +*/ + +#include "owncp.h" +#include "pcpaesm.h" + +#if(_IPP32E>=_IPP32E_K1) + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(disable: 4305) // zmmintrin.h bug: conversion from int to _mmask8 +#endif + +#if !defined(_PCP_AES_DECRYPT_VAES512_H_) +#define _PCP_AES_DECRYPT_VAES512_H_ +//////////////////////////////////////////////////////////////////////////////// + +static void cpAESDecrypt1_VAES_NI(__m512i* blk0, + const __m128i* pRkey, + int cipherRounds) +{ + int nr; + + __m512i rKey0 = _mm512_broadcast_i64x2(pRkey[0]); + __m512i rKey1 = _mm512_broadcast_i64x2(pRkey[-1]); + + __m512i b0 = _mm512_xor_si512(*blk0, rKey0); + rKey0 = _mm512_broadcast_i64x2(pRkey[-2]); + + for (nr = 1, pRkey--; nr < cipherRounds; nr += 2, pRkey -= 2) { + b0 = _mm512_aesdec_epi128(b0, rKey1); + rKey1 = _mm512_broadcast_i64x2(pRkey[-2]); + b0 = _mm512_aesdec_epi128(b0, rKey0); + rKey0 = _mm512_broadcast_i64x2(pRkey[-3]); + } + + b0 = _mm512_aesdec_epi128(b0, rKey1); + *blk0 = _mm512_aesdeclast_epi128(b0, rKey0); + + rKey0 = _mm512_setzero_si512(); + rKey1 = _mm512_setzero_si512(); +} + +static void cpAESDecrypt2_VAES_NI(__m512i* blk0, + __m512i* blk1, + const __m128i* pRkey, + int cipherRounds) +{ + int nr; + + __m512i rKey0 = _mm512_broadcast_i64x2(pRkey[0]); + __m512i rKey1 = _mm512_broadcast_i64x2(pRkey[-1]); + + __m512i b0 = _mm512_xor_si512(*blk0, rKey0); + __m512i b1 = _mm512_xor_si512(*blk1, rKey0); + rKey0 = _mm512_broadcast_i64x2(pRkey[-2]); + + for (nr = 1, pRkey--; nr < cipherRounds; nr += 2, pRkey -= 2) { + b0 = _mm512_aesdec_epi128(b0, rKey1); + b1 = _mm512_aesdec_epi128(b1, rKey1); + rKey1 = _mm512_broadcast_i64x2(pRkey[-2]); + + b0 = _mm512_aesdec_epi128(b0, rKey0); + b1 = _mm512_aesdec_epi128(b1, rKey0); + rKey0 = _mm512_broadcast_i64x2(pRkey[-3]); + } + b0 = _mm512_aesdec_epi128(b0, rKey1); + b1 = _mm512_aesdec_epi128(b1, rKey1); + + *blk0 = _mm512_aesdeclast_epi128(b0, rKey0); + *blk1 = _mm512_aesdeclast_epi128(b1, rKey0); + + rKey0 = _mm512_setzero_si512(); + rKey1 = _mm512_setzero_si512(); +} + +static void cpAESDecrypt3_VAES_NI(__m512i* blk0, + __m512i* blk1, + __m512i* blk2, + const __m128i* pRkey, + int cipherRounds) +{ + int nr; + + __m512i rKey0 = _mm512_broadcast_i64x2(pRkey[0]); + __m512i rKey1 = _mm512_broadcast_i64x2(pRkey[-1]); + + __m512i b0 = _mm512_xor_si512(*blk0, rKey0); + __m512i b1 = _mm512_xor_si512(*blk1, rKey0); + __m512i b2 = _mm512_xor_si512(*blk2, rKey0); + rKey0 = _mm512_broadcast_i64x2(pRkey[-2]); + + for (nr = 1, pRkey--; nr < cipherRounds; nr += 2, pRkey -= 2) { + b0 = _mm512_aesdec_epi128(b0, rKey1); + b1 = _mm512_aesdec_epi128(b1, rKey1); + b2 = _mm512_aesdec_epi128(b2, rKey1); + rKey1 = _mm512_broadcast_i64x2(pRkey[-2]); + + b0 = _mm512_aesdec_epi128(b0, rKey0); + b1 = _mm512_aesdec_epi128(b1, rKey0); + b2 = _mm512_aesdec_epi128(b2, rKey0); + rKey0 = _mm512_broadcast_i64x2(pRkey[-3]); + } + b0 = _mm512_aesdec_epi128(b0, rKey1); + b1 = _mm512_aesdec_epi128(b1, rKey1); + b2 = _mm512_aesdec_epi128(b2, rKey1); + + *blk0 = _mm512_aesdeclast_epi128(b0, rKey0); + *blk1 = _mm512_aesdeclast_epi128(b1, rKey0); + *blk2 = _mm512_aesdeclast_epi128(b2, rKey0); + + rKey0 = _mm512_setzero_si512(); + rKey1 = _mm512_setzero_si512(); +} + +static void cpAESDecrypt4_VAES_NI(__m512i* blk0, + __m512i* blk1, + __m512i* blk2, + __m512i* blk3, + const __m128i* pRkey, + int cipherRounds) +{ + int nr; + + __m512i rKey0 = _mm512_broadcast_i64x2(pRkey[0]); + __m512i rKey1 = _mm512_broadcast_i64x2(pRkey[-1]); + + __m512i b0 = _mm512_xor_si512(*blk0, rKey0); + __m512i b1 = _mm512_xor_si512(*blk1, rKey0); + __m512i b2 = _mm512_xor_si512(*blk2, rKey0); + __m512i b3 = _mm512_xor_si512(*blk3, rKey0); + + rKey0 = _mm512_broadcast_i64x2(pRkey[-2]); + + for (nr = 1, pRkey--; nr < cipherRounds; nr += 2, pRkey -= 2) { + b0 = _mm512_aesdec_epi128(b0, rKey1); + b1 = _mm512_aesdec_epi128(b1, rKey1); + b2 = _mm512_aesdec_epi128(b2, rKey1); + b3 = _mm512_aesdec_epi128(b3, rKey1); + rKey1 = _mm512_broadcast_i64x2(pRkey[-2]); + + b0 = _mm512_aesdec_epi128(b0, rKey0); + b1 = _mm512_aesdec_epi128(b1, rKey0); + b2 = _mm512_aesdec_epi128(b2, rKey0); + b3 = _mm512_aesdec_epi128(b3, rKey0); + rKey0 = _mm512_broadcast_i64x2(pRkey[-3]); + } + + b0 = _mm512_aesdec_epi128(b0, rKey1); + b1 = _mm512_aesdec_epi128(b1, rKey1); + b2 = _mm512_aesdec_epi128(b2, rKey1); + b3 = _mm512_aesdec_epi128(b3, rKey1); + + *blk0 = _mm512_aesdeclast_epi128(b0, rKey0); + *blk1 = _mm512_aesdeclast_epi128(b1, rKey0); + *blk2 = _mm512_aesdeclast_epi128(b2, rKey0); + *blk3 = _mm512_aesdeclast_epi128(b3, rKey0); + + rKey0 = _mm512_setzero_si512(); + rKey1 = _mm512_setzero_si512(); +} + +#endif /* _PCP_AES_DECRYPT_VAES512_H_ */ + +#endif /* _IPP32E>=_IPP32E_K1 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ecb_vaes512.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ecb_vaes512.c new file mode 100644 index 0000000..b45f5fd --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ecb_vaes512.c @@ -0,0 +1,199 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption/decryption (ECB mode) +// +// Contents: +// EncryptECB_RIJ128pipe_VAES_NI +// DecryptECB_RIJ128pipe_VAES_NI +// +// +*/ + +#include "owncp.h" +#include "pcpaesm.h" +#include "pcpaes_encrypt_vaes512.h" +#include "pcpaes_decrypt_vaes512.h" + +#if (_IPP32E>=_IPP32E_K1) + +IPP_OWN_DEFN (void, EncryptECB_RIJ128pipe_VAES_NI, (const Ipp8u* pSrc, // pointer to the plaintext + Ipp8u* pDst, // pointer to the ciphertext buffer + int len, // text length in bytes + const IppsAESSpec* pCtx)) // pointer to the context +{ + int cipherRounds = RIJ_NR(pCtx) - 1; + + __m128i* pRkey = (__m128i*)RIJ_EKEYS(pCtx); + __m512i* pInp512 = (__m512i*)pSrc; + __m512i* pOut512 = (__m512i*)pDst; + + int blocks; + for (blocks = len / MBS_RIJ128; blocks >= (4 * 4); blocks -= (4 * 4)) { + __m512i blk0 = _mm512_loadu_si512(pInp512); + __m512i blk1 = _mm512_loadu_si512(pInp512 + 1); + __m512i blk2 = _mm512_loadu_si512(pInp512 + 2); + __m512i blk3 = _mm512_loadu_si512(pInp512 + 3); + + cpAESEncrypt4_VAES_NI(&blk0, &blk1, &blk2, &blk3, pRkey, cipherRounds); + + _mm512_storeu_si512(pOut512, blk0); + _mm512_storeu_si512(pOut512 + 1, blk1); + _mm512_storeu_si512(pOut512 + 2, blk2); + _mm512_storeu_si512(pOut512 + 3, blk3); + + pInp512 += 4; + pOut512 += 4; + } + + if ((3 * 4) <= blocks) { + __m512i blk0 = _mm512_loadu_si512(pInp512); + __m512i blk1 = _mm512_loadu_si512(pInp512 + 1); + __m512i blk2 = _mm512_loadu_si512(pInp512 + 2); + + cpAESEncrypt3_VAES_NI(&blk0, &blk1, &blk2, pRkey, cipherRounds); + + _mm512_storeu_si512(pOut512, blk0); + _mm512_storeu_si512(pOut512 + 1, blk1); + _mm512_storeu_si512(pOut512 + 2, blk2); + + pInp512 += 3; + pOut512 += 3; + blocks -= (3 * 4); + } + + if ((4 * 2) <= blocks) { + __m512i blk0 = _mm512_loadu_si512(pInp512); + __m512i blk1 = _mm512_loadu_si512(pInp512 + 1); + + cpAESEncrypt2_VAES_NI(&blk0, &blk1, pRkey, cipherRounds); + + _mm512_storeu_si512(pOut512, blk0); + _mm512_storeu_si512(pOut512 + 1, blk1); + + pInp512 += 2; + pOut512 += 2; + blocks -= (2 * 4); + } + + for (; blocks >= 4; blocks -= 4) { + __m512i blk0 = _mm512_loadu_si512(pInp512); + + cpAESEncrypt1_VAES_NI(&blk0, pRkey, cipherRounds); + + _mm512_storeu_si512(pOut512, blk0); + + pInp512 += 1; + pOut512 += 1; + } + + if (blocks) { + __mmask8 k = (__mmask8)((1 << (blocks + blocks)) - 1); + __m512i blk0 = _mm512_maskz_loadu_epi64(k, pInp512); + + cpAESEncrypt1_VAES_NI(&blk0, pRkey, cipherRounds); + + _mm512_mask_storeu_epi64(pOut512, k, blk0); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +IPP_OWN_DEFN (void, DecryptECB_RIJ128pipe_VAES_NI, (const Ipp8u* pSrc, // pointer to the plaintext + Ipp8u* pDst, // pointer to the ciphertext buffer + int len, // text length in bytes + const IppsAESSpec* pCtx)) // pointer to the context +{ + int cipherRounds = RIJ_NR(pCtx) - 1; + + __m128i* pRkey = (__m128i*)RIJ_DKEYS(pCtx) + cipherRounds + 1; + __m512i* pInp512 = (__m512i*)pSrc; + __m512i* pOut512 = (__m512i*)pDst; + + int blocks; + for (blocks = len / MBS_RIJ128; blocks >= (4 * 4); blocks -= (4 * 4)) { + __m512i blk0 = _mm512_loadu_si512(pInp512); + __m512i blk1 = _mm512_loadu_si512(pInp512 + 1); + __m512i blk2 = _mm512_loadu_si512(pInp512 + 2); + __m512i blk3 = _mm512_loadu_si512(pInp512 + 3); + + cpAESDecrypt4_VAES_NI(&blk0, &blk1, &blk2, &blk3, pRkey, cipherRounds); + + _mm512_storeu_si512(pOut512, blk0); + _mm512_storeu_si512(pOut512 + 1, blk1); + _mm512_storeu_si512(pOut512 + 2, blk2); + _mm512_storeu_si512(pOut512 + 3, blk3); + + pInp512 += 4; + pOut512 += 4; + } + + if ((3 * 4) <= blocks) { + __m512i blk0 = _mm512_loadu_si512(pInp512); + __m512i blk1 = _mm512_loadu_si512(pInp512 + 1); + __m512i blk2 = _mm512_loadu_si512(pInp512 + 2); + + cpAESDecrypt3_VAES_NI(&blk0, &blk1, &blk2, pRkey, cipherRounds); + + _mm512_storeu_si512(pOut512, blk0); + _mm512_storeu_si512(pOut512 + 1, blk1); + _mm512_storeu_si512(pOut512 + 2, blk2); + + pInp512 += 3; + pOut512 += 3; + blocks -= (3 * 4); + } + + if ((4 * 2) <= blocks) { + __m512i blk0 = _mm512_loadu_si512(pInp512); + __m512i blk1 = _mm512_loadu_si512(pInp512 + 1); + + cpAESDecrypt2_VAES_NI(&blk0, &blk1, pRkey, cipherRounds); + + _mm512_storeu_si512(pOut512, blk0); + _mm512_storeu_si512(pOut512 + 1, blk1); + + pInp512 += 2; + pOut512 += 2; + blocks -= (2 * 4); + } + + for (; blocks >= 4; blocks -= 4) { + __m512i blk0 = _mm512_loadu_si512(pInp512); + + cpAESDecrypt1_VAES_NI(&blk0, pRkey, cipherRounds); + + _mm512_storeu_si512(pOut512, blk0); + pInp512 += 1; + pOut512 += 1; + } + + if (blocks) { + __mmask8 k = (__mmask8)((1 << (blocks + blocks)) - 1); + __m512i blk0 = _mm512_maskz_loadu_epi64(k, pInp512); + + cpAESDecrypt1_VAES_NI(&blk0, pRkey, cipherRounds); + + _mm512_mask_storeu_epi64(pOut512, k, blk0); + } +} + +#endif /* _IPP32E>=_IPP32E_K1 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ecbdecrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ecbdecrypt.c new file mode 100644 index 0000000..44ccfdf --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ecbdecrypt.c @@ -0,0 +1,157 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption/decryption (ECB mode) +// +// Contents: +// ippsAESDecryptECB() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + +/* +// AES-ECB denryption +// +// Parameters: +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// nBlocks number of decrypted data blocks +// pCtx pointer to the AES context +*/ +static +void cpDecryptAES_ecb(const Ipp8u* pSrc, Ipp8u* pDst, int nBlocks, const IppsAESSpec* pCtx) +{ +#if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + /* use pipelined version is possible */ + if(AES_NI_ENABLED==RIJ_AESNI(pCtx)) { + DecryptECB_RIJ128pipe_AES_NI(pSrc, pDst, RIJ_NR(pCtx), RIJ_DKEYS(pCtx), nBlocks*MBS_RIJ128); + } + else +#endif + { + /* block-by-block decryption */ + RijnCipher decoder = RIJ_DECODER(pCtx); + + while(nBlocks) { + //decoder((const Ipp32u*)pSrc, (Ipp32u*)pDst, RIJ_NR(pCtx), RIJ_DKEYS(pCtx), (const Ipp32u (*)[256])RIJ_DEC_SBOX(pCtx)); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + decoder(pSrc, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), RijDecSbox/*NULL*/); + #else + decoder(pSrc, pDst, RIJ_NR(pCtx), RIJ_DKEYS(pCtx), NULL); + #endif + + pSrc += MBS_RIJ128; + pDst += MBS_RIJ128; + nBlocks--; + } + } +} + +static void cpDecryptAES_ecb_dispatch(const Ipp8u *pSrc, Ipp8u *pDst, int len, const IppsAESSpec *pCtx) +{ +#if (_IPP32E >= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512VAES)) + DecryptECB_RIJ128pipe_VAES_NI(pSrc, pDst, len, pCtx); + else +#endif + { + int nBlocks = len / MBS_RIJ128; + cpDecryptAES_ecb(pSrc, pDst, nBlocks, pCtx); + } +} + +/*F* +// Name: ippsAESDecryptECB +// +// Purpose: AES-ECB decryption. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// pSrc == NULL +// pDst == NULL +// ippStsContextMatchErr !VALID_AES_ID() +// ippStsLengthErr dataLen <1 +// ippStsUnderRunErr 0!=(dataLen%MBS_RIJ128) +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// len input/output buffer length (in bytes) +// pCtx pointer to the AES context +// +*F*/ +IPPFUN(IppStatus, ippsAESDecryptECB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx)) +{ + /* test context */ + IPP_BAD_PTR1_RET(pCtx); + /* test the context ID */ + IPP_BADARG_RET(!VALID_AES_ID(pCtx), ippStsContextMatchErr); + + /* test source and target buffer pointers */ + IPP_BAD_PTR2_RET(pSrc, pDst); + /* test stream length */ + IPP_BADARG_RET((len<1), ippStsLengthErr); + /* test stream integrity */ + IPP_BADARG_RET((len&(MBS_RIJ128-1)), ippStsUnderRunErr); + + /* do encryption */ +#if (_AES_PROB_NOISE == _FEATURE_ON_) + cpAESNoiseParams *params = (cpAESNoiseParams*)&RIJ_NOISE_PARAMS(pCtx); + /* Mistletoe3 mitigation */ + if (AES_NOISE_LEVEL(params) > 0) { + /* Number of bytes allowed for operation without adding noise */ + int chunk_size; + /* Number of bytes remaining for operation */ + int remaining_size = len; + + while (remaining_size > 0) { + /* How many bytes to encrypt in this operation */ + chunk_size = (remaining_size >= MISTLETOE3_MAX_CHUNK_SIZE) ? MISTLETOE3_MAX_CHUNK_SIZE : remaining_size; + + cpDecryptAES_ecb_dispatch(pSrc, pDst, chunk_size, pCtx); + + cpAESRandomNoise(NULL, + MISTLETOE3_BASE_NOISE_LEVEL + AES_NOISE_LEVEL(params), + MISTLETOE3_NOISE_RATE, + &AES_NOISE_RAND(params)); + + pSrc += chunk_size; + pDst += chunk_size; + remaining_size -= chunk_size; + } + } else +#endif + { + cpDecryptAES_ecb_dispatch(pSrc, pDst, len, pCtx); + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ecbencrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ecbencrypt.c new file mode 100644 index 0000000..640a390 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ecbencrypt.c @@ -0,0 +1,159 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption/decryption (ECB mode) +// +// Contents: +// ippsAESEncryptECB() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" +#include "pcprij.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + + +/* +// AES-ECB ecnryption +// +// Parameters: +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// nBlocks number of ecnrypted data blocks +// pCtx pointer to the AES context +*/ +static +void cpEncryptAES_ecb(const Ipp8u* pSrc, Ipp8u* pDst, int nBlocks, const IppsAESSpec* pCtx) +{ +#if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + /* use pipelined version is possible */ + if(AES_NI_ENABLED==RIJ_AESNI(pCtx)) { + EncryptECB_RIJ128pipe_AES_NI(pSrc, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), nBlocks*MBS_RIJ128); + } + else +#endif + { + /* block-by-block encryption */ + RijnCipher encoder = RIJ_ENCODER(pCtx); + + while(nBlocks) { + //encoder((const Ipp32u*)pSrc, (Ipp32u*)pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), (const Ipp32u (*)[256])RIJ_ENC_SBOX(pCtx)); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder(pSrc, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), RijEncSbox/*NULL*/); + #else + encoder(pSrc, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), NULL); + #endif + + pSrc += MBS_RIJ128; + pDst += MBS_RIJ128; + nBlocks--; + } + } +} + +static void cpEncryptAES_ecb_dispatch(const Ipp8u *pSrc, Ipp8u *pDst, int len, const IppsAESSpec *pCtx) +{ +#if (_IPP32E >= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512VAES)) + EncryptECB_RIJ128pipe_VAES_NI(pSrc, pDst, len, pCtx); + else +#endif + { + int nBlocks = len / MBS_RIJ128; + cpEncryptAES_ecb(pSrc, pDst, nBlocks, pCtx); + } +} + +/*F* +// Name: ippsAESEncryptECB +// +// Purpose: AES-ECB encryption. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// pSrc == NULL +// pDst == NULL +// ippStsContextMatchErr !VALID_AES_ID() +// ippStsLengthErr dataLen <1 +// ippStsUnderRunErr 0!=(dataLen%MBS_RIJ128) +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// len input/output buffer length (in bytes) +// pCtx pointer to the AES context +// +*F*/ +IPPFUN(IppStatus, ippsAESEncryptECB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsAESSpec* pCtx)) +{ + /* test context */ + IPP_BAD_PTR1_RET(pCtx); + /* test the context ID */ + IPP_BADARG_RET(!VALID_AES_ID(pCtx), ippStsContextMatchErr); + + /* test source and target buffer pointers */ + IPP_BAD_PTR2_RET(pSrc, pDst); + /* test stream length */ + IPP_BADARG_RET((len<1), ippStsLengthErr); + /* test stream integrity */ + IPP_BADARG_RET((len&(MBS_RIJ128-1)), ippStsUnderRunErr); + + /* do encryption */ +#if (_AES_PROB_NOISE == _FEATURE_ON_) + /* Mistletoe3 mitigation */ + cpAESNoiseParams *params = (cpAESNoiseParams*)&RIJ_NOISE_PARAMS(pCtx); + if (AES_NOISE_LEVEL(params) > 0) { + /* Number of bytes allowed for operation without adding noise */ + int chunk_size; + /* Number of bytes remaining for operation */ + int remaining_size = len; + + while (remaining_size > 0) { + /* How many bytes to encrypt in this operation */ + chunk_size = (remaining_size >= MISTLETOE3_MAX_CHUNK_SIZE) ? MISTLETOE3_MAX_CHUNK_SIZE : remaining_size; + + cpEncryptAES_ecb_dispatch(pSrc, pDst, chunk_size, pCtx); + + cpAESRandomNoise(NULL, + MISTLETOE3_BASE_NOISE_LEVEL + AES_NOISE_LEVEL(params), + MISTLETOE3_NOISE_RATE, + &AES_NOISE_RAND(params)); + + pSrc += chunk_size; + pDst += chunk_size; + remaining_size -= chunk_size; + } + } else +#endif + { + cpEncryptAES_ecb_dispatch(pSrc, pDst, len, pCtx); + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_encrypt_vaes512.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_encrypt_vaes512.h new file mode 100644 index 0000000..88a2671 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_encrypt_vaes512.h @@ -0,0 +1,193 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption (VAES-512 kernels) +// +// Contents: +// cpAESEncrypt1_VAES_NI +// cpAESEncrypt2_VAES_NI +// cpAESEncrypt3_VAES_NI +// cpAESEncrypt4_VAES_NI +// +*/ + +#include "owncp.h" +#include "pcpaesm.h" + +#if(_IPP32E>=_IPP32E_K1) + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(disable: 4305) // zmmintrin.h bug: conversion from int to _mmask8 +#endif + +#if !defined(_PCP_AES_ENCRYPT_VAES512_H_) +#define _PCP_AES_ENCRYPT_VAES512_H_ + +//////////////////////////////////////////////////////////////////////////////// + +static void cpAESEncrypt1_VAES_NI(__m512i* blk0, + const __m128i* pRkey, + int cipherRounds) +{ + int nr; + + __m512i rKey0 = _mm512_broadcast_i64x2(pRkey[0]); + __m512i rKey1 = _mm512_broadcast_i64x2(pRkey[1]); + + __m512i b0 = _mm512_xor_si512(*blk0, rKey0); + rKey0 = _mm512_broadcast_i64x2(pRkey[2]); + + for (nr = 1, pRkey++; nr < cipherRounds; nr += 2, pRkey += 2) { + b0 = _mm512_aesenc_epi128(b0, rKey1); + rKey1 = _mm512_broadcast_i64x2(pRkey[2]); + b0 = _mm512_aesenc_epi128(b0, rKey0); + rKey0 = _mm512_broadcast_i64x2(pRkey[3]); + } + + b0 = _mm512_aesenc_epi128(b0, rKey1); + *blk0 = _mm512_aesenclast_epi128(b0, rKey0); + + rKey0 = _mm512_setzero_si512(); + rKey1 = _mm512_setzero_si512(); +} + +static void cpAESEncrypt2_VAES_NI(__m512i* blk0, + __m512i* blk1, + const __m128i* pRkey, + int cipherRounds) +{ + int nr; + + __m512i rKey0 = _mm512_broadcast_i64x2(pRkey[0]); + __m512i rKey1 = _mm512_broadcast_i64x2(pRkey[1]); + + __m512i b0 = _mm512_xor_si512(*blk0, rKey0); + __m512i b1 = _mm512_xor_si512(*blk1, rKey0); + rKey0 = _mm512_broadcast_i64x2(pRkey[2]); + + for (nr = 1, pRkey++; nr < cipherRounds; nr += 2, pRkey += 2) { + b0 = _mm512_aesenc_epi128(b0, rKey1); + b1 = _mm512_aesenc_epi128(b1, rKey1); + rKey1 = _mm512_broadcast_i64x2(pRkey[2]); + + b0 = _mm512_aesenc_epi128(b0, rKey0); + b1 = _mm512_aesenc_epi128(b1, rKey0); + rKey0 = _mm512_broadcast_i64x2(pRkey[3]); + } + + b0 = _mm512_aesenc_epi128(b0, rKey1); + b1 = _mm512_aesenc_epi128(b1, rKey1); + + *blk0 = _mm512_aesenclast_epi128(b0, rKey0); + *blk1 = _mm512_aesenclast_epi128(b1, rKey0); + + rKey0 = _mm512_setzero_si512(); + rKey1 = _mm512_setzero_si512(); +} + +static void cpAESEncrypt3_VAES_NI(__m512i* blk0, + __m512i* blk1, + __m512i* blk2, + const __m128i* pRkey, + int cipherRounds) +{ + int nr; + + __m512i rKey0 = _mm512_broadcast_i64x2(pRkey[0]); + __m512i rKey1 = _mm512_broadcast_i64x2(pRkey[1]); + + __m512i b0 = _mm512_xor_si512(*blk0, rKey0); + __m512i b1 = _mm512_xor_si512(*blk1, rKey0); + __m512i b2 = _mm512_xor_si512(*blk2, rKey0); + rKey0 = _mm512_broadcast_i64x2(pRkey[2]); + + for (nr = 1, pRkey++; nr < cipherRounds; nr += 2, pRkey += 2) { + b0 = _mm512_aesenc_epi128(b0, rKey1); + b1 = _mm512_aesenc_epi128(b1, rKey1); + b2 = _mm512_aesenc_epi128(b2, rKey1); + rKey1 = _mm512_broadcast_i64x2(pRkey[2]); + + b0 = _mm512_aesenc_epi128(b0, rKey0); + b1 = _mm512_aesenc_epi128(b1, rKey0); + b2 = _mm512_aesenc_epi128(b2, rKey0); + rKey0 = _mm512_broadcast_i64x2(pRkey[3]); + } + + b0 = _mm512_aesenc_epi128(b0, rKey1); + b1 = _mm512_aesenc_epi128(b1, rKey1); + b2 = _mm512_aesenc_epi128(b2, rKey1); + + *blk0 = _mm512_aesenclast_epi128(b0, rKey0); + *blk1 = _mm512_aesenclast_epi128(b1, rKey0); + *blk2 = _mm512_aesenclast_epi128(b2, rKey0); + + rKey0 = _mm512_setzero_si512(); + rKey1 = _mm512_setzero_si512(); +} + +static void cpAESEncrypt4_VAES_NI(__m512i* blk0, + __m512i* blk1, + __m512i* blk2, + __m512i* blk3, + const __m128i* pRkey, + int cipherRounds) +{ + int nr; + + __m512i rKey0 = _mm512_broadcast_i64x2(pRkey[0]); + __m512i rKey1 = _mm512_broadcast_i64x2(pRkey[1]); + + __m512i b0 = _mm512_xor_si512(*blk0, rKey0); + __m512i b1 = _mm512_xor_si512(*blk1, rKey0); + __m512i b2 = _mm512_xor_si512(*blk2, rKey0); + __m512i b3 = _mm512_xor_si512(*blk3, rKey0); + rKey0 = _mm512_broadcast_i64x2(pRkey[2]); + + for (nr = 1, pRkey++; nr < cipherRounds; nr += 2, pRkey += 2) { + b0 = _mm512_aesenc_epi128(b0, rKey1); + b1 = _mm512_aesenc_epi128(b1, rKey1); + b2 = _mm512_aesenc_epi128(b2, rKey1); + b3 = _mm512_aesenc_epi128(b3, rKey1); + rKey1 = _mm512_broadcast_i64x2(pRkey[2]); + + b0 = _mm512_aesenc_epi128(b0, rKey0); + b1 = _mm512_aesenc_epi128(b1, rKey0); + b2 = _mm512_aesenc_epi128(b2, rKey0); + b3 = _mm512_aesenc_epi128(b3, rKey0); + rKey0 = _mm512_broadcast_i64x2(pRkey[3]); + } + b0 = _mm512_aesenc_epi128(b0, rKey1); + b1 = _mm512_aesenc_epi128(b1, rKey1); + b2 = _mm512_aesenc_epi128(b2, rKey1); + b3 = _mm512_aesenc_epi128(b3, rKey1); + + *blk0 = _mm512_aesenclast_epi128(b0, rKey0); + *blk1 = _mm512_aesenclast_epi128(b1, rKey0); + *blk2 = _mm512_aesenclast_epi128(b2, rKey0); + *blk3 = _mm512_aesenclast_epi128(b3, rKey0); + + rKey0 = _mm512_setzero_si512(); + rKey1 = _mm512_setzero_si512(); +} + +#endif /* _PCP_AES_ENCRYPT_VAES512_H_ */ + +#endif /* _IPP32E>=_IPP32E_K1 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_expandkey_ni.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_expandkey_ni.c new file mode 100644 index 0000000..d6f6c2b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_expandkey_ni.c @@ -0,0 +1,51 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. AES keys expansion +// +// Contents: +// cpExpandAesKey_NI() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcpaes_keys_ni.h" + +#if (_AES_NI_ENABLING_==_FEATURE_ON_) || (_AES_NI_ENABLING_==_FEATURE_TICKTOCK_) + +IPP_OWN_DEFN (void, cpExpandAesKey_NI, (const Ipp8u* pSecret, IppsAESSpec* pCtx)) +{ + int nRounds = RIJ_NR(pCtx); + Ipp8u* pEncKeys = RIJ_EKEYS(pCtx); + Ipp8u* pDecKeys = RIJ_DKEYS(pCtx); + + switch (nRounds) { + case 12: aes192_KeyExpansion_NI(pEncKeys, pSecret); break; + case 14: aes256_KeyExpansion_NI(pEncKeys, pSecret); break; + default: aes128_KeyExpansion_NI(pEncKeys, pSecret); break; + } + + aes_DecKeyExpansion_NI(pDecKeys, pEncKeys, nRounds); +} + +#endif /* #if (_AES_NI_ENABLING_==_FEATURE_ON_) || (_AES_NI_ENABLING_==_FEATURE_TICKTOCK_) */ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcm_vaes512.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcm_vaes512.h new file mode 100644 index 0000000..0d2368b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcm_vaes512.h @@ -0,0 +1,146 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-GCM auxilary +// +// Contents: +// +*/ + +#if 0 // Not used + +#if !defined(_CP_AES_GCM_VAES512_H) +#define _CP_AES_GCM_VAES512_H + +#if (_IPP32E>=_IPP32E_K1) +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(disable: 4310) // cast truncates constant value in MSVC +#endif + +#define M128(mem) (*((__m128i*)((Ipp8u*)(mem)))) +#define M256(mem) (*((__m256i*)((Ipp8u*)(mem)))) +#define M512(mem) (*((__m512i*)((Ipp8u*)(mem)))) + +static const __ALIGN64 Ipp64u POLY2[] = { 0x1, 0xC200000000000000, 0x1, 0xC200000000000000, + 0x1, 0xC200000000000000, 0x1, 0xC200000000000000 }; + +static __ALIGN64 Ipp8u swapBytes[] = { + 15,14,13,12, 11,10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 31,30,29,28, 27,26,25,24, 23,22,21,20, 19,18,17,16, + 47,46,45,44, 43,42,41,40, 39,38,37,36, 35,34,33,32, + 63,62,61,60, 59,58,57,56, 55,54,53,52, 51,50,49,48, +}; + +/* The function performs careless Karatsuba multiplication of 4x128-bit blocks stored + * in 512-bit inputs: + * A = [A3 A2 A1 A0], B = [B3 B2 B1 B0] + * Ai=[a1i:a0i] and Bi=[b1i:b0i], i=0..3 + * and returns parts of the multiplication Hi, Mi and Li, i=0..3, where + * Hi = a1i*b1i + * Mi = (a1i^a0i)*(b1i^b0i) + * Li = a0i*b0i + * + * NB: make sure unused parts of input registers are zeroed to avoid issues with further horizontal XOR. + */ +__INLINE void AesGcmKaratsubaMul4(const __m512i * const pA, /* A3 A2 A1 A0 */ + const __m512i * const pHKeys, /* B3 B2 B1 B0 */ + const __m512i * const pHKeysKaratsuba, /* precomputed (b1i^b0i) */ + __m512i * const pH, + __m512i * const pM, + __m512i * const pL) +{ + *pL = _mm512_clmulepi64_epi128(*pA, *pHKeys, 0x00); // L = [a0i*b0i] + *pH = _mm512_clmulepi64_epi128(*pA, *pHKeys, 0x11); // H = [a1i*b1i] + + *pM = _mm512_shuffle_epi32(*pA, 78); // M = [a0i:a1i] + *pM = _mm512_xor_epi32(*pM, *pA); // M = [a1i^a0i:a1i^a0i] + *pM = _mm512_clmulepi64_epi128(*pM, *pHKeysKaratsuba, 0x00); // M = (a1i^a0i)*(b1i^b0i) +} + +/* The function performs horizontal XOR for 4 128-bit values in 512-bit register + 128-bit result value saved in the low part of the 512-bit register + */ +__INLINE void HXor4x128(const __m512i * const zmm, + __m128i * const xmm) +{ + __m256i ymm; + + ymm = _mm512_extracti64x4_epi64(*zmm, 0x1); // zmm = [3 2 1 0]; ymm = [3 2] + ymm = _mm256_xor_si256(_mm512_castsi512_si256(*zmm), ymm); // ymm = [1^3 0^2] + + *xmm = _mm256_extracti32x4_epi32(ymm, 0x1); // xmm = [1^3] + *xmm = _mm_maskz_xor_epi64(0xFF, _mm256_castsi256_si128(ymm), *xmm); +} + +/* The function performs Montgomery reduction of 256-bit polynomial to 128-bit one + with irreducible polynomial + */ +__INLINE void ReducePoly2x128(const __m128i * const pHI, + const __m128i * const pLO, + __m128i * const result) +{ + __m256i tmp1, tmp2, HI, LO; + + HI = _mm256_set_m128i(_mm_setzero_si128(), *pHI); + LO = _mm256_set_m128i(_mm_setzero_si128(), *pLO); + + tmp1 = _mm256_clmulepi64_epi128(LO, M256(POLY2), 0x10); + tmp2 = _mm256_shuffle_epi32(LO, 78); // 78 = 01001110b + LO = _mm256_xor_si256(tmp1, tmp2); + + tmp1 = _mm256_clmulepi64_epi128(LO, M256(POLY2), 0x10); + tmp2 = _mm256_shuffle_epi32(LO, 78); // 78 = 01001110b + LO = _mm256_xor_si256(tmp1, tmp2); + + tmp1 = _mm256_xor_si256(HI, LO); + *result = _mm256_castsi256_si128(tmp1); +} + +/* The function aggregates partial products of Karatsuba multiplication into final ghash value */ +__INLINE void AggregateKaratsubaPartialProducts(const __m512i * const pH, + const __m512i * const pM, + const __m512i * const pL, + __m128i * const result) +{ + __m512i H, M, L; + __m128i H128, L128; + + /* Aggregation step1 - combine multiplication results H,M,L into Hi and Lo 128-bit parts */ + M = _mm512_xor_si512(*pM, *pH); + M = _mm512_xor_si512(M, *pL); + H = _mm512_bsrli_epi128(M, 8); + H = _mm512_xor_si512(*pH, H); + L = _mm512_bslli_epi128(M, 8); + L = _mm512_xor_si512(*pL, L); + + /* Aggregation step2 - horizontal XOR for H, L*/ + HXor4x128(&H, &H128); + HXor4x128(&L, &L128); + + /* Reduction of 256-bit poly to 128-bit poly using irreducible polynomial */ + ReducePoly2x128(&H128, &L128, result); +} + +#endif /* #if (_IPP32E>=_IPP32E_K1) */ + +#endif /* _CP_AES_GCM_VAES512_H*/ + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmauth_vaes512.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmauth_vaes512.c new file mode 100644 index 0000000..e8b789f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmauth_vaes512.c @@ -0,0 +1,231 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES authentication (GCM mode) +// +// Contents: +// +*/ + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(disable: 4206) // empty translation unit in MSVC +#endif + +#if 0 // Not used + +#include "owncp.h" +#include "pcpaesm.h" +#include "pcpaes_encrypt_vaes512.h" +#include "pcpaes_gcm_vaes512.h" +#include "pcpaesauthgcm.h" + +#if (_IPP32E>=_IPP32E_K1) +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(disable: 4310) // cast truncates constant value in MSVC +#endif + +/* AES-GCM authentication function. It calculates GHASH of the source input */ +IPP_OWN_DEFN (void, AesGcmAuth_vaes, (Ipp8u* pGHash, const Ipp8u* pSrc, int len, const Ipp8u* pHKey, const void* pParam)) +{ + IPP_UNREFERENCED_PARAMETER(pParam); + + __m512i* pSrc512 = (__m512i*)pSrc; + __m512i* pHKey512 = (__m512i*)pHKey; + + /* Load hKeys vectors */ + __m512i hKeys0 = _mm512_loadu_si512(pHKey512); + __m512i hKeys1 = _mm512_loadu_si512(pHKey512 + 1); + __m512i hKeys2 = _mm512_loadu_si512(pHKey512 + 2); + __m512i hKeys3 = _mm512_loadu_si512(pHKey512 + 3); + + /* Load precomputed multipliers for Karatsuba multiplication */ + __m512i hKeysKaratsuba0 = _mm512_loadu_si512(pHKey512 + 4); + __m512i hKeysKaratsuba1 = _mm512_loadu_si512(pHKey512 + 5); + __m512i hKeysKaratsuba2 = _mm512_loadu_si512(pHKey512 + 6); + __m512i hKeysKaratsuba3 = _mm512_loadu_si512(pHKey512 + 7); + + /* Current GHASH value */ + __m128i ghash128; + __m512i ghash512 = _mm512_maskz_loadu_epi64(0x03, pGHash); + ghash512 = _mm512_shuffle_epi8(ghash512, M512(swapBytes)); + + int blocks; + for (blocks = len / MBS_RIJ128; blocks >= (4 * 4); blocks -= (4 * 4)) { + __m512i blk0 = _mm512_loadu_si512(pSrc512); + __m512i blk1 = _mm512_loadu_si512(pSrc512 + 1); + __m512i blk2 = _mm512_loadu_si512(pSrc512 + 2); + __m512i blk3 = _mm512_loadu_si512(pSrc512 + 3); + + /* Reflect inputs */ + blk0 = _mm512_shuffle_epi8(blk0, M512(swapBytes)); + blk1 = _mm512_shuffle_epi8(blk1, M512(swapBytes)); + blk2 = _mm512_shuffle_epi8(blk2, M512(swapBytes)); + blk3 = _mm512_shuffle_epi8(blk3, M512(swapBytes)); + + /* Add current GHASH to src[0] */ + blk0 = _mm512_mask_xor_epi64(blk0, 0x03, blk0, ghash512); + + /* Karatsuba multiplication for 16 blocks with postponed aggregation */ + __m512i H, M, L, tmp1, tmp2, tmp3; + AesGcmKaratsubaMul4(&blk0, &hKeys3, &hKeysKaratsuba3, &H, &M, &L); + AesGcmKaratsubaMul4(&blk1, &hKeys2, &hKeysKaratsuba2, &tmp1, &tmp2, &tmp3); + H = _mm512_xor_si512(H, tmp1); + M = _mm512_xor_si512(M, tmp2); + L = _mm512_xor_si512(L, tmp3); + AesGcmKaratsubaMul4(&blk2, &hKeys1, &hKeysKaratsuba1, &tmp1, &tmp2, &tmp3); + H = _mm512_xor_si512(H, tmp1); + M = _mm512_xor_si512(M, tmp2); + L = _mm512_xor_si512(L, tmp3); + AesGcmKaratsubaMul4(&blk3, &hKeys0, &hKeysKaratsuba0, &tmp1, &tmp2, &tmp3); + H = _mm512_xor_si512(H, tmp1); + M = _mm512_xor_si512(M, tmp2); + L = _mm512_xor_si512(L, tmp3); + + AggregateKaratsubaPartialProducts(&H, &M, &L, &ghash128); + + /* save current ghash value */ + ghash512 = _mm512_castsi128_si512(ghash128); + + pSrc512 += 4; + } + + if ((3 * 4) <= blocks) { + __m512i blk0 = _mm512_loadu_si512(pSrc512); + __m512i blk1 = _mm512_loadu_si512(pSrc512+1); + __m512i blk2 = _mm512_loadu_si512(pSrc512+2); + + /* Reflect inputs */ + blk0 = _mm512_shuffle_epi8(blk0, M512(swapBytes)); + blk1 = _mm512_shuffle_epi8(blk1, M512(swapBytes)); + blk2 = _mm512_shuffle_epi8(blk2, M512(swapBytes)); + + /* Add current GHASH to src[0] */ + blk0 = _mm512_mask_xor_epi64(blk0, 0x03, blk0, ghash512); + + __m512i H, M, L, tmp1, tmp2, tmp3; + AesGcmKaratsubaMul4(&blk0, &hKeys2, &hKeysKaratsuba2, &H, &M, &L); + AesGcmKaratsubaMul4(&blk1, &hKeys1, &hKeysKaratsuba1, &tmp1, &tmp2, &tmp3); + H = _mm512_xor_si512(H, tmp1); + M = _mm512_xor_si512(M, tmp2); + L = _mm512_xor_si512(L, tmp3); + AesGcmKaratsubaMul4(&blk2, &hKeys0, &hKeysKaratsuba0, &tmp1, &tmp2, &tmp3); + H = _mm512_xor_si512(H, tmp1); + M = _mm512_xor_si512(M, tmp2); + L = _mm512_xor_si512(L, tmp3); + + AggregateKaratsubaPartialProducts(&H, &M, &L, &ghash128); + + /* save current ghash value */ + ghash512 = _mm512_castsi128_si512(ghash128); + + pSrc512 += 3; + } + + if ((4 * 2) <= blocks) { + __m512i blk0 = _mm512_loadu_si512(pSrc512); + __m512i blk1 = _mm512_loadu_si512(pSrc512+1); + + /* Reflect inputs */ + blk0 = _mm512_shuffle_epi8(blk0, M512(swapBytes)); + blk1 = _mm512_shuffle_epi8(blk1, M512(swapBytes)); + + /* Add current GHASH to src[0] */ + blk0 = _mm512_mask_xor_epi64(blk0, 0x03, blk0, ghash512); + + __m512i H, M, L, tmp1, tmp2, tmp3; + AesGcmKaratsubaMul4(&blk0, &hKeys1, &hKeysKaratsuba1, &H, &M, &L); + AesGcmKaratsubaMul4(&blk1, &hKeys0, &hKeysKaratsuba0, &tmp1, &tmp2, &tmp3); + H = _mm512_xor_si512(H, tmp1); + M = _mm512_xor_si512(M, tmp2); + L = _mm512_xor_si512(L, tmp3); + + AggregateKaratsubaPartialProducts(&H, &M, &L, &ghash128); + + /* save current ghash value */ + ghash512 = _mm512_castsi128_si512(ghash128); + + pSrc512 += 2; + } + + for (; blocks >= 4; blocks -= 4) { + __m512i blk0 = _mm512_loadu_si512(pSrc512); + blk0 = _mm512_shuffle_epi8(blk0, M512(swapBytes)); + + /* Add current GHASH to src[0] */ + blk0 = _mm512_mask_xor_epi64(blk0, 0x03, blk0, ghash512); + + __m512i H, M, L; + AesGcmKaratsubaMul4(&blk0, &hKeys0, &hKeysKaratsuba0, &H, &M, &L); + + AggregateKaratsubaPartialProducts(&H, &M, &L, &ghash128); + + /* save current ghash value */ + ghash512 = _mm512_castsi128_si512(ghash128); + + pSrc512 += 1; + } + + // at least one block left (max 3 blocks) + if (blocks) { + __mmask8 k8 = (__mmask8)((1 << (blocks + blocks)) - 1); // 64-bit chunks + + __m512i blk0 = _mm512_maskz_loadu_epi64(k8, pSrc512); + blk0 = _mm512_shuffle_epi8(blk0, M512(swapBytes)); + + /* Add current GHASH to src[0] */ + blk0 = _mm512_mask_xor_epi64(blk0, 0x03, blk0, ghash512); + + __m512i H = _mm512_setzero_si512(); + __m512i M = _mm512_setzero_si512(); + __m512i L = _mm512_setzero_si512(); + + // NB: we need immediate parameter in alignr function + switch (blocks) { + case 1: { + hKeys0 = _mm512_alignr_epi64(_mm512_setzero_si512(), hKeys0, 6); + hKeysKaratsuba0 = _mm512_alignr_epi64(_mm512_setzero_si512(), hKeysKaratsuba0, 6); + break; + } + case 2: { + hKeys0 = _mm512_alignr_epi64(_mm512_setzero_si512(), hKeys0, 4); + hKeysKaratsuba0 = _mm512_alignr_epi64(_mm512_setzero_si512(), hKeysKaratsuba0, 4); + break; + } + default: { + hKeys0 = _mm512_alignr_epi64(_mm512_setzero_si512(), hKeys0, 2); + hKeysKaratsuba0 = _mm512_alignr_epi64(_mm512_setzero_si512(), hKeysKaratsuba0, 2); + } + } + AesGcmKaratsubaMul4(&blk0, &hKeys0, &hKeysKaratsuba0, &H, &M, &L); + + AggregateKaratsubaPartialProducts(&H, &M, &L, &ghash128); + + /* save current ghash value */ + ghash512 = _mm512_castsi128_si512(ghash128); + } + + ghash512 = _mm512_shuffle_epi8(ghash512, M512(swapBytes)); + _mm512_mask_storeu_epi64(pGHash, 0x03, ghash512); +} + +#endif /* #if (_IPP32E>=_IPP32E_K1) */ + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmdecrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmdecrypt.c new file mode 100644 index 0000000..4e4b29e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmdecrypt.c @@ -0,0 +1,216 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-GCM +// +// Contents: +// ippsAES_GCMDecrypt() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + +#if(_IPP32E>=_IPP32E_K0) +#include "pcpaesauthgcm_avx512.h" +#else +#include "pcpaesauthgcm.h" +#endif /* #if(_IPP32E>=_IPP32E_K0) */ + +/*F* +// Name: ippsAES_GCMDecrypt +// +// Purpose: Decrypts a data buffer in the GCM mode. +// +// Returns: Reason: +// ippStsNullPtrErr pSrc == NULL +// pDst == NULL +// pState == NULL +// ippStsContextMatchErr !AESGCM_VALID_ID() +// ippStsLengthErr len<0 +// ippStsNoErr no errors +// +// Parameters: +// pSrc Pointer to ciphertext. +// pDst Pointer to plaintext. +// len Length of the plaintext and ciphertext in bytes +// pState pointer to the context +// +*F*/ + +IPPFUN(IppStatus, ippsAES_GCMDecrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int len, IppsAES_GCMState* pState)) +{ + /* test pState pointer */ + IPP_BAD_PTR1_RET(pState); + /* use aligned context */ + pState = (IppsAES_GCMState*)( IPP_ALIGNED_PTR(pState, AESGCM_ALIGNMENT) ); + /* test state ID */ + IPP_BADARG_RET(!AESGCM_VALID_ID(pState), ippStsContextMatchErr); + /* test context validity */ + IPP_BADARG_RET(!(GcmAADprocessing==AESGCM_STATE(pState) || GcmTXTprocessing==AESGCM_STATE(pState)), ippStsBadArgErr); + + /* test text pointers and length */ + IPP_BAD_PTR2_RET(pSrc, pDst); + IPP_BADARG_RET(len<0, ippStsLengthErr); + + /* According to the NIST Special Publication 800-38D (Recommendation for GCM + * mode, p.5.2.1.1 Input Data) the input text shall be between 0 and 2^39-256 + * bits. */ + const Ipp64u MAX_TXT_LEN = ((Ipp64u)1 << 36) - 32; /* length in bytes */ + IPP_BADARG_RET(((AESGCM_TXT_LEN(pState) > MAX_TXT_LEN - (Ipp64u)len) || + ((AESGCM_TXT_LEN(pState) + (Ipp64u)len) < (Ipp64u)len)), + ippStsScaleRangeErr); + +#if (_IPP32E < _IPP32E_K0) + /* get method */ + IppsAESSpec *pAES = AESGCM_CIPHER(pState); + RijnCipher encoder = RIJ_ENCODER(pAES); + MulGcm_ hashFunc = AESGCM_HASH(pState); +#endif + + if( GcmAADprocessing==AESGCM_STATE(pState) ) { +#if(_IPP32E>=_IPP32E_K0) + if(AESGCM_BUFLEN(pState)) { + MulGcm_ ghashFunc = AES_GCM_GMUL(pState); + ghashFunc(&AES_GCM_KEY_DATA(pState), AESGCM_GHASH(pState)); + } +#else + /* complete AAD processing */ + if(AESGCM_BUFLEN(pState)) + hashFunc(AESGCM_GHASH(pState), AESGCM_HKEY(pState), AesGcmConst_table); + + /* increment counter block */ + IncrementCounter32(AESGCM_COUNTER(pState)); + /* and encrypt counter */ + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder(AESGCM_COUNTER(pState), AESGCM_ECOUNTER(pState), RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder(AESGCM_COUNTER(pState), AESGCM_ECOUNTER(pState), RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif +#endif /* #if(_IPP32E>=_IPP32E_K0) */ + + /* switch mode and init counters */ + AESGCM_BUFLEN(pState) = 0; + AESGCM_TXT_LEN(pState) = CONST_64(0); + AESGCM_STATE(pState) = GcmTXTprocessing; + } + + /* + // process text (authenticate and decrypt) + */ + +#if(_IPP32E>=_IPP32E_K0) + DecryptUpdate_ dec = AES_GCM_DECRYPT_UPDATE(pState); +#if (_AES_PROB_NOISE == _FEATURE_ON_) + /* Mistletoe3 mitigation */ + cpAESNoiseParams *params = (cpAESNoiseParams*)&AESGCM_NOISE_PARAMS(pState); + if (AES_NOISE_LEVEL(params) > 0) { + /* Number of bytes allowed for operation without adding noise */ + int chunk_size; + /* Number of bytes remaining for operation */ + int remaining_size = len; + + while (remaining_size > 0) { + /* How many bytes to encrypt in this operation */ + chunk_size = (remaining_size >= MISTLETOE3_MAX_CHUNK_SIZE) ? MISTLETOE3_MAX_CHUNK_SIZE : remaining_size; + + dec(&AES_GCM_KEY_DATA(pState), &AES_GCM_CONTEXT_DATA(pState), pDst, pSrc, (Ipp64u)chunk_size); + + cpAESRandomNoise(NULL, + MISTLETOE3_BASE_NOISE_LEVEL + AES_NOISE_LEVEL(params), + MISTLETOE3_NOISE_RATE, + &AES_NOISE_RAND(params)); + + pSrc += chunk_size; + pDst += chunk_size; + remaining_size -= chunk_size; + } + } else +#endif + { + dec(&AES_GCM_KEY_DATA(pState), &AES_GCM_CONTEXT_DATA(pState), pDst, pSrc, (Ipp64u)len); + } +#else + /* process partial block */ + if(AESGCM_BUFLEN(pState)) { + int locLen = IPP_MIN(len, BLOCK_SIZE-AESGCM_BUFLEN(pState)); + /* authentication */ + XorBlock(pSrc, AESGCM_GHASH(pState)+AESGCM_BUFLEN(pState), AESGCM_GHASH(pState)+AESGCM_BUFLEN(pState), locLen); + /* ctr decryption */ + XorBlock(pSrc, AESGCM_ECOUNTER(pState)+AESGCM_BUFLEN(pState), pDst, locLen); + + AESGCM_BUFLEN(pState) += locLen; + AESGCM_TXT_LEN(pState) += (Ipp64u)locLen; + pSrc += locLen; + pDst += locLen; + len -= locLen; + + /* if buffer full */ + if(BLOCK_SIZE==AESGCM_BUFLEN(pState)) { + /* hash buffer */ + hashFunc(AESGCM_GHASH(pState), AESGCM_HKEY(pState), AesGcmConst_table); + AESGCM_BUFLEN(pState) = 0; + + /* increment counter block */ + IncrementCounter32(AESGCM_COUNTER(pState)); + /* and encrypt counter */ + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder(AESGCM_COUNTER(pState), AESGCM_ECOUNTER(pState), RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder(AESGCM_COUNTER(pState), AESGCM_ECOUNTER(pState), RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif + } + } + + /* process the main part of text */ + { + int lenBlks = len & (-BLOCK_SIZE); + if(lenBlks) { + Decrypt_ decFunc = AESGCM_DEC(pState); + + decFunc(pDst, pSrc, lenBlks, pState); + + AESGCM_TXT_LEN(pState) += (Ipp64u)lenBlks; + pSrc += lenBlks; + pDst += lenBlks; + len -= lenBlks; + } + } + + /* process the rest of text */ + if(len) { + /* ctr encryption */ + XorBlock(pSrc, AESGCM_GHASH(pState)+AESGCM_BUFLEN(pState), AESGCM_GHASH(pState)+AESGCM_BUFLEN(pState), len); + XorBlock(pSrc, AESGCM_ECOUNTER(pState)+AESGCM_BUFLEN(pState), pDst, len); + + AESGCM_BUFLEN(pState) += len; + AESGCM_TXT_LEN(pState) += (Ipp64u)len; + } +#endif /* #if(_IPP32E>=_IPP32E_K0) */ + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmdecrypt_vaes512.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmdecrypt_vaes512.c new file mode 100644 index 0000000..4d21be3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmdecrypt_vaes512.c @@ -0,0 +1,347 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES decryption (GCM mode) +// +// Contents: +// +*/ + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(disable: 4206) // empty translation unit in MSVC +#endif + +#if 0 // Not used + +#include "owncp.h" +#include "pcpaesm.h" +#include "pcpaes_encrypt_vaes512.h" +#include "pcpaes_gcm_vaes512.h" +#include "pcpaesauthgcm.h" + +#if (_IPP32E>=_IPP32E_K1) + +static __ALIGN64 Ipp32u inc_lo32x4[] = { 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0 }; +static __ALIGN64 Ipp32u inc1_lo32x4[] = { 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0 }; +static __ALIGN64 Ipp32u inc4_lo32x4[] = { 4, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0 }; + +IPP_OWN_DEFN (void, AesGcmDec_vaes, (Ipp8u* pDst, const Ipp8u* pSrc, int length, IppsAES_GCMState* pCtx)) +{ + IppsAESSpec* pAES = AESGCM_CIPHER(pCtx); + int cipherRounds = RIJ_NR(pAES) - 1; + const Ipp8u* pRKey = RIJ_EKEYS(pAES); + + Ipp8u* pCtrValue = AESGCM_COUNTER(pCtx); + Ipp8u* pGHash = AESGCM_GHASH(pCtx); + const Ipp8u* pHKey = AESGCM_HKEY(pCtx); + + __m128i* pKeys = (__m128i*)pRKey; + __m512i* pSrc512 = (__m512i*)pSrc; + __m512i* pDst512 = (__m512i*)pDst; + __m512i* pHKey512 = (__m512i*)pHKey; + + __m512i incMsk = M512(inc_lo32x4); + + /* Load hKeys vectors */ + __m512i hKeys0 = _mm512_loadu_si512(pHKey512); + __m512i hKeys1 = _mm512_loadu_si512(pHKey512 + 1); + __m512i hKeys2 = _mm512_loadu_si512(pHKey512 + 2); + __m512i hKeys3 = _mm512_loadu_si512(pHKey512 + 3); + + /* Load precomputed multipliers for Karatsuba multiplication */ + __m512i hKeysKaratsuba0 = _mm512_loadu_si512(pHKey512 + 4); + __m512i hKeysKaratsuba1 = _mm512_loadu_si512(pHKey512 + 5); + __m512i hKeysKaratsuba2 = _mm512_loadu_si512(pHKey512 + 6); + __m512i hKeysKaratsuba3 = _mm512_loadu_si512(pHKey512 + 7); + + /* Current GHASH value */ + __m128i ghash128; + __m512i ghash512 = _mm512_maskz_loadu_epi64(0x03, pGHash); + ghash512 = _mm512_shuffle_epi8(ghash512, M512(swapBytes)); + + /* Load initial counter */ + __m128i ctr128 = _mm_maskz_loadu_epi64(0x03, pCtrValue); + __m512i ctr512 = _mm512_broadcast_i64x2(ctr128); + + // convert counter to little-endian + ctr512 = _mm512_shuffle_epi8(ctr512, M512(swapBytes)); + + int blocks; + for (blocks = length / MBS_RIJ128; blocks >= (4 * 4); blocks -= (4 * 4)) { + __m512i counter0 = _mm512_add_epi32(incMsk, ctr512); + __m512i counter1 = _mm512_add_epi32(M512(inc4_lo32x4), counter0); + __m512i counter2 = _mm512_add_epi32(M512(inc4_lo32x4), counter1); + __m512i counter3 = _mm512_add_epi32(M512(inc4_lo32x4), counter2); + + incMsk = M512(inc4_lo32x4); + ctr512 = counter3; + + // convert back to big-endian + counter0 = _mm512_shuffle_epi8(counter0, M512(swapBytes)); + counter1 = _mm512_shuffle_epi8(counter1, M512(swapBytes)); + counter2 = _mm512_shuffle_epi8(counter2, M512(swapBytes)); + counter3 = _mm512_shuffle_epi8(counter3, M512(swapBytes)); + + cpAESEncrypt4_VAES_NI(&counter0, &counter1, &counter2, &counter3, pKeys, cipherRounds); + + __m512i blk0 = _mm512_loadu_si512(pSrc512); + __m512i blk1 = _mm512_loadu_si512(pSrc512 + 1); + __m512i blk2 = _mm512_loadu_si512(pSrc512 + 2); + __m512i blk3 = _mm512_loadu_si512(pSrc512 + 3); + + _mm512_storeu_si512(pDst512, _mm512_xor_si512(blk0, counter0)); + _mm512_storeu_si512(pDst512 + 1, _mm512_xor_si512(blk1, counter1)); + _mm512_storeu_si512(pDst512 + 2, _mm512_xor_si512(blk2, counter2)); + _mm512_storeu_si512(pDst512 + 3, _mm512_xor_si512(blk3, counter3)); + + /* Authenticate */ + /* Reflect inputs */ + blk0 = _mm512_shuffle_epi8(blk0, M512(swapBytes)); + blk1 = _mm512_shuffle_epi8(blk1, M512(swapBytes)); + blk2 = _mm512_shuffle_epi8(blk2, M512(swapBytes)); + blk3 = _mm512_shuffle_epi8(blk3, M512(swapBytes)); + + /* Add current GHASH to src[0] */ + blk0 = _mm512_mask_xor_epi64(blk0, 0x03, blk0, ghash512); + + /* Karatsuba multiplication for 16 blocks with postponed aggregation */ + __m512i H, M, L, tmp1, tmp2, tmp3; + AesGcmKaratsubaMul4(&blk0, &hKeys3, &hKeysKaratsuba3, &H, &M, &L); + AesGcmKaratsubaMul4(&blk1, &hKeys2, &hKeysKaratsuba2, &tmp1, &tmp2, &tmp3); + H = _mm512_xor_si512(H, tmp1); + M = _mm512_xor_si512(M, tmp2); + L = _mm512_xor_si512(L, tmp3); + AesGcmKaratsubaMul4(&blk2, &hKeys1, &hKeysKaratsuba1, &tmp1, &tmp2, &tmp3); + H = _mm512_xor_si512(H, tmp1); + M = _mm512_xor_si512(M, tmp2); + L = _mm512_xor_si512(L, tmp3); + AesGcmKaratsubaMul4(&blk3, &hKeys0, &hKeysKaratsuba0, &tmp1, &tmp2, &tmp3); + H = _mm512_xor_si512(H, tmp1); + M = _mm512_xor_si512(M, tmp2); + L = _mm512_xor_si512(L, tmp3); + + AggregateKaratsubaPartialProducts(&H, &M, &L, &ghash128); + + /* save current ghash value */ + ghash512 = _mm512_castsi128_si512(ghash128); + + pSrc512 += 4; + pDst512 += 4; + } + + if ((3 * 4) <= blocks) { + __m512i counter0 = _mm512_add_epi32(incMsk, ctr512); + __m512i counter1 = _mm512_add_epi32(M512(inc4_lo32x4), counter0); + __m512i counter2 = _mm512_add_epi32(M512(inc4_lo32x4), counter1); + + incMsk = M512(inc4_lo32x4); + ctr512 = counter2; + + // convert back to big-endian + counter0 = _mm512_shuffle_epi8(counter0, M512(swapBytes)); + counter1 = _mm512_shuffle_epi8(counter1, M512(swapBytes)); + counter2 = _mm512_shuffle_epi8(counter2, M512(swapBytes)); + + cpAESEncrypt3_VAES_NI(&counter0, &counter1, &counter2, pKeys, cipherRounds); + + __m512i blk0 = _mm512_loadu_si512(pSrc512); + __m512i blk1 = _mm512_loadu_si512(pSrc512 + 1); + __m512i blk2 = _mm512_loadu_si512(pSrc512 + 2); + + _mm512_storeu_si512(pDst512, _mm512_xor_si512(blk0, counter0)); + _mm512_storeu_si512(pDst512 + 1, _mm512_xor_si512(blk1, counter1)); + _mm512_storeu_si512(pDst512 + 2, _mm512_xor_si512(blk2, counter2)); + + /* Authenticate */ + /* Reflect inputs */ + blk0 = _mm512_shuffle_epi8(blk0, M512(swapBytes)); + blk1 = _mm512_shuffle_epi8(blk1, M512(swapBytes)); + blk2 = _mm512_shuffle_epi8(blk2, M512(swapBytes)); + + /* Add current GHASH to src[0] */ + blk0 = _mm512_mask_xor_epi64(blk0, 0x03, blk0, ghash512); + + /* Karatsuba multiplication for 16 blocks with postponed aggregation */ + __m512i H, M, L, tmp1, tmp2, tmp3; + AesGcmKaratsubaMul4(&blk0, &hKeys3, &hKeysKaratsuba2, &H, &M, &L); + AesGcmKaratsubaMul4(&blk1, &hKeys2, &hKeysKaratsuba1, &tmp1, &tmp2, &tmp3); + H = _mm512_xor_si512(H, tmp1); + M = _mm512_xor_si512(M, tmp2); + L = _mm512_xor_si512(L, tmp3); + AesGcmKaratsubaMul4(&blk2, &hKeys1, &hKeysKaratsuba0, &tmp1, &tmp2, &tmp3); + H = _mm512_xor_si512(H, tmp1); + M = _mm512_xor_si512(M, tmp2); + L = _mm512_xor_si512(L, tmp3); + + AggregateKaratsubaPartialProducts(&H, &M, &L, &ghash128); + + /* save current ghash value */ + ghash512 = _mm512_castsi128_si512(ghash128); + + pSrc512 += 3; + pDst512 += 3; + blocks -= (3 * 4); + } + + if ((4 * 2) <= blocks) { + __m512i counter0 = _mm512_add_epi32(incMsk, ctr512); + __m512i counter1 = _mm512_add_epi32(M512(inc4_lo32x4), counter0); + + incMsk = M512(inc4_lo32x4); + ctr512 = counter1; + + // convert back to big-endian + counter0 = _mm512_shuffle_epi8(counter0, M512(swapBytes)); + counter1 = _mm512_shuffle_epi8(counter1, M512(swapBytes)); + + cpAESEncrypt2_VAES_NI(&counter0, &counter1, pKeys, cipherRounds); + + __m512i blk0 = _mm512_loadu_si512(pSrc512); + __m512i blk1 = _mm512_loadu_si512(pSrc512 + 1); + + _mm512_storeu_si512(pDst512, _mm512_xor_si512(blk0, counter0)); + _mm512_storeu_si512(pDst512 + 1, _mm512_xor_si512(blk1, counter1)); + + /* Authenticate */ + /* Reflect inputs */ + blk0 = _mm512_shuffle_epi8(blk0, M512(swapBytes)); + blk1 = _mm512_shuffle_epi8(blk1, M512(swapBytes)); + + /* Add current GHASH to src[0] */ + blk0 = _mm512_mask_xor_epi64(blk0, 0x03, blk0, ghash512); + + __m512i H, M, L, tmp1, tmp2, tmp3; + AesGcmKaratsubaMul4(&blk0, &hKeys1, &hKeysKaratsuba1, &H, &M, &L); + AesGcmKaratsubaMul4(&blk1, &hKeys0, &hKeysKaratsuba0, &tmp1, &tmp2, &tmp3); + H = _mm512_xor_si512(H, tmp1); + M = _mm512_xor_si512(M, tmp2); + L = _mm512_xor_si512(L, tmp3); + + AggregateKaratsubaPartialProducts(&H, &M, &L, &ghash128); + + /* save current ghash value */ + ghash512 = _mm512_castsi128_si512(ghash128); + + pSrc512 += 2; + pDst512 += 2; + blocks -= (2 * 4); + } + + for (; blocks >= 4; blocks -= 4) { + __m512i counter0 = _mm512_add_epi32(incMsk, ctr512); + + incMsk = M512(inc4_lo32x4); + ctr512 = counter0; + + // convert back to big-endian + counter0 = _mm512_shuffle_epi8(counter0, M512(swapBytes)); + + cpAESEncrypt1_VAES_NI(&counter0, pKeys, cipherRounds); + + __m512i blk0 = _mm512_loadu_si512(pSrc512); + _mm512_storeu_si512(pDst512, _mm512_xor_si512(blk0, counter0)); + + /* Authenticate */ + blk0 = _mm512_shuffle_epi8(blk0, M512(swapBytes)); + /* Add current GHASH to src[0] */ + blk0 = _mm512_mask_xor_epi64(blk0, 0x03, blk0, ghash512); + + __m512i H, M, L; + AesGcmKaratsubaMul4(&blk0, &hKeys0, &hKeysKaratsuba0, &H, &M, &L); + + AggregateKaratsubaPartialProducts(&H, &M, &L, &ghash128); + + /* save current ghash value */ + ghash512 = _mm512_castsi128_si512(ghash128); + + pSrc512 += 1; + pDst512 += 1; + } + + if (blocks) { + __mmask8 k8 = (__mmask8)((1 << (blocks + blocks)) - 1); // 64-bit chunks + __mmask16 k16 = (__mmask16)((1 << (blocks << 2)) - 1); // 32-bit chunks + __mmask64 k64 = (__mmask64)((1LL << (blocks << 4)) - 1); // 8-bit chunks + + __m512i counter0 = _mm512_maskz_add_epi32(k16, incMsk, ctr512); + ctr512 = counter0; + + // convert back to big-endian + counter0 = _mm512_maskz_shuffle_epi8(k64, counter0, M512(swapBytes)); + + cpAESEncrypt1_VAES_NI(&counter0, pKeys, cipherRounds); + + __m512i blk0 = _mm512_maskz_loadu_epi64(k8, pSrc512); + _mm512_mask_storeu_epi64(pDst512, k8, _mm512_maskz_xor_epi64(k8, blk0, counter0)); + + /* Authenticate */ + blk0 = _mm512_shuffle_epi8(blk0, M512(swapBytes)); + /* Add current GHASH to src[0] */ + blk0 = _mm512_mask_xor_epi64(blk0, 0x03, blk0, ghash512); + + __m512i H = _mm512_setzero_si512(); + __m512i M = _mm512_setzero_si512(); + __m512i L = _mm512_setzero_si512(); + + // NB: we need immediate parameter in alignr function + switch (blocks) { + case 1: { + hKeys0 = _mm512_alignr_epi64(_mm512_setzero_si512(), hKeys0, 6); + hKeysKaratsuba0 = _mm512_alignr_epi64(_mm512_setzero_si512(), hKeysKaratsuba0, 6); + break; + } + case 2: { + hKeys0 = _mm512_alignr_epi64(_mm512_setzero_si512(), hKeys0, 4); + hKeysKaratsuba0 = _mm512_alignr_epi64(_mm512_setzero_si512(), hKeysKaratsuba0, 4); + break; + } + default: { + hKeys0 = _mm512_alignr_epi64(_mm512_setzero_si512(), hKeys0, 2); + hKeysKaratsuba0 = _mm512_alignr_epi64(_mm512_setzero_si512(), hKeysKaratsuba0, 2); + } + } + AesGcmKaratsubaMul4(&blk0, &hKeys0, &hKeysKaratsuba0, &H, &M, &L); + + AggregateKaratsubaPartialProducts(&H, &M, &L, &ghash128); + + /* save current ghash value */ + ghash512 = _mm512_castsi128_si512(ghash128); + } + + // return last counter + __mmask8 lastCtrK8 = blocks == 0 ? 0xC0 : (__mmask8)((Ipp8u)0x03<<((blocks-1)<<1)); + __mmask16 lastCtrK16 = blocks == 0 ? 0xF000 : (__mmask16)((Ipp16u)0xF<<((blocks-1)<<2)); + __mmask64 lastCtrK64 = blocks == 0 ? 0xFFFF000000000000 : (__mmask64)((Ipp64u)0xFFFF<<((blocks-1)<<4)); + + ctr512 = _mm512_maskz_add_epi32(lastCtrK16, M512(inc1_lo32x4), ctr512); + ctr512 = _mm512_maskz_shuffle_epi8(lastCtrK64, ctr512, M512(swapBytes)); + + /* save next counter */ + _mm512_mask_compressstoreu_epi64(pCtrValue, lastCtrK8, ctr512); + + /* save current ghash value */ + ghash512 = _mm512_shuffle_epi8(ghash512, M512(swapBytes)); + _mm512_mask_storeu_epi64(pGHash, 0x03, ghash512); +} + +#endif + +#endif /* #if (_IPP32E>=_IPP32E_K1) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmencrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmencrypt.c new file mode 100644 index 0000000..9312a29 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmencrypt.c @@ -0,0 +1,216 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-GCM +// +// Contents: +// ippsAES_GCMEncrypt() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + +#if(_IPP32E>=_IPP32E_K0) +#include "pcpaesauthgcm_avx512.h" +#else +#include "pcpaesauthgcm.h" +#endif /* #if(_IPP32E>=_IPP32E_K0) */ + +/*F* +// Name: ippsAES_GCMEncrypt +// +// Purpose: Encrypts a data buffer in the GCM mode. +// +// Returns: Reason: +// ippStsNullPtrErr pSrc == NULL +// pDst == NULL +// pState == NULL +// ippStsContextMatchErr !AESGCM_VALID_ID() +// ippStsLengthErr len<0 +// ippStsNoErr no errors +// +// Parameters: +// pSrc Pointer to plaintext. +// pDst Pointer to ciphertext. +// len Length of the plaintext and ciphertext in bytes +// pState pointer to the context +// +*F*/ +IPPFUN(IppStatus, ippsAES_GCMEncrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + IppsAES_GCMState* pState)) +{ + /* test pState pointer */ + IPP_BAD_PTR1_RET(pState); + /* use aligned context */ + pState = (IppsAES_GCMState*)( IPP_ALIGNED_PTR(pState, AESGCM_ALIGNMENT) ); + /* test state ID */ + IPP_BADARG_RET(!AESGCM_VALID_ID(pState), ippStsContextMatchErr); + /* test context validity */ + IPP_BADARG_RET(!(GcmAADprocessing==AESGCM_STATE(pState) || GcmTXTprocessing==AESGCM_STATE(pState)), ippStsBadArgErr); + + /* test text pointers and length */ + IPP_BAD_PTR2_RET(pSrc, pDst); + IPP_BADARG_RET(len<0, ippStsLengthErr); + + /* According to the NIST Special Publication 800-38D (Recommendation for GCM + * mode, p.5.2.1.1 Input Data) the input text shall be between 0 and 2^39-256 + * bits. */ + const Ipp64u MAX_TXT_LEN = ((Ipp64u)1 << 36) - 32; /* length in bytes */ + IPP_BADARG_RET(((AESGCM_TXT_LEN(pState) > MAX_TXT_LEN - (Ipp64u)len) || + ((AESGCM_TXT_LEN(pState) + (Ipp64u)len) < (Ipp64u)len)), + ippStsScaleRangeErr); + +#if(_IPP32E<_IPP32E_K0) + /* get method */ + IppsAESSpec* pAES = AESGCM_CIPHER(pState); + RijnCipher encoder = RIJ_ENCODER(pAES); + MulGcm_ hashFunc = AESGCM_HASH(pState); +#endif + + if( GcmAADprocessing==AESGCM_STATE(pState) ) { +#if(_IPP32E>=_IPP32E_K0) + if(AESGCM_BUFLEN(pState)) { + MulGcm_ ghashFunc = AES_GCM_GMUL(pState); + ghashFunc(&AES_GCM_KEY_DATA(pState), AESGCM_GHASH(pState)); + } +#else + /* complete AAD processing */ + if(AESGCM_BUFLEN(pState)) + hashFunc(AESGCM_GHASH(pState), AESGCM_HKEY(pState), AesGcmConst_table); + + /* increment counter block */ + IncrementCounter32(AESGCM_COUNTER(pState)); + /* and encrypt counter */ + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder(AESGCM_COUNTER(pState), AESGCM_ECOUNTER(pState), RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder(AESGCM_COUNTER(pState), AESGCM_ECOUNTER(pState), RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif +#endif /* #if(_IPP32E>=_IPP32E_K0) */ + + /* switch mode and init counters */ + AESGCM_STATE(pState) = GcmTXTprocessing; + AESGCM_TXT_LEN(pState) = CONST_64(0); + AESGCM_BUFLEN(pState) = 0; + } + + /* execute encryption with code from Intel IPsec if possible */ +#if(_IPP32E>=_IPP32E_K0) + EncryptUpdate_ enc = AES_GCM_ENCRYPT_UPDATE(pState); +#if (_AES_PROB_NOISE == _FEATURE_ON_) + /* Mistletoe3 mitigation */ + cpAESNoiseParams *params = (cpAESNoiseParams*)&AESGCM_NOISE_PARAMS(pState); + if (AES_NOISE_LEVEL(params) > 0) { + /* Number of bytes allowed for operation without adding noise */ + int chunk_size; + /* Number of bytes remaining for operation */ + int remaining_size = len; + + while (remaining_size > 0) { + /* How many bytes to encrypt in this operation */ + chunk_size = (remaining_size >= MISTLETOE3_MAX_CHUNK_SIZE) ? MISTLETOE3_MAX_CHUNK_SIZE : remaining_size; + + enc(&AES_GCM_KEY_DATA(pState), &AES_GCM_CONTEXT_DATA(pState), pDst, pSrc, (Ipp64u)chunk_size); + + cpAESRandomNoise(NULL, + MISTLETOE3_BASE_NOISE_LEVEL + AES_NOISE_LEVEL(params), + MISTLETOE3_NOISE_RATE, + &AES_NOISE_RAND(params)); + + pSrc += chunk_size; + pDst += chunk_size; + remaining_size -= chunk_size; + } + } else +#endif + { + enc(&AES_GCM_KEY_DATA(pState), &AES_GCM_CONTEXT_DATA(pState), pDst, pSrc, (Ipp64u)len); + } +#else + /* + // process text (encrypt and authenticate) + */ + + /* process partial block */ + if(AESGCM_BUFLEN(pState)) { + int locLen = IPP_MIN(len, BLOCK_SIZE-AESGCM_BUFLEN(pState)); + /* ctr encryption */ + XorBlock(pSrc, AESGCM_ECOUNTER(pState)+AESGCM_BUFLEN(pState), pDst, locLen); + /* authentication */ + XorBlock(pDst, AESGCM_GHASH(pState)+AESGCM_BUFLEN(pState), AESGCM_GHASH(pState)+AESGCM_BUFLEN(pState), locLen); + + AESGCM_BUFLEN(pState) += locLen; + AESGCM_TXT_LEN(pState) += (Ipp64u)locLen; + pSrc += locLen; + pDst += locLen; + len -= locLen; + + /* if buffer full */ + if(BLOCK_SIZE==AESGCM_BUFLEN(pState)) { + /* hash buffer */ + hashFunc(AESGCM_GHASH(pState), AESGCM_HKEY(pState), AesGcmConst_table); + AESGCM_BUFLEN(pState) = 0; + + /* increment counter block */ + IncrementCounter32(AESGCM_COUNTER(pState)); + /* and encrypt counter */ + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder(AESGCM_COUNTER(pState), AESGCM_ECOUNTER(pState), RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder(AESGCM_COUNTER(pState), AESGCM_ECOUNTER(pState), RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif + } + } + + /* process the main part of text */ + { + int lenBlks = len & (-BLOCK_SIZE); + if(lenBlks) { + Encrypt_ encFunc = AESGCM_ENC(pState); + + encFunc(pDst, pSrc, lenBlks, pState); + + AESGCM_TXT_LEN(pState) += (Ipp64u)lenBlks; + pSrc += lenBlks; + pDst += lenBlks; + len -= lenBlks; + } + } + + /* process the rest of text */ + if(len) { + XorBlock(pSrc, AESGCM_ECOUNTER(pState)+AESGCM_BUFLEN(pState), pDst, len); + XorBlock(pDst, AESGCM_GHASH(pState)+AESGCM_BUFLEN(pState), AESGCM_GHASH(pState)+AESGCM_BUFLEN(pState), len); + + AESGCM_BUFLEN(pState) += len; + AESGCM_TXT_LEN(pState) += (Ipp64u)len; + } +#endif /* #if(_IPP32E>=_IPP32E_K0) */ + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmencrypt_vaes512.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmencrypt_vaes512.c new file mode 100755 index 0000000..3778cf0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmencrypt_vaes512.c @@ -0,0 +1,362 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption (GCM mode) +// +// Contents: +// +*/ + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(disable: 4206) // empty translation unit in MSVC +#endif + +#if 0 // Not used + +#include "owncp.h" +#include "pcpaesm.h" +#include "pcpaes_encrypt_vaes512.h" +#include "pcpaes_gcm_vaes512.h" +#include "pcpaesauthgcm.h" + +#if (_IPP32E>=_IPP32E_K1) + +static __ALIGN64 Ipp32u inc_lo32x4[] = { 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0 }; +static __ALIGN64 Ipp32u inc1_lo32x4[] = { 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0 }; +static __ALIGN64 Ipp32u inc4_lo32x4[] = { 4, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0 }; + +/* Encryption with authentication */ +IPP_OWN_DEFN (void, AesGcmEnc_vaes, (Ipp8u* pDst, const Ipp8u* pSrc, int length, IppsAES_GCMState* pCtx)) +{ + IppsAESSpec* pAES = AESGCM_CIPHER(pCtx); + int cipherRounds = RIJ_NR(pAES) - 1; + const Ipp8u* pRKey = RIJ_EKEYS(pAES); + + Ipp8u* pCtrValue = AESGCM_COUNTER(pCtx); + Ipp8u* pGHash = AESGCM_GHASH(pCtx); + const Ipp8u* pHKey = AESGCM_HKEY(pCtx); + + __m128i* pKeys = (__m128i*)pRKey; + __m512i* pSrc512 = (__m512i*)pSrc; + __m512i* pDst512 = (__m512i*)pDst; + __m512i* pHKey512 = (__m512i*)pHKey; + + __m512i incMsk = M512(inc_lo32x4); + + __m512i H, M, L, tmp1, tmp2, tmp3; + __m512i counter0, counter1, counter2, counter3; + __m512i blk0, blk1, blk2, blk3; + + /* Load hKeys vectors */ + __m512i hKeys0 = _mm512_loadu_si512(pHKey512); + __m512i hKeys1 = _mm512_loadu_si512(pHKey512 + 1); + __m512i hKeys2 = _mm512_loadu_si512(pHKey512 + 2); + __m512i hKeys3 = _mm512_loadu_si512(pHKey512 + 3); + + /* Load precomputed multipliers for Karatsuba multiplication */ + __m512i hKeysKaratsuba0 = _mm512_loadu_si512(pHKey512 + 4); + __m512i hKeysKaratsuba1 = _mm512_loadu_si512(pHKey512 + 5); + __m512i hKeysKaratsuba2 = _mm512_loadu_si512(pHKey512 + 6); + __m512i hKeysKaratsuba3 = _mm512_loadu_si512(pHKey512 + 7); + + /* Current GHASH value */ + __m128i ghash128; + __m512i ghash512 = _mm512_maskz_loadu_epi64(0x03, pGHash); + ghash512 = _mm512_shuffle_epi8(ghash512, M512(swapBytes)); + + /* Load initial counter */ + __m128i ctr128 = _mm_maskz_loadu_epi64(0x03, pCtrValue); + __m512i ctr512 = _mm512_broadcast_i64x2(ctr128); + + // convert counter to little-endian + ctr512 = _mm512_shuffle_epi8(ctr512, M512(swapBytes)); + + int blocks; + for (blocks = length / MBS_RIJ128; blocks >= (4 * 4); blocks -= (4 * 4)) { + counter0 = _mm512_add_epi32(incMsk, ctr512); + counter1 = _mm512_add_epi32(M512(inc4_lo32x4), counter0); + counter2 = _mm512_add_epi32(M512(inc4_lo32x4), counter1); + counter3 = _mm512_add_epi32(M512(inc4_lo32x4), counter2); + + incMsk = M512(inc4_lo32x4); + ctr512 = counter3; + + // convert back to big-endian + counter0 = _mm512_shuffle_epi8(counter0, M512(swapBytes)); + counter1 = _mm512_shuffle_epi8(counter1, M512(swapBytes)); + counter2 = _mm512_shuffle_epi8(counter2, M512(swapBytes)); + counter3 = _mm512_shuffle_epi8(counter3, M512(swapBytes)); + + cpAESEncrypt4_VAES_NI(&counter0, &counter1, &counter2, &counter3, pKeys, cipherRounds); + + blk0 = _mm512_loadu_si512(pSrc512); + blk1 = _mm512_loadu_si512(pSrc512 + 1); + blk2 = _mm512_loadu_si512(pSrc512 + 2); + blk3 = _mm512_loadu_si512(pSrc512 + 3); + + blk0 = _mm512_xor_si512(blk0, counter0); + blk1 = _mm512_xor_si512(blk1, counter1); + blk2 = _mm512_xor_si512(blk2, counter2); + blk3 = _mm512_xor_si512(blk3, counter3); + + _mm512_storeu_si512(pDst512, blk0); + _mm512_storeu_si512(pDst512 + 1, blk1); + _mm512_storeu_si512(pDst512 + 2, blk2); + _mm512_storeu_si512(pDst512 + 3, blk3); + + /* Authenticate */ + /* Reflect inputs */ + blk0 = _mm512_shuffle_epi8(blk0, M512(swapBytes)); + blk1 = _mm512_shuffle_epi8(blk1, M512(swapBytes)); + blk2 = _mm512_shuffle_epi8(blk2, M512(swapBytes)); + blk3 = _mm512_shuffle_epi8(blk3, M512(swapBytes)); + + /* Add current GHASH to src[0] */ + blk0 = _mm512_mask_xor_epi64(blk0, 0x03, blk0, ghash512); + + /* Karatsuba multiplication for 16 blocks with postponed aggregation */ + AesGcmKaratsubaMul4(&blk0, &hKeys3, &hKeysKaratsuba3, &H, &M, &L); + AesGcmKaratsubaMul4(&blk1, &hKeys2, &hKeysKaratsuba2, &tmp1, &tmp2, &tmp3); + H = _mm512_xor_si512(H, tmp1); + M = _mm512_xor_si512(M, tmp2); + L = _mm512_xor_si512(L, tmp3); + AesGcmKaratsubaMul4(&blk2, &hKeys1, &hKeysKaratsuba1, &tmp1, &tmp2, &tmp3); + H = _mm512_xor_si512(H, tmp1); + M = _mm512_xor_si512(M, tmp2); + L = _mm512_xor_si512(L, tmp3); + AesGcmKaratsubaMul4(&blk3, &hKeys0, &hKeysKaratsuba0, &tmp1, &tmp2, &tmp3); + H = _mm512_xor_si512(H, tmp1); + M = _mm512_xor_si512(M, tmp2); + L = _mm512_xor_si512(L, tmp3); + + AggregateKaratsubaPartialProducts(&H, &M, &L, &ghash128); + + /* save current ghash value */ + ghash512 = _mm512_castsi128_si512(ghash128); + + pSrc512 += 4; + pDst512 += 4; + } + + if ((3 * 4) <= blocks) { + counter0 = _mm512_add_epi32(incMsk, ctr512); + counter1 = _mm512_add_epi32(M512(inc4_lo32x4), counter0); + counter2 = _mm512_add_epi32(M512(inc4_lo32x4), counter1); + + incMsk = M512(inc4_lo32x4); + ctr512 = counter2; + + // convert back to big-endian + counter0 = _mm512_shuffle_epi8(counter0, M512(swapBytes)); + counter1 = _mm512_shuffle_epi8(counter1, M512(swapBytes)); + counter2 = _mm512_shuffle_epi8(counter2, M512(swapBytes)); + + cpAESEncrypt3_VAES_NI(&counter0, &counter1, &counter2, pKeys, cipherRounds); + + blk0 = _mm512_loadu_si512(pSrc512); + blk1 = _mm512_loadu_si512(pSrc512 + 1); + blk2 = _mm512_loadu_si512(pSrc512 + 2); + + blk0 = _mm512_xor_si512(blk0, counter0); + blk1 = _mm512_xor_si512(blk1, counter1); + blk2 = _mm512_xor_si512(blk2, counter2); + + _mm512_storeu_si512(pDst512, blk0); + _mm512_storeu_si512(pDst512 + 1, blk1); + _mm512_storeu_si512(pDst512 + 2, blk2); + + /* Authenticate */ + /* Reflect inputs */ + blk0 = _mm512_shuffle_epi8(blk0, M512(swapBytes)); + blk1 = _mm512_shuffle_epi8(blk1, M512(swapBytes)); + blk2 = _mm512_shuffle_epi8(blk2, M512(swapBytes)); + + /* Add current GHASH to src[0] */ + blk0 = _mm512_mask_xor_epi64(blk0, 0x03, blk0, ghash512); + + /* Karatsuba multiplication for 16 blocks with postponed aggregation */ + AesGcmKaratsubaMul4(&blk0, &hKeys3, &hKeysKaratsuba2, &H, &M, &L); + AesGcmKaratsubaMul4(&blk1, &hKeys2, &hKeysKaratsuba1, &tmp1, &tmp2, &tmp3); + H = _mm512_xor_si512(H, tmp1); + M = _mm512_xor_si512(M, tmp2); + L = _mm512_xor_si512(L, tmp3); + AesGcmKaratsubaMul4(&blk2, &hKeys1, &hKeysKaratsuba0, &tmp1, &tmp2, &tmp3); + H = _mm512_xor_si512(H, tmp1); + M = _mm512_xor_si512(M, tmp2); + L = _mm512_xor_si512(L, tmp3); + + AggregateKaratsubaPartialProducts(&H, &M, &L, &ghash128); + + /* save current ghash value */ + ghash512 = _mm512_castsi128_si512(ghash128); + + pSrc512 += 3; + pDst512 += 3; + blocks -= (3 * 4); + } + + if ((4 * 2) <= blocks) { + counter0 = _mm512_add_epi32(incMsk, ctr512); + counter1 = _mm512_add_epi32(M512(inc4_lo32x4), counter0); + + incMsk = M512(inc4_lo32x4); + ctr512 = counter1; + + // convert back to big-endian + counter0 = _mm512_shuffle_epi8(counter0, M512(swapBytes)); + counter1 = _mm512_shuffle_epi8(counter1, M512(swapBytes)); + + cpAESEncrypt2_VAES_NI(&counter0, &counter1, pKeys, cipherRounds); + + blk0 = _mm512_loadu_si512(pSrc512); + blk1 = _mm512_loadu_si512(pSrc512 + 1); + + blk0 = _mm512_xor_si512(blk0, counter0); + blk1 = _mm512_xor_si512(blk1, counter1); + + _mm512_storeu_si512(pDst512, blk0); + _mm512_storeu_si512(pDst512 + 1, blk1); + + /* Authenticate */ + /* Reflect inputs */ + blk0 = _mm512_shuffle_epi8(blk0, M512(swapBytes)); + blk1 = _mm512_shuffle_epi8(blk1, M512(swapBytes)); + + /* Add current GHASH to src[0] */ + blk0 = _mm512_mask_xor_epi64(blk0, 0x03, blk0, ghash512); + + AesGcmKaratsubaMul4(&blk0, &hKeys1, &hKeysKaratsuba1, &H, &M, &L); + AesGcmKaratsubaMul4(&blk1, &hKeys0, &hKeysKaratsuba0, &tmp1, &tmp2, &tmp3); + H = _mm512_xor_si512(H, tmp1); + M = _mm512_xor_si512(M, tmp2); + L = _mm512_xor_si512(L, tmp3); + + AggregateKaratsubaPartialProducts(&H, &M, &L, &ghash128); + + /* save current ghash value */ + ghash512 = _mm512_castsi128_si512(ghash128); + + pSrc512 += 2; + pDst512 += 2; + blocks -= (2 * 4); + } + + for (; blocks >= 4; blocks -= 4) { + counter0 = _mm512_add_epi32(incMsk, ctr512); + + incMsk = M512(inc4_lo32x4); + ctr512 = counter0; + + // convert back to big-endian + counter0 = _mm512_shuffle_epi8(counter0, M512(swapBytes)); + + cpAESEncrypt1_VAES_NI(&counter0, pKeys, cipherRounds); + + blk0 = _mm512_loadu_si512(pSrc512); + blk0 = _mm512_xor_si512(blk0, counter0); + _mm512_storeu_si512(pDst512, blk0); + + /* Authenticate */ + blk0 = _mm512_shuffle_epi8(blk0, M512(swapBytes)); + /* Add current GHASH to src[0] */ + blk0 = _mm512_mask_xor_epi64(blk0, 0x03, blk0, ghash512); + + AesGcmKaratsubaMul4(&blk0, &hKeys0, &hKeysKaratsuba0, &H, &M, &L); + + AggregateKaratsubaPartialProducts(&H, &M, &L, &ghash128); + + /* save current ghash value */ + ghash512 = _mm512_castsi128_si512(ghash128); + + pSrc512 += 1; + pDst512 += 1; + } + + if (blocks) { + __mmask8 k8 = (__mmask8)((1 << (blocks + blocks)) - 1); // 64-bit chunks + __mmask16 k16 = (__mmask16)((1 << (blocks << 2)) - 1); // 32-bit chunks + __mmask64 k64 = (__mmask64)((1LL << (blocks << 4)) - 1); // 8-bit chunks + + counter0 = _mm512_maskz_add_epi32(k16, incMsk, ctr512); + ctr512 = counter0; + + // convert back to big-endian + counter0 = _mm512_maskz_shuffle_epi8(k64, counter0, M512(swapBytes)); + + cpAESEncrypt1_VAES_NI(&counter0, pKeys, cipherRounds); + + blk0 = _mm512_maskz_loadu_epi64(k8, pSrc512); + blk0 = _mm512_maskz_xor_epi64(k8, blk0, counter0); + _mm512_mask_storeu_epi64(pDst512, k8, blk0); + + /* Authenticate */ + blk0 = _mm512_shuffle_epi8(blk0, M512(swapBytes)); + /* Add current GHASH to src[0] */ + blk0 = _mm512_mask_xor_epi64(blk0, 0x03, blk0, ghash512); + + H = _mm512_setzero_si512(); + M = _mm512_setzero_si512(); + L = _mm512_setzero_si512(); + + // NB: we need immediate parameter in alignr function + switch (blocks) { + case 1: { + hKeys0 = _mm512_alignr_epi64(_mm512_setzero_si512(), hKeys0, 6); + hKeysKaratsuba0 = _mm512_alignr_epi64(_mm512_setzero_si512(), hKeysKaratsuba0, 6); + break; + } + case 2: { + hKeys0 = _mm512_alignr_epi64(_mm512_setzero_si512(), hKeys0, 4); + hKeysKaratsuba0 = _mm512_alignr_epi64(_mm512_setzero_si512(), hKeysKaratsuba0, 4); + break; + } + default: { + hKeys0 = _mm512_alignr_epi64(_mm512_setzero_si512(), hKeys0, 2); + hKeysKaratsuba0 = _mm512_alignr_epi64(_mm512_setzero_si512(), hKeysKaratsuba0, 2); + } + } + AesGcmKaratsubaMul4(&blk0, &hKeys0, &hKeysKaratsuba0, &H, &M, &L); + + AggregateKaratsubaPartialProducts(&H, &M, &L, &ghash128); + + /* save current ghash value */ + ghash512 = _mm512_castsi128_si512(ghash128); + } + + // return last counter + __mmask8 lastCtrK8 = blocks == 0 ? 0xC0 : (__mmask8)((Ipp8u)0x03<<((blocks-1)<<1)); + __mmask16 lastCtrK16 = blocks == 0 ? 0xF000 : (__mmask16)((Ipp16u)0xF<<((blocks-1)<<2)); + __mmask64 lastCtrK64 = blocks == 0 ? 0xFFFF000000000000 : (__mmask64)((Ipp64u)0xFFFF<<((blocks-1)<<4)); + + ctr512 = _mm512_maskz_add_epi32(lastCtrK16, M512(inc1_lo32x4), ctr512); + ctr512 = _mm512_maskz_shuffle_epi8(lastCtrK64, ctr512, M512(swapBytes)); + + /* save next counter */ + _mm512_mask_compressstoreu_epi64(pCtrValue, lastCtrK8, ctr512); + + /* save current ghash value */ + ghash512 = _mm512_shuffle_epi8(ghash512, M512(swapBytes)); + _mm512_mask_storeu_epi64(pGHash, 0x03, ghash512); +} + +#endif + +#endif /* #if (_IPP32E>=_IPP32E_K1) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmgetsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmgetsize.c new file mode 100644 index 0000000..e5b566c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmgetsize.c @@ -0,0 +1,62 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-GCM +// +// Contents: +// ippsAES_GCMGetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" + +#if(_IPP32E>=_IPP32E_K0) +#include "pcpaesauthgcm_avx512.h" +#else +#include "pcpaesauthgcm.h" +#endif /* #if(_IPP32E>=_IPP32E_K0) */ + +/*F* +// Name: ippsAES_GCMGetSize +// +// Purpose: Returns size of AES_GCM state (in bytes). +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to size of context +// +*F*/ + +IPPFUN(IppStatus, ippsAES_GCMGetSize,(int* pSize)) +{ + /* test size's pointer */ + IPP_BAD_PTR1_RET(pSize); + + *pSize = cpSizeofCtx_AESGCM(); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmgettag.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmgettag.c new file mode 100644 index 0000000..de23cf5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmgettag.c @@ -0,0 +1,111 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-GCM +// +// Contents: +// ippsAES_GCMGetTag() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" + +#if(_IPP32E>=_IPP32E_K0) +#include "pcpaesauthgcm_avx512.h" +#else +#include "pcpaesauthgcm.h" +#endif /* #if(_IPP32E>=_IPP32E_K0) */ + +/*F* +// Name: ippsAES_GCMGetTag +// +// Purpose: Generates authentication tag in the GCM mode. +// +// Returns: Reason: +// ippStsNullPtrErr pDstTag == NULL +// pState == NULL +// ippStsLengthErr tagLen<=0 || tagLen>16 +// ippStsContextMatchErr !AESGCM_VALID_ID() +// ippStsNoErr no errors +// +// Parameters: +// pDstTag pointer to the authentication tag. +// tagLen length of the authentication tag *pDstTag in bytes +// pState pointer to the context +// +*F*/ + +IPPFUN(IppStatus, ippsAES_GCMGetTag,(Ipp8u* pDstTag, int tagLen, const IppsAES_GCMState* pState)) +{ + /* test State pointer */ + IPP_BAD_PTR1_RET(pState); + /* use aligned context */ + pState = (IppsAES_GCMState*)( IPP_ALIGNED_PTR(pState, AESGCM_ALIGNMENT) ); + /* test state ID */ + IPP_BADARG_RET(!AESGCM_VALID_ID(pState), ippStsContextMatchErr); + + /* test tag pointer and length */ + IPP_BAD_PTR1_RET(pDstTag); + IPP_BADARG_RET(tagLen<=0 || tagLen>BLOCK_SIZE, ippStsLengthErr); + +#if(_IPP32E>=_IPP32E_K0) + GetTag_ getTag = AES_GCM_GET_TAG(pState); + getTag(&AES_GCM_KEY_DATA(pState), &AES_GCM_CONTEXT_DATA(pState), pDstTag, (Ipp64u)tagLen); +#else + /* get method */ + MulGcm_ hashFunc = AESGCM_HASH(pState); + + __ALIGN16 Ipp8u tmpHash[BLOCK_SIZE]; + Ipp8u tmpCntr[BLOCK_SIZE]; + + /* local copy of AAD and text counters (in bits) */ + Ipp64u aadBitLen = AESGCM_AAD_LEN(pState)*BYTESIZE; + Ipp64u txtBitLen = AESGCM_TXT_LEN(pState)*BYTESIZE; + + /* do local copy of ghash */ + CopyBlock16(AESGCM_GHASH(pState), tmpHash); + + /* complete text processing */ + if(AESGCM_BUFLEN(pState)) { + hashFunc(tmpHash, AESGCM_HKEY(pState), AesGcmConst_table); + } + + /* process lengths of AAD and text */ + U32_TO_HSTRING(tmpCntr, IPP_HIDWORD(aadBitLen)); + U32_TO_HSTRING(tmpCntr+4, IPP_LODWORD(aadBitLen)); + U32_TO_HSTRING(tmpCntr+8, IPP_HIDWORD(txtBitLen)); + U32_TO_HSTRING(tmpCntr+12,IPP_LODWORD(txtBitLen)); + + XorBlock16(tmpHash, tmpCntr, tmpHash); + hashFunc(tmpHash, AESGCM_HKEY(pState), AesGcmConst_table); + + /* add encrypted initial counter */ + XorBlock16(tmpHash, AESGCM_ECOUNTER0(pState), tmpHash); + + /* return tag of required length */ + CopyBlock(tmpHash, pDstTag, tagLen); +#endif /* #if(_IPP32E>=_IPP32E_K0) */ + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcminit.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcminit.c new file mode 100644 index 0000000..92d81f6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcminit.c @@ -0,0 +1,179 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-GCM +// +// Contents: +// ippsAES_GCMInit() +// +*/ + +#include "aes_gcm_avx512.h" +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" +#include "pcpaesminit_internal.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + +#if(_IPP32E>=_IPP32E_K0) +#include "pcpaesauthgcm_avx512.h" +#include "aes_keyexp.h" +#else +#include "pcpaesauthgcm.h" +#endif /* #if(_IPP32E>=_IPP32E_K0) */ + +/*F* +// Name: ippsAES_GCMInit +// +// Purpose: Init AES_GCM context for future usage. +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL +// ippStsMemAllocErr size of buffer is not match fro operation +// ippStsLengthErr keyLen != 16 && +// != 24 && +// != 32 +// ippStsNoErr no errors +// +// Parameters: +// pKey pointer to the secret key +// keyLen length of secret key +// pState pointer to the AES-GCM context +// ctxSize available size (in bytes) of buffer above +// +*F*/ +IPPFUN(IppStatus, ippsAES_GCMInit,(const Ipp8u* pKey, int keyLen, IppsAES_GCMState* pState, int ctxSize)) +{ + /* test pCtx pointer */ + IPP_BAD_PTR1_RET(pState); + + /* test available size of context buffer */ + IPP_BADARG_RET(ctxSize=_IPP32E_K0) + AES_GCM_KEY_LEN(pState) = (Ipp64u)keyLen; + + Ipp8u zeroKey[32] = {0}; + const Ipp8u* pActualKey = pKey? pKey : zeroKey; + +#if (_AES_PROB_NOISE == _FEATURE_ON_) + /* Reset AES noise parameters */ + cpAESNoiseParams *params = (cpAESNoiseParams *)&AESGCM_NOISE_PARAMS(pState); + + AES_NOISE_RAND(params) = 0; + AES_NOISE_LEVEL(params) = 0; +#endif + if (IsFeatureEnabled(ippCPUID_AVX512VAES) && IsFeatureEnabled(ippCPUID_AVX512VCLMUL)) { + + switch AES_GCM_KEY_LEN(pState) { + case 16: + aes_keyexp_128_enc(pActualKey, &AES_GCM_KEY_DATA(pState)); + aes_gcm_precomp_128_vaes_avx512(&AES_GCM_KEY_DATA(pState)); + break; + case 24: + aes_keyexp_192_enc(pActualKey, &AES_GCM_KEY_DATA(pState)); + aes_gcm_precomp_192_vaes_avx512(&AES_GCM_KEY_DATA(pState)); + break; + case 32: + aes_keyexp_256_enc(pActualKey, &AES_GCM_KEY_DATA(pState)); + aes_gcm_precomp_256_vaes_avx512(&AES_GCM_KEY_DATA(pState)); + break; + } + } else { + + switch AES_GCM_KEY_LEN(pState) { + case 16: + aes_keyexp_128_enc(pActualKey, &AES_GCM_KEY_DATA(pState)); + aes_gcm_precomp_128_avx512(&AES_GCM_KEY_DATA(pState)); + break; + case 24: + aes_keyexp_192_enc(pActualKey, &AES_GCM_KEY_DATA(pState)); + aes_gcm_precomp_192_avx512(&AES_GCM_KEY_DATA(pState)); + break; + case 32: + aes_keyexp_256_enc(pActualKey, &AES_GCM_KEY_DATA(pState)); + aes_gcm_precomp_256_avx512(&AES_GCM_KEY_DATA(pState)); + break; + } + } + +#else + + /* init cipher */ + { + IppStatus sts = ippsAESInit(pKey, keyLen, AESGCM_CIPHER(pState), cpSizeofCtx_AES()); + if(ippStsNoErr!=sts) + return sts; + } + + /* precomputations (for constant multiplier(s)) */ + { + IppsAESSpec* pAES = AESGCM_CIPHER(pState); + RijnCipher encoder = RIJ_ENCODER(pAES); + + /* multiplier c = Enc({0}) */ + PadBlock(0, AESGCM_HKEY(pState), BLOCK_SIZE); + //encoder((Ipp32u*)AESGCM_HKEY(pState), (Ipp32u*)AESGCM_HKEY(pState), RIJ_NR(pAES), RIJ_EKEYS(pAES), (const Ipp32u (*)[256])RIJ_ENC_SBOX(pAES)); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder(AESGCM_HKEY(pState), AESGCM_HKEY(pState), RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder(AESGCM_HKEY(pState), AESGCM_HKEY(pState), RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif + } + + #if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + // the dead code that currently is unused + //#if(_IPP32E>=_IPP32E_K0) + //if (IsFeatureEnabled(ippCPUID_AVX512VAES)) { + // /* pre-compute hKey<<1, (hKey<<1)^2, (hKey<<1)^3, ... , (hKey<<1)^15 and corresponding + // Karatsuba constant multipliers for aggregated reduction */ + // AesGcmPrecompute_vaes(AESGCM_CPWR(pState), AESGCM_HKEY(pState)); + //} + //else + //#endif /* #if(_IPP32E>=_IPP32E_K0) */ + if(IsFeatureEnabled(ippCPUID_AES|ippCPUID_CLMUL) || IsFeatureEnabled(ippCPUID_AVX2VAES|ippCPUID_AVX2VCLMUL)) { + /* pre-compute reflect(hkey) and hKey<<1, (hKey<<1)^2 and (hKey<<1)^4 powers of hKey */ + AesGcmPrecompute_avx(AESGCM_CPWR(pState), AESGCM_HKEY(pState)); + } + else + #endif + AesGcmPrecompute_table2K(AES_GCM_MTBL(pState), AESGCM_HKEY(pState)); + #endif /* #if(_IPP32E>=_IPP32E_K0) */ + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmmul_vaes512.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmmul_vaes512.c new file mode 100644 index 0000000..8f5fa85 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmmul_vaes512.c @@ -0,0 +1,242 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES GFMUL and GHASH (GCM mode) +// +// Contents: +// +*/ + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(disable: 4206) // empty translation unit in MSVC +#endif + +#if 0 // Not used + +#include "owncp.h" +#include "pcpaesm.h" +#include "pcpaes_encrypt_vaes512.h" +#include "pcpaes_gcm_vaes512.h" +#include "pcpaesauthgcm.h" + +#if (_IPP32E>=_IPP32E_K1) +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(disable: 4310) // cast truncates constant value in MSVC +#endif + +/* The function calculates multiplication of two 2x128-bit polynomials C = A*B with + polynomial reduction. 2 polynomials can be processed at one call. + The inputs are bit-reflected. The result is bit-reflected. + */ +__INLINE void AesGcmGhash2(const __m256i* const src1, + const __m256i* const src2, + __m256i * const result) +{ + __m256i tmp1, tmp2, tmp3, tmp4; + +#if 0 + /* School-book multiplication */ + tmp1 = _mm256_clmulepi64_epi128(src2, src1, 0x00); // [C1:C0] + tmp2 = _mm256_clmulepi64_epi128(src2, src1, 0x11); // [D1:D0] + tmp3 = _mm256_clmulepi64_epi128(src2, src1, 0x10); // [F1:F0] + tmp4 = _mm256_clmulepi64_epi128(src2, src1, 0x01); // [E1:E0] + + tmp3 = _mm256_xor_si256(tmp4, tmp3); + + tmp4 = _mm256_bslli_epi128(tmp3, 8); + tmp3 = _mm256_bsrli_epi128(tmp3, 8); + tmp1 = _mm256_xor_si256(tmp4, tmp1); + tmp4 = _mm256_xor_si256(tmp3, tmp2); +#endif + + /* Karatsuba multiplication */ + tmp1 = _mm256_clmulepi64_epi128(*src2, *src1, 0x00); + tmp4 = _mm256_clmulepi64_epi128(*src2, *src1, 0x11); + + tmp2 = _mm256_shuffle_epi32(*src2, 78); // 78 = 01001110b + tmp3 = _mm256_shuffle_epi32(*src1, 78); // 78 = 01001110b + tmp2 = _mm256_xor_si256(tmp2, *src2); + tmp3 = _mm256_xor_si256(tmp3, *src1); + + tmp2 = _mm256_clmulepi64_epi128(tmp2, tmp3, 0x00); + tmp2 = _mm256_xor_si256(tmp2, tmp1); + tmp2 = _mm256_xor_si256(tmp2, tmp4); + + tmp3 = _mm256_bslli_epi128(tmp2, 8); + tmp2 = _mm256_bsrli_epi128(tmp2, 8); + + tmp1 = _mm256_xor_si256(tmp1, tmp3); + tmp4 = _mm256_xor_si256(tmp4, tmp2); + + /* Montgomery reduction */ + tmp2 = _mm256_clmulepi64_epi128(tmp1, M256(POLY2), 0x10); + tmp3 = _mm256_shuffle_epi32(tmp1, 78); // 78 = 01001110b + tmp1 = _mm256_xor_si256(tmp2, tmp3); + + tmp2 = _mm256_clmulepi64_epi128(tmp1, M256(POLY2), 0x10); + tmp3 = _mm256_shuffle_epi32(tmp1, 78); // 78 = 01001110b + tmp1 = _mm256_xor_si256(tmp2, tmp3); + + *result = _mm256_xor_si256(tmp1, tmp4); +} + +/* The function calculates multiplication of two 128-bit polynomials C = A*B with + polynomial reduction. + The inputs are bit-reflected. The result is bit-reflected. + */ +__INLINE void AesGcmGhash(const __m128i* const a, + const __m128i* const b, + __m128i * const result) +{ + __m256i res256; + + __m256i src1 = _mm256_set_m128i(_mm_setzero_si128(), *a); + __m256i src2 = _mm256_set_m128i(_mm_setzero_si128(), *b); + + AesGcmGhash2(&src1, &src2, &res256); + + *result = _mm256_castsi256_si128(res256); +} + +/* The function calculates multiplication of 128-bit polynomials C = A*B with + polynomial reduction. 4 polynomials can be processed at one call. + The inputs are bit-reflected. The result is bit-reflected. + */ +__INLINE void AesGcmGhash4(const __m512i* const src1, + const __m512i* const src2, + __m512i * const result) +{ + __m512i tmp1, tmp2, tmp3, tmp4; + + /* Karatsuba multiplication */ + tmp1 = _mm512_clmulepi64_epi128(*src2, *src1, 0x00); + tmp4 = _mm512_clmulepi64_epi128(*src2, *src1, 0x11); + + tmp2 = _mm512_shuffle_epi32(*src2, 78); // 78 = 01001110b + tmp3 = _mm512_shuffle_epi32(*src1, 78); // 78 = 01001110b + tmp2 = _mm512_xor_si512(tmp2, *src2); + tmp3 = _mm512_xor_si512(tmp3, *src1); + + tmp2 = _mm512_clmulepi64_epi128(tmp2, tmp3, 0x00); + tmp2 = _mm512_xor_si512(tmp2, tmp1); + tmp2 = _mm512_xor_si512(tmp2, tmp4); + + tmp3 = _mm512_bslli_epi128(tmp2, 8); + tmp2 = _mm512_bsrli_epi128(tmp2, 8); + + tmp1 = _mm512_xor_si512(tmp1, tmp3); + tmp4 = _mm512_xor_si512(tmp4, tmp2); + + /* Montgomery reduction */ + tmp2 = _mm512_clmulepi64_epi128(tmp1, M512(POLY2), 0x10); + tmp3 = _mm512_shuffle_epi32(tmp1, 78); // 78 = 01001110b + tmp1 = _mm512_xor_si512(tmp2, tmp3); + + tmp2 = _mm512_clmulepi64_epi128(tmp1, M512(POLY2), 0x10); + tmp3 = _mm512_shuffle_epi32(tmp1, 78); // 78 = 01001110b + tmp1 = _mm512_xor_si512(tmp2, tmp3); + + *result = _mm512_xor_si512(tmp1, tmp4); +} + +/* The function performs single A*(Hash<<1 mod poly) multiplication */ +IPP_OWN_DEFN (void, AesGcmMulGcm_vaes, (Ipp8u* pGHash, const Ipp8u* pHKey, const void * pParam)) +{ + IPP_UNREFERENCED_PARAMETER(pParam); + + __m128i ghash = _mm_maskz_loadu_epi64(0x03, pGHash); + __m128i hkey = _mm_maskz_loadu_epi64(0x03, pHKey + 16*3); // NB: hKey is at index 3 in the array + + ghash = _mm_maskz_shuffle_epi8((Ipp16u)(-1), ghash, M128(swapBytes)); + AesGcmGhash(&ghash, &hkey, &ghash); + ghash = _mm_maskz_shuffle_epi8((Ipp16u)(-1), ghash, M128(swapBytes)); + + _mm_mask_storeu_epi64(pGHash, (Ipp8u)(-1), ghash); +} + +/* The function computes reflected hKey<<1, hKey^2<<1, hKey^3<<1, ..., hKey^16<<1 - all mod poly + * to use in batch GHASH calculation. + * It also pre-computes parts of Karatsuba multipliers that are fixed and derived from the hKey. + */ +IPP_OWN_DEFN (void, AesGcmPrecompute_vaes, (Ipp8u* const pHtbl, const Ipp8u* const hKey)) +{ + /* Initial hKey = E_ctr(key, 0^16) */ + __m128i* pDst = (__m128i*)pHtbl; + + __m128i xmm0, xmm1; + __m256i ymm0, ymm1, ymm3, ymm4, ymm5; + __m512i zmm0, zmm1, zmm2; + + /* Load initial hKey = E_ctr(key,0^16) */ + ymm0 = _mm256_maskz_loadu_epi64(0x03, hKey); + /* Reflect hKey */ + ymm0 = _mm256_shuffle_epi8(ymm0, M256(swapBytes)); + + /* Compute reflected hKey<<1 mod poly */ + ymm3 = _mm256_srai_epi32(ymm0, 31); + ymm3 = _mm256_shuffle_epi32(ymm3, 0xFF); + ymm5 = _mm256_and_si256(ymm3, M256(POLY2)); + ymm3 = _mm256_srli_epi32(ymm0, 31); + ymm4 = _mm256_slli_epi32(ymm0, 1); + ymm3 = _mm256_bslli_epi128(ymm3, 4); + ymm0 = _mm256_xor_si256(ymm4, ymm3); + ymm0 = _mm256_xor_si256(ymm0,ymm5); + + xmm0 = _mm256_castsi256_si128(ymm0); + + AesGcmGhash(&xmm0, &xmm0, &xmm1); /* xmm1 = hKey^2 */ + + ymm0 = _mm256_broadcast_i64x2(xmm1); /* ymm0 = hKey^2 hKey^2 */ + ymm1 = _mm256_set_m128i(xmm0, xmm1); /* ymm1 = hKey hKey^2 */ + AesGcmGhash2(&ymm0, &ymm1, &ymm0); /* ymm0 = hKey^3 hKey^4 */ + + xmm1 = _mm256_extracti64x2_epi64(ymm0, 0); /* xmm1 = hKey^4 */ + + zmm0 = _mm512_setzero_si512(); + zmm0 = _mm512_inserti64x4(zmm0, ymm1, 1); /* zmm0 = hKey hKey^2 0 0 */ + zmm0 = _mm512_inserti64x4(zmm0, ymm0, 0); /* zmm0 = hKey hKey^2 hKey^3 hKey^4 */ + zmm1 = _mm512_broadcast_i64x2(xmm1); /* zmm1 = hKey^4 hKey^4 hKey^4 hKey^4 */ + + /* Store 4xhKey<<1 mod poly degrees in precompute table */ + _mm512_storeu_si512(pDst, zmm0); + + /* Prepare constant multipliers for Karatsuba (Bh^Bl) */ + zmm2 = _mm512_shuffle_epi32(zmm0, 78); + zmm2 = _mm512_xor_si512(zmm0, zmm2); + _mm512_storeu_si512(pDst + 16, zmm2); + + for (int i = 1; i < 4; i++) + { + AesGcmGhash4(&zmm0, &zmm1, &zmm0); + + /* Store 4xhKey<<1 mod poly degrees in precompute table */ + _mm512_storeu_si512(pDst + 4*i, zmm0); + + /* Prepare constant multipliers for Karatsuba (Bh^Bl) */ + zmm2 = _mm512_shuffle_epi32(zmm0, 78); + zmm2 = _mm512_xor_si512(zmm0, zmm2); + _mm512_storeu_si512(pDst + 4*i + 16, zmm2); + } +} + +#endif /* #if (_IPP32E>=_IPP32E_K1) */ + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmprocessaad.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmprocessaad.c new file mode 100644 index 0000000..f87fdea --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmprocessaad.c @@ -0,0 +1,224 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-GCM +// +// Contents: +// ippsAES_GCMProcessAAD() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + +#if(_IPP32E>=_IPP32E_K0) +#include "pcpaesauthgcm_avx512.h" +#else +#include "pcpaesauthgcm.h" +#endif /* #if(_IPP32E>=_IPP32E_K0) */ + +/*F* +// Name: ippsAES_GCMProcessAAD +// +// Purpose: AAD processing. +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL +// pAAD == NULL, aadLen>0 +// ippStsContextMatchErr !AESGCM_VALID_ID() +// ippStsLengthErr aadLen <0 +// ippStsBadArgErr illegal sequence call +// ippStsNoErr no errors +// +// Parameters: +// pAAD pointer to the AAD +// aadlen length of AAD (it could be 0) +// pState pointer to the context +// +*F*/ +IPPFUN(IppStatus, ippsAES_GCMProcessAAD,(const Ipp8u* pAAD, int aadLen, IppsAES_GCMState* pState)) +{ + /* test pState pointer */ + IPP_BAD_PTR1_RET(pState); + /* use aligned context */ + pState = (IppsAES_GCMState*)( IPP_ALIGNED_PTR(pState, AESGCM_ALIGNMENT) ); + /* test if context is valid */ + IPP_BADARG_RET(!AESGCM_VALID_ID(pState), ippStsContextMatchErr); + + /* test AAD pointer and length */ + IPP_BADARG_RET(aadLen && !pAAD, ippStsNullPtrErr); + IPP_BADARG_RET(aadLen<0, ippStsLengthErr); + + /* According to the NIST Special Publication 800-38D (Recommendation for GCM + * mode, p.5.2.1.1 Input Data) the AAD shall be between 0 and 2^64 bits. */ + IPP_BADARG_RET(((AESGCM_AAD_LEN(pState) + (Ipp64u)aadLen) < (Ipp64u)aadLen), ippStsScaleRangeErr); + + IPP_BADARG_RET(!(GcmIVprocessing==AESGCM_STATE(pState) || GcmAADprocessing==AESGCM_STATE(pState)), ippStsBadArgErr); + + if( GcmIVprocessing==AESGCM_STATE(pState) ) { + IPP_BADARG_RET(0==AESGCM_IV_LEN(pState), ippStsBadArgErr); + +#if(_IPP32E>=_IPP32E_K0) + IvFinalize_ ivHashFinalize = AES_GCM_IV_FINALIZE(pState); + + /* complete IV processing */ + ivHashFinalize(&AES_GCM_KEY_DATA(pState), &AES_GCM_CONTEXT_DATA(pState), + AESGCM_COUNTER(pState), (Ipp64u)AESGCM_BUFLEN(pState), AESGCM_IV_LEN(pState)); +#else + /* complete IV processing */ + if(CTR_POS==AESGCM_IV_LEN(pState)) { + /* apply special format if IV length is 12 bytes */ + AESGCM_COUNTER(pState)[12] = 0; + AESGCM_COUNTER(pState)[13] = 0; + AESGCM_COUNTER(pState)[14] = 0; + AESGCM_COUNTER(pState)[15] = 1; + } else { + /* get method */ + MulGcm_ hashFunc = AESGCM_HASH(pState); + + /* process the rest of IV */ + if(AESGCM_BUFLEN(pState)) + hashFunc(AESGCM_COUNTER(pState), AESGCM_HKEY(pState), AesGcmConst_table); + + /* add IV bit length */ + { + Ipp64u ivBitLen = AESGCM_IV_LEN(pState)*BYTESIZE; + Ipp8u tmp[BLOCK_SIZE]; + PadBlock(0, tmp, BLOCK_SIZE-8); + U32_TO_HSTRING(tmp+8, IPP_HIDWORD(ivBitLen)); + U32_TO_HSTRING(tmp+12, IPP_LODWORD(ivBitLen)); + XorBlock16(tmp, AESGCM_COUNTER(pState), AESGCM_COUNTER(pState)); + hashFunc(AESGCM_COUNTER(pState), AESGCM_HKEY(pState), AesGcmConst_table); + } + } + + /* prepare initial counter */ + { + IppsAESSpec* pAES = AESGCM_CIPHER(pState); + RijnCipher encoder = RIJ_ENCODER(pAES); + //encoder((Ipp32u*)AESGCM_COUNTER(pState), (Ipp32u*)AESGCM_ECOUNTER0(pState), RIJ_NR(pAES), RIJ_EKEYS(pAES), (const Ipp32u (*)[256])RIJ_ENC_SBOX(pAES)); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder(AESGCM_COUNTER(pState), AESGCM_ECOUNTER0(pState), RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder(AESGCM_COUNTER(pState), AESGCM_ECOUNTER0(pState), RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif + } +#endif /* #if(_IPP32E>=_IPP32E_K0) */ + + /* switch mode and init counters */ + AESGCM_STATE(pState) = GcmAADprocessing; + AESGCM_AAD_LEN(pState) = CONST_64(0); + AESGCM_BUFLEN(pState) = 0; + } + + /* + // AAD processing + */ + +#if(_IPP32E>=_IPP32E_K0) + /* test if buffer is not empty */ + if(AESGCM_BUFLEN(pState)) { + /* Cast to int here does not produce loss of data as AESGCM_BUFLEN <= BLOCK_SIZE, which is 16 bytes */ + int bufCapacity = BLOCK_SIZE-(int)AESGCM_BUFLEN(pState); + int locLen = IPP_MIN(aadLen, bufCapacity); + XorBlockMirror(pAAD, AESGCM_GHASH(pState), AESGCM_GHASH(pState), bufCapacity, locLen); + AESGCM_BUFLEN(pState) += (Ipp64u)locLen; + + /* if buffer full */ + if(BLOCK_SIZE==AESGCM_BUFLEN(pState)) { + MulGcm_ ghashFunc = AES_GCM_GMUL(pState); + ghashFunc(&AES_GCM_KEY_DATA(pState), AESGCM_GHASH(pState)); + AESGCM_BUFLEN(pState) = 0; + } + + AESGCM_AAD_LEN(pState) += (Ipp64u)locLen; + pAAD += locLen; + aadLen -= locLen; + } + + /* process main part of AAD */ + int lenBlks = aadLen & (-BLOCK_SIZE); + if(lenBlks) { + AadUpdate_ aadHashUpdate = AES_GCM_AAD_UPDATE(pState); + + aadHashUpdate(&AES_GCM_KEY_DATA(pState), &AES_GCM_CONTEXT_DATA(pState), pAAD, (Ipp64u)lenBlks); + + AESGCM_AAD_LEN(pState) += (Ipp64u)lenBlks; + pAAD += lenBlks; + aadLen -= lenBlks; + } + + /* copy the rest of AAD into the buffer */ + if(aadLen) { + /* Note: GHASH in the IPsec context is byte-reflected */ + XorBlockMirror(pAAD, AESGCM_GHASH(pState), AESGCM_GHASH(pState), BLOCK_SIZE, aadLen); + AESGCM_AAD_LEN(pState) += (Ipp64u)aadLen; + AESGCM_BUFLEN(pState) = (Ipp64u)aadLen; + } +#else + /* test if buffer is not empty */ + if(AESGCM_BUFLEN(pState)) { + int locLen = IPP_MIN(aadLen, BLOCK_SIZE-AESGCM_BUFLEN(pState)); + XorBlock(pAAD, AESGCM_GHASH(pState)+AESGCM_BUFLEN(pState), AESGCM_GHASH(pState)+AESGCM_BUFLEN(pState), locLen); + AESGCM_BUFLEN(pState) += locLen; + + /* if buffer full */ + if(BLOCK_SIZE==AESGCM_BUFLEN(pState)) { + MulGcm_ hashFunc = AESGCM_HASH(pState); + hashFunc(AESGCM_GHASH(pState), AESGCM_HKEY(pState), AesGcmConst_table); + AESGCM_BUFLEN(pState) = 0; + } + + AESGCM_AAD_LEN(pState) += (Ipp64u)locLen; + pAAD += locLen; + aadLen -= locLen; + } + + /* process main part of AAD */ + { + int lenBlks = aadLen & (-BLOCK_SIZE); + if(lenBlks) { + Auth_ authFunc = AESGCM_AUTH(pState); + + authFunc(AESGCM_GHASH(pState), pAAD, lenBlks, AESGCM_HKEY(pState), AesGcmConst_table); + + AESGCM_AAD_LEN(pState) += (Ipp64u)lenBlks; + pAAD += lenBlks; + aadLen -= lenBlks; + } + } + + /* copy the rest of AAD into the buffer */ + if(aadLen) { + XorBlock(pAAD, AESGCM_GHASH(pState), AESGCM_GHASH(pState), aadLen); + AESGCM_AAD_LEN(pState) += (Ipp64u)aadLen; + AESGCM_BUFLEN(pState) = aadLen; + } +#endif /* #if(_IPP32E>=_IPP32E_K0) */ + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmprocessiv.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmprocessiv.c new file mode 100644 index 0000000..03854d8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmprocessiv.c @@ -0,0 +1,158 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-GCM +// +// Contents: +// ippsAES_GCMProcessIV() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" + +#if(_IPP32E>=_IPP32E_K0) +#include "pcpaesauthgcm_avx512.h" +#else +#include "pcpaesauthgcm.h" +#endif /* #if(_IPP32E>=_IPP32E_K0) */ + +/*F* +// Name: ippsAES_GCMProcessIV +// +// Purpose: IV processing. +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL +// pIV ==NULL && ivLen>0 +// ippStsContextMatchErr !AESGCM_VALID_ID() +// ippStsLengthErr ivLen <0 +// ippStsBadArgErr illegal sequence call +// ippStsNoErr no errors +// +// Parameters: +// pIV pointer to the IV +// ivLen length of IV (it could be 0) +// pState pointer to the context +// +*F*/ +IPPFUN(IppStatus, ippsAES_GCMProcessIV,(const Ipp8u* pIV, int ivLen, IppsAES_GCMState* pState)) +{ + /* test pState pointer */ + IPP_BAD_PTR1_RET(pState); + + /* test IV pointer and length */ + IPP_BADARG_RET(ivLen && !pIV, ippStsNullPtrErr); + IPP_BADARG_RET(ivLen<0, ippStsLengthErr); + + /* use aligned context */ + pState = (IppsAES_GCMState*)( IPP_ALIGNED_PTR(pState, AESGCM_ALIGNMENT) ); + /* test context validity */ + IPP_BADARG_RET(!AESGCM_VALID_ID(pState), ippStsContextMatchErr); + + /* According to the NIST Special Publication 800-38D (Recommendation for GCM + * mode, p.5.2.1.1 Input Data) the IV shall be between 1 and 2^64 bits. */ + IPP_BADARG_RET(((AESGCM_IV_LEN(pState) + (Ipp64u)ivLen) < (Ipp64u)ivLen), ippStsScaleRangeErr); + + IPP_BADARG_RET(!(GcmInit==AESGCM_STATE(pState) || GcmIVprocessing==AESGCM_STATE(pState)), ippStsBadArgErr); + + /* switch IVprocessing on */ + AESGCM_STATE(pState) = GcmIVprocessing; + +#if(_IPP32E>=_IPP32E_K0) + IvUpdate_ ivHashUpdate = AES_GCM_IV_UPDATE(pState); + + /* test if buffer is not empty */ + if(AESGCM_BUFLEN(pState)) { + /* Cast to int here does not produce loss of data as AESGCM_BUFLEN <= BLOCK_SIZE, which is 16 bytes */ + int locLen = IPP_MIN(ivLen, BLOCK_SIZE-(int)AESGCM_BUFLEN(pState)); + CopyBlock((void*)pIV, (void*)(AESGCM_COUNTER(pState)+AESGCM_BUFLEN(pState)), locLen); + AESGCM_BUFLEN(pState) += (Ipp64u)locLen; + + /* if buffer full */ + if(BLOCK_SIZE==AESGCM_BUFLEN(pState)) { + ivHashUpdate(&AES_GCM_KEY_DATA(pState), &AES_GCM_CONTEXT_DATA(pState), AESGCM_COUNTER(pState), BLOCK_SIZE); + AESGCM_BUFLEN(pState) = 0; + } + + AESGCM_IV_LEN(pState) += (Ipp64u)locLen; + pIV += locLen; + ivLen -= locLen; + } + + /* process main part of IV */ + int lenBlks = ivLen & (-BLOCK_SIZE); + if(lenBlks) { + ivHashUpdate(&AES_GCM_KEY_DATA(pState), &AES_GCM_CONTEXT_DATA(pState), pIV, (Ipp64u)lenBlks); + AESGCM_IV_LEN(pState) += (Ipp64u)lenBlks; + pIV += lenBlks; + ivLen -= lenBlks; + } + + /* copy the rest of IV into the buffer */ + if(ivLen) { + CopyBlock((void*)pIV, (void*)(AESGCM_COUNTER(pState)), ivLen); + AESGCM_IV_LEN(pState) += (Ipp64u)ivLen; + AESGCM_BUFLEN(pState) = (Ipp64u)ivLen; + } +#else + /* test if buffer is not empty */ + if(AESGCM_BUFLEN(pState)) { + int locLen = IPP_MIN(ivLen, BLOCK_SIZE-AESGCM_BUFLEN(pState)); + XorBlock(pIV, AESGCM_COUNTER(pState)+AESGCM_BUFLEN(pState), AESGCM_COUNTER(pState)+AESGCM_BUFLEN(pState), locLen); + AESGCM_BUFLEN(pState) += locLen; + + /* if buffer full */ + if(BLOCK_SIZE==AESGCM_BUFLEN(pState)) { + MulGcm_ ghashFunc = AESGCM_HASH(pState); + ghashFunc(AESGCM_COUNTER(pState), AESGCM_HKEY(pState), AesGcmConst_table); + AESGCM_BUFLEN(pState) = 0; + } + + AESGCM_IV_LEN(pState) += (Ipp64u)locLen; + pIV += locLen; + ivLen -= locLen; + } + + /* process main part of IV */ + { + int lenBlks = ivLen & (-BLOCK_SIZE); + if(lenBlks) { + Auth_ authFunc = AESGCM_AUTH(pState); + authFunc(AESGCM_COUNTER(pState), pIV, lenBlks, AESGCM_HKEY(pState), AesGcmConst_table); + AESGCM_IV_LEN(pState) += (Ipp64u)lenBlks; + pIV += lenBlks; + ivLen -= lenBlks; + } + } + + /* copy the rest of IV into the buffer */ + if(ivLen) { + XorBlock(pIV, AESGCM_COUNTER(pState), AESGCM_COUNTER(pState), ivLen); + AESGCM_IV_LEN(pState) += (Ipp64u)ivLen; + AESGCM_BUFLEN(pState) += ivLen; + } +#endif /* #if(_IPP32E>=_IPP32E_K0) */ + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmreinit.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmreinit.c new file mode 100644 index 0000000..61206e5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmreinit.c @@ -0,0 +1,87 @@ +/******************************************************************************* +* Copyright (C) 2023 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-GCM functionality +// +// Contents: +// ippsAES_GCMReinit() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcpaesminit_internal.h" +#include "pcptool.h" + +#if (_IPP32E >= _IPP32E_K0) +#include "pcpaesauthgcm_avx512.h" +#else +#include "pcpaesauthgcm.h" +#endif /* #if(_IPP32E>=_IPP32E_K0) */ + +/*F* +// Name: ippsAES_GCMReinit +// +// Purpose: Re-init AES_GCM context for future usage - +// the state of the context left unchanged, just the internal stuff is updated +// (it's useful when the context physically lies in the same memory, but its +// virtual address changes - some pointers inside become stale and need to be re-initialized). +// Important note: this API shouldn't be used to re-initialize a context that was copied +// from some original context, computations in this case may be incorrect. +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL +// ippStsNoErr no errors +// +// Parameters: +// pState pointer to the AES-GCM context to be re-initialized +// +*F*/ +IPPFUN(IppStatus, ippsAES_GCMReinit, (IppsAES_GCMState * pState)) +{ + /* test pState pointer */ + IPP_BAD_PTR1_RET(pState); + + /* use aligned context */ + pState = (IppsAES_GCMState *)(IPP_ALIGNED_PTR(pState, AESGCM_ALIGNMENT)); + + /* set proper GCM context id */ + AESGCM_SET_ID(pState); + + Ipp64u keyByteLen; +/* re-init pointers inside the cipher context */ +#if (_IPP32E >= _IPP32E_K0) + keyByteLen = AES_GCM_KEY_LEN(pState); +#else + IppsAESSpec *pAesCtx = AESGCM_CIPHER(pState); + /* set proper cipher context id*/ + RIJ_SET_ID(pAesCtx); + keyByteLen = (Ipp64u)RIJ_NK(pAesCtx) * RIJ_BYTES_IN_WORD; + + cpAes_setup_ptrs_and_methods(pAesCtx); +#endif + + /* re-init pointers inside the AES-GCM context */ + cpAesGCM_setup_ptrs_and_methods(pState, keyByteLen); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmreset.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmreset.c new file mode 100644 index 0000000..835a857 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmreset.c @@ -0,0 +1,83 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-GCM +// +// Contents: +// ippsAES_GCMReset() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" + +#if(_IPP32E>=_IPP32E_K0) +#include "pcpaesauthgcm_avx512.h" +#else +#include "pcpaesauthgcm.h" +#endif /* #if(_IPP32E>=_IPP32E_K0) */ + +/*F* +// Name: ippsAES_GCMReset +// +// Purpose: Resets AES_GCM context. +// +// Returns: Reason: +// ippStsNullPtrErr pState== NULL +// ippStsContextMatchErr pState points on invalid context +// ippStsNoErr no errors +// +// Parameters: +// pState pointer to the context +// +*F*/ +IPPFUN(IppStatus, ippsAES_GCMReset,(IppsAES_GCMState* pState)) +{ + /* test pState pointer */ + IPP_BAD_PTR1_RET(pState); + + /* use aligned context */ + pState = (IppsAES_GCMState*)( IPP_ALIGNED_PTR(pState, AESGCM_ALIGNMENT) ); + /* test context validity */ + IPP_BADARG_RET(!AESGCM_VALID_ID(pState), ippStsContextMatchErr); + + /* reset GCM */ + AESGCM_STATE(pState) = GcmInit; + AESGCM_IV_LEN(pState) = CONST_64(0); + AESGCM_AAD_LEN(pState) = CONST_64(0); + AESGCM_TXT_LEN(pState) = CONST_64(0); + + AESGCM_BUFLEN(pState) = 0; + PadBlock(0, AESGCM_COUNTER(pState), BLOCK_SIZE); + PadBlock(0, AESGCM_ECOUNTER(pState), BLOCK_SIZE); + PadBlock(0, AESGCM_ECOUNTER0(pState), BLOCK_SIZE); + PadBlock(0, AESGCM_GHASH(pState), BLOCK_SIZE); + + #if(_IPP32E>=_IPP32E_K0) + + PadBlock(0, (void*)&AES_GCM_CONTEXT_DATA(pState), sizeof(struct gcm_context_data)); + + #endif /* #if(_IPP32E>=_IPP32E_K0) */ + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmstart.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmstart.c new file mode 100644 index 0000000..7b98f1c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_gcmstart.c @@ -0,0 +1,72 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-GCM +// +// Contents: +// ippsAES_GCMStart() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" + +#if(_IPP32E>=_IPP32E_K0) +#include "pcpaesauthgcm_avx512.h" +#else +#include "pcpaesauthgcm.h" +#endif /* #if(_IPP32E>=_IPP32E_K1) */ + +/*F* +// Name: ippsAES_GCMStart +// +// Purpose: Start the process of encryption or decryption and authentication tag generation. +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL +// pIV == NULL, ivLen>0 +// pAAD == NULL, aadLen>0 +// ippStsContextMatchErr !AESGCM_VALID_ID() +// ippStsLengthErr ivLen < 0 +// aadLen < 0 +// ippStsNoErr no errors +// +// Parameters: +// pIV pointer to the IV (nonce) +// ivLen length of the IV in bytes +// pAAD pointer to the Addition Authenticated Data (header) +// aadLen length of the AAD in bytes +// pState pointer to the AES-GCM state +// +*F*/ +IPPFUN(IppStatus, ippsAES_GCMStart,(const Ipp8u* pIV, int ivLen, + const Ipp8u* pAAD, int aadLen, + IppsAES_GCMState* pState)) +{ + IppStatus sts = ippsAES_GCMReset(pState); + if(ippStsNoErr==sts) + sts = ippsAES_GCMProcessIV(pIV, ivLen, pState); + if(ippStsNoErr==sts) + sts = ippsAES_GCMProcessAAD(pAAD, aadLen, pState); + return sts; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_keys_ni.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_keys_ni.h new file mode 100644 index 0000000..2bf119d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_keys_ni.h @@ -0,0 +1,51 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. AES keys expansion +// +// Contents: +// cpExpandAesKey_NI() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" + +#if !defined (_PCP_AES_KEYS_NI_H) +#define _PCP_AES_KEYS_NI_H + +#if (_AES_NI_ENABLING_==_FEATURE_ON_) || (_AES_NI_ENABLING_==_FEATURE_TICKTOCK_) + +////////////////////////////////////////////////////////////////////// +#define cpExpandAesKey_NI OWNAPI(cpExpandAesKey_NI) + IPP_OWN_DECL (void, cpExpandAesKey_NI, (const Ipp8u* pSecret, IppsAESSpec* pCtx)) +#define aes_DecKeyExpansion_NI OWNAPI(aes_DecKeyExpansion_NI) + IPP_OWN_DECL (void, aes_DecKeyExpansion_NI, (Ipp8u* decKeys, const Ipp8u* encKeys, int nr)) +#define aes128_KeyExpansion_NI OWNAPI(aes128_KeyExpansion_NI) + IPP_OWN_DECL (void, aes128_KeyExpansion_NI, (Ipp8u* keyExp, const Ipp8u* userkey)) +#define aes192_KeyExpansion_NI OWNAPI(aes192_KeyExpansion_NI) + IPP_OWN_DECL (void, aes192_KeyExpansion_NI, (Ipp8u* keyExp, const Ipp8u* userkey)) +#define aes256_KeyExpansion_NI OWNAPI(aes256_KeyExpansion_NI) + IPP_OWN_DECL (void, aes256_KeyExpansion_NI, (Ipp8u* keyExp, const Ipp8u* userkey)) + +#endif /* #if (_AES_NI_ENABLING_==_FEATURE_ON_) || (_AES_NI_ENABLING_==_FEATURE_TICKTOCK_) */ + +#endif /* #if !defined (_PCP_AES_KEYS_NI_H) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ofb.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ofb.h new file mode 100644 index 0000000..75ba3da --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_ofb.h @@ -0,0 +1,49 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption/decryption (OFB mode) +// +// Contents: +// cpProcessAES_ofb8() +// +*/ + +#include "owndefs.h" +#include "owncp.h" + +#if !defined(_PCP_AES_OFB_H) +#define _PCP_AES_OFB_H + +/* +// AES-OFB ecnryption/decryption +// +// Parameters: +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// dataLen input/output buffer length (in bytes) +// ofbBlkSize ofb block size (in bytes) +// pCtx pointer to the AES context +// pIV pointer to the initialization vector +*/ +#define cpProcessAES_ofb8 OWNAPI(cpProcessAES_ofb8) + IPP_OWN_DECL (void, cpProcessAES_ofb8, (const Ipp8u *pSrc, Ipp8u *pDst, int dataLen, int ofbBlkSize, const IppsAESSpec* pCtx, Ipp8u* pIV)) + +#endif /* #if !defined(_PCP_AES_OFB_H) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_process_ofb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_process_ofb8.c new file mode 100644 index 0000000..d2a84c8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_process_ofb8.c @@ -0,0 +1,95 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption/decryption (OFB mode) +// +// Contents: +// cpProcessAES_ofb8() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" +#include "pcpaes_ofb.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + + +/* +// AES-OFB ecnryption/decryption +// +// Parameters: +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// dataLen input/output buffer length (in bytes) +// ofbBlkSize ofb block size (in bytes) +// pCtx pointer to the AES context +// pIV pointer to the initialization vector +*/ +IPP_OWN_DEFN (void, cpProcessAES_ofb8, (const Ipp8u *pSrc, Ipp8u *pDst, int dataLen, int ofbBlkSize, const IppsAESSpec* pCtx, Ipp8u* pIV)) +{ + /* setup encoder method */ + RijnCipher encoder = RIJ_ENCODER(pCtx); + + Ipp32u tmpInpOut[2*NB(128)]; + + CopyBlock16(pIV, tmpInpOut); + + while(dataLen>=ofbBlkSize) { + /* block-by-block processing */ + //encoder(tmpInpOut, tmpInpOut+NB(128), RIJ_NR(pCtx), RIJ_EKEYS(pCtx), (const Ipp32u (*)[256])RIJ_ENC_SBOX(pCtx)); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder((Ipp8u*)tmpInpOut, (Ipp8u*)(tmpInpOut+NB(128)), RIJ_NR(pCtx), RIJ_EKEYS(pCtx), RijEncSbox/*NULL*/); + #else + encoder((Ipp8u*)tmpInpOut, (Ipp8u*)(tmpInpOut+NB(128)), RIJ_NR(pCtx), RIJ_EKEYS(pCtx), NULL); + #endif + + /* store output and shift inpBuffer for the next OFB operation */ + if(ofbBlkSize==MBS_RIJ128) { + ((Ipp32u*)pDst)[0] = tmpInpOut[0+NB(128)]^((Ipp32u*)pSrc)[0]; + ((Ipp32u*)pDst)[1] = tmpInpOut[1+NB(128)]^((Ipp32u*)pSrc)[1]; + ((Ipp32u*)pDst)[2] = tmpInpOut[2+NB(128)]^((Ipp32u*)pSrc)[2]; + ((Ipp32u*)pDst)[3] = tmpInpOut[3+NB(128)]^((Ipp32u*)pSrc)[3]; + tmpInpOut[0] = tmpInpOut[0+NB(128)]; + tmpInpOut[1] = tmpInpOut[1+NB(128)]; + tmpInpOut[2] = tmpInpOut[2+NB(128)]; + tmpInpOut[3] = tmpInpOut[3+NB(128)]; + } + else { + XorBlock(pSrc, tmpInpOut+NB(128), pDst, ofbBlkSize); + CopyBlock16((Ipp8u*)tmpInpOut+ofbBlkSize, tmpInpOut); + } + + pSrc += ofbBlkSize; + pDst += ofbBlkSize; + dataLen -= ofbBlkSize; + } + + /* update pIV */ + CopyBlock16((Ipp8u*)tmpInpOut, pIV); + + /* clear secret data */ + PurgeBlock(tmpInpOut, sizeof(tmpInpOut)); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_randomnoise.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_randomnoise.c new file mode 100644 index 0000000..a8a66a8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_randomnoise.c @@ -0,0 +1,120 @@ +/******************************************************************************* +* Copyright 2022 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +/* +// +// Purpose: +// AES noise function +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcpbnumisc.h" +#include "pcptool.h" +#include "pcpprng.h" + +/*F* +// Name: cpAESRandomNoise +// +// Purpose: AES Random Noise +// +// Returns: Reason: +// ippStsNullPtrErr pRndValue == NULL +// ippStsLengthErr 29 < nBits > 32 +// ippStsScaleRangeErr noiseRate > 1 +// (MISTLETOE3_MAX_CHUNK_SIZE > MISTLETOE3_TARGET_SIZE - invalid) +// ippStsNotSupportedModeErr Mistletoe3 mitigation isn't applicable for current CPU +// (no support of Intel® Advanced Encryption Standard New Instructions (Intel® AES-NI) or +// vector extensions of Intel® AES-NI) +// ippStsErr random bit sequence can't be generated +// ippStsNoErr no errors +// +// Parameters: +// rndFunc external random generator +// nBits number of bits that should be taken +// from generated 32-bit random value +// noiseRate probability of refreshing the random +// number pRndValue in the context +// pRndValue pointer to random number value from previous noise injection +// +*F*/ +IPP_OWN_DEFN(IppStatus, cpAESRandomNoise, (IppBitSupplier rndFunc, Ipp32u nBits, Ipp64f noiseRate, Ipp32u *pRndValue)) +{ +#if (_AES_PROB_NOISE == _FEATURE_ON_) + /* test that pRndValue pointer isn't NULL */ + IPP_BAD_PTR1_RET(pRndValue); + + /* test nBits and noiseRate values */ + IPP_BADARG_RET(nBits < 29 || nBits > 32, ippStsLengthErr); + IPP_BADARG_RET((noiseRate > 1), ippStsScaleRangeErr); + + Ipp32u rand = 0; + IppStatus status = ippStsNoErr; + + IppsPRNGState Ctx; + IppsPRNGState* pCtx = NULL; + + /* Use internal PRNG implementations if no external random source provided */ + if (NULL == rndFunc) { + if( IsFeatureEnabled(ippCPUID_RDRAND) ) + rndFunc = ippsPRNGenRDRAND; + else // RDRAND feature isn't available + { + pCtx = &Ctx; + ippsPRNGInit(160, pCtx); + rndFunc = ippsPRNGen; + } + } + + Ipp32u ctxRand = *pRndValue; + + /* Get the threshold to generate random noise */ + const Ipp32u randMax = (Ipp32u)(-1); + Ipp32u noiseThreshold = (Ipp32u)((Ipp64f)randMax * noiseRate); + + /* Get a random value, which is used to decide whether a new random should be + * generated. This allows generate random latency with probability */ + status = rndFunc(&rand, 32, pCtx); + + /* Check if new rand value required */ + if ((ippStsNoErr == status) && ((rand < noiseThreshold) || (ctxRand == 0))) { + status = rndFunc(&ctxRand, (int)nBits, pCtx); + if (ippStsNoErr == status) { + /* Write back a new random */ + *pRndValue = ctxRand; + } + } + + /* Scale down based on noise rate */ + rand = (Ipp32u)((Ipp64f)ctxRand * noiseRate); + + if (ippStsNoErr == status) { + _ippcpDelay(rand); + } + + return status; +#else + /* To remove MSVC warning C4100: 'XXX': unreferenced formal parameter*/ + IPP_UNREFERENCED_PARAMETER(rndFunc); + IPP_UNREFERENCED_PARAMETER(nBits); + IPP_UNREFERENCED_PARAMETER(noiseRate); + IPP_UNREFERENCED_PARAMETER(pRndValue); + + return ippStsNotSupportedModeErr; +#endif /* #if (_AES_PROB_NOISE == _FEATURE_ON_) */ +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_s2v_cmac.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_s2v_cmac.c new file mode 100644 index 0000000..2a34fb8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_s2v_cmac.c @@ -0,0 +1,107 @@ +/******************************************************************************* +* Copyright (C) 2015 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-SIV Functions (RFC 5297) +// +// Contents: +// ippsAES_S2V_CMAC() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpcmac.h" +#include "pcpaesm.h" +#include "pcptool.h" +#include "pcpaes_sivstuff.h" + +/*F* +// Name: ippsAES_S2V_CMAC +// +// Purpose: Converts strings to vector - +// performs S2V operation as defined RFC 5297. +// +// Returns: Reason: +// ippStsNullPtrErr pV == NULL +// pAD== NULL +// pADlen==NULL +// pADlen[i]!=0 && pAD[i]==0 +// ippStsLengthErr keyLen != 16 +// keyLen != 24 +// keyLen != 32 +// pADlen[i]<0 +// 0 > numAD +// ippStsNoErr no errors +// +// Parameters: +// pKey pointer to the secret key +// keyLen length of secret key +// pAD[] array of pointers to input strings +// pADlen[] array of input string lengths +// numAD number of pAD[] and pADlen[] terms +// pV pointer to output vector +// +*F*/ +IPPFUN(IppStatus, ippsAES_S2V_CMAC,(const Ipp8u* pKey, int keyLen, + const Ipp8u* pAD[], const int pADlen[], int numAD, + Ipp8u V[MBS_RIJ128])) +{ + /* test output vector */ + IPP_BAD_PTR1_RET(V); + + /* make sure that number of input string is legal */ + IPP_BADARG_RET(0>numAD, ippStsLengthErr); + + /* test arrays of input */ + IPP_BAD_PTR2_RET(pAD, pADlen); + + int n; + for(n=0; n 4 +// ippStsNotSupportedModeErr Mistletoe3 mitigation isn't applicable for current CPU +// (no support of Intel® Advanced Encryption Standard New Instructions (Intel® AES-NI) or +// vector extensions of Intel® AES-NI) +// ippStsNoErr no errors +// +// Parameters: +// noiseLevel the value of this parameter is directly +// proportional to the amount of noise injected +// Increasing noise level by 1 means the delay +// (performance impact) is doubled +// pCtx pointer to the AES context +// +*F*/ +IPPFUN(IppStatus, ippsAESSetupNoise, (Ipp32u noiseLevel, IppsAESSpec* pCtx)) +{ +#if (_AES_PROB_NOISE == _FEATURE_ON_) + /* test context */ + IPP_BAD_PTR1_RET(pCtx); + /* test the context ID */ + IPP_BADARG_RET(!VALID_AES_ID(pCtx), ippStsContextMatchErr); + + /* test noise level range */ + IPP_BADARG_RET(noiseLevel > 4, ippStsLengthErr); + + cpAESNoiseParams *params = (cpAESNoiseParams *)&RIJ_NOISE_PARAMS(pCtx); + + /* set up the parameters with initial values */ + AES_NOISE_RAND(params) = 0; + AES_NOISE_LEVEL(params) = noiseLevel; + + return ippStsNoErr; +#else + /* To remove MSVC warning C4100: 'XXX': unreferenced formal parameter*/ + IPP_UNREFERENCED_PARAMETER(noiseLevel); + IPP_UNREFERENCED_PARAMETER(pCtx); + return ippStsNotSupportedModeErr; +#endif +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_sivdecrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_sivdecrypt.c new file mode 100644 index 0000000..54e7541 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_sivdecrypt.c @@ -0,0 +1,140 @@ +/******************************************************************************* +* Copyright (C) 2015 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-SIV Functions (RFC 5297) +// +// Contents: +// ippsAES_SIVDecrypt() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpcmac.h" +#include "pcpaesm.h" +#include "pcptool.h" +#include "pcpaes_sivstuff.h" + +/*F* +// Name: ippsAES_SIVDecrypt +// +// Purpose: RFC 5297 authenticated decryption +// +// Returns: Reason: +// ippStsNullPtrErr pSrc==NULL, pDst==NULL +// pAuthPassed==NULL +// pAuthKey==NULL, pConfKey==NULL +// pAD== NULL, pADlen==NULL +// pADlen[i]!=0 && pAD[i]==0 +// pSIV == NULL +// ippStsLengthErr keyLen != 16 +// keyLen != 24 +// keyLen != 32 +// pADlen[i]<0 +// numAD<0 +// len<=0 +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to ciphertext +// pDst pointer to plaintext +// len length (in bytes) of plaintext/ciphertext +// pAuthPassed "authentication passed" flag +// pAuthKey pointer to the authentication key +// pConfKey pointer to the confidendat key +// keyLen length of keys +// pAD[] array of pointers to input strings +// pADlen[] array of input string lengths +// numAD number of pAD[] and pADlen[] terms +// pSIV pointer to input SIV +// +*F*/ +IPPFUN(IppStatus, ippsAES_SIVDecrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + int* pAuthPassed, + const Ipp8u* pAuthKey, const Ipp8u* pConfKey, int keyLen, + const Ipp8u* pAD[], const int pADlen[], int numAD, + const Ipp8u* pSIV)) +{ + /* test ciphertext, plaintex and length */ + IPP_BAD_PTR2_RET(pSrc, pDst); + IPP_BADARG_RET(0>=len, ippStsLengthErr); + + /* test keys & keyLen */ + IPP_BAD_PTR2_RET(pAuthKey, pConfKey); + IPP_BADARG_RET(keyLen!=16 && keyLen!=24 && keyLen!=32, ippStsLengthErr); + + /* test passed flag & auth vector */ + IPP_BAD_PTR2_RET(pAuthPassed, pSIV); + + /* test arrays of input AD[] */ + IPP_BAD_PTR2_RET(pAD, pADlen); + IPP_BADARG_RET(0>numAD, ippStsLengthErr); + + { + int n; + for (n = 0; n < numAD; n++) { + /* test input message and it's length */ + IPP_BADARG_RET((pADlen[n] < 0), ippStsLengthErr); + /* test source pointer */ + IPP_BADARG_RET((pADlen[n] && !pAD[n]), ippStsNullPtrErr); + } + } + + { + int n; + + /* iv an dmask */ + Ipp8u iv[MBS_RIJ128]; + Ipp8u vmask[MBS_RIJ128] = {0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0x7f,0xff,0xff,0xff,0x7f,0xff,0xff,0xff}; + /* AES context */ + Ipp8u aesBlob[sizeof(IppsAESSpec)]; + IppsAESSpec* paesCtx = (IppsAESSpec*)aesBlob; + + ippsAESInit(pConfKey, keyLen, paesCtx, sizeof(aesBlob)); + + /* construct iv */ + for(n=0; n=len, ippStsLengthErr); + + /* test keys & keyLen */ + IPP_BAD_PTR2_RET(pAuthKey, pConfKey); + IPP_BADARG_RET(keyLen!=16 && keyLen!=24 && keyLen!=32, ippStsLengthErr); + + /* test output vector */ + IPP_BAD_PTR1_RET(pSIV); + + /* test arrays of input AD[] */ + IPP_BAD_PTR2_RET(pAD, pADlen); + IPP_BADARG_RET(0>numAD, ippStsLengthErr); + + { + int n; + for (n = 0; n < numAD; n++) { + /* test input message and it's length */ + IPP_BADARG_RET((pADlen[n] < 0), ippStsLengthErr); + /* test source pointer */ + IPP_BADARG_RET((pADlen[n] && !pAD[n]), ippStsNullPtrErr); + } + } + + { + int n; + + /* iv and mask */ + Ipp8u iv[MBS_RIJ128]; + Ipp8u vmask[MBS_RIJ128] = {0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0x7f,0xff,0xff,0xff,0x7f,0xff,0xff,0xff}; + /* AES context */ + Ipp8u aesBlob[sizeof(IppsAESSpec)]; + IppsAESSpec* paesCtx = (IppsAESSpec*)aesBlob; + + { + Ipp8u ctxBlob[sizeof(IppsAES_CMACState)]; + IppsAES_CMACState* pCtx = (IppsAES_CMACState*)ctxBlob; + cpAES_S2V_init(pSIV, pAuthKey, keyLen, pCtx, sizeof(ctxBlob)); + + for(n=0; n>8) & 0xFF; + } + + out[MBS_RIJ128-1] ^= ((Ipp8u)(0-carry) & 0x87); + return out; +} +__INLINE void cpAES_S2V_update(Ipp8u v[MBS_RIJ128], const Ipp8u* pSrc, int len, IppsAES_CMACState* pCtx) +{ + Ipp8u t[MBS_RIJ128]; + cpAES_CMAC(t, pSrc, len, pCtx); + double16(v, v); + XorBlock16(v, t, v); +} + +static void cpAES_S2V_final(Ipp8u v[MBS_RIJ128], const Ipp8u* pSrc, int len, IppsAES_CMACState* pCtx) +{ + Ipp8u t[MBS_RIJ128]; + + if(len>=MBS_RIJ128) { + ippsAES_CMACUpdate(pSrc, len-MBS_RIJ128, pCtx); + XorBlock16(pSrc+len-MBS_RIJ128, v, t); + } + else { + double16(t, v); + XorBlock(pSrc, t, t, len); + t[len] ^= 0x80; + } + cpAES_CMAC(v, t, MBS_RIJ128, pCtx); +} + +#endif /*_PCP_AES_SIV_STUFF_H_*/ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_wrpaesgcmdec_avx.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_wrpaesgcmdec_avx.c new file mode 100644 index 0000000..2904b74 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_wrpaesgcmdec_avx.c @@ -0,0 +1,78 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-GCM +// +// Contents: +// wrpAesGcmDec_avx() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesauthgcm.h" +#include "pcpaesm.h" +#include "pcptool.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + + +#if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) +#if(_IPP32E<_IPP32E_K0) + +/*F* +// Name: ippsAES_GCMDecrypt +// +// Purpose: Decrypts a data buffer in the GCM mode. +// +// Returns: Reason: +// ippStsNullPtrErr pSrc == NULL +// pDst == NULL +// pState == NULL +// ippStsContextMatchErr !AESGCM_VALID_ID() +// ippStsLengthErr txtLen<0 +// ippStsNoErr no errors +// +// Parameters: +// pSrc Pointer to ciphertext. +// pDst Pointer to plaintext. +// len Length of the plaintext and ciphertext in bytes +// pState pointer to the context +// +*F*/ +IPP_OWN_DEFN (void, wrpAesGcmDec_avx, (Ipp8u* pDst, const Ipp8u* pSrc, int lenBlks, IppsAES_GCMState* pState)) +{ + IppsAESSpec* pAES = AESGCM_CIPHER(pState); + RijnCipher encoder = RIJ_ENCODER(pAES); + + AesGcmDec_avx(pDst, pSrc, lenBlks, + encoder, RIJ_NR(pAES), RIJ_EKEYS(pAES), + AESGCM_GHASH(pState), + AESGCM_COUNTER(pState), + AESGCM_ECOUNTER(pState), + AESGCM_HKEY(pState)); +} + +#endif /* (_IPP32E<_IPP32E_K0) */ +#endif /* #if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) */ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_wrpaesgcmenc_avx.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_wrpaesgcmenc_avx.c new file mode 100644 index 0000000..204bd0e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_wrpaesgcmenc_avx.c @@ -0,0 +1,59 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-GCM +// +// Contents: +// wrpAesGcmEnc_avx() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesauthgcm.h" +#include "pcpaesm.h" +#include "pcptool.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + + +#if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) +#if(_IPP32E<_IPP32E_K0) + +/* encrypts and authenticates n*BLOCK_SIZE bytes */ +IPP_OWN_DEFN (void, wrpAesGcmEnc_avx, (Ipp8u* pDst, const Ipp8u* pSrc, int lenBlks, IppsAES_GCMState* pState)) +{ + IppsAESSpec* pAES = AESGCM_CIPHER(pState); + RijnCipher encoder = RIJ_ENCODER(pAES); + + AesGcmEnc_avx(pDst, pSrc, lenBlks, + encoder, RIJ_NR(pAES), RIJ_EKEYS(pAES), + AESGCM_GHASH(pState), + AESGCM_COUNTER(pState), + AESGCM_ECOUNTER(pState), + AESGCM_HKEY(pState)); +} + +#endif /* (_IPP32E<_IPP32E_K0) */ +#endif /* #if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) */ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_xts_vaes512.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_xts_vaes512.c new file mode 100644 index 0000000..ee5a6f0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_xts_vaes512.c @@ -0,0 +1,506 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-XTS VAES512 Functions (IEEE P1619) +// +// Contents: +// cpAESEncryptXTS_VAES() +// cpAESDecryptXTS_VAES() +// +*/ + +#include "owncp.h" +#include "pcpaesmxts.h" +#include "pcptool.h" +#include "pcpaesmxtsstuff.h" + +#include "pcpaes_encrypt_vaes512.h" +#include "pcpaes_decrypt_vaes512.h" + +#if (_IPP32E>=_IPP32E_K1) +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(disable: 4310) // cast truncates constant value in MSVC +#endif + +#define M512(mem) (*((__m512i*)(mem))) + +/* Generate next 4 tweaks with 2^8 multiplier */ +__INLINE __m512i nextTweaks_x8(__m512i tweak128x4) +{ + const __m512i poly = _mm512_set_epi64(0, 0x87, 0, 0x87, 0, 0x87, 0, 0x87); + + __m512i highBytes = _mm512_bsrli_epi128(tweak128x4, 15); + __m512i tmp = _mm512_clmulepi64_epi128(highBytes, poly, 0); + tweak128x4 = _mm512_bslli_epi128(tweak128x4, 1); + tweak128x4 = _mm512_xor_si512(tweak128x4, tmp); + + return tweak128x4; +} + +/* Generate next 4 tweaks with 2^32 multiplier */ +__INLINE __m512i nextTweaks_x32(__m512i tweak128x4) +{ + const __m512i poly = _mm512_set_epi64(0, 0x87, 0, 0x87, 0, 0x87, 0, 0x87); + + /* Shift 128-bit lanes right by 12 bytes */ + __m512i highBytes = _mm512_bsrli_epi128(tweak128x4, 12); + + __m512i tmp = _mm512_clmulepi64_epi128(highBytes, poly, 0); + + /* Shift 128-bit lanes left by 4 bytes */ + tweak128x4 = _mm512_bslli_epi128(tweak128x4, 4); + + /* Xor low 4 bytes in each 128-bit lane with 0x87-modified ones */ + tweak128x4 = _mm512_xor_si512(tweak128x4, tmp); + + return tweak128x4; +} + +IPP_OWN_DEFN (void, cpAESEncryptXTS_VAES, (Ipp8u* outBlk, const Ipp8u* inpBlk, int nBlks, const Ipp8u* pRKey, int nr, Ipp8u* pTweak)) +{ + if (0 == nBlks) { + return; // do not modify tweak value + } + + int cipherRounds = nr - 1; + + __m128i* pRkey = (__m128i*)pRKey; + __m512i* pInp512 = (__m512i*)inpBlk; + __m512i* pOut512 = (__m512i*)outBlk; + + /* Produce initial 32 tweaks */ + __ALIGN64 Ipp8u tempTweakBuffer[AES_BLK_SIZE * 4 * 8]; // 32 tweaks + cpXTSwhitening(tempTweakBuffer, 8, pTweak); // generate 8 tweaks + + const __m512i* pInitialTweaks = (const __m512i*)tempTweakBuffer; + + int tailTweaksConsumedCount = 0; + + __m512i tweakBlk0 = M512(pInitialTweaks); + __m512i tweakBlk1 = M512(pInitialTweaks + 1); + __m512i tweakBlk2 = M512(pInitialTweaks + 2); + __m512i tweakBlk3 = M512(pInitialTweaks + 3); + __m512i tweakBlk4 = M512(pInitialTweaks + 4); + __m512i tweakBlk5 = M512(pInitialTweaks + 5); + __m512i tweakBlk6 = M512(pInitialTweaks + 6); + __m512i tweakBlk7 = M512(pInitialTweaks + 7); + + // generate other 24 tweaks + tweakBlk2 = nextTweaks_x8(tweakBlk0); + tweakBlk3 = nextTweaks_x8(tweakBlk1); + tweakBlk4 = nextTweaks_x8(tweakBlk2); + tweakBlk5 = nextTweaks_x8(tweakBlk3); + tweakBlk6 = nextTweaks_x8(tweakBlk4); + tweakBlk7 = nextTweaks_x8(tweakBlk5); + + int blocks; + for (blocks = nBlks; blocks >= (4 * 8); blocks -= (4 * 8)) { + __m512i blk0 = _mm512_loadu_si512(pInp512); + __m512i blk1 = _mm512_loadu_si512(pInp512 + 1); + __m512i blk2 = _mm512_loadu_si512(pInp512 + 2); + __m512i blk3 = _mm512_loadu_si512(pInp512 + 3); + __m512i blk4 = _mm512_loadu_si512(pInp512 + 4); + __m512i blk5 = _mm512_loadu_si512(pInp512 + 5); + __m512i blk6 = _mm512_loadu_si512(pInp512 + 6); + __m512i blk7 = _mm512_loadu_si512(pInp512 + 7); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + blk1 = _mm512_xor_epi64(tweakBlk1, blk1); + blk2 = _mm512_xor_epi64(tweakBlk2, blk2); + blk3 = _mm512_xor_epi64(tweakBlk3, blk3); + blk4 = _mm512_xor_epi64(tweakBlk4, blk4); + blk5 = _mm512_xor_epi64(tweakBlk5, blk5); + blk6 = _mm512_xor_epi64(tweakBlk6, blk6); + blk7 = _mm512_xor_epi64(tweakBlk7, blk7); + + cpAESEncrypt4_VAES_NI(&blk0, &blk1, &blk2, &blk3, pRkey, cipherRounds); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + blk1 = _mm512_xor_epi64(tweakBlk1, blk1); + blk2 = _mm512_xor_epi64(tweakBlk2, blk2); + blk3 = _mm512_xor_epi64(tweakBlk3, blk3); + + tweakBlk0 = nextTweaks_x32(tweakBlk0); + tweakBlk1 = nextTweaks_x32(tweakBlk1); + tweakBlk2 = nextTweaks_x32(tweakBlk2); + tweakBlk3 = nextTweaks_x32(tweakBlk3); + + cpAESEncrypt4_VAES_NI(&blk4, &blk5, &blk6, &blk7, pRkey, cipherRounds); + + blk4 = _mm512_xor_epi64(tweakBlk4, blk4); + blk5 = _mm512_xor_epi64(tweakBlk5, blk5); + blk6 = _mm512_xor_epi64(tweakBlk6, blk6); + blk7 = _mm512_xor_epi64(tweakBlk7, blk7); + + tweakBlk4 = nextTweaks_x32(tweakBlk4); + tweakBlk5 = nextTweaks_x32(tweakBlk5); + tweakBlk6 = nextTweaks_x32(tweakBlk6); + tweakBlk7 = nextTweaks_x32(tweakBlk7); + + _mm512_storeu_si512(pOut512, blk0); + _mm512_storeu_si512(pOut512 + 1, blk1); + _mm512_storeu_si512(pOut512 + 2, blk2); + _mm512_storeu_si512(pOut512 + 3, blk3); + _mm512_storeu_si512(pOut512 + 4, blk4); + _mm512_storeu_si512(pOut512 + 5, blk5); + _mm512_storeu_si512(pOut512 + 6, blk6); + _mm512_storeu_si512(pOut512 + 7, blk7); + + pInp512 += 8; + pOut512 += 8; + } + + if ((4 * 4) <= blocks) { + __m512i blk0 = _mm512_loadu_si512(pInp512); + __m512i blk1 = _mm512_loadu_si512(pInp512 + 1); + __m512i blk2 = _mm512_loadu_si512(pInp512 + 2); + __m512i blk3 = _mm512_loadu_si512(pInp512 + 3); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + blk1 = _mm512_xor_epi64(tweakBlk1, blk1); + blk2 = _mm512_xor_epi64(tweakBlk2, blk2); + blk3 = _mm512_xor_epi64(tweakBlk3, blk3); + + cpAESEncrypt4_VAES_NI(&blk0, &blk1, &blk2, &blk3, pRkey, cipherRounds); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + blk1 = _mm512_xor_epi64(tweakBlk1, blk1); + blk2 = _mm512_xor_epi64(tweakBlk2, blk2); + blk3 = _mm512_xor_epi64(tweakBlk3, blk3); + + _mm512_storeu_si512(pOut512, blk0); + _mm512_storeu_si512(pOut512 + 1, blk1); + _mm512_storeu_si512(pOut512 + 2, blk2); + _mm512_storeu_si512(pOut512 + 3, blk3); + + tailTweaksConsumedCount += 4; + pInp512 += 4; + pOut512 += 4; + blocks -= (4 * 4); + } + + if ((3 * 4) <= blocks) { + __m512i blk0 = _mm512_loadu_si512(pInp512); + __m512i blk1 = _mm512_loadu_si512(pInp512 + 1); + __m512i blk2 = _mm512_loadu_si512(pInp512 + 2); + + tweakBlk0 = M512(pInitialTweaks + tailTweaksConsumedCount); + tweakBlk1 = M512(pInitialTweaks + tailTweaksConsumedCount + 1); + tweakBlk2 = M512(pInitialTweaks + tailTweaksConsumedCount + 2); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + blk1 = _mm512_xor_epi64(tweakBlk1, blk1); + blk2 = _mm512_xor_epi64(tweakBlk2, blk2); + + cpAESEncrypt3_VAES_NI(&blk0, &blk1, &blk2, pRkey, cipherRounds); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + blk1 = _mm512_xor_epi64(tweakBlk1, blk1); + blk2 = _mm512_xor_epi64(tweakBlk2, blk2); + + _mm512_storeu_si512(pOut512, blk0); + _mm512_storeu_si512(pOut512 + 1, blk1); + _mm512_storeu_si512(pOut512 + 2, blk2); + + tailTweaksConsumedCount += 3; + pInp512 += 3; + pOut512 += 3; + blocks -= (3 * 4); + } + else if ((2 * 4) <= blocks) { + __m512i blk0 = _mm512_loadu_si512(pInp512); + __m512i blk1 = _mm512_loadu_si512(pInp512 + 1); + + tweakBlk0 = M512(pInitialTweaks + tailTweaksConsumedCount); + tweakBlk1 = M512(pInitialTweaks + tailTweaksConsumedCount + 1); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + blk1 = _mm512_xor_epi64(tweakBlk1, blk1); + + cpAESEncrypt2_VAES_NI(&blk0, &blk1, pRkey, cipherRounds); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + blk1 = _mm512_xor_epi64(tweakBlk1, blk1); + + _mm512_storeu_si512(pOut512, blk0); + _mm512_storeu_si512(pOut512 + 1, blk1); + + tailTweaksConsumedCount += 2; + pInp512 += 2; + pOut512 += 2; + blocks -= (2 * 4); + } + else if ((1 * 4) <= blocks) { + __m512i blk0 = _mm512_loadu_si512(pInp512); + + tweakBlk0 = M512(pInitialTweaks + tailTweaksConsumedCount); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + + cpAESEncrypt1_VAES_NI(&blk0, pRkey, cipherRounds); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + + _mm512_storeu_si512(pOut512, blk0); + + tailTweaksConsumedCount += 1; + pInp512 += 1; + pOut512 += 1; + blocks -= (1 * 4); + } + + if (blocks) { + __mmask8 k = (__mmask8)((1 << (blocks + blocks)) - 1); + __m512i blk0 = _mm512_maskz_loadu_epi64(k, pInp512); + + tweakBlk0 = M512(pInitialTweaks + tailTweaksConsumedCount); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + + cpAESEncrypt1_VAES_NI(&blk0, pRkey, cipherRounds); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + + _mm512_mask_storeu_epi64(pOut512, k, blk0); + } + + { + __mmask8 maskTweakToReturn = (__mmask8)(((Ipp8u)0x03u << (blocks << 1))); + _mm512_mask_compressstoreu_epi64(pTweak, maskTweakToReturn /* the first unused tweak */, tweakBlk0); + } + +} + +IPP_OWN_DEFN (void, cpAESDecryptXTS_VAES, (Ipp8u* outBlk, const Ipp8u* inpBlk, int nBlks, const Ipp8u* pRKey, int nr, Ipp8u* pTweak)) +{ + if (0 == nBlks) { + return; // do not modify tweak value + } + + int cipherRounds = nr - 1; + + __m128i* pRkey = (__m128i*)pRKey + cipherRounds + 1; + __m512i* pInp512 = (__m512i*)inpBlk; + __m512i* pOut512 = (__m512i*)outBlk; + + /* Produce initial 32 tweaks */ + __ALIGN64 Ipp8u tempTweakBuffer[AES_BLK_SIZE * 4 * 8]; // 32 tweaks + cpXTSwhitening(tempTweakBuffer, 8, pTweak); // generate 8 tweaks + + const __m512i* pInitialTweaks = (const __m512i*)tempTweakBuffer; + + int tailTweaksConsumedCount = 0; + + __m512i tweakBlk0 = M512(pInitialTweaks); + __m512i tweakBlk1 = M512(pInitialTweaks + 1); + __m512i tweakBlk2 = M512(pInitialTweaks + 2); + __m512i tweakBlk3 = M512(pInitialTweaks + 3); + __m512i tweakBlk4 = M512(pInitialTweaks + 4); + __m512i tweakBlk5 = M512(pInitialTweaks + 5); + __m512i tweakBlk6 = M512(pInitialTweaks + 6); + __m512i tweakBlk7 = M512(pInitialTweaks + 7); + + // generate other 24 tweaks + tweakBlk2 = nextTweaks_x8(tweakBlk0); + tweakBlk3 = nextTweaks_x8(tweakBlk1); + tweakBlk4 = nextTweaks_x8(tweakBlk2); + tweakBlk5 = nextTweaks_x8(tweakBlk3); + tweakBlk6 = nextTweaks_x8(tweakBlk4); + tweakBlk7 = nextTweaks_x8(tweakBlk5); + + int blocks; + for (blocks = nBlks; blocks >= (4 * 8); blocks -= (4 * 8)) { + __m512i blk0 = _mm512_loadu_si512(pInp512); + __m512i blk1 = _mm512_loadu_si512(pInp512 + 1); + __m512i blk2 = _mm512_loadu_si512(pInp512 + 2); + __m512i blk3 = _mm512_loadu_si512(pInp512 + 3); + __m512i blk4 = _mm512_loadu_si512(pInp512 + 4); + __m512i blk5 = _mm512_loadu_si512(pInp512 + 5); + __m512i blk6 = _mm512_loadu_si512(pInp512 + 6); + __m512i blk7 = _mm512_loadu_si512(pInp512 + 7); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + blk1 = _mm512_xor_epi64(tweakBlk1, blk1); + blk2 = _mm512_xor_epi64(tweakBlk2, blk2); + blk3 = _mm512_xor_epi64(tweakBlk3, blk3); + blk4 = _mm512_xor_epi64(tweakBlk4, blk4); + blk5 = _mm512_xor_epi64(tweakBlk5, blk5); + blk6 = _mm512_xor_epi64(tweakBlk6, blk6); + blk7 = _mm512_xor_epi64(tweakBlk7, blk7); + + cpAESDecrypt4_VAES_NI(&blk0, &blk1, &blk2, &blk3, pRkey, cipherRounds); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + blk1 = _mm512_xor_epi64(tweakBlk1, blk1); + blk2 = _mm512_xor_epi64(tweakBlk2, blk2); + blk3 = _mm512_xor_epi64(tweakBlk3, blk3); + + tweakBlk0 = nextTweaks_x32(tweakBlk0); + tweakBlk1 = nextTweaks_x32(tweakBlk1); + tweakBlk2 = nextTweaks_x32(tweakBlk2); + tweakBlk3 = nextTweaks_x32(tweakBlk3); + + cpAESDecrypt4_VAES_NI(&blk4, &blk5, &blk6, &blk7, pRkey, cipherRounds); + + blk4 = _mm512_xor_epi64(tweakBlk4, blk4); + blk5 = _mm512_xor_epi64(tweakBlk5, blk5); + blk6 = _mm512_xor_epi64(tweakBlk6, blk6); + blk7 = _mm512_xor_epi64(tweakBlk7, blk7); + + tweakBlk4 = nextTweaks_x32(tweakBlk4); + tweakBlk5 = nextTweaks_x32(tweakBlk5); + tweakBlk6 = nextTweaks_x32(tweakBlk6); + tweakBlk7 = nextTweaks_x32(tweakBlk7); + + _mm512_storeu_si512(pOut512, blk0); + _mm512_storeu_si512(pOut512 + 1, blk1); + _mm512_storeu_si512(pOut512 + 2, blk2); + _mm512_storeu_si512(pOut512 + 3, blk3); + _mm512_storeu_si512(pOut512 + 4, blk4); + _mm512_storeu_si512(pOut512 + 5, blk5); + _mm512_storeu_si512(pOut512 + 6, blk6); + _mm512_storeu_si512(pOut512 + 7, blk7); + + pInp512 += 8; + pOut512 += 8; + } + + if ((4 * 4) <= blocks) { + __m512i blk0 = _mm512_loadu_si512(pInp512); + __m512i blk1 = _mm512_loadu_si512(pInp512 + 1); + __m512i blk2 = _mm512_loadu_si512(pInp512 + 2); + __m512i blk3 = _mm512_loadu_si512(pInp512 + 3); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + blk1 = _mm512_xor_epi64(tweakBlk1, blk1); + blk2 = _mm512_xor_epi64(tweakBlk2, blk2); + blk3 = _mm512_xor_epi64(tweakBlk3, blk3); + + cpAESDecrypt4_VAES_NI(&blk0, &blk1, &blk2, &blk3, pRkey, cipherRounds); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + blk1 = _mm512_xor_epi64(tweakBlk1, blk1); + blk2 = _mm512_xor_epi64(tweakBlk2, blk2); + blk3 = _mm512_xor_epi64(tweakBlk3, blk3); + + _mm512_storeu_si512(pOut512, blk0); + _mm512_storeu_si512(pOut512 + 1, blk1); + _mm512_storeu_si512(pOut512 + 2, blk2); + _mm512_storeu_si512(pOut512 + 3, blk3); + + tailTweaksConsumedCount += 4; + pInp512 += 4; + pOut512 += 4; + blocks -= (4 * 4); + } + + if ((3 * 4) <= blocks) { + __m512i blk0 = _mm512_loadu_si512(pInp512); + __m512i blk1 = _mm512_loadu_si512(pInp512 + 1); + __m512i blk2 = _mm512_loadu_si512(pInp512 + 2); + + tweakBlk0 = M512(pInitialTweaks + tailTweaksConsumedCount); + tweakBlk1 = M512(pInitialTweaks + tailTweaksConsumedCount + 1); + tweakBlk2 = M512(pInitialTweaks + tailTweaksConsumedCount + 2); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + blk1 = _mm512_xor_epi64(tweakBlk1, blk1); + blk2 = _mm512_xor_epi64(tweakBlk2, blk2); + + cpAESDecrypt3_VAES_NI(&blk0, &blk1, &blk2, pRkey, cipherRounds); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + blk1 = _mm512_xor_epi64(tweakBlk1, blk1); + blk2 = _mm512_xor_epi64(tweakBlk2, blk2); + + _mm512_storeu_si512(pOut512, blk0); + _mm512_storeu_si512(pOut512 + 1, blk1); + _mm512_storeu_si512(pOut512 + 2, blk2); + + tailTweaksConsumedCount += 3; + pInp512 += 3; + pOut512 += 3; + blocks -= (3 * 4); + } + else if ((2 * 4) <= blocks) { + __m512i blk0 = _mm512_loadu_si512(pInp512); + __m512i blk1 = _mm512_loadu_si512(pInp512 + 1); + + tweakBlk0 = M512(pInitialTweaks + tailTweaksConsumedCount); + tweakBlk1 = M512(pInitialTweaks + tailTweaksConsumedCount + 1); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + blk1 = _mm512_xor_epi64(tweakBlk1, blk1); + + cpAESDecrypt2_VAES_NI(&blk0, &blk1, pRkey, cipherRounds); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + blk1 = _mm512_xor_epi64(tweakBlk1, blk1); + + _mm512_storeu_si512(pOut512, blk0); + _mm512_storeu_si512(pOut512 + 1, blk1); + + tailTweaksConsumedCount += 2; + pInp512 += 2; + pOut512 += 2; + blocks -= (2 * 4); + } + else if ((1 * 4) <= blocks) { + __m512i blk0 = _mm512_loadu_si512(pInp512); + + tweakBlk0 = M512(pInitialTweaks + tailTweaksConsumedCount); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + + cpAESDecrypt1_VAES_NI(&blk0, pRkey, cipherRounds); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + + _mm512_storeu_si512(pOut512, blk0); + + tailTweaksConsumedCount += 1; + pInp512 += 1; + pOut512 += 1; + blocks -= (1 * 4); + } + + if (blocks) { + __mmask8 k = (__mmask8)((1 << (blocks + blocks)) - 1); + __m512i blk0 = _mm512_maskz_loadu_epi64(k, pInp512); + + tweakBlk0 = M512(pInitialTweaks + tailTweaksConsumedCount); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + + cpAESDecrypt1_VAES_NI(&blk0, pRkey, cipherRounds); + + blk0 = _mm512_xor_epi64(tweakBlk0, blk0); + + _mm512_mask_storeu_epi64(pOut512, k, blk0); + } + + { + __mmask8 maskTweakToReturn = (__mmask8)(((Ipp8u)0x03u << (blocks << 1))); + _mm512_mask_compressstoreu_epi64(pTweak, maskTweakToReturn /* the first unused tweak */, tweakBlk0); + } +} + +#endif /* (_IPP32E>=_IPP32E_K1) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_xtsdecrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_xtsdecrypt.c new file mode 100644 index 0000000..a21aa72 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_xtsdecrypt.c @@ -0,0 +1,181 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-XTS Functions (IEEE P1619) +// +// Contents: +// ippsAES_XTSDecrypt() +// +*/ + +#include "owncp.h" +#include "pcpaesmxts.h" +#include "pcptool.h" +#include "pcpaesmxtsstuff.h" + + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + +// +// +static void cpAES_XTS_DecBlock(Ipp8u* ctxt, const Ipp8u* ptxt, const Ipp8u* tweak, const IppsAESSpec* pEncCtx) +{ + /* pre-whitening */ + XorBlock16(ptxt, tweak, ctxt); + /* encryption */ + ippsAESDecryptECB(ctxt, ctxt, AES_BLK_SIZE, pEncCtx); + /* post-whitening */ + XorBlock16(ctxt, tweak, ctxt); +} + +/*F* +// Name: ippsAES_XTSDecrypt +// +// Purpose: AES-XTS decryption. +// +// Returns: Reason: +// ippStsNullPtrErr pSrc == NULL +// pDst == NULL +// pTweak ==NULL +// pCtx == NULL +// ippStsLengthErr bitLen <128 +// ippStsContextMatchErr !VALID_AES_XTS_ID(pCtx) +// ippStsBadArgErr !IsLegalGeometry(startCipherBlkNo, +// bitLen, pCtx->duBitsize) +// ippStsNoErr no errors +// +// Parameters: +// pSrc points input buffer +// pDst points output buffer +// bitLen length of the input buffer in bits +// startCipherBlkNo number of the first block for data unit +// pTweak points tweak value +// pCtx points AES_XTS context +// +*F*/ + +IPPFUN(IppStatus, ippsAES_XTSDecrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int bitLen, + const IppsAES_XTSSpec* pCtx, + const Ipp8u* pTweak, + int startCipherBlkNo)) +{ + /* test pointers */ + IPP_BAD_PTR1_RET(pCtx); + /* test the context ID */ + IPP_BADARG_RET(!VALID_AES_XTS_ID(pCtx), ippStsContextMatchErr); + + /* test data pointers */ + IPP_BAD_PTR3_RET(pSrc, pDst, pTweak); + + /* test startCipherBlkNo and bitLen */ + IPP_BADARG_RET(bitLen < IPP_AES_BLOCK_BITSIZE, ippStsLengthErr); + IPP_BADARG_RET(!IsLegalGeometry(startCipherBlkNo, bitLen, pCtx->duBitsize), ippStsBadArgErr); + + { + __ALIGN16 Ipp8u tweakCT[AES_BLK_SIZE]; + + { /* encrypt tweak */ + const IppsAESSpec* ptwkAES = &pCtx->tweakAES; + + RijnCipher encoder = RIJ_ENCODER(ptwkAES); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder(pTweak, tweakCT, RIJ_NR(ptwkAES), RIJ_EKEYS(ptwkAES), RijEncSbox/*NULL*/); + #else + encoder(pTweak, tweakCT, RIJ_NR(ptwkAES), RIJ_EKEYS(ptwkAES), NULL); + #endif + + /* update tweakCT */ + for(; startCipherBlkNo>0; startCipherBlkNo--) + gf_mul_by_primitive(tweakCT); + } + + /* XTS decryption */ + { + const IppsAESSpec* pdatAES = &pCtx->datumAES; + + int encBlocks = bitLen/IPP_AES_BLOCK_BITSIZE; + int encBlocklast = bitLen%IPP_AES_BLOCK_BITSIZE; + if(encBlocklast) encBlocks--; + + /* decrypt data blocks */ + if( encBlocks>0) { + #if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + /* use Intel(R) AES New Instructions version if possible */ + if(AES_NI_ENABLED==RIJ_AESNI(pdatAES)) { + #if(_IPP32E>=_IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512VAES)) { + cpAESDecryptXTS_VAES(pDst, pSrc, encBlocks, RIJ_DKEYS(pdatAES), RIJ_NR(pdatAES), tweakCT); + } + else + #endif + cpAESDecryptXTS_AES_NI(pDst, pSrc, encBlocks, RIJ_DKEYS(pdatAES), RIJ_NR(pdatAES), tweakCT); + pSrc += encBlocks*AES_BLK_SIZE; + pDst += encBlocks*AES_BLK_SIZE; + } + else + #endif + { + for(; encBlocks>0; encBlocks--) { + cpAES_XTS_DecBlock(pDst, pSrc, tweakCT, pdatAES); + gf_mul_by_primitive(tweakCT); + pSrc += AES_BLK_SIZE; + pDst += AES_BLK_SIZE; + } + } + } + + /* "stealing" - decrypt last partial block if is */ + if(encBlocklast) { + int partBlockSize = encBlocklast/BYTESIZE; + + __ALIGN16 Ipp8u cc[AES_BLK_SIZE]; + __ALIGN16 Ipp8u pp[AES_BLK_SIZE]; + CopyBlock16(tweakCT, cc); + gf_mul_by_primitive(cc); + cpAES_XTS_DecBlock(pp, pSrc, cc, pdatAES); + + CopyBlock16(pp, cc); + CopyBlock(pSrc+AES_BLK_SIZE, cc, partBlockSize); + + encBlocklast %= BYTESIZE; + if(encBlocklast) { + Ipp8u partBlockMask = (Ipp8u)((0xFF)<<((BYTESIZE -encBlocklast) %BYTESIZE)); + Ipp8u x = pSrc[AES_BLK_SIZE+partBlockSize]; + Ipp8u y = cc[partBlockSize]; + x = (x & partBlockMask) | (y & ~partBlockMask); + cc[partBlockSize] = x; + pp[partBlockSize] &= partBlockMask; + partBlockSize++; + } + cpAES_XTS_DecBlock(pDst, cc, tweakCT, pdatAES); + + CopyBlock(pp, pDst+AES_BLK_SIZE, partBlockSize); + + /* clear secret data */ + PurgeBlock(pp, sizeof(pp)); + } + return ippStsNoErr; + } + } +} + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_xtsdecrypt_direct.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_xtsdecrypt_direct.c new file mode 100644 index 0000000..37df5d4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_xtsdecrypt_direct.c @@ -0,0 +1,229 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-XTS Functions (IEEE P1619) +// +// Contents: +// ippsAESDecryptXTS_Direct() +// +*/ + +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" +#include "pcpaesmxtsstuff.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + +#if !defined AES_BLK_SIZE +#define AES_BLK_SIZE (IPP_AES_BLOCK_BITSIZE/BITSIZE(Ipp8u)) +#endif + +#define AES_BLKS_PER_BUFFER (32) + + +/*F* +// Name: ippsAESDecryptXTS_Direct +// +// Purpose: AES-XTS decryption (see IEEE P1619-2007). +// +// Returns: Reason: +// ippStsNullPtrErr pSrc == NULL +// pDst == NULL +// pTweak ==NULL +// pKey ==NULL +// ippStsLengthErr dataUnitBitsize <128 +// keyBitsize != 256, !=512 +// ippStsBadArgErr aesBlkNo >= dataUnitBitsize/IPP_AES_BLOCK_BITSIZE +// ippStsNoErr no errors +// +// Parameters: +// pSrc points input buffer +// pDst points output buffer +// encBitsize lendth of the input/output buffer in bits +// aesBlkNo number of the first block for data unit +// ptweakPT points tweak value +// pKey pointer to the XTS key +// keyBitsize length of the key in bits +// dataUnitBitsize length of Data Unit in bits +// +*F*/ +static IppStatus cpAES_XTS_DecBlock(Ipp8u* ctxt, const Ipp8u* ptxt, const Ipp8u* tweak, const IppsAESSpec* pEncCtx) +{ + IppStatus sts; + /* pre-whitening */ + XorBlock16(ptxt, tweak, ctxt); + /* decryption */ + sts = ippsAESDecryptECB(ctxt, ctxt, AES_BLK_SIZE, pEncCtx); + /* post-whitening */ + XorBlock16(ctxt, tweak, ctxt); + return sts; +} + +// +// To Do: advance parameter check!! +// +IPPFUN(IppStatus, ippsAESDecryptXTS_Direct,(const Ipp8u* pSrc, Ipp8u* pDst, int encBitsize, int aesBlkNo, + const Ipp8u* pTweakPT, + const Ipp8u* pKey, int keyBitsize, + int dataUnitBitsize)) +{ + /* test dataUnitBitsize */ + IPP_BADARG_RET(dataUnitBitsize (1<<27), ippStsBadArgErr); + + /* test dataUnitBitsize and aesBlkNo */ + IPP_BADARG_RET(((dataUnitBitsize/IPP_AES_BLOCK_BITSIZE)<=aesBlkNo) || (0>aesBlkNo), ippStsBadArgErr); + + { + IppStatus sts = ippStsNoErr; + + int keySize = keyBitsize/2/8; + const Ipp8u* pConfKey = pKey; + const Ipp8u* pTweakKey = pKey+keySize; + + do { + int encBlocks = encBitsize/IPP_AES_BLOCK_BITSIZE; + int encBlocklast = encBitsize%IPP_AES_BLOCK_BITSIZE; + + __ALIGN16 IppsAESSpec aesCtx; + __ALIGN16 Ipp8u tweakCT[AES_BLK_SIZE]; + __ALIGN16 Ipp8u tmp[AES_BLKS_PER_BUFFER*AES_BLK_SIZE]; + + sts = ippsAESInit(pTweakKey, keySize, &aesCtx, sizeof(aesCtx)); + if(ippStsNoErr!=sts) break; + + { + RijnCipher encoder = RIJ_ENCODER(&aesCtx); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder(pTweakPT, tweakCT, RIJ_NR(&aesCtx), RIJ_EKEYS(&aesCtx), RijEncSbox/*NULL*/); + #else + encoder(pTweakPT, tweakCT, RIJ_NR(&aesCtx), RIJ_EKEYS(&aesCtx), NULL); + #endif + } + + sts = ippsAESInit(pConfKey, keySize, &aesCtx, sizeof(aesCtx)); + if(ippStsNoErr!=sts) break; + + for(; aesBlkNo>0; aesBlkNo--) + gf_mul_by_primitive(tweakCT); + + if(encBlocklast) encBlocks--; + + /* + // decrypt data + */ + #if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + /* use Intel(R) AES New Instructions version if possible */ + if(AES_NI_ENABLED==RIJ_AESNI(&aesCtx)) { + #if(_IPP32E>=_IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512VAES)) { + cpAESDecryptXTS_VAES(pDst, pSrc, encBlocks, RIJ_DKEYS(&aesCtx), RIJ_NR(&aesCtx), tweakCT); + } + else + #endif + cpAESDecryptXTS_AES_NI(pDst, pSrc, encBlocks, RIJ_DKEYS(&aesCtx), RIJ_NR(&aesCtx), tweakCT); + pSrc += encBlocks*AES_BLK_SIZE; + pDst += encBlocks*AES_BLK_SIZE; + } + else + #endif + { + for(; encBlocks>=AES_BLKS_PER_BUFFER && ippStsNoErr==sts; encBlocks-=AES_BLKS_PER_BUFFER) { + /* compute whitening tweaks */ + cpXTSwhitening(tmp, AES_BLKS_PER_BUFFER, tweakCT); + /* pre-whitening */ + cpXTSxor16(pDst, pSrc, tmp, AES_BLKS_PER_BUFFER); + + sts = ippsAESDecryptECB(pDst, pDst, AES_BLKS_PER_BUFFER*AES_BLK_SIZE, &aesCtx); + + /* post-whitening */ + cpXTSxor16(pDst, pDst, tmp, AES_BLKS_PER_BUFFER); + + pSrc += AES_BLKS_PER_BUFFER*AES_BLK_SIZE; + pDst += AES_BLKS_PER_BUFFER*AES_BLK_SIZE; + } + if(ippStsNoErr!=sts) break; + + if(encBlocks) { + cpXTSwhitening(tmp, encBlocks, tweakCT); + + /* pre-whitening */ + cpXTSxor16(pDst, pSrc, tmp, encBlocks); + + ippsAESDecryptECB(pDst, pDst, AES_BLK_SIZE*encBlocks, &aesCtx); + + /* post-whitening */ + cpXTSxor16(pDst, pDst, tmp, encBlocks); + + pSrc += AES_BLK_SIZE*encBlocks; + pDst += AES_BLK_SIZE*encBlocks; + } + } + + /* "stealing" - encrypt last partial block if is */ + if(encBlocklast) { + int partBlockSize = encBlocklast/BYTESIZE; + + __ALIGN16 Ipp8u cc[AES_BLK_SIZE]; + __ALIGN16 Ipp8u pp[AES_BLK_SIZE]; + CopyBlock16(tweakCT, cc); + gf_mul_by_primitive(cc); + cpAES_XTS_DecBlock(pp, pSrc, cc, &aesCtx); + + CopyBlock16(pp, cc); + CopyBlock(pSrc+AES_BLK_SIZE, cc, partBlockSize); + + encBlocklast %= BYTESIZE; + if(encBlocklast) { + Ipp8u partBlockMask = (Ipp8u)((0xFF)<<((BYTESIZE -encBlocklast) %BYTESIZE)); + Ipp8u x = pSrc[AES_BLK_SIZE+partBlockSize]; + Ipp8u y = cc[partBlockSize]; + x = (x & partBlockMask) | (y & ~partBlockMask); + cc[partBlockSize] = x; + pp[partBlockSize] &= partBlockMask; + partBlockSize++; + } + cpAES_XTS_DecBlock(pDst, cc, tweakCT, &aesCtx); + + CopyBlock(pp, pDst+AES_BLK_SIZE, partBlockSize); + + /* clear secret data */ + PurgeBlock(pp, sizeof(pp)); + } + + } while(0); + + return sts; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_xtsencrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_xtsencrypt.c new file mode 100644 index 0000000..5c45f21 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_xtsencrypt.c @@ -0,0 +1,175 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-XTS Functions (IEEE P1619) +// +// Contents: +// ippsAES_XTSEncrypt() +// +*/ + +#include "owncp.h" +#include "pcpaesmxts.h" +#include "pcptool.h" +#include "pcpaesmxtsstuff.h" + + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + +static void cpAES_XTS_EncBlock(Ipp8u* ctxt, const Ipp8u* ptxt, const Ipp8u* tweak, const IppsAESSpec* pEncCtx) +{ + /* pre-whitening */ + XorBlock16(ptxt, tweak, ctxt); + /* encryption */ + ippsAESEncryptECB(ctxt, ctxt, AES_BLK_SIZE, pEncCtx); + /* post-whitening */ + XorBlock16(ctxt, tweak, ctxt); +} + +/*F* +// Name: ippsAES_XTSEncrypt +// +// Purpose: AES-XTS encryption. +// +// Returns: Reason: +// ippStsNullPtrErr pSrc == NULL +// pDst == NULL +// pTweak ==NULL +// pCtx == NULL +// ippStsLengthErr bitLen <128 +// ippStsContextMatchErr !VALID_AES_XTS_ID(pCtx) +// ippStsBadArgErr !IsLegalGeometry(startCipherBlkNo, +// bitLen, pCtx->duBitsize) +// ippStsNoErr no errors +// +// Parameters: +// pSrc points input buffer +// pDst points output buffer +// bitLen length of the input buffer in bits +// startCipherBlkNo number of the first block for data unit +// pTweak points tweak value +// pCtx points AES_XTS context +// +*F*/ + +IPPFUN(IppStatus, ippsAES_XTSEncrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int bitLen, + const IppsAES_XTSSpec* pCtx, + const Ipp8u* pTweak, + int startCipherBlkNo)) +{ + /* test pointers */ + IPP_BAD_PTR1_RET(pCtx); + /* test the context ID */ + IPP_BADARG_RET(!VALID_AES_XTS_ID(pCtx), ippStsContextMatchErr); + + /* test data pointers */ + IPP_BAD_PTR3_RET(pSrc, pDst, pTweak); + + /* test startCipherBlkNo and bitLen */ + IPP_BADARG_RET(bitLen < IPP_AES_BLOCK_BITSIZE, ippStsLengthErr); + IPP_BADARG_RET(!IsLegalGeometry(startCipherBlkNo, bitLen, pCtx->duBitsize), ippStsBadArgErr); + + { + __ALIGN16 Ipp8u tweakCT[AES_BLK_SIZE]; + + { /* encrypt tweak */ + const IppsAESSpec* ptwkAES = &pCtx->tweakAES; + + RijnCipher encoder = RIJ_ENCODER(ptwkAES); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder(pTweak, tweakCT, RIJ_NR(ptwkAES), RIJ_EKEYS(ptwkAES), RijEncSbox/*NULL*/); + #else + encoder(pTweak, tweakCT, RIJ_NR(ptwkAES), RIJ_EKEYS(ptwkAES), NULL); + #endif + + /* update tweakCT */ + for(; startCipherBlkNo>0; startCipherBlkNo--) + gf_mul_by_primitive(tweakCT); + } + + /* XTS encryption */ + { + const IppsAESSpec* pdatAES = &pCtx->datumAES; + + int encBlocks = bitLen/IPP_AES_BLOCK_BITSIZE; + int encBlocklast = bitLen%IPP_AES_BLOCK_BITSIZE; + if(encBlocklast) encBlocks--; + + /* encrypt data blocks */ + if( encBlocks>0) { + #if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + /* use Intel(R) AES New Instructions version if possible */ + if(AES_NI_ENABLED==RIJ_AESNI(pdatAES)) { + #if(_IPP32E>=_IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512VAES)) { + cpAESEncryptXTS_VAES(pDst, pSrc, encBlocks, RIJ_EKEYS(pdatAES), RIJ_NR(pdatAES), tweakCT); + } + else + #endif + cpAESEncryptXTS_AES_NI(pDst, pSrc, encBlocks, RIJ_EKEYS(pdatAES), RIJ_NR(pdatAES), tweakCT); + pSrc += encBlocks*AES_BLK_SIZE; + pDst += encBlocks*AES_BLK_SIZE; + } + else + #endif + { + for(; encBlocks>0; encBlocks--) { + cpAES_XTS_EncBlock(pDst, pSrc, tweakCT, pdatAES); + gf_mul_by_primitive(tweakCT); + pSrc += AES_BLK_SIZE; + pDst += AES_BLK_SIZE; + } + } + } + + /* "stealing" - encrypt last partial block if is */ + if(encBlocklast) { + int partBlockSize = encBlocklast/BYTESIZE; + + __ALIGN16 Ipp8u cc[AES_BLK_SIZE]; + __ALIGN16 Ipp8u pp[AES_BLK_SIZE]; + cpAES_XTS_EncBlock(cc, pSrc, tweakCT, pdatAES); + gf_mul_by_primitive(tweakCT); + + CopyBlock16(cc, pp); + CopyBlock(pSrc+AES_BLK_SIZE, pp, partBlockSize); + + encBlocklast %= BYTESIZE; + if(encBlocklast) { + Ipp8u partBlockMask = (Ipp8u)((0xFF)<<((BYTESIZE -encBlocklast) %BYTESIZE)); + Ipp8u x = pSrc[AES_BLK_SIZE+partBlockSize]; + Ipp8u y = cc[partBlockSize]; + x = (x & partBlockMask) | (y & ~partBlockMask); + pp[partBlockSize] = x; + cc[partBlockSize] &= partBlockMask; + partBlockSize++; + } + cpAES_XTS_EncBlock(pDst, pp, tweakCT, pdatAES); + + CopyBlock(cc, pDst+AES_BLK_SIZE, partBlockSize); + } + return ippStsNoErr; + } + } +} + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_xtsencrypt_direct.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_xtsencrypt_direct.c new file mode 100644 index 0000000..d5b5cc1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_xtsencrypt_direct.c @@ -0,0 +1,224 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-XTS Functions (IEEE P1619) +// +// Contents: +// ippsAESEncryptXTS_Direct() +// +*/ + +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" +#include "pcpaesmxtsstuff.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + +#if !defined AES_BLK_SIZE +#define AES_BLK_SIZE (IPP_AES_BLOCK_BITSIZE/BITSIZE(Ipp8u)) +#endif + +#define AES_BLKS_PER_BUFFER (32) + + +/*F* +// Name: ippsAESEncryptXTS_Direct +// +// Purpose: AES-XTS encryption (see IEEE P1619-2007). +// +// Returns: Reason: +// ippStsNullPtrErr pSrc == NULL +// pDst == NULL +// pTweak ==NULL +// pKey ==NULL +// ippStsLengthErr dataUnitBitsize <128 +// keyBitsize != 256, !=512 +// ippStsBadArgErr aesBlkNo >= dataUnitBitsize/IPP_AES_BLOCK_BITSIZE +// aesBlkNo < 0 +// ippStsNoErr no errors +// +// Parameters: +// pSrc points input buffer +// pDst points output buffer +// encBitsize length of the input/output buffer in bits +// aesBlkNo number of the first block for data unit +// ptweakPT points tweak value +// pKey pointer to the XTS key +// keyBitsize length of the key in bits +// dataUnitBitsize length of Data Unit in bits +// +*F*/ +static IppStatus cpAES_XTS_EncBlock(Ipp8u* ctxt, const Ipp8u* ptxt, const Ipp8u* tweak, const IppsAESSpec* pEncCtx) +{ + IppStatus sts; + /* pre-whitening */ + XorBlock16(ptxt, tweak, ctxt); + /* encryption */ + sts = ippsAESEncryptECB(ctxt, ctxt, AES_BLK_SIZE, pEncCtx); + /* post-whitening */ + XorBlock16(ctxt, tweak, ctxt); + return sts; +} + +IPPFUN(IppStatus, ippsAESEncryptXTS_Direct,(const Ipp8u* pSrc, Ipp8u* pDst, int encBitsize, int aesBlkNo, + const Ipp8u* pTweakPT, + const Ipp8u* pKey, int keyBitsize, + int dataUnitBitsize)) +{ + /* test dataUnitBitsize */ + IPP_BADARG_RET(dataUnitBitsize (1<<27), ippStsBadArgErr); + + /* test dataUnitBitsize and aesBlkNo */ + IPP_BADARG_RET(((dataUnitBitsize/IPP_AES_BLOCK_BITSIZE)<=aesBlkNo) || (0>aesBlkNo), ippStsBadArgErr); + + { + IppStatus sts = ippStsNoErr; + + int keySize = keyBitsize/2/8; + const Ipp8u* pConfKey = pKey; + const Ipp8u* pTweakKey = pKey+keySize; + + do { + int encBlocks = encBitsize/IPP_AES_BLOCK_BITSIZE; + int encBlocklast = encBitsize%IPP_AES_BLOCK_BITSIZE; + + __ALIGN16 IppsAESSpec aesCtx; + __ALIGN16 Ipp8u tweakCT[AES_BLK_SIZE]; + __ALIGN16 Ipp8u tmp[AES_BLKS_PER_BUFFER*AES_BLK_SIZE]; + __ALIGN16 Ipp8u tmpDst[AES_BLKS_PER_BUFFER*AES_BLK_SIZE]; + + sts = ippsAESInit(pTweakKey, keySize, &aesCtx, sizeof(aesCtx)); + if(ippStsNoErr!=sts) break; + + { + RijnCipher encoder = RIJ_ENCODER(&aesCtx); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder(pTweakPT, tweakCT, RIJ_NR(&aesCtx), RIJ_EKEYS(&aesCtx), RijEncSbox/*NULL*/); + #else + encoder(pTweakPT, tweakCT, RIJ_NR(&aesCtx), RIJ_EKEYS(&aesCtx), NULL); + #endif + } + + sts = ippsAESInit(pConfKey, keySize, &aesCtx, sizeof(aesCtx)); + if(ippStsNoErr!=sts) break; + + for(; aesBlkNo>0; aesBlkNo--) + gf_mul_by_primitive(tweakCT); + + if(encBlocklast) encBlocks--; + + /* + // encrypt data + */ + #if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + /* use Intel(R) AES New Instructions version if possible */ + if(AES_NI_ENABLED==RIJ_AESNI(&aesCtx)) { + #if(_IPP32E>=_IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512VAES)) { + cpAESEncryptXTS_VAES(pDst, pSrc, encBlocks, RIJ_EKEYS(&aesCtx), RIJ_NR(&aesCtx), tweakCT); + } + else + #endif + cpAESEncryptXTS_AES_NI(pDst, pSrc, encBlocks, RIJ_EKEYS(&aesCtx), RIJ_NR(&aesCtx), tweakCT); + pSrc += encBlocks*AES_BLK_SIZE; + pDst += encBlocks*AES_BLK_SIZE; + } + else + #endif + { + for(; encBlocks>=AES_BLKS_PER_BUFFER && ippStsNoErr==sts; encBlocks-=AES_BLKS_PER_BUFFER) { + /* compute whitening tweaks */ + cpXTSwhitening(tmp, AES_BLKS_PER_BUFFER, tweakCT); + /* pre-whitening */ + cpXTSxor16(tmpDst, pSrc, tmp, AES_BLKS_PER_BUFFER); + + sts = ippsAESEncryptECB(tmpDst, pDst, AES_BLKS_PER_BUFFER*AES_BLK_SIZE, &aesCtx); + + /* post-whitening */ + cpXTSxor16(pDst, pDst, tmp, AES_BLKS_PER_BUFFER); + + pSrc += AES_BLKS_PER_BUFFER*AES_BLK_SIZE; + pDst += AES_BLKS_PER_BUFFER*AES_BLK_SIZE; + } + if(ippStsNoErr!=sts) break; + + if(encBlocks) { + cpXTSwhitening(tmp, encBlocks, tweakCT); + + /* pre-whitening */ + cpXTSxor16(tmpDst, pSrc, tmp, encBlocks); + + ippsAESEncryptECB(tmpDst, pDst, AES_BLK_SIZE*encBlocks, &aesCtx); + + /* post-whitening */ + cpXTSxor16(pDst, pDst, tmp, encBlocks); + + pSrc += AES_BLK_SIZE*encBlocks; + pDst += AES_BLK_SIZE*encBlocks; + } + } + + /* "stealing" - encrypt last partial block if is */ + if(encBlocklast) { + int partBlockSize = encBlocklast/BYTESIZE; + + __ALIGN16 Ipp8u cc[AES_BLK_SIZE]; + __ALIGN16 Ipp8u pp[AES_BLK_SIZE]; + cpAES_XTS_EncBlock(cc, pSrc, tweakCT, &aesCtx); + gf_mul_by_primitive(tweakCT); + + CopyBlock16(cc, pp); + CopyBlock(pSrc+AES_BLK_SIZE, pp, partBlockSize); + + encBlocklast %= BYTESIZE; + if(encBlocklast) { + Ipp8u partBlockMask = (Ipp8u)((0xFF)<<((BYTESIZE -encBlocklast) %BYTESIZE)); + Ipp8u x = pSrc[AES_BLK_SIZE+partBlockSize]; + Ipp8u y = cc[partBlockSize]; + x = (x & partBlockMask) | (y & ~partBlockMask); + pp[partBlockSize] = x; + cc[partBlockSize] &= partBlockMask; + partBlockSize++; + } + cpAES_XTS_EncBlock(pDst, pp, tweakCT, &aesCtx); + + CopyBlock(cc, pDst+AES_BLK_SIZE, partBlockSize); + } + + } while(0); + + return sts; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_xtsinit.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_xtsinit.c new file mode 100644 index 0000000..106b72a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaes_xtsinit.c @@ -0,0 +1,89 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-XTS Functions (IEEE P1619) +// +// Contents: +// ippsAES_XTSInit() +// +*/ + +#include "owncp.h" +#include "pcpaesmxts.h" +#include "pcptool.h" + +/*F* +// Name: ippsAES_XTSInit +// +// Purpose: Init AES_XTS context for future usage. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// ippStsMemAllocErr size of buffer is not match fro operation +// ippStsLengthErr keyLen != 16*8*2 && +// != 32*8*2 +// ippStsNoErr no errors +// +// Parameters: +// pKey pointer to the secret key +// keyLen length of the secret key in bits +// pCtx pointer to the AES-XTS context +// ctxSize available size (in bytes) of buffer above +// duBitSize length of Data Unit in bits +// +*F*/ + +IPPFUN(IppStatus, ippsAES_XTSInit,(const Ipp8u* pKey, int keyLen, + int duBitsize, + IppsAES_XTSSpec* pCtx, int ctxSize)) +{ + /* test key and keyLenBits */ + IPP_BAD_PTR1_RET(pKey); + IPP_BADARG_RET(keyLen!=16*BYTESIZE*2 && keyLen!=32*BYTESIZE*2, ippStsLengthErr); + + /* test DU parameters */ + IPP_BADARG_RET(duBitsize ctxSize, ippStsMemAllocErr); + + { + IppsAESSpec* pdatAES = &pCtx->datumAES; + IppsAESSpec* ptwkAES = &pCtx->tweakAES; + + int keySize = keyLen/2/BYTESIZE; + const Ipp8u* pdatKey = pKey; + const Ipp8u* ptwkKey = pKey+keySize; + + IppStatus sts = ippStsNoErr; + sts = ippsAESInit(pdatKey, keySize, pdatAES, sizeof(IppsAESSpec)); + if(ippStsNoErr!=sts) return sts; + + sts = ippsAESInit(ptwkKey, keySize, ptwkAES, sizeof(IppsAESSpec)); + if(ippStsNoErr!=sts) return sts; + + AES_XTS_SET_ID(pCtx); + pCtx->duBitsize = duBitsize; + return sts; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesauthccm.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesauthccm.h new file mode 100644 index 0000000..a6b6d15 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesauthccm.h @@ -0,0 +1,99 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Message Authentication Algorithm +// Internal Definitions and Internal Functions Prototypes +// +// +*/ + +#if !defined(_CP_AES_CCM_H) +#define _CP_AES_CCM_H + +#include "pcprij.h" +#include "pcpaesm.h" + +struct _cpAES_CCM { + Ipp32u idCtx; /* CCM ID */ + + Ipp64u msgLen; /* length of message to be processed */ + Ipp64u lenProcessed; /* message length has been processed */ + Ipp32u tagLen; /* length of authentication tag */ + Ipp32u counterVal; /* current counter value */ + Ipp8u ctr0[MBS_RIJ128]; /* counter value */ + Ipp8u s0[MBS_RIJ128]; /* S0 = ENC(CTR0) content */ + Ipp8u si[MBS_RIJ128]; /* Si = ENC(CTRi) content */ + Ipp8u blk[MBS_RIJ128]; /* temporary data container */ + Ipp8u mac[MBS_RIJ128]; /* current MAC value */ + + Ipp8u cipher[sizeof(IppsAESSpec)]; +}; + +/* alignment */ +#define AESCCM_ALIGNMENT ((int)(sizeof(void*))) + +/* +// access macros +*/ +#define AESCCM_SET_ID(stt) ((stt)->idCtx = (Ipp32u)idCtxAESCCM ^ (Ipp32u)IPP_UINT_PTR(stt)) +#define AESCCM_MSGLEN(stt) ((stt)->msgLen) +#define AESCCM_LENPRO(stt) ((stt)->lenProcessed) +#define AESCCM_TAGLEN(stt) ((stt)->tagLen) +#define AESCCM_COUNTER(stt) ((stt)->counterVal) +#define AESCCM_CTR0(stt) ((stt)->ctr0) +#define AESCCM_S0(stt) ((stt)->s0) +#define AESCCM_Si(stt) ((stt)->si) +#define AESCCM_BLK(stt) ((stt)->blk) +#define AESCCM_MAC(stt) ((stt)->mac) +#define AESCCM_CIPHER(stt) (IppsAESSpec*)(&((stt)->cipher)) + +/* valid context ID */ +#define VALID_AESCCM_ID(ctx) ((((ctx)->idCtx) ^ (Ipp32u)IPP_UINT_PTR((ctx))) == (Ipp32u)idCtxAESCCM) + +/* +// Internal functions +*/ +#if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) +#define AuthEncrypt_RIJ128_AES_NI OWNAPI(AuthEncrypt_RIJ128_AES_NI) + IPP_OWN_DECL (void, AuthEncrypt_RIJ128_AES_NI, (const Ipp8u* inpBlk, Ipp8u* outBlk, int nr, const void* pRKey, Ipp32u len, void* pLocalCtx)) +#define DecryptAuth_RIJ128_AES_NI OWNAPI(DecryptAuth_RIJ128_AES_NI) + IPP_OWN_DECL (void, DecryptAuth_RIJ128_AES_NI, (const Ipp8u* inpBlk, Ipp8u* outBlk, int nr, const void* pRKey, Ipp32u len, void* pLocalCtx)) +#endif + +/* Counter block formatter */ +static Ipp8u* CounterEnc(Ipp32u* pBuffer, int fmt, Ipp64u counter) +{ + #if (IPP_ENDIAN == IPP_LITTLE_ENDIAN) + pBuffer[0] = ENDIANNESS(IPP_HIDWORD(counter)); + pBuffer[1] = ENDIANNESS(IPP_LODWORD(counter)); + #else + pBuffer[0] = IPP_HIDWORD(counter); + pBuffer[1] = IPP_LODWORD(counter); + #endif + return (Ipp8u*)pBuffer + 8 - fmt; +} + +static int cpSizeofCtx_AESCCM(void) +{ + return sizeof(IppsAES_CCMState); +} + +#endif /* _CP_AES_CCM_H*/ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesauthgcm.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesauthgcm.h new file mode 100644 index 0000000..d33a144 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesauthgcm.h @@ -0,0 +1,237 @@ +/******************************************************************************* +* Copyright (C) 2015 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Message Authentication Algorithm +// Internal Definitions and Internal Functions Prototypes +// +// +*/ + +#if !defined(_CP_AESAUTH_GCM_H) +#define _CP_AESAUTH_GCM_H + +#include "owncp.h" +#include "pcpaesm.h" + +#if(_IPP32E < _IPP32E_K0) + +#define BLOCK_SIZE (MBS_RIJ128) + +/* GCM Hash prototype: GHash = GHash*HKey mod G() */ +IPP_OWN_FUNPTR (void, MulGcm_, (Ipp8u* pGHash, const Ipp8u* pHKey, const void* pParam)) + +/* GCM Authentication prototype: GHash = (GHash^src[])*HKey mod G() */ +IPP_OWN_FUNPTR (void, Auth_, (Ipp8u* pHash, const Ipp8u* pSrc, int len, const Ipp8u* pHKey, const void* pParam)) + +/* GCM Encrypt_Authentication prototype */ +IPP_OWN_FUNPTR (void, Encrypt_, (Ipp8u* pDst, const Ipp8u* pSrc, int len, IppsAES_GCMState* pCtx)) + +/* GCM Authentication_Decrypt prototype */ +IPP_OWN_FUNPTR (void, Decrypt_, (Ipp8u* pDst, const Ipp8u* pSrc, int len, IppsAES_GCMState* pCtx)) + +typedef enum { + GcmInit, + GcmIVprocessing, + GcmAADprocessing, + GcmTXTprocessing +} GcmState; + +struct _cpAES_GCM { + + Ipp32u idCtx; /* AES-GCM id */ + GcmState state; /* GCM state: Init, IV|AAD|TXT processing */ + Ipp64u ivLen; /* IV length (bytes) */ + Ipp64u aadLen; /* header length (bytes) */ + Ipp64u txtLen; /* text length (bytes) */ + + int bufLen; /* stuff buffer length */ + __ALIGN16 /* aligned buffers */ + Ipp8u counter[BLOCK_SIZE]; /* counter */ + Ipp8u ecounter0[BLOCK_SIZE]; /* encrypted initial counter */ + Ipp8u ecounter[BLOCK_SIZE]; /* encrypted counter */ + Ipp8u ghash[BLOCK_SIZE]; /* ghash accumulator */ + + MulGcm_ hashFun; /* AES-GCM mul function */ + Auth_ authFun; /* authentication function */ + Encrypt_ encFun; /* encryption & authentication */ + Decrypt_ decFun; /* authentication & decryption */ + + __ALIGN16 /* aligned AES context */ + IppsAESSpec cipher; + +#if (_AES_PROB_NOISE == _FEATURE_ON_) + __ALIGN16 + cpAESNoiseParams noiseParams; +#endif + + __ALIGN16 /* aligned pre-computed data: */ + Ipp8u multiplier[BLOCK_SIZE]; /* - (default) hKey */ + /* - (aes_ni) hKey*t, (hKey*t)^2, (hKey*t)^4 */ + /* - (vaes_ni) 8 reverted ordered vectors by 4 128-bit values. + hKeys derivations in the multiplier[] array in order of appearance + (zero-index starts from the left): + hKey^4<<1, hKey^3<<1, hKey^2<<1, hKey<<1, + hKey^8<<1, hKey^7<<1, hKey^6<<1, hKey^5<<1, + hKey^12<<1, hKey^11<<1, hKey^10<<1, hKey^9<<1, + hKey^16<<1, hKey^15<<1, hKey^14<<1, hKey^13<<1, + ... ... + */ + /* - (safe) hKey*(t^i), i=0,...,127 */ +}; + +#define CTR_POS 12 + +/* alignment */ +#define AESGCM_ALIGNMENT (16) + +#define PRECOMP_DATA_SIZE_AES_NI_AESGCM (BLOCK_SIZE*4) +#define PRECOMP_DATA_SIZE_VAES_NI_AESGCM (BLOCK_SIZE*16*2) +#define PRECOMP_DATA_SIZE_FAST2K (BLOCK_SIZE*128) + +/* +// Useful macros +*/ +#define AESGCM_SET_ID(context) ((context)->idCtx = (Ipp32u)idCtxAESGCM ^ (Ipp32u)IPP_UINT_PTR(context)) +#define AESGCM_STATE(context) ((context)->state) + +#define AESGCM_IV_LEN(context) ((context)->ivLen) +#define AESGCM_AAD_LEN(context) ((context)->aadLen) +#define AESGCM_TXT_LEN(context) ((context)->txtLen) + +#define AESGCM_BUFLEN(context) ((context)->bufLen) +#define AESGCM_COUNTER(context) ((context)->counter) +#define AESGCM_ECOUNTER0(context) ((context)->ecounter0) +#define AESGCM_ECOUNTER(context) ((context)->ecounter) +#define AESGCM_GHASH(context) ((context)->ghash) + +#define AESGCM_HASH(context) ((context)->hashFun) +#define AESGCM_AUTH(context) ((context)->authFun) +#define AESGCM_ENC(context) ((context)->encFun) +#define AESGCM_DEC(context) ((context)->decFun) + +#define AESGCM_CIPHER(context) (IppsAESSpec*)(&((context)->cipher)) + +#if (_AES_PROB_NOISE == _FEATURE_ON_) +#define AESGCM_NOISE_PARAMS(context) ((context)->noiseParams) +#endif + +#define AESGCM_HKEY(context) ((context)->multiplier) +#define AESGCM_CPWR(context) ((context)->multiplier) +#define AES_GCM_MTBL(context) ((context)->multiplier) + +#define AESGCM_VALID_ID(context) ((((context)->idCtx) ^ (Ipp32u)IPP_UINT_PTR((context))) == (Ipp32u)idCtxAESGCM) + +#if 0 +__INLINE void IncrementCounter32(Ipp8u* pCtr) +{ + int i; + for(i=BLOCK_SIZE-1; i>=CTR_POS && 0==(Ipp8u)(++pCtr[i]); i--) ; +} +#endif +__INLINE void IncrementCounter32(Ipp8u* pCtr) +{ + Ipp32u* pCtr32 = (Ipp32u*)pCtr; + Ipp32u ctrVal = pCtr32[3]; + ctrVal = ENDIANNESS32(ctrVal); + ctrVal++; + ctrVal = ENDIANNESS32(ctrVal); + pCtr32[3] = ctrVal; +} + +#if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) +#define AesGcmPrecompute_avx OWNAPI(AesGcmPrecompute_avx) + IPP_OWN_DECL (void, AesGcmPrecompute_avx, (Ipp8u* pPrecomputeData, const Ipp8u* pHKey)) +#define AesGcmMulGcm_avx OWNAPI(AesGcmMulGcm_avx) + IPP_OWN_DECL (void, AesGcmMulGcm_avx, (Ipp8u* pGhash, const Ipp8u* pHkey, const void* pParam)) +#define AesGcmAuth_avx OWNAPI(AesGcmAuth_avx) + IPP_OWN_DECL (void, AesGcmAuth_avx, (Ipp8u* pGhash, const Ipp8u* pSrc, int len, const Ipp8u* pHkey, const void* pParam)) +#define wrpAesGcmEnc_avx OWNAPI(wrpAesGcmEnc_avx) + IPP_OWN_DECL (void, wrpAesGcmEnc_avx, (Ipp8u* pDst, const Ipp8u* pSrc, int len, IppsAES_GCMState* pCtx)) +#define wrpAesGcmDec_avx OWNAPI(wrpAesGcmDec_avx) + IPP_OWN_DECL (void, wrpAesGcmDec_avx, (Ipp8u* pDst, const Ipp8u* pSrc, int len, IppsAES_GCMState* pCtx)) +#define AesGcmEnc_avx OWNAPI(AesGcmEnc_avx) + IPP_OWN_DECL (void, AesGcmEnc_avx, (Ipp8u* pDst, const Ipp8u* pSrc, int len, RijnCipher cipher, int nr, const Ipp8u* pKeys, Ipp8u* pGhash, Ipp8u* pCnt, Ipp8u* pECnt, const Ipp8u* pMuls)) +#define AesGcmDec_avx OWNAPI(AesGcmDec_avx) + IPP_OWN_DECL (void, AesGcmDec_avx, (Ipp8u* pDst, const Ipp8u* pSrc, int len, RijnCipher cipher, int nr, const Ipp8u* pKeys, Ipp8u* pGhash, Ipp8u* pCnt, Ipp8u* pECnt, const Ipp8u* pMuls)) +#endif + +#if(_IPP32E>=_IPP32E_K0) +#define AesGcmPrecompute_vaes OWNAPI(AesGcmPrecompute_vaes) + IPP_OWN_DECL (void, AesGcmPrecompute_vaes, (Ipp8u* const pPrecomputeData, const Ipp8u* const pHKey)) +#define AesGcmMulGcm_vaes OWNAPI(AesGcmMulGcm_vaes) + IPP_OWN_DECL (void, AesGcmMulGcm_vaes, (Ipp8u* pGhash, const Ipp8u* pHkey, const void* pParam)) +#define AesGcmAuth_vaes OWNAPI(AesGcmAuth_vaes) + IPP_OWN_DECL (void, AesGcmAuth_vaes, (Ipp8u* pGhash, const Ipp8u* pSrc, int len, const Ipp8u* pHkey, const void* pParam)) +#define AesGcmEnc_vaes OWNAPI(AesGcmEnc_vaes) + IPP_OWN_DECL (void, AesGcmEnc_vaes, (Ipp8u* pDst, const Ipp8u* pSrc, int len, IppsAES_GCMState* pCtx)) +#define AesGcmDec_vaes OWNAPI(AesGcmDec_vaes) + IPP_OWN_DECL (void, AesGcmDec_vaes, (Ipp8u* pDst, const Ipp8u* pSrc, int len, IppsAES_GCMState* pCtx)) +#endif /* _IPP32E>=_IPP32E_K0 */ + +#define AesGcmPrecompute_table2K OWNAPI(AesGcmPrecompute_table2K) + IPP_OWN_DECL (void, AesGcmPrecompute_table2K, (Ipp8u* pPrecomputeData, const Ipp8u* pHKey)) + +/* #define AesGcmMulGcm_table2K OWNAPI(AesGcmMulGcm_table2K) */ +/* IPP_OWN_DECL (void, AesGcmMulGcm_table2K, (Ipp8u* pGhash, const Ipp8u* pHkey, const void* pParam)) */ +#define AesGcmMulGcm_table2K_ct OWNAPI(AesGcmMulGcm_table2K_ct) + IPP_OWN_DECL (void, AesGcmMulGcm_table2K_ct, (Ipp8u* pGhash, const Ipp8u* pHkey, const void* pParam)) + +/* #define AesGcmAuth_table2K OWNAPI(AesGcmAuth_table2K) */ +/* IPP_OWN_DECL (void, AesGcmAuth_table2K, (Ipp8u* pGhash, const Ipp8u* pSrc, int len, const Ipp8u* pHkey, const void* pParam)) */ +#define AesGcmAuth_table2K_ct OWNAPI(AesGcmAuth_table2K_ct) + IPP_OWN_DECL (void, AesGcmAuth_table2K_ct, (Ipp8u* pGhash, const Ipp8u* pSrc, int len, const Ipp8u* pHkey, const void* pParam)) + +#define wrpAesGcmEnc_table2K OWNAPI(wrpAesGcmEnc_table2K) + IPP_OWN_DECL (void, wrpAesGcmEnc_table2K, (Ipp8u* pDst, const Ipp8u* pSrc, int len, IppsAES_GCMState* pCtx)) +#define wrpAesGcmDec_table2K OWNAPI(wrpAesGcmDec_table2K) + IPP_OWN_DECL (void, wrpAesGcmDec_table2K, (Ipp8u* pDst, const Ipp8u* pSrc, int len, IppsAES_GCMState* pCtx)) + +#if (_IPP==_IPP_H9) || (_IPP32E==_IPP32E_L9) +#define AesGcmEnc_vaes_avx2 OWNAPI(AesGcmEnc_vaes_avx2) + IPP_OWN_DECL (void, AesGcmEnc_vaes_avx2, (Ipp8u* pDst, const Ipp8u* pSrc, int len, IppsAES_GCMState* pCtx)) +#define AesGcmDec_vaes_avx2 OWNAPI(AesGcmDec_vaes_avx2) + IPP_OWN_DECL (void, AesGcmDec_vaes_avx2, (Ipp8u* pDst, const Ipp8u* pSrc, int len, IppsAES_GCMState* pCtx)) +#endif /* #if (_IPP==_IPP_H9) || (_IPP32E==_IPP32E_L9) */ + +extern const Ipp16u AesGcmConst_table[256]; /* precomputed reduction table */ + +static int cpSizeofCtx_AESGCM(void) +{ + int precomp_size; + + #if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + if(IsFeatureEnabled(ippCPUID_AES|ippCPUID_CLMUL) || IsFeatureEnabled(ippCPUID_AVX2VAES|ippCPUID_AVX2VCLMUL)) + precomp_size = PRECOMP_DATA_SIZE_AES_NI_AESGCM; + else + #endif + precomp_size = PRECOMP_DATA_SIZE_FAST2K; + + /* decrease precomp_size as soon as BLOCK_SIZE bytes already reserved in context */ + precomp_size -= BLOCK_SIZE; + + return (Ipp32s)sizeof(IppsAES_GCMState) + +precomp_size + +AESGCM_ALIGNMENT-1; +} + +#endif // (_IPP32E < _IPP32E_K0) + +#endif /* _CP_AESAUTH_GCM_H*/ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesauthgcm_avx512.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesauthgcm_avx512.h new file mode 100644 index 0000000..e81a8a7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesauthgcm_avx512.h @@ -0,0 +1,153 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES GCM otimized for AVX512 and AVX512-VAES features +// Internal Definitions +// +// +*/ + +#ifndef _CP_AESAUTH_GCM_AVX512_H +#define _CP_AESAUTH_GCM_AVX512_H + +#include "owndefs.h" +#include "owncp.h" + +#if(_IPP32E>=_IPP32E_K0) + +#include "pcprij.h" + +#include "aes_gcm_vaes.h" +#include "aes_gcm_avx512.h" +#include "aes_gcm_avx512_structures.h" + +// Prototypes for internal functions from IPsec + +// IV processing +IPP_OWN_FUNPTR (void, IvUpdate_, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const Ipp8u *iv, const Ipp64u iv_len)) +IPP_OWN_FUNPTR (void, IvFinalize_, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const Ipp8u *iv, const Ipp64u iv_len, const Ipp64u iv_general_len)) + +// AAD processing +IPP_OWN_FUNPTR (void, AadUpdate_, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const Ipp8u *aad, const Ipp64u aad_len)) + +// GCM multiplication +IPP_OWN_FUNPTR (void, MulGcm_, (const struct gcm_key_data *key_data, Ipp8u *ghash)) + +// Encryption-authentication +IPP_OWN_FUNPTR (void, EncryptUpdate_, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, Ipp8u *out, const Ipp8u *in, Ipp64u len)) + +// Decryption-verification +IPP_OWN_FUNPTR (void, DecryptUpdate_, (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, Ipp8u *out, const Ipp8u *in, Ipp64u len)) + +// Get tag +IPP_OWN_FUNPTR (void, GetTag_, (const struct gcm_key_data *key_data, const struct gcm_context_data *context_data, Ipp8u *auth_tag, Ipp64u auth_tag_len)) + +typedef enum { + GcmInit, + GcmIVprocessing, + GcmAADprocessing, + GcmTXTprocessing +} GcmState; + +#define BLOCK_SIZE (MBS_RIJ128) + +// Structure modified to work with functions from IPsec + +struct _cpAES_GCM { + + Ipp32u idCtx; /* AES-GCM id */ + GcmState state; /* GCM state: Init, IV|AAD|TXT processing */ + Ipp64u ivLen; /* IV length (bytes) */ + Ipp64u aadLen; /* header length (bytes) */ + Ipp64u txtLen; /* text length (bytes) */ + + int bufLen; /* stuff buffer length */ + __ALIGN16 /* aligned buffers */ + Ipp8u counter[BLOCK_SIZE]; /* counter */ + Ipp8u ecounter0[BLOCK_SIZE]; /* encrypted initial counter */ + Ipp8u ecounter[BLOCK_SIZE]; /* encrypted counter */ + Ipp8u ghash[BLOCK_SIZE]; /* ghash accumulator */ + + __ALIGN16 + struct gcm_key_data key_data; + __ALIGN16 + struct gcm_context_data context_data; + Ipp64u keyLen; /* key length (bytes) */ + + IvUpdate_ ivUpdateFunc; // IV processing + IvFinalize_ ivFinalizeFunc; + AadUpdate_ aadUpdateFunc; // AAD processing + MulGcm_ gcmMulFunc; // GCM multiplication + EncryptUpdate_ encryptUpdateFunc; // Encryption-authentication + DecryptUpdate_ decryptUpdateFunc; // Decryption-verification + GetTag_ getTagFunc; // Get tag + +#if (_AES_PROB_NOISE == _FEATURE_ON_) + __ALIGN16 + cpAESNoiseParams noiseParams; +#endif +}; + +// Alignment +#define AESGCM_ALIGNMENT (16) + +// Useful macros +#define AESGCM_SET_ID(context) ((context)->idCtx = (Ipp32u)idCtxAESGCM ^ (Ipp32u)IPP_UINT_PTR(context)) +#define AESGCM_STATE(context) ((context)->state) + +#define AESGCM_IV_LEN(context) ((context)->ivLen) + +#define AESGCM_COUNTER(context) ((context)->counter) +#define AESGCM_ECOUNTER0(context) ((context)->ecounter0) +#define AESGCM_ECOUNTER(context) ((context)->ecounter) + +#define AES_GCM_KEY_DATA(context) ((context)->key_data) +#define AES_GCM_CONTEXT_DATA(context) ((context)->context_data) +#define AES_GCM_KEY_LEN(context) ((context)->keyLen) + +#define AES_GCM_IV_UPDATE(context) ((context)->ivUpdateFunc) +#define AES_GCM_IV_FINALIZE(context) ((context)->ivFinalizeFunc) +#define AES_GCM_AAD_UPDATE(context) ((context)->aadUpdateFunc) +#define AES_GCM_GMUL(context) ((context)->gcmMulFunc) +#define AES_GCM_ENCRYPT_UPDATE(context) ((context)->encryptUpdateFunc) +#define AES_GCM_DECRYPT_UPDATE(context) ((context)->decryptUpdateFunc) +#define AES_GCM_GET_TAG(context) ((context)->getTagFunc) + +// Fields retargeted to IPsec context +#define AESGCM_GHASH(context) (&(AES_GCM_CONTEXT_DATA(context).aad_hash[0])) +#define AESGCM_TXT_LEN(context) (AES_GCM_CONTEXT_DATA(context).in_length) +#define AESGCM_AAD_LEN(context) (AES_GCM_CONTEXT_DATA(context).aad_length) +#define AESGCM_BUFLEN(context) (AES_GCM_CONTEXT_DATA(context).partial_block_length) + +#if (_AES_PROB_NOISE == _FEATURE_ON_) +#define AESGCM_NOISE_PARAMS(ctx) ((ctx)->noiseParams) +#endif + +#define AESGCM_VALID_ID(context) ((((context)->idCtx) ^ (Ipp32u)IPP_UINT_PTR((context))) == (Ipp32u)idCtxAESGCM) + +static int cpSizeofCtx_AESGCM(void) +{ + return (Ipp32s)sizeof(IppsAES_GCMState) + AESGCM_ALIGNMENT-1; +} + +#endif // (_IPP32E>=_IPP32E_K0) + +#endif // _CP_AESAUTH_GCM_AVX512_H diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaescmac_setupnoise.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaescmac_setupnoise.c new file mode 100644 index 0000000..7981e53 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaescmac_setupnoise.c @@ -0,0 +1,79 @@ +/******************************************************************************* +* Copyright 2022 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-CMAC noise setup function +// +// Contents: +// ippsAES_CMACSetupNoise +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpcmac.h" +#include "pcptool.h" + +/*F* +// Name: ippsAES_CMACSetupNoise +// +// Purpose: AES-CMAC Setup Noise +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL +// ippStsContextMatchErr AES context is invalid +// ippStsLengthErr noiseLevel > 4 +// ippStsNotSupportedModeErr Mistletoe3 mitigation isn't applicable for current CPU +// (no support of Intel® Advanced Encryption Standard New Instructions (Intel® AES-NI) or +// vector extensions of Intel® AES-NI) +// ippStsNoErr no errors +// +// Parameters: +// noiseLevel the value of this parameter is directly +// proportional to the amount of noise injected +// Increasing noise level by 1 means the delay +// (performance impact) is doubled +// pState pointer to the AES context +// +*F*/ +IPPFUN(IppStatus, ippsAES_CMACSetupNoise,(Ipp32u noiseLevel, IppsAES_CMACState* pState)) +{ +#if (_AES_PROB_NOISE == _FEATURE_ON_) + /* test context */ + IPP_BAD_PTR1_RET(pState); + /* test ID */ + IPP_BADARG_RET(!VALID_AESCMAC_ID(pState), ippStsContextMatchErr); + + /* test noise level range */ + IPP_BADARG_RET(noiseLevel > 4, ippStsLengthErr); + + cpAESNoiseParams *params = (cpAESNoiseParams *)&AESCMAC_NOISE_PARAMS(pState); + + /* set up the parameters with initial values */ + AES_NOISE_RAND(params) = 0; + AES_NOISE_LEVEL(params) = noiseLevel; + + return ippStsNoErr; +#else + /* To remove MSVC warning C4100: 'XXX': unreferenced formal parameter*/ + IPP_UNREFERENCED_PARAMETER(noiseLevel); + IPP_UNREFERENCED_PARAMETER(pState); + return ippStsNotSupportedModeErr; +#endif +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesdecryptofb.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesdecryptofb.c new file mode 100644 index 0000000..962985c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesdecryptofb.c @@ -0,0 +1,91 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES decryption (OFB mode) +// +// Contents: +// ippsAESDecryptOFB() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" +#include "pcpaes_ofb.h" + +/*F* +// Name: ippsAESDecryptOFB +// +// Purpose: Decrypts byte data stream according to Rijndael in OFB mode. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// pSrc == NULL +// pDst == NULL +// pIV == NULL +// ippStsContextMatchErr !VALID_AES_ID() +// ippStsLengthErr len <1 +// ippStsOFBSizeErr (1>ofbBlkSize || ofbBlkSize>MBS_RIJ128) +// ippStsUnderRunErr (len%ofbBlkSize) +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// len output buffer length (in bytes) +// ofbBlkSize OFB block size (in bytes) +// pCtx pointer to the AES context +// pIV pointer to the initialization vector +*F*/ +IPPFUN(IppStatus, ippsAESDecryptOFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int ofbBlkSize, + const IppsAESSpec* pCtx, + Ipp8u* pIV)) +{ + /* test context */ + IPP_BAD_PTR1_RET(pCtx); + /* test the context ID */ + IPP_BADARG_RET(!VALID_AES_ID(pCtx), ippStsContextMatchErr); + + /* test source, target buffers and initialization pointers */ + IPP_BAD_PTR3_RET(pSrc, pIV, pDst); + /* test stream length */ + IPP_BADARG_RET((len<1), ippStsLengthErr); + /* test OFB value */ + IPP_BADARG_RET(((1>ofbBlkSize) || (MBS_RIJ128=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + if(AES_NI_ENABLED==RIJ_AESNI(pCtx)) { + if(ofbBlkSize==MBS_RIJ128) + EncryptOFB128_RIJ128_AES_NI(pSrc, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), len, pIV); + else + EncryptOFB_RIJ128_AES_NI(pSrc, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), len, ofbBlkSize, pIV); + return ippStsNoErr; + } + else +#endif + { + cpProcessAES_ofb8(pSrc, pDst, len, ofbBlkSize, pCtx, pIV); + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesencryptofb.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesencryptofb.c new file mode 100644 index 0000000..78362d2 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesencryptofb.c @@ -0,0 +1,91 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES encryption (OFB mode) +// +// Contents: +// ippsAESEncryptOFB() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" +#include "pcpaes_ofb.h" + +/*F* +// Name: ippsAESEncryptOFB +// +// Purpose: Encrypts byte data stream according to Rijndael in OFB mode. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// pSrc == NULL +// pDst == NULL +// pIV == NULL +// ippStsContextMatchErr !VALID_AES_ID() +// ippStsLengthErr len <1 +// ippStsOFBSizeErr (1>ofbBlkSize || ofbBlkSize>MBS_RIJ128) +// ippStsUnderRunErr (len%ofbBlkSize) +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// len input buffer length (in bytes) +// ofbBlkSize OFB block size (in bytes) +// pCtx pointer to the AES context +// pIV pointer to the initialization vector +*F*/ +IPPFUN(IppStatus, ippsAESEncryptOFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int ofbBlkSize, + const IppsAESSpec* pCtx, + Ipp8u* pIV)) +{ + /* test context */ + IPP_BAD_PTR1_RET(pCtx); + /* test the context ID */ + IPP_BADARG_RET(!VALID_AES_ID(pCtx), ippStsContextMatchErr); + + /* test source, target buffers and initialization pointers */ + IPP_BAD_PTR3_RET(pSrc, pIV, pDst); + /* test stream length */ + IPP_BADARG_RET((len<1), ippStsLengthErr); + /* test OFB value */ + IPP_BADARG_RET(((1>ofbBlkSize) || (MBS_RIJ128=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + if(AES_NI_ENABLED==RIJ_AESNI(pCtx)) { + if(ofbBlkSize==MBS_RIJ128) + EncryptOFB128_RIJ128_AES_NI(pSrc, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), len, pIV); + else + EncryptOFB_RIJ128_AES_NI(pSrc, pDst, RIJ_NR(pCtx), RIJ_EKEYS(pCtx), len, ofbBlkSize, pIV); + return ippStsNoErr; + } + else +#endif + { + cpProcessAES_ofb8(pSrc, pDst, len, ofbBlkSize, pCtx, pIV); + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesgcm_setupnoise.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesgcm_setupnoise.c new file mode 100644 index 0000000..fbf3c63 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesgcm_setupnoise.c @@ -0,0 +1,86 @@ +/******************************************************************************* +* Copyright 2022 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-GCM noise setup function +// +// Contents: +// ippsAES_GCMSetupNoise +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" + +#include "pcpaesauthgcm.h" +#include "pcpaesauthgcm_avx512.h" + +/*F* +// Name: ippsAES_GCMSetupNoise +// +// Purpose: AES-GCM Setup Noise +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL +// ippStsContextMatchErr AES context is invalid +// ippStsLengthErr noiseLevel > 4 +// ippStsNotSupportedModeErr Mistletoe3 mitigation isn't applicable for current CPU +// (no support of Intel® Advanced Encryption Standard New Instructions (Intel® AES-NI) or +// vector extensions of Intel® AES-NI) +// ippStsNoErr no errors +// +// Parameters: +// noiseLevel the value of this parameter is directly +// proportional to the amount of noise injected +// Increasing noise level by 1 means the delay +// (performance impact) is doubled +// pState pointer to the AES context +// +*F*/ +IPPFUN(IppStatus, ippsAES_GCMSetupNoise,(Ipp32u noiseLevel, IppsAES_GCMState* pState)) +{ +#if (_AES_PROB_NOISE == _FEATURE_ON_) + /* test context */ + IPP_BAD_PTR1_RET(pState); + + /* use aligned context */ + pState = (IppsAES_GCMState*)(IPP_ALIGNED_PTR(pState, AESGCM_ALIGNMENT)); + + /* test state ID */ + IPP_BADARG_RET(!AESGCM_VALID_ID(pState), ippStsContextMatchErr); + + /* test noise level range */ + IPP_BADARG_RET(noiseLevel > 4, ippStsLengthErr); + + cpAESNoiseParams *params = (cpAESNoiseParams *)&AESGCM_NOISE_PARAMS(pState); + + /* set up the parameters with initial values */ + AES_NOISE_RAND(params) = 0; + AES_NOISE_LEVEL(params) = noiseLevel; + + return ippStsNoErr; +#else + /* To remove MSVC warning C4100: 'XXX': unreferenced formal parameter*/ + IPP_UNREFERENCED_PARAMETER(noiseLevel); + IPP_UNREFERENCED_PARAMETER(pState); + return ippStsNotSupportedModeErr; +#endif +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesgcmtableca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesgcmtableca.c new file mode 100644 index 0000000..7ab98d0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesgcmtableca.c @@ -0,0 +1,54 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Encrypt/Decrypt byte data stream according to Rijndael128 (GCM mode) +// +// "fast" stuff +// +// Contents: +// +// +*/ + + +#include "owndefs.h" +#include "owncp.h" + +#if defined( _IPP_DATA ) +__ALIGN64 const Ipp16u AesGcmConst_table[256] = { +0x0000, 0xc201, 0x8403, 0x4602, 0x0807, 0xca06, 0x8c04, 0x4e05, 0x100e, 0xd20f, 0x940d, 0x560c, 0x1809, 0xda08, 0x9c0a, 0x5e0b, +0x201c, 0xe21d, 0xa41f, 0x661e, 0x281b, 0xea1a, 0xac18, 0x6e19, 0x3012, 0xf213, 0xb411, 0x7610, 0x3815, 0xfa14, 0xbc16, 0x7e17, +0x4038, 0x8239, 0xc43b, 0x063a, 0x483f, 0x8a3e, 0xcc3c, 0x0e3d, 0x5036, 0x9237, 0xd435, 0x1634, 0x5831, 0x9a30, 0xdc32, 0x1e33, +0x6024, 0xa225, 0xe427, 0x2626, 0x6823, 0xaa22, 0xec20, 0x2e21, 0x702a, 0xb22b, 0xf429, 0x3628, 0x782d, 0xba2c, 0xfc2e, 0x3e2f, +0x8070, 0x4271, 0x0473, 0xc672, 0x8877, 0x4a76, 0x0c74, 0xce75, 0x907e, 0x527f, 0x147d, 0xd67c, 0x9879, 0x5a78, 0x1c7a, 0xde7b, +0xa06c, 0x626d, 0x246f, 0xe66e, 0xa86b, 0x6a6a, 0x2c68, 0xee69, 0xb062, 0x7263, 0x3461, 0xf660, 0xb865, 0x7a64, 0x3c66, 0xfe67, +0xc048, 0x0249, 0x444b, 0x864a, 0xc84f, 0x0a4e, 0x4c4c, 0x8e4d, 0xd046, 0x1247, 0x5445, 0x9644, 0xd841, 0x1a40, 0x5c42, 0x9e43, +0xe054, 0x2255, 0x6457, 0xa656, 0xe853, 0x2a52, 0x6c50, 0xae51, 0xf05a, 0x325b, 0x7459, 0xb658, 0xf85d, 0x3a5c, 0x7c5e, 0xbe5f, +0x00e1, 0xc2e0, 0x84e2, 0x46e3, 0x08e6, 0xcae7, 0x8ce5, 0x4ee4, 0x10ef, 0xd2ee, 0x94ec, 0x56ed, 0x18e8, 0xdae9, 0x9ceb, 0x5eea, +0x20fd, 0xe2fc, 0xa4fe, 0x66ff, 0x28fa, 0xeafb, 0xacf9, 0x6ef8, 0x30f3, 0xf2f2, 0xb4f0, 0x76f1, 0x38f4, 0xfaf5, 0xbcf7, 0x7ef6, +0x40d9, 0x82d8, 0xc4da, 0x06db, 0x48de, 0x8adf, 0xccdd, 0x0edc, 0x50d7, 0x92d6, 0xd4d4, 0x16d5, 0x58d0, 0x9ad1, 0xdcd3, 0x1ed2, +0x60c5, 0xa2c4, 0xe4c6, 0x26c7, 0x68c2, 0xaac3, 0xecc1, 0x2ec0, 0x70cb, 0xb2ca, 0xf4c8, 0x36c9, 0x78cc, 0xbacd, 0xfccf, 0x3ece, +0x8091, 0x4290, 0x0492, 0xc693, 0x8896, 0x4a97, 0x0c95, 0xce94, 0x909f, 0x529e, 0x149c, 0xd69d, 0x9898, 0x5a99, 0x1c9b, 0xde9a, +0xa08d, 0x628c, 0x248e, 0xe68f, 0xa88a, 0x6a8b, 0x2c89, 0xee88, 0xb083, 0x7282, 0x3480, 0xf681, 0xb884, 0x7a85, 0x3c87, 0xfe86, +0xc0a9, 0x02a8, 0x44aa, 0x86ab, 0xc8ae, 0x0aaf, 0x4cad, 0x8eac, 0xd0a7, 0x12a6, 0x54a4, 0x96a5, 0xd8a0, 0x1aa1, 0x5ca3, 0x9ea2, +0xe0b5, 0x22b4, 0x64b6, 0xa6b7, 0xe8b2, 0x2ab3, 0x6cb1, 0xaeb0, 0xf0bb, 0x32ba, 0x74b8, 0xb6b9, 0xf8bc, 0x3abd, 0x7cbf, 0xbebe +}; +#endif /* _IPP_DATA */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesgcmtbl2k_mulpx.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesgcmtbl2k_mulpx.c new file mode 100644 index 0000000..e01d604 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesgcmtbl2k_mulpx.c @@ -0,0 +1,350 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Encrypt/Decrypt byte data stream according to Rijndael128 (GCM mode) +// +// "fast" stuff +// +// Contents: +// AesGcmMulGcm_table2K() +// +*/ + + +#include "owndefs.h" +#include "owncp.h" + +#include "pcpaesauthgcm.h" +#include "pcptool.h" +#include "pcpmask_ct.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + +#if(_IPP32E<_IPP32E_K0) + +typedef struct{ + Ipp8u b[16]; +} AesGcmPrecompute_GF; + + +#if !((_IPP==_IPP_V8) || (_IPP==_IPP_P8) || \ + (_IPP==_IPP_S8) || (_IPP>=_IPP_G9) || \ + (_IPP32E==_IPP32E_U8) || (_IPP32E==_IPP32E_Y8) || \ + (_IPP32E==_IPP32E_N8) || (_IPP32E>=_IPP32E_E9)) +/* +// AesGcmMulGcm_def|safe(Ipp8u* pGhash, const Ipp8u* pHKey) +// +// Ghash = Ghash * HKey mod G() +*/ +__INLINE Ipp16u getAesGcmConst_table_ct(int idx) +{ + #define TBL_SLOTS_REP_READ (Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(AesGcmConst_table[0])) + const BNU_CHUNK_T* TblEntry = (BNU_CHUNK_T*)AesGcmConst_table; + + BNU_CHUNK_T idx_sel = (BNU_CHUNK_T)(idx / TBL_SLOTS_REP_READ); /* selection index */ + BNU_CHUNK_T i; + BNU_CHUNK_T selection = 0; + for (i = 0; i>= (idx & (TBL_SLOTS_REP_READ-1)) * sizeof(Ipp16u)*8; + return (Ipp16u)(selection & 0xFFffFF); + #undef TBL_SLOTS_REP_READ +} + +#if 0 +void AesGcmMulGcm_table2K(Ipp8u* pGhash, const Ipp8u* pPrecomputeData, const void* pParam) +{ + __ALIGN16 Ipp8u t5[BLOCK_SIZE]; + __ALIGN16 Ipp8u t4[BLOCK_SIZE]; + __ALIGN16 Ipp8u t3[BLOCK_SIZE]; + __ALIGN16 Ipp8u t2[BLOCK_SIZE]; + + int nw; + Ipp32u a; + + IPP_UNREFERENCED_PARAMETER(pParam); + + #if 0 + XorBlock16(t5, t5, t5); + XorBlock16(t4, t4, t4); + XorBlock16(t3, t3, t3); + XorBlock16(t2, t2, t2); + #endif + PadBlock(0, t5, sizeof(t5)); + PadBlock(0, t4, sizeof(t4)); + PadBlock(0, t3, sizeof(t3)); + PadBlock(0, t2, sizeof(t2)); + + for(nw=0; nw<4; nw++) { + Ipp32u hashdw = ((Ipp32u*)pGhash)[nw]; + + a = hashdw & 0xf0f0f0f0; + XorBlock16(t5, pPrecomputeData+1024+EBYTE(a,1)+256*nw, t5); + XorBlock16(t4, pPrecomputeData+1024+EBYTE(a,0)+256*nw, t4); + XorBlock16(t3, pPrecomputeData+1024+EBYTE(a,3)+256*nw, t3); + XorBlock16(t2, pPrecomputeData+1024+EBYTE(a,2)+256*nw, t2); + + a = (hashdw<<4) & 0xf0f0f0f0; + XorBlock16(t5, pPrecomputeData+EBYTE(a,1)+256*nw, t5); + XorBlock16(t4, pPrecomputeData+EBYTE(a,0)+256*nw, t4); + XorBlock16(t3, pPrecomputeData+EBYTE(a,3)+256*nw, t3); + XorBlock16(t2, pPrecomputeData+EBYTE(a,2)+256*nw, t2); + } + + XorBlock(t2+1, t3, t2+1, BLOCK_SIZE-1); + XorBlock(t5+1, t2, t5+1, BLOCK_SIZE-1); + XorBlock(t4+1, t5, t4+1, BLOCK_SIZE-1); + + nw = t3[BLOCK_SIZE-1]; + //a = (Ipp32u)AesGcmConst_table[nw]; + a = (Ipp32u)getAesGcmConst_table_ct(nw); + a <<= 8; + nw = t2[BLOCK_SIZE-1]; + //a ^= (Ipp32u)AesGcmConst_table[nw]; + a ^= (Ipp32u)getAesGcmConst_table_ct(nw); + a <<= 8; + nw = t5[BLOCK_SIZE-1]; + //a ^= (Ipp32u)AesGcmConst_table[nw]; + a ^= (Ipp32u)getAesGcmConst_table_ct(nw); + + XorBlock(t4, &a, t4, sizeof(Ipp32u)); + CopyBlock16(t4, pGhash); +} +#endif + +/* +// CTE version of AesGcmMulGcm_table2K() +*/ +#if (_IPP_ARCH ==_IPP_ARCH_EM64T) +__INLINE void MaskedXorBlock16(const Ipp8u* pSrc1, const Ipp8u* pSrc2, Ipp8u* pDst, Ipp64u src2mask) +{ + ((Ipp64u*)pDst)[0] = ((Ipp64u*)pSrc1)[0] ^ (((Ipp64u*)pSrc2)[0] & src2mask); + ((Ipp64u*)pDst)[1] = ((Ipp64u*)pSrc1)[1] ^ (((Ipp64u*)pSrc2)[1] & src2mask); +} +#else /* IPP_ARCH == IPP_ARCH_IA32 */ +__INLINE void MaskedXorBlock16(const Ipp8u* pSrc1, const Ipp8u* pSrc2, Ipp8u* pDst, Ipp32u src2mask) +{ + ((Ipp32u*)pDst)[0] = ((Ipp32u*)pSrc1)[0] ^ (((Ipp32u*)pSrc2)[0] & src2mask); + ((Ipp32u*)pDst)[1] = ((Ipp32u*)pSrc1)[1] ^ (((Ipp32u*)pSrc2)[1] & src2mask); + ((Ipp32u*)pDst)[2] = ((Ipp32u*)pSrc1)[2] ^ (((Ipp32u*)pSrc2)[2] & src2mask); + ((Ipp32u*)pDst)[3] = ((Ipp32u*)pSrc1)[3] ^ (((Ipp32u*)pSrc2)[3] & src2mask); +} +#endif + +IPP_OWN_DEFN (void, AesGcmMulGcm_table2K_ct, (Ipp8u* pGhash, const Ipp8u* pPrecomputeData, const void* pParam)) +{ + __ALIGN16 Ipp8u t5[BLOCK_SIZE]; + __ALIGN16 Ipp8u t4[BLOCK_SIZE]; + __ALIGN16 Ipp8u t3[BLOCK_SIZE]; + __ALIGN16 Ipp8u t2[BLOCK_SIZE]; + + int nw; + + IPP_UNREFERENCED_PARAMETER(pParam); + +#if 0 + XorBlock16(t5, t5, t5); + XorBlock16(t4, t4, t4); + XorBlock16(t3, t3, t3); + XorBlock16(t2, t2, t2); +#endif + PadBlock(0, t5, sizeof(t5)); + PadBlock(0, t4, sizeof(t4)); + PadBlock(0, t3, sizeof(t3)); + PadBlock(0, t2, sizeof(t2)); + + for(nw=0; nw<4; nw++) { + Ipp32u hashdw = ((Ipp32u*)pGhash)[nw]; + Ipp32u a = hashdw & 0xf0f0f0f0; + + Ipp32u a0 = EBYTE(a,0); + Ipp32u a1 = EBYTE(a,1); + Ipp32u a2 = EBYTE(a,2); + Ipp32u a3 = EBYTE(a,3); + + int idx; + for(idx=0; idx<256; idx+=16) { + BNU_CHUNK_T mask0 = cpIsEqu_ct(a0, (BNU_CHUNK_T)idx); + BNU_CHUNK_T mask1 = cpIsEqu_ct(a1, (BNU_CHUNK_T)idx); + BNU_CHUNK_T mask2 = cpIsEqu_ct(a2, (BNU_CHUNK_T)idx); + BNU_CHUNK_T mask3 = cpIsEqu_ct(a3, (BNU_CHUNK_T)idx); + MaskedXorBlock16(t5, pPrecomputeData+1024 +256*nw +idx, t5, mask1); + MaskedXorBlock16(t4, pPrecomputeData+1024 +256*nw +idx, t4, mask0); + MaskedXorBlock16(t3, pPrecomputeData+1024 +256*nw +idx, t3, mask3); + MaskedXorBlock16(t2, pPrecomputeData+1024 +256*nw +idx, t2, mask2); + } + + a = (hashdw << 4) & 0xf0f0f0f0; + a0 = EBYTE(a, 0); + a1 = EBYTE(a, 1); + a2 = EBYTE(a, 2); + a3 = EBYTE(a, 3); + for (idx = 0; idx < 256; idx += 16) { + BNU_CHUNK_T mask0 = cpIsEqu_ct(a0, (BNU_CHUNK_T)idx); + BNU_CHUNK_T mask1 = cpIsEqu_ct(a1, (BNU_CHUNK_T)idx); + BNU_CHUNK_T mask2 = cpIsEqu_ct(a2, (BNU_CHUNK_T)idx); + BNU_CHUNK_T mask3 = cpIsEqu_ct(a3, (BNU_CHUNK_T)idx); + MaskedXorBlock16(t5, pPrecomputeData +256*nw +idx, t5, mask1); + MaskedXorBlock16(t4, pPrecomputeData +256*nw +idx, t4, mask0); + MaskedXorBlock16(t3, pPrecomputeData +256*nw +idx, t3, mask3); + MaskedXorBlock16(t2, pPrecomputeData +256*nw +idx, t2, mask2); + } + } + + XorBlock(t2 + 1, t3, t2 + 1, BLOCK_SIZE - 1); + XorBlock(t5 + 1, t2, t5 + 1, BLOCK_SIZE - 1); + XorBlock(t4 + 1, t5, t4 + 1, BLOCK_SIZE - 1); + + nw = t3[BLOCK_SIZE - 1]; + { + //a = (Ipp32u)AesGcmConst_table[nw]; + Ipp32u a = (Ipp32u)getAesGcmConst_table_ct(nw); + a <<= 8; + nw = t2[BLOCK_SIZE - 1]; + //a ^= (Ipp32u)AesGcmConst_table[nw]; + a ^= (Ipp32u)getAesGcmConst_table_ct(nw); + a <<= 8; + nw = t5[BLOCK_SIZE - 1]; + //a ^= (Ipp32u)AesGcmConst_table[nw]; + a ^= (Ipp32u)getAesGcmConst_table_ct(nw); + + XorBlock(t4, &a, t4, sizeof(Ipp32u)); + CopyBlock16(t4, pGhash); + } +} + +#endif + +#if ((_IPP>=_IPP_V8) || (_IPP32E>=_IPP32E_N8)) + +__INLINE Ipp16u getAesGcmConst_table_ct(int idx) +{ + /* init current indexes */ + __ALIGN16 Ipp16u idx_start[] = { 0,1,2,3,4,5,6,7 }; + __m128i idx_curr = _mm_load_si128((__m128i*)idx_start); + /* indexes step */ + __m128i idx_step = _mm_set1_epi16(sizeof(__m128i) / sizeof(AesGcmConst_table[0])); + /* broadcast idx */ + __m128i idx_bcst = _mm_set1_epi16((Ipp16s)idx); + + /* init accumulator */ + __m128i acc = _mm_setzero_si128(); + + int i; + for (i = 0; i < (int)sizeof(AesGcmConst_table); i += sizeof(__m128i)) { + /* read 16 entries of AesGcmConst_table[] */ + __m128i tbl = _mm_load_si128((__m128i*)((Ipp8u*)AesGcmConst_table + i)); + /* set mask if idx==idx_curr[] */ + __m128i mask = _mm_cmpeq_epi16(idx_bcst, idx_curr); + mask = _mm_and_si128(mask, tbl); + /* accumulates masked */ + acc = _mm_or_si128(acc, mask); + /* ad advance idx_curr[] indexes */ + idx_curr = _mm_add_epi16(idx_curr, idx_step); + } + + /* shift accumulator to get AesGcmConst_table[idx] in low word */ + acc = _mm_or_si128(acc, _mm_srli_si128(acc, sizeof(__m128i) / 2)); /* pack result into dword */ + acc = _mm_or_si128(acc, _mm_srli_si128(acc, sizeof(__m128i) / 4)); + acc = _mm_or_si128(acc, _mm_srli_si128(acc, sizeof(__m128i) / 8)); + i = _mm_cvtsi128_si32(acc); + + return (Ipp16u)i; +} + +IPP_OWN_DEFN (void, AesGcmMulGcm_table2K_ct, (Ipp8u* pHash, const Ipp8u* pPrecomputedData, const void* pParam)) +{ + __m128i t5 = _mm_setzero_si128(); + __m128i t4 = _mm_setzero_si128(); + __m128i t3 = _mm_setzero_si128(); + __m128i t2 = _mm_setzero_si128(); + + IPP_UNREFERENCED_PARAMETER(pParam); + + { + int nw; + for (nw = 0; nw < 4; nw++) { + Ipp32u hashdw = ((Ipp32u*)pHash)[nw]; + + Ipp32u a = hashdw & 0xf0f0f0f0; + Ipp32u a0 = EBYTE(a, 0); + Ipp32u a1 = EBYTE(a, 1); + Ipp32u a2 = EBYTE(a, 2); + Ipp32u a3 = EBYTE(a, 3); + int idx; + for (idx = 0; idx < 256; idx += 16) { + __m128i mask0 = _mm_set1_epi32((Ipp32s)cpIsEqu_ct(a0, (BNU_CHUNK_T)idx)); + __m128i mask1 = _mm_set1_epi32((Ipp32s)cpIsEqu_ct(a1, (BNU_CHUNK_T)idx)); + __m128i mask2 = _mm_set1_epi32((Ipp32s)cpIsEqu_ct(a2, (BNU_CHUNK_T)idx)); + __m128i mask3 = _mm_set1_epi32((Ipp32s)cpIsEqu_ct(a3, (BNU_CHUNK_T)idx)); + t5 = _mm_xor_si128(t5, _mm_and_si128(mask1, _mm_load_si128((__m128i*)(pPrecomputedData + 1024 + 256 * nw + idx)))); + t4 = _mm_xor_si128(t4, _mm_and_si128(mask0, _mm_load_si128((__m128i*)(pPrecomputedData + 1024 + 256 * nw + idx)))); + t3 = _mm_xor_si128(t3, _mm_and_si128(mask3, _mm_load_si128((__m128i*)(pPrecomputedData + 1024 + 256 * nw + idx)))); + t2 = _mm_xor_si128(t2, _mm_and_si128(mask2, _mm_load_si128((__m128i*)(pPrecomputedData + 1024 + 256 * nw + idx)))); + } + + a = (hashdw << 4) & 0xf0f0f0f0; + a0 = EBYTE(a, 0); + a1 = EBYTE(a, 1); + a2 = EBYTE(a, 2); + a3 = EBYTE(a, 3); + for (idx = 0; idx < 256; idx += 16) { + __m128i mask0 = _mm_set1_epi32((Ipp32s)cpIsEqu_ct(a0, (BNU_CHUNK_T)idx)); + __m128i mask1 = _mm_set1_epi32((Ipp32s)cpIsEqu_ct(a1, (BNU_CHUNK_T)idx)); + __m128i mask2 = _mm_set1_epi32((Ipp32s)cpIsEqu_ct(a2, (BNU_CHUNK_T)idx)); + __m128i mask3 = _mm_set1_epi32((Ipp32s)cpIsEqu_ct(a3, (BNU_CHUNK_T)idx)); + t5 = _mm_xor_si128(t5, _mm_and_si128(mask1, _mm_load_si128((__m128i*)(pPrecomputedData + 256 * nw + idx)))); + t4 = _mm_xor_si128(t4, _mm_and_si128(mask0, _mm_load_si128((__m128i*)(pPrecomputedData + 256 * nw + idx)))); + t3 = _mm_xor_si128(t3, _mm_and_si128(mask3, _mm_load_si128((__m128i*)(pPrecomputedData + 256 * nw + idx)))); + t2 = _mm_xor_si128(t2, _mm_and_si128(mask2, _mm_load_si128((__m128i*)(pPrecomputedData + 256 * nw + idx)))); + } + } + + { + Ipp32u a; + t2 = _mm_xor_si128(t2, _mm_slli_si128(t3, 1)); + t5 = _mm_xor_si128(t5, _mm_slli_si128(t2, 1)); + t4 = _mm_xor_si128(t4, _mm_slli_si128(t5, 1)); + + nw = _mm_cvtsi128_si32(_mm_srli_si128(t3, 15)); + a = (Ipp32u)getAesGcmConst_table_ct(nw); + a <<= 8; + + nw = _mm_cvtsi128_si32(_mm_srli_si128(t2, 15)); + a ^= (Ipp32u)getAesGcmConst_table_ct(nw); + a <<= 8; + + nw = _mm_cvtsi128_si32(_mm_srli_si128(t5, 15)); + a ^= (Ipp32u)getAesGcmConst_table_ct(nw); + + t2 = _mm_cvtsi32_si128((Ipp32s)a); + t4 = _mm_xor_si128(t4, t2); + _mm_storeu_si128((__m128i*)pHash, t4); + } + } +} +#endif + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesgcmtbl2k_precom.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesgcmtbl2k_precom.c new file mode 100644 index 0000000..5d81607 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesgcmtbl2k_precom.c @@ -0,0 +1,93 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Encrypt/Decrypt byte data stream according to Rijndael128 (GCM mode) +// +// "fast" stuff +// +// Contents: +// AesGcmPrecompute_table2K() +// +*/ + + +#include "owndefs.h" +#include "owncp.h" + +#include "pcpaesauthgcm.h" +#include "pcptool.h" + +#if(_IPP32E<_IPP32E_K0) + +/* +// AES-GCM precomputations. +*/ +static void RightShiftBlock16(Ipp8u* pBlock) +{ + Ipp8u v0 = 0; + int i; + for(i=0; i<16; i++) { + Ipp8u v1 = pBlock[i]; + Ipp8u tmp = (Ipp8u)( (v1>>1) | (v0<<7) ); + pBlock[i] = tmp; + v0 = v1; + } +} + +IPP_OWN_DEFN (void, AesGcmPrecompute_table2K, (Ipp8u* pPrecomputeData, const Ipp8u* pHKey)) +{ + Ipp8u t[BLOCK_SIZE]; + int n; + + CopyBlock16(pHKey, t); + + for(n=0; n<128-24; n++) { + /* get msb */ + int hBit = t[15]&1; + + int k = n%32; + if(k<4) { + CopyBlock16(t, pPrecomputeData +1024 +(n/32)*256 +(Ipp32u)(1<<(7-k))); + } + else if(k<8) { + CopyBlock16(t, pPrecomputeData +(n/32)*256 +(Ipp32u)(1<<(11-k))); + } + + /* shift */ + RightShiftBlock16(t); + /* xor if msb=1 */ + if(hBit) + t[0] ^= 0xe1; + } + + for(n=0; n<4; n++) { + int m, k; + XorBlock16(pPrecomputeData +n*256, pPrecomputeData +n*256, pPrecomputeData +n*256); + XorBlock16(pPrecomputeData +1024 +n*256, pPrecomputeData +1024 +n*256, pPrecomputeData +1024 +n*256); + for(m=2; m<=8; m*=2) + for(k=1; k=_IPP_G9) || \ +// (_IPP32E==_IPP32E_U8) || (_IPP32E==_IPP32E_Y8) || \ +// (_IPP32E==_IPP32E_N8) || (_IPP32E>=_IPP32E_E9)) +#if 0 +IPP_OWN_DEFN (void, AesGcmAuth_table2K, (Ipp8u* pHash, const Ipp8u* pSrc, int len, const Ipp8u* pHKey, const void* pParam)) +{ + IPP_UNREFERENCED_PARAMETER(pParam); + + while(len>=BLOCK_SIZE) { + /* add src */ + XorBlock16(pSrc, pHash, pHash); + /* hash it */ + AesGcmMulGcm_table2K(pHash, pHKey, AesGcmConst_table); + + pSrc += BLOCK_SIZE; + len -= BLOCK_SIZE; + } +} +#endif + +#if(_IPP32E<_IPP32E_K0) + +IPP_OWN_DEFN (void, AesGcmAuth_table2K_ct, (Ipp8u* pHash, const Ipp8u* pSrc, int len, const Ipp8u* pHKey, const void* pParam)) +{ + IPP_UNREFERENCED_PARAMETER(pParam); + + while (len >= BLOCK_SIZE) { + /* add src */ + XorBlock16(pSrc, pHash, pHash); + /* hash it */ + AesGcmMulGcm_table2K_ct(pHash, pHKey, AesGcmConst_table); + + pSrc += BLOCK_SIZE; + len -= BLOCK_SIZE; + } +} + +#endif + +//#endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesgcmtbl2kca_decpx.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesgcmtbl2kca_decpx.c new file mode 100644 index 0000000..bc5b37b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesgcmtbl2kca_decpx.c @@ -0,0 +1,80 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Encrypt/Decrypt byte data stream according to Rijndael128 (GCM mode) +// +// "fast" stuff +// +// Contents: +// wrpAesGcmDec_table2K() +// +*/ + + +#include "owndefs.h" +#include "owncp.h" + +#include "pcpaesauthgcm.h" +#include "pcptool.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + +#if(_IPP32E<_IPP32E_K0) + +/* +// authenticates and decrypts n*BLOCK_SIZE bytes +*/ +IPP_OWN_DEFN (void, wrpAesGcmDec_table2K, (Ipp8u* pDst, const Ipp8u* pSrc, int len, IppsAES_GCMState* pState)) +{ + //AesGcmAuth_table2K(AESGCM_GHASH(pState), pSrc, len, AESGCM_HKEY(pState), AesGcmConst_table); + AesGcmAuth_table2K_ct(AESGCM_GHASH(pState), pSrc, len, AESGCM_HKEY(pState), AesGcmConst_table); + + { + Ipp8u* pCounter = AESGCM_COUNTER(pState); + Ipp8u* pECounter = AESGCM_ECOUNTER(pState); + + IppsRijndael128Spec* pAES = AESGCM_CIPHER(pState); + RijnCipher encoder = RIJ_ENCODER(pAES); + + while(len>=BLOCK_SIZE) { + /* encrypt whole AES block */ + XorBlock16(pSrc, pECounter, pDst); + + pSrc += BLOCK_SIZE; + pDst += BLOCK_SIZE; + len -= BLOCK_SIZE; + + /* increment counter block */ + IncrementCounter32(pCounter); + /* and encrypt counter */ + //encoder((Ipp32u*)pCounter, (Ipp32u*)pECounter, RIJ_NR(pAES), RIJ_EKEYS(pAES), (const Ipp32u (*)[256])RIJ_ENC_SBOX(pAES)); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder(pCounter, pECounter, RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder(pCounter, pECounter, RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif + } + } +} + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesgcmtbl2kca_encpx.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesgcmtbl2kca_encpx.c new file mode 100644 index 0000000..83092ad --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesgcmtbl2kca_encpx.c @@ -0,0 +1,81 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Encrypt/Decrypt byte data stream according to Rijndael128 (GCM mode) +// +// "fast" stuff +// +// Contents: +// wrpAesGcmEnc_table2K() +// +*/ + + +#include "owndefs.h" +#include "owncp.h" + +#include "pcpaesauthgcm.h" +#include "pcptool.h" + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +# include "pcprijtables.h" +#endif + +#if(_IPP32E<_IPP32E_K0) + +/* +// encrypts and authenticates n*BLOCK_SIZE bytes +*/ +IPP_OWN_DEFN (void, wrpAesGcmEnc_table2K, (Ipp8u* pDst, const Ipp8u* pSrc, int len, IppsAES_GCMState* pState)) +{ + Ipp8u* pHashedData = pDst; + int hashedDataLen = len; + + Ipp8u* pCounter = AESGCM_COUNTER(pState); + Ipp8u* pECounter = AESGCM_ECOUNTER(pState); + + IppsRijndael128Spec* pAES = AESGCM_CIPHER(pState); + RijnCipher encoder = RIJ_ENCODER(pAES); + + while(len>=BLOCK_SIZE) { + /* encrypt whole AES block */ + XorBlock16(pSrc, pECounter, pDst); + + pSrc += BLOCK_SIZE; + pDst += BLOCK_SIZE; + len -= BLOCK_SIZE; + + /* increment counter block */ + IncrementCounter32(pCounter); + /* and encrypt counter */ + //encoder((Ipp32u*)pCounter, (Ipp32u*)pECounter, RIJ_NR(pAES), RIJ_EKEYS(pAES), (const Ipp32u (*)[256])RIJ_ENC_SBOX(pAES)); + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + encoder(pCounter, pECounter, RIJ_NR(pAES), RIJ_EKEYS(pAES), RijEncSbox/*NULL*/); + #else + encoder(pCounter, pECounter, RIJ_NR(pAES), RIJ_EKEYS(pAES), NULL); + #endif + } + + //AesGcmAuth_table2K(AESGCM_GHASH(pState), pHashedData, hashedDataLen, AESGCM_HKEY(pState), AesGcmConst_table); + AesGcmAuth_table2K_ct(AESGCM_GHASH(pState), pHashedData, hashedDataLen, AESGCM_HKEY(pState), AesGcmConst_table); +} + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesgetsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesgetsize.c new file mode 100644 index 0000000..2bb59ef --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesgetsize.c @@ -0,0 +1,56 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Initialization of AES +// +// Contents: +// ippsAESGetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcprij128safe.h" +#include "pcptool.h" + +/*F* +// Name: ippsAESGetSize +// +// Purpose: Returns size of AES context (in bytes). +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to AES size of context(in bytes) +// +*F*/ +IPPFUN(IppStatus, ippsAESGetSize,(int* pSize)) +{ + /* test size's pointer */ + IPP_BAD_PTR1_RET(pSize); + + *pSize = cpSizeofCtx_AES(); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesm.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesm.h new file mode 100644 index 0000000..8954683 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesm.h @@ -0,0 +1,80 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Internal Definitions and +// Internal AES Function Prototypes +// +// +*/ + +#if !defined(_PCP_AES_H) +#define _PCP_AES_H + +#include "pcprij.h" + +/* Intel(R) AES New Instructions (Intel(R) AES-NI) flag */ +#define AES_NI_ENABLED (ippCPUID_AES) + +/* alignment of AES context */ +#define AES_ALIGNMENT (RIJ_ALIGNMENT) + +/* valid AES context ID */ +#define VALID_AES_ID(ctx) ((((ctx)->idCtx) ^ (Ipp32u)IPP_UINT_PTR((ctx))) == (Ipp32u)idCtxRijndael) + +/* number of rounds (use [NK] for access) */ +static int rij128nRounds[3] = {NR128_128, NR128_192, NR128_256}; + +/* +// number of keys (estimation only!) (use [NK] for access) +// +// accurate number of keys necassary for encrypt/decrypt are: +// nKeys = NB * (NR+1) +// where NB - data block size (32-bit words) +// NR - number of rounds (depend on NB and keyLen) +// +// but the estimation +// estnKeys = (NK*n) >= nKeys +// or +// estnKeys = ( (NB*(NR+1) + (NK-1)) / NK) * NK +// where NK - key length (words) +// NB - data block size (word) +// NR - number of rounds (depend on NB and keyLen) +// nKeys - accurate numner of keys +// is more convinient when calculates key extension +*/ +static int rij128nKeys[3] = {44, 52, 60 }; + +/* +// helper for nRounds[] and estnKeys[] access +// note: x is length in 32-bits words +*/ +__INLINE int rij_index(int x) +{ + return (x-NB(128))>>1; +} + +/* size of AES context */ +__INLINE int cpSizeofCtx_AES(void) +{ + return sizeof(IppsAESSpec); +} + +#endif /* _PCP_AES_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesminit_internal.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesminit_internal.c new file mode 100644 index 0000000..1aaaaab --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesminit_internal.c @@ -0,0 +1,172 @@ +/******************************************************************************* +* Copyright (C) 2023 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Initialization functions for internal methods and pointers +// AES cipher context +// AES-GCM context +// +*/ + +#include "pcpaesminit_internal.h" +#include "aes_gcm_avx512.h" +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcptool.h" + +#if (_IPP32E >= _IPP32E_K0) +#include "pcpaesauthgcm_avx512.h" +#else +#include "pcpaesauthgcm.h" +#endif /* #if(_IPP32E>=_IPP32E_K0) */ + +/* + * This function set up pointers to encryption and decryption key schedules, + * dispatches to the right internal methods and sets pointers to them inside the AES state. + */ +IPP_OWN_DEFN(void, cpAes_setup_ptrs_and_methods, (IppsAESSpec * pCtx)) +{ + int nExpKeys = rij128nKeys[rij_index(RIJ_NK(pCtx))]; + + RIJ_EKEYS(pCtx) = (Ipp8u *)(IPP_ALIGNED_PTR(RIJ_KEYS_BUFFER(pCtx), AES_ALIGNMENT)); + RIJ_DKEYS(pCtx) = (Ipp8u *)((Ipp32u *)RIJ_EKEYS(pCtx) + nExpKeys); + +#if (_AES_NI_ENABLING_ == _FEATURE_ON_) + RIJ_AESNI(pCtx) = AES_NI_ENABLED; + RIJ_ENCODER(pCtx) = Encrypt_RIJ128_AES_NI; /* AES_NI based encoder */ + RIJ_DECODER(pCtx) = Decrypt_RIJ128_AES_NI; /* AES_NI based decoder */ +#else +#if (_AES_NI_ENABLING_ == _FEATURE_TICKTOCK_) + if (IsFeatureEnabled(ippCPUID_AES) || IsFeatureEnabled(ippCPUID_AVX2VAES)) { + RIJ_AESNI(pCtx) = AES_NI_ENABLED; + RIJ_ENCODER(pCtx) = Encrypt_RIJ128_AES_NI; /* AES_NI based encoder */ + RIJ_DECODER(pCtx) = Decrypt_RIJ128_AES_NI; /* AES_NI based decoder */ + } else +#endif + { +#if (_ALG_AES_SAFE_ == _ALG_AES_SAFE_COMPOSITE_GF_) + { + RIJ_ENCODER(pCtx) = SafeEncrypt_RIJ128; /* safe encoder (composite GF) */ + RIJ_DECODER(pCtx) = SafeDecrypt_RIJ128; /* safe decoder (composite GF)*/ + } +#else + { + RIJ_ENCODER(pCtx) = Safe2Encrypt_RIJ128; /* safe encoder (compact Sbox)) */ + RIJ_DECODER(pCtx) = Safe2Decrypt_RIJ128; /* safe decoder (compact Sbox)) */ + } +#endif + } +#endif +} + +/* + * This function dispatches to the right internal methods and sets pointers to them inside the AES-GCM state. + */ +IPP_OWN_DEFN(void, cpAesGCM_setup_ptrs_and_methods, (IppsAES_GCMState * pState, Ipp64u keyByteLen)) +{ +#if (_IPP32E >= _IPP32E_K0) + if (IsFeatureEnabled(ippCPUID_AVX512VAES) && IsFeatureEnabled(ippCPUID_AVX512VCLMUL)) { + switch (keyByteLen) { + case 16: + AES_GCM_ENCRYPT_UPDATE(pState) = aes_gcm_enc_128_update_vaes_avx512; + AES_GCM_DECRYPT_UPDATE(pState) = aes_gcm_dec_128_update_vaes_avx512; + AES_GCM_GET_TAG(pState) = aes_gcm_gettag_128_vaes_avx512; + break; + case 24: + AES_GCM_ENCRYPT_UPDATE(pState) = aes_gcm_enc_192_update_vaes_avx512; + AES_GCM_DECRYPT_UPDATE(pState) = aes_gcm_dec_192_update_vaes_avx512; + AES_GCM_GET_TAG(pState) = aes_gcm_gettag_192_vaes_avx512; + break; + case 32: + AES_GCM_ENCRYPT_UPDATE(pState) = aes_gcm_enc_256_update_vaes_avx512; + AES_GCM_DECRYPT_UPDATE(pState) = aes_gcm_dec_256_update_vaes_avx512; + AES_GCM_GET_TAG(pState) = aes_gcm_gettag_256_vaes_avx512; + break; + } + + AES_GCM_IV_UPDATE(pState) = aes_gcm_iv_hash_update_vaes512; + AES_GCM_IV_FINALIZE(pState) = aes_gcm_iv_hash_finalize_vaes512; + AES_GCM_AAD_UPDATE(pState) = aes_gcm_aad_hash_update_vaes512; + AES_GCM_GMUL(pState) = aes_gcm_gmult_vaes512; + } else { + switch (keyByteLen) { + case 16: + AES_GCM_ENCRYPT_UPDATE(pState) = aes_gcm_enc_128_update_avx512; + AES_GCM_DECRYPT_UPDATE(pState) = aes_gcm_dec_128_update_avx512; + AES_GCM_GET_TAG(pState) = aes_gcm_gettag_128_avx512; + break; + case 24: + AES_GCM_ENCRYPT_UPDATE(pState) = aes_gcm_enc_192_update_avx512; + AES_GCM_DECRYPT_UPDATE(pState) = aes_gcm_dec_192_update_avx512; + AES_GCM_GET_TAG(pState) = aes_gcm_gettag_192_avx512; + break; + case 32: + AES_GCM_ENCRYPT_UPDATE(pState) = aes_gcm_enc_256_update_avx512; + AES_GCM_DECRYPT_UPDATE(pState) = aes_gcm_dec_256_update_avx512; + AES_GCM_GET_TAG(pState) = aes_gcm_gettag_256_avx512; + break; + } + + AES_GCM_IV_UPDATE(pState) = aes_gcm_iv_hash_update_avx512; + AES_GCM_IV_FINALIZE(pState) = aes_gcm_iv_hash_finalize_avx512; + AES_GCM_AAD_UPDATE(pState) = aes_gcm_aad_hash_update_avx512; + AES_GCM_GMUL(pState) = aes_gcm_gmult_avx512; + } +#else + IPP_UNREFERENCED_PARAMETER(keyByteLen); + + /* set up: + // - ghash function + // - authentication function + */ + AESGCM_HASH(pState) = AesGcmMulGcm_table2K_ct; // AesGcmMulGcm_table2K; + AESGCM_AUTH(pState) = AesGcmAuth_table2K_ct; // AesGcmAuth_table2K; + AESGCM_ENC(pState) = wrpAesGcmEnc_table2K; + AESGCM_DEC(pState) = wrpAesGcmDec_table2K; + +#if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) +// the dead code that currently is unused +//#if (_IPP32E >= _IPP32E_K0) +// if (IsFeatureEnabled(ippCPUID_AVX512VAES)) { +// AESGCM_HASH(pState) = AesGcmMulGcm_vaes; +// AESGCM_AUTH(pState) = AesGcmAuth_vaes; +// AESGCM_ENC(pState) = AesGcmEnc_vaes; +// AESGCM_DEC(pState) = AesGcmDec_vaes; +// } else +//#endif /* #if(_IPP32E>=_IPP32E_K0) */ + if (IsFeatureEnabled(ippCPUID_AES | ippCPUID_CLMUL)) { + AESGCM_HASH(pState) = AesGcmMulGcm_avx; + AESGCM_AUTH(pState) = AesGcmAuth_avx; + AESGCM_ENC(pState) = wrpAesGcmEnc_avx; + AESGCM_DEC(pState) = wrpAesGcmDec_avx; + } +#if (_IPP==_IPP_H9) || (_IPP32E==_IPP32E_L9) + if (IsFeatureEnabled(ippCPUID_AVX2VAES | ippCPUID_AVX2VCLMUL)) { + AESGCM_HASH(pState) = AesGcmMulGcm_avx; + AESGCM_AUTH(pState) = AesGcmAuth_avx; + AESGCM_ENC(pState) = AesGcmEnc_vaes_avx2; + AESGCM_DEC(pState) = AesGcmDec_vaes_avx2; + } +#endif /* #if(_IPP==_IPP_H9) || (_IPP32E==_IPP32E_L9) */ +#endif /* #if(_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) */ + +#endif /* #if(_IPP32E>=_IPP32E_K0) */ +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesminit_internal.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesminit_internal.h new file mode 100644 index 0000000..55985ce --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesminit_internal.h @@ -0,0 +1,39 @@ +/******************************************************************************* +* Copyright (C) 2023 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Initialization functions for internal methods and pointers +// AES cipher context +// AES-GCM context +// +*/ + +#if !defined(_PCP_AES_INIT_INTERNAL_H) +#define _PCP_AES_INIT_INTERNAL_H + +#include "owndefs.h" + +#define cpAes_setup_ptrs_and_methods OWNAPI(cpAes_setup_ptrs_and_methods) +IPP_OWN_DECL(void, cpAes_setup_ptrs_and_methods, (IppsAESSpec * pCtx)) + +#define cpAesGCM_setup_ptrs_and_methods OWNAPI(cpAesGCM_setup_ptrs_and_methods) +IPP_OWN_DECL(void, cpAesGCM_setup_ptrs_and_methods, (IppsAES_GCMState * pCtx, Ipp64u keyByteLen)) + +#endif /* _PCP_AES_INIT_INTERNAL_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesminitca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesminitca.c new file mode 100644 index 0000000..65d4500 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesminitca.c @@ -0,0 +1,146 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Initialization of AES +// +// Contents: +// ippsAESInit() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcprij128safe.h" +#include "pcptool.h" +#include "pcpaesminit_internal.h" + +/*F* +// Name: ippsAESInit +// +// Purpose: Init AES context for future usage +// and setup secret key. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// ippStsMemAllocErr size of buffer is not match for operation +// ippStsLengthErr keyLen != 16 +// keyLen != 24 +// keyLen != 32 +// ippStsNoErr no errors +// +// Parameters: +// pKey secret key +// keyLen length of the secret key (in bytes) +// pCtx pointer to buffer initialized as AES context +// ctxSize available size (in bytes) of buffer above +// +// Note: +// if pKey==NULL, then AES initialized by zero value key +// +*F*/ +IPPFUN(IppStatus, ippsAESInit,(const Ipp8u* pKey, int keyLen, + IppsAESSpec* pCtx, int ctxSize)) +{ + /* test context pointer */ + IPP_BAD_PTR1_RET(pCtx); + + /* make sure in legal keyLen */ + IPP_BADARG_RET(keyLen!=16 && keyLen!=24 && keyLen!=32, ippStsLengthErr); + + IPP_BADARG_RET(((Ipp8u*)pCtx+sizeof(IppsAESSpec)) > ((Ipp8u*)pCtx+ctxSize), ippStsMemAllocErr); + + { + int keyWords = NK(keyLen*BITSIZE(Ipp8u)); + int nExpKeys = rij128nKeys [ rij_index(keyWords) ]; + int nRounds = rij128nRounds[ rij_index(keyWords) ]; + + Ipp8u zeroKey[32] = {0}; + const Ipp8u* pActualKey = pKey? pKey : zeroKey; + + /* clear context */ + PadBlock(0, pCtx, sizeof(IppsAESSpec)); + + /* init spec */ + /* light trick to prevent context copy: add low 32-bit part of its current address to the context Id and + * check if is changed later in processing functions */ + RIJ_SET_ID(pCtx); + RIJ_NB(pCtx) = NB(128); + RIJ_NK(pCtx) = keyWords; + RIJ_NR(pCtx) = nRounds; + RIJ_SAFE_INIT(pCtx) = 1; + +#if (_AES_PROB_NOISE == _FEATURE_ON_) + /* Reset AES noise parameters */ + cpAESNoiseParams *params = (cpAESNoiseParams *)&RIJ_NOISE_PARAMS(pCtx); + + AES_NOISE_RAND(params) = 0; + AES_NOISE_LEVEL(params) = 0; +#endif + + cpAes_setup_ptrs_and_methods(pCtx); + + #if (_AES_NI_ENABLING_==_FEATURE_ON_) + cpExpandAesKey_NI(pActualKey, pCtx); /* AES_NI based key expansion */ + #else + #if (_AES_NI_ENABLING_==_FEATURE_TICKTOCK_) + if(IsFeatureEnabled(ippCPUID_AES) || IsFeatureEnabled(ippCPUID_AVX2VAES)) { + cpExpandAesKey_NI(pActualKey, pCtx); /* AES_NI based key expansion */ + } + else + #endif + { + ExpandRijndaelKey(pActualKey, keyWords, NB(128), nRounds, nExpKeys, + RIJ_EKEYS(pCtx), + RIJ_DKEYS(pCtx)); + + #if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPOSITE_GF_) + { + int nr; + Ipp8u* pEnc_key = (Ipp8u*)(RIJ_EKEYS(pCtx)); + Ipp8u* pDec_key = (Ipp8u*)(RIJ_DKEYS(pCtx)); + /* update key material: convert into GF((2^4)2) */ + for(nr=0; nr<(1+nRounds); nr++) { + TransformNative2Composite(pEnc_key+16*nr, pEnc_key+16*nr); + TransformNative2Composite(pDec_key+16*nr, pDec_key+16*nr); + } + } + #else + { + int nr; + Ipp8u* pEnc_key = (Ipp8u*)(RIJ_EKEYS(pCtx)); + /* update key material: transpose inplace */ + for(nr=0; nr<(1+nRounds); nr++, pEnc_key+=16) { + SWAP(pEnc_key[ 1], pEnc_key[ 4]); + SWAP(pEnc_key[ 2], pEnc_key[ 8]); + SWAP(pEnc_key[ 3], pEnc_key[12]); + SWAP(pEnc_key[ 6], pEnc_key[ 9]); + SWAP(pEnc_key[ 7], pEnc_key[13]); + SWAP(pEnc_key[11], pEnc_key[14]); + } + } + #endif + } + #endif + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesmxts.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesmxts.h new file mode 100644 index 0000000..dac0464 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesmxts.h @@ -0,0 +1,79 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// AES-XTS Internal Definitions +// +// +*/ + +#if !defined(_PCP_AES_XTS_H) +#define _PCP_AES_XTS_H + +#include "owncp.h" +#include "pcpaesm.h" + +#define _NEW_XTS_API_3 +#if defined (_NEW_XTS_API_3) +/* +// AES-XTS State +*/ +#if !defined(AES_BLK_SIZE) +#define AES_BLK_SIZE (IPP_AES_BLOCK_BITSIZE/BITSIZE(Ipp8u)) +#endif + +#define AES_BLKS_PER_BUFFER (32) + +struct _cpAES_XTS +{ + Ipp32u idCtx; + int duBitsize; /* size of data unit (in bits) */ + IppsAESSpec datumAES; /* datum AES context */ + IppsAESSpec tweakAES; /* tweak AES context */ +}; + +#define AES_XTS_SET_ID(ctx) ((ctx)->idCtx = (Ipp32u)idCtxAESXTS ^ (Ipp32u)IPP_UINT_PTR(ctx)) +/* valid AES_XTS context ID */ +#define VALID_AES_XTS_ID(ctx) ((((ctx)->idCtx) ^ (Ipp32u)IPP_UINT_PTR((ctx))) == (Ipp32u)idCtxAESXTS) + +/* size of AES-XTS context */ +__INLINE int cpSizeof_AES_XTS_Ctx(void) +{ + return sizeof(IppsAES_XTSSpec); +} + +static int IsLegalGeometry(int startCipherBlkNo, int bitLen, int duBitsize) +{ + int duBlocks = (duBitsize+IPP_AES_BLOCK_BITSIZE-1)/IPP_AES_BLOCK_BITSIZE; + int legalBlk = (0<=startCipherBlkNo && startCipherBlkNo> 63) & GF_POLY; + Ipp64u addH = ((Ipp64s)x64[0] >> 63) & 1; + x64[0] = (x64[0]+x64[0]) ^ xorL; + x64[1] = (x64[1]+x64[1]) + addH; +} + +/* + the following are especially for multi-block processing +*/ +static void cpXTSwhitening(Ipp8u* buffer, int nblk, Ipp8u* ptwk) +{ + Ipp64u* pbuf64 = (Ipp64u*)buffer; + Ipp64u* ptwk64 = (Ipp64u*)ptwk; + + pbuf64[0] = ptwk64[0]; + pbuf64[1] = ptwk64[1]; + + for(nblk--, pbuf64+=2; nblk>0; nblk--, pbuf64+=2) { + gf_mul_by_primitive(ptwk64); + pbuf64[0] = ptwk64[0]; + pbuf64[1] = ptwk64[1]; + } + gf_mul_by_primitive(ptwk64); +} + +static void cpXTSxor16(Ipp8u* pDst, const Ipp8u* pSrc1, const Ipp8u* pSrc2, int nblk) +{ + Ipp64u* pdst64 = (Ipp64u*)pDst; + const Ipp64u* ps1_64 = (const Ipp64u*)pSrc1; + const Ipp64u* ps2_64 = (const Ipp64u*)pSrc2; + for(; nblk>0; nblk--, pdst64+=2, ps1_64+=2, ps2_64+=2) { + pdst64[0] = ps1_64[0] ^ ps2_64[0]; + pdst64[1] = ps1_64[1] ^ ps2_64[1]; + } +} + +#endif /* _PCP_AES_XTS_STUFF_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesnoise.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesnoise.h new file mode 100644 index 0000000..f783446 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaesnoise.h @@ -0,0 +1,61 @@ +/******************************************************************************* +* Copyright 2022 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#if !defined(_PCP_AES_NOISE_H) +#define _PCP_AES_NOISE_H + +/* + * The parameters below are empirical and choosen in advance to guarantee + * the high level of security protection against Mistletoe3 attack. + */ +#define MISTLETOE3_MAX_CHUNK_SIZE (16000) /* maximum chunks size allowed to be processed without noise injection (in bytes) \ + 16000 bytes = 16*1000 bytes = 1000 AES blocks */ +#define MISTLETOE3_TARGET_SIZE (800000000) /* expected sampling interval of the attacker. During the attack \ + the adversary measures how long it takes to encrypt MISTLETOE3_TARGET_SIZE of bytes. \ + MISTLETOE3_TARGET_SIZE is much greater than MISTLETOE3_MAX_CHUNK_SIZE \ + 800000000 bytes = 16*50000000 bytes = 50000000 AES blocks */ +#define MISTLETOE3_BASE_NOISE_LEVEL (28) /* noiseLevel adjusts the random number of what bitsize will be generated, this number will be \ + used as a parameter for _ippcpDelay function. Level of noise needed depends on many factors, such \ + as processor, code implementation, power limit setting by the attacker, etc. \ + For base noise level was selected a relatively safe value of 28, for user this parameter was \ + introduced as abstract with tunable range [0,4] */ + +#define MISTLETOE3_NOISE_RATE ((double)MISTLETOE3_MAX_CHUNK_SIZE / \ + (double)MISTLETOE3_TARGET_SIZE) + +/* Structure containing noise parameters required for Mistletoe3 mitigation */ +typedef struct _cpAESNoiseParams { + Ipp32u rnd; /* Random number value from previous noise injection */ + Ipp32u noiseLevel; /* Number of bits that should be taken from generated \ + 32-bit random value. noiseLevel == 0 -> mitigation is off */ +} cpAESNoiseParams; + +#define AES_NOISE_RAND(ctx) ((ctx)->rnd) +#define AES_NOISE_LEVEL(ctx) ((ctx)->noiseLevel) + +/* size of _cpAESNoiseParams structure */ +__INLINE int cpSizeofNoise_Params(void) +{ + return sizeof(cpAESNoiseParams); +} + +#define _ippcpDelay OWNAPI(_ippcpDelay) + IPP_OWN_DECL(void, _ippcpDelay, (Ipp32u value)) + +#define cpAESRandomNoise OWNAPI(cpAESRandomNoise) + IPP_OWN_DECL(IppStatus, cpAESRandomNoise, (IppBitSupplier rndFunc, Ipp32u nBits, Ipp64f noiseRate, Ipp32u *pRndValue)) + +#endif /* _PCP_AES_NOISE_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaespack.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaespack.c new file mode 100644 index 0000000..3236d91 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpaespack.c @@ -0,0 +1,83 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Initialization of AES +// +// Contents: +// ippsAESPack() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpaesm.h" +#include "pcprij128safe.h" +#include "pcptool.h" + +/*F* +// Name: ippsAESPack +// +// Purpose: Serialize AES context into the buffer. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// pBuffer == NULL +// ippStsContextMatchErr RIJ_ID(pCtx) != idCtxRijndael +// ippStsLengthErr avaliable size of buffer is not enough for operation +// ippStsNoErr no errors +// +// Parameters: +// pCtx pointer RIJ spec +// pBuffer pointer to the buffer +// bufsize available size of buffer above +// +*F*/ +IPPFUN(IppStatus, ippsAESPack,(const IppsAESSpec* pCtx, Ipp8u* pBuffer, int bufsize)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pCtx, pBuffer); + + /* test the context */ + IPP_BADARG_RET(!VALID_AES_ID(pCtx), ippStsContextMatchErr); + + /* test available size of destination buffer */ + IPP_BADARG_RET(bufsize=_IPP_V8) || (_IPP32E>=_IPP32E_M7)) + #define rc4word Ipp32u + +#else + #define rc4word Ipp8u +#endif + +/* +// ARCFOUR context +*/ +struct _cpARCfour { + Ipp32u idCtx; /* RC4 identifier */ + + Ipp32u cntX; /* algorithm's counter x */ + Ipp32u cntY; /* algorithm's counter y */ + rc4word Sbox[256]; /* current state block.*/ + Ipp8u Sbox0[256]; /* initial state block */ +}; + +/* alignment */ +#define RC4_ALIGNMENT ((int)(sizeof(Ipp32u))) + +/* +// Useful macros +*/ +#define RC4_SET_ID(ctx) ((ctx)->idCtx = (Ipp32u)idCtxARCFOUR ^ (Ipp32u)IPP_UINT_PTR(ctx)) +#define RC4_RESET_ID(ctx) ((ctx)->idCtx = (Ipp32u)idCtxARCFOUR) +#define RC4_CNTX(ctx) ((ctx)->cntX) +#define RC4_CNTY(ctx) ((ctx)->cntY) +#define RC4_SBOX(ctx) ((ctx)->Sbox) +#define RC4_SBOX0(ctx) ((ctx)->Sbox0) + +#define RC4_VALID_ID(ctx) ((((ctx)->idCtx) ^ (Ipp32u)IPP_UINT_PTR((ctx))) == (Ipp32u)idCtxARCFOUR) + +/* +// internal functions +*/ +#define ARCFourProcessData OWNAPI(ARCFourProcessData) + IPP_OWN_DECL (void, ARCFourProcessData, (const Ipp8u *pSrc, Ipp8u *pDst, int length, IppsARCFourState *pCtx)) + +#endif /* _ARCFOUR_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfour_processdata.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfour_processdata.c new file mode 100644 index 0000000..e797fe0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfour_processdata.c @@ -0,0 +1,69 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RC4 implementation +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcparcfour.h" +#include "pcptool.h" + + +/* +// data processing function. +*/ +#if !((_IPP>=_IPP_M5) || (_IPP32E>=_IPP32E_M7)) + +IPP_OWN_DEFN (void, ARCFourProcessData, (const Ipp8u *pSrc, Ipp8u *pDst, int length, IppsARCFourState *pCtx)) +{ + if(length) { + rc4word tx, ty; + + Ipp32u x = RC4_CNTX(pCtx); + Ipp32u y = RC4_CNTY(pCtx); + rc4word* pSbox = RC4_SBOX(pCtx); + + x = (x+1) &0xFF; + tx = pSbox[x]; + + while(length) { + y = (y+tx) & 0xFF; + ty = pSbox[y]; + pSbox[x] = ty; + x = (x+1) & 0xFF; + ty = (ty+tx) & 0xFF; + pSbox[y] = tx; + tx = pSbox[x]; + ty = pSbox[ty]; + *pDst = (Ipp8u)( *pSrc ^ ty ); + pDst++; + pSrc++; + length--; + } + RC4_CNTX(pCtx) = (x-1) & 0xFF; + RC4_CNTY(pCtx) = y; + } +} + +#endif /* #if !((_IPP>=_IPP_M5) || (_IPP32E>=_IPP32E_M7)) */ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourca.c new file mode 100644 index 0000000..73a3bc0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourca.c @@ -0,0 +1,78 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RC4 implementation +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcparcfour.h" +#include "pcptool.h" + +/*F* +// Name: ippsARCFourInit +// +// Purpose: Init ARCFOUR spec for future usage. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// ippStsLengthErr 1 > keyLen +// keyLen > IPP_ARCFOUR_KEYMAX_SIZE +// ippStsNoErr no errors +// +// Parameters: +// key security key +// keyLen length of key (bytes) +// pCtx pointer to the ARCFOUR context +// +*F*/ +IPPFUN(IppStatus, ippsARCFourInit, (const Ipp8u *pKey, int keyLen, IppsARCFourState *pCtx)) +{ + /* test context pointer */ + IPP_BAD_PTR1_RET(pCtx); + + /* test key */ + IPP_BAD_PTR1_RET(pKey); + IPP_BADARG_RET(((1>keyLen)||(IPP_ARCFOUR_KEYMAX_SIZE< keyLen)), ippStsLengthErr); + + { + int i; + Ipp8u kblk[256], j, tmp; + + /* init RC4 context */ + RC4_SET_ID(pCtx); + + for(i=0; i<256; i++) { + pCtx->Sbox0[i] = (Ipp8u)i; + kblk[i] = pKey[i%keyLen]; + } + j=0; + for(i=0; i<256; i++) { + j += pCtx->Sbox0[i] + kblk[i]; + tmp = pCtx->Sbox0[j]; + pCtx->Sbox0[j] = pCtx->Sbox0[i]; + pCtx->Sbox0[i] = tmp; + } + + return ippsARCFourReset(pCtx); + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourcheckkey.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourcheckkey.c new file mode 100644 index 0000000..42ff699 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourcheckkey.c @@ -0,0 +1,67 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RC4 implementation +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcparcfour.h" +#include "pcptool.h" + +/*F* +// Name: ippsARCFourCheckKey +// +// Purpose: Checks key for weakness. +// +// Returns: Reason: +// ippStsNullPtrErr pKey== NULL +// ippStsNullPtrErr pIsWeak== NULL +// ippStsLengthErr 1 > keyLen +// keyLen > IPP_ARCFOUR_KEYMAX_SIZE +// ippStsNoErr no errors +// +// Parameters: +// key security key +// keyLen length of key (bytes) +// pIsWeak pointer to the result +// +// Note: +// See ANDREW ROOS "A CLASS OF WEAK KEYS IN THE RC4 STREAM CIPHER" +// (http://marcel.wanda.ch/Archive/WeakKeys) +*F*/ +IPPFUN(IppStatus, ippsARCFourCheckKey, (const Ipp8u *pKey, int keyLen, IppBool* pIsWeak)) +{ + /* test key */ + IPP_BAD_PTR1_RET(pKey); + IPP_BADARG_RET(((1>keyLen)||(IPP_ARCFOUR_KEYMAX_SIZE< keyLen)), ippStsLengthErr); + + /* test result*/ + IPP_BAD_PTR1_RET(pIsWeak); + + if(1==keyLen) + *pIsWeak = (pKey[0]==128)? ippTrue : ippFalse; + else + *pIsWeak = (pKey[0] + pKey[1])%256 == 0 ? ippTrue : ippFalse; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourdecrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourdecrypt.c new file mode 100644 index 0000000..10aef07 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourdecrypt.c @@ -0,0 +1,69 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Cryptography Primitive. +// RC4 implementation +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcparcfour.h" +#include "pcptool.h" + +/*F* +// Name: ippsARCFourDecrypt +// +// Purpose: Decrypt data stream. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// pSrc == NULL +// pDst == NULL +// ippStsContextMatchErr pCtx->idCtx != idCtxARCFOUR +// ippStsLengthErr length<1 +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source byte data block stream +// pDst pointer to the destination byte data block stream +// length stream length (bytes) +// pCtx pointer to the ARCFOUR context +// +// Note: +// Convenience function only +*F*/ +IPPFUN(IppStatus, ippsARCFourDecrypt, (const Ipp8u *pSrc, Ipp8u *pDst, int length, + IppsARCFourState *pCtx)) +{ + /* test context */ + IPP_BAD_PTR1_RET(pCtx); + IPP_BADARG_RET(!RC4_VALID_ID(pCtx), ippStsContextMatchErr); + + /* test source and destination pointers */ + IPP_BAD_PTR2_RET(pSrc, pDst); + /* test stream length */ + IPP_BADARG_RET((length<1), ippStsLengthErr); + + + /* process data */ + ARCFourProcessData(pSrc, pDst, length, pCtx); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourencrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourencrypt.c new file mode 100644 index 0000000..b0560b5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourencrypt.c @@ -0,0 +1,69 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RC4 implementation +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcparcfour.h" +#include "pcptool.h" + +/*F* +// Name: ippsARCFourEncrypt +// +// Purpose: Encrypt data stream. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// pSrc == NULL +// pDst == NULL +// ippStsContextMatchErr pCtx->idCtx != idCtxARCFOUR +// ippStsLengthErr length<1 +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source byte data block stream +// pDst pointer to the destination byte data block stream +// length stream length (bytes) +// pCtx ponter to the ARCFOUR context +// +// Note: +// Convenience function only +*F*/ +IPPFUN(IppStatus, ippsARCFourEncrypt, (const Ipp8u *pSrc, Ipp8u *pDst, int length, + IppsARCFourState *pCtx)) +{ + /* test context */ + IPP_BAD_PTR1_RET(pCtx); + IPP_BADARG_RET(!RC4_VALID_ID(pCtx), ippStsContextMatchErr); + + /* test source and destination pointers */ + IPP_BAD_PTR2_RET(pSrc, pDst); + /* test stream length */ + IPP_BADARG_RET((length<1), ippStsLengthErr); + + /* process data */ + ARCFourProcessData(pSrc, pDst, length, pCtx); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourgetsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourgetsize.c new file mode 100644 index 0000000..baa4305 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourgetsize.c @@ -0,0 +1,51 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcparcfour.h" +#include "pcptool.h" + +/* +// +// Purpose: +// Cryptography Primitive. +// RC4 implementation +// +*/ + +/*F* +// Name: ippsARCFourGetSize +// +// Purpose: Returns size of ARCFOUR context (bytes). +// +// Returns: Reason: +// ippStsNullPtrErr pSzie == NULL +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to the size of ARCFOUR context +// +*F*/ +IPPFUN(IppStatus, ippsARCFourGetSize, (int* pSize)) +{ + /* test size's pointer */ + IPP_BAD_PTR1_RET(pSize); + + *pSize = sizeof(IppsARCFourState); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourpack.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourpack.c new file mode 100644 index 0000000..e8e3c6a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourpack.c @@ -0,0 +1,58 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RC4 implementation +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcparcfour.h" +#include "pcptool.h" + +/*F* +// Name: ippsARCFourPack +// +// Purpose: Copy initialized context to the buffer. +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL +// pCtx == NULL +// ippStsNoErr no errors +// +// Parameters: +// pCtx pointer ARCFour spec +// pSize pointer to the packed spec size +// +*F*/ +IPPFUN(IppStatus, ippsARCFourPack,(const IppsARCFourState* pCtx, Ipp8u* pBuffer)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pCtx, pBuffer); + /* test the context */ + IPP_BADARG_RET(!RC4_VALID_ID(pCtx), ippStsContextMatchErr); + + CopyBlock(pCtx, pBuffer, sizeof(IppsARCFourState)); + IppsARCFourState* pCopy = (IppsARCFourState*)pBuffer; + RC4_RESET_ID(pCopy); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourreset.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourreset.c new file mode 100644 index 0000000..093ba28 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourreset.c @@ -0,0 +1,65 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RC4 implementation +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcparcfour.h" +#include "pcptool.h" + +/*F* +// Name: ippsARCFourReset +// +// Purpose: Resrt ARCFOUR context: +// set current state block to the initial one +// and zeroes counters +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// ippStsContextMatchErr pCtx->idCtx != idCtxARCFOUR +// ippStsNoErr no errors +// +// Parameters: +// pCtx pointer to the ARCFOUR context +// +*F*/ +IPPFUN(IppStatus, ippsARCFourReset, (IppsARCFourState* pCtx)) +{ + /* test context */ + IPP_BAD_PTR1_RET(pCtx); + IPP_BADARG_RET(!RC4_VALID_ID(pCtx), ippStsContextMatchErr); + + { + /* reset Sbox */ + int n; + for(n=0; n<256; n++) + RC4_SBOX(pCtx)[n] = RC4_SBOX0(pCtx)[n]; + + /* reset counters */ + RC4_CNTX(pCtx) = 0; + RC4_CNTY(pCtx) = 0; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourunpack.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourunpack.c new file mode 100644 index 0000000..9f9df52 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcparcfourunpack.c @@ -0,0 +1,55 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RC4 implementation +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcparcfour.h" +#include "pcptool.h" + +/*F* +// Name: ippsARCFourUnpack +// +// Purpose: Unpack buffer content into the initialized context. +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL +// pCtx == NULL +// ippStsNoErr no errors +// +// Parameters: +// pCtx pointer ARCFour spec +// pSize pointer to the packed spec size +// +*F*/ +IPPFUN(IppStatus, ippsARCFourUnpack,(const Ipp8u* pBuffer, IppsARCFourState* pCtx)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pCtx, pBuffer); + + CopyBlock(pBuffer, pCtx, sizeof(IppsARCFourState)); + RC4_SET_ID(pCtx); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbn.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbn.h new file mode 100644 index 0000000..4b47281 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbn.h @@ -0,0 +1,207 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// +// +*/ + +#if !defined(_CP_BN_H) +#define _CP_BN_H + +#include "pcpbnuimpl.h" +#include "pcpbnuarith.h" +#include "pcpbnumisc.h" +#include "pcpbnu32arith.h" +#include "pcpbnu32misc.h" + +/* +// Big Number context +*/ +struct _cpBigNum +{ + Ipp32u idCtx; /* BigNum ctx id */ + IppsBigNumSGN sgn; /* sign */ + cpSize size; /* BigNum size (BNU_CHUNK_T) */ + cpSize room; /* BigNum max size (BNU_CHUNK_T) */ + BNU_CHUNK_T* number; /* BigNum value */ + BNU_CHUNK_T* buffer; /* temporary buffer */ +}; + +/* BN accessory macros */ +#define BN_SET_ID(pBN) ((pBN)->idCtx = (Ipp32u)idCtxBigNum ^ (Ipp32u)IPP_UINT_PTR(pBN)) +#define BN_SIGN(pBN) ((pBN)->sgn) +#define BN_POSITIVE(pBN) (BN_SIGN(pBN)==ippBigNumPOS) +#define BN_NEGATIVE(pBN) (BN_SIGN(pBN)==ippBigNumNEG) +#define BN_NUMBER(pBN) ((pBN)->number) +#define BN_BUFFER(pBN) ((pBN)->buffer) +#define BN_ROOM(pBN) ((pBN)->room) +#define BN_SIZE(pBN) ((pBN)->size) +#define BN_SIZE32(pBN) ((pBN)->size*((Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(Ipp32u)))) +//#define BN_SIZE32(pBN) (BITS2WORD32_SIZE( BITSIZE_BNU(BN_NUMBER((pBN)),BN_SIZE((pBN))))) + +#define BN_VALID_ID(pBN) ((((pBN)->idCtx) ^ (Ipp32u)IPP_UINT_PTR((pBN))) == (Ipp32u)idCtxBigNum) + +#define INVERSE_SIGN(s) (((s)==ippBigNumPOS)? ippBigNumNEG : ippBigNumPOS) + +#define BN_ALIGNMENT ((int)sizeof(void*)) + + +/* pack-unpack context */ +#define cpPackBigNumCtx OWNAPI(cpPackBigNumCtx) + IPP_OWN_DECL (void, cpPackBigNumCtx, (const IppsBigNumState* pBN, Ipp8u* pBuffer)) +#define cpUnpackBigNumCtx OWNAPI(cpUnpackBigNumCtx) + IPP_OWN_DECL (void, cpUnpackBigNumCtx, (const Ipp8u* pBuffer, IppsBigNumState* pBN)) + +/* copy BN */ +__INLINE IppsBigNumState* cpBN_copy(IppsBigNumState* pDst, const IppsBigNumState* pSrc) +{ + BN_SIGN(pDst) = BN_SIGN(pSrc); + BN_SIZE(pDst) = BN_SIZE(pSrc); + ZEXPAND_COPY_BNU(BN_NUMBER(pDst), BN_ROOM(pDst), BN_NUMBER(pSrc), BN_SIZE(pSrc)); + return pDst; +} +/* set BN to zero */ +__INLINE IppsBigNumState* cpBN_zero(IppsBigNumState* pBN) +{ + BN_SIGN(pBN) = ippBigNumPOS; + BN_SIZE(pBN) = 1; + ZEXPAND_BNU(BN_NUMBER(pBN),0, (int)BN_ROOM(pBN)); + return pBN; +} +/* fixup BN */ +__INLINE IppsBigNumState* cpBN_fix(IppsBigNumState* pBN) +{ + cpSize len = BN_SIZE(pBN); + FIX_BNU(BN_NUMBER(pBN), len); + BN_SIZE(pBN) = len; + return pBN; +} +/* set BN to chunk */ +__INLINE IppsBigNumState* cpBN_chunk(IppsBigNumState* pBN, BNU_CHUNK_T a) +{ + BN_SIGN(pBN) = ippBigNumPOS; + BN_SIZE(pBN) = 1; + ZEXPAND_BNU(BN_NUMBER(pBN),0, (int)BN_ROOM(pBN)); + BN_NUMBER(pBN)[0] = a; + return pBN; +} +/* set BN to 2^m */ +__INLINE IppsBigNumState* cpBN_power2(IppsBigNumState* pBN, int power) +{ + cpSize size = BITS_BNU_CHUNK(power+1); + if(BN_ROOM(pBN) >= size) { + BN_SIGN(pBN) = ippBigNumPOS; + BN_SIZE(pBN) = size; + ZEXPAND_BNU(BN_NUMBER(pBN),0, BN_ROOM(pBN)); + SET_BIT(BN_NUMBER(pBN), power); + return pBN; + } + else return NULL; +} + +/* bitsize of BN */ +__INLINE int cpBN_bitsize(const IppsBigNumState* pA) +{ + int bitsize = BITSIZE_BNU(BN_NUMBER(pA), BN_SIZE(pA)); + return bitsize; +} + +/* returns -1/0/+1 depemding on A~B comparison */ +__INLINE int cpBN_cmp(const IppsBigNumState* pA, const IppsBigNumState* pB) +{ + IppsBigNumSGN signA = BN_SIGN(pA); + IppsBigNumSGN signB = BN_SIGN(pB); + + if(signA==signB) { + int result = cpCmp_BNU(BN_NUMBER(pA), BN_SIZE(pA), BN_NUMBER(pB), BN_SIZE(pB)); + return (ippBigNumPOS==signA)? result : -result; + } + return (ippBigNumPOS==signA)? 1 : -1; +} + +/* returns -1/0/+1 depemding on A comparison 00 */ +__INLINE int cpBN_tst(const IppsBigNumState* pA) +{ + if(1==BN_SIZE(pA) && 0==BN_NUMBER(pA)[0]) + return 0; + else + return BN_POSITIVE(pA)? 1 : -1; +} + + +// some addtition functions +__INLINE int IsZero_BN(const IppsBigNumState* pA) +{ + return ( BN_SIZE(pA)==1 ) && ( BN_NUMBER(pA)[0]==0 ); +} +__INLINE int IsOdd_BN(const IppsBigNumState* pA) +{ + return BN_NUMBER(pA)[0] & 1; +} + +__INLINE IppsBigNumState* BN_Word(IppsBigNumState* pBN, BNU_CHUNK_T w) +{ + BN_SIGN(pBN) = ippBigNumPOS; + BN_SIZE(pBN) = 1; + ZEXPAND_BNU(BN_NUMBER(pBN),0, BN_ROOM(pBN)); + BN_NUMBER(pBN)[0] = w; + return pBN; +} +__INLINE IppsBigNumState* BN_Set(const BNU_CHUNK_T* pData, cpSize len, IppsBigNumState* pBN) +{ + BN_SIGN(pBN) = ippBigNumPOS; + BN_SIZE(pBN) = len; + ZEXPAND_COPY_BNU(BN_NUMBER(pBN), BN_ROOM(pBN), pData, len); + return pBN; +} +__INLINE IppsBigNumState* BN_Make(BNU_CHUNK_T* pData, BNU_CHUNK_T* pBuffer, cpSize len, IppsBigNumState* pBN) +{ + BN_SET_ID(pBN); + BN_SIGN(pBN) = ippBigNumPOS; + BN_SIZE(pBN) = 1; + BN_ROOM(pBN) = len; + BN_NUMBER(pBN) = pData; + BN_BUFFER(pBN) = pBuffer; + return pBN; +} + +/* +// fixed single chunk BN +*/ +typedef struct _ippcpBigNumChunk { + IppsBigNumState bn; + BNU_CHUNK_T value; + BNU_CHUNK_T temporary; +} IppsBigNumStateChunk; + +/* reference to BN(1) and BN(2) */ +#define cpBN_OneRef OWNAPI(cpBN_OneRef) + IPP_OWN_DECL (IppsBigNumState*, cpBN_OneRef, (void)) +#define cpBN_TwoRef OWNAPI(cpBN_TwoRef) + IPP_OWN_DECL (IppsBigNumState*, cpBN_TwoRef, (void)) +#define cpBN_ThreeRef OWNAPI(cpBN_ThreeRef) + IPP_OWN_DECL (IppsBigNumState*, cpBN_ThreeRef, (void)) + +#define BN_ONE_REF() cpBN_OneRef() +#define BN_TWO_REF() cpBN_TwoRef() +#define BN_THREE_REF() cpBN_ThreeRef() + +#endif /* _CP_BN_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbn_pack.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbn_pack.c new file mode 100644 index 0000000..1abd056 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbn_pack.c @@ -0,0 +1,55 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// cpPackBigNumCtx() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcptool.h" + + +/*F* +// Name: cpPackBigNumCtx +// +// Purpose: Serialize bigNum context +// +// Parameters: +// pBN BigNum +// pBuffer buffer +*F*/ +IPP_OWN_DEFN (void, cpPackBigNumCtx, (const IppsBigNumState* pBN, Ipp8u* pBuffer)) +{ + IppsBigNumState* pB = (IppsBigNumState*)(pBuffer); + CopyBlock(pBN, pB, sizeof(IppsBigNumState)); + + cpSize dataAlignment = (cpSize)(IPP_INT_PTR(BN_NUMBER(pBN)) - IPP_INT_PTR(pBN) - (IPP_INT64)sizeof(IppsBigNumState)); + + BN_NUMBER(pB) = (BNU_CHUNK_T*)((Ipp8u*)NULL + IPP_INT_PTR(BN_NUMBER(pBN))-IPP_INT_PTR(pBN) - dataAlignment); + BN_BUFFER(pB) = (BNU_CHUNK_T*)((Ipp8u*)NULL + IPP_INT_PTR(BN_BUFFER(pBN))-IPP_INT_PTR(pBN) - dataAlignment); + + CopyBlock(BN_NUMBER(pBN), (Ipp8u*)pB+IPP_UINT_PTR(BN_NUMBER(pB)), BN_ROOM(pBN)*(Ipp32s)sizeof(BNU_CHUNK_T)); + CopyBlock(BN_BUFFER(pBN), (Ipp8u*)pB+IPP_UINT_PTR(BN_BUFFER(pB)), BN_ROOM(pBN)*(Ipp32s)sizeof(BNU_CHUNK_T)); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbn_unpack.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbn_unpack.c new file mode 100644 index 0000000..7740c3f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbn_unpack.c @@ -0,0 +1,61 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// cpUnpackBigNumCtx() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcptool.h" + + +/*F* +// Name: cpUnpackBigNumCtx +// +// Purpose: Deserialize bigNum context +// +// Parameters: +// pBN BigNum +// pBuffer buffer +*F*/ + + +IPP_OWN_DEFN (void, cpUnpackBigNumCtx, (const Ipp8u* pBuffer, IppsBigNumState* pBN)) +{ + IppsBigNumState* pB = (IppsBigNumState*)(pBuffer); + CopyBlock(pBuffer, pBN, sizeof(IppsBigNumState)); + + Ipp8u* ptr = (Ipp8u*)pBN; + ptr += sizeof(IppsBigNumState); + ptr = IPP_ALIGNED_PTR(ptr, BN_ALIGNMENT); + BN_NUMBER(pBN) = (BNU_CHUNK_T*)(ptr); + ptr += BN_ROOM(pBN)*(Ipp32s)sizeof(BNU_CHUNK_T); + BN_BUFFER(pBN) = (BNU_CHUNK_T*)(ptr); + + cpSize bufferOffset = (cpSize)(IPP_INT_PTR(BN_BUFFER(pBN)) - IPP_INT_PTR(pBN)); + + CopyBlock((Ipp8u*)pB+sizeof(IppsBigNumState), BN_NUMBER(pBN), BN_ROOM(pBN)*(Ipp32s)sizeof(BNU_CHUNK_T)); + CopyBlock((Ipp8u*)pB+bufferOffset, BN_BUFFER(pBN), BN_ROOM(pBN)*(Ipp32s)sizeof(BNU_CHUNK_T)); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbn_val1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbn_val1.c new file mode 100644 index 0000000..0e0f7e4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbn_val1.c @@ -0,0 +1,56 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// cpBN_OneRef() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcptool.h" + + +/*F* +// Name: cpBN_OneRef +// +// Purpose: BN(1) and reference +// +// Return: +// BigNum = 1 +*F*/ + +/* BN(1) and reference */ +IPP_OWN_DEFN (IppsBigNumState*, cpBN_OneRef, (void)) +{ + static IppsBigNumStateChunk cpChunk_BN1 = { + { + idCtxUnknown, + ippBigNumPOS, + 1,1, + &cpChunk_BN1.value,&cpChunk_BN1.temporary + }, + 1,0 + }; + BN_SET_ID(&cpChunk_BN1.bn); + return &cpChunk_BN1.bn; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbn_val2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbn_val2.c new file mode 100644 index 0000000..82c2779 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbn_val2.c @@ -0,0 +1,55 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// cpBN_TwoRef() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcptool.h" + + +/*F* +// Name: cpBN_TwoRef +// +// Purpose: BN(2) and reference +// +// Return: +// BigNum = 2 +*F*/ + +IPP_OWN_DEFN (IppsBigNumState*, cpBN_TwoRef, (void)) +{ + static IppsBigNumStateChunk cpChunk_BN2 = { + { + idCtxUnknown, + ippBigNumPOS, + 1,1, + &cpChunk_BN2.value,&cpChunk_BN2.temporary + }, + 2,0 + }; + BN_SET_ID(&cpChunk_BN2.bn); + return &cpChunk_BN2.bn; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbn_val3.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbn_val3.c new file mode 100644 index 0000000..bde9f8f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbn_val3.c @@ -0,0 +1,55 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// cpBN_ThreeRef() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcptool.h" + + +/*F* +// Name: cpBN_ThreeRef +// +// Purpose: BN(3) and reference +// +// Return: +// BigNum = 3 +*F*/ + +IPP_OWN_DEFN (IppsBigNumState*, cpBN_ThreeRef, (void)) +{ + static IppsBigNumStateChunk cpChunk_BN3 = { + { + idCtxUnknown, + ippBigNumPOS, + 1,1, + &cpChunk_BN3.value,&cpChunk_BN3.temporary + }, + 3,0 + }; + BN_SET_ID(&cpChunk_BN3.bn); + return &cpChunk_BN3.bn; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithadd.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithadd.c new file mode 100644 index 0000000..2a32343 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithadd.c @@ -0,0 +1,124 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// ippsAdd_BN() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcptool.h" + + +/*F* +// Name: ippsAdd_BN +// +// Purpose: Add BigNums. +// +// Returns: Reason: +// ippStsNullPtrErr pA == NULL +// pB == NULL +// pR == NULL +// ippStsContextMatchErr !BN_VALID_ID(pA) +// !BN_VALID_ID(pB) +// !BN_VALID_ID(pR) +// ippStsOutOfRangeErr pR can not hold result +// ippStsNoErr no errors +// +// Parameters: +// pA source BigNum A +// pB source BigNum B +// pR resultant BigNum +// +*F*/ +IPPFUN(IppStatus, ippsAdd_BN, (IppsBigNumState* pA, IppsBigNumState* pB, IppsBigNumState* pR)) +{ + IPP_BAD_PTR3_RET(pA, pB, pR); + + IPP_BADARG_RET(!BN_VALID_ID(pA), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pB), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pR), ippStsContextMatchErr); + + { + cpSize nsA = BN_SIZE(pA); + cpSize nsB = BN_SIZE(pB); + cpSize nsR = BN_ROOM(pR); + IPP_BADARG_RET(nsR < IPP_MAX(nsA, nsB), ippStsOutOfRangeErr); + + { + BNU_CHUNK_T* pDataR = BN_NUMBER(pR); + + IppsBigNumSGN sgnA = BN_SIGN(pA); + IppsBigNumSGN sgnB = BN_SIGN(pB); + BNU_CHUNK_T* pDataA = BN_NUMBER(pA); + BNU_CHUNK_T* pDataB = BN_NUMBER(pB); + + BNU_CHUNK_T carry; + + if(sgnA==sgnB) { + if(nsA < nsB) { + SWAP(nsA, nsB); + SWAP_PTR(BNU_CHUNK_T, pDataA, pDataB); + } + + carry = cpAdd_BNU(pDataR, pDataA, pDataB, nsB); + if(nsA>nsB) + carry = cpInc_BNU(pDataR+nsB, pDataA+nsB, nsA-nsB, carry); + if(carry) { + if(nsR>nsA) + pDataR[nsA++] = carry; + else + IPP_ERROR_RET(ippStsOutOfRangeErr); + } + BN_SIGN(pR) = sgnA; + } + + else { + int cmpRes = cpCmp_BNU(pDataA, nsA, pDataB, nsB); + + if(0==cmpRes) { + pDataR[0] = 0; + BN_SIZE(pR) = 1; + BN_SIGN(pR) = ippBigNumPOS; + return ippStsNoErr; + } + + if(0>cmpRes) { + SWAP(nsA, nsB); + SWAP_PTR(BNU_CHUNK_T, pDataA, pDataB); + } + + carry = cpSub_BNU(pDataR, pDataA, pDataB, nsB); + if(nsA>nsB) + cpDec_BNU(pDataR+nsB, pDataA+nsB, nsA-nsB, carry); + + BN_SIGN(pR) = cmpRes>0? sgnA : INVERSE_SIGN(sgnA); + } + + FIX_BNU(pDataR, nsA); + BN_SIZE(pR) = nsA; + + return ippStsNoErr; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithcmp.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithcmp.c new file mode 100644 index 0000000..5a8bc76 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithcmp.c @@ -0,0 +1,86 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// ippsCmp_BN() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcptool.h" + +/*F* +// Name: ippsCmp_BN +// +// Purpose: Compare two BigNums. +// +// Returns: Reason: +// ippStsNullPtrErr pA == NULL +// pB == NULL +// pResult == NULL +// ippStsContextMatchErr !BN_VALID_ID(pA) +// !BN_VALID_ID(pB) +// ippStsNoErr no errors +// +// Parameters: +// pA BigNum ctx +// pB BigNum ctx +// pResult result of comparison +// +*F*/ +IPPFUN(IppStatus, ippsCmp_BN,(const IppsBigNumState* pA, const IppsBigNumState* pB, Ipp32u *pResult)) +{ + IPP_BAD_PTR3_RET(pA, pB, pResult); + + IPP_BADARG_RET(!BN_VALID_ID(pA), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pB), ippStsContextMatchErr); + + { + BNU_CHUNK_T positiveA = cpIsEqu_ct(ippBigNumPOS, BN_SIGN(pA)); + BNU_CHUNK_T positiveB = cpIsEqu_ct(ippBigNumPOS, BN_SIGN(pB)); + BNU_CHUNK_T signMask; + + /* (ippBigNumPOS == BN_SIGN(pA)) && (ippBigNumPOS==BN_SIGN(pB)) => res = cpCmp_BNU() */ + BNU_CHUNK_T res = (BNU_CHUNK_T)( cpCmp_BNU(BN_NUMBER(pA), BN_SIZE(pA), BN_NUMBER(pB), BN_SIZE(pB)) ); + + /* (ippBigNumNEG == BN_SIGN(pA)) && (ippBigNumNEG==BN_SIGN(pB)) => invert res value */ + signMask = ~positiveA & ~positiveB; + res = (res & ~signMask) | ((0-res) & signMask); + + /* (ippBigNumPOS == BN_SIGN(pA)) && (ippBigNumNEG==BN_SIGN(pB)) => res = 1 */ + signMask = positiveA & ~positiveB; + res = (res & ~signMask) | ((1) & signMask); + + /* (ippBigNumNEG == BN_SIGN(pA)) && (ippBigNumPOS==BN_SIGN(pB)) => res = -1 */ + signMask = ~positiveA & positiveB; + res = (res & ~signMask) | ((BNU_CHUNK_T)(-1) & signMask); + + // map res into IPP_IS_LT/EQ/GT + Ipp32u cmpResult = (Ipp32u)( (cpIsEqu_ct(res, (BNU_CHUNK_T)(-1)) & IPP_IS_LT) + | (cpIsEqu_ct(res, (BNU_CHUNK_T)(0)) & IPP_IS_EQ) + | (cpIsEqu_ct(res, (BNU_CHUNK_T)(1)) & IPP_IS_GT) ); + *pResult = cmpResult; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithcmpz.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithcmpz.c new file mode 100644 index 0000000..bd848b7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithcmpz.c @@ -0,0 +1,63 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// ippsCmpZero_BN() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcptool.h" + + +/*F* +// Name: ippsCmpZero_BN +// +// Purpose: Compare BigNum value with zero. +// +// Returns: Reason: +// ippStsNullPtrErr pBN == NULL +// pResult == NULL +// ippStsContextMatchErr !BN_VALID_ID(pBN) +// ippStsNoErr no errors +// +// Parameters: +// pBN BigNum ctx +// pResult result of comparison +// +*F*/ +IPPFUN(IppStatus, ippsCmpZero_BN, (const IppsBigNumState* pBN, Ipp32u* pResult)) +{ + IPP_BAD_PTR2_RET(pBN, pResult); + + IPP_BADARG_RET(!BN_VALID_ID(pBN), ippStsContextMatchErr); + + if(BN_SIZE(pBN)==1 && BN_NUMBER(pBN)[0]==0) + *pResult = IS_ZERO; + else if (BN_SIGN(pBN)==ippBigNumPOS) + *pResult = GREATER_THAN_ZERO; + else if (BN_SIGN(pBN)==ippBigNumNEG) + *pResult = LESS_THAN_ZERO; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithdiv.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithdiv.c new file mode 100644 index 0000000..22ed974 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithdiv.c @@ -0,0 +1,98 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// ippsDiv_BN() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcptool.h" + + +/*F* +// Name: ippsDiv_BN +// +// Purpose: Divide BigNums. +// +// Returns: Reason: +// ippStsNullPtrErr pA == NULL +// pB == NULL +// pQ == NULL +// pR == NULL +// ippStsContextMatchErr !BN_VALID_ID(pA) +// !BN_VALID_ID(pB) +// !BN_VALID_ID(pQ) +// !BN_VALID_ID(pR) +// ippStsDivByZeroErr BN_SIZE(pB) == 1 && BN_NUMBER(pB)[0] == 0 +// ippStsOutOfRangeErr pQ and/or pR can not hold result +// ippStsNoErr no errors +// +// Parameters: +// pA source BigNum +// pB source BigNum +// pQ quotient BigNum +// pR reminder BigNum +// +// A = Q*B + R, 0 <= val(R) < val(B), sgn(A)==sgn(R) +// +*F*/ +IPPFUN(IppStatus, ippsDiv_BN, (IppsBigNumState* pA, IppsBigNumState* pB, IppsBigNumState* pQ, IppsBigNumState* pR)) +{ + IPP_BAD_PTR4_RET(pA, pB, pQ, pR); + + IPP_BADARG_RET(!BN_VALID_ID(pA), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pB), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pQ), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pR), ippStsContextMatchErr); + + IPP_BADARG_RET(BN_SIZE(pB)== 1 && BN_NUMBER(pB)[0]==0, ippStsDivByZeroErr); + + IPP_BADARG_RET(BN_ROOM(pR)cmpRes) + SWAP_PTR(IppsBigNumState, x, y); + if(0==cmpRes) { + COPY_BNU(BN_NUMBER(g), BN_NUMBER(x), BN_SIZE(x)); + BN_SIGN(g) = ippBigNumPOS; + BN_SIZE(g) = BN_SIZE(x); + return ippStsNoErr; + } + if(BN_SIZE(x)==1) { + BNU_CHUNK_T gcd = cpGcd_BNU(BN_NUMBER(x)[0], BN_NUMBER(y)[0]); + BN_NUMBER(g)[0] = gcd; + BN_SIZE(g) = 1; + return ippStsNoErr; + } + } + + { + Ipp32u* xBuffer = (Ipp32u*)BN_BUFFER(x); + Ipp32u* yBuffer = (Ipp32u*)BN_BUFFER(y); + Ipp32u* gBuffer = (Ipp32u*)BN_BUFFER(g); + Ipp32u* xData = (Ipp32u*)BN_NUMBER(x); + Ipp32u* yData = (Ipp32u*)BN_NUMBER(y); + Ipp32u* gData = (Ipp32u*)BN_NUMBER(g); + cpSize nsXmax = BN_ROOM(x)*((Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(Ipp32u))); + cpSize nsYmax = BN_ROOM(y)*((Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(Ipp32u))); + cpSize nsGmax = BN_ROOM(g)*((Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(Ipp32u))); + cpSize nsX = BN_SIZE(x)*((Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(Ipp32u))); + cpSize nsY = BN_SIZE(y)*((Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(Ipp32u))); + + Ipp32u* T; + Ipp32u* u; + + FIX_BNU32(xData, nsX); + FIX_BNU32(yData, nsY); + + /* init buffers */ + ZEXPAND_COPY_BNU(xBuffer, nsXmax, xData, nsX); + ZEXPAND_COPY_BNU(yBuffer, nsYmax, yData, nsY); + + T = gBuffer; + u = gData; + ZEXPAND_BNU(T, 0, nsGmax); + ZEXPAND_BNU(u, 0, nsGmax); + + while(nsX > (cpSize)(sizeof(BNU_CHUNK_T)/sizeof(Ipp32u))) { + /* xx and yy is the high-order digits of x and y (yy could be 0) */ + + Ipp64u xx = (Ipp64u)(xBuffer[nsX-1]); + Ipp64u yy = (nsY < nsX)? 0 : (Ipp64u)(yBuffer[nsY-1]); + + Ipp64s AA = 1; + Ipp64s BB = 0; + Ipp64s CC = 0; + Ipp64s DD = 1; + Ipp64s t; + + while((yy+(Ipp64u)CC)!=0 && (yy+(Ipp64u)DD)!=0) { + Ipp64u q = ( xx + (Ipp64u)AA ) / ( yy + (Ipp64u)CC ); + Ipp64u q1 = ( xx + (Ipp64u)BB ) / ( yy + (Ipp64u)DD ); + if(q!=q1) + break; + t = AA - (Ipp64s)q*CC; + AA = CC; + CC = t; + t = BB - (Ipp64s)q*DD; + BB = DD; + DD = t; + t = (Ipp64s)(xx - q*yy); + xx = yy; + yy = (Ipp64u)t; + } + + if(BB == 0) { + /* T = x mod y */ + cpSize nsT = cpMod_BNU32(xBuffer, nsX, yBuffer, nsY); + ZEXPAND_BNU(T, 0, nsGmax); + COPY_BNU(T, xBuffer, nsT); + /* a = b; b = T; */ + ZEXPAND_BNU(xBuffer, 0, nsXmax); + COPY_BNU(xBuffer, yBuffer, nsY); + ZEXPAND_BNU(yBuffer, 0, nsYmax); + COPY_BNU(yBuffer, T, nsY); + } + + else { + Ipp32u carry; + /* + // T = AA*x + BB*y; + // u = CC*x + DD*y; + // b = u; a = T; + */ + if((AA <= 0)&&(BB>=0)) { + Ipp32u a1 = (Ipp32u)(-AA); + carry = cpMulDgt_BNU32(T, yBuffer, nsY, (Ipp32u)BB); + carry = cpMulDgt_BNU32(u, xBuffer, nsY, a1); + /* T = BB*y - AA*x; */ + carry = cpSub_BNU32(T, T, u, nsY); + } + else { + if((AA >= 0)&&(BB<=0)) { + Ipp32u b1 = (Ipp32u)(-BB); + carry = cpMulDgt_BNU32(T, xBuffer, nsY, (Ipp32u)AA); + carry = cpMulDgt_BNU32(u, yBuffer, nsY, b1); + /* T = AA*x - BB*y; */ + carry = cpSub_BNU32(T, T, u, nsY); + } + else { + /*AA*BB>=0 */ + carry = cpMulDgt_BNU32(T, xBuffer, nsY, (Ipp32u)AA); + carry = cpMulDgt_BNU32(u, yBuffer, nsY, (Ipp32u)BB); + /* T = AA*x + BB*y; */ + carry = cpAdd_BNU32(T, T, u, nsY); + } + } + + /* Now T is reserved. We use only u for intermediate results. */ + if((CC <= 0)&&(DD>=0)){ + Ipp32u c1 = (Ipp32u)(-CC); + /* u = x*CC; x = u; */ + carry = cpMulDgt_BNU32(u, xBuffer, nsY, c1); + COPY_BNU(xBuffer, u, nsY); + /* u = y*DD; */ + carry = cpMulDgt_BNU32(u, yBuffer, nsY, (Ipp32u)DD); + /* u = DD*y - CC*x; */ + carry = cpSub_BNU32(u, u, xBuffer, nsY); + } + else { + if((CC >= 0)&&(DD<=0)){ + Ipp32u d1 = (Ipp32u)(-DD); + /* u = y*DD; y = u */ + carry = cpMulDgt_BNU32(u, yBuffer, nsY, d1); + COPY_BNU(yBuffer, u, nsY); + /* u = CC*x; */ + carry = cpMulDgt_BNU32(u, xBuffer, nsY, (Ipp32u)CC); + /* u = CC*x - DD*y; */ + carry = cpSub_BNU32(u, u, yBuffer, nsY); + } + else { + /*CC*DD>=0 */ + /* y = y*DD */ + carry = cpMulDgt_BNU32(u, yBuffer, nsY, (Ipp32u)DD); + COPY_BNU(yBuffer, u, nsY); + /* u = x*CC */ + carry = cpMulDgt_BNU32(u, xBuffer, nsY, (Ipp32u)CC); + /* u = x*CC + y*DD */ + carry = cpAdd_BNU32(u, u, yBuffer, nsY); + } + } + + IPP_UNREFERENCED_PARAMETER(carry); + + /* y = u; x = T; */ + COPY_BNU(yBuffer, u, nsY); + COPY_BNU(xBuffer, T, nsY); + } + + FIX_BNU32(xBuffer, nsX); + FIX_BNU32(yBuffer, nsY); + + if (nsY > nsX) { + SWAP_PTR(IppsBigNumState, x, y); + SWAP(nsX, nsY); + } + + if (nsY==1 && yBuffer[nsY-1]==0) { + /* End evaluation */ + ZEXPAND_BNU(gData, 0, nsGmax); + COPY_BNU(gData, xBuffer, nsX); + BN_SIZE(g) = INTERNAL_BNU_LENGTH(nsX); + BN_SIGN(g) = ippBigNumPOS; + return ippStsNoErr; + } + } + + BN_NUMBER(g)[0] = cpGcd_BNU(((BNU_CHUNK_T*)xBuffer)[0], ((BNU_CHUNK_T*)yBuffer)[0]); + BN_SIZE(g) = 1; + BN_SIGN(g) = ippBigNumPOS; + return ippStsNoErr; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithmac.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithmac.c new file mode 100644 index 0000000..6fafc37 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithmac.c @@ -0,0 +1,125 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// ippsMAC_BN_I() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcptool.h" + + +/*F* +// Name: ippsMAC_BN_I +// +// Purpose: Multiply and Accumulate BigNums. +// +// Returns: Reason: +// ippStsNullPtrErr pA == NULL +// pB == NULL +// pR == NULL +// ippStsContextMatchErr !BN_VALID_ID(pA) +// !BN_VALID_ID(pB) +// !BN_VALID_ID(pR) +// ippStsOutOfRangeErr pR can not fit result +// ippStsNoErr no errors +// +// Parameters: +// pA source BigNum +// pB source BigNum +// pR resultant BigNum +// +*F*/ +IPPFUN(IppStatus, ippsMAC_BN_I, (IppsBigNumState* pA, IppsBigNumState* pB, IppsBigNumState* pR)) +{ + IPP_BAD_PTR3_RET(pA, pB, pR); + + IPP_BADARG_RET(!BN_VALID_ID(pA), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pB), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pR), ippStsContextMatchErr); + + { + BNU_CHUNK_T* pDataA = BN_NUMBER(pA); + BNU_CHUNK_T* pDataB = BN_NUMBER(pB); + + cpSize nsA = BN_SIZE(pA); + cpSize nsB = BN_SIZE(pB); + + cpSize bitSizeA = BITSIZE_BNU(pDataA, nsA); + cpSize bitSizeB = BITSIZE_BNU(pDataB, nsB); + /* size of temporary pruduct */ + cpSize nsP = BITS_BNU_CHUNK(bitSizeA+bitSizeB); + + /* test if multiplicant/multiplier is zero */ + if(!bitSizeA || !bitSizeB) return ippStsNoErr; + /* test if product can't fit to the result */ + IPP_BADARG_RET(BN_ROOM(pR)cmpRes) { + SWAP_PTR(BNU_CHUNK_T, pTmp, pDataP); + } + cpSub_BNU(pDataR, pTmp, pDataP, room); + + BN_SIGN(pR) = cmpRes>0? sgnR : INVERSE_SIGN(sgnR); + } + + FIX_BNU(pDataR, room); + BN_SIZE(pR) = room; + + return ippStsNoErr; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithminv.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithminv.c new file mode 100644 index 0000000..4bc4038 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithminv.c @@ -0,0 +1,84 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// ippsModInv_BN() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcptool.h" + + +/*F* +// Name: ippsModInv_BN +// +// Purpose: Multiplicative Inversion BigNum. +// +// Returns: Reason: +// ippStsNullPtrErr pA == NULL +// pM == NULL +// pInv == NULL +// ippStsContextMatchErr !BN_VALID_ID(pA) +// !BN_VALID_ID(pM) +// !BN_VALID_ID(pInv) +// ippStsBadArgErr A<=0 +// ippStsBadModulusErr M<=0 +// ippStsScaleRangeErr A>=M +// ippStsOutOfRangeErr pInv can not hold result +// ippStsNoErr no errors +// ippStsBadModulusErr inversion not found +// +// Parameters: +// pA source (value) BigNum +// pM source (modulus) BigNum +// pInv result BigNum +// +*F*/ +IPPFUN(IppStatus, ippsModInv_BN, (IppsBigNumState* pA, IppsBigNumState* pM, IppsBigNumState* pInv) ) +{ + IPP_BAD_PTR3_RET(pA, pM, pInv); + + IPP_BADARG_RET(!BN_VALID_ID(pA), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pM), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pInv), ippStsContextMatchErr); + + IPP_BADARG_RET(BN_ROOM(pInv) < BN_SIZE(pM), ippStsOutOfRangeErr); + IPP_BADARG_RET(BN_NEGATIVE(pA) || (BN_SIZE(pA)==1 && BN_NUMBER(pA)[0]==0), ippStsBadArgErr); + IPP_BADARG_RET(BN_NEGATIVE(pM) || (BN_SIZE(pM)==1 && BN_NUMBER(pM)[0]==0), ippStsBadModulusErr); + IPP_BADARG_RET(cpCmp_BNU(BN_NUMBER(pA), BN_SIZE(pA), BN_NUMBER(pM), BN_SIZE(pM)) >= 0, ippStsScaleRangeErr); + + { + cpSize nsR = cpModInv_BNU(BN_NUMBER(pInv), + BN_NUMBER(pA), BN_SIZE(pA), + BN_NUMBER(pM), BN_SIZE(pM), + BN_BUFFER(pInv), BN_BUFFER(pA), BN_BUFFER(pM)); + if(nsR) { + BN_SIGN(pInv) = ippBigNumPOS; + BN_SIZE(pInv) = nsR; + return ippStsNoErr; + } + else + return ippStsBadModulusErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithmod.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithmod.c new file mode 100644 index 0000000..76e88b5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithmod.c @@ -0,0 +1,101 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// ippsMod_BN() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcptool.h" + + +/*F* +// Name: ippsMod_BN +// +// Purpose: reduction BigNum. +// +// Returns: Reason: +// ippStsNullPtrErr pA == NULL +// pM == NULL +// pR == NULL +// ippStsContextMatchErr !BN_VALID_ID(pA) +// !BN_VALID_ID(pM) +// !BN_VALID_ID(pR) +// ippStsOutOfRangeErr pR can not hold result +// ippStsBadModulusErr modulus IppsBigNumState* pM +// is not a positive integer +// ippStsNoErr no errors +// +// Parameters: +// pA source BigNum +// pB source BigNum +// pR reminder BigNum +// +// A = Q*M + R, 0 <= R < B +// +*F*/ +IPPFUN(IppStatus, ippsMod_BN, (IppsBigNumState* pA, IppsBigNumState* pM, IppsBigNumState* pR)) +{ + IPP_BAD_PTR3_RET(pA, pM, pR); + + IPP_BADARG_RET(!BN_VALID_ID(pA), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pM), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pR), ippStsContextMatchErr); + + IPP_BADARG_RET(BN_NEGATIVE(pM), ippStsBadModulusErr); + IPP_BADARG_RET(BN_SIZE(pM)== 1 && BN_NUMBER(pM)[0]==0, ippStsBadModulusErr); + + IPP_BADARG_RET(BN_ROOM(pR)BN_ROOM(pR), ippStsOutOfRangeErr); + + BN_SIZE(pR) = nsR; + BN_SIGN(pR) = (BN_SIGN(pA)==BN_SIGN(pB)? ippBigNumPOS : ippBigNumNEG); + return ippStsNoErr; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithsub.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithsub.c new file mode 100644 index 0000000..5b1a2e6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnarithsub.c @@ -0,0 +1,124 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// ippsSub_BN() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcptool.h" + + +/*F* +// Name: ippsSub_BN +// +// Purpose: Subtract BigNums. +// +// Returns: Reason: +// ippStsNullPtrErr pA == NULL +// pB == NULL +// pR == NULL +// ippStsContextMatchErr !BN_VALID_ID(pA) +// !BN_VALID_ID(pB) +// !BN_VALID_ID(pR) +// ippStsOutOfRangeErr pR can not hold result +// ippStsNoErr no errors +// +// Parameters: +// pA source BigNum +// pB source BigNum +// pR resultant BigNum +// +*F*/ +IPPFUN(IppStatus, ippsSub_BN, (IppsBigNumState* pA, IppsBigNumState* pB, IppsBigNumState* pR)) +{ + IPP_BAD_PTR3_RET(pA, pB, pR); + + IPP_BADARG_RET(!BN_VALID_ID(pA), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pB), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pR), ippStsContextMatchErr); + + { + cpSize nsA = BN_SIZE(pA); + cpSize nsB = BN_SIZE(pB); + cpSize nsR = BN_ROOM(pR); + IPP_BADARG_RET(nsR < IPP_MAX(nsA, nsB), ippStsOutOfRangeErr); + + { + BNU_CHUNK_T* pDataR = BN_NUMBER(pR); + + IppsBigNumSGN sgnA = BN_SIGN(pA); + IppsBigNumSGN sgnB = BN_SIGN(pB); + BNU_CHUNK_T* pDataA = BN_NUMBER(pA); + BNU_CHUNK_T* pDataB = BN_NUMBER(pB); + + BNU_CHUNK_T carry; + + if(sgnA!=sgnB) { + if(nsA < nsB) { + SWAP(nsA, nsB); + SWAP_PTR(BNU_CHUNK_T, pDataA, pDataB); + } + + carry = cpAdd_BNU(pDataR, pDataA, pDataB, nsB); + if(nsA>nsB) + carry = cpInc_BNU(pDataR+nsB, pDataA+nsB, nsA-nsB, carry); + if(carry) { + if(nsR > nsA) + pDataR[nsA++] = carry; + else + IPP_ERROR_RET(ippStsOutOfRangeErr); + } + BN_SIGN(pR) = sgnA; + } + + else { + int cmpRes= cpCmp_BNU(pDataA, nsA, pDataB, nsB); + + if(0==cmpRes) { + ZEXPAND_BNU(pDataR,0, nsR); + BN_SIZE(pR) = 1; + BN_SIGN(pR) = ippBigNumPOS; + return ippStsNoErr; + } + + if(0>cmpRes) { + SWAP(nsA, nsB); + SWAP_PTR(BNU_CHUNK_T, pDataA, pDataB); + } + + carry = cpSub_BNU(pDataR, pDataA, pDataB, nsB); + if(nsA>nsB) + cpDec_BNU(pDataR+nsB, pDataA+nsB, nsA-nsB, carry); + + BN_SIGN(pR) = cmpRes>0? sgnA : INVERSE_SIGN(sgnA); + } + + FIX_BNU(pDataR, nsA); + BN_SIZE(pR) = nsA; + + return ippStsNoErr; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbngetext.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbngetext.c new file mode 100644 index 0000000..075dd0d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbngetext.c @@ -0,0 +1,72 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// ippsExtGet_BN() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcptool.h" + +/*F* +// Name: ippsExtGet_BN +// +// Purpose: Extracts the specified combination of the sign, data +// length, and value characteristics of the integer big +// number from the input structure. +// +// Returns: Reason: +// ippStsNullPtrErr pBN == NULL +// ippStsContextMatchErr !BN_VALID_ID(pBN) +// ippStsNoErr no errors +// +// Parameters: +// pSgn pointer to the sign +// pBitSize pointer to the data size (in bits) +// pData pointer to the data buffer +// pBN BigNum ctx +// +*F*/ + +IPPFUN(IppStatus, ippsExtGet_BN, (IppsBigNumSGN* pSgn, int* pBitSize, Ipp32u* pData, + const IppsBigNumState* pBN)) +{ + IPP_BAD_PTR1_RET(pBN); + + IPP_BADARG_RET(!BN_VALID_ID(pBN), ippStsContextMatchErr); + + { + cpSize bitSize = BITSIZE_BNU(BN_NUMBER(pBN), BN_SIZE(pBN)); + if(0==bitSize) + bitSize = 1; + if(pData) + COPY_BNU(pData, (Ipp32u*)BN_NUMBER(pBN), BITS2WORD32_SIZE(bitSize)); + if(pSgn) + *pSgn = BN_SIGN(pBN); + if(pBitSize) + *pBitSize = bitSize; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbngetlen.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbngetlen.c new file mode 100644 index 0000000..e3ab59e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbngetlen.c @@ -0,0 +1,58 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// ippsGetSize_BN() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcptool.h" + + +/*F* +// Name: ippsGetSize_BN +// +// Purpose: Returns BigNum room. +// +// Returns: Reason: +// ippStsNullPtrErr pBN == NULL +// pSize == NULL +// ippStsContextMatchErr !BN_VALID_ID(pBN) +// ippStsNoErr no errors +// +// Parameters: +// pBN BigNum ctx +// pSize max BigNum length (in Ipp32u chunks) +// +*F*/ +IPPFUN(IppStatus, ippsGetSize_BN, (const IppsBigNumState* pBN, int* pSize)) +{ + IPP_BAD_PTR2_RET(pBN, pSize); + + IPP_BADARG_RET(!BN_VALID_ID(pBN), ippStsContextMatchErr); + + *pSize = BN_ROOM(pBN)*(Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(Ipp32u)); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbngetoctstr.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbngetoctstr.c new file mode 100644 index 0000000..c96927f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbngetoctstr.c @@ -0,0 +1,66 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Big Number Operations +// +// Contents: +// ippsGetOctString_BN() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" + + +/*F* +// Name: ippsGetOctString_BN +// +// Purpose: Convert BN value into the octet string. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pOctStr +// NULL == pBN +// +// ippStsContextMatchErr !BN_VALID_ID(pBN) +// +// ippStsRangeErr BN <0 +// +// ippStsLengthErr strLen is enough for keep BN value +// +// ippStsNoErr no errors +// +// Parameters: +// pBN pointer to the source BN +// pOctStr pointer to the target octet string +// strLen octet string length +*F*/ +IPPFUN(IppStatus, ippsGetOctString_BN,(Ipp8u* pOctStr, int strLen, + const IppsBigNumState* pBN)) +{ + IPP_BAD_PTR2_RET(pOctStr, pBN); + + IPP_BADARG_RET(!BN_VALID_ID(pBN), ippStsContextMatchErr); + IPP_BADARG_RET(BN_NEGATIVE(pBN), ippStsRangeErr); + IPP_BADARG_RET((0>strLen), ippStsLengthErr); + + return cpToOctStr_BNU(pOctStr,strLen, BN_NUMBER(pBN),BN_SIZE(pBN))? ippStsNoErr : ippStsLengthErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbngetref.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbngetref.c new file mode 100644 index 0000000..ddb44f7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbngetref.c @@ -0,0 +1,90 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// ippsBigNumGetSize() +// ippsBigNumInit() +// +// ippsSet_BN() +// ippsGet_BN() +// ippsGetSize_BN() +// ippsExtGet_BN() +// ippsRef_BN() +// +// ippsCmpZero_BN() +// ippsCmp_BN() +// +// ippsAdd_BN() +// ippsSub_BN() +// ippsMul_BN() +// ippsMAC_BN_I() +// ippsDiv_BN() +// ippsMod_BN() +// ippsGcd_BN() +// ippsModInv_BN() +// +// cpPackBigNumCtx(), cpUnpackBigNumCtx() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcptool.h" + + +/*F* +// Name: ippsRef_BN +// +// Purpose: Get BigNum info. +// +// Returns: Reason: +// ippStsNullPtrErr pBN == NULL +// ippStsContextMatchErr !BN_VALID_ID(pBN) +// ippStsNoErr no errors +// +// Parameters: +// pSgn pointer to the sign +// pBitSize pointer to the data size (in bits) +// ppData pointer to the data buffer +// pBN BigNum ctx +// +*F*/ +IPPFUN(IppStatus, ippsRef_BN, (IppsBigNumSGN* pSgn, int* pBitSize, Ipp32u** const ppData, + const IppsBigNumState *pBN)) +{ + IPP_BAD_PTR1_RET(pBN); + + IPP_BADARG_RET(!BN_VALID_ID(pBN), ippStsContextMatchErr); + + if(pSgn) + *pSgn = BN_SIGN(pBN); + if(pBitSize) { + cpSize bitLen = BITSIZE_BNU(BN_NUMBER(pBN), BN_SIZE(pBN)); + *pBitSize = bitLen? bitLen : 1; + } + + if(ppData) + *ppData = (Ipp32u*)BN_NUMBER(pBN); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbngetsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbngetsize.c new file mode 100644 index 0000000..ae07e9a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbngetsize.c @@ -0,0 +1,70 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// ippsBigNumGetSize() +// ippsBigNumInit() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcptool.h" + + +/*F* +// Name: ippsBigNumGetSize +// +// Purpose: Returns size of BigNum ctx (bytes). +// +// Returns: Reason: +// ippStsNullPtrErr pCtxSize == NULL +// ippStsLengthErr length < 1 +// length > BITS2WORD32_SIZE(BN_MAXBITSIZE) +// ippStsNoErr no errors +// +// Parameters: +// length max BN length (32-bits segments) +// pSize pointer BigNum ctx size +// +*F*/ +IPPFUN(IppStatus, ippsBigNumGetSize, (int length, cpSize *pCtxSize)) +{ + IPP_BAD_PTR1_RET(pCtxSize); + IPP_BADARG_RET(length<1 || length>BITS2WORD32_SIZE(BN_MAXBITSIZE), ippStsLengthErr); + + { + /* convert length to the number of BNU_CHUNK_T */ + cpSize len = INTERNAL_BNU_LENGTH(length); + + /* reserve one BNU_CHUNK_T more for cpDiv_BNU, + mul, mont exp operations */ + len++; + + *pCtxSize = (Ipp32s)sizeof(IppsBigNumState) + + len*(Ipp32s)sizeof(BNU_CHUNK_T) + + len*(Ipp32s)sizeof(BNU_CHUNK_T) + + BN_ALIGNMENT-1; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbngetwords.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbngetwords.c new file mode 100644 index 0000000..b665466 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbngetwords.c @@ -0,0 +1,71 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// ippsGet_BN() +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcptool.h" + + +/*F* +// Name: ippsGet_BN +// +// Purpose: Get BigNum. +// +// Returns: Reason: +// ippStsNullPtrErr pBN == NULL +// pData == NULL +// pSgn == NULL +// pLengthInBits ==NULL +// ippStsContextMatchErr !BN_VALID_ID(pBN) +// ippStsNoErr no errors +// +// Parameters: +// pSgn pointer to the sign +// pLengthInBits pointer to the data size (in Ipp32u chunks) +// pData pointer to the data buffer +// pBN BigNum ctx +// +*F*/ +IPPFUN(IppStatus, ippsGet_BN, (IppsBigNumSGN* pSgn, int* pLengthInBits, Ipp32u* pData, + const IppsBigNumState* pBN)) +{ + IPP_BAD_PTR4_RET(pSgn, pLengthInBits, pData, pBN); + + IPP_BADARG_RET(!BN_VALID_ID(pBN), ippStsContextMatchErr); + + { + cpSize len32 = BN_SIZE(pBN)*(Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(Ipp32u)); + Ipp32u* bnData = (Ipp32u*)BN_NUMBER(pBN); + + FIX_BNU32(bnData, len32); + COPY_BNU(pData, bnData, len32); + + *pSgn = BN_SIGN(pBN); + *pLengthInBits = len32; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbninit.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbninit.c new file mode 100644 index 0000000..2d7185b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbninit.c @@ -0,0 +1,83 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// ippsBigNumGetSize() +// ippsBigNumInit() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcptool.h" + +/*F* +// Name: ippsBigNumInit +// +// Purpose: Init BigNum spec for future usage. +// +// Returns: Reason: +// ippStsNullPtrErr pBN == NULL +// ippStsLengthErr length<1 +// length > BITS2WORD32_SIZE(BN_MAXBITSIZE) +// ippStsNoErr no errors +// +// Parameters: +// length max BN length (32-bits segments) +// pBN BigNum ctx +// +*F*/ +IPPFUN(IppStatus, ippsBigNumInit, (int length, IppsBigNumState* pBN)) +{ + IPP_BADARG_RET(length<1 || length>BITS2WORD32_SIZE(BN_MAXBITSIZE), ippStsLengthErr); + IPP_BAD_PTR1_RET(pBN); + + { + Ipp8u* ptr = (Ipp8u*)pBN; + + /* convert length to the number of BNU_CHUNK_T */ + cpSize len = INTERNAL_BNU_LENGTH(length); + + BN_SIGN(pBN) = ippBigNumPOS; + BN_SIZE(pBN) = 1; /* initial valie is zero */ + BN_ROOM(pBN) = len; /* close to what has been passed by user */ + + /* reserve one BNU_CHUNK_T more for cpDiv_BNU, + mul, mont exp operations */ + len++; + + ptr += sizeof(IppsBigNumState); + + /* allocate buffers */ + ptr = (Ipp8u*)(IPP_ALIGNED_PTR(ptr, BN_ALIGNMENT)); + BN_NUMBER(pBN) = (BNU_CHUNK_T*)ptr; + ptr += len * (Ipp32s)sizeof(BNU_CHUNK_T); + BN_BUFFER(pBN) = (BNU_CHUNK_T*)(ptr); /* use expanded length here */ + + /* set BN value and buffer to zero */ + ZEXPAND_BNU(BN_NUMBER(pBN), 0, len); + ZEXPAND_BNU(BN_BUFFER(pBN), 0, len); + + BN_SET_ID(pBN); + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnresource.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnresource.h new file mode 100644 index 0000000..5fa0e21 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnresource.h @@ -0,0 +1,49 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Internal BN Resource Definitions & Function Prototypes +// +// +*/ + +#if !defined(_PCP_BNRESOURCE_H) +#define _PCP_BNRESOURCE_H + + +typedef struct { + void* pNext; + IppsBigNumState* pBN; +} BigNumNode; + + +/* size (byte) of BN resource */ +#define cpBigNumListGetSize OWNAPI(cpBigNumListGetSize) + IPP_OWN_DECL (int, cpBigNumListGetSize, (int feBitSize, int nodes)) + +/* init BN resource */ +#define cpBigNumListInit OWNAPI(cpBigNumListInit) + IPP_OWN_DECL (void, cpBigNumListInit, (int feBitSize, int nodes, BigNumNode* pList)) + +/* get BN from resource */ +#define cpBigNumListGet OWNAPI(cpBigNumListGet) + IPP_OWN_DECL (IppsBigNumState*, cpBigNumListGet, (BigNumNode** pList)) + +#endif /* _PCP_BNRESOURCE_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnresourceca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnresourceca.c new file mode 100644 index 0000000..1f5202d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnresourceca.c @@ -0,0 +1,121 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Internal ECC (prime) Resource List Function +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbnresource.h" +#include "pcpbn.h" + + + +/*F* +// Name: cpBigNumListGetSize +// +// Purpose: Size of BigNum List Buffer +// +// Returns: +// Size of List Buffer +// +// Parameters: +// feBitSize size in bits +// nodes number of nodes +// +*F*/ + +IPP_OWN_DEFN (int, cpBigNumListGetSize, (int feBitSize, int nodes)) +{ + /* size of buffer per single big number */ + int bnSize; + ippsBigNumGetSize(BITS2WORD32_SIZE(feBitSize), &bnSize); + + /* size of buffer for whole list */ + return ((Ipp32s)sizeof(BigNumNode) + bnSize) * nodes; +} + +/*F* +// Name: cpBigNumListInit +// +// Purpose: Init list +// +// +// Parameters: +// feBitSize size in bit +// nodes number of nodes +// pList pointer to list +// +// Note: buffer for BN list must have appropriate alignment +// +*F*/ +IPP_OWN_DEFN (void, cpBigNumListInit, (int feBitSize, int nodes, BigNumNode* pList)) +{ + int itemSize; + /* length of Big Num */ + int bnLen = BITS2WORD32_SIZE(feBitSize); + /* size of buffer per single big number */ + ippsBigNumGetSize(bnLen, &itemSize); + /* size of list item */ + itemSize += sizeof(BigNumNode); + + { + int n; + /* init all nodes */ + BigNumNode* pNode = (BigNumNode*)( (Ipp8u*)pList + (nodes-1)*itemSize ); + BigNumNode* pNext = NULL; + for(n=0; npNext = pNext; + pNode->pBN = (IppsBigNumState*)(tbnPtr); + ippsBigNumInit(bnLen, pNode->pBN); + pNext = pNode; + pNode = (BigNumNode*)( (Ipp8u*)pNode - itemSize); + } + } +} + + +/*F* +// Name: cpBigNumListGet +// +// Purpose: Get BigNum reference +// +// Returns: +// BigNum reference +// +// Parameters: +// ppList pointer to pointer to List +// +*F*/ + +IPP_OWN_DEFN (IppsBigNumState*, cpBigNumListGet, (BigNumNode** ppList)) +{ + if(*ppList) { + IppsBigNumState* ret = (*ppList)->pBN; + *ppList = (*ppList)->pNext; + return ret; + } + else + return NULL; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnsetoctstr.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnsetoctstr.c new file mode 100644 index 0000000..8be4c1e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnsetoctstr.c @@ -0,0 +1,83 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Big Number Operations +// +// Contents: +// ippsSetOctString_BN() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" + + +/*F* +// Name: ippsSetOctString_BN +// +// Purpose: Convert octet string into the BN value. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pOctStr +// NULL == pBN +// +// ippStsContextMatchErr !BN_VALID_ID(pBN) +// +// ippStsLengthErr 0 > strLen +// +// ippStsSizeErr BN_ROOM(pBN) is enough for keep actual strLen +// +// ippStsNoErr no errors +// +// Parameters: +// pOctStr pointer to the source octet string +// strLen octet string length +// pBN pointer to the target BN +// +*F*/ +IPPFUN(IppStatus, ippsSetOctString_BN,(const Ipp8u* pOctStr, int strLen, + IppsBigNumState* pBN)) +{ + IPP_BAD_PTR2_RET(pOctStr, pBN); + + IPP_BADARG_RET(!BN_VALID_ID(pBN), ippStsContextMatchErr); + + IPP_BADARG_RET((0>strLen), ippStsLengthErr); + + /* remove leading zeros */ + while(strLen && (0==pOctStr[0])) { + strLen--; + pOctStr++; + } + + /* test BN size */ + IPP_BADARG_RET((int)((Ipp32s)sizeof(BNU_CHUNK_T)*BN_ROOM(pBN)) BN_ROOM(pBN) +// ippStsNoErr no errors +// +// Parameters: +// sgn sign +// length data size (in Ipp32u chunks) +// pData source data pointer +// pBN BigNum ctx +// +*F*/ +IPPFUN(IppStatus, ippsSet_BN, (IppsBigNumSGN sgn, int length, const Ipp32u* pData, + IppsBigNumState* pBN)) +{ + IPP_BAD_PTR2_RET(pData, pBN); + + IPP_BADARG_RET(!BN_VALID_ID(pBN), ippStsContextMatchErr); + + IPP_BADARG_RET(length<1, ippStsLengthErr); + + /* compute real size */ + FIX_BNU32(pData, length); + + { + cpSize len = INTERNAL_BNU_LENGTH(length); + IPP_BADARG_RET(len > BN_ROOM(pBN), ippStsOutOfRangeErr); + + ZEXPAND_COPY_BNU((Ipp32u*)BN_NUMBER(pBN), BN_ROOM(pBN)*(int)(sizeof(BNU_CHUNK_T)/sizeof(Ipp32u)), pData, length); + + BN_SIZE(pBN) = len; + + if(length==1 && pData[0] == 0) + sgn = ippBigNumPOS; /* consider zero value as positive */ + BN_SIGN(pBN) = sgn; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32_arith_add.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32_arith_add.c new file mode 100644 index 0000000..7391928 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32_arith_add.c @@ -0,0 +1,59 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal BNU32 arithmetic. +// +// Contents: +// cpAdd_BNU32() +// +*/ + +#include "owncp.h" +#include "pcpbnumisc.h" +#include "pcpbnu32misc.h" +#include "pcpbnu32arith.h" + + +/*F* +// Name: cpSub_BNU32 +// +// Purpose: addition BNU32. +// +// Returns: +// carry +// +// Parameters: +// pA source +// pB source +// ns size +// pR result +// +*F*/ +IPP_OWN_DEFN (Ipp32u, cpAdd_BNU32, (Ipp32u* pR, const Ipp32u* pA, const Ipp32u* pB, cpSize ns)) +{ + Ipp32u carry = 0; + cpSize i; + for(i=0; i>(32-1); + } + return borrow; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32_arith_div.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32_arith_div.c new file mode 100644 index 0000000..59fdbc6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32_arith_div.c @@ -0,0 +1,175 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal BNU32 arithmetic. +// +// Contents: +// cpDiv_BNU32() +// +*/ + +#include "owncp.h" +#include "pcpbnumisc.h" +#include "pcpbnu32misc.h" +#include "pcpbnu32arith.h" + + +/*F* +// Name: cpDiv_BNU32 +// +// Purpose: BNU32 division. +// +// Returns: +// size of result +// +// Parameters: +// pX source X +// pY source Y +// pQ source quotient +// sizeQ pointer to max size of Q +// sizeX size of A +// sizeY size of B +// +*F*/ + +#if !((_IPP32E==_IPP32E_M7) || \ + (_IPP32E==_IPP32E_U8) || \ + (_IPP32E==_IPP32E_Y8) || \ + (_IPP32E>=_IPP32E_E9) || \ + (_IPP32E==_IPP32E_N8)) +IPP_OWN_DEFN (int, cpDiv_BNU32, (Ipp32u* pQ, cpSize* sizeQ, Ipp32u* pX, cpSize sizeX, Ipp32u* pY, cpSize sizeY)) +{ + FIX_BNU32(pY,sizeY); + FIX_BNU32(pX,sizeX); + + /* special case */ + if(sizeX < sizeY) { + + if(pQ) { + pQ[0] = 0; + *sizeQ = 1; + } + + return sizeX; + } + + /* special case */ + if(1 == sizeY) { + int i; + Ipp32u r = 0; + for(i=(int)sizeX-1; i>=0; i--) { + Ipp64u tmp = IPP_MAKEDWORD(pX[i],r); + Ipp32u q = IPP_LODWORD(tmp / pY[0]); + r = IPP_LODWORD(tmp - q*pY[0]); + if(pQ) pQ[i] = q; + } + + pX[0] = r; + + if(pQ) { + FIX_BNU32(pQ,sizeX); + *sizeQ = sizeX; + } + + return 1; + } + + + /* common case */ + { + cpSize qs = sizeX-sizeY+1; + + cpSize nlz = cpNLZ_BNU32(pY[sizeY-1]); + + /* normalization */ + pX[sizeX] = 0; + if(nlz) { + cpSize ni; + + pX[sizeX] = pX[sizeX-1] >> (32-nlz); + for(ni=sizeX-1; ni>0; ni--) + pX[ni] = (pX[ni]<>(32-nlz)); + pX[0] <<= nlz; + + for(ni=sizeY-1; ni>0; ni--) + pY[ni] = (pY[ni]<>(32-nlz)); + pY[0] <<= nlz; + } + + /* + // division + */ + { + Ipp32u yHi = pY[sizeY-1]; + + int i; + for(i=(int)qs-1; i>=0; i--) { + Ipp32u extend; + + /* estimate digit of quotient */ + Ipp64u tmp = IPP_MAKEDWORD(pX[i+sizeY-1], pX[i+sizeY]); + Ipp64u q = tmp / yHi; + Ipp64u r = tmp - q*yHi; + + /* tune estimation above */ + //for(; (q>=CONST_64(0x100000000)) || (Ipp64u)q*pY[sizeY-2] > IPP_MAKEDWORD(pX[i+sizeY-2],r); ) { + for(; IPP_HIDWORD(q) || (Ipp64u)q*pY[sizeY-2] > IPP_MAKEDWORD(pX[i+sizeY-2],r); ) { + q -= 1; + r += yHi; + if( IPP_HIDWORD(r) ) + break; + } + + /* multiply and subtract */ + extend = cpSubMulDgt_BNU32(pX+i, pY, sizeY, (Ipp32u)q); + extend = (pX[i+sizeY] -= extend); + + if(extend) { /* subtracted too much */ + q -= 1; + extend = cpAdd_BNU32(pX+i, pY, pX+i, sizeY); + pX[i+sizeY] += extend; + } + + /* store quotation digit */ + if(pQ) pQ[i] = IPP_LODWORD(q); + } + } + + /* de-normalization */ + if(nlz) { + cpSize ni; + for(ni=0; ni>nlz) | (pX[ni+1]<<(32-nlz)); + for(ni=0; ni>nlz) | (pY[ni+1]<<(32-nlz)); + pY[sizeY-1] >>= nlz; + } + + FIX_BNU32(pX,sizeX); + + if(pQ) { + FIX_BNU32(pQ,qs); + *sizeQ = qs; + } + + return sizeX; + } +} +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32_arith_inc.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32_arith_inc.c new file mode 100644 index 0000000..c2a18db --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32_arith_inc.c @@ -0,0 +1,59 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal BNU32 arithmetic. +// +// Contents: +// cpInc_BNU32() +// +*/ + +#include "owncp.h" +#include "pcpbnumisc.h" +#include "pcpbnu32misc.h" +#include "pcpbnu32arith.h" + + +/*F* +// Name: cpInc_BNU32 +// +// Purpose: BNU32 increment. +// +// Returns: +// carry +// +// Parameters: +// pA source +// pR result +// ns size +// v borrow +// +*F*/ +IPP_OWN_DEFN (Ipp32u, cpInc_BNU32, (Ipp32u* pR, const Ipp32u* pA, cpSize ns, Ipp32u v)) +{ + Ipp32u carry = v; + cpSize i; + for(i=0; i=_IPP32E_E9) || \ + (_IPP32E==_IPP32E_N8)) +IPP_OWN_DEFN (Ipp32u, cpSubMulDgt_BNU32, (Ipp32u* pR, const Ipp32u* pA, cpSize nsA, Ipp32u val)) +{ + Ipp32u carry = 0; + for(; nsA>0; nsA--) { + Ipp64u r = (Ipp64u)*pR - (Ipp64u)(*pA++) * val - carry; + *pR++ = IPP_LODWORD(r); + carry = 0-IPP_HIDWORD(r); + } + return carry; +} +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32_getoctstr.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32_getoctstr.c new file mode 100644 index 0000000..03e4c83 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32_getoctstr.c @@ -0,0 +1,77 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Unsigned internal BNU32 misc functionality +// +// Contents: +// cpToOctStr_BNU32() +// +*/ + +#include "owncp.h" +#include "pcpbnuimpl.h" +#include "pcpbnumisc.h" +#include "pcpbnu32misc.h" + +/*F* +// Name: cpToOctStr_BNU32 +// +// Purpose: Convert BNU into HexString representation. +// +// Returns: +// length of the string or 0 if no success +// +// Parameters: +// pBNU pointer to the source BN +// bnuSize size of BN +// pStr pointer to the target octet string +// strLen octet string length +*F*/ + +IPP_OWN_DEFN (cpSize, cpToOctStr_BNU32, (Ipp8u* pStr, cpSize strLen, const Ipp32u* pBNU, cpSize bnuSize)) +{ + FIX_BNU32(pBNU, bnuSize); + { + int bnuBitSize = BITSIZE_BNU32(pBNU, bnuSize); + if(bnuBitSize <= strLen*BYTESIZE) { + Ipp32u x = pBNU[bnuSize-1]; + + ZEXPAND_BNU(pStr, 0, strLen); + pStr += strLen - BITS2WORD8_SIZE(bnuBitSize); + + if(x) { + int nb; + for(nb=cpNLZ_BNU32(x)/BYTESIZE; nb<4; nb++) + *pStr++ = EBYTE(x,3-nb); + + for(--bnuSize; bnuSize>0; bnuSize--) { + x = pBNU[bnuSize-1]; + *pStr++ = EBYTE(x,3); + *pStr++ = EBYTE(x,2); + *pStr++ = EBYTE(x,1); + *pStr++ = EBYTE(x,0); + } + } + return strLen; + } + else + return 0; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32_nlz.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32_nlz.c new file mode 100644 index 0000000..a14d3ac --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32_nlz.c @@ -0,0 +1,60 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Unsigned internal BNU32 misc functionality +// +// Contents: +// cpNLZ_BNU32() +// +*/ + +#include "owncp.h" +#include "pcpbnuimpl.h" +#include "pcpbnumisc.h" +#include "pcpbnu32misc.h" + +/*F* +// Name: cpNLZ_BNU32 +// +// Purpose: Returns number of leading zeros of the 32-bit BN chunk. +// +// Returns: +// number of leading zeros of the 32-bit BN chunk +// +// Parameters: +// x BigNum x +// +*F*/ + +#if (_IPP < _IPP_H9) + IPP_OWN_DEFN (cpSize, cpNLZ_BNU32, (Ipp32u x)) + { + cpSize nlz = BITSIZE(Ipp32u); + if(x) { + nlz = 0; + if( 0==(x & 0xFFFF0000) ) { nlz +=16; x<<=16; } + if( 0==(x & 0xFF000000) ) { nlz += 8; x<<= 8; } + if( 0==(x & 0xF0000000) ) { nlz += 4; x<<= 4; } + if( 0==(x & 0xC0000000) ) { nlz += 2; x<<= 2; } + if( 0==(x & 0x80000000) ) { nlz++; } + } + return nlz; + } +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32_setoctstr.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32_setoctstr.c new file mode 100644 index 0000000..629a340 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32_setoctstr.c @@ -0,0 +1,74 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Unsigned internal BNU32 misc functionality +// +// Contents: +// cpToOctStr_BNU32() +// +*/ + +#include "owncp.h" +#include "pcpbnuimpl.h" +#include "pcpbnumisc.h" +#include "pcpbnu32misc.h" + +/*F* +// Name: cpFromOctStr_BNU32 +// +// Purpose: Convert Oct String into BNU representation. +// +// Returns: +// size of BNU in BNU_CHUNK_T chunks +// +// Parameters: +// pOctStr pointer to the source octet string +// strLen octet string length +// pBNU pointer to the target BN +// +*F*/ + +IPP_OWN_DEFN (cpSize, cpFromOctStr_BNU32, (Ipp32u* pBNU, const Ipp8u* pOctStr, cpSize strLen)) +{ + cpSize bnuSize=0; + *pBNU = 0; + + /* start from the end of string */ + for(; strLen>=4; bnuSize++,strLen-=4) { + /* pack 4 bytes into single Ipp32u value*/ + *pBNU++ = (Ipp32u)(( pOctStr[strLen-4]<<(8*3) ) + +( pOctStr[strLen-3]<<(8*2) ) + +( pOctStr[strLen-2]<<(8*1) ) + + pOctStr[strLen-1]); + } + + /* convert the beginning of the string */ + if(strLen) { + Ipp32u x; + for(x=0; strLen>0; strLen--) { + Ipp32u d = *pOctStr++; + x = x*256 + d; + } + *pBNU++ = x; + bnuSize++; + } + + return bnuSize? bnuSize : 1; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32arith.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32arith.h new file mode 100644 index 0000000..1118c96 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32arith.h @@ -0,0 +1,51 @@ +/******************************************************************************* +* Copyright (C) 2012 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. +// Internal BNU32 arithmetic +// +// +*/ + +#if !defined(_CP_BNU32_ARITH_H) +#define _CP_BNU32_ARITH_H + +#define cpAdd_BNU32 OWNAPI(cpAdd_BNU32) + IPP_OWN_DECL (Ipp32u, cpAdd_BNU32, (Ipp32u* pR, const Ipp32u* pA, const Ipp32u* pB, int ns)) +#define cpSub_BNU32 OWNAPI(cpSub_BNU32) + IPP_OWN_DECL (Ipp32u, cpSub_BNU32, (Ipp32u* pR, const Ipp32u* pA, const Ipp32u* pB, int ns)) +#define cpInc_BNU32 OWNAPI(cpInc_BNU32) + IPP_OWN_DECL (Ipp32u, cpInc_BNU32, (Ipp32u* pR, const Ipp32u* pA, cpSize ns, Ipp32u val)) +#define cpDec_BNU32 OWNAPI(cpDec_BNU32) + IPP_OWN_DECL (Ipp32u, cpDec_BNU32, (Ipp32u* pR, const Ipp32u* pA, cpSize ns, Ipp32u val)) +#define cpMulDgt_BNU32 OWNAPI(cpMulDgt_BNU32) + IPP_OWN_DECL (Ipp32u, cpMulDgt_BNU32, (Ipp32u* pR, const Ipp32u* pA, int ns, Ipp32u val)) +#define cpSubMulDgt_BNU32 OWNAPI(cpSubMulDgt_BNU32) + IPP_OWN_DECL (Ipp32u, cpSubMulDgt_BNU32, (Ipp32u* pR, const Ipp32u* pA, int nsA, Ipp32u val)) + +#define cpDiv_BNU32 OWNAPI(cpDiv_BNU32) + IPP_OWN_DECL (int, cpDiv_BNU32, (Ipp32u* pQ, int* nsQ, Ipp32u* pX, int nsX, Ipp32u* pY, int nsY)) +#define cpMod_BNU32(pX,sizeX, pM,sizeM) cpDiv_BNU32(NULL,NULL, (pX),(sizeX), (pM),(sizeM)) + +#define cpFromOS_BNU32 OWNAPI(cpFromOS_BNU32) + IPP_OWN_DECL (int, cpFromOS_BNU32, (Ipp32u* pBNU, const Ipp8u* pOctStr, int strLen)) +#define cpToOS_BNU32 OWNAPI(cpToOS_BNU32) + IPP_OWN_DECL (int, cpToOS_BNU32, (Ipp8u* pStr, int strLen, const Ipp32u* pBNU, int bnuSize)) + +#endif /* _CP_BNU32_ARITH_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32misc.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32misc.h new file mode 100644 index 0000000..e6a62c8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu32misc.h @@ -0,0 +1,98 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal Miscellaneous BNU 32 bit Definitions & Function Prototypes +// +// +*/ + +#if !defined(_CP_BNU32_MISC_H) +#define _CP_BNU32_MISC_H + + +/* bit operations */ +#define BITSIZE_BNU32(p,ns) ((ns)*BNU_CHUNK_32BIT-cpNLZ_BNU32((p)[(ns)-1])) + +/* number of leading/trailing zeros */ +#if (_IPP < _IPP_H9) +#define cpNLZ_BNU32 OWNAPI(cpNLZ_BNU32) + IPP_OWN_DECL (cpSize, cpNLZ_BNU32, (Ipp32u x)) +#else + __INLINE cpSize cpNLZ_BNU32(Ipp32u x) + { + return (cpSize)_lzcnt_u32(x); + } +#endif + +/* Name: cpFix_BNU32 +// +// Purpose: fix up BNU. +// +// Returns: +// fixed nsA +// +// Parameters: +// pA BNU ptr +// nsA size of BNU +// +*/ +__INLINE int cpFix_BNU32(const Ipp32u* pA, int nsA) +{ + Ipp32u zscan = (Ipp32u)(-1); + int outLen = nsA; + for(; nsA>0; nsA--) { + zscan &= (Ipp32u)cpIsZero_ct((BNU_CHUNK_T)pA[nsA-1]); + outLen -= 1 & zscan; + } + return (int)((1 & zscan) | ((BNU_CHUNK_T)outLen & ~(BNU_CHUNK_T)zscan)); // change to scanz +} + +#define FIX_BNU32(src,srcLen) ((srcLen) = cpFix_BNU32((src), (srcLen))) + +/* most significant BNU bit */ +#if 0 +__INLINE int cpMSBit_BNU32(const Ipp32u* pA, cpSize nsA) +{ + FIX_BNU(pA, nsA); + return nsA*BITSIZE(Ipp32u) - cpNLZ_BNU32(pA[nsA-1]) -1; +} +#endif + +#if 0 +__INLINE int cpCmp_BNU32(const Ipp32u* pA, cpSize nsA, const Ipp32u* pB, cpSize nsB) +{ + if(nsA!=nsB) + return nsA>nsB? 1 : -1; + else { + BNU_CHUNK_T idx = 0; + for(; nsA>0; nsA--) + idx |= ~cpIsEqu_ct(pA[nsA-1], pB[nsA-1]) & cpIsZero_ct(idx) & (nsA-1); + return pA[idx] < pB[idx] ? -1 : (pA[idx] > pB[idx] ? 1 : 0); + } +} +#endif + +/* to/from oct string conversion */ +#define cpToOctStr_BNU32 OWNAPI(cpToOctStr_BNU32) + IPP_OWN_DECL (cpSize, cpToOctStr_BNU32, (Ipp8u* pStr, cpSize strLen, const Ipp32u* pBNU, cpSize bnuSize)) +#define cpFromOctStr_BNU32 OWNAPI(cpFromOctStr_BNU32) + IPP_OWN_DECL (cpSize, cpFromOctStr_BNU32, (Ipp32u* pBNU, const Ipp8u* pOctStr, cpSize strLen)) + +#endif /* _CP_BNU32_MISC_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu_arith_add.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu_arith_add.c new file mode 100644 index 0000000..1ada073 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu_arith_add.c @@ -0,0 +1,68 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal Unsigned arithmetic +// +// Contents: +// cpAdd_BNU() +// +*/ + +#include "owncp.h" +#include "pcpbnuarith.h" +#include "pcpbnumisc.h" + + +/*F* +// Name: cpAdd_BNU +// +// Purpose: Addition of two BigNums. +// +// Returns: +// carry of result of add two BigNums. +// +// Parameters: +// pA source BigNum A +// pB source BigNum B +// pR resultant BigNum +// ns size of BigNums +*F*/ +#if !((_IPP==_IPP_W7) || \ + (_IPP==_IPP_T7) || \ + (_IPP==_IPP_V8) || \ + (_IPP==_IPP_P8) || \ + (_IPP>=_IPP_G9) || \ + (_IPP==_IPP_S8) || \ + (_IPP32E==_IPP32E_M7) || \ + (_IPP32E==_IPP32E_U8) || \ + (_IPP32E==_IPP32E_Y8) || \ + (_IPP32E>=_IPP32E_E9) || \ + (_IPP32E==_IPP32E_N8)) || \ + defined(_USE_C_cpAdd_BNU_) +IPP_OWN_DEFN (BNU_CHUNK_T, cpAdd_BNU, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, cpSize ns)) +{ + BNU_CHUNK_T carry = 0; + cpSize i; + for(i=0; i=_IPP_G9) || \ + (_IPP==_IPP_S8) || \ + (_IPP32E==_IPP32E_M7) || \ + (_IPP32E==_IPP32E_U8) || \ + (_IPP32E==_IPP32E_Y8) || \ + (_IPP32E>=_IPP32E_E9) || \ + (_IPP32E==_IPP32E_N8)) || \ + defined(_USE_C_cpAddMulDgt_BNU_) +IPP_OWN_DEFN (BNU_CHUNK_T, cpAddMulDgt_BNU, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, cpSize ns, BNU_CHUNK_T val)) +{ + BNU_CHUNK_T extension = 0; + cpSize i; + for(i=0; i=_IPP32E_E9) || \ + (_IPP32E==_IPP32E_N8)) + +IPP_OWN_DEFN (BNU_CHUNK_T, cpDec_BNU, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, cpSize ns, BNU_CHUNK_T val)) +{ + cpSize i; + for(i=0; i b){ + gcd = a; + t = b; + } else { + t = a; + gcd = b; + } + + while (t != 0) { + r = gcd % t; + gcd = t; + t = r; + } + return gcd; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu_arith_inc.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu_arith_inc.c new file mode 100644 index 0000000..5e30f37 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu_arith_inc.c @@ -0,0 +1,72 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal Unsigned arithmetic +// +// Contents: +// cpInc_BNU() +// +*/ + +#include "owncp.h" +#include "pcpbnuarith.h" +#include "pcpbnumisc.h" + + +/*F* +// Name: cpInc_BNU +// +// Purpose: increment BigNum. +// +// Returns: +// carry of result of inc BigNum. +// +// Parameters: +// pA source BigNum A +// pR resultant BigNum +// ns size of BigNum +// val carry +*F*/ +#if !((_IPP==_IPP_W7) || \ + (_IPP==_IPP_T7) || \ + (_IPP==_IPP_V8) || \ + (_IPP==_IPP_P8) || \ + (_IPP>=_IPP_G9) || \ + (_IPP==_IPP_S8) || \ + (_IPP32E==_IPP32E_M7) || \ + (_IPP32E==_IPP32E_U8) || \ + (_IPP32E==_IPP32E_Y8) || \ + (_IPP32E>=_IPP32E_E9) || \ + (_IPP32E==_IPP32E_N8)) || \ + defined(_USE_C_cpInc_BNU_) +IPP_OWN_DEFN (BNU_CHUNK_T, cpInc_BNU, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, cpSize ns, BNU_CHUNK_T val)) +{ + cpSize i; + for(i=0; i=_IPP_G9) || \ + (_IPP==_IPP_S8) || \ + (_IPP32E==_IPP32E_M7) || \ + (_IPP32E==_IPP32E_U8) || \ + (_IPP32E==_IPP32E_Y8) || \ + (_IPP32E>=_IPP32E_E9) || \ + (_IPP32E==_IPP32E_N8)) || \ + defined(_USE_C_cpMulAdc_BNU_school_) +IPP_OWN_DEFN (BNU_CHUNK_T, cpMulAdc_BNU_school, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, cpSize nsA, const BNU_CHUNK_T* pB, cpSize nsB)) +{ + const BNU_CHUNK_T* pa = (BNU_CHUNK_T*)pA; + const BNU_CHUNK_T* pb = (BNU_CHUNK_T*)pB; + BNU_CHUNK_T* pr = (BNU_CHUNK_T*)pR; + + BNU_CHUNK_T extension = 0; + cpSize i, j; + + ZEXPAND_BNU(pr, 0, nsA+nsB); + + for(i=0; i=_IPP_G9) || \ + (_IPP==_IPP_S8) || \ + (_IPP32E==_IPP32E_M7) || \ + (_IPP32E==_IPP32E_U8) || \ + (_IPP32E==_IPP32E_Y8) || \ + (_IPP32E>=_IPP32E_E9) || \ + (_IPP32E==_IPP32E_N8)) || \ + defined(_USE_C_cpSqrAdc_BNU_school_) +IPP_OWN_DEFN (BNU_CHUNK_T, cpSqrAdc_BNU_school, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, cpSize nsA)) +{ + cpSize i; + + BNU_CHUNK_T extension; + BNU_CHUNK_T rH, rL; + + /* init result */ + pR[0] = 0; + for(i=1, extension=0; i=_IPP_G9) || \ + (_IPP==_IPP_S8) || \ + (_IPP32E==_IPP32E_M7) || \ + (_IPP32E==_IPP32E_U8) || \ + (_IPP32E==_IPP32E_Y8) || \ + (_IPP32E>=_IPP32E_E9) || \ + (_IPP32E==_IPP32E_N8)) || \ + defined(_USE_C_cpSub_BNU_) +IPP_OWN_DEFN (BNU_CHUNK_T, cpSub_BNU, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, cpSize ns)) +{ + BNU_CHUNK_T borrow = 0; + cpSize i; + for(i=0; i=_IPP_G9) || \ + (_IPP==_IPP_S8) || \ + (_IPP32E==_IPP32E_M7) || \ + (_IPP32E==_IPP32E_U8) || \ + (_IPP32E==_IPP32E_Y8) || \ + (_IPP32E>=_IPP32E_E9) || \ + (_IPP32E==_IPP32E_N8)) || \ + defined(_USE_C_cpSubMulDgt_BNU_) +BNU_CHUNK_T cpSubMulDgt_BNU(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, cpSize ns, BNU_CHUNK_T val) +{ + BNU_CHUNK_T extension = 0; + cpSize i; + for(i=0; i0; cnvLen+=(Ipp32s)sizeof(BNU_CHUNK_T), nsA--) { + x = pA[nsA-1]; + #if (BNU_CHUNK_BITS==BNU_CHUNK_64BIT) + *pStr++ = EBYTE(x,7); + *pStr++ = EBYTE(x,6); + *pStr++ = EBYTE(x,5); + *pStr++ = EBYTE(x,4); + #endif + *pStr++ = EBYTE(x,3); + *pStr++ = EBYTE(x,2); + *pStr++ = EBYTE(x,1); + *pStr++ = EBYTE(x,0); + } + } + IPP_UNREFERENCED_PARAMETER(cnvLen); + + return strLen; + } + else + return 0; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu_lsr.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu_lsr.c new file mode 100644 index 0000000..b9e9928 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu_lsr.c @@ -0,0 +1,76 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal Unsigned BNU misc functionality +// +// Contents: +// cpLSL_BNU() +// +*/ + +#include "owncp.h" +#include "pcpbnumisc.h" + + +/*F* +// Name: cpLSR_BNU +// +// Purpose: Logical shift right (including inplace). +// +// Returns: +// new length +// +// Parameters: +// pA BigNum A +// pR result BigNum +// nsA size of A +// nBits size of shift in bits +*F*/ + +IPP_OWN_DEFN (cpSize, cpLSR_BNU, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, cpSize nsA, cpSize nBits)) +{ + cpSize nw = nBits/BNU_CHUNK_BITS; + cpSize n; + + pA += nw; + nsA -= nw; + + nBits %= BNU_CHUNK_BITS; + if(nBits) { + BNU_CHUNK_T hi; + BNU_CHUNK_T lo = pA[0]; + + for(n=0; n<(nsA-1); n++) { + hi = pA[n+1]; + pR[n] = (lo>>nBits) | (hi<<(BNU_CHUNK_BITS-nBits)); + lo = hi; + } + pR[nsA-1] = (lo>>nBits); + } + else { + for(n=0; n= _IPP_H9) || (_IPP32E >= _IPP32E_L9)) +#if 0 + IPP_OWN_DEFN (cpSize, cpNLZ_BNU, (BNU_CHUNK_T x)) + { + cpSize nlz = BNU_CHUNK_BITS; + if(x) { + nlz = 0; + #if (BNU_CHUNK_BITS == BNU_CHUNK_64BIT) + if( 0==(x & 0xFFFFFFFF00000000) ) { nlz +=32; x<<=32; } + if( 0==(x & 0xFFFF000000000000) ) { nlz +=16; x<<=16; } + if( 0==(x & 0xFF00000000000000) ) { nlz += 8; x<<= 8; } + if( 0==(x & 0xF000000000000000) ) { nlz += 4; x<<= 4; } + if( 0==(x & 0xC000000000000000) ) { nlz += 2; x<<= 2; } + if( 0==(x & 0x8000000000000000) ) { nlz++; } + #else + if( 0==(x & 0xFFFF0000) ) { nlz +=16; x<<=16; } + if( 0==(x & 0xFF000000) ) { nlz += 8; x<<= 8; } + if( 0==(x & 0xF0000000) ) { nlz += 4; x<<= 4; } + if( 0==(x & 0xC0000000) ) { nlz += 2; x<<= 2; } + if( 0==(x & 0x80000000) ) { nlz++; } + #endif + } + return nlz; + } +#endif +/* cte version */ +IPP_OWN_DEFN (cpSize, cpNLZ_BNU, (BNU_CHUNK_T x)) +{ + cpSize nlz = 0; + BNU_CHUNK_T + #if (BNU_CHUNK_BITS == BNU_CHUNK_64BIT) + mask = cpIsZero_ct(x & 0xFFFFFFFF00000000); + nlz += 32 & mask; x = ((x<<32) & mask) | (x & ~mask); + + mask = cpIsZero_ct(x & 0xFFFF000000000000); + nlz += 16 & mask; x = ((x<<16) & mask) | (x & ~mask); + + mask = cpIsZero_ct(x & 0xFF00000000000000); + nlz += 8 & mask; x = ((x << 8) & mask) | (x & ~mask); + + mask = cpIsZero_ct(x & 0xF000000000000000); + nlz += 4 & mask; x = ((x << 4) & mask) | (x & ~mask); + + mask = cpIsZero_ct(x & 0xC000000000000000); + nlz += 2 & mask; x = ((x << 2) & mask) | (x & ~mask); + + mask = cpIsZero_ct(x & 0x8000000000000000); + nlz += 1 & mask; x = ((x << 1) & mask) | (x & ~mask); + + mask = cpIsZero_ct(x & 0x8000000000000000); + nlz += 1 & mask; +#else + mask = cpIsZero_ct(x & 0xFFFF0000); + nlz += 16 & mask; x = ((x << 16) & mask) | (x & ~mask); + + mask = cpIsZero_ct(x & 0xFF000000); + nlz += 8 & mask; x = ((x << 8) & mask) | (x & ~mask); + + mask = cpIsZero_ct(x & 0xF0000000); + nlz += 4 & mask; x = ((x << 4) & mask) | (x & ~mask); + + mask = cpIsZero_ct(x & 0xC0000000); + nlz += 2 & mask; x = ((x << 2) & mask) | (x & ~mask); + + mask = cpIsZero_ct(x & 0x80000000); + nlz += 1 & mask; x = ((x << 1) & mask) | (x & ~mask); + + mask = cpIsZero_ct(x & 0x80000000); + nlz += 1 & mask; +#endif + return nlz; +} + +#endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu_ntz.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu_ntz.c new file mode 100644 index 0000000..ff40518 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu_ntz.c @@ -0,0 +1,66 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal Unsigned BNU misc functionality +// +// Contents: +// cpNTZ_BNU() +// +*/ + +#include "owncp.h" +#include "pcpbnumisc.h" + + +/*F* +// Name: cpNTZ_BNU +// +// Purpose: Returns number of trailing zeros of the BNU. +// +// Returns: +// number of trailing zeros of the BNU +// +// Parameters: +// x BigNum x +// +*F*/ + +IPP_OWN_DEFN (cpSize, cpNTZ_BNU, (BNU_CHUNK_T x)) +{ + cpSize ntz = BNU_CHUNK_BITS; + if(x) { + ntz = 0; + #if (BNU_CHUNK_BITS==BNU_CHUNK_64BIT) + if( 0==(x & 0x00000000FFFFFFFF) ) { ntz+=32; x>>=32; } + if( 0==(x & 0x000000000000FFFF) ) { ntz+=16; x>>=16; } + if( 0==(x & 0x00000000000000FF) ) { ntz+= 8; x>>= 8; } + if( 0==(x & 0x000000000000000F) ) { ntz+= 4; x>>= 4; } + if( 0==(x & 0x0000000000000003) ) { ntz+= 2; x>>= 2; } + if( 0==(x & 0x0000000000000001) ) { ntz++; } + #else + if( 0==(x & 0x0000FFFF) ) { ntz+=16; x>>=16; } + if( 0==(x & 0x000000FF) ) { ntz+= 8; x>>= 8; } + if( 0==(x & 0x0000000F) ) { ntz+= 4; x>>= 4; } + if( 0==(x & 0x00000003) ) { ntz+= 2; x>>= 2; } + if( 0==(x & 0x00000001) ) { ntz++; } + #endif + } + return ntz; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu_setoctstr.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu_setoctstr.c new file mode 100644 index 0000000..d0cefe6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnu_setoctstr.c @@ -0,0 +1,78 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal Unsigned BNU misc functionality +// +// Contents: +// cpFromOctStr_BNU() +// +*/ + +#include "owncp.h" +#include "pcpbnumisc.h" + +/*F* +// Name: cpFromOctStr_BNU +// +// Purpose: Convert Oct String into BNU representation. +// +// Returns: +// size of BNU in BNU_CHUNK_T chunks +// +// Parameters: +// pStr pointer to the source octet string +// strLen octet string length +// pA pointer to the target BN +// +*F*/ + +IPP_OWN_DEFN (cpSize, cpFromOctStr_BNU, (BNU_CHUNK_T* pA, const Ipp8u* pStr, cpSize strLen)) +{ + int nsA =0; + + /* start from the end of string */ + for(; strLen>=(int)sizeof(BNU_CHUNK_T); nsA++,strLen-=(int)(sizeof(BNU_CHUNK_T))) { + /* pack sizeof(BNU_CHUNK_T) bytes into single BNU_CHUNK_T value*/ + *pA++ = + #if (BNU_CHUNK_BITS==BNU_CHUNK_64BIT) + +( (BNU_CHUNK_T)pStr[strLen-8]<<(8*7) ) + +( (BNU_CHUNK_T)pStr[strLen-7]<<(8*6) ) + +( (BNU_CHUNK_T)pStr[strLen-6]<<(8*5) ) + +( (BNU_CHUNK_T)pStr[strLen-5]<<(8*4) ) + #endif + +( (BNU_CHUNK_T)pStr[strLen-4]<<(8*3) ) + +( (BNU_CHUNK_T)pStr[strLen-3]<<(8*2) ) + +( (BNU_CHUNK_T)pStr[strLen-2]<<(8*1) ) + + (BNU_CHUNK_T)pStr[strLen-1]; + } + + /* convert the beginning of the string */ + if(strLen) { + BNU_CHUNK_T x = 0; + for(x=0; strLen>0; strLen--) { + BNU_CHUNK_T d = *pStr++; + x = (x<<8) + d; + } + *pA++ = x; + nsA++; + } + + return nsA; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnuarith.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnuarith.h new file mode 100644 index 0000000..4a49f52 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnuarith.h @@ -0,0 +1,189 @@ +/******************************************************************************* +* Copyright (C) 2012 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. +// Internal Unsigned internal arithmetic +// +// +*/ + +#if !defined(_CP_BNU_ARITH_H) +#define _CP_BNU_ARITH_H + +#include "pcpbnuimpl.h" +#include "pcpbnu32arith.h" + +#define cpAdd_BNU OWNAPI(cpAdd_BNU) + IPP_OWN_DECL (BNU_CHUNK_T, cpAdd_BNU, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, cpSize ns)) +#define cpSub_BNU OWNAPI(cpSub_BNU) + IPP_OWN_DECL (BNU_CHUNK_T, cpSub_BNU, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, cpSize ns)) +#define cpInc_BNU OWNAPI(cpInc_BNU) + IPP_OWN_DECL (BNU_CHUNK_T, cpInc_BNU, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, cpSize ns, BNU_CHUNK_T val)) +#define cpDec_BNU OWNAPI(cpDec_BNU) + IPP_OWN_DECL (BNU_CHUNK_T, cpDec_BNU, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, cpSize ns, BNU_CHUNK_T val)) +#define cpAddMulDgt_BNU OWNAPI(cpAddMulDgt_BNU) + IPP_OWN_DECL (BNU_CHUNK_T, cpAddMulDgt_BNU, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, cpSize ns, BNU_CHUNK_T val)) +#define cpMulAdc_BNU_school OWNAPI(cpMulAdc_BNU_school) + IPP_OWN_DECL (BNU_CHUNK_T, cpMulAdc_BNU_school, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, cpSize nsA, const BNU_CHUNK_T* pB, cpSize nsB)) +#define cpMulAdx_BNU_school OWNAPI(cpMulAdx_BNU_school) + IPP_OWN_DECL (BNU_CHUNK_T, cpMulAdx_BNU_school, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, cpSize nsA, const BNU_CHUNK_T* pB, cpSize nsB)) + +/*F* +// Name: cpMul_BNU_school +// +// Purpose: Multiply 2 BigNums. +// +// Returns: +// extension of result of multiply 2 BigNums +// +// Parameters: +// pA source BigNum A +// nsA size of A +// pB source BigNum B +// nsB size of B +// pR resultant BigNum +// +*F*/ + +__INLINE BNU_CHUNK_T cpMul_BNU_school(BNU_CHUNK_T* pR, + const BNU_CHUNK_T* pA, cpSize nsA, + const BNU_CHUNK_T* pB, cpSize nsB) +{ +#if(_ADCOX_NI_ENABLING_==_FEATURE_ON_) + return cpMulAdx_BNU_school(pR, pA,nsA, pB,nsB); +#elif(_ADCOX_NI_ENABLING_==_FEATURE_TICKTOCK_) + return IsFeatureEnabled(ippCPUID_ADCOX)? cpMulAdx_BNU_school(pR, pA,nsA, pB,nsB) + : cpMulAdc_BNU_school(pR, pA,nsA, pB,nsB); +#else + return cpMulAdc_BNU_school(pR, pA,nsA, pB,nsB); +#endif +} + + +#define cpSqrAdc_BNU_school OWNAPI(cpSqrAdc_BNU_school) + IPP_OWN_DECL (BNU_CHUNK_T, cpSqrAdc_BNU_school, (BNU_CHUNK_T * pR, const BNU_CHUNK_T * pA, cpSize nsA)) +#define cpSqrAdx_BNU_school OWNAPI(cpSqrAdx_BNU_school) + IPP_OWN_DECL (BNU_CHUNK_T, cpSqrAdx_BNU_school, (BNU_CHUNK_T * pR, const BNU_CHUNK_T * pA, cpSize nsA)) + +/*F* +// Name: cpSqr_BNU_school +// +// Purpose: Square BigNum. +// +// Returns: +// extension of result of square BigNum +// +// Parameters: +// pA source BigNum +// pR resultant BigNum +// +*F*/ + +__INLINE BNU_CHUNK_T cpSqr_BNU_school(BNU_CHUNK_T * pR, const BNU_CHUNK_T * pA, cpSize nsA) +{ +#if(_ADCOX_NI_ENABLING_==_FEATURE_ON_) + return cpSqrAdx_BNU_school(pR, pA,nsA); +#elif(_ADCOX_NI_ENABLING_==_FEATURE_TICKTOCK_) + return IsFeatureEnabled(ippCPUID_ADCOX)? cpSqrAdx_BNU_school(pR, pA,nsA) + : cpSqrAdc_BNU_school(pR, pA,nsA); +#else + return cpSqrAdc_BNU_school(pR, pA,nsA); +#endif +} + +#define cpGcd_BNU OWNAPI(cpGcd_BNU) + IPP_OWN_DECL (BNU_CHUNK_T, cpGcd_BNU, (BNU_CHUNK_T a, BNU_CHUNK_T b)) +#define cpModInv_BNU OWNAPI(cpModInv_BNU) + IPP_OWN_DECL (int, cpModInv_BNU, (BNU_CHUNK_T* pInv, const BNU_CHUNK_T* pA, cpSize nsA, const BNU_CHUNK_T* pM, cpSize nsM, BNU_CHUNK_T* bufInv, BNU_CHUNK_T* bufA, BNU_CHUNK_T* bufM)) + +/* +// multiplication/squaring wrappers +*/ +__INLINE BNU_CHUNK_T cpMul_BNU(BNU_CHUNK_T* pR, + const BNU_CHUNK_T* pA, cpSize nsA, + const BNU_CHUNK_T* pB, cpSize nsB, + BNU_CHUNK_T* pBuffer) +{ + IPP_UNREFERENCED_PARAMETER(pBuffer); + return cpMul_BNU_school(pR, pA,nsA, pB,nsB); +} +__INLINE BNU_CHUNK_T cpSqr_BNU(BNU_CHUNK_T * pR, + const BNU_CHUNK_T * pA, cpSize nsA, + BNU_CHUNK_T* pBuffer) +{ + IPP_UNREFERENCED_PARAMETER(pBuffer); + return cpSqr_BNU_school(pR, pA,nsA); +} + +/*F* +// Name: cpDiv_BNU +// +// Purpose: division/reduction BigNums. +// +// Returns: +// size of result +// +// Parameters: +// pA source BigNum +// pB source BigNum +// pQ quotient BigNum +// pnsQ pointer to max size of Q +// nsA size of A +// nsB size of B +// +*F*/ + +__INLINE cpSize cpDiv_BNU(BNU_CHUNK_T* pQ, cpSize* pnsQ, BNU_CHUNK_T* pA, cpSize nsA, BNU_CHUNK_T* pB, cpSize nsB) +{ + int nsR = cpDiv_BNU32((Ipp32u*)pQ, pnsQ, + (Ipp32u*)pA, nsA*(Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(Ipp32u)), + (Ipp32u*)pB, nsB*(Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(Ipp32u))); + #if (BNU_CHUNK_BITS == BNU_CHUNK_64BIT) + if(nsR&1) ((Ipp32u*)pA)[nsR] = 0; + nsR = INTERNAL_BNU_LENGTH(nsR); + if(pQ) { + if(*pnsQ&1) ((Ipp32u*)pQ)[*pnsQ] = 0; + *pnsQ = INTERNAL_BNU_LENGTH(*pnsQ); + } + #endif + return nsR; +} + +/*F* +// Name: cpMod_BNU +// +// Purpose: reduction BigNums. +// +// Returns: +// cpDiv_BNU(NULL,NULL, pX,nsX, pModulus, nsM) +// +// Parameters: +// pX source BigNum +// pModulus source BigNum +// nsX size of X +// nsM size of Modulus +// +*F*/ + +__INLINE cpSize cpMod_BNU(BNU_CHUNK_T* pX, cpSize nsX, BNU_CHUNK_T* pModulus, cpSize nsM) +{ + return cpDiv_BNU(NULL,NULL, pX,nsX, pModulus, nsM); +} + +#endif /* _CP_BNU_ARITH_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnuimpl.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnuimpl.h new file mode 100644 index 0000000..6c7d30c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnuimpl.h @@ -0,0 +1,133 @@ +/******************************************************************************* +* Copyright (C) 2012 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. +// BNU data type definition +// +// +// +*/ + +#if !defined(_CP_BNU_IMPL_H) +#define _CP_BNU_IMPL_H + +#define BNU_CHUNK_64BIT (64) +#define BNU_CHUNK_32BIT (32) + + +/* +// define BNU chunk data type +*/ +#if ((_IPP_ARCH == _IPP_ARCH_EM64T) || (_IPP_ARCH == _IPP_ARCH_LP64) || (_IPP_ARCH == _IPP_ARCH_LRB) || (_IPP_ARCH == _IPP_ARCH_LRB2)) + typedef Ipp64u BNU_CHUNK_T; + typedef Ipp64s BNS_CHUNK_T; + #define BNU_CHUNK_LOG2 (6) + #define BNU_CHUNK_BITS BNU_CHUNK_64BIT + +#else + typedef Ipp32u BNU_CHUNK_T; + typedef Ipp32s BNS_CHUNK_T; + #define BNU_CHUNK_LOG2 (5) + #define BNU_CHUNK_BITS BNU_CHUNK_32BIT +#endif + +#define BNU_CHUNK_MASK (~(BNU_CHUNK_T)(0)) + +#if (BNU_CHUNK_BITS != BNU_CHUNK_64BIT && BNU_CHUNK_BITS != BNU_CHUNK_32BIT) + #error BNU_CHUNK_BITS should be either 64 or 32 bit! +#endif + + +#ifdef _MSC_VER +// #pragma warning( disable : 4127 4711 4206) +# pragma warning( disable : 4127) +#endif + +/* user's API BNU chunk data type */ +typedef Ipp32u API_BNU_CHUNK_T; + +/* convert API_BNU_CHUNK_T (usual Ipp32u) length into the BNU_CHUNK_T length */ +#define INTERNAL_BNU_LENGTH(apiLen) \ + ((apiLen) + (Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(API_BNU_CHUNK_T)) -1)/((Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(API_BNU_CHUNK_T))) + +/* Low and High parts of BNU_CHUNK_T value */ +#define BNU_CHUNK_2H ((BNU_CHUNK_T)1 << (BNU_CHUNK_BITS/2)) +#define LO_CHUNK(c) ((BNU_CHUNK_T)(c) & (BNU_CHUNK_2H - 1)) +#define HI_CHUNK(c) ((BNU_CHUNK_T)(c) >> (BNU_CHUNK_BITS/2)) + +/* (carry,R) = A+B */ +#define ADD_AB(CARRY,R, A,B) \ +do { \ + BNU_CHUNK_T __s = (A) + (B); \ + (CARRY) = __s < (A); \ + (R) = __s; \ +} while(0) + +/* (carry,R) = A+B+C */ +#define ADD_ABC(CARRY,R, A,B,C) \ +do { \ + BNU_CHUNK_T __s = (A) + (B); \ + BNU_CHUNK_T __t1= __s < (A); \ + BNU_CHUNK_T __r = __s + (C); \ + BNU_CHUNK_T __t2 = __r < __s; \ + (CARRY) = __t1 + __t2; \ + (R) = __r; \ +} while(0) + +/* (borrow,R) = A-B */ +#define SUB_AB(BORROW,R, A,B) \ +do { \ + (BORROW) = (A)<(B); \ + (R) = (A)-(B); \ +} while(0) + +/* (borrow,R) = A-B-C */ +#define SUB_ABC(BORROW,R, A,B,C) \ +do { \ + BNU_CHUNK_T __s = (A) -( B); \ + BNU_CHUNK_T __t1= __s > (A); \ + BNU_CHUNK_T __r = __s - (C); \ + BNU_CHUNK_T __t2 = __r > __s; \ + (BORROW) = __t1 + __t2; \ + (R) = __r; \ +} while(0) + +/* (RH,RL) = A*B */ +#define MUL_AB(RH, RL, A, B) \ + do { \ + BNU_CHUNK_T __aL = LO_CHUNK((A)); \ + BNU_CHUNK_T __aH = HI_CHUNK((A)); \ + BNU_CHUNK_T __bL = LO_CHUNK((B)); \ + BNU_CHUNK_T __bH = HI_CHUNK((B)); \ + \ + BNU_CHUNK_T __x0 = (BNU_CHUNK_T) __aL * __bL; \ + BNU_CHUNK_T __x1 = (BNU_CHUNK_T) __aL * __bH; \ + BNU_CHUNK_T __x2 = (BNU_CHUNK_T) __aH * __bL; \ + BNU_CHUNK_T __x3 = (BNU_CHUNK_T) __aH * __bH; \ + \ + __x1 += HI_CHUNK(__x0); \ + __x1 += __x2; \ + if(__x1 < __x2) \ + __x3 += BNU_CHUNK_2H; \ + \ + (RH) = __x3 + HI_CHUNK(__x1); \ + (RL) = (__x1 << BNU_CHUNK_BITS/2) + LO_CHUNK(__x0); \ + } while (0) + +#endif /* _CP_BNU_IMPL_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnumisc.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnumisc.h new file mode 100644 index 0000000..3fb6788 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpbnumisc.h @@ -0,0 +1,239 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal Miscellaneous BNU Definitions & Function Prototypes +// +// +*/ + +#if !defined(_PCP_BNUMISC_H) +#define _PCP_BNUMISC_H + +#include "pcpbnuimpl.h" +#include "pcpmask_ct.h" + + +/* bit operations */ +#define BITSIZE_BNU(p,ns) ((ns)*BNU_CHUNK_BITS-cpNLZ_BNU((p)[(ns)-1])) +#define BIT_BNU(bnu, ns,nbit) ((((nbit)>>BNU_CHUNK_LOG2) < (ns))? ((((bnu))[(nbit)>>BNU_CHUNK_LOG2] >>((nbit)&(BNU_CHUNK_BITS))) &1) : 0) + +#define TST_BIT(bnu, nbit) (((Ipp8u*)(bnu))[(nbit)/8] & ((1<<((nbit)%8)) &0xFF)) +#define SET_BIT(bnu, nbit) (((Ipp8u*)(bnu))[(nbit)/8] |= ((1<<((nbit)%8)) &0xFF)) +#define CLR_BIT(bnu, nbit) (((Ipp8u*)(bnu))[(nbit)/8] &=~((1<<((nbit)%8)) &0xFF)) + +/* convert bitsize nbits into the number of BNU_CHUNK_T */ +#define BITS_BNU_CHUNK(nbits) (((nbits)+BNU_CHUNK_BITS-1)/BNU_CHUNK_BITS) + +/* mask for top BNU_CHUNK_T */ +#define MASK_BNU_CHUNK(nbits) ((BNU_CHUNK_T)(-1) >>((BNU_CHUNK_BITS- ((nbits)&(BNU_CHUNK_BITS-1))) &(BNU_CHUNK_BITS-1))) + +/* copy BNU content */ +#define COPY_BNU(dst, src, len) \ +{ \ + cpSize __idx; \ + for(__idx=0; __idx<(len); __idx++) (dst)[__idx] = (src)[__idx]; \ +} + +/* expand by zeros */ +#define ZEXPAND_BNU(srcdst,srcLen, dstLen) \ +{ \ + cpSize __idx; \ + for(__idx=(srcLen); __idx<(dstLen); __idx++) (srcdst)[__idx] = 0; \ +} + +/* copy and expand by zeros */ +#define ZEXPAND_COPY_BNU(dst,dstLen, src,srcLen) \ +{ \ + cpSize __idx; \ + for(__idx=0; __idx<(srcLen); __idx++) (dst)[__idx] = (src)[__idx]; \ + for(; __idx<(dstLen); __idx++) (dst)[__idx] = 0; \ +} + + +/* copy and set */ +__INLINE void cpCpy_BNU(BNU_CHUNK_T* pDst, const BNU_CHUNK_T* pSrc, cpSize ns) +{ COPY_BNU(pDst, pSrc, ns); } + +__INLINE void cpSet_BNU(BNU_CHUNK_T* pDst, cpSize ns, BNU_CHUNK_T val) +{ + ZEXPAND_BNU(pDst, 0, ns); + pDst[0] = val; +} + +/* fix up */ + +/* Name: cpFix_BNU +// +// Purpose: fix up BigNums. +// +// Returns: +// fixed nsA +// +// Parameters: +// pA BigNum ctx +// nsA Size of pA +// +*/ +__INLINE int cpFix_BNU(const BNU_CHUNK_T* pA, int nsA) +{ + BNU_CHUNK_T zscan = (BNU_CHUNK_T)(-1); + int outLen = nsA; + for(; nsA>0; nsA--) { + zscan &= cpIsZero_ct(pA[nsA-1]); + outLen -= 1 & zscan; + } + return (int)((1 & zscan) | ((BNU_CHUNK_T)outLen & ~zscan)); // change to scanz +} + +#define FIX_BNU(src,srcLen) ((srcLen) = cpFix_BNU((src), (srcLen))) + +/* Name: cpCmp_BNU +// +// Purpose: Compare two BigNums. +// +// Returns: +// negative, if A < B +// 0, if A = B +// positive, if A > B +// +// Parameters: +// pA BigNum ctx +// nsA Size of pA +// pB BigNum ctx +// nsB Size of pB +// +*/ +#if 0 +__INLINE int cpCmp_BNU(const BNU_CHUNK_T* pA, cpSize nsA, const BNU_CHUNK_T* pB, cpSize nsB) +{ + if(nsA!=nsB) + return nsA>nsB? 1 : -1; + else { + BNU_CHUNK_T idx = 0; + for(; nsA>0; nsA--) + idx |= ~cpIsEqu_ct(pA[nsA-1], pB[nsA-1]) & cpIsZero_ct(idx) & (BNU_CHUNK_T)(nsA-1); + return pA[idx] < pB[idx] ? -1 : (pA[idx] > pB[idx] ? 1 : 0); + } +} +#endif + +__INLINE int cpCmp_BNU0(const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, int len) +{ + const Ipp32u* a32 = (const Ipp32u*)a; + const Ipp32u* b32 = (const Ipp32u*)b; + len *= (sizeof(BNU_CHUNK_T))/sizeof(Ipp32u); + + // borrow, difference |= (a[]-b[]) + BNU_CHUNK_T borrow = 0; + BNU_CHUNK_T difference = 0; + for(int n=0; n>63); + } + + int resb = (int)( cpIsEqu_ct(borrow, 1) ); + int resd = (int)(~cpIsZero_ct(difference) ) &1; + return (int)(resb|resd); +} + +__INLINE int cpCmp_BNU(const BNU_CHUNK_T* a, int aLen, const BNU_CHUNK_T* b, int bLen) +{ + BNU_CHUNK_T aLen_eq_bLen = cpIsZero_ct((BNU_CHUNK_T)(aLen-bLen)); // FFFF/0000 if (aLen=bLen) / (aLen!=bLen) + BNU_CHUNK_T aLen_gt_bLen = cpIsMsb_ct((BNU_CHUNK_T)(bLen-aLen)) & 1; // 1/0 if (aLen>bLen) / (aLenbLen) + + + int len = (int)(((Ipp32u)aLen & aLen_lt_bLen) | ((Ipp32u)bLen & ~aLen_lt_bLen)); + int cmp_res = cpCmp_BNU0(a, b, len); + + return (int)( aLen_gt_bLen | (aLen_eq_bLen & (Ipp32u)cmp_res) | aLen_lt_bLen ); +} + +/* Name: cpEqu_BNU_CHUNK +// +// Purpose: Compare two BNU_CHUNKs. +// +// Returns: +// positive, if A = b +// 0 , if A != b +// +// Parameters: +// pA BigNum ctx +// nsA Size of pA +// b BNU_CHUNK_T to compare +// +*/ +__INLINE int cpEqu_BNU_CHUNK(const BNU_CHUNK_T* pA, cpSize nsA, BNU_CHUNK_T b) +{ + BNU_CHUNK_T res = pA[0] ^ b; + int n; + for(n=1; n0, if A > 0 +// <0, looks like impossible (or error) case +*/ +__INLINE int cpTst_BNU(const BNU_CHUNK_T* pA, int nsA) +{ + for(; (nsA>0) && (0==pA[nsA-1]); nsA--) ; + return nsA; +} + +/* number of leading/trailing zeros */ +#if !((_IPP >= _IPP_H9) || (_IPP32E >= _IPP32E_L9)) +#define cpNLZ_BNU OWNAPI(cpNLZ_BNU) + IPP_OWN_DECL (cpSize, cpNLZ_BNU, (BNU_CHUNK_T x)) +#else + __INLINE cpSize cpNLZ_BNU(BNU_CHUNK_T x) + { + #if (BNU_CHUNK_BITS == BNU_CHUNK_64BIT) + return (cpSize)_lzcnt_u64(x); + #else + return (cpSize)_lzcnt_u32(x); + #endif + } +#endif + +#define cpNTZ_BNU OWNAPI(cpNTZ_BNU) + IPP_OWN_DECL (cpSize, cpNTZ_BNU, (BNU_CHUNK_T x)) + +/* logical shift left/right */ +#define cpLSR_BNU OWNAPI(cpLSR_BNU) + IPP_OWN_DECL (int, cpLSR_BNU, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, cpSize nsA, cpSize nBits)) + +/* most significant BNU bit */ +#define cpMSBit_BNU OWNAPI(cpMSBit_BNU) + IPP_OWN_DECL (int, cpMSBit_BNU, (const BNU_CHUNK_T* pA, cpSize nsA)) + +/* BNU <-> hex-string conversion */ +#define cpToOctStr_BNU OWNAPI(cpToOctStr_BNU) + IPP_OWN_DECL (int, cpToOctStr_BNU, (Ipp8u* pStr, cpSize strLen, const BNU_CHUNK_T* pA, cpSize nsA)) +#define cpFromOctStr_BNU OWNAPI(cpFromOctStr_BNU) + IPP_OWN_DECL (int, cpFromOctStr_BNU, (BNU_CHUNK_T* pA, const Ipp8u* pStr, cpSize strLen)) + +#endif /* _PCP_BNUMISC_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpcmac.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpcmac.h new file mode 100644 index 0000000..9ba036f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpcmac.h @@ -0,0 +1,71 @@ +/******************************************************************************* +* Copyright (C) 2007 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Ciper-based Message Authentication Code (CMAC) see SP800-38B +// Internal Definitions and Internal Functions Prototypes +// +*/ + +#if !defined(_PCP_CMAC_H) +#define _PCP_CMAC_H + +#include "pcprij.h" + + +/* +// Rijndael128 based CMAC context +*/ +struct _cpAES_CMAC { + Ipp32u idCtx; /* CMAC identifier */ + int index; /* internal buffer entry (free) */ + Ipp8u k1[MBS_RIJ128]; /* k1 subkey */ + Ipp8u k2[MBS_RIJ128]; /* k2 subkey */ + Ipp8u mBuffer[MBS_RIJ128];/* buffer */ + Ipp8u mMAC[MBS_RIJ128]; /* intermediate digest */ + IppsRijndael128Spec mCipherCtx; +}; + +/* alignment */ +//#define CMACRIJ_ALIGNMENT (RIJ_ALIGNMENT) +#define AESCMAC_ALIGNMENT (RIJ_ALIGNMENT) + +/* +// Useful macros +*/ +#define CMAC_SET_ID(stt) ((stt)->idCtx = (Ipp32u)idCtxCMAC ^ (Ipp32u)IPP_UINT_PTR(stt)) +#define CMAC_INDX(stt) ((stt)->index) +#define CMAC_K1(stt) ((stt)->k1) +#define CMAC_K2(stt) ((stt)->k2) +#define CMAC_BUFF(stt) ((stt)->mBuffer) +#define CMAC_MAC(stt) ((stt)->mMAC) +#define CMAC_CIPHER(stt) ((stt)->mCipherCtx) + +#if (_AES_PROB_NOISE == _FEATURE_ON_) +#define AESCMAC_NOISE_PARAMS(ctx) (CMAC_CIPHER(ctx).noiseParams) +#endif + +/* valid context ID */ +#define VALID_AESCMAC_ID(ctx) ((((ctx)->idCtx) ^ (Ipp32u)IPP_UINT_PTR((ctx))) == (Ipp32u)idCtxCMAC) + +#define cpAESCMAC_Update_AES_NI OWNAPI(cpAESCMAC_Update_AES_NI) + IPP_OWN_DECL (void, cpAESCMAC_Update_AES_NI, (Ipp8u* pMac, const Ipp8u* inpBlk, int nBlks, int nr, const Ipp8u* pKeys)) + +#endif /* _PCP_CMAC_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdes.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdes.h new file mode 100644 index 0000000..cd6fdb1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdes.h @@ -0,0 +1,96 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Internal Definitions and +// Internal DES based Encrypt/Decrypt Function Prototypes +// +// +*/ + + +#if !defined(_PCP_DES_H) +#define _PCP_DES_H + + +/* +// really DES round key saved in terms of pear of 24-bit half mentioned in FIPS 46-3 +*/ +typedef Ipp64u RoundKeyDES; +typedef Ipp32u HalfRoundKeyDES; + +/* +// DES context +*/ +struct _cpDES { + Ipp32u idCtx; /* DES spec identifier */ + RoundKeyDES enc_keys[16]; /* array of keys for encryprion */ + RoundKeyDES dec_keys[16]; /* array of keys for decryprion */ +}; + +/* alignment */ +#define DES_ALIGNMENT ((int)(sizeof(Ipp64u))) + +#define MBS_DES (8) /* data block (bytes) */ + +/* +// Useful macros +*/ +#define DES_SET_ID(ctx) ((ctx)->idCtx = (Ipp32u)idCtxDES ^ (Ipp32u)IPP_UINT_PTR(ctx)) +#define DES_RESET_ID(ctx) ((ctx)->idCtx = (Ipp32u)idCtxDES) +#define DES_EKEYS(ctx) ((ctx)->enc_keys) +#define DES_DKEYS(ctx) ((ctx)->dec_keys) + +#define VALID_DES_ID(ctx) ((((ctx)->idCtx) ^ (Ipp32u)IPP_UINT_PTR((ctx))) == (Ipp32u)idCtxDES) + +/* +// Internal Tables +*/ +#define MITIGATED (4) + +#define IMPLEMENTATION MITIGATED + + +#if (IMPLEMENTATION == MITIGATED) + extern const __ALIGN64 Ipp32u DESspbox[16*16]; +#endif + + +/* +// internal functions +*/ +#define SetKey_DES OWNAPI(SetKey_DES) + IPP_OWN_DECL (void, SetKey_DES, (const Ipp8u* pKey, IppsDESSpec* pCtx)) + +#define Cipher_DES OWNAPI(Cipher_DES) + IPP_OWN_DECL (Ipp64u, Cipher_DES, (Ipp64u inpBlk, const RoundKeyDES* pRKey, const Ipp32u spbox[])) + +#define ENCRYPT_DES(blk, pCtx) Cipher_DES((blk), DES_EKEYS((pCtx)), DESspbox) +#define DECRYPT_DES(blk, pCtx) Cipher_DES((blk), DES_DKEYS((pCtx)), DESspbox) + +/* TDES prototypes */ +#define ECB_TDES OWNAPI(ECB_TDES) + IPP_OWN_DECL (void, ECB_TDES, (const Ipp64u* pSrc, Ipp64u* pDst, int nBlocks, const RoundKeyDES* pRKey[3], const Ipp32u spbox[])) +#define EncryptCBC_TDES OWNAPI(EncryptCBC_TDES) + IPP_OWN_DECL (void, EncryptCBC_TDES, (const Ipp64u* pSrc, Ipp64u* pDst, int nBlocks, const RoundKeyDES* pRKey[3], Ipp64u iv, const Ipp32u spbox[])) +#define DecryptCBC_TDES OWNAPI(DecryptCBC_TDES) + IPP_OWN_DECL (void, DecryptCBC_TDES, (const Ipp64u* pSrc, Ipp64u* pDst, int nBlocks, const RoundKeyDES* pRKey[3], Ipp64u iv, const Ipp32u spbox[])) + +#endif /* _PCP_DES_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdescipherm.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdescipherm.c new file mode 100644 index 0000000..81ba39c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdescipherm.c @@ -0,0 +1,177 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// DES Cipher function (MemJam mitigation incuded) +// +// Contents: +// initial permutation: ip() +// final permutation: fp() +// round function: rndm() +// DES block encypt/decrypt: Chipher_DES() +// +// +*/ + + +#include "owndefs.h" +#include "owncp.h" +#include "pcpdes.h" +#include "pcpmask_ct.h" + +/* +// Following implementation of ip(), fp() and rndm() +// assumes input and output of DES as sequence of bytes. +*/ +#define ip(x,y) { \ + Ipp32u t; \ + (x) = ROR32((x), 4); \ + (t) = ((x)^(y)) & 0x0f0f0f0fL; \ + (y)^= (t); \ + (x) = ROR32((x)^(t), 12); \ + (t) = ((y)^(x)) & 0xffff0000L; \ + (y)^= (t); \ + (x) = ROR32((x)^(t), 18); \ + (t) = ((x)^(y)) & 0x33333333L; \ + (y)^= (t); \ + (x) = ROR32((x)^(t), 22); \ + (t) = ((y)^(x)) & 0xff00ff00L; \ + (y)^= (t); \ + (x) = ROR32((x)^(t), 9); \ + (t) = ((x)^(y)) & 0x55555555L; \ + (x) = ROL32((x)^(t), 2); \ + (y) = ROL32((y)^(t), 1); \ +} + +#define fp(x,y) { \ + Ipp32u t; \ + (y) = ROR32((y), 1); \ + (x) = ROR32((x), 2); \ + (t) = ((x)^(y)) & 0x55555555L; \ + (y)^= (t); \ + (x) = ROL32((x)^(t), 9); \ + (t) = ((y)^(x)) & 0xff00ff00L; \ + (y)^= (t); \ + (x) = ROL32((x)^(t), 22); \ + (t) = ((x)^(y)) & 0x33333333L; \ + (y)^= (t); \ + (x) = ROL32((x)^(t), 18); \ + (t) = ((y)^(x)) & 0xffff0000L; \ + (y)^= (t); \ + (x) = ROL32((x)^(t), 12); \ + (t) = ((x)^(y)) & 0x0f0f0f0fL; \ + (y)^= (t); \ + (x) = ROL32((x)^(t), 4); \ +} + + +static BNU_CHUNK_T getSbox(const BNU_CHUNK_T sbox[], int idx) +{ + unsigned int i; + BNU_CHUNK_T res = 0; + for(i=0; i<(64/sizeof(BNU_CHUNK_T)); i++) { + BNU_CHUNK_T mask = cpIsEqu_ct(i, (BNU_CHUNK_T)idx); + res |= sbox[i] & mask; + } + return res; +} +static Ipp8u getSbox8u(const Ipp8u sbox[], int idx) +{ + int shift = idx % (Ipp32s)((sizeof(BNU_CHUNK_T))/sizeof(Ipp8u)); + idx /= ((sizeof(BNU_CHUNK_T))/sizeof(Ipp8u)); + return (Ipp8u)( getSbox((BNU_CHUNK_T*)sbox, idx) >>(shift*8) ); +} +static Ipp32u getSbox32u(const Ipp8u sbox[], int idx) +{ + int shift = idx % (Ipp32s)((sizeof(BNU_CHUNK_T))/sizeof(Ipp32u)); + idx /= ((sizeof(BNU_CHUNK_T))/sizeof(Ipp32u)); + return (Ipp32u)( getSbox((BNU_CHUNK_T*)sbox, idx)>>shift*32 ); +} + +static Ipp32u rndm(Ipp32u x0, Ipp32u x1, Ipp32u* key, const Ipp8u* sbox) +{ + Ipp32u + tt = key[0] ^ (x1 &0x3F3F3F3F); + x0 ^= getSbox32u(sbox+512+64*0, getSbox8u(sbox+64*0, (Ipp8u)tt)); + tt >>= 8; + x0 ^= getSbox32u(sbox+512+64*2, getSbox8u(sbox+64*2, (Ipp8u)tt)); + tt >>= 8; + x0 ^= getSbox32u(sbox+512+64*4, getSbox8u(sbox+64*4, (Ipp8u)tt)); + tt >>= 8; + x0 ^= getSbox32u(sbox+512+64*6, getSbox8u(sbox+64*6, (Ipp8u)tt)); + + tt = (key)[1] ^ (ROR32((x1 &0xF3F3F3F3),4)); + x0 ^= getSbox32u(sbox+512+64*1, getSbox8u(sbox+64*1, (Ipp8u)tt)); + tt >>= 8; + x0 ^= getSbox32u(sbox+512+64*3, getSbox8u(sbox+64*3, (Ipp8u)tt)); + tt >>= 8; + x0 ^= getSbox32u(sbox+512+64*5, getSbox8u(sbox+64*5, (Ipp8u)tt)); + tt >>= 8; + x0 ^= getSbox32u(sbox+512+64*7, getSbox8u(sbox+64*7, (Ipp8u)tt)); + + return x0; +} + +IPP_OWN_DEFN (Ipp64u, Cipher_DES, (Ipp64u inpBlk, const RoundKeyDES* pRKey, const Ipp32u sbox[])) +{ + const Ipp8u* sbox8 = (const Ipp8u*)sbox; + + #if (IPP_ENDIAN == IPP_BIG_ENDIAN) + Ipp32u q0 = IPP_LODWORD(inpBlk); + Ipp32u q1 = IPP_HIDWORD(inpBlk); + q0 = ENDIANNESS(q0); + q1 = ENDIANNESS(q1); + #else + Ipp32u q0 = IPP_HIDWORD(inpBlk); + Ipp32u q1 = IPP_LODWORD(inpBlk); + #endif + + /* apply inverse permutation IP */ + ip(q0,q1); + + /* 16 magic encrypt iterations */ + q0 = rndm(q0,q1, ((HalfRoundKeyDES*)(pRKey)), sbox8); + q1 = rndm(q1,q0, ((HalfRoundKeyDES*)(pRKey+1 )), sbox8); + q0 = rndm(q0,q1, ((HalfRoundKeyDES*)(pRKey+2 )), sbox8); + q1 = rndm(q1,q0, ((HalfRoundKeyDES*)(pRKey+3 )), sbox8); + q0 = rndm(q0,q1, ((HalfRoundKeyDES*)(pRKey+4 )), sbox8); + q1 = rndm(q1,q0, ((HalfRoundKeyDES*)(pRKey+5 )), sbox8); + q0 = rndm(q0,q1, ((HalfRoundKeyDES*)(pRKey+6 )), sbox8); + q1 = rndm(q1,q0, ((HalfRoundKeyDES*)(pRKey+7 )), sbox8); + q0 = rndm(q0,q1, ((HalfRoundKeyDES*)(pRKey+8 )), sbox8); + q1 = rndm(q1,q0, ((HalfRoundKeyDES*)(pRKey+9 )), sbox8); + q0 = rndm(q0,q1, ((HalfRoundKeyDES*)(pRKey+10)), sbox8); + q1 = rndm(q1,q0, ((HalfRoundKeyDES*)(pRKey+11)), sbox8); + q0 = rndm(q0,q1, ((HalfRoundKeyDES*)(pRKey+12)), sbox8); + q1 = rndm(q1,q0, ((HalfRoundKeyDES*)(pRKey+13)), sbox8); + q0 = rndm(q0,q1, ((HalfRoundKeyDES*)(pRKey+14)), sbox8); + q1 = rndm(q1,q0, ((HalfRoundKeyDES*)(pRKey+15)), sbox8); + + /* apply forward permutation FP */ + fp(q1,q0); + + #if (IPP_ENDIAN == IPP_BIG_ENDIAN) + q0 = ENDIANNESS(q0); + q1 = ENDIANNESS(q1); + return IPP_MAKEDWORD(q1,q0); + #else + return IPP_MAKEDWORD(q0,q1); + #endif +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdesgetsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdesgetsize.c new file mode 100644 index 0000000..a284407 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdesgetsize.c @@ -0,0 +1,55 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Set Round Keys for DES/TDES +// +// Contents: +// ippsDESGetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpdes.h" +#include "pcptool.h" + +/*F* +// Name: ippsDESGetSize +// +// Purpose: Returns size of DES spec (bytes). +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer DES spec size +// +*F*/ +IPPFUN(IppStatus, ippsDESGetSize, (int* pSize)) +{ + /* test size's pointer */ + IPP_BAD_PTR1_RET(pSize); + + *pSize = sizeof(IppsDESSpec); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdesinitca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdesinitca.c new file mode 100644 index 0000000..7c66b04 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdesinitca.c @@ -0,0 +1,61 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Set Round Keys for DES/TDES +// +// Contents: +// ippsDESInit() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpdes.h" +#include "pcptool.h" + +/*F* +// Name: ippsDESInit +// +// Purpose: Init DES spec for future usage. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// pKey == NULL +// ippStsNoErr no errors +// +// Parameters: +// pKey pointer to security key +// pCtx pointer DES spec +// +*F*/ +IPPFUN(IppStatus, ippsDESInit,(const Ipp8u* pKey, IppsDESSpec* pCtx)) +{ + /* test key's and spec's pointers */ + IPP_BAD_PTR2_RET(pKey, pCtx); + + /* init DES spec */ + DES_SET_ID(pCtx); + + /* set round keys */ + SetKey_DES(pKey, pCtx); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdesinitpxca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdesinitpxca.c new file mode 100644 index 0000000..6cdbd29 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdesinitpxca.c @@ -0,0 +1,159 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Set Round Keys for DES/TDES +// +// Contents: +// SetKey_DES() +// ippcSetKey_DES() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpdes.h" + + +/* +// bit 0 is left-most in byte (reference FIPS 46-3) +*/ +static int bytebit[] = { + 0x80,0x40,0x20,0x10,0x8,0x4,0x2,0x1 +}; + +/* +// Key schedule-related tables (reference FIPS 46-3) +*/ + +/* PC-1 permuted table (for the user key) */ +static Ipp8u pc1[] = { + 57, 49, 41, 33, 25, 17, 9, + 1, 58, 50, 42, 34, 26, 18, + 10, 2, 59, 51, 43, 35, 27, + 19, 11, 3, 60, 52, 44, 36, + + 63, 55, 47, 39, 31, 23, 15, + 7, 62, 54, 46, 38, 30, 22, + 14, 6, 61, 53, 45, 37, 29, + 21, 13, 5, 28, 20, 12, 4 +}; + +/* number left rotations of PC-1 key */ +static int rotations[] = { + 1,2,4,6,8,10,12,14,15,17,19,21,23,25,27,28 +}; + +/* PC-2 table (for round key constuction) */ +static Ipp8u pc2[] = { + 14, 17, 11, 24, 1, 5, + 3, 28, 15, 6, 21, 10, + 23, 19, 12, 4, 26, 8, + 16, 7, 27, 20, 13, 2, + 41, 52, 31, 37, 47, 55, + 30, 40, 51, 45, 33, 48, + 44, 49, 39, 56, 34, 53, + 46, 42, 50, 36, 29, 32 +}; + +#define SWAP8U(a,b) { Ipp8u x=(a); (a)=(b); (b)=x; } +/* +// Generate key schedule for encryption and decryption +*/ +IPP_OWN_DEFN (void, SetKey_DES, (const Ipp8u* pKey, IppsDESSpec* pCtx)) +{ + RoundKeyDES* pEncRoundKey = DES_EKEYS(pCtx); + RoundKeyDES* pDecRoundKey = DES_DKEYS(pCtx); + + __ALIGN64 Ipp8u pc1key[56]; /* PC-1 permuted user key bits */ + __ALIGN64 Ipp8u rndkey[56]; /* rounded key bits */ + + int nkey; + int i; + + /* + // apply permutation PC-1 + */ + for(i=0; i<56; i++) { + /* location into the user key (bit number) */ + int nBit = pc1[i]-1; + /* location into the user key (byte and offset in the byte) */ + int nByte = nBit>>3; + int offset = nBit & 0x07; + + /* test bit in the user key and set into the permuted */ + pc1key[i] = (Ipp8u)( (pKey[nByte] & bytebit[offset] ) >>(7-offset)); + } + + /* + // key schedule for encryption + */ + for(nkey=0; nkey<16; nkey++) { + Ipp64u tmp = 0; + Ipp8u* rkeyNibble = (Ipp8u*)(&tmp); + + int pc1keyBit, pc1keyBit_m28; + Ipp32u mask; + + /* rotate right pc1key */ + for (i=0; i<28; i++) { + pc1keyBit = i+rotations[nkey]; + pc1keyBit_m28 = pc1keyBit-28; + mask = (Ipp32u)(pc1keyBit_m28>>(BITSIZE(Ipp32u)-1)); + pc1keyBit = (Ipp32s)((Ipp32u)pc1keyBit & mask) | (Ipp32s)((Ipp32u)pc1keyBit_m28 &~mask); + rndkey[i] = pc1key[pc1keyBit]; + } + for (; i<56; i++) { + pc1keyBit = i+rotations[nkey]; + pc1keyBit_m28 = pc1keyBit-28; + mask = (Ipp32u)((pc1keyBit-56)>>(BITSIZE(Ipp32u)-1)); + pc1keyBit = (Ipp32s)((Ipp32u)pc1keyBit & mask) | (Ipp32s)((Ipp32u)pc1keyBit_m28 &~mask); + rndkey[i] = pc1key[pc1keyBit]; + } + + /* + // construct eight 6-bit nibbles of round key + // applying PC-2 permutation to rndkey[] + */ + for(i=0; i<48; i++) { + int offset = i%6; + rkeyNibble[i/6] |= (rndkey[pc2[i]-1]<idCtx = (Ipp32u)idCtxDLP ^ (Ipp32u)IPP_UINT_PTR(ctx)) +#define DLP_RESET_ID(ctx) ((ctx)->idCtx = (Ipp32u)idCtxDLP) +#define DLP_FLAG(ctx) ((ctx)->flag) +#define DLP_BITSIZEP(ctx) ((ctx)->bitSizeP) +#define DLP_BITSIZER(ctx) ((ctx)->bitSizeR) +#define DLP_EXPMETHOD(ctx) ((ctx)->method) + +#define DLP_MONTP0(ctx) ((ctx)->pMontP0) +#define DLP_MONTP1(ctx) ((ctx)->pMontP1) +#define DLP_MONTR(ctx) ((ctx)->pMontR) + +#define DLP_P(ctx) (MOD_MODULUS(DLP_MONTP0((ctx)))) +#define DLP_R(ctx) (MOD_MODULUS(DLP_MONTR((ctx)))) +#define DLP_GENC(ctx) ((ctx)->pGenc) +#define DLP_X(ctx) ((ctx)->pX) +#define DLP_YENC(ctx) ((ctx)->pYenc) + +#define DLP_PRIMEGEN(ctx) ((ctx)->pPrimeGen) + +#define DLP_METBL(ctx) ((ctx)->pMeTable) +#define DLP_BNCTX(ctx) ((ctx)->pBnList) +#if defined(_USE_WINDOW_EXP_) +#define DLP_BNUCTX0(ctx) ((ctx)->pBnuList0) +#define DLP_BNUCTX1(ctx) ((ctx)->pBnuList1) +#endif + +#define DLP_VALID_ID(ctx) ((((ctx)->idCtx) ^ (Ipp32u)IPP_UINT_PTR((ctx))) == (Ipp32u)idCtxDLP) +#define DLP_COMPLETE(ctx) (DLP_FLAG((ctx))==(IppDLPkeyP|IppDLPkeyR|IppDLPkeyG)) + +/* alignment */ +#define DLP_ALIGNMENT ((int)(sizeof(void*))) + +/* pool size for gsModEngine */ +#define DLP_MONT_POOL_LENGTH (6) + +#define cpPackDLPCtx OWNAPI(cpPackDLPCtx) + IPP_OWN_DECL (void, cpPackDLPCtx, (const IppsDLPState* pDLP, Ipp8u* pBuffer)) +#define cpUnpackDLPCtx OWNAPI(cpUnpackDLPCtx) + IPP_OWN_DECL (void, cpUnpackDLPCtx, (const Ipp8u* pBuffer, IppsDLPState* pDLP)) + +#endif /* _PCP_DLP_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlp_packctx.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlp_packctx.c new file mode 100644 index 0000000..f70e8f5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlp_packctx.c @@ -0,0 +1,65 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// DL over Prime Field (initialization) +// +// Contents: +// cpPackDLPCtx() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpdlp.h" +#include "pcptool.h" + +IPP_OWN_DEFN (void, cpPackDLPCtx, (const IppsDLPState* pDLP, Ipp8u* pBuffer)) +{ + IppsDLPState* pB = (IppsDLPState*)(pBuffer); + + CopyBlock(pDLP, pB, sizeof(IppsDLPState)); + DLP_MONTP0(pB) = (gsModEngine*)((Ipp8u*)NULL + IPP_UINT_PTR(DLP_MONTP0(pDLP))-IPP_UINT_PTR(pDLP)); + DLP_MONTP1(pB) = NULL; + DLP_MONTR(pB) = (gsModEngine*)((Ipp8u*)NULL + IPP_UINT_PTR(DLP_MONTR(pDLP)) -IPP_UINT_PTR(pDLP)); + + DLP_GENC(pB) = (IppsBigNumState*)((Ipp8u*)NULL + IPP_UINT_PTR(DLP_GENC(pDLP)) -IPP_UINT_PTR(pDLP)); + DLP_X(pB) = (IppsBigNumState*)((Ipp8u*)NULL + IPP_UINT_PTR(DLP_X(pDLP)) -IPP_UINT_PTR(pDLP)); + DLP_YENC(pB) = (IppsBigNumState*)((Ipp8u*)NULL + IPP_UINT_PTR(DLP_YENC(pDLP)) -IPP_UINT_PTR(pDLP)); + + DLP_PRIMEGEN(pB)= (IppsPrimeState*)((Ipp8u*)NULL + IPP_UINT_PTR(DLP_PRIMEGEN(pDLP))-IPP_UINT_PTR(pDLP)); + + DLP_METBL(pB) = (BNU_CHUNK_T*)((Ipp8u*)NULL + IPP_UINT_PTR(DLP_METBL(pDLP)) -IPP_UINT_PTR(pDLP)); + + DLP_BNCTX(pB) = (BigNumNode*)((Ipp8u*)NULL + IPP_UINT_PTR(DLP_BNCTX(pDLP)) -IPP_UINT_PTR(pDLP)); + #if defined(_USE_WINDOW_EXP_) + DLP_BNUCTX0(pB) = (WINDOW==DLP_EXPMETHOD(pDLP))?(BNU_CHUNK_T*)((Ipp8u*)NULL + IPP_UINT_PTR(DLP_BNUCTX0(pDLP))-IPP_UINT_PTR(pDLP)) : NULL; + DLP_BNUCTX1(pB) = NULL; + #endif + + gsPackModEngineCtx(DLP_MONTP0(pDLP), (Ipp8u*)pB+IPP_UINT_PTR(DLP_MONTP0(pB))); + gsPackModEngineCtx(DLP_MONTR(pDLP), (Ipp8u*)pB+IPP_UINT_PTR(DLP_MONTR(pB))); + + cpPackBigNumCtx(DLP_GENC(pDLP), (Ipp8u*)pB+IPP_UINT_PTR(DLP_GENC(pB))); + cpPackBigNumCtx(DLP_X(pDLP), (Ipp8u*)pB+IPP_UINT_PTR(DLP_X(pB))); + cpPackBigNumCtx(DLP_YENC(pDLP), (Ipp8u*)pB+IPP_UINT_PTR(DLP_YENC(pB))); + + cpPackPrimeCtx(DLP_PRIMEGEN(pDLP), (Ipp8u*)pB+IPP_UINT_PTR(DLP_PRIMEGEN(pB))); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlp_unpackctx.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlp_unpackctx.c new file mode 100644 index 0000000..7bf5b90 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlp_unpackctx.c @@ -0,0 +1,63 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// DL over Prime Field (initialization) +// +// Contents: +// cpUnpackDLPCtx() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpdlp.h" +#include "pcptool.h" + +IPP_OWN_DEFN (void, cpUnpackDLPCtx, (const Ipp8u* pBuffer, IppsDLPState* pDLP)) +{ + IppsDLPState* pB = (IppsDLPState*)(pBuffer); + + CopyBlock(pB, pDLP, sizeof(IppsDLPState)); + DLP_MONTP0(pDLP) = (gsModEngine*)((Ipp8u*)pDLP+ IPP_UINT_PTR(DLP_MONTP0(pB))); + DLP_MONTP1(pDLP) = NULL; + DLP_MONTR(pDLP) = (gsModEngine*)((Ipp8u*)pDLP+ IPP_UINT_PTR(DLP_MONTR(pB))); + + DLP_GENC(pDLP) = (IppsBigNumState*)((Ipp8u*)pDLP+ IPP_UINT_PTR(DLP_GENC(pB))); + DLP_X(pDLP) = (IppsBigNumState*)((Ipp8u*)pDLP+ IPP_UINT_PTR(DLP_X(pB))); + DLP_YENC(pDLP) = (IppsBigNumState*)((Ipp8u*)pDLP+ IPP_UINT_PTR(DLP_YENC(pB))); + + DLP_PRIMEGEN(pDLP)= (IppsPrimeState*)((Ipp8u*)pDLP+ IPP_UINT_PTR(DLP_PRIMEGEN(pB))); + + DLP_METBL(pDLP) = (BNU_CHUNK_T*)((Ipp8u*)pDLP+ IPP_UINT_PTR(DLP_METBL(pB))); + DLP_BNCTX(pDLP) = (BigNumNode*)((Ipp8u*)pDLP+ IPP_UINT_PTR(DLP_BNCTX(pB))); + #if defined(_USE_WINDOW_EXP_) + DLP_BNUCTX0(pDLP) = (WINDOW==DLP_EXPMETHOD(pDLP))?(BNU_CHUNK_T*)((Ipp8u*)pDLP+ IPP_UINT_PTR(DLP_BNUCTX0(pB))) : NULL; + DLP_BNUCTX1(pDLP) = NULL; + #endif + + gsUnpackModEngineCtx((Ipp8u*)pB+IPP_UINT_PTR(DLP_MONTP0(pB)), DLP_MONTP0(pDLP)); + gsUnpackModEngineCtx((Ipp8u*)pB+IPP_UINT_PTR(DLP_MONTR(pB)), DLP_MONTR(pDLP)); + cpUnpackBigNumCtx((Ipp8u*)pB+IPP_UINT_PTR(DLP_GENC(pB)), DLP_GENC(pDLP)); + cpUnpackBigNumCtx((Ipp8u*)pB+IPP_UINT_PTR(DLP_X(pB)), DLP_X(pDLP)); + cpUnpackBigNumCtx((Ipp8u*)pB+IPP_UINT_PTR(DLP_YENC(pB)), DLP_YENC(pDLP)); + cpUnpackPrimeCtx((Ipp8u*)pB+IPP_UINT_PTR(DLP_PRIMEGEN(pB)),DLP_PRIMEGEN(pDLP)); + cpBigNumListInit(DLP_BITSIZEP(pDLP)+1, BNLISTSIZE, DLP_BNCTX(pDLP)); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpgeneratedh.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpgeneratedh.c new file mode 100644 index 0000000..b95f8d7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpgeneratedh.c @@ -0,0 +1,346 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// DL over Prime Finite Field (generate domain parameters) +// +// Contents: +// ippsDLPGenerateDH() +// ippsDLPGenerateDSA() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpdlp.h" + + +/*F* +// Name: ippsDLPGenerateDH +// +// Purpose: Generate DL (DH) Domain Parameters. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pDH +// NULL == rndFunc +// +// ippStsContextMatchErr illegal pDH->idCtx +// illegal pSeedIn->idCtx +// illegal pSeedOut->idCtx +// +// ippStsSizeErr !(MIN_DLPDH_BITSIZE <= DLP_BITSIZEP()) +// !(256|DLP_BITSIZEP()) +// !(MIN_DLPDH_BITSIZER <= DLP_BITSIZER()) +// +// ippStsRangeErr BitSize(pSeedIn) < DH_BITSIZER() +// no room for pSeedOut +// +// ippStsBadArgErr nTrials <=0 +// +// ippStsInsufficientEntropy +// genration failure due to poor random generator +// +// ippStsNoErr no errors +// +// Parameters: +// pSeedIn pointer to the input seed (probably null) +// nTrials number of trials of primality test +// pDL pointer to the DL context +// pSeedOut pointer to the output seed (probably null, or ==pSeedIn) +// pCounter pointer to the generator loop counter (probably null) +// rndFunc external random generator +// pRndParam pointer to the external random generator params +// +// Note: +// 1) pSeedIn==NULL means, that rndFunc will be used for input seed generation +// 2) PseedIn!=NULL limited by DL bitsizeP parameter! +*F*/ +#define MAXLOOP (4096) + +IPPFUN(IppStatus, ippsDLPGenerateDH,(const IppsBigNumState* pSeedIn, + int nTrials, IppsDLPState* pDL, + IppsBigNumState* pSeedOut, int* pCounter, + IppBitSupplier rndFunc, void* pRndParam)) +{ + /* test DL context */ + IPP_BAD_PTR1_RET(pDL); + IPP_BADARG_RET(!DLP_VALID_ID(pDL), ippStsContextMatchErr); + + /* test DL sizes */ + IPP_BADARG_RET(MIN_DLPDH_BITSIZER>DLP_BITSIZER(pDL), ippStsSizeErr); + IPP_BADARG_RET(MIN_DLPDH_BITSIZE >DLP_BITSIZEP(pDL), ippStsSizeErr); + IPP_BADARG_RET(DLP_BITSIZEP(pDL)%256, ippStsSizeErr); + + /* test number of trials for primality check */ + IPP_BADARG_RET(nTrials<=0, ippStsBadArgErr); + + IPP_BAD_PTR1_RET(rndFunc); + + { + /* allocate BN resources */ + BigNumNode* pList = DLP_BNCTX(pDL); + IppsBigNumState* pP = cpBigNumListGet(&pList); + IppsBigNumState* pR = cpBigNumListGet(&pList); + IppsBigNumState* pG = cpBigNumListGet(&pList); + + IppsBigNumState* pSeed1 = cpBigNumListGet(&pList); + IppsBigNumState* pSeed2 = cpBigNumListGet(&pList); + + /* interally generates SeedIn value by default */ + IppBool seed_is_random = ippTrue; + int seedBitSize = DLP_BITSIZER(pDL); + + DLP_FLAG(pDL) = 0; + /* + // DH generator uses input SEED + // either defined by user or generated internally + */ + if(pSeedIn) { + /* test SeedIn */ + IPP_BADARG_RET(!BN_VALID_ID(pSeedIn), ippStsContextMatchErr); + seedBitSize = BITSIZE_BNU(BN_NUMBER(pSeedIn), BN_SIZE(pSeedIn)); + IPP_BADARG_RET(DLP_BITSIZER(pDL)>seedBitSize, ippStsRangeErr); + IPP_BADARG_RET(BN_ROOM(pSeed1)BITSIZE(BNU_CHUNK_T)*BN_ROOM(pSeedOut), ippStsRangeErr); + } + + /* + // generation DSA domain parameters + */ + { + int feBitsize = DLP_BITSIZEP(pDL); + int ordBitsize = DLP_BITSIZER(pDL); + int m = (ordBitsize + IPP_SHA1_DIGEST_BITSIZE -1)/IPP_SHA1_DIGEST_BITSIZE; + int l = (feBitsize + IPP_SHA1_DIGEST_BITSIZE -1)/IPP_SHA1_DIGEST_BITSIZE; + int n = (feBitsize + 1023)/1024; + + IppsPrimeState* pPrimeGen = DLP_PRIMEGEN(pDL); + + /* pointers to the BNU32-value of SEED */ + Ipp32u* pSeed1BNU32 = (Ipp32u*)BN_NUMBER(pSeed1); + Ipp32u* pSeed2BNU32 = (Ipp32u*)BN_NUMBER(pSeed2); + int seedSize32 = BITS2WORD32_SIZE(seedBitSize); + /* pointers to the octet-string-value of SEED */ + Ipp8u* pSeed1Oct = (Ipp8u*)BN_BUFFER(pSeed1); + Ipp8u* pSeed2Oct = (Ipp8u*)BN_BUFFER(pSeed2); + + int octSize; + Ipp8u shaDgst1[BITS2WORD8_SIZE(IPP_SHA1_DIGEST_BITSIZE)]; + Ipp8u shaDgst2[BITS2WORD8_SIZE(IPP_SHA1_DIGEST_BITSIZE)]; + + int primeGenerated = 0; + int genCounter; + + ippsSet_BN(ippBigNumPOS, 1, (Ipp32u*)&m, pP); /* P = m */ + + /* + // generate prime Q + */ + for(genCounter=0; genCounteridCtx +// illegal pSeedIn->idCtx +// illegal pSeedOut->idCtx +// +// ippStsSizeErr !(MIN_DLPDLP_BITSIZE <= DLP_BITSIZEP() <= MAX_DLPDLP_BITSIZE) +// !(64|DLP_BITSIZEP()) +// !(DEF_DLPDLP_BITSIZER == DLP_BITSIZER()) +// +// ippStsRangeErr BitSize(pSeedIn) < MIN_DLPDLP_SEEDSIZE +// BitSize(pSeedIn) > DLP_BITSIZEP() +// no room for pSeedOut +// +// ippStsBadArgErr nTrials <=0 +// +// ippStsInsufficientEntropy +// genration failure due to poor random generator +// +// ippStsNoErr no errors +// +// Parameters: +// pSeedIn pointer to the input seed (probably null) +// nTrials number of trials of primality test +// pDL pointer to the DL context +// pSeedOut pointer to the output seed (probably null, or ==pSeedIn) +// pCounter pointer to the generator loop counter (probably null) +// rndFunc external random generator +// pRndParam pointer to the external random generator params +// +// Note: +// 1) pSeedIn==NULL means, that rndFunc will be used for input seed generation +// 2) PseedIn!=NULL limited by DSA bitsize (L) parameter! +*F*/ +#define R_MAXLOOP (100) +#define P_MAXLOOP (4096) +#define G_MAXLOOP (100) + +IPPFUN(IppStatus, ippsDLPGenerateDSA,(const IppsBigNumState* pSeedIn, + int nTrials, IppsDLPState *pDL, + IppsBigNumState* pSeedOut, int* pCounter, + IppBitSupplier rndFunc, void* pRndParam)) +{ + /* test DL context */ + IPP_BAD_PTR1_RET(pDL); + IPP_BADARG_RET(!DLP_VALID_ID(pDL), ippStsContextMatchErr); + + /* test DL sizes */ + IPP_BADARG_RET(DEF_DLPDSA_BITSIZER != DLP_BITSIZER(pDL),ippStsSizeErr); + IPP_BADARG_RET(MIN_DLPDSA_BITSIZE >DLP_BITSIZEP(pDL), ippStsSizeErr); + IPP_BADARG_RET(MAX_DLPDSA_BITSIZE = MIN_DLPDSA_SEEDSIZE (==160 bits) - it's FIPS-186 claim + - divisible by 8 + (week request, because provided automatically by BITS2WORD8_SIZE(seedBitSize) conversion) + + If seedBitSize (been calculated above) <160 this means + that there are some zero msb bits in SeedIn representation. + For example, if SeedIn hex string is + is 0000000000000000000000000000000000000001 + or even 01 + we'll think about 160 bit length + **/ + if(seedBitSizeidCtx +// invalid pPrivate->idCtx +// invalid pPublic->idCtx +// +// ippStsIncompleteContextErr +// incomplete context +// +// ippStsRangeErr not enough room for: +// pPrivate, +// pPublic +// +// ippStsNoErr no error +// +// Parameters: +// pPrvKey pointer to the new privatea key +// pPubKey pointer to the corrsponding public key +// pDL pointer to the DL context +// rndFunc external random generator +// pRndParam pointer to the external random generator params +*F*/ +IPPFUN(IppStatus, ippsDLPGenKeyPair,(IppsBigNumState* pPrvKey, IppsBigNumState* pPubKey, + IppsDLPState* pDL, + IppBitSupplier rndFunc, void* pRndParam)) +{ + /* test DL context */ + IPP_BAD_PTR1_RET(pDL); + IPP_BADARG_RET(!DLP_VALID_ID(pDL), ippStsContextMatchErr); + + /* test flag */ + IPP_BADARG_RET(!DLP_COMPLETE(pDL), ippStsIncompleteContextErr); + + /* test random generator */ + IPP_BAD_PTR1_RET(rndFunc); + + /* test private/public keys */ + IPP_BAD_PTR2_RET(pPrvKey, pPubKey); + IPP_BADARG_RET(!BN_VALID_ID(pPrvKey), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pPubKey), ippStsContextMatchErr); + IPP_BADARG_RET(DLP_BITSIZER(pDL)>BITSIZE(BNU_CHUNK_T)*BN_ROOM(pPrvKey), ippStsRangeErr); + IPP_BADARG_RET(DLP_BITSIZEP(pDL)>BITSIZE(BNU_CHUNK_T)*BN_ROOM(pPubKey), ippStsRangeErr); + + { + /* + // generate random private key X: 0 < X < R + */ + BNU_CHUNK_T* pOrder = DLP_R(pDL); + cpSize ordBitSize = DLP_BITSIZER(pDL); + cpSize ordLen = BITS_BNU_CHUNK(ordBitSize); + BNU_CHUNK_T xMask = MASK_BNU_CHUNK(ordBitSize); + + //BNU_CHUNK_T* pY = BN_NUMBER(pPubKey); + BNU_CHUNK_T* pX = BN_NUMBER(pPrvKey); + + //gsModEngine* pME = DLP_MONTP0(pDL); + + do { + rndFunc((Ipp32u*)pX, ordBitSize, pRndParam); + pX[ordLen-1] &= xMask; + } while( cpEqu_BNU_CHUNK(pX, ordLen, 0) || cpCmp_BNU(pX,ordLen, pOrder,ordLen)>=0 ); + BN_SIZE(pPrvKey) = ordLen; + BN_SIGN(pPrvKey) = ippBigNumPOS; + + /* + // compute public key: G^prvKey (mod P) + */ + cpMontExpBin_BN_sscm(pPubKey, DLP_GENC(pDL), pPrvKey, DLP_MONTP0(pDL)); + cpMontDec_BN(pPubKey, pPubKey, DLP_MONTP0(pDL)); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpget.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpget.c new file mode 100644 index 0000000..b0e5009 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpget.c @@ -0,0 +1,96 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// DL over Prime Finite Field (setup/retrieve domain parameters) +// +// Contents: +// ippsDLPGet() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpdlp.h" + +/*F* +// Name: ippsDLPGet +// +// Purpose: Retrieve DLP Domain Parameter. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pP +// NULL == pR +// NULL == pG +// NULL == pDL +// +// ippStsContextMatchErr illegal pP->idCtx +// illegal pR->idCtx +// illegal pG->idCtx +// illegal pDL->idCtx +// +// ippStsIncompleteContextErr +// incomplete context +// +// ippStsRangeErr not enough room for: +// pP, +// pR, +// pG +// +// ippStsNoErr no errors +// +// Parameters: +// pP pointer to the retrieval P +// pR pointer to the retrieval R +// pG pointer to the retrieval G +// pDSA pointer to the DL context +// +*F*/ +IPPFUN(IppStatus, ippsDLPGet,(IppsBigNumState* pP, + IppsBigNumState* pR, + IppsBigNumState* pG, + IppsDLPState* pDL)) +{ + /* test DL context */ + IPP_BAD_PTR1_RET(pDL); + IPP_BADARG_RET(!DLP_VALID_ID(pDL), ippStsContextMatchErr); + + /* test flag */ + IPP_BADARG_RET(!DLP_COMPLETE(pDL), ippStsIncompleteContextErr); + + /* test DL parameters */ + IPP_BAD_PTR3_RET(pP, pR, pG); + IPP_BADARG_RET(!BN_VALID_ID(pP), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pR), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pG), ippStsContextMatchErr); + + /* test size of retrieval parameters */ + IPP_BADARG_RET(BN_ROOM(pP)idCtx +// illegal pDL->idCtx +// +// ippStsIncompleteContextErr requested parameter hasn't set up +// +// ippStsOutOfRangeErr BN_ROOM(pDP) < BN_ROOM(DLP_{P|R|G}(pDL)) +// +// ippStsBadArgErr invalid key tag +// +// errors produced by ippsSet_BN() +// +// ippStsNoErr no errors +// +// Parameters: +// pDP pointer to the DL domain parameter +// tag DLP key component tag +// pDL pointer to the DL context +// +*F*/ +IPPFUN(IppStatus, ippsDLPGetDP,(IppsBigNumState* pDP, IppDLPKeyTag tag, const IppsDLPState* pDL)) +{ + /* test DL context */ + IPP_BAD_PTR1_RET(pDL); + IPP_BADARG_RET(!DLP_VALID_ID(pDL), ippStsContextMatchErr); + + /* test DL parameter to be set */ + IPP_BAD_PTR1_RET(pDP); + IPP_BADARG_RET(!BN_VALID_ID(pDP), ippStsContextMatchErr); + + { + IppStatus sts = ippStsNoErr; + + switch(tag) { + case ippDLPkeyP: + if(DLP_FLAG(pDL)&ippDLPkeyP) + sts = ippsSet_BN(ippBigNumPOS, BITS2WORD32_SIZE(DLP_BITSIZEP(pDL)), (Ipp32u*)DLP_P(pDL), pDP); + else + sts = ippStsIncompleteContextErr; + break; + case ippDLPkeyR: + if(DLP_FLAG(pDL)&ippDLPkeyR) + sts = ippsSet_BN(ippBigNumPOS, BITS2WORD32_SIZE(DLP_BITSIZER(pDL)), (Ipp32u*)DLP_R(pDL), pDP); + else + sts = ippStsIncompleteContextErr; + break; + case ippDLPkeyG: + if(DLP_FLAG(pDL)&ippDLPkeyG) { + cpMontDec_BN(pDP, DLP_GENC(pDL), DLP_MONTP0(pDL)); + } + else + sts = ippStsIncompleteContextErr; + break; + default: + sts = ippStsBadArgErr; + } + + return sts; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpgetsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpgetsize.c new file mode 100644 index 0000000..6c74f27 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpgetsize.c @@ -0,0 +1,125 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// DL over Prime Field (initialization) +// +// Contents: +// ippsDLPGetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpdlp.h" +#include "pcptool.h" + +#include "pcpmont_exp_bufsize.h" + + + +/*F* +// Name: ippsDLPGetSize +// +// Purpose: Returns size of DL context (bytes). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pSize +// +// ippStsSizeErr MIN_DLP_BITSIZE > feBitSize +// MIN_DLP_BITSIZER > ordBitSize +// ordBitSize >= feBitSize +// +// ippStsNoErr no errors +// +// Parameters: +// feBitSize size (bits) of field element (GF(p)) of DL system +// ordBitSize size (bits) of subgroup (GF(p)) generator order +// pSize pointer to the size of DLP context (bytes) +// +*F*/ +IPPFUN(IppStatus, ippsDLPGetSize,(int feBitSize, int ordBitSize, int *pSize)) +{ + /* test size's pointer */ + IPP_BAD_PTR1_RET(pSize); + + /* test sizes of DL system */ + IPP_BADARG_RET(MIN_DLP_BITSIZE >feBitSize, ippStsSizeErr); + IPP_BADARG_RET(MIN_DLP_BITSIZER>ordBitSize, ippStsSizeErr); + IPP_BADARG_RET(ordBitSize>=feBitSize, ippStsSizeErr); + + { + int bnSizeP; + int bnSizeR; + int montSizeP; + int montSizeR; + int primeGenSize; + int bn_resourceSize; + + #if defined(_USE_WINDOW_EXP_) + int window = cpMontExp_WinSize(ordBitSize); + int bnu_resourceSize = window==1? 0 : cpMontExpScratchBufferSize(feBitSize, ordBitSize, 1); + #endif + + /* size of GF(P) element */ + int sizeP = BITS2WORD32_SIZE(feBitSize); + /* size of GF(R) element */ + int sizeR = BITS2WORD32_SIZE(ordBitSize); + /* sizeof multi-exp table */ + int sizeMeTable = cpMontExpScratchBufferSize(feBitSize, ordBitSize, 2); + + /* size of BigNum over GF(P) */ + ippsBigNumGetSize(sizeP, &bnSizeP); + /* size of BigNum over GF(R) */ + ippsBigNumGetSize(sizeR, &bnSizeR); + + /* size of montgomery engine over GF(P) */ + gsModEngineGetSize(feBitSize, DLP_MONT_POOL_LENGTH, &montSizeP); + + /* size of montgomery engine over GF(R) */ + gsModEngineGetSize(ordBitSize, DLP_MONT_POOL_LENGTH, &montSizeR); + + /* size of prime engine */ + ippsPrimeGetSize(feBitSize, &primeGenSize); + + /* size of big num list (big num in the list preserve 32 bit word) */ + bn_resourceSize = cpBigNumListGetSize(feBitSize+1, BNLISTSIZE); + + *pSize = (Ipp32s)sizeof(IppsDLPState) + +montSizeP /* montgomery(P) */ + +montSizeR /* montgomery(Q) */ + + +bnSizeP /* Genc */ + +bnSizeR /* X */ + +bnSizeP /* Y */ + + +primeGenSize /* prime engine */ + + +sizeMeTable /* pre-computed multi-exp table */ + + +bn_resourceSize /* BN resource */ + #if defined(_USE_WINDOW_EXP_) + +bnu_resourceSize /* BNU resource */ + #endif + ; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpinit.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpinit.c new file mode 100644 index 0000000..5ded86f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpinit.c @@ -0,0 +1,160 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// DL over Prime Field (initialization) +// +// Contents: +// ippsDLPInit() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpdlp.h" +#include "pcptool.h" + +#include "pcpmont_exp_bufsize.h" + +/*F* +// Name: ippsDLPInit +// +// Purpose: Init DL context. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pDSA +// +// ippStsSizeErr MIN_DLP_BITSIZE > feBitSize +// MIN_DLP_BITSIZER > ordBitSize +// ordBitSize >= feBitSize +// +// ippStsNoErr no errors +// +// Parameters: +// feBitSize size (bits) of field element (GF(p)) of DL system +// ordBitSize size (bits) of subgroup (GF(p)) generator order +// pDL pointer to the DL context +// +*F*/ +IPPFUN(IppStatus, ippsDLPInit,(int feBitSize, int ordBitSize, IppsDLPState* pDL)) +{ + /* test DSA context */ + IPP_BAD_PTR1_RET(pDL); + + /* test sizes of DL system */ + IPP_BADARG_RET(MIN_DLP_BITSIZE >feBitSize, ippStsSizeErr); + IPP_BADARG_RET(MIN_DLP_BITSIZER>ordBitSize, ippStsSizeErr); + IPP_BADARG_RET(ordBitSize>=feBitSize, ippStsSizeErr); + + DLP_SET_ID(pDL); + DLP_FLAG(pDL) = 0; + DLP_BITSIZEP(pDL) = feBitSize; + DLP_BITSIZER(pDL) = ordBitSize; + DLP_EXPMETHOD(pDL)= BINARY; + + /* + // init other context fields + */ + { + int bnSizeP; + int bnSizeR; + int montSizeP; + int montSizeR; + int primeGenSize; + int bn_resourceSize; + + Ipp8u* ptr = (Ipp8u*)pDL; + + /* size of GF(P) element */ + int sizeP = BITS2WORD32_SIZE(feBitSize); + /* size of GF(R) element */ + int sizeR = BITS2WORD32_SIZE(ordBitSize); + /* size of pre-computed multi-exp table */ + int sizeMeTable = cpMontExpScratchBufferSize(feBitSize, ordBitSize, 2); + + /* size of window for exponentiation */ + #if defined(_USE_WINDOW_EXP_) + int window = cpMontExp_WinSize(ordBitSize); + DLP_EXPMETHOD(pDL) = window>1? WINDOW : BINARY; + #endif + + /* size of BigNum over GF(P) */ + ippsBigNumGetSize(sizeP, &bnSizeP); + /* size of BigNum over GF(R) */ + ippsBigNumGetSize(sizeR, &bnSizeR); + + /* size of montgomery engine over GF(P) */ + gsModEngineGetSize(feBitSize, DLP_MONT_POOL_LENGTH, &montSizeP); + + /* size of montgomery engine over GF(R) */ + gsModEngineGetSize(ordBitSize, DLP_MONT_POOL_LENGTH, &montSizeR); + + /* size of prime engine */ + ippsPrimeGetSize(feBitSize, &primeGenSize); + + /* size of big num list (big num in the list preserve 32 bit word) */ + bn_resourceSize = cpBigNumListGetSize(feBitSize+1, BNLISTSIZE); + + /* allocate buffers */ + ptr += sizeof(IppsDLPState); + DLP_MONTP0(pDL) = (gsModEngine*)(ptr); + ptr += montSizeP; + DLP_MONTP1(pDL) = 0; + DLP_MONTR(pDL) = (gsModEngine*)(ptr); + ptr += montSizeR; + + DLP_GENC(pDL) = (IppsBigNumState*)(ptr); + ptr += bnSizeP; + DLP_X(pDL) = (IppsBigNumState*)(ptr); + ptr += bnSizeR; + DLP_YENC(pDL) = (IppsBigNumState*)(ptr); + ptr += bnSizeP; + + DLP_PRIMEGEN(pDL)= (IppsPrimeState*)(ptr); + ptr += primeGenSize; + + DLP_METBL(pDL) = (BNU_CHUNK_T*)( IPP_ALIGNED_PTR(ptr, CACHE_LINE_SIZE) ); + ptr += sizeMeTable; + + DLP_BNCTX(pDL) = (BigNumNode*)(ptr); + ptr += bn_resourceSize; + + #if defined(_USE_WINDOW_EXP_) + DLP_BNUCTX0(pDL) = 0; + DLP_BNUCTX1(pDL) = 0; + DLP_BNUCTX0(pDL) = window>1? (BNU_CHUNK_T*)( IPP_ALIGNED_PTR(ptr, (int)sizeof(BNU_CHUNK_T)) ) : 0; + #endif + + /* init buffers */ + gsModEngineInit(DLP_MONTP0(pDL), NULL, feBitSize, DLP_MONT_POOL_LENGTH, gsModArithDLP()); + + gsModEngineInit(DLP_MONTR(pDL), NULL, ordBitSize, DLP_MONT_POOL_LENGTH, gsModArithDLP()); + + ippsBigNumInit(sizeP, DLP_GENC(pDL)); + ippsBigNumInit(sizeP, DLP_YENC(pDL)); + ippsBigNumInit(sizeR, DLP_X(pDL)); + + ippsPrimeInit(feBitSize, DLP_PRIMEGEN(pDL)); + + cpBigNumListInit(feBitSize+1, BNLISTSIZE, DLP_BNCTX(pDL)); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlppack.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlppack.c new file mode 100644 index 0000000..745cca5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlppack.c @@ -0,0 +1,61 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// DL over Prime Field (initialization) +// +// Contents: +// ippsDLPPack() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpdlp.h" +#include "pcptool.h" + +/*F* +// Name: ippsDLPPack +// +// Purpose: Copy initialized context to the buffer. +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL +// pCtx == NULL +// ippStsNoErr no errors +// +// Parameters: +// pCtx pointer DLP ctx +// pSize pointer to the packed spec size +// +*F*/ +IPPFUN(IppStatus, ippsDLPPack,(const IppsDLPState* pDL, Ipp8u* pBuffer)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pDL, pBuffer); + /* test the context */ + IPP_BADARG_RET(!DLP_VALID_ID(pDL), ippStsContextMatchErr); + + cpPackDLPCtx(pDL, pBuffer); + IppsDLPState* pCopy = (IppsDLPState*)pBuffer; + DLP_RESET_ID(pCopy); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlppublickey.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlppublickey.c new file mode 100644 index 0000000..279a6a9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlppublickey.c @@ -0,0 +1,103 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// DL over Prime Finite Field (EC Key Generation, Validation and Set Up) +// +// Contents: +// ippsDLPPublicKey() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpdlp.h" + +/*F* +// Name: ippsDLPPublicKey +// +// Purpose: Compute DL public key +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pDL +// NULL == pPrvKey +// NULL == pPubKey +// +// ippStsContextMatchErr invalid pDL->idCtx +// invalid pPrvKey->idCtx +// invalid pPubKey->idCtx +// +// ippStsIncompleteContextErr +// incomplete context +// +// ippStsInvalidPrivateKey !(0 < pPrivate < DLP_R()) +// +// ippStsRangeErr not enough room for pPubKey +// +// ippStsNoErr no error +// +// Parameters: +// pPrvKey pointer to the private key +// pPubKey pointer to the public key +// pDL pointer to the DL context +*F*/ +IPPFUN(IppStatus, ippsDLPPublicKey,(const IppsBigNumState* pPrvKey, + IppsBigNumState* pPubKey, + IppsDLPState* pDL)) +{ + /* test DL context */ + IPP_BAD_PTR1_RET(pDL); + IPP_BADARG_RET(!DLP_VALID_ID(pDL), ippStsContextMatchErr); + + /* test flag */ + IPP_BADARG_RET(!DLP_COMPLETE(pDL), ippStsIncompleteContextErr); + + /* test private/public keys */ + IPP_BAD_PTR2_RET(pPrvKey, pPubKey); + IPP_BADARG_RET(!BN_VALID_ID(pPrvKey), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pPubKey), ippStsContextMatchErr); + + /* test private key */ + IPP_BADARG_RET((0<=cpBN_cmp(cpBN_OneRef(), pPrvKey))|| + (0<=cpCmp_BNU(BN_NUMBER(pPrvKey),BN_SIZE(pPrvKey), DLP_R(pDL),BITS_BNU_CHUNK(DLP_BITSIZER(pDL)))), ippStsInvalidPrivateKey); + + /* test public key's room */ + IPP_BADARG_RET(BN_ROOM(pPubKey)idCtx +// illegal pPubKey->idCtx +// illegal pSecret->idCtx +// +// ippStsIncompleteContextErr +// incomplete context +// +// ippStsRangeErr no room for pSecret +// +// ippStsNoErr no errors +// +// Parameters: +// pPrvKeyA pointer to the own private key +// pPubKeyB pointer to the partner's public key +// pSecret pointer to the secret value +// pDL pointer to the DL context +*F*/ +IPPFUN(IppStatus, ippsDLPSharedSecretDH,(const IppsBigNumState* pPrvKeyA, + const IppsBigNumState* pPubKeyB, + IppsBigNumState* pSecret, + IppsDLPState* pDL)) +{ + /* test DL context */ + IPP_BAD_PTR1_RET(pDL); + IPP_BADARG_RET(!DLP_VALID_ID(pDL), ippStsContextMatchErr); + + /* test flag */ + IPP_BADARG_RET(!DLP_COMPLETE(pDL), ippStsIncompleteContextErr); + + /* test public key */ + IPP_BAD_PTR1_RET(pPrvKeyA); + IPP_BADARG_RET(!BN_VALID_ID(pPrvKeyA), ippStsContextMatchErr); + + /* test public key */ + IPP_BAD_PTR1_RET(pPubKeyB); + IPP_BADARG_RET(!BN_VALID_ID(pPubKeyB), ippStsContextMatchErr); + + /* test secret */ + IPP_BAD_PTR1_RET(pSecret); + IPP_BADARG_RET(!BN_VALID_ID(pSecret), ippStsContextMatchErr); + IPP_BADARG_RET(BITS_BNU_CHUNK(DLP_BITSIZEP(pDL))>BN_ROOM(pSecret), ippStsRangeErr); + + cpMontEnc_BN(pSecret, pPubKeyB, DLP_MONTP0(pDL)); + + { + gsModEngine* pMEorder = DLP_MONTR(pDL); + int ordLen = MOD_LEN(pMEorder); + + /* expand privKeyA */ + BigNumNode* pList = DLP_BNCTX(pDL); + IppsBigNumState* pTmpPrivKey = cpBigNumListGet(&pList); + ZEXPAND_COPY_BNU(BN_NUMBER(pTmpPrivKey), ordLen, BN_NUMBER(pPrvKeyA), BN_SIZE(pPrvKeyA)); + BN_SIZE(pTmpPrivKey) = ordLen; + + #if !defined(_USE_WINDOW_EXP_) + cpMontExpBin_BN_sscm(pSecret, pSecret, pTmpPrivKey, DLP_MONTP0(pDL)); + #else + (DLP_EXPMETHOD(pDL)==BINARY) || (1==cpMontExp_WinSize(BITSIZE_BNU(BN_NUMBER(pTmpPrivKey), BN_SIZE(pTmpPrivKey))))? + cpMontExpBin_BN_sscm(pSecret, pSecret, pTmpPrivKey, DLP_MONTP0(pDL)) : + cpMontExpWin_BN_sscm(pSecret, pSecret, pTmpPrivKey, DLP_MONTP0(pDL), DLP_BNUCTX0(pDL)); + #endif + + cpMontDec_BN(pSecret, pSecret, DLP_MONTP0(pDL)); + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpset.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpset.c new file mode 100644 index 0000000..68dd7d3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpset.c @@ -0,0 +1,109 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// DL over Prime Finite Field (setup/retrieve domain parameters) +// +// Contents: +// ippsDLPSet() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpdlp.h" + +/*F* +// Name: ippsDLPSet +// +// Purpose: Set DL Domain Parameters. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pP +// NULL == pR +// NULL == pG +// NULL == pDL +// +// ippStsContextMatchErr illegal pP->idCtx +// illegal pR->idCtx +// illegal pG->idCtx +// illegal pDL->idCtx +// +// ippStsRangeErr not enough room for: +// pP, +// pR, +// pG +// +// errors produced by ippsMontSet() +// ippsMontForm() +// +// ippStsNoErr no errors +// +// Parameters: +// pP pointer to the DL domain parameter (P) +// pR pointer to the DL domain parameter (R) +// pG pointer to the DL domain parameter (G) +// pDSA pointer to the DL context +// +*F*/ +IPPFUN(IppStatus, ippsDLPSet,(const IppsBigNumState* pP, + const IppsBigNumState* pR, + const IppsBigNumState* pG, + IppsDLPState* pDL)) +{ + /* test DL context */ + IPP_BAD_PTR1_RET(pDL); + IPP_BADARG_RET(!DLP_VALID_ID(pDL), ippStsContextMatchErr); + + /* test DL domain parameters */ + IPP_BAD_PTR3_RET(pP, pR, pG); + IPP_BADARG_RET(!BN_VALID_ID(pP), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pR), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pG), ippStsContextMatchErr); + + /* test size of DL domain parameters */ + IPP_BADARG_RET(BN_SIZE(pP)>BITS_BNU_CHUNK(DLP_BITSIZEP(pDL)), ippStsRangeErr); + IPP_BADARG_RET(BN_SIZE(pR)>BITS_BNU_CHUNK(DLP_BITSIZER(pDL)), ippStsRangeErr); + IPP_BADARG_RET(BN_SIZE(pG)>BITS_BNU_CHUNK(DLP_BITSIZEP(pDL)), ippStsRangeErr); + + /* + // set up DL domain parameters + */ + { + IppStatus sts; + + DLP_FLAG(pDL) = 0; + + cpBN_zero(DLP_X(pDL)); + cpBN_zero(DLP_YENC(pDL)); + + sts = gsModEngineInit(DLP_MONTP0(pDL), (Ipp32u*)BN_NUMBER(pP), cpBN_bitsize(pP), DLP_MONT_POOL_LENGTH, gsModArithDLP()); + + if(ippStsNoErr==sts) { + sts = gsModEngineInit(DLP_MONTR(pDL), (Ipp32u*)BN_NUMBER(pR), cpBN_bitsize(pR), DLP_MONT_POOL_LENGTH, gsModArithDLP()); + if(ippStsNoErr==sts) { + cpMontEnc_BN(DLP_GENC(pDL), pG, DLP_MONTP0(pDL)); + DLP_FLAG(pDL) = ippDLPkeyP|ippDLPkeyR|ippDLPkeyG; + return ippStsNoErr; + } + } + return sts; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpsetdp.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpsetdp.c new file mode 100644 index 0000000..284335d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpsetdp.c @@ -0,0 +1,106 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// DL over Prime Finite Field (setup/retrieve domain parameters) +// +// Contents: +// ippsDLPSetDP() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpdlp.h" + +/*F* +// Name: ippsDLPSetDP +// +// Purpose: Set tagged DL Domain Parameter. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pDP +// NULL == pDL +// +// ippStsContextMatchErr illegal pDP->idCtx +// illegal pDL->idCtx +// +// ippStsRangeErr not enough room for pDP +// +// ippStsBadArgErr invalid key tag +// +// errors produced by ippsMontSet() +// ippsMontForm() +// +// ippStsNoErr no errors +// +// Parameters: +// pDP pointer to the DL domain parameter +// tag DLP key component tag +// pDL pointer to the DL context +// +*F*/ +IPPFUN(IppStatus, ippsDLPSetDP,(const IppsBigNumState* pDP, IppDLPKeyTag tag, IppsDLPState* pDL)) +{ + /* test DL context */ + IPP_BAD_PTR1_RET(pDL); + IPP_BADARG_RET(!DLP_VALID_ID(pDL), ippStsContextMatchErr); + + /* test DL parameter to be set */ + IPP_BAD_PTR1_RET(pDP); + IPP_BADARG_RET(!BN_VALID_ID(pDP), ippStsContextMatchErr); + IPP_BADARG_RET(BN_NEGATIVE(pDP), ippStsBadArgErr); + + { + IppStatus sts = ippStsNoErr; + + cpBN_zero(DLP_X(pDL)); + cpBN_zero(DLP_YENC(pDL)); + + switch(tag) { + case ippDLPkeyP: + DLP_FLAG(pDL) &=(Ipp32u)~ippDLPkeyP; + sts = gsModEngineInit(DLP_MONTP0(pDL), (Ipp32u*)BN_NUMBER(pDP), cpBN_bitsize(pDP), DLP_MONT_POOL_LENGTH, gsModArithDLP()); + if(ippStsNoErr==sts) { + DLP_FLAG(pDL) |= ippDLPkeyP; + } + break; + case ippDLPkeyR: + DLP_FLAG(pDL) &=(Ipp32u)~ippDLPkeyR; + sts = gsModEngineInit(DLP_MONTR(pDL), (Ipp32u*)BN_NUMBER(pDP), cpBN_bitsize(pDP), DLP_MONT_POOL_LENGTH, gsModArithDLP()); + if(ippStsNoErr==sts) + DLP_FLAG(pDL) |= ippDLPkeyR; + break; + case ippDLPkeyG: + DLP_FLAG(pDL) &=(Ipp32u)~ippDLPkeyG; + if(DLP_FLAG(pDL)&ippDLPkeyP) { + cpMontEnc_BN(DLP_GENC(pDL), pDP, DLP_MONTP0(pDL)); + DLP_FLAG(pDL) |= ippDLPkeyG; + } + else + sts = ippStsIncompleteContextErr; + break; + default: + sts = ippStsBadArgErr; + } + + return sts; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpsetkeypair.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpsetkeypair.c new file mode 100644 index 0000000..abb0359 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpsetkeypair.c @@ -0,0 +1,116 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// DL over Prime Finite Field (EC Key Generation, Validation and Set Up) +// +// Contents: +// ippsDLPSetKeyPair() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpdlp.h" + +/*F* +// Name: ippsDSASetKeyPair +// +// Purpose: Set up Key Pair into the DL context +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pDL +// NULL == pPrvKey +// NULL == pPubKey +// +// ippStsContextMatchErr invalid pDL->idCtx +// invalid pPrvKey->idCtx +// invalid pPubKey->idCtx +// +// ippStsIncompleteContextErr +// incomplete context: P and/or R and/or G is not set +// +// ippStsInvalidPrivateKey PrvKey >= R +// PrvKey < 0 +// +// ippStsRangeErr PubKey >= P +// PubKey < 0 +// +// ippStsNoErr no error +// +// Parameters: +// pPrvKey pointer to the private key +// pPubKey pointer to the public key +// pDL pointer to the DL context +*F*/ +IPPFUN(IppStatus, ippsDLPSetKeyPair,(const IppsBigNumState* pPrvKey, + const IppsBigNumState* pPubKey, + IppsDLPState* pDL)) +{ + /* test DL context */ + IPP_BAD_PTR1_RET(pDL); + IPP_BADARG_RET(!DLP_VALID_ID(pDL), ippStsContextMatchErr); + + /* test flag */ + IPP_BADARG_RET(!DLP_COMPLETE(pDL), ippStsIncompleteContextErr); + + /* set up private key */ + if(pPrvKey) { + IPP_BADARG_RET(!BN_VALID_ID(pPrvKey), ippStsContextMatchErr); + IPP_BADARG_RET(BN_NEGATIVE(pPrvKey), ippStsInvalidPrivateKey); + { + gsModEngine* pMontR = DLP_MONTR(pDL); + BNU_CHUNK_T* pOrder = MOD_MODULUS(pMontR); + int ordLen = MOD_LEN(pMontR); + + BNU_CHUNK_T* pPriData = BN_NUMBER(pPrvKey); + int priLen = BN_SIZE(pPrvKey); + + /* make sure regular 0 < private < order */ + IPP_BADARG_RET(cpEqu_BNU_CHUNK(pPriData, priLen, 0) || + 0<=cpCmp_BNU(pPriData, priLen, pOrder, ordLen), ippStsInvalidPrivateKey); + + cpBN_copy(DLP_X(pDL), pPrvKey); + BN_SIZE(DLP_X(pDL)) = ordLen; + } + } + + /* set up public key */ + if(pPubKey) { + IPP_BADARG_RET(!BN_VALID_ID(pPubKey), ippStsContextMatchErr); + IPP_BADARG_RET(BN_NEGATIVE(pPubKey), ippStsRangeErr); + { + gsModEngine* pMontP = DLP_MONTP0(pDL); + BNU_CHUNK_T* pPrime = MOD_MODULUS(pMontP); + int primeLen = MOD_LEN(pMontP); + + BNU_CHUNK_T* pPubData = BN_NUMBER(pPubKey); + int pubLen = BN_SIZE(pPubKey); + + /* make sure regular 0 < public < prime */ + IPP_BADARG_RET(cpEqu_BNU_CHUNK(pPubData, pubLen, 0) || + 0<=cpCmp_BNU(pPubData, pubLen, pPrime, primeLen), ippStsRangeErr); + + cpMontEnc_BN(DLP_YENC(pDL), pPubKey, DLP_MONTP0(pDL)); + } + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpsigndsaca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpsigndsaca.c new file mode 100644 index 0000000..c5ea361 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpsigndsaca.c @@ -0,0 +1,193 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// DL over Prime Finite Field (Sign, DSA version) +// +// Contents: +// ippsDLPSignDSA() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpdlp.h" +#include "pcpeccp.h" + + +/*F* +// Name: ippsDLPSignDSA +// +// Purpose: Signing of message representative +// (DSA version). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pDL +// NULL == pPrvKey +// NULL == pMsgDigest +// NULL == pSignR +// NULL == pSignS +// +// ippStsContextMatchErr illegal pDL->idCtx +// illegal pPrvKey->idCtx +// illegal pMsgDigest->idCtx +// illegal pSignR->idCtx +// illegal pSignS->idCtx +// +// ippStsIncompleteContextErr +// incomplete context: P and/or R and/or G is not set +// +// ippStsMessageErr MsgDigest >= R +// MsgDigest < 0 +// +// ippStsInvalidPrivateKey PrvKey >= R +// PrvKey < 0 +// +// ippStsRangeErr not enough room for: +// signR +// signS +// +// ippStsNoErr no errors +// +// Parameters: +// pMsgDigest pointer to the message representative to be signed +// pPevKey pointer to the (signatory's) regular private key +// pSignR,pSignS pointer to the signature +// pDL pointer to the DL context +// +// Primitive sequence call: +// 1) set up domain parameters +// 2) generate (signatory's) ephemeral key pair +// 3) set up (signatory's) ephemeral key pair +// 4) use primitive with (signatory's) private key +*F*/ +IPPFUN(IppStatus, ippsDLPSignDSA,(const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pPrvKey, + IppsBigNumState* pSignR, + IppsBigNumState* pSignS, + IppsDLPState *pDL)) +{ + /* test DL context */ + IPP_BAD_PTR1_RET(pDL); + IPP_BADARG_RET(!DLP_VALID_ID(pDL), ippStsContextMatchErr); + + /* test flag */ + IPP_BADARG_RET(!DLP_COMPLETE(pDL), ippStsIncompleteContextErr); + + /* test message representative */ + IPP_BAD_PTR1_RET(pMsgDigest); + IPP_BADARG_RET(!BN_VALID_ID(pMsgDigest), ippStsContextMatchErr); + IPP_BADARG_RET(BN_NEGATIVE(pMsgDigest), ippStsMessageErr); + + /* test regular private key */ + IPP_BAD_PTR1_RET(pPrvKey); + IPP_BADARG_RET(!BN_VALID_ID(pPrvKey), ippStsContextMatchErr); + IPP_BADARG_RET(BN_NEGATIVE(pPrvKey), ippStsInvalidPrivateKey); + + /* test signature */ + IPP_BAD_PTR2_RET(pSignR,pSignS); + IPP_BADARG_RET(!BN_VALID_ID(pSignR), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pSignS), ippStsContextMatchErr); + IPP_BADARG_RET(BITSIZE(BNU_CHUNK_T)*BN_ROOM(pSignR)decode(buffer, buffer, pMontP); + ns = cpMod_BNU(buffer, elmLen, pOrder, ordLen); + ZEXPAND_COPY_BNU(dataR, ordLen, buffer, ns); + gsModPoolFree(pMontP, 1); + + if(!cpEqu_BNU_CHUNK(dataR, ordLen, 0)) { + /* + // signS = ((1/eX)*(MsgDigest + X*signR)) (mod R) + */ + + /* private representation in Montgomery domain */ + ZEXPAND_COPY_BNU(dataS, ordLen, pPriData, priLen); + MOD_METHOD(pMontR)->encode(dataS, dataS, pMontR); + + /* (X*signR) in regular domain */ + MOD_METHOD(pMontR)->mul(dataS, dataS, dataR, pMontR); + + /* pMsgDigest + (X*signR) */ + ZEXPAND_COPY_BNU(buffS, ordLen, pMsgData, msgLen); + cpModAdd_BNU(dataS, dataS, buffS, pOrder, ordLen, buffS); + + if(!cpEqu_BNU_CHUNK(dataS, ordLen, 0)) { + + ZEXPAND_COPY_BNU(buffS, ordLen, BN_NUMBER(DLP_X(pDL)), BN_SIZE(DLP_X(pDL))); + /* (1/eX) in Montgomery domain */ + gs_mont_inv(buffS, buffS, pMontR, alm_mont_inv_ct); + + /* signS = (1/eX)*(MsgDigest + X*signR) */ + MOD_METHOD(pMontR)->mul(dataS, dataS, buffS, pMontR); + + /* signR */ + ns = ordLen; + FIX_BNU(dataR, ns); + BN_SIGN(pSignR) = ippBigNumPOS; + BN_SIZE(pSignR) = ns; + /* signS */ + ns = ordLen; + FIX_BNU(dataS, ns); + BN_SIGN(pSignS) = ippBigNumPOS; + BN_SIZE(pSignS) = ns; + + return ippStsNoErr; + } + } + + return ippStsEphemeralKeyErr; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpunpack.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpunpack.c new file mode 100644 index 0000000..4824ac2 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpunpack.c @@ -0,0 +1,58 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// DL over Prime Field (initialization) +// +// Contents: +// ippsDLPUnpack() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpdlp.h" +#include "pcptool.h" + +/*F* +// Name: ippsDLPUnpack +// +// Purpose: Unpack buffer content into the initialized context. +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL +// pCtx == NULL +// ippStsNoErr no errors +// +// Parameters: +// pCtx pointer DLP ctx +// pSize pointer to the packed spec size +// +*F*/ +IPPFUN(IppStatus, ippsDLPUnpack,(const Ipp8u* pBuffer, IppsDLPState* pDL)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pDL, pBuffer); + + cpUnpackDLPCtx(pBuffer, pDL); + DLP_SET_ID(pDL); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpvalidatedh.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpvalidatedh.c new file mode 100644 index 0000000..a6e8132 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpvalidatedh.c @@ -0,0 +1,208 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// DL over Prime Finite Field (validate domain parameters) +// +// Contents: +// ippsDLPValidateDH() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpdlp.h" + + +/* +// Name: DLPValidate +// +// Purpose: Validate DL Domain Parameters. +// +// Parameters: +// nTrials number of trials of primality test +// pResult pointer to the validation result +// pDL pointer to the DL context +// rndFunc external random generator +// pRndParam pointer to the external random generator params +*/ +static +IppDLResult DLPValidate(int nTrials, IppsDLPState* pDL, + IppBitSupplier rndFunc, void* pRndParam) +{ + /* + // validate DL parameters: + // check that P is odd and P > 2 + // check that R is odd and R > 2 + // check that R | (P-1) + // check that 1 < G < P + // check that 1 == G^R (mod P) + */ + + BNU_CHUNK_T* pP = DLP_P(pDL); + BNU_CHUNK_T* pR = DLP_R(pDL); + cpSize lenP = BITS_BNU_CHUNK(DLP_BITSIZEP(pDL)); + cpSize lenR = BITS_BNU_CHUNK(DLP_BITSIZER(pDL)); + + IppsPrimeState* pPrimeCtx = DLP_PRIMEGEN(pDL); + + /* allocate BN resources */ + BigNumNode* pList = DLP_BNCTX(pDL); + IppsBigNumState* pTmp = cpBigNumListGet(&pList); + BNU_CHUNK_T* pT = BN_NUMBER(pTmp); + + /* P is odd and prime */ + if(0 == (pP[0] & 1)) + return ippDLBaseIsEven; + if(0==cpPrimeTest(pP, lenP, nTrials, pPrimeCtx, rndFunc,pRndParam)) + return ippDLCompositeBase; + + /* R is odd and prime */ + if(0 == (pR[0] & 1)) + return ippDLOrderIsEven; + if(0==cpPrimeTest(pR, lenR, nTrials, pPrimeCtx, rndFunc,pRndParam)) + return ippDLCompositeOrder; + + /* R|(P-1) */ + cpDec_BNU(pT, pP, lenP, 1); + cpMod_BNU(pT, lenP, pR, lenR); + if(!cpEqu_BNU_CHUNK(pT, lenP, 0)) + return ippDLInvalidCofactor; + + /* 1 < G < P */ + cpMontDec_BN(pTmp, DLP_GENC(pDL), DLP_MONTP0(pDL)); + if( 0>=cpBN_cmp(pTmp, cpBN_OneRef()) || cpCmp_BNU(pT, BN_SIZE(pTmp), DLP_P(pDL), lenP)>=0 ) + return ippDLInvalidGenerator; + + /* G^R = 1 (mod P) */ + cpMontExpBin_BNU(pT, BN_NUMBER(DLP_GENC(pDL)),lenP, pR, lenR, DLP_MONTP0(pDL) ); + if( cpCmp_BNU(pT,lenP, MOD_MNT_R(DLP_MONTP0(pDL)), lenP) ) + return ippDLInvalidGenerator; + + return ippDLValid; +} + +/*F* +// Name: ippsDLPValidateDH +// +// Purpose: Validate DL (DH) Domain Parameters. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pDL +// NULL == pResult +// NULL == rndFunc +// +// ippStsContextMatchErr illegal pDL->idCtx +// +// ippStsIncompleteContextErr +// incomplete context +// +// ippStsBadArgErr nTrials <=0 +// +// ippStsNoErr no errors +// +// Parameters: +// nTrials number of trials of primality test +// pResult pointer to the validation result +// pDL pointer to the DL context +// rndFunc external random generator +// pRndParam pointer to the external random generator params +*F*/ +IPPFUN(IppStatus, ippsDLPValidateDH,(int nTrials, IppDLResult* pResult, IppsDLPState* pDL, + IppBitSupplier rndFunc, void* pRndParam)) +{ + /* test DL context */ + IPP_BAD_PTR1_RET(pDL); + IPP_BADARG_RET(!DLP_VALID_ID(pDL), ippStsContextMatchErr); + + /* test operation flag */ + IPP_BADARG_RET(!DLP_COMPLETE(pDL), ippStsIncompleteContextErr); + + /* test number of trials for primality check */ + IPP_BADARG_RET(nTrials<=0, ippStsBadArgErr); + + /* test another pointers */ + IPP_BAD_PTR3_RET(pResult, rndFunc, pRndParam); + + /* execute genetal DL validation */ + *pResult = DLPValidate(nTrials, pDL, rndFunc, pRndParam); + + /* + // DH specific validation + */ + if(ippDLValid == *pResult) { + /* allocate BN resources */ + BigNumNode* pList = DLP_BNCTX(pDL); + IppsBigNumState* pT = cpBigNumListGet(&pList); + + BNU_CHUNK_T* pP = DLP_P(pDL); + BNU_CHUNK_T* pR = DLP_R(pDL); + cpSize feBitSize = DLP_BITSIZEP(pDL); + cpSize ordBitSize= DLP_BITSIZER(pDL); + cpSize lenP = BITS_BNU_CHUNK(feBitSize); + cpSize lenR = BITS_BNU_CHUNK(ordBitSize); + + /* DLP_BITSIZEP() >= 512, 256|DLP_BITSIZEP() */ + if( (MIN_DLPDH_BITSIZE > feBitSize) || + (feBitSize % 256) ) { + *pResult = ippDLInvalidBaseRange; + return ippStsNoErr; + } + + /* 2^(DLP_BITSIZEP()-1) < P < 2^DLP_BITSIZEP() */ + cpBN_power2(pT, feBitSize-1); + if( 0<=cpCmp_BNU(BN_NUMBER(pT),BN_SIZE(pT), pP,lenP) ) { + *pResult = ippDLInvalidBaseRange; + return ippStsNoErr; + } + cpBN_power2(pT, feBitSize); + if( 0>=cpCmp_BNU(BN_NUMBER(pT),BN_SIZE(pT), pP,lenP) ) { + *pResult = ippDLInvalidBaseRange; + return ippStsNoErr; + } + + /* DLP_BITSIZER() >= 160 */ + if( (MIN_DLPDH_BITSIZER > ordBitSize) ) { + *pResult = ippDLInvalidOrderRange; + return ippStsNoErr; + } + /* 2^(DLP_BITSIZER()-1) < R < 2^DLP_BITSIZER() */ + cpBN_power2(pT, ordBitSize-1); + if( 0<=cpCmp_BNU(BN_NUMBER(pT),BN_SIZE(pT), pR,lenR) ) { + *pResult = ippDLInvalidOrderRange; + return ippStsNoErr; + } + cpBN_power2(pT, ordBitSize); + if( 0>=cpCmp_BNU(BN_NUMBER(pT),BN_SIZE(pT), pR,lenR) ) { + *pResult = ippDLInvalidOrderRange; + return ippStsNoErr; + } + + /* 1 < G < (P-1) */ + cpMontDec_BN(pT, DLP_GENC(pDL), DLP_MONTP0(pDL)); + if( !(0 < cpBN_cmp(pT, cpBN_OneRef()) && + 0 > cpCmp_BNU(BN_NUMBER(pT),BN_SIZE(pT), pP,lenP)) ) { + *pResult = ippDLInvalidGenerator; + return ippStsNoErr; + } + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpvalidatedsa.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpvalidatedsa.c new file mode 100644 index 0000000..a36c366 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpvalidatedsa.c @@ -0,0 +1,208 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// DL over Prime Finite Field (validate domain parameters) +// +// Contents: +// ippsDLPValidateDSA() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpdlp.h" + + +/* +// Name: DLPValidate +// +// Purpose: Validate DL Domain Parameters. +// +// Parameters: +// nTrials number of trials of primality test +// pResult pointer to the validation result +// pDL pointer to the DL context +// rndFunc external random generator +// pRndParam pointer to the external random generator params +*/ +static +IppDLResult DLPValidate(int nTrials, IppsDLPState* pDL, + IppBitSupplier rndFunc, void* pRndParam) +{ + /* + // validate DL parameters: + // check that P is odd and P > 2 + // check that R is odd and R > 2 + // check that R | (P-1) + // check that 1 < G < P + // check that 1 == G^R (mod P) + */ + + BNU_CHUNK_T* pP = DLP_P(pDL); + BNU_CHUNK_T* pR = DLP_R(pDL); + cpSize lenP = BITS_BNU_CHUNK(DLP_BITSIZEP(pDL)); + cpSize lenR = BITS_BNU_CHUNK(DLP_BITSIZER(pDL)); + + IppsPrimeState* pPrimeCtx = DLP_PRIMEGEN(pDL); + + /* allocate BN resources */ + BigNumNode* pList = DLP_BNCTX(pDL); + IppsBigNumState* pTmp = cpBigNumListGet(&pList); + BNU_CHUNK_T* pT = BN_NUMBER(pTmp); + + /* P is odd and prime */ + if(0 == (pP[0] & 1)) + return ippDLBaseIsEven; + if(0==cpPrimeTest(pP, lenP, nTrials, pPrimeCtx, rndFunc,pRndParam)) + return ippDLCompositeBase; + + /* R is odd and prime */ + if(0 == (pR[0] & 1)) + return ippDLOrderIsEven; + if(0==cpPrimeTest(pR, lenR, nTrials, pPrimeCtx, rndFunc,pRndParam)) + return ippDLCompositeOrder; + + /* R|(P-1) */ + cpDec_BNU(pT, pP, lenP, 1); + cpMod_BNU(pT, lenP, pR, lenR); + if(!cpEqu_BNU_CHUNK(pT, lenP, 0)) + return ippDLInvalidCofactor; + + /* 1 < G < P */ + cpMontDec_BN(pTmp, DLP_GENC(pDL), DLP_MONTP0(pDL)); + if( 0>=cpBN_cmp(pTmp, cpBN_OneRef()) || cpCmp_BNU(pT, BN_SIZE(pTmp), DLP_P(pDL), lenP)>=0 ) + return ippDLInvalidGenerator; + + /* G^R = 1 (mod P) */ + cpMontExpBin_BNU(pT, BN_NUMBER(DLP_GENC(pDL)),lenP, pR, lenR, DLP_MONTP0(pDL) ); + if( cpCmp_BNU(pT,lenP, MOD_MNT_R(DLP_MONTP0(pDL)), lenP) ) + return ippDLInvalidGenerator; + + return ippDLValid; +} + +/*F* +// Name: ippsDLPValidateDSA +// +// Purpose: Validate DL (DSA) Domain Parameters. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pDL +// NULL == pResult +// NULL == rndFunc +// +// ippStsContextMatchErr illegal pDL->idCtx +// +// ippStsIncompleteContextErr +// incomplete context +// +// ippStsBadArgErr nTrials <=0 +// +// ippStsNoErr no errors +// +// Parameters: +// nTrials number of trials of primality test +// pResult pointer to the validation result +// pDL pointer to the DL context +// rndFunc external random generator +// pRndParam pointer to the external random generator params +*F*/ +IPPFUN(IppStatus, ippsDLPValidateDSA,(int nTrials, IppDLResult* pResult, IppsDLPState* pDL, + IppBitSupplier rndFunc, void* pRndParam)) +{ + /* test DL context */ + IPP_BAD_PTR1_RET(pDL); + IPP_BADARG_RET(!DLP_VALID_ID(pDL), ippStsContextMatchErr); + + /* test operation flag */ + IPP_BADARG_RET(!DLP_COMPLETE(pDL), ippStsIncompleteContextErr); + + /* test number of trials for primality check */ + IPP_BADARG_RET(nTrials<=0, ippStsBadArgErr); + + /* test another pointers */ + IPP_BAD_PTR2_RET(pResult, rndFunc); + + + /* execute genetal DL validation */ + *pResult = DLPValidate(nTrials, pDL, rndFunc, pRndParam); + + /* + // DSA specific validation + */ + if(ippDLValid == *pResult) { + /* allocate BN resources */ + BigNumNode* pList = DLP_BNCTX(pDL); + IppsBigNumState* pT = cpBigNumListGet(&pList); + + BNU_CHUNK_T* pP = DLP_P(pDL); + BNU_CHUNK_T* pR = DLP_R(pDL); + cpSize feBitSize = DLP_BITSIZEP(pDL); + cpSize ordBitSize= DLP_BITSIZER(pDL); + cpSize lenP = BITS_BNU_CHUNK(feBitSize); + cpSize lenR = BITS_BNU_CHUNK(ordBitSize); + + /* 512 <= DLP_BITSIZEP() <= 1024, 64|DLP_BITSIZEP() */ + if( (MIN_DLPDSA_BITSIZE > feBitSize) || (MAX_DLPDSA_BITSIZE < feBitSize) || + (feBitSize % 64) ) { + *pResult = ippDLInvalidBaseRange; + return ippStsNoErr; + } + /* 2^(DLP_BITSIZEP()-1) < P < 2^DLP_BITSIZEP() */ + cpBN_power2(pT, feBitSize-1); + if( 0<=cpCmp_BNU(BN_NUMBER(pT),BN_SIZE(pT), pP,lenP) ) { + *pResult = ippDLInvalidBaseRange; + return ippStsNoErr; + } + cpBN_power2(pT, feBitSize); + if( 0>=cpCmp_BNU(BN_NUMBER(pT),BN_SIZE(pT), pP,lenP) ) { + *pResult = ippDLInvalidBaseRange; + return ippStsNoErr; + } + + /* DLP_BITSIZER() == 160 */ + if( (DEF_DLPDSA_BITSIZER != ordBitSize) ) { + *pResult = ippDLInvalidOrderRange; + return ippStsNoErr; + } + /* 2^(DLP_BITSIZER()-1) < R < 2^DLP_BITSIZER() */ + cpBN_power2(pT, ordBitSize-1); + if( 0<=cpCmp_BNU(BN_NUMBER(pT),BN_SIZE(pT), pR,lenR) ) { + *pResult = ippDLInvalidOrderRange; + return ippStsNoErr; + } + cpBN_power2(pT, ordBitSize); + if( 0>=cpCmp_BNU(BN_NUMBER(pT),BN_SIZE(pT), pR,lenR) ) { + *pResult = ippDLInvalidOrderRange; + return ippStsNoErr; + } + + /* 1 < G < (P-1) */ + cpMontDec_BN(pT, DLP_GENC(pDL), DLP_MONTP0(pDL)); + if( !(0 < cpBN_cmp(pT, cpBN_OneRef()) && + 0 > cpCmp_BNU(BN_NUMBER(pT),BN_SIZE(pT), pP,lenP)) ) { + *pResult = ippDLInvalidGenerator; + return ippStsNoErr; + } + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpvalidatekeypair.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpvalidatekeypair.c new file mode 100644 index 0000000..eaeeea9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpvalidatekeypair.c @@ -0,0 +1,129 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// DL over Prime Finite Field (EC Key Generation, Validation and Set Up) +// +// Contents: +// ippsDLPValidateKeyPair() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpdlp.h" + +/*F* +// Name: ippsDLPValidateKeyPair +// +// Purpose: Validate DL Key Pair +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pDL +// NULL == pPrvKey +// NULL == pPubKey +// +// ippStsContextMatchErr invalid pDL->idCtx +// invalid pPrvKey->idCtx +// invalid pPubKey->idCtx +// +// ippStsIncompleteContextErr +// incomplete context +// +// ippStsNoErr no error +// +// Parameters: +// pPrvKey pointer to the private key +// pPubKey pointer to the public key +// pResult pointer to the result: ippDLValid/ +// ippDLInvalidPrivateKey/ippDLInvalidPublicKey/ +// ippDLInvalidKeyPair +// pDL pointer to the DL context +*F*/ +IPPFUN(IppStatus, ippsDLPValidateKeyPair,(const IppsBigNumState* pPrvKey, + const IppsBigNumState* pPubKey, + IppDLResult* pResult, + IppsDLPState* pDL)) +{ + /* test DL context */ + IPP_BAD_PTR2_RET(pResult, pDL); + IPP_BADARG_RET(!DLP_VALID_ID(pDL), ippStsContextMatchErr); + + /* test flag */ + IPP_BADARG_RET(!DLP_COMPLETE(pDL), ippStsIncompleteContextErr); + + { + /* allocate BN resources */ + BigNumNode* pList = DLP_BNCTX(pDL); + IppsBigNumState* pTmp = cpBigNumListGet(&pList); + BNU_CHUNK_T* pT = BN_NUMBER(pTmp); + + /* assume keys are OK */ + *pResult = ippDLValid; + + /* private key validation request */ + if(pPrvKey) { + cpSize lenR = BITS_BNU_CHUNK(DLP_BITSIZER(pDL)); + IPP_BADARG_RET(!BN_VALID_ID(pPrvKey), ippStsContextMatchErr); + + /* test private key: 1 < pPrvKey < (R-1) */ + cpDec_BNU(pT, DLP_R(pDL),lenR, 1); + if( 0>=cpBN_cmp(pPrvKey, cpBN_OneRef()) || + cpCmp_BNU(BN_NUMBER(pPrvKey),BN_SIZE(pPrvKey), pT,lenR)>=0 ) { + *pResult = ippDLInvalidPrivateKey; + return ippStsNoErr; + } + } + + /* public key validation request */ + if(pPubKey) { + cpSize lenP = BITS_BNU_CHUNK(DLP_BITSIZEP(pDL)); + IPP_BADARG_RET(!BN_VALID_ID(pPubKey), ippStsContextMatchErr); + + /* test public key: 1 < pPubKey < (P-1) */ + cpDec_BNU(pT, DLP_P(pDL),lenP, 1); + if( 0>=cpBN_cmp(pPubKey, cpBN_OneRef()) || + cpCmp_BNU(BN_NUMBER(pPubKey),BN_SIZE(pPubKey), pT,lenP)>=0 ) { + *pResult = ippDLInvalidPublicKey; + return ippStsNoErr; + } + + /* addition test: pPubKey = G^pPrvKey (mod P) */ + if(pPrvKey) { + int ordLen = MOD_LEN( DLP_MONTR(pDL) ); + IppsBigNumState* pTmpPrivate = cpBigNumListGet(&pList); + ZEXPAND_COPY_BNU(BN_NUMBER(pTmpPrivate), ordLen, BN_NUMBER(pPrvKey), BN_SIZE(pPrvKey)); + BN_SIZE(pTmpPrivate) = ordLen; + + /* recompute public key */ + cpMontExpBin_BN_sscm(pTmp, DLP_GENC(pDL), pTmpPrivate, DLP_MONTP0(pDL)); + cpMontDec_BN(pTmp, pTmp, DLP_MONTP0(pDL)); + + /* and compare */ + if( cpBN_cmp(pTmp, pPubKey) ) { + *pResult = ippDLInvalidKeyPair; + return ippStsNoErr; + } + } + } + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpverifydsaca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpverifydsaca.c new file mode 100644 index 0000000..6bfd759 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpdlpverifydsaca.c @@ -0,0 +1,185 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// DL over Prime Finite Field (Verify, DSA version) +// +// Contents: +// ippsDLPVerifyDSA() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpdlp.h" + + +/*F* +// Name: ippsDLPVerifyDSA +// +// Purpose: Verify Signature (DSA version) +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pDL +// NULL == pMsgDigest +// NULL == pSignR +// NULL == pSignS +// NULL == pResult +// +// ippStsContextMatchErr illegal pDL->idCtx +// illegal pMsgDigest->idCtx +// illegal pSignR->idCtx +// illegal pSignS->idCtx +// +// ippStsIncompleteContextErr +// incomplete context +// +// ippStsMessageErr MsgDigest >= R +// MsgDigest < 0 +// +// ippStsNoErr no errors +// +// Parameters: +// pMsgDigest pointer to the message representative to be signed +// pSignR,pSignS pointer to the signature +// pResult pointer to the result: IppSignIsValid/IppSignIsInvalid +// pDSA pointer to the DL context +// +// Primitive sequence call: +// 1) set up domain parameters +// 2) set up (signatory's) public key +*F*/ + +IPPFUN(IppStatus, ippsDLPVerifyDSA,(const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pSignR, const IppsBigNumState* pSignS, + IppDLResult* pResult, + IppsDLPState* pDL)) +{ + /* test context*/ + IPP_BAD_PTR2_RET(pDL,pResult); + IPP_BADARG_RET(!DLP_VALID_ID(pDL), ippStsContextMatchErr); + + /* test operation flag */ + IPP_BADARG_RET(!DLP_COMPLETE(pDL), ippStsIncompleteContextErr); + + /* test message representative */ + IPP_BAD_PTR1_RET(pMsgDigest); + IPP_BADARG_RET(!BN_VALID_ID(pMsgDigest), ippStsContextMatchErr); + IPP_BADARG_RET(BN_NEGATIVE(pMsgDigest), ippStsMessageErr); + + /* test signature */ + IPP_BAD_PTR2_RET(pSignR,pSignS); + IPP_BADARG_RET(!BN_VALID_ID(pSignR), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pSignS), ippStsContextMatchErr); + + /* test signature range */ + if(0idCtx +// illegal pA->idCtx +// illegal pB->idCtx +// illegal pGX->idCtx +// illegal pGY->idCtx +// illegal pOrder->idCtx +// illegal pECC->idCtx +// +// ippStsRangeErr not enough room for: +// pPrime +// pA, pB, +// pGX,pGY +// pOrder +// +// ippStsRangeErr 0>= cofactor +// +// ippStsNoErr no errors +// +// Parameters: +// pPrime pointer to the prime (specify FG(p)) +// pA pointer to the A coefficient of EC equation +// pB pointer to the B coefficient of EC equation +// pGX,pGY pointer to the Base Point (x and y coordinates) of EC +// pOrder pointer to the Base Point order +// cofactor cofactor value +// pECC pointer to the ECC context +// +*F*/ +IPP_OWN_DEFN (IppStatus, ECCPSetDP, (const IppsGFpMethod* method, int pLen, const BNU_CHUNK_T* pP, int aLen, const BNU_CHUNK_T* pA, int bLen, const BNU_CHUNK_T* pB, int xLen, const BNU_CHUNK_T* pX, int yLen, const BNU_CHUNK_T* pY, int rLen, const BNU_CHUNK_T* pR, BNU_CHUNK_T h, IppsGFpECState* pEC)) +{ + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + { + IppsGFpState * pGF = ECP_GFP(pEC); + + IppStatus sts = ippStsNoErr; + __ALIGN8 IppsBigNumState P; + __ALIGN8 IppsBigNumState H; + int primeBitSize = BITSIZE_BNU(pP, pLen); + //cpConstructBN(&P, pLen, (BNU_CHUNK_T*)pP, NULL); + //sts = cpGFpSetGFp(&P, primeBitSize, method, pGF); + cpGFpSetGFp(pP, primeBitSize, method, pGF); + + if(ippStsNoErr==sts) { + gsModEngine* pGFE = GFP_PMA(pGF); + + do { + int elemLen = GFP_FELEN(pGFE); + IppsGFpElement elmA, elmB; + + /* convert A ans B coeffs into GF elements */ + cpGFpElementConstruct(&elmA, cpGFpGetPool(1, pGFE), elemLen); + cpGFpElementConstruct(&elmB, cpGFpGetPool(1, pGFE), elemLen); + sts = ippsGFpSetElement((Ipp32u*)pA, BITS2WORD32_SIZE(BITSIZE_BNU(pA,aLen)), &elmA, pGF); + if(ippStsNoErr!=sts) break; + sts = ippsGFpSetElement((Ipp32u*)pB, BITS2WORD32_SIZE(BITSIZE_BNU(pB,bLen)), &elmB, pGF); + if(ippStsNoErr!=sts) break; + /* and set EC */ + sts = ippsGFpECSet(&elmA, &elmB, pEC); + if(ippStsNoErr!=sts) break; + + /* convert GX ans GY coeffs into GF elements */ + cpConstructBN(&P, rLen, (BNU_CHUNK_T*)pR, NULL); + cpConstructBN(&H, 1, &h, NULL); + sts = ippsGFpSetElement((Ipp32u*)pX, BITS2WORD32_SIZE(BITSIZE_BNU(pX,xLen)), &elmA, pGF); + if(ippStsNoErr!=sts) break; + sts = ippsGFpSetElement((Ipp32u*)pY, BITS2WORD32_SIZE(BITSIZE_BNU(pY,yLen)), &elmB, pGF); + if(ippStsNoErr!=sts) break; + /* and init EC subgroup */ + sts = ippsGFpECSetSubgroup(&elmA, &elmB, &P, &H, pEC); + } while(0); + + cpGFpReleasePool(2, pGFE); + } + + return sts; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccp.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccp.h new file mode 100644 index 0000000..26b9533 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccp.h @@ -0,0 +1,190 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Internal ECC (prime) basic Definitions & Function Prototypes +// +// +*/ + +#if !defined(_NEW_PCP_ECCP_H) +#define _NEW_PCP_ECCP_H + +#include "pcpgfpecstuff.h" + + +__INLINE IppsBigNumState* cpConstructBN(IppsBigNumState* pBN, cpSize len, BNU_CHUNK_T* pData, BNU_CHUNK_T* pBuffer) +{ + BN_SET_ID(pBN); + BN_SIGN(pBN) = ippBigNumPOS; + BN_SIZE(pBN) = len; + BN_ROOM(pBN) = len; + BN_NUMBER(pBN) = pData; + BN_BUFFER(pBN) = pBuffer; + return pBN; +} + +/* set EC parameters */ +#define ECCPSetDP OWNAPI(ECCPSetDP) + IPP_OWN_DECL (IppStatus, ECCPSetDP, (const IppsGFpMethod* method, int pLen, const BNU_CHUNK_T* pP, int aLen, const BNU_CHUNK_T* pA, int bLen, const BNU_CHUNK_T* pB, int xLen, const BNU_CHUNK_T* pX, int yLen, const BNU_CHUNK_T* pY, int rLen, const BNU_CHUNK_T* pR, BNU_CHUNK_T h, IppsGFpECState* pEC)) + +/* +// Recommended (Standard) Domain Parameters +*/ +extern const BNU_CHUNK_T secp112r1_p[]; // (2^128 -3)/76439 +extern const BNU_CHUNK_T secp112r1_a[]; +extern const BNU_CHUNK_T secp112r1_b[]; +extern const BNU_CHUNK_T secp112r1_gx[]; +extern const BNU_CHUNK_T secp112r1_gy[]; +extern const BNU_CHUNK_T secp112r1_r[]; +extern BNU_CHUNK_T secp112r1_h; + +extern const BNU_CHUNK_T secp112r2_p[]; // (2^128 -3)/76439 +extern const BNU_CHUNK_T secp112r2_a[]; +extern const BNU_CHUNK_T secp112r2_b[]; +extern const BNU_CHUNK_T secp112r2_gx[]; +extern const BNU_CHUNK_T secp112r2_gy[]; +extern const BNU_CHUNK_T secp112r2_r[]; +extern BNU_CHUNK_T secp112r2_h; + +extern const BNU_CHUNK_T secp128r1_p[]; // 2^128 -2^97 -1 +extern const BNU_CHUNK_T secp128r1_a[]; +extern const BNU_CHUNK_T secp128r1_b[]; +extern const BNU_CHUNK_T secp128r1_gx[]; +extern const BNU_CHUNK_T secp128r1_gy[]; +extern const BNU_CHUNK_T secp128r1_r[]; +extern BNU_CHUNK_T secp128r1_h; + +extern const BNU_CHUNK_T* secp128_mx[]; + +extern const BNU_CHUNK_T secp128r2_p[]; // 2^128 -2^97 -1 +extern const BNU_CHUNK_T secp128r2_a[]; +extern const BNU_CHUNK_T secp128r2_b[]; +extern const BNU_CHUNK_T secp128r2_gx[]; +extern const BNU_CHUNK_T secp128r2_gy[]; +extern const BNU_CHUNK_T secp128r2_r[]; +extern BNU_CHUNK_T secp128r2_h; + +extern const BNU_CHUNK_T secp160r1_p[]; // 2^160 -2^31 -1 +extern const BNU_CHUNK_T secp160r1_a[]; +extern const BNU_CHUNK_T secp160r1_b[]; +extern const BNU_CHUNK_T secp160r1_gx[]; +extern const BNU_CHUNK_T secp160r1_gy[]; +extern const BNU_CHUNK_T secp160r1_r[]; +extern BNU_CHUNK_T secp160r1_h; + +extern const BNU_CHUNK_T secp160r2_p[]; // 2^160 -2^32 -2^14 -2^12 -2^9 -2^8 -2^7 -2^2 -1 +extern const BNU_CHUNK_T secp160r2_a[]; +extern const BNU_CHUNK_T secp160r2_b[]; +extern const BNU_CHUNK_T secp160r2_gx[]; +extern const BNU_CHUNK_T secp160r2_gy[]; +extern const BNU_CHUNK_T secp160r2_r[]; +extern BNU_CHUNK_T secp160r2_h; + +extern const BNU_CHUNK_T secp192r1_p[]; // 2^192 -2^64 -1 +extern const BNU_CHUNK_T secp192r1_a[]; +extern const BNU_CHUNK_T secp192r1_b[]; +extern const BNU_CHUNK_T secp192r1_gx[]; +extern const BNU_CHUNK_T secp192r1_gy[]; +extern const BNU_CHUNK_T secp192r1_r[]; +extern BNU_CHUNK_T secp192r1_h; + +extern const BNU_CHUNK_T secp224r1_p[]; // 2^224 -2^96 +1 +extern const BNU_CHUNK_T secp224r1_a[]; +extern const BNU_CHUNK_T secp224r1_b[]; +extern const BNU_CHUNK_T secp224r1_gx[]; +extern const BNU_CHUNK_T secp224r1_gy[]; +extern const BNU_CHUNK_T secp224r1_r[]; +extern BNU_CHUNK_T secp224r1_h; + +extern const BNU_CHUNK_T secp256r1_p[]; // 2^256 -2^224 +2^192 +2^96 -1 +extern const BNU_CHUNK_T secp256r1_a[]; +extern const BNU_CHUNK_T secp256r1_b[]; +extern const BNU_CHUNK_T secp256r1_gx[]; +extern const BNU_CHUNK_T secp256r1_gy[]; +extern const BNU_CHUNK_T secp256r1_r[]; +extern BNU_CHUNK_T secp256r1_h; + +extern const BNU_CHUNK_T secp384r1_p[]; // 2^384 -2^128 -2^96 +2^32 -1 +extern const BNU_CHUNK_T secp384r1_a[]; +extern const BNU_CHUNK_T secp384r1_b[]; +extern const BNU_CHUNK_T secp384r1_gx[]; +extern const BNU_CHUNK_T secp384r1_gy[]; +extern const BNU_CHUNK_T secp384r1_r[]; +extern BNU_CHUNK_T secp384r1_h; + +extern const BNU_CHUNK_T secp521r1_p[]; // 2^521 -1 +extern const BNU_CHUNK_T secp521r1_a[]; +extern const BNU_CHUNK_T secp521r1_b[]; +extern const BNU_CHUNK_T secp521r1_gx[]; +extern const BNU_CHUNK_T secp521r1_gy[]; +extern const BNU_CHUNK_T secp521r1_r[]; +extern BNU_CHUNK_T secp521r1_h; + +extern const BNU_CHUNK_T tpmBN_p256p_p[]; // TPM BN_P256 +extern const BNU_CHUNK_T tpmBN_p256p_a[]; +extern const BNU_CHUNK_T tpmBN_p256p_b[]; +extern const BNU_CHUNK_T tpmBN_p256p_gx[]; +extern const BNU_CHUNK_T tpmBN_p256p_gy[]; +extern const BNU_CHUNK_T tpmBN_p256p_r[]; +extern BNU_CHUNK_T tpmBN_p256p_h; + +extern const BNU_CHUNK_T tpmSM2_p256_p[]; // TPM SM2_P256 +extern const BNU_CHUNK_T tpmSM2_p256_a[]; +extern const BNU_CHUNK_T tpmSM2_p256_b[]; +extern const BNU_CHUNK_T tpmSM2_p256_gx[]; +extern const BNU_CHUNK_T tpmSM2_p256_gy[]; +extern const BNU_CHUNK_T tpmSM2_p256_r[]; +extern BNU_CHUNK_T tpmSM2_p256_h; + +extern const BNU_CHUNK_T* tpmSM2_p256_p_mx[]; + +/* half of some std modulus */ +extern const BNU_CHUNK_T h_secp128r1_p[]; +extern const BNU_CHUNK_T h_secp192r1_p[]; +extern const BNU_CHUNK_T h_secp224r1_p[]; +extern const BNU_CHUNK_T h_secp256r1_p[]; +extern const BNU_CHUNK_T h_secp384r1_p[]; +extern const BNU_CHUNK_T h_secp521r1_p[]; +extern const BNU_CHUNK_T h_tpmSM2_p256_p[]; + +__INLINE BNU_CHUNK_T* cpModAdd_BNU(BNU_CHUNK_T* pR, + const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, + const BNU_CHUNK_T* pM, int ns, + BNU_CHUNK_T* pBuffer) +{ + BNU_CHUNK_T e = cpAdd_BNU(pR, pA, pB, ns); + e -= cpSub_BNU(pBuffer, pR, pM, ns); + MASKED_COPY_BNU(pR, e, pR, pBuffer, ns); + return pR; +} + +__INLINE BNU_CHUNK_T* cpModSub_BNU(BNU_CHUNK_T* pR, + const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, + const BNU_CHUNK_T* pM, int ns, + BNU_CHUNK_T* pBuffer) +{ + BNU_CHUNK_T e = cpSub_BNU(pR, pA, pB, ns); + cpAdd_BNU(pBuffer, pR, pM, ns); + MASKED_COPY_BNU(pR, (0-e), pBuffer, pR, ns); + return pR; +} + +#endif /* _NEW_PCP_ECCP_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpaddpoint.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpaddpoint.c new file mode 100644 index 0000000..37edbf9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpaddpoint.c @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (EC Point operations) +// +// Contents: +// ippsECCPAddPoint() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPAddPoint +// +// Purpose: Perforn EC point operation: R = P+Q +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pP +// NULL == pQ +// NULL == pR +// +// ippStsContextMatchErr illegal pEC->idCtx +// illegal pP->idCtx +// illegal pQ->idCtx +// illegal pR->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pP pointer to the source EC Point context +// pQ pointer to the source EC Point context +// pR pointer to the resultant EC Point context +// pEC pointer to the ECCP context +// +*F*/ +IPPFUN(IppStatus, ippsECCPAddPoint,(const IppsECCPPointState* pP, + const IppsECCPPointState* pQ, + IppsECCPPointState* pR, + IppsECCPState* pEC)) +{ + return ippsGFpECAddPoint(pP, pQ, pR, pEC); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpbindgxytblstd192r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpbindgxytblstd192r1.c new file mode 100644 index 0000000..53ce58d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpbindgxytblstd192r1.c @@ -0,0 +1,59 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (setup/retrieve domain parameters) +// +// Contents: +// ippsECCPBindGxyTblStd192r1() +// +*/ + +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPBindGxyTblStd192r1 +// +// Purpose: Enable the use of base point-based pre-computed +// tables of standard elliptic curves for EC192r1 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr illegal pEC->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pEC pointer to the ECC context +// +*F*/ + +IPPFUN(IppStatus, ippsECCPBindGxyTblStd192r1,(IppsECCPState* pEC)) +{ + /* test pEC */ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + ECP_PREMULBP(pEC) = gfpec_precom_nistP192r1_fun(); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpbindgxytblstd224r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpbindgxytblstd224r1.c new file mode 100644 index 0000000..2384ac7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpbindgxytblstd224r1.c @@ -0,0 +1,59 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (setup/retrieve domain parameters) +// +// Contents: +// ippsECCPBindGxyTblStd224r1() +// +*/ + +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPBindGxyTblStd224r1 +// +// Purpose: Enable the use of base point-based pre-computed +// tables of standard elliptic curves for EC224r1 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr illegal pEC->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pEC pointer to the ECC context +// +*F*/ + +IPPFUN(IppStatus, ippsECCPBindGxyTblStd224r1,(IppsECCPState* pEC)) +{ + /* test pEC */ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + ECP_PREMULBP(pEC) = gfpec_precom_nistP224r1_fun(); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpbindgxytblstd256r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpbindgxytblstd256r1.c new file mode 100644 index 0000000..a70d76f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpbindgxytblstd256r1.c @@ -0,0 +1,60 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (setup/retrieve domain parameters) +// +// Contents: +// ippsECCPBindGxyTblStd256r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPBindGxyTblStd256r1 +// +// Purpose: Enable the use of base point-based pre-computed +// tables of standard elliptic curves for EC256r1 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr illegal pEC->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pEC pointer to the ECC context +// +*F*/ + +IPPFUN(IppStatus, ippsECCPBindGxyTblStd256r1,(IppsECCPState* pEC)) +{ + /* test pEC */ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + ECP_PREMULBP(pEC) = gfpec_precom_nistP256r1_fun(); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpbindgxytblstd384r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpbindgxytblstd384r1.c new file mode 100644 index 0000000..97b59cb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpbindgxytblstd384r1.c @@ -0,0 +1,60 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (setup/retrieve domain parameters) +// +// Contents: +// ippsECCPBindGxyTblStd384r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPBindGxyTblStd384r1 +// +// Purpose: Enable the use of base point-based pre-computed +// tables of standard elliptic curves for EC384r1 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr illegal pEC->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pEC pointer to the ECC context +// +*F*/ + +IPPFUN(IppStatus, ippsECCPBindGxyTblStd384r1,(IppsECCPState* pEC)) +{ + /* test pEC */ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + ECP_PREMULBP(pEC) = gfpec_precom_nistP384r1_fun(); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpbindgxytblstd521r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpbindgxytblstd521r1.c new file mode 100644 index 0000000..04bd5a9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpbindgxytblstd521r1.c @@ -0,0 +1,60 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (setup/retrieve domain parameters) +// +// Contents: +// ippsECCPBindGxyTblStd521r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPBindGxyTblStd521r1 +// +// Purpose: Enable the use of base point-based pre-computed +// tables of standard elliptic curves for EC521r1 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr illegal pEC->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pEC pointer to the ECC context +// +*F*/ + +IPPFUN(IppStatus, ippsECCPBindGxyTblStd521r1,(IppsECCPState* pEC)) +{ + /* test pEC */ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + ECP_PREMULBP(pEC) = gfpec_precom_nistP521r1_fun(); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpbindgxytblstdsm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpbindgxytblstdsm2.c new file mode 100644 index 0000000..9097f63 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpbindgxytblstdsm2.c @@ -0,0 +1,60 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (setup/retrieve domain parameters) +// +// Contents: +// ippsECCPBindGxyTblStdSM2() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPBindGxyTblStdSM2 +// +// Purpose: Enable the use of base point-based pre-computed +// tables of standard elliptic curves for ECSM2 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr illegal pEC->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pEC pointer to the ECC context +// +*F*/ + +IPPFUN(IppStatus, ippsECCPBindGxyTblStdSM2,(IppsECCPState* pEC)) +{ + /* test pEC */ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + ECP_PREMULBP(pEC) = gfpec_precom_sm2_fun(); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpcheckpoint.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpcheckpoint.c new file mode 100644 index 0000000..7799a83 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpcheckpoint.c @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (EC Point operations) +// +// Contents: +// ippsECCPCheckPoint() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPCheckPoint +// +// Purpose: Check EC point: +// - is point lie on EC +// - is point at infinity +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pP +// NULL == pResult +// +// ippStsContextMatchErr illegal pEC->idCtx +// illegal pP->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pPoint pointer to the EC Point context +// pEC pointer to the ECCP context +// pResult pointer to the result: +// ippECValid +// ippECPointIsNotValid +// ippECPointIsAtInfinite +// +*F*/ +IPPFUN(IppStatus, ippsECCPCheckPoint,(const IppsECCPPointState* pP, + IppECResult* pResult, + IppsECCPState* pEC)) +{ + return ippsGFpECTstPoint(pP, pResult, pEC); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpcomparepoint.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpcomparepoint.c new file mode 100644 index 0000000..85499cc --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpcomparepoint.c @@ -0,0 +1,65 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (EC Point operations) +// +// Contents: +// ippsECCPComparePoint() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPComparePoint +// +// Purpose: Compare two EC points +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pP +// NULL == pQ +// NULL == pResult +// +// ippStsContextMatchErr illegal pEC->idCtx +// illegal pP->idCtx +// illegal pQ->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pP pointer to the EC Point context +// pQ pointer to the EC Point context +// pEC pointer to the EC context +// pResult pointer to the result: +// ippECPointIsEqual +// ippECPointIsNotEqual +// +*F*/ +IPPFUN(IppStatus, ippsECCPComparePoint,(const IppsECCPPointState* pP, + const IppsECCPPointState* pQ, + IppECResult* pResult, + IppsECCPState* pEC)) +{ + return ippsGFpECCmpPoint(pP, pQ, pResult, pEC); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgenkeyca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgenkeyca.c new file mode 100644 index 0000000..1058d64 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgenkeyca.c @@ -0,0 +1,108 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (EC Key Generation) +// +// Contents: +// ippsECCPGenKeyPair() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + + +/*F* +// Name: ippsECCPGenKeyPair +// +// Purpose: Generate (private,public) Key Pair +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pPrivate +// NULL == pPublic +// +// ippStsContextMatchErr illegal pEC->idCtx +// illegal pPrivate->idCtx +// illegal pPublic->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pPrivate pointer to the resultant private key +// pPublic pointer to the resultant public key +// pEC pointer to the ECCP context +// rndFunc Specified Random Generator. +// pRndParam Pointer to the Random Generator context. +// +*F*/ +IPPFUN(IppStatus, ippsECCPGenKeyPair, (IppsBigNumState* pPrivate, IppsECCPPointState* pPublic, + IppsECCPState* pEC, + IppBitSupplier rndFunc, void* pRndParam)) +{ + IPP_BAD_PTR2_RET(pEC, rndFunc); + + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + /* test private/public keys */ + IPP_BAD_PTR2_RET(pPrivate,pPublic); + IPP_BADARG_RET(!BN_VALID_ID(pPrivate), ippStsContextMatchErr); + IPP_BADARG_RET((BN_ROOM(pPrivate)*BITSIZE(BNU_CHUNK_T)idCtx +// illegal pA->idCtx +// illegal pB->idCtx +// illegal pGX->idCtx +// illegal pGY->idCtx +// illegal pOrder->idCtx +// illegal pEC->idCtx +// +// ippStsRangeErr not enough room for: +// pPrime +// pA, pB, +// pGX,pGY +// pOrder +// +// ippStsNoErr no errors +// +// Parameters: +// pPrime pointer to the retrieval prime (specify FG(p)) +// pA pointer to the retrieval A coefficient of EC equation +// pB pointer to the retrieval B coefficient of EC equation +// pGX,pGY pointer to the retrieval Base Point (x and y coordinates) of EC +// pOrder pointer to the retrieval Base Point order +// cofactor pointer to the retrieval cofactor value +// pEC pointer to the ECC context +// +*F*/ +IPPFUN(IppStatus, ippsECCPGet, (IppsBigNumState* pPrime, + IppsBigNumState* pA, IppsBigNumState* pB, + IppsBigNumState* pGX,IppsBigNumState* pGY, + IppsBigNumState* pOrder, int* cofactor, + IppsECCPState* pEC)) +{ + IppsGFpState* pGF; + gsModEngine* pGFE; + + /* test pEC */ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + pGF = ECP_GFP(pEC); + pGFE = GFP_PMA(pGF); + + /* test pPrime */ + IPP_BAD_PTR1_RET(pPrime); + IPP_BADARG_RET(!BN_VALID_ID(pPrime), ippStsContextMatchErr); + IPP_BADARG_RET(BN_ROOM(pPrime)decode; /* gf decode method */ + BNU_CHUNK_T* tmp = cpGFpGetPool(1, pGFE); + + /* retrieve EC parameter */ + ippsSet_BN(ippBigNumPOS, GFP_FELEN32(pGFE), (Ipp32u*)GFP_MODULUS(pGFE), pPrime); + + decode(tmp, ECP_A(pEC), pGFE); + ippsSet_BN(ippBigNumPOS, GFP_FELEN32(pGFE), (Ipp32u*)tmp, pA); + decode(tmp, ECP_B(pEC), pGFE); + ippsSet_BN(ippBigNumPOS, GFP_FELEN32(pGFE), (Ipp32u*)tmp, pB); + + decode(tmp, ECP_G(pEC), pGFE); + ippsSet_BN(ippBigNumPOS, GFP_FELEN32(pGFE), (Ipp32u*)tmp, pGX); + decode(tmp, ECP_G(pEC)+GFP_FELEN(pGFE), pGFE); + ippsSet_BN(ippBigNumPOS, GFP_FELEN32(pGFE), (Ipp32u*)tmp, pGY); + + { + gsModEngine* pR = ECP_MONT_R(pEC); + ippsSet_BN(ippBigNumPOS, MOD_LEN(pR)*(Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(Ipp32u)), (Ipp32u*)MOD_MODULUS(pR), pOrder); + } + + *cofactor = (int)ECP_COFACTOR(pEC)[0]; + + cpGFpReleasePool(1, pGFE); + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetorderbitsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetorderbitsize.c new file mode 100644 index 0000000..f3e34be --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetorderbitsize.c @@ -0,0 +1,62 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (setup/retrieve domain parameters) +// +// Contents: +// ippsECCPGetOrderBitSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPGetOrderBitSize +// +// Purpose: Retrieve size of Base Point Order (in bits). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pBitSize +// +// ippStsContextMatchErr illegal pEC->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pBitSize pointer to the size of base point order +// pEC pointer to the EC context +// +*F*/ +IPPFUN(IppStatus, ippsECCPGetOrderBitSize,(int* pBitSize, IppsECCPState* pEC)) +{ + /* test pEC */ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + /* test pBitSize*/ + IPP_BAD_PTR1_RET(pBitSize); + + *pBitSize = ECP_ORDBITSIZE(pEC); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetpoint.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetpoint.c new file mode 100644 index 0000000..84a08f9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetpoint.c @@ -0,0 +1,105 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (EC Point operations) +// +// Contents: +// ippsECCPGetPoint() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPGetPoint +// +// Purpose: Converts internal presentation EC point - montgomery projective +// into regular affine coordinates EC point (pX,pY) +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pPoint +// +// ippStsContextMatchErr illegal pEC->idCtx +// illegal pPoint->idCtx +// NULL != pX, illegal pX->idCtx +// NULL != pY, illegal pY->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pX pointer to the regular affine coordinate X +// pY pointer to the regular affine coordinate Y +// pPoint pointer to the EC Point context +// pEC pointer to the ECCP context +// +*F*/ +IPPFUN(IppStatus, ippsECCPGetPoint,(IppsBigNumState* pX, IppsBigNumState* pY, + const IppsECCPPointState* pPoint, + IppsECCPState* pEC)) +{ + /* test pEC */ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + /* test pX and pY */ + if(pX) { + IPP_BADARG_RET(!BN_VALID_ID(pX), ippStsContextMatchErr); + } + if(pY) { + IPP_BADARG_RET(!BN_VALID_ID(pY), ippStsContextMatchErr); + } + + { + IppStatus sts; + + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elemLen = GFP_FELEN(pGFE); + + mod_decode decode = GFP_METHOD(pGFE)->decode; /* gf decode method */ + + IppsGFpElement elmX, elmY; + + cpGFpElementConstruct(&elmX, cpGFpGetPool(1, pGFE), elemLen); + cpGFpElementConstruct(&elmY, cpGFpGetPool(1, pGFE), elemLen); + do { + sts = ippsGFpECGetPoint(pPoint, pX? &elmX:NULL, pY? &elmY:NULL, pEC); + if(ippStsNoErr!=sts) break; + + if(pX) { + decode(elmX.pData, elmX.pData, pGFE); + sts = ippsSet_BN(ippBigNumPOS, GFP_FELEN32(pGFE), (Ipp32u*)elmX.pData, pX); + if(ippStsNoErr!=sts) break; + } + if(pY) { + decode(elmY.pData, elmY.pData, pGFE); + sts = ippsSet_BN(ippBigNumPOS, GFP_FELEN32(pGFE), (Ipp32u*)elmY.pData, pY); + if(ippStsNoErr!=sts) break; + } + } while(0); + + cpGFpReleasePool(2, pGFE); + return sts; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsize.c new file mode 100644 index 0000000..8f8aa93 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsize.c @@ -0,0 +1,76 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (initialization) +// +// Contents: +// ippsECCPGetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPGetSize +// +// Purpose: Returns size of ECC context (bytes). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pSize +// +// ippStsSizeErr 2 > feBitSize +// feBitSize > EC_GFP_MAXBITSIZE +// ippStsNoErr no errors +// +// Parameters: +// feBitSize size of field element (bits) +// pSize pointer to the size of internal ECC context +// +*F*/ +IPPFUN(IppStatus, ippsECCPGetSize, (int feBitSize, int *pSize)) +{ + /* test size's pointer */ + IPP_BAD_PTR1_RET(pSize); + + /* test size of field element */ + IPP_BADARG_RET((2>feBitSize || feBitSize>EC_GFP_MAXBITSIZE), ippStsSizeErr); + + { + /* size of GF context */ + //int gfCtxSize = cpGFpGetSize(feBitSize); + int gfCtxSize = cpGFpGetSize(feBitSize, feBitSize+BITSIZE(BNU_CHUNK_T), GFP_POOL_SIZE); + /* size of EC context */ + int ecCtxSize = cpGFpECGetSize(1, feBitSize); + + /* size of EC scratch buffer: 16 points of BITS_BNU_CHUNK(feBitSize)*3 length each */ + int ecScratchBufferSize = 16*(BITS_BNU_CHUNK(feBitSize)*3)*(Ipp32s)sizeof(BNU_CHUNK_T); + + *pSize = ecCtxSize /* EC context */ + +gfCtxSize /* GF context */ + +ecScratchBufferSize /* *scratch buffer */ + +ecScratchBufferSize /* should be enough for 2 tables */ + +CACHE_LINE_SIZE; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestd128r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestd128r1.c new file mode 100644 index 0000000..73099d5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestd128r1.c @@ -0,0 +1,49 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (initialization) +// +// Contents: +// ippsECCPGetSizeStd128r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPGetSizeStd128r1 +// +// Purpose: Returns size of ECC context for secp128r1 [SEC2] (bytes). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pSize +// +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to the size of internal ECC context +*F*/ +IPPFUN(IppStatus, ippsECCPGetSizeStd128r1, (int *pSize)) +{ + return ippsECCPGetSize(128, pSize); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestd128r2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestd128r2.c new file mode 100644 index 0000000..f558941 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestd128r2.c @@ -0,0 +1,50 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (initialization) +// +// Contents: +// ippsECCPGetSizeStd128r2() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPGetSizeStd128r2 +// +// Purpose: Returns size of ECC context for secp128r2 [SEC2] (bytes). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pSize +// +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to the size of internal ECC context +*F*/ + +IPPFUN(IppStatus, ippsECCPGetSizeStd128r2, (int *pSize)) +{ + return ippsECCPGetSize(128, pSize); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestd192r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestd192r1.c new file mode 100644 index 0000000..505af2f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestd192r1.c @@ -0,0 +1,50 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (initialization) +// +// Contents: +// ippsECCPGetSizeStd192r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPGetSizeStd192r1 +// +// Purpose: Returns size of ECC context for secp192r1 [SEC2] (bytes). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pSize +// +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to the size of internal ECC context +*F*/ + +IPPFUN(IppStatus, ippsECCPGetSizeStd192r1, (int *pSize)) +{ + return ippsECCPGetSize(192, pSize); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestd224r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestd224r1.c new file mode 100644 index 0000000..c3b0ed6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestd224r1.c @@ -0,0 +1,50 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (initialization) +// +// Contents: +// ippsECCPGetSizeStd224r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPGetSizeStd224r1 +// +// Purpose: Returns size of ECC context for secp224r1 [SEC2] (bytes). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pSize +// +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to the size of internal ECC context +*F*/ + +IPPFUN(IppStatus, ippsECCPGetSizeStd224r1, (int *pSize)) +{ + return ippsECCPGetSize(224, pSize); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestd256r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestd256r1.c new file mode 100644 index 0000000..3ea269e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestd256r1.c @@ -0,0 +1,50 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (initialization) +// +// Contents: +// ippsECCPGetSizeStd256r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPGetSizeStd256r1 +// +// Purpose: Returns size of ECC context for secp256r1 [SEC2] (bytes). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pSize +// +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to the size of internal ECC context +*F*/ + +IPPFUN(IppStatus, ippsECCPGetSizeStd256r1, (int *pSize)) +{ + return ippsECCPGetSize(256, pSize); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestd384r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestd384r1.c new file mode 100644 index 0000000..b8cd7e3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestd384r1.c @@ -0,0 +1,50 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (initialization) +// +// Contents: +// ippsECCPGetSizeStd384r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPGetSizeStd384r1 +// +// Purpose: Returns size of ECC context for secp384r1 [SEC2] (bytes). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pSize +// +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to the size of internal ECC context +*F*/ + +IPPFUN(IppStatus, ippsECCPGetSizeStd384r1, (int *pSize)) +{ + return ippsECCPGetSize(384, pSize); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestd521r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestd521r1.c new file mode 100644 index 0000000..225aefd --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestd521r1.c @@ -0,0 +1,50 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (initialization) +// +// Contents: +// ippsECCPGetSizeStd521r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPGetSizeStd521r1 +// +// Purpose: Returns size of ECC context for secp521r1 [SEC2] (bytes). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pSize +// +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to the size of internal ECC context +*F*/ + +IPPFUN(IppStatus, ippsECCPGetSizeStd521r1, (int *pSize)) +{ + return ippsECCPGetSize(521, pSize); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestdsm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestdsm2.c new file mode 100644 index 0000000..a083c0a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpgetsizestdsm2.c @@ -0,0 +1,50 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (initialization) +// +// Contents: +// ippsECCPGetSizeStdSM2() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPGetSizeStdSM2 +// +// Purpose: Returns size of ECC context for SM2 (bytes). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pSize +// +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to the size of internal ECC context +*F*/ + +IPPFUN(IppStatus, ippsECCPGetSizeStdSM2, (int *pSize)) +{ + return ippsECCPGetSize(256, pSize); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinit.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinit.c new file mode 100644 index 0000000..65cd71d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinit.c @@ -0,0 +1,81 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (initialization) +// +// Contents: +// ippsECCPInit() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPInit +// +// Purpose: Init ECC context. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsSizeErr 2>feBitSize +// feBitSize>EC_GFP_MAXBITSIZE +// ippStsNoErr no errors +// +// Parameters: +// feBitSize size of field element (bits) +// pEC pointer to the ECC context +// +*F*/ +IPPFUN(IppStatus, ippsECCPInit, (int feBitSize, IppsECCPState* pEC)) +{ + /* test pEC pointer */ + IPP_BAD_PTR1_RET(pEC); + + /* test size of field element */ + IPP_BADARG_RET((2>feBitSize || feBitSize>EC_GFP_MAXBITSIZE), ippStsSizeErr); + + { + /* size of GF context */ + //int gfCtxSize = cpGFpGetSize(feBitSize); + int gfCtxSize = cpGFpGetSize(feBitSize, feBitSize+BITSIZE(BNU_CHUNK_T), GFP_POOL_SIZE); + /* size of EC context */ + int ecCtxSize = cpGFpECGetSize(1, feBitSize); + + IppsGFpState* pGF = (IppsGFpState*)((Ipp8u*)pEC+ecCtxSize); + BNU_CHUNK_T* pScratchBuffer = (BNU_CHUNK_T*)IPP_ALIGNED_PTR((Ipp8u*)pGF+gfCtxSize, CACHE_LINE_SIZE); + + /* set up contexts */ + IppStatus sts; + do { + sts = cpGFpInitGFp(feBitSize, pGF); + if(ippStsNoErr!=sts) break; + sts = ippsGFpECInit(pGF, NULL, NULL, pEC); + } while (0); + + /* save scratch buffer pointer */ + ECP_SBUFFER(pEC) = pScratchBuffer; + + return sts; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstd128r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstd128r1.c new file mode 100644 index 0000000..cd0dccb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstd128r1.c @@ -0,0 +1,50 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (initialization) +// +// Contents: +// ippsECCPInitStd128r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPInitStd128r1 +// +// Purpose: Init ECC context for secp128r1 [SEC2]. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pECC +// +// ippStsNoErr no errors +// +// Parameters: +// pECC pointer to the ECC context +// +*F*/ +IPPFUN(IppStatus, ippsECCPInitStd128r1, (IppsECCPState* pEC)) +{ + return ippsECCPInit(128, pEC); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstd128r2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstd128r2.c new file mode 100644 index 0000000..24bf7cc --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstd128r2.c @@ -0,0 +1,50 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (initialization) +// +// Contents: +// ippsECCPInitStd128r2() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPInitStd128r2 +// +// Purpose: Init ECC context for secp128r2 [SEC2]. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pECC +// +// ippStsNoErr no errors +// +// Parameters: +// pECC pointer to the ECC context +// +*F*/ +IPPFUN(IppStatus, ippsECCPInitStd128r2, (IppsECCPState* pEC)) +{ + return ippsECCPInit(128, pEC); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstd192r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstd192r1.c new file mode 100644 index 0000000..dc03e49 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstd192r1.c @@ -0,0 +1,50 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (initialization) +// +// Contents: +// ippsECCPInitStd192r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPInitStd192r1 +// +// Purpose: Init ECC context for secp192r1 [SEC2]. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pECC +// +// ippStsNoErr no errors +// +// Parameters: +// pECC pointer to the ECC context +// +*F*/ +IPPFUN(IppStatus, ippsECCPInitStd192r1, (IppsECCPState* pEC)) +{ + return ippsECCPInit(192, pEC); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstd224r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstd224r1.c new file mode 100644 index 0000000..817c35d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstd224r1.c @@ -0,0 +1,50 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (initialization) +// +// Contents: +// ippsECCPInitStd224r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPInitStd224r1 +// +// Purpose: Init ECC context for secp224r1 [SEC2]. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pECC +// +// ippStsNoErr no errors +// +// Parameters: +// pECC pointer to the ECC context +// +*F*/ +IPPFUN(IppStatus, ippsECCPInitStd224r1, (IppsECCPState* pEC)) +{ + return ippsECCPInit(224, pEC); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstd256r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstd256r1.c new file mode 100644 index 0000000..b97f90d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstd256r1.c @@ -0,0 +1,50 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (initialization) +// +// Contents: +// ippsECCPInitStd256r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPInitStd256r1 +// +// Purpose: Init ECC context for secp256r1 [SEC2]. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pECC +// +// ippStsNoErr no errors +// +// Parameters: +// pECC pointer to the ECC context +// +*F*/ +IPPFUN(IppStatus, ippsECCPInitStd256r1, (IppsECCPState* pEC)) +{ + return ippsECCPInit(256, pEC); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstd384r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstd384r1.c new file mode 100644 index 0000000..9f7dbbc --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstd384r1.c @@ -0,0 +1,50 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (initialization) +// +// Contents: +// ippsECCPInitStd384r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPInitStd384r1 +// +// Purpose: Init ECC context for secp384r1 [SEC2]. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pECC +// +// ippStsNoErr no errors +// +// Parameters: +// pECC pointer to the ECC context +// +*F*/ +IPPFUN(IppStatus, ippsECCPInitStd384r1, (IppsECCPState* pEC)) +{ + return ippsECCPInit(384, pEC); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstd521r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstd521r1.c new file mode 100644 index 0000000..c11c636 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstd521r1.c @@ -0,0 +1,50 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (initialization) +// +// Contents: +// ippsECCPInitStd521r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPInitStd521r1 +// +// Purpose: Init ECC context for secp521r1 [SEC2]. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pECC +// +// ippStsNoErr no errors +// +// Parameters: +// pECC pointer to the ECC context +// +*F*/ +IPPFUN(IppStatus, ippsECCPInitStd521r1, (IppsECCPState* pEC)) +{ + return ippsECCPInit(521, pEC); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstdsm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstdsm2.c new file mode 100644 index 0000000..9975638 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpinitstdsm2.c @@ -0,0 +1,50 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (initialization) +// +// Contents: +// ippsECCPInitStdSM2() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPInitStdSM2 +// +// Purpose: Init ECC context for SM2. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pECC +// +// ippStsNoErr no errors +// +// Parameters: +// pECC pointer to the ECC context +// +*F*/ +IPPFUN(IppStatus, ippsECCPInitStdSM2, (IppsECCPState* pEC)) +{ + return ippsECCPInit(256, pEC); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpmulpointscalar.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpmulpointscalar.c new file mode 100644 index 0000000..405f3c1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpmulpointscalar.c @@ -0,0 +1,68 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (EC Point operations) +// +// Contents: +// ippsECCPMulPointScalar() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPMulPointScalar +// +// Purpose: Perforn EC point operation: R = k*P +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pP +// NULL == pK +// NULL == pR +// +// ippStsContextMatchErr illegal pEC->idCtx +// illegal pP->idCtx +// illegal pK->idCtx +// illegal pR->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pP pointer to the source EC Point context +// pK pointer to the source BigNum multiplier context +// pR pointer to the resultant EC Point context +// pEC pointer to the ECCP context +// +*F*/ +IPPFUN(IppStatus, ippsECCPMulPointScalar,(const IppsECCPPointState* pP, + const IppsBigNumState* pK, + IppsECCPPointState* pR, + IppsECCPState* pEC)) +{ + /* use aligned EC context */ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + return ippsGFpECMulPoint(pP, pK, pR, pEC, (Ipp8u*)ECP_SBUFFER(pEC)); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpnegativepoint.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpnegativepoint.c new file mode 100644 index 0000000..53b413a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpnegativepoint.c @@ -0,0 +1,60 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (EC Point operations) +// +// Contents: +// ippsECCPNegativePoint() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPNegativePoint +// +// Purpose: Perforn EC point operation: R = -P +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pP +// NULL == pR +// +// ippStsContextMatchErr illegal pEC->idCtx +// illegal pP->idCtx +// illegal pR->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pP pointer to the source EC Point context +// pR pointer to the resultant EC Point context +// pEC pointer to the ECCP context +// +*F*/ +IPPFUN(IppStatus, ippsECCPNegativePoint, (const IppsECCPPointState* pP, + IppsECCPPointState* pR, + IppsECCPState* pEC)) +{ + return ippsGFpECNegPoint(pP, pR, pEC); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccppointgetsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccppointgetsize.c new file mode 100644 index 0000000..b440c98 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccppointgetsize.c @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC (prime) Point +// +// Contents: +// ippsECCPPointGetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPPointGetSize +// +// Purpose: Returns size of EC Point context (bytes). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pSzie +// ippStsSizeErr 2>feBitSize +// ippStsNoErr no errors +// +// Parameters: +// feBitSize size of field element (bits) +// pSize pointer to the size of EC Point context +// +*F*/ +IPPFUN(IppStatus, ippsECCPPointGetSize, (int feBitSize, int* pSize)) +{ + /* test size's pointer */ + IPP_BAD_PTR1_RET(pSize); + + /* test size of field element */ + IPP_BADARG_RET((2>feBitSize), ippStsSizeErr); + + { + int elemLen = BITS_BNU_CHUNK(feBitSize); + *pSize= (Ipp32s)sizeof(IppsGFpECPoint) + +elemLen*(Ipp32s)sizeof(BNU_CHUNK_T) /* X */ + +elemLen*(Ipp32s)sizeof(BNU_CHUNK_T) /* Y */ + +elemLen*(Ipp32s)sizeof(BNU_CHUNK_T);/* Z */ + } + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccppointinit.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccppointinit.c new file mode 100644 index 0000000..f13b894 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccppointinit.c @@ -0,0 +1,69 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC (prime) Point +// +// Contents: +// ippsECCPPointInit() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPPointInit +// +// Purpose: Init EC Point context. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pPoint +// ippStsSizeErr 2>feBitSize +// ippStsNoErr no errors +// +// Parameters: +// feBitSize size of field element (bits) +// pECC pointer to ECC context +// +*F*/ +IPPFUN(IppStatus, ippsECCPPointInit, (int feBitSize, IppsECCPPointState* pPoint)) +{ + /* test pEC pointer */ + IPP_BAD_PTR1_RET(pPoint); + + /* test size of field element */ + IPP_BADARG_RET((2>feBitSize), ippStsSizeErr); + + { + int elemLen = BITS_BNU_CHUNK(feBitSize); + Ipp8u* ptr = (Ipp8u*)pPoint; + + ECP_POINT_SET_ID(pPoint); + ECP_POINT_FLAGS(pPoint) = 0; + ECP_POINT_FELEN(pPoint) = elemLen; + ptr += sizeof(IppsGFpECPoint); + ECP_POINT_DATA(pPoint) = (BNU_CHUNK_T*)(ptr); + + gfec_SetPointAtInfinity(pPoint); + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccppublickeyca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccppublickeyca.c new file mode 100644 index 0000000..406e746 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccppublickeyca.c @@ -0,0 +1,67 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (EC Key Generation) +// +// Contents: +// ippsECCPPublicKey() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + + +/*F* +// Name: ippsECCPPublicKey +// +// Purpose: Calculate Public Key +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pPrivate +// NULL == pPublic +// +// ippStsContextMatchErr illegal pEC->idCtx +// illegal pPrivate->idCtx +// illegal pPublic->idCtx +// +// ippStsInvalidPrivateKey !(0 < pPrivate < order) +// +// ippStsNoErr no errors +// +// Parameters: +// pPrivate pointer to the private key +// pPublic pointer to the resultant public key +// pEC pointer to the ECCP context +// +*F*/ +IPPFUN(IppStatus, ippsECCPPublicKey, (const IppsBigNumState* pPrivate, + IppsECCPPointState* pPublic, + IppsECCPState* pEC)) +{ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + return ippsGFpECPublicKey(pPrivate, pPublic, pEC, (Ipp8u*)ECP_SBUFFER(pEC)); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsecretdhca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsecretdhca.c new file mode 100644 index 0000000..1f884a3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsecretdhca.c @@ -0,0 +1,74 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (EC Key Ecxhange, Diffie-Hellman version) +// +// Contents: +// ippsECCPSharedSecretDH() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + + +/*F* +// Name: ippsECCPSharedSecretDH +// +// Purpose: Shared Secret Value Derivation +// (Diffie-Hellman version). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pPrivateA +// NULL == pPublicB +// NULL == pShare +// +// ippStsContextMatchErr illegal pEC->idCtx +// illegal pPrivateA->idCtx +// illegal pPublicB->idCtx +// illegal pShare->idCtx +// +// ippStsRangeErr not enough room for share key +// +// ippStsShareKeyErr (infinity) => z +// +// ippStsNoErr no errors +// +// Parameters: +// pPrivateA pointer to own private key +// pPublicB pointer to alien public key +// pShare pointer to the shareds secret value +// pEC pointer to the ECCP context +// +*F*/ +IPPFUN(IppStatus, ippsECCPSharedSecretDH,(const IppsBigNumState* pPrivateA, + const IppsECCPPointState* pPublicB, + IppsBigNumState* pShare, + IppsECCPState* pEC)) +{ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + return ippsGFpECSharedSecretDH(pPrivateA, pPublicB, pShare, pEC, (Ipp8u*)ECP_SBUFFER(pEC)); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsecretdhcca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsecretdhcca.c new file mode 100644 index 0000000..d97cbbe --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsecretdhcca.c @@ -0,0 +1,74 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (EC Key Ecxhange, Diffie-Hellman version with cofactor) +// +// Contents: +// ippsECCPSharedSecretDHC() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + + +/*F* +// Name: ippsECCPSharedSecretDHC +// +// Purpose: Shared Secret Value Derivation +// (Diffie-Hellman version with cofactor). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pPrivateA +// NULL == pPublicB +// NULL == pShare +// +// ippStsContextMatchErr illegal pEC->idCtx +// illegal pPrivateA->idCtx +// illegal pPublicB->idCtx +// illegal pShare->idCtx +// +// ippStsRangeErr not enough room for share key +// +// ippStsShareKeyErr (infinity) => z +// +// ippStsNoErr no errors +// +// Parameters: +// pPrivateA pointer to own private key +// pPublicB pointer to alien public key +// pShare pointer to the shared secret value +// pEC pointer to the ECCP context +// +*F*/ +IPPFUN(IppStatus, ippsECCPSharedSecretDHC,(const IppsBigNumState* pPrivateA, + const IppsECCPPointState* pPublicB, + IppsBigNumState* pShare, + IppsECCPState* pEC)) +{ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + return ippsGFpECSharedSecretDHC(pPrivateA, pPublicB, pShare, pEC, (Ipp8u*)ECP_SBUFFER(pEC)); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpset.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpset.c new file mode 100644 index 0000000..82ad4a8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpset.c @@ -0,0 +1,119 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (setup/retrieve domain parameters) +// +// Contents: +// ippsECCPSet() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPInitStdSM2 +// +// Purpose: Sets up the elliptic curve domain parameters +// over a prime finite field GF(p) +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pPrime +// NULL == pA +// NULL == pB +// NULL == pGX +// NULL == pGY +// NULL == pOrder +// NULL == cofactor +// NULL == pEC +// +// ippStsContextMatchErr illegal pPrime->idCtx +// illegal pA->idCtx +// illegal pB->idCtx +// illegal pGX->idCtx +// illegal pGY->idCtx +// illegal pOrder->idCtx +// illegal pEC->idCtx +// +// ippStsRangeErr not enough room for: +// pPrime +// pA, pB, +// pGX, pGY +// pOrder +// +// ippStsNoErr no errors +// +// Parameters: +// pPrime pointer to the retrieval prime (specify FG(p)) +// pA pointer to the retrieval A coefficient of EC equation +// pB pointer to the retrieval B coefficient of EC equation +// pGX,pGY pointer to the retrieval Base Point (x and y coordinates) of EC +// pOrder pointer to the retrieval Base Point order +// cofactor pointer to the retrieval cofactor value +// pEC pointer to the ECC context +// +*F*/ + +IPPFUN(IppStatus, ippsECCPSet, (const IppsBigNumState* pPrime, + const IppsBigNumState* pA, const IppsBigNumState* pB, + const IppsBigNumState* pGX,const IppsBigNumState* pGY, + const IppsBigNumState* pOrder, int cofactor, + IppsECCPState* pEC)) +{ + /* test pEC */ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + /* test pPrime */ + IPP_BAD_PTR1_RET(pPrime); + IPP_BADARG_RET(!BN_VALID_ID(pPrime), ippStsContextMatchErr); + IPP_BADARG_RET((cpBN_bitsize(pPrime)>GFP_FEBITLEN(GFP_PMA(ECP_GFP(pEC)))), ippStsRangeErr); + + /* test pA and pB */ + IPP_BAD_PTR2_RET(pA,pB); + IPP_BADARG_RET(!BN_VALID_ID(pA), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pB), ippStsContextMatchErr); + IPP_BADARG_RET(BN_NEGATIVE(pA) || 0<=cpBN_cmp(pA,pPrime), ippStsRangeErr); + IPP_BADARG_RET(BN_NEGATIVE(pB) || 0<=cpBN_cmp(pB,pPrime), ippStsRangeErr); + + /* test pG and pGorder pointers */ + IPP_BAD_PTR3_RET(pGX,pGY, pOrder); + IPP_BADARG_RET(!BN_VALID_ID(pGX), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pGY), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pOrder), ippStsContextMatchErr); + IPP_BADARG_RET(BN_NEGATIVE(pGX) || 0<=cpBN_cmp(pGX,pPrime), ippStsRangeErr); + IPP_BADARG_RET(BN_NEGATIVE(pGY) || 0<=cpBN_cmp(pGY,pPrime), ippStsRangeErr); + IPP_BADARG_RET((cpBN_bitsize(pOrder)>ECP_ORDBITSIZE(pEC)), ippStsRangeErr); + + /* test cofactor */ + IPP_BADARG_RET(!(0idCtx +// illegal pPrivate->idCtx +// illegal pPublic->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pPrivate pointer to the private key +// pPublic pointer to the public key +// regular flag regular/ephemeral keys +// pEC pointer to the ECCP context +// +*F*/ +IPPFUN(IppStatus, ippsECCPSetKeyPair, (const IppsBigNumState* pPrivate, const IppsECCPPointState* pPublic, + IppBool regular, + IppsECCPState* pEC)) +{ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + { + BNU_CHUNK_T* targetPrivate; + BNU_CHUNK_T* targetPublic; + + if(regular) { + targetPrivate = ECP_PRIVAT(pEC); + targetPublic = ECP_PUBLIC(pEC); + } + else { + targetPrivate = ECP_PRIVAT_E(pEC); + targetPublic = ECP_PUBLIC_E(pEC); + } + + /* set up private key request */ + if( pPrivate ) { + IPP_BADARG_RET(!BN_VALID_ID(pPrivate), ippStsContextMatchErr); + { + int privateLen = BITS_BNU_CHUNK(ECP_ORDBITSIZE(pEC)); + cpGFpElementCopyPad(targetPrivate, privateLen, BN_NUMBER(pPrivate), BN_SIZE(pPrivate)); + } + } + + /* set up public key request */ + if( pPublic ) { + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pPublic), ippStsContextMatchErr ); + { + BNU_CHUNK_T* targetPublicX = targetPublic; + BNU_CHUNK_T* targetPublicY = targetPublic+ECP_POINT_FELEN(pPublic); + gfec_GetPoint(targetPublicX, targetPublicY, pPublic, pEC); + gfec_SetPoint(targetPublic, targetPublicX, targetPublicY, pEC); + + //cpGFpElementCopy(targetPublic, ECP_POINT_DATA(pPublic), publicLen); + } + } + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetpoint.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetpoint.c new file mode 100644 index 0000000..ea295c2 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetpoint.c @@ -0,0 +1,106 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (EC Point operations) +// +// Contents: +// ippsECCPSetPoint() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPSetPoint +// +// Purpose: Converts regular affine coordinates EC point (pX,pY) +// into internal presentation - montgomery projective. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pPoint +// NULL == pX +// NULL == pY +// +// ippStsContextMatchErr illegal pEC->idCtx +// illegal pX->idCtx +// illegal pY->idCtx +// illegal pPoint->idCtx +// +// ippStsOutOfECErr point out-of EC +// +// ippStsNoErr no errors +// +// Parameters: +// pX pointer to the regular affine coordinate X +// pY pointer to the regular affine coordinate Y +// pPoint pointer to the EC Point context +// pEC pointer to the ECCP context +// +// Note: +// if B==0 and (x,y)=(0,y) then point at Infinity will be set up +// if B!=0 and (x,y)=(0,0) then point at Infinity will be set up +// else point with requested coordinates (x,y) wil be set up +// There are no check validation inside! +// +*F*/ +IPPFUN(IppStatus, ippsECCPSetPoint,(const IppsBigNumState* pX, + const IppsBigNumState* pY, + IppsECCPPointState* pPoint, + IppsECCPState* pEC)) +{ + /* test pEC */ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + /* test pX and pY */ + IPP_BAD_PTR2_RET(pX,pY); + IPP_BADARG_RET(!BN_VALID_ID(pX), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pY), ippStsContextMatchErr); + + { + IppStatus sts; + + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elemLen = GFP_FELEN(pGFE); + IppsGFpElement elmX, elmY; + + cpGFpElementConstruct(&elmX, cpGFpGetPool(1, pGFE), elemLen); + cpGFpElementConstruct(&elmY, cpGFpGetPool(1, pGFE), elemLen); + do { + BNU_CHUNK_T* pData = BN_NUMBER(pX); + int ns = BN_SIZE(pX); + sts = ippsGFpSetElement((Ipp32u*)pData, BITS2WORD32_SIZE(BITSIZE_BNU(pData, ns)), &elmX, pGF); + if(ippStsNoErr!=sts) break; + pData = BN_NUMBER(pY); + ns = BN_SIZE(pY); + sts = ippsGFpSetElement((Ipp32u*)pData, BITS2WORD32_SIZE(BITSIZE_BNU(pData, ns)), &elmY, pGF); + if(ippStsNoErr!=sts) break; + sts = ippsGFpECSetPoint(&elmX, &elmY, pPoint, pEC); + } while(0); + + cpGFpReleasePool(2, pGFE); + return sts; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetpointatinfinity.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetpointatinfinity.c new file mode 100644 index 0000000..a6f5a2c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetpointatinfinity.c @@ -0,0 +1,55 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (EC Point operations) +// +// Contents: +// ippsECCPSetPointAtInfinity() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPSetPointAtInfinity +// +// Purpose: Set point at Infinity +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pPoint +// +// ippStsContextMatchErr illegal pEC->idCtx +// illegal pPoint->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pPoint pointer to the EC Point context +// pEC pointer to the ECCP context +// +*F*/ +IPPFUN(IppStatus, ippsECCPSetPointAtInfinity,(IppsECCPPointState* pPoint, IppsECCPState* pEC)) +{ + return ippsGFpECSetPointAtInfinity(pPoint, pEC); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd.c new file mode 100644 index 0000000..d51bdac --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd.c @@ -0,0 +1,127 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (setup/retrieve domain parameters) +// +// Contents: +// ippsECCPSetStd() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPSetStd +// +// Purpose: Set Standard ECC Domain Parameter. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr illegal pEC->idCtx +// +// ippStsECCInvalidFlagErr invalid flag +// +// ippStsNoErr no errors +// +// Parameters: +// flag specify standard ECC parameter(s) to be setup +// pEC pointer to the ECC context +// +*F*/ +IPPFUN(IppStatus, ippsECCPSetStd, (IppECCType flag, IppsECCPState* pEC)) +{ + /* test pEC */ + IPP_BAD_PTR1_RET(pEC); + + switch(flag) { + case IppECCPStd112r1: + return ECCPSetDP(ippsGFpMethod_pArb(), + BITS_BNU_CHUNK(112), secp112r1_p, + BITS_BNU_CHUNK(112), secp112r1_a, + BITS_BNU_CHUNK(112), secp112r1_b, + BITS_BNU_CHUNK(112), secp112r1_gx, + BITS_BNU_CHUNK(112), secp112r1_gy, + BITS_BNU_CHUNK(112), secp112r1_r, + secp112r1_h, pEC); + + case IppECCPStd112r2: + return ECCPSetDP(ippsGFpMethod_pArb(), + BITS_BNU_CHUNK(112), secp112r2_p, + BITS_BNU_CHUNK(112), secp112r2_a, + BITS_BNU_CHUNK(112), secp112r2_b, + BITS_BNU_CHUNK(112), secp112r2_gx, + BITS_BNU_CHUNK(112), secp112r2_gy, + BITS_BNU_CHUNK(112), secp112r2_r, + secp112r2_h, pEC); + + case IppECCPStd128r1: return ippsECCPSetStd128r1(pEC); + + case IppECCPStd128r2: return ippsECCPSetStd128r2(pEC); + + case IppECCPStd160r1: + return ECCPSetDP(ippsGFpMethod_pArb(), + BITS_BNU_CHUNK(160), secp160r1_p, + BITS_BNU_CHUNK(160), secp160r1_a, + BITS_BNU_CHUNK(160), secp160r1_b, + BITS_BNU_CHUNK(160), secp160r1_gx, + BITS_BNU_CHUNK(160), secp160r1_gy, + BITS_BNU_CHUNK(161), secp160r1_r, + secp160r1_h, pEC); + + case IppECCPStd160r2: + return ECCPSetDP(ippsGFpMethod_pArb(), + BITS_BNU_CHUNK(160), secp160r2_p, + BITS_BNU_CHUNK(160), secp160r2_a, + BITS_BNU_CHUNK(160), secp160r2_b, + BITS_BNU_CHUNK(160), secp160r2_gx, + BITS_BNU_CHUNK(160), secp160r2_gy, + BITS_BNU_CHUNK(161), secp160r2_r, + secp160r2_h, pEC); + + case IppECCPStd192r1: return ippsECCPSetStd192r1(pEC); + + case IppECCPStd224r1: return ippsECCPSetStd224r1(pEC); + + case IppECCPStd256r1: return ippsECCPSetStd256r1(pEC); + + case IppECCPStd384r1: return ippsECCPSetStd384r1(pEC); + + case IppECCPStd521r1: return ippsECCPSetStd521r1(pEC); + + case ippEC_TPM_BN_P256: + return ECCPSetDP(ippsGFpMethod_pArb(), + BITS_BNU_CHUNK(256), tpmBN_p256p_p, + BITS_BNU_CHUNK(32), tpmBN_p256p_a, + BITS_BNU_CHUNK(32), tpmBN_p256p_b, + BITS_BNU_CHUNK(32), tpmBN_p256p_gx, + BITS_BNU_CHUNK(32), tpmBN_p256p_gy, + BITS_BNU_CHUNK(256), tpmBN_p256p_r, + tpmBN_p256p_h, pEC); + + case ippECPstdSM2: return ippsECCPSetStdSM2(pEC); + + default: + return ippStsECCInvalidFlagErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd128r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd128r1.c new file mode 100644 index 0000000..09c8944 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd128r1.c @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (setup/retrieve domain parameters) +// +// Contents: +// ippsECCPSetStd128r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPSetStd128r1 +// +// Purpose: Set EC128r1 parameters +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr illegal pEC->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pEC pointer to the ECC context +// +*F*/ +IPPFUN(IppStatus, ippsECCPSetStd128r1, (IppsECCPState* pEC)) +{ + /* test pEC */ + IPP_BAD_PTR1_RET(pEC); + + /* set domain parameters */ + return ECCPSetDP(ippsGFpMethod_pArb(), + BITS_BNU_CHUNK(128), secp128r1_p, + BITS_BNU_CHUNK(128), secp128r1_a, + BITS_BNU_CHUNK(128), secp128r1_b, + BITS_BNU_CHUNK(128), secp128r1_gx, + BITS_BNU_CHUNK(128), secp128r1_gy, + BITS_BNU_CHUNK(128), secp128r1_r, + secp128r1_h, + pEC); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd128r2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd128r2.c new file mode 100644 index 0000000..2a015b7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd128r2.c @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (setup/retrieve domain parameters) +// +// Contents: +// ippsECCPSetStd128r2() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPSetStd128r2 +// +// Purpose: Set EC128r2 parameters +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr illegal pEC->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pEC pointer to the ECC context +// +*F*/ +IPPFUN(IppStatus, ippsECCPSetStd128r2, (IppsECCPState* pEC)) +{ + /* test pEC */ + IPP_BAD_PTR1_RET(pEC); + + /* set domain parameters */ + return ECCPSetDP(ippsGFpMethod_pArb(), + BITS_BNU_CHUNK(128), secp128r2_p, + BITS_BNU_CHUNK(128), secp128r2_a, + BITS_BNU_CHUNK(128), secp128r2_b, + BITS_BNU_CHUNK(128), secp128r2_gx, + BITS_BNU_CHUNK(128), secp128r2_gy, + BITS_BNU_CHUNK(128), secp128r2_r, + secp128r2_h, + pEC); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd192r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd192r1.c new file mode 100644 index 0000000..dc56b55 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd192r1.c @@ -0,0 +1,63 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (setup/retrieve domain parameters) +// +// Contents: +// ippsECCPSetStd192r1() +// +*/ + +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPSetStd192r1 +// +// Purpose: Set EC192r1 parameters +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr illegal pEC->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pEC pointer to the ECC context +// +*F*/ +IPPFUN(IppStatus, ippsECCPSetStd192r1, (IppsECCPState* pEC)) +{ + /* test pEC */ + IPP_BAD_PTR1_RET(pEC); + + /* set domain parameters */ + return ECCPSetDP(ippsGFpMethod_p192r1(), + BITS_BNU_CHUNK(192), secp192r1_p, + BITS_BNU_CHUNK(192), secp192r1_a, + BITS_BNU_CHUNK(192), secp192r1_b, + BITS_BNU_CHUNK(192), secp192r1_gx, + BITS_BNU_CHUNK(192), secp192r1_gy, + BITS_BNU_CHUNK(192), secp192r1_r, + secp192r1_h, + pEC); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd224r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd224r1.c new file mode 100644 index 0000000..977427d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd224r1.c @@ -0,0 +1,63 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (setup/retrieve domain parameters) +// +// Contents: +// ippsECCPSetStd224r1() +// +*/ + +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPSetStd224r1 +// +// Purpose: Set EC224r1 parameters +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr illegal pEC->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pEC pointer to the ECC context +// +*F*/ +IPPFUN(IppStatus, ippsECCPSetStd224r1, (IppsECCPState* pEC)) +{ + /* test pEC */ + IPP_BAD_PTR1_RET(pEC); + + /* set domain parameters */ + return ECCPSetDP(ippsGFpMethod_p224r1(), + BITS_BNU_CHUNK(224), secp224r1_p, + BITS_BNU_CHUNK(224), secp224r1_a, + BITS_BNU_CHUNK(224), secp224r1_b, + BITS_BNU_CHUNK(224), secp224r1_gx, + BITS_BNU_CHUNK(224), secp224r1_gy, + BITS_BNU_CHUNK(224), secp224r1_r, + secp224r1_h, + pEC); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd256r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd256r1.c new file mode 100644 index 0000000..7038cb9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd256r1.c @@ -0,0 +1,63 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (setup/retrieve domain parameters) +// +// Contents: +// ippsECCPSetStd256r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPSetStd256r1 +// +// Purpose: Set EC256r1 parameters +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr illegal pEC->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pEC pointer to the ECC context +// +*F*/ +IPPFUN(IppStatus, ippsECCPSetStd256r1, (IppsECCPState* pEC)) +{ + /* test pEC */ + IPP_BAD_PTR1_RET(pEC); + /* set domain parameters */ + return ECCPSetDP(ippsGFpMethod_p256r1(), + BITS_BNU_CHUNK(256), secp256r1_p, + BITS_BNU_CHUNK(256), secp256r1_a, + BITS_BNU_CHUNK(256), secp256r1_b, + BITS_BNU_CHUNK(256), secp256r1_gx, + BITS_BNU_CHUNK(256), secp256r1_gy, + BITS_BNU_CHUNK(256), secp256r1_r, + secp256r1_h, + pEC); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd384r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd384r1.c new file mode 100644 index 0000000..6e8ff25 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd384r1.c @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (setup/retrieve domain parameters) +// +// Contents: +// ippsECCPSetStd384r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPSetStd384r1 +// +// Purpose: Set EC384r1 parameters +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr illegal pEC->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pEC pointer to the ECC context +// +*F*/ +IPPFUN(IppStatus, ippsECCPSetStd384r1, (IppsECCPState* pEC)) +{ + /* test pEC */ + IPP_BAD_PTR1_RET(pEC); + + /* set domain parameters */ + return ECCPSetDP(ippsGFpMethod_p384r1(), + BITS_BNU_CHUNK(384), secp384r1_p, + BITS_BNU_CHUNK(384), secp384r1_a, + BITS_BNU_CHUNK(384), secp384r1_b, + BITS_BNU_CHUNK(384), secp384r1_gx, + BITS_BNU_CHUNK(384), secp384r1_gy, + BITS_BNU_CHUNK(384), secp384r1_r, + secp384r1_h, + pEC); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd521r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd521r1.c new file mode 100644 index 0000000..77c1d2f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstd521r1.c @@ -0,0 +1,63 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (setup/retrieve domain parameters) +// +// Contents: +// ippsECCPSetStd521r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPSetStd521r1 +// +// Purpose: Set EC521r1 parameters +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr illegal pEC->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pEC pointer to the ECC context +// +*F*/ +IPPFUN(IppStatus, ippsECCPSetStd521r1, (IppsECCPState* pEC)) +{ + /* test pEC */ + IPP_BAD_PTR1_RET(pEC); + /* set domain parameters */ + return ECCPSetDP(ippsGFpMethod_p521r1(), + BITS_BNU_CHUNK(521), secp521r1_p, + BITS_BNU_CHUNK(521), secp521r1_a, + BITS_BNU_CHUNK(521), secp521r1_b, + BITS_BNU_CHUNK(521), secp521r1_gx, + BITS_BNU_CHUNK(521), secp521r1_gy, + BITS_BNU_CHUNK(521), secp521r1_r, + secp521r1_h, + pEC); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstdsm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstdsm2.c new file mode 100644 index 0000000..b2cba14 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsetstdsm2.c @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (setup/retrieve domain parameters) +// +// Contents: +// ippsECCPSetStdSM2() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsECCPSetStdSM2 +// +// Purpose: Set ECSM2 parameters +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr illegal pEC->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pEC pointer to the ECC context +// +*F*/ +IPPFUN(IppStatus, ippsECCPSetStdSM2, (IppsECCPState* pEC)) +{ + /* test pEC */ + IPP_BAD_PTR1_RET(pEC); + + /* set domain parameters */ + return ECCPSetDP(ippsGFpMethod_p256sm2(), + BITS_BNU_CHUNK(256), tpmSM2_p256_p, + BITS_BNU_CHUNK(256), tpmSM2_p256_a, + BITS_BNU_CHUNK(256), tpmSM2_p256_b, + BITS_BNU_CHUNK(256), tpmSM2_p256_gx, + BITS_BNU_CHUNK(256), tpmSM2_p256_gy, + BITS_BNU_CHUNK(256), tpmSM2_p256_r, + tpmSM2_p256_h, + pEC); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsigndsaca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsigndsaca.c new file mode 100644 index 0000000..28542dd --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsigndsaca.c @@ -0,0 +1,205 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (Sign, DSA version) +// +// Contents: +// ippsECCPSignDSA() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + + +/*F* +// Name: ippsECCPSignDSA +// +// Purpose: Signing of message representative. +// (DSA version). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pMsgDigest +// NULL == pPrivate +// NULL == pSignX +// NULL == pSignY +// +// ippStsContextMatchErr illegal pEC->idCtx +// illegal pMsgDigest->idCtx +// illegal pPrivate->idCtx +// illegal pSignX->idCtx +// illegal pSignY->idCtx +// +// ippStsInvalidPrivateKey 0 >= Private +// Private >= order +// +// ippStsMessageErr MsgDigest >= order +// MsgDigest < 0 +// +// ippStsRangeErr not enough room for: +// signX +// signY +// +// ippStsEphemeralKeyErr (0==signX) || (0==signY) +// +// ippStsNoErr no errors +// +// Parameters: +// pMsgDigest pointer to the message representative to be signed +// pPrivate pointer to the regular private key +// pSignX,pSignY pointer to the signature +// pEC pointer to the ECCP context +// +// Note: +// - ephemeral key pair extracted from pEC and +// must be generated and before ippsECCPDSASign() usage +// - ephemeral key pair destroy before exit +// +*F*/ +IPPFUN(IppStatus, ippsECCPSignDSA,(const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pPrivate, + IppsBigNumState* pSignX, IppsBigNumState* pSignY, + IppsECCPState* pEC)) +{ + /* use aligned EC context */ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + /* test private key*/ + IPP_BAD_PTR1_RET(pPrivate); + IPP_BADARG_RET(!BN_VALID_ID(pPrivate), ippStsContextMatchErr); + IPP_BADARG_RET(BN_NEGATIVE(pPrivate), ippStsInvalidPrivateKey); + + /* test message representative: pMsgDigest>=0 */ + IPP_BAD_PTR1_RET(pMsgDigest); + IPP_BADARG_RET(!BN_VALID_ID(pMsgDigest), ippStsContextMatchErr); + IPP_BADARG_RET(BN_NEGATIVE(pMsgDigest), ippStsMessageErr); + /* make sure bisize(pMsgDigest) <= bitsize(order) */ + IPP_BADARG_RET(ECP_ORDBITSIZE(pEC) < cpBN_bitsize(pMsgDigest), ippStsMessageErr); + + /* test signature */ + IPP_BAD_PTR2_RET(pSignX,pSignY); + IPP_BADARG_RET(!BN_VALID_ID(pSignX), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pSignY), ippStsContextMatchErr); + IPP_BADARG_RET((BN_ROOM(pSignX)*BITSIZE(BNU_CHUNK_T)decode(buffer, buffer, pMontP); + ns = cpMod_BNU(buffer, elmLen, pOrder, ordLen); + cpGFpElementCopyPad(dataC, ordLen, buffer, ns); + gsModPoolFree(pMontP, 1); + } + + if(!GFP_IS_ZERO(dataC, ordLen)) { + /* + // signY = (1/ephPrivate)*(pMsgDigest + private*signX) (mod order) + */ + + /* copy and expand message is being signed and reduce just in case */ + ZEXPAND_COPY_BNU(buffF, ordLen, pMsgData, msgLen); + cpModSub_BNU(buffF, buffF, pOrder, pOrder, ordLen, buffT); + + /* private representation in Montgomery domain */ + ZEXPAND_COPY_BNU(dataD, ordLen, pPriData, priLen); + GFP_METHOD(pMontR)->encode(dataD, dataD, pMontR); + + /* (private*signX) in regular domain */ + GFP_METHOD(pMontR)->mul(dataD, dataD, dataC, pMontR); + + /* pMsgDigest + private*signX */ + cpModAdd_BNU(dataD, dataD, buffF, pOrder, ordLen, buffT); + + if(!GFP_IS_ZERO(dataD, ordLen)) { + /* (1/ephPrivate) in Montgomery domain */ + gs_mont_inv(buffT, ECP_PRIVAT_E(pEC), pMontR, alm_mont_inv_ct); + + /* (1/ephPrivate)*(pMsgDigest + private*signX) */ + GFP_METHOD(pMontR)->mul(dataD, dataD, buffT, pMontR); + + /* signX */ + ns = ordLen; + FIX_BNU(dataC, ns); + BN_SIGN(pSignX) = ippBigNumPOS; + BN_SIZE(pSignX) = ns; + /* signY */ + ns = ordLen; + FIX_BNU(dataD, ns); + BN_SIGN(pSignY) = ippBigNumPOS; + BN_SIZE(pSignY) = ns; + + sts = ippStsNoErr; + } + } + + /* clear ephemeral keys pair */ + { + BNU_CHUNK_T* pEphPrivate = ECP_PRIVAT_E(pEC); + BNU_CHUNK_T* pEphPublic = ECP_PUBLIC_E(pEC); + cpGFpElementSetChunk(pEphPrivate, BITS_BNU_CHUNK(ECP_ORDBITSIZE(pEC)), 0); + cpGFpElementSetChunk(pEphPublic, ECP_POINTLEN(pEC), 0); + } + + return sts; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsignnrca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsignnrca.c new file mode 100644 index 0000000..163d5bb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsignnrca.c @@ -0,0 +1,188 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (Sign, NR version) +// +// Contents: +// ippsECCPSignNR() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + + +/*F* +// Name: ippsECCPSignNR +// +// Purpose: Signing of message representative. +// (NR version). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pMsgDigest +// NULL == pPrivate +// NULL == pSignX +// NULL == pSignY +// +// ippStsContextMatchErr illegal pEC->idCtx +// illegal pMsgDigest->idCtx +// illegal pPrivate->idCtx +// illegal pSignX->idCtx +// illegal pSignY->idCtx +// +// ippStsInvalidPrivateKey 0 >= Private +// Private >= order +// +// ippStsMessageErr MsgDigest >= order +// MsgDigest < 0 +// +// ippStsRangeErr not enough room for: +// signX +// signY +// +// ippStsEphemeralKeyErr (0==signX) || (0==signY) +// +// ippStsNoErr no errors +// +// Parameters: +// pMsgDigest pointer to the message representative to be signed +// pPrivate pointer to the regular private key +// pSignX,pSignY pointer to the signature +// pEC pointer to the ECCP context +// +// Note: +// - ephemeral key pair extracted from pEC and +// must be generated and before ippsECCPNRSign() usage +// - ephemeral key pair destroy before exit +// +*F*/ +IPPFUN(IppStatus, ippsECCPSignNR,(const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pPrivate, + IppsBigNumState* pSignX, IppsBigNumState* pSignY, + IppsECCPState* pEC)) +{ + /* use aligned EC context */ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + /* test private key*/ + IPP_BAD_PTR1_RET(pPrivate); + IPP_BADARG_RET(!BN_VALID_ID(pPrivate), ippStsContextMatchErr); + IPP_BADARG_RET(BN_NEGATIVE(pPrivate), ippStsInvalidPrivateKey); + + /* test message representative: msg>=0 */ + IPP_BAD_PTR1_RET(pMsgDigest); + IPP_BADARG_RET(!BN_VALID_ID(pMsgDigest), ippStsContextMatchErr); + IPP_BADARG_RET(BN_NEGATIVE(pMsgDigest), ippStsMessageErr); + + /* test signature */ + IPP_BAD_PTR2_RET(pSignX,pSignY); + IPP_BADARG_RET(!BN_VALID_ID(pSignX), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pSignY), ippStsContextMatchErr); + IPP_BADARG_RET((BN_ROOM(pSignX)*BITSIZE(BNU_CHUNK_T)decode(buffer, buffer, pMontP); + ns = cpMod_BNU(buffer, elmLen, pOrder, ordLen); + cpGFpElementCopyPad(dataC, ordLen, buffer, ns); + gsModPoolFree(pMontP, 1); + } + + /* signX = (pMsgDigest + C) (mod order) */ + ZEXPAND_COPY_BNU(buffF, ordLen, pMsgData, msgLen); + cpModAdd_BNU(dataC, dataC, buffF, pOrder, ordLen, dataD); + + if(!GFP_IS_ZERO(dataC, ordLen)) { + + /* signY = (eph_private - private*signX) (mod order) */ + ZEXPAND_COPY_BNU(dataD, ordLen, pPriData, priLen); + GFP_METHOD(pMontR)->encode(dataD, dataD, pMontR); + GFP_METHOD(pMontR)->mul(dataD, dataD, dataC, pMontR); + cpModSub_BNU(dataD, ECP_PRIVAT_E(pEC), dataD, pOrder, ordLen, buffF); + + /* signX */ + ns = ordLen; + FIX_BNU(dataC, ns); + BN_SIGN(pSignX) = ippBigNumPOS; + BN_SIZE(pSignX) = ns; + + /* signY */ + ns = ordLen; + FIX_BNU(dataD, ns); + BN_SIGN(pSignY) = ippBigNumPOS; + BN_SIZE(pSignY) = ns; + + sts= ippStsNoErr; + } + + /* clear ephemeral keys pair */ + { + BNU_CHUNK_T* pEphPrivate = ECP_PRIVAT_E(pEC); + BNU_CHUNK_T* pEphPublic = ECP_PUBLIC_E(pEC); + cpGFpElementSetChunk(pEphPrivate, BITS_BNU_CHUNK(ECP_ORDBITSIZE(pEC)), 0); + cpGFpElementSetChunk(pEphPublic, ECP_POINTLEN(pEC), 0); + } + + return sts; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsignsm2ca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsignsm2ca.c new file mode 100644 index 0000000..499da15 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpsignsm2ca.c @@ -0,0 +1,95 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (Sign, SM2 version) +// +// Contents: +// ippsECCPSignSM2() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + + +/*F* +// Name: ippsECCPSignSM2 +// +// Purpose: Signing of message representative. +// (SM2 version). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pMsgDigest +// NULL == pRegPrivate +// NULL == pEphPrivate +// NULL == pSignR +// NULL == pSignS +// +// ippStsContextMatchErr illegal pEC->idCtx +// illegal pMsgDigest->idCtx +// illegal pRegPrivate->idCtx +// illegal pEphPrivate->idCtx +// illegal pSignR->idCtx +// illegal pSignS->idCtx +// +// ippStsInvalidPrivateKey (1 + regPrivate) >= order +// +// ippStsMessageErr MsgDigest >= order +// MsgDigest < 0 +// +// ippStsRangeErr not enough room for: +// signR +// signS +// +// ippStsEphemeralKeyErr (signR + ephPrivate) == order +// (0==signR) || (0==signS) +// +// ippStsNoErr no errors +// +// Parameters: +// pMsgDigest pointer to the message representative to be signed +// pRegPrivate pointer to the regular private key +// pEphPrivate pointer to the ephemeral private key +// pSignR,pSignS pointer to the signature +// pEC pointer to the ECCP context +// +// Note: +// ephemeral key computes inside ippsECCPSignSM2 in contrast with ippsECCPSignDSA, +// where ephemeral key pair computed before call and setup in context +// +*F*/ +IPPFUN(IppStatus, ippsECCPSignSM2,(const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pRegPrivate, + IppsBigNumState* pEphPrivate, + IppsBigNumState* pSignR, IppsBigNumState* pSignS, + IppsECCPState* pEC)) +{ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + return ippsGFpECSignSM2(pMsgDigest, + pRegPrivate, pEphPrivate, + pSignR, pSignS, + pEC, (Ipp8u*)ECP_SBUFFER(pEC)); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_112r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_112r1.c new file mode 100644 index 0000000..687bcae --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_112r1.c @@ -0,0 +1,53 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// ECC over Prime Finite Field (recommended ECC parameters) +// +// Contents: +// secp112r1 +// +*/ + +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" + +#if defined( _IPP_DATA ) + +/* +// Recommended Parameters secp112r1 +*/ +const BNU_CHUNK_T secp112r1_p[] = { // (2^128 -3)/76439 + LL(0xBEAD208B, 0x5E668076), LL(0x2ABF62E3, 0xDB7C)}; +const BNU_CHUNK_T secp112r1_a[] = { + LL(0xBEAD2088, 0x5E668076), LL(0x2ABF62E3, 0xDB7C)}; +const BNU_CHUNK_T secp112r1_b[] = { + LL(0x11702B22, 0x16EEDE89), LL(0xF8BA0439, 0x659E)}; +const BNU_CHUNK_T secp112r1_gx[] = { + LL(0xF9C2F098, 0x5EE76B55), LL(0x7239995A, 0x0948)}; +const BNU_CHUNK_T secp112r1_gy[] = { + LL(0x0FF77500, 0xC0A23E0E), LL(0xE5AF8724, 0xA89C)}; +const BNU_CHUNK_T secp112r1_r[] = { + LL(0xAC6561C5, 0x5E7628DF), LL(0x2ABF62E3, 0xDB7C)}; +BNU_CHUNK_T secp112r1_h = 1; + +#endif /* _IPP_DATA */ \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_112r2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_112r2.c new file mode 100644 index 0000000..488ba2f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_112r2.c @@ -0,0 +1,53 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// ECC over Prime Finite Field (recommended ECC parameters) +// +// Contents: +// secp112r2 +// +*/ + +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" + +#if defined( _IPP_DATA ) + +/* +// Recommended Parameters secp112r2 +*/ +const BNU_CHUNK_T secp112r2_p[] = { // (2^128 -3)/76439 + LL(0xBEAD208B, 0x5E668076), LL(0x2ABF62E3, 0xDB7C)}; +const BNU_CHUNK_T secp112r2_a[] = { + LL(0x5C0EF02C, 0x8A0AAAF6), LL(0xC24C05F3, 0x6127)}; +const BNU_CHUNK_T secp112r2_b[] = { + LL(0x4C85D709, 0xED74FCC3), LL(0xF1815DB5, 0x51DE)}; +const BNU_CHUNK_T secp112r2_gx[] = { + LL(0xD0928643, 0xB4E1649D), LL(0x0AB5E892, 0x4BA3)}; +const BNU_CHUNK_T secp112r2_gy[] = { + LL(0x6E956E97, 0x3747DEF3), LL(0x46F5882E, 0xADCD)}; +const BNU_CHUNK_T secp112r2_r[] = { + LL(0x0520D04B, 0xD7597CA1), LL(0x0AAFD8B8, 0x36DF)}; +BNU_CHUNK_T secp112r2_h = 4; + +#endif /* _IPP_DATA */ \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_128r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_128r1.c new file mode 100644 index 0000000..97e7e6f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_128r1.c @@ -0,0 +1,56 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// ECC over Prime Finite Field (recommended ECC parameters) +// +// Contents: +// secp128r1 (* Montgomery Friendly Modulus (+1) *) +// +*/ + +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" + +#if defined( _IPP_DATA ) + +/* +// Recommended Parameters secp128r1 +*/ +const BNU_CHUNK_T h_secp128r1_p[] = { // halpf of secp128r1_p + LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0x7FFFFFFE)}; + +const BNU_CHUNK_T secp128r1_p[] = { // 2^128 -2^97 -1 + LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFD), LL(0, 0)}; +const BNU_CHUNK_T secp128r1_a[] = { + LL(0xFFFFFFFC, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFD)}; +const BNU_CHUNK_T secp128r1_b[] = { + LL(0x2CEE5ED3, 0xD824993C), LL(0x1079F43D, 0xE87579C1)}; +const BNU_CHUNK_T secp128r1_gx[] = { + LL(0xA52C5B86, 0x0C28607C), LL(0x8B899B2D, 0x161FF752)}; +const BNU_CHUNK_T secp128r1_gy[] = { + LL(0xDDED7A83, 0xC02DA292), LL(0x5BAFEB13, 0xCF5AC839)}; +const BNU_CHUNK_T secp128r1_r[] = { + LL(0x9038A115, 0x75A30D1B), LL(0x00000000, 0xFFFFFFFE)}; +BNU_CHUNK_T secp128r1_h = 1; + +#endif /* _IPP_DATA */ \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_128r2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_128r2.c new file mode 100644 index 0000000..4f47f5c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_128r2.c @@ -0,0 +1,53 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// ECC over Prime Finite Field (recommended ECC parameters) +// +// Contents: +// secp128r2 (* Montgomery Friendly Modulus (+1) *) +// +*/ + +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" + +#if defined( _IPP_DATA ) + +/* +// Recommended Parameters secp128r2 +*/ +const BNU_CHUNK_T secp128r2_p[] = { // 2^128 -2^97 -1 + LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFD), LL(0, 0)}; +const BNU_CHUNK_T secp128r2_a[] = { + LL(0xBFF9AEE1, 0xBF59CC9B), LL(0xD1B3BBFE, 0xD6031998)}; +const BNU_CHUNK_T secp128r2_b[] = { + LL(0xBB6D8A5D, 0xDC2C6558), LL(0x80D02919, 0x5EEEFCA3)}; +const BNU_CHUNK_T secp128r2_gx[] = { + LL(0xCDEBC140, 0xE6FB32A7), LL(0x5E572983, 0x7B6AA5D8)}; +const BNU_CHUNK_T secp128r2_gy[] = { + LL(0x5FC34B44, 0x7106FE80), LL(0x894D3AEE, 0x27B6916A)}; +const BNU_CHUNK_T secp128r2_r[] = { + LL(0x0613B5A3, 0xBE002472), LL(0x7FFFFFFF, 0x3FFFFFFF)}; +BNU_CHUNK_T secp128r2_h = 4; + +#endif /* _IPP_DATA */ \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_160r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_160r1.c new file mode 100644 index 0000000..b72e5d3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_160r1.c @@ -0,0 +1,53 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// ECC over Prime Finite Field (recommended ECC parameters) +// +// Contents: +// secp160r1 +// +*/ + +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" + +#if defined( _IPP_DATA ) + +/* +// Recommended Parameters secp160r1 +*/ +const BNU_CHUNK_T secp160r1_p[] = { // 2^160 -2^31 -1 + LL(0x7FFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), L_(0xFFFFFFFF)}; +const BNU_CHUNK_T secp160r1_a[] = { + LL(0x7FFFFFFC, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), L_(0xFFFFFFFF)}; +const BNU_CHUNK_T secp160r1_b[] = { + LL(0xC565FA45, 0x81D4D4AD), LL(0x65ACF89F, 0x54BD7A8B), L_(0x1C97BEFC)}; +const BNU_CHUNK_T secp160r1_gx[] = { + LL(0x13CBFC82, 0x68C38BB9), LL(0x46646989, 0x8EF57328), L_(0x4A96B568)}; +const BNU_CHUNK_T secp160r1_gy[] = { + LL(0x7AC5FB32, 0x04235137), LL(0x59DCC912, 0x3168947D), L_(0x23A62855)}; +const BNU_CHUNK_T secp160r1_r[] = { + LL(0xCA752257, 0xF927AED3), LL(0x0001F4C8, 0x00000000), LL(0x00000000, 0x1)}; +BNU_CHUNK_T secp160r1_h = 1; + +#endif /* _IPP_DATA */ \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_160r2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_160r2.c new file mode 100644 index 0000000..164eca8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_160r2.c @@ -0,0 +1,53 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// ECC over Prime Finite Field (recommended ECC parameters) +// +// Contents: +// secp160r2 +// +*/ + +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" + +#if defined( _IPP_DATA ) + +/* +// Recommended Parameters secp160r2 +*/ +const BNU_CHUNK_T secp160r2_p[] = { // 2^160 -2^32 -2^14 -2^12 -2^9 -2^8 -2^7 -2^2 -1 + LL(0xFFFFAC73, 0xFFFFFFFE), LL(0xFFFFFFFF, 0xFFFFFFFF), L_(0xFFFFFFFF)}; +const BNU_CHUNK_T secp160r2_a[] = { + LL(0xFFFFAC70, 0xFFFFFFFE), LL(0xFFFFFFFF, 0xFFFFFFFF), L_(0xFFFFFFFF)}; +const BNU_CHUNK_T secp160r2_b[] = { + LL(0xF50388BA, 0x04664D5A), LL(0xAB572749, 0xFB59EB8B), L_(0xB4E134D3)}; +const BNU_CHUNK_T secp160r2_gx[] = { + LL(0x3144CE6D, 0x30F7199D), LL(0x1F4FF11B, 0x293A117E), L_(0x52DCB034)}; +const BNU_CHUNK_T secp160r2_gy[] = { + LL(0xA7D43F2E, 0xF9982CFE), LL(0xE071FA0D, 0xE331F296), L_(0xFEAFFEF2)}; +const BNU_CHUNK_T secp160r2_r[] = { + LL(0xF3A1A16B, 0xE786A818), LL(0x0000351E, 0x00000000), LL(0x00000000, 0x1)}; +BNU_CHUNK_T secp160r2_h = 1; + +#endif /* _IPP_DATA */ \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_192r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_192r1.c new file mode 100644 index 0000000..3a33d8f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_192r1.c @@ -0,0 +1,56 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// ECC over Prime Finite Field (recommended ECC parameters) +// +// Contents: +// secp192r1 (* Montgomery Friendly Modulus (+1) *) +// +*/ + +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" + +#if defined( _IPP_DATA ) + +/* +// Recommended Parameters secp192r1 +*/ +const BNU_CHUNK_T h_secp192r1_p[] = { // half of secp192r1_p + LL(0xFFFFFFFF, 0x7FFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0x7FFFFFFF)}; + +const BNU_CHUNK_T secp192r1_p[] = { // 2^192 -2^64 -1 + LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFE, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0x0, 0x0)}; +const BNU_CHUNK_T secp192r1_a[] = { + LL(0xFFFFFFFC, 0xFFFFFFFF), LL(0xFFFFFFFE, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF)}; +const BNU_CHUNK_T secp192r1_b[] = { + LL(0xC146B9B1, 0xFEB8DEEC), LL(0x72243049, 0x0FA7E9AB), LL(0xE59C80E7, 0x64210519)}; +const BNU_CHUNK_T secp192r1_gx[] = { + LL(0x82FF1012, 0xF4FF0AFD), LL(0x43A18800, 0x7CBF20EB), LL(0xB03090F6, 0x188DA80E)}; +const BNU_CHUNK_T secp192r1_gy[] = { + LL(0x1E794811, 0x73F977A1), LL(0x6B24CDD5, 0x631011ED), LL(0xFFC8DA78, 0x07192B95)}; +const BNU_CHUNK_T secp192r1_r[] = { + LL(0xB4D22831, 0x146BC9B1), LL(0x99DEF836, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF)}; +BNU_CHUNK_T secp192r1_h = 1; + +#endif /* _IPP_DATA */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_224r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_224r1.c new file mode 100644 index 0000000..492f907 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_224r1.c @@ -0,0 +1,63 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// ECC over Prime Finite Field (recommended ECC parameters) +// +// Contents: +// secp224r1 (* Montgomery Friendly Modulus (-1) *) +// +*/ + +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" + +#if defined( _IPP_DATA ) + +/* +// Recommended Parameters secp224r1 +*/ +const BNU_CHUNK_T h_secp224r1_p[] = { // half of secp224r1_p + LL(0x00000000, 0x00000000), LL(0x80000000, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), + LL(0x7FFFFFFF, 0x0)}; + +const BNU_CHUNK_T secp224r1_p[] = { // 2^224 -2^96 +1 + LL(0x00000001, 0x00000000), LL(0x00000000, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), + LL(0xFFFFFFFF, 0x0)}; +const BNU_CHUNK_T secp224r1_a[] = { + LL(0xFFFFFFFE, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFE), LL(0xFFFFFFFF, 0xFFFFFFFF), + L_(0xFFFFFFFF)}; +const BNU_CHUNK_T secp224r1_b[] = { + LL(0x2355FFB4, 0x270B3943), LL(0xD7BFD8BA, 0x5044B0B7), LL(0xF5413256, 0x0C04B3AB), + L_(0xB4050A85)}; +const BNU_CHUNK_T secp224r1_gx[] = { + LL(0x115C1D21, 0x343280D6), LL(0x56C21122, 0x4A03C1D3), LL(0x321390B9, 0x6BB4BF7F), + L_(0xB70E0CBD)}; +const BNU_CHUNK_T secp224r1_gy[] = { + LL(0x85007E34, 0x44D58199), LL(0x5A074764, 0xCD4375A0), LL(0x4C22DFE6, 0xB5F723FB), + L_(0xBD376388)}; +const BNU_CHUNK_T secp224r1_r[] = { + LL(0x5C5C2A3D, 0x13DD2945), LL(0xE0B8F03E, 0xFFFF16A2), LL(0xFFFFFFFF, 0xFFFFFFFF), + L_(0xFFFFFFFF)}; +BNU_CHUNK_T secp224r1_h = 1; + +#endif /* _IPP_DATA */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_256r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_256r1.c new file mode 100644 index 0000000..ddeb777 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_256r1.c @@ -0,0 +1,63 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// ECC over Prime Finite Field (recommended ECC parameters) +// +// Contents: +// secp256r1 (* Montgomery Friendly Modulus (+1) *) +// +*/ + +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" + +#if defined( _IPP_DATA ) + +/* +// Recommended Parameters secp256r1 +*/ +const BNU_CHUNK_T h_secp256r1_p[] = { // half of secp256r1_p + LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0x7FFFFFFF, 0x00000000), LL(0x00000000, 0x80000000), + LL(0x80000000, 0x7FFFFFFF)}; + +const BNU_CHUNK_T secp256r1_p[] = { // 2^256 -2^224 +2^192 +2^96 -1 + LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0x00000000), LL(0x00000000, 0x00000000), + LL(0x00000001, 0xFFFFFFFF), LL(0x0, 0x0)}; +const BNU_CHUNK_T secp256r1_a[] = { + LL(0xFFFFFFFC, 0xFFFFFFFF), LL(0xFFFFFFFF, 0x00000000), LL(0x00000000, 0x00000000), + LL(0x00000001, 0xFFFFFFFF)}; +const BNU_CHUNK_T secp256r1_b[] = { + LL(0x27D2604B, 0x3BCE3C3E), LL(0xCC53B0F6, 0x651D06B0), LL(0x769886BC, 0xB3EBBD55), + LL(0xAA3A93E7, 0x5AC635D8)}; +const BNU_CHUNK_T secp256r1_gx[] = { + LL(0xD898C296, 0xF4A13945), LL(0x2DEB33A0, 0x77037D81), LL(0x63A440F2, 0xF8BCE6E5), + LL(0xE12C4247, 0x6B17D1F2)}; +const BNU_CHUNK_T secp256r1_gy[] = { + LL(0x37BF51F5, 0xCBB64068), LL(0x6B315ECE, 0x2BCE3357), LL(0x7C0F9E16, 0x8EE7EB4A), + LL(0xFE1A7F9B, 0x4FE342E2)}; +const BNU_CHUNK_T secp256r1_r[] = { + LL(0xFC632551, 0xF3B9CAC2), LL(0xA7179E84, 0xBCE6FAAD), LL(0xFFFFFFFF, 0xFFFFFFFF), + LL(0x00000000, 0xFFFFFFFF)}; +BNU_CHUNK_T secp256r1_h = 1; + +#endif /* _IPP_DATA */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_384r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_384r1.c new file mode 100644 index 0000000..cce5e91 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_384r1.c @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// ECC over Prime Finite Field (recommended ECC parameters) +// +// Contents: +// secp384r1 (* Montgomery Friendly Modulus (0x0000000100000001) *) +// +*/ + +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" + +#if defined( _IPP_DATA ) + +/* +// Recommended Parameters secp384r1 +*/ +const BNU_CHUNK_T h_secp384r1_p[] = { // half of secp384r1_p + LL(0x7FFFFFFF, 0x00000000), LL(0x80000000, 0x7FFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), + LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0x7FFFFFFF)}; + +const BNU_CHUNK_T secp384r1_p[] = { // 2^384 -2^128 -2^96 +2^32 -1 + LL(0xFFFFFFFF, 0x00000000), LL(0x00000000, 0xFFFFFFFF), LL(0xFFFFFFFE, 0xFFFFFFFF), + LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), + LL(0x0, 0x0)}; +const BNU_CHUNK_T secp384r1_a[] = { + LL(0xFFFFFFFC, 0x00000000), LL(0x00000000, 0xFFFFFFFF), LL(0xFFFFFFFE, 0xFFFFFFFF), + LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF)}; +const BNU_CHUNK_T secp384r1_b[] = { + LL(0xD3EC2AEF, 0x2A85C8ED), LL(0x8A2ED19D, 0xC656398D), LL(0x5013875A, 0x0314088F), + LL(0xFE814112, 0x181D9C6E), LL(0xE3F82D19, 0x988E056B), LL(0xE23EE7E4, 0xB3312FA7)}; +const BNU_CHUNK_T secp384r1_gx[] = { + LL(0x72760AB7, 0x3A545E38), LL(0xBF55296C, 0x5502F25D), LL(0x82542A38, 0x59F741E0), + LL(0x8BA79B98, 0x6E1D3B62), LL(0xF320AD74, 0x8EB1C71E), LL(0xBE8B0537, 0xAA87CA22)}; +const BNU_CHUNK_T secp384r1_gy[] = { + LL(0x90EA0E5F, 0x7A431D7C), LL(0x1D7E819D, 0x0A60B1CE), LL(0xB5F0B8C0, 0xE9DA3113), + LL(0x289A147C, 0xF8F41DBD), LL(0x9292DC29, 0x5D9E98BF), LL(0x96262C6F, 0x3617DE4A)}; +const BNU_CHUNK_T secp384r1_r[] = { + LL(0xCCC52973, 0xECEC196A), LL(0x48B0A77A, 0x581A0DB2), LL(0xF4372DDF, 0xC7634D81), + LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF)}; +BNU_CHUNK_T secp384r1_h = 1; + +#endif /* _IPP_DATA */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_521r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_521r1.c new file mode 100644 index 0000000..dd5151c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_521r1.c @@ -0,0 +1,70 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// ECC over Prime Finite Field (recommended ECC parameters) +// +// Contents: +// secp521r1 (* Montgomery Friendly Modulus (+1) *) +// +*/ + +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" + +#if defined( _IPP_DATA ) + +/* +// Recommended Parameters secp521r1 +*/ +const BNU_CHUNK_T h_secp521r1_p[] = { // half of secp521r1_p + LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), + LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), + LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), L_(0x000000FF)}; + +const BNU_CHUNK_T secp521r1_p[] = { // 2^521 -1 + LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), + LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), + LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), L_(0x000001FF)}; +const BNU_CHUNK_T secp521r1_a[] = { + LL(0xFFFFFFFC, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), + LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), + LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), L_(0x000001FF)}; +const BNU_CHUNK_T secp521r1_b[] = { + LL(0x6B503F00, 0xEF451FD4), LL(0x3D2C34F1, 0x3573DF88), LL(0x3BB1BF07, 0x1652C0BD), + LL(0xEC7E937B, 0x56193951), LL(0x8EF109E1, 0xB8B48991), LL(0x99B315F3, 0xA2DA725B), + LL(0xB68540EE, 0x929A21A0), LL(0x8E1C9A1F, 0x953EB961), L_(0x00000051)}; +const BNU_CHUNK_T secp521r1_gx[] = { + LL(0xC2E5BD66, 0xF97E7E31), LL(0x856A429B, 0x3348B3C1), LL(0xA2FFA8DE, 0xFE1DC127), + LL(0xEFE75928, 0xA14B5E77), LL(0x6B4D3DBA, 0xF828AF60), LL(0x053FB521, 0x9C648139), + LL(0x2395B442, 0x9E3ECB66), LL(0x0404E9CD, 0x858E06B7), L_(0x000000C6)}; +const BNU_CHUNK_T secp521r1_gy[] = { + LL(0x9FD16650, 0x88BE9476), LL(0xA272C240, 0x353C7086), LL(0x3FAD0761, 0xC550B901), + LL(0x5EF42640, 0x97EE7299), LL(0x273E662C, 0x17AFBD17), LL(0x579B4468, 0x98F54449), + LL(0x2C7D1BD9, 0x5C8A5FB4), LL(0x9A3BC004, 0x39296A78), L_(0x00000118)}; +const BNU_CHUNK_T secp521r1_r[] = { + LL(0x91386409, 0xBB6FB71E), LL(0x899C47AE, 0x3BB5C9B8), LL(0xF709A5D0, 0x7FCC0148), + LL(0xBF2F966B, 0x51868783), LL(0xFFFFFFFA, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), + LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), L_(0x000001FF)}; +BNU_CHUNK_T secp521r1_h = 1; + +#endif /* _IPP_DATA */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_BN_256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_BN_256.c new file mode 100644 index 0000000..0d6fbf4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_BN_256.c @@ -0,0 +1,55 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// ECC over Prime Finite Field (recommended ECC parameters) +// +// Contents: +// tpm_BN_p256 (BN, TPM) +// +*/ + +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" + +#if defined( _IPP_DATA ) + +/* +// Recommended Parameters tpm_BN_p256 (Barreto-Naehrig) +*/ +const BNU_CHUNK_T tpmBN_p256p_p[] = { + LL(0xAED33013, 0xD3292DDB), LL(0x12980A82, 0x0CDC65FB), LL(0xEE71A49F, 0x46E5F25E), + LL(0xFFFCF0CD, 0xFFFFFFFF)}; +const BNU_CHUNK_T tpmBN_p256p_a[] = { + LL(0, 0)}; +const BNU_CHUNK_T tpmBN_p256p_b[] = { + LL(3, 0)}; +const BNU_CHUNK_T tpmBN_p256p_gx[] = { + LL(1, 0)}; +const BNU_CHUNK_T tpmBN_p256p_gy[] = { + LL(2, 0)}; +const BNU_CHUNK_T tpmBN_p256p_r[] = { + LL(0xD10B500D, 0xF62D536C), LL(0x1299921A, 0x0CDC65FB), LL(0xEE71A49E, 0x46E5F25E), + LL(0xFFFCF0CD, 0xFFFFFFFF)}; +BNU_CHUNK_T tpmBN_p256p_h = 1; + +#endif /* _IPP_DATA */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_SM2_256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_SM2_256.c new file mode 100644 index 0000000..a7f2c0c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpstdca_SM2_256.c @@ -0,0 +1,71 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// ECC over Prime Finite Field (recommended ECC parameters) +// +// Contents: +// tpmSM2_p256_p (SM2) (* Montgomery Friendly Modulus (+1) *) +// +*/ + +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" + +#if defined( _IPP_DATA ) + +/* +// Recommended Parameters tpm_SM2_p256 +*/ +#ifdef _SM2_SIGN_DEBUG_ +const BNU_CHUNK_T tpmSM2_p256_p[] = { + LL(0x08F1DFC3, 0x722EDB8B), LL(0x5C45517D, 0x45728391), LL(0xBF6FF7DE, 0xE8B92435), LL(0x4C044F18, 0x8542D69E), LL(0x0, 0x0)}; +const BNU_CHUNK_T tpmSM2_p256_a[] = { + LL(0x3937E498, 0xEC65228B), LL(0x6831D7E0, 0x2F3C848B), LL(0x73BBFEFF, 0x2417842E), LL(0xFA32C3FD, 0x787968B4)}; +const BNU_CHUNK_T tpmSM2_p256_b[] = { + LL(0x27C5249A, 0x6E12D1DA), LL(0xB16BA06E, 0xF61D59A5), LL(0x484BFE48, 0x9CF84241), LL(0xB23B0C84, 0x63E4C6D3)}; +const BNU_CHUNK_T tpmSM2_p256_gx[] = { + LL(0x7FEDD43D, 0x4C4E6C14), LL(0xADD50BDC, 0x32220B3B), LL(0xC3CC315E, 0x746434EB), LL(0x1B62EAB6, 0x421DEBD6)}; +const BNU_CHUNK_T tpmSM2_p256_gy[] = { + LL(0xE46E09A2, 0xA85841B9), LL(0xBFA36EA1, 0xE5D7FDFC), LL(0x153B70C4, 0xD47349D2), LL(0xCBB42C07, 0x0680512B)}; +const BNU_CHUNK_T tpmSM2_p256_r[] = { + LL(0xC32E79B7, 0x5AE74EE7), LL(0x0485628D, 0x29772063), LL(0xBF6FF7DD, 0xE8B92435), LL(0x4C044F18, 0x8542D69E)}; +#else +const BNU_CHUNK_T h_tpmSM2_p256_p[] = { // half of tpmSM2_p256_p + LL(0xFFFFFFFF, 0x7FFFFFFF), LL(0x80000000, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0x7FFFFFFF, 0x7FFFFFFF)}; + +const BNU_CHUNK_T tpmSM2_p256_p[] = { // 2^256 -2^224 -2^96 +2^64 -1 + LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0x00000000, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFE), LL(0x0, 0x0)}; +const BNU_CHUNK_T tpmSM2_p256_a[] = { + LL(0xFFFFFFFC, 0xFFFFFFFF), LL(0x00000000, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFE)}; +const BNU_CHUNK_T tpmSM2_p256_b[] = { + LL(0x4D940E93, 0xDDBCBD41), LL(0x15AB8F92, 0xF39789F5), LL(0xCF6509A7, 0x4D5A9E4B), LL(0x9D9F5E34, 0x28E9FA9E)}; +const BNU_CHUNK_T tpmSM2_p256_gx[] = { + LL(0x334C74C7, 0x715A4589), LL(0xF2660BE1, 0x8FE30BBF), LL(0x6A39C994, 0x5F990446), LL(0x1F198119, 0x32C4AE2C)}; +const BNU_CHUNK_T tpmSM2_p256_gy[] = { + LL(0x2139F0A0, 0x02DF32E5), LL(0xC62A4740, 0xD0A9877C), LL(0x6B692153, 0x59BDCEE3), LL(0xF4F6779C, 0xBC3736A2)}; +const BNU_CHUNK_T tpmSM2_p256_r[] = { + LL(0x39D54123, 0x53BBF409), LL(0x21C6052B, 0x7203DF6B), LL(0xFFFFFFFF, 0xFFFFFFFF), LL(0xFFFFFFFF, 0xFFFFFFFE)}; +#endif +BNU_CHUNK_T tpmSM2_p256_h = 1; + +#endif /* _IPP_DATA */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpvalidateca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpvalidateca.c new file mode 100644 index 0000000..8314d39 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpvalidateca.c @@ -0,0 +1,91 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (validate domain parameters) +// +// Contents: +// ippsECCPValidate() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + + +/*F* +// Name: ippsECCPValidate +// +// Purpose: Validate ECC Domain Parameters. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == c +// NULL == pSeed +// NULL == pResult +// +// ippStsContextMatchErr illegal pEC->idCtx +// illegal c->idCtx +// +// ippStsBadArgErr nTrials <=0 +// BN_SIZE(c)<5 +// +// ippStsNoErr no errors +// +// Parameters: +// nTrials number of trials of primality test +// pResult pointer to the validation result +// pEC pointer to the ECC context +// rndFunc Specified Random Generator. +// pRndParam Pointer to Random Generator context. +// +// Note +// Validation Domain Parameters folowing by P1363 recommendation +// (reference A.16.8) +// +*F*/ + +#define _DISABLE_TEST_PRIMALITY_ +// validation has been reduced in comparison with legacy version: +// - no primality test of underlying prime +// - no primality test of base point order +// - no MOV test + +IPPFUN(IppStatus, ippsECCPValidate, (int nTrials, IppECResult* pResult, IppsECCPState* pEC, + IppBitSupplier rndFunc, void* pRndParam)) +{ + #if defined(_DISABLE_TEST_PRIMALITY_) + IPP_UNREFERENCED_PARAMETER(nTrials); + IPP_UNREFERENCED_PARAMETER(rndFunc); + IPP_UNREFERENCED_PARAMETER(pRndParam); + #else + /* test number of trials for primality check */ + IPP_BADARG_RET(nTrials<=0, ippStsBadArgErr); + IPP_BAD_PTR2_RET(rndFunc, pRndParam); + #endif + + IPP_BAD_PTR2_RET(pResult, pEC); + + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + return ippsGFpECVerify(pResult, pEC, (Ipp8u*)ECP_SBUFFER(pEC)); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpvalidatekeyca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpvalidatekeyca.c new file mode 100644 index 0000000..e936640 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpvalidatekeyca.c @@ -0,0 +1,69 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (EC Key Generation) +// +// Contents: +// ippsECCPValidateKeyPair() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + + +/*F* +// Name: ippsECCPValidateKeyPair +// +// Purpose: Validate (private,public) Key Pair +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pPrivate +// NULL == pPublic +// NULL == pResult +// +// ippStsContextMatchErr illegal pEC->idCtx +// illegal pPrivate->idCtx +// illegal pPublic->idCtx +// +// ippStsNoErr no errors +// +// Parameters: +// pPrivate pointer to the private key +// pPublic pointer to the public key +// pResult pointer to the result: +// ippECValid/ippECInvalidPrivateKey/ippECPointIsAtInfinite/ippECInvalidPublicKey +// pEC pointer to the ECCP context +// +*F*/ +IPPFUN(IppStatus, ippsECCPValidateKeyPair, (const IppsBigNumState* pPrivate, + const IppsECCPPointState* pPublic, + IppECResult* pResult, + IppsECCPState* pEC)) +{ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + return ippsGFpECTstKeyPair(pPrivate, pPublic, pResult, pEC, (Ipp8u*)ECP_SBUFFER(pEC)); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpverifydsaca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpverifydsaca.c new file mode 100644 index 0000000..622954d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpverifydsaca.c @@ -0,0 +1,178 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (Verify Signature, DSA version) +// +// Contents: +// ippsECCPVerifyDSA() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + + +/*F* +// Name: ippsECCPVerifyDSA +// +// Purpose: Verify Signature (DSA version). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pMsgDigest +// NULL == pSignX +// NULL == pSignY +// NULL == pResult +// +// ippStsContextMatchErr illegal pEC->idCtx +// illegal pMsgDigest->idCtx +// illegal pSignX->idCtx +// illegal pSignY->idCtx +// +// ippStsMessageErr MsgDigest >= order +// MsgDigest < 0 +// +// ippStsRangeErr SignX < 0 or SignY < 0 +// +// ippStsNoErr no errors +// +// Parameters: +// pMsgDigest pointer to the message representative to be signed +// pSignX,pSignY pointer to the signature +// pResult pointer to the result: ippECValid/ippECInvalidSignature +// pEC pointer to the ECCP context +// +// Note: +// - signer's key must be set up in ECCP context +// before ippsECCPVerifyDSA() usage +// +*F*/ +IPPFUN(IppStatus, ippsECCPVerifyDSA,(const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pSignX, const IppsBigNumState* pSignY, + IppECResult* pResult, + IppsECCPState* pEC)) +{ + /* use aligned EC context */ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + /* test message representative: pMsgDigest>=0 */ + IPP_BAD_PTR1_RET(pMsgDigest); + IPP_BADARG_RET(!BN_VALID_ID(pMsgDigest), ippStsContextMatchErr); + IPP_BADARG_RET(BN_NEGATIVE(pMsgDigest), ippStsMessageErr); + /* make sure bisize(pMsgDigest) <= bitsize(order) */ + IPP_BADARG_RET(ECP_ORDBITSIZE(pEC) < cpBN_bitsize(pMsgDigest), ippStsMessageErr); + + /* test result */ + IPP_BAD_PTR1_RET(pResult); + + /* test signature */ + IPP_BAD_PTR2_RET(pSignX,pSignY); + IPP_BADARG_RET(!BN_VALID_ID(pSignX), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pSignY), ippStsContextMatchErr); + IPP_BADARG_RET(BN_NEGATIVE(pSignX), ippStsRangeErr); + IPP_BADARG_RET(BN_NEGATIVE(pSignY), ippStsRangeErr); + + { + IppECResult vResult = ippECInvalidSignature; + + gsModEngine* pModEngine = ECP_MONT_R(pEC); + BNU_CHUNK_T* pOrder = MOD_MODULUS(pModEngine); + int orderLen = MOD_LEN(pModEngine); + + /* test signature value */ + if(!cpEqu_BNU_CHUNK(BN_NUMBER(pSignX), BN_SIZE(pSignX), 0) && + !cpEqu_BNU_CHUNK(BN_NUMBER(pSignY), BN_SIZE(pSignY), 0) && + 0>cpCmp_BNU(BN_NUMBER(pSignX), BN_SIZE(pSignX), pOrder, orderLen) && + 0>cpCmp_BNU(BN_NUMBER(pSignY), BN_SIZE(pSignY), pOrder, orderLen)) { + + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + + int elmLen = GFP_FELEN(pGFE); + int pelmLen = GFP_PELEN(pGFE); + + BNU_CHUNK_T* h1 = cpGFpGetPool(3, pGFE); + BNU_CHUNK_T* h2 = h1+pelmLen; + BNU_CHUNK_T* redMsg = h2+pelmLen; + + IppsGFpECPoint P, G, Public; + + /* Y = 1/signY mod order */ + __ALIGN8 IppsBigNumState Y; + __ALIGN8 IppsBigNumState R; + BNU_CHUNK_T* buffer = ECP_SBUFFER(pEC); + BN_Make(buffer, buffer+orderLen+1, orderLen, &Y); + BN_Make(buffer+(orderLen+1)*2, buffer+(orderLen+1)*3, orderLen, &R); + /* BN(order) */ + BN_Set(pOrder, orderLen, &R); + ippsModInv_BN((IppsBigNumState*)pSignY, &R, &Y); + /* h1 = 1/signY mod order */ + cpGFpElementCopyPad(h1, orderLen, BN_NUMBER(&Y), BN_SIZE(&Y)); + cpMontEnc_BNU_EX(h1, h1, orderLen, pModEngine); + + /* validate signature */ + cpEcGFpInitPoint(&P, cpEcGFpGetPool(1, pEC),0, pEC); + cpEcGFpInitPoint(&G, ECP_G(pEC), ECP_AFFINE_POINT|ECP_FINITE_POINT, pEC); + cpEcGFpInitPoint(&Public, ECP_PUBLIC(pEC), ECP_FINITE_POINT, pEC); + + /* reduce message just in case */ + ZEXPAND_COPY_BNU(redMsg, orderLen, BN_NUMBER(pMsgDigest), BN_SIZE(pMsgDigest)); + cpModSub_BNU(redMsg, redMsg, pOrder, pOrder, orderLen, h2); + + /* h2 = pSignX * h1 (mod order) */ + cpMontMul_BNU_EX(h2, + h1,orderLen, BN_NUMBER(pSignX), BN_SIZE(pSignX), + pModEngine); + /* h1 = pMsgDigest * h1 (mod order) */ + cpMontMul_BNU_EX(h1, + h1,orderLen, BN_NUMBER(pMsgDigest), BN_SIZE(pMsgDigest), + //h1,orderLen, BN_NUMBER(pMsgDigest), BN_SIZE(pMsgDigest), + pModEngine); + + /* compute h1*BasePoint + h2*publicKey */ + gfec_BasePointProduct(&P, + h1, orderLen, &Public, h2, orderLen, + pEC, (Ipp8u*)ECP_SBUFFER(pEC)); + + /* check that P!=O */ + if( !gfec_IsPointAtInfinity(&P)) { + /* get P.X */ + gfec_GetPoint(h1, NULL, &P, pEC); + /* C' = int(P.x) mod order */ + GFP_METHOD(pGFE)->decode(h1, h1, pGFE); + elmLen = cpMod_BNU(h1, elmLen, pOrder, orderLen); + cpGFpElementPad(h1+elmLen, orderLen-elmLen, 0); + + /* and make sure signX==P.X */ + cpGFpElementCopyPad(h2, orderLen, BN_NUMBER(pSignX), BN_SIZE(pSignX)); + if(GFP_EQ(h1, h2, orderLen)) + vResult = ippECValid; + } + + cpEcGFpReleasePool(1, pEC); + cpGFpReleasePool(3, pGFE); + } + + *pResult = vResult; + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpverifynrca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpverifynrca.c new file mode 100644 index 0000000..92fa865 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpverifynrca.c @@ -0,0 +1,168 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (Verify Signature, NR version) +// +// Contents: +// ippsECCPVerifyNR() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + + +/*F* +// Name: ippsECCPVerifyNR +// +// Purpose: Verify Signature (NR version). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pMsgDigest +// NULL == pSignX +// NULL == pSignY +// NULL == pResult +// +// ippStsContextMatchErr illegal pEC->idCtx +// illegal pMsgDigest->idCtx +// illegal pSignX->idCtx +// illegal pSignY->idCtx +// +// ippStsMessageErr 0> MsgDigest +// order<= MsgDigest +// +// ippStsRangeErr SignX < 0 or SignY < 0 +// +// ippStsNoErr no errors +// +// Parameters: +// pMsgDigest pointer to the message representative to be signed +// pSignX,pSignY pointer to the signature +// pResult pointer to the result: ippECValid/ippECInvalidSignature +// pEC pointer to the ECCP context +// +// Note: +// - signer's key must be set up in ECCP context +// before ippsECCPVerifyNR() usage +// +*F*/ +IPPFUN(IppStatus, ippsECCPVerifyNR,(const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pSignX, const IppsBigNumState* pSignY, + IppECResult* pResult, + IppsECCPState* pEC)) +{ + gsModEngine* pModEngine; + BNU_CHUNK_T* pOrder; + int orderLen; + + BNU_CHUNK_T* pMsgData; + int msgLen; + + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + pModEngine = ECP_MONT_R(pEC); + pOrder = MOD_MODULUS(pModEngine); + orderLen = MOD_LEN(pModEngine); + + /* test message representative */ + IPP_BAD_PTR1_RET(pMsgDigest); + IPP_BADARG_RET(!BN_VALID_ID(pMsgDigest), ippStsContextMatchErr); + IPP_BADARG_RET(BN_NEGATIVE(pMsgDigest), ippStsMessageErr); + + pMsgData = BN_NUMBER(pMsgDigest); + msgLen = BN_SIZE(pMsgDigest); + IPP_BADARG_RET(0<=cpCmp_BNU(pMsgData, msgLen, pOrder, orderLen), ippStsMessageErr); + + /* test result */ + IPP_BAD_PTR1_RET(pResult); + + /* test signature */ + IPP_BAD_PTR2_RET(pSignX,pSignY); + IPP_BADARG_RET(!BN_VALID_ID(pSignX), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pSignY), ippStsContextMatchErr); + IPP_BADARG_RET(BN_NEGATIVE(pSignX), ippStsRangeErr); + IPP_BADARG_RET(BN_NEGATIVE(pSignY), ippStsRangeErr); + + { + IppECResult vResult = ippECInvalidSignature; + + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + + int elmLen = GFP_FELEN(pGFE); + int pelmLen = GFP_PELEN(pGFE); + + BNU_CHUNK_T* pH1 = cpGFpGetPool(3, pGFE); + BNU_CHUNK_T* pH2 = pH1 + pelmLen; + BNU_CHUNK_T* pR1 = pH2 + pelmLen; + BNU_CHUNK_T* pF = pR1 + pelmLen; + + /* test signature value */ + if(0cpCmp_BNU(BN_NUMBER(pSignX),BN_SIZE(pSignX), pOrder,orderLen) && + 0>cpCmp_BNU(BN_NUMBER(pSignY),BN_SIZE(pSignY), pOrder,orderLen)) { + + /* validate signature */ + IppsGFpECPoint P, G, Public; + cpEcGFpInitPoint(&P, cpEcGFpGetPool(1, pEC),0, pEC); + cpEcGFpInitPoint(&G, ECP_G(pEC), ECP_AFFINE_POINT|ECP_FINITE_POINT, pEC); + cpEcGFpInitPoint(&Public, ECP_PUBLIC(pEC), ECP_FINITE_POINT, pEC); + + /* expand signature: H1 = signY, H2 = signX */ + cpGFpElementCopyPad(pH1, orderLen, BN_NUMBER(pSignY), BN_SIZE(pSignY)); + cpGFpElementCopyPad(pH2, orderLen, BN_NUMBER(pSignX), BN_SIZE(pSignX)); + + /* compute H1*BasePoint + H2*publicKey */ + gfec_BasePointProduct(&P, + pH1, orderLen, &Public, pH2, orderLen, + pEC, (Ipp8u*)ECP_SBUFFER(pEC)); + + /* check that P!=O */ + if( !gfec_IsPointAtInfinity(&P)) { + /* P.X */ + gfec_GetPoint(pH1, NULL, &P, pEC); + /* H1 = int(P.X) mod order */ + GFP_METHOD(pGFE)->decode(pH1, pH1, pGFE); + elmLen = cpMod_BNU(pH1, elmLen, pOrder, orderLen); + cpGFpElementPad(pH1+elmLen, orderLen-elmLen, 0); + + /* recovered message: (SignX - H1) mod order */ + cpModSub_BNU(pH1, pH2, pH1, pOrder, orderLen, pF); + + /* and compare with input message*/ + cpGFpElementCopyPad(pH2, orderLen, pMsgData, msgLen); + if(GFP_EQ(pH1, pH2, orderLen)) + vResult = ippECValid; + } + + cpEcGFpReleasePool(1, pEC); + } + + cpGFpReleasePool(3, pGFE); + + *pResult = vResult; + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpverifysm2ca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpverifysm2ca.c new file mode 100644 index 0000000..a192bb3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccpverifysm2ca.c @@ -0,0 +1,173 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (Verify Signature, SM2 version) +// +// Contents: +// ippsECCPVerifySM2() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + + +/*F* +// Name: ippsECCPVerifySM2 +// +// Purpose: Verify Signature (SM2 version). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pMsgDigest +// NULL == pRegPublic +// NULL == pSignR +// NULL == pSignS +// NULL == pResult +// +// ippStsContextMatchErr illegal pEC->idCtx +// illegal pMsgDigest->idCtx +// illegal pRegPublic->idCtx +// illegal pSignR->idCtx +// illegal pSignS->idCtx +// +// ippStsMessageErr 0> MsgDigest +// order<= MsgDigest +// +// ippStsRangeErr SignR < 0 or SignS < 0 +// +// ippStsNoErr no errors +// +// Parameters: +// pMsgDigest pointer to the message representative to being signed +// pRegPublic pointer to the regular public key +// pSignR,pSignS pointer to the signature +// pResult pointer to the result: ippECValid/ippECInvalidSignature +// pEC pointer to the ECCP context +// +*F*/ +IPPFUN(IppStatus, ippsECCPVerifySM2,(const IppsBigNumState* pMsgDigest, + const IppsECCPPointState* pRegPublic, + const IppsBigNumState* pSignR, const IppsBigNumState* pSignS, + IppECResult* pResult, + IppsECCPState* pEC)) +{ + IppsGFpState* pGF; + gsModEngine* pGFE; + + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + pGF = ECP_GFP(pEC); + pGFE = GFP_PMA(pGF); + + /* test message representative: pMsgDigest>=0 */ + IPP_BAD_PTR1_RET(pMsgDigest); + IPP_BADARG_RET(!BN_VALID_ID(pMsgDigest), ippStsContextMatchErr); + IPP_BADARG_RET(BN_NEGATIVE(pMsgDigest), ippStsMessageErr); + /* make sure bisize(pMsgDigest) <= bitsize(order) */ + IPP_BADARG_RET(ECP_ORDBITSIZE(pEC) < cpBN_bitsize(pMsgDigest), ippStsMessageErr); + + /* test regular public key */ + IPP_BAD_PTR1_RET(pRegPublic); + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pRegPublic), ippStsContextMatchErr ); + IPP_BADARG_RET( ECP_POINT_FELEN(pRegPublic)!=GFP_FELEN(pGFE), ippStsOutOfRangeErr); + + /* test result */ + IPP_BAD_PTR1_RET(pResult); + + /* test signature */ + IPP_BAD_PTR2_RET(pSignR, pSignS); + IPP_BADARG_RET(!BN_VALID_ID(pSignR), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pSignS), ippStsContextMatchErr); + IPP_BADARG_RET(BN_NEGATIVE(pSignR), ippStsRangeErr); + IPP_BADARG_RET(BN_NEGATIVE(pSignS), ippStsRangeErr); + + { + IppECResult vResult = ippECInvalidSignature; + + gsModEngine* pMontR = ECP_MONT_R(pEC); + BNU_CHUNK_T* pOrder = MOD_MODULUS(pMontR); + int orderLen = MOD_LEN(pMontR); + + BNU_CHUNK_T* pMsgData = BN_NUMBER(pMsgDigest); + int msgLen = BN_SIZE(pMsgDigest); + + /* test signature value */ + if(!cpEqu_BNU_CHUNK(BN_NUMBER(pSignR), BN_SIZE(pSignR), 0) && + !cpEqu_BNU_CHUNK(BN_NUMBER(pSignS), BN_SIZE(pSignS), 0) && + 0>cpCmp_BNU(BN_NUMBER(pSignR), BN_SIZE(pSignR), pOrder, orderLen) && + 0>cpCmp_BNU(BN_NUMBER(pSignS), BN_SIZE(pSignS), pOrder, orderLen)) { + + int elmLen = GFP_FELEN(pGFE); + int ns; + + BNU_CHUNK_T* r = cpGFpGetPool(4, pGFE); + BNU_CHUNK_T* s = r+orderLen; + BNU_CHUNK_T* t = s+orderLen; + BNU_CHUNK_T* f = t+orderLen; + + /* expand signatire's components */ + cpGFpElementCopyPad(r, orderLen, BN_NUMBER(pSignR), BN_SIZE(pSignR)); + cpGFpElementCopyPad(s, orderLen, BN_NUMBER(pSignS), BN_SIZE(pSignS)); + + /* t = (r+s) mod order */ + cpModAdd_BNU(t, r, s, pOrder, orderLen, f); + + /* check if t!=0 */ + if( !cpIsGFpElemEquChunk_ct(t, orderLen, 0) ) { + + /* P = [s]G +[t]regPublic, t = P.x */ + IppsGFpECPoint P, G; + cpEcGFpInitPoint(&P, cpEcGFpGetPool(1, pEC),0, pEC); + cpEcGFpInitPoint(&G, ECP_G(pEC), ECP_AFFINE_POINT|ECP_FINITE_POINT, pEC); + + gfec_BasePointProduct(&P, + s, orderLen, pRegPublic, t, orderLen, + pEC, (Ipp8u*)ECP_SBUFFER(pEC)); + + gfec_GetPoint(t, NULL, &P, pEC); + GFP_METHOD(pGFE)->decode(t, t, pGFE); + ns = cpMod_BNU(t, elmLen, pOrder, orderLen); + + cpEcGFpReleasePool(1, pEC); + + /* t = (msg+t) mod order */ + cpGFpElementCopyPad(f, orderLen, pMsgData, msgLen); + cpModSub_BNU(f, f, pOrder, pOrder, orderLen, s); + cpModAdd_BNU(t, t, f, pOrder, orderLen, s); + + if(GFP_EQ(t, r, orderLen)) + vResult = ippECValid; + + IPP_UNREFERENCED_PARAMETER(ns); + } + + cpGFpReleasePool(4, pGFE); + + } + + *pResult = vResult; + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccresultca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccresultca.c new file mode 100644 index 0000000..69f8434 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpeccresultca.c @@ -0,0 +1,67 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// ECC operation results +// +// Contents: +// ippsECCGetResultString() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" + +/*F* +// Name: ippsECCGetResultString +// +// Purpose: Returns the string corresponding to code +// that represents the result of validation +// +// Parameters: +// code The code of the validation result +// +*F*/ + +IPPFUN( const char*, ippsECCGetResultString, (IppECResult code)) +{ + switch(code) { + case ippECValid: return "Validation pass successfully"; + case ippECCompositeBase: return "Finite Field produced by Composite"; + case ippECComplicatedBase: return "Too much non-zero terms in the polynomial"; + case ippECIsZeroDiscriminant: return "Zero discriminamt"; + case ippECCompositeOrder: return "Composite Base Point order"; + case ippECInvalidOrder: return "Invalid Base Point order"; + case ippECIsWeakMOV: return "EC cover by MOV Reduction Test"; + case ippECIsWeakSSSA: return "EC cover by SS-SA Reduction Test"; + case ippECIsSupersingular: return "EC is supersingular curve"; + case ippECInvalidPrivateKey: return "Invalid Private Key"; + case ippECInvalidPublicKey: return "Invalid Public Key"; + case ippECInvalidKeyPair: return "Invalid Key Pair"; + case ippECPointOutOfGroup: return "Point is out of group"; + case ippECPointIsAtInfinite: return "Point at Infinity"; + case ippECPointIsNotValid: return "Invalid EC Point"; + case ippECPointIsEqual: return "Points are equal"; + case ippECPointIsNotEqual: return "Points are different"; + case ippECInvalidSignature: return "Invalid Signature"; + default: return "Unknown ECC result"; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpecprime.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpecprime.h new file mode 100644 index 0000000..6bbd875 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpecprime.h @@ -0,0 +1,56 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Fixed EC primes +// +// +*/ + +#if !defined(_PCP_ECPRIME_H) +#define _PCP_ECPRIME_H + +#include "owndefs.h" +#include "pcpbnuimpl.h" + + +/* +// Recommended (NIST's) underlying EC Primes +*/ +extern const BNU_CHUNK_T secp112r1_p[]; // (2^128 -3)/76439 +extern const BNU_CHUNK_T secp112r2_p[]; // (2^128 -3)/76439 +extern const BNU_CHUNK_T secp128r1_p[]; // 2^128 -2^97 -1 +extern const BNU_CHUNK_T secp128r2_p[]; // 2^128 -2^97 -1 +extern const BNU_CHUNK_T secp160r1_p[]; // 2^160 -2^31 -1 +extern const BNU_CHUNK_T secp160r2_p[]; // 2^160 -2^32 -2^14 -2^12 -2^9 -2^8 -2^7 -2^2 -1 +extern const BNU_CHUNK_T secp192r1_p[]; // 2^192 -2^64 -1 +extern const BNU_CHUNK_T secp224r1_p[]; // 2^224 -2^96 +1 +extern const BNU_CHUNK_T secp256r1_p[]; // 2^256 -2^224 +2^192 +2^96 -1 +extern const BNU_CHUNK_T secp384r1_p[]; // 2^384 -2^128 -2^96 +2^32 -1 +extern const BNU_CHUNK_T secp521r1_p[]; // 2^521 -1 + +extern const BNU_CHUNK_T tpmBN_p256p_p[]; // TPM BN_P256 + +/* +// Recommended (SM2) underlying EC Prime +*/ +extern const BNU_CHUNK_T tpmSM2_p256_p[]; // TPM SM2_P256 + +#endif /* _PCP_ECPRIME_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp.c new file mode 100644 index 0000000..9856330 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp.c @@ -0,0 +1,109 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// cpGFpGetSize() +// cpGFpInitGFp() +// +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + + +/* +// size of GFp engine context (Montgomery) +*/ +IPP_OWN_DEFN (int, cpGFpGetSize, (int feBitSize, int peBitSize, int numpe)) +{ + int ctxSize = 0; + int elemLen = BITS_BNU_CHUNK(feBitSize); + int pelmLen = BITS_BNU_CHUNK(peBitSize); + + /* size of GFp engine */ + ctxSize = (Ipp32s)sizeof(gsModEngine) + + elemLen*(Ipp32s)sizeof(BNU_CHUNK_T) /* modulus */ + + elemLen*(Ipp32s)sizeof(BNU_CHUNK_T) /* mont_R */ + + elemLen*(Ipp32s)sizeof(BNU_CHUNK_T) /* mont_R^2 */ + + elemLen*(Ipp32s)sizeof(BNU_CHUNK_T) /* half of modulus */ + + elemLen*(Ipp32s)sizeof(BNU_CHUNK_T) /* quadratic non-residue */ + + pelmLen*(Ipp32s)sizeof(BNU_CHUNK_T)*numpe; /* pool */ + + ctxSize += sizeof(IppsGFpState); /* size of IppsGFPState */ + return ctxSize; +} + +/* +// init GFp engine context (Montgomery) +*/ +static void cpGFEInit(gsModEngine* pGFE, int modulusBitSize, int peBitSize, int numpe) +{ + int modLen = BITS_BNU_CHUNK(modulusBitSize); + int pelmLen = BITS_BNU_CHUNK(peBitSize); + + Ipp8u* ptr = (Ipp8u*)pGFE; + + /* clear whole context */ + PadBlock(0, ptr, sizeof(gsModEngine)); + ptr += sizeof(gsModEngine); + + GFP_PARENT(pGFE) = NULL; + GFP_EXTDEGREE(pGFE) = 1; + GFP_FEBITLEN(pGFE) = modulusBitSize; + GFP_FELEN(pGFE) = modLen; + GFP_FELEN32(pGFE) = BITS2WORD32_SIZE(modulusBitSize); + GFP_PELEN(pGFE) = pelmLen; + //GFP_METHOD(pGFE) = method; + GFP_MODULUS(pGFE) = (BNU_CHUNK_T*)(ptr); ptr += modLen*(Ipp32s)sizeof(BNU_CHUNK_T); + GFP_MNT_R(pGFE) = (BNU_CHUNK_T*)(ptr); ptr += modLen*(Ipp32s)sizeof(BNU_CHUNK_T); + GFP_MNT_RR(pGFE) = (BNU_CHUNK_T*)(ptr); ptr += modLen*(Ipp32s)sizeof(BNU_CHUNK_T); + GFP_HMODULUS(pGFE) = (BNU_CHUNK_T*)(ptr); ptr += modLen*(Ipp32s)sizeof(BNU_CHUNK_T); + GFP_QNR(pGFE) = (BNU_CHUNK_T*)(ptr); ptr += modLen*(Ipp32s)sizeof(BNU_CHUNK_T); + GFP_POOL(pGFE) = (BNU_CHUNK_T*)(ptr);/* ptr += modLen*(Ipp32s)sizeof(BNU_CHUNK_T);*/ + GFP_MAXPOOL(pGFE) = numpe; + GFP_USEDPOOL(pGFE) = 0; + + cpGFpElementPad(GFP_MODULUS(pGFE), modLen, 0); + cpGFpElementPad(GFP_MNT_R(pGFE), modLen, 0); + cpGFpElementPad(GFP_MNT_RR(pGFE), modLen, 0); + cpGFpElementPad(GFP_HMODULUS(pGFE), modLen, 0); + cpGFpElementPad(GFP_QNR(pGFE), modLen, 0); +} + +IPP_OWN_DEFN (IppStatus, cpGFpInitGFp, (int primeBitSize, IppsGFpState* pGF)) +{ + IPP_BADARG_RET((primeBitSize< IPP_MIN_GF_BITSIZE) || (primeBitSize> IPP_MAX_GF_BITSIZE), ippStsSizeErr); + IPP_BAD_PTR1_RET(pGF); + + { + Ipp8u* ptr = (Ipp8u*)pGF; + + GFP_SET_ID(pGF); + GFP_PMA(pGF) = (gsModEngine*)(ptr+sizeof(IppsGFpState)); + cpGFEInit(GFP_PMA(pGF), primeBitSize, primeBitSize+BITSIZE(BNU_CHUNK_T), GFP_POOL_SIZE); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_exp.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_exp.c new file mode 100644 index 0000000..a2c1cef --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_exp.c @@ -0,0 +1,37 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal operations over prime GF(p). +// +// Context: +// cpGFpExp +// +*/ +#include "owncp.h" + +#include "pcpbn.h" +#include "pcpgfpstuff.h" + + +IPP_OWN_DEFN (BNU_CHUNK_T*, cpGFpExp, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pE, int nsE, gsModEngine* pGFE)) +{ + int elemLen = GFP_FELEN(pGFE); + cpMontExpBin_BNU(pR, pA,cpFix_BNU(pA, elemLen), pE,cpFix_BNU(pE, nsE), pGFE); + return pR; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_get.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_get.c new file mode 100644 index 0000000..e180c86 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_get.c @@ -0,0 +1,45 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal operations over prime GF(p). +// +// Context: +// cpGFpGet +// +*/ +#include "owncp.h" + +#include "pcpbn.h" +#include "pcpgfpstuff.h" + +//tbcd: temporary excluded: #include + +IPP_OWN_DEFN (BNU_CHUNK_T*, cpGFpGet, (BNU_CHUNK_T* pDataA, int nsA, const BNU_CHUNK_T* pElm, gsModEngine* pGFE)) +{ + int elemLen = GFP_FELEN(pGFE); + + BNU_CHUNK_T* pTmp = cpGFpGetPool(1, pGFE); + //tbcd: temporary excluded: assert(pTmp !=NULL); + + GFP_METHOD(pGFE)->decode(pTmp, pElm, pGFE); + ZEXPAND_COPY_BNU(pDataA, nsA, pTmp, elemLen); + + cpGFpReleasePool(1, pGFE); + return pDataA; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_getoctstring.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_getoctstring.c new file mode 100644 index 0000000..921294b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_getoctstring.c @@ -0,0 +1,46 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal operations over prime GF(p). +// +// Context: +// cpGFpSetOctString +// +*/ +#include "owncp.h" + +#include "pcpbn.h" +#include "pcpgfpstuff.h" + +//tbcd: temporary excluded: #include + +IPP_OWN_DEFN (Ipp8u*, cpGFpGetOctString, (Ipp8u* pStr, int strSize, const BNU_CHUNK_T* pElm, gsModEngine* pGFE)) +{ + int elemLen = GFP_FELEN(pGFE); + int error; + + BNU_CHUNK_T* pTmp = cpGFpGetPool(1, pGFE); + //tbcd: temporary excluded: assert(pTmp !=NULL); + + GFP_METHOD(pGFE)->decode(pTmp, pElm, pGFE); + error = (0 == cpToOctStr_BNU(pStr, strSize, pTmp, elemLen)); + + cpGFpReleasePool(1, pGFE); + return error ? NULL : pStr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_gfeqnr.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_gfeqnr.c new file mode 100644 index 0000000..140c214 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_gfeqnr.c @@ -0,0 +1,60 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// cpGFEqnr() +// +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + +//tbcd: temporary excluded: #include + +IPP_OWN_DEFN (void, cpGFEqnr, (gsModEngine* pGFE)) +{ + BNU_CHUNK_T* pQnr = GFP_QNR(pGFE); + + int elemLen = GFP_FELEN(pGFE); + BNU_CHUNK_T* e = cpGFpGetPool(3, pGFE); + BNU_CHUNK_T* t = e+elemLen; + BNU_CHUNK_T* p1 = t+elemLen; + //tbcd: temporary excluded: assert(NULL!=e); + + cpGFpElementCopyPad(p1, elemLen, GFP_MNT_R(pGFE), elemLen); + + /* (modulus-1)/2 */ + cpLSR_BNU(e, GFP_MODULUS(pGFE), elemLen, 1); + + /* find a non-square g, where g^{(modulus-1)/2} = -1 */ + cpGFpElementCopy(pQnr, p1, elemLen); + do { + cpGFpAdd(pQnr, pQnr, p1, pGFE); + cpGFpExp(t, pQnr, e, elemLen, pGFE); + cpGFpNeg(t, t, pGFE); + } while( !GFP_EQ(p1, t, elemLen) ); + + cpGFpReleasePool(3, pGFE); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_inv.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_inv.c new file mode 100644 index 0000000..3531679 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_inv.c @@ -0,0 +1,37 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal operations over prime GF(p). +// +// Context: +// cpGFpInv +// +*/ +#include "owncp.h" + +#include "pcpbn.h" +#include "pcpgfpstuff.h" + +IPP_OWN_DEFN (BNU_CHUNK_T*, cpGFpInv, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsModEngine* pGFE)) +{ + GFP_METHOD(pGFE)->decode(pR, pA, pGFE); + /* gs_mont_inv(pR, pR, pGFE, alm_mont_inv); */ + gs_mont_inv(pR, pR, pGFE, alm_mont_inv_ct); /* switch on CTE internal function */ + return pR; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_rand.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_rand.c new file mode 100644 index 0000000..0a901e6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_rand.c @@ -0,0 +1,56 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal operations over prime GF(p). +// +// Context: +// cpGFpRand +// +*/ +#include "owncp.h" + +#include "pcpbn.h" +#include "pcpgfpstuff.h" + +//tbcd: temporary excluded: #include + +IPP_OWN_DEFN (BNU_CHUNK_T*, cpGFpRand, (BNU_CHUNK_T* pR, gsModEngine* pGFE, IppBitSupplier rndFunc, void* pRndParam)) +{ + int elemLen = GFP_FELEN(pGFE); + int reqBitSize = GFP_FEBITLEN(pGFE)+GFP_RAND_ADD_BITS; + int nsR = (reqBitSize +BITSIZE(BNU_CHUNK_T)-1)/BITSIZE(BNU_CHUNK_T); + + int internal_err; + + BNU_CHUNK_T* pPool = cpGFpGetPool(2, pGFE); + //tbcd: temporary excluded: assert(pPool!=NULL); + + cpGFpElementPad(pPool, nsR, 0); + + internal_err = ippStsNoErr != rndFunc((Ipp32u*)pPool, reqBitSize, pRndParam); + + if(!internal_err) { + nsR = cpMod_BNU(pPool, nsR, GFP_MODULUS(pGFE), elemLen); + cpGFpElementPad(pPool+nsR, elemLen-nsR, 0); + GFP_METHOD(pGFE)->encode(pR, pPool, pGFE); + } + + cpGFpReleasePool(2, pGFE); + return internal_err? NULL : pR; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_set.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_set.c new file mode 100644 index 0000000..636456f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_set.c @@ -0,0 +1,50 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal operations over prime GF(p). +// +// Context: +// cpGFpSet +// +*/ +#include "owncp.h" + +#include "pcpbn.h" +#include "pcpgfpstuff.h" + +//tbcd: temporary excluded: #include + +IPP_OWN_DEFN (BNU_CHUNK_T*, cpGFpSet, (BNU_CHUNK_T* pElm, const BNU_CHUNK_T* pDataA, int nsA, gsModEngine* pGFE)) +{ + const BNU_CHUNK_T* pModulus = GFP_MODULUS(pGFE); + int elemLen = GFP_FELEN(pGFE); + + if(0 <= cpCmp_BNU(pDataA, nsA, pModulus, elemLen)) + return NULL; + else { + BNU_CHUNK_T* pTmp = cpGFpGetPool(1, pGFE); + //tbcd: temporary excluded: assert(pTmp !=NULL); + + ZEXPAND_COPY_BNU(pTmp, elemLen, pDataA, nsA); + GFP_METHOD(pGFE)->encode(pElm, pTmp, pGFE); + + cpGFpReleasePool(1, pGFE); + return pElm; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_setgfp.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_setgfp.c new file mode 100644 index 0000000..aed6dcc --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_setgfp.c @@ -0,0 +1,70 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// cpGFpSetGFp() +// +// +*/ +#include "gsmodmethod.h" +#include "gsmodstuff.h" +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + + +IPP_OWN_DEFN (IppStatus, cpGFpSetGFp, (const BNU_CHUNK_T* pPrime, int primeBitSize, const IppsGFpMethod* method, IppsGFpState* pGF)) +{ + gsModEngine* pGFE = GFP_PMA(pGF); + + int primeLen = BITS_BNU_CHUNK(primeBitSize); + + /* arithmetic methods */ + GFP_METHOD(pGFE) = method->arith; + pGFE->method_alt = method->arith_alt; + + /* store modulus */ + COPY_BNU(GFP_MODULUS(pGFE), pPrime, primeLen); + + /* montgomery factor */ + GFP_MNT_FACTOR(pGFE) = gsMontFactor(GFP_MODULUS(pGFE)[0]); + + /* montgomery identity (R) */ + ZEXPAND_BNU(GFP_MNT_R(pGFE), 0, primeLen); + GFP_MNT_R(pGFE)[primeLen] = 1; + cpMod_BNU(GFP_MNT_R(pGFE), primeLen+1, GFP_MODULUS(pGFE), primeLen); + + /* montgomery domain converter (RR) */ + ZEXPAND_BNU(GFP_MNT_RR(pGFE), 0, primeLen); + COPY_BNU(GFP_MNT_RR(pGFE)+primeLen, GFP_MNT_R(pGFE), primeLen); + cpMod_BNU(GFP_MNT_RR(pGFE), 2*primeLen, GFP_MODULUS(pGFE), primeLen); + + /* half of modulus */ + cpLSR_BNU(GFP_HMODULUS(pGFE), GFP_MODULUS(pGFE), primeLen, 1); + + /* set qnr value */ + cpGFEqnr(pGFE); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_setoctstring.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_setoctstring.c new file mode 100644 index 0000000..ce65422 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_setoctstring.c @@ -0,0 +1,51 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal operations over prime GF(p). +// +// Context: +// cpGFpSetOctString +// +*/ +#include "owncp.h" + +#include "pcpbn.h" +#include "pcpgfpstuff.h" + +//tbcd: temporary excluded: #include + +IPP_OWN_DEFN (BNU_CHUNK_T*, cpGFpSetOctString, (BNU_CHUNK_T* pElm, const Ipp8u* pStr, int strSize, gsModEngine* pGFE)) +{ + int elemLen = GFP_FELEN(pGFE); + + if((int)(elemLen*(Ipp32s)sizeof(BNU_CHUNK_T)) < strSize) + return NULL; + + { + BNU_CHUNK_T* pTmp = cpGFpGetPool(1, pGFE); + //tbcd: temporary excluded: assert(pTmp !=NULL); + { + int nsTmp = cpFromOctStr_BNU(pTmp, pStr, strSize); + BNU_CHUNK_T* ret = cpGFpSet(pElm, pTmp, nsTmp, pGFE); + + cpGFpReleasePool(1, pGFE); + return ret==NULL? NULL : pElm; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_sqrt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_sqrt.c new file mode 100644 index 0000000..d0983e6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfp_sqrt.c @@ -0,0 +1,148 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal operations over prime GF(p). +// +// Context: +// cpGFpSqrt +// +*/ +#include "owncp.h" + +#include "pcpbn.h" +#include "pcpgfpstuff.h" + +static int factor2(BNU_CHUNK_T* pA, int nsA) +{ + int factor = 0; + int bits; + + int i; + for(i=0; i= BITSIZE(BNU_CHUNK_T)) { + int nchunk = bits/BITSIZE(BNU_CHUNK_T); + cpGFpElementCopyPad(pA, nsA, pA+nchunk, nsA-nchunk); + bits %= BITSIZE(BNU_CHUNK_T); + } + if(bits) + cpLSR_BNU(pA, pA, nsA, bits); + + return factor; +} + +static BNU_CHUNK_T* cpGFpExp2(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, int e, gsModEngine* pGFE) +{ + cpGFpElementCopy(pR, pA, GFP_FELEN(pGFE)); + while(e--) { + GFP_METHOD(pGFE)->sqr(pR, pR, pGFE); + } + return pR; +} + + +/* returns: + 0, if a - qnr + 1, if sqrt is found +*/ +IPP_OWN_DEFN (int, cpGFpSqrt, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsModEngine* pGFE)) +{ + int elemLen = GFP_FELEN(pGFE); + int poolelementLen = GFP_PELEN(pGFE); + int resultFlag = 1; + + /* case A==0 */ + if( GFP_IS_ZERO(pA, elemLen) ) + cpGFpElementPad(pR, elemLen, 0); + + /* general case */ + else { + BNU_CHUNK_T* q = cpGFpGetPool(4, pGFE); + BNU_CHUNK_T* x = q + poolelementLen; + BNU_CHUNK_T* y = x + poolelementLen; + BNU_CHUNK_T* z = y + poolelementLen; + + int s; + + //tbcd: temporary excluded: assert(q!=NULL); + + /* z=1 */ + GFP_ONE(z, elemLen); + + /* (modulus-1) = 2^s*q */ + cpSub_BNU(q, GFP_MODULUS(pGFE), z, elemLen); + s = factor2(q, elemLen); + + /* + // initialization + */ + + /* y = qnr^q */ + cpGFpExp(y, GFP_QNR(pGFE), q,elemLen, pGFE); + /* x = a^((q-1)/2) */ + cpSub_BNU(q, q, z, elemLen); + cpLSR_BNU(q, q, elemLen, 1); + cpGFpExp(x, pA, q, elemLen, pGFE); + /* z = a*x^2 */ + GFP_METHOD(pGFE)->mul(z, x, x, pGFE); + GFP_METHOD(pGFE)->mul(z, pA, z, pGFE); + /* R = a*x */ + GFP_METHOD(pGFE)->mul(pR, pA, x, pGFE); + + while( !GFP_EQ(z, MOD_MNT_R(pGFE), elemLen) ) { + int m = 0; + cpGFpElementCopy(q, z, elemLen); + + for(m=1; mmul(q, q, q, pGFE); + if( GFP_EQ(q, MOD_MNT_R(pGFE), elemLen) ) + break; + } + + if(m==s) { + /* A is quadratic non-residue */ + resultFlag = 0; + break; + } + else { + /* exponent reduction */ + cpGFpExp2(q, y, (s-m-1), pGFE); /* q = y^(2^(s-m-1)) */ + GFP_METHOD(pGFE)->mul(y, q, q, pGFE); /* y = q^2 */ + GFP_METHOD(pGFE)->mul(pR, q, pR, pGFE); /* R = q*R */ + GFP_METHOD(pGFE)->mul(z, y, z, pGFE); /* z = z*y */ + s = m; + } + } + + /* choose smallest between R and (modulus-R) */ + GFP_METHOD(pGFE)->decode(q, pR, pGFE); + if(1 == cpCmp_BNU(q, elemLen, GFP_HMODULUS(pGFE), elemLen)) + GFP_METHOD(pGFE)->neg(pR, pR, pGFE); + + cpGFpReleasePool(4, pGFE); + } + + return resultFlag; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpadd.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpadd.c new file mode 100644 index 0000000..720cef1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpadd.c @@ -0,0 +1,77 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpAdd() +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + + +/*F* +// Name: ippsGFpAdd +// +// Purpose: Sum of GF elements +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFp +// NULL == pA +// NULL == pR +// NULL == pB +// +// ippStsContextMatchErr invalid pGFp->idCtx +// invalid pA->idCtx +// invalid pR->idCtx +// invalid pB->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM() != GFP_FELEN() +// +// ippStsNoErr no error +// +// Parameters: +// pA Pointer to the context of the first finite field element. +// pB Pointer to the context of the second finite field element. +// pR Pointer to the context of the result finite field element. +// pGFp Pointer to the context of the finite field. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpAdd,(const IppsGFpElement* pA, const IppsGFpElement* pB, + IppsGFpElement* pR, IppsGFpState* pGFp)) +{ + IPP_BAD_PTR4_RET(pA, pB, pR, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pA), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pB), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pR), ippStsContextMatchErr ); + { + gsModEngine* pGFE = GFP_PMA(pGFp); + IPP_BADARG_RET( (GFPE_ROOM(pA)!=GFP_FELEN(pGFE)) || (GFPE_ROOM(pB)!=GFP_FELEN(pGFE)) || (GFPE_ROOM(pR)!=GFP_FELEN(pGFE)), ippStsOutOfRangeErr); + + GFP_METHOD(pGFE)->add(GFPE_DATA(pR), GFPE_DATA(pA), GFPE_DATA(pB), pGFE); + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpaddpe.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpaddpe.c new file mode 100644 index 0000000..5a74a2e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpaddpe.c @@ -0,0 +1,81 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpAdd_PE() +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + + +/*F* +// Name: ippsGFpAdd_PE +// +// Purpose: Sum of GF element and parent GF element +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFp +// NULL == pA +// NULL == pR +// NULL == pParentB +// +// ippStsContextMatchErr invalid pGFp->idCtx +// invalid pA->idCtx +// invalid pR->idCtx +// invalid pParentB->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM() != GFP_FELEN() +// +// ippStsBadArgErr pGFp does not specify prime field +// +// ippStsNoErr no error +// +// Parameters: +// pA Pointer to the context of the first finite field element. +// pParentB Pointer to the context of the second finite field element. +// pR Pointer to the context of the result finite field element. +// pGFp Pointer to the context of the finite field. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpAdd_PE,(const IppsGFpElement* pA, const IppsGFpElement* pParentB, + IppsGFpElement* pR, IppsGFpState* pGFp)) +{ + IPP_BAD_PTR4_RET(pA, pParentB, pR, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pA), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pParentB), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pR), ippStsContextMatchErr ); + { + gsModEngine* pGFE = GFP_PMA(pGFp); + IPP_BADARG_RET( GFP_IS_BASIC(pGFE), ippStsBadArgErr ) + IPP_BADARG_RET( (GFPE_ROOM(pA)!=GFP_FELEN(pGFE)) || (GFPE_ROOM(pR)!=GFP_FELEN(pGFE)), ippStsOutOfRangeErr); + IPP_BADARG_RET( (GFPE_ROOM(pParentB)!=GFP_FELEN(GFP_PARENT(pGFE))), ippStsOutOfRangeErr); + + cpGFpxAdd_GFE(GFPE_DATA(pR), GFPE_DATA(pA), GFPE_DATA(pParentB), pGFE); + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpbufsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpbufsize.c new file mode 100644 index 0000000..5628723 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpbufsize.c @@ -0,0 +1,79 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpScratchBufferSize +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + + +/*F* +// Name: ippsGFpScratchBufferSize +// +// Purpose: Gets the size of the scratch buffer. +// +// Returns: Reason: +// ippStsNullPtrErr pGFp == NULL +// pBufferSize == NULL +// ippStsContextMatchErr incorrect pGFp's context id +// ippStsBadArgErr 0>=nExponents +// nExponents>6 +// ippStsNoErr no error +// +// Parameters: +// nExponents Number of exponents. +// ExpBitSize Maximum bit size of the exponents. +// pGFp Pointer to the context of the finite field. +// pBufferSize Pointer to the calculated buffer size in bytes. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpScratchBufferSize,(int nExponents, int ExpBitSize, const IppsGFpState* pGFp, int* pBufferSize)) +{ + IPP_BAD_PTR2_RET(pGFp, pBufferSize); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + + IPP_BADARG_RET( 0>=nExponents ||nExponents>IPP_MAX_EXPONENT_NUM, ippStsBadArgErr); + IPP_BADARG_RET( 0>=ExpBitSize, ippStsBadArgErr); + + /* gres 06/10/2019: ExpBirSize=BNU_CHUNK_BITS*n -- meet CTE implementation */ + ExpBitSize = ((ExpBitSize + BNU_CHUNK_BITS-1)/BNU_CHUNK_BITS) * BNU_CHUNK_BITS; + { + int elmDataSize = GFP_FELEN(GFP_PMA(pGFp))*(Ipp32s)sizeof(BNU_CHUNK_T); + + /* get window_size */ + int w = (nExponents==1)? cpGFpGetOptimalWinSize(ExpBitSize) : /* use optimal window size, if single-scalar operation */ + nExponents; /* or pseudo-oprimal if multi-scalar operation */ + + /* number of table entries */ + int nPrecomputed = 1<idCtx +// invalid pA->idCtx +// invalid pB->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM() != GFP_FELEN() +// +// ippStsNoErr no error +// +// Parameters: +// pA Pointer to the context of the first finite field element. +// pB Pointer to the context of the second finite field element. +// pResult Pointer to the result of the comparison. +// pGFp Pointer to the context of the finite field. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpCmpElement,(const IppsGFpElement* pA, const IppsGFpElement* pB, + int* pResult, + const IppsGFpState* pGFp)) +{ + IPP_BAD_PTR4_RET(pA, pB, pResult, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pA), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pB), ippStsContextMatchErr ); + { + gsModEngine* pGFE = GFP_PMA(pGFp); + IPP_BADARG_RET( (GFPE_ROOM(pA)!=GFP_FELEN(pGFE)) || (GFPE_ROOM(pB)!=GFP_FELEN(pGFE)), ippStsOutOfRangeErr); + { + BNU_CHUNK_T* a = cpGFpGetPool(2, pGFE); + BNU_CHUNK_T* b = a + GFP_PELEN(pGFE); + + GFP_METHOD(pGFE)->decode(a, GFPE_DATA(pA), pGFE); + GFP_METHOD(pGFE)->decode(b, GFPE_DATA(pB), pGFE); + + ZEXPAND_BNU(a, GFP_FELEN(pGFE), GFP_PELEN(pGFE)); + ZEXPAND_BNU(b, GFP_FELEN(pGFE), GFP_PELEN(pGFE)); + + int flag = cpCmp_BNU(a, GFP_PELEN(pGFE), b, GFP_PELEN(pGFE)); + if( GFP_IS_BASIC(pGFE) ) + *pResult = (0==flag)? IPP_IS_EQ : (0idCtx +// invalid pA->idCtx +// invalid pR->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM() != GFP_FELEN() +// +// ippStsBadArgErr 2!=GFP_EXTDEGREE(pGFE) +// +// ippStsNoErr no error +// +// Parameters: +// pA Pointer to the context of the source finite field element. +// pR Pointer to the context of the result finite field element. +// pGFp Pointer to the context of the finite field. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpConj,(const IppsGFpElement* pA, + IppsGFpElement* pR, IppsGFpState* pGFp)) +{ + IPP_BAD_PTR3_RET(pA, pR, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pA), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pR), ippStsContextMatchErr ); + { + gsModEngine* pGFE = GFP_PMA(pGFp); + IPP_BADARG_RET( (GFPE_ROOM(pA)!=GFP_FELEN(pGFE)) || (GFPE_ROOM(pR)!=GFP_FELEN(pGFE)), ippStsOutOfRangeErr); + IPP_BADARG_RET( 2!=GFP_EXTDEGREE(pGFE), ippStsBadArgErr ) + + cpGFpxConj(GFPE_DATA(pR), GFPE_DATA(pA), pGFE); + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpcpyelem.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpcpyelem.c new file mode 100644 index 0000000..4f32c65 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpcpyelem.c @@ -0,0 +1,71 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpCpyElement() +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + + + +/*F* +// Name: ippsGFpCpyElement +// +// Purpose: Copy GF Element +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFp +// NULL == pA +// NULL == pR +// +// ippStsContextMatchErr invalid pGFp->idCtx +// invalid pA->idCtx +// invalid pR->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM() != GFP_FELEN() +// +// ippStsNoErr no error +// +// Parameters: +// pA pointer to the source element +// pR pointer to the result element +// pGFp pointer to Finite Field context +*F*/ + +IPPFUN(IppStatus, ippsGFpCpyElement, (const IppsGFpElement* pA, IppsGFpElement* pR, IppsGFpState* pGFp)) +{ + IPP_BAD_PTR3_RET(pA, pR, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pA), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pR), ippStsContextMatchErr ); + { + gsModEngine* pGFE = GFP_PMA(pGFp); + IPP_BADARG_RET( (GFPE_ROOM(pA)!=GFP_FELEN(pGFE)) || (GFPE_ROOM(pR)!=GFP_FELEN(pGFE)), ippStsOutOfRangeErr); + cpGFpElementCopy(GFPE_DATA(pR), GFPE_DATA(pA), GFP_FELEN(pGFE)); + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec.c new file mode 100644 index 0000000..e545f8c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec.c @@ -0,0 +1,61 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// cpGFpECGetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" + + +IPP_OWN_DEFN (int, cpGFpECGetSize, (int basicDeg, int basicElmBitSize)) +{ + int ctxSize = 0; + int elemLen = basicDeg*BITS_BNU_CHUNK(basicElmBitSize); + + int maxOrderBits = 1+ basicDeg*basicElmBitSize; + #if defined(_LEGACY_ECCP_SUPPORT_) + int maxOrderLen = BITS_BNU_CHUNK(maxOrderBits); + #endif + + int modEngineCtxSize; + if(ippStsNoErr==gsModEngineGetSize(maxOrderBits, MONT_DEFAULT_POOL_LENGTH, &modEngineCtxSize)) { + + ctxSize = (Ipp32s)sizeof(IppsGFpECState) + +elemLen*(Ipp32s)sizeof(BNU_CHUNK_T) /* EC coeff A */ + +elemLen*(Ipp32s)sizeof(BNU_CHUNK_T) /* EC coeff B */ + +elemLen*(Ipp32s)sizeof(BNU_CHUNK_T) /* generator G.x */ + +elemLen*(Ipp32s)sizeof(BNU_CHUNK_T) /* generator G.y */ + +elemLen*(Ipp32s)sizeof(BNU_CHUNK_T) /* generator G.z */ + +modEngineCtxSize /* mont engine (R) */ + +elemLen*(Ipp32s)sizeof(BNU_CHUNK_T) /* cofactor */ + #if defined(_LEGACY_ECCP_SUPPORT_) + +2*elemLen*3*(Ipp32s)sizeof(BNU_CHUNK_T) /* regular and ephemeral public keys */ + +2*maxOrderLen*(Ipp32s)sizeof(BNU_CHUNK_T) /* regular and ephemeral private keys */ + #endif + +elemLen*(Ipp32s)sizeof(BNU_CHUNK_T)*3*EC_POOL_SIZE; + } + return ctxSize; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_add.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_add.c new file mode 100644 index 0000000..17f0e73 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_add.c @@ -0,0 +1,138 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal EC over GF(p^m) basic Definitions & Function Prototypes +// +// Context: +// gfec_point_add() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpmask_ct.h" + + +#if ( ECP_PROJECTIVE_COORD == JACOBIAN ) +/* +// S1 = y1*z2^3 +// S2 = y2*z1^3 +// +// U1 = x1*z2^2 +// U2 = x2*z1^2 + +// R = S2-S1 +// H = U2-U1 +// +// x3 = -H^3 -2*U1*H^2 +R2 +// y3 = -S1*H^3 +R*(U1*H^2 -x3) +// z3 = z1*z2*H +// +// complexity = 4s+12m +*/ + +IPP_OWN_DEFN (void, gfec_point_add, (BNU_CHUNK_T* pRdata, const BNU_CHUNK_T* pPdata, const BNU_CHUNK_T* pQdata, IppsGFpECState* pEC)) +{ + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elemLen = GFP_FELEN(pGFE); + + mod_sub sub = GFP_METHOD(pGFE)->sub; /* gf sub */ + mod_mul2 mul2= GFP_METHOD(pGFE)->mul2; /* gf mul2 */ + mod_mul mul = GFP_METHOD(pGFE)->mul; /* gf mul */ + mod_sqr sqr = GFP_METHOD(pGFE)->sqr; /* gf sqr */ + + /* coordinates of P */ + const BNU_CHUNK_T* px1 = pPdata; + const BNU_CHUNK_T* py1 = pPdata+elemLen; + const BNU_CHUNK_T* pz1 = pPdata+2*elemLen; + + /* coordinates of Q */ + const BNU_CHUNK_T* px2 = pQdata; + const BNU_CHUNK_T* py2 = pQdata+elemLen; + const BNU_CHUNK_T* pz2 = pQdata+2*elemLen; + + BNU_CHUNK_T inftyP = GFPE_IS_ZERO_CT(pz1, elemLen); + BNU_CHUNK_T inftyQ = GFPE_IS_ZERO_CT(pz2, elemLen); + + /* get temporary from top of EC point pool */ + BNU_CHUNK_T* U1 = pEC->pPool; + BNU_CHUNK_T* U2 = U1 + elemLen; + BNU_CHUNK_T* S1 = U2 + elemLen; + BNU_CHUNK_T* S2 = S1 + elemLen; + BNU_CHUNK_T* H = S2 + elemLen; + BNU_CHUNK_T* R = H + elemLen; + + BNU_CHUNK_T* pRx = R + elemLen; /* temporary result */ + BNU_CHUNK_T* pRy = pRx+ elemLen; + BNU_CHUNK_T* pRz = pRy+ elemLen; + + mul(S1, py1, pz2, pGFE); // S1 = Y1*Z2 + sqr(U1, pz2, pGFE); // U1 = Z2^2 + + mul(S2, py2, pz1, pGFE); // S2 = Y2*Z1 + sqr(U2, pz1, pGFE); // U2 = Z1^2 + + mul(S1, S1, U1, pGFE); // S1 = Y1*Z2^3 + mul(S2, S2, U2, pGFE); // S2 = Y2*Z1^3 + + mul(U1, px1, U1, pGFE); // U1 = X1*Z2^2 + mul(U2, px2, U2, pGFE); // U2 = X2*Z1^2 + + sub(R, S2, S1, pGFE); // R = S2-S1 + sub(H, U2, U1, pGFE); // H = U2-U1 + + { + BNU_CHUNK_T mask_zeroH = GFPE_IS_ZERO_CT(H, elemLen); + BNU_CHUNK_T mask = mask_zeroH & ~inftyP & ~inftyQ; + if(mask) { + if( GFPE_IS_ZERO_CT(R, elemLen) ) + gfec_point_double(pRdata, pPdata, pEC); + else + cpGFpElementPad(pRdata, 3*elemLen, 0); + return; + } + } + + mul(pRz, pz1, pz2, pGFE); // Z3 = Z1*Z2 + sqr(U2, H, pGFE); // U2 = H^2 + mul(pRz, pRz, H, pGFE); // Z3 = (Z1*Z2)*H + sqr(S2, R, pGFE); // S2 = R^2 + mul(H, H, U2, pGFE); // H = H^3 + + mul(U1, U1, U2, pGFE); // U1 = U1*H^2 + sub(pRx, S2, H, pGFE); // X3 = R^2 - H^3 + mul2(U2, U1, pGFE); // U2 = 2*U1*H^2 + mul(S1, S1, H, pGFE); // S1 = S1*H^3 + sub(pRx, pRx, U2, pGFE); // X3 = (R^2 - H^3) -2*U1*H^2 + + sub(pRy, U1, pRx, pGFE); // Y3 = R*(U1*H^2 - X3) -S1*H^3 + mul(pRy, pRy, R, pGFE); + sub(pRy, pRy, S1, pGFE); + + cpMaskedReplace_ct(pRx, px2, elemLen*3, inftyP); + cpMaskedReplace_ct(pRx, px1, elemLen*3, inftyQ); + + cpGFpElementCopy(pRdata, pRx, 3*elemLen); +} +#endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_addaffine.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_addaffine.c new file mode 100644 index 0000000..145e488 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_addaffine.c @@ -0,0 +1,110 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal EC over GF(p^m) basic Definitions & Function Prototypes +// +// Context: +// gfec_affine_point_add() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpmask_ct.h" + + +#if ( ECP_PROJECTIVE_COORD == JACOBIAN ) +/* +// complexity = 3s+8m +*/ +IPP_OWN_DEFN (void, gfec_affine_point_add, (BNU_CHUNK_T* pRdata, const BNU_CHUNK_T* pPdata, const BNU_CHUNK_T* pAdata, IppsGFpECState* pEC)) +{ + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elemLen = GFP_FELEN(pGFE); + + mod_sub sub = GFP_METHOD(pGFE)->sub; /* gf sub */ + mod_mul2 mul2= GFP_METHOD(pGFE)->mul2; /* gf mul2 */ + mod_mul mul = GFP_METHOD(pGFE)->mul; /* gf mul */ + mod_sqr sqr = GFP_METHOD(pGFE)->sqr; /* gf sqr */ + + BNU_CHUNK_T* mont1 = GFP_MNT_R(pGFE); + + /* coordinates of projective P point */ + const BNU_CHUNK_T* px = pPdata; /* x1 */ + const BNU_CHUNK_T* py = pPdata+elemLen; /* y1 */ + const BNU_CHUNK_T* pz = pPdata+2*elemLen; /* z1 */ + + /* coordinates of affine A point, az==mont(1) */ + const BNU_CHUNK_T* ax = pAdata; /* x2 */ + const BNU_CHUNK_T* ay = pAdata+elemLen; /* y2 */ + + BNU_CHUNK_T inftyP = GFPE_IS_ZERO_CT(px, elemLen) & GFPE_IS_ZERO_CT(py, elemLen); + BNU_CHUNK_T inftyA = GFPE_IS_ZERO_CT(ax, elemLen) & GFPE_IS_ZERO_CT(ay, elemLen); + + /* get temporary from top of EC point pool */ + BNU_CHUNK_T* U2 = pEC->pPool; + BNU_CHUNK_T* S2 = U2 + elemLen; + BNU_CHUNK_T* H = S2 + elemLen; + BNU_CHUNK_T* R = H + elemLen; + + BNU_CHUNK_T* pRx = R + elemLen; /* temporary result */ + BNU_CHUNK_T* pRy = pRx+ elemLen; + BNU_CHUNK_T* pRz = pRy+ elemLen; + + sqr(R, pz, pGFE); // R = Z1^2 + mul(S2, ay, pz, pGFE); // S2 = Y2*Z1 + mul(U2, ax, R, pGFE); // U2 = X2*Z1^2 + mul(S2, S2, R, pGFE); // S2 = Y2*Z1^3 + + sub(H, U2, px, pGFE); // H = U2-X1 + sub(R, S2, py, pGFE); // R = S2-Y1 + + mul(pRz, H, pz, pGFE); // Z3 = H*Z1 + + sqr(U2, H, pGFE); // U2 = H^2 + sqr(S2, R, pGFE); // S2 = R^2 + mul(H, H, U2, pGFE); // H = H^3 + + mul(U2, U2, px, pGFE); // U2 = X1*H^2 + + mul(pRy, H, py, pGFE); // T = Y1*H^3 + + mul2(pRx, U2, pGFE); // X3 = 2*X1*H^2 + sub(pRx, S2, pRx, pGFE); // X3 = R^2 - 2*X1*H^2 + sub(pRx, pRx, H, pGFE); // X3 = R^2 - 2*X1*H^2 -H^3 + + sub(U2, U2, pRx, pGFE); // U2 = X1*H^2 - X3 + mul(U2, U2, R, pGFE); // U2 = R*(X1*H^2 - X3) + sub(pRy, U2, pRy, pGFE); // Y3 = -Y1*H^3 + R*(X1*H^2 - X3) + + cpMaskedReplace_ct(pRx, ax, elemLen, inftyP); + cpMaskedReplace_ct(pRy, ay, elemLen, inftyP); + cpMaskedReplace_ct(pRz, mont1, elemLen, inftyP); + cpMaskedReplace_ct(pRz, ax, elemLen, inftyP&inftyA); + + cpMaskedReplace_ct(pRx, px, elemLen*3, inftyA); + + cpGFpElementCopy(pRdata, pRx, 3*elemLen); +} +#endif + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_baseprod.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_baseprod.c new file mode 100644 index 0000000..823fb7d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_baseprod.c @@ -0,0 +1,74 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal EC over GF(p^m) basic Definitions & Function Prototypes +// +// Context: +// gfec_BasePointProduct() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "gsscramble.h" + + +IPP_OWN_DEFN (IppsGFpECPoint*, gfec_BasePointProduct, (IppsGFpECPoint* pR, const BNU_CHUNK_T* pScalarG, int scalarGlen, const IppsGFpECPoint* pP, const BNU_CHUNK_T* pScalarP, int scalarPlen, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +{ + FIX_BNU(pScalarG, scalarGlen); + FIX_BNU(pScalarP, scalarPlen); + + { + gsModEngine* pGForder = ECP_MONT_R(pEC); + int orderBits = MOD_BITSIZE(pGForder); + int orderLen = MOD_LEN(pGForder); + BNU_CHUNK_T* tmpScalarG = cpGFpGetPool(2, pGForder); + BNU_CHUNK_T* tmpScalarP = tmpScalarG+orderLen+1; + + cpGFpElementCopyPad(tmpScalarG, orderLen+1, pScalarG,scalarGlen); + cpGFpElementCopyPad(tmpScalarP, orderLen+1, pScalarP,scalarPlen); + + if(ECP_PREMULBP(pEC)) { + BNU_CHUNK_T* productG = cpEcGFpGetPool(2, pEC); + BNU_CHUNK_T* productP = productG+ECP_POINTLEN(pEC); + + gfec_base_point_mul(productG, (Ipp8u*)tmpScalarG, orderBits, pEC); + gfec_point_mul(productP, ECP_POINT_X(pP), (Ipp8u*)tmpScalarP, orderBits, pEC, pScratchBuffer); + gfec_point_add(ECP_POINT_X(pR), productG, productP, pEC); + + cpEcGFpReleasePool(2, pEC); + } + + else { + gfec_point_prod(ECP_POINT_X(pR), + ECP_G(pEC), (Ipp8u*)tmpScalarG, + ECP_POINT_X(pP), (Ipp8u*)tmpScalarP, + orderBits, + pEC, pScratchBuffer); + } + + cpGFpReleasePool(2, pGForder); + } + + ECP_POINT_FLAGS(pR) = gfec_IsPointAtInfinity(pR)? 0 : ECP_FINITE_POINT; + return pR; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_comppont.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_comppont.c new file mode 100644 index 0000000..f4c163f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_comppont.c @@ -0,0 +1,103 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal EC over GF(p^m) basic Definitions & Function Prototypes +// +// Context: +// gfec_ComparePoint() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "gsscramble.h" + + +#if ( ECP_PROJECTIVE_COORD == JACOBIAN ) +IPP_OWN_DEFN (int, gfec_ComparePoint, (const IppsGFpECPoint* pP, const IppsGFpECPoint* pQ, IppsGFpECState* pEC)) +{ + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elemLen = GFP_FELEN(pGFE); + + /* P or/and Q at Infinity */ + if( !IS_ECP_FINITE_POINT(pP) ) + return !IS_ECP_FINITE_POINT(pQ)? 1:0; + if( !IS_ECP_FINITE_POINT(pQ) ) + return !IS_ECP_FINITE_POINT(pP)? 1:0; + + /* Px==Qx && Py==Qy && Pz==Qz */ + if( GFP_EQ(ECP_POINT_Z(pP), ECP_POINT_Z(pQ), elemLen) + &&GFP_EQ(ECP_POINT_X(pP), ECP_POINT_X(pQ), elemLen) + &&GFP_EQ(ECP_POINT_Y(pP), ECP_POINT_Y(pQ), elemLen)) + return 1; + + else { + mod_mul mulF = GFP_METHOD(pGFE)->mul; + mod_sqr sqrF = GFP_METHOD(pGFE)->sqr; + + int isEqu = 1; + + BNU_CHUNK_T* pPtmp = cpGFpGetPool(1, pGFE); + BNU_CHUNK_T* pQtmp = cpGFpGetPool(1, pGFE); + BNU_CHUNK_T* pPz = cpGFpGetPool(1, pGFE); + BNU_CHUNK_T* pQz = cpGFpGetPool(1, pGFE); + + if(isEqu) { + /* Px*Qz^2 ~ Qx*Pz^2 */ + if( IS_ECP_AFFINE_POINT(pQ) ) /* Ptmp = Px * Qz^2 */ + cpGFpElementCopy(pPtmp, ECP_POINT_X(pP), elemLen); + else { + sqrF(pQz, ECP_POINT_Z(pQ), pGFE); + mulF(pPtmp, ECP_POINT_X(pP), pQz, pGFE); + } + if( IS_ECP_AFFINE_POINT(pP) ) /* Qtmp = Qx * Pz^2 */ + cpGFpElementCopy(pQtmp, ECP_POINT_X(pQ), elemLen); + else { + sqrF(pPz, ECP_POINT_Z(pP), pGFE); + mulF(pQtmp, ECP_POINT_X(pQ), pPz, pGFE); + } + isEqu = GFP_EQ(pPtmp, pQtmp, elemLen); + } + + if(isEqu) { + /* Py*Qz^3 ~ Qy*Pz^3 */ + if( IS_ECP_AFFINE_POINT(pQ) ) /* Ptmp = Py * Qz^3 */ + cpGFpElementCopy(pPtmp, ECP_POINT_Y(pP), elemLen); + else { + mulF(pQz, ECP_POINT_Z(pQ), pQz, pGFE); + mulF(pPtmp, pQz, ECP_POINT_Y(pP), pGFE); + } + if( IS_ECP_AFFINE_POINT(pP) ) /* Qtmp = Qy * Pz^3 */ + cpGFpElementCopy(pQtmp, ECP_POINT_Y(pQ), elemLen); + else { + mulF(pPz, ECP_POINT_Z(pP), pPz, pGFE); + mulF(pQtmp, pPz, ECP_POINT_Y(pQ), pGFE); + } + isEqu = GFP_EQ(pPtmp, pQtmp, elemLen); + } + + cpGFpReleasePool(4, pGFE); + return isEqu; + } +} +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_dblpoint.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_dblpoint.c new file mode 100644 index 0000000..31139ff --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_dblpoint.c @@ -0,0 +1,111 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal EC over GF(p^m) basic Definitions & Function Prototypes +// +// Context: +// gfec_point_double() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "gsscramble.h" + + + +#if ( ECP_PROJECTIVE_COORD == JACOBIAN ) +/* +// A = 4*x*y^2 +// B = 3*x^2 + a*z^4 +// +// x3 = -2*A + B^2 +// y3 = -8y^4 +B*(A-x3) +// z3 = 2*y*z +// +// complexity: = 4s+4m (NIST's, SM2 curves) +// = (EPID2 curve) +// = 6s+4m (arbitrary curves) +*/ +IPP_OWN_DEFN (void, gfec_point_double, (BNU_CHUNK_T* pRdata, const BNU_CHUNK_T* pPdata, IppsGFpECState* pEC)) +{ + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elemLen = GFP_FELEN(pGFE); + + mod_add add = GFP_METHOD(pGFE)->add; /* gf add */ + mod_sub sub = GFP_METHOD(pGFE)->sub; /* gf sub */ + mod_div2 div2= GFP_METHOD(pGFE)->div2; /* gf div2 */ + mod_mul2 mul2= GFP_METHOD(pGFE)->mul2; /* gf mul2 */ + mod_mul3 mul3= GFP_METHOD(pGFE)->mul3; /* gf mul3 */ + mod_mul mul = GFP_METHOD(pGFE)->mul; /* gf mul */ + mod_sqr sqr = GFP_METHOD(pGFE)->sqr; /* gf sqr */ + + const BNU_CHUNK_T* pX = pPdata; + const BNU_CHUNK_T* pY = pPdata+elemLen; + const BNU_CHUNK_T* pZ = pPdata+2*+elemLen; + + BNU_CHUNK_T* rX = pRdata; + BNU_CHUNK_T* rY = pRdata+elemLen; + BNU_CHUNK_T* rZ = pRdata+2*elemLen; + + /* get temporary from top of EC point pool */ + BNU_CHUNK_T* U = pEC->pPool; + BNU_CHUNK_T* M = U+elemLen; + BNU_CHUNK_T* S = M+elemLen; + + mul2(S, pY, pGFE); /* S = 2*Y */ + sqr(U, pZ, pGFE); /* U = Z^2 */ + + sqr(M, S, pGFE); /* M = 4*Y^2 */ + mul(rZ, S, pZ, pGFE); /* Zres = 2*Y*Z */ + + sqr(rY, M, pGFE); /* Yres = 16*Y^4 */ + + mul(S, M, pX, pGFE); /* S = 4*X*Y^2 */ + div2(rY, rY, pGFE); /* Yres = 8*Y^4 */ + + if(ECP_STD==ECP_SPECIFIC(pEC)) { + add(M, pX, U, pGFE); /* M = 3*(X^2-Z^4) */ + sub(U, pX, U, pGFE); + mul(M, M, U, pGFE); + mul3(M, M, pGFE); + } + else { + sqr(M, pX, pGFE); /* M = 3*X^2 */ + mul3(M, M, pGFE); + if(ECP_EPID2!=ECP_SPECIFIC(pEC)) { + sqr(U, U, pGFE); /* M = 3*X^2+a*Z4 */ + mul(U, U, ECP_A(pEC), pGFE); + add(M, M, U, pGFE); + } + } + + mul2(U, S, pGFE); /* U = 8*X*Y^2 */ + sqr(rX, M, pGFE); /* Xres = M^2 */ + sub(rX, rX, U, pGFE); /* Xres = M^2-U */ + + sub(S, S, rX, pGFE); /* S = 4*X*Y^2-Xres */ + mul(S, S, M, pGFE); /* S = M*(4*X*Y^2-Xres) */ + sub(rY, S, rY, pGFE); /* Yres = M*(4*X*Y^2-Xres) -8*Y^4 */ +} +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_getpoint.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_getpoint.c new file mode 100644 index 0000000..f854df3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_getpoint.c @@ -0,0 +1,84 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal EC over GF(p^m) basic Definitions & Function Prototypes +// +// Context: +// gfec_GetPoint() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "gsscramble.h" + + +#if ( ECP_PROJECTIVE_COORD == JACOBIAN ) +/* returns 1/0 if point is finite/infinite */ +IPP_OWN_DEFN (int, gfec_GetPoint, (BNU_CHUNK_T* pX, BNU_CHUNK_T* pY, const IppsGFpECPoint* pPoint, IppsGFpECState* pEC)) +{ + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elemLen = GFP_FELEN(pGFE); + + if( !IS_ECP_FINITE_POINT(pPoint) ) { + if(pX) cpGFpElementPad(pX, elemLen, 0); + if(pY) cpGFpElementPad(pY, elemLen, 0); + return 0; + } + + /* affine point (1==Z) */ + if( IS_ECP_AFFINE_POINT(pPoint) ) { + if(pX) + cpGFpElementCopy(pX, ECP_POINT_X(pPoint), elemLen); + if(pY) + cpGFpElementCopy(pY, ECP_POINT_Y(pPoint), elemLen); + return 1; + } + + /* projective point (1!=Z) */ + { + mod_mul mulF = GFP_METHOD(pGFE)->mul; + mod_sqr sqrF = GFP_METHOD(pGFE)->sqr; + + /* T = (1/Z)*(1/Z) */ + BNU_CHUNK_T* pT = cpGFpGetPool(1, pGFE); + BNU_CHUNK_T* pZinv = cpGFpGetPool(1, pGFE); + BNU_CHUNK_T* pU = cpGFpGetPool(1, pGFE); + cpGFpxInv(pZinv, ECP_POINT_Z(pPoint), pGFE); + sqrF(pT, pZinv, pGFE); + + if(pX) { + mulF(pU, ECP_POINT_X(pPoint), pT, pGFE); + cpGFpElementCopy(pX, pU, elemLen); + } + if(pY) { + mulF(pT, pZinv, pT, pGFE); + mulF(pU, ECP_POINT_Y(pPoint), pT, pGFE); + cpGFpElementCopy(pY, pU, elemLen); + } + + cpGFpReleasePool(3, pGFE); + return 1; + } +} +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_makepoint.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_makepoint.c new file mode 100644 index 0000000..bf88d6e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_makepoint.c @@ -0,0 +1,72 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal EC over GF(p^m) basic Definitions & Function Prototypes +// +// Context: +// gfec_MakePoint() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "gsscramble.h" + +IPP_OWN_DEFN (int, gfec_MakePoint, (IppsGFpECPoint* pPoint, const BNU_CHUNK_T* pElm, IppsGFpECState* pEC)) +{ + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elemLen = GFP_FELEN(pGFE); + + mod_mul mulF = GFP_METHOD(pGFE)->mul; + mod_sqr sqrF = GFP_METHOD(pGFE)->sqr; + mod_add addF = GFP_METHOD(pGFE)->add; + + BNU_CHUNK_T* pX = ECP_POINT_X(pPoint); + BNU_CHUNK_T* pY = ECP_POINT_Y(pPoint); + BNU_CHUNK_T* pZ = ECP_POINT_Z(pPoint); + + /* set x-coordinate */ + cpGFpElementCopy(pX, pElm, elemLen); + + /* T = X^3 + A*X + B */ + sqrF(pY, pX, pGFE); + mulF(pY, pY, pX, pGFE); + if(ECP_SPECIFIC(pEC)!=ECP_EPID2) { + mulF(pZ, ECP_A(pEC), pX, pGFE); + addF(pY, pY, pZ, pGFE); + } + addF(pY, pY, ECP_B(pEC), pGFE); + + /* set z-coordinate =1 */ + cpGFpElementCopyPad(pZ, elemLen, GFP_MNT_R(pGFE), elemLen); + + /* Y = sqrt(Y) */ + if( cpGFpSqrt(pY, pY, pGFE) ) { + ECP_POINT_FLAGS(pPoint) = ECP_AFFINE_POINT | ECP_FINITE_POINT; + return 1; + } + else { + gfec_SetPointAtInfinity(pPoint); + return 0; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_mul.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_mul.c new file mode 100644 index 0000000..e073abd --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_mul.c @@ -0,0 +1,112 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal EC over GF(p^m) basic Definitions & Function Prototypes +// +// Context: +// gfec_MulPoint() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "gsscramble.h" +#include "pcpmask_ct.h" + + +IPP_OWN_DEFN (void, gfec_point_mul, (BNU_CHUNK_T* pRdata, const BNU_CHUNK_T* pPdata, const Ipp8u* pScalar8, int scalarBitSize, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +{ + int pointLen = ECP_POINTLEN(pEC); + + /* optimal size of window */ + const int window_size = 5; + + /* aligned pre-computed table */ + BNU_CHUNK_T* pTable = (BNU_CHUNK_T*)IPP_ALIGNED_PTR(pScratchBuffer, CACHE_LINE_SIZE); + setupTable(pTable, pPdata, pEC); + + { + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elemLen = GFP_FELEN(pGFE); + + mod_neg negF = GFP_METHOD(pGFE)->neg; + + BNU_CHUNK_T* pHy = cpGFpGetPool(1, pGFE); + + BNU_CHUNK_T* pTdata = cpEcGFpGetPool(1, pEC); /* points from the pool */ + BNU_CHUNK_T* pHdata = cpEcGFpGetPool(1, pEC); + + int wvalue; + Ipp8u digit, sign; + int mask = (1<<(window_size+1)) -1; + int bit = scalarBitSize-(scalarBitSize%window_size); + + /* first window */ + if(bit) { + wvalue = *((Ipp16u*)&pScalar8[(bit-1)/8]); + wvalue = (wvalue>> ((bit-1)%8)) & mask; + } + else + wvalue = 0; + booth_recode(&sign, &digit, (Ipp8u)wvalue, window_size); + gsScrambleGet_sscm(pTdata, pointLen, pTable, digit-1, 5-1); + + for(bit-=window_size; bit>=window_size; bit-=window_size) { + gfec_point_double(pTdata, pTdata, pEC); /* probably it's better to have separate calls */ + gfec_point_double(pTdata, pTdata, pEC); /* instead of gfec_point_double_k() */ + gfec_point_double(pTdata, pTdata, pEC); + gfec_point_double(pTdata, pTdata, pEC); + gfec_point_double(pTdata, pTdata, pEC); + + wvalue = *((Ipp16u*)&pScalar8[(bit-1)/8]); + wvalue = (wvalue>> ((bit-1)%8)) & mask; + booth_recode(&sign, &digit, (Ipp8u)wvalue, window_size); + gsScrambleGet_sscm(pHdata, pointLen, pTable, digit-1, 5-1); + + negF(pHy, pHdata+elemLen, pGFE); + cpMaskedReplace_ct(pHdata+elemLen, pHy, elemLen, ~cpIsZero_ct(sign)); + gfec_point_add(pTdata, pTdata, pHdata, pEC); + } + + /* last window */ + gfec_point_double(pTdata, pTdata, pEC); + gfec_point_double(pTdata, pTdata, pEC); + gfec_point_double(pTdata, pTdata, pEC); + gfec_point_double(pTdata, pTdata, pEC); + gfec_point_double(pTdata, pTdata, pEC); + + wvalue = *((Ipp16u*)&pScalar8[0]); + wvalue = (wvalue << 1) & mask; + booth_recode(&sign, &digit, (Ipp8u)wvalue, window_size); + gsScrambleGet_sscm(pHdata, pointLen, pTable, digit-1, 5-1); + + negF(pHy, pHdata+elemLen, pGFE); + cpMaskedReplace_ct(pHdata+elemLen, pHy, elemLen, ~cpIsZero_ct(sign)); + gfec_point_add(pTdata, pTdata, pHdata, pEC); + + cpGFpElementCopy(pRdata, pTdata, pointLen); + + cpEcGFpReleasePool(2, pEC); + cpGFpReleasePool(1, pGFE); + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_mul1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_mul1.c new file mode 100644 index 0000000..a555c1c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_mul1.c @@ -0,0 +1,75 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal EC over GF(p^m) basic Definitions & Function Prototypes +// +// Context: +// gfec_MulPoint() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "gsscramble.h" + +#if 0 +IPP_OWN_DEFN (IppsGFpECPoint*, gfec_MulPoint, (IppsGFpECPoint* pR, const IppsGFpECPoint* pP, const BNU_CHUNK_T* pScalar, int scalarLen, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +{ + FIX_BNU(pScalar, scalarLen); + { + gsModEngine* pGForder = ECP_MONT_R(pEC); + + BNU_CHUNK_T* pTmpScalar = cpGFpGetPool(1, pGForder); /* length of scalar does not exceed length of order */ + int orderBits = MOD_BITSIZE(pGForder); + int orderLen = MOD_LEN(pGForder); + cpGFpElementCopyPad(pTmpScalar,orderLen+1, pScalar,scalarLen); + + gfec_point_mul(ECP_POINT_X(pR), ECP_POINT_X(pP), + (Ipp8u*)pTmpScalar, orderBits, + pEC, pScratchBuffer); + cpGFpReleasePool(1, pGForder); + + ECP_POINT_FLAGS(pR) = gfec_IsPointAtInfinity(pR)? 0 : ECP_FINITE_POINT; + return pR; + } +} +#endif +IPP_OWN_DEFN (IppsGFpECPoint*, gfec_MulPoint, (IppsGFpECPoint* pR, const IppsGFpECPoint* pP, const BNU_CHUNK_T* pScalar, int scalarLen, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +{ + FIX_BNU(pScalar, scalarLen); + { + gsModEngine* pME = GFP_PMA(ECP_GFP(pEC)); + + BNU_CHUNK_T* pTmpScalar = cpGFpGetPool(2, pME); + int orderBits = ECP_ORDBITSIZE(pEC); + int orderLen = BITS_BNU_CHUNK(orderBits); + cpGFpElementCopyPad(pTmpScalar, orderLen + 1, pScalar, scalarLen); + + gfec_point_mul(ECP_POINT_X(pR), ECP_POINT_X(pP), + (Ipp8u*)pTmpScalar, orderBits, + pEC, pScratchBuffer); + cpGFpReleasePool(2, pME); + + ECP_POINT_FLAGS(pR) = gfec_IsPointAtInfinity(pR) ? 0 : ECP_FINITE_POINT; + return pR; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_mulbase.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_mulbase.c new file mode 100644 index 0000000..5f58927 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_mulbase.c @@ -0,0 +1,92 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal EC over GF(p^m) basic Definitions & Function Prototypes +// +// Context: +// gfec_base_point_mul() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpmask_ct.h" + +IPP_OWN_DEFN (void, gfec_base_point_mul, (BNU_CHUNK_T* pRdata, const Ipp8u* pScalar8, int scalarBitSize, IppsGFpECState* pEC)) +{ + /* size of window, get function and pre-computed table */ + int window_size = ECP_PREMULBP(pEC)->w; + selectAP select_affine_point = ECP_PREMULBP(pEC)->select_affine_point; + const BNU_CHUNK_T* pTbl = ECP_PREMULBP(pEC)->pTbl; + + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elmLen = GFP_FELEN(pGFE); + + mod_neg negF = GFP_METHOD(pGFE)->neg; + + BNU_CHUNK_T* mont1 = GFP_MNT_R(pGFE); + + /* number of points per table slot */ + int tslot_point = 1<<(window_size-1); + int tslot_size = tslot_point * (elmLen*2); + + BNU_CHUNK_T* negtmp = cpGFpGetPool(1, pGFE); /* temporary element */ + BNU_CHUNK_T* pointT = cpEcGFpGetPool(1, pEC); /* temporary point */ + + Ipp8u digit, sign; + int mask = (1<<(window_size+1)) -1; + int bit = 0; + + /* processing of window[0] */ + int wvalue = *((Ipp16u*)&pScalar8[0]); + wvalue = (wvalue << 1) & mask; + + booth_recode(&sign, &digit, (Ipp8u)wvalue, window_size); + select_affine_point(pRdata, pTbl, digit); + + /* if(sign) R.y = -R.y */ + negF(negtmp, pRdata+elmLen, pGFE); + cpMaskedReplace_ct(pRdata+elmLen, negtmp, elmLen, ~cpIsZero_ct(sign)); + /* R.z = R!=O? mont(1) : 0 */ + cpGFpElementCopy(pRdata+elmLen*2, mont1, elmLen); + cpGFpElementSetChunk(negtmp, elmLen, 0); + cpMaskedReplace_ct(pRdata+elmLen*2, negtmp, elmLen, cpIsZero_ct(digit)); + //T afine cpGFpElementCopy(pointT+elmLen*2, mont1, elmLen); + + /* processing of other windows.. [1],[2],... */ + for(bit+=window_size, pTbl+=tslot_size; bit<=scalarBitSize; bit+=window_size, pTbl+=tslot_size) { + wvalue = *((Ipp16u*)&pScalar8[(bit-1)/8]); + wvalue = (wvalue>> ((bit-1)%8)) & mask; + + booth_recode(&sign, &digit, (Ipp8u)wvalue, window_size); + select_affine_point(pointT, pTbl, digit); + + negF(negtmp, pointT+elmLen, pGFE); + cpMaskedReplace_ct(pointT+elmLen, negtmp, elmLen, ~cpIsZero_ct(sign)); + + gfec_affine_point_add(pRdata, pRdata, pointT, pEC); + } + + cpEcGFpReleasePool(1, pEC); + cpGFpReleasePool(1, pGFE); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_mulbase1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_mulbase1.c new file mode 100644 index 0000000..659b092 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_mulbase1.c @@ -0,0 +1,58 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal EC over GF(p^m) basic Definitions & Function Prototypes +// +// Context: +// gfec_MulBasePoint() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "gsscramble.h" + +IPP_OWN_DEFN (IppsGFpECPoint*, gfec_MulBasePoint, (IppsGFpECPoint* pR, const BNU_CHUNK_T* pScalar, int scalarLen, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +{ + FIX_BNU(pScalar, scalarLen); + { + gsModEngine* pGForder = ECP_MONT_R(pEC); + + BNU_CHUNK_T* pTmpScalar = cpGFpGetPool(1, pGForder); /* length of scalar does not exceed length of order */ + int orderBits = MOD_BITSIZE(pGForder); + int orderLen = MOD_LEN(pGForder); + cpGFpElementCopyPad(pTmpScalar,orderLen+1, pScalar,scalarLen); + + if(ECP_PREMULBP(pEC)) + gfec_base_point_mul(ECP_POINT_X(pR), + (Ipp8u*)pTmpScalar, orderBits, + pEC); + else + gfec_point_mul(ECP_POINT_X(pR), ECP_G(pEC), + (Ipp8u*)pTmpScalar, orderBits, + pEC, pScratchBuffer); + cpGFpReleasePool(1, pGForder); + + ECP_POINT_FLAGS(pR) = gfec_IsPointAtInfinity(pR)? 0 : ECP_FINITE_POINT; + return pR; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_negpoint.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_negpoint.c new file mode 100644 index 0000000..bcb5875 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_negpoint.c @@ -0,0 +1,44 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal EC over GF(p^m) basic Definitions & Function Prototypes +// +// Context: +// gfec_NegPoint() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "gsscramble.h" + + +IPP_OWN_DEFN (IppsGFpECPoint*, gfec_NegPoint, (IppsGFpECPoint* pR, const IppsGFpECPoint* pP, IppsGFpECState* pEC)) +{ + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elmLen = GFP_FELEN(pGFE); + if(pR!=pP) + gfec_CopyPoint(pR, pP, elmLen); + GFP_METHOD(pGFE)->neg(ECP_POINT_Y(pR), ECP_POINT_Y(pP), pGFE); + return pR; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_prod.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_prod.c new file mode 100644 index 0000000..d961dde --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_prod.c @@ -0,0 +1,145 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal EC over GF(p^m) basic Definitions & Function Prototypes +// +// Context: +// gfec_point_prod() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "gsscramble.h" +#include "pcpmask_ct.h" + + +IPP_OWN_DEFN (void, gfec_point_prod, (BNU_CHUNK_T* pointR, const BNU_CHUNK_T* pointA, const Ipp8u* scalarA, const BNU_CHUNK_T* pointB, const Ipp8u* scalarB, int scalarBitSize, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +{ + int pointLen = ECP_POINTLEN(pEC); + + /* optimal size of window */ + const int window_size = 5; + /* number of table entries */ + const int tableLen = 1<<(window_size-1); + + /* aligned pre-computed tables */ + BNU_CHUNK_T* pTableA = (BNU_CHUNK_T*)IPP_ALIGNED_PTR(pScratchBuffer, CACHE_LINE_SIZE); + BNU_CHUNK_T* pTableB = pTableA+pointLen*tableLen; + + setupTable(pTableA, pointA, pEC); + setupTable(pTableB, pointB, pEC); + + { + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elemLen = GFP_FELEN(pGFE); + + mod_neg negF = GFP_METHOD(pGFE)->neg; + + BNU_CHUNK_T* pHy = cpGFpGetPool(1, pGFE); + + BNU_CHUNK_T* pTdata = cpEcGFpGetPool(1, pEC); /* points from the pool */ + BNU_CHUNK_T* pHdata = cpEcGFpGetPool(1, pEC); + + int wvalue; + Ipp8u digit, sign; + int mask = (1<<(window_size+1)) -1; + int bit = scalarBitSize-(scalarBitSize%window_size); + + /* first window */ + if(bit) { + wvalue = *((Ipp16u*)&scalarA[(bit-1)/8]); + wvalue = (wvalue>> ((bit-1)%8)) & mask; + } + else + wvalue = 0; + booth_recode(&sign, &digit, (Ipp8u)wvalue, window_size); + gsScrambleGet_sscm(pTdata, pointLen, pTableA, digit-1, 5-1); + + if(bit) { + wvalue = *((Ipp16u*)&scalarB[(bit-1)/8]); + wvalue = (wvalue>> ((bit-1)%8)) & mask; + } + else + wvalue = 0; + booth_recode(&sign, &digit, (Ipp8u)wvalue, window_size); + gsScrambleGet_sscm(pHdata, pointLen, pTableB, digit-1, 5-1); + + gfec_point_add(pTdata, pTdata, pHdata, pEC); + + for(bit-=window_size; bit>=window_size; bit-=window_size) { + gfec_point_double(pTdata, pTdata, pEC); + gfec_point_double(pTdata, pTdata, pEC); + gfec_point_double(pTdata, pTdata, pEC); + gfec_point_double(pTdata, pTdata, pEC); + gfec_point_double(pTdata, pTdata, pEC); + + wvalue = *((Ipp16u*)&scalarA[(bit-1)/8]); + wvalue = (wvalue>> ((bit-1)%8)) & mask; + booth_recode(&sign, &digit, (Ipp8u)wvalue, window_size); + gsScrambleGet_sscm(pHdata, pointLen, pTableA, digit-1, 5-1); + + negF(pHy, pHdata+elemLen, pGFE); + cpMaskedReplace_ct(pHdata+elemLen, pHy, elemLen, ~cpIsZero_ct(sign)); + gfec_point_add(pTdata, pTdata, pHdata, pEC); + + wvalue = *((Ipp16u*)&scalarB[(bit-1)/8]); + wvalue = (wvalue>> ((bit-1)%8)) & mask; + booth_recode(&sign, &digit, (Ipp8u)wvalue, window_size); + gsScrambleGet_sscm(pHdata, pointLen, pTableB, digit-1, 5-1); + + negF(pHy, pHdata+elemLen, pGFE); + cpMaskedReplace_ct(pHdata+elemLen, pHy, elemLen, ~cpIsZero_ct(sign)); + gfec_point_add(pTdata, pTdata, pHdata, pEC); + } + /* last window */ + gfec_point_double(pTdata, pTdata, pEC); + gfec_point_double(pTdata, pTdata, pEC); + gfec_point_double(pTdata, pTdata, pEC); + gfec_point_double(pTdata, pTdata, pEC); + gfec_point_double(pTdata, pTdata, pEC); + + wvalue = *((Ipp16u*)&scalarA[0]); + wvalue = (wvalue << 1) & mask; + booth_recode(&sign, &digit, (Ipp8u)wvalue, window_size); + gsScrambleGet_sscm(pHdata, pointLen, pTableA, digit-1, 5-1); + + negF(pHy, pHdata+elemLen, pGFE); + cpMaskedReplace_ct(pHdata+elemLen, pHy, elemLen, ~cpIsZero_ct(sign)); + gfec_point_add(pTdata, pTdata, pHdata, pEC); + + wvalue = *((Ipp16u*)&scalarB[0]); + wvalue = (wvalue << 1) & mask; + booth_recode(&sign, &digit, (Ipp8u)wvalue, window_size); + gsScrambleGet_sscm(pHdata, pointLen, pTableB, digit-1, 5-1); + + negF(pHy, pHdata+elemLen, pGFE); + cpMaskedReplace_ct(pHdata+elemLen, pHy, elemLen, ~cpIsZero_ct(sign)); + gfec_point_add(pTdata, pTdata, pHdata, pEC); + + cpGFpElementCopy(pointR, pTdata, pointLen); + + cpEcGFpReleasePool(2, pEC); + cpGFpReleasePool(1, pGFE); + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_selectp192r1w7.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_selectp192r1w7.c new file mode 100644 index 0000000..ba56000 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpec_selectp192r1w7.c @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal EC over GF(p^m) basic Definitions & Function Prototypes +// +// Context: +// p192r1_select_ap_w7() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpmask_ct.h" + +/* +// select affine point +*/ +#if (_IPP32E < _IPP32E_M7) +IPP_OWN_DEFN (void, p192r1_select_ap_w7, (BNU_CHUNK_T* pVal, const BNU_CHUNK_T* pTbl, int idx)) +{ + #define OPERAND_BITSIZE (192) + #define LEN_P192 (BITS_BNU_CHUNK(OPERAND_BITSIZE)) + #define LEN_P192_APOINT (2*LEN_P192) + + const int tblLen = 64; + int i; + unsigned int n; + + /* clear output affine point */ + for(n=0; nidCtx +// invalid pP->idCtx +// invalid pQ->idCtx +// invalid pR->idCtx +// +// ippStsOutOfRangeErr ECP_POINT_FELEN(pP)!=GFP_FELEN() +// ECP_POINT_FELEN(pQ)!=GFP_FELEN() +// ECP_POINT_FELEN(pR)!=GFP_FELEN() +// +// ippStsNoErr no error +// +// Parameters: +// pP Pointer to the context of the first elliptic curve point +// pQ Pointer to the context of the second elliptic curve point +// pR Pointer to the context of the resulting elliptic curve point +// pEC Pointer to the context of the elliptic curve +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECAddPoint,(const IppsGFpECPoint* pP, const IppsGFpECPoint* pQ, IppsGFpECPoint* pR, + IppsGFpECState* pEC)) +{ + IPP_BAD_PTR4_RET(pP, pQ, pR, pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + IPP_BADARG_RET(!ECP_POINT_VALID_ID(pP), ippStsContextMatchErr); + IPP_BADARG_RET(!ECP_POINT_VALID_ID(pQ), ippStsContextMatchErr); + IPP_BADARG_RET(!ECP_POINT_VALID_ID(pR), ippStsContextMatchErr); + + IPP_BADARG_RET(ECP_POINT_FELEN(pP) != GFP_FELEN(GFP_PMA(ECP_GFP(pEC))), ippStsOutOfRangeErr); + IPP_BADARG_RET(ECP_POINT_FELEN(pQ) != GFP_FELEN(GFP_PMA(ECP_GFP(pEC))), ippStsOutOfRangeErr); + IPP_BADARG_RET(ECP_POINT_FELEN(pR) != GFP_FELEN(GFP_PMA(ECP_GFP(pEC))), ippStsOutOfRangeErr); + +#if (_IPP32E >= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512IFMA)) { + switch (ECP_MODULUS_ID(pEC)) { + case cpID_PrimeP256r1: { + gfec_AddPoint_nistp256_avx512(pR, pP, pQ, pEC); + return ippStsNoErr; + } + case cpID_PrimeP384r1: { + gfec_AddPoint_nistp384_avx512(pR, pP, pQ, pEC); + return ippStsNoErr; + } + case cpID_PrimeP521r1: { + gfec_AddPoint_nistp521_avx512(pR, pP, pQ, pEC); + return ippStsNoErr; + } + case cpID_PrimeTPM_SM2: { + gfec_AddPoint_sm2_avx512(pR, pP, pQ, pEC); + return ippStsNoErr; + } + default: + /* Go to default implementation below */ + break; + } + } /* no else */ +#endif // (_IPP32E >= _IPP32E_K1) + + if (pP == pQ) + gfec_DblPoint(pR, pP, pEC); + else + gfec_AddPoint(pR, pP, pQ, pEC); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecbindstd192r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecbindstd192r1.c new file mode 100644 index 0000000..88ed640 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecbindstd192r1.c @@ -0,0 +1,98 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECBindGxyTblStd192r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" + + + +static IppStatus cpGFpECBindGxyTbl(const BNU_CHUNK_T* pPrime, + const cpPrecompAP* preComp, + IppsGFpECState* pEC) +{ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + { + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + Ipp32u elemLen = (Ipp32u)GFP_FELEN(pGFE); + + /* test if GF is prime GF */ + IPP_BADARG_RET(!GFP_IS_BASIC(pGFE), ippStsBadArgErr); + /* test underlying prime value*/ + IPP_BADARG_RET(cpCmp_BNU(pPrime, (cpSize)elemLen, GFP_MODULUS(pGFE), (cpSize)elemLen), ippStsBadArgErr); + + { + BNU_CHUNK_T* pbp_ec = ECP_G(pEC); + int cmpFlag; + BNU_CHUNK_T* pbp_tbl = cpEcGFpGetPool(1, pEC); + + selectAP select_affine_point = preComp->select_affine_point; + const BNU_CHUNK_T* pTbl = preComp->pTbl; + select_affine_point(pbp_tbl, pTbl, 1); + + /* check if EC's and G-table's Base Point is the same */ + cmpFlag = cpCmp_BNU(pbp_ec, (cpSize)elemLen*2, pbp_tbl, (cpSize)elemLen*2); + + cpEcGFpReleasePool(1, pEC); + + return cmpFlag? ippStsBadArgErr : ippStsNoErr; + } + } +} + +/*F* +// Name: ippsGFpECBindGxyTblStd192r1 +// +// Purpose: Enables the use of base point-based pre-computed tables of EC192r1 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr invalid pEC->idCtx +// +// ippStsBadArgErr pEC is not EC192r1 +// +// ippStsNoErr no error +// +// Parameters: +// pEC Pointer to the context of the elliptic curve +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECBindGxyTblStd192r1,(IppsGFpECState* pEC)) +{ + IppStatus sts = cpGFpECBindGxyTbl(secp192r1_p, gfpec_precom_nistP192r1_fun(), pEC); + + /* setup pre-computed g-table and point access function */ + if(ippStsNoErr==sts) + ECP_PREMULBP(pEC) = gfpec_precom_nistP192r1_fun(); + + return sts; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecbindstd224r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecbindstd224r1.c new file mode 100644 index 0000000..0bc322a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecbindstd224r1.c @@ -0,0 +1,98 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECBindGxyTblStd224r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" + + + +static IppStatus cpGFpECBindGxyTbl(const BNU_CHUNK_T* pPrime, + const cpPrecompAP* preComp, + IppsGFpECState* pEC) +{ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + { + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + Ipp32u elemLen = (Ipp32u)GFP_FELEN(pGFE); + + /* test if GF is prime GF */ + IPP_BADARG_RET(!GFP_IS_BASIC(pGFE), ippStsBadArgErr); + /* test underlying prime value*/ + IPP_BADARG_RET(cpCmp_BNU(pPrime, (cpSize)elemLen, GFP_MODULUS(pGFE), (cpSize)elemLen), ippStsBadArgErr); + + { + BNU_CHUNK_T* pbp_ec = ECP_G(pEC); + int cmpFlag; + BNU_CHUNK_T* pbp_tbl = cpEcGFpGetPool(1, pEC); + + selectAP select_affine_point = preComp->select_affine_point; + const BNU_CHUNK_T* pTbl = preComp->pTbl; + select_affine_point(pbp_tbl, pTbl, 1); + + /* check if EC's and G-table's Base Point is the same */ + cmpFlag = cpCmp_BNU(pbp_ec, (cpSize)elemLen*2, pbp_tbl, (cpSize)elemLen*2); + + cpEcGFpReleasePool(1, pEC); + + return cmpFlag? ippStsBadArgErr : ippStsNoErr; + } + } +} + +/*F* +// Name: ippsGFpECBindGxyTblStd224r1 +// +// Purpose: Enables the use of base point-based pre-computed tables of EC224r1 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr invalid pEC->idCtx +// +// ippStsBadArgErr pEC is not EC224r1 +// +// ippStsNoErr no error +// +// Parameters: +// pEC Pointer to the context of the elliptic curve +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECBindGxyTblStd224r1,(IppsGFpECState* pEC)) +{ + IppStatus sts = cpGFpECBindGxyTbl(secp224r1_p, gfpec_precom_nistP224r1_fun(), pEC); + + /* setup pre-computed g-table and point access function */ + if(ippStsNoErr==sts) + ECP_PREMULBP(pEC) = gfpec_precom_nistP224r1_fun(); + + return sts; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecbindstd256r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecbindstd256r1.c new file mode 100644 index 0000000..0640964 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecbindstd256r1.c @@ -0,0 +1,110 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECBindGxyTblStd256r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" +#include "pcpgfpmethod.h" + + +static IppStatus cpGFpECBindGxyTbl(const BNU_CHUNK_T* pPrime, + const cpPrecompAP* preComp, + IppsGFpECState* pEC) +{ + IppsGFpState *pGF = ECP_GFP(pEC); + gsModEngine *pGFE = GFP_PMA(pGF); + Ipp32u elemLen = (Ipp32u)GFP_FELEN(pGFE); + + /* test if GF is prime GF */ + IPP_BADARG_RET(!GFP_IS_BASIC(pGFE), ippStsBadArgErr); + /* test underlying prime value*/ + IPP_BADARG_RET(cpCmp_BNU(pPrime, (cpSize)elemLen, GFP_MODULUS(pGFE), (cpSize)elemLen), ippStsBadArgErr); + + { + BNU_CHUNK_T *pbp_ec = ECP_G(pEC); + int cmpFlag; + BNU_CHUNK_T *pbp_tbl = cpEcGFpGetPool(1, pEC); + + selectAP select_affine_point = preComp->select_affine_point; + const BNU_CHUNK_T *pTbl = preComp->pTbl; + select_affine_point(pbp_tbl, pTbl, 1); + + /* check if EC's and G-table's Base Point is the same */ + cmpFlag = cpCmp_BNU(pbp_ec, (cpSize)elemLen * 2, pbp_tbl, (cpSize)elemLen * 2); + + cpEcGFpReleasePool(1, pEC); + + return cmpFlag ? ippStsBadArgErr : ippStsNoErr; + } +} + +/*F* +// Name: ippsGFpECBindGxyTblStd256r1 +// +// Purpose: Enables the use of base point-based pre-computed tables of EC256r1 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr invalid pEC->idCtx +// +// ippStsBadArgErr pEC is not EC256r1 +// +// ippStsNoErr no error +// +// Parameters: +// pEC Pointer to the context of the elliptic curve +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECBindGxyTblStd256r1, (IppsGFpECState * pEC)) +{ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + +#if (_IPP32E >= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512IFMA)) { + /* ModulusId as well as IFMA-based GF(p) method are assigned in + * ippsGFpECInitStd* function. If modulusId is not set (i.e. GFp method is + * not IFMA-based), then bind regular table (not for IFMA implementation). + */ + if (ECP_MODULUS_ID(pEC) == cpID_PrimeP256r1) { + ECP_PREMULBP(pEC) = gfpec_precom_nistP256r1_radix52_fun(); + return ippStsNoErr; + } + } +#endif + + const cpPrecompAP *precomp = gfpec_precom_nistP256r1_fun(); + IppStatus sts = cpGFpECBindGxyTbl(secp256r1_p, precomp, pEC); + + /* setup pre-computed g-table and point access function */ + if (ippStsNoErr == sts) + ECP_PREMULBP(pEC) = precomp; + + return sts; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecbindstd384r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecbindstd384r1.c new file mode 100644 index 0000000..d4d404c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecbindstd384r1.c @@ -0,0 +1,110 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECBindGxyTblStd384r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" +#include "pcpgfpmethod.h" + + +static IppStatus cpGFpECBindGxyTbl(const BNU_CHUNK_T* pPrime, + const cpPrecompAP* preComp, + IppsGFpECState* pEC) +{ + IppsGFpState *pGF = ECP_GFP(pEC); + gsModEngine *pGFE = GFP_PMA(pGF); + Ipp32u elemLen = (Ipp32u)GFP_FELEN(pGFE); + + /* test if GF is prime GF */ + IPP_BADARG_RET(!GFP_IS_BASIC(pGFE), ippStsBadArgErr); + /* test underlying prime value*/ + IPP_BADARG_RET(cpCmp_BNU(pPrime, (cpSize)elemLen, GFP_MODULUS(pGFE), (cpSize)elemLen), ippStsBadArgErr); + + { + BNU_CHUNK_T *pbp_ec = ECP_G(pEC); + int cmpFlag; + BNU_CHUNK_T *pbp_tbl = cpEcGFpGetPool(1, pEC); + + selectAP select_affine_point = preComp->select_affine_point; + const BNU_CHUNK_T *pTbl = preComp->pTbl; + select_affine_point(pbp_tbl, pTbl, 1); + + /* check if EC's and G-table's Base Point is the same */ + cmpFlag = cpCmp_BNU(pbp_ec, (cpSize)elemLen * 2, pbp_tbl, (cpSize)elemLen * 2); + + cpEcGFpReleasePool(1, pEC); + + return cmpFlag ? ippStsBadArgErr : ippStsNoErr; + } +} + +/*F* +// Name: ippsGFpECBindGxyTblStd384r1 +// +// Purpose: Enables the use of base point-based pre-computed tables of EC384r1 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr invalid pEC->idCtx +// +// ippStsBadArgErr pEC is not EC384r1 +// +// ippStsNoErr no error +// +// Parameters: +// pEC Pointer to the context of the elliptic curve +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECBindGxyTblStd384r1, (IppsGFpECState * pEC)) +{ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + +#if (_IPP32E >= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512IFMA)) { + /* ModulusId as well as IFMA-based GF(p) method are assigned in + * ippsGFpECInitStd* function. If modulusId is not set (i.e. GFp method is + * not IFMA-based), then bind regular table (not for IFMA implementation). + */ + if (ECP_MODULUS_ID(pEC) == cpID_PrimeP384r1) { + ECP_PREMULBP(pEC) = gfpec_precom_nistP384r1_radix52_fun(); + return ippStsNoErr; + } + } +#endif + + const cpPrecompAP *precomp = gfpec_precom_nistP384r1_fun(); + IppStatus sts = cpGFpECBindGxyTbl(secp384r1_p, precomp, pEC); + + /* setup pre-computed g-table and point access function */ + if (ippStsNoErr == sts) + ECP_PREMULBP(pEC) = precomp; + + return sts; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecbindstd521r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecbindstd521r1.c new file mode 100644 index 0000000..b8a8753 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecbindstd521r1.c @@ -0,0 +1,110 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECBindGxyTblStd521r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" +#include "pcpgfpmethod.h" + + +static IppStatus cpGFpECBindGxyTbl(const BNU_CHUNK_T* pPrime, + const cpPrecompAP* preComp, + IppsGFpECState* pEC) +{ + IppsGFpState *pGF = ECP_GFP(pEC); + gsModEngine *pGFE = GFP_PMA(pGF); + Ipp32u elemLen = (Ipp32u)GFP_FELEN(pGFE); + + /* test if GF is prime GF */ + IPP_BADARG_RET(!GFP_IS_BASIC(pGFE), ippStsBadArgErr); + /* test underlying prime value*/ + IPP_BADARG_RET(cpCmp_BNU(pPrime, (cpSize)elemLen, GFP_MODULUS(pGFE), (cpSize)elemLen), ippStsBadArgErr); + + { + BNU_CHUNK_T *pbp_ec = ECP_G(pEC); + int cmpFlag; + BNU_CHUNK_T *pbp_tbl = cpEcGFpGetPool(1, pEC); + + selectAP select_affine_point = preComp->select_affine_point; + const BNU_CHUNK_T *pTbl = preComp->pTbl; + select_affine_point(pbp_tbl, pTbl, 1); + + /* check if EC's and G-table's Base Point is the same */ + cmpFlag = cpCmp_BNU(pbp_ec, (cpSize)elemLen * 2, pbp_tbl, (cpSize)elemLen * 2); + + cpEcGFpReleasePool(1, pEC); + + return cmpFlag ? ippStsBadArgErr : ippStsNoErr; + } +} + +/*F* +// Name: ippsGFpECBindGxyTblStd521r1 +// +// Purpose: Enables the use of base point-based pre-computed tables of EC521r1 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr invalid pEC->idCtx +// +// ippStsBadArgErr pEC is not EC521r1 +// +// ippStsNoErr no error +// +// Parameters: +// pEC Pointer to the context of the elliptic curve +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECBindGxyTblStd521r1, (IppsGFpECState * pEC)) +{ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + +#if (_IPP32E >= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512IFMA)) { + /* ModulusId as well as IFMA-based GF(p) method are assigned in + * ippsGFpECInitStd* function. If modulusId is not set (i.e. GFp method is + * not IFMA-based), then bind regular table (not for IFMA implementation). + */ + if (ECP_MODULUS_ID(pEC) == cpID_PrimeP521r1) { + ECP_PREMULBP(pEC) = gfpec_precom_nistP521r1_radix52_fun(); + return ippStsNoErr; + } + } +#endif + + const cpPrecompAP *precomp = gfpec_precom_nistP521r1_fun(); + IppStatus sts = cpGFpECBindGxyTbl(secp521r1_p, precomp, pEC); + + /* setup pre-computed g-table and point access function */ + if (ippStsNoErr == sts) + ECP_PREMULBP(pEC) = precomp; + + return sts; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecbindstdsm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecbindstdsm2.c new file mode 100644 index 0000000..ad4322e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecbindstdsm2.c @@ -0,0 +1,109 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECBindGxyTblStdSM2() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" +#include "pcpgfpmethod.h" + + +static IppStatus cpGFpECBindGxyTbl(const BNU_CHUNK_T* pPrime, + const cpPrecompAP* preComp, + IppsGFpECState* pEC) +{ + IppsGFpState *pGF = ECP_GFP(pEC); + gsModEngine *pGFE = GFP_PMA(pGF); + Ipp32u elemLen = (Ipp32u)GFP_FELEN(pGFE); + + /* test if GF is prime GF */ + IPP_BADARG_RET(!GFP_IS_BASIC(pGFE), ippStsBadArgErr); + /* test underlying prime value*/ + IPP_BADARG_RET(cpCmp_BNU(pPrime, (cpSize)elemLen, GFP_MODULUS(pGFE), (cpSize)elemLen), ippStsBadArgErr); + + { + BNU_CHUNK_T *pbp_ec = ECP_G(pEC); + int cmpFlag; + BNU_CHUNK_T *pbp_tbl = cpEcGFpGetPool(1, pEC); + + selectAP select_affine_point = preComp->select_affine_point; + const BNU_CHUNK_T *pTbl = preComp->pTbl; + select_affine_point(pbp_tbl, pTbl, 1); + + /* check if EC's and G-table's Base Point is the same */ + cmpFlag = cpCmp_BNU(pbp_ec, (cpSize)elemLen * 2, pbp_tbl, (cpSize)elemLen * 2); + + cpEcGFpReleasePool(1, pEC); + + return cmpFlag ? ippStsBadArgErr : ippStsNoErr; + } +} + +/*F* +// Name: ippsGFpECBindGxyTblStdSM2 +// +// Purpose: Enables the use of base point-based pre-computed tables of ECSM2 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr invalid pEC->idCtx +// +// ippStsBadArgErr pEC is not ECSM2 +// +// ippStsNoErr no error +// +// Parameters: +// pEC Pointer to the context of the elliptic curve +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECBindGxyTblStdSM2,(IppsGFpECState* pEC)) +{ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + +#if (_IPP32E >= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512IFMA)) { + /* ModulusId as well as IFMA-based GF(p) method are assigned in + * ippsGFpECInitStd* function. If modulusId is not set (i.e. GFp method is + * not IFMA-based), then bind regular table (not for IFMA implementation). + */ + if (ECP_MODULUS_ID(pEC) == cpID_PrimeTPM_SM2) { + ECP_PREMULBP(pEC) = gfpec_precom_sm2_radix52_fun(); + return ippStsNoErr; + } + } +#endif + const cpPrecompAP* precomp = gfpec_precom_sm2_fun(); + IppStatus sts = cpGFpECBindGxyTbl(tpmSM2_p256_p, precomp, pEC); + + /* setup pre-computed g-table and point access function */ + if(ippStsNoErr == sts) + ECP_PREMULBP(pEC) = precomp; + + return sts; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecbufsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecbufsize.c new file mode 100644 index 0000000..b1dcc1a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecbufsize.c @@ -0,0 +1,71 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECScratchBufferSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsGFpScratchBufferSize +// +// Purpose: Gets the size of the scratch buffer. +// +// Returns: Reason: +// ippStsNullPtrErr pEC == NULL +// pBufferSize == NULL +// ippStsContextMatchErr invalid pEC->idCtx +// ippStsBadArgErr 0>=nScalars +// nScalars>6 +// ippStsNoErr no error +// +// Parameters: +// nScalars Number of scalar values. +// pEC Pointer to the context of the elliptic curve +// pBufferSize Pointer to the calculated buffer size in bytes. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECScratchBufferSize,(int nScalars, const IppsGFpECState* pEC, int* pBufferSize)) +{ + IPP_BAD_PTR2_RET(pEC, pBufferSize); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + + IPP_BADARG_RET( (0>=nScalars)||(nScalars>IPP_MAX_EXPONENT_NUM), ippStsBadArgErr); + + { + /* select constant size of window */ + const int w = 5; + /* number of table entries */ + const int nPrecomputed = 1<<(w-1); /* because of signed digit representation of scalar is uses */ + + int pointDataSize = ECP_POINTLEN(pEC)*(Ipp32s)sizeof(BNU_CHUNK_T); + + *pBufferSize = nScalars * pointDataSize*nPrecomputed + CACHE_LINE_SIZE; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpeccmppoint.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpeccmppoint.c new file mode 100644 index 0000000..d61bd3b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpeccmppoint.c @@ -0,0 +1,77 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECCmpPoint() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcphash.h" +#include "pcphash_rmf.h" + +/*F* +// Name: ippsGFpECCmpPoint +// +// Purpose: Compares two points +// +// Returns: Reason: +// ippStsNullPtrErr pP == NULL +// pQ == NULL +// pEC == NULL +// pResult == NULL +// +// ippStsContextMatchErr invalid pEC->idCtx +// invalid pP->idCtx +// invalid pQ->idCtx +// +// ippStsOutOfRangeErr ECP_POINT_FELEN(pP)!=GFP_FELEN() +// ECP_POINT_FELEN(pQ)!=GFP_FELEN() +// +// ippStsNoErr no error +// +// Parameters: +// pP Pointer to the context of the first elliptic curve point +// pQ Pointer to the context of the second elliptic curve point +// pEC Pointer to the context of the elliptic curve +// pResult Pointer to the result of the comparison +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECCmpPoint,(const IppsGFpECPoint* pP, const IppsGFpECPoint* pQ, + IppECResult* pResult, + IppsGFpECState* pEC)) +{ + IPP_BAD_PTR4_RET(pP, pQ, pResult, pEC); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pP), ippStsContextMatchErr ); + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pQ), ippStsContextMatchErr ); + + IPP_BADARG_RET( ECP_POINT_FELEN(pP)!=GFP_FELEN(GFP_PMA(ECP_GFP(pEC))), ippStsOutOfRangeErr); + IPP_BADARG_RET( ECP_POINT_FELEN(pQ)!=GFP_FELEN(GFP_PMA(ECP_GFP(pEC))), ippStsOutOfRangeErr); + + *pResult = gfec_ComparePoint(pP, pQ, pEC)? ippECPointIsEqual : ippECPointIsNotEqual; + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpeccpypoint.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpeccpypoint.c new file mode 100644 index 0000000..437973a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpeccpypoint.c @@ -0,0 +1,75 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECCpyPoint() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcphash.h" +#include "pcphash_rmf.h" + +/*F* +// Name: ippsGFpECCpyPoint +// +// Purpose: Copies one point to another +// +// Returns: Reason: +// ippStsNullPtrErr pA == NULL +// pR == NULL +// pEC == NULL +// +// ippStsContextMatchErr invalid pEC->idCtx +// invalid pA->idCtx +// invalid pR->idCtx +// +// ippStsOutOfRangeErr ECP_POINT_FELEN(pA)!=GFP_FELEN() +// ECP_POINT_FELEN(pR)!=GFP_FELEN() +// +// ippStsNoErr no error +// +// Parameters: +// pA Pointer to the context of the elliptic curve point being copied +// pR Pointer to the context of the elliptic curve point being changed +// pEC Pointer to the context of the elliptic curve +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECCpyPoint,(const IppsGFpECPoint* pA, + IppsGFpECPoint* pR, + IppsGFpECState* pEC)) +{ + IPP_BAD_PTR3_RET(pA, pR, pEC); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pA), ippStsContextMatchErr ); + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pR), ippStsContextMatchErr ); + + IPP_BADARG_RET( ECP_POINT_FELEN(pA)!=GFP_FELEN(GFP_PMA(ECP_GFP(pEC))), ippStsOutOfRangeErr); + IPP_BADARG_RET( ECP_POINT_FELEN(pR)!=GFP_FELEN(GFP_PMA(ECP_GFP(pEC))), ippStsOutOfRangeErr); + + gfec_CopyPoint(pR, pA, GFP_FELEN(GFP_PMA(ECP_GFP(pEC)))); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecdh.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecdh.c new file mode 100644 index 0000000..3c1c6a0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecdh.c @@ -0,0 +1,184 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// ippsGFpECSharedSecretDH() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" + +/*F* +// Name: ippsGFpECSharedSecretDHC +// +// Purpose: Compute Shared Secret (Diffie-Hellman) +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pPrivateA +// NULL == pPublicB +// NULL == pShare +// +// ippStsContextMatchErr illegal pEC->idCtx +// pEC->subgroup == NULL +// illegal pPrivateA->idCtx +// illegal pPublicB->idCtx +// illegal pShare->idCtx +// +// ippStsRangeErr not enough room for share key +// +// ippStsShareKeyErr (infinity) => z +// +// ippStsNoErr no errors +// +// Parameters: +// pPrivateA pointer to own private key +// pPublicB pointer to alien public key +// pShare pointer to the shared secret value +// pEC pointer to the EC context +// pScratchBuffer pointer to the scratch buffer +// +*F*/ +IPPFUN(IppStatus, ippsGFpECSharedSecretDH,(const IppsBigNumState* pPrivateA, const IppsGFpECPoint* pPublicB, + IppsBigNumState* pShare, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +{ + IppsGFpState *pGF; + gsModEngine *pGFE; + + /* EC context and buffer */ + IPP_BAD_PTR2_RET(pEC, pScratchBuffer); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + IPP_BADARG_RET(!ECP_SUBGROUP(pEC), ippStsContextMatchErr); + + pGF = ECP_GFP(pEC); + pGFE = GFP_PMA(pGF); + + /* test private (own) key */ + IPP_BAD_PTR1_RET(pPrivateA); + IPP_BADARG_RET(!BN_VALID_ID(pPrivateA), ippStsContextMatchErr); + /* test if 0 < pPrivateA < Order */ + IPP_BADARG_RET(0==gfec_CheckPrivateKey(pPrivateA, pEC), ippStsInvalidPrivateKey); + + /* test public (other party) key */ + IPP_BAD_PTR1_RET(pPublicB); + IPP_BADARG_RET(!ECP_POINT_VALID_ID(pPublicB), ippStsContextMatchErr); + /* test if pPublicB belongs EC */ + IPP_BADARG_RET(0==gfec_IsPointOnCurve(pPublicB, pEC), ippStsInvalidPoint); + + /* test share secret value */ + IPP_BAD_PTR1_RET(pShare); + IPP_BADARG_RET(!BN_VALID_ID(pShare), ippStsContextMatchErr); + IPP_BADARG_RET((BN_ROOM(pShare) < GFP_FELEN(pGFE)), ippStsRangeErr); + + { + /* init tmp Point */ + IppsGFpECPoint T; + cpEcGFpInitPoint(/* pPoint = */ &T, + /* pData = */ cpEcGFpGetPool(1, pEC), + /* flags = */ 0, + /* pEC = */ pEC); + + int finite_point = 0; + const int elmLen = GFP_FELEN(pGFE); + /* share data */ + BNU_CHUNK_T *pShareData = BN_NUMBER(pShare); + int nsShare = BN_ROOM(pShare); + +#if (_IPP32E >= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512IFMA)) { + switch (ECP_MODULUS_ID(pEC)) { + case cpID_PrimeP256r1: { + finite_point = gfec_SharedSecretDH_nistp256_avx512(&T, pPublicB, BN_NUMBER(pPrivateA), BN_SIZE(pPrivateA), pEC, pScratchBuffer); + if (finite_point) { + cpGFpElementCopy(pShareData, ECP_POINT_X(&T), elmLen); + cpGFpElementPad(pShareData + elmLen, nsShare - elmLen, 0); + } + goto exit; + break; + } + case cpID_PrimeP384r1: { + finite_point = gfec_SharedSecretDH_nistp384_avx512(&T, pPublicB, BN_NUMBER(pPrivateA), BN_SIZE(pPrivateA), pEC, pScratchBuffer); + if (finite_point) { + cpGFpElementCopy(pShareData, ECP_POINT_X(&T), elmLen); + cpGFpElementPad(pShareData + elmLen, nsShare - elmLen, 0); + } + goto exit; + break; + } + case cpID_PrimeP521r1: { + finite_point = gfec_SharedSecretDH_nistp521_avx512(&T, pPublicB, BN_NUMBER(pPrivateA), BN_SIZE(pPrivateA), pEC, pScratchBuffer); + if (finite_point) { + cpGFpElementCopy(pShareData, ECP_POINT_X(&T), elmLen); + cpGFpElementPad(pShareData + elmLen, nsShare - elmLen, 0); + } + goto exit; + break; + } + case cpID_PrimeTPM_SM2: { + finite_point = gfec_SharedSecretDH_sm2_avx512(&T, pPublicB, BN_NUMBER(pPrivateA), BN_SIZE(pPrivateA), pEC, pScratchBuffer); + if (finite_point) { + cpGFpElementCopy(pShareData, ECP_POINT_X(&T), elmLen); + cpGFpElementPad(pShareData + elmLen, nsShare - elmLen, 0); + } + goto exit; + break; + } + default: + /* Go to default implementation below */ + break; + } + } /* no else */ +#endif // (_IPP32E >= _IPP32E_K1) + { + /* T = [privateA]pPublicB */ + gfec_MulPoint(&T, pPublicB, BN_NUMBER(pPrivateA), BN_SIZE(pPrivateA), /*ECP_ORDBITSIZE(pEC),*/ pEC, pScratchBuffer); + /* share = T.x */ + IppsGFpElement elm; + /* Buffer by GFpElement - get in Pool data */ + cpGFpElementConstruct(&elm, cpGFpGetPool(1, pGFE), elmLen); + + finite_point = gfec_GetPoint(GFPE_DATA(&elm), NULL, &T, pEC); + + /* check finit point */ + if (finite_point) { + /* share = decode(T.x) */ + GFP_METHOD(pGFE)->decode(pShareData, GFPE_DATA(&elm), pGFE); + cpGFpElementPad(pShareData + elmLen, nsShare - elmLen, 0); + } + + cpGFpReleasePool(1, pGFE); /* GFpElement */ + } + +#if (_IPP32E >= _IPP32E_K1) +exit: +#endif + + if (finite_point) { + BN_SIGN(pShare) = ippBigNumPOS; + FIX_BNU(pShareData, nsShare); + BN_SIZE(pShare) = nsShare; + } + cpEcGFpReleasePool(1, pEC); /* ECPoint */ + return finite_point ? ippStsNoErr : ippStsShareKeyErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecdhc.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecdhc.c new file mode 100644 index 0000000..8bff02e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecdhc.c @@ -0,0 +1,141 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// ippsGFpECSharedSecretDHC() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" + + +/*F* +// Name: ippsGFpECSharedSecretDHC +// +// Purpose: Compute Shared Secret (Diffie-Hellman with cofactor) +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pPrivateA +// NULL == pPublicB +// NULL == pShare +// +// ippStsContextMatchErr illegal pEC->idCtx +// pEC->subgroup == NULL +// illegal pPrivateA->idCtx +// illegal pPublicB->idCtx +// illegal pShare->idCtx +// +// ippStsRangeErr not enough room for share key +// +// ippStsShareKeyErr (infinity) => z +// +// ippStsNoErr no errors +// +// Parameters: +// pPrivateA pointer to own private key +// pPublicB pointer to alien public key +// pShare pointer to the shared secret value +// pEC pointer to the EC context +// +*F*/ +IPPFUN(IppStatus, ippsGFpECSharedSecretDHC,(const IppsBigNumState* pPrivateA, const IppsGFpECPoint* pPublicB, + IppsBigNumState* pShare, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +{ + IppsGFpState* pGF; + gsModEngine* pGFE; + + /* EC context and buffer */ + IPP_BAD_PTR2_RET(pEC, pScratchBuffer); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + IPP_BADARG_RET(!ECP_SUBGROUP(pEC), ippStsContextMatchErr); + + pGF = ECP_GFP(pEC); + pGFE = GFP_PMA(pGF); + + /* test private (own) key */ + IPP_BAD_PTR1_RET(pPrivateA); + IPP_BADARG_RET(!BN_VALID_ID(pPrivateA), ippStsContextMatchErr); + /* test if 0 < pPrivateA < Order */ + IPP_BADARG_RET(0 == gfec_CheckPrivateKey(pPrivateA, pEC), ippStsInvalidPrivateKey); + + /* test public (other party) key */ + IPP_BAD_PTR1_RET(pPublicB); + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pPublicB), ippStsContextMatchErr ); + /* test if pPublicB belongs EC */ + IPP_BADARG_RET(0 == gfec_IsPointOnCurve(pPublicB, pEC), ippStsInvalidPoint); + + /* test share key */ + IPP_BAD_PTR1_RET(pShare); + IPP_BADARG_RET(!BN_VALID_ID(pShare), ippStsContextMatchErr); + IPP_BADARG_RET((BN_ROOM(pShare)decode(pShareData, GFPE_DATA(&elm), pGFE); + cpGFpElementPad(pShareData+elmLen, nsShare-elmLen, 0); + + BN_SIGN(pShare) = ippBigNumPOS; + FIX_BNU(pShareData, nsShare); + BN_SIZE(pShare) = nsShare; + } + + cpGFpReleasePool(2, pGFE); + cpEcGFpReleasePool(1, pEC); + + return finite_point? ippStsNoErr : ippStsShareKeyErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecesdecryptsm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecesdecryptsm2.c new file mode 100644 index 0000000..053268c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecesdecryptsm2.c @@ -0,0 +1,66 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECESDecrypt_SM2() +// +*/ + +#include "pcpgfpecessm2.h" +#include "pcpgfpecstuff.h" + +/*F* +// Name: ippsGFpECESDecrypt_SM2 +// +// Purpose: Decrypts the given buffer, updates the auth tag +// +// Returns: Reason: +// ippStsNullPtrErr pInput == NULL / pOutput == NULL / pState == NULL +// ippStsContextMatchErr pState invalid context or the algorithm is in an invalid state +// ippStsSizeErr dataLen < 0 +// ippStsNoErr no errors +// +// Parameters: +// pInput Pointer to input data +// pOutput Pointer to output data +// dataLen Size of input and output buffers +// pState Pointer to a SM2 algorithm state +// +*F*/ +IPPFUN(IppStatus, ippsGFpECESDecrypt_SM2, (const Ipp8u* pInput, Ipp8u* pOutput, int dataLen, IppsECESState_SM2* pState)) { + IPP_BAD_PTR3_RET(pInput, pOutput, pState); + IPP_BADARG_RET(!VALID_ECES_SM2_ID(pState), ippStsContextMatchErr); + /* a shared secret should be computed and the process should not be finished by getTag */ + IPP_BADARG_RET(pState->state != ECESAlgoProcessing, ippStsIncompleteContextErr); + IPP_BADARG_RET(dataLen < 0, ippStsSizeErr); + + { + int i; + for (i = 0; i < dataLen; ++i) { + pOutput[i] = pInput[i] ^ cpECES_SM2KdfNextByte(pState); + } + } + ippsHashUpdate_rmf(pOutput, dataLen, pState->pTagHasher); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecesencryptsm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecesencryptsm2.c new file mode 100644 index 0000000..42a0f01 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecesencryptsm2.c @@ -0,0 +1,66 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECESEncrypt_SM2() +// +*/ + +#include "pcpgfpecessm2.h" +#include "pcpgfpecstuff.h" + +/*F* +// Name: ippsGFpECESEncrypt_SM2 +// +// Purpose: Encrypts the given buffer, updates the auth tag +// +// Returns: Reason: +// ippStsNullPtrErr pInput == NULL / pOutput == NULL / pState == NULL +// ippStsContextMatchErr pState invalid context or the algorithm is in an invalid state +// ippStsSizeErr dataLen < 0 +// ippStsNoErr no errors +// +// Parameters: +// pInput Pointer to input data +// pOutput Pointer to output data +// dataLen Size of input and output buffers +// pState Pointer to a SM2 algorithm state +// +*F*/ +IPPFUN(IppStatus, ippsGFpECESEncrypt_SM2, (const Ipp8u* pInput, Ipp8u* pOutput, int dataLen, IppsECESState_SM2* pState)) { + IPP_BAD_PTR3_RET(pInput, pOutput, pState); + IPP_BADARG_RET(!VALID_ECES_SM2_ID(pState), ippStsContextMatchErr); + /* a shared secret should be computed and the process should not be finished by getTag */ + IPP_BADARG_RET(pState->state != ECESAlgoProcessing, ippStsIncompleteContextErr); + IPP_BADARG_RET(dataLen < 0, ippStsSizeErr); + + ippsHashUpdate_rmf(pInput, dataLen, pState->pTagHasher); + { + int i; + for (i = 0; i < dataLen; ++i) { + pOutput[i] = pInput[i] ^ cpECES_SM2KdfNextByte(pState); + } + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecesfinalsm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecesfinalsm2.c new file mode 100644 index 0000000..a6a5598 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecesfinalsm2.c @@ -0,0 +1,73 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECESFinal_SM2() +// +*/ + +#include "pcpgfpecessm2.h" +#include "pcpgfpecstuff.h" + +/*F* +// Name: ippsGFpECESFinal_SM2 +// +// Purpose: Ends SM2 algorithm chain and returns an auth tag +// +// Returns: Reason: +// ippStsNullPtrErr pTag == NULL / pState == NULL +// ippStsContextMatchErr pState invalid context or the algorithm is in an invalid state +// ippStsSizeErr tagLen < 0 || tagLen > IPP_SM3_DIGEST_BITSIZE / BYTESIZE +// ippStsShareKeyErr All the kdf provided bytes were 0. The operation is completed successfully, but the warning is given +// ippStsNoErr no errors +// +// Parameters: +// pTag Pointer to a tag buffer to write to +// tagLen Size of the tag [0; IPP_SM3_DIGEST_BITSIZE / BYTESIZE] +// pState Pointer to a SM2 algorithm state +// +*F*/ +IPPFUN(IppStatus, ippsGFpECESFinal_SM2, (Ipp8u* pTag, int tagLen, IppsECESState_SM2* pState)) { + IPP_BAD_PTR2_RET(pTag, pState); + IPP_BADARG_RET(!VALID_ECES_SM2_ID(pState), ippStsContextMatchErr); + /* a shared secret should be computed and the process should not be finished by getTag */ + IPP_BADARG_RET(pState->state != ECESAlgoProcessing, ippStsIncompleteContextErr); + IPP_BADARG_RET(tagLen < 0 || tagLen > IPP_SM3_DIGEST_BITSIZE / BYTESIZE, ippStsSizeErr); + + ippsHashUpdate_rmf(pState->pSharedSecret + pState->sharedSecretLen / 2, pState->sharedSecretLen / 2, pState->pTagHasher); + if (tagLen == IPP_SM3_DIGEST_BITSIZE / BYTESIZE) { + ippsHashFinal_rmf(pTag, pState->pTagHasher); + } else { + Ipp8u pFinal[IPP_SM3_DIGEST_BITSIZE / BYTESIZE]; + int i; + ippsHashFinal_rmf(pFinal, pState->pTagHasher); + for (i = 0; i < tagLen; ++i) { + pTag[i] = pFinal[i]; + } + } + + pState->state = ECESAlgoFinished; /* cannot proceed futher due to closing ippsSM3Update */ + + /* do the operation, but return an error code in 0-case */ + return pState->wasNonZero ? ippStsNoErr : ippStsShareKeyErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecesgetbufferssizesm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecesgetbufferssizesm2.c new file mode 100644 index 0000000..bd0f0ee --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecesgetbufferssizesm2.c @@ -0,0 +1,61 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECESGetBuffersSize_SM2() +// +*/ + +#include "pcpgfpecessm2.h" +#include "pcpgfpecstuff.h" + +/*F* +// Name: ippsGFpECESGetBuffersSize_SM2 +// +// Purpose: Returns sizes of used buffers +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL if pPublicKeySize != NULL or all the pointers are NULLs +// ippStsContextMatchErr pState invalid context if pPublicKeySize != NULL +// ippStsNoErr no errors +// +// Parameters: +// pPublicKeySize Pointer to write public (x||y) key length in bytes +// pMaximumTagSize Pointer to write maximum tag size in bytes +// pState Pointer to a state to get pPublicKeySize from +// +*F*/ +IPPFUN(IppStatus, ippsGFpECESGetBuffersSize_SM2, (int* pPublicKeySize, + int* pMaximumTagSize, const IppsECESState_SM2* pState)) { + IPP_BADARG_RET(pPublicKeySize == NULL && pMaximumTagSize == NULL && pState == NULL, ippStsNullPtrErr); + + if (pMaximumTagSize) + *pMaximumTagSize = IPP_SM3_DIGEST_BITSIZE / BYTESIZE; + if (pPublicKeySize) { + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!VALID_ECES_SM2_ID(pState), ippStsContextMatchErr); + *pPublicKeySize = pState->sharedSecretLen; + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecesgetsizesm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecesgetsizesm2.c new file mode 100644 index 0000000..1eee866 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecesgetsizesm2.c @@ -0,0 +1,62 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECESGetSize_SM2() +// +*/ + +#include "pcpgfpecessm2.h" +#include "pcpgfpecstuff.h" + +/*F* +// Name: ippsGFpECESGetSize_SM2 +// +// Purpose: Computes space required to allocate a SM2 algorithm state +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL / pEC == NULL +// ippStsContextMatchErr pEC invalid context +// ippStsNotSupportedModeErr pGFE->extdegree > 1 +// ippStsNoErr no errors +// +// Parameters: +// pEC Pointer to an EC to calculate a shared secret size +// pSize Pointer to write a SM2 algorithm state size +// +*F*/ +IPPFUN(IppStatus, ippsGFpECESGetSize_SM2, (const IppsGFpECState* pEC, int* pSize)) { + IPP_BAD_PTR2_RET(pEC, pSize); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + IPP_BADARG_RET(!pEC->subgroup, ippStsContextMatchErr); + IPP_BADARG_RET(1 < pEC->pGF->pGFE->extdegree, ippStsNotSupportedModeErr); + + { + int sm3size; + ippsHashGetSize_rmf(&sm3size); + + *pSize = (Ipp32s)sizeof(IppsECESState_SM2) + sm3size * 2 + BITS2WORD8_SIZE(pEC->pGF->pGFE->modBitLen) * 2; + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecesinitsm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecesinitsm2.c new file mode 100644 index 0000000..fffea84 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecesinitsm2.c @@ -0,0 +1,78 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECESInit_SM2() +// +*/ + +#include "pcpgfpecessm2.h" +#include "pcpgfpecstuff.h" + +/*F* +// Name: ippsGFpECESInit_SM2 +// +// Purpose: Inits the SM2 algorithm state +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL / pEC == NULL +// ippStsContextMatchErr pEC, pPoint invalid context +// ippStsNotSupportedModeErr pGFE->extdegree > 1 +// ippStsSizeErr size of the provided state is less than needed +// ippStsNoErr no errors +// +// Parameters: +// pEC Pointer to an EC to calculate a shared secret size +// pState Pointer to a SM2 algorithm state buffer +// avaliableCtxSize Count of avaliable bytes in the context allocation +// +*F*/ +IPPFUN(IppStatus, ippsGFpECESInit_SM2, (IppsGFpECState* pEC, IppsECESState_SM2* pState, int avaliableCtxSize)) { + IPP_BAD_PTR2_RET(pEC, pState); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + IPP_BADARG_RET(!pEC->subgroup, ippStsContextMatchErr); + IPP_BADARG_RET(1 < pEC->pGF->pGFE->extdegree, ippStsNotSupportedModeErr); + + { + int realCtxSize; + ippsGFpECESGetSize_SM2(pEC, &realCtxSize); + IPP_BADARG_RET(avaliableCtxSize < realCtxSize, ippStsSizeErr); + + { + int sm3size; + ippsHashGetSize_rmf(&sm3size); + + ECES_SM2_SET_ID(pState); + pState->sharedSecretLen = BITS2WORD8_SIZE(pEC->pGF->pGFE->modBitLen) * 2; + pState->pSharedSecret = ((Ipp8u*)pState) + sizeof(IppsECESState_SM2); + pState->pKdfHasher = (IppsHashState_rmf*)(((Ipp8u*)pState) + sizeof(IppsECESState_SM2) + pState->sharedSecretLen); + pState->pTagHasher = (IppsHashState_rmf*)(((Ipp8u*)pState) + sizeof(IppsECESState_SM2) + pState->sharedSecretLen + sm3size); + + ippsHashInit_rmf(pState->pKdfHasher, ippsHashMethod_SM3()); + + pState->state = ECESAlgoInit; + + return ippStsNoErr; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecessetkeysm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecessetkeysm2.c new file mode 100644 index 0000000..3f50051 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecessetkeysm2.c @@ -0,0 +1,98 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECESSetKey_SM2() +// +*/ + +#include "pcpgfpecessm2.h" +#include "pcpgfpecstuff.h" + +/*F* +// Name: ippsGFpECESSetKey_SM2 +// +// Purpose: Resets all counters, computes shared secret and saves it into the SM2 algorithm state +// +// Returns: Reason: +// ippStsNullPtrErr pPrivate == NULL / pPublic == NULL / pState == NULL / pEC == NULL +// ippStsContextMatchErr the algorithm is in an invalid state or any of the specified contexts does not match the operation +// ippStsOutOfRangeErr private key does not belong to the EC's finite field or public key/result does not belong to EC +// ippStsNotSupportedModeErr pGFE->extdegree > 1 +// ippStsBadArgErr curve element size is not the same as in init / pPrivate is negative / pPrivate > pEC GFp mod +// ippStsPointAtInfinity shared secret is a point at infinity +// ippStsNoErr no errors +// +// Parameters: +// pEC Pointer to an EC to calculate a shared secret size +// pState Pointer to a SM2 algorithm state buffer +// pPrivate Pointer to a private component of shared secret +// pPublic Pointer to a public component +// pEcScratchBuffer Pointer to a scratch buffer for computations on the EC +// +*F*/ +IPPFUN(IppStatus, ippsGFpECESSetKey_SM2, (const IppsBigNumState* pPrivate, + const IppsGFpECPoint* pPublic, IppsECESState_SM2* pState, + IppsGFpECState* pEC, Ipp8u* pEcScratchBuffer)) { + IPP_BAD_PTR4_RET(pPrivate, pPublic, pState, pEC); + IPP_BADARG_RET(!VALID_ECES_SM2_ID(pState), ippStsContextMatchErr); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + IPP_BADARG_RET(!pEC->subgroup, ippStsContextMatchErr); + IPP_BADARG_RET(1 < pEC->pGF->pGFE->extdegree, ippStsNotSupportedModeErr); + + { + gsModEngine* pGFE = pEC->pGF->pGFE; + /* curve element size is not the same */ + IPP_BADARG_RET(BITS2WORD8_SIZE(pGFE->modBitLen) * 2 != pState->sharedSecretLen, ippStsBadArgErr); + + { + IppStatus multResult; + IppsGFpECPoint PT; + IppsGFpElement ptX, ptY; + int finitePoint = 0; + + cpEcGFpInitPoint(&PT, cpEcGFpGetPool(1, pEC), 0, pEC); + multResult = ippsGFpECMulPoint(pPublic, pPrivate, &PT, pEC, pEcScratchBuffer); + if (ippStsNoErr == multResult) { + cpGFpElementConstruct(&ptX, cpGFpGetPool(1, pGFE), pGFE->modLen); + cpGFpElementConstruct(&ptY, cpGFpGetPool(1, pGFE), pGFE->modLen); + finitePoint = gfec_GetPoint(ptX.pData, ptY.pData, &PT, pEC); + if (finitePoint) { + ippsGFpGetElementOctString(&ptX, pState->pSharedSecret, pState->sharedSecretLen / 2, pEC->pGF); + ippsGFpGetElementOctString(&ptY, pState->pSharedSecret + pState->sharedSecretLen / 2, pState->sharedSecretLen / 2, pEC->pGF); + + pState->kdfCounter = 0; + pState->kdfIndex = IPP_SM3_DIGEST_BITSIZE / BYTESIZE; /* will generate a kdf window */ + pState->wasNonZero = 0; + pState->state = ECESAlgoKeySet; + } + cpGFpReleasePool(2, pGFE); /* release ptX and ptY from the pool */ + } + cpEcGFpReleasePool(1, pEC); /* release PT from the pool */ + + if (multResult) + return multResult; + return finitePoint ? ippStsNoErr : ippStsPointAtInfinity; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecessm2.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecessm2.h new file mode 100644 index 0000000..e45b9ae --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecessm2.h @@ -0,0 +1,77 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// ES encryption/decryption API +// +// +*/ + +#if !defined(_CP_GFP_ES_SM2_H) +#define _CP_GFP_ES_SM2_H + +#include "owncp.h" + +typedef enum { + ECESAlgoInit, + ECESAlgoKeySet, + ECESAlgoProcessing, + ECESAlgoFinished +} ECESAlgoState; + +struct _cpStateECES_SM2 { + Ipp32u idCtx; + Ipp8u* pSharedSecret; + Ipp32s sharedSecretLen; + + ECESAlgoState state; + + Ipp32u kdfCounter; + Ipp8u pKdfWindow[IPP_SM3_DIGEST_BITSIZE / BYTESIZE]; + Ipp8u wasNonZero; + Ipp8u kdfIndex; + + IppsHashState_rmf* pKdfHasher; + IppsHashState_rmf* pTagHasher; +}; + +#define ECES_SM2_SET_ID(stt) ((stt)->idCtx = (Ipp32u)idxCtxECES_SM2 ^ (Ipp32u)IPP_UINT_PTR(stt)) +#define VALID_ECES_SM2_ID(stt) ((((stt)->idCtx) ^ (Ipp32u)IPP_UINT_PTR((stt))) == (Ipp32u)idxCtxECES_SM2) + +/* get a byte, update 0-kdf status */ +__INLINE Ipp8u cpECES_SM2KdfNextByte(IppsECESState_SM2* pState) { + if (pState->kdfIndex == IPP_SM3_DIGEST_BITSIZE / BYTESIZE) { + ++pState->kdfCounter; + pState->kdfIndex = 0; + + { + Ipp8u ctnStr[sizeof(Ipp32u)]; + ippsHashUpdate_rmf(pState->pSharedSecret, pState->sharedSecretLen, pState->pKdfHasher); + U32_TO_HSTRING(ctnStr, pState->kdfCounter); + ippsHashUpdate_rmf(ctnStr, sizeof(Ipp32u), pState->pKdfHasher); + ippsHashFinal_rmf(pState->pKdfWindow, pState->pKdfHasher); + } + } + + pState->wasNonZero |= pState->pKdfWindow[pState->kdfIndex]; + + return pState->pKdfWindow[pState->kdfIndex++]; +} + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecesstartsm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecesstartsm2.c new file mode 100644 index 0000000..27b7ac1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecesstartsm2.c @@ -0,0 +1,56 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECESStart_SM2() +// +*/ + +#include "pcpgfpecessm2.h" + +/*F* +// Name: ippsGFpECESStart_SM2 +// +// Purpose: Starts an SM2 encryption chain. +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL +// ippStsContextMatchErr pState invalid context or the algorithm is in an invalid state +// ippStsNoErr no errors +// +// Parameters: +// pState Pointer to a SM2 algorithm state +// +*F*/ +IPPFUN(IppStatus, ippsGFpECESStart_SM2, (IppsECESState_SM2* pState)) { + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!VALID_ECES_SM2_ID(pState), ippStsContextMatchErr); + IPP_BADARG_RET(pState->state != ECESAlgoKeySet, ippStsContextMatchErr); + + ippsHashInit_rmf(pState->pTagHasher, ippsHashMethod_SM3()); + ippsHashUpdate_rmf(pState->pSharedSecret, pState->sharedSecretLen / 2, pState->pTagHasher); + + pState->state = ECESAlgoProcessing; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecget.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecget.c new file mode 100644 index 0000000..1359fb6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecget.c @@ -0,0 +1,86 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECGet() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsGFpECGet +// +// Purpose: Extracts the parameters of an elliptic curve +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr invalid pEC->idCtx +// invalid pA->idCtx +// invalid pB->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM(pA)!=GFP_FELEN(pGFE) +// GFPE_ROOM(pB)!=GFP_FELEN(pGFE) +// +// ippStsNoErr no error +// +// Parameters: +// ppGFp Pointer to the pointer to the context of underlying finite field +// pA Pointer to a copy of the coefficient A of the equation defining the elliptic curve +// pB Pointer to a copy of the coefficient B of the equation defining the elliptic curve +// pEC Pointer to the context of the elliptic curve +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECGet,(IppsGFpState** const ppGFp, + IppsGFpElement* pA, IppsGFpElement* pB, + const IppsGFpECState* pEC)) +{ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + + { + const IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + Ipp32u elementSize = (Ipp32u)GFP_FELEN(pGFE); + + if(ppGFp) { + *ppGFp = (IppsGFpState*)pGF; + } + + if(pA) { + IPP_BADARG_RET( !GFPE_VALID_ID(pA), ippStsContextMatchErr ); + IPP_BADARG_RET( GFPE_ROOM(pA)!=GFP_FELEN(pGFE), ippStsOutOfRangeErr); + cpGFpElementCopy(GFPE_DATA(pA), ECP_A(pEC), (cpSize)elementSize); + } + if(pB) { + IPP_BADARG_RET( !GFPE_VALID_ID(pB), ippStsContextMatchErr ); + IPP_BADARG_RET( GFPE_ROOM(pB)!=GFP_FELEN(pGFE), ippStsOutOfRangeErr); + cpGFpElementCopy(GFPE_DATA(pB), ECP_B(pEC), (cpSize)elementSize); + } + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecgetinfo.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecgetinfo.c new file mode 100644 index 0000000..3678c49 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecgetinfo.c @@ -0,0 +1,67 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECGetInfo_GF() +// +*/ + +#include "owncp.h" +#include "pcpeccp.h" + + +/*F* +// Name: ippsGFpECGetInfo_GF +// +// Purpose: Returns info regarding underlying GF +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pInfo +// +// ippStsContextMatchErr invalid pEC->idCtx +// +// ippStsNoErr no error +// +// Parameters: +// pInfo Pointer to the info structure +// pEC Pointer to the context of the elliptic curve being initialized. +// +*F*/ +IPPFUN(IppStatus, ippsGFpECGetInfo_GF,(IppsGFpInfo* pInfo, const IppsGFpECState* pEC)) +{ + IPP_BAD_PTR2_RET(pInfo, pEC); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + + return ippsGFpGetInfo(pInfo, ECP_GFP(pEC)); + #if 0 + { + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFpx = GFP_PMA(pGF); /* current */ + gsModEngine* pGFp = cpGFpBasic(pGFpx); /* basic */ + pInfo->parentGFdegree = MOD_EXTDEG(pGFpx); /* parent extension */ + pInfo->basicGFdegree = cpGFpBasicDegreeExtension(pGFpx); /* total basic extention */ + pInfo->basicElmBitSize = GFP_FEBITLEN(pGFp); /* basic bitsise */ + + return ippStsNoErr; + } + #endif +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecgetpoint.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecgetpoint.c new file mode 100644 index 0000000..56772fb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecgetpoint.c @@ -0,0 +1,85 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECGetPoint() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcphash.h" +#include "pcphash_rmf.h" + +/*F* +// Name: ippsGFpECGetPoint +// +// Purpose: Retrieves coordinates of a point on an elliptic curve +// +// Returns: Reason: +// ippStsNullPtrErr pPoint == NULL +// pEC == NULL +// +// ippStsContextMatchErr invalid pEC->idCtx +// invalid pPoint->idCtx +// pX != NULL && invalid pX->idCtx +// pY != NULL && invalid pY->idCtx +// pX != NULL && GFPE_ROOM(pX)!=GFP_FELEN() +// pY != NULL && GFPE_ROOM(pY)!=GFP_FELEN() +// +// ippStsOutOfRangeErr ECP_POINT_FELEN(pPoint)!=GFP_FELEN() +// pX != NULL && GFPE_ROOM(pX)!=GFP_FELEN() +// pY != NULL && GFPE_ROOM(pY)!=GFP_FELEN() +// +// ippStsNoErr no error +// +// Parameters: +// pPoint Pointer to the IppsGFpECPoint context +// pEC Pointer to the context of the elliptic curve +// pX, pY Pointers to the X and Y coordinates of a point on the elliptic curve +// +// Note: +// Is not a fact that computed point belongs to BP-related subgroup BP +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECGetPoint,(const IppsGFpECPoint* pPoint, + IppsGFpElement* pX, IppsGFpElement* pY, + IppsGFpECState* pEC)) +{ + IPP_BAD_PTR2_RET(pPoint, pEC); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pPoint), ippStsContextMatchErr ); + + IPP_BADARG_RET( pX && !GFPE_VALID_ID(pX), ippStsContextMatchErr ); + IPP_BADARG_RET( pY && !GFPE_VALID_ID(pY), ippStsContextMatchErr ); + + IPP_BADARG_RET( pX && GFPE_ROOM(pX)!=GFP_FELEN(GFP_PMA(ECP_GFP(pEC))), ippStsOutOfRangeErr); + IPP_BADARG_RET( pY && GFPE_ROOM(pY)!=GFP_FELEN(GFP_PMA(ECP_GFP(pEC))), ippStsOutOfRangeErr); + IPP_BADARG_RET( ECP_POINT_FELEN(pPoint)!=GFP_FELEN(GFP_PMA(ECP_GFP(pEC))), ippStsOutOfRangeErr); + + /* returns (X,Y) == (0,0) if Point is at infinity */ + gfec_GetPoint((pX)? GFPE_DATA(pX):NULL, (pY)? GFPE_DATA(pY):NULL, pPoint, pEC); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecgetpointoctstring.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecgetpointoctstring.c new file mode 100644 index 0000000..37b103b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecgetpointoctstring.c @@ -0,0 +1,89 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECGetPointOctString() +// +*/ +#include "pcpgfpecessm2.h" +#include "pcpgfpecstuff.h" + +/*F* +// Name: ippsGFpECGetPointOctString +// +// Purpose: Converts a point on EC into x||y octstring +// +// Returns: Reason: +// ippStsNullPtrErr pPoint == NULL / pEC == NULL / pStr == NULL +// ippStsContextMatchErr pEC, pPoint invalid context +// ippStsNotSupportedModeErr pGFE->extdegree > 1 +// ippStsSizeErr strLen is not equal to double GFp element +// ippStsOutOfRangeErr the point does not belong to the EC +// ippStsPointAtInfinity a point on infinity cannot be converted to a string +// ippStsNoErr no errors +// +// Parameters: +// pStr pointer to the string to read from +// strLen length of the string +// pPoint pointer to output point +// pEC EC ctx +// +*F*/ +IPPFUN(IppStatus, ippsGFpECGetPointOctString, (const IppsGFpECPoint* pPoint, + Ipp8u* pStr, int strLen, IppsGFpECState* pEC)) { + IPP_BAD_PTR3_RET(pPoint, pEC, pStr); + IPP_BADARG_RET(!ECP_POINT_VALID_ID(pPoint), ippStsContextMatchErr); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + { + gsModEngine* pGFE = pEC->pGF->pGFE; + IppsGFpInfo gfi; + ippsGFpGetInfo(&gfi, pEC->pGF); + + { + int elemLenBits = gfi.basicGFdegree * gfi.basicElmBitSize; + int elemLenBytes = BITS2WORD8_SIZE(elemLenBits); + int elemLenChunks = BITS_BNU_CHUNK(elemLenBits); + IPP_BADARG_RET(strLen != elemLenBytes * 2, ippStsSizeErr); + IPP_BADARG_RET(pPoint->elementSize != elemLenChunks, ippStsOutOfRangeErr); + + { + int finitePoint; + IppsGFpElement ptX, ptY; + + cpGFpElementConstruct(&ptX, cpGFpGetPool(1, pGFE), elemLenChunks); + cpGFpElementConstruct(&ptY, cpGFpGetPool(1, pGFE), elemLenChunks); + finitePoint = gfec_GetPoint(ptX.pData, ptY.pData, pPoint, pEC); + if (finitePoint) { + ippsGFpGetElementOctString(&ptX, pStr, elemLenBytes, pEC->pGF); + pStr += elemLenBytes; + ippsGFpGetElementOctString(&ptY, pStr, elemLenBytes, pEC->pGF); + } + + cpGFpReleasePool(2, pGFE); /* release ptX and ptY from the pool */ + + return finitePoint ? ippStsNoErr : ippStsPointAtInfinity; + } + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecgetpointreg.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecgetpointreg.c new file mode 100644 index 0000000..8dad83b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecgetpointreg.c @@ -0,0 +1,107 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECGetPointRegular() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcphash.h" +#include "pcphash_rmf.h" + +/*F* +// Name: ippsGFpECGetPointRegular +// +// Purpose: Retrieves coordinates of a point on an elliptic curve in the regular domain +// +// Returns: Reason: +// ippStsNullPtrErr pPoint == NULL +// pEC == NULL +// +// +// ippStsContextMatchErr invalid pEC->idCtx +// invalid pPoint->idCtx +// invalid pX->idCtx +// invalid pY->idCtx +// +// ippStsOutOfRangeErr ECP_POINT_FELEN(pPoint)!=GFP_FELEN() +// BN_ROOM(pX)*BNU_CHUNK_BITSdecode(x, x, pGFE); + ippsSet_BN(ippBigNumPOS, GFP_FELEN32(pGFE), (Ipp32u*)x, pX); + } + if(pY) { + GFP_METHOD(pGFE)->decode(y, y, pGFE); + ippsSet_BN(ippBigNumPOS, GFP_FELEN32(pGFE), (Ipp32u*)y, pY); + } + + cpGFpReleasePool(2, pGFE); + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecgetsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecgetsize.c new file mode 100644 index 0000000..3b00f1c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecgetsize.c @@ -0,0 +1,61 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECGetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsGFpECGetSize +// +// Purpose: Gets the size of an elliptic curve over the finite field +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFp +// NULL == pSize +// +// ippStsContextMatchErr invalid pGFp->idCtx +// +// ippStsNoErr no error +// +// Parameters: +// pGFp Pointer to the IppsGFpState context of the underlying finite field +// pSize Buffer size in bytes needed for the IppsGFpECState context +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECGetSize,(const IppsGFpState* pGFp, int* pSize)) +{ + IPP_BAD_PTR2_RET(pGFp, pSize); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + + { + gsModEngine* pGFE = GFP_PMA(pGFp); + *pSize = cpGFpECGetSize(cpGFpBasicDegreeExtension(pGFE), GFP_FEBITLEN(cpGFpBasic(pGFE))); + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecgetsubgroup.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecgetsubgroup.c new file mode 100644 index 0000000..d6ba814 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecgetsubgroup.c @@ -0,0 +1,121 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECGetSubgroup() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsGFpECGet +// +// Purpose: Extracts the parameters (base point and its order) of an elliptic curve +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// +// ippStsContextMatchErr invalid pEC->idCtx +// NULL == pEC->subgroup +// invalid pX->idCtx +// invalid pY->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM(pX)!=GFP_FELEN(pGFE) +// GFPE_ROOM(pY)!=GFP_FELEN(pGFE) +// +// ippStsLengthErr BN_ROOM(pOrder) < orderLen +// BN_ROOM(pCofactor) < cofactorLen +// +// ippStsNoErr no error +// +// Parameters: +// ppGFp Pointer to the pointer to the context of underlying finite field +// pX, pY Pointers to the X and Y coordinates of the base point of the elliptic curve +// pOrder Pointer to the big number context storing the order of the base point. +// pCofactor Pointer to the big number context storing the cofactor. +// pEC Pointer to the context of the elliptic curve. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECGetSubgroup,(IppsGFpState** const ppGFp, + IppsGFpElement* pX, IppsGFpElement* pY, + IppsBigNumState* pOrder, + IppsBigNumState* pCofactor, + const IppsGFpECState* pEC)) +{ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + IPP_BADARG_RET(!ECP_SUBGROUP(pEC), ippStsContextMatchErr); + + { + const IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + Ipp32u elementSize = (Ipp32u)GFP_FELEN(pGFE); + + if(ppGFp) { + *ppGFp = (IppsGFpState*)pGF; + } + + if(pX) { + IPP_BADARG_RET( !GFPE_VALID_ID(pX), ippStsContextMatchErr ); + IPP_BADARG_RET( GFPE_ROOM(pX)!=GFP_FELEN(pGFE), ippStsOutOfRangeErr); + cpGFpElementCopy(GFPE_DATA(pX), ECP_G(pEC), (cpSize)elementSize); + } + if(pY) { + IPP_BADARG_RET( !GFPE_VALID_ID(pY), ippStsContextMatchErr ); + IPP_BADARG_RET( GFPE_ROOM(pY)!=GFP_FELEN(pGFE), ippStsOutOfRangeErr); + cpGFpElementCopy(GFPE_DATA(pY), ECP_G(pEC)+elementSize, (cpSize)elementSize); + } + + if(pOrder) { + BNU_CHUNK_T* pOrderData = MOD_MODULUS(ECP_MONT_R(pEC)); + int orderBitSize = ECP_ORDBITSIZE(pEC); + int orderLen = BITS_BNU_CHUNK(orderBitSize); + FIX_BNU(pOrderData, orderLen); + + IPP_BADARG_RET(!BN_VALID_ID(pOrder), ippStsContextMatchErr); + IPP_BADARG_RET(BN_ROOM(pOrder) < orderLen, ippStsLengthErr); + + ZEXPAND_COPY_BNU(BN_NUMBER(pOrder), BN_ROOM(pOrder), pOrderData, orderLen); + BN_SIZE(pOrder) = orderLen; + BN_SIGN(pOrder) = ippBigNumPOS; + } + + if(pCofactor) { + BNU_CHUNK_T* pCofactorData = ECP_COFACTOR(pEC); + int cofactorLen = (cpSize)elementSize; + FIX_BNU(pCofactorData, cofactorLen); + + IPP_BADARG_RET(!BN_VALID_ID(pCofactor), ippStsContextMatchErr); + IPP_BADARG_RET(BN_ROOM(pCofactor) < cofactorLen, ippStsLengthErr); + + ZEXPAND_COPY_BNU(BN_NUMBER(pCofactor), BN_ROOM(pCofactor), pCofactorData, cofactorLen); + BN_SIZE(pCofactor) = cofactorLen; + BN_SIGN(pCofactor) = ippBigNumPOS; + } + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinit.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinit.c new file mode 100644 index 0000000..75a1098 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinit.c @@ -0,0 +1,122 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECInit() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" +#include "pcpgfpmethod.h" + +/*F* +// Name: ippsGFpECInit +// +// Purpose: Initializes the context of an elliptic curve over a finite field. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pA +// NULL == pB +// +// ippStsContextMatchErr invalid pEC->idCtx +// invalid pA->idCtx +// invalid pB->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM(pA)!=GFP_FELEN(pGFE) +// GFPE_ROOM(pB)!=GFP_FELEN(pGFE) +// +// ippStsNoErr no error +// +// Parameters: +// pGFp Pointer to the IppsGFpState context of the underlying finite field +// pA Pointer to the coefficient A of the equation defining the elliptic curve +// pB Pointer to the coefficient B of the equation defining the elliptic curve +// pEC Pointer to the context of the elliptic curve being initialized +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECInit,(const IppsGFpState* pGFp, + const IppsGFpElement* pA, const IppsGFpElement* pB, + IppsGFpECState* pEC)) +{ + IPP_BAD_PTR2_RET(pGFp, pEC); + + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + + { + Ipp8u* ptr = (Ipp8u*)pEC; + + gsModEngine* pGFE = GFP_PMA(pGFp); + int elemLen = GFP_FELEN(pGFE); + + int maxOrderBits = 1+ cpGFpBasicDegreeExtension(pGFE) * GFP_FEBITLEN(cpGFpBasic(pGFE)); /* Hasse's theorem */ + #if defined(_LEGACY_ECCP_SUPPORT_) + int maxOrdLen = BITS_BNU_CHUNK(maxOrderBits); + #endif + + int modEngineCtxSize; + gsModEngineGetSize(maxOrderBits, MONT_DEFAULT_POOL_LENGTH, &modEngineCtxSize); + + ECP_SET_ID(pEC); + ECP_MODULUS_ID(pEC) = cpID_Prime; + ECP_GFP(pEC) = (IppsGFpState*)pGFp; + ECP_SUBGROUP(pEC) = 0; + ECP_POINTLEN(pEC) = elemLen*3; + ECP_ORDBITSIZE(pEC) = maxOrderBits; + ECP_SPECIFIC(pEC) = ECP_ARB; + + ptr += sizeof(IppsGFpECState); + ECP_A(pEC) = (BNU_CHUNK_T*)(ptr); ptr += elemLen*(Ipp32s)sizeof(BNU_CHUNK_T); + ECP_B(pEC) = (BNU_CHUNK_T*)(ptr); ptr += elemLen*(Ipp32s)sizeof(BNU_CHUNK_T); + ECP_G(pEC) = (BNU_CHUNK_T*)(ptr); ptr += ECP_POINTLEN(pEC)*(Ipp32s)sizeof(BNU_CHUNK_T); + ECP_PREMULBP(pEC) = (cpPrecompAP*)NULL; + ECP_MONT_R(pEC) = (gsModEngine*)(ptr); ptr += modEngineCtxSize; + ECP_COFACTOR(pEC) = (BNU_CHUNK_T*)(ptr); ptr += elemLen*(Ipp32s)sizeof(BNU_CHUNK_T); + #if defined(_LEGACY_ECCP_SUPPORT_) + ECP_PUBLIC(pEC) = (BNU_CHUNK_T*)(ptr); ptr += 3*elemLen*(Ipp32s)sizeof(BNU_CHUNK_T); + ECP_PUBLIC_E(pEC) = (BNU_CHUNK_T*)(ptr); ptr += 3*elemLen*(Ipp32s)sizeof(BNU_CHUNK_T); + ECP_PRIVAT(pEC) = (BNU_CHUNK_T*)(ptr); ptr += maxOrdLen*(Ipp32s)sizeof(BNU_CHUNK_T); + ECP_PRIVAT_E(pEC) = (BNU_CHUNK_T*)(ptr); ptr += maxOrdLen*(Ipp32s)sizeof(BNU_CHUNK_T); + ECP_SBUFFER(pEC) = (BNU_CHUNK_T*)0; + #endif + ECP_POOL(pEC) = (BNU_CHUNK_T*)(ptr); //ptr += ECP_POINTLEN(pEC)*sizeof(BNU_CHUNK_T)*EC_POOL_SIZE; + + cpGFpElementPad(ECP_A(pEC), elemLen, 0); + cpGFpElementPad(ECP_B(pEC), elemLen, 0); + cpGFpElementPad(ECP_G(pEC), elemLen*3, 0); + //gsModEngineInit(ECP_MONT_R(pEC), NULL, maxOrderBits, MONT_DEFAULT_POOL_LENGTH, gsModArithMont()); + gsModEngineInit(ECP_MONT_R(pEC), NULL, maxOrderBits, MONT_DEFAULT_POOL_LENGTH, NULL); + + cpGFpElementPad(ECP_COFACTOR(pEC), elemLen, 0); + + cpGFpElementPad(ECP_POOL(pEC), elemLen*3*EC_POOL_SIZE, 0); + + /* set up EC if possible */ + if(pA && pB) + return ippsGFpECSet(pA,pB, pEC); + else + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstd128r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstd128r1.c new file mode 100644 index 0000000..c0038eb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstd128r1.c @@ -0,0 +1,117 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECInitStd128r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" + + + + +static void cpGFpECSetStd(int aLen, const BNU_CHUNK_T* pA, + int bLen, const BNU_CHUNK_T* pB, + int xLen, const BNU_CHUNK_T* pX, + int yLen, const BNU_CHUNK_T* pY, + int rLen, const BNU_CHUNK_T* pR, + BNU_CHUNK_T h, + IppsGFpECState* pEC) +{ + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elemLen = GFP_FELEN(pGFE); + + IppsGFpElement elmA, elmB; + __ALIGN8 IppsBigNumState R; + __ALIGN8 IppsBigNumState H; + + /* convert A ans B coeffs into GF elements */ + cpGFpElementConstruct(&elmA, cpGFpGetPool(1, pGFE), elemLen); + cpGFpElementConstruct(&elmB, cpGFpGetPool(1, pGFE), elemLen); + ippsGFpSetElement((Ipp32u*)pA, BITS2WORD32_SIZE(BITSIZE_BNU(pA,aLen)), &elmA, pGF); + ippsGFpSetElement((Ipp32u*)pB, BITS2WORD32_SIZE(BITSIZE_BNU(pB,bLen)), &elmB, pGF); + /* and set EC */ + ippsGFpECSet(&elmA, &elmB, pEC); + + /* construct R and H */ + cpConstructBN(&R, rLen, (BNU_CHUNK_T*)pR, NULL); + cpConstructBN(&H, 1, &h, NULL); + /* convert GX ans GY coeffs into GF elements */ + ippsGFpSetElement((Ipp32u*)pX, BITS2WORD32_SIZE(BITSIZE_BNU(pX,xLen)), &elmA, pGF); + ippsGFpSetElement((Ipp32u*)pY, BITS2WORD32_SIZE(BITSIZE_BNU(pY,yLen)), &elmB, pGF); + /* and init EC subgroup */ + ippsGFpECSetSubgroup(&elmA, &elmB, &R, &H, pEC); + cpGFpReleasePool(2, pGFE); +} + +/*F* +// Name: ippsGFpECInitStd128r1 +// +// Purpose: Initializes the context of EC128r1 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pGFp +// +// ippStsContextMatchErr invalid pGFp->idCtx +// +// ippStsBadArgErr pGFp does not specify the finite field over which the given +// standard elliptic curve is defined +// +// ippStsNoErr no error +// +// Parameters: +// pGFp Pointer to the IppsGFpState context of the underlying finite field +// pEC Pointer to the context of the elliptic curve being initialized. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECInitStd128r1,(const IppsGFpState* pGFp, IppsGFpECState* pEC)) +{ + IPP_BAD_PTR2_RET(pGFp, pEC); + + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + + { + gsModEngine* pGFE = GFP_PMA(pGFp); + + /* test if GF is prime GF */ + IPP_BADARG_RET(!GFP_IS_BASIC(pGFE), ippStsBadArgErr); + /* test underlying prime value*/ + IPP_BADARG_RET(cpCmp_BNU(secp128r1_p, BITS_BNU_CHUNK(128), GFP_MODULUS(pGFE), BITS_BNU_CHUNK(128)), ippStsBadArgErr); + + ippsGFpECInit(pGFp, NULL, NULL, pEC); + cpGFpECSetStd(BITS_BNU_CHUNK(128), secp128r1_a, + BITS_BNU_CHUNK(128), secp128r1_b, + BITS_BNU_CHUNK(128), secp128r1_gx, + BITS_BNU_CHUNK(128), secp128r1_gy, + BITS_BNU_CHUNK(128), secp128r1_r, + secp128r1_h, + pEC); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstd128r2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstd128r2.c new file mode 100644 index 0000000..ee0cd58 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstd128r2.c @@ -0,0 +1,117 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECInitStd128r2() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" + + + + +static void cpGFpECSetStd(int aLen, const BNU_CHUNK_T* pA, + int bLen, const BNU_CHUNK_T* pB, + int xLen, const BNU_CHUNK_T* pX, + int yLen, const BNU_CHUNK_T* pY, + int rLen, const BNU_CHUNK_T* pR, + BNU_CHUNK_T h, + IppsGFpECState* pEC) +{ + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elemLen = GFP_FELEN(pGFE); + + IppsGFpElement elmA, elmB; + __ALIGN8 IppsBigNumState R; + __ALIGN8 IppsBigNumState H; + + /* convert A ans B coeffs into GF elements */ + cpGFpElementConstruct(&elmA, cpGFpGetPool(1, pGFE), elemLen); + cpGFpElementConstruct(&elmB, cpGFpGetPool(1, pGFE), elemLen); + ippsGFpSetElement((Ipp32u*)pA, BITS2WORD32_SIZE(BITSIZE_BNU(pA,aLen)), &elmA, pGF); + ippsGFpSetElement((Ipp32u*)pB, BITS2WORD32_SIZE(BITSIZE_BNU(pB,bLen)), &elmB, pGF); + /* and set EC */ + ippsGFpECSet(&elmA, &elmB, pEC); + + /* construct R and H */ + cpConstructBN(&R, rLen, (BNU_CHUNK_T*)pR, NULL); + cpConstructBN(&H, 1, &h, NULL); + /* convert GX ans GY coeffs into GF elements */ + ippsGFpSetElement((Ipp32u*)pX, BITS2WORD32_SIZE(BITSIZE_BNU(pX,xLen)), &elmA, pGF); + ippsGFpSetElement((Ipp32u*)pY, BITS2WORD32_SIZE(BITSIZE_BNU(pY,yLen)), &elmB, pGF); + /* and init EC subgroup */ + ippsGFpECSetSubgroup(&elmA, &elmB, &R, &H, pEC); + cpGFpReleasePool(2, pGFE); +} + +/*F* +// Name: ippsGFpECInitStd128r2 +// +// Purpose: Initializes the context of EC128r2 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pGFp +// +// ippStsContextMatchErr invalid pGFp->idCtx +// +// ippStsBadArgErr pGFp does not specify the finite field over which the given +// standard elliptic curve is defined +// +// ippStsNoErr no error +// +// Parameters: +// pGFp Pointer to the IppsGFpState context of the underlying finite field +// pEC Pointer to the context of the elliptic curve being initialized. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECInitStd128r2,(const IppsGFpState* pGFp, IppsGFpECState* pEC)) +{ + IPP_BAD_PTR2_RET(pGFp, pEC); + + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + + { + gsModEngine* pGFE = GFP_PMA(pGFp); + + /* test if GF is prime GF */ + IPP_BADARG_RET(!GFP_IS_BASIC(pGFE), ippStsBadArgErr); + /* test underlying prime value*/ + IPP_BADARG_RET(cpCmp_BNU(secp128r2_p, BITS_BNU_CHUNK(128), GFP_MODULUS(pGFE), BITS_BNU_CHUNK(128)), ippStsBadArgErr); + + ippsGFpECInit(pGFp, NULL, NULL, pEC); + cpGFpECSetStd(BITS_BNU_CHUNK(128), secp128r2_a, + BITS_BNU_CHUNK(128), secp128r2_b, + BITS_BNU_CHUNK(128), secp128r2_gx, + BITS_BNU_CHUNK(128), secp128r2_gy, + BITS_BNU_CHUNK(128), secp128r2_r, + secp128r2_h, + pEC); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstd192r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstd192r1.c new file mode 100644 index 0000000..0862517 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstd192r1.c @@ -0,0 +1,117 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECInitStd192r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" + + + + +static void cpGFpECSetStd(int aLen, const BNU_CHUNK_T* pA, + int bLen, const BNU_CHUNK_T* pB, + int xLen, const BNU_CHUNK_T* pX, + int yLen, const BNU_CHUNK_T* pY, + int rLen, const BNU_CHUNK_T* pR, + BNU_CHUNK_T h, + IppsGFpECState* pEC) +{ + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elemLen = GFP_FELEN(pGFE); + + IppsGFpElement elmA, elmB; + __ALIGN8 IppsBigNumState R; + __ALIGN8 IppsBigNumState H; + + /* convert A ans B coeffs into GF elements */ + cpGFpElementConstruct(&elmA, cpGFpGetPool(1, pGFE), elemLen); + cpGFpElementConstruct(&elmB, cpGFpGetPool(1, pGFE), elemLen); + ippsGFpSetElement((Ipp32u*)pA, BITS2WORD32_SIZE(BITSIZE_BNU(pA,aLen)), &elmA, pGF); + ippsGFpSetElement((Ipp32u*)pB, BITS2WORD32_SIZE(BITSIZE_BNU(pB,bLen)), &elmB, pGF); + /* and set EC */ + ippsGFpECSet(&elmA, &elmB, pEC); + + /* construct R and H */ + cpConstructBN(&R, rLen, (BNU_CHUNK_T*)pR, NULL); + cpConstructBN(&H, 1, &h, NULL); + /* convert GX ans GY coeffs into GF elements */ + ippsGFpSetElement((Ipp32u*)pX, BITS2WORD32_SIZE(BITSIZE_BNU(pX,xLen)), &elmA, pGF); + ippsGFpSetElement((Ipp32u*)pY, BITS2WORD32_SIZE(BITSIZE_BNU(pY,yLen)), &elmB, pGF); + /* and init EC subgroup */ + ippsGFpECSetSubgroup(&elmA, &elmB, &R, &H, pEC); + cpGFpReleasePool(2, pGFE); +} + +/*F* +// Name: ippsGFpECInitStd192r1 +// +// Purpose: Initializes the context of EC192r1 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pGFp +// +// ippStsContextMatchErr invalid pGFp->idCtx +// +// ippStsBadArgErr pGFp does not specify the finite field over which the given +// standard elliptic curve is defined +// +// ippStsNoErr no error +// +// Parameters: +// pGFp Pointer to the IppsGFpState context of the underlying finite field +// pEC Pointer to the context of the elliptic curve being initialized. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECInitStd192r1,(const IppsGFpState* pGFp, IppsGFpECState* pEC)) +{ + IPP_BAD_PTR2_RET(pGFp, pEC); + + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + + { + gsModEngine* pGFE = GFP_PMA(pGFp); + + /* test if GF is prime GF */ + IPP_BADARG_RET(!GFP_IS_BASIC(pGFE), ippStsBadArgErr); + /* test underlying prime value*/ + IPP_BADARG_RET(cpCmp_BNU(secp192r1_p, BITS_BNU_CHUNK(192), GFP_MODULUS(pGFE), BITS_BNU_CHUNK(192)), ippStsBadArgErr); + + ippsGFpECInit(pGFp, NULL, NULL, pEC); + cpGFpECSetStd(BITS_BNU_CHUNK(192), secp192r1_a, + BITS_BNU_CHUNK(192), secp192r1_b, + BITS_BNU_CHUNK(192), secp192r1_gx, + BITS_BNU_CHUNK(192), secp192r1_gy, + BITS_BNU_CHUNK(192), secp192r1_r, + secp192r1_h, + pEC); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstd224r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstd224r1.c new file mode 100644 index 0000000..46e2750 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstd224r1.c @@ -0,0 +1,117 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECInitStd224r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" + + + + +static void cpGFpECSetStd(int aLen, const BNU_CHUNK_T* pA, + int bLen, const BNU_CHUNK_T* pB, + int xLen, const BNU_CHUNK_T* pX, + int yLen, const BNU_CHUNK_T* pY, + int rLen, const BNU_CHUNK_T* pR, + BNU_CHUNK_T h, + IppsGFpECState* pEC) +{ + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elemLen = GFP_FELEN(pGFE); + + IppsGFpElement elmA, elmB; + __ALIGN8 IppsBigNumState R; + __ALIGN8 IppsBigNumState H; + + /* convert A ans B coeffs into GF elements */ + cpGFpElementConstruct(&elmA, cpGFpGetPool(1, pGFE), elemLen); + cpGFpElementConstruct(&elmB, cpGFpGetPool(1, pGFE), elemLen); + ippsGFpSetElement((Ipp32u*)pA, BITS2WORD32_SIZE(BITSIZE_BNU(pA,aLen)), &elmA, pGF); + ippsGFpSetElement((Ipp32u*)pB, BITS2WORD32_SIZE(BITSIZE_BNU(pB,bLen)), &elmB, pGF); + /* and set EC */ + ippsGFpECSet(&elmA, &elmB, pEC); + + /* construct R and H */ + cpConstructBN(&R, rLen, (BNU_CHUNK_T*)pR, NULL); + cpConstructBN(&H, 1, &h, NULL); + /* convert GX ans GY coeffs into GF elements */ + ippsGFpSetElement((Ipp32u*)pX, BITS2WORD32_SIZE(BITSIZE_BNU(pX,xLen)), &elmA, pGF); + ippsGFpSetElement((Ipp32u*)pY, BITS2WORD32_SIZE(BITSIZE_BNU(pY,yLen)), &elmB, pGF); + /* and init EC subgroup */ + ippsGFpECSetSubgroup(&elmA, &elmB, &R, &H, pEC); + cpGFpReleasePool(2, pGFE); +} + +/*F* +// Name: ippsGFpECInitStd224r1 +// +// Purpose: Initializes the context of EC224r1 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pGFp +// +// ippStsContextMatchErr invalid pGFp->idCtx +// +// ippStsBadArgErr pGFp does not specify the finite field over which the given +// standard elliptic curve is defined +// +// ippStsNoErr no error +// +// Parameters: +// pGFp Pointer to the IppsGFpState context of the underlying finite field +// pEC Pointer to the context of the elliptic curve being initialized. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECInitStd224r1,(const IppsGFpState* pGFp, IppsGFpECState* pEC)) +{ + IPP_BAD_PTR2_RET(pGFp, pEC); + + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + + { + gsModEngine* pGFE = GFP_PMA(pGFp); + + /* test if GF is prime GF */ + IPP_BADARG_RET(!GFP_IS_BASIC(pGFE), ippStsBadArgErr); + /* test underlying prime value*/ + IPP_BADARG_RET(cpCmp_BNU(secp224r1_p, BITS_BNU_CHUNK(224), GFP_MODULUS(pGFE), BITS_BNU_CHUNK(224)), ippStsBadArgErr); + + ippsGFpECInit(pGFp, NULL, NULL, pEC); + cpGFpECSetStd(BITS_BNU_CHUNK(224), secp224r1_a, + BITS_BNU_CHUNK(224), secp224r1_b, + BITS_BNU_CHUNK(224), secp224r1_gx, + BITS_BNU_CHUNK(224), secp224r1_gy, + BITS_BNU_CHUNK(224), secp224r1_r, + secp224r1_h, + pEC); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstd256r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstd256r1.c new file mode 100644 index 0000000..a895425 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstd256r1.c @@ -0,0 +1,126 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECInitStd256r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" + +#include "ecnist/ifma_arith_method.h" +#include "pcpgfpmethod.h" + + +static void cpGFpECSetStd(int aLen, const BNU_CHUNK_T* pA, + int bLen, const BNU_CHUNK_T* pB, + int xLen, const BNU_CHUNK_T* pX, + int yLen, const BNU_CHUNK_T* pY, + int rLen, const BNU_CHUNK_T* pR, + BNU_CHUNK_T h, + IppsGFpECState* pEC) +{ + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elemLen = GFP_FELEN(pGFE); + + IppsGFpElement elmA, elmB; + __ALIGN8 IppsBigNumState R; + __ALIGN8 IppsBigNumState H; + + /* convert A ans B coeffs into GF elements */ + cpGFpElementConstruct(&elmA, cpGFpGetPool(1, pGFE), elemLen); + cpGFpElementConstruct(&elmB, cpGFpGetPool(1, pGFE), elemLen); + ippsGFpSetElement((Ipp32u*)pA, BITS2WORD32_SIZE(BITSIZE_BNU(pA,aLen)), &elmA, pGF); + ippsGFpSetElement((Ipp32u*)pB, BITS2WORD32_SIZE(BITSIZE_BNU(pB,bLen)), &elmB, pGF); + /* and set EC */ + ippsGFpECSet(&elmA, &elmB, pEC); + + /* construct R and H */ + cpConstructBN(&R, rLen, (BNU_CHUNK_T*)pR, NULL); + cpConstructBN(&H, 1, &h, NULL); + /* convert GX ans GY coeffs into GF elements */ + ippsGFpSetElement((Ipp32u*)pX, BITS2WORD32_SIZE(BITSIZE_BNU(pX,xLen)), &elmA, pGF); + ippsGFpSetElement((Ipp32u*)pY, BITS2WORD32_SIZE(BITSIZE_BNU(pY,yLen)), &elmB, pGF); + /* and init EC subgroup */ + ippsGFpECSetSubgroup(&elmA, &elmB, &R, &H, pEC); + cpGFpReleasePool(2, pGFE); + +#if (_IPP32E >= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512IFMA)) { + ECP_MONT_R(pEC)->method_alt = gsArithGF_n256r1_avx512(); + } +#endif +} + +/*F* +// Name: ippsGFpECInitStd256r1 +// +// Purpose: Initializes the context of EC256r1 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pGFp +// +// ippStsContextMatchErr invalid pGFp->idCtx +// +// ippStsBadArgErr pGFp does not specify the finite field over which the given +// standard elliptic curve is defined +// +// ippStsNoErr no error +// +// Parameters: +// pGFp Pointer to the IppsGFpState context of the underlying finite field +// pEC Pointer to the context of the elliptic curve being initialized. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECInitStd256r1,(const IppsGFpState* pGFp, IppsGFpECState* pEC)) +{ + IPP_BAD_PTR2_RET(pGFp, pEC); + + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + + { + gsModEngine* pGFE = GFP_PMA(pGFp); + + /* test if GF is prime GF */ + IPP_BADARG_RET(!GFP_IS_BASIC(pGFE), ippStsBadArgErr); + /* test underlying prime value*/ + IPP_BADARG_RET(cpCmp_BNU(secp256r1_p, BITS_BNU_CHUNK(256), GFP_MODULUS(pGFE), BITS_BNU_CHUNK(256)), ippStsBadArgErr); + + ippsGFpECInit(pGFp, NULL, NULL, pEC); + cpGFpECSetStd(BITS_BNU_CHUNK(256), secp256r1_a, + BITS_BNU_CHUNK(256), secp256r1_b, + BITS_BNU_CHUNK(256), secp256r1_gx, + BITS_BNU_CHUNK(256), secp256r1_gy, + BITS_BNU_CHUNK(256), secp256r1_r, + secp256r1_h, + pEC); + + ECP_MODULUS_ID(pEC) = cpID_PrimeP256r1; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstd384r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstd384r1.c new file mode 100644 index 0000000..9309ba9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstd384r1.c @@ -0,0 +1,125 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECInitStd384r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" + +#include "ifma_arith_method.h" + + +static void cpGFpECSetStd(int aLen, const BNU_CHUNK_T* pA, + int bLen, const BNU_CHUNK_T* pB, + int xLen, const BNU_CHUNK_T* pX, + int yLen, const BNU_CHUNK_T* pY, + int rLen, const BNU_CHUNK_T* pR, + BNU_CHUNK_T h, + IppsGFpECState* pEC) +{ + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elemLen = GFP_FELEN(pGFE); + + IppsGFpElement elmA, elmB; + __ALIGN8 IppsBigNumState R; + __ALIGN8 IppsBigNumState H; + + /* convert A ans B coeffs into GF elements */ + cpGFpElementConstruct(&elmA, cpGFpGetPool(1, pGFE), elemLen); + cpGFpElementConstruct(&elmB, cpGFpGetPool(1, pGFE), elemLen); + ippsGFpSetElement((Ipp32u*)pA, BITS2WORD32_SIZE(BITSIZE_BNU(pA,aLen)), &elmA, pGF); + ippsGFpSetElement((Ipp32u*)pB, BITS2WORD32_SIZE(BITSIZE_BNU(pB,bLen)), &elmB, pGF); + /* and set EC */ + ippsGFpECSet(&elmA, &elmB, pEC); + + /* construct R and H */ + cpConstructBN(&R, rLen, (BNU_CHUNK_T*)pR, NULL); + cpConstructBN(&H, 1, &h, NULL); + /* convert GX ans GY coeffs into GF elements */ + ippsGFpSetElement((Ipp32u*)pX, BITS2WORD32_SIZE(BITSIZE_BNU(pX,xLen)), &elmA, pGF); + ippsGFpSetElement((Ipp32u*)pY, BITS2WORD32_SIZE(BITSIZE_BNU(pY,yLen)), &elmB, pGF); + /* and init EC subgroup */ + ippsGFpECSetSubgroup(&elmA, &elmB, &R, &H, pEC); + cpGFpReleasePool(2, pGFE); + +#if (_IPP32E >= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512IFMA)) { + ECP_MONT_R(pEC)->method_alt = gsArithGF_n384r1_avx512(); + } +#endif +} + +/*F* +// Name: ippsGFpECInitStd384r1 +// +// Purpose: Initializes the context of EC384r1 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pGFp +// +// ippStsContextMatchErr invalid pGFp->idCtx +// +// ippStsBadArgErr pGFp does not specify the finite field over which the given +// standard elliptic curve is defined +// +// ippStsNoErr no error +// +// Parameters: +// pGFp Pointer to the IppsGFpState context of the underlying finite field +// pEC Pointer to the context of the elliptic curve being initialized. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECInitStd384r1,(const IppsGFpState* pGFp, IppsGFpECState* pEC)) +{ + IPP_BAD_PTR2_RET(pGFp, pEC); + + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + + { + gsModEngine* pGFE = GFP_PMA(pGFp); + + /* test if GF is prime GF */ + IPP_BADARG_RET(!GFP_IS_BASIC(pGFE), ippStsBadArgErr); + /* test underlying prime value*/ + IPP_BADARG_RET(cpCmp_BNU(secp384r1_p, BITS_BNU_CHUNK(384), GFP_MODULUS(pGFE), BITS_BNU_CHUNK(384)), ippStsBadArgErr); + + ippsGFpECInit(pGFp, NULL, NULL, pEC); + cpGFpECSetStd(BITS_BNU_CHUNK(384), secp384r1_a, + BITS_BNU_CHUNK(384), secp384r1_b, + BITS_BNU_CHUNK(384), secp384r1_gx, + BITS_BNU_CHUNK(384), secp384r1_gy, + BITS_BNU_CHUNK(384), secp384r1_r, + secp384r1_h, + pEC); + + ECP_MODULUS_ID(pEC) = cpID_PrimeP384r1; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstd521r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstd521r1.c new file mode 100644 index 0000000..fe5bc7e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstd521r1.c @@ -0,0 +1,126 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECInitStd521r1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" + + +#include "ecnist/ifma_arith_method_p521.h" + + +static void cpGFpECSetStd(int aLen, const BNU_CHUNK_T* pA, + int bLen, const BNU_CHUNK_T* pB, + int xLen, const BNU_CHUNK_T* pX, + int yLen, const BNU_CHUNK_T* pY, + int rLen, const BNU_CHUNK_T* pR, + BNU_CHUNK_T h, + IppsGFpECState* pEC) +{ + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elemLen = GFP_FELEN(pGFE); + + IppsGFpElement elmA, elmB; + __ALIGN8 IppsBigNumState R; + __ALIGN8 IppsBigNumState H; + + /* convert A ans B coeffs into GF elements */ + cpGFpElementConstruct(&elmA, cpGFpGetPool(1, pGFE), elemLen); + cpGFpElementConstruct(&elmB, cpGFpGetPool(1, pGFE), elemLen); + ippsGFpSetElement((Ipp32u*)pA, BITS2WORD32_SIZE(BITSIZE_BNU(pA,aLen)), &elmA, pGF); + ippsGFpSetElement((Ipp32u*)pB, BITS2WORD32_SIZE(BITSIZE_BNU(pB,bLen)), &elmB, pGF); + /* and set EC */ + ippsGFpECSet(&elmA, &elmB, pEC); + + /* construct R and H */ + cpConstructBN(&R, rLen, (BNU_CHUNK_T*)pR, NULL); + cpConstructBN(&H, 1, &h, NULL); + /* convert GX ans GY coeffs into GF elements */ + ippsGFpSetElement((Ipp32u*)pX, BITS2WORD32_SIZE(BITSIZE_BNU(pX,xLen)), &elmA, pGF); + ippsGFpSetElement((Ipp32u*)pY, BITS2WORD32_SIZE(BITSIZE_BNU(pY,yLen)), &elmB, pGF); + /* and init EC subgroup */ + ippsGFpECSetSubgroup(&elmA, &elmB, &R, &H, pEC); + cpGFpReleasePool(2, pGFE); + +#if (_IPP32E >= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512IFMA)) { + ECP_MONT_R(pEC)->method_alt = gsArithGF_n521r1_avx512(); + } +#endif +} + +/*F* +// Name: ippsGFpECInitStd521r1 +// +// Purpose: Initializes the context of EC521r1 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pGFp +// +// ippStsContextMatchErr invalid pGFp->idCtx +// +// ippStsBadArgErr pGFp does not specify the finite field over which the given +// standard elliptic curve is defined +// +// ippStsNoErr no error +// +// Parameters: +// pGFp Pointer to the IppsGFpState context of the underlying finite field +// pEC Pointer to the context of the elliptic curve being initialized. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECInitStd521r1,(const IppsGFpState* pGF, IppsGFpECState* pEC)) +{ + IPP_BAD_PTR2_RET(pGF, pEC); + + IPP_BADARG_RET( !GFP_VALID_ID(pGF), ippStsContextMatchErr ); + + { + gsModEngine* pGFE = GFP_PMA(pGF); + + /* test if GF is prime GF */ + IPP_BADARG_RET(!GFP_IS_BASIC(pGFE), ippStsBadArgErr); + /* test underlying prime value*/ + IPP_BADARG_RET(cpCmp_BNU(secp521r1_p, BITS_BNU_CHUNK(521), GFP_MODULUS(pGFE), BITS_BNU_CHUNK(521)), ippStsBadArgErr); + + ippsGFpECInit(pGF, NULL, NULL, pEC); + cpGFpECSetStd(BITS_BNU_CHUNK(521), secp521r1_a, + BITS_BNU_CHUNK(521), secp521r1_b, + BITS_BNU_CHUNK(521), secp521r1_gx, + BITS_BNU_CHUNK(521), secp521r1_gy, + BITS_BNU_CHUNK(521), secp521r1_r, + secp521r1_h, + pEC); + + ECP_MODULUS_ID(pEC) = cpID_PrimeP521r1; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstdbn256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstdbn256.c new file mode 100644 index 0000000..4c62398 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstdbn256.c @@ -0,0 +1,117 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECInitStdBN256() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" + + + + +static void cpGFpECSetStd(int aLen, const BNU_CHUNK_T* pA, + int bLen, const BNU_CHUNK_T* pB, + int xLen, const BNU_CHUNK_T* pX, + int yLen, const BNU_CHUNK_T* pY, + int rLen, const BNU_CHUNK_T* pR, + BNU_CHUNK_T h, + IppsGFpECState* pEC) +{ + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elemLen = GFP_FELEN(pGFE); + + IppsGFpElement elmA, elmB; + __ALIGN8 IppsBigNumState R; + __ALIGN8 IppsBigNumState H; + + /* convert A ans B coeffs into GF elements */ + cpGFpElementConstruct(&elmA, cpGFpGetPool(1, pGFE), elemLen); + cpGFpElementConstruct(&elmB, cpGFpGetPool(1, pGFE), elemLen); + ippsGFpSetElement((Ipp32u*)pA, BITS2WORD32_SIZE(BITSIZE_BNU(pA,aLen)), &elmA, pGF); + ippsGFpSetElement((Ipp32u*)pB, BITS2WORD32_SIZE(BITSIZE_BNU(pB,bLen)), &elmB, pGF); + /* and set EC */ + ippsGFpECSet(&elmA, &elmB, pEC); + + /* construct R and H */ + cpConstructBN(&R, rLen, (BNU_CHUNK_T*)pR, NULL); + cpConstructBN(&H, 1, &h, NULL); + /* convert GX ans GY coeffs into GF elements */ + ippsGFpSetElement((Ipp32u*)pX, BITS2WORD32_SIZE(BITSIZE_BNU(pX,xLen)), &elmA, pGF); + ippsGFpSetElement((Ipp32u*)pY, BITS2WORD32_SIZE(BITSIZE_BNU(pY,yLen)), &elmB, pGF); + /* and init EC subgroup */ + ippsGFpECSetSubgroup(&elmA, &elmB, &R, &H, pEC); + cpGFpReleasePool(2, pGFE); +} + +/*F* +// Name: ippsGFpECInitStdBN256 +// +// Purpose: Initializes the context of ECBN256 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pGFp +// +// ippStsContextMatchErr invalid pGFp->idCtx +// +// ippStsBadArgErr pGFp does not specify the finite field over which the given +// standard elliptic curve is defined +// +// ippStsNoErr no error +// +// Parameters: +// pGFp Pointer to the IppsGFpState context of the underlying finite field +// pEC Pointer to the context of the elliptic curve being initialized. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECInitStdBN256,(const IppsGFpState* pGF, IppsGFpECState* pEC)) +{ + IPP_BAD_PTR2_RET(pGF, pEC); + + IPP_BADARG_RET( !GFP_VALID_ID(pGF), ippStsContextMatchErr ); + + { + gsModEngine* pGFE = GFP_PMA(pGF); + + /* test if GF is prime GF */ + IPP_BADARG_RET(!GFP_IS_BASIC(pGFE), ippStsBadArgErr); + /* test underlying prime value*/ + IPP_BADARG_RET(cpCmp_BNU(tpmBN_p256p_p, BITS_BNU_CHUNK(256), GFP_MODULUS(pGFE), BITS_BNU_CHUNK(256)), ippStsBadArgErr); + + ippsGFpECInit(pGF, NULL, NULL, pEC); + cpGFpECSetStd(BITS_BNU_CHUNK(BNU_CHUNK_BITS), tpmBN_p256p_a, + BITS_BNU_CHUNK(BNU_CHUNK_BITS), tpmBN_p256p_b, + BITS_BNU_CHUNK(BNU_CHUNK_BITS), tpmBN_p256p_gx, + BITS_BNU_CHUNK(BNU_CHUNK_BITS), tpmBN_p256p_gy, + BITS_BNU_CHUNK(256), tpmBN_p256p_r, + tpmBN_p256p_h, + pEC); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstdsm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstdsm2.c new file mode 100644 index 0000000..83ba0ff --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecinitstdsm2.c @@ -0,0 +1,128 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECInitStdSM2() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" + +#if (_IPP32E >= _IPP32E_K1) +#include "sm2/ifma_arith_method_sm2.h" +#include "pcpgfpmethod.h" +#endif // (_IPP32E >= _IPP32E_K1) + + +static void cpGFpECSetStd(int aLen, const BNU_CHUNK_T* pA, + int bLen, const BNU_CHUNK_T* pB, + int xLen, const BNU_CHUNK_T* pX, + int yLen, const BNU_CHUNK_T* pY, + int rLen, const BNU_CHUNK_T* pR, + BNU_CHUNK_T h, + IppsGFpECState* pEC) +{ + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elemLen = GFP_FELEN(pGFE); + + IppsGFpElement elmA, elmB; + __ALIGN8 IppsBigNumState R; + __ALIGN8 IppsBigNumState H; + + /* convert A ans B coeffs into GF elements */ + cpGFpElementConstruct(&elmA, cpGFpGetPool(1, pGFE), elemLen); + cpGFpElementConstruct(&elmB, cpGFpGetPool(1, pGFE), elemLen); + ippsGFpSetElement((Ipp32u*)pA, BITS2WORD32_SIZE(BITSIZE_BNU(pA,aLen)), &elmA, pGF); + ippsGFpSetElement((Ipp32u*)pB, BITS2WORD32_SIZE(BITSIZE_BNU(pB,bLen)), &elmB, pGF); + /* and set EC */ + ippsGFpECSet(&elmA, &elmB, pEC); + + /* construct R and H */ + cpConstructBN(&R, rLen, (BNU_CHUNK_T*)pR, NULL); + cpConstructBN(&H, 1, &h, NULL); + /* convert GX ans GY coeffs into GF elements */ + ippsGFpSetElement((Ipp32u*)pX, BITS2WORD32_SIZE(BITSIZE_BNU(pX,xLen)), &elmA, pGF); + ippsGFpSetElement((Ipp32u*)pY, BITS2WORD32_SIZE(BITSIZE_BNU(pY,yLen)), &elmB, pGF); + /* and init EC subgroup */ + ippsGFpECSetSubgroup(&elmA, &elmB, &R, &H, pEC); + cpGFpReleasePool(2, pGFE); + +#if (_IPP32E >= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512IFMA)) { + ECP_MONT_R(pEC)->method_alt = gsArithGF_nsm2_avx512(); + } +#endif // (_IPP32E >= _IPP32E_K1) +} + +/*F* +// Name: ippsGFpECInitStdSM2 +// +// Purpose: Initializes the context of ECSM2 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pGFp +// +// ippStsContextMatchErr invalid pGFp->idCtx +// +// ippStsBadArgErr pGFp does not specify the finite field over which the given +// standard elliptic curve is defined +// +// ippStsNoErr no error +// +// Parameters: +// pGFp Pointer to the IppsGFpState context of the underlying finite field +// pEC Pointer to the context of the elliptic curve being initialized. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECInitStdSM2,(const IppsGFpState* pGF, IppsGFpECState* pEC)) +{ + IPP_BAD_PTR2_RET(pGF, pEC); + + IPP_BADARG_RET( !GFP_VALID_ID(pGF), ippStsContextMatchErr ); + + { + gsModEngine* pGFE = GFP_PMA(pGF); + + /* test if GF is prime GF */ + IPP_BADARG_RET(!GFP_IS_BASIC(pGFE), ippStsBadArgErr); + /* test underlying prime value*/ + IPP_BADARG_RET(cpCmp_BNU(tpmSM2_p256_p, BITS_BNU_CHUNK(256), GFP_MODULUS(pGFE), BITS_BNU_CHUNK(256)), ippStsBadArgErr); + + ippsGFpECInit(pGF, NULL, NULL, pEC); + cpGFpECSetStd(BITS_BNU_CHUNK(256), tpmSM2_p256_a, + BITS_BNU_CHUNK(256), tpmSM2_p256_b, + BITS_BNU_CHUNK(256), tpmSM2_p256_gx, + BITS_BNU_CHUNK(256), tpmSM2_p256_gy, + BITS_BNU_CHUNK(256), tpmSM2_p256_r, + tpmSM2_p256_h, + pEC); + + ECP_MODULUS_ID(pEC) = cpID_PrimeTPM_SM2; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpeckeys.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpeckeys.c new file mode 100644 index 0000000..5023628 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpeckeys.c @@ -0,0 +1,157 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// ippsGFpECTstKeyPair() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" + + +/* +// checks privateKey +// +// returns 1 if private_key belongs (0,order) range +// 0 if otherwise +*/ +int gfec_CheckPrivateKey(const IppsBigNumState* pPrivate, IppsGFpECState* pEC) +{ + /* order */ + BNU_CHUNK_T* pOrder = MOD_MODULUS(ECP_MONT_R(pEC)); + int orderLen = BITS_BNU_CHUNK(ECP_ORDBITSIZE(pEC)); + + /* key under check */ + BNU_CHUNK_T* pKey = BN_NUMBER(pPrivate); + int keyLen = BN_SIZE(pPrivate); + + IppsGFpState *pGF = ECP_GFP(pEC); + gsModEngine *pGFE = GFP_PMA(pGF); + + BNU_CHUNK_T* F = cpGFpGetPool(1, pGFE); + + int ret = BN_POSITIVE(pPrivate); + if (ret) + ret = !IsZero_BN(pPrivate); + if (ret) + ret = ECP_ORDBITSIZE(pEC) >= cpBN_bitsize(pPrivate); + if (ret) { + cpGFpElementCopyPad(F, orderLen, pKey, keyLen); + /* cpSub_BNU() returns borrow bit, so any non - zero value corresponds to a valid key range(key < order) */ + ret = 0 != cpSub_BNU(F, F, pOrder, orderLen); + } + + cpGFpReleasePool(1, pGFE); + + return ret; +} + +/*F* +// Name: ippsGFpECTstKeyPair +// +// Purpose: Test Key Pair +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pPrivate +// NULL == pPublic +// NULL == pResult +// NULL == pScratchBuffer +// +// ippStsContextMatchErr illegal pEC->idCtx +// pEC->subgroup == NULL +// illegal pPrivate->idCtx +// illegal pPublic->idCtx +// +// ippStsRangeErr ECP_POINT_FELEN(pPublic)idCtx +// invalid pPoint->idCtx +// invalid pX->idCtx +// +// ippStsBadArgErr !GFP_IS_BASIC(GFP_PMA(ECP_GFP(pEC))) +// +// ippStsOutOfRangeErr ECP_POINT_FELEN(pPoint)!=GFP_FELEN() +// GFPE_ROOM(pX)!=GFP_FELEN() +// +// ippStsQuadraticNonResidueErr square of the Y-coordinate of +// the pPoint is a quadratic non-residue modulo +// +// +// ippStsNoErr no error +// +// Parameters: +// pPoint Pointer to the IppsGFpECPoint context +// pEC Pointer to the context of the elliptic curve +// pX Pointer to the X-coordinate of the point on the elliptic curve +// +// Note: +// Is not a fact that computed point belongs to BP-related subgroup BP +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECMakePoint,(const IppsGFpElement* pX, IppsGFpECPoint* pPoint, IppsGFpECState* pEC)) +{ + IPP_BAD_PTR3_RET(pX, pPoint, pEC); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFP_IS_BASIC(GFP_PMA(ECP_GFP(pEC))), ippStsBadArgErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pX), ippStsContextMatchErr ); + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pPoint), ippStsContextMatchErr ); + + IPP_BADARG_RET( GFPE_ROOM(pX)!=GFP_FELEN(GFP_PMA(ECP_GFP(pEC))), ippStsOutOfRangeErr); + IPP_BADARG_RET( ECP_POINT_FELEN(pPoint)!=GFP_FELEN(GFP_PMA(ECP_GFP(pEC))), ippStsOutOfRangeErr); + + return gfec_MakePoint(pPoint, GFPE_DATA(pX), pEC)? ippStsNoErr : ippStsQuadraticNonResidueErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecmulpoint.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecmulpoint.c new file mode 100644 index 0000000..1af04d7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecmulpoint.c @@ -0,0 +1,155 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECMulPoint() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" + +/*F* +// Name: ippsGFpECMulPoint +// +// Purpose: Multiplies a point on an elliptic curve by a scalar +// +// Returns: Reason: +// ippStsNullPtrErr pP == NULL +// pQ == NULL +// pR == NULL +// pEC == NULL +// +// ippStsContextMatchErr invalid pEC->idCtx +// pEC->subgroup == NULL +// invalid pP->idCtx +// invalid pQ->idCtx +// invalid pR->idCtx +// +// ippStsOutOfRangeErr ECP_POINT_FELEN(pP)!=GFP_FELEN() +// ECP_POINT_FELEN(pR)!=GFP_FELEN() +// +// ippStsBadArgErr pN is negative +// pN > MOD_MODULUS(ECP_MONT_R(pEC)) +// +// ippStsNoErr no error +// +// Parameters: +// pP Pointer to the context of the given point on the elliptic curve +// pN Pointer to the Big Number context storing the scalar value +// pR Pointer to the context of the resulting elliptic curve point +// pEC Pointer to the context of the elliptic curve +// pScratchBuffer Pointer to the scratch buffer +// +// Note: +// computes [N]*P, 0 < N < order +// +*F*/ +#if 0 +IPPFUN(IppStatus, ippsGFpECMulPoint,(const IppsGFpECPoint* pP, + const IppsBigNumState* pN, + IppsGFpECPoint* pR, + IppsGFpECState* pEC, + Ipp8u* pScratchBuffer)) +{ + IPP_BAD_PTR4_RET(pP, pR, pEC, pScratchBuffer); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + IPP_BADARG_RET(!ECP_SUBGROUP(pEC), ippStsContextMatchErr); + + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pP), ippStsContextMatchErr ); + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pR), ippStsContextMatchErr ); + + IPP_BADARG_RET( ECP_POINT_FELEN(pP)!=GFP_FELEN(GFP_PMA(ECP_GFP(pEC))), ippStsOutOfRangeErr); + IPP_BADARG_RET( ECP_POINT_FELEN(pR)!=GFP_FELEN(GFP_PMA(ECP_GFP(pEC))), ippStsOutOfRangeErr); + + IPP_BAD_PTR1_RET(pN); + IPP_BADARG_RET(!BN_VALID_ID(pN), ippStsContextMatchErr ); + IPP_BADARG_RET( BN_NEGATIVE(pN), ippStsBadArgErr ); + + { + BNU_CHUNK_T* pScalar = BN_NUMBER(pN); + int scalarLen = BN_SIZE(pN); + IPP_BADARG_RET(0= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512IFMA)) { + switch (ECP_MODULUS_ID(pEC)) { + case cpID_PrimeP256r1: { + gfec_MulPoint_nistp256_avx512(pR, pP, pScalar, scalarLen, pEC, pScratchBuffer); + return ippStsNoErr; + } + case cpID_PrimeP384r1: { + gfec_MulPoint_nistp384_avx512(pR, pP, pScalar, scalarLen, pEC, pScratchBuffer); + return ippStsNoErr; + } + case cpID_PrimeP521r1: { + gfec_MulPoint_nistp521_avx512(pR, pP, pScalar, scalarLen, pEC, pScratchBuffer); + return ippStsNoErr; + } + case cpID_PrimeTPM_SM2: { + gfec_MulPoint_sm2_avx512(pR, pP, pScalar, scalarLen, pEC, pScratchBuffer); + return ippStsNoErr; + } + default: + /* Go to default implementation below */ + break; + } + } /* no else */ +#endif // (_IPP32E >= _IPP32E_K1) + + gfec_MulPoint(pR, pP, pScalar, scalarLen, pEC, pScratchBuffer); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecnegpoint.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecnegpoint.c new file mode 100644 index 0000000..bb151ed --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecnegpoint.c @@ -0,0 +1,76 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECNegPoint() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcphash.h" +#include "pcphash_rmf.h" + +/*F* +// Name: ippsGFpECNegPoint +// +// Purpose: Computes the inverse of a point +// +// Returns: Reason: +// ippStsNullPtrErr pP == NULL +// pR == NULL +// pEC == NULL +// +// ippStsContextMatchErr invalid pEC->idCtx +// invalid pP->idCtx +// invalid pR->idCtx +// +// ippStsOutOfRangeErr ECP_POINT_FELEN(pP)!=GFP_FELEN() +// ECP_POINT_FELEN(pR)!=GFP_FELEN() +// +// ippStsNoErr no error +// +// Parameters: +// pP Pointer to the context of the first elliptic curve point +// pR Pointer to the context of the resulting elliptic curve point +// pEC Pointer to the context of the elliptic curve +// +*F*/ + + +IPPFUN(IppStatus, ippsGFpECNegPoint,(const IppsGFpECPoint* pP, + IppsGFpECPoint* pR, + IppsGFpECState* pEC)) +{ + IPP_BAD_PTR3_RET(pP, pR, pEC); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pP), ippStsContextMatchErr ); + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pR), ippStsContextMatchErr ); + + IPP_BADARG_RET( ECP_POINT_FELEN(pP)!=GFP_FELEN(GFP_PMA(ECP_GFP(pEC))), ippStsOutOfRangeErr); + IPP_BADARG_RET( ECP_POINT_FELEN(pR)!=GFP_FELEN(GFP_PMA(ECP_GFP(pEC))), ippStsOutOfRangeErr); + + gfec_NegPoint(pR, pP, pEC); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecpoint.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecpoint.c new file mode 100644 index 0000000..31fbef3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecpoint.c @@ -0,0 +1,112 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECPointGetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcphash.h" +#include "pcphash_rmf.h" + +/*F* +// Name: ippsGFpECPointGetSize +// +// Purpose: Gets the size of the IppsGFpECPoint context +// +// Returns: Reason: +// ippStsNullPtrErr pEC == NULL +// pSize == NULL +// +// ippStsContextMatchErr invalid pEC->idCtx +// +// ippStsNoErr no error +// +// Parameters: +// pEC Pointer to the context of the elliptic curve +// pSize Pointer to the buffer size +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECPointGetSize,(const IppsGFpECState* pEC, int* pSize)) +{ + IPP_BAD_PTR2_RET(pEC, pSize); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + + { + int elemLen = GFP_FELEN(GFP_PMA(ECP_GFP(pEC))); + *pSize = (Ipp32s)sizeof(IppsGFpECPoint) + +elemLen*(Ipp32s)sizeof(BNU_CHUNK_T) /* X */ + +elemLen*(Ipp32s)sizeof(BNU_CHUNK_T) /* Y */ + +elemLen*(Ipp32s)sizeof(BNU_CHUNK_T);/* Z */ + return ippStsNoErr; + } +} + +/*F* +// Name: ippsGFpECPointInit +// +// Purpose: Initializes the context of a point on an elliptic curve +// +// Returns: Reason: +// ippStsNullPtrErr pPoint == NULL +// pEC == NULL +// +// ippStsContextMatchErr invalid pEC->idCtx +// +// ippStsNoErr no error +// +// Parameters: +// pX, pY Pointers to the X and Y coordinates of a point on the elliptic curve +// pPoint Pointer to the IppsGFpECPoint context being initialized +// pEC Pointer to the context of the elliptic curve +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECPointInit,(const IppsGFpElement* pX, const IppsGFpElement* pY, + IppsGFpECPoint* pPoint, IppsGFpECState* pEC)) +{ + IPP_BAD_PTR2_RET(pPoint, pEC); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + + { + Ipp8u* ptr = (Ipp8u*)pPoint; + int elemLen = GFP_FELEN(GFP_PMA(ECP_GFP(pEC))); + + ECP_POINT_SET_ID(pPoint); + ECP_POINT_FLAGS(pPoint) = 0; + ECP_POINT_FELEN(pPoint) = elemLen; + ptr += sizeof(IppsGFpECPoint); + ECP_POINT_DATA(pPoint) = (BNU_CHUNK_T*)(ptr); + + if(pX && pY) + return ippsGFpECSetPoint(pX, pY, pPoint, pEC); + else { + gfec_SetPointAtInfinity(pPoint); + return ippStsNoErr; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecpointstuff.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecpointstuff.c new file mode 100644 index 0000000..9b8b70a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecpointstuff.c @@ -0,0 +1,133 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal EC over GF(p^m) basic Definitions & Function Prototypes +// +// Context: +// gfec_IsPointOnCurve() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "gsscramble.h" + + +#if ( ECP_PROJECTIVE_COORD == JACOBIAN ) +IPP_OWN_DEFN (int, gfec_IsPointOnCurve, (const IppsGFpECPoint* pPoint, IppsGFpECState* pEC)) +{ + /* point at infinity does not belong curve */ + if( !IS_ECP_FINITE_POINT(pPoint) ) + //return 1; + return 0; + + /* test that 0 == R = (Y^2) - (X^3 + A*X*(Z^4) + B*(Z^6)) */ + else { + int isOnCurve = 0; + + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + + mod_mul mulF = GFP_METHOD(pGFE)->mul; + mod_sqr sqrF = GFP_METHOD(pGFE)->sqr; + mod_sub subF = GFP_METHOD(pGFE)->sub; + + BNU_CHUNK_T* pX = ECP_POINT_X(pPoint); + BNU_CHUNK_T* pY = ECP_POINT_Y(pPoint); + BNU_CHUNK_T* pZ = ECP_POINT_Z(pPoint); + + BNU_CHUNK_T* pR = NULL; + BNU_CHUNK_T* pT = NULL; + + BNU_CHUNK_T* pZ4 = NULL; + BNU_CHUNK_T* pZ6 = NULL; + +#if (_IPP32E >= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512IFMA)) { + switch (ECP_MODULUS_ID(pEC)) { + case cpID_PrimeP256r1: { + isOnCurve = gfec_point_on_curve_nistp256_avx512(pPoint, pEC); + goto exit; + break; + } + case cpID_PrimeP384r1: { + isOnCurve = gfec_point_on_curve_nistp384_avx512(pPoint, pEC); + goto exit; + break; + } + case cpID_PrimeP521r1: { + isOnCurve = gfec_point_on_curve_nistp521_avx512(pPoint, pEC); + goto exit; + break; + } + case cpID_PrimeTPM_SM2: { + isOnCurve = gfec_point_on_curve_sm2_avx512(pPoint, pEC); + goto exit; + break; + } + default: + /* Go to default implementation below */ + break; + } + } +#endif // (_IPP32E >= _IPP32E_K1) + pR = cpGFpGetPool(1, pGFE); + pT = cpGFpGetPool(1, pGFE); + + sqrF(pR, pY, pGFE); /* R = Y^2 */ + sqrF(pT, pX, pGFE); /* T = X^3 */ + mulF(pT, pX, pT, pGFE); + subF(pR, pR, pT, pGFE); /* R -= T */ + + if( IS_ECP_AFFINE_POINT(pPoint) ) { + mulF(pT, pX, ECP_A(pEC), pGFE); /* T = A*X */ + subF(pR, pR, pT, pGFE); /* R -= T */ + subF(pR, pR, ECP_B(pEC), pGFE); /* R -= B */ + } + else { + pZ4 = cpGFpGetPool(1, pGFE); + pZ6 = cpGFpGetPool(1, pGFE); + + sqrF(pZ6, pZ, pGFE); /* Z^2 */ + sqrF(pZ4, pZ6, pGFE); /* Z^4 */ + mulF(pZ6, pZ6, pZ4, pGFE); /* Z^6 */ + + mulF(pZ4, pZ4, pX, pGFE); /* X*(Z^4) */ + mulF(pZ4, pZ4, ECP_A(pEC), pGFE); /* A*X*(Z^4) */ + mulF(pZ6, pZ6, ECP_B(pEC), pGFE); /* B*(Z^4) */ + + subF(pR, pR, pZ4, pGFE); /* R -= A*X*(Z^4) */ + subF(pR, pR, pZ6, pGFE); /* R -= B*(Z^6) */ + + cpGFpReleasePool(2, pGFE); + } + + isOnCurve = GFP_IS_ZERO(pR, GFP_FELEN(pGFE)); + + cpGFpReleasePool(2, pGFE); +#if (_IPP32E >= _IPP32E_K1) + exit: +#endif // (_IPP32E >= _IPP32E_K1) + return isOnCurve; + } +} +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecprivatekey.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecprivatekey.c new file mode 100644 index 0000000..805767a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecprivatekey.c @@ -0,0 +1,95 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// ippsGFpECPrivateKey() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" + +/*F* +// Name: ippsGFpECPrivateKey +// +// Purpose: Generate random private key +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pPrivate +// +// ippStsContextMatchErr illegal pEC->idCtx +// pEC->subgroup == NULL +// illegal pPrivate->idCtx +// +// ippStsSizeErr BN_ROOM(pPrivate)*BITSIZE(BNU_CHUNK_T)idCtx +// pEC->subgroup == NULL +// illegal pPrivate->idCtx +// illegal pPublic->idCtx +// +// ippStsInvalidPrivateKey !(0 < pPrivate < order) +// +// ippStsRangeErr ECP_POINT_FELEN(pPublic)= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512IFMA)) { + switch (ECP_MODULUS_ID(pEC)) { + case cpID_PrimeP256r1: { + gfec_PubKey_nist256_avx512(pPublic, pS, nsS, pEC, pScratchBuffer); + return ippStsNoErr; + } + case cpID_PrimeP384r1: { + gfec_PubKey_nist384_avx512(pPublic, pS, nsS, pEC, pScratchBuffer); + return ippStsNoErr; + } + case cpID_PrimeP521r1: { + gfec_PubKey_nist521_avx512(pPublic, pS, nsS, pEC, pScratchBuffer); + return ippStsNoErr; + } + case cpID_PrimeTPM_SM2: { + gfec_PubKey_sm2_avx512(pPublic, pS, nsS, pEC, pScratchBuffer); + return ippStsNoErr; + } + default: + /* Go to default implementation below */ + break; + } + } /* no else */ +#endif // (_IPP32E >= _IPP32E_K1) + { + gfec_MulBasePoint(pPublic, pS, nsS, pEC, pScratchBuffer); + } + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecset.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecset.c new file mode 100644 index 0000000..9d17ea9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecset.c @@ -0,0 +1,99 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECSet() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsGFpECSet +// +// Purpose: Sets up the parameters of an elliptic curve over a finite field +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pA +// NULL == pB +// +// ippStsContextMatchErr invalid pEC->idCtx +// invalid pA->idCtx +// invalid pB->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM(pA)!=GFP_FELEN(pGFE) +// GFPE_ROOM(pB)!=GFP_FELEN(pGFE) +// +// ippStsNoErr no error +// +// Parameters: +// pA Pointer to the coefficient A of the equation defining the elliptic curve +// pB Pointer to the coefficient B of the equation defining the elliptic curve +// pEC Pointer to the context of the elliptic curve +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECSet,(const IppsGFpElement* pA, + const IppsGFpElement* pB, + IppsGFpECState* pEC)) +{ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + + IPP_BAD_PTR2_RET(pA, pB); + IPP_BADARG_RET( !GFPE_VALID_ID(pA), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pB), ippStsContextMatchErr ); + + { + gsModEngine* pGFE = GFP_PMA(ECP_GFP(pEC)); + int elemLen = GFP_FELEN(pGFE); + + IPP_BADARG_RET( GFPE_ROOM(pA)!=GFP_FELEN(pGFE), ippStsOutOfRangeErr); + IPP_BADARG_RET( GFPE_ROOM(pB)!=GFP_FELEN(pGFE), ippStsOutOfRangeErr); + + /* copy A */ + cpGFpElementPad(ECP_A(pEC), elemLen, 0); + cpGFpElementCopy(ECP_A(pEC), GFPE_DATA(pA), elemLen); + /* and set up A-specific (a==0 or a==-3) if is */ + if(GFP_IS_ZERO(ECP_A(pEC), elemLen)) + ECP_SPECIFIC(pEC) = ECP_EPID2; + + cpGFpElementSetChunk(ECP_B(pEC), elemLen, 3); + GFP_METHOD(pGFE)->encode(ECP_B(pEC), ECP_B(pEC), pGFE); + GFP_METHOD(pGFE)->add(ECP_B(pEC), ECP_A(pEC), ECP_B(pEC), pGFE); + if(GFP_IS_ZERO(ECP_B(pEC), elemLen)) + ECP_SPECIFIC(pEC) = ECP_STD; + + /* copy B */ + cpGFpElementPad(ECP_B(pEC), elemLen, 0); + cpGFpElementCopy(ECP_B(pEC), GFPE_DATA(pB), elemLen); + /* and set type of affine infinity representation: + // (0,1) if B==0 + // (0,0) if B!=0 */ + ECP_INFINITY(pEC) = GFP_IS_ZERO(ECP_B(pEC), elemLen); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpoint.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpoint.c new file mode 100644 index 0000000..3d2fe26 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpoint.c @@ -0,0 +1,85 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECSetPoint() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcphash.h" +#include "pcphash_rmf.h" + +/*F* +// Name: ippsGFpECSetPoint +// +// Purpose: Sets a point on an elliptic curve +// +// Returns: Reason: +// ippStsNullPtrErr pPoint == NULL +// pEC == NULL +// pX == NULL +// pY == NULL +// +// ippStsContextMatchErr invalid pEC->idCtx +// invalid pPoint->idCtx +// invalid pX->idCtx +// invalid pY->idCtx +// +// ippStsOutOfRangeErr ECP_POINT_FELEN(pX)!=GFP_FELEN() +// ECP_POINT_FELEN(pY)!=GFP_FELEN() +// ECP_POINT_FELEN(pPoint)!=GFP_FELEN() +// +// ippStsNoErr no error +// +// Parameters: +// pX, pY Pointers to the X and Y coordinates of a point on the elliptic curve +// pPoint Pointer to the IppsGFpECPoint context +// pEC Pointer to the context of the elliptic curve +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECSetPoint,(const IppsGFpElement* pX, const IppsGFpElement* pY, + IppsGFpECPoint* pPoint, + IppsGFpECState* pEC)) +{ + IPP_BAD_PTR2_RET(pPoint, pEC); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pPoint), ippStsContextMatchErr ); + + IPP_BAD_PTR2_RET(pX, pY); + IPP_BADARG_RET( !GFPE_VALID_ID(pX), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pY), ippStsContextMatchErr ); + + IPP_BADARG_RET( GFPE_ROOM(pX)!=GFP_FELEN(GFP_PMA(ECP_GFP(pEC))), ippStsOutOfRangeErr); + IPP_BADARG_RET( GFPE_ROOM(pY)!=GFP_FELEN(GFP_PMA(ECP_GFP(pEC))), ippStsOutOfRangeErr); + IPP_BADARG_RET( ECP_POINT_FELEN(pPoint)!=GFP_FELEN(GFP_PMA(ECP_GFP(pEC))), ippStsOutOfRangeErr); + + if(gfec_SetPoint(ECP_POINT_DATA(pPoint), GFPE_DATA(pX), GFPE_DATA(pY), pEC)) + ECP_POINT_FLAGS(pPoint) = ECP_AFFINE_POINT | ECP_FINITE_POINT; + else + ECP_POINT_FLAGS(pPoint) = 0; + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointatinf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointatinf.c new file mode 100644 index 0000000..6ac685f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointatinf.c @@ -0,0 +1,67 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECSetPointAtInfinity() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcphash.h" +#include "pcphash_rmf.h" + +/*F* +// Name: ippsGFpECSetPointAtInfinity +// +// Purpose: Sets a point on an elliptic curve as a point at infinity +// +// Returns: Reason: +// ippStsNullPtrErr pPoint == NULL +// pEC == NULL +// +// ippStsContextMatchErr invalid pEC->idCtx +// invalid pPoint->idCtx +// +// ippStsOutOfRangeErr ECP_POINT_FELEN()!=GFP_FELEN() +// +// ippStsNoErr no error +// +// Parameters: +// pPoint Pointer to the IppsGFpECPoint context +// pEC Pointer to the context of the elliptic curve +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECSetPointAtInfinity,(IppsGFpECPoint* pPoint, IppsGFpECState* pEC)) +{ + IPP_BAD_PTR2_RET(pPoint, pEC); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pPoint), ippStsContextMatchErr ); + + IPP_BADARG_RET( ECP_POINT_FELEN(pPoint)!=GFP_FELEN(GFP_PMA(ECP_GFP(pEC))), ippStsOutOfRangeErr); + + gfec_SetPointAtInfinity(pPoint); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointhash.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointhash.c new file mode 100644 index 0000000..5aa4172 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointhash.c @@ -0,0 +1,154 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECSetPointHash() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcphash.h" +#include "pcphash_rmf.h" + +/*F* +// Name: ippsGFpECSetPointHash +// +// Purpose: Constructs a point on an elliptic curve based on the hash of the input message +// +// Returns: Reason: +// ippStsNullPtrErr pPoint == NULL +// pEC == NULL +// pScratchBuffer == NULL +// (msgLen && !pMsg) +// +// ippStsContextMatchErr invalid pEC->idCtx +// invalid pPoint->idCtx +// +// ippStsBadArgErr !GFP_IS_BASIC(pGFE) +// +// ippStsOutOfRangeErr ECP_POINT_FELEN(pPoint)!=GFP_FELEN() +// +// ippStsQuadraticNonResidueErr square of the Y-coordinate of +// the pPoint is a quadratic non-residue modulo +// +// ippStsLengthErr msgLen<0 +// +// ippStsNoErr no error +// +// Parameters: +// hdr Header of the input message +// pMsg Pointer to the input message +// msgLen Length of the input message +// pPoint Pointer to the IppsGFpECPoint context +// pEC Pointer to the context of the elliptic curve +// hashID ID of the hash algorithm used +// pScratchBuffer Pointer to the scratch buffer +// +// Note: +// Is not a fact that computed point belongs to BP-related subgroup BP +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECSetPointHash,(Ipp32u hdr, const Ipp8u* pMsg, int msgLen, IppsGFpECPoint* pPoint, + IppsGFpECState* pEC, IppHashAlgId hashID, + Ipp8u* pScratchBuffer)) +{ + IppsGFpState* pGF; + gsModEngine* pGFE; + + /* get algorithm id */ + hashID = cpValidHashAlg(hashID); + IPP_BADARG_RET(ippHashAlg_Unknown==hashID, ippStsNotSupportedModeErr); + + /* test message length */ + IPP_BADARG_RET((msgLen<0), ippStsLengthErr); + /* test message pointer */ + IPP_BADARG_RET((msgLen && !pMsg), ippStsNullPtrErr); + + IPP_BAD_PTR3_RET(pPoint, pEC, pScratchBuffer); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + + pGF = ECP_GFP(pEC); + pGFE = GFP_PMA(pGF); + + IPP_BADARG_RET( !GFP_IS_BASIC(pGFE), ippStsBadArgErr ); + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pPoint), ippStsContextMatchErr ); + + IPP_BADARG_RET( ECP_POINT_FELEN(pPoint)!=GFP_FELEN(pGFE), ippStsOutOfRangeErr); + + { + int elemLen = GFP_FELEN(pGFE); + BNU_CHUNK_T* pModulus = GFP_MODULUS(pGFE); + + Ipp8u md[IPP_SHA512_DIGEST_BITSIZE/BYTESIZE]; + int hashLen = cpHashAlgAttr[hashID].hashSize; + BNU_CHUNK_T hashVal[BITS_BNU_CHUNK(IPP_SHA512_DIGEST_BITSIZE)+1]; + int hashValLen; + + IppsHashState hashCtx; + ippsHashInit(&hashCtx, hashID); + + { + BNU_CHUNK_T* pPoolElm = cpGFpGetPool(1, pGFE); + + /* convert hdr => hdrStr */ + BNU_CHUNK_T locHdr = (BNU_CHUNK_T)hdr; + Ipp8u hdrOctStr[sizeof(hdr/*locHdr*/)]; + cpToOctStr_BNU(hdrOctStr, sizeof(hdrOctStr), &locHdr, 1); + + /* compute md = hash(hrd||msg) */ + ippsHashUpdate(hdrOctStr, sizeof(hdrOctStr), &hashCtx); + ippsHashUpdate(pMsg, msgLen, &hashCtx); + ippsHashFinal(md, &hashCtx); + + /* convert hash into the integer */ + hashValLen = cpFromOctStr_BNU(hashVal, md, hashLen); + hashValLen = cpMod_BNU(hashVal, hashValLen, pModulus, elemLen); + cpGFpSet(pPoolElm, hashVal, hashValLen, pGFE); + + if( gfec_MakePoint(pPoint, pPoolElm, pEC)) { + /* choose even y-coordinate of the point (see SafeID Specs v2) */ + BNU_CHUNK_T* pY = ECP_POINT_Y(pPoint); + GFP_METHOD(pGFE)->decode(pPoolElm, pY, pGFE); /* due to P(X,Y,Z=1) just decode Y->y */ + if(pPoolElm[0] & 1) + cpGFpNeg(pY, pY, pGFE); + + /* R = [cofactor]R */ + if(ECP_SUBGROUP(pEC)) { + BNU_CHUNK_T* pCofactor = ECP_COFACTOR(pEC); + int cofactorLen = GFP_FELEN(pGFE); + if(!GFP_IS_ONE(pCofactor, cofactorLen)) + gfec_MulPoint(pPoint, pPoint, pCofactor, cofactorLen, /*0,*/ pEC, pScratchBuffer); + } + + cpGFpReleasePool(1, pGFE); + return ippStsNoErr; + } + } + + cpGFpReleasePool(1, pGFE); + return ippStsQuadraticNonResidueErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointhash_backc.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointhash_backc.c new file mode 100644 index 0000000..15201b0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointhash_backc.c @@ -0,0 +1,155 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECSetPointHash() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcphash.h" +#include "pcphash_rmf.h" + +/*F* +// Name: ippsGFpECSetPointHashBackCompatible +// +// Purpose: Constructs a point on an elliptic curve based on the hash of the input message +// This version provide back compatibility with ipp2019update5 and before +// +// Returns: Reason: +// ippStsNullPtrErr pPoint == NULL +// pEC == NULL +// pScratchBuffer == NULL +// (msgLen && !pMsg) +// +// ippStsContextMatchErr invalid pEC->idCtx +// invalid pPoint->idCtx +// +// ippStsBadArgErr !GFP_IS_BASIC(pGFE) +// +// ippStsOutOfRangeErr ECP_POINT_FELEN(pPoint)!=GFP_FELEN() +// +// ippStsQuadraticNonResidueErr square of the Y-coordinate of +// the pPoint is a quadratic non-residue modulo +// +// ippStsLengthErr msgLen<0 +// +// ippStsNoErr no error +// +// Parameters: +// hdr Header of the input message +// pMsg Pointer to the input message +// msgLen Length of the input message +// pPoint Pointer to the IppsGFpECPoint context +// pEC Pointer to the context of the elliptic curve +// hashID ID of the hash algorithm used +// pScratchBuffer Pointer to the scratch buffer +// +// Note: +// Is not a fact that computed point belongs to BP-related subgroup BP +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECSetPointHashBackCompatible,(Ipp32u hdr, const Ipp8u* pMsg, int msgLen, IppsGFpECPoint* pPoint, + IppsGFpECState* pEC, IppHashAlgId hashID, + Ipp8u* pScratchBuffer)) +{ + IppsGFpState* pGF; + gsModEngine* pGFE; + + /* get algorithm id */ + hashID = cpValidHashAlg(hashID); + IPP_BADARG_RET(ippHashAlg_Unknown==hashID, ippStsNotSupportedModeErr); + + /* test message length */ + IPP_BADARG_RET((msgLen<0), ippStsLengthErr); + /* test message pointer */ + IPP_BADARG_RET((msgLen && !pMsg), ippStsNullPtrErr); + + IPP_BAD_PTR3_RET(pPoint, pEC, pScratchBuffer); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + + pGF = ECP_GFP(pEC); + pGFE = GFP_PMA(pGF); + + IPP_BADARG_RET( !GFP_IS_BASIC(pGFE), ippStsBadArgErr ); + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pPoint), ippStsContextMatchErr ); + + IPP_BADARG_RET( ECP_POINT_FELEN(pPoint)!=GFP_FELEN(pGFE), ippStsOutOfRangeErr); + + { + int elemLen = GFP_FELEN(pGFE); + BNU_CHUNK_T* pModulus = GFP_MODULUS(pGFE); + + Ipp8u md[IPP_SHA512_DIGEST_BITSIZE/BYTESIZE]; + int hashLen = cpHashAlgAttr[hashID].hashSize; + BNU_CHUNK_T hashVal[BITS_BNU_CHUNK(IPP_SHA512_DIGEST_BITSIZE)+1]; + int hashValLen; + + IppsHashState hashCtx; + ippsHashInit(&hashCtx, hashID); + + { + BNU_CHUNK_T* pPoolElm = cpGFpGetPool(1, pGFE); + + /* convert hdr => hdrStr */ + BNU_CHUNK_T locHdr = (BNU_CHUNK_T)hdr; + Ipp8u hdrOctStr[sizeof(hdr/*locHdr*/)]; + cpToOctStr_BNU(hdrOctStr, sizeof(hdrOctStr), &locHdr, 1); + + /* compute md = hash(hrd||msg) */ + ippsHashUpdate(hdrOctStr, sizeof(hdrOctStr), &hashCtx); + ippsHashUpdate(pMsg, msgLen, &hashCtx); + ippsHashFinal(md, &hashCtx); + + /* convert hash into the integer */ + hashValLen = cpFromOctStr_BNU(hashVal, md, hashLen); + hashValLen = cpMod_BNU(hashVal, hashValLen, pModulus, elemLen); + cpGFpSet(pPoolElm, hashVal, hashValLen, pGFE); + + if( gfec_MakePoint(pPoint, pPoolElm, pEC)) { + /* set y-coordinate of the point (positive or negative) */ + /* because y is in Montgomery domain the code below is not matched to SafeID Specs v2 */ + BNU_CHUNK_T* pY = ECP_POINT_Y(pPoint); + if(pY[0] & 1) + cpGFpNeg(pY, pY, pGFE); + + /* R = [cofactor]R */ + if(ECP_SUBGROUP(pEC)) { + BNU_CHUNK_T* pCofactor = ECP_COFACTOR(pEC); + int cofactorLen = GFP_FELEN(pGFE); + if(!GFP_IS_ONE(pCofactor, cofactorLen)) + gfec_MulPoint(pPoint, pPoint, pCofactor, cofactorLen, /*0,*/ pEC, pScratchBuffer); + } + + cpGFpReleasePool(1, pGFE); + return ippStsNoErr; + } + } + + cpGFpReleasePool(1, pGFE); + return ippStsQuadraticNonResidueErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointhash_backc_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointhash_backc_rmf.c new file mode 100644 index 0000000..498d490 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointhash_backc_rmf.c @@ -0,0 +1,151 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECSetPointHashBackCompatible_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcphash.h" +#include "pcphash_rmf.h" + + +/*F* +// Name: ippsGFpECSetPointHashBackCompatible_rmf +// +// Purpose: Constructs a point on an elliptic curve based on the hash of the input message +// This version provide back compatibility with ipp2019update5 and before +// +// Returns: Reason: +// ippStsNullPtrErr pPoint == NULL +// pEC == NULL +// pScratchBuffer == NULL +// (msgLen && !pMsg) +// pMethod == NULL +// +// ippStsContextMatchErr invalid pEC->idCtx +// invalid pPoint->idCtx +// +// ippStsBadArgErr !GFP_IS_BASIC(pGFE) +// +// ippStsOutOfRangeErr ECP_POINT_FELEN(pPoint)!=GFP_FELEN() +// +// ippStsQuadraticNonResidueErr square of the Y-coordinate of +// the pPoint is a quadratic non-residue modulo +// +// ippStsLengthErr msgLen<0 +// +// ippStsNoErr no error +// +// Parameters: +// hdr Header of the input message +// pMsg Pointer to the input message +// msgLen Length of the input message +// pPoint Pointer to the IppsGFpECPoint context +// pEC Pointer to the context of the elliptic curve +// pMethod Pointer to the hash method +// pScratchBuffer Pointer to the scratch buffer +// +// Note: +// Is not a fact that computed point belongs to BP-related subgroup BP +// +*F*/ +IPPFUN(IppStatus, ippsGFpECSetPointHashBackCompatible_rmf,(Ipp32u hdr, const Ipp8u* pMsg, int msgLen, IppsGFpECPoint* pPoint, + IppsGFpECState* pEC, const IppsHashMethod* pMethod, + Ipp8u* pScratchBuffer)) +{ + IppsGFpState* pGF; + gsModEngine* pGFE; + + /* test method pointer */ + IPP_BAD_PTR1_RET(pMethod); + + /* test message length */ + IPP_BADARG_RET((msgLen<0), ippStsLengthErr); + /* test message pointer */ + IPP_BADARG_RET((msgLen && !pMsg), ippStsNullPtrErr); + + IPP_BAD_PTR3_RET(pPoint, pEC, pScratchBuffer); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + + pGF = ECP_GFP(pEC); + pGFE = GFP_PMA(pGF); + + IPP_BADARG_RET( !GFP_IS_BASIC(pGFE), ippStsBadArgErr ); + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pPoint), ippStsContextMatchErr ); + + IPP_BADARG_RET( ECP_POINT_FELEN(pPoint)!=GFP_FELEN(pGFE), ippStsOutOfRangeErr); + + { + int elemLen = GFP_FELEN(pGFE); + BNU_CHUNK_T* pModulus = GFP_MODULUS(pGFE); + + Ipp8u md[IPP_SHA512_DIGEST_BITSIZE/BYTESIZE]; + int hashLen = pMethod->hashLen; + BNU_CHUNK_T hashVal[BITS_BNU_CHUNK(IPP_SHA512_DIGEST_BITSIZE)+1]; + int hashValLen; + + IppsHashState_rmf hashCtx; + ippsHashInit_rmf(&hashCtx, pMethod); + + { + BNU_CHUNK_T* pPoolElm = cpGFpGetPool(1, pGFE); + + /* convert hdr => hdrStr */ + BNU_CHUNK_T locHdr = (BNU_CHUNK_T)hdr; + Ipp8u hdrOctStr[sizeof(hdr/*locHdr*/)]; + cpToOctStr_BNU(hdrOctStr, sizeof(hdrOctStr), &locHdr, 1); + + /* compute md = hash(hrd||msg) */ + ippsHashUpdate_rmf(hdrOctStr, sizeof(hdrOctStr), &hashCtx); + ippsHashUpdate_rmf(pMsg, msgLen, &hashCtx); + ippsHashFinal_rmf(md, &hashCtx); + + /* convert hash into the integer */ + hashValLen = cpFromOctStr_BNU(hashVal, md, hashLen); + hashValLen = cpMod_BNU(hashVal, hashValLen, pModulus, elemLen); + cpGFpSet(pPoolElm, hashVal, hashValLen, pGFE); + + if( gfec_MakePoint(pPoint, pPoolElm, pEC)) { + /* set y-coordinate of the point (positive or negative) */ + /* because y is in Montgomery domain the code below is not matched to SafeID Specs v2 */ + BNU_CHUNK_T* pY = ECP_POINT_Y(pPoint); + if(pY[0] & 1) + cpGFpNeg(pY, pY, pGFE); + + /* update point if cofactor>1 */ + if(ECP_SUBGROUP(pEC)) + gfec_MulPoint(pPoint, pPoint, ECP_COFACTOR(pEC), GFP_FELEN(pGFE), /*0,*/ pEC, pScratchBuffer); + + cpGFpReleasePool(1, pGFE); + return ippStsNoErr; + } + } + + cpGFpReleasePool(1, pGFE); + return ippStsQuadraticNonResidueErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointhash_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointhash_rmf.c new file mode 100644 index 0000000..044ddb5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointhash_rmf.c @@ -0,0 +1,150 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECSetPointHash_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcphash.h" +#include "pcphash_rmf.h" + + +/*F* +// Name: ippsGFpECSetPointHash_rmf +// +// Purpose: Constructs a point on an elliptic curve based on the hash of the input message +// +// Returns: Reason: +// ippStsNullPtrErr pPoint == NULL +// pEC == NULL +// pScratchBuffer == NULL +// (msgLen && !pMsg) +// pMethod == NULL +// +// ippStsContextMatchErr invalid pEC->idCtx +// invalid pPoint->idCtx +// +// ippStsBadArgErr !GFP_IS_BASIC(pGFE) +// +// ippStsOutOfRangeErr ECP_POINT_FELEN(pPoint)!=GFP_FELEN() +// +// ippStsQuadraticNonResidueErr square of the Y-coordinate of +// the pPoint is a quadratic non-residue modulo +// +// ippStsLengthErr msgLen<0 +// +// ippStsNoErr no error +// +// Parameters: +// hdr Header of the input message +// pMsg Pointer to the input message +// msgLen Length of the input message +// pPoint Pointer to the IppsGFpECPoint context +// pEC Pointer to the context of the elliptic curve +// pMethod Pointer to the hash method +// pScratchBuffer Pointer to the scratch buffer +// +// Note: +// Is not a fact that computed point belongs to BP-related subgroup BP +// +*F*/ +IPPFUN(IppStatus, ippsGFpECSetPointHash_rmf,(Ipp32u hdr, const Ipp8u* pMsg, int msgLen, IppsGFpECPoint* pPoint, + IppsGFpECState* pEC, const IppsHashMethod* pMethod, + Ipp8u* pScratchBuffer)) +{ + IppsGFpState* pGF; + gsModEngine* pGFE; + + /* test method pointer */ + IPP_BAD_PTR1_RET(pMethod); + + /* test message length */ + IPP_BADARG_RET((msgLen<0), ippStsLengthErr); + /* test message pointer */ + IPP_BADARG_RET((msgLen && !pMsg), ippStsNullPtrErr); + + IPP_BAD_PTR3_RET(pPoint, pEC, pScratchBuffer); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + + pGF = ECP_GFP(pEC); + pGFE = GFP_PMA(pGF); + + IPP_BADARG_RET( !GFP_IS_BASIC(pGFE), ippStsBadArgErr ); + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pPoint), ippStsContextMatchErr ); + + IPP_BADARG_RET( ECP_POINT_FELEN(pPoint)!=GFP_FELEN(pGFE), ippStsOutOfRangeErr); + + { + int elemLen = GFP_FELEN(pGFE); + BNU_CHUNK_T* pModulus = GFP_MODULUS(pGFE); + + Ipp8u md[IPP_SHA512_DIGEST_BITSIZE/BYTESIZE]; + int hashLen = pMethod->hashLen; + BNU_CHUNK_T hashVal[BITS_BNU_CHUNK(IPP_SHA512_DIGEST_BITSIZE)+1]; + int hashValLen; + + IppsHashState_rmf hashCtx; + ippsHashInit_rmf(&hashCtx, pMethod); + + { + BNU_CHUNK_T* pPoolElm = cpGFpGetPool(1, pGFE); + + /* convert hdr => hdrStr */ + BNU_CHUNK_T locHdr = (BNU_CHUNK_T)hdr; + Ipp8u hdrOctStr[sizeof(hdr/*locHdr*/)]; + cpToOctStr_BNU(hdrOctStr, sizeof(hdrOctStr), &locHdr, 1); + + /* compute md = hash(hrd||msg) */ + ippsHashUpdate_rmf(hdrOctStr, sizeof(hdrOctStr), &hashCtx); + ippsHashUpdate_rmf(pMsg, msgLen, &hashCtx); + ippsHashFinal_rmf(md, &hashCtx); + + /* convert hash into the integer */ + hashValLen = cpFromOctStr_BNU(hashVal, md, hashLen); + hashValLen = cpMod_BNU(hashVal, hashValLen, pModulus, elemLen); + cpGFpSet(pPoolElm, hashVal, hashValLen, pGFE); + + if( gfec_MakePoint(pPoint, pPoolElm, pEC)) { + /* choose even y-coordinate of the point (see SafeID Specs v2) */ + BNU_CHUNK_T* pY = ECP_POINT_Y(pPoint); + GFP_METHOD(pGFE)->decode(pPoolElm, pY, pGFE); /* due to P(X,Y,Z=1) just decode Y->y */ + if(pPoolElm[0] & 1) + cpGFpNeg(pY, pY, pGFE); + + /* update point if cofactor>1 */ + if(ECP_SUBGROUP(pEC)) + gfec_MulPoint(pPoint, pPoint, ECP_COFACTOR(pEC), GFP_FELEN(pGFE), /*0,*/ pEC, pScratchBuffer); + + cpGFpReleasePool(1, pGFE); + return ippStsNoErr; + } + } + + cpGFpReleasePool(1, pGFE); + return ippStsQuadraticNonResidueErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointoctstring.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointoctstring.c new file mode 100644 index 0000000..fd449f6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointoctstring.c @@ -0,0 +1,89 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECSetPointOctString() +// +*/ + +#include "pcpgfpecessm2.h" +#include "pcpgfpecstuff.h" + +/*F* +// Name: ippsGFpECSetPointOctString +// +// Purpose: Converts x||y octstring into a point on EC +// +// Returns: Reason: +// ippStsNullPtrErr pPoint == NULL / pEC == NULL / pStr == NULL +// ippStsContextMatchErr pEC, pPoint invalid context +// ippStsNotSupportedModeErr pGFE->extdegree > 1 +// ippStsSizeErr strLen is not equal to double GFp element +// ippStsOutOfRangeErr X or Y from the string exceeds the EC's field modulus or the point does not belong to the EC +// ippStsNoErr no errors +// +// Parameters: +// pStr pointer to the string to read from +// strLen length of the string +// pPoint pointer to output point +// pEC EC ctx +// +*F*/ +IPPFUN(IppStatus, ippsGFpECSetPointOctString, (const Ipp8u* pStr, + int strLen, IppsGFpECPoint* pPoint, IppsGFpECState* pEC)) { + IPP_BAD_PTR3_RET(pPoint, pEC, pStr); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + + { + gsModEngine* pGFE = pEC->pGF->pGFE; + IppsGFpInfo gfi; + ippsGFpGetInfo(&gfi, pEC->pGF); + + { + int elemLenBits = gfi.basicGFdegree * gfi.basicElmBitSize; + int elemLenBytes = BITS2WORD8_SIZE(elemLenBits); + int elemLenChunks = BITS_BNU_CHUNK(elemLenBits); + IPP_BADARG_RET(strLen != elemLenBytes * 2, ippStsSizeErr); + + { + IppStatus ret; + IppsGFpElement ptX, ptY; + cpGFpElementConstruct(&ptX, cpGFpGetPool(1, pGFE), elemLenChunks); + cpGFpElementConstruct(&ptY, cpGFpGetPool(1, pGFE), elemLenChunks); + + ret = ippsGFpSetElementOctString(pStr, elemLenBytes, &ptX, pEC->pGF); + if (ippStsNoErr == ret) { + pStr += elemLenBytes; + ret = ippsGFpSetElementOctString(pStr, elemLenBytes, &ptY, pEC->pGF); + } + if (ippStsNoErr == ret) { + ret = ippsGFpECSetPoint(&ptX, &ptY, pPoint, pEC); + } + + cpGFpReleasePool(2, pGFE); /* release ptX and ptY from the pool */ + + return ret; + } + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointrand.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointrand.c new file mode 100644 index 0000000..5e281a4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointrand.c @@ -0,0 +1,135 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECSetPointRandom() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcphash.h" +#include "pcphash_rmf.h" + +/*F* +// Name: ippsGFpECSetPointRandom +// +// Purpose: Sets the coordinates of a point on an elliptic curve to random values +// +// Returns: Reason: +// ippStsNullPtrErr pPoint == NULL +// pEC == NULL +// pScratchBuffer == NULL +// +// ippStsContextMatchErr invalid pEC->idCtx +// NULL == pEC->subgroup +// invalid pPoint->idCtx +// +// ippStsOutOfRangeErr ECP_POINT_FELEN(pPoint)!=GFP_FELEN() +// +// ippStsNoErr no error +// +// Parameters: +// pPoint Pointer to the IppsGFpECPoint context +// pEC Pointer to the context of the elliptic curve +// rndFunc Pesudorandom number generator +// pRndParam Pointer to the pseudorandom number generator context +// pScratchBuffer Pointer to the scratch buffer +// +// Note: +// Is not a fact that computed point belongs to BP-related subgroup BP +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECSetPointRandom,(IppsGFpECPoint* pPoint, IppsGFpECState* pEC, + IppBitSupplier rndFunc, void* pRndParam, + Ipp8u* pScratchBuffer)) +{ + IppsGFpState* pGF; + gsModEngine* pGFE; + + IPP_BAD_PTR3_RET(pPoint, pEC, pScratchBuffer); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pPoint), ippStsContextMatchErr ); + + pGF = ECP_GFP(pEC); + pGFE = GFP_PMA(pGF); + + IPP_BADARG_RET( ECP_POINT_FELEN(pPoint)!=GFP_FELEN(pGFE), ippStsOutOfRangeErr); + + IPP_BAD_PTR2_RET(rndFunc, pRndParam); + + { + int internal_err; + + if( GFP_IS_BASIC(pGFE) ) { + BNU_CHUNK_T* pElm = cpGFpGetPool(1, pGFE); + + do { /* get random X */ + internal_err = NULL==cpGFpRand(pElm, pGFE, rndFunc, pRndParam); + } while( !internal_err && !gfec_MakePoint(pPoint, pElm, pEC) ); + + cpGFpReleasePool(1, pGFE); + + /* R = [cofactor]R */ + if(!internal_err && ECP_SUBGROUP(pEC)) { + BNU_CHUNK_T* pCofactor = ECP_COFACTOR(pEC); + int cofactorLen = GFP_FELEN(pGFE); + if(!GFP_IS_ONE(pCofactor, cofactorLen)) + gfec_MulPoint(pPoint, pPoint, ECP_COFACTOR(pEC), GFP_FELEN(pGFE), /*0,*/ pEC, pScratchBuffer); + } + } + + else { + IPP_BADARG_RET(!ECP_SUBGROUP(pEC), ippStsContextMatchErr); + { + /* number of bits being generated */ + int generatedBits = ECP_ORDBITSIZE(pEC) + GFP_RAND_ADD_BITS; + int generatedLen = BITS_BNU_CHUNK(generatedBits); + + /* allocate random exponent */ + int poolElements = (generatedLen + GFP_PELEN(pGFE) -1) / GFP_PELEN(pGFE); + BNU_CHUNK_T* pExp = cpGFpGetPool(poolElements, pGFE); + + /* setup copy of the base point */ + IppsGFpECPoint G; + cpEcGFpInitPoint(&G, ECP_G(pEC),ECP_AFFINE_POINT|ECP_FINITE_POINT, pEC); + + /* get random bits */ + internal_err = ippStsNoErr != rndFunc((Ipp32u*)pExp, generatedBits, pRndParam); + + if(!internal_err) { + /* reduce with respect to order value */ + int nsE = cpMod_BNU(pExp, generatedLen, MOD_MODULUS(ECP_MONT_R(pEC)), BITS_BNU_CHUNK(ECP_ORDBITSIZE(pEC))); + /* compute random point */ + gfec_MulPoint(pPoint, &G, pExp, nsE, pEC, pScratchBuffer); + } + + cpGFpReleasePool(poolElements, pGFE); + } + } + + return internal_err? ippStsErr : ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointreg.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointreg.c new file mode 100644 index 0000000..eed0ecf --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetpointreg.c @@ -0,0 +1,106 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECSetPointRegular() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcphash.h" +#include "pcphash_rmf.h" + +/*F* +// Name: ippsGFpECSetPointRegular +// +// Purpose: Sets a point with Big Number coordinates +// +// Returns: Reason: +// ippStsNullPtrErr pPoint == NULL +// pEC == NULL +// pX == NULL +// pY == NULL +// +// ippStsContextMatchErr invalid pEC->idCtx +// invalid pPoint->idCtx +// invalid pX->idCtx +// invalid pY->idCtx +// +// ippStsOutOfRangeErr GFP_FELEN() < pX <= 0 +// GFP_FELEN() < pY <= 0 +// ECP_POINT_FELEN(pPoint)!=GFP_FELEN() +// +// ippStsNoErr no error +// +// Parameters: +// pX, pY Pointers to the X and Y coordinates of a point on the elliptic curve +// pPoint Pointer to the IppsGFpECPoint context +// pEC Pointer to the context of the elliptic curve +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECSetPointRegular,(const IppsBigNumState* pX, const IppsBigNumState* pY, + IppsGFpECPoint* pPoint, + IppsGFpECState* pEC)) +{ + IPP_BAD_PTR2_RET(pPoint, pEC); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pPoint), ippStsContextMatchErr ); + + IPP_BAD_PTR2_RET(pX, pY); + IPP_BADARG_RET( !BN_VALID_ID(pX), ippStsContextMatchErr ); + IPP_BADARG_RET( !BN_VALID_ID(pY), ippStsContextMatchErr ); + IPP_BADARG_RET( !BN_POSITIVE(pX), ippStsOutOfRangeErr); + IPP_BADARG_RET( !BN_POSITIVE(pY), ippStsOutOfRangeErr); + + { + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elemLen = GFP_FELEN(pGFE); + + IPP_BADARG_RET( !GFP_IS_BASIC(pGFE), ippStsBadArgErr ) + + IPP_BADARG_RET( BN_SIZE(pX) > elemLen, ippStsOutOfRangeErr); + IPP_BADARG_RET( BN_SIZE(pY) > elemLen, ippStsOutOfRangeErr); + IPP_BADARG_RET( ECP_POINT_FELEN(pPoint)!=elemLen, ippStsOutOfRangeErr); + + { + BNU_CHUNK_T* pointX = ECP_POINT_X(pPoint); + BNU_CHUNK_T* pointY = ECP_POINT_Y(pPoint); + BNU_CHUNK_T* pointZ = ECP_POINT_Z(pPoint); + + cpGFpElementCopyPad(pointX, elemLen, BN_NUMBER(pX), BN_SIZE(pX)); + cpGFpElementCopyPad(pointY, elemLen, BN_NUMBER(pY), BN_SIZE(pY)); + cpGFpElementCopy(pointZ, MOD_MNT_R(pGFE), elemLen); + + if( cpGFpSet(pointX, pointX, elemLen, pGFE) && cpGFpSet(pointY, pointY, elemLen, pGFE) ) + ECP_POINT_FLAGS(pPoint) = ECP_AFFINE_POINT | ECP_FINITE_POINT; + else + gfec_SetPointAtInfinity(pPoint); + + return ippStsNoErr; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetsubgroup.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetsubgroup.c new file mode 100644 index 0000000..0b820bc --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsetsubgroup.c @@ -0,0 +1,120 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECSetSubgroup() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsGFpECSetSubgroup +// +// Purpose: Sets up the parameters defining an elliptic curve points subgroup. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pX +// NULL == pY +// NULL == pOrder +// NULL == pCofactor +// +// ippStsContextMatchErr invalid pEC->idCtx +// invalid pX->idCtx +// invalid pY->idCtx +// invalid pOrder->idCtx +// invalid pCofactor->idCtx +// +// ippStsBadArgErr pOrder <= 0 +// pCofactor <= 0 +// +// ippStsOutOfRangeErr GFPE_ROOM(pX)!=GFP_FELEN(pGFE) +// GFPE_ROOM(pY)!=GFP_FELEN(pGFE) +// +// ippStsRangeErr orderBitSize>maxOrderBits +// cofactorBitSize>elemLen*BITSIZE(BNU_CHUNK_T) +// +// ippStsNoErr no error +// +// Parameters: +// pX, pY Pointers to the X and Y coordinates of the base point of the elliptic curve +// pOrder Pointer to the big number context storing the order of the base point. +// pCofactor Pointer to the big number context storing the cofactor. +// pEC Pointer to the context of the elliptic curve. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECSetSubgroup,(const IppsGFpElement* pX, const IppsGFpElement* pY, + const IppsBigNumState* pOrder, + const IppsBigNumState* pCofactor, + IppsGFpECState* pEC)) +{ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + + IPP_BAD_PTR2_RET(pX, pY); + IPP_BADARG_RET( !GFPE_VALID_ID(pX), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pY), ippStsContextMatchErr ); + + IPP_BAD_PTR2_RET(pOrder, pCofactor); + IPP_BADARG_RET(!BN_VALID_ID(pOrder), ippStsContextMatchErr); + IPP_BADARG_RET(BN_SIGN(pOrder)!= IppsBigNumPOS, ippStsBadArgErr); + + IPP_BADARG_RET(!BN_VALID_ID(pCofactor), ippStsContextMatchErr); + IPP_BADARG_RET(BN_SIGN(pCofactor)!= IppsBigNumPOS, ippStsBadArgErr); + + { + gsModEngine* pGFE = GFP_PMA(ECP_GFP(pEC)); + int elemLen = GFP_FELEN(pGFE); + + IPP_BADARG_RET( GFPE_ROOM(pX)!=GFP_FELEN(pGFE), ippStsOutOfRangeErr); + IPP_BADARG_RET( GFPE_ROOM(pY)!=GFP_FELEN(pGFE), ippStsOutOfRangeErr); + + gfec_SetPoint(ECP_G(pEC), GFPE_DATA(pX), GFPE_DATA(pY), pEC); + + { + int maxOrderBits = 1+ cpGFpBasicDegreeExtension(pGFE) * GFP_FEBITLEN(cpGFpBasic(pGFE)); + BNU_CHUNK_T* pOrderData = BN_NUMBER(pOrder); + int orderLen= BN_SIZE(pOrder); + int orderBitSize = BITSIZE_BNU(pOrderData, orderLen); + IPP_BADARG_RET(orderBitSize>maxOrderBits, ippStsRangeErr) + + /* set actual size of order and re-init engine */ + ECP_ORDBITSIZE(pEC) = orderBitSize; + gsModEngineInit(ECP_MONT_R(pEC),(Ipp32u*)pOrderData, orderBitSize, MONT_DEFAULT_POOL_LENGTH, gsModArithMont()); + } + + { + BNU_CHUNK_T* pCofactorData = BN_NUMBER(pCofactor); + int cofactorLen= BN_SIZE(pCofactor); + int cofactorBitSize = BITSIZE_BNU(pCofactorData, cofactorLen); + IPP_BADARG_RET(cofactorBitSize>elemLen*BITSIZE(BNU_CHUNK_T), ippStsRangeErr) + COPY_BNU(ECP_COFACTOR(pEC), pCofactorData, cofactorLen); + } + + ECP_SUBGROUP(pEC) = 1; + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsigndsa.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsigndsa.c new file mode 100644 index 0000000..a2241fa --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsigndsa.c @@ -0,0 +1,252 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// ippsGFpECSignDSA() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" +#include "pcpgfpmethod.h" +#include "pcpgfpstuff.h" + +/*F* +// Name: ippsGFpECSignDSA +// +// Purpose: DSA Signature Generation. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pMsgDigest +// NULL == pRegPrivate +// NULL == pEphPrivate +// NULL == pSignR +// NULL == pSignS +// NULL == pScratchBuffer +// +// ippStsContextMatchErr illegal pEC->idCtx +// pEC->subgroup == NULL +// illegal pMsgDigest->idCtx +// illegal pRegPrivate->idCtx +// illegal pEphPrivate->idCtx +// illegal pSignR->idCtx +// illegal pSignS->idCtx +// +// ippStsInvalidPrivateKey 0 >= RegPrivate +// RegPrivate >= order +// +// 0 >= EphPrivate +// EphPrivate >= order +// +// ippStsMessageErr MsgDigest >= order +// MsgDigest < 0 +// +// ippStsRangeErr not enough room for: +// signR +// signS +// +// ippStsEphemeralKeyErr (0==signR) || (0==signS) +// +// ippStsNotSupportedModeErr pGFE->extdegree > 1 +// +// +// ippStsNoErr no errors +// +// Parameters: +// pMsgDigest pointer to the message representative to be signed +// pRegPrivate pointer to the regular private key +// pEphPrivate pointer to the ephemeral private key +// pSignR,pSignS pointer to the signature +// pEC pointer to the EC context +// pScratchBuffer pointer to buffer (1 mul_point operation) +// +*F*/ +IPPFUN(IppStatus, ippsGFpECSignDSA,(const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pRegPrivate, + IppsBigNumState* pEphPrivate, + IppsBigNumState* pSignR, IppsBigNumState* pSignS, + IppsGFpECState* pEC, + Ipp8u* pScratchBuffer)) +{ + IppsGFpState* pGF; + gsModEngine* pMontP; + + /* EC context and buffer */ + IPP_BAD_PTR2_RET(pEC, pScratchBuffer); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + IPP_BADARG_RET(!ECP_SUBGROUP(pEC), ippStsContextMatchErr); + + pGF = ECP_GFP(pEC); + pMontP = GFP_PMA(pGF); + IPP_BADARG_RET(1= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512IFMA)) { + switch (ECP_MODULUS_ID(pEC)) { + case cpID_PrimeP256r1: { + sts = gfec_SignDSA_nistp256_avx512(pMsgDigest, pRegPrivate, pEphPrivate, pSignR, pSignS, pEC, pScratchBuffer); + goto exit; + break; + } + case cpID_PrimeP384r1: { + sts = gfec_SignDSA_nistp384_avx512(pMsgDigest, pRegPrivate, pEphPrivate, pSignR, pSignS, pEC, pScratchBuffer); + goto exit; + break; + } + case cpID_PrimeP521r1: { + sts = gfec_SignDSA_nistp521_avx512(pMsgDigest, pRegPrivate, pEphPrivate, pSignR, pSignS, pEC, pScratchBuffer); + goto exit; + break; + } + default: + /* Go to default implementation below */ + break; + } + } /* no else */ +#endif // (_IPP32E >= _IPP32E_K1) + { + int elmLen = GFP_FELEN(pMontP); + int ns; + + /* compute ephemeral public key */ + IppsGFpECPoint ephPublic; + cpEcGFpInitPoint(&ephPublic, cpEcGFpGetPool(1, pEC), 0, pEC); + gfec_MulBasePoint(&ephPublic, + BN_NUMBER(pEphPrivate), BN_SIZE(pEphPrivate), + pEC, pScratchBuffer); + + /* + // signR = int(ephPublic.x) (mod order) + */ + { + BNU_CHUNK_T* buffer = gsModPoolAlloc(pMontP, 1); + IPP_BAD_PTR1_RET(buffer); // buffer can be NULL, stop processing + gfec_GetPoint(buffer, NULL, &ephPublic, pEC); + GFP_METHOD(pMontP)->decode(buffer, buffer, pMontP); + ns = cpMod_BNU(buffer, elmLen, pOrder, ordLen); + cpGFpElementCopyPad(dataC, ordLen, buffer, ns); + gsModPoolFree(pMontP, 1); + } + cpEcGFpReleasePool(1, pEC); + + if(!GFP_IS_ZERO(dataC, ordLen)) { + /* + // signS = (1/ephPrivate)*(pMsgDigest + private*signR) (mod order) + */ + + /* copy and expand message is being signed and reduce just in case */ + ZEXPAND_COPY_BNU(buffF, ordLen, pMsgData, msgLen); + cpModSub_BNU(buffF, buffF, pOrder, pOrder, ordLen, buffT); + + /* private representation in Montgomery domain */ + ZEXPAND_COPY_BNU(dataD, ordLen, pPriData, priLen); + GFP_METHOD(pMontR)->encode(dataD, dataD, pMontR); + + /* (private*signX) in regular domain */ + GFP_METHOD(pMontR)->mul(dataD, dataD, dataC, pMontR); + + /* pMsgDigest + private*signX */ + cpModAdd_BNU(dataD, dataD, buffF, pOrder, ordLen, buffT); + + if(!GFP_IS_ZERO(dataD, ordLen)) { + /* (1/ephPrivate) in Montgomery domain */ + ZEXPAND_COPY_BNU(buffT, ordLen, pEphData, ephLen); + gs_mont_inv(buffT, buffT, pMontR, alm_mont_inv_ct); + + /* (1/ephPrivate)*(pMsgDigest + private*signS) */ + GFP_METHOD(pMontR)->mul(dataD, dataD, buffT, pMontR); + + /* signR */ + ns = ordLen; + FIX_BNU(dataC, ns); + BN_SIGN(pSignR) = ippBigNumPOS; + BN_SIZE(pSignR) = ns; + /* signS */ + ns = ordLen; + FIX_BNU(dataD, ns); + BN_SIGN(pSignS) = ippBigNumPOS; + BN_SIZE(pSignS) = ns; + + sts = ippStsNoErr; + } + } + } + +#if (_IPP32E >= _IPP32E_K1) +exit: +#endif + /* clear ephemeral private key */ + cpBN_zero(pEphPrivate); + + return sts; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsignnr.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsignnr.c new file mode 100644 index 0000000..72869df --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsignnr.c @@ -0,0 +1,206 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// ippsGFpECSignNR() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsGFpECSignNR +// +// Purpose: NR Signature Generation. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pMsgDigest +// NULL == pRegPrivate +// NULL == pEphPrivate +// NULL == pSignR +// NULL == pSignS +// NULL == pScratchBuffer +// +// ippStsContextMatchErr illegal pEC->idCtx +// pEC->subgroup == NULL +// illegal pMsgDigest->idCtx +// illegal pRegPrivate->idCtx +// illegal pEphPrivate->idCtx +// illegal pSignR->idCtx +// illegal pSignS->idCtx +// +// ippStsInvalidPrivateKey 0 >= RegPrivate +// RegPrivate >= order +// +// 0 >= EphPrivate +// EphPrivate >= order +// +// ippStsMessageErr pMsgDigest < 0 +// pMsgDigest >= order +// +// ippStsRangeErr not enough room for: +// signC +// signD +// +// ippStsEphemeralKeyErr (0==signR) || (0==signS) +// +// ippStsNotSupportedModeErr 1decode(buffer, buffer, pMontP); + ns = cpMod_BNU(buffer, elmLen, pOrder, ordLen); + cpGFpElementCopyPad(dataC, ordLen, buffer, ns); + gsModPoolFree(pMontP, 1); + } + cpEcGFpReleasePool(1, pEC); + + /* C = (ephPublic.x + msg) mod order */ + ZEXPAND_COPY_BNU(buffF, ordLen, pMsgData, msgLen); + cpModAdd_BNU(dataC, dataC, buffF, pOrder, ordLen, dataD); + + if(!GFP_IS_ZERO(dataC, ordLen)) { + + /* signS = (eph_private - private*signR) (mod order) */ + ZEXPAND_COPY_BNU(dataD, ordLen, pPriData, priLen); + GFP_METHOD(pMontR)->encode(dataD, dataD, pMontR); + GFP_METHOD(pMontR)->mul(dataD, dataD, dataC, pMontR); + ZEXPAND_COPY_BNU(buffF, ordLen, pEphData, ephLen); + cpModSub_BNU(dataD, buffF, dataD, pOrder, ordLen, buffT); + + /* signR */ + ns = ordLen; + FIX_BNU(dataC, ns); + BN_SIGN(pSignR) = ippBigNumPOS; + BN_SIZE(pSignR) = ns; + /* signS */ + ns = ordLen; + FIX_BNU(dataD, ns); + BN_SIGN(pSignS) = ippBigNumPOS; + BN_SIZE(pSignS) = ns; + + sts = ippStsNoErr; + } + + /* clear ephemeral private key */ + cpBN_zero(pEphPrivate); + + return sts; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsignsm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsignsm2.c new file mode 100644 index 0000000..cba8e9d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecsignsm2.c @@ -0,0 +1,226 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// ippsGFpECSignSM2() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsGFpECSignSM2 +// +// Purpose: SM2 Signature Generation. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pMsgDigest +// NULL == pRegPrivate +// NULL == pEphPrivate +// NULL == pSignR +// NULL == pSignS +// NULL == pScratchBuffer +// +// ippStsContextMatchErr illegal pEC->idCtx +// pEC->subgroup == NULL +// illegal pMsgDigest->idCtx +// illegal pRegPrivate->idCtx +// illegal pEphPrivate->idCtx +// illegal pSignR->idCtx +// illegal pSignS->idCtx +// +// ippStsInvalidPrivateKey (1 + regPrivate) >= order +// +// ippStsMessageErr MsgDigest >= order +// MsgDigest < 0 +// +// ippStsRangeErr not enough room for: +// signR +// signS +// +// ippStsEphemeralKeyErr (signR + ephPrivate) == order +// (0==signR) || (0==signS) +// +// ippStsNotSupportedModeErr 1= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512IFMA) && ECP_MODULUS_ID(pEC) == cpID_PrimeTPM_SM2) { + sts = gfec_Sign_sm2_avx512(pMsgDigest, pRegPrivate, pEphPrivate, pSignR, pSignS, pEC, pScratchBuffer); + goto exit; + } /* no else */ +#endif // (_IPP32E >= _IPP32E_K1) + { + int elmLen = GFP_FELEN(pMontP); + int ns; + + /* compute ephemeral public key */ + IppsGFpECPoint ephPublic; + cpEcGFpInitPoint(&ephPublic, cpEcGFpGetPool(1, pEC), 0, pEC); + gfec_MulBasePoint(&ephPublic, + pEphData, ephLen, + pEC, pScratchBuffer); + + /* extract X component: ephPublicX = (ephPublic.x) mod order */ + { + BNU_CHUNK_T* buffer = gsModPoolAlloc(pMontP, 1); + IPP_BAD_PTR1_RET(buffer); // buffer can be NULL, stop processing + gfec_GetPoint(buffer, NULL, &ephPublic, pEC); + GFP_METHOD(pMontP)->decode(buffer, buffer, pMontP); + ns = cpMod_BNU(buffer, elmLen, pOrder, ordLen); + cpGFpElementCopyPad(dataR, ordLen, buffer, ns); + gsModPoolFree(pMontP, 1); + } + cpEcGFpReleasePool(1, pEC); + + /* compute R signature component: r = (msg + ephPublic.X) mod order */ + ZEXPAND_COPY_BNU(buffS, ordLen, pMsgData, msgLen); + cpModSub_BNU(buffS, buffS, pOrder, pOrder, ordLen, buffR); + cpModAdd_BNU(dataR, dataR, buffS, pOrder, ordLen, buffR); + + /* t = (r+ephPrivate) mod order */ + ZEXPAND_COPY_BNU(buffR,ordLen, pEphData, ephLen); + cpModAdd_BNU(buffR, buffR, dataR, pOrder, ordLen, buffS); + + /* check if r!=0 and t!=0 */ + if(!GFP_IS_ZERO(dataR, ordLen) && !GFP_IS_ZERO(buffR, ordLen)) { + + /* S = (1+regPrivate)^1 *(ephPrivate-r*regPrivate) mod order */ + ZEXPAND_COPY_BNU(buffS, ordLen, pPriData, priLen); + GFP_METHOD(pMontR)->encode(buffS, buffS, pMontR); /* mont(regPrivate) */ + GFP_METHOD(pMontR)->mul(buffS, buffS, dataR, pMontR); /* r*mont(regPrivate) */ + ZEXPAND_COPY_BNU(buffR, ordLen, pEphData, ephLen); + cpModSub_BNU(buffR, buffR, buffS, pOrder, ordLen, buffS); /* ephPrivate-r*mont(regPrivate) */ + + gs_mont_inv(dataS, dataS, pMontR, alm_mont_inv_ct); /* 1/(1+regPrivate) */ + GFP_METHOD(pMontR)->mul(dataS, dataS, buffR, pMontR); + + if( !GFP_IS_ZERO(dataS, ordLen)) { + /* signR */ + ns = ordLen; + FIX_BNU(dataR, ns); + BN_SIGN(pSignR) = ippBigNumPOS; + BN_SIZE(pSignR) = ns; + /* signS */ + ns = ordLen; + FIX_BNU(dataS, ns); + BN_SIGN(pSignS) = ippBigNumPOS; + BN_SIZE(pSignS) = ns; + + sts = ippStsNoErr; + } + } + } + +#if (_IPP32E >= _IPP32E_K1) +exit: +#endif + /* clear ephemeral private key */ + cpBN_zero(pEphPrivate); + + return sts; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecstuff.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecstuff.h new file mode 100644 index 0000000..3dec864 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecstuff.h @@ -0,0 +1,407 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal EC over GF(p^m) basic Definitions & Function Prototypes +// +// +*/ + +#if !defined(_CP_ECGFP_H_) +#define _CP_ECGFP_H_ + +#include "pcpgfpmethod.h" +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcpmask_ct.h" + +#define _LEGACY_ECCP_SUPPORT_ + +/* +// EC over GF(p) Point context +*/ +typedef struct _cpGFpECPoint { + Ipp32u idCtx; /* EC Point identifier */ + int flags; /* flags: affine */ + int elementSize; /* size of each coordinate */ + BNU_CHUNK_T* pData; /* coordinate X, Y, Z */ +} cpGFPECPoint; + +/* +// Contet Access Macros +*/ +#define ECP_POINT_SET_ID(ctx) ((ctx)->idCtx = (Ipp32u)idCtxGFPPoint ^ (Ipp32u)IPP_UINT_PTR(ctx)) +#define ECP_POINT_FLAGS(ctx) ((ctx)->flags) +#define ECP_POINT_FELEN(ctx) ((ctx)->elementSize) +#define ECP_POINT_DATA(ctx) ((ctx)->pData) +#define ECP_POINT_X(ctx) ((ctx)->pData) +#define ECP_POINT_Y(ctx) ((ctx)->pData+(ctx)->elementSize) +#define ECP_POINT_Z(ctx) ((ctx)->pData+(ctx)->elementSize*2) +#define ECP_POINT_VALID_ID(ctx) ((((ctx)->idCtx) ^ (Ipp32u)IPP_UINT_PTR(ctx)) == (Ipp32u)idCtxGFPPoint) + +/* point flags */ +#define ECP_AFFINE_POINT (1) +#define ECP_FINITE_POINT (2) + +#define IS_ECP_AFFINE_POINT(ctx) (ECP_POINT_FLAGS((ctx))&ECP_AFFINE_POINT) +#define SET_ECP_AFFINE_POINT(ctx) (ECP_POINT_FLAGS((ctx))|ECP_AFFINE_POINT) +#define SET_ECP_PROJECTIVE_POINT(ctx) (ECP_POINT_FLAGS((ctx))&~ECP_AFFINE_POINT) + +#define IS_ECP_FINITE_POINT(ctx) (ECP_POINT_FLAGS((ctx))&ECP_FINITE_POINT) +#define SET_ECP_FINITE_POINT(ctx) (ECP_POINT_FLAGS((ctx))|ECP_FINITE_POINT) +#define SET_ECP_INFINITE_POINT(ctx) (ECP_POINT_FLAGS((ctx))&~ECP_FINITE_POINT) + +/* +// define using projective coordinates +*/ +#define JACOBIAN (0) +#define HOMOGENEOUS (1) +#define ECP_PROJECTIVE_COORD JACOBIAN +//#define ECP_PROJECTIVE_COORD HOMOGENEOUS + +#if (ECP_PROJECTIVE_COORD != JACOBIAN && ECP_PROJECTIVE_COORD != HOMOGENEOUS) + #error ECP_PROJECTIVE_COORD should be either JACOBIAN or HOMOGENEOUS type +#endif + + +/* +// pre-computed Base Point descriptor +*/ +IPP_OWN_FUNPTR (void, selectAP, (BNU_CHUNK_T* pAP, const BNU_CHUNK_T* pAPtbl, int index)) + +typedef struct _cpPrecompAP { + int w; /* scalar's window bitsize */ + selectAP select_affine_point; /* get affine point function */ + const BNU_CHUNK_T* pTbl; /* pre-computed table */ +} cpPrecompAP; + + +/* EC over GF(p) context */ +typedef struct _cpGFpEC { + Ipp32u idCtx; /* EC identifier */ + cpModulusID idModulus; /* Modulus type */ + + IppsGFpState* pGF; /* arbitrary GF(p^d)*/ + + int subgroup; /* set up subgroup */ + int elementSize; /* length of EC point */ + int orderBitSize; /* base_point order bitsize */ + BNU_CHUNK_T* pA; /* EC parameter A */ + BNU_CHUNK_T* pB; /* B */ + BNU_CHUNK_T* pG; /* base_point */ + BNU_CHUNK_T* cofactor; /* cofactor = #E/base_point order */ + int parmAspc; /* NIST's, EPIDv2.0 A-parameter specific */ + int infinity; /* 0/1 if B !=0/==0 */ + const cpPrecompAP* pBaseTbl; /* address of pre-computed [n]G tabble */ + gsModEngine* pMontR; /* EC order montgomery engine */ + + BNU_CHUNK_T* pPool; /* pool of points */ + #if defined(_LEGACY_ECCP_SUPPORT_) + BNU_CHUNK_T* pPublic; /* regular public key */ + BNU_CHUNK_T* pPublicE; /* ephemeral public key */ + BNU_CHUNK_T* pPrivat; /* regular private key */ + BNU_CHUNK_T* pPrivatE; /* ephemeral private key */ + BNU_CHUNK_T* pBuffer; /* pointer to scaratch buffer (for legacy ECCP only) */ + #endif +} cpGFPEC; + +#define ECGFP_ALIGNMENT ((int)(sizeof(void*))) + +/* Local definitions */ +#define EC_POOL_SIZE (10) /* num of points into the pool */ + +#define EC_MONT_POOL_SIZE (4) /* num of temp values for modular arithmetic */ + +#define ECP_SET_ID(pCtx) ((pCtx)->idCtx = (Ipp32u)idCtxGFPEC ^ (Ipp32u)IPP_UINT_PTR(pCtx)) +#define ECP_MODULUS_ID(pCtx) ((pCtx)->idModulus) +#define ECP_GFP(pCtx) ((pCtx)->pGF) +#define ECP_SUBGROUP(pCtx) ((pCtx)->subgroup) +#define ECP_POINTLEN(pCtx) ((pCtx)->elementSize) +#define ECP_ORDBITSIZE(pCtx) ((pCtx)->orderBitSize) +#define ECP_COFACTOR(pCtx) ((pCtx)->cofactor) +#define ECP_SPECIFIC(pCtx) ((pCtx)->parmAspc) +#define ECP_INFINITY(pCtx) ((pCtx)->infinity) +#define ECP_A(pCtx) ((pCtx)->pA) +#define ECP_B(pCtx) ((pCtx)->pB) +#define ECP_G(pCtx) ((pCtx)->pG) +#define ECP_PREMULBP(pCtx) ((pCtx)->pBaseTbl) +#define ECP_MONT_R(pCtx) ((pCtx)->pMontR) +#define ECP_POOL(pCtx) ((pCtx)->pPool) +#if defined(_LEGACY_ECCP_SUPPORT_) + #define ECP_PUBLIC(pCtx) ((pCtx)->pPublic) + #define ECP_PUBLIC_E(pCtx) ((pCtx)->pPublicE) + #define ECP_PRIVAT(pCtx) ((pCtx)->pPrivat) + #define ECP_PRIVAT_E(pCtx) ((pCtx)->pPrivatE) + #define ECP_SBUFFER(pCtx) ((pCtx)->pBuffer) +#endif + +#define VALID_ECP_ID(pCtx) ((((pCtx)->idCtx) ^ (Ipp32u)IPP_UINT_PTR((pCtx))) == (Ipp32u)idCtxGFPEC) + +/* EC curve specific (a-parameter) */ +#define ECP_Acom (0) /* commont case */ +#define ECP_Ami3 (1) /* a=-3 NIST's and SM2 curve */ +#define ECP_Aeq0 (2) /* a=0 EPIDv2.0 curve */ + +#define ECP_ARB ECP_Acom +#define ECP_STD ECP_Ami3 +#define ECP_EPID2 ECP_Aeq0 + +/* std ec pre-computed tables */ +#define gfpec_precom_nistP192r1_fun OWNAPI(gfpec_precom_nistP192r1_fun) +#define gfpec_precom_nistP224r1_fun OWNAPI(gfpec_precom_nistP224r1_fun) +#define gfpec_precom_nistP256r1_fun OWNAPI(gfpec_precom_nistP256r1_fun) +#define gfpec_precom_nistP256r1_radix52_fun OWNAPI(gfpec_precom_nistP256r1_radix52_fun) +#define gfpec_precom_nistP384r1_fun OWNAPI(gfpec_precom_nistP384r1_fun) +#define gfpec_precom_nistP384r1_radix52_fun OWNAPI(gfpec_precom_nistP384r1_radix52_fun) +#define gfpec_precom_nistP521r1_fun OWNAPI(gfpec_precom_nistP521r1_fun) +#define gfpec_precom_nistP521r1_radix52_fun OWNAPI(gfpec_precom_nistP521r1_radix52_fun) +#define gfpec_precom_sm2_fun OWNAPI(gfpec_precom_sm2_fun) +#define gfpec_precom_sm2_radix52_fun OWNAPI(gfpec_precom_sm2_radix52_fun) + +IPP_OWN_DECL (const cpPrecompAP*, gfpec_precom_nistP192r1_fun, (void)) +IPP_OWN_DECL (const cpPrecompAP*, gfpec_precom_nistP224r1_fun, (void)) +IPP_OWN_DECL (const cpPrecompAP*, gfpec_precom_nistP256r1_fun, (void)) +IPP_OWN_DECL (const cpPrecompAP*, gfpec_precom_nistP256r1_radix52_fun, (void)) +IPP_OWN_DECL (const cpPrecompAP*, gfpec_precom_nistP384r1_fun, (void)) +IPP_OWN_DECL (const cpPrecompAP*, gfpec_precom_nistP384r1_radix52_fun, (void)) +IPP_OWN_DECL (const cpPrecompAP*, gfpec_precom_nistP521r1_fun, (void)) +IPP_OWN_DECL (const cpPrecompAP*, gfpec_precom_nistP521r1_radix52_fun, (void)) +IPP_OWN_DECL (const cpPrecompAP*, gfpec_precom_sm2_fun, (void)) +IPP_OWN_DECL (const cpPrecompAP*, gfpec_precom_sm2_radix52_fun, (void)) + +/* +// get/release n points from/to the pool +*/ +__INLINE BNU_CHUNK_T* cpEcGFpGetPool(int n, IppsGFpECState* pEC) +{ + BNU_CHUNK_T* pPool = ECP_POOL(pEC); + ECP_POOL(pEC) += n*GFP_FELEN(GFP_PMA(ECP_GFP(pEC)))*3; + return pPool; +} +__INLINE void cpEcGFpReleasePool(int n, IppsGFpECState* pEC) +{ + int chunk_size = n*GFP_FELEN(GFP_PMA(ECP_GFP(pEC)))*3; + ECP_POOL(pEC) -= chunk_size; + // Clean the pool for the security reasons + // (intermediate sensetive data may be stored here) + ZEXPAND_BNU(ECP_POOL(pEC), 0, chunk_size); +} + +__INLINE IppsGFpECPoint* cpEcGFpInitPoint(IppsGFpECPoint* pPoint, BNU_CHUNK_T* pData, int flags, const IppsGFpECState* pEC) +{ + ECP_POINT_SET_ID(pPoint); + ECP_POINT_FLAGS(pPoint) = flags; + ECP_POINT_FELEN(pPoint) = GFP_FELEN(GFP_PMA(ECP_GFP(pEC))); + ECP_POINT_DATA(pPoint) = pData; + return pPoint; +} + +/* copy one point into another */ +__INLINE IppsGFpECPoint* gfec_CopyPoint(IppsGFpECPoint* pPointR, const IppsGFpECPoint* pPointA, int elemLen) +{ + cpGFpElementCopy(ECP_POINT_DATA(pPointR), ECP_POINT_DATA(pPointA), 3*elemLen); + ECP_POINT_FLAGS(pPointR) = ECP_POINT_FLAGS(pPointA); + return pPointR; +} + + +__INLINE IppsGFpECPoint* gfec_SetPointAtInfinity(IppsGFpECPoint* pPoint) +{ + int elemLen = ECP_POINT_FELEN(pPoint); + cpGFpElementPad(ECP_POINT_X(pPoint), elemLen, 0); + cpGFpElementPad(ECP_POINT_Y(pPoint), elemLen, 0); + cpGFpElementPad(ECP_POINT_Z(pPoint), elemLen, 0); + ECP_POINT_FLAGS(pPoint) = 0; + return pPoint; +} + +/* +// test infinity: +// IsProjectivePointAtInfinity +*/ +__INLINE int gfec_IsPointAtInfinity(const IppsGFpECPoint* pPoint) +{ + return GFP_IS_ZERO( ECP_POINT_Z(pPoint), ECP_POINT_FELEN(pPoint)); +} + + + +/* signed encode */ +__INLINE void booth_recode(Ipp8u* sign, Ipp8u* digit, Ipp8u in, int w) +{ + Ipp8u s = (Ipp8u)(~((in >> w) - 1)); + int d = (1 << (w+1)) - in - 1; + d = (d & s) | (in & ~s); + d = (d >> 1) + (d & 1); + *sign = s & 1; + *digit = (Ipp8u)d; +} + + +#define gfec_point_add OWNAPI(gfec_point_add) +#define gfec_affine_point_add OWNAPI(gfec_affine_point_add) +#define gfec_point_double OWNAPI(gfec_point_double) +#define gfec_point_mul OWNAPI(gfec_point_mul) +#define gfec_point_prod OWNAPI(gfec_point_prod) +#define gfec_base_point_mul OWNAPI(gfec_base_point_mul) +#define setupTable OWNAPI(setupTable) + +IPP_OWN_DECL (void, gfec_point_add, (BNU_CHUNK_T* pRdata, const BNU_CHUNK_T* pPdata, const BNU_CHUNK_T* pQdata, IppsGFpECState* pEC)) +IPP_OWN_DECL (void, gfec_affine_point_add, (BNU_CHUNK_T* pRdata, const BNU_CHUNK_T* pPdata, const BNU_CHUNK_T* pAdata, IppsGFpECState* pEC)) +IPP_OWN_DECL (void, gfec_point_double, (BNU_CHUNK_T* pRdata, const BNU_CHUNK_T* pPdata, IppsGFpECState* pEC)) +IPP_OWN_DECL (void, gfec_point_mul, (BNU_CHUNK_T* pRdata, const BNU_CHUNK_T* pPdata, const Ipp8u* pScalar8, int scalarBitSize, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPP_OWN_DECL (void, gfec_point_prod, (BNU_CHUNK_T* pointR, const BNU_CHUNK_T* pointA, const Ipp8u* pScalarA, const BNU_CHUNK_T* pointB, const Ipp8u* pScalarB, int scalarBitSize, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPP_OWN_DECL (void, gfec_base_point_mul, (BNU_CHUNK_T* pRdata, const Ipp8u* pScalarB, int scalarBitSize, IppsGFpECState* pEC)) +IPP_OWN_DECL (void, setupTable, (BNU_CHUNK_T* pTbl, const BNU_CHUNK_T* pPdata, IppsGFpECState* pEC)) + + +/* size of context */ +#define cpGFpECGetSize OWNAPI(cpGFpECGetSize) +IPP_OWN_DECL (int, cpGFpECGetSize, (int deg, int basicElmBitSize)) + +/* point operations */ +#define gfec_GetPoint OWNAPI(gfec_GetPoint) +#define gfec_SetPoint OWNAPI(gfec_SetPoint) +#define gfec_MakePoint OWNAPI(gfec_MakePoint) +#define gfec_ComparePoint OWNAPI(gfec_ComparePoint) +#define gfec_IsPointOnCurve OWNAPI(gfec_IsPointOnCurve) + +IPP_OWN_DECL (int, gfec_GetPoint, (BNU_CHUNK_T* pX, BNU_CHUNK_T* pY, const IppsGFpECPoint* pPoint, IppsGFpECState* pEC)) +IPP_OWN_DECL (int, gfec_SetPoint, (BNU_CHUNK_T* pP, const BNU_CHUNK_T* pX, const BNU_CHUNK_T* pY, IppsGFpECState* pEC)) +IPP_OWN_DECL (int, gfec_MakePoint, (IppsGFpECPoint* pPoint, const BNU_CHUNK_T* pElm, IppsGFpECState* pEC)) +IPP_OWN_DECL (int, gfec_ComparePoint, (const IppsGFpECPoint* pP, const IppsGFpECPoint* pQ, IppsGFpECState* pEC)) +IPP_OWN_DECL (int, gfec_IsPointOnCurve, (const IppsGFpECPoint* pP, IppsGFpECState* pEC)) + +__INLINE IppsGFpECPoint* gfec_DblPoint(IppsGFpECPoint* pR, + const IppsGFpECPoint* pP, IppsGFpECState* pEC) +{ + gfec_point_double(ECP_POINT_X(pR), ECP_POINT_X(pP), pEC); + ECP_POINT_FLAGS(pR) = gfec_IsPointAtInfinity(pR)? 0 : ECP_FINITE_POINT; + return pR; +} + +__INLINE IppsGFpECPoint* gfec_AddPoint(IppsGFpECPoint* pR, + const IppsGFpECPoint* pP, const IppsGFpECPoint* pQ, + IppsGFpECState* pEC) +{ + gfec_point_add(ECP_POINT_X(pR), ECP_POINT_X(pP), ECP_POINT_X(pQ), pEC); + ECP_POINT_FLAGS(pR) = gfec_IsPointAtInfinity(pR)? 0 : ECP_FINITE_POINT; + return pR; +} + + +#define gfec_NegPoint OWNAPI(gfec_NegPoint) +#define gfec_MulPoint OWNAPI(gfec_MulPoint) +#define gfec_MulBasePoint OWNAPI(gfec_MulBasePoint) +/* #define gfec_PointProduct OWNAPI(gfec_PointProduct) */ +#define gfec_BasePointProduct OWNAPI(gfec_BasePointProduct) +#define p192r1_select_ap_w7 OWNAPI(p192r1_select_ap_w7) +#define p224r1_select_ap_w7 OWNAPI(p224r1_select_ap_w7) +#define p256r1_select_ap_w7 OWNAPI(p256r1_select_ap_w7) +#define p384r1_select_ap_w5 OWNAPI(p384r1_select_ap_w5) +#define p521r1_select_ap_w5 OWNAPI(p521r1_select_ap_w5) + +IPP_OWN_DECL (IppsGFpECPoint*, gfec_NegPoint, (IppsGFpECPoint* pR, const IppsGFpECPoint* pP, IppsGFpECState* pEC)) +IPP_OWN_DECL (IppsGFpECPoint*, gfec_MulPoint, (IppsGFpECPoint* pR, const IppsGFpECPoint* pP, const BNU_CHUNK_T* pScalar, int scalarLen, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPP_OWN_DECL (IppsGFpECPoint*, gfec_MulBasePoint, (IppsGFpECPoint* pR, const BNU_CHUNK_T* pScalar, int scalarLen, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +/* IPP_OWN_DECL (IppsGFpECPoint*, gfec_PointProduct, (IppsGFpECPoint* pR, const IppsGFpECPoint* pP, const BNU_CHUNK_T* pScalarP, int scalarPlen, const IppsGFpECPoint* pQ, const BNU_CHUNK_T* pScalarQ, int scalarQlen, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) */ +IPP_OWN_DECL (IppsGFpECPoint*, gfec_BasePointProduct, (IppsGFpECPoint* pR, const BNU_CHUNK_T* pScalarG, int scalarGlen, const IppsGFpECPoint* pP, const BNU_CHUNK_T* pScalarP, int scalarPlen, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) + +IPP_OWN_DECL (void, p192r1_select_ap_w7, (BNU_CHUNK_T* pAffinePoint, const BNU_CHUNK_T* pTable, int index)) +IPP_OWN_DECL (void, p224r1_select_ap_w7, (BNU_CHUNK_T* pAffinePoint, const BNU_CHUNK_T* pTable, int index)) +IPP_OWN_DECL (void, p256r1_select_ap_w7, (BNU_CHUNK_T* pAffinePoint, const BNU_CHUNK_T* pTable, int index)) +IPP_OWN_DECL (void, p384r1_select_ap_w5, (BNU_CHUNK_T* pAffinePoint, const BNU_CHUNK_T* pTable, int index)) +IPP_OWN_DECL (void, p521r1_select_ap_w5, (BNU_CHUNK_T* pAffinePoint, const BNU_CHUNK_T* pTable, int index)) + +/* AVX512-IFMA implementations */ +#define gfec_SignDSA_nistp256_avx512 OWNAPI(gfec_SignDSA_nistp256_avx512) +#define gfec_SignDSA_nistp384_avx512 OWNAPI(gfec_SignDSA_nistp384_avx512) +#define gfec_SignDSA_nistp521_avx512 OWNAPI(gfec_SignDSA_nistp521_avx512) +#define gfec_Sign_sm2_avx512 OWNAPI(gfec_Sign_sm2_avx512) + +#define gfec_VerifyDSA_nistp256_avx512 OWNAPI(gfec_VerifyDSA_nistp256_avx512) +#define gfec_VerifyDSA_nistp384_avx512 OWNAPI(gfec_VerifyDSA_nistp384_avx512) +#define gfec_VerifyDSA_nistp521_avx512 OWNAPI(gfec_VerifyDSA_nistp521_avx512) +#define gfec_Verify_sm2_avx512 OWNAPI(gfec_Verify_sm2_avx512) + +#define gfec_SharedSecretDH_nistp256_avx512 OWNAPI(gfec_SharedSecretDH_nistp256_avx512) +#define gfec_SharedSecretDH_nistp384_avx512 OWNAPI(gfec_SharedSecretDH_nistp384_avx512) +#define gfec_SharedSecretDH_nistp521_avx512 OWNAPI(gfec_SharedSecretDH_nistp521_avx512) +#define gfec_SharedSecretDH_sm2_avx512 OWNAPI(gfec_SharedSecretDH_sm2_avx512) + +#define gfec_PubKey_nist256_avx512 OWNAPI(gfec_PubKey_nist256_avx512) +#define gfec_PubKey_nist384_avx512 OWNAPI(gfec_PubKey_nist384_avx512) +#define gfec_PubKey_nist521_avx512 OWNAPI(gfec_PubKey_nist521_avx512) +#define gfec_PubKey_sm2_avx512 OWNAPI(gfec_PubKey_sm2_avx512) + +#define gfec_MulPoint_nistp256_avx512 OWNAPI(gfec_MulPoint_nistp256_avx512) +#define gfec_MulPoint_nistp384_avx512 OWNAPI(gfec_MulPoint_nistp384_avx512) +#define gfec_MulPoint_nistp521_avx512 OWNAPI(gfec_MulPoint_nistp521_avx512) +#define gfec_MulPoint_sm2_avx512 OWNAPI(gfec_MulPoint_sm2_avx512) + +#define gfec_AddPoint_nistp256_avx512 OWNAPI(gfec_AddPoint_nistp256_avx512) +#define gfec_AddPoint_nistp384_avx512 OWNAPI(gfec_AddPoint_nistp384_avx512) +#define gfec_AddPoint_nistp521_avx512 OWNAPI(gfec_AddPoint_nistp521_avx512) +#define gfec_AddPoint_sm2_avx512 OWNAPI(gfec_AddPoint_sm2_avx512) + +#define gfec_point_on_curve_nistp256_avx512 OWNAPI(gfec_point_on_curve_nistp256_avx512) +#define gfec_point_on_curve_nistp384_avx512 OWNAPI(gfec_point_on_curve_nistp384_avx512) +#define gfec_point_on_curve_nistp521_avx512 OWNAPI(gfec_point_on_curve_nistp521_avx512) +#define gfec_point_on_curve_sm2_avx512 OWNAPI(gfec_point_on_curve_sm2_avx512) + +IPP_OWN_DECL (IppStatus, gfec_SignDSA_nistp256_avx512, (const IppsBigNumState* pMsgDigest, const IppsBigNumState* pRegPrivate, IppsBigNumState* pEphPrivate, IppsBigNumState* pSignR, IppsBigNumState* pSignS, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPP_OWN_DECL (IppStatus, gfec_SignDSA_nistp384_avx512, (const IppsBigNumState* pMsgDigest, const IppsBigNumState* pRegPrivate, IppsBigNumState* pEphPrivate, IppsBigNumState* pSignR, IppsBigNumState* pSignS, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPP_OWN_DECL (IppStatus, gfec_SignDSA_nistp521_avx512, (const IppsBigNumState* pMsgDigest, const IppsBigNumState* pRegPrivate, IppsBigNumState* pEphPrivate, IppsBigNumState* pSignR, IppsBigNumState* pSignS, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPP_OWN_DECL (IppStatus, gfec_Sign_sm2_avx512, (const IppsBigNumState* pMsgDigest, const IppsBigNumState* pRegPrivate, IppsBigNumState* pEphPrivate, IppsBigNumState* pSignR, IppsBigNumState* pSignS, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) + +IPP_OWN_DECL (IppECResult, gfec_VerifyDSA_nistp256_avx512, (const IppsBigNumState* pMsgDigest, const IppsGFpECPoint* pRegPublic, const IppsBigNumState* pSignR, const IppsBigNumState* pSignS, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPP_OWN_DECL (IppECResult, gfec_VerifyDSA_nistp384_avx512, (const IppsBigNumState* pMsgDigest, const IppsGFpECPoint* pRegPublic, const IppsBigNumState* pSignR, const IppsBigNumState* pSignS, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPP_OWN_DECL (IppECResult, gfec_VerifyDSA_nistp521_avx512, (const IppsBigNumState* pMsgDigest, const IppsGFpECPoint* pRegPublic, const IppsBigNumState* pSignR, const IppsBigNumState* pSignS, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPP_OWN_DECL (IppECResult, gfec_Verify_sm2_avx512, (const IppsBigNumState* pMsgDigest, const IppsGFpECPoint* pRegPublic, const IppsBigNumState* pSignR, const IppsBigNumState* pSignS, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) + +IPP_OWN_DECL (int, gfec_SharedSecretDH_nistp256_avx512, (IppsGFpECPoint * pR, const IppsGFpECPoint* pP, const BNU_CHUNK_T* pScalar, int scalarLen, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPP_OWN_DECL (int, gfec_SharedSecretDH_nistp384_avx512, (IppsGFpECPoint * pR, const IppsGFpECPoint* pP, const BNU_CHUNK_T* pScalar, int scalarLen, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPP_OWN_DECL (int, gfec_SharedSecretDH_nistp521_avx512, (IppsGFpECPoint * pR, const IppsGFpECPoint* pP, const BNU_CHUNK_T* pScalar, int scalarLen, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPP_OWN_DECL (int, gfec_SharedSecretDH_sm2_avx512, (IppsGFpECPoint * pR, const IppsGFpECPoint* pP, const BNU_CHUNK_T* pScalar, int scalarLen, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) + +IPP_OWN_DECL (IppsGFpECPoint*, gfec_PubKey_nist256_avx512, (IppsGFpECPoint * pR, const BNU_CHUNK_T* pScalar, int scalarLen, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPP_OWN_DECL (IppsGFpECPoint*, gfec_PubKey_nist384_avx512, (IppsGFpECPoint * pR, const BNU_CHUNK_T* pScalar, int scalarLen, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPP_OWN_DECL (IppsGFpECPoint*, gfec_PubKey_nist521_avx512, (IppsGFpECPoint * pR, const BNU_CHUNK_T* pScalar, int scalarLen, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPP_OWN_DECL (IppsGFpECPoint*, gfec_PubKey_sm2_avx512, (IppsGFpECPoint * pR, const BNU_CHUNK_T* pScalar, int scalarLen, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) + +IPP_OWN_DECL (IppsGFpECPoint*, gfec_MulPoint_nistp256_avx512, (IppsGFpECPoint* pR, const IppsGFpECPoint* pP, const BNU_CHUNK_T* pScalar, int scalarLen, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPP_OWN_DECL (IppsGFpECPoint*, gfec_MulPoint_nistp384_avx512, (IppsGFpECPoint* pR, const IppsGFpECPoint* pP, const BNU_CHUNK_T* pScalar, int scalarLen, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPP_OWN_DECL (IppsGFpECPoint*, gfec_MulPoint_nistp521_avx512, (IppsGFpECPoint* pR, const IppsGFpECPoint* pP, const BNU_CHUNK_T* pScalar, int scalarLen, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +IPP_OWN_DECL (IppsGFpECPoint*, gfec_MulPoint_sm2_avx512, (IppsGFpECPoint* pR, const IppsGFpECPoint* pP, const BNU_CHUNK_T* pScalar, int scalarLen, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) + +IPP_OWN_DECL (IppsGFpECPoint*, gfec_AddPoint_nistp256_avx512, (IppsGFpECPoint* pR, const IppsGFpECPoint* pP, const IppsGFpECPoint* pQ, IppsGFpECState* pEC)) +IPP_OWN_DECL (IppsGFpECPoint*, gfec_AddPoint_nistp384_avx512, (IppsGFpECPoint* pR, const IppsGFpECPoint* pP, const IppsGFpECPoint* pQ, IppsGFpECState* pEC)) +IPP_OWN_DECL (IppsGFpECPoint*, gfec_AddPoint_nistp521_avx512, (IppsGFpECPoint* pR, const IppsGFpECPoint* pP, const IppsGFpECPoint* pQ, IppsGFpECState* pEC)) +IPP_OWN_DECL (IppsGFpECPoint*, gfec_AddPoint_sm2_avx512, (IppsGFpECPoint* pR, const IppsGFpECPoint* pP, const IppsGFpECPoint* pQ, IppsGFpECState* pEC)) + +IPP_OWN_DECL (int, gfec_point_on_curve_nistp256_avx512, (const IppsGFpECPoint *pPoint, IppsGFpECState *pEC)) +IPP_OWN_DECL (int, gfec_point_on_curve_nistp384_avx512, (const IppsGFpECPoint *pPoint, IppsGFpECState *pEC)) +IPP_OWN_DECL (int, gfec_point_on_curve_nistp521_avx512, (const IppsGFpECPoint *pPoint, IppsGFpECState *pEC)) +IPP_OWN_DECL (int, gfec_point_on_curve_sm2_avx512, (const IppsGFpECPoint *pPoint, IppsGFpECState *pEC)) + +#define gfec_CheckPrivateKey OWNAPI(gfec_CheckPrivateKey) +IPP_OWN_DECL(int, gfec_CheckPrivateKey, (const IppsBigNumState* pPrivate, IppsGFpECState* pEC)) + +#endif /* _CP_ECGFP_H_ */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpectstpoint.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpectstpoint.c new file mode 100644 index 0000000..47bd4ca --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpectstpoint.c @@ -0,0 +1,80 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECTstPoint() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcphash.h" +#include "pcphash_rmf.h" + +/*F* +// Name: ippsGFpECTstPoint +// +// Purpose: Checks if a point belongs to an elliptic curve +// +// Returns: Reason: +// ippStsNullPtrErr pP == NULL +// pEC == NULL +// pResult == NULL +// +// ippStsContextMatchErr invalid pEC->idCtx +// invalid pP->idCtx +// +// ippStsOutOfRangeErr ECP_POINT_FELEN(pP)!=GFP_FELEN() +// +// ippStsNoErr no error +// +// Parameters: +// pP Pointer to the IppsGFpECPoint context +// pEC Pointer to the context of the elliptic curve +// pResult Pointer to the result of the check +// +// Note: +// Even if test passed is not a fact that the point belongs to BP-related subgroup BP +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECTstPoint,(const IppsGFpECPoint* pP, + IppECResult* pResult, + IppsGFpECState* pEC)) +{ + IPP_BAD_PTR3_RET(pP, pResult, pEC); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pP), ippStsContextMatchErr ); + + IPP_BADARG_RET( ECP_POINT_FELEN(pP)!=GFP_FELEN(GFP_PMA(ECP_GFP(pEC))), ippStsOutOfRangeErr); + + if( gfec_IsPointAtInfinity(pP) ) + *pResult = ippECPointIsAtInfinite; + else if( !gfec_IsPointOnCurve(pP, pEC) ) + *pResult = ippECPointIsNotValid; + else + *pResult = ippECValid; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpectstpointinsubgroup.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpectstpointinsubgroup.c new file mode 100644 index 0000000..abae071 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpectstpointinsubgroup.c @@ -0,0 +1,91 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p) Operations +// +// Context: +// ippsGFpECTstPointInSubgroup() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcphash.h" +#include "pcphash_rmf.h" + + +/*F* +// Name: ippsGFpECTstPointInSubgroup +// +// Purpose: Checks if a point belongs to a specified subgroup +// +// Returns: Reason: +// ippStsNullPtrErr pP == NULL +// pEC == NULL +// pResult == NULL +/ pScratchBuffer == NULL +// +// ippStsContextMatchErr invalid pEC->idCtx +// pEC->subgroup == NULL +// invalid pP->idCtx +// +// ippStsOutOfRangeErr ECP_POINT_FELEN(pP)!=GFP_FELEN() +// +// ippStsNoErr no error +// +// Parameters: +// pP Pointer to the IppsGFpECPoint context +// pEC Pointer to the context of the elliptic curve +// pResult Pointer to the result of the check +// pScratchBuffer Pointer to the scratch buffer +// +*F*/ +IPPFUN(IppStatus, ippsGFpECTstPointInSubgroup,(const IppsGFpECPoint* pP, + IppECResult* pResult, + IppsGFpECState* pEC, + Ipp8u* pScratchBuffer)) +{ + IPP_BAD_PTR4_RET(pP, pResult, pEC, pScratchBuffer); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + IPP_BADARG_RET(!ECP_SUBGROUP(pEC), ippStsContextMatchErr); + IPP_BADARG_RET( !ECP_POINT_VALID_ID(pP), ippStsContextMatchErr ); + + IPP_BADARG_RET( ECP_POINT_FELEN(pP)!=GFP_FELEN(GFP_PMA(ECP_GFP(pEC))), ippStsOutOfRangeErr); + + { + IppECResult tstResult; + ippsGFpECTstPoint(pP, &tstResult, pEC); + + if(ippECValid==tstResult) { + IppsGFpECPoint T; + cpEcGFpInitPoint(&T, cpEcGFpGetPool(1, pEC),0, pEC); + + gfec_MulPoint(&T, pP, MOD_MODULUS(ECP_MONT_R(pEC)), BITS_BNU_CHUNK(ECP_ORDBITSIZE(pEC)), /*0,*/ pEC, pScratchBuffer); + tstResult = gfec_IsPointAtInfinity(&T)? ippECValid : ippECPointOutOfGroup; + + cpEcGFpReleasePool(1, pEC); + } + *pResult = tstResult; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecverify.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecverify.c new file mode 100644 index 0000000..f84988e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecverify.c @@ -0,0 +1,142 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// EC over GF(p^m) definitinons +// +// Context: +// ippsGFpECVerify() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpecstuff.h" +#include "pcpeccp.h" + +//tbcd: temporary excluded: #include +/*F* +// Name: ippsGFpECVerify +// +// Purpose: Verifies the parameters of an elliptic curve. +// +// Returns: Reason: +// ippStsNullPtrErr pEC == NULL +// pResult == NULL +// pScratchBuffer == NULL +// ippStsContextMatchErr invalid pEC->idCtx +// ippStsNoErr no error +// +// Parameters: +// pResult Pointer to the verification result +// pEC Pointer to the context of the elliptic curve +// pScratchBuffer Pointer to the scratch buffer +// +*F*/ + +IPPFUN(IppStatus, ippsGFpECVerify,(IppECResult* pResult, IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +{ + IPP_BAD_PTR3_RET(pEC, pResult, pScratchBuffer); + IPP_BADARG_RET( !VALID_ECP_ID(pEC), ippStsContextMatchErr ); + + *pResult = ippECValid; + + { + IppsGFpState* pGF = ECP_GFP(pEC); + gsModEngine* pGFE = GFP_PMA(pGF); + int elemLen = GFP_FELEN(pGFE); + + mod_mul mulF = GFP_METHOD(pGFE)->mul; + mod_sqr sqrF = GFP_METHOD(pGFE)->sqr; + mod_add addF = GFP_METHOD(pGFE)->add; + + /* + // check discriminant ( 4*A^3 + 27*B^2 != 0 mod P) + */ + if(ippECValid == *pResult) { + BNU_CHUNK_T* pT = cpGFpGetPool(1, pGFE); + BNU_CHUNK_T* pU = cpGFpGetPool(1, pGFE); + //tbcd: temporary excluded: assert(NULL!=pT && NULL!=pU); + + if(ECP_SPECIFIC(pEC)==ECP_EPID2) + cpGFpElementPad(pT, elemLen, 0); /* T = 4*A^3 = 0 */ + else { + addF(pT, ECP_A(pEC), ECP_A(pEC), pGFE); /* T = 4*A^3 */ + sqrF(pT, pT, pGFE); + mulF(pT, ECP_A(pEC), pT, pGFE); + } + + addF(pU, ECP_B(pEC), ECP_B(pEC), pGFE); /* U = 9*B^2 */ + addF(pU, pU, ECP_B(pEC), pGFE); + sqrF(pU, pU, pGFE); + + addF(pT, pU, pT, pGFE); /* T += 3*U */ + addF(pT, pU, pT, pGFE); + addF(pT, pU, pT, pGFE); + + *pResult = GFP_IS_ZERO(pT, elemLen)? ippECIsZeroDiscriminant: ippECValid; + + cpGFpReleasePool(2, pGFE); + } + + if(ECP_SUBGROUP(pEC)) { + /* + // check base point and it order + */ + if(ippECValid == *pResult) { + IppsGFpECPoint G; + cpEcGFpInitPoint(&G, ECP_G(pEC), ECP_AFFINE_POINT|ECP_FINITE_POINT, pEC); + + /* check G != infinity */ + *pResult = gfec_IsPointAtInfinity(&G)? ippECPointIsAtInfinite : ippECValid; + + /* check G lies on EC */ + if(ippECValid == *pResult) + *pResult = gfec_IsPointOnCurve(&G, pEC)? ippECValid : ippECPointIsNotValid; + + /* check Gorder*G = infinity */ + if(ippECValid == *pResult) { + IppsGFpECPoint T; + cpEcGFpInitPoint(&T, cpEcGFpGetPool(1, pEC),0, pEC); + + gfec_MulBasePoint(&T, MOD_MODULUS(ECP_MONT_R(pEC)), BITS_BNU_CHUNK(ECP_ORDBITSIZE(pEC)), pEC, pScratchBuffer); + + *pResult = gfec_IsPointAtInfinity(&T)? ippECValid : ippECInvalidOrder; + + cpEcGFpReleasePool(1, pEC); + } + } + + /* + // check order==P + */ + if(ippECValid == *pResult) { + BNU_CHUNK_T* pPrime = GFP_MODULUS(pGFE); + int primeLen = GFP_FELEN(pGFE); + + gsModEngine* pR = ECP_MONT_R(pEC); + BNU_CHUNK_T* pOrder = MOD_MODULUS(pR); + int orderLen = MOD_LEN(pR); + + *pResult = (primeLen==orderLen && GFP_EQ(pPrime, pOrder, primeLen))? ippECIsWeakSSSA : ippECValid; + } + } + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecverifydsa.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecverifydsa.c new file mode 100644 index 0000000..6789e68 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecverifydsa.c @@ -0,0 +1,205 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// ippsGFpECVerifyDSA() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsGFpECVerifyDSA +// +// Purpose: DSA Signature Verification. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pMsgDigest +// NULL == pRegPublic +// NULL == pSignR +// NULL == pSignS +// NULL == pResult +// NULL == pScratchBuffer +// +// ippStsContextMatchErr illegal pECC->idCtx +// pEC->subgroup == NULL +// illegal pMsgDigestDigest->idCtx +// illegal pRegPublic->idCtx +// illegal pSignR->idCtx +// illegal pSignS->idCtx +// +// ippStsMessageErr 0> MsgDigest +// order<= MsgDigest +// +// ippStsRangeErr SignR < 0 or SignS < 0 +// +// ippStsOutOfRangeErr bitsize(pRegPublic) != bitsize(prime) +// +// ippStsNotSupportedModeErr 1 cpCmp_BNU(BN_NUMBER(pSignR), BN_SIZE(pSignR), pOrder, orderLen) && + 0 > cpCmp_BNU(BN_NUMBER(pSignS), BN_SIZE(pSignS), pOrder, orderLen)) { + +#if (_IPP32E >= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512IFMA)) { + switch (ECP_MODULUS_ID(pEC)) { + case cpID_PrimeP256r1: { + verifyResult = gfec_VerifyDSA_nistp256_avx512(pMsgDigest, pRegPublic, pSignR, pSignS, pEC, pScratchBuffer); + goto exit; + break; + } + case cpID_PrimeP384r1: { + verifyResult = gfec_VerifyDSA_nistp384_avx512(pMsgDigest, pRegPublic, pSignR, pSignS, pEC, pScratchBuffer); + goto exit; + break; + } + case cpID_PrimeP521r1: { + verifyResult = gfec_VerifyDSA_nistp521_avx512(pMsgDigest, pRegPublic, pSignR, pSignS, pEC, pScratchBuffer); + goto exit; + break; + } + default: + /* Go to default implementation below */ + break; + } + } /* no else */ +#endif // (_IPP32E >= _IPP32E_K1) + { + int elmLen = GFP_FELEN(pGFE); + int pelmLen = GFP_PELEN(pGFE); + BNU_CHUNK_T *h1 = cpGFpGetPool(3, pGFE); + BNU_CHUNK_T *h2 = h1 + pelmLen; + BNU_CHUNK_T *h = h2 + pelmLen; + + IppsGFpECPoint P; + cpEcGFpInitPoint(&P, cpEcGFpGetPool(1, pEC), 0, pEC); + + /* copy message and reduce */ + ZEXPAND_COPY_BNU(h1, orderLen, BN_NUMBER(pMsgDigest), BN_SIZE(pMsgDigest)); + cpModSub_BNU(h1, h1, pOrder, pOrder, orderLen, h2); + + /* h = d^-1, h1 = msg*h, h2 = c*h */ + ZEXPAND_COPY_BNU(h, orderLen, BN_NUMBER(pSignS), BN_SIZE(pSignS)); + gs_mont_inv(h, h, pMontR, alm_mont_inv); + + cpMontMul_BNU(h1, h, h1, pMontR); + ZEXPAND_COPY_BNU(h2, orderLen, BN_NUMBER(pSignR), BN_SIZE(pSignR)); + cpMontMul_BNU(h2, h, h2, pMontR); + + /* P = [h1]BasePoint + [h2]publicKey */ + gfec_BasePointProduct(&P, + h1, orderLen, pRegPublic, h2, orderLen, + pEC, pScratchBuffer); + + /* check that P!=O */ + if (!gfec_IsPointAtInfinity(&P)) { + /* get P.X */ + gfec_GetPoint(h1, NULL, &P, pEC); + /* c' = int(P.x) mod order */ + GFP_METHOD(pGFE)->decode(h1, h1, pGFE); + elmLen = cpMod_BNU(h1, elmLen, pOrder, orderLen); + cpGFpElementPad(h1 + elmLen, orderLen - elmLen, 0); + + /* and make sure c' = signC */ + cpGFpElementCopyPad(h2, orderLen, BN_NUMBER(pSignR), BN_SIZE(pSignR)); + if (GFP_EQ(h1, h2, orderLen)) + verifyResult = ippECValid; + } + + cpEcGFpReleasePool(1, pEC); + cpGFpReleasePool(3, pGFE); + } + } + +#if (_IPP32E >= _IPP32E_K1) +exit: +#endif + *pResult = verifyResult; + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecverifynr.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecverifynr.c new file mode 100644 index 0000000..545db07 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecverifynr.c @@ -0,0 +1,166 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// ippsGFpECVerifyNR() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsGFpECVerifyNR +// +// Purpose: NR Signature Verification. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pMsgDigest +// NULL == pRegPublic +// NULL == pSignR +// NULL == pSignS +// NULL == pResult +// NULL == pScratchBuffer +// +// ippStsContextMatchErr illegal pEC->idCtx +// pEC->subgroup == NULL +// illegal pMsgDigestDigest->idCtx +// illegal pRegPublic->idCtx +// illegal pSignR->idCtx +// illegal pSignS->idCtx +// +// ippStsMessageErr 0> MsgDigest +// order<= MsgDigest +// +// ippStsRangeErr SignR < 0 or SignS < 0 +// +// ippStsOutOfRangeErr bitsize(pRegPublic) != bitsize(prime) +// +// ippStsNotSupportedModeErr 1cpCmp_BNU(BN_NUMBER(pSignR), BN_SIZE(pSignR), pOrder, orderLen) && + 0>cpCmp_BNU(BN_NUMBER(pSignS), BN_SIZE(pSignS), pOrder, orderLen)) { + + int elmLen = GFP_FELEN(pGFE); + int pelmLen = GFP_PELEN(pGFE); + + BNU_CHUNK_T* h1 = cpGFpGetPool(3, pGFE); + BNU_CHUNK_T* h2 = h1+pelmLen; + BNU_CHUNK_T* f = h2+pelmLen; + + IppsGFpECPoint P; + cpEcGFpInitPoint(&P, cpEcGFpGetPool(1, pEC),0, pEC); + + /* P = [d]BasePoint + [c]publicKey */ + ZEXPAND_COPY_BNU(h1, orderLen, BN_NUMBER(pSignS),BN_SIZE(pSignS)); + ZEXPAND_COPY_BNU(h2, orderLen, BN_NUMBER(pSignR),BN_SIZE(pSignR)); + gfec_BasePointProduct(&P, + h1, orderLen, pRegPublic, h2, orderLen, + pEC, pScratchBuffer); + + /* check that P!=O */ + if( !gfec_IsPointAtInfinity(&P)) { + /* get P.X */ + gfec_GetPoint(h1, NULL, &P, pEC); + /* x = int(P.x) mod order */ + GFP_METHOD(pGFE)->decode(h1, h1, pGFE); + elmLen = cpMod_BNU(h1, elmLen, pOrder, orderLen); + cpGFpElementPad(h1+elmLen, orderLen-elmLen, 0); + + /* and recover msg f = (signC -x) mod order */ + ZEXPAND_COPY_BNU(f, orderLen, BN_NUMBER(pMsgDigest),BN_SIZE(pMsgDigest)); + cpModSub_BNU(h1, h2, h1, pOrder, orderLen, h2); + if(GFP_EQ(f, h1, orderLen)) + vResult = ippECValid; + } + + cpEcGFpReleasePool(1, pEC); + cpGFpReleasePool(3, pGFE); + } + + *pResult = vResult; + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecverifysm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecverifysm2.c new file mode 100644 index 0000000..ea0b5b9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpecverifysm2.c @@ -0,0 +1,184 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// ippsGFpECVerifySM2() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpeccp.h" + +/*F* +// Name: ippsGFpECVerifySM2 +// +// Purpose: SM2 Signature Verification. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pEC +// NULL == pMsgDigest +// NULL == pRegPublic +// NULL == pSignR +// NULL == pSignS +// NULL == pResult +// NULL == pScratchBuffer +// +// ippStsContextMatchErr illegal pECC->idCtx +// illegal pMsgDigestDigest->idCtx +// illegal pRegPublic->idCtx +// illegal pSignR->idCtx +// illegal pSignS->idCtx +// +// ippStsMessageErr 0> MsgDigest +// order<= MsgDigest +// +// ippStsRangeErr SignR < 0 or SignS < 0 +// +// ippStsOutOfRangeErr bitsize(pRegPublic) != bitsize(prime) +// +// ippStsNotSupportedModeErr 1cpCmp_BNU(BN_NUMBER(pSignR), BN_SIZE(pSignR), pOrder, orderLen) && + 0>cpCmp_BNU(BN_NUMBER(pSignS), BN_SIZE(pSignS), pOrder, orderLen)) { + +#if (_IPP32E >= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512IFMA) && ECP_MODULUS_ID(pEC) == cpID_PrimeTPM_SM2) { + vResult = gfec_Verify_sm2_avx512(pMsgDigest, pRegPublic, pSignR, pSignS, pEC, pScratchBuffer); + goto exit; + } /* no else */ +#endif // (_IPP32E >= _IPP32E_K1) + { + int elmLen = GFP_FELEN(pGFE); + int ns; + + BNU_CHUNK_T* r = cpGFpGetPool(4, pGFE); + BNU_CHUNK_T* s = r+orderLen; + BNU_CHUNK_T* t = s+orderLen; + BNU_CHUNK_T* f = t+orderLen; + + /* expand signatire's components */ + cpGFpElementCopyPad(r, orderLen, BN_NUMBER(pSignR), BN_SIZE(pSignR)); + cpGFpElementCopyPad(s, orderLen, BN_NUMBER(pSignS), BN_SIZE(pSignS)); + + /* t = (r+s) mod order */ + cpModAdd_BNU(t, r, s, pOrder, orderLen, f); + + /* check if t!=0 */ + if( !cpIsGFpElemEquChunk_ct(t, orderLen, 0) ) { + + /* P = [s]G +[t]regPublic, t = P.x */ + IppsGFpECPoint P, G; + cpEcGFpInitPoint(&P, cpEcGFpGetPool(1, pEC),0, pEC); + cpEcGFpInitPoint(&G, ECP_G(pEC), ECP_AFFINE_POINT|ECP_FINITE_POINT, pEC); + + gfec_BasePointProduct(&P, + s, orderLen, pRegPublic, t, orderLen, + pEC, pScratchBuffer); + + gfec_GetPoint(t, NULL, &P, pEC); + GFP_METHOD(pGFE)->decode(t, t, pGFE); + ns = cpMod_BNU(t, elmLen, pOrder, orderLen); + + cpEcGFpReleasePool(1, pEC); + IPP_UNREFERENCED_PARAMETER(ns); + } + + /* t = (msg+t) mod order */ + cpGFpElementCopyPad(f, orderLen, BN_NUMBER(pMsgDigest), BN_SIZE(pMsgDigest)); + cpModSub_BNU(f, f, pOrder, pOrder, orderLen, s); + cpModAdd_BNU(t, t, f, pOrder, orderLen, f); + + if(GFP_EQ(t, r, orderLen)) + vResult = ippECValid; + + cpGFpReleasePool(4, pGFE); + } + } + +#if (_IPP32E >= _IPP32E_K1) +exit: +#endif + *pResult = vResult; + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpelemgetsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpelemgetsize.c new file mode 100644 index 0000000..c28069a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpelemgetsize.c @@ -0,0 +1,62 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpElementGetSize() +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + +//tbcd: temporary excluded: #include + +/*F* +// Name: ippsGFpElementGetSize +// +// Purpose: Gets the size of the context for an element of the finite field. +// +// Returns: Reason: +// ippStsNullPtrErr pGFp == NULL +// pBufferSize == NULL +// ippStsContextMatchErr incorrect pGFp's context id +// ippStsNoErr no error +// +// Parameters: +// nExponents Number of exponents. +// ExpBitSize Maximum bit size of the exponents. +// pGFp Pointer to the context of the finite field. +// pBufferSize Pointer to the calculated buffer size in bytes. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpElementGetSize,(const IppsGFpState* pGFp, int* pElementSize)) +{ + IPP_BAD_PTR2_RET(pElementSize, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + + *pElementSize = (Ipp32s)sizeof(IppsGFpElement) + +GFP_FELEN(GFP_PMA(pGFp))*(Ipp32s)sizeof(BNU_CHUNK_T); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpeleminit.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpeleminit.c new file mode 100644 index 0000000..ee8e352 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpeleminit.c @@ -0,0 +1,78 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpElementInit() +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + +//tbcd: temporary excluded: #include + +/*F* +// Name: ippsGFpElementInit +// +// Purpose: Initializes the context of an element of the finite field. +// +// Returns: Reason: +// ippStsNullPtrErr pGFp == NULL +// pR == NULL +// pA && nsA>0 == NULL +// +// ippStsContextMatchErr invalid pGFp->idCtx +// invalid pR->idCtx +// +// ippStsSizeErr pA && !(0<=lenA && lenA= modulus +// +// ippStsNoErr no error +// +// Parameters: +// pA Pointer to the data array storing the finite field element. +// lenA Length of the element. +// pR Pointer to the context of the finite field element being initialized. +// pGFp Pointer to the context of the finite field. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpElementInit,(const Ipp32u* pA, int lenA, IppsGFpElement* pR, IppsGFpState* pGFp)) +{ + IPP_BAD_PTR2_RET(pR, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + + IPP_BADARG_RET(0>lenA, ippStsSizeErr); + + { + int elemLen = GFP_FELEN(GFP_PMA(pGFp)); + + Ipp8u* ptr = (Ipp8u*)pR; + ptr += sizeof(IppsGFpElement); + cpGFpElementConstruct(pR, (BNU_CHUNK_T*)ptr, elemLen); + return ippsGFpSetElement(pA, lenA, pR, pGFp); + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpexp.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpexp.c new file mode 100644 index 0000000..8363db9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpexp.c @@ -0,0 +1,80 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpExp() +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + + +/*F* +// Name: ippsGFpExp +// +// Purpose: Raise GF element to the specified power +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFp +// NULL == pA +// NULL == pR +// NULL == pE +// +// ippStsContextMatchErr invalid pGFp->idCtx +// invalid pA->idCtx +// invalid pR->idCtx +// invalid pE->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM() != GFP_FELEN() +// +// ippStsNoErr no error +// +// Parameters: +// pA Pointer to the context of the first finite field element. +// pE Pointer to the Big Number context storing the exponent +// pR Pointer to the context of the result finite field element. +// pGFp Pointer to the context of the finite field. +// pScratchBuffer Pointer to the scratch buffer. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpExp,(const IppsGFpElement* pA, const IppsBigNumState* pE, + IppsGFpElement* pR, IppsGFpState* pGFp, + Ipp8u* pScratchBuffer)) +{ + IPP_BAD_PTR4_RET(pA, pE, pR, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pA), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pR), ippStsContextMatchErr ); + + IPP_BADARG_RET( !BN_VALID_ID(pE), ippStsContextMatchErr ); + { + gsModEngine* pGFE = GFP_PMA(pGFp); + IPP_BADARG_RET( (GFPE_ROOM(pA)!=GFP_FELEN(pGFE)) || (GFPE_ROOM(pR)!=GFP_FELEN(pGFE)), ippStsOutOfRangeErr); + + cpGFpxExp(GFPE_DATA(pR), GFPE_DATA(pA), BN_NUMBER(pE), BN_SIZE(pE), pGFE, pScratchBuffer); + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpgetelem.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpgetelem.c new file mode 100644 index 0000000..83899e3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpgetelem.c @@ -0,0 +1,83 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpGetElement() +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + +//tbcd: temporary excluded: #include + +/*F* +// Name: ippsGFpGetElement +// +// Purpose: Get GF Element +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFp +// NULL == pA +// NULL == pDataA +// +// ippStsContextMatchErr invalid pGFp->idCtx +// invalid pA->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM() != GFP_FELEN() +// +// ippStsSizeErr !(0=GFP_FELEN32(pGFE)) +// +// ippStsNoErr no error +// +// Parameters: +// pA Pointer to the context of the finite field element. +// pDataA Pointer to the data array to copy the finite field element from. +// lenA Length of the data array. +// pGFp Pointer to the context of the finite field. +*F*/ + +IPPFUN(IppStatus, ippsGFpGetElement, (const IppsGFpElement* pA, Ipp32u* pDataA, int lenA, IppsGFpState* pGFp)) +{ + IPP_BAD_PTR3_RET(pA, pDataA, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pA), ippStsContextMatchErr ); + { + gsModEngine* pGFE = GFP_PMA(pGFp); + IPP_BADARG_RET( GFPE_ROOM(pA)!=GFP_FELEN(pGFE), ippStsOutOfRangeErr); + IPP_BADARG_RET( !(0=GFP_FELEN32(pGFE)), ippStsSizeErr ); + + { + int elemLen = GFP_FELEN(pGFE); + BNU_CHUNK_T* pTmp = cpGFpGetPool(1, pGFE); + //tbcd: temporary excluded: assert(NULL!=pTmp); + + cpGFpxGet(pTmp, elemLen, GFPE_DATA(pA), pGFE); + cpGFpxCopyFromChunk(pDataA, pTmp, pGFE); + + cpGFpReleasePool(1, pGFE); + return ippStsNoErr; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpgetelemoctstr.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpgetelemoctstr.c new file mode 100644 index 0000000..b1f0654 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpgetelemoctstr.c @@ -0,0 +1,91 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpGetElementOctString() +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + + +/*F* +// Name: ippsGFpGetElementOctString +// +// Purpose: Get GF Element to the octet string +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFp +// NULL == pStr +// NULL == pA +// +// ippStsContextMatchErr invalid pGFp->idCtx +// invalid pA->idCtx +// invalid pR->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM() != GFP_FELEN() +// +// ippStsSizeErr !(0=GFP_FELEN32(pGFE)) +// +// ippStsNoErr no error +// +// Parameters: +// pA Pointer to the context of the finite field element. +// pStr Pointer to the octet string. +// strSize Size of the octet string buffer in bytes. +// pGFp Pointer to the context of the finite field. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpGetElementOctString,(const IppsGFpElement* pA, Ipp8u* pStr, int strSize, IppsGFpState* pGFp)) +{ + IPP_BAD_PTR3_RET(pStr, pA, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pA), ippStsContextMatchErr ); + IPP_BADARG_RET( 0>=strSize, ippStsSizeErr ); + { + gsModEngine* pGFE = GFP_PMA(pGFp); + IPP_BADARG_RET( GFPE_ROOM(pA)!=GFP_FELEN(pGFE), ippStsOutOfRangeErr); + { + gsModEngine* pBasicGFE = cpGFpBasic(pGFE); + int basicDeg = cpGFpBasicDegreeExtension(pGFE); + int basicElemLen = GFP_FELEN(pBasicGFE); + int basicSize = BITS2WORD8_SIZE(BITSIZE_BNU(GFP_MODULUS(pBasicGFE),GFP_FELEN(pBasicGFE))); + + BNU_CHUNK_T* pDataElm = GFPE_DATA(pA); + int deg, error; + for(deg=0, error=0; deg GFP_MAX_BITSIZE), ippStsSizeErr); + + *pSize = cpGFpGetSize(feBitSize, feBitSize+BITSIZE(BNU_CHUNK_T), GFP_POOL_SIZE); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpinfo.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpinfo.c new file mode 100644 index 0000000..d3351f5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpinfo.c @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p) ectension. +// +// Context: +// ippsGFpInfo() +// +*/ + +#include "owncp.h" +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" + + +/*F* +// Name: ippsGFpInit +// +// Purpose: finite field info +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFp +// NULL == pInfo +// +// ippStsContextMatchErr invalid pGFp->idCtx +// +// ippStsNoErr no error +// +// Parameters: +// pInfo pointer to finite field infon +// pGFp Pointer to the context of the finite field. +*F*/ +IPPFUN(IppStatus, ippsGFpGetInfo,(IppsGFpInfo* pInfo, const IppsGFpState* pGFp)) +{ + IPP_BAD_PTR2_RET(pGFp, pInfo); + + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + + { + gsModEngine* pGFpx = GFP_PMA(pGFp); /* current */ + gsModEngine* pGFpBasic = cpGFpBasic(pGFpx); /* basic */ + pInfo->parentGFdegree = MOD_EXTDEG(pGFpx); /* parent extension */ + pInfo->basicGFdegree = cpGFpBasicDegreeExtension(pGFpx); /* total basic extention */ + pInfo->basicElmBitSize = GFP_FEBITLEN(pGFpBasic); /* basic bitsise */ + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpinit.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpinit.c new file mode 100644 index 0000000..4764a0a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpinit.c @@ -0,0 +1,110 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpInit +// +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + + +/*F* +// Name: ippsGFpInit +// +// Purpose: initializes prime finite field GF(p) +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFpMethod +// NULL == pGFp +// +// ippStsSizeErr !(IPP_MIN_GF_BITSIZE <= primeBitSize <=IPP_MAX_GF_BITSIZE +// +// ippStsContextMatchErr invalid pPrime->idCtx +// +// ippStsBadArgErr pGFpMethod != ippsGFpMethod_pXXX() or != ippsGFpMethod_pArb() +// prime != pGFpMethod->modulus +// prime <0 +// bitsize(prime) != primeBitSize +// prime IPP_MAX_GF_BITSIZE), ippStsSizeErr); + + /* use ippsGFpInitFixed() if NULL==pPrimeBN */ + if(!pPrimeBN) + return ippsGFpInitFixed(primeBitSize, pGFpMethod, pGFp); + + /* use ippsGFpInitArbitrary() if NULL==pGFpMethod */ + if(!pGFpMethod) + return ippsGFpInitArbitrary(pPrimeBN, primeBitSize, pGFp); + + /* test parameters if both pPrimeBN and method are defined */ + else { + IppStatus sts; + + /* test input prime */ + IPP_BADARG_RET(!BN_VALID_ID(pPrimeBN), ippStsContextMatchErr); + IPP_BADARG_RET(BN_SIGN(pPrimeBN)!= IppsBigNumPOS, ippStsBadArgErr); /* prime is negative */ + IPP_BADARG_RET(BITSIZE_BNU(BN_NUMBER(pPrimeBN),BN_SIZE(pPrimeBN)) != primeBitSize, ippStsBadArgErr); /* primeBitSize == bitsize(prime) */ + IPP_BADARG_RET((BN_SIZE(pPrimeBN)==1) && (BN_NUMBER(pPrimeBN)[0]modulusID & cpID_Prime), ippStsBadArgErr); + + /* test if size of the prime is matched to method's prime */ + IPP_BADARG_RET(pGFpMethod->modulusBitDeg && (primeBitSize!=pGFpMethod->modulusBitDeg), ippStsBadArgErr); + + /* if method assumes fixed prime value */ + if(pGFpMethod->modulus) { + int primeLen = BITS_BNU_CHUNK(primeBitSize); + IPP_BADARG_RET(cpCmp_BNU(BN_NUMBER(pPrimeBN), primeLen, pGFpMethod->modulus, primeLen), ippStsBadArgErr); + } + + /* init GF */ + sts = cpGFpInitGFp(primeBitSize, pGFp); + + /* set up GF and find quadratic nonresidue */ + if(ippStsNoErr==sts) { + cpGFpSetGFp(BN_NUMBER(pPrimeBN), primeBitSize, pGFpMethod, pGFp); + } + + return sts; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpinitarbitrary.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpinitarbitrary.c new file mode 100644 index 0000000..ccdd9ce --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpinitarbitrary.c @@ -0,0 +1,83 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpInitArbitrary() +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + + +/*F* +// Name: ippsGFpInitArbitrary +// +// Purpose: initializes prime finite field GF(p) +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pPrime +// NULL == pGFp +// +// ippStsSizeErr !(IPP_MIN_GF_BITSIZE <= primeBitSize <=IPP_MAX_GF_BITSIZE) +// +// ippStsContextMatchErr incorrect pPrime context ID +// +// ippStsBadArgErr prime <0 +// bitsize(prime) != primeBitSize +// prime IPP_MAX_GF_BITSIZE), ippStsSizeErr); + + IPP_BAD_PTR1_RET(pPrime); + IPP_BADARG_RET(!BN_VALID_ID(pPrime), ippStsContextMatchErr); + IPP_BADARG_RET(BN_SIGN(pPrime)!= IppsBigNumPOS, ippStsBadArgErr); /* prime is negative */ + IPP_BADARG_RET(BITSIZE_BNU(BN_NUMBER(pPrime),BN_SIZE(pPrime)) != primeBitSize, ippStsBadArgErr); /* primeBitSize == bitsize(prime) */ + IPP_BADARG_RET((BN_SIZE(pPrime)==1) && (BN_NUMBER(pPrime)[0]modulusID & cpID_Prime), ippStsBadArgErr); + /* test if method is not prime based arbitrary */ + IPP_BADARG_RET(!pGFpMethod->modulus, ippStsBadArgErr); + /* size of the underlying prime must be equal to primeBitSize parameter*/ + IPP_BADARG_RET(pGFpMethod->modulusBitDeg!=primeBitSize, ippStsBadArgErr); + + { + /* init GF */ + IppStatus sts = cpGFpInitGFp(primeBitSize, pGFp); + + /* set up GF engine */ + if(ippStsNoErr==sts) { + cpGFpSetGFp(pGFpMethod->modulus, primeBitSize, pGFpMethod, pGFp); + } + + return sts; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpinv.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpinv.c new file mode 100644 index 0000000..7f40814 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpinv.c @@ -0,0 +1,77 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpInv() +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + + +/*F* +// Name: ippsGFpInv +// +// Purpose: Multiplicative inverse GF element +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFp +// NULL == pA +// NULL == pR +// +// ippStsContextMatchErr invalid pGFp->idCtx +// invalid pA->idCtx +// invalid pR->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM() != GFP_FELEN() +// +// ippStsDivByZeroErr pA is zero +// +// ippStsBadArgErr computational error +// +// ippStsNoErr no error +// +// Parameters: +// pA Pointer to the context of the source finite field element. +// pR Pointer to the context of the result finite field element. +// pGFp Pointer to the context of the finite field. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpInv,(const IppsGFpElement* pA, + IppsGFpElement* pR, IppsGFpState* pGFp)) +{ + IPP_BAD_PTR3_RET(pA, pR, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pA), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pR), ippStsContextMatchErr ); + { + gsModEngine* pGFE = GFP_PMA(pGFp); + IPP_BADARG_RET( (GFPE_ROOM(pA)!=GFP_FELEN(pGFE)) || (GFPE_ROOM(pR)!=GFP_FELEN(pGFE)), ippStsOutOfRangeErr); + IPP_BADARG_RET( GFP_IS_ZERO(GFPE_DATA(pA),GFP_FELEN(pGFE)), ippStsDivByZeroErr ); + + return NULL != cpGFpxInv(GFPE_DATA(pR), GFPE_DATA(pA), pGFE)? ippStsNoErr : ippStsBadArgErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpisunityelem.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpisunityelem.c new file mode 100644 index 0000000..614b8ff --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpisunityelem.c @@ -0,0 +1,84 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpIsUnityElement() +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + + +/*F* +// Name: ippsGFpIsUnityElement +// +// Purpose: Compare GF Element with unity element +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFp +// NULL == pA +// NULL == pResult +// +// ippStsContextMatchErr invalid pGFp->idCtx +// invalid pA->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM() != GFP_FELEN() +// +// ippStsNoErr no error +// +// Parameters: +// pA Pointer to the context of the finite field element. +// pResult Pointer to the result of the comparison +// pGFp Pointer to the context of the finite field. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpIsUnityElement,(const IppsGFpElement* pA, + int* pResult, + const IppsGFpState* pGFp)) +{ + IPP_BAD_PTR3_RET(pA, pResult, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pA), ippStsContextMatchErr ); + { + gsModEngine* pGFE = GFP_PMA(pGFp); + IPP_BADARG_RET( GFPE_ROOM(pA)!=GFP_FELEN(pGFE), ippStsOutOfRangeErr); + { + gsModEngine* pBasicGFE = cpGFpBasic(pGFE); + int basicElmLen = GFP_FELEN(pBasicGFE); + BNU_CHUNK_T* pUnity = GFP_MNT_R(pBasicGFE); + + int elmLen = GFP_FELEN(pGFE); + int flag; + + FIX_BNU(pUnity, basicElmLen); + FIX_BNU(GFPE_DATA(pA), elmLen); + + flag = (basicElmLen==elmLen) && (GFP_EQ(GFPE_DATA(pA), pUnity, elmLen)); + *pResult = (1==flag)? IPP_IS_EQ : IPP_IS_NE; + return ippStsNoErr; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpiszeroelem.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpiszeroelem.c new file mode 100644 index 0000000..13e82a5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpiszeroelem.c @@ -0,0 +1,74 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpIsZeroElement() +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + + +/*F* +// Name: ippsGFpIsZeroElement +// +// Purpose: Compare GF Element with zero element +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFp +// NULL == pA +// NULL == pResult +// +// ippStsContextMatchErr invalid pGFp->idCtx +// invalid pA->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM() != GFP_FELEN() +// +// ippStsNoErr no error +// +// Parameters: +// pA Pointer to the context of the finite field element. +// pResult Pointer to the result of the comparison +// pGFp Pointer to the context of the finite field. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpIsZeroElement,(const IppsGFpElement* pA, + int* pResult, + const IppsGFpState* pGFp)) +{ + IPP_BAD_PTR3_RET(pA, pResult, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pA), ippStsContextMatchErr ); + { + gsModEngine* pGFE = GFP_PMA(pGFp); + IPP_BADARG_RET( GFPE_ROOM(pA)!=GFP_FELEN(pGFE), ippStsOutOfRangeErr); + { + int flag = GFP_IS_ZERO(GFPE_DATA(pA), GFP_FELEN(pGFE)); + *pResult = (1==flag)? IPP_IS_EQ : IPP_IS_NE; + return ippStsNoErr; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod.h new file mode 100644 index 0000000..6e79cb0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod.h @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives +// Internal GF(p) basic Definitions & Function Prototypes +// +*/ +#if !defined(_CP_GFP_METHOD_H) +#define _CP_GFP_METHOD_H + +#include "owncp.h" + +#include "pcpbnuimpl.h" +#include "gsmodmethod.h" + +/* modulus ID */ +typedef enum { + cpID_Prime = 0x1000, + cpID_PrimeP192r1 = cpID_Prime+6, + cpID_PrimeP224r1 = cpID_Prime+7, + cpID_PrimeP256r1 = cpID_Prime+8, + cpID_PrimeP384r1 = cpID_Prime+9, + cpID_PrimeP521r1 = cpID_Prime+10, + cpID_PrimeTPM_SM2 = cpID_Prime+11, + cpID_PrimeTPM_BN = cpID_Prime+12, + + cpID_Poly = 0x10000000, /* id=0x10000000: general polynomial */ + cpID_Binom = 0x01000000, /* id=0x11000000: x^d+a */ + + cpID_Binom2_epid20 = cpID_Binom|0x220000, /* 0x11220000 */ + cpID_Binom3_epid20 = cpID_Binom|0x230000 /* 0x11230000 */ + +} cpModulusID; + +typedef struct _cpGFpMethod { + cpModulusID modulusID; + int modulusBitDeg; + const BNU_CHUNK_T* modulus; + const gsModMethod* arith; + const void* arith_alt; // alternative radix implementation +} cpGFpMethod; + +/* common GF arith methods */ +#define gsArithGFp OWNAPI(gsArithGFp) + IPP_OWN_DECL (gsModMethod*, gsArithGFp, (void)) + +#endif /* _CP_GFP_METHOD_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_192r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_192r1.c new file mode 100644 index 0000000..eab0fa2 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_192r1.c @@ -0,0 +1,207 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// GF(p) methods +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpbnumisc.h" +#include "gsmodstuff.h" +#include "pcpgfpstuff.h" +#include "pcpgfpmethod.h" +#include "pcpecprime.h" + +//tbcd: temporary excluded: #include + +#if(_IPP >= _IPP_P8) || (_IPP32E >= _IPP32E_M7) + +/* arithmetic over P-192r1 NIST modulus */ +#define p192r1_add OWNAPI(p192r1_add) + IPP_OWN_DECL (BNU_CHUNK_T*, p192r1_add, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +#define p192r1_sub OWNAPI(p192r1_sub) + IPP_OWN_DECL (BNU_CHUNK_T*, p192r1_sub, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +#define p192r1_neg OWNAPI(p192r1_neg) + IPP_OWN_DECL (BNU_CHUNK_T*, p192r1_neg, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p192r1_div_by_2 OWNAPI(p192r1_div_by_2) + IPP_OWN_DECL (BNU_CHUNK_T*, p192r1_div_by_2, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p192r1_mul_by_2 OWNAPI(p192r1_mul_by_2) + IPP_OWN_DECL (BNU_CHUNK_T*, p192r1_mul_by_2, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p192r1_mul_by_3 OWNAPI(p192r1_mul_by_3) + IPP_OWN_DECL (BNU_CHUNK_T*, p192r1_mul_by_3, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) + +#if(_IPP_ARCH ==_IPP_ARCH_EM64T) +#define p192r1_mul_montl OWNAPI(p192r1_mul_montl) + IPP_OWN_DECL (BNU_CHUNK_T*, p192r1_mul_montl, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +#define p192r1_mul_montx OWNAPI(p192r1_mul_montx) + IPP_OWN_DECL (BNU_CHUNK_T*, p192r1_mul_montx, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +#define p192r1_sqr_montl OWNAPI(p192r1_sqr_montl) + IPP_OWN_DECL (BNU_CHUNK_T*, p192r1_sqr_montl, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p192r1_sqr_montx OWNAPI(p192r1_sqr_montx) + IPP_OWN_DECL (BNU_CHUNK_T*, p192r1_sqr_montx, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p192r1_to_mont OWNAPI(p192r1_to_mont) + IPP_OWN_DECL (BNU_CHUNK_T*, p192r1_to_mont, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p192r1_mont_back OWNAPI(p192r1_mont_back) + IPP_OWN_DECL (BNU_CHUNK_T*, p192r1_mont_back, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#endif + +#if(_IPP_ARCH ==_IPP_ARCH_IA32) +#define p192r1_mul_mont_slm OWNAPI(p192r1_mul_mont_slm) + IPP_OWN_DECL (BNU_CHUNK_T*, p192r1_mul_mont_slm, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +#define p192r1_sqr_mont_slm OWNAPI(p192r1_sqr_mont_slm) + IPP_OWN_DECL (BNU_CHUNK_T*, p192r1_sqr_mont_slm, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p192r1_mred OWNAPI(p192r1_mred) + IPP_OWN_DECL (BNU_CHUNK_T*, p192r1_mred, (BNU_CHUNK_T* res, BNU_CHUNK_T* product)) +#endif + +#define OPERAND_BITSIZE (192) +#define LEN_P192 (BITS_BNU_CHUNK(OPERAND_BITSIZE)) + +/* +// ia32 multiplicative methods +*/ +#if (_IPP_ARCH ==_IPP_ARCH_IA32) +IPP_OWN_DEFN (static BNU_CHUNK_T*, p192r1_mul_montl, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFE)) +{ + BNU_CHUNK_T* product = cpGFpGetPool(2, pGFE); + //tbcd: temporary excluded: assert(NULL!=product); + + cpMulAdc_BNU_school(product, pA,LEN_P192, pB,LEN_P192); + p192r1_mred(pR, product); + + cpGFpReleasePool(2, pGFE); + return pR; +} + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p192r1_sqr_montl, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + BNU_CHUNK_T* product = cpGFpGetPool(2, pGFE); + //tbcd: temporary excluded: assert(NULL!=product); + + cpSqrAdc_BNU_school(product, pA,LEN_P192); + p192r1_mred(pR, product); + + cpGFpReleasePool(2, pGFE); + return pR; +} + + +/* +// Montgomery domain conversion constants +*/ +static BNU_CHUNK_T RR[] = { + 0x00000001,0x00000000,0x00000002,0x00000000, + 0x00000001,0x00000000}; + +static BNU_CHUNK_T one[] = { + 1,0,0,0,0,0}; + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p192r1_to_mont, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return p192r1_mul_montl(pR, pA, (BNU_CHUNK_T*)RR, pGFE); +} + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p192r1_mont_back, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return p192r1_mul_montl(pR, pA, (BNU_CHUNK_T*)one, pGFE); +} + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p192r1_to_mont_slm, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return p192r1_mul_mont_slm(pR, pA, (BNU_CHUNK_T*)RR, pGFE); +} + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p192r1_mont_back_slm, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return p192r1_mul_mont_slm(pR, pA, (BNU_CHUNK_T*)one, pGFE); +} +#endif /* _IPP >= _IPP_P8 */ + +/* +// return specific gf p192r1 arith methods, +// p192r1 = 2^192 -2^64 -1 (NIST P192r1) +*/ +static gsModMethod* gsArithGF_p192r1 (void) +{ + static gsModMethod m = { + p192r1_to_mont, + p192r1_mont_back, + p192r1_mul_montl, + p192r1_sqr_montl, + NULL, + p192r1_add, + p192r1_sub, + p192r1_neg, + p192r1_div_by_2, + p192r1_mul_by_2, + p192r1_mul_by_3, + }; + + #if(_IPP_ARCH==_IPP_ARCH_EM64T) && ((_ADCOX_NI_ENABLING_==_FEATURE_ON_) || (_ADCOX_NI_ENABLING_==_FEATURE_TICKTOCK_)) + if(IsFeatureEnabled(ippCPUID_ADCOX)) { + m.mul = p192r1_mul_montx; + m.sqr = p192r1_sqr_montx; + } + #endif + + #if(_IPP_ARCH==_IPP_ARCH_IA32) + if(IsFeatureEnabled(ippCPUID_SSSE3|ippCPUID_MOVBE) && !IsFeatureEnabled(ippCPUID_AVX)) { + m.mul = p192r1_mul_mont_slm; + m.sqr = p192r1_sqr_mont_slm; + m.encode = p192r1_to_mont_slm; + m.decode = p192r1_mont_back_slm; + } + #endif + + return &m; +} +#endif /* (_IPP >= _IPP_P8) || (_IPP32E >= _IPP32E_M7) */ + +/*F* +// Name: ippsGFpMethod_p192r1 +// +// Purpose: Returns a reference to an implementation of +// arithmetic operations over GF(q). +// +// Returns: Pointer to a structure containing an implementation of arithmetic +// operations over GF(q). q = 2^192 - 2^64 - 1 +*F*/ + +IPPFUN( const IppsGFpMethod*, ippsGFpMethod_p192r1, (void) ) +{ + static IppsGFpMethod method = { + cpID_PrimeP192r1, + 192, + secp192r1_p, + NULL, + NULL + }; + + #if(_IPP >= _IPP_P8) || (_IPP32E >= _IPP32E_M7) + method.arith = gsArithGF_p192r1(); + #else + method.arith = gsArithGFp(); + #endif + + return &method; +} + +#undef LEN_P192 +#undef OPERAND_BITSIZE diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_224r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_224r1.c new file mode 100644 index 0000000..66278e5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_224r1.c @@ -0,0 +1,207 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// GF(p) methods +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpbnumisc.h" +#include "gsmodstuff.h" +#include "pcpgfpstuff.h" +#include "pcpgfpmethod.h" +#include "pcpecprime.h" + +//tbcd: temporary excluded: #include + +#if(_IPP >= _IPP_P8) || (_IPP32E >= _IPP32E_M7) + +/* arithmetic over P-224r1 NIST modulus */ +#define p224r1_add OWNAPI(p224r1_add) + IPP_OWN_DECL (BNU_CHUNK_T*, p224r1_add, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +#define p224r1_sub OWNAPI(p224r1_sub) + IPP_OWN_DECL (BNU_CHUNK_T*, p224r1_sub, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +#define p224r1_neg OWNAPI(p224r1_neg) + IPP_OWN_DECL (BNU_CHUNK_T*, p224r1_neg, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p224r1_div_by_2 OWNAPI(p224r1_div_by_2) + IPP_OWN_DECL (BNU_CHUNK_T*, p224r1_div_by_2, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p224r1_mul_by_2 OWNAPI(p224r1_mul_by_2) + IPP_OWN_DECL (BNU_CHUNK_T*, p224r1_mul_by_2, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p224r1_mul_by_3 OWNAPI(p224r1_mul_by_3) + IPP_OWN_DECL (BNU_CHUNK_T*, p224r1_mul_by_3, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) + +#if(_IPP_ARCH ==_IPP_ARCH_EM64T) +#define p224r1_mul_montl OWNAPI(p224r1_mul_montl) + IPP_OWN_DECL (BNU_CHUNK_T*, p224r1_mul_montl, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +#define p224r1_mul_montx OWNAPI(p224r1_mul_montx) + IPP_OWN_DECL (BNU_CHUNK_T*, p224r1_mul_montx, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +#define p224r1_sqr_montl OWNAPI(p224r1_sqr_montl) + IPP_OWN_DECL (BNU_CHUNK_T*, p224r1_sqr_montl, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p224r1_sqr_montx OWNAPI(p224r1_sqr_montx) + IPP_OWN_DECL (BNU_CHUNK_T*, p224r1_sqr_montx, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p224r1_to_mont OWNAPI(p224r1_to_mont) + IPP_OWN_DECL (BNU_CHUNK_T*, p224r1_to_mont, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p224r1_mont_back OWNAPI(p224r1_mont_back) + IPP_OWN_DECL (BNU_CHUNK_T*, p224r1_mont_back, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#endif + +#if(_IPP_ARCH ==_IPP_ARCH_IA32) +#define p224r1_mul_mont_slm OWNAPI(p224r1_mul_mont_slm) + IPP_OWN_DECL (BNU_CHUNK_T*, p224r1_mul_mont_slm, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +#define p224r1_sqr_mont_slm OWNAPI(p224r1_sqr_mont_slm) + IPP_OWN_DECL (BNU_CHUNK_T*, p224r1_sqr_mont_slm, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p224r1_mred OWNAPI(p224r1_mred) + IPP_OWN_DECL (BNU_CHUNK_T*, p224r1_mred, (BNU_CHUNK_T* res, BNU_CHUNK_T* product)) +#endif + +#define OPERAND_BITSIZE (224) +#define LEN_P224 (BITS_BNU_CHUNK(OPERAND_BITSIZE)) + +/* +// ia32 multiplicative methods +*/ +#if (_IPP_ARCH ==_IPP_ARCH_IA32) +IPP_OWN_DEFN (static BNU_CHUNK_T*, p224r1_mul_montl, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFE)) +{ + BNU_CHUNK_T* product = cpGFpGetPool(2, pGFE); + //tbcd: temporary excluded: assert(NULL!=product); + + cpMulAdc_BNU_school(product, pA,LEN_P224, pB,LEN_P224); + p224r1_mred(pR, product); + + cpGFpReleasePool(2, pGFE); + return pR; +} + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p224r1_sqr_montl, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + BNU_CHUNK_T* product = cpGFpGetPool(2, pGFE); + //tbcd: temporary excluded: assert(NULL!=product); + + cpSqrAdc_BNU_school(product, pA,LEN_P224); + p224r1_mred(pR, product); + + cpGFpReleasePool(2, pGFE); + return pR; +} + + +/* +// Montgomery domain conversion constants +*/ +static BNU_CHUNK_T RR[] = { + 0x00000001,0x00000000,0x00000000,0xfffffffe, + 0xffffffff,0xffffffff,0x00000000}; + +static BNU_CHUNK_T one[] = { + 1,0,0,0,0,0,0}; + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p224r1_to_mont, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return p224r1_mul_montl(pR, pA, (BNU_CHUNK_T*)RR, pGFE); +} + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p224r1_mont_back, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return p224r1_mul_montl(pR, pA, (BNU_CHUNK_T*)one, pGFE); +} + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p224r1_to_mont_slm, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return p224r1_mul_mont_slm(pR, pA, (BNU_CHUNK_T*)RR, pGFE); +} + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p224r1_mont_back_slm, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return p224r1_mul_mont_slm(pR, pA, (BNU_CHUNK_T*)one, pGFE); +} +#endif /* _IPP >= _IPP_P8 */ + +/* +// return specific gf p224r1 arith methods, +// p224r1 = 2^224 -2^96 +1 (NIST P224r1) +*/ +static gsModMethod* gsArithGF_p224r1 (void) +{ + static gsModMethod m = { + p224r1_to_mont, + p224r1_mont_back, + p224r1_mul_montl, + p224r1_sqr_montl, + NULL, + p224r1_add, + p224r1_sub, + p224r1_neg, + p224r1_div_by_2, + p224r1_mul_by_2, + p224r1_mul_by_3, + }; + + #if(_IPP_ARCH==_IPP_ARCH_EM64T) && ((_ADCOX_NI_ENABLING_==_FEATURE_ON_) || (_ADCOX_NI_ENABLING_==_FEATURE_TICKTOCK_)) + if(IsFeatureEnabled(ippCPUID_ADCOX)) { + m.mul = p224r1_mul_montx; + m.sqr = p224r1_sqr_montx; + } + #endif + + #if(_IPP_ARCH==_IPP_ARCH_IA32) + if(IsFeatureEnabled(ippCPUID_SSSE3|ippCPUID_MOVBE) && !IsFeatureEnabled(ippCPUID_AVX)) { + m.mul = p224r1_mul_mont_slm; + m.sqr = p224r1_sqr_mont_slm; + m.encode = p224r1_to_mont_slm; + m.decode = p224r1_mont_back_slm; + } + #endif + + return &m; +} +#endif /* (_IPP >= _IPP_P8) || (_IPP32E >= _IPP32E_M7) */ + +/*F* +// Name: ippsGFpMethod_p224r1 +// +// Purpose: Returns a reference to an implementation of +// arithmetic operations over GF(q). +// +// Returns: Pointer to a structure containing an implementation of arithmetic +// operations over GF(q). q = 2^224 - 2^96 - 1 +*F*/ + +IPPFUN( const IppsGFpMethod*, ippsGFpMethod_p224r1, (void) ) +{ + static IppsGFpMethod method = { + cpID_PrimeP224r1, + 224, + secp224r1_p, + NULL, + NULL + }; + + #if(_IPP >= _IPP_P8) || (_IPP32E >= _IPP32E_M7) + method.arith = gsArithGF_p224r1(); + #else + method.arith = gsArithGFp(); + #endif + + return &method; +} + +#undef LEN_P224 +#undef OPERAND_BITSIZE diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_256.c new file mode 100644 index 0000000..317f326 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_256.c @@ -0,0 +1,51 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// GF(p) methods +// +*/ +#include "pcpgfpmethod_256.h" + +/*F* +// Name: ippsGFpMethod_p256 +// +// Purpose: Returns a reference to an implementation of +// arithmetic operations over GF(q). +// +// Returns: Pointer to a structure containing an implementation of arithmetic +// operations over GF(q). Arbitrary 256 bit modulus. +*F*/ + +IPPFUN( const IppsGFpMethod*, ippsGFpMethod_p256, (void) ) +{ + static IppsGFpMethod method = { + cpID_Prime, + 256, + NULL, + NULL, + NULL + }; + + #if(_IPP32E >= _IPP32E_M7) + method.arith = gsArithGF_p256(); + #else + method.arith = gsArithGFp(); + #endif + return &method; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_256.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_256.h new file mode 100644 index 0000000..5a5318d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_256.h @@ -0,0 +1,130 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// GF(p) methods +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpbnumisc.h" +#include "pcpgfpstuff.h" +#include "pcpgfpmethod.h" +#include "pcpecprime.h" + +#if !defined(_PCP_GFPMETHOD_256_H_) +#define _PCP_GFPMETHOD_256_H_ + +#if(_IPP32E >= _IPP32E_M7) + +/* arithmetic over arbitrary 256r-bit modulus */ +#define gf256_add OWNAPI(gf256_add) + IPP_OWN_DECL (BNU_CHUNK_T*, gf256_add, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, const BNU_CHUNK_T* pModulus)) +#define gf256_sub OWNAPI(gf256_sub) + IPP_OWN_DECL (BNU_CHUNK_T*, gf256_sub, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, const BNU_CHUNK_T* pModulus)) +#define gf256_neg OWNAPI(gf256_neg) + IPP_OWN_DECL (BNU_CHUNK_T*, gf256_neg, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pModulus)) +#define gf256_mulm OWNAPI(gf256_mulm) + IPP_OWN_DECL (BNU_CHUNK_T*, gf256_mulm, (BNU_CHUNK_T* pR,const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, const BNU_CHUNK_T* pModulus, BNU_CHUNK_T m0)) +#define gf256_sqrm OWNAPI(gf256_sqrm) + IPP_OWN_DECL (BNU_CHUNK_T*, gf256_sqrm, (BNU_CHUNK_T* pR,const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pModulus, BNU_CHUNK_T m0)) +#define gf256_div2 OWNAPI(gf256_div2) + IPP_OWN_DECL (BNU_CHUNK_T*, gf256_div2, (BNU_CHUNK_T* pR,const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pModulus)) + +#define OPERAND_BITSIZE (256) +#define LEN_P256 (BITS_BNU_CHUNK(OPERAND_BITSIZE)) + +static BNU_CHUNK_T* p256_add(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFE) +{ + return gf256_add(pR, pA, pB, GFP_MODULUS(pGFE)); +} + +static BNU_CHUNK_T* p256_sub(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFE) +{ + return gf256_sub(pR, pA, pB, GFP_MODULUS(pGFE)); +} + +static BNU_CHUNK_T* p256_neg(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE) +{ + return gf256_neg(pR, pA, GFP_MODULUS(pGFE)); +} + +static BNU_CHUNK_T* p256_div_by_2(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE) +{ + return gf256_div2(pR, pA, GFP_MODULUS(pGFE)); +} + +static BNU_CHUNK_T* p256_mul_by_2(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE) +{ + return gf256_add(pR, pA, pA, GFP_MODULUS(pGFE)); +} + +static BNU_CHUNK_T* p256_mul_by_3(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE) +{ + BNU_CHUNK_T tmp[LEN_P256]; + gf256_add(tmp, pA, pA, GFP_MODULUS(pGFE)); + return gf256_add(pR, tmp, pA, GFP_MODULUS(pGFE)); +} + +static BNU_CHUNK_T* p256_mul_montl(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFE) +{ + return gf256_mulm(pR, pA, pB, GFP_MODULUS(pGFE), GFP_MNT_FACTOR(pGFE)); +} + +static BNU_CHUNK_T* p256_sqr_montl(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE) +{ + return gf256_sqrm(pR, pA, GFP_MODULUS(pGFE), GFP_MNT_FACTOR(pGFE)); +} + +static BNU_CHUNK_T* p256_to_mont(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE) +{ + return gf256_mulm(pR, pA, GFP_MNT_RR(pGFE), GFP_MODULUS(pGFE), GFP_MNT_FACTOR(pGFE)); +} + +static BNU_CHUNK_T one[] = {1,0,0,0}; + +static BNU_CHUNK_T* p256_mont_back(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE) +{ + return gf256_mulm(pR, pA, one, GFP_MODULUS(pGFE), GFP_MNT_FACTOR(pGFE)); +} + +/* return specific gf p256 arith methods */ +static gsModMethod* gsArithGF_p256(void) +{ + static gsModMethod m = { + p256_to_mont, + p256_mont_back, + p256_mul_montl, + p256_sqr_montl, + NULL, + p256_add, + p256_sub, + p256_neg, + p256_div_by_2, + p256_mul_by_2, + p256_mul_by_3, + }; + return &m; +} +#endif /* _IPP32E >= _IPP32E_M7 */ + +#undef LEN_P256 +#undef OPERAND_BITSIZE + +#endif /* #if !defined(_PCP_GFPMETHOD_256_H_) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_256bn.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_256bn.c new file mode 100644 index 0000000..3f77193 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_256bn.c @@ -0,0 +1,53 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// GF(p) methods +// +*/ + +#include "pcpgfpmethod_256.h" + +/*F* +// Name: ippsGFpMethod_p256bn +// +// Purpose: Returns a reference to an implementation of +// arithmetic operations over GF(q). +// +// Returns: Pointer to a structure containing an implementation of arithmetic +// operations over GF(q). q = +// 0xFFFFFFFFFFFCF0CD46E5F25EEE71A49F0CDC65FB12980A82D3292DDBAED33013 +*F*/ + +IPPFUN( const IppsGFpMethod*, ippsGFpMethod_p256bn, (void) ) +{ + static IppsGFpMethod method = { + cpID_Prime, + 256, + tpmBN_p256p_p, + NULL, + NULL + }; + + #if(_IPP32E >= _IPP32E_M7) + method.arith = gsArithGF_p256(); + #else + method.arith = gsArithGFp(); + #endif + return &method; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_256r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_256r1.c new file mode 100644 index 0000000..1b8c017 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_256r1.c @@ -0,0 +1,215 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// GF(p) methods +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpbnumisc.h" +#include "gsmodstuff.h" +#include "pcpgfpstuff.h" +#include "pcpgfpmethod.h" +#include "pcpecprime.h" + +#include "ifma_arith_method.h" + +//tbcd: temporary excluded: #include + +#if(_IPP >= _IPP_P8) || (_IPP32E >= _IPP32E_M7) + +/* arithmetic over P-256r1 NIST modulus */ +#define p256r1_add OWNAPI(p256r1_add) + IPP_OWN_DECL (BNU_CHUNK_T*, p256r1_add, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +#define p256r1_sub OWNAPI(p256r1_sub) + IPP_OWN_DECL (BNU_CHUNK_T*, p256r1_sub, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +#define p256r1_neg OWNAPI(p256r1_neg) + IPP_OWN_DECL (BNU_CHUNK_T*, p256r1_neg, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p256r1_div_by_2 OWNAPI(p256r1_div_by_2) + IPP_OWN_DECL (BNU_CHUNK_T*, p256r1_div_by_2, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p256r1_mul_by_2 OWNAPI(p256r1_mul_by_2) + IPP_OWN_DECL (BNU_CHUNK_T*, p256r1_mul_by_2, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p256r1_mul_by_3 OWNAPI(p256r1_mul_by_3) + IPP_OWN_DECL (BNU_CHUNK_T*, p256r1_mul_by_3, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) + +#if(_IPP_ARCH ==_IPP_ARCH_EM64T) +#define p256r1_mul_montl OWNAPI(p256r1_mul_montl) + IPP_OWN_DECL (BNU_CHUNK_T*, p256r1_mul_montl, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +#define p256r1_mul_montx OWNAPI(p256r1_mul_montx) + IPP_OWN_DECL (BNU_CHUNK_T*, p256r1_mul_montx, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +#define p256r1_sqr_montl OWNAPI(p256r1_sqr_montl) + IPP_OWN_DECL (BNU_CHUNK_T*, p256r1_sqr_montl, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p256r1_sqr_montx OWNAPI(p256r1_sqr_montx) + IPP_OWN_DECL (BNU_CHUNK_T*, p256r1_sqr_montx, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p256r1_to_mont OWNAPI(p256r1_to_mont) + IPP_OWN_DECL (BNU_CHUNK_T*, p256r1_to_mont, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p256r1_mont_back OWNAPI(p256r1_mont_back) + IPP_OWN_DECL (BNU_CHUNK_T*, p256r1_mont_back, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#endif + +#if(_IPP_ARCH ==_IPP_ARCH_IA32) +#define p256r1_mul_mont_slm OWNAPI(p256r1_mul_mont_slm) + IPP_OWN_DECL (BNU_CHUNK_T*, p256r1_mul_mont_slm, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +#define p256r1_sqr_mont_slm OWNAPI(p256r1_sqr_mont_slm) + IPP_OWN_DECL (BNU_CHUNK_T*, p256r1_sqr_mont_slm, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p256r1_mred OWNAPI(p256r1_mred) + IPP_OWN_DECL (BNU_CHUNK_T*, p256r1_mred, (BNU_CHUNK_T* res, BNU_CHUNK_T* product)) +#endif + +#define OPERAND_BITSIZE (256) +#define LEN_P256 (BITS_BNU_CHUNK(OPERAND_BITSIZE)) + +/* +// ia32 multiplicative methods +*/ +#if (_IPP_ARCH ==_IPP_ARCH_IA32) +IPP_OWN_DEFN (static BNU_CHUNK_T*, p256r1_mul_montl, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFE)) +{ + BNU_CHUNK_T* product = cpGFpGetPool(2, pGFE); + //tbcd: temporary excluded: assert(NULL!=product); + + cpMulAdc_BNU_school(product, pA,LEN_P256, pB,LEN_P256); + p256r1_mred(pR, product); + + cpGFpReleasePool(2, pGFE); + return pR; +} + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p256r1_sqr_montl, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + BNU_CHUNK_T* product = cpGFpGetPool(2, pGFE); + //tbcd: temporary excluded: assert(NULL!=product); + + cpSqrAdc_BNU_school(product, pA,LEN_P256); + p256r1_mred(pR, product); + + cpGFpReleasePool(2, pGFE); + return pR; +} + + +/* +// Montgomery domain conversion constants +*/ +static BNU_CHUNK_T RR[] = { + 0x00000003,0x00000000, 0xffffffff,0xfffffffb, + 0xfffffffe,0xffffffff, 0xfffffffd,0x00000004}; + +static BNU_CHUNK_T one[] = { + 1,0,0,0,0,0,0,0}; + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p256r1_to_mont, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return p256r1_mul_montl(pR, pA, (BNU_CHUNK_T*)RR, pGFE); +} + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p256r1_mont_back, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return p256r1_mul_montl(pR, pA, (BNU_CHUNK_T*)one, pGFE); +} + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p256r1_to_mont_slm, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return p256r1_mul_mont_slm(pR, pA, (BNU_CHUNK_T*)RR, pGFE); +} + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p256r1_mont_back_slm, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return p256r1_mul_mont_slm(pR, pA, (BNU_CHUNK_T*)one, pGFE); +} +#endif /* _IPP >= _IPP_P8 */ + +/* +// return specific gf p256r1 arith methods, +// p256r1 = 2^256 -2^224 +2^192 +2^96 -1 (NIST P256r1) +*/ +static gsModMethod* gsArithGF_p256r1 (void) +{ + static gsModMethod m = { + p256r1_to_mont, + p256r1_mont_back, + p256r1_mul_montl, + p256r1_sqr_montl, + NULL, + p256r1_add, + p256r1_sub, + p256r1_neg, + p256r1_div_by_2, + p256r1_mul_by_2, + p256r1_mul_by_3, + }; + + #if(_IPP_ARCH==_IPP_ARCH_EM64T) && ((_ADCOX_NI_ENABLING_==_FEATURE_ON_) || (_ADCOX_NI_ENABLING_==_FEATURE_TICKTOCK_)) + if(IsFeatureEnabled(ippCPUID_ADCOX)) { + m.mul = p256r1_mul_montx; + m.sqr = p256r1_sqr_montx; + } + #endif + + #if(_IPP_ARCH==_IPP_ARCH_IA32) + if(IsFeatureEnabled(ippCPUID_SSSE3|ippCPUID_MOVBE) && !IsFeatureEnabled(ippCPUID_AVX)) { + m.mul = p256r1_mul_mont_slm; + m.sqr = p256r1_sqr_mont_slm; + m.encode = p256r1_to_mont_slm; + m.decode = p256r1_mont_back_slm; + } + #endif + + return &m; +} +#endif /* (_IPP >= _IPP_P8) || (_IPP32E >= _IPP32E_M7) */ + +/*F* +// Name: ippsGFpMethod_p256r1 +// +// Purpose: Returns a reference to an implementation of +// arithmetic operations over GF(q). +// +// Returns: Pointer to a structure containing an implementation of arithmetic +// operations over GF(q). q = 2^256 - 2^224 + 2^192 + 2^96 - 1 +*F*/ + +IPPFUN( const IppsGFpMethod*, ippsGFpMethod_p256r1, (void) ) +{ + static IppsGFpMethod method = { + cpID_PrimeP256r1, + 256, + secp256r1_p, + NULL, + NULL + }; + +#if (_IPP >= _IPP_P8) || (_IPP32E >= _IPP32E_M7) + method.arith = gsArithGF_p256r1(); +#else + method.arith = gsArithGFp(); +#endif + +#if (_IPP32E >= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512IFMA)) { + method.arith_alt = gsArithGF_p256r1_avx512(); + } +#endif + + return &method; +} + +#undef LEN_P256 +#undef OPERAND_BITSIZE diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_384r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_384r1.c new file mode 100644 index 0000000..f98a0f9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_384r1.c @@ -0,0 +1,213 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// GF(p) methods +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "ecnist/ifma_arith_method.h" + +#include "pcpbnumisc.h" +#include "gsmodstuff.h" +#include "pcpgfpstuff.h" +#include "pcpgfpmethod.h" +#include "pcpbnuarith.h" +#include "pcpecprime.h" + +#if(_IPP >= _IPP_P8) || (_IPP32E >= _IPP32E_M7) + +/* arithmetic over P-384r1 NIST modulus */ +#define p384r1_add OWNAPI(p384r1_add) + IPP_OWN_DECL (BNU_CHUNK_T*, p384r1_add, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +#define p384r1_sub OWNAPI(p384r1_sub) + IPP_OWN_DECL (BNU_CHUNK_T*, p384r1_sub, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +#define p384r1_neg OWNAPI(p384r1_neg) + IPP_OWN_DECL (BNU_CHUNK_T*, p384r1_neg, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p384r1_div_by_2 OWNAPI(p384r1_div_by_2) + IPP_OWN_DECL (BNU_CHUNK_T*, p384r1_div_by_2, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p384r1_mul_by_2 OWNAPI(p384r1_mul_by_2) + IPP_OWN_DECL (BNU_CHUNK_T*, p384r1_mul_by_2, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p384r1_mul_by_3 OWNAPI(p384r1_mul_by_3) + IPP_OWN_DECL (BNU_CHUNK_T*, p384r1_mul_by_3, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) + +#if(_IPP_ARCH ==_IPP_ARCH_EM64T) +//BNU_CHUNK_T* p384r1_mul_montl(BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE); +//BNU_CHUNK_T* p384r1_sqr_montl(BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE); +//BNU_CHUNK_T* p384r1_mul_montx(BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE); +//BNU_CHUNK_T* p384r1_sqr_montx(BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE); +//BNU_CHUNK_T* p384r1_to_mont (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE); +//BNU_CHUNK_T* p384r1_mont_back(BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE); +#endif + +#define p384r1_mred OWNAPI(p384r1_mred) + IPP_OWN_DECL (BNU_CHUNK_T*, p384r1_mred, (BNU_CHUNK_T* res, BNU_CHUNK_T* product)) + +#if(_IPP_ARCH ==_IPP_ARCH_IA32) +#define p384r1_mul_mont_slm OWNAPI(p384r1_mul_mont_slm) + IPP_OWN_DECL (BNU_CHUNK_T*, p384r1_mul_mont_slm, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +#define p384r1_sqr_mont_slm OWNAPI(p384r1_sqr_mont_slm) + IPP_OWN_DECL (BNU_CHUNK_T*, p384r1_sqr_mont_slm, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#endif + +#define OPERAND_BITSIZE (384) +#define LEN_P384 (BITS_BNU_CHUNK(OPERAND_BITSIZE)) + +/* +// multiplicative methods +*/ +IPP_OWN_DEFN (static BNU_CHUNK_T*, p384r1_mul_montl, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFE)) +{ + BNU_CHUNK_T* product = cpGFpGetPool(2, pGFE); + //tbcd: temporary excluded: assert(NULL!=product); + + cpMul_BNU_school(product, pA,LEN_P384, pB,LEN_P384); + p384r1_mred(pR, product); + + cpGFpReleasePool(2, pGFE); + return pR; +} + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p384r1_sqr_montl, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + BNU_CHUNK_T* product = gsModPoolAlloc((gsModEngine*)pGFE, 2); + if(NULL == product) + return NULL; + + cpSqr_BNU_school(product, pA,LEN_P384); + p384r1_mred(pR, product); + + cpGFpReleasePool(2, pGFE); + return pR; +} + + +/* +// Montgomery domain conversion constants +*/ +static Ipp64u RR[] = { + 0xfffffffe00000001,0x0000000200000000,0xfffffffe00000000, + 0x0000000200000000,0x0000000000000001,0x0000000000000000 +}; + +static BNU_CHUNK_T one[] = { +#if(_IPP_ARCH == _IPP_ARCH_EM64T) + 1,0,0,0,0,0}; +#elif(_IPP_ARCH == _IPP_ARCH_IA32) + 1,0,0,0,0,0,0,0,0,0,0,0}; +#endif + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p384r1_to_mont, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return p384r1_mul_montl(pR, pA, (BNU_CHUNK_T*)RR, pGFE); +} + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p384r1_mont_back, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return p384r1_mul_montl(pR, pA, (BNU_CHUNK_T*)one, pGFE); +} + +#if (_ADCOX_NI_ENABLING_==_FEATURE_ON_) || (_ADCOX_NI_ENABLING_==_FEATURE_TICKTOCK_) +//BNU_CHUNK_T* p384r1_mul_montx(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFE) +//BNU_CHUNK_T* p384r1_sqr_montx(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFE) +#endif + +#if(_IPP_ARCH ==_IPP_ARCH_IA32) +IPP_OWN_DEFN (static BNU_CHUNK_T*, p384r1_to_mont_slm, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return p384r1_mul_mont_slm(pR, pA, (BNU_CHUNK_T*)RR, pGFE); +} + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p384r1_mont_back_slm, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return p384r1_mul_mont_slm(pR, pA, (BNU_CHUNK_T*)one, pGFE); +} +#endif /* _IPP_ARCH ==_IPP_ARCH_IA32 */ + +/* +// return specific gf p384r1 arith methods, +// p384r1 = 2^384 -2^128 -2^96 +2^32 -1 (NIST P384r1) +*/ +static gsModMethod* gsArithGF_p384r1 (void) +{ + static gsModMethod m = { + p384r1_to_mont, + p384r1_mont_back, + p384r1_mul_montl, + p384r1_sqr_montl, + NULL, + p384r1_add, + p384r1_sub, + p384r1_neg, + p384r1_div_by_2, + p384r1_mul_by_2, + p384r1_mul_by_3, + }; + + #if(_IPP_ARCH==_IPP_ARCH_IA32) + if(IsFeatureEnabled(ippCPUID_SSSE3|ippCPUID_MOVBE) && !IsFeatureEnabled(ippCPUID_AVX)) { + m.mul = p384r1_mul_mont_slm; + m.sqr = p384r1_sqr_mont_slm; + m.encode = p384r1_to_mont_slm; + m.decode = p384r1_mont_back_slm; + } + #endif + + return &m; +} +#endif /* (_IPP >= _IPP_P8) || (_IPP32E >= _IPP32E_M7) */ + +/*F* +// Name: ippsGFpMethod_p384r1 +// +// Purpose: Returns a reference to an implementation of +// arithmetic operations over GF(q). +// +// Returns: Pointer to a structure containing an implementation of arithmetic +// operations over GF(q). q = 2^384 - 2^128 - 2^96 + 2^32 - 1 +*F*/ + +IPPFUN(const IppsGFpMethod *, ippsGFpMethod_p384r1, (void)) +{ + static IppsGFpMethod method = { + cpID_PrimeP384r1, + 384, + secp384r1_p, + NULL, + NULL + }; + +#if (_IPP >= _IPP_P8) || (_IPP32E >= _IPP32E_M7) + method.arith = gsArithGF_p384r1(); +#else + method.arith = gsArithGFp(); +#endif + +#if (_IPP32E >= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512IFMA)) { + method.arith_alt = gsArithGF_p384r1_avx512(); + } +#endif + + return &method; +} + +#undef LEN_P384 +#undef OPERAND_BITSIZE diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_521r1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_521r1.c new file mode 100644 index 0000000..a30345d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_521r1.c @@ -0,0 +1,220 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// GF(p) methods +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpbnumisc.h" +#include "gsmodstuff.h" +#include "pcpgfpstuff.h" +#include "pcpgfpmethod.h" +#include "pcpbnuarith.h" +#include "pcpecprime.h" + +#include "ecnist/ifma_arith_method_p521.h" + +//tbcd: temporary excluded: #include + +#if(_IPP >= _IPP_P8) || (_IPP32E >= _IPP32E_M7) + +/* arithmetic over P-521r1 NIST modulus */ +#define p521r1_add OWNAPI(p521r1_add) + IPP_OWN_DECL (BNU_CHUNK_T*, p521r1_add, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +#define p521r1_sub OWNAPI(p521r1_sub) + IPP_OWN_DECL (BNU_CHUNK_T*, p521r1_sub, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +#define p521r1_neg OWNAPI(p521r1_neg) + IPP_OWN_DECL (BNU_CHUNK_T*, p521r1_neg, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p521r1_div_by_2 OWNAPI(p521r1_div_by_2) + IPP_OWN_DECL (BNU_CHUNK_T*, p521r1_div_by_2, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p521r1_mul_by_2 OWNAPI(p521r1_mul_by_2) + IPP_OWN_DECL (BNU_CHUNK_T*, p521r1_mul_by_2, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#define p521r1_mul_by_3 OWNAPI(p521r1_mul_by_3) + IPP_OWN_DECL (BNU_CHUNK_T*, p521r1_mul_by_3, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) + +#if(_IPP_ARCH ==_IPP_ARCH_EM64T) +//BNU_CHUNK_T* p521r1_to_mont (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE); +//BNU_CHUNK_T* p521r1_mont_back(BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE); +//BNU_CHUNK_T* p521r1_mul_montl(BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE); +//BNU_CHUNK_T* p521r1_sqr_montl(BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE); +//BNU_CHUNK_T* p521r1_mul_montx(BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE); +//BNU_CHUNK_T* p521r1_sqr_montx(BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE); +#endif + +#define p521r1_mred OWNAPI(p521r1_mred) + IPP_OWN_DECL (BNU_CHUNK_T*, p521r1_mred, (BNU_CHUNK_T* res, BNU_CHUNK_T* product)) + +#if(_IPP_ARCH ==_IPP_ARCH_IA32) +#define p521r1_mul_mont_slm OWNAPI(p521r1_mul_mont_slm) + IPP_OWN_DECL (BNU_CHUNK_T*, p521r1_mul_mont_slm, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +#define p521r1_sqr_mont_slm OWNAPI(p521r1_sqr_mont_slm) + IPP_OWN_DECL (BNU_CHUNK_T*, p521r1_sqr_mont_slm, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#endif + +#define OPERAND_BITSIZE (521) +#define LEN_P521 (BITS_BNU_CHUNK(OPERAND_BITSIZE)) + +/* +// multiplicative methods +*/ +IPP_OWN_DEFN (static BNU_CHUNK_T*, p521r1_mul_montl, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFE)) +{ + BNU_CHUNK_T* product = cpGFpGetPool(2, pGFE); + //tbcd: temporary excluded: assert(NULL!=product); + + cpMul_BNU_school(product, pA,LEN_P521, pB,LEN_P521); + p521r1_mred(pR, product); + + cpGFpReleasePool(2, pGFE); + return pR; +} + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p521r1_sqr_montl, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + BNU_CHUNK_T* product = cpGFpGetPool(2, pGFE); + //tbcd: temporary excluded: assert(NULL!=product); + + cpSqr_BNU_school(product, pA,LEN_P521); + p521r1_mred(pR, product); + + cpGFpReleasePool(2, pGFE); + return pR; +} + + +/* +// Montgomery domain conversion constants +*/ +static BNU_CHUNK_T RR[] = { +#if(_IPP_ARCH == _IPP_ARCH_EM64T) + 0x0000000000000000,0x0000400000000000,0x0000000000000000, + 0x0000000000000000,0x0000000000000000,0x0000000000000000, + 0x0000000000000000,0x0000000000000000,0x0000000000000000}; +#elif(_IPP_ARCH == _IPP_ARCH_IA32) + 0x00000000,0x00004000,0x00000000,0x00000000,0x00000000,0x00000000, + 0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000, + 0x00000000,0x00000000,0x00000000,0x00000000,0x00000000}; +#endif + +static BNU_CHUNK_T one[] = { +#if(_IPP_ARCH == _IPP_ARCH_EM64T) + 1,0,0,0,0,0,0,0,0}; +#elif(_IPP_ARCH == _IPP_ARCH_IA32) + 1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; +#endif + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p521r1_to_mont, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return p521r1_mul_montl(pR, pA, (BNU_CHUNK_T*)RR, pGFE); +} + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p521r1_mont_back, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return p521r1_mul_montl(pR, pA, (BNU_CHUNK_T*)one, pGFE); +} + +#if (_ADCOX_NI_ENABLING_==_FEATURE_ON_) || (_ADCOX_NI_ENABLING_==_FEATURE_TICKTOCK_) +//BNU_CHUNK_T* p521r1_mul_montx(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFE) +//BNU_CHUNK_T* p521r1_sqr_montx(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFE) +#endif + +#if(_IPP_ARCH ==_IPP_ARCH_IA32) +IPP_OWN_DEFN (static BNU_CHUNK_T*, p521r1_to_mont_slm, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return p521r1_mul_mont_slm(pR, pA, (BNU_CHUNK_T*)RR, pGFE); +} + +IPP_OWN_DEFN (static BNU_CHUNK_T*, p521r1_mont_back_slm, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return p521r1_mul_mont_slm(pR, pA, (BNU_CHUNK_T*)one, pGFE); +} +#endif /* _IPP_ARCH ==_IPP_ARCH_IA32*/ + +/* +// return specific gf p521r1 arith methods, +// p521r1 = 2^521 -1 (NIST P521r1) +*/ +static gsModMethod* gsArithGF_p521r1 (void) +{ + static gsModMethod m = { + p521r1_to_mont, + p521r1_mont_back, + p521r1_mul_montl, + p521r1_sqr_montl, + NULL, + p521r1_add, + p521r1_sub, + p521r1_neg, + p521r1_div_by_2, + p521r1_mul_by_2, + p521r1_mul_by_3, + }; + + #if(_IPP_ARCH==_IPP_ARCH_IA32) + if(IsFeatureEnabled(ippCPUID_SSSE3|ippCPUID_MOVBE) && !IsFeatureEnabled(ippCPUID_AVX)) { + m.mul = p521r1_mul_mont_slm; + m.sqr = p521r1_sqr_mont_slm; + m.encode = p521r1_to_mont_slm; + m.decode = p521r1_mont_back_slm; + } + #endif + + return &m; +} +#endif /* (_IPP >= _IPP_P8) || (_IPP32E >= _IPP32E_M7) */ + +/*F* +// Name: ippsGFpMethod_p521r1 +// +// Purpose: Returns a reference to an implementation of +// arithmetic operations over GF(q). +// +// Returns: Pointer to a structure containing an implementation of arithmetic +// operations over GF(q). q = 2^521 - 1 +*F*/ + +IPPFUN( const IppsGFpMethod*, ippsGFpMethod_p521r1, (void) ) +{ + static IppsGFpMethod method = { + cpID_PrimeP521r1, + 521, + secp521r1_p, + NULL, + NULL + }; + +#if (_IPP >= _IPP_P8) || (_IPP32E >= _IPP32E_M7) + method.arith = gsArithGF_p521r1(); +#else + method.arith = gsArithGFp(); +#endif + +#if (_IPP32E >= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512IFMA)) { + method.arith_alt = gsArithGF_p521r1_avx512(); + } +#endif + + return &method; +} + +#undef LEN_P521 +#undef OPERAND_BITSIZE diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_com.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_com.c new file mode 100644 index 0000000..bb66933 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_com.c @@ -0,0 +1,48 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// GF(p) methods +// +*/ +#include "owncp.h" + +#include "pcpgfpmethod.h" + +/*F* +// Name: ippsGFpMethod_pArb +// +// Purpose: Returns a reference to an implementation of +// arithmetic operations over GF(q). +// +// Returns: Pointer to a structure containing an implementation of arithmetic +// operations over GF(q). Arbitrary modulus q. +*F*/ + +IPPFUN( const IppsGFpMethod*, ippsGFpMethod_pArb, (void) ) +{ + static IppsGFpMethod method = { + cpID_Prime, + 0, + NULL, + NULL, + NULL + }; + method.arith = gsArithGFp(); + return &method; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_sm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_sm2.c new file mode 100644 index 0000000..fe2a959 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmethod_sm2.c @@ -0,0 +1,219 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// GF(p) methods +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpbnumisc.h" +#include "gsmodstuff.h" +#include "pcpgfpstuff.h" +#include "pcpgfpmethod.h" +#include "pcpecprime.h" + +#if (_IPP32E >= _IPP32E_K1) +#include "sm2/ifma_arith_method_sm2.h" +#endif // (_IPP32E >= _IPP32E_K1) +//tbcd: temporary excluded: #include + +#if(_IPP >= _IPP_P8) || (_IPP32E >= _IPP32E_M7) + +/* arithmetic over 256-bit SM2 modulus */ +#define sm2_add OWNAPI(sm2_add) +#define sm2_sub OWNAPI(sm2_sub) +#define sm2_neg OWNAPI(sm2_neg) +#define sm2_div_by_2 OWNAPI(sm2_div_by_2) +#define sm2_mul_by_2 OWNAPI(sm2_mul_by_2) +#define sm2_mul_by_3 OWNAPI(sm2_mul_by_3) + +IPP_OWN_DECL (BNU_CHUNK_T*, sm2_add, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +IPP_OWN_DECL (BNU_CHUNK_T*, sm2_sub, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +IPP_OWN_DECL (BNU_CHUNK_T*, sm2_neg, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +IPP_OWN_DECL (BNU_CHUNK_T*, sm2_div_by_2, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +IPP_OWN_DECL (BNU_CHUNK_T*, sm2_mul_by_2, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +IPP_OWN_DECL (BNU_CHUNK_T*, sm2_mul_by_3, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) + +#if(_IPP_ARCH ==_IPP_ARCH_EM64T) +#define sm2_mul_montl OWNAPI(sm2_mul_montl) +#define sm2_mul_montx OWNAPI(sm2_mul_montx) +#define sm2_sqr_montl OWNAPI(sm2_sqr_montl) +#define sm2_sqr_montx OWNAPI(sm2_sqr_montx) +#define sm2_to_mont OWNAPI(sm2_to_mont) +#define sm2_mont_back OWNAPI(sm2_mont_back) + +IPP_OWN_DECL (BNU_CHUNK_T*, sm2_mul_montl, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +IPP_OWN_DECL (BNU_CHUNK_T*, sm2_mul_montx, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +IPP_OWN_DECL (BNU_CHUNK_T*, sm2_sqr_montl, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +IPP_OWN_DECL (BNU_CHUNK_T*, sm2_sqr_montx, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +IPP_OWN_DECL (BNU_CHUNK_T*, sm2_to_mont, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +IPP_OWN_DECL (BNU_CHUNK_T*, sm2_mont_back, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +#endif + +#if(_IPP_ARCH ==_IPP_ARCH_IA32) +#define sm2_mul_mont_slm OWNAPI(sm2_mul_mont_slm) +#define sm2_sqr_mont_slm OWNAPI(sm2_sqr_mont_slm) +#define sm2_mred OWNAPI(sm2_mred) + +IPP_OWN_DECL (BNU_CHUNK_T*, sm2_mul_mont_slm, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, const BNU_CHUNK_T* b, gsEngine* pGFE)) +IPP_OWN_DECL (BNU_CHUNK_T*, sm2_sqr_mont_slm, (BNU_CHUNK_T* res, const BNU_CHUNK_T* a, gsEngine* pGFE)) +IPP_OWN_DECL (void, sm2_mred, (BNU_CHUNK_T* res, BNU_CHUNK_T* product)) +#endif + +#define OPERAND_BITSIZE (256) +#define LEN_SM2 (BITS_BNU_CHUNK(OPERAND_BITSIZE)) + +/* +// ia32 multiplicative methods +*/ +#if (_IPP_ARCH ==_IPP_ARCH_IA32 ) +IPP_OWN_DEFN (static BNU_CHUNK_T*, sm2_mul_montl, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFE)) +{ + BNU_CHUNK_T* product = cpGFpGetPool(2, pGFE); + //tbcd: temporary excluded: assert(NULL!=product); + + cpMulAdc_BNU_school(product, pA, LEN_SM2, pB, LEN_SM2); + sm2_mred(pR, product); + + cpGFpReleasePool(2, pGFE); + return pR; +} + +IPP_OWN_DEFN (static BNU_CHUNK_T*, sm2_sqr_montl, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + BNU_CHUNK_T* product = cpGFpGetPool(2, pGFE); + //tbcd: temporary excluded: assert(NULL!=product); + + cpSqrAdc_BNU_school(product, pA, LEN_SM2); + sm2_mred(pR, product); + + cpGFpReleasePool(2, pGFE); + return pR; +} + + +/* +// Montgomery domain conversion constants +*/ +static BNU_CHUNK_T RR[] = { + 0x00000003,0x00000002, 0xffffffff,0x00000002, + 0x00000001,0x00000001, 0x00000002,0x00000004}; + +static BNU_CHUNK_T one[] = { + 1,0,0,0,0,0,0,0}; + +IPP_OWN_DEFN (static BNU_CHUNK_T*, sm2_to_mont, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return sm2_mul_montl(pR, pA, (BNU_CHUNK_T*)RR, pGFE); +} + +IPP_OWN_DEFN (static BNU_CHUNK_T*, sm2_mont_back, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return sm2_mul_montl(pR, pA, (BNU_CHUNK_T*)one, pGFE); +} + +IPP_OWN_DEFN (static BNU_CHUNK_T*, sm2_to_mont_slm, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return sm2_mul_mont_slm(pR, pA, (BNU_CHUNK_T*)RR, pGFE); +} + +IPP_OWN_DEFN (static BNU_CHUNK_T*, sm2_mont_back_slm, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFE)) +{ + return sm2_mul_mont_slm(pR, pA, (BNU_CHUNK_T*)one, pGFE); +} +#endif /* _IPP >= _IPP_P8 */ + +/* +// return specific gf p256sm2 arith methods, +// p256sm2 = 2^256 -2^224 -2^96 +2^64 -1 (SM2 curve) +*/ +static gsModMethod* gsArithGF_p256sm2 (void) +{ + static gsModMethod m = { + sm2_to_mont, + sm2_mont_back, + sm2_mul_montl, + sm2_sqr_montl, + NULL, + sm2_add, + sm2_sub, + sm2_neg, + sm2_div_by_2, + sm2_mul_by_2, + sm2_mul_by_3, + }; + + #if(_IPP_ARCH==_IPP_ARCH_EM64T) && ((_ADCOX_NI_ENABLING_==_FEATURE_ON_) || (_ADCOX_NI_ENABLING_==_FEATURE_TICKTOCK_)) + if(IsFeatureEnabled(ippCPUID_ADCOX)) { + m.mul = sm2_mul_montx; + m.sqr = sm2_sqr_montx; + } + #endif + + #if(_IPP_ARCH==_IPP_ARCH_IA32) + if(IsFeatureEnabled(ippCPUID_SSSE3|ippCPUID_MOVBE) && !IsFeatureEnabled(ippCPUID_AVX)) { + m.mul = sm2_mul_mont_slm; + m.sqr = sm2_sqr_mont_slm; + m.encode = sm2_to_mont_slm; + m.decode = sm2_mont_back_slm; + } + #endif + + return &m; +} +#endif /* (_IPP >= _IPP_P8) || (_IPP32E >= _IPP32E_M7) */ + +/*F* +// Name: ippsGFpMethod_p256sm2 +// +// Purpose: Returns a reference to an implementation of +// arithmetic operations over GF(q). +// +// Returns: Pointer to a structure containing an implementation of arithmetic +// operations over GF(q). q = 2^256 - 2^224 - 2^96 + 2^64 - 1 +*F*/ + +IPPFUN( const IppsGFpMethod*, ippsGFpMethod_p256sm2, (void) ) +{ + static IppsGFpMethod method = { + cpID_PrimeTPM_SM2, + 256, + tpmSM2_p256_p, + NULL, + NULL + }; + +#if(_IPP >= _IPP_P8) || (_IPP32E >= _IPP32E_M7) + method.arith = gsArithGF_p256sm2(); +#else + method.arith = gsArithGFp(); +#endif + +#if (_IPP32E >= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512IFMA)) { + method.arith_alt = gsArithGF_psm2_avx512(); + } +#endif // (_IPP32E >= _IPP32E_K1) + + return &method; +} + +#undef LEN_SM2 +#undef OPERAND_BITSIZE diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmul.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmul.c new file mode 100644 index 0000000..0e3611d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmul.c @@ -0,0 +1,79 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpMul() +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + + +/*F* +// Name: ippsGFpMul +// +// Purpose: Multiply of GF elements +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFp +// NULL == pA +// NULL == pR +// NULL == pB +// +// ippStsContextMatchErr invalid pGFp->idCtx +// invalid pA->idCtx +// invalid pR->idCtx +// invalid pB->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM() != GFP_FELEN() +// +// ippStsNoErr no error +// +// Parameters: +// pA Pointer to the context of the first finite field element. +// pB Pointer to the context of the second finite field element. +// pR Pointer to the context of the result finite field element. +// pGFp Pointer to the context of the finite field. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpMul,(const IppsGFpElement* pA, const IppsGFpElement* pB, + IppsGFpElement* pR, IppsGFpState* pGFp)) +{ + IPP_BAD_PTR4_RET(pA, pB, pR, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pA), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pB), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pR), ippStsContextMatchErr ); + { + gsModEngine* pGFE = GFP_PMA(pGFp); + IPP_BADARG_RET( (GFPE_ROOM(pA)!=GFP_FELEN(pGFE)) || (GFPE_ROOM(pB)!=GFP_FELEN(pGFE)) || (GFPE_ROOM(pR)!=GFP_FELEN(pGFE)), ippStsOutOfRangeErr); + + GFP_METHOD(pGFE)->mul(GFPE_DATA(pR), GFPE_DATA(pA), GFPE_DATA(pB),pGFE); + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmulexp.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmulexp.c new file mode 100644 index 0000000..2f7c3bc --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpmulexp.c @@ -0,0 +1,133 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpMultiExp() +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + +//tbcd: temporary excluded: #include + +/*F* +// Name: ippsGFpMultiExp +// +// Purpose: Multiplies exponents of GF elements +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFp +// NULL == ppElmA +// NULL == pR +// NULL == ppE +// +// ippStsContextMatchErr invalid pGFp->idCtx +// invalid ppElmA[i]->idCtx +// invalid pR->idCtx +// invalid ppE[i]->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM() != GFP_FELEN() +// +// ippStsBadArgErr 1>nItems +// nItems>6 +// +// ippStsNoErr no error +// +// Parameters: +// ppElmA Pointer to the array of contexts of the finite field elements representing the base of the exponentiation. +// ppE Pointer to the array of the Big Number contexts storing the exponents. +// nItems Number of exponents. +// pR Pointer to the context of the resulting element of the finite field. +// pGFp Pointer to the context of the finite field. +// pScratchBuffer Pointer to the scratch buffer. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpMultiExp,(const IppsGFpElement* const ppElmA[], const IppsBigNumState* const ppE[], int nItems, + IppsGFpElement* pR, IppsGFpState* pGFp, + Ipp8u* pScratchBuffer)) +{ + IPP_BAD_PTR2_RET(ppElmA, ppE); + + if(nItems==1) + return ippsGFpExp(ppElmA[0], ppE[0], pR, pGFp, pScratchBuffer); + + else { + /* test number of exponents */ + IPP_BADARG_RET(1>nItems || nItems>IPP_MAX_EXPONENT_NUM, ippStsBadArgErr); + + IPP_BAD_PTR2_RET(pR, pGFp); + + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pR), ippStsContextMatchErr ); + { + int n; + + gsModEngine* pGFE = GFP_PMA(pGFp); + IPP_BADARG_RET( GFPE_ROOM(pR)!=GFP_FELEN(pGFE), ippStsOutOfRangeErr); + + /* test all ppElmA[] and ppE[] pairs */ + for(n=0; nmul; + + BNU_CHUNK_T* pTmpR = cpGFpGetPool(1, pGFE); + //tbcd: temporary excluded: assert(NULL!=pTmpR); + + cpGFpxExp(GFPE_DATA(pR), GFPE_DATA(ppElmA[0]), BN_NUMBER(ppE[0]), BN_SIZE(ppE[0]), pGFE, 0); + for(n=1; nidCtx +// invalid pA->idCtx +// invalid pR->idCtx +// invalid pParentB->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM() != GFP_FELEN() +// +// ippStsBadArgErr pGFp does not specify prime field +// +// ippStsNoErr no error +// +// Parameters: +// pA Pointer to the context of the first finite field element. +// pParentB Pointer to the context of the second finite field element. +// pR Pointer to the context of the result finite field element. +// pGFp Pointer to the context of the finite field. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpMul_PE,(const IppsGFpElement* pA, const IppsGFpElement* pParentB, + IppsGFpElement* pR, IppsGFpState* pGFp)) +{ + IPP_BAD_PTR4_RET(pA, pParentB, pR, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pA), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pParentB), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pR), ippStsContextMatchErr ); + { + gsModEngine* pGFE = GFP_PMA(pGFp); + IPP_BADARG_RET( GFP_IS_BASIC(pGFE), ippStsBadArgErr ) + IPP_BADARG_RET( (GFPE_ROOM(pA)!=GFP_FELEN(pGFE)) || (GFPE_ROOM(pR)!=GFP_FELEN(pGFE)), ippStsOutOfRangeErr); + IPP_BADARG_RET( (GFPE_ROOM(pParentB)!=GFP_FELEN(GFP_PARENT(pGFE))), ippStsOutOfRangeErr); + + cpGFpxMul_GFE(GFPE_DATA(pR), GFPE_DATA(pA), GFPE_DATA(pParentB), pGFE); + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpneg.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpneg.c new file mode 100644 index 0000000..0f0b8f9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpneg.c @@ -0,0 +1,73 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpNeg() +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + + +/*F* +// Name: ippsGFpNeg +// +// Purpose: Additive inverse GF element +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFp +// NULL == pA +// NULL == pR +// +// ippStsContextMatchErr invalid pGFp->idCtx +// invalid pA->idCtx +// invalid pR->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM() != GFP_FELEN() +// +// ippStsNoErr no error +// +// Parameters: +// pA Pointer to the context of the source finite field element. +// pR Pointer to the context of the result finite field element. +// pGFp Pointer to the context of the finite field. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpNeg,(const IppsGFpElement* pA, + IppsGFpElement* pR, IppsGFpState* pGFp)) +{ + IPP_BAD_PTR3_RET(pA, pR, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pA), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pR), ippStsContextMatchErr ); + { + gsModEngine* pGFE = GFP_PMA(pGFp); + IPP_BADARG_RET( (GFPE_ROOM(pA)!=GFP_FELEN(pGFE)) || (GFPE_ROOM(pR)!=GFP_FELEN(pGFE)), ippStsOutOfRangeErr); + + GFP_METHOD(pGFE)->neg(GFPE_DATA(pR), GFPE_DATA(pA), pGFE); + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsetelem.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsetelem.c new file mode 100644 index 0000000..7655242 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsetelem.c @@ -0,0 +1,89 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpSetElement() +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + +//tbcd: temporary excluded: #include + +/*F* +// Name: ippsGFpSetElement +// +// Purpose: Set GF Element +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFp +// NULL == pR +// NULL == pA && lenA>0 +// +// ippStsContextMatchErr invalid pGFp->idCtx +// invalid pR->idCtx +// +// ippStsSizeErr pA && !(0<=lenA && lenA= modulus +// +// ippStsNoErr no error +// +// Parameters: +// pA pointer to the data representation Finite Field element +// lenA length of Finite Field data representation array +// pR pointer to Finite Field Element context +// pGFp pointer to Finite Field context +*F*/ +IPPFUN(IppStatus, ippsGFpSetElement,(const Ipp32u* pA, int lenA, IppsGFpElement* pR, IppsGFpState* pGFp)) +{ + IPP_BAD_PTR2_RET(pR, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pR), ippStsContextMatchErr ); + + IPP_BADARG_RET( !pA && (00 +// +// ippStsNotSupportedModeErr hashID is not supported +// +// ippStsContextMatchErr invalid pGFp->idCtx +// invalid pElm->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM() != GFP_FELEN() +// +// ippStsBadArgErr !GFP_IS_BASIC(pGFE) +// +// ippStsLengthErr msgLen<0 +// +// ippStsNoErr no error +// +// Parameters: +// pMsg pointer to the message is being hashed +// msgLen length of the message above +// pElm pointer to Finite Field Element context +// pGFp pointer to Finite Field context +// hashID applied hash algorithm ID +*F*/ +IPPFUN(IppStatus, ippsGFpSetElementHash,(const Ipp8u* pMsg, int msgLen, IppsGFpElement* pElm, IppsGFpState* pGFp, IppHashAlgId hashID)) +{ + /* get algorithm id */ + hashID = cpValidHashAlg(hashID); + IPP_BADARG_RET(ippHashAlg_Unknown==hashID, ippStsNotSupportedModeErr); + + /* test message length and pointer */ + IPP_BADARG_RET((msgLen<0), ippStsLengthErr); + IPP_BADARG_RET((msgLen && !pMsg), ippStsNullPtrErr); + + IPP_BAD_PTR2_RET(pElm, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr); + IPP_BADARG_RET( !GFPE_VALID_ID(pElm), ippStsContextMatchErr); + { + gsModEngine* pGFE = GFP_PMA(pGFp); + IPP_BADARG_RET( !GFP_IS_BASIC(pGFE), ippStsBadArgErr); + IPP_BADARG_RET( GFPE_ROOM(pElm)!=GFP_FELEN(pGFE), ippStsOutOfRangeErr); + + { + Ipp8u md[MAX_HASH_SIZE]; + BNU_CHUNK_T hashVal[(MAX_HASH_SIZE*8)/BITSIZE(BNU_CHUNK_T)+1]; /* +1 to meet cpMod_BNU() implementtaion specific */ + IppStatus sts = ippsHashMessage(pMsg, msgLen, md, hashID); + + if(ippStsNoErr==sts) { + int elemLen = GFP_FELEN(pGFE); + int hashLen = cpHashAlgAttr[hashID].hashSize; + int hashValLen = cpFromOctStr_BNU(hashVal, md, hashLen); + hashValLen = cpMod_BNU(hashVal, hashValLen, GFP_MODULUS(pGFE), elemLen); + cpGFpSet(GFPE_DATA(pElm), hashVal, hashValLen, pGFE); + } + + return sts; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsetelemhashrmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsetelemhashrmf.c new file mode 100644 index 0000000..0c43ca7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsetelemhashrmf.c @@ -0,0 +1,99 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpSetElementHash_rmf +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + + +/*F* +// Name: ippsGFpSetElementHash_rmf +// +// Purpose: Set GF Element Hash of the Message +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFp +// NULL == pElm +// NULL == pMsg if msgLen>0 +// NULL = pMethod +// +// ippStsContextMatchErr invalid pGFp->idCtx +// invalid pElm->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM() != GFP_FELEN() +// +// ippStsBadArgErr !GFP_IS_BASIC(pGFE) +// +// ippStsLengthErr msgLen<0 +// +// ippStsNoErr no error +// +// Parameters: +// pMsg pointer to the message is being hashed +// msgLen length of the message above +// pElm pointer to Finite Field Element context +// pGFp pointer to Finite Field context +// pMethod pointer to hash method +*F*/ + +IPPFUN(IppStatus, ippsGFpSetElementHash_rmf,(const Ipp8u* pMsg, int msgLen, IppsGFpElement* pElm, IppsGFpState* pGFp, const IppsHashMethod* pMethod)) +{ + /* test method pointer */ + IPP_BAD_PTR1_RET(pMethod); + + /* test message length and pointer */ + IPP_BADARG_RET((msgLen<0), ippStsLengthErr); + IPP_BADARG_RET((msgLen && !pMsg), ippStsNullPtrErr); + + IPP_BAD_PTR2_RET(pElm, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr); + IPP_BADARG_RET( !GFPE_VALID_ID(pElm), ippStsContextMatchErr); + { + gsModEngine* pGFE = GFP_PMA(pGFp); + IPP_BADARG_RET( !GFP_IS_BASIC(pGFE), ippStsBadArgErr); + IPP_BADARG_RET( GFPE_ROOM(pElm)!=GFP_FELEN(pGFE), ippStsOutOfRangeErr); + + { + Ipp8u md[MAX_HASH_SIZE]; + BNU_CHUNK_T hashVal[(MAX_HASH_SIZE*8)/BITSIZE(BNU_CHUNK_T)+1]; /* +1 to meet cpMod_BNU() implementtaion specific */ + IppStatus sts = ippsHashMessage_rmf(pMsg, msgLen, md, pMethod); + + if(ippStsNoErr==sts) { + int elemLen = GFP_FELEN(pGFE); + int hashLen = pMethod->hashLen; + int hashValLen = cpFromOctStr_BNU(hashVal, md, hashLen); + hashValLen = cpMod_BNU(hashVal, hashValLen, GFP_MODULUS(pGFE), elemLen); + cpGFpSet(GFPE_DATA(pElm), hashVal, hashValLen, pGFE); + } + + return sts; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsetelemoctstr.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsetelemoctstr.c new file mode 100644 index 0000000..2b737cf --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsetelemoctstr.c @@ -0,0 +1,96 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpSetElementOctString() +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + + +/*F* +// Name: ippsGFpSetElementOctString +// +// Purpose: Set GF Element from the input octet string +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFp +// NULL == pR +// NULL == pStr && strSize>0 +// +// ippStsContextMatchErr invalid pGFp->idCtx +// invalid pR->idCtx +// +// ippStsSizeErr !(0= modulus +// +// ippStsNoErr no error +// +// Parameters: +// pStr Pointer to the octet string +// strSize Size of the octet string buffer in bytes. +// pR pointer to Finite Field Element context +// pGFp pointer to Finite Field context +*F*/ +IPPFUN(IppStatus, ippsGFpSetElementOctString,(const Ipp8u* pStr, int strSize, IppsGFpElement* pR, IppsGFpState* pGFp)) +{ + IPP_BAD_PTR2_RET(pR, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pR), ippStsContextMatchErr ); + + IPP_BADARG_RET( (!pStr && 0idCtx +// invalid pR->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM() != GFP_FELEN() +// +// ippStsErr internal error caused by call of rndFunc() +// +// ippStsNoErr no error +// +// Parameters: +// pR Pointer to the context of the finite field element. +// pGFp Pointer to the context of the finite field. +// rndFunc Pseudorandom number generator. +// pRndParam Pointer to the context of the pseudorandom number generator. +*F*/ +IPPFUN(IppStatus, ippsGFpSetElementRandom,(IppsGFpElement* pR, IppsGFpState* pGFp, + IppBitSupplier rndFunc, void* pRndParam)) +{ + IPP_BAD_PTR3_RET(pR, pGFp, rndFunc); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pR), ippStsContextMatchErr ); + + { + gsModEngine* pGFE = GFP_PMA(pGFp); + IPP_BADARG_RET( GFPE_ROOM(pR)!=GFP_FELEN(pGFE), ippStsOutOfRangeErr); + return cpGFpxRand(GFPE_DATA(pR), pGFE, rndFunc, pRndParam)? ippStsNoErr : ippStsErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsetelemreg.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsetelemreg.c new file mode 100644 index 0000000..10bcc5e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsetelemreg.c @@ -0,0 +1,63 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpSetElementRegular() +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + + +/*F* +// Name: ippsGFpSetElement +// +// Purpose: Set GF Element from the input Big Number +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFp +// NULL == pR +// NULL == pBN +// +// ippStsContextMatchErr invalid pBN->idCtx +// +// ippStsOutOfRangeErr BN is not positive +// +// ippStsNoErr no error +// +// Parameters: +// pBN pointer to the Big Number context +// pR pointer to Finite Field Element context +// pGFp pointer to Finite Field context +*F*/ + +IPPFUN(IppStatus, ippsGFpSetElementRegular,(const IppsBigNumState* pBN, IppsGFpElement* pR, IppsGFpState* pGFp)) +{ + IPP_BAD_PTR1_RET(pBN); + IPP_BADARG_RET( !BN_VALID_ID(pBN), ippStsContextMatchErr ); + IPP_BADARG_RET( !BN_POSITIVE(pBN), ippStsOutOfRangeErr); + + return ippsGFpSetElement((Ipp32u*)BN_NUMBER(pBN), BITS2WORD32_SIZE( BITSIZE_BNU(BN_NUMBER((pBN)),BN_SIZE((pBN)))), pR, pGFp); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsqr.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsqr.c new file mode 100644 index 0000000..408c508 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsqr.c @@ -0,0 +1,74 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpSqr() +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsGFpSqr +// +// Purpose: Square of GF element +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFp +// NULL == pA +// NULL == pR +// +// ippStsContextMatchErr invalid pGFp->idCtx +// invalid pA->idCtx +// invalid pR->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM() != GFP_FELEN() +// +// ippStsNoErr no error +// +// Parameters: +// pA Pointer to the context of the source finite field element. +// pR Pointer to the context of the result finite field element. +// pGFp Pointer to the context of the finite field. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpSqr,(const IppsGFpElement* pA, + IppsGFpElement* pR, IppsGFpState* pGFp)) +{ + IPP_BAD_PTR3_RET(pA, pR, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pA), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pR), ippStsContextMatchErr ); + { + gsModEngine* pGFE = GFP_PMA(pGFp); + IPP_BADARG_RET( (GFPE_ROOM(pA)!=GFP_FELEN(pGFE)) || (GFPE_ROOM(pR)!=GFP_FELEN(pGFE)), ippStsOutOfRangeErr); + + GFP_METHOD(pGFE)->sqr(GFPE_DATA(pR), GFPE_DATA(pA), pGFE); + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsqrt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsqrt.c new file mode 100644 index 0000000..e9f9661 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsqrt.c @@ -0,0 +1,79 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpSqrt() +// +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + +/*F* +// Name: ippsGFpSqrt +// +// Purpose: Square root of GF element +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFp +// NULL == pA +// NULL == pR +// +// ippStsContextMatchErr invalid pGFp->idCtx +// invalid pA->idCtx +// invalid pR->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM() != GFP_FELEN() +// +// ippStsDivByZeroErr pA is zero +// +// ippStsBadArgErr pGFp is not prime +// +// ippStsQuadraticNonResidueErr pA is a square non-residue +// +// ippStsNoErr no error +// +// Parameters: +// pA Pointer to the context of the source finite field element. +// pR Pointer to the context of the result finite field element. +// pGFp Pointer to the context of the finite field. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpSqrt,(const IppsGFpElement* pA, + IppsGFpElement* pR, IppsGFpState* pGFp)) +{ + IPP_BAD_PTR3_RET(pA, pR, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pA), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pR), ippStsContextMatchErr ); + { + gsModEngine* pGFE = GFP_PMA(pGFp); + IPP_BADARG_RET( !GFP_IS_BASIC(pGFE), ippStsBadArgErr ) + IPP_BADARG_RET( (GFPE_ROOM(pA)!=GFP_FELEN(pGFE)) || (GFPE_ROOM(pR)!=GFP_FELEN(pGFE)), ippStsOutOfRangeErr); + + return cpGFpSqrt(GFPE_DATA(pR), GFPE_DATA(pA), pGFE)? ippStsNoErr : ippStsQuadraticNonResidueErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpstuff.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpstuff.h new file mode 100644 index 0000000..f15f350 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpstuff.h @@ -0,0 +1,210 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives +// Internal GF(p) basic Definitions & Function Prototypes +// +*/ + +#if !defined(_PCP_GFP_H_) +#define _PCP_GFP_H_ + +#include "owncp.h" +#include "pcpgfpmethod.h" +#include "pcpmontgomery.h" +#include "pcpmask_ct.h" + +/* GF element */ +typedef struct _cpGFpElement { + Ipp32u idCtx; /* GF() element ident */ + int length; /* length of element (in BNU_CHUNK_T) */ + BNU_CHUNK_T* pData; +} cpGFpElement; + +#define GFPE_SET_ID(pCtx) ((pCtx)->idCtx = (Ipp32u)idCtxGFPE ^ (Ipp32u)IPP_UINT_PTR(pCtx)) +#define GFPE_ROOM(pCtx) ((pCtx)->length) +#define GFPE_DATA(pCtx) ((pCtx)->pData) + +#define GFPE_VALID_ID(pCtx) ((((pCtx)->idCtx) ^ (Ipp32u)IPP_UINT_PTR(pCtx)) == idCtxGFPE) + + +/* GF(p) context */ +typedef struct _cpGFp { + Ipp32u idCtx; /* GFp spec ident */ + cpModulusID modulusID; + gsModEngine* pGFE; /* arithmethic engine */ +} cpGFp; + +#define GFP_ALIGNMENT ((int)(sizeof(void*))) + +/* Local definitions */ +#define GFP_MAX_BITSIZE (IPP_MAX_GF_BITSIZE) /* max bitsize for GF element */ +#define GFP_POOL_SIZE (16)//(IPP_MAX_EXPONENT_NUM+3) /* num of elements into the pool */ +#define GFP_RAND_ADD_BITS (128) /* parameter of random element generation ?? == febits/2 */ + +#define GFP_SET_ID(pCtx) ((pCtx)->idCtx = (Ipp32u)idCtxGFP ^ (Ipp32u)IPP_UINT_PTR(pCtx)) +#define GFP_PMA(pCtx) ((pCtx)->pGFE) + +#define GFP_PARENT(pCtx) MOD_PARENT((pCtx)) +#define GFP_EXTDEGREE(pCtx) MOD_EXTDEG((pCtx)) +#define GFP_FEBITLEN(pCtx) MOD_BITSIZE((pCtx)) +#define GFP_FELEN(pCtx) MOD_LEN((pCtx)) +#define GFP_FELEN32(pCtx) MOD_LEN32((pCtx)) +#define GFP_PELEN(pCtx) MOD_PELEN((pCtx)) +#define GFP_METHOD(pCtx) MOD_METHOD((pCtx)) +#define GFP_METHOD_ALT(pCtx) MOD_METHOD_ALT((pCtx)) +#define GFP_MODULUS(pCtx) MOD_MODULUS((pCtx)) +#define GFP_MNT_FACTOR(pCtx) MOD_MNT_FACTOR((pCtx)) +#define GFP_MNT_R(pCtx) MOD_MNT_R((pCtx)) +#define GFP_MNT_RR(pCtx) MOD_MNT_R2((pCtx)) +#define GFP_HMODULUS(pCtx) MOD_HMODULUS((pCtx)) +#define GFP_QNR(pCtx) MOD_QNR((pCtx)) +#define GFP_POOL(pCtx) MOD_POOL_BUF((pCtx)) +#define GFP_MAXPOOL(pCtx) MOD_MAXPOOL((pCtx)) +#define GFP_USEDPOOL(pCtx) MOD_USEDPOOL((pCtx)) + +#define GFP_IS_BASIC(pCtx) (GFP_PARENT((pCtx))==NULL) +#define GFP_VALID_ID(pCtx) ((((pCtx)->idCtx) ^ (Ipp32u)IPP_UINT_PTR(pCtx)) == idCtxGFP) + +/* +// get/release n element from/to the pool +*/ +#define cpGFpGetPool(n, gfe) gsModPoolAlloc((gfe), (n)) +#define cpGFpReleasePool(n, gfe) gsModPoolFree((gfe), (n)) + + +__INLINE int cpGFpElementLen(const BNU_CHUNK_T* pE, int nsE) +{ + for(; nsE>1 && 0==pE[nsE-1]; nsE--) ; + return nsE; +} +__INLINE BNU_CHUNK_T* cpGFpElementCopy(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pE, int nsE) +{ + int n; + for(n=0; nadd(pR, pA, pB, pGFE); +} + +__INLINE BNU_CHUNK_T* cpGFpSub(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsModEngine* pGFE) +{ + return GFP_METHOD(pGFE)->sub(pR, pA, pB, pGFE); +} + +__INLINE BNU_CHUNK_T* cpGFpNeg(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsModEngine* pGFE) +{ + return GFP_METHOD(pGFE)->neg(pR, pA, pGFE); +} + +__INLINE BNU_CHUNK_T* cpGFpMul(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsModEngine* pGFE) +{ + return GFP_METHOD(pGFE)->mul(pR, pA, pB, pGFE); +} + +__INLINE BNU_CHUNK_T* cpGFpSqr(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsModEngine* pGFE) +{ + return GFP_METHOD(pGFE)->sqr(pR, pA, pGFE); +} + +__INLINE BNU_CHUNK_T* cpGFpHalve(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsModEngine* pGFE) +{ + return GFP_METHOD(pGFE)->div2(pR, pA, pGFE); +} + + +#define GFP_EQ(a,b,size) (0 == cpCmp_BNU((a), (size), (b), (size))) + +#define GFP_IS_ZERO(a,size) cpGFpElementIsEquChunk((a),(size), 0) +#define GFP_IS_ONE(a,size) cpGFpElementIsEquChunk((a),(size), 1) + +#define GFP_ZERO(a,size) cpGFpElementSetChunk((a),(size), 0) +#define GFP_ONE(a,size) cpGFpElementSetChunk((a),(size), 1) + + +/* construct GF element */ +__INLINE IppsGFpElement* cpGFpElementConstruct(IppsGFpElement* pR, BNU_CHUNK_T* pDataBufer, int ns) +{ + GFPE_SET_ID(pR); + GFPE_ROOM(pR) = ns; + GFPE_DATA(pR) = pDataBufer; + return pR; +} + + +/* size of GFp context, init and setup */ +#define cpGFpGetSize OWNAPI(cpGFpGetSize) + IPP_OWN_DECL (int, cpGFpGetSize, (int feBitSize, int peBitSize, int numpe)) +#define cpGFpInitGFp OWNAPI(cpGFpInitGFp) + IPP_OWN_DECL (IppStatus, cpGFpInitGFp, (int primeBitSize, IppsGFpState* pGF)) +#define cpGFpSetGFp OWNAPI(cpGFpSetGFp) + IPP_OWN_DECL (IppStatus, cpGFpSetGFp, (const BNU_CHUNK_T* pPrime, int primeBitSize, const IppsGFpMethod* method, IppsGFpState* pGF)) + +/* operations */ +#define cpGFpRand OWNAPI(cpGFpRand) + IPP_OWN_DECL (BNU_CHUNK_T*, cpGFpRand, (BNU_CHUNK_T* pR, gsModEngine* pGFE, IppBitSupplier rndFunc, void* pRndParam)) +#define cpGFpSet OWNAPI(cpGFpSet) + IPP_OWN_DECL (BNU_CHUNK_T*, cpGFpSet, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pDataA, int nsA, gsModEngine* pGFE)) +#define cpGFpGet OWNAPI(cpGFpGet) + IPP_OWN_DECL (BNU_CHUNK_T*, cpGFpGet, (BNU_CHUNK_T* pDataA, int nsA, const BNU_CHUNK_T* pR, gsModEngine* pGFE)) +#define cpGFpSetOctString OWNAPI(cpGFpSetOctString) + IPP_OWN_DECL (BNU_CHUNK_T*, cpGFpSetOctString, (BNU_CHUNK_T* pR, const Ipp8u* pStr, int strSize, gsModEngine* pGFE)) +#define cpGFpGetOctString OWNAPI(cpGFpGetOctString) + IPP_OWN_DECL (Ipp8u*, cpGFpGetOctString, (Ipp8u* pStr, int strSize, const BNU_CHUNK_T* pA, gsModEngine* pGFE)) +#define cpGFpInv OWNAPI(cpGFpInv) + IPP_OWN_DECL (BNU_CHUNK_T*, cpGFpInv, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsModEngine* pGFE)) +#define cpGFpExp OWNAPI(cpGFpExp) + IPP_OWN_DECL (BNU_CHUNK_T*, cpGFpExp, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pE, int nsE, gsModEngine* pGFE)) +#define cpGFpSqrt OWNAPI(cpGFpSqrt) + IPP_OWN_DECL (int, cpGFpSqrt, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsModEngine* pGFE)) +#define cpGFEqnr OWNAPI(cpGFEqnr) + IPP_OWN_DECL (void, cpGFEqnr, (gsModEngine* pGFE)) + +#endif /* _PCP_GFP_H_ */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsub.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsub.c new file mode 100644 index 0000000..c3aeb2b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsub.c @@ -0,0 +1,76 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpSub() +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + +/*F* +// Name: ippsGFpSub +// +// Purpose: Subtract of GF elements +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFp +// NULL == pA +// NULL == pR +// NULL == pB +// +// ippStsContextMatchErr invalid pGFp->idCtx +// invalid pA->idCtx +// invalid pR->idCtx +// invalid pB->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM() != GFP_FELEN() +// +// ippStsNoErr no error +// +// Parameters: +// pA Pointer to the context of the first finite field element. +// pB Pointer to the context of the second finite field element. +// pR Pointer to the context of the result finite field element. +// pGFp Pointer to the context of the finite field. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpSub,(const IppsGFpElement* pA, const IppsGFpElement* pB, + IppsGFpElement* pR, IppsGFpState* pGFp)) +{ + IPP_BAD_PTR4_RET(pA, pB, pR, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pA), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pB), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pR), ippStsContextMatchErr ); + { + gsModEngine* pGFE = GFP_PMA(pGFp); + IPP_BADARG_RET( (GFPE_ROOM(pA)!=GFP_FELEN(pGFE)) || (GFPE_ROOM(pB)!=GFP_FELEN(pGFE)) || (GFPE_ROOM(pR)!=GFP_FELEN(pGFE)), ippStsOutOfRangeErr); + + GFP_METHOD(pGFE)->sub(GFPE_DATA(pR), GFPE_DATA(pA), GFPE_DATA(pB), pGFE); + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsubpe.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsubpe.c new file mode 100644 index 0000000..e8aa3e9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpsubpe.c @@ -0,0 +1,80 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p). +// +// Context: +// ippsGFpSub_PE() +// +*/ +#include "owndefs.h" +#include "owncp.h" + +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + +/*F* +// Name: ippsGFpSub_PE +// +// Purpose: Subtract of GF element and parent GF element +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFp +// NULL == pA +// NULL == pR +// NULL == pParentB +// +// ippStsContextMatchErr invalid pGFp->idCtx +// invalid pA->idCtx +// invalid pR->idCtx +// invalid pParentB->idCtx +// +// ippStsOutOfRangeErr GFPE_ROOM() != GFP_FELEN() +// +// ippStsBadArgErr pGFp does not specify prime field +// +// ippStsNoErr no error +// +// Parameters: +// pA Pointer to the context of the first finite field element. +// pParentB Pointer to the context of the second finite field element. +// pR Pointer to the context of the result finite field element. +// pGFp Pointer to the context of the finite field. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpSub_PE,(const IppsGFpElement* pA, const IppsGFpElement* pParentB, + IppsGFpElement* pR, IppsGFpState* pGFp)) +{ + IPP_BAD_PTR4_RET(pA, pParentB, pR, pGFp); + IPP_BADARG_RET( !GFP_VALID_ID(pGFp), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pA), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pParentB), ippStsContextMatchErr ); + IPP_BADARG_RET( !GFPE_VALID_ID(pR), ippStsContextMatchErr ); + { + gsModEngine* pGFE = GFP_PMA(pGFp); + IPP_BADARG_RET( GFP_IS_BASIC(pGFE), ippStsBadArgErr ) + IPP_BADARG_RET( (GFPE_ROOM(pA)!=GFP_FELEN(pGFE)) || (GFPE_ROOM(pR)!=GFP_FELEN(pGFE)), ippStsOutOfRangeErr); + IPP_BADARG_RET( (GFPE_ROOM(pParentB)!=GFP_FELEN(GFP_PARENT(pGFE))), ippStsOutOfRangeErr); + + cpGFpxSub_GFE(GFPE_DATA(pR), GFPE_DATA(pA), GFPE_DATA(pParentB), pGFE); + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx.c new file mode 100644 index 0000000..6beddf4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx.c @@ -0,0 +1,71 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p) ectension. +// +// Context: +// pcpgfpec_initgfpxctx.c() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + + +/* the "static" specificator removed because of incorrect result under Linux-32, p8 + what's wrong? not know maybe compiler (icl 2017) + need to check after switchng on icl 2018 + */ +/*static*/ +IPP_OWN_DEFN (void, InitGFpxCtx, (const IppsGFpState* pGroundGF, int extDeg, const IppsGFpMethod* method, IppsGFpState* pGFpx)) +{ + gsModEngine* pGFEp = GFP_PMA(pGroundGF); + int elemLen = extDeg * GFP_FELEN(pGFEp); + int elemLen32 = extDeg * GFP_FELEN32(pGFEp); + + Ipp8u* ptr = (Ipp8u*)pGFpx + sizeof(IppsGFpState); + + /* context identifier */ + GFP_SET_ID(pGFpx); + GFP_PMA(pGFpx) = (gsModEngine*)ptr; + { + gsModEngine* pGFEx = GFP_PMA(pGFpx); + + /* clear whole context */ + PadBlock(0, ptr, sizeof(gsModEngine)); + ptr += sizeof(gsModEngine); + + GFP_PARENT(pGFEx) = pGFEp; + GFP_EXTDEGREE(pGFEx) = extDeg; + GFP_FEBITLEN(pGFEx) = 0;//elemBitLen; + GFP_FELEN(pGFEx) = elemLen; + GFP_FELEN32(pGFEx) = elemLen32; + GFP_PELEN(pGFEx) = elemLen; + GFP_METHOD(pGFEx) = method->arith; + GFP_MODULUS(pGFEx) = (BNU_CHUNK_T*)(ptr); ptr += elemLen * (Ipp32s)sizeof(BNU_CHUNK_T); /* field polynomial */ + GFP_POOL(pGFEx) = (BNU_CHUNK_T*)(ptr); /* pool */ + GFP_MAXPOOL(pGFEx) = GFPX_POOL_SIZE; + GFP_USEDPOOL(pGFEx) = 0; + + cpGFpElementPad(GFP_MODULUS(pGFEx), elemLen, 0); + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_add_gfe.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_add_gfe.c new file mode 100644 index 0000000..c03407f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_add_gfe.c @@ -0,0 +1,43 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal operations over GF(p) extension. +// +// Context: +// cpGFpxAdd_GFE() +// +*/ + +#include "owncp.h" +#include "pcpbnumisc.h" +#include "pcpgfpxstuff.h" +#include "gsscramble.h" + +IPP_OWN_DEFN (BNU_CHUNK_T*, cpGFpxAdd_GFE, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pGroundB, gsModEngine* pGFEx)) +{ + gsModEngine* pGroundGFE = GFP_PARENT(pGFEx); + mod_add addF = MOD_METHOD(pGroundGFE)->add; + + if(pR != pA) { + int groundElemLen = GFP_FELEN(pGroundGFE); + int deg = GFP_EXTDEGREE(pGFEx); + cpGFpElementCopy(pR+groundElemLen, pA+groundElemLen, groundElemLen*(deg-1)); + } + return addF(pR, pA, pGroundB, pGroundGFE); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_conj.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_conj.c new file mode 100644 index 0000000..2ebe7f6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_conj.c @@ -0,0 +1,42 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal operations over GF(p) extension. +// +// Context: +// cpGFpxConj() +// +*/ + +#include "owncp.h" +#include "pcpbnumisc.h" +#include "pcpgfpxstuff.h" +#include "gsscramble.h" + +IPP_OWN_DEFN (BNU_CHUNK_T*, cpGFpxConj, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsModEngine* pGFEx)) +{ + gsModEngine* pGroundGFE = GFP_PARENT(pGFEx); + int groundElemLen = GFP_FELEN(pGroundGFE); + + if(pR != pA) + cpGFpElementCopy(pR, pA, groundElemLen); + MOD_METHOD(pGroundGFE)->neg(pR+groundElemLen, pA+groundElemLen, pGroundGFE); + + return pR; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_exp.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_exp.c new file mode 100644 index 0000000..1357616 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_exp.c @@ -0,0 +1,142 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal operations over GF(p) extension. +// +// Context: +// cpGFpxExp() +// +*/ + +#include "owncp.h" +#include "pcpbnumisc.h" +#include "pcpgfpxstuff.h" +#include "gsscramble.h" + +//tbcd: temporary excluded: #include + +static int div_upper(int a, int d) +{ return (a+d-1)/d; } + +/* sscm version */ +IPP_OWN_DEFN (BNU_CHUNK_T*, cpGFpxExp, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pE, int nsE, gsModEngine* pGFEx, Ipp8u* pScratchBuffer)) +{ + gsModEngine* pBasicGFE = cpGFpBasic(pGFEx); + + /* remove leding zeros */ + /* gres 06/10/2019: FIX_BNU(pE, nsE); */ + + { + mod_mul mulF = GFP_METHOD(pGFEx)->mul; /* mul and sqr methods */ + mod_sqr sqrF = GFP_METHOD(pGFEx)->sqr; + + BNU_CHUNK_T* pScratchAligned; /* aligned scratch buffer */ + int nAllocation = 0; /* points from the pool */ + + /* size of element */ + int elmLen = GFP_FELEN(pGFEx); + + /* exponent bitsize */ + int expBitSize = nsE * BNU_CHUNK_BITS; /* gres 06/102019: BITSIZE_BNU(pE, nsE); */ + /* optimal size of window */ + int w = (NULL==pScratchBuffer)? 1 : cpGFpGetOptimalWinSize(expBitSize); + /* number of table entries */ + int nPrecomputed = 1<>shift) & dmask; + + /* initialize result */ + //cpScrambleGet((Ipp8u*)pR, elmDataSize, pScratchAligned+windowVal, nPrecomputed); + gsScrambleGet_sscm(pR, elmLen, pScratchAligned, (Ipp32s)windowVal, w); + + for(wPosition-=w; wPosition>=0; wPosition-=w) { + int k; + /* w times squaring */ + for(k=0; k>shift) & dmask; + + /* extract value from the pre-computed table */ + //cpScrambleGet((Ipp8u*)pTmp, elmDataSize, pScratchAligned+windowVal, nPrecomputed); + gsScrambleGet_sscm(pTmp, elmLen, pScratchAligned, (Ipp32s)windowVal, w); + + /* and multiply */ + mulF(pR, pR, pTmp, pGFEx); + } + } + + } + + cpGFpReleasePool(nAllocation+2, pGFEx); + + return pR; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_get.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_get.c new file mode 100644 index 0000000..1aacdf9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_get.c @@ -0,0 +1,58 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal operations over GF(p) extension. +// +// Context: +// cpGFpxGet() +// +*/ + +#include "owncp.h" +#include "pcpbnumisc.h" +#include "pcpgfpxstuff.h" +#include "gsscramble.h" + +IPP_OWN_DEFN (BNU_CHUNK_T*, cpGFpxGet, (BNU_CHUNK_T* pDataA, int nsA, const BNU_CHUNK_T* pE, gsModEngine* pGFEx)) +{ + cpGFpElementPad(pDataA, nsA, 0); + + if( GFP_IS_BASIC(pGFEx) ) + return cpGFpGet(pDataA, nsA, pE, pGFEx); + + else { + gsModEngine* pBasicGFE = cpGFpBasic(pGFEx); + int basicElemLen = GFP_FELEN(pBasicGFE); + + BNU_CHUNK_T* pTmp = pDataA; + int basicDeg = cpGFpBasicDegreeExtension(pGFEx); + + int deg; + for(deg=0; deg0; deg++) { + int pieceA = IPP_MIN(nsA, basicElemLen); + + cpGFpGet(pTmp, pieceA, pE, pBasicGFE); + pE += basicElemLen; + pTmp += pieceA; + nsA -= pieceA; + } + + return pDataA; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_getpolyterm.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_getpolyterm.c new file mode 100644 index 0000000..2dc972a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_getpolyterm.c @@ -0,0 +1,36 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal operations over GF(p) extension. +// +// Context: +// cpGFpxGetPolyTerm() +// +*/ + +#include "owncp.h" +#include "pcpbnumisc.h" +#include "pcpgfpxstuff.h" +#include "gsscramble.h" + +IPP_OWN_DEFN (BNU_CHUNK_T*, cpGFpxGetPolyTerm, (BNU_CHUNK_T* pDataA, int nsA, const BNU_CHUNK_T* pE, int deg, gsModEngine* pGFEx)) +{ + pE += deg * GFP_FELEN(pGFEx); + return cpGFpxGet(pDataA, nsA, pE, pGFEx); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_inv.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_inv.c new file mode 100644 index 0000000..927dd1d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_inv.c @@ -0,0 +1,223 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal operations over GF(p) extension. +// +// Context: +// cpGFpxInv() +// +*/ + +#include "owncp.h" +#include "pcpbnumisc.h" +#include "pcpgfpxstuff.h" +#include "gsscramble.h" + +//tbcd: temporary excluded: #include + +static BNU_CHUNK_T* gfpxPolyDiv(BNU_CHUNK_T* pQ, BNU_CHUNK_T* pR, + const BNU_CHUNK_T* pA, + const BNU_CHUNK_T* pB, + gsModEngine* pGFEx) +{ + if( GFP_IS_BASIC(pGFEx) ) + return NULL; + + else { + int elemLen = GFP_FELEN(pGFEx); + gsModEngine* pGroundGFE = GFP_PARENT(pGFEx); + int termLen = GFP_FELEN(pGroundGFE); + + int degA = degree(pA, pGFEx); + int degB = degree(pB, pGFEx); + + if(degB==0) { + if( GFP_IS_ZERO(pB, termLen) ) + return NULL; + else { + gsModEngine* pBasicGFE = cpGFpBasic(pGroundGFE); + + cpGFpInv(pR, pB, pBasicGFE); + cpGFpElementPad(pR+GFP_FELEN(pGroundGFE), termLen-GFP_FELEN(pGroundGFE), 0); + cpGFpxMul_GFE(pQ, pA, pR, pGFEx); + cpGFpElementPad(pR, elemLen, 0); + return pR; + } + } + + if(degA < degB) { + cpGFpElementPad(pQ, elemLen, 0); + cpGFpElementCopyPad(pR, elemLen, pA, (degA+1)*termLen); + return pR; + } + + else { + mod_mul mulF = GFP_METHOD(pGroundGFE)->mul; + mod_sub subF = GFP_METHOD(pGroundGFE)->sub; + + int i, j; + BNU_CHUNK_T* pProduct = cpGFpGetPool(2, pGroundGFE); + BNU_CHUNK_T* pInvB = pProduct + GFP_PELEN(pGroundGFE); + //tbcd: temporary excluded: assert(NULL!=pProduct); + + cpGFpElementCopyPad(pR, elemLen, pA, (degA+1)*termLen); + cpGFpElementPad(pQ, elemLen, 0); + + cpGFpxInv(pInvB, GFPX_IDX_ELEMENT(pB, degB, termLen), pGroundGFE); + + for(i=0; i<=degA-degB && !GFP_IS_ZERO(GFPX_IDX_ELEMENT(pR, degA-i, termLen), termLen); i++) { + /* compute q term */ + mulF(GFPX_IDX_ELEMENT(pQ, degA-degB-i, termLen), + GFPX_IDX_ELEMENT(pR, degA-i, termLen), + pInvB, + pGroundGFE); + + /* R -= B * q */ + cpGFpElementPad(GFPX_IDX_ELEMENT(pR, degA-i, termLen), termLen, 0); + for(j=0; jmul; + mod_sub subF = GFP_METHOD(pGroundGFE)->sub; + + int termLen = GFP_FELEN(pGroundGFE); + + BNU_CHUNK_T* pInvB = cpGFpGetPool(2, pGroundGFE); + BNU_CHUNK_T* pTmp = pInvB + GFP_PELEN(pGroundGFE); + + int degB = degree(pB, pGFEx); + int i; + + //tbcd: temporary excluded: assert(NULL!=pInvB); + + cpGFpElementCopy(pR, GFP_MODULUS(pGFEx), elemLen); + cpGFpElementPad(pQ, elemLen, 0); + + cpGFpxInv(pInvB, GFPX_IDX_ELEMENT(pB, degB, termLen), pGroundGFE); + + for(i=0; i 0) { + gfpxPolyDiv(quo, temp, lastrem, rem, pGFEx); + SWAP_PTR(BNU_CHUNK_T, rem, lastrem); // + SWAP_PTR(BNU_CHUNK_T, temp, rem); + + GFP_METHOD(pGFEx)->neg(quo, quo, pGFEx); + GFP_METHOD(pGFEx)->mul(temp, quo, aux, pGFEx); + GFP_METHOD(pGFEx)->add(temp, lastaux, temp, pGFEx); + SWAP_PTR(BNU_CHUNK_T, aux, lastaux); + SWAP_PTR(BNU_CHUNK_T, temp, aux); + } + if (GFP_IS_ZERO(rem, elemLen)) { /* gcd != 1 */ + cpGFpReleasePool(pxVars, pGFEx); + return NULL; + } + + { + BNU_CHUNK_T* invRem = cpGFpGetPool(1, pGroundGFE); + //tbcd: temporary excluded: assert(NULL!=invRem); + + cpGFpxInv(invRem, rem, pGroundGFE); + cpGFpxMul_GFE(pR, aux, invRem, pGFEx); + + cpGFpReleasePool(1, pGroundGFE); + } + + cpGFpReleasePool(pxVars, pGFEx); + + return pR; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_mul_gfe.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_mul_gfe.c new file mode 100644 index 0000000..e147986 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_mul_gfe.c @@ -0,0 +1,48 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal operations over GF(p) extension. +// +// Context: +// cpGFpxMul_GFE() +// +*/ + +#include "owncp.h" +#include "pcpbnumisc.h" +#include "pcpgfpxstuff.h" +#include "gsscramble.h" + +IPP_OWN_DEFN (BNU_CHUNK_T*, cpGFpxMul_GFE, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pGroundB, gsModEngine* pGFEx)) +{ + gsModEngine* pGroundGFE = GFP_PARENT(pGFEx); + mod_mul mulF = MOD_METHOD(pGroundGFE)->mul; + + int grounfElemLen = GFP_FELEN(pGroundGFE); + + BNU_CHUNK_T* pTmp = pR; + + int deg; + for(deg=0; deg + +static int GetIndex(const BNU_CHUNK_T* ppE[], int nItems, int nBit) +{ + int shift = nBit%BYTESIZE; + int offset= nBit/BYTESIZE; + int index = 0; + + int n; + for(n=nItems; n>0; n--) { + const Ipp8u* pE = ((Ipp8u*)ppE[n-1]) + offset; + Ipp8u e = pE[0]; + index <<= 1; + index += (e>>shift) &1; + } + return index; +} + + +static void cpPrecomputeMultiExp(BNU_CHUNK_T* pTable, const BNU_CHUNK_T* ppA[], int nItems, gsModEngine* pGFEx) +{ + gsModEngine* pBasicGFE = cpGFpBasic(pGFEx); + + //int nPrecomputed = 1<mul; /* mul method */ + + int i, baseIdx; + for(i=1, baseIdx=2; i=0; k--) { + int tblIdx = baseIdx; + + int n; + for(n=0; nmul; /* mul and sqr methods and parameter */ + mod_sqr sqrF = GFP_METHOD(pGFEx)->sqr; + int elmLen = GFP_FELEN(pGFEx); + + /* find out the longest exponent */ + int expBitSize = cpGetMaxBitsizeExponent(ppE, nsE, nItems); + + /* allocate resource and copy expanded exponents into */ + const BNU_CHUNK_T* ppExponent[IPP_MAX_EXPONENT_NUM]; + { + int n; + for(n=0; n=0; expBitSize--) { + sqrF(pR, pR, pGFEx); + tblIdx = GetIndex(ppExponent, nItems, expBitSize); + //cpScrambleGet((Ipp8u*)pT, elmDataSize, pScratchBuffer+tblIdx, nPrecomputed); + gsScrambleGet_sscm(pT, elmLen, pTable, tblIdx, nItems); + mulF(pR, pR, pT, pGFEx); + } + + /* release resourse */ + cpGFpReleasePool(1, pGFEx); + } + + /* release resourse */ + cpGFpReleasePool(nItems, pGFEx); + + return pR; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_neg.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_neg.c new file mode 100644 index 0000000..5e4f8fa --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_neg.c @@ -0,0 +1,46 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal operations over GF(p) extension. +// +// Context: +// cpGFpxNeg() +// +*/ + +#include "owncp.h" +#include "pcpbnumisc.h" +#include "pcpgfpxstuff.h" +#include "gsscramble.h" + +IPP_OWN_DEFN (BNU_CHUNK_T*, cpGFpxNeg, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsModEngine* pGFEx)) +{ + gsModEngine* pBasicGFE = cpGFpBasic(pGFEx); + int basicElemLen = GFP_FELEN(pBasicGFE); + int basicDeg = cpGFpBasicDegreeExtension(pGFEx); + + BNU_CHUNK_T* pTmp = pR; + int deg; + for(deg=0; degneg(pTmp, pA, pBasicGFE); + pTmp += basicElemLen; + pA += basicElemLen; + } + return pR; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_rand.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_rand.c new file mode 100644 index 0000000..397b1bf --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpx_rand.c @@ -0,0 +1,52 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Internal operations over GF(p) extension. +// +// Context: +// cpGFpxRand() +// +*/ + +#include "owncp.h" +#include "pcpbnumisc.h" +#include "pcpgfpxstuff.h" +#include "gsscramble.h" + + +IPP_OWN_DEFN (BNU_CHUNK_T*, cpGFpxRand, (BNU_CHUNK_T* pR, gsModEngine* pGFEx, IppBitSupplier rndFunc, void* pRndParam)) +{ + if( GFP_IS_BASIC(pGFEx) ) + return cpGFpRand(pR, pGFEx, rndFunc, pRndParam); + + else { + gsModEngine* pBasicGFE = cpGFpBasic(pGFEx); + int basicElemLen = GFP_FELEN(pBasicGFE); + int basicDeg = cpGFpBasicDegreeExtension(pGFEx); + + BNU_CHUNK_T* pTmp = pR; + int deg; + for(deg=0; degsub; + + if(pR != pA) { + int groundElemLen = GFP_FELEN(pGroundGFE); + int deg = GFP_EXTDEGREE(pGFEx); + cpGFpElementCopy(pR+groundElemLen, pA+groundElemLen, groundElemLen*(deg-1)); + } + return subF(pR, pA, pGroundB, pGroundGFE); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxgetsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxgetsize.c new file mode 100644 index 0000000..4ede120 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxgetsize.c @@ -0,0 +1,85 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p) ectension. +// +// Context: +// pcpgfpxgetsize.c() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + +/* Get context size */ +static int cpGFExGetSize(int elemLen, int pelmLen, int numpe) +{ + int ctxSize = 0; + + /* size of GFp engine */ + ctxSize = (Ipp32s)sizeof(gsModEngine) + + elemLen*(Ipp32s)sizeof(BNU_CHUNK_T) /* modulus */ + + pelmLen*(Ipp32s)sizeof(BNU_CHUNK_T)*numpe; /* pool */ + + ctxSize = (Ipp32s)sizeof(IppsGFpState) /* size of IppsGFPState*/ + + ctxSize; /* GFpx engine */ + return ctxSize; +} + +/*F* +// Name: ippsGFpxGetSize +// +// Purpose: Gets the size of the context of a GF(p^d) field. +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL. +// ippStsContextMatchErr !GFP_VALID_ID(pGroundGF) +// ippStsBadArgErr degree is greater than or equal to 9 or is less than 2. +// ippStsNoErr no error +// +// Parameters: +// pGroundGF Pointer to the context of the finite field GF(p) being extended. +// degree Degree of the extension. +// pSize Pointer to the buffer size, in bytes, needed for the IppsGFpState +// context. +// +*F*/ + +IPPFUN(IppStatus, ippsGFpxGetSize, (const IppsGFpState* pGroundGF, int degree, int* pSize)) +{ + IPP_BAD_PTR2_RET(pGroundGF, pSize); + IPP_BADARG_RET( degreeIPP_MAX_GF_EXTDEG, ippStsBadArgErr); + IPP_BADARG_RET( !GFP_VALID_ID(pGroundGF), ippStsContextMatchErr ); + + #define MAX_GFx_SIZE (1<<15) /* max size (bytes) of GF element (32KB) */ + { + int groundElmLen = GFP_FELEN(GFP_PMA(pGroundGF)); + Ipp64u elmLen64 = (Ipp64u)groundElmLen * (Ipp64u)sizeof(BNU_CHUNK_T) * (Ipp64u)degree; + int elemLen = (int)IPP_LODWORD(elmLen64); + *pSize = 0; + IPP_BADARG_RET(elmLen64> MAX_GFx_SIZE, ippStsBadArgErr); + + *pSize = cpGFExGetSize(elemLen, elemLen, GFPX_POOL_SIZE); + return ippStsNoErr; + } + #undef MAX_GFx_SIZE +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxinit.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxinit.c new file mode 100644 index 0000000..515f868 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxinit.c @@ -0,0 +1,108 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// Operations over GF(p) ectension. +// +// Context: +// pcpgfpxinit.c() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpstuff.h" +#include "pcpgfpxstuff.h" +#include "pcptool.h" + + + +/*F* +// Name: ippsGFpxInit +// +// Purpose: initializes finite field extension GF(p^d) +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pGFpx +// NULL == pGroundGF +// NULL == ppGroundElm +// NULL == pGFpMethod +// +// ippStsContextMatchErr incorrect pGroundGF's context ID +// incorrect ppGroundElm[i]'s context ID +// +// ippStsOutOfRangeErr size of ppGroundElm[i] does not equal to size of pGroundGF element +// +// ippStsBadArgErr IPP_MIN_GF_EXTDEG > extDeg || extDeg > IPP_MAX_GF_EXTDEG +// (IPP_MIN_GF_EXTDEG==2, IPP_MAX_GF_EXTDEG==8) +// 1>nElm || nElm>extDeg +// +// cpID_Poly!=pGFpMethod->modulusID -- method does not refferenced to polynomial one +// pGFpMethod->modulusBitDeg!=extDeg -- fixed method does not match to degree extension +// +// ippStsNoErr no error +// +// Parameters: +// pGroundGF pointer to the context of the finite field is being extension +// extDeg degree of extension +// ppGroundElm[] pointer to the array of extension field polynomial +// nElm number of coefficients above +// pGFpMethod pointer to the basic arithmetic metods +// pGFpx pointer to Finite Field context is being initialized +*F*/ +IPPFUN(IppStatus, ippsGFpxInit,(const IppsGFpState* pGroundGF, int extDeg, + const IppsGFpElement* const ppGroundElm[], int nElm, + const IppsGFpMethod* pGFpMethod, IppsGFpState* pGFpx)) +{ + IPP_BAD_PTR4_RET(pGFpx, pGroundGF, ppGroundElm, pGFpMethod); + + IPP_BADARG_RET( !GFP_VALID_ID(pGroundGF), ippStsContextMatchErr ); + + /* test extension degree */ + IPP_BADARG_RET( extDegIPP_MAX_GF_EXTDEG, ippStsBadArgErr); + /* coeffs at (x^0), (x^1), ..., (x^(deg-1)) passed acually */ + /* considering normilized f(x), the coeff at (x^deg) is 1 and so could neither stored no passed */ + /* test if 1<=nElm<=extDeg */ + IPP_BADARG_RET( 1>nElm || nElm>extDeg, ippStsBadArgErr); + + /* test if method is polynomial based */ + IPP_BADARG_RET(cpID_Poly != (pGFpMethod->modulusID & cpID_Poly), ippStsBadArgErr); + /* test if method is fixed polynomial based */ + IPP_BADARG_RET(pGFpMethod->modulusBitDeg && (pGFpMethod->modulusBitDeg!=extDeg), ippStsBadArgErr); + + InitGFpxCtx(pGroundGF, extDeg, pGFpMethod, pGFpx); + + { + BNU_CHUNK_T* pPoly = GFP_MODULUS(GFP_PMA(pGFpx)); + int polyTermlen = GFP_FELEN(GFP_PMA(pGroundGF)); + int n; + for(n=0; n extDeg || extDeg > IPP_MAX_GF_EXTDEG +// (IPP_MIN_GF_EXTDEG==2, IPP_MAX_GF_EXTDEG==8) +// +// cpID_Poly!=pGFpMethod->modulusID -- method does not refferenced to polynomial one +// pGFpMethod->modulusBitDeg!=extDeg -- fixed method does not match to degree extension +// +// ippStsNoErr no error +// +// Parameters: +// pGroundGF pointer to the context of the finite field is being extension +// extDeg degree of extension +// pGroundElm pointer to the IppsGFpElement context containing the trailing coefficient of the field binomial. +// pGFpMethod pointer to the basic arithmetic metods +// pGFpx pointer to Finite Field context is being initialized +*F*/ +IPPFUN(IppStatus, ippsGFpxInitBinomial,(const IppsGFpState* pGroundGF, int extDeg, + const IppsGFpElement* pGroundElm, + const IppsGFpMethod* pGFpMethod, + IppsGFpState* pGFpx)) +{ + IPP_BAD_PTR4_RET(pGFpx, pGroundGF, pGroundElm, pGFpMethod); + + IPP_BADARG_RET( !GFP_VALID_ID(pGroundGF), ippStsContextMatchErr ); + + IPP_BADARG_RET( !GFPE_VALID_ID(pGroundElm), ippStsContextMatchErr ); + IPP_BADARG_RET(GFPE_ROOM(pGroundElm)!=GFP_FELEN(GFP_PMA(pGroundGF)), ippStsOutOfRangeErr); + + IPP_BADARG_RET( extDegIPP_MAX_GF_EXTDEG, ippStsBadArgErr); + + /* test method is binomial based */ + IPP_BADARG_RET(cpID_Binom != (pGFpMethod->modulusID & cpID_Binom), ippStsBadArgErr); + + /* test if method assums fixed degree extension */ + IPP_BADARG_RET(pGFpMethod->modulusBitDeg && (extDeg!=pGFpMethod->modulusBitDeg), ippStsBadArgErr); + + /* init context */ + InitGFpxCtx(pGroundGF, extDeg, pGFpMethod, pGFpx); + + /* store low-order coefficient of irresucible into the context */ + cpGFpElementCopy(GFP_MODULUS(GFP_PMA(pGFpx)), GFPE_DATA(pGroundElm), GFP_FELEN(GFP_PMA(pGroundGF))); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_binom.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_binom.c new file mode 100644 index 0000000..0e44995 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_binom.c @@ -0,0 +1,123 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// GF(p^d) methods, if binomial generator +// +*/ +#include "owncp.h" + +#include "pcpgfpxstuff.h" +#include "pcpgfpxmethod_com.h" + +//tbcd: temporary excluded: #include + +/* +// Multiplication in GF(p^d), if field polynomial: g(x) = x^d + beta => binominal +*/ +IPP_OWN_DEFN (static BNU_CHUNK_T*, cpGFpxMul_pd_binom, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFEx)) +{ + BNU_CHUNK_T* pGFpolynomial = GFP_MODULUS(pGFEx); + int deg = GFP_EXTDEGREE(pGFEx); + int elemLen= GFP_FELEN(pGFEx); + int groundElemLen = GFP_FELEN(GFP_PARENT(pGFEx)); + int d; + + BNU_CHUNK_T* R = cpGFpGetPool(4, pGFEx); + BNU_CHUNK_T* X = R+elemLen; + BNU_CHUNK_T* T0= X+elemLen; + BNU_CHUNK_T* T1= T0+elemLen; + //tbcd: temporary excluded: assert(NULL!=R); + + /* T0 = A * beta */ + cpGFpxMul_GFE(T0, pA, pGFpolynomial, pGFEx); + /* T1 = A */ + cpGFpElementCopy(T1, pA, elemLen); + + /* R = A * B[0] */ + cpGFpxMul_GFE(R, pA, pB, pGFEx); + + /* R += (A*B[d]) mod g() */ + for(d=1; dadd(R, R, X, pGFEx); + } + cpGFpElementCopy(pR, R, elemLen); + + cpGFpReleasePool(4, pGFEx); + return pR; +} + +/* +// Squaring in GF(p^d), if field polynomial: g(x) = x^d + beta => binominal +*/ +IPP_OWN_DEFN (static BNU_CHUNK_T*, cpGFpxSqr_pd_binom, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFEx)) +{ + #if defined(__INTEL_COMPILER) + #pragma noinline + #endif + return cpGFpxMul_pd_binom(pR, pA, pA, pGFEx); +} + +/* +// return specific polynomial arith methods +// polynomial - general binomial +*/ +static gsModMethod* gsPolyArith_binom (void) +{ + static gsModMethod m = { + cpGFpxEncode_com, + cpGFpxDecode_com, + cpGFpxMul_pd_binom, + cpGFpxSqr_pd_binom, + NULL, + cpGFpxAdd_com, + cpGFpxSub_com, + cpGFpxNeg_com, + cpGFpxDiv2_com, + cpGFpxMul2_com, + cpGFpxMul3_com, + //cpGFpxInv + }; + return &m; +} + +/*F* +// Name: ippsGFpxMethod_binom2 +// +// Purpose: Returns a reference to the implementation of arithmetic operations over GF(pd). +// +// Returns: pointer to a structure containing +// an implementation of arithmetic operations over GF(pd) +// g(x) = x^d - a0, a0 from GF(p) +// +// +*F*/ + +IPPFUN( const IppsGFpMethod*, ippsGFpxMethod_binom, (void) ) +{ + static IppsGFpMethod method = { + cpID_Binom, + 0, + NULL, + NULL, + NULL + }; + method.arith = gsPolyArith_binom(); + return &method; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_binom2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_binom2.c new file mode 100644 index 0000000..0ab1801 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_binom2.c @@ -0,0 +1,191 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// GF(p^d) methods, if binomial generator +// +*/ +#include "owncp.h" + +#include "pcpgfpxmethod_binom_mulc.h" +#include "pcpgfpxmethod_com.h" + +//tbcd: temporary excluded: #include + +/* +// Multiplication in GF(p^2), if field polynomial: g(x) = x^2 + beta => binominal +*/ +IPP_OWN_DEFN (static BNU_CHUNK_T*, cpGFpxMul_p2_binom, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFEx)) +{ + gsEngine* pGroundGFE = GFP_PARENT(pGFEx); + int groundElemLen = GFP_FELEN(pGroundGFE); + + mod_mul mulF = GFP_METHOD(pGroundGFE)->mul; + mod_add addF = GFP_METHOD(pGroundGFE)->add; + mod_sub subF = GFP_METHOD(pGroundGFE)->sub; + + const BNU_CHUNK_T* pA0 = pA; + const BNU_CHUNK_T* pA1 = pA+groundElemLen; + + const BNU_CHUNK_T* pB0 = pB; + const BNU_CHUNK_T* pB1 = pB+groundElemLen; + + BNU_CHUNK_T* pR0 = pR; + BNU_CHUNK_T* pR1 = pR+groundElemLen; + + BNU_CHUNK_T* t0 = cpGFpGetPool(4, pGroundGFE); + BNU_CHUNK_T* t1 = t0+groundElemLen; + BNU_CHUNK_T* t2 = t1+groundElemLen; + BNU_CHUNK_T* t3 = t2+groundElemLen; + //tbcd: temporary excluded: assert(NULL!=t0); + + #if defined GS_DBG + BNU_CHUNK_T* arg0 = cpGFpGetPool(1, pGroundGFE); + BNU_CHUNK_T* arg1 = cpGFpGetPool(1, pGroundGFE); + #endif + #if defined GS_DBG + cpGFpxGet(arg0, groundElemLen, pA0, pGroundGFE); + cpGFpxGet(arg1, groundElemLen, pB0, pGroundGFE); + #endif + + mulF(t0, pA0, pB0, pGroundGFE); /* t0 = a[0]*b[0] */ + + #if defined GS_DBG + cpGFpxGet(arg0, groundElemLen, pA1, pGroundGFE); + cpGFpxGet(arg1, groundElemLen, pB1, pGroundGFE); + #endif + + mulF(t1, pA1, pB1, pGroundGFE); /* t1 = a[1]*b[1] */ + addF(t2, pA0, pA1, pGroundGFE); /* t2 = a[0]+a[1] */ + addF(t3, pB0, pB1, pGroundGFE); /* t3 = b[0]+b[1] */ + + #if defined GS_DBG + cpGFpxGet(arg0, groundElemLen, t2, pGroundGFE); + cpGFpxGet(arg1, groundElemLen, t3, pGroundGFE); + #endif + + mulF(pR1, t2, t3, pGroundGFE); /* r[1] = (a[0]+a[1]) * (b[0]+b[1]) */ + subF(pR1, pR1, t0, pGroundGFE); /* r[1] -= a[0]*b[0]) + a[1]*b[1] */ + subF(pR1, pR1, t1, pGroundGFE); + + cpGFpxMul_G0(t1, t1, pGFEx); + subF(pR0, t0, t1, pGroundGFE); + + #if defined GS_DBG + cpGFpReleasePool(2, pGroundGFE); + #endif + + cpGFpReleasePool(4, pGroundGFE); + return pR; +} + +/* +// Squaring in GF(p^2), if field polynomial: g(x) = x^2 + beta => binominal +*/ +IPP_OWN_DEFN (static BNU_CHUNK_T*, cpGFpxSqr_p2_binom, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFEx)) +{ + gsEngine* pGroundGFE = GFP_PARENT(pGFEx); + int groundElemLen = GFP_FELEN(pGroundGFE); + + mod_mul mulF = GFP_METHOD(pGroundGFE)->mul; + mod_sqr sqrF = GFP_METHOD(pGroundGFE)->sqr; + mod_add addF = GFP_METHOD(pGroundGFE)->add; + mod_sub subF = GFP_METHOD(pGroundGFE)->sub; + + const BNU_CHUNK_T* pA0 = pA; + const BNU_CHUNK_T* pA1 = pA+groundElemLen; + + BNU_CHUNK_T* pR0 = pR; + BNU_CHUNK_T* pR1 = pR+groundElemLen; + + BNU_CHUNK_T* t0 = cpGFpGetPool(3, pGroundGFE); + BNU_CHUNK_T* t1 = t0+groundElemLen; + BNU_CHUNK_T* u0 = t1+groundElemLen; + //tbcd: temporary excluded: assert(NULL!=t0); + + #if defined GS_DBG + BNU_CHUNK_T* arg0 = cpGFpGetPool(1, pGroundGFE); + BNU_CHUNK_T* arg1 = cpGFpGetPool(1, pGroundGFE); + #endif + #if defined GS_DBG + cpGFpxGet(arg0, groundElemLen, pA0, pGroundGFE); + cpGFpxGet(arg1, groundElemLen, pA1, pGroundGFE); + #endif + + mulF(u0, pA0, pA1, pGroundGFE); /* u0 = a[0]*a[1] */ + sqrF(t0, pA0, pGroundGFE); /* t0 = a[0]*a[0] */ + sqrF(t1, pA1, pGroundGFE); /* t1 = a[1]*a[1] */ + cpGFpxMul_G0(t1, t1, pGFEx); + subF(pR0, t0, t1, pGroundGFE); + addF(pR1, u0, u0, pGroundGFE); /* r[1] = 2*a[0]*a[1] */ + + #if defined GS_DBG + cpGFpReleasePool(2, pGroundGFE); + #endif + + cpGFpReleasePool(3, pGroundGFE); + return pR; +} + +/* +// return specific polynomi alarith methods +// polynomial - deg 2 binomial +*/ +static gsModMethod* gsPolyArith_binom2 (void) +{ + static gsModMethod m = { + cpGFpxEncode_com, + cpGFpxDecode_com, + cpGFpxMul_p2_binom, + cpGFpxSqr_p2_binom, + NULL, + cpGFpxAdd_com, + cpGFpxSub_com, + cpGFpxNeg_com, + cpGFpxDiv2_com, + cpGFpxMul2_com, + cpGFpxMul3_com, + //cpGFpxInv + }; + return &m; +} + +/*F* +// Name: ippsGFpxMethod_binom2 +// +// Purpose: Returns a reference to the implementation of arithmetic operations over GF(pd). +// +// Returns: pointer to a structure containing +// an implementation of arithmetic operations over GF(pd) +// g(x) = x^2 - a0, a0 from GF(p) +// +// +*F*/ + +IPPFUN( const IppsGFpMethod*, ippsGFpxMethod_binom2, (void) ) +{ + static IppsGFpMethod method = { + cpID_Binom, + 2, + NULL, + NULL, + NULL + }; + method.arith = gsPolyArith_binom2(); + return &method; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_binom3.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_binom3.c new file mode 100644 index 0000000..5c83152 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_binom3.c @@ -0,0 +1,195 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// GF(p^d) methods, if binomial generator +// +*/ +#include "owncp.h" + +#include "pcpgfpxmethod_binom_mulc.h" +#include "pcpgfpxmethod_com.h" + +//tbcd: temporary excluded: #include + +/* +// Multiplication in GF(p^3), if field polynomial: g(x) = x^3 + beta => binominal +*/ +IPP_OWN_DEFN (static BNU_CHUNK_T*, cpGFpxMul_p3_binom, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFEx)) +{ + gsEngine* pGroundGFE = GFP_PARENT(pGFEx); + int groundElemLen = GFP_FELEN(pGroundGFE); + + mod_mul mulF = GFP_METHOD(pGroundGFE)->mul; + mod_add addF = GFP_METHOD(pGroundGFE)->add; + mod_sub subF = GFP_METHOD(pGroundGFE)->sub; + + const BNU_CHUNK_T* pA0 = pA; + const BNU_CHUNK_T* pA1 = pA+groundElemLen; + const BNU_CHUNK_T* pA2 = pA+groundElemLen*2; + + const BNU_CHUNK_T* pB0 = pB; + const BNU_CHUNK_T* pB1 = pB+groundElemLen; + const BNU_CHUNK_T* pB2 = pB+groundElemLen*2; + + BNU_CHUNK_T* pR0 = pR; + BNU_CHUNK_T* pR1 = pR+groundElemLen; + BNU_CHUNK_T* pR2 = pR+groundElemLen*2; + + BNU_CHUNK_T* t0 = cpGFpGetPool(6, pGroundGFE); + BNU_CHUNK_T* t1 = t0+groundElemLen; + BNU_CHUNK_T* t2 = t1+groundElemLen; + BNU_CHUNK_T* u0 = t2+groundElemLen; + BNU_CHUNK_T* u1 = u0+groundElemLen; + BNU_CHUNK_T* u2 = u1+groundElemLen; + //tbcd: temporary excluded: assert(NULL!=t0); + + addF(u0 ,pA0, pA1, pGroundGFE); /* u0 = a[0]+a[1] */ + addF(t0 ,pB0, pB1, pGroundGFE); /* t0 = b[0]+b[1] */ + mulF(u0, u0, t0, pGroundGFE); /* u0 = (a[0]+a[1])*(b[0]+b[1]) */ + mulF(t0, pA0, pB0, pGroundGFE); /* t0 = a[0]*b[0] */ + + addF(u1 ,pA1, pA2, pGroundGFE); /* u1 = a[1]+a[2] */ + addF(t1 ,pB1, pB2, pGroundGFE); /* t1 = b[1]+b[2] */ + mulF(u1, u1, t1, pGroundGFE); /* u1 = (a[1]+a[2])*(b[1]+b[2]) */ + mulF(t1, pA1, pB1, pGroundGFE); /* t1 = a[1]*b[1] */ + + addF(u2 ,pA2, pA0, pGroundGFE); /* u2 = a[2]+a[0] */ + addF(t2 ,pB2, pB0, pGroundGFE); /* t2 = b[2]+b[0] */ + mulF(u2, u2, t2, pGroundGFE); /* u2 = (a[2]+a[0])*(b[2]+b[0]) */ + mulF(t2, pA2, pB2, pGroundGFE); /* t2 = a[2]*b[2] */ + + subF(u0, u0, t0, pGroundGFE); /* u0 = a[0]*b[1]+a[1]*b[0] */ + subF(u0, u0, t1, pGroundGFE); + subF(u1, u1, t1, pGroundGFE); /* u1 = a[1]*b[2]+a[2]*b[1] */ + subF(u1, u1, t2, pGroundGFE); + subF(u2, u2, t2, pGroundGFE); /* u2 = a[2]*b[0]+a[0]*b[2] */ + subF(u2, u2, t0, pGroundGFE); + + cpGFpxMul_G0(u1, u1, pGFEx); /* u1 = (a[1]*b[2]+a[2]*b[1]) * beta */ + cpGFpxMul_G0(t2, t2, pGFEx); /* t2 = a[2]*b[2] * beta */ + + subF(pR0, t0, u1, pGroundGFE); /* r[0] = a[0]*b[0] - (a[2]*b[1]+a[1]*b[2])*beta */ + subF(pR1, u0, t2, pGroundGFE); /* r[1] = a[1]*b[0] + a[0]*b[1] - a[2]*b[2]*beta */ + + addF(pR2, u2, t1, pGroundGFE); /* r[2] = a[2]*b[0] + a[1]*b[1] + a[0]*b[2] */ + + cpGFpReleasePool(6, pGroundGFE); + return pR; +} + +/* +// Squaring in GF(p^3), if field polynomial: g(x) = x^3 + beta => binominal +*/ +IPP_OWN_DEFN (static BNU_CHUNK_T*, cpGFpxSqr_p3_binom, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFEx)) +{ + gsEngine* pGroundGFE = GFP_PARENT(pGFEx); + int groundElemLen = GFP_FELEN(pGroundGFE); + + mod_mul mulF = GFP_METHOD(pGroundGFE)->mul; + mod_sqr sqrF = GFP_METHOD(pGroundGFE)->sqr; + mod_add addF = GFP_METHOD(pGroundGFE)->add; + mod_sub subF = GFP_METHOD(pGroundGFE)->sub; + + const BNU_CHUNK_T* pA0 = pA; + const BNU_CHUNK_T* pA1 = pA+groundElemLen; + const BNU_CHUNK_T* pA2 = pA+groundElemLen*2; + + BNU_CHUNK_T* pR0 = pR; + BNU_CHUNK_T* pR1 = pR+groundElemLen; + BNU_CHUNK_T* pR2 = pR+groundElemLen*2; + + BNU_CHUNK_T* s0 = cpGFpGetPool(5, pGroundGFE); + BNU_CHUNK_T* s1 = s0+groundElemLen; + BNU_CHUNK_T* s2 = s1+groundElemLen; + BNU_CHUNK_T* s3 = s2+groundElemLen; + BNU_CHUNK_T* s4 = s3+groundElemLen; + //tbcd: temporary excluded: assert(NULL!=s0); + + addF(s2, pA0, pA2, pGroundGFE); + subF(s2, s2, pA1, pGroundGFE); + sqrF(s2, s2, pGroundGFE); + sqrF(s0, pA0, pGroundGFE); + sqrF(s4, pA2, pGroundGFE); + mulF(s1, pA0, pA1, pGroundGFE); + mulF(s3, pA1, pA2, pGroundGFE); + addF(s1, s1, s1, pGroundGFE); + addF(s3, s3, s3, pGroundGFE); + + addF(pR2, s1, s2, pGroundGFE); + addF(pR2, pR2, s3, pGroundGFE); + subF(pR2, pR2, s0, pGroundGFE); + subF(pR2, pR2, s4, pGroundGFE); + + cpGFpxMul_G0(s4, s4, pGFEx); + subF(pR1, s1, s4, pGroundGFE); + + cpGFpxMul_G0(s3, s3, pGFEx); + subF(pR0, s0, s3, pGroundGFE); + + cpGFpReleasePool(5, pGroundGFE); + return pR; +} + + +/* +// return specific polynomi alarith methods +// polynomial - deg 3 binomial +*/ +static gsModMethod* gsPolyArith_binom3 (void) +{ + static gsModMethod m = { + cpGFpxEncode_com, + cpGFpxDecode_com, + cpGFpxMul_p3_binom, + cpGFpxSqr_p3_binom, + NULL, + cpGFpxAdd_com, + cpGFpxSub_com, + cpGFpxNeg_com, + cpGFpxDiv2_com, + cpGFpxMul2_com, + cpGFpxMul3_com, + //cpGFpxInv + }; + return &m; +} + +/*F* +// Name: ippsGFpxMethod_binom2 +// +// Purpose: Returns a reference to the implementation of arithmetic operations over GF(pd). +// +// Returns: pointer to a structure containing +// an implementation of arithmetic operations over GF(pd) +// g(x) = x^3 - a0, a0 from GF(p) +// +// +*F*/ +IPPFUN( const IppsGFpMethod*, ippsGFpxMethod_binom3, (void) ) +{ + static IppsGFpMethod method = { + cpID_Binom, + 3, + NULL, + NULL, + NULL + }; + method.arith = gsPolyArith_binom3(); + return &method; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_binom3_epid2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_binom3_epid2.c new file mode 100644 index 0000000..4252d85 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_binom3_epid2.c @@ -0,0 +1,254 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// GF(p^d) methods, if binomial generator over GF((p^2)^3) +// +*/ +#include "owncp.h" + +#include "pcpgfpxstuff.h" +#include "pcpgfpxmethod_com.h" +#include "pcpgfpxmethod_binom_epid2.h" + +//tbcd: temporary excluded: #include + +/* +// Intel(R) Enhanced Privacy ID (Intel(R) EPID) 2.0 specific. +// +// Intel(R) EPID 2.0 uses the following finite field hierarchy: +// +// 1) prime field GF(p), +// p = 0xFFFFFFFFFFFCF0CD46E5F25EEE71A49F0CDC65FB12980A82D3292DDBAED33013 +// +// 2) 2-degree extension of GF(p): GF(p^2) == GF(p)[x]/g(x), g(x) = x^2 -beta, +// beta =-1 mod p, so "beta" represents as {1} +// +// 3) 3-degree extension of GF(p^2) ~ GF(p^6): GF((p^2)^3) == GF(p)[v]/g(v), g(v) = v^3 -xi, +// xi belongs GF(p^2), xi=x+2, so "xi" represents as {2,1} ---- "2" is low- and "1" is high-order coefficients +// +// 4) 2-degree extension of GF((p^2)^3) ~ GF(p^12): GF(((p^2)^3)^2) == GF(p)[w]/g(w), g(w) = w^2 -vi, +// psi belongs GF((p^2)^3), vi=0*v^2 +1*v +0, so "vi" represents as {0,1,0}---- "0", '1" and "0" are low-, middle- and high-order coefficients +// +// See representations in t_gfpparam.cpp +// +*/ + +/* +// Intel(R) EPID 2.0 specific +// ~~~~~~~~~~~~~~~ +// +// Multiplication over GF((p^2)^3) +// - field polynomial: g(v) = v^3 - xi => binominal with specific value of "xi" +// - xi = x+2 +*/ +IPP_OWN_DEFN (static BNU_CHUNK_T*, cpGFpxMul_p3_binom_epid2, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFEx)) +{ + gsEngine* pGroundGFE = GFP_PARENT(pGFEx); + int groundElemLen = GFP_FELEN(pGroundGFE); + + mod_mul mulF = GFP_METHOD(pGroundGFE)->mul; + mod_add addF = GFP_METHOD(pGroundGFE)->add; + mod_sub subF = GFP_METHOD(pGroundGFE)->sub; + + const BNU_CHUNK_T* pA0 = pA; + const BNU_CHUNK_T* pA1 = pA+groundElemLen; + const BNU_CHUNK_T* pA2 = pA+groundElemLen*2; + + const BNU_CHUNK_T* pB0 = pB; + const BNU_CHUNK_T* pB1 = pB+groundElemLen; + const BNU_CHUNK_T* pB2 = pB+groundElemLen*2; + + BNU_CHUNK_T* pR0 = pR; + BNU_CHUNK_T* pR1 = pR+groundElemLen; + BNU_CHUNK_T* pR2 = pR+groundElemLen*2; + + BNU_CHUNK_T* t0 = cpGFpGetPool(6, pGroundGFE); + BNU_CHUNK_T* t1 = t0+groundElemLen; + BNU_CHUNK_T* t2 = t1+groundElemLen; + BNU_CHUNK_T* u0 = t2+groundElemLen; + BNU_CHUNK_T* u1 = u0+groundElemLen; + BNU_CHUNK_T* u2 = u1+groundElemLen; + //tbcd: temporary excluded: assert(NULL!=t0); + + addF(u0 ,pA0, pA1, pGroundGFE); /* u0 = a[0]+a[1] */ + addF(t0 ,pB0, pB1, pGroundGFE); /* t0 = b[0]+b[1] */ + mulF(u0, u0, t0, pGroundGFE); /* u0 = (a[0]+a[1])*(b[0]+b[1]) */ + mulF(t0, pA0, pB0, pGroundGFE); /* t0 = a[0]*b[0] */ + + addF(u1 ,pA1, pA2, pGroundGFE); /* u1 = a[1]+a[2] */ + addF(t1 ,pB1, pB2, pGroundGFE); /* t1 = b[1]+b[2] */ + mulF(u1, u1, t1, pGroundGFE); /* u1 = (a[1]+a[2])*(b[1]+b[2]) */ + mulF(t1, pA1, pB1, pGroundGFE); /* t1 = a[1]*b[1] */ + + addF(u2 ,pA2, pA0, pGroundGFE); /* u2 = a[2]+a[0] */ + addF(t2 ,pB2, pB0, pGroundGFE); /* t2 = b[2]+b[0] */ + mulF(u2, u2, t2, pGroundGFE); /* u2 = (a[2]+a[0])*(b[2]+b[0]) */ + mulF(t2, pA2, pB2, pGroundGFE); /* t2 = a[2]*b[2] */ + + subF(u0, u0, t0, pGroundGFE); /* u0 = a[0]*b[1]+a[1]*b[0] */ + subF(u0, u0, t1, pGroundGFE); + subF(u1, u1, t1, pGroundGFE); /* u1 = a[1]*b[2]+a[2]*b[1] */ + subF(u1, u1, t2, pGroundGFE); + subF(u2, u2, t2, pGroundGFE); /* u2 = a[2]*b[0]+a[0]*b[2] */ + subF(u2, u2, t0, pGroundGFE); + + /* Intel(R) EPID 2.0 specific */ + { + int basicExtDegree = cpGFpBasicDegreeExtension(pGFEx); + + /* deal with GF(p^2^3) */ + if(basicExtDegree==6) { + cpFq2Mul_xi(u1, u1, pGroundGFE); + cpFq2Mul_xi(t2, t2, pGroundGFE); + addF(pR0, t0, u1, pGroundGFE); /* r[0] = a[0]*b[0] - (a[2]*b[1]+a[1]*b[2])*beta */ + addF(pR1, u0, t2, pGroundGFE); /* r[1] = a[1]*b[0] + a[0]*b[1] - a[2]*b[2]*beta */ + } + /* just a case */ + else { + cpGFpxMul_G0(u1, u1, pGFEx); /* u1 = (a[1]*b[2]+a[2]*b[1]) * beta */ + cpGFpxMul_G0(t2, t2, pGFEx); /* t2 = a[2]*b[2] * beta */ + subF(pR0, t0, u1, pGroundGFE); /* r[0] = a[0]*b[0] - (a[2]*b[1]+a[1]*b[2])*beta */ + subF(pR1, u0, t2, pGroundGFE); /* r[1] = a[1]*b[0] + a[0]*b[1] - a[2]*b[2]*beta */ + } + } + + addF(pR2, u2, t1, pGroundGFE); /* r[2] = a[2]*b[0] + a[1]*b[1] + a[0]*b[2] */ + + cpGFpReleasePool(6, pGroundGFE); + return pR; +} + +/* +// Intel(R) EPID 2.0 specific +// ~~~~~~~~~~~~~~~ +// +// Squaring over GF((p^2)^3) +// - field polynomial: g(v) = v^3 - xi => binominal with specific value of "xi" +// - xi = x+2 +*/ +IPP_OWN_DEFN (static BNU_CHUNK_T*, cpGFpxSqr_p3_binom_epid2, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFEx)) +{ + gsEngine* pGroundGFE = GFP_PARENT(pGFEx); + int groundElemLen = GFP_FELEN(pGroundGFE); + + mod_mul mulF = GFP_METHOD(pGroundGFE)->mul; + mod_sqr sqrF = GFP_METHOD(pGroundGFE)->sqr; + mod_add addF = GFP_METHOD(pGroundGFE)->add; + mod_sub subF = GFP_METHOD(pGroundGFE)->sub; + + const BNU_CHUNK_T* pA0 = pA; + const BNU_CHUNK_T* pA1 = pA+groundElemLen; + const BNU_CHUNK_T* pA2 = pA+groundElemLen*2; + + BNU_CHUNK_T* pR0 = pR; + BNU_CHUNK_T* pR1 = pR+groundElemLen; + BNU_CHUNK_T* pR2 = pR+groundElemLen*2; + + BNU_CHUNK_T* s0 = cpGFpGetPool(5, pGroundGFE); + BNU_CHUNK_T* s1 = s0+groundElemLen; + BNU_CHUNK_T* s2 = s1+groundElemLen; + BNU_CHUNK_T* s3 = s2+groundElemLen; + BNU_CHUNK_T* s4 = s3+groundElemLen; + + addF(s2, pA0, pA2, pGroundGFE); + subF(s2, s2, pA1, pGroundGFE); + sqrF(s2, s2, pGroundGFE); + sqrF(s0, pA0, pGroundGFE); + sqrF(s4, pA2, pGroundGFE); + mulF(s1, pA0, pA1, pGroundGFE); + mulF(s3, pA1, pA2, pGroundGFE); + addF(s1, s1, s1, pGroundGFE); + addF(s3, s3, s3, pGroundGFE); + + addF(pR2, s1, s2, pGroundGFE); + addF(pR2, pR2, s3, pGroundGFE); + subF(pR2, pR2, s0, pGroundGFE); + subF(pR2, pR2, s4, pGroundGFE); + + /* Intel(R) EPID 2.0 specific */ + { + int basicExtDegree = cpGFpBasicDegreeExtension(pGFEx); + + /* deal with GF(p^2^3) */ + if(basicExtDegree==6) { + cpFq2Mul_xi(s4, s4, pGroundGFE); + cpFq2Mul_xi(s3, s3, pGroundGFE); + addF(pR1, s1, s4, pGroundGFE); + addF(pR0, s0, s3, pGroundGFE); + } + /* just a case */ + else { + cpGFpxMul_G0(s4, s4, pGFEx); + cpGFpxMul_G0(s3, s3, pGFEx); + subF(pR1, s1, s4, pGroundGFE); + subF(pR0, s0, s3, pGroundGFE); + } + } + + cpGFpReleasePool(5, pGroundGFE); + return pR; +} + +/* +// return specific polynomi alarith methods +// polynomial - deg 3 binomial (Intel(R) EPID 2.0) +*/ +static gsModMethod* gsPolyArith_binom3_epid2 (void) +{ + static gsModMethod m = { + cpGFpxEncode_com, + cpGFpxDecode_com, + cpGFpxMul_p3_binom_epid2, + cpGFpxSqr_p3_binom_epid2, + NULL, + cpGFpxAdd_com, + cpGFpxSub_com, + cpGFpxNeg_com, + cpGFpxDiv2_com, + cpGFpxMul2_com, + cpGFpxMul3_com, + //cpGFpxInv + }; + return &m; +} + +/*F* +// Name: ippsGFpxMethod_binom3_epid2 +// +// Purpose: Returns a reference to the implementation of arithmetic operations over GF(pd). +// +// Returns: pointer to a structure containing +// an implementation of arithmetic operations over GF(pd) +// g(v) = v^3 - U0, U0 from GF(q^2), U0 = u + 2 +// +// +*F*/ + +IPPFUN( const IppsGFpMethod*, ippsGFpxMethod_binom3_epid2, (void) ) +{ + static IppsGFpMethod method = { + cpID_Binom3_epid20, + 3, + NULL, + NULL, + NULL + }; + method.arith = gsPolyArith_binom3_epid2(); + return &method; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_binom_epid2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_binom_epid2.c new file mode 100644 index 0000000..030bfb9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_binom_epid2.c @@ -0,0 +1,274 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// GF(p^d) methods, if binomial generator +// +*/ +#include "owncp.h" + +#include "pcpgfpxstuff.h" +#include "pcpgfpxmethod_com.h" +#include "pcpgfpxmethod_binom_epid2.h" + +//tbcd: temporary excluded: #include + +/* +// Intel(R) Enhanced Privacy ID (Intel(R) EPID) 2.0 specific. +// +// Intel(R) EPID 2.0 uses the following finite field hierarchy: +// +// 1) prime field GF(p), +// p = 0xFFFFFFFFFFFCF0CD46E5F25EEE71A49F0CDC65FB12980A82D3292DDBAED33013 +// +// 2) 2-degree extension of GF(p): GF(p^2) == GF(p)[x]/g(x), g(x) = x^2 -beta, +// beta =-1 mod p, so "beta" represents as {1} +// +// 3) 3-degree extension of GF(p^2) ~ GF(p^6): GF((p^2)^3) == GF(p)[v]/g(v), g(v) = v^3 -xi, +// xi belongs GF(p^2), xi=x+2, so "xi" represents as {2,1} ---- "2" is low- and "1" is high-order coefficients +// +// 4) 2-degree extension of GF((p^2)^3) ~ GF(p^12): GF(((p^2)^3)^2) == GF(p)[w]/g(w), g(w) = w^2 -vi, +// psi belongs GF((p^2)^3), vi=0*v^2 +1*v +0, so "vi" represents as {0,1,0}---- "0", '1" and "0" are low-, middle- and high-order coefficients +// +// See representations in t_gfpparam.cpp +// +*/ + +/* +// Multiplication case: mul(a, vi) over GF((p^2)^3), +// where: +// a, belongs to GF((p^2)^3) +// xi belongs to GF((p^2)^3), vi={0,1,0} +// +// The case is important in GF(((p^2)^3)^2) arithmetic for Intel(R) EPID 2.0. +// +*/ +__INLINE BNU_CHUNK_T* cpFq6Mul_vi(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFEx) +{ + gsEngine* pGroundGFE = GFP_PARENT(pGFEx); + int termLen = GFP_FELEN(pGroundGFE); + + const BNU_CHUNK_T* pA0 = pA; + const BNU_CHUNK_T* pA1 = pA+termLen; + const BNU_CHUNK_T* pA2 = pA+termLen*2; + BNU_CHUNK_T* pR0 = pR; + BNU_CHUNK_T* pR1 = pR+termLen; + BNU_CHUNK_T* pR2 = pR+termLen*2; + + BNU_CHUNK_T* t = cpGFpGetPool(1, pGroundGFE); + //tbcd: temporary excluded: assert(NULL!=t); + + cpFq2Mul_xi(t, pA2, pGroundGFE); + cpGFpElementCopy(pR2, pA1, termLen); + cpGFpElementCopy(pR1, pA0, termLen); + cpGFpElementCopy(pR0, t, termLen); + + cpGFpReleasePool(1, pGroundGFE); + + return pR; +} + +/* +// Intel(R) EPID 2.0 specific +// ~~~~~~~~~~~~~~~ +// +// Multiplication over GF(p^2) +// - field polynomial: g(x) = x^2 - beta => binominal with specific value of "beta" +// - beta = p-1 +// +// Multiplication over GF(((p^2)^3)^2) ~ GF(p^12) +// - field polynomial: g(w) = w^2 - vi => binominal with specific value of "vi" +// - vi = 0*v^2 + 1*v + 0 - i.e vi={0,1,0} belongs to GF((p^2)^3) +*/ +IPP_OWN_DEFN (static BNU_CHUNK_T*, cpGFpxMul_p2_binom_epid2, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFEx)) +{ + gsEngine* pGroundGFE = GFP_PARENT(pGFEx); + mod_mul mulF = GFP_METHOD(pGroundGFE)->mul; + mod_add addF = GFP_METHOD(pGroundGFE)->add; + mod_sub subF = GFP_METHOD(pGroundGFE)->sub; + + int groundElemLen = GFP_FELEN(pGroundGFE); + + const BNU_CHUNK_T* pA0 = pA; + const BNU_CHUNK_T* pA1 = pA+groundElemLen; + + const BNU_CHUNK_T* pB0 = pB; + const BNU_CHUNK_T* pB1 = pB+groundElemLen; + + BNU_CHUNK_T* pR0 = pR; + BNU_CHUNK_T* pR1 = pR+groundElemLen; + + BNU_CHUNK_T* t0 = cpGFpGetPool(4, pGroundGFE); + BNU_CHUNK_T* t1 = t0+groundElemLen; + BNU_CHUNK_T* t2 = t1+groundElemLen; + BNU_CHUNK_T* t3 = t2+groundElemLen; + //tbcd: temporary excluded: assert(NULL!=t0); + + mulF(t0, pA0, pB0, pGroundGFE); /* t0 = a[0]*b[0] */ + mulF(t1, pA1, pB1, pGroundGFE); /* t1 = a[1]*b[1] */ + addF(t2, pA0, pA1, pGroundGFE); /* t2 = a[0]+a[1] */ + addF(t3, pB0, pB1, pGroundGFE); /* t3 = b[0]+b[1] */ + + mulF(pR1, t2, t3, pGroundGFE); /* r[1] = (a[0]+a[1]) * (b[0]+b[1]) */ + subF(pR1, pR1, t0, pGroundGFE); /* r[1] -= a[0]*b[0]) + a[1]*b[1] */ + subF(pR1, pR1, t1, pGroundGFE); + + /* Intel(R) EPID 2.0 specific */ + { + int basicExtDegree = cpGFpBasicDegreeExtension(pGFEx); + + /* deal with GF(p^2) */ + if(basicExtDegree==2) { + subF(pR0, t0, t1, pGroundGFE); + } + /* deal with GF(p^6^2) */ + else if(basicExtDegree==12) { + cpFq6Mul_vi(t1, t1, pGroundGFE); + addF(pR0, t0, t1, pGroundGFE); + } + /* deal with GF(p^x^2) - it's not Intel(R) EPID 2.0 case, just a case */ + else { + cpGFpxMul_G0(t1, t1, pGFEx); + subF(pR0, t0, t1, pGroundGFE); + } + } + + cpGFpReleasePool(4, pGroundGFE); + return pR; +} + +/* +// Intel(R) EPID 2.0 specific +// ~~~~~~~~~~~~~~~ +// +// Squaring over GF(p^2) +// - field polynomial: g(x) = x^2 - beta => binominal with specific value of "beta" +// - beta = p-1 +// +// Squaring in GF(((p^2)^3)^2) ~ GF(p^12) +// - field polynomial: g(w) = w^2 - vi => binominal with specific value of "vi" +// - vi = 0*v^2 + 1*v + 0 - i.e vi={0,1,0} belongs to GF((p^2)^3) +*/ +IPP_OWN_DEFN (static BNU_CHUNK_T*, cpGFpxSqr_p2_binom_epid2, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFEx)) +{ + gsEngine* pGroundGFE = GFP_PARENT(pGFEx); + mod_mul mulF = GFP_METHOD(pGroundGFE)->mul; + mod_sqr sqrF = GFP_METHOD(pGroundGFE)->sqr; + mod_add addF = GFP_METHOD(pGroundGFE)->add; + mod_sub subF = GFP_METHOD(pGroundGFE)->sub; + + int groundElemLen = GFP_FELEN(pGroundGFE); + + const BNU_CHUNK_T* pA0 = pA; + const BNU_CHUNK_T* pA1 = pA+groundElemLen; + + BNU_CHUNK_T* pR0 = pR; + BNU_CHUNK_T* pR1 = pR+groundElemLen; + + BNU_CHUNK_T* t0 = cpGFpGetPool(3, pGroundGFE); + BNU_CHUNK_T* t1 = t0+groundElemLen; + BNU_CHUNK_T* u0 = t1+groundElemLen; + //tbcd: temporary excluded: assert(NULL!=t0); + + mulF(u0, pA0, pA1, pGroundGFE); /* u0 = a[0]*a[1] */ + + /* Intel(R) EPID 2.0 specific */ + { + int basicExtDegree = cpGFpBasicDegreeExtension(pGFEx); + + /* deal with GF(p^2) */ + if(basicExtDegree==2) { + addF(t0, pA0, pA1, pGroundGFE); + subF(t1, pA0, pA1, pGroundGFE); + mulF(pR0, t0, t1, pGroundGFE); + addF(pR1, u0, u0, pGroundGFE); /* r[1] = 2*a[0]*a[1] */ + } + /* deal with GF(p^6^2) */ + else if(basicExtDegree==12) { + subF(t0, pA0, pA1, pGroundGFE); + cpFq6Mul_vi(t1, pA1, pGroundGFE); + subF(t1, pA0, t1, pGroundGFE); + mulF(t0, t0, t1, pGroundGFE); + addF(t0, t0, u0, pGroundGFE); + cpFq6Mul_vi(t1, u0, pGroundGFE); + addF(pR0, t0, t1, pGroundGFE); + addF(pR1, u0, u0, pGroundGFE); + } + /* just a case */ + else { + sqrF(t0, pA0, pGroundGFE); /* t0 = a[0]*a[0] */ + sqrF(t1, pA1, pGroundGFE); /* t1 = a[1]*a[1] */ + cpGFpxMul_G0(t1, t1, pGFEx); + subF(pR0, t0, t1, pGroundGFE); + addF(pR1, u0, u0, pGroundGFE); /* r[1] = 2*a[0]*a[1] */ + } + } + + cpGFpReleasePool(3, pGroundGFE); + return pR; +} + +/* +// return specific polynomi alarith methods +// polynomial - deg 2 binomial (Intel(R) EPID 2.0) +*/ +static gsModMethod* gsPolyArith_binom2_epid2 (void) +{ + static gsModMethod m = { + cpGFpxEncode_com, + cpGFpxDecode_com, + cpGFpxMul_p2_binom_epid2, + cpGFpxSqr_p2_binom_epid2, + NULL, + cpGFpxAdd_com, + cpGFpxSub_com, + cpGFpxNeg_com, + cpGFpxDiv2_com, + cpGFpxMul2_com, + cpGFpxMul3_com, + //cpGFpxInv + }; + return &m; +} + +/*F* +// Name: ippsGFpxMethod_binom2_epid2 +// +// Purpose: Returns a reference to the implementation of arithmetic operations over GF(pd). +// +// Returns: pointer to a structure containing +// an implementation of arithmetic operations over GF(pd) +// g(x) = x^2 - a0, a0 from GF(q), a0 = 1 +// g(w) = w^2 - V0, v0 from GF((q^2)^3), V0 = 0*s^2 + v + 0 +// +// +*F*/ + +IPPFUN( const IppsGFpMethod*, ippsGFpxMethod_binom2_epid2, (void) ) +{ + static IppsGFpMethod method = { + cpID_Binom2_epid20, + 2, + NULL, + NULL, + NULL + }; + method.arith = gsPolyArith_binom2_epid2(); + return &method; +} + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_binom_epid2.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_binom_epid2.h new file mode 100644 index 0000000..f7e4a00 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_binom_epid2.h @@ -0,0 +1,99 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// internal functions for GF(p^d) methods, if binomial generator +// with Intel(R) Enhanced Privacy ID (Intel(R) EPID) 2.0 specific +// +*/ +#include "owncp.h" + +#include "pcpgfpxstuff.h" +#include "pcpgfpxmethod_com.h" + +//tbcd: temporary excluded: #include + +/* +// Intel(R) EPID 2.0 specific. +// +// Intel(R) EPID 2.0 uses the following finite field hierarchy: +// +// 1) prime field GF(p), +// p = 0xFFFFFFFFFFFCF0CD46E5F25EEE71A49F0CDC65FB12980A82D3292DDBAED33013 +// +// 2) 2-degree extension of GF(p): GF(p^2) == GF(p)[x]/g(x), g(x) = x^2 -beta, +// beta =-1 mod p, so "beta" represents as {1} +// +// 3) 3-degree extension of GF(p^2) ~ GF(p^6): GF((p^2)^3) == GF(p)[v]/g(v), g(v) = v^3 -xi, +// xi belongs GF(p^2), xi=x+2, so "xi" represents as {2,1} ---- "2" is low- and "1" is high-order coefficients +// +// 4) 2-degree extension of GF((p^2)^3) ~ GF(p^12): GF(((p^2)^3)^2) == GF(p)[w]/g(w), g(w) = w^2 -vi, +// psi belongs GF((p^2)^3), vi=0*v^2 +1*v +0, so "vi" represents as {0,1,0}---- "0", '1" and "0" are low-, middle- and high-order coefficients +// +// See representations in t_gfpparam.cpp +// +*/ + +/* +// Multiplication case: mul(a, xi) over GF(p^2), +// where: +// a, belongs to GF(p^2) +// xi belongs to GF(p^2), xi={2,1} +// +// The case is important in GF((p^2)^3) arithmetic for Intel(R) EPID 2.0. +// +*/ +__INLINE BNU_CHUNK_T* cpFq2Mul_xi(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFEx) +{ + gsEngine* pGroundGFE = GFP_PARENT(pGFEx); + mod_mul addF = GFP_METHOD(pGroundGFE)->add; + mod_sub subF = GFP_METHOD(pGroundGFE)->sub; + + int termLen = GFP_FELEN(pGroundGFE); + BNU_CHUNK_T* t0 = cpGFpGetPool(2, pGroundGFE); + BNU_CHUNK_T* t1 = t0+termLen; + + const BNU_CHUNK_T* pA0 = pA; + const BNU_CHUNK_T* pA1 = pA+termLen; + BNU_CHUNK_T* pR0 = pR; + BNU_CHUNK_T* pR1 = pR+termLen; + + //tbcd: temporary excluded: assert(NULL!=t0); + addF(t0, pA0, pA0, pGroundGFE); + addF(t1, pA0, pA1, pGroundGFE); + subF(pR0, t0, pA1, pGroundGFE); + addF(pR1, t1, pA1, pGroundGFE); + + cpGFpReleasePool(2, pGroundGFE); + return pR; +} + +/* +// Multiplication case: mul(a, g0) over GF(()), +// where: +// a and g0 belongs to GF(()) - field is being extension +// +// The case is important in GF(()^d) arithmetic if constructed polynomial is generic binomial g(t) = t^d +g0. +// +*/ +static BNU_CHUNK_T* cpGFpxMul_G0(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFEx) +{ + gsEngine* pGroundGFE = GFP_PARENT(pGFEx); + BNU_CHUNK_T* pGFpolynomial = GFP_MODULUS(pGFEx); /* g(x) = t^d + g0 */ + return GFP_METHOD(pGroundGFE)->mul(pR, pA, pGFpolynomial, pGroundGFE); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_binom_mulc.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_binom_mulc.h new file mode 100644 index 0000000..61043a2 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_binom_mulc.h @@ -0,0 +1,57 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// GF(p^d) methods, if binomial generator +// +*/ +#if !defined(_CP_GFP_METHOD_BINOM_H) +#define _CP_GFP_METHOD_BINOM_H + +#include "owncp.h" +#include "pcpgfpxstuff.h" + + +static BNU_CHUNK_T* cpGFpxMul_G0(BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFEx) +{ + gsEngine* pGroundGFE = GFP_PARENT(pGFEx); + mod_mul mulF = GFP_METHOD(pGroundGFE)->mul; + + BNU_CHUNK_T* pGFpolynomial = GFP_MODULUS(pGFEx); /* g(x) = t^d + g0 */ + + #if defined GS_DBG + BNU_CHUNK_T* arg0 = cpGFpGetPool(1, pGroundGFE); + BNU_CHUNK_T* arg1 = cpGFpGetPool(1, pGroundGFE); + int groundElemLen = GFP_FELEN(pGroundGFE); + #endif + + #if defined GS_DBG + cpGFpxGet(arg0, groundElemLen, pA, pGroundGFE); + cpGFpxGet(arg1, groundElemLen, pGFpolynomial, pGroundGFE); + #endif + + mulF(pR, pA, pGFpolynomial, pGroundGFE); + + #if defined GS_DBG + cpGFpReleasePool(2, pGroundGFE); + #endif + + return pR; +} + +#endif /* _CP_GFP_METHOD_BINOM_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_com.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_com.c new file mode 100644 index 0000000..c6eda7b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_com.c @@ -0,0 +1,74 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// GF(p^d) methods +// +*/ +#include "owncp.h" + +#include "pcpgfpxstuff.h" +#include "pcpgfpxmethod_com.h" + + +/* +// return common polynomi alarith methods +*/ +static gsModMethod* gsPolyArith(void) +{ + static gsModMethod m = { + cpGFpxEncode_com, + cpGFpxDecode_com, + cpGFpxMul_com, + cpGFpxSqr_com, + NULL, + cpGFpxAdd_com, + cpGFpxSub_com, + cpGFpxNeg_com, + cpGFpxDiv2_com, + cpGFpxMul2_com, + cpGFpxMul3_com, + //cpGFpxInv + }; + return &m; +} + +/*F* +// Name: ippsGFpxMethod_binom2 +// +// Purpose: Returns a reference to the implementation of arithmetic operations over GF(pd). +// +// Returns: pointer to a structure containing +// an implementation of arithmetic operations over GF(pd) +// g(x) = x^d + x(d - 1) a_(d-1) + x^(d - 2)a_(d - 2) + ... + x1a1 + a0, ai from GF(p) +// +// +*F*/ + +IPPFUN( const IppsGFpMethod*, ippsGFpxMethod_com, (void) ) +{ + static IppsGFpMethod method = { + cpID_Poly, + 0, + NULL, + NULL, + NULL + }; + method.arith = gsPolyArith(); + return &method; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_com.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_com.h new file mode 100644 index 0000000..5bb03ab --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_com.h @@ -0,0 +1,53 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// GF(p^d) methods +// +*/ +#if !defined(_CP_GFP_METHOD_COM_H) +#define _CP_GFP_METHOD_COM_H + +#include "owncp.h" +#include "pcpgfpstuff.h" + +#define cpGFpxAdd_com OWNAPI(cpGFpxAdd_com) + IPP_OWN_DECL (BNU_CHUNK_T*, cpGFpxAdd_com, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFEx)) +#define cpGFpxSub_com OWNAPI(cpGFpxSub_com) + IPP_OWN_DECL (BNU_CHUNK_T*, cpGFpxSub_com, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFEx)) +#define cpGFpxNeg_com OWNAPI(cpGFpxNeg_com) + IPP_OWN_DECL (BNU_CHUNK_T*, cpGFpxNeg_com, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFEx)) + +#define cpGFpxMul_com OWNAPI(cpGFpxMul_com) + IPP_OWN_DECL (BNU_CHUNK_T*, cpGFpxMul_com, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFEx)) +#define cpGFpxSqr_com OWNAPI(cpGFpxSqr_com) + IPP_OWN_DECL (BNU_CHUNK_T*, cpGFpxSqr_com, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFEx)) + +#define cpGFpxDiv2_com OWNAPI(cpGFpxDiv2_com) + IPP_OWN_DECL (BNU_CHUNK_T*, cpGFpxDiv2_com, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFEx)) +#define cpGFpxMul2_com OWNAPI(cpGFpxMul2_com) + IPP_OWN_DECL (BNU_CHUNK_T*, cpGFpxMul2_com, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFEx)) +#define cpGFpxMul3_com OWNAPI(cpGFpxMul3_com) + IPP_OWN_DECL (BNU_CHUNK_T*, cpGFpxMul3_com, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFEx)) + +#define cpGFpxEncode_com OWNAPI(cpGFpxEncode_com) + IPP_OWN_DECL (BNU_CHUNK_T*, cpGFpxEncode_com, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFEx)) +#define cpGFpxDecode_com OWNAPI(cpGFpxDecode_com) + IPP_OWN_DECL (BNU_CHUNK_T*, cpGFpxDecode_com, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFEx)) + +#endif /* _CP_GFP_METHOD_COM_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_com_add.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_com_add.c new file mode 100644 index 0000000..3f51a7e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_com_add.c @@ -0,0 +1,45 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// GF(p^d) methods +// +*/ +#include "owncp.h" + +#include "pcpgfpxstuff.h" +#include "pcpgfpxmethod_com.h" + + +IPP_OWN_DEFN (BNU_CHUNK_T*, cpGFpxAdd_com, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFEx)) +{ + gsEngine* pBasicGFE = cpGFpBasic(pGFEx); + mod_add addF = GFP_METHOD(pBasicGFE)->add; + int basicElemLen = GFP_FELEN(pBasicGFE); + int basicDeg = cpGFpBasicDegreeExtension(pGFEx); + + BNU_CHUNK_T* pTmp = pR; + int deg; + for(deg=0; degdecode; + int basicElemLen = GFP_FELEN(pBasicGFE); + int basicDeg = cpGFpBasicDegreeExtension(pGFEx); + + BNU_CHUNK_T* pTmp = pR; + int deg; + for(deg=0; degdiv2; + int basicElemLen = GFP_FELEN(pBasicGFE); + int basicDeg = cpGFpBasicDegreeExtension(pGFEx); + + BNU_CHUNK_T* pTmp = pR; + int deg; + for(deg=0; degencode; + int basicElemLen = GFP_FELEN(pBasicGFE); + int basicDeg = cpGFpBasicDegreeExtension(pGFEx); + + BNU_CHUNK_T* pTmp = pR; + int deg; + for(deg=0; deg + +IPP_OWN_DEFN (BNU_CHUNK_T*, cpGFpxMul_com, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFEx)) +{ + int extDegree = GFP_EXTDEGREE(pGFEx); + + BNU_CHUNK_T* pGFpolynomial = GFP_MODULUS(pGFEx); + int degR = extDegree-1; + int elemLen= GFP_FELEN(pGFEx); + + int degB = degR; + BNU_CHUNK_T* pTmpProduct = cpGFpGetPool(2, pGFEx); + BNU_CHUNK_T* pTmpResult = pTmpProduct + GFP_PELEN(pGFEx); + + gsEngine* pGroundGFE = GFP_PARENT(pGFEx); + BNU_CHUNK_T* r = cpGFpGetPool(1, pGroundGFE); + int groundElemLen = GFP_FELEN(pGroundGFE); + + const BNU_CHUNK_T* pTmpB = GFPX_IDX_ELEMENT(pB, degB, groundElemLen); + + //tbcd: temporary excluded: assert(NULL!=pTmpProduct && NULL!=r); + + /* clear temporary */ + cpGFpElementPad(pTmpProduct, elemLen, 0); + + /* R = A * B[degB-1] */ + cpGFpxMul_GFE(pTmpResult, pA, pTmpB, pGFEx); + + for(degB-=1; degB>=0; degB--) { + /* save R[degR-1] */ + cpGFpElementCopy(r, GFPX_IDX_ELEMENT(pTmpResult, degR, groundElemLen), groundElemLen); + + { /* R = R * x */ + int j; + for (j=degR; j>=1; j--) + cpGFpElementCopy(GFPX_IDX_ELEMENT(pTmpResult, j, groundElemLen), GFPX_IDX_ELEMENT(pTmpResult, j-1, groundElemLen), groundElemLen); + cpGFpElementPad(pTmpResult, groundElemLen, 0); + } + + cpGFpxMul_GFE(pTmpProduct, pGFpolynomial, r, pGFEx); + GFP_METHOD(pGFEx)->sub(pTmpResult, pTmpResult, pTmpProduct, pGFEx); + + /* B[degB-i] */ + pTmpB -= groundElemLen; + cpGFpxMul_GFE(pTmpProduct, pA, pTmpB, pGFEx); + GFP_METHOD(pGFEx)->add(pTmpResult, pTmpResult, pTmpProduct, pGFEx); + } + + /* copy result */ + cpGFpElementCopy(pR, pTmpResult, elemLen); + + /* release pools */ + cpGFpReleasePool(1, pGroundGFE); + cpGFpReleasePool(2, pGFEx); + + return pR; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_com_mul2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_com_mul2.c new file mode 100644 index 0000000..465389a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_com_mul2.c @@ -0,0 +1,44 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// GF(p^d) methods +// +*/ +#include "owncp.h" + +#include "pcpgfpxstuff.h" +#include "pcpgfpxmethod_com.h" + + +IPP_OWN_DEFN (BNU_CHUNK_T*, cpGFpxMul2_com, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFEx)) +{ + gsEngine* pBasicGFE = cpGFpBasic(pGFEx); + mod_mul2 mul2F = GFP_METHOD(pBasicGFE)->mul2; + int basicElemLen = GFP_FELEN(pBasicGFE); + int basicDeg = cpGFpBasicDegreeExtension(pGFEx); + + BNU_CHUNK_T* pTmp = pR; + int deg; + for(deg=0; degmul3; + int basicElemLen = GFP_FELEN(pBasicGFE); + int basicDeg = cpGFpBasicDegreeExtension(pGFEx); + + BNU_CHUNK_T* pTmp = pR; + int deg; + for(deg=0; degneg; + int basicElemLen = GFP_FELEN(pBasicGFE); + int basicDeg = cpGFpBasicDegreeExtension(pGFEx); + + BNU_CHUNK_T* pTmp = pR; + int deg; + for(deg=0; deg + +IPP_OWN_DEFN (BNU_CHUNK_T*, cpGFpxSqr_com, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, gsEngine* pGFEx)) +{ + int extDegree = GFP_EXTDEGREE(pGFEx); + + BNU_CHUNK_T* pGFpolynomial = GFP_MODULUS(pGFEx); + int degR = extDegree-1; + int elemLen= GFP_FELEN(pGFEx); + + int degA = degR; + BNU_CHUNK_T* pTmpProduct = cpGFpGetPool(2, pGFEx); + BNU_CHUNK_T* pTmpResult = pTmpProduct + GFP_PELEN(pGFEx); + + gsEngine* pGroundGFE = GFP_PARENT(pGFEx); + BNU_CHUNK_T* r = cpGFpGetPool(1, pGroundGFE); + int groundElemLen = GFP_FELEN(pGroundGFE); + + const BNU_CHUNK_T* pTmpA = GFPX_IDX_ELEMENT(pA, degA, groundElemLen); + + //tbcd: temporary excluded: assert(NULL!=pTmpProduct && NULL!=r); + + /* clear temporary */ + cpGFpElementPad(pTmpProduct, elemLen, 0); + + /* R = A * A[degA-1] */ + cpGFpxMul_GFE(pTmpResult, pA, pTmpA, pGFEx); + + for(degA-=1; degA>=0; degA--) { + /* save R[degR-1] */ + cpGFpElementCopy(r, GFPX_IDX_ELEMENT(pTmpResult, degR, groundElemLen), groundElemLen); + + { /* R = R * x */ + int j; + for (j=degR; j>=1; j--) + cpGFpElementCopy(GFPX_IDX_ELEMENT(pTmpResult, j, groundElemLen), GFPX_IDX_ELEMENT(pTmpResult, j-1, groundElemLen), groundElemLen); + cpGFpElementPad(pTmpResult, groundElemLen, 0); + } + + cpGFpxMul_GFE(pTmpProduct, pGFpolynomial, r, pGFEx); + GFP_METHOD(pGFEx)->sub(pTmpResult, pTmpResult, pTmpProduct, pGFEx); + + /* A[degA-i] */ + pTmpA -= groundElemLen; + cpGFpxMul_GFE(pTmpProduct, pA, pTmpA, pGFEx); + GFP_METHOD(pGFEx)->add(pTmpResult, pTmpResult, pTmpProduct, pGFEx); + } + + /* copy result */ + cpGFpElementCopy(pR, pTmpResult, elemLen); + + /* release pools */ + cpGFpReleasePool(1, pGroundGFE); + cpGFpReleasePool(2, pGFEx); + + return pR; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_com_sub.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_com_sub.c new file mode 100644 index 0000000..ff50b67 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxmethod_com_sub.c @@ -0,0 +1,45 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// GF(p^d) methods +// +*/ +#include "owncp.h" + +#include "pcpgfpxstuff.h" +#include "pcpgfpxmethod_com.h" + + +IPP_OWN_DEFN (BNU_CHUNK_T*, cpGFpxSub_com, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, const BNU_CHUNK_T* pB, gsEngine* pGFEx)) +{ + gsEngine* pBasicGFE = cpGFpBasic(pGFEx); + mod_sub subF = GFP_METHOD(pBasicGFE)->sub; + int basicElemLen = GFP_FELEN(pBasicGFE); + int basicDeg = cpGFpBasicDegreeExtension(pGFEx); + + BNU_CHUNK_T* pTmp = pR; + int deg; + for(deg=0; deg=n_opt) break; + w_opt = w_trial; + n_opt = n_trial; + } + return w_opt; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxstuff.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxstuff.h new file mode 100644 index 0000000..ca03fc9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpgfpxstuff.h @@ -0,0 +1,144 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippCP) +// GF(p) extension internal +// +*/ + +#if !defined(_PCP_GFPEXT_H_) +#define _PCP_GFPEXT_H_ + +#include "pcpgfpstuff.h" + + +/* GF(p^d) pool */ +#define GFPX_PESIZE(pGF) GFP_FELEN((pGF)) +#define GFPX_POOL_SIZE (14) //(8) /* Number of temporary variables in pool */ + +/* address of ground field element inside expanded field element */ +#define GFPX_IDX_ELEMENT(pxe, idx, eleSize) ((pxe)+(eleSize)*(idx)) + + +__INLINE int degree(const BNU_CHUNK_T* pE, const gsModEngine* pGFEx) +{ + int groundElemLen = GFP_FELEN(GFP_PARENT(pGFEx)); + int deg; + for(deg=GFP_EXTDEGREE(pGFEx)-1; deg>=0; deg--) { + if(!GFP_IS_ZERO(pE+groundElemLen*deg, groundElemLen)) break; + } + return deg; +} + +__INLINE gsModEngine* cpGFpBasic(const gsModEngine* pGFEx) +{ + while( !GFP_IS_BASIC(pGFEx) ) { + pGFEx = GFP_PARENT(pGFEx); + } + return (gsModEngine*)pGFEx; +} +__INLINE int cpGFpBasicDegreeExtension(const gsModEngine* pGFEx) +{ + int degree = GFP_EXTDEGREE(pGFEx); + while( !GFP_IS_BASIC(pGFEx) ) { + pGFEx = GFP_PARENT(pGFEx); + degree *= GFP_EXTDEGREE(pGFEx); + } + return degree; +} + +/* convert external data (Ipp32u) => internal element (BNU_CHUNK_T) representation + returns length of element (in BNU_CHUNK_T) +*/ +__INLINE int cpGFpxCopyToChunk(BNU_CHUNK_T* pElm, const Ipp32u* pA, int nsA, const gsModEngine* pGFEx) +{ + gsModEngine* pBasicGFE = cpGFpBasic(pGFEx); + int basicExtension = cpGFpBasicDegreeExtension(pGFEx); + int basicElmLen32 = GFP_FELEN32(pBasicGFE); + int basicElmLen = GFP_FELEN(pBasicGFE); + int deg; + for(deg=0; deg0; deg++, nsA -= basicElmLen32) { + int srcLen = IPP_MIN(nsA, basicElmLen32); + ZEXPAND_COPY_BNU((Ipp32u*)pElm, basicElmLen*(int)(sizeof(BNU_CHUNK_T)/sizeof(Ipp32u)), pA,srcLen); + pElm += basicElmLen; + pA += basicElmLen32; + } + return basicElmLen*deg; +} + +/* convert internal element (BNU_CHUNK_T) => external data (Ipp32u) representation + returns length of data (in Ipp32u) +*/ +__INLINE int cpGFpxCopyFromChunk(Ipp32u* pA, const BNU_CHUNK_T* pElm, const gsModEngine* pGFEx) +{ + gsModEngine* pBasicGFE = cpGFpBasic(pGFEx); + int basicExtension = cpGFpBasicDegreeExtension(pGFEx); + int basicElmLen32 = GFP_FELEN32(pBasicGFE); + int basicElmLen = GFP_FELEN(pBasicGFE); + int deg; + for(deg=0; degidCtx = (Ipp32u)ctxid ^ (Ipp32u)IPP_UINT_PTR(stt)) +#define HASH_RESET_ID(stt,ctxid) ((stt)->idCtx = (Ipp32u)ctxid) +#define HASH_ALG_ID(stt) ((stt)->algID) +#define HASH_LENLO(stt) ((stt)->msgLenLo) +#define HASH_LENHI(stt) ((stt)->msgLenHi) +#define HASH_FUNC(stt) ((stt)->hashProc) +#define HASH_FUNC_PAR(stt) ((stt)->pParam) +#define HASH_VALUE(stt) ((stt)->msgHash) +#define HASH_BUFFIDX(stt) ((stt)->msgBuffIdx) +#define HASH_BUFF(stt) ((stt)->msgBuffer) +#define HASH_VALID_ID(stt,ctxId) ((((stt)->idCtx) ^ (Ipp32u)IPP_UINT_PTR((stt))) == (Ipp32u)ctxId) + + +/* initial hash values */ +extern const Ipp32u SHA1_IV[]; +extern const Ipp32u SHA256_IV[]; +extern const Ipp32u SHA224_IV[]; +extern const Ipp64u SHA512_IV[]; +extern const Ipp64u SHA384_IV[]; +extern const Ipp32u MD5_IV[]; +extern const Ipp32u SM3_IV[]; +extern const Ipp64u SHA512_224_IV[]; +extern const Ipp64u SHA512_256_IV[]; + +/* hash alg additive constants */ +extern __ALIGN16 const Ipp32u SHA1_cnt[]; +extern __ALIGN16 const Ipp32u SHA256_cnt[]; +extern __ALIGN16 const Ipp64u SHA512_cnt[]; +extern __ALIGN16 const Ipp32u MD5_cnt[]; +extern __ALIGN16 const Ipp32u SM3_cnt[]; + +/* hash alg opt argument */ +extern const void* cpHashProcFuncOpt[]; + +/* enabled hash alg */ +extern const IppHashAlgId cpEnabledHashAlgID[]; + +/* hash alg IV (init value) */ +extern const Ipp8u* cpHashIV[]; + +/* hash alg attribute DB */ +extern const cpHashAttr cpHashAlgAttr[]; + +/* IV size helper */ +__INLINE int cpHashIvSize(IppHashAlgId algID) +{ return cpHashAlgAttr[algID].ivSize; } + +/* hash size helper */ +__INLINE int cpHashSize(IppHashAlgId algID) +{ return cpHashAlgAttr[algID].hashSize; } + +/* message block size helper */ +__INLINE int cpHashMBS(IppHashAlgId algID) +{ return cpHashAlgAttr[algID].msgBlkSize; } + +/* maps algID into enabled IppHashAlgId value */ +__INLINE IppHashAlgId cpValidHashAlg(IppHashAlgId algID) +{ + /* maps algID into the valid range */ + algID = (((int)ippHashAlg_Unknown < (int)algID) && ((int)algID < (int)ippHashAlg_MaxNo))? algID : ippHashAlg_Unknown; + return cpEnabledHashAlgID[algID]; +} + +/* common functions */ +#define cpComputeDigest OWNAPI(cpComputeDigest) + IPP_OWN_DECL (void, cpComputeDigest, (Ipp8u* pHashTag, int hashTagLen, const IppsHashState* pCtx)) + +/* processing functions */ +#define UpdateSHA1 OWNAPI(UpdateSHA1) + IPP_OWN_DECL (void, UpdateSHA1, (void* pHash, const Ipp8u* mblk, int mlen, const void* pParam)) +#define UpdateSHA256 OWNAPI(UpdateSHA256) + IPP_OWN_DECL (void, UpdateSHA256, (void* pHash, const Ipp8u* mblk, int mlen, const void* pParam)) +#define UpdateSHA512 OWNAPI(UpdateSHA512) + IPP_OWN_DECL (void, UpdateSHA512, (void* pHash, const Ipp8u* mblk, int mlen, const void* pParam)) +#define UpdateMD5 OWNAPI(UpdateMD5) + IPP_OWN_DECL (void, UpdateMD5, (void* pHash, const Ipp8u* mblk, int mlen, const void* pParam)) +#define UpdateSM3 OWNAPI(UpdateSM3) + IPP_OWN_DECL (void, UpdateSM3, (void* pHash, const Ipp8u* mblk, int mlen, const void* pParam)) + +#if (_SHA_NI_ENABLING_ == _FEATURE_TICKTOCK_) || (_SHA_NI_ENABLING_ == _FEATURE_ON_) +#define UpdateSHA1ni OWNAPI(UpdateSHA1ni) + IPP_OWN_DECL (void, UpdateSHA1ni, (void* pHash, const Ipp8u* mblk, int mlen, const void* pParam)) +#define UpdateSHA256ni OWNAPI(UpdateSHA256ni) + IPP_OWN_DECL (void, UpdateSHA256ni, (void* pHash, const Ipp8u* mblk, int mlen, const void* pParam)) +#endif + +/* general methods */ +#define cpInitHash OWNAPI(cpInitHash) + IPP_OWN_DECL (int, cpInitHash, (IppsHashState* pCtx, IppHashAlgId algID)) +#define cpReInitHash OWNAPI(cpReInitHash) + IPP_OWN_DECL (int, cpReInitHash, (IppsHashState* pCtx, IppHashAlgId algID)) + +#endif /* _PCP_HASH_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphash_digest.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphash_digest.c new file mode 100644 index 0000000..94819e3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphash_digest.c @@ -0,0 +1,126 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// General Functionality +// +// Contents: +// cpComputeDigest() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcptool.h" + + +IPP_OWN_DEFN (void, cpComputeDigest, (Ipp8u* pHashTag, int hashTagLen, const IppsHashState* pCtx)) +{ + /* hash alg and parameters */ + cpHashProc hashFunc = HASH_FUNC(pCtx); /* processing function */ + const void* pParam = HASH_FUNC_PAR(pCtx); /* and it's addition params */ + + /* attributes */ + const cpHashAttr* pAttr = &cpHashAlgAttr[HASH_ALG_ID(pCtx)]; + int mbs = pAttr->msgBlkSize; /* data block size */ + int ivSize = pAttr->ivSize; /* size of hash's IV */ + int msgLenRepSize = pAttr->msgLenRepSize; /* length of the message representation */ + + /* number of bytes in context buffer */ + int n = HASH_BUFFIDX(pCtx); + /* buffer and it actual length */ + Ipp8u buffer[MBS_HASH_MAX*2]; + int bufferLen = n < (mbs-msgLenRepSize)? mbs : mbs*2; + + /* copy current hash value */ + cpHash hash; + CopyBlock(HASH_VALUE(pCtx), hash, ivSize); + + /* copy of state's buffer */ + CopyBlock(HASH_BUFF(pCtx), buffer, n); + /* end of message bit */ + buffer[n++] = 0x80; + /* padd buffer */ + PadBlock(0, buffer+n, bufferLen-n-msgLenRepSize); + + /* message length representation in bits (remember about big endian) */ + { + /* convert processed message length bytes ->bits */ + Ipp64u lo = HASH_LENLO(pCtx); + Ipp64u hi = HASH_LENHI(pCtx); + hi = LSL64(hi,3) | LSR64(lo,63-3); + lo = LSL64(lo,3); + + if(msgLenRepSize>(int)(sizeof(Ipp64u))) { + #if (IPP_ENDIAN == IPP_BIG_ENDIAN) + ((Ipp64u*)(buffer+bufferLen))[-2] = hi; + #else + ((Ipp64u*)(buffer+bufferLen))[-2] = ENDIANNESS64(hi); + #endif + } + + /* recall about MD5 specific */ + if(ippHashAlg_MD5!=HASH_ALG_ID(pCtx)) { + #if (IPP_ENDIAN == IPP_BIG_ENDIAN) + ((Ipp64u*)(buffer+bufferLen))[-1] = lo; + #else + ((Ipp64u*)(buffer+bufferLen))[-1] = ENDIANNESS64(lo); + #endif + } + else { + #if (IPP_ENDIAN == IPP_BIG_ENDIAN) + ((Ipp64u*)(buffer+bufferLen))[-1] = ENDIANNESS64(lo); + #else + ((Ipp64u*)(buffer+bufferLen))[-1] = lo; + #endif + } + } + + /* copmplete hash computation */ + hashFunc(hash, buffer, bufferLen, pParam); + + /* store digest into the user buffer (remember digest in big endian) */ + if(msgLenRepSize>(int)(sizeof(Ipp64u))) { + /* ippHashAlg_SHA384, ippHashAlg_SHA512, ippHashAlg_SHA512_224 and ippHashAlg_SHA512_256 */ + hash[0] = ENDIANNESS64(hash[0]); + hash[1] = ENDIANNESS64(hash[1]); + hash[2] = ENDIANNESS64(hash[2]); + hash[3] = ENDIANNESS64(hash[3]); + hash[4] = ENDIANNESS64(hash[4]); + hash[5] = ENDIANNESS64(hash[5]); + hash[6] = ENDIANNESS64(hash[6]); + hash[7] = ENDIANNESS64(hash[7]); + } + else if(ippHashAlg_MD5!=HASH_ALG_ID(pCtx)) { + ((Ipp32u*)hash)[0] = ENDIANNESS32(((Ipp32u*)hash)[0]); + ((Ipp32u*)hash)[1] = ENDIANNESS32(((Ipp32u*)hash)[1]); + ((Ipp32u*)hash)[2] = ENDIANNESS32(((Ipp32u*)hash)[2]); + ((Ipp32u*)hash)[3] = ENDIANNESS32(((Ipp32u*)hash)[3]); + ((Ipp32u*)hash)[4] = ENDIANNESS32(((Ipp32u*)hash)[4]); + if(ippHashAlg_SHA1!=HASH_ALG_ID(pCtx)) { + ((Ipp32u*)hash)[5] = ENDIANNESS32(((Ipp32u*)hash)[5]); + ((Ipp32u*)hash)[6] = ENDIANNESS32(((Ipp32u*)hash)[6]); + ((Ipp32u*)hash)[7] = ENDIANNESS32(((Ipp32u*)hash)[7]); + } + } + CopyBlock(hash, pHashTag, hashTagLen); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphash_func.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphash_func.h new file mode 100644 index 0000000..6b4254e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphash_func.h @@ -0,0 +1,108 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// Internal Definitions and Internal Functions Prototypes +// +// +*/ + +#if !defined(_PCP_HASH_FUNC_H) +#define _PCP_HASH_FUNC_H + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" + +/* +// hash alg default processing functions and opt argument +*/ +static cpHashProc cpHashProcFunc[] = { + (cpHashProc)NULL, + + #if defined(_ENABLE_ALG_SHA1_) + #if(_SHA_NI_ENABLING_==_FEATURE_ON_) + UpdateSHA1ni, + #else + UpdateSHA1, + #endif + #else + NULL, + #endif + + #if defined(_ENABLE_ALG_SHA256_) + #if(_SHA_NI_ENABLING_==_FEATURE_ON_) + UpdateSHA256ni, + #else + UpdateSHA256, + #endif + #else + NULL, + #endif + + #if defined(_ENABLE_ALG_SHA224_) + #if(_SHA_NI_ENABLING_==_FEATURE_ON_) + UpdateSHA256ni, + #else + UpdateSHA256, + #endif + #else + NULL, + #endif + + #if defined(_ENABLE_ALG_SHA512_) + UpdateSHA512, + #else + NULL, + #endif + + #if defined(_ENABLE_ALG_SHA384_) + UpdateSHA512, + #else + NULL, + #endif + + #if defined(_ENABLE_ALG_MD5_) + UpdateMD5, + #else + NULL, + #endif + + #if defined(_ENABLE_ALG_SM3_) + UpdateSM3, + #else + NULL, + #endif + + #if defined(_ENABLE_ALG_SHA512_224_) + UpdateSHA512, + #else + NULL, + #endif + + #if defined(_ENABLE_ALG_SHA512_256_) + UpdateSHA512, + #else + NULL, + #endif +}; + +#endif /* _PCP_HASH_FUNC_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphash_init.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphash_init.c new file mode 100644 index 0000000..0278c61 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphash_init.c @@ -0,0 +1,61 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// General Functionality +// +// Contents: +// cpInitHash() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_func.h" +#include "pcptool.h" + +IPP_OWN_DEFN (int, cpInitHash, (IppsHashState* pCtx, IppHashAlgId algID)) +{ + /* setup default processing function */ + HASH_FUNC(pCtx) = cpHashProcFunc[algID]; + + /* update default processing function if Intel® Secure Hash Algorithm - New Instructions (Intel® SHA-NI) enabled */ + #if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + if( IsFeatureEnabled(ippCPUID_SHA) ) { + + #if defined(_ENABLE_ALG_SHA1_) + if(ippHashAlg_SHA1==algID) + HASH_FUNC(pCtx) = UpdateSHA1ni; + #endif + + #if defined(_ENABLE_ALG_SHA256_) || defined(_ENABLE_ALG_SHA224_) + if(ippHashAlg_SHA256==algID || ippHashAlg_SHA224==algID) + HASH_FUNC(pCtx) = UpdateSHA256ni; + #endif + } + #endif + + /* setup optional agr of processing function */ + HASH_FUNC_PAR(pCtx) = cpHashProcFuncOpt[algID]; + + return cpReInitHash(pCtx, algID); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphash_reinit.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphash_reinit.c new file mode 100644 index 0000000..0a88722 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphash_reinit.c @@ -0,0 +1,61 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// General Functionality +// +// Contents: +// cpReInitHash() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcptool.h" + +/*F* +// Name: ippsHashInit +// +// Purpose: Init Hash state. +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL +// ippStsNotSupportedModeErr if algID is not match to supported hash alg +// ippStsNoErr no errors +// +// Parameters: +// pCtx pointer to the Hash state +// algID hash alg ID +// +*F*/ +IPP_OWN_DEFN (int, cpReInitHash, (IppsHashState* pCtx, IppHashAlgId algID)) +{ + int hashIvSize = cpHashIvSize(algID); + const Ipp8u* iv = cpHashIV[algID]; + + HASH_LENLO(pCtx) = CONST_64(0); + HASH_LENHI(pCtx) = CONST_64(0); + HASH_BUFFIDX(pCtx) = 0; + CopyBlock(iv, HASH_VALUE(pCtx), hashIvSize); + + return hashIvSize; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphash_rmf.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphash_rmf.h new file mode 100644 index 0000000..b3741f4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphash_rmf.h @@ -0,0 +1,48 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// Internal Definitions and Internal Functions Prototypes +// +*/ + +#if !defined(_CP_HASH_RMF_H) +#define _CP_HASH_RMF_H + +#include "pcphash.h" +#include "pcphashmethod_rmf.h" + +struct _cpHashCtx_rmf { + Ipp32u idCtx; /* hash identifier */ + const cpHashMethod_rmf* pMethod; /* hash methods */ + int msgBuffIdx; /* buffer index */ + Ipp8u msgBuffer[MBS_HASH_MAX]; /* buffer */ + Ipp64u msgLenLo; /* processed message */ + Ipp64u msgLenHi; /* length (bytes) */ + cpHash msgHash; /* hash value */ +}; + +/* accessors (see others in pcphash.h) */ +#define HASH_METHOD(stt) ((stt)->pMethod) + +#define cpFinalize_rmf OWNAPI(cpFinalize_rmf) + IPP_OWN_DECL (void, cpFinalize_rmf, (DigestSHA512 pHash, const Ipp8u* inpBuffer, int inpLen, Ipp64u lenLo, Ipp64u lenHi, const IppsHashMethod* method)) + +#endif /* _CP_HASH_RMF_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashca_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashca_rmf.c new file mode 100644 index 0000000..5240fdd --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashca_rmf.c @@ -0,0 +1,57 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// Generalized Functionality +// +// Contents: +// cpFinalize_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + + +IPP_OWN_DEFN (void, cpFinalize_rmf, (DigestSHA512 pHash, const Ipp8u* inpBuffer, int inpLen, Ipp64u lenLo, Ipp64u lenHi, const IppsHashMethod* method)) +{ + int mbs = method->msgBlkSize; /* messge block size */ + int mrl = method->msgLenRepSize; /* processed length representation size */ + + /* local buffer and it length */ + Ipp8u buffer[MBS_SHA512*2]; + int bufferLen = inpLen < (mbs-mrl)? mbs : mbs*2; + + /* copy rest of message into internal buffer */ + CopyBlock(inpBuffer, buffer, inpLen); + + /* padd message */ + buffer[inpLen++] = 0x80; + PadBlock(0, buffer+inpLen, bufferLen-inpLen-mrl); + + /* message length representation */ + method->msgLenRep(buffer+bufferLen-mrl, lenLo, lenHi); + + /* copmplete hash computation */ + method->hashUpdate(pHash, buffer, bufferLen); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashcnt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashcnt.c new file mode 100644 index 0000000..6a4c6f5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashcnt.c @@ -0,0 +1,435 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// Constants +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" + +#if defined( _IPP_DATA ) + +/* +// enabled hash alg IDs +*/ +const IppHashAlgId cpEnabledHashAlgID[] = { + IPP_ALG_HASH_UNKNOWN, + +#if defined(_ENABLE_ALG_SHA1_) + IPP_ALG_HASH_SHA1, +#else + IPP_ALG_HASH_UNKNOWN, +#endif + +#if defined(_ENABLE_ALG_SHA256_) + IPP_ALG_HASH_SHA256, +#else + IPP_ALG_HASH_UNKNOWN, +#endif + +#if defined(_ENABLE_ALG_SHA224_) + IPP_ALG_HASH_SHA224, +#else + IPP_ALG_HASH_UNKNOWN, +#endif + +#if defined(_ENABLE_ALG_SHA512_) + IPP_ALG_HASH_SHA512, +#else + IPP_ALG_HASH_UNKNOWN, +#endif + +#if defined(_ENABLE_ALG_SHA384_) + IPP_ALG_HASH_SHA384, +#else + IPP_ALG_HASH_UNKNOWN, +#endif + +#if defined(_ENABLE_ALG_MD5_) + IPP_ALG_HASH_MD5, +#else + IPP_ALG_HASH_UNKNOWN, +#endif + +#if defined(_ENABLE_ALG_SM3_) + IPP_ALG_HASH_SM3, +#else + IPP_ALG_HASH_UNKNOWN, +#endif + +#if defined(_ENABLE_ALG_SHA512_224_) + IPP_ALG_HASH_SHA512_224, +#else + IPP_ALG_HASH_UNKNOWN, +#endif + +#if defined(_ENABLE_ALG_SHA512_256_) + IPP_ALG_HASH_SHA512_256 +#else + IPP_ALG_HASH_UNKNOWN +#endif +}; +//////////////////////////////////////////////////////////// + +/* +// hash init values +*/ +const Ipp32u UnknownHash_IV[] = { + 0}; + +#if defined(_ENABLE_ALG_SHA1_) +const Ipp32u SHA1_IV[] = { + 0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476, 0xC3D2E1F0}; +#endif + +#if defined(_ENABLE_ALG_SHA256_) +const Ipp32u SHA256_IV[] = { + 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, + 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19}; +#endif + +#if defined(_ENABLE_ALG_SHA224_) +const Ipp32u SHA224_IV[] = { + 0xC1059ED8, 0x367CD507, 0x3070DD17, 0xF70E5939, + 0xFFC00B31, 0x68581511, 0x64F98FA7, 0xBEFA4FA4}; +#endif + +#if defined(_ENABLE_ALG_SHA512_) +const Ipp64u SHA512_IV[] = { + CONST_64(0x6A09E667F3BCC908), CONST_64(0xBB67AE8584CAA73B), + CONST_64(0x3C6EF372FE94F82B), CONST_64(0xA54FF53A5F1D36F1), + CONST_64(0x510E527FADE682D1), CONST_64(0x9B05688C2B3E6C1F), + CONST_64(0x1F83D9ABFB41BD6B), CONST_64(0x5BE0CD19137E2179)}; +#endif + +#if defined(_ENABLE_ALG_SHA384_) +const Ipp64u SHA384_IV[] = { + CONST_64(0xCBBB9D5DC1059ED8), CONST_64(0x629A292A367CD507), + CONST_64(0x9159015A3070DD17), CONST_64(0x152FECD8F70E5939), + CONST_64(0x67332667FFC00B31), CONST_64(0x8EB44A8768581511), + CONST_64(0xDB0C2E0D64F98FA7), CONST_64(0x47B5481DBEFA4FA4)}; +#endif + +#if defined(_ENABLE_ALG_MD5_) +const Ipp32u MD5_IV[] = { + 0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476}; +#endif + +#if defined(_ENABLE_ALG_SM3_) +const Ipp32u SM3_IV[] = { + 0x7380166F, 0x4914B2B9, 0x172442D7, 0xDA8A0600, + 0xA96F30BC, 0x163138AA, 0xE38DEE4D, 0xB0FB0E4E}; +#endif + +#if defined(_ENABLE_ALG_SHA512_224_) +const Ipp64u SHA512_224_IV[] = { + CONST_64(0x8C3D37C819544DA2), CONST_64(0x73E1996689DCD4D6), + CONST_64(0x1DFAB7AE32FF9C82), CONST_64(0x679DD514582F9FCF), + CONST_64(0x0F6D2B697BD44DA8), CONST_64(0x77E36F7304C48942), + CONST_64(0x3F9D85A86A1D36C8), CONST_64(0x1112E6AD91D692A1)}; +#endif + +#if defined(_ENABLE_ALG_SHA512_256_) +const Ipp64u SHA512_256_IV[] = { + CONST_64(0x22312194FC2BF72C), CONST_64(0x9F555FA3C84C64C2), + CONST_64(0x2393B86B6F53B151), CONST_64(0x963877195940EABD), + CONST_64(0x96283EE2A88EFFE3), CONST_64(0xBE5E1E2553863992), + CONST_64(0x2B0199FC2C85B8AA), CONST_64(0x0EB72DDC81C52CA2)}; +#endif + +const Ipp8u* cpHashIV[] = { + (Ipp8u*)UnknownHash_IV, + + #if defined(_ENABLE_ALG_SHA1_) + (Ipp8u*)SHA1_IV, + #else + (Ipp8u*)UnknownHash_IV, + #endif + + #if defined(_ENABLE_ALG_SHA256_) + (Ipp8u*)SHA256_IV, + #else + (Ipp8u*)UnknownHash_IV, + #endif + + #if defined(_ENABLE_ALG_SHA224_) + (Ipp8u*)SHA224_IV, + #else + (Ipp8u*)UnknownHash_IV, + #endif + + #if defined(_ENABLE_ALG_SHA512_) + (Ipp8u*)SHA512_IV, + #else + (Ipp8u*)UnknownHash_IV, + #endif + + #if defined(_ENABLE_ALG_SHA384_) + (Ipp8u*)SHA384_IV, + #else + (Ipp8u*)UnknownHash_IV, + #endif + + #if defined(_ENABLE_ALG_MD5_) + (Ipp8u*)MD5_IV, + #else + (Ipp8u*)UnknownHash_IV, + #endif + + #if defined(_ENABLE_ALG_SM3_) + (Ipp8u*)SM3_IV, + #else + (Ipp8u*)UnknownHash_IV, + #endif + + #if defined(_ENABLE_ALG_SHA512_224_) + (Ipp8u*)SHA512_224_IV, + #else + (Ipp8u*)UnknownHash_IV, + #endif + + #if defined(_ENABLE_ALG_SHA512_256_) + (Ipp8u*)SHA512_256_IV, + #else + (Ipp8u*)UnknownHash_IV, + #endif +}; + +//////////////////////////////////////////////////////////// + +/* +// additive constatns +*/ +#if defined(_ENABLE_ALG_SHA1_) +__ALIGN16 const Ipp32u SHA1_cnt[] = { + 0x5A827999, 0x6ED9EBA1, 0x8F1BBCDC, 0xCA62C1D6 +}; +#endif + +#if defined(_ENABLE_ALG_SHA256_) || defined(_ENABLE_ALG_SHA224_) +__ALIGN16 const Ipp32u SHA256_cnt[] = { + 0x428A2F98, 0x71374491, 0xB5C0FBCF, 0xE9B5DBA5, + 0x3956C25B, 0x59F111F1, 0x923F82A4, 0xAB1C5ED5, + 0xD807AA98, 0x12835B01, 0x243185BE, 0x550C7DC3, + 0x72BE5D74, 0x80DEB1FE, 0x9BDC06A7, 0xC19BF174, + 0xE49B69C1, 0xEFBE4786, 0x0FC19DC6, 0x240CA1CC, + 0x2DE92C6F, 0x4A7484AA, 0x5CB0A9DC, 0x76F988DA, + 0x983E5152, 0xA831C66D, 0xB00327C8, 0xBF597FC7, + 0xC6E00BF3, 0xD5A79147, 0x06CA6351, 0x14292967, + 0x27B70A85, 0x2E1B2138, 0x4D2C6DFC, 0x53380D13, + 0x650A7354, 0x766A0ABB, 0x81C2C92E, 0x92722C85, + 0xA2BFE8A1, 0xA81A664B, 0xC24B8B70, 0xC76C51A3, + 0xD192E819, 0xD6990624, 0xF40E3585, 0x106AA070, + 0x19A4C116, 0x1E376C08, 0x2748774C, 0x34B0BCB5, + 0x391C0CB3, 0x4ED8AA4A, 0x5B9CCA4F, 0x682E6FF3, + 0x748F82EE, 0x78A5636F, 0x84C87814, 0x8CC70208, + 0x90BEFFFA, 0xA4506CEB, 0xBEF9A3F7, 0xC67178F2 +}; +#endif + +#if defined(_ENABLE_ALG_SHA512_) || defined(_ENABLE_ALG_SHA384_) || defined(_ENABLE_ALG_SHA512_224_) || defined(_ENABLE_ALG_SHA512_256_) +__ALIGN16 const Ipp64u SHA512_cnt[] = { + CONST_64(0x428A2F98D728AE22), CONST_64(0x7137449123EF65CD), CONST_64(0xB5C0FBCFEC4D3B2F), CONST_64(0xE9B5DBA58189DBBC), + CONST_64(0x3956C25BF348B538), CONST_64(0x59F111F1B605D019), CONST_64(0x923F82A4AF194F9B), CONST_64(0xAB1C5ED5DA6D8118), + CONST_64(0xD807AA98A3030242), CONST_64(0x12835B0145706FBE), CONST_64(0x243185BE4EE4B28C), CONST_64(0x550C7DC3D5FFB4E2), + CONST_64(0x72BE5D74F27B896F), CONST_64(0x80DEB1FE3B1696B1), CONST_64(0x9BDC06A725C71235), CONST_64(0xC19BF174CF692694), + CONST_64(0xE49B69C19EF14AD2), CONST_64(0xEFBE4786384F25E3), CONST_64(0x0FC19DC68B8CD5B5), CONST_64(0x240CA1CC77AC9C65), + CONST_64(0x2DE92C6F592B0275), CONST_64(0x4A7484AA6EA6E483), CONST_64(0x5CB0A9DCBD41FBD4), CONST_64(0x76F988DA831153B5), + CONST_64(0x983E5152EE66DFAB), CONST_64(0xA831C66D2DB43210), CONST_64(0xB00327C898FB213F), CONST_64(0xBF597FC7BEEF0EE4), + CONST_64(0xC6E00BF33DA88FC2), CONST_64(0xD5A79147930AA725), CONST_64(0x06CA6351E003826F), CONST_64(0x142929670A0E6E70), + CONST_64(0x27B70A8546D22FFC), CONST_64(0x2E1B21385C26C926), CONST_64(0x4D2C6DFC5AC42AED), CONST_64(0x53380D139D95B3DF), + CONST_64(0x650A73548BAF63DE), CONST_64(0x766A0ABB3C77B2A8), CONST_64(0x81C2C92E47EDAEE6), CONST_64(0x92722C851482353B), + CONST_64(0xA2BFE8A14CF10364), CONST_64(0xA81A664BBC423001), CONST_64(0xC24B8B70D0F89791), CONST_64(0xC76C51A30654BE30), + CONST_64(0xD192E819D6EF5218), CONST_64(0xD69906245565A910), CONST_64(0xF40E35855771202A), CONST_64(0x106AA07032BBD1B8), + CONST_64(0x19A4C116B8D2D0C8), CONST_64(0x1E376C085141AB53), CONST_64(0x2748774CDF8EEB99), CONST_64(0x34B0BCB5E19B48A8), + CONST_64(0x391C0CB3C5C95A63), CONST_64(0x4ED8AA4AE3418ACB), CONST_64(0x5B9CCA4F7763E373), CONST_64(0x682E6FF3D6B2B8A3), + CONST_64(0x748F82EE5DEFB2FC), CONST_64(0x78A5636F43172F60), CONST_64(0x84C87814A1F0AB72), CONST_64(0x8CC702081A6439EC), + CONST_64(0x90BEFFFA23631E28), CONST_64(0xA4506CEBDE82BDE9), CONST_64(0xBEF9A3F7B2C67915), CONST_64(0xC67178F2E372532B), + CONST_64(0xCA273ECEEA26619C), CONST_64(0xD186B8C721C0C207), CONST_64(0xEADA7DD6CDE0EB1E), CONST_64(0xF57D4F7FEE6ED178), + CONST_64(0x06F067AA72176FBA), CONST_64(0x0A637DC5A2C898A6), CONST_64(0x113F9804BEF90DAE), CONST_64(0x1B710B35131C471B), + CONST_64(0x28DB77F523047D84), CONST_64(0x32CAAB7B40C72493), CONST_64(0x3C9EBE0A15C9BEBC), CONST_64(0x431D67C49C100D4C), + CONST_64(0x4CC5D4BECB3E42B6), CONST_64(0x597F299CFC657E2A), CONST_64(0x5FCB6FAB3AD6FAEC), CONST_64(0x6C44198C4A475817) +}; +#endif + +#if defined(_ENABLE_ALG_MD5_) +__ALIGN16 const Ipp32u MD5_cnt[] = { + 0xD76AA478, 0xE8C7B756, 0x242070DB, 0xC1BDCEEE, + 0xF57C0FAF, 0x4787C62A, 0xA8304613, 0xFD469501, + 0x698098D8, 0x8B44F7AF, 0xFFFF5BB1, 0x895CD7BE, + 0x6B901122, 0xFD987193, 0xA679438E, 0x49B40821, + + 0xF61E2562, 0xC040B340, 0x265E5A51, 0xE9B6C7AA, + 0xD62F105D, 0x02441453, 0xD8A1E681, 0xE7D3FBC8, + 0x21E1CDE6, 0xC33707D6, 0xF4D50D87, 0x455A14ED, + 0xA9E3E905, 0xFCEFA3F8, 0x676F02D9, 0x8D2A4C8A, + + 0xFFFA3942, 0x8771F681, 0x6D9D6122, 0xFDE5380C, + 0xA4BEEA44, 0x4BDECFA9, 0xF6BB4B60, 0xBEBFBC70, + 0x289B7EC6, 0xEAA127FA, 0xD4EF3085, 0x04881D05, + 0xD9D4D039, 0xE6DB99E5, 0x1FA27CF8, 0xC4AC5665, + + 0xF4292244, 0x432AFF97, 0xAB9423A7, 0xFC93A039, + 0x655B59C3, 0x8F0CCC92, 0xFFEFF47D, 0x85845DD1, + 0x6FA87E4F, 0xFE2CE6E0, 0xA3014314, 0x4E0811A1, + 0xF7537E82, 0xBD3AF235, 0x2AD7D2BB, 0xEB86D391 +}; +#endif + +#if defined(_ENABLE_ALG_SM3_) +__ALIGN16 const Ipp32u SM3_cnt[] = { + 0x79CC4519,0xF3988A32,0xE7311465,0xCE6228CB,0x9CC45197,0x3988A32F,0x7311465E,0xE6228CBC, + 0xCC451979,0x988A32F3,0x311465E7,0x6228CBCE,0xC451979C,0x88A32F39,0x11465E73,0x228CBCE6, + 0x9D8A7A87,0x3B14F50F,0x7629EA1E,0xEC53D43C,0xD8A7A879,0xB14F50F3,0x629EA1E7,0xC53D43CE, + 0x8A7A879D,0x14F50F3B,0x29EA1E76,0x53D43CEC,0xA7A879D8,0x4F50F3B1,0x9EA1E762,0x3D43CEC5, + 0x7A879D8A,0xF50F3B14,0xEA1E7629,0xD43CEC53,0xA879D8A7,0x50F3B14F,0xA1E7629E,0x43CEC53D, + 0x879D8A7A,0x0F3B14F5,0x1E7629EA,0x3CEC53D4,0x79D8A7A8,0xF3B14F50,0xE7629EA1,0xCEC53D43, + 0x9D8A7A87,0x3B14F50F,0x7629EA1E,0xEC53D43C,0xD8A7A879,0xB14F50F3,0x629EA1E7,0xC53D43CE, + 0x8A7A879D,0x14F50F3B,0x29EA1E76,0x53D43CEC,0xA7A879D8,0x4F50F3B1,0x9EA1E762,0x3D43CEC5 +}; +#endif + +/* +// hash alg default processing opt argument +*/ +const void* cpHashProcFuncOpt[] = { + NULL, + + #if defined(_ENABLE_ALG_SHA1_) + SHA1_cnt, + #else + NULL, + #endif + + #if defined(_ENABLE_ALG_SHA256_) + SHA256_cnt, + #else + NULL, + #endif + + #if defined(_ENABLE_ALG_SHA224_) + SHA256_cnt, + #else + NULL, + #endif + + #if defined(_ENABLE_ALG_SHA512_) + SHA512_cnt, + #else + NULL, + #endif + + #if defined(_ENABLE_ALG_SHA384_) + SHA512_cnt, + #else + NULL, + #endif + + #if defined(_ENABLE_ALG_MD5_) + MD5_cnt, + #else + NULL, + #endif + + #if defined(_ENABLE_ALG_SM3_) + SM3_cnt, + #else + NULL, + #endif + + #if defined(_ENABLE_ALG_SHA512_224_) + SHA512_cnt, + #else + NULL, + #endif + + #if defined(_ENABLE_ALG_SHA512_256_) + SHA512_cnt, + #else + NULL, + #endif +}; +//////////////////////////////////////////////////////////// + +/* hash alg attributes */ +const cpHashAttr cpHashAlgAttr[] = { + {0, 0, 0, 0, {CONST_64(0),CONST_64(0)}}, /* unknown */ + +#if defined(_ENABLE_ALG_SHA1_) /* sha1 / unknown */ + {IPP_SHA1_DIGEST_BITSIZE/8, IPP_SHA1_DIGEST_BITSIZE/8, MBS_SHA1, sizeof(Ipp64u), {CONST_64(0x2000000000000000-1),CONST_64(0)}}, +#else + {0, 0, 0, 0, {CONST_64(0),CONST_64(0)}}, +#endif + +#if defined(_ENABLE_ALG_SHA256_) /* sha256 / unknown */ + {IPP_SHA256_DIGEST_BITSIZE/8,IPP_SHA256_DIGEST_BITSIZE/8, MBS_SHA256, sizeof(Ipp64u), {CONST_64(0x2000000000000000-1),CONST_64(0)}}, +#else + {0, 0, 0, 0, {CONST_64(0),CONST_64(0)}}, +#endif + +#if defined(_ENABLE_ALG_SHA224_) /* sha224 / unknown */ + {IPP_SHA256_DIGEST_BITSIZE/8,IPP_SHA224_DIGEST_BITSIZE/8, MBS_SHA224, sizeof(Ipp64u), {CONST_64(0x2000000000000000-1),CONST_64(0)}}, +#else + {0, 0, 0, 0, {CONST_64(0),CONST_64(0)}}, +#endif + +#if defined(_ENABLE_ALG_SHA512_) /* sha512 / unknown */ + {IPP_SHA512_DIGEST_BITSIZE/8,IPP_SHA512_DIGEST_BITSIZE/8, MBS_SHA512, sizeof(Ipp64u)*2, {CONST_64(0xFFFFFFFFFFFFFFFF),CONST_64(0x2000000000000000-1)}}, +#else + {0, 0, 0, 0, {CONST_64(0),CONST_64(0)}}, +#endif + +#if defined(_ENABLE_ALG_SHA384_) /* sha384 / unknown */ + {IPP_SHA512_DIGEST_BITSIZE/8,IPP_SHA384_DIGEST_BITSIZE/8, MBS_SHA384, sizeof(Ipp64u)*2, {CONST_64(0xFFFFFFFFFFFFFFFF),CONST_64(0x2000000000000000-1)}}, +#else + {0, 0, 0, 0, {CONST_64(0),CONST_64(0)}}, +#endif + +#if defined(_ENABLE_ALG_MD5_) /* md5 / unknown */ + {IPP_MD5_DIGEST_BITSIZE/8,IPP_MD5_DIGEST_BITSIZE/8, MBS_MD5, sizeof(Ipp64u), {CONST_64(0x2000000000000000-1),CONST_64(0)}}, +#else + {0, 0, 0, 0, {CONST_64(0),CONST_64(0)}}, +#endif + +#if defined(_ENABLE_ALG_SM3_) /* sm3 / unknown */ + {IPP_SM3_DIGEST_BITSIZE/8,IPP_SM3_DIGEST_BITSIZE/8, MBS_SM3, sizeof(Ipp64u), {CONST_64(0x2000000000000000-1),CONST_64(0)}}, +#else + {0, 0, 0, 0, {CONST_64(0),CONST_64(0)}}, +#endif + +#if defined(_ENABLE_ALG_SHA512_224_) /* sha512/224 / unknown */ + {IPP_SHA512_DIGEST_BITSIZE/8,IPP_SHA512_224_DIGEST_BITSIZE/8, MBS_SHA512, sizeof(Ipp64u)*2, {CONST_64(0xFFFFFFFFFFFFFFFF),CONST_64(0x2000000000000000-1)}}, +#else + {0, 0, 0, 0, {CONST_64(0),CONST_64(0)}}, +#endif + +#if defined(_ENABLE_ALG_SHA512_256_) /* sha512/256 / unknown */ + {IPP_SHA512_DIGEST_BITSIZE/8,IPP_SHA512_256_DIGEST_BITSIZE/8, MBS_SHA512, sizeof(Ipp64u)*2, {CONST_64(0xFFFFFFFFFFFFFFFF),CONST_64(0x2000000000000000-1)}} +#else + {0, 0, 0, 0, {CONST_64(0),CONST_64(0)}} +#endif +}; + +#endif /* _IPP_DATA */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashduplicate.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashduplicate.c new file mode 100644 index 0000000..c1d636e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashduplicate.c @@ -0,0 +1,67 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// General Functionality +// +// Contents: +// ippsHashDuplicate() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcptool.h" + +/*F* +// Name: ippsHashDuplicate +// +// Purpose: Clone Hash context. +// +// Returns: Reason: +// ippStsNullPtrErr pSrcState == NULL +// pDstState == NULL +// ippStsContextMatchErr pSrcState->idCtx != idCtxHash +// pDstState->idCtx != idCtxHash +// ippStsNoErr no errors +// +// Parameters: +// pSrcState pointer to the source Hash context +// pDstState pointer to the target Hash context +// +// Note: +// pDstState may to be uninitialized by ippsHashInit() +// +*F*/ +IPPFUN(IppStatus, ippsHashDuplicate,(const IppsHashState* pSrcState, IppsHashState* pDstState)) +{ + /* test state pointers */ + IPP_BAD_PTR2_RET(pSrcState, pDstState); + /* test states ID */ + IPP_BADARG_RET(!HASH_VALID_ID(pSrcState, idCtxHash), ippStsContextMatchErr); + + /* copy state */ + CopyBlock(pSrcState, pDstState, sizeof(IppsHashState)); + HASH_SET_ID(pDstState, idCtxHash); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashduplicate_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashduplicate_rmf.c new file mode 100644 index 0000000..866367e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashduplicate_rmf.c @@ -0,0 +1,67 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// Generalized Functionality +// +// Contents: +// ippsHashDuplicate_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsHashDuplicate_rmf +// +// Purpose: Clone Hash context. +// +// Returns: Reason: +// ippStsNullPtrErr pSrcState == NULL +// pDstState == NULL +// ippStsContextMatchErr pSrcState->idCtx != idCtxHash +// pDstState->idCtx != idCtxHash +// ippStsNoErr no errors +// +// Parameters: +// pSrcState pointer to the source Hash context +// pDstState pointer to the target Hash context +// +// Note: +// pDstState may to be uninitialized by ippsHashInit_rmf() +// +*F*/ +IPPFUN(IppStatus, ippsHashDuplicate_rmf,(const IppsHashState_rmf* pSrcState, IppsHashState_rmf* pDstState)) +{ + /* test state pointers */ + IPP_BAD_PTR2_RET(pSrcState, pDstState); + /* test states ID */ + IPP_BADARG_RET(!HASH_VALID_ID(pSrcState,idCtxHash), ippStsContextMatchErr); + + /* copy state */ + CopyBlock(pSrcState, pDstState, sizeof(IppsHashState_rmf)); + HASH_SET_ID(pDstState, idCtxHash); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashfinal.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashfinal.c new file mode 100644 index 0000000..97cc4fb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashfinal.c @@ -0,0 +1,68 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// General Functionality +// +// Contents: +// ippsHashFinal() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcptool.h" + + +/*F* +// Name: ippsHashFinal +// +// Purpose: Complete message digesting and return digest. +// +// Returns: Reason: +// ippStsNullPtrErr pMD == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxHash +// ippStsNoErr no errors +// +// Parameters: +// pMD address of the output digest +// pState pointer to the SHS state +// +*F*/ +IPPFUN(IppStatus, ippsHashFinal,(Ipp8u* pMD, IppsHashState* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR2_RET(pMD, pState); + /* test the context */ + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxHash), ippStsContextMatchErr); + + { + IppHashAlgId algID = HASH_ALG_ID(pState); + int hashSize = cpHashAlgAttr[algID].hashSize; + + cpComputeDigest(pMD, hashSize, pState); + cpReInitHash(pState, algID); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashfinal_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashfinal_rmf.c new file mode 100644 index 0000000..5c1132d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashfinal_rmf.c @@ -0,0 +1,76 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// Generalized Functionality +// +// Contents: +// ippsHashFinal_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + + +/*F* +// Name: ippsHashFinal_rmf +// +// Purpose: Complete message digesting and return digest. +// +// Returns: Reason: +// ippStsNullPtrErr pMD == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxHash +// ippStsNoErr no errors +// +// Parameters: +// pMD address of the output digest +// pState pointer to the SHS state +// +*F*/ +IPPFUN(IppStatus, ippsHashFinal_rmf,(Ipp8u* pMD, IppsHashState_rmf* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR2_RET(pMD, pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxHash), ippStsContextMatchErr); + + { + const IppsHashMethod* method = HASH_METHOD(pState); + + cpFinalize_rmf(HASH_VALUE(pState), + HASH_BUFF(pState), HASH_BUFFIDX(pState), + HASH_LENLO(pState), HASH_LENHI(pState), + method); + /* convert hash into oct string */ + method->hashOctStr(pMD, HASH_VALUE(pState)); + + /* re-init hash value */ + HASH_BUFFIDX(pState) = 0; + HASH_LENLO(pState) = 0; + HASH_LENHI(pState) = 0; + method->hashInit(HASH_VALUE(pState)); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashgetinfo_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashgetinfo_rmf.c new file mode 100644 index 0000000..11109a9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashgetinfo_rmf.c @@ -0,0 +1,89 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// Generalized Functionality +// +// Contents: +// ippsHashMethodGetInfo() +// ippsHashGetInfo_rmf() +// +*/ + +#include "owncp.h" +#include "pcphash_rmf.h" + +/*F* +// Name: ippsHashMethodGetInfo +// +// Purpose: Returns info of the Hash algorithm +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pMethod +// NULL == pInfo +// +// ippStsNoErr no error +// +// Parameters: +// pInfo Pointer to the info structure +// pMethod Pointer to the IppsHashMethod +// +*F*/ +IPPFUN(IppStatus, ippsHashMethodGetInfo,(IppsHashInfo* pInfo, const IppsHashMethod* pMethod)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pInfo, pMethod); + + pInfo->hashSize = pMethod->hashLen; + pInfo->msgBlockSize = pMethod->msgBlkSize; + + return ippStsNoErr; +} + +/*F* +// Name: ippsHashGetInfo_rmf +// +// Purpose: Returns info of the using Hash +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pState +// NULL == pInfo +// +// ippStsContextMatchErr invalid pState->idCtx +// +// ippStsNoErr no error +// +// Parameters: +// pInfo Pointer to the info structure +// pState Pointer to the state context +// +*F*/ +IPPFUN(IppStatus, ippsHashGetInfo_rmf,(IppsHashInfo* pInfo, const IppsHashState_rmf* pState)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pInfo, pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxHash), ippStsContextMatchErr); + + pInfo->hashSize = HASH_METHOD(pState)->hashLen; + pInfo->msgBlockSize = HASH_METHOD(pState)->msgBlkSize; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashgetsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashgetsize.c new file mode 100644 index 0000000..015a79d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashgetsize.c @@ -0,0 +1,55 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// General Functionality +// +// Contents: +// ippsHashGetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcptool.h" + +/*F* +// Name: ippsHashGetSize +// +// Purpose: Returns size (bytes) of IppsHashState state. +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to state size +// +*F*/ +IPPFUN(IppStatus, ippsHashGetSize,(int* pSize)) +{ + /* test pointers */ + IPP_BAD_PTR1_RET(pSize); + + *pSize = sizeof(IppsHashState); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashgetsize_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashgetsize_rmf.c new file mode 100644 index 0000000..cdf2f5b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashgetsize_rmf.c @@ -0,0 +1,56 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// Generalized Functionality +// +// Contents: +// ippsHashGetSize_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsHashGetSize_rmf +// +// Purpose: Returns size (bytes) of IppsHashState state. +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to state size +// +*F*/ +IPPFUN(IppStatus, ippsHashGetSize_rmf,(int* pSize)) +{ + /* test pointers */ + IPP_BAD_PTR1_RET(pSize); + + *pSize = sizeof(IppsHashState_rmf); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashgettag.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashgettag.c new file mode 100644 index 0000000..d01cab5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashgettag.c @@ -0,0 +1,69 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// General Functionality +// +// Contents: +// ippsHashGetTag() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcptool.h" + +/*F* +// Name: ippsHashGetTag +// +// Purpose: Compute digest based on current state. +// Note, that futher digest update is possible +// +// Returns: Reason: +// ippStsNullPtrErr pTag == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxHash +// ippStsLengthErr hashSize < tagLen <1 +// ippStsNoErr no errors +// +// Parameters: +// pTag address of the output digest +// tagLen length of digest +// pState pointer to the SHS state +// +*F*/ +IPPFUN(IppStatus, ippsHashGetTag,(Ipp8u* pTag, int tagLen, const IppsHashState* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR2_RET(pTag, pState); + /* test the context */ + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxHash), ippStsContextMatchErr); + + { + /* size of hash */ + int hashSize = cpHashAlgAttr[HASH_ALG_ID(pState)].hashSize; + if(tagLen<1||hashSizeidCtx != idCtxHash +// ippStsLengthErr hashSize < tagLen <1 +// ippStsNoErr no errors +// +// Parameters: +// pTag address of the output digest +// tagLen length of digest +// pState pointer to the SHS state +// +*F*/ +IPPFUN(IppStatus, ippsHashGetTag_rmf,(Ipp8u* pTag, int tagLen, const IppsHashState_rmf* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxHash), ippStsContextMatchErr); + + /* test digest pointer */ + IPP_BAD_PTR1_RET(pTag); + IPP_BADARG_RET((tagLen <1) || HASH_METHOD(pState)->hashLenhashOctStr(pTag, hash); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashinit.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashinit.c new file mode 100644 index 0000000..18a076f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashinit.c @@ -0,0 +1,68 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// General Functionality +// +// Contents: +// ippsHashInit() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcptool.h" + +/*F* +// Name: ippsHashInit +// +// Purpose: Init Hash state. +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL +// ippStsNoErr no errors +// +// Parameters: +// pState pointer to the SHA1 state +// hashAlg identifier of the hash algorithm +// +*F*/ + +IPPFUN(IppStatus, ippsHashInit,(IppsHashState* pState, IppHashAlgId hashAlg)) +{ + /* get algorithm id */ + hashAlg = cpValidHashAlg(hashAlg); + /* test hash alg */ + IPP_BADARG_RET(ippHashAlg_Unknown==hashAlg, ippStsNotSupportedModeErr); + + /* test ctx pointer */ + IPP_BAD_PTR1_RET(pState); + /* test hash alg */ + + /* set ctx ID */ + HASH_SET_ID(pState, idCtxHash); + HASH_ALG_ID(pState) = hashAlg; + + /* init context */ + cpInitHash(pState, hashAlg); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashinit_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashinit_rmf.c new file mode 100644 index 0000000..ece390c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashinit_rmf.c @@ -0,0 +1,60 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// Generalized Functionality +// +// Contents: +// ippsHashInit_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsHashInit_rmf +// +// Purpose: Init Hash state. +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL +// pMethod == NULL +// ippStsNoErr no errors +// +// Parameters: +// pState pointer to the Hash state +// pMethod hash method +// +*F*/ +IPPFUN(IppStatus, ippsHashInit_rmf,(IppsHashState_rmf* pState, const IppsHashMethod* pMethod)) +{ + /* test ctx pointers */ + IPP_BAD_PTR2_RET(pState, pMethod); + + PadBlock(0, pState, sizeof(IppsHashState_rmf)); + HASH_METHOD(pState) = pMethod; + HASH_SET_ID(pState, idCtxHash); + pMethod->hashInit(HASH_VALUE(pState)); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmd5px.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmd5px.c new file mode 100644 index 0000000..b9c6aef --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmd5px.c @@ -0,0 +1,206 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Message block processing according to MD5 +// +// Contents: +// UpdateMD5() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcptool.h" + +#if defined(_ENABLE_ALG_MD5_) + +#if !((_IPP==_IPP_M5) || \ + (_IPP==_IPP_W7) || (_IPP==_IPP_T7) || \ + (_IPP==_IPP_V8) || (_IPP==_IPP_P8) || \ + (_IPP==_IPP_S8) || (_IPP>=_IPP_G9) || \ + (_IPP32E==_IPP32E_M7) || \ + (_IPP32E==_IPP32E_U8) || (_IPP32E==_IPP32E_Y8) || \ + (_IPP32E==_IPP32E_N8) || (_IPP32E>=_IPP32E_E9)) + + +/* +// Magic functions defined in RFC 1321 +// +*/ +#define F(X,Y,Z) ((Z) ^ ((X) & ((Y) ^ (Z)))) /* sightly optimized form of (((X) & (Y)) | ((~(X) & (Z)))*/ +#define G(X,Y,Z) F((Z),(X),(Y)) /* replace the original (((X) & (Z)) | ((Y) & ~(Z))) */ +#define H(X,Y,Z) ((X) ^ (Y) ^ (Z)) +#define I(X,Y,Z) ((Y) ^ ((X) | ~(Z))) + +/* +// MD5 step +*/ +#define MD5_STEP(MAGIC, A,B,C,D, data, constant, nrot) \ + (A = B +ROL32((A +MAGIC(B,C,D) +data +constant), nrot)) + +/* +// MD5 left rotations (number of bits) +// depends on round type +*/ +#define F1 7 +#define F2 12 +#define F3 17 +#define F4 22 + +#define G1 5 +#define G2 9 +#define G3 14 +#define G4 20 + +#define H1 4 +#define H2 11 +#define H3 16 +#define H4 23 + +#define I1 6 +#define I2 10 +#define I3 15 +#define I4 21 + +/*F* +// Name: UpdateMD5 +// +// Purpose: Update internal hash according to input message stream. +// +// Parameters: +// uniHash pointer to in/out hash +// mblk pointer to message stream +// mlen message stream length (multiple by message block size) +// uniParam pointer to the optional parameter +// +*F*/ +IPP_OWN_DEFN (void, UpdateMD5, (void* uinHash, const Ipp8u* mblk, int mlen, const void* uniParam)) +{ + Ipp32u* digest = (Ipp32u*)uinHash; + Ipp32u* MD5_cnt_loc = (Ipp32u*)uniParam; + + for(; mlen>=MBS_MD5; mblk += MBS_MD5, mlen -= MBS_MD5) { + + /* allocate data */ + #if (IPP_ENDIAN == IPP_BIG_ENDIAN) + Ipp32u data[MBS_MD5/sizeof(Ipp32u)]; + #else + /* or just word alias */ + Ipp32u* data = (Ipp32u*)mblk; + #endif + + /* init variables */ + Ipp32u a = digest[0]; + Ipp32u b = digest[1]; + Ipp32u c = digest[2]; + Ipp32u d = digest[3]; + + #if (IPP_ENDIAN == IPP_BIG_ENDIAN) + int t; + for(t=0; t<16; t++) { + data[t] = ENDIANNESS(((Ipp32u*)mblk)[t]); + } + #endif + + /* rounds type F */ + MD5_STEP(F, a,b,c,d, data[ 0], MD5_cnt_loc[ 0], F1); + MD5_STEP(F, d,a,b,c, data[ 1], MD5_cnt_loc[ 1], F2); + MD5_STEP(F, c,d,a,b, data[ 2], MD5_cnt_loc[ 2], F3); + MD5_STEP(F, b,c,d,a, data[ 3], MD5_cnt_loc[ 3], F4); + MD5_STEP(F, a,b,c,d, data[ 4], MD5_cnt_loc[ 4], F1); + MD5_STEP(F, d,a,b,c, data[ 5], MD5_cnt_loc[ 5], F2); + MD5_STEP(F, c,d,a,b, data[ 6], MD5_cnt_loc[ 6], F3); + MD5_STEP(F, b,c,d,a, data[ 7], MD5_cnt_loc[ 7], F4); + MD5_STEP(F, a,b,c,d, data[ 8], MD5_cnt_loc[ 8], F1); + MD5_STEP(F, d,a,b,c, data[ 9], MD5_cnt_loc[ 9], F2); + MD5_STEP(F, c,d,a,b, data[10], MD5_cnt_loc[10], F3); + MD5_STEP(F, b,c,d,a, data[11], MD5_cnt_loc[11], F4); + MD5_STEP(F, a,b,c,d, data[12], MD5_cnt_loc[12], F1); + MD5_STEP(F, d,a,b,c, data[13], MD5_cnt_loc[13], F2); + MD5_STEP(F, c,d,a,b, data[14], MD5_cnt_loc[14], F3); + MD5_STEP(F, b,c,d,a, data[15], MD5_cnt_loc[15], F4); + + /* rounds type G */ + MD5_STEP(G, a,b,c,d, data[ 1], MD5_cnt_loc[16], G1); + MD5_STEP(G, d,a,b,c, data[ 6], MD5_cnt_loc[17], G2); + MD5_STEP(G, c,d,a,b, data[11], MD5_cnt_loc[18], G3); + MD5_STEP(G, b,c,d,a, data[ 0], MD5_cnt_loc[19], G4); + MD5_STEP(G, a,b,c,d, data[ 5], MD5_cnt_loc[20], G1); + MD5_STEP(G, d,a,b,c, data[10], MD5_cnt_loc[21], G2); + MD5_STEP(G, c,d,a,b, data[15], MD5_cnt_loc[22], G3); + MD5_STEP(G, b,c,d,a, data[ 4], MD5_cnt_loc[23], G4); + MD5_STEP(G, a,b,c,d, data[ 9], MD5_cnt_loc[24], G1); + MD5_STEP(G, d,a,b,c, data[14], MD5_cnt_loc[25], G2); + MD5_STEP(G, c,d,a,b, data[ 3], MD5_cnt_loc[26], G3); + MD5_STEP(G, b,c,d,a, data[ 8], MD5_cnt_loc[27], G4); + MD5_STEP(G, a,b,c,d, data[13], MD5_cnt_loc[28], G1); + MD5_STEP(G, d,a,b,c, data[ 2], MD5_cnt_loc[29], G2); + MD5_STEP(G, c,d,a,b, data[ 7], MD5_cnt_loc[30], G3); + MD5_STEP(G, b,c,d,a, data[12], MD5_cnt_loc[31], G4); + + /* rounds type H */ + MD5_STEP(H, a,b,c,d, data[ 5], MD5_cnt_loc[32], H1); + MD5_STEP(H, d,a,b,c, data[ 8], MD5_cnt_loc[33], H2); + MD5_STEP(H, c,d,a,b, data[11], MD5_cnt_loc[34], H3); + MD5_STEP(H, b,c,d,a, data[14], MD5_cnt_loc[35], H4); + MD5_STEP(H, a,b,c,d, data[ 1], MD5_cnt_loc[36], H1); + MD5_STEP(H, d,a,b,c, data[ 4], MD5_cnt_loc[37], H2); + MD5_STEP(H, c,d,a,b, data[ 7], MD5_cnt_loc[38], H3); + MD5_STEP(H, b,c,d,a, data[10], MD5_cnt_loc[39], H4); + MD5_STEP(H, a,b,c,d, data[13], MD5_cnt_loc[40], H1); + MD5_STEP(H, d,a,b,c, data[ 0], MD5_cnt_loc[41], H2); + MD5_STEP(H, c,d,a,b, data[ 3], MD5_cnt_loc[42], H3); + MD5_STEP(H, b,c,d,a, data[ 6], MD5_cnt_loc[43], H4); + MD5_STEP(H, a,b,c,d, data[ 9], MD5_cnt_loc[44], H1); + MD5_STEP(H, d,a,b,c, data[12], MD5_cnt_loc[45], H2); + MD5_STEP(H, c,d,a,b, data[15], MD5_cnt_loc[46], H3); + MD5_STEP(H, b,c,d,a, data[ 2], MD5_cnt_loc[47], H4); + + /* rounds type I */ + MD5_STEP(I, a,b,c,d, data[ 0], MD5_cnt_loc[48], I1); + MD5_STEP(I, d,a,b,c, data[ 7], MD5_cnt_loc[49], I2); + MD5_STEP(I, c,d,a,b, data[14], MD5_cnt_loc[50], I3); + MD5_STEP(I, b,c,d,a, data[ 5], MD5_cnt_loc[51], I4); + MD5_STEP(I, a,b,c,d, data[12], MD5_cnt_loc[52], I1); + MD5_STEP(I, d,a,b,c, data[ 3], MD5_cnt_loc[53], I2); + MD5_STEP(I, c,d,a,b, data[10], MD5_cnt_loc[54], I3); + MD5_STEP(I, b,c,d,a, data[ 1], MD5_cnt_loc[55], I4); + MD5_STEP(I, a,b,c,d, data[ 8], MD5_cnt_loc[56], I1); + MD5_STEP(I, d,a,b,c, data[15], MD5_cnt_loc[57], I2); + MD5_STEP(I, c,d,a,b, data[ 6], MD5_cnt_loc[58], I3); + MD5_STEP(I, b,c,d,a, data[13], MD5_cnt_loc[59], I4); + MD5_STEP(I, a,b,c,d, data[ 4], MD5_cnt_loc[60], I1); + MD5_STEP(I, d,a,b,c, data[11], MD5_cnt_loc[61], I2); + MD5_STEP(I, c,d,a,b, data[ 2], MD5_cnt_loc[62], I3); + MD5_STEP(I, b,c,d,a, data[ 9], MD5_cnt_loc[63], I4); + + /* update digest */ + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + } +} + +#endif +#endif /* IPP_ALG_HASH_MD5 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmessage.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmessage.c new file mode 100644 index 0000000..5a73822 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmessage.c @@ -0,0 +1,145 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// General Functionality +// +// Contents: +// ippsHashMessage() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_func.h" +#include "pcptool.h" + +/*F* +// Name: ippsHashMessage +// +// Purpose: Hash of the whole message. +// +// Returns: Reason: +// ippStsNullPtrErr pMD == NULL +// pMsg == NULL but len!=0 +// ippStsLengthErr len <0 +// ippStsNotSupportedModeErr if hashAlg is not match to supported hash alg +// ippStsNoErr no errors +// +// Parameters: +// pMsg pointer to the input message +// len input message length +// pMD address of the output digest +// hashAlg hash alg ID +// +*F*/ +IPPFUN(IppStatus, ippsHashMessage,(const Ipp8u* pMsg, int len, Ipp8u* pMD, IppHashAlgId hashAlg)) +{ + /* get algorithm id */ + hashAlg = cpValidHashAlg(hashAlg); + /* test hash alg */ + IPP_BADARG_RET(ippHashAlg_Unknown==hashAlg, ippStsNotSupportedModeErr); + + /* test digest pointer */ + IPP_BAD_PTR1_RET(pMD); + /* test message length */ + IPP_BADARG_RET((len<0), ippStsLengthErr); + /* test message pointer */ + IPP_BADARG_RET((len && !pMsg), ippStsNullPtrErr); + + { + /* processing function and parameter */ + cpHashProc hashFunc = cpHashProcFunc[hashAlg]; + const void* pParam = cpHashProcFuncOpt[hashAlg]; + + /* attributes */ + const cpHashAttr* pAttr = &cpHashAlgAttr[hashAlg]; + int mbs = pAttr->msgBlkSize; /* data block size */ + int ivSize = pAttr->ivSize; /* size of hash's IV */ + int hashSize = pAttr->hashSize; /* hash size */ + int msgLenRepSize = pAttr->msgLenRepSize; /* length of the message representation */ + + /* message bitlength representation */ + Ipp64u msgLenBits = (Ipp64u)len*8; + /* length of main message part */ + int msgLenBlks = len & (-mbs); + /* rest of message length */ + int msgLenRest = len - msgLenBlks; + + /* end of message buffer */ + Ipp8u buffer[MBS_HASH_MAX*2]; + int bufferLen = (msgLenRest < (mbs-msgLenRepSize))? mbs : mbs*2; + + /* init hash */ + cpHash hash; + const Ipp8u* iv = cpHashIV[hashAlg]; + CopyBlock(iv, hash, ivSize); + + /*construct last messge block(s) */ + #define MSG_LEN_REP (sizeof(Ipp64u)) + + /* copy end of message */ + CopyBlock(pMsg+len-msgLenRest, buffer, msgLenRest); + /* end of message bit */ + buffer[msgLenRest++] = 0x80; + /* padd buffer */ + PadBlock(0, buffer+msgLenRest, (cpSize)(bufferLen-msgLenRest-(int)MSG_LEN_REP)); + /* copy message bitlength representation */ + if(ippHashAlg_MD5!=hashAlg) + msgLenBits = ENDIANNESS64(msgLenBits); + ((Ipp64u*)(buffer+bufferLen))[-1] = msgLenBits; + + #undef MSG_LEN_REP + + /* message processing */ + if(msgLenBlks) + hashFunc(hash, pMsg, msgLenBlks, pParam); + hashFunc(hash, buffer, bufferLen, pParam); + + /* store digest into the user buffer (remember digest in big endian) */ + if(msgLenRepSize > (int)(sizeof(Ipp64u))) { + /* ippHashAlg_SHA384, ippHashAlg_SHA512, ippHashAlg_SHA512_224 and ippHashAlg_SHA512_256 */ + hash[0] = ENDIANNESS64(hash[0]); + hash[1] = ENDIANNESS64(hash[1]); + hash[2] = ENDIANNESS64(hash[2]); + hash[3] = ENDIANNESS64(hash[3]); + hash[4] = ENDIANNESS64(hash[4]); + hash[5] = ENDIANNESS64(hash[5]); + hash[6] = ENDIANNESS64(hash[6]); + hash[7] = ENDIANNESS64(hash[7]); + } + else if(ippHashAlg_MD5!=hashAlg) { + /* ippHashAlg_SHA1, ippHashAlg_SHA224, ippHashAlg_SHA256 and ippHashAlg_SM3 */ + ((Ipp32u*)hash)[0] = ENDIANNESS32(((Ipp32u*)hash)[0]); + ((Ipp32u*)hash)[1] = ENDIANNESS32(((Ipp32u*)hash)[1]); + ((Ipp32u*)hash)[2] = ENDIANNESS32(((Ipp32u*)hash)[2]); + ((Ipp32u*)hash)[3] = ENDIANNESS32(((Ipp32u*)hash)[3]); + ((Ipp32u*)hash)[4] = ENDIANNESS32(((Ipp32u*)hash)[4]); + ((Ipp32u*)hash)[5] = ENDIANNESS32(((Ipp32u*)hash)[5]); + ((Ipp32u*)hash)[6] = ENDIANNESS32(((Ipp32u*)hash)[6]); + ((Ipp32u*)hash)[7] = ENDIANNESS32(((Ipp32u*)hash)[7]); + } + CopyBlock(hash, pMD, hashSize); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmessage_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmessage_rmf.c new file mode 100644 index 0000000..389557b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmessage_rmf.c @@ -0,0 +1,86 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// Generalized Functionality +// +// Contents: +// ippsHashMessage_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsHashMessage_rmf +// +// Purpose: Hash of the whole message. +// +// Returns: Reason: +// ippStsNullPtrErr pMD == NULL +// pMsg == NULL but len!=0 +// ippStsLengthErr len <0 +// ippStsNoErr no errors +// +// Parameters: +// pMsg pointer to the input message +// len input message length +// pMD address of the output digest +// pMethod hash methods +// +*F*/ +IPPFUN(IppStatus, ippsHashMessage_rmf,(const Ipp8u* pMsg, int len, Ipp8u* pMD, const IppsHashMethod* pMethod)) +{ + /* test method pointer */ + IPP_BAD_PTR1_RET(pMethod); + /* test digest pointer */ + IPP_BAD_PTR1_RET(pMD); + /* test message length */ + IPP_BADARG_RET(0>len, ippStsLengthErr); + IPP_BADARG_RET((len && !pMsg), ippStsNullPtrErr); + + { + /* message length in the multiple MBS and the rest */ + int msgLenBlks = len &(-pMethod->msgBlkSize); + int msgLenRest = len - msgLenBlks; + + /* init hash */ + DigestSHA512 hash; + pMethod->hashInit(hash); + + /* process main part of the message */ + if(msgLenBlks) { + pMethod->hashUpdate(hash, pMsg, msgLenBlks); + pMsg += msgLenBlks; + } + cpFinalize_rmf(hash, + pMsg, msgLenRest, + (Ipp64u)len, 0, + pMethod); + + pMethod->hashOctStr(pMD, hash); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_md5.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_md5.c new file mode 100644 index 0000000..f6c9d49 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_md5.c @@ -0,0 +1,67 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to MD5 +// (derived from the RSA Data Security, Inc. MD5 Message-Digest Algorithm) +// +// Equivalent code is available from RFC 1321. +// +// Contents: +// ippsHashMethod_MD5() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpmd5stuff.h" + +/*F* +// Name: ippsHashMethod_MD5 +// +// Purpose: Return MD5 method. +// +// Returns: +// Pointer to MD5 hash-method. +// +*F*/ +IPPFUN( const IppsHashMethod*, ippsHashMethod_MD5, (void) ) +{ + static IppsHashMethod method = { + ippHashAlg_MD5, + IPP_MD5_DIGEST_BITSIZE/8, + MBS_MD5, + MLR_MD5, + 0, + 0, + 0, + 0 + }; + + method.hashInit = md5_hashInit; + method.hashUpdate = md5_hashUpdate; + method.hashOctStr = md5_hashOctString; + method.msgLenRep = md5_msgRep; + + return &method; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_rmf.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_rmf.h new file mode 100644 index 0000000..bf4814f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_rmf.h @@ -0,0 +1,45 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// Internal Definitions and Internal Functions Prototypes +// +*/ +#if !defined(_PCP_HASH_METHOD_RMF_H) +#define _PCP_HASH_METHOD_RMF_H + +/* hash alg methods */ +IPP_OWN_FUNPTR (void, hashInitF, (void* pHash)) +IPP_OWN_FUNPTR (void, hashUpdateF, (void* pHash, const Ipp8u* pMsg, int msgLen)) +IPP_OWN_FUNPTR (void, hashOctStrF, (Ipp8u* pDst, void* pHash)) +IPP_OWN_FUNPTR (void, msgLenRepF, (Ipp8u* pDst, Ipp64u lenLo, Ipp64u lenHi)) + +typedef struct _cpHashMethod_rmf { + IppHashAlgId hashAlgId; /* algorithm ID */ + int hashLen; /* hash length in bytes */ + int msgBlkSize; /* message blkock size in bytes */ + int msgLenRepSize; /* length of processed msg length representation in bytes */ + hashInitF hashInit; /* set initial hash value */ + hashUpdateF hashUpdate; /* hash compressor */ + hashOctStrF hashOctStr; /* convert hash into oct string */ + msgLenRepF msgLenRep; /* processed mgs length representation */ +} cpHashMethod_rmf; + +#endif /* _PCP_HASH_METHOD_RMF_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha1.c new file mode 100644 index 0000000..84ffbc6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha1.c @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA1 +// +// Contents: +// ippsHashMethod_SHA1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha1stuff.h" + +/*F* +// Name: ippsHashMethod_SHA1 +// +// Purpose: Return SHA1 method. +// +// Returns: +// Pointer to SHA1 hash-method. +// +*F*/ +IPPFUN( const IppsHashMethod*, ippsHashMethod_SHA1, (void) ) +{ + static IppsHashMethod method = { + ippHashAlg_SHA1, + IPP_SHA1_DIGEST_BITSIZE/8, + MBS_SHA1, + MLR_SHA1, + 0, + 0, + 0, + 0 + }; + + method.hashInit = sha1_hashInit; + method.hashUpdate = sha1_hashUpdate; + method.hashOctStr = sha1_hashOctString; + method.msgLenRep = sha1_msgRep; + + return &method; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha1_ni.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha1_ni.c new file mode 100644 index 0000000..1c15e73 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha1_ni.c @@ -0,0 +1,69 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA1 +// +// Contents: +// ippsHashMethod_SHA1_NI() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha1stuff.h" + +/*F* +// Name: ippsHashMethod_SHA1_NI +// +// Purpose: Return SHA1 method (using the Intel® Secure Hash Algorithm - New Instructions (Intel® SHA-NI) instruction set). +// +// Returns: +// Pointer to SHA1 hash-method (using the Intel SHA-NI instruction set). +// +*F*/ + +IPPFUN( const IppsHashMethod*, ippsHashMethod_SHA1_NI, (void) ) +{ + #if (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_ || _SHA_NI_ENABLING_==_FEATURE_ON_) + static IppsHashMethod method = { + ippHashAlg_SHA1, + IPP_SHA1_DIGEST_BITSIZE/8, + MBS_SHA1, + MLR_SHA1, + 0, + 0, + 0, + 0 + }; + + method.hashInit = sha1_hashInit; + method.hashUpdate = sha1_ni_hashUpdate; + method.hashOctStr = sha1_hashOctString; + method.msgLenRep = sha1_msgRep; + + return &method; + #else + return NULL; + #endif +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha1_tt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha1_tt.c new file mode 100644 index 0000000..ca15272 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha1_tt.c @@ -0,0 +1,73 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA1 +// +// Contents: +// ippsHashMethod_SHA1_TT() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha1stuff.h" + +/*F* +// Name: ippsHashMethod_SHA1_TT +// +// Purpose: Return SHA1 method +// (using the Intel® Secure Hash Algorithm - New Instructions (Intel® SHA-NI) instructions set +// if it is available at run time) +// +// Returns: +// Pointer to SHA1 hash-method +// (using the Intel SHA-NI instructions set +// if it is available at run time) +// +*F*/ + +IPPFUN( const IppsHashMethod*, ippsHashMethod_SHA1_TT, (void) ) +{ + static IppsHashMethod method = { + ippHashAlg_SHA1, + IPP_SHA1_DIGEST_BITSIZE/8, + MBS_SHA1, + MLR_SHA1, + 0, + 0, + 0, + 0 + }; + + method.hashInit = sha1_hashInit; + method.hashUpdate = sha1_hashUpdate; + method.hashOctStr = sha1_hashOctString; + method.msgLenRep = sha1_msgRep; + + #if (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_ || _SHA_NI_ENABLING_==_FEATURE_ON_) + if(IsFeatureEnabled(ippCPUID_SHA)) + method.hashUpdate = sha1_ni_hashUpdate; + #endif + return &method; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha224.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha224.c new file mode 100644 index 0000000..cad8c88 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha224.c @@ -0,0 +1,65 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsHashMethod_SHA224() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +/*F* +// Name: ippsHashMethod_SHA224 +// +// Purpose: Return SHA224 method. +// +// Returns: +// Pointer to SHA224 hash-method. +// +*F*/ + +IPPFUN( const IppsHashMethod*, ippsHashMethod_SHA224, (void) ) +{ + static IppsHashMethod method = { + ippHashAlg_SHA224, + IPP_SHA224_DIGEST_BITSIZE/8, + MBS_SHA256, + MLR_SHA256, + 0, + 0, + 0, + 0 + }; + + method.hashInit = sha224_hashInit; + method.hashUpdate = sha256_hashUpdate; + method.hashOctStr = sha224_hashOctString; + method.msgLenRep = sha256_msgRep; + + return &method; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha224_ni.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha224_ni.c new file mode 100644 index 0000000..e1133f4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha224_ni.c @@ -0,0 +1,70 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsHashMethod_SHA224_NI() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +/*F* +// Name: ippsHashMethod_SHA224_NI +// +// Purpose: Return SHA224 method (using the Intel® Secure Hash Algorithm - New Instructions (Intel® SHA-NI) instruction set). +// +// Returns: +// Pointer to SHA224 hash-method (using the Intel SHA-NI instruction set). +// +*F*/ + + +IPPFUN( const IppsHashMethod*, ippsHashMethod_SHA224_NI, (void) ) +{ + #if (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_ || _SHA_NI_ENABLING_==_FEATURE_ON_) + static IppsHashMethod method = { + ippHashAlg_SHA224, + IPP_SHA224_DIGEST_BITSIZE/8, + MBS_SHA256, + MLR_SHA256, + 0, + 0, + 0, + 0 + }; + + method.hashInit = sha224_hashInit; + method.hashUpdate = sha256_ni_hashUpdate; + method.hashOctStr = sha224_hashOctString; + method.msgLenRep = sha256_msgRep; + + return &method; + #else + return NULL; + #endif +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha224_tt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha224_tt.c new file mode 100644 index 0000000..5b17dd8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha224_tt.c @@ -0,0 +1,73 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsHashMethod_SHA224_TT() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +/*F* +// Name: ippsHashMethod_SHA224_TT +// +// Purpose: Return SHA224 method +// (using the Intel® Secure Hash Algorithm - New Instructions (Intel® SHA-NI) instructions set +// if it is available at run time) +// +// Returns: +// Pointer to SHA224 hash-method +// (using the Intel SHA-NI instructions set +// if it is available at run time) +// +*F*/ + +IPPFUN( const IppsHashMethod*, ippsHashMethod_SHA224_TT, (void) ) +{ + static IppsHashMethod method = { + ippHashAlg_SHA224, + IPP_SHA224_DIGEST_BITSIZE/8, + MBS_SHA256, + MLR_SHA256, + 0, + 0, + 0, + 0 + }; + + method.hashInit = sha224_hashInit; + method.hashUpdate = sha256_hashUpdate; + method.hashOctStr = sha224_hashOctString; + method.msgLenRep = sha256_msgRep; + + #if (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_ || _SHA_NI_ENABLING_==_FEATURE_ON_) + if(IsFeatureEnabled(ippCPUID_SHA)) + method.hashUpdate = sha256_ni_hashUpdate; + #endif + return &method; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha256.c new file mode 100644 index 0000000..efe78c5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha256.c @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsHashMethod_SHA256() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +/*F* +// Name: ippsHashMethod_SHA256 +// +// Purpose: Return SHA256 method. +// +// Returns: +// Pointer to SHA256 hash-method. +// +*F*/ +IPPFUN( const IppsHashMethod*, ippsHashMethod_SHA256, (void) ) +{ + static IppsHashMethod method = { + ippHashAlg_SHA256, + IPP_SHA256_DIGEST_BITSIZE/8, + MBS_SHA256, + MLR_SHA256, + 0, + 0, + 0, + 0 + }; + + method.hashInit = sha256_hashInit; + method.hashUpdate = sha256_hashUpdate; + method.hashOctStr = sha256_hashOctString; + method.msgLenRep = sha256_msgRep; + + return &method; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha256_ni.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha256_ni.c new file mode 100644 index 0000000..fd43666 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha256_ni.c @@ -0,0 +1,69 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsHashMethod_SHA256_NI() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +/*F* +// Name: ippsHashMethod_SHA256_NI +// +// Purpose: Return SHA256 method (using the Intel® Secure Hash Algorithm - New Instructions (Intel® SHA-NI) instruction set). +// +// Returns: +// Pointer to SHA256 hash-method (using the Intel SHA-NI instruction set). +// +*F*/ + +IPPFUN( const IppsHashMethod*, ippsHashMethod_SHA256_NI, (void) ) +{ + #if (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_ || _SHA_NI_ENABLING_==_FEATURE_ON_) + static IppsHashMethod method = { + ippHashAlg_SHA256, + IPP_SHA256_DIGEST_BITSIZE/8, + MBS_SHA256, + MLR_SHA256, + 0, + 0, + 0, + 0 + }; + + method.hashInit = sha256_hashInit; + method.hashUpdate = sha256_ni_hashUpdate; + method.hashOctStr = sha256_hashOctString; + method.msgLenRep = sha256_msgRep; + + return &method; + #else + return NULL; + #endif +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha256_tt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha256_tt.c new file mode 100644 index 0000000..0f8f6bb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha256_tt.c @@ -0,0 +1,73 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsHashMethod_SHA256_TT() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +/*F* +// Name: ippsHashMethod_SHA256_TT +// +// Purpose: Return SHA256 method +// (using the Intel® Secure Hash Algorithm - New Instructions (Intel® SHA-NI) instructions set +// if it is available at run time) +// +// Returns: +// Pointer to SHA256 hash-method +// (using the Intel SHA-NI instructions set +// if it is available at run time) +// +*F*/ + +IPPFUN( const IppsHashMethod*, ippsHashMethod_SHA256_TT, (void) ) +{ + static IppsHashMethod method = { + ippHashAlg_SHA256, + IPP_SHA256_DIGEST_BITSIZE/8, + MBS_SHA256, + MLR_SHA256, + 0, + 0, + 0, + 0 + }; + + method.hashInit = sha256_hashInit; + method.hashUpdate = sha256_hashUpdate; + method.hashOctStr = sha256_hashOctString; + method.msgLenRep = sha256_msgRep; + + #if (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_ || _SHA_NI_ENABLING_==_FEATURE_ON_) + if(IsFeatureEnabled(ippCPUID_SHA)) + method.hashUpdate = sha256_ni_hashUpdate; + #endif + return &method; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha384.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha384.c new file mode 100644 index 0000000..bdd0082 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha384.c @@ -0,0 +1,65 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsHashMethod_SHA384() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha512stuff.h" + +/*F* +// Name: ippsHashMethod_SHA384 +// +// Purpose: Return SHA384 method. +// +// Returns: +// Pointer to SHA384 hash-method. +// +*F*/ + +IPPFUN( const IppsHashMethod*, ippsHashMethod_SHA384, (void) ) +{ + static IppsHashMethod method = { + ippHashAlg_SHA384, + IPP_SHA384_DIGEST_BITSIZE/8, + MBS_SHA512, + MLR_SHA512, + 0, + 0, + 0, + 0 + }; + + method.hashInit = sha512_384_hashInit; + method.hashUpdate = sha512_hashUpdate; + method.hashOctStr = sha512_384_hashOctString; + method.msgLenRep = sha512_msgRep; + + return &method; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha512.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha512.c new file mode 100644 index 0000000..a612672 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha512.c @@ -0,0 +1,65 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsHashMethod_SHA512() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha512stuff.h" + +/*F* +// Name: ippsHashMethod_SHA512 +// +// Purpose: Return SHA512 method. +// +// Returns: +// Pointer to SHA512 hash-method. +// +*F*/ + +IPPFUN( const IppsHashMethod*, ippsHashMethod_SHA512, (void) ) +{ + static IppsHashMethod method = { + ippHashAlg_SHA512, + IPP_SHA512_DIGEST_BITSIZE/8, + MBS_SHA512, + MLR_SHA512, + 0, + 0, + 0, + 0 + }; + + method.hashInit = sha512_hashInit; + method.hashUpdate = sha512_hashUpdate; + method.hashOctStr = sha512_hashOctString; + method.msgLenRep = sha512_msgRep; + + return &method; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha512_224.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha512_224.c new file mode 100644 index 0000000..6b7e887 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha512_224.c @@ -0,0 +1,65 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsHashMethod_SHA512_224() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha512stuff.h" + +/*F* +// Name: ippsHashMethod_SHA512_224 +// +// Purpose: Return SHA512_224 method. +// +// Returns: +// Pointer to SHA512_224 hash-method. +// +*F*/ + +IPPFUN( const IppsHashMethod*, ippsHashMethod_SHA512_224, (void) ) +{ + static IppsHashMethod method = { + ippHashAlg_SHA512_224, + IPP_SHA224_DIGEST_BITSIZE/8, + MBS_SHA512, + MLR_SHA512, + 0, + 0, + 0, + 0 + }; + + method.hashInit = sha512_224_hashInit; + method.hashUpdate = sha512_hashUpdate; + method.hashOctStr = sha512_224_hashOctString; + method.msgLenRep = sha512_msgRep; + + return &method; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha512_256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha512_256.c new file mode 100644 index 0000000..a7c5144 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sha512_256.c @@ -0,0 +1,65 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsHashMethod_SHA512_256() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha512stuff.h" + +/*F* +// Name: ippsHashMethod_SHA512_256 +// +// Purpose: Return SHA512_256 method. +// +// Returns: +// Pointer to SHA512_256 hash-method. +// +*F*/ + +IPPFUN( const IppsHashMethod*, ippsHashMethod_SHA512_256, (void) ) +{ + static IppsHashMethod method = { + ippHashAlg_SHA512_256, + IPP_SHA256_DIGEST_BITSIZE/8, + MBS_SHA512, + MLR_SHA512, + 0, + 0, + 0, + 0 + }; + + method.hashInit = sha512_256_hashInit; + method.hashUpdate = sha512_hashUpdate; + method.hashOctStr = sha512_256_hashOctString; + method.msgLenRep = sha512_msgRep; + + return &method; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sm3.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sm3.c new file mode 100644 index 0000000..cc23353 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethod_sm3.c @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SM3 +// +// Contents: +// ippsHashMethod_SM3() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsm3stuff.h" + +/*F* +// Name: ippsHashMethod_SM3 +// +// Purpose: Return SM3 method. +// +// Returns: +// Pointer to SM3 hash-method. +// +*F*/ +IPPFUN( const IppsHashMethod*, ippsHashMethod_SM3, (void) ) +{ + static IppsHashMethod method = { + ippHashAlg_SM3, + IPP_SM3_DIGEST_BITSIZE/8, + MBS_SM3, + MLR_SM3, + 0, + 0, + 0, + 0 + }; + + method.hashInit = sm3_hashInit; + method.hashUpdate = sm3_hashUpdate; + method.hashOctStr = sm3_hashOctString; + method.msgLenRep = sm3_msgRep; + + return &method; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodgetsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodgetsize.c new file mode 100644 index 0000000..c55985d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodgetsize.c @@ -0,0 +1,55 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// Generalized Functionality +// +// Contents: +// ippsHashMethodGetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphashmethod_rmf.h" + +/*F* +// Name: ippsHashMethodGetSize +// +// Purpose: Returns size (bytes) of IppsHashMethod structure. +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to state size +// +*F*/ +IPPFUN(IppStatus, ippsHashMethodGetSize,(int* pSize)) +{ + /* test pointers */ + IPP_BAD_PTR1_RET(pSize); + + *pSize = sizeof(IppsHashMethod); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_md5.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_md5.c new file mode 100644 index 0000000..e71c0ae --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_md5.c @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to MD5 +// (derived from the RSA Data Security, Inc. MD5 Message-Digest Algorithm) +// +// Equivalent code is available from RFC 1321. +// +// Contents: +// ippsHashMethodSet_MD5() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpmd5stuff.h" + +/*F* +// Name: ippsHashMethodSet_MD5 +// +// Purpose: Setup MD5 method. +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL +// ippStsNoErr no errors +// +*F*/ +IPPFUN( IppStatus, ippsHashMethodSet_MD5, (IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR1_RET(pMethod); + + pMethod->hashAlgId = ippHashAlg_MD5; + pMethod->hashLen = IPP_MD5_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_MD5; + pMethod->msgLenRepSize = MLR_MD5; + pMethod->hashInit = md5_hashInit; + pMethod->hashUpdate = md5_hashUpdate; + pMethod->hashOctStr = md5_hashOctString; + pMethod->msgLenRep = md5_msgRep; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha1.c new file mode 100644 index 0000000..6255bc3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha1.c @@ -0,0 +1,62 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA1 +// +// Contents: +// ippsHashMethodSet_SHA1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha1stuff.h" + +/*F* +// Name: ippsHashMethodSet_SHA1 +// +// Purpose: Setup SHA1 method. +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL +// ippStsNoErr no errors + +// +*F*/ +IPPFUN( IppStatus, ippsHashMethodSet_SHA1, (IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR1_RET(pMethod); + + pMethod->hashAlgId = ippHashAlg_SHA1; + pMethod->hashLen = IPP_SHA1_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA1; + pMethod->msgLenRepSize = MLR_SHA1; + pMethod->hashInit = sha1_hashInit; + pMethod->hashUpdate = sha1_hashUpdate; + pMethod->hashOctStr = sha1_hashOctString; + pMethod->msgLenRep = sha1_msgRep; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha1_ni.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha1_ni.c new file mode 100644 index 0000000..f798113 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha1_ni.c @@ -0,0 +1,76 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA1 +// +// Contents: +// ippsHashMethodSet_SHA1_NI() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha1stuff.h" + +/*F* +// Name: ippsHashMethodSet_SHA1_NI +// +// Purpose: Setup SHA1 method (using the Intel® Secure Hash Algorithm - New Instructions (Intel® SHA-NI) instruction set). +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL +// ippStsNotSupportedModeErr mode disabled by configuration +// ippStsNoErr no errors +// +*F*/ + +IPPFUN( IppStatus, ippsHashMethodSet_SHA1_NI, (IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR1_RET(pMethod); + +#if (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_ || _SHA_NI_ENABLING_==_FEATURE_ON_) + pMethod->hashAlgId = ippHashAlg_SHA1; + pMethod->hashLen = IPP_SHA1_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA1; + pMethod->msgLenRepSize = MLR_SHA1; + pMethod->hashInit = sha1_hashInit; + pMethod->hashUpdate = sha1_ni_hashUpdate; + pMethod->hashOctStr = sha1_hashOctString; + pMethod->msgLenRep = sha1_msgRep; + + return ippStsNoErr; +#else + pMethod->hashAlgId = ippHashAlg_Unknown; + pMethod->hashLen = 0; + pMethod->msgBlkSize = 0; + pMethod->msgLenRepSize = 0; + pMethod->hashInit = 0; + pMethod->hashUpdate = 0; + pMethod->hashOctStr = 0; + pMethod->msgLenRep = 0; + + return ippStsNotSupportedModeErr; +#endif +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha1_tt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha1_tt.c new file mode 100644 index 0000000..940b98e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha1_tt.c @@ -0,0 +1,69 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA1 +// +// Contents: +// ippsHashMethodSet_SHA1_TT() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha1stuff.h" + +/*F* +// Name: ippsHashMethodSet_SHA1_TT +// +// Purpose: Setup SHA1 method +// (using the Intel® Secure Hash Algorithm - New Instructions (Intel® SHA-NI) instructions set +// if it is available at run time) +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL +// ippStsNoErr no errors +// +*F*/ + +IPPFUN( IppStatus, ippsHashMethodSet_SHA1_TT, (IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR1_RET(pMethod); + + pMethod->hashAlgId = ippHashAlg_SHA1; + pMethod->hashLen = IPP_SHA1_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA1; + pMethod->msgLenRepSize = MLR_SHA1; + pMethod->hashInit = sha1_hashInit; + pMethod->hashUpdate = sha1_hashUpdate; + pMethod->hashOctStr = sha1_hashOctString; + pMethod->msgLenRep = sha1_msgRep; + +#if (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_ || _SHA_NI_ENABLING_==_FEATURE_ON_) + if(IsFeatureEnabled(ippCPUID_SHA)) + pMethod->hashUpdate = sha1_ni_hashUpdate; +#endif + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha224.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha224.c new file mode 100644 index 0000000..1e9231d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha224.c @@ -0,0 +1,62 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsHashMethodSet_SHA224() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +/*F* +// Name: ippsHashMethodSet_SHA224 +// +// Purpose: Setup SHA224 method. +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL +// ippStsNoErr no errors +// +*F*/ + +IPPFUN( IppStatus, ippsHashMethodSet_SHA224, (IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR1_RET(pMethod); + + pMethod->hashAlgId = ippHashAlg_SHA224; + pMethod->hashLen = IPP_SHA224_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA256; + pMethod->msgLenRepSize = MLR_SHA256; + pMethod->hashInit = sha224_hashInit; + pMethod->hashUpdate = sha256_hashUpdate; + pMethod->hashOctStr = sha224_hashOctString; + pMethod->msgLenRep = sha256_msgRep; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha224_ni.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha224_ni.c new file mode 100644 index 0000000..6fa60e3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha224_ni.c @@ -0,0 +1,77 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsHashMethodSet_SHA224_NI() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +/*F* +// Name: ippsHashMethodSet_SHA224_NI +// +// Purpose: Setup SHA224 method (using the Intel® Secure Hash Algorithm - New Instructions (Intel® SHA-NI) instruction set). +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL +// ippStsNotSupportedModeErr mode disabled by configuration +// ippStsNoErr no errors +// +*F*/ + + +IPPFUN( IppStatus, ippsHashMethodSet_SHA224_NI, (IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR1_RET(pMethod); + +#if (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_ || _SHA_NI_ENABLING_==_FEATURE_ON_) + pMethod->hashAlgId = ippHashAlg_SHA224; + pMethod->hashLen = IPP_SHA224_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA256; + pMethod->msgLenRepSize = MLR_SHA256; + pMethod->hashInit = sha224_hashInit; + pMethod->hashUpdate = sha256_ni_hashUpdate; + pMethod->hashOctStr = sha224_hashOctString; + pMethod->msgLenRep = sha256_msgRep; + + return ippStsNoErr; +#else + pMethod->hashAlgId = ippHashAlg_Unknown; + pMethod->hashLen = 0; + pMethod->msgBlkSize = 0; + pMethod->msgLenRepSize = 0; + pMethod->hashInit = 0; + pMethod->hashUpdate = 0; + pMethod->hashOctStr = 0; + pMethod->msgLenRep = 0; + + return ippStsNotSupportedModeErr; +#endif +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha224_tt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha224_tt.c new file mode 100644 index 0000000..5e20bbe --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha224_tt.c @@ -0,0 +1,69 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsHashMethodSet_SHA224_TT() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +/*F* +// Name: ippsHashMethodSet_SHA224_TT +// +// Purpose: Setup SHA224 method +// (using the Intel® Secure Hash Algorithm - New Instructions (Intel® SHA-NI) instructions set +// if it is available at run time) +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL +// ippStsNoErr no errors +// +*F*/ + +IPPFUN( IppStatus, ippsHashMethodSet_SHA224_TT, (IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR1_RET(pMethod); + + pMethod->hashAlgId = ippHashAlg_SHA224; + pMethod->hashLen = IPP_SHA224_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA256; + pMethod->msgLenRepSize = MLR_SHA256; + pMethod->hashInit = sha224_hashInit; + pMethod->hashUpdate = sha256_hashUpdate; + pMethod->hashOctStr = sha224_hashOctString; + pMethod->msgLenRep = sha256_msgRep; + +#if (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_ || _SHA_NI_ENABLING_==_FEATURE_ON_) + if(IsFeatureEnabled(ippCPUID_SHA)) + pMethod->hashUpdate = sha256_ni_hashUpdate; +#endif + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha256.c new file mode 100644 index 0000000..3ffcd28 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha256.c @@ -0,0 +1,61 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsHashMethodSet_SHA256() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +/*F* +// Name: ippsHashMethodSet_SHA256 +// +// Purpose: Setup SHA256 method. +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL +// ippStsNoErr no errors +// +*F*/ +IPPFUN( IppStatus, ippsHashMethodSet_SHA256, (IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR1_RET(pMethod); + + pMethod->hashAlgId = ippHashAlg_SHA256; + pMethod->hashLen = IPP_SHA256_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA256; + pMethod->msgLenRepSize = MLR_SHA256; + pMethod->hashInit = sha256_hashInit; + pMethod->hashUpdate = sha256_hashUpdate; + pMethod->hashOctStr = sha256_hashOctString; + pMethod->msgLenRep = sha256_msgRep; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha256_ni.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha256_ni.c new file mode 100644 index 0000000..7da64de --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha256_ni.c @@ -0,0 +1,76 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsHashMethodSet_SHA256_NI() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +/*F* +// Name: ippsHashMethodSet_SHA256_NI +// +// Purpose: Setup SHA256 method (using the Intel® Secure Hash Algorithm - New Instructions (Intel® SHA-NI) instruction set). +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL +// ippStsNotSupportedModeErr mode disabled by configuration +// ippStsNoErr no errors +// +*F*/ + +IPPFUN( IppStatus, ippsHashMethodSet_SHA256_NI, (IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR1_RET(pMethod); + +#if (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_ || _SHA_NI_ENABLING_==_FEATURE_ON_) + pMethod->hashAlgId = ippHashAlg_SHA256; + pMethod->hashLen = IPP_SHA256_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA256; + pMethod->msgLenRepSize = MLR_SHA256; + pMethod->hashInit = sha256_hashInit; + pMethod->hashUpdate = sha256_ni_hashUpdate; + pMethod->hashOctStr = sha256_hashOctString; + pMethod->msgLenRep = sha256_msgRep; + + return ippStsNoErr; +#else + pMethod->hashAlgId = ippHashAlg_Unknown; + pMethod->hashLen = 0; + pMethod->msgBlkSize = 0; + pMethod->msgLenRepSize = 0; + pMethod->hashInit = 0; + pMethod->hashUpdate = 0; + pMethod->hashOctStr = 0; + pMethod->msgLenRep = 0; + + return ippStsNotSupportedModeErr; +#endif +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha256_tt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha256_tt.c new file mode 100644 index 0000000..e77c29a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha256_tt.c @@ -0,0 +1,69 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsHashMethodSet_SHA256_TT() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +/*F* +// Name: ippsHashMethodSet_SHA256_TT +// +// Purpose: Setup SHA256 method +// (using the Intel® Secure Hash Algorithm - New Instructions (Intel® SHA-NI) instructions set +// if it is available at run time) +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL +// ippStsNoErr no errors +// +*F*/ + +IPPFUN( IppStatus, ippsHashMethodSet_SHA256_TT, (IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR1_RET(pMethod); + + pMethod->hashAlgId = ippHashAlg_SHA256; + pMethod->hashLen = IPP_SHA256_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA256; + pMethod->msgLenRepSize = MLR_SHA256; + pMethod->hashInit = sha256_hashInit; + pMethod->hashUpdate = sha256_hashUpdate; + pMethod->hashOctStr = sha256_hashOctString; + pMethod->msgLenRep = sha256_msgRep; + +#if (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_ || _SHA_NI_ENABLING_==_FEATURE_ON_) + if(IsFeatureEnabled(ippCPUID_SHA)) + pMethod->hashUpdate = sha256_ni_hashUpdate; +#endif + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha384.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha384.c new file mode 100644 index 0000000..8a69082 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha384.c @@ -0,0 +1,62 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsHashMethodSet_SHA384() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha512stuff.h" + +/*F* +// Name: ippsHashMethodSet_SHA384 +// +// Purpose: Setup SHA384 method. +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL +// ippStsNoErr no errors +// +*F*/ + +IPPFUN( IppStatus, ippsHashMethodSet_SHA384, (IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR1_RET(pMethod); + + pMethod->hashAlgId = ippHashAlg_SHA384; + pMethod->hashLen = IPP_SHA384_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA512; + pMethod->msgLenRepSize = MLR_SHA512; + pMethod->hashInit = sha512_384_hashInit; + pMethod->hashUpdate = sha512_hashUpdate; + pMethod->hashOctStr = sha512_384_hashOctString; + pMethod->msgLenRep = sha512_msgRep; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha512.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha512.c new file mode 100644 index 0000000..422f64d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha512.c @@ -0,0 +1,61 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsHashMethodSet_SHA512() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha512stuff.h" + +/*F* +// Name: ippsHashMethodSet_SHA512 +// +// Purpose: Setup SHA512 method. +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL +// ippStsNoErr no errors +// +*F*/ + +IPPFUN( IppStatus, ippsHashMethodSet_SHA512, (IppsHashMethod* pMethod) ) +{ + IPP_BAD_PTR1_RET(pMethod); + + pMethod->hashAlgId = ippHashAlg_SHA512; + pMethod->hashLen = IPP_SHA512_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA512; + pMethod->msgLenRepSize = MLR_SHA512; + pMethod->hashInit = sha512_hashInit; + pMethod->hashUpdate = sha512_hashUpdate; + pMethod->hashOctStr = sha512_hashOctString; + pMethod->msgLenRep = sha512_msgRep; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha512_224.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha512_224.c new file mode 100644 index 0000000..2978ffb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha512_224.c @@ -0,0 +1,62 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsHashMethodSet_SHA512_224() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha512stuff.h" + +/*F* +// Name: ippsHashMethodSet_SHA512_224 +// +// Purpose: Return SHA512_224 method. +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL +// ippStsNoErr no errors +// +*F*/ + +IPPFUN( IppStatus, ippsHashMethodSet_SHA512_224, (IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR1_RET(pMethod); + + pMethod->hashAlgId = ippHashAlg_SHA512_224; + pMethod->hashLen = IPP_SHA224_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA512; + pMethod->msgLenRepSize = MLR_SHA512; + pMethod->hashInit = sha512_224_hashInit; + pMethod->hashUpdate = sha512_hashUpdate; + pMethod->hashOctStr = sha512_224_hashOctString; + pMethod->msgLenRep = sha512_msgRep; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha512_256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha512_256.c new file mode 100644 index 0000000..7bbff90 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sha512_256.c @@ -0,0 +1,62 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsHashMethodSet_SHA512_256() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha512stuff.h" + +/*F* +// Name: ippsHashMethodSet_SHA512_256 +// +// Purpose: Setup SHA512_256 method. +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL +// ippStsNoErr no errors +// +*F*/ + +IPPFUN( IppStatus, ippsHashMethodSet_SHA512_256, (IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR1_RET(pMethod); + + pMethod->hashAlgId = ippHashAlg_SHA512_256; + pMethod->hashLen = IPP_SHA256_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA512; + pMethod->msgLenRepSize = MLR_SHA512; + pMethod->hashInit = sha512_256_hashInit; + pMethod->hashUpdate = sha512_hashUpdate; + pMethod->hashOctStr = sha512_256_hashOctString; + pMethod->msgLenRep = sha512_msgRep; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sm3.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sm3.c new file mode 100644 index 0000000..7a06678 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashmethodset_sm3.c @@ -0,0 +1,61 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SM3 +// +// Contents: +// ippsHashMethodSet_SM3() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsm3stuff.h" + +/*F* +// Name: ippsHashMethodSet_SM3 +// +// Purpose: Setup SM3 method. +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL +// ippStsNoErr no errors +// +*F*/ +IPPFUN( IppStatus, ippsHashMethodSet_SM3, (IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR1_RET(pMethod); + + pMethod->hashAlgId = ippHashAlg_SM3; + pMethod->hashLen = IPP_SM3_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SM3; + pMethod->msgLenRepSize = MLR_SM3; + pMethod->hashInit = sm3_hashInit; + pMethod->hashUpdate = sm3_hashUpdate; + pMethod->hashOctStr = sm3_hashOctString; + pMethod->msgLenRep = sm3_msgRep; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashpack.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashpack.c new file mode 100644 index 0000000..1c13a3b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashpack.c @@ -0,0 +1,67 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// General Functionality +// +// Contents: +// ippsHashPack() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcptool.h" + +/*F* +// Name: ippsHashPack +// +// Purpose: Copy initialized context to the buffer. +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL +// pBuffer == NULL +// ippStsContextMatchErr pState->idCtx != idCtxHash +// ippStsNoMemErr bufSize < sizeof(IppsHashState) +// ippStsNoErr no errors +// +// Parameters: +// pState pointer hash state +// pBuffer pointer to the destination buffer +// bufSize size of the destination buffer +// +*F*/ +IPPFUN(IppStatus, ippsHashPack,(const IppsHashState* pState, Ipp8u* pBuffer, int bufSize)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pState, pBuffer); + /* test the context */ + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxHash), ippStsContextMatchErr); + /* test buffer length */ + IPP_BADARG_RET((int)(sizeof(IppsHashState))>bufSize, ippStsNoMemErr); + + CopyBlock(pState, pBuffer, sizeof(IppsHashState)); + IppsHashState* pCopy = (IppsHashState*)pBuffer; + HASH_RESET_ID(pCopy, idCtxHash); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashpack_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashpack_rmf.c new file mode 100644 index 0000000..34962f0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashpack_rmf.c @@ -0,0 +1,65 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// Generalized Functionality +// +// Contents: +// ippsHashPack_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsHashPack_rmf +// +// Purpose: Copy initialized context to the buffer. +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL +// pBuffer == NULL +// ippStsContextMatchErr pState->idCtx != idCtxHash +// ippStsNoMemErr bufSize < sizeof(IppsHashState_rmf) +// ippStsNoErr no errors +// +// Parameters: +// pState pointer hash state +// pBuffer pointer to the destination buffer +// bufSize size of the destination buffer +// +*F*/ +IPPFUN(IppStatus, ippsHashPack_rmf,(const IppsHashState_rmf* pState, Ipp8u* pBuffer, int bufSize)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pState, pBuffer); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxHash), ippStsContextMatchErr); + /* test buffer length */ + IPP_BADARG_RET((int)(sizeof(IppsHashState_rmf))>bufSize, ippStsNoMemErr); + + CopyBlock(pState, pBuffer, sizeof(IppsHashState_rmf)); + IppsHashState_rmf* pCopy = (IppsHashState_rmf*)pBuffer; + HASH_RESET_ID(pCopy, idCtxHash); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashsha1px.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashsha1px.c new file mode 100644 index 0000000..c189e62 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashsha1px.c @@ -0,0 +1,177 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Message block processing according to SHA1 +// +// Contents: +// UpdateSHA1() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcptool.h" + +#if !((_IPP==_IPP_M5) || \ + (_IPP==_IPP_W7) || (_IPP==_IPP_T7) || \ + (_IPP==_IPP_V8) || (_IPP==_IPP_P8) || \ + (_IPP==_IPP_S8) || (_IPP>=_IPP_G9) || \ + (_IPP32E==_IPP32E_M7) || \ + (_IPP32E==_IPP32E_U8) || (_IPP32E==_IPP32E_Y8) || \ + (_IPP32E==_IPP32E_N8) || (_IPP32E>=_IPP32E_E9)) + +/* +// Magic functions defined in FIPS 180-1 +// +*/ +#define MAGIC_F0(B,C,D) (((B) & (C)) | ((~(B)) & (D))) +#define MAGIC_F1(B,C,D) ((B) ^ (C) ^ (D)) +#define MAGIC_F2(B,C,D) (((B) & (C)) | ((B) & (D)) | ((C) & (D))) +#define MAGIC_F3(B,C,D) ((B) ^ (C) ^ (D)) + +#define SHA1_STEP(A,B,C,D,E, MAGIC_FUN, W,K) \ + (E)+= ROL32((A),5) + MAGIC_FUN((B),(C),(D)) + (W) + (K); \ + (B) = ROL32((B),30) + +#define COMPACT_SHA1_STEP(A,B,C,D,E, MAGIC_FUN, W,K, t) { \ + Ipp32u _T = ROL32((A),5) + MAGIC_FUN((t)/20, (B),(C),(D)) + (E) + (W)[(t)] + (K)[(t)/20]; \ + (E) = (D); \ + (D) = (C); \ + (C) = ROL32((B),30); \ + (B) = (A); \ + (A) = _T; \ +} + +#if defined(_ALG_SHA1_COMPACT_) +__INLINE Ipp32u MagicFun(int s, Ipp32u b, Ipp32u c, Ipp32u d) +{ + switch(s) { + case 0: return MAGIC_F0(b,c,d); + case 2: return MAGIC_F2(b,c,d); + default:return MAGIC_F1(b,c,d); + } +} +#endif + + +/*F* +// Name: UpdateSHA1 +// +// Purpose: Update internal hash according to input message stream. +// +// Parameters: +// uniHash pointer to in/out hash +// mblk pointer to message stream +// mlen message stream length (multiple by message block size) +// uniParam pointer to the optional parameter +// +*F*/ +IPP_OWN_DEFN (void, UpdateSHA1, (void* uinHash, const Ipp8u* mblk, int mlen, const void *uniParam)) +{ + Ipp32u* data = (Ipp32u*)mblk; + + Ipp32u* digest = (Ipp32u*)uinHash; + Ipp32u* SHA1_cnt_loc = (Ipp32u*)uniParam; + + for(; mlen>=MBS_SHA1; data += MBS_SHA1/sizeof(Ipp32u), mlen -= MBS_SHA1) { + int t; + + /* + // expand message block + */ + Ipp32u W[80]; + /* initialize the first 16 words in the array W (remember about endian) */ + for(t=0; t<16; t++) { + #if (IPP_ENDIAN == IPP_BIG_ENDIAN) + W[t] = data[t]; + #else + W[t] = ENDIANNESS(data[t]); + #endif + } + /* schedule another 80-16 words in the array W */ + for(; t<80; t++) { + W[t] = ROL32(W[t-3] ^ W[t-8] ^ W[t-14] ^ W[t-16], 1); + } + + /* + // update hash + */ + { + /* init A, B, C, D, E by the the input hash */ + Ipp32u A = digest[0]; + Ipp32u B = digest[1]; + Ipp32u C = digest[2]; + Ipp32u D = digest[3]; + Ipp32u E = digest[4]; + + #if defined(_ALG_SHA1_COMPACT_) + /* steps 0-79 */ + for(t=0; t<80; t++) + COMPACT_SHA1_STEP(A,B,C,D,E, MagicFun, W, SHA1_cnt_loc, t); + + #else + /* perform 0-19 steps */ + for(t=0; t<20; t+=5) { + SHA1_STEP(A,B,C,D,E, MAGIC_F0, W[t ],SHA1_cnt_loc[0]); + SHA1_STEP(E,A,B,C,D, MAGIC_F0, W[t+1],SHA1_cnt_loc[0]); + SHA1_STEP(D,E,A,B,C, MAGIC_F0, W[t+2],SHA1_cnt_loc[0]); + SHA1_STEP(C,D,E,A,B, MAGIC_F0, W[t+3],SHA1_cnt_loc[0]); + SHA1_STEP(B,C,D,E,A, MAGIC_F0, W[t+4],SHA1_cnt_loc[0]); + } + /* perform 20-39 steps */ + for(; t<40; t+=5) { + SHA1_STEP(A,B,C,D,E, MAGIC_F1, W[t ],SHA1_cnt_loc[1]); + SHA1_STEP(E,A,B,C,D, MAGIC_F1, W[t+1],SHA1_cnt_loc[1]); + SHA1_STEP(D,E,A,B,C, MAGIC_F1, W[t+2],SHA1_cnt_loc[1]); + SHA1_STEP(C,D,E,A,B, MAGIC_F1, W[t+3],SHA1_cnt_loc[1]); + SHA1_STEP(B,C,D,E,A, MAGIC_F1, W[t+4],SHA1_cnt_loc[1]); + } + /* perform 40-59 steps */ + for(; t<60; t+=5) { + SHA1_STEP(A,B,C,D,E, MAGIC_F2, W[t ],SHA1_cnt_loc[2]); + SHA1_STEP(E,A,B,C,D, MAGIC_F2, W[t+1],SHA1_cnt_loc[2]); + SHA1_STEP(D,E,A,B,C, MAGIC_F2, W[t+2],SHA1_cnt_loc[2]); + SHA1_STEP(C,D,E,A,B, MAGIC_F2, W[t+3],SHA1_cnt_loc[2]); + SHA1_STEP(B,C,D,E,A, MAGIC_F2, W[t+4],SHA1_cnt_loc[2]); + } + /* perform 60-79 steps */ + for(; t<80; t+=5) { + SHA1_STEP(A,B,C,D,E, MAGIC_F3, W[t ],SHA1_cnt_loc[3]); + SHA1_STEP(E,A,B,C,D, MAGIC_F3, W[t+1],SHA1_cnt_loc[3]); + SHA1_STEP(D,E,A,B,C, MAGIC_F3, W[t+2],SHA1_cnt_loc[3]); + SHA1_STEP(C,D,E,A,B, MAGIC_F3, W[t+3],SHA1_cnt_loc[3]); + SHA1_STEP(B,C,D,E,A, MAGIC_F3, W[t+4],SHA1_cnt_loc[3]); + } + #endif + + /* update digest */ + digest[0] += A; + digest[1] += B; + digest[2] += C; + digest[3] += D; + digest[4] += E; + } + } +} + +#endif diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashsha256px.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashsha256px.c new file mode 100644 index 0000000..b56834e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashsha256px.c @@ -0,0 +1,208 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Message block processing according to SHA256 +// +// Contents: +// UpdateSHA256() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcptool.h" + +#if defined(_ENABLE_ALG_SHA256_) || defined(_ENABLE_ALG_SHA224_) + +#if !((_IPP==_IPP_M5) || \ + (_IPP==_IPP_W7) || (_IPP==_IPP_T7) || \ + (_IPP==_IPP_V8) || (_IPP==_IPP_P8) || \ + (_IPP==_IPP_S8) || (_IPP>=_IPP_G9) || \ + (_IPP32E==_IPP32E_M7) || \ + (_IPP32E==_IPP32E_U8) || (_IPP32E==_IPP32E_Y8) || \ + (_IPP32E==_IPP32E_N8) || (_IPP32E>=_IPP32E_E9)) + +/* +// SHA256 Specific Macros (reference proposal 256-384-512) +*/ +#define CH(x,y,z) (((x) & (y)) ^ (~(x) & (z))) +#define MAJ(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) + +#define SUM0(x) (ROR32((x), 2) ^ ROR32((x),13) ^ ROR32((x),22)) +#define SUM1(x) (ROR32((x), 6) ^ ROR32((x),11) ^ ROR32((x),25)) + +#define SIG0(x) (ROR32((x), 7) ^ ROR32((x),18) ^ LSR32((x), 3)) +#define SIG1(x) (ROR32((x),17) ^ ROR32((x),19) ^ LSR32((x),10)) + +#define SHA256_UPDATE(i) \ + wdat[i & 15] += SIG1(wdat[(i+14)&15]) + wdat[(i+9)&15] + SIG0(wdat[(i+1)&15]) + +#define SHA256_STEP(i,j) \ + v[(7 - i) & 7] += (j ? SHA256_UPDATE(i) : wdat[i&15]) \ + + SHA256_cnt_loc[i + j] \ + + SUM1(v[(4-i)&7]) \ + + CH(v[(4-i)&7], v[(5-i)&7], v[(6-i)&7]); \ + v[(3-i)&7] += v[(7-i)&7]; \ + v[(7-i)&7] += SUM0(v[(0-i)&7]) + MAJ(v[(0-i)&7], v[(1-i)&7], v[(2-i)&7]) + +#define COMPACT_SHA256_STEP(A,B,C,D,E,F,G,H, W,K, r) { \ + Ipp32u _T1 = (H) + SUM1((E)) + CH((E),(F),(G)) + (W)[(r)] + (K)[(r)]; \ + Ipp32u _T2 = SUM0((A)) + MAJ((A),(B),(C)); \ + (H) = (G); \ + (G) = (F); \ + (F) = (E); \ + (E) = (D)+_T1; \ + (D) = (C); \ + (C) = (B); \ + (B) = (A); \ + (A) = _T1+_T2; \ +} + +/*F* +// Name: UpdateSHA256 +// +// Purpose: Update internal hash according to input message stream. +// +// Parameters: +// uniHash pointer to in/out hash +// mblk pointer to message stream +// mlen message stream length (multiple by message block size) +// uniParam pointer to the optional parameter +// +*F*/ +#if defined(_ALG_SHA256_COMPACT_) + +IPP_OWN_DEFN (void, UpdateSHA256, (void* uniHash, const Ipp8u* mblk, int mlen, const void* uniParam)) +{ + Ipp32u* data = (Ipp32u*)mblk; + + Ipp32u* digest = (Ipp32u*)uniHash; + Ipp32u* SHA256_cnt_loc = (Ipp32u*)uniParam; + + for(; mlen>=MBS_SHA256; data += MBS_SHA256/sizeof(Ipp32u), mlen -= MBS_SHA256) { + int t; + + /* + // expand message block + */ + Ipp32u W[64]; + /* initialize the first 16 words in the array W (remember about endian) */ + for(t=0; t<16; t++) { + #if (IPP_ENDIAN == IPP_BIG_ENDIAN) + W[t] = data[t]; + #else + W[t] = ENDIANNESS( data[t] ); + #endif + } + for(; t<64; t++) + W[t] = SIG1(W[t-2]) + W[t-7] + SIG0(W[t-15]) + W[t-16]; + + /* + // update hash + */ + { + /* init A, B, C, D, E, F, G, H by the input hash */ + Ipp32u A = digest[0]; + Ipp32u B = digest[1]; + Ipp32u C = digest[2]; + Ipp32u D = digest[3]; + Ipp32u E = digest[4]; + Ipp32u F = digest[5]; + Ipp32u G = digest[6]; + Ipp32u H = digest[7]; + + for(t=0; t<64; t++) + COMPACT_SHA256_STEP(A,B,C,D,E,F,G,H, W,SHA256_cnt_loc, t); + + /* update hash*/ + digest[0] += A; + digest[1] += B; + digest[2] += C; + digest[3] += D; + digest[4] += E; + digest[5] += F; + digest[6] += G; + digest[7] += H; + } + } +} + +#else +IPP_OWN_DEFN (void, UpdateSHA256, (void* uniHash, const Ipp8u* mblk, int mlen, const void* uniParam)) +{ + Ipp32u* data = (Ipp32u*)mblk; + + Ipp32u* digest = (Ipp32u*)uniHash; + Ipp32u* SHA256_cnt_loc = (Ipp32u*)uniParam; + + for(; mlen>=MBS_SHA256; data += MBS_SHA256/sizeof(Ipp32u), mlen -= MBS_SHA256) { + Ipp32u wdat[16]; + int j; + + /* copy digest */ + Ipp32u v[8]; + CopyBlock(digest, v, IPP_SHA256_DIGEST_BITSIZE/BYTESIZE); + + /* initialize the first 16 words in the array W (remember about endian) */ + for(j=0; j<16; j++) { + #if (IPP_ENDIAN == IPP_BIG_ENDIAN) + wdat[j] = data[j]; + #else + wdat[j] = ENDIANNESS( data[j] ); + #endif + } + + for(j=0; j<64; j+=16) { + SHA256_STEP( 0, j); + SHA256_STEP( 1, j); + SHA256_STEP( 2, j); + SHA256_STEP( 3, j); + SHA256_STEP( 4, j); + SHA256_STEP( 5, j); + SHA256_STEP( 6, j); + SHA256_STEP( 7, j); + SHA256_STEP( 8, j); + SHA256_STEP( 9, j); + SHA256_STEP(10, j); + SHA256_STEP(11, j); + SHA256_STEP(12, j); + SHA256_STEP(13, j); + SHA256_STEP(14, j); + SHA256_STEP(15, j); + } + + /* update digest */ + digest[0] += v[0]; + digest[1] += v[1]; + digest[2] += v[2]; + digest[3] += v[3]; + digest[4] += v[4]; + digest[5] += v[5]; + digest[6] += v[6]; + digest[7] += v[7]; + } +} +#endif + +#endif +#endif /* IPP_ALG_HASH_SHA256 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashsha512px.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashsha512px.c new file mode 100644 index 0000000..42d5b59 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashsha512px.c @@ -0,0 +1,215 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Message block processing according to SHA512 +// +// Contents: +// UpdateSHA512() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcptool.h" + +#if defined(_ENABLE_ALG_SHA512_) || defined(_ENABLE_ALG_SHA384_) || defined(_ENABLE_ALG_SHA512_224_) || defined(_ENABLE_ALG_SHA512_256_) + +#if !((_IPP==_IPP_W7) || (_IPP==_IPP_T7) || \ + (_IPP==_IPP_V8) || (_IPP==_IPP_P8) || \ + (_IPP==_IPP_S8) || (_IPP>=_IPP_G9) || \ + (_IPP32E==_IPP32E_M7) || \ + (_IPP32E==_IPP32E_U8) || (_IPP32E==_IPP32E_Y8) || \ + (_IPP32E==_IPP32E_N8) || (_IPP32E>=_IPP32E_E9)) + +/* +// SHA512 Specific Macros (reference proposal 256-384-512) +// +// Note: All operations act on DWORDs (64-bits) +*/ +#define CH(x,y,z) (((x) & (y)) ^ (~(x) & (z))) +#define MAJ(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) + +#define SUM0(x) (ROR64((x),28) ^ ROR64((x),34) ^ ROR64((x),39)) +#define SUM1(x) (ROR64((x),14) ^ ROR64((x),18) ^ ROR64((x),41)) + +#define SIG0(x) (ROR64((x), 1) ^ ROR64((x), 8) ^ LSR64((x), 7)) +#define SIG1(x) (ROR64((x),19) ^ ROR64((x),61) ^ LSR64((x), 6)) + +#define SHA512_UPDATE(i) \ + wdat[i&15] += SIG1(wdat[(i+14)&15]) + wdat[(i+9)&15] + SIG0(wdat[(i+1)&15]) + +#define SHA512_STEP(i,j) \ + v[(7-i)&7] += (j ? SHA512_UPDATE(i) : wdat[i&15]) \ + + SHA512_cnt_loc[i+j] \ + + SUM1(v[(4-i)&7]) \ + + CH(v[(4-i)&7], v[(5-i)&7], v[(6-i)&7]); \ + v[(3-i)&7] += v[(7-i)&7]; \ + v[(7-i)&7] += SUM0(v[(0-i)&7]) + MAJ(v[(0-i)&7], v[(1-i)&7], v[(2-i)&7]) + +#define COMPACT_SHA512_STEP(A,B,C,D,E,F,G,H, W,K, r) { \ + Ipp64u _T1 = (H) + SUM1((E)) + CH((E),(F),(G)) + (W)[(r)] + (K)[(r)]; \ + Ipp64u _T2 = SUM0((A)) + MAJ((A),(B),(C)); \ + (H) = (G); \ + (G) = (F); \ + (F) = (E); \ + (E) = (D)+_T1; \ + (D) = (C); \ + (C) = (B); \ + (B) = (A); \ + (A) = _T1+_T2; \ +} + +/*F* +// Name: UpdateSHA512 +// +// Purpose: Update internal hash according to input message stream. +// +// Parameters: +// uniHash pointer to in/out hash +// mblk pointer to message stream +// mlen message stream length (multiple by message block size) +// uniParam pointer to the optional parameter +// +*F*/ +#if defined(_ALG_SHA512_COMPACT_) + +IPP_OWN_DEFN (void, UpdateSHA512, (void* uniHash, const Ipp8u* mblk, int mlen, const void* uniPraram)) +{ + Ipp32u* data = (Ipp32u*)mblk; + + Ipp64u* digest = (Ipp64u*)uniHash; + Ipp64u* SHA512_cnt_loc = (Ipp64u*)uniPraram; + + + for(; mlen>=MBS_SHA512; data += MBS_SHA512/sizeof(Ipp32u), mlen -= MBS_SHA512) { + int t; + Ipp64u W[80]; + + /* + // expand message block + */ + /* initialize the first 16 words in the array W (remember about endian) */ + for(t=0; t<16; t++) { + Ipp32u hiX = data[2*t]; + Ipp32u loX = data[2*t+1]; + #if (IPP_ENDIAN == IPP_BIG_ENDIAN) + W[t] = IPP_MAKEDWORD(loX, hiX); + #else + W[t] = IPP_MAKEDWORD( ENDIANNESS(loX), ENDIANNESS(hiX) ); + #endif + } + for(; t<80; t++) + W[t] = SIG1(W[t-2]) + W[t-7] + SIG0(W[t-15]) + W[t-16]; + + /* + // update hash + */ + { + /* init A, B, C, D, E, F, G, H by the input hash */ + Ipp64u A = digest[0]; + Ipp64u B = digest[1]; + Ipp64u C = digest[2]; + Ipp64u D = digest[3]; + Ipp64u E = digest[4]; + Ipp64u F = digest[5]; + Ipp64u G = digest[6]; + Ipp64u H = digest[7]; + + for(t=0; t<80; t++) + COMPACT_SHA512_STEP(A,B,C,D,E,F,G,H, W,SHA512_cnt_loc, t); + + /* update hash*/ + digest[0] += A; + digest[1] += B; + digest[2] += C; + digest[3] += D; + digest[4] += E; + digest[5] += F; + digest[6] += G; + digest[7] += H; + } + } +} + +#else +IPP_OWN_DEFN (void, UpdateSHA512, (void* uniHash, const Ipp8u* mblk, int mlen, const void* uniPraram)) +{ + Ipp32u* data = (Ipp32u*)mblk; + + Ipp64u* digest = (Ipp64u*)uniHash; + Ipp64u* SHA512_cnt_loc = (Ipp64u*)uniPraram; + + for(; mlen>=MBS_SHA512; data += MBS_SHA512/sizeof(Ipp32u), mlen -= MBS_SHA512) { + Ipp64u wdat[16]; + int j; + + Ipp64u v[8]; + + /* initialize the first 16 words in the array W (remember about endian) */ + for(j=0; j<16; j++) { + Ipp32u hiX = data[2*j]; + Ipp32u loX = data[2*j+1]; + #if (IPP_ENDIAN == IPP_BIG_ENDIAN) + wdat[j] = IPP_MAKEDWORD(loX, hiX); + #else + wdat[j] = IPP_MAKEDWORD( ENDIANNESS(loX), ENDIANNESS(hiX) ); + #endif + } + + /* copy digest */ + CopyBlock(digest, v, IPP_SHA512_DIGEST_BITSIZE/BYTESIZE); + + for(j=0; j<80; j+=16) { + SHA512_STEP( 0, j); + SHA512_STEP( 1, j); + SHA512_STEP( 2, j); + SHA512_STEP( 3, j); + SHA512_STEP( 4, j); + SHA512_STEP( 5, j); + SHA512_STEP( 6, j); + SHA512_STEP( 7, j); + SHA512_STEP( 8, j); + SHA512_STEP( 9, j); + SHA512_STEP(10, j); + SHA512_STEP(11, j); + SHA512_STEP(12, j); + SHA512_STEP(13, j); + SHA512_STEP(14, j); + SHA512_STEP(15, j); + } + + /* update digest */ + digest[0] += v[0]; + digest[1] += v[1]; + digest[2] += v[2]; + digest[3] += v[3]; + digest[4] += v[4]; + digest[5] += v[5]; + digest[6] += v[6]; + digest[7] += v[7]; + } +} +#endif + +#endif +#endif /* IPP_ALG_HASH_SHA512 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashsm3px.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashsm3px.c new file mode 100644 index 0000000..892f2f1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashsm3px.c @@ -0,0 +1,338 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Message block processing according to SM5 +// +// Contents: +// UpdateSM3() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcptool.h" + +#if defined(_ENABLE_ALG_SM3_) + +#if !((_IPP32E>=_IPP32E_U8) || (_IPP32E==_IPP32E_N8) ) + +/* +// SM3 Specific Macros +// (reference SM3 Cryptographic Hash Algorithm, +// Chinese Commercial Cryptography Administration Office, 2010.12) +*/ + +/* T1 and T2 are base for additive const generation */ +#define T1 (0x79CC4519) +#define T2 (0x7A879D8A) + +// boolean functions (0<=nr<16) +#define FF1(x,y,z) ((x)^(y)^(z)) +#define GG1(x,y,z) ((x)^(y)^(z)) +// boolean functions (16<=nr<64) +#define FF2(x,y,z) (((x)&(y)) | ((x)&(z)) | ((y)&(z))) +#define GG2(x,y,z) (((x)&(y)) | (~(x)&(z))) + +// P0 permutation: +#define P0(x) ((x) ^ ROL32((x),9) ^ ROL32((x),17)) +// P1 permutation: +#define P1(x) ((x) ^ ROL32((x),15) ^ ROL32((x),23)) + +// update W +#define WUPDATE(nr, W) (P1(W[((nr)-16)&15] ^ W[((nr)-9)&15] ^ ROL32(W[((nr)-3)&15],15)) ^ ROL32(W[((nr)-13)&15],7) ^ W[((nr)-6)&15]) + +// SM3 steps +#define SM3_STEP1(nr, A,B,C,D,E,F,G,H, Tj, W) { \ + TT1 = FF1(A,B,C) + D + (W[nr&15] ^ W[(nr+4)&15]); \ + TT2 = GG1(E,F,G) + H + W[nr&15]; \ + H = ROL32(A,12); \ + D = ROL32(H + E +Tj, 7); \ + H ^= D; \ + D += TT2; \ + H += TT1; \ + B = ROL32(B, 9); \ + D = P0(D); \ + F = ROL32(F, 19); \ + /*Tj = ROL32(Tj, 1);*/ \ + W[(nr)&15] = WUPDATE(nr, W); \ +} + +#define SM3_STEP2(nr, A,B,C,D,E,F,G,H, Tj, W) { \ + TT1 = FF2(A,B,C) + D + (W[nr&15] ^ W[(nr+4)&15]); \ + TT2 = GG2(E,F,G) + H + W[nr&15]; \ + H = ROL32(A,12); \ + D = ROL32(H + E +Tj, 7); \ + H ^= D; \ + D += TT2; \ + H += TT1; \ + B = ROL32(B, 9); \ + D = P0(D); \ + F = ROL32(F, 19); \ + /*Tj = ROL32(Tj, 1);*/ \ + W[(nr)&15] = WUPDATE(nr, W); \ +} + +#define SM3_STEP3(nr, A,B,C,D,E,F,G,H, Tj, W) { \ + TT1 = FF2(A,B,C) + D + (W[nr&15] ^ W[(nr+4)&15]); \ + TT2 = GG2(E,F,G) + H + W[nr&15]; \ + H = ROL32(A,12); \ + D = ROL32(H + E +Tj, 7); \ + H ^= D; \ + D += TT2; \ + H += TT1; \ + B = ROL32(B, 9); \ + D = P0(D); \ + F = ROL32(F, 19); \ + /*Tj = ROL32(Tj, 1);*/ \ +} + +#define COMPACT_SM3_STEP(A,B,C,D,E,F,G,H, FF, GG, W,Tj, r) { \ + TT1 = FF((r)&0x30, A,B,C) + D + (W[(r)] ^ W[(r)+4]); \ + TT2 = GG((r)&0x30, E,F,G) + H + W[(r)]; \ + \ + _H = ROL32(A,12); \ + _D = ROL32(_H + E +Tj[(r)], 7); \ + _H ^= _D; \ + _D += TT2; \ + _H += TT1; \ + _D = P0(_D);\ + \ + H = G; \ + G = ROL32(F,19); \ + F = E; \ + E =_D; \ + D = C; \ + C = ROL32(B, 9); \ + B = A; \ + A =_H; \ +} + + +/*F* +// Name: UpdateSM3 +// +// Purpose: Update internal hash according to input message stream. +// +// Parameters: +// uniHash pointer to in/out hash +// mblk pointer to message stream +// mlen message stream length (multiple by message block size) +// uniParam pointer to the optional parameter +// +*F*/ +#if defined(_ALG_SM3_COMPACT_) + +__INLINE Ipp32u MagicFF(int s, Ipp32u a, Ipp32u b, Ipp32u c) +{ + switch(s) { + case 0: return FF1(a,b,c); + default:return FF2(a,b,c); + } +} +__INLINE Ipp32u MagicGG(int s, Ipp32u e, Ipp32u f, Ipp32u g) +{ + switch(s) { + case 0: return GG1(e,f,g); + default:return GG2(e,f,g); + } +} + +IPP_OWN_DEFN (void, UpdateSM3, (void* uniHash, const Ipp8u* mblk, int mlen, const void* uniParam)) +{ + Ipp32u* data = (Ipp32u*)mblk; + + Ipp32u* hash = (Ipp32u*)uniHash; + Ipp32u* SM3_cnt_loc = (Ipp32u*)uniParam; + + for(; mlen>=MBS_SM3; data += MBS_SM3/sizeof(Ipp32u), mlen -= MBS_SM3) { + int r; + + /* + // expand message block + */ + Ipp32u W[68]; + /* initialize the first 16 words in the array W (remember about endian) */ + for(r=0; r<16; r++) { + #if (IPP_ENDIAN == IPP_BIG_ENDIAN) + W[r] = data[r]; + #else + W[r] = ENDIANNESS( data[r] ); + #endif + } + for(; r<68; r++) + W[r] = P1(W[r-16] ^ W[r-9] ^ ROL32(W[r-3],15)) ^ ROL32(W[r-13],7) ^ W[r-6]; + + /* + // update hash + */ + { + /* init A, B, C, D, E, F, G, H by the input hash */ + Ipp32u A = hash[0]; + Ipp32u B = hash[1]; + Ipp32u C = hash[2]; + Ipp32u D = hash[3]; + Ipp32u E = hash[4]; + Ipp32u F = hash[5]; + Ipp32u G = hash[6]; + Ipp32u H = hash[7]; + + Ipp32u TT1, TT2, _H, _D; + for(r=0; r<64; r++) + COMPACT_SM3_STEP(A,B,C,D,E,F,G,H, MagicFF,MagicGG, W, SM3_cnt_loc, r); + + /* update hash */ + hash[0] ^= A; + hash[1] ^= B; + hash[2] ^= C; + hash[3] ^= D; + hash[4] ^= E; + hash[5] ^= F; + hash[6] ^= G; + hash[7] ^= H; + } + } +} + +#else +IPP_OWN_DEFN (void, UpdateSM3, (void* uniHash, const Ipp8u* mblk, int mlen, const void* uniParam)) +{ + Ipp32u* data = (Ipp32u*)mblk; + + Ipp32u* hash = (Ipp32u*)uniHash; + Ipp32u* SM3_cnt_loc = (Ipp32u*)uniParam; + + for(; mlen>=MBS_SM3; data += MBS_SM3/sizeof(Ipp32u), mlen -= MBS_SM3) { + + /* copy input hash */ + Ipp32u A = hash[0]; + Ipp32u B = hash[1]; + Ipp32u C = hash[2]; + Ipp32u D = hash[3]; + Ipp32u E = hash[4]; + Ipp32u F = hash[5]; + Ipp32u G = hash[6]; + Ipp32u H = hash[7]; + + Ipp32u W[16]; + int j; + + + /* initialize the first 16 words in the array W (remember about endian) */ + for(j=0; j<16; j++) { + #if (IPP_ENDIAN == IPP_BIG_ENDIAN) + W[j] = data[j]; + #else + W[j] = ENDIANNESS( data[j] ); + #endif + } + + /* apply compression function */ + { + Ipp32u TT1, TT2; + SM3_STEP1( 0, A,B,C,D,E,F,G,H, SM3_cnt_loc[0], W); + SM3_STEP1( 1, H,A,B,C,D,E,F,G, SM3_cnt_loc[1], W); + SM3_STEP1( 2, G,H,A,B,C,D,E,F, SM3_cnt_loc[2], W); + SM3_STEP1( 3, F,G,H,A,B,C,D,E, SM3_cnt_loc[3], W); + SM3_STEP1( 4, E,F,G,H,A,B,C,D, SM3_cnt_loc[4], W); + SM3_STEP1( 5, D,E,F,G,H,A,B,C, SM3_cnt_loc[5], W); + SM3_STEP1( 6, C,D,E,F,G,H,A,B, SM3_cnt_loc[6], W); + SM3_STEP1( 7, B,C,D,E,F,G,H,A, SM3_cnt_loc[7], W); + + SM3_STEP1( 8, A,B,C,D,E,F,G,H, SM3_cnt_loc[ 8], W); + SM3_STEP1( 9, H,A,B,C,D,E,F,G, SM3_cnt_loc[ 9], W); + SM3_STEP1(10, G,H,A,B,C,D,E,F, SM3_cnt_loc[10], W); + SM3_STEP1(11, F,G,H,A,B,C,D,E, SM3_cnt_loc[11], W); + SM3_STEP1(12, E,F,G,H,A,B,C,D, SM3_cnt_loc[12], W); + SM3_STEP1(13, D,E,F,G,H,A,B,C, SM3_cnt_loc[13], W); + SM3_STEP1(14, C,D,E,F,G,H,A,B, SM3_cnt_loc[14], W); + SM3_STEP1(15, B,C,D,E,F,G,H,A, SM3_cnt_loc[15], W); + + SM3_STEP2(16, A,B,C,D,E,F,G,H, SM3_cnt_loc[16], W); + SM3_STEP2(17, H,A,B,C,D,E,F,G, SM3_cnt_loc[17], W); + SM3_STEP2(18, G,H,A,B,C,D,E,F, SM3_cnt_loc[18], W); + SM3_STEP2(19, F,G,H,A,B,C,D,E, SM3_cnt_loc[19], W); + SM3_STEP2(20, E,F,G,H,A,B,C,D, SM3_cnt_loc[20], W); + SM3_STEP2(21, D,E,F,G,H,A,B,C, SM3_cnt_loc[21], W); + SM3_STEP2(22, C,D,E,F,G,H,A,B, SM3_cnt_loc[22], W); + SM3_STEP2(23, B,C,D,E,F,G,H,A, SM3_cnt_loc[23], W); + + SM3_STEP2(24, A,B,C,D,E,F,G,H, SM3_cnt_loc[24], W); + SM3_STEP2(25, H,A,B,C,D,E,F,G, SM3_cnt_loc[25], W); + SM3_STEP2(26, G,H,A,B,C,D,E,F, SM3_cnt_loc[26], W); + SM3_STEP2(27, F,G,H,A,B,C,D,E, SM3_cnt_loc[27], W); + SM3_STEP2(28, E,F,G,H,A,B,C,D, SM3_cnt_loc[28], W); + SM3_STEP2(29, D,E,F,G,H,A,B,C, SM3_cnt_loc[29], W); + SM3_STEP2(30, C,D,E,F,G,H,A,B, SM3_cnt_loc[30], W); + SM3_STEP2(31, B,C,D,E,F,G,H,A, SM3_cnt_loc[31], W); + + SM3_STEP2(32, A,B,C,D,E,F,G,H, SM3_cnt_loc[32], W); + SM3_STEP2(33, H,A,B,C,D,E,F,G, SM3_cnt_loc[33], W); + SM3_STEP2(34, G,H,A,B,C,D,E,F, SM3_cnt_loc[34], W); + SM3_STEP2(35, F,G,H,A,B,C,D,E, SM3_cnt_loc[35], W); + SM3_STEP2(36, E,F,G,H,A,B,C,D, SM3_cnt_loc[36], W); + SM3_STEP2(37, D,E,F,G,H,A,B,C, SM3_cnt_loc[37], W); + SM3_STEP2(38, C,D,E,F,G,H,A,B, SM3_cnt_loc[38], W); + SM3_STEP2(39, B,C,D,E,F,G,H,A, SM3_cnt_loc[39], W); + + SM3_STEP2(40, A,B,C,D,E,F,G,H, SM3_cnt_loc[40], W); + SM3_STEP2(41, H,A,B,C,D,E,F,G, SM3_cnt_loc[41], W); + SM3_STEP2(42, G,H,A,B,C,D,E,F, SM3_cnt_loc[42], W); + SM3_STEP2(43, F,G,H,A,B,C,D,E, SM3_cnt_loc[43], W); + SM3_STEP2(44, E,F,G,H,A,B,C,D, SM3_cnt_loc[44], W); + SM3_STEP2(45, D,E,F,G,H,A,B,C, SM3_cnt_loc[45], W); + SM3_STEP2(46, C,D,E,F,G,H,A,B, SM3_cnt_loc[46], W); + SM3_STEP2(47, B,C,D,E,F,G,H,A, SM3_cnt_loc[47], W); + + SM3_STEP2(48, A,B,C,D,E,F,G,H, SM3_cnt_loc[48], W); + SM3_STEP2(49, H,A,B,C,D,E,F,G, SM3_cnt_loc[49], W); + SM3_STEP2(50, G,H,A,B,C,D,E,F, SM3_cnt_loc[50], W); + SM3_STEP2(51, F,G,H,A,B,C,D,E, SM3_cnt_loc[51], W); + SM3_STEP3(52, E,F,G,H,A,B,C,D, SM3_cnt_loc[52], W); + SM3_STEP3(53, D,E,F,G,H,A,B,C, SM3_cnt_loc[53], W); + SM3_STEP3(54, C,D,E,F,G,H,A,B, SM3_cnt_loc[54], W); + SM3_STEP3(55, B,C,D,E,F,G,H,A, SM3_cnt_loc[55], W); + + SM3_STEP3(56, A,B,C,D,E,F,G,H, SM3_cnt_loc[56], W); + SM3_STEP3(57, H,A,B,C,D,E,F,G, SM3_cnt_loc[57], W); + SM3_STEP3(58, G,H,A,B,C,D,E,F, SM3_cnt_loc[58], W); + SM3_STEP3(59, F,G,H,A,B,C,D,E, SM3_cnt_loc[59], W); + SM3_STEP3(60, E,F,G,H,A,B,C,D, SM3_cnt_loc[60], W); + SM3_STEP3(61, D,E,F,G,H,A,B,C, SM3_cnt_loc[61], W); + SM3_STEP3(62, C,D,E,F,G,H,A,B, SM3_cnt_loc[62], W); + SM3_STEP3(63, B,C,D,E,F,G,H,A, SM3_cnt_loc[63], W); + } + /* update hash */ + hash[0] ^= A; + hash[1] ^= B; + hash[2] ^= C; + hash[3] ^= D; + hash[4] ^= E; + hash[5] ^= F; + hash[6] ^= G; + hash[7] ^= H; + } +} +#endif + +#endif /* _PX/_W7/_T7, _MX/_M7 versions */ +#endif /* IPP_ALG_HASH_SM3 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha224.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha224.c new file mode 100644 index 0000000..64f5cf8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha224.c @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2022 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA224 +// +// Contents: +// ippsHashStateMethodSet_SHA224() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +/*F* +// Name: ippsHashStateMethodSet_SHA224 +// +// Purpose: Setup SHA224 method inside the hash state. +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL or pState == NULL +// ippStsNoErr no errors +// +*F*/ + +IPPFUN( IppStatus, ippsHashStateMethodSet_SHA224, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pState, pMethod); + + HASH_METHOD(pState) = pMethod; + + pMethod->hashAlgId = ippHashAlg_SHA224; + pMethod->hashLen = IPP_SHA224_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA256; + pMethod->msgLenRepSize = MLR_SHA256; + pMethod->hashInit = sha224_hashInit; + pMethod->hashUpdate = sha256_hashUpdate; + pMethod->hashOctStr = sha224_hashOctString; + pMethod->msgLenRep = sha256_msgRep; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha224_ni.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha224_ni.c new file mode 100644 index 0000000..650b19d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha224_ni.c @@ -0,0 +1,79 @@ +/******************************************************************************* +* Copyright (C) 2022 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA224 +// +// Contents: +// ippsHashStateMethodSet_SHA224_NI() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +/*F* +// Name: ippsHashStateMethodSet_SHA224_NI +// +// Purpose: Setup SHA224 method inside the hash state (using the Intel® Secure Hash Algorithm - New Instructions (Intel® SHA-NI) instruction set). +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL or pState == NULL +// ippStsNotSupportedModeErr mode disabled by configuration +// ippStsNoErr no errors +// +*F*/ + + +IPPFUN( IppStatus, ippsHashStateMethodSet_SHA224_NI, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pState, pMethod); + + HASH_METHOD(pState) = pMethod; + +#if (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_ || _SHA_NI_ENABLING_==_FEATURE_ON_) + pMethod->hashAlgId = ippHashAlg_SHA224; + pMethod->hashLen = IPP_SHA224_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA256; + pMethod->msgLenRepSize = MLR_SHA256; + pMethod->hashInit = sha224_hashInit; + pMethod->hashUpdate = sha256_ni_hashUpdate; + pMethod->hashOctStr = sha224_hashOctString; + pMethod->msgLenRep = sha256_msgRep; + + return ippStsNoErr; +#else + pMethod->hashAlgId = ippHashAlg_Unknown; + pMethod->hashLen = 0; + pMethod->msgBlkSize = 0; + pMethod->msgLenRepSize = 0; + pMethod->hashInit = 0; + pMethod->hashUpdate = 0; + pMethod->hashOctStr = 0; + pMethod->msgLenRep = 0; + + return ippStsNotSupportedModeErr; +#endif +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha224_tt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha224_tt.c new file mode 100644 index 0000000..0092d0d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha224_tt.c @@ -0,0 +1,71 @@ +/******************************************************************************* +* Copyright (C) 2022 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA224 +// +// Contents: +// ippsHashStateMethodSet_SHA224_TT() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +/*F* +// Name: ippsHashStateMethodSet_SHA224_TT +// +// Purpose: Setup SHA224 method inside the hash state +// (using the Intel® Secure Hash Algorithm - New Instructions (Intel® SHA-NI) instructions set +// if it is available at run time) +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL or pState == NULL +// ippStsNoErr no errors +// +*F*/ + +IPPFUN( IppStatus, ippsHashStateMethodSet_SHA224_TT, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pState, pMethod); + + HASH_METHOD(pState) = pMethod; + + pMethod->hashAlgId = ippHashAlg_SHA224; + pMethod->hashLen = IPP_SHA224_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA256; + pMethod->msgLenRepSize = MLR_SHA256; + pMethod->hashInit = sha224_hashInit; + pMethod->hashUpdate = sha256_hashUpdate; + pMethod->hashOctStr = sha224_hashOctString; + pMethod->msgLenRep = sha256_msgRep; + +#if (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_ || _SHA_NI_ENABLING_==_FEATURE_ON_) + if(IsFeatureEnabled(ippCPUID_SHA)) + pMethod->hashUpdate = sha256_ni_hashUpdate; +#endif + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha256.c new file mode 100644 index 0000000..10d12f6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha256.c @@ -0,0 +1,63 @@ +/******************************************************************************* +* Copyright (C) 2022 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsHashStateMethodSet_SHA256() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +/*F* +// Name: ippsHashStateMethodSet_SHA256 +// +// Purpose: Setup SHA256 method inside the hash state. +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL or pState == NULL +// ippStsNoErr no errors +// +*F*/ +IPPFUN( IppStatus, ippsHashStateMethodSet_SHA256, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pState, pMethod); + + HASH_METHOD(pState) = pMethod; + + pMethod->hashAlgId = ippHashAlg_SHA256; + pMethod->hashLen = IPP_SHA256_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA256; + pMethod->msgLenRepSize = MLR_SHA256; + pMethod->hashInit = sha256_hashInit; + pMethod->hashUpdate = sha256_hashUpdate; + pMethod->hashOctStr = sha256_hashOctString; + pMethod->msgLenRep = sha256_msgRep; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha256_ni.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha256_ni.c new file mode 100644 index 0000000..accb348 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha256_ni.c @@ -0,0 +1,78 @@ +/******************************************************************************* +* Copyright (C) 2022 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsHashStateMethodSet_SHA256_NI() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +/*F* +// Name: ippsHashStateMethodSet_SHA256_NI +// +// Purpose: Setup SHA256 method inside the hash state (using the Intel® Secure Hash Algorithm - New Instructions (Intel® SHA-NI) instruction set). +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL or pState == NULL +// ippStsNotSupportedModeErr mode disabled by configuration +// ippStsNoErr no errors +// +*F*/ + +IPPFUN( IppStatus, ippsHashStateMethodSet_SHA256_NI, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pState, pMethod); + + HASH_METHOD(pState) = pMethod; + +#if (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_ || _SHA_NI_ENABLING_==_FEATURE_ON_) + pMethod->hashAlgId = ippHashAlg_SHA256; + pMethod->hashLen = IPP_SHA256_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA256; + pMethod->msgLenRepSize = MLR_SHA256; + pMethod->hashInit = sha256_hashInit; + pMethod->hashUpdate = sha256_ni_hashUpdate; + pMethod->hashOctStr = sha256_hashOctString; + pMethod->msgLenRep = sha256_msgRep; + + return ippStsNoErr; +#else + pMethod->hashAlgId = ippHashAlg_Unknown; + pMethod->hashLen = 0; + pMethod->msgBlkSize = 0; + pMethod->msgLenRepSize = 0; + pMethod->hashInit = 0; + pMethod->hashUpdate = 0; + pMethod->hashOctStr = 0; + pMethod->msgLenRep = 0; + + return ippStsNotSupportedModeErr; +#endif +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha256_tt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha256_tt.c new file mode 100644 index 0000000..4c62917 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha256_tt.c @@ -0,0 +1,71 @@ +/******************************************************************************* +* Copyright (C) 2022 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsHashStateMethodSet_SHA256_TT() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +/*F* +// Name: ippsHashStateMethodSet_SHA256_TT +// +// Purpose: Setup SHA256 method inside the hash state +// (using the Intel® Secure Hash Algorithm - New Instructions (Intel® SHA-NI) instructions set +// if it is available at run time) +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL or pState == NULL +// ippStsNoErr no errors +// +*F*/ + +IPPFUN( IppStatus, ippsHashStateMethodSet_SHA256_TT, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pState, pMethod); + + HASH_METHOD(pState) = pMethod; + + pMethod->hashAlgId = ippHashAlg_SHA256; + pMethod->hashLen = IPP_SHA256_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA256; + pMethod->msgLenRepSize = MLR_SHA256; + pMethod->hashInit = sha256_hashInit; + pMethod->hashUpdate = sha256_hashUpdate; + pMethod->hashOctStr = sha256_hashOctString; + pMethod->msgLenRep = sha256_msgRep; + +#if (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_ || _SHA_NI_ENABLING_==_FEATURE_ON_) + if(IsFeatureEnabled(ippCPUID_SHA)) + pMethod->hashUpdate = sha256_ni_hashUpdate; +#endif + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha384.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha384.c new file mode 100644 index 0000000..4203d63 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha384.c @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2022 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA384 message digest +// +// Contents: +// ippsHashStateMethodSet_SHA384() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha512stuff.h" + +/*F* +// Name: ippsHashStateMethodSet_SHA384 +// +// Purpose: Setup SHA384 method inside the hash state. +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL or pState == NULL +// ippStsNoErr no errors +// +*F*/ + +IPPFUN( IppStatus, ippsHashStateMethodSet_SHA384, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pState, pMethod); + + HASH_METHOD(pState) = pMethod; + + pMethod->hashAlgId = ippHashAlg_SHA384; + pMethod->hashLen = IPP_SHA384_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA512; + pMethod->msgLenRepSize = MLR_SHA512; + pMethod->hashInit = sha512_384_hashInit; + pMethod->hashUpdate = sha512_hashUpdate; + pMethod->hashOctStr = sha512_384_hashOctString; + pMethod->msgLenRep = sha512_msgRep; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha512.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha512.c new file mode 100644 index 0000000..bffc445 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha512.c @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2022 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsHashStateMethodSet_SHA512() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha512stuff.h" + +/*F* +// Name: ippsHashStateMethodSet_SHA512 +// +// Purpose: Setup SHA512 method inside the hash state. +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL or pState == NULL +// ippStsNoErr no errors +// +*F*/ + +IPPFUN( IppStatus, ippsHashStateMethodSet_SHA512, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pState, pMethod); + + HASH_METHOD(pState) = pMethod; + + pMethod->hashAlgId = ippHashAlg_SHA512; + pMethod->hashLen = IPP_SHA512_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA512; + pMethod->msgLenRepSize = MLR_SHA512; + pMethod->hashInit = sha512_hashInit; + pMethod->hashUpdate = sha512_hashUpdate; + pMethod->hashOctStr = sha512_hashOctString; + pMethod->msgLenRep = sha512_msgRep; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha512_224.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha512_224.c new file mode 100644 index 0000000..165cd36 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha512_224.c @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2022 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsHashStateMethodSet_SHA512_224() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha512stuff.h" + +/*F* +// Name: ippsHashStateMethodSet_SHA512_224 +// +// Purpose: Return SHA512_224 method inside the hash state. +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL or pState == NULL +// ippStsNoErr no errors +// +*F*/ + +IPPFUN( IppStatus, ippsHashStateMethodSet_SHA512_224, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pState, pMethod); + + HASH_METHOD(pState) = pMethod; + + pMethod->hashAlgId = ippHashAlg_SHA512_224; + pMethod->hashLen = IPP_SHA224_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA512; + pMethod->msgLenRepSize = MLR_SHA512; + pMethod->hashInit = sha512_224_hashInit; + pMethod->hashUpdate = sha512_hashUpdate; + pMethod->hashOctStr = sha512_224_hashOctString; + pMethod->msgLenRep = sha512_msgRep; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha512_256.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha512_256.c new file mode 100644 index 0000000..fb4fe5b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sha512_256.c @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2022 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsHashStateMethodSet_SHA512_256() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha512stuff.h" + +/*F* +// Name: ippsHashStateMethodSet_SHA512_256 +// +// Purpose: Setup SHA512_256 method inside the hash state. +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL or pState == NULL +// ippStsNoErr no errors +// +*F*/ + +IPPFUN( IppStatus, ippsHashStateMethodSet_SHA512_256, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pState, pMethod); + + HASH_METHOD(pState) = pMethod; + + pMethod->hashAlgId = ippHashAlg_SHA512_256; + pMethod->hashLen = IPP_SHA256_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SHA512; + pMethod->msgLenRepSize = MLR_SHA512; + pMethod->hashInit = sha512_256_hashInit; + pMethod->hashUpdate = sha512_hashUpdate; + pMethod->hashOctStr = sha512_256_hashOctString; + pMethod->msgLenRep = sha512_msgRep; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sm3.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sm3.c new file mode 100644 index 0000000..7fc83d6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashstatemethodset_sm3.c @@ -0,0 +1,63 @@ +/******************************************************************************* +* Copyright (C) 2022 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SM3 +// +// Contents: +// ippsHashStateMethodSet_SM3() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsm3stuff.h" + +/*F* +// Name: ippsHashStateMethodSet_SM3 +// +// Purpose: Setup SM3 method inside the hash state. +// +// Returns: Reason: +// ippStsNullPtrErr pMethod == NULL or pState == NULL +// ippStsNoErr no errors +// +*F*/ +IPPFUN( IppStatus, ippsHashStateMethodSet_SM3, (IppsHashState_rmf* pState, IppsHashMethod* pMethod) ) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pState, pMethod); + + HASH_METHOD(pState) = pMethod; + + pMethod->hashAlgId = ippHashAlg_SM3; + pMethod->hashLen = IPP_SM3_DIGEST_BITSIZE/8; + pMethod->msgBlkSize = MBS_SM3; + pMethod->msgLenRepSize = MLR_SM3; + pMethod->hashInit = sm3_hashInit; + pMethod->hashUpdate = sm3_hashUpdate; + pMethod->hashOctStr = sm3_hashOctString; + pMethod->msgLenRep = sm3_msgRep; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashunpack.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashunpack.c new file mode 100644 index 0000000..557086b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashunpack.c @@ -0,0 +1,58 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// General Functionality +// +// Contents: +// ippsHashUnpack() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcptool.h" + +/*F* +// Name: ippsHashUnpack +// +// Purpose: Unpack buffer content into the initialized context. +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL +// pBuffer == NULL +// ippStsNoErr no errors +// +// Parameters: +// pBuffer pointer to the source buffer +// pState pointer hash state +// +*F*/ +IPPFUN(IppStatus, ippsHashUnpack,(const Ipp8u* pBuffer, IppsHashState* pState)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pState, pBuffer); + + CopyBlock(pBuffer, pState, sizeof(IppsHashState)); + HASH_SET_ID(pState, idCtxHash); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashunpack_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashunpack_rmf.c new file mode 100644 index 0000000..fe75e2e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashunpack_rmf.c @@ -0,0 +1,58 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// Generalized Functionality +// +// Contents: +// ippsHashUnpack_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsHashUnpack_rmf +// +// Purpose: Unpack buffer content into the initialized context. +// +// Returns: Reason: +// ippStsNullPtrErr pBuffer == NULL +// pState == NULL +// ippStsNoErr no errors +// +// Parameters: +// pBuffer pointer to the source buffer +// pState pointer hash state +// +*F*/ +IPPFUN(IppStatus, ippsHashUnpack_rmf,(const Ipp8u* pBuffer, IppsHashState_rmf* pState)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pState, pBuffer); + + CopyBlock(pBuffer, pState, sizeof(IppsHashState_rmf)); + HASH_SET_ID(pState, idCtxHash); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashupdate.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashupdate.c new file mode 100644 index 0000000..68ed9b8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashupdate.c @@ -0,0 +1,137 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// General Functionality +// +// Contents: +// ippsHashUpdate() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcptool.h" + + +/*F* +// Name: ippsHashUpdate +// +// Purpose: Updates intermediate hash value based on input stream. +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL +// pSrc==0 but len!=0 +// ippStsContextMatchErr pState->idCtx != idCtxHash +// ippStsLengthErr len <0 +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the input stream +// len input stream length +// pState pointer to the Hash context +// +*F*/ +__INLINE int IsExceedMsgLen(Ipp64u maxLo, Ipp64u maxHi, Ipp64u lenLo, Ipp64u lenHi) +{ + int isExceed = lenLo > maxLo; + isExceed = (lenHi+(Ipp64u)isExceed) > maxHi; + return isExceed; +} + +IPPFUN(IppStatus, ippsHashUpdate,(const Ipp8u* pSrc, int len, IppsHashState* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pState); + /* test the context */ + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxHash), ippStsContextMatchErr); + /* test input length */ + IPP_BADARG_RET((len<0 && pSrc), ippStsLengthErr); + /* test source pointer */ + IPP_BADARG_RET((len && !pSrc), ippStsNullPtrErr); + + /* handle non empty input */ + if(len) { + const cpHashAttr* pAttr = &cpHashAlgAttr[HASH_ALG_ID(pState)]; + + /* test if size of message is being processed not exceeded yet */ + Ipp64u lenLo = HASH_LENLO(pState); + Ipp64u lenHi = HASH_LENHI(pState); + lenLo += (Ipp64u)len; + if(lenLo < HASH_LENLO(pState)) lenHi++; + if(IsExceedMsgLen(pAttr->msgLenMax[0],pAttr->msgLenMax[1], lenLo,lenHi)) + IPP_ERROR_RET(ippStsLengthErr); + + else { + cpHashProc hashFunc = HASH_FUNC(pState); /* processing function */ + const void* pParam = HASH_FUNC_PAR(pState); /* and it's addition params */ + int mbs = pAttr->msgBlkSize; /* data block size */ + + /* + // processing + */ + { + int procLen; + + /* test if internal buffer is not empty */ + int n = HASH_BUFFIDX(pState); + if(n) { + procLen = IPP_MIN(len, (mbs-n)); + CopyBlock(pSrc, HASH_BUFF(pState)+n, procLen); + HASH_BUFFIDX(pState) = n += procLen; + + /* block processing */ + if(mbs==n) { + hashFunc(HASH_VALUE(pState), HASH_BUFF(pState), mbs, pParam); + HASH_BUFFIDX(pState) = 0; + } + + /* update message pointer and length */ + pSrc += procLen; + len -= procLen; + } + + /* main processing part */ + procLen = len & ~(mbs-1); + if(procLen) { + hashFunc(HASH_VALUE(pState), pSrc, procLen, pParam); + pSrc += procLen; + len -= procLen; + } + + /* rest of input message */ + if(len) { + CopyBlock(pSrc, HASH_BUFF(pState), len); + HASH_BUFFIDX(pState) += len; + } + } + + /* update length of processed message */ + HASH_LENLO(pState) = lenLo; + HASH_LENHI(pState) = lenHi; + + return ippStsNoErr; + } + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashupdate_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashupdate_rmf.c new file mode 100644 index 0000000..78bbdf2 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphashupdate_rmf.c @@ -0,0 +1,115 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Security Hash Standard +// Generalized Functionality +// +// Contents: +// ippsHashUpdate_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsHashUpdate_rmf +// +// Purpose: Updates intermediate hash value based on input stream. +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL +// ippStsNullPtrErr pSrc==0 but len!=0 +// ippStsContextMatchErr pState->idCtx != idCtxHash +// ippStsLengthErr len <0 +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the input stream +// len input stream length +// pState pointer to the Hash context +// +*F*/ +IPPFUN(IppStatus, ippsHashUpdate_rmf,(const Ipp8u* pSrc, int len, IppsHashState_rmf* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxHash), ippStsContextMatchErr); + + /* test input length */ + IPP_BADARG_RET((len<0), ippStsLengthErr); + /* test source pointer */ + IPP_BADARG_RET((len && !pSrc), ippStsNullPtrErr); + + if(len) { + const IppsHashMethod* method = HASH_METHOD(pState); + hashUpdateF hashFunc = method->hashUpdate; /* processing function */ + int msgBlkSize = method->msgBlkSize; /* messge block size */ + + int procLen; + + int idx = HASH_BUFFIDX(pState); + Ipp64u lenLo = HASH_LENLO(pState); + Ipp64u lenHi = HASH_LENHI(pState); + lenLo += (Ipp64u)len; + if(lenLo < HASH_LENLO(pState)) lenHi++; + + /* if internal buffer is not empty */ + if(idx) { + procLen = IPP_MIN(len, (msgBlkSize-idx)); + CopyBlock(pSrc, HASH_BUFF(pState)+idx, procLen); + idx += procLen; + + /* process complete message block */ + if(msgBlkSize==idx) { + hashFunc(HASH_VALUE(pState), HASH_BUFF(pState), msgBlkSize); + idx = 0; + } + + /* update message pointer and length */ + pSrc += procLen; + len -= procLen; + } + + /* process main part of the input*/ + procLen = len & ~(msgBlkSize-1); + if(procLen) { + hashFunc(HASH_VALUE(pState), pSrc, procLen); + pSrc += procLen; + len -= procLen; + } + + /* store the rest of input in the buffer */ + if(len) { + CopyBlock(pSrc, HASH_BUFF(pState), len); + idx += len; + } + + /* update length of processed message */ + HASH_LENLO(pState) = lenLo; + HASH_LENHI(pState) = lenHi; + HASH_BUFFIDX(pState) = idx; + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac.h new file mode 100644 index 0000000..0eb3691 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac.h @@ -0,0 +1,52 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Hash Message Authentication Code +// Internal Definitions and Internal Functions Prototypes +// +// +*/ + +#if !defined(_PCP_HMAC_H) +#define _PCP_HMAC_H + +#include "pcphash.h" + +/* +// HMAC context +*/ +struct _cpHMAC { + Ipp32u idCtx; /* HMAC identifier */ + Ipp8u ipadKey[MBS_HASH_MAX]; /* inner padding key */ + Ipp8u opadKey[MBS_HASH_MAX]; /* outer padding key */ + IppsHashState hashCtx; /* hash context */ +}; + +/* accessors */ +#define HMAC_SET_CTX_ID(stt) ((stt)->idCtx = (Ipp32u)idCtxHMAC ^ (Ipp32u)IPP_UINT_PTR(stt)) +#define HMAC_RESET_CTX_ID(stt) ((stt)->idCtx = idCtxHMAC) +#define HASH_CTX(stt) ((stt)->hashCtx) +#define HMAC_VALID_ID(stt) ((((stt)->idCtx) ^ (Ipp32u)IPP_INT_PTR((stt))) == (Ipp32u)idCtxHMAC) + +#define IPAD (0x36) /* inner padding value */ +#define OPAD (0x5C) /* outer padding value */ + +#endif /* _PCP_HMAC_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_duplicate.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_duplicate.c new file mode 100644 index 0000000..82d9d14 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_duplicate.c @@ -0,0 +1,68 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// HMAC General Functionality +// +// Contents: +// ippsHMAC_Duplicate() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphmac.h" +#include "pcptool.h" + +/*F* +// Name: ippsHMAC_Duplicate +// +// Purpose: Clone HMAC state. +// +// Returns: Reason: +// ippStsNullPtrErr pSrcState == NULL +// pDstState == NULL +// ippStsContextMatchErr pSrcState->idCtx != idCtxHMAC +// pDstState->idCtx != idCtxHMAC +// ippStsNoErr no errors +// +// Parameters: +// pSrcState pointer to the source HMAC state +// pDstState pointer to the target HMAC state +// +// Note: +// pDstState may not to be initialized by ippsHMACInit() +// +*F*/ +IPPFUN(IppStatus, ippsHMAC_Duplicate,(const IppsHMACState* pSrcCtx, IppsHMACState* pDstCtx)) +{ + /* test state pointers */ + IPP_BAD_PTR2_RET(pSrcCtx, pDstCtx); + /* test states ID */ + IPP_BADARG_RET(!HMAC_VALID_ID(pSrcCtx), ippStsContextMatchErr); + + /* copy HMAC state without Hash context */ + CopyBlock(pSrcCtx, pDstCtx, (int)(IPP_UINT_PTR(&HASH_CTX(pSrcCtx)) - IPP_UINT_PTR(pSrcCtx))); + HMAC_SET_CTX_ID(pDstCtx); + /* copy Hash context separately */ + ippsHashDuplicate(&HASH_CTX(pSrcCtx), &HASH_CTX(pDstCtx)); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_final.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_final.c new file mode 100644 index 0000000..b478d1c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_final.c @@ -0,0 +1,93 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// HMAC General Functionality +// +// Contents: +// ippsHMAC_Final() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphmac.h" +#include "pcptool.h" + +/*F* +// Name: ippsHMAC_Final +// +// Purpose: Stop message digesting and return digest. +// +// Returns: Reason: +// ippStsNullPtrErr pMD == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxHMAC +// ippStsLengthErr sizeof(DigestMD5) < mdLen <1 +// ippStsNoErr no errors +// +// Parameters: +// pMD address of the output digest +// pState pointer to the HMAC state +// +*F*/ +IPPFUN(IppStatus, ippsHMAC_Final,(Ipp8u* pMD, int mdLen, IppsHMACState* pCtx)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pCtx); + IPP_BADARG_RET(!HMAC_VALID_ID(pCtx), ippStsContextMatchErr); + + /* test MD pointer and length */ + IPP_BAD_PTR1_RET(pMD); + IPP_BADARG_RET(mdLen<=0, ippStsLengthErr); + + { + /* hash specific */ + IppsHashState* pHashCtx = &HASH_CTX(pCtx); + int mbs = cpHashMBS(HASH_ALG_ID(pHashCtx)); + int hashSize = cpHashSize(HASH_ALG_ID(pHashCtx)); + if(mdLen>hashSize) + IPP_ERROR_RET(ippStsLengthErr); + + /* + // finalize hmac + */ + { + /* finalize 1-st step */ + Ipp8u md[IPP_SHA512_DIGEST_BITSIZE/8]; + IppStatus sts = ippsHashFinal(md, pHashCtx); + + if(ippStsNoErr==sts) { + /* perform outer hash */ + ippsHashUpdate(pCtx->opadKey, mbs, pHashCtx); + ippsHashUpdate(md, hashSize, pHashCtx); + + /* complete HMAC */ + ippsHashFinal(md, pHashCtx); + CopyBlock(md, pMD, IPP_MIN(hashSize, mdLen)); + + /* ready to the next HMAC computation */ + ippsHashUpdate(pCtx->ipadKey, mbs, pHashCtx); + } + + return sts; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_getsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_getsize.c new file mode 100644 index 0000000..83a0ad6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_getsize.c @@ -0,0 +1,55 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// HMAC General Functionality +// +// Contents: +// ippsHMAC_GetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphmac.h" +#include "pcptool.h" + +/*F* +// Name: ippsHMAC_GetSize +// +// Purpose: Returns size of HMAC state (bytes). +// +// Returns: Reason: +// ippStsNullPtrErr pSzie == NULL +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to the HMAC state size +// +*F*/ +IPPFUN(IppStatus, ippsHMAC_GetSize,(int* pSize)) +{ + /* test size's pointer */ + IPP_BAD_PTR1_RET(pSize); + + *pSize = sizeof(IppsHMACState); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_gettag.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_gettag.c new file mode 100644 index 0000000..a4ca40e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_gettag.c @@ -0,0 +1,70 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// HMAC General Functionality +// +// Contents: +// ippsHMAC_GetTag() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphmac.h" +#include "pcptool.h" + +/*F* +// Name: ippsHMAC_GetTag +// +// Purpose: Compute digest with further digesting ability. +// +// Returns: Reason: +// ippStsNullPtrErr pMD == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxHMAC +// ippStsLengthErr size_of_digest < mdLen <1 +// ippStsNoErr no errors +// +// Parameters: +// pMD address of the output digest +// mdLen length of the digest +// pState pointer to the HMAC state +// +*F*/ +IPPFUN(IppStatus, ippsHMAC_GetTag,(Ipp8u* pMD, int mdLen, const IppsHMACState* pCtx)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pCtx); + IPP_BADARG_RET(!HMAC_VALID_ID(pCtx), ippStsContextMatchErr); + + /* test MD pointer */ + IPP_BAD_PTR1_RET(pMD); + + { /* TBD: consider implementation without copy of context */ + IppStatus sts; + IppsHMACState tmpCtx; + ippsHMAC_Duplicate(pCtx, &tmpCtx); + sts = ippsHMAC_Final(pMD, mdLen, &tmpCtx); + + PurgeBlock(&tmpCtx, sizeof(IppsHMACState)); + return sts; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_init.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_init.c new file mode 100644 index 0000000..8a4d815 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_init.c @@ -0,0 +1,113 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// HMAC General Functionality +// +// Contents: +// ippsHMAC_Init() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphmac.h" +#include "pcptool.h" + +/*F* +// Name: ippsHMAC_Init +// +// Purpose: Init HMAC state. +// +// Returns: Reason: +// ippStsNullPtrErr pKey == NULL +// pState == NULL +// ippStsLengthErr keyLen <0 +// ippStsNotSupportedModeErr if algID is not match to supported hash alg +// ippStsNoErr no errors +// +// Parameters: +// pKey pointer to the secret key +// keyLen length (bytes) of the secret key +// pState pointer to the HMAC state +// hashAlg hash alg ID +// +*F*/ +IPPFUN(IppStatus, ippsHMAC_Init,(const Ipp8u* pKey, int keyLen, IppsHMACState* pCtx, IppHashAlgId hashAlg)) +{ + //int mbs; + + /* get algorithm id */ + hashAlg = cpValidHashAlg(hashAlg); + /* test hash alg */ + IPP_BADARG_RET(ippHashAlg_Unknown==hashAlg, ippStsNotSupportedModeErr); + //mbs = cpHashMBS(hashAlg); + + /* test pState pointer */ + IPP_BAD_PTR1_RET(pCtx); + + /* test key pointer and key length */ + IPP_BAD_PTR1_RET(pKey); + IPP_BADARG_RET(0>keyLen, ippStsLengthErr); + + /* set state ID */ + HMAC_SET_CTX_ID(pCtx); + + /* init hash context */ + ippsHashInit(&HASH_CTX(pCtx), hashAlg); + + { + int n; + + /* hash specific */ + IppsHashState* pHashCtx = &HASH_CTX(pCtx); + int mbs = cpHashMBS(hashAlg); + int hashSize = cpHashSize(hashAlg); + + /* copyMask = keyLen>mbs? 0xFF : 0x00 */ + int copyMask = (mbs-keyLen) >>(BITSIZE(int)-1); + + /* actualKeyLen = keyLen>mbs? hashSize:keyLen */ + int actualKeyLen = (hashSize & copyMask) | (keyLen & ~copyMask); + + /* compute hash(key, keyLen) just in case */ + ippsHashUpdate(pKey, keyLen, pHashCtx); + ippsHashFinal(HASH_BUFF(pHashCtx), pHashCtx); + + /* copy either key or hash(key) into ipad- and opad- buffers */ + MASKED_COPY_BNU(pCtx->ipadKey, (Ipp8u)copyMask, HASH_BUFF(pHashCtx), pKey, actualKeyLen); + MASKED_COPY_BNU(pCtx->opadKey, (Ipp8u)copyMask, HASH_BUFF(pHashCtx), pKey, actualKeyLen); + + /* XOR-ing key */ + for(n=0; nipadKey[n] ^= (Ipp8u)IPAD; + pCtx->opadKey[n] ^= (Ipp8u)OPAD; + } + for(; nipadKey[n] = (Ipp8u)IPAD; + pCtx->opadKey[n] = (Ipp8u)OPAD; + } + + /* ipad key processing */ + ippsHashUpdate(pCtx->ipadKey, mbs, pHashCtx); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_message.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_message.c new file mode 100644 index 0000000..e3d41c0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_message.c @@ -0,0 +1,95 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// HMAC General Functionality +// +// Contents: +// ippsHMAC_Message() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphmac.h" +#include "pcptool.h" + +/*F* +// Name: ippsHMAC_Message +// +// Purpose: MAC (MD5) of the whole message. +// +// Returns: Reason: +// ippStsNullPtrErr pMsg == NULL +// pKey == NULL +// pMD == NULL +// ippStsLengthErr msgLen <0 +// keyLen <0 +// size_of_digest < mdLen <1 +// ippStsNotSupportedModeErr if algID is not match to supported hash alg +// ippStsNoErr no errors +// +// Parameters: +// pMsg pointer to the input message +// msgLen input message length +// pKey pointer to the secret key +// keyLen secret key length +// pMD pointer to message digest +// mdLen MD length +// hashAlg hash alg ID +// +*F*/ +IPPFUN(IppStatus, ippsHMAC_Message,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pKey, int keyLen, + Ipp8u* pMD, int mdLen, + IppHashAlgId hashAlg)) +{ + /* get algorithm id */ + hashAlg = cpValidHashAlg(hashAlg); + /* test hash alg */ + IPP_BADARG_RET(ippHashAlg_Unknown==hashAlg, ippStsNotSupportedModeErr); + + /* test secret key pointer and length */ + IPP_BAD_PTR1_RET(pKey); + IPP_BADARG_RET((keyLen<0), ippStsLengthErr); + + /* test input message pointer and length */ + IPP_BADARG_RET((msgLen<0), ippStsLengthErr); + IPP_BADARG_RET((msgLen && !pMsg), ippStsNullPtrErr); + + /* test MD pointer and length */ + IPP_BAD_PTR1_RET(pMD); + IPP_BADARG_RET(0>=mdLen || mdLen>cpHashSize(hashAlg), ippStsLengthErr); + + { + __ALIGN8 IppsHMACState ctx; + IppStatus sts = ippsHMAC_Init(pKey, keyLen, &ctx, hashAlg); + if(ippStsNoErr!=sts) goto exit; + + sts = ippsHashUpdate(pMsg,msgLen, &HASH_CTX(&ctx)); + if(ippStsNoErr!=sts) goto exit; + + sts = ippsHMAC_Final(pMD, mdLen, &ctx); + + exit: + PurgeBlock(&ctx, sizeof(IppsHMACState)); + return sts; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_pack.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_pack.c new file mode 100644 index 0000000..1188ebb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_pack.c @@ -0,0 +1,74 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// HMAC General Functionality +// +// Contents: +// ippsHMAC_Pack() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphmac.h" +#include "pcptool.h" + +/*F* +// Name: ippsHMAC_Pack +// +// Purpose: Copy initialized context to the buffer. +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL +// pCtx == NULL +// ippStsNoErr no errors +// +// Parameters: +// pCtx pointer keyed hash state +// pBuffer pointer to the destination buffer +// bufSize size of destination buffer +// +*F*/ +IPPFUN(IppStatus, ippsHMAC_Pack,(const IppsHMACState* pCtx, Ipp8u* pBuffer, int bufSize)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pCtx, pBuffer); + /* test the context */ + IPP_BADARG_RET(!HMAC_VALID_ID(pCtx), ippStsContextMatchErr); + + { + int ctxSize; + ippsHMAC_GetSize(&ctxSize); + /* test buffer length */ + IPP_BADARG_RET(ctxSize>bufSize, ippStsNoMemErr); + + CopyBlock(pCtx, pBuffer, ctxSize); + + /* Reset IppsHMACState context id */ + IppsHMACState* pCopy = (IppsHMACState*)pBuffer; + HMAC_RESET_CTX_ID(pCopy); + /* Reset context id for IppsHashState, which is the part of IppsHMACState */ + IppsHashState* pHashCopy = (IppsHashState*)&HASH_CTX(pCopy); + HASH_RESET_ID(pHashCopy,idCtxHash); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_rmf.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_rmf.h new file mode 100644 index 0000000..9ddb5ce --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_rmf.h @@ -0,0 +1,41 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Cryptography Primitive. +// Hash Message Authentication Code +// Internal Definitions and Internal Functions Prototypes +*/ + +#if !defined(_PCP_HMAC_RMF_H) +#define _PCP_HMAC_RMF_H + +#include "pcphash_rmf.h" + +/* +// HMAC context +*/ +struct _cpHMAC_rmf { + Ipp32u idCtx; /* HMAC identifier */ + Ipp8u ipadKey[MBS_HASH_MAX]; /* inner padding key */ + Ipp8u opadKey[MBS_HASH_MAX]; /* outer padding key */ + IppsHashState_rmf hashCtx; /* hash context */ +}; + + +#endif /* _PCP_HMAC_RMF_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_unpack.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_unpack.c new file mode 100644 index 0000000..40d045f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_unpack.c @@ -0,0 +1,60 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// HMAC General Functionality +// +// Contents: +// ippsHMAC_Unpack() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphmac.h" +#include "pcptool.h" + +/*F* +// Name: ippsHMAC_Unpack +// +// Purpose: Unpack buffer content into the initialized context. +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL +// pCtx == NULL +// ippStsNoErr no errors +// +// Parameters: +// pCtx pointer keyed hash state +// pSize pointer to the packed spec size +// +*F*/ +IPPFUN(IppStatus, ippsHMAC_Unpack,(const Ipp8u* pBuffer, IppsHMACState* pCtx)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pCtx, pBuffer); + + CopyBlock(pBuffer, pCtx, sizeof(IppsHMACState)); + /* Set IppsHMACState context id */ + HMAC_SET_CTX_ID(pCtx); + /* Set context id for IppsHashState, which is the part of IppsHMACState */ + HASH_SET_ID(&HASH_CTX(pCtx),idCtxHash); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_update.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_update.c new file mode 100644 index 0000000..51295d4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmac_update.c @@ -0,0 +1,68 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// HMAC General Functionality +// +// Contents: +// ippsHMAC_Update() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphmac.h" +#include "pcptool.h" + +/*F* +// Name: ippsHMAC_Update +// +// Purpose: Updates intermadiate MAC based on input stream. +// +// Returns: Reason: +// ippStsNullPtrErr pSrc == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxHMAC +// ippStsLengthErr len <0 +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the input stream +// len input stream length +// pState pointer to the HMAC state +// +*F*/ +IPPFUN(IppStatus, ippsHMAC_Update,(const Ipp8u* pSrc, int len, IppsHMACState* pCtx)) +{ + /* test state pointers */ + IPP_BAD_PTR1_RET(pCtx); + + /* test state ID */ + IPP_BADARG_RET(!HMAC_VALID_ID(pCtx), ippStsContextMatchErr); + /* test input length */ + IPP_BADARG_RET((len<0), ippStsLengthErr); + /* test source pointer */ + IPP_BADARG_RET((len && !pSrc), ippStsNullPtrErr); + + if(len) + return ippsHashUpdate(pSrc, len, &HASH_CTX(pCtx)); + else + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacca_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacca_rmf.c new file mode 100644 index 0000000..7b080b4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacca_rmf.c @@ -0,0 +1,107 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// HMAC General Functionality +// +// Contents: +// ippsHMACInit_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphmac.h" +#include "pcphmac_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsHMACInit_rmf +// +// Purpose: Init HMAC state. +// +// Returns: Reason: +// ippStsNullPtrErr pKey == NULL +// pState == NULL +// ippStsLengthErr keyLen <0 +// ippStsNoErr no errors +// +// Parameters: +// pKey pointer to the secret key +// keyLen length (bytes) of the secret key +// pState pointer to the HMAC state +// pMethod hash method +// +*F*/ +IPPFUN(IppStatus, ippsHMACInit_rmf,(const Ipp8u* pKey, int keyLen, + IppsHMACState_rmf* pCtx, + const IppsHashMethod* pMethod)) +{ + /* test pointer */ + IPP_BAD_PTR2_RET(pCtx, pMethod); + + /* test key pointer and key length */ + IPP_BAD_PTR1_RET(pKey); + IPP_BADARG_RET(0>keyLen, ippStsLengthErr); + + /* set state ID */ + HMAC_SET_CTX_ID(pCtx); + + /* init hash context */ + ippsHashInit_rmf(&HASH_CTX(pCtx), pMethod); + + { + int n; + + /* hash specific */ + IppsHashState_rmf* pHashCtx = &HASH_CTX(pCtx); + int mbs = pMethod->msgBlkSize; + int hashSize = pMethod->hashLen; + + /* copyMask = keyLen>mbs? 0xFF : 0x00 */ + int copyMask = (mbs-keyLen) >>(BITSIZE(int)-1); + + /* actualKeyLen = keyLen>mbs? hashSize:keyLen */ + int actualKeyLen = (hashSize & copyMask) | (keyLen & ~copyMask); + + /* compute hash(key, keyLen) just in case */ + ippsHashUpdate_rmf(pKey, keyLen, pHashCtx); + ippsHashFinal_rmf(HASH_BUFF(pHashCtx), pHashCtx); + + /* copy either key or hash(key) into ipad- and opad- buffers */ + MASKED_COPY_BNU(pCtx->ipadKey, (Ipp8u)copyMask, HASH_BUFF(pHashCtx), pKey, actualKeyLen); + MASKED_COPY_BNU(pCtx->opadKey, (Ipp8u)copyMask, HASH_BUFF(pHashCtx), pKey, actualKeyLen); + + /* XOR-ing key */ + for(n=0; nipadKey[n] ^= (Ipp8u)IPAD; + pCtx->opadKey[n] ^= (Ipp8u)OPAD; + } + for(; nipadKey[n] = (Ipp8u)IPAD; + pCtx->opadKey[n] = (Ipp8u)OPAD; + } + + /* ipad key processing */ + ippsHashUpdate_rmf(pCtx->ipadKey, mbs, pHashCtx); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacduplicate_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacduplicate_rmf.c new file mode 100644 index 0000000..ad5660e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacduplicate_rmf.c @@ -0,0 +1,69 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// HMAC General Functionality +// +// Contents: +// ippsHMACDuplicate_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphmac.h" +#include "pcphmac_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsHMACDuplicate_rmf +// +// Purpose: Clone HMAC state. +// +// Returns: Reason: +// ippStsNullPtrErr pSrcState == NULL +// pDstState == NULL +// ippStsContextMatchErr pSrcState->idCtx != idCtxHMAC +// pDstState->idCtx != idCtxHMAC +// ippStsNoErr no errors +// +// Parameters: +// pSrcState pointer to the source HMAC state +// pDstState pointer to the target HMAC state +// +// Note: +// pDstState may not to be initialized by ippsHMACInit_rmf() +// +*F*/ +IPPFUN(IppStatus, ippsHMACDuplicate_rmf,(const IppsHMACState_rmf* pSrcCtx, IppsHMACState_rmf* pDstCtx)) +{ + /* test state pointers */ + IPP_BAD_PTR2_RET(pSrcCtx, pDstCtx); + /* test states ID */ + IPP_BADARG_RET(!HMAC_VALID_ID(pSrcCtx), ippStsContextMatchErr); + + /* copy HMAC state without Hash context */ + CopyBlock(pSrcCtx, pDstCtx, (int)(IPP_UINT_PTR(&HASH_CTX(pSrcCtx)) - IPP_UINT_PTR(pSrcCtx))); + HMAC_SET_CTX_ID(pDstCtx); + /* copy Hash context separately */ + ippsHashDuplicate_rmf(&HASH_CTX(pSrcCtx), &HASH_CTX(pDstCtx)); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacfinal_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacfinal_rmf.c new file mode 100644 index 0000000..99504fa --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacfinal_rmf.c @@ -0,0 +1,95 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// HMAC General Functionality +// +// Contents: +// ippsHMACFinal_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphmac.h" +#include "pcphmac_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsHMACFinal_rmf +// +// Purpose: Stop message digesting and return digest. +// +// Returns: Reason: +// ippStsNullPtrErr pMD == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxHMAC +// ippStsLengthErr sizeof(DigestMD5) < mdLen <1 +// ippStsNoErr no errors +// +// Parameters: +// pMD address of the output digest +// pState pointer to the HMAC state +// +*F*/ +IPPFUN(IppStatus, ippsHMACFinal_rmf,(Ipp8u* pMD, int mdLen, IppsHMACState_rmf* pCtx)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pCtx); + IPP_BADARG_RET(!HMAC_VALID_ID(pCtx), ippStsContextMatchErr); + + /* test MD pointer and length */ + IPP_BAD_PTR1_RET(pMD); + IPP_BADARG_RET(mdLen<=0, ippStsLengthErr); + + { + /* hash specific */ + IppsHashState_rmf* pHashCtx = &HASH_CTX(pCtx); + const IppsHashMethod* method = HASH_METHOD(pHashCtx); + int mbs = method->msgBlkSize; + int hashSize = method->hashLen; + if(mdLen>hashSize) + IPP_ERROR_RET(ippStsLengthErr); + + /* + // finalize hmac + */ + { + /* finalize 1-st step */ + Ipp8u md[IPP_SHA512_DIGEST_BITSIZE/8]; + IppStatus sts = ippsHashFinal_rmf(md, pHashCtx); + + if(ippStsNoErr==sts) { + /* perform outer hash */ + ippsHashUpdate_rmf(pCtx->opadKey, mbs, pHashCtx); + ippsHashUpdate_rmf(md, hashSize, pHashCtx); + + /* complete HMAC */ + ippsHashFinal_rmf(md, pHashCtx); + CopyBlock(md, pMD, IPP_MIN(hashSize, mdLen)); + + /* ready to the next HMAC computation */ + ippsHashUpdate_rmf(pCtx->ipadKey, mbs, pHashCtx); + } + + return sts; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacgetsize_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacgetsize_rmf.c new file mode 100644 index 0000000..736e6ff --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacgetsize_rmf.c @@ -0,0 +1,56 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// HMAC General Functionality +// +// Contents: +// ippsHMACGetSize_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphmac.h" +#include "pcphmac_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsHMACGetSize_rmf +// +// Purpose: Returns size of HMAC state (bytes). +// +// Returns: Reason: +// ippStsNullPtrErr pSzie == NULL +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to the HMAC state size +// +*F*/ +IPPFUN(IppStatus, ippsHMACGetSize_rmf,(int* pSize)) +{ + /* test size's pointer */ + IPP_BAD_PTR1_RET(pSize); + + *pSize = sizeof(IppsHMACState_rmf); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacgettag_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacgettag_rmf.c new file mode 100644 index 0000000..ef4e363 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacgettag_rmf.c @@ -0,0 +1,72 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// HMAC General Functionality +// +// Contents: +// ippsHMACGetTag_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphmac.h" +#include "pcphmac_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsHMACGetTag_rmf +// +// Purpose: Compute digest with further digesting ability. +// +// Returns: Reason: +// ippStsNullPtrErr pMD == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxHMAC +// ippStsLengthErr size_of_digest < mdLen <1 +// ippStsNoErr no errors +// +// Parameters: +// pMD address of the output digest +// mdLen length of the digest +// pState pointer to the HMAC state +// +*F*/ +IPPFUN(IppStatus, ippsHMACGetTag_rmf,(Ipp8u* pMD, int mdLen, const IppsHMACState_rmf* pCtx)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pCtx); + IPP_BADARG_RET(!HMAC_VALID_ID(pCtx), ippStsContextMatchErr); + + /* test MD pointer */ + IPP_BAD_PTR1_RET(pMD); + + { /* TBD: consider implementation without copy of context */ + IppStatus sts; + IppsHMACState_rmf tmpCtx; + ippsHMACDuplicate_rmf(pCtx, &tmpCtx); + + sts = ippsHMACFinal_rmf(pMD, mdLen, &tmpCtx); + + PurgeBlock(&tmpCtx, sizeof(IppsHMACState_rmf)); + return sts; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacmessage_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacmessage_rmf.c new file mode 100644 index 0000000..e47d772 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacmessage_rmf.c @@ -0,0 +1,93 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// HMAC General Functionality +// +// Contents: +// ippsHMACMessage_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphmac.h" +#include "pcphmac_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsHMACMessage_rmf +// +// Purpose: MAC (MD5) of the whole message. +// +// Returns: Reason: +// ippStsNullPtrErr pMsg == NULL +// pKey == NULL +// pMD == NULL +// ippStsLengthErr msgLen <0 +// keyLen <0 +// size_of_digest < mdLen <1 +// ippStsNotSupportedModeErr if algID is not match to supported hash alg +// ippStsNoErr no errors +// +// Parameters: +// pMsg pointer to the input message +// msgLen input message length +// pKey pointer to the secret key +// keyLen secret key length +// pMD pointer to message digest +// mdLen MD length +// pMethod hash method +// +*F*/ +IPPFUN(IppStatus, ippsHMACMessage_rmf,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pKey, int keyLen, + Ipp8u* pMD, int mdLen, + const IppsHashMethod* pMethod)) +{ + /* test method pointer */ + IPP_BAD_PTR1_RET(pMethod); + /* test secret key pointer and length */ + IPP_BAD_PTR1_RET(pKey); + IPP_BADARG_RET((keyLen<0), ippStsLengthErr); + /* test input message pointer and length */ + IPP_BADARG_RET((msgLen<0), ippStsLengthErr); + IPP_BADARG_RET((msgLen && !pMsg), ippStsNullPtrErr); + + /* test MD pointer and length */ + IPP_BAD_PTR1_RET(pMD); + IPP_BADARG_RET(0>=mdLen || mdLen>pMethod->hashLen, ippStsLengthErr); + + { + __ALIGN8 IppsHMACState_rmf ctx; + IppStatus sts; + + ippsHMACInit_rmf(pKey, keyLen, &ctx, pMethod); + + sts = ippsHashUpdate_rmf(pMsg,msgLen, &HASH_CTX(&ctx)); + if(ippStsNoErr!=sts) goto exit; + + sts = ippsHMACFinal_rmf(pMD, mdLen, &ctx); + + exit: + PurgeBlock(&ctx, sizeof(IppsHMACState_rmf)); + return sts; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacpack_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacpack_rmf.c new file mode 100644 index 0000000..9a6a05d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacpack_rmf.c @@ -0,0 +1,75 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// HMAC General Functionality +// +// Contents: +// ippsHMACPack_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphmac.h" +#include "pcphmac_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsHMACPack_rmf +// +// Purpose: Copy initialized context to the buffer. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// pBuffer == NULL +// ippStsNoErr no errors +// +// Parameters: +// pCtx pointer keyed hash state +// pBuffer pointer to the destination buffer +// bufSize size of destination buffer +// +*F*/ +IPPFUN(IppStatus, ippsHMACPack_rmf,(const IppsHMACState_rmf* pCtx, Ipp8u* pBuffer, int bufSize)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pCtx, pBuffer); + /* test the context */ + IPP_BADARG_RET(!HMAC_VALID_ID(pCtx), ippStsContextMatchErr); + + { + int ctxSize; + ippsHMACGetSize_rmf(&ctxSize); + /* test buffer length */ + IPP_BADARG_RET(ctxSize>bufSize, ippStsNoMemErr); + + CopyBlock(pCtx, pBuffer, ctxSize); + + /* Reset IppsHMACState_rmf context id */ + IppsHMACState_rmf* pCopy = (IppsHMACState_rmf*)pBuffer; + HMAC_RESET_CTX_ID(pCopy); + /* Reset context id for IppsHashState_rmf, which is the part of IppsHMACState_rmf */ + IppsHashState_rmf* pHashCopy = (IppsHashState_rmf*)&HASH_CTX(pCopy); + HASH_RESET_ID(pHashCopy,idCtxHash); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacunpack_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacunpack_rmf.c new file mode 100644 index 0000000..2aec545 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacunpack_rmf.c @@ -0,0 +1,63 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// HMAC General Functionality +// +// Contents: +// ippsHMACUnpack_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphmac.h" +#include "pcphmac_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsHMACUnpack_rmf +// +// Purpose: Unpack buffer content into the initialized context. +// +// Returns: Reason: +// ippStsNullPtrErr pBuffer == NULL +// pCtx == NULL +// ippStsNoErr no errors +// +// Parameters: +// pBuffer pointer to the source buffer +// pSize pointer to the packed spec size +// +*F*/ +IPPFUN(IppStatus, ippsHMACUnpack_rmf,(const Ipp8u* pBuffer, IppsHMACState_rmf* pCtx)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pCtx, pBuffer); + + CopyBlock(pBuffer, pCtx, sizeof(IppsHMACState_rmf)); + + /* Set IppsHMACState_rmf context id */ + HMAC_SET_CTX_ID(pCtx); + /* Set context id for IppsHashState_rmf, which is the part of IppsHMACState_rmf */ + HASH_SET_ID(&HASH_CTX(pCtx),idCtxHash); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacupdate_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacupdate_rmf.c new file mode 100644 index 0000000..898b213 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcphmacupdate_rmf.c @@ -0,0 +1,69 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// HMAC General Functionality +// +// Contents: +// ippsHMACUpdate_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphmac.h" +#include "pcphmac_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsHMACUpdate_rmf +// +// Purpose: Updates intermadiate MAC based on input stream. +// +// Returns: Reason: +// ippStsNullPtrErr pSrc == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxHMAC +// ippStsLengthErr len <0 +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the input stream +// len input stream length +// pState pointer to the HMAC state +// +*F*/ +IPPFUN(IppStatus, ippsHMACUpdate_rmf,(const Ipp8u* pSrc, int len, IppsHMACState_rmf* pCtx)) +{ + /* test state pointers */ + IPP_BAD_PTR1_RET(pCtx); + + /* test state ID */ + IPP_BADARG_RET(!HMAC_VALID_ID(pCtx), ippStsContextMatchErr); + /* test input length */ + IPP_BADARG_RET((len<0), ippStsLengthErr); + /* test source pointer */ + IPP_BADARG_RET((len && !pSrc), ippStsNullPtrErr); + + if(len) + return ippsHashUpdate_rmf(pSrc, len, &HASH_CTX(pCtx)); + else + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmask_ct.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmask_ct.h new file mode 100644 index 0000000..0dc033d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmask_ct.h @@ -0,0 +1,162 @@ +/******************************************************************************* +* Copyright (C) 2018 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Constant time Mask operations +// +// +*/ + +#if !defined(_PCP_MASK_CT_H) +#define _PCP_MASK_CT_H + +#include "owncp.h" +#include "pcpbnuimpl.h" + +/* +// The following functions test particular conditions +// and returns either 0 or 0xffffffff. +// +// The result is suitable for boolean and masked operations. +// +// Inspite of operation below are using BNU_CHUNK_T operand(s) it can be applied to Ipp32u, Ipp32s, Ipp16u, Ipp16s, Ipp8u and Ipp8s too. +// For example, if +// Ipp32u uns_int; +// Ipp32s sgn_int; +// Ipp8u uns_char; +// Ipp8s sgn_char; +// then +// cpIs_msb_ct((Ipp32s)uns_int) tests 31 bit of uns_int +// cpIs_msb_ct( sgn_int) tests 31 bit of sgn_int +// cpIs_msb_ct((Ipp8u)uns_char) tests 7 bit of uns_char +// cpIs_msb_ct( sgn_char) tests 7 bit of sgn_char +*/ + + +/* Disable optimization for Clang compiler to produce constant execution time code */ +#if defined( __clang__ ) && !defined (__INTEL_COMPILER) + #pragma clang optimize off +#endif + +/* replace under mask: dst[] = replaceFlag? src[] : dst[] */ +__INLINE void cpMaskedReplace_ct(BNU_CHUNK_T* dst, const BNU_CHUNK_T* src, int len, BNU_CHUNK_T replaceMask) +{ + BNU_CHUNK_T dstMask = ~replaceMask; + int n; + for(n=0; n> (sizeof(a) * 8 - 1)); +} + +#if defined( __clang__ ) && !defined (__INTEL_COMPILER) + #pragma clang optimize on +#endif + +/* tests if LSB(a)==1 */ +__INLINE BNU_CHUNK_T cpIsLsb_ct(BNU_CHUNK_T a) +{ + return (BNU_CHUNK_T)0 - (a & 1); +} + +/* tests if a is odd */ +__INLINE BNU_CHUNK_T cpIsOdd_ct(BNU_CHUNK_T a) +{ + return cpIsLsb_ct(a); +} + +/* tests if a is even */ +__INLINE BNU_CHUNK_T cpIsEven_ct(BNU_CHUNK_T a) +{ + return ~cpIsLsb_ct(a); +} + +/* tests if a==0 */ +__INLINE BNU_CHUNK_T cpIsZero_ct(BNU_CHUNK_T a) +{ + return cpIsMsb_ct(~a & (a - 1)); +} + +/* tests if a==b */ +__INLINE BNU_CHUNK_T cpIsEqu_ct(BNU_CHUNK_T a, BNU_CHUNK_T b) +{ + return cpIsZero_ct(a ^ b); +} + +/* test if aidCtx != idCtxMD5 +// pDstState->idCtx != idCtxMD5 +// ippStsNoErr no errors +// +// Parameters: +// pSrcState pointer to the source MD5 state +// pDstState pointer to the target MD5 state +// +// Note: +// pDstState may to be uninitialized by ippsMD5Init() +// +*F*/ +IPPFUN(IppStatus, ippsMD5Duplicate,(const IppsMD5State* pSrcState, IppsMD5State* pDstState)) +{ + /* test state pointers */ + IPP_BAD_PTR2_RET(pSrcState, pDstState); + IPP_BADARG_RET(!HASH_VALID_ID(pSrcState, idCtxMD5), ippStsContextMatchErr); + + /* copy state */ + CopyBlock(pSrcState, pDstState, sizeof(IppsMD5State)); + HASH_SET_ID(pDstState, idCtxMD5); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmd5final.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmd5final.c new file mode 100644 index 0000000..4d4dd8e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmd5final.c @@ -0,0 +1,73 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to MD5 +// (derived from the RSA Data Security, Inc. MD5 Message-Digest Algorithm) +// +// Equivalent code is available from RFC 1321. +// +// Contents: +// ippsMD5Final() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpmd5stuff.h" + +/*F* +// Name: ippsMD5Final +// +// Purpose: Stop message digesting and return digest. +// +// Returns: Reason: +// ippStsNullPtrErr pMD == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxMD5 +// ippStsNoErr no errors +// +// Parameters: +// pMD address of the output digest +// pState pointer to the MD5 state +// +*F*/ +IPPFUN(IppStatus, ippsMD5Final,(Ipp8u* pMD, IppsMD5State* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxMD5), ippStsContextMatchErr); + + /* test digest pointer */ + IPP_BAD_PTR1_RET(pMD); + + cpFinalizeMD5(HASH_VALUE(pState), HASH_BUFF(pState), HASH_BUFFIDX(pState), HASH_LENLO(pState)); + CopyBlock(HASH_VALUE(pState), pMD, sizeof(DigestMD5)); + + /* re-init hash value */ + HASH_BUFFIDX(pState) = 0; + HASH_LENLO(pState) = 0; + md5_hashInit(HASH_VALUE(pState)); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmd5getsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmd5getsize.c new file mode 100644 index 0000000..b931f56 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmd5getsize.c @@ -0,0 +1,59 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to MD5 +// (derived from the RSA Data Security, Inc. MD5 Message-Digest Algorithm) +// +// Equivalent code is available from RFC 1321. +// +// Contents: +// ippsMD5GetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsMD5GetSize +// +// Purpose: Returns size (bytes) of IppsMD5State state. +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to size +// +*F*/ +IPPFUN(IppStatus, ippsMD5GetSize,(int* pSize)) +{ + /* test pointer */ + IPP_BAD_PTR1_RET(pSize); + + *pSize = sizeof(IppsMD5State); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmd5gettag.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmd5gettag.c new file mode 100644 index 0000000..f772a60 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmd5gettag.c @@ -0,0 +1,76 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to MD5 +// (derived from the RSA Data Security, Inc. MD5 Message-Digest Algorithm) +// +// Equivalent code is available from RFC 1321. +// +// Contents: +// ippsMD5GetTag() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpmd5stuff.h" + +/*F* +// Name: ippsMD5GetTag +// +// Purpose: Compute digest based on current state. +// Note, that futher digest update is possible +// +// Returns: Reason: +// ippStsNullPtrErr pTag == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxMD5 +// ippStsLengthErr max_MD5_digestLen < tagLen <1 +// ippStsNoErr no errors +// +// Parameters: +// pTag address of the output digest +// tagLen length of digest +// pState pointer to the MD5 state +// +*F*/ +IPPFUN(IppStatus, ippsMD5GetTag,(Ipp8u* pTag, Ipp32u tagLen, const IppsMD5State* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxMD5), ippStsContextMatchErr); + + /* test digest pointer */ + IPP_BAD_PTR1_RET(pTag); + IPP_BADARG_RET((tagLen<1)||(sizeof(DigestMD5)idCtx != idCtxMD5 +// ippStsNoErr no errors +// +// Parameters: +// pState pointer hash state +// pBuffer pointer to the destination buffer +// +*F*/ +IPPFUN(IppStatus, ippsMD5Pack,(const IppsMD5State* pState, Ipp8u* pBuffer)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pState, pBuffer); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxMD5), ippStsContextMatchErr); + + CopyBlock(pState, pBuffer, sizeof(IppsMD5State)); + IppsMD5State* pCopy = (IppsMD5State*)pBuffer; + HASH_RESET_ID(pCopy, idCtxMD5); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmd5stuff.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmd5stuff.h new file mode 100644 index 0000000..f5a86bc --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmd5stuff.h @@ -0,0 +1,100 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to MD5 +// (derived from the RSA Data Security, Inc. MD5 Message-Digest Algorithm) +// +// Equivalent code is available from RFC 1321. +// +// Contents: +// MD4 methods and constants +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +#if !defined(_PCP_MD5_STUFF_H) +#define _PCP_MD5_STUFF_H + +/* MD5 constants */ +static const Ipp32u md5_iv[] = { + 0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476}; + +static __ALIGN16 const Ipp32u md5_cnt[] = { + 0xD76AA478, 0xE8C7B756, 0x242070DB, 0xC1BDCEEE, + 0xF57C0FAF, 0x4787C62A, 0xA8304613, 0xFD469501, + 0x698098D8, 0x8B44F7AF, 0xFFFF5BB1, 0x895CD7BE, + 0x6B901122, 0xFD987193, 0xA679438E, 0x49B40821, + + 0xF61E2562, 0xC040B340, 0x265E5A51, 0xE9B6C7AA, + 0xD62F105D, 0x02441453, 0xD8A1E681, 0xE7D3FBC8, + 0x21E1CDE6, 0xC33707D6, 0xF4D50D87, 0x455A14ED, + 0xA9E3E905, 0xFCEFA3F8, 0x676F02D9, 0x8D2A4C8A, + + 0xFFFA3942, 0x8771F681, 0x6D9D6122, 0xFDE5380C, + 0xA4BEEA44, 0x4BDECFA9, 0xF6BB4B60, 0xBEBFBC70, + 0x289B7EC6, 0xEAA127FA, 0xD4EF3085, 0x04881D05, + 0xD9D4D039, 0xE6DB99E5, 0x1FA27CF8, 0xC4AC5665, + + 0xF4292244, 0x432AFF97, 0xAB9423A7, 0xFC93A039, + 0x655B59C3, 0x8F0CCC92, 0xFFEFF47D, 0x85845DD1, + 0x6FA87E4F, 0xFE2CE6E0, 0xA3014314, 0x4E0811A1, + 0xF7537E82, 0xBD3AF235, 0x2AD7D2BB, 0xEB86D391 +}; + +IPP_OWN_DEFN (static void, md5_hashInit, (void* pHash)) +{ + /* setup initial digest */ + ((Ipp32u*)pHash)[0] = md5_iv[0]; + ((Ipp32u*)pHash)[1] = md5_iv[1]; + ((Ipp32u*)pHash)[2] = md5_iv[2]; + ((Ipp32u*)pHash)[3] = md5_iv[3]; +} + +IPP_OWN_DEFN (static void, md5_hashUpdate, (void* pHash, const Ipp8u* pMsg, int msgLen)) +{ + UpdateMD5(pHash, pMsg, msgLen, md5_cnt); +} + +IPP_OWN_DEFN (static void, md5_hashOctString, (Ipp8u* pMD, void* pHashVal)) +{ + /* md5 does not need conversion into big endian */ + ((Ipp32u*)pMD)[0] = ((Ipp32u*)pHashVal)[0]; + ((Ipp32u*)pMD)[1] = ((Ipp32u*)pHashVal)[1]; + ((Ipp32u*)pMD)[2] = ((Ipp32u*)pHashVal)[2]; + ((Ipp32u*)pMD)[3] = ((Ipp32u*)pHashVal)[3]; +} + +IPP_OWN_DEFN (static void, md5_msgRep, (Ipp8u* pDst, Ipp64u lenLo, Ipp64u lenHi)) +{ + IPP_UNREFERENCED_PARAMETER(lenHi); + lenLo <<= 3; + ((Ipp64u*)(pDst))[0] = lenLo; +} + +#define cpFinalizeMD5 OWNAPI(cpFinalizeMD5) + IPP_OWN_DECL (void, cpFinalizeMD5, (DigestMD5 pHash, const Ipp8u* inpBuffer, int inpLen, Ipp64u processedMsgLen)) + +#endif /* #if !defined(_PCP_MD5_STUFF_H) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmd5unpack.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmd5unpack.c new file mode 100644 index 0000000..94b3c08 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmd5unpack.c @@ -0,0 +1,60 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to MD5 +// (derived from the RSA Data Security, Inc. MD5 Message-Digest Algorithm) +// +// Equivalent code is available from RFC 1321. +// +// Contents: +// ippsMD5Unpack() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsMD5Unpack +// +// Purpose: Unpack buffer content into the initialized context. +// +// Returns: Reason: +// ippStsNullPtrErr pBuffer == NULL +// ippStsNoErr no errors +// +// Parameters: +// pBuffer pointer to the input buffer +// pState pointer hash state +// +*F*/ +IPPFUN(IppStatus, ippsMD5Unpack,(const Ipp8u* pBuffer, IppsMD5State* pState)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pState, pBuffer); + + CopyBlock(pBuffer, pState, sizeof(IppsMD5State)); + HASH_SET_ID(pState, idCtxMD5); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmd5update.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmd5update.c new file mode 100644 index 0000000..ea1ab76 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmd5update.c @@ -0,0 +1,116 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to MD5 +// (derived from the RSA Data Security, Inc. MD5 Message-Digest Algorithm) +// +// Equivalent code is available from RFC 1321. +// +// Contents: +// ippsMD5Update() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpmd5stuff.h" + +/*F* +// Name: ippsMD5Update +// +// Purpose: Updates intermadiate digest based on input stream. +// +// Returns: Reason: +// ippStsNullPtrErr pSrc == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxMD5 +// ippStsLengthErr len <0 +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the input stream +// len input stream length +// pState pointer to the MD5 state +// +*F*/ +IPPFUN(IppStatus, ippsMD5Update,(const Ipp8u* pSrc, int len, IppsMD5State* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxMD5), ippStsContextMatchErr); + + /* test input length */ + IPP_BADARG_RET((len<0), ippStsLengthErr); + /* test source pointer */ + IPP_BADARG_RET((len && !pSrc), ippStsNullPtrErr); + + /* + // handle non empty message + */ + if(len) { + int procLen; + + int idx = HASH_BUFFIDX(pState); + Ipp8u* pBuffer = HASH_BUFF(pState); + Ipp64u lenLo = HASH_LENLO(pState) + (Ipp64u)len; + + /* if non empty internal buffer filling */ + if(idx) { + /* copy from input stream to the internal buffer as match as possible */ + procLen = IPP_MIN(len, (MBS_MD5 -idx)); + CopyBlock(pSrc, pBuffer+idx, procLen); + + /* update message pointer and length */ + idx += procLen; + pSrc += procLen; + len -= procLen; + + /* update digest if buffer full */ + if( MBS_MD5 == idx) { + UpdateMD5(HASH_VALUE(pState), pBuffer, MBS_MD5, md5_cnt); + idx = 0; + } + } + + /* main message part processing */ + procLen = len & ~(MBS_MD5-1); + if(procLen) { + UpdateMD5(HASH_VALUE(pState), pSrc, procLen, md5_cnt); + pSrc += procLen; + len -= procLen; + } + + /* store rest of message into the internal buffer */ + if(len) { + CopyBlock(pSrc, pBuffer, len); + idx += len; + } + + /* update length of processed message */ + HASH_LENLO(pState) = lenLo; + HASH_BUFFIDX(pState) = idx; + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmgf1ca_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmgf1ca_rmf.c new file mode 100644 index 0000000..02af6d8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmgf1ca_rmf.c @@ -0,0 +1,99 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Cryptography Primitive. +// HASH based Mask Generation Functions +// +// Contents: +// ippsMGF1_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsMGF1 +// +// Purpose: Mask Generation Functios. +// +// Returns: Reason: +// ippStsNullPtrErr pMask == NULL +// pMethod ==NULL +// ippStsLengthErr seedLen <0 +// maskLen <0 +// ippStsNoErr no errors +// +// Parameters: +// pSeed pointer to the input stream +// seedLen input stream length (bytes) +// pMaske pointer to the ouput mask +// maskLen desired length of mask (bytes) +// pMethod hash method +// +// +// Note. +// MGF1 defined in the IEEE P1363 standard. +// MGF1 defined in the ANSI X9.63 standard and frequently called KDF (key Generation Function). +// The fifference between MGF1 and MGF2 is negligible - counter i runs from 0 (in MGF1) and from 1 (in MGF2) +*F*/ +IPPFUN(IppStatus, ippsMGF1_rmf,(const Ipp8u* pSeed, int seedLen, Ipp8u* pMask, int maskLen, const IppsHashMethod* pMethod)) +{ + IPP_BAD_PTR2_RET(pMask, pMethod); + IPP_BADARG_RET((seedLen<0)||(maskLen<0), ippStsLengthErr); + + { + /* hash specific */ + int hashSize = pMethod->hashLen; + + int i, outLen; + + __ALIGN8 IppsHashState_rmf hashCtx; + ippsHashInit_rmf(&hashCtx, pMethod); + + if(!pSeed) + seedLen = 0; + + for(i=0,outLen=0; outLen>24) & 0xFF); + cnt[1] = (Ipp8u)((i>>16) & 0xFF); + cnt[2] = (Ipp8u)((i>>8) & 0xFF); + cnt[3] = (Ipp8u)(i & 0xFF); + + ippsHashUpdate_rmf(pSeed, seedLen, &hashCtx); + ippsHashUpdate_rmf(cnt, 4, &hashCtx); + + if((outLen + hashSize) <= maskLen) { + ippsHashFinal_rmf(pMask+outLen, &hashCtx); + outLen += hashSize; + } + else { + Ipp8u md[MAX_HASH_SIZE]; + ippsHashFinal_rmf(md, &hashCtx); + CopyBlock(md, pMask+outLen, maskLen-outLen); + outLen = maskLen; + } + } + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmgf2ca_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmgf2ca_rmf.c new file mode 100644 index 0000000..8ee272f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmgf2ca_rmf.c @@ -0,0 +1,100 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Cryptography Primitive. +// HASH based Mask Generation Functions +// +// Contents: +// ippsMGF2_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsMGF2 +// +// Purpose: Mask Generation Functios. +// +// Returns: Reason: +// ippStsNullPtrErr pMask == NULL +// pMethod ==NULL +// ippStsLengthErr seedLen <0 +// maskLen <0 +// ippStsNoErr no errors +// +// Parameters: +// pSeed pointer to the input stream +// seedLen input stream length (bytes) +// pMaske pointer to the ouput mask +// maskLen desired length of mask (bytes) +// pMethod hash method +// +// +// Note. +// MGF1 defined in the IEEE P1363 standard. +// MGF1 defined in the ANSI X9.63 standard and frequently called KDF (key Generation Function). +// The fifference between MGF1 and MGF2 is negligible - counter i runs from 0 (in MGF1) and from 1 (in MGF2) +*F*/ + +IPPFUN(IppStatus, ippsMGF2_rmf,(const Ipp8u* pSeed, int seedLen, Ipp8u* pMask, int maskLen, const IppsHashMethod* pMethod)) +{ + IPP_BAD_PTR2_RET(pMask, pMethod); + IPP_BADARG_RET((seedLen<0)||(maskLen<0), ippStsLengthErr); + + { + /* hash specific */ + int hashSize = pMethod->hashLen; + + int i, outLen; + + __ALIGN8 IppsHashState_rmf hashCtx; + ippsHashInit_rmf(&hashCtx, pMethod); + + if(!pSeed) + seedLen = 0; + + for(i=1,outLen=0; outLen>24) & 0xFF); + cnt[1] = (Ipp8u)((i>>16) & 0xFF); + cnt[2] = (Ipp8u)((i>>8) & 0xFF); + cnt[3] = (Ipp8u)(i & 0xFF); + + ippsHashUpdate_rmf(pSeed, seedLen, &hashCtx); + ippsHashUpdate_rmf(cnt, 4, &hashCtx); + + if((outLen + hashSize) <= maskLen) { + ippsHashFinal_rmf(pMask+outLen, &hashCtx); + outLen += hashSize; + } + else { + Ipp8u md[MAX_HASH_SIZE]; + ippsHashFinal_rmf(md, &hashCtx); + CopyBlock(md, pMask+outLen, maskLen-outLen); + outLen = maskLen; + } + } + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_exp_bufsize.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_exp_bufsize.h new file mode 100644 index 0000000..4b52c2d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_exp_bufsize.h @@ -0,0 +1,51 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// DL over Prime Field +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpdlp.h" +#include "pcptool.h" + +/* +// Size of scratch buffer, involved in MontExp operation +// +// nExponents - number of exponents +// expBitSize - (max) sizeof exponent (in bits) +// modulusBitSize - size of modulus (bits) +*/ +static cpSize cpMontExpScratchBufferSize(cpSize modulusBitSize, cpSize expBitSize, cpSize nExponents) +{ + /* sizeof table element */ + cpSize elmDataSize = BITS_BNU_CHUNK(modulusBitSize) * (Ipp32s)sizeof(BNU_CHUNK_T); + /* get window_size */ + cpSize w = (nExponents == 1) ? cpMontExp_WinSize(expBitSize) : /* use optimal window size, if single-scalar operation */ + nExponents; /* or pseudo-oprimal if multi-scalar operation */ + + /* number of table entries */ + cpSize nPrecomputed = 1 << w; + + cpSize bufferSize = elmDataSize*nPrecomputed + (CACHE_LINE_SIZE - 1); + return bufferSize; +} \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_expbinbnu.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_expbinbnu.c new file mode 100644 index 0000000..35038bd --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_expbinbnu.c @@ -0,0 +1,117 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Modular Exponentiation (binary version) +// +// Contents: +// cpMontExpBin_BNU() +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpmontgomery.h" + +/*F* +// Name: cpMontExpBin_BNU +// +// Purpose: computes the Montgomery exponentiation with exponent +// BNU_CHUNK_T *dataE to the given big number integer of Montgomery form +// BNU_CHUNK_T *dataX with respect to the modulus gsModEngine *pModEngine. +// +// Returns: +// Length of modulus +// +// +// Parameters: +// dataX big number integer of Montgomery form within the +// range [0,m-1] +// dataE big number exponent +// pModEngine Montgomery modulus of IppsMontState. +/ dataY the Montgomery exponentation result. +// +// Notes: IppsBigNumState *r should possess enough memory space as to hold the result +// of the operation. i.e. both pointers r->d and r->buffer should possess +// no less than (m->n->length) number of 32-bit words. +*F*/ + +IPP_OWN_DEFN (cpSize, cpMontExpBin_BNU, (BNU_CHUNK_T* dataY, const BNU_CHUNK_T* dataX, cpSize nsX, const BNU_CHUNK_T* dataE, cpSize nsE, gsModEngine* pModEngine)) +{ + cpSize nsM = MOD_LEN( pModEngine ); + + /* + // test for special cases: + // x^0 = 1 + // 0^e = 0 + */ + if( cpEqu_BNU_CHUNK(dataE, nsE, 0) ) { + COPY_BNU(dataY, MOD_MNT_R( pModEngine ), nsM); + } + else if( cpEqu_BNU_CHUNK(dataX, nsX, 0) ) { + ZEXPAND_BNU(dataY, 0, nsM); + } + + /* general case */ + else { + /* Montgomery engine buffers */ + const int usedPoolLen = 1; + BNU_CHUNK_T* dataT = gsModPoolAlloc(pModEngine, usedPoolLen); + if(NULL == dataT) + return -1; + + { + /* execute most significant part pE */ + BNU_CHUNK_T eValue = dataE[nsE-1]; + int n = cpNLZ_BNU(eValue)+1; + + /* expand base and init result */ + ZEXPAND_COPY_BNU(dataT, nsM, dataX, nsX); + COPY_BNU(dataY, dataT, nsM); + + eValue <<= n; + for(; nsqr(dataY, dataY, pModEngine); + + /* and multiply R = R*X mod Modulus */ + if(eValue & ((BNU_CHUNK_T)1<<(BNU_CHUNK_BITS-1))) + MOD_METHOD( pModEngine )->mul(dataY, dataY, dataT, pModEngine); + } + + /* execute rest bits of E */ + for(--nsE; nsE>0; nsE--) { + eValue = dataE[nsE-1]; + + for(n=0; nsqr(dataY, dataY, pModEngine); + + if(eValue & ((BNU_CHUNK_T)1<<(BNU_CHUNK_BITS-1))) + MOD_METHOD( pModEngine )->mul(dataY, dataY, dataT, pModEngine); + } + } + } + + gsModPoolFree(pModEngine, usedPoolLen); + } + + return nsM; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_expbinbnu_sscm.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_expbinbnu_sscm.c new file mode 100644 index 0000000..15da4fb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_expbinbnu_sscm.c @@ -0,0 +1,122 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Modular Exponentiation (binary version) +// +// Contents: +// cpMontExpBin_BNU_sscm() +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpmontgomery.h" +#include "pcpmask_ct.h" + +#if defined(_USE_IPP_OWN_CBA_MITIGATION_) +/* +// The reason was to mitigate "cache monitoring" attack on RSA +// +// This is improved version of modular exponentiation. +// Current version provide both either mitigation and perrformance. +// This version in comparison with previous (Intel(R) Integrated Performance Primitives (Intel(R) IPP) 4.1.3) one ~30-40% faster, +// i.e the the performance stayed as was for pre-mitigated version +// +*/ + +/*F* +// Name: cpMontExpBin_BNU_sscm +// +// Purpose: computes the Montgomery exponentiation with exponent +// BNU_CHUNK_T *dataE to the given big number integer of Montgomery form +// BNU_CHUNK_T *dataX with respect to the modulus gsModEngine *pModEngine. +// +// Returns: +// Length of modulus +// +// +// Parameters: +// dataX big number integer of Montgomery form within the +// range [0,m-1] +// dataE big number exponent +// pMont Montgomery modulus of IppsMontState. +/ dataY the Montgomery exponentation result. +// +*F*/ +IPP_OWN_DEFN (cpSize, cpMontExpBin_BNU_sscm, (BNU_CHUNK_T* dataY, const BNU_CHUNK_T* dataX, cpSize nsX, const BNU_CHUNK_T* dataE, cpSize nsE, gsModEngine* pMont)) +{ + cpSize nsM = MOD_LEN(pMont); + + /* + // test for special cases: + // x^0 = 1 + // 0^e = 0 + */ + if( cpIsGFpElemEquChunk_ct(dataE, nsE, 0) ) { + COPY_BNU(dataY, MOD_MNT_R(pMont), nsM); + } + else if( cpIsGFpElemEquChunk_ct(dataX, nsX, 0) ) { + ZEXPAND_BNU(dataY, 0, nsM); + } + + /* general case */ + else { + /* Montgomery engine buffers */ + const int usedPoolLen = 2; + BNU_CHUNK_T* dataT = gsModPoolAlloc(pMont, usedPoolLen); + if(NULL == dataT) + return -1; + + BNU_CHUNK_T* sscmB = dataT + nsM; + + /* mont(1) */ + BNU_CHUNK_T* pR = MOD_MNT_R(pMont); + + /* copy base */ + ZEXPAND_COPY_BNU(dataT, nsM, dataX, nsX); + /* init result, Y=1 */ + COPY_BNU(dataY, pR, nsM); + + /* execute bits of E */ + for(; nsE>0; nsE--) { + BNU_CHUNK_T eValue = dataE[nsE-1]; + + int n; + for(n=BNU_CHUNK_BITS; n>0; n--) { + /* sscmB = ( msb(eValue) )? X : mont(1) */ + BNU_CHUNK_T mask = cpIsMsb_ct(eValue); + eValue <<= 1; + cpMaskedCopyBNU_ct(sscmB, mask, dataT, pR, nsM); + + /* squaring Y = Y^2 */ + MOD_METHOD(pMont)->sqr(dataY, dataY, pMont); + /* and multiplication: Y = Y * sscmB */ + MOD_METHOD(pMont)->mul(dataY, dataY, sscmB, pMont); + } + } + + gsModPoolFree(pMont, usedPoolLen); + } + + return nsM; +} + +#endif /* _USE_IPP_OWN_CBA_MITIGATION_ */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_expsafebinary.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_expsafebinary.c new file mode 100644 index 0000000..e8f9994 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_expsafebinary.c @@ -0,0 +1,341 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Modular Exponentiation (binary version) +// +// Contents: +// cpSafeMontExp_Binary() +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpmontgomery.h" + +//tbcd: temporary excluded: #include + +/*F* +// Name: cpSafeMontExp_Binary +// +// Purpose: Binary method of Exponentiation +// +// +// Parameters: +// pX big number integer of Montgomery form within the +// range [0,m-1] +// pE big number exponent +// pMont Montgomery modulus of IppsMontState. +/ pY the Montgomery exponentation result. +// +*F*/ + +#if !defined(_USE_VERSION1_CBA_MITIGATION_) && !defined(_USE_IPP_OWN_CBA_MITIGATION_) // unsafe version +void cpSafeMontExp_Binary(IppsBigNumState* pY, + const IppsBigNumState* pX, const IppsBigNumState* pE, + IppsMontState* pMont) +{ + int k; + + /* if E==0 then Y=R mod m */ + if (pE->size == 1 && pE->number[0] == 0) { + int len = IPP_MULTIPLE_OF(pMont->n->size, BNUBASE_TYPE_SIZE); + cpMemset32u(pMont->wb->number, 0, len); + pMont->wb->number[len] = 1; + + cpMod_BNU(pMont->wb->number, len + 1, pMont->n->number, pMont->n->size, &pY->size); + + cpMemcpy32u(pY->number, pMont->wb->number, pY->size); + pY->sgn = ippBigNumPOS; + return; + } + + else { + Ipp32u* r_number = pY->workBuffer; + int r_size = pY->size; + + int flag=1; + Ipp32u power = pE->number[pE->size-1]; + + for( k = 31; k >= 0; k-- ) { + Ipp32u powd = power & 0x80000000;/* from top to bottom*/ + power <<= 1; + + if((flag == 1) && (powd == 0)) + continue; + + else if (flag == 0) { + #if defined(_USE_NN_MONTMUL_) + cpMontMul(r_number, r_size, r_number,r_size, + pMont->n->number, pMont->n->size, + r_number,&r_size, pMont->n0, pMont->wb->number); + #else + cpMontMul(r_number, r_size, r_number,r_size, + pMont->n->number, pMont->n->size, + r_number,&r_size, pMont->n0, pMont->wb->number, pMont->pBuffer); + #endif + if (powd) + #if defined(_USE_NN_MONTMUL_) + cpMontMul(r_number, r_size, pX->number,pX->size, + pMont->n->number, pMont->n->size, + r_number,&r_size, pMont->n0, pMont->wb->number); + #else + cpMontMul(r_number, r_size, pX->number,pX->size, + pMont->n->number, pMont->n->size, + r_number,&r_size, pMont->n0, pMont->wb->number, pMont->pBuffer); + #endif + } + + else { + int i; + flag = 0; + r_size = pMont->n->size; + if( pX->size < pMont->n->size ) + for(i = r_size - 1; i >= pX->size; i-- ) + r_number[i] = 0; + + for( i = pX->size - 1; i >= 0; i-- ) + r_number[i] = pX->number[i]; + } + } + + if (pE->size > 1) { + struct BNU { + Ipp32u *number; + int *size; + } BNUs[2]; + BNUs[0].number = r_number; + BNUs[0].size = &r_size; + BNUs[1].number = pX->number; + BNUs[1].size = &(((IppsBigNumState*)pX)->size); + + for( k = pE->size - 2; k >= 0; k-- ) { + int j; + Ipp32u powd = 0; + power = pE->number[k]; + + for( j = 31; j >= 0; j-- ) { + #if defined(_USE_NN_MONTMUL_) + cpMontMul(r_number, r_size, BNUs[powd].number, *(BNUs[powd].size), + pMont->n->number, pMont->n->size, + r_number,&r_size, pMont->n0, pMont->wb->number); + #else + cpMontMul(r_number, r_size, BNUs[powd].number, *(BNUs[powd].size), + pMont->n->number, pMont->n->size, + r_number,&r_size, pMont->n0, pMont->wb->number, pMont->pBuffer); + #endif + powd = ((power >> j) & 0x1) & (powd ^ 1); + j += powd; + } + } + } + + for(k=r_size-1; k>= 0; k--) + pY->number[k] = r_number[k]; + + pY->sgn = ippBigNumPOS; + pY->size = r_size; + + while((pY->size > 1) && (pY->number[pY->size-1] == 0)) + pY->size--; + + return; + } +} +#endif /* _USE_VERSION1_CBA_MITIGATION_, _xUSE_IPP_OWN_CBA_MITIGATION_ */ + + +#if defined(_USE_VERSION1_CBA_MITIGATION_) +/* +// The version below was designed according to recommendation +// from Crypto experts. +// The reason was to mitigate "cache monitoring" attack on RSA +// Note: this version slower than pre-mitigated version ~ 30-40% +*/ + + +#define SET_BNU(dst,val,len) \ +{ \ + int n; \ + for(n=0; n<(len); n++) (dst)[n] = (val); \ +} + +#define AND_BNU(dst,src1,src2,len) \ +{ \ + int n; \ + for(n=0; n<(len); n++) (dst)[n] = (src1)[n] & (src2)[n]; \ +} + +/*F* +// Name: cpSafeMontExp_Binary +// +// Purpose: Binary method of Exponentiation +// +// +// Parameters: +// pX big number integer of Montgomery form within the +// range [0,m-1] +// pE big number exponent +// pMont Montgomery modulus of IppsMontState. +/ pY the Montgomery exponentation result. +// +*F*/ + +void cpSafeMontExp_Binary(IppsBigNumState* pY, + const IppsBigNumState* pX, const IppsBigNumState* pE, + IppsMontState* pMont) +{ + Ipp32u* eData = BN_NUMBER(pE); + int eSize = BN_SIZE(pE); + + /* + // if e==0 then r=R mod m (i.e MontEnc(1)) + */ + if (eSize == 1 && eData[0] == 0) { + cpBN_copy(MNT_1(pMont), pY); + return; + } + + /* + // modulo exponentiation + */ + if(pY!=pX) /* init result */ + cpBN_copy(pX, pY); + + { + Ipp32u eValue; + int nBits; + + Ipp32u* pModulus = BN_NUMBER(MNT_MODULO(pMont)); + int mSize = BN_SIZE(MNT_MODULO(pMont)); + Ipp32u* pHelper = MNT_HELPER(pMont); + + Ipp32u* yData = BN_NUMBER(pY); + Ipp32u* xData = BN_BUFFER(pY); + int ySize = BN_SIZE(pY); + + Ipp32u* tData = BN_NUMBER(MNT_PRODUCT(pMont)); + Ipp32u* pBuffer = BN_BUFFER(MNT_PRODUCT(pMont)); + Ipp32u* pMontOne= BN_NUMBER(MNT_1(pMont)); + + /* expand Mont(1) */ + ZEXPAND_BNU(pMontOne, BN_SIZE(MNT_1(pMont)), mSize); + + /* copy base */ + ZEXPAND_COPY_BNU(yData,ySize, xData,mSize); + + + /* execute most significant part pE */ + eValue = eData[eSize-1]; + nBits = 32-NLZ32u(eValue); + eValue <<= (32-nBits); + + nBits--; + eValue <<=1; + for(; nBits>0; nBits--, eValue<<=1) { + Ipp32u carry; + + /* squaring: R^2 mod Modulus */ + #if defined(_USE_NN_MONTMUL_) + cpMontMul(yData, ySize, + yData, ySize, + pModulus, mSize, + yData, &ySize, + pHelper, pBuffer); + #else + cpMontMul(yData, ySize, + yData, ySize, + pModulus, mSize, + yData, &ySize, + pHelper, pBuffer, MNT_BUFFER(pMont)); + #endif + + /* T = (X-1)*bitof(E,j) + 1 */ + SET_BNU(pBuffer, ((Ipp32s)eValue)>>31, mSize); + carry = cpSub_BNU(tData, xData, pMontOne, mSize); + AND_BNU(tData, tData, pBuffer, mSize); + carry = cpAdd_BNU(tData, tData, pMontOne, mSize); + + /* multiply: Y*T mod Modulus */ + #if defined(_USE_NN_MONTMUL_) + cpMontMul(yData, ySize, + tData, mSize, + pModulus, mSize, + yData, &ySize, + pHelper, pBuffer); + #else + cpMontMul(yData, ySize, + tData, mSize, + pModulus, mSize, + yData, &ySize, + pHelper, pBuffer, MNT_BUFFER(pMont)); + #endif + } + + /* execute rest bits of E */ + eSize--; + for(; eSize>0; eSize--) { + eValue = eData[eSize-1]; + + for(nBits=32; nBits>0; nBits--, eValue<<=1) { + Ipp32u carry; + + /* squaring: R^2 mod Modulus */ + #if defined(_USE_NN_MONTMUL_) + cpMontMul(yData, ySize, + yData, ySize, + pModulus, mSize, + yData, &ySize, + pHelper, pBuffer); + #else + cpMontMul(yData, ySize, + yData, ySize, + pModulus, mSize, + yData, &ySize, + pHelper, pBuffer, MNT_BUFFER(pMont)); + #endif + + /* T = (X-1)*bitof(E,j) + 1 */ + SET_BNU(pBuffer, ((Ipp32s)eValue)>>31, mSize); + carry = cpSub_BNU(tData, xData, pMontOne, mSize); + AND_BNU(tData, tData, pBuffer, mSize); + carry = cpAdd_BNU(tData, tData, pMontOne, mSize); + + /* multiply: R*T mod Modulus */ + #if defined(_USE_NN_MONTMUL_) + cpMontMul(yData, ySize, + tData, mSize, + pModulus, mSize, + yData, &ySize, + pHelper, pBuffer); + #else + cpMontMul(yData, ySize, + tData, mSize, + pModulus, mSize, + yData, &ySize, + pHelper, pBuffer, MNT_BUFFER(pMont)); + #endif + } + } + + BN_SIZE(pY) = ySize; + } +} +#endif /* _USE_VERSION1_CBA_MITIGATION_ */ \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_expwinbnu.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_expwinbnu.c new file mode 100644 index 0000000..179e2e9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_expwinbnu.c @@ -0,0 +1,161 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Contents: +// cpMontExpWin_BN() +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpmontgomery.h" +#include "gsscramble.h" + +#if defined(_USE_WINDOW_EXP_) + +IPP_OWN_DEFN (static void, gsMul_school, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pX, const BNU_CHUNK_T* pY, cpSize len, BNU_CHUNK_T* pKbuffer)) +{ + IPP_UNREFERENCED_PARAMETER(pKbuffer); + cpMul_BNU_school(pR, pX, len, pY, len); +} +IPP_OWN_DEFN (static void, gsSqr_school, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pX, cpSize len, BNU_CHUNK_T* pKbuffer)) +{ + IPP_UNREFERENCED_PARAMETER(pKbuffer); + cpSqr_BNU_school(pR, pX, len); +} + +IPP_OWN_FUNPTR (void, gsMul, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pX, const BNU_CHUNK_T* pY, cpSize len, BNU_CHUNK_T* pKbuffer)) +IPP_OWN_FUNPTR (void, gsSqr, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pX, cpSize len, BNU_CHUNK_T* pKbuffer)) + +/*F* +// Name: cpMontExpWin_BN +// +// Purpose: Binary method of Exponentiation +// +// +// Parameters: +// pX big number integer of Montgomery form within the +// range [0,m-1] +// pE big number exponent +// pMont Montgomery modulus of IppsMontState. +/ pY the Montgomery exponentation result. +// pResource pointer to resource +// +*F*/ + +IPP_OWN_DEFN (void, cpMontExpWin_BN, (IppsBigNumState* pY, const IppsBigNumState* pX, const IppsBigNumState* pE, gsModEngine* pMont, BNU_CHUNK_T* pResource)) +{ + BNU_CHUNK_T* dataX = BN_NUMBER(pX); + cpSize nsX = BN_SIZE(pX); + + BNU_CHUNK_T* dataE = BN_NUMBER(pE); + cpSize nsE = BN_SIZE(pE); + + BNU_CHUNK_T* dataY = BN_NUMBER(pY); + cpSize nsM = MOD_LEN(pMont); + + /* + // test for special cases: + // x^0 = 1 + // 0^e = 0 + */ + if( cpEqu_BNU_CHUNK(dataE, nsE, 0) ) { + COPY_BNU(dataY, MOD_MNT_R(pMont), nsM); + } + else if( cpEqu_BNU_CHUNK(dataX, nsX, 0) ) { + ZEXPAND_BNU(dataY, 0, nsM); + } + + /* general case */ + else { + /* Montgomery engine buffers */ + const int usedPoolLen = 2; + BNU_CHUNK_T* pBuffer = gsModPoolAlloc(pMont, usedPoolLen); + if(NULL == pBuffer) + return; + + BNU_CHUNK_T* pKBuffer = pBuffer + nsM; + + /* mul & sqr functions */ + gsMul mulFun = gsMul_school; + gsSqr sqrFun = gsSqr_school; + + /* fixed window param */ + cpSize bitsizeE = BITSIZE_BNU(dataE, nsE); + cpSize window = cpMontExp_WinSize(bitsizeE); + BNU_CHUNK_T mask = (1<>shift) &mask; + + /* initialize result */ + COPY_BNU(dataY, pResource+windowVal*(BNU_CHUNK_T)nsM, nsM); + + for(eBit-=window; eBit>=0; eBit-=window) { + /* do square window times */ + for(n=0,windowVal=0; n>shift) &mask; + + if(windowVal) { + /* extract precomputed value and muptiply */ + mulFun(pBuffer, dataY, pResource+windowVal*(BNU_CHUNK_T)nsM, nsM, pKBuffer); + cpMontRed_BNU(dataY, pBuffer, pMont); + } + } + } + gsModPoolFree(pMont, usedPoolLen); + } + + FIX_BNU(dataY, nsM); + BN_SIZE(pY) = nsM; + BN_SIGN(pY) = ippBigNumPOS; +} + +#endif /* _USE_WINDOW_EXP_ */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_expwinbnu_sscm.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_expwinbnu_sscm.c new file mode 100644 index 0000000..ec5be55 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_expwinbnu_sscm.c @@ -0,0 +1,171 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Contents: +// cpMontExpWin_BN_sscm() +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpmontgomery.h" +#include "gsscramble.h" + +#if defined(_USE_WINDOW_EXP_) + +IPP_OWN_DEFN (static void, gsMul_school, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pX, const BNU_CHUNK_T* pY, cpSize len, BNU_CHUNK_T* pKbuffer)) +{ + IPP_UNREFERENCED_PARAMETER(pKbuffer); + cpMul_BNU_school(pR, pX, len, pY, len); +} +IPP_OWN_DEFN (static void, gsSqr_school, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pX, cpSize len, BNU_CHUNK_T* pKbuffer)) +{ + IPP_UNREFERENCED_PARAMETER(pKbuffer); + cpSqr_BNU_school(pR, pX, len); +} + +IPP_OWN_FUNPTR (void, gsMul, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pX, const BNU_CHUNK_T* pY, cpSize len, BNU_CHUNK_T* pKbuffer)) +IPP_OWN_FUNPTR (void, gsSqr, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pX, cpSize len, BNU_CHUNK_T* pKbuffer)) + +/*F* +// Name: cpMontExpWin_BN_sscm +// +// Purpose: Binary method of Exponentiation +// +// +// Parameters: +// pX big number integer of Montgomery form within the +// range [0,m-1] +// pE big number exponent +// pMont Montgomery modulus of IppsMontState. +/ pY the Montgomery exponentation result. +// +*F*/ + +IPP_OWN_DEFN (void, cpMontExpWin_BN_sscm, (IppsBigNumState* pY, const IppsBigNumState* pX, const IppsBigNumState* pE, gsModEngine* pMont, BNU_CHUNK_T* pResource)) +{ + BNU_CHUNK_T* dataX = BN_NUMBER(pX); + cpSize nsX = BN_SIZE(pX); + + BNU_CHUNK_T* dataE = BN_NUMBER(pE); + cpSize nsE = BN_SIZE(pE); + + BNU_CHUNK_T* dataY = BN_NUMBER(pY); + cpSize nsM = MOD_LEN(pMont); + + /* + // test for special cases: + // x^0 = 1 + // 0^e = 0 + */ + if( cpEqu_BNU_CHUNK(dataE, nsE, 0) ) { + COPY_BNU(dataY, MOD_MNT_R(pMont), nsM); + } + else if( cpEqu_BNU_CHUNK(dataX, nsX, 0) ) { + ZEXPAND_BNU(dataY, 0, nsM); + } + + /* general case */ + else { + /* Montgomery engine buffers */ + const int usedPoolLen = 2; + BNU_CHUNK_T* pBuffer = gsModPoolAlloc(pMont, usedPoolLen); + if(NULL == pBuffer) + return; + + BNU_CHUNK_T* pKBuffer = pBuffer + nsM; + + /* mul & sqr functions */ + gsMul mulFun = gsMul_school; + gsSqr sqrFun = gsSqr_school; + + /* fixed window param */ + cpSize bitsizeE = BITSIZE_BNU(dataE, nsE); + cpSize window = cpMontExp_WinSize(bitsizeE); + BNU_CHUNK_T mask = (1<>shift) &mask); + + /* initialize result */ + //cpScrambleGet((Ipp32u*)dataY, nsM*sizeof(BNU_CHUNK_T)/sizeof(Ipp32u), ((Ipp8u*)pResource)+windowVal*chunkSize, chunkSize); + gsScrambleGet_sscm(dataY, nsM, pResource, windowVal, window); + + for(eBit-=window; eBit>=0; eBit-=window) { + /* do squaring window-times */ + for(n=0,windowVal=0; n>shift) &mask); + + /* exptact precomputed value and muptiply */ + //cpScrambleGet((Ipp32u*)dataT, nsM*sizeof(BNU_CHUNK_T)/sizeof(Ipp32u), ((Ipp8u*)pResource)+windowVal*chunkSize, chunkSize); + gsScrambleGet_sscm(dataT, nsM, pResource, windowVal, window); + + mulFun(pBuffer, dataY, dataT, nsM, pKBuffer); + cpMontRed_BNU(dataY, pBuffer, pMont); + } + } + + gsModPoolFree(pMont, usedPoolLen); + } + + FIX_BNU(dataY, nsM); + BN_SIZE(pY) = nsM; + BN_SIGN(pY) = ippBigNumPOS; +} + +#endif /* _USE_WINDOW_EXP_ */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_getsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_getsize.c new file mode 100644 index 0000000..30931fa --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_getsize.c @@ -0,0 +1,61 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// cpMontGetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpmontgomery.h" + +#include "pcptool.h" + +/*F* +// Name: cpMontGetSize +// +// Purpose: Specifies size of buffer in bytes. +// +// Returns: Reason: +// ippStsNoErr no errors +// +// Parameters: +// poolLength length of pool +// maxLen32 max modulus length (in Ipp32u chunks) +// pCtxSize pointer to size of context +// +*F*/ + +IPP_OWN_DEFN (IppStatus, cpMontGetSize, (cpSize maxLen32, int poolLength, cpSize* pCtxSize)) +{ + { + int size = 0; + int maxBitSize = maxLen32 << 5; + gsModEngineGetSize(maxBitSize, poolLength, &size); + + *pCtxSize = (Ipp32s)sizeof(IppsMontState) + + (cpSize)size; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_init.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_init.c new file mode 100644 index 0000000..f59b32c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_init.c @@ -0,0 +1,63 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// cpMontInit() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpmontgomery.h" +#include "pcptool.h" + +/*F* +// Name: cpMontInit +// +// Purpose: Initializes the symbolic data structure and partitions the +// specified buffer space. +// +// Returns: Reason: +// ippStsNoErr no errors +// +// Parameters: +// poolLength length of pool +// maxLen32 max modulus length (in Ipp32u chunks) +// pMont pointer to Montgomery context +// +*F*/ + +IPP_OWN_DEFN (IppStatus, cpMontInit, (int maxLen32, int poolLength, IppsMontState* pMont)) +{ + { + int maxBitSize = ((maxLen32) << 5); + + MNT_ROOM( pMont ) = INTERNAL_BNU_LENGTH(maxLen32); + MNT_ENGINE ( pMont ) = (gsModEngine*)((Ipp8u*)pMont + sizeof(IppsMontState)); + + MNT_SET_ID(pMont); + + gsModEngineInit(MNT_ENGINE(pMont), NULL, maxBitSize, poolLength, gsModArithMont()); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_multiexp_fast.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_multiexp_fast.c new file mode 100644 index 0000000..aa527b9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_multiexp_fast.c @@ -0,0 +1,85 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// cpFastMontMultiExp() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpmontgomery.h" + + +static cpSize GetIndex(const Ipp8u** ppE, cpSize numItems, cpSize nBit) +{ + cpSize shift = nBit%BYTESIZE; + cpSize offset= nBit/BYTESIZE; + cpSize index = 0; + + cpSize n; + for(n=numItems; n>0; n--) { + const Ipp8u* pE = ppE[n-1] + offset; + Ipp8u e = pE[0]; + index <<= 1; + index += (e>>shift) &1; + } + return index; +} + +/* +// Computes multi-exponentiation +// y = x[0]^e[0] * x[1]^e[1] *...* x[numItems-1]^e[numItems-1] mod M +// +// Input: +// - table pTbl of precomuted values pTbl[i] = x[0]^i[0] * x[1]^i[1] *...* x[numItems-1]^i[numItems-1] mod M, +// where i[0], i[1], ..., i[numItems-1] are bits of i value; +// each entry has sizeM length (i.e. equial to modulo M size) +// - array of pointers to the BNU exponents e[0], e[1],...,e[numItems-1] +// - pointer to the Montgomery engine +*/ +IPP_OWN_DEFN (void, cpFastMontMultiExp, (BNU_CHUNK_T* pY, const BNU_CHUNK_T* pPrecomTbl, const Ipp8u** ppE, cpSize eItemBitSize, cpSize numItems, gsModEngine* pModEngine)) +{ + cpSize sizeM = MOD_LEN(pModEngine); + + /* find 1-st non zero index */ + cpSize eBitNumber; + cpSize tblIdx; + for(eBitNumber=eItemBitSize-1, tblIdx=0; !tblIdx && eBitNumber>=0; eBitNumber--) + tblIdx =GetIndex(ppE, numItems, eBitNumber); + + COPY_BNU(pY, pPrecomTbl+tblIdx*sizeM, sizeM); + + for(; eBitNumber>=0; eBitNumber--) { + cpMontMul_BNU(pY, + pY, + pY, + pModEngine); + + tblIdx = GetIndex(ppE, numItems, eBitNumber); + + if(tblIdx) + cpMontMul_BNU(pY, + pY, + pPrecomTbl+tblIdx*sizeM, + pModEngine); + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_multiexpinitarray.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_multiexpinitarray.c new file mode 100644 index 0000000..02ed6ba --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_multiexpinitarray.c @@ -0,0 +1,87 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// cpMontMultiExpInitArray() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpmontgomery.h" + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// +*/ +/* +// Initialize multi-exponentiation computation +// y = x[0]^e[0] * x[1]^e[1] *...* x[numItems-1]^e[numItems-1] mod M +// +// Output: +// - table pTbl of precomuted values pTbl[i] = x[0]^i[0] * x[1]^i[1] *...* x[numItems-1]^i[numItems-1] mod M, +// where i[0], i[1], ..., i[numItems-1] are bits of i value; +// +// Input: +// - array of pointers to the BNU bases x[0], x[1],...,x[numItems-1] +// - pointer to the Montgomery engine +*/ +IPP_OWN_DEFN (void, cpMontMultiExpInitArray, (BNU_CHUNK_T* pPrecomTbl, const BNU_CHUNK_T** ppX, cpSize xItemBitSize, cpSize numItems, gsModEngine* pModEngine)) +{ + cpSize sizeM = MOD_LEN(pModEngine); + + cpSize i, base; + cpSize sizeX = BITS_BNU_CHUNK(xItemBitSize); + + /* buff[0] = mont(1) */ + COPY_BNU(pPrecomTbl, MOD_MNT_R(pModEngine), sizeM); + /* buff[1] = X[0] */ + ZEXPAND_COPY_BNU(pPrecomTbl+sizeM, sizeM, ppX[0], sizeX); + + for(i=1,base=2*sizeM; i=0; k--) { + const BNU_CHUNK_T* pXterm = ppX[k]; + + BNU_CHUNK_T* pBufferBase = pPrecomTbl+base; + int n; + for(n=1; n<=nPasses; n++, pBufferBase+=2*step) { + cpMontMul_BNU_EX(pBufferBase+step, + pBufferBase, sizeM, + pXterm, sizeX, + pModEngine); + } + + nPasses *= 2; + step /= 2; + } + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_packctx.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_packctx.c new file mode 100644 index 0000000..e1f8015 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_packctx.c @@ -0,0 +1,52 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// cpPackMontCtx() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpmontgomery.h" +#include "pcptool.h" + +/*F* +// Name: cpPackMontCtx +// +// Purpose: Serialize mont context +// +// Parameters: +// pCtx context +// pBuffer buffer +*F*/ + +IPP_OWN_DEFN (void, cpPackMontCtx, (const IppsMontState* pCtx, Ipp8u* pBuffer)) +{ + /* size of context (bytes) */ + int ctxSize = sizeof(IppsMontState); + CopyBlock(pCtx, pBuffer, ctxSize); + + pBuffer = (Ipp8u*)pBuffer + sizeof(IppsMontState); + + gsPackModEngineCtx(MNT_ENGINE(pCtx), pBuffer); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_set.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_set.c new file mode 100644 index 0000000..90f3627 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_set.c @@ -0,0 +1,76 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// cpMontSet() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpmontgomery.h" +#include "pcptool.h" + + +/* Auxilirary function */ +__INLINE int cpGetBitSize(Ipp32u offset, Ipp32u val) +{ + int bitSize = 31; + if (val == 0) return 0; + while ((val & (1 << bitSize)) == 0) bitSize--; + return (int)offset + bitSize; +} + +/*F* +// Name: cpMontSet +// +// Purpose: Setup modulus value +// +// Returns: Reason: +// ippStsBadModulusErr (pModulus[0] & 1) == 0 +// ippStsOutOfRangeErr ((Ipp32u)MNT_ROOM(pMont) < INTERNAL_BNU_LENGTH(len32)) +// ippStsLengthErr len32<1 +// ippStsNoErr no errors +// +// Parameters: +// pModulus pointer to the modulus buffer +// len32 length of the modulus (in Ipp32u chunks). +// pMont pointer to the context +*F*/ + +IPP_OWN_DEFN (IppStatus, cpMontSet, (const Ipp32u* pModulus, cpSize len32, IppsMontState* pMont)) +{ + IPP_BADARG_RET(len32<1, ippStsLengthErr); + + /* modulus is not an odd number */ + IPP_BADARG_RET((pModulus[0] & 1) == 0, ippStsBadModulusErr); + IPP_BADARG_RET(MNT_ROOM(pMont)<(int)(INTERNAL_BNU_LENGTH(len32)), ippStsOutOfRangeErr); + + { + const int poolLen = MOD_MAXPOOL(MNT_ENGINE(pMont)); + int modulusBitSize = cpGetBitSize((Ipp32u)((len32 - 1) << 5), pModulus[len32-1]); + + gsModEngineInit(MNT_ENGINE(pMont), pModulus, modulusBitSize, poolLen, gsModArithMont()); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_unpackctx.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_unpackctx.c new file mode 100644 index 0000000..f05bd83 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmont_unpackctx.c @@ -0,0 +1,52 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// cpUnpackMontCtx() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpmontgomery.h" +#include "pcptool.h" + +/*F* +// Name: cpUnpackMontCtx +// +// Purpose: Deserialize mont context +// +// Parameters: +// pCtx context +// pBuffer buffer +*F*/ + +IPP_OWN_DEFN (void, cpUnpackMontCtx, (const Ipp8u* pBuffer, IppsMontState* pCtx)) +{ + /* size of context (bytes) */ + int ctxSize = sizeof(IppsMontState); + CopyBlock(pBuffer, pCtx, ctxSize); + + pBuffer = (Ipp8u*)pBuffer + sizeof(IppsMontState); + + gsUnpackModEngineCtx(pBuffer, MNT_ENGINE(pCtx)); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontexp.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontexp.c new file mode 100644 index 0000000..ef9bc65 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontexp.c @@ -0,0 +1,82 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// ippsMontExp() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpmontgomery.h" +#include "pcptool.h" + +/*F* +// Name: ippsMontExp +// +// Purpose: computes the Montgomery exponentiation with exponent +// IppsBigNumState *pE to the given big number integer of Montgomery form +// IppsBigNumState *pA with respect to the modulus IppsMontState *pCtx. +// +// Returns: Reason: +// ippStsNoErr Returns no error. +// ippStsNullPtrErr Returns an error when pointers are null. +// ippStsBadArgErr Returns an error when a or b is a negative integer. +// ippStsScaleRangeErr Returns an error when a or b is more than m. +// ippStsOutOfRangeErr Returns an error when IppsBigNumState *r is larger than +// IppsMontState *m. +// ippStsContextMatchErr Returns an error when the context parameter does +// not match the operation. +// +// +// Parameters: +// pA big number integer of Montgomery form within the +// range [0,m-1] +// pE big number exponent +// pCtx Montgomery modulus of IppsMontState +/ pR the Montgomery exponentation result. +// +// Notes: IppsBigNumState *r should possess enough memory space as to hold the result +// of the operation. i.e. both pointers r->d and r->buffer should possess +// no less than (m->n->length) number of 32-bit words. +*F*/ + +IPPFUN(IppStatus, ippsMontExp, (const IppsBigNumState* pA, const IppsBigNumState* pE, IppsMontState* pCtx, IppsBigNumState* pR)) +{ + IPP_BAD_PTR4_RET(pA, pE, pCtx, pR); + + IPP_BADARG_RET(!MNT_VALID_ID(pCtx), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pA), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pE), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pR), ippStsContextMatchErr); + + IPP_BADARG_RET(BN_ROOM(pR) < MOD_LEN( MNT_ENGINE(pCtx) ), ippStsOutOfRangeErr); + /* check a */ + IPP_BADARG_RET(BN_NEGATIVE(pA), ippStsBadArgErr); + IPP_BADARG_RET(cpCmp_BNU(BN_NUMBER(pA), BN_SIZE(pA), MOD_MODULUS( MNT_ENGINE(pCtx) ), MOD_LEN( MNT_ENGINE(pCtx) )) >= 0, ippStsScaleRangeErr); + /* check e */ + IPP_BADARG_RET(BN_NEGATIVE(pE), ippStsBadArgErr); + + cpMontExpBin_BN(pR, pA, pE, MNT_ENGINE( pCtx) ); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontexp_winsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontexp_winsize.c new file mode 100644 index 0000000..3db2cb8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontexp_winsize.c @@ -0,0 +1,80 @@ +/******************************************************************************* +* Copyright (C) 2003 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Contents: +// cpMontExp_WinSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpmontgomery.h" +#include "gsscramble.h" + + +#if !defined(_USE_WINDOW_EXP_) +IPP_OWN_DEFN (cpSize, cpMontExp_WinSize, (int bitsize)) +{ + IPP_UNREFERENCED_PARAMETER(bitsize); + return 1; +} + +#else +/* +// returns (optimal) window width +// Because of safety Window width depend on CACHE LINE size: +// P4,EM64T, ITP - 64 bytes +// XScale - 32 bytes +// Blend - no cache +*/ +#if !((_IPP==_IPP_W7) || (_IPP==_IPP_T7) || \ + (_IPP==_IPP_V8) || (_IPP==_IPP_P8) || \ + (_IPP==_IPP_S8) || (_IPP>=_IPP_G9) || \ + (_IPP32E==_IPP32E_M7) || \ + (_IPP32E==_IPP32E_U8) || (_IPP32E==_IPP32E_Y8) || \ + (_IPP32E==_IPP32E_N8) || (_IPP32E>=_IPP32E_E9)) +IPP_OWN_DEFN (cpSize, cpMontExp_WinSize, (int bitsize)) +{ + IPP_UNREFERENCED_PARAMETER(bitsize); + return 1; +} +#endif + +#if ((_IPP==_IPP_W7) || (_IPP==_IPP_T7) || \ + (_IPP==_IPP_V8) || (_IPP==_IPP_P8) || \ + (_IPP==_IPP_S8) || (_IPP>=_IPP_G9) || \ + (_IPP32E==_IPP32E_M7) || \ + (_IPP32E==_IPP32E_U8) || (_IPP32E==_IPP32E_Y8) || \ + (_IPP32E==_IPP32E_N8) || (_IPP32E>=_IPP32E_E9)) +IPP_OWN_DEFN (cpSize, cpMontExp_WinSize, (int bitsize)) +{ + return + //bitsize>3715? 8 : /*limited by 6 or 4 (LOG_CACHE_LINE_SIZE); we use it for windowing-exp imtigation */ + //bitsize>1434? 7 : + #if (_IPP !=_IPP_M5) + bitsize> 539? 6 : + bitsize> 197? 5 : + #endif + bitsize> 70? 4 : + bitsize> 25? 3 : + bitsize> 9? 2 : 1; +} +#endif + +#endif /* _USE_WINDOW_EXP_ */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontform.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontform.c new file mode 100644 index 0000000..567587e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontform.c @@ -0,0 +1,85 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// ippsMontForm() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpmontgomery.h" +#include "pcptool.h" + +/*F* +// Name: ippsMontForm +// +// Purpose: Converts input into Montgomery domain. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx==NULL +// pA==NULL +// pR==NULL +// ippStsContextMatchErr !MNT_VALID_ID(pCtx) +// !BN_VALID_ID(pA) +// !BN_VALID_ID(pR) +// ippStsBadArgErr A < 0. +// ippStsScaleRangeErr A >= Modulus. +// ippStsOutOfRangeErr R can't hold result +// ippStsNoErr no errors +// +// Parameters: +// pA pointer to the input [0, modulus-1] +// pCtx Montgomery context +// pR pointer to the output (A*R mod modulus) +*F*/ +IPPFUN(IppStatus, ippsMontForm,(const IppsBigNumState* pA, IppsMontState* pCtx, IppsBigNumState* pR)) +{ + IPP_BAD_PTR3_RET(pCtx, pA, pR); + + IPP_BADARG_RET(!MNT_VALID_ID(pCtx), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pA), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pR), ippStsContextMatchErr); + + IPP_BADARG_RET(BN_SIGN(pA) != ippBigNumPOS, ippStsBadArgErr); + IPP_BADARG_RET(cpCmp_BNU(BN_NUMBER(pA), BN_SIZE(pA), MOD_MODULUS( MNT_ENGINE(pCtx) ), MOD_LEN( MNT_ENGINE(pCtx) )) >= 0, ippStsScaleRangeErr); + IPP_BADARG_RET(BN_ROOM(pR) < MOD_LEN( MNT_ENGINE(pCtx) ), ippStsOutOfRangeErr); + + { + const int usedPoolLen = 1; + cpSize nsM = MOD_LEN( MNT_ENGINE(pCtx) ); + BNU_CHUNK_T* pDataA = gsModPoolAlloc(MNT_ENGINE(pCtx), usedPoolLen); + IPP_BAD_PTR1_RET(pDataA); // pDataA can be NULL, stop processing + + ZEXPAND_COPY_BNU(pDataA, nsM, BN_NUMBER(pA), BN_SIZE(pA)); + + MOD_METHOD( MNT_ENGINE(pCtx) )->encode(BN_NUMBER(pR), pDataA, MNT_ENGINE(pCtx)); + + FIX_BNU(BN_NUMBER(pR), nsM); + BN_SIZE(pR) = nsM; + BN_SIGN(pR) = ippBigNumPOS; + + gsModPoolFree(MNT_ENGINE(pCtx), usedPoolLen); + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontget.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontget.c new file mode 100644 index 0000000..cb874f1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontget.c @@ -0,0 +1,66 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// ippsMontGet() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpmontgomery.h" +#include "pcptool.h" + +/*F* +// Name: ippsMontGet +// +// Purpose: Extracts modulus. +// +// Returns: Reason: +// ippStsNullPtrErr pMont==NULL +// pModulus==NULL +// pSize==NULL +// ippStsContextMatchErr !MNT_VALID_ID() +// ippStsNoErr no errors +// +// Parameters: +// pModulus pointer to the modulus buffer +// pSize pointer to the modulus length (in Ipp32u chunks). +// pMont pointer to the context +*F*/ +IPPFUN(IppStatus, ippsMontGet,(Ipp32u* pModulus, int* pSize, const IppsMontState* pMont)) +{ + IPP_BAD_PTR3_RET(pMont, pModulus, pSize); + + IPP_BADARG_RET(!MNT_VALID_ID(pMont), ippStsContextMatchErr); + + { + cpSize len32 = MOD_LEN(MNT_ENGINE(pMont))*(Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(Ipp32u)); + Ipp32u* bnData = (Ipp32u*) MOD_MODULUS( MNT_ENGINE(pMont) ); + + FIX_BNU32(bnData, len32); + COPY_BNU(pModulus, bnData, len32); + *pSize = len32; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontgetsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontgetsize.c new file mode 100644 index 0000000..d154913 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontgetsize.c @@ -0,0 +1,62 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// ippsMontGetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpmontgomery.h" +#include "pcptool.h" + +/*F* +// Name: ippsMontGetSize +// +// Purpose: Specifies size of buffer in bytes. +// +// Returns: Reason: +// ippStsNullPtrErr pSize==NULL +// ippStsLengthErr length < 1 +// length > BITS2WORD32_SIZE(BN_MAXBITSIZE) +// ippStsNoErr no errors +// +// Parameters: +// method selected exponential method (unused parameter) +// length max modulus length (in Ipp32u chunks) +// pSize size of context +// +// Notes: Function always use method=ippBinaryMethod, +// so this parameter is ignored +*F*/ +IPPFUN(IppStatus, ippsMontGetSize, (IppsExpMethod method, int length, int* pSize)) +{ + IPP_BAD_PTR1_RET(pSize); + IPP_BADARG_RET(length<1 || length>BITS2WORD32_SIZE(BN_MAXBITSIZE), ippStsLengthErr); + + IPP_UNREFERENCED_PARAMETER(method); + + { + return cpMontGetSize(length, MONT_DEFAULT_POOL_LENGTH, pSize); + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontgomery.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontgomery.h new file mode 100644 index 0000000..e0b4c63 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontgomery.h @@ -0,0 +1,285 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// +*/ + +#if !defined(_CP_MONTGOMETRY_H) +#define _CP_MONTGOMETRY_H + +#include "pcpbn.h" +#include "gsmodstuff.h" + +#define MONT_DEFAULT_POOL_LENGTH (6) + +/* +// Montgomery spec structure +*/ +struct _cpMontgomery +{ + Ipp32u idCtx; /* Montgomery spec identifier */ + cpSize maxLen; /* Maximum length of modulus being stored */ + gsModEngine* pEngine; /* Modular arith engine structure */ +}; + +/* accessory macros */ +#define MNT_SET_ID(eng) ((eng)->idCtx = (Ipp32u)idCtxMontgomery ^ (Ipp32u)IPP_UINT_PTR(eng)) +#define MNT_ROOM(eng) ((eng)->maxLen) +#define MNT_ENGINE(eng) ((eng)->pEngine) + +#define MNT_VALID_ID(eng) ((((eng)->idCtx) ^ (Ipp32u)IPP_UINT_PTR((eng))) == (Ipp32u)idCtxMontgomery) + +/* default methos */ +#define EXPONENT_METHOD (ippBinaryMethod) + +/* alignment */ +#define MONT_ALIGNMENT ((int)(sizeof(void*))) + + +/* +// Pacp/unpack Montgomery context +*/ +#define cpPackMontCtx OWNAPI(cpPackMontCtx) + IPP_OWN_DECL (void, cpPackMontCtx, (const IppsMontState* pCtx, Ipp8u* pBuffer)) +#define cpUnpackMontCtx OWNAPI(cpUnpackMontCtx) + IPP_OWN_DECL (void, cpUnpackMontCtx, (const Ipp8u* pBuffer, IppsMontState* pCtx)) + +/* +// Montgomery reduction, multiplication and squaring +*/ +__INLINE void cpMontRed_BNU(BNU_CHUNK_T* pR, + BNU_CHUNK_T* pProduct, + gsModEngine* pModEngine) +{ + MOD_METHOD( pModEngine )->red(pR, pProduct, pModEngine); +} + +__INLINE void cpMontMul_BNU(BNU_CHUNK_T* pR, + const BNU_CHUNK_T* pA, + const BNU_CHUNK_T* pB, + gsModEngine* pModEngine) +{ + MOD_METHOD( pModEngine )->mul(pR, pA, pB, pModEngine); +} + +__INLINE cpSize cpMontMul_BNU_EX(BNU_CHUNK_T* pR, + const BNU_CHUNK_T* pA, cpSize nsA, + const BNU_CHUNK_T* pB, cpSize nsB, + gsModEngine* pModEngine) +{ + const int usedPoolLen = 1; + cpSize nsM = MOD_LEN( pModEngine ); + BNU_CHUNK_T* pDataR = pR; + BNU_CHUNK_T* pDataA = gsModPoolAlloc(pModEngine, usedPoolLen); + if(NULL == pDataA) + return -1; + + ZEXPAND_COPY_BNU(pDataA, nsM, pA, nsA); + ZEXPAND_COPY_BNU(pDataR, nsM, pB, nsB); + + MOD_METHOD( pModEngine )->mul(pDataR, pDataA, pDataR, pModEngine); + + gsModPoolFree(pModEngine, usedPoolLen); + return nsM; +} + +__INLINE void cpMontSqr_BNU(BNU_CHUNK_T* pR, + const BNU_CHUNK_T* pA, + gsModEngine* pModEngine) +{ + MOD_METHOD( pModEngine )->sqr(pR, pA, pModEngine); +} + +__INLINE void cpMontSqr_BNU_EX(BNU_CHUNK_T* pR, + const BNU_CHUNK_T* pA, cpSize nsA, + gsModEngine* pModEngine) +{ + cpSize nsM = MOD_LEN( pModEngine ); + ZEXPAND_COPY_BNU(pR, nsM, pA, nsA); + + MOD_METHOD( pModEngine )->sqr(pR, pR, pModEngine); +} + +/* +// Montgomery encoding/decoding +*/ +__INLINE cpSize cpMontEnc_BNU(BNU_CHUNK_T* pR, + const BNU_CHUNK_T* pXreg, + gsModEngine* pModEngine) +{ + cpSize nsM = MOD_LEN(pModEngine); + + MOD_METHOD( pModEngine )->encode(pR, pXreg, pModEngine); + + FIX_BNU(pR, nsM); + return nsM; +} + +__INLINE cpSize cpMontEnc_BNU_EX(BNU_CHUNK_T* pR, + const BNU_CHUNK_T* pXreg, cpSize nsX, + gsModEngine* pModEngine) +{ + cpSize nsM = MOD_LEN(pModEngine); + + ZEXPAND_COPY_BNU(pR, nsM, pXreg, nsX); + + MOD_METHOD( pModEngine )->encode(pR, pR, pModEngine); + + FIX_BNU(pR, nsM); + + return nsM; +} + +__INLINE cpSize cpMontDec_BNU(BNU_CHUNK_T* pR, + const BNU_CHUNK_T* pXmont, cpSize nsX, + gsModEngine* pModEngine) +{ + cpSize nsM = MOD_LEN( pModEngine ); + + ZEXPAND_COPY_BNU(pR, nsM, pXmont, nsX); + + MOD_METHOD( pModEngine )->decode(pR, pR, pModEngine); + + FIX_BNU(pR, nsM); + return nsM; +} + +__INLINE void cpMontMul_BN(IppsBigNumState* pRbn, + const IppsBigNumState* pXbn, + const IppsBigNumState* pYbn, + gsModEngine* pModEngine) +{ + cpSize nsM = cpMontMul_BNU_EX(BN_NUMBER(pRbn), + BN_NUMBER(pXbn), BN_SIZE(pXbn), + BN_NUMBER(pYbn), BN_SIZE(pYbn), + pModEngine); + + FIX_BNU(BN_NUMBER(pRbn), nsM); + BN_SIZE(pRbn) = nsM; + BN_SIGN(pRbn) = ippBigNumPOS; +} + +__INLINE void cpMontEnc_BN(IppsBigNumState* pRbn, + const IppsBigNumState* pXbn, + gsModEngine* pModEngine) +{ + cpSize nsM = cpMontEnc_BNU_EX(BN_NUMBER(pRbn), + BN_NUMBER(pXbn), BN_SIZE(pXbn), + pModEngine); + + BN_SIZE(pRbn) = nsM; + BN_SIGN(pRbn) = ippBigNumPOS; +} + +__INLINE void cpMontDec_BN(IppsBigNumState* pRbn, + const IppsBigNumState* pXbn, + gsModEngine* pModEngine) +{ + cpSize nsM = MOD_LEN(pModEngine); + cpMontDec_BNU(BN_NUMBER(pRbn), BN_NUMBER(pXbn), BN_SIZE(pXbn), pModEngine); + + BN_SIZE(pRbn) = nsM; + BN_SIGN(pRbn) = ippBigNumPOS; +} + +/* +// Montgomery exponentiation (binary) "fast" and "safe" versions +*/ +#define cpMontExpBin_BNU OWNAPI(cpMontExpBin_BNU) + IPP_OWN_DECL (cpSize, cpMontExpBin_BNU, (BNU_CHUNK_T* pY, const BNU_CHUNK_T* pX, cpSize nsX, const BNU_CHUNK_T* pE, cpSize nsE, gsModEngine* pModEngine)) +#define cpMontExpBin_BNU_sscm OWNAPI(cpMontExpBin_BNU_sscm) + IPP_OWN_DECL (cpSize, cpMontExpBin_BNU_sscm, (BNU_CHUNK_T* pY, const BNU_CHUNK_T* pX, cpSize nsX, const BNU_CHUNK_T* pE, cpSize nsE, gsModEngine* pModEngine)) + +__INLINE void cpMontExpBin_BN_sscm(IppsBigNumState* pYbn, + const IppsBigNumState* pXbn, + const IppsBigNumState* pEbn, + gsModEngine* pMont) +{ + BNU_CHUNK_T* pX = BN_NUMBER(pXbn); + cpSize nsX = BN_SIZE(pXbn); + BNU_CHUNK_T* pE = BN_NUMBER(pEbn); + cpSize nsE = BN_SIZE(pEbn); + BNU_CHUNK_T* pY = BN_NUMBER(pYbn); + cpSize nsY = cpMontExpBin_BNU_sscm(pY, pX,nsX, pE,nsE, pMont); + FIX_BNU(pY, nsY); + BN_SIZE(pYbn) = nsY; + BN_SIGN(pYbn) = ippBigNumPOS; +} + +__INLINE void cpMontExpBin_BN(IppsBigNumState* pYbn, + const IppsBigNumState* pXbn, + const IppsBigNumState* pEbn, + gsModEngine* pModEngine) +{ + BNU_CHUNK_T* pX = BN_NUMBER(pXbn); + cpSize nsX = BN_SIZE(pXbn); + BNU_CHUNK_T* pE = BN_NUMBER(pEbn); + cpSize nsE = BN_SIZE(pEbn); + BNU_CHUNK_T* pY = BN_NUMBER(pYbn); + cpSize nsY = cpMontExpBin_BNU(pY, pX,nsX, pE,nsE, pModEngine); + FIX_BNU(pY, nsY); + BN_SIZE(pYbn) = nsY; + BN_SIGN(pYbn) = ippBigNumPOS; +} + +/* +// Montgomery exponentiation (fixed window) +*/ +#define cpMontExp_WinSize OWNAPI(cpMontExp_WinSize) + IPP_OWN_DECL (cpSize, cpMontExp_WinSize, (int bitsize)) + +#if defined(_USE_WINDOW_EXP_) +#define cpMontExpWin_BN_sscm OWNAPI(cpMontExpWin_BN_sscm) + IPP_OWN_DECL (void, cpMontExpWin_BN_sscm, (IppsBigNumState* pY, const IppsBigNumState* pX, const IppsBigNumState* pE, gsModEngine* pMont, BNU_CHUNK_T* pPrecompResource)) +#define cpMontExpWin_BN OWNAPI(cpMontExpWin_BN) + IPP_OWN_DECL (void, cpMontExpWin_BN, (IppsBigNumState* pY, const IppsBigNumState* pX, const IppsBigNumState* pE, gsModEngine* pMont, BNU_CHUNK_T* pPrecompResource)) +#endif + +/* +// Montgomery multi-exponentiation +*/ +/* precompute table for multi-exponentiation */ +#define cpMontMultiExpInitArray OWNAPI(cpMontMultiExpInitArray) + IPP_OWN_DECL (void, cpMontMultiExpInitArray, (BNU_CHUNK_T* pPrecomTbl, const BNU_CHUNK_T** ppX, cpSize xItemBitSize, cpSize numItems, gsModEngine* pMont)) + +/* multi-exponentiation */ +#define cpFastMontMultiExp OWNAPI(cpFastMontMultiExp) + IPP_OWN_DECL (void, cpFastMontMultiExp, (BNU_CHUNK_T* pY, const BNU_CHUNK_T* pPrecomTbl, const Ipp8u** ppE, cpSize eItemBitSize, cpSize numItems, gsModEngine* pMont)) +/* +// Montgomery inversion +*/ +#define cpMontInv_BNU OWNAPI(cpMontInv_BNU) + IPP_OWN_DECL (BNU_CHUNK_T*, cpMontInv_BNU, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, IppsMontState* pMont)) +#define cpRegInv_BNU OWNAPI(cpRegInv_BNU) + IPP_OWN_DECL (BNU_CHUNK_T*, cpRegInv_BNU, (BNU_CHUNK_T* pR, const BNU_CHUNK_T* pA, IppsMontState* pMont)) + + +/* +// Montgomery internal GetSize/Init functions +*/ +#define cpMontGetSize OWNAPI(cpMontGetSize) + IPP_OWN_DECL (IppStatus, cpMontGetSize, (cpSize maxLen32, int poolLength, cpSize* pCtxSize)) +#define cpMontInit OWNAPI(cpMontInit) + IPP_OWN_DECL (IppStatus, cpMontInit, (int maxLen32, int poolLength, IppsMontState* pMont)) +#define cpMontSet OWNAPI(cpMontSet) + IPP_OWN_DECL (IppStatus, cpMontSet, (const Ipp32u* pModulus, cpSize len32, IppsMontState* pMont)) + +#endif /* _CP_MONTGOMETRY_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontinit.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontinit.c new file mode 100644 index 0000000..e500ef5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontinit.c @@ -0,0 +1,61 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// ippsMontInit() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpmontgomery.h" +#include "pcptool.h" + +/*F* +// Name: ippsMontInit +// +// Purpose: Initializes the symbolic data structure and partitions the +// specified buffer space. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// ippStsLengthErr length < 1 +// length > BITS2WORD32_SIZE(BN_MAXBITSIZE) +// ippStsNoErr no errors +// +// Parameters: +// method selected exponential method (unused parameter) +// length max modulus length (in Ipp32u chunks) +// pCtx pointer to Montgomery context +*F*/ +IPPFUN(IppStatus, ippsMontInit,(IppsExpMethod method, int length, IppsMontState* pCtx)) +{ + IPP_BADARG_RET(length<1 || length>BITS2WORD32_SIZE(BN_MAXBITSIZE), ippStsLengthErr); + + IPP_BAD_PTR1_RET(pCtx); + + IPP_UNREFERENCED_PARAMETER(method); + + { + return cpMontInit(length, MONT_DEFAULT_POOL_LENGTH, pCtx); + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontmul.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontmul.c new file mode 100644 index 0000000..0124332 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontmul.c @@ -0,0 +1,96 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Contents: +// ippsMontMul() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpmontgomery.h" +#include "pcptool.h" + +/*F* +// Name: ippsMontMul +// +// Purpose: Computes Montgomery modular multiplication for positive big +// number integers of Montgomery form. The following pseudocode +// represents this function: +// r <- ( a * b * R^(-1) ) mod m +// +// Returns: Reason: +// ippStsNoErr Returns no error. +// ippStsNullPtrErr Returns an error when pointers are null. +// ippStsBadArgErr Returns an error when a or b is a negative integer. +// ippStsScaleRangeErr Returns an error when a or b is more than m. +// ippStsOutOfRangeErr Returns an error when IppsBigNumState *r is larger than +// IppsMontState *m. +// ippStsContextMatchErr Returns an error when the context parameter does +// not match the operation. +// +// Parameters: +// pA Multiplicand within the range [0, m - 1]. +// pB Multiplier within the range [0, m - 1]. +// pCtx Modulus. +// pR Montgomery multiplication result. +// +// Notes: The size of IppsBigNumState *r should not be less than the data +// length of the modulus m. +*F*/ +IPPFUN(IppStatus, ippsMontMul, (const IppsBigNumState* pA, const IppsBigNumState* pB, IppsMontState* pCtx, IppsBigNumState* pR)) +{ + IPP_BAD_PTR4_RET(pA, pB, pCtx, pR); + + IPP_BADARG_RET(!MNT_VALID_ID(pCtx), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pA), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pB), ippStsContextMatchErr); + IPP_BADARG_RET(!BN_VALID_ID(pR), ippStsContextMatchErr); + + IPP_BADARG_RET(BN_NEGATIVE(pA) || BN_NEGATIVE(pB), ippStsBadArgErr); + IPP_BADARG_RET(cpCmp_BNU(BN_NUMBER(pA), BN_SIZE(pA), MOD_MODULUS( MNT_ENGINE(pCtx) ), MOD_LEN( MNT_ENGINE(pCtx) )) >= 0, ippStsScaleRangeErr); + IPP_BADARG_RET(cpCmp_BNU(BN_NUMBER(pB), BN_SIZE(pB), MOD_MODULUS( MNT_ENGINE(pCtx) ), MOD_LEN( MNT_ENGINE(pCtx) )) >= 0, ippStsScaleRangeErr); + IPP_BADARG_RET(BN_ROOM(pR) < MOD_LEN( MNT_ENGINE(pCtx) ), ippStsOutOfRangeErr); + + { + const int usedPoolLen = 2; + cpSize nsM = MOD_LEN( MNT_ENGINE(pCtx) ); + BNU_CHUNK_T* pDataR = BN_NUMBER(pR); + BNU_CHUNK_T* pDataA = gsModPoolAlloc(MNT_ENGINE(pCtx), usedPoolLen); + IPP_BAD_PTR1_RET(pDataA); // pDataA can be NULL, stop processing + + BNU_CHUNK_T* pDataB = pDataA + nsM; + + ZEXPAND_COPY_BNU(pDataA, nsM, BN_NUMBER(pA), BN_SIZE(pA)); + ZEXPAND_COPY_BNU(pDataB, nsM, BN_NUMBER(pB), BN_SIZE(pB)); + + MOD_METHOD( MNT_ENGINE(pCtx) )->mul(pDataR, pDataA, pDataB, MNT_ENGINE(pCtx)); + + gsModPoolFree(MNT_ENGINE(pCtx), usedPoolLen); + + FIX_BNU(pDataR, nsM); + BN_SIZE(pR) = nsM; + BN_SIGN(pR) = ippBigNumPOS; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontred.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontred.c new file mode 100644 index 0000000..cf6ab1c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpmontred.c @@ -0,0 +1,70 @@ +/******************************************************************************* +* Copyright (C) 2010 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +*/ + +#include "owncp.h" +#include "pcpbnuarith.h" + + +#if !((_IPP==_IPP_W7) || \ + (_IPP==_IPP_T7) || \ + (_IPP==_IPP_V8) || \ + (_IPP==_IPP_P8) || \ + (_IPP>=_IPP_G9) || \ + (_IPP==_IPP_S8) || \ + (_IPP32E==_IPP32E_M7) || \ + (_IPP32E==_IPP32E_U8) || \ + (_IPP32E==_IPP32E_Y8) || \ + (_IPP32E>=_IPP32E_E9) || \ + (_IPP32E==_IPP32E_N8)) || \ + defined(_USE_C_cpMontRedAdc_BNU_) + +#define cpMontRedAdc_BNU OWNAPI(cpMontRedAdc_BNU) + +IPP_OWN_DEFN (void, cpMontRedAdc_BNU, (BNU_CHUNK_T* pR, BNU_CHUNK_T* pProduct, const BNU_CHUNK_T* pModulus, cpSize nsM, BNU_CHUNK_T m0)) +{ + BNU_CHUNK_T carry; + BNU_CHUNK_T extension; + + cpSize n; + for(n=0, carry = 0; n<(nsM-1); n++) { + BNU_CHUNK_T u = pProduct[n]*m0; + BNU_CHUNK_T t = pProduct[nsM +n +1] + carry; + + extension = cpAddMulDgt_BNU(pProduct+n, pModulus, nsM, u); + ADD_AB(carry, pProduct[nsM+n], pProduct[nsM+n], extension); + t += carry; + + carry = t 4096? 6 : /* 4096- .. . */ + bitsize> 2666? 5 : /* 2666 - 4095 */ + #endif + bitsize> 717? 4 : /* 717 - 2665 */ + bitsize> 178? 3 : /* 178 - 716 */ + bitsize> 41? 2 : 1; /* 41 - 177 */ + #else + IPP_UNREFERENCED_PARAMETER(bitsize); + return 1; + #endif +} + +/* +// Montgomery encoding/decoding +*/ +__INLINE cpSize gsMontEnc_BNU(BNU_CHUNK_T* pR, + const BNU_CHUNK_T* pXreg, cpSize nsX, + const gsModEngine* pMont) +{ + cpSize nsM = MOD_LEN( pMont ); + ZEXPAND_COPY_BNU(pR, nsM, pXreg, nsX); + MOD_METHOD( pMont )->encode(pR, pR, (gsModEngine*)pMont); + return nsM; +} + +__INLINE cpSize gsMontDec_BNU(BNU_CHUNK_T* pR, + const BNU_CHUNK_T* pXmont, + gsModEngine* pMont) +{ + cpSize nsM = MOD_LEN(pMont); + MOD_METHOD( pMont )->decode(pR, pXmont, (gsModEngine*)pMont); + return nsM; +} + +__INLINE void gsMontEnc_BN(IppsBigNumState* pRbn, + const IppsBigNumState* pXbn, + gsModEngine* pMont) +{ + BNU_CHUNK_T* pR = BN_NUMBER(pRbn); + cpSize nsM = MOD_LEN(pMont); + + gsMontEnc_BNU(pR, BN_NUMBER(pXbn), BN_SIZE(pXbn), pMont); + + FIX_BNU(pR, nsM); + BN_SIZE(pRbn) = nsM; + BN_SIGN(pRbn) = ippBigNumPOS; +} + +/* exponentiation buffer size */ +#define gsMontExpBinBuffer OWNAPI(gsMontExpBinBuffer) + IPP_OWN_DECL (cpSize, gsMontExpBinBuffer, (int modulusBits)) +#define gsMontExpWinBuffer OWNAPI(gsMontExpWinBuffer) + IPP_OWN_DECL (cpSize, gsMontExpWinBuffer, (int modulusBits)) + +/* exponentiation prototype */ +IPP_OWN_FUNPTR (cpSize, ngMontExp, (BNU_CHUNK_T* dataY, const BNU_CHUNK_T* dataX, cpSize nsX, const BNU_CHUNK_T* dataE, cpSize nbitsE, gsModEngine* pMont, BNU_CHUNK_T* pBuffer)) +IPP_OWN_FUNPTR (cpSize, ngMontDualExp, (BNU_CHUNK_T* dataY[2], const BNU_CHUNK_T* dataX[2], cpSize nsX[2], const BNU_CHUNK_T* dataE[2], gsModEngine* pMont[2], BNU_CHUNK_T* pBuffer)) + +/* +// "fast" and "safe" binary montgomery exponentiation ("fast" version) +*/ +#define gsMontExpBin_BNU OWNAPI(gsMontExpBin_BNU) + IPP_OWN_DECL (cpSize, gsMontExpBin_BNU, (BNU_CHUNK_T* dataY, const BNU_CHUNK_T* dataX, cpSize nsX, const BNU_CHUNK_T* dataE, cpSize nbitsE, gsModEngine* pMont, BNU_CHUNK_T* pBuffer)) +#define gsModExpBin_BNU OWNAPI(gsModExpBin_BNU) + IPP_OWN_DECL (cpSize, gsModExpBin_BNU, (BNU_CHUNK_T* dataY, const BNU_CHUNK_T* dataX, cpSize nsX, const BNU_CHUNK_T* dataE, cpSize nbitsE, gsModEngine* pMont, BNU_CHUNK_T* pBuffer)) + +#define gsMontExpBin_BNU_sscm OWNAPI(gsMontExpBin_BNU_sscm) + IPP_OWN_DECL (cpSize, gsMontExpBin_BNU_sscm, (BNU_CHUNK_T* pY, const BNU_CHUNK_T* pX, cpSize nsX, const BNU_CHUNK_T* pE, cpSize nbitsE, gsModEngine* pMont, BNU_CHUNK_T* pBuffer)) +#define gsModExpBin_BNU_sscm OWNAPI(gsModExpBin_BNU_sscm) + IPP_OWN_DECL (cpSize, gsModExpBin_BNU_sscm, (BNU_CHUNK_T* pY, const BNU_CHUNK_T* pX, cpSize nsX, const BNU_CHUNK_T* pE, cpSize nbitsE, gsModEngine* pMont, BNU_CHUNK_T* pBuffer)) + +/* +// "fast" and "safe" fixed-size window montgomery exponentiation +*/ +#define gsMontExpWin_BNU OWNAPI(gsMontExpWin_BNU) + IPP_OWN_DECL (cpSize, gsMontExpWin_BNU, (BNU_CHUNK_T* pY, const BNU_CHUNK_T* pX, cpSize nsX, const BNU_CHUNK_T* dataE, cpSize nbitsE, gsModEngine* pMont, BNU_CHUNK_T* pBuffer)) +#define gsModExpWin_BNU OWNAPI(gsModExpWin_BNU) + IPP_OWN_DECL (cpSize, gsModExpWin_BNU, (BNU_CHUNK_T* pY, const BNU_CHUNK_T* pX, cpSize nsX, const BNU_CHUNK_T* dataE, cpSize nbitsE, gsModEngine* pMont, BNU_CHUNK_T* pBuffer)) + +#define gsMontExpWin_BNU_sscm OWNAPI(gsMontExpWin_BNU_sscm) + IPP_OWN_DECL (cpSize, gsMontExpWin_BNU_sscm, (BNU_CHUNK_T* dataY, const BNU_CHUNK_T* dataX, cpSize nsX, const BNU_CHUNK_T* dataE, cpSize nbitsE, gsModEngine* pMont, BNU_CHUNK_T* pBuffer)) +#define gsModExpWin_BNU_sscm OWNAPI(gsModExpWin_BNU_sscm) + IPP_OWN_DECL (cpSize, gsModExpWin_BNU_sscm, (BNU_CHUNK_T* dataY, const BNU_CHUNK_T* dataX, cpSize nsX, const BNU_CHUNK_T* dataE, cpSize nbitsE, gsModEngine* pMont, BNU_CHUNK_T* pBuffer)) + +#endif /* _CP_NG_MONT_EXP_STUFF_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_avx2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_avx2.c new file mode 100644 index 0000000..1efd129 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_avx2.c @@ -0,0 +1,682 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "owncp.h" + +#if (_IPP32E>=_IPP32E_L9) + +#include "pcpbnuimpl.h" +#include "pcpbnumisc.h" + +#include "pcpngmontexpstuff.h" +#include "pcpngmontexpstuff_avx2.h" +#include "gsscramble.h" +#include "pcpmask_ct.h" + + +//tbcd: temporary excluded: #include +/* + converts regular (base = 2^32) representation (pRegular, regLen) + into "redundant" (base = 2^DIGIT_SIZE) represenrartion (pRep, repLen) + + return 1 + + note: + 1) repLen >= (bitsize +DIGIT_SIZE-1)/DIGIT_SIZE for complete conversion + 2) regular representation should expanded by at least one zero value, + pre-requisite: pRegular[regLen] == 0 to make conversion correct + 3) caller must provide suitable lengths of regular and redundant respresentations + so, conversion does correct +*/ +static int regular_dig27(Ipp64u* pRep27, int repLen, const Ipp32u* pRegular, int regLen) +{ + /* expected number of digit in redundant representation */ + int n = cpDigitNum_avx2(regLen*BITSIZE(Ipp32u), EXP_DIGIT_SIZE_AVX2); + + //tbcd: temporary excluded: assert(pRegular[regLen]==0); /* test pre-requisite */ + + { + int redBit; /* output representatin bit */ + int i; + for(i=0, redBit=0; i>= shift; + pRep27[i] = x & EXP_DIGIT_MASK_AVX2; + } + + /* expands by zeros if necessary */ + for(; i=NORM_DIGSIZE_AVX2) { + pRegular[idx++] = (Ipp32u)(x & NORM_MASK_AVX2); + x >>= NORM_DIGSIZE_AVX2; + shift -= NORM_DIGSIZE_AVX2; + } + } + + if(idx0; nsE--) { + eValue = dataE[nsE-1]; + + for(n=0; n0; nsE--) { + BNU_CHUNK_T eValue = dataE[nsE-1]; + + int n; + for(n=BNU_CHUNK_BITS; n>0; n--) { + /* T = ( msb(eValue) )? X : mont(1) */ + BNU_CHUNK_T mask = cpIsMsb_ct(eValue); + eValue <<= 1; + cpMaskedCopyBNU_ct(redT, mask, redX, redR, redLen); + + /* squaring: Y = Y^2 */ + cpMontSqr_avx2(redY, redY, redM, redLen, k0, redBuffer); + #ifdef _EXP_AVX2_DEBUG_ + debugToConvMontDomain(dbgValue, redY, redM, redLen, dataM, dataRR, nsM, k0, redBuffer); + #endif + /* and multiply: Y = Y * T */ + cpMontMul_avx2(redY, redY, redT, redM, redLen, k0, redBuffer); + } + } + + /* convert result back to regular domain */ + ZEXPAND_BNU(redT, 0, redBufferLen); + redT[0] = 1; + cpMontMul_avx2(redY, redY, redT, redM, redLen, k0, redBuffer); + dig27_regular((Ipp32u*)dataY, nsM*sizeof(BNU_CHUNK_T)/sizeof(ipp32u), redY, redLen); + + return nsM; +} + +#endif /* !_USE_WINDOW_EXP_ */ + + +#if defined(_USE_WINDOW_EXP_) +/* +// "fast" fixed-size window montgomery exponentiation +// +// scratch buffer structure: +// precomuted table of multipliers[(1< bitSizeE > 0), + it is checked in initialization phase by (ippsRSA_GetSizePublickey() and ippsRSA_InitPublicKey). + Buffer "redE" assigned for copy of dataE, is 1 (64-bit) chunk longer than size of RSA modulus, + therefore the access "*((Ipp32u*)((Ipp16u*)redE+ eBit/BITSIZE(Ipp16u)))" is always inside the boundary. + */ + /* extract 1-st window value */ + Ipp32u eChunk = *((Ipp32u*)((Ipp16u*)redE+ eBit/BITSIZE(Ipp16u))); + int shift = eBit & 0xF; + cpSize windowVal = (cpSize)((eChunk>>shift) &wmask); + + /* initialize result */ + COPY_BNU(redY, redTable+windowVal*redLen, redLen); + #ifdef _EXP_AVX2_DEBUG_ + debugToConvMontDomain(dbgValue, redY, redM, redLen, dataM, dataRR, nsM, k0, redBuffer); + #endif + + for(eBit-=window; eBit>=0; eBit-=window) { + /* do squaring window-times */ + for(n=0; n>shift) &wmask); + + /* precomputed value muptiplication */ + if(windowVal) { + cpMontMul_avx2(redY, redY, redTable+windowVal*redLen, redM, redLen, k0, redBuffer); + #ifdef _EXP_AVX2_DEBUG_ + COPY_BNU(redT, redTable+windowVal*redBufferLen, redBufferLen); + cpMontMul_avx2(redY, redY, redT, redM, redLen, k0, redBuffer); + debugToConvMontDomain(dbgValue, redY, redM, redLen, dataM, dataRR, nsM, k0, redBuffer); + #endif + } + } + } + #ifdef _EXP_AVX2_DEBUG_ + debugToConvMontDomain(dbgValue, redY, redM, redLen, dataM, dataRR, nsM, k0, redBuffer); + #endif + + /* convert result back */ + ZEXPAND_BNU(redT, 0, redBufferLen); + redT[0] = 1; + cpMontMul_avx2(redY, redY, redT, redM, redLen, k0, redBuffer); + dig27_regular((Ipp32u*)dataY, nsM*(Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(ipp32u)), redY, redLen); + + return nsM; +} + +/* +// "safe" fixed-size window montgomery exponentiation +// +// scratch buffer structure: +// precomuted table of multipliers[(1<>shift) &wmask); + + /* initialize result */ + gsScrambleGet_sscm(redY, redLen, redTable, windowVal, window); + #ifdef _EXP_AVX2_DEBUG_ + debugToConvMontDomain(dbgValue, redY, redM, redLen, dataM, dataRR, nsM, k0, redBuffer); + #endif + + for(eBit-=window; eBit>=0; eBit-=window) { + /* do squaring window-times */ + for(n=0; n>shift) &wmask); + /* exptact precomputed value and muptiply */ + gsScrambleGet_sscm(redT, redLen, redTable, windowVal, window); + cpMontMul_avx2(redY, redY, redT, redM, redLen, k0, redBuffer); + #ifdef _EXP_AVX2_DEBUG_ + debugToConvMontDomain(dbgValue, redY, redM, redLen, dataM, dataRR, nsM, k0, redBuffer); + #endif + } + } + #ifdef _EXP_AVX2_DEBUG_ + debugToConvMontDomain(dbgValue, redY, redM, redLen, dataM, dataRR, nsM, k0, redBuffer); + #endif + + /* convert result back */ + ZEXPAND_BNU(redT, 0, redBufferLen); + redT[0] = 1; + cpMontMul_avx2(redY, redY, redT, redM, redLen, k0, redBuffer); + dig27_regular((Ipp32u*)dataY, nsM*(Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(ipp32u)), redY, redLen); + + return nsM; +} +#endif /* _USE_WINDOW_EXP_ */ + +#endif /* _IPP32E_L9 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_avx2.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_avx2.h new file mode 100644 index 0000000..e4c0adb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_avx2.h @@ -0,0 +1,100 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Internal Definitions of AVX2 Montgomery Exp +// +*/ +#include "owncp.h" + +#if (_IPP32E>=_IPP32E_L9) + +#include "pcpbnuimpl.h" +#include "pcpngmontexpstuff.h" + +#define RSA_AVX2_MIN_BITSIZE (1024) +#define RSA_AVX2_MAX_BITSIZE (13*1024) + +#define NORM_DIGSIZE_AVX2 (BITSIZE(Ipp32u)) +#define NORM_BASE_AVX2 ((Ipp64u)1<=_IPP32E_K1) +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(disable: 4310) // cast truncates constant value in MSVC +#endif + +#include "pcptool.h" + +#include "pcpbnuimpl.h" +#include "pcpbnumisc.h" + +#include "pcpngmontexpstuff.h" +#include "pcpngmontexpstuff_avx512.h" +#include "gsscramble.h" +#include "pcpmask_ct.h" + +/* ams functions */ +IPP_OWN_FUNPTR (void, cpAMM52, (Ipp64u* out, const Ipp64u* a, const Ipp64u* b, const Ipp64u* m, Ipp64u k0, int len, Ipp64u* res)) + +static void AMM52(Ipp64u* out, const Ipp64u* a, const Ipp64u* b, const Ipp64u* m, Ipp64u k0, int len, Ipp64u* res) +{ + #define NUM64 ((Ipp32s)(sizeof(__m512i)/sizeof(Ipp64u))) + + __mmask8 k1 = (__mmask8)_mm512_kmov(0x02); /* mask of the 2-nd elment */ + + __m512i zero = _mm512_setzero_si512(); /* zeros */ + + int n; + int tail = len & (NUM64 -1); + int expLen = len; + if(tail) expLen += (NUM64 - tail); + + /* make sure not inplace operation */ + //tbcd: temporary excluded: assert(res!=a); + //tbcd: temporary excluded: assert(res!=b); + + /* set result to zero */ + for(n=0; n>= EXP_DIGIT_SIZE_AVX512; + } + } +} + +static void AMM52x20(Ipp64u* out, const Ipp64u* a, const Ipp64u* b, const Ipp64u* m, Ipp64u k0, int len, Ipp64u* res) +{ + __mmask8 k2 = (__mmask8)_mm512_kmov(0x0f); /* mask of the 0-3 elments */ + + /* load a */ + __m512i A0 = _mm512_loadu_si512(a); + __m512i A1 = _mm512_loadu_si512(a+NUM64); + __m512i A2 = _mm512_maskz_loadu_epi64(k2, a+2*NUM64); + + /* load m */ + __m512i M0 = _mm512_loadu_si512(m); + __m512i M1 = _mm512_loadu_si512(m+NUM64); + __m512i M2 = _mm512_maskz_loadu_epi64(k2, m+2*NUM64); + + /* R0, R1, R2 holds temporary result */ + __m512i R0 = _mm512_setzero_si512(); + __m512i R1 = _mm512_setzero_si512(); + __m512i R2 = _mm512_setzero_si512(); + + __m512i ZERO = _mm512_setzero_si512(); /* zeros */ + __m512i K = _mm512_set1_epi64((Ipp64s)k0); + + IPP_UNREFERENCED_PARAMETER(len); + IPP_UNREFERENCED_PARAMETER(res); + + __mmask8 k1 = (__mmask8)_mm512_kmov(0x01); /* mask of the 0 elment */ + int i; + for(i=0; i<20; i++) { + __m512i Bi = _mm512_set1_epi64((Ipp64s)b[i]); /* bloadcast(b[i]) */ + __m512i Yi = _mm512_setzero_si512(); /* Yi = 0 */ + __m512i tmp; + + R0 = _mm512_madd52lo_epu64(R0, A0, Bi); /* R += A*Bi (lo) */ + R1 = _mm512_madd52lo_epu64(R1, A1, Bi); + R2 = _mm512_madd52lo_epu64(R2, A2, Bi); + + Yi = _mm512_madd52lo_epu64(ZERO, K, R0); /* Yi = R0*K */ + Yi = _mm512_permutexvar_epi64(ZERO, Yi); /* broadcast Yi */ + + R0 = _mm512_madd52lo_epu64(R0, M0, Yi); /* R += M*Yi (lo) */ + R1 = _mm512_madd52lo_epu64(R1, M1, Yi); + R2 = _mm512_madd52lo_epu64(R2, M2, Yi); + + /* shift R */ + tmp = _mm512_maskz_srli_epi64(k1, R0, EXP_DIGIT_SIZE_AVX512); + R0 = _mm512_alignr_epi64(R1, R0, 1); + R1 = _mm512_alignr_epi64(R2, R1, 1); + R2 = _mm512_alignr_epi64(ZERO, R2, 1); + R0 = _mm512_add_epi64(R0, tmp); + + R0 = _mm512_madd52hi_epu64(R0, A0, Bi); /* R += A*Bi (hi) */ + R1 = _mm512_madd52hi_epu64(R1, A1, Bi); + R2 = _mm512_madd52hi_epu64(R2, A2, Bi); + + R0 = _mm512_madd52hi_epu64(R0, M0, Yi); /* R += M*Yi (hi) */ + R1 = _mm512_madd52hi_epu64(R1, M1, Yi); + R2 = _mm512_madd52hi_epu64(R2, M2, Yi); + } + + /* store de-normilized result */ + _mm512_storeu_si512(out, R0); + _mm512_storeu_si512(out+NUM64, R1); + _mm512_mask_storeu_epi64(out+2*NUM64, k2, R2); + + /* normalize result */ + { + Ipp64u acc = 0; + #if !defined(_MSC_VER) || defined(__INTEL_COMPILER) // unkonwn for msvc + #pragma nounroll + #endif + for(i=0; i<20; i++) { + acc += out[i]; + out[i] = acc & EXP_DIGIT_MASK_AVX512; + acc >>= EXP_DIGIT_SIZE_AVX512; + } + } +} + +static void AMM52x40(Ipp64u* out, const Ipp64u* a, const Ipp64u* b, const Ipp64u* m, Ipp64u k0, int len, Ipp64u* res) +{ + /* load a */ + __m512i A0 = _mm512_loadu_si512(a); + __m512i A1 = _mm512_loadu_si512(a+NUM64); + __m512i A2 = _mm512_loadu_si512(a+2*NUM64); + __m512i A3 = _mm512_loadu_si512(a+3*NUM64); + __m512i A4 = _mm512_loadu_si512(a+4*NUM64); + + /* load m */ + __m512i M0 = _mm512_loadu_si512(m); + __m512i M1 = _mm512_loadu_si512(m+NUM64); + __m512i M2 = _mm512_loadu_si512(m+2*NUM64); + __m512i M3 = _mm512_loadu_si512(m+3*NUM64); + __m512i M4 = _mm512_loadu_si512(m+4*NUM64); + + /* R0, R1, R2, R3, R4 holds temporary result */ + __m512i R0 = _mm512_setzero_si512(); + __m512i R1 = _mm512_setzero_si512(); + __m512i R2 = _mm512_setzero_si512(); + __m512i R3 = _mm512_setzero_si512(); + __m512i R4 = _mm512_setzero_si512(); + + __m512i ZERO = _mm512_setzero_si512(); /* zeros */ + __m512i K = _mm512_set1_epi64((Ipp64s)k0); + + IPP_UNREFERENCED_PARAMETER(len); + IPP_UNREFERENCED_PARAMETER(res); + + __mmask8 k1 = (__mmask8)_mm512_kmov(0x01); /* mask of the 0 elment */ + int i; + for(i=0; i<40; i++) { + __m512i Bi = _mm512_set1_epi64((Ipp64s)b[i]); /* bloadcast(b[i]) */ + __m512i Yi = _mm512_setzero_si512(); /* Yi = 0 */ + __m512i tmp; + + R0 = _mm512_madd52lo_epu64(R0, A0, Bi); /* R += A*Bi (lo) */ + R1 = _mm512_madd52lo_epu64(R1, A1, Bi); + R2 = _mm512_madd52lo_epu64(R2, A2, Bi); + R3 = _mm512_madd52lo_epu64(R3, A3, Bi); + R4 = _mm512_madd52lo_epu64(R4, A4, Bi); + + Yi = _mm512_madd52lo_epu64(ZERO, K, R0); /* Yi = R0*K */ + Yi = _mm512_permutexvar_epi64(ZERO, Yi); /* broadcast Yi */ + + R0 = _mm512_madd52lo_epu64(R0, M0, Yi); /* R += M*Yi (lo) */ + R1 = _mm512_madd52lo_epu64(R1, M1, Yi); + R2 = _mm512_madd52lo_epu64(R2, M2, Yi); + R3 = _mm512_madd52lo_epu64(R3, M3, Yi); + R4 = _mm512_madd52lo_epu64(R4, M4, Yi); + + /* shift R */ + tmp = _mm512_maskz_srli_epi64(k1, R0, EXP_DIGIT_SIZE_AVX512); + R0 = _mm512_alignr_epi64(R1, R0, 1); + R1 = _mm512_alignr_epi64(R2, R1, 1); + R2 = _mm512_alignr_epi64(R3, R2, 1); + R3 = _mm512_alignr_epi64(R4, R3, 1); + R4 = _mm512_alignr_epi64(ZERO, R4, 1); + R0 = _mm512_add_epi64(R0, tmp); + + R0 = _mm512_madd52hi_epu64(R0, A0, Bi); /* R += A*Bi (hi) */ + R1 = _mm512_madd52hi_epu64(R1, A1, Bi); + R2 = _mm512_madd52hi_epu64(R2, A2, Bi); + R3 = _mm512_madd52hi_epu64(R3, A3, Bi); + R4 = _mm512_madd52hi_epu64(R4, A4, Bi); + + R0 = _mm512_madd52hi_epu64(R0, M0, Yi); /* R += M*Yi (hi) */ + R1 = _mm512_madd52hi_epu64(R1, M1, Yi); + R2 = _mm512_madd52hi_epu64(R2, M2, Yi); + R3 = _mm512_madd52hi_epu64(R3, M3, Yi); + R4 = _mm512_madd52hi_epu64(R4, M4, Yi); + } + + /* store de-normilized result */ + _mm512_storeu_si512(out, R0); + _mm512_storeu_si512(out+NUM64, R1); + _mm512_storeu_si512(out+2*NUM64, R2); + _mm512_storeu_si512(out+3*NUM64, R3); + _mm512_storeu_si512(out+4*NUM64, R4); + + /* normalize result */ + { + Ipp64u acc = 0; + #if !defined(_MSC_VER) || defined(__INTEL_COMPILER) // unkonwn for msvc + #pragma nounroll + #endif + for(i=0; i<40; i++) { + acc += out[i]; + out[i] = acc & EXP_DIGIT_MASK_AVX512; + acc >>= EXP_DIGIT_SIZE_AVX512; + } + } +} + +static void AMM52x60(Ipp64u* out, const Ipp64u* a, const Ipp64u* b, const Ipp64u* m, Ipp64u k0, int len, Ipp64u* res) +{ + __mmask8 k2 = (__mmask8)_mm512_kmov(0x0f); /* mask of the 0-3 elments */ + + /* load a */ + __m512i A0 = _mm512_loadu_si512(a); + __m512i A1 = _mm512_loadu_si512(a+NUM64); + __m512i A2 = _mm512_loadu_si512(a+2*NUM64); + __m512i A3 = _mm512_loadu_si512(a+3*NUM64); + __m512i A4 = _mm512_loadu_si512(a+4*NUM64); + __m512i A5 = _mm512_loadu_si512(a+5*NUM64); + __m512i A6 = _mm512_loadu_si512(a+6*NUM64); + __m512i A7 = _mm512_maskz_loadu_epi64(k2, a+7*NUM64); + + /* load m */ + __m512i M0 = _mm512_loadu_si512(m); + __m512i M1 = _mm512_loadu_si512(m+NUM64); + __m512i M2 = _mm512_loadu_si512(m+2*NUM64); + __m512i M3 = _mm512_loadu_si512(m+3*NUM64); + __m512i M4 = _mm512_loadu_si512(m+4*NUM64); + __m512i M5 = _mm512_loadu_si512(m+5*NUM64); + __m512i M6 = _mm512_loadu_si512(m+6*NUM64); + __m512i M7 = _mm512_maskz_loadu_epi64(k2, m+7*NUM64); + + /* R0, R1, R2, R3, R4, R5, R6, R7 holds temporary result */ + __m512i R0 = _mm512_setzero_si512(); + __m512i R1 = _mm512_setzero_si512(); + __m512i R2 = _mm512_setzero_si512(); + __m512i R3 = _mm512_setzero_si512(); + __m512i R4 = _mm512_setzero_si512(); + __m512i R5 = _mm512_setzero_si512(); + __m512i R6 = _mm512_setzero_si512(); + __m512i R7 = _mm512_setzero_si512(); + + __m512i ZERO = _mm512_setzero_si512(); /* zeros */ + __m512i K = _mm512_set1_epi64((Ipp64s)k0); + + IPP_UNREFERENCED_PARAMETER(len); + IPP_UNREFERENCED_PARAMETER(res); + + __mmask8 k1 = (__mmask8)_mm512_kmov(0x01); /* mask of the 0 elment */ + int i; + for(i=0; i<60; i++) { + __m512i Bi = _mm512_set1_epi64((Ipp64s)b[i]); /* bloadcast(b[i]) */ + __m512i Yi = _mm512_setzero_si512(); /* Yi = 0 */ + __m512i tmp; + + R0 = _mm512_madd52lo_epu64(R0, A0, Bi); /* R += A*Bi (lo) */ + R1 = _mm512_madd52lo_epu64(R1, A1, Bi); + R2 = _mm512_madd52lo_epu64(R2, A2, Bi); + R3 = _mm512_madd52lo_epu64(R3, A3, Bi); + R4 = _mm512_madd52lo_epu64(R4, A4, Bi); + R5 = _mm512_madd52lo_epu64(R5, A5, Bi); + R6 = _mm512_madd52lo_epu64(R6, A6, Bi); + R7 = _mm512_madd52lo_epu64(R7, A7, Bi); + + Yi = _mm512_madd52lo_epu64(ZERO, K, R0); /* Yi = R0*K */ + Yi = _mm512_permutexvar_epi64(ZERO, Yi); /* broadcast Yi */ + + R0 = _mm512_madd52lo_epu64(R0, M0, Yi); /* R += M*Yi (lo) */ + R1 = _mm512_madd52lo_epu64(R1, M1, Yi); + R2 = _mm512_madd52lo_epu64(R2, M2, Yi); + R3 = _mm512_madd52lo_epu64(R3, M3, Yi); + R4 = _mm512_madd52lo_epu64(R4, M4, Yi); + R5 = _mm512_madd52lo_epu64(R5, M5, Yi); + R6 = _mm512_madd52lo_epu64(R6, M6, Yi); + R7 = _mm512_madd52lo_epu64(R7, M7, Yi); + + /* shift R */ + tmp = _mm512_maskz_srli_epi64(k1, R0, EXP_DIGIT_SIZE_AVX512); + R0 = _mm512_alignr_epi64(R1, R0, 1); + R1 = _mm512_alignr_epi64(R2, R1, 1); + R2 = _mm512_alignr_epi64(R3, R2, 1); + R3 = _mm512_alignr_epi64(R4, R3, 1); + R4 = _mm512_alignr_epi64(R5, R4, 1); + R5 = _mm512_alignr_epi64(R6, R5, 1); + R6 = _mm512_alignr_epi64(R7, R6, 1); + R7 = _mm512_alignr_epi64(ZERO, R7, 1); + R0 = _mm512_add_epi64(R0, tmp); + + R0 = _mm512_madd52hi_epu64(R0, A0, Bi); /* R += A*Bi (hi) */ + R1 = _mm512_madd52hi_epu64(R1, A1, Bi); + R2 = _mm512_madd52hi_epu64(R2, A2, Bi); + R3 = _mm512_madd52hi_epu64(R3, A3, Bi); + R4 = _mm512_madd52hi_epu64(R4, A4, Bi); + R5 = _mm512_madd52hi_epu64(R5, A5, Bi); + R6 = _mm512_madd52hi_epu64(R6, A6, Bi); + R7 = _mm512_madd52hi_epu64(R7, A7, Bi); + + R0 = _mm512_madd52hi_epu64(R0, M0, Yi); /* R += M*Yi (hi) */ + R1 = _mm512_madd52hi_epu64(R1, M1, Yi); + R2 = _mm512_madd52hi_epu64(R2, M2, Yi); + R3 = _mm512_madd52hi_epu64(R3, M3, Yi); + R4 = _mm512_madd52hi_epu64(R4, M4, Yi); + R5 = _mm512_madd52hi_epu64(R5, M5, Yi); + R6 = _mm512_madd52hi_epu64(R6, M6, Yi); + R7 = _mm512_madd52hi_epu64(R7, M7, Yi); + } + + /* store de-normilized result */ + _mm512_storeu_si512(out, R0); + _mm512_storeu_si512(out+NUM64, R1); + _mm512_storeu_si512(out+2*NUM64, R2); + _mm512_storeu_si512(out+3*NUM64, R3); + _mm512_storeu_si512(out+4*NUM64, R4); + _mm512_storeu_si512(out+5*NUM64, R5); + _mm512_storeu_si512(out+6*NUM64, R6); + _mm512_mask_storeu_epi64(out+7*NUM64, k2, R7); + + /* normalize result */ + { + Ipp64u acc = 0; + #if !defined(_MSC_VER) || defined(__INTEL_COMPILER) // unkonwn for msvc + #pragma nounroll + #endif + for(i=0; i<60; i++) { + acc += out[i]; + out[i] = acc & EXP_DIGIT_MASK_AVX512; + acc >>= EXP_DIGIT_SIZE_AVX512; + } + } +} + +static void AMM52x79(Ipp64u* out, const Ipp64u* a, const Ipp64u* b, const Ipp64u* m, Ipp64u k0, int len, Ipp64u* res) +{ + __mmask8 k2 = (__mmask8)_mm512_kmov(0x7f); /* mask of the 0-7 elments */ + + /* load a */ + __m512i A0 = _mm512_loadu_si512(a); + __m512i A1 = _mm512_loadu_si512(a+NUM64); + __m512i A2 = _mm512_loadu_si512(a+2*NUM64); + __m512i A3 = _mm512_loadu_si512(a+3*NUM64); + __m512i A4 = _mm512_loadu_si512(a+4*NUM64); + __m512i A5 = _mm512_loadu_si512(a+5*NUM64); + __m512i A6 = _mm512_loadu_si512(a+6*NUM64); + __m512i A7 = _mm512_loadu_si512(a+7*NUM64); + __m512i A8 = _mm512_loadu_si512(a+8*NUM64); + __m512i A9 = _mm512_maskz_loadu_epi64(k2, a+9*NUM64); + + /* load m */ + __m512i M0 = _mm512_loadu_si512(m); + __m512i M1 = _mm512_loadu_si512(m+NUM64); + __m512i M2 = _mm512_loadu_si512(m+2*NUM64); + __m512i M3 = _mm512_loadu_si512(m+3*NUM64); + __m512i M4 = _mm512_loadu_si512(m+4*NUM64); + __m512i M5 = _mm512_loadu_si512(m+5*NUM64); + __m512i M6 = _mm512_loadu_si512(m+6*NUM64); + __m512i M7 = _mm512_loadu_si512(m+7*NUM64); + __m512i M8 = _mm512_loadu_si512(m+8*NUM64); + __m512i M9 = _mm512_maskz_loadu_epi64(k2, m+9*NUM64); + + /* R0, R1, R2, R3, R4, R5, R6, R7, R8, R9 holds temporary result */ + __m512i R0 = _mm512_setzero_si512(); + __m512i R1 = _mm512_setzero_si512(); + __m512i R2 = _mm512_setzero_si512(); + __m512i R3 = _mm512_setzero_si512(); + __m512i R4 = _mm512_setzero_si512(); + __m512i R5 = _mm512_setzero_si512(); + __m512i R6 = _mm512_setzero_si512(); + __m512i R7 = _mm512_setzero_si512(); + __m512i R8 = _mm512_setzero_si512(); + __m512i R9 = _mm512_setzero_si512(); + + __m512i ZERO = _mm512_setzero_si512(); /* zeros */ + __m512i K = _mm512_set1_epi64((Ipp64s)k0); + + IPP_UNREFERENCED_PARAMETER(len); + IPP_UNREFERENCED_PARAMETER(res); + + __mmask8 k1 = (__mmask8)_mm512_kmov(0x01); /* mask of the 0 elment */ + int i; + for(i=0; i<79; i++) { + __m512i Bi = _mm512_set1_epi64((Ipp64s)b[i]); /* bloadcast(b[i]) */ + __m512i Yi = _mm512_setzero_si512(); /* Yi = 0 */ + __m512i tmp; + + R0 = _mm512_madd52lo_epu64(R0, A0, Bi); /* R += A*Bi (lo) */ + R1 = _mm512_madd52lo_epu64(R1, A1, Bi); + R2 = _mm512_madd52lo_epu64(R2, A2, Bi); + R3 = _mm512_madd52lo_epu64(R3, A3, Bi); + R4 = _mm512_madd52lo_epu64(R4, A4, Bi); + R5 = _mm512_madd52lo_epu64(R5, A5, Bi); + R6 = _mm512_madd52lo_epu64(R6, A6, Bi); + R7 = _mm512_madd52lo_epu64(R7, A7, Bi); + R8 = _mm512_madd52lo_epu64(R8, A8, Bi); + R9 = _mm512_madd52lo_epu64(R9, A9, Bi); + + Yi = _mm512_madd52lo_epu64(ZERO, K, R0); /* Yi = R0*K */ + Yi = _mm512_permutexvar_epi64(ZERO, Yi); /* broadcast Yi */ + + R0 = _mm512_madd52lo_epu64(R0, M0, Yi); /* R += M*Yi (lo) */ + R1 = _mm512_madd52lo_epu64(R1, M1, Yi); + R2 = _mm512_madd52lo_epu64(R2, M2, Yi); + R3 = _mm512_madd52lo_epu64(R3, M3, Yi); + R4 = _mm512_madd52lo_epu64(R4, M4, Yi); + R5 = _mm512_madd52lo_epu64(R5, M5, Yi); + R6 = _mm512_madd52lo_epu64(R6, M6, Yi); + R7 = _mm512_madd52lo_epu64(R7, M7, Yi); + R8 = _mm512_madd52lo_epu64(R8, M8, Yi); + R9 = _mm512_madd52lo_epu64(R9, M9, Yi); + + /* shift R */ + tmp = _mm512_maskz_srli_epi64(k1, R0, EXP_DIGIT_SIZE_AVX512); + R0 = _mm512_alignr_epi64(R1, R0, 1); + R1 = _mm512_alignr_epi64(R2, R1, 1); + R2 = _mm512_alignr_epi64(R3, R2, 1); + R3 = _mm512_alignr_epi64(R4, R3, 1); + R4 = _mm512_alignr_epi64(R5, R4, 1); + R5 = _mm512_alignr_epi64(R6, R5, 1); + R6 = _mm512_alignr_epi64(R7, R6, 1); + R7 = _mm512_alignr_epi64(R8, R7, 1); + R8 = _mm512_alignr_epi64(R9, R8, 1); + R9 = _mm512_alignr_epi64(ZERO, R9, 1); + R0 = _mm512_add_epi64(R0, tmp); + + R0 = _mm512_madd52hi_epu64(R0, A0, Bi); /* R += A*Bi (hi) */ + R1 = _mm512_madd52hi_epu64(R1, A1, Bi); + R2 = _mm512_madd52hi_epu64(R2, A2, Bi); + R3 = _mm512_madd52hi_epu64(R3, A3, Bi); + R4 = _mm512_madd52hi_epu64(R4, A4, Bi); + R5 = _mm512_madd52hi_epu64(R5, A5, Bi); + R6 = _mm512_madd52hi_epu64(R6, A6, Bi); + R7 = _mm512_madd52hi_epu64(R7, A7, Bi); + R8 = _mm512_madd52hi_epu64(R8, A8, Bi); + R9 = _mm512_madd52hi_epu64(R9, A9, Bi); + + R0 = _mm512_madd52hi_epu64(R0, M0, Yi); /* R += M*Yi (hi) */ + R1 = _mm512_madd52hi_epu64(R1, M1, Yi); + R2 = _mm512_madd52hi_epu64(R2, M2, Yi); + R3 = _mm512_madd52hi_epu64(R3, M3, Yi); + R4 = _mm512_madd52hi_epu64(R4, M4, Yi); + R5 = _mm512_madd52hi_epu64(R5, M5, Yi); + R6 = _mm512_madd52hi_epu64(R6, M6, Yi); + R7 = _mm512_madd52hi_epu64(R7, M7, Yi); + R8 = _mm512_madd52hi_epu64(R8, M8, Yi); + R9 = _mm512_madd52hi_epu64(R9, M9, Yi); + } + + /* store de-normilized result */ + _mm512_storeu_si512(out, R0); + _mm512_storeu_si512(out+NUM64, R1); + _mm512_storeu_si512(out+2*NUM64, R2); + _mm512_storeu_si512(out+3*NUM64, R3); + _mm512_storeu_si512(out+4*NUM64, R4); + _mm512_storeu_si512(out+5*NUM64, R5); + _mm512_storeu_si512(out+6*NUM64, R6); + _mm512_storeu_si512(out+7*NUM64, R7); + _mm512_storeu_si512(out+8*NUM64, R8); + _mm512_mask_storeu_epi64(out+9*NUM64, k2, R9); + + /* normalize result */ + { + Ipp64u acc = 0; + #if !defined(_MSC_VER) || defined(__INTEL_COMPILER) // unkonwn for msvc + #pragma nounroll + #endif + for(i=0; i<79; i++) { + acc += out[i]; + out[i] = acc & EXP_DIGIT_MASK_AVX512; + acc >>= EXP_DIGIT_SIZE_AVX512; + } + } +} + +/* ======= degugging section =========================================*/ +//#define _EXP_AVX512_DEBUG_ +#ifdef _EXP_AVX512_DEBUG_ +#include "pcpmontred.h" +void debugToConvMontDomain(BNU_CHUNK_T* pR, + const BNU_CHUNK_T* redInp, const BNU_CHUNK_T* redM, int almMM_bitsize, + const BNU_CHUNK_T* pM, const BNU_CHUNK_T* pRR, int nsM, BNU_CHUNK_T k0, + BNU_CHUNK_T* pBuffer) +{ + Ipp64u one[32] = { + 1,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0, 0,0 + }; + Ipp64u redT[32]; + int redLen = NUMBER_OF_DIGITS(almMM_bitsize, EXP_DIGIT_SIZE_AVX512); + AMM52(redT, redInp, one, redM, k0, redLen, pBuffer); + dig52_regular(pR, redT, almMM_bitsize); + + //cpMontMul_BNU(pR, should be changed + // redT, nsM, + // pRR, nsM, + // pM, nsM, k0, + // pBuffer, 0); + cpMul_BNU(pBuffer, pR,nsM, pRR,nsM, 0); + cpMontRed_BNU_opt(pR, pBuffer, pM, nsM, k0); +} +#endif +/* ===================================================================*/ + +IPP_OWN_DEFN (cpSize, gsMontExpBinBuffer_avx512, (int modulusBits)) +{ + cpSize redNum = numofVariable_avx512(modulusBits); /* "sizeof" variable */ + cpSize redBufferNum = numofVariableBuff_avx512(redNum,8); /* "sizeof" variable buffer */ + return redBufferNum *8; +} + +#if defined(_USE_WINDOW_EXP_) +IPP_OWN_DEFN (cpSize, gsMontExpWinBuffer_avx512, (int modulusBits)) +{ + cpSize w = gsMontExp_WinSize(modulusBits); + + cpSize redNum = numofVariable_avx512(modulusBits); /* "sizeof" variable */ + cpSize redBufferNum = numofVariableBuff_avx512(redNum,8); /* "sizeof" variable buffer */ + + cpSize bufferNum = CACHE_LINE_SIZE/(Ipp32s)sizeof(BNU_CHUNK_T) + + gsGetScrambleBufferSize(redNum, w) /* pre-computed table */ + + redBufferNum *7; /* addition 7 variables */ + return bufferNum; +} +#endif /* _USE_WINDOW_EXP_ */ + +/* +// "fast" binary montgomery exponentiation +// +// scratch buffer structure: +// redX[redBufferLen] +// redT[redBufferLen] +// redY[redBufferLen] +// redM[redBufferLen] +// redBuffer[redBufferLen*3] +*/ +IPP_OWN_DEFN (cpSize, gsMontExpBin_BNU_avx512, (BNU_CHUNK_T* dataY, const BNU_CHUNK_T* dataX, cpSize nsX, const BNU_CHUNK_T* dataE, cpSize bitsizeE, gsModEngine* pMont, BNU_CHUNK_T* pBuffer)) +{ + const BNU_CHUNK_T* dataM = MOD_MODULUS(pMont); + const BNU_CHUNK_T* dataRR= MOD_MNT_R2(pMont); + cpSize nsM = MOD_LEN(pMont); + BNU_CHUNK_T k0 = MOD_MNT_FACTOR(pMont); + + cpSize nsE = BITS_BNU_CHUNK(bitsizeE); + + int modulusBitSize = BITSIZE_BNU(dataM, nsM); + int cnvMM_bitsize = NUMBER_OF_DIGITS(modulusBitSize, BITSIZE(BNU_CHUNK_T)) * BITSIZE(BNU_CHUNK_T); + int almMM_bitsize = cnvMM_bitsize+2; + int redLen = NUMBER_OF_DIGITS(almMM_bitsize, EXP_DIGIT_SIZE_AVX512); + int redBufferLen = numofVariableBuff_avx512(redLen,8); + + /* allocate buffers */ + BNU_CHUNK_T* redX = pBuffer; + BNU_CHUNK_T* redT = redX+redBufferLen; + BNU_CHUNK_T* redY = redT+redBufferLen; + BNU_CHUNK_T* redM = redY+redBufferLen; + BNU_CHUNK_T* redBuffer = redM+redBufferLen; + + cpAMM52 ammFunc; + switch (modulusBitSize) { + case 1024: ammFunc = AMM52x20; break; + case 2048: ammFunc = AMM52x40; break; + case 3072: ammFunc = AMM52x60; break; + case 4096: ammFunc = AMM52x79; break; + default: ammFunc = AMM52; break; + } + + /* convert modulus into reduced domain */ + ZEXPAND_COPY_BNU(redBuffer, redBufferLen, dataM, nsM); + regular_dig52(redM, redBufferLen, redBuffer, almMM_bitsize); + + /* compute taget domain Montgomery converter RR' */ + ZEXPAND_BNU(redBuffer, 0, redBufferLen); + SET_BIT(redBuffer, (4*redLen*EXP_DIGIT_SIZE_AVX512- 4*cnvMM_bitsize)); + regular_dig52(redY, redBufferLen, redBuffer, almMM_bitsize); + + ZEXPAND_COPY_BNU(redBuffer, redBufferLen, dataRR, nsM); + regular_dig52(redT, redBufferLen, redBuffer, almMM_bitsize); + ammFunc(redT, redT, redT, redM, k0, redLen, redBuffer); + ammFunc(redT, redT, redY, redM, k0, redLen, redBuffer); + + /* convert base to Montgomery domain */ + ZEXPAND_COPY_BNU(redY, redBufferLen/*nsX+1*/, dataX, nsX); + regular_dig52(redX, redBufferLen, redY, almMM_bitsize); + ammFunc(redX, redX, redT, redM, k0, redLen, redBuffer); + + /* init result */ + COPY_BNU(redY, redX, redLen); + + FIX_BNU(dataE, nsE); + { + /* execute most significant part pE */ + BNU_CHUNK_T eValue = dataE[nsE-1]; + int n = cpNLZ_BNU(eValue)+1; + + eValue <<= n; + for(; n0; nsE--) { + eValue = dataE[nsE-1]; + + for(n=0; n0; nsE--) { + BNU_CHUNK_T eValue = dataE[nsE-1]; + + int n; + for(n=BNU_CHUNK_BITS; n>0; n--) { + /* T = ( msb(eValue) )? X : mont(1) */ + BNU_CHUNK_T mask = cpIsMsb_ct(eValue); + eValue <<= 1; + cpMaskedCopyBNU_ct(redT, mask, redX, redR, redLen); + + /* squaring: Y = Y*Y */ + ammFunc(redY, redY, redY, redM, k0, redLen, redBuffer); + /* and multiply: Y = Y * T */ + ammFunc(redY, redY, redT, redM, k0, redLen, redBuffer); + } + } + + /* convert result back to regular domain */ + ZEXPAND_BNU(redT, 0, redBufferLen); + redT[0] = 1; + ammFunc(redY, redY, redT, redM, k0, redLen, redBuffer); + dig52_regular(dataY, redY, cnvMM_bitsize); + + return nsM; +} + +#endif /* !_USE_WINDOW_EXP_ */ + + +#if defined(_USE_WINDOW_EXP_) +/* +// "fast" fixed-size window montgomery exponentiation +// +// scratch buffer structure: +// precomuted table of multipliers[(1< bitSizeE > 0), + it is checked in initialization phase by (ippsRSA_GetSizePublickey() and ippsRSA_InitPublicKey). + Buffer "redE" assigned for copy of dataE, is 1 (64-bit) chunk longer than size of RSA modulus, + therefore the access "*((Ipp32u*)((Ipp16u*)redE+ eBit/BITSIZE(Ipp16u)))" is always inside the boundary. + */ + /* extract 1-st window value */ + Ipp32u eChunk = *((Ipp32u*)((Ipp16u*)redE+ eBit/BITSIZE(Ipp16u))); + int shift = eBit & 0xF; + cpSize windowVal = (cpSize)((eChunk>>shift) &wmask); + + /* initialize result */ + ZEXPAND_COPY_BNU(redY, redBufferLen, redTable+windowVal*redLen, redLen); + + for(eBit-=window; eBit>=0; eBit-=window) { + /* do squaring window-times */ + for(n=0; n>shift) &wmask); + + /* extract precomputed value and muptiply */ + if(windowVal) { + ammFunc(redY, redY, redTable+windowVal*redLen, redM, k0, redLen, redBuffer); + } + } + } + + /* convert result back */ + ZEXPAND_BNU(redT, 0, redBufferLen); + redT[0] = 1; + ammFunc(redY, redY, redT, redM, k0, redLen, redBuffer); + dig52_regular(dataY, redY, cnvMM_bitsize); + + return nsM; +} + +/* +// "safe" fixed-size window montgomery exponentiation +// +// scratch buffer structure: +// precomuted table of multipliers[(1<>shift) &wmask); + + /* initialize result */ + gsScrambleGet_sscm(redY, redLen, redTable, windowVal, window); + #ifdef _EXP_AVX512_DEBUG_ + debugToConvMontDomain(dbgValue, redY, redM, almMM_bitsize, dataM, dataRR, nsM, k0, redBuffer); + #endif + + for(eBit-=window; eBit>=0; eBit-=window) { + /* do squaring window-times */ + for(n=0; n>shift) &wmask); + + /* exptact precomputed value and muptiply */ + gsScrambleGet_sscm(redT, redLen, redTable, windowVal, window); + /* muptiply */ + ammFunc(redY, redY, redT, redM, k0, redLen, redBuffer); + } + } + + /* convert result back */ + ZEXPAND_BNU(redT, 0, redBufferLen); + redT[0] = 1; + ammFunc(redY, redY, redT, redM, k0, redLen, redBuffer); + dig52_regular(dataY, redY, cnvMM_bitsize); + + return nsM; +} +#endif /* _USE_WINDOW_EXP_ */ + +#endif /* _IPP32E_K1 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_avx512.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_avx512.h new file mode 100644 index 0000000..97d2a2e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_avx512.h @@ -0,0 +1,171 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Internal Definitions of AVX512 Montgomery Exp +// +*/ +#include "owncp.h" + +#if (_IPP32E>=_IPP32E_K1) + +#include "pcpbnuimpl.h" +#include "pcpngmontexpstuff.h" + +#define RSA_AVX512_MIN_BITSIZE (1024) +#define RSA_AVX512_MAX_BITSIZE (13*1024) + +#define EXP_DIGIT_SIZE_AVX512 (52) +#define EXP_DIGIT_BASE_AVX512 (1<0; strLen--) { + digit <<= 8; + digit += (Ipp64u)(pStr[strLen-1]); + } + return digit; +} + +/* regular => redundant conversion */ +static void regular_dig52(Ipp64u* out, int outLen /* in qwords */, const Ipp64u* in, int inBitSize) +{ + Ipp8u* inStr = (Ipp8u*)in; + + for(; inBitSize>=(2*EXP_DIGIT_SIZE_AVX512); inBitSize-=(2*EXP_DIGIT_SIZE_AVX512), out+=2) { + out[0] = (*(Ipp64u*)inStr) & EXP_DIGIT_MASK_AVX512; + inStr += 6; + out[1] = ((*(Ipp64u*)inStr) >> 4) & EXP_DIGIT_MASK_AVX512; + inStr += 7; + outLen -= 2; + } + if(inBitSize>EXP_DIGIT_SIZE_AVX512) { + Ipp64u digit = getDig52(inStr, 7); + out[0] = digit & EXP_DIGIT_MASK_AVX512; + inStr += 6; + inBitSize -= EXP_DIGIT_SIZE_AVX512; + digit = getDig52(inStr, BITS2WORD8_SIZE(inBitSize)); + out[1] = digit>>4; + out += 2; + outLen -= 2; + } + else if(inBitSize>0) { + out[0] = getDig52(inStr, BITS2WORD8_SIZE(inBitSize)); + out++; + outLen--; + } + for(; outLen>0; outLen--,out++) out[0] = 0; +} + +/* + converts "redundant" (base = 2^DIGIT_SIZE) representation + into regular (base = 2^64) +*/ +__INLINE void putDig52(Ipp8u* pStr, int strLen, Ipp64u digit) +{ + for(; strLen>0; strLen--) { + *pStr++ = (Ipp8u)(digit&0xFF); + digit >>= 8; + } +} + +static void dig52_regular(Ipp64u* out, const Ipp64u* in, int outBitSize) +{ + int i; + int outLen = BITS2WORD64_SIZE(outBitSize); + for(i=0; i=(2*EXP_DIGIT_SIZE_AVX512); outBitSize-=(2*EXP_DIGIT_SIZE_AVX512), in+=2) { + (*(Ipp64u*)outStr) = in[0]; + outStr+=6; + (*(Ipp64u*)outStr) ^= in[1] << 4; + outStr+=7; + } + if(outBitSize>EXP_DIGIT_SIZE_AVX512) { + putDig52(outStr, 7, in[0]); + outStr+=6; + outBitSize -= EXP_DIGIT_SIZE_AVX512; + putDig52(outStr, BITS2WORD8_SIZE(outBitSize), (in[1]<<4 | in[0]>>48)); + } + else if(outBitSize) { + putDig52(outStr, BITS2WORD8_SIZE(outBitSize), in[0]); + } + } +} + +/* exponentiation buffer size */ +#define gsMontExpBinBuffer_avx512 OWNAPI(gsMontExpBinBuffer_avx512) + IPP_OWN_DECL (cpSize, gsMontExpBinBuffer_avx512, (int modulusBits)) +#define gsMontExpWinBuffer_avx512 OWNAPI(gsMontExpWinBuffer_avx512) + IPP_OWN_DECL (cpSize, gsMontExpWinBuffer_avx512, (int modulusBits)) + +/* exponentiations */ +#define gsMontExpBin_BNU_avx512 OWNAPI(gsMontExpBin_BNU_avx512) + IPP_OWN_DECL (cpSize, gsMontExpBin_BNU_avx512, (BNU_CHUNK_T* dataY, const BNU_CHUNK_T* dataX, cpSize nsX, const BNU_CHUNK_T* dataE, cpSize nsE, gsModEngine* pMont, BNU_CHUNK_T* pBuffer)) +#define gsMontExpWin_BNU_avx512 OWNAPI(gsMontExpWin_BNU_avx512) + IPP_OWN_DECL (cpSize, gsMontExpWin_BNU_avx512, (BNU_CHUNK_T* dataY, const BNU_CHUNK_T* dataX, cpSize nsX, const BNU_CHUNK_T* dataE, cpSize nsE, gsModEngine* pMont, BNU_CHUNK_T* pBuffer)) +#define gsMontExpBin_BNU_sscm_avx512 OWNAPI(gsMontExpBin_BNU_sscm_avx512) + IPP_OWN_DECL (cpSize, gsMontExpBin_BNU_sscm_avx512, (BNU_CHUNK_T* dataY, const BNU_CHUNK_T* dataX, cpSize nsX, const BNU_CHUNK_T* dataE, cpSize nsE, gsModEngine* pMont, BNU_CHUNK_T* pBuffer)) +#define gsMontExpWin_BNU_sscm_avx512 OWNAPI(gsMontExpWin_BNU_sscm_avx512) + IPP_OWN_DECL (cpSize, gsMontExpWin_BNU_sscm_avx512, (BNU_CHUNK_T* dataY, const BNU_CHUNK_T* dataX, cpSize nsX, const BNU_CHUNK_T* dataE, cpSize nsE, gsModEngine* pMont, BNU_CHUNK_T* pBuffer)) + +/* dual exponentiation buffer size */ +#define gsMontDualExpWinBuffer_avx512 OWNAPI(gsMontDualExpWinBuffer_avx512) + IPP_OWN_DECL (cpSize, gsMontDualExpWinBuffer_avx512, (int modulusBits)) + +/* dual exponentiation */ +#define gsMontDualExpWin_BNU_sscm_avx512 OWNAPI(gsMontDualExpWin_BNU_sscm_avx512) + IPP_OWN_DECL(cpSize, gsMontDualExpWin_BNU_sscm_avx512, (BNU_CHUNK_T* dataY[2], const BNU_CHUNK_T* dataX[2], cpSize nsX[2], const BNU_CHUNK_T* dataE[2], gsModEngine* pMont[2], BNU_CHUNK_T* pBuffer)) + +#endif /* _IPP32E_K1 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_bin.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_bin.c new file mode 100644 index 0000000..3266986 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_bin.c @@ -0,0 +1,122 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Modular Exponentiation +*/ + +#include "owncp.h" +#include "pcpngmontexpstuff.h" +#include "gsscramble.h" + +IPP_OWN_DEFN (cpSize, gsMontExpBinBuffer, (int modulusBits)) +{ + cpSize nsM = BITS_BNU_CHUNK(modulusBits); + cpSize bufferNum = nsM; + return bufferNum; +} + +IPP_OWN_DEFN (cpSize, gsMontExpBin_BNU, (BNU_CHUNK_T* dataY, const BNU_CHUNK_T* dataX, cpSize nsX, const BNU_CHUNK_T* dataE, cpSize bitsizeE, gsModEngine* pMont, BNU_CHUNK_T* pBuffer) ) +{ + cpSize nsM = MOD_LEN(pMont); + cpSize nsE = BITS_BNU_CHUNK(bitsizeE); + + /* + // test for special cases: + // x^0 = 1 + // 0^e = 0 + */ + if (cpEqu_BNU_CHUNK(dataE, nsE, 0)) { + COPY_BNU(dataY, MOD_MNT_R(pMont), nsM); + } + else if (cpEqu_BNU_CHUNK(dataX, nsX, 0)) { + ZEXPAND_BNU(dataY, 0, nsM); + } + + /* general case */ + else { + /* allocate buffers */ + BNU_CHUNK_T* dataT = pBuffer; + + /* copy and expand base to the modulus length */ + ZEXPAND_COPY_BNU(dataT, nsM, dataX, nsX); + /* copy */ + COPY_BNU(dataY, dataT, nsM); + + FIX_BNU(dataE, nsE); + + /* execute most significant part pE */ + { + BNU_CHUNK_T eValue = dataE[nsE - 1]; + int n = cpNLZ_BNU(eValue) + 1; + + eValue <<= n; + for (; nsqr(dataY, dataY, pMont); + /* and multiply R = R*X mod Modulus */ + if (eValue & ((BNU_CHUNK_T)1 << (BNU_CHUNK_BITS - 1))) + MOD_METHOD(pMont)->mul(dataY, dataY, dataT, pMont); + } + + /* execute rest bits of E */ + for (--nsE; nsE>0; nsE--) { + eValue = dataE[nsE - 1]; + + for (n = 0; nsqr(dataY, dataY, pMont); + + if (eValue & ((BNU_CHUNK_T)1 << (BNU_CHUNK_BITS - 1))) + MOD_METHOD(pMont)->mul(dataY, dataY, dataT, pMont); + } + } + } + } + + return nsM; +} + +/* +// "fast" binary montgomery exponentiation +// +// - input/output are in Regular Domain +// - possible inplace mode +// +// scratch buffer structure: +// dataT[nsM] copy of base (in case of inplace operation) +*/ +IPP_OWN_DEFN (cpSize, gsModExpBin_BNU, (BNU_CHUNK_T* dataY, const BNU_CHUNK_T* dataX, cpSize nsX, const BNU_CHUNK_T* dataE, cpSize bitsizeE, gsModEngine* pMont, BNU_CHUNK_T* pBuffer)) +{ + cpSize nsM = MOD_LEN(pMont); + + /* copy and expand base to the modulus length */ + ZEXPAND_COPY_BNU(dataY, nsM, dataX, nsX); + /* convert base to Montgomery domain */ + MOD_METHOD(pMont)->encode(dataY, dataY, pMont); + + /* exponentiation */ + gsMontExpBin_BNU(dataY, dataY, nsM, dataE, bitsizeE, pMont, pBuffer); + + /* convert result back to regular domain */ + MOD_METHOD(pMont)->decode(dataY, dataY, pMont); + + return nsM; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_bin_sscm.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_bin_sscm.c new file mode 100644 index 0000000..4342912 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_bin_sscm.c @@ -0,0 +1,112 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Modular Exponentiation +*/ + +#include "owncp.h" +#include "pcpngmontexpstuff.h" +#include "gsscramble.h" +#include "pcpmask_ct.h" + +IPP_OWN_DEFN (cpSize, gsMontExpBin_BNU_sscm, (BNU_CHUNK_T* dataY, const BNU_CHUNK_T* dataX, cpSize nsX, const BNU_CHUNK_T* dataE, cpSize bitsizeE, gsModEngine* pMont, BNU_CHUNK_T* pBuffer)) +{ + + cpSize nsM = MOD_LEN(pMont); + cpSize nsE = BITS_BNU_CHUNK(bitsizeE); + + /* + // test for special cases: + // x^0 = 1 + // 0^e = 0 + */ + if (cpEqu_BNU_CHUNK(dataE, nsE, 0)) { + COPY_BNU(dataY, MOD_MNT_R(pMont), nsM); + } + else if (cpEqu_BNU_CHUNK(dataX, nsX, 0)) { + ZEXPAND_BNU(dataY, 0, nsM); + } + + /* general case */ + else { + + /* allocate buffers */ + BNU_CHUNK_T* dataT = pBuffer; + BNU_CHUNK_T* sscmB = dataT + nsM; + + /* mont(1) */ + BNU_CHUNK_T* pR = MOD_MNT_R(pMont); + + /* copy and expand base to the modulus length */ + ZEXPAND_COPY_BNU(dataT, nsM, dataX, nsX); + /* init result */ + COPY_BNU(dataY, MOD_MNT_R(pMont), nsM); + + /* execute bits of E */ + for (; nsE>0; nsE--) { + BNU_CHUNK_T eValue = dataE[nsE-1]; + + int n; + for(n=BNU_CHUNK_BITS; n>0; n--) { + /* sscmB = ( msb(eValue) )? X : mont(1) */ + BNU_CHUNK_T mask = cpIsMsb_ct(eValue); + eValue <<= 1; + cpMaskedCopyBNU_ct(sscmB, mask, dataT, pR, nsM); + + /* squaring Y = Y^2 */ + MOD_METHOD(pMont)->sqr(dataY, dataY, pMont); + /* and multiplication: Y = Y * sscmB */ + MOD_METHOD(pMont)->mul(dataY, dataY, sscmB, pMont); + } + } + } + + return nsM; +} + +/* +// "safe" binary montgomery exponentiation +// +// - input/output are in Regular Domain +// - possible inplace mode +// +// scratch buffer structure: +// dataT[nsM] +// sscm[nsM] +*/ +IPP_OWN_DEFN (cpSize, gsModExpBin_BNU_sscm, (BNU_CHUNK_T* dataY, const BNU_CHUNK_T* dataX, cpSize nsX, const BNU_CHUNK_T* dataE, cpSize bitsizeE, gsModEngine* pMont, BNU_CHUNK_T* pBuffer)) +{ + cpSize nsM = MOD_LEN(pMont); + + /* copy and expand base to the modulus length */ + ZEXPAND_COPY_BNU(dataY, nsM, dataX, nsX); + + /* convert base to Montgomery domain */ + MOD_METHOD(pMont)->encode(dataY, dataY, pMont); + + /* exponentiation */ + gsMontExpBin_BNU_sscm(dataY, dataY, nsM, dataE, bitsizeE, pMont, pBuffer); + + /* convert result back to regular domain */ + MOD_METHOD(pMont)->decode(dataY, dataY, pMont); + + return nsM; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_sse2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_sse2.c new file mode 100644 index 0000000..7b17a78 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_sse2.c @@ -0,0 +1,879 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "owncp.h" + +#if (_IPP>=_IPP_W7) + +#include "pcpbnuimpl.h" +#include "pcpbnumisc.h" + +#include "pcpngmontexpstuff.h" +#include "pcpngmontexpstuff_sse2.h" +#include "gsscramble.h" +#include "pcpmask_ct.h" + + +//tbcd: temporary excluded: #include +/* + converts regular (base = 2^32) representation (norm, normLen) + into "redundant" (base = 2^EXP_DIGIT_SIZE_SSE2) represenrartion (red, redLen) + + return 1 + + note: + 1) repLen >= (bitsize +DIGIT_SIZE-1)/DIGIT_SIZE for complete conversion + 2) regular representation should expanded by at least one zero value, + pre-requisite: pRegular[regLen] == 0 to make conversion correct + 3) caller must provide suitable lengths of regular and redundant respresentations + so, conversion does correct +*/ +static int regular_dig27(Ipp64u* pRep27, int repLen, const Ipp32u* pRegular, int regLen) +{ + /* expected number of digit in redundant representation */ + int n = cpDigitNum_sse2(regLen*BITSIZE(Ipp32u), EXP_DIGIT_SIZE_SSE2); + + //tbcd: temporary excluded: assert(pRegular[regLen]==0); /* test pre-requisite */ + + { + int redBit; /* output representatin bit */ + int i; + for(i=0, redBit=0; i>= shift; + pRep27[i] = x & EXP_DIGIT_MASK_SSE2; + } + + /* expands by zeros if necessary */ + for(; i= NORM_DIGSIZE_SSE2) { + pRegular[idx++] = (Ipp32u)(x & NORM_MASK_SSE2); + x >>= NORM_DIGSIZE_SSE2; + shift -= NORM_DIGSIZE_SSE2; + } + } + + if(idx>= EXP_DIGIT_SIZE_SSE2; + } + return tmp; +} + +/* + Montgomery multiplication of the redundant operands +*/ +static void cpMontMul_sse2(Ipp64u* pR, const Ipp64u* pA, const Ipp64u* pB, const Ipp64u* pModulus, int mLen, BNU_CHUNK_T k0, Ipp64u* pBuffer) +{ + int extLen = mLen+1; /* len extension */ + int i; + + __m128i v_k0 = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)k0), 0x44); + __m128i v_digMask = _mm_srli_epi64(_mm_set1_epi32(-1), 64-EXP_DIGIT_SIZE_SSE2); + __m128i lowQword = _mm_srli_si128(_mm_set1_epi32(-1), sizeof(Ipp64u)); + __m128i v_b0, v_y0, v_ac; + + /* clear buffer */ + v_ac = _mm_setzero_si128(); + for(i=0; i1; pB+=2, mLen-=2) { + __m128i v_b1, v_y1; + + /* v_b0 = {pB[i]:pB[i]}, v_b1 = {pB[i+1]:pB[i+1]} */ + v_b1 = _mm_load_si128((__m128i*)pB); + v_b0 = _mm_shuffle_epi32(v_b1, 0x44); + /* v_ac = {pA[1]:pA[0]} */ + v_ac = _mm_load_si128((__m128i*)pA); + + v_b1 = _mm_shuffle_epi32(v_b1, 0xEE); + + /* v_ac = {pA[1]:pA[0]} * {pB[i]:pB[i]} + {pR[1]:pR[0]} */ + v_ac = _mm_add_epi64(_mm_mul_epu32(v_ac, v_b0), ((__m128i*)pBuffer)[0]); + /* v_y0 = {v_ac[0]:v_ac[0]} * {k0:k0} ) & {digMask:digMask} */ + v_y0 = _mm_mul_epu32(_mm_shuffle_epi32(v_ac, 0x44),v_k0); + v_y0 = _mm_and_si128(v_y0, v_digMask); + /* v_ac += v_y0 * {pM[1]:pM[0]}*/ + v_ac = _mm_add_epi64(v_ac, _mm_mul_epu32(v_y0, ((__m128i*)pModulus)[0])); + + /* v_ac = {0: v_ac[1]+v_ac[0]>>EXP_DIGIT_SIZE_SSE2}*/ + v_ac = _mm_add_epi64(_mm_srli_si128(v_ac, sizeof(Ipp64u)), + _mm_srli_epi64(_mm_and_si128(v_ac, lowQword), EXP_DIGIT_SIZE_SSE2)); + /* v_ac += {0: pA[0]} * v_b1 */ + v_ac = _mm_add_epi64(v_ac, _mm_mul_epu32(v_b1, _mm_cvtsi64_si128((IPP_INT64)pA[0]))); + + /* v_y1 = (v_ac * v_k0) & v_digMask */ + v_y1 = _mm_and_si128(_mm_mul_epu32(v_ac, v_k0), v_digMask); + + /* v_ac += pM[0] * v_y1 */ + v_ac = _mm_add_epi64(v_ac, _mm_mul_epu32(_mm_cvtsi64_si128((IPP_INT64)pModulus[0]), v_y1)); + + v_y1 = _mm_shuffle_epi32(v_y1, 0x44); + + /* pR[2] += (v_ac>>EXP_DIGIT_SIZE_SSE2) */ + v_ac = _mm_add_epi64(_mm_srli_epi64(v_ac, EXP_DIGIT_SIZE_SSE2), ((__m128i*)pBuffer)[1]); + _mm_store_si128(((__m128i*)pBuffer)+1, v_ac); + + for(i=2; i1; pProduct+=2, mLen-=2) { + __m128i v_y1; + + /* v_ac = {pProd[1]:pProd[0]} */ + v_ac = _mm_load_si128((__m128i*)pProduct); + + /* v_y0 = {v_ac[0]:v_ac[0]} * {k0:k0} ) & {digMask:digMask} */ + v_y0 = _mm_mul_epu32(_mm_shuffle_epi32(v_ac, 0x44),v_k0); + v_y0 = _mm_and_si128(v_y0, v_digMask); + + /* v_ac += v_y0 * {pMod[1]:pMod[0]}*/ + v_ac = _mm_add_epi64(v_ac, _mm_mul_epu32(v_y0, ((__m128i*)pModulus)[0])); + + /* v_ac = {0: v_ac[1]+v_ac[0]>>EXP_DIGIT_SIZE_SSE2}*/ + v_ac = _mm_add_epi64(_mm_srli_si128(v_ac, sizeof(Ipp64u)), + _mm_srli_epi64(_mm_and_si128(v_ac, lowQword), EXP_DIGIT_SIZE_SSE2)); + + /* v_y1 = (v_ac * v_k0) & v_digMask */ + v_y1 = _mm_and_si128(_mm_mul_epu32(v_ac, v_k0), v_digMask); + + /* v_ac += pM[0] * v_y1 */ + v_ac = _mm_add_epi64(v_ac, _mm_mul_epu32(_mm_cvtsi64_si128((IPP_INT64)pModulus[0]), v_y1)); + + v_y1 = _mm_shuffle_epi32(v_y1, 0x44); + + /* pProd[2] += (v_ac>>EXP_DIGIT_SIZE_SSE2) */ + v_ac = _mm_add_epi64(_mm_srli_epi64(v_ac, EXP_DIGIT_SIZE_SSE2), ((__m128i*)pProduct)[1]); + _mm_store_si128(((__m128i*)pProduct)+1, v_ac); + + for(i=2; i>EXP_DIGIT_SIZE_SSE2 */ + v_ac = _mm_add_epi64(v_ac, _mm_srli_epi64(_mm_shuffle_epi32(v_ac, 0x44), EXP_DIGIT_SIZE_SSE2)); + _mm_store_si128(((__m128i*)pProduct), v_ac); + + for(i=2; i1; i+=4,hcounter-=2, tmpa+=4,ps+=4) { + __m128i t; + __m128i a0 = _mm_shuffle_epi32(_mm_cvtsi64_si128 ((IPP_INT64)tmpa[0]), 0x44); /* {a0:a0} */ + __m128i a2 = _mm_shuffle_epi32(_mm_cvtsi64_si128 ((IPP_INT64)tmpa[2]), 0x44); /* {a2:a2} */ + + __m128i s0 = _mm_load_si128((__m128i*)ps); + __m128i s2 = _mm_load_si128((__m128i*)(ps+2)); + + __m128i d0 = _mm_load_si128((__m128i*)(pDbl+i+2)); + __m128i d2 = _mm_load_si128((__m128i*)(pDbl+i+4)); + + /* px[2*i] += a0*s00; px[2*i+1] += a0*s01; */ + _mm_storeu_si128((__m128i*)(px+2*i), _mm_add_epi64(_mm_mul_epu32(a0,s0), _mm_loadu_si128((__m128i*)(px+2*i)))); + + /* px[2*i+2] += a0*d00; px[2*i+3] += a0*d01; */ + _mm_storeu_si128((__m128i*)(px+2*i+2), _mm_add_epi64(_mm_mul_epu32(a0,d0), _mm_loadu_si128((__m128i*)(px+2*i+2)))); + + /* px[2*i+4] += a0*d20 + a2*s20; px[2*i+5] += a0*d21 + a2*s21; */ + t = _mm_add_epi64(_mm_mul_epu32(a0,d2), _mm_mul_epu32(a2,s2)); + _mm_storeu_si128((__m128i*)(px+2*i+4), _mm_add_epi64(t, _mm_loadu_si128((__m128i*)(px+2*i+4)))); + + for(j=i+6; j0; nsE--) { + eValue = dataE[nsE-1]; + + for(n=0; n0; nsE--) { + BNU_CHUNK_T eValue = dataE[nsE-1]; + + + int n; + for(n=BNU_CHUNK_BITS; n>0; n--) { + /* T = ( msb(eValue) )? X : mont(1) */ + BNU_CHUNK_T mask = cpIsMsb_ct(eValue); + eValue <<= 1; + cpMaskedCopyBNU_ct((BNU_CHUNK_T*)redT, mask, (BNU_CHUNK_T*)redX, (BNU_CHUNK_T*)redR, redLen*sizeof(Ipp64u)/sizeof(BNU_CHUNK_T)); + + /* squaring: Y = Y^2 */ + cpMontSqr_sse2(redY, redY, redM, redLen, k0, redBuffer); + /* and multiply: Y = Y * T */ + cpMontMul_sse2(redY, redY, redT, redM, redLen, k0, redBuffer); + } + } + + /* convert result back to regular domain */ + ZEXPAND_BNU(redT, 0, redBufferLen); + redT[0] = 1; + cpMontMul_sse2(redY, redY, redT, redM, redLen, k0, redBuffer); + dig27_regular((Ipp32u*)dataY, nsM*sizeof(BNU_CHUNK_T)/sizeof(ipp32u), redY, redLen); + + return nsM; +} +#endif /* !_USE_WINDOW_EXP_ */ + + +#if defined(_USE_WINDOW_EXP_) +/* +// "fast" fixed-size window montgomery exponentiation +// +// scratch buffer structure: +// precomuted table of multipliers[(1<>shift) &wmask); + + /* initialize result */ + COPY_BNU(redY, redTable+windowVal*redLen, redLen); + + for(eBit-=window; eBit>=0; eBit-=window) { + /* do squaring window-times */ + for(n=0; n>shift) &wmask); + + /* precomputed value muptiplication */ + if(windowVal) { + COPY_BNU(redT, redTable+windowVal*redLen, redLen); + cpMontMul_sse2(redY, redY, redT, redM, redLen, k0, redBuffer); + } + } + } + + /* convert result back */ + ZEXPAND_BNU(redT, 0, redBufferLen); + redT[0] = 1; + cpMontMul_sse2(redY, redY, redT, redM, redLen, k0, redBuffer); + dig27_regular((Ipp32u*)dataY, nsM*(Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(ipp32u)), redY, redLen); + + return nsM; +} + +/* +// "safe" fixed-size window montgomery exponentiation +// +// scratch buffer structure: +// pre-computed table +// redM[redBufferLen] +// redY[redBufferLen] +// redT[redBufferLen] +// redBuffer[redBufferLen*3] +// redE[redBufferLen] +*/ +IPP_OWN_DEFN (cpSize, gsMontExpWin_BNU_sscm_sse2, (BNU_CHUNK_T* dataY, const BNU_CHUNK_T* dataX, cpSize nsX, const BNU_CHUNK_T* dataE, cpSize bitsizeE, gsModEngine* pMont, BNU_CHUNK_T* pBuffer)) +{ + const BNU_CHUNK_T* dataM = MOD_MODULUS(pMont); + const BNU_CHUNK_T* dataRR= MOD_MNT_R2(pMont); + cpSize nsM = MOD_LEN(pMont); + BNU_CHUNK_T k0 = MOD_MNT_FACTOR(pMont); + + cpSize nsE = BITS_BNU_CHUNK(bitsizeE); + + int modulusBitSize = MOD_BITSIZE(pMont); + int convModulusBitSize = cpDigitNum_sse2(modulusBitSize, BITSIZE(BNU_CHUNK_T)) * BITSIZE(BNU_CHUNK_T); + int modulusLen32 = BITS2WORD32_SIZE(modulusBitSize); + int redLen = cpDigitNum_sse2(convModulusBitSize+2, EXP_DIGIT_SIZE_SSE2); + int redBufferLen = numofVariableBuff_sse2(redLen); + + cpSize window = gsMontExp_WinSize(bitsizeE); + cpSize nPrecomute= 1<>shift) &wmask); + + /* initialize result */ + gsScrambleGet_sscm((BNU_CHUNK_T*)redY, redLen*(Ipp32s)(sizeof(Ipp64u)/sizeof(BNU_CHUNK_T)), (BNU_CHUNK_T*)redTable, windowVal, window); + + for(eBit-=window; eBit>=0; eBit-=window) { + /* do squaring window-times */ + for(n=0; n>shift) &wmask); + /* exptact precomputed value and muptiply */ + gsScrambleGet_sscm((BNU_CHUNK_T*)redT, redLen*(Ipp32s)(sizeof(Ipp64u)/sizeof(BNU_CHUNK_T)), (BNU_CHUNK_T*)redTable, windowVal, window); + cpMontMul_sse2(redY, redY, redT, redM, redLen, k0, redBuffer); + } + } + + /* convert result back */ + ZEXPAND_BNU(redT, 0, redBufferLen); + redT[0] = 1; + cpMontMul_sse2(redY, redY, redT, redM, redLen, k0, redBuffer); + dig27_regular((Ipp32u*)dataY, nsM*(Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(ipp32u)), redY, redLen); + + return nsM; +} +#endif /* _USE_WINDOW_EXP_ */ + +#endif /* _IPP_W7 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_sse2.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_sse2.h new file mode 100644 index 0000000..c1e6d41 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_sse2.h @@ -0,0 +1,78 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Internal Definitions of SSE Montgomery Exp +// +*/ +#include "owncp.h" + +#if (_IPP>=_IPP_W7) + +#include "pcpbnuimpl.h" +#include "pcpngmontexpstuff.h" + +#define RSA_SSE2_MIN_BITSIZE (256) +#define RSA_SSE2_MAX_BITSIZE (13*1024) + +#define NORM_DIGSIZE_SSE2 (BITSIZE(Ipp32u)) +#define NORM_BASE_SSE2 ((Ipp64u)1<mul(pTable+n*nsM, pTable+(n-1)*nsM, dataTT, pMont); + } + + /* expand exponent*/ + ZEXPAND_COPY_BNU(dataEE, nsE+1, dataExp, nsE); + bitsizeE = ((bitsizeE+winSize-1)/winSize) *winSize; + + /* exponentiation */ + { + /* position of the 1-st (left) window */ + int eBit = bitsizeE-winSize; + + /* Note: Static analysis can generate error/warning on the expression below. + + The value of "bitSizeE" is limited, ((modulusBitSize > bitSizeE > 0), + it is checked in initialization phase by (ippsRSA_GetSizePublickey() and ippsRSA_InitPublicKey). + Buffer "dataEE" assigned for copy of dataExp, is 1 (64-bit) chunk longer than size of RSA modulus, + therefore the access "*((Ipp32u*)((Ipp16u*)dataEE+ eBit/BITSIZE(Ipp16u)))" is always inside the boundary. + */ + /* extract 1-st window value */ + Ipp32u eChunk = *((Ipp32u*)((Ipp16u*)dataEE + eBit/BITSIZE(Ipp16u))); + int shift = eBit & 0xF; + Ipp32u winVal = (eChunk>>shift) &mask; + + /* initialize result */ + COPY_BNU(dataY, pTable+winVal*(Ipp32u)nsM, nsM); + + for(eBit-=winSize; eBit>=0; eBit-=winSize) { + /* do square window times */ + for(n=0,winVal=0; nsqr(dataY, dataY, pMont); + } + + /* extract next window value */ + eChunk = *((Ipp32u*)((Ipp16u*)dataEE + eBit/BITSIZE(Ipp16u))); + shift = eBit & 0xF; + winVal = (eChunk>>shift) &mask; + + /* muptiply precomputed value */ + MOD_METHOD( pMont )->mul(dataY, dataY, pTable+winVal*(Ipp32u)nsM, pMont); + } + + } + } + + return nsM; +} + +/* +// "fast" fixed-size window montgomery exponentiation +// - input/output are in Regular Domain +// - possible inplace mode +*/ +IPP_OWN_DEFN (cpSize, gsModExpWin_BNU, (BNU_CHUNK_T* dataY, const BNU_CHUNK_T* dataX, cpSize nsX, const BNU_CHUNK_T* dataExp, cpSize bitsizeE, gsModEngine* pMont, BNU_CHUNK_T* pBuffer)) +{ + cpSize nsM = MOD_LEN(pMont); + + /* copy and expand base to the modulus length */ + ZEXPAND_COPY_BNU(dataY, nsM, dataX, nsX); + + /* convert base to Montgomery domain */ + MOD_METHOD(pMont)->encode(dataY, dataY, pMont); + + /* exponentiation */ + gsMontExpWin_BNU(dataY, dataY, nsM, dataExp, bitsizeE, pMont, pBuffer); + + /* convert result back to regular domain */ + MOD_METHOD(pMont)->decode(dataY, dataY, pMont); + + return nsM; +} + +#endif /* _USE_WINDOW_EXP_ */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_win_sscm.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_win_sscm.c new file mode 100644 index 0000000..42021c1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngmontexpstuff_win_sscm.c @@ -0,0 +1,149 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Modular Exponentiation (windowed "safe" version) +*/ + +#include "owncp.h" + +#if defined(_USE_WINDOW_EXP_) + +#include "pcpngmontexpstuff.h" +#include "gsscramble.h" +#include "pcpmask_ct.h" + +/* +// "safe" fixed-size window montgomery exponentiation +// +// - input/output are in Montgomery Domain +// - possible inplace mode +// +// scratch buffer structure: +// precomuted table of multipliers[(1<mul(dataTT, dataTT, dataRR, pMont); + gsScramblePut(pTable, n, dataTT, nsM, winSize); + } + + /* expand exponent*/ + ZEXPAND_COPY_BNU(dataEE, nsM+1, dataExp, nsE); + bitsizeE = ((bitsizeE+winSize-1)/winSize) *winSize; + + /* exponentiation */ + { + /* position of the 1-st (left) window */ + int eBit = bitsizeE-winSize; + + /* extract 1-st window value */ + Ipp32u eChunk = *((Ipp32u*)((Ipp16u*)dataEE + eBit/BITSIZE(Ipp16u))); + int shift = eBit & 0xF; + Ipp32u winVal = (eChunk>>shift) &mask; + + /* initialize result */ + gsScrambleGet_sscm(dataY, nsM, pTable, (int)winVal, winSize); + + for(eBit-=winSize; eBit>=0; eBit-=winSize) { + /* do square window times */ + for(n=0,winVal=0; nsqr(dataY, dataY, pMont); + } + + /* extract next window value */ + eChunk = *((Ipp32u*)((Ipp16u*)dataEE + eBit/BITSIZE(Ipp16u))); + shift = eBit & 0xF; + winVal = (eChunk>>shift) &mask; + + /* exptact precomputed value and muptiply */ + gsScrambleGet_sscm(dataTT, nsM, pTable, (int)winVal, winSize); + + MOD_METHOD( pMont )->mul(dataY, dataY, dataTT, pMont); + } + } + } + + return nsM; +} + +/* +// "safe" fixed-size window exponentiation +// - input/output are in Regular Domain +// - possible inplace mode +*/ +IPP_OWN_DEFN (cpSize, gsModExpWin_BNU_sscm, (BNU_CHUNK_T* dataY, const BNU_CHUNK_T* dataX, cpSize nsX, const BNU_CHUNK_T* dataExp, cpSize bitsizeE, gsModEngine* pMont, BNU_CHUNK_T* pBuffer)) +{ + cpSize nsM = MOD_LEN(pMont); + + /* copy and expand base to the modulus length */ + ZEXPAND_COPY_BNU(dataY, nsM, dataX, nsX); + + /* convert base to Montgomery domain */ + MOD_METHOD(pMont)->encode(dataY, dataY, pMont); + + /* exponentiation */ + gsMontExpWin_BNU_sscm(dataY, dataY, nsM, dataExp, bitsizeE, pMont, pBuffer); + + /* convert result back to regular domain */ + MOD_METHOD(pMont)->decode(dataY, dataY, pMont); + + return nsM; +} + +#endif /* _USE_WINDOW_EXP_ */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngrsa.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngrsa.h new file mode 100644 index 0000000..b15a08a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngrsa.h @@ -0,0 +1,123 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Internal Definitions and +// Internal ng RSA Function Prototypes +// +// +*/ + +#if !defined(_CP_NG_RSA_H) +#define _CP_NG_RSA_H + +#include "pcpbn.h" +#include "pcpmontgomery.h" +#include "pcpngmontexpstuff.h" + +struct _cpRSA_public_key { + Ipp32u id; /* key ID */ + int maxbitSizeN; + int maxbitSizeE; + int bitSizeN; /* RSA modulus bitsize */ + int bitSizeE; /* RSA public exp bitsize */ + + BNU_CHUNK_T* pDataE; /* public exp */ + gsModEngine* pMontN; /* montgomery engine (N) */ +}; + +#define RSA_PUB_KEY_SET_ID(x) ((x)->id = (Ipp32u)idCtxRSA_PubKey ^ (Ipp32u)IPP_UINT_PTR(x)) +/* access */ +#define RSA_PUB_KEY_MAXSIZE_N(x) ((x)->maxbitSizeN) +#define RSA_PUB_KEY_MAXSIZE_E(x) ((x)->maxbitSizeE) +#define RSA_PUB_KEY_BITSIZE_N(x) ((x)->bitSizeN) +#define RSA_PUB_KEY_BITSIZE_E(x) ((x)->bitSizeE) +#define RSA_PUB_KEY_E(x) ((x)->pDataE) +#define RSA_PUB_KEY_NMONT(x) ((x)->pMontN) + +#define RSA_PUB_KEY_VALID_ID(x) ((((x)->id) ^ (Ipp32u)IPP_UINT_PTR((x))) == (Ipp32u)idCtxRSA_PubKey) +#define RSA_PUB_KEY_IS_SET(x) (RSA_PUB_KEY_BITSIZE_N((x))>0) + +/* alignment */ +#define RSA_PUBLIC_KEY_ALIGNMENT ((int)(sizeof(void*))) + +struct _cpRSA_private_key { + Ipp32u id; /* key ID */ + int maxbitSizeN; + int maxbitSizeD; + int bitSizeN; /* RSA modulus bitsize */ + int bitSizeD; /* RSA private exp bitsize */ + int bitSizeP; /* RSA p-factor bitsize */ + int bitSizeQ; /* RSA q-factor bitsize */ + + BNU_CHUNK_T* pDataD; /* private exp */ + BNU_CHUNK_T* pDataDp; /* dp private exp */ + BNU_CHUNK_T* pDataDq; /* dq private exp */ + BNU_CHUNK_T* pDataQinv; /* qinv coeff in residue domain */ + + gsModEngine* pMontP; /* montgomery engine (P) */ + gsModEngine* pMontQ; /* montgomery engine (Q) */ + gsModEngine* pMontN; /* montgomery engine (N) */ +}; + +#define RSA_PRV_KEY1_SET_ID(x) ((x)->id = (Ipp32u)idCtxRSA_PrvKey1 ^ (Ipp32u)IPP_UINT_PTR(x)) +#define RSA_PRV_KEY2_SET_ID(x) ((x)->id = (Ipp32u)idCtxRSA_PrvKey2 ^ (Ipp32u)IPP_UINT_PTR(x)) +/* access */ +#define RSA_PRV_KEY_MAXSIZE_N(x) ((x)->maxbitSizeN) +#define RSA_PRV_KEY_MAXSIZE_D(x) ((x)->maxbitSizeD) +#define RSA_PRV_KEY_BITSIZE_N(x) ((x)->bitSizeN) +#define RSA_PRV_KEY_BITSIZE_D(x) ((x)->bitSizeD) +#define RSA_PRV_KEY_BITSIZE_P(x) ((x)->bitSizeP) +#define RSA_PRV_KEY_BITSIZE_Q(x) ((x)->bitSizeQ) +#define RSA_PRV_KEY_D(x) ((x)->pDataD) +#define RSA_PRV_KEY_DP(x) ((x)->pDataDp) +#define RSA_PRV_KEY_DQ(x) ((x)->pDataDq) +#define RSA_PRV_KEY_INVQ(x) ((x)->pDataQinv) +#define RSA_PRV_KEY_PMONT(x) ((x)->pMontP) +#define RSA_PRV_KEY_QMONT(x) ((x)->pMontQ) +#define RSA_PRV_KEY_NMONT(x) ((x)->pMontN) + +#define RSA_PRV_KEY1_VALID_ID(x) ((((x)->id) ^ (Ipp32u)IPP_UINT_PTR((x))) == (Ipp32u)idCtxRSA_PrvKey1) +#define RSA_PRV_KEY2_VALID_ID(x) ((((x)->id) ^ (Ipp32u)IPP_UINT_PTR((x))) == (Ipp32u)idCtxRSA_PrvKey2) +#define RSA_PRV_KEY_VALID_ID(x) (RSA_PRV_KEY1_VALID_ID((x)) || RSA_PRV_KEY2_VALID_ID((x))) +#define RSA_PRV_KEY_IS_SET(x) (RSA_PRV_KEY_BITSIZE_N((x))>0) + +/* alignment */ +#define RSA_PRIVATE_KEY_ALIGNMENT ((int)(sizeof(void*))) + +#define MOD_ENGINE_RSA_POOL_SIZE (2) + +/* +// Montgomery engine preparation (GetSize/init/Set) +*/ +#define rsaMontExpGetSize OWNAPI(rsaMontExpGetSize) + IPP_OWN_DECL (void, rsaMontExpGetSize, (int length, int* pSize)) + +/* +// pubic and private key operations +*/ +#define gsRSApub_cipher OWNAPI(gsRSApub_cipher) + IPP_OWN_DECL (void, gsRSApub_cipher, (IppsBigNumState* pY, const IppsBigNumState* pX, const IppsRSAPublicKeyState* pKey, BNU_CHUNK_T* pScratchBuffer)) +#define gsRSAprv_cipher OWNAPI(gsRSAprv_cipher) + IPP_OWN_DECL (void, gsRSAprv_cipher, (IppsBigNumState* pY, const IppsBigNumState* pX, const IppsRSAPrivateKeyState* pKey, BNU_CHUNK_T* pScratchBuffer)) +#define gsRSAprv_cipher_crt OWNAPI(gsRSAprv_cipher_crt) + IPP_OWN_DECL (void, gsRSAprv_cipher_crt, (IppsBigNumState* pY, const IppsBigNumState* pX, const IppsRSAPrivateKeyState* pKey, BNU_CHUNK_T* pScratchBuffer)) + +#endif /* _CP_NG_RSA_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngrsamethod.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngrsamethod.h new file mode 100644 index 0000000..59593aa --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpngrsamethod.h @@ -0,0 +1,76 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Cryptography Primitive. +// Internal Definitions and +// Internal ng RSA methods +*/ + +#if !defined(_CP_NG_RSA_METHOD_H) +#define _CP_NG_RSA_METHOD_H + +#include "pcpngmontexpstuff.h" + +/* +// declaration of RSA exponentiation +*/ +IPP_OWN_FUNPTR (cpSize, ngBufNum, (int modulusBits)) + +typedef struct _gsMethod_RSA { + int loModulusBisize; // application area (lowew + int hiModulusBisize; // and upper) + ngBufNum bufferNumFunc; // pub operation buffer in BNU_CHUNK_T + ngMontExp expFun; // exponentiation + ngMontDualExp dualExpFun; // dual exponentiation +} gsMethod_RSA; + + +/* GPR exponentiation */ +#define gsMethod_RSA_gpr_public OWNAPI(gsMethod_RSA_gpr_public) + IPP_OWN_DECL (gsMethod_RSA*, gsMethod_RSA_gpr_public, (void)) +#define gsMethod_RSA_gpr_private OWNAPI(gsMethod_RSA_gpr_private) + IPP_OWN_DECL (gsMethod_RSA*, gsMethod_RSA_gpr_private, (void)) + +/* SSE2 exponentiation */ +#if (_IPP>=_IPP_W7) +#define gsMethod_RSA_sse2_public OWNAPI(gsMethod_RSA_sse2_public) + IPP_OWN_DECL (gsMethod_RSA*, gsMethod_RSA_sse2_public, (void)) +#define gsMethod_RSA_sse2_private OWNAPI(gsMethod_RSA_sse2_private) + IPP_OWN_DECL (gsMethod_RSA*, gsMethod_RSA_sse2_private, (void)) +#endif /* _IPP_W7 */ + +/* AVX2 exponentiation */ +#if (_IPP32E>=_IPP32E_L9) +#define gsMethod_RSA_avx2_public OWNAPI(gsMethod_RSA_avx2_public) + IPP_OWN_DECL (gsMethod_RSA*, gsMethod_RSA_avx2_public, (void)) +#define gsMethod_RSA_avx2_private OWNAPI(gsMethod_RSA_avx2_private) + IPP_OWN_DECL (gsMethod_RSA*, gsMethod_RSA_avx2_private, (void)) +#endif /* _IPP32E_L9 */ + +/* AVX512 exponentiation */ +#if (_IPP32E>=_IPP32E_K1) +#define gsMethod_RSA_avx512_public OWNAPI(gsMethod_RSA_avx512_public) + IPP_OWN_DECL (gsMethod_RSA*, gsMethod_RSA_avx512_public, (void)) +#define gsMethod_RSA_avx512_private OWNAPI(gsMethod_RSA_avx512_private) + IPP_OWN_DECL (gsMethod_RSA*, gsMethod_RSA_avx512_private, (void)) +#define gsMethod_RSA_avx512_crt_private OWNAPI(gsMethod_RSA_avx512_crt_private) + IPP_OWN_DECL (gsMethod_RSA*, gsMethod_RSA_avx512_crt_private, (int privExpBitSize)) +#endif /* _IPP32E_K1 */ + +#endif /* _CP_NG_RSA_METHOD_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpp192r1precomca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpp192r1precomca.c new file mode 100644 index 0000000..ddb9b66 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpp192r1precomca.c @@ -0,0 +1,1937 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (P192r1 precomputed) +// +// +*/ +#include "owncp.h" +#include "pcpgfpecstuff.h" + + +#define OPERAND_BITSIZE (192) +#define LEN_P192 (BITS_BNU_CHUNK(OPERAND_BITSIZE)) + +/* P192 affine point */ +typedef struct{ + BNU_CHUNK_T X[LEN_P192]; + BNU_CHUNK_T Y[LEN_P192]; +} P192_POINT_AFFINE; + +extern const __ALIGN64 P192_POINT_AFFINE ec_p192r1_precomputed[28][64]; + + +#if defined ( _IPP_DATA ) + +#if !defined(_DISABLE_ECP_192R1_HARDCODED_BP_TBL_) +/* see ippcp_baseptbl.cpp test for generation details */ + +const __ALIGN64 P192_POINT_AFFINE ec_p192r1_precomputed[28][64] = { +/* digit=0 base_pwr=2^0 */ +{ + LL(0x332fa108,0x0d8cb30c),LL(0x76d12909,0x8a4bd3f7),LL(0xf3d218f7,0x954cc8f9), LL(0x1e422289,0x7b12a337),LL(0x8966f05e,0xde22b524),LL(0x6aeda84d,0x6a293d83), + LL(0x91fae7b3,0x04a5cf09),LL(0xa7838baa,0xd7fb036c),LL(0x3e00e321,0xae53f3bb), LL(0xf5bc34a6,0x369ef507),LL(0x09d7bb36,0x7d5170c4),LL(0xad59278a,0x241e55c9), + LL(0x230c0249,0x56b3fd7f),LL(0x42bebbf4,0x3386809f),LL(0x770c581a,0x53b5ad45), LL(0x7fb74b1e,0x6be17b49),LL(0x50d91fbd,0x16445c48),LL(0x43dc19c0,0x228f18e2), + LL(0x9a01a832,0xda417f6b),LL(0xd148d209,0x8a42d66e),LL(0x60c3ed50,0xe544900a), LL(0x0fc71a62,0xc79d34b6),LL(0x8733ef53,0xe640e166),LL(0x69803121,0xc1084635), + LL(0x1d8490a8,0x69bca784),LL(0x4d92a6ad,0xa7c43520),LL(0x7012b11d,0x4ec31c34), LL(0x843e401c,0x6261825c),LL(0x6a65a2da,0x102b7a93),LL(0x2d96f43a,0xdf00083f), + LL(0xd6f1d9ee,0xae494b37),LL(0x81585220,0x4708a709),LL(0xed8612f6,0x3c3a183d), LL(0xda5571cc,0x8d3ab905),LL(0xf6131ada,0x8c11bd0f),LL(0xe556b1ea,0x83f240b8), + LL(0xe5efca73,0xeb8ad91f),LL(0x1650b841,0x4c83fc44),LL(0xa63ebb45,0xeea07d43), LL(0xcc6ab309,0x2c8262c8),LL(0xca1e7327,0x44a67081),LL(0x66147372,0x6fef6d88), + LL(0x5264d6fe,0x7afb70b5),LL(0x1e79609b,0x8f728a49),LL(0xeae172bd,0x441912c0), LL(0x8ec468d4,0x96163a6d),LL(0x07b3dc14,0xbd51c5c7),LL(0x707ae554,0xca743b38), + LL(0x449ce446,0x9f27848b),LL(0xed6f60e4,0xedc613b6),LL(0x33ef2855,0xd028dc5c), LL(0x97566f06,0x32bbfe7f),LL(0x529bebe5,0xaee52dfc),LL(0xdbd304ac,0x7dfad9da), + LL(0xcb9cdc1c,0x301800ef),LL(0xa4c0645f,0x01c6df1b),LL(0xd2c1c6da,0x7c2b2dca), LL(0x91015dd9,0x92fe0217),LL(0x43f5e36d,0x94ec3870),LL(0xfae24bbd,0x3400d9bd), + LL(0xf19ed4f9,0x44a299cc),LL(0x3ec18b4b,0x5ce790e3),LL(0x3898e8a1,0x34de50ac), LL(0x52948777,0xa2641e2f),LL(0x538eff35,0xa5037393),LL(0x38e46331,0xf1b6bb32), + LL(0xa53425b9,0xa02e1119),LL(0x9dfe0d6b,0x6a2f4991),LL(0x360f54c1,0xda626cb7), LL(0x39cdd976,0xc224f8ac),LL(0x21725fbe,0x5b23f190),LL(0x164b3680,0xf305458e), + LL(0x0f8904a9,0x153f3b78),LL(0x14a75349,0x44a7bd96),LL(0xd85d4a3f,0x4093735f), LL(0x3f7db015,0xed1d5ed0),LL(0x1fff90b3,0x174975f7),LL(0xdcd47496,0x9838d562), + LL(0x24fc4afc,0xf299b94a),LL(0x1cc29901,0x4bb0fffe),LL(0x3eb21741,0x6cd077ba), LL(0x95c82991,0x6e97b4f9),LL(0x8209cb7d,0x848b6921),LL(0xa082e4c9,0x34d09e8b), + LL(0xb0cf681b,0xc373e084),LL(0x6610b602,0x7da92737),LL(0xf098041b,0x46cadc98), LL(0x6cdec872,0xd3e0adf8),LL(0xa7842f28,0x50d43591),LL(0xcdf46778,0xa35efdc3), + LL(0x7944ad9c,0x90c57eb8),LL(0x1e7a0999,0xb9c5cf4b),LL(0xee3159ba,0xe0315bd7), LL(0x06a19eaf,0x17e47405),LL(0x4be877f9,0x5abccc61),LL(0x22195568,0x427706fb), + LL(0xe7056d4e,0xcfdba265),LL(0xa9b98e89,0xc9d1189c),LL(0xa4a88eff,0x3e1cd309), LL(0xd040ec58,0x8b2429c3),LL(0x7859d5e6,0x6967afd6),LL(0xa5fbf616,0xcdf1095b), + LL(0xfe4697b7,0x78060e67),LL(0x2648e5f8,0x1ed0242c),LL(0x4f235455,0x687ef0c6), LL(0x80964fe2,0x95a99ec5),LL(0xa71449df,0xe59296c6),LL(0xe31fca8c,0x6e0faa87), + LL(0x417692ee,0x37b23549),LL(0x7a4db977,0x3fafb8db),LL(0x5d550c5c,0xc85fef61), LL(0xc591dfcc,0x88f70525),LL(0xadbba7d1,0x20d82c9e),LL(0x3bc105c7,0x34bdc150), + LL(0x312e2b25,0xa96781ce),LL(0xc28a98fb,0x21d19734),LL(0xb32fbb94,0x33d91d89), LL(0xf1a39faa,0x4b423b1e),LL(0xd6d73b09,0xd6b91a11),LL(0x5cf7ee7b,0x1d5bbc7d), + LL(0xc7b622a0,0x09a14d89),LL(0x04ac07cd,0x719fb611),LL(0x485b1f75,0x487f1ad3), LL(0xf779459c,0x6604b185),LL(0x79ffa446,0x633d924a),LL(0xe2ed4e42,0x0a48ac49), + LL(0x97a2ac99,0x8f8141df),LL(0x559a156a,0x2f85aea4),LL(0x0de172c5,0x045569d1), LL(0x70195a60,0xd31e98ed),LL(0x31901571,0x639f8252),LL(0x017861e5,0xddfb3017), + LL(0x8d76bd59,0x791db75f),LL(0xa5c667f8,0x664088fb),LL(0x5a1f032a,0x7cc0335c), LL(0xbf031b03,0xabb460b3),LL(0xc8f559b0,0xcb61b85e),LL(0xfed307dc,0x282335ba), + LL(0xe9b47661,0x398b31e5),LL(0x4785fbe8,0x17a770ad),LL(0x2bf4567e,0xdb5e0ce3), LL(0xc82fa654,0xd8db9c86),LL(0x1b760f9c,0xdc7dbdfc),LL(0x65df38f7,0x7dc26c5a), + LL(0xbfd973a2,0x422b5a19),LL(0x7de885a1,0xe637c08a),LL(0x28af843c,0x493d27a7), LL(0x27d0f709,0x0635af16),LL(0xa46b8241,0xfaa0731d),LL(0xde8513de,0x30152936), + LL(0xc2c6a98b,0xe5b1f5c5),LL(0xdd1f1a85,0x9999e0ab),LL(0xf09f0de9,0x0f017d6e), LL(0xffad11c5,0x2cd84536),LL(0x97ac8fd1,0xc150b469),LL(0xf4a156d0,0x7275b4b6), + LL(0xdea3626f,0xeaa0614e),LL(0x51b201c7,0xb0778899),LL(0xbc24cf40,0xe1e17ccf), LL(0x6f6a4f09,0x5b9a2356),LL(0x81b35be5,0x6b43dd89),LL(0x8eb6864b,0x7134a900), + LL(0x6256a868,0x1fbfbc8d),LL(0x3b84fa71,0xc3c97795),LL(0x75676360,0xaf1330f2), LL(0x5ab6bd0c,0xaa33c244),LL(0xbd5e407e,0xece4da7e),LL(0x2f438421,0x8e4be95e), + LL(0x267c3b3b,0xb550886c),LL(0x072ada2a,0xe4ef0b6a),LL(0x16cf9337,0xefb23bc9), LL(0x035d6cdd,0xf4945a35),LL(0x24d8682c,0x7c2fbc75),LL(0xb35b5932,0xb4b6f148), + LL(0x1704593e,0x5ea32695),LL(0x487113d8,0xcb897688),LL(0x48e6fb62,0x9a382e50), LL(0x59da6a81,0x845eeeee),LL(0x72ab659e,0x427e258f),LL(0x0186211c,0x8d145e39), + LL(0xc1a83d71,0xa5549752),LL(0x07c519da,0xe9b2c58f),LL(0x286a27e2,0x6f5e6aee), LL(0x7cfc6dd4,0x99cece8d),LL(0xbbc24c84,0x18c9b5f7),LL(0xd5a88338,0x7e5875ca), + LL(0xde0dd9bb,0x655f1b75),LL(0x2ef38753,0x19df3f60),LL(0xeceb06b1,0x7a25bb17), LL(0xce909ba8,0x1359f3db),LL(0x65603d61,0xaa2802b7),LL(0xe147b10c,0xf7070a53), + LL(0x4fbffaeb,0x41095e96),LL(0x8b3c6012,0x422337e2),LL(0x563eef5d,0xb731e12b), LL(0xc07cc789,0x7e950bdd),LL(0x9d9cc3aa,0xf97c2753),LL(0x5a7f7345,0x7af21a9a), + LL(0xe3a0c0e2,0x7e91f2d6),LL(0x54a1cd29,0xa286afb7),LL(0xa6160c29,0x8c0ddde8), LL(0x8c5398c8,0xb03e8586),LL(0xfe6503e1,0x4db7075d),LL(0x7ec6bed6,0x999eec3e), + LL(0x16036842,0xb35bcfb0),LL(0xecedc764,0x036d725d),LL(0x2c6f8668,0xcbcb0161), LL(0x1b04bc1a,0x0348f719),LL(0xe1e99a42,0x0b2d84f8),LL(0x9c1ec6e6,0xf098e97a), + LL(0x3f5a8497,0x17f2106b),LL(0xf56d582b,0xc4c8ac85),LL(0x514207b2,0xde518ab8), LL(0x2e30b7c4,0x33018834),LL(0x643786e1,0xf867de48),LL(0x735735ab,0x72d64984), + LL(0x2775c0c5,0x9d83fd01),LL(0x065da96c,0x8658cc5a),LL(0x604210f1,0x466cf509), LL(0xe8fc750b,0xdc4d8237),LL(0x1477a111,0xb13663ff),LL(0x4252c38b,0x6a0ef58f), + LL(0xe5bcf759,0x717f1389),LL(0xa8ecb8d4,0x2146493b),LL(0x3e3c9d15,0xd51427c9), LL(0x77c1c32f,0x7ec8afc4),LL(0xf63856de,0xfa0a577e),LL(0x32750be7,0x95a6c860), + LL(0xcaa33960,0x49b1bb22),LL(0xe647c5a4,0x45871855),LL(0x167d6801,0x6d71e970), LL(0xa2764f1f,0xbbeae386),LL(0x1284c39a,0xc7df73b8),LL(0xe8a8b423,0x3ddc63be), + LL(0x6c2484c5,0x4cf0be08),LL(0xbff601e9,0x59028533),LL(0xcc241c87,0x94453c36), LL(0x0b18c6d5,0xb6e2065d),LL(0xfb4927d4,0xafde2953),LL(0x51875c38,0x69d1f3bc), + LL(0x6709fe0c,0xc4bdbeb2),LL(0x2aa46589,0x86c2f225),LL(0x1c6d96c6,0x6e215f98), LL(0x1900c952,0x3b102255),LL(0x0b476bea,0x09b6f447),LL(0x1643c833,0x88877356), + LL(0xe1f4c47f,0x897e15e4),LL(0x3deb58b0,0x9055b4df),LL(0x5d73f210,0xb189fd7b), LL(0x344e98cb,0xabbc3bfa),LL(0x17fe8809,0x355345a8),LL(0xb8a63413,0x848ea408), + LL(0x0ee8588b,0x111d94b7),LL(0x1cae2230,0x5f7ba2ed),LL(0x8b428b68,0xd1119eb2), LL(0xc5901d8a,0x5ac72be9),LL(0x1a7e213d,0xd11c39be),LL(0xae1574db,0x5eba5216), + LL(0x78e7a47d,0x2c6b4874),LL(0x03871f7c,0x5cbfeb97),LL(0x8dfcd4af,0xdb875c76), LL(0x6202ffc6,0x922461a3),LL(0xf1463db1,0x084e44fe),LL(0xa6b3ba89,0x55a890d4), + LL(0x0d359302,0xbbb38087),LL(0x3bf7212f,0xe0c552b5),LL(0xce70eda4,0x6873aa54), LL(0xf0dcd6fe,0x42eb0476),LL(0x202c6fba,0xe354ae37),LL(0xbb182481,0xd2eeb9a4), + LL(0xac565505,0xfb13f470),LL(0xe0d9811e,0x8e6be184),LL(0xb7fb0ea8,0xc727bd2b), LL(0x2a49df43,0xe0cc92d4),LL(0x6c43c82b,0x551ef99f),LL(0x449a2247,0x50d3acd2), + LL(0x8959dc53,0x2e246f36),LL(0x3f449349,0xc6882bd2),LL(0x52d56c22,0xfbcda062), LL(0x037c14b7,0xaf1525bc),LL(0xf51db774,0x79f2c364),LL(0x397c0db5,0x51d5156e), + LL(0x88ea2408,0x15fdfc20),LL(0xc31c09fc,0x39ad6859),LL(0xa7479cce,0x23bf58d5), LL(0x6df4616c,0x200420e7),LL(0x0bb71847,0x6516f75d),LL(0x60976e77,0x478ec1a4), + LL(0x2d623962,0x38f56704),LL(0xb8e018b4,0x1d5927f2),LL(0xb7159417,0x60f41495), LL(0xe71a9c08,0xa531058e),LL(0xc4427fd5,0x038edfcf),LL(0x16bcd9b9,0xf40f0c3f), + LL(0x4e8b0660,0x5c3875f8),LL(0xba508d58,0x4835397b),LL(0xa986f017,0x28f855c2), LL(0x947b88a2,0xab86055f),LL(0x4038096a,0xf6aab3c8),LL(0xc66df877,0x5babb42e), + LL(0x218251e9,0xbc40bae0),LL(0xff92ffab,0x86a6f175),LL(0x15c92a68,0xc538fbe8), LL(0xce8e05fa,0x94b037e5),LL(0x38df007b,0x1e23b8c7),LL(0x6dfc21b0,0xcf60919b), + LL(0xeaab0281,0x88c58a61),LL(0xfd515dfa,0xbdef49bd),LL(0x7b29a823,0xafb4049d), LL(0x51fb148d,0x803b4141),LL(0xefcde52e,0xa607366b),LL(0x756dd392,0xdc243912), + LL(0x07b07222,0x6c66ac7c),LL(0xabbb5a7e,0x7a2251bf),LL(0xea506766,0x6898eec1), LL(0x0c8100c3,0x8ee773e9),LL(0xb5ac1e10,0xe267812a),LL(0x45ea2616,0x51d3e17e), + LL(0xc5d7bcee,0xec435b31),LL(0x692565f6,0x45068195),LL(0xee8a64da,0xabbe69a5), LL(0xf3259fd4,0x10b23048),LL(0x6cf98be2,0x0df0afcf),LL(0xce0764fa,0xf317669c), + LL(0xab0c2496,0xe0cad6d0),LL(0x3e58a13e,0xad9d4a4b),LL(0xcb2f20c6,0xb4009b9f), LL(0xfbd04c92,0xf586adaa),LL(0x65e4a2c2,0x27f06af1),LL(0xe6ac820a,0x5efc658f), + LL(0x24c1a379,0x14a80bf6),LL(0xab134b06,0x0879a1fe),LL(0xe46fa8bc,0x6425c07b), LL(0xf328660c,0x626b781e),LL(0xa6f4006e,0x32c537fd),LL(0xa5cee8a3,0x3de55b30), + LL(0x45111281,0xd072005c),LL(0xc6dd923a,0xc38687f7),LL(0xe7e55c30,0xc5493c3b), LL(0x1e7bcaf3,0xc801a0dd),LL(0x66af003d,0x7d896378),LL(0x5c5fc040,0x88c92d2e), + LL(0xb5ac249b,0xd7ac3494),LL(0xd031be50,0x646f3e8c),LL(0x925d3ebe,0x6a6d2984), LL(0x0e7ed329,0x4696646d),LL(0x1822a37a,0xe4167a29),LL(0x3f6245b2,0x9419d1d2), + LL(0x489578d4,0x752de436),LL(0x6c0f251c,0x0ade8b1c),LL(0x59ad5609,0x29b34ffc), LL(0x857c870b,0xd4af9bf6),LL(0xe6b0527e,0xb779fab3),LL(0x70ef261f,0x20970dfe), + LL(0x392b4efc,0x2bcf9ad9),LL(0x8a796449,0x24ce4be0),LL(0xdc2b03ad,0x099a3cad), LL(0x223fe0fc,0x35749d6b),LL(0x8c2e618e,0xe9a66616),LL(0x3910e994,0xc7df5779), + LL(0x14729183,0x4af80891),LL(0x2650cc8e,0x44458190),LL(0x587ad69a,0x738c0bd4), LL(0xefc4b5e7,0xdab8713a),LL(0x902363c0,0x7d6d6ae3),LL(0x45acff6b,0xad593898), + LL(0xedc9ecd6,0x4c3723e7),LL(0x9731c5bf,0xc8fa6afb),LL(0xcc9bf5cb,0x162d7a77), LL(0xd28f51e8,0x623c7ce2),LL(0x45d36aad,0xb68fedf5),LL(0x2e95d0ea,0xc840c399), + LL(0xa5542c7d,0xcc1674d5),LL(0x1fb770dd,0xc8ebdbdd),LL(0x0a3b16ce,0x7d70a1ed), LL(0x685db534,0xa74f8148),LL(0x37c0b157,0xa0eab0ad),LL(0xa3971e22,0xcd8063a7), + LL(0x794542c1,0x9bded711),LL(0x3360a470,0xa56154fa),LL(0x79e1003c,0xf0bdb57b), LL(0x36c1f48f,0x48e694c5),LL(0x0eb1a568,0x3b71c023),LL(0x6090a9d3,0x4f574953), +}, +/* digit=1 base_pwr=2^7 */ +{ + LL(0xb3da4d9d,0x9cb08eb5),LL(0x8c573da9,0x95707fee),LL(0xe85fccd9,0xa5ad07fb), LL(0x769af379,0x5d3b50b3),LL(0xb9761267,0x79e01d7e),LL(0xee770876,0xdd24ac5b), + LL(0x492d1c7c,0x7808129c),LL(0x6b626f03,0x97ad42c7),LL(0x5d974059,0xbf9a52c8), LL(0xeaa619c8,0xa888802d),LL(0xbbe19555,0x171450c4),LL(0xb980305f,0x173483c8), + LL(0xa8a831fc,0xa5fa35ab),LL(0xf7ffad71,0xe4f05b10),LL(0x3f8d1cc3,0xc1ed613b), LL(0x6c60e44a,0x9d0f2a96),LL(0xfc64d797,0x8ece4c89),LL(0x8a315b0e,0x6fbdadb4), + LL(0xc352becd,0x824863da),LL(0xa2c14e2f,0xd5d43090),LL(0x46bc9bb9,0x85272175), LL(0xaa9a25da,0x75eb8695),LL(0x85c7c93c,0x4a05c961),LL(0xa9043492,0xcb5861df), + LL(0x62da9c26,0x0a04a3c6),LL(0x6e6542fd,0x9907614b),LL(0x8e9ada48,0x39c0b3b7), LL(0xe51bb6c4,0xf7dffbcc),LL(0xe566dea8,0x0316edb7),LL(0xb0ac507f,0x8b1a59cd), + LL(0xd1581b68,0xbfa7fc59),LL(0xdd5c0d6d,0xdd9906b2),LL(0xfadb3203,0x50570aa0), LL(0xb207b0c6,0x026b8114),LL(0xcf4b11ff,0xf3829145),LL(0xc3abfb69,0x97eef9fe), + LL(0x82a8e3aa,0x3ced44cc),LL(0x8cfe1e20,0x991f72d1),LL(0xeed3e6cb,0xce242a8b), LL(0xf6a15c38,0x70905e2f),LL(0x1a2a9a53,0x0d5db763),LL(0xa414c46c,0xe60a8854), + LL(0xaf1c73a9,0x0b6b1174),LL(0xc0678153,0xaef24601),LL(0x63cc9eb3,0x5dac22ed), LL(0xa0f27800,0x80606754),LL(0xf3dc2a70,0xe3154503),LL(0xec01fbfa,0x6ba23ce4), + LL(0xdc5f2b20,0x53cf654b),LL(0x3e7d94fb,0x4203ff82),LL(0x43547670,0x1f39db20), LL(0xf6716d31,0xbf42ccca),LL(0xa8ef535f,0x7be941f0),LL(0xaaac51de,0x890245d9), + LL(0x2c853305,0x3aba1480),LL(0x43e3e158,0xd4554887),LL(0x135360e9,0xcbae9038), LL(0x6e23c4f0,0xe8d3054b),LL(0xe66d94c6,0x19bd813e),LL(0xafcd895d,0xf4397ee0), + LL(0xc23cf227,0x5526335b),LL(0x6e23ef80,0x1053b920),LL(0x06c75842,0x656efa8a), LL(0x6397c380,0x7b7d1a0a),LL(0xe8cbe845,0xfea5cd22),LL(0x47a7e673,0x620c0df1), + LL(0x7aaf742d,0x8d07f60f),LL(0xeda0b762,0x74469872),LL(0xe237de09,0x3f90d096), LL(0xf4d8b55e,0x679535d3),LL(0xbc04ae92,0xa463084f),LL(0xf1202dff,0x2a0d536d), + LL(0x3ac85f57,0x27649c81),LL(0x235fa2cf,0x8b41bf96),LL(0x9df79933,0xe866618f), LL(0xadc59139,0x6d476615),LL(0xd404e889,0x09b1345f),LL(0x0f7eb70f,0x6bd51082), + LL(0xdc3366d9,0x63fe2f64),LL(0x7732bce9,0xf48f309a),LL(0x6f637459,0xbcd3fb82), LL(0xa9255eaf,0x6d9b5c4b),LL(0x2e1fdd05,0x01ff5ae4),LL(0x72e33bef,0x66e45b33), + LL(0xe080d6b8,0x1d521fcf),LL(0x14f56cf4,0x4e5d581b),LL(0x9a50ece4,0xea6d7579), LL(0xe1c8a0af,0x2d34a6d6),LL(0x28e5f588,0xd459901c),LL(0xf52fe058,0x8d41ab2b), + LL(0x436cba75,0x780fd8f6),LL(0x09f798f7,0x7216f9a2),LL(0xa3720506,0x72c9e7a9), LL(0xfa919d28,0x72cbdd01),LL(0x053500b6,0x045474ba),LL(0xaf3af74d,0x6630329d), + LL(0x5fa22efe,0x548b87c5),LL(0x62b464fe,0xcf449871),LL(0x7637a7dc,0x1eb2ccd0), LL(0x693b51ae,0x6ae5b6b1),LL(0x0a635765,0xcd15e6cc),LL(0x32160012,0xb8139da2), + LL(0x88eff9b5,0xf4477fb6),LL(0x9b8264aa,0x1abe49f0),LL(0x6f59e67b,0x2ed2f447), LL(0x598889ff,0x71460c4d),LL(0x94f3f05f,0x04b1e2ed),LL(0xf04b29b7,0x9b1b5eef), + LL(0x0a36430d,0x2de30220),LL(0x00e9adf9,0xe1df5595),LL(0x1d56ee5c,0x4642ab6f), LL(0x321cd0e6,0x32675e76),LL(0xbd822a36,0x37eaa735),LL(0xff9b1023,0xb807c5a0), + LL(0xb77035f4,0xc1637964),LL(0x2e67810c,0x627deaa6),LL(0xe68721d1,0x11c73b51), LL(0xddd6a178,0x00b6c78f),LL(0x43326826,0x63414a2f),LL(0x9370cb95,0x3f3157a5), + LL(0x28cac9b8,0xa6f2bedb),LL(0xdb9143d1,0xa8d660f1),LL(0xff740aca,0x28161efc), LL(0x81b9607f,0x6a8c99b0),LL(0x3379b6fd,0x6f6991fb),LL(0x54b4b97f,0x3463ae20), + LL(0x5901b54f,0x85e79db9),LL(0x55a0c83e,0xffbd105b),LL(0xef647c2f,0x3b871868), LL(0x18cedd8a,0x5aa96847),LL(0xb6ac9ecc,0xabd9b3d6),LL(0x93bddbd9,0x707f61d7), + LL(0xfcf1d3f7,0xa3e82462),LL(0x0b854fa4,0x619933c3),LL(0xd12e3bed,0xdceb86e0), LL(0x5ac77287,0x8e500715),LL(0x65cf6840,0xe5cb2c78),LL(0xc874ed06,0x6a2fcd3a), + LL(0x75c74d72,0xa9af4af2),LL(0x7df05ffc,0x0f341f15),LL(0x31713e68,0x13bae06e), LL(0xd8e1e3c0,0xbeb2661f),LL(0x9d5b5ff4,0xffe49ef1),LL(0x61d5cae6,0x04eac206), + LL(0xedd6a166,0x2f4fee08),LL(0x2eb29647,0x646cc729),LL(0x4f0f6cd5,0xb5a3c6bb), LL(0x2779dead,0xd1a9c06b),LL(0x4aed0996,0x066e3452),LL(0xbdc98d70,0xcbab5149), + LL(0x88f89e9b,0xf4185c2e),LL(0xf9985e9e,0xc8e5dd64),LL(0xbb24492e,0xb8d5b8d3), LL(0x0fab80d7,0xad79a272),LL(0x5b3133e3,0xb766095b),LL(0x8b2b7fda,0xc83eaf38), + LL(0x4448648e,0xedc5a97c),LL(0x6ae43bdd,0x2658d407),LL(0x48a8e439,0x766de9f3), LL(0x127fb314,0xca58b583),LL(0xb49667b5,0x65246221),LL(0x99398ac5,0x5b180e85), + LL(0xcdf0d01f,0x75910da1),LL(0xae61cd5f,0xb0fbade7),LL(0x17e45ebd,0x0e94c843), LL(0x72944c2a,0xe13b1cad),LL(0xa8546c01,0x80e5d5cc),LL(0xd98df1d8,0x8cb5077e), + LL(0x8e8cdd4c,0xc73d42b7),LL(0xf107a930,0x3a4ade51),LL(0x0da1e241,0x775acdae), LL(0xe97e3d45,0x1919504b),LL(0xb09e94e4,0x15d0774a),LL(0x4d58aaf7,0x3bd3c7f9), + LL(0x7998dd8f,0x07eb2872),LL(0x2cf2179a,0xe9813bff),LL(0x06921a45,0xfa9171da), LL(0xcf58180f,0xb8c04797),LL(0x8721e9b4,0x31db6313),LL(0x91b2c60c,0x381c63f7), + LL(0x66013a01,0x11b0d72b),LL(0x6c92f9e0,0xaf00d7aa),LL(0x47dcc4fb,0x2a4a91be), LL(0x0fa92641,0xb049b8d4),LL(0x448bd3a0,0x069abbac),LL(0xe74c7a8c,0xca288262), + LL(0x385633ba,0xdffd6286),LL(0x19ddfcd5,0xdad4d394),LL(0x4bc0ef1f,0xe90ccfaa), LL(0xc1fc6f55,0x016bf2e9),LL(0xab3c2edb,0x879b641f),LL(0x89695f01,0xd5a122fc), + LL(0xc302a272,0x12a1065f),LL(0x782cef51,0x368397a3),LL(0xbc4ad296,0xebac9fb3), LL(0xe0227d62,0x0caffee7),LL(0x88e685f2,0x63679dcd),LL(0x6aff1b35,0x85ced1a6), + LL(0xa3155a1c,0x8b4e0205),LL(0x8310b45e,0x53afa029),LL(0xce35c0c9,0xa7571202), LL(0x06f892ac,0xa0b7a35a),LL(0x796af8cf,0xe4fbab54),LL(0x9d086c19,0x14ac6bfc), + LL(0xc6267432,0x2e6b041b),LL(0xd1a15019,0x60267755),LL(0x50c3b7c6,0x7d6c36c2), LL(0x9bbe459a,0xf25437d5),LL(0x23468209,0xe1eff41c),LL(0x046c931a,0xe95608fd), + LL(0x3a8012a2,0xd410363c),LL(0x62137413,0x2c5a7c08),LL(0x711e77de,0xaeca7f42), LL(0x08ce5358,0x492fb4fb),LL(0x48f0de03,0x715abe73),LL(0x5f625701,0x726faa61), + LL(0x739b9455,0x2262f752),LL(0x511abdd2,0x8cd71649),LL(0xdeb366db,0x6765aca3), LL(0x3e0a005c,0x686587e3),LL(0x33f1710e,0x029b3605),LL(0x828cc559,0xf7b50e02), + LL(0x66442cbd,0xe50f5b8e),LL(0x84763a5c,0xf27cc25d),LL(0x3120d35d,0xa92103eb), LL(0xfcef0800,0x91c0542f),LL(0x2d72f1ce,0x8bd12bb5),LL(0xf8086d1d,0xe97d290f), + LL(0x4ef89505,0xab4b24f7),LL(0xfe72dd92,0x2bc14e79),LL(0xc60af192,0x94226552), LL(0xfd45454c,0x474c2b8d),LL(0xc7753a83,0xac99b5d3),LL(0xc8f1d3ae,0x6946a61d), + LL(0x0c6a72ee,0xcb9c16c5),LL(0x64f9d8b2,0x96e95912),LL(0xdaf45f48,0x7c59cb24), LL(0x79b6515b,0xe520df3f),LL(0xded63e3c,0xb79c6ab5),LL(0xec685103,0x49767609), + LL(0x6d892dae,0xbae68558),LL(0x3e48a96b,0xf07e8b59),LL(0x49e4d777,0xa7115219), LL(0x56e5e52c,0x724d7d29),LL(0x6fe58f07,0xb2483de9),LL(0x67836a1d,0xb28ffc89), + LL(0x507bc006,0xaebcc451),LL(0xce8569b0,0x27b4e0db),LL(0x02bf59f6,0x579ae452), LL(0x5cff7495,0xc53b052a),LL(0x4cc9201b,0xc816b206),LL(0x540dd63b,0xd3c7350a), + LL(0x26022a40,0xd9df51ca),LL(0x92be30ea,0xc1842bc2),LL(0xfa161a61,0xcdd5cee3), LL(0x554031f4,0xdaefdb75),LL(0x4935fd8a,0xd28afc65),LL(0x0a32f4c9,0x439d6d68), + LL(0xeb1dc395,0xbe55112c),LL(0x531a3e8c,0x90def44c),LL(0xf77a7b59,0xf5b9ef12), LL(0x4c8ce124,0xc3ec1edf),LL(0x1c16391a,0x4edffe21),LL(0xe9ada0cf,0x40b8ec6f), + LL(0xae68790a,0xc586619e),LL(0x0e0ba4ca,0x5bbe744a),LL(0x70ee1ec0,0x21a82b73), LL(0x7b9850f1,0x82066320),LL(0xee9bcc2c,0x2a7536c0),LL(0x9d661f43,0x9632c632), + LL(0x1724c849,0x6d3b65ad),LL(0x8cfb77ce,0x3bef3b0e),LL(0x71fb5452,0x670e88e5), LL(0x1035a015,0x82d57c5f),LL(0x23736115,0x340c145b),LL(0xb1c2dc13,0xf1a704ea), + LL(0x0ed1952e,0xd3aa33da),LL(0x9a32b9c9,0x459e7596),LL(0x15d05a80,0x15dcd976), LL(0x75895b93,0x85eea023),LL(0xc2fcd91d,0x8daa32c3),LL(0xba5669ce,0x2290ba90), + LL(0x66e7e5af,0x85ac1742),LL(0x05bff18e,0x9ba521e0),LL(0x556a9133,0x9f081540), LL(0x929cd03f,0xb913fd04),LL(0xf46a514d,0x3065b31b),LL(0xe351ba04,0xe2fe761d), + LL(0x139c5933,0xaf1e2230),LL(0xc129bd42,0x0c6c8096),LL(0xf4cf79c9,0x56f406b5), LL(0x4e86f542,0x772cb3d0),LL(0x843d10bd,0xbe007c1b),LL(0xe8e6693c,0xd3a201f5), + LL(0xcfee362e,0x5a6556de),LL(0xa9264918,0x00dafb80),LL(0xc2ab4fcd,0x608dfd33), LL(0x091a15a3,0x29946cea),LL(0xfcbef4a0,0xb8caca39),LL(0x27c3eb56,0xb0c39fce), + LL(0x21b66fdb,0x9fad053d),LL(0x74192d67,0xb529c9b6),LL(0xf21d3c4f,0x34c31800), LL(0x33cb3cc5,0x80797948),LL(0xb20d1f9b,0x18c66091),LL(0xe49755a6,0xc9fe30a8), + LL(0x4beb71b8,0x90ae4d45),LL(0x4083ee91,0x2fc24099),LL(0xab8d6bb8,0xbcb9c603), LL(0xbca4c1b4,0x72d50046),LL(0x1a6be188,0xbc5fbf1c),LL(0x3da03c7d,0x4bc93b91), + LL(0x3931092b,0xcbc53ee8),LL(0x9af003b5,0x455bb703),LL(0x9ac0e7d7,0x8da6de21), LL(0x9ccbcb20,0x6a4900cb),LL(0x73963c8e,0x3146af37),LL(0xd86a3d10,0xfc9469ab), + LL(0x8078ecda,0xcd49df84),LL(0x45a4ff92,0x9caf52e4),LL(0x5a8fc8e7,0x4e8bd802), LL(0xdfeff51d,0x4339ea5f),LL(0xc4433bd8,0x25169385),LL(0xebe52ca1,0xdb3a7ae3), + LL(0x09e9bb4b,0x8cb174c9),LL(0x224cf62c,0x44657124),LL(0xe1f6dbe1,0x70c991d3), LL(0xb10cd328,0x561771b8),LL(0xb5242c51,0x4e08ed68),LL(0x9e7d3968,0xd3eb6498), + LL(0xbef053e4,0xaef71aa7),LL(0x49740ae4,0x3c3c52b5),LL(0x8eca22b9,0x5b6f510e), LL(0xf2645282,0xf1b861ae),LL(0xe96d07f6,0x670312de),LL(0x0c04dcfb,0x413388fe), + LL(0x7734d6a7,0x7b924e76),LL(0xa86fc636,0x3616aaaa),LL(0xb9d584ca,0xd600f983), LL(0x67c54321,0x6324b5b2),LL(0x843f1742,0xee50247c),LL(0xde7f93af,0xf92e7c7a), + LL(0x0aab4e71,0x1e668209),LL(0x236d6f67,0x76b4b879),LL(0x1fe85adc,0x8358a8ae), LL(0x550e1974,0x7567b0d6),LL(0xbd316399,0x3d193081),LL(0x92a0d80d,0x383ef613), + LL(0x26ad5111,0x04a8ee34),LL(0x2efe5eeb,0x56e28325),LL(0xcd888cec,0x88e3f877), LL(0x0de0fb50,0x9ce73666),LL(0x7be9e17e,0xc84bc9e9),LL(0xbc67f215,0xf3a496a6), + LL(0x41f7e964,0x548c525b),LL(0xafc8296f,0x18b03c78),LL(0xc171d822,0x14e4c1df), LL(0x03b39f89,0x090e2c4e),LL(0xe157c072,0xaf0feb62),LL(0x77227f20,0xe10a4331), + LL(0xc72c5d07,0x40314b18),LL(0x1407f165,0xb2b01e30),LL(0x512cd1ec,0x4ad000d1), LL(0xcf5ce913,0xd7310420),LL(0x865fb1f8,0xa38701df),LL(0xb1db3c80,0xac4c2674), + LL(0x36bf878b,0x07697637),LL(0x6ddd10b4,0x662c148d),LL(0x8d97a563,0x75b55463), LL(0x7f1198b4,0x34bf8bd9),LL(0x643001f6,0x31155240),LL(0x0d3f4f13,0xda176596), + LL(0xcdcf7cab,0x44c61a3b),LL(0xecb257b3,0xcda79f8f),LL(0x8a15332d,0x7f701080), LL(0x4472224a,0x6f48e031),LL(0x31fafaff,0x3e89e0e6),LL(0xbfc3f192,0x1f066674), + LL(0xa2df8c34,0x132f7d0e),LL(0xbc28bb95,0x66d020ad),LL(0xa0c4325a,0xb942cb64), LL(0x07a82d3a,0x4922c971),LL(0xad5d63e9,0x22ce594b),LL(0xc2e553d3,0xdf369ada), +}, +/* digit=2 base_pwr=2^14 */ +{ + LL(0xa96c5146,0x773311a3),LL(0xafc15c15,0x8ff8dce1),LL(0x3e382d13,0x59b9e79c), LL(0x3e0a552a,0x0ed1f12e),LL(0xce4662af,0x487503ec),LL(0x7000683e,0x624bc8ab), + LL(0x58cb54ac,0x59ffacde),LL(0x492dc39c,0xef64164f),LL(0x337cd1ec,0x16b13c76), LL(0x47ca44a2,0x1e2033a7),LL(0x193743a8,0x8477b54e),LL(0x8e016823,0x557a7155), + LL(0x5da83913,0x96d2f426),LL(0xc427cf5a,0x41062046),LL(0x1dd15a14,0x8016f670), LL(0x78840ddb,0xacd78288),LL(0x7feae4af,0x21529372),LL(0xc77acb08,0xdb65ffee), + LL(0x9f1f177a,0x23d068f6),LL(0xbec8525d,0x508a8bed),LL(0xabb44bb0,0xa9de104e), LL(0xaef83c7c,0xb1ad4ce8),LL(0x9b842889,0xf045e04d),LL(0xa931cb25,0xca1ba541), + LL(0x87b11b0a,0x377c8c86),LL(0x70d0248a,0x4eb952ff),LL(0xec2537f4,0xbde06ebe), LL(0x48f400cf,0xfc17552b),LL(0x84cd18c3,0x73e1b8a5),LL(0x4e54b85c,0xa3a8b445), + LL(0xa64a281a,0xa9a806ba),LL(0x7b3bc32a,0xbc16fb01),LL(0x8ab7aede,0x8d2a8d5a), LL(0x2d7db806,0x7db641a7),LL(0x2d8c945e,0x8afe3e84),LL(0x1aefc1de,0x3e700391), + LL(0xf8271efc,0xd9e4e55a),LL(0xecac958d,0xd50a0d2a),LL(0x3293a393,0xc58b1b30), LL(0xca65ac88,0x3dfed217),LL(0x13dbff4e,0xe308b3d8),LL(0x7a70bcfa,0x185024cb), + LL(0x73d7e444,0xd7ca2cc9),LL(0x5d9cfe00,0x2ab7d94e),LL(0xd1333221,0x24a41faa), LL(0x6b27127b,0x9ce62b32),LL(0x20aca054,0xf66d0897),LL(0x079bb62d,0x41f3f259), + LL(0xc11087c3,0xd74686ed),LL(0x99d4d141,0xd199b1e3),LL(0xdb7b97dc,0x2e3a551d), LL(0x8910fa3f,0xb59f58e6),LL(0x7ecfc3ef,0xdef98365),LL(0x349b65ff,0x29ec545a), + LL(0x0598f0ef,0x7fc08cb5),LL(0x4400c739,0xa5208904),LL(0x4c9f98b7,0xb02f25b3), LL(0x901fc002,0x4d9901cf),LL(0x49babf27,0xdde7d571),LL(0xf2d65888,0x13b3e92c), + LL(0xb6dc4b6d,0xd55436e9),LL(0xc04a2f37,0x746c5407),LL(0x4485d943,0x84d53bb7), LL(0x16b241f8,0x0ba98680),LL(0xf8feecce,0xbdd3b788),LL(0x87dd6420,0x1801aa7c), + LL(0xcd59a045,0x047bdddb),LL(0xb8464ecc,0x0b74793b),LL(0x37cce6df,0x95661e95), LL(0xfc3170e4,0x20d1535b),LL(0x68c8b253,0x14546482),LL(0xc42e9c3d,0xbae5cef0), + LL(0xe87e6d86,0x0201464f),LL(0xbd67f280,0x1bab0e73),LL(0x64600ebe,0x69b188cb), LL(0x5f9c98da,0x3641dea5),LL(0x9aa645be,0x94d2a2cd),LL(0xbd545f72,0x1ef74b42), + LL(0x914d779c,0x19faba59),LL(0x27ba5f19,0x90007928),LL(0xe1ee88ff,0x46f12e72), LL(0x726a44cc,0xa9e0848d),LL(0x285108c9,0x13d4e943),LL(0xdf77e165,0xb5721271), + LL(0x492ca661,0xa444123a),LL(0x8e0769ae,0x9ff4a82c),LL(0x718014db,0xc6786b05), LL(0x19f577ac,0x702027a5),LL(0x56e9b5ef,0x349ca959),LL(0x4f6e4b5a,0x1f746299), + LL(0x4f0efa59,0xb23ca413),LL(0x15a3b35c,0x003f4a5b),LL(0xe4a4d655,0xa0fe8a5a), LL(0x9e5467af,0x044479ec),LL(0x6d4b3c89,0x5abb290c),LL(0x1c2f5364,0xfd845d3c), + LL(0x6837a39b,0x21cdab37),LL(0xf18d4f99,0x7037f403),LL(0x6621328c,0x964d48a7), LL(0x930c4f4e,0xd70553ec),LL(0x5274352b,0xca35a6fd),LL(0x6199ce61,0xff584ff2), + LL(0x479ebc39,0x60a7a4ce),LL(0xaa703602,0xaf3c24ce),LL(0x89689136,0xfe79ffc9), LL(0xc7ede59c,0xaab98ecc),LL(0xbf5659bd,0x933f24a2),LL(0x1d702fc6,0x68fd0d06), + LL(0xedbb3147,0xbb51d576),LL(0x5d358026,0x038e2a2c),LL(0x45ed9fed,0x10fef0db), LL(0x55be7ba9,0x78b92751),LL(0x69bd1f87,0xae493919),LL(0x186699c2,0x632e8ffd), + LL(0x9085fbe9,0xb0b4b604),LL(0x808be4f0,0xf93f9bf1),LL(0x6d8bf009,0x2b58f977), LL(0xd6188f6c,0x020fe5ea),LL(0xd25805ca,0x85d4a860),LL(0x07bb3697,0x7547f724), + LL(0xe0b4ea32,0xb225b950),LL(0x33f7d58d,0xbc6e0982),LL(0xcb7418f4,0xb81e161f), LL(0xc0020282,0xfa1572b5),LL(0x8ab14fe1,0x3fababab),LL(0x08b2a18e,0x3a033779), + LL(0x6495bacb,0xe816e0fb),LL(0x5d8888be,0x238bd498),LL(0xe3df2fc2,0x7ab0b8ca), LL(0x978404d8,0x88ee40ec),LL(0x26a463b7,0xbc3d1a07),LL(0x65d21535,0xa4c92dcd), + LL(0x2928f5f1,0x819bd554),LL(0xbf6255d1,0x82906c78),LL(0x406326b1,0xc710bf1a), LL(0x09f4cd28,0x3a9bbfcb),LL(0xded76f80,0x83e41584),LL(0xbbe53844,0xca1c61fe), + LL(0xae17eaf5,0x80c38892),LL(0x61b16da4,0x3670cf14),LL(0xa795a9c9,0x9f9ddd50), LL(0x1255afff,0x6f1841d7),LL(0xb69e10fc,0xc733cd3f),LL(0xdaf931ce,0x515ff527), + LL(0xcc8a9a7a,0x14969bd6),LL(0x2c1d2761,0x579c2fb7),LL(0xf09d2a18,0xb0bfb370), LL(0x073b22bd,0x0baf361d),LL(0x1a52d79f,0xb5e11353),LL(0x22d425c2,0x9d475b0d), + LL(0xecdc5b6e,0x24d4fea9),LL(0x1731ad86,0x8b3af04e),LL(0x7dfe43b4,0x0a1c0e04), LL(0xaf2af7bb,0x5cf225bc),LL(0x69ef4bed,0xb0885ab2),LL(0x830333e4,0x90999bc9), + LL(0x985d6058,0xa8c2e9f0),LL(0xd1b5a3f0,0x8e9e9162),LL(0x3e407711,0x1055feff), LL(0xaaba085f,0x5a33934f),LL(0x23b30a1b,0xf961622f),LL(0x6dafb1dc,0xb1a2ee28), + LL(0xa448ee27,0xe49caf6b),LL(0x4560f7a7,0xb8552bfb),LL(0x93f90be8,0xb68fd6ca), LL(0x1f2499d1,0xbb85cfca),LL(0x093c01a1,0x1a51059e),LL(0xd231b79e,0xdf98052f), + LL(0xc076155c,0xf696a525),LL(0xf93ce35a,0x3a519224),LL(0x2e706fd0,0x2771d43b), LL(0x51a451c7,0xde78cd08),LL(0xae569dba,0xc9a60017),LL(0x22a0b1b7,0xa3cafd39), + LL(0x422b409c,0x4afddf96),LL(0x75f5eb64,0x6313a926),LL(0x626ac381,0xf9420ee7), LL(0xe9858ee9,0xfbe23e3f),LL(0x3c678a64,0x658013be),LL(0xf2ad2c6c,0xbbcfddc5), + LL(0xd05334aa,0xf9303aca),LL(0xd7626b98,0xad3e45cf),LL(0x8ca3a1f8,0x56c81f5d), LL(0x4aab3342,0xe892a0b4),LL(0xa1db86fc,0xf8a3f244),LL(0x97566251,0xdee7ed3e), + LL(0x5efced68,0x0c65a22c),LL(0x9ce653ac,0xfa181866),LL(0x1e590014,0xa46d7d86), LL(0xc79656dc,0x69d234da),LL(0x00d2b7a7,0x4dbb31d4),LL(0x3452ca10,0xc9636e5d), + LL(0x0778d7f8,0x5850e572),LL(0x31b9202b,0x4323e55b),LL(0x5e8d8006,0xaea618dd), LL(0x999bf27e,0xe0f4408d),LL(0x8fab0cc5,0x4bd06762),LL(0x5a75c171,0xddb647f2), + LL(0x452f35a5,0x6ce09fdf),LL(0x91938a04,0xe6f5d9dd),LL(0x4cab01a5,0x79821593), LL(0xdc0ff9f5,0x054bf7a2),LL(0x613b5386,0xdb52d927),LL(0xf35b7179,0x661fe231), + LL(0xa035b792,0x8a4adcd0),LL(0x2790ce5c,0x854bed6f),LL(0xac3fa847,0x64b0eee2), LL(0xfeb32887,0x7b752cc2),LL(0x09fcfa29,0x5e34ee86),LL(0xac0cd2e1,0xd06594aa), + LL(0x10995206,0x6151e607),LL(0x93bb033b,0x93532d80),LL(0x7a33bd69,0x0beac8c5), LL(0xfb8f97c6,0x6f85d5ec),LL(0x0517b3eb,0x661303e8),LL(0xbc3f62c8,0xe17cc0cf), + LL(0x4f4d78fb,0x279803d5),LL(0x66c81c4e,0xca63bac0),LL(0xc854f59a,0x6ec15cd1), LL(0x0f159ecf,0xb345c1f6),LL(0x3763e854,0x84807c78),LL(0x8cc6374a,0x691abd9b), + LL(0xe4f37f9a,0x77c0e93d),LL(0x611d93dc,0xc60578c7),LL(0x1aff93a6,0x7c98f95b), LL(0x5bb698ce,0x534f1e2d),LL(0xd8bad550,0x0cdc34b6),LL(0x16afd591,0x597794f0), + LL(0x571c8a4e,0xb795a174),LL(0xe651a52f,0x15ce6b83),LL(0xf49edec9,0x9394c28a), LL(0xf091dfa7,0x0f74e17e),LL(0x65c44142,0xcc9e8180),LL(0xa4df670a,0xc33d77e9), + LL(0xc1633318,0x44536321),LL(0xc0b9deff,0xefb782b9),LL(0xc6b86a19,0x24fda05e), LL(0xf868535b,0xba601337),LL(0x91c21fc3,0xd3d126a5),LL(0x0663ffb0,0x10b1b7b1), + LL(0x7fa1153d,0x8162598c),LL(0x688e7c55,0xc3c97288),LL(0x89c8fc1b,0xb0c2872d), LL(0xa859461e,0xfbc2a074),LL(0x82f7d203,0x91591a5e),LL(0xa909de2c,0xe278dac7), + LL(0x7d4e49c7,0xad689f7f),LL(0x940bf215,0x12072de0),LL(0xde694ba0,0x7ec61fdc), LL(0xfed04942,0x4a4ee787),LL(0x0f665159,0x0497d6c5),LL(0x94136075,0xe9e08707), + LL(0x19850a32,0x391c4103),LL(0x50d61027,0x0ad15740),LL(0xd883b3e7,0x9ca3e6c7), LL(0xf0d758fb,0x76891ec0),LL(0x04d6f53d,0x89ae897b),LL(0xcffefb93,0x77e9b63a), + LL(0x216ee99a,0xf201c95e),LL(0x4fb9bed8,0x65648551),LL(0xda5d4d62,0xda9c97f4), LL(0x6aab7149,0xfae41f2e),LL(0x56aa67d1,0xfb649545),LL(0xc024449c,0x493a5061), + LL(0x4df4c709,0xd0aa84eb),LL(0x06a93789,0x69c9246f),LL(0x917b88eb,0xe19b7340), LL(0x9fd99349,0x7f7cfe6c),LL(0xa8ebc191,0x6c82e590),LL(0xade7110e,0x7c50c13d), + LL(0x50ffdce6,0x17155bf9),LL(0x5281bb39,0xab2bb8dd),LL(0xc4ba3181,0xe73086fe), LL(0xa736d737,0x41d9f1f7),LL(0x3df7fbbd,0x831b8099),LL(0x3b6ffde2,0x566467da), + LL(0x36bb3e58,0x6cb9c1b9),LL(0xb8a84147,0x450324ab),LL(0x8bedffc5,0x99535f56), LL(0xa7e14db3,0xb0a718ef),LL(0x304dcfa7,0x65a62f9f),LL(0xd623bb6e,0xfb2f289a), + LL(0x9d546bd5,0x0888c3e7),LL(0x772582a9,0x1a97b3cb),LL(0x7645a4c8,0x50937134), LL(0x93ea1519,0x70d77533),LL(0x079ff6e5,0x5837b53c),LL(0x45bcc030,0x8f5c8745), + LL(0x1370e46d,0x28549138),LL(0x0bcf90ad,0xf0c3a434),LL(0x639db090,0x4d42664a), LL(0x0158f802,0x345c315a),LL(0x0463e159,0xef82afb0),LL(0xd3e83b12,0xfcf2c419), + LL(0x9e3fc0b3,0xe04ea5a2),LL(0x37fd1788,0x98b3901d),LL(0x2d603830,0xc6326a28), LL(0xbe5e9c25,0x6e6c5c6b),LL(0x0fc60fb7,0xac753036),LL(0x12b1dad0,0xc4cf690c), + LL(0x46768f3f,0x273e19a9),LL(0x4414d65a,0x93aa3754),LL(0x5fabe262,0x4ecff706), LL(0x7ff1403d,0x6d9cfcee),LL(0x33284115,0xf6e4192e),LL(0x5e218033,0x80aa8768), + LL(0x485363f3,0x0067ee42),LL(0xfab8b185,0x5fcd61f5),LL(0x8e63d384,0xe3c345c8), LL(0x23ff551f,0x80847073),LL(0xe339aa9e,0x6a804b17),LL(0x6bb64f96,0xb9142088), + LL(0x7679428f,0x855916bb),LL(0x9c4cf460,0xbab9e9be),LL(0x55dad941,0x0c3bdf26), LL(0xfedb73be,0x6b8e8c3a),LL(0x0355a4c2,0x07fdbfa0),LL(0x5e3c1b3d,0x8e9137e5), + LL(0x57927722,0xb4afd653),LL(0x5f997d66,0xd48cbfc8),LL(0x7719d80d,0x5125ce07), LL(0x654d2350,0x19d47bce),LL(0x4d7a3060,0xfc671097),LL(0x15d775f7,0xddad1eae), + LL(0xc6a285a3,0x9e8ea16e),LL(0xc535adfb,0xc2f0c22d),LL(0x217f73ce,0x3fc464b9), LL(0x73a8ee71,0xc543babb),LL(0x0da708c9,0xce410235),LL(0xc2f48649,0x99849442), + LL(0xc83b501e,0x6b420f3d),LL(0x642f062d,0xe08c411c),LL(0xf03d2cf5,0xc79fd495), LL(0x07236a83,0x2184fa72),LL(0xc9a0ea3b,0x31b0e4f4),LL(0x5b82a3b3,0xd7795d4b), + LL(0x3fe60842,0xb2520494),LL(0x45bdeed9,0x751336f1),LL(0x9718fa41,0x1b66bb31), LL(0xda9a21c4,0x5a50a96b),LL(0x615dcdec,0x94cc1fb5),LL(0xe39c3066,0x556b641f), + LL(0x15ea6075,0x69d00e4c),LL(0x83005a21,0x3e707e72),LL(0x767ca1f8,0x84e2a469), LL(0x3f8c4c84,0x94b82739),LL(0x81a2db60,0xe00dd19b),LL(0xd7cabd28,0x16f30fec), + LL(0x6305314b,0xb08563f9),LL(0x7ac22e44,0x73d8fb6a),LL(0x55919896,0x9b969f2f), LL(0xa3460685,0xcc9dd3fa),LL(0x1226dbee,0x377b5d9e),LL(0xaaa83ccc,0x0b4bbd51), + LL(0x61f910de,0x929c87b0),LL(0x5f2bdd34,0x0595e87f),LL(0xb587735f,0xa5b54a4e), LL(0x6ac0195c,0xa3bbb78b),LL(0x1d44758a,0xe6b38b87),LL(0xac01f679,0xfd575f0c), + LL(0xb21a6382,0x223f54eb),LL(0x672c43f7,0xf6f79b9d),LL(0xb8b258f5,0xcf8054d3), LL(0xa119b37d,0x88ec08ee),LL(0xafda5f1f,0x4b0fd837),LL(0xad4546ad,0xc0dde1b4), + LL(0xe673eb95,0xbf7186d6),LL(0xc9d6a6a8,0xa324e752),LL(0xc117d914,0x5ebf8238), LL(0xad6715dc,0x9c33641e),LL(0x1d597adb,0x99439b58),LL(0x9c6a2ef6,0x83ba580e), + LL(0xb17b3816,0xbf87295c),LL(0xee1db90d,0x05c753ce),LL(0x2554bda2,0xa31eccaa), LL(0x81176cf2,0x66691105),LL(0xb4e7bb4c,0x67eb9bec),LL(0xc294982a,0x6443f703), + LL(0xd0ab699f,0x86c4e0ba),LL(0x748ed429,0xce3d72f1),LL(0x53e24aed,0x0ed09a39), LL(0x4db6e8ed,0x0ac3cda5),LL(0xeed93b68,0x26b6ee99),LL(0x2d53c5f0,0x3707c0fb), +}, +/* digit=3 base_pwr=2^21 */ +{ + LL(0xc3bba334,0x549606d0),LL(0x013ca242,0x6dcb5519),LL(0xdcf78255,0x784d65cc), LL(0xdbd76716,0x73d0de0c),LL(0x1010fb15,0x2bc13bf7),LL(0xe8d77e91,0x21d2c3cc), + LL(0xa396d29f,0xf8efa6ac),LL(0x767470ec,0x33ec678b),LL(0x7241f5fc,0x23f69526), LL(0xd2a4a560,0xd0338dfe),LL(0x9d30be61,0xce5baa02),LL(0x4db7b6e5,0xb39ee0e3), + LL(0x4268d7ee,0x93adf307),LL(0x2c9a12bf,0xfcbae0f0),LL(0x534822a7,0xb76859b8), LL(0xf961bd84,0xab38334c),LL(0xd09e4e94,0xb17e54bd),LL(0x43d3a64a,0x0fe2b812), + LL(0x9577e8d9,0xe17445a1),LL(0xd2d34121,0x330f6634),LL(0xa1aa8076,0xcea03741), LL(0xfcfafd3e,0xfddda6ff),LL(0xe05b9369,0x440df5db),LL(0xbb0d7ca7,0x51fbac8e), + LL(0xc9b4ce1e,0x320accbd),LL(0x99292bb2,0x3798c1f3),LL(0xa402db93,0x8371a2e6), LL(0x65d07e1c,0x716af227),LL(0x23851a3e,0xd06eaa2f),LL(0x61f48045,0xc42ee506), + LL(0xc40da094,0x538f7dc5),LL(0x54a44890,0x4c651fc4),LL(0xee59f9d2,0x80876d6c), LL(0x66c8ac59,0xad6aff6e),LL(0x257e1b91,0x32b4e3ba),LL(0xec717e05,0x957bedc7), + LL(0x7459bbfe,0xe45216bb),LL(0x2ef69f90,0x3dac0bfe),LL(0x6161cb8b,0x2cc6a2b9), LL(0xf6e7e3b3,0x472e91fd),LL(0x304e8482,0x4d2e40f0),LL(0xaf1042aa,0xf4afc219), + LL(0xd64583ec,0xffc53726),LL(0xeef5bce9,0x6b462381),LL(0x24829cb3,0x753a4bef), LL(0x404af02f,0x976d6897),LL(0x448a7822,0x93313219),LL(0x399e5d55,0x79449ea9), + LL(0xc5ecabd1,0x3574a11c),LL(0x34e48369,0x11ee8559),LL(0x915873b6,0x7fa30b5d), LL(0x2ae5b9b4,0xe1a71608),LL(0x2e04edad,0x57296722),LL(0x30d27fbf,0x84cdfbff), + LL(0xbe3bd4b2,0x0c29de57),LL(0x2898b5b6,0xfdc6a64c),LL(0xc8bb55be,0x5604fc5b), LL(0x2e98dac3,0x1e6f616c),LL(0x8e9baefb,0x494a5143),LL(0x5501fe87,0x251c23c9), + LL(0x6cdda2c2,0xebcbef56),LL(0xe3ee721e,0x85b1e422),LL(0x9dcdd732,0x32bcca8f), LL(0x72410bf8,0x182e8ce7),LL(0x36f1dca3,0xaa7f4322),LL(0xdcd402ab,0x893d6417), + LL(0x43cd1278,0x87f47e84),LL(0xe1831ddd,0xc8b878ac),LL(0x093ba446,0x4d0a3597), LL(0x40080bf9,0xafc47c89),LL(0x3832f14c,0x3157bb67),LL(0xb8eb8baf,0x2abb24b3), + LL(0x063399bf,0x30752625),LL(0x2979063f,0x47abe9c9),LL(0x55165854,0x01c9def8), LL(0x6fb4555d,0x0bab44d6),LL(0x567eb3a7,0x8a53411f),LL(0x726f9e4e,0xb8aab552), + LL(0x2440a85b,0xad782e43),LL(0x2fbb1ce6,0xfa5a74b7),LL(0x1ae20947,0xc49e442b), LL(0x772117aa,0xd3fca276),LL(0x6115a3ed,0x303be2dc),LL(0x1a6f32b8,0x3b808eb6), + LL(0x71faca97,0x673ae260),LL(0x086041a0,0x32f3b688),LL(0x1367ca73,0x26acc3be), LL(0x8c66b759,0x48400913),LL(0x9b73c0e4,0xc88e5241),LL(0xd44b961b,0x17a8a61d), + LL(0xe56aeb1a,0x0a3a7387),LL(0x461fdd3f,0xa4010610),LL(0x016484bb,0x601e6822), LL(0x04bbe3e8,0xde403502),LL(0x9f071625,0x6a1b8c7d),LL(0xdd3f60b3,0xd1e9028a), + LL(0x1496f098,0x7b626130),LL(0xbf161daa,0x62dffbe1),LL(0x9c44fcc7,0x401ddc78), LL(0xb9872e73,0xc81c3a33),LL(0xc751f26b,0x0e48c326),LL(0xa0829b6c,0xffaeec0e), + LL(0x77a5b096,0xf5530fa7),LL(0x19734245,0xb008b72c),LL(0x848b3785,0x7462bc45), LL(0x6cf9e5f2,0x29fa43a4),LL(0x9854d78b,0xd8b27b11),LL(0x9097f3d4,0x6f8fe5a5), + LL(0x30c97aa5,0x63ec5b71),LL(0xc6488713,0xfa8462af),LL(0xd7b335fa,0x51f2d273), LL(0xcd075cb1,0xf2891fbd),LL(0xd007c58e,0xffe2576c),LL(0xd8bc9492,0xaeec8e91), + LL(0x6f905b81,0x305b6908),LL(0xbf19b3bd,0x5d023c2e),LL(0x22ae8649,0x9c503cbe), LL(0x25255b41,0xde8be78f),LL(0xe809567c,0xb1e4b2de),LL(0xb09e4638,0x986a2fd5), + LL(0x213fb6cf,0x0b36dcd5),LL(0x834d7c84,0x5f340cf2),LL(0x42696c00,0xf835cd54), LL(0x71e8739d,0x7f02199e),LL(0xb4f38aed,0x0663d540),LL(0x1f7c9176,0xed21b27e), + LL(0xbc7e8681,0x1810377d),LL(0x1545ee0d,0x242d18eb),LL(0x45281613,0x3360ba32), LL(0x808ebc75,0xb14e805f),LL(0xad26b94c,0x965c6c59),LL(0xebf7f5e2,0x24e3cd91), + LL(0xc0623d66,0x493d0777),LL(0x05131c2a,0xafe840f2),LL(0xfe23fc98,0xb282512d), LL(0x3735071c,0x47342277),LL(0x105f0570,0xfe0900ce),LL(0xad56ac6e,0x7f80e17f), + LL(0x7da80215,0x5509a924),LL(0xe2039096,0x0d1b8304),LL(0x32330f7c,0xb8d360d9), LL(0x866bf185,0x8b6b538a),LL(0x60f48919,0x65565d06),LL(0x4a88485d,0x457d3e36), + LL(0xb4d530a4,0x21ffdbfe),LL(0x134babd4,0xb68a01ac),LL(0x3c545d8d,0xeac3a42a), LL(0x525bd6e8,0x2ec4920e),LL(0x11415be9,0x91d90569),LL(0xc859b9a9,0x27f2bf78), + LL(0x3a492c73,0xdef766d6),LL(0xbd647210,0xd395535f),LL(0x7a9398e6,0x2cf03d25), LL(0xa10ca7f8,0x00d0d0ec),LL(0x05dde357,0x384e9692),LL(0x66d09f76,0x86dfa64d), + LL(0x97c72efb,0x949c71bd),LL(0x3942aaad,0xeaebb176),LL(0x6b83fb91,0x2998f398), LL(0xccd511b6,0xa3fa1c7e),LL(0x8636cb8b,0x30452ba9),LL(0x0c63c213,0xfd01b34f), + LL(0x654f4b1f,0x161634df),LL(0xba6b7612,0x786d3d9b),LL(0xa3f145c5,0x7b731aff), LL(0x81bf5f5d,0x318ec0fd),LL(0x28ec1847,0x5aac3946),LL(0xef3f2a56,0xb5b34960), + LL(0x8a25550d,0xb8603afb),LL(0xca58e8f8,0x7d76ca1f),LL(0x52edd227,0xb124c5a5), LL(0x749a0bb0,0x4ff6045a),LL(0x166a72cb,0x9d1c4729),LL(0x18c9dd50,0xa1165c3d), + LL(0x32ec01e6,0xaf916e4f),LL(0x74270c7a,0x9b18b797),LL(0x2bde0960,0xc31c12bd), LL(0xe3a52afc,0x6fdbedeb),LL(0x99dcd3e1,0x2fcf26c9),LL(0x2ec4570a,0x15d634a7), + LL(0x02e98f6e,0xe27fb84e),LL(0x016e8348,0x19815362),LL(0x5ee51a3d,0x72e0c390), LL(0x36959aff,0x5f3ea99c),LL(0xe880a6fb,0xec5b0f2d),LL(0xac887faf,0xaecf25fd), + LL(0xca7c485f,0xd0640e47),LL(0x8c826885,0xa0eae2dd),LL(0xf0750194,0xfcd0e169), LL(0xe05f5115,0x04c965ed),LL(0xd7ba9d0d,0xe9482a7a),LL(0xd261b83f,0xf2660cd3), + LL(0xf53ab896,0x722e7324),LL(0x83e9fccc,0xcabeeaad),LL(0x0886fae1,0x332ff124), LL(0xfa23970c,0x0e8c2e7d),LL(0x433ab568,0x26f580ee),LL(0xcdcf0e3e,0x7cdcc1f4), + LL(0x513d4695,0x83ac29d5),LL(0x9101ca09,0x525837bc),LL(0xe8e10f27,0xecd93292), LL(0xeea1edfd,0x312f9d84),LL(0x83f862a3,0xdf9354ea),LL(0x15909df4,0x0f9f120a), + LL(0x5c61ed01,0x16fa2ab0),LL(0x816f7ce0,0xeab42f96),LL(0xf1cbe36b,0x1aca9b66), LL(0x1763d0c8,0x7fb3d70e),LL(0x7c64a8fc,0x5fb8567b),LL(0x45fb302c,0x08b686bd), + LL(0x6270b30e,0x0124d430),LL(0xcf5b5235,0x566f1fd2),LL(0xbee07063,0x923da430), LL(0xe2c6736c,0x2ecebb06),LL(0xad6d6ee4,0x3a3fdafe),LL(0x82bd6945,0x16522430), + LL(0xd6d11f78,0x0d162ae9),LL(0x69047eb7,0xddb375c5),LL(0x926e0966,0xd125f66b), LL(0x100fb84c,0x1cfafcac),LL(0x40c50ecf,0x1f4c7fd4),LL(0x499513b2,0xbd8ac3ec), + LL(0x5feaf69f,0x937412eb),LL(0x641cace4,0x760d3378),LL(0x203e5367,0x9bcec28e), LL(0x45fa3774,0x457f3a0d),LL(0xdf85b5f1,0xec51fd4f),LL(0x37d174ba,0x47390235), + LL(0x0a893990,0x7df7dee7),LL(0x8f08f682,0xa525af4c),LL(0xfae95f9b,0x9edbb45a), LL(0xcdc01949,0x58f3b29d),LL(0x773a1cde,0x9afdfd03),LL(0x8b16ab71,0xd6077f9e), + LL(0x13611c0d,0xc96eabbc),LL(0x7581f50a,0x8af0d512),LL(0x93c963cd,0xf75a7835), LL(0x9060a883,0xb7487030),LL(0x17ceb72c,0xfc03c9b1),LL(0x79773be4,0x64773851), + LL(0xd416e391,0x8f73e056),LL(0xb0289136,0xf9c29801),LL(0x1ebcb0de,0x85fd38b0), LL(0xf0af703b,0x9928f60a),LL(0xbbd1143a,0xd77a236c),LL(0x2b99b61f,0x120472d4), + LL(0x5f4a504c,0x0e448282),LL(0x5cc9b39f,0x1fca16bf),LL(0xdf0166de,0x4e47e87a), LL(0x76644e8f,0xf11ccae7),LL(0x4b20ddcf,0x84113220),LL(0xe274c42f,0xae2e75b4), + LL(0xf97bdda6,0x604a226f),LL(0xadbee2bb,0xadd35dc6),LL(0x986e403e,0x8b7db9ea), LL(0x6ad6318f,0x9007482c),LL(0x891688d9,0x22b19d05),LL(0x4d848a95,0xe974ab42), + LL(0x88cc05d4,0x339df4aa),LL(0x5855f83a,0x14e21ec8),LL(0xd790d568,0xddcd7ff2), LL(0xcf09873b,0x5c8a6e40),LL(0x134b7a3c,0xb76d74a0),LL(0x07549e84,0xd05b80d6), + LL(0xc5f26048,0x845e4b79),LL(0xeac9a450,0xcb338886),LL(0x9b807f9a,0x78f027d1), LL(0xb02ce220,0xf81fb903),LL(0xcd541a4a,0x2bc100d8),LL(0xb3d3cbfd,0xf10cf19f), + LL(0xb198d5f3,0x74645ba6),LL(0xcebf9671,0x9fbabd70),LL(0x5d7a8816,0x57cf26bc), LL(0xc18901bd,0x415d9e95),LL(0x08448312,0xa71715a6),LL(0xe80f10b3,0xbb1d05fa), + LL(0x488ffac2,0x6065bcb2),LL(0x21393e2d,0x02526899),LL(0x4f000282,0x259f0d34), LL(0xf12f0afa,0x10a4333c),LL(0x62a1dba3,0xaa0b50d7),LL(0x8cf13d7e,0xe09542dd), + LL(0xc2ba51fa,0xf15ba587),LL(0xcdcc4e52,0x72f2bc15),LL(0xe3ca3356,0xd1c3673d), LL(0x3028612e,0xabbfd1f0),LL(0x9def1d98,0x4ab9792d),LL(0x4815b9f7,0x02e11fbf), + LL(0xf804ca53,0x24940965),LL(0xd69ecbb9,0x5073162c),LL(0xdccb4f88,0xe078c3a3), LL(0x1f34a5f8,0x06f13d89),LL(0xfca14e1e,0xa848dd8e),LL(0x8c4c3fac,0xb989d0a4), + LL(0x3f31ee3d,0x670aa037),LL(0xc8999458,0xf72dfbba),LL(0x4698301e,0x36e3a8b0), LL(0x1a579aef,0x26b32ad4),LL(0x98b2b956,0xdff0c7ec),LL(0x77953721,0xa732a2c4), + LL(0x436265d5,0x0ff83c66),LL(0x06547c6f,0xfa5166e1),LL(0x5bfd456d,0x03018748), LL(0xf306cdd5,0x1b4ec2b6),LL(0x4b56f91f,0xe064b766),LL(0xa5805ee3,0xfdeb932d), + LL(0xc49208e8,0xa583f200),LL(0x369d91c7,0x34a07e82),LL(0xa89b1fd5,0x3fe43f6f), LL(0xb439d920,0x76aecf08),LL(0x4547896e,0xf76a9539),LL(0x3019148f,0x3c69e2e4), + LL(0x41fc2ca3,0x79e74959),LL(0x6a5af93e,0x09a99e14),LL(0x8293aacb,0x6610c0ad), LL(0x84dff17a,0xc56e9614),LL(0x0ff4cf52,0x0a4f0290),LL(0x35902cdb,0x50fd4a21), + LL(0x7297ff6b,0xc526cdd2),LL(0x47459c97,0x2c8409af),LL(0xb7537567,0xbd3097f3), LL(0xb47675e9,0x5b29ab79),LL(0xfd4e344a,0xded7e0b2),LL(0x0eb1e2c4,0x40324c92), + LL(0x9c381ce8,0x617df28b),LL(0x4475066c,0x6771e091),LL(0x82a57c2e,0x7ffb1259), LL(0x90dbf30a,0x6997a10e),LL(0x4603a795,0xa412e99c),LL(0x6da0de96,0xfb49f3ca), + LL(0x24c43f70,0x28fb7939),LL(0x9b346573,0xa7d31444),LL(0x20545584,0xdf8f9b6b), LL(0x8a9b1477,0x54464b03),LL(0xad8c0fa5,0x623ad546),LL(0xae0601f7,0x5cf44ed1), + LL(0x0f36e763,0xbdc4a6e7),LL(0xd3fb2441,0xaa5058c6),LL(0xbd11e44f,0x7603b5a1), LL(0x6c72527d,0x65a896f3),LL(0xd59c82f0,0x214ffbf2),LL(0x0ec375a1,0xf886b405), + LL(0x6933ecb2,0x12e89a15),LL(0x2c1f2f43,0x3ed0ad89),LL(0xe0d5c6de,0xcafb442c), LL(0xe8cd1af4,0xb06cf2fd),LL(0x9c4265bc,0xb06e7452),LL(0xa73a0639,0x6af8b9d8), + LL(0x513e4297,0xc18ced3e),LL(0x7fab97a1,0xb4f2f3ca),LL(0x79cc3e70,0x16a64cee), LL(0x64e9eb0e,0x1f3006b4),LL(0xfedf5a33,0xf997d410),LL(0xffe83ec6,0x7fed67a8), + LL(0x3d30aa08,0x2452d81d),LL(0x1cd85802,0xcac330c3),LL(0x79a76f43,0x9a89abac), LL(0x17c27a7a,0xa965909b),LL(0xa62a311a,0xa1534cd2),LL(0x71a72a64,0xe4fe0317), + LL(0x261133ef,0x2d7c475e),LL(0xff4f2dd3,0x37952ead),LL(0xe360fb3a,0x10b0a5a5), LL(0xea3f4d28,0xb88323f0),LL(0xebeff23e,0xf1488cef),LL(0x8d265c93,0xa3b9477f), + LL(0xb76e90d9,0x57b098a1),LL(0x0449bf9c,0x988fbb42),LL(0x062f911b,0x95642d7d), LL(0x61b08116,0x788996fd),LL(0x09d205af,0x5b02272e),LL(0xb3d8f31e,0xdea85f72), + LL(0x09b907f5,0xdcc15267),LL(0xf7971f9b,0x784511d8),LL(0x20119354,0xe309478a), LL(0x4ce572f8,0x4d5e16a3),LL(0x2c62f607,0x7eb6eeed),LL(0x57f7d26b,0x6bcb2bff), + LL(0xd75c445c,0x2559d692),LL(0xb7443959,0x49d40028),LL(0x55e49cd0,0x5ee0d518), LL(0x33bb1b6d,0x0cade75f),LL(0x32bf7888,0xd8bac375),LL(0xa0531d90,0xa47d4b3b), +}, +/* digit=4 base_pwr=2^28 */ +{ + LL(0x58d5549f,0x881d4415),LL(0x7328bce3,0xed748e21),LL(0x2f1f8481,0x56f0d441), LL(0x021c9c2c,0x74ff865e),LL(0x93500f82,0xbaa63b47),LL(0xc2e3b807,0x0c40fade), + LL(0x9568fad3,0x6f93d120),LL(0x7ff0a77f,0x291875f3),LL(0xdd42f359,0x0608b32f), LL(0x644f9985,0xfee522d9),LL(0x05996cc8,0x4beb2bfd),LL(0x769aa367,0x5a77212a), + LL(0x504a6f07,0x96957fc9),LL(0x6acad37b,0xfd75c98d),LL(0x548f17da,0x2cc5e7b2), LL(0x7bdfaedc,0x039b865d),LL(0x1970dc2b,0x1ed21afd),LL(0xb4169fbc,0x93b309be), + LL(0xc373ba13,0xca0eaf9d),LL(0x798e41ad,0xe25f5f11),LL(0x1eca3ea4,0x5af4b3c7), LL(0xa23b9c8a,0x47bc86f1),LL(0xfae9d7b2,0xe9768f07),LL(0xed3e806e,0xf94f89f5), + LL(0xe4f24c73,0x43f87b32),LL(0x643fb20c,0x41c6cddb),LL(0x602b32c9,0x7093785c), LL(0xa13dc1f1,0xdf9d7247),LL(0xd42502e0,0x3a967995),LL(0x4df43de6,0xa0e28ce3), + LL(0xf53d962a,0xf3011fff),LL(0x1767f31f,0x9ef38fb2),LL(0x5f883ec2,0xa11413c6), LL(0x4c344917,0x84383633),LL(0x3f5cae22,0xca855976),LL(0x15b2fddf,0x929a1484), + LL(0xccfc0e1d,0x58946718),LL(0x809ffae5,0x651d4a6d),LL(0x87a653a0,0x6eeb8892), LL(0x6881e966,0x2c4c3c55),LL(0x6dfb6d35,0xbdb6d87e),LL(0x92113e28,0x7cf86719), + LL(0x6203f1ec,0x6451f3e3),LL(0xab59d68f,0x40681d08),LL(0x66af6fee,0xd0bf9608), LL(0xc1e4b6a6,0x5df24a1f),LL(0x5e6a069f,0x64c1b0f2),LL(0x63fbf5bd,0x124e2748), + LL(0xedf146a2,0xe6cbf0bf),LL(0x085c71de,0x85e47e89),LL(0x603e3ba3,0x39330eae), LL(0x6841460f,0xbcd9f912),LL(0x55ffa24e,0x4212f2ba),LL(0x3fb12ad2,0xadf49ddb), + LL(0xadb7dfa6,0xfb02103e),LL(0xbabeeea2,0x4dce5edb),LL(0x622cfe41,0xe4e76c1e), LL(0x5d2cde0b,0x318e8d34),LL(0x4448a054,0x5d3e168a),LL(0xa23e5457,0x444ff058), + LL(0xa8083fb8,0x06020b4b),LL(0x73e1d9fe,0xc6d07af1),LL(0x137f9872,0x2056f42e), LL(0x523b9b18,0x5ee31758),LL(0x39042106,0x58771cbe),LL(0xa0e5b696,0x114531cf), + LL(0x8a9552c6,0xa4eedc90),LL(0x6eb5fedf,0x23be4e8c),LL(0x5ad0744a,0xe5f6ada1), LL(0x4ca7d6f4,0x373a4a9e),LL(0x57cea5a6,0xb5a5a64d),LL(0x7f48cb04,0x3a65f463), + LL(0xc83aaf37,0x73506b7f),LL(0xd4bce539,0x9785f792),LL(0x7df31e2d,0xebbfa1b8), LL(0xd5a2ea75,0x3b66c6b3),LL(0xf2196549,0x32ac5d33),LL(0xf8082490,0x89d1486d), + LL(0xf9fe7377,0xc0c2ec19),LL(0x39415cfb,0x07911ff1),LL(0xa1c91f71,0x1b54a66d), LL(0xd4f59851,0x29646a9e),LL(0xe209ab6d,0x0a7fcf62),LL(0xfca0de15,0x6ea260fe), + LL(0x5fcc220a,0xd026ef4d),LL(0xd9364e42,0x7303fde9),LL(0xd35f5fad,0xcf2f621c), LL(0xd439a9f6,0x6e346476),LL(0xccf2de6e,0xb108eab2),LL(0x6d3db6c6,0xca46cb58), + LL(0x1de59604,0xc89fccf4),LL(0x59f8e37a,0x72ee2825),LL(0xd52aa0eb,0x9423de94), LL(0x1110f4b8,0xb6837166),LL(0x5e0ca5e7,0x00b22047),LL(0x7f72b3f8,0x79573c27), + LL(0xe67b0c29,0x6dfbc44f),LL(0x05177565,0xa0a8e62f),LL(0xbf5b1bbb,0x7f494f35), LL(0xd472f2c7,0xcc727439),LL(0x48ea2e18,0xcafc4607),LL(0xc5b2f1e5,0x25499b27), + LL(0x86c1e086,0x83fb7e04),LL(0xafbb1111,0x19eeb012),LL(0xa0c8df49,0x620d5c9f), LL(0xd8adc0b6,0x85368dc4),LL(0x0c2f7efe,0x174dcd3c),LL(0x52fe89c3,0x71d50509), + LL(0xbbeb2341,0x03ec18cc),LL(0xc49153aa,0xf386e999),LL(0xd01e004b,0x58b7a90b), LL(0x3bf5753d,0x66713f74),LL(0x23ce4a5f,0x96fa5445),LL(0xa7a2e6a6,0x06b17741), + LL(0x1fa5345b,0x93e18885),LL(0x2b10748b,0xc3faca64),LL(0xdf987dbc,0xc1b82b06), LL(0x205e25f4,0x8f3aa4d9),LL(0xb7dc364e,0x31bdcd03),LL(0xb91f4dfe,0x5a21cbcc), + LL(0xbc5b1ae6,0xc9f8dd3e),LL(0x09390a2d,0xfd14096e),LL(0xd6fbd5f0,0xcf2461e3), LL(0x02a263e4,0xaa775c22),LL(0x0fbb901d,0xc7829c8b),LL(0x1840557a,0x338e09b6), + LL(0x1c6d5d1d,0x6aa505da),LL(0x5646e930,0xf7906394),LL(0xa52fc317,0xfc0035dd), LL(0xb185221a,0x9fefcec6),LL(0x49de64e9,0x692267cc),LL(0xc316de61,0x2d105157), + LL(0xd4e3b784,0x57ac5de5),LL(0x13ab394f,0xac9c4222),LL(0x027f1648,0x5aa27972), LL(0xc83390e1,0x7a4b5ae2),LL(0xa1a47d58,0xba9ff075),LL(0x4a3d0cd8,0xf9ad2400), + LL(0xede14e49,0x70ac2786),LL(0xbdf4b5ad,0xd79e0890),LL(0x01edd96f,0x7b6dd833), LL(0x565fd682,0x1b4f533d),LL(0x474f71f9,0x9222c9a2),LL(0x83ebda25,0xa56581a9), + LL(0x25ee5573,0x7050f5eb),LL(0xdb88f4cd,0x4758c876),LL(0xc852b937,0x9abc5336), LL(0xa5b25a0b,0x88d32963),LL(0xf471412c,0x3e117bb5),LL(0x081986d4,0xf4fe7f16), + LL(0x976cf433,0x1a1189da),LL(0x7202a031,0x8d95553c),LL(0x844752a4,0x59d33ccf), LL(0xb361b89b,0x8145a133),LL(0x43423d20,0xf78e0d37),LL(0x61a44de5,0x522e94dd), + LL(0x8e540c0b,0x7c2237f8),LL(0xe30c14a2,0x89d8e57c),LL(0xbf6aa5e6,0x456a042a), LL(0xf3216fe6,0x22ab0e28),LL(0x5cb9a52f,0x54763652),LL(0x0b10595d,0x5ee60169), + LL(0x3800818b,0x06cc9e76),LL(0x7f5020ad,0x426cc669),LL(0x18577c5f,0x53ff7b9e), LL(0x30789f9e,0xbecc9a3d),LL(0xac798e8a,0x685f1d84),LL(0xcf85daa7,0x3c0745ce), + LL(0xdfd3615b,0xd516bc63),LL(0x9ad06650,0x36e8a2de),LL(0x9a930d91,0x71f2ade0), LL(0x88d6e03e,0xe237231a),LL(0x80cd199b,0x8af22a7b),LL(0x68638e0e,0x999ac96b), + LL(0x3b855f82,0x92421da5),LL(0x229553e1,0xdf39524a),LL(0x389ff1eb,0x388c4741), LL(0x49806f26,0xb73b3db2),LL(0x116a75da,0x0066ae14),LL(0x7d37c100,0x224f589b), + LL(0x4defbdd3,0xc60c22e1),LL(0x5449bd64,0x6d574468),LL(0x46ea4c3b,0xee1e887e), LL(0xc608f1e7,0x16b1d73e),LL(0x87301c1a,0x30a39cb8),LL(0x76725f2b,0x5e7728f2), + LL(0xbeda23d1,0xd3a7aed2),LL(0xaf09fa81,0x2d4adeac),LL(0xf998500d,0xcc705a59), LL(0x97b62945,0x2ec67209),LL(0x2156a3f4,0xfefbf4e8),LL(0x0a5fb347,0x20684eab), + LL(0x785d1eb6,0xeb66a59a),LL(0x82fd2480,0x98279fc5),LL(0x1a00554b,0x6b259c82), LL(0x913e2492,0xa46b9ebf),LL(0xea3691cc,0xc3524af6),LL(0x6746a57e,0x00ec234a), + LL(0x71718339,0x33a6b51e),LL(0x206c1075,0xbffbd973),LL(0x0461a56a,0xd7489944), LL(0x5ee41c8e,0xb4440567),LL(0x7b86fe5a,0x2bf38abf),LL(0x8cd45ef5,0xf70c63a1), + LL(0x845aa2a3,0x80c95a21),LL(0xd508e765,0x4882eaac),LL(0xcb5bb20c,0xc5f7b8fe), LL(0x104b3369,0x96426128),LL(0xd131d49d,0x23598393),LL(0x27af509a,0x059d6d87), + LL(0x330dc205,0xecb4bd8d),LL(0x793b3824,0x3934dce1),LL(0xf505c154,0x2b8cedbe), LL(0x7a873a69,0xba92295b),LL(0x386ee396,0x8449398b),LL(0x4717b4b5,0x72584bae), + LL(0x73765fd1,0xa18a909b),LL(0x7db9952c,0x2494a304),LL(0xe91c490d,0x2747b58d), LL(0xb6d10153,0x1178cdf0),LL(0xbf4c5889,0x35235125),LL(0x90f823d5,0xdf63fa73), + LL(0x4066c1a7,0x38748fb7),LL(0xf5cf90a5,0xfbfbb8e2),LL(0x9b83fd71,0xa828018b), LL(0xb34ce140,0x3de7f633),LL(0xd54824f7,0xd760a3e5),LL(0x94e2e012,0xea872fd7), + LL(0x479fbc3f,0xd2aa4fdf),LL(0x1979cfcb,0x11a9a799),LL(0x03e344fe,0x01733a8f), LL(0xb5b3cc49,0x0c61b034),LL(0x10c7d2cf,0x27483deb),LL(0x1884d1f1,0x6dc92d21), + LL(0xdce73a42,0xebf1943b),LL(0xa62405be,0x91a018b4),LL(0x5684989d,0x844d759b), LL(0x4678912e,0xbdc609b4),LL(0xd12582c1,0x4b151529),LL(0x7f8e1390,0x7ac0cfc4), + LL(0x987acd67,0x5245ca01),LL(0xf023cfd4,0xd4688d7c),LL(0x4b96af1c,0x799e7c06), LL(0x0a9e14c2,0xa96c4571),LL(0x2ded73d6,0xc2dc67eb),LL(0x2783e2b5,0x43588841), + LL(0xeef15ae5,0xa3b115c6),LL(0x87b86741,0xf305a27e),LL(0x4711a4a3,0xa5fdc849), LL(0x7b6ef648,0xeb58f0f1),LL(0x6f3ea084,0xdccddcbe),LL(0xa1b862a7,0x6f7470fd), + LL(0xd88e9f3d,0x141947dd),LL(0xfbb2d23f,0x1b677675),LL(0x16da518b,0x12a4c8c1), LL(0x032c0bc7,0x1cccf6b7),LL(0x20ed64ed,0x5b7cfe57),LL(0x5269cf39,0x7d427f2c), + LL(0x8d53b039,0x7faea5da),LL(0xacb5b11a,0xf19f5f61),LL(0xfe27af9c,0xfd679e18), LL(0xc0e391aa,0xbe29ec0c),LL(0x8cef3cd7,0x42a815d2),LL(0x42c2564f,0x3dd43747), + LL(0x6370d284,0xbaa0ccdf),LL(0xeb70e670,0xb68dae3c),LL(0x9917aadb,0x5e25db51), LL(0x57c8020e,0x27888369),LL(0x0cd601a7,0x4ca76fa7),LL(0x4509eeb6,0x46794e0a), + LL(0x4538b5b5,0x9f21619a),LL(0xd6754ebf,0x6c1f3f7b),LL(0xb7b2f926,0xa2a0e2e7), LL(0x42728394,0xc5747d30),LL(0x0a0328e3,0x2662cd33),LL(0x3bfbcc49,0x89c445cf), + LL(0x96bba2f7,0xce07c166),LL(0x93dfae9a,0x6828eab6),LL(0x4036abaf,0xa907dcce), LL(0x6cd5853d,0x61e23034),LL(0x331dc277,0x541d6f4e),LL(0x82796b5a,0xdaa10e8c), + LL(0xfac61d43,0x339d6949),LL(0x75b34910,0xedc8bc2f),LL(0x1ee6f866,0x4c686fe9), LL(0x5f02368e,0x9bc462bf),LL(0x638abf60,0xc3902774),LL(0xb7ed5413,0xc403b179), + LL(0x664e0b41,0x35bf0ed9),LL(0x889af4e3,0xc3ad5755),LL(0x5153a58d,0x14291828), LL(0x64798a76,0xb3d9c453),LL(0x827c237a,0x71e6ce15),LL(0x41e52b67,0xbbcce06b), + LL(0x1a05cd67,0x523df124),LL(0xf104c7c7,0x969598b8),LL(0x7f7b1fa6,0xc16d7bc1), LL(0xfb1a2aa4,0x935b18e9),LL(0x4f99ebf4,0xba7ff2a2),LL(0x862cab62,0x0613fa7b), + LL(0xd8d26e75,0x02f5cfb1),LL(0x98906c57,0x5e0d4363),LL(0x1032e39f,0xca80873c), LL(0x204c0680,0x2fa5b881),LL(0x7c178dd3,0xca227906),LL(0xdf2e321b,0xba33bb48), + LL(0xd9a13914,0x4cbde5bb),LL(0x621b5b3c,0xaa89d2d4),LL(0x75821515,0x66fd1db4), LL(0x66154f11,0x3e616070),LL(0xed85f2f5,0x2dde1732),LL(0x528e1b57,0xfa1fe78f), + LL(0x600e571d,0xb51f5c95),LL(0x15059c58,0x13f0e99e),LL(0x72e8f443,0xe2a1a34e), LL(0x2285a044,0x5c484292),LL(0x908d9855,0xac4b6bdd),LL(0x8cffdb0a,0x4b4ac5ae), + LL(0xd691153c,0xf8f74a29),LL(0xd21533bb,0xc0cb0e05),LL(0x86067d1d,0x52ceb52e), LL(0x9cd93bda,0xeb021228),LL(0xd7c98d3b,0x27d7ef30),LL(0x78fae7b9,0x7c18f1bf), + LL(0x5ef7655c,0x1b3562c7),LL(0xb552fd07,0x62ce6adb),LL(0xb9c6217d,0xac6f179d), LL(0x7fb8394e,0x3b329c45),LL(0x9b340753,0x418d2ac5),LL(0x68c5c9d3,0x5adca639), + LL(0x15b71702,0xbfd6f204),LL(0x569ed998,0x292bfd32),LL(0x003d0141,0x5b5d5b09), LL(0xefcd36d1,0xd512f4d8),LL(0x16c87812,0xc81f1c34),LL(0x1777dcaa,0xa54cef3e), + LL(0x76585d99,0x53830298),LL(0x30d1c285,0xe2d41a7b),LL(0xad60f2ff,0xb6eabff3), LL(0x3d361b12,0x073639c4),LL(0x16f921f3,0x10e605bf),LL(0x3394e5eb,0xff665c90), + LL(0x359bf1b9,0xadae33b1),LL(0x61554e45,0x183beb7e),LL(0x559e3f90,0x28d50fcb), LL(0x4bc0b8cd,0xa9a26299),LL(0xb403463a,0x25b24e6a),LL(0xf076ebd6,0x715055a6), + LL(0xd381a2e0,0xa6b7fe33),LL(0x6b248193,0xc2aacec9),LL(0xc9d6ba70,0x1f617084), LL(0xcd06c9a1,0xce945304),LL(0x1b4c3d5b,0xb957b98b),LL(0xfa6b8f04,0x50c61d1f), + LL(0xdaa47b46,0xbd69e6d9),LL(0x7a9479bb,0xc12cbd8a),LL(0xa8b92f63,0xf07cbcd9), LL(0x27d2f85e,0x4998f584),LL(0xbf43ba36,0x0ab2e076),LL(0xbad0d0a2,0xbed73de0), + LL(0xad129e1b,0x3e02902b),LL(0xa008c72c,0xd2e0bd57),LL(0x30e61c2b,0xe88a075a), LL(0x5d9402a8,0x136e1b7f),LL(0x7bc0a653,0xaa06f30f),LL(0x068e6e0b,0x51b52693), + LL(0x695404da,0x0bb86a49),LL(0xb1059de5,0x339cbbca),LL(0xdfc1bfa7,0xd45db64b), LL(0xa98831bc,0xad07e52d),LL(0x51a7a71a,0x5ef54a92),LL(0xee02c742,0x7fda785f), + LL(0x6686a65a,0x05b7848e),LL(0x6f478ff9,0xc12a7893),LL(0x2d137586,0xf1f54c31), LL(0xd50ea873,0xfe8799f4),LL(0x5fabdd98,0x62d996af),LL(0x5181afce,0xf0fe39fd), + LL(0xef010cef,0x646b35f9),LL(0x7d54d120,0xa63852dd),LL(0xadec6e34,0x3e5c14f4), LL(0xc2f4b7e6,0x030270aa),LL(0xec560a92,0xdedf4875),LL(0x9da64c53,0xd6d97f65), +}, +/* digit=5 base_pwr=2^35 */ +{ + LL(0x1386b66e,0x18754788),LL(0x9a2eb0d8,0x783dc08a),LL(0xf074e5e1,0xfe2682d1), LL(0xcbf4f5b8,0xb6ab7696),LL(0x6da5b0aa,0x382401e6),LL(0x0f93d45d,0xfd5a6375), + LL(0x6fefd17e,0x51d517b8),LL(0x18fba339,0x35e97b40),LL(0xcf221c7b,0xbdc8467c), LL(0x9eaee11c,0xba6035ff),LL(0x18128100,0xb417857d),LL(0x4f40acec,0x383235c3), + LL(0x4197bc61,0xdcfc9f38),LL(0xfe9c1104,0xeb61eb5e),LL(0x5486c84c,0x7ba40c29), LL(0x1377a769,0x2f6a8ebc),LL(0x914c369a,0xe4e70f07),LL(0x57afc5f6,0x8d954007), + LL(0x0fa7e4d2,0x56b010d6),LL(0x1d9544ef,0x91cd7d72),LL(0x48b3fb02,0xe2c51636), LL(0x683588bd,0xa2440769),LL(0x98239b52,0x7054bb40),LL(0x3bb5090c,0x2019db99), + LL(0xf043605e,0x6da23d19),LL(0x69d16792,0x4f2d4a4f),LL(0x6d5127a3,0x5061d4c8), LL(0x71c846bd,0xa0ef332c),LL(0xca929e1f,0x7948a241),LL(0x0baf2f1a,0x4152aec5), + LL(0xdc72a138,0xf15dd5cb),LL(0x91a32dbf,0x6d2d2597),LL(0xfd5852dc,0x7f46cffd), LL(0xd817364b,0xda270e8e),LL(0xa1df99e0,0x97e5e85f),LL(0x5a90c0ae,0xc77a3a0a), + LL(0x19a3eb23,0x1636651c),LL(0x12294385,0x34a5e78a),LL(0xec25d3d6,0x71cac725), LL(0xe5aa03cc,0x98956393),LL(0xf1d2e766,0xcda62f63),LL(0xf7c046ed,0xb8c8909f), + LL(0x1578b888,0xb68a6a3a),LL(0x30e1dbfd,0x01e39894),LL(0x71dafb3f,0x03ca0487), LL(0xf8241fab,0x7690a577),LL(0xe64d2bb7,0xac71ef0b),LL(0x3eae3938,0x4fd69e13), + LL(0xa2e02fc2,0x7a04ff5d),LL(0x75b01d9c,0x11114bbf),LL(0xe08f4299,0x1233f026), LL(0xf491d93a,0xd888bb1c),LL(0x07c1f414,0x7c6fc026),LL(0x97d8d7df,0x753a2913), + LL(0xea21ec86,0x0fe566b4),LL(0x75ddae93,0x743f5133),LL(0xee43927e,0x0173fdd9), LL(0x8b3a99d1,0xe9491f6e),LL(0x90e9d60e,0x6af11d34),LL(0x373fafeb,0x7ca50de3), + LL(0xb53b609f,0x3a3eaa6d),LL(0x1cf1fa53,0xcafff213),LL(0x2cc998b7,0xb64116ef), LL(0x8eac5d3a,0x63148b28),LL(0x1dc633b7,0xff5f7662),LL(0xc94b635d,0xf174dcd1), + LL(0x013644fb,0xdf0f0994),LL(0x526fa9f2,0x77369779),LL(0x4197eb1b,0x41ef1970), LL(0x22cb653d,0x7bf4bf00),LL(0xb7dd509f,0x94f0f226),LL(0xbe77682f,0x5fd7dd2b), + LL(0xf658d2b2,0x80435f56),LL(0x5f8cc779,0x1e864847),LL(0x8b56aaf0,0xdd4cb7e9), LL(0x72c3024f,0x458c8bb7),LL(0x8cffdca0,0xa5c186c5),LL(0x377f96ba,0xc0022799), + LL(0x1f7d9343,0x7e4d4404),LL(0x299cd36e,0xa496646a),LL(0x600decf3,0x06372825), LL(0x08bf05fa,0xe19b609d),LL(0x1b7e153a,0xfe6a7182),LL(0x9808f5c7,0xfe191f76), + LL(0x2a8c6b66,0x2c13eeb6),LL(0x0fcafcfc,0xcd3cc6d3),LL(0x236f08ff,0xb85a615d), LL(0x14e4e83c,0x92187d5e),LL(0x21979d04,0x5fdaaa79),LL(0xe519147a,0xed2d2008), + LL(0x02fe620c,0xd8083896),LL(0xde8984fb,0x277803e8),LL(0x7058d34c,0x2698ed9f), LL(0xbce1dcc3,0x00b9c0d3),LL(0xbc2c1c85,0x356b310e),LL(0x67cacabb,0x41e412dd), + LL(0x2790dc4a,0x396267ca),LL(0xd5bfe8c6,0x5d4593a3),LL(0x335a7f8d,0x5fddd3d4), LL(0xab0ac92c,0x92f002c0),LL(0x6226c031,0x590a93b2),LL(0xdfe775be,0xb3c10c49), + LL(0x85889144,0x0209bd99),LL(0xb9b83e2d,0x77df1ced),LL(0x4cca6a2d,0xbf9f5870), LL(0x35978ee6,0x2c4f310e),LL(0xeae8acff,0x98debae5),LL(0x778c9461,0xf2f40fc1), + LL(0x9d363814,0x4c6c9e7e),LL(0x7b9d3d53,0x52fe5e06),LL(0x9cecbe7a,0x19bd3915), LL(0x272c9325,0xcbce0a07),LL(0x6a7c9eaa,0x48336f5c),LL(0x77a17578,0xd234b427), + LL(0x3ca7e0eb,0xb22492a2),LL(0xc5a32881,0x91ba7704),LL(0x00d1425b,0x5a2a7f1e), LL(0xbc09eeae,0x6b15d61f),LL(0x86b7bff1,0x2e8f822c),LL(0x32c4dd9e,0x56357b66), + LL(0x51e3d663,0x74a1f2c6),LL(0xa42a3633,0xf42faf2f),LL(0xf6e193fb,0x1b1be8b6), LL(0x4200b0ee,0xb647f284),LL(0x02d4935e,0x0950b424),LL(0x4deee646,0xdbdd7c52), + LL(0x77c3a95a,0x15c8ddba),LL(0x6ba71f9c,0x901a11bd),LL(0x440741f5,0x54fc905b), LL(0xf1358db4,0x4b6b0fc5),LL(0x6ed93612,0x3c4c39a3),LL(0x395cb400,0xa764c9a8), + LL(0x2b54f47d,0x089cc68e),LL(0x4fa19e45,0xd6f94042),LL(0x62803448,0x52875bc1), LL(0xdf571d9b,0x1f43a201),LL(0xff9a0750,0x9e8645cd),LL(0x61a710d0,0xa177f4bb), + LL(0xddae695a,0xd55cc687),LL(0xe7471217,0x90d2c914),LL(0x917d18a0,0x884f4e6d), LL(0x744d11d9,0xd2871c7d),LL(0x02972782,0xef60b7d8),LL(0x2531f9a0,0x491aa401), + LL(0x8f0f3404,0xf881970a),LL(0xe6a7678b,0xb681fd66),LL(0x45c5eec8,0x673a8192), LL(0x216460f1,0x39e65db1),LL(0xee6d42eb,0x28cd720e),LL(0x08badd82,0xcf155154), + LL(0xf5c7d936,0x35ab0302),LL(0x7eee29e7,0x0a6d5f94),LL(0x7ed08644,0x03232350), LL(0x6d2a8356,0x505064a6),LL(0x979af3f3,0x6e9937d1),LL(0x0758cb3d,0xec148fcc), + LL(0xa047cb43,0x46a50ee9),LL(0x2c02de4c,0x6c6b569f),LL(0x3d95c934,0xcd21154d), LL(0xd55720da,0x3a14afa6),LL(0x5512e601,0x6ceee63b),LL(0x4c47b641,0xafe6bbff), + LL(0xf19e974d,0x717ac089),LL(0xf71eadf4,0xfb0cebe8),LL(0xe85c30e2,0x503535ba), LL(0x072130ee,0x05e57133),LL(0xa8037e5f,0xf33e81ed),LL(0x3f0c555a,0x4b33d96d), + LL(0xd71f6adf,0x47ff2d79),LL(0x7a985d02,0x88a9de0d),LL(0xb2dc0cc5,0xc9cc2822), LL(0xfbc2972b,0x38052b28),LL(0xbc8465ba,0xdaecc9da),LL(0x7ff5a249,0xaaffa0d1), + LL(0x60361946,0x4f326618),LL(0xae1695c7,0xe7eaffff),LL(0x1311036b,0xa7aed10b), LL(0x9eb2dfa7,0x6eb81e71),LL(0xf680fea3,0xeda95d34),LL(0x83630d7a,0xd2ab66c5), + LL(0xb891fc22,0xa7433cf1),LL(0x1111e0db,0xdbea5480),LL(0x658341c1,0x823a8595), LL(0xe2d8917b,0x7a171afa),LL(0xe713e015,0x79e4154d),LL(0x3ebc430e,0xbd07aa2c), + LL(0x0523f8b0,0x091a19d2),LL(0x0950ab93,0x9fbf9c8c),LL(0xfbb731ea,0xd8377e50), LL(0x18b26813,0xe3b8b5bd),LL(0x1bc1257d,0x78a16f16),LL(0xcfd383ea,0x837106a0), + LL(0x11c7579c,0x01e9427c),LL(0x1580ef3e,0x24504560),LL(0x0b447ba0,0xd7408db5), LL(0x0e1cfc39,0x6749f5e6),LL(0xdfe158ad,0xd26d7dbf),LL(0xe7b17909,0x66d731fa), + LL(0xea86e123,0x46d3addc),LL(0xe46c37d8,0x3b52f72b),LL(0x8036a8e6,0xda02c7ee), LL(0xfeeaba0d,0x9f141b52),LL(0x0049be34,0x7a66cbac),LL(0xb9de2e84,0xc5b5d9d0), + LL(0x6d15b711,0x190e0e24),LL(0xc7ca01a7,0x4eec20f0),LL(0x7803ca5d,0x12ff14dd), LL(0xd59fb3a4,0x84991c92),LL(0xc64b50b6,0x86a00ae1),LL(0x175bbf50,0x6bf38341), + LL(0x5310953f,0x3cd33d82),LL(0xef7027f4,0xcf3d011e),LL(0x3eee08ca,0x10779dfa), LL(0x31f432ec,0x2c3228c4),LL(0x7f319ca8,0x94c51072),LL(0x3cfe5d7c,0x2ac3a2fe), + LL(0x4acc7cc4,0xe2ccc5e9),LL(0xff58443f,0xfedc0da8),LL(0x9567fea7,0x776014d0), LL(0xa2b90e62,0xbca6d7b2),LL(0x164be21a,0x11148f7f),LL(0xfec4a5f1,0xebbdf677), + LL(0xca5a2ef0,0x4bd9a436),LL(0xfd095c81,0x320744ae),LL(0xc307a816,0x3da008cf), LL(0x2f6801e7,0xc5cc3ff9),LL(0x4407869e,0xf0ba2f46),LL(0x213338b4,0x6eed1a15), + LL(0x274dd179,0x7fa61ef4),LL(0xa631cf50,0x6e6453fd),LL(0xb86291a1,0xaf1017f5), LL(0x9cf7ace4,0x757564ec),LL(0x2535cbaa,0x0fa1ec1c),LL(0xbc9b95bd,0xc03cec64), + LL(0x7686669e,0xfbcb2d77),LL(0x6cdcb0f1,0xf9bed26f),LL(0xb5a8f081,0x37aff505), LL(0x6053e4cc,0x1f258d7c),LL(0x45bc699e,0x86384c5f),LL(0x25a2f865,0xe0025c0a), + LL(0xb24a8eb1,0xea42c8e1),LL(0xdc2dd956,0xf670d8ea),LL(0x035bfb42,0x71d4c32b), LL(0xe7824aa9,0x213fce6d),LL(0x4b22f111,0x9b34d5b0),LL(0x3a14d9cd,0xa299994c), + LL(0x7ba4a4bd,0x975d551f),LL(0x32d65fb4,0x862bc216),LL(0x151762af,0xfaaf16ad), LL(0xcd1fe1b3,0x931eac72),LL(0x00f9e6e2,0xcd730742),LL(0x85ccbcf2,0x581b13c3), + LL(0x559cb856,0x013ed0f3),LL(0xd22cf5f3,0x5c1af1b5),LL(0x3badc3ca,0x11b231ec), LL(0x68ce4eac,0xc22972b9),LL(0x1a2f55b5,0xcb92e129),LL(0x038e230a,0xa61289eb), + LL(0x309e8cec,0x625ac394),LL(0x49b13bac,0x99dab1a7),LL(0xdf48a436,0x77940cbf), LL(0x78881636,0x963cb651),LL(0x7541e4c5,0x889ac28e),LL(0x5407fbf2,0x39918bd5), + LL(0x69209c9c,0x63d26894),LL(0x255dc3aa,0xb6b122cb),LL(0x932b582c,0x12866e42), LL(0xc373f8ad,0x0b0a7dd0),LL(0x1fe0f36b,0x2fed9f9a),LL(0x2345d1f2,0x6dbbf47f), + LL(0xdefc4c20,0xdf69b818),LL(0x2e3eb2cb,0xf31ff9a8),LL(0x0be0bfa6,0x946578a6), LL(0xe8b31f47,0xbecddbf2),LL(0xa45fadaa,0x86e3594c),LL(0x999c547a,0x56d2d2e3), + LL(0x3b5a1df0,0xcfe909ee),LL(0x94fd4014,0x4b62c64a),LL(0x258a8f81,0x7d6e3ae9), LL(0x5804ee7d,0xfc15381b),LL(0x71710702,0x3b444911),LL(0x5f18965e,0x46497ae3), + LL(0x33dd9555,0xfbb99cfe),LL(0xfcbeecde,0xc6e14198),LL(0x51da2e20,0x8d4e83b3), LL(0x75a0719b,0x2bc54a6b),LL(0x56b03328,0x1aba20d3),LL(0xede40907,0xd84a7db6), + LL(0xeae9291d,0x6f52a996),LL(0x33d0f1a3,0x507c3701),LL(0xafebeab2,0xb7809c3c), LL(0x80b978c3,0x301a5ed5),LL(0xd10f0edd,0x1ba9c42c),LL(0x150d6504,0x0e9f9f93), + LL(0xee90dfeb,0x56ff221e),LL(0xc6815c60,0x279a7940),LL(0x1b0208a3,0xa119572a), LL(0xd2710835,0x5bb04e27),LL(0x67414bf2,0x61d8d945),LL(0x6aec6a5c,0x440a1d39), + LL(0x1ecc000e,0x07e60a25),LL(0xe6efdc85,0x2325aab6),LL(0xf4872ceb,0x29438ae3), LL(0xe8d10241,0xeea2a3f2),LL(0x548a7556,0xd2ff366b),LL(0x2a6e27bf,0x85e8ada2), + LL(0x11a6e295,0xb10332e7),LL(0x609a424a,0x44c9ba0e),LL(0x986e93f0,0x08f222e9), LL(0x9dcb7c9b,0xeb8bea94),LL(0x1d5aa8a0,0x36e44e93),LL(0xa3e0dccc,0xe0deccc6), + LL(0xc22e314b,0x92be41b1),LL(0xaabbe524,0x388494bc),LL(0xbde2517d,0xaaf52dfe), LL(0x186fedfc,0x4ce61d88),LL(0x300b7bc2,0x9f29e844),LL(0xedab8ca6,0x03a2221d), + LL(0xda0cae4d,0x7c3f687b),LL(0xe7da757e,0x84dc6167),LL(0x85d25286,0xc0c08f69), LL(0xb4889367,0x17f7921d),LL(0xf81fddec,0x223c9181),LL(0xa6a05bf7,0x3b72e938), + LL(0x60326fa4,0x0ff61ea1),LL(0x97c51293,0xa54cc228),LL(0xf976ab17,0x7d25531f), LL(0xa8f84662,0x1e8c3f37),LL(0x007ac2f1,0x4d8c3063),LL(0x4598d053,0x80c8de31), + LL(0xe4198b95,0x3ed83a96),LL(0x90ef6fa9,0x4435dc6a),LL(0x9ab2a2d6,0x47791ebf), LL(0x1564b80a,0x3e0c8b07),LL(0xa0ec1ad9,0xb4079d88),LL(0xee9bfef8,0x6fc566af), + LL(0x7f1a5c21,0xbfc3ee41),LL(0x2727a00a,0xe3694ec6),LL(0xa4087b70,0x3ff4d181), LL(0x37de763e,0xda656043),LL(0x8a50ec66,0xc2da4f75),LL(0xb94a5c19,0x096c88ff), + LL(0x3121ec7a,0x05e200b8),LL(0xf23f234f,0xc5f282e5),LL(0x94aebbc2,0x0d95e5bc), LL(0x6922306e,0x389ff8bd),LL(0x0c458f95,0x949a4362),LL(0xa8a67759,0x2ed544e9), + LL(0xd643944f,0x46d3e81e),LL(0xa5cfb8de,0x36211db8),LL(0x95b4f959,0x8a5ee9eb), LL(0xcd19ebcd,0xc3d6941b),LL(0x61306b29,0x4bc41c81),LL(0xeac1ab81,0x0093af39), + LL(0xe6622883,0x301a7d6e),LL(0x270ac148,0x5c49e719),LL(0x4bbec99b,0xff3296f0), LL(0x80061376,0x880c775d),LL(0x322403c9,0x587357fe),LL(0xc24a4307,0x4db2f9fb), + LL(0x76f5e74e,0x3a8b93a4),LL(0xaa1b2dd8,0x5e9e5474),LL(0x5ad377ae,0x43c84363), LL(0x8b7ef6de,0x3ef3ceea),LL(0x7be9f363,0xa3457c60),LL(0x6c4f30eb,0x8a0c2c94), + LL(0x3eef5158,0xc50fa97a),LL(0x86d6a054,0x52c0d8a2),LL(0x3be5c327,0xebdf472b), LL(0x0852c07e,0xc34b020b),LL(0x42fa2f1c,0x5b2b3552),LL(0x3a4f7817,0xbff68be9), + LL(0x959b0d24,0x2cf2220a),LL(0x56dd4a59,0xeb164f48),LL(0x9d8fff97,0xd9927903), LL(0x233eaa77,0x0ddf5e1b),LL(0x7969f9e5,0xf32d29d8),LL(0x5562cc50,0x26ee53ae), + LL(0xf53c8cf7,0xcbc8f03f),LL(0xc7ff11a2,0xb6321218),LL(0xb9346795,0xb37d632a), LL(0xaf7e83a5,0xbc24e219),LL(0xe6dbd3da,0x95b4863a),LL(0xcfb3f25e,0xc3f00af4), +}, +/* digit=6 base_pwr=2^42 */ +{ + LL(0x097b6df3,0x807a7671),LL(0x0fb780e0,0x41faf5e2),LL(0x157c3ef1,0xdaa12450), LL(0xd7cb0c23,0x31d72356),LL(0xb251a877,0xd5d2c185),LL(0x92fa894a,0xdbdaee99), + LL(0x98704ce6,0x85e18802),LL(0x306d7501,0x25a349dc),LL(0x03de69b2,0x6c5ba821), LL(0xc403903d,0x9fd2d533),LL(0xbf5698ac,0xe3b96ca6),LL(0xcc20c102,0x7d5e41c1), + LL(0xcc888fdc,0x6d434dd0),LL(0xed221a04,0x47e731b4),LL(0xa107a3c9,0xda0d85b5), LL(0xebf163d0,0x489780a7),LL(0x9892bf9b,0x1ddd62aa),LL(0x04e0db7d,0x7f988bcb), + LL(0xe492a9fc,0xb55c1496),LL(0x597a9eb3,0xd0cd80f2),LL(0x9f1d8391,0xeaf34b51), LL(0x785286c4,0x81e7ca50),LL(0x6c0c5297,0x5b63c071),LL(0xb1b52561,0xa09ca533), + LL(0x19fafaf2,0x082de97f),LL(0x25cc03ed,0x92de16ec),LL(0x0fabb7cf,0x415b7e1c), LL(0x721659a0,0xb121761c),LL(0x4a3f2f1d,0x2c646d3f),LL(0x13cef73d,0x6c441f7c), + LL(0x26cc7cca,0x7d4281a3),LL(0xe302e412,0xf9d8fdf7),LL(0xed1531b3,0x9b29a767), LL(0x8daae632,0x9713865b),LL(0x51561f0a,0x5c4d8193),LL(0x74daf2a8,0x786d917a), + LL(0x08c24f78,0xf4b82a3f),LL(0xe3565350,0x82c57c65),LL(0xc56bf74b,0x92213214), LL(0x066173e2,0x51cf2f2a),LL(0x22931b9e,0xb67e7f28),LL(0x0e448ae2,0x87983d74), + LL(0xcd93498b,0xa08d39d8),LL(0x81fb94c6,0x98c24fae),LL(0xe2ec50b6,0x2c1f6da0), LL(0x703366fa,0x96b60285),LL(0x6ce66461,0xa46d2ed9),LL(0xc7bef52c,0xaf14665e), + LL(0x614dfb0d,0x03487de0),LL(0x950342d5,0xae8d0e16),LL(0x77195a73,0x4f2ab2f7), LL(0x9757d4ae,0x0cfdcfec),LL(0xe42519f0,0x20cca44c),LL(0x4b2eb661,0x3a973637), + LL(0xf0c80fdb,0x5ed30224),LL(0x9d39980f,0xb74201d0),LL(0xc2a6b759,0x705dd1b2), LL(0x080941d1,0xd6535dc8),LL(0xd279a85b,0x0d532b08),LL(0xa6c95800,0x55279407), + LL(0xed2f99c0,0xa5e5dccd),LL(0x26335c8f,0x1d51b65a),LL(0x08067e36,0x930c04c4), LL(0x0583c23c,0xb14d764c),LL(0xe5be4e67,0xe9f1f8b8),LL(0xd3b9a111,0x2b684b5a), + LL(0xce971c7a,0x9420fabb),LL(0xc95e25ab,0x3ade0eab),LL(0xa490d8c4,0x350f2e80), LL(0x3f6eb3a6,0xa76804d4),LL(0xe63fd72e,0xd2ce7028),LL(0x85be7c39,0xb27222e2), + LL(0xfccf002e,0xe372c921),LL(0x514dc3ea,0x96a39294),LL(0xdad77509,0x4e6aeb46), LL(0xd37b54ed,0x9a5799a8),LL(0xfc0e43fe,0x64a87cee),LL(0x753b6b4a,0x7f05b1aa), + LL(0x549af75f,0x807a2862),LL(0xd359bf07,0x1c8b5f7e),LL(0x95d01de7,0xf52d3291), LL(0x5ffe6146,0x7a461465),LL(0xe83b1659,0x224a8202),LL(0x470e8470,0x01236b76), + LL(0x61865ca5,0x6208d875),LL(0x9571dfd5,0xa1a42854),LL(0x2d2d7940,0x6d5edb28), LL(0x6ff2b93c,0x8af66e50),LL(0x1baad60c,0xb6eb8e7d),LL(0x4a16a63e,0xbe114d2c), + LL(0x388a91ae,0xbb783662),LL(0xc9cc2382,0x34e00a1b),LL(0x2aa28a74,0x66d0d031), LL(0x6fbf826c,0x5c0cc1a7),LL(0xc1ac7477,0x78033af5),LL(0x9a21f372,0x1d7e2839), + LL(0x5af2db66,0x036516b8),LL(0xab38cfc3,0xd3fec155),LL(0x416d0d54,0xa6e7d1b4), LL(0xb397c8ca,0x9d762998),LL(0x4467324b,0x73e4c3a2),LL(0xbf7ada2d,0x9f7dd282), + LL(0xc9c10ad9,0x3f452c2a),LL(0xb2609a4d,0x7a827181),LL(0xe31734f3,0xfb87d393), LL(0x2bbf4efe,0xdafffdda),LL(0x9837686b,0x53193aba),LL(0xd61a044e,0x17401c18), + LL(0xf7fe6834,0x8561d6d5),LL(0x040422c5,0x20eee370),LL(0xc924342e,0xa16cd1c2), LL(0x0aba73e6,0xc017dbd6),LL(0x9e7b1683,0x209b1479),LL(0xf14b41af,0x4a639bbb), + LL(0x0be4ccf0,0x59308c11),LL(0x49cc5fa5,0xa69616fd),LL(0x35696487,0xed6dfed1), LL(0x1657ce60,0xdef27e77),LL(0xab4a8d76,0x346c3382),LL(0x98a008a6,0x54c10f0e), + LL(0x9d792671,0xbda7a3ff),LL(0xae4fafb1,0xab2ba801),LL(0x79f9c043,0xe27406e3), LL(0x14bcd570,0xefff1790),LL(0x80a00e1c,0xae57b991),LL(0x7b38c603,0x63b01e35), + LL(0x388ceb24,0x7f1b2eef),LL(0xadc25ce7,0xbf81721a),LL(0x6ae36f35,0x1fd0c684), LL(0xfa611602,0xdae9c8db),LL(0x3f40a562,0x9d7b2264),LL(0x9e4cec86,0x64de9a7e), + LL(0x396229d2,0x413e342b),LL(0x0afa76eb,0x97291d8c),LL(0x790ad9ca,0x18f63384), LL(0x307ad8f6,0x90deccfd),LL(0xcc49b61a,0xefb21d4b),LL(0x3d724666,0xcda971d5), + LL(0xb10df2cb,0xe1a0e3dc),LL(0xe9334feb,0xe5509cd4),LL(0x38f21828,0x4dae1419), LL(0xe9309400,0x932c62f1),LL(0x289fd6f2,0x14acca95),LL(0x4e1325f1,0x92934423), + LL(0x0edb12da,0xbc9bd40a),LL(0x3cd6bd8d,0x78f3256c),LL(0x644bdf08,0xb8fe8191), LL(0x314f2c73,0x3ba94523),LL(0xf3d8e905,0x33ddf165),LL(0xdd9721f4,0x603ecb74), + LL(0x04e15cb9,0xa1a8d802),LL(0xe4eef1ba,0xb3bf09c2),LL(0xf7ea8116,0xfcaecf4f), LL(0xe68eb8dd,0x72f4dc39),LL(0x761271cf,0xeab7abaa),LL(0x51b8259a,0xfe99e9d8), + LL(0x625f396b,0x6325bcfc),LL(0x7ad03440,0x36560d57),LL(0xaa639416,0x76246bab), LL(0x8fd064b6,0x381de8c9),LL(0x95537e9e,0xe4c3f8ac),LL(0x6e56f0e3,0x0dd1b6fa), + LL(0xf065498d,0x96a10f94),LL(0x0038c2dc,0xf88a1867),LL(0xc58b18bc,0x0de21830), LL(0x9a5da75f,0xc86e1dd4),LL(0xb092c397,0x05e00843),LL(0x3154c419,0x2839c7bd), + LL(0x18ef22bc,0xf6457fe0),LL(0x1df42e96,0xbb096001),LL(0xdef286c9,0x1a337c91), LL(0x4580b8e7,0xd4bb225d),LL(0xa2ce3586,0x6bba9b93),LL(0xf2d150c2,0x9d51106c), + LL(0x943953ef,0x0a3f8284),LL(0xe9c8b50e,0xa6f7e5d6),LL(0x1bfa459d,0xe75546eb), LL(0x34529d4a,0x3846abb5),LL(0xe1da3b07,0xbbbd1f98),LL(0xfc1e8935,0x05a92469), + LL(0x6e52f511,0x3edb02c2),LL(0xe6cea9d0,0x0306e54f),LL(0x5fe7cc33,0x1944a8aa), LL(0x53fd6127,0x3da98252),LL(0xce628fe6,0x9d5fa436),LL(0xb3f9ba33,0xe7ccb20b), + LL(0x2b5eabc9,0x22d21323),LL(0x8ddb58d2,0xfa85732a),LL(0xc0a4b7a7,0xf80ab9d0), LL(0xefe1b0aa,0xe505f00a),LL(0xd93f46b0,0x1388c1d5),LL(0x94416a57,0xf44495f8), + LL(0xd1b9d175,0x3e5e07cd),LL(0x28229715,0x713fb4a2),LL(0x750b4e8c,0x6a1dc0df), LL(0xb1837ba8,0x2b9fe39c),LL(0x8a4e51e1,0xad0b8e77),LL(0xed0e4aae,0x823a02e0), + LL(0x5f16b838,0x8a9de11c),LL(0x95820340,0xfda67f12),LL(0xe5b3813a,0xaf424ffa), LL(0x1961cfba,0x8eccd1ab),LL(0xf3022e67,0xacea8ccc),LL(0x1522dd90,0x799ce2cb), + LL(0x85c6d5b4,0xad3a9564),LL(0x11a4286d,0xbf57a898),LL(0x86df6b27,0xeb453d87), LL(0x4baef190,0x676b9e41),LL(0xa947a521,0xf94b92ee),LL(0x71257aaf,0xdc1c303e), + LL(0xe1f3b4df,0x84384afe),LL(0xc6f889c1,0xb9f7ae71),LL(0x0dc943bb,0x2f227f6c), LL(0x994e3369,0xd7a6ccfd),LL(0x4042431f,0xf4480749),LL(0x54ff110a,0x4abda3ff), + LL(0x9882e341,0xd50de25e),LL(0xed0228e4,0x7e68284a),LL(0x794de327,0xcb751d4a), LL(0x99ac62a1,0xb3cb0de3),LL(0x0001de50,0x870815cf),LL(0x82c9f5fd,0x7957d4ee), + LL(0xbda3fcde,0x8760035a),LL(0x5734a5bc,0xf131439d),LL(0x1de171b9,0xbef8fa6e), LL(0xb076e2ab,0x35d32607),LL(0x8bfa83eb,0xb132de84),LL(0x1b272008,0x7ca46b88), + LL(0x4437ac91,0xbc0e5b55),LL(0x7381e979,0xd7cb1a53),LL(0xd154616b,0x906abe2a), LL(0x4c1b6f06,0xebc02151),LL(0xf2e0afa1,0x6ede04a9),LL(0xa50c46b3,0x7f666548), + LL(0x4abfde1a,0xda143ed3),LL(0x5fb5c163,0x968f5f5c),LL(0xc56b0f29,0xe9641696), LL(0x40766adb,0xcaade44d),LL(0x41a6905e,0xfac69f2b),LL(0x96dd9b54,0x8a2a6278), + LL(0xead8c9c2,0x3a498bf3),LL(0x38527bb3,0x3f5fdc4f),LL(0x5a898416,0x78259bc3), LL(0xd2bc2a01,0xd7f94979),LL(0x058e4b40,0xb28701a4),LL(0xddf1c2fa,0x9908452f), + LL(0xd85e5adf,0x30be2fcb),LL(0x492081e6,0x740f18cf),LL(0xd2f40d30,0x4d4090f9), LL(0xf3be491b,0x3770d2ac),LL(0xd47a45ef,0xad71d7b8),LL(0x85b820f4,0x0b2cd763), + LL(0xa83085f6,0x1d1f6f65),LL(0xf7b897d7,0x4643738e),LL(0x209fede2,0xa9ae5e94), LL(0x929a2b44,0xf63bafe4),LL(0xef7d17e3,0xeb605e21),LL(0x526ff05c,0xa858b5cb), + LL(0x3d3a82b9,0xd365dd2e),LL(0xcc2cfd70,0xfeccda5c),LL(0x344ed0f5,0x36ffafe6), LL(0x04ae99a2,0x934a88f7),LL(0xc20491ef,0x0af5b488),LL(0x7a66a1c8,0xcda2acb7), + LL(0xdd5e1098,0x2b25a1ea),LL(0x909058c4,0x345012e3),LL(0xb5670aae,0x965a49e2), LL(0x1766fa93,0xc31d4480),LL(0xd3941b2a,0xd6f8bac6),LL(0x6eb3985e,0x46cd2776), + LL(0xbf004633,0x4c45b908),LL(0xdec0c1bf,0xe6b9457e),LL(0x78c0aa4f,0x2c6f62d8), LL(0xf80fdbd0,0x32bd6508),LL(0x97f86a7a,0xaa4f54c5),LL(0x2eee225e,0xc3186238), + LL(0xca5d7fb0,0xc11da036),LL(0x79e6d224,0x2bc98024),LL(0x779b9f68,0xed3b8e2b), LL(0x2c18b348,0xac7614f9),LL(0x36c756d6,0xc2072c63),LL(0x35e4340e,0xa6c5e629), + LL(0x8d338dff,0x09c64919),LL(0x8d7f5e34,0x347f9f9d),LL(0x8044621a,0xd3012dad), LL(0x27327013,0x4520210b),LL(0xe20f7a66,0x9f76824c),LL(0x7b70408e,0xa136be56), + LL(0x455a2af9,0x4b1f9940),LL(0x48945af9,0x1385b4c9),LL(0xac68fc09,0x3da47592), LL(0x5f05a04d,0x533995d0),LL(0xd7c5146e,0xc20d45f0),LL(0x22d2ae20,0x351ffb64), + LL(0x7ab5d0f9,0xb5eb2af1),LL(0x7762a3a5,0xc5cc2409),LL(0x6b374f6a,0xc65f2281), LL(0x0f46e6ce,0x440d2fe0),LL(0xb868814b,0xde15331e),LL(0xee22f041,0xd1d2f6c0), + LL(0xee1b9004,0x9ff4734b),LL(0x2e67a085,0x062dc3a3),LL(0xde8f3162,0xb7ea5b38), LL(0x824c31e7,0x19ce04cd),LL(0xbc61fd9e,0x4252601c),LL(0xf45482d3,0x95c7e121), + LL(0x31057137,0xde58e0a9),LL(0x0cac0d1b,0x30c96d91),LL(0x9104641a,0x0daab9d7), LL(0x1fd0668e,0x9ea156fc),LL(0x977683f4,0x0f6b404b),LL(0xa30e9a24,0x7966c84d), + LL(0xbff9caf0,0xe79728e1),LL(0x90e27f5d,0x82d66ec6),LL(0xbc650306,0x5ea57c74), LL(0xb2a82b96,0xcbe5e4ba),LL(0xfdfca395,0x24199d5f),LL(0x06991511,0x0ccbd94e), + LL(0x7876e2ff,0x4c91e7e0),LL(0x5d650fe2,0xba74af49),LL(0x5c7b9759,0x6f6df70a), LL(0xed511d0e,0xa5be2df5),LL(0x0c9af38f,0x0f540f6d),LL(0xb785f2e4,0x646f1fef), + LL(0xcb1eb7ef,0xf07cd946),LL(0xdd475a7a,0x41582424),LL(0xeccd0cd0,0xecabb646), LL(0x20672fdc,0xb08c9043),LL(0xd4fabfa8,0x0a0e852e),LL(0x08465cac,0x2df86c08), + LL(0xaddeb802,0x4a9d718e),LL(0x19257426,0x442b8a24),LL(0xf292c043,0xe1bf4df6), LL(0xf420119a,0xd846f55a),LL(0xb81b55b6,0x883b5be3),LL(0x7fa04e04,0xd233b497), + LL(0xb25f824c,0xa72a9e9b),LL(0x148d4350,0xcd9fa1a4),LL(0xe649b6b6,0xfe7be423), LL(0x1f055a16,0x54ff5456),LL(0x99348b7f,0xc73dc1b8),LL(0xb58ae286,0x072f9ae0), + LL(0xa2f354be,0x17fbb428),LL(0x8e54a9f4,0xf07c8cc9),LL(0x26d6dff2,0xc5dce60a), LL(0xfb7068ab,0x73ba83b1),LL(0xd6ccfb08,0x7f786cdc),LL(0x298a0fd6,0x57ea8ba3), + LL(0x6c93cbfc,0xb4637d1a),LL(0xbdb72d64,0x3b43839c),LL(0xa8ecb34f,0x80874787), LL(0xd5e642b2,0x57d9b566),LL(0x7a9c42a5,0x923772c0),LL(0xff8b1b47,0x24629516), + LL(0x07ceeae9,0x120c441a),LL(0x07740927,0xe82c0626),LL(0x91dda975,0x82b56342), LL(0xe8904b5f,0x5ccadd6d),LL(0x72da2998,0x55dc6fed),LL(0x5595c1fd,0xa6d506c2), + LL(0xeab8eea0,0x1a6a3b0f),LL(0xcafabd92,0xdf46f04a),LL(0x297dbb29,0xc5e2814d), LL(0x71d1e312,0x39d8e04c),LL(0x265b4075,0xc7017a13),LL(0xf78fbd26,0x4c21493b), + LL(0x0b9f158c,0x5c28c603),LL(0x9a07acaf,0x33de6098),LL(0x0e9c4c73,0x7b77cebd), LL(0x498db066,0x62e1c44c),LL(0xe28af701,0x57f684b7),LL(0x1f8b8dfd,0xac841f0c), + LL(0x6bc2d75f,0x9ca86d3a),LL(0xe7876500,0xab63e1ca),LL(0x436712c4,0xab42a0ee), LL(0x1d001327,0x8aead0f1),LL(0xf0a70715,0xf563e655),LL(0x6f079f79,0x58f26dd9), + LL(0x2922bad4,0x2b795683),LL(0x83b5370a,0x16a63631),LL(0x2638ee32,0x530fec6c), LL(0x44d0b441,0x23b918a6),LL(0x4b00c9b9,0xf1ebe8e2),LL(0x5e410b3b,0x77681a00), +}, +/* digit=7 base_pwr=2^49 */ +{ + LL(0x1db6b479,0x22b1c723),LL(0x39e84714,0xd62f06f1),LL(0xe23256f7,0xdee9ec7e), LL(0x95968194,0xe889db11),LL(0xa9e6d7d7,0x11d66cd0),LL(0x62416021,0x09b2418b), + LL(0x6adf5658,0x0be70575),LL(0x73b1138c,0x17f6b55a),LL(0x2a9fc1a8,0xce3bcb88), LL(0x01211935,0x4edfa8ef),LL(0x9b82e1b6,0x8972dc1c),LL(0x3e9d7b44,0x84866f69), + LL(0xcee3c64a,0x3f2da340),LL(0xf63dc83e,0xfcedb4bf),LL(0x56ff2a25,0xdedd2aac), LL(0x250762ff,0x9b5af4ff),LL(0xcbf6e216,0xd67cbdc0),LL(0xae6521ed,0x1b147e2c), + LL(0xcfcb0905,0x5315d6ee),LL(0x1a685aaa,0x269200d9),LL(0xc1f724bd,0x96f1c3a3), LL(0x3025b425,0x3a231812),LL(0x39d51119,0xff7672e6),LL(0x85a58c8f,0x1280679d), + LL(0x1ee4c1e9,0xe004f1c3),LL(0x6695e2c7,0x51f5f62b),LL(0xad1a015a,0x4c8db9dd), LL(0x0175ea4c,0x141a6846),LL(0xdb97814c,0xf8e3d91e),LL(0xf678fa80,0xdb30a211), + LL(0x0a4bf8e8,0x6e08e548),LL(0x95d4a1f9,0xd1e1533e),LL(0x74eb6623,0x01c6b2d2), LL(0x584b99ba,0x9e3dcf2f),LL(0x44a872c9,0x1211df9b),LL(0x13cf855a,0x44d0a919), + LL(0x7a71a32a,0x689a8dc1),LL(0x0969c604,0x6862a375),LL(0xbe398e22,0xd324e9b9), LL(0xfe84fe60,0x2175c004),LL(0xa817cdb4,0xa607ef1f),LL(0x721cb883,0xcd9d9495), + LL(0x3a36ef0d,0x4f8ebef8),LL(0xf4ef55c3,0x5c910c5a),LL(0x2a959873,0x8ce26809), LL(0x376d6a96,0xc2132558),LL(0x026083f4,0xa6c1cfb2),LL(0x0d455653,0x46c5dbd9), + LL(0x8ba48c45,0x48969551),LL(0xc643319e,0xba14588e),LL(0x6164409f,0x79cc5f79), LL(0x7cd52144,0xbbcfa5ac),LL(0x01be08de,0x291192cf),LL(0x1b5535ee,0xe09fcd9b), + LL(0x8b222a3d,0x4e761342),LL(0x9ee6a22d,0x874f88e9),LL(0xdbb503a4,0xfb29bb24), LL(0x27a88472,0x1e5923d8),LL(0x27cc978b,0x968ac6a8),LL(0xb31d7b22,0x8cd1a928), + LL(0x0643e7f0,0xc14f22a2),LL(0x97a5e590,0xa920ca01),LL(0x2c3fef97,0x4e4e7c4b), LL(0x8e7baa94,0x0200b159),LL(0x16b1757c,0x24b9c68b),LL(0x6ca9d309,0x51ff6499), + LL(0x96b7c630,0x7b132fca),LL(0x63d2264b,0x8df911e7),LL(0x164dbc12,0xff71dd08), LL(0xcd036c72,0x2f7b92af),LL(0x24d891cc,0xf1336105),LL(0x66b47dc3,0xe614aee7), + LL(0x12ead645,0x1c53214c),LL(0x3c7f65a7,0x32aa24b9),LL(0x118a5225,0xfd7b67e9), LL(0xdad33487,0xf58a2515),LL(0x3a563a0b,0xa9ce4387),LL(0xd92455a8,0xad5dc427), + LL(0xaca6b802,0xfa852455),LL(0x26bfb5b4,0x448d6d2d),LL(0x0accc4e4,0xa76981e9), LL(0xfbedc1b3,0x2c43f05a),LL(0xdfa0cdbc,0x684e84d3),LL(0xb07a9af9,0x60fa3028), + LL(0x46cbf3c2,0x55adc2d2),LL(0x0ab9aba5,0x5ddaea63),LL(0xf06d91eb,0x457318d3), LL(0x3face92b,0x210c0bda),LL(0xc8cca904,0xcb6be0b9),LL(0xac932c9e,0xcc059269), + LL(0x3b076836,0x35440607),LL(0xcfbc594d,0x6e139ab7),LL(0x0f818173,0x24c24eb1), LL(0x4c3d9a9f,0x26b87e2a),LL(0x1fcc4242,0x03671da7),LL(0x643cbca5,0xd08f6b5e), + LL(0x69cce513,0x3b1962bf),LL(0xbbceb26d,0xe0091d0c),LL(0xf374c0a2,0xda304365), LL(0xa099ddf8,0x27a29bfc),LL(0x969c677a,0x5f53ba53),LL(0xf0627255,0x77b8d2a7), + LL(0x3819f9dd,0x221375dd),LL(0xe08b0ca2,0x04a971e7),LL(0xd46c54ac,0x5f7345a7), LL(0x93d0f966,0x8f01f13d),LL(0x6ef17236,0xf5cf276c),LL(0x0dd038f0,0xca0bba4c), + LL(0x525bd2b0,0x86ab6164),LL(0xa0a0d81f,0x0fddac9e),LL(0xe3827450,0xc5322927), LL(0x7081e340,0x5965a1a9),LL(0x427b0701,0x46c3aef7),LL(0x6692b94a,0x98a9fa7c), + LL(0xa495e99e,0xd2932e7f),LL(0xd82f6584,0xdeb528fd),LL(0xbe305394,0x93b3c1c1), LL(0x7d8622a3,0xa4cb19b6),LL(0xb6d5eb26,0x7b46a163),LL(0x9239f879,0x247e327b), + LL(0x57a9df34,0xa102051f),LL(0x1aaf0cff,0xe1fbc51e),LL(0x9fd0f53a,0xe533029e), LL(0x75a0f971,0xda45f13c),LL(0xfe0d2922,0xadee768d),LL(0xfc9bca7d,0xdcf7b7bb), + LL(0x39f2a51e,0xa84b4adc),LL(0xa4c4231a,0x9c257e5f),LL(0xe6835319,0x5dd04181), LL(0x276cf633,0xb2ea7ce6),LL(0xd0c18fe5,0xf245702e),LL(0xb338e5a9,0x994cccff), + LL(0x33e5c5c0,0x6ba4e93a),LL(0x258e8029,0xa7b7d1f8),LL(0xad202aff,0x304ea934), LL(0x54bb2c25,0x7b28d6b0),LL(0x50b1a4e9,0x624bdf85),LL(0x7ddc1ccd,0x2c1079d6), + LL(0x7bc06924,0xd757678b),LL(0x4a9230a8,0x761f6494),LL(0xa38af7ae,0x4a74e816), LL(0x55556207,0x5cb7e1ad),LL(0xf5417b00,0x51a88664),LL(0x78c7935e,0x48886cf1), + LL(0xc5ae666d,0x112a047b),LL(0xe0d72464,0xe3e8915c),LL(0x88f740ce,0x18fd8fe6), LL(0x67d49be0,0xa5843ed0),LL(0x91b85641,0x44616e3f),LL(0x40e3c66c,0xfc394c04), + LL(0xce689251,0x4d6ed19e),LL(0xf28dfceb,0x81078128),LL(0xc82c2dd5,0x71bade6c), LL(0xc7bf5e3a,0x20cfbae4),LL(0xd1983380,0x0d1c23fb),LL(0xfb5cb3bc,0xd56434b8), + LL(0x496c7720,0xdbdd0273),LL(0xd7c99031,0x20b010b3),LL(0x06d61176,0xf5ff610c), LL(0xd3631b15,0xa0b6ea34),LL(0xe950e355,0x1528b1e0),LL(0xf0c84a6d,0x6e27fab8), + LL(0xf55bab95,0x8e66ed20),LL(0xbc6b300a,0x63e73fb5),LL(0xab652058,0x753ebe56), LL(0x46319ffa,0x0d97e216),LL(0xfbb99b70,0xf605457b),LL(0x8829b2aa,0x03a28a89), + LL(0x0073b797,0x3eac149e),LL(0x9bfeb2fd,0x95792d96),LL(0x7dc85ebc,0x0ccf0041), LL(0xecc61042,0x449bf3dc),LL(0x986664d6,0xaa4fce55),LL(0xa8c146ef,0xbc5ee8ae), + LL(0x1ff1dcbf,0xd904f9b2),LL(0x5d8a1197,0x35a18d0d),LL(0x6e3d4745,0x3341db2d), LL(0x4b050939,0x6b1eec19),LL(0xd70a0a15,0x9c1fe3f4),LL(0xcdfdd83e,0x654fc4dd), + LL(0xe9ba38d1,0x4ebcf740),LL(0x7de3b4a7,0xb42141e4),LL(0x58b093ca,0x09454a65), LL(0xa475ef3c,0xa87bd8d2),LL(0xcd2de97b,0xa1e89c23),LL(0xcb134c1e,0xe3000fa9), + LL(0xe03d89fc,0x56e13148),LL(0x560317fc,0x737b874d),LL(0xaee37ff7,0x28ce8556), LL(0x600002bd,0x21a8f876),LL(0xd20a7373,0x11a46449),LL(0x849772fb,0x280be2ee), + LL(0x44d8341d,0x81aa4216),LL(0xcfc93e1e,0x413bbc51),LL(0x5cbb96ed,0xc61e7fbd), LL(0x348b93bf,0x183ffd78),LL(0x1a6cac3d,0xbc237f9a),LL(0x075f2c01,0xcbe09c38), + LL(0x05a97f5f,0x337c5514),LL(0xf04dfeee,0xc2e77233),LL(0x5bb4ca71,0x400f4852), LL(0x8553221b,0x9626c894),LL(0xb57b0765,0x11aa8bb3),LL(0xa12d3f73,0x543ce8fa), + LL(0x8faba6a0,0xf77adb46),LL(0x435376cf,0x1174ee49),LL(0xedd2bd00,0x19430a06), LL(0xe4cbbaef,0xbfdb0a6f),LL(0xfe5c89b0,0x3b520354),LL(0xbb7ab998,0x8acd5ed9), + LL(0x28bc63d9,0x4358e216),LL(0x6d84054f,0x1d99f606),LL(0x923248e3,0x8e24aa2e), LL(0x2d2082d7,0x004093da),LL(0x0c19a153,0x180e72ac),LL(0xa4539eb7,0xb4d8b7b3), + LL(0xab583068,0x850a27b8),LL(0xeacb954e,0x85790f5d),LL(0x533b4d63,0x49567edd), LL(0x2c83d758,0xeedc0e79),LL(0xfcced827,0xe007c1d7),LL(0x2000bb00,0x68232c77), + LL(0x2ab39a53,0xdce3a7d4),LL(0xe408976d,0x5401ad0e),LL(0xb9ffd974,0xfcd5a5e8), LL(0x06214587,0x6adae145),LL(0x3b0e57df,0x15d783fc),LL(0x35f73e2f,0x8d4a1061), + LL(0xb2e9c8be,0xf6b94404),LL(0xe122287a,0x57269f9d),LL(0x7f0b3575,0x3f9849e2), LL(0x691d4ea8,0x1d85bc98),LL(0xc811fe44,0xbf34f840),LL(0x6fdeeca9,0x188d1f53), + LL(0xda6d3dcf,0x78294002),LL(0xaddf5b48,0x3f86f8c4),LL(0x6f8761ea,0xca9c8dbb), LL(0xb56d54cf,0x6d4be379),LL(0x3d32e3d0,0x9504874e),LL(0x1b4d470d,0xe0959574), + LL(0x60c89192,0x03618e26),LL(0xec22e3da,0xa067fbde),LL(0x202d430d,0x5656229a), LL(0xc7703466,0x1f782bd8),LL(0xa0026a97,0x3b278178),LL(0xf1c08768,0xc86a636a), + LL(0x754cbc65,0x175ae3ec),LL(0x4f765d06,0xc6f3677d),LL(0xa11dc7c6,0x699a40ca), LL(0xc2c20678,0xf85e4e41),LL(0x3ee230c1,0xc4fd583c),LL(0xf604d6e1,0xfbf97e3e), + LL(0xc6e43d77,0xc2a7b60e),LL(0x5e09cfba,0x5c44a7ab),LL(0x20323a01,0x31279294), LL(0xd83e9122,0xbef510c2),LL(0x910635c8,0xce89cd72),LL(0x6525654f,0x40c3123c), + LL(0xabb9fa3a,0xe5efa39c),LL(0xd54d0553,0x0dd062d8),LL(0x5f654ea6,0x2447ebea), LL(0xb205ee7f,0xc6b30d78),LL(0x6b327d54,0xac800865),LL(0x2d494627,0x3564831f), + LL(0xbf996e35,0xe5612ddb),LL(0xb2c8a71e,0x82b679c1),LL(0xb8522766,0xadfb7763), LL(0x6edb7f6b,0x31f681bc),LL(0x6c1396b8,0x0ac77865),LL(0xc16391a7,0x4fec5edc), + LL(0x6a8ba5e0,0x79be8744),LL(0x76a3f5dc,0x9cad48d3),LL(0xd3a4a677,0x3b4ddd6a), LL(0x2427a393,0x53d9dd5a),LL(0x99061179,0xe3992757),LL(0xc5ab35de,0x9a76a0ad), + LL(0x270ecf31,0x18e8dbf5),LL(0x3f3da910,0x6723152b),LL(0xe687db02,0x74b53e35), LL(0x89fccc8e,0x72dd231b),LL(0x6ca0a007,0xdf98fe28),LL(0x9b8fc127,0x5c469a08), + LL(0xf7c47e53,0x3d45d6b3),LL(0x2bca9e62,0x830867c6),LL(0xcef81d38,0x84276d68), LL(0xcc30ac2b,0x60d227cc),LL(0xaa747e61,0x514c1992),LL(0x4ed869e9,0x90083817), + LL(0xf15ff76b,0x9c199d37),LL(0x4a4fc2fd,0x0e5ab91a),LL(0x25ed5ca0,0x142d9700), LL(0xc7398261,0x417bd5e9),LL(0xca5133ba,0x074f5397),LL(0xbbf69817,0xda80ec1f), + LL(0x29f549a1,0x66cc5bce),LL(0xd6af775a,0xbe183389),LL(0x7b066b44,0x492f66bf), LL(0x431fc807,0x0f1397b4),LL(0xe83dd557,0xb6f675ba),LL(0x62489a2a,0x972d9038), + LL(0x8eff888d,0x71c228f7),LL(0x75586fd7,0xcd59c5d1),LL(0xdd3143e7,0xa966bb14), LL(0xbf353a1f,0xa7f87d54),LL(0xf484d5f8,0xf9cbceff),LL(0xb68ff698,0x698dae86), + LL(0x6f3235f2,0x66286731),LL(0x1579c24d,0xcba86933),LL(0xba144ac8,0x8864f364), LL(0x3ffd19b1,0xaad3e749),LL(0x6fdf894e,0x148d7a44),LL(0xb448c223,0x3aefa6dc), + LL(0x84e0548b,0xc6c90b42),LL(0xa6072bb0,0x9234d1be),LL(0xafb14b2a,0x3b28ca69), LL(0xcfbcc4d3,0x80eda096),LL(0xac4275fd,0x8bf29e73),LL(0xa9e0838e,0x10655c00), + LL(0x8eff3cbe,0x75e78f0c),LL(0xb0a9ad9e,0x2d549d6a),LL(0x50d83c74,0xd9179f56), LL(0x45988bd0,0xdf61dfe9),LL(0xe8f87cbb,0xb76286ea),LL(0x033c1c00,0xa64bca3d), + LL(0x664f67a7,0xd82e91bd),LL(0x19c561de,0x7329573d),LL(0xce6bab53,0x5faf92d6), LL(0xcc978c65,0x715cc536),LL(0xe1e82686,0x843e802e),LL(0xbb3c2514,0xd7a82374), + LL(0x46e3c384,0x7dec5bcc),LL(0xb5c8409d,0x874ece30),LL(0x0fb8869d,0xbc0f588d), LL(0x65d88ad2,0x11c4d95d),LL(0x349f7fc1,0xef6a5587),LL(0x5d3c89c7,0x8ceb5e7b), + LL(0xfdbac03d,0x8326fccd),LL(0xc0e3814f,0x8592d5ac),LL(0x92780f4e,0x4c4f9181), LL(0xcbd77d47,0xdfa95b95),LL(0x116d8d69,0x5c695658),LL(0x45d71633,0x3b9e5958), + LL(0x1f933a54,0xf15d931e),LL(0x572c2915,0x16a8e689),LL(0x6078e8b9,0xe37672fe), LL(0xb7252a68,0x37af9fd5),LL(0x6b8ac148,0x9e17b01d),LL(0x942a5a17,0x5ad4c09c), + LL(0xef904c15,0xcf9f44da),LL(0x1b8bd3f2,0xfd9b0973),LL(0x1a01c966,0xa2209dbc), LL(0x4d678467,0x0878c83b),LL(0x41418cd8,0xdfcce3e9),LL(0x24b1f3e9,0x10ed020b), + LL(0x6eb0a25e,0x1d05d913),LL(0x6ba1b2f1,0xd8dfea69),LL(0x4570c9ce,0x224f88a7), LL(0xb8c5a0d1,0x82087d08),LL(0xcdf8086e,0x2d689252),LL(0x33330715,0x7f5cbba0), + LL(0x143708d2,0x793030ec),LL(0x3aa671de,0xe1e44a09),LL(0x6d0c6b60,0xf8d14831), LL(0x4c6b4751,0x43814c65),LL(0x07c35ce6,0x23531a9b),LL(0x30f97153,0xec6a9878), + LL(0xa300f8ce,0xd0ced276),LL(0x1ef83f9a,0x7eb2f7db),LL(0x3ff788a7,0x8bdc98f4), LL(0x29e0e1c2,0x8c5f9af2),LL(0x7ac14e4e,0xb6cd8700),LL(0x073f9d19,0x54909c4d), + LL(0x3fb6efd9,0x9481d096),LL(0xb50cc9b5,0x04a156f3),LL(0x8baffff7,0x4b8e541d), LL(0xdb644174,0xe7f5e90c),LL(0x530c3696,0x754d867a),LL(0xf47978bb,0xfb20bb76), + LL(0xd5e0c70e,0x16986cc1),LL(0x52e7e6c7,0x42ce3058),LL(0xa618254e,0xf31ea341), LL(0xae1ae829,0xaf2001d4),LL(0xd97c4da7,0x3f93eca5),LL(0x8b5c3229,0x01dd7a1c), +}, +/* digit=8 base_pwr=2^56 */ +{ + LL(0x0c6a351e,0x6c78a1f4),LL(0x807a4e21,0x5192b2cf),LL(0x72479dc8,0x531cf1e5), LL(0x15863b65,0x78f3e85f),LL(0x6f917ccf,0x008524f6),LL(0xd0dccb94,0x531192d9), + LL(0xc490a7f0,0xf7d5ce6e),LL(0x06cc1312,0x646319a8),LL(0xbc37af6c,0x01d71545), LL(0x7f32c096,0x92077f6a),LL(0x02223fa6,0x3c4192cf),LL(0x0b59de1f,0x72b36fe0), + LL(0xee192ea3,0x6cc9d6a8),LL(0x4008c57e,0x951ca552),LL(0x0ccb74a8,0x0906ec42), LL(0x09e39599,0x80c4c111),LL(0xa5246b67,0x4a79b1c3),LL(0x5ead0c0f,0xff7242ea), + LL(0xf2d97c8d,0x9138c2cd),LL(0x2650a9b4,0x3caecb47),LL(0xf8760ddf,0x23a73447), LL(0xc1d98546,0x1ebf0619),LL(0xcc7f0b0a,0xd9f11b6e),LL(0xde8d96f1,0x16ab2c05), + LL(0xeffca98d,0x0dd3b147),LL(0xe1abc2a9,0x10c4fdcc),LL(0xf74f036e,0xc4b7b199), LL(0x7301d513,0x6db9d5ac),LL(0x774b8020,0x154acf22),LL(0xa60f3784,0x0030085f), + LL(0x859de975,0x6b73749b),LL(0x67883183,0xc97b9ead),LL(0xbfb919b1,0xaaae5948), LL(0xf5c8912d,0x8150189a),LL(0x67fe0faf,0x6b988a5c),LL(0xf5839146,0x21d1f6d9), + LL(0xa3635200,0xaf8c2c49),LL(0x1d377a92,0x0400e2f7),LL(0xfa0d421c,0xaa0ec06c), LL(0x0b0d997b,0x6e3c3b69),LL(0x6756ee3e,0x7d3832dd),LL(0xe78e3204,0x30d62b54), + LL(0x5174cd2b,0xcb5e1dcb),LL(0x18f6e4b5,0x8984cb7b),LL(0x72a12150,0x363d0f1f), LL(0xcc62f17b,0xb5c36272),LL(0x3a2f964b,0x4a992731),LL(0x4e8f1ce1,0xbca57238), + LL(0xff1a436b,0xdfcf7d1d),LL(0xee098e33,0xf6028d75),LL(0x85091828,0xf98fd92a), LL(0x8cbf1897,0x535e81c3),LL(0x100e48e4,0x4b9286c9),LL(0xa517213e,0x6dcb2c57), + LL(0xb451f187,0x1bb72786),LL(0xac01e718,0xbfa69395),LL(0x9acebb3e,0x2c1852ea), LL(0xddabfc41,0x5eae9e20),LL(0x1bd561f4,0x661e7254),LL(0x270a6fa5,0x71f20425), + LL(0x961036e2,0x15abdd47),LL(0x35b92111,0xe3edb318),LL(0xfeae7403,0x6a5f093b), LL(0x1595973a,0x4bca351b),LL(0x3a5ff8b1,0x10b5c69d),LL(0x826996cd,0x61699ca4), + LL(0x78bdb771,0xef90144f),LL(0xa6fa7f1d,0xc15f7dab),LL(0xb45483c8,0x367a4d90), LL(0x770b4b1a,0x3524f21b),LL(0x0446fddb,0x31111556),LL(0xf304737e,0x77b6be1c), + LL(0xabde9561,0x37c8901e),LL(0x122cf2de,0x607d5c3b),LL(0x8479673d,0x31a6f869), LL(0xf1df773f,0x1bc908c7),LL(0x6e7f32aa,0xb13f1008),LL(0xd9fc2d4c,0xd0b64b4e), + LL(0xf338ea23,0x4c0c5170),LL(0x49fbff41,0xb0639fbf),LL(0x6cdb87d1,0x0c71578a), LL(0x5d00e5a8,0x6a0fac6e),LL(0xfb624d14,0xf923d1e7),LL(0x65ec474b,0xe2af600c), + LL(0x0afa00ec,0x96a0e907),LL(0xe93d5449,0xa58ae31f),LL(0x1e050306,0xf0090575), LL(0xce3c1478,0x3570e2bc),LL(0xd154329a,0x6c779b82),LL(0x8a7caddf,0x2e65670c), + LL(0x428ffdee,0x7850b688),LL(0x1e3103a7,0x5630065c),LL(0xdc08408c,0xd774279e), LL(0x6eaf0bcb,0x85499490),LL(0x72f72173,0xb1954318),LL(0x4ac8220e,0x2e5e8eda), + LL(0xe86f4f00,0xbd47e5b5),LL(0x8338414d,0x312a0a57),LL(0x5dc0dd30,0x8832fc40), LL(0x8be0076e,0x5b2ec051),LL(0x45348887,0x3064cc96),LL(0x61596442,0x9d0e00fe), + LL(0x747e8db9,0xa49b588c),LL(0x1c9851ef,0x91bd49ac),LL(0x21c7c36b,0x547ad2e7), LL(0xcf0a07e7,0x91725ea2),LL(0xcbbcb82e,0xf7fc5911),LL(0x5646ba9c,0xd5f7714f), + LL(0x8edccbb0,0x0f86777c),LL(0x7068d3ce,0x4df433f8),LL(0x77731ed1,0xfd2d02a0), LL(0x1f323ab3,0x9ce12310),LL(0xabf47059,0xab127e12),LL(0x4257eed4,0xe37b4198), + LL(0x61b7a7bc,0x96bff478),LL(0x24a24d11,0xf70c91dc),LL(0x2ed5f8ef,0xf25c49f7), LL(0x8b8cd0bb,0x66103231),LL(0x337a16a3,0x9af494be),LL(0xbcdf06c9,0x8c8b1723), + LL(0xb8d9c7fd,0x5370d17d),LL(0x1320d963,0xa9848001),LL(0x3ab21366,0x4b08ede7), LL(0x8ee6ae31,0x3640d969),LL(0x7deca729,0x782297a2),LL(0x8cf1b471,0xb99c1450), + LL(0xc55e2971,0x3a206eb5),LL(0xe3b5b1a3,0xe65b0a7a),LL(0x7f789c24,0x611ca3ef), LL(0xaee050b4,0xffbed20f),LL(0xf7561895,0x81e86366),LL(0xc2fa8046,0x64671984), + LL(0x0c3c9013,0xade34ec5),LL(0x5eb41025,0x50bd68e1),LL(0x0d4894dc,0x78808518), LL(0x7c29a657,0x7ed39b6b),LL(0x241517d5,0xbafdbdd0),LL(0x188218bf,0xd6f50092), + LL(0xf03ef732,0x81177e12),LL(0x56a3507b,0x1574fe49),LL(0xab053718,0x6e960496), LL(0xc3a21218,0xbcd97662),LL(0x3bf70829,0x0cf68dea),LL(0x25b800e2,0x32d30c54), + LL(0xc9428b22,0x95452160),LL(0x72a22c82,0xc2622e6b),LL(0x5b27cb42,0xb6731ec6), LL(0x4fce33d5,0x0f8a1fc2),LL(0x6407716e,0xa9838902),LL(0xfa98bc54,0x5e553dc0), + LL(0xcaa69408,0x59683c61),LL(0x6598df18,0x82bc08db),LL(0x6634b238,0x154e5197), LL(0x43fc4c77,0x70ad310c),LL(0x835272f8,0xf9144e70),LL(0x9b7d0d58,0xf41beaab), + LL(0x7a860980,0x7aef7397),LL(0x7175ed79,0xab751037),LL(0x797c12f0,0x7dd8fb29), LL(0xebd2f077,0xa63e2440),LL(0xce865a7d,0x645fe5ce),LL(0x317de834,0x5e57db53), + LL(0x2bb8134c,0xc28c2ef9),LL(0x520d82a8,0x3cfa28dc),LL(0x7103b732,0xc31217da), LL(0xfef50290,0x9daf07ed),LL(0xb5fbffea,0x5b6cc3a2),LL(0x710a59db,0x98d2a6f3), + LL(0x9174bf48,0x44b0ff44),LL(0xeef16e79,0xb5562ba6),LL(0xea9adffa,0x129bbad4), LL(0x8ccd2a64,0xc93aa7fc),LL(0x2ce754e9,0x0c252964),LL(0x88a4559c,0xbaf7f806), + LL(0x548d0e57,0x924da3ef),LL(0xbe13e6c9,0xde8c8973),LL(0xd73979cb,0xe5dc0c62), LL(0xf5d1e78f,0xc2f92566),LL(0x95a8fdc6,0x3152a7cc),LL(0xdc528326,0x6601c017), + LL(0xd38d9fea,0x1235bcd4),LL(0xd899201d,0xe4d00d99),LL(0x2709bee4,0xf0a9da65), LL(0xf33d4d62,0x69266ab1),LL(0x7a7c073b,0xc5c102a1),LL(0x1fddcaa7,0x42e277b8), + LL(0x176de7d5,0xa3b38048),LL(0x3ae77e40,0xcc01f3c4),LL(0x1b8310bc,0xec1be1f1), LL(0xb0b6554b,0xbdc976c3),LL(0x9c4bcd13,0xd86117f4),LL(0xd41ce646,0x9feb9193), + LL(0x2120685b,0x7d1547dd),LL(0x610034f4,0xb3b1ec97),LL(0x947b29d6,0xe98a9207), LL(0xdb48b264,0x3dfa24b2),LL(0xa5afaebc,0x6c8e33db),LL(0x0db735ad,0xfcad214a), + LL(0x3a1e3514,0xbc366d47),LL(0xd584ab3c,0x67228d9c),LL(0x90009ee1,0xd8abe402), LL(0x68d15f9b,0xb06fcb6e),LL(0x3cbc8586,0x9f4fab36),LL(0xac1ca214,0xeb6fc7d3), + LL(0x212701d8,0x644ffbc6),LL(0x3e7f5a64,0x0dd77816),LL(0xbba949bd,0x4182d12c), LL(0x9dd68a5b,0x566f707c),LL(0x3374e3d4,0xc32b41f7),LL(0x33a887f2,0xe25506ae), + LL(0x7d747270,0x6809dda9),LL(0x34b9c161,0xe0ff644d),LL(0x5735d1ce,0xedce5894), LL(0x3c299049,0x9a14955a),LL(0xe770315b,0xdf5340ba),LL(0x09ab6581,0x21d8b57d), + LL(0xa3a84876,0x6e2ebe98),LL(0x33705d95,0xb25b877f),LL(0x17be7330,0xfb32cded), LL(0xb3db892e,0x91cd62d6),LL(0x782eca61,0xca99ae59),LL(0xf953f71e,0xe927b38e), + LL(0xfdefd2fc,0x932d6c0e),LL(0xd45bc513,0xc02f4213),LL(0x97a9f940,0xb9554eff), LL(0xe0f84e1a,0x4e5d7ebe),LL(0x2f4302e3,0x77035f14),LL(0x9555b2f5,0x45bcd681), + LL(0xb757b6a7,0x55313f4d),LL(0xf62039e3,0xc1b3ca66),LL(0xa68faba3,0xbc76d877), LL(0x5fc39ebe,0x8addb47a),LL(0x2205e6df,0x52e4f8b8),LL(0xe999baf6,0xa4963df4), + LL(0x01e9090d,0x43ec78a5),LL(0x670ed5c0,0xada7c41a),LL(0xca0ccd63,0xf441fc60), LL(0x9f8a35c9,0x2b6b14cb),LL(0xd67b0c7a,0xea90296f),LL(0x475bfd04,0x9ec66ada), + LL(0xae9c5b50,0x785fcd1c),LL(0x43f8004f,0x1793f480),LL(0xff4b7c93,0x4ac0e11d), LL(0xbdbd07ff,0xfe1f7829),LL(0xb36c82d1,0x928d4b94),LL(0xedb0863e,0x9afbd51b), + LL(0x35ef7543,0x45b8e6c7),LL(0xefcbf15e,0x04bedfec),LL(0xd26c073f,0xe5d2b221), LL(0x9eeedb4d,0xbb74aed4),LL(0x8329872b,0xbdeee909),LL(0xd5027bee,0xde0ef4fd), + LL(0xfdec460d,0x4373d235),LL(0x5367ccc3,0x889db5d3),LL(0x6ba366f2,0x75b7adbd), LL(0x59e47152,0xda4f82c6),LL(0x74645dce,0x2f5d2404),LL(0x72715896,0x73bc8042), + LL(0xc2538c33,0xd97aa542),LL(0xc4176743,0xb0d4f42f),LL(0xd9117f2c,0x4e46a104), LL(0xa1dbb89c,0x749b728c),LL(0xbb2d1a5c,0xc91c80f8),LL(0xc41a94ef,0x3668d74b), + LL(0xc88dbe72,0x64a248da),LL(0x8c9c23c7,0x90d78a4c),LL(0xa56cbc8b,0xd49e40ab), LL(0x305322ae,0x764a4a3b),LL(0xd2f6d2ca,0xf13655d0),LL(0x0ad04588,0x5e206b78), + LL(0xa4e2f277,0xb295d1f4),LL(0x2b0f59d4,0x1d49ae57),LL(0xfc85f875,0x77ccdf3f), LL(0x40f7816d,0x98ca5c5e),LL(0x2cb0558a,0xea990a23),LL(0x7a723586,0x76ef8645), + LL(0xd4b23830,0x1618d79d),LL(0x0f9647c2,0x0a1387ea),LL(0x166ce29c,0xd2ba4d03), LL(0xacf33156,0xcc88e78f),LL(0xcaf5f200,0x57da063f),LL(0xae63a300,0xa5cbc1c3), + LL(0x017ded2b,0x8abdda43),LL(0x02c5993c,0xabbe3891),LL(0xfd7ffd34,0x52df96de), LL(0xd7e89e8c,0x6c4d980c),LL(0xbd659dc7,0x8e3c41be),LL(0xfe1fe1f8,0x7f6f7496), + LL(0xa4a51425,0xecae5b7b),LL(0xdd9d83af,0x81e2c971),LL(0xf7b85342,0x39b1d2a3), LL(0xfa8bf8c0,0xc45ad2c1),LL(0x7f18d82c,0xf495b15f),LL(0x9ccdc3b1,0x8b260d45), + LL(0x1d241682,0x5a2e6fa8),LL(0xa49fd352,0x313e6fc8),LL(0xa7ddd234,0x75139843), LL(0xd3d7609e,0x6549aefc),LL(0x8f6fd0cd,0x1b477e03),LL(0x70f7a7f4,0x6140a881), + LL(0x6718b3b6,0xaf1106f1),LL(0x44eaa8c1,0x767e6522),LL(0xf735f07a,0x527f2eed), LL(0x14ea415a,0x45ad0cfb),LL(0xabde5ef6,0xc09636ac),LL(0x9fcee205,0x88d7fa8d), + LL(0x95543299,0x004861d5),LL(0xa61e24e5,0xd594bc3c),LL(0xb3b7cffa,0x2bcafd63), LL(0x5669358b,0xcb375694),LL(0xcb56b897,0xc5c022e4),LL(0x12de2824,0xb2b30bdf), + LL(0x2cfdf68c,0xce6b8b13),LL(0x910076c2,0x942efa8e),LL(0x4719c7c3,0x661fc5fb), LL(0x82ebe6b8,0xcfdf4b8d),LL(0x3bb3c18e,0xdeee7446),LL(0xd641180b,0x00ed3964), + LL(0xcb3382da,0x1a1ea271),LL(0x30bf39fb,0x0256f492),LL(0xaf66aa74,0xc817dad0), LL(0xf968fb2d,0x2b653957),LL(0xf0fbf6ca,0x5d0b99f4),LL(0x80454299,0xe59af736), + LL(0xc2755f73,0xec4992e6),LL(0x4ca484fc,0xdfc3c447),LL(0x7217c264,0xa1f98659), LL(0x6549d8ea,0x43da1644),LL(0xee1b8b43,0x5a967704),LL(0xd404bff3,0x2a8a337d), + LL(0xbe4b6d8b,0x94c51eb5),LL(0xc07e4713,0x8fbc9eb8),LL(0xd390eddc,0xa19d59bb), LL(0xe0be24bf,0x0ccb795a),LL(0x340f70c0,0x84f1de5a),LL(0x582c2b9b,0xa699c480), + LL(0x7414a8d3,0x5eb36220),LL(0xe5364537,0xf828a75c),LL(0xc830d477,0x1082ff06), LL(0x037d83d0,0xd1528c4d),LL(0x45baf298,0x2b40c250),LL(0x35ee8eae,0xe2e4f82d), + LL(0x12e4391e,0x6d36ed91),LL(0x4f322b3c,0x1ea89876),LL(0xfbc19d19,0x60200183), LL(0xda185be5,0xea5a5069),LL(0x248f835b,0x0e2393fc),LL(0x15ee6f20,0x61d051fa), + LL(0xe1b5b6fb,0xf886470a),LL(0x60b4fdb1,0xda09259d),LL(0xf88455b3,0x088e9c86), LL(0xbc2676c1,0x52c2860a),LL(0xd1a102f0,0x6d6fa953),LL(0x7cc062e0,0x9299dce6), + LL(0xf1a2b5ca,0x812c18ab),LL(0x21bc03a1,0x9b74e2b7),LL(0xc81ccc62,0xbca14295), LL(0x05d9bf48,0xfb6c986d),LL(0xb169ef26,0xc284d240),LL(0x901172ad,0x6292fc73), + LL(0x5ef17d8f,0xeddaa5d3),LL(0x7c406cd1,0x15267457),LL(0x9531e2d8,0x2e303a5e), LL(0x75ca396a,0xf7bf1c5f),LL(0xc12acc0d,0xb10e432e),LL(0xeac37f33,0x2ffed1db), + LL(0x8e561f72,0x1fef1ecd),LL(0xdd1089ac,0x38fe7b1b),LL(0x3bc9bed2,0x2f24cac2), LL(0x8beb4a49,0x0d99d3cd),LL(0x9f925b5f,0x7dd5cfb6),LL(0xfd9b7744,0xa23894f0), + LL(0x14747d9d,0x1f353554),LL(0x75836ce2,0x31bcdf39),LL(0xf2ace220,0xea43017e), LL(0x0246a810,0x55ea2317),LL(0x4ad388ad,0x14671291),LL(0x6a13372a,0x93633c21), + LL(0x3a2d7c85,0xcb969eff),LL(0x73683dda,0xe59df301),LL(0xd6450189,0x07e6d627), LL(0x5bb295f6,0xdddae32b),LL(0xe18b49b4,0x6746310c),LL(0x9b220769,0x828865f2), +}, +/* digit=9 base_pwr=2^63 */ +{ + LL(0x06290b5b,0x76334d9f),LL(0x1c02aaf4,0x096fa00d),LL(0xb1c345b9,0xb29c7abc), LL(0xdb7dfee2,0x91b1e9d2),LL(0x28848019,0x0c63c40d),LL(0x3e9d93a9,0xa131323f), + LL(0xf13eb9c2,0x1935ea95),LL(0x2c20a03b,0x37b07e4b),LL(0x2897bd62,0x7ccebf49), LL(0xae088f35,0x671df286),LL(0x1a339948,0x9b826c19),LL(0x5b176e1c,0x2e503224), + LL(0x6ba550e2,0x674ed6b8),LL(0x1cda326c,0xccbad8d2),LL(0x99aa3448,0xd1542f83), LL(0xbf457736,0xcf8eee5d),LL(0xcebef35d,0x40007023),LL(0x3c3cacd6,0x4f6c8f17), + LL(0x4971b9d2,0x5794be45),LL(0x22f3503e,0x64a2a44e),LL(0x337993cc,0x62069df9), LL(0x8cf5aa65,0xb5a8be91),LL(0x4266a4f2,0xdcf77c1c),LL(0x479a02fc,0xbbe137f3), + LL(0xf8b77a15,0xbd930e7c),LL(0x565819b0,0x1f604624),LL(0xc4a43c29,0x9a01433c), LL(0x8a108d0f,0x8cb9e675),LL(0x86e761d4,0xa4cf1e24),LL(0x30951662,0xf54b8c4e), + LL(0xa45636a1,0x5b66a405),LL(0xaaad9d66,0xaaddb23d),LL(0xb2f86dd3,0x896cbfcb), LL(0xdb9ca210,0x01d8e6d4),LL(0x6cca4804,0x6623a1bd),LL(0x51bacaa2,0xf657c1fc), + LL(0x3447e4e1,0x94172f67),LL(0x9c87c415,0x54f322cc),LL(0x9e98c65c,0xdc8ef479), LL(0x45d91721,0xd3538e9b),LL(0x1506e6bd,0x6291aa36),LL(0x13f2dbec,0xf61e47e5), + LL(0x1a8522ff,0x95dc6196),LL(0xb2a0c2fd,0x7800e4de),LL(0x493d2023,0xb2d8b5c9), LL(0xb3d564a2,0x2c595ab6),LL(0x4e856f46,0xee5106ea),LL(0x0d824ff7,0xdf6ac56c), + LL(0x8d5303f8,0x0cc840ed),LL(0x019b0602,0xb756ad36),LL(0x723c0494,0xe56dfdab), LL(0xd268efe1,0x8ba759dd),LL(0xc3466388,0x6c72c1b7),LL(0x85cf6c47,0x7a5f5d09), + LL(0x4fe757ec,0xdbab4d4a),LL(0xdb01add0,0xfb7fb787),LL(0x6922f7b1,0x521bd604), LL(0x6b994d8e,0x4c317222),LL(0x9e46ab73,0xa7a17e9e),LL(0x20326632,0xa39d84d6), + LL(0xa2841ffc,0xa302b6c0),LL(0x4e85da4c,0x4809d2c3),LL(0x16eeee34,0xf8bea692), LL(0xfda40073,0x049d0306),LL(0xbffc68be,0xe8956671),LL(0x21e27827,0xf5ae3b98), + LL(0xe56f2481,0xcf5a16d7),LL(0xcb81a7ba,0x3def2594),LL(0x032f3df9,0xe07beed8), LL(0xc369a737,0xab17091a),LL(0x438b3620,0x0956e828),LL(0x10f465fc,0xab6ecf14), + LL(0x6b2649d8,0x6e182bdb),LL(0xb09ecfad,0xc7601742),LL(0xeb6286a3,0xb43cc8c7), LL(0xb948e18a,0x89b48a00),LL(0x42ed648d,0xe337b779),LL(0xe4fc127d,0x1da0987e), + LL(0x88ae8b76,0x45aa621b),LL(0x3162f9f7,0xb127ad6f),LL(0xc8f2cb42,0xeb56c7f7), LL(0xcddb2b13,0x6b7993cd),LL(0xfba0d637,0x6f2f2d51),LL(0xb16542d8,0xcabe79c9), + LL(0xbde530d9,0x6753d960),LL(0x9b8986a8,0x77b91029),LL(0x9cefc1c4,0x5af09d8a), LL(0xa78c0b08,0xb396dcb1),LL(0xb5928a9c,0x44dc48ef),LL(0x2f48a964,0x9fafb33c), + LL(0x408cae9b,0xa2a98dd1),LL(0x61c6bec6,0xb3ff192c),LL(0x3ead88da,0x607b0c64), LL(0x63184290,0x0039f8e6),LL(0x6dd9836f,0x0b9f09db),LL(0xc51a47fa,0x5d9d8f22), + LL(0x166920d2,0xe9cf521f),LL(0xbec39b63,0xaf5b1d19),LL(0xee934b22,0x3099fced), LL(0x959116f1,0xef6f72af),LL(0xacea8ba9,0xf710ef28),LL(0x13990b15,0x588d2bbc), + LL(0x86049369,0xcbe2755c),LL(0x46e988e6,0x738787b9),LL(0x4a2e3319,0x520ae3e4), LL(0x932e0896,0xf7e43ece),LL(0x2d3989f8,0x906f0e67),LL(0x9e24fec1,0xc4ce28e2), + LL(0x9519aaac,0xd4391f09),LL(0x1c2d3eb4,0x923a951d),LL(0xeaa8d831,0xbba9e206), LL(0x3b0369ed,0x86a8f819),LL(0x84f45ee3,0xf08f5db8),LL(0xc7fd4306,0xde9d97b7), + LL(0x82162e5e,0x4abae764),LL(0x820c91df,0xda046c14),LL(0xcac95307,0x9be5686d), LL(0x1926ce21,0x8d90109e),LL(0xa11ddecb,0x346eafc4),LL(0xd72ecc75,0x3ce252dd), + LL(0x3d30e9b1,0xb559793c),LL(0x21faa664,0xcf651f10),LL(0x0af936f8,0x58827716), LL(0x3390af36,0x2e460004),LL(0x2e93e7e8,0x355c08cc),LL(0xda2c6727,0x84eb137e), + LL(0x2625e961,0x9e901d5b),LL(0x499b3624,0x53244f50),LL(0x3c07fdd0,0xf7b5e924), LL(0xc8b4fccb,0x5e1b6db8),LL(0xd9a4ede5,0x4d6122ee),LL(0xf719582d,0x96df7864), + LL(0xc9be27d1,0xeafcb131),LL(0x985a8e07,0x86458dd3),LL(0x63ff40d4,0x3e039563), LL(0xd1407667,0x2541540f),LL(0xd9795e3f,0x8b27f2d9),LL(0xd60d0743,0x5dbbd4b3), + LL(0x605be24c,0x2b496309),LL(0x496bfcf4,0x3afbea78),LL(0x46ed79bc,0xfe6069c6), LL(0xad7bb67a,0x3cbeb331),LL(0x37463aa4,0xeedcaf16),LL(0x7c12f87f,0xc41a07ca), + LL(0x2fccd260,0xbd41b63e),LL(0xc658b3a5,0x3bb5676c),LL(0x15c9ccb5,0x84c591dd), LL(0xa5fcc6ca,0xc5573715),LL(0x9fcba543,0xf1800487),LL(0x63da6fef,0xfb887cbf), + LL(0x9f37dc0e,0xfd99db21),LL(0x5783e53d,0x85bbbefe),LL(0x09d5d09d,0xa10360ba), LL(0x3cc09837,0x12e9679e),LL(0x793efd44,0x81a4b2dd),LL(0x8153dba4,0x0c93ed4d), + LL(0x2883764f,0xb7b5a250),LL(0x75b781bc,0xd3ca90ea),LL(0xc5198673,0x8da40a3f), LL(0x237103fa,0x434744ad),LL(0xcbd65a76,0x16d7466c),LL(0x9d0ce6ec,0x97f5ff82), + LL(0xc3b3f666,0x0134d746),LL(0x7c3910eb,0x62a3254f),LL(0x6906613f,0x989510d0), LL(0xe75bfea7,0xa63aead6),LL(0xf6f48b5d,0x2c7bb201),LL(0x0cf11cdd,0x17c485c4), + LL(0x01346a36,0x44072775),LL(0xeccf5986,0xd81f9e8b),LL(0xffca3195,0xcedbaa7d), LL(0x5744e89d,0xeea2a618),LL(0xad358476,0x65f3b598),LL(0x49cae477,0x31ef0e2a), + LL(0x82f6a471,0x0f618b7c),LL(0x437cc826,0xac49eab7),LL(0x7209d33e,0x6ac9e2b8), LL(0xa161856e,0x14857b21),LL(0x638a1099,0x2c515ae8),LL(0xe8f5e86f,0xd34ba334), + LL(0xdf15afb4,0x96dc51a8),LL(0x5b39ee73,0x0db5cc4d),LL(0x59face85,0xb166ae3d), LL(0xb3c9bf88,0x94ee7aca),LL(0x34ba27bc,0x0724cb81),LL(0x8aa869b4,0xac1a10a4), + LL(0x621e0951,0xe85de117),LL(0xf75855ac,0x7b464098),LL(0x53d1abe6,0x06b33c28), LL(0xb538c2cf,0x095bdc74),LL(0xd859366c,0xa64b61e0),LL(0xe9922d24,0xa5c92136), + LL(0xc221fe54,0xa1285a95),LL(0x7964a5fc,0x50c3e10e),LL(0x497b1e90,0xdbaa4092), LL(0xa9408c0f,0x3f4e7d97),LL(0x92b70f2b,0x1e83c2f5),LL(0x6911dac9,0x2ed8ab9c), + LL(0x73b888d5,0x0bc16a46),LL(0x8a011451,0x4305504a),LL(0x5de19cb5,0xea5d6a02), LL(0xa7166a08,0x8571e2f0),LL(0x6ea97470,0x9edcb0ba),LL(0xd6fc7b1d,0x1ed74e9b), + LL(0x48e4b5fe,0x960652fe),LL(0xd0e6f19c,0x391d4162),LL(0xb59dd2ef,0x8c6e00ef), LL(0x95fd71ce,0x0cc08f13),LL(0x8fedafc2,0x6d81789b),LL(0xe498c253,0x3ec56b1a), + LL(0xb7edb5da,0xb9bf335a),LL(0x693e2fe3,0x1a3b9636),LL(0x94c320a4,0xb12ae440), LL(0x35ef6698,0xf7d1986e),LL(0x273f0a93,0xc1db1a91),LL(0xadbd81a4,0x72fd29a3), + LL(0x1dcb4e55,0x57a9af55),LL(0xa46743c4,0xa430b677),LL(0x4459d987,0x2e2d09e0), LL(0xc1831698,0x58bc26b8),LL(0x8e8a86f9,0x02cdfd4d),LL(0x86532393,0x81824a71), + LL(0xfdf7070a,0x4b7d4311),LL(0x46b2d2cd,0x790ff04e),LL(0xa669dca9,0x5f194b6e), LL(0x6ee0a73c,0x8d27f077),LL(0x3f03f66f,0x7db8300e),LL(0x52542218,0x894792c2), + LL(0x320a391f,0x709e970a),LL(0xd301afda,0xffc55682),LL(0x187a421b,0x8c7c2ac7), LL(0x05c2a76b,0xb9413568),LL(0x50d693ea,0x4ce64d45),LL(0xf8118a62,0x133a1d85), + LL(0x1634f569,0xb78ee791),LL(0x0985837d,0xc8fdd735),LL(0x42337dc2,0xb40ab147), LL(0x2ece8062,0x58ee37a9),LL(0xb4fb1ae1,0x1b80e3de),LL(0xddf42642,0x55797958), + LL(0x3a4a5d2b,0x4ced5be1),LL(0xf46469b5,0x537cc02f),LL(0x9140aadc,0x369faf66), LL(0xbba54c56,0x91646050),LL(0xf1627bac,0xb6ae4660),LL(0x03bf4ace,0xd39a1183), + LL(0x5ad40b74,0x9a746086),LL(0x09825b65,0x8d116e80),LL(0x4c9e508b,0x66642780), LL(0xca3af3bd,0x10c1db38),LL(0x67df4e51,0x91ca8866),LL(0xadf6517e,0x3b4ebf22), + LL(0x69754168,0x906ece9b),LL(0x3842db37,0x78a4999a),LL(0x45118f97,0xa54934b5), LL(0xd6600715,0x17c76b7f),LL(0xfef1a9a9,0xec8d0be6),LL(0xdaf01223,0x80902814), + LL(0xe7382aa9,0x8ffad587),LL(0x64489fbc,0xff48de9c),LL(0xc8d3ef62,0xe539dabd), LL(0x6e44245a,0x037fa21e),LL(0x6ffc89e7,0x44aeb6c6),LL(0xc24d18dd,0x903826c9), + LL(0xfa798920,0x7e6202ad),LL(0x97dc220c,0xa69a704f),LL(0x27b68255,0xd7e7cafe), LL(0x52b2a274,0xa6aa0342),LL(0xe49cef76,0x72ecec87),LL(0x82afed60,0xee4bbb4a), + LL(0x7c8213f0,0xb7bfa6d1),LL(0xe2e5eb2d,0x55e77d8a),LL(0xecb57d00,0xde5c925a), LL(0x91bfc135,0xc67d7eea),LL(0x14df6915,0x6915f948),LL(0x8e5e081a,0xbc1fffee), + LL(0x4f0a6b86,0x1ca04a6c),LL(0x07a69d26,0x3f736642),LL(0x8eb91e1c,0xbcf425f5), LL(0xf7a230eb,0x0ee5e1c6),LL(0x11f1574d,0xafdd21cd),LL(0x31d6dc6d,0xb5ef9bf3), + LL(0x7cc6bcf3,0x6774ed5f),LL(0x0a75d23e,0x6feadf8b),LL(0x81ec8c8a,0x5898828e), LL(0x414cd64b,0x3e9103bb),LL(0x1542efe4,0x703f0fb2),LL(0x7388d6c8,0x3df2c7c6), + LL(0x60a7e99b,0x02e73081),LL(0x7986406b,0x1b3db98b),LL(0x3fff6009,0x9959bac0), LL(0x75e1d2fa,0x80625a36),LL(0xa198da2b,0x95492b2e),LL(0x59742464,0x81018f71), + LL(0x3cdf8db3,0xc1fc725f),LL(0x59326d48,0xd58228f1),LL(0x1d74eaaa,0x4c0a442e), LL(0x1b886d60,0xa814c005),LL(0xfaa58d26,0x9756df79),LL(0x176bed0c,0xc17f3346), + LL(0x7ec82b2a,0x874840bd),LL(0x1e1e2156,0x8b253112),LL(0xd1a58bca,0x8264f8ad), LL(0x23a4105c,0xc2a71227),LL(0xf0882599,0x8add8a3d),LL(0xd491e94b,0xfadbbc8e), + LL(0x0f86b5cf,0x8d48becb),LL(0xdd83a12b,0x700c6428),LL(0x87164a6a,0x9175e238), LL(0x300636b7,0xdd10d85e),LL(0x080c01ea,0xec0b0e20),LL(0x18c937fc,0x5b27d26f), + LL(0xaba54a99,0xe45deb1a),LL(0x88f0f0e9,0xea11e26a),LL(0x8d1c5c7c,0x3f29d26a), LL(0x261af059,0xaad1697c),LL(0x8bc6950e,0x65b29644),LL(0x3940ae67,0x33e4b7c4), + LL(0x18f510ad,0xe7e52f31),LL(0x4aba84d1,0xecf901f6),LL(0x1d0d87ae,0xe6d9492c), LL(0xf4371e3c,0x21861ebd),LL(0xac94b579,0xd7f34636),LL(0xe45e7910,0x72e22e1d), + LL(0x0d44e5c0,0xe0a30cd5),LL(0x672259aa,0x2d74d417),LL(0x48769d5d,0x208c07a6), LL(0x3ab04b13,0x0182f535),LL(0xd1493acf,0xa88ddf33),LL(0xe65f0e82,0xd692f755), + LL(0x200e00e2,0x57480257),LL(0x80f4a13a,0xd24d38be),LL(0x8b6535bd,0xe2d4702a), LL(0x6d6d0cd9,0x597abaed),LL(0x1a32e8b7,0x2eaf61f2),LL(0x06a3acc5,0x48a99a4d), + LL(0x39489527,0x25304144),LL(0x832761c7,0xb36d9b74),LL(0xaa187256,0xe33bc0a7), LL(0xa72fb33d,0x4d4e2c96),LL(0x7ed273fe,0x1dba3ef7),LL(0x577de798,0x2b9978d8), + LL(0x54644ced,0x467e81b5),LL(0xc2a02440,0xd4ef3e1b),LL(0x15b3a380,0x0df99eec), LL(0x250b43d1,0x549afec6),LL(0x1598f520,0x0c4fc7e2),LL(0x67422c36,0x9f8e42d7), + LL(0x6c0213e3,0x3a3886aa),LL(0x8b5b752a,0xc12533ce),LL(0x814f4729,0xbe235906), LL(0xbffc94e3,0xdf45bc94),LL(0xf2a001db,0xccdad604),LL(0xb8cdcec3,0x11ddcaa6), + LL(0x25ddf146,0x7eee967e),LL(0x0bf6e4d3,0x9a2d1d73),LL(0xabd48b07,0x57a43f05), LL(0x32b3a8a7,0x77d9ad80),LL(0x55872fbd,0xfbab22cf),LL(0xd02e7501,0x2100eede), + LL(0x25faa7bf,0xf4f4c899),LL(0xaa9fc84a,0x235a5736),LL(0xfae4fdb9,0x37647c20), LL(0xa26ae472,0xfc3f8ca6),LL(0xa2f519e3,0x01c6ea11),LL(0xc2fc610f,0x1ce14f58), + LL(0xb767a1fd,0xb6f923fc),LL(0x07e07c0c,0x31a760f3),LL(0x6a7085a9,0xe913fcfb), LL(0x40442930,0x86adb2bb),LL(0xe41a1b14,0xc8c20759),LL(0xd8d1924a,0x399d5ad6), + LL(0x76ac6874,0x6f0508e8),LL(0xafba3d82,0xd96bd7cd),LL(0x56cc2c07,0x336b813e), LL(0xb0c7906c,0x0fca9b73),LL(0x232e056d,0xb85518fe),LL(0xd1c21ece,0xa97bdc68), + LL(0xfbc66efc,0xf3327dfd),LL(0x34a1c591,0xa401731e),LL(0x3371b4b7,0x8c1ca43a), LL(0x526198b2,0x835393dc),LL(0x686e9c85,0xa8d2d86f),LL(0x7a0c3d07,0xa4491de7), +}, +/* digit=10 base_pwr=2^70 */ +{ + LL(0x309b2b98,0x790e100d),LL(0x50a4f09b,0xaf54cb0b),LL(0x956ec705,0xbfd5af9f), LL(0x3bab82f5,0x4692a125),LL(0xec637260,0xac0be759),LL(0x580441bf,0xaf835bc0), + LL(0x513c3f3c,0x80ab94a2),LL(0x77e64a9c,0x39c245d0),LL(0xa7543bd5,0xc3ba4002), LL(0x7eacb224,0x6658e2bd),LL(0x1f419c54,0x7e005bba),LL(0x58b62644,0xd54bc0f7), + LL(0x9194cd13,0x5c1c461d),LL(0x95f13f43,0x843427bf),LL(0xba86794a,0x6844d36f), LL(0x393e443b,0x90e5c334),LL(0x9f72ba01,0x8cec77f9),LL(0x145d81fc,0x28ef5fba), + LL(0xdd9dc934,0xa6b331c2),LL(0xde2f138b,0x32a15ddf),LL(0x2b7697a8,0x490117ee), LL(0xb3b1e935,0x5eaf7f20),LL(0x9b807876,0x4b750ed0),LL(0x864be6f5,0x98f1673d), + LL(0x9a5d2cee,0x4a36329f),LL(0x373aae9d,0xd6288e69),LL(0xd918ee97,0xbf2fd4fb), LL(0x870b1cdd,0x020ef080),LL(0x22a9b3e7,0x5578e9ae),LL(0xeb392f0b,0x8887e7e2), + LL(0xf5b8a656,0x533c438d),LL(0xbc6d32ae,0x98531dda),LL(0x1af7184c,0xc8e1927f), LL(0x393cf3c4,0xb9f8c2c8),LL(0x73ac1607,0x37cffce8),LL(0x3312e5ff,0x76d8c894), + LL(0x2b57f886,0x9a855c17),LL(0x3f649b7d,0xa334a009),LL(0x6e284b9c,0x90468704), LL(0x008bb313,0x633d15c5),LL(0x0ec3ca93,0x384f430f),LL(0x83b412c8,0x109e0fec), + LL(0x1a53d7a9,0x53582049),LL(0xf0099876,0xa35696c9),LL(0xfab32bde,0x34991882), LL(0x9092df43,0xa6e48215),LL(0x0f2f3443,0xf674d02f),LL(0x4d862a69,0xeeb91808), + LL(0x8ece35fc,0xdc9c1776),LL(0x9ab87051,0x7e6e4720),LL(0x7fd83120,0xf6d82143), LL(0xa0bcd27e,0x7ee28542),LL(0xd6a86383,0x5e5835d3),LL(0x3ddae47d,0x1b6caf5f), + LL(0x48ce8d7a,0x66cc8b26),LL(0x5f4e54c1,0xb8f25b70),LL(0x8abade9b,0x9c222105), LL(0xe91600e2,0x6a5880be),LL(0x55c01418,0xd8214454),LL(0x3e0670f2,0x1510a747), + LL(0xefd35bbf,0xe70de601),LL(0x01575c49,0x23fea84f),LL(0x04ef42e0,0xc9dc4bb5), LL(0xb139d368,0x01d78685),LL(0x4c1d31f0,0x6acc3ed7),LL(0xbeac2fcd,0x7ebf6809), + LL(0x8b17cc56,0xb221b461),LL(0x6c918f42,0x3a388a6c),LL(0x61ec2c4a,0x43b22feb), LL(0xdd8892c4,0x1f31f72b),LL(0x7aae5ed6,0xff9e63db),LL(0xc5aa19b7,0xcf33f8e0), + LL(0xb6d01e70,0xb2b290ae),LL(0x70c988ca,0x32377051),LL(0x5c47c52d,0xb49ee722), LL(0x73ed265f,0xa41919e1),LL(0xf4708b44,0x41428dd0),LL(0xda4fedfa,0x15d760a1), + LL(0xe8537c4b,0xfed502b6),LL(0xfc7dde03,0x8bab770b),LL(0x7d1f76d0,0x05c46da4), LL(0xc0db9267,0xc8dcbdf0),LL(0xcf6393d4,0x01cc7b6a),LL(0xeaf6dc69,0xab93ff59), + LL(0x9da53ff6,0xc026d3db),LL(0xd6c5a028,0x6b07911d),LL(0x9f01eec9,0xf2e88f7a), LL(0x35fb3be7,0x30ac879a),LL(0x9c8e073c,0x92871e9a),LL(0x44eb456d,0xc18518f1), + LL(0xe32b5a2f,0x2ca049f6),LL(0x3c2dc1f8,0x164d195b),LL(0x0eafa9d1,0xd1364e77), LL(0x59bac901,0x87a85a1f),LL(0x31b28912,0x040599eb),LL(0x73989f3d,0xe5138d57), + LL(0x59e18159,0xda271207),LL(0x87bdce2f,0x62d2e07e),LL(0x9c9da191,0x00bc160e), LL(0x9f9f3edd,0xc2aa6fab),LL(0x3e0cc568,0xd0bd4459),LL(0x6d2f3dbc,0x80621f6a), + LL(0x2608caf0,0xc02d8317),LL(0x3b8d5f88,0xb2b7cf8e),LL(0x97aa5235,0xe268620c), LL(0x53373262,0xc5d88092),LL(0xcbdc45c7,0xb80c631a),LL(0x3758fbc8,0xa82fa7f0), + LL(0x61946f84,0x71ca7112),LL(0x26f634dc,0xdbb91c74),LL(0x48b33bf4,0xc61feaa2), LL(0x0f0b622c,0x8c1f4e50),LL(0x4a976276,0x54a1b037),LL(0x9ac35cd3,0x46fdaca0), + LL(0x62a4a464,0xa78d6498),LL(0xbe50e07c,0x8185cc99),LL(0x046a15b3,0xe0df6a84), LL(0xb1747017,0x3fbd2fe4),LL(0xd8a28d37,0x40bd174e),LL(0x3ba62fe3,0xeb67a57c), + LL(0x9cd4dbf3,0xc674a040),LL(0x8ef4b42f,0x342823c9),LL(0x82a815c3,0x51bb59b7), LL(0x56fc0c39,0x4b239627),LL(0xc5f5d825,0x28ec23b4),LL(0x6c8e1e27,0xf7c1b3d8), + LL(0x27ec4b30,0x9635ffa2),LL(0x100ea34e,0xc42039eb),LL(0x90076fbc,0xd912642e), LL(0x1f77514a,0xaafcaa31),LL(0x0fb82d1f,0x79d17151),LL(0xb2f1317a,0x7c0989e3), + LL(0x89f462e7,0xb4998e80),LL(0xe6fbd086,0xb36cee42),LL(0x2313e077,0xc8121a21), LL(0x2726f97c,0x5989f410),LL(0xcd4d494f,0x0186977b),LL(0x5a5b30ab,0xc3fb6b78), + LL(0x424d3124,0xf9bb1829),LL(0x22c5cfed,0x4def4592),LL(0x116b8380,0x635090f0), LL(0xedc582ce,0xdc36aaf6),LL(0xcb88543f,0x8c97c600),LL(0x79d7d839,0xea4cedf6), + LL(0x3a7fe603,0x8e22dd53),LL(0xb6e13252,0x4f6646e7),LL(0xcc10cc0b,0xecb71522), LL(0x8fe0bd6f,0xb14b31d5),LL(0x7a6c5efb,0x2fe52f94),LL(0x1d6b6945,0x46144410), + LL(0x41fe4d44,0xd9cc6cd5),LL(0x3e28dc8d,0x358e33f3),LL(0x521ff50b,0x1a26ddf8), LL(0xa3eab01f,0x3ee649e8),LL(0x03050e22,0x98968099),LL(0x12ed1dd8,0x49721509), + LL(0x1e67197b,0x384a1493),LL(0x56c98a4b,0x0d7943cf),LL(0xaf2f7e74,0x23d774ac), LL(0x79355a55,0x2c21ca0b),LL(0x1fad2d85,0xf5e43dea),LL(0xa8d0b6bd,0x57ab8f61), + LL(0xd56cc2a3,0x0e518afc),LL(0xe2dc0d52,0xc6a83b40),LL(0x809ea39f,0x6f5059aa), LL(0x2721bc4d,0x94390dc2),LL(0xfb429338,0x879b35f1),LL(0x14546ddd,0x3891b090), + LL(0xc56f4571,0x7b43040e),LL(0x5fa40c73,0x0b5a0144),LL(0x274f5c74,0xbe5cf035), LL(0x9af86d27,0x9e879540),LL(0xd3c0d8b2,0x77e49fb8),LL(0xc770f9db,0x61fb43b6), + LL(0x92022253,0x4ad13b39),LL(0x71a9d129,0x8396b4e5),LL(0xdce6f688,0x8182898f), LL(0x9bab196f,0xfc0794cd),LL(0x7d4e5580,0x4100be42),LL(0x73474464,0x1417be29), + LL(0x0b909777,0x50d8f53e),LL(0xb81656d9,0x9e5bcdce),LL(0x68e77a3f,0xd3e75c90), LL(0x22b5d762,0x3e315195),LL(0x1511374c,0x1fe3bea5),LL(0x39e43804,0x7dcf6bb6), + LL(0x98b4883d,0xaa84b4ab),LL(0x5003afb8,0xfb1203a4),LL(0xf113ef60,0x49b549bd), LL(0x421a26db,0xb8a534eb),LL(0x4176bcfa,0xae3abef5),LL(0xc9fd2e50,0x65ad7ddc), + LL(0xc9927e31,0x32999234),LL(0xc5f17855,0xcbc9be39),LL(0x91ce524e,0x4189b3bb), LL(0xedfa4e44,0xc72add92),LL(0xfa0ae77e,0x59dfa5aa),LL(0xd3b54994,0x1bc2e1a2), + LL(0x743258e5,0xc7638300),LL(0x4e7252c9,0x508d7943),LL(0x39297969,0x882f689a), LL(0x5a87da5a,0x87d951d3),LL(0x24116564,0x21c31a28),LL(0xbf2be0ff,0x76451670), + LL(0x500e0aee,0xeb561121),LL(0x89b20709,0x1d9b8fb4),LL(0xc9f555ae,0x7ba3f437), LL(0x276c8c91,0xc08e6586),LL(0x6d0f76b9,0x7daa15eb),LL(0x3c2bff70,0x1769cb69), + LL(0x242a43b4,0xbd62fc3b),LL(0xe4ceb019,0x853d3ce1),LL(0x267cdfd1,0x81780739), LL(0x9cd6475c,0x60ecb687),LL(0x48a209f2,0xf3e199ed),LL(0x02dae3a1,0x3cf6b9f3), + LL(0xe7cec18a,0xb0e37780),LL(0x98f8a592,0xaa6eb4e7),LL(0x2d0d06e7,0x0427150e), LL(0x911cd620,0xef4a40dc),LL(0x6e271737,0x71162ad1),LL(0x700d088c,0x82437eca), + LL(0xf1cc972e,0xee1967ba),LL(0xd4eddaa0,0x920e2ab5),LL(0x985d7040,0x5c5ee245), LL(0x3950b6ed,0xb0b435d4),LL(0x364257b2,0x3d814c4e),LL(0xec06a200,0x5baa5fcd), + LL(0x79c5f683,0xea62de1d),LL(0xf682ff65,0x02d1dab0),LL(0xad34060a,0x001e1565), LL(0xbc38c15a,0x09c411dd),LL(0xda6f1853,0x5f688345),LL(0xdd138b16,0x3a60f9df), + LL(0x0bcf4074,0xb21d2687),LL(0x66ea60fa,0xcb9d6c7f),LL(0x86febc79,0x33b4a213), LL(0x9bb71bf2,0x6c5e209c),LL(0x63580404,0x53f871bf),LL(0x0c55bce3,0x4c4a3634), + LL(0x50a5c1d5,0xd05e1b75),LL(0xeda43a63,0xac2eb90d),LL(0xff9c26a8,0x66eba3d3), LL(0x3c837bb1,0xe4f6c13e),LL(0x6e9b77ce,0xee6c1ed4),LL(0xf727fe74,0x7107e1fe), + LL(0x88d7d553,0xbd0548b8),LL(0x4afe5186,0x78876c17),LL(0xa7285f30,0x621d5e10), LL(0x5a304147,0x051e0777),LL(0x334a494d,0x5390f3bd),LL(0x4579e150,0xbebd4f2a), + LL(0xc9672b55,0x0aa436fe),LL(0xaf1550c6,0x52a75c26),LL(0x3858b864,0xd0cbe739), LL(0x0d008d67,0x8f10eef9),LL(0x72b9fc05,0x8f75f104),LL(0x364e6b20,0x143ef589), + LL(0x533b4c98,0x2be2a8f3),LL(0xc7ae49a0,0x982b2540),LL(0xefebfa00,0x546e27b9), LL(0x900dd378,0xb5d294cc),LL(0x3658dbf4,0x3be2150d),LL(0xb34a20e7,0x02874ed2), + LL(0x40cc25ec,0x478a065c),LL(0xada695ef,0xbeaf90cb),LL(0x862215ea,0x4aae42ff), LL(0x165174a2,0x381fc293),LL(0x3d459c99,0xe86a1fe7),LL(0x401dd67b,0xe890d568), + LL(0x82e7aada,0xb46876e1),LL(0xd804209c,0xecca3439),LL(0x1d006b77,0x2c44843b), LL(0x8dd030ac,0x61576007),LL(0xfcb53cac,0x43b0a3df),LL(0xdebeb1f9,0xc5ca915d), + LL(0x0ca9fabd,0x261ec1c6),LL(0xcd7cb986,0xef1e6df1),LL(0x2bbbbc23,0xa827cc65), LL(0xcbd255f5,0xbe5fb899),LL(0x512312f5,0x909568d8),LL(0xa6726a82,0x681dd29a), + LL(0x488ad19a,0x0de34649),LL(0xa836d120,0xaedbbf66),LL(0xc3d2bb79,0x3db458a5), LL(0xf1b23abb,0xa516213a),LL(0x1fb13cf4,0x57cb0ebe),LL(0x2a71fb5b,0xb43b7de1), + LL(0x5a77c99c,0x4136b8b1),LL(0x8ea5e01e,0x7e568b1a),LL(0x63a9ae35,0x218b1234), LL(0xe124bab5,0x31875171),LL(0xccc750f2,0x8b5a214c),LL(0x085b0bf6,0xcda64152), + LL(0x65aa7c4a,0x823a1144),LL(0xd237a179,0x887834cc),LL(0x79b0e325,0xa2884253), LL(0x71245d03,0x05200952),LL(0xefa5fe27,0x6b7e9f1d),LL(0xd8acec2b,0xda6faffe), + LL(0xf6bd56f2,0x8136bac4),LL(0x996005c4,0x13e6d8cc),LL(0xa4d43df3,0xd7ea43e1), LL(0x2f377e2d,0xfc227ad7),LL(0xa8878d26,0x90549e9b),LL(0x4aa1f76a,0xb578b012), + LL(0x4a188fe4,0x42ca87e7),LL(0x5d8e390f,0x4ba3638d),LL(0x9476562a,0xb3568c45), LL(0x0c0a6cc4,0x0c426680),LL(0x3e63129f,0x64115513),LL(0xcec9a6eb,0x711c89d3), + LL(0xd14dddc5,0x923db3a9),LL(0x7dac544b,0xd001ba98),LL(0x692ca6d5,0xc5b16136), LL(0xcdf3aa4c,0x105a9b51),LL(0x52023455,0x909991a3),LL(0xe71eda94,0x8699a77a), + LL(0x20abff43,0xde5a86c2),LL(0x5e0fe305,0x51a3ccb0),LL(0x9ea73b83,0xb76c1d2b), LL(0xf9a975a1,0x9ecfed6c),LL(0xfb0c036d,0x9c0332e0),LL(0x6889bbce,0x4e7d344a), + LL(0xeff8ef6a,0x4f5b02ba),LL(0x165b7828,0xca84ea74),LL(0x6fd1b432,0xc50328c5), LL(0x4166147f,0x56d815b8),LL(0x3da09b08,0xabfb8a90),LL(0x2d02b8d2,0x6b35b6b2), + LL(0x398c7dbd,0xbd664dac),LL(0xd96e1ba7,0xf61eb365),LL(0x65e6672a,0x4063219c), LL(0x8c7ea7dc,0x4bacb326),LL(0x40eb1485,0xcb7c53dc),LL(0xc8724e95,0x75993332), + LL(0xb8af3e69,0x65ace7ac),LL(0x92a2146c,0xc1b257bf),LL(0xcd5a446f,0x5901f7b5), LL(0x8c135291,0xfd4954c0),LL(0x7287802d,0xf9489d56),LL(0x79ca70bb,0xe671da6c), + LL(0x7342afb2,0xcf1f68a7),LL(0xd2662b38,0xb7c95d16),LL(0xb90fd0c0,0xfb1d7741), LL(0x836a9bb4,0xfd7f66be),LL(0x3f666fd7,0x9196236b),LL(0xf2666854,0x97d5dc11), + LL(0x13be0faf,0x53b96191),LL(0x4aad75e2,0x8984caab),LL(0x8c638642,0x5f016b43), LL(0xfe4962b3,0xbe9ab98b),LL(0xf41d7325,0xce94bc4e),LL(0x2727019c,0xa8761364), + LL(0xae07f4a8,0xbb80b445),LL(0x8294ccdf,0xe5881e44),LL(0xad112cda,0x3a4c8e39), LL(0x35d8c72f,0xa5340d8e),LL(0x48955a80,0x47142cce),LL(0x86d4ad66,0xb5ddcf06), + LL(0x2ba33bbf,0xdf4d64b8),LL(0x38311d97,0xb562f723),LL(0x3ae0b587,0x9f7f5f29), LL(0xc4c06982,0x7233a026),LL(0x72ac8ffe,0xe33867b4),LL(0x508eade6,0xf85acdb1), + LL(0x67a2a49a,0x8261c3a7),LL(0x9775813d,0xcb58ffc5),LL(0x932bb91b,0x06c28465), LL(0xb5e56211,0xd07de2e2),LL(0x8f5bfd9a,0x6e0b5a38),LL(0x6f9d74ef,0x1cccc9f7), + LL(0xe3a75abb,0x48de8e6e),LL(0x39374610,0x0ebaaa1e),LL(0x2106e8ab,0x003aaedd), LL(0x92e2817c,0x6ae46c78),LL(0xfbd7d633,0xc52e217b),LL(0xc280871c,0x804476ea), + LL(0x8f4ad647,0x1399bf87),LL(0x2f0e2288,0x81692c0a),LL(0x4c8fbcb7,0x4eb07838), LL(0xe3874f86,0xdb2f862c),LL(0x89f62e6a,0x9e0ee74a),LL(0xb8e8633f,0xa0fef663), +}, +/* digit=11 base_pwr=2^77 */ +{ + LL(0x596b6e25,0x3628c4e5),LL(0xe3c48c5e,0xc4406956),LL(0x2e8ae234,0xe7c4cb00), LL(0x2ed43c07,0xa11d2ed3),LL(0x77b2799d,0xc33b975f),LL(0xcaefccd6,0x3715cc2d), + LL(0x29a801e4,0x98e5a9fd),LL(0xd84bf0a4,0xfa4d3abb),LL(0x2c426bed,0x8f4dc6f8), LL(0x1b83f02d,0x9decebac),LL(0x31afcc78,0x27563177),LL(0x28657c91,0x25efdee0), + LL(0x10bed657,0x44f7577b),LL(0x05089a71,0xd2686801),LL(0x687d749e,0x44e84ecc), LL(0x2ffe47f3,0xeba9b10c),LL(0x977d2767,0xed6470da),LL(0x70a6ec4e,0x7568cede), + LL(0x5fdc4b62,0xb3dfb691),LL(0x55109b96,0xd851f390),LL(0x24e13631,0x55c4dac4), LL(0xb02dcba6,0x12c87918),LL(0xb6a6220f,0x082a25d8),LL(0xcc90ccd7,0xf8abc990), + LL(0xb4ca91b2,0xef95af0a),LL(0xa99a156c,0x693d8362),LL(0x4cc32c97,0x8b1f5532), LL(0xf16e87af,0xff44233f),LL(0x09622892,0x6fffb3de),LL(0x676ed99f,0x9c976523), + LL(0x19ea3118,0x486075b2),LL(0xe8b84a78,0x288a821c),LL(0x603792cd,0x53ce6139), LL(0xcd02301a,0xcc3dfb91),LL(0xafc2d623,0xd6745a56),LL(0x2bdb691f,0x55aa143c), + LL(0x38ddf5e1,0xb104e58a),LL(0x2b17bd48,0x504256db),LL(0x1af21432,0x9e5bdc70), LL(0x364f820d,0x478233b6),LL(0x6c7d0127,0x150c2f89),LL(0xd8d44844,0xdf664b62), + LL(0x15f1c173,0x5dd16958),LL(0xc46a9115,0x3312d6a3),LL(0xfe869bbc,0xc9527a68), LL(0xf6cbcbb4,0x6e5effcd),LL(0x362c0478,0x35bd9c75),LL(0x55bb0376,0x02094198), + LL(0x86c1d92c,0x20c0085c),LL(0xb4122bba,0xbd007e1e),LL(0x25befafd,0xd7c99c19), LL(0xa3ef65e4,0xf07b6cd4),LL(0xcfed174f,0x670355d0),LL(0xd9b990a8,0xff36e219), + LL(0xdc2a6498,0xdc527d73),LL(0x6e1a780d,0x08934ec0),LL(0x70e3a7c9,0xc2577255), LL(0xc59481bd,0x3e9b3c1c),LL(0x33ca9c3c,0x148a477f),LL(0xdf421578,0x5ac5a917), + LL(0x0d689c44,0xecd077c7),LL(0xb91f2750,0xb9946fa4),LL(0xf06d9aee,0x6a48a5de), LL(0x09b44876,0x90d6c760),LL(0x8d0df7fc,0x4768b79a),LL(0x528d023b,0x894c061b), + LL(0xeb1f23e5,0xc155c9fc),LL(0x87fb0d4d,0x30da65cc),LL(0xfe29aaf9,0x39ddc6f1), LL(0x9b7d2157,0xe1559681),LL(0x62eec022,0xe62ee793),LL(0xa973795b,0x6841ff58), + LL(0x8c5f6a6f,0x5041ffe5),LL(0xa1e49e9e,0xb047d430),LL(0xa1870cf1,0x55c67f67), LL(0xfdc4137f,0xa52cc630),LL(0xeeacaf6a,0x708f8900),LL(0xf29a770c,0x4489b888), + LL(0x427f459d,0xa9893015),LL(0x3af11c58,0x29c3a358),LL(0x3159a1a0,0x9fe19047), LL(0x16e111ce,0x3590c6ca),LL(0xb144f41e,0xbb21fd6b),LL(0x48ff3048,0x668bc4a8), + LL(0xcd902ca6,0xf80a9619),LL(0xf2ba49cb,0x75608dc6),LL(0x0d7ead99,0x1ac32280), LL(0x7b3c121e,0x0f8cc5ca),LL(0x541e43fb,0x70c044bb),LL(0x6838ded3,0xfb10ee49), + LL(0x3fe39fff,0x75b96345),LL(0xf45a96d7,0xb4aeb917),LL(0x8932cc8f,0xcc29a1ee), LL(0x682514c2,0xd1317dff),LL(0xa9ef556d,0x5cecce58),LL(0x80d7dfc4,0x51b5fe83), + LL(0x496e1c06,0xef91106b),LL(0xf293c76e,0x28e0bd07),LL(0x504a0bfc,0x42b5d427), LL(0x507248f9,0xe4090019),LL(0x391b523a,0xf41922a2),LL(0xa9b23a0a,0xf8bf6bbc), + LL(0x03c42d14,0x4538e84f),LL(0x125c5ecc,0xc3304e31),LL(0x0528582e,0xdb87c51f), LL(0x63617411,0xd5eb0dd8),LL(0xbab74fe4,0xf4502fb9),LL(0xbda4957c,0xce6a62a7), + LL(0xcc524a7e,0xa34e113a),LL(0x0e326637,0x256f2404),LL(0xe9251c73,0xeb005bad), LL(0xd5be4e06,0x3ac0298f),LL(0x97f2b09f,0xb76eb4f5),LL(0x71ef690a,0x93965d52), + LL(0xc620384d,0x1050467a),LL(0x466e292b,0x8b677cf7),LL(0xe11cabe6,0xf2ac4850), LL(0xb63594f1,0xcec62350),LL(0xe4d91805,0x9afedaa5),LL(0x27dff452,0xd7455a8e), + LL(0x1f700bc7,0x19af1010),LL(0xc8359c70,0x5cd5ad5d),LL(0x70f71fce,0xc8be23e9), LL(0x467d70f8,0x80764bb7),LL(0xf6206a40,0x62069652),LL(0xdd25262a,0x17473085), + LL(0x1ba99063,0x32fe5625),LL(0xdb47d5bd,0xac325742),LL(0x30faac68,0x1621ec3c), LL(0x1c66729b,0x1956c250),LL(0x073aef1f,0xb76bcafe),LL(0x5159e850,0xeea7292a), + LL(0x090f7e37,0xc1a04582),LL(0x3f65d4c2,0x8a13821a),LL(0x10d83d97,0xce32c6c5), LL(0x26738cf6,0xedabac6c),LL(0x1200113a,0xf2615b30),LL(0x4c0415f6,0x7cee46f2), + LL(0x6d603672,0xabe36235),LL(0x4bf4eb2c,0x5f4f1ae8),LL(0xa26e6c0e,0x9a8e44c8), LL(0x167b41d3,0x0944d401),LL(0x76f5c7d2,0xb60f0fd0),LL(0x0f6051df,0x5efdc9a4), + LL(0xd2fd3d8c,0x456e8957),LL(0x5374a59a,0x22dbddfd),LL(0x2a3a89ab,0xbac78cef), LL(0xc10c1d79,0x9c3ba82a),LL(0xa3dd19d1,0x1f935fcc),LL(0x1919803f,0xeb08bb8f), + LL(0x6500b9f4,0xddb31f74),LL(0xe0649288,0xaf383638),LL(0x861c8b10,0xe0dbe97a), LL(0x76552cfe,0xca7ec939),LL(0xfe33f78b,0x7aec0efb),LL(0x11123c97,0xf4548120), + LL(0x53e3f6a8,0x900bdaa4),LL(0x03a79d16,0xbe717d84),LL(0x5f55a064,0x3c99d359), LL(0x7553c946,0xdfd860a0),LL(0xf4133310,0xb4480bb2),LL(0x4f946025,0x6401d83d), + LL(0x1f75cad5,0xa03d7c76),LL(0xf4042451,0x97da3fcd),LL(0xcbdbfbe2,0x6a1b945d), LL(0x7a8cce9d,0x516a95b8),LL(0x30905cc3,0xb16e3a1e),LL(0xfcfc0d25,0x6adefe9e), + LL(0xae7626a8,0x3d86d527),LL(0xfeb4e78c,0xdecc9c13),LL(0xf94a1fd5,0x3ba67e62), LL(0x8e7a588c,0xf5ac7572),LL(0x0fd2201e,0x2b741d8f),LL(0x20073536,0x3be1b453), + LL(0xcba034d7,0xe2b80fd1),LL(0xb0031b65,0x6edad8d5),LL(0x927a36ec,0x3aee1cb4), LL(0xe39b9efd,0xbb88d751),LL(0xc92d3641,0x0129b42f),LL(0xd09b53ac,0x4b434faa), + LL(0xdde24bc8,0x15b64975),LL(0xb1d4c6ee,0x481df832),LL(0xb358244d,0xae6cc85d), LL(0x6e6afac6,0xd54043d7),LL(0xc89eec23,0x50af2efc),LL(0xb2d29f99,0xf5fab1cc), + LL(0xb6f646fa,0x41e0d32a),LL(0x6faa2688,0xec8093eb),LL(0x42550962,0x991ad76f), LL(0x07681a7f,0xcd41aff9),LL(0x0304701d,0x1316c11d),LL(0x133a0046,0xcce7ab79), + LL(0x79249a06,0x34db09fa),LL(0x48f2c060,0xf3356247),LL(0xfc942369,0xe8272db6), LL(0x9849b02d,0x2b844d9c),LL(0x5af3f78c,0x8cc29c86),LL(0x79d64275,0x0bf13430), + LL(0x5c5ea1b6,0xa299919b),LL(0xd6ef8e0e,0xb6184c83),LL(0xf2084644,0x3628b342), LL(0xe68eb3e3,0x017b62a9),LL(0x6f608feb,0x6d5b1e74),LL(0x8218965a,0x1b45f2e2), + LL(0x3529b0e2,0x76707d69),LL(0x60a070db,0x64fb0f14),LL(0x83396a82,0x42737c50), LL(0x3b5d56c9,0x74698c2d),LL(0x284fca9f,0xa8fceed2),LL(0xe3d44208,0x30c3e857), + LL(0xe3bfd480,0x78614fa4),LL(0xa0e38787,0x51ce5be0),LL(0x4d57abb0,0xa92d75fa), LL(0x9fad5db2,0x8d4d97f2),LL(0x1ed2bf25,0x701a80b0),LL(0x3a2db6d9,0x8bac6dfd), + LL(0x3bb9e4df,0x092ecae1),LL(0x1a72623d,0xb57cfcce),LL(0xdac4d79c,0x38943831), LL(0x1340b17f,0x7bf10d57),LL(0x89300f7c,0xac6ec65d),LL(0xf71b46d5,0x1d4109cb), + LL(0xcb8aeb3f,0xa00a19e7),LL(0x18b1c766,0x850b5102),LL(0x6318ebe3,0x3ec2566d), LL(0x42b9bbaf,0x8e70f71f),LL(0xb35f50b3,0x777d9ed5),LL(0xccf46394,0x332280ce), + LL(0x87ceef93,0xf49974f8),LL(0x371f9494,0x06bef26e),LL(0xe9ac3207,0xa5e9736d), LL(0x9df3a98e,0x83769455),LL(0x8dd1f6f9,0x1ccdbfca),LL(0xe399f893,0x8b554d16), + LL(0xefc22c30,0x36752ea8),LL(0xff7e7724,0x104614bf),LL(0x3a74e7a5,0x306ea93d), LL(0x1c1b929a,0xae54fe5c),LL(0x289c4ec7,0xd8ed29e0),LL(0x105ca5d9,0x6153b2a0), + LL(0x29494bd7,0x5e854f89),LL(0x04a28ba0,0x85a970a5),LL(0x394d6453,0x9422f2ac), LL(0x4df93f6b,0x8e084e65),LL(0x9cbe0ba0,0x3963ac8e),LL(0x84ce0da3,0xd8aec4c5), + LL(0x9cb36f40,0xd3d12931),LL(0x36f44ee4,0x5868644a),LL(0x92d4f69a,0xa2106e73), LL(0xa33cf53e,0x2987838c),LL(0xe1a0a530,0x8085e7c7),LL(0x906c8b5a,0xfe92589b), + LL(0x0056310a,0x67d14723),LL(0x9e76d3b8,0xbb0a63ad),LL(0x299a1026,0xe8012f4b), LL(0x9d1ee6ac,0x620ecb4b),LL(0xd16c0542,0xd7f0c277),LL(0xe42209e3,0x8dc7db24), + LL(0xb0f70b40,0xbc708816),LL(0x979dc8d3,0xefe99e05),LL(0xc18443d4,0x80932dc0), LL(0xd18a619c,0xea8bf51a),LL(0x9468aa4b,0xbc18f6ca),LL(0xc3414f98,0xa296c07d), + LL(0xe4242d98,0x6cf05251),LL(0x656af78b,0x3e220300),LL(0x23a061af,0x0fdbbc02), LL(0xe31e3724,0xcd04e892),LL(0x629b6426,0xf5c0c669),LL(0xf4bf0309,0x9499d7ec), + LL(0x685aa7b3,0x0e8c0c01),LL(0xd6abde69,0xf9179568),LL(0x8273d4cf,0x9bab3bd4), LL(0x90b3b1df,0x21b5831c),LL(0xfe3fe362,0xa86800a5),LL(0x244e7750,0x1b1f0560), + LL(0x6ec31a03,0x45e0b747),LL(0xe1072284,0xe2ebe7fd),LL(0xec603fd9,0x9f5c1060), LL(0xf78eec39,0xab1556b3),LL(0x9ad1144b,0xbc3da215),LL(0x3ac47d57,0x84785cf3), + LL(0x49ffcfa9,0x9c91a508),LL(0x609ea085,0xc6f3b880),LL(0x673b6c64,0x9883274f), LL(0x8302014d,0x3b40d18c),LL(0x599dff26,0x9c68a876),LL(0x1d6601fc,0x689e6aea), + LL(0xac0742df,0x80d1dbe3),LL(0x08f7ebbd,0xba614de7),LL(0x9d2a62b1,0x69a683fb), LL(0xc964cf44,0x2c1a9700),LL(0x6bdd35c1,0xeb51e86a),LL(0x02fff771,0xcea21c66), + LL(0x42d0fe7d,0x89b1dfdd),LL(0x85376650,0xa2a98bb3),LL(0x94d35fef,0x4f94ad00), LL(0xbd9991f1,0xe483a5b9),LL(0xce3c5861,0x8a2f4bb3),LL(0x9b125141,0x0fa548d3), + LL(0x914022da,0xec7c9526),LL(0x44ce5265,0x21974559),LL(0x11bbfb98,0xfdc0f3b4), LL(0x077f573e,0x59c23e06),LL(0x4c97db66,0xf09dfcc0),LL(0x56abb779,0x537a65cc), + LL(0x08a4b325,0x11440278),LL(0x0980ea50,0xf848026a),LL(0x676accd3,0x4c3ebe77), LL(0xb9eaeece,0x560720d4),LL(0x0940b1b7,0x78f3fbdb),LL(0x3d9d9db6,0xed41f518), + LL(0xd0762b40,0x7098160c),LL(0xd83f532c,0xb70a2692),LL(0xa861e3ae,0x4d388dfa), LL(0xb6eb9422,0x4c663b91),LL(0x867c33b3,0x16423348),LL(0xa8a6c969,0x291f41c6), + LL(0xd5028ec8,0xd9ad7bad),LL(0x02886a38,0xdaaa8d81),LL(0x2a6977bc,0xa1f8c88c), LL(0xad4366c0,0xad549474),LL(0xae44ac0b,0x36c0cffa),LL(0xeff81937,0x98fbbad5), + LL(0x23a08ab0,0xbe19d7e3),LL(0xf3339f11,0xc5d656d6),LL(0x1c1ecd0d,0xf11fa3f3), LL(0x3cadfca3,0xe59f4653),LL(0xaeb0abcb,0x6901c263),LL(0x46ba47b6,0xe409428d), + LL(0x3e3c4443,0xec64fc99),LL(0xefabc7a2,0x906e0f24),LL(0xcb0e7955,0xd3d83a34), LL(0x2801f179,0x932e43c1),LL(0xecfcc1d0,0xa02e934b),LL(0xea7f5e30,0xfe881c78), + LL(0x96e7b595,0x2be7b0b5),LL(0xc93de911,0x87aed5b7),LL(0xf98e09ee,0xe0574143), LL(0x84396d55,0x7dca05ee),LL(0xbe0f509b,0xf643767c),LL(0xc4d026c7,0x3e216685), + LL(0x4bf02e63,0xbd80154b),LL(0x9eb6f151,0x195227a0),LL(0x3ea46c97,0x07e13a17), LL(0x2363d5e9,0x9dcd5c80),LL(0x48c4488f,0x511f8482),LL(0x6cad16d3,0x7c333567), + LL(0x46373252,0x2b07ac5a),LL(0xa0decd5f,0x90999421),LL(0xb9e96cce,0x96747330), LL(0x5632ceab,0x82d5a2c1),LL(0x636dddf2,0x73828eca),LL(0xf65213f2,0xd6797c5d), + LL(0x64911f54,0x20daab28),LL(0xc600c4de,0x01e50c2a),LL(0x63487592,0x2111ded1), LL(0x8f857962,0xcd1073ae),LL(0xb7ae56ad,0xb4caf0a9),LL(0xd86137f2,0x416f957f), + LL(0x11b1bc91,0x8f563687),LL(0xaa0794bf,0x4e72baac),LL(0x91c1bc51,0x932208a9), LL(0x2ae20dc1,0x5e2bb84b),LL(0xea1e042f,0x598ceb61),LL(0x5004d259,0xd635344b), + LL(0x1a52940d,0xed0255c2),LL(0xe03eca1f,0x4150996f),LL(0x23f5b4d1,0x3aacce4e), LL(0xed0cf35f,0xcb550cd0),LL(0xa494561e,0xea36f4b7),LL(0xebef6e80,0x41cb8c5f), + LL(0x879159f8,0xb73d9294),LL(0x7ca6da8e,0xc9eb2492),LL(0x1844cdd2,0xba2e5973), LL(0x7f9e35dc,0x544261cc),LL(0xea359cba,0xd5e6aff7),LL(0x24c0084a,0x066e78f8), + LL(0xeedeaac4,0xd7604eb4),LL(0xa5c24ee9,0xaf1638b2),LL(0xfa90be8f,0xe4a2346d), LL(0x3004fb1e,0xbd43097a),LL(0xb073fbcc,0x442aafb2),LL(0xfdc31cd4,0x290829b1), +}, +/* digit=12 base_pwr=2^84 */ +{ + LL(0x514a7898,0xe8d780ad),LL(0x4494b63e,0x328ab404),LL(0x8744b64b,0xd2f0e876), LL(0x04b2c586,0xbd70e64c),LL(0xf8a4ba4f,0x5658ddee),LL(0x6ac85e37,0x3219d4c7), + LL(0xa9c333d1,0xfc159a9c),LL(0x40607a08,0x02ba3b00),LL(0xbab9d4d1,0x6912063f), LL(0xdfb380c0,0x206a4fc7),LL(0x9a973b24,0xdaf80877),LL(0xb68651e7,0x95fd7ff4), + LL(0xe8792caa,0x188e1e12),LL(0x02bc1f2b,0x5fa2294a),LL(0x69b0d8b0,0xe3949d79), LL(0x216824ea,0x7d86c141),LL(0x8b1e5097,0x7f37c42b),LL(0x92789c7b,0x08d30e6e), + LL(0xa1ef9f43,0x07cfa020),LL(0x930de093,0xc17f6f2a),LL(0x241ddea8,0x9589ec4c), LL(0xf3c510a8,0xfd97e3b2),LL(0x182c452d,0xbce604aa),LL(0x40027066,0xf22902d5), + LL(0x3293d199,0xc5644219),LL(0xf219caa6,0x393795af),LL(0xd5c761c5,0x7db08017), LL(0x3853454c,0x8d7e1f84),LL(0x29caaa33,0xe327130c),LL(0xb25301f0,0x5f09f900), + LL(0x4e8c95fc,0xf87caee2),LL(0xd0ce4d35,0x631480fd),LL(0x79b8b374,0x4449a646), LL(0xd413863e,0x1eed07c7),LL(0x013cba81,0x0f3da431),LL(0xbc56baf3,0x8cc5bf37), + LL(0x70315569,0x15bb751f),LL(0x676c5956,0xdff9fa34),LL(0x9e87acdf,0x0f196a3c), LL(0xf121946f,0xab7ac330),LL(0xab0ad7f7,0x5e9ba277),LL(0xd0c2d9c0,0x4951ba09), + LL(0x9ed1500e,0x22bf13fb),LL(0x59f5da53,0x67273921),LL(0xfe704a34,0xe840d89a), LL(0xda4b1fa3,0xb4d02cc7),LL(0x2895a3f9,0xba32df4e),LL(0xdea82804,0x6986cfc6), + LL(0x5da22871,0xe56de760),LL(0xa8a67254,0xe13cb688),LL(0xcdcd1ae1,0x3834e31a), LL(0xf18fd6d9,0x9c68e6a3),LL(0xfc21c435,0x695c7ff0),LL(0xe7b0f948,0x99254013), + LL(0xebb3db6a,0x229fd0ae),LL(0x22db0b74,0xae94f977),LL(0xae623467,0xa05744c8), LL(0x8556c819,0x3bbcfbe2),LL(0x60bdc20c,0x39d79df9),LL(0x59b1b053,0xdc97b133), + LL(0xd4d6ee50,0x9f703270),LL(0xa184f3ed,0xd444c71d),LL(0x978e07a9,0x7918b4b1), LL(0xbc8d8792,0x970363af),LL(0x02b0f92e,0x17f5dc50),LL(0xc8c384c4,0x2554094f), + LL(0x2ccd5235,0x9637b2a2),LL(0x27e2ac09,0x75dba5f0),LL(0xbae90634,0xda15c76b), LL(0xe38c9c8e,0xa8374267),LL(0x2c3af338,0x062f336b),LL(0x66c7b787,0x2fa98651), + LL(0x0837c3ac,0xc2a0d764),LL(0x62680588,0x43ba0369),LL(0x5c8c094c,0x8ec53f20), LL(0x5e289f3c,0x0ff5b7bb),LL(0xa5268547,0x8785c02f),LL(0xf5cd3301,0x1f17344d), + LL(0x6269e364,0x8b71131f),LL(0xe5772b51,0xe8bda70e),LL(0x9ec4dbc0,0x76ca9cf6), LL(0x6b091b64,0xe6f8b934),LL(0x8b563cf7,0xda109f46),LL(0x4b3bb11d,0xb109f7e0), + LL(0x0beddcf4,0xf97b7930),LL(0x9c3c946b,0x342f1b05),LL(0x0204a23f,0x4a73cecf), LL(0xe3adbd26,0x15f75cd1),LL(0x41c9110c,0xf09a81fb),LL(0xf5d8cda5,0x88dc3f03), + LL(0x8c96a49d,0x960481f1),LL(0x464ab9fc,0xc6b88ef5),LL(0x8849c44e,0x67e3f698), LL(0x5956b294,0x32068517),LL(0xd22cadb5,0x3705b95a),LL(0x836f6d0e,0x567b470d), + LL(0xcd97cefe,0x441ff140),LL(0x54db07ef,0x659b7aa0),LL(0xfe435aa3,0x8784a63a), LL(0xf04a616d,0x1d970140),LL(0x53af3b98,0xe29b21bb),LL(0x6bb3c12a,0xe6d0f2b8), + LL(0x368f5fc6,0x02132f5d),LL(0x66fdcb81,0x49d6a524),LL(0x9c2be80a,0x05aa3ab5), LL(0xc2500560,0xb7b2f0f3),LL(0x37df204b,0x6359ab2c),LL(0x6d32b4ab,0x25a10b66), + LL(0xf6591fb3,0x03360968),LL(0xc52b20c7,0x16a7fad4),LL(0xb2062758,0x4140a2e8), LL(0xd82f2254,0x905dee52),LL(0x53363565,0x29405a93),LL(0x416c034d,0x1ba1ab88), + LL(0xddd5cfd7,0x8bc8af26),LL(0xc77da82b,0x658c496b),LL(0x1827a062,0xd77384a1), LL(0xfce01201,0xee17f994),LL(0x2d3b276d,0x46605636),LL(0xff5d6088,0x02a31f1b), + LL(0x9e7536a0,0xdb49e057),LL(0x99c35008,0xbcc58d33),LL(0xb687e5e3,0x1f8bc7f4), LL(0x72027d06,0x8638a410),LL(0x2049ced9,0x0c6a070f),LL(0x69ef4623,0xd2729607), + LL(0x3355ca14,0x32aef267),LL(0x848620d7,0x8bbdfbda),LL(0xb0bcc374,0x65eccd58), LL(0x0370119b,0x15730aad),LL(0x349eba95,0xc7d99e64),LL(0x27105839,0xd30d076f), + LL(0x9f258a16,0xcccc32e2),LL(0x535d3667,0x1dacc394),LL(0x81f623e7,0x3f955c62), LL(0x38cb46b2,0xfe599a1f),LL(0x831c4abc,0x88fa6156),LL(0x1708497b,0xb9242cc8), + LL(0xffd874ce,0x9bf9b706),LL(0xa0f974f9,0xf632747c),LL(0xd17df676,0x3f1263b7), LL(0x44df1a8a,0xd7c0eb1a),LL(0x382c181a,0xaaac2337),LL(0xef3d9b5d,0x8cf66156), + LL(0x5bd8a32b,0x1d1d11dc),LL(0x8663b96a,0x57cdb2bf),LL(0xd7aca4c0,0x0983005a), LL(0x50c2f986,0xb3cffa36),LL(0x65f29ae6,0x069eb437),LL(0xec49900e,0xd896f1aa), + LL(0x455dcf4f,0x7152f4c1),LL(0x238cda30,0xa6b6e460),LL(0x2b553cad,0xb4b732b3), LL(0xe87524d0,0xa6d73263),LL(0x378032e6,0x6d48d14c),LL(0x0366d951,0xde0def0d), + LL(0x258e8fa1,0xf3647476),LL(0xd3d8e3c9,0x11f581d6),LL(0x3da389c4,0xf95d4219), LL(0xdd4c30f4,0xfe37e7d7),LL(0xcf2e5808,0x9a47035d),LL(0xbcccbd4e,0xe167952c), + LL(0xf19a3807,0x26816e58),LL(0x34036c34,0xbcdfe39a),LL(0x559a4898,0xa9d96a3c), LL(0x5922653b,0x2be83f20),LL(0xdb63d585,0x31e57750),LL(0x90d8cdb6,0xe25a5789), + LL(0x07016f2e,0x7598794f),LL(0x5b52794b,0x8631b406),LL(0x0473326c,0x4a7a3e13), LL(0x38098ff0,0xfa8445dc),LL(0x224ae6d5,0xe0829d6c),LL(0xa34dbb8b,0xb4ecb6ec), + LL(0x41a5e88e,0x505e3c0d),LL(0x3428e632,0xedcfa76d),LL(0x7378d468,0x1a6892e9), LL(0x3387bfd7,0xcf8f2e12),LL(0xcfedce94,0x75c21720),LL(0x64252690,0x92bd9a2d), + LL(0x7db48f12,0xb3dc859a),LL(0xa7e11cf8,0xf26b3a7f),LL(0x4b8bcf8c,0x38994648), LL(0xf70596d4,0xbe5e497b),LL(0x475a1d46,0xb0e98337),LL(0xb6229496,0x15d374ea), + LL(0xaffa1419,0x10c53832),LL(0xca139934,0x41d4f8c7),LL(0x2df85ac8,0x39e8b454), LL(0x835eedf8,0x8c98e7d9),LL(0xa2d40a9b,0xd931771c),LL(0x7f3156e2,0xf172ea3d), + LL(0x85791984,0x551aa33e),LL(0x5a936719,0xe7900736),LL(0x706f62f6,0x6cf471c1), LL(0x35562192,0xee2f4edb),LL(0x23457cb9,0xa3b54c39),LL(0xbfa40e0a,0x893cc476), + LL(0x84db691f,0x9fdf4d5a),LL(0x1b4ae453,0x9c0411b9),LL(0xa116f967,0x49d4f2d8), LL(0x0b67725e,0x76a1f188),LL(0xc88bd7ed,0xe7d9ffcd),LL(0x68698380,0xc0d8b68d), + LL(0x689cd83d,0xa6a6cba8),LL(0x9cff9c33,0x2f3e63e2),LL(0x86212397,0xf4698015), LL(0x84afdbb8,0xbdda0fc7),LL(0x582808e9,0x03350c9b),LL(0xb37b0173,0x2682ff4a), + LL(0xbc9fa923,0x8bd44537),LL(0x62a20c38,0x68ce391a),LL(0xa248616b,0x22585962), LL(0xe3f811fb,0x5dd9a480),LL(0x8909e98b,0xea0859e8),LL(0xa790c444,0x5f7e1b54), + LL(0x62155fc6,0xb10a93d1),LL(0x0700ee23,0x32320e65),LL(0xc17d480b,0x9eb37705), LL(0xae0b68e3,0x21f11ede),LL(0xda53ea55,0x999286c3),LL(0xa359fe14,0x97dee225), + LL(0x1f39aaa7,0xa59acbcd),LL(0xaaf5a212,0x5386d1a7),LL(0xcbc7628e,0xbda27b0b), LL(0x8a7e5aa7,0x6bd51dca),LL(0x59bbf399,0xb7dbe2e2),LL(0x81cc4d35,0xc15c94bf), + LL(0xaeb9c83c,0x600a3d7b),LL(0x8659b786,0x2b13cbc2),LL(0x5cc8f7f6,0xeac2aed7), LL(0x6bf18dea,0xfc7f4bb5),LL(0x11ea3f69,0xc2d16cb7),LL(0x341cbeea,0x69784d06), + LL(0xbe212759,0x5b823c92),LL(0x0f3cd127,0xa65cc417),LL(0xe67e7aeb,0xd7ba38a5), LL(0xe796850c,0x4e572182),LL(0xf9fc752f,0x36463b09),LL(0x1b698241,0x26827970), + LL(0x7c17865d,0x4cb9ca41),LL(0x7e6cfd97,0x214fe7d9),LL(0x4f587903,0x61d96b63), LL(0xec411bc5,0xb0101b58),LL(0x77f687cf,0x36692f2c),LL(0xcc9bfb45,0xe54d3eac), + LL(0x99723bab,0x121bfab7),LL(0x0884ebfb,0x893ab325),LL(0x6502d3f7,0x0e6788ad), LL(0xb7a108f2,0xb0513c5b),LL(0x4b2eef9c,0xa4f73e16),LL(0x6a483c20,0x342415c6), + LL(0xe752defe,0x39a27b3f),LL(0x9126ae15,0x0d86022f),LL(0x68883326,0xd8ba4a8e), LL(0x3b100c9a,0x715f52c8),LL(0x4addc384,0x336e496a),LL(0x8a6f3e95,0xec21f02c), + LL(0x7769257d,0x2894dfef),LL(0x5ac4bb61,0x2fd58cea),LL(0x708fac5f,0x3cf6d8b6), LL(0xe836e867,0xa76d99fc),LL(0x42a369eb,0x0d863adf),LL(0x15d821ff,0x82c333e2), + LL(0x4042c21a,0x9143b876),LL(0xc737f640,0x0c242403),LL(0xc719ca5a,0x7c397a93), LL(0x022b09e8,0xcf7897df),LL(0x12178c00,0x5c92e3dc),LL(0x4bd00abc,0x4fb2687d), + LL(0x911220ce,0x5750f3ec),LL(0xfb86ec9f,0x58bb4fbf),LL(0x94d00517,0xe33fcc7d), LL(0x5d9eefa0,0xa09ae695),LL(0x9756d5a4,0x3a331936),LL(0xf5c16f2c,0xc6da863b), + LL(0x07532fc8,0x05c5e926),LL(0x9d0d96bb,0xfe0c58e9),LL(0xcff62c16,0x98056d33), LL(0x346a4ec2,0x77cbe60b),LL(0xf43fd4c2,0x40989b21),LL(0xfd0645ec,0x1a3c47ee), + LL(0x5f51dc39,0x91089174),LL(0xb5a23122,0x7cbbe6a0),LL(0x8c2818e7,0x83b5abb0), LL(0xb9e654d4,0x24081e1a),LL(0x5ba06cb3,0x63528af7),LL(0x69d7dcc6,0xf570710c), + LL(0xb34b5890,0x583afeef),LL(0xc813cc91,0x6095edb9),LL(0xd3c329f1,0x166a5e48), LL(0x33293a13,0xee878e89),LL(0xeb1ad806,0xe6c0325e),LL(0x210c80b5,0x8add3663), + LL(0x3811f849,0x6b9517b7),LL(0x9d8bad91,0xd0aa5e75),LL(0x3160bda4,0xf70a8172), LL(0x916ce0d5,0x24b52ea7),LL(0xac05f7b8,0x13da7006),LL(0xbd773df2,0x1213f1bf), + LL(0x72d96f4d,0x22b2b776),LL(0x98ecfb1e,0xb0c964b2),LL(0xba7dd824,0xed9d0828), LL(0x869c0f63,0x6faddff3),LL(0xf8b13355,0x2b409d2c),LL(0x31237c9a,0x53431320), + LL(0x72a9170c,0xde41027d),LL(0x3a82f137,0x40f7b012),LL(0x099b7d4a,0x343a36ac), LL(0x2606178e,0xbe82d933),LL(0x19c93f1e,0x741f76ea),LL(0x27fd9465,0xd13d492f), + LL(0xf2518396,0x9b09dc67),LL(0x14669921,0xa79dc92b),LL(0xbea82992,0x0eb149ce), LL(0x13c60cbc,0x62ff22de),LL(0x6141a946,0x610724f4),LL(0x0793021b,0xf57b976b), + LL(0xc699a47f,0x0e516c18),LL(0x5d6f8d8f,0xd719adfb),LL(0xbdb43f6e,0xf067a17e), LL(0x0b490085,0xce5eb760),LL(0xcfebbb1d,0x15113e6a),LL(0xee6d2951,0x2194fd0a), + LL(0xd1abb57b,0x0bddf14e),LL(0x3416ad68,0xe0e768c2),LL(0xc4534e1b,0x0326b07d), LL(0x3da9b4be,0x796b6192),LL(0x474dbec9,0x85976f8b),LL(0x64001998,0xe1b892ef), + LL(0x3ddc43d1,0xb387d9c2),LL(0xdcc58653,0xd879f295),LL(0x316adb79,0x680e4335), LL(0x771da848,0x6ebecfdd),LL(0x98e7ae92,0x21e44da1),LL(0x2a3e21ba,0xf7db9238), + LL(0x6d480c11,0x7be805ba),LL(0x5ed87195,0x657824d5),LL(0x152a427d,0xad9bb855), LL(0xd479e14d,0x54fa5b2b),LL(0x967d66b3,0x393ff9fd),LL(0x1f8cde27,0x2e20e269), + LL(0xced8a78d,0xbc607c2a),LL(0xa8eed88d,0x191f87df),LL(0x5861efa0,0xae1d6b52), LL(0xcafa53fe,0x57c60dc3),LL(0x67c35922,0xb27b8932),LL(0x9b29125d,0x0fa6246f), + LL(0xb808e2df,0x50a069c2),LL(0x9a88a445,0x84e557c0),LL(0x38d713c5,0x27e6460d), LL(0x3c86b058,0x7b410b71),LL(0xb1b113ec,0x93fa53b8),LL(0x7441167f,0xc634cba5), + LL(0x282c12ce,0x1e671bc2),LL(0xa604f3aa,0x39b311ce),LL(0x78a148f0,0x98bd47d2), LL(0x371a0da8,0x5e27388b),LL(0x0fd9e315,0xfa513fbd),LL(0x34376073,0x611e38e3), + LL(0xb58100d4,0xd5ebd78f),LL(0x6e9f40fc,0x274fae3f),LL(0x8942e656,0xc1a74338), LL(0x46604337,0x6c53a23e),LL(0x660df2a9,0x2ed93bd4),LL(0xbf37f8e3,0xdfa53649), + LL(0xa0632b8b,0x02511435),LL(0x3a69fc4e,0xdf7abcb9),LL(0xd9e52727,0x6717e277), LL(0x929a903a,0x990834f2),LL(0x80c4abbb,0x8b89a584),LL(0x94e5ac4c,0xd57fcb53), + LL(0x6acebef8,0x7293af18),LL(0xdc7aa8d3,0x12646628),LL(0x409608e9,0x89165145), LL(0x01ea6183,0x9086b84f),LL(0x78ce39d6,0xc508268d),LL(0x6ff226ec,0x1d30f560), + LL(0xdd6833a4,0x49bf690d),LL(0xa575c955,0xf290aff6),LL(0x7bfea4f5,0x0e0b673b), LL(0x8e59e02b,0x3fa210fa),LL(0xbbfb09d6,0xc1cecbf6),LL(0x670a5aa4,0xa76eef60), +}, +/* digit=13 base_pwr=2^91 */ +{ + LL(0x01c06ca9,0xa1498bce),LL(0x7a08bab0,0x21c01611),LL(0xca17bd74,0x2c85a0b7), LL(0x37d0f77a,0x7b903162),LL(0x6e81e65b,0xd14898cd),LL(0x218cd115,0xe0ed1f2f), + LL(0x1731d3e6,0x78c6c92d),LL(0x7d3b6343,0xee373594),LL(0x614f0ed3,0xc8e90901), LL(0x680df68d,0x9dd54b4b),LL(0x983f94c5,0xd0c3cba6),LL(0xe9dd9140,0x80983823), + LL(0x9da2d6f5,0x3de0421a),LL(0xe135d7df,0x933d779e),LL(0x45fbc83f,0xdcc10777), LL(0x552833c4,0x437d64cc),LL(0x581bf745,0x616eb481),LL(0xaf9d7fdf,0xc3d7614b), + LL(0x4b010379,0x72673639),LL(0xb40b8ccd,0x784d95cd),LL(0x32191a11,0x3647d089), LL(0x83ad425e,0x1383335b),LL(0x5ed3ff39,0x3c5b0250),LL(0xa814df0f,0xf1f78c9e), + LL(0xd81332fb,0x48ee4921),LL(0x75c4a231,0x62f08af2),LL(0x52b5815b,0x81cee592), LL(0x47ff2530,0x5ee938f3),LL(0xbb73d595,0x9a58ec1e),LL(0x29b94b64,0x473405f4), + LL(0x0f89bcf3,0x423677b2),LL(0x3746d402,0xaafec85e),LL(0x8cec15af,0xb61ce3f8), LL(0x5ba02dd3,0x7618d63f),LL(0xc28a6515,0x4f774238),LL(0xe74d6f15,0xf6a3b5c5), + LL(0x0a141a66,0x695b4444),LL(0x09e7251e,0x9b502847),LL(0xa2883803,0x52285abb), LL(0x5f2f7fb4,0x71858bb4),LL(0xa5943828,0xbf5eef5c),LL(0x81bbe54b,0xb29d0c74), + LL(0xaa092e25,0x83536ebb),LL(0xb4e5d213,0x4768e469),LL(0x8289c98b,0xb540b589), LL(0x080c0531,0x350a6e40),LL(0x590b38c3,0x94d12807),LL(0x40fe5f65,0x83badade), + LL(0xe86caac6,0xd77e33c7),LL(0xb8ead427,0xd1935c95),LL(0x5183cc83,0xde18e630), LL(0x261521a1,0xbdcd00f1),LL(0xc82809bb,0xbd18e1d9),LL(0xf960fc2b,0xe142711a), + LL(0x1da8931c,0xdfe9e3bf),LL(0x6e1829f7,0xeb943b45),LL(0xe6eeee44,0xa1c817bb), LL(0x4fdd6559,0x0f852c30),LL(0xa27f2549,0xd9ad9b4d),LL(0xfb111f65,0x266fb914), + LL(0x917fc712,0xa20f882d),LL(0xf7566719,0xd05d7f13),LL(0x12c11721,0xd5c196b9), LL(0xc566b104,0x3eba0c8a),LL(0xecae1039,0xe301bd90),LL(0xfb24c348,0xd64eb499), + LL(0xb4738f00,0xa0c38991),LL(0xe4dd35c2,0x63bc012b),LL(0xf72495e7,0x980e89b0), LL(0x68ad6dd8,0xb530252c),LL(0x2642e8d1,0x35d6d8ef),LL(0xc235ef4f,0x1309691e), + LL(0x7262cd15,0x64a61d20),LL(0x04da445b,0xf2916951),LL(0x4ed50bd4,0x3beb5815), LL(0xe0204220,0x35638863),LL(0x69f21c08,0x8929e2ab),LL(0x14756735,0x5f273190), + LL(0x819c11ec,0x47befaa3),LL(0x90deb435,0x3bcfa2d2),LL(0x19569f95,0x4caca5ba), LL(0xd48453ad,0xbc78fd0f),LL(0xfa2100f2,0x7e4c3646),LL(0xf96c19da,0xc75a1a82), + LL(0xed646507,0x6e45f6c3),LL(0x86558b67,0xae0601e1),LL(0x3ccec830,0x20634c8e), LL(0x7c87c192,0xc3fea47e),LL(0xb66a402d,0xf14aba2e),LL(0x5bb3f7e1,0x8dcecf35), + LL(0x9751dc6a,0x33396804),LL(0x1c107cd8,0xc56b5879),LL(0x5b41235b,0xd8a41e90), LL(0x35971626,0xa956148b),LL(0x79168709,0x185bfe69),LL(0xeba02351,0x32829b76), + LL(0x39f2bf71,0x28f979a3),LL(0x53316bad,0x212c0931),LL(0xc1f8d9f1,0xf02710c6), LL(0xbb871267,0x6e6cbf36),LL(0x9c425ef5,0x11c79382),LL(0xb94f8b01,0x6ce18819), + LL(0xbd5bf1f3,0xd9a44d46),LL(0xda2c3d1d,0x662f70a1),LL(0xb13563ca,0x43d5374b), LL(0xcbcce692,0x8f4ad04c),LL(0x0c72e84b,0x3151ee4d),LL(0x5accc2ed,0x50c80454), + LL(0x74697919,0xdce044f6),LL(0xedbef994,0xfce0b586),LL(0x8b92c529,0x0d47e307), LL(0x6ac46081,0x64904019),LL(0xad80c124,0x6f52cad2),LL(0x996d42ee,0x02f7e44d), + LL(0x6aa2e142,0x257eae2c),LL(0xba53beb1,0x747a167a),LL(0xa8f5eba2,0x3b3ac5e3), LL(0xb6fa720b,0x6c6ee79a),LL(0x3a7bd99a,0xf6dc5ff5),LL(0x1a36d3b5,0x2447970d), + LL(0x7b6c6e86,0xd717e889),LL(0xc64d98d4,0xb808d76c),LL(0x7f8c0bca,0x9a06f942), LL(0xb253f5b4,0x58b8b04b),LL(0x0c103d51,0x82d27325),LL(0x1810ceba,0x7dd6d9a6), + LL(0xa27bfff3,0x0a1f1bb3),LL(0x3f63695b,0x3ba841c4),LL(0x4dba6a84,0x0e4c6ef4), LL(0xc6d82a09,0xe283f321),LL(0x78f4a01f,0xd8bcb661),LL(0x504f48b4,0xf527ac07), + LL(0x750dfc07,0xade1c74f),LL(0xa0611072,0xb8acc3bd),LL(0xad13e407,0x3886a58c), LL(0x0011bec7,0xdf6dd4c5),LL(0x350b7b68,0x01d96ff3),LL(0x76a879ea,0x2be73b43), + LL(0xca38d7cf,0xc1eca6c4),LL(0xe4217860,0xed1d926b),LL(0x0c476828,0xffc2433d), LL(0xf0c18d59,0xee65a719),LL(0x0ca1dca8,0x6a00e59f),LL(0x10811b54,0xc137d01e), + LL(0x66442ac3,0x74f38545),LL(0x1ecdd1f4,0x4947a37b),LL(0x5407e0c6,0xdaf73a23), LL(0x14140508,0x1b6175f0),LL(0x7b23a861,0x628c409f),LL(0x60c7c2f4,0x2550fdb1), + LL(0xd23a25b8,0x0e026ec7),LL(0x2b9ed7bc,0x435dbf9e),LL(0x027cd584,0x16c87162), LL(0xd7f56db4,0xa3ab5d25),LL(0x7f477034,0x8c9fa8ef),LL(0xac46696a,0x2eb3909a), + LL(0x7bc6b191,0x6f288477),LL(0x72cf52e6,0x7857879c),LL(0x4438a334,0x9ccb9b67), LL(0x0110d13e,0x016f2b86),LL(0xa623a4ce,0x7355fff7),LL(0x88c1b0b9,0x1dd555dd), + LL(0x543e3d58,0xecc48b76),LL(0x63049aab,0x251ea3ca),LL(0x1753f7aa,0xd091864f), LL(0xd65af237,0x0fd2d1af),LL(0x0ac895b0,0x70403da1),LL(0xa89e9386,0x6c3bb93a), + LL(0x48e9bf38,0x4d1df04d),LL(0x36a1b91c,0x4d05792d),LL(0x6405251a,0xd07633cc), LL(0x3ef7d87d,0x0077621d),LL(0x0c74697c,0xb35da825),LL(0x2698f465,0x521ee502), + LL(0xf4454069,0x2f0af2e4),LL(0x0799d8f7,0x1c6595ad),LL(0x57d14169,0x8cc1b16b), LL(0x660a073c,0x6ef9a078),LL(0xcad9cb79,0x4d04db63),LL(0x2d73e47f,0xbd1843d6), + LL(0x9838b1cc,0xc34b6a45),LL(0xe4d57b01,0x37705ff3),LL(0x8dde6264,0x7b15a361), LL(0x7bf8c818,0x22c4dd55),LL(0xfccc811a,0x404bd481),LL(0x601ccc8a,0x6f6b37a1), + LL(0xd8ecaa77,0x033ee5b1),LL(0x59bdb383,0x4a6fbaa6),LL(0x991bd043,0x98d656cd), LL(0xc754b213,0x489d8614),LL(0xf6bcf12a,0xc57d54b4),LL(0x1023de85,0xd8b7ed83), + LL(0xdc84066f,0xcc88e0d0),LL(0x112d5d04,0x563deb7a),LL(0xe6715a55,0xcf71acb5), LL(0x29268b7f,0xf49a247f),LL(0x5b36fd67,0x8e53cf14),LL(0x06da22fa,0xa4093a91), + LL(0x2dfb4f20,0xfbe6fb96),LL(0xd6946271,0xadb8875c),LL(0xc95a344b,0x50528b99), LL(0xd2627469,0xd7442d08),LL(0x47e943cf,0xb8b049e1),LL(0x65cd148f,0xa47b0b3a), + LL(0x50579b69,0x1cd28588),LL(0xbb9f9f10,0x38acf2f6),LL(0x0b35f238,0x67eaec1e), LL(0x00a3119a,0x72fc9c42),LL(0x39f88c39,0xa1cfc76e),LL(0x14c7fda2,0x67b702d8), + LL(0xeeafce5a,0xe782ecf6),LL(0x5eccfbde,0x16b562b4),LL(0x2a70e93b,0x23a8eaf5), LL(0x64855ed4,0x0fb734f6),LL(0x2bd5f2b8,0xde37f9b5),LL(0x959a5858,0x3d01215d), + LL(0x667d4abf,0x5a4a7d0c),LL(0x68d13116,0xf4f05c66),LL(0xdb565813,0x09cc67cd), LL(0xb34d0f55,0x53d0e496),LL(0x5a179043,0xa4f11437),LL(0x39b26f7e,0xf4ed66b3), + LL(0xbf723512,0xc670ce6a),LL(0xb17e4ba3,0x5bbf65a7),LL(0x16e38386,0xb5ddc6ff), LL(0xb0dd0439,0x9c28d121),LL(0x66dfee98,0x06d89034),LL(0xf7d33983,0x547138ac), + LL(0x92a04c6b,0x8503bf04),LL(0x008a5092,0x8761e5ef),LL(0x13902f4b,0xfa9bfa8d), LL(0xd7c9b5cb,0xbc37af37),LL(0x9f7a27b8,0xcfb05e37),LL(0x1d5b124f,0x81418e4d), + LL(0x03353c76,0x0d57da43),LL(0xdd7feecd,0x3f36caaf),LL(0xdb56957c,0xdcaadf69), LL(0xb652acff,0x2c36792e),LL(0x4213d519,0xc5d03a97),LL(0x98734fe1,0x11c6aa92), + LL(0xf714efdd,0xeca42e87),LL(0x9655a0c3,0x9d65e8e3),LL(0xe741a619,0x5be55b09), LL(0x1066c180,0x53b47016),LL(0x805e1ebf,0xe9ddc1b3),LL(0x79a748ef,0xc1057ad9), + LL(0x66d0174b,0xb2d62622),LL(0x37c05be2,0x10a0512c),LL(0x63e47f0e,0xc2eeab52), LL(0x6f08ceef,0xce6e28b5),LL(0x2f1a5a13,0x7716afb7),LL(0xc55ab74e,0x83942c61), + LL(0x6539037b,0xcef92b51),LL(0xdc615b4b,0x04a4f1e8),LL(0x49e546ee,0x0d56014a), LL(0x94d44546,0xbc1437a1),LL(0x147375c2,0xfb1e3aca),LL(0xdfc71c1d,0xb0f2abf0), + LL(0x44424b7c,0xb5679c9d),LL(0x36c89cc1,0xde54d53d),LL(0xd9a702ed,0x23252ab5), LL(0xf775d90f,0xdccb09e3),LL(0x15cac902,0x28e652e9),LL(0x08ca0b18,0x193bd66a), + LL(0x7abafda3,0xb81c2e5e),LL(0xbe90ac49,0x34a1192a),LL(0x9d1512b5,0x5d6945d3), LL(0x9cb6c182,0x0184a059),LL(0x868f5d65,0x329ea8fd),LL(0x66410f60,0x03783cc2), + LL(0x1f3d38ad,0x8524ab85),LL(0x3ad4fe62,0x58a7ccec),LL(0x37ece281,0x6a00a442), LL(0x16a1a2ac,0xfa3d6fc3),LL(0xb727b625,0xf6fc8af9),LL(0x4c3b3361,0x42699573), + LL(0x22414f66,0x2f55c436),LL(0x512968a1,0x8869eec6),LL(0xd98ecda1,0x925a1ae7), LL(0x9fb914fe,0xf112e9e3),LL(0x60e41857,0x16496689),LL(0xc8f300d8,0x1f93ed91), + LL(0xc8208043,0xb3e2c2fc),LL(0x7f1acebb,0x19f61c61),LL(0xc1e96d31,0x1ff2f9dd), LL(0xf44de6f3,0xd956fdfb),LL(0x0685ff3a,0x101656c8),LL(0xb98bc38c,0xe725cebe), + LL(0x354b318c,0x77e2edcd),LL(0xe1580079,0x9ac1cfbd),LL(0xedff64a0,0xc3c197e8), LL(0xaa113966,0xa2da7af6),LL(0x0aa54ff6,0x8f9df560),LL(0x88555266,0x5aaaa100), + LL(0x4885a4fa,0x01e3a2fe),LL(0xc469f05d,0x4aea2d91),LL(0xb3ea443c,0xf5947f39), LL(0x34d7b0fc,0x6f22b634),LL(0x93042129,0xc843cc3e),LL(0x2f14f438,0xf884bf04), + LL(0xa710c7fe,0xa0a19c29),LL(0x3057ad61,0x7fabfd39),LL(0xfe6ff084,0x94e9125d), LL(0x100a8cf8,0xd4fdf49a),LL(0x55eac3de,0x1839418e),LL(0x1aa6a2ce,0x59253c55), + LL(0xee768081,0xe01ff1f4),LL(0x144d7219,0x083342a6),LL(0x51fe8705,0x34cfa650), LL(0x75c5938a,0x499516d1),LL(0xf2c8f347,0x6350e5fe),LL(0x73f4eb28,0xe030f2a7), + LL(0x035b04cd,0xcd5808db),LL(0xc6f54f17,0x93d6ac3f),LL(0xbae00379,0x56d72b91), LL(0xc7448e15,0x211f276f),LL(0xaecaad32,0x979d0736),LL(0xe64e8a11,0xe581cfa6), + LL(0x2904cb1e,0x94ac596c),LL(0x626952e7,0x53e1e42d),LL(0x775ae6d8,0xcc989b89), LL(0x44343d68,0x0fb5c605),LL(0x0f175322,0x87608356),LL(0x6db60843,0x182f1dd6), + LL(0xe8867c7c,0x0ad3c4a8),LL(0x103cb06f,0x5479684c),LL(0x14756b01,0xe2961491), LL(0x9dfc5ad6,0x5207d54f),LL(0x3c865a45,0x184260a4),LL(0xbea2f905,0xd1158cb4), + LL(0xc78b0c7f,0x9de51643),LL(0x31357d76,0x1a3b3589),LL(0x60f16852,0xc2175848), LL(0x4d90d2d7,0x5eb827a7),LL(0x79abe6d4,0xe83be898),LL(0xb0974ddc,0x7c03935b), + LL(0xd54dae93,0x7ae1a72b),LL(0x0144c969,0xd4c8e333),LL(0xac7744c4,0xb3b92e3c), LL(0x751865f4,0x81a6f8e6),LL(0xb9b51064,0x11e0f64a),LL(0x8cfce4fc,0xedbcfd08), + LL(0xfe811795,0xe33545de),LL(0x33edeb4a,0x663a0fc4),LL(0x23e2eaeb,0x0fb9d61f), LL(0x99b52602,0xf136b9ba),LL(0x766a8136,0x6a15e5ae),LL(0x05333f8f,0x5a1a79b6), + LL(0x5a85d6be,0xda1cb3bb),LL(0x835d8007,0x28e30bff),LL(0xef4cb0ee,0x6a8c3d85), LL(0xd6781054,0xe8517460),LL(0x48b0c592,0xa0234065),LL(0x4d7b7aa5,0x5cf1f874), + LL(0x9e511ada,0x007bf10f),LL(0x8687e7c5,0xc5edcfb1),LL(0x047a613c,0x507ca4c5), LL(0x8f117e83,0x3c8707d9),LL(0xe75c9755,0xda6e38e5),LL(0x6dbd6711,0xb3ffd623), + LL(0xb9e2ce9a,0x8530669d),LL(0x020d181a,0xffc85696),LL(0x02c1a8c3,0xfa2f2979), LL(0xee4a1a98,0x989146d8),LL(0x76ac6289,0xc3f98bc9),LL(0x76d28437,0x1d491a8a), + LL(0x3fb6cb28,0xf9d04b0c),LL(0xbc28bce9,0xf246e5c6),LL(0x48fd1173,0xd3d455ed), LL(0xe3e9d40a,0x00ea713a),LL(0xf5af8e29,0x4d4c9262),LL(0x9e95cf17,0xd5e49303), + LL(0x685350ef,0x9f5fe067),LL(0x788ffe11,0xe22b2f70),LL(0xe6ff086f,0x74d4e6e8), LL(0xa937a949,0xef631c8e),LL(0xd122a378,0x707bb8ff),LL(0xff54817b,0xc1ee7dda), + LL(0xfebc7fc2,0x328f6f5a),LL(0xd933fdc0,0x8333e284),LL(0x51ee741b,0x9c135c87), LL(0x90923a1a,0x856760be),LL(0x44a46bc0,0xbe7905c7),LL(0x77415a22,0xbdc7248e), +}, +/* digit=14 base_pwr=2^98 */ +{ + LL(0xd646c05b,0xd1893bfc),LL(0xa963fee4,0x4e61d76b),LL(0xbdcc95da,0x71adc08a), LL(0xdf5decf2,0x42f481e9),LL(0x0872ca53,0xbdfbba8b),LL(0x21ece5ce,0x0eabd6ac), + LL(0x45d4fc03,0xb2367fe3),LL(0x75cbdc62,0xb115bdad),LL(0xef8b952a,0x07522180), LL(0xfd23c45d,0xc0448864),LL(0x87eb202e,0x8c068193),LL(0x313c2e61,0x7fe87505), + LL(0xf2ede850,0xbfa39552),LL(0x93cab795,0xa40a845d),LL(0x09e97bd9,0x56ca4f4e), LL(0xccbdb84b,0x207a1e71),LL(0x1893a956,0xd0ce1e11),LL(0x84c34b10,0x7a22a433), + LL(0xe404586f,0x48becb6a),LL(0xf5d6372a,0xa136232d),LL(0x87ba83a5,0x438af2bd), LL(0xf4e459ff,0x37075ede),LL(0x582d1ce8,0xaa42aae7),LL(0x0daeb82b,0x973dba33), + LL(0xc40861a3,0x48bc98ae),LL(0xefd9a196,0xc100f037),LL(0x052eb352,0xda4029f6), LL(0xef34169d,0xd61b4933),LL(0xa357cc0b,0xcedde9cc),LL(0x4d006fcc,0x8983c01e), + LL(0x92f2a1c1,0x31ad5f19),LL(0x924bcb08,0x7c7a6319),LL(0xd0c36056,0xcbee249b), LL(0x5ac4f6eb,0x63135518),LL(0x37f8cd7c,0x8b2aaaed),LL(0x0b9bcd79,0xe71c855a), + LL(0x5828ecb9,0x1f12da90),LL(0xbed745a9,0xbd094b68),LL(0x59a321b0,0x27d4a84e), LL(0x71683549,0xd05c2695),LL(0xc9c9f2ae,0x56e4dcf2),LL(0x199abda7,0xdfff6393), + LL(0x576fff20,0xc28ae07c),LL(0xb9024805,0x88ed10b6),LL(0x851ed43d,0x81944c6a), LL(0x158549cd,0x31192f87),LL(0xff08eac3,0x36dce6e4),LL(0x9ffacd59,0x7837d3e2), + LL(0xc8329917,0xe820d2e4),LL(0xed56083c,0xc343156c),LL(0x67c118a9,0x60116d83), LL(0x699583a8,0xbf2c1598),LL(0xc03c3370,0x77da3db5),LL(0x88522e50,0x35b0e741), + LL(0x3e4cf4e3,0xcb945a8b),LL(0x45fbb1c3,0x1ac37aba),LL(0x7e016a40,0x05b3d362), LL(0x93c0799e,0x175cb9f3),LL(0x9102d347,0xad1068c4),LL(0x8c6b55d4,0x01200565), + LL(0x1fcfd55e,0x6396a275),LL(0xc76eb5a2,0x26c5c201),LL(0xc49cc617,0x9eafc039), LL(0x62fdc2b8,0xecf4f6fd),LL(0x73f3babc,0x795af8b3),LL(0xb456ddc8,0x9d6380f0), + LL(0x00481dbf,0x8435b4c8),LL(0x586fec04,0xd68ed151),LL(0xbad5a7ec,0x904b3a7d), LL(0x755a599b,0x5ac0c68f),LL(0xeed07240,0x8202bd5e),LL(0xc2111eb1,0x50108f7a), + LL(0x64114206,0x7966f061),LL(0x08252097,0xc7e5a500),LL(0x4a27d84f,0x1efdcfb5), LL(0x992fb0b4,0x05327d63),LL(0x109f651f,0x33c1800c),LL(0x4858bb75,0xe90ccb5c), + LL(0xdfd79c9e,0x3f05f6b9),LL(0x94808d4d,0xb0db9df5),LL(0xda56ea30,0x170d0bfa), LL(0x46ea52b5,0xfc08aa5f),LL(0xd7247824,0xf5eb81e2),LL(0x26357344,0xab23fe73), + LL(0xb37efb0c,0x8d9e1ced),LL(0xd7b33644,0x3b126789),LL(0xa25195f0,0xee7f591f), LL(0xe6380d90,0x5572f635),LL(0x6381422f,0x570f7d77),LL(0xe69759c5,0x44378d7d), + LL(0xa08babdc,0x1b30b2aa),LL(0x3cc52896,0x9ad9a870),LL(0xd7cb91be,0xd1afe3d1), LL(0x4a5535b4,0x1f838c05),LL(0xcfab1e86,0x86b14aaa),LL(0x6f77d53e,0xe64af8a4), + LL(0xebdc38c4,0x2028f1f2),LL(0xf31ac53e,0x534fc79a),LL(0x1a0e531f,0x987d5d41), LL(0x76fd27ac,0xf9d7d8c3),LL(0x40a77dbd,0x930da5d5),LL(0x57ef7f4f,0x2c9fae32), + LL(0xdb6f852a,0x7232dd8f),LL(0xddea21cd,0xcacd16a9),LL(0x21f8ec9e,0xf04c4671), LL(0x4688eec7,0xfec90470),LL(0xba277998,0xd49d9514),LL(0xb4cf1ab4,0xbec829aa), + LL(0xe594415f,0x6fb54104),LL(0x2c3a2383,0xcd39d8af),LL(0xe44a8388,0xd50673e0), LL(0xf14f72d0,0x55137ed9),LL(0x1abb3c7b,0x4475a031),LL(0x246ece39,0x73e9fa03), + LL(0xee6222b2,0x7017567a),LL(0x03448302,0x603051f2),LL(0x5aaa55f0,0x8e7a299a), LL(0xc61d2cb5,0x6a43ef05),LL(0xd1923efe,0xc780996b),LL(0x29e3847b,0x71c3dd7d), + LL(0xfdc2519e,0x34bff2b3),LL(0x4e5b30b6,0x55e13340),LL(0xdf3b31e6,0xf0159d7d), LL(0xffa85b11,0xe041606a),LL(0x437094a8,0xc9438445),LL(0x8b1d800e,0x13df929b), + LL(0xdf21b623,0x0e2925cb),LL(0x9010bc70,0x24051cf5),LL(0x891c2a68,0xc2176b95), LL(0x01e795a1,0x991bf219),LL(0x30ef7d3d,0xc069d0c2),LL(0x1bc05ca1,0x6021da66), + LL(0x1e52bf74,0xfcf4cf04),LL(0xd74c9731,0xd7cb6e14),LL(0x7e857c02,0x7628ac80), LL(0x126c5b91,0x0cc391d5),LL(0x7e2d15bd,0x731062b4),LL(0xabdfd049,0x1e87cea9), + LL(0x363f0203,0x10006c7d),LL(0xf63971b4,0x51ea9151),LL(0x80734bc5,0xd951b96d), LL(0x89a7c10f,0x1d164750),LL(0xcae9c4ae,0x488652d2),LL(0x185669d2,0xb4ca6a36), + LL(0xbb8f19e9,0x6e84424a),LL(0xc9d19687,0x8b9aa75e),LL(0x0b88d08b,0xc120a35c), LL(0x2a8cf8ec,0xdeeb61da),LL(0x6bebce59,0xfcc8ccf8),LL(0x8de4133a,0x679e7f58), + LL(0xe3c5a444,0x0f3eb897),LL(0x5751461e,0xce575c28),LL(0xdc20e60f,0xe8d1e5b3), LL(0x7d07536e,0x9dc103d7),LL(0x2f685999,0x86a2a40a),LL(0x47c6f8fe,0x15ae9ff4), + LL(0xaa9b291a,0x885babc0),LL(0xfedd2942,0xe3f51080),LL(0xb5d39214,0x304ccff4), LL(0x0a3b3582,0xf107db2f),LL(0xea86a325,0xe2fa620c),LL(0x6b20d46f,0xa0b718ec), + LL(0x72969183,0x9d14762b),LL(0xfce848f5,0x6b2baf0b),LL(0x968f6133,0xe7660ec7), LL(0x8299819f,0xc6ee65e4),LL(0xef1b9b29,0xb57828af),LL(0xa50176ea,0x5b1c99f3), + LL(0x62f3aae5,0xc35057fa),LL(0x46f3c966,0xa5437e70),LL(0x8864f29f,0x6bc5d22a), LL(0x86323fd9,0x143c464b),LL(0xd1570c61,0x30852173),LL(0xb5b6f3d4,0xc188ba63), + LL(0xc8406bdc,0x8b26d419),LL(0x3fafc9be,0xc0c95def),LL(0x5d5eb753,0xcd58ca48), LL(0x57e8a8a8,0xb937d969),LL(0x8d184ad0,0x8c18d5f4),LL(0xf5485ef3,0xbd764f21), + LL(0xecb39885,0xc5483510),LL(0x705df629,0xf0122752),LL(0x93d33824,0xe43d6cca), LL(0x49923c5e,0xe00a9514),LL(0xd8080e76,0x6993aecc),LL(0x633b9fba,0x4dc6d79e), + LL(0xefcd68d8,0xa33bbb48),LL(0xe68f5543,0xacaebe65),LL(0xa37a2bbd,0x7383a726), LL(0x770555c6,0x5d3e8c9c),LL(0x9b90f1ad,0x86e220c9),LL(0xab26ba0b,0xfc32885e), + LL(0x89e7a1a1,0x35837dd0),LL(0x08889b69,0xb8c46a1e),LL(0x1957ef47,0xbf3d0f2f), LL(0xf0a25602,0x7b89c2b9),LL(0x0656bcd8,0xab5068da),LL(0xf4298e4b,0x0c5c3a38), + LL(0xcaf9bcee,0xd20c9cc5),LL(0xbd6c6fd4,0x783987bd),LL(0x93ba43c5,0x696e2d85), LL(0x391285c2,0x356cbc57),LL(0xb533c31e,0x536b15a5),LL(0xc4c4ba41,0xc375dc50), + LL(0x405fe9b5,0xb961c694),LL(0xd5283673,0xb3983e9c),LL(0x96e95f05,0xd455be9e), LL(0xc75e8014,0xb38412e7),LL(0x96860cdb,0x1033ac0e),LL(0x00b0a103,0x303881b4), + LL(0xb3b3e63d,0xd1fc3bf7),LL(0xda50174a,0x35ab8665),LL(0x01a402c8,0x2944d50f), LL(0x4d26ddfc,0xba75bddb),LL(0x80efbf29,0x530839a3),LL(0xb93462eb,0xbf8adca3), + LL(0x84d08e74,0xd534a5d0),LL(0x4efdf525,0x436eb254),LL(0xdca89d2b,0xc653af06), LL(0x41a37013,0x0895e017),LL(0x91294447,0xb9f0b192),LL(0xb7f407e1,0xfd2cb9b0), + LL(0xe6cc78dc,0xc5323fdc),LL(0x2a74ed72,0x9c863011),LL(0x1aa90d73,0x86d391c8), LL(0x308de318,0xae9c2f27),LL(0xcc2d2cad,0x133f4d65),LL(0x42a244ba,0x0bfd5ed6), + LL(0xa7843758,0xb5b22ff6),LL(0xd93d7ed0,0xbdd3456d),LL(0xe0432d41,0x4bb279a7), LL(0x14acb55c,0x375da883),LL(0xad01ebbf,0xf9493a0b),LL(0xf0543ccf,0x5063d083), + LL(0xac19b11f,0x5c97aefb),LL(0x820ab407,0x253042d0),LL(0x550d30a7,0xf6b2204e), LL(0xa702ecbd,0xa091b635),LL(0xb34417cf,0xbd011942),LL(0xd3f92344,0xcb969929), + LL(0xeca70aa5,0x46c47688),LL(0x383cfca9,0x057cd161),LL(0x7ea5b03d,0xe538b089), LL(0xf15b8704,0x0f3fc549),LL(0xc4be9475,0x0bdcf648),LL(0x40851b7d,0x31fb0690), + LL(0x7b90be33,0x8dcd148f),LL(0xb5e956ad,0x3720de48),LL(0x2a303f7c,0xa0ce39c4), LL(0x89d0fdb7,0x054206d6),LL(0xb76fa220,0xe9991e88),LL(0xbac4c9ee,0xa0eb60ae), + LL(0x01065ebf,0x9feaf043),LL(0x23e6a889,0x001f921d),LL(0x526d4d12,0xbe2a6e6a), LL(0x762aaed0,0xae7aa8bd),LL(0xde1e43cb,0xa8aa4f13),LL(0x626356ac,0xea279354), + LL(0xc65bd80c,0x54727769),LL(0xf135559b,0x0f18555c),LL(0xc8a92ba2,0x3ef9e205), LL(0xf4e4b870,0x32f97cc9),LL(0x9fdfc295,0x764e6c3b),LL(0xa0be0176,0x97f95ca0), + LL(0x3547707f,0xa25af6ad),LL(0xae94e108,0xc46e1845),LL(0x3a29996f,0x481c7fd4), LL(0x1aac0d60,0x4e9a38c3),LL(0x2d811b3d,0x40172f5e),LL(0x58ba5cdb,0x69750750), + LL(0xd402cb8c,0x29404a7b),LL(0xc4a0deb6,0x8698ab1c),LL(0xf10f49fa,0xb0336bae), LL(0x4d34533b,0x27cbe364),LL(0x53d57740,0xb22e812c),LL(0x90f4aa01,0x82b80434), + LL(0x479f2e14,0x1b20606f),LL(0xb817e38d,0x5fa6e4bb),LL(0xcedd7aac,0x860fdf9c), LL(0xb63a00f6,0x66fa3783),LL(0x73c45ad2,0x2719710a),LL(0x64de62a2,0xa3db3619), + LL(0xf04aa960,0xb2e191c9),LL(0xf57411c2,0xd419a1dd),LL(0xa4a57d93,0xd860a8ea), LL(0xc1f433bb,0x5001230c),LL(0x8ec92a7f,0xf28ce31b),LL(0x9a4d25a6,0x8c582d34), + LL(0x96219ee4,0xf2826086),LL(0x4b4f0c24,0x0c21e1e2),LL(0x387213d6,0xd1804821), LL(0x1cce49ca,0xfa342c74),LL(0x4341ef6f,0x880ea569),LL(0xe968aa8c,0xa6fdc07f), + LL(0x91a41ded,0x293a6180),LL(0xe5d87356,0x51412d3b),LL(0xf0b2d509,0x925d4c9e), LL(0xd251fb90,0x80c77ead),LL(0x90e53a2d,0x1d6e3cd3),LL(0xf9a2ab15,0x6e11e4d0), + LL(0x1d009ea4,0x3535dc7b),LL(0x75c6c6f0,0xb6b48635),LL(0x72e3c50c,0x12ac6480), LL(0x6464430a,0x6d71a720),LL(0xe1dfdda2,0xf553de43),LL(0xcc6c345d,0xa1ee9a4e), + LL(0x2ac5a8c8,0x46f7bd04),LL(0x063f2739,0x62c11b7e),LL(0x69a36c58,0xaa7522c7), LL(0x06a4df65,0x6a10fbe0),LL(0xdf06bc54,0xd5f96d18),LL(0x38151768,0x545f76b0), + LL(0x712eb633,0xe9dd6bff),LL(0xf6972761,0x6c1e035a),LL(0x7a4d81a8,0x060bfcaf), LL(0xd2653839,0xa262cba7),LL(0x95a4c747,0x561af30d),LL(0xe33e339a,0xb9a3b8ac), + LL(0x5f371d0d,0x2fc5342b),LL(0xefbb2027,0xf9654b68),LL(0xf7520a2a,0x14820afb), LL(0x43d0ea24,0xb24c0b99),LL(0xc37df573,0xa1a8898f),LL(0x7bed1103,0xa1c07bb8), + LL(0xc4d993a3,0x664a8ae7),LL(0xfe44732b,0x1c2e8847),LL(0xb2426ed1,0xdb1e5a22), LL(0x7a45d008,0x2bdd7cb9),LL(0xf3d91e11,0x59477a34),LL(0x4cd742a4,0xf53dcb02), + LL(0x6c709f3b,0xa28f6d2b),LL(0x9ac0b1d9,0x479b31b7),LL(0xd568d9ff,0xe8237a0f), LL(0x0580975a,0x6895a188),LL(0x7b234cf3,0x450b5fe5),LL(0x70629bf6,0x1b4e0f89), + LL(0xe5bcec01,0x455ac64e),LL(0xc6bf296c,0xaf051300),LL(0x2306584f,0x119cb4c6), LL(0x7c0dee75,0x3242a0e6),LL(0xae1fd52a,0xd09fce7c),LL(0x2d5cf9ca,0x53608dc2), + LL(0xc8506617,0xaf6edd22),LL(0x3241e286,0x5ed21b88),LL(0x81282c66,0xf27d3439), LL(0x308d6ca8,0xbd393fac),LL(0x9032cca3,0x5155e229),LL(0x56a04a1c,0x29acd1b2), + LL(0xf78b6573,0x95c60d2b),LL(0xed576414,0xe53bb875),LL(0x452f8871,0x389200b3), LL(0x5af9dd53,0x69a42e38),LL(0xbf0327d1,0x01e3b18e),LL(0xba8d03df,0x985d811f), + LL(0x559c7bda,0x0cfef9cf),LL(0x5bb4cd8c,0x5096e369),LL(0xc8011afa,0x8d16d0e2), LL(0xf25b0ff3,0xa816a8be),LL(0x2f161a22,0x7edbd05e),LL(0x8ba1f83c,0x1ae570e5), + LL(0x690bc4ad,0x7c0084c0),LL(0xad2fd216,0xe0a5451c),LL(0xb130c417,0xb4926a84), LL(0x9309e50a,0x39e4ec36),LL(0x5c1bc674,0x094fb1e0),LL(0x67878735,0x19572f99), + LL(0xd011ed00,0x8f62726f),LL(0x0793cd2b,0x430d8c6c),LL(0xb8f172e8,0x26226dc9), LL(0x11935136,0x550ff9fa),LL(0x8c9798d3,0xb415bbd9),LL(0x83b3b2c1,0xb136921d), + LL(0x669e0918,0x940bce84),LL(0x298448d1,0xca050170),LL(0xa8e4042d,0x1fa3041a), LL(0x1e2abce5,0x0f73a3aa),LL(0x448c4fcf,0xe5196a8a),LL(0x2774a5b1,0xfe25d513), + LL(0x7192ab9c,0x6799b1b3),LL(0x7f02ea02,0xfd1cd774),LL(0x01b1e71f,0x1b4e5cd1), LL(0x13c4b9f9,0xb09c08b2),LL(0x87870c13,0x6be2f294),LL(0x3c2d5610,0xb74d7045), +}, +/* digit=15 base_pwr=2^105 */ +{ + LL(0x7c1f0b12,0x950114b6),LL(0xd9406636,0x4862f5b4),LL(0xe79e2ad1,0x4b94f772), LL(0x2162024c,0x283df03b),LL(0x1eae9865,0x730f9f5b),LL(0x1260cbf5,0x70b3f304), + LL(0xe16c6f09,0x850addcf),LL(0x5592d8d7,0x36a07642),LL(0x74af4f32,0xcdbf2690), LL(0xf354ed72,0xf452488f),LL(0xa99eb244,0x8ceb8c62),LL(0x92a47252,0x471774a0), + LL(0xf938b030,0xc7285307),LL(0x51cc80fd,0x81689ad0),LL(0x483aeb1a,0xbb8e3d60), LL(0x8286229d,0x6dec5453),LL(0x0cc83f74,0x3edd27c2),LL(0x152e37a2,0x1cd6ed5f), + LL(0xd377bce5,0x537244ab),LL(0x92bbe196,0x333760b5),LL(0xd7b6c795,0x373f1920), LL(0xb99b6824,0xd28c3c25),LL(0x501ea35f,0x14e96723),LL(0x8bebdafd,0xb3c3f757), + LL(0xd82a0a63,0x4a680526),LL(0x0558bb75,0xeeb46a35),LL(0x7589e42c,0xc27b9513), LL(0x59267a86,0xaf5a87ec),LL(0x5c4b67d0,0xbbf1e057),LL(0x09598a27,0xe37e471c), + LL(0x2a71504c,0x894af9cd),LL(0xd1e8512f,0x38e0d48f),LL(0xfc2cdbf4,0x96db568c), LL(0xeac2f293,0x911f7b8d),LL(0x5c965527,0x5821420d),LL(0x078559b0,0x832e957e), + LL(0xfbf84029,0xff114368),LL(0xd21a72b0,0xd8dd9e2d),LL(0x057c15f6,0x4ba812a4), LL(0x64fa748e,0x976cf8b5),LL(0x73812963,0x1801ab26),LL(0x394d2cf3,0x4a748d37), + LL(0xb8ffb76d,0x6b23479d),LL(0x898a63b9,0x401bf60b),LL(0xdc64023e,0xee58a7ae), LL(0x6c70a3b4,0xd288cadf),LL(0x5cf80831,0xc1da3aea),LL(0x5ee941bf,0x941b411d), + LL(0x1029b199,0xc4cccd83),LL(0x6d56502f,0x683863df),LL(0x43f9e9f5,0xe362783b), LL(0xa34a9ef2,0xb224d946),LL(0x72b438e0,0xccabcf49),LL(0xc4af3f0f,0x78d92dda), + LL(0xceee3620,0x123b5621),LL(0x90e8a27d,0x513add4e),LL(0xe45a02ac,0xaf127aef), LL(0xc0d7ba62,0x21a78e17),LL(0xf98d0ce5,0xcf09e816),LL(0xae64e8c5,0xc28cab36), + LL(0xef7a2cc1,0x0912071a),LL(0x3b451d17,0xd3f2bc6c),LL(0x35ea478f,0xf3796f91), LL(0xc9c756e8,0x9b5e64d7),LL(0x2d58e1bc,0x7ed6f7fe),LL(0xeae6b6de,0x503d9af2), + LL(0x5e314295,0x57cc1308),LL(0x89230510,0x74c77119),LL(0x9968ac6f,0xa314891f), LL(0xf033b995,0x8dd4df22),LL(0xe0d71497,0x7ffe0a44),LL(0xac968d35,0x5c5f54f3), + LL(0x406b0bca,0x246a971a),LL(0xd805a357,0x90eedbcb),LL(0x5c690ec1,0xa6dbc124), LL(0x7bc65cbe,0xbe369f3d),LL(0x96065417,0xe0ff616a),LL(0x337f567d,0x3654f681), + LL(0x9bf8c833,0xffa30743),LL(0xafe911c4,0x6bd11b77),LL(0x6762a958,0x30eee614), LL(0xa9e63f35,0xfba9fbc0),LL(0x332389bf,0x70edc028),LL(0xb9798769,0xb5d35ca9), + LL(0xb8c620c1,0xcde63d0f),LL(0x06942335,0xcdeaad60),LL(0xbdef1579,0xefb74b79), LL(0xd8b128a9,0x13dd5771),LL(0x56343492,0x00e9fe16),LL(0xd6baa45a,0xf4ffba1d), + LL(0x19d575fd,0xfdb1ece3),LL(0x61ad3345,0x231c53ac),LL(0x523e013e,0x3f3df394), LL(0x71c3b97d,0x254a291f),LL(0x396eb0c1,0x645a1aea),LL(0x0282f254,0x680b79e1), + LL(0xf3ef6b21,0xcededfad),LL(0x577a8e6e,0xa57437f5),LL(0xf5f051f3,0x413575b4), LL(0x4d1ca0ed,0xd88fe7f3),LL(0xdcceffdf,0x1849d754),LL(0x141422af,0x40e7733f), + LL(0x00721b87,0xcd5a690a),LL(0x0b2801ae,0xf0e84ac9),LL(0x17c73f20,0xaf0765ff), LL(0x33595f95,0x976d23d8),LL(0x72c8f457,0xdece9d39),LL(0x6604f618,0x3dddbfa2), + LL(0x2bb19559,0xef6c219c),LL(0xd57790b8,0xfbf7ac31),LL(0xc8f6c246,0x676751ad), LL(0x4c3b7ff6,0x0f4384d0),LL(0x39ee5f11,0x040a4f5a),LL(0x698f2f7f,0x30c58138), + LL(0x0d1a6b11,0x097f5ccf),LL(0xf9b7701b,0xf6cdd1f6),LL(0x4786ad28,0xb965d0ce), LL(0x70cd6a22,0xadd506d0),LL(0xd6299b81,0x9c4667a5),LL(0x1c032341,0x64056720), + LL(0xeb125185,0xfd01d07b),LL(0x48235596,0xbec6636b),LL(0x293de94e,0x9017825d), LL(0xe499b69d,0x888e08c6),LL(0x3b6a7ced,0x4f4a6117),LL(0x58f8d42c,0xd6d2a6fc), + LL(0x79062ff2,0x66a2ad0a),LL(0x9bc01a36,0xf187ed18),LL(0xb3f3d332,0x4fce4f5a), LL(0x38533148,0x903af8c9),LL(0xd058ab8e,0x34ff5da6),LL(0x6b2b6b4d,0x8fc66e30), + LL(0xd4eb310c,0x7e0843bb),LL(0xbefbb992,0xcd48672b),LL(0x75c258ee,0x839c6b0b), LL(0xd70a705e,0x11733171),LL(0x15042807,0x92fb3004),LL(0x9ca0a17a,0xe091def9), + LL(0xaa917028,0x48c33a95),LL(0xb87bb667,0xdef22091),LL(0x106fa5c7,0xb6223052), LL(0x6934d6e3,0x12491351),LL(0xbce2aa51,0x55c0b4d5),LL(0xb70985ca,0x734ec89c), + LL(0xbc8fd35d,0x000e0cd7),LL(0xb50502f2,0x05b488af),LL(0x1252793c,0xb56958a5), LL(0x8a310520,0x4e8c9cc1),LL(0x7f1db7bd,0x0a6e45d6),LL(0x5ecdeb31,0x72d3eb5a), + LL(0x85f7d20f,0x1ac14fe9),LL(0xb430d3cc,0x0e020f68),LL(0x409668ed,0x273e515f), LL(0x78e0031b,0xbf622f1b),LL(0x9bf7fcd7,0x5aa6dd5e),LL(0xb6b977f5,0x30573193), + LL(0x275d217e,0x7c4023a7),LL(0xcca3f6f1,0xac6a6b67),LL(0xf0fcc9b9,0x176bacc3), LL(0xb0858d73,0xb280416f),LL(0x6fec46e4,0xe48e3c6d),LL(0xf3235cfd,0x9aeb3a09), + LL(0xa486a8eb,0x33c068c6),LL(0x5def6704,0xc60c21cb),LL(0xbefdd067,0xff4ac34f), LL(0xe174348b,0x4f1427e6),LL(0x9f27fd60,0xc80afd96),LL(0x35955ccc,0x1799ed29), + LL(0xddebd485,0xdbf71628),LL(0x4b8207f2,0x13897ccd),LL(0xfd8365e9,0xe12c2822), LL(0xb003566c,0x705fe026),LL(0xf12863e8,0x8a97247d),LL(0x1748fc9e,0x698a3477), + LL(0x0669c4d8,0x90917d8d),LL(0x80746165,0xc65e4aa5),LL(0x12c0f44c,0x96763d21), LL(0x97318a9b,0x00a7db8f),LL(0x5fd1a0e9,0x29079037),LL(0xa9b6ec4f,0x2f4f1c47), + LL(0x0bd63292,0xd861004e),LL(0xca40a1da,0x0176d94c),LL(0x59fb9a7d,0x5e91abe8), LL(0xf6d4f1ff,0xa881a8f5),LL(0xa24ce9a9,0xded77256),LL(0xc492f4f1,0xacf4db67), + LL(0x391ca168,0x7146928e),LL(0x11e6bfe4,0x11981f6f),LL(0x32f997ad,0xfce0f814), LL(0xe85cd202,0x7cf18ed9),LL(0x03c315e4,0xa94f2b6d),LL(0xb5f8a4d2,0xe2a93315), + LL(0x2fab5a21,0xcd259208),LL(0x928dcc30,0x3d24f816),LL(0x9d66a41a,0x779e1c36), LL(0xc2e41af4,0xa8248120),LL(0x06fc8b0e,0x243a6bbb),LL(0xed8d12b1,0xb9de9d41), + LL(0x30f64b0e,0x23ff17d9),LL(0x8f083ea7,0x1eab5984),LL(0x554ae817,0x9965f2c0), LL(0xf4d8956a,0xb0a399b6),LL(0x0ebf134d,0x315694aa),LL(0x7fde4077,0x53567f64), + LL(0x5c71762b,0xfdfeb029),LL(0x1eb1144d,0x968d0bd7),LL(0x1f08b2a6,0x171297f4), LL(0x897a2666,0x7944101d),LL(0xda77b1a1,0xf4936746),LL(0x44547953,0x0e752c75), + LL(0x492b2fa0,0x69d5b44f),LL(0x5d483119,0xda5f6651),LL(0xac522988,0x40814f2b), LL(0x4e1f60df,0x344aeee8),LL(0x5ff6f053,0x2ddba5e0),LL(0xa4008621,0x4453d9ee), + LL(0x179e2967,0x37ab4aec),LL(0xba87af44,0xe91479f7),LL(0x0b55c02f,0x4b292d1e), LL(0xb69ad1a8,0x9deeddd2),LL(0xe91e913c,0xb610d598),LL(0x912b8e9d,0xa6b91331), + LL(0x20cdff63,0xaa819620),LL(0xaa6d0ed4,0x8d70fbba),LL(0xcb318c6a,0x5c951edc), LL(0x57b1fd8f,0xa4cc6573),LL(0xe00bbceb,0xd7a23636),LL(0x8d179500,0x33bb3513), + LL(0xe0168751,0x86da6dfe),LL(0x66604b83,0x940ba0c2),LL(0x22e9e186,0x15a8aa02), LL(0x65a6f19d,0xe5b39c58),LL(0x32f441b2,0xf591dc15),LL(0x9d3a2de0,0xb85cad59), + LL(0xfb422575,0x35abc7e2),LL(0x4b8acd9c,0x517b6718),LL(0x2ee6263b,0x3e303015), LL(0x7393f470,0x51c8c440),LL(0x974d3dad,0x032cce2d),LL(0x4c104799,0x68e57ee5), + LL(0xc0514589,0xb95807db),LL(0x0402a3fb,0x09a79c01),LL(0xd3fdd054,0x92bf681e), LL(0x393adbfc,0x9f2f00f4),LL(0x1ca244f6,0x13789212),LL(0x00d3a9db,0xe2f45a02), + LL(0x207a53c9,0x29ea3e07),LL(0x163549d5,0x61213e9a),LL(0x4656d770,0xe0b025d0), LL(0xbd0d85da,0x050f7501),LL(0xe3b43784,0xe95027f4),LL(0xf1a6b6b5,0xca3907cc), + LL(0x4a7a00be,0x670062e7),LL(0xace7ec67,0x9ea4759e),LL(0xc1d92037,0xf5d29365), LL(0x9c39e2de,0x6d421c87),LL(0x1593c52d,0xb3e3d2c7),LL(0xcd74d083,0x6733eb50), + LL(0x5ad1d924,0x30029e9b),LL(0x84925a40,0xc15c63d1),LL(0x17e5f0bb,0xb2c098b9), LL(0xda6f02f1,0x2646d78b),LL(0xb8ae8095,0x1f8b74a3),LL(0xb03283d5,0x2bfdcbee), + LL(0xb7d7ef7f,0xc78a8e01),LL(0x574b90f6,0x7e0a26ab),LL(0x26861a59,0x9be1c8b4), LL(0xc787cf5d,0x5dddc986),LL(0x217b8936,0x77e06114),LL(0x94d79885,0x0b58deb0), + LL(0x6ef1f554,0xb89d079b),LL(0xf1deef2a,0x87ee6d78),LL(0x7a7b9098,0x8985bc42), LL(0x165957c3,0x0843395d),LL(0xccf6daf8,0x11293e7b),LL(0xfce740f4,0xb2527ca8), + LL(0x12fbd080,0x54f19e66),LL(0x4aed0e22,0x82f0835d),LL(0xb56935f6,0x5f73c822), LL(0x5b0f4a59,0x4b255bef),LL(0x6b23634c,0xc2542ee4),LL(0x858eb55c,0x41a4fe31), + LL(0x70d71c90,0x5c19755b),LL(0xdee1b0d6,0xb15b8f0d),LL(0xeab4b7ef,0x07e40808), LL(0x6faa90c6,0x9f4f2665),LL(0x9fcb51fb,0x9e72096a),LL(0x541dbffb,0x1660e2f9), + LL(0x9aff8d55,0xfb1cf7e0),LL(0x7a1815da,0x7276df1a),LL(0x913d43ca,0x604cd6bf), LL(0xe0227dec,0x4f016034),LL(0x0e990a9e,0xb70eb020),LL(0x83b0d458,0x9db0e5f0), + LL(0x4bda5c36,0x137d2ba6),LL(0xc9d84ed7,0xf2ad5444),LL(0xb4878d48,0x37ba4976), LL(0x0c6237c0,0xe3518c97),LL(0xb5e9c50f,0x32e71c0a),LL(0xa325250d,0xfa2c1d25), + LL(0x80be6431,0xfa8cf7f2),LL(0xe5a2035e,0x74e8f698),LL(0x03eff799,0xa032a4e4), LL(0xc53ac08f,0xbdfd91ab),LL(0xebf217bc,0xe17b20eb),LL(0xdf84edd2,0x40de3e73), + LL(0xdf81c5d3,0x4110f517),LL(0x53f20019,0xbefb623d),LL(0xf346f4d3,0x67061890), LL(0x26ff57dc,0x096120d3),LL(0xccb068a2,0x00b49cf6),LL(0xf00d1e3e,0x613293d4), + LL(0x467e8604,0x248b65d5),LL(0x3ab4a1b3,0xcd983257),LL(0x1e96833c,0x1dc552b2), LL(0x89592574,0x851b4255),LL(0x1da99f23,0x62d85452),LL(0x401ba0d8,0x4cdbe71f), + LL(0x9760901c,0xdd6660af),LL(0xb8ad97c4,0x503a92d0),LL(0x6fbd51d5,0xdd313e4e), LL(0x6e919a43,0x441e5529),LL(0x65e9a71d,0x8613c8d8),LL(0x3d2336b4,0xc0c8b396), + LL(0x7b97021e,0x811cc3a9),LL(0xf50aa806,0x6d1a1d14),LL(0xf7f8f8f8,0x85a4588e), LL(0xe67b610a,0x23ba0ee5),LL(0xe3f78d9b,0x77df65eb),LL(0x99a02de1,0x0295f1aa), + LL(0xae21de63,0xd3284a43),LL(0x6e23bdc6,0xa976e9a5),LL(0xa757b390,0xcc8cfb98), LL(0xa4697fde,0x5e6a1ae0),LL(0x5aafbacc,0x5c62485d),LL(0x27e4f7d3,0x804597be), + LL(0x53bc971a,0xea91a442),LL(0x436f90ba,0x41484d9b),LL(0xd87d03a9,0x21b57dab), LL(0x8f1888f0,0x935c940c),LL(0x4c82401f,0xe1c2a3eb),LL(0x7847c8a6,0xf42fb0f8), + LL(0xb4347b62,0xa649770f),LL(0xb29590ef,0x3bf77343),LL(0x6a205e84,0x92b6e5a1), LL(0x63cac357,0x8c973844),LL(0x7e8ab06a,0x058f0f22),LL(0x58d62ed7,0xaddca6e4), + LL(0x1160ca95,0xf4b969c6),LL(0x0d73f647,0x60e6b905),LL(0x9b98ce7e,0xd8413d40), LL(0x9ec67398,0xebd6924f),LL(0x628424b1,0xe4c95b85),LL(0xc7f2b86c,0x157759fb), + LL(0x195680a9,0xd6f1e165),LL(0x965d62c7,0xd433c1dd),LL(0xd3dd177d,0x0890fc5d), LL(0x8c44dbc2,0x3e0f43d4),LL(0x9cb07f9d,0xaa0c14ed),LL(0x6ff8b939,0x6d7feea2), + LL(0xc17b0751,0xa1c5b89f),LL(0x46164d7b,0xf47724f4),LL(0x60ec3b26,0x9b1dd929), LL(0xcb50b27b,0x6c3c835c),LL(0xf86980fd,0x259fde39),LL(0xde92e229,0x84e73227), + LL(0xd05a1ec4,0x1f1e9817),LL(0xe6e48ece,0x2692c83d),LL(0xf4903352,0xa2c5bd53), LL(0xaf3eb235,0xa1f9c3ff),LL(0x71158432,0x52c141c2),LL(0x16d971ea,0xeddd3c85), + LL(0x3724189a,0x7554e3b9),LL(0x2e63e62b,0x5eea6e3b),LL(0xe23434b0,0xe28ad863), LL(0x5f932bc7,0x6016f1e8),LL(0x7d76d1fd,0x1240c223),LL(0x5124123a,0x2ce14c4f), + LL(0x6d50fbd1,0x6c89d0a8),LL(0x31779189,0x4a38b313),LL(0x8d8124b8,0xa1718ddc), LL(0x0e02b2d0,0x58f48b51),LL(0x6114c7e3,0xd38936a6),LL(0x1b70ec0a,0xb5e924e8), +}, +/* digit=16 base_pwr=2^112 */ +{ + LL(0x1f47a973,0xcf447109),LL(0x627fd943,0x24c191c4),LL(0x32c6da19,0x4f5f134d), LL(0x9d08667d,0x1b92e5f9),LL(0x1d9684b0,0xe6b9a4e8),LL(0x553bcb2a,0xd60fdeba), + LL(0x46cc2ad3,0x840f76a1),LL(0x6235d7a8,0x63b85026),LL(0xb2ef5ad3,0x5616a569), LL(0xc3f03ea7,0xa056663a),LL(0x38ce234b,0xd93ddef0),LL(0x57a8650d,0x0f729e40), + LL(0x360595c1,0x1b437776),LL(0x080dc311,0x6aa3a75d),LL(0x11147932,0xd1288177), LL(0x8a241c89,0x8c953ffc),LL(0x7de54cc4,0x7de8a558),LL(0xa5a3ed5a,0xa8dc2b2f), + LL(0x084ee344,0xf6b53652),LL(0x4048d28e,0xb51a92cb),LL(0xd688ff26,0x58fb44be), LL(0x9247a804,0x136a7d51),LL(0x3ef5f22c,0xf2176fba),LL(0xa999af5d,0xb4a457be), + LL(0xf23f79c8,0xb4cf845c),LL(0x1ad85366,0x8233b3b2),LL(0x78e68d8c,0x4efef3d6), LL(0x22c7941c,0x17b1e851),LL(0x761dde7d,0x40a27661),LL(0xcbd06300,0xac29e1a1), + LL(0x0ced7008,0xff48c690),LL(0xebfce074,0xbed07115),LL(0x27aa6e6f,0x4ac10aa5), LL(0x5e777e33,0x7652ebb3),LL(0x4dd30e6c,0x255c16ab),LL(0x9159219a,0xc0e1b073), + LL(0xec32e62a,0x6d2fc306),LL(0xe70096e4,0x426dd67b),LL(0x70d9383d,0x994877cc), LL(0xd011ca1f,0x8bc75d3a),LL(0xaa1362ae,0xcb18fa82),LL(0x2c292316,0x54fb1473), + LL(0x8c720bd1,0x45aa7854),LL(0xc5257a6e,0xceba0e5d),LL(0x796e4320,0x0e924974), LL(0x866fc34b,0xf2ce13c7),LL(0x6a58601d,0xbe8606f3),LL(0xbf56d38c,0xa25d6005), + LL(0xbdbaa117,0x9432504c),LL(0xba422130,0x4dfc9c38),LL(0x500ef320,0x32d66ac1), LL(0x0679d51e,0x1a38a920),LL(0x93a9c6f2,0x3e2a3c53),LL(0x0a59702f,0xe0b71451), + LL(0x4cefcb03,0x04e4087a),LL(0x4ab1d062,0x68686886),LL(0xdd24b69f,0x67054513), LL(0xa89e5edb,0x21d5ff58),LL(0xd8e0f856,0x9412a213),LL(0x3faa0ffa,0x1a05bd39), + LL(0x14033a58,0xee2aec92),LL(0x5161890c,0x38e267ef),LL(0x8c51e740,0x2f0ae797), LL(0xea57871c,0x6d2143ac),LL(0xc5475c3e,0xef3ce79b),LL(0x1fa55fd9,0xdd9cfa6a), + LL(0x412965e9,0x6ae0ef90),LL(0xe9188107,0x2430c1f0),LL(0x42b98992,0x9682e74d), LL(0xb638ccb1,0x3094b342),LL(0x16948a40,0x6aa863ef),LL(0x60f1cca5,0x2ef984e5), + LL(0x694a80f4,0x4980d35b),LL(0x2c3d3326,0x4dbbe36d),LL(0x1c899dfb,0x1c719190), LL(0xe9938032,0x96febca8),LL(0xa63775ac,0x7b17130d),LL(0xd12cfb2c,0x17e68ea4), + LL(0xfde8a210,0xf111a32f),LL(0x83b56b17,0xc3d61a7e),LL(0x22b86813,0x48125662), LL(0x410c959f,0x1ddf2f99),LL(0x68591a75,0x886a1893),LL(0xa21da760,0x92dc4bbf), + LL(0xb9b71c82,0xa2d0b7f2),LL(0xe59c5616,0x206fb318),LL(0xa9f1f448,0x01112ace), LL(0xd40341f9,0xbc6ee063),LL(0x90590590,0xe2c50a8d),LL(0x795c4ef5,0x3aa25a83), + LL(0xf5016294,0x582102f1),LL(0x8ecaf2bb,0x6c6a8cc5),LL(0x9e733710,0x7887a218), LL(0x82da339d,0x18ea2c41),LL(0xef8def66,0x428afd13),LL(0x5f923e0a,0x171ec875), + LL(0x98756704,0xd9233e20),LL(0x86a8b3bb,0x5b3a223a),LL(0x216d0c55,0x844ef9e9), LL(0xb7f4d4b6,0xb8a1a350),LL(0x64bb191f,0xf70cb37c),LL(0xefb8ae25,0x7c42a6c0), + LL(0xc8074a8b,0xf789197b),LL(0xbd20de1c,0xf7984f1b),LL(0x50eb451d,0xc0b1f821), LL(0xa88538d0,0xeb36fa99),LL(0xaa22b727,0x87c78bb1),LL(0x3644f1eb,0xfe697599), + LL(0x6fb08720,0x775050a3),LL(0x7c92825b,0x6547216d),LL(0x4a525878,0xe4c237f8), LL(0x63d4f3fd,0x584a8f77),LL(0x80b593cd,0xfefbf7fb),LL(0x89778016,0x8f52fc2a), + LL(0x275beb93,0xdf1d7c82),LL(0x7512f5a7,0x4e3e1e22),LL(0x833674bc,0x0a8c3404), LL(0xabc4edcb,0x92c482d9),LL(0x9fc05328,0x90d5620d),LL(0x72e82254,0x49facc26), + LL(0x2ecb1fc1,0xc5acdaaf),LL(0x6e74dd57,0x2a6d3302),LL(0xda6dfd6a,0x7ac86ef8), LL(0x4074218a,0x24edf1a2),LL(0xf9162dd2,0x9010034e),LL(0xae5e58fb,0x035ed18c), + LL(0x55f3fe0b,0xa961e6d8),LL(0x46a86b10,0xcd5156e2),LL(0xc2988880,0xc4fc85c9), LL(0x5e00c084,0x68c21329),LL(0x9af42336,0x505ffd96),LL(0x2d4346e2,0x2d3d12ff), + LL(0x81aaf463,0x19a0b708),LL(0xe4d6eec0,0xc956aa02),LL(0x897c0465,0xa8dba669), LL(0xbee2b8a9,0x5c8a3e27),LL(0x80c032bf,0x8ae8552d),LL(0x482aaf27,0xe7526fc1), + LL(0xc12240f9,0x4767cb3f),LL(0xfe66cca2,0xa31fe2bb),LL(0x0c03119a,0x15f6f333), LL(0x9283ee27,0x67a68e9a),LL(0x53e4d988,0xe2d7d349),LL(0x0150cf1a,0xaf919ee4), + LL(0x4bc9e09b,0xcd0b84c0),LL(0x8f7286bb,0x0e18d19f),LL(0x5aa9039c,0xf51f3b13), LL(0xb25604be,0xe0bf876a),LL(0x60e8118a,0xaf35db67),LL(0x6dca47f8,0xd6f2ebd8), + LL(0xe4942689,0xdb226347),LL(0xd061e06d,0xda1e15d6),LL(0x3d043dec,0x2a99ad74), LL(0x7a5cac1c,0x78a33189),LL(0xd8bfd02e,0xb3b6c52b),LL(0x7aaec4f4,0xcae776f5), + LL(0xe13ee1b0,0x00bd80ca),LL(0x5570567c,0x678837f3),LL(0xe22c402b,0x379e4e13), LL(0xb68e8230,0x7f51fdac),LL(0x5d23aa66,0xc76acd65),LL(0x2caaa8e7,0x4b8a0d89), + LL(0xb2c47709,0xeeb913e9),LL(0x01b76a21,0xc2683d5a),LL(0xec8c8ce7,0x58e53311), LL(0x0654cd8a,0x0a179454),LL(0x5999ac6c,0x011152f5),LL(0xc8b4fa8f,0x15eb6f48), + LL(0xfdb6b880,0xe5363d68),LL(0x3ad457bf,0xef83e507),LL(0x6fd08edd,0x63874010), LL(0x85975c5f,0x64b77762),LL(0xbed19eac,0x11cf13e2),LL(0x19f07470,0x823d8c2e), + LL(0xdc4e8957,0xd091f448),LL(0xcf0950b8,0x9825fdfb),LL(0x0d1e5204,0x1e297e4b), LL(0x499cbea9,0xab53b2a8),LL(0x4917c840,0xca4a1f80),LL(0x2a785cab,0xe6506492), + LL(0x0f900918,0x9c532f97),LL(0xcb00118b,0xfc209bf8),LL(0x6290d47a,0xdd4a28aa), LL(0xcb4c67cc,0x8a7aa0f1),LL(0x3a3a7acc,0x1d7cc682),LL(0x504f910f,0x085b0335), + LL(0xc3978c89,0x2a3e44bc),LL(0x3724d81a,0xbbc69641),LL(0x8e42053e,0x8cd0bcf6), LL(0xebd9205c,0x9a84e877),LL(0x8bbf687b,0x6c153f89),LL(0xdee7732a,0x730b14c7), + LL(0x3e54212a,0x71f21e01),LL(0x2044a858,0x2fce6291),LL(0xd5b7e3b0,0xb3b630f4), LL(0x59a7edb1,0x42ceac09),LL(0xbddf2781,0x6a40528a),LL(0x8954daf0,0x68b67f7b), + LL(0xd4cc9c00,0x6f07450a),LL(0x243d9b91,0xac4f418f),LL(0xb3930836,0x862c4d7c), LL(0xb829804c,0xad9509d4),LL(0x35d6a88a,0x5448355a),LL(0xa02a8977,0xb1fd6cf6), + LL(0x12fda311,0x926dc0b6),LL(0xa713c6b6,0x3858aa1e),LL(0xb59d713b,0x32532f12), LL(0x0f81bc84,0xe0a3bd86),LL(0x34b57212,0x28bfc374),LL(0x59e6b4f4,0xc1d92564), + LL(0xd1adc5f0,0x749961eb),LL(0x6897fc6f,0x070b3de4),LL(0xb71a0ade,0x0b95a35d), LL(0x4c6cf54f,0x993a0e3f),LL(0x07030f4a,0x57dd5c90),LL(0x12298384,0x191f3d38), + LL(0xfe98a51d,0xd30028ba),LL(0xf0f77aef,0x536fceb1),LL(0x15f7d4d9,0x58a8d8cd), LL(0xafa42baa,0x498dd178),LL(0x8ad8d1db,0x107ae753),LL(0xf4a6e0d7,0x79957dae), + LL(0x098aa8a4,0xdefb4a06),LL(0x5a536615,0xd45f01be),LL(0x2175624e,0xdfb9ab8f), LL(0x432b8122,0xb7f76e8a),LL(0x66ba6ee3,0x838ee280),LL(0x275d1fcf,0xe6911bea), + LL(0xe82bfc51,0x5ddd5dc2),LL(0x126dd7dc,0x36994ad7),LL(0x520c3a43,0xa3be5780), LL(0x6b94a716,0x093dec86),LL(0x9ba657b4,0x5189af95),LL(0x2c605098,0x188fb2ee), + LL(0xb76392f7,0x14886c31),LL(0xdecb5423,0x304dd2c9),LL(0x0214664e,0x853a2344), LL(0xca815581,0xa150c793),LL(0x5edee2f8,0x1334dabe),LL(0x58d437d1,0xfa263569), + LL(0x31b3f896,0x348804e5),LL(0x4bc1df4d,0xf9778c68),LL(0x5ba2cff5,0x9174fb72), LL(0x3156fc57,0xdae23f11),LL(0x0eaf5c36,0x146fe941),LL(0x6954d2fa,0xa756199e), + LL(0xfc879267,0xd875fdac),LL(0xeac024c1,0x504843c3),LL(0x9abdbb10,0x2880c735), LL(0x3b9acd44,0x45f8969d),LL(0x6261a810,0x86744967),LL(0xda56c33d,0x2889aa10), + LL(0xcca4b465,0x4cd3e586),LL(0x0438f6dc,0xb9858004),LL(0xff4ba237,0x5ecb0167), LL(0x2de8c238,0xbf2dfbac),LL(0x86067a4e,0xd1a1f341),LL(0x514110d5,0x6e001905), + LL(0x32515872,0xcc0b6188),LL(0x9db67e8d,0x8a5a4765),LL(0x4032cd76,0xfc5572bd), LL(0x4ced4c98,0xa8148877),LL(0x7e29acf7,0x3a3f4af1),LL(0x4878a56e,0x19e7ba4b), + LL(0x6ffe2a99,0x7d54fa14),LL(0x602fe247,0xd9f414c7),LL(0x284f46df,0x31abd712), LL(0xe6a9ef58,0xf55a66ce),LL(0xb2d9463d,0x8d76dacb),LL(0xc14ace92,0xac7aebc3), + LL(0x7fc941bf,0xa7b46744),LL(0x3f70d808,0x0c420029),LL(0x177bab8e,0x3da5d17e), LL(0xb3a67013,0x3ce9a56e),LL(0x34c0d947,0x4fc18745),LL(0x4571d7d0,0x1295a40c), + LL(0x59a9c46e,0x55635c3d),LL(0xbfe9682c,0x425f4119),LL(0x47824565,0xd6486529), LL(0x8a96f0bc,0x741c7a34),LL(0x382a3617,0x9cb36316),LL(0x7e0bb8c3,0xe8704d9b), + LL(0x261438fc,0xba5fc6e5),LL(0x304dce0b,0xd455c9e0),LL(0x5d776157,0x2c5cd9cc), LL(0x58000464,0x2ad5cc3e),LL(0x9efd5253,0x88b007d4),LL(0x031f40e8,0xc5024b19), + LL(0x89f85ced,0x43e291bc),LL(0x2565e514,0x704c3d56),LL(0x6f19bf54,0x401941da), LL(0xc42b17b9,0x407ffe45),LL(0xee6116a1,0x68753698),LL(0x0f1e7e3c,0xf69c5def), + LL(0x59d50565,0x7e95369e),LL(0xc3f575f3,0x089278bf),LL(0xd7d4ed14,0xfbf02dc1), LL(0xc063b9b3,0xe98848a5),LL(0xb3d74962,0x49058917),LL(0x937ac3e7,0xbaee0b60), + LL(0x9517cd36,0xd3f4711d),LL(0x5ab34ba6,0x72055716),LL(0x8054f1b6,0x55781094), LL(0x854d305f,0xe3ad29bc),LL(0xa21b9037,0xc6f2e79f),LL(0x0478e475,0xa01b493a), + LL(0xf078fc36,0x80124211),LL(0x203d5414,0x29e04253),LL(0x1a40dfb6,0x53737763), LL(0x19d3137d,0x698334c3),LL(0x22b7e891,0x02c4658d),LL(0x285de1ab,0xe33063e7), + LL(0xe76c0ac6,0x56eb9f2e),LL(0x90aef3e0,0x3898b635),LL(0x3e409c12,0x016a7234), LL(0x32cd68c5,0x488d72fe),LL(0x809dd2bd,0xc9cfcd28),LL(0x0b99c1d2,0xec284a02), + LL(0xaf16bcb5,0x7cc64eaf),LL(0xdb4ffe42,0x6a629feb),LL(0x2d775fbd,0xdf4a9495), LL(0x41ee0c37,0xde89a2db),LL(0x2b89d6f5,0x4bcb9b55),LL(0xaa29912c,0xabeefb50), + LL(0xde0b2010,0x026c18f7),LL(0x43b3d1d2,0x39495f8b),LL(0x30288eb8,0x63546dd7), LL(0x68b69732,0x0a49e9b2),LL(0x0ffd4a14,0x9ef54b7e),LL(0xd1fd4c67,0x834add8b), + LL(0x95444724,0xaf5a35ac),LL(0x98374f56,0xa4f56e13),LL(0xbad358c4,0x779136c5), LL(0x0b77d7db,0xda98e86d),LL(0xd83f4464,0x664c97a2),LL(0xa95543ed,0x9e4c6820), + LL(0xa63fde03,0x4189250d),LL(0x252242a2,0xfdf0c2da),LL(0x04072ae9,0xc46d8769), LL(0x6679d6e8,0xac1026c9),LL(0x30b3ab9e,0x2932cf08),LL(0x52f5c3ef,0x543f55db), + LL(0x14b85fd4,0xbea36af7),LL(0x5bcce101,0x32771ea0),LL(0x798ec0fd,0xd9a59a8c), LL(0x4d437c35,0x165e1bfe),LL(0xedfb6d49,0xb55f1e8b),LL(0xec21ec21,0x362e3826), + LL(0xd4c60ba2,0x3aebfb11),LL(0xc7e3f014,0x77426fe0),LL(0x8a27fdc3,0xe457c3e5), LL(0x5e5bc8a5,0xb301854c),LL(0xd258898f,0xe6b8c9d9),LL(0x415129b5,0x3a7ead26), + LL(0x1e2c52bd,0xe3760794),LL(0x36b33409,0x383b64a3),LL(0xe476721f,0x3e25c3ed), LL(0x636f7572,0x209893c6),LL(0xe5f8134f,0x7e26bf15),LL(0x155ae70b,0x27e0a220), + LL(0x9e9f2836,0x8f6b7ae3),LL(0xf9826937,0xf5eb2b96),LL(0x27c80d30,0x6b213e6c), LL(0x5320af31,0x52d85c63),LL(0x719df6d3,0x468df649),LL(0x2744251b,0x69403a14), + LL(0xd9a2d948,0x28b1b9c5),LL(0x8e0c0b46,0x171331b5),LL(0xc359284b,0x84749e1b), LL(0x2e820b7b,0x184eb606),LL(0x740d7291,0x4f45113f),LL(0xaacb8d25,0xd1743386), + LL(0xf464caa0,0x003c4b6d),LL(0x6280078a,0xe1e5b15f),LL(0xbfdf8ef2,0x2c0ace42), LL(0xbdcc429b,0x695d68df),LL(0xc9b7c777,0xb9a74231),LL(0xe2ee83bc,0x72976a32), + LL(0x649ab654,0x72625a05),LL(0x40672830,0x4c950283),LL(0x6265499f,0xaa52b9e0), LL(0x9e60c50f,0x82c299fa),LL(0x697cef66,0x32ed0262),LL(0xf4c1412d,0xa5fad068), +}, +/* digit=17 base_pwr=2^119 */ +{ + LL(0xa11d1043,0x5db5f2ac),LL(0x3144df6d,0xb1054ec2),LL(0x484fcf47,0x4d5655f8), LL(0x9dbb1826,0x0650cfdb),LL(0x21f5feda,0x27dd6048),LL(0xd7202fb5,0x0ac68e0b), + LL(0x1af46fd9,0xc42442cd),LL(0xfcf7cac7,0xb01213f0),LL(0x7eee2308,0x36e25d39), LL(0x17b3a506,0xcbf7331b),LL(0xcbb09169,0x9cb57ff6),LL(0x2928d485,0x819296b4), + LL(0x8f4e94b8,0x22111a83),LL(0xb5d0778a,0x808a8a94),LL(0xe721738a,0x6dc353a0), LL(0x622c3cad,0xfbc21338),LL(0x2a4f4740,0x33ad7aa1),LL(0x7bd23daf,0xc3ce2ea5), + LL(0x17755e9d,0x5214c0df),LL(0x22952389,0xd0989763),LL(0x2069bb1e,0x51accd0b), LL(0xe9cdd0be,0xe8ff9789),LL(0xbb212bd4,0x7e57a08a),LL(0x31edf24e,0x6b3e08ed), + LL(0x6be00a20,0xf20543a1),LL(0x190d57cb,0xb8bcaa66),LL(0xef2b91a2,0xdd7c09be), LL(0x5592a7fb,0xd0e3d7f0),LL(0xa4e5380b,0x71dd3f1f),LL(0x7ad60960,0xe1a6c123), + LL(0xd3ca68f9,0x57bc69bd),LL(0x52100876,0x4ada2eed),LL(0x8c3bb3a7,0x2e966e5b), LL(0xe177995d,0x507cdf48),LL(0x62dec6cf,0xa34769db),LL(0x615eb17f,0xd9cd4150), + LL(0x7de68ea6,0x0135519a),LL(0xe8a86804,0xe966ae0c),LL(0x4d390e2d,0x63738258), LL(0x91e8e57c,0xfd018375),LL(0xd49915f4,0x31e5d307),LL(0xd1e1542b,0x173b719a), + LL(0xfc46d6d5,0xb492fccc),LL(0x46eddcd3,0x973958f7),LL(0xb0f21fba,0x7222fff6), LL(0x41f0f218,0x8eb1db20),LL(0x9d5451aa,0x7353ea3b),LL(0xf054ffc5,0xf2cd3afc), + LL(0x25b37d1b,0x148268ee),LL(0x6c58898f,0x9d597241),LL(0x5ef964a3,0x516c6871), LL(0x73756f60,0x777d95dd),LL(0xce04cb87,0x79b9b5e6),LL(0x371ab2f1,0xdbe50dca), + LL(0xb9c32a0e,0xc12a3617),LL(0xa6d8dedd,0xd368a553),LL(0xef101236,0xdf472f4d), LL(0xa749514e,0xd6bc5900),LL(0xc2a203dd,0x7531976c),LL(0x983e67ab,0xa5bbf91a), + LL(0x5ee73780,0xad4ada0e),LL(0x41fe693b,0xdf363734),LL(0x1a88c54c,0x996727c5), LL(0x6f0cd51a,0x635e6cd5),LL(0x4e75f89b,0x93d7e6cd),LL(0x2326e236,0x761b08f9), + LL(0x27a21294,0x719c6065),LL(0x0f25f450,0x5fd2859d),LL(0xd52b85df,0xd10bf835), LL(0x40de93c0,0xa6fd2831),LL(0xf7c3a0d4,0x89c1bd7d),LL(0xefebe6be,0x75fed863), + LL(0x17c97de1,0x921b88ad),LL(0x2cd59e91,0xb48dc81c),LL(0x73f3504e,0xda03b882), LL(0x5f2ee761,0x39ff606f),LL(0x5ad4f4de,0x608772a2),LL(0xc72d7dbc,0x2b4d5732), + LL(0x17a8bdb6,0xd5d5f3d4),LL(0xde7a418e,0xcdadbfcd),LL(0x73f2b0da,0x2fbe10bc), LL(0xdd2b933a,0xcc436783),LL(0x120a26f4,0x7d464c7e),LL(0x3295a8b8,0x2e25a06a), + LL(0x4edfcc33,0xce566ba6),LL(0xb883585e,0x72ced5ee),LL(0xe257f582,0xc7a1597f), LL(0x42dd036e,0x469e4698),LL(0xcf916dd4,0xf534e335),LL(0x5409b755,0x2a1a0ac7), + LL(0x9c7ba23a,0x2ee09586),LL(0x2d9a1fbb,0xd9dae4aa),LL(0x0701d2cc,0xad876506), LL(0x35d64ab2,0xb9eaf2f9),LL(0xa2ca2cf4,0xc8a57b67),LL(0x0938f8b2,0x7e81d43e), + LL(0xf1f4c06c,0x517ac75d),LL(0xd2fb833a,0xebd83a3b),LL(0xd0ceba1a,0xe14412b0), LL(0xc203df53,0x14bfd5f7),LL(0xa3cc4222,0x8eddee82),LL(0xe0413bc6,0x987370db), + LL(0x2c70f714,0xc3aa5190),LL(0x1e87511c,0x35550f45),LL(0x207b35cc,0xacdb541a), LL(0xddc40e1e,0xd645313b),LL(0x5d7745fb,0xfadf08df),LL(0x63f5d14b,0x34c4a2da), + LL(0x74f5662b,0x3bf9a89c),LL(0xc3293614,0x26fa24f5),LL(0x93645819,0x2232f2bc), LL(0xb642a3d2,0xae2493be),LL(0x4354eadc,0x67888dfb),LL(0xd5a343ce,0x7917ebd9), + LL(0x22e18ac9,0x83f1a578),LL(0x96227250,0x754ac8ad),LL(0x02f84593,0x8720c7b5), LL(0x6077017a,0xbe8034ce),LL(0x125f0aba,0x0e38bcc9),LL(0xd7c001b7,0x491175d9), + LL(0xdc6d9d65,0xdc830479),LL(0x8402b7f8,0xd961ba98),LL(0x85d31cef,0x9a870f49), LL(0x747a1a30,0x3f08d97d),LL(0x5e845976,0xf5cd0560),LL(0x39c404c9,0x34b2e606), + LL(0x479f9f28,0xc6be3e89),LL(0x8639cfd5,0x277e7391),LL(0x6eac029f,0x90018430), LL(0xccc296ef,0xdeebdb98),LL(0x6c909542,0x225d9267),LL(0x37a527ab,0xcf151ac4), + LL(0xa65460ad,0x40e22878),LL(0xd388bcc5,0x0a345fb2),LL(0x01c72449,0xfdbb8d02), LL(0xb73e0d19,0xeef0f1e3),LL(0xbbfdd87f,0x4ec39f63),LL(0x5ef57d3b,0x79501995), + LL(0x60873c20,0xd6dbf138),LL(0x6715b03a,0xeee2c9aa),LL(0xa7482c51,0x2f5e5ce2), LL(0xc3dd1233,0x927e5fb2),LL(0xd64c61c6,0x72dde264),LL(0xcd32e0b4,0x605f8de1), + LL(0x5b9091af,0x8ca120c2),LL(0x0f0fb8bb,0x6f7fa55e),LL(0x87264d00,0x78e97110), LL(0xd0f9cca4,0x63be0590),LL(0xe863c20c,0x4825ad2d),LL(0x12449414,0x4c34ba2d), + LL(0x21fd0243,0x0256c27d),LL(0xa2406f04,0xbe85c36f),LL(0xb457d240,0xa30e427f), LL(0x3b4a0eb7,0x4bc8ea0f),LL(0xbfb3b1d4,0xf84a988b),LL(0x20a37ec6,0x1e9d2665), + LL(0xb27e7722,0x07d6c0ef),LL(0x6729edd8,0xb81d03e2),LL(0xecf969a0,0x75b9e7d4), LL(0x72ad86dd,0xef9c8251),LL(0x9292331b,0xe609a51d),LL(0x35e1a81c,0x0df83add), + LL(0xc5478838,0x73e31c76),LL(0x44828fc8,0x1a131a98),LL(0xb5e70dd2,0x14e09dce), LL(0x4a72dd31,0xb38126a3),LL(0x5b0e772f,0x3d06723b),LL(0xb7d1198d,0xca600717), + LL(0x57b91e05,0x047cb901),LL(0x7e21d7a5,0x1946f1b6),LL(0xccec26a7,0x042e3eca), LL(0xfd416a33,0x0d9f9119),LL(0x49019bf8,0xdf57814c),LL(0x6527b664,0xe01c04b8), + LL(0x359b599d,0xc0fb1444),LL(0xbd4e90c3,0x09858bf6),LL(0xe4adda27,0x1ad5019b), LL(0xe2069275,0xa8229db9),LL(0x2e9a9c96,0x4af49c04),LL(0x32909ac0,0x2eba27df), + LL(0xba29ac20,0x44771234),LL(0xa8f7d70a,0x91271936),LL(0x35e3439b,0x1d2553dd), LL(0x66de658c,0xa35e4915),LL(0x5505ca57,0x958c5d53),LL(0xefac11c3,0x5806c454), + LL(0x68947de6,0x480af451),LL(0x20ed7c76,0x1970a51c),LL(0xfd273a2b,0x13be8cf8), LL(0xde3fb216,0xdaa99fc9),LL(0x007e796d,0x9e1ecd60),LL(0x1145bac9,0x0d61162b), + LL(0x1a7240fd,0xd324f3e5),LL(0x02dead6f,0xb652ceb5),LL(0x945e4927,0x9d2efe09), LL(0x09d3fe5a,0x505409cd),LL(0xd0b1d40f,0x4068d39b),LL(0x3d2add11,0xb169ceb8), + LL(0xf87110f6,0x95209c85),LL(0x9d8c8473,0xce5e933a),LL(0x2dac6c5c,0x50e9c87d), LL(0x6c95d63f,0x5ad288c5),LL(0xc2e80fdb,0x9801bde5),LL(0xcf50ed4c,0x6d577a05), + LL(0x5c341a0f,0xb2952c0b),LL(0x08fc2f1a,0x5296d67b),LL(0x595be428,0xb671036e), LL(0xf1974091,0xca815a34),LL(0xd7822f17,0xe4a2f88a),LL(0xaeba3ed7,0xdfc557f7), + LL(0xf1f5523e,0x78c30cae),LL(0xa828197b,0x467ad201),LL(0x41532182,0xccf347b4), LL(0x84ef4bc9,0x67c4ff37),LL(0x98cade6e,0xebeb3d71),LL(0xa59ab992,0x979dba66), + LL(0x0a363201,0x5ff9bcb7),LL(0xcc8b691a,0xb6995231),LL(0x62797899,0xc9aa7830), LL(0xcf5ece20,0x57d1c888),LL(0x963c0086,0x688d3d70),LL(0xc85a248d,0x91091fbe), + LL(0xe39ec208,0x4d6e7182),LL(0x8d5cf62a,0xda235ace),LL(0xde7dbbdb,0x33f70b00), LL(0xe66a3b86,0x60f841c4),LL(0x42d7e874,0xbf8660af),LL(0x80f2c57c,0x073c5f84), + LL(0xbc4f1599,0xd80d4f36),LL(0x185811a7,0xfd5e23b6),LL(0x7ac38f06,0xc8b0fa81), LL(0x43e96490,0x652d79b3),LL(0x73d98971,0xe222aa1d),LL(0xd121248f,0x4b6278dc), + LL(0x3c14b2d0,0xce20c25e),LL(0x98193e2e,0xd1379c97),LL(0x6d804130,0x21102482), LL(0xf20fb7a3,0x577f8229),LL(0xb4f08660,0xdef46260),LL(0xe4dffb3e,0x8bb54fac), + LL(0xc0ecadfb,0xbc45d692),LL(0xd3435c47,0x2d92602f),LL(0xa5dfe7a5,0x29e826c0), LL(0x42a0c989,0x9272dc04),LL(0xe3b3a440,0xc1eddc74),LL(0x65741581,0x741fa074), + LL(0x75dbc772,0x0e2a8f97),LL(0xcc5e88b1,0xed5c6a60),LL(0x715808ac,0x099753c4), LL(0x9cd4d1fa,0x515d4892),LL(0x78393560,0xc4e5c9fe),LL(0x59206312,0x7ce9ac16), + LL(0xba61c10c,0x05e139c7),LL(0x56285997,0x83c2806d),LL(0x80f93027,0xf308c086), LL(0x3d95ff97,0x192570cb),LL(0xb69f20a2,0xd7677bc9),LL(0x9a9416a8,0xfea2554b), + LL(0x2ac3e62c,0xe26cdbf8),LL(0xb719cfa9,0x99cfefda),LL(0x7c7c2f30,0x67fc003a), LL(0x529cd8a8,0xfb8eb991),LL(0x8902bddd,0xe63830d5),LL(0x87d9a82d,0x8d0dd890), + LL(0xcdd4c46f,0x239f32f2),LL(0x4de15fb1,0xf9517e3f),LL(0xbb8c38ff,0xc8183a19), LL(0x4f41bc08,0xefe4c3ef),LL(0x36ec7fd4,0x900d74fe),LL(0xfbf56005,0x49b24c64), + LL(0x2aae4376,0x8c1c4bb0),LL(0xf9c1a936,0xe9eadc2b),LL(0x9b62791b,0xea437c5a), LL(0xafe6482d,0xb43a409d),LL(0x5ae0e629,0x405189df),LL(0xb6b6e06a,0xb60dbb63), + LL(0x122f06d7,0x015b080a),LL(0xe52d69ac,0x8fe18e03),LL(0xbe34ac7f,0x54894766), LL(0xd5131f42,0x60d01c43),LL(0x24b2ae7b,0x179b14a6),LL(0x6e7c925c,0x2478209d), + LL(0x7667ec3c,0x037b49bb),LL(0x0ddcd693,0xb4751c25),LL(0xe82b8e88,0xa9933bfb), LL(0x741344e2,0x14f812fc),LL(0x25e24d12,0x7253361b),LL(0x071df26d,0xaba4f18b), + LL(0x05bcc791,0xfe1f3f2f),LL(0x33608ce5,0xe3ae1232),LL(0x672aa21b,0x3325659b), LL(0x9a01ebde,0x6d572c9a),LL(0xa10425ad,0x733c2067),LL(0xf7e7b67a,0x80f1da8b), + LL(0x2e21e1c3,0xdcba0c7d),LL(0x79eb4307,0x08883499),LL(0x27cd2b91,0x5a5ee0f8), LL(0x13a4c6d0,0xcc6494e2),LL(0xb3270d14,0xdc1c75e3),LL(0x66eb04f8,0x64e17db8), + LL(0x3e232e2c,0x1b8e2672),LL(0x0e99fc0c,0x1553fed2),LL(0x3c3d73af,0xe4e6608a), LL(0x3ed25820,0x8dc28c2a),LL(0xcadc377d,0x5f375146),LL(0x4b113c00,0x09e1c08d), + LL(0x2d096c20,0x567cec1c),LL(0x9c346dd2,0x0b3fb247),LL(0x79a71044,0x1e4c1e43), LL(0xd86defcc,0x4980b895),LL(0x7175491a,0x9ee53061),LL(0x979cdece,0x28b5bb15), + LL(0x16b49afc,0x36cb6dd6),LL(0x9372557b,0x498a1aef),LL(0xfa192344,0x4e51a28e), LL(0x6cdd93bc,0xf5a4373e),LL(0xec6a751e,0x54a73a75),LL(0x841194aa,0x9a563884), + LL(0x91ad93f9,0xa95a8a06),LL(0x540e0c19,0xb0681b8a),LL(0x4f815aec,0xb0cb9c4d), LL(0x9b9c744a,0x5acf61c1),LL(0xfa7f4dc9,0x72c1126c),LL(0xcc2a7aa2,0xa5afea44), + LL(0xf614aace,0x5a9fbecf),LL(0x794afb09,0x2e940ee6),LL(0x0ecaf20e,0xcc5a62de), LL(0x7a41866c,0x57634449),LL(0x4335ac0c,0x365bb619),LL(0x537165fb,0xe8c5699a), + LL(0x0db61f1e,0x61966988),LL(0xf31a42d9,0x9a86bd4a),LL(0x93826e9b,0x712f2cdb), LL(0x17d5867b,0xd8edee16),LL(0x8710c50b,0x1e25628f),LL(0x7daab4cc,0x4f0c8f12), + LL(0xd3fea70e,0x0ec1ef9f),LL(0x6591a1bc,0xdff00f61),LL(0x126ace6d,0xb680207f), LL(0xc5b9722e,0xfaa3fd69),LL(0x0b3e7f42,0x4a46ce3d),LL(0x49ba8836,0xdb76aec5), + LL(0x77738737,0xeaa46cad),LL(0xe1b12f15,0x5c0f2ebd),LL(0x0ca12f78,0x1b07bf63), LL(0x8dd854dc,0xbe19584f),LL(0x11c1c918,0x9ac8971a),LL(0xb3600069,0x72f4b00d), + LL(0x43f232a4,0x8c0c1e19),LL(0x6a116ed0,0xbd774022),LL(0xbebaeb71,0xee2ead3a), LL(0x861ab743,0xf224296d),LL(0x650a381e,0x37b4d979),LL(0x8d64e6f5,0x7a8da726), + LL(0xd9e956fe,0x528c8c28),LL(0xa1f5f87c,0x1bb92006),LL(0xff91e588,0xf2e662c7), LL(0xbb5cfe01,0x36ad72ae),LL(0x19e772d5,0xf489b95d),LL(0x8b6ddd88,0x77675dbf), + LL(0xf10d59a9,0xa40a5023),LL(0x250fb5de,0x15965ab7),LL(0xb539afcc,0x27808277), LL(0x8cab9918,0xf7f7ee31),LL(0x6c0353bf,0xd5b05608),LL(0x5255c6a1,0x20a150e8), + LL(0x192fc198,0xdc6a57e6),LL(0xc3c7ed17,0x3506fc9d),LL(0xa4a271bc,0x916219f6), LL(0x964b3816,0x03f8b160),LL(0xf1a09190,0x454bae16),LL(0x6fab73b1,0x6d316b41), + LL(0x5c22fc9b,0x768669f6),LL(0xe0219ce7,0xc2bbb0c5),LL(0xaf224185,0x77d99a44), LL(0xcead6ae8,0x1bdcaaa5),LL(0x80bcad3d,0xa0015109),LL(0xac464ccb,0xde74800f), + LL(0x7c5f58b6,0x419fcc04),LL(0x423d4004,0xa28d071d),LL(0x30014ed7,0x2b51b7b8), LL(0x1e910824,0x51d2e74a),LL(0x049aba24,0x46142c53),LL(0x1c4c9bfc,0xeee290c9), +}, +/* digit=18 base_pwr=2^126 */ +{ + LL(0x3d7eba56,0x1432c61d),LL(0x9f3bcbe3,0x3e5748be),LL(0x70d4b4cf,0xd060396a), LL(0xd56cbe10,0x5cf4dd3a),LL(0x8c26eccd,0x6e120e45),LL(0xd44782b8,0x5a349466), + LL(0xece48d34,0x0aa69704),LL(0x72c3e801,0x5ae78e58),LL(0xdbbf3257,0x8cae7cdf), LL(0xad51d342,0x7f33e06d),LL(0x8c54f76e,0x842b3c9b),LL(0xd9738bf1,0x27439730), + LL(0x8d92d2ec,0x606dcdf3),LL(0x47bce9e6,0x80661389),LL(0x20424ac9,0xf9e37e1c), LL(0x1d4308c4,0xb4271df8),LL(0x2524d693,0x32991fcb),LL(0x3eb1eb87,0x1b3e3739), + LL(0xb01d8c67,0x1366f4ea),LL(0x12c079e1,0xfc0e2aaa),LL(0xc73eb944,0xde8828da), LL(0x4fb7452a,0x1be389c3),LL(0x3ab20ed9,0x1b47de61),LL(0xe88a74b5,0x0a0e4dd5), + LL(0x9f3a0e52,0x42086dea),LL(0xdea9b529,0x2b27362c),LL(0x0f6b324a,0xa601b101), LL(0x8d8999b3,0x50deb5db),LL(0x0f97d636,0xb5146466),LL(0x7452ad66,0xe400edf5), + LL(0xf40e2c2a,0xcea8e0f5),LL(0x5c7a6851,0xd7a70dcf),LL(0x3c3eef95,0xb07f27bf), LL(0x6b31486d,0xebeaedec),LL(0xae6fd833,0xa6218332),LL(0x4c2f458c,0x19ce8eaa), + LL(0xb1bfa9ec,0xfaae66f3),LL(0x2742714b,0xc7b516dd),LL(0x8f55fcc6,0x6bd999fa), LL(0x1b07a49e,0xbdf9e3dc),LL(0xdb922eb3,0x8c8fe856),LL(0xe3b05757,0xf0e446dc), + LL(0xcab82ecb,0x12bfbae5),LL(0xe279d4d5,0xae24cbb6),LL(0xaaa064ed,0x2ae08e15), LL(0x053194f4,0xe4f4d9f8),LL(0x4fb349c2,0x66627f60),LL(0x7d673df8,0xa57270c4), + LL(0x8de75f76,0x79c8e28f),LL(0xb5103e06,0xc46d7632),LL(0xb99c0669,0xee23de7a), LL(0x112100cb,0xb6a05082),LL(0x559c6fe6,0xd3aaf2ba),LL(0x310d9cda,0x1fa98099), + LL(0xd68c0cdd,0x66b15c32),LL(0x316bb565,0xc1a8dcd1),LL(0x66388e62,0xfe124230), LL(0x6aa0caf8,0x87421a82),LL(0x45ae8aba,0x163ed8a0),LL(0xf8869a95,0x799ecc06), + LL(0xb1f08b18,0x9b5f811f),LL(0x8a8e6e51,0x21df77e4),LL(0x41906e3b,0x0ece8f82), LL(0xb358dbd7,0xcc8f1a7f),LL(0x348f81d2,0x7eae1659),LL(0xba2fc496,0xaeb9bf86), + LL(0xe8ee3475,0x1d24ece4),LL(0x0333898d,0x3a9629c6),LL(0xaa4b4d26,0x51fe2f63), LL(0x45450f91,0x65713078),LL(0xed53b1a6,0x9fe570de),LL(0xccfc903e,0x49d5947a), + LL(0xd12646d4,0xd1a4ee1f),LL(0x108a021d,0x8be0f550),LL(0x0c9a397c,0xadca94e2), LL(0x7a116b52,0x443e40d2),LL(0x4a7e4f8c,0x8922a749),LL(0x11b9faaf,0x13dd25d4), + LL(0x677337c6,0x23169fd1),LL(0xf4275e2f,0xe8b155ff),LL(0xaa432608,0x199fd451), LL(0xd6a8a707,0x43fae1e4),LL(0x22598f5c,0x30b74dae),LL(0x4412a37e,0x3231c5fb), + LL(0xa6ef8dc3,0x242076a6),LL(0x3531d9ba,0xa91fa776),LL(0x15395cc0,0x77f95454), LL(0xc333f3fd,0x2d82f42d),LL(0xd57091bf,0x0ebcdf5c),LL(0x04a7020c,0xced34ce1), + LL(0x8a8c3087,0x996010a0),LL(0x8731577a,0x2e13009d),LL(0xfb82ff62,0x4a13fc99), LL(0x54ffd5e2,0x03887d55),LL(0x72a2d703,0x81a63ca5),LL(0x61a70095,0xffd0fd49), + LL(0xa9ab8d73,0x08cb1949),LL(0x30fcdafe,0xbcd463e1),LL(0xcb2946e3,0x1bd9c440), LL(0x7b685a62,0x67df0c20),LL(0xe6a0b599,0xc85033b3),LL(0xdad9fc5e,0xc26cb4c0), + LL(0xd189ee56,0x2915b334),LL(0x7df4385e,0xec153690),LL(0x74e10903,0x78c1233a), LL(0x99e6424b,0x1aad36db),LL(0x8e6c1b2a,0xf0985d3c),LL(0x94d7d170,0x2378ac4b), + LL(0x4f4a6728,0x2b339c66),LL(0x682d8695,0xb36bf9a0),LL(0xea6efc7f,0xe5077906), LL(0x8c8272d1,0xbe19ad2b),LL(0x80ba118b,0x2273dd26),LL(0xbc2ebef4,0x1afaebc8), + LL(0xd967a41e,0xebfd1e2b),LL(0xc3fded13,0xf89bd7eb),LL(0x5e968c2e,0xfa2e6197), LL(0xd2842061,0x75e0b500),LL(0x81e69daa,0x484219aa),LL(0xe5989ba9,0x3a9fc283), + LL(0x827deaa4,0xc54392cb),LL(0xf39f1cc6,0x09b5364b),LL(0xd2711eb8,0xccce7212), LL(0x9b8acd72,0xd4aab549),LL(0x0cb6dec2,0x030af88b),LL(0xc5188526,0x9852b073), + LL(0x8119cabd,0x0cd88f26),LL(0x92f36a37,0x09ad060f),LL(0xea4ffb39,0x60e5c4a0), LL(0x44aa9c46,0x29530c7c),LL(0x98565223,0xbc1d4e89),LL(0xd46de9ba,0x1aa44a30), + LL(0x2ac56598,0x33cdcbc5),LL(0xdb40e83e,0xe16fab9e),LL(0xa7fb4b01,0x2e90d12b), LL(0xfe18a04f,0x80c85443),LL(0xf59d5ddb,0x27fae391),LL(0xa121c824,0xd75f7ffa), + LL(0x21d0d7f1,0xacf5993c),LL(0x04b7c788,0x159623b7),LL(0x35cea4b4,0xc631a623), LL(0x36f64979,0xc75060d1),LL(0xde81f49b,0x93d67ee5),LL(0xb1c080dd,0xc8b7ddec), + LL(0x88fc80b3,0x86602cee),LL(0xa52e2b85,0x8d378408),LL(0x7b43597c,0xb66f2489), LL(0x34ead5db,0x8fa5fe75),LL(0x872b559f,0xa91b7b3d),LL(0x969b74d2,0x34007f49), + LL(0x31a991bc,0x7ac027f0),LL(0x231f0bd6,0xce3e5a83),LL(0xa724434a,0x6059c8c4), LL(0xbf199050,0x3949f4fc),LL(0x50f4c59e,0xc8d1ce42),LL(0x2c2e840e,0x406e976e), + LL(0xff33a74e,0x70c69777),LL(0xbce579fb,0x0caa3c59),LL(0xe3cc09ca,0xcba746d0), LL(0x7493a1a1,0xe544dacb),LL(0x74ebeba1,0x84d251ac),LL(0x7b5a26cb,0x24c351ad), + LL(0x438babd3,0x882cde8d),LL(0x4a0c3725,0xc88a8ff9),LL(0xa80a190f,0xee17ee44), LL(0xb11442f3,0x50337a67),LL(0x7f7eed2a,0xc6803f55),LL(0x36750d2d,0xee2e0425), + LL(0xe5eccbeb,0x9614df9b),LL(0x6d8a00a6,0xa42fa89b),LL(0x5cb36d70,0x90d5f6eb), LL(0x06790958,0xcc2994e9),LL(0xbda37f56,0x4c4820a0),LL(0x109395c2,0x4ac16bf5), + LL(0x4d5e93c7,0x57f12bfa),LL(0xae11e7bf,0x6f57c3c7),LL(0x805ee169,0xe0c53938), LL(0x27da73c9,0xb1ea61d1),LL(0xf1ca8fcf,0x812387b6),LL(0x9aaad2cb,0x812bbf9b), + LL(0x673f3249,0x103ab65e),LL(0xf91c5bac,0xbccd32cd),LL(0xb7502b99,0x2cd17117), LL(0xc6ee643f,0xacbc6b98),LL(0x7e3a9007,0xe45beed2),LL(0x39540d2d,0xd6e7cbb3), + LL(0x735d7a7e,0xf5ba275a),LL(0x939bb080,0x7d56e5bb),LL(0xa316d8f3,0x632ae0df), LL(0x44834265,0xb16f0da3),LL(0xc0e774a9,0x2ac41378),LL(0x963763b2,0x2d1368a7), + LL(0x2910cb9b,0xbb909572),LL(0x7ec8764f,0x9b97d7d3),LL(0x0a1f7187,0x45ed9127), LL(0x91607114,0x4911eae6),LL(0xbec9af64,0x5a3283f3),LL(0x24181e7a,0xfad12d4a), + LL(0x304aa2f6,0x7130416a),LL(0xa93ea2a4,0x3d8ec32a),LL(0xb42283de,0xf746be9c), LL(0xfaf3861d,0x122fff14),LL(0xfc5c58f9,0x85c1a762),LL(0xa1051cf0,0x096434ec), + LL(0x7f5300f5,0x991bef56),LL(0xdb18ccbb,0x8a974cb7),LL(0xd51e5e8d,0x95e6464d), LL(0xb24414cc,0x7891a137),LL(0x78ca6881,0x614852b1),LL(0x11512b7e,0x8ce344f9), + LL(0x14180a17,0x225c5475),LL(0xe0402e95,0x20f7198f),LL(0x7dc14a78,0x834328aa), LL(0xf00864aa,0xf9f3f9d9),LL(0x5c929f4e,0xc14ac9fd),LL(0x67caed5a,0x565edd57), + LL(0xa3116bfd,0xc3febfae),LL(0x25b0091a,0xd28b9297),LL(0x43d43e9d,0x71940761), LL(0xa6743cde,0x0b0dcd52),LL(0x85916ef5,0x0d9c9307),LL(0xed0ea307,0x16d7fa15), + LL(0xd874f8fb,0xcd73c3b4),LL(0xdd4872db,0x89f049a0),LL(0xd6c574e0,0xa3c1e344), LL(0x1501c2b5,0x42aa48a1),LL(0x4379b187,0x08c07386),LL(0xdb94daef,0x651b8ea0), + LL(0x2a46e40e,0xb29008a4),LL(0xafe5ce10,0xf8cfacfa),LL(0x1f36cf21,0xc20de58c), LL(0x77586b4e,0x1667beb5),LL(0xdb697901,0x549fbdad),LL(0xbec7838a,0x05d84e16), + LL(0x86573220,0xca40cc00),LL(0xce72bdb3,0x2b43ccde),LL(0xf187d609,0xd8d285d3), LL(0xcbb90bdf,0x975d5583),LL(0xdb3ce7c8,0xd8e1edb1),LL(0x1be6a0d8,0xf24a6f87), + LL(0xc5db734e,0x72d204bf),LL(0x1ead06dd,0x4b794609),LL(0xd899503a,0xab7e900a), LL(0x808698fa,0x682a67a7),LL(0x81d2be81,0x8d5f7a3f),LL(0x6db17983,0x5ac7f636), + LL(0xf5d8bc59,0x58e88381),LL(0xbc6677a0,0x0c14b7eb),LL(0x75f0beea,0x26d7fb70), LL(0x3c789c7d,0xf71d6c28),LL(0x31a64d6a,0x0e1c3ef5),LL(0x3a025a05,0x930f15cb), + LL(0xf84e9a66,0xa848b469),LL(0x0a6acdfb,0x8b3620ef),LL(0xc0f3c653,0x803dc9e4), LL(0x70d4aaad,0x87b9452d),LL(0x2e5341df,0xd2039635),LL(0xcee3e654,0x1518ec72), + LL(0x75e30599,0xd89bb62e),LL(0x358eb463,0xffb8be40),LL(0xac2b9c1d,0xcc7b8c90), LL(0xbe967747,0x14076fcd),LL(0x9e9118cc,0x4c590a0d),LL(0x7fdedab7,0x255af971), + LL(0x0bf36a0e,0x7cb0703b),LL(0x2f6e1ad6,0x9d4f60d3),LL(0x80851e9d,0x0ddfa57d), LL(0x320a950b,0x0d7e34d2),LL(0x3cf9b367,0x344c00bb),LL(0x7f5fb665,0xbf5c2104), + LL(0x5c85310d,0x01355f1f),LL(0x430840fb,0xa17a28ca),LL(0xbddb8e53,0x1573b14f), LL(0x1be8bb25,0xb6e9f88e),LL(0x68f882ed,0xf8283ea2),LL(0x9f8a6c36,0x97b1f85c), + LL(0xf96965eb,0x16d8df1e),LL(0xc2b9fd6d,0x347159dd),LL(0xc973dc04,0x5a8ff74e), LL(0xe385ad6d,0x6b89c04a),LL(0x76c7bedd,0x1604bc1a),LL(0x09914658,0xc1e216de), + LL(0xe5d7f3b0,0xe285c4b9),LL(0xde0ec465,0x340ea40f),LL(0x51eaa759,0xddc7e286), LL(0x2a2118e1,0x3d1f99a9),LL(0x99fa6ada,0xf8dbccb1),LL(0x2e831534,0xdcb7d5d1), + LL(0xa33e53ea,0x90c5048d),LL(0xc03b6f58,0xe68e4543),LL(0x0585ccce,0x2c9860a2), LL(0x15f92a46,0x948b7875),LL(0xa62a0f0f,0x913f01ea),LL(0xa35eeb9b,0x06623f96), + LL(0x9c77ae5f,0x403740c1),LL(0xd975126c,0x25df8030),LL(0xaf1bd02a,0x762b3d5f), LL(0x7a17ca9e,0xa9ef424f),LL(0x5bbdefcb,0xf0e425ee),LL(0x6ed00a8b,0xafa22cf9), + LL(0xfdcc0852,0xdcb9848b),LL(0xe4e8b45d,0x22851be9),LL(0x195c527a,0x25543398), LL(0xc4ffcb5c,0x39a1709a),LL(0xc84717ea,0x9d6bc1dc),LL(0x3080227e,0xc5a9436a), + LL(0xafcc481e,0xfcc2dbcb),LL(0xe8bdacd0,0x936af760),LL(0xa208d23c,0x37762746), LL(0xcf4cfb3d,0x7c32898d),LL(0xbe8d86f7,0x4c7103db),LL(0x29137736,0x0e8e29e9), + LL(0x5fb1f4df,0x4dddcd9d),LL(0xf8e40af7,0xcee0b177),LL(0xd5ed5e8c,0x2d5eed04), LL(0x4436fde2,0x8c57cd9f),LL(0x59431816,0x6d9ae88a),LL(0xfcd05844,0x410d80be), + LL(0x4f2e214e,0x72cb7d76),LL(0x50149bdc,0x84aecdf4),LL(0x29a10599,0x55c54af4), LL(0xff356cf8,0x15b4a13c),LL(0x94c1cdb7,0xe251ef48),LL(0x3c294aa2,0x3d868b7d), + LL(0x38ac42ad,0x61562246),LL(0x9857cdde,0x35bea87c),LL(0x3bff3a31,0xe288cd86), LL(0xfeff112b,0x6de40901),LL(0xfb86b4fa,0x4c3ff10e),LL(0xa2760141,0xefb2ebbb), + LL(0xb651d24d,0xfe675b81),LL(0x1bf57d70,0x58decfa4),LL(0x3e407d11,0x6d577314), LL(0xc363e64b,0x99100a53),LL(0xd5b05ad2,0x3700486d),LL(0x3d16a4f6,0xd725f452), + LL(0x4369d7d0,0x7987446d),LL(0x082a8d9a,0x6f73c9d4),LL(0x90c45db8,0x50f2643b), LL(0xa8b8a26e,0xc0547960),LL(0x0419c5f9,0xcf0cc729),LL(0xa88c242d,0x36ec0c68), + LL(0xab61869c,0xc39acd0f),LL(0x52d501d6,0xb3676312),LL(0xe963c20d,0xcce446cb), LL(0xd5427311,0x171ef8f9),LL(0x0a43e236,0xd2250f20),LL(0xbf664fa3,0x3734c3a6), + LL(0x3a865bc1,0x4debd2e7),LL(0xbffefe51,0x48d3b6e2),LL(0x39246490,0x63d793f1), LL(0x07251f37,0xf2e568cb),LL(0xb86ac8ba,0x0543ce7c),LL(0xf0233d6d,0x34c94bb8), + LL(0x6b22f5d4,0x825d20cc),LL(0x87e8b87c,0x3008f0d6),LL(0x6fe24ecc,0x1f179a44), LL(0x3732b8f4,0x45b86fc1),LL(0x5bf33be2,0x280c34e0),LL(0xacc6b35b,0x3656ee7c), + LL(0x3de8e3cf,0xef5aea9a),LL(0x13295f9a,0xdaf53c2f),LL(0x37faa7d5,0xef4cb920), LL(0x7b389099,0x7cc42259),LL(0xf4bbc5ab,0xa229e4b0),LL(0x369283c9,0x75a6dd11), + LL(0x208de900,0x2472c7bd),LL(0xfe199477,0xa707b3a7),LL(0xe84304aa,0x4eeabcfd), LL(0x69fb0f07,0x8a125707),LL(0xcff32ccb,0x8146ef85),LL(0x6874ae86,0x924e6627), + LL(0x5baf5149,0x7ebfdd60),LL(0x6fd26802,0x060d10ff),LL(0x433383ae,0xed4cab01), LL(0x6ae53b2f,0xf6529a62),LL(0x17680c05,0xfb786ea5),LL(0x6a368e73,0xe22d7ea8), + LL(0xd5b48338,0x52e49947),LL(0x0b58cffa,0x70575502),LL(0x3097c699,0x0f568fcf), LL(0x2ebb3f51,0x3a78100a),LL(0x6ebf4f52,0x8ca651ed),LL(0xaf034c4a,0xc21be721), +}, +/* digit=19 base_pwr=2^133 */ +{ + LL(0xb439db86,0x318e0141),LL(0x6ba0c36d,0x24dccf9d),LL(0x278f654b,0xe30ecdee), LL(0xcc96cfd2,0x3b53ec8b),LL(0xffdef7f9,0xb15365ab),LL(0xb77c2bdb,0x2886538d), + LL(0xdd6dd374,0x16beccc0),LL(0xe0de912b,0xab022ba4),LL(0x9fa6c113,0x0fa1e066), LL(0x5602f255,0xa62f3440),LL(0xde5080c3,0xf1744743),LL(0x959de74a,0x5dc10f57), + LL(0x30bcd508,0x3fea4a00),LL(0xd7784a23,0xc4b99502),LL(0x22777f72,0xc2aae8e2), LL(0xb738b26a,0x0998acd7),LL(0x284a4687,0x01c21758),LL(0x60aa1544,0x86331b2c), + LL(0x044902c4,0xb138ec30),LL(0xbbc5aa92,0xedbb10a5),LL(0x8cf968b8,0x78dab5f0), LL(0x5582adf1,0xc522aa4d),LL(0x757f6d77,0x27b2cad3),LL(0x8496e06e,0xd99cc418), + LL(0x14b10f6b,0x353a03a9),LL(0xd618e82c,0x0e0360ad),LL(0x5c8ba6c1,0xe95b9c21), LL(0xa2aca406,0x420fcef8),LL(0x3d0af393,0x511e7327),LL(0x4d7c2d87,0x240d0932), + LL(0x0431f61e,0x72bc67e3),LL(0x35cbaced,0xe0b8c3f5),LL(0x811b966d,0xec526634), LL(0xedbca0fb,0x994e956d),LL(0xec733f7b,0x1d1d456a),LL(0x10d8156a,0x481983f5), + LL(0x25b3e1e1,0x2d4c36da),LL(0x63bd55c6,0xf81e5e71),LL(0x1d80e9a3,0x8ebaa53f), LL(0x5d2b6568,0xe162ca96),LL(0x58264463,0xae3e0e1d),LL(0xca65ecfa,0x4d50c694), + LL(0x2b0d1ddf,0x9dbc08f7),LL(0xabf627f7,0xe9e345da),LL(0xa7badd66,0x110cebe6), LL(0x48702def,0x8055893e),LL(0x987a98ba,0x04470e1b),LL(0x1d380312,0xc27475da), + LL(0x9fea7b29,0x8a275184),LL(0xcaf14cdd,0xdc9a9a94),LL(0x5aa56285,0x6c8859b5), LL(0x242a5eee,0x9eede9fa),LL(0x0b4d4c44,0xb738b0f2),LL(0xda57c6cc,0xcd8174f8), + LL(0x3cb5df38,0x32b55a3c),LL(0x0cbf8de1,0x2bf1c097),LL(0x9b7812a9,0x1f27c2f2), LL(0xbb4241cd,0x41cfb699),LL(0xb8cefac0,0xf25fb9ad),LL(0x4b462fe6,0x0e8ea882), + LL(0xb2fa7ed3,0x6d1fe0cd),LL(0xb805f909,0xa5afec06),LL(0x50929ff7,0x7114c77f), LL(0x41e73d2a,0xb70a0449),LL(0x412afd6a,0x17353c86),LL(0xe777054a,0x4b1ab341), + LL(0xb6b4736a,0x205f30a3),LL(0xec96f97b,0xdc57c0dc),LL(0x50892f3d,0x6026e016), LL(0x9872ee46,0x5587a900),LL(0xea0f599b,0xfa2e40d1),LL(0x5698483e,0xd0dfe28e), + LL(0xc978c548,0x96ea419d),LL(0xd41fdcca,0xc78106e5),LL(0x2c3d871f,0x1fcd9bd6), LL(0x8f18504e,0xde3a3b13),LL(0x147e11bb,0x41af768d),LL(0xbea88154,0x57db2d46), + LL(0x359605ca,0xa03b50ed),LL(0x8d2e6f9e,0x102d624a),LL(0x021fbaee,0x75846c7d), LL(0xfde66141,0x7ce93a17),LL(0xcb75846b,0xff36566b),LL(0xdfe8687b,0x13f15dbc), + LL(0xb6900145,0xa1762cdc),LL(0xd04f6298,0xc52758d8),LL(0x05fb29ed,0x2388b649), LL(0x952caba9,0xf559602d),LL(0x612d83d0,0x7a57aa1c),LL(0x798f6741,0xb0a70b52), + LL(0x718b8cfc,0xd977e424),LL(0x9724fb0c,0x0bd2e7ad),LL(0xb4b77d96,0x62055643), LL(0xf3b93b52,0xabe7d1ae),LL(0x29f1f8a5,0x2573a99e),LL(0xf1ae95ae,0x9d861485), + LL(0xf37551bf,0xff5d9174),LL(0xeed4fbcc,0x2120b50e),LL(0x61f2e529,0xe442a255), LL(0x2f0b3221,0x1991c2ac),LL(0x598ea6d0,0x39e9fba8),LL(0x38f4004e,0x3e215ecb), + LL(0x7112f48d,0xab373c03),LL(0x0cbf0146,0xb36e09a1),LL(0xf8d04ac7,0x750d400d), LL(0x1a257ba4,0x7a522a7e),LL(0xa6fe3d3c,0x19d94e30),LL(0xab24c6d5,0x8f563b6d), + LL(0xce3f76db,0x4e8f7ae4),LL(0x951e6883,0xf041cd61),LL(0x677e20c7,0xd5346ef4), LL(0x4f2fa361,0x2ecde00a),LL(0x24e52b5a,0xda39ea83),LL(0xa4037195,0xf13a1d6a), + LL(0x8122da8e,0xc4ef50c0),LL(0x1e4dfe54,0x44245ba2),LL(0x549eb736,0x51a3afe6), LL(0x149e53e2,0xb389a041),LL(0x019f7f33,0x59e1806e),LL(0x207e8b03,0xf0bf62ca), + LL(0xf023e16c,0x2afa74f0),LL(0xe45110c3,0xc0e3985f),LL(0xef4dc058,0x30d3c844), LL(0x0932e063,0x74a8fe78),LL(0x8d089afd,0x1694b37f),LL(0xc02cfea5,0x688b8d05), + LL(0x00b00b60,0x82d73a13),LL(0xec1be174,0x9914b5f8),LL(0x260cca25,0x4b3aa37e), LL(0xdb9eedcc,0x80edd7e4),LL(0xfac065fd,0xd734f788),LL(0x95ca6a6d,0x3bad348e), + LL(0x394ac74a,0xcc754ddb),LL(0xfdf2392c,0x499071de),LL(0xbcd3b3a9,0x64980287), LL(0x8ac174ca,0xf9ad94b1),LL(0x48adbfcc,0x7d8487b5),LL(0xeaa22b97,0x3b94b5a7), + LL(0xd830a7f8,0xdd6a5e09),LL(0x0a707d6b,0xe88ab4ee),LL(0x6f282e77,0x65064114), LL(0x05630c1d,0x732b3f5f),LL(0xa32278e4,0xedc7f608),LL(0x40aa906b,0xf9cc5a48), + LL(0x84cb7f2e,0x22cc8910),LL(0x492f6b49,0xe72c3a7e),LL(0xd9172542,0xcdfb7abb), LL(0x5379490b,0x332ca313),LL(0xc019dc08,0x74e34e7e),LL(0xffac012c,0xae01f427), + LL(0x6589755c,0x28a5ee38),LL(0xf5484971,0xc3c09b28),LL(0xebdc2e2d,0x98c22f2b), LL(0xb739eaf1,0xf34f7446),LL(0x40de5322,0x71c9f8f5),LL(0x8cf40eb6,0x94ace574), + LL(0x660ffcd8,0x14a05e05),LL(0x384c2a8e,0x13f0de7d),LL(0x192dd045,0xa52ddb4d), LL(0xea4aaf10,0xf4d36e86),LL(0xe77d2f45,0xfd9db24d),LL(0x904e7b10,0x55f85a8b), + LL(0x260ee499,0x315e7fcb),LL(0x15cbbecf,0xa3bb055e),LL(0xfda09d93,0x2634a003), LL(0x71c4d7c5,0x5b3c4fe2),LL(0xc9380c8d,0x6805f34b),LL(0xc8a539f9,0x587eeb71), + LL(0x51635be4,0xdd9571b7),LL(0x9e7aabb0,0x321a0e22),LL(0x55781706,0x8b2b23f5), LL(0xaea3c730,0x7cca33ac),LL(0xf95cb480,0x3f6401d1),LL(0x26a55858,0xa5ef93a5), + LL(0xc8b7c736,0x1eaf9f4c),LL(0xeb5b2b70,0x1dcb1722),LL(0x65aab50b,0x0d948725), LL(0x15e7c8ef,0x193c8d06),LL(0x6c391c08,0xf7745835),LL(0x897265d5,0xcbd15810), + LL(0xb54eb895,0x205436fe),LL(0x059b35b6,0x4300c84f),LL(0x9d231cce,0xab267da3), LL(0x20fa381d,0x25239fea),LL(0x004d10be,0x5781e9de),LL(0x14641496,0x36c78b47), + LL(0x99963302,0x0c2bd51b),LL(0x6afdcb5d,0x1958d45e),LL(0x25a00883,0xaa86d186), LL(0xe705abde,0x14852c1f),LL(0xc056e6f3,0x352304ed),LL(0x83c10ff8,0xd05eedf0), + LL(0x702067ff,0xd0cd6dc3),LL(0xd3d111ac,0x3ee83612),LL(0x5ef57788,0xad67877e), LL(0xff6b4f08,0xda249b82),LL(0xd696f4f4,0x9dbbe132),LL(0x51804942,0xfe5ba023), + LL(0x060154d9,0xdc19d20b),LL(0x522b27ce,0x2c6233ef),LL(0x20fde2af,0x8b235bec), LL(0x3bb960c1,0x2adcfe01),LL(0x603b8fb9,0xc79aa84e),LL(0x6b460288,0xe204af13), + LL(0x21d24338,0xe9c83cec),LL(0x644ee8c2,0x768ea11b),LL(0x94fcc664,0x59088ada), LL(0x9374c03c,0x900c7bb8),LL(0x43cb2818,0xad48000f),LL(0x9145786e,0xa2041cca), + LL(0x02582993,0x1765e1a6),LL(0x34c86e0f,0xe74f0372),LL(0x2bfc967b,0x5ba6c193), LL(0x154fde15,0xa09688b9),LL(0xd02973ed,0xcc002cb5),LL(0x92f64238,0x01037f55), + LL(0x8597bf52,0xcef12f0f),LL(0x323d2edc,0xd1bbb357),LL(0x6cac9c11,0x06254d2f), LL(0xa913fbcd,0x13b6a932),LL(0xa97bee41,0x9865d3ba),LL(0x224479fb,0xab2a703b), + LL(0xbe515d1a,0xfd4472bf),LL(0x1dc983d4,0x7ca21984),LL(0x2a2db967,0x1f62c4c0), LL(0xcf71a671,0x4bcade52),LL(0xdfadf5eb,0x50fdda39),LL(0x650fac39,0xb020e598), + LL(0x6d401575,0x05292bde),LL(0x1448bd42,0x0c3c2b31),LL(0x526536c1,0xa76d46cd), LL(0x2a839d2c,0xaa0c74ff),LL(0x62f94ad3,0x7283bfb0),LL(0xe710c108,0x063159c2), + LL(0x11840343,0xbf96687c),LL(0xd26b7f84,0x36468056),LL(0x362582a6,0x6ab84c43), LL(0x7d43997a,0x3bb7a3cb),LL(0x19bc5d68,0xfafa8874),LL(0x042ff6c3,0x73dd38cb), + LL(0xe5d2eb17,0xab12eac0),LL(0x07bac547,0x4b8a41cb),LL(0xffa6df36,0xe9bfd303), LL(0x3c9a6423,0x6bb4517b),LL(0x9ca20f86,0xd393d1f0),LL(0x2231716f,0xb8e17b6c), + LL(0x35417747,0x4a3a2eeb),LL(0xc8c6b7c4,0x9b638e84),LL(0x16cbe067,0xc1215698), LL(0x322fa78f,0xfe96edae),LL(0x35c0584c,0x7b161bc3),LL(0xb32db702,0x86d59110), + LL(0xdc2fd5f6,0x232b8b24),LL(0x3c7926e0,0x082d2045),LL(0xb2c6285b,0xa73041dc), LL(0x8dcded40,0x60172712),LL(0x04a70ed1,0xc62c9174),LL(0x962a378d,0x6fb37121), + LL(0x823c3d77,0xb6f17b32),LL(0x9f7e5c97,0x5c45e3a8),LL(0xc3396a05,0x2daa3c77), LL(0xb063c7fa,0xbb2c63d7),LL(0x2ce90d3a,0x7a1c8227),LL(0xe4b3c32a,0x109b4b69), + LL(0x35f82f23,0xfdf6efba),LL(0x35381294,0x805d2980),LL(0x35aa14e0,0x894b62fa), LL(0x83d15a96,0x9b0d470a),LL(0x562cbafe,0x16a073f7),LL(0xdd734859,0x7851278e), + LL(0xd678f3a7,0xd0bf1f91),LL(0x5c227a14,0xe3d6d2e9),LL(0xf043aca3,0xd262c755), LL(0xee561894,0x60988c89),LL(0x73ca3bc8,0x64ec6b35),LL(0xc42baa7e,0x69a377b3), + LL(0xc96115ea,0x71695d26),LL(0x5b21f1d3,0x72271453),LL(0x39bafd21,0x8697c31b), LL(0x90197919,0xbd705026),LL(0x24ac9116,0x3bd494e4),LL(0xdfc15b14,0xc80bb868), + LL(0xf71afdd8,0x279ba623),LL(0x033c4ae8,0x9f980a73),LL(0x470c4517,0x21606d51), LL(0xe11ef4db,0x6d179aa7),LL(0xf1ca7162,0x869eac72),LL(0x86deb5b6,0xbc4a973d), + LL(0x200897e5,0x162b8299),LL(0xfccd9352,0xd6d516d0),LL(0xabe7feae,0x77eb73ea), LL(0xc3484609,0x8c10355c),LL(0x3426fefb,0xeea4f0b5),LL(0x1520bff2,0x140ba2d5), + LL(0xa781a4b6,0x8e02c736),LL(0xc1a1dc16,0x42c46ee6),LL(0x40321197,0xcae4633e), LL(0x98f07389,0xf04c64f2),LL(0x037c647a,0x6ee9ba3c),LL(0x48084ba5,0x4b003962), + LL(0xe74b1cd4,0x858e1b6d),LL(0x32cb7b0f,0x1d3abad5),LL(0xed92d1b4,0x5b519e77), LL(0xc30f4257,0x1c92c6d0),LL(0x4987bf78,0x5f383d3b),LL(0x4a7f80cc,0xd1c583d1), + LL(0xb56713fe,0x93af26d2),LL(0xa2ab05fd,0xc1226000),LL(0x7254b6e9,0x841945df), LL(0x5061c175,0xc5109d82),LL(0x00ff256d,0x62d1e3dd),LL(0xf0de138f,0x843b00ee), + LL(0xf374c5cf,0xf615bb55),LL(0x48c4c8a3,0xf0ad394d),LL(0x4d11c95a,0xdc7346c8), LL(0x4c5c8014,0x458a607e),LL(0xa5494d82,0xfe616aeb),LL(0xec4b9fb5,0xb0b346d1), + LL(0x06273f56,0x69c25150),LL(0x0c65034e,0x67589cbf),LL(0x93836375,0xd9a2c967), LL(0xa738e250,0x65e7cf03),LL(0xf4d22339,0x96e92577),LL(0x81400fed,0x4bfc9f81), + LL(0xbe5747bd,0x6e69c9c5),LL(0x2c6c18e6,0xe0472ac1),LL(0x083b884c,0xba3a6bdf), LL(0x52e63728,0x68985297),LL(0xeec09362,0x953f2dd9),LL(0xf975e840,0x3f3a2694), + LL(0x3edfaffd,0x77325f02),LL(0xff7d543b,0xc74420f8),LL(0xf9f15b53,0xb3944991), LL(0x0aac0254,0x86959454),LL(0x3b374e03,0x6f43d3e6),LL(0xb30e1624,0x844012c1), + LL(0x65028b0a,0xd064ea02),LL(0x3b4d2ee7,0x806c09ba),LL(0x7fe4f230,0x8a1f8901), LL(0xeda1696a,0xbd498bb0),LL(0x3bc968a5,0xe51465d8),LL(0x2bd1dae9,0xd28bdafe), + LL(0xf62ea381,0x65982b48),LL(0xcd73058f,0x4807eaa8),LL(0x240c2f9b,0x345875b6), LL(0x65a17c9e,0x778bc20a),LL(0x23054c23,0x6fe407b8),LL(0x75ba53f6,0xfae57b0f), + LL(0x78d45156,0x32fbbcb8),LL(0x71e3fd03,0xaefa78d7),LL(0x0f74090b,0xd7c46ea4), LL(0xcc57cee3,0x89b357ff),LL(0x5840243b,0x9e5cedb2),LL(0x0d0e8ac0,0x98a48d58), + LL(0x610ce375,0xcff03f53),LL(0xc9bfbd54,0x7b3c5728),LL(0xcd327836,0xeb868866), LL(0xae712c4e,0x79819aee),LL(0xd7f04c0b,0x0b11d0e4),LL(0xf8a5a59a,0x95bd124d), + LL(0xcfda2cc6,0x924e7e79),LL(0x212e1d27,0x3ec19199),LL(0xe70982f2,0x596b8e05), LL(0x08cd6128,0xd664a1a5),LL(0xa30c9229,0x374d9700),LL(0xdea4ef83,0x44226abe), + LL(0x49136355,0x9b6659c1),LL(0xc404424c,0x138972a0),LL(0x38bb5edb,0x75c55b59), LL(0x16af7574,0xcf1bc9c8),LL(0x3386b6af,0xaad6e8c6),LL(0x76fe803f,0x5d136ae2), + LL(0x9d03f245,0x3912bca6),LL(0xc3a1fb1b,0xe6ff0efa),LL(0xbb5db166,0x7876e5c1), LL(0xf0b460b5,0x23c904b1),LL(0xe69528b2,0xee8e2bf6),LL(0x5191c468,0x2e8d3640), + LL(0x06516227,0x980138d8),LL(0x861c3973,0x2b71e24d),LL(0x5ca35d8d,0x2580dd9c), LL(0x5c9c05ae,0xac4df0a9),LL(0x3c68c54a,0x63916a87),LL(0xf536c869,0x3a6ffdbe), +}, +/* digit=20 base_pwr=2^140 */ +{ + LL(0xe9cb9264,0x1012e415),LL(0x14ab08a4,0x4f91e85d),LL(0x35df1c1d,0x61431287), LL(0xe79281d7,0xbbd3824f),LL(0xaf54a4fc,0x99d4e30c),LL(0x642e1333,0xc4948701), + LL(0xbae8e3bd,0xe93eda3e),LL(0xf46b38aa,0x4ef30da4),LL(0x4151dbfa,0xd6cab5b2), LL(0xcf5c0161,0x04fd2b14),LL(0x80166092,0x7c8a0389),LL(0xd94e32be,0x47096659), + LL(0x6156b873,0xf7feeab3),LL(0x5be66206,0x4a2f2b83),LL(0xdcfc9038,0xdf3277f1), LL(0xf1d66fb4,0x9b170776),LL(0x076dcabf,0x25ebdd78),LL(0x38b46ab5,0xa7c6ee6f), + LL(0x79f1a500,0x4cfc4785),LL(0xaf098654,0x3ba69756),LL(0xc21ecc8e,0x41491597), LL(0x0c8f0d93,0x4982c841),LL(0x2b324495,0x03c63423),LL(0x697188c1,0x7c3952ab), + LL(0x56b9ce4d,0xe770f45f),LL(0xa9a47842,0x07148130),LL(0xced2b9b0,0xbc38458c), LL(0x047d7534,0xbe545be0),LL(0x1c6ff074,0x6768b949),LL(0xde3dbd8d,0x1dca8c16), + LL(0xcc3bcbc4,0x0636cefc),LL(0x5159e643,0xc92b9ff8),LL(0x91bb0476,0x00a152ac), LL(0x42dea60d,0x3fc87a4d),LL(0x64d6fc10,0x6803f0e3),LL(0xf9f370c8,0x99409032), + LL(0x5826fd73,0xc6616960),LL(0x5517ee73,0xf1955831),LL(0x51c70ff4,0x88b7cec6), LL(0x366d0443,0xb0888a9e),LL(0x3af869e2,0xa9181fd0),LL(0x0a248328,0x26076aef), + LL(0xbaed8f78,0xad53d93a),LL(0xd71de257,0xa0b9be84),LL(0x603a33a2,0x4630f086), LL(0xb911d04a,0x5b44638b),LL(0x6e3fa849,0x02150fde),LL(0x4184d866,0x8421d2e5), + LL(0xe5897526,0x52557cb2),LL(0x502c34ec,0xdfe0933e),LL(0x72eb3dba,0x4c1d921f), LL(0xc524a1ed,0x1989729a),LL(0x18b3826f,0xc702ac61),LL(0x0d5cd8f7,0xaa34f109), + LL(0xfa609590,0xe4ef5cf9),LL(0x6f3d8102,0x215b5177),LL(0xccb2d3ca,0xb1886914), LL(0x65884534,0xcda19104),LL(0x870769b4,0x2cf8baf3),LL(0x7339dc8e,0x22f2d15b), + LL(0x21a810cf,0x35c9c17a),LL(0xfb6bdd26,0x24cd74b4),LL(0x1a7c7f5a,0x6786a13c), LL(0x817609c4,0xec7e4bb7),LL(0x256f4592,0x6de1e630),LL(0x8326a507,0x923f4ca6), + LL(0xb5bfac37,0xbe78c4a0),LL(0x197cb928,0xedbb3fc8),LL(0x8b563167,0x477ac821), LL(0xd224f360,0x70cb3ee4),LL(0x53d0e1fe,0xe1b338d6),LL(0xf10e26d1,0x7a5c3f25), + LL(0x745f0869,0xb0c7846e),LL(0xc667523a,0x50becb24),LL(0x14521b55,0xc6acff6b), LL(0x0c4a3c6a,0x70ed855c),LL(0x8e159fe3,0x1ec2b9d5),LL(0x74b5c29a,0x06249cb5), + LL(0x2dd180d8,0x0f3b2f65),LL(0x7b859a81,0x56200850),LL(0x4b18d95c,0x9d0f51df), LL(0x6f0b29af,0xe82ce6d3),LL(0x2a4310d4,0x75a0fcf0),LL(0xc7bed80c,0xdeebf877), + LL(0x64d180d0,0xcf0d6afa),LL(0xa6434a9a,0xb8f7c1c9),LL(0x8cfb2948,0x6cdb576a), LL(0x863bb933,0xed57f57a),LL(0x4df11d72,0xc7f06882),LL(0x4c2b933c,0xe29a2fa1), + LL(0xa17d0463,0xbba79104),LL(0x5a81b9f9,0xd7cbed07),LL(0x4e242aa9,0x20e07b74), LL(0xbb9e790d,0xb9d12856),LL(0x47c00f37,0x45bdbce1),LL(0x898b8efb,0xa0ed2b72), + LL(0xb466dfb9,0xe8318ed6),LL(0x94a4baab,0x9faa662b),LL(0x051206c6,0x01503060), LL(0x3c19ae55,0xa2ebeefc),LL(0x84d2a9c1,0x210ae455),LL(0x82e43956,0x562df0b5), + LL(0x17080cf3,0x89be703d),LL(0xf1ca3642,0xe83c11cb),LL(0x13271110,0xe61d20e1), LL(0xd21d05b8,0xd631b1d8),LL(0x6452ed8c,0x620fc76f),LL(0xe58a415b,0xcc6794e6), + LL(0x15a94078,0x0a5c8a7e),LL(0x86688f1d,0xd4226082),LL(0xa85bc83e,0x8ede9371), LL(0x0df393c7,0xecd1f5e2),LL(0xd2fc7359,0xc32847c6),LL(0x08190efe,0x1e9fd4bc), + LL(0xeea5827a,0x83dd7278),LL(0xcbc88ba5,0x1ddaadd7),LL(0x218e4199,0x588e4903), LL(0xc8310baf,0xb685d516),LL(0x74bb3deb,0xc9aa64ca),LL(0x931bdaa3,0x0cb7a0ea), + LL(0xcdf5ecc3,0xc1ecf641),LL(0xf8c7946c,0xdeac2eff),LL(0x02ca92e6,0x73c8e009), LL(0x7361770a,0x19bc9700),LL(0xacc0e88b,0x455770d3),LL(0x7ddafde7,0xf7cd145c), + LL(0x3d17e97a,0xa33b896c),LL(0x30a84be8,0x238a70a9),LL(0x044f68ec,0xb629bdd9), LL(0x5614b150,0x6d180693),LL(0x7fde6cd3,0x54ba0757),LL(0x4980c20b,0x19edd13b), + LL(0x1073d079,0x4b1584e9),LL(0xa794787f,0x6e60302e),LL(0x8088839b,0xfeb8358c), LL(0x1a701ad3,0x86f51cbf),LL(0xd07f6815,0x61b9c974),LL(0x653f1425,0x00a0d546), + LL(0x11894381,0xf75ced35),LL(0x319c7001,0x49e325d8),LL(0xa67066ad,0xd20b2435), LL(0x4c608028,0xdd23ebc4),LL(0xba2205ea,0x51d78928),LL(0xfb1fb5cc,0x737b9cb1), + LL(0xc03586f9,0xbbb8ad72),LL(0xd27fea7c,0x402a9fff),LL(0x5dda8a10,0x831e331a), LL(0x1eb0456e,0x6d5a2c5e),LL(0x9f7b81ed,0x65290756),LL(0xbec1e59d,0x976b7327), + LL(0x419baf59,0xea1a80a5),LL(0x57278748,0xbc517958),LL(0x32e85407,0x29245035), LL(0xc0d421ac,0x812b0397),LL(0xd7a01665,0xa9f235bf),LL(0x24652cd8,0x9d164236), + LL(0x7f2c6c70,0xbf6ae2aa),LL(0xee9e0e91,0xa021d9a9),LL(0xf1766121,0x409464f1), LL(0x4a73446a,0xe180eb42),LL(0xc5f86d28,0xe7e1ed8b),LL(0x903583e2,0x2e4a2f9a), + LL(0xc5e91397,0xba644637),LL(0xd74dad5e,0xfc63ab71),LL(0x824f5cb0,0x1baa70ad), LL(0x72783f79,0x5f89e95b),LL(0x3813fcb5,0x45bb2a10),LL(0xf3165835,0xab43fa7f), + LL(0x14c22638,0xb7c86dac),LL(0xdae71747,0x3656ca1f),LL(0xc534e20e,0x85da5ae4), LL(0x49153a00,0x3aa49d12),LL(0xbd68b553,0x34857651),LL(0x47329cb6,0xe4fb4c67), + LL(0xd2659f96,0xf91390b8),LL(0xae3ada10,0x507dcc75),LL(0xea42c05c,0xec3156e1), LL(0x28627de0,0xa851f6db),LL(0x3f5bd276,0x10f1771e),LL(0x29bb086a,0xda6fb38d), + LL(0x72b97a47,0x9bdec367),LL(0x657aba3c,0x033db4d1),LL(0x0a26ba3d,0x6d7dcdad), LL(0xb13b21c6,0x93253eae),LL(0xf3e1e435,0xf49e4546),LL(0xfc1d9651,0xda2f3b67), + LL(0x98a48009,0xf35a51b4),LL(0x74249dd0,0x6d03d805),LL(0xcb813640,0x3a2ea471), LL(0x8fb0ba2e,0xf40ccd7e),LL(0x967a4bda,0xe1511909),LL(0x9c401b43,0x92f26a91), + LL(0x9b338eaa,0xc7019ac8),LL(0x1ff7bf17,0x9d084573),LL(0x565ad4f0,0x64807759), LL(0x0a7bdf41,0x1c42c251),LL(0x9e01efd3,0x63caad5e),LL(0xc624e5ba,0x55785f80), + LL(0xee0bde78,0x7db8ae10),LL(0xb62c8387,0x0e6313ac),LL(0xcd166c86,0xd4ec5c51), LL(0x39448747,0x1a9c939d),LL(0x731d066a,0x3a75230d),LL(0x73962926,0x91bf93ae), + LL(0x61e7a7be,0x1485af16),LL(0x71c2640c,0x112d3d53),LL(0xcbbb3b9b,0x277a24c5), LL(0xfa95ddae,0xa17a1980),LL(0x7a40ef8e,0x4ae12d96),LL(0xfb1c3fd7,0x339601a8), + LL(0x74e8c96d,0x05fc1df3),LL(0x8ad8de59,0x004fd161),LL(0x747f9809,0x0ab94cbb), LL(0x341b687f,0xc689e96d),LL(0x03d6ec78,0xc3ed2540),LL(0xc677b947,0x58414ee5), + LL(0x79d557f9,0xd4b38421),LL(0xb80fd2ca,0xe5f1e680),LL(0x5b2a650d,0x403029fa), LL(0x6ea522b1,0xbc0c8825),LL(0xa186a767,0xcaa2d945),LL(0x254875d3,0xb85ecf45), + LL(0xf6031493,0x098fe2ee),LL(0x95f3b2dd,0xd8a4c0c1),LL(0xf565fbfa,0x84d7151f), LL(0x0d46e2ab,0xab92d636),LL(0x7fcbe589,0x822ee95c),LL(0xd061506c,0x855320c5), + LL(0xd8c4c89b,0x82458f67),LL(0x4a55190e,0xb7f499f0),LL(0x2be3b154,0x8f90b016), LL(0x13d9c5d2,0xf943056e),LL(0xdf50c757,0x7de98cd6),LL(0xc1785279,0x97d69ee8), + LL(0x3fe0c0ab,0x140bc9a7),LL(0x0452da51,0xbf26b0db),LL(0xdad75d1d,0x31658423), LL(0xeb0c6035,0x0f75fa69),LL(0xe800bb3f,0x4cbb16f7),LL(0x66429e7b,0x4892f0a7), + LL(0x4c6c9c18,0x218c66a2),LL(0x08cad414,0x97355790),LL(0xb15e573a,0x3b5b1298), LL(0x0e6834ba,0x7aca61bf),LL(0x3f5ac90a,0x025883d3),LL(0x9dff4949,0x6462025c), + LL(0x701e063f,0x6370b52c),LL(0xfcbed6d5,0x575b74fa),LL(0x54464ddc,0x87196ca2), LL(0x6aded252,0xabf56ac6),LL(0xe859df43,0x195e68db),LL(0xd3b0c609,0xd7d0b724), + LL(0xc7d916d9,0x68fee7bc),LL(0xd2261a1d,0x7c342ab5),LL(0x2d528b45,0x6dc620c3), LL(0x0ae80fdf,0x74e628a7),LL(0xc87865c1,0xb6024eaa),LL(0xb87c19a5,0xcfbcc11e), + LL(0x7fc82526,0x16e10271),LL(0x4e53a7eb,0x1894ff31),LL(0xd6fd2213,0x483dd45d), LL(0xe86573ca,0xf832f47b),LL(0x93e56334,0x3fba3845),LL(0xedd7db47,0x88af40db), + LL(0x48c963ee,0x478c4ff6),LL(0x41711eb6,0xe3c63604),LL(0x40ebd5fb,0xdc23721b), LL(0x38c5e8a3,0x73b730cb),LL(0xa14de57f,0x379e6a4b),LL(0xc789f412,0xed816ee8), + LL(0x0ce5eda0,0x22553add),LL(0xae4119b5,0x2cced448),LL(0xaa185ab7,0x2a352ea6), LL(0x29a2bdee,0x01bd04b4),LL(0xe26e2a0c,0xf76ade92),LL(0xd94aaa8d,0xda8577d6), + LL(0x07d56f07,0xc7f17e2f),LL(0x7d87d713,0xc5e3f9d2),LL(0x20f51bc9,0x1497d098), LL(0xed2a6960,0xf4d19f47),LL(0xc38a9534,0x1c57c5ec),LL(0x18c80df7,0xdbffd9a1), + LL(0x5f168647,0xf0cc9844),LL(0x1593d662,0x2e63ed9f),LL(0x0ac8146a,0xf5fa425d), LL(0xce76be64,0x67f11299),LL(0xf7727692,0xd42902cf),LL(0xf1f0d4a5,0x06ce330a), + LL(0x799a13c8,0xfd425657),LL(0xcf29d9a8,0x2c8a973c),LL(0x1e78973e,0x40ec7ae0), LL(0x78699406,0x4d9856a5),LL(0x08dfdad4,0x7247ba62),LL(0xa468c80b,0xc9427c34), + LL(0x3aeaaf13,0xe79cd000),LL(0x9ec3462c,0x6018f62b),LL(0x2ace3a5a,0x1192c3d9), LL(0x64a64ae3,0x715b684d),LL(0x34bf6e70,0x9d318356),LL(0xfdc656ed,0xa23a9958), + LL(0x77357c1f,0xe6acb14b),LL(0xe9438466,0x554e9c51),LL(0x8ba9a286,0x97a5abd9), LL(0xd3eed3bb,0xfc1cd0de),LL(0x700f6eee,0xf311fa4a),LL(0x8a99cd1d,0x4dd6d73d), + LL(0x6b41a2bb,0x43f68039),LL(0x083ab1a3,0xba3ca572),LL(0x1b173556,0xea841d92), LL(0x928c1fd4,0xb23978ab),LL(0xb985008a,0xd6d7c468),LL(0x2248ceb2,0xa6d40f4e), + LL(0xb9ad7efa,0xc9627a3a),LL(0x6759753d,0xbebc0a32),LL(0x7cf0a0fb,0xa3bacd1a), LL(0xdb7cc4c0,0x4ac922fd),LL(0xd7afe1c2,0xfc69f5fa),LL(0x7afcfb11,0x6dde5c15), + LL(0xbbd42505,0x9df0ef70),LL(0x464d44cf,0xa4274df4),LL(0x57e809eb,0x8113b3de), LL(0xc10dcf7e,0xaeab98c6),LL(0x1e6f2975,0x58a34894),LL(0x8f9ccf40,0x9cb5ea9d), + LL(0xb579d154,0x1c4eac89),LL(0x4b706046,0xafbc0bf0),LL(0x7f6fc44a,0x5ecdfe60), LL(0x0030d612,0x3d9c58e8),LL(0x7811ca4f,0x3d9b1e58),LL(0x78e80d2a,0xd3ace80a), + LL(0x48e70c10,0x5a2c98a4),LL(0x1f1028cf,0xa72ad42e),LL(0x2fe8c440,0x30739ff7), LL(0x5fe64d92,0xe83f9edf),LL(0x2fd3764b,0x7cb62281),LL(0x51ab555f,0x9f114c85), + LL(0xba081442,0x3a838e28),LL(0x27db39bd,0x71ea2048),LL(0xaf2896ba,0x3cdb99dc), LL(0x962c484b,0x00d6fcb4),LL(0xcd7377d4,0xe72a7bcf),LL(0x5164a634,0x3b6505a3), + LL(0x8bddb5d0,0xaec6f498),LL(0x7f02675d,0x92b09fb7),LL(0xa83cefd0,0x79345be5), LL(0x1f828e29,0xea0f3427),LL(0x7cadddfa,0x1ea0786e),LL(0xd159e1b8,0x75ca82d5), + LL(0x62c0533a,0xaaaf5a5c),LL(0xe510bc9f,0xa96e494b),LL(0xf2debee9,0x46b1aedc), LL(0xf98911db,0xedd51f34),LL(0xf9056ce1,0x83a0eab8),LL(0x7eae269d,0xa541abff), + LL(0xd5d26026,0xb68b8dce),LL(0x28bf2023,0x6ceb559f),LL(0x74bc843f,0x7dd76130), LL(0x35f97a24,0x79c99ea9),LL(0x7b478eaf,0xcc786699),LL(0xc10dc8c8,0x7788ee3c), + LL(0xbcba6716,0x88ad66c4),LL(0x95300687,0x2bdf72c8),LL(0xb8fa1b0f,0xdd8629e6), LL(0xe183a10a,0x93d5a8e4),LL(0xae5d5956,0xe3e2d85f),LL(0xddd1b643,0x8b623a95), + LL(0x8cc90e24,0xfaf59c8f),LL(0x7f1c7245,0xb7173e32),LL(0x9baad443,0x568e30bf), LL(0x718cbe51,0x7eb7c7cb),LL(0xc1f0f949,0x2a3a214d),LL(0x96a84ad4,0x21a777be), + LL(0xb2081c31,0xcdb78081),LL(0x7a9d57b5,0x2bfa8d65),LL(0xb750969c,0x9db52d99), LL(0x645d4012,0x6ebcd855),LL(0x86245e59,0xbcd2fcd1),LL(0x812b3b4b,0x8fff22a6), + LL(0x6ffb620b,0x1130ffec),LL(0x22eb38ff,0x279544f9),LL(0x940699a7,0xaeb474fd), LL(0x4c85071c,0x33329bf0),LL(0xaf3d6bd2,0x56b019ec),LL(0x8b3cf686,0xc4aaaf7c), +}, +/* digit=21 base_pwr=2^147 */ +{ + LL(0xac2ec7bf,0x8824e17f),LL(0x9bc85814,0x131057c0),LL(0xdbe08716,0x7e43698b), LL(0xab93ba96,0x42ad5ab0),LL(0x7c518be6,0x1ecf5fda),LL(0x673d6c18,0x1a02e6f8), + LL(0xb3503cee,0xbf0f89da),LL(0x331d9e6b,0x9169467b),LL(0x3b1900fc,0xe3aa5241), LL(0x3f13a573,0xae45a71e),LL(0x1940c9af,0xec133c7c),LL(0x55c2087c,0x0409942c), + LL(0x24e5717c,0xbac3bea8),LL(0x9b83ae02,0xebbc708e),LL(0x7e9ea1f2,0xfa57a0ad), LL(0x704d566a,0x107a38cd),LL(0x85e59024,0x18e24faf),LL(0x0f84a27a,0xec3157c2), + LL(0x58b3bb0c,0x17889e93),LL(0xb4031a98,0xbaa2495a),LL(0xb10778db,0x5d3a3a63), LL(0x8f7c139d,0x2fd443d8),LL(0x5ea5ad03,0x83c71634),LL(0xce562cfd,0xaba81b6a), + LL(0xd98102aa,0x73e43632),LL(0xc1c41614,0x86932d35),LL(0xeec41de3,0x1c66a427), LL(0x72427be4,0xef9616dc),LL(0xd48ea6d2,0x206de291),LL(0x8160c0e5,0x755f3ab3), + LL(0x376fc02d,0x23668837),LL(0xde1f643f,0xc4608c43),LL(0x05775439,0x6f04d8d7), LL(0x0f5452a2,0xb32fa6bc),LL(0x3b09e219,0x6a200395),LL(0xfb0a3adb,0xf58e4d28), + LL(0x5f9f2818,0x56285629),LL(0x594adc3c,0x19ba097b),LL(0x0d5212fe,0x6a73f84f), LL(0xb0680c11,0x5106510c),LL(0x13904ea1,0x4d287906),LL(0x8ed4bb25,0x9e4b6db5), + LL(0x3efd07b3,0x5c58a701),LL(0x8a62c9c4,0x5569ea8b),LL(0xe3eb4a21,0xbabcab23), LL(0x2ed936e5,0x9f05b68c),LL(0x25d25b7a,0x5b8fcbe4),LL(0x40745f94,0x5104d275), + LL(0xe96ebb1c,0xbf9a348a),LL(0xadcee8f6,0xde39269a),LL(0x1891395c,0x92012c94), LL(0x6584db0f,0x3b71f247),LL(0x8f7e3a23,0x9fc90016),LL(0xbe7d1077,0xb5f10362), + LL(0x137efed8,0xf06fe20d),LL(0x096f2e2c,0xf037f990),LL(0x416e7f9e,0x7e828e25), LL(0xd17229cc,0xb9066cdf),LL(0x74e2f3cf,0xe1b5d7b4),LL(0xcbc63300,0xa21dc884), + LL(0x73213695,0x09291e46),LL(0x7d591bb2,0x599398e7),LL(0x8693f17f,0xbc316181), LL(0x323b2584,0x29e970f3),LL(0x8253ea0d,0x07db9ebe),LL(0xaf8d9499,0x62d8e321), + LL(0xce2b6bb7,0x468f741b),LL(0x423f1cec,0xb8703663),LL(0xa1385cee,0x045ddc8d), LL(0x9b1abc32,0xfb61bc91),LL(0x8b0a1f38,0xdc221abc),LL(0x5f8c7e7c,0x174b6f77), + LL(0x8aedb4c2,0xf975efc4),LL(0x150dc348,0xb7126f5b),LL(0x8f316668,0x1826a434), LL(0x47b0020f,0x89a0a561),LL(0xddf906f1,0x7c814d16),LL(0x5cbe397c,0xae729976), + LL(0xac73b7b2,0x73507bd6),LL(0xd77d1311,0xdc109e64),LL(0xd07e0b84,0x40745a92), LL(0xeae6ea71,0x80a25d92),LL(0xdba1caf3,0x7d16cdaf),LL(0x178d162b,0x532678a1), + LL(0x89ce3929,0xf04f0f69),LL(0xfccffe8c,0x623a881a),LL(0xc3cc5b57,0xe35d518d), LL(0xd7cbfd70,0xcddb9b25),LL(0x517679db,0xcec3736a),LL(0x3964a183,0xff07c63c), + LL(0x1ee7af47,0xb6f1e8e5),LL(0x774114f6,0xdae8bedd),LL(0xe9b23727,0x7e40fd87), LL(0x4e0b09b4,0x677a0144),LL(0xecabbeea,0x37b376fb),LL(0x05a3a145,0x3d536296), + LL(0xf07be04d,0x0b60e86c),LL(0xb0f7efb6,0x72210de8),LL(0x44de2248,0xe661f7e3), LL(0x5e1a2e6a,0xa50972cd),LL(0xf3c16459,0x8ea7ea6c),LL(0x3d5cc6fa,0xce1eac26), + LL(0x3fb24e2e,0x12057921),LL(0xe703d891,0x2c6418a6),LL(0x85a84ce2,0xba1fdc36), LL(0xd7969f76,0x15c19ded),LL(0xce8a554f,0x0ef1f54f),LL(0x9dae18ed,0x45215796), + LL(0xea05050a,0xe4fb6484),LL(0x249b636e,0xf0ac280e),LL(0xa3b0a746,0x622dc53a), LL(0xc2c4b6e2,0xcbd0132e),LL(0x608cf966,0x71f38ed9),LL(0x47ac6ed1,0x733cdc1c), + LL(0xedbd5f2c,0xeec38461),LL(0xb053878a,0x02367abe),LL(0x366f0019,0xbc7279d2), LL(0x3917f0f2,0x383b04a6),LL(0xd170c6d6,0x7a5ac16b),LL(0x6e32c8f9,0x4f3c4a0e), + LL(0x0bd61a8f,0xcc2c1858),LL(0x34f527e4,0x87c8a389),LL(0xf23dc622,0x618a666d), LL(0x364108ca,0x40fecc5b),LL(0x98a0bf79,0x29742c4f),LL(0x272071ba,0x37d45180), + LL(0x685f2d3f,0x1625e4f4),LL(0x14c1f55c,0x3470dfe8),LL(0x44d91a5a,0x969f3ad2), LL(0x9f30454d,0x6a5be95e),LL(0x7281af65,0xed104a4e),LL(0x36d5ad8c,0xb23076ad), + LL(0x5eb808b3,0x4ed6c763),LL(0x60e8d7fb,0x70f86e24),LL(0xde84f30f,0x2b18c950), LL(0xfaf0ac4d,0x435aaafe),LL(0x3276078f,0x6f3e4e4e),LL(0xb8977a67,0xf3b9b7dc), + LL(0x3bcf4e15,0x28ef1bcf),LL(0x358303c9,0xbe939b53),LL(0x21a319d2,0x204114a7), LL(0xaa81d7a8,0x259ed8fa),LL(0x18fe7097,0x7d9a642b),LL(0x184765b8,0x9acf2e2a), + LL(0x9149e0ef,0x77900481),LL(0x8dfdc1e8,0x0ae5deb5),LL(0xd6f33c8e,0x23b0cfeb), LL(0xa26da632,0x9a6ca40b),LL(0xc537ce62,0x34b9c619),LL(0x5a4ec3f7,0xd37f4428), + LL(0xcbf87088,0x16d03a12),LL(0xb1c521c9,0x872be585),LL(0x1e8e4fc6,0xff44095d), LL(0x5c32815d,0xa7e6bc74),LL(0x8d3db724,0xdf3b021b),LL(0x101b1403,0x6b363726), + LL(0xbe2d31d8,0x4cceb610),LL(0xa171cad9,0x4b4a33ca),LL(0x37cd3a7d,0x8e335bfb), LL(0x5185da89,0xa4922968),LL(0x5914cb63,0x45f929dd),LL(0xa3aa3f6a,0x040366eb), + LL(0x2d373ab6,0x2ae211b1),LL(0x671cfd45,0x81bd0544),LL(0xcb962cdb,0x0c2e8457), LL(0x64ce8ebd,0x58fa7e29),LL(0xcdb6f3cd,0xfdc40b43),LL(0x1c69bac1,0x17d10e47), + LL(0x86421e7c,0x292c5ae4),LL(0x6758a6b1,0x32584cff),LL(0xaf7a9c3f,0x0288b16a), LL(0xeba67e24,0x03aadb8c),LL(0xc87df2d4,0x0f3e777b),LL(0xbb91d9ea,0x6ba2d690), + LL(0x404ff38f,0x4a5352c0),LL(0x5340d952,0x0f04982f),LL(0x2798c5c2,0xcfa86bde), LL(0x94dd1186,0x1ab8b19e),LL(0xccfd8af2,0x83ce68ee),LL(0x06cb2c1b,0xb6f99034), + LL(0x6ecd5c6a,0xadfb667e),LL(0x59a0fb16,0xb5401239),LL(0x5b9e22cb,0xe1100936), LL(0x21d0430d,0x34d54f0a),LL(0xa6197440,0xcb34b278),LL(0xc927d3c9,0x1c07cf85), + LL(0x31e24e54,0xa386ba5c),LL(0xa48e552b,0xdfa72d1c),LL(0x075794d0,0xcb7594cc), LL(0x0b3dc6ce,0xa52aceb3),LL(0xc6d190de,0x16c80954),LL(0x203a1303,0x7cd54e57), + LL(0xd5313dd9,0x72983edf),LL(0x4bf35d2e,0x7557de74),LL(0xfde38dd0,0x29719af0), LL(0x6838b644,0xe3ce6ea4),LL(0x39486a01,0x70976ef8),LL(0xb7d8d84d,0x7b16d6b6), + LL(0xc80b2253,0x44cffea0),LL(0x53e978f5,0x39b12203),LL(0x76e654ca,0x6efe5c6c), LL(0xf5ecd362,0x9ce55175),LL(0x32904094,0x945bc283),LL(0x419b4c47,0x16be694e), + LL(0xb25ff2d8,0xc6d695b8),LL(0xfbc1c346,0x95848769),LL(0xf78df5b6,0x4cd68253), LL(0x07a395a6,0x366208a9),LL(0xb300e26e,0x6a100464),LL(0x2c4c9774,0xcd75f9ef), + LL(0x58587dee,0xa6dde508),LL(0xaba27af9,0x1fb5756e),LL(0x2435fdfc,0x886588cb), LL(0xafa3a40b,0x8836939e),LL(0x23550a04,0x8e4ea6e4),LL(0x13129070,0x1f60a305), + LL(0xafd87664,0xcd87217f),LL(0x62eaab37,0x33999750),LL(0x63e77715,0xc5c71386), LL(0xadc2dc58,0x8f7ec765),LL(0x06ec10e5,0x0166d81b),LL(0x4bab2eec,0xd978b136), + LL(0xabc9d990,0x518d8792),LL(0x848bc38a,0xb5e9214e),LL(0x890c38ca,0xbc62bb10), LL(0xa5d74a39,0x24395eb5),LL(0xe81ca0ef,0xbaacb032),LL(0xf968482f,0xaa7d10d1), + LL(0x3c8d1134,0x55fb3866),LL(0xc866fb41,0x2f0c11d9),LL(0x60d9da04,0x470f0d0c), LL(0x30199fda,0x2ce76abb),LL(0xf1754859,0x68239b6e),LL(0x36c8db28,0xabcbfa61), + LL(0x15fd8cdd,0x59bc69c7),LL(0x7dc71686,0x8c35acbe),LL(0xdf56f4b5,0xa864fdf4), LL(0x73cb6096,0x7b019a3f),LL(0x6d0212a9,0x9bec6ec0),LL(0xd403dbf3,0x96f43d54), + LL(0x25f69dd1,0x1f8ef8de),LL(0xd4c811f4,0x01ea5842),LL(0x7bd1206b,0x6b00713a), LL(0xc77c3c64,0xcc267e59),LL(0xe68a4f20,0x788a8c0d),LL(0xf4b3c37a,0x968e777b), + LL(0xf39053e8,0xef230e41),LL(0x7839732e,0x3acdd9f9),LL(0x815c2a4c,0xa80f263e), LL(0x544f6fe5,0x20c61e68),LL(0x2c146750,0xd0a8228e),LL(0x4770e4ae,0x30857c49), + LL(0x42a4aa18,0xcba4107a),LL(0x68f36209,0xe72e3f7e),LL(0xf0a19740,0x42a02b75), LL(0x9946daf5,0xd4783068),LL(0x862f2de7,0xc01f64b0),LL(0x6209f44c,0xf00cfcca), + LL(0x7beb13fe,0x69d1c1f7),LL(0xfa01681f,0xf547c381),LL(0xfe808e67,0xe7fbdc0a), LL(0xc9b56b2e,0xa0fe9ead),LL(0xb2682ce9,0x1c53a580),LL(0x556a1034,0xe6cae9a2), + LL(0x7ff5d034,0xf4497140),LL(0xd0b40f9d,0x95b5f6f9),LL(0xc68600e2,0xec27b942), LL(0xdbb02501,0xd8f6e734),LL(0xf3ced49b,0x1e62a674),LL(0xd9620b28,0xd64cb872), + LL(0x418878cc,0x4506163a),LL(0xa5575e5a,0x96d1754d),LL(0x9a25f013,0xbf853bbe), LL(0x03c3aa89,0xfd8c5194),LL(0x14a549d4,0x0eb1954d),LL(0x0dbe140e,0xa88b972e), + LL(0x476facb7,0x5b45ac78),LL(0xcb7dbe77,0x8dc145b4),LL(0x4e47ee7a,0x0dd76bfc), LL(0xc1569361,0x06f554cf),LL(0xf665649a,0xbd841afa),LL(0x6e9773f4,0xe27421d7), + LL(0xe76c5baf,0x65ee7fbc),LL(0xb8dfe92d,0x6ddd2b14),LL(0xda22cc76,0xb9f4b2b4), LL(0xd9c02353,0x36ffd8da),LL(0xb62c010e,0xc23f7f57),LL(0x3308d355,0x00c51af6), + LL(0xa8fb84f5,0x7cdc4353),LL(0xbd3b9efa,0x431a25d9),LL(0xe921fd8b,0xe357451e), LL(0xa7d0d22c,0x3607231e),LL(0x7ebf0d0b,0xd8d0370f),LL(0xc5ac6f7c,0xfdbb3c4f), + LL(0xc826f1d9,0xfaa73e4f),LL(0x3e954cb4,0x95f6b20c),LL(0xc5935901,0x56f618ef), LL(0xc1141c9f,0x3a55c846),LL(0x27edf041,0xbc5bd1dc),LL(0x0e373045,0x51933533), + LL(0xb5f4dca2,0xacf42b35),LL(0x7eac22dd,0x0eedba21),LL(0x75502f43,0x2945fad0), LL(0x04c50653,0x94995ef4),LL(0x373f4e93,0x7992d051),LL(0x1b1c94b7,0x8a7c3436), + LL(0xaa607973,0xe636537b),LL(0x4e2f86bb,0x1c845ea2),LL(0x81f6999a,0x99f38921), LL(0xfed9173e,0xc1716d20),LL(0x1e801b36,0x792b9ffc),LL(0xca8f7a6b,0x06765f7d), + LL(0x79b19761,0xc6771baa),LL(0x29433d11,0x80d67617),LL(0x6b423d37,0x196fe0e0), LL(0x47bec8f6,0xe752e3bd),LL(0x52fbbe1c,0xd63ec3fc),LL(0x2ebe6309,0x30d727b9), + LL(0x1d17b14e,0xd9f486d8),LL(0x6761b05a,0x47c47d53),LL(0x7041158f,0xdf1f5e86), LL(0xac78e6bd,0x626756b3),LL(0xd777224d,0x01aedca6),LL(0x4a443a1f,0x025d3c43), + LL(0x5634214b,0xa12d740e),LL(0x375af247,0x4bc99161),LL(0xbe3e2882,0x346bf9f0), LL(0x4211297b,0xdad14a72),LL(0x1c532542,0x6554266b),LL(0x15d18e75,0x030daf54), + LL(0x51583335,0x7cdc99eb),LL(0x4e821ee0,0xd2239bce),LL(0xb41ded13,0x8974b1a0), LL(0xfe676f40,0x17dcd329),LL(0x9d22a9f9,0xc3253117),LL(0x36c57e47,0xdd5a6256), + LL(0xeb7827f5,0x04fcaeb7),LL(0x97baafab,0xcbba55a8),LL(0xd29a034b,0xccd2a2c8), LL(0x34045d21,0x37a19e21),LL(0x20ebeb66,0x1319ad06),LL(0xaa2041fe,0xfed9eef7), + LL(0x5ef2c6ce,0xa3fa82ba),LL(0x60524b3f,0x764801ce),LL(0x714ec445,0xdcc5c27d), LL(0x885e5fc2,0x5cdbaa30),LL(0xc779e89b,0x0aa6c9a8),LL(0xb797f8a8,0x9db3b836), + LL(0xa3ddb5c8,0x8c8b5a06),LL(0xc17aa963,0x1904c79d),LL(0x4a40b438,0xd6e2cd45), LL(0xa689e7d5,0x90b1ed27),LL(0x3f834964,0x350a4374),LL(0x1d9e5301,0x4c0cbe62), + LL(0x193559c9,0x9959ff0e),LL(0x3a77311c,0x0a6908c4),LL(0x714d000c,0x93d61072), LL(0xe471b9f7,0xc054c8c1),LL(0xfb602066,0x78534fd5),LL(0x614fcf16,0x9a5f7b8b), + LL(0xc459f1a8,0xd1d42f2a),LL(0xb574b235,0x9403a4ae),LL(0x42ef8a7f,0x1335a414), LL(0x6a4e1b79,0x4eb9a067),LL(0xd7a319db,0x90dd6eca),LL(0x8cfa5579,0xd54bf9e1), + LL(0x72331044,0x063b4a7e),LL(0xc4e13d1e,0xb82357d4),LL(0x38e9b37d,0xfb4981b7), LL(0xee950c17,0x0c64d898),LL(0xa553cc9d,0x16ac1d7c),LL(0x6e7d9643,0x27624002), + LL(0x62dc7931,0x70e5badb),LL(0x3d4149b7,0x92048bb3),LL(0x626d709c,0xa45d15d5), LL(0xe83bc3f4,0xc1e9c751),LL(0x07a26fe9,0x35bb7bd8),LL(0xb8fee30d,0x2713c9e1), + LL(0xc4895195,0xb26a41dd),LL(0xbec62d17,0x3695b237),LL(0x5bd95196,0x272d55e9), LL(0x89b5e05f,0xcb659d58),LL(0xa5687e17,0xcba46e22),LL(0x623cf306,0x2ece876a), +}, +/* digit=22 base_pwr=2^154 */ +{ + LL(0xce363c03,0xd16b25a3),LL(0x69b561cd,0xe21621f4),LL(0xabf845bf,0x03a8e393), LL(0x45c545c4,0xf68a5989),LL(0x492c1276,0x0d669f1b),LL(0x239a2436,0x762cdc78), + LL(0x4e33515c,0x32e5a4ad),LL(0x107349fc,0xfaf120c4),LL(0x42b588b2,0x18f529a5), LL(0xad41f013,0x53afbf6a),LL(0xfce98199,0x7e107d23),LL(0x49fc022a,0xb3e6ca6e), + LL(0xf43c63d0,0x2f0fa15f),LL(0x3c5f1062,0x2ee7972b),LL(0x1251d981,0x217bc709), LL(0x4f44f67b,0x869cc500),LL(0x6e74d49a,0x410ce8d8),LL(0x295b72ee,0xa2668d2e), + LL(0xc2e1ab9c,0x377d1c96),LL(0xf02f335e,0xc4d44912),LL(0x17483b67,0x91a1b453), LL(0x0a35ed35,0xd87bfb05),LL(0x06388074,0xda51aa7f),LL(0x3bc985d2,0xe8eded68), + LL(0x62093566,0x53a65438),LL(0xd40d4ada,0x1a184bde),LL(0xb1ce835f,0xcf398a06), LL(0x2e3db6b2,0x3e0368f2),LL(0x768efcf7,0x5b1e672f),LL(0xce1191fc,0xcde23025), + LL(0x1965b94c,0x7d18a84c),LL(0x42798e4d,0xa92514b7),LL(0xffa944d6,0x6df7a966), LL(0xc415a2b6,0x7c9cf75d),LL(0x0048e5ed,0xac0c7c55),LL(0xe02f7dae,0xd5c826e6), + LL(0xf728251a,0xee934c6f),LL(0xbd48f9f8,0xb7ec512a),LL(0x2ec83189,0xc5f704a8), LL(0xed82fce0,0x3c184301),LL(0x378c8f4d,0x8cbc0d6f),LL(0xa97c2ab3,0xc0ff9d1b), + LL(0xd18c264e,0xa6c6c290),LL(0x768df841,0xe5061160),LL(0x519df852,0x7913aaad), LL(0xf67253ef,0x61958563),LL(0xd45ffadd,0x36c367f5),LL(0x9e16d8cd,0xf1b45f01), + LL(0xec831ea0,0x3d1cecd3),LL(0xfa15690f,0x2bde5c72),LL(0x38b353ee,0xba0efaa9), LL(0xe8d4b6ee,0x15f17a68),LL(0x954159dd,0x9699e58a),LL(0xe8f17b7a,0x819578f2), + LL(0x61ee7443,0xc7eeaf29),LL(0x1b16a9dd,0xfa9c2747),LL(0x45e6c8f4,0xf723daa8), LL(0x80e39749,0x2c263cb4),LL(0xe1796ef8,0x69b29a11),LL(0x41cf69fe,0x71b39ec0), + LL(0xc93aacf4,0x277c0653),LL(0x90aa54e4,0xe7cedf79),LL(0xfbb3e9e4,0x5f3dacd8), LL(0xd0335cd7,0xebf8b3d2),LL(0x54b9f51a,0x320526dc),LL(0xda07d715,0x670feaee), + LL(0xc815957e,0xed31303c),LL(0x9dc47dbd,0xe3fd01ce),LL(0x7796d887,0x69884abe), LL(0x91801969,0x5eeac021),LL(0x1fede637,0x3f7f3b0a),LL(0xb577b6b1,0x94177a5d), + LL(0xeab50a36,0xffccab96),LL(0xc61d85f9,0x0d130c6e),LL(0x7da399fb,0xef804dc2), LL(0xa18f07c8,0x24faa196),LL(0x04fcf0bf,0x64bb949f),LL(0xd5aeb209,0xb1fcfc87), + LL(0x32651bcd,0xf00f320d),LL(0x06c32d6c,0x9df50f13),LL(0x5f7039a8,0xe5521172), LL(0x637e1a26,0x1ea8cc96),LL(0xe1edd4f0,0x2b163456),LL(0x0a1c1b35,0xf5bf574c), + LL(0xe6a6c9d7,0xd11076c0),LL(0x892f314b,0xc2f5ad9a),LL(0x5737733a,0x1ce795fb), LL(0x7ff3d547,0x2ea960a2),LL(0x1b76089a,0x44189130),LL(0xa3a6506c,0x6ef7149a), + LL(0x1032f79c,0x5903c244),LL(0xdcc0e2de,0xf21d72a3),LL(0x1762ce11,0x9ba803fb), LL(0xca5ae539,0x76e31b74),LL(0x9fd4f343,0x4e502c66),LL(0xd2eaa2ab,0xb6a9a1fc), + LL(0x073efd81,0x5888451f),LL(0x9927bf88,0x398711b5),LL(0x018be818,0xbb190a8b), LL(0x79e61240,0xc797d10a),LL(0x3beaa53f,0x8696dba6),LL(0xcfb7766d,0xb542db1a), + LL(0xd903933d,0x6f4e1530),LL(0xcd7c2795,0x12359176),LL(0x5a5fca32,0x755f299a), LL(0xc4a38b20,0xd72339f0),LL(0x842195c0,0x375cc0c0),LL(0x30d23f6c,0xbc55da2b), + LL(0x9aac817d,0xce0ba5f7),LL(0xa1a45388,0x3b669115),LL(0xf409cf17,0x1dedbc8b), LL(0x3ffa552b,0x31a461a0),LL(0x382ba937,0x86758b16),LL(0x795dc73f,0x4a479210), + LL(0x48aba8e6,0x83c44b08),LL(0x62517119,0x17c26d0a),LL(0xd01e1f9d,0x5b018ac9), LL(0xe9efcfe5,0x4397fd63),LL(0xfe0829a3,0xca651042),LL(0x13eb60a0,0x46cee0c3), + LL(0x8600caea,0x27529486),LL(0x35717139,0xd38810c6),LL(0x41461bde,0x63178e7e), LL(0x045c484c,0x227b3172),LL(0x97348e92,0xbcfee10d),LL(0xa51d3833,0x3fde1f78), + LL(0xf10b4b60,0x50b3debe),LL(0xa5ad595f,0x7901e0b9),LL(0x47e58c1f,0x87df2160), LL(0xfa209201,0xad4fd6ef),LL(0xa55d0556,0x9335bbb9),LL(0x00fee680,0xe8d815f3), + LL(0xd99bdf5c,0x3e3ecfe1),LL(0x9bc2df7b,0xcf690949),LL(0x102d61ba,0x7e6008b1), LL(0xe00329c7,0xc1283828),LL(0xeb7b35fd,0x5b105f4e),LL(0x3dcb3985,0x9509ed61), + LL(0x81c63288,0x045a1445),LL(0x76cc9ac1,0xa4e7b140),LL(0x48acdd5e,0xa2f07d10), LL(0xa2b81e15,0x90c45e5f),LL(0x308acd63,0xa933ffa1),LL(0xd7e46f9d,0x17365897), + LL(0x466f886f,0x8852a2bf),LL(0x4d14f3d8,0x4ace368b),LL(0xe440d441,0x8d94e2a7), LL(0x72866999,0xfcb3c915),LL(0x1b47c102,0x9760bf19),LL(0xe720c285,0x6611674e), + LL(0x11d915d4,0xdcb98a2d),LL(0x6beac9be,0xf990fa0f),LL(0x6dd7048a,0x681bfe50), LL(0x689f0ae5,0xb41ece47),LL(0x44ff0641,0xe6fdbd83),LL(0x0e9753dd,0x42e0fbcc), + LL(0x3ffa710c,0xc7877d3d),LL(0x9919eb3c,0x7f0c21c6),LL(0x290c9014,0xc85786a5), LL(0x5689cb11,0x093aae39),LL(0xb5d01c9f,0xfb777943),LL(0x8cebd9c1,0x319aaf33), + LL(0x1a2fd277,0x902b0342),LL(0x9af2dbda,0x0912afff),LL(0x48589a78,0x15e0d236), LL(0x1fa3327f,0xf8b35812),LL(0xec713d1d,0x8990816e),LL(0x8e343f2d,0x5e1a860b), + LL(0x3614d67e,0xf61322f7),LL(0x29935fae,0x4d6b23b9),LL(0xf4a58207,0x5c8bdcc6), LL(0x9265fa5c,0x750d615a),LL(0x40ca78a9,0x2022ff1f),LL(0x19732ba7,0xd9585db6), + LL(0xf29d3bbc,0x694abdf0),LL(0x24a568a9,0x03a757fe),LL(0xeb85bcf9,0x69e64a5f), LL(0x3a6d438e,0x11c2527c),LL(0xe22585b6,0x4aa39cb8),LL(0x20f51f14,0x696f234b), + LL(0xcd374897,0x796f3388),LL(0x747f55ca,0x9cae05cb),LL(0xc3244db7,0x0178bf8f), LL(0xcad92264,0x2cf6b0b2),LL(0xd9851a5c,0xa5eaffed),LL(0xe5613e1a,0x677ef509), + LL(0xb8a7827b,0xd774ccd5),LL(0x30c2fdb5,0x1a9035d8),LL(0x634655f8,0xeed95b80), LL(0x56b0b0bc,0xc2b700c0),LL(0xc9f0aa76,0x37259c87),LL(0x4c9ae4a6,0x7f6b6b1a), + LL(0xf72a765b,0x23bfe902),LL(0x838d2dcf,0xf220ca7d),LL(0xfa8c5fe9,0x6aa1a258), LL(0xdd1fbec2,0x80a3ac57),LL(0xbfd1c7fa,0xa9d70cd6),LL(0x997ee6ac,0x6c5953bc), + LL(0x6a40f84b,0x3b3747a2),LL(0xb83105a1,0xfd1a969b),LL(0xbbcb761d,0x3c5c4b12), LL(0xda6bebae,0x7c7f4e65),LL(0xa14f1495,0x7a4fdbf2),LL(0x49b37a93,0x7f59536b), + LL(0x3240d78b,0x0c50a5f1),LL(0xb32a0cce,0xb51ad026),LL(0x53101002,0x4ccf9cdb), LL(0x0073f0a9,0x2fa22f3b),LL(0xbd1d7831,0x3a7fe366),LL(0x740322c6,0x5f31b6ce), + LL(0x013aca29,0x5cbc9486),LL(0xe542ffae,0x03e60edc),LL(0x8457faa5,0x80539a8f), LL(0x6355b3b2,0xeba394b1),LL(0x2249fcd3,0x0bd1b305),LL(0x4e3f192b,0x5454e867), + LL(0x4416b978,0x78e9316d),LL(0x81c4173a,0xf2a15f86),LL(0xb6731547,0xa6971d20), LL(0x98dbfdb5,0x2c5b69c5),LL(0xfeac68cc,0xf7b1130c),LL(0x0e84299e,0xb1cbec33), + LL(0xc44d578e,0xb354773a),LL(0x712087f3,0xcc55ac98),LL(0xba9554a8,0xd9b75e7a), LL(0xe80e95c0,0x93eb6a79),LL(0x99fb7e95,0x667e67eb),LL(0x14b32858,0x105a22cd), + LL(0x6f70ee86,0x0ded3bd7),LL(0xc1a43b13,0x7e0e5a6c),LL(0xc41e1afa,0x893c3ede), LL(0xeaed250c,0x21e88f45),LL(0x93d3ef9c,0x2cfc36f3),LL(0x7da31990,0x5b4e1c71), + LL(0xc4126c1f,0x1b61be56),LL(0x5abc20ba,0x274763f8),LL(0xed1afb78,0xb49d2c4d), LL(0x3df52e62,0x1bdeb4ea),LL(0xfff32461,0x8dcdd1d1),LL(0xbb1ca9df,0xe3e523f7), + LL(0x809ca4e9,0x4672ae1d),LL(0xdde4d528,0x58d55485),LL(0x8b74e2f5,0x19428043), LL(0x203314f7,0x1745d62a),LL(0xd4c37bee,0x6157bba4),LL(0x7dd72c9f,0x49805c53), + LL(0xa37d788c,0xca78b3b0),LL(0xf475a121,0x17a44e3f),LL(0x1ad2fff6,0xe5c1c959), LL(0x70139e2b,0xd0312c1e),LL(0x8b27ca8c,0x6724a7cc),LL(0xd962035e,0xaf28caf8), + LL(0x9570d848,0xbcc2ea7f),LL(0x7f265e1e,0x05fc9759),LL(0x244ce79b,0xb4954372), LL(0x084d1046,0x164e75ef),LL(0x89adc16e,0x4defb358),LL(0xfa4dc623,0x550e503a), + LL(0x5564a17f,0x370b2b55),LL(0x55a566cf,0xe074512c),LL(0x78ab0d7a,0x110463ca), LL(0x5d2460f6,0xefd6c296),LL(0x9c33b5d1,0x9abe3d42),LL(0x08c3f981,0x4e016175), + LL(0xc21b36dc,0xbf03ca84),LL(0x0b8c07f1,0xf34003bf),LL(0x63eb0039,0xc537b09e), LL(0xb82887e2,0xcacbf643),LL(0xbd3bdbf4,0x9d053615),LL(0x61dda14a,0x5c36ffc2), + LL(0x528c2064,0xe1664301),LL(0xbcae3cc1,0x2e16357e),LL(0x52311eed,0x025d8360), LL(0x2f0a83b4,0x76f15854),LL(0x06dc8ff7,0x10d87d9e),LL(0xf989d477,0xaded98a6), + LL(0xa52127c0,0x3ed6c171),LL(0xa69dee80,0x0847d132),LL(0xd8445a15,0xb2e7e29f), LL(0x71f974e1,0x156e4663),LL(0xe5bd0273,0xd7512685),LL(0xc73100df,0x6f515794), + LL(0xc2ab5306,0x951131c1),LL(0x6c73f4f5,0x89255bf9),LL(0xe69b21c9,0x6ac84582), LL(0x0ab9dce0,0x018b49af),LL(0x7954a835,0x27e5b722),LL(0xb1229414,0x865a648f), + LL(0xc8dd070b,0x388cdc70),LL(0xf52522ad,0x75642aeb),LL(0xcc49bc29,0xe25ec387), LL(0x41ee6cb0,0x214f62c1),LL(0x041d74c3,0xc5ee3195),LL(0x449dbc9b,0x04d43b7f), + LL(0xc7e8f6b2,0xb3a64235),LL(0x323a93a3,0xeeb2edcd),LL(0x4938cf75,0x67e22e11), LL(0xa914b04d,0xd64b0693),LL(0x88b818df,0x688fa54c),LL(0x591d0c01,0xd481be4f), + LL(0xcf3d8ba4,0xaecd7d60),LL(0xa97c0b6b,0xc7a415c8),LL(0xeebbc7e4,0x3839dd69), LL(0xe2ff8cc5,0x0691d0ee),LL(0xe92d7a4a,0x0c253257),LL(0x36ca20eb,0x740e2dd7), + LL(0x2a1939f7,0x5d07cf91),LL(0x5cf3ecae,0x3f6c26a8),LL(0xcb547d04,0x92001c06), LL(0xdf774ad1,0x4175ff52),LL(0x6b5f9c2c,0x7abd12ff),LL(0x4d6d8453,0x1658d69d), + LL(0x40729414,0xcd926583),LL(0x8ebcd09c,0x8a3d6ad5),LL(0x95dc70c7,0x83f72907), LL(0xb5be9edc,0x200031c9),LL(0xd05c8986,0x472b56e6),LL(0x07e0faca,0x9899422b), + LL(0xf88b02ea,0xd7bf0be1),LL(0x9bffc69c,0xb622e02a),LL(0xbc6df9ca,0x1e5b8cb4), LL(0x350dfef3,0x048f54ec),LL(0x0a312a13,0xe1f864f8),LL(0xd6ca4ce3,0xd4bfba4d), + LL(0xb0c052cf,0xc3ff21f8),LL(0xfdfba872,0x19f1a2b6),LL(0x5b694edc,0xd6f10827), LL(0x68a8a596,0x6097aff6),LL(0x954b1904,0x518e1cd2),LL(0x7c51ccd6,0xfd80ae29), + LL(0x1a7e9066,0xbc7a7b59),LL(0x23db5c12,0xcce91b75),LL(0xa3c0a935,0xba7bf0ae), LL(0xda23595c,0x653f64a6),LL(0x08f45389,0x2ae29a0f),LL(0x88589fca,0x724f46cd), + LL(0x3372c3ee,0x45b34ad7),LL(0x985555fa,0x0f609319),LL(0x9f3dcb8f,0x7fd63b9f), LL(0x0bd28eba,0x13a518c5),LL(0xf5706481,0x57cffe62),LL(0xd032721f,0x11718aca), + LL(0x9dd7bfaa,0x32ef81d0),LL(0x7adeeac3,0x3ed2d217),LL(0x6157a51e,0xac2de8bf), LL(0x4d16615b,0x7a7ca0e7),LL(0x217f0fad,0x5bdd6a51),LL(0x099a123e,0xae2064e5), + LL(0x6f3b0a4c,0xd8eb2d51),LL(0x59883368,0x556551c4),LL(0x5f48ad2f,0x6c8195a1), LL(0x62de8d83,0x0a88379b),LL(0xd0b6ecc9,0xaaa811b2),LL(0x83c9d046,0xba49bf0f), + LL(0xe6eed978,0x11f53192),LL(0x41560616,0x0cdd4a81),LL(0x55689932,0x106a49de), LL(0xa791e389,0x79fdb67e),LL(0xe28a67d9,0xba7af653),LL(0xc7aa61e7,0x44db5d83), + LL(0xfee81242,0xe86f2877),LL(0xe05b4568,0x1ebfa60c),LL(0x66c639bb,0x6131750f), LL(0x3da39228,0x2f9b4000),LL(0xb25c8ded,0x8e37e3c0),LL(0xaa3c48d1,0x8c3858ea), + LL(0x37de8469,0x98bf7c76),LL(0x9d051f00,0xd78e0afb),LL(0xc71691e0,0x4553e903), LL(0x539040a0,0x807d903d),LL(0xf73fcb62,0x30fdeee9),LL(0xcc1519f0,0x9c01a657), + LL(0x0b7f4996,0xc5c73b26),LL(0x50750843,0x2ca19471),LL(0x5a92f717,0xe4d8d372), LL(0x4abe9028,0x832ebb12),LL(0xe09a7f03,0x7f0123dd),LL(0xe8c1d6a0,0xa8456dbc), + LL(0x28302bb6,0xc8639947),LL(0x037bebfc,0x1fc1410a),LL(0x9f0b40a6,0x7432946a), LL(0xe06af4e8,0xff9565e1),LL(0x8b9882d9,0x02d05ead),LL(0x13e179cd,0x6e5ca86c), +}, +/* digit=23 base_pwr=2^161 */ +{ + LL(0x4584bdf6,0x5fd3ebe9),LL(0x847693d1,0xd8a4299d),LL(0x40b5d01a,0xc9c0c5d6), LL(0x3afa9eda,0x70b51263),LL(0x805fb55b,0xde94920e),LL(0xb5a46cc6,0xf0c432e1), + LL(0xf6dcaedc,0x0ce0859d),LL(0x3a3f7708,0xa55b2e15),LL(0xc0a8b787,0x200ce3fc), LL(0x3c7bddae,0x9e0f9782),LL(0x0b81aa87,0xe9d43e19),LL(0x4666b8f8,0x9c47c1bb), + LL(0x228ed42e,0xa9c19cef),LL(0xc0249ead,0x22c955e1),LL(0x264c2cb1,0x40d5d287), LL(0xa4070d06,0x420e5723),LL(0xc0604717,0xc2a3bf15),LL(0x4a963f1c,0x7ab3da61), + LL(0xda80e5fc,0x739b4f37),LL(0xcf587883,0xb45a6fe6),LL(0xcadd521f,0xe4a9b791), LL(0x57562a1c,0x27e1b300),LL(0x2fd0f712,0xb04af2a5),LL(0x93409099,0x12cd55e1), + LL(0xad7c2bed,0xe518dc8f),LL(0x65eedead,0x52acef20),LL(0xb2efa9d0,0x4227e093), LL(0x99c38a81,0x8b7ca42d),LL(0x4c509cec,0x30c1a73e),LL(0x51635a28,0x3dc9300c), + LL(0x46be452b,0x92d5ea3d),LL(0x215ba744,0x40841923),LL(0x9930df86,0xd8335b49), LL(0x92c4868b,0x12da53b2),LL(0x60a4383f,0x2400365b),LL(0xe190b71c,0x646771ed), + LL(0xf14687de,0x30e142c5),LL(0x9403fa68,0xad298203),LL(0x2c2dd3e0,0x3e98cfd3), LL(0x5bd48575,0xe2426602),LL(0xc9d41416,0xc7a5a2b7),LL(0x4ade8bbf,0xdcaa3d31), + LL(0x5f6b0fff,0xb8792b08),LL(0xfac1431d,0x17e00f0f),LL(0x479b79be,0xcb5f6f9f), LL(0x32c49e8c,0xbfecbd08),LL(0x654100b0,0x6a6be7f1),LL(0x0890c3b8,0x2e545747), + LL(0xb07447fd,0xa559c912),LL(0x56728d8c,0xb33a2b44),LL(0xf526bd48,0xa027ee4c), LL(0xc58f6af2,0x12d8001f),LL(0x374fa755,0x79e6db7c),LL(0xf25a1455,0x2cbc98cc), + LL(0xd10a0b59,0x597f4f1e),LL(0x84bc0000,0xe166c79f),LL(0xf74bc520,0x5ae7890a), LL(0x8660d37d,0x3303f3aa),LL(0x405c0955,0x2d0ab6f5),LL(0x1f6fea86,0xe98a1550), + LL(0xb1fec6f3,0x58dac0ba),LL(0x0018597d,0xc24ea4df),LL(0xebe0dd78,0x9ec12302), LL(0x2bec23bd,0xfa749d3c),LL(0xd47a29a6,0x91a607e5),LL(0x273be752,0xb0e81d66), + LL(0xb3a95f3e,0x6975c26e),LL(0x1d7290dd,0xcb4ee68d),LL(0x4c6ba170,0x8120bbe9), LL(0x1a407679,0xf428065b),LL(0xc12ea429,0x92b64255),LL(0x56d99374,0xc94c96d3), + LL(0x0103f0e7,0x1bd6a47d),LL(0x1affc26a,0xce0f3821),LL(0x1b702d5d,0x284426b0), LL(0xc65050e4,0xe5ba26be),LL(0x77a391dd,0x29e14db3),LL(0x8ee16508,0xb917678f), + LL(0x0ddf3f74,0x18c4fe3a),LL(0xd8e5db90,0x477e39cb),LL(0x56614560,0xaf43028b), LL(0x8c492bee,0xffdf3688),LL(0xac548919,0x595987ef),LL(0x74dc2b98,0x4d0d250a), + LL(0x3c574a92,0x25fa0aa2),LL(0xfaa1a418,0x89be8ba2),LL(0xb69739a1,0x1b002c0d), LL(0xe3f5b3a7,0xa6b854ef),LL(0x1204504d,0xcaa17499),LL(0xa58cedd3,0xad836998), + LL(0xc4d5d262,0x3cddb768),LL(0x4251792d,0x0a0801ad),LL(0x0d3897eb,0xd704ec49), LL(0x9c91baee,0x0e13d4f4),LL(0x82831726,0x090f2c69),LL(0x76422a7f,0x829b8edd), + LL(0x49398523,0xe83170b2),LL(0x1b62a533,0x253100b2),LL(0x5e36efc0,0xc18431d0), LL(0xc4c34823,0x26325714),LL(0x628d7b24,0x74c3cfd0),LL(0x9d919045,0xa961e996), + LL(0x41d95f75,0xe3a642ca),LL(0xf0a8d5c1,0x4c53ea80),LL(0xa9d3eedc,0x07463073), LL(0xc4e66d00,0xa1439f9e),LL(0xb0694462,0x451a25c4),LL(0x8dfe8fe6,0x713bf4e8), + LL(0xc9e24326,0xae9ef904),LL(0xe8bfd9ba,0xe039837f),LL(0x59511a8e,0xaca967be), LL(0x7acac091,0xd2386b22),LL(0xe9d27517,0x63bdb595),LL(0xce41bbd3,0x51f95292), + LL(0x9ec42924,0xee0ee680),LL(0xb77fdef0,0x4af753cb),LL(0x7cb4d8a9,0x15078aba), LL(0xb4177c3e,0xe19806b9),LL(0x88f090af,0x3e1d4fc5),LL(0xeeb90099,0xc7b2cc43), + LL(0xae6a6f66,0x24b3ffd7),LL(0x308cc05f,0xab2000ed),LL(0x107b362e,0xd89d6e75), LL(0x1fa91ce6,0x835736db),LL(0x390c487c,0x6429da58),LL(0x0d312fc5,0xbaed5289), + LL(0x4a018455,0x0feb44af),LL(0xad578332,0x1af273ce),LL(0x5f625cc3,0x31349adb), LL(0x2d2f3877,0x3848e62e),LL(0x1bfe8dd6,0x620372c5),LL(0x16c9a24e,0xbcc4d459), + LL(0x0a45de42,0x40c8630c),LL(0x30cc053a,0x74488edb),LL(0x119b7194,0x9cdb7fad), LL(0xd629e05d,0x02121283),LL(0xfa4bbe6d,0x727585f8),LL(0xea1f9677,0xc2a6e653), + LL(0xbc14220c,0x4656ea75),LL(0x00b087d6,0xc70a76ac),LL(0x3d83a9ec,0x037ac35f), LL(0xaac1649a,0xf8f1e014),LL(0x4c5f804f,0xaa6de597),LL(0x7d523732,0x8cd45ec3), + LL(0xa04c2d63,0x4540fa7b),LL(0x9eb857a9,0x544d7dec),LL(0xe8f40c8a,0x8ada66ad), LL(0x4598d9e3,0xb8c4fa18),LL(0x516e5110,0xd3ef8a28),LL(0xd56843a4,0xb761d4d5), + LL(0x91f59c5b,0x0aed399e),LL(0x43f5de7f,0xe4868fd3),LL(0xe58c53b6,0x01b524d0), LL(0x712c183e,0x77219f8f),LL(0x036dd938,0x6371e9d4),LL(0xa8cc4985,0xd4de675a), + LL(0xc894130a,0x7da252c3),LL(0x85384ea0,0x9e4041cc),LL(0xeba418c6,0xd0a69993), LL(0xaf043892,0x9a9ebb66),LL(0xc703f8bc,0x9baa32db),LL(0x0bf6fcfd,0xc250e1c4), + LL(0xf56a38f1,0x5913cd6a),LL(0x61ecfa22,0x1e16a4f0),LL(0xb5c22ada,0xeb036c2e), LL(0x95892d2d,0xca1a39c5),LL(0xd31c9856,0x4688a1cc),LL(0x29e69ad8,0x3a6ea062), + LL(0xda5d2bab,0x283f3d5c),LL(0xa82a537d,0xc4f0d483),LL(0x81e8516a,0x953f037a), LL(0x5a8f3330,0x6bf902cd),LL(0x9216e842,0xda96e730),LL(0xd1f1d62f,0xa8ca4ed6), + LL(0x50bbcd76,0xd9521c88),LL(0x5533084d,0x3d8ab30a),LL(0x40a22bb5,0x4328e72f), LL(0x680fccc1,0x6594e079),LL(0xbd6afc46,0xc578705e),LL(0x724db349,0x677a7e9e), + LL(0x77b5d2f3,0x376f9327),LL(0xc9b18afe,0x87bd1401),LL(0x7395cc2c,0x8f06b3fb), LL(0x0f11a8a7,0x5d2d7a7c),LL(0xa3a0d64e,0x84cc49c1),LL(0xf720e6f2,0x0b5a5743), + LL(0x80df0a37,0xaeedece7),LL(0x9fa00fed,0x78b407d3),LL(0xfb5d3331,0x8ada177e), LL(0x8af3669a,0xa526660c),LL(0x26f37f61,0x95b36f2f),LL(0x1e1206f2,0x21aeaae2), + LL(0xade1a30f,0x0fbc3955),LL(0x70682b66,0x27c9b059),LL(0x2bfa453b,0x2fce5ba7), LL(0x1316e4e1,0xc90c4a00),LL(0x761c39fd,0xd6355b7b),LL(0x2532c4da,0xf0e4ff59), + LL(0xd62425cc,0xa9d571ca),LL(0x842bd560,0xdce97426),LL(0xc461be1b,0xbb38f10b), LL(0x822d623f,0x3d2249f7),LL(0xe1eeb405,0x202ec2c7),LL(0xb64d9541,0x4527855d), + LL(0xc433bf19,0x6fb57a38),LL(0x2a004581,0xa01483cc),LL(0xf1fb8cac,0x7cb982cf), LL(0x7ea92be5,0x8e24b22a),LL(0xc526028a,0x44dc6883),LL(0xc3c10ecd,0x3ccba012), + LL(0x6e378a16,0xfe5d98dd),LL(0xd7a9f035,0x959df6be),LL(0x856a9b08,0x9123863f), LL(0x4a4861a1,0x37a49207),LL(0xe726e927,0xf3abe482),LL(0xe950eead,0x80ca1765), + LL(0xf21dac1d,0xa899447b),LL(0x62bb8a55,0xca34599d),LL(0x599442ee,0x3ace3361), LL(0x9b2bdec8,0x19fb35d6),LL(0x1ba85c82,0x60e552f0),LL(0x0ecd12b9,0xe293d542), + LL(0xa2c1ef2f,0x6effd23e),LL(0x114661d2,0x5cd387d8),LL(0xdffe2dc7,0x8e8c7a48), LL(0x16c4945a,0xa2c03c10),LL(0x9644389a,0x9d7fd950),LL(0x30888d62,0x64455c93), + LL(0x910d7674,0xfda175a4),LL(0x233d1789,0xa7147ea4),LL(0x9013a09d,0xf101e4a2), LL(0x07139a87,0x67af7675),LL(0xf5941b5c,0x94837a17),LL(0x2afba5b9,0x8d2a3dc4), + LL(0xbfe0067a,0xe2c65840),LL(0xe666562e,0xd0b29ae1),LL(0x3f69d5ff,0x994f3344), LL(0xb38b5225,0x4895550c),LL(0x67443db8,0x317f6133),LL(0x6a3efbb9,0xe78c5ffd), + LL(0xf20ec153,0xb98ff8ef),LL(0x54026d0e,0xbb891684),LL(0xe7d12956,0x54ec1dc1), LL(0x8a5739e8,0x9c22e187),LL(0x2ada1917,0x1afc3eb0),LL(0xc839e43b,0xff8822a3), + LL(0x7425abda,0x9d4a9d18),LL(0x7bd3fdc1,0xf8e9fbc5),LL(0x9ebefe1b,0x697f7987), LL(0x1e6ed20f,0xa9e72448),LL(0xcdf74ccf,0xa345725d),LL(0xbe698fa7,0x7896df8e), + LL(0x3ba72507,0xa0c0c8b7),LL(0x0fa2a2a3,0x65954bbd),LL(0x16257ac3,0xa5e65d96), LL(0xbba0d229,0x3818228d),LL(0x021796d5,0xb8be11d8),LL(0xb97dc1f3,0xe6c3332a), + LL(0xdfda1f51,0xb405e455),LL(0xcbe599bf,0xa82537f5),LL(0x9f755190,0xb60bf1e8), LL(0x32a7e242,0x38f7bb06),LL(0x98acbf9b,0x3212ccc5),LL(0xe7ff0a94,0xeebc9ac0), + LL(0xd6c9a5f3,0xbb8969ed),LL(0x5bd66149,0x12b8c0a4),LL(0xd0b87b2f,0xb58f5e63), LL(0xd834d568,0x3567a0dd),LL(0x19662011,0x9a569d08),LL(0xf86f6443,0x089e7840), + LL(0x632d7db6,0xbdbee0e9),LL(0xc3b20b5a,0x6d9f02c0),LL(0x998a8d90,0x1050c608), LL(0x2f06ab1d,0xd8d0282e),LL(0xab0aa37e,0x4c54a44d),LL(0xd082d2ae,0x6e1b0aa1), + LL(0x25ecff7f,0xfeabb634),LL(0xd1a47294,0x97ef1829),LL(0x45d079b6,0x478931f1), LL(0x90a34a1a,0x0c0a2fa3),LL(0x25caf0d5,0x152a763f),LL(0x0e40b542,0x35d8af73), + LL(0x2007d582,0xc49fbdb0),LL(0x26d43be1,0xbba36d92),LL(0xd8e1f054,0x4e8feb86), LL(0xdf679100,0x2d7a4eb1),LL(0xa594368f,0xd99cc183),LL(0x7a3bbb39,0x5953cee1), + LL(0x8b7ac847,0x9db6811e),LL(0x2196bcd8,0x611682a4),LL(0xab14055b,0x5d9d1f5a), LL(0xec03a248,0xde8b16e7),LL(0xd7f2292b,0x8fdd20f4),LL(0x55aedc44,0x0ab41c2a), + LL(0x4c2182b6,0xdcca73f6),LL(0x236096b4,0x8fb4b87f),LL(0xd5653730,0xce326270), LL(0x76943c44,0x5c9e40b8),LL(0x0ab1c0c4,0xde4cd5b3),LL(0x3940c729,0xf6ed23bd), + LL(0x77a7afa6,0x5c07d98d),LL(0x5fb74010,0x3b469a44),LL(0xd7860cfb,0xfd595617), LL(0x59e07d3b,0x98976412),LL(0xba3430ef,0x90454a37),LL(0x0a3e1885,0x46a2a6dd), + LL(0xfb32d876,0x25adf7fa),LL(0x774e5344,0x256577cf),LL(0x1805d5e4,0xfd44d54d), LL(0x8fb4e6b6,0x92207fea),LL(0x11c5d5d5,0x4a220d0a),LL(0xf8e91bdc,0x3be12723), + LL(0xc245b8f3,0x2ca7da35),LL(0xa88aee56,0x2870eaac),LL(0x46354fd1,0x550b013d), LL(0x79eff102,0x42ddcbc1),LL(0x111f871c,0x66a3ed1f),LL(0x3e002a53,0xbddf6a08), + LL(0x89d6175b,0xdadeac86),LL(0xdedeed02,0xedba73cd),LL(0x5fae1ea3,0xf56ed9fa), LL(0x46fb11d2,0xd27a3c2e),LL(0x8111c477,0xe8196e09),LL(0x4511b448,0xff123cde), + LL(0xe26a279a,0x0f843c1c),LL(0xa3576554,0xc5e0d5bb),LL(0x41361ba5,0x23b9fb1b), LL(0xe963cc9e,0x23b72bfd),LL(0xb847fc14,0xdce7e767),LL(0xf14fcef9,0x162fc75f), + LL(0x5495b1c0,0x835fb910),LL(0x9d67f026,0x4bf8073b),LL(0x8899fab2,0xbf47587e), LL(0x82303583,0x5d3d606d),LL(0x3016c2cb,0x8dc9e515),LL(0x70ad2ecd,0x1941fb0d), + LL(0xaaf9bf1a,0x52d3382e),LL(0xd97fd623,0x82ca0b62),LL(0x421adaf5,0xd415fb55), LL(0x5bae728f,0x9e737a64),LL(0xa226fda8,0x8bb1f7ec),LL(0xc8fc211a,0xbbe52052), + LL(0x98b18dd9,0x111f6855),LL(0xfc1a8284,0x07963287),LL(0x049bc41f,0xadd96aeb), LL(0xa4b9431a,0xf9cdc9cf),LL(0x3186f178,0xed472952),LL(0x9e6b099c,0x7c483bff), + LL(0xecf00f66,0x0f259f0b),LL(0xdc4d7da1,0x42adeebe),LL(0x8347831b,0xa84be35a), LL(0x1cb75e95,0xfcc87599),LL(0xaf570741,0x4d352430),LL(0x560a12e1,0x3985430b), + LL(0x269c3158,0x0d101d88),LL(0x62a2e7c9,0xf4f4fd03),LL(0x5bd4e966,0x2868c3a9), LL(0x3f2fe7d2,0x28560f6f),LL(0x9f97e5c5,0x33e9acc4),LL(0xaa6fdb21,0x13e585da), + LL(0xa9d10acd,0x8c93261c),LL(0x39ee8556,0xaa0c4ecf),LL(0xa1ae594e,0x24174c20), LL(0xe22539a2,0x3741dd7f),LL(0x592618e2,0x2d22e571),LL(0xdf3458f1,0x2f7bfd39), + LL(0xd69e123f,0x0a50b744),LL(0x16a94796,0x64461277),LL(0xe18a1d4d,0xd71b262f), LL(0x5e180b7b,0xd9040b30),LL(0x47d6ce9d,0x26f0f403),LL(0x67a15aa3,0x78d16700), + LL(0x31529faa,0xa97d1af6),LL(0x2ce2270e,0x6598f797),LL(0xb178d3c3,0xfeb8f446), LL(0x16f5a2ab,0xa967e2c7),LL(0x04b96d78,0x205fdcb6),LL(0xe986b7e7,0x0e580f2a), + LL(0x2dd71da3,0x5b769229),LL(0x88f598fb,0xfaf8b016),LL(0x143492ae,0x8f8d4b70), LL(0x96a9c03f,0xa7f5e617),LL(0xa0c34682,0x6bd67a8e),LL(0xa12c32cc,0x7b459945), +}, +/* digit=24 base_pwr=2^168 */ +{ + LL(0x4fee65e2,0x00b598c5),LL(0xf18f37b1,0xf85f1a2c),LL(0xe3b72548,0x8659b393), LL(0x696e8d4b,0x3f89080e),LL(0x219f661f,0x4b0794d1),LL(0xa6fda57a,0x95783cf3), + LL(0xacf31bb7,0xd47dbc28),LL(0x44076b4a,0x228ba361),LL(0xeff372cb,0xc98f0d65), LL(0xed79eb73,0x73179523),LL(0x9c75c7fb,0xc172348a),LL(0xfd4dc8f8,0x56a7598b), + LL(0x611e59a4,0x59648973),LL(0xa9f595e0,0x2151777b),LL(0xf5551b53,0x4c47c966), LL(0x3972a3d3,0x021253a3),LL(0xb4befe84,0x90bc0c61),LL(0x77515e8a,0x16d84bd2), + LL(0x94f40d96,0x5c350e2e),LL(0xa5864e3f,0xdfcbcae9),LL(0xfb5d92bb,0x9a2e4e30), LL(0x44281abe,0xd9bde421),LL(0xea539b37,0xa7f9d3a7),LL(0x27b92e31,0x61c3ae50), + LL(0xf2aa70a9,0x2244e258),LL(0x35be44a9,0x73df3f53),LL(0xa6238039,0xdcefe8bb), LL(0xe47497a3,0x3a811b6d),LL(0x25465973,0x620c900a),LL(0x4aa632c7,0xe81d4109), + LL(0x81a533c4,0x49894327),LL(0x9c521884,0xb510e537),LL(0xd2db22d8,0x8e091dec), LL(0x2112921b,0xe40883cb),LL(0x3023be06,0x77a182a5),LL(0x344bcdce,0x9406af56), + LL(0x5f5b1245,0xb0c7d9e5),LL(0x01cd2020,0xf2cae30e),LL(0x4f24138a,0x300d3991), LL(0xe5eb6668,0xb1a3ab1f),LL(0x1468e662,0x1c21f927),LL(0xe4627789,0x2cd78fd3), + LL(0xa2475808,0x6bb937ce),LL(0x89fbe7fa,0x190dea20),LL(0xa4a879ab,0x2b19299e), LL(0x02e58d85,0xd08bb8b3),LL(0x44d8e850,0x5f3c807f),LL(0xa807b92b,0x7163052b), + LL(0xf139c859,0x33e3fbb7),LL(0xb61d1651,0xae851c86),LL(0x711ebfa9,0xd19bb9d2), LL(0xab556ef8,0x04eb2dec),LL(0x26e75498,0xa90032a2),LL(0xad236dde,0x1b6e1eac), + LL(0x72b36422,0x45e0d154),LL(0x440553f6,0x505c7e2a),LL(0xcfaf2624,0x51c21ee7), LL(0xeead1c76,0x28cfa503),LL(0x21322f7a,0xe7e08256),LL(0xca8f7baf,0xb50012c8), + LL(0x0e8b3583,0x3ec7b6f2),LL(0x40191db4,0x75562e4b),LL(0xd97200b3,0x2de64fe8), LL(0x67766784,0x9035e032),LL(0x00818616,0x5f64b295),LL(0x9d38c296,0xd62bff91), + LL(0x20e7da86,0xc1c38d7c),LL(0xaa79a592,0x29d1701d),LL(0xcb4964c4,0x43853290), LL(0x558d0744,0x157c8a28),LL(0xd845fdae,0x9359aa13),LL(0x0c16e633,0x6938504e), + LL(0x303aaf9c,0x7551d181),LL(0xa6ee9450,0x4f75790a),LL(0x5a5ee5ae,0x30cda47c), LL(0xcaeeda67,0x5cecbe96),LL(0xb6acf1ec,0xbb304fb4),LL(0x23d4e0e9,0xa8d438de), + LL(0x55bbf072,0xe3c174cd),LL(0x096526d5,0x4c192612),LL(0xa3622362,0x72b1886e), LL(0x38ad3224,0xf9ff34ac),LL(0xde248da9,0x3c0e6f03),LL(0x904019ba,0xaed9b48d), + LL(0xeb4c2fe6,0x8b931dfc),LL(0x6fb1fd3f,0xb59385ae),LL(0xcdbf8b2c,0x34d15d09), LL(0x501dfe27,0x354fb5bd),LL(0x8678f10a,0x3244c239),LL(0xdefc3322,0x27fb966c), + LL(0xd835d9d2,0xa7bfa23b),LL(0x3d77d0c6,0xe2e0879f),LL(0x7198caba,0xcd307f9f), LL(0x8ef6dc8a,0xaaf47ac7),LL(0xe089b5a0,0x79d488a1),LL(0xd3898ccd,0x9c5910d1), + LL(0x99fedbbc,0xe1a69186),LL(0x8c8fedb7,0x1cb035cb),LL(0xb456f31c,0x2cd27f26), LL(0xde4d4827,0x5fe6041c),LL(0xc126f7bf,0x3566eaa3),LL(0x4361f83f,0xafba18a2), + LL(0x5b63f6d1,0x79cf0a94),LL(0xd989b598,0x00ce6a85),LL(0x03f0b2c6,0x0f8d928b), LL(0x018ff12d,0x4a4299b9),LL(0x5f379fa7,0x59ab07e4),LL(0x5381194b,0x5e5deef5), + LL(0x3c4ab2c9,0x6e7f659c),LL(0x65654e5a,0x8a446f62),LL(0x6327f61a,0xc3cccd42), LL(0xd14cb9b1,0xc9e4ca54),LL(0x6bde8567,0xeee5364e),LL(0xdd7a0868,0xac8150fc), + LL(0x5f134595,0xa719652f),LL(0xae508424,0xfcfec81c),LL(0x11ba658e,0x9e16ae3f), LL(0x60511e27,0x88494b81),LL(0x66979b9a,0x495d153e),LL(0x49fad3d1,0x55611c7a), + LL(0x8e3ffac8,0xfc730512),LL(0x83da2d11,0x1b6bf5a3),LL(0xac075bc5,0x99558261), LL(0x7587269d,0xd44f89c7),LL(0xdee8b114,0x05413990),LL(0xe309d884,0x35258a3c), + LL(0x6a76d0d3,0xba07c28f),LL(0x5067e16c,0x18a52a08),LL(0xc175e5e2,0x6ae1d870), LL(0x9b314f1f,0x5404c2f8),LL(0xb85b02be,0x22c45028),LL(0x69fe8fb5,0xbfa91557), + LL(0xd5e20541,0xf420471f),LL(0x93b4597e,0xb7cd0563),LL(0x9e8fced4,0xcd762134), LL(0x667246ed,0x93f1a293),LL(0xcc37aa97,0xa67f1015),LL(0xccc76489,0x54ecfc1d), + LL(0xffed38d3,0x2867d280),LL(0x339ff836,0x34e2e617),LL(0x843d9ee5,0x90c7a0a0), LL(0x0bc5ad9d,0x38be6f0d),LL(0x61784c24,0x5be128b1),LL(0xf5c92d27,0x6d1d7024), + LL(0xf28e8e61,0xb93c9e42),LL(0x2e7b7f4b,0x1cb53b6a),LL(0xe4e0fef9,0xe1f605e9), LL(0x1f88f7d1,0x069d28f6),LL(0x22c71a8d,0x96265f16),LL(0xdab828ef,0xac157d26), + LL(0x341a67b9,0x98b3a5ed),LL(0x3f3639df,0x9aaa605c),LL(0x8b92ce1e,0x59297d2f), LL(0x36504280,0xb9812805),LL(0xa4416a3d,0xe5ae2be0),LL(0x41919442,0xb31daa16), + LL(0x0feb1f18,0x1634ffbc),LL(0xd33d0a9d,0xa51dcbe0),LL(0x767a41f1,0x0e03bb87), LL(0x148212a0,0x01041281),LL(0x0e55fb63,0x07f9ce13),LL(0x9bbfabe9,0x9dcfb9b9), + LL(0x4eb575e2,0x3ac5febc),LL(0xc5aedbca,0x8aaae188),LL(0xd5870b65,0xc14938bb), LL(0x37600276,0x05a4c6b7),LL(0x6227e53f,0xa3b7f153),LL(0x4703e577,0x3b37d9f1), + LL(0x6aea82a4,0x7354429d),LL(0xd061ccd4,0xa8b693b4),LL(0x52d0b458,0x3120ff37), LL(0x92f80ea7,0x254dbf7f),LL(0x89430f4b,0xd229a339),LL(0x94de0c1e,0x97b253ff), + LL(0xe4b60a58,0x6556cb19),LL(0xf3b79531,0xaf91e209),LL(0x5bd7c1e7,0x4600c789), LL(0x4100417e,0xe92bfc4e),LL(0xb6d8a9d9,0x772261d9),LL(0xa9a6916b,0x2e573e06), + LL(0x98b95e3a,0x05709266),LL(0x1dbd0d3c,0xef67fdbf),LL(0x7552cee7,0x950e2fa7), LL(0xf4ae00d3,0x03b57561),LL(0x46ba7e71,0xf070c70a),LL(0xe39a20da,0x196843b1), + LL(0x60091a66,0x4213bea2),LL(0xa0e25486,0xbed0b79e),LL(0xe5c60f95,0xd2bc9dfb), LL(0x3ce65338,0x7aa159d5),LL(0xc8ec70b6,0xef1f0d85),LL(0x28d23ccb,0xb47100d7), + LL(0x2f8c540a,0x7c602da6),LL(0x854edfb6,0x9e3193eb),LL(0xa18a05e6,0xd1b97d97), LL(0xc430fcd8,0xff9bdf8d),LL(0x3b7fb6c4,0xdfde4b67),LL(0xa97725fd,0x3d8b2185), + LL(0x0cd2fdea,0x20f59fe1),LL(0x5923a8af,0xfe8a4dc0),LL(0xfa9e9992,0xf9d94a99), LL(0xd35b2020,0xb9c441eb),LL(0x07668b98,0xcce38d28),LL(0x2bf90eab,0x582f54bd), + LL(0xab0e3f41,0x9f1d9413),LL(0x3a412ee4,0x5e8c119b),LL(0xf030ad58,0x8879b87f), LL(0xde74321c,0xe48dc015),LL(0xf6cee695,0x522fb0ae),LL(0xb8d5054e,0x40162c77), + LL(0xc129d829,0x2f8ae88f),LL(0x3f3ab771,0x999f091f),LL(0xffb12ab6,0xa7e1a5ad), LL(0x6c51306e,0xbf7d078b),LL(0x3307b832,0xb084c854),LL(0x64450a20,0x46824c00), + LL(0xff4b278e,0xafad7324),LL(0xb780a046,0xf2aa88cf),LL(0x03143e85,0x429e746d), LL(0x91dc0962,0x5727ab2b),LL(0x67771a60,0x489517bd),LL(0xd28b48a1,0x2d01a677), + LL(0x68e73dfd,0xcecba270),LL(0xcdb43f75,0x2988378a),LL(0x8a6f1487,0x617e4cc8), LL(0x77958a86,0x7b3bbea4),LL(0x00baf816,0x097fa14d),LL(0xf3dd7875,0xaad04d0d), + LL(0x6ac04cd8,0x03d57abe),LL(0xc1971431,0xe65c48d6),LL(0x57dbcf69,0xc762049d), LL(0x408042bd,0x967f3d11),LL(0x0ff3e3b8,0x8b9583ed),LL(0x11593e70,0xafcb3c34), + LL(0xc3f9b58c,0x41df50a9),LL(0x55f9cc35,0x259d92b8),LL(0x2f2195a7,0x670ff896), LL(0x242e6aed,0x1a7c5825),LL(0x0b8b7fa6,0x04cc2dba),LL(0x0f24c794,0x20eb7a8c), + LL(0xab8c47fd,0xbb5b87fa),LL(0xe425d990,0x24a897b6),LL(0x680c630a,0x86cc5725), LL(0x31fb1789,0x98095baf),LL(0xe7c13217,0xc3655f2e),LL(0xf43a1aa1,0x014736b8), + LL(0x52deee88,0x62bcaf35),LL(0x5b973eba,0xb396c4e7),LL(0xbda679ab,0x51c1903d), LL(0xd5c3a5aa,0xae30078a),LL(0x40876de9,0x8b17d5cd),LL(0x86473f13,0x150e6256), + LL(0x09da0150,0x3afdb04f),LL(0xa6c1fbb6,0x944dc9f4),LL(0xd6d37139,0xf978d7e1), LL(0xd7688302,0x294df22d),LL(0x050fc353,0xcb4a2989),LL(0x4d693ea6,0xb7355cc1), + LL(0x705e6c1d,0x603b9990),LL(0x6e20c02e,0xbc22cb07),LL(0xda4a714d,0xcf26fedd), LL(0xc5f7dd2d,0x499a5020),LL(0x870ff472,0xd5f6d465),LL(0xd7242e9f,0x64105bcd), + LL(0xdfc15966,0xe8651d8a),LL(0x3f36c08c,0x65f62567),LL(0xd7cc1614,0x7e1b118a), LL(0x5f8c1b51,0xe547ba77),LL(0x40fd1c4a,0xbe829b43),LL(0x7393994c,0x27ef6fa1), + LL(0xf06e1517,0x47b06a9b),LL(0x9538a94f,0xfc4162b2),LL(0x8787dbb2,0xda0b09f1), LL(0x69ed3599,0xaf40f743),LL(0x20668861,0xd0539dd8),LL(0x4d25d347,0xd97dd381), + LL(0x8ed5efa1,0xf37b0062),LL(0xf0d1e9a1,0x33b8ee0e),LL(0x4e88bddd,0xfc5ed264), LL(0xebc6518d,0x9c8a10bf),LL(0xed07b158,0x3bacc4d1),LL(0x38679a57,0xf3a21ff7), + LL(0x28d869ca,0xb696e37a),LL(0x5e14048e,0x1c92d21f),LL(0x09011d46,0xbd612912), LL(0x344de336,0x4253b283),LL(0xb1b9160e,0xc5500601),LL(0xe5dcbf35,0x6a09d20c), + LL(0x94adc78d,0x0d0de35b),LL(0x0dc1d07c,0x77e19853),LL(0x0bdd2c5b,0x34b0dc75), LL(0x753ad557,0xfb35d357),LL(0x63471eb9,0x5d25872a),LL(0xc1012690,0x0deb74f6), + LL(0xaaab12f1,0x8af62a30),LL(0x1013c549,0xb94f6b3e),LL(0xe29c422c,0xc66fd2e8), LL(0x5ac31b52,0xf14b58ae),LL(0x222c930c,0x341a40c6),LL(0x79810099,0x97a887df), + LL(0x4d81c717,0xd365310a),LL(0x01f352f5,0x03f4d8f5),LL(0x7eea6a1f,0xf252f189), LL(0xf8638d81,0xb159cdf9),LL(0xe3f109b7,0x984ed9c3),LL(0x782d6bf4,0xef2542fb), + LL(0x587be148,0xd0c19f88),LL(0x86172fd1,0x607cef08),LL(0x75ee2bbe,0x7d46f8eb), LL(0xf39fe2f1,0x0d243305),LL(0x362a46ff,0x595aba8f),LL(0xe19fa4d4,0x5a9e9116), + LL(0x9d07c0e7,0x7c3998c0),LL(0x3ab74d57,0x5dab40f7),LL(0xc6ea1153,0xb0deb5d1), LL(0xdba8086e,0x2f3a9092),LL(0x9dd0d434,0xdf0db996),LL(0x30e52883,0x63af92c7), + LL(0xfb8379d7,0xce592f6b),LL(0x57237edd,0xf182db14),LL(0xa33ff0fb,0x435ce079), LL(0x16e22f7b,0xf67443b4),LL(0xd01a4364,0x1a603d76),LL(0x0d3849b6,0xf37e32ca), + LL(0xe6dac621,0x2456397b),LL(0x1585b538,0xa98d7499),LL(0xd2fb419a,0x249b8fe2), LL(0x2d722c8e,0x1971929c),LL(0xeb0dfb04,0xb835ed94),LL(0x7786927d,0xe0ec2ab9), + LL(0x96808ab1,0xda420127),LL(0xae1677d6,0x9d1d0cfe),LL(0x4381a97e,0x641b4256), LL(0xcf4f8245,0x57a2072c),LL(0x24192acf,0xc038aab8),LL(0x6ee05997,0xf708b6d9), + LL(0x82628553,0xd0f52e74),LL(0x96add683,0xd6dfc4ae),LL(0xe70500d4,0x5197189e), LL(0x82ee5eac,0x28fe6d10),LL(0x24145176,0x6c314538),LL(0x7a0ce5ad,0x0b507a03), + LL(0xbf1b56ed,0x28341dba),LL(0x2bd1775a,0xa704c00d),LL(0xcd969de8,0xe8c68f7f), LL(0xad40ac8e,0x64fd4b91),LL(0xfbb5f7b8,0xbc8f2d47),LL(0xb0338932,0x43d6bbe3), + LL(0x81bc04b1,0x72af7125),LL(0x4e2f9274,0x88d535a1),LL(0x3974e880,0xf0578277), LL(0x15b341a6,0x1539a26b),LL(0xb3e593cb,0x603717a3),LL(0x7d61dcd7,0x477f833b), + LL(0x032e59e7,0xee7a6462),LL(0x3532e58d,0x70ac18e5),LL(0x0e36ad7b,0x9382dda0), LL(0xf911a5a7,0xf4c500c9),LL(0x800f8273,0x6d46aa73),LL(0x3e10963e,0x670e4f56), + LL(0x29429437,0x4164dbd4),LL(0x41281f62,0xd9358c9b),LL(0xcbf5eb0a,0x176d1b60), LL(0x2ccdd253,0x3a70e691),LL(0xa25d63e9,0xc53385f2),LL(0x2388d4cf,0xecfe446a), + LL(0xbc93d305,0x5fe1afcf),LL(0xf307c07f,0x59c6b4d8),LL(0xec492ace,0xa805d89d), LL(0x7c31b553,0x572efc64),LL(0xd5bf85fd,0xb15afb14),LL(0xfa431d83,0xb8a6a24c), + LL(0x3057e6e1,0x031dd1e6),LL(0x919f1a50,0xac205cef),LL(0x6bf4b8fe,0x3236d945), LL(0xd11eb95a,0x3cdc048e),LL(0x2b4d6ce2,0xaaa916ac),LL(0x7d395898,0x381aca94), + LL(0x9ec42177,0xc5146299),LL(0x2d66626c,0x6b043df3),LL(0x82ea3769,0x9ed7174d), LL(0x9da03903,0xf8f18beb),LL(0xcd642f4c,0x72c11f7a),LL(0xd4de2f90,0x4bdad457), +}, +/* digit=25 base_pwr=2^175 */ +{ + LL(0x58d3510e,0x5641060f),LL(0x364db85d,0x8e313a7f),LL(0x5cafd58c,0x203122d9), LL(0xccb848a1,0xf79b8cdf),LL(0x67e06429,0xda00c247),LL(0x50dbd8ef,0xe8ddf520), + LL(0xd06db4a0,0x13b4d6c6),LL(0x1c320a37,0x9ca098dd),LL(0x4660ce85,0x23635473), LL(0x70e85a9d,0xcb8bdc7f),LL(0x92b6fa4b,0xd96a49de),LL(0x6f48c32b,0xd22ee1cc), + LL(0x526a7a48,0xbaf0570d),LL(0xfd4d36d1,0x9436cae8),LL(0x24fac988,0x44427e92), LL(0xb39cb050,0x935deddc),LL(0x95237984,0x2db73da9),LL(0xe0403a31,0x6d812614), + LL(0xa4968444,0xf1cb1e84),LL(0x15156d72,0x8624f8a4),LL(0x66276ea6,0x83d9f12a), LL(0x3b28aaa8,0xd15883d2),LL(0x06aba758,0x4043aea5),LL(0x19ad8608,0x3b9cffbe), + LL(0x71c764ac,0x19b195dd),LL(0x16c89a6f,0xb10cdd8e),LL(0xdee07ce5,0x51f783dc), LL(0xbf7febaa,0x05b98ceb),LL(0x0026e40f,0xcbdf20ec),LL(0x8ea46069,0xf4ff428d), + LL(0x19829b4e,0x204f3d40),LL(0xfa6cc5ed,0xd37d49f5),LL(0x07a5325c,0x55bfb8db), LL(0x7c3cffdf,0x349c9305),LL(0x44e85a40,0x15868ca4),LL(0xe664b649,0xab792855), + LL(0xa8931c2a,0x1488def0),LL(0x93e47cf2,0x20b6b035),LL(0xc9c0e09a,0xfd90bf99), LL(0x9e4cf905,0x28fe0815),LL(0x48dbf990,0xf397729a),LL(0xb09e4e66,0x82000550), + LL(0x152fdc18,0x7c7b2c74),LL(0x81529a8b,0x2e09d47c),LL(0xdffec896,0x116d1ef9), LL(0x9ea4077e,0x448891e4),LL(0x9566eb00,0xadad99e4),LL(0x9df75199,0x7d6c3d02), + LL(0x1d7fffc5,0xbc3f75a2),LL(0x741fd001,0x47e4ce19),LL(0x45b7d08c,0x24659725), LL(0xc74dcf3a,0x1a4b43cd),LL(0xdc7dd457,0xa1e3c71d),LL(0xe49194dc,0x37aeb7d2), + LL(0xbe8af7ed,0xf6372965),LL(0x2e04049f,0x4e36da0e),LL(0x34f4f1d8,0xcc4e4c0f), LL(0x387c3ce4,0x185a80da),LL(0xf81efae2,0x999c4880),LL(0xa0cc97d1,0xd2002692), + LL(0x2a469d56,0x97b45c6f),LL(0x4932fa9a,0x0c918aab),LL(0x43aff643,0xc2a2d43e), LL(0x33806c7d,0x685eab5b),LL(0x69645ab9,0xa44c2171),LL(0xa8df46a4,0x128ed8da), + LL(0x650a4592,0x561ca2b2),LL(0x3ca61b0b,0x01673084),LL(0x680d8c8f,0x79ebf7d1), LL(0xc216ae2a,0x33cd5c09),LL(0x260a7797,0xdc44dbeb),LL(0xe5886569,0x80046982), + LL(0x01f3457c,0x6ffe44e9),LL(0xcbd42d4f,0x9f2fb631),LL(0x77426e19,0x372de1ab), LL(0xb5b9cfe8,0xe73bea79),LL(0x105e1632,0xd2af87d0),LL(0x91def158,0x562e0ebb), + LL(0xe8eadc48,0xb3051141),LL(0x214796b8,0xe9797985),LL(0xb0cf5916,0xc7174b9f), LL(0x7c17697a,0xccdaf130),LL(0xe6a3f8cb,0x2ff86ffa),LL(0xe0e4d911,0x655d7ff4), + LL(0x7f8a7546,0xeb590d5a),LL(0xd84598ae,0x2e33657c),LL(0x1cf22291,0x6865f925), LL(0x80d54d83,0xf33b0db7),LL(0xd095724d,0xd2045fa3),LL(0x37a729ca,0x2a65650a), + LL(0x9b7d8eb7,0xe5a6dff7),LL(0xdd67daf1,0x104734b8),LL(0x503bfa5d,0x49050eee), LL(0x49376195,0xa3422472),LL(0xd85aa682,0x9d58a5f9),LL(0x27b209d1,0xef14a582), + LL(0x10c74322,0x72142664),LL(0x161a9f7f,0xed4f37db),LL(0x5f7f95dc,0x0f64b4a6), LL(0x3907231a,0x66c0fe9a),LL(0xc7664836,0xeeadbbaa),LL(0x7907bf1b,0xece73bcd), + LL(0x9c17a6d5,0x16aded71),LL(0x08ca25bb,0x2a92a505),LL(0xe1061e03,0x180b69be), LL(0xd61b7fbe,0xad3ee8dc),LL(0xe5e8b274,0xf8d14495),LL(0x3bb4fd37,0xa5269565), + LL(0xfb62dc03,0x170bf89b),LL(0x18b9c79a,0x1fe9859a),LL(0x839e677b,0x7605d9e1), LL(0xaa1b4663,0x6d265c97),LL(0x48202008,0xb34d878c),LL(0xa03d375a,0x2b696896), + LL(0x166a71cd,0x789e3b3d),LL(0x10983ef5,0xa9f4f3ca),LL(0xdb908c7d,0xf634e8df), LL(0x8192d652,0x0375b69c),LL(0x7e4b6e1c,0x2f5cbad4),LL(0x45be1a16,0x4518ff99), + LL(0x408d39c0,0xb664989d),LL(0x2e4280b7,0xa947849c),LL(0xece6ea89,0xbad0f997), LL(0x3a7571ed,0x434fd6c9),LL(0xcf47df20,0x46da8ae4),LL(0x638076c4,0x7708eff6), + LL(0xc63c8ac2,0x17e1b4e7),LL(0x36830de3,0x0fff916b),LL(0x55092352,0x2fe8eb88), LL(0xd0af6ad5,0x8b72540b),LL(0xf26f7df4,0xf41a415f),LL(0xb3259e07,0xb7be4c73), + LL(0xb3b6ce27,0x4d668949),LL(0xf8705df2,0x332b0818),LL(0x2b168b67,0x11736e3f), LL(0x96ab51a4,0xb7a67c69),LL(0x57e547e7,0x00855b88),LL(0x17aac6bc,0xe9ff07b6), + LL(0x2efd47f8,0xeebd36d2),LL(0x288c8f9e,0x4fe7bc31),LL(0x034e796a,0xe82f1e28), LL(0xf0e8922a,0xd25c0279),LL(0xf0f8dcb2,0x9ec4ded4),LL(0x860e5a78,0xa66ac462), + LL(0xb8a729d3,0xef72f5c5),LL(0x13b9c03c,0xb6915595),LL(0x2ed45b90,0xa4e66b65), LL(0x0845bf8a,0x763c3414),LL(0xac93c4fb,0x13ac686d),LL(0x2095313b,0x923efc3c), + LL(0x3349c905,0xac570164),LL(0x5ca04e5a,0xc85ebcd6),LL(0x6a5d9dc0,0x616a35c7), LL(0x68d2818e,0xfbce6895),LL(0x59769d61,0xd85a342b),LL(0x025ba76e,0x0aa5703c), + LL(0xe391b459,0x205537fb),LL(0x5d410558,0x8d2a3d0b),LL(0x64a2d47d,0x71444a23), LL(0x822b0c30,0x0c86c2f6),LL(0x64c2f548,0x8b276842),LL(0x94dbb24c,0x51a73b54), + LL(0x29e9c4f7,0xf0ead209),LL(0xe1401652,0x5eddd090),LL(0x190c1898,0x6d6b0600), LL(0xef639412,0x762be19e),LL(0xe78dd0d2,0xe322a3e9),LL(0x824c6d20,0xde53b7e9), + LL(0xc90500e8,0xc6b2dd77),LL(0x5a53da9e,0x3a399dc1),LL(0xd09ab46b,0x0969d406), LL(0x249ac549,0x526ec7ed),LL(0x1a09dad3,0x4ae8309f),LL(0xb0c3ad12,0x143165c0), + LL(0x610d6c08,0xd1d7d354),LL(0xd1dbf837,0x12d3da0b),LL(0x01a98f82,0x73fb7218), LL(0xa8666766,0x71fe7d63),LL(0xdc316634,0x74ea9c07),LL(0x55aa5452,0xad9f5798), + LL(0x92ab7a01,0xeb52fc54),LL(0xfa5e4eea,0x6ec23c07),LL(0x01a397ea,0xa9cb3e84), LL(0x69945c66,0xebcac5af),LL(0x3f194ceb,0xdf57e59f),LL(0xa296c88e,0xb5ec451f), + LL(0x794f2a13,0x9cf79425),LL(0x310e1c2d,0xff079313),LL(0x5317d195,0x4d8c0f20), LL(0xf670d56c,0x0d854923),LL(0x31cfadcf,0x7564177d),LL(0x0845bd19,0x67d58b2f), + LL(0x6ce7d0f0,0xed2fd628),LL(0xa2ff8523,0x0a338e2e),LL(0x6968e5ed,0x4cd9123a), LL(0xf2f06ab4,0x14edcc8f),LL(0x6e52fb45,0x23c39e14),LL(0x41a9c448,0x529a89d3), + LL(0xfa8c503f,0x436e8ebc),LL(0xc7ec91f5,0x2d692350),LL(0x2f6a42f3,0xe847216b), LL(0xa68613e7,0xe12297b3),LL(0x10dbc984,0x16e343fb),LL(0x16d59123,0x7dbec088), + LL(0xd64ff51c,0xf3e6abe1),LL(0xd3afe2c2,0x87accfb2),LL(0x4dba0813,0x57b5f905), LL(0xb3b8c3fe,0x17038f08),LL(0x247ea7e7,0xfe435876),LL(0xa6d8a7a0,0x61415630), + LL(0x72efb994,0x685e474d),LL(0x495c513e,0x05b09ce3),LL(0x36da77db,0xf03f489e), LL(0xd1920e81,0x5bce5842),LL(0x4ba12592,0xc3fe249f),LL(0x0a5eed91,0x9671b08a), + LL(0xaac51e01,0x387fcc9a),LL(0x9c3e2419,0xf8c9ecb3),LL(0x754e0ecb,0x1017d74c), LL(0x9a76367c,0xb1528d0c),LL(0xa2e5c47a,0x4dcd49c1),LL(0x3751eba3,0x7b635160), + LL(0xae011f19,0x3879777a),LL(0xf839016e,0x6a63aafe),LL(0xe3944a17,0x199a76e4), LL(0xb399afa0,0xa0455c72),LL(0xf9479065,0x95450e12),LL(0x82b89c67,0x90479fcb), + LL(0xe392cb40,0xdc585a30),LL(0xafe2f1cd,0xbd161799),LL(0xaf671659,0xf0794ecb), LL(0x3e558726,0x7144b4e9),LL(0x0e8fb051,0x61aa80d1),LL(0x0a97d49b,0xd2a41b0b), + LL(0x0a74e744,0x10f7c811),LL(0x96ab89c0,0xdd94a330),LL(0x53525096,0xb617d078), LL(0xcfdfa40c,0x2e5b46f0),LL(0xcedc53d0,0x6edea8ab),LL(0x6399837f,0xaee682f0), + LL(0xf2adc050,0x398c7fd2),LL(0xc7638b0a,0x1030cdae),LL(0x8867f1d6,0x1cd2be4b), LL(0x1ba68d19,0x2f97a204),LL(0x18cd2ec7,0x47f3b28a),LL(0x8e77c014,0xf164c987), + LL(0xc33772e4,0x8489b718),LL(0x2f4128d3,0x08ac1911),LL(0xda0369a0,0x8370f1f5), LL(0xf1d75833,0x50ff1409),LL(0x0f83a8e7,0xa314c13c),LL(0x5a766258,0x776de80d), + LL(0x77ba6858,0xffe94ccf),LL(0xcc2a35c8,0x1b87cbeb),LL(0x57a1fa64,0xd5528288), LL(0x5937abfc,0x2e32d797),LL(0x284ca1f0,0x89a76fe3),LL(0x482441bc,0xdbeb8eda), + LL(0x397926fd,0x6bd5d9e9),LL(0xf5fbd488,0x3e97744f),LL(0x4ff64738,0x020f694d), LL(0x968aa895,0x34b3915d),LL(0xb88572cd,0x60b03cd4),LL(0x67dcf49f,0x0cd44b01), + LL(0xac9b8b5f,0xc6f1228c),LL(0x7822905f,0x5f49fb8e),LL(0x950e82e9,0xa61a8520), LL(0x6f44f547,0xec78c658),LL(0xaa640c3e,0x86d54b78),LL(0x54c16691,0xf6aab7a4), + LL(0x44fbab19,0x0cae07b3),LL(0xcc4d821d,0x61e61d3c),LL(0x5beae87e,0x5c2ff6ed), LL(0x9ba67712,0x719d81cf),LL(0xcb1e1329,0x8fb7e1c8),LL(0x32f7e9b7,0x82a0507a), + LL(0x0d8af7c0,0x94bf5956),LL(0x3dde66aa,0xeb94e708),LL(0xcc491968,0x2924bca4), LL(0x1bae682d,0x348c6486),LL(0x0da0931b,0x5490288c),LL(0xa923db17,0x79cae045), + LL(0x4ed8cc93,0x99ae0e06),LL(0x1f211af5,0xe0dca803),LL(0x54413c07,0x3e0da321), LL(0xd2e5771c,0x11bf91ee),LL(0x5d9efe6d,0xde57fde7),LL(0x2bae2eca,0x140cd249), + LL(0xebe25fb3,0x8c25772c),LL(0x5ae86876,0x9ebd7151),LL(0x227c8fc6,0x9e01abfa), LL(0xe8279d68,0x10316800),LL(0xa07da963,0x394a6fff),LL(0xc8b9c7e0,0xd8316e12), + LL(0x4866b100,0xe34a3346),LL(0x64640c51,0xbe82e03e),LL(0x8c26b45a,0x2228e85c), LL(0xaaec11b1,0x4f53739a),LL(0xed773bc3,0xedcebe7c),LL(0xfcef0ce2,0xfee917ad), + LL(0x44c7c595,0xdf6dca1f),LL(0x374d0d04,0x421ac2e7),LL(0x9e3ed6ef,0x842f0ded), LL(0xe9dcc612,0x0e20b04a),LL(0x2040e59d,0xbe403a6f),LL(0x8611c050,0x0ed4eb43), + LL(0x559e8de2,0x8479f4bf),LL(0x851f7b34,0xb12ebeb5),LL(0x3a3d71ea,0xc34bfd18), LL(0x24e9273c,0x56d9d399),LL(0xb9f63b13,0x7f404c7f),LL(0xe50afb2d,0x9b679748), + LL(0x09f98b35,0x9be55951),LL(0xff869a57,0xde925893),LL(0x21a9054e,0x387f817a), LL(0x70ee95b2,0x1e674786),LL(0x36b7d89e,0x628421b4),LL(0xe95fc254,0x5c66493f), + LL(0x3f27517a,0x68f7f079),LL(0xa2634bb9,0x44e0f99f),LL(0xc49ed6bb,0xff40ec9d), LL(0xd4c15f22,0x308b3e76),LL(0xddaa3039,0xb3f77d17),LL(0x7ea641c5,0x3dc87e6e), + LL(0x004a3af7,0x8d01afc4),LL(0x5cb71d7c,0x118e5970),LL(0x48ba3691,0x4facd336), LL(0xa796a7d3,0x30530fed),LL(0xcc7913f8,0x437ad3b9),LL(0x8adebd12,0xe0c151c6), + LL(0x2e553c5f,0xf37ea5cf),LL(0x45ebc28e,0xd35512e6),LL(0x05331492,0xd4f2b6b1), LL(0xaba58423,0x0ab0b63f),LL(0xd8791541,0x0c6d3499),LL(0x81abb7ed,0xc911fda1), + LL(0x53f134aa,0x4316e48a),LL(0x492e43e8,0xc26f2068),LL(0x126d373f,0xaea01397), LL(0x37d4baed,0xdfb016dc),LL(0x68550352,0x70fd4cd9),LL(0x90c02756,0x82a6c302), + LL(0x66cf543d,0xbbfbd662),LL(0x9f372f9c,0x9198ae05),LL(0x0d3c6392,0x548bc0b0), LL(0x71100572,0x5e227e8c),LL(0x189496d9,0xbd10e2ef),LL(0xb6a6798f,0x66b7ddd2), + LL(0x1d9b4627,0xdbfa89d2),LL(0xe9722687,0xa4144f05),LL(0x62a9f0b3,0xa03a6658), LL(0x09a6ee11,0x11e24e34),LL(0x610d808b,0x02a2f96d),LL(0xb68d12eb,0x3434da69), + LL(0x420733a0,0x224f3f9b),LL(0x99244104,0xa3f7e5c2),LL(0x9dd4dd08,0x2018ee4f), LL(0x230f84ef,0xb993d248),LL(0x69d385df,0xbb1c8f5b),LL(0xddffd0f2,0x9242e20a), + LL(0x09654b6a,0x6d842a62),LL(0xfe652314,0x2f79695e),LL(0x193c89e2,0x0c65ea9f), LL(0x6b79b085,0x009cf4f9),LL(0x2268da91,0x390256cc),LL(0xb23a9a13,0x9a5602a1), + LL(0xc087ed5f,0x7e6b4f66),LL(0xa1c63549,0x14a8e4d1),LL(0xfed4b006,0x01ca8c96), LL(0x6933bc42,0x1f5e13f7),LL(0xcc727c9e,0xd642cd3f),LL(0xa4a52672,0x60a81922), + LL(0xcb37c6b1,0x21d4016a),LL(0x6aaea4e2,0xbf9c3ad6),LL(0x4d227e29,0x9bd42e82), LL(0x66fe8ded,0x7b43dfb3),LL(0x4f6d7cab,0xccc64f49),LL(0x709cf722,0x9fb84f64), + LL(0x99a34b60,0x657bba39),LL(0x5bf3ff4c,0x54c86454),LL(0x6a3fc2c9,0x91cab991), LL(0x9d937181,0xad3baab9),LL(0xa318a96e,0x9b39648e),LL(0xd5a8dd54,0x416fe984), +}, +/* digit=26 base_pwr=2^182 */ +{ + LL(0x10e89d46,0xa5d2d3fc),LL(0x267decf4,0x426ccb6f),LL(0xf65161a7,0xc430a435), LL(0x45ff08ed,0xf32e4fd9),LL(0x52fce424,0x29bd88e4),LL(0x7df49592,0xb58dca75), + LL(0x2de63b4f,0x08b2eb9d),LL(0xc5a70a53,0x2dd60214),LL(0x183b9f0a,0x8bf59ae3), LL(0xa4d75e6a,0x127db136),LL(0x2dd19a85,0x70a4c22d),LL(0x6bb9e133,0xa04ca68b), + LL(0x1e0ac0de,0x7c1f8f54),LL(0xcc70b75d,0x33fa9eda),LL(0x1661ad09,0x190f2578), LL(0xc6af0599,0xc8d29939),LL(0xd76f1d21,0x8f0f6073),LL(0x28d0432c,0xda9f9fbc), + LL(0x9c65f79d,0xa73ce4f4),LL(0xcb12b4d2,0x81728494),LL(0x34a30e52,0xb3709952), LL(0xcc5be142,0x34e61905),LL(0x0d1b9183,0x0237c5ce),LL(0x51d9f58a,0xcd502078), + LL(0xb77a9580,0x5935c009),LL(0x60590762,0x109f262c),LL(0xa9bef637,0x40cd7b02), LL(0xe1e887cb,0xbcf02172),LL(0x22fc3d18,0x23ab267e),LL(0x2b110de6,0xbb9632d8), + LL(0x1e71e027,0x939849be),LL(0x0916d6dc,0x04493ee8),LL(0x7a4c25ff,0x2456a4fa), LL(0x80e2c3af,0xbd7a130d),LL(0x19ebceb4,0x63b50b49),LL(0xca679a32,0xcf3af9f6), + LL(0x568806a1,0x8bf5ebcd),LL(0xd6906912,0x9d9e980d),LL(0x2224ffdf,0xbc01567e), LL(0x9b382f20,0xec45fad8),LL(0x2ae9b081,0xc81ce8e7),LL(0x98932797,0xe492d96d), + LL(0x29a231ea,0x5d2a1162),LL(0x59e196df,0xa3a8d097),LL(0x7f4b3fb1,0xe049a755), LL(0xc6316d29,0x5a3a8033),LL(0x927eae96,0xcfa30505),LL(0xbe935448,0x08b8a76c), + LL(0xedc1da3d,0xae5b906d),LL(0xe511be7d,0x7e4c4f50),LL(0xbf36de87,0x8d50de21), LL(0x0ed128b9,0xa3e529b5),LL(0x39d260df,0x902d56ab),LL(0x5e3d15b3,0xaafa263e), + LL(0x0e6cecf4,0x201392c1),LL(0x49f3f177,0xf3e51d03),LL(0xcb103c3c,0xa778f941), LL(0x7666e871,0x663a85f9),LL(0x67c98f13,0xa46b0c85),LL(0x3d7ea941,0x3419ecca), + LL(0x0cd251b9,0x6e9c319c),LL(0x740ebef4,0xe03a5e58),LL(0xeb45dea1,0xef0db2a0), LL(0xe6e4e097,0x12e7626f),LL(0x19914347,0x27cfc38c),LL(0x6046e4f4,0x92b17248), + LL(0xfd260bad,0xad9a809e),LL(0x84cf1f3c,0x0faa61f5),LL(0x6afd71d3,0x7b60a72c), LL(0xd1fd427b,0x67f98a21),LL(0x45809f86,0x55e94be4),LL(0x8e08a8fc,0x55f38c03), + LL(0x78ac4a59,0xe1bd82eb),LL(0x80599bf6,0xc195517f),LL(0xcba34b90,0xd57d73bd), LL(0x68e80682,0x70b22ccf),LL(0xfbe46433,0x9ef082ba),LL(0x4e1de964,0x0e3b2665), + LL(0x629523a0,0x40c1bd3d),LL(0xb3c7fa46,0x55c4d1e4),LL(0xc3ac04c5,0xe731d1dd), LL(0xb98ea38f,0xee1fa82d),LL(0xc3da8420,0x2a53d82c),LL(0xf22acfea,0x368c36c9), + LL(0x29b755d0,0x0266ab97),LL(0xcd211255,0xf06c3dd8),LL(0xda4b92c3,0x815c935c), LL(0x7c35910f,0xc8b3d452),LL(0xe14a590c,0x0f3aed26),LL(0x8e1ce1c0,0xf6ecbf3a), + LL(0x574f2fbb,0x552b5d7a),LL(0x8de8d85f,0xfa3986c3),LL(0x9614d372,0xaa53bc82), LL(0xc58f40f9,0x1118e769),LL(0xe3a00d02,0xa39c3653),LL(0x92267b25,0xb0de6989), + LL(0xc7b64858,0xa306b7c8),LL(0xb64b2f3b,0xf22489d0),LL(0x277347b5,0xb4d82d56), LL(0xb3c23db8,0xc6120600),LL(0xeb1d672f,0xdd104bc6),LL(0x91821d86,0xf46c3e19), + LL(0x182aa649,0xbd1809b8),LL(0x9def0d8a,0xa14bcb07),LL(0x150bc2b2,0xbc355f52), LL(0x303a6966,0x9b74419e),LL(0xeb2b3cb5,0xf922f6bf),LL(0x18f7ca08,0x5d7aa60c), + LL(0x0be5968b,0x42346530),LL(0x2a740a40,0xe0a37f25),LL(0xe665427d,0x03bd64c1), LL(0xe4074164,0xe3911c26),LL(0x7960eb4e,0xc9770b1c),LL(0x5c67feb2,0xbe470d6d), + LL(0x3894d144,0x3b30ad55),LL(0x16768125,0x51da7959),LL(0xbed864b3,0xe8cfebda), LL(0x833b7cb3,0x49a98925),LL(0x6f174ec4,0x830c736e),LL(0xcda2e7e0,0xf9efa80a), + LL(0x376e70ba,0x01f86bbf),LL(0x0eddf0bb,0x686b6cb6),LL(0x64d59f33,0xdc9eec10), LL(0xa46828be,0xdb7037cf),LL(0xd2c7ca5a,0xceeb8c28),LL(0xe6196bda,0xe40cbf09), + LL(0x8bd42f71,0x0ac2d045),LL(0xb62d83fe,0xd4215f7e),LL(0x03acf7bd,0xaab77a58), LL(0xc2eeb2e0,0x7e1aa3e8),LL(0xd86c3480,0x16dbe858),LL(0x8a7fb0b3,0x65a386cd), + LL(0x8894f88b,0xc426e3e1),LL(0x4593936f,0xa748da73),LL(0xca40633d,0x9e640d2a), LL(0x02ac44dc,0x84fd2be4),LL(0xd437cf7f,0x0471f362),LL(0x7306c123,0x8f4155e7), + LL(0x21dd0f82,0xc9871691),LL(0x28d633a3,0x2bd96ea5),LL(0xfc2ba2af,0x1906ae24), LL(0x1ee92586,0x2b4efac7),LL(0xf7f07861,0x4b97cbfa),LL(0xf7281f7e,0x4dbf54e2), + LL(0x25b5b11c,0xc39fffbf),LL(0x98a6b660,0x41973093),LL(0x0f7b07f8,0xd02dcfda), LL(0xfacfc8d6,0x42bd5635),LL(0xfa119a0d,0x49c754a0),LL(0x7a67703b,0xf3900a31), + LL(0xb7648cd6,0xffe76894),LL(0x973e597b,0xa790612b),LL(0xd6d78341,0x5a817ff0), LL(0xc9960802,0xe4a6c1a8),LL(0x6884bf58,0xa296fc0a),LL(0xcaee199c,0x8387842f), + LL(0x78a21751,0x505a5fbc),LL(0x04338ed3,0x8d557dd4),LL(0xf8316933,0xc6daa6eb), LL(0x9fbe57c9,0xce8b52fc),LL(0x82ffca53,0x93375f08),LL(0xe48a4172,0xd2e09822), + LL(0x2d8b1f67,0x59d6d11a),LL(0xf984b355,0x3df9851e),LL(0xeef602c2,0xfad1f4e1), LL(0x1df4b332,0x6c07fd2b),LL(0x0a339f9f,0x1672e0ae),LL(0x3e2e73f1,0xa7aac7c0), + LL(0x53de8d30,0xd015e91f),LL(0xdeee9e90,0x9cef3987),LL(0x9b54ea0f,0xcae1fbee), LL(0x0b2e4bfc,0x5541f0dd),LL(0x00c1a228,0x15f9dffc),LL(0xd9dfb7f7,0xeed05a2b), + LL(0xf8bcb7fb,0xbeb350fb),LL(0x3fb74891,0xa6a2efe1),LL(0x4216f4a5,0x1cfaf6a0), LL(0xccd45bce,0xd268b24b),LL(0xe3db6dfa,0x5c85e954),LL(0xf04e8161,0x127ce38c), + LL(0xfc6c17ef,0x749016b3),LL(0xc849c80f,0x840e079d),LL(0x55aebfa6,0xef3493b4), LL(0xbdddb794,0x5645586b),LL(0x0a0da6a8,0x1e1ddb6d),LL(0x33397277,0xbd31053b), + LL(0x257efe71,0x64c4bcd5),LL(0xf7abc98c,0xd35b6fad),LL(0x49479a50,0x3b0df461), LL(0x0c01f68e,0x7aa9ea28),LL(0xfc247d85,0xa37c8d40),LL(0x82d5d062,0x4214db8f), + LL(0xd4665035,0x615f976f),LL(0xb2fd9161,0x39f27e8e),LL(0xce638414,0xa1ae7fee), LL(0xbee083df,0x0717218c),LL(0x80a73a7f,0x66d2bccf),LL(0x77555a78,0xc1cfe44c), + LL(0x151f9185,0xab6467ac),LL(0x25664569,0x6df53a18),LL(0x466dd4b8,0xa54fbfd4), LL(0x8b3c8dc6,0x6f063338),LL(0x41623458,0x64adde6f),LL(0x112ad869,0x08056bf1), + LL(0x3434a31e,0xfdd32b7a),LL(0x04cdeecc,0x5728361d),LL(0x3a7b8da1,0x2aa81490), LL(0x09a2731d,0x87c2d7d2),LL(0x89ebca03,0x1f3344b9),LL(0x19158d11,0x3b4eb6aa), + LL(0x4aea58b5,0xc2bc01be),LL(0x19f12b28,0xf7d53cd7),LL(0xdf3da230,0x475fcd61), LL(0x09990316,0xeb3dd742),LL(0x28ea4a85,0xb1c62ec2),LL(0x71770efc,0xd9ca3e02), + LL(0x285cb4ae,0x92099143),LL(0x0d8f3f4f,0xfac421c8),LL(0x31e4afb4,0xb20b6d02), LL(0x41735275,0xb896f118),LL(0xe7a40b45,0x9f8d1d92),LL(0xc574911e,0x578117a2), + LL(0xc8ab5906,0xe049b850),LL(0x90ebe14c,0x3dc8761d),LL(0x3d9b21b1,0x01b959ad), LL(0xa1be26d8,0x69e0edfb),LL(0x90bffa45,0x23061d51),LL(0x5432258c,0x12c49398), + LL(0xec1ab63d,0xcf6318e5),LL(0x78baa5c1,0xfabf5dcc),LL(0x6529af85,0x6fe5da1b), LL(0x0d696bed,0xc02b8cbe),LL(0xc83d0478,0x5865de25),LL(0xf422a123,0xaef1328d), + LL(0x700b22a4,0x0895f5d0),LL(0xf78d8bb2,0xb52e7e2d),LL(0xe0972d87,0x5771d2c8), LL(0xa67e68d5,0x45181514),LL(0x29446b80,0x11ad6a2c),LL(0x3c18bc56,0x17b2b4e7), + LL(0x075ee3b2,0x3d4755d1),LL(0x0fcec9d9,0x1505789c),LL(0x398f0c91,0x6e00b31e), LL(0x57048d0c,0x54cea710),LL(0x6443fcff,0xf40ba9be),LL(0x05b2a1b2,0x04075b8f), + LL(0xf3aa488f,0xcb23f2f6),LL(0xb3aa9bab,0xd48b8a3e),LL(0xa8b58c26,0xe1dfa993), LL(0x58d04b7c,0xe2641538),LL(0x32ad4a3c,0x1159caa2),LL(0x00256f11,0xa19210d5), + LL(0xeb897a63,0x52787df1),LL(0x3afae814,0xe4f787d2),LL(0x9a665638,0x3c391a2a), LL(0x49d18f8c,0xa7cbf5cb),LL(0xd351d0bd,0xb159a6c2),LL(0x57b9faae,0xe501bee6), + LL(0x7ffa6fb7,0xfd21c745),LL(0x6eb966ff,0x9f082577),LL(0x6cf4bbb6,0x7bc7fd6a), LL(0xadae7e60,0x01fbbec5),LL(0xdb5201fd,0x8ede95fa),LL(0x505dc527,0xfe027d78), + LL(0x4ab8d622,0x556427f0),LL(0x2d8a1b65,0xac06be29),LL(0xc1517a5a,0x24ffe89e), LL(0xbb687bbe,0x8ed6619f),LL(0x5e6229f5,0xc391ab29),LL(0x6668e363,0xc6accdea), + LL(0x2236abab,0xaa1a8aae),LL(0xbbbb5c34,0x7ff0f90e),LL(0xcdbbe595,0x342421e1), LL(0x158fa276,0x3a43eb45),LL(0xb1c8d9e2,0x3f206b59),LL(0xd1fb731b,0x926495ee), + LL(0x772ae335,0x64dee795),LL(0x27bb58cf,0x5536e14e),LL(0x68065d10,0x559824cb), LL(0x5fca59f5,0x6780d28c),LL(0xc5672db7,0xb1f51778),LL(0xe64c87fb,0xc295e9dc), + LL(0x6a759ba8,0x43aa71ca),LL(0xed6e67a5,0x296cde1c),LL(0x2481f6c7,0x97eff2f8), LL(0xee4bf211,0x311c89d7),LL(0x20c5e8fb,0xa03cf7db),LL(0xa30c9a13,0xf181425c), + LL(0xef8041bd,0xe9164a69),LL(0xf14e75d4,0x10616709),LL(0x31432c48,0x3d0fda80), LL(0xedfce733,0xff34f02e),LL(0x97785106,0x47e53d79),LL(0x0c42b429,0xb73df841), + LL(0x16d00d56,0x430590ae),LL(0x0b6db145,0xb36bf8a4),LL(0x631b8af0,0xf0fbf1b8), LL(0x7c97735a,0xd285813a),LL(0x233bbcb4,0x3be81538),LL(0xece4421e,0x5dce4f9e), + LL(0x115acbd2,0xdf3aafe8),LL(0x4e168fd2,0x01352bc5),LL(0xafcfd633,0x7948b02b), LL(0x559eebfe,0xd8976862),LL(0x8de4fafd,0x752a202a),LL(0x47600461,0xf9964960), + LL(0x45629c12,0xd8e2a4f5),LL(0x0e8d9c56,0x53d8b1f9),LL(0xce8a7e4b,0x69f85fa3), LL(0x50234689,0x78e35e04),LL(0x8ffbcfd7,0xe0274e82),LL(0x0c75b636,0x3bed2d8f), + LL(0x373fcc37,0x5ad6f7e7),LL(0x347b8ead,0x3ae4bbd9),LL(0xdf9414dd,0x3ef24805), LL(0xed9355e0,0x5d655d51),LL(0x12d96970,0x93e831a1),LL(0xa1cc5116,0x9d02f97d), + LL(0xc8216057,0xad5bc00d),LL(0xa7efc99a,0x7aa8806c),LL(0x63fe1b91,0x1abc67a2), LL(0xe3315539,0xdcef61b3),LL(0xbd3f1ea2,0x31a563f8),LL(0xb76036c6,0x86e84857), + LL(0xc967c01d,0x22a2bca9),LL(0x6184d879,0x15300b29),LL(0x24018b6c,0xcb4eece3), LL(0x4bebca46,0xee715ca1),LL(0x0bbbd9c5,0x8d9daf6a),LL(0xd918c1bd,0x683a7647), + LL(0x53eb0ffd,0xeda60e7d),LL(0x71a51e14,0x537aefa1),LL(0xcd7ff005,0x7927da15), LL(0xf7f04011,0x6191e2df),LL(0x4bba4547,0x7bedb9cb),LL(0x89b2a74a,0xa50562a9), + LL(0x606c84ad,0x17eb0b47),LL(0x7c97dbb9,0xab88aa51),LL(0xd1818d8b,0x07ea03ae), LL(0x9374b72b,0xf73e2f04),LL(0x86858702,0xcb29129c),LL(0x98608a71,0x3620afda), + LL(0x716a5874,0xb810b51e),LL(0xa889add3,0x9ec17162),LL(0x19089d83,0xfd276227), LL(0xbb1c7343,0x063aee05),LL(0xe784bbe9,0x87cb9fc6),LL(0xd7489e40,0x782d098e), + LL(0x8f090898,0x7c43b0a9),LL(0xbdca71ed,0xf8f3b085),LL(0x3f9f4e6e,0xc9b19fd1), LL(0xc47961e6,0x6e42f7f0),LL(0x31020f60,0x45aaf72a),LL(0x2550f424,0x661616e1), + LL(0x20db7e4a,0xb16f561f),LL(0xa4281dd3,0x83f2a22b),LL(0x43f5d715,0x31f11530), LL(0x1a9f9532,0xbb3cc73e),LL(0xf1fe503c,0xe86e672b),LL(0x6997be05,0xf4c9b96c), + LL(0xe3cb641a,0x4b0bb45e),LL(0xb5ddb389,0x741adc8d),LL(0xbc920f0d,0xd87c5699), LL(0x15b6ef33,0x55bcf8a4),LL(0x1245ec9c,0x68eb1b09),LL(0x4c03f712,0x7b692b42), + LL(0x829adb45,0x835194bf),LL(0xe4bd2b50,0x5f3fed6a),LL(0xfffe5c2e,0x32e24c5e), LL(0xb9a052b8,0xed361d44),LL(0x0b43353b,0x512fea09),LL(0xd5a71bbb,0x1d35cd1f), + LL(0x05b5e6d9,0xba48388b),LL(0x3b0b203b,0x07136b21),LL(0x765b8607,0x8ab4af31), LL(0x74816f5a,0x5107c05c),LL(0x4a5d27b7,0x8d7cfad6),LL(0xd9c2aa8d,0xbc507c9c), + LL(0x467fe1f2,0x6ed919c9),LL(0xfb1bf823,0x9a75c404),LL(0xfefebd90,0x5eff17b8), LL(0x521a3668,0xa8b6ed77),LL(0x251c6e8c,0x0f8b8ae7),LL(0x631669c0,0xe45c3453), +}, +/* digit=27 base_pwr=2^189 */ +{ + LL(0x65fd193b,0x443763e3),LL(0x98d2ccd9,0x23523a88),LL(0xff0d967e,0x0a6f8f92), LL(0x009db3b0,0x1263e468),LL(0xbd562f47,0xee4dc8c1),LL(0xafc849c5,0x76f03c5c), + LL(0x99bd9bfe,0x8d976c26),LL(0x58ed4760,0xb19a85a7),LL(0x1e34868e,0x13dca814), LL(0x54cc6d8a,0x056e40ad),LL(0x8fcf3241,0x861dc2cf),LL(0xd3afc6cd,0xa6a8a21d), + LL(0x7b77442c,0xe6eeae3d),LL(0x8aa82aef,0x057453a1),LL(0x9a9a0869,0x5a1dd863), LL(0x8372157f,0x15b34a60),LL(0x892b8e9a,0xc35f2257),LL(0xcdd030f9,0x0476b29a), + LL(0x490b43e5,0x1f623ab9),LL(0x6d3b5faa,0x0fb153e0),LL(0x126be417,0x11ecf7af), LL(0xf9d44cdb,0xc5588fe3),LL(0x80576c99,0x93bd13f4),LL(0xf7cca387,0x458f50aa), + LL(0x05cadd86,0x33e0840f),LL(0x1288d2ab,0xa4ceb96c),LL(0x2574cecd,0xf18de0a1), LL(0xbd606235,0xb1eecb58),LL(0x243a5639,0x46f1c0e3),LL(0x2f8617f4,0xfbf9ada6), + LL(0xfdc33e8a,0x29778c36),LL(0x7c7a4b81,0xe96d19b3),LL(0x87bd6ef2,0x011f2a94), LL(0x7b36e1c8,0xd84b0388),LL(0x6e6d91ea,0x5f21366d),LL(0x670039fe,0x0c884a47), + LL(0x5485e744,0xdff19ddb),LL(0x826523ab,0xb626116c),LL(0x10d50cd3,0x2f4f0fd8), LL(0x24125350,0x5dfd1acb),LL(0x53f2ab5e,0x09c128b0),LL(0x4f9b2256,0x9f940ced), + LL(0x917ef295,0x347c7bb6),LL(0xe7525a0c,0x3578fbbf),LL(0xdaade6c5,0xd64bdb43), LL(0xeff35295,0xdb9014a1),LL(0x8bd89301,0x1c18a193),LL(0x8573ec51,0xccd8059d), + LL(0xff89a745,0xd263620d),LL(0x8b578c94,0x485093e4),LL(0x831165c3,0x6cc8df06), LL(0x72a1e4bb,0x623c1f86),LL(0x1ae4359b,0x1857c157),LL(0xf8c83db9,0x51252b88), + LL(0xf3eaa85f,0x75c0f153),LL(0x296331ba,0x8524e8a3),LL(0xf153b5e3,0xbdfb652d), LL(0xf3645389,0x650adac3),LL(0xe5150ccd,0x59382d93),LL(0x40e57d00,0xf3310865), + LL(0x3135c8cc,0x43c89d80),LL(0x8ef6e51f,0x5f0b989e),LL(0xe67cd4d4,0xc6b0f722), LL(0x2433fada,0x35eb83b7),LL(0x688247c4,0xb18db37b),LL(0x51703d00,0x0a444f19), + LL(0x57840670,0x8361e5ab),LL(0x42af05a2,0x4f46eb2d),LL(0x4520d223,0x34632e05), LL(0x66bdbe8a,0x4aa80691),LL(0xb5742363,0x467d5d3a),LL(0x0894ba88,0x5af9783e), + LL(0x0cb7f96b,0x84644ff9),LL(0x522d6aaf,0x0ae33ac3),LL(0x2cc8944b,0x017c4b01), LL(0x6a2ddda6,0x9923bca6),LL(0x8c9b621e,0x6627fcc2),LL(0x17bdfa4f,0xa29f1b1e), + LL(0x1cad89b6,0x534ff925),LL(0xc2042459,0x0863e52b),LL(0xaf25d3a6,0x5306b082), LL(0xbbbd9f5b,0xabb1a0b0),LL(0xb65cde23,0x7e3bd9f2),LL(0xc228a0ec,0x001c50b1), + LL(0xb9339a10,0x86cb78d0),LL(0x1caa6cdf,0xa4b51028),LL(0x21d335bb,0xc52bad81), LL(0xc2fe7de5,0x4301dba7),LL(0x7d6b198e,0xc7b39fd3),LL(0x2a96f21b,0x28e0fdea), + LL(0xbabe8679,0x59373e25),LL(0x9fd9ed30,0xd44cef09),LL(0x58210099,0xfafb37c0), LL(0x1174c9f1,0x19aea5a6),LL(0xaa0d14c2,0x95527d4f),LL(0xabaf1d12,0x3f19e7e2), + LL(0xcb8a82f6,0xe6d77312),LL(0x63ded2df,0x1e5c2945),LL(0xa2eb913f,0xe6eba908), LL(0xd2a8cdaf,0xe7a562f9),LL(0xa209420a,0xc659bdc9),LL(0xc59d4edf,0x9d016dd4), + LL(0x74da1b13,0x4bdd6da2),LL(0xe9df9fa8,0xcdee393f),LL(0x2e6bac46,0x64890274), LL(0xb2d61427,0x8843a6bd),LL(0x6efc5b51,0xfd6c2073),LL(0x035128e4,0x12029497), + LL(0xbe06abde,0x680d1bf9),LL(0x452f6c1c,0xb37decb2),LL(0x6ec32edc,0x981e840c), LL(0x84360e72,0xd5257167),LL(0x18300091,0x558abf57),LL(0x72ecaad4,0x3a7b592d), + LL(0xf7312883,0xcdc53ab7),LL(0x0571634d,0x69af637c),LL(0xc7cc70ca,0xec10a5e8), LL(0x424399ce,0xa90c48d4),LL(0x341ddfbc,0xe05856ae),LL(0xd3e4c39c,0x3a1976b2), + LL(0x90220209,0x9ed5893d),LL(0x81b6ced6,0x45ae0a2a),LL(0x546a8d70,0xe929ce12), LL(0x455cdddf,0x267c3d5b),LL(0xcea5169b,0x4d9fa92f),LL(0xe94a3b42,0xb925af46), + LL(0xe118cce2,0xe404af85),LL(0x8fc5766e,0xd5a91ab7),LL(0x5e924307,0x71c251eb), LL(0xf7a25052,0xe3cb6d55),LL(0x18b850c9,0x968fed5f),LL(0x2174cb4b,0x623893ba), + LL(0xd3d0e873,0x67d096e1),LL(0x2541a85b,0x4487615c),LL(0xfa518d4c,0xe87a5c0f), LL(0x8371fee6,0xb0b2a717),LL(0xf0d1d1fa,0x0c2e5455),LL(0x11189a5d,0x42cf786c), + LL(0x0075ca1a,0x6477d9cb),LL(0x0bada61a,0xff28d896),LL(0x7d671caf,0xea81a421), LL(0xe5e640ea,0x40762d23),LL(0xd43891af,0xfda94df7),LL(0x527b4662,0x6349fb9b), + LL(0x3b2603b3,0x3b5ff95f),LL(0xf23c44e6,0x6b633ea1),LL(0xe90ab3cb,0xb54db154), LL(0x37f6bcde,0xa92deb1d),LL(0x4eb88b8c,0x7b328bf5),LL(0xde5545e4,0x4ce336ee), + LL(0x1750434d,0x072846ed),LL(0xef744765,0xef46faee),LL(0xf439980c,0x7a858840), LL(0x6aecf70c,0xec75222e),LL(0x1a4ef06e,0x2ece8f59),LL(0x3d233bad,0x82784973), + LL(0x3b3b6956,0xaee6f0e5),LL(0xf4eb1f77,0x04ceab91),LL(0xdcbb09d4,0xc6977800), LL(0xc8203c8b,0x8948d95e),LL(0x4f31a369,0xa354e6e3),LL(0x42785a63,0x10e7ebf6), + LL(0x426a2da6,0x5c233e59),LL(0x5aff8031,0xfc7c21ac),LL(0xae2b7618,0x713ae9f6), LL(0x8ffa178c,0x958babfe),LL(0x404c1ece,0xb21fe414),LL(0x1e4f7c28,0x1fd8cf6a), + LL(0x482a1f04,0xb68bc919),LL(0xfbee302f,0x7ff1f4be),LL(0x6f98c545,0x5d510b76), LL(0x04a84a52,0x0a54942b),LL(0x8d306d4d,0xdc70958c),LL(0x638a365c,0xd5564cd0), + LL(0x295ed5b7,0x7275aaed),LL(0xa7a642b1,0xc6c05b8a),LL(0xa499ac12,0x5572818c), LL(0x038c961d,0xdf0a6fa3),LL(0x798ec56d,0x7ed5630f),LL(0x4acc0009,0x206c39e3), + LL(0x63cc9e62,0xc371e275),LL(0x6e06e663,0x425ae991),LL(0x155a4978,0xa2aa623a), LL(0x82b7846e,0x021e0916),LL(0x07ccc4b1,0x0500f013),LL(0x891bba46,0xd62661e5), + LL(0xd612756b,0x38909c56),LL(0x090f7ae5,0xd362d7a3),LL(0x9a871772,0xaf3ba39c), LL(0x575c94d9,0x28dfce6b),LL(0xfaf575ae,0xb556c52c),LL(0x710956bf,0x6c0131b7), + LL(0x071961ab,0x33ce3aed),LL(0x69b35991,0x226aa630),LL(0x2398b3af,0xc42f173d), LL(0xfc2e6625,0x5d4d9cca),LL(0x4790b7b4,0x767e6911),LL(0x184e1458,0x81a0782f), + LL(0x4a451326,0x77aab96a),LL(0xe916fe1d,0x8ba50ba3),LL(0xea83134c,0x4e839eec), LL(0x10e0b57b,0xc1f0f693),LL(0xe2b978ff,0xfd32f86a),LL(0x207746dd,0xa4737150), + LL(0xa3db3bbd,0x454c0df4),LL(0xb0f36f88,0x0edad6a0),LL(0x5dcf2b92,0xe233816f), LL(0x53dab969,0x25098373),LL(0x102254b6,0x436f8133),LL(0x4eedcc6f,0x9ce8f3ae), + LL(0x4b60ac27,0xf64e0fad),LL(0xe3f9cebf,0xea44c9f8),LL(0x84fe9a3a,0x719d958b), LL(0x27bf9bd3,0x5a5e47e8),LL(0xf50de48a,0x6f8c6ee1),LL(0x1971c6f2,0x5dc81c5f), + LL(0xe435f073,0x1e41a11d),LL(0x49ef9817,0x7b20a534),LL(0x111d8548,0x1ab1fb5a), LL(0x43450972,0x40d481ef),LL(0xddcf8585,0x28382f8b),LL(0x1b0f6b65,0x5a75f9e6), + LL(0xb1f8052b,0x41ac6def),LL(0xff4c9377,0xc417b22b),LL(0x3ba0240f,0x1c7a1067), LL(0xc99883cb,0x61005fe1),LL(0x6433295e,0xde900b52),LL(0x97d9ccb1,0x1eb1f666), + LL(0x65928ad1,0x86410ced),LL(0x05383f3a,0x281a9d46),LL(0x11cdecb0,0xa2b718a9), LL(0x6fb1cbee,0x67043aa6),LL(0x94dc2a31,0x5e2de7f3),LL(0x18e2d13b,0x7796cacb), + LL(0x413f1989,0x18e05e2d),LL(0x3e5cc915,0x2b2b27e9),LL(0x8836a712,0xa32587cf), LL(0x34c64b87,0x0cf77bb0),LL(0x6cafe545,0xf4d7f7ca),LL(0xa18bc173,0xf2191e2f), + LL(0xaf600892,0xc338b3de),LL(0x35ee6915,0x43c76ec6),LL(0x5564fc33,0x4fdad82e), LL(0xd93b474f,0x089a676d),LL(0x62b06515,0x44a02780),LL(0x2067dba5,0x65fe8051), + LL(0xf6549736,0xf1c24fd9),LL(0xc063acd4,0xf9d94656),LL(0x95a57c69,0x20c24772), LL(0x8993633e,0x91f7d4d6),LL(0x4444d45b,0xe9d75b4d),LL(0xb5bcc9cf,0x3b5a8bf8), + LL(0x1b093e28,0xf93e1494),LL(0xfcd88fcf,0x8ce5f30d),LL(0xbfcff34f,0x8fe20824), LL(0x7eec26d2,0x23a22164),LL(0x72404851,0x9b0f8456),LL(0x0e6c550f,0x135ff4a9), + LL(0x6ef7ad8c,0x70a42efc),LL(0xd82f0cec,0x2f6af7cd),LL(0x7a83b569,0xa0e0549b), LL(0xd3adf095,0xf580e86c),LL(0x8712a121,0x3c2d891a),LL(0xb9d07525,0xf1f2cd22), + LL(0x461ab8fe,0x7e90bc98),LL(0xc3e7e4df,0xf27a52f3),LL(0xe71da883,0x99813684), LL(0x5c5d56d6,0x2bc98a5a),LL(0x9d789345,0x17248209),LL(0x7dd92590,0x3312dfe6), + LL(0x26c45ea7,0xfeb5bd64),LL(0xea91cffa,0xfd9bbc06),LL(0xb59f6e6f,0x8be51e9c), LL(0x12663168,0x2082f3c9),LL(0xb78ce9a5,0xe57a2832),LL(0xe4f54287,0xb0d7aac6), + LL(0x338a630c,0xdad61bc3),LL(0xe66f4efa,0xcffda1bb),LL(0xa61cc9ef,0xccb2e20e), LL(0x3e09551f,0x8d2efcc4),LL(0x617bb37e,0x4e8f46b8),LL(0x2d9b4751,0xc516bc0f), + LL(0x8992f590,0x753b4f58),LL(0x994a51af,0x55461379),LL(0x0d16de56,0x9f0a37e6), LL(0x204a2e7c,0xb346ff69),LL(0xce235822,0xfaef890d),LL(0x98b923c7,0x5b046b64), + LL(0x29e950a8,0xb55daedf),LL(0xd3a1ff37,0x79b58c3a),LL(0xec3ed694,0x72c9d2d9), LL(0x9ddf8225,0xd119d76b),LL(0xce66c9a2,0x330952af),LL(0xb0e5e0dc,0xe062e16a), + LL(0x2d1cc8bd,0xe8cd1c24),LL(0x799a5052,0xf2f0da35),LL(0xbe0be8af,0xe52e337a), LL(0x6a2cbabf,0xeae205cf),LL(0x0acc8002,0xdddc1a94),LL(0x3d1031df,0xf559ec0f), + LL(0xefc11888,0x5d22ffc2),LL(0x4ff02455,0xdbb1c1ef),LL(0xc63b8f6f,0x543b4752), LL(0x883a7c99,0xd08ef2e5),LL(0xf38b3b3c,0x58ac93a0),LL(0xd8cac2ad,0x3c6c8105), + LL(0xdf1587c5,0x538bffc6),LL(0xa1a2eb88,0x0d5a4dd0),LL(0xc155f055,0xb9730932), LL(0xb9b8cd61,0xa21c36d2),LL(0x5320c28d,0xc435c223),LL(0x38120687,0x3230ce02), + LL(0xc37ce54a,0xea0530df),LL(0x06b232a0,0xb1894861),LL(0xf018336a,0xca2122b1), LL(0x38a380e9,0x08135d85),LL(0xe3028bd7,0xf256c6e5),LL(0x663fdf24,0xdacc9271), + LL(0x053cff11,0x31da52c7),LL(0xcc61252b,0x39051bad),LL(0x317a0ba8,0xf380f3e8), LL(0xd42451b7,0x212de129),LL(0x98e4747e,0x146f3c25),LL(0x2acac2c4,0x844da479), + LL(0x4cabc97f,0x9b8a6fb0),LL(0x0fbc8135,0xba6da1e3),LL(0x561f0e02,0xb076c93d), LL(0xf54d9d7d,0x2f940cf1),LL(0xec9daef2,0x14a542cb),LL(0x19d5ffa1,0x0579ad14), + LL(0xd4065d34,0xf8f179dd),LL(0x30ff0d6d,0x355fc75d),LL(0xc8fac2e6,0x91f6f5fb), LL(0x3e3fd917,0x8baf55f3),LL(0x077d4cef,0x6c4a7378),LL(0x192e1ce6,0x16be9fa8), + LL(0x92e5aec0,0xa3a9011c),LL(0xf37a5ab0,0x4fa0764f),LL(0xb1e1e01a,0x75ffe5ac), LL(0x51756cb8,0x3cd4ee19),LL(0xe2636498,0xc557a98c),LL(0x2c691a68,0x53a49769), + LL(0x752dbe26,0xdcdf8e6f),LL(0x7564740f,0xeaa9a260),LL(0x31a5df7f,0x9298d7e2), LL(0xa1b1f37a,0x47cdc94f),LL(0x3c25a836,0x26578ece),LL(0xec817acd,0x0aab1d74), + LL(0x5911f2ae,0xdd0678c2),LL(0x126b83a4,0xb903d7a7),LL(0xecd46d83,0x4efeef39), LL(0xb0d49ea1,0xbb4b0af7),LL(0xb6c2ae9c,0xc62fa8f1),LL(0x3e453617,0x897a8d06), + LL(0x54820219,0x335e7a19),LL(0x9104367e,0x8ae7926b),LL(0xcf0f8e5e,0xae2225b5), LL(0xd3fddb3b,0xc276e1af),LL(0x6a507a40,0x7b05028e),LL(0xef73bdac,0x85d2bd18), + LL(0xc71dd981,0x6953e0e2),LL(0x0e9a352f,0x7f739d63),LL(0xbe280f30,0xb6620388), LL(0xd020979e,0x89d5450f),LL(0x66d59c07,0x0370df8a),LL(0x0e91d327,0x76a60330), + LL(0x12f802fb,0x56161a39),LL(0x2ca0151f,0xe9e9e58c),LL(0xde14a0f7,0xc6da98d6), LL(0xe70f9647,0xdc21514c),LL(0x523f5cb8,0xa95c863d),LL(0x25f1c818,0x3e1cc20c), + LL(0x8d6f8ca1,0xeb6c9c82),LL(0xb9d65ec1,0xc57e3312),LL(0x0f121596,0x8b4328c3), LL(0x067036a8,0x1affe768),LL(0xe3091467,0x42586aa8),LL(0x30b7ad74,0x12de5d18), + LL(0x34933969,0x06705f70),LL(0x7267ab88,0x4ce99cd6),LL(0x1f916ab1,0x2bc4cfb9), LL(0xcd9b8de7,0xe458fea7),LL(0xedafc50a,0x59778ef8),LL(0x86ea0de9,0x58651c07), +} +}; +#endif /* _DISABLE_ECP_192R1_HARDCODED_BP_TBL_ */ +#endif /* _IPP_DATA */ + + +IPP_OWN_DEFN (const cpPrecompAP*, gfpec_precom_nistP192r1_fun, (void)) +{ + static cpPrecompAP t = { + /* w */ 7, + /* select function */ p192r1_select_ap_w7, + /* precomputed data */ (BNU_CHUNK_T*)ec_p192r1_precomputed + }; + return &t; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpp224r1precomca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpp224r1precomca.c new file mode 100644 index 0000000..4e752a5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpp224r1precomca.c @@ -0,0 +1,4489 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (P224r1 precomputed) +// +// +*/ +#include "owncp.h" +#include "pcpgfpecstuff.h" + + +#define OPERAND_BITSIZE (224) +#define LEN_P224 (BITS_BNU_CHUNK(OPERAND_BITSIZE)) + +/* P192 affine point */ +typedef struct{ + BNU_CHUNK_T X[LEN_P224]; + BNU_CHUNK_T Y[LEN_P224]; +} P224_POINT_AFFINE; + +extern const __ALIGN64 P224_POINT_AFFINE ec_p224r1_precomputed[33][64]; + + +#if defined ( _IPP_DATA ) + +#if !defined(_DISABLE_ECP_224R1_HARDCODED_BP_TBL_) +/* see ippcp_baseptbl.cpp test for generation details */ + +#if ((_IPP_ARCH == _IPP_ARCH_EM64T) || (_IPP_ARCH == _IPP_ARCH_LP64) || (_IPP_ARCH == _IPP_ARCH_LRB) || (_IPP_ARCH == _IPP_ARCH_LRB2)) +const __ALIGN64 P224_POINT_AFFINE ec_p224r1_precomputed[33][64] = { +/* digit=0 base_pwr=2^0 */ +{ + LL(0x6d0a4aea,0xbc905226),LL(0x6018bfaa,0x85259736),LL(0xf96bec05,0x6dd3af9b),L_(0xa21b5e60), LL(0xeff3ede8,0x2edca1e5),LL(0x05335a6b,0xf8cd672b),LL(0x03dfe878,0xaea9c5ae),L_(0x614786f1), + LL(0xe722ff54,0x4ca9a1ec),LL(0x5650df9b,0xd704f84f),LL(0x2a0f1689,0x10c911f7),L_(0xcafb50f5), LL(0x95a24e5e,0x02aac79b),LL(0xc90ae186,0x518c11b1),LL(0x76cc101a,0x0c84ced0),L_(0x5cabb880), + LL(0x86614a9a,0xe37710d7),LL(0xe0b02223,0x24c17ffc),LL(0xc3cf1c5b,0x00505bb6),L_(0xaf8b6496), LL(0x73e4e25d,0x9a73b173),LL(0x04144e62,0x70501632),LL(0x357940ac,0xafe10c35),L_(0x6dfa492c), + LL(0x840f10e9,0x0a72faf6),LL(0x0860856c,0x3c2dbf31),LL(0xcb021706,0x4256e88a),L_(0x89f0c6a0), LL(0x8c1f2b39,0x6743e273),LL(0x98933269,0xd8933d1c),LL(0x2058f0ba,0x7ee104d3),L_(0x8c87db62), + LL(0xd42fac58,0x0b10cc50),LL(0xf8dfc283,0xd80cdd0a),LL(0x975a451e,0x39211b77),L_(0xf7f4fe95), LL(0x64fade9e,0x2b552e73),LL(0x61a9e695,0x5e5610f3),LL(0x11bcf038,0x42b54f79),L_(0xc858da82), + LL(0xde4901db,0xde577c0d),LL(0x1a8aec73,0x09e4f789),LL(0xe0cd01d4,0x1addb3a1),L_(0xf25fe17c), LL(0xdc7c211c,0x5fae6c32),LL(0x0f5c7709,0x5603672c),LL(0xf2d2c41c,0x9b6731a6),L_(0x162f9e1c), + LL(0x899abac2,0x90462820),LL(0x68646e35,0xc6adf683),LL(0x1e2526ba,0xc81df812),L_(0x7ed50814), LL(0x5dd6407c,0x610f2185),LL(0x1fb32889,0x3f3879aa),LL(0xed8ceeee,0xce5eb25e),L_(0x0e5952e3), + LL(0xde1707ec,0xb4c9d865),LL(0x7bd696a2,0x3fcec855),LL(0xa3bebfe9,0x8e4a5712),L_(0x1b59038c), LL(0x29a5f205,0x1bc1ff36),LL(0x7558734a,0x0efc11fd),LL(0x09762590,0xc4e19066),L_(0x2199c364), + LL(0x7b94d3c7,0x1c1ecc77),LL(0x743eda4a,0x23767b5f),LL(0x2f993a50,0x94f5e9ba),L_(0x79d4095a), LL(0x8f776409,0xe07e55b6),LL(0x909bf3f1,0x2b57fa61),LL(0x77e1db6d,0xe733d627),L_(0x5114ac9d), + LL(0xa53cb542,0xb808145c),LL(0xb4f2672c,0x8d51557e),LL(0x3e8abf6a,0xc1f86e2c),L_(0x9d0cdd53), LL(0x8dd67e37,0xccddceff),LL(0x02883b6f,0x94892c1d),LL(0x873d7e4c,0xdbc0f8ee),L_(0x2a687506), + LL(0x6c6825d9,0xac358ca8),LL(0xe1bdd5ca,0x71f50674),LL(0xd0dee306,0x0f060758),L_(0xd6d89093), LL(0xf1b7227e,0x649ea142),LL(0x08181c2c,0x87d166a2),LL(0x6c63713a,0xc1182bf0),L_(0x45e48d79), + LL(0x8f86d9da,0xf6495da1),LL(0xa37b7441,0x5dc634c9),LL(0x2b0b8e1a,0x571d38d9),L_(0x517af941), LL(0x92dd72b5,0x1305b612),LL(0x8cf21835,0x8db28ab1),LL(0x5f59deb1,0xbc243af0),L_(0xa71404ae), + LL(0x3a1f6a38,0x489c8db2),LL(0xffbe563f,0x4943a5ae),LL(0xbc99fb0b,0x68ade63c),L_(0x398eacf0), LL(0xed549cc4,0x83fdacb5),LL(0x800df824,0xb6ab4a8b),LL(0x442ed616,0xb4dc6bf8),L_(0x3f8060b4), + LL(0xb83ee7d6,0xf4109221),LL(0xcdbd6eec,0x565559e4),LL(0x1ef814eb,0xf4550d2e),L_(0xb3a9b5e4), LL(0x6d218d7b,0x1544e90e),LL(0x3f51f9f9,0x31b0bc68),LL(0x825b5795,0x6516bbc3),L_(0x3d1419d2), + LL(0xe977f339,0x55bfd519),LL(0xb16ed1af,0x5cd307d1),LL(0xd91345bb,0xc5e26f62),L_(0x64a35d22), LL(0xa243f1ee,0x05015842),LL(0x70203aa0,0x6fa10db3),LL(0x05a83a63,0xdaf3972b),L_(0x9422f10e), + LL(0x73b437e2,0x6a4bd1f9),LL(0xb3f66de6,0xbb0f749a),LL(0xfdf9a0e9,0x471a2a7f),L_(0xb9c51b5f), LL(0x44574243,0xfa29681a),LL(0x9c409bd0,0x9522774b),LL(0x169405c3,0x48be80cb),L_(0xf22d3fc0), + LL(0xf845831a,0x86ade48b),LL(0x79cec3fb,0x473e361f),LL(0xb9622253,0x5523244b),L_(0x538d543e), LL(0xf8570c03,0x12bb38bf),LL(0xb4e20b8c,0x784e1b25),LL(0xe6873448,0x2a2e7758),L_(0x2954321a), + LL(0x47b54bc5,0x3591891e),LL(0x27a72a86,0x3ca319e3),LL(0x482460f6,0x70114363),L_(0x3b204e52), LL(0xcca0ee1f,0xd5f98053),LL(0xc4a1a96b,0xd84149c4),LL(0x0c0e97bb,0x6bc4b6e9),L_(0xfb212c8c), + LL(0x1bfad739,0x716b6add),LL(0xea007122,0x0e68b721),LL(0x08a75056,0x4e35e1f6),L_(0xe28f8fa3), LL(0x93c21468,0x7393ad16),LL(0x4e27959e,0x4533e928),LL(0x433c708d,0xe5b33144),L_(0x4c003b64), + LL(0x32804a04,0xc57bd47a),LL(0x238addb6,0x5ae81313),LL(0x703c82d5,0xa0691b17),L_(0x14a5d58e), LL(0x825037c4,0xf9cefcb5),LL(0xfd4764c3,0x38e801b7),LL(0x7b23c9fe,0x23cf8d21),L_(0x1d7b0c07), + LL(0x8738f5d4,0x869c61a1),LL(0x8d8b998b,0xe83f1dc1),LL(0x5cc8116d,0x1f271989),L_(0xa8992ade), LL(0x88f6e037,0xb2334f8e),LL(0x1fadf19a,0x9fcaafe1),LL(0x71d5904e,0x939bd9ec),L_(0x534ec549), + LL(0xa2cbcd92,0x22be0a44),LL(0xb12549d7,0x275cd50c),LL(0xb64feadd,0xfa13b698),L_(0xd61f5f42), LL(0xedfc9326,0x7ea2fed9),LL(0xd797283d,0xdbd2171d),LL(0x5e1f2529,0x113c4da8),L_(0x4d22be7a), + LL(0xd7a234b5,0x746daf5f),LL(0x210782c4,0x8d13c8a1),LL(0x358ace9d,0x1008fd0f),L_(0xaf231c63), LL(0xeedf77aa,0x17203da0),LL(0x3534d554,0x0974abbe),LL(0x30962d48,0xf7552ee7),L_(0xa1ab0080), + LL(0x77890e5b,0x7708ee68),LL(0x981a94f1,0x624e8fe3),LL(0x50822536,0x2632b017),L_(0x21031b41), LL(0x0b603325,0xa43cc164),LL(0xfb24577f,0xc20ef5b2),LL(0x67a5fdbd,0x583ccbb4),L_(0xdf198e9a), + LL(0x2390b30d,0xec729f5b),LL(0x347bddd0,0x5467eb45),LL(0x4576657b,0x79a42c59),L_(0x3a4d5c62), LL(0x871cf39c,0x72672c8d),LL(0x34edf6de,0x22af9ec1),LL(0xc615c176,0x264b7c52),L_(0x6b6c91f7), + LL(0x3dc0652c,0x3c93deb4),LL(0x454fe5e1,0x83476556),LL(0xdaedb455,0x895933d6),L_(0xc3d92afc), LL(0xbabeca3c,0x62afc582),LL(0xb1ad4196,0xd877f8d1),LL(0x122626f8,0x56df6aa3),L_(0x88c5db33), + LL(0xc8d94c12,0xb5862dd8),LL(0xfb7f7ec7,0x3488de74),LL(0xf9ca08fd,0xf3fb356b),L_(0xa3379ec2), LL(0x27d39af3,0x5680ab94),LL(0x1f4b780d,0x4aa3a013),LL(0x181ad0f0,0x9c77acf2),L_(0xc5ed6c19), + LL(0xa57812ca,0x57cdf956),LL(0x2e235ce3,0xec26e698),LL(0x3b66736f,0x02924c24),L_(0x9611c7f6), LL(0x20dcabee,0x2cd39c40),LL(0x0ca01257,0x6c0fb396),LL(0x36cea89c,0x576c9d6d),L_(0xc0386b9b), + LL(0x69c717f0,0x3ff1d620),LL(0xb26ad028,0x975088a3),LL(0x3395a449,0xbb9b7453),L_(0xac8575b5), LL(0x0cb11f18,0x19b52294),LL(0x61b7e04c,0x84cd1038),LL(0xea595a3a,0xb7e04864),L_(0xf641867a), + LL(0x68c289a6,0xf9c67812),LL(0x0d5cd555,0x64da8983),LL(0x15609314,0x071a1814),L_(0xcec729a1), LL(0x2725b13c,0xd82b0a63),LL(0xe9fd6113,0xba49569c),LL(0xf4a523e8,0x095fdd3b),L_(0x601388e8), + LL(0x382f2331,0x5d150588),LL(0x6e0c12f8,0xabac1d43),LL(0xcc4b0247,0xe25d613a),L_(0x98b62cd3), LL(0xc09181c3,0xca8e4ca9),LL(0x1f7fb23f,0xae6dcc27),LL(0x41da9f53,0x44527d49),L_(0x9e1f47c6), + LL(0xbf5c291a,0xb17f74ec),LL(0xe460275b,0x7fe47107),LL(0x56dc0e69,0xaef13524),L_(0x055bebff), LL(0x85eeba4b,0x5a0a6115),LL(0x52aae002,0xd72b6016),LL(0x65c91bf7,0x55229ca9),L_(0x837c7497), + LL(0x05073659,0x724287d6),LL(0x86eb5493,0x266f1738),LL(0x70786420,0x72b46e9e),L_(0x0bd8afe8), LL(0x2e7a6b37,0xeea543b6),LL(0x7400cd52,0x1221cd6e),LL(0x0a6c2e1c,0x9604a54b),L_(0xe2c2c60a), + LL(0xaacc2f1d,0x771d897e),LL(0x80cd8b3c,0xa4304d46),LL(0x3b961927,0x56c6e625),L_(0x8d0c1d5a), LL(0xf0576bd1,0x4b4c8802),LL(0xee49b988,0xd528b6a1),LL(0x617aabd8,0xb63d84cc),L_(0xb309e31a), + LL(0x21a4d862,0x6ef3ab05),LL(0xfbca1b97,0x7fe6fa59),LL(0xd8911f8e,0x639e011c),L_(0xb731d607), LL(0x654578a1,0x0706cd08),LL(0x8e5b6703,0x5d08a7fc),LL(0x4d443d17,0xdc2ea468),L_(0xccc5dbca), + LL(0x2986a64c,0xf86e4ee6),LL(0xb38b3cb9,0xb2a50e4f),LL(0xd001bff6,0xad920b0c),L_(0x5f833048), LL(0xceab268e,0x3c687112),LL(0x9cac7077,0xb6207b74),LL(0x2bf74e82,0x92b4f4b1),L_(0xfcfe6f73), + LL(0x4a1a48c2,0x9afdccc1),LL(0xf2645ccf,0x9befe15b),LL(0x170bc3de,0xc06a289b),L_(0x8b3174cc), LL(0xcb55af1c,0xc289c4c5),LL(0xfcbc66cf,0x73ec1671),LL(0xea32e7fb,0xfdc7bcec),L_(0x2d89d32d), + LL(0xc3672949,0x31801b58),LL(0xcf102dd0,0x2e36c7e0),LL(0x28f8286a,0x1600629e),L_(0x6d3edc02), LL(0x16920812,0xf21cabe6),LL(0x31851ac8,0xf3657a9a),LL(0x037f5693,0x3f33781d),L_(0xb0a0ce6c), + LL(0x15c4b334,0x4821f125),LL(0x43b3629d,0x3a678868),LL(0x33190540,0xa3b32151),L_(0x4038845d), LL(0xcf5706d9,0xedfaa06c),LL(0x00044014,0x08de20a4),LL(0xd5eb20d6,0x1a8a108b),L_(0x38cb0a6b), + LL(0x15f2ae4d,0xb6e363d4),LL(0xa45c3df2,0x80d28480),LL(0x00c26ab1,0x85098966),L_(0x6d03e325), LL(0x36056e89,0x8eec84d1),LL(0x9134e03a,0xce2d0f6b),LL(0xbbc8d92d,0x4ffdb217),L_(0x6e97a4bb), + LL(0xb60e66c5,0x231d20c5),LL(0xa36c8848,0x89bcc285),LL(0xcb7e8b54,0xc4b2c9f8),L_(0x2f8b6de0), LL(0x243f2d3f,0xb8860525),LL(0xfdcf530d,0x945673bf),LL(0x49da0f6b,0xefc627b6),L_(0x4e9fb807), + LL(0x21bc7f20,0x64eae113),LL(0x23074bd5,0x2718ea4a),LL(0xb246633c,0xefdb4759),L_(0xc6af484b), LL(0xcdbec413,0xe5997f28),LL(0x8979b2ec,0x82bcb2f5),LL(0x530f4963,0x55ee3298),L_(0xccf127a1), + LL(0xba2c7ded,0x687018fc),LL(0x7c723f5d,0xcc9d7b65),LL(0x2221f721,0x93828714),L_(0x13d720d3), LL(0xe6d02ad4,0xe61c9b05),LL(0xc198db55,0x72b115de),LL(0xbe38af8f,0x621f284e),L_(0x18f09bf6), + LL(0x7c8299c2,0x65cd0db6),LL(0xbb33fff7,0x7d479cee),LL(0x44ac0a56,0x7a9b66be),L_(0xc9308cae), LL(0x43b7cc76,0x1b4048cb),LL(0x5b7ecc1b,0x0b2ccca8),LL(0x7fa30aae,0x7e09ad23),L_(0xff1f76e3), + LL(0x13e6bef2,0xa178a33a),LL(0xdd724842,0x7d66c450),LL(0xa5d7a87c,0x30befc70),L_(0x69874db8), LL(0x3ef58d85,0xcfcdbfce),LL(0xe3d6346a,0x9fa94e38),LL(0x84c65476,0xde39cef5),L_(0xe4efb3c1), + LL(0x5b582473,0x493ae0ba),LL(0xbc282cc0,0x8c93a914),LL(0xa024a0ea,0xfa3a15cd),L_(0x240e6a89), LL(0x572dde87,0xbba5d02d),LL(0x95b6f1e0,0x2f1b39cd),LL(0x15b8172f,0x158b8956),L_(0x5eb7385c), + LL(0x4b30aee1,0x92f507cd),LL(0x9137f80a,0xa95f0372),LL(0xcb924116,0x2be56c53),L_(0x0703cdda), LL(0xbf64ed3a,0x0b8faa7f),LL(0xb73eae3f,0x9943cbed),LL(0xf227ad25,0xdbca2fc4),L_(0x06231927), + LL(0x92d256df,0x5983f1a7),LL(0xee7ac222,0x640d13e7),LL(0x48c69cc0,0xa7dc8e21),L_(0xbd701252), LL(0xaa2ef84d,0x483083a1),LL(0xc0d30797,0x560bf32f),LL(0xd4ec2e63,0xbda6fdc0),L_(0x15074c14), + LL(0xc0aebb53,0x29e4a335),LL(0x600e1a17,0x155e8020),LL(0xa542f638,0x556905e7),L_(0xc7209080), LL(0xf57b447b,0x0b55b0fd),LL(0xa6cf1f95,0xc37cde3e),LL(0x9d44e115,0x1acfe880),L_(0xe820af09), + LL(0x08d29dee,0xc0bba382),LL(0x02857752,0x40ac12bb),LL(0x8b30f556,0xf120846d),L_(0xac6401e1), LL(0x41706845,0xcb20c290),LL(0x84fbf362,0x6005431c),LL(0xecddc9db,0xe1839421),L_(0x63952a13), + LL(0x93d07eac,0x24f4b135),LL(0xf0524e07,0xfc097ef4),LL(0x138ee6ed,0x1770de3d),L_(0xedc6985e), LL(0x34e15a3c,0x997ae14f),LL(0xb5a50098,0xa168adf0),LL(0x05ed2b31,0xe84702e1),L_(0x4701e013), + LL(0x991691d6,0xa2d01aa3),LL(0x5c0291c5,0xe3a7de4e),LL(0x87f294c6,0xdc575065),L_(0xf7c66733), LL(0x1ae8eb0b,0x236da942),LL(0xb2bb31c0,0xd46c809a),LL(0xefda656f,0x0d4ba22a),L_(0xd90a6c4a), + LL(0x009f6ea1,0xd7b84536),LL(0x6ae52eb4,0x52644093),LL(0x4989ea92,0x58155fb2),L_(0x51865588), LL(0xd29d86e0,0xb2f24a3e),LL(0x4a65a557,0x6e88be4d),LL(0xc5c558ab,0x39d391fe),L_(0x71637238), + LL(0x551e6187,0xc6f1655b),LL(0xda9557a2,0x49fef895),LL(0xad9724f5,0x02bd8dec),L_(0x286ae065), LL(0x519ec0a6,0x66f88628),LL(0x409bbff9,0xb11907a8),LL(0xa035e3ce,0xfa9ce7d1),L_(0xb11a05b6), + LL(0x835832b0,0x7d3144d7),LL(0xeeb7e587,0xca5729ee),LL(0x63105e87,0x24b31279),L_(0x358ad95d), LL(0x44854e57,0x8b96f12f),LL(0xa1f8699a,0x43d92d9e),LL(0x36720d2f,0x19da3cf7),L_(0x0c3e259d), + LL(0x4f9b1862,0x3ebef321),LL(0x18a10476,0x898e24fc),LL(0x93fc7b18,0x2dd5dfa6),L_(0x453f9b19), LL(0x9fe968bd,0x2e375ef1),LL(0xde52e14c,0xc56e3c8e),LL(0xb86d2dac,0xcdf1057b),L_(0x8978f37a), + LL(0xa47a47e4,0x01ae0872),LL(0xf209f351,0x1ffda591),LL(0x1c3d4f64,0xfd2dee37),L_(0x23082359), LL(0x92135e7e,0x226bc4e4),LL(0xc1dc2c14,0x68af001c),LL(0x4a88888d,0xd11ebfc0),L_(0x5bb60dfa), + LL(0xbdac7377,0x6d7442e6),LL(0x1a37fe3d,0x111de286),LL(0x64b6847f,0x54c71408),L_(0x9321abab), LL(0x605934d9,0x8f6064c1),LL(0x0fe08518,0xab526e1f),LL(0x48846076,0x18d88b42),L_(0x5027824f), + LL(0x58d7da2d,0x2fefa18a),LL(0x0a5f6d1a,0x28fd1689),LL(0xaeecb1e4,0xe2cc44ed),L_(0x1a3c357a), LL(0x0093ad3c,0xe129986c),LL(0x78fb4e9b,0x8691e9c2),LL(0x6b166a07,0x1f456692),L_(0x55649b1b), + LL(0x9465a494,0xbb9f7822),LL(0x0c7e73f7,0x720482c7),LL(0x4c6140bb,0x1d71b6bb),L_(0xc0d4cee9), LL(0x846e827c,0x8f080a16),LL(0xf403aa8c,0xa7eed738),LL(0xb53cebd4,0x45561f5f),L_(0x176891fe), + LL(0xd85db49c,0xffa27852),LL(0x4d6d2499,0xebaf8c05),LL(0xb5248259,0x5a3e6570),L_(0xb7125d0c), LL(0xf6311485,0x37979bfc),LL(0x785afdfc,0x732a179e),LL(0x1a4c7732,0xe1b6b02f),L_(0xa84e1866), + LL(0x70e07038,0x11dbcdcd),LL(0x4243fd62,0xf7e91887),LL(0x9892ddcf,0xdc0c4d01),L_(0x5638c3c1), LL(0x71bade10,0xa42313ce),LL(0xfcf531d5,0xe28bc38a),LL(0x1417aa25,0x7e0b8323),L_(0x41b2315e), + LL(0xac4962cb,0xd6c61ac1),LL(0x9d056208,0xb7765b3a),LL(0xa70e6aa2,0x566c60a7),L_(0x483b996a), LL(0x05f2cab3,0xa38d32bc),LL(0xc38fefeb,0x21c5e288),LL(0x57f15c48,0x689ae7bd),L_(0x3e0d8010), + LL(0xbb871aa2,0x2587df14),LL(0x904f01e9,0xc5d6ae2a),LL(0xbce7f04f,0x88aa4d08),L_(0xd7f1a354), LL(0x75d9b4b6,0x5bfae677),LL(0x299925d3,0xf0375024),LL(0x6a708cd7,0x74d5a757),L_(0x97968c25), +}, +/* digit=1 base_pwr=2^7 */ +{ + LL(0x44a6cdeb,0xd1a6df98),LL(0x084a7e0f,0x89ef410c),LL(0x0b68659d,0xdf5b6590),L_(0xe1cee8dc), LL(0xb48fcec5,0xf5210071),LL(0xccbda01d,0x01031fdf),LL(0x4c7a3392,0x01bb7a1c),L_(0xa7484df7), + LL(0x698ab732,0xcb74232b),LL(0xb62e25de,0x4f40ee59),LL(0xc5c8bed1,0xfb8ee6c4),L_(0x4617192d), LL(0x626b516c,0xeaf8b0a3),LL(0xdc2453a5,0x0f8cc094),LL(0x4d7f2e28,0xf43e697c),L_(0x2e81523e), + LL(0x212c04e5,0x37eb6d84),LL(0x29543b70,0xa63887c8),LL(0xe67d4662,0xbf221e31),L_(0xde581875), LL(0x5a447606,0xc3db23ff),LL(0x82bba001,0x0b2a5765),LL(0x60a68af2,0x5e5ccc27),L_(0x619f6149), + LL(0x008cfd4a,0xe80619b5),LL(0x25f0799b,0xd4f88882),LL(0xeb6dfe0d,0x9bf5f7d0),L_(0x63a366a0), LL(0x3703627a,0x927212be),LL(0x02c160de,0xbd5509cb),LL(0x09ea0af6,0xfe05e98d),L_(0xbae9ad71), + LL(0xb26352aa,0x65b1d33d),LL(0x7f5ed157,0x84979794),LL(0x1ed1728d,0x39babd58),L_(0x0137cf58), LL(0x836ccdc1,0x2546cd9b),LL(0xd0dfb090,0xbfa9bb59),LL(0xb59a51ab,0x6476def4),L_(0xc5dce06b), + LL(0xceaa7c3c,0x23138239),LL(0x80c0cfc1,0xa694e0b5),LL(0x27734f88,0x587454f5),L_(0xb527a9af), LL(0xee6f2afb,0x049823f2),LL(0x25a1f7dc,0xe574bc64),LL(0xb3ea257d,0x1d11fd76),L_(0xafd0ae51), + LL(0x9b56a4c2,0xc1bc485c),LL(0x407824fd,0xbe617f5c),LL(0xa0e87d81,0x71256aba),L_(0x76217f94), LL(0x3e2877fa,0x8b3e508b),LL(0xe501c9b4,0xbda0977d),LL(0x08a19d4e,0xfb512fa2),L_(0x59ab4cb8), + LL(0x71544a12,0xe4756696),LL(0xaf2eb7f6,0x93d2ed81),LL(0x5c0e9df7,0xc52db668),L_(0x2b7064bd), LL(0xded1b829,0x43184720),LL(0xd4b9fc59,0xad215e80),LL(0x1343040b,0x88ef261a),L_(0xb9864942), + LL(0x8eee1fca,0x0f601216),LL(0xc3ce93c3,0x8e99383b),LL(0x40d758a5,0xc0aa59c7),L_(0x54ae6327), LL(0x9df0859e,0x111b9e3c),LL(0x09cd3a96,0x2e522f76),LL(0x6bcd40ef,0xb04993c8),L_(0xbcda6d48), + LL(0x0deb6f15,0xbabee791),LL(0x030c3495,0x66179f3c),LL(0xd426f7b5,0x457bca6d),L_(0x81319526), LL(0xd35cd40e,0x2498c158),LL(0x0b2bd04a,0xaec4ff20),LL(0x12985a92,0xcff74849),L_(0x837d8e0e), + LL(0xf9b841f9,0x139de2db),LL(0xf41adf6c,0xf5c32cc1),LL(0xb7dc7dc5,0xede2ea87),L_(0x6e2033f0), LL(0x706a5efb,0xcd213864),LL(0x287f4ece,0xdccf34ae),LL(0x50f400e9,0x0fc6614f),L_(0x89db6d9e), + LL(0x4782e689,0xc42a6b83),LL(0x9e9a2ddd,0xa767b1e1),LL(0xf3f5ab2e,0xabdd54a4),L_(0xc69f9a88), LL(0x5eeb634c,0x8affd662),LL(0xb4602516,0x434e3998),LL(0xc75f7a3f,0xfbd56bb7),L_(0xe4aa58aa), + LL(0xfdddeaed,0xbb083ae3),LL(0x96aff3db,0x8b900a50),LL(0x2d9b4c87,0x77208838),L_(0x2b1d390f), LL(0xeb40774a,0x408ee7f4),LL(0xc9a98c1e,0x491fbb55),LL(0xb6789b9e,0xf0a9cf2d),L_(0xcc0aecb1), + LL(0x4d8759b7,0xf99e1bc1),LL(0xf9d97ac1,0xe489d308),LL(0x31242e73,0xe520dfcf),L_(0xba519709), LL(0x502d066a,0xaada88da),LL(0xdf453d98,0x2db805ff),LL(0x48d83f43,0xa35584c3),L_(0x7efe383e), + LL(0x1ce90c15,0x18351405),LL(0x987fc6a6,0x16ded7d8),LL(0xf2b213ba,0x17bc0ecf),L_(0xd5c3d1eb), LL(0x4a42fef2,0xc603801c),LL(0x037728cd,0xbe4db3d6),LL(0xc5cf397d,0xde6243a1),L_(0xaf004bfd), + LL(0xee10b9c9,0x81c21906),LL(0x140e4fb4,0x5c74da45),LL(0xb07d29b4,0x65c368fc),L_(0x6829786b), LL(0x7aba9db1,0x7c01e8b7),LL(0x5c2ad4ba,0x597dbeb5),LL(0x8b692603,0xca528c39),L_(0x09175739), + LL(0xd1bd09d7,0x3e778b35),LL(0x332aa21b,0x97ef919b),LL(0x7761d7c9,0xf6af5223),L_(0xded7e546), LL(0xe8470307,0xde909622),LL(0x254a8b2a,0xe9be14c8),LL(0x6fa060cc,0x7cccaea0),L_(0x723e3222), + LL(0x6fa0e5d8,0x00e54d7e),LL(0x80274626,0x53d70b13),LL(0xf63104d3,0x448cb4f0),L_(0x86f7822c), LL(0x567fec1a,0x635d1d19),LL(0x74c9d852,0xad5841d3),LL(0x2f27a18d,0xa7638f56),L_(0x7df0f814), + LL(0x2f7d4c38,0xe3919fa1),LL(0x11b8e6be,0x426c3ac2),LL(0x2038bc39,0x1a7c24cb),L_(0x547f1d30), LL(0xad339016,0x4937319b),LL(0x14498e9f,0xe9acdbc9),LL(0xd9e8ae36,0x71231dd7),L_(0x099cc29c), + LL(0x0f0c5086,0xbfa21d69),LL(0x9934b257,0x0d29b38c),LL(0x2552b462,0xfad28b1a),L_(0x87244f2d), LL(0x042aa2fb,0xe091c63b),LL(0x224579d6,0x86f0f8da),LL(0x7927774b,0xa4177fae),L_(0x39e5970f), + LL(0x8ecd5fa1,0xf8a953db),LL(0x00e1559f,0x8ef3b9f7),LL(0xa70e63b9,0xf8851e82),L_(0x48191bdc), LL(0xefc3b637,0x6d398f41),LL(0xc96682f5,0x40655248),LL(0xc2927516,0xd7d600fe),L_(0x6435359a), + LL(0x4a966ff2,0x43c60bf9),LL(0xbe98cc32,0x94a73ea6),LL(0xe899fd7f,0x6f6a67d4),L_(0x4805ca93), LL(0x49a69773,0x532ddf8d),LL(0xb5f6725b,0x4785c103),LL(0x9818fcf0,0xa6151612),L_(0xa9419c51), + LL(0x351340ab,0x544bdd84),LL(0xf568cb9c,0x1359eeae),LL(0xe070e7d3,0xa9680c77),L_(0xf00849de), LL(0x3f049e65,0xc159fff4),LL(0x9f8b9614,0xc00e514e),LL(0x0503644f,0xf325f0bc),L_(0x58ef4f26), + LL(0x0afb5680,0xbb5a090b),LL(0x2a905098,0x29e6e124),LL(0xeb89d752,0xb46f0674),L_(0xc23b6e08), LL(0x7d46e8c8,0x84d934a5),LL(0x7446341f,0x814623ef),LL(0x2d200178,0x428fe6ef),L_(0x663531ec), + LL(0xc36811cc,0xf0622226),LL(0xe7d3d99c,0x26dbef18),LL(0x87323666,0x0421f1d4),L_(0xb81113de), LL(0x2f1ac158,0x121ae440),LL(0x14b9bfe5,0x7f1d1f81),LL(0xab8ce457,0xfb0b5381),L_(0xeebf8b75), + LL(0x1ed0f16e,0x4ab996f9),LL(0x3bca93de,0xf3972d52),LL(0x9c224825,0x51f0a250),L_(0x4390fa94), LL(0x52d41c04,0xa17f9bcf),LL(0x87ca94cd,0x1da254f1),LL(0x8eb876e4,0x8c10d532),L_(0x6ab8e456), + LL(0x251d18df,0x00191031),LL(0xbe58c480,0xfb757330),LL(0xf0df6b45,0x51d6cd1a),L_(0x3a2b0d2e), LL(0x72d910fc,0x6e0dcc32),LL(0x17ad59a0,0xd56ef620),LL(0x24ff8cf7,0xb1342fd7),L_(0x514eda6b), + LL(0x3d2d0473,0xaede35a8),LL(0x43fadb36,0xc3bc4ec3),LL(0xda078971,0x89ff847e),L_(0x757ad40d), LL(0x3367d69b,0x063bed04),LL(0xfbec9b9f,0x96916bd1),LL(0xa2938401,0xed5c65dd),L_(0xd3c116f5), + LL(0x4ab81e63,0xbf6d76ab),LL(0x4a94b590,0x92ace598),LL(0x43e3ebf0,0x6d7848f4),L_(0xe0fe0161), LL(0xd3008b3b,0x503ad71b),LL(0x14c43727,0x1bf34193),LL(0xafbcb091,0xbb5f8c82),L_(0x3bf8d893), + LL(0x9e3f46b2,0xa2d0d277),LL(0xd8be2204,0x6e8eb285),LL(0xc9fa15ec,0xac17e25a),L_(0x91b10a2f), LL(0x897ae118,0x12c8ef7f),LL(0x087fa01d,0x919f4309),LL(0xb6f728bd,0xd198666a),L_(0xcef80aee), + LL(0xf8d94d59,0x6112903c),LL(0x89add916,0xda5061e7),LL(0x1c13960f,0xf15aa481),L_(0x48b52034), LL(0x16623c8b,0x2d1bf700),LL(0xb5d87e2f,0x9a5f3c4f),LL(0xc7e5ba8d,0x6e4a90fb),L_(0xfd04fbda), + LL(0x16d9fe00,0xce2f6f2e),LL(0xae50cdc8,0x205e3401),LL(0x5c73b50f,0x59a0da00),L_(0x6b833239), LL(0xb1b01bc6,0x54c575fa),LL(0xfdd6dfac,0x38a3a688),LL(0x761a3a3e,0xb3095407),L_(0xca8c1a6e), + LL(0x2aa45623,0x56e12b9e),LL(0xa3fc4119,0xd0bffaa7),LL(0xf5bdb564,0xe34b58ec),L_(0x178622a7), LL(0xcda19c33,0x9e9ffc05),LL(0xfbbaa19e,0x51b7b9c6),LL(0x42f09c2b,0xb2d6157b),L_(0x0c83351e), + LL(0x80fda4ab,0x623598c1),LL(0xae4a61b7,0x488f724a),LL(0x47014b1c,0xddd6a618),L_(0xc1476c47), LL(0xbc2cc2d5,0x7eb38a43),LL(0x31a26c96,0x7b20f4d7),LL(0xc5ea2b2a,0xa5a136e8),L_(0x13b6043f), + LL(0x3199bbf6,0x07cb2293),LL(0x210180c0,0xe5a1b306),LL(0x782987ea,0xab40f28e),L_(0x4a4f4f9e), LL(0x95fa6103,0x591b5ef8),LL(0xf6832896,0xb357ced7),LL(0x38f56fa2,0x5abb2a56),L_(0x5a65a98b), + LL(0xf1c770e6,0xca04bf7b),LL(0x022f4557,0x88cfe334),LL(0x657a19e3,0xc772f98c),L_(0xa19ab123), LL(0x3ab78500,0x82db0eac),LL(0xd6b0ec00,0xf668d4b5),LL(0xdeadd9cc,0x9dab24ce),L_(0x1c2e2051), + LL(0x0ac55884,0x7647407a),LL(0x489801b8,0xfa51e70a),LL(0x56034023,0x26d5313c),L_(0x29b7a999), LL(0x834f99e6,0x6f0fc77c),LL(0xfdc7c2f1,0x647839fb),LL(0x006982c9,0xcb5af1f9),L_(0x9323d960), + LL(0x91a088be,0x76954497),LL(0xa4de751e,0x393750b7),LL(0x6dc8702a,0x805eb5f7),L_(0x1cfb5ce3), LL(0xaa971870,0x94bbb1b2),LL(0x43b64965,0x939fd8c5),LL(0xaf8d3158,0xc4fd0925),L_(0xeb438d66), + LL(0x0e5bf508,0xc7a170c1),LL(0xb2973b56,0x43a0c49d),LL(0x9e4dbaca,0x2cb87666),L_(0xcf3bacaf), LL(0xef7d14b1,0xa2effdce),LL(0x20a60ee8,0x586540be),LL(0x60509402,0x676121bd),L_(0x745c0ff2), + LL(0x6d713b1e,0x3407acc5),LL(0xb6d85b13,0x4730880c),LL(0x7e9049e1,0x9fc877ce),L_(0x64c1005f), LL(0xe9cfd783,0xb606ab7b),LL(0xe272e2ce,0xc2eb602b),LL(0x97ac2d38,0x954f3f1f),L_(0xa0e40e6d), + LL(0x479a41f2,0xdf2474e7),LL(0x1f05663e,0x3574feb4),LL(0x234012a9,0x89989069),L_(0xc2481dbc), LL(0x24c35b2a,0xf495b125),LL(0x452948bf,0x0325369d),LL(0xa43ea9bc,0x280bca9e),L_(0x8a34a29a), + LL(0x2326ccfd,0xae198471),LL(0x848b5387,0x840ea47e),LL(0xcce778e1,0x4288a820),L_(0x34cf4dad), LL(0x53f54930,0x9e011fed),LL(0x30c9f97b,0x29fd0d6f),LL(0xcba3ae4e,0xdb99670d),L_(0x1db11c4c), + LL(0x59d694e8,0x9d825f38),LL(0x2980d39c,0xc504447d),LL(0x5a3ddebc,0x9c96ad62),L_(0x7f86e997), LL(0x516005e5,0xac938f78),LL(0x7a80b2ac,0x3bbc6276),LL(0xe8b1101b,0x055eb004),L_(0x6c8daec0), + LL(0x50fa66d9,0xcaf9f5b8),LL(0x88ad0652,0x4681401f),LL(0x35f6f9f8,0xbced09e3),L_(0x51bdbe8d), LL(0x7b25a089,0xd40e66f7),LL(0xa071947a,0x63257022),LL(0x2ee8b520,0x9757d231),L_(0x4f80de84), + LL(0x822c5f31,0xdf7efaf0),LL(0x3121c652,0x39218972),LL(0xd7749654,0xb988c422),L_(0x493a2d8b), LL(0x76360207,0x98e08450),LL(0x2760de2f,0x8f841286),LL(0xcf07c50e,0xe5f81246),L_(0x1a00e240), + LL(0x9a2b0329,0x9d26d17c),LL(0x89b4fc25,0x9bc053f8),LL(0xc765d35e,0x45f96752),L_(0xbfefae3b), LL(0x6da731c5,0xf9a4089b),LL(0x22ecc154,0xcdbfce36),LL(0x5053964b,0xdcd4205f),L_(0x1e216001), + LL(0x30c80608,0x1db56abf),LL(0x7ae4875c,0xacce82ac),LL(0x4a976988,0x4fa88f83),L_(0xd67fc048), LL(0xd5a17ca7,0x49b6eedb),LL(0xedcaacaf,0x635d8bdf),LL(0xd26b609b,0x29f6f1cc),L_(0xa9ceab78), + LL(0x8a159d18,0xf6f8a0a1),LL(0x95c6ea4c,0x766b602d),LL(0x61916ad2,0x8497eb60),L_(0xb0ce6cae), LL(0x1a0eedff,0x50c80948),LL(0xa31d0581,0x811b1e1b),LL(0x4cec302c,0x0bfcd85b),L_(0x36484967), + LL(0x37db6bc4,0x6e5e0c25),LL(0x6df2fa39,0x21b92740),LL(0x248a89e5,0xdcc52026),L_(0x67b816f2), LL(0xbd55bfd0,0xee7ad3eb),LL(0x5bb4e55c,0x3cdb6fe7),LL(0xdaa70d9f,0xacc63cd9),L_(0x9155d520), + LL(0x868a4cc8,0x6abedf02),LL(0x22a83e65,0xa18aad01),LL(0xb90ac85d,0x088ae4b8),L_(0x65e3e716), LL(0x60fc849a,0xf19b0bfb),LL(0xfbebb9d9,0x9c3ff4ac),LL(0x7355b4ab,0xef331b93),L_(0xfcb3b647), + LL(0xaf2dc94a,0x4e49fdb7),LL(0x76e1b0d3,0xf8a1df9c),LL(0x94293ef4,0x588501b4),L_(0xd8359bfc), LL(0x1ef75655,0x3cd45f3a),LL(0xee63deea,0x2a08f333),LL(0x98af1b2f,0xd1f2806f),L_(0x12a331f8), + LL(0xd2c7e263,0x3577e68b),LL(0x7e2770f6,0x355f7336),LL(0xe8913f4b,0x77ead765),L_(0xe027fdcb), LL(0xac403db5,0x9a6f6902),LL(0xf459aa5e,0x8c922198),LL(0x24bf5d49,0x130316d1),L_(0xd808c825), + LL(0x672ccabd,0xe048c7d8),LL(0x33437c46,0x674f7855),LL(0x2ce661a8,0xe77ba9cf),L_(0x46b5165f), LL(0x9cd0b0e7,0xaa89fa66),LL(0xd3057aed,0xe440af7a),LL(0x142332ba,0xb60e8644),L_(0x3eb16842), + LL(0xa0b4b4f4,0x35fc5d9b),LL(0x9f0177bd,0xc346b37c),LL(0x63dcb419,0x4fc638bc),L_(0x0d4a9edd), LL(0xe8dc39c5,0xd3a1ffca),LL(0x2be34520,0x6520ec71),LL(0x6ee7420c,0x449c7ffa),L_(0xca54fb79), + LL(0x391c4163,0x447c4950),LL(0x07a8fa48,0x212fb092),LL(0xfe67f50a,0x0d70f3ac),L_(0xdbc48919), LL(0xec4b9c9f,0xc5e2455d),LL(0xc9b41ae6,0x306c9f8e),LL(0x1f329372,0xaab1f6eb),L_(0xa76cbb84), + LL(0x50bb236e,0x7f11366c),LL(0xd38d4421,0x865c47f2),LL(0x9815fc32,0x1e6e5fa0),L_(0x42bf5236), LL(0x980283c7,0x15c5a008),LL(0x03ac6a2a,0x13c27e86),LL(0x8f3eaacf,0x9eb61a43),L_(0x1a7ff3ad), + LL(0x0c067d55,0xf6ee6117),LL(0x4446a48b,0xc9f90069),LL(0x08776dbb,0x62dd1551),L_(0x856288fd), LL(0x2c6743db,0xb1d08622),LL(0x59e29beb,0x3e4f8e95),LL(0xca32b4fd,0x1744a1d3),L_(0xad3745b8), + LL(0x4ef78890,0x3831190a),LL(0xaaeece81,0x44c9a76b),LL(0xd465548a,0x6dc6a194),L_(0xdbb627e1), LL(0x18f3c7f1,0x91571bcb),LL(0xf5fb9008,0x31e00308),LL(0xf421b41f,0xeaac6d91),L_(0x01604108), + LL(0x72bdb3aa,0xa2f0a871),LL(0x276d7a20,0x680da72c),LL(0xa823fb58,0x507ee8a9),L_(0xe6fef46d), LL(0x9c541554,0xa385d033),LL(0x2785c811,0xdf53b551),LL(0xbcf3686a,0x8ac8344a),L_(0x3bf9ea41), + LL(0x450260e1,0x8075a440),LL(0xfea1ac4e,0xe6e29d49),LL(0x0fedea77,0xb3c70edf),L_(0x2398f701), LL(0xa3d35062,0xbbbf63ab),LL(0x9c1eb75a,0x62e94dcf),LL(0x9c3a2d57,0xee68eb75),L_(0x4d73d0da), + LL(0xbcef4afe,0xe663f1d7),LL(0x31f69d33,0x6b81992e),LL(0x46cf68c3,0xc7fa33cc),L_(0xb4e3665a), LL(0x0a54ba22,0x48f183b6),LL(0x154b0c79,0x17fddba7),LL(0x32607f61,0x1e044a43),L_(0xc1030470), + LL(0x5565823c,0x25aa9de0),LL(0x88786097,0x3798fe3f),LL(0x045b969a,0x2e64a0a5),L_(0x921c1980), LL(0x89334dcd,0x8b058ded),LL(0xbe46c84c,0xbe78b8bd),LL(0x5b271860,0xfe16e851),L_(0x48f0acc8), + LL(0x11ee6162,0x31bafd7e),LL(0x7c0c6e0d,0xc8efb7fd),LL(0xe2d4fff9,0x2b1da468),L_(0x95ee5ad6), LL(0x82ecd1e6,0xfebeb584),LL(0x307f35dd,0x3c68f532),LL(0x406d6f3a,0xe2245982),L_(0xd6be6d55), + LL(0x979df4a2,0x002c4e27),LL(0x4f02beba,0xf600aa2e),LL(0xc26d73f0,0x21de614f),L_(0x38b705d3), LL(0x13b06b78,0x23429db6),LL(0xf4967923,0xa3e9a4f2),LL(0x198f9eed,0x9eb6af93),L_(0x6bb259cc), +}, +/* digit=2 base_pwr=2^14 */ +{ + LL(0x6f5b4a5a,0xdcabdf3a),LL(0xd382ee31,0x1e2db542),LL(0xeeccf680,0xe97abee6),L_(0xc6941824), LL(0xc6749744,0x47a3192f),LL(0x7f45fc06,0xc0f578ba),LL(0x7b1af995,0x4d9341e8),L_(0x248e916d), + LL(0xdde84f14,0x082378c2),LL(0x4bf3bcae,0xaf15229f),LL(0x82c65899,0xb15f6e52),L_(0x5919e8e7), LL(0x6798e12c,0x6c251212),LL(0xce97fdbe,0x696c3757),LL(0xf8d74875,0x1738524d),L_(0x44e69e4f), + LL(0x9c3b9c4f,0x54e9eb41),LL(0x1bc6447d,0x2ee02fd1),LL(0x65482175,0x63694975),L_(0x72869bff), LL(0xb7191008,0xf6aea0b9),LL(0xa70d16cd,0x5bd0a826),LL(0xa212780d,0x14730725),L_(0xdc25b909), + LL(0x09a50ad3,0x8d673442),LL(0x01c6af2b,0xc4f517cc),LL(0x24e6fbcf,0xc6a67d3a),L_(0xbbe5e1ed), LL(0x8492568c,0x964e75d1),LL(0xb97b0d10,0x09dfd2ce),LL(0x09007531,0x67a932fe),L_(0xc85f611d), + LL(0x7d086ac1,0xfaef942c),LL(0xfc2eb46d,0x4a2d4ce4),LL(0x51096873,0x75df38bf),L_(0x75ab062b), LL(0x6825bc8a,0x1f5151d2),LL(0x587ef818,0x0a14774b),LL(0xf3f93877,0x0eae8822),L_(0x5b5cb708), + LL(0x2f159690,0x36982470),LL(0x165fe970,0x2b470b0b),LL(0x2ff27509,0x89cd99ad),L_(0x98c66330), LL(0xc33eb1a9,0x0397edc7),LL(0xd2a0cfc6,0x868918fb),LL(0x7e36813d,0x722feffe),L_(0x829c6b89), + LL(0x5c8de701,0xe76a6990),LL(0x394ea602,0x568ac304),LL(0x1348f9f4,0x34e40071),L_(0x3b2e0279), LL(0x6e909cab,0x5da0385c),LL(0x0b77484a,0xd9ed956a),LL(0xbbb8ff2e,0x7e2ac222),L_(0xf54f007f), + LL(0xe5f3820b,0x306867f0),LL(0x271f814c,0x5fe6163a),LL(0x455790c2,0x038f4fe6),L_(0x58008770), LL(0x2a2eebb4,0xa3ffa255),LL(0x2ae5134e,0x4eecbe86),LL(0xfa061ce0,0x2bb3a978),L_(0xa4fe1b53), + LL(0xcfa6c306,0x1840dd3f),LL(0x24b14e0e,0xbd1f6ada),LL(0xc72a3b9b,0x5ca5c5d6),L_(0x8625f308), LL(0xdf5f8793,0x6afba384),LL(0x5e8c195d,0xd59557a2),LL(0x6bdd2d33,0x0975350a),L_(0xeff53d99), + LL(0x35527ee1,0x3fb95972),LL(0xc84c9bda,0xaea90e2e),LL(0x172afe3a,0xd069e505),L_(0x2d143bfb), LL(0x7231a957,0xaf5715f4),LL(0x9c2db5f2,0x7e8468f0),LL(0x6e7b80b4,0x71ec23d9),L_(0x4f3f9687), + LL(0x1034850b,0xf5d591b7),LL(0x0c9c04f2,0x3e07210b),LL(0x6d31163d,0x530f24db),L_(0x66c42b68), LL(0x7457ed53,0x1378c67e),LL(0x777097e8,0x8164d6d2),LL(0x3f1062b9,0x9ee5175c),L_(0xb09a9492), + LL(0x20bad002,0x12b71603),LL(0x0ee3063d,0xd5045d6c),LL(0x2261149a,0x853772f7),L_(0x8f349671), LL(0x9ec1ef24,0xdd3b39e1),LL(0x3acb7c05,0x02d7c1cd),LL(0x34c3e767,0x9381a0b0),L_(0x74081fec), + LL(0xfcd9c832,0x49667bc0),LL(0xd57108d0,0x9658b33b),LL(0x6834cfcd,0xf7fb9009),L_(0x9b0baa9c), LL(0x352f5036,0xd797b7f8),LL(0x13c1353f,0x3152ce4a),LL(0x6f581f80,0x48a60c42),L_(0x7f9e960b), + LL(0x00259aa6,0x474d3703),LL(0xcfd42398,0xd3db7f63),LL(0x1c1a54ad,0x3c4fd36b),L_(0xd3eac930), LL(0x79efe57a,0x7c4dc3a3),LL(0xdce7096a,0xfbb39dec),LL(0x732dfde0,0xa1787513),L_(0xe804b476), + LL(0x0535a13e,0x926b6b5c),LL(0xe5f713d5,0x96e21c9c),LL(0x3d019dde,0xd52c15ec),L_(0x3e7f7e62), LL(0x883a0c0e,0x6609394e),LL(0x694f2696,0x9ff80d9a),LL(0x69aa76bd,0x07cee438),L_(0x234e3949), + LL(0xae230adf,0x90a49b52),LL(0xf501aeb8,0xdfc8b644),LL(0xc50fe003,0x8b89e79b),L_(0xa04a6f7a), LL(0xc225c287,0xff3922dd),LL(0xc4ed1911,0x0cd4b29f),LL(0x0ce76929,0x8af23a3e),L_(0x918974b5), + LL(0x3d52ecd2,0xdfd72a7f),LL(0xdea21ead,0x7da62126),LL(0x8e3f7ba7,0x1f678c3b),L_(0x6df4e74e), LL(0x22dc8168,0xc5a010e6),LL(0xd7eea56a,0x54e30f4f),LL(0x389d8453,0xc6ed7b9d),L_(0x2e489cfc), + LL(0x80f6e62d,0x914f8bfd),LL(0xbcee9591,0xa0be9a1f),LL(0x7fab5561,0xec5af129),L_(0xd4d265a9), LL(0x3327e887,0xc74bfc93),LL(0xd2bab14c,0xe716458b),LL(0xe6b2e9c5,0x77d00d82),L_(0x161eafab), + LL(0xe139a461,0x8fb0ef8c),LL(0x5b2aa48c,0x622967d6),LL(0xe9937cc2,0x00ef4c97),L_(0xb862b86d), LL(0x42bfb0a9,0x97617d08),LL(0xfe0c2928,0x4f55cab8),LL(0xce44e424,0x625a38ef),L_(0x8594c3af), + LL(0x239b2f0b,0x1b735668),LL(0xbbf9ef82,0x066337ae),LL(0x754e211c,0xf058688a),L_(0x95864d69), LL(0x39a1596b,0x8beee136),LL(0x9150359b,0xd3b81681),LL(0x0ab949fd,0x3f4ac39d),L_(0xf6d5f8ee), + LL(0xad05618b,0x4aaed3cb),LL(0xb594cda0,0x3d341ce5),LL(0x0216bf14,0x9e271d0d),L_(0x7efe2b03), LL(0x68f95031,0x6a084fdc),LL(0x824376da,0xcda2825c),LL(0x6acd2b86,0x2deabee7),L_(0x55b4713a), + LL(0x7d01995f,0x9b5408d1),LL(0x5ffa275a,0x5103e5b4),LL(0xfa6d3b64,0x3f3eff6d),L_(0x5bf0abda), LL(0x13a776d1,0x97b27239),LL(0xcf4c8232,0x22a3f2f2),LL(0x170c8de4,0x047d7294),L_(0x43f7fc78), + LL(0xab797a07,0x0b00d0bd),LL(0x4d42a1fe,0x92ead9f6),LL(0xc3907782,0xd65498ad),L_(0x716502c0), LL(0x814b71a4,0xaa83d0a9),LL(0xd8e4c681,0xaf90bf17),LL(0xb2730e91,0x3b7c8d6f),L_(0xc3d966b3), + LL(0x8da671b5,0xfee99849),LL(0x2e099be3,0xdbbe577f),LL(0x5f8e6df6,0x31915ae6),L_(0x2ba98b17), LL(0x0d5b6ffc,0xe2f20ab0),LL(0x791aa104,0xdc69cc94),LL(0x2ae23401,0x5c5045ab),L_(0xd646c528), + LL(0x93077179,0x50b8339a),LL(0x2874cd90,0x05d7c4a6),LL(0xcb1961a7,0x221ee1b5),L_(0x43f06891), LL(0x0a41aae9,0xf9998038),LL(0xeac42395,0xd6ac7543),LL(0x042a0af5,0x36489fc2),L_(0x5fdad7bc), + LL(0x3c56133d,0x9f01cd48),LL(0xc0ba445f,0x29263bac),LL(0x3e5d8713,0xc6107ee1),L_(0xecfad04a), LL(0xb648509b,0x45f6365f),LL(0xeda195e8,0x4716fd8d),LL(0x2f2caf71,0x4fc9d505),L_(0x0c644ae1), + LL(0x510aa881,0xbc63e799),LL(0x658bc0b0,0x4716a31d),LL(0x2713db77,0xabf827dd),L_(0x12824e7f), LL(0x5c30bb60,0xb2616194),LL(0xfcdeef4c,0xf6fd28f3),LL(0x3fa68909,0xe9395e01),L_(0x42dfc220), + LL(0x91f7da3d,0xe6bee7b8),LL(0xb1959a03,0x303e2e61),LL(0x9b779901,0xe7e57056),L_(0x1c2d553f), LL(0x9abaa6a1,0x47995b48),LL(0xab4d653f,0x6d28c851),LL(0xf8e40f18,0xb70d2132),L_(0xbaae5b9b), + LL(0xc8d7fa04,0x325e475b),LL(0x8ad5b01f,0x8d0c0363),LL(0x8fb55f0b,0xa691b273),L_(0xee832613), LL(0x3a711dab,0xb146b4e3),LL(0x748d793c,0x81182539),LL(0x2911a867,0xdb940971),L_(0x24df2ed3), + LL(0x0b071d1f,0x25961d64),LL(0x87754320,0xc47ba4de),LL(0x15d5a2ff,0xee280f81),L_(0xfc4877e3), LL(0x1fbba89c,0xd08afabd),LL(0x8dd2825d,0xc881c4eb),LL(0xa36fe3f2,0x20840934),L_(0x398c3272), + LL(0x13806339,0x51502992),LL(0x5f9bec8a,0x1bb87fb3),LL(0x878f8530,0xbcaa2cbf),L_(0xe99dba0e), LL(0xde118aef,0x7a185dc3),LL(0x7f95d77f,0x2ea5905d),LL(0x3f419246,0x2b04f2d5),L_(0x7b04c078), + LL(0x0fd96b35,0x597ba0a8),LL(0x0eb981ea,0xb6624fc6),LL(0xe6e26240,0xbfc4dac6),L_(0x35dd20af), LL(0x61745187,0xe7b40563),LL(0x8d381cfb,0x554c1adc),LL(0x4013b9ec,0x354e81ee),L_(0x50a8d84c), + LL(0x098f1196,0xccc6dc56),LL(0xe6735e77,0xe9fcb8bf),LL(0x629efebf,0x2ffffa9f),L_(0xc520680b), LL(0x6d812701,0x732a53da),LL(0x4d7d1ede,0x248c465e),LL(0xa1f8c1a5,0x3dffe6e5),L_(0xc519ac0b), + LL(0xcaf0a9d6,0xaef46030),LL(0xdd8a57c6,0x739000fd),LL(0xb0540672,0x8ef45968),L_(0x7dc6abd9), LL(0x33da5d1c,0x5aafa6a3),LL(0x60865fae,0x6662d3dc),LL(0x9c28cea0,0xd3cb6b76),L_(0xbbb47176), + LL(0x93070274,0xfe93910c),LL(0x86c3c000,0xd939b811),LL(0xe4bdec34,0xa7194bcc),L_(0x308782a6), LL(0x640b22a9,0x710f1668),LL(0x1c395e5f,0xec57d6a5),LL(0x2f41d5d8,0xe7981956),L_(0x53f886b7), + LL(0x67aad45f,0x6d08786e),LL(0x2cc15260,0x8bf6b065),LL(0x84169282,0xa53ab3f4),L_(0x49111025), LL(0x714a4d69,0x8db73ccf),LL(0x49ee5fc0,0xaec354d3),LL(0xeb0a609a,0x26497f34),L_(0x752e2221), + LL(0xbc1c0e24,0x23ccf8a7),LL(0xb6e96781,0x8d254256),LL(0x183ac489,0x907018f8),L_(0xf0ef92fc), LL(0xc98e1d84,0xac3eb092),LL(0x63c56eb0,0x61b00609),LL(0xf8deac28,0x95f55a52),L_(0x5ae62b60), + LL(0xa101bad5,0x8575f396),LL(0x643e6356,0x75613315),LL(0x3b84d8ca,0x87159fb6),L_(0xb1bb3787), LL(0xeb901a9a,0x04257ae5),LL(0x76cc23a1,0xc722a719),LL(0xf70dd339,0xd9298acd),L_(0x91c26db4), + LL(0xbcfc15f8,0xac24fa28),LL(0xa4b011f9,0x9cf8c10a),LL(0x77f2a9a2,0x897d9044),L_(0x0ed14d52), LL(0xe210939f,0xa6397972),LL(0xe2308c70,0xc34aac84),LL(0x72030316,0x664a26ca),L_(0x291e9903), + LL(0xd67a8d4c,0xd3c9cb6a),LL(0x38755541,0x115ea4ff),LL(0x4ad0effb,0x5c237f2b),L_(0x75646bb1), LL(0x1c0c1001,0xa07ca1d3),LL(0x7f647322,0xc00e9dcd),LL(0xe24ec66d,0x819d18b8),L_(0x95c255a0), + LL(0xd48e2f95,0x8209864f),LL(0xddb2d835,0x9d8a81eb),LL(0x41d055f3,0x92d42e0f),L_(0xe2f17df9), LL(0xdaf29a59,0xc2c8d2a6),LL(0x061a667a,0xd108615b),LL(0xcc790c1c,0x0eb1df27),L_(0x6dc9536e), + LL(0x5d7251ce,0xf7ca409f),LL(0x55a94793,0x8791f913),LL(0x6bbc768b,0x1ef210b3),L_(0x166d8931), LL(0x24baa65a,0xa0bcf014),LL(0x621f63a2,0xa8abe2d3),LL(0xf790fc9d,0x60203dd4),L_(0x667b1fa7), + LL(0x77f64ca2,0x2884e5da),LL(0xc59a883c,0x38a09cd0),LL(0x47f89d65,0x780aeefd),L_(0xd528d23d), LL(0x9945faad,0x90523929),LL(0x6e9ee32f,0xa460502a),LL(0xb5fe2f01,0x1d21d327),L_(0x2883e925), + LL(0xc3ff6cb2,0x8d7cbdb6),LL(0x32e5cd9d,0x2baad720),LL(0x2385860f,0x43f0f911),L_(0x4b6038f7), LL(0xe096aed9,0x8ece6e04),LL(0x40da7078,0x964d3a26),LL(0xa20a6649,0x1a09dbaf),L_(0x709cfd74), + LL(0x4a5af8b1,0xa2ce00d8),LL(0xa63a8b02,0x57b8f935),LL(0x35cb5ede,0x706b0821),L_(0x4173e4a2), LL(0x74594a56,0xaae1c286),LL(0xaa0caca3,0xc1e054bb),LL(0x3c3b3e24,0x34698533),L_(0xe3b66255), + LL(0xcb1b2f2a,0x4bf57c51),LL(0x164519e7,0x6b4ba459),LL(0xb55fe2b9,0xd37e29a7),L_(0x68e8549f), LL(0x9b0eb441,0x9f4b4c99),LL(0x8f691f6b,0x59078e54),LL(0xa382964e,0x49524978),L_(0x7d3fe7f2), + LL(0x3a14e17c,0xb90d4bd4),LL(0x087b646e,0xac93ca7a),LL(0x53865f43,0x2d659d4e),L_(0xb9db500d), LL(0xdc91bb2e,0x17597d7a),LL(0x6ae5d43e,0xb1613ba6),LL(0x3a5e9fe1,0x4204e314),L_(0xd91a271a), + LL(0xbf7ca431,0xcfce86aa),LL(0x16c2854c,0x51585545),LL(0x3999a9a1,0x5fd18a95),L_(0xa1e55106), LL(0xd29f7bb5,0xd7192594),LL(0x7c17c721,0xb7a8c7e2),LL(0xf6856acd,0xdb82b841),L_(0x39017815), + LL(0xe9631773,0xf155113d),LL(0xc2b2be30,0x09bb3adb),LL(0x1e571839,0xa98b5f59),L_(0x17a7266e), LL(0xde672c40,0x02e9f20a),LL(0x9c53a705,0x68d00619),LL(0x476b9b10,0x144091ad),L_(0x98e56e88), + LL(0x4746f916,0x02f6a9c6),LL(0x461d2e37,0x6a692ac8),LL(0x6fe1c2f9,0x8835c958),L_(0xaae29f67), LL(0xaa57aa25,0x1573f8b0),LL(0x2aae0885,0x8359860b),LL(0xc9a1230d,0x930776e9),L_(0xd3f32fd7), + LL(0xda3f170d,0xdbf89586),LL(0xfeac4726,0x6f39cb19),LL(0xe907285f,0x084c5773),L_(0xc72ade03), LL(0x2f55c6f0,0x155c7e90),LL(0xe4fb90c2,0x50f9f0f0),LL(0x833af29a,0x87b3ac91),L_(0xeeb712d2), + LL(0x1b2f2123,0x08fa5175),LL(0xad400dee,0xe9ac51f9),LL(0xdb50d10c,0x8bbf71ad),L_(0x1322cb4e), LL(0x3efa91c3,0xb7ef36d2),LL(0xa5571f80,0x4c6187ca),LL(0x938aa540,0x6a106d1c),L_(0xfee7b99f), + LL(0xc717d42e,0x0d738fbd),LL(0x9340ad37,0xf74cbf89),LL(0x923ef854,0x1a934a84),L_(0xdc9bcb14), LL(0xf0ce91b3,0x3ed1d40f),LL(0x3e3d69c5,0x35833a46),LL(0x8cf302d1,0x60492caf),L_(0xee0ad716), + LL(0xf9f98ca9,0x310d3206),LL(0xc40b6b35,0x67ddb804),LL(0xf6b370b9,0xcf2da667),L_(0x4a7e6061), LL(0x57730157,0xd8d48240),LL(0x186ae81f,0x002b8bc3),LL(0x2133c198,0xa42e3f3d),L_(0xf8b9b6ca), + LL(0x1ba04233,0x9ac56b2a),LL(0x55759656,0x927ea140),LL(0xd47c70b0,0x0f258220),L_(0x06bffc7e), LL(0xe94dda94,0x453179d9),LL(0x92ffe53d,0x2b54517d),LL(0x2ac0af28,0xb924dbee),L_(0x7afd903c), + LL(0x7c5bc163,0x7d2123b3),LL(0x61435520,0x1d58d680),LL(0x2b094145,0xeb4a95b5),L_(0x09b8c6a3), LL(0x3c690f29,0x54e52454),LL(0x646b89b4,0x6bd00cf3),LL(0x2640a633,0x17b930ad),L_(0xa457233d), + LL(0x05fb72f6,0xbfa90c2b),LL(0xfad26494,0x9f2c3cd4),LL(0xbfe60269,0xcd907456),L_(0xb3e48eca), LL(0x6374f2bd,0xdf9ab844),LL(0x94fe7551,0x36c69779),LL(0x8ed01908,0x5a97c45a),L_(0x5dbf5641), + LL(0xa14499cb,0x9dc2ad99),LL(0xe9019e0a,0x64554a27),LL(0x588acbee,0xfff03f99),L_(0xc48117a2), LL(0x46bb4b9c,0x6e713362),LL(0x59d13072,0xfb7747ec),LL(0xc63c7628,0xefa02cb7),L_(0x39d7f934), + LL(0x69d702d8,0x5f6d09ba),LL(0x2361ada1,0x920262ae),LL(0x5419445e,0x6f750612),L_(0xa30fa392), LL(0xcf460268,0xd59c34bc),LL(0x32785073,0x76b69628),LL(0x72c99605,0xb431b539),L_(0xeec4de29), + LL(0xc39fd75e,0x97507c26),LL(0x0c5debbb,0xd82ad3a6),LL(0x3c42310d,0xa01f4d9e),L_(0x5fdffd07), LL(0xd1fedd01,0x5e2bab3e),LL(0x4ce43e20,0x8abe3a41),LL(0xe71cd8a2,0x0c4858ed),L_(0x8bd12b19), + LL(0xbb2c8805,0xb6380f03),LL(0x59339488,0x7a463ca9),LL(0x920e2111,0x0fbdd870),L_(0xcf927c12), LL(0x9fa1e546,0x4ea01a50),LL(0x20ee4123,0x38b747c2),LL(0xeeedf23b,0x65fa7940),L_(0x82b08346), + LL(0x6a22e6e3,0x34651055),LL(0x1098cb76,0x269d82cd),LL(0xd6a624c2,0x5fd458b6),L_(0xb856ae59), LL(0x6c36cae7,0xd687ba3d),LL(0x33a82abc,0x6963f047),LL(0xa949d735,0x1de89776),L_(0xc44fd2cc), + LL(0x23ce7894,0xc54039f7),LL(0x1ad9c99d,0xaba8109a),LL(0xa0ae59e6,0x5269a308),L_(0x65e4eb8f), LL(0x63914a92,0xe212c370),LL(0x2c3e1f0e,0xcaf748c2),LL(0x8869d4e8,0xc88e85fd),L_(0xbc2a9592), + LL(0x0874ad96,0x0f4e6e42),LL(0x2bfeb37b,0x3e5a96cd),LL(0x6bb45b1b,0x53c918ad),L_(0xb8bd0546), LL(0xfd07d3ba,0x0e33f7bc),LL(0x9e58beb5,0x58f58e4a),LL(0x26578e47,0x986f5206),L_(0x90135d8a), +}, +/* digit=3 base_pwr=2^21 */ +{ + LL(0x089bae53,0x4190a603),LL(0xec5c8b6b,0xa64a64f3),LL(0x8bca6d9c,0xba9eddb4),L_(0x3ebc0dcc), LL(0x9555731f,0xa709d0a0),LL(0xc3d3be2f,0xa147a9eb),LL(0xdae0ded6,0xf474acd5),L_(0x965b8988), + LL(0xa9158749,0xfb05e5ec),LL(0x7df5ec60,0x5b8ef888),LL(0x2d8170fd,0xd08f651c),L_(0xe08fb325), LL(0x1ccffe04,0xa2b2a173),LL(0x20fa30ea,0xe5f33422),LL(0xd169c10d,0xc7e20c19),L_(0x366327ee), + LL(0x5d85ed76,0xf62aa1bd),LL(0xe359773b,0x38737578),LL(0x2b782c08,0x31df9439),L_(0x80e166af), LL(0x673b4b07,0xa7a7bd40),LL(0x50ca47c9,0x95ad7602),LL(0x8ed45ebe,0x4a7eebb4),L_(0x03e9fc9d), + LL(0x33fde338,0xaba113fc),LL(0x0e6fb34d,0xc0022adf),LL(0xe2da2a97,0x491a39ca),L_(0x54e2ec96), LL(0xb883d291,0x3d0946d9),LL(0x6f792b36,0xd8794d71),LL(0xd78ca8ec,0x13094248),L_(0x286898ab), + LL(0x0c894527,0xb354ae4c),LL(0xf18c6f77,0xa39fcad9),LL(0xf685126a,0x7bd039dc),L_(0xfb3809c0), LL(0xe8ec1ffb,0xc99f8944),LL(0x7c71a341,0x08ed706a),LL(0x21f4ed22,0x4c371e86),L_(0xa650cd97), + LL(0x64586632,0x8e7cfcb0),LL(0xfcae4f5c,0xe83d4510),LL(0x62196ce2,0xc9a4fa0c),L_(0x83e1de7b), LL(0x4871e08f,0x8f0bc09a),LL(0xfa78f5d2,0x1e32cbe6),LL(0xda32a5ee,0x23d217eb),L_(0xf8df814e), + LL(0xefdc00b8,0x80a1f116),LL(0x670e56db,0x0b205db0),LL(0xd1c521ab,0xcaa4c48a),L_(0x41b37b7f), LL(0xde1b3d20,0xf6421dce),LL(0xafa1a3e7,0x906ec304),LL(0x5c6bc442,0xbc2935cf),L_(0xbe169cc3), + LL(0xa597c946,0xe5876159),LL(0x308bf446,0xcc637e99),LL(0x8047e56a,0xdb6a36f6),L_(0xe8293346), LL(0x51f3fb7c,0x93a30fbd),LL(0xcd2f5b41,0x3e0a22c7),LL(0x0b7b25a0,0x3d9fe740),L_(0xaafd7816), + LL(0xe59f76ea,0xc167cd8d),LL(0x0d3f30f7,0x07b77d41),LL(0x3b140b84,0x6554fb45),L_(0xc105ccc8), LL(0x72d13268,0x69c72227),LL(0xf0450467,0x0ba87831),LL(0x88e4350b,0x1e0bbd29),L_(0x6f9f2dff), + LL(0x8eb3482c,0x1e9d015e),LL(0xf4db6e3d,0x3a6f5318),LL(0x60df1945,0x91e766b6),L_(0x2499a2cf), LL(0xe58a4308,0x0468acbe),LL(0x7a3c83da,0x9eeaeaca),LL(0xfeedfeca,0x95c8fd77),L_(0x9f35aedd), + LL(0xdf4b1836,0x1d85a174),LL(0xefd882f6,0x2cdd973c),LL(0xfcbc0529,0xff17a943),L_(0x6d3cb4e6), LL(0x7dbddba8,0x1d223311),LL(0x680691e3,0x6b1fb1d6),LL(0x2d7d7413,0x3d6056c3),L_(0xf379f1df), + LL(0x75f0d3c3,0x5a2e3385),LL(0x66a60d5f,0xa99c6fca),LL(0x08231783,0x2621fe6f),L_(0x282e92dc), LL(0x9beb8a2d,0xa53267dc),LL(0x48813229,0x2abc54c5),LL(0x18f63ff9,0x25a10917),L_(0x06fdd515), + LL(0xd5250739,0x866c8a6c),LL(0xb47ff8e8,0xe99cef30),LL(0x360eff22,0x64925640),L_(0x392f59d5), LL(0x58d96359,0x18c35e2f),LL(0xe8c753dd,0x5ea13624),LL(0xeef6fc01,0xdf425955),L_(0x3afc5278), + LL(0x6599bea3,0xedc68cb7),LL(0x69d044d7,0x3950ba3f),LL(0xb3b58152,0x8eb64dda),L_(0x49c378c0), LL(0xfcacec23,0xaf6b16d6),LL(0x47e931cc,0xbc7d2577),LL(0xcb7189bc,0xa9051236),L_(0x2aca5249), + LL(0xb4223fb7,0xad339986),LL(0x9bcc65a5,0xe221979a),LL(0x278964b2,0x7810f1da),L_(0xf8f2be96), LL(0x225ed2c6,0x7e983036),LL(0x59fa9f23,0xfcfa0c31),LL(0xbc86f69d,0x918a9c4a),L_(0xb726eddd), + LL(0xcabaac34,0xac70d023),LL(0xe4e20984,0x7fc6e9c4),LL(0x0ffac935,0xcce8d103),L_(0x6efdaa99), LL(0xd5d8263d,0xea3876d9),LL(0xa0b5e29a,0x1319056a),LL(0x65ae7e78,0x9fa645d3),L_(0x93cb4927), + LL(0x09cc8e5f,0xac6cb27b),LL(0x414e945f,0xb503f62a),LL(0x73d9e98e,0x4809a03f),L_(0x2d14c85e), LL(0xbf213bd6,0x0b05bc27),LL(0x74a3b39b,0x4314e710),LL(0x003b8462,0xed2336e3),L_(0xc6f085ad), + LL(0x3c7d36a1,0xd9687e2e),LL(0xd82a965c,0x7a3f9fe9),LL(0x088401a6,0x068fa075),L_(0x07ba4f19), LL(0x0f0c8a66,0xf91cec2f),LL(0xb6cda991,0xb14b14fb),LL(0xa7b1febe,0x50d069b1),L_(0x5d3d8cd6), + LL(0x10626da8,0xf8949e74),LL(0x5c71b954,0xbe5cd298),LL(0x28006acc,0x6ba62a11),L_(0xe08cb4da), LL(0xe8cb22bc,0xc4d932f1),LL(0x9059e525,0xb085b630),LL(0x72f4258d,0x4cc4c854),L_(0x0a971745), + LL(0x1acb63c1,0xdaf48ca2),LL(0xa543ba94,0xe2648935),LL(0xd71707e5,0x809ef386),L_(0xe30d9ba3), LL(0x12555c2b,0x69eeeae4),LL(0x6910f9ee,0xea668f6a),LL(0xe8c90ba6,0x1896385f),L_(0x12c1a610), + LL(0x9f5a846d,0x2d268c8b),LL(0xd0a57e22,0xf1a092e2),LL(0x907afb7b,0x927d9d98),L_(0x4fb8839b), LL(0x54540741,0x967900e2),LL(0xa9ff207a,0x628af581),LL(0xee9dbe19,0x1bad27e2),L_(0x37d4f91b), + LL(0xd5bb2ea9,0x6069565b),LL(0x7e61e032,0x973dd1c6),LL(0x1034f977,0xa6921003),L_(0xf41e571f), LL(0x58bdabb3,0xeaa8bfa5),LL(0x77a67171,0xe71c31a6),LL(0xf9f34af4,0xa78106eb),L_(0x7aed89ac), + LL(0x9de5d621,0x8c1c931d),LL(0x2befbabd,0x5b08557f),LL(0x7e64aafe,0xb97c5d4d),L_(0xf99509a4), LL(0x10d5a953,0x7372eddc),LL(0x47ce48d3,0x24c1d7d6),LL(0x28b7f369,0x1fd874f8),L_(0xfdab8a95), + LL(0x886ba3df,0x7f6337ce),LL(0xe5943606,0x7b3e3023),LL(0x21b5b09f,0xa28353d5),L_(0x917b4519), LL(0xd1005ca4,0xfd5ad004),LL(0x73dbf462,0xc7335825),LL(0x7f25c9d4,0x3b50b66b),L_(0x35b3bf78), + LL(0x9b2e5ded,0x4b668d6b),LL(0xbe5e047a,0xd58c1f6d),LL(0x2bc0144b,0xee508f32),L_(0x9d2116d4), LL(0x7efb1e4f,0x5d9a5637),LL(0x8ab6e9cc,0xcd52f6ad),LL(0x954172ff,0x7ca23907),L_(0x9fe1053c), + LL(0x0b042690,0x711be7f8),LL(0x4c525b55,0x549592f5),LL(0xac72c62f,0x962efa73),L_(0x92c58056), LL(0xe8cda600,0xa38c53ed),LL(0x917ebbe6,0xd1e2d2e6),LL(0x6ab0ae35,0x25682606),L_(0x0603c2dc), + LL(0xd25fb822,0xb1a2e953),LL(0x22b420c7,0x030fabe0),LL(0xffe309b6,0xda6ba55a),L_(0xdce7859f), LL(0x9c46c82d,0x96d10a75),LL(0x057f1caa,0xd24573e9),LL(0x72ae9f6f,0x8794d9eb),L_(0xab64c20f), + LL(0xfc0d12fe,0xb289f4d7),LL(0xd9e2a94a,0x133fa143),LL(0x4c9fd385,0xecfdb1d1),L_(0x0675dcf1), LL(0xa5804aec,0xaccd6ef1),LL(0xfad7706e,0x577c5c78),LL(0x0667c6b3,0x7a5f797a),L_(0x2207359c), + LL(0x03a638e7,0xf3f519e2),LL(0x06628ac7,0x06647de5),LL(0xe38c99a3,0x86774ade),L_(0x8f775fea), LL(0xe57995ae,0x1bf25ffc),LL(0xf9325842,0xcccd3a9f),LL(0x34f4502c,0xabcab086),L_(0x064e829c), + LL(0x27f04a80,0x5fefab0b),LL(0x06295d26,0x2a4f8979),LL(0x234674d8,0x9c70b1f8),L_(0x01e05c18), LL(0x49f25ab4,0x352749e2),LL(0xee248ea9,0x9909ef81),LL(0x3b3c0b4c,0x7556a699),L_(0x4df0080e), + LL(0xeb394e8f,0xc8a345b1),LL(0xaf868ebc,0xbf72406a),LL(0x9a7fcfbb,0x959c94fa),L_(0xea271ece), LL(0xb72d1e17,0x1ea91da8),LL(0x252b7410,0x004276b7),LL(0x560e8135,0x8ada4812),L_(0x4d0eaa23), + LL(0x857c7d7b,0x11e78f96),LL(0x55ff3762,0xcd19ea8e),LL(0xa90b8b11,0xb557ab25),L_(0x98bfd175), LL(0x436277e5,0x453f5ec0),LL(0xbd8cc123,0x74f19c3e),LL(0xe53f50d1,0xaf6d61de),L_(0xb809c27b), + LL(0x70f6cada,0x22d1e307),LL(0xcafcf6b1,0xc39e2653),LL(0xb5d40ef0,0x602ae873),L_(0xb5bef707), LL(0x37b8abfa,0x47aa28a0),LL(0xc7e7a44e,0xfe824073),LL(0x9f424f8c,0x39a88470),L_(0x08171c6a), + LL(0x80811691,0xd43ff3e6),LL(0x5bfd96fc,0xb502656b),LL(0xa9d5f891,0xeb92efe6),L_(0x5ac4853d), LL(0x1f5ef580,0x90443655),LL(0xe8ead180,0xb1c5c5f0),LL(0x84a289e5,0x98bbea4f),L_(0x8bd0878a), + LL(0x7035018f,0xd233645a),LL(0x2cec3fe7,0x567fe1b2),LL(0xc6974f80,0xc7a73902),L_(0x0b481a08), LL(0x9adde606,0x9eb32485),LL(0x9b2ec275,0x16392d5c),LL(0xdb3bea46,0x0653d374),L_(0xab7f0ce3), + LL(0xe1cac3cf,0x89e007fd),LL(0xa77a41f1,0x3300e242),LL(0x6059351c,0xed2ec819),L_(0x02bfa9e8), LL(0xba1e89c6,0xa9065e31),LL(0xd4e23efc,0x6a1da501),LL(0x9c16b8bc,0x465d2c9d),L_(0xa1b1cda1), + LL(0x9ecf8570,0x54df839d),LL(0xb93666fd,0xb9514f4e),LL(0x1fed47ac,0x7e8268de),L_(0xbda2efca), LL(0x5da57404,0x428af14c),LL(0x9ee2b07c,0x1471974b),LL(0xda44c29f,0xfffd22e4),L_(0x296f1761), + LL(0x917860b8,0xb42b17e2),LL(0x6d4c5ddd,0xf2571ed2),LL(0xabe94b27,0x89be3310),L_(0xde98e510), LL(0x346eec5a,0x9d4ca35e),LL(0x96df2661,0x11588b07),LL(0xb28f2b86,0x96395d9e),L_(0xc8d9fdcf), + LL(0x031a12c5,0xdbc075c3),LL(0x54d26d6d,0x6ab8c6ef),LL(0x23c171f2,0xca47c96b),L_(0xecedbb06), LL(0x8c7cf280,0x31539156),LL(0xe9bac065,0xb7662b30),LL(0x552f0920,0xef1cbf3a),L_(0x2cb42440), + LL(0x9fba2bd9,0xed539df1),LL(0xd5c7355c,0x718a4f45),LL(0x5ced43ea,0x10d60144),L_(0x9618c03b), LL(0x55e1556d,0x8b9bcb4a),LL(0xe3767fbb,0xf1a46d41),LL(0x2a5c6f77,0x8a009344),L_(0xa23cb6b8), + LL(0x0db9a0c8,0xfd4a0f59),LL(0x7ea78a85,0x4d12f34e),LL(0x258d360e,0x951a2674),L_(0x7b3935b0), LL(0x4a231357,0x75e447f0),LL(0x80a9bfbf,0xc988c231),LL(0x8cd933d2,0x1c914702),L_(0x8da2914d), + LL(0xbe785bb8,0xbb8f1e8e),LL(0xbb61ba00,0xa9e04405),LL(0x018ee54b,0xf3ed084b),L_(0x9b33e2c5), LL(0x796aad53,0x90c49201),LL(0xdac991d0,0x9174f367),LL(0x06286fbc,0xfa92d2f7),L_(0x8e51716d), + LL(0xce4d1ca0,0xcd0c67a7),LL(0xe3fcc974,0x5197d346),LL(0x4936eae1,0x38aec941),L_(0x49b20db5), LL(0xf9c12e70,0x8806a867),LL(0xfd73d3b4,0xfe592936),LL(0x8553e67e,0xbb731824),L_(0x5f399b3f), + LL(0x7afb78c1,0x04dbb078),LL(0xd2ce86a5,0xbe71dc62),LL(0x636ee6c6,0x5c029f32),L_(0x90ef1fcb), LL(0xa90f74fc,0x05c18efb),LL(0x812addf4,0x86cc7880),LL(0xdf2ff660,0x34c03bb8),L_(0xb1740945), + LL(0x642146f4,0x18fd7ff7),LL(0x8a416ad5,0x8b9a8120),LL(0xb7d2ecb8,0x8226e0af),L_(0xbb5fb220), LL(0x7d62e2a5,0xcaee42bb),LL(0xb76b0d06,0x9ae1502d),LL(0x31322587,0x678f208f),L_(0xa07b89ea), + LL(0xdd6c08b2,0x5899ed84),LL(0x45083585,0x50d7aafb),LL(0x7217b173,0x04f9e7f1),L_(0x59198da1), LL(0x5dd6befe,0x11d6cf0e),LL(0x908d5768,0xa9128791),LL(0x51f15b45,0x7dd32a0d),L_(0x58045077), + LL(0xafc5b319,0x885c6636),LL(0xf7bef8ba,0xc50dd479),LL(0x478b8ac8,0x58aa9fce),L_(0x9d21686d), LL(0xe233d315,0x541006e3),LL(0xdfc86dcf,0xdfefb8ef),LL(0xe98f7f2c,0x73e745a7),L_(0xfc620932), + LL(0xbda630b0,0xd49816e2),LL(0x68c388d9,0xabd8c48b),LL(0x741de91e,0x3b976a04),L_(0xf24259fb), LL(0x5a923f4f,0xcbbf2684),LL(0x56659522,0x8a51b8af),LL(0xea31954e,0x43c8b17e),L_(0x3d1de35f), + LL(0x1067b268,0xddd685da),LL(0xa178f2e3,0xe840e9b4),LL(0x859daa46,0xc3965546),L_(0x65f74f36), LL(0x3ebc87fc,0x31952a50),LL(0xd0b694c6,0x2ba1d2d4),LL(0x4f51b3a3,0x20b2748b),L_(0x54b61458), + LL(0x450b7ea8,0x91be987e),LL(0x92390774,0x530b2e3f),LL(0x31cf3730,0xd8dc53f9),L_(0x490c450f), LL(0xe4e70ece,0x604fd792),LL(0xb689243f,0xa20bf532),LL(0xb2d8f482,0xdc31ef3b),L_(0x506dd3c6), + LL(0x89cb2010,0x1196c48c),LL(0x6755b53e,0x59cee16a),LL(0x6a0e74ef,0x2da63e64),L_(0xaa8ea0cd), LL(0x661a0e7f,0xb013649e),LL(0x1ab0635d,0x56e8f0f1),LL(0x9858a79d,0x65938865),L_(0x3acb5b92), + LL(0xf39d8f51,0x28b2de13),LL(0x6b120542,0xd10ea1ae),LL(0x5ea493a4,0xbe9b2f46),L_(0xcdd3cbfb), LL(0x9af8b9c3,0x543265a5),LL(0xf3a605c4,0xe8ee990d),LL(0x92bcbe09,0x1d71ae8b),L_(0x4ad0176d), + LL(0x77ad60e7,0x183493f0),LL(0x3e8ef423,0x4d9cc856),LL(0x065effba,0xa9ccf6fd),L_(0xb0500b2e), LL(0x122bf6db,0xfb31da5d),LL(0xe16bd325,0x3de8eb0a),LL(0x2bb46b58,0xcdab125c),L_(0x8179b8bc), + LL(0x72f249a0,0x54823e45),LL(0x263960f2,0xb1c1bcb0),LL(0x21ab8a5f,0x193b2d98),L_(0xb940533d), LL(0xc80b7cd9,0x7586193c),LL(0x671090ab,0x40d0049f),LL(0x216acee5,0x6bb91540),L_(0xcc4fc3a7), + LL(0xc26826e4,0x0e69184d),LL(0xe86f6c8e,0xfa8a33b4),LL(0xe88688e1,0x9beec95a),L_(0xd008257c), LL(0xa638dd0b,0x35395ef4),LL(0xacddfb2b,0x005b4fcb),LL(0x086efe98,0x8a1a0ad3),L_(0xb4d51fd4), + LL(0x9613f782,0xd8510368),LL(0xd5dedcbd,0x7ea65e3d),LL(0xab06f6a4,0xbd599498),L_(0xc5aa099e), LL(0xf3eb18eb,0x32f40f7b),LL(0xad93fbe0,0xfde611a8),LL(0x40b951f1,0xf5751752),L_(0xfc37b6e1), + LL(0xc4dba0cc,0xaae77bea),LL(0x58b91be1,0x7a9f5f3a),LL(0x6ad767a4,0x9b47cd71),L_(0x24d37cee), LL(0x7268bfef,0x70ac8b25),LL(0x2da13b60,0x16423d93),LL(0xbadf18ad,0x62a037c8),L_(0x78224e07), + LL(0x334baa48,0x6a2f2198),LL(0x1578e706,0x50c74263),LL(0xa273c223,0xe95f128a),L_(0x2b3feceb), LL(0x0c8d88cc,0x4d1450aa),LL(0x247fea5e,0xb5c004a3),LL(0xcc56e421,0x32f2d692),L_(0x001d3250), + LL(0x63905c2b,0x3d1a2b08),LL(0x1192eb20,0x7a96aa59),LL(0x194cd936,0xcbfbf813),L_(0xe3ba3ce4), LL(0xdc1fe33e,0x8d6355cd),LL(0x36393a36,0x3c74701a),LL(0x6d3b3d2c,0x50d98fcf),L_(0x2da52d36), + LL(0xa9690fb6,0xd68d0be2),LL(0x87600722,0x2fa3c8b8),LL(0x6a898b0f,0x64b1c25c),L_(0xf26df29e), LL(0x18b74321,0x0562d2ac),LL(0x116fb9cd,0x1b89331d),LL(0xadee6e40,0x19e073f7),L_(0xdcd869ef), + LL(0x39df6bf7,0xd9747842),LL(0x484e4a18,0x9ca1c1ce),LL(0x2181acfd,0x09d6f90f),L_(0xae25cfad), LL(0xc2644ecc,0xb6aa912f),LL(0xeb2922c9,0x767976e7),LL(0xbe3728a6,0x3b8ce6be),L_(0x04eb6141), + LL(0xd6e6d0b5,0x618ad07e),LL(0xc862e8d5,0x926d38e4),LL(0xca247f9d,0xaa268736),L_(0xbded92f5), LL(0x1a6031fd,0x9481e659),LL(0x42b36d96,0xc8b3617c),LL(0xabfa2ad2,0xd3f46f02),L_(0x1cbbeccc), + LL(0x2057edf3,0x2b94f0fe),LL(0x608a4b71,0x7ca57706),LL(0xdd35cef8,0x236745ba),L_(0xb4d5d169), LL(0x99412994,0x03f0b33a),LL(0x826e0d5f,0x0c5801a2),LL(0x31eb9951,0xf2cc0540),L_(0xd954458a), + LL(0xf649bf08,0x2c486c0b),LL(0xa5afc024,0x9417e436),LL(0xb063a531,0x38aa1ed3),L_(0x1558aff7), LL(0x004fbb10,0x904e9ed7),LL(0x3494f0b6,0xd2555c07),LL(0x365e3d05,0x0f32df2c),L_(0x5c8f30d5), +}, +/* digit=4 base_pwr=2^28 */ +{ + LL(0x2f432139,0x6a7ebdd1),LL(0x676d690b,0x25d6c68a),LL(0xf169ac1a,0x70ef4a18),L_(0x3fe9e66e), LL(0xa193f6f9,0x9b515115),LL(0x6b6b0581,0x4399b66b),LL(0xd0e88db2,0x52c4fcf1),L_(0x31c324fa), + LL(0x62e391aa,0x403677bb),LL(0xe67af8a2,0xcd3fd0e0),LL(0x0b4be307,0xdcb51813),L_(0x88ce9bf6), LL(0xe4877d66,0x0f788e98),LL(0x8400154f,0x1742afd0),LL(0x768edb4a,0x80428f26),L_(0x277da6c6), + LL(0x0b15cc70,0x6933e246),LL(0x8f4bec34,0x8bc92958),LL(0x13b62772,0xc9121c92),L_(0x543662c4), LL(0x4549aa77,0x1090a59c),LL(0xc62e7c6f,0x066a0956),LL(0xa38394a8,0x19c57ed5),L_(0x4823e8cf), + LL(0x003e564b,0x61b100d2),LL(0x416f9398,0xaf3087cc),LL(0x8d0f5a48,0x11e5f34b),L_(0xe047da80), LL(0xe946d928,0xc598885a),LL(0x455959c9,0x838b9ed4),LL(0xe69bdd02,0x6bdbe44c),L_(0xd3377883), + LL(0xd1de4464,0xe93c7baa),LL(0x23daaa17,0xb54baf63),LL(0xcf91e74b,0x64e1431b),L_(0x1ae9e235), LL(0x43fd6c3d,0x2c00590f),LL(0x0ba0b46e,0x98d151a0),LL(0x1ee4dc18,0xe61cc51e),L_(0x93805289), + LL(0x15b970a3,0xa33f496c),LL(0xc65b315e,0x3f6d708b),LL(0x25723787,0x2ca2c551),L_(0xc7e943c6), LL(0x9a10432d,0x1845a431),LL(0xb33bb1fe,0x81764017),LL(0xf5b5a1e6,0xbb41c15a),L_(0xf5bd0a9c), + LL(0x042a23c5,0xe9864244),LL(0x73ab2381,0xbb7b2648),LL(0x0ed12b18,0xe3536d47),L_(0x0a040c86), LL(0x9c5a68b2,0x550f5800),LL(0xa16b2d10,0xfa0aebe0),LL(0xd20e3020,0x40239ea3),L_(0x13feb317), + LL(0x8e882fb8,0x12206859),LL(0xd53242c9,0x3e1a95f6),LL(0x1c2921f5,0x765eedc1),L_(0x84f582be), LL(0x525bb499,0xf8de62be),LL(0xf9eee520,0xd0a27199),LL(0x209f3c9d,0x721633cd),L_(0xc2e9678c), + LL(0xc8a35690,0x852acfe8),LL(0x92b626ca,0xdde1d1be),LL(0xf571ade2,0x918c76fa),L_(0x5b7490f5), LL(0x38c8b0c0,0xdf1662bd),LL(0xbb1e6d8b,0x470868d3),LL(0xd821c745,0x9a06de8a),L_(0xcaec5c2c), + LL(0x64f233d6,0xc1321d2a),LL(0x74538108,0x30546bc8),LL(0x2f1d65e6,0xc8d1c486),L_(0x8ecd7a1e), LL(0xccea4b95,0x3466245a),LL(0x700ed4ab,0xf5d50910),LL(0xfa32badf,0xb5a66db1),L_(0xfc4883b2), + LL(0xb4589512,0x962b479c),LL(0xc4a7efe4,0xb97069b4),LL(0xa0d20fc9,0xd0ef4040),L_(0x76eabbd4), LL(0x38b07e76,0x17a75081),LL(0x4bc6d910,0x3d3dbafe),LL(0xff0f721e,0x36e896fb),L_(0x6146e5ef), + LL(0xab1ff699,0x6fd3001f),LL(0xe508cb13,0x5b228934),LL(0x80a73390,0xcc02c93c),L_(0x6ad8073c), LL(0x9444deec,0x5f92bcf6),LL(0x8847a6eb,0x06c7d42b),LL(0x758f4aa4,0x40b45a83),L_(0xdb4ada91), + LL(0x0083de14,0xfff55c75),LL(0x661c826d,0x317fc21d),LL(0xb311e146,0x6bdaa80c),L_(0xcad158e9), LL(0x69952fc9,0x4502f454),LL(0x239ecee1,0x74f37d8f),LL(0x2b344eb3,0x161e1cf8),L_(0x02ce3f80), + LL(0x645e7684,0x0009b919),LL(0x53d14c44,0x8105efea),LL(0xcbb0101a,0xdf767e4c),L_(0xecc4eabb), LL(0x0c2ac7fe,0x70cf6d6c),LL(0x0ede1159,0x17089527),LL(0x4162258f,0x5ed679cb),L_(0x5e706cdc), + LL(0xcd1206e2,0x33b4ee64),LL(0x78fd4645,0xcab93e67),LL(0xb807ac0a,0x7e760bcf),L_(0x923934bf), LL(0xfa2159f5,0xaf2ed832),LL(0x918e957c,0xe1d5b548),LL(0xdf411692,0xde9c3716),L_(0x2c76553a), + LL(0x3d4c49ae,0xd3a72d08),LL(0xa5e4f840,0x03aca3cc),LL(0xd9bcbe83,0xf00706d6),L_(0x986b79c3), LL(0x412a3304,0xefb01a68),LL(0xa0bb40d2,0x19e3729e),LL(0x2d182974,0x57b4bca1),L_(0x43c566de), + LL(0xb3db024c,0x2f1c7c7b),LL(0xfb3c8bb1,0x86b88161),LL(0x1addae50,0x29206628),L_(0xa115d8d7), LL(0xc7b9fa3c,0xcf3f272c),LL(0xf2a0db00,0xb0b04d71),LL(0x1a242dce,0x048ec06d),L_(0xbe72542f), + LL(0xc6eb32f1,0x515f5aba),LL(0x995a0552,0x9e89cf10),LL(0x809459ef,0x2a52fccb),L_(0x4bf08352), LL(0x31a7e6c7,0x94c73471),LL(0x507e309b,0xc1732a6e),LL(0x006ea760,0xd3410c08),L_(0xde332d38), + LL(0xe9a863af,0x05608149),LL(0x971e9a21,0x80261bb1),LL(0xcf890a46,0x607c23ca),L_(0x8c2f3995), LL(0xee178996,0xb8358125),LL(0xc384ccce,0x8630d610),LL(0x62e3cda6,0x637377fe),L_(0x36145889), + LL(0xebe66e5d,0xa7ccaba3),LL(0x66ee0e61,0xe5d23ef3),LL(0x8b52d5b2,0x9b192439),L_(0xe2507e26), LL(0xdebe6661,0xf0320f46),LL(0x164afe7b,0x06272a4d),LL(0xa9cd4c00,0x53be168f),L_(0xbd05717a), + LL(0xb9a090f6,0x82377241),LL(0x84720302,0x8cab5c0e),LL(0x3e8eab14,0x8131dcb9),L_(0x8dffb402), LL(0xe82db9aa,0xca87536f),LL(0x7476d50d,0xc99eb71b),LL(0x40036a93,0xad40e0b3),L_(0xae5c279a), + LL(0x203e82ae,0xce0565d4),LL(0x4590aef4,0xf8303097),LL(0x813e461c,0x4d52d217),L_(0x64b75926), LL(0x1774cb88,0x51483218),LL(0x4046569c,0x804c4220),LL(0xbd282e8b,0x394837d5),L_(0x5a6b2c1e), + LL(0x7eb4bbcf,0x58408340),LL(0xa28c5b85,0x9cd1f588),LL(0xac9ddd98,0xdf706c18),L_(0x62a72bba), LL(0x1ecd1293,0x7437fa2c),LL(0xafe0e361,0x5b13b49c),LL(0x29283b54,0x0c099515),L_(0x469c2258), + LL(0x611af3b1,0x1869604e),LL(0x69cd9356,0xb7d10b48),LL(0x161651d5,0x8a8472c4),L_(0xa23f6947), LL(0x8a84a570,0x9ca82d05),LL(0xdd9ffcc5,0xbdf0cb1e),LL(0x7c9cfaaf,0x8b227092),L_(0xc52e4c68), + LL(0x344a8ce2,0x66b5d593),LL(0x661a0707,0x277d6af9),LL(0xf0ed8bfb,0x9f814a89),L_(0x68ff82da), LL(0xff4e0f6a,0x54b6136e),LL(0x125a9d0a,0xe9e8996a),LL(0xdbfa6d51,0x0914dd93),L_(0xacae0da7), + LL(0x326565a3,0x41cc5966),LL(0x8e32f402,0xa314cfc5),LL(0x99f4bcb7,0x0ba8496e),L_(0xcbcf93d4), LL(0x58afb0bc,0xf2f01549),LL(0x22877383,0x775e4843),LL(0xf270905d,0xfeb2c8a5),L_(0x15d8ffe9), + LL(0x111a2a90,0xc56f7437),LL(0x435cbc99,0x64f9c9bd),LL(0xedb87fdc,0xc1d19083),L_(0x702066e8), LL(0xc8dbc37a,0x2c1c4233),LL(0x672ee8b0,0xc183ca5e),LL(0xfc342416,0xaa518aba),L_(0x34e2bf87), + LL(0x13de8a41,0x3bd53f9f),LL(0x38707713,0x326f7696),LL(0x6251ddca,0x4c9a0c1a),L_(0xe7e323fd), LL(0x6876f934,0xd33612f2),LL(0x93f34e7a,0x1b94f160),LL(0x0c42a981,0xdc0b91ab),L_(0x548c92c1), + LL(0x4cbbb4e6,0x72ed1d97),LL(0x0de1c184,0x5a3987bf),LL(0x86904dc8,0x45379ece),L_(0xbe4c089a), LL(0x87ac4c8e,0x6d5238ae),LL(0xa27b3f88,0xbf067204),LL(0x21e17b1e,0x122542d6),L_(0xd6e7acc3), + LL(0x84631900,0x01b419d0),LL(0xdb03ac70,0x4d15d553),LL(0xc5304466,0x5f684940),L_(0x11655587), LL(0x8f896988,0x196f70de),LL(0x771ca6aa,0x79277c70),LL(0x4a94022c,0xb69fe2d3),L_(0x076b1dc4), + LL(0xb8ac3536,0xecd18f22),LL(0x225dd231,0x86439125),LL(0x23c21f5b,0x43b63acb),L_(0xf5b2b43c), LL(0x9b9dfcde,0x7ec7a341),LL(0x362a2c90,0xd7db9d5b),LL(0x308c8fcb,0x608d86ad),L_(0x8858eb99), + LL(0x89b3a627,0x82f0ce08),LL(0x8ffc3544,0x4c9d1c88),LL(0x1414581d,0xf2cd74da),L_(0x2ab74c29), LL(0x47673821,0xd4c61a1b),LL(0x8b9a0584,0x4166899a),LL(0xd7114188,0x7b53c8e6),L_(0x4e91e1e4), + LL(0x83a918f0,0xe472f5b7),LL(0x120a4cb4,0xa54211cb),LL(0xd29feaef,0xa568a4a5),L_(0x1f0a0815), LL(0x80033652,0x52d9147c),LL(0x1dd52d78,0x73e66c07),LL(0x99e7538c,0x25ad4a06),L_(0x8a35cc7c), + LL(0x45837327,0x3daaae35),LL(0x214d493c,0x0495de90),LL(0xc0f2cff5,0x5e7107a4),L_(0x46024f37), LL(0x4ceabd92,0x0f39120b),LL(0xd7f5199d,0x26eb2e5e),LL(0x2dba9f4d,0x2755fd79),L_(0x54915591), + LL(0xf3749f34,0xbd4c1fd7),LL(0xbe148855,0x0fc24339),LL(0x7f36ce43,0x41ceba92),L_(0x40c686c2), LL(0xd073af02,0xa2355b2f),LL(0xeb937195,0xa16c5398),LL(0xa1f3ab05,0xf7d38672),L_(0x90828e20), + LL(0x7c7ef0f9,0x37b7db08),LL(0x4de172b0,0x2ce3e250),LL(0x3f8b6d32,0x1253fef8),L_(0xc76ebff6), LL(0x54208a07,0xc53e233e),LL(0x9a8ef0b3,0x6f68e154),LL(0x2d43fb48,0x9a94edf5),L_(0xc8891237), + LL(0x85578ac6,0xf170445f),LL(0x115d7eb6,0xd46f93ef),LL(0x71666f11,0xdfdf4d27),L_(0xcd09d0bd), LL(0x742f2abc,0x658ac044),LL(0xe75c6300,0x0f93b4da),LL(0xa0de2978,0x5ea75aec),L_(0xd9cc6143), + LL(0x7e1b6b82,0x2a662d9d),LL(0xcd959485,0x38780971),LL(0x5ee58f00,0x69a1ad5b),L_(0x417599f7), LL(0xfe8f4d61,0xc5110167),LL(0x471554ab,0x3201dfa1),LL(0x4e2c7d96,0xa6a4af88),L_(0xe49c13ae), + LL(0x44dc3bbe,0x0a1a147d),LL(0x23d766f5,0x3c2cbc33),LL(0x5446ea58,0x737348f7),L_(0x49f6d997), LL(0x2488e5f7,0x17464a0c),LL(0xf5ac023b,0x5e39103b),LL(0x1c89d9f7,0x5a4700e1),L_(0x918b1cd7), + LL(0x1a52af4c,0xd9c4d8a5),LL(0xfb136a04,0x62eae24c),LL(0x627e23b3,0x2a546df5),L_(0x9518b383), LL(0xc577abda,0xe74b275b),LL(0x9e8dd1b3,0xbe223e4a),LL(0xb28d50a9,0xc1bf5392),L_(0xf0c4dae3), + LL(0x12450901,0x60cd114e),LL(0x19f30dd8,0xb57b5e92),LL(0x0a04952b,0xa5b0d428),L_(0x9fbf17a2), LL(0x998f1456,0xdc88c74f),LL(0x2eaf445d,0x39f1ad33),LL(0x24d076c8,0xb7cb2b9f),L_(0x4da895c0), + LL(0x2bde5f9c,0x296bb0ce),LL(0x1e460fe9,0x8b7ed12e),LL(0x0a2b9d60,0x779ddda6),L_(0x70ad87c8), LL(0xda6b433e,0x1216c7ea),LL(0x015e5322,0xfbda2b1c),LL(0x08a2c2b4,0x5c4f2da2),L_(0xda0125d8), + LL(0x7e27d6a4,0x5cbb2271),LL(0x0e27b02e,0xf69dabcb),LL(0x651b4dc0,0xf5b545b4),L_(0x89864ae4), LL(0xe7bf1ebd,0xbfa85080),LL(0x0162c99f,0x358c27ef),LL(0xc148b34a,0x39fddb94),L_(0x7de290f5), + LL(0xbe0c0f85,0x8d61f3ca),LL(0xe4f10dce,0x4cbdbb45),LL(0xd40a9135,0xdb4a6076),L_(0x869e961b), LL(0x191ed5b2,0xc8ea7a1c),LL(0xaed2c272,0xe62f26a5),LL(0xdd267430,0x49228bdc),L_(0x196c9b69), + LL(0xda4dc900,0x5f4b0d32),LL(0x357d26fc,0x47de3ca9),LL(0x8bc3e1fc,0x44a306a1),L_(0x6ebde62d), LL(0xc4fcbc98,0x66c5b1ca),LL(0xc2a2d52f,0x6365559c),LL(0x67f40398,0x6a1916f6),L_(0x9eb2b762), + LL(0xd3f693c6,0x97ae197e),LL(0xe2705942,0x4294f4ca),LL(0xb717c180,0xfe30cc2e),L_(0x7aa89b4d), LL(0xb42643a1,0x3d1c1e8b),LL(0x1731a1a7,0x4e550c0f),LL(0x4cdde0b3,0x85e85391),L_(0xab7103b7), + LL(0xe4b061b6,0x39f1ba7e),LL(0x21c91792,0xcfcb8f5f),LL(0xd1e29983,0x8102c46e),L_(0xaa0fed15), LL(0xf3e438cb,0x1df20d6a),LL(0x659104e2,0x43818f1f),LL(0xd4f49239,0x3a84be64),L_(0xca582a23), + LL(0x3118786e,0xce03f04a),LL(0x827606a4,0x107aaf6b),LL(0x10bcfdd2,0x277b88b9),L_(0xf2e4c5e4), LL(0xb9b85218,0xb9213c6d),LL(0x161c54e8,0xebb4f0be),LL(0x2814c386,0x859e6045),L_(0xac244cf4), + LL(0x7292e148,0xd6446675),LL(0xab795487,0x71b2851d),LL(0x5a242339,0x899b5dad),L_(0x8520bc7b), LL(0x72df7577,0xa366ee1a),LL(0xc17b463a,0x30d222b1),LL(0xa0af7c17,0xddf53842),L_(0x85b5501f), + LL(0x124bc359,0xc8d63689),LL(0xcd7b3fe1,0xbfb69ae2),LL(0x124ab94c,0x6971b236),L_(0x961e282f), LL(0x7f403da9,0xb92ac7ad),LL(0xc76e5b2f,0x5d04c417),LL(0x8df11606,0x0055901f),L_(0x666181f1), + LL(0xf49645d2,0x96fcbeb8),LL(0x745bf74b,0xe0b2a4ea),LL(0x24675bb4,0x531c5deb),L_(0xff3b4dca), LL(0x560275f8,0xd74deba2),LL(0x97100731,0x02589d49),LL(0xa26f7f22,0xb1b02a6f),L_(0x396cb1a0), + LL(0x7de707a4,0xc6165fe0),LL(0xc07709d3,0x6be7081c),LL(0xfdb883bd,0xb1a5a67b),L_(0xfd535ec8), LL(0x25257079,0x8234b458),LL(0x45bb7b81,0x9c1b3f29),LL(0x1f4c2d8a,0xa1cee750),L_(0xcf55f516), + LL(0x206c7d17,0x0e3c9acd),LL(0x75c44136,0xd3de8638),LL(0xbe92eabc,0xf4ca1355),L_(0x9e459c32), LL(0xbf750404,0x4bc1af9e),LL(0xc06b8bfe,0x20fc4197),LL(0xe0b85f76,0x7d557c99),L_(0xff091685), + LL(0xbe45dd17,0x90ad8901),LL(0x5513303a,0x62ba8a0a),LL(0x9b18a004,0x8286d6c6),L_(0x73c22592), LL(0x126eaadf,0xf6402ecc),LL(0x2bbee8cd,0xa309b99e),LL(0xab06f9cc,0x42089652),L_(0xc4a5e5a0), + LL(0xe1391274,0x6925cad9),LL(0x6d3b1102,0xe366ed4c),LL(0x2e19a20a,0x1593ba2f),L_(0x8c091708), LL(0x9083afa5,0x711ad8d6),LL(0x1a65086c,0x5e8645a5),LL(0xc052325d,0x182117dd),L_(0x865d46f7), + LL(0x6f70a66e,0xfb548313),LL(0x7a6b6eef,0xae61c7f4),LL(0xec447933,0x6a3d4ff3),L_(0x74866913), LL(0x35042671,0x38316d02),LL(0xb9635b34,0x6a9889f8),LL(0x2db27177,0xac233369),L_(0x567be1c0), + LL(0xbe9b9347,0xcf5edce0),LL(0xa18e3f0f,0xe93ce044),LL(0x05816a5b,0xa26ca8e1),L_(0x00da101c), LL(0xacb5542b,0xe04bccb7),LL(0xa1f2efbe,0x470e98a4),LL(0x67b00a65,0xdd0dbd8b),L_(0xc0be2197), + LL(0x788904b0,0xe5a39b63),LL(0x0a586a5d,0xc10656ab),LL(0xe22752b9,0xfc9ab49c),L_(0x72fc1c09), LL(0x31249974,0x8b426974),LL(0xd51e5946,0x0c7e10a4),LL(0xa79d2b5d,0x8d8169eb),L_(0xe4ca8644), + LL(0x93ac6205,0x923206a9),LL(0xd8894d89,0x311a6950),LL(0x992f9c30,0xd0679c8b),L_(0x0cde00fd), LL(0x5d5f7311,0xf07baea8),LL(0x3f8e094d,0x00100a63),LL(0x3d3b3316,0xbfbe9e92),L_(0xd269b8cc), + LL(0x3357ad87,0x88e7f567),LL(0x8dcb5c06,0x7dc56ca1),LL(0xc8f6e06c,0xa6e90095),L_(0x905b4a44), LL(0x99de2654,0x63aa3a4e),LL(0x1e51e461,0x486476c9),LL(0x5fb4cf43,0x35d139e9),L_(0x8725096b), + LL(0x016940d8,0x2005767b),LL(0x95ce3a0f,0x0dce7b68),LL(0xb593a6ed,0x2c5a09aa),L_(0x762f3669), LL(0xe1f753d0,0x52242366),LL(0x88d38ecc,0x05e8fe24),LL(0x759292df,0x8843ff7b),L_(0x041ca14a), + LL(0x8edf1adc,0x0062aa12),LL(0x4bff8a1d,0xb1aa7c7a),LL(0x28b33bc2,0x966aee37),L_(0xb4b3e7e7), LL(0x2d9ceec2,0x7f31f05a),LL(0xc7be6aaf,0x10ae244f),LL(0xc2c97f7b,0x7401d050),L_(0x1475b928), + LL(0xd008f9fc,0x3150b4aa),LL(0xd53ede1b,0xef6ad2c4),LL(0xe427c40d,0xf4e2d20a),L_(0xe5dd8b02), LL(0xb370a814,0xac4f8ba7),LL(0xaaacf4ec,0x44dbb479),LL(0x8e6b2784,0xd9aeb4f2),L_(0x4019cd79), + LL(0x004de7f7,0x680b826b),LL(0xc0a2ee02,0xc1ff25d4),LL(0xb25a4e2a,0x8b33b701),L_(0x3a2939d0), LL(0x9621d94e,0xe6feb01b),LL(0xd1311810,0xca0e2b81),LL(0xa701c8f2,0xdbcecc81),L_(0xeee95871), +}, +/* digit=5 base_pwr=2^35 */ +{ + LL(0x22d3206b,0xb5c4c69b),LL(0xf507ee25,0x886af51e),LL(0x454a0f10,0xf21fe543),L_(0x29d7720a), LL(0x22d2999a,0x7c33ec2a),LL(0xb4054a86,0xa4d8106e),LL(0xdb971595,0x8a74b5b0),L_(0x8850caf4), + LL(0x3bc6ae19,0xa04cb662),LL(0xf30acc6a,0x4f889fa2),LL(0xd7560ec2,0xb8b6110c),L_(0x4f8fab99), LL(0x6bc71596,0x4308e608),LL(0x2c9d6af5,0x10d12c43),LL(0xca6486c0,0x64c7b963),L_(0x28ca7953), + LL(0xb662cfb6,0xbcc43ce9),LL(0x3055b2de,0x39c5bde1),LL(0x68461c60,0x4caf7c30),L_(0x84c1743c), LL(0xab0e6d70,0xb5b3a77f),LL(0x0e121552,0xa5bc5e90),LL(0xc38f8d3d,0x6cee1666),L_(0xe3236cdc), + LL(0x88d7b871,0x4220e847),LL(0xa785596f,0xc1fb7864),LL(0xfa4258dc,0xf4276ec4),L_(0xfc15e9f1), LL(0x293a6fc6,0xa58a670d),LL(0x9382335f,0xd56aed8a),LL(0x3477647c,0x311f9766),L_(0x4f5dbddc), + LL(0x6de91978,0x69933c7b),LL(0x21ea5af6,0xf13eda55),LL(0x4b132119,0x12e00aff),L_(0x0ff8497a), LL(0x3253a2e3,0xb8a8554d),LL(0x7d054f07,0x9338ab00),LL(0xb896b1fc,0x91dbc7f3),L_(0x6470c920), + LL(0x51929e02,0xc7b39de9),LL(0x93ff36ec,0x2b86f570),LL(0xa6f2dced,0x802aa4a0),L_(0xdbd9b943), LL(0xa8bbc105,0x8c1a0edb),LL(0x7cecf63b,0x54aa43d1),LL(0x54af7ee1,0x78b1b385),L_(0x0498d501), + LL(0xe2a3944b,0xf34b8646),LL(0xe5ba9123,0x61656a11),LL(0xc14ff9dd,0x42cbd03c),L_(0x2891aad9), LL(0xd76dd4f7,0xf01f7308),LL(0x55808c68,0x793b012a),LL(0x2ceb5051,0x9d313349),L_(0x2f3b809e), + LL(0x1cd0a45c,0x59881514),LL(0x919ddc75,0xa32929ee),LL(0xf87e0bab,0x3037e369),L_(0xa090d36b), LL(0xb3841cb6,0x579ffa96),LL(0x38e224e4,0x904c8cca),LL(0x2cadda68,0x702d5e89),L_(0x12eff4f2), + LL(0x92015c38,0x52239088),LL(0xfabf2649,0xd339e361),LL(0xe9f6b8df,0xe84977d6),L_(0x3f58519b), LL(0xaa4bb20f,0x4357d0b8),LL(0x6c5351ad,0x8521ac57),LL(0x1253d822,0x9524c5cc),L_(0xfbce31d7), + LL(0xc07ab316,0x2e93f070),LL(0xfcee759c,0x25c23713),LL(0x132e8c43,0x6a329ada),L_(0xec4a7a29), LL(0xe8c7f287,0xd5e2b374),LL(0x36a40bf0,0xcf0c0149),LL(0x9da99c9f,0x25e16e85),L_(0x7730bda6), + LL(0x4bd943c9,0x4e15a70d),LL(0x37a376cc,0x82e59c18),LL(0xad1329bc,0xe14b9f7a),L_(0xac8b6391), LL(0x60924a1b,0xe412693b),LL(0xb1a885d1,0x20e7bad4),LL(0xdd7cdcc0,0x06667f09),L_(0xfb4d7bf2), + LL(0xb6cb033c,0x350411eb),LL(0xc83be0ea,0x4b4e8d93),LL(0xc7612043,0xbb338804),L_(0xfb1e56ce), LL(0xef5ca7d8,0x4b75557b),LL(0xfed92eee,0x5e0d7736),LL(0xc3259e9d,0x61eea795),L_(0xc28b498a), + LL(0xbe56f422,0x7363a98e),LL(0xe21a8af4,0xb0ad92ac),LL(0x2ee0c526,0x081ad4e5),L_(0x50ce73f9), LL(0xecb59a8c,0xfce29612),LL(0x52824a9e,0xdcb3b781),LL(0xb689cfc9,0xac6e83b5),L_(0x006e1f4f), + LL(0x1f147d23,0xd0a972a8),LL(0xe471b9e4,0x346ed362),LL(0xcfd1906e,0xd7accdbc),L_(0x293a4390), LL(0x79ac99cd,0x86a243d8),LL(0x06c24954,0xca928f18),LL(0x3453aba8,0x25495810),L_(0x36f891b2), + LL(0xd91398ce,0xba90aff9),LL(0x4d72eeba,0x14e030a4),LL(0x81c31d9d,0xe70f3373),L_(0xa716b2aa), LL(0x2213b99f,0x42ac4252),LL(0x6f657c45,0x415aca82),LL(0x7d074644,0x461026d7),L_(0x0dde683d), + LL(0xa01e18e0,0x4b805781),LL(0x8dd02690,0x35c19d90),LL(0x1d04223e,0x83277841),L_(0x7ea96551), LL(0xe06c5996,0x4c78c04d),LL(0x70dde026,0xc34eaf3c),LL(0xb58c0bf7,0xae645b00),L_(0x3afcf537), + LL(0x76eb1d35,0x708a8083),LL(0x5f7d6f8d,0xc51a45c7),LL(0x141c5fc5,0x61a1cb60),L_(0x78ecbf19), LL(0xba4b17cb,0x4cb03ebc),LL(0x9be1423d,0xf30899fe),LL(0xd63cb771,0x51acdaa0),L_(0xa32591dd), + LL(0x88195009,0x8de09e86),LL(0x76e08928,0x70957a85),LL(0xa653c234,0x59256cc9),L_(0xd8d51507), LL(0x5d7b9252,0x47309994),LL(0x0f159c21,0x3ff7ccc1),LL(0xfe9fe5bc,0x0339c473),L_(0x16784f00), + LL(0x24c3bafd,0xf2b37280),LL(0x1cbd8aaf,0x2790844b),LL(0x8a600a73,0x2324a737),L_(0x644765ee), LL(0x4ad1ba6a,0xb4dd3d30),LL(0xa23d1c2e,0x9d90ab4e),LL(0x953388e0,0x864110e9),L_(0xe9be065f), + LL(0x9dfdba3f,0x9c32af38),LL(0x69ee0a06,0xf93a2f14),LL(0x1fd2a13b,0x38105c26),L_(0xfe12d778), LL(0x3947424e,0xd4a50774),LL(0xd5b96217,0x8f7c85be),LL(0x0f0582fb,0x9f4dc144),L_(0x87629c9c), + LL(0x6470ccfc,0x377dc9b6),LL(0x9e11a84c,0x09816331),LL(0x610e3f5c,0x9becf104),L_(0x87cc9a9b), LL(0x62783e9f,0xbefa56b8),LL(0x89fa53e3,0x18a4bfcc),LL(0x0b3ed787,0xdb99856d),L_(0x68d6dbd1), + LL(0x9620574e,0xdd0b49af),LL(0x7e561904,0x68703702),LL(0x3056fb70,0xab404bf8),L_(0x4911d8a5), LL(0x0f24dcd1,0x91807631),LL(0x606c33cb,0x6847c0d5),LL(0x791f57f0,0x525adb32),L_(0x818f0a1f), + LL(0xa84199ec,0x1094f386),LL(0xd1afe50b,0x558c85a0),LL(0xab294ef9,0x69998dcc),L_(0xe0d8fa53), LL(0xa8d822a6,0x6ccabafd),LL(0x64a8c67c,0x4f2dce66),LL(0xd4a601a5,0x8082a5ca),L_(0x1c89a181), + LL(0x49e538af,0x4855546c),LL(0x6d3c05ab,0x11f89f31),LL(0x44996dfb,0x9e6dd61e),L_(0xe5054752), LL(0x053ae779,0x4c1c7978),LL(0xa7675178,0x49d3b92e),LL(0x72b800a4,0x307c1a34),L_(0xafc9ca2a), + LL(0x73bd9113,0xf1254531),LL(0x1e52b09c,0xb3c304d4),LL(0xfc186009,0x579623ed),L_(0x0b5f7250), LL(0x0a3ddce4,0xa4d79ef7),LL(0xff4c374a,0xd69618ef),LL(0x252cb608,0xf8e76342),L_(0xfcc58682), + LL(0x1cbe9e0c,0x586d393a),LL(0x480cf461,0x2e2acea1),LL(0x13a215d4,0xe3cf0a14),L_(0x82460da2), LL(0x38bbf760,0x188125b7),LL(0xcd5956cb,0xa80c3de5),LL(0x9e63be59,0xbfc652be),L_(0x275a3495), + LL(0xccb821df,0x965c6e19),LL(0x0b8b60d7,0x03204648),LL(0xd6030341,0xc11f7050),L_(0x4e201a9c), LL(0x5815f324,0xdd1a6a75),LL(0x73f70f06,0x7aaff126),LL(0xebc85955,0x11a05e07),L_(0x0cd2e37d), + LL(0x34fb8b2f,0x7afa362d),LL(0xf7c5f96b,0x995871ed),LL(0xfa20c1e4,0x6be4e8e8),L_(0x02b63df7), LL(0xf9949e2f,0x3453fb74),LL(0xf6fdf958,0x376ae7a8),LL(0x502b7475,0xadf32795),L_(0x20557243), + LL(0xb1daffbe,0x446cdc61),LL(0x8eb329bf,0xefdaaa71),LL(0x369f125e,0x6c894dd3),L_(0x9db7bb62), LL(0x4dd5f542,0xc4277490),LL(0x31b2dbea,0xfe4eccd1),LL(0xb650ae0a,0x20fa0957),L_(0x29da364e), + LL(0x937b1868,0x3d0c5313),LL(0xaa1e00ca,0xb2ed18b4),LL(0xe23097a6,0x18e7fcd5),L_(0x790fdadc), LL(0xf18eb299,0x00c47190),LL(0xf4068839,0x1797157e),LL(0x5b080743,0x257f4df1),L_(0xd4b97dd1), + LL(0xcf771fe7,0x9e9f2679),LL(0x7fa3e72a,0x0a76d555),LL(0xec547443,0x303f3000),L_(0x74ba5f3a), LL(0x56040059,0xe1305459),LL(0xf97b4e0b,0x58cce0ba),LL(0x7b3e3dc5,0xb3342459),L_(0x849b0cbe), + LL(0xc043c4d3,0x156bffa3),LL(0xef4fdf72,0xc568dd1a),LL(0x0fb07e25,0x373456aa),L_(0x0c136024), LL(0x9c743fd6,0x75063485),LL(0x09872cea,0x512889e4),LL(0x1f75eff7,0x2982d676),L_(0x46918abe), + LL(0x19d6d0db,0x62d87801),LL(0x8ac63776,0xe6ac10ed),LL(0x3df57bb4,0xfb67a75d),L_(0x872c8f87), LL(0xb22e582b,0x49548c67),LL(0xa4ab6141,0x6cb6b04c),LL(0x5ae2f1d3,0xccb21fe7),L_(0xcc4c5d51), + LL(0x95de93f8,0x552b1d5a),LL(0xc91b6f0a,0xf6ce5a61),LL(0x26244e6c,0xebe1c6fb),L_(0xec4bbe5c), LL(0x4fe79929,0x35cfef8c),LL(0x09fa8c0f,0xc7ce6957),LL(0x77fdabaa,0x4bbc3792),L_(0x94421437), + LL(0xa29405cd,0x820d6808),LL(0x9b55ccda,0xf32ab874),LL(0x1046f37c,0xed3f6fb8),L_(0x07a02e6b), LL(0x6ed01e06,0x6676830d),LL(0xf82f2f79,0x1cd99760),LL(0x35492433,0x79d53ab5),L_(0x16a996a8), + LL(0x1f2f75b4,0xaa0f7a8f),LL(0xe4d29021,0x44a728bb),LL(0x9b86e6e0,0xdb26a514),L_(0xf567f4e0), LL(0x47f066f6,0xc1decce5),LL(0x9de6495a,0xbb842ffb),LL(0xda23fbb5,0x6557417b),L_(0x7d23ff08), + LL(0x5767ef80,0xc3c2aab8),LL(0x2e3254ff,0xb5203eb2),LL(0x48576b69,0x2e3a1aff),L_(0x4e27496a), LL(0xd5e96792,0x497974dc),LL(0xb0b36f57,0x8169a5cf),LL(0x1d0467dd,0xbd12d521),L_(0xee061337), + LL(0xa2a324b7,0x3a9a26ff),LL(0xfa637395,0x664652b4),LL(0xbc2048f0,0xf8ae9c4f),L_(0x7229d31e), LL(0x499224d1,0xd86d5de2),LL(0xd7e8cbb1,0x33a4a4fb),LL(0xa98c305e,0x382a852f),L_(0x7668edae), + LL(0xdf02b971,0x6d33747e),LL(0x6bae97b0,0x468b6590),LL(0xc0b8420d,0x0d1395e0),L_(0x5d523a37), LL(0xb00603ce,0xf196c7bc),LL(0x940889f2,0x7b8dc7b3),LL(0xa914e0ea,0xce35713d),L_(0xcb59a392), + LL(0x5a648ea8,0x984bd24c),LL(0x17b7b322,0x88be2da8),LL(0x33279a92,0xa03ade42),L_(0x3ac696ec), LL(0xa75e5e17,0x36f23ece),LL(0x4faba2b9,0x7b477179),LL(0xf70fdee7,0x7bbc4522),L_(0xeb035a5f), + LL(0x52c14e4f,0x5ddc6b6c),LL(0x150eef02,0xffc4fbef),LL(0xdd3f93bb,0x3696d783),L_(0x15012933), LL(0xe0cad04f,0xff6e69b5),LL(0xc84755b9,0xd70bfdb4),LL(0x118ab913,0x2da1eb13),L_(0x99325702), + LL(0x0bcc9e8a,0xdfd09cd8),LL(0xd7ddee82,0xf1564d08),LL(0x93460444,0x87b5bd4a),L_(0x0d5541f4), LL(0x87dcbe2a,0x2e83bcba),LL(0xd3e283ed,0xd947e2cf),LL(0xe52b803b,0xb28eb570),L_(0x4a8c77d7), + LL(0x7f6e4fa5,0x76362059),LL(0x18df4a69,0x41a44792),LL(0x70a8c001,0xf96be149),L_(0x69d0bc39), LL(0xef115578,0x59f09aa4),LL(0x53e28bf7,0xc1354a01),LL(0x3ef52081,0x28ac324e),L_(0xa781c57c), + LL(0xf28899e9,0xb145dd4d),LL(0xf8de49f2,0xc929d158),LL(0x7a359488,0xa5fe71f0),L_(0xea4e5a13), LL(0x62263448,0x8252a9a9),LL(0x9870bd6d,0xa478d63b),LL(0xd7e75649,0x73608571),L_(0xb0499f6c), + LL(0xf3fc02d9,0x63626f38),LL(0xee89a981,0xecb2dedb),LL(0x45b932da,0x25321bbf),L_(0xdd6b2382), LL(0x73c34e68,0xbd649606),LL(0xdbca6a1d,0xf7cc4ce3),LL(0x1e699d4b,0x940460cb),L_(0x7938144e), + LL(0x6625374c,0xa571f0bd),LL(0x50d17043,0x67316b9e),LL(0x7b24e59a,0x15047c8b),L_(0xe06a40d9), LL(0x71fd751b,0x5bc5aeb6),LL(0xaeb386d4,0x77f99ebe),LL(0x40fabb40,0x62bdfdf5),L_(0x6aace159), + LL(0x2c7b8857,0xde610cbe),LL(0xf6b35535,0xd1d9c553),LL(0xb7d2826b,0xb4edc29d),L_(0x03e1e5e0), LL(0x05ce2cac,0x789b199e),LL(0x03331148,0x44bc5992),LL(0xa966c64a,0x0851cfcf),L_(0x1622ec1d), + LL(0xb82bff1a,0x888b7c2b),LL(0x07cb3e37,0xade27f4e),LL(0xb017d5a9,0x7f7c933f),L_(0x42119d1d), LL(0x2b1e659a,0x5ef83e45),LL(0x0109f6b8,0x91081a86),LL(0x6e139181,0x1b624dea),L_(0xa2bd127f), + LL(0xa90158d0,0x54ae0fcd),LL(0x9a79307a,0x5d77cf42),LL(0x6d342e22,0xb177e174),L_(0x2e3ee119), LL(0xf32ebada,0x5e84ad5e),LL(0x08e8f89c,0x1f21876e),LL(0x425e69c5,0x42b47533),L_(0x0b2e547d), + LL(0x8c73ff85,0x041c2bca),LL(0xd43f468a,0xdff69743),LL(0x497dff29,0x58417877),L_(0x9551c8f0), LL(0xea9a5efb,0x1e1b3a51),LL(0x7aa44dae,0x62164b97),LL(0x82e3854e,0xe5d6128d),L_(0x2f25adda), + LL(0xce69fd62,0x92365b99),LL(0xdd4e8322,0x64c55482),LL(0xe083981e,0xbf7947a0),L_(0x5b678747), LL(0xe12f61e8,0xc98f9109),LL(0x0ca71f21,0x7084eee8),LL(0xe31a901e,0x922f7c66),L_(0xeed6566d), + LL(0x830c51a8,0x274ea5de),LL(0x52fe1ee1,0x6f8c93f7),LL(0x7dc25d86,0xd6285877),L_(0xda84cd8b), LL(0x48aed2f0,0xbb649ac1),LL(0x1e3ba4b5,0x6976562f),LL(0x6f9c137d,0xbd251ef1),L_(0x5e5495d3), + LL(0x239cae69,0x8392e4dc),LL(0xdc8ee05c,0x79a24fdc),LL(0x4eeeb0db,0x0d91ecaf),L_(0x31e89bf7), LL(0x4b9c499d,0xeca9047c),LL(0xee9a09ba,0x2d00d0dc),LL(0x81d64f7f,0xea9e0b50),L_(0xc7b60917), + LL(0xb40fe472,0xb48d4d3e),LL(0xb23798db,0x4985b85c),LL(0x4a87382d,0x829e3bb6),L_(0x492aac4a), LL(0x59bdf861,0xf58a6463),LL(0x96935f47,0x28f41a65),LL(0x3396e951,0x6d21b8d5),L_(0xe244afe7), + LL(0xe019d807,0x890b3389),LL(0x01b0423f,0xaf7bf53a),LL(0x6d8cdb00,0x68218ce9),L_(0x7809fc35), LL(0xd3b4c83e,0x446c771a),LL(0xaae3f3c1,0x8e7a1eda),LL(0xfb834953,0x89128662),L_(0xff1c7590), + LL(0xb5425304,0xe1d9d533),LL(0x52f4f30f,0xdcd37dd8),LL(0x3a3e06d4,0x6d0d3574),L_(0x9ec8c4a0), LL(0x48b20906,0x06630324),LL(0x8d559ed9,0x334cec45),LL(0x295b5edf,0xd0a2d07d),L_(0x8d6e5cd0), + LL(0x2b1a895f,0x42c70b2b),LL(0x4f4ae671,0x36254863),LL(0x536e7d65,0xfaa3003d),L_(0x94dc0a21), LL(0xc4355888,0x1592950e),LL(0xb4369ba0,0x6d4496df),LL(0x5f932085,0x6309e764),L_(0xe5462315), + LL(0xe10bf24d,0x08fa6e2e),LL(0xec60e2f4,0x4a8f6f4a),LL(0xa152beaf,0xcc5f8968),L_(0xbce9a53b), LL(0xedca75db,0x076de5d8),LL(0x31df6025,0xfb2b47e7),LL(0xce591c62,0xead89b13),L_(0x52e8a741), + LL(0x94fdce56,0x179fcba0),LL(0xc81fc0c7,0xa0dcd6da),LL(0x8eb90798,0xbdb2ded4),L_(0x820ef61d), LL(0x9c335535,0x2b8a3cbb),LL(0xf522badb,0x7214d9a1),LL(0x93d7057a,0x0ff17306),L_(0x70098d33), + LL(0x50cac57f,0x8e019361),LL(0x2f914dd7,0xa3854afa),LL(0x2acdf295,0x2848b0db),L_(0x0f138ea9), LL(0x8c64c319,0x4ea1b039),LL(0x49ae634d,0x47440022),LL(0x113138a5,0x8ed0557e),L_(0x2b0adbad), + LL(0x931ca67d,0xb33942ae),LL(0x1ecf958e,0xc163a1f5),LL(0xc66d2148,0x1f27ea10),L_(0x011365d7), LL(0x267a659a,0xc4208dd5),LL(0xb8f4d08e,0x61b362a6),LL(0x3e553027,0x29499eb7),L_(0xea028ba1), + LL(0xd2551fb7,0xa6f2df89),LL(0x5bf03737,0x40fa3c8b),LL(0x924c5819,0xce383c9e),L_(0x910aeacb), LL(0x641dc85b,0xbfdc9b58),LL(0xe45c96a9,0xd63db8d5),LL(0x726cada6,0x8177cf88),L_(0xa738df5c), + LL(0x0cdee643,0x21f30c42),LL(0x8cd0ef97,0xf0f5ccbd),LL(0xe6a63e2e,0xd4df1941),L_(0x57a01e9c), LL(0x498e2890,0x9c025c36),LL(0xdc96eafb,0xc9df9cce),LL(0x95fe2aa2,0xae8f18c9),L_(0x239e5894), + LL(0x52954a16,0x9c68e7cc),LL(0x22874bf5,0xac850b44),LL(0x64db3074,0x3b1f7071),L_(0xa14665fa), LL(0x87ef6099,0x78fc6e68),LL(0x77eb0618,0x1edffe76),LL(0x6d531161,0xf5f0fda1),L_(0x429c094e), +}, +/* digit=6 base_pwr=2^42 */ +{ + LL(0x0a953500,0x570bd76b),LL(0x42617e4e,0x493624f9),LL(0xec1fce2d,0xbd807766),L_(0x5098d9d8), LL(0xcbc6ada2,0x468a4f16),LL(0xada13650,0xb78fc3ac),LL(0xb5aa694d,0x1dce66de),L_(0x5318219a), + LL(0xfedbc51c,0xd30a5800),LL(0x97115f9f,0x2745e2b8),LL(0xbb10a57f,0xeff5c174),L_(0xadf87884), LL(0x589f5f94,0x162e3c5b),LL(0xa2bc843d,0x16aad5e5),LL(0x25478287,0xd6f94dfb),L_(0x31bb4258), + LL(0xc180bac0,0x6c5c8427),LL(0x46351cc8,0xdf01fd21),LL(0xb964daab,0xdd1d4004),L_(0xc4d0e945), LL(0x72fb966c,0xb82dabad),LL(0x7d7ac429,0x2268903b),LL(0xeed99c4c,0x7dafa986),L_(0xcbdad15b), + LL(0x60c5d65f,0xe5e0aad8),LL(0xc7b54b13,0xc4b812b6),LL(0xe9d8af38,0x008e0bc3),L_(0x71a411c5), LL(0x27cb0dc7,0xc482b232),LL(0x07289ac0,0x5042230e),LL(0x68d763c4,0xa128b566),L_(0xc589940b), + LL(0x62562a0c,0x533b21ed),LL(0xb86a434f,0x398ba505),LL(0x5f1dadf5,0x5b2f618e),L_(0x35caca2b), LL(0xcaf4b249,0x74cf7bbc),LL(0xb14dae65,0xd924780d),LL(0x27c04cc6,0xf5af0266),L_(0xc41f13b5), + LL(0xfd136860,0x5727ef4b),LL(0x9fcc3538,0x2af297df),LL(0x957682ab,0xe33ed1c3),L_(0x9f83349f), LL(0x64693518,0x5961423c),LL(0x5148178e,0x8650191e),LL(0x66e2c6e6,0x5986c53b),L_(0xd5c63ac2), + LL(0x11b49c5a,0x9082e761),LL(0x5661867b,0x1815dbbc),LL(0x8ece3938,0x30442089),L_(0xeddde973), LL(0xbf2eb7b4,0x3e2f678f),LL(0xd4f64e91,0x15fb42e7),LL(0x49fdfa46,0x0b104178),L_(0x78b6687a), + LL(0x391a31fe,0x9f724691),LL(0x0ca4bf22,0x79581fa0),LL(0x730e0882,0x4bbc9bf5),L_(0x91d1e24a), LL(0xd756f110,0x26de9371),LL(0xd72ae23c,0xf050810b),LL(0x6ca80e46,0x75b289b3),L_(0x777ca3a2), + LL(0x8185a290,0x7d3fcac6),LL(0xebb7ace4,0x1f109f32),LL(0x48bb32eb,0x0be22833),L_(0x3f76d58e), LL(0x1868a043,0x8e3df32b),LL(0x00f1bf36,0xd5e2b1fa),LL(0x8b204654,0xdaeaa58b),L_(0x3263322d), + LL(0x742ad365,0x648b273d),LL(0x2e05e12c,0x9a066a23),LL(0x3a732a3d,0x691aae3b),L_(0x6cd91464), LL(0x04fe3ff0,0x208c6c3b),LL(0xe4c05b8f,0x61fcbdb4),LL(0xd9e2218e,0x59cd9a06),L_(0x835930df), + LL(0x33eb0d51,0xd984c49e),LL(0x888fc640,0xd7d25d0d),LL(0x9bb00fea,0x4da4ce0e),L_(0xaffa26c5), LL(0x72396942,0x2307b843),LL(0xc6cd9202,0xc1d18e97),LL(0x3f8ed768,0x518df91e),L_(0xeadd2347), + LL(0xc1d3ad41,0xaf9c00eb),LL(0xb90b0248,0x6b575b77),LL(0x45a5507d,0x2e1dac5f),L_(0x8db688a8), LL(0xab53ef89,0xa655bbf0),LL(0x0fdb8f93,0xea6e6754),LL(0xfdb2b439,0x9751d174),L_(0xaef8bdee), + LL(0x0aa602c4,0xb601c379),LL(0xdaadd78b,0x3a6533ad),LL(0xb65d4dfb,0xb3e4d63b),L_(0x653e4965), LL(0xf35d8a67,0x8cbfa3ec),LL(0xbaaa17fd,0xd9cd6c4a),LL(0xe29a388e,0x50eade40),L_(0x2ec73453), + LL(0xd1812921,0xbac1435a),LL(0xaa859487,0xf5d2dc01),LL(0xb5b2445b,0xe22a73a2),L_(0xaa1116ed), LL(0x39d4f53c,0xf9d72a85),LL(0x616cb03b,0x27cfb1ab),LL(0xcdf54ee8,0x4cf9a426),L_(0xe0208859), + LL(0x0749a748,0x5300262c),LL(0x7faf0467,0xbb0d7807),LL(0xff47f117,0x93b14cbc),L_(0x484a66c2), LL(0x91118c52,0x654db6c9),LL(0x20220d13,0x6b9ef224),LL(0x1162cfdf,0x897e3ec2),L_(0xf29d6809), + LL(0xf7481883,0x2c67eef0),LL(0x1a833a88,0x49af9414),LL(0x8aa0f2d7,0x5fc1c69f),L_(0x9f06aae9), LL(0x2b6f561b,0x4ecfeaa5),LL(0x17fda35e,0xbf5973f7),LL(0x338ff3e0,0x0013029b),L_(0xe6215b79), + LL(0x45cc9ed7,0x45530707),LL(0x3d6ab800,0xa1f8d140),LL(0x34dca331,0x5aa02638),L_(0x4dc086c8), LL(0x5be308ce,0x6df633bb),LL(0x90bbb8d8,0xd7e74378),LL(0xc7c28c06,0x8dcb3bae),L_(0xa0c24314), + LL(0x2b4b775f,0x7f227ccf),LL(0x8cd51cfe,0x8cb18663),LL(0x68ea58e3,0x9f7bcc04),L_(0x3003cc72), LL(0xdb7fa1bc,0x5b982b6a),LL(0xd5ebd68b,0x64a1222b),LL(0x675b50df,0x9679336c),L_(0xa8fee552), + LL(0x1db92746,0xe51e0a9e),LL(0x75eb0c92,0x584f0a75),LL(0x9af943f8,0xa54c748c),L_(0x28a759fb), LL(0x59c61e20,0x1cea2c6d),LL(0xb0384781,0xd54f4a86),LL(0xf0f73ecd,0xb65d1ecc),L_(0xd26a77df), + LL(0x6f8d4351,0xcb23a541),LL(0x51dd59db,0x3cb31aea),LL(0x24f458a5,0x7d0b3dda),L_(0x26b341e1), LL(0xb5440aa5,0x80d2e582),LL(0x9e2fd334,0x1b5ad25e),LL(0x114215cd,0x794f6e44),L_(0xe3bb54b5), + LL(0x119ef86c,0x01429cdc),LL(0x7fb2c053,0x994ee3e5),LL(0xee0b3bcd,0xcf236537),L_(0xca6a0781), LL(0x9d76f90c,0x28ee066e),LL(0xe18e5f40,0x18225dd3),LL(0xb1e0ccfd,0x2a9ed4ef),L_(0xa109a3af), + LL(0x9f94c8a8,0xad211acd),LL(0xe621f731,0xfa0ba148),LL(0x072030c8,0xa71de5a1),L_(0x106812c7), LL(0x57d3fed5,0xeac9452f),LL(0x4ce2ef8a,0xff6172a2),LL(0xac163954,0x778bc2dd),L_(0x2e5cbdab), + LL(0x6d06223c,0x0baf79a2),LL(0x9c81a4be,0xe2bee5ec),LL(0x9363638d,0x8a4d69e0),L_(0x687f0dee), LL(0xdceee166,0xa6d3323c),LL(0x202ed4a9,0x354badbe),LL(0x1c483cb2,0x5bbe8c1a),L_(0x433b79ed), + LL(0x2c4c3668,0x3828dbf7),LL(0xb8a0c6af,0x17810cb9),LL(0x9f12e96e,0xdd7d6cea),L_(0xe595dcab), LL(0x5b28c732,0xad54b44b),LL(0xe8aa36bb,0xb904d92f),LL(0x11dcc1c4,0xcbe63415),L_(0xeff9d401), + LL(0x29014f0c,0xfef9f39e),LL(0x1d099df3,0x4d9cac37),LL(0xbf9c0427,0xe387610f),L_(0x0cce0331), LL(0x95f296eb,0x31cb09c4),LL(0xd62d1375,0x7a5fdf3c),LL(0x53290cd4,0xa90e25d2),L_(0xf1396054), + LL(0x83b59271,0xf603ca7e),LL(0xe57e3a51,0xa2fe4449),LL(0x547b9c97,0x5234c386),L_(0x44edaf6d), LL(0x6f332534,0xcea690d6),LL(0xf13946cc,0xdad5199c),LL(0x6ad6bb4e,0x9374003a),L_(0x36c2bacd), + LL(0x9a4d019e,0x341c30ad),LL(0x7e1c2c9b,0x96ee26ce),LL(0x17e56dfd,0xb65db139),L_(0x9888703d), LL(0x4d9f2cbf,0xe07d7042),LL(0x3e36f84d,0x2223785f),LL(0xb4b9c74d,0xbbd020f9),L_(0xed1bc384), + LL(0x08bd06c5,0x4432f2cf),LL(0xa3a488e9,0xec0de874),LL(0x9efd00d7,0xd17c4829),L_(0xf5128936), LL(0x83542187,0x35e93b48),LL(0xae41e05a,0x22977d11),LL(0xe030a465,0x41a1010a),L_(0x1db1bce3), + LL(0xe96cbfbc,0x6fb1c1be),LL(0xfe5cfb57,0x46e0f43b),LL(0x401be370,0x29b356b7),L_(0xef6ca52e), LL(0x1769847a,0x131c52a8),LL(0x3084c2e8,0x9a3bea64),LL(0x58a36c5b,0x8cfd7a22),L_(0xa076e835), + LL(0xc50d8952,0x74382b8b),LL(0xecf54e6c,0x22c47953),LL(0x80a6564f,0x3378864a),L_(0xf29ac0df), LL(0x87ce0bb5,0x9549b954),LL(0x71036c3c,0xfdefd453),LL(0xd3203c72,0x57b0b3ec),L_(0x7ee244a8), + LL(0xc911b9a4,0x2a2ae110),LL(0xe00edfa7,0xc23eb58f),LL(0x6b68f39e,0x8f5215f7),L_(0x0b132723), LL(0x750f7a6d,0xe7d50418),LL(0x85673786,0xd615c040),LL(0x231c0709,0x487f22bb),L_(0xf5e2a237), + LL(0x2e68e2c2,0x95692ad2),LL(0x0b27bb26,0x72e6eaf5),LL(0x090be6f6,0x8b45b55f),L_(0xea84dd7c), LL(0x8e110b28,0x928497fa),LL(0x971d5f28,0x100a822e),LL(0x6339a104,0x9ddbd5c4),L_(0x3fe08c7f), + LL(0xdbfc1ac8,0x987de692),LL(0x1683a455,0xee7012c8),LL(0xb38e1de8,0xdd0a3684),L_(0xf01af5d9), LL(0x1923ae0d,0x86ab28a0),LL(0xd4b30cec,0x3a16a7ed),LL(0x22c7c599,0x01cadc9e),L_(0xb5dd36ac), + LL(0xa274d3dd,0x8baf2808),LL(0x60cc335f,0x7c5d7b61),LL(0xa633db28,0x732a899d),L_(0x7f9cd569), LL(0xe146b7a9,0x04bce2d5),LL(0x1f59a0d7,0x8db165c3),LL(0x6d10e528,0xcbb166ea),L_(0x7fe10619), + LL(0xa561b19d,0x04d81b3f),LL(0x88a7293b,0x9822961c),LL(0x1732c2f0,0x057b8676),L_(0x320d8861), LL(0x8a935dc3,0x5bfbc590),LL(0x3e5d907e,0x1bdbcf0b),LL(0x2b35a3ff,0x1597d44b),L_(0x7605acfa), + LL(0xe27abdf0,0x6112f38e),LL(0xf5ab47b4,0x12fd1d52),LL(0x5984ae07,0x4fccb7e8),L_(0x81972b3a), LL(0x95532439,0x5d4f964b),LL(0x905ded37,0x60b2cf53),LL(0x6a31fa69,0xfb6ad50f),L_(0xd83dca88), + LL(0x8f19b88b,0x1b0ac422),LL(0x711dde1a,0x9363a89c),LL(0xb5e309b2,0xac0aec46),L_(0x17ee236d), LL(0x5719b962,0x600531c2),LL(0xce093e55,0xee69dc7c),LL(0xec04dd4e,0x59639b52),L_(0xb9e01d45), + LL(0xca086bb7,0x1b94ace6),LL(0xb0fee2ec,0x3ed113f9),LL(0x5c0b8f2f,0xdfd59ee2),L_(0x6a154d90), LL(0x462e146c,0x8fb75d59),LL(0xb692d5e7,0xc31c3f9e),LL(0x080f7773,0x1fbd50fb),L_(0xa4f8f9e4), + LL(0x7b7f1009,0xf63f6da7),LL(0x2aa45a5e,0xf59e459b),LL(0x2d7d0b41,0x29c1bc69),L_(0x8aabebdc), LL(0xaf1dea2d,0xb0063dbe),LL(0xc38e3c41,0x335900c9),LL(0x661ba0dd,0x99541913),L_(0xc19809da), + LL(0x7540a87d,0xff252d35),LL(0xa1530a6c,0xaffcab72),LL(0xc8e9b570,0x9793a371),L_(0x3489107c), LL(0xdcf94c42,0xf0b49c6c),LL(0x879022b6,0x35c46e5c),LL(0x069dd468,0x2c618b82),L_(0x64bde16d), + LL(0x4335b182,0x2d08df94),LL(0xed7d7c9a,0x919ddb27),LL(0xe80a7f85,0x357baf28),L_(0xbb968f21), LL(0xbcb38a46,0x8315da92),LL(0x96e91923,0x925476f6),LL(0x4e306972,0xc5ed2a12),L_(0x950bbe41), + LL(0xcd3c90b3,0x39668cb5),LL(0x077167b6,0xaea10320),LL(0xfce255b9,0xb37bc6b8),L_(0xf3809a3f), LL(0x860aeccb,0x7f61816d),LL(0x44d57109,0x80e57881),LL(0xb19cf5fd,0xdb202875),L_(0x42100d63), + LL(0x5a155cef,0xaf246d29),LL(0x7f3965a6,0x329527e8),LL(0x8d8e3138,0x26bde524),L_(0x11b1a769), LL(0x66b0792b,0xd87dfaed),LL(0x707808be,0x02a38ada),LL(0x04bb89d1,0xe52cbe87),L_(0x7ec2d88f), + LL(0xded9227f,0x5423908e),LL(0x43d7285e,0xa6f72d9e),LL(0x49363917,0x6a49fc82),L_(0xf5d8d60b), LL(0x067bfaaf,0xa0f34345),LL(0x665a2267,0x19294466),LL(0x1ac20d98,0xffa2e5fc),L_(0x4e6d33c1), + LL(0x558d4d1f,0x9731dc1f),LL(0xf1f7a37e,0x6a85be9c),LL(0x69d32744,0xea51b61d),L_(0x05851633), LL(0xa0b1e63d,0x04e8d8ba),LL(0x1c8bd021,0xa75506f1),LL(0xef76ca45,0xf099b725),L_(0xd230047c), + LL(0xf47e0086,0x44613936),LL(0x62171491,0x0105d7a8),LL(0xa07a7b6e,0x9015ad40),L_(0x306c9555), LL(0x166467b1,0x18925a9d),LL(0xaac18583,0x8a300595),LL(0x7d220577,0xbbd4708e),L_(0xf151aaec), + LL(0x6807a39d,0x6e04c9ec),LL(0xb57b3b22,0xd4d3310c),LL(0xa3593866,0xa01b71df),L_(0xa63bab57), LL(0x378ccde9,0x773f947b),LL(0xefc9bc76,0xe29288bb),LL(0xcb328f94,0xe12b780c),L_(0xb068b3ce), + LL(0x3bf9b78d,0xc1916a89),LL(0x62555b0b,0xad0d163d),LL(0x00203bb0,0x82d7d36f),L_(0xd8764be5), LL(0x91a430b0,0xf446925f),LL(0xa23e8076,0xedc1e316),LL(0xb903dea6,0x11610026),L_(0xd83fceb2), + LL(0x8f55d6ab,0x2739ed0a),LL(0xcff10abf,0x3b4b462d),LL(0xe0b269ec,0x92761d90),L_(0xc1c0ca0e), LL(0x2403dfd8,0xab5ce41d),LL(0x62aaaef4,0x9ebcf2d7),LL(0x5a3ff0da,0x3aa250ea),L_(0x8f10b784), + LL(0x2fbc9b42,0xe5bbf62c),LL(0xa53f3f3f,0x2c19f9e2),LL(0x27dd1952,0xeada36e1),L_(0x40a88ecc), LL(0x8eac382d,0xca0a31a3),LL(0xd318529b,0xcc8ff4c6),LL(0xefec04d1,0x49d3e4dd),L_(0x55c20eb0), + LL(0x54347369,0xc5f830ce),LL(0xbf310209,0x66ba5a3d),LL(0x1372fe7d,0xab6a0dc0),L_(0xaf6004cc), LL(0x65a2727f,0x8803af60),LL(0xa6088f51,0x91e29b12),LL(0x53f144ad,0xe77f4280),L_(0xd7d06ab8), + LL(0x978295dd,0xc4969a0d),LL(0xdf19cc2f,0xabbb112c),LL(0x8bd7557f,0x554a3a06),L_(0x8fe8b8dc), LL(0x516a1ab6,0xf261d772),LL(0x82e07bc5,0x4d16bca0),LL(0xd1d7da5e,0x45447ad3),L_(0x19044418), + LL(0xe167f824,0x3956922e),LL(0xf66675c8,0xfcc5cc95),LL(0x20469977,0xf5f39465),L_(0x011863a2), LL(0xaf5616e5,0x960a1daa),LL(0x66b9d7bb,0x2c7f8f70),LL(0x074a85f0,0xd9f9bf2e),L_(0x7dd3b230), + LL(0x4e5dc71e,0xe5be9e2a),LL(0xb5995728,0xb7e24b2c),LL(0xa127aac9,0x23c8cef0),L_(0xf11df0db), LL(0xe494dece,0xefcf6a97),LL(0x38cfdd79,0x936e7033),LL(0x899c799e,0x8c265685),L_(0x7cab4cd9), + LL(0xdfd2bdce,0xe6160916),LL(0x480a36fd,0x01a7795c),LL(0x3f7c1b1d,0x91f84a8c),L_(0x85381912), LL(0x14463148,0xb7408da8),LL(0xe263f58c,0xbd7b174a),LL(0x8ca85525,0x8c2f9aa1),L_(0x687fc42f), + LL(0xa86b0be3,0xf5f78e41),LL(0x3455a3e9,0x135a0c71),LL(0x30c1c2c7,0xddf065a7),L_(0x3692f51b), LL(0xebc66e40,0x0f5906c0),LL(0xbfc2af37,0x5229db63),LL(0x5fa05ed1,0xd9f88327),L_(0xfad22122), + LL(0x6a2bd6b4,0xcbee3324),LL(0xe6fcf3f6,0xbfd3822c),LL(0xc3e92fcb,0x7c43be45),L_(0x6def0ea9), LL(0xe2791c4e,0x72baebe0),LL(0xabf2c782,0xad049ecd),LL(0xff1c5bf6,0x21f7b8ae),L_(0xcada1649), + LL(0xa204b629,0x2e618a14),LL(0xe44e287b,0x1ee56452),LL(0x9ec08fe0,0x3cbeeaa0),L_(0x41378416), LL(0x65a854a6,0x3defdbe3),LL(0xdd8dc723,0xe7d6715a),LL(0x6d23302b,0xf1849878),L_(0x922ac03f), + LL(0x826900c1,0xf4361242),LL(0x5150fa93,0xbc384de9),LL(0x1bf1d7e3,0x0106a9fc),L_(0x90827c46), LL(0xb45c094f,0xd4951e89),LL(0x046acfdb,0xf1ea11f5),LL(0x38e792fd,0x2d8ca425),L_(0x07e228cc), + LL(0x2b9cf3d8,0x889c6c44),LL(0x12d9fd78,0xc8a37703),LL(0xa6aed948,0x46f4af45),L_(0x4de1be3f), LL(0x229dbecf,0x932b6857),LL(0x4b284bc1,0xa90d9c3e),LL(0xe27e82bc,0x9200852f),L_(0xe41775aa), + LL(0x762bd870,0x50a8e4b7),LL(0x9d917e78,0xdf55385a),LL(0x4ac6dc2a,0xeb63b95b),L_(0x585b0c05), LL(0xf896eb5a,0x876d7f21),LL(0xdd6e7325,0xbc4c0b28),LL(0x34a78682,0x55d49e0e),L_(0x7de26ccf), + LL(0x0018e94a,0xf3430043),LL(0x09139313,0x6fccb693),LL(0x32e29483,0x9f0a6227),L_(0x1c584d56), LL(0x4da9fbc2,0x11cca785),LL(0x53203116,0x349e04d6),LL(0xb1c8440b,0x0ec04b4a),L_(0xb4fd311a), + LL(0x4e394c99,0x4a367cf0),LL(0x82ea7247,0x419778aa),LL(0x35a8f8e5,0xee0c8d61),L_(0xeb47eaf0), LL(0xb8c3b039,0x7fc38516),LL(0xb4361b1e,0xedd99fe4),LL(0x24e2fb10,0x94b8860e),L_(0x1e4ac934), + LL(0xb83e5822,0x9c7f7bc2),LL(0x85016181,0xec2c9c48),LL(0xae2e08c9,0x1d49bd53),L_(0xd51d0f76), LL(0xe02702a5,0x574b1fa2),LL(0x275045e3,0x4b568161),LL(0xe63f9064,0xb270bab6),L_(0xcbacaf10), +}, +/* digit=7 base_pwr=2^49 */ +{ + LL(0x28c9f54b,0x55cc3b93),LL(0x81366649,0x9f91db20),LL(0x0bb5529e,0x66b83a45),L_(0x020a3d7b), LL(0x84c6fb9a,0x6d9c718d),LL(0x4c39583c,0x6e5e8c08),LL(0x063ccb67,0x8cb7eb45),L_(0x01328b11), + LL(0xc3d2cc22,0x7e06eec9),LL(0x2a100811,0x5a959809),LL(0xf4997f3a,0xf5c07857),L_(0xad98c158), LL(0x5ae7c1cf,0xf408207d),LL(0xdb7d94f4,0x8cd5c2c0),LL(0x6379ef0a,0xf3bfa4da),L_(0xde775bfd), + LL(0xd39752a4,0xd08dfe44),LL(0x9b19df92,0x208bc436),LL(0xfd14af8d,0xcc3f5ace),L_(0x9cb59da4), LL(0x97f49ac2,0x880b584a),LL(0xe1fa2671,0x72d9f8ff),LL(0x76bf0449,0x0c870e3e),L_(0x6fb6d427), + LL(0xbad01fec,0x70db0532),LL(0x12b3f8dd,0xf82bbcfa),LL(0x167ab18c,0x92abaaa2),L_(0x5e4d357c), LL(0x4304ac9f,0x1fb69d3d),LL(0x7bd8f0e1,0x15e5d010),LL(0x47351320,0x84b31510),L_(0x6a03c6b3), + LL(0x739b3599,0x5a2c8035),LL(0x91588e92,0xdac9ebee),LL(0x1b69af1a,0x451b95e7),L_(0x3517cee5), LL(0xb64f8f05,0x1e09f842),LL(0x90930327,0x099f2550),LL(0xefa175e0,0x9e48451e),L_(0xedef520b), + LL(0x43b4b068,0xa2492064),LL(0x1e8946dc,0xeddb27dd),LL(0x9916090e,0x82135311),L_(0x98fdccc1), LL(0x4585ddd8,0x31f673e3),LL(0xd529f3ea,0xac4307ee),LL(0xb119ca51,0x428145ce),L_(0xcf53a57b), + LL(0x3728e89e,0x38640e71),LL(0xa872dbd5,0x745bd8a2),LL(0x8f90052e,0xffeb6e87),L_(0x9a484b77), LL(0xb85a7c0b,0xe045b6ce),LL(0xb3a39b3e,0xafd92044),LL(0xdbc53298,0x462a1b9b),L_(0x60a23eea), + LL(0x050b6e45,0x8a1c84ea),LL(0xc38a15ff,0xc2315710),LL(0x5b502e99,0x67d1f3e4),L_(0xf85c7d6e), LL(0xd3018572,0x7c34a744),LL(0x94062e0f,0x8b89ec47),LL(0x9579c983,0x35f7a266),L_(0x63ed902e), + LL(0xbfd3d5c1,0x7e6607ad),LL(0x54fa21c1,0xbc8c84d5),LL(0x2c3baf13,0x177b275b),L_(0x39001a70), LL(0x71f9447d,0xf4928a3c),LL(0x9356e91a,0xd23bd45a),LL(0x30b1d7ca,0x02772478),L_(0x8c5ecfb7), + LL(0x5717dffc,0xcaa887a3),LL(0x4f7bef32,0xb1687f81),LL(0x5dec8cf9,0x8965403a),L_(0xcc362cae), LL(0xb0dbc045,0x37f6ba20),LL(0x35e7cde5,0x6c266c6c),LL(0x0d54e5e6,0x02d7913b),L_(0xf4a0addb), + LL(0xdde83971,0xcc348a37),LL(0x58983641,0x18fb2236),LL(0x237be719,0xd585a061),L_(0xdb2539f1), LL(0x35a8a0dd,0x403565cc),LL(0x78351808,0x011a3765),LL(0x6202e956,0x7f54736b),L_(0x660d4fb3), + LL(0x90a83185,0x6bc9ce97),LL(0xb22311a5,0xf3cc61c2),LL(0xe993a41b,0x85025abd),L_(0x432e7fa6), LL(0x3a136918,0x42d19cdc),LL(0x18144346,0xe5b41dc3),LL(0x2bd73670,0xede484f4),L_(0xa93261b3), + LL(0x62a3c5f3,0x38a7a728),LL(0x2660c15e,0x05a1c27b),LL(0xcbfe2670,0xe7715ea4),L_(0x803ef610), LL(0xfff730b3,0x6f2eb06b),LL(0x3e73d7f3,0xafffff2e),LL(0x8fe4228f,0x2703c35b),L_(0x801d9246), + LL(0xcf3f9f32,0xa7b3253d),LL(0xb347bed9,0xb88a3be1),LL(0x23367d55,0x50ebab37),L_(0xe6bc8d5f), LL(0xb7d8a5f9,0x9df003d1),LL(0xd78ada2c,0x0bb08555),LL(0x4989b92e,0xf9b3e9f5),L_(0x2dbbe952), + LL(0x1b0c7f5b,0xdc3c1c11),LL(0xfb949b08,0x8d1cf002),LL(0x010b8835,0x69483176),L_(0x73825d94), LL(0x95d368f2,0x7caf4449),LL(0x3cd86c08,0xd50eaa5e),LL(0x8ba59225,0x6d5a74bd),L_(0x1e6e636f), + LL(0xda9ad1f1,0x51eea6ca),LL(0x69de6020,0x3afc7be4),LL(0xf383a0c6,0xa7aeef7a),L_(0xec5ab9f7), LL(0xe3d53f58,0xb56f14b0),LL(0x7b78f2d9,0xda9d3372),LL(0xb4e91769,0x0886a015),L_(0xb8c7fe1f), + LL(0xbb4c1b2b,0x7b2d2c19),LL(0x3f05a7cf,0xc24e67f7),LL(0x8a978fa5,0x2541de9c),L_(0xb09f953f), LL(0xbd943ed2,0xa6872621),LL(0x0275a86c,0x9f103457),LL(0xc3073baf,0xe68e138a),L_(0x70608786), + LL(0x94c887ac,0x22def543),LL(0xc1d1a75a,0x70845389),LL(0x9d05dd03,0x13908388),L_(0xcac954e6), LL(0xcee91dc6,0xfb37d06c),LL(0x42873dd7,0x0976cad9),LL(0x77686ddc,0xbd6e2bfe),L_(0x0a7aae21), + LL(0xf93e73a3,0xd00f349f),LL(0x6d3f30d8,0xbdf0b725),LL(0x793a6f93,0x8e54b8c8),L_(0xa3ebe240), LL(0xef42120a,0xd3635d7e),LL(0x7527fefe,0x954dc6a7),LL(0x7ecf5710,0xe5c193ac),L_(0xc5b359a4), + LL(0xcee954fa,0x6a69e07e),LL(0x904b6a99,0x28557756),LL(0x74691c8a,0x377aafb2),L_(0xf8dfe49e), LL(0xaf1c65b9,0xbc4bbf13),LL(0x32c37416,0xa07e7595),LL(0xa6bc7861,0xb481fbff),L_(0xab667972), + LL(0x4ade6d39,0xd40e7121),LL(0x03318c55,0xe035cb6d),LL(0x5cc23063,0x7d21722c),L_(0x353dafc9), LL(0x57f87e8e,0x47d32da3),LL(0x8e5c3c5f,0xdd769bba),LL(0x4c620a4a,0xc2e2dc49),L_(0x6ac8f16a), + LL(0xe2c62094,0xe0aae2f2),LL(0x27e1fc19,0xe8068cdc),LL(0x9a5089ff,0xd066d3ec),L_(0x83ce796d), LL(0x15285915,0x3d7bd85d),LL(0x2109b271,0xbfaee64f),LL(0x032bfaab,0x4c64918b),L_(0x28f7e9fb), + LL(0xe9c1a122,0x81a3a910),LL(0x1ec15d92,0x1d6baa6b),LL(0x0de6b8aa,0x25d33d78),L_(0xdb94f1aa), LL(0x84537b54,0xcda754fc),LL(0x0aa41bac,0x07667978),LL(0x26a33ede,0xaed5fcc3),L_(0x03b383eb), + LL(0xc6a270c7,0x48a0860f),LL(0xe3c9d6e7,0xe8f8a0be),LL(0x1c9dda6d,0x37a0510c),L_(0x87290f47), LL(0xb746897c,0xe0b9b992),LL(0x04968462,0x691f74eb),LL(0x79bcdef3,0xba7c23dd),L_(0x3c9dfbbc), + LL(0xb9c1ad2a,0x4867362c),LL(0x9b7dcff4,0xea98891d),LL(0xf6be4fd8,0x9dd99069),L_(0xfc837cc5), LL(0xc4b7695b,0xa104db38),LL(0xe73440a8,0xc1d73652),LL(0x557bc30e,0xca38a194),L_(0x0b9107de), + LL(0x08821106,0xf9b3c662),LL(0x0572aa2e,0xa9b0c578),LL(0x6cf8f9ea,0xdc3bb3a4),L_(0x578fa229), LL(0x935620c9,0x7b780978),LL(0xe2e55595,0xaa7bd737),LL(0x1ae5674a,0x7d70eb0e),L_(0x6287e785), + LL(0x8161a173,0x60277149),LL(0x6418b451,0xe9fdf471),LL(0x008c73fe,0xc7a41b44),L_(0xb52d2690), LL(0x6217c51d,0x07d5913e),LL(0x28af32f1,0x584cce8f),LL(0x70ace437,0x249d5a1f),L_(0xe2634904), + LL(0x2aa8ec2f,0x91c3f75e),LL(0x7ac06d6f,0x3279d284),LL(0x8cbbcc96,0x6114dddf),L_(0xb3f952c3), LL(0x18133afe,0x50a376bd),LL(0x95633a5e,0x281108f7),LL(0xf0277529,0x900af3d2),L_(0x2aac428e), + LL(0x34f686e3,0xede089c0),LL(0xafe013ca,0x66ca4aa6),LL(0x24f9b482,0x7d9ab065),L_(0x7064130f), LL(0x59e2f96d,0x88599e65),LL(0x1a8684d1,0x81cccdd4),LL(0x5b828088,0xa7784039),L_(0x274e2738), + LL(0xa482a439,0x04a64674),LL(0xabf54d81,0x3a3f5066),LL(0x46144f06,0xc232e58f),L_(0x30cddd92), LL(0x4aba099f,0xc2b48797),LL(0xfb8c420d,0x6eb5b117),LL(0xb4d97ee4,0x8d97d0d5),L_(0x8101b1e7), + LL(0x3dd4c04f,0x98af21ef),LL(0xcd4340b9,0x57885c4e),LL(0xda56271d,0xcf1eb8ab),L_(0xcfc8c025), LL(0x42508147,0x99747ee9),LL(0xdff889cf,0xbf581471),LL(0x16fb6025,0x22ae0ed8),L_(0x56d4b8ed), + LL(0x3d08ef7e,0x0745f517),LL(0xc7781923,0x732c8246),LL(0x6cd5a87c,0xe1112158),L_(0x734d96e5), LL(0x0ea91035,0xfbc17aeb),LL(0x7d6ae9b9,0x9fdb5814),LL(0x6b340fbc,0x7a81eba8),L_(0xa462be47), + LL(0xd9df5a31,0xbad5359e),LL(0xcc17a45f,0x137fb217),LL(0x3962aaec,0x18daea3d),L_(0x24dea50b), LL(0x585f316e,0x2db980d4),LL(0x94722851,0x890d4e8a),LL(0x17a80fda,0x0cd73f37),L_(0x0ef75926), + LL(0x7ddc10cf,0x2994a0c5),LL(0x0d862f7d,0xe8d6d38f),LL(0xa326bcf2,0xb39a090d),L_(0xebe228c9), LL(0x3fb2d1f5,0x4fe39e13),LL(0x292a329c,0xa1c6d77e),LL(0xa1f5d625,0x3f828ea8),L_(0x70ad621c), + LL(0x997cd647,0xb7237a23),LL(0x94de2084,0xf796b3ba),LL(0x0c93caa4,0xd97e1daa),L_(0xb74fa915), LL(0x4dfd621a,0x146706c7),LL(0x23c7b716,0xd56307f2),LL(0x54c6c8af,0xbe011ca0),L_(0x2ef6e4a4), + LL(0xa3b6fdcc,0x33851601),LL(0xbee34e4c,0xd7da0ee1),LL(0x8e7c506d,0x2b92ce18),L_(0x62b378cc), LL(0xc8c0d60c,0x95f16f73),LL(0x661c0891,0xdc466782),LL(0xf7df962d,0x6fdadac1),L_(0x8f3ef4f4), + LL(0x81a3d47e,0x23206f02),LL(0x235bc318,0xf375259d),LL(0xec43017e,0xc5d4dab6),L_(0xf547238a), LL(0x6f0f9561,0x25d0ebfb),LL(0x8357f1bb,0x8c478548),LL(0xe137e309,0x996a92a8),L_(0x629e8aa7), + LL(0xa2b3a713,0x4a17680e),LL(0xa8c7b143,0xe242039e),LL(0xb84edf8c,0xf2acd3a2),L_(0x8b3a07a1), LL(0x4d9e5baf,0x2f0d5861),LL(0xcd2f47fd,0xe42d316f),LL(0x0d4da406,0x0cd04a92),L_(0x8ec4fcad), + LL(0x6dcda792,0x91a38da5),LL(0x76eb9c38,0xa4b8df93),LL(0x0a5b71ac,0x2d328a16),L_(0xf6122daa), LL(0x9fdf7693,0xa1eceeb6),LL(0xe9555051,0x2371d345),LL(0x55020183,0xae8df945),L_(0xcee3a23b), + LL(0x4fcaf8f6,0x88241ba4),LL(0x6839ac95,0x25f7328a),LL(0xb01a4c73,0x4e73d41b),L_(0xdbce70d9), LL(0x470b6777,0x25457ffa),LL(0x4155bcf6,0x2545dcdb),LL(0x79b75849,0x3c25b38e),L_(0x8c33a19b), + LL(0x5375d125,0xb2bc7aac),LL(0xc7b2eb56,0x434f55ea),LL(0x8112c880,0x15208222),L_(0x612138e3), LL(0xd30babc8,0xa6392e60),LL(0x7fcbfb2b,0xc2c437cd),LL(0xb4541276,0x134be404),L_(0x3de33a7c), + LL(0x2cb35099,0x27c80981),LL(0x6d995371,0x15e5f210),LL(0x1e16c2dd,0x0e59bd58),L_(0xfbad79fb), LL(0x819dc1a9,0x481ca9ce),LL(0x6cb52c72,0x7ae12702),LL(0x7e9afc86,0xfda3af10),L_(0x41a661b7), + LL(0xbab0c519,0xc421cd20),LL(0xb68b5cf7,0x1e9b3f9e),LL(0x107cec0d,0x81208f8b),L_(0x383162ed), LL(0xf0c38656,0x2a8bfdfd),LL(0x0f977619,0xedf48fbf),LL(0x382decc1,0x8e5df4ce),L_(0xf0414c84), + LL(0x3083dd2a,0x15f29276),LL(0x89f9b2a7,0x717244c4),LL(0x817b0a58,0x6ee121aa),L_(0xf56eeafc), LL(0xcb4b4066,0x0bbfabbe),LL(0x52eb19cc,0x3641db33),LL(0xc2b09e5e,0xa82ae35b),L_(0xd9cf95b8), + LL(0x8dada775,0x017073a4),LL(0x4f731dd0,0x7488377c),LL(0xd07b63f1,0x5e605b0d),L_(0x4769aa10), LL(0xf9e533a6,0x26cfa7a4),LL(0xfac43672,0x077deba1),LL(0xbb074c5c,0xe02c4623),L_(0xb6536982), + LL(0x711f0c09,0x0bd8823b),LL(0xca3a4054,0xe696bd3c),LL(0xbee559f1,0xac8c4662),L_(0x54b8d320), LL(0xff20eb5d,0xfc9e136f),LL(0xafaff913,0x7fd1bf6f),LL(0x15a9349a,0x81208c2d),L_(0x0d06fd25), + LL(0x5b98ceba,0xc4906f43),LL(0x3cb8d3d5,0x671d2405),LL(0xef621a84,0x42fee26b),L_(0xcb150abc), LL(0xfa8ba5fe,0x8c081900),LL(0x2e00a64e,0x548ca8ab),LL(0x4b9fa7b4,0x7c4d0b48),L_(0x60c91243), + LL(0x612a3e39,0x36d03f9a),LL(0x7049620b,0xf3a56627),LL(0x29c0b51d,0x50448eab),L_(0x3f0b2dd9), LL(0x673df2ce,0x217c3b7c),LL(0x8eca2e20,0x17355feb),LL(0x20883efb,0x3afc2315),L_(0xc23211b5), + LL(0x4fe29bd3,0x3c1d5c97),LL(0x00ef3bc9,0xb31d8854),LL(0xf1f069cc,0xc5bebf01),L_(0xf661ac1a), LL(0x08af7536,0x54219e41),LL(0x25a7fa91,0x10eb2c79),LL(0xf217cbfb,0xf0d4c0bf),L_(0x4f2df7eb), + LL(0x0daa5bcd,0x8f2773da),LL(0xb21c4cfa,0x88c3cc76),LL(0x52c04b62,0x590a0567),L_(0xf35f4a84), LL(0xdf2a6679,0xcc2e1148),LL(0x04ae8b1c,0x340ba5a3),LL(0x92f66f03,0x4053731c),L_(0x20aba0c9), + LL(0x55e48460,0x0a4bb9e8),LL(0x9ad08ec7,0x4d8d0e03),LL(0x54a4bb3f,0xc9bcb020),L_(0xf5564618), LL(0x0bfda4fa,0x2f0cb8a9),LL(0x64275534,0xc49d2847),LL(0x33405050,0xd7e50869),L_(0xaf97cbcd), + LL(0x6faccf10,0x96f1d3cc),LL(0x9f391b14,0x4db1c834),LL(0x5f6bed3c,0x008d20fd),L_(0xc79a2572), LL(0xe2180f45,0x77d13f01),LL(0xb71f049c,0xf95c462e),LL(0xe088bd21,0xbd449f9b),L_(0x3a76432d), + LL(0x0405bf97,0x12dbb0ce),LL(0x4f53adb9,0x2a497b10),LL(0x13ff9791,0x61674465),L_(0xa80fdde3), LL(0xc0cc1d8e,0x50e102a8),LL(0x28391810,0x621706f7),LL(0xd119b153,0x47aca745),L_(0xeb9f0102), + LL(0xc5273792,0x6dc8e415),LL(0xb01555de,0xe0184e8d),LL(0xc5669efc,0x14d3a409),L_(0x0c68a597), LL(0x41d1a98a,0xed8dbd5f),LL(0xc5474f33,0x98419695),LL(0xd21a5750,0xf6b7aed6),L_(0x0d2d58af), + LL(0x4cd02456,0xb63ca144),LL(0xa81eba20,0x84079574),LL(0xf6e2c559,0x4e439115),L_(0xe8f8d978), LL(0x283b6289,0x8cf406a6),LL(0x23a5c7d1,0x4837878b),LL(0x4e16a72a,0x3db551d5),L_(0x88436ae6), + LL(0x334d8d79,0x32172eb5),LL(0x7d243c47,0x1ed58d88),LL(0x4a4714a6,0xed3510a8),L_(0xfde7f4a6), LL(0xb3d82e22,0x1ed13734),LL(0x4721f266,0xa661cacd),LL(0xf2889694,0x99b1258f),L_(0x45218687), + LL(0xd82826c7,0xe91d500b),LL(0xcf933e7f,0xdfae0ba5),LL(0x2d1ad273,0x572c7767),L_(0x79dbaa6e), LL(0x6b8ab0cc,0x17390031),LL(0x6c991f09,0x81e5fbb5),LL(0xbaab3192,0xfc86b907),L_(0x93c673b6), + LL(0xbe8c8998,0x3543430b),LL(0xc2f96d5f,0x50196f53),LL(0xac730ca9,0x376185cf),L_(0xfeac300d), LL(0xe73fa407,0x96df0a24),LL(0xf035d502,0xcdda229d),LL(0x9ca93571,0xd64df17a),L_(0x01970aaf), + LL(0x7ca6efb8,0x471f2098),LL(0xa5fb90b3,0xd70e4a72),LL(0x2aa370a3,0x09b936b9),L_(0xcfb37658), LL(0x3b357def,0xb2dfe05b),LL(0x1e8ce8f7,0x3594e46b),LL(0xcf8806b7,0xf062523d),L_(0xa1c74034), + LL(0x03cfe393,0x02fd45e8),LL(0xf0d5ce43,0xf575564b),LL(0x86092a2d,0xc28ad111),L_(0x0990658e), LL(0x70cbd16a,0x850acec1),LL(0x097c875b,0xc2704919),LL(0x79052ae9,0x651a1ec4),L_(0xdf5b00e0), + LL(0xd20da620,0x7c4aacf3),LL(0x12306aef,0x51b1a962),LL(0x0369e1f2,0x86d193a6),L_(0x3bdb2181), LL(0x93dcf8af,0x4d9c6a9a),LL(0x2e587a72,0xae32ac0a),LL(0xe30d76a5,0xaf97ed7f),L_(0x20fbaabf), + LL(0x94bc93d0,0x044baaf3),LL(0x04047c5f,0xf8872ab5),LL(0xe4e3879a,0x436ef7e0),L_(0xed64bf04), LL(0x11c91ee8,0xa9c7bdcc),LL(0xc1679bbd,0x006ae67f),LL(0x38ba10d2,0x83b598cb),L_(0xa99537cc), + LL(0x01ef166f,0x18eb43c9),LL(0x5aa367ce,0x93100d60),LL(0x2dc9b947,0x066d6c04),L_(0x3a41602a), LL(0xb9778f57,0xfcc4d377),LL(0x3dbf71fc,0x1d448cab),LL(0xc9ccbaae,0x0aacb390),L_(0x985954f5), + LL(0xa4179427,0x1f6e5aab),LL(0x4d58f138,0x64717e7d),LL(0x72e75855,0x389e7a2b),L_(0x4b5aa1e6), LL(0x12a92070,0x29d90bc2),LL(0xb35b2d2a,0x553f857a),LL(0xaca3b9cc,0x832c687a),L_(0xe2911a57), +}, +/* digit=8 base_pwr=2^56 */ +{ + LL(0xf505ff0a,0x56b1b68c),LL(0xa766a468,0x85630476),LL(0x7f0a8ccb,0x15b9ce0d),L_(0x8bfb375d), LL(0x8741f066,0xe7944b6a),LL(0xe502fd6a,0x8026fb64),LL(0x079ac7ac,0x956dea42),L_(0x3fc3f258), + LL(0xbd407e63,0x7157aab7),LL(0xde649372,0x62a53378),LL(0xec032ca1,0xdc9ac491),L_(0xaf00af85), LL(0x5a1b0fb3,0x3d3a586c),LL(0xb1b4ed91,0x6cce27ca),LL(0x0da4ec47,0x2fb5dda7),L_(0xe69eb235), + LL(0xa2c9ee34,0x4a718a90),LL(0x6813c47e,0xaf44fcce),LL(0xcc1aaacb,0xbae4540e),L_(0xfd6dde20), LL(0xf4786198,0xa1399699),LL(0x47c1ef53,0xabb23a85),LL(0xff24f01d,0xa1779b32),L_(0xafe60f72), + LL(0xa9ef8159,0x6e7fd2ab),LL(0x206a6f14,0x802458a0),LL(0x43a11c7f,0xf31bbf6c),L_(0xe2ff97fb), LL(0x893d191b,0xb83101a9),LL(0x0b2c24c6,0x52ca6e06),LL(0xd00a24aa,0xc5ea4420),L_(0x80054822), + LL(0x3173ee97,0x5d302b85),LL(0xa3ec0bc2,0x275a0fe0),LL(0x7469b929,0xde033220),L_(0x9823907b), LL(0x06ebb14d,0x8d35908d),LL(0x668b8357,0x8a0b74c2),LL(0xcd96886f,0x388da84d),L_(0x8e759b7d), + LL(0x82c45d8b,0xb0068631),LL(0xb87f256b,0x51283056),LL(0xb7562059,0x554d7fad),L_(0x68b8bed1), LL(0xe9ccc163,0x5349ed31),LL(0xaa23f48d,0x0ef896e4),LL(0xac744603,0x125ee820),L_(0x150da1ca), + LL(0x546f326b,0x16a4c78d),LL(0x26b49a2d,0x42a579ae),LL(0x69e5dc3a,0x50449596),L_(0x2af6b2c1), LL(0xde7ccc7f,0x07439dc4),LL(0xf5736c67,0xacbc3d69),LL(0xeeb790dd,0x0cc523dd),L_(0xae2f8bbf), + LL(0xf1c49667,0x1c12b2d5),LL(0x7628ecee,0x7bc41dbb),LL(0x0692cc8d,0x65398808),L_(0x5048f64f), LL(0x2cde35bc,0xd880351d),LL(0xae75c4fc,0x131f80e2),LL(0x3e2ff89e,0x5f9ca860),L_(0x435a454d), + LL(0xfc6b3603,0x1dbe11f0),LL(0x805cdd89,0x0ed82341),LL(0x0562fe66,0x5d6a45d5),L_(0xc9eb9f56), LL(0x02dcc377,0x1fbcb272),LL(0x4b25d574,0x873a72d5),LL(0xb78a31c0,0x1466b27f),L_(0x468c47f5), + LL(0xba96c374,0xb836e3e5),LL(0xf6743281,0x2c1ed972),LL(0x13f41d4f,0xd77d4340),L_(0x397a6846), LL(0x748b4159,0xbd735496),LL(0x6560f3f3,0xeadd97bc),LL(0x61405ead,0x119c162c),L_(0x9870a7f1), + LL(0x7c46ea86,0x4f81eb05),LL(0xdd4548bd,0xaad784b0),LL(0x49ea4452,0xff17a13f),L_(0x54ebcb9f), LL(0x810464d0,0x3e7236a8),LL(0x4d44b2ce,0xfe62fc5f),LL(0xfeac4aaf,0x0d85cf3f),L_(0x7d7cb315), + LL(0x4ea682b5,0xb0f78fa9),LL(0x82bbcdcf,0xa319bd52),LL(0x01559ec4,0x7cdb7afd),L_(0x86ff3957), LL(0xc4e5e26b,0x6c320e71),LL(0x45b1afec,0x8d455eef),LL(0x9d12c298,0x6b93f2de),L_(0x3e21f849), + LL(0xbb797884,0xc12b655d),LL(0x219c2865,0xa6ec705d),LL(0xc6daf8cf,0xf3ec2977),L_(0x10269fa7), LL(0xf0951e7f,0x8de615f1),LL(0xb448dafb,0x0e454e35),LL(0xac03ce6a,0x0fd25b12),L_(0xb96afd44), + LL(0xc1091d62,0x069f14f0),LL(0xc970df0c,0xa04696ef),LL(0x8cddee54,0x0475ddb9),L_(0xac228ae6), LL(0xda3d6fc8,0xc73f5026),LL(0xbdeb5cb1,0x475ad247),LL(0x82098200,0x8694111c),L_(0xc11479ab), + LL(0xf181f7b2,0x0c73fe98),LL(0x4aa502d3,0xe9de471e),LL(0x6c27909f,0x4f620050),L_(0xf76957c1), LL(0xe0c4eaa1,0x1565b30a),LL(0xfba38431,0xb518ef0e),LL(0x65c7038d,0x82a6a124),L_(0xe7db47e4), + LL(0x56f2fd55,0xfdab8dbd),LL(0x773fef23,0x198d2eb9),LL(0x011efbb9,0x8698e8b3),L_(0xa4f63ad2), LL(0x8629eaf2,0x60003ff6),LL(0x5aebc019,0xd2455886),LL(0x00bcb9de,0x859f79cb),L_(0x6a9d21d2), + LL(0x7a576508,0x8a59362e),LL(0x21480f43,0xbd8af994),LL(0x4a37a459,0xe55838d3),L_(0x36aab717), LL(0xd4ef1a0b,0x1d33aa1b),LL(0x86dc1133,0xe457ffeb),LL(0x0a1e4a76,0xf9d8ea27),L_(0x7a9f7b61), + LL(0xc6332530,0xf5ab77c1),LL(0xb7c034fa,0xcae4753a),LL(0x89204639,0xd0dd8159),L_(0x4ba725f9), LL(0xae3159be,0xa510602d),LL(0xc72415fe,0xd6d54251),LL(0xa6bc7ccc,0x7d585ae6),L_(0xc90e9462), + LL(0xb6723f1d,0xaed13815),LL(0x84f02e2a,0x6de52d8b),LL(0xffdbb5f9,0x3c9269f7),L_(0x5c61743e), LL(0xb086030f,0xd5bc6378),LL(0xa9132224,0x607634ca),LL(0x12f65fd8,0x817445f8),L_(0xd0c982b8), + LL(0xfb02a22b,0x40f7fe3d),LL(0xcb5018ef,0x34d86234),LL(0xf32b39b6,0x621a4bd4),L_(0xe7c33edc), LL(0x974f5440,0x7972a383),LL(0x586d4bef,0x8b71269c),LL(0x81211aa8,0x8c46dfe1),L_(0x5b192417), + LL(0xe3e15473,0x30ddafb0),LL(0xe56640a6,0x71464526),LL(0xe01ca53a,0x6b7735ff),L_(0xa4ba7379), LL(0xfee843b7,0x184d5072),LL(0x1f2853f9,0x0b71f2dd),LL(0x0c65322e,0x77546dfc),L_(0x814529cf), + LL(0xbb7e4da9,0xe44092c0),LL(0xdcf36025,0x6e08eec1),LL(0xa38b0382,0x2beaabb1),L_(0x57dab020), LL(0x30b6f340,0x3fc25cd9),LL(0x16190d52,0x0d13ca42),LL(0x632514c8,0xc69b7cce),L_(0xcb82be1e), + LL(0x3eb4e07f,0x79d31336),LL(0x78cc588c,0x986322bf),LL(0x5d3ca505,0x0951b466),L_(0x82b9278f), LL(0x20636a0f,0x6c160f3d),LL(0x308d56c2,0x62ceeabd),LL(0x460ed898,0xcccc63ee),L_(0xd27b55ac), + LL(0x81017a2d,0x2097cb5a),LL(0x6fb48059,0x423d9fb9),LL(0x8b4a82f5,0x3bf0f71b),L_(0xda304a5b), LL(0x0620aba2,0x5d609c9d),LL(0xb32599e2,0x77106961),LL(0xea5b4567,0xc072d5ff),L_(0xf9f2dc57), + LL(0xbe531971,0xdacba735),LL(0xbc32b9c9,0x4c64a7d8),LL(0x154849a8,0x29e4dd9b),L_(0x11dfc29e), LL(0xbaa9dc51,0xc8616f73),LL(0x971ba9a3,0xf61450f4),LL(0x15776b0b,0x8325c930),L_(0x294a4ec5), + LL(0x86967b01,0x16f8969b),LL(0x235d5c34,0x3233bbff),LL(0x57aa97db,0x5c79afae),L_(0x417467c4), LL(0x2f6f6091,0xe7950968),LL(0x59ca92ce,0x9d943f83),LL(0xb1a7c34e,0x6203a921),L_(0x066a9647), + LL(0x759f1a19,0xfb7e2683),LL(0x84d55477,0x96a642c6),LL(0xbdee722d,0x74bfcbf4),L_(0xf514f2bf), LL(0x80d0ae79,0xba7058a3),LL(0x4713d1ae,0x6c24158b),LL(0xa6c17b5f,0x8941ddd8),L_(0x261d9ac7), + LL(0x943bc0de,0xac5e5ce6),LL(0x76370390,0xd093dc4b),LL(0xfce7f694,0x7c34fbe8),L_(0xe3b42849), LL(0x4b4bfe6f,0x8781925a),LL(0xe273ae40,0xfe87a94c),LL(0x777881ae,0x6fc64556),L_(0xb0f8e854), + LL(0x498275e5,0x024a93d4),LL(0xa6b66d66,0x05a83965),LL(0x56c335af,0x0805e125),L_(0xe1e39e4e), LL(0x059f9143,0x9bb438e6),LL(0x0e8dde7c,0x1ec8aae3),LL(0x06a38af8,0xa234eb6c),L_(0xd74870f8), + LL(0x0883fffc,0x05b51c39),LL(0x62d8ac43,0x839148f1),LL(0x8eefcd87,0x3b151865),L_(0x3c047e9d), LL(0x197a8a18,0x76ea2960),LL(0x82998a41,0x8a3902cd),LL(0x110b3b2f,0xbcfe60ba),L_(0x89b19d6d), + LL(0xab6f3543,0x8765ec48),LL(0x36e47913,0x1b75763b),LL(0x767e806b,0x81bc9bd7),L_(0x38103684), LL(0x0f165768,0xb8107b90),LL(0xbe9c24b5,0x54d331ef),LL(0x43623ea1,0xf06a675d),L_(0xcbeab5be), + LL(0x8e21820b,0xd064269b),LL(0x1ccbf4e2,0x7020a8a8),LL(0xe21fbca7,0x5cee0a5c),L_(0x263913a5), LL(0x8a6b9ebd,0xa3fc8bad),LL(0x047f1949,0x838a0019),LL(0xd1a5d116,0x35a0d475),L_(0x9e3419f5), + LL(0x742ad243,0x36c9f808),LL(0x631145ae,0x2db0aada),LL(0x0c423277,0x4bf70210),L_(0x2f1584d5), LL(0x6f50ff50,0xa19699d1),LL(0x8015f024,0xc8530fc5),LL(0xf9755c59,0x25538f41),L_(0x387ddb31), + LL(0x1985efbe,0xe8509933),LL(0x9e321e61,0xb8ff1d78),LL(0x7dc68e42,0xf8206e77),L_(0x58e783b2), LL(0xad8c32f2,0x906883cc),LL(0x6f41c769,0xa42d4644),LL(0xbb9527f5,0x38e2f27d),L_(0xcf17789e), + LL(0xf5dd9f3b,0x1bf5a9f9),LL(0xae648e71,0xe8b2f305),LL(0x3f5f8275,0x93245241),L_(0x80482c3b), LL(0xb0e6f457,0xef9f19aa),LL(0xaa9e665a,0x6dcb7ba1),LL(0xad522975,0xce379c80),L_(0xf8653e32), + LL(0xeea9e807,0x19d1e570),LL(0x4023698a,0x5fb46b4e),LL(0xa19db2ab,0xd2800586),L_(0xaee5fc4c), LL(0xe6f0c329,0x378c9ac9),LL(0xbcb82922,0x142d0bcf),LL(0xc3b8fa15,0x1b19035c),L_(0x1c3a4f74), + LL(0x074cbefd,0xf201d616),LL(0x88de2216,0x703ef032),LL(0x409f13a6,0xfc1697ff),L_(0x1f4ee383), LL(0x4f0ae8bc,0xc70f2c0a),LL(0x596702b2,0xf1e207d2),LL(0xaa0989d3,0xcc60a8c2),L_(0x5ba1fcbb), + LL(0x4e7d0964,0x3f768dfd),LL(0xee929e0b,0x0dda310f),LL(0x171719ee,0x91e6ec5d),L_(0x48549719), LL(0xb61e9fad,0xa3e2860b),LL(0xa30f11a1,0xa7e2c8e0),LL(0xf804fa42,0x58c435e7),L_(0xd81ff657), + LL(0xd7d22038,0x3ee264ba),LL(0x6d74b27a,0xf3577276),LL(0x776b6b64,0xa97538c5),L_(0xdc494d53), LL(0x89e972aa,0xe4acc7fd),LL(0xe5d1ac83,0xf7c63ea2),LL(0xf0a59879,0x9b5f787d),L_(0x0730aa6b), + LL(0x6c13249b,0x5d202340),LL(0x15ca4cef,0xfbbc39ae),LL(0x03078647,0x7cbaad05),L_(0x7a152a3a), LL(0xc0834b0b,0x5c759c2f),LL(0x1d48b5cf,0xc983dd84),LL(0x954731e0,0x67babf9e),L_(0xb890cb9b), + LL(0xeefdbd63,0xfbda2538),LL(0xd314af3a,0x77f9df03),LL(0x333f8c94,0x07bcc285),L_(0xd37bbc81), LL(0xadb373f3,0x7ba7309e),LL(0xe5686c59,0xb6c90f3a),LL(0x96218ea6,0xbfb2d48e),L_(0x7bc6fa2a), + LL(0x5c8452f6,0x91273fc0),LL(0x9f330845,0x12ba358a),LL(0x8d3155c9,0x229515d9),L_(0x4e0d0091), LL(0xaeed3a8c,0x3cca8a70),LL(0xc538ea5c,0x8528e080),LL(0x7f68ef37,0x412db800),L_(0xfc39477c), + LL(0xc1f84a4a,0x202d77cf),LL(0x3f50f1ae,0x94eeda74),LL(0xc9d43dee,0x2163dc89),L_(0x82838e71), LL(0xc916bedb,0x68b2f3b4),LL(0x028beb58,0x32389e93),LL(0x2f67136b,0x63c11941),L_(0xf0576d72), + LL(0x5b467cc7,0x79a7908a),LL(0xfe08b743,0x237e35d8),LL(0xe4c9aa02,0x3121fc40),L_(0xe6d6660d), LL(0x1c0060e4,0xcc2b942e),LL(0x54acd180,0xab155509),LL(0x4b3ac4d5,0xde75824c),L_(0x91ca23fa), + LL(0x4201e6e0,0x46582482),LL(0xa927f3c9,0x78c500df),LL(0xe88a4310,0x1f445777),L_(0x3e33ca59), LL(0x2a806dc7,0x71dd2626),LL(0x54a8126d,0xc14947d3),LL(0x9c986434,0xbec3ed13),L_(0x71db698f), + LL(0x422ef39a,0x731eb54c),LL(0xa93b1752,0x61227804),LL(0x6f7bfa09,0x873d3a45),L_(0x4f66d06f), LL(0x6fc87ed2,0xc97be3e6),LL(0xf6435b79,0x338ab476),LL(0x207b9b6d,0x1c6c74ae),L_(0xb64b6b70), + LL(0xfd951bda,0x74ccc359),LL(0x4be5a10b,0x44c6bf22),LL(0xb790b940,0xc32a94b3),L_(0x098b38b5), LL(0x57ac1bc0,0x4ef7c296),LL(0x18feb7b4,0x82d9c658),LL(0x6f3016a9,0xad72b80e),L_(0xbdf2db87), + LL(0xcffa9aa8,0x0ca1adfd),LL(0xb13f2859,0xed4f750d),LL(0xd393e869,0x277e9cd1),L_(0x90b15b52), LL(0x9386d23f,0xf705f504),LL(0x2c2d2fb1,0x98b6973f),LL(0xf49260e8,0xcb991398),L_(0xfed48a8d), + LL(0xb2b27562,0x2687c047),LL(0xc3ca0fc2,0xe54d1276),LL(0xbb9d5b5b,0xe241d7c5),L_(0xef9d2be9), LL(0xe9d4f361,0xfe8acc56),LL(0xc5011f32,0x5fc31f72),LL(0x8545eca9,0xfa8d83be),L_(0xc270dd4a), + LL(0xe5151a0f,0xb3951e08),LL(0x78a942de,0xdd13a9d6),LL(0x57fae761,0xbc46ee6e),L_(0x83169c42), LL(0x2d1c6395,0x340fc521),LL(0x669821ab,0x24a2322a),LL(0x09548072,0x92a505ce),L_(0x83ec59bb), + LL(0x0a4dfff3,0x3ce8adae),LL(0x41ed0956,0x7f33cce3),LL(0xc0e55577,0xba8eb552),L_(0xa018b90e), LL(0xa0f2bcda,0x67316cff),LL(0x5703fba5,0x58c433f9),LL(0x6078c1e4,0x99f221d3),L_(0x047eec80), + LL(0x41f43d8c,0x81739c7f),LL(0x6f11b810,0xae918a0f),LL(0x1e314860,0x9ddc21df),L_(0x18e4d189), LL(0x558db016,0xe78c5637),LL(0x439dc1dd,0x0de5812f),LL(0xa22acbf8,0x9f469794),L_(0x161460c0), + LL(0xb19e5bf0,0xcd4a85b0),LL(0x7e2cff5d,0xfd828b89),LL(0x998dca20,0x3d5bd0f7),L_(0xbfc65978), LL(0x176f7759,0x388987a0),LL(0x9b689b7c,0xed6c490d),LL(0x3ea487dd,0x5b3b94d0),L_(0xa77864e1), + LL(0xdff87175,0x39413fad),LL(0x796a4cff,0xf20c6ce8),LL(0x978f94f4,0x2c42836e),L_(0xdabb2333), LL(0x344fb7ef,0x683de92c),LL(0x5fe8df50,0xe072aff6),LL(0x705a1d2a,0xf68dcb2c),L_(0x16e49003), + LL(0x04c5b257,0xc7d4f66d),LL(0x02ce592d,0x679405b4),LL(0x449fe24f,0x49708f6f),L_(0xe167be96), LL(0x3f036191,0x7456adb8),LL(0xafdb07bc,0x63213856),LL(0xfc9ec5eb,0xb52ff28e),L_(0x55b069a6), + LL(0x0dfb54c3,0x1b0cec1a),LL(0xf4484ca3,0xa32d497c),LL(0xde798874,0x05b5ac5f),L_(0x87ada89d), LL(0x771e433f,0xbc022aca),LL(0x93b1313e,0x5af43a1b),LL(0xa4d9094a,0x71a6efc0),L_(0xd897f8e5), + LL(0x8e365581,0x5f811bb2),LL(0x9a725a78,0x04edaa41),LL(0x1f6cfcf4,0xe1abeb72),L_(0x6d3872f8), LL(0x6636d80e,0x90cec4c4),LL(0x400da758,0x751efdbb),LL(0x74d4d54d,0x6fed8555),L_(0x4bc5cda0), + LL(0xed247186,0xb4a54351),LL(0xa9b678d4,0x349f3364),LL(0xf079b22c,0xcd89ee25),L_(0xec3738e6), LL(0x9a75388b,0x1754341f),LL(0x3b3edf75,0xe93c6d27),LL(0x46732c66,0xcfead94a),L_(0xa689fe7e), + LL(0x46839b8d,0x91cb6be2),LL(0xa3af0a2d,0x68643eeb),LL(0x37e8c56e,0x44cf93c9),L_(0xc55af358), LL(0xb74e3e18,0x2941f1b6),LL(0x85525c49,0xe580c8b8),LL(0x7fe18358,0x32c35949),L_(0xbf4a513e), + LL(0xf3da6204,0x8adf6261),LL(0x6619bed5,0xe7cd8eab),LL(0x5bb1a013,0xdac22d18),L_(0xe70d7542), LL(0x64ef4466,0x0bfa91a1),LL(0x195a05ff,0xb9ce90fc),LL(0x52bb429e,0x34d0f00f),L_(0x904ed07d), + LL(0x335940e2,0x4331ebb8),LL(0xf071319d,0x6f7c1728),LL(0x05e7c4f2,0x1da3b676),L_(0x7abf809c), LL(0x1a1e3a9f,0x9290a717),LL(0xf68174d5,0xab7b1f25),LL(0xa93e4d8a,0x7f4cbc44),L_(0x93b784f5), + LL(0x5640e88e,0x7b8373a5),LL(0x1bf6e31b,0xcdec2eb7),LL(0xb8888fa2,0x010ee8af),L_(0xa7cd3971), LL(0x2d9ffc40,0x7e33265d),LL(0xc3866a61,0x26755d7a),LL(0x9e1bb3f0,0x9be25bb9),L_(0x1c158895), + LL(0xcbffc265,0x187246e4),LL(0xcb1700db,0xcb116afc),LL(0x3aa73e7a,0x93e55e2b),L_(0xb2ba60e5), LL(0xfa8aa297,0xbcfa0106),LL(0x253d2d5c,0x4919fb96),LL(0x9fa1e7c7,0x04ad0152),L_(0xd483d4ee), + LL(0xce14b1f7,0xd95b8ba8),LL(0x1d6eb61c,0x20d43399),LL(0xc7ef0d06,0x47b3133d),L_(0xf255931e), LL(0x5d5a0b15,0x60b98074),LL(0x2344f6b6,0x8b1f2bd2),LL(0x789f9baa,0x00302e85),L_(0xccdbabda), +}, +/* digit=9 base_pwr=2^63 */ +{ + LL(0xfb42e9d6,0x228b3a54),LL(0xd1e183b4,0x3efd699d),LL(0x4948ad88,0x18288fb9),L_(0xf45f24f0), LL(0x23d7207d,0x37574acd),LL(0xe86d0db8,0x57c32be7),LL(0x1ccef3f9,0x451b09b7),L_(0xee7d7495), + LL(0xa61d5325,0xd52a1c1e),LL(0xd08a8ae3,0xcb34540a),LL(0x750d587a,0x03e86cff),L_(0x27e73149), LL(0x156ef6cf,0x527a9877),LL(0x96f4a96b,0x6ee444f1),LL(0xe4a1f249,0x038153e5),L_(0xd9998bb3), + LL(0x2bb80640,0x281ef606),LL(0xee8331e0,0x73bbb0b4),LL(0x71264953,0xa04c6f5a),L_(0xecfe722b), LL(0x85f7bcc2,0xbb4ca37a),LL(0x8ff53f4b,0x6e5931f6),LL(0x42ab81a0,0xb862be64),L_(0x8ba40994), + LL(0xe1ee20f3,0x3c280032),LL(0xa9734b49,0xc6f2aff8),LL(0xb12f26d4,0x3686d2f0),L_(0xe0111b72), LL(0x8bf62e5c,0x7a48daba),LL(0x069099d5,0x63c661a3),LL(0xaba55aed,0x6cdc05f3),L_(0x4ea7440f), + LL(0x33a9ba23,0x4a15994f),LL(0xe99a66d4,0x022ffe96),LL(0x4d054e49,0x924ebb42),L_(0x155f0d8c), LL(0x4d774618,0xe319a1e3),LL(0xe6d591f2,0x8f71c6dc),LL(0xe9b13d22,0x0834867a),L_(0x39cb42c0), + LL(0x4edd01ad,0xb4a0a7e3),LL(0xd28d49c3,0xb472df44),LL(0xab2cbbd4,0x4cb47055),L_(0xcd15bd1f), LL(0x31800301,0x16ef99e3),LL(0xb349d462,0x78a8538e),LL(0x746c4b41,0x4dce044a),L_(0x56cf52da), + LL(0x7bf3d9bf,0xb1b9c2e9),LL(0x0196cebe,0xaaa33f41),LL(0xdb961af6,0x51023370),L_(0x68f286d7), LL(0x8d6b25f5,0xad763cf9),LL(0xa07bb401,0xe4c3391f),LL(0xab25bfde,0xac71bbe8),L_(0x06a272b6), + LL(0x33ad9d0f,0xd727b458),LL(0x6aab9dfc,0x340cf8e0),LL(0x695de110,0x078f906d),L_(0xa987a86e), LL(0x65ed9ae8,0xd50493d4),LL(0x3eb5588c,0xb2673304),LL(0xce9abc84,0x101a021e),L_(0x2ec773ad), + LL(0x492a4dda,0x7bdb0ab2),LL(0xb55a240a,0x08dd2523),LL(0x06998948,0xf475a7f2),L_(0x04f12f32), LL(0xd2a94b0e,0x688143bd),LL(0x73021050,0x5bdf7bb4),LL(0x325ed38e,0x8114699d),L_(0x467cd567), + LL(0x3e7baca8,0xcbfe120b),LL(0xe69958d1,0x56a88710),LL(0xa2e1385c,0x9586b08e),L_(0x8d8876e4), LL(0xa05f5596,0xe749a357),LL(0x85028619,0x62d3a285),LL(0xf41d9b86,0x8430e884),L_(0xe0fe7436), + LL(0x449eb800,0xe20a701b),LL(0xbc42935f,0x63dac155),LL(0xaf260958,0xb55c4fab),L_(0xb5f1b7b4), LL(0xbaf2a69d,0x3e2c715a),LL(0x61856276,0x158e35b0),LL(0x447c4419,0xfaf57b99),L_(0xa5a859f6), + LL(0xe7457d94,0xa50c7d9f),LL(0xaaf263e7,0x5bd93ac0),LL(0x27bdb99e,0x28bc6b58),L_(0xa291a83e), LL(0xfed995c1,0xe155fe7a),LL(0xa92f7c72,0x441255d1),LL(0xfdebd3d9,0x54ea2480),L_(0x1d9b3247), + LL(0x8cfadc9d,0x3bb14260),LL(0x40205f43,0xf6fea697),LL(0x1e5b99d9,0xf68ea4a3),L_(0x99449ff0), LL(0xc42cccaf,0xb4680c53),LL(0x6506393b,0x1cded754),LL(0x9fa16281,0x355805ad),L_(0xa8869c1e), + LL(0xa2b679a7,0x1fe90870),LL(0x60faf2a7,0x841d50a4),LL(0x74445704,0x97761552),L_(0xc6294a8b), LL(0x88aef1a6,0x8306971e),LL(0x79313f41,0x8a08da74),LL(0x459c33b3,0x5ff5c231),L_(0xa47413df), + LL(0x528acbac,0x8931a009),LL(0x1bb13907,0x64ec461c),LL(0x6e5d6bd9,0xf62b4e1f),L_(0x419ecc6e), LL(0xb5b1b6f0,0x2e05fe26),LL(0xa9d7ebb7,0x5484c570),LL(0x46afc58b,0x2c8381f8),L_(0x243d0786), + LL(0x67588c0e,0xa5a5886a),LL(0xd9832e54,0xf569db1f),LL(0xc764b06f,0x38ed1085),L_(0x7eb63b3e), LL(0x11a2a8db,0x585bdb20),LL(0x8a62c89d,0x3d5436af),LL(0x5e301e9d,0x35b91587),L_(0x4e488867), + LL(0x2e0bb083,0x9471249c),LL(0xcfed9152,0xd02dc9b6),LL(0xf83f6a42,0x859a1f08),L_(0x67c5e1b0), LL(0xfa3c9fbb,0x40557a70),LL(0xa37c1374,0x180e4d9f),LL(0xecd05a92,0x7635cd74),L_(0xccd0d7f7), + LL(0x0ad7acbd,0x02596d63),LL(0x81ed48b5,0xf3957977),LL(0x8ec54e25,0xfd23aeec),L_(0xa7513cf4), LL(0xfa5cf86c,0x8f6097dd),LL(0x2a1c180b,0xea60f2b6),LL(0xacdc4130,0x253dbb93),L_(0x9daa0c3e), + LL(0xece03d73,0x5f61e12d),LL(0x3336043a,0x1b5a6c4a),LL(0xa83c1fea,0x77fedd08),L_(0x64002d5d), LL(0x6ed15df8,0x08aaa0a9),LL(0x210df7fe,0x6ccfd556),LL(0x6b43ad21,0x98ade169),L_(0x42a1a05e), + LL(0x701d31e8,0x6e92d485),LL(0x8a6e2064,0x53a09da0),LL(0xc92637e7,0x4ee89ae2),L_(0xa8cbe2a0), LL(0xc8b9f5ad,0x688111d2),LL(0x1fafd02e,0x75a9c058),LL(0x315d9dc5,0x0cd0654b),L_(0x02890b79), + LL(0x24746253,0x131fab03),LL(0x1ac884b5,0xda0c0ec6),LL(0x8a1a27c7,0x66f76b97),L_(0x254a6db2), LL(0x8c4da1c1,0x17450bac),LL(0xd2c4396b,0x988596ca),LL(0x2691986c,0xd9864a47),L_(0x31c0544b), + LL(0x24fed0d9,0x2f0f74a0),LL(0xf3ddaad3,0x28c66344),LL(0xa0f85074,0x55736fde),L_(0x9f28ac0a), LL(0x0c8f2cd4,0x5b08d2ff),LL(0xaaec443d,0x96848bd3),LL(0x9fd17e0f,0xb4213134),L_(0xf10e3114), + LL(0x0077c0a8,0xd9301e91),LL(0xa6577039,0xdf36f0d6),LL(0x50e6f1b9,0xf5e86ab5),L_(0x58a7b434), LL(0xe0f8d5dc,0x2f0371f4),LL(0x1d3d9546,0xeabede08),LL(0x39c8d544,0x12bf2df1),L_(0x57a123a0), + LL(0x3878d553,0x817dc769),LL(0xfd0338c3,0x60a4d34c),LL(0xb99ba802,0x36e30856),L_(0x1dcffb49), LL(0x8ba72940,0x665a28b4),LL(0xf1016ea1,0x9d6822ea),LL(0x86a30740,0xc3331f1b),L_(0x0205d1d8), + LL(0x69c5e4bf,0x8628180f),LL(0xd7ae93c1,0x7ead547c),LL(0x227716ff,0x736914e5),L_(0x605a49ab), LL(0xec82f273,0xa3fd7cc4),LL(0x99cf069a,0x60d1dc75),LL(0x93c53857,0xdc5b96cb),L_(0x080bcc0e), + LL(0x5adf052d,0xf2335efb),LL(0x398d73b0,0x9cde465e),LL(0x00bb4ee5,0x212ffe0a),L_(0x7ab2ea4d), LL(0x9a9ad333,0xe2f332a2),LL(0xeeeaddc8,0xbb4703c4),LL(0x2962d309,0x143e5fcc),L_(0x49553c63), + LL(0x99cf8b3c,0xc941479d),LL(0x203c4316,0xe6b52643),LL(0x855dab4e,0x717ef700),L_(0x4ed7e83c), LL(0xe00ddd9b,0x494ccfde),LL(0xe90530ff,0x94ed4ea6),LL(0x8222ad9a,0x4430a4d7),L_(0x252e55b2), + LL(0x53d9c3c5,0x96506aa3),LL(0x3f484d6d,0x4d8a0741),LL(0x10e92da9,0xbd490fb2),L_(0xffd03214), LL(0x5fa8cebc,0x95792257),LL(0x7dfe3775,0xe4e706c8),LL(0x9120d23f,0xe2412ec1),L_(0x1de70984), + LL(0xa39f223e,0xcb479a89),LL(0xe7cc4f73,0x0ea9b422),LL(0x1382187c,0x9dc7e0af),L_(0x01c1bd80), LL(0x530c072d,0x7d0256e7),LL(0xbd971088,0xbcc29991),LL(0x7ac3d22e,0x36e29a92),L_(0xecf0d361), + LL(0x28113fdf,0xbb9273c9),LL(0x4c568852,0x6ab4dc1c),LL(0xf3f274f5,0xb258fcff),L_(0xdee7fe95), LL(0xf7dd8fb6,0xea16df71),LL(0x3f7227a1,0xf774dfb9),LL(0x8865b774,0xc554b491),L_(0x10280ec0), + LL(0x32e80670,0x0aaf5932),LL(0x600576a1,0x0829e9bc),LL(0x63228df7,0x7b64399d),L_(0xb12890cc), LL(0xce0e14e3,0xf166eabe),LL(0xa3e9c521,0xa17a6f3d),LL(0x01f649df,0x33266d1a),L_(0xb179110c), + LL(0x0d9efc66,0xaedb7e51),LL(0x2d780714,0x3d4cd6e3),LL(0x76c48d11,0x95533ca3),L_(0xef834a13), LL(0xfcf0269c,0xbb0ea84c),LL(0x8bf5f008,0x2e995288),LL(0x0c5e907a,0x5462d73b),L_(0x1d6aecec), + LL(0x80c6b892,0xb5bf76ef),LL(0x2e91623e,0xaf49e877),LL(0x1415663c,0x15fea421),L_(0x1f1c3707), LL(0x71645fb0,0x03dd8cca),LL(0xb068031e,0x639980b2),LL(0xd50a50f4,0xbde1692d),L_(0xc8df6655), + LL(0xb536b786,0xa49675be),LL(0x923698c8,0x410ff3c8),LL(0x9ea67a42,0x67f23bf8),L_(0x4444d11f), LL(0xbb783ec9,0x05829e19),LL(0x18b3a870,0x79d71996),LL(0x8b8364e8,0x917428eb),L_(0xafd1b685), + LL(0x391fe0d9,0x5cdc38f5),LL(0x8bdefd85,0x653fae2f),LL(0x60854dd0,0x5650f1c2),L_(0x3d41abd0), LL(0xb709cc99,0x338d5d07),LL(0xd45495b6,0x840a9918),LL(0x10d2d56f,0x05c210e8),L_(0x2709cd44), + LL(0xd04c44b0,0x3d513962),LL(0xb778f7da,0xedf97e8d),LL(0x237a1a75,0xb843c30f),L_(0xfe6c9001), LL(0x52dcbf0f,0xb7f1d85e),LL(0xdfc37735,0x3f355190),LL(0x63ca258b,0xbcb01029),L_(0xaad826a1), + LL(0x74f4c74e,0xf2d99edc),LL(0x3824eb60,0xcbcc9613),LL(0x96cabc5a,0xdb4188d8),L_(0xe5012748), LL(0xbf865e18,0x100a7041),LL(0x73b6c9c4,0x922a5abc),LL(0x4b2e4ffc,0x3c179919),L_(0x35fd801d), + LL(0x274742c8,0xfba7fe0c),LL(0x71487144,0x4d838b83),LL(0xa5684725,0xde29806b),L_(0xaf1f4b15), LL(0x3c9d276d,0x0418df98),LL(0x3da078d2,0x970cee18),LL(0x9504ef04,0xd6fca127),L_(0x68282ff5), + LL(0x2c2165ac,0xa4371233),LL(0xf24ea908,0x340a8ba9),LL(0xbf610642,0x751d8536),L_(0x169f256b), LL(0xa6be26ab,0xa7afba2b),LL(0x66d0d844,0x20c6f4fd),LL(0x292f0814,0xb9773f57),L_(0xc95c32ed), + LL(0x975db7a6,0x931f72a8),LL(0x2d2921a4,0x75be82f0),LL(0xaabec6ae,0x93be322a),L_(0xe0530ac5), LL(0x4d6dd8a4,0x21ed4c4a),LL(0x3539fb21,0x702437c5),LL(0x54c5601d,0xff29a4db),L_(0x33f41e0a), + LL(0x957fea7b,0x92370d49),LL(0x2b5f5d00,0x1feb014c),LL(0x9ce3670e,0xc246a235),L_(0xcfeee196), LL(0x58ffdb8b,0xc688f198),LL(0xc992ad9e,0xee953054),LL(0x3828369e,0xc2172706),L_(0x9016c246), + LL(0xf2b16a57,0xd8546f62),LL(0xf288a32a,0xf4996e54),LL(0xd866770d,0xa8e0a566),L_(0x41c5b1a5), LL(0xa81aa974,0xc8eca336),LL(0x06f5713c,0x7317dc66),LL(0xaf5394d4,0x1c420a16),L_(0x05bdbb33), + LL(0x3d683b90,0x8e6a5c3e),LL(0xf167321f,0xd4ab4b02),LL(0x4806da87,0x98fd2767),L_(0x23e3b0df), LL(0xf9faf1d6,0xde00e6e4),LL(0xa65aaac7,0xb82c1e61),LL(0x603b9246,0x39526356),L_(0x6098239e), + LL(0x1471049c,0xdf432ea3),LL(0x989bae16,0x08aae4b3),LL(0xcdff8338,0x1f3f8b31),L_(0xe12f868b), LL(0xa96b85c5,0xc59a6e02),LL(0x3e2e2a6e,0x679432f4),LL(0x35623aa8,0x66bc9d0b),L_(0x419d9c04), + LL(0xd12c1c29,0x3e403fad),LL(0x80a49b71,0xb3fa108c),LL(0xa16aaa21,0xc0552283),L_(0x941ac341), LL(0x5f01091e,0x78d10e07),LL(0x9e2010ad,0x78aaa1d0),LL(0xc3d626dc,0xad4eebe0),L_(0x616d3812), + LL(0x77d51621,0xa85af39f),LL(0xc44a4210,0x41d18ad2),LL(0xd700b9c9,0x00fd9c15),L_(0xaaeaf056), LL(0xcb08ab64,0xcc55572b),LL(0x58e16fee,0xdedd8934),LL(0x224b2130,0x27e82ee6),L_(0xe838331f), + LL(0xc76dfaad,0xf7314697),LL(0xc2d9578c,0x651c0514),LL(0x28f766e4,0xe5782e17),L_(0xdb1237a2), LL(0x6bb64cbb,0xdcb19e7b),LL(0x382be491,0x9ac56c1c),LL(0xf76d352a,0x0749b544),L_(0x7968d6a8), + LL(0x07203eb5,0xceb29d96),LL(0xe4a0f31c,0x3f66d60f),LL(0x62bd6086,0xc9e0da34),L_(0xd2a91a43), LL(0xeb1e3a08,0xf153aa4a),LL(0x13715918,0x07234640),LL(0x5c701eb4,0x3cc2d39d),L_(0x21bc78c7), + LL(0x481eb504,0xd3a03cad),LL(0x5f6a9d91,0x1d390d95),LL(0x3753e069,0x733ae857),L_(0x58532343), LL(0x4de25ac2,0xc858b2c5),LL(0x423a2e5e,0xd0e29a40),LL(0x03c6d3e7,0xbd4e4f93),L_(0xb99af7a6), + LL(0xf347240e,0x74665b47),LL(0x00f3a821,0xd4995fe7),LL(0xbb254497,0xae564546),L_(0x09235190), LL(0x8aa0bb23,0x148c2b64),LL(0x44c9ade6,0xd4712255),LL(0x01120f40,0x2095b0d0),L_(0xc2cfde05), + LL(0x7058189d,0x9096ca15),LL(0x0b95bea1,0xf2ebd4a0),LL(0x92a3cb12,0x403109c3),L_(0x67ede54a), LL(0xc8d15887,0xd330e122),LL(0xe94c1522,0xbc6cec87),LL(0xcf570b75,0x8c60e503),L_(0x7b779c33), + LL(0xc396d1d3,0xb7a09eba),LL(0xb7df5c36,0xa4ad2686),LL(0xda7c16c6,0xea76230d),L_(0xe12538a4), LL(0x5c153671,0x35168d66),LL(0x22a3c5aa,0x7317fdf8),LL(0x90a1a3b8,0x488f516d),L_(0x4a17692d), + LL(0x427a3fed,0x979163af),LL(0x5261367c,0x67750f39),LL(0x2b64c77f,0x121123d3),L_(0x57b5327b), LL(0x1ae75801,0x60be5340),LL(0x4320f770,0x1c82e2fe),LL(0x80d5de38,0xbfa35c85),L_(0x804730a8), + LL(0x7c65d352,0xfec3e94c),LL(0xe49a8416,0xe15daed0),LL(0xebea12bc,0xd561b8e1),L_(0x1b6939a7), LL(0x80b4e8f4,0x689cf9a4),LL(0x0a52d61b,0xe93a4cce),LL(0x2707af62,0x00163171),L_(0x888d6c2f), + LL(0xdaeed59d,0xcfab7c64),LL(0xd2cae7f9,0x5781a043),LL(0x10e60343,0x1e1d00e2),L_(0xb1b2d682), LL(0x8a649d20,0xbc56c4a9),LL(0x87923cc3,0xa8b43dfb),LL(0x05fa3aa6,0x07ea55ed),L_(0x08a82010), + LL(0x208d5b3c,0x5178b723),LL(0x10a520c7,0x2d1f55b2),LL(0xb1fffc0d,0xbbad0a50),L_(0xab6af890), LL(0xce6aec24,0xf8407690),LL(0x2bcfca02,0x1c5424aa),LL(0x56944f46,0xaab35dc1),L_(0xa5c174f8), + LL(0x957be638,0xd8227572),LL(0xfd6ac99b,0x7fcc54ba),LL(0x4081f3db,0x7e998f3d),L_(0x08e76dcc), LL(0x3ffd1f3b,0x1c7b8f2b),LL(0x0551cbe8,0x76e4b694),LL(0x045398f7,0x93712a63),L_(0x31b716ac), + LL(0x9ccd7f23,0xf2590188),LL(0x1074bc24,0x14031de7),LL(0x9d9df6f6,0x1a06a2f7),L_(0xf880ea5b), LL(0x9cc577d5,0xe509ec7a),LL(0xc7021265,0x864b62e6),LL(0xe9189561,0x09271f5a),L_(0x910bdd23), + LL(0x1f12f8d6,0x258fd822),LL(0x28518543,0xa7361b32),LL(0x338b1788,0xc4796256),L_(0x636fbcb3), LL(0x4f5097e8,0x0aff7ca0),LL(0x9306ff92,0x3deb3281),LL(0x6a18552c,0x5ba5cce7),L_(0x563c7d80), + LL(0xf184055c,0x0a2d6941),LL(0x682b34d2,0xba0b737d),LL(0x467f7fe4,0xa7e4e833),L_(0xfb7c4f69), LL(0xceeed886,0xd5513495),LL(0xc4ac863c,0xcc4688b1),LL(0x74229c34,0xd5e5f34b),L_(0x729a99ef), + LL(0x33e6967e,0xca92d3be),LL(0xacf9987e,0x27e6ecfd),LL(0x84c9d53f,0xa434a89c),L_(0x0ac3dbe1), LL(0x3f16aded,0x05a4b104),LL(0xbbe7f4a2,0x9d1faddd),LL(0x2222c720,0x0091add4),L_(0x5e600780), + LL(0x6e5923eb,0xe0a35b29),LL(0x5acac230,0x1ca3d3f2),LL(0x5089506f,0x57de9730),L_(0xf05a525b), LL(0xc963a675,0x8c76bd08),LL(0x74d9f43b,0xdeecdd8c),LL(0xdca2c05b,0x4bcbac2f),L_(0xdc6633b9), + LL(0x60601ff6,0x1bc30bde),LL(0xa7ddd558,0xcac4e4e0),LL(0xe21dd977,0x66baebc4),L_(0xb4d573cb), LL(0x79ee6d3c,0x352d2cee),LL(0x3c8cf82d,0xe3db9857),LL(0x12474a96,0x8d29a0c1),L_(0x55049164), + LL(0x3ff1fac3,0x8806d22d),LL(0x827b3141,0xe5b92d9d),LL(0xc0bcac45,0x45f1bee6),L_(0xe2c1046e), LL(0x368d7809,0x0b973abe),LL(0xc60d4681,0x6f45c847),LL(0x05dd04f4,0x1ae3e5bf),L_(0x64bbe857), +}, +/* digit=10 base_pwr=2^70 */ +{ + LL(0xad2244ee,0xa40697e3),LL(0x6dcf122e,0x379dd18c),LL(0x179d681f,0x177fcb32),L_(0x2e1c6f2e), LL(0xd9f2b14a,0xdfe5e464),LL(0x4c70a028,0xfcfc8017),LL(0x6183ea61,0x4d64db2b),L_(0xd053bce3), + LL(0xa9c848c0,0xe6f3cecd),LL(0x3c37feb1,0x1953cde4),LL(0x857f0263,0xbbc0b11e),L_(0x091b8ddb), LL(0x46acc3ea,0x8541cfa6),LL(0x772c3ab4,0x7c142776),LL(0x71e71be0,0xcd2d8bee),L_(0xd29d11b6), + LL(0xc6b24ae6,0x4b9ffc6a),LL(0x8ee5427c,0x4a42092a),LL(0x997ab138,0xff6b61a7),L_(0x3b574c18), LL(0xd6cb4977,0xbeade6c6),LL(0x315f47ce,0x2e2d5dc5),LL(0xc77bac27,0x39f2d0a5),L_(0x28b192db), + LL(0xa902a170,0xdfa7909e),LL(0x374a8799,0x93925c6a),LL(0x121579dd,0xb4c9c6a6),L_(0x2b2a2fe1), LL(0x421343f7,0x2dec18b9),LL(0x467e925a,0x4a7ce3d6),LL(0x51b39839,0x33d05082),L_(0x6dd45eec), + LL(0xaf22f4e9,0xe4793da0),LL(0xce68ae65,0x6d70ff7a),LL(0xdf1eb924,0x67519764),L_(0xa6d12a17), LL(0x9c9b86f4,0x1e13ad01),LL(0x43f0840b,0xebcc7b33),LL(0x648dece5,0x0b8a73e1),L_(0x9e1099c5), + LL(0x3ee5fb2f,0x00e5b507),LL(0x16e7b26d,0x02eb10ab),LL(0x2fb5b17d,0xf5f94483),L_(0x964b96f7), LL(0xb10c75ea,0xb577452b),LL(0x8083550f,0xcf241d80),LL(0x2f106bf6,0xe11783de),L_(0x0cfd54c0), + LL(0x598729b4,0xd3bf6d02),LL(0x6afe9c51,0xfb3e35f6),LL(0xbb0f1cd8,0xb0816531),L_(0x27df4a0c), LL(0x2f8214c2,0x72f5b39b),LL(0xc24bf200,0xdd44ad34),LL(0x39525315,0x2d46495a),L_(0x758c33c9), + LL(0x6fa7c2c7,0x9c7bf299),LL(0x2cb77bd8,0x4a4734a4),LL(0xb7245ac9,0x0a33b879),L_(0xe50ef27d), LL(0x3e4bd3ce,0xb91f6979),LL(0x1ca8036c,0xf8b8d90d),LL(0xbee50919,0xf053ecd4),L_(0xece7c952), + LL(0xe88a9300,0x990b1bdd),LL(0x0420911c,0x516f881a),LL(0x4138b100,0xafe94d6f),L_(0x24ff9ef4), LL(0x4c68e2d4,0x528a709b),LL(0xa50cb647,0x19ff7b20),LL(0xbb306765,0x907cbbc6),L_(0x74217b89), + LL(0x0a371fe5,0x0bab31b6),LL(0xcccd2b33,0x69aaf848),LL(0x5f8e6b05,0xb7efd33b),L_(0x4cd501f3), LL(0x117abd54,0x83d20ea2),LL(0xf475121c,0xa34d2ec5),LL(0x47427d11,0x6eb988c6),L_(0x7acc59a0), + LL(0x96650b0d,0x591ee335),LL(0x16ee1f0b,0x26a19153),LL(0x27985f50,0x7a51d8bb),L_(0xfe74d9ab), LL(0x276b185c,0xaa92cd9d),LL(0x9ed60020,0xb66c17eb),LL(0x2a0eac87,0x18315901),L_(0x8dbf60b9), + LL(0xe73f9dd1,0xaaf7ef5f),LL(0x3d3b7078,0x315302f9),LL(0xf09fd9b4,0x00b8f278),L_(0xf7dd85ca), LL(0x4140a60e,0x54e7df9d),LL(0xe305da30,0xaef051a2),LL(0x67dd41ab,0x88c0e226),L_(0xc437eba8), + LL(0xaecd7f62,0x938be7f6),LL(0x7f0d8c8a,0xe465c032),LL(0x74f9489c,0xba77e9c0),L_(0x529b3458), LL(0xbea77798,0xfea9f3ec),LL(0x93d7e4fe,0x6ec5677f),LL(0xce0cdb8b,0x1c3846b0),L_(0xe0a036d5), + LL(0xa818d0c9,0x3b25f1dc),LL(0x640f0895,0xf998244a),LL(0x6a36628c,0x0a1631ee),L_(0x22c92416), LL(0x79522c82,0xb5997002),LL(0xa3c8067e,0xbaf242be),LL(0xe28f2eda,0xca50379b),L_(0x0c2d44ef), + LL(0x7e8a5165,0x1879da9a),LL(0xf1be18cc,0x4b28b798),LL(0x5708ec80,0x088da37b),L_(0xa743ca6b), LL(0xa1588631,0xd86f2e95),LL(0x9bf8faf1,0x97dcfd6d),LL(0xae7389fc,0x784c1096),L_(0x1aa41e81), + LL(0xf9a13e9f,0x2380678e),LL(0xe93e1c94,0x3f5ce533),LL(0x68003338,0xa4954b13),L_(0xfa7015a9), LL(0x63905ece,0x3e892821),LL(0x19b8125b,0x4b7bb992),LL(0xd66a6ac0,0x8579e7ef),L_(0x7516c1ae), + LL(0x21560c47,0x8051f521),LL(0xfd3b4b27,0xa4371e4f),LL(0x8947385a,0x8c0cdac3),L_(0xd7eaeef0), LL(0xc46bf169,0x0639dd0c),LL(0x862785aa,0x9ec7e487),LL(0xd61bc685,0xd861b3a5),L_(0x83d3ee5e), + LL(0x930059f4,0xb145ba07),LL(0x2aaf9072,0x30f8ae06),LL(0x2f3077a1,0xbc426169),L_(0xc99c4c21), LL(0x6c57976c,0x6ee68683),LL(0x236b0866,0xb9cd4e70),LL(0xe6b22138,0xaa9f8c57),L_(0xc8cde662), + LL(0xed492587,0xcedbdd1a),LL(0x51eec28a,0xe84b42a7),LL(0xd4deae6d,0x9e368887),L_(0xaded26ea), LL(0x1bd18dec,0xf79fe229),LL(0xa610dbd9,0x0d650fde),LL(0xc8e53036,0x8d4db547),L_(0x6d7b9831), + LL(0xb55dd50b,0x6d88e9da),LL(0xaf929b4d,0xbb1af641),LL(0x09f3ff8a,0x2c88cea6),L_(0x95aa1478), LL(0xf6e2f2aa,0x4d819416),LL(0xe1f9b46c,0xce813081),LL(0x3fdecbde,0x368fe183),L_(0xce1f9cb4), + LL(0x55d27a4e,0xb55224a5),LL(0xc92c61ce,0xbd06485e),LL(0xe42fc5df,0xbae2229f),L_(0x3afc274c), LL(0xa273191e,0x85cd8bfc),LL(0x97118ed0,0x9c9c61a0),LL(0x5e361376,0xaaf47545),L_(0xcad4d556), + LL(0xfb2444ed,0x18d8ff5c),LL(0x18b51ba2,0x3dc74bfd),LL(0xaf0eee18,0xfc1b2868),L_(0xbb73ea5a), LL(0xd68ff57a,0x88d94a2f),LL(0x730a7041,0x162bcf15),LL(0xd183ab03,0x9867c6b2),L_(0xf198711f), + LL(0x817437d0,0xf225ca3a),LL(0xebadfce9,0x2afa4968),LL(0x7b328600,0x09069b77),L_(0x1091afcf), LL(0x82fb3189,0xdcc4bc5d),LL(0xf1844abf,0x49ace070),LL(0xf655f51c,0x7e393926),L_(0xa022dcd5), + LL(0x13b3c6d8,0x32d143d6),LL(0x07002476,0x37eefa5a),LL(0x028e4eac,0xf7ed9d04),L_(0x65e3cb55), LL(0x22c86f9d,0x1cab2511),LL(0xdf940bc6,0x50c1e3f3),LL(0x734bf045,0x2643f52d),L_(0x11593f75), + LL(0xeaec70bc,0x1759457b),LL(0xcee75815,0x1ef41bc8),LL(0xbfd1d973,0x5f64050d),L_(0x6b3087eb), LL(0xda22e615,0xc045345f),LL(0x776dfb51,0x3f9aff61),LL(0x07f4f3a2,0xb17003a3),L_(0xb6a6cf4d), + LL(0x9276438a,0x53f04642),LL(0x8525ef52,0x1397f8d6),LL(0x9a0888e4,0xde6f955e),L_(0xda028b89), LL(0x4292050e,0x2711d946),LL(0xd109e67e,0x70541c9f),LL(0x19b5f6f0,0x75754b8c),L_(0x85c8d6d5), + LL(0x3073bee0,0x0a0de3cd),LL(0x725322a6,0x3ecd54ab),LL(0xabf6f3c4,0xeb4d3502),L_(0x6dfedcca), LL(0x12c56dbf,0x08adcae7),LL(0x33d4bd69,0x0b5e164a),LL(0xcfe1b693,0x659e68b8),L_(0x82c86924), + LL(0xc347067e,0xd351d4d7),LL(0x141df0b6,0x5fe38868),LL(0xaaee8341,0x6a97a114),L_(0xaafe4f6c), LL(0xc0912e86,0x3ce36b09),LL(0x4106201e,0xd6174b63),LL(0x40106385,0xfda3c463),L_(0x9e471096), + LL(0x21aa62bd,0x15f6014c),LL(0xb5217d60,0xf4c26c02),LL(0x421c3d60,0xcc23ecec),L_(0xc0f18f53), LL(0x08c04fe6,0x1ae450ba),LL(0xf66ae880,0x7520566d),LL(0x2e31cf5b,0x1b3a8046),L_(0x916c70ac), + LL(0x47d17a16,0xd287d1ca),LL(0x3e47dc5d,0x291a00ca),LL(0x42cffa9b,0xc6ddb9b7),L_(0xb751c3c5), LL(0x6b576fb2,0x7f3bfdc3),LL(0x8676c057,0x05c5c887),LL(0x2f6a12cc,0x4e43cafd),L_(0x273385a8), + LL(0xae17a9c0,0x1d472cf8),LL(0xb53ac163,0x1569367f),LL(0xf1c57e24,0x5f7dfde3),L_(0x53f9522b), LL(0x45eaeed1,0xf3d29713),LL(0x3c47cbaf,0x7a261ef7),LL(0x561e0755,0xbe952c4a),L_(0x6bcd178c), + LL(0x63cc0fbd,0x24b41db6),LL(0x36a421ac,0x34bd0f29),LL(0xdc178877,0x619cb9a4),L_(0xa266089e), LL(0x7b84d003,0x9fd8767d),LL(0xba9a6d1f,0xa5381512),LL(0x8cbdf583,0x46c198f4),L_(0x23f49a00), + LL(0x65bc3562,0x281cc4e9),LL(0xc3d2804e,0x7562f859),LL(0x22a16679,0x638a7949),L_(0x7ec2aa7b), LL(0xe6c7330d,0x187243a5),LL(0xb99ca6d8,0xee353b2a),LL(0x2d4040d0,0xbd604fb9),L_(0x8cefbced), + LL(0x1fc30151,0xfbd9e1c4),LL(0xd10937a9,0x0f7ffd5e),LL(0x2fa67094,0xe7b103db),L_(0xb57b1840), LL(0x39530ea9,0x1368261d),LL(0x719e1d06,0xc1336cad),LL(0x192c11f5,0xa6cd197f),L_(0x9b41db84), + LL(0x37505def,0xd68a5a49),LL(0xddd06586,0xb0511d2a),LL(0x52664a68,0x2621ebc3),L_(0x6f5736f5), LL(0x2fcaf173,0xd267f258),LL(0xbacb7a85,0xe5e678b7),LL(0x5c0b0c67,0xaa7c18e3),L_(0x166ad375), + LL(0x669558fe,0x9a21bdfe),LL(0x8411fa8b,0x06c5b604),LL(0xde70b24d,0xf5bd9bc8),L_(0xd7443ed1), LL(0x0a439533,0x2b84584e),LL(0x3ea3f8f2,0x2cf8fe06),LL(0x099fde13,0xbc0565bd),L_(0x8fd32c81), + LL(0xc291b778,0x7e78ce16),LL(0x12d214f6,0x6d2b8e47),LL(0x5f204c64,0xaf966aac),L_(0x62d03da4), LL(0xe9be215a,0x0fc086e5),LL(0x14494699,0x77fd96e7),LL(0xcc71f940,0xec60a687),L_(0x65805046), + LL(0x4b6c94b4,0x1ee6996b),LL(0x8eb81a1a,0x5a56077f),LL(0x547ea701,0x71308d2f),L_(0x9126c06e), LL(0x9fdcaccb,0x5fecdc40),LL(0x49cacc43,0xd04d1212),LL(0x89bc7a7f,0x6331b109),L_(0x93db1b8f), + LL(0x2b6470fb,0x18292505),LL(0x93a8f9cc,0xe0d6f3f0),LL(0x8d658c23,0x33eef875),L_(0xc22c9665), LL(0xb09b3b96,0x2081f85d),LL(0xbea17435,0x11959ada),LL(0x6f44daf5,0x5ff2900d),L_(0xdea368bd), + LL(0x224521dd,0xfcb125d5),LL(0x097e443e,0x7d3c0cbb),LL(0x96ff5378,0x0cce254b),L_(0x562b2166), LL(0xc45e09f3,0x73e36236),LL(0x992060e2,0xf844b9b5),LL(0xa968d30e,0x139e42f9),L_(0xe5b89488), + LL(0xb57b80af,0x3e86ab5b),LL(0xb177e3f9,0x8e7a956e),LL(0x95050abc,0x580bb577),L_(0x75270515), LL(0xa96f2da7,0x0ce33d22),LL(0xcab8647e,0xf5e4958e),LL(0x139f525e,0x9b1f28f4),L_(0xa2e38250), + LL(0xf097a8be,0x193d7b50),LL(0xbe57f373,0x326744fe),LL(0x192cee98,0x66699aba),L_(0x69e0895a), LL(0x51e1d17f,0xe787c995),LL(0xb3f66818,0x7acab4f1),LL(0x5ee5c7cd,0xe1645254),L_(0xb7d10719), + LL(0x29d8986c,0x17022e01),LL(0x2f237075,0x0ccc4d78),LL(0x01453fbb,0xba3615f3),L_(0xa703f343), LL(0xf538646c,0xb5aab8cf),LL(0x30b0a7e8,0x4a42c722),LL(0x33588983,0x8a8547cc),L_(0x64a4028c), + LL(0x82ecbbdc,0xbbdec907),LL(0x6c0cb953,0x1807ae9c),LL(0xf8e40104,0xca5b89e3),L_(0x5e56c8f6), LL(0x41f32d88,0x040b1ce4),LL(0x76bd7772,0xbd9bda0d),LL(0x396f023e,0x6bb455f6),L_(0x4243e710), + LL(0x82b34b90,0xff00cfce),LL(0x7a3a34e7,0xfea1370a),LL(0x59745afe,0x13dec9bb),L_(0x00460ef4), LL(0xc0ccb5ce,0x664b5767),LL(0x467ad23b,0xe91ea77c),LL(0xd6ac872c,0xc30f32ed),L_(0x7527a687), + LL(0x4a73f787,0xf2877171),LL(0xa577b4e7,0x05b283db),LL(0x7ac6ab0b,0xbbb22aab),L_(0xce8c4374), LL(0x2cf4237f,0x5d2444bb),LL(0x1362bb86,0x354270cf),LL(0xa58498c9,0x03a3218e),L_(0x4585f77c), + LL(0x17d3d4da,0xe1c0eb99),LL(0xf69271e6,0xde536a3e),LL(0x239476a2,0x3701952a),L_(0xb94d2063), LL(0x25bb5fc2,0x39ed615a),LL(0x44f390c3,0x6a9dd585),LL(0xa4418ea6,0x1ce9df6c),L_(0x76cd5c55), + LL(0xa10e49d6,0xf5dcc900),LL(0x62fd7874,0x2e74ef98),LL(0x74f61b6c,0x9d3a5873),L_(0x10ca6b61), LL(0x470595b7,0x848244dc),LL(0x81922f08,0xf7db6c87),LL(0xbc236621,0xb2a50ed6),L_(0x0cbc779a), + LL(0x274cea83,0x4914c419),LL(0xb8a19540,0x1bc1a1ba),LL(0x957adbf1,0xd2aac8dc),L_(0x64d6ca82), LL(0x96ac9d7f,0xdb015eca),LL(0x42372196,0x03b15d50),LL(0x05451b09,0xe16c7207),L_(0x463bc66d), + LL(0x01c4dfe9,0xfc7e965f),LL(0xb8358066,0x8fc594e9),LL(0xa1915c8c,0x8d26603b),L_(0xfdddd6dc), LL(0xc4b1e185,0x2166a1d2),LL(0x79912871,0xad0b86ad),LL(0xe96aa265,0x98a560dc),L_(0x43b8858b), + LL(0xa1e7a822,0x2d3f3785),LL(0x95813382,0xcc85095a),LL(0x89e2322f,0x3568d961),L_(0xfbe11c03), LL(0xe15474b7,0xb0011b61),LL(0xb6d17af8,0x94e2b66a),LL(0xfa35ee62,0x88f30207),L_(0x496e0cfb), + LL(0x00a8605a,0x233a4ea2),LL(0xed225296,0xfb49bd9b),LL(0x18a0b19f,0x49f70f9b),L_(0xfd4a3974), LL(0x3d19e5f7,0x6935d1e0),LL(0x4ca49b02,0x167ee2db),LL(0x465d1799,0x83dab70c),L_(0x3925587b), + LL(0x06b4ef1a,0xadc74707),LL(0x97b00e28,0x7b776a0d),LL(0x4ea78215,0x29de362e),L_(0xe0075db8), LL(0xd1a8437d,0xcd9ea162),LL(0x4071fbdd,0xd6f65966),LL(0x5861958d,0xb0f23643),L_(0xfb6d160c), + LL(0xe1dbc405,0x1cf01490),LL(0xfd9abd17,0x781604fe),LL(0xfdabdb45,0xfe106ce3),L_(0x28aec19b), LL(0x0765d40f,0x5dea7b0e),LL(0x9dd7eace,0x05e88870),LL(0x7aa326d2,0x58144fba),L_(0x0c0b55eb), + LL(0x4b6bbeb3,0xf492b1fe),LL(0x80211303,0x00d1a0a3),LL(0xa0a42f81,0x812c4f34),L_(0x0513dab5), LL(0x701129ed,0x2b196b3d),LL(0x8292825d,0x14497eea),LL(0xfccdef2f,0x8309d41f),L_(0xa6c098eb), + LL(0x4e01759a,0xe5e506e5),LL(0x34984e59,0x5bdb0568),LL(0xc78ed701,0x023cf8a4),L_(0x8818f370), LL(0xadc05868,0xedb22c00),LL(0x61bbfcfe,0x1eb0f72a),LL(0x4b1c28bf,0x87de5c20),L_(0x13f66db8), + LL(0x2c5c67ae,0x311b814c),LL(0x18478ee8,0x98f391ee),LL(0x5fc08d55,0x3d6c85b2),L_(0x2a4234ef), LL(0xc63b3540,0xef0ad114),LL(0xfa0e9b48,0x19926812),LL(0xded54ff1,0x8d2d8068),L_(0xe6033e94), + LL(0xfcdb3c46,0x567ac24a),LL(0x31e079f2,0x5c7d3b6c),LL(0x876a06e1,0xd77d6297),L_(0xabca81b2), LL(0xcd9c326f,0x27382ebd),LL(0xbd7f7ff0,0x8c9f6089),LL(0xb11d9de0,0x31e36d28),L_(0xba48fe41), + LL(0x8c0ffdf0,0x845700c1),LL(0x659750c3,0x9a4535d8),LL(0xbfa2263b,0x46779254),L_(0x426c9332), LL(0xea64e4f2,0xc0627926),LL(0x919159e1,0xd72486ed),LL(0xb8633c7d,0x62049111),L_(0x57a02e38), + LL(0x6d5a56c1,0xc0ef9c32),LL(0x87339367,0x1398d5b9),LL(0xd690f5fe,0x4cb8372b),L_(0x74004a4c), LL(0xa8b7608c,0x62fc7f7e),LL(0xc75eca95,0x7fcc6560),LL(0x87991514,0x1a3939cc),L_(0x9eeab34e), + LL(0xee335372,0x6dda426f),LL(0xe95ee7c3,0x7a84b92e),LL(0x5637d166,0x0cc0434e),L_(0x108326a5), LL(0x4c69a1c3,0x66908f8e),LL(0x768a4dc5,0x93e7c968),LL(0x24086621,0x0390f0a6),L_(0x34cd95fd), + LL(0xffd9518c,0xa579c74a),LL(0xf4d1cd8b,0x245e5d80),LL(0xa682fc6c,0xcf5b3a95),L_(0x13389ce6), LL(0x8944bc8c,0xc24a47b4),LL(0xf37dbb35,0x67296189),LL(0x9d47aaf7,0x196773ed),L_(0xab42affb), + LL(0x6c1fae2e,0x78457340),LL(0x0583dc84,0xea2de616),LL(0x6a0c10e4,0xbf46a7bb),L_(0xe6adebac), LL(0xb7cad754,0x3e924b67),LL(0xbf5e7329,0x93303eef),LL(0x9cb33af1,0x5460942b),L_(0x1005e77d), + LL(0x5dd3bfdc,0xa399de31),LL(0x867305ce,0xbc0f8f54),LL(0x2200477c,0xd31a5a81),L_(0x9ac2ed5d), LL(0x59ceee51,0xa5a011eb),LL(0x9cb94041,0x53fce407),LL(0xf7a7e8fe,0x26131e4c),L_(0xbcae3bd4), +}, +/* digit=11 base_pwr=2^77 */ +{ + LL(0xe3b818f5,0xe00bddec),LL(0x44e43c61,0x4681ea2b),LL(0xc584d310,0xb6fe7f05),L_(0x951b39e7), LL(0x81eb4c72,0x1eae5fc5),LL(0x518f5441,0xdae90140),LL(0xc505aeda,0x216fb5f2),L_(0xbb781e43), + LL(0x3cf8ec7d,0xf9a62c2a),LL(0xe0d93869,0x23fabd24),LL(0x6d3f0de8,0xb3106bbb),L_(0xb0f77641), LL(0x6c7c7f7d,0x9b298ebb),LL(0x184aed09,0x9f30709a),LL(0x334394d4,0x79c1d22f),L_(0xd95473d2), + LL(0xae76cd6a,0xcb7cb589),LL(0xf3fc070e,0x3ecf8591),LL(0x60dba7d6,0xe19d60f2),L_(0xe420c869), LL(0x59b7cb97,0x097e4daa),LL(0x707e030c,0x7f52c8de),LL(0x0dcda4fd,0x9803c9fe),L_(0x2bbc44ce), + LL(0x2ff400c0,0xfabcb249),LL(0x10355326,0xacc122ab),LL(0x91ed19cf,0x4f316ca1),L_(0xd20efc4e), LL(0x940a18e1,0x1a2e490c),LL(0x2b83ac01,0x029cb6e5),LL(0x0fd81ad0,0x8da694f8),L_(0xfcad677f), + LL(0xb1b32829,0x248ec9b3),LL(0x11ef88d2,0x35de7b93),LL(0xda678713,0x370ca0ea),L_(0x7a293625), LL(0x1a056e86,0xddfa0a4d),LL(0x1f856294,0xfe05d52e),LL(0x6bd2e637,0xf728ccb4),L_(0x91baf444), + LL(0x74014196,0x9e4adc25),LL(0x8981411a,0xf964b6c4),LL(0x7eb4dd2d,0x58e82e86),L_(0x97c312ce), LL(0x153d2149,0xd7b84e08),LL(0xc95c8ff2,0x4e547787),LL(0xd7ac3655,0x7546fc40),L_(0x504a9f59), + LL(0x3f1e5f11,0x9edb0425),LL(0x476b8958,0x63c34818),LL(0xf14565b6,0x4bab1b96),L_(0x3d3ca262), LL(0x18ef180c,0xfd4e9a47),LL(0x0043c8ff,0xa1174dbb),LL(0xafaf8911,0x0b1e0a8d),L_(0xcbeb2d8e), + LL(0x486cbfa7,0xebbefd50),LL(0x44fc634e,0xe20aab71),LL(0x9ddeace8,0x08a9107e),L_(0x909eff4c), LL(0x746a522f,0xf816fa9e),LL(0xefd70514,0x864324da),LL(0x598db9df,0x0094b64c),L_(0xc6634c12), + LL(0xe65d8204,0xd6fb5d4b),LL(0xadb1a9c2,0xec38f10c),LL(0xc8ec0f39,0x1908657d),L_(0xa25710b3), LL(0x87e1d90e,0x311947c3),LL(0xe187cb1c,0x04d9654b),LL(0x1bae3f87,0xafb6e7b5),L_(0x44ea2977), + LL(0xe4a3b821,0xee18a382),LL(0x60b120f0,0x4002bc41),LL(0x91636563,0x29fd66f4),L_(0x6538f012), LL(0x300af347,0xda52b57a),LL(0x9a0bd35b,0xa08ae1b7),LL(0xcee45c95,0xafb1fcae),L_(0x042232d0), + LL(0xe06efe03,0x54397157),LL(0xd06ed8f6,0xf4ce3dfb),LL(0x67644c3e,0xaa8e6af8),L_(0xa26425e0), LL(0xbbd40e3b,0x6fb8009d),LL(0xdfa5a71f,0xbf5753ee),LL(0xc4fb8d84,0x413c2883),L_(0xa71e26d8), + LL(0xe358be61,0xae3de433),LL(0x45610301,0x9f32cd87),LL(0x4ec1365e,0xb95b637d),L_(0x41dbce62), LL(0x0840486b,0x9637a66e),LL(0xc6d80952,0x522810a8),LL(0x928915a0,0x761cc88a),L_(0x1da0d501), + LL(0x208620d1,0x37a59562),LL(0xb4ca54d1,0xc246b32e),LL(0x5128864c,0x8aad87e4),L_(0x559dacdc), LL(0x9d93608a,0x89718ac7),LL(0xea353321,0x48d40286),LL(0x2012406b,0xc599f213),L_(0x1eb2fe94), + LL(0x6807d450,0xba62fad2),LL(0xa7fededf,0x07ea62b0),LL(0xd58195f9,0xd3d6b5bf),L_(0x1ad1505f), LL(0x87583b1c,0x6f93c77a),LL(0xaf96dc4e,0xa959c022),LL(0x20624dad,0x4fa52512),L_(0x3481aabe), + LL(0x9310eef9,0xba1c1390),LL(0x9fb8e966,0xcce0400b),LL(0xb9c0dea0,0xd59d1993),L_(0xe8a99c53), LL(0x243bcc60,0xb44f7477),LL(0x97335f28,0x1bd266f4),LL(0x6f550c82,0x3749bb24),L_(0x67501a21), + LL(0x35f741c9,0x2ddbdad6),LL(0x5e95c8ee,0xbeac98ef),LL(0x74fa6bf8,0xf68aac5d),L_(0xe404df62), LL(0x13d824a6,0xf3252763),LL(0x934f4963,0x4c5b99db),LL(0x65ea023c,0x39f3ad81),L_(0xf2ab45c6), + LL(0x513ad1f3,0x6c9e60dd),LL(0xd82ebdb2,0xeb595e39),LL(0xf4278e99,0xf76b6d5d),L_(0x177bfa74), LL(0xda5b699a,0x0698c8f7),LL(0x8af11d92,0x022b9e4d),LL(0xdcfeb87d,0xdf966b87),L_(0x790b8b3b), + LL(0xe6576cdc,0x0d632c2d),LL(0x82b42706,0xe4079724),LL(0xeb7e23be,0x67d13334),L_(0x6bef8f5e), LL(0xab6f7a84,0x5b9d7907),LL(0xb943b3e3,0x3b2e4de9),LL(0xe1142403,0xe61e180e),L_(0x3186f5ab), + LL(0x54c38bc7,0xee73f3c8),LL(0x1c021526,0xcdd71fb7),LL(0xd5f0330f,0xbcd8ae19),L_(0x34f15f5c), LL(0x3f1e795b,0x0f2d33f8),LL(0xf2452d93,0xe4d47a55),LL(0x0c4b0f70,0x322c677d),L_(0x311431b5), + LL(0xe32f7152,0xd222f30a),LL(0x32c8e92e,0xd5265ce0),LL(0xf9051d2c,0x34079f7e),L_(0xca51b73e), LL(0x752d0df7,0xe4529fd5),LL(0x3ee05e30,0xd39fd02a),LL(0x762211ba,0x957b54a8),L_(0x7bd842d1), + LL(0xca5c5b9b,0x8578a05a),LL(0xf931c6ff,0x5854ceb6),LL(0x9c1a24ae,0xc7ad186b),L_(0x3aae7e80), LL(0x405ad033,0xcd94ffe8),LL(0x02d60546,0xc24532db),LL(0x6d8dfc51,0xf70532cd),L_(0xa7523214), + LL(0xf6e5bcb6,0x3898ac49),LL(0x7ef15aa2,0x0dc0b7cb),LL(0xc92d3e61,0x6f146470),L_(0x20a65e93), LL(0xa99d1e34,0x40c2ff2e),LL(0x5f5f4b1f,0x8bce3569),LL(0xef24d37f,0xf5eb8c60),L_(0xd0f3625a), + LL(0x770ff147,0x04af9657),LL(0x0ddb7f8a,0x5d6a9432),LL(0xf5679a0e,0xfa164f3a),L_(0x75f9e25f), LL(0xb341ef73,0xd2b96f53),LL(0xc75b8bfb,0x111da06d),LL(0x141dae1e,0xf86c989c),L_(0x1a015ae9), + LL(0xd89c49d6,0x7440165b),LL(0x252d3538,0xbc50c381),LL(0xc7935e6e,0xfc485299),L_(0xafc2b1d4), LL(0x7b5b7847,0x8b5606a8),LL(0x8714fa4e,0xc8ca9e3a),LL(0xd015855f,0x03679b60),L_(0x14d9de29), + LL(0x5d13fc8a,0x7fd6372a),LL(0x1fc76dfc,0xd3c20f3f),LL(0x835ac98b,0xec54c6b2),L_(0x98cbc171), LL(0xe5e2a3e4,0x7ab5c98a),LL(0x577a8e9f,0x6c990ebd),LL(0x84028fa5,0xed0d2362),L_(0xb28e8d67), + LL(0x277610eb,0xd8c85ac2),LL(0x8e20b175,0xc11f2394),LL(0xbc82df9f,0x763f7da0),L_(0xc32cc81d), LL(0x85f8381c,0xb02c89cd),LL(0xd305dbab,0x2a33f683),LL(0x7f0a99f0,0xd40823f0),L_(0xf958bb39), + LL(0x08d09d3c,0xee84ea33),LL(0xc5f043e5,0xe195e3e9),LL(0x13f7ae30,0x18438628),L_(0xb6ffaed5), LL(0x4012f22a,0x9cb2856d),LL(0x9a5c926d,0x6dd2b787),LL(0xad05632b,0x4d11a1b7),L_(0xf0f2c69d), + LL(0x8e9a6388,0x60b10c54),LL(0x79943f17,0x0474b959),LL(0x15056515,0x70dc2681),L_(0xc9a17b25), LL(0xb6a005e4,0x4d9c29fe),LL(0x5978aada,0x9f751987),LL(0x6e84fa9d,0xb01cfa96),L_(0xd9553cbf), + LL(0xe72b7959,0x14645ddf),LL(0x50569b84,0x6a35f872),LL(0x76814e1a,0x6931ce3c),L_(0xd1694dea), LL(0x9877446f,0x8fb4ec92),LL(0xcbd84ece,0xd0d64612),LL(0x70c2d8a8,0x6f1a63f5),L_(0x2ca687fd), + LL(0x73d59d85,0x66719258),LL(0x78b1b1f1,0x3528fe8e),LL(0x372aba7b,0xf72aecd5),L_(0x049424c7), LL(0x0a8225e9,0x1299651f),LL(0x7a3d9b21,0xcdfa1798),LL(0xeea3b43d,0x6296bed4),L_(0x965623a1), + LL(0xe79e4875,0x19faf24e),LL(0xee2a75e2,0xfce9de0f),LL(0x2608d8d7,0x5dbd9aa4),L_(0x8dd2d8bb), LL(0xa1d5ac1f,0x96024e09),LL(0x59608369,0x88c11a15),LL(0x3109b985,0x4c72f34b),L_(0x073affe7), + LL(0xa7d5999d,0x7163417c),LL(0xe75a8d8d,0x58690b9f),LL(0x34a71e84,0xf46ea0c1),L_(0xa472172f), LL(0xdb5e6f96,0x5941f31e),LL(0x45ef8849,0xff03761f),LL(0x813628f8,0xff6e6be2),L_(0x2f6a6923), + LL(0x37f746c4,0xf2dab7c0),LL(0xee4fac0c,0x94b2339d),LL(0x442b5427,0xd1572489),L_(0x1b63f7f9), LL(0x77b93d36,0x86beafa0),LL(0x931a009d,0x89006e01),LL(0xc54e5dcf,0xce454abb),L_(0xe3d31b67), + LL(0x5dbfaab4,0x2bfb9672),LL(0x00a46605,0x0f7940ce),LL(0xd0e900f4,0xac391030),L_(0x56b396f4), LL(0xd65db7d6,0x111e6518),LL(0x0e375257,0x574c6063),LL(0x0569eb0d,0x6386031f),L_(0xa43939dc), + LL(0xe79fa54d,0x790064ff),LL(0x19bae3a4,0x040699eb),LL(0x278b676c,0x2bfc1210),L_(0x5fd221ef), LL(0xc41696b3,0xaddaabbc),LL(0x3e58e744,0x4ef37496),LL(0x86211a5e,0x27a20439),L_(0x06b96609), + LL(0xa9a97859,0x31e7bef8),LL(0x6b521dbf,0x39d7de6d),LL(0xbb74a0f3,0x41f4bd94),L_(0x79699c57), LL(0xfd6b25ba,0x6af2b090),LL(0xb9a903de,0x17aa75d4),LL(0x250d78d4,0x565e3c29),L_(0x674edb33), + LL(0x580583bb,0xfa502a79),LL(0x4f9ab886,0x9fc3a870),LL(0xb5e6f604,0xcd18776f),L_(0xf25cc6e4), LL(0xea80a2a0,0x9822aa1a),LL(0x2c1f0788,0x3a8b80d3),LL(0xb4be8a38,0x671f0157),L_(0x5b11123b), + LL(0x9b6b4d48,0x54914216),LL(0x1256ef4e,0xb5f866f9),LL(0x075b53ab,0x5ecb1889),L_(0x56e20726), LL(0x69455333,0x09f26254),LL(0xd571b91b,0x2b2fad5a),LL(0x328d62f6,0xde6fb9c6),L_(0xe065c610), + LL(0xfd0dd09b,0x45144de7),LL(0xdc428059,0x74b10792),LL(0x9aff2cca,0x9af32379),L_(0x7429380e), LL(0x49635e31,0xe3565603),LL(0xdea72af4,0xe2386988),LL(0x2bf6a43f,0x5a2add5c),L_(0x6b6ae6f7), + LL(0x13448c75,0x9ab52f76),LL(0x061d7ef8,0x4392015e),LL(0x010840be,0xae0d756d),L_(0x97508e7b), LL(0xd15d5084,0xadc78971),LL(0x4d5b80a4,0x7c713bfc),LL(0xf9aaef5c,0x256809bb),L_(0xa1ec2e50), + LL(0xfbb42651,0x74a6dd49),LL(0x0393f719,0x977434ad),LL(0x4e57160b,0xac7e00ee),L_(0xe124fa0e), LL(0x7a6f9567,0x2389bd87),LL(0x80b4129f,0xfb1db8c0),LL(0x0e8aaba3,0x1051debe),L_(0x4a190f97), + LL(0x477f6b19,0x2156959b),LL(0x37e2ab37,0x910a6757),LL(0xfe9b7158,0x540652e0),L_(0xd3215a27), LL(0x7976f02e,0xc0c597ef),LL(0xceb6dd56,0xe44ded06),LL(0x09436cb3,0x07e11a08),L_(0xc752116d), + LL(0xf6b2fb7d,0xe154e3b5),LL(0x3dd689de,0xc3b95a13),LL(0xecec9673,0xf4ebc67c),L_(0xdf67ffb0), LL(0x01c1c880,0x33ce85c6),LL(0x58ffaf1b,0x033751df),LL(0x741f05bf,0x2535c38a),L_(0xffe3294f), + LL(0x308eb02c,0x4c921d98),LL(0x1c787299,0x5bd1a752),LL(0xbf10f0e6,0x4f5e9e39),L_(0xdd4daa07), LL(0x18aefaaa,0x92fcbe07),LL(0xf29c8618,0x200b4b85),LL(0x84305705,0x8d7c9a43),L_(0x19694aa8), + LL(0x3a51e7ce,0x680c4324),LL(0xb1ac0982,0x0b339790),LL(0xa4032aca,0xdf576261),L_(0x92007d10), LL(0x50fc383f,0x755cfb3b),LL(0xb9ea853a,0x702d89aa),LL(0x28725f39,0xa32de41e),L_(0x8310cc56), + LL(0xe3293ae8,0x4319f044),LL(0x6a31f14c,0xd8080331),LL(0xf0a68d54,0xa79121b3),L_(0xf8ad5c75), LL(0x3c85f6ea,0x1aca6f0d),LL(0xd728ef75,0xb8a8d44f),LL(0x66ed37bf,0x244f6df5),L_(0x5253f731), + LL(0x40e52539,0x7e5a8bd2),LL(0x9d03a5af,0x2a775548),LL(0xa4eb71ab,0xa305f15d),L_(0x1dc7fb76), LL(0x68a7c637,0x20487a25),LL(0x6deaadda,0x8c18ae20),LL(0xe0f4999d,0x9bb84678),L_(0xc451bb12), + LL(0x27b6061c,0xc977cb30),LL(0xd39b851a,0x430b3684),LL(0xcd656ca9,0xa19898ab),L_(0xafd81332), LL(0xfa31c77d,0x2bb5e6b0),LL(0xc3157a16,0x60d12981),LL(0x8acd8d9d,0x69cbb158),L_(0x6d76f690), + LL(0xe9d8d3f2,0xc8ff4785),LL(0x2bca8c99,0x2e87c4ea),LL(0xa2caaf78,0xf6be10a7),L_(0x2100372c), LL(0x8cdcc69a,0x761e39c6),LL(0xdb53800d,0xd8745be4),LL(0xc599f6e5,0xc995d474),L_(0xcb2cc9b4), + LL(0x35b78d83,0x39687ef6),LL(0xe8d0ac03,0x91a4e746),LL(0x921b3cbc,0xc64c35ba),L_(0xa500e776), LL(0x4f3feb91,0xe79c45af),LL(0x60ad3863,0xdbd9485d),LL(0x0ce33aaf,0xaee36834),L_(0x4ce8962e), + LL(0x8023ad1d,0xb38c8506),LL(0xc359f6e1,0x0673f87d),LL(0x31837b1d,0x98af1b3a),L_(0xf716fe6e), LL(0x3024d432,0x3830cd5b),LL(0x52930fa5,0x60d8b1ad),LL(0x0ad62261,0x097dfa97),L_(0xebeb447d), + LL(0xf447ab0f,0xe409d4ec),LL(0xe80a1b86,0x58d6899f),LL(0x1aaed1d3,0x9eac7e33),L_(0x80bb831f), LL(0x2a9866ba,0x986f7e0b),LL(0x005a4dc6,0x43a73638),LL(0x52827ff0,0x293c91f5),L_(0xcdd9e93e), + LL(0x691e28ae,0x2c6558ef),LL(0xd0ac15f2,0xe56ad3e9),LL(0xa247b120,0x573e9688),L_(0xd0627f7e), LL(0x375dee4b,0x22887f9e),LL(0xd31e0ba7,0x7d20b431),LL(0x1eea7f55,0x4423561d),L_(0xf9581051), + LL(0xd4243ea6,0x42a54363),LL(0xeb2affe7,0x619c06c6),LL(0x972ce84c,0xaeca0047),L_(0x4d9efd23), LL(0x4e0c1d41,0x173fea08),LL(0x1d890ba5,0x4bd9d779),LL(0xfea0ad92,0x5f0b0f42),L_(0x5619484b), + LL(0xe4e12bd4,0x6cdbcee2),LL(0xc11ff4f3,0xfe8d06f2),LL(0xa260adc3,0x5bb4d1a2),L_(0x33e0dbb6), LL(0x27018d5f,0xcaa2ba14),LL(0x415b8922,0x74b9b719),LL(0xb1fbde1d,0xe7f63f33),L_(0xb014bfff), + LL(0xeb046732,0xab510c97),LL(0xee7b323c,0x9bf44d3f),LL(0x37854732,0x23d6cc1a),L_(0xdbbcc348), LL(0xf773172b,0x64d5f135),LL(0x6fb100cc,0xb0c07c01),LL(0xb27f28f5,0x980b1419),L_(0xca8834c2), + LL(0x01f6b795,0x1b1c75ca),LL(0x8c28e4a6,0x6f2aae0d),LL(0x2cbdb244,0x1341e16e),L_(0x5a726ab8), LL(0xd482dc14,0x32264865),LL(0x4f4a819e,0x1e1e75b0),LL(0xba655f22,0x37367c7d),L_(0xd157593e), + LL(0xaa3ab9fd,0x46d94f54),LL(0xc9397bab,0x3628a867),LL(0x65e56a09,0xf946f4e0),L_(0xa259e975), LL(0x372139d2,0xb5060783),LL(0xccc37d6a,0xeef39a01),LL(0xf709343a,0xfb056c2e),L_(0x89705d36), + LL(0xb9d0b061,0xeb841e0e),LL(0x7663ad0e,0x0dc12b25),LL(0xb1420c31,0x5792fb1a),L_(0xaebf5ced), LL(0xfb7eb601,0xeaecfdd5),LL(0xd5142858,0xc9d7159e),LL(0xe56ab53a,0xdbaff47d),L_(0xb56ba899), + LL(0x64ac917b,0xd69321cf),LL(0xa06234ff,0xe8fd532c),LL(0x98b8cc13,0x4be3eee6),L_(0xadbfb76c), LL(0xb07602f3,0x57f0cdf3),LL(0x01e45447,0xa4676560),LL(0xb307918b,0xa0a5d2a4),L_(0xf7e16814), + LL(0x87a7be06,0x6c92c96b),LL(0x865c8f6c,0x666baece),LL(0xf96358e6,0xc0b20165),L_(0x10dbd508), LL(0xc18e3e53,0x990a46d4),LL(0x1d1a7d70,0x0002a616),LL(0x6c2d572e,0xa87b8f93),L_(0x03a7ba98), + LL(0x8d933af2,0x6d9f3e8c),LL(0x2221d899,0x3a4a28c9),LL(0x7c844ad4,0x142035bd),L_(0x11c04771), LL(0x2de90904,0xf02e1ad1),LL(0x78ab7d85,0x9793442b),LL(0x301c5cee,0x1feac0e2),L_(0x95568955), + LL(0x48a5ccac,0xc5b0ee88),LL(0xf045ebf3,0x1ecaeea9),LL(0xc1ea77c4,0x4e10f859),L_(0xa44993af), LL(0xf6bfe532,0x38d373de),LL(0x5c460d85,0x9d4beaec),LL(0x2bdad16c,0x264105cb),L_(0xb8fc3b3e), + LL(0xd715e784,0x2edb771c),LL(0xd4ae16cd,0x5451781f),LL(0xc8055a0a,0xdee3f3e2),L_(0xc08d1ca2), LL(0x3a1c4bfb,0x4252a1b8),LL(0x6d687314,0xbca7011a),LL(0xba935b81,0x3b383652),L_(0x52c1b85d), +}, +/* digit=12 base_pwr=2^84 */ +{ + LL(0x793c7e62,0xb92b8aef),LL(0x6354d2c9,0xd358ac7c),LL(0x5a4d9e35,0x0381968a),L_(0x52d9f13a), LL(0x36ed6955,0xc63b9fe6),LL(0x3743142f,0x726db145),LL(0x6e186e53,0xaea91efc),L_(0x3de25aa0), + LL(0x4c1ff250,0xf8921757),LL(0xced4216d,0xb6db16c1),LL(0x28cf0cfe,0xd524c877),L_(0x03bdd0b5), LL(0xdfd0f98a,0xd91831dd),LL(0xd131c59b,0x980f4d7d),LL(0x47533185,0xf492c767),L_(0x63c30af1), + LL(0x0cc8e4dd,0x79a5ddcb),LL(0x495cdcc7,0x26e26606),LL(0x7f199f53,0x9d2ff41e),L_(0xbe613bf2), LL(0x36bd8348,0x967f0753),LL(0x76c36728,0xc4ce6462),LL(0x02af3c45,0xee5a166b),L_(0x0bda3025), + LL(0xff47da80,0x63718529),LL(0x473e976c,0xa699d496),LL(0x88aeb436,0x368ce839),L_(0x94aabe19), LL(0x0bf4b797,0xb19a3ec6),LL(0x7d50b7ff,0xcd07cc98),LL(0x81c2bf46,0x8c4eb78c),L_(0xb0a7cd1f), + LL(0x2ddf8350,0x56f0849c),LL(0x5739e59e,0x4be4bf01),LL(0x1fc6493a,0x8b91a3a3),L_(0xd401b24b), LL(0xe89ec9f3,0x9decd9c5),LL(0x17f178e9,0xe63f9369),LL(0xfa904473,0x7a09efc0),L_(0xca5c00a9), + LL(0xe3f640d4,0x6aca6f60),LL(0x4bda0113,0x574789e1),LL(0x12ccd894,0xb4db3fdf),L_(0xb39be04f), LL(0xf1e4603b,0xb4705565),LL(0xccbe5154,0xef713fb6),LL(0xd84fec44,0x11964906),L_(0x9ceede3f), + LL(0x0244c62d,0x13111bdb),LL(0x175d3f77,0x14dda4d9),LL(0xe0971342,0x21244b52),L_(0xd6fa5642), LL(0x2ee8560d,0x68da34f1),LL(0xb9933942,0x387164b3),LL(0xd7d7f791,0xee292b1d),L_(0x4382d48d), + LL(0x2a7dfaac,0xe877a208),LL(0xaf9b8432,0x77afbba8),LL(0x1257224a,0xbb09b923),L_(0x1501cefc), LL(0x38985dbe,0xe061421f),LL(0xaa863b6a,0x00e2d3d6),LL(0x2733782a,0x1a9bf0b7),L_(0x6912b5aa), + LL(0x70c5c0c6,0x38d01224),LL(0xa0259e70,0x84a934e4),LL(0x8e85d7ef,0xfd49df96),L_(0x6bb4a230), LL(0xa15b7d76,0x58b99e52),LL(0x3c2be4ba,0x2dc32c4c),LL(0xe49fcb40,0x4937a76d),L_(0xdc9b73dc), + LL(0x5ce8f858,0xa516c9ed),LL(0x41610138,0xd3f7221c),LL(0x71d5b83a,0x8795351f),L_(0x9a45e682), LL(0xef8c4cff,0x9edacc8d),LL(0x17bd9c0b,0x9166ced1),LL(0x871cc69c,0x57013f03),L_(0xb6b7b6e2), + LL(0xf3f01792,0x4855e4b1),LL(0x124c67e8,0x7556de49),LL(0xd4d7f50b,0x494ae118),L_(0xf16b9834), LL(0xab9ad587,0x3a6a8d30),LL(0x18d69af9,0xa950fac0),LL(0x7f8ed091,0x9daffa88),L_(0xba6e7c59), + LL(0x8f831a7c,0x011788f5),LL(0x9c64aa0c,0xf480aaea),LL(0xf896522c,0xcc8f1ea7),L_(0x772627b7), LL(0xe1227c86,0x72bb812a),LL(0x57f8a759,0x6e202e06),LL(0xc7e7a397,0x4c06faeb),L_(0xeea7a14b), + LL(0x46853977,0x0c456e84),LL(0xbacf3d29,0x94a82d15),LL(0x861ef909,0x17a88c53),L_(0x25d01a4d), LL(0xaa44d1ac,0x0b6e87fe),LL(0x066301cc,0xce900509),LL(0x51763eac,0xe4c44c3a),L_(0x243f5364), + LL(0xf02ccf3e,0x69fe2b3e),LL(0x623f7c06,0xa6634ac0),LL(0xa81b9914,0xba6dd8e5),L_(0xd33612e7), LL(0xcf35703c,0xc948497c),LL(0x7184b686,0x3725232b),LL(0xd5e74279,0xfa105e4b),L_(0x4bfe5bea), + LL(0xbbed4591,0x03d46274),LL(0xfcb464d3,0xc3c74b90),LL(0xdcb4ccb3,0xf6934477),L_(0xbdf563a1), LL(0xdae24867,0x7c07acc7),LL(0x4d4331eb,0x05fd6424),LL(0x768b2706,0x022d151f),L_(0xf65a4027), + LL(0xfff51bbb,0x5cbe2653),LL(0x32707706,0xbec1f2c5),LL(0x9739588e,0x1d1ff202),L_(0xe51c696a), LL(0xfa60c3f4,0xdd740417),LL(0x25fcc7e6,0x6a0e351e),LL(0xac2dfbc7,0x0d72b69e),L_(0x1ff6f6ad), + LL(0x3eaa7b81,0x3ade75e7),LL(0x3ac64888,0x4036d025),LL(0xd957b110,0x7be01ff5),L_(0x33f835d6), LL(0xc6af558a,0x45081b72),LL(0x0c08887d,0x137ffdce),LL(0x4fc01e35,0x84d416f7),L_(0x15716e5d), + LL(0xb47ca6c4,0xc4cafbc2),LL(0xcd118e18,0xac25eb7f),LL(0x78d7034b,0x4d198a7c),L_(0x9ba63f25), LL(0xdb8a8725,0x81c83f82),LL(0x9eb5e40b,0xdd6d729d),LL(0x3b1941e5,0x27b87b90),L_(0xb7229f73), + LL(0x492e405b,0x59afdea3),LL(0xf6e5451f,0x74cd3ea2),LL(0x3c7977fd,0x8262d80b),L_(0x0ff79532), LL(0xd5eea209,0x85d46815),LL(0x41a59faa,0xe5277e25),LL(0x815bfdff,0xf25c4d84),L_(0x26a648ca), + LL(0x1ce66149,0xc47be93e),LL(0x8807158f,0x150ece78),LL(0x71da23a8,0xf790aca9),L_(0x26957d5b), LL(0xeaf83d16,0x571af637),LL(0x3a3ff68a,0xbbf0d967),LL(0xb02eef0e,0x78555759),L_(0x1609b893), + LL(0x38ec85bb,0xa99f1706),LL(0x7a4aae59,0x12f3c66b),LL(0xc84493c2,0xf1b134ce),L_(0x8f2fb8ff), LL(0xbfc251dd,0x8f6f7a7f),LL(0x73cd2591,0x7b6bc242),LL(0xf68459a2,0xf4b9ddab),L_(0x1d8a7278), + LL(0x995f94c3,0xa1fd53d1),LL(0x3ad117de,0xae71c050),LL(0x75e31939,0x58a5dfbd),L_(0x3251c208), LL(0xda1a36d0,0x8436c37c),LL(0x65ab378a,0x80258188),LL(0x3a685733,0xe6172e78),L_(0xfa11b001), + LL(0x54abde7c,0x18e18996),LL(0x48c2dc2c,0x3b62d97b),LL(0x814d21da,0x198afe84),L_(0xf76ecf91), LL(0x0ca9c69a,0xb918572e),LL(0xf53020dd,0x249d565a),LL(0x365cb3e0,0xa6a9ebf4),L_(0xf8ad1ad6), + LL(0xb5e96d0f,0xf1859230),LL(0xcb1be872,0x64eca0e1),LL(0xfaa10d6a,0x11273c1d),L_(0x240336fa), LL(0xe56d43c3,0x6bfe9508),LL(0x88653660,0x12a7c212),LL(0x7b0d301c,0xd427be59),L_(0xbd85a9ef), + LL(0xbc70a5f7,0x72600cd7),LL(0xe9cae1c4,0xee97ffd5),LL(0x3d1457c8,0x96865fe9),L_(0x9a1bc1e7), LL(0x1518b86c,0x7727a4f5),LL(0x09a2a9db,0x1ed0a203),LL(0x028d0a99,0x9656e0c0),L_(0x7f3b01db), + LL(0xcb2d1081,0xb1f41608),LL(0x060c3752,0xf5804976),LL(0x6efb7250,0x0bd8bd31),L_(0x79016736), LL(0xc3ebc844,0x8739f62c),LL(0xf14247f9,0xb2896f39),LL(0x29b78fc9,0x5b2e893a),L_(0x5f2b1e7f), + LL(0x058470b5,0x380ddadd),LL(0x014c7c14,0xddf90df8),LL(0xc2d7837b,0x78897ca3),L_(0x5567dd34), LL(0xe3308b08,0x00642d55),LL(0xf735f752,0x9ca8b38a),LL(0x8afd8b87,0x6eb02b4e),L_(0x120f5319), + LL(0x2f0ecf3f,0x8fc31698),LL(0x41d511e3,0x8873639c),LL(0xbdc4fb65,0x9841bbd5),L_(0x207bfaea), LL(0x8e82da75,0x38fcb8d9),LL(0x6ba5dcee,0x05619b1f),LL(0x21f2d336,0xf78fd1c0),L_(0xc6f634fd), + LL(0x8258ddc6,0x09a676ca),LL(0x414f2daa,0x23041ee7),LL(0x4cbc9d7b,0x3b973302),L_(0xb409e95d), LL(0x7027e9ce,0xd16a5ca8),LL(0xf92a6554,0x473c41d1),LL(0x153e5025,0xa08470dd),L_(0xae4dfb2b), + LL(0xb57dcbad,0x5b3c44c3),LL(0x45031642,0x1130bf58),LL(0x335c400b,0xd4b606ca),L_(0x6b34a475), LL(0xbd2244f8,0xf3ab94fe),LL(0x746295e2,0x68723d0a),LL(0x4b9b612a,0x9d957566),L_(0x510e67f8), + LL(0x9a9a5ab2,0xdc56f264),LL(0x9f7dfe87,0x8565f7b7),LL(0xe3386448,0x104ea5b5),L_(0xac4c4feb), LL(0x5964e7d7,0xb2f8650c),LL(0x186e3514,0xf781aadb),LL(0x58c2701b,0x81927ca6),L_(0x81220e7e), + LL(0x6cffb688,0x727a95e9),LL(0xfb0e9702,0x83b28617),LL(0xcca5f38c,0xdb71dd75),L_(0xad31c5dd), LL(0xf33897ae,0x3ff02651),LL(0x683b0e57,0x380268df),LL(0xc6e32a5b,0x4ab25a51),L_(0x283f183b), + LL(0x48906979,0x08ea3767),LL(0xf7f76eae,0x744d1934),LL(0x222dd487,0x5b8927a0),L_(0xdc4a955f), LL(0xf36b1f09,0x605b6b77),LL(0xc80579f6,0xc4a4588d),LL(0x4cf36f45,0x94402fad),L_(0x0a226060), + LL(0xe0e79649,0x77db1b5d),LL(0x4fe36153,0xd0b5aff6),LL(0xb32ea8ef,0xa09373a7),L_(0xd4f68719), LL(0x116381ec,0xd8828433),LL(0x8c0026e9,0x7e2c0fab),LL(0x16af4c4a,0x8f7a154b),L_(0xd4672394), + LL(0x32df0ada,0xe432397e),LL(0xd0196172,0xd07b25cb),LL(0x114f01ee,0x12143827),L_(0x2f06e8aa), LL(0x9a3c110f,0x00bcfcbc),LL(0x76ee873b,0xb9f30c05),LL(0x8ff02da4,0x786da906),L_(0x7e7e6c65), + LL(0x0cb411eb,0xc8866c5b),LL(0x0d7b5118,0x5022b82e),LL(0xef5be83e,0xb37c41ce),L_(0x4b76c64d), LL(0x17ee0e9a,0x3d28da11),LL(0x8ef7cb6d,0xdd638761),LL(0xa3c8a2d5,0x4369465e),L_(0xc8c85ab8), + LL(0x5cda4d54,0x08818978),LL(0x17f1f556,0x62350317),LL(0x251cc26a,0x686c57b8),L_(0x865fb805), LL(0xe9b31ebe,0x19c7f60e),LL(0x23566eda,0x56fbedbe),LL(0x47f95f9f,0x867b619f),L_(0x1958dbaa), + LL(0x6427adad,0xfc7b0692),LL(0x44b01395,0x899725d1),LL(0x0d676d87,0x075607b2),L_(0xe2ae6225), LL(0xf9c3cc31,0x56d68f4a),LL(0x7e636075,0xb883986d),LL(0x1ffba2be,0x5c1ff02f),L_(0xc237448d), + LL(0x06c3092f,0xc3ea5575),LL(0x52c5f87e,0x2926437b),LL(0xf135c025,0x593535ee),L_(0xe0965b4b), LL(0x33672517,0x54fedc17),LL(0x28d4b11a,0x39103346),LL(0xe5297d9a,0xb0d741c2),L_(0x39caed90), + LL(0xf87698f2,0x340fdc0a),LL(0x1b8201fc,0x4ba2f34c),LL(0x248c333c,0x5f236490),L_(0x12ee4946), LL(0x52913c22,0xcbc6b04f),LL(0xee23385a,0xe9ac2163),LL(0x630f7175,0x0000016a),L_(0x3f6e71ce), + LL(0x953882bf,0xc6c3de30),LL(0xf99b211a,0x34fd7b9c),LL(0x549ffc42,0x5ffb8aa4),L_(0xbc4e1e52), LL(0x8cc9069f,0xe552f859),LL(0xf54af34b,0xe153c954),LL(0xd61618d2,0x5bb20166),L_(0x5776aea6), + LL(0x546f1f93,0x535ea11d),LL(0x565a5471,0x3aceac15),LL(0x4200ddef,0x8ee2aacd),L_(0x2493fe70), LL(0xd0228894,0x048f6036),LL(0xc2e41220,0x37388c77),LL(0x3193a2ea,0x3c56b2e4),L_(0xc4609ea4), + LL(0xcae3c391,0x4276f085),LL(0x57d5dc17,0x5fbdd403),LL(0xa8a6408a,0xd70d74f7),L_(0x98acac12), LL(0x75cbb8e7,0x93cd39a9),LL(0xb4c10685,0xe460014e),LL(0x56a7276e,0x2685e5f9),L_(0x3eeed659), + LL(0x4c4c8866,0x2d701e0d),LL(0x22e61da3,0x73a590ba),LL(0x45111480,0x403fd833),L_(0x6080d968), LL(0x41cddacd,0x2957083a),LL(0x38eb9ccb,0xed0801ce),LL(0x1be2ace4,0x726301f8),L_(0x002d8909), + LL(0xdf6a5e6d,0xd1f92359),LL(0xf591d929,0x221a78d5),LL(0xdc207a6c,0x5e691f61),L_(0x20b8f8e8), LL(0xa52ee137,0xaeb5ac09),LL(0x7eacabb9,0x95e85345),LL(0x0985df07,0x04b8e782),L_(0xf8047be0), + LL(0x2128e372,0x84928309),LL(0x4f347f1e,0x62f5aa58),LL(0x7d0fc140,0xf365c6c6),L_(0xa0e662a2), LL(0x7f46460f,0x8d246f44),LL(0x9dcc1860,0x05e6f03d),LL(0x5bfa3ba3,0xd9a27ded),L_(0xb281e327), + LL(0x5bf0784f,0x6bab642b),LL(0xbff128cd,0x0d3bef83),LL(0x268f3894,0xfc73c613),L_(0xcd24728c), LL(0xdefd474c,0xd40e3e59),LL(0xf065f5d7,0xb7f90c59),LL(0x15d0cf49,0x4ad712bb),L_(0x70e00818), + LL(0xd2fe399c,0xf1c993ca),LL(0xa2f0722e,0x4e4c53ef),LL(0xe7c142ac,0x8af34ce1),L_(0xce47aba3), LL(0x7f0fff87,0xb6c1f1fe),LL(0x49dac9d9,0x0f23c16d),LL(0x1baa4d4d,0xdbfb3b98),L_(0xd927e1a8), + LL(0xe7a09b61,0xde4413aa),LL(0xad27b0b5,0xbaf8e054),LL(0x0077242e,0x5d3ac419),L_(0xe50d3185), LL(0xb407f82a,0x6b5c4ee9),LL(0x4a8b1fad,0x6d04cfb4),LL(0x30ac4e43,0xe0253dce),L_(0xf07461c4), + LL(0x5a25f143,0x01947898),LL(0x818e99b8,0x35e2c880),LL(0x05ed8f7c,0x09f78f1a),L_(0x2fb78a6e), LL(0xc9d82b2c,0x4c2f0ccd),LL(0xfdd48d7b,0xaf9e328c),LL(0xe380dbfc,0x9600539c),L_(0xbb3a3bc6), + LL(0x54969bd7,0xea2bbe07),LL(0x394029b6,0x262c8bfa),LL(0xc8ab0425,0x90339a91),L_(0xda368bdd), LL(0xd540c3e3,0x0fe39b20),LL(0xc20f1366,0x5dbdfc74),LL(0x8075ba9a,0xdc668935),L_(0xa20bdd31), + LL(0x457e42b6,0x513bd423),LL(0x02a9ea32,0x37ec475b),LL(0xe477baa7,0x52b0bc16),L_(0x4583bc9a), LL(0xcce869a0,0x287b6484),LL(0xf11b84b0,0xe1aa5cec),LL(0x7925b118,0x4ab367b2),L_(0x8ff3e481), + LL(0xe318b3c1,0xb6f137ff),LL(0x0ffc4673,0x73596e44),LL(0xf9b0dfda,0x2bd4103b),L_(0x56acfc3b), LL(0xd8a47cb2,0xa87de433),LL(0x046c39cd,0x69413a97),LL(0xdebd1720,0x11fa933d),L_(0x3f6c6fec), + LL(0x47ed0eca,0xae4601fc),LL(0x06034734,0x110171a4),LL(0x104d85dd,0xca3c77a5),L_(0x8bca0d2e), LL(0xaee2efa8,0xc78c356e),LL(0x70ae3402,0xcedf0097),LL(0x322e72ad,0x07c1fe3f),L_(0xd0253c44), + LL(0x26e6bed6,0xc2f400a6),LL(0x2291a21d,0x7258c335),LL(0x287eac13,0x1680907d),L_(0x03a00f86), LL(0x4aa4df86,0xbadddbc2),LL(0xc5e15b28,0xc0fcaacd),LL(0xf30664e1,0xb1606671),L_(0xc87b6603), + LL(0x81f0973b,0x1873a8e4),LL(0x5f55c7e2,0xecb9e193),LL(0x33f083bf,0x95c9fed8),L_(0xe416fc1c), LL(0x13f90aa7,0x02778f81),LL(0x07eef5cc,0xc73195fa),LL(0xa94149e0,0xc8c44cfd),L_(0x2d270d3f), + LL(0x1a3813b8,0x7ca9f957),LL(0xdb514ec8,0xb674cd7d),LL(0x9718dd0a,0xe2bf5a99),L_(0x325fe3bd), LL(0x25692bfa,0x4f0b0620),LL(0x78990d74,0x5191688d),LL(0x5e70b57e,0xdfbda68c),L_(0x6e9d11e2), + LL(0x134142aa,0xd61361ba),LL(0xf97cf589,0xb62b767a),LL(0x7b21af77,0xf64ebc1a),L_(0x03a72d66), LL(0x6883434c,0x4b021c90),LL(0x39df1cca,0x2fe04352),LL(0xb84c6c06,0x9b6a3082),L_(0x62692bf5), + LL(0x33f0c495,0x94bf9bf4),LL(0x79c2f57b,0x85cdb0a8),LL(0xe0d00e0a,0x49f82bc9),L_(0xb9c9a088), LL(0xce773e24,0x1f589bd0),LL(0x11cd1677,0x1892643b),LL(0x8f952d00,0xb5eec978),L_(0x016b826e), + LL(0x87c07fe6,0x8cd03f0a),LL(0xf4d9e042,0xa3443289),LL(0xb07e4630,0x361c9252),L_(0x5bbca5ce), LL(0xb600853c,0xe687c31f),LL(0x7bd9a485,0x90550b79),LL(0xe200e18a,0x0e254cdc),L_(0x50dce2e6), + LL(0x3c92fc46,0x4b5e8398),LL(0x284e641b,0xf7347214),LL(0x0556fa64,0xa75c4fae),L_(0x4cc4b972), LL(0xaebb3e9c,0x90fa7977),LL(0x6a40b04c,0xa133ea2d),LL(0x97401c28,0x613bfe61),L_(0x443845a0), + LL(0xe2f9a715,0x532b2d67),LL(0x482b7779,0x065c4d2c),LL(0x123aa16e,0x6886e7e9),L_(0x9147c317), LL(0x40c6dfd4,0x1530f2ae),LL(0xaed04ac0,0xd86577c2),LL(0xa4111d16,0xd86157ff),L_(0x7406594c), + LL(0x23598a12,0xafae6c6e),LL(0xed33d4b3,0xa2858c95),LL(0x008ec0c9,0xfe186e4e),L_(0x7e4ed374), LL(0x9cb08278,0x2534f6e7),LL(0x798a9b30,0xe5f0b67d),LL(0xfcf997cc,0xb58b2b67),L_(0x2f620d55), + LL(0x3d465f68,0xe8013df7),LL(0x5dcfcfb0,0x3e4bfa75),LL(0x7a765b73,0xe0fbd7bb),L_(0x4e7374be), LL(0x186b6aed,0xfe9b1e77),LL(0x375f54f1,0xf7011562),LL(0x55dc463e,0x202fbb6d),L_(0x8d697af5), +}, +/* digit=13 base_pwr=2^91 */ +{ + LL(0xe165ca1e,0xbcdbe444),LL(0xcb4a833d,0x76643977),LL(0xb07c1b51,0xef7d725e),L_(0xdc3b82a8), LL(0x21f29146,0xaa5a7ac8),LL(0x01e0c54e,0xa51481d9),LL(0x2b2d5197,0xd220aca6),L_(0x736d91c9), + LL(0x87d54b6b,0x74d6d0ba),LL(0x9af89e67,0x723937b3),LL(0xada42e0e,0xda49ba83),L_(0x83b9c4ec), LL(0xa3055000,0x12dfdcad),LL(0x7e0a2cb4,0xa98d5243),LL(0xf7e3a9a3,0x6ed0d332),L_(0x1f8c692d), + LL(0x6e661a1d,0x634cef8a),LL(0xff5ab01d,0x35d65531),LL(0x6fe3ecd9,0xc1015db1),L_(0xfcc7ca4c), LL(0x769b7e20,0x795d4926),LL(0xed944d52,0x6db9a053),LL(0xf5f4129a,0x8f02bb95),L_(0x4007343b), + LL(0x75cd1b48,0xab70364f),LL(0x6b243c97,0xf0408ddf),LL(0x8b9a0873,0x0ef0ffd5),L_(0xb6e8c899), LL(0xbe3a87dc,0x2e1c11d8),LL(0x154379bf,0x6ed01611),LL(0x4382cb75,0xc876e0d3),L_(0xc42ec7d1), + LL(0x0e4e37cf,0x1ce64f93),LL(0xa2eb2c00,0x1061f84c),LL(0xe1a22446,0x450ed02e),L_(0x4205e69b), LL(0x644aa51b,0xc76dda32),LL(0x62db01d8,0xb244a09f),LL(0x633bdf3e,0x95c86470),L_(0xf9a6b309), + LL(0xab41ec83,0x71aba88e),LL(0x590711c2,0x3366c35b),LL(0x0b6ab428,0x87bfe939),L_(0x1d7e1090), LL(0x1d44d3e7,0x7ebcdd74),LL(0x2a937fd4,0x42a83b75),LL(0x3da3197e,0xc4155a57),L_(0xf1cea938), + LL(0xb74ca945,0x6a4807e3),LL(0xc0c52eb7,0x474fa11d),LL(0x12ba26fc,0xda7a7914),L_(0x001d4756), LL(0x34ddd360,0xce833467),LL(0x3290dbb2,0x0e517e33),LL(0x2b0a342a,0x579c91eb),L_(0xc27c2657), + LL(0xb129c61a,0x6983b44b),LL(0xf27d3e69,0x463c956f),LL(0xe4a043b6,0x5a97c010),L_(0xc8f8a3c7), LL(0x6cfa78d3,0xb9a3137c),LL(0x5655ed33,0xfc8d858e),LL(0x0b6dca4a,0xc1a3c9bb),L_(0x630b3026), + LL(0xe0a6194c,0x082a09e7),LL(0xffe89fc6,0x8699cb3c),LL(0x9f4f85f6,0x0b13732a),L_(0xba6e1981), LL(0xa2917bd0,0x26eb2b17),LL(0xa4afe844,0x0b4bc106),LL(0x91a5d7b8,0xeeac1c21),L_(0xe71f5fe0), + LL(0x2f9e0c0c,0x3f0bf1e3),LL(0x031e908e,0xbd33fa00),LL(0xbc16afc6,0x6e2d4301),L_(0x90a23e58), LL(0x99df250f,0x2549fd20),LL(0x52f4ef15,0xa724a018),LL(0x5b4c8dd8,0x1a5fbbc3),L_(0x7457c84e), + LL(0xbdbe5cc7,0xa3742f59),LL(0xe0a87786,0x6287ad4f),LL(0xe7aa9e66,0xba62daa3),L_(0x65327959), LL(0xa06c023b,0x7da31b20),LL(0x7fbf3e8d,0x94bea960),LL(0xe1a7865a,0x877c379e),L_(0xea576b89), + LL(0xca000d60,0xaa616a39),LL(0x662a3f35,0xcacdc37a),LL(0xdd7d18e3,0xee30f511),L_(0xf25f9c46), LL(0x7bee7ee6,0x3e51e652),LL(0x81e804d2,0xdd692c2b),LL(0xb0aaaab1,0xa52da544),L_(0x89ee1376), + LL(0x3e7d3b9f,0x97063082),LL(0xf2d42cdd,0x846e4346),LL(0x9cb9423c,0x6ed009de),L_(0x746461d4), LL(0xd30da2a3,0x6b0d52f7),LL(0xe78fe609,0x0b06f17a),LL(0x6887a9cd,0x8730c3cd),L_(0x0f586a94), + LL(0x91370dc2,0xf21a3a39),LL(0x38aa8cf8,0x4835ded5),LL(0x3dd4e3e5,0xf1f81f18),L_(0xee849585), LL(0x8aa2dfcc,0xb30df028),LL(0x1e4514ba,0x24f52ca4),LL(0x757e6772,0x3b2305c3),L_(0xa3525eaa), + LL(0x0e45f1c6,0xb94b1aa1),LL(0xebe6c1cc,0xa3e2b74f),LL(0x1c4d6e37,0xfa33270b),L_(0x5d3856a1), LL(0x62ec6306,0x77b4e047),LL(0x63f53943,0x36b59c35),LL(0x7466b59e,0xaea29806),L_(0xb1050d57), + LL(0x4fc503ac,0x0fd330ac),LL(0x0f394d0b,0xd80c9e62),LL(0xe4d013eb,0x99b62540),L_(0xaa1c17a9), LL(0x0a2e0bc8,0x8f6f2742),LL(0x7366a618,0x3a715ba2),LL(0x88e9ae3e,0x8818aa14),L_(0xd2fe924a), + LL(0x1cf5fda8,0xfd68c56e),LL(0xd02cd00e,0x77ce9ac0),LL(0x5f260a21,0x9c693311),L_(0x7c1f3266), LL(0x878fca42,0x5b2551ab),LL(0x9f545dde,0x365af76c),LL(0x19f87afe,0x019bf350),L_(0x58397798), + LL(0x2ced9c6c,0x46ea826a),LL(0xc4da4263,0xe4112274),LL(0x43744264,0xbfc3581d),L_(0xcc933457), LL(0xb80bff4c,0x9cbecb36),LL(0xe0071ca1,0xb9509e61),LL(0xcf0cfc02,0x67a2c9e1),L_(0x2f9bd24a), + LL(0x48a08314,0xa57e540e),LL(0xd472cc15,0xd2110f43),LL(0x0b60e719,0xcd92d76f),L_(0xca1c9c30), LL(0x44377160,0x8103a772),LL(0xa305744b,0xf08411da),LL(0x54b38b68,0xd6e1cdb8),L_(0xb252c82f), + LL(0xf1e98090,0x935833d6),LL(0xd0e4995f,0x530418e5),LL(0x87f60eef,0x933733ca),L_(0x379092ba), LL(0x423ea862,0xd4945af5),LL(0x60eeff1a,0x165baec1),LL(0x1d6d1df0,0x4bee290d),L_(0xe87f42f5), + LL(0xcf66861f,0xb222f293),LL(0x080ba4e8,0xb45f15c2),LL(0x8ed3f3c1,0xfdee70ba),L_(0x58ce1c82), LL(0x46cc54a7,0x8490a252),LL(0x6ed1e744,0x7d319886),LL(0x0602a509,0x585e149e),L_(0x7fa41773), + LL(0xcd741439,0xf0a3e291),LL(0x74dc9c31,0x41b8528c),LL(0xc78baacd,0x01ba1bb0),L_(0xe5b73558), LL(0x1ce4898f,0x9cf50575),LL(0x5421bd61,0x5a0b632c),LL(0x241cb4c7,0xc421cfb2),L_(0x9ba297a8), + LL(0x9bac4e41,0xf0b66ab7),LL(0xd87b31c9,0x9f0067bf),LL(0xcf835fb0,0x2f9c67c3),L_(0xaee5e6cf), LL(0x228157ee,0xd04298dd),LL(0x0411e445,0xeac052a3),LL(0xbec90b82,0xe6adb2bb),L_(0xb30b9c0e), + LL(0xdcd6bf3b,0xda605905),LL(0x5f01a30b,0x0b1bb0c1),LL(0x7c053c41,0x40f7d79d),L_(0xc39feb03), LL(0x98a4669a,0x3d73c359),LL(0xe472e81d,0x4a94f93e),LL(0xdea2bf34,0xed66fd6f),L_(0xddc57518), + LL(0x904c89ad,0x7095dc91),LL(0x07f66d27,0x2423fe30),LL(0x9c4ec652,0xfd209ae0),L_(0x35803afd), LL(0x94522cf2,0x51ed5eae),LL(0xc9356091,0xb3740f2a),LL(0x19e52bee,0x0676dac6),L_(0xd87e6c27), + LL(0x4ef0a025,0x83877c70),LL(0x39e61abc,0x09e709b4),LL(0xa88e05f1,0x7885ad28),L_(0x53bfba53), LL(0x9b73b3c4,0x8e433b9e),LL(0xd6a78477,0x7394d932),LL(0xcefdc6a1,0x3ce9d748),L_(0x27d9ed71), + LL(0xda4320d3,0xe57aeaa8),LL(0x7dbbbc39,0x06fa3110),LL(0x5c942e4e,0x7d7aed19),L_(0xe660d0ba), LL(0xadbc0281,0xff40dd89),LL(0xcdf314b5,0xb1ee8372),LL(0xa625d183,0x9d4344c2),L_(0x54f2035c), + LL(0x73c8d3ee,0xd120f88d),LL(0xe3275570,0x060ff4c7),LL(0xe344b66d,0x8d5e8c7f),L_(0x1cdb23ae), LL(0x23a7070c,0xce526de2),LL(0xe64e6355,0x4b694305),LL(0x5edf30dc,0x7e00559a),L_(0x3501ee2b), + LL(0x93c5d80a,0xa6698b0c),LL(0x91c3e912,0xd21b90b5),LL(0xbb24391d,0xe1e797f6),L_(0x0526a13b), LL(0xe2efd168,0x41e4a8d3),LL(0x63a82271,0x92a0c47e),LL(0xf6fd3e09,0xd1ade148),L_(0x85041faa), + LL(0xd58b2165,0x7675921a),LL(0xb30c5e67,0xa292beda),LL(0x9a004497,0xb5aac131),L_(0xcaf13420), LL(0xa8c50521,0xced86c69),LL(0xc60fb50a,0x9774900b),LL(0x8cd4dd3d,0xdec98657),L_(0x5092d4ea), + LL(0x829ecf62,0x9fb10f97),LL(0x67b904af,0xef94ed3f),LL(0xeebc687c,0xdae52182),L_(0x13aadd13), LL(0x964738db,0x59a01f8a),LL(0xc7db8080,0x608e9fae),LL(0x5433ce08,0x8b26a71a),L_(0x0c3a71bb), + LL(0x09326b40,0xd29e534e),LL(0x215aa2f4,0x727f40f5),LL(0x5f00b26c,0x3c54a3cb),L_(0x71ec2585), LL(0x14b424a2,0x4c553b92),LL(0xf86c23e2,0x06c97184),LL(0xdd6cfc3e,0x0fb40ab0),L_(0x4647a750), + LL(0xb818ca3b,0xa51a2b16),LL(0x1bbce0ea,0xb716ceb6),LL(0x7e78ccf9,0xc6736591),L_(0xbaa1d2e3), LL(0xe68eaa6c,0x6b8866a9),LL(0xc5b9e6fb,0xa4607970),LL(0xc39a4975,0x712336e6),L_(0x86d2dbfa), + LL(0x37f2fc9e,0x106db861),LL(0xc495534c,0x4b840f41),LL(0x75c25359,0x05a076ec),L_(0xf2a1fa9b), LL(0x22526757,0xbab703f5),LL(0x2b054aeb,0x8e748f87),LL(0x9e9ed0df,0x42646b39),L_(0xd95a6334), + LL(0x22dfe0f7,0xbcf088fd),LL(0x373df018,0x58f99bf3),LL(0x2646d8ce,0xa20f1b7c),L_(0x3bc4db1d), LL(0xc7206a4f,0xf3f061d0),LL(0x7fc330d4,0x97fe5b1c),LL(0xd64fa0dc,0x93be4afe),L_(0x239a905a), + LL(0x465eb830,0x9491ca54),LL(0x3a2368e4,0x096991e5),LL(0x840339bc,0x60229ac3),L_(0x0f74bf45), LL(0x167045a8,0x6bb23f63),LL(0x6df7548f,0x7c1e871a),LL(0xbd4ae1af,0xa5c7651d),L_(0x85fa24af), + LL(0xc10316ff,0xd9ea606e),LL(0x72e722f7,0x1d7722d9),LL(0x02dae11f,0x6f289df9),L_(0x4f148db1), LL(0xb66ca6f2,0xe8df6026),LL(0x0dc6bfc0,0x6f25ab93),LL(0xc2c848fa,0x726e5bf2),L_(0x0bc8372a), + LL(0x29431ba8,0xe513cc60),LL(0xb368f5d3,0xf213ba74),LL(0x2acdbaae,0x077919ee),L_(0x69129848), LL(0x3dbdea30,0x5c03af05),LL(0x6892cac4,0x81b53f34),LL(0x13156876,0x86df2c46),L_(0x15d63bed), + LL(0x20319c6f,0x85ef7365),LL(0x7ea45182,0x2f38b49b),LL(0x7d4f7ca6,0x2e79b2d0),L_(0x5befddd6), LL(0xcfcb5a29,0xeb6e8177),LL(0xd7c05293,0xeb5b17b7),LL(0xfebc1ad3,0xc2ef9919),L_(0x5ef848b7), + LL(0x628b133f,0xec0fd6a2),LL(0x4834ba2e,0x71d618c4),LL(0x70062560,0xee7cbb5c),L_(0xa240c28a), LL(0x0c5f4e11,0xe8bfcc1d),LL(0x36f2363a,0xdaf65ead),LL(0x2c7fdead,0xee2e792a),L_(0xfe7f2c9f), + LL(0xb9045bc3,0x21f1b865),LL(0x7a207076,0xf7c65eaf),LL(0xa289580c,0x26c6a4fb),L_(0xac8c38be), LL(0x2c1dd729,0x603a204c),LL(0xc96f5b0f,0x7ec2e50b),LL(0x07974ac0,0xdd93d467),L_(0xd384c831), + LL(0x27da0d82,0x130cf277),LL(0xb2d57bad,0x7cd69a13),LL(0x2e92cb98,0x0b8251c6),L_(0x04325b28), LL(0x35f746a8,0x630e8e9d),LL(0xcda5985f,0xe5c307ba),LL(0xf6775b8c,0x9cc53c03),L_(0x466530c7), + LL(0x660bcc85,0x9b3d46a0),LL(0x6911dd6e,0xfec92d25),LL(0xf5c72cad,0xaf7d9d93),L_(0x53a3ea3c), LL(0x244c7ae6,0xd879f3e1),LL(0xeaba3d43,0x57858904),LL(0x4c921338,0x3b59edc0),L_(0x7ee8ea79), + LL(0x0d019290,0x64c557a0),LL(0x70b7979e,0xa66689a7),LL(0x482d3b9a,0x08f6017c),L_(0xecf693f0), LL(0x90c3c46a,0xe8816977),LL(0x7c8670c4,0xf94ce04b),LL(0x05b8eed1,0xa8713371),L_(0x0ebddf9f), + LL(0xe8f0dfc1,0x50dc1442),LL(0x884f22ec,0x104f20a7),LL(0x5f885c0b,0x4da67f62),L_(0x6b4ecc34), LL(0x8460d391,0x0181b70c),LL(0x3f227a67,0xffedd7e8),LL(0xafb114cc,0x5b40835b),L_(0x2f5bb436), + LL(0x2da90ee3,0xdaec76bb),LL(0x6deef6a5,0x05e4b337),LL(0xe70560f3,0x8e70fab8),L_(0xdf0c450d), LL(0x76bc05d5,0x916202a7),LL(0x05568dde,0x84cf17af),LL(0x89f0919f,0xb5670da6),L_(0x97387cce), + LL(0x54cdc952,0xc269a349),LL(0xae4d60ad,0xa15fdc88),LL(0x27a5a550,0x560e5ad7),L_(0xfd2940f8), LL(0x23e1393c,0x86446f69),LL(0x2f2ee978,0x918a434e),LL(0xa481c71a,0xd1f9e2af),L_(0x99fe0c80), + LL(0xecc46dd9,0x4c354b31),LL(0x9d2731ca,0x5917267d),LL(0x9583d77a,0x549fab4d),L_(0x20743202), LL(0xfa4579de,0xc9afa8d5),LL(0x6ee70bf6,0x679e9b14),LL(0x71181b55,0x34dfc171),L_(0x186cb701), + LL(0x92faa6da,0x784904dc),LL(0xef91aeb1,0x99e8f15a),LL(0x466ca745,0x9dba0e94),L_(0x3efbbed0), LL(0x9c629c4a,0x818f1160),LL(0x238404b3,0x1649074d),LL(0x7f378102,0x9026e5ef),L_(0xd9ddeb02), + LL(0xcb042e26,0x50f516e2),LL(0x359a1ffe,0xf69c04f0),LL(0xea7c7fb8,0x2be043ec),L_(0xe26d37f4), LL(0x4f292184,0xd8d23c81),LL(0x97f967f3,0xa3d950fe),LL(0xf97e68cf,0x0cf39dbb),L_(0x0f3321ca), + LL(0xf1e4bd62,0xd200dfdf),LL(0x890af600,0x9e7aabf2),LL(0x1e4962b1,0xd80a6211),L_(0x78562fa6), LL(0x2ca32240,0xdb5a70a7),LL(0x68abbf8d,0xb27e8e16),LL(0xbffea5c8,0x99114570),L_(0x6e5308c1), + LL(0x4fe00738,0xdfd77bdc),LL(0x16448af4,0x0bebe0b7),LL(0xcb49f26a,0x9f4c424c),L_(0x7c95aa03), LL(0x84b25349,0xd5948dce),LL(0x93acdc43,0xa92b381d),LL(0x7efb93e4,0xa8e311a0),L_(0xe08e348f), + LL(0x8f67c168,0x96c4f394),LL(0xfa10af74,0x457899a3),LL(0x489d01bc,0x1a4d0c16),L_(0x5457e489), LL(0x5c85d68a,0x1331b4d9),LL(0x84eb66f4,0x97024d48),LL(0x8054cf7a,0xb191cf89),L_(0x3e7eb0cb), + LL(0xd307d077,0x8d0cc3ce),LL(0xa6a534ca,0xa8bd8bc1),LL(0xa917d88e,0xc01d22dc),L_(0xba878e3c), LL(0x6dc0cccd,0x927b4947),LL(0xfb3f69e6,0x8cef47b8),LL(0x44185c64,0x586c00a9),L_(0x151ed75b), + LL(0x2be07ecc,0x72a46aa9),LL(0x519ea031,0x5b289abe),LL(0x75f6f741,0x1a09f429),L_(0x0df7057c), LL(0xe83d8445,0x0511f341),LL(0xb8c291d5,0xcd82da51),LL(0xbd814e25,0xa548461b),L_(0xdfcbbc5c), + LL(0x481967d8,0xf029f707),LL(0x3ec594c8,0x9b2f8b93),LL(0xaacdcc45,0x76cebcbf),L_(0x6e7c6e83), LL(0xaec54ae3,0xa2a8a4fe),LL(0xb47c1c29,0xea7ebbf6),LL(0xebdc1ae0,0xed49a01f),L_(0x63ac12b4), + LL(0x532eddda,0xaa4b3b46),LL(0x7e957ed9,0x54c46d48),LL(0xe3393b10,0xde08b775),L_(0x8522afee), LL(0x04a30d0e,0x5f73d8d1),LL(0x39d26476,0xa070b5a5),LL(0x7e58e0f3,0x213036c0),L_(0xa0e99b0f), + LL(0x180c99f6,0x8a626b91),LL(0xbe1888a3,0xa37b0a55),LL(0xc627fd2e,0xd55c88b6),L_(0x75c0ce87), LL(0x143ea75e,0x96806d01),LL(0xdce56555,0xb31db013),LL(0xa0f7c5ea,0x286d7145),L_(0xd7ff4a5f), + LL(0x7eb22eef,0x9f5793c8),LL(0xf0bdad25,0x3795ea86),LL(0xde77b0de,0x308c67ea),L_(0x506de99f), LL(0xa761521a,0x9d70efe9),LL(0xf8b6908d,0xbeee111a),LL(0xa22f9208,0xabb48b09),L_(0x305250b9), + LL(0x54347d1c,0x04d51ba3),LL(0x69790978,0x0cf10a39),LL(0x55c72c45,0x340e8617),L_(0xcd9b6116), LL(0x897f38e5,0x80b0004a),LL(0x8b257151,0xa9ff195d),LL(0xae7d889b,0xd95f9a8b),L_(0xf7035183), + LL(0xf9445254,0x96f17da2),LL(0x21e71bcf,0x87497b69),LL(0x1dde1b45,0x3f515a1d),L_(0x1f620fc3), LL(0xbb05c2ca,0xd2651945),LL(0xa7f48789,0x5426bd20),LL(0xa9e9b7f9,0x048d20b9),L_(0x2efd58e7), + LL(0xfddc7154,0x095a8f25),LL(0x6fa2edc8,0x675fd35e),LL(0xad7dc081,0xf2b8fbbb),L_(0x88a7b80f), LL(0x6cabea84,0x2ca3a2a5),LL(0x00c09332,0x1047dcd4),LL(0x0d577a61,0x54d150a9),L_(0x1837cf45), + LL(0xe243bea6,0xec967631),LL(0x281b5c05,0x7c40da6a),LL(0x580362ba,0x832277a8),L_(0x702cc617), LL(0x1ca409f6,0x7335bdc8),LL(0x26119f95,0x735f11ed),LL(0xa27ab674,0xc7055ffb),L_(0x652580b5), + LL(0x3dba43f3,0x8c6adeda),LL(0xebd0f498,0xa42d63e8),LL(0x993107d0,0xb75b4edf),L_(0x0232f8ad), LL(0x720ca302,0xa2592945),LL(0x48d03da4,0x473bf917),LL(0xda871c01,0x8ff4aaa7),L_(0x746e818b), +}, +/* digit=14 base_pwr=2^98 */ +{ + LL(0x3e28ba8b,0x23800fab),LL(0x8b12655a,0x99a4d041),LL(0x66a4179d,0x33023ec9),L_(0x1205108d), LL(0xdbb13a66,0xa87c96ac),LL(0x402e3b9a,0x98998891),LL(0xd3dbe7de,0x6b327511),L_(0x5723ac5b), + LL(0xa70cb5da,0x7ab235bd),LL(0xbede2315,0xac5f3af5),LL(0x5f9a6aaf,0xe6d56464),L_(0x357d2d21), LL(0x3156d6fe,0x95a60b40),LL(0x49f4f0e2,0x9e63e29f),LL(0xdca3ea62,0x4c739342),L_(0xa16d4457), + LL(0xe1948d0e,0xdcc5b89a),LL(0x463327ef,0xd5a2f46b),LL(0xadb9328a,0xbfce0e4d),L_(0x0d62b545), LL(0x1dd89a4b,0x349d4bd5),LL(0xa76133fc,0xc19125f9),LL(0xb4284724,0xf5a817aa),L_(0xf7ca3909), + LL(0x7fc121e6,0x0655039e),LL(0xbc44db37,0x42a0f87f),LL(0x3d4e983e,0x67d89bbe),L_(0x85435ace), LL(0x18346e4d,0xbe29cd2e),LL(0xed39f9e8,0xe8b5a373),LL(0x55cac6ca,0x3d4406d2),L_(0xde2aeda4), + LL(0x6b7d56c0,0x8fabc04a),LL(0xc39c7ee8,0xbb51ed42),LL(0xc619fe58,0x0816fbf5),L_(0xf268e46c), LL(0xe0f981bd,0x3b1bba1f),LL(0xce226865,0x27136de9),LL(0x4f9ab1d9,0xfcbc42d6),L_(0x13efb949), + LL(0x4f54831f,0xf8119c34),LL(0x75d60569,0x24503e1c),LL(0xb3a82995,0x063e48b4),L_(0xc7bc1aec), LL(0xf21ff6a7,0xd626619e),LL(0xa7f182b8,0x5a64395d),LL(0x533f72eb,0x1d2aa02e),L_(0x69953a2d), + LL(0x64b57fc3,0x91516c9b),LL(0x0913a322,0xb1936549),LL(0xa0961a2f,0xa7d10cb1),L_(0x06e4869d), LL(0x8f465d18,0xdc8793e6),LL(0x9f17e3a2,0x9a3dd9de),LL(0x06fc6243,0xe925133a),L_(0xb7622bbe), + LL(0x784462ed,0x84e7489a),LL(0xc981ba62,0xe5d5e019),LL(0x792db847,0xa77e3ed1),L_(0xf4b6792a), LL(0x36f5a622,0xed208d22),LL(0xfc7dc4a1,0x953f894f),LL(0x87c2b024,0x095b0ea7),L_(0x88016bc4), + LL(0x486f7168,0x59119396),LL(0x4b8755e3,0xb37c8ef0),LL(0x19aa2514,0x554f2e58),L_(0x1d544a58), LL(0xaa9858fe,0x1d4dba6e),LL(0xcc8e5a35,0xa400007e),LL(0x4036ed62,0xf70ca123),L_(0x0fd793e9), + LL(0x292ba598,0x269da084),LL(0x2503b0ce,0xdd28f9d0),LL(0xb3025472,0xeb061c62),L_(0xfe9ae248), LL(0x79d8d16b,0xd2906b24),LL(0xeaa5d6f9,0xdee30126),LL(0xaf62d50d,0xc2b140a8),L_(0x4dbbcce0), + LL(0x57f41736,0xa7cf5774),LL(0x3293ff9c,0x238c8acf),LL(0x1f10127e,0x739c1785),L_(0xc01f9711), LL(0x761037ad,0x68df6a03),LL(0x50fcfcf4,0xc797349c),LL(0x340f3712,0x53cced84),L_(0xba988480), + LL(0xc57ef65f,0xed47d3b7),LL(0x6051e486,0x623f7228),LL(0x2e29a4c3,0xfb40723c),L_(0x77353f89), LL(0xda6cb60f,0xf4eea8fd),LL(0x672e091a,0xd086a7ab),LL(0xef34d20e,0x96e8e707),L_(0x2e5db0f7), + LL(0xd47d7453,0x089e80e8),LL(0x259bf71c,0x99cb2528),LL(0xcd52c81c,0x9b67f891),L_(0x106aa44c), LL(0xb3fd57c9,0x84cca31f),LL(0x492159ac,0xd208e111),LL(0x99fc08c2,0x9a577c2f),L_(0xde194e1f), + LL(0x3ffcd5a2,0x1a869e94),LL(0xac2dd0d1,0x0101716e),LL(0x0d98495e,0x926fca45),L_(0x230d5720), LL(0xd7a376f8,0xda97aa6f),LL(0xe0f36b43,0x043a0c8c),LL(0xbdee74a9,0x9f4f1e97),L_(0x616d2e77), + LL(0xc462e14f,0x4ce190f6),LL(0xe408ab06,0x8983aaf7),LL(0xaec26bc8,0xea864657),L_(0xa9c6ad27), LL(0xe823a8e1,0xab862943),LL(0x327c78b5,0x42d64507),LL(0x751ec6f1,0x18bed81e),L_(0x1c240eff), + LL(0xc5905237,0x291c3272),LL(0x766b4a0b,0x7507a05b),LL(0x6e56a096,0xafb97a2d),L_(0xe3cbbbe3), LL(0x44beffab,0x2369de34),LL(0x4c78c1e4,0x287f5c70),LL(0x698e8f90,0x14e1a4eb),L_(0xa87a74e8), + LL(0x4ca2c62e,0x0fb3705f),LL(0xcbf75eae,0xe59c7ef1),LL(0x58dee01f,0x1a149a9e),L_(0x67ff1a09), LL(0xa2bee6a8,0xf63b9864),LL(0x7e111af7,0x602716b7),LL(0xf6f23814,0xc6967a32),L_(0x682916a3), + LL(0x94c36bb8,0x53aa5941),LL(0x7dc26ca6,0x57977663),LL(0x3a7f3859,0x4c6d1720),L_(0x87d2b98e), LL(0xce189ddd,0x379e4e5c),LL(0xc6bec0ca,0xafc9ee7e),LL(0x69db9b01,0x4e3aaa80),L_(0x4f1a3e1f), + LL(0x9f2327ea,0x948cf978),LL(0xe1a7d9ed,0x932e601a),LL(0xabd060c2,0xe2c8026f),L_(0xd02845ba), LL(0x4568e0ed,0xd2fcd20c),LL(0x9fc8ca8b,0x224394d0),LL(0x0b403598,0xfa897231),L_(0xd2ef4cab), + LL(0x61ffdada,0xc9608223),LL(0xd3a9e728,0xbfde5d50),LL(0x9fcf28b0,0x23db313b),L_(0xe226f1f0), LL(0xc362ae30,0xdd26c697),LL(0xa3a7479f,0xbe0de20e),LL(0x4871cbd5,0xc0ad648f),L_(0x30414c9b), + LL(0x93a58cef,0xe22f8df4),LL(0xdef38d08,0x40b0ba7f),LL(0xfacc23b1,0x7b3f5e97),L_(0x00fb9a2e), LL(0x926c141d,0xa6cbdc36),LL(0x87cb811c,0xefb43047),LL(0x6214d9f5,0x06806fd9),L_(0x5ba8f23b), + LL(0xc5b2ff25,0x86e33c2c),LL(0x23423298,0xde9bc0ea),LL(0xf6e30434,0xc242970e),L_(0xbf53cf6b), LL(0x3df6ea77,0x84f61ee9),LL(0x8140c42b,0x1d03d30e),LL(0xe917c40a,0x70d1d6e0),L_(0x95de99fd), + LL(0xca391889,0x396774da),LL(0xc249bd59,0xc8c93775),LL(0x24f81111,0xfe365e8c),L_(0xdcd6e625), LL(0x680035a8,0xe01d308c),LL(0x65e9e9bf,0xe2741d32),LL(0x5cab99f6,0x441a1645),L_(0xf759c45d), + LL(0x858ad4cf,0x1d7326e1),LL(0x52fad307,0x7ee589d6),LL(0x3aa389bc,0x9cfd5023),L_(0x3feb82b3), LL(0x03b8a60e,0x4a98e744),LL(0x22a198b9,0x2c871f83),LL(0x5fd22f5d,0x95dcd425),L_(0xe40ca457), + LL(0xb6c77ce3,0x0baec1d0),LL(0x899726d2,0xe604c5b2),LL(0x9503a4a3,0x8faff5a9),L_(0x735bcc96), LL(0x6c5c0a3c,0x294b5dc8),LL(0x27659e3e,0x63327cba),LL(0xb11539c5,0xf82de3fd),L_(0xdafe9d6c), + LL(0x2b031402,0xb712f885),LL(0x4ba7d548,0xc6d1f271),LL(0x627d0816,0xebc722ce),L_(0x4e21b679), LL(0x2f1104b0,0x3d488791),LL(0x654d615e,0x7a45f251),LL(0x38e30e5a,0xbe0c2b4a),L_(0xd07392c8), + LL(0xc3944e46,0x91582e10),LL(0x245a5f24,0xbf72cd72),LL(0x1d74fe95,0x32fc2d8c),L_(0x8d68efa6), LL(0xc63d3e55,0xb73ed81a),LL(0xd7c363e3,0x81d7988d),LL(0x4b18aee3,0x8563cd35),L_(0xf296ee0a), + LL(0xde9cd058,0x6115b5de),LL(0x12e07056,0x494a27b8),LL(0xab33f925,0xdadb7f44),L_(0x9611fc0f), LL(0x7c525248,0x135db673),LL(0x0014d0b2,0x13d5d03c),LL(0x34df44be,0x498b5aab),L_(0x3b9bf0a3), + LL(0xb6d60031,0x8b47b859),LL(0x7583945e,0xd5d1a023),LL(0xe1eb9f0d,0xdc6bf5ac),L_(0x7f50540d), LL(0xd48c1aad,0x7dcd0fa2),LL(0xd6e884e8,0x7ff3b8ae),LL(0x0fda3c41,0x42d34849),L_(0x156b5074), + LL(0xf41aefe8,0x9db1951d),LL(0x6f1aaef3,0xc78e5a0a),LL(0xe07d36c2,0xaf67c45f),L_(0x598114fb), LL(0x491a5763,0x1883388f),LL(0xb079f72e,0xfa868602),LL(0xee4b11df,0x20bd4ca5),L_(0x9dd3598b), + LL(0x4fe58726,0x2f515e06),LL(0x0a130c64,0x03928fde),LL(0x21d929ee,0xed92766f),L_(0x2370b592), LL(0x26410775,0x461ee104),LL(0x11fd327c,0xbf2200cc),LL(0xaac6ed8b,0xaad362f8),L_(0xe2c21338), + LL(0xa0369424,0x7702c304),LL(0x90cc96b4,0x88180dc5),LL(0xec9108c9,0xea844162),L_(0x79d4d2e8), LL(0xf7c7120d,0x9b79933f),LL(0x9b1c7dfd,0x73191637),LL(0xf8a798d5,0x4a0a1e22),L_(0x7439503e), + LL(0xa7716be7,0x25112356),LL(0x41a91a95,0xf0234300),LL(0x12be9cf2,0xa09c00f4),L_(0x87b213f6), LL(0xb0626c0f,0x3cc23f7c),LL(0xa44cb6f8,0xee84c654),LL(0xda068194,0x999bf774),L_(0xe19968dc), + LL(0x20c86fd9,0xa3bbb5a6),LL(0xe07280d1,0xdced85e1),LL(0xf0df9658,0xbd50dd87),L_(0x4715fc8b), LL(0x90915daf,0xf3d02094),LL(0x64a96f25,0x5a8039b5),LL(0x661822c3,0x8b298e54),L_(0xd8ec764d), + LL(0x4146852b,0xcef9767a),LL(0x3dfab91a,0x3b280141),LL(0x85522368,0xdbf3bb79),L_(0x79b36fcb), LL(0x730e7127,0x05fa18c1),LL(0xdbafd1f2,0xf1172af0),LL(0xdd549dff,0x83c7657f),L_(0xd82dc78f), + LL(0x35ef27c5,0xad652989),LL(0x58b32f27,0xeb421196),LL(0x976af3d3,0x4b505f2e),L_(0x64c1cfc3), LL(0xf3dae0f8,0x59913cfd),LL(0xbab37657,0x2039f0fc),LL(0xf066519c,0x8887e569),L_(0x3d975cff), + LL(0xb1eb152a,0x425ddccb),LL(0xd5368e78,0xe11b71d6),LL(0x12b23c3e,0x33076aa1),L_(0x4e626b1c), LL(0x4b18cb13,0x23f1e78c),LL(0xe86079cd,0x875b887a),LL(0x908738ce,0x244ce9b9),L_(0xe51d741d), + LL(0xd6758071,0xb2d63b63),LL(0xf0e2cdd8,0x30007109),LL(0x90f8143d,0x75829dd6),L_(0x2d23a43f), LL(0x5e5327c1,0x47ca8667),LL(0xa526104e,0xf04921aa),LL(0x85467645,0xb0f0e275),L_(0xafc61eeb), + LL(0x87b4b9ac,0x7ee79f80),LL(0xf367484b,0x27ef21cc),LL(0xbfc8c258,0x7fe068cc),L_(0x7c8262c2), LL(0x572f615c,0xc96069f0),LL(0xd5c6e6e1,0x5f2c8d88),LL(0xd8f70ecc,0x24073300),L_(0xc55f13ed), + LL(0x33b47b28,0x82d22a51),LL(0xe38ffddd,0x480198ae),LL(0x03df60cb,0xa964fa9d),L_(0x9bfb3890), LL(0xee5cec96,0x96707a63),LL(0xf9f9338a,0x22ae2467),LL(0x5d9e9ac0,0x0acd29e1),L_(0x854609dd), + LL(0xb29eb007,0x84bcad3a),LL(0x9a527f75,0x761f8288),LL(0xc16f294f,0xdaf38a99),L_(0xfdebcc4a), LL(0x9c537eba,0x8a1d04d0),LL(0x760244a9,0x03627bce),LL(0x97af96d2,0x79ecc48b),L_(0xebf508a7), + LL(0xa6400e31,0xa9338a43),LL(0x4ccbe898,0x4312a928),LL(0xf79ce659,0xa7f6528b),L_(0x0efdd8f6), LL(0x0c3ef14c,0x7002f54a),LL(0x859eefc9,0xacd6c70b),LL(0x3146204e,0x25b282fd),L_(0x9e570ac1), + LL(0x7cfc8794,0x50227a05),LL(0x96cc2182,0x5a9a9895),LL(0xed491384,0xa5e9dc33),L_(0xef480409), LL(0xf767a743,0x48fc5412),LL(0x7b84b8b7,0xb18a17fb),LL(0x22c04cfa,0xa2e99567),L_(0x5b8dc942), + LL(0x45e7789f,0xb80a1876),LL(0x7a0670f5,0xaff77eda),LL(0x99c0dcff,0xceba003e),L_(0x737b1478), LL(0x169db41d,0x88ff60f7),LL(0xa8a91790,0xe2c4f8ba),LL(0xd0c75bfd,0x8ebfd0f8),L_(0xaf779b34), + LL(0x0401de06,0x62fec3e1),LL(0xd969780a,0x99a90daf),LL(0xafc171d9,0x647f2bf8),L_(0x16ea68f8), LL(0x41dbf4ac,0x83f12d02),LL(0xb63e2588,0xa64d6924),LL(0xb69cbad9,0x08b7948f),L_(0x1a740ee4), + LL(0x4eff7cff,0xd3ad093d),LL(0x97774b8e,0x520858a7),LL(0x1438ff5b,0x90acd42f),L_(0x0c55b2c6), LL(0xdb613141,0xf3524dc0),LL(0x1f19f151,0x4d56c7a8),LL(0xc5b4f62c,0xe1899b7c),L_(0x7240d638), + LL(0xeb1a717d,0xf9ad69aa),LL(0x9a96c604,0x32f476a8),LL(0xd851c4f0,0x816d9822),L_(0xf895d68e), LL(0x6f708c56,0x914c2f2f),LL(0xdd941975,0x64e1cdce),LL(0x1765af52,0x9bdc107c),L_(0x770ebc8f), + LL(0xc745e953,0xe5bd052c),LL(0x70e13a7c,0x1d26e2b6),LL(0x0c4379c6,0x0121f8bc),L_(0x71124e71), LL(0x87d6954b,0xe43a3a54),LL(0xd9134113,0x9bfa2ae6),LL(0xbbbe7daf,0x3a1c2b24),L_(0x3950f72a), + LL(0x6a18f59a,0xa71550b0),LL(0xdab1491f,0x73428fb9),LL(0x12caa6e9,0xb08da5e0),L_(0xf505a6eb), LL(0x3f48afb0,0x79842176),LL(0xcf510150,0x2962dfb0),LL(0xafdce054,0x87a8f6f2),L_(0x8158b7f4), + LL(0x438e4a3a,0xe2eb279d),LL(0x6af97a49,0x42b5d181),LL(0xd6c5159e,0xcf78aa46),L_(0x572c05de), LL(0x50428178,0x0bf9ef59),LL(0x5b7148bf,0x479ccacb),LL(0xbe4b9632,0x101cdc13),L_(0x7ed1a6b8), + LL(0xf0214364,0xb580b66f),LL(0xc6feaf5c,0x4e810e3e),LL(0xa5cb0934,0x02cfec16),L_(0xc2e35b6b), LL(0xddb45407,0x6e09365d),LL(0x5b4d0976,0x76d217cd),LL(0x405714ef,0x892fc650),L_(0x75be4796), + LL(0xf30d07fb,0xe024a3b5),LL(0xffdc8787,0xe4416883),LL(0xbb8a1baa,0x16ab4631),L_(0xf8688223), LL(0x2f0f8322,0x09c1692b),LL(0x0dd7a6f9,0x13a2837f),LL(0x3a94d17d,0x784e210d),L_(0x58b82c4e), + LL(0x75a9d2ec,0x4df5c032),LL(0xa0b09f2e,0x9989a169),LL(0x8ad59d7f,0x92ef323e),L_(0x9dfd994c), LL(0x641354fe,0x9c294c51),LL(0x0784cc2a,0xa2a1bccd),LL(0xa7c397c2,0x9d3b5c1e),L_(0x5217298a), + LL(0x1d43acad,0x5fffdecb),LL(0x15425166,0xd4c107c2),LL(0x3ad2c54b,0x29b56266),L_(0xcbfffc28), LL(0x2c3f6f4b,0xb4051ff3),LL(0xe925b548,0x65af8f85),LL(0xd94421d3,0x61d3c552),L_(0x0c2f7520), + LL(0x6d77a3b7,0x70ed4768),LL(0xfa81c450,0x9fe1ff87),LL(0xbb762aa7,0x72d6d774),L_(0x7b08eae2), LL(0x7dbd50f7,0x1432d1e1),LL(0xc28ea0a8,0xd0acc833),LL(0x57bf54b2,0x163d3f1c),L_(0x996fa17f), + LL(0x4583d724,0xb5fbaafe),LL(0x53d923d7,0x051dd3b4),LL(0x8aeef402,0xc59594fb),L_(0xd7267ef5), LL(0x5e8a7497,0xede0bede),LL(0xb019e637,0x6171e5a5),LL(0x1be98aa9,0xb69b47a8),L_(0xd848486b), + LL(0xddef3d83,0x95f000be),LL(0xc417bef7,0x615e1192),LL(0xef4c07bd,0x74d082e1),L_(0xc8cd53f2), LL(0xc2fe846c,0xfb026020),LL(0xaab1ecd2,0x31978a04),LL(0x47b08bd0,0xf6ef41f3),L_(0x386f4e41), + LL(0x0974e562,0x806b3be2),LL(0xd292e87d,0xb4c666b3),LL(0xb31604a0,0x08cba3d2),L_(0x5a2daba1), LL(0xd6e9ea3f,0xff881f9f),LL(0x4da3f147,0xbaac2563),LL(0x8b3ea4f6,0x7f14ef98),L_(0x931e6a5d), + LL(0x09ed1548,0xe8b9b58f),LL(0x3a4e460d,0xbdf4bff4),LL(0x410540d4,0xba715bf7),L_(0x9faf6e57), LL(0x58d1763d,0x13d8563b),LL(0x83a48da3,0x3effa46a),LL(0xbc420fce,0x54fecfb6),L_(0x8ea8b02c), + LL(0xcc8fc2ff,0x9f084f50),LL(0x15421912,0x4d3ec704),LL(0x69e1810f,0xa93a4b35),L_(0x66d7f71e), LL(0x75523ca2,0x9b5d9b14),LL(0x85c36f3f,0x8615c97c),LL(0xf837f0c4,0x1acc3982),L_(0x4526c2f7), + LL(0x61301784,0x91bdbe3d),LL(0x5c462b80,0xe5304d0f),LL(0x16f4a53a,0x2452c556),L_(0xf2ad5a49), LL(0x92965e0e,0x961c76d6),LL(0x92031bde,0x3e75de1d),LL(0x43fc1b46,0xba2806c7),L_(0x7ea59306), + LL(0xe2e84c48,0xaf7fedc1),LL(0x44f370ac,0x7473cfd6),LL(0x7fa03a08,0x366491a9),L_(0x65caef0d), LL(0x8b501288,0x3a469595),LL(0x6dc5f00f,0x67981e43),LL(0x87cfba83,0xebb7f488),L_(0x1fc8150f), + LL(0x0adc88a9,0x70a3e6c5),LL(0xf79b97ae,0xc2b70fcd),LL(0x0c92525b,0xfd612292),L_(0x1f877d7a), LL(0x2017f6eb,0x2b249ae0),LL(0x78a6422a,0x083dff80),LL(0x0659a5e4,0x12934c87),L_(0x7a1fca11), + LL(0x2ef56139,0xe4d85de6),LL(0x0b0acedc,0xf2c33f4b),LL(0x6091bc19,0x04526cb6),L_(0xac1ce3ea), LL(0x24bfc4bb,0xb54024a6),LL(0x57196c24,0x909d86a2),LL(0x42e04322,0x1b1d5d2a),L_(0xbb31992a), +}, +/* digit=15 base_pwr=2^105 */ +{ + LL(0xb476bc0f,0x5f4861c3),LL(0x9a1b156b,0xed89263c),LL(0x45b930e9,0xbeb5a955),L_(0x29358e8a), LL(0xaad0495f,0x7923968b),LL(0x127f5983,0x3abc6adf),LL(0xf5aef19c,0x22713358),L_(0x0329be36), + LL(0x97c8de6b,0x842632ce),LL(0xd5e766ad,0xfcb4c194),LL(0xbab1e908,0xdb59f894),L_(0xa103e5b9), LL(0xe0644e8b,0xc4c67972),LL(0x2cd3e628,0x340af5c5),LL(0xe2690b88,0x787c5ec2),L_(0x153e0f49), + LL(0x7e2abe1e,0x01d9f5c6),LL(0x9acab18d,0x4187566a),LL(0x96fbb4bc,0x43543f3e),L_(0x19e99acc), LL(0x911c12bc,0x4d4f6942),LL(0x2373babd,0x05a419a8),LL(0x0c662933,0xdd524336),L_(0x60570f05), + LL(0x5607a4f3,0x98a32e87),LL(0x9cbcb156,0x841d648d),LL(0x00a72b88,0x85f705a8),L_(0xb60a7751), LL(0x30e3dc7e,0x3ef7d9b6),LL(0xc5078f17,0xeec26796),LL(0x20110dec,0xb521d60e),L_(0x324aa16f), + LL(0x547a71b7,0x7053cd14),LL(0xc75d38b5,0x3ecd5cad),LL(0xff26fb53,0xfa6fa1ea),L_(0x89904b74), LL(0xba1aaedf,0xb9800605),LL(0x0d1c07cb,0x517f26e6),LL(0x7605e424,0xe982130b),L_(0xc4ed9c66), + LL(0x06117460,0xb3f704f4),LL(0x2c1c043d,0xed0726e9),LL(0x3e68dbd8,0xbe354e8d),L_(0xa6382386), LL(0xa4136478,0x26378a40),LL(0x04646666,0xcc8a8909),LL(0x88fbb7ce,0xc603785d),L_(0xb1f815a2), + LL(0xee89450b,0x43fe75c8),LL(0x9a29229c,0x16a051c0),LL(0x9b1a9170,0x7d4b9fc3),L_(0x133dd4ca), LL(0x738df325,0x53ac8646),LL(0x6dcfad8d,0x5c600cb6),LL(0xe4115708,0x6ff11203),L_(0xbc9689c3), + LL(0xaab0cefc,0xd81789dc),LL(0x7b457d74,0x67ca91b0),LL(0x5a87275f,0x096f997d),L_(0x5d4f53a6), LL(0x3d1c72aa,0xc86a176a),LL(0x0c5247ba,0xbb03ebf7),LL(0xb389c0f5,0x325d67b1),L_(0xdc511ddb), + LL(0xe0e68838,0xa0dc02ac),LL(0xca8b2e56,0xcb1f32e9),LL(0x59515fbd,0x152ad207),L_(0xa0ec8c40), LL(0xaf08ac54,0x990ce0e8),LL(0x094829b8,0x62062c81),LL(0xdb93ce78,0x53af9d31),L_(0x980f663f), + LL(0xa663c4b2,0x0afcea40),LL(0x1410da31,0xeb72dea5),LL(0xc4dd3dce,0xefc18ea4),L_(0xc844e9d6), LL(0x827776e0,0xb2b1ab80),LL(0x89238185,0xeef3941c),LL(0xc007c089,0xcc238164),L_(0x10df0d3c), + LL(0xccb1a15d,0xd9103e73),LL(0xec1702f8,0x34d31501),LL(0x67e64440,0x4b15a941),L_(0xaf4f5d2e), LL(0x8b86c479,0x82fa4170),LL(0xbc874d39,0xb17316c7),LL(0x63f968d3,0x7b71afa1),L_(0xc6c567c6), + LL(0x8941fa38,0x1b353d4b),LL(0xfbfeb69e,0x76d964d5),LL(0xaecee209,0xdaae33fb),L_(0xc8d8f5e9), LL(0x47c07e3b,0x4e52dc91),LL(0x87741a88,0x92359ae5),LL(0xa5009004,0x72c650f3),L_(0xc505e3b9), + LL(0x660afa5f,0x2c204e5a),LL(0x9f6af3aa,0x70aa9a5d),LL(0x0423f2f9,0xa67cf879),L_(0xe29129c8), LL(0x58a3703e,0x47ebe4a0),LL(0xc220fd1a,0x8f060a95),LL(0xa34e4a66,0x1fd2a0dc),L_(0x7c1fb322), + LL(0x1bfb2217,0xcad10b11),LL(0xca3e4b04,0xc1ab5423),LL(0x00ce7df7,0x9c84d269),L_(0xa1f31fb1), LL(0x8a721b19,0x45e72360),LL(0xf80ab752,0xa59a87d1),LL(0x38935f38,0x36966819),L_(0xebd1b2a6), + LL(0x657a1f10,0x28dbeb1d),LL(0x4f41e2ad,0xafbe6ad3),LL(0x1c0782c2,0x60b0cb1e),L_(0xb609d7c2), LL(0xae37eb9c,0x403c7b8e),LL(0x3d308948,0x374fa1b2),LL(0xd8503866,0x0326fd01),L_(0xd472fecf), + LL(0xa5b224c1,0x98673100),LL(0x35c900a8,0x7e857bdd),LL(0xdb5df675,0x63415d28),L_(0x6b676a73), LL(0x04cd81e5,0x3e21afd0),LL(0xa33929eb,0x2e610def),LL(0x717e43a4,0x6bf414b5),L_(0x9c7a3c47), + LL(0xd475f817,0xf9ea4384),LL(0x6b738d34,0x671b40ad),LL(0xac317163,0xd197efee),L_(0x2438948a), LL(0x66a989c1,0xebcc3f73),LL(0x1fc5f207,0x7913fae7),LL(0x9fdbdbb6,0xddc39008),L_(0x663a5b4e), + LL(0xe781f5a2,0xa670dec4),LL(0xaeb71ef5,0xb1196ff6),LL(0xa4973f38,0xf981cb1a),L_(0xe1b496e0), LL(0xcfa69883,0x8185f4e4),LL(0xdcd9014d,0xfea1489d),LL(0x01c7c910,0x33bfde50),L_(0x0a1d6d42), + LL(0x0b6d7bc6,0x0c74186c),LL(0xf93c6e65,0x21c134ed),LL(0x5c4b9d40,0x9b7f55b4),L_(0x62194b5f), LL(0xf0967474,0x07af0f2c),LL(0xead94ac6,0x43b53483),LL(0xfc351f8f,0x6a02d6dc),L_(0x96c34c95), + LL(0x8a77d8e2,0x06eb2a9e),LL(0xb29f094a,0x5a50f6db),LL(0x8cc86e58,0x40844e37),L_(0x2f8aef05), LL(0x3ed6ee27,0x864a532f),LL(0x8ade6f97,0xbafa3f05),LL(0x78f8e230,0xadc126ed),L_(0x8c70fd00), + LL(0x61f69a9b,0xecdb5c0e),LL(0x3c35a3f6,0xac81a7f8),LL(0xe0f4235e,0x49ad3737),L_(0x162c0cf0), LL(0x9bbaae92,0x6d262e32),LL(0x3611d970,0x92ea2e05),LL(0x56f9b02c,0x569e559b),L_(0xd700ade3), + LL(0x1248f104,0xab2f2393),LL(0xf45846e5,0x2e1305dc),LL(0xaa46d2c4,0x1222d842),L_(0xf5f64176), LL(0xfa36ca68,0x1b067c95),LL(0xb8990097,0xbdbafd4f),LL(0xd2f67212,0xe2b50ce3),L_(0x7ac5ce8c), + LL(0x2f610f1e,0x754843cd),LL(0x77e241fc,0xeb4f1114),LL(0xa0d9cbeb,0x55cb5b7b),L_(0x98136c68), LL(0x4f9041bc,0xef262223),LL(0x523e661c,0x38002545),LL(0x30661570,0x3e7c7958),L_(0x5ea1ccfb), + LL(0x43726846,0x736e6684),LL(0xa762a548,0x4ea61054),LL(0x4ea95a8f,0xa90f1dd7),L_(0xf1a7ecac), LL(0x052ff525,0xdb175bab),LL(0x176f58b3,0x113349c6),LL(0x66d2f2dd,0xf30e4d83),L_(0x047e3e70), + LL(0x3053884b,0x2523bfbe),LL(0xdb1ab97e,0x32d988c1),LL(0xa7f43905,0xed30067f),L_(0xe0063138), LL(0x315aac55,0x0fe3afe0),LL(0xef7cd5f9,0x25c62d9c),LL(0xbceebc63,0x7475ff1c),L_(0x37aef5f2), + LL(0xd9f33d01,0x21d9dd88),LL(0xfe303f84,0x0e4f3aea),LL(0x403a10c7,0xdc250e4e),L_(0xcf99ada4), LL(0x20afdb06,0x16293016),LL(0xdf77e00f,0x21ef71a7),LL(0x017aea99,0xb51701db),L_(0xb9a51f04), + LL(0xb223220b,0xaefa0a3e),LL(0x3c6160d1,0xd7af87c7),LL(0xb6871dd0,0xcb336211),L_(0x30015933), LL(0x605c6481,0x87d9707a),LL(0x18fd4882,0x367d0209),LL(0xf138a307,0xeeb3123d),L_(0xce3f684e), + LL(0x1ad5ce5f,0xdb674dfd),LL(0xd843e892,0xc8958516),LL(0xedfd1c19,0xc7988b87),L_(0x3bdd2895), LL(0x9b80b679,0x09bd2af2),LL(0x294ff4e2,0x3150d204),LL(0x2ad4535f,0x2fc2c26d),L_(0x1ebd13d4), + LL(0x0284d812,0x0bdfba7b),LL(0x94e42bd5,0xabd7b483),LL(0x512c1be7,0x10be5ddc),L_(0x6bc6d702), LL(0x0bf271cb,0xd21e8820),LL(0x636561c6,0x0b1793f9),LL(0xe4ef4a35,0xda161673),L_(0x3d69a37a), + LL(0xf8513215,0xc206f7b0),LL(0xa0f11bca,0x9944badb),LL(0x23eeb3f7,0xe6050ed7),L_(0x98889901), LL(0x2ce1ebe3,0xc76ba3ec),LL(0x4edc3ee6,0x3e534941),LL(0xaab7f4a9,0xc6e7d7b5),L_(0xf6442856), + LL(0x6d4746dd,0x5b3638e6),LL(0xd78b290a,0x6b3e0d8f),LL(0x77aef8ca,0xfac422d4),L_(0x0feb906d), LL(0x3bc30209,0xdc0b9001),LL(0x331327a2,0xe68c779f),LL(0x5d146cb4,0x6fb5da21),L_(0x3f887bb5), + LL(0x13fd6d90,0xb80d181c),LL(0x2be6d108,0x288db1c9),LL(0xb342ddca,0x7004d85b),L_(0xbe7db7ec), LL(0x0ba4c981,0xc1d565da),LL(0xb82063a3,0xaf5c7b86),LL(0xaa90bf00,0xbf4ed46e),L_(0x3a15f177), + LL(0x7f0b547c,0x49eb74f8),LL(0x146e10ea,0x95205278),LL(0x1256c788,0x346ea27d),L_(0x3f761b6a), LL(0x6a1a4872,0x4ec42ca7),LL(0x150f9515,0xec53ab18),LL(0xda268f4d,0xe31f64a6),L_(0xe4706b89), + LL(0x331bfc4e,0x0b4a9c48),LL(0xc8ae2f2d,0xa61d3700),LL(0x58cc7b40,0x3e9cb1a4),L_(0x31facd26), LL(0x10a69f57,0x3b34d496),LL(0xd018498f,0xb4f2bc1e),LL(0xcbad3ed9,0xa8867a65),L_(0x0fb7952a), + LL(0x060d829d,0xe832b583),LL(0xa4bc1e98,0x376e4266),LL(0x4ca3a485,0x2da2b54a),L_(0xbbcd02db), LL(0xce23e38a,0xd59bf323),LL(0x02c44a56,0x71d50f01),LL(0x366f36f1,0x0dbb411d),L_(0xf30d32a2), + LL(0x000bf15a,0x52b2e706),LL(0x19599554,0x83c05edb),LL(0xe133dbc2,0x659be3d5),L_(0x90ee1c4f), LL(0xe38dcd45,0x70cd081f),LL(0xba72fb3b,0xc7e5be8d),LL(0xff5688ad,0x3ae40925),L_(0xb751810a), + LL(0x7b159833,0x071a076f),LL(0x01bb4b01,0x67a90721),LL(0xd05cc7e4,0xc3b59e15),L_(0x14e1dc86), LL(0xbb61bd7a,0x1ee66adf),LL(0x743d3ae3,0xdf5f0047),LL(0x75487d37,0xb272a24e),L_(0x18ad2d25), + LL(0xecfd1c78,0x69935fe6),LL(0xefec1497,0x0149938a),LL(0xb3e61f40,0x1edeaf36),L_(0x2b4abccf), LL(0xe6be1d5b,0x6554f5f2),LL(0x0fe91c0b,0x443c9e6c),LL(0x613c238b,0xf8ba68b0),L_(0x0b0ae516), + LL(0xed42491a,0xbf0bc611),LL(0x9c608ede,0x7040894e),LL(0xc0f244de,0x0b0057eb),L_(0xd601464b), LL(0xf956aa49,0x825a361b),LL(0x4ff78a4b,0xb7556c66),LL(0x6aed48bf,0x1ae6fb3a),L_(0xba8a9318), + LL(0x87f39f99,0x09e379b4),LL(0x9e14e886,0x3fce44cd),LL(0x2fb4d3e3,0x75711ae3),L_(0x036fce90), LL(0x85e9ac2b,0x2542f70e),LL(0xa4a903d2,0x8303b7ee),LL(0x95852fe8,0x637b247b),L_(0x012141cc), + LL(0x2165af3e,0x9af74288),LL(0x41ab3d86,0x8945cab0),LL(0x29c26ab0,0xcccef77c),L_(0xea5535b0), LL(0x8648e931,0xc11f99b1),LL(0xb7f340c5,0x7dcfe320),LL(0xfc229594,0x8f5c0876),L_(0xf20b5874), + LL(0x48b7cd0a,0xba865a55),LL(0x6fae5df7,0xf611d6ed),LL(0x2fe39b65,0x8329316a),L_(0xc07b24cb), LL(0xdf303162,0xd09a26e9),LL(0x1af7d473,0xb6a75d27),LL(0x375af56e,0x62993ad0),L_(0x957236cb), + LL(0xd5155d78,0x3e2c48a3),LL(0x427f5e06,0x0f15720c),LL(0x25837ce2,0xcae65a88),L_(0x966e8b1e), LL(0x78a5dd4d,0x9e77f90a),LL(0x306bd895,0xd3918029),LL(0xd4ed09b8,0xa2585522),L_(0x47a8a9ab), + LL(0xd18d6ff6,0x1e6ed174),LL(0x8c957fe6,0xfb94845d),LL(0x54818bf4,0x2495a8a9),L_(0xdd977b92), LL(0x5993b230,0x05a5967d),LL(0x2647b9d2,0x3902c0f8),LL(0x53df8e0c,0x496697af),L_(0x5247a639), + LL(0x7b7c8590,0x8424e43f),LL(0x682c252e,0xf452918a),LL(0x0bb6bcff,0x17581f41),L_(0x040d50d1), LL(0x91b07886,0xd070dc3a),LL(0x686cbe64,0x9e8c81ee),LL(0x3485fd98,0xdfd534bf),L_(0xf0aec283), + LL(0xc3e77fdc,0x4458831d),LL(0xbfa63a77,0xdcd45b1b),LL(0xe6ad6640,0xeffc865e),L_(0xa2922966), LL(0xf55dbb61,0xae34f5fe),LL(0xec39a211,0xcad5ffd6),LL(0x5d5de65c,0x1ffcbe06),L_(0xae549602), + LL(0xc0237c46,0xf7d9eb1a),LL(0x008bf1c7,0x578b9779),LL(0xdce3e9ef,0x7341adb2),L_(0xe6aba225), LL(0xbfdcbb1f,0x52473c28),LL(0x58df1458,0xcea6114e),LL(0xcbca23e4,0x64a68f69),L_(0xdc93a2bf), + LL(0x2ad64bbf,0x12f4285d),LL(0x289cd434,0xe31b4490),LL(0x2614d391,0x5132c5c4),L_(0xd24f8d84), LL(0x0a9c7374,0xb496fb6f),LL(0x945d4d6b,0xaa619fb0),LL(0x64d58ea0,0x226ba20e),L_(0x78fadda9), + LL(0x328976b5,0x15c734ef),LL(0x71f5ab8b,0x49d5ba93),LL(0xd644b74d,0x8a5a08f7),L_(0x5b9eaf3f), LL(0xcd33e0a1,0x40783f59),LL(0xfc35396c,0x91276e7f),LL(0xdf4e44eb,0x698ace20),L_(0x85de5335), + LL(0xbe2724e6,0x4ca6d828),LL(0xc01805fa,0xab7f4b0a),LL(0x9b659d62,0xb0bfec7f),L_(0x21ba53df), LL(0xcb83946a,0x3ca3ffdb),LL(0xaaee16f0,0xe089ddef),LL(0x2829c022,0x8266c12f),L_(0x570a8b63), + LL(0x7f62cdfb,0x27c7435e),LL(0xc2862c3c,0x56a78e59),LL(0x577136cc,0xeb454a5e),L_(0x41bf3bc1), LL(0x76737cb9,0xd5ff2a5c),LL(0xce2b3472,0xa9a9af2c),LL(0xabfed878,0xdae3ae7c),L_(0x076254af), + LL(0x8125e260,0xfbee92e0),LL(0x288eeccf,0xeb8bf90b),LL(0x9cfb5c57,0x059c298f),L_(0x53598af7), LL(0xc5e99b56,0x888186ae),LL(0x3084705f,0x696f873d),LL(0x86f1d164,0xc59e084e),L_(0xa44fd074), + LL(0xeb77883e,0xd2343b34),LL(0xa40a6bf8,0xc2267f63),LL(0x798ceee2,0x27b865e6),L_(0xe4570eb1), LL(0x60596c23,0x8f87d684),LL(0xf0ce5967,0x06984779),LL(0x66b1f5c5,0xcbcc053f),L_(0x2f8869b5), + LL(0x7620dc06,0x0d7b69de),LL(0x17f0c4dc,0x9d39bfa6),LL(0xa20e8614,0xabbfb492),L_(0xc7dc6a2c), LL(0xe61a635a,0x2759d405),LL(0xcc56f093,0x966b6ab3),LL(0x049b5aa5,0x4e1f566b),L_(0xbec87c76), + LL(0x57acbf9f,0x40e840f5),LL(0x1f1c6ca8,0x0ffd84c8),LL(0x2d21e27b,0x9ca6123e),L_(0x1d5e2c46), LL(0xda38f612,0x5531352e),LL(0x009d49a6,0x3be34fe2),LL(0x43dffb38,0xd020075d),L_(0x2f8a3d38), + LL(0x89bc38a6,0x5ded0d34),LL(0xd70c960d,0x82054d24),LL(0x08b78436,0xbcbebb9e),L_(0x662ab949), LL(0xea6945f0,0x1d7a378c),LL(0xcd6abb3b,0x2b017a9f),LL(0x991364b4,0x4dbff4b2),L_(0x6e088465), + LL(0xca476359,0xf7fff479),LL(0x8b812163,0xe17baaee),LL(0xad939c3e,0xd700d28b),L_(0xe8837ac3), LL(0xbb3a964c,0x26712ef2),LL(0xd89edbf6,0xe0390b08),LL(0xeaa73bdc,0x536ef396),L_(0x19770554), + LL(0xb20e2aec,0x7f3d231f),LL(0xe904178b,0xa6924ffd),LL(0x7ff297bf,0xad6fdd69),L_(0x6deef5b5), LL(0x6c6f7f42,0xda650bfd),LL(0xc9fb4677,0x702cfe74),LL(0xd99a935f,0x93d1a532),L_(0xbfdd1320), + LL(0x49e4811b,0x0e07db7d),LL(0xfed77923,0x0dbb53c0),LL(0x0f05670c,0x112a7ade),L_(0x6e79da66), LL(0x3436a7c3,0x201e8aa2),LL(0x567d0dd8,0xd3bae1e2),LL(0x9a0b34dd,0x6a730912),L_(0x5c656713), + LL(0x8ee826b6,0x9522c2ac),LL(0x1cf99a2e,0x6bbbf4b2),LL(0xbfb9ae13,0x1fe851c4),L_(0x91215bb4), LL(0x8e9f0b90,0xbf89747c),LL(0x6286cdba,0x0e20c83e),LL(0x792aeb4f,0x25b00fa8),L_(0x61f48d5b), + LL(0x5762ff25,0xa1ce4a2b),LL(0xbe45da68,0xe3b7ba58),LL(0x63ee425c,0x98b17978),L_(0xd26e1784), LL(0x8832d22b,0x5fcc8cf0),LL(0xecb80c24,0x54bb743c),LL(0x3e21c95d,0x1df7f32b),L_(0xb9cc6c4a), + LL(0xc522b190,0x61766f4c),LL(0xa5778295,0x2fc3322e),LL(0x34307914,0x57f4defb),L_(0x71c105c8), LL(0xe02ecefc,0x2096768e),LL(0xce51bf21,0xad887721),LL(0x3117821e,0xf475048e),L_(0x2b073db2), + LL(0xa7c80b61,0x7067894f),LL(0x4426eaf6,0x99c9ba1c),LL(0xd1c53be2,0x68f854e3),L_(0xce0157db), LL(0x5b23fe91,0x5112a49d),LL(0xa0471f83,0x97b550cf),LL(0x87e0a2d1,0x9f64998d),L_(0x344b223b), + LL(0xb77380be,0x3ad984a7),LL(0x3bd4bc94,0x216f1748),LL(0xc17ca930,0x4aee7a72),L_(0xfcd30768), LL(0x02746210,0xa1f41f76),LL(0x6fdb5587,0x670f4013),LL(0x3c2763bb,0x0145b604),L_(0x6e18b2a2), +}, +/* digit=16 base_pwr=2^112 */ +{ + LL(0xca8e1890,0xa72d343b),LL(0x006aff79,0x07464235),LL(0x1ec739c9,0x136d5ba5),L_(0xce9ce64b), LL(0x721071fb,0xb11ea220),LL(0x7bfe943d,0xfb7789dc),LL(0x57dad905,0xac86bd3f),L_(0x5eeba8a6), + LL(0xac243d83,0xb0b49013),LL(0x31f50c04,0x6bbaa790),LL(0xd8fab554,0x1d543cec),L_(0x2fb0a285), LL(0x15602d83,0x3256cf69),LL(0xf3bd35dd,0x4a5d9d40),LL(0x8a264f24,0x7dff3a00),L_(0x24830923), + LL(0x4400cf66,0x3267fa61),LL(0x7dfb1e52,0xfe343741),LL(0x34e86055,0x552bcdda),L_(0x131f5b56), LL(0x6065029d,0x40256e8c),LL(0x61212776,0x6fac4167),LL(0x3807bd7c,0xbba74ace),L_(0xa2ce6ab3), + LL(0x954e93a7,0xfd51954f),LL(0xd0706483,0xbe73a498),LL(0x89215ea5,0x45e8a87b),L_(0xf3cf4080), LL(0xcdccb303,0x85c8552d),LL(0x09212e02,0xdb8ab951),LL(0x10c2badf,0x19c8db39),L_(0x7b08661c), + LL(0xdafc0f6f,0x9e15fa46),LL(0x148c6f0e,0x89a68d97),LL(0xdb00031e,0x4cc0f2bb),L_(0xc3df9d3a), LL(0x340e9460,0x1ff47bb7),LL(0x620b581f,0x7ef8908d),LL(0x57588030,0x4277ac9d),L_(0xfc122be2), + LL(0x5631310d,0x1336d2ba),LL(0xcd5bed69,0x039396d8),LL(0xedd85963,0xac1f77dc),L_(0xea62aa43), LL(0x40cf73fd,0xf970f801),LL(0x25fc53f3,0x7927be46),LL(0xf9d1654e,0x9f9753cc),L_(0xb61c3720), + LL(0x1f400c41,0x96dca6f7),LL(0x6461f973,0x29585547),LL(0x1f57a9e0,0xb90ca0df),L_(0xb010f7d3), LL(0xd1cf0605,0x21188094),LL(0xf21e0d4d,0xcfd8c7ab),LL(0x5f3468a5,0xde0d0f24),L_(0x4e99f334), + LL(0x0e472341,0x70d9012f),LL(0x7c2bf741,0x876213ff),LL(0x47c7e115,0xb309c07a),L_(0x977513df), LL(0x7682fd32,0xbe1ac3e8),LL(0xb7e947d5,0x1c6b708a),LL(0x09c3181e,0x41e0c220),L_(0x8b26dfc1), + LL(0x4c02bbfb,0xfa332320),LL(0xbb0f1fc0,0x1dd657b9),LL(0x68034aed,0x784ebda5),L_(0x58fb05ab), LL(0x0677d37f,0x32cca1fc),LL(0x83e2269c,0x2c9407c6),LL(0xea0312b9,0x33fa100f),L_(0x1761c87e), + LL(0x40a8cd2a,0xf4f00cb5),LL(0x58c8e52b,0x05e2b3e0),LL(0x99489e99,0x5593b9ee),L_(0x7924f02b), LL(0x464ae656,0x0334e2ed),LL(0x2a2bf206,0x55ec8d2c),LL(0x41ac277d,0xa7b08209),L_(0x0c5c176d), + LL(0x3ac595c7,0x1aabcb3f),LL(0x1450434d,0x93706f44),LL(0xcf82a49f,0x0e6c721c),L_(0xec192907), LL(0x92676891,0xfecc2ffd),LL(0x451e72f3,0xc8f49751),LL(0x81bf062a,0xe8852ddc),L_(0xf84db443), + LL(0xb1dfeb18,0x32a2e886),LL(0x8b6ac261,0x074b9898),LL(0x3c40d7a6,0x1ea4ca4c),L_(0xf7653f1d), LL(0x6940eab6,0x2b2347ff),LL(0x56338adb,0xca0ffd59),LL(0x314b0e58,0x8e442705),L_(0x79527139), + LL(0xbe4dbd9f,0xd63c1700),LL(0x1101a669,0x9b252cda),LL(0xd06390a9,0x501cc189),L_(0x0fb428cd), LL(0x8dcdeb0d,0x0a3108aa),LL(0x47aa9e0e,0x71a44ce5),LL(0x9c9d8b17,0xe86804cc),L_(0x61ad5482), + LL(0xfe987606,0x4432954f),LL(0xebe782ea,0x4d668855),LL(0x11e7dc71,0xf9379d6a),L_(0xfb5f9f2d), LL(0xebb7e729,0x98c8071e),LL(0x3aaef819,0x23ed5017),LL(0x3d2ba5a3,0x0cd97039),L_(0x2ceebe22), + LL(0xf7bd7afe,0x6646c443),LL(0xc3bd60cf,0x242cedce),LL(0x7c4048fa,0x5e74dbe8),L_(0xd29b1f65), LL(0xc5c513c7,0x91eb1130),LL(0xbb7b08f7,0x9edbdadd),LL(0x3c5f214f,0x6e37d0fb),L_(0x85ea7863), + LL(0x46abff2c,0x87af66ad),LL(0x26b2f70c,0x6dfaf224),LL(0x5494f6a8,0xf84408cd),L_(0xa24b6575), LL(0xb3e6fa13,0xfb2baa35),LL(0x7fdee2e4,0x48b82a51),LL(0x82709ae1,0xd231d3ab),L_(0x8897d9e8), + LL(0x9b54a957,0x43e3a795),LL(0xc93592ff,0x7df94193),LL(0x9aebe35d,0x3c64092f),L_(0x1d7432bf), LL(0xd212a26f,0x608467be),LL(0x61dd14ba,0x92d589f9),LL(0x1c6431e0,0xb4aaeed1),L_(0x53076cd4), + LL(0x20a03c9c,0x064ac09e),LL(0x3e9b39b0,0xe7dae697),LL(0xd8bc5839,0xc348b809),L_(0x8ab0974b), LL(0xcf8f0968,0xf886797d),LL(0x9cb572eb,0xeaa08b56),LL(0x9a9ce85b,0xa4a7eaf7),L_(0x0e1605e0), + LL(0x7cf927b4,0xbe868d7f),LL(0xeca36ddb,0xd5b365e4),LL(0x86bfcb64,0x9b8656b8),L_(0x2ab84426), LL(0x82271896,0x01fe86cb),LL(0x15f75f0e,0x672910dd),LL(0x4c2d70f5,0xf239468a),L_(0x365ce9d3), + LL(0xfbc74fe2,0xd342696f),LL(0xe0e930b3,0x1de7ca12),LL(0xcb67c28f,0x163565e8),L_(0x312a9e2c), LL(0x197ec1e3,0x54d76d68),LL(0x3b508871,0x11c4ebab),LL(0x71ee55a2,0x4dfa9969),L_(0x0a025c8a), + LL(0xe7055396,0xc327e039),LL(0xf0b0b5d0,0x5f514a6a),LL(0xb462d687,0xa70b2c73),L_(0x842f549f), LL(0x161da539,0x864e3870),LL(0x0b583d9c,0xde80546e),LL(0x1693a8f3,0x5135df56),L_(0xadecd5ac), + LL(0xc575df9d,0xe8f3d99a),LL(0x04c439d1,0xbba7ca9c),LL(0x79057450,0xed15dbf2),L_(0x557c06cf), LL(0x7e76a8b5,0x09b41a1e),LL(0x0bdc4dcf,0xf356b834),LL(0xf085eab4,0xdab0c931),L_(0xb48d6df1), + LL(0x2605d170,0x4a27c44b),LL(0x2e80ab05,0x5425f3dc),LL(0xdd9e1376,0x242b815a),L_(0xa910e496), LL(0x8b98999a,0x3f111c1c),LL(0x0c5343f8,0x29db22da),LL(0x10be67ce,0xf1c6ef62),L_(0x4ae541a2), + LL(0xbec6e451,0xc5669a27),LL(0xff0e3e5f,0xabcdf52c),LL(0xe911fae7,0x85cd9be2),L_(0x4150b30c), LL(0x9e505adc,0xab441ae6),LL(0x92888eaf,0x3ad89345),LL(0x0dd62a7e,0x1b7b37eb),L_(0xe9e642a8), + LL(0xc78817e4,0x15d73eae),LL(0x70f4b3ee,0x4965b1f3),LL(0x70ff29ef,0xe8787785),L_(0xe5dce106), LL(0x6012d2ef,0x36b7ab1a),LL(0x21a140c0,0x943b6ec7),LL(0xffda816f,0x9bd299e0),L_(0x5968ca97), + LL(0x6cb8a41d,0xb0d9d8cc),LL(0x6b2cc953,0xa58bcb0c),LL(0x932fd885,0x8fc62139),L_(0xb782380e), LL(0x5794203a,0x83692f14),LL(0xb2525747,0x8e1b841e),LL(0x2213b2d6,0xc34bb49f),L_(0x0270ba84), + LL(0x4fa6af14,0x3b925a7e),LL(0xe649f779,0x23113463),LL(0xa3785af0,0x20ce9dd7),L_(0xdc5dbf9f), LL(0x3c0ce37f,0xa5d677f4),LL(0xcdc8d1f8,0x793a5749),LL(0xd8d3e9cd,0xb698eb06),L_(0x5cbd0981), + LL(0xfe510f94,0x1e752015),LL(0xf090a1eb,0x65fa8e80),LL(0xfc5f6857,0x111a1741),L_(0xbaf89d9f), LL(0x4cc335c5,0x1b1a1910),LL(0xbfa9551b,0x24829f96),LL(0x24a70dba,0xc5ef4f06),L_(0x17130b26), + LL(0x1febf14a,0xfcb5b042),LL(0x911935f4,0x2a9fcf44),LL(0x53ed6fc0,0x90634826),L_(0x4421e383), LL(0x98983bff,0xc4143e7a),LL(0x12ccf145,0xfc4163c9),LL(0x5bf8b6c9,0x4f765081),L_(0x73883a6d), + LL(0xef044be0,0x997c821a),LL(0xe367cf8a,0x3560d6df),LL(0x3d0455b7,0xe8c7bcd6),L_(0x92643ab4), LL(0x26239fb1,0x547dbeb1),LL(0xb5294491,0x2538d97d),LL(0x07450e45,0x472a47f1),L_(0x5e3d3784), + LL(0x23d0e3a3,0x6372566a),LL(0x4d62de9c,0x9858907c),LL(0x7a323978,0x78458eba),L_(0x4b2a04ff), LL(0xae2094af,0x6819a84c),LL(0x0eccbf7e,0xe0c337be),LL(0xb93eec58,0x2a02a086),L_(0x03b76658), + LL(0xbfc94c0f,0x6aa0f607),LL(0x0cbee27c,0x4123775a),LL(0xae1d1601,0xbbef8dbb),L_(0xe7149786), LL(0xf56932af,0xbe686d03),LL(0x91ce3c48,0x3e33d4ba),LL(0x3d6535b6,0xa72c8f85),L_(0x2c9fb75e), + LL(0x4a47050e,0x39dd7c74),LL(0xeebc8c18,0x2f4a6be4),LL(0xa2f0fb3a,0xba0f68a4),L_(0x1c5cad2c), LL(0x85d6ef4d,0x3f59abfa),LL(0x3a302769,0x47cf5f7e),LL(0xed44c056,0xcd7fd1b9),L_(0x2e974b80), + LL(0x691eccfe,0xc8f85c87),LL(0xf503e456,0x34a4ce86),LL(0xb53e74a3,0x8eb76194),L_(0xc4c55423), LL(0x22da99fd,0x153fefba),LL(0xdbbf4e12,0x5438e1b0),LL(0x80b3f694,0x65187422),L_(0x9f8c212c), + LL(0xab8f55ff,0x0fcf5a38),LL(0x6da81ce6,0x0b057c12),LL(0xdff4777f,0x4a9fa32b),L_(0x602640c6), LL(0x2624de9e,0xa3b241c6),LL(0x2a545f4f,0x0dd2feaa),LL(0x4cf94266,0x8a894ff1),L_(0xb5260296), + LL(0x2194d8b6,0x7320211c),LL(0x391c4f62,0xd2de2551),LL(0x6d039386,0xbaa68de6),L_(0xc3a5b977), LL(0xf3b1162b,0xbca0bb5c),LL(0x3643a502,0xd1a3ad93),LL(0xe1f5b66b,0xa430cadc),L_(0xb32e7ba4), + LL(0x4c8d2ee8,0xd36efcf0),LL(0xb6b2c682,0xc22137cd),LL(0x1024c516,0x8688f303),L_(0x99e728ff), LL(0x3fb84765,0x777d2fde),LL(0xd972734f,0x9a00c6af),LL(0x25251454,0xf34097fd),L_(0x424404cf), + LL(0xdfdcead3,0x3079239d),LL(0xb5ff46f3,0x80c77858),LL(0x36a3735f,0xc9986723),L_(0xd572091f), LL(0x6645ed98,0x0dabffd5),LL(0x0da45042,0x4b0c6093),LL(0xf6ffaf50,0xbffef0a3),L_(0xbc330b9d), + LL(0x7d149925,0x3b961737),LL(0x8d236f27,0xc18d27e0),LL(0xa5f014ae,0xee0ae6b2),L_(0xe50d6b9c), LL(0xc1b92639,0x2b5f008e),LL(0x36e3aea7,0x8996a3a6),LL(0x2f8e4e4e,0x5ccc02df),L_(0x9f683f7c), + LL(0x3a45a099,0x7caa8790),LL(0x6c2cdda1,0xf82f4e65),LL(0xa9cdf2c5,0xb634f00e),L_(0xd5eec010), LL(0x4012fecf,0xa6a18c31),LL(0xde736bf0,0x153b6c27),LL(0xc1059186,0xb7a29e2b),L_(0x9b23923d), + LL(0xc90ac0c7,0x7000c5a2),LL(0xb80400e7,0x2f6fc4cf),LL(0x147ccc10,0x9e3b070e),L_(0xfef094f1), LL(0x0ab36cd7,0xefcc38e0),LL(0xf5b77a62,0xa70d4a46),LL(0x2c523e42,0x02efb3f7),L_(0x210639da), + LL(0xda4106f5,0x330449c9),LL(0x7b06f635,0x30d85516),LL(0x9efabcec,0xe879f10c),L_(0x3bd206ba), LL(0x75c081fb,0x0312bdf3),LL(0x88e64b25,0xb372be76),LL(0x3a8f90eb,0x3e832f84),L_(0x14f38ac0), + LL(0xc7331aa0,0x4bafc9dc),LL(0xb6461cb6,0xae9ec59c),LL(0x266ba391,0xf40aa6ef),L_(0x4715355a), LL(0x33ae64ed,0x1adab377),LL(0x716f224c,0x04c2b6cd),LL(0x32b71c3a,0x7e89026d),L_(0xccbd7f4e), + LL(0x095925d5,0x7de43130),LL(0x0bbfe1df,0xa8605297),LL(0x10701822,0x0bc2e01c),L_(0xf68b86eb), LL(0x816f32a8,0xad3f4af4),LL(0x67e192cc,0xe9dc6c49),LL(0xd5fef609,0x52b649b2),L_(0xf428af30), + LL(0x9f45ede9,0xe78176a5),LL(0x0b536242,0x0ba30aad),LL(0xdbdd8b20,0x503b274a),L_(0x51151bfa), LL(0x4d7ac01b,0x393d4fa5),LL(0xdc4e6926,0x8e9d7c61),LL(0xe68387c4,0xaaf0a35e),L_(0x06168aa4), + LL(0x35b79ca0,0x2143ebc0),LL(0x4be98451,0x85664008),LL(0x1016322a,0x5db7c140),L_(0x8229e729), LL(0x4e10aefd,0xc19bdf57),LL(0xc8df52e4,0x07fc2589),LL(0x89b605d1,0x2a200cdd),L_(0x0c94090c), + LL(0x02211b9e,0x4588f75f),LL(0x0c217b21,0x90c9218a),LL(0x90f7c009,0xef37649d),L_(0xddfaf799), LL(0x35885f85,0x3d0294b0),LL(0x510fffdd,0xe646d7e9),LL(0x4a21ba9d,0x6c9d2cfa),L_(0xa7ada78e), + LL(0xcf1bb365,0x1db377d9),LL(0x63e01d3e,0x8b74ddb8),LL(0x8c9ad4b9,0x6bb31b8b),L_(0x2b26fec2), LL(0x858449ed,0xfeaf9d41),LL(0xe2652b65,0x700223fb),LL(0x03b48402,0xee2505a9),L_(0x2438e370), + LL(0x9e8ea415,0x900098e5),LL(0xd8a6e8e7,0xe478cc9a),LL(0xd8452a93,0x2e2cd25b),L_(0x569bbc5f), LL(0x5d22bbf4,0x80e8caa9),LL(0x3f76b251,0x10dc6eaf),LL(0xeba89ffb,0x18acb601),L_(0xf763dfc0), + LL(0xfed0a316,0x24370f95),LL(0xf5f4a36c,0xc06ba373),LL(0x527f5322,0x43417948),L_(0xe0b12bf9), LL(0xa7bf2afb,0x6aa04127),LL(0xb3183162,0x87eaa68e),LL(0x1070f9c1,0xe002ccee),L_(0x5b0c2b4a), + LL(0xc7a653c7,0xfca649b9),LL(0xb223ef3b,0xf66cb526),LL(0x4aa0b0e6,0x10592f0a),L_(0x3d9b366f), LL(0x61473660,0xab61a041),LL(0xaa721fec,0x1b173f43),LL(0x0a96a721,0x06be6fd2),L_(0xb3b3cccc), + LL(0x87a37531,0xfda2de36),LL(0xd8576cc3,0x94a1bbc9),LL(0x22b5be7f,0xaad70136),L_(0xc40c9235), LL(0xfb07dd6f,0x9edbb744),LL(0x501faa81,0xccf5dbd3),LL(0x60c831a6,0x6f69ca6e),L_(0x813a83b7), + LL(0xa96bdf09,0xdbe3e245),LL(0x4f871623,0xacfa6c70),LL(0xd2c645ee,0x18c48541),L_(0xc037f50e), LL(0xcb36c731,0x08342ef7),LL(0x68cc0cb0,0x4e89abf4),LL(0x0c4fa826,0x186c66ac),L_(0x8f28f4b4), + LL(0x2d46a77a,0x837493ce),LL(0xf41079e1,0x15f6dc1a),LL(0x1a861704,0x0e538567),L_(0x03e474ae), LL(0xa4564b51,0xa188c7ea),LL(0x76610ad6,0x3c36716c),LL(0x706eb273,0x8025f126),L_(0xb5435d24), + LL(0x28638fa1,0xc4faeda2),LL(0xd9ddd6dd,0xcb83b696),LL(0xe2be7ac0,0x4eea6f06),L_(0x2e6abaae), LL(0xaad7f53d,0x99f8dc45),LL(0xc8502a73,0x8477e1b0),LL(0xa0aa0199,0xcbc5ed2a),L_(0x38689f49), + LL(0xd7981750,0xfbd4773e),LL(0xb3548b14,0x01d933bf),LL(0x18cea1f5,0xb3c65f79),L_(0x3bc435fc), LL(0x3414a439,0x0706d138),LL(0xca5b9f90,0xb2166606),LL(0x14b87e77,0x864343f5),L_(0x8a82e1fd), + LL(0xa2f61a19,0x18cc1c82),LL(0x0026972b,0x0e88aaf9),LL(0x1da022c6,0x2265f4c6),L_(0xb898d6b7), LL(0xc0e9ecae,0xa53eabdc),LL(0xef23c0bd,0xd1e853a6),LL(0x98e844f5,0x05f9b046),L_(0x7d2e87fd), + LL(0xf9a4fe94,0xe7c2033d),LL(0x1c9b4b3f,0x3037306e),LL(0xf61d0ee2,0xdb5834bf),L_(0x28b5943b), LL(0x4c22b3ea,0x5de6ece6),LL(0xf70c350f,0xa4e1deaa),LL(0x1c3bfb3c,0xd4c8be79),L_(0xe75564bc), + LL(0x16c64cef,0xab2cc7d7),LL(0xa4b41a9f,0xa5534f63),LL(0x912b7248,0xe6362cfb),L_(0x6a214d2e), LL(0x9c7005e2,0xc25479e5),LL(0xf1652743,0x6a7f2578),LL(0x3bcb10e6,0x9be1e8e0),L_(0xbede8ea6), + LL(0x459cf3fc,0x31630308),LL(0x706d16a7,0xf60bf227),LL(0x9f2f55c9,0xed0db8c6),L_(0xaf0d2787), LL(0x6dd2c868,0xb9e7f043),LL(0xce8001ee,0xa2ee914c),LL(0xb8247bef,0x658e5458),L_(0x78f60572), + LL(0x6bc3d716,0x5c6843cd),LL(0xacaada1c,0x1bd0e72e),LL(0xcabaa3a2,0x36370b66),L_(0x507a17a3), LL(0xebae3c4d,0x481b026d),LL(0x48821bb8,0x4cd76456),LL(0x5ed8e5b9,0x78f46113),L_(0x6817009d), + LL(0x029887e4,0x7560c538),LL(0x75119a00,0x92989cd8),LL(0xe00d03bb,0xef71ab39),L_(0xb863df63), LL(0x5f6dd06c,0xfedea07c),LL(0xddc4f0d5,0xd3b8a96e),LL(0xc43e8f08,0x36d08634),L_(0x9efb1f02), + LL(0x3215c001,0xba05b2e5),LL(0x949d9990,0xa3503f0e),LL(0x76e4e629,0x6c281fa8),L_(0xdccad172), LL(0xbcf0bc10,0x6b13eba6),LL(0x70f4d75c,0x1b941283),LL(0xd34756fd,0x70986d34),L_(0x8f71b0bd), + LL(0x2576d875,0xe10a7afc),LL(0x0a9b3e69,0x411dab1f),LL(0xf0719c41,0x046cc610),L_(0x9d1611b2), LL(0x1ba334cc,0x6b7a1ff0),LL(0x24d9cfba,0x47e8a2e5),LL(0x5bd30854,0xa740e80a),L_(0x1159e67f), +}, +/* digit=17 base_pwr=2^119 */ +{ + LL(0x7c2ca549,0x5898fc38),LL(0xa3566efe,0xa757b52e),LL(0x24bb02bf,0x91ab4316),L_(0x00e46ee0), LL(0x9ec42857,0x8c02881c),LL(0x1d20ad25,0xb9a28540),LL(0xb01ce79f,0x40d6ae86),L_(0x40b2c06e), + LL(0x4fd7f402,0x259779c5),LL(0xafc1d993,0x3aa4fd10),LL(0x86dcdc83,0x77170d32),L_(0x8441f4f3), LL(0x9892b126,0x46144a81),LL(0x58df739b,0xc6549963),LL(0x344ddb0c,0x01761502),L_(0xd9bcbeb6), + LL(0xaf8e61b1,0x66de5a2d),LL(0xbc92b6dd,0xcb8a0bac),LL(0x1127f258,0x3d18ed03),L_(0x3982ee92), LL(0x8ca9b244,0x0deb6e5b),LL(0xcf198f32,0x7cefc1f3),LL(0xdbaf1a5d,0x373c1801),L_(0xf524b8bb), + LL(0x746fcfe2,0x1bb01e23),LL(0xdc5e2b63,0x38352329),LL(0x5df11c5b,0x68e597b5),L_(0x44ec319c), LL(0x84b36d32,0x0e27f5b0),LL(0xe5706d53,0x06c3753d),LL(0xe67b3281,0xa7111b5d),L_(0x8fa5ee02), + LL(0x5141455b,0xe429fedc),LL(0x0035c84e,0x353b3d90),LL(0x2ea6f135,0xa3e74746),L_(0xb9e352c0), LL(0xa8f82237,0x220bb555),LL(0xb02520ea,0xecc581df),LL(0xe4b94f8e,0x16dbb87f),L_(0xfbff49d6), + LL(0xc2eb5b9f,0xda91cf48),LL(0xdb33cda7,0x821b171b),LL(0x006b3e24,0x489cbfa3),L_(0x4461fe49), LL(0xc4e7389d,0x4d93f4e1),LL(0xfe8e5310,0xf28f63e5),LL(0xf3d0d29d,0x94deaf7c),L_(0x3d33e2ad), + LL(0x1e948480,0x453d6838),LL(0x8f1b71e2,0xd842ed96),LL(0x92302500,0xf76bc681),L_(0x17e6ad54), LL(0x5d04302d,0x66dcecf9),LL(0xe97810bb,0x37c278e5),LL(0x13de9f06,0x30c9e7f2),L_(0xf16b622f), + LL(0xe415829a,0x436d6b94),LL(0x794d49e6,0x541c7bda),LL(0x702ede17,0x5afaacb1),L_(0x3b5ca68c), LL(0xc9496ea8,0x9ca7b73f),LL(0x50a1e66e,0xc5ce627a),LL(0x762e4f6f,0xc90185b1),L_(0x6705418b), + LL(0x52188496,0x52070a5e),LL(0x9065afef,0x510e288b),LL(0xe6df0c7e,0x441709b3),L_(0xa1396423), LL(0xa8018a37,0x66868b26),LL(0x14f84f84,0x28ce78c1),LL(0x1fa90793,0x19c083f9),L_(0xfd8d88ee), + LL(0x2458c92b,0x5330a979),LL(0x77ea9ee9,0xdba786fa),LL(0xce00962f,0xae5f476a),L_(0x05915824), LL(0x1219b9cd,0x8cdf0300),LL(0x660a19e4,0xdc707120),LL(0xc70e0b11,0xfdd0563d),L_(0x41d445a6), + LL(0x6102cfd1,0x53d54a0a),LL(0xa5729cf8,0x4fc590fa),LL(0xea907455,0xea0c5a91),L_(0x59a871ce), LL(0xb4020096,0xe025ffb0),LL(0x7e7fb9d6,0x140025ac),LL(0xb7954f0e,0x46ca7ee6),L_(0x010dbf77), + LL(0x6a28fa6b,0x2d7e46eb),LL(0x034b69ed,0x1ae23996),LL(0x51bc6b67,0x3c25755a),L_(0xe9bf6717), LL(0x579d75fc,0x06b03b1f),LL(0xfc3e6ce3,0xe0b2026f),LL(0x723756d9,0x58938357),L_(0xb4731ae9), + LL(0x5cbab1e2,0x42ddfb70),LL(0x33bbd9fd,0x6d346512),LL(0xeb7aa22f,0x725fdf2d),L_(0x88eb52e0), LL(0x48e63519,0xf1fc169f),LL(0x297a7972,0x91aa8dc7),LL(0xb92c9671,0x9c2b4849),L_(0xa557aadd), + LL(0x4fe2181c,0x58182a19),LL(0x27b6bd7a,0x0c4b58a3),LL(0x1c72e55a,0xd4f2f098),L_(0x6b53e542), LL(0x0bdee531,0xb5e11022),LL(0xa8f7b191,0xa8bb8315),LL(0xe62460d4,0xbdf84150),L_(0x74ea071f), + LL(0x1ca83a09,0xe03c9ae4),LL(0x54fef132,0x7e2ef688),LL(0xa900e0bd,0x1a602274),L_(0x3698f6bb), LL(0xc6654794,0x57e365a9),LL(0xbb500025,0x2b39b18b),LL(0xf16e71f6,0xa0dfe4bb),L_(0x9d6899d1), + LL(0x1d2f288c,0x8e577078),LL(0x8e833532,0x8ebb1ef6),LL(0xfb9fbcd0,0xf5ead3c9),L_(0xe31f0c20), LL(0xf16fa294,0xc462ce8b),LL(0x6466c045,0xc1068645),LL(0x85e7447b,0x3e1abc03),L_(0xbba798a0), + LL(0x41e55979,0x122d19ea),LL(0x158f4312,0xfd5b3495),LL(0x297c5747,0xa7b750fd),L_(0x735aa3f7), LL(0xd2156123,0x312d6128),LL(0x31e8bfa3,0x4456ccf3),LL(0xcc9cdadc,0x5094cff5),L_(0x9ea9beca), + LL(0x1ea96169,0x616ee979),LL(0x78257a1a,0x60b3f06d),LL(0xec98f98d,0x609d2d2d),L_(0xd1b3e514), LL(0x1100f9ce,0x2e59e232),LL(0x8087f780,0x34e1a9a4),LL(0x0b40b853,0x95c2865d),L_(0x8fa97627), + LL(0x297f6423,0x1eac58fa),LL(0x7eb748d9,0x6fa36840),LL(0x0d0b673e,0x955ae146),L_(0x72be4e8e), LL(0xb20412e2,0x22b08d06),LL(0xa4616bf8,0x57a2dcba),LL(0xc55b68a6,0x42138fbd),L_(0x8fa529ef), + LL(0xdb26dfa2,0xf1766b07),LL(0xdfd72d39,0x050e85e5),LL(0xad6b741f,0x59466485),L_(0x5fceaf74), LL(0xbc9edaeb,0x0c6b2254),LL(0x453c9eae,0xfca1477c),LL(0x9c00d050,0xa3817166),L_(0xa87cf553), + LL(0xdbbc12a2,0xad146ff5),LL(0x39990110,0x33dcec4a),LL(0x3d48a808,0xfe9fea01),L_(0x8c4aac12), LL(0x2cefb34a,0x0ad5c79b),LL(0x0a9a20f7,0xaca81076),LL(0x942b38be,0x9475c474),L_(0x218031b5), + LL(0xc173f9cf,0x4c0dcf94),LL(0xcc4c491e,0x61d12afb),LL(0xfa790f55,0x12a6c289),L_(0x37abf7be), LL(0xd04d2ae3,0xcb586fd2),LL(0x192166df,0x3c04c5b4),LL(0x7ee1f233,0xbba9c768),L_(0xb69b46cc), + LL(0x7c1e6077,0x1443ddcc),LL(0x6345a210,0x70d81228),LL(0x5f035127,0xc8273e38),L_(0xdafd6230), LL(0xcf227ac1,0x4e794379),LL(0xb3e2d215,0x916e6825),LL(0x138aca38,0x4afb3539),L_(0x4ce64d55), + LL(0x463d181b,0x6b581764),LL(0xc440084f,0xa9bac44f),LL(0x088767cd,0xf0d004bb),L_(0x94e1ba30), LL(0x55abae1e,0xa1a18412),LL(0xa732c539,0x6d71f955),LL(0x896710ea,0x3608f75d),L_(0xa067bb07), + LL(0xbc080af4,0x9891098f),LL(0x65c3a1bc,0x00befda2),LL(0x3321e866,0xb04ac43d),L_(0x438e5b9d), LL(0x5fd8c4d4,0xe260d024),LL(0xc4dcba5f,0x19886f71),LL(0x5e5f5f4c,0x37b1a0c5),L_(0xd3c102f6), + LL(0x58ee2dea,0x4e2d8d4c),LL(0xe6546299,0x8042631e),LL(0x262d0cb2,0x2df96d37),L_(0x349292cc), LL(0x04a3c0d5,0x2fddedd3),LL(0x452a0a52,0xd5418050),LL(0x2293a069,0xc6053861),L_(0xfe3040eb), + LL(0x3a50a7c9,0x02a166d5),LL(0x861af22b,0x8a435126),LL(0x449c202c,0xb053bb62),L_(0xbb664aff), LL(0x9083b8c2,0xd6f78a70),LL(0x14a57ed2,0x8e5943f4),LL(0x6c5eedb1,0x591d0e00),L_(0x0f079d9e), + LL(0x2036fb32,0x4ee2bac6),LL(0xc62978b5,0xce8c9892),LL(0x0ee684b9,0x56d30723),L_(0xb8993d5e), LL(0x2fee5c92,0xfc172755),LL(0xc8622a7c,0x43c48f91),LL(0x4279a0bd,0x78cfee0e),L_(0x3826553c), + LL(0x4315c335,0x9060272c),LL(0xe29ecb65,0x5d0bbf12),LL(0xdd976fea,0x53d3e99f),L_(0x278ebed1), LL(0x7fc69ec0,0x7a33fe7e),LL(0x13d33afb,0x9b8a7301),LL(0x19d16dca,0xe09a47ad),L_(0x6e0ed440), + LL(0xd0327c5a,0xb55799e4),LL(0xf145e8f9,0x7408bf29),LL(0x97e6d3d7,0x4580043d),L_(0x5f599a51), LL(0x517f2eda,0x38414d94),LL(0xe09d35b1,0x37f611a1),LL(0x31551461,0xce073290),L_(0x1044bab9), + LL(0x24eb973b,0x045e586d),LL(0x4d8f2d61,0xa3318071),LL(0x1d2b5b32,0xb68b7f81),L_(0x10bccde6), LL(0xcd3e0e0e,0x0578898c),LL(0x1e460079,0x927cc37b),LL(0x31bad31c,0x3d22477a),L_(0x8c371b62), + LL(0xa374b8d9,0xbc19767b),LL(0x3ed7db29,0xa97161c1),LL(0xcc843c60,0x45c30a38),L_(0x508e3e86), LL(0x433b6dbe,0x1457fcb6),LL(0xd8c78825,0xed4ed464),LL(0xc3997703,0x2b88e6d2),L_(0xa1f2140c), + LL(0x953f63bf,0x2a452961),LL(0x62444dca,0x6a62e296),LL(0x2d1e976e,0x1d95f51c),L_(0x2bbeb1e0), LL(0x62f53f4b,0x052daa65),LL(0x827170a2,0x5cb51b06),LL(0xc319b1bb,0x3a8ec55d),L_(0xda0c72a5), + LL(0x71e73523,0x898419da),LL(0x35d9c257,0x49a774fb),LL(0x55758231,0x4e49ddae),L_(0xd5c3703e), LL(0x4d4edb63,0x28d3f23e),LL(0xa38dc679,0x41807b47),LL(0x749ea0b8,0x15724faa),L_(0x68bb7f4b), + LL(0x5af98b88,0x0f227def),LL(0x24af0afb,0x27396d04),LL(0x1cd97cef,0xa2ad5165),L_(0x678f117f), LL(0xce22032e,0x03432db3),LL(0x5cb93ad2,0x92eb23b1),LL(0xcc96e20a,0x47d3d48c),L_(0x5f1e84c2), + LL(0x163136e0,0xc4293aee),LL(0x7f0db6e5,0xca0a189e),LL(0x9069c8c2,0xa088318f),L_(0xf4210d55), LL(0xb0d80206,0x13bd2b27),LL(0x34f3150f,0xcd0efd69),LL(0xe152eb4e,0x0fcf1527),L_(0x32e6885b), + LL(0xcb923f09,0xed8906b9),LL(0x7eba0ca2,0x95cab4ce),LL(0xc91dbf27,0xcf44aa02),L_(0xf0948c8c), LL(0x1a9b31dd,0xbe7b7487),LL(0x88539394,0xc84b72c8),LL(0x938fd6cd,0x15b31f6a),L_(0x1a1c6b6b), + LL(0xce67988c,0xc2869ed0),LL(0x61ab3c9f,0x882c2379),LL(0x3bb8c353,0xb69781cd),L_(0x95d9d719), LL(0xedcef064,0x3de229ea),LL(0xc3895533,0x84ff6e10),LL(0xb6955022,0x5ac0d29e),L_(0x6e2ba9f7), + LL(0x3cc87419,0xfe2a24c3),LL(0xe3406a2f,0x3d474db8),LL(0x67beb236,0xf6867bb0),L_(0x5b2cd123), LL(0x74e31d6b,0x2ff473bf),LL(0xed6d88b5,0xeadd2ad4),LL(0x046a1f51,0x9b3025d4),L_(0xf4a61d8b), + LL(0x40eaccf0,0x99e19903),LL(0x7e9f5e57,0x1b67fa53),LL(0x00b84a60,0xbabc0faa),L_(0x6de1d67d), LL(0x06fe7a95,0x04041366),LL(0x06672f11,0x5f522ae3),LL(0x0dd51cb4,0xd4012aa6),L_(0xd5ce357e), + LL(0xfbc38722,0x538e45da),LL(0x5f595162,0xf57a7cab),LL(0xe917805d,0x56a8bf52),L_(0xf639993d), LL(0x31ba0865,0xe876b4e9),LL(0x3eaf3c58,0xb08987af),LL(0x0ef4538a,0xbd9e742b),L_(0x95404cfd), + LL(0x9eb63b57,0x1453ad21),LL(0xf6e400ed,0x627008e7),LL(0xa4ababfc,0xd78f1c8d),L_(0x4ebc5852), LL(0xcc21b23c,0x4e1ef090),LL(0x7415270e,0xb8b3701a),LL(0x921cd6ee,0x472b1f2f),L_(0x765126f0), + LL(0x59e9fe85,0xade7ead3),LL(0x1b413797,0xb8da2677),LL(0xeabbca1c,0xdb705e2f),L_(0xa1e12595), LL(0x65e95e7d,0xd1990099),LL(0x79fa6f2e,0x2b7f7bbd),LL(0x4d4d9e0a,0xbde588b7),L_(0xc0ac97e6), + LL(0x20ff9acf,0x23cf39f7),LL(0xca0fac28,0x1f72fbb4),LL(0x4b179b83,0xb04c01ae),L_(0xfaca0fdf), LL(0x7173749c,0x7b3cc769),LL(0x3d2bf7fd,0x0cbf34b6),LL(0xf3f2fea4,0x6369c1d9),L_(0x866e7f8e), + LL(0xb873e85f,0x6eb23ad8),LL(0x8b9475ca,0x2855072f),LL(0x22d9fd01,0x4e9a3adc),L_(0x813270cc), LL(0xfbf27bf5,0x292b8bdc),LL(0x3db7f857,0xe9f00375),LL(0xf6ba5b62,0x1b8a570a),L_(0xe20966e7), + LL(0x05a5a71b,0x19b8e7ac),LL(0xe13277a1,0x9a4abf1f),LL(0x6d26fe41,0x8f8b7d68),L_(0x0ce42976), LL(0xaa0c781d,0x48e32e14),LL(0x0ee98d37,0x13c9b527),LL(0x3bb6c2c9,0x4db74aeb),L_(0x208cd0d9), + LL(0xc2685698,0x47ef78f2),LL(0x6c1b0959,0x9bb6bd13),LL(0x2ccd97f7,0xa303c278),L_(0xf9abf5b1), LL(0xafdb9d40,0x0276fd8a),LL(0x2be1d396,0xdb26cd04),LL(0x726beb4a,0xe09517f7),L_(0x4e0b499e), + LL(0x6f715206,0x5979197a),LL(0x82ffa270,0xb44c8267),LL(0x52cb678a,0x260e7e25),L_(0xc541f700), LL(0x18439449,0x33983071),LL(0xb1d2f59c,0x19d945b4),LL(0x11eef46b,0xf2b04fe3),L_(0x9fba737f), + LL(0x214431d6,0xbcadb425),LL(0x4d1d9a21,0x941b766d),LL(0xe2a1f314,0x2170c6a6),L_(0xc15b9466), LL(0x6c1b9a4b,0xd605b2bc),LL(0x1578a3c5,0x65c9904b),LL(0x96b6befc,0xbe0b58ef),L_(0xc4f7e33c), + LL(0x5a93994e,0x3fcf43e1),LL(0xb19e11a6,0xf5f793fc),LL(0x5ad2ed77,0x6ff2a142),L_(0xe587c79f), LL(0x23c53b77,0xb2ee2d92),LL(0xa6fa95d1,0x023b2c75),LL(0xac5ec115,0x67b40c94),L_(0x41c55249), + LL(0x05d09dcb,0x6b551c3c),LL(0xc6c2dc20,0x2e81c56f),LL(0x1b5233cb,0x0e6faafe),L_(0x4565862d), LL(0x43368b7b,0x08edb0b1),LL(0xb2d31417,0x29c77be9),LL(0xa0fa599c,0x77052491),L_(0xb2ac3ec1), + LL(0x3a2c0bb0,0x0302d5f3),LL(0x04cf0b83,0xfda301ea),LL(0xbd701c39,0xfb565431),L_(0x5eee993f), LL(0x2230bcaa,0xb5c3681a),LL(0x5d285d50,0xa9dd4df3),LL(0x8618542b,0x8c1d7c52),L_(0xd80996cc), + LL(0x85ebe7cd,0xf0fe462f),LL(0x12c3c8ab,0x752eb1b0),LL(0xb6349271,0x2f148e5a),L_(0xc0513e4e), LL(0xdaa709f7,0xd043e9e0),LL(0x340f7e04,0x7842c3a1),LL(0xb4acedf4,0x5d6d0c14),L_(0xfa56de6f), + LL(0x807fa877,0x435b6be9),LL(0xdb7a9959,0x65c2c690),LL(0x24fae090,0x3cd0982b),L_(0xcc274d62), LL(0x721b08d7,0x02f3dea6),LL(0x2a9bf937,0x052c0001),LL(0xadae4c04,0x61de7f6d),L_(0xba3c48a3), + LL(0xdfe50f8c,0x2faaf032),LL(0x7e8bb31a,0x51dbadf4),LL(0xd937aa83,0x072c01c9),L_(0xbfe57ab1), LL(0x5d66c435,0x87c5d208),LL(0x4ce95747,0xfb211069),LL(0x7ee66e9b,0xf01cc54c),L_(0x48478c99), + LL(0xb35f128c,0x957c7ee6),LL(0x50dc83b5,0xf963430c),LL(0xf0f63a80,0xb4578760),L_(0x842ee70e), LL(0x57d57cbd,0xc891093b),LL(0x24bc2867,0xa548c5ff),LL(0x0f85fa2f,0x47bb6a4a),L_(0x18cd7a2e), + LL(0x813aaaa5,0x40773919),LL(0x798cdd8c,0xb6d9db4e),LL(0x68858372,0xd2c5ede9),L_(0xb240c94c), LL(0x79a51f60,0x62fa25b7),LL(0x89e94f52,0xac054f6f),LL(0x0332347b,0xf207ad5e),L_(0x9e15c0d4), + LL(0x19041ef2,0x6d748e07),LL(0x298a62fb,0x3b9bdf40),LL(0x8f7fb37f,0x35f944a9),L_(0x92926ab1), LL(0x51a4355a,0xdc488cfe),LL(0x3f355ad1,0x814db3fc),LL(0x84b0d885,0x29d8cb57),L_(0x98d39682), + LL(0xfe4b50e2,0x07c8c799),LL(0x4957a527,0x3418ab11),LL(0xec94fd37,0x601424db),L_(0xf147c20f), LL(0x420c1861,0x1adc80ee),LL(0x7d9440bd,0x09ee96a9),LL(0xee7355ab,0xd1adde68),L_(0xba4d5d2f), + LL(0x14379251,0x4dc18b12),LL(0xebadf0ef,0xfee19a43),LL(0xbc8d4cc9,0x8da37771),L_(0xd555d58a), LL(0xd71b4107,0x42fdc281),LL(0x2e05719e,0xac68f304),LL(0xdc3ed2bf,0x4c1b070d),L_(0x03356492), + LL(0x0c94988c,0x8da99bff),LL(0xa1e580f6,0xe79a8b42),LL(0xcf6d6c6e,0xfdabb4f2),L_(0x80f7da41), LL(0x1a4121f1,0x6a7dfe84),LL(0x36467882,0x6b06eea9),LL(0x30e07e6a,0x943abc5d),L_(0x8ae444d2), + LL(0x2f87568d,0x2441fb99),LL(0x0599179e,0xe8a9e90a),LL(0x54669582,0x47767b19),L_(0x6fa66e89), LL(0x19290f64,0x4596bd1b),LL(0x93f0e068,0x9dab54b3),LL(0xe4e3af20,0xff7a84e2),L_(0x951ce962), + LL(0x0b49df59,0x466ca295),LL(0x16afa866,0xc6b06127),LL(0x1f38aad5,0xf4e1677b),L_(0x672e7252), LL(0x29732848,0x3a4e5076),LL(0xfcdff811,0xcc6ceb5b),LL(0xfe6adcac,0x8a4243ce),L_(0xb04666fc), + LL(0xe74fffdd,0xc1e32f38),LL(0xd93030cb,0xf9561e97),LL(0x15f3f603,0x39bbca36),L_(0x90c8c914), LL(0x3734c1a1,0xa53fc681),LL(0xb0c4475f,0x158ec0bb),LL(0xed39d189,0xa57cf95c),L_(0xd562a81e), +}, +/* digit=18 base_pwr=2^126 */ +{ + LL(0x5cfdbf76,0x02c6b679),LL(0xace24453,0x8195dc3a),LL(0xa58b929b,0xbe2f8e64),L_(0x9a3a7efb), LL(0x9f7af479,0x7fa4b042),LL(0x8ad04303,0xb747bbcf),LL(0x426eb7ce,0xa67998c5),L_(0xdb040e40), + LL(0x7d551504,0x3bfd76b2),LL(0x0361658a,0xa3a2cf8c),LL(0xa3b9f1bd,0x66db4cfa),L_(0x4799c86b), LL(0x27afe24e,0x01c04fa0),LL(0x0689618b,0xaa0383bc),LL(0x5b052124,0x524802bb),L_(0x25748110), + LL(0xfd9c3522,0xb542168a),LL(0xf1dd3dc0,0x435f3e05),LL(0x3556cca8,0xf3b21a79),L_(0x0d1838e7), LL(0x0a1d87d1,0xbbfa360f),LL(0x43bd0f7a,0xdcacc277),LL(0xfdb15d31,0xa293f5a9),L_(0xf9285692), + LL(0x960b1fe7,0x7da35372),LL(0xefe3b9e7,0xb15275ca),LL(0x08b0fe30,0x9c0da2e3),L_(0xcef9bd5d), LL(0xfc9156e7,0x087c3eea),LL(0x4386ea9f,0x13e94876),LL(0x5543dce0,0x057ad0af),L_(0xc5162840), + LL(0xf9c98a4a,0xfeb313d1),LL(0x398fba1b,0x0e02712b),LL(0x52f53c92,0x98909181),L_(0xfa320278), LL(0xebd66101,0x906d44a0),LL(0x427688fa,0xc8c39420),LL(0x1c7282db,0x3901fb30),L_(0xaea2389a), + LL(0x6775078d,0x1424671a),LL(0xd8f3ae0f,0x4ebcdd7f),LL(0x80279f72,0xd5a77238),L_(0x602ce4df), LL(0xc083918a,0x12e9402d),LL(0xdd095d1b,0x02858419),LL(0x6cb15ad2,0x0927adfc),L_(0xa38c1285), + LL(0xfc6fbaf6,0xec392752),LL(0x1ec20f01,0xeed1ebd2),LL(0x5cac318e,0x47520f8c),L_(0x407d9032), LL(0x8ec0f4fd,0xe2f3dac1),LL(0x43dd79ec,0x35a74b40),LL(0x9236a81d,0x6bccf29e),L_(0x00ac0a42), + LL(0xc4755991,0x163c66e0),LL(0x849061a0,0xdd950bf5),LL(0x70208a9e,0x7bf95257),L_(0x07a10de2), LL(0xb0813c25,0x45c2891a),LL(0x15468f60,0x450f087f),LL(0x73d5ebe9,0x387e234b),L_(0xa32ac9f8), + LL(0xe8ff8e91,0x962ab070),LL(0x570fcd62,0xd3ce94dc),LL(0xb1757c3e,0x1ad55d1b),L_(0x7fa592e8), LL(0xa8ad3f09,0x76fcda7d),LL(0x73d3b392,0x348ab2da),LL(0xd6cf8e99,0xca16fca1),L_(0xac456eec), + LL(0xc7486909,0x5f929900),LL(0x240b92b4,0x27ccfc23),LL(0x0f7d51a1,0x632731fb),L_(0xf12e9902), LL(0x735b1778,0x47671b3c),LL(0x985803eb,0xec4d7c8a),LL(0x63e1c219,0x608cf2d5),L_(0x109df03c), + LL(0x09438c99,0xb25c71f5),LL(0x0829cedc,0xbe60921f),LL(0x483d485d,0x15305f6f),L_(0x60591120), LL(0xba32c7ab,0xbcdea716),LL(0xbeeee884,0x2b65df3c),LL(0x88e6ef43,0xc3d80182),L_(0x4b2dd1d3), + LL(0x1bdc3884,0x482c3679),LL(0x6b421466,0x6f172c00),LL(0xa58b570f,0x2587649f),L_(0x3e4ee4bd), LL(0xef99f5b4,0xf5a1388a),LL(0x98494de5,0x5a8f54da),LL(0x8f25a37c,0x50e9cc72),L_(0x1ce8e0f0), + LL(0x72cca43e,0x119587d3),LL(0x8e668562,0xa3b06524),LL(0x0554888c,0xd291f0fd),L_(0xae58bc85), LL(0xd1139d9b,0x48a506c1),LL(0x034f448b,0x04ec28f0),LL(0x94dd32bc,0x2bd2abc9),L_(0xb9e588a6), + LL(0x4988c3b7,0xe1e13901),LL(0x24b16627,0x4231b66d),LL(0x4f52ca62,0x749a749b),L_(0x8bb11e14), LL(0x03150154,0x3dfafe07),LL(0x71bdac04,0x73419c6b),LL(0x40bd5232,0x1587ab8c),L_(0x7d2d4a5e), + LL(0xf231ad9a,0xe7ffdae6),LL(0x0f11fde4,0xa13a1e41),LL(0xc78059f3,0xf4713c18),L_(0x2d2f0ff7), LL(0xfde9d338,0x5fe86c6a),LL(0xa381beae,0x9d423777),LL(0x734b9092,0x77f56796),L_(0x82131b3f), + LL(0x3a2c7153,0x612650a1),LL(0xc29b399c,0x72e7ea2b),LL(0xd673cbc5,0x60425f21),L_(0xece9a8ea), LL(0xfd08bc21,0x394b09dd),LL(0x1b69fef5,0x58e594d5),LL(0x141fc2ff,0xeab0baec),L_(0x85999248), + LL(0x5c8c18cf,0xf4b4d1ab),LL(0xe1f44cb9,0xdca5073e),LL(0xd86fbb1f,0x32861f51),L_(0x242811d3), LL(0x038cf05a,0x20b39e70),LL(0xc64c1332,0xd6198042),LL(0xeb68ae09,0x93c66ba1),L_(0x7aee0eb8), + LL(0xc4c03565,0xf47a18f7),LL(0x992398ea,0x808ebaad),LL(0x7414fdc9,0x325c8a8e),L_(0x42a7e664), LL(0x0900ad27,0x9f053caf),LL(0xc8869d15,0x2b52f07a),LL(0x1878089f,0x11f347a4),L_(0x19ee0c05), + LL(0xf8345a24,0x3e51ac6c),LL(0xb254a9d3,0x57f4ec7a),LL(0xe2b48c1d,0x916d1214),L_(0x3732c0ab), LL(0x5045ad37,0x0cb5ec90),LL(0x010093ba,0x1eea394a),LL(0x4a4cfff6,0x4aaf2a44),L_(0x7feda10c), + LL(0x370622e3,0x58625a1a),LL(0xea895fc8,0xe2d5d70a),LL(0xcac0e4d3,0x53bc48b9),L_(0x1d43df08), LL(0x681301c8,0xee75a210),LL(0xe1a8bb95,0x097c56b0),LL(0x7e7b6ce5,0x4a384a6d),L_(0x26d0db83), + LL(0x3ebba7eb,0x2bb95cad),LL(0x9e42faba,0x5398a161),LL(0x0f33c612,0xf67c4703),L_(0x6ef93df3), LL(0xbb0fa5e1,0x8ad685b2),LL(0x8655685c,0xb1c226c7),LL(0xdd2eb2bd,0x5e194627),L_(0x523e0fd5), + LL(0x3d42f37f,0x301fcb42),LL(0x80fb78c3,0x647b20c7),LL(0xfaa1d374,0x3d0a850d),L_(0x051bb6db), LL(0xbe9d92c9,0xc7fc5cad),LL(0x38f0078b,0xae641b58),LL(0x91a544af,0x0c9540c5),L_(0x6cae5443), + LL(0x33b2702e,0x93ebe56e),LL(0x71946cb7,0xff835a5d),LL(0xc82366c4,0xbfe6dc77),L_(0x27a97fdc), LL(0x2bb71ad5,0x80d9e527),LL(0x16d42ed2,0x21ce1d14),LL(0x18abf494,0x0398f8d3),L_(0xa190c5d3), + LL(0x87b49d63,0x495ac575),LL(0x6bf10350,0xb50a8115),LL(0x2674f39c,0x90ac41f9),L_(0xaedc843c), LL(0x20e79ef7,0x60ca4b9c),LL(0x2b7ba708,0x67e85483),LL(0xc070ae10,0x2e4b0764),L_(0x0c3aa1f7), + LL(0x9b6c96e8,0x555cdffb),LL(0xd2310fa6,0x1116970b),LL(0x42f99112,0x9d84f404),L_(0x8d7bdc74), LL(0x452face1,0x43e3cff2),LL(0x1c149edf,0xf602e240),LL(0x442dde2f,0x1288dcc9),L_(0xb3089de8), + LL(0x1ad5fcdc,0x3fe61e5f),LL(0x20fa93a3,0x2e793d8a),LL(0xb50f92b7,0xbfce0907),L_(0x632ab7a5), LL(0x8a0e5e0a,0xc89531e2),LL(0x6f57401d,0x4059a329),LL(0xf7bd6eb0,0x5942987b),L_(0xceffd87f), + LL(0xacfc439f,0x74161b3d),LL(0x647fc617,0x3b17df59),LL(0xac7f6cc8,0xc485b2c8),L_(0x0414a8d3), LL(0x69c118ce,0xd7de72d0),LL(0x1d6064d4,0x1eff6d80),LL(0x9fd6a735,0x8ee9cb9d),L_(0xf104d7a9), + LL(0x252fb3dd,0x776f7b1a),LL(0x2b058af6,0xb6f63fd9),LL(0x0d06abe2,0xb0461b36),L_(0x90f6c358), LL(0x5c9d58a0,0x9a72e438),LL(0x24e0b184,0xce596569),LL(0x6facf219,0x5f6753e5),L_(0x5f23a218), + LL(0x87cb85e1,0x4fd2cc25),LL(0xadb7e3c5,0x53c5cae5),LL(0x4cc67b4e,0x4c80d589),L_(0x6a578011), LL(0x4f08551a,0xc04163a3),LL(0x41978a17,0x52d7161e),LL(0x4701d626,0x7acb9d4e),L_(0x7040eb22), + LL(0x25c4f3fa,0xb117faf3),LL(0x734cd26d,0xb2bfac77),LL(0x071d299e,0x1ad0de2a),L_(0x826417df), LL(0x1049bb39,0x487c15ef),LL(0x5432a8df,0x74da9484),LL(0xa5c29870,0xe2f76737),L_(0x89d1f699), + LL(0xe58a9a06,0xe366cc79),LL(0xcd96f1e7,0x497f4836),LL(0x19aa24f5,0xbe89a16c),L_(0xea6d5b99), LL(0x4105a3ce,0x1afa0bd4),LL(0x8da4ca09,0x4aa54b7f),LL(0xb278c556,0x07697581),L_(0xea0b2435), + LL(0x69260167,0x233ddda4),LL(0xbce451f2,0x055176b1),LL(0xe21ca9dd,0x33fc9573),L_(0x608477ee), LL(0xb1111c08,0x37f4a8b6),LL(0x2f253ba7,0x60c86d48),LL(0xec1f4a44,0x3ae7c4bb),L_(0x1c649260), + LL(0x418877ba,0xf186ffa7),LL(0xca88dd8a,0xe435d41d),LL(0x2e8c5c07,0xd65638d7),L_(0x8d28689a), LL(0x50763ccc,0xa2ef4b71),LL(0xfa5e89e0,0xbab1be9a),LL(0xea4b66ad,0x2a17ea7b),L_(0x5744fef5), + LL(0x511ae7ab,0x1c88c915),LL(0xd377e6af,0x55991086),LL(0xbd38c04f,0x43c62134),L_(0x7e11eacb), LL(0x715ff436,0x3aa19a12),LL(0xe65b9643,0xc019b147),LL(0x0069d240,0x7d7a8265),L_(0xc6fc4f7d), + LL(0x00db9dd8,0xb94a17c8),LL(0x95308d83,0xb2b6edac),LL(0x88fbad84,0xd14a8ca8),L_(0xcec36252), LL(0x7824a283,0xf71bccba),LL(0xd90035ec,0xd172d468),LL(0x55a73d3d,0x07f0cdbc),L_(0x6c21797b), + LL(0x3b33cd4c,0x5162797e),LL(0xa87e9702,0x6fc60b78),LL(0x2ab93ff3,0xe2a31c19),L_(0x4288abf4), LL(0x297e2ccf,0x80be51bb),LL(0x388680ed,0x058de4bc),LL(0x8dedbe92,0x7da1a829),L_(0xbffcb08c), + LL(0x7008c033,0xac1e2c00),LL(0x1d6fb3ed,0xd70668f0),LL(0x9f1545b9,0xfb03795c),L_(0xbfecfd6e), LL(0x83ab63e2,0xac0e74d0),LL(0xcbb46331,0x481489c9),LL(0x01849565,0x79c317d8),L_(0x9186b0db), + LL(0x1b42ef85,0xe3b9e5c1),LL(0xa44cb510,0x4bcaad24),LL(0xc72bcdea,0xd47f22e3),L_(0x23ac79ad), LL(0xffacee46,0x1062a849),LL(0xd5193e66,0xfb9adffd),LL(0xbb25e4cf,0x600c0353),L_(0x5ab3c0ce), + LL(0xd8f9674f,0x1f36e1c6),LL(0x2a414d7c,0xc2f2d11a),LL(0x23202a91,0xbdd27859),L_(0xd77095b4), LL(0xe447fde6,0xe5a99ae1),LL(0xd0fffc9e,0x5835b80b),LL(0x490dac48,0xacfdbe9c),L_(0x9079d090), + LL(0x65c70a3a,0x550c3e82),LL(0x448361bb,0x40b9aa3e),LL(0xfeab6ae4,0x41969a52),L_(0x74461138), LL(0x37a53426,0x47381f18),LL(0x2be030b8,0xd6e199e3),LL(0xb091abda,0x4da4cd53),L_(0x7df8f1e3), + LL(0xb6274829,0x9fea92c3),LL(0xab2fcd01,0xcf88fa14),LL(0xd804cba2,0xed8709e8),L_(0x19141717), LL(0x820090bf,0x88fb9ad9),LL(0xe105506b,0x370d09cc),LL(0x8cb2c13e,0x4e6bda23),L_(0xcaba970f), + LL(0x5ad6bf8f,0x9fcd4871),LL(0x160191eb,0xa47d5c67),LL(0x826242d1,0x99634c20),L_(0xdfe7289d), LL(0xbd0af1f7,0x4e07eba8),LL(0xc0910d55,0x18abee83),LL(0x7e042c87,0x6d7f5229),L_(0xd74d6e82), + LL(0x4747a07d,0x6efbc0e2),LL(0xcc3c4e8e,0x9088fde2),LL(0xf21ea1eb,0x05c108d8),L_(0x37aae682), LL(0x0d08ac9d,0xf2c37fd2),LL(0x9a51bef8,0x4d3bd6c1),LL(0x5ac834a7,0x2a412b12),L_(0xac985dae), + LL(0xf5833d2b,0x39bd0099),LL(0xb0ef3648,0x886d33b3),LL(0xc1936e74,0x97226670),L_(0x3eb64bfc), LL(0x408e784b,0xfa065545),LL(0x6cb29cd8,0x96246832),LL(0x010fbc36,0x6b24b9f4),L_(0x72e08b5c), + LL(0xaa5c541e,0xe0daf350),LL(0x2fb0ee94,0x86c5cee7),LL(0xac57f680,0xb753971e),L_(0x6b0ccf83), LL(0xbcb76004,0xc3b1be67),LL(0x48b857e0,0x3f637d0f),LL(0x81e2b0c6,0x595ebecb),L_(0x7151293e), + LL(0x9a2c561d,0xb788bb43),LL(0x65e9c6ef,0x6cf04373),LL(0x10062256,0x891a7719),L_(0xe6f377fd), LL(0x9d5d462c,0x7355204a),LL(0xe23556ab,0x08f2e4fc),LL(0x188872ce,0x84eba15d),L_(0xea03bca4), + LL(0x37bcd5fb,0x159bb9de),LL(0x08807416,0x412b6267),LL(0x496eba6c,0x5b5b2f5c),L_(0xff4f34bf), LL(0xd57ab42b,0x376ec0fc),LL(0xe48904ea,0x62f02fea),LL(0x96bf9fdc,0x9b04b325),L_(0x003a468c), + LL(0x85de6fe5,0x3d1be5ab),LL(0x62509881,0xf11ef926),LL(0x65d5f607,0x244caa05),L_(0xc38d7540), LL(0xc1a149a8,0x0b239ba6),LL(0x19b8e994,0x53224bb3),LL(0x1f25f647,0xd2cd0a34),L_(0x7f481820), + LL(0x303ac802,0xdceb71b5),LL(0xf395c0f2,0x9bd5860b),LL(0x6febd95b,0x01d8efcf),L_(0x0b358ed0), LL(0xc95f4540,0xdadfd46c),LL(0xdc5d961e,0xc1f026dd),LL(0x221b24a2,0x6d21a125),L_(0xc0f16b10), + LL(0x03e43472,0x3ece8318),LL(0x54252e96,0x430e9b3f),LL(0x97edd631,0x020e1e99),L_(0x90685aae), LL(0xad7d0431,0xb7771434),LL(0x6df64310,0x9647f95c),LL(0xa647e2b0,0xd6ddd8f3),L_(0x52ff5927), + LL(0xe48eb4a9,0x3fd7f726),LL(0xbb02013a,0xcd6dd255),LL(0x6e17def8,0x5ef8f2c6),L_(0x1b970bb2), LL(0x867ed64e,0x1236fa4f),LL(0xf7a7e274,0x459207e6),LL(0xffa4c978,0x5a8179b1),L_(0x99219388), + LL(0xcb87940a,0x4b407e59),LL(0x844408fa,0xa05e3d79),LL(0x742669b3,0x0751f86d),L_(0xb00b365a), LL(0x7860c6e5,0x3a8360fb),LL(0x4147cefc,0x10209a6b),LL(0x0393026e,0x7a7f6231),L_(0x2e39fa34), + LL(0xe7f6f33d,0x149f6c77),LL(0xbbcd3afa,0x9a9bdb39),LL(0x742b5b3a,0x899f0118),L_(0xb8dc2766), LL(0x573badd0,0x14c00e4c),LL(0x6924c12d,0x190dfe39),LL(0x4e4111e9,0xdf09100c),L_(0x2987ca68), + LL(0xe5fa2136,0x65fad088),LL(0x01cff054,0x9c1c2f99),LL(0x3002769e,0xdb9e3912),L_(0xe9ad46db), LL(0xa7493cf2,0x6e70449f),LL(0x34ab7ab5,0x1ed50609),LL(0xfda4dfd0,0x91982d5a),L_(0x89e4ea03), + LL(0xb358f84b,0x132822e9),LL(0x1ad76044,0x9d9ae4a1),LL(0x3e8a1d95,0xb981b100),L_(0x37f04df1), LL(0x364692e2,0x0b17bd09),LL(0x0b34cd00,0x5d63a307),LL(0x447278ad,0x07bde57a),L_(0x6e16122f), + LL(0xa6d64780,0x3221832c),LL(0x8fc6ce6c,0x3dbd0cb9),LL(0x6582853a,0x9cdf65d9),L_(0x708e9258), LL(0xf6bfe6cf,0x273611ec),LL(0xef43a507,0xa78d98b3),LL(0x86406043,0x88f84624),L_(0x4396769b), + LL(0xbca28b2c,0x25903255),LL(0x6b8a2c7e,0x255e1a29),LL(0x567e51fc,0x06f1e0ff),L_(0xdbcad515), LL(0xf590a256,0xe7fa9d15),LL(0x627c717b,0xe236aabd),LL(0x3efff1d7,0xa592bf53),L_(0xe58ce7d7), + LL(0x47f49527,0x6bc49db6),LL(0x2993ad98,0xc9fd48de),LL(0xcd8ed14b,0x104437d1),L_(0x419b6d76), LL(0x80c19d84,0x5ceae531),LL(0x2692fc9f,0x7b8ba2ce),LL(0x96208c12,0x5f17f19b),L_(0x34a83cc6), + LL(0x35c24843,0x3d238cb1),LL(0x6bb879b5,0x818fcd3b),LL(0xeaa5d8cb,0x137d82f5),L_(0x769e2a68), LL(0x1bcbebc0,0x0ddf279d),LL(0x2ae26372,0xda0b62e6),LL(0x79e90c0c,0x27e773c7),L_(0x3451edcf), + LL(0x8b2767f2,0x65ddd79a),LL(0x34988e35,0x9ab07761),LL(0xd93aac10,0x814a0e96),L_(0x76ec0b96), LL(0x802069a0,0xe6639b9a),LL(0x9143cd9d,0x23a6da3d),LL(0xfb9b096c,0x77b915d6),L_(0x4353464a), + LL(0xc93ba38f,0x103b0148),LL(0x52d3720b,0x9d45cf17),LL(0x3c3a2d72,0x0ddae18b),L_(0x3543af35), LL(0x10ee7a77,0x90846157),LL(0x51cb6d6c,0xa8824c71),LL(0x5541df28,0x9f0df2e6),L_(0xa96cf0e6), + LL(0xdb058979,0x51a147b3),LL(0xbd4c751a,0xba41fdd4),LL(0x5b132575,0x81839452),L_(0x9b0ddfdb), LL(0x785de02d,0x2e14f24c),LL(0x54dad63a,0x052a799c),LL(0xd7c22290,0xdcbca7aa),L_(0x77c13acc), + LL(0xe8859b95,0x4c326ed2),LL(0xe3bb354a,0x57333802),LL(0xa148004d,0xb7008ce3),L_(0x85367d7f), LL(0x2d4febd4,0xe73d1a3f),LL(0x93d871ad,0x1f4f245d),LL(0x7768c969,0xd9c87876),L_(0xb38a617a), + LL(0x085dbc17,0xaf8f400d),LL(0x2a94dc1a,0x93399c4a),LL(0xf8d2e936,0x370e4c06),L_(0xca35bd61), LL(0x0fb73083,0x2c1682bc),LL(0x00b6e1a8,0x4742b84d),LL(0x6d92e1f4,0x9bb1736d),L_(0x3e9b6293), +}, +/* digit=19 base_pwr=2^133 */ +{ + LL(0x2a85b5cc,0x6c35afbc),LL(0xf6f30e96,0xb02b80b7),LL(0x669a8252,0x4dd8bbbe),L_(0xb9b1b311), LL(0x28535ef6,0xbbcbf46b),LL(0xe4d59c99,0x20d09a27),LL(0x63242255,0x877172dc),L_(0x853f9c6a), + LL(0x432984a1,0xd5de71e9),LL(0xe7f42edd,0xb421581f),LL(0x0b9c53bf,0xc8c4ee60),L_(0x82b1e37e), LL(0x51492ea8,0x4a10fc29),LL(0x7502a8c1,0x69016e9c),LL(0x2dc30077,0xcd3e0527),L_(0x2e0b7f76), + LL(0x161f4880,0xc21b4aab),LL(0x95f715ce,0x44db3fe0),LL(0xff0073cd,0xdb4528e5),L_(0xdbc69fc1), LL(0x5161785c,0x8bfc4101),LL(0xe2c44c67,0x22e31d16),LL(0xa7624b31,0x1b45072c),L_(0xebf2704a), + LL(0xfd1ff163,0x540d3ef9),LL(0xe3c8a7e4,0xd48da416),LL(0x39e285d6,0xccac1ee5),L_(0x8d098605), LL(0x09862475,0xaae2485c),LL(0x424cb101,0x4df1fb56),LL(0xd81ebf03,0xe045acd6),L_(0xec4b922d), + LL(0x3fbc727e,0xe3ace3b3),LL(0xfb5f0701,0x39644794),LL(0x1534cde5,0x25904a79),L_(0xf9421b0d), LL(0xd3275fa8,0x0528f7ca),LL(0x8a9c39d8,0xdea09c8b),LL(0xdd383932,0x458a7c5f),L_(0xbaf06aaf), + LL(0x1adf0fac,0x193b780b),LL(0x5abd6eef,0x5042f2da),LL(0xd76b78fd,0x06442ef6),L_(0x0cdf9e58), LL(0xfef055d2,0x8a5c2657),LL(0xfae0a442,0x0bf16e33),LL(0x6c2b778b,0xb5adbc98),L_(0xf374d05a), + LL(0x846ef971,0x0c6f4ae2),LL(0xb657d5f6,0xd2466841),LL(0xabf1920f,0x70a6775e),L_(0x15909e3c), LL(0xcede3a4d,0xe0c73f43),LL(0x563843f3,0x15282f23),LL(0xe15912d0,0x2e80308d),L_(0xd16c8835), + LL(0x767329dd,0xb00d2849),LL(0x96ae8e24,0x10d39711),LL(0x118cc563,0x4ed443bf),L_(0x9157acc3), LL(0x755122ae,0x14b54c12),LL(0x07d6ccc1,0x4fd87692),LL(0x05ec4685,0xf50f2de3),L_(0x1a923061), + LL(0x3480f7d1,0xc692cc02),LL(0xbe07dcf7,0x71999891),LL(0x5f04bf18,0x0d99300c),L_(0xa062deba), LL(0xb9eb55fc,0xa3451ea5),LL(0x02666d4c,0x1acb5351),LL(0x2fc90780,0x564f33fe),L_(0xa2bd47b9), + LL(0xb70680a0,0x6da6ee27),LL(0xdab5f17d,0x76f48c1f),LL(0x36f25faf,0x16a32d1e),L_(0xbe5fdc6d), LL(0xcd1b3276,0xd0d79fc8),LL(0x0ee979a6,0x2e5c89f7),LL(0xc670e327,0x6435ef83),L_(0x55425ae2), + LL(0x8e0dd4c4,0xcf395e16),LL(0xd8954092,0x86079f76),LL(0x516938e6,0xc57f3db0),L_(0xe85d4e50), LL(0x4eabd92c,0x611ea78f),LL(0xfafbe19a,0x8d26fbc1),LL(0xe5ff8f07,0xe41d9e6e),L_(0x9e9c0da2), + LL(0xcfcf1446,0x86e3cdb5),LL(0x8cb41178,0x3fbc43f0),LL(0x5615e2b5,0x74d58d7a),L_(0x4e6b82ce), LL(0x0c36a5a8,0xd4c6f917),LL(0x374bbfd9,0xc3b3ae3b),LL(0x15aadf46,0x363a6cec),L_(0x30dd6649), + LL(0xe325634b,0xe43c3ade),LL(0x120bb9ae,0xb587b6a1),LL(0x73ef55fc,0xbe8a8508),L_(0xbb18ab2b), LL(0xc135e4d5,0x84facee1),LL(0xb277aa2c,0x67fba354),LL(0x742a25ca,0x1cc06cae),L_(0xd0d24d20), + LL(0x3aa2917b,0x0714ac71),LL(0x2ffe0519,0x178aa56a),LL(0x843d58c9,0x7eddcb6d),L_(0x1e60abe4), LL(0x4b221bdc,0xb10fd184),LL(0xccf9c609,0x113f37e1),LL(0xa0d4b627,0xe398ed34),L_(0x7b6aa811), + LL(0x95472b76,0xa960c0c2),LL(0x735c5d62,0xf7232900),LL(0x0f8b2a35,0x54dfa089),L_(0x0b186f7a), LL(0x7962dd9a,0x3e7eb8a1),LL(0xc1614c55,0x0188ff65),LL(0x8c7e1e1d,0x390e7c1c),L_(0xf9b0b2f9), + LL(0x66e2227a,0x11277d69),LL(0xd9cc122b,0x5181f585),LL(0x3f2ab21d,0x2cbd87e4),L_(0x494cfb1e), LL(0xd57e150f,0xae7bbb8f),LL(0x9977c3ac,0xee685251),LL(0x397ea438,0xb224446c),L_(0x7ed9dec7), + LL(0x3c14cf6c,0xc59accd1),LL(0xb5432f0f,0xb6446b96),LL(0x8b88a937,0xc5589b93),L_(0x02d9f9ac), LL(0x007ea9ac,0x8302d1bb),LL(0x43908c95,0x087096dc),LL(0xb05f1092,0x83349c39),L_(0x5061c674), + LL(0xf6734878,0xf0350dc8),LL(0x9371eb01,0x4edf29b1),LL(0xce86a9ba,0x47861a0c),L_(0x9aad7247), LL(0x16443994,0xa559d5e8),LL(0x979093ec,0xa58da55e),LL(0x0fe17efc,0xb512322d),L_(0xae7b8c8e), + LL(0xf66cc404,0x3ad82960),LL(0x9f1ec261,0x07a3082c),LL(0x77a36e96,0x3914c9d2),L_(0x9b1a4daf), LL(0x238844ea,0xec701058),LL(0xcfa77625,0xc70140df),LL(0xe60b3f2a,0x0c5b4abf),L_(0x138b4b1c), + LL(0xa5d35437,0x174442e5),LL(0x1802aa28,0x2aa23227),LL(0x26f71e89,0x4a4c5681),L_(0x7ae33021), LL(0x7106d5d6,0x5d0cad43),LL(0x2893d185,0xe47a47cc),LL(0x7e9947d1,0xc9f7bf57),L_(0xf2c79dea), + LL(0x607d7e89,0x3a838f3a),LL(0xfbe1be02,0xf73959fe),LL(0x229d491d,0x207a3044),L_(0x9f62e957), LL(0xd08dcf90,0x2e1d3c53),LL(0x86b7e5aa,0x229a31a5),LL(0x3fbda02f,0x7c96f0d5),L_(0xbda44f9b), + LL(0x6e006c73,0xf1aeae69),LL(0xae77f574,0x08648f34),LL(0x0423c397,0xf1f4d153),L_(0x47451433), LL(0xcd40d0b0,0x3a807b62),LL(0xe8e961f8,0xa745762a),LL(0xd89c3d2a,0x467e86a7),L_(0x6ac80359), + LL(0xb9798433,0xb36ecce6),LL(0x3b7324a4,0x5be87e7d),LL(0xbb9babf8,0x98c1904a),L_(0xf1e380ce), LL(0x03c9dc36,0x01b08224),LL(0x2eeb6052,0x1ae12594),LL(0xd0d4fef9,0x19322045),L_(0x256a539e), + LL(0xdcff0296,0xc94659b0),LL(0x2f0568f5,0x39aab164),LL(0xed8a79a8,0x50d8c266),L_(0x757c7d99), LL(0x4445c1ec,0x66c509e4),LL(0xb792533a,0x70d28954),LL(0x9e0c5aad,0x34caeea9),L_(0xe5b193ca), + LL(0x8188c9ad,0x195602fb),LL(0x701a9997,0x76ac1458),LL(0x3992cbca,0xe1e8b8e7),L_(0x8ec536e5), LL(0x4d168e61,0x29c79b39),LL(0x29bff6f9,0xca81be8e),LL(0x596ff95f,0x0a0ae347),L_(0x661b4a6c), + LL(0xd0970633,0x8e767a03),LL(0x473ec4b4,0x7b96613b),LL(0xeb4d7d16,0x98a8ebb4),L_(0xc4e28600), LL(0x1487660b,0x6beb5ab7),LL(0x1ef6cbd4,0x71463757),LL(0x83dde9df,0x1762ff65),L_(0x9ddef882), + LL(0x0d56a9f5,0xe6f97e52),LL(0x2cfac1c4,0xe924c291),LL(0x9480052a,0x61546993),L_(0xcee96715), LL(0x4783b246,0xdc612b8a),LL(0x01036872,0x40ce9da3),LL(0x9e6d56f5,0x0543bf02),L_(0x40b8eb7d), + LL(0xaa29fa33,0xbb67b973),LL(0xd7ceddb4,0x733e1da7),LL(0x7bd3859f,0xe95a7ea2),L_(0xc85424d7), LL(0xc42d076f,0x5d21362a),LL(0x2591a7e7,0x8674ee1d),LL(0x52817220,0xc9d8453d),L_(0xa286acd4), + LL(0x7fece4b3,0xaf830115),LL(0x6e587151,0x0a71eb17),LL(0x3d8da1f0,0x7afbb075),L_(0xf8e3bab4), LL(0xea1fecf8,0x3ab8344c),LL(0x6bb840b0,0xb4392fd5),LL(0xae7bd8be,0x73177038),L_(0xed8b2ca2), + LL(0x50dfe6f9,0xadd91143),LL(0x680d82d3,0x88a2c506),LL(0x88f503de,0x37393204),L_(0x746b676f), LL(0xc6591d03,0xaf979503),LL(0x5a9be716,0x79a8b931),LL(0x0e682183,0x609ab748),L_(0xfe8299c8), + LL(0xc3535c6b,0x193fd320),LL(0x8682c4c0,0xff4f04b8),LL(0xd566872d,0x1407f3c3),L_(0x16439d5d), LL(0xac2e95ba,0xd6726481),LL(0xb7bc2fb3,0xb0b0ea32),LL(0xb4c2b53d,0xfeaaece6),L_(0x4349edc0), + LL(0x649e4b51,0xae9bd277),LL(0x6c678048,0x4268b40c),LL(0x52da3782,0xae4a71b1),L_(0x1e1cd89e), LL(0x4f079bbd,0x2f691fc6),LL(0xf69884c3,0xfe5d33d3),LL(0x19020a41,0xdda1c483),L_(0x77d02be4), + LL(0x7ea53289,0x8529164c),LL(0x5d80aafd,0xd09dfaea),LL(0x1e830c7c,0xa9f0f09e),L_(0xfc3140e2), LL(0x5135dc82,0xdc2ab04a),LL(0x59befe02,0xa0b35f50),LL(0xa44595d6,0xc1e15529),L_(0xf19abe7c), + LL(0x3a5d5f42,0xa12bcced),LL(0xb9b8ec3a,0xb87d7dc6),LL(0xad1e4dcf,0xb63e0921),L_(0xfcf65288), LL(0x0c0e6332,0x863fe95a),LL(0x5cd2ee3e,0x42a677d4),LL(0x60da7b43,0xb8a5bd09),L_(0xc426f7df), + LL(0x51ea4dd8,0x6e5213a1),LL(0xfd69d7c4,0xdf5f1e1a),LL(0xd30cf23b,0x37d44e1e),L_(0x7097147f), LL(0x321e5347,0x12d98f14),LL(0x32dad5dc,0x76d7e7b8),LL(0xdfe1a803,0xc8d86798),L_(0xeccd5411), + LL(0xa7aae910,0x74006fc6),LL(0xd7d803d8,0xa34c1849),LL(0xec84dc01,0xae309a70),L_(0xa34e0e32), LL(0x42af799e,0x235ed631),LL(0xcfee558c,0x8802c5f3),LL(0xcd97224f,0x1640e7f8),L_(0x92280213), + LL(0x535c56a8,0xc7ccf419),LL(0xd687e53b,0x31ef82e6),LL(0x9a24176e,0x01470242),L_(0x01aae44b), LL(0xf12919c6,0x9fcc8bac),LL(0xc772707a,0x2bbb2bc0),LL(0xeeb32998,0x1fd556e7),L_(0x6b1da8b4), + LL(0x10943f72,0x3fe0c227),LL(0x929fe46b,0xf1163a98),LL(0x55829b86,0xb0593728),L_(0x46968b3a), LL(0x98063840,0x861c28fa),LL(0x49238b1c,0x238ae42f),LL(0xe2c2f08f,0xc3669826),L_(0xd635b05a), + LL(0x0b716b89,0x1cbd7071),LL(0xac172491,0x9bedfd64),LL(0x70337bd6,0x5806f542),L_(0xba62de08), LL(0xff8ac4cd,0xc789f899),LL(0x778e6f6e,0xcabb9b96),LL(0x7cb61f89,0x9e11ca0e),L_(0x54802afa), + LL(0xb4f60ac0,0xa293c1cf),LL(0x98d81eae,0x3ce25589),LL(0x4a05fd65,0xf7776c88),L_(0x31edd830), LL(0x7102bdbe,0x6b003741),LL(0x530442c5,0x619c469b),LL(0x2717506f,0x993316f5),L_(0x977de534), + LL(0x731f0430,0x1f4b647b),LL(0x63a5f62f,0xa8649b7d),LL(0x184bd1bf,0x9eb8bb02),L_(0xd878238c), LL(0x2268c556,0xee715bb4),LL(0x0c09795f,0x5ba6433a),LL(0x4a0ef111,0x5e80a1d2),L_(0xa6ed70be), + LL(0x1a0ec85a,0x02e27463),LL(0x66b5fa13,0x7351bd1f),LL(0xf1afa0ff,0x262f06e4),L_(0x55328bec), LL(0x17a8c5cc,0x359db888),LL(0x56654a88,0xbbf24e90),LL(0x31866e60,0xe08705c4),L_(0x615a5b05), + LL(0x2741b093,0xd9bae789),LL(0xa18e5f26,0x959b66a0),LL(0xfb66461f,0x52d89638),L_(0xf79de1ee), LL(0x3b41960d,0x63f080c4),LL(0xf0c1209c,0x1356ad3c),LL(0x8927edab,0x1cca87fa),L_(0x0d7779de), + LL(0xed47b255,0x4ffe196c),LL(0x591b255f,0x0bda5bcf),LL(0xf529da71,0xa919472f),L_(0xc172114d), LL(0x4edb3f70,0x727df75d),LL(0x0b0bba01,0x7a3418a3),LL(0x18ae3fad,0x7de531b5),L_(0x60f6010b), + LL(0xeec6a320,0xc8ab7f39),LL(0x4d9c100a,0x42812137),LL(0xb3ed07de,0x2ce9d1a7),L_(0xf4f877cc), LL(0xbf31e6a0,0xe5d2ab30),LL(0x7ebabf38,0xc0c95215),LL(0xce3cfa6e,0x85ffa6d1),L_(0xfa3306d0), + LL(0xfe179a15,0x014e69fa),LL(0xe40ec6e7,0x949a6258),LL(0x8f04f43e,0xf3db5670),L_(0xb9883ee7), LL(0xf31a374c,0xeab3fc5d),LL(0xedf8720a,0x70831223),LL(0xf1d8e66c,0x32cbb568),L_(0xc1af6691), + LL(0x955bb333,0xb78b642a),LL(0x65a42ec3,0x18885922),LL(0xdde23959,0x216642c3),L_(0xcdc7c4de), LL(0x4a6ce38f,0x82ee3788),LL(0xec771036,0xd5430b69),LL(0xefdcbefc,0xe82b1033),L_(0x5b9fd0f1), + LL(0x856e9dad,0xd7c8b168),LL(0x94661f99,0x034f792d),LL(0x968a0514,0x5e1f223f),L_(0xf2936613), LL(0x9468ee68,0xf109d747),LL(0x8c1958b8,0x35d5c367),LL(0xd1baaa07,0x58b0e5bd),L_(0x04265560), + LL(0x03eb3ddb,0x3c1c78f2),LL(0x73aa994d,0x77b06810),LL(0x9d13c879,0x5ffbf962),L_(0x329365bf), LL(0xeb311f82,0x151d1b5b),LL(0x9b162681,0xa0010f38),LL(0x67d4adbd,0x2d7119cb),L_(0x112111da), + LL(0x3a3f8979,0x8de63a52),LL(0x0bc054e9,0xcecf35b9),LL(0xac3474f8,0xd39c2e22),L_(0x8a24d244), LL(0xe1c3169e,0x72de5749),LL(0xf7cd28ca,0x1f3311ce),LL(0xaba378ca,0xacf3d959),L_(0xc4c178a4), + LL(0x41028a7a,0x4c196a71),LL(0xeddb3096,0x07052fa2),LL(0x78d05008,0x4a5cea44),L_(0x4e82cf55), LL(0xe4836382,0xbc157855),LL(0x42628bb0,0x6906bb4c),LL(0x3bec9a03,0x414298ee),L_(0x71c76fb4), + LL(0x2dc6d80f,0x451c89a8),LL(0x0eb5ca81,0xb205946f),LL(0x85231812,0xdd102f79),L_(0xcfe70175), LL(0x79cbba78,0x74f5620f),LL(0xa68f72ba,0x925ad282),LL(0x107305c7,0xa0b90dac),L_(0x71b34217), + LL(0xd11e6b55,0xdf55e61e),LL(0x74013fe7,0x39b4240c),LL(0x384b1c58,0x117f4d5d),L_(0xb417f0cc), LL(0xdee67f19,0xa5e101d1),LL(0x6a59354e,0x057e8fde),LL(0xdd613a29,0x3d053f50),L_(0x49698601), + LL(0x0e1e0dac,0xbc28da28),LL(0x350510df,0x6476441e),LL(0xa5492aa3,0x488b3ebb),L_(0x7335580b), LL(0xe4198903,0x150fa7db),LL(0x5080e530,0x1e0113ef),LL(0x41fe0adc,0x151c555b),L_(0xefba1701), + LL(0x4e602721,0xda6ab8ed),LL(0x01c286ca,0x82be4f7e),LL(0x8f870fa0,0x67983e93),L_(0x011517ba), LL(0xd1bb4bf4,0x2ffdf949),LL(0xb8ff2b68,0x4a318fa3),LL(0x57a7f08b,0x2df51505),L_(0x7f4a552d), + LL(0xc07ee494,0x8a38d900),LL(0x15f78485,0xd40347a2),LL(0x40936fbc,0xdc403083),L_(0x26d9bf80), LL(0x87df50e9,0x6682494a),LL(0x658eecc3,0x88a94156),LL(0xe2c9cc47,0xf4a7dc3b),L_(0x6742c074), + LL(0x58b3b3f6,0xaf6f6f61),LL(0x1be73cf3,0xb37f3459),LL(0x74ffb428,0xb29ede9a),L_(0x92e686e4), LL(0xa4431924,0xdfa5e88f),LL(0xe1001a22,0x01fe6a9c),LL(0xf029cdcc,0x55c88ddd),L_(0xd03747ad), + LL(0xc359e1a8,0x69bc7a5b),LL(0x0a7f469b,0x79ced652),LL(0x8cb61ff4,0x860090cf),L_(0x25e6acf5), LL(0x5ae7246e,0x7c9a5e58),LL(0xd04ea633,0x62e6dab4),LL(0x3dccdf35,0x679b9a9c),L_(0xe0d5601a), + LL(0x9f6c163f,0x75393ffa),LL(0x503ff7fc,0x8a5ba74c),LL(0xad84a980,0x6a8d3ebd),L_(0xb711116c), LL(0xdb3d29d6,0x508cd2da),LL(0x298e39d1,0xadc29bed),LL(0xd182ffce,0xf35bcd9e),L_(0x56f60d5a), + LL(0x18ec5621,0xbb37476c),LL(0x126c6b62,0x00881f6d),LL(0x12e961ae,0xc23fa1b7),L_(0x620a85d2), LL(0x116a94f1,0x18d50f3c),LL(0x90b6bf4d,0xc1b565ee),LL(0xcce6cb44,0xb9db57be),L_(0x43232f2b), + LL(0x7e516745,0x33f18556),LL(0x34242913,0x303ca557),LL(0xecfa2caa,0xe5298223),L_(0x54c9b89f), LL(0x5a38eea9,0x2bb85e07),LL(0x3640c336,0xbe45ca34),LL(0x23f72dcc,0xe33bfd7f),L_(0x1138d09f), + LL(0x2532b3de,0x9635cec4),LL(0x99decb38,0xecdd16cc),LL(0xb6e94853,0xc90493fd),L_(0xe03d588e), LL(0x7c31a9ad,0xbbcd5eac),LL(0x1a5efdc4,0x24031f80),LL(0xbd3e5f35,0xc0bbee30),L_(0x91354a2e), + LL(0x46fe272f,0xe4fe7dd5),LL(0x40ccc046,0xcb3af2d4),LL(0xef734198,0xe37fbe0f),L_(0xd7d730ce), LL(0xc1b5100f,0xe81b74b4),LL(0xccbbf03c,0xfbbba07c),LL(0xaf1eca00,0xb202f7d1),L_(0x6c3a5371), + LL(0xb759588a,0x3c293b4f),LL(0x50ebe82c,0xccc60f4b),LL(0xe5c199da,0x9c0164e0),L_(0x09ae68d1), LL(0xdbb33342,0xe57a6b34),LL(0x764aa682,0x71131364),LL(0xd2b124d2,0xc37e19f6),L_(0x0684de69), +}, +/* digit=20 base_pwr=2^140 */ +{ + LL(0x2e5b54d1,0x3a2fc0ae),LL(0x3a0ae219,0x810b8800),LL(0x71d39325,0xeb53d991),L_(0x1b364d9f), LL(0xc225e922,0xe634fa4b),LL(0xfa979c3b,0x48c1302d),LL(0x0315e213,0x7a46c430),L_(0x7d3dfc00), + LL(0xf75af9f3,0x69c6dab4),LL(0xcad74055,0xbd0a143f),LL(0x5f980b30,0xf457bc61),L_(0x08ab6cd4), LL(0xa472e295,0x5a3b5bb9),LL(0x4a0eec60,0x1e647946),LL(0xa4e2c617,0xeb13a610),L_(0xea8ba726), + LL(0xb1f39e26,0x8208d95b),LL(0x1d126957,0xd33801fe),LL(0x38314570,0x2b3db8bc),L_(0x68f43b86), LL(0x06523bb4,0x2fa1baa7),LL(0xb3650b8b,0x7b7200e3),LL(0x222d46b5,0x5ab609c3),L_(0xe25b7f78), + LL(0xf09c3b4a,0x2601c0c2),LL(0x10c7e593,0xb71c88f8),LL(0xd433020b,0x163a4b4f),L_(0x6ea15982), LL(0xd7a199df,0x559abf4b),LL(0x168cc6db,0x8c58a3fd),LL(0x5e92d42d,0xcad8eab0),L_(0x0f718cd3), + LL(0x5222ebab,0x1eccb2dd),LL(0x45eee5ae,0x6cca2e8f),LL(0xf0f806c3,0x93a2e216),L_(0x0b896b9f), LL(0x2753730f,0x21ac6aa6),LL(0xe860a374,0x93ff82bd),LL(0xa6d03d5f,0xc336e305),L_(0x5a3ad08c), + LL(0x05094bb6,0x7e65ba97),LL(0x927ff44f,0x69a96369),LL(0xcfc2ebd4,0x8e5841d8),L_(0xcd919ed2), LL(0x7d67ba45,0xd72479fa),LL(0xf9079703,0xb458951d),LL(0x11b2590d,0x12b5f89b),L_(0x9e78739c), + LL(0xb68ddf1d,0x64971999),LL(0xcc5dc1ba,0x0f78a155),LL(0xc7f1ee3f,0xbbfeebe2),L_(0xa475b47a), LL(0xae6b1a67,0xd00f3342),LL(0xaa2739d6,0xcbc8eae8),LL(0x6bcba57f,0x08421638),L_(0xf50c5c89), + LL(0x21fce16e,0x56ea720e),LL(0xc14eb574,0xb07c02a7),LL(0xe5e93b80,0x1a141ce9),L_(0xf206da59), LL(0x400443af,0xff6ca106),LL(0x85cacb30,0xe7e89a41),LL(0xe54c35f6,0x73777848),L_(0xeecb8a06), + LL(0x27098970,0x27c35c5a),LL(0x88bff7c0,0xd3495797),LL(0x2aa31cbd,0x84083f2a),L_(0xee8007b7), LL(0xd736e6e1,0x7c2ce5db),LL(0x46d1faf9,0x7eccf318),LL(0x0d96af5f,0x7894ccc6),L_(0x143733e6), + LL(0x38e5c20b,0xf8d1a779),LL(0x74b4b8c0,0x1407c96c),LL(0xaa226358,0x0f677dc4),L_(0xbce88332), LL(0xae06373b,0xa1eff8f8),LL(0x6f4d4505,0x46e8175d),LL(0x37fa6b50,0x12414006),L_(0x078af83c), + LL(0xaa9911e2,0x6038786a),LL(0x043fa7bc,0x9ec4cf47),LL(0x39594053,0xeeffb86e),L_(0xf30130d5), LL(0x93573962,0x8e9bddbb),LL(0x6b822690,0xa5656b7e),LL(0xfd18acdc,0xf63c9ead),L_(0xa29c50ef), + LL(0x7a68f812,0x875129d8),LL(0x23706eff,0x43107ab7),LL(0x343d01b4,0x0f0278ce),L_(0xd1d2ff3f), LL(0x947dc2e4,0x62e80eed),LL(0x76f993b1,0xb237e733),LL(0xd8eb588f,0xc6cb2916),L_(0x3e93bfcf), + LL(0x2323cfa1,0x40dc414f),LL(0x02fc59aa,0x81bf9064),LL(0xf4b7afef,0x2f863790),L_(0xdf1eb2ce), LL(0x2da3a955,0xc2be20ad),LL(0x05119c9a,0x826e16e4),LL(0xa27da69d,0x59ba9dd2),L_(0x50be3370), + LL(0x702f4a30,0xe5b817ac),LL(0x62b0b4a9,0x732be8dd),LL(0x0b744a18,0x49eb11c0),L_(0xdb8066e2), LL(0x61315a81,0x60bc3f3b),LL(0x9db8094e,0xa033694a),LL(0x1a336970,0xe5728d57),L_(0x98764bb8), + LL(0xa31f05a5,0x471336b0),LL(0xc97e0c25,0xe1b3c5c4),LL(0x40016340,0xd00385b4),L_(0x8681b939), LL(0xf3dbd1c9,0xfb0504e6),LL(0x190db5e8,0xf84aaa64),LL(0x73d7879b,0x5e7d8612),L_(0x8f037a48), + LL(0x4bb632f0,0xb920e089),LL(0x81a11e46,0x5fd26f9a),LL(0xad7ce91c,0xd873ff38),L_(0x463f5eef), LL(0xe973b348,0xa997a44a),LL(0x67e4b48b,0x7c4b4d96),LL(0x76b8b69f,0xc007a35a),L_(0x0ebb0cc7), + LL(0x066bf5fc,0xc6331e3d),LL(0xd49fdc60,0x01bb84a6),LL(0x3f1c848c,0xae53c6d2),L_(0xa68e50ed), LL(0x64a19a00,0xa6271228),LL(0x1c7298ae,0x6345be32),LL(0x34a3a582,0x6666c91c),L_(0x1bb3ce87), + LL(0xe3fc9fee,0xabcce7ab),LL(0xfa1beb7d,0xdf35c272),LL(0x9cab965d,0x7c86ce27),L_(0x145e311e), LL(0xadb37a6d,0x3f4563ca),LL(0xbd40ee7b,0xc5e44069),LL(0x077a6fa1,0x6ab23a8b),L_(0xdc9e2258), + LL(0xe14e323f,0x27ccdc44),LL(0x8d1e93bc,0x9fd47cad),LL(0xac370974,0xf5fc7d6a),L_(0xbb95cf97), LL(0x3c654b26,0x65e7b71e),LL(0x0d3d8936,0x69b3aaf9),LL(0x6cfc89cd,0xcfd74f9d),L_(0x46c8cccc), + LL(0xe59aef1b,0x82c35cac),LL(0x4b6e4cd2,0xd343c61a),LL(0x5bf959ed,0x77b74152),L_(0x14153467), LL(0xabc9828c,0xc645fe14),LL(0x44ed7180,0x251c7bd5),LL(0xb00a2c9f,0x94c68e2e),L_(0xcd40eadd), + LL(0x90c9c5ec,0xdb397bc5),LL(0x0bb14c2d,0x0029fc46),LL(0x2727c32b,0xbdde6547),L_(0xf5120538), LL(0x8748603f,0xca65a20c),LL(0xb71dac7a,0x79d195d6),LL(0x7b4ac903,0xf1e5f556),L_(0x83bcccf1), + LL(0xd8d5a0e2,0x35177eb8),LL(0xb2a6346d,0xe17edde4),LL(0xcdb1c427,0xc5c607ff),L_(0xf6acdcca), LL(0x7045642b,0xf198276e),LL(0x5ddfa215,0xa8387d90),LL(0xc1a3402a,0xc070af80),L_(0x0cf324a8), + LL(0x89428955,0x4be4c6c0),LL(0xddf4319c,0x076b38c2),LL(0xcc96208b,0xd1d87cd0),L_(0xc3e2cfee), LL(0xb63b328c,0x1fa3dfb1),LL(0x89c310af,0xbcb33f40),LL(0xab955eb2,0x1e3849a6),L_(0x4e9da2a4), + LL(0x572060d6,0x4a409b86),LL(0x4a32be16,0x01fee4f3),LL(0x5593d29d,0x1f66b255),L_(0xa18a71de), LL(0xdc92993c,0x871121b8),LL(0xd5a52a49,0x2885efde),LL(0xdb77ca39,0x35178007),L_(0xe87f7b56), + LL(0xb0b54185,0x7b11bf25),LL(0x78cced92,0x5f580048),LL(0x568cf974,0x1aebc2f4),L_(0x90437303), LL(0x380e1852,0xe1d4cf60),LL(0x81708f04,0x335221a8),LL(0xfcd5e396,0xe8e310f1),L_(0x67164e9b), + LL(0x5cc646e4,0x4435eb3f),LL(0x309bf5d1,0xc4516a98),LL(0xfa094c38,0x0f0f519b),L_(0xa7712b6a), LL(0x063b90c9,0x13b443e8),LL(0x3fc80f73,0x588eabab),LL(0xc2640659,0x6f1b036b),L_(0xcc979824), + LL(0x15f68a6c,0x2c10fe82),LL(0xfd4280b4,0x4be94dbf),LL(0x0f01bc51,0x8ebbdd0a),L_(0xe8a94761), LL(0xf5c8cf6e,0x3331db35),LL(0xe9e32c12,0xee34fb22),LL(0xdfdd390f,0x7d9acfa0),L_(0xf46a833d), + LL(0x4db03d63,0xb1f7d640),LL(0x50d5c66d,0x77b9e67a),LL(0xd311bbb8,0x2df06964),L_(0x4d4065dd), LL(0xb1ba8be7,0xc5f2e7e7),LL(0xa84be5a0,0x3c7c5264),LL(0x704fb715,0x72ab97c4),L_(0xe211a72c), + LL(0xde6eb928,0x22e459d0),LL(0x262ef282,0x78e351ef),LL(0xcdae125f,0x400cd2e5),L_(0xc83f210b), LL(0x1982c6e5,0xb3e5fe6c),LL(0x87c8591a,0x13591995),LL(0x83fd93d2,0x072fa5b0),L_(0x7d701f58), + LL(0xae57420d,0x2dda42fb),LL(0xa6490705,0x0c8f6d52),LL(0x5e3ec836,0x974661b3),L_(0x77bc552c), LL(0xca185c95,0xf43d8647),LL(0x351407f7,0x3094919d),LL(0x93522ec1,0x2effc179),L_(0x49a92d02), + LL(0xad6aa522,0x1178f9fd),LL(0x88ddd25b,0x7c35a7b4),LL(0x4985102d,0x0d3de452),L_(0xf0277afe), LL(0x9952ce22,0xf3571744),LL(0x53a80f8c,0x33bbd8c5),LL(0xa11d84cd,0x8cf3c9a4),L_(0x7920692b), + LL(0x19d5d3aa,0x3259f507),LL(0x8dd2cea1,0xfaa534d0),LL(0xb1de59c3,0x964e43aa),L_(0xd9cd158a), LL(0xb8fe9ccf,0xf7812a29),LL(0x483a8ca3,0x5a40b462),LL(0xf93b681f,0xeafa2982),L_(0x22d0852e), + LL(0xccafe782,0x4e3898e7),LL(0x178b364e,0x122c0954),LL(0x1e46cbf3,0x07005a6a),L_(0x017cc249), LL(0x1f834707,0xe3d63870),LL(0x9df8269b,0x84c46f73),LL(0xf32797a5,0x1360de40),L_(0x70a39a32), + LL(0xe6611eb8,0x58005efe),LL(0xefaea42e,0x9d406314),LL(0xcdf53ee0,0x8d9f77b8),L_(0x2be3de21), LL(0x3152090c,0x219fc4d6),LL(0xdfbb4054,0x2e867eb0),LL(0x370b0058,0x0df48ecb),L_(0x67c50061), + LL(0xd35d5628,0x7d760756),LL(0x3cb96c10,0xaff6e332),LL(0x87378266,0xc87eb4e6),L_(0xbb1b1686), LL(0x719b7dba,0xf625cc5d),LL(0x7f6f6bfb,0xc5c21343),LL(0xd1702bfc,0xd7459a05),L_(0xb5757892), + LL(0x07b102d7,0x2285b7c5),LL(0x77777ad0,0x90a0453e),LL(0x2a9f4627,0xa5cb03f7),L_(0xbdaaa412), LL(0xd6bb4325,0x6fb99eeb),LL(0x8aa2c4b5,0xc61bae52),LL(0xec729246,0x9b9f8d87),L_(0x995887b7), + LL(0x714a54b4,0x786ccb7c),LL(0xae2ffd66,0x1df96294),LL(0xb5661480,0x0b294654),L_(0x42cc5e69), LL(0xc6f02a04,0x1634f1da),LL(0x21f0caa7,0x75d2244a),LL(0x80410ce5,0x6597b349),L_(0xb9e50c3a), + LL(0xd19e2b9f,0xbefea27e),LL(0xca541338,0x53b70277),LL(0x0bc9bd26,0xd300e1fc),L_(0xed2c5b84), LL(0x935f9273,0x731c6b5d),LL(0x10fed3e2,0xac105b56),LL(0x6d0357ab,0x30c7c4b6),L_(0xfdf04b77), + LL(0x32c68e8f,0x6198d0bf),LL(0x49f0ac87,0x02928249),LL(0x5908bfa6,0x2b72f231),L_(0x1e0626b8), LL(0x87d56448,0xb70c5fdd),LL(0xaffd0d74,0xc8176ae0),LL(0x30464156,0xbe7c0da7),L_(0xf7d2ffb9), + LL(0xffa548f9,0xd71029f6),LL(0x2ac8267b,0x5922838c),LL(0x4f8afc3b,0x2d787218),L_(0x10f9a786), LL(0xb358d2a8,0xf19c8b54),LL(0x5209c3c7,0xf590721f),LL(0x0212109d,0x4e16a382),L_(0x3cd77b54), + LL(0xef4b9410,0x4e89b491),LL(0x77501651,0x417ff9fd),LL(0xdbb5243c,0xd24a6908),L_(0xc09c5df1), LL(0x2c39f4c4,0xeb16d02d),LL(0xb28822ac,0x4cd4a553),LL(0xbf46b5bb,0xde8a18e1),L_(0x47566077), + LL(0x0937292b,0xa4a4f179),LL(0xdff0cf1f,0x01b86895),LL(0x34f0c71b,0x3fe5a3b7),L_(0xae1fff30), LL(0xe080e0da,0x95b01982),LL(0x5317a578,0x3064e918),LL(0x8872e118,0x154f511a),L_(0x28ad7f46), + LL(0x9dbd0087,0x3c4162c3),LL(0x88636424,0x58631293),LL(0x69f32e25,0xf3a9fd92),L_(0x049b5c0f), LL(0x8d93c391,0xd8613d4d),LL(0xcf94b297,0x74bd10f5),LL(0xf99f0a15,0xf6d69ab3),L_(0x8d722881), + LL(0x67ba729a,0x2829f720),LL(0xc6f3cc19,0x88fffa70),LL(0xf0c5f76b,0x0d44b294),L_(0xda7df6fd), LL(0x033be0dd,0x35516898),LL(0x4c9c487f,0x50dfac1a),LL(0x312edd81,0x14a64ecf),L_(0x2db649b2), + LL(0x4933ac70,0x4e65d480),LL(0x17bb4dfc,0x8f9141c2),LL(0x7b8827b2,0x6ffab99a),L_(0x788afe5d), LL(0x15192ba1,0x6ed82d80),LL(0x75adf2d6,0x7be9df26),LL(0x99ce22b1,0x957976c9),L_(0xf8f37372), + LL(0x2204ff70,0x3ad3db41),LL(0xa4617ca1,0x7ca2697e),LL(0x8f228ced,0xc881f9da),L_(0x86ddd01a), LL(0x6e35e7e8,0xd1467d74),LL(0x41eb80aa,0xa4eadfce),LL(0x875ce5dd,0xaf17e700),L_(0x398d4c2f), + LL(0x3e59ab08,0x165575cd),LL(0x7c2f636f,0x55e6cf6d),LL(0x89f1290e,0xeeb00b1e),L_(0x2cccc148), LL(0x264e5c6c,0x7200e28c),LL(0x5409cce2,0xaf2b4dc4),LL(0xf8e920e1,0x3586fdc4),L_(0x11976312), + LL(0xab519f43,0x45934ad0),LL(0xa0a5bb1b,0x0eda4d94),LL(0xbcff29b2,0x9f749e30),L_(0x2f73a7d6), LL(0x2a833e74,0x9ed5bbc8),LL(0x111b6548,0x83945e37),LL(0x7b77f3f9,0x9b568171),L_(0xf1563350), + LL(0x999312c1,0xbde3fb88),LL(0xaa153efc,0x4ead50eb),LL(0xf24dc769,0x91718221),L_(0x3207691f), LL(0x1445f929,0xa962b883),LL(0x8a1416f3,0x6db04bb2),LL(0x2992be20,0x388328fb),L_(0xa7c4a359), + LL(0x92efc55b,0x709d7f62),LL(0xf678058c,0x08e33ff6),LL(0x9db19687,0x8d882bae),L_(0x81afb521), LL(0x462b58d3,0xce4ef2a4),LL(0xc6ecf132,0x21f09167),LL(0xab9bbd79,0x3831d678),L_(0x4c11a697), + LL(0xda7c3371,0x8a72049c),LL(0xc3e5138e,0xc7decc6d),LL(0x9235657e,0xf870194a),L_(0xeddbefb7), LL(0x306c9631,0x85d843d8),LL(0x912ed2a5,0x27180693),LL(0xdb45568f,0x453cbf16),L_(0x410259cc), + LL(0x3f11cacd,0x10263ac7),LL(0x69fa2805,0xb832a378),LL(0x6120289b,0x86d5285d),L_(0xece730a0), LL(0xabd38f00,0xbf1197bb),LL(0x286f4724,0xdbbb1a26),LL(0x0b0ef425,0xf397181d),L_(0x24bb4f4f), + LL(0xe24a9a47,0x162331a8),LL(0x536a5ea6,0x2bfbcebe),LL(0x88c29411,0x3da5a14f),L_(0x9cb76097), LL(0x5145ce76,0xef6b8093),LL(0x59e86f3c,0xe0a56e13),LL(0x817764eb,0x58eed1b6),L_(0x8010fc3d), + LL(0x0b1262b6,0x9a5b60b7),LL(0x108b3068,0x14ec42e4),LL(0x1312c3e2,0x8dd4b81a),L_(0x0253e9dc), LL(0x253ff39b,0x5eadccd1),LL(0xbd75013a,0x87febbae),LL(0xc60daeca,0x20b2a500),L_(0xbea0d3fb), + LL(0x20482528,0xfb3fb2a4),LL(0x72db2c7a,0x3620dd94),LL(0x864ce0ff,0xe0956799),L_(0x2d7ec9e2), LL(0x86bb3cca,0xdfc7056e),LL(0x77bfd4dc,0xf75d4a37),LL(0x0f42ffde,0x110fb370),L_(0xf2040e52), + LL(0x38389a23,0x2100e0ba),LL(0x6fd68890,0xbf38ce28),LL(0x67978ca8,0xb3a94049),L_(0x279171ad), LL(0x4b5e0e0d,0x0230458f),LL(0x41165c1f,0xb73bda2e),LL(0x8eadb8c9,0xe9eaef42),L_(0x2345e24e), + LL(0x9fe296bd,0x41f15f29),LL(0xaa889c09,0x8ccda12e),LL(0x6adf563e,0x559edd7a),L_(0xdd414cb0), LL(0x84d15464,0xad8084b6),LL(0xbef296d5,0x3238cf0a),LL(0x0ee342aa,0x074068ff),L_(0xc9f3a4e6), + LL(0x374364db,0x4db1a4b3),LL(0x983e29af,0x190ecab8),LL(0xc26d69a7,0x8e174c76),L_(0x2739f19a), LL(0xea361aa4,0x4c902613),LL(0xcd082b89,0xbbc1e2c1),LL(0xa0661a39,0xd0352acc),L_(0x6d5bd738), + LL(0x6b165ee3,0x0f46deb2),LL(0xe05a9d3e,0xbaae3236),LL(0x68d899f5,0x2fcbd485),L_(0xd426fbf8), LL(0xfce9e8db,0x70b0b8e7),LL(0xbb075381,0x24c285b0),LL(0xeb323221,0x933c7339),L_(0x90f21e83), + LL(0xa437ec78,0x65c3ea6f),LL(0x26f5292f,0x16eebdc4),LL(0x2f10ff0d,0xd0e0dfce),L_(0xe59106aa), LL(0x47d3151e,0x334ef2fb),LL(0xe1d13288,0xa14975cc),LL(0xb2dd6c80,0xe941b704),L_(0x428af6d0), + LL(0xac3814f9,0xf4357fa6),LL(0xe9634ed9,0x3e28518f),LL(0x92f270b2,0xb8fe4e04),L_(0x4ce90044), LL(0xc454c0af,0x9137e5a6),LL(0xb5cf536c,0xff154124),LL(0xaa1f4551,0x0024f90b),L_(0x981784b9), + LL(0x9e485500,0xb04608de),LL(0x5269aa1a,0x6eb17af7),LL(0x8ce1ebee,0x3d279465),L_(0xc66381e8), LL(0x0b5ed745,0xc4fa7ed6),LL(0xdeb29303,0x601d19f0),LL(0x510a3c8d,0x2ce166ff),L_(0xaa227120), + LL(0x1c8b8392,0x992398c6),LL(0xa3c32177,0x98eae201),LL(0xcd19850d,0x70b1a326),L_(0x9505296f), LL(0x958225ae,0x41bcad26),LL(0x0d11515a,0x938c115e),LL(0x04883b49,0x5a5bf602),L_(0x72d1e180), + LL(0x597adc60,0x5898ed4e),LL(0x63896a2f,0xa9c6e64f),LL(0x82a0ed92,0x71349776),L_(0x71b9b0ee), LL(0x216fc62a,0xd360630c),LL(0xbd440771,0xa2285bdf),LL(0xa49907da,0x28e79328),L_(0xad498ec7), +}, +/* digit=21 base_pwr=2^147 */ +{ + LL(0x9dfbcf8e,0x5e338a94),LL(0x0324aba6,0x9f1b5252),LL(0x970769c2,0x32ea5aeb),L_(0x82c3b64b), LL(0xfeabe87b,0x80346bd1),LL(0x98111ddd,0x63f11cc5),LL(0x2ef9e31e,0x75819a03),L_(0x6e0e0fd4), + LL(0xead2c5e0,0xd481d37a),LL(0x676ef7cb,0x83660242),LL(0x6ddd542a,0x4bcdf56d),L_(0x0807b6ee), LL(0x88b2540c,0x99cfbaa1),LL(0x440a4586,0x92cdeb4c),LL(0xaab76f82,0x62bde53d),L_(0x31ca293a), + LL(0x88edf264,0x279a665d),LL(0xc73f3e56,0x4837e985),LL(0x51cce02b,0x925d55fe),L_(0xba48033b), LL(0xfbc2fa3e,0x3c3ccfff),LL(0x09219d1a,0xa32d9d41),LL(0x7bba7cbf,0xf5c59984),L_(0x1f9a5c08), + LL(0x1175b9ba,0x4079f189),LL(0x150e6d32,0xe9e1ab30),LL(0x8b6cc92b,0x2d425400),L_(0x274ec0c2), LL(0x8f0527d9,0x6d261b01),LL(0x4ef086f3,0x63cc5685),LL(0x29dec1b9,0x7f963f71),L_(0xea97e94e), + LL(0xd50b2092,0x9e517139),LL(0xf9867b99,0x32ba3680),LL(0xbd760d63,0xafdf07e9),L_(0xa9b6e9ec), LL(0x8b644aef,0x6d61dc07),LL(0x93df1b2e,0xe8d9afc6),LL(0x3387d03a,0x32d7f165),L_(0x759fb34d), + LL(0x41d99c15,0x08345264),LL(0x34f3b1e1,0x17c124bd),LL(0x20468107,0xc8425e31),L_(0xa035ed92), LL(0x19019adc,0x15129a66),LL(0xd78546b5,0x4c880bae),LL(0x2a03f320,0x04faf0ec),L_(0xf88d60ed), + LL(0xcbda1d0c,0x99a20d9a),LL(0x31f03c7f,0xc7f9c96f),LL(0xbdab0f5d,0xcfc1004b),L_(0xc665b5f0), LL(0x24cf476d,0x57bee516),LL(0xbaab1072,0xcf987028),LL(0x64de4da0,0xa4efa899),L_(0xb3c284d3), + LL(0xec9ac930,0xb6ef6e4f),LL(0x7bd6b2eb,0xf97982f6),LL(0xb87b9f50,0x3506344d),L_(0xe38cc59c), LL(0x0df2358f,0x98614bb4),LL(0x4daab44e,0xa73a33ba),LL(0x3957de53,0xc680567a),L_(0xcfa42f96), + LL(0x6e9aa82b,0xad09cde6),LL(0x89749594,0x88dae77c),LL(0x686f22a8,0xe7736848),L_(0xd741b21a), LL(0x9e20d489,0x08ab4cce),LL(0xb87c7bef,0xcc175c96),LL(0x982331fd,0x177b6473),L_(0x63140d6e), + LL(0xb3fe0b33,0x061731af),LL(0x785b32f1,0xdee055b2),LL(0xc296cafa,0x1de52e72),L_(0xb10ce663), LL(0xc5a7f1be,0xc6a06418),LL(0x50016567,0x1713586c),LL(0x4502900b,0x2c001a59),L_(0x450b28d5), + LL(0xcab8d38d,0xc5913490),LL(0x9e1e0ce7,0x343c325b),LL(0xc5f9091b,0xe7590273),L_(0x44739896), LL(0xee5b12cc,0x449da236),LL(0x3913a8d2,0xd0e8b775),LL(0xd9db43d7,0x0204d37f),L_(0xbcc28709), + LL(0xfda66aba,0x0ba00094),LL(0x00c25d60,0x034594b0),LL(0xb0ccb9b4,0xaecf0812),L_(0xebecd41b), LL(0x95fb80f1,0xe2cb6f9c),LL(0xe9f56be8,0x33f77310),LL(0xab12b176,0x77e2c3e0),L_(0xa7a303fc), + LL(0xaa4bbfa4,0x0ca6bd41),LL(0x829f1bab,0x9f37381d),LL(0xf8aa85f0,0x63a37ba6),L_(0xe64c6ed3), LL(0xb1db7703,0x9ba22c56),LL(0x14c74bf2,0x26369a5a),LL(0x57cdabdc,0xa2f36ff4),L_(0x69603458), + LL(0xd1e5c2d6,0x049ce7f9),LL(0xdda045cf,0x6a8702d5),LL(0xae642723,0x444a4ca8),L_(0xc9e8efd0), LL(0xbc74c0b8,0xee8ff912),LL(0xa5f0a182,0x7cacfd02),LL(0xf34c0a0e,0x0bdcdecc),L_(0xf174835c), + LL(0xad6aaeda,0x262941fe),LL(0x50dd89e9,0x052230cb),LL(0x52bfaad6,0x3d3651a5),L_(0x42da9f6d), LL(0x1803593d,0x72f24771),LL(0xfb5c67bc,0xf8d5c69c),LL(0x6f3ebdef,0xa8074e96),L_(0x5d95664e), + LL(0x3cfb3680,0xfbd1a191),LL(0x14bfa4e1,0xd4f7de76),LL(0xf38ce345,0xfcdb2abd),L_(0x45ebd21e), LL(0x1d84093e,0xb59f480a),LL(0x3957e934,0xcc218197),LL(0x7c1b8e5b,0xe4604f76),L_(0xc58f2c48), + LL(0xd2d407e1,0xc8a88d72),LL(0x21606f53,0x9385e572),LL(0xfba9c9f7,0xfb7d83f0),L_(0xf57dce48), LL(0x37eda284,0xfb80fd4f),LL(0xc8023a07,0xf3549107),LL(0x71a25cd3,0x7a9a0b44),L_(0xb3c48734), + LL(0x885801a9,0xf3798b62),LL(0x9856b7c0,0x08f77ad9),LL(0x04ab2722,0x471ef449),L_(0xe5f3587d), LL(0xc09f1c74,0x21b96d5c),LL(0xb0c73d23,0x5e2a228b),LL(0xa3366107,0x69323a1b),L_(0x759bf017), + LL(0x11492138,0x80978740),LL(0x4e3559fa,0x456876b0),LL(0xe7559081,0x56ad6e70),L_(0xdd9852be), LL(0x8ce4faa0,0x0e7594de),LL(0xe4523a8a,0x3999a1a1),LL(0x4a40d4e5,0x8ca5687b),L_(0xfaf902fd), + LL(0x854da328,0xd2ade9c1),LL(0x683ea003,0xb2369f45),LL(0x6bbf8ead,0x26a2a12e),L_(0x65fa7d53), LL(0x44971ebd,0x6154c3a8),LL(0xd7f809d8,0x9e3b11ad),LL(0x20ed9167,0x24d19af8),L_(0xa692a83a), + LL(0x26ad917a,0xd8e0ff42),LL(0x90e27172,0xd10a1313),LL(0xa90bf44d,0x8cb2abd0),L_(0x130d21df), LL(0xea4790cd,0xa3cb4c6a),LL(0x65186f8e,0xfd47c901),LL(0x0c9c373c,0x3514478f),L_(0x5307b692), + LL(0x80001060,0xd8568305),LL(0x34f0a25d,0x6a578b8b),LL(0x039f50cc,0x0eadf2ee),L_(0x6a3cf739), LL(0x8ca51f3b,0x6a9a2a61),LL(0x5c707fb6,0x92f53d4f),LL(0x22360abf,0x756df232),L_(0xa7a12676), + LL(0xa54ee092,0x232b0eae),LL(0xfac26fa0,0x2030a397),LL(0x91c0d8b3,0xc85b9885),L_(0xcd36972d), LL(0xf99f3cb8,0x2b28cccb),LL(0xa2197bb4,0xd5dc11dd),LL(0xb00034b6,0x22200d9d),L_(0x398b8d08), + LL(0x48067657,0x3e7e898a),LL(0x14230734,0x429f6872),LL(0x7bef4c47,0xe3a1abc8),L_(0xb3564180), LL(0xb41998c2,0xeed81406),LL(0xa7ce6e62,0xe40d75be),LL(0xf02dd5b1,0x9d22967d),L_(0xa3ff259d), + LL(0x669ca4f5,0x25d2230b),LL(0x2d777318,0x7178f19a),LL(0x80147bac,0x6f460f52),L_(0x22279805), LL(0x7e64abfe,0x20550e61),LL(0x7880c924,0x79d1e82f),LL(0x6428d5fc,0x9a19cfda),L_(0xd1beb76b), + LL(0xba10e2eb,0xe36b37d3),LL(0x5d87cbda,0x2be422df),LL(0x404781e8,0xc8f5e0fe),L_(0xa1bc9eda), LL(0xd68a56ff,0x23299058),LL(0x970d11fc,0x7d7df62c),LL(0x785e8445,0x8ed8b412),L_(0xbad1345d), + LL(0xea68889d,0xce7aa168),LL(0xd74f6437,0x77a09f9a),LL(0x1f1dc772,0xf497718e),L_(0x36827efd), LL(0x53f8a4a5,0x978b617c),LL(0x4cb29feb,0x11c9e4d8),LL(0x6a64b734,0xe04077a2),L_(0x86b8cf3a), + LL(0x81d83b84,0xed710d97),LL(0x3a39d170,0x9d798cd7),LL(0xef236d5b,0x0683a7e0),L_(0xb9141322), LL(0x295bb2f8,0x05109f73),LL(0xdade7c00,0xb994c8eb),LL(0xf251df45,0xc11452de),L_(0xb48e9a05), + LL(0xab19c0be,0x4aaf45f4),LL(0x17039bf0,0x12b12bdf),LL(0xb97b72ee,0x30ce8053),L_(0xa7a81755), LL(0xa5e18f4a,0xc81db67a),LL(0x80753cae,0x1c4a9982),LL(0xe3019a4d,0xd1c81400),L_(0x80f8c3da), + LL(0xeb75d498,0x165046c8),LL(0x030a06a1,0x232d37a1),LL(0x09c6053c,0x3932bda3),L_(0xcb39f998), LL(0x95de7723,0x87fe7976),LL(0x0499f81d,0x216feab0),LL(0x7ae298d0,0x2959c264),L_(0x1edb6b93), + LL(0x916476f9,0x6ab52744),LL(0xdc732139,0x45a91c78),LL(0xbd7da6c3,0x5ac1ef95),L_(0xe2e0217e), LL(0x874209cf,0x63f9997a),LL(0x5cecf1c0,0x9d1320f0),LL(0x5082d35e,0x7b11e2cc),L_(0x04cc9aa3), + LL(0x481e157c,0xfecff2a6),LL(0x4f8b44cc,0xa01ff932),LL(0xd783714c,0x9c0232a4),L_(0x90d1c36e), LL(0xad070ffb,0xbd5e004d),LL(0xf22c1c14,0xec3d0dd1),LL(0x77b24ec6,0xc4fac858),L_(0xf15be1db), + LL(0xa4425706,0xfa634b73),LL(0xc11ab7e2,0xe2196b67),LL(0x39a0b0f1,0xa62656eb),L_(0xdf5e8e89), LL(0x10e754f8,0x35af495a),LL(0x059e7fc2,0x8885a6c4),LL(0x5721bb41,0xfcc7a8b4),L_(0x18530aec), + LL(0xce8caf63,0x1efc0864),LL(0x7e7fb20d,0x45a5d6d7),LL(0x319cbee0,0x32603543),L_(0x216780a7), LL(0x728ec24d,0x771c5008),LL(0x332b0e8d,0x88a3ec11),LL(0x27842539,0xfd61d211),L_(0x176fee74), + LL(0x636c68f5,0x6d5e6c03),LL(0x8f5f36df,0x99e754bc),LL(0xff864830,0x04fc8d4b),L_(0x35ae923b), LL(0x8c2df569,0xd03159ad),LL(0x6c4574ce,0x802cc48f),LL(0x95e44ad6,0xecc940e1),L_(0xacd13e60), + LL(0xaeaeda46,0x24bc05c7),LL(0x55d8d9f7,0x6d33fac1),LL(0xba1700c6,0xb1dbe56e),L_(0x12671c92), LL(0xa3906992,0x7c873618),LL(0x28f4b581,0x5d43aa50),LL(0xc27395ee,0x31a82a43),L_(0x84bd2bdf), + LL(0x85fa613e,0x19e04eb7),LL(0xd6d7b2f3,0xacf5f895),LL(0x7d71e73c,0x5dd55589),L_(0x6b4ddaaf), LL(0x5cbbc28d,0xb3874051),LL(0xb395499e,0xf0404258),LL(0x56ce4cbb,0x5961b289),L_(0x2960f1ca), + LL(0x08af573f,0x739615c5),LL(0x72405988,0xdcdfa5f7),LL(0x03300371,0xe254b688),L_(0x6195855d), LL(0xe4e87561,0xd298a295),LL(0xf9205a13,0x156d1dbc),LL(0x11ffb781,0xe00ca07c),L_(0xd638a7b0), + LL(0x9e23a234,0x970fe955),LL(0x62d115d2,0x4374bbb8),LL(0x4db47695,0xb3652081),L_(0x6565367b), LL(0x49a6f033,0x53a17dff),LL(0xbb00a05a,0xef6118bc),LL(0xc7f0a623,0x2971e39b),L_(0x091d591c), + LL(0xde40fbc4,0x3c51221c),LL(0xd273fed4,0xda08dbf2),LL(0x7eaf4c1f,0xc3cadac4),L_(0x155b21cd), LL(0x8e6c2580,0x824bf439),LL(0x6d0eb93f,0x16f47b06),LL(0x2b5af50d,0xe79201e3),L_(0x50f56685), + LL(0x2cbaab9e,0xaf65e93d),LL(0x64cb8361,0xdcc125a4),LL(0x2b08724e,0x5d31d941),L_(0x128434b7), LL(0x11800db2,0xd87b7933),LL(0x6349bb61,0x41b8d379),LL(0x8eabd8ce,0x46f9abc2),L_(0xf5ec4590), + LL(0x226b273d,0xde41b0d7),LL(0xfcdd55ff,0x16980cb0),LL(0x5f8a8d74,0xad3ba05b),L_(0x5c84894a), LL(0x8dc5f338,0x1041454e),LL(0x7425541f,0xc1a5e43c),LL(0xf6c08550,0xbe1bce56),L_(0x48f611af), + LL(0xf3ceb53f,0x31e207aa),LL(0x505aaa25,0x7b5d7c2b),LL(0x9df3ed0d,0x6ff3aa7e),L_(0xecdfa689), LL(0xe6296cd5,0x4660e301),LL(0xaac1b188,0x8e86f81b),LL(0xe2709e66,0x1690d7e4),L_(0x9ad43bd5), + LL(0x31a8090d,0x9e4966d9),LL(0xe9803ed7,0x2923afcc),LL(0x94a51c0a,0xc6ffb2f1),L_(0xac8d73e5), LL(0x7e9362e3,0x87ee51c6),LL(0x0f2f29a5,0x9df72731),LL(0xcf07fe4d,0xf7993328),L_(0x11a086fc), + LL(0x366308d2,0x482c8576),LL(0x7008cc6c,0x04061b0e),LL(0x4ea34321,0xea139dd1),L_(0x0db793be), LL(0xd163362c,0x2f88c485),LL(0x46daeb88,0x20d2b379),LL(0xa9529777,0x8fc9a2fe),L_(0x685903b2), + LL(0x4733e75d,0x160fbbaf),LL(0x85dc20a6,0x8dfd433f),LL(0x40ce263c,0x33ee3772),L_(0xf18c955f), LL(0xbc745c54,0x7c56f68e),LL(0x629b2ed0,0x7ba316a5),LL(0xd3751f88,0x9be35b3d),L_(0x87658375), + LL(0x65718fca,0x4c71f19d),LL(0x2e6a930b,0x58ad9bf9),LL(0x6ce370be,0x9c7c9090),L_(0x9fe1a225), LL(0xd3e1755a,0xdfc19acc),LL(0xbc4095c2,0xeb61289d),LL(0x102343b7,0xa07aed0a),L_(0x75593f8f), + LL(0x892da157,0x6e87ec4f),LL(0xe6abe1ed,0xe3973fff),LL(0x32e1d48d,0x9c1aa3b0),L_(0xbc16fee9), LL(0x64341200,0x1a098c5d),LL(0xa549cfc8,0xabb4ca0a),LL(0xdccd7aa4,0xe2b2179c),L_(0xf0addee3), + LL(0xa357e9f5,0xec32e6a1),LL(0x4e4d505b,0x0d55532a),LL(0xd584c5e2,0x708100e5),L_(0xf05981fa), LL(0x9188d927,0xff71c065),LL(0x30b8c50e,0x099da401),LL(0xa5cf657a,0x00705bdb),L_(0x003593a1), + LL(0x634a3d0c,0x1e13fadd),LL(0xbfa0d04d,0xbc86e569),LL(0xcc81bd28,0x39bdb34f),L_(0x7ba2330c), LL(0xa09b0aa7,0xb5b45e68),LL(0xf331bfd7,0xf478e7fb),LL(0xdd8ec1b5,0xb3f734e7),L_(0x9064cd97), + LL(0x156f54eb,0x6a4afc5f),LL(0x4194a70e,0x4e2a403a),LL(0x523cff28,0x03d08e2c),L_(0xa27e7edc), LL(0x994e73a6,0xfc7c7e86),LL(0x38873c54,0xa0320256),LL(0x805abf3c,0x414e3976),L_(0x93991ee1), + LL(0xcf5e586e,0xa5d47503),LL(0xb3fe93fc,0xcd1bfca9),LL(0xb7d168b6,0x4a438e8c),L_(0x581ebd7a), LL(0xbbc7084b,0x1e7c3d14),LL(0x7749a42d,0x66b2f40c),LL(0x86e109ed,0x853c9b29),L_(0x069c5f0b), + LL(0x51236167,0x2fb43299),LL(0x2b3c8790,0x48986b12),LL(0xc7e0dd08,0x9d9741bf),L_(0x51f341ad), LL(0x9e792a48,0x6eb8a76d),LL(0x1cd0ce38,0x20f5141d),LL(0x2698a77b,0x3099a97a),L_(0x6b8968f5), + LL(0xdcec6ecf,0x5f3a4ac2),LL(0xf86e098a,0x8c8b8e0a),LL(0x3194f379,0xc98e48aa),L_(0x886d05b8), LL(0xe568b10f,0xbd9be136),LL(0x488df6bf,0xa20174d8),LL(0x9833b00c,0xfb9f6810),L_(0x5fb8ce68), + LL(0xb887fe68,0x0996c579),LL(0xe74e34d6,0xdd48ea52),LL(0xbc8f1cc6,0x49ddbd76),L_(0x3e421dda), LL(0xd57b1ea4,0x2fcb399b),LL(0xf599d67b,0xde52325a),LL(0x1de7c3a8,0x60ffdfcb),L_(0x41018e1d), + LL(0xda8c07fc,0x5803c900),LL(0x1834f6f5,0xab321f3b),LL(0x0269aa98,0x51b8942e),L_(0xbd9cdd7b), LL(0xad131077,0xf11dba7e),LL(0x62d1e404,0xf313e57d),LL(0xa355486e,0x3e7a1a96),L_(0xa6d151c2), + LL(0x7855f86c,0x5bfcbb65),LL(0xd3cf6d12,0x06e90560),LL(0x509c593c,0x36beb1c6),L_(0x50bb9be7), LL(0x89a08a0b,0x461e9080),LL(0xf6ebcef8,0x278adf68),LL(0x53fb4ce1,0x1d3a4bb3),L_(0x84afbe92), + LL(0xda5ddb0e,0x36690883),LL(0x66809ebd,0xa2cf5571),LL(0x6d7b75a0,0xb7561328),L_(0x7a0b342b), LL(0xdbdf8401,0xc6b75bd0),LL(0x8b464a87,0xbcc0ebce),LL(0x145091bc,0x33cace99),L_(0x723fe6fa), + LL(0xcfc2a3f9,0xfa386bf6),LL(0x5ea0874e,0x7749b51c),LL(0xa8387298,0xbc4e6a04),L_(0x02c73e59), LL(0x42c5707c,0xa4b7c2f1),LL(0x2fdbbf21,0xea7c0fdd),LL(0xb933bbc7,0x874e47bf),L_(0x4c88b532), + LL(0xc90a29b6,0xb79dad0a),LL(0x94b27e28,0x833287b1),LL(0x47dac83a,0xb3a5c35a),L_(0xace3bd6b), LL(0x318b5daa,0x0d91b2e7),LL(0x00f91b6c,0x2490c7d0),LL(0xd6d7f96d,0x390f1d25),L_(0x92ea6b8b), + LL(0x4f12bc87,0xa7f8fb95),LL(0xf0e6068b,0x6fd2151b),LL(0xed56b92b,0x64cd2ac4),L_(0xd3d78730), LL(0x6abfc96c,0xf0d95041),LL(0x8db9e812,0xa3638d2c),LL(0x6447eba8,0xd45b6fa7),L_(0x3b769af7), + LL(0x54f9cda5,0x26f167e9),LL(0x8aa2617b,0x869821a5),LL(0x52ead3d1,0x41988291),L_(0x5273a693), LL(0x32358d32,0x18ca4d80),LL(0xb1bbd253,0xb8619557),LL(0x58878e33,0x64d8aee8),L_(0xe8908e43), + LL(0x64ed9a67,0x5a517622),LL(0x6d83d019,0x742a30e3),LL(0x969711c0,0xa7fc9902),L_(0x225d81d7), LL(0x8b9e5575,0x25981f3f),LL(0xe02b87e7,0x78b544b4),LL(0xaec410d7,0x35630262),L_(0xb1115eaa), + LL(0xb95b1070,0x83f05d8d),LL(0x4dc83cf2,0x225fcf2b),LL(0x523db1cd,0x09d299d6),L_(0xe968d6f7), LL(0xb132ecd5,0xa5406081),LL(0xc545ab6a,0x645c2ab4),LL(0xcf6e254a,0x592bcabb),L_(0x599530e8), +}, +/* digit=22 base_pwr=2^154 */ +{ + LL(0xda598881,0xa46f08d8),LL(0xb9fe1aa5,0x4cf987a8),LL(0x32c26e96,0x1b37edc0),L_(0xcc6e653e), LL(0x8ea05891,0x37d78fa4),LL(0x44dcd6fa,0xb6453df3),LL(0x637ed305,0x72834a3e),L_(0x6cab1cc9), + LL(0x1ff1de02,0x2efe30a1),LL(0xd9e804cc,0xfc2c0f38),LL(0xb9c440fa,0xb6ba8131),L_(0xd2f7d77a), LL(0x60e1b4f3,0x88842b69),LL(0x96f7f20a,0x008db8f2),LL(0x9a618fff,0xf13ceac5),L_(0x6577f60c), + LL(0xd0b89306,0xbd482556),LL(0x1653fafd,0xc3361427),LL(0x39c09ddd,0x10b62eeb),L_(0x79f6d014), LL(0xd744c03c,0x78d81b00),LL(0xe8963c96,0xec51067e),LL(0x8c8f6a4c,0x63b70385),L_(0x551fa28d), + LL(0xf611c717,0x433a33e9),LL(0x2417a7ce,0x5f8001f9),LL(0x1d9155b1,0x7a6ca298),L_(0xd78e16eb), LL(0xc11dfcf9,0x3ab289c1),LL(0xee48adb0,0x14996507),LL(0xc998c5c9,0x8dd6fbf2),L_(0x959f66af), + LL(0x82c3a705,0x23a0b239),LL(0x63c09a4f,0xa1f25d1e),LL(0x021b685f,0x7c69331c),L_(0xb1b0820e), LL(0xf86bcb46,0x81992f9e),LL(0x370cbdbc,0x0d158241),LL(0xeecaf1b3,0xaeba071a),L_(0x5ebd44d9), + LL(0x17d040bb,0x193df858),LL(0x7a008e70,0x858d93f7),LL(0x8d7ec281,0x81377001),L_(0x3062b2db), LL(0xffd70fd5,0x11e91ce3),LL(0xdb62847d,0xf3cecc63),LL(0x9a492d59,0x42b1e033),L_(0x3c5e448f), + LL(0x9609b50b,0x48abe62e),LL(0xf00a790d,0x5b827b1e),LL(0xe9c56ab8,0xcfc6643d),L_(0x7a101d9e), LL(0x63e008bb,0xe901d5c6),LL(0xc8b5cd29,0x26d03ac2),LL(0x5a6c0d3c,0xf3eacfba),L_(0x437c7e62), + LL(0x42de8327,0x7ef25e15),LL(0xc3c0b7b4,0xf84d2f14),LL(0x2d411980,0x9d372287),L_(0xb86b3e9a), LL(0x7f225094,0x6682e29b),LL(0xc7ddb5da,0x7638e99e),LL(0x80ad8746,0xc5272ca9),L_(0x67e42112), + LL(0xb19ef327,0x2db21213),LL(0x26d65fa2,0xa797a69c),LL(0x27bc92ce,0x780c1507),L_(0x62bfbfb4), LL(0x02fe91fa,0x90f53f41),LL(0x306b2065,0xd9ad25ed),LL(0x70add874,0x97f0fe05),L_(0xef3f82de), + LL(0xf3beb3c0,0xdb734eed),LL(0x49ac57da,0x4035d15e),LL(0x0575235f,0x12e6b4bb),L_(0x84ba90a7), LL(0x2a740735,0x2e72cdc8),LL(0xde178a6e,0x32f40243),LL(0x2c8f9b53,0x04316bb2),L_(0x4d9021c4), + LL(0xbcf71c73,0x4a88c71f),LL(0x7c1da7fb,0x646fc8a7),LL(0x7be262ad,0x26f4415e),L_(0x8d16df29), LL(0xbe32e30e,0x6e047967),LL(0xf7050746,0x935f128c),LL(0x40048ee7,0x671c053f),L_(0x7e714450), + LL(0xa89d04f2,0x126208fb),LL(0xc3f913fb,0xcea88f22),LL(0x81ebbd24,0xfbf9d09c),L_(0x60be0397), LL(0x554a7a99,0xab78592d),LL(0xee7a8b54,0xbca7876a),LL(0x58636e68,0x5cb0cea7),L_(0x56d9eb18), + LL(0xf22ebb87,0xe2adb3e2),LL(0xe08bf721,0x9944dfb5),LL(0xcf7b1348,0x88809e01),L_(0x9dd0883a), LL(0x57335ddf,0xfb2d7aec),LL(0x9e8ac3f6,0x91eff023),LL(0x3aa9be67,0x87a97238),L_(0x56210cb7), + LL(0x88f902c7,0x515e6712),LL(0x989ff215,0x47c3afa5),LL(0x8011a541,0x16b0e517),L_(0xc8b38bfd), LL(0x4bc31c65,0x23c98753),LL(0xc13358b8,0x3b82c767),LL(0x5edc147b,0xae944131),L_(0xbf36526b), + LL(0x8561f47c,0xbb74f4a2),LL(0xf57da9fb,0x4788db1a),LL(0xe6c34f9a,0xae99e920),L_(0x72b76a37), LL(0xbdc2a795,0xd46a4b66),LL(0x377b9f4d,0x1b102f7a),LL(0x2dbc27cc,0x1c0d51f0),L_(0x63d0ccbb), + LL(0xf20ce9cb,0x76b14e73),LL(0x5173ac48,0xe87b13ab),LL(0xac29f6c0,0xde2af466),L_(0xbf4405a4), LL(0x5cd87093,0xe0a96171),LL(0xc72bdfba,0x6f175bde),LL(0x14f78dd7,0x2c991165),L_(0xf55ecef7), + LL(0x94e2d286,0x6d393d5e),LL(0x05e626ee,0x250d4b04),LL(0x225ed4d7,0x53d41ea6),L_(0x5d5b37e5), LL(0x6770d691,0x70c4e492),LL(0xf4527dfb,0xdeeb2466),LL(0x1522deea,0x333509e1),L_(0xad711cce), + LL(0x5bd9660b,0x072fecfa),LL(0xba0c11d9,0xeac96039),LL(0xa6fa784d,0xa105395f),L_(0x3a914aba), LL(0x7ecb3cf0,0x5b5ec37c),LL(0x6c8c0e8b,0x8123cd86),LL(0x79696f79,0x69daa4dd),L_(0xbf1748ba), + LL(0xa78decff,0x16919c0f),LL(0x43014c2d,0xfbc474bc),LL(0xd14b6a12,0x6d705b75),L_(0x395d7526), LL(0xa7378944,0x4c3aec7f),LL(0x3f4fd283,0x8737fb98),LL(0x4114b4f9,0x6a63c71b),L_(0xcf962aae), + LL(0x99ce3cf2,0x085cede3),LL(0x39a2603a,0x87ec3837),LL(0xb203a75d,0xbdc1e809),L_(0xf3640f89), LL(0xd93d4aae,0x8d9d5264),LL(0xd480817b,0xd455e58c),LL(0x8492b58b,0xe3368ba5),L_(0x24b8bfe6), + LL(0x5ec2512d,0x64ef9737),LL(0x6907b7bc,0x78d38667),LL(0xf6e9c509,0x33b42737),L_(0x55ab9706), LL(0xfaa748d1,0xf586dde6),LL(0x76b6d961,0x154becfd),LL(0xad1c2111,0x13031322),L_(0x1f36dd8c), + LL(0xa1af32ba,0xe83a229c),LL(0xe6ae70d0,0x4ec7d7fe),LL(0x5b9ca0d3,0x06479e73),L_(0x17cdf808), LL(0xad7c2138,0xd020f0ab),LL(0xca397995,0x85bbbbc2),LL(0x70369103,0x3d37e46a),L_(0x67d7bcef), + LL(0xb1610c52,0x920986d3),LL(0x9e2b8580,0xbe1fdaa6),LL(0x1a1086a7,0xf038b9fd),L_(0x76630d46), LL(0x62b5dc1c,0xf2780d63),LL(0x479ba46d,0x97c26f70),LL(0x41a1c690,0x953b6640),L_(0xeee5927b), + LL(0x50cfff54,0x7a6dcf7f),LL(0x2905feea,0xd9932f1f),LL(0x5bba1fd8,0xd274195e),L_(0x596a73e6), LL(0xd4297f7a,0xfa2e2812),LL(0x83723184,0xc5b16141),LL(0x21f09629,0x7ce090e3),L_(0xb6dc2849), + LL(0x7d63db18,0xb254b1d4),LL(0xfee577f0,0xaf830093),LL(0x6accfc0b,0x775f693f),L_(0x483bf6a8), LL(0x0e851bfe,0x74582d9f),LL(0x6731bfc7,0x7ffa6bf2),LL(0xf7d39569,0xa67faa6c),L_(0xc9418804), + LL(0xf9316a6d,0xb5eb4f28),LL(0x7a12fe1e,0x8cfe6abf),LL(0x4a7fc70e,0xd820b6bc),L_(0x319719c3), LL(0x54fb2143,0xbd26f428),LL(0x66648099,0x64ab8443),LL(0xb662d063,0x7fbe15d7),L_(0xf16d93b4), + LL(0x0457afa4,0x951453e4),LL(0x8f99c3ae,0xfd74995c),LL(0x422aab2c,0xec8547fb),L_(0x73be771b), LL(0xe916d5c5,0x49e82e71),LL(0x7b15c573,0x3a725205),LL(0xaa7e7ca5,0xfd867aa0),L_(0x58e19238), + LL(0x31434cbf,0x31e49c5c),LL(0x2d0ee2e6,0xaf660f6c),LL(0x3aaf91fd,0x271cbc87),L_(0xcb833e79), LL(0x520b6d31,0x36952e46),LL(0xf7a6bc06,0x6695996b),LL(0xda163497,0x1f6ccb97),L_(0xb5f271bf), + LL(0xa5259a0d,0x606962f8),LL(0x82d18205,0xf314ac21),LL(0x4a97cf98,0xde06a403),L_(0x23969744), LL(0x43ff18b9,0xba5e083d),LL(0x22229508,0x3789db89),LL(0x376f004b,0x1c53a79e),L_(0x8ecb5e4a), + LL(0x3757ad7d,0xfde8f5af),LL(0xdaab7a45,0x313f26c0),LL(0x6cba2e40,0x0b7de7e1),L_(0x31db2861), LL(0x90144bb6,0xf98e6b34),LL(0xa8d3e811,0x0ec080af),LL(0xc4961d3b,0x4bad9114),L_(0x10d64cdd), + LL(0x1ff6dfd6,0xadd9fbd0),LL(0xca3624bb,0x4c069f73),LL(0x73a43aed,0x91301b03),L_(0xc382f632), LL(0x9d2aab1b,0x83cfddff),LL(0xf9e8d9a9,0x66a4ad61),LL(0xb9fbd92f,0x25a8b014),L_(0x760229d4), + LL(0xc29bf199,0x95ce63b8),LL(0x225d1531,0x1e469d80),LL(0xf1862d72,0x143d1930),L_(0x89faba0c), LL(0xac073163,0x36978268),LL(0x2f3bfacc,0x077ad847),LL(0x8ecd6948,0x817ba1fc),L_(0xad14a0f1), + LL(0x4b5edd16,0x9cf6404b),LL(0x0fcc7993,0xf9fdd94b),LL(0x94227ce7,0x36c49b8b),L_(0x1add1a97), LL(0x06eb3245,0xddf10442),LL(0x478ddbe9,0xd0e75910),LL(0x598d307b,0xa7d77b58),L_(0x7c2d4285), + LL(0x640b82b6,0x6ae40efb),LL(0x7296f54f,0xc265b407),LL(0xbb8c6473,0xa3248f4d),L_(0x096d9531), LL(0xb52d1b77,0x72de2ce6),LL(0x71d14af5,0x2df2b5bc),LL(0x8bb3132f,0xd5a9918e),L_(0x1ee08068), + LL(0x7efa31af,0x3a3e84c4),LL(0xdb4c764e,0x41d38660),LL(0x06967d91,0x9a2fe8e7),L_(0x3d0cd72c), LL(0x81428c4a,0x663d93ab),LL(0xd9a81336,0x463057c2),LL(0xe092d510,0x7d2f5c90),L_(0x45eb443a), + LL(0x482cdab4,0x29b44f02),LL(0x09859196,0x88f3dd7a),LL(0x274e7582,0x8a9337d3),L_(0xe9cba9ed), LL(0x30787ea1,0xab1d6ae1),LL(0x1aa44fa6,0x72445ee4),LL(0x8201c08b,0x2f9c1703),L_(0xd128d342), + LL(0xdf8f55c0,0x04dbb488),LL(0x4b130afb,0xa603695f),LL(0x010d7b25,0xf211bcec),L_(0xb7f03740), LL(0xf354eee1,0x24a2aec6),LL(0xf116e286,0x473d0ffe),LL(0x4b6af9fa,0x974e5b38),L_(0x19cbf327), + LL(0x3d701f1c,0x8816e065),LL(0x52f1347b,0xd04d2137),LL(0x733b5b5e,0xe33961b0),L_(0x8db340e4), LL(0x02cb30d8,0x8fdad9c6),LL(0xb3cc8f09,0x63fea3f5),LL(0x36371a82,0x36432409),L_(0x9dd5d017), + LL(0x3ac09ce0,0xc45e9f48),LL(0xce25491e,0x0b828bf6),LL(0xf8ea793c,0x27eb6c22),L_(0x6371b206), LL(0xb1d7932a,0x80e32e78),LL(0x61eaf6df,0x8cf6f356),LL(0xdc4f44ca,0x387b7804),L_(0x4b85bf6e), + LL(0x1b431c86,0x2c7dc474),LL(0x18a6a123,0x6c31f68d),LL(0x53665bca,0xaa43dbfb),L_(0x8d40fe26), LL(0x7d646aab,0x6723ea8e),LL(0xbade3e08,0xcd9dea27),LL(0xc7adbb73,0x2f3a4dc9),L_(0xa74a2565), + LL(0x7fa9020f,0x9ab672ef),LL(0x9cc7bfa2,0xa0040dfb),LL(0x2ac7eb07,0x368be66e),L_(0x5665ccf4), LL(0xba527a40,0xb250a7ab),LL(0x465d7f98,0xa9c591cc),LL(0x8d2b97aa,0x7b979097),L_(0xa50d476d), + LL(0x4b3aef47,0x8b68fb1a),LL(0x542fb7f1,0xadc18f71),LL(0xd96b7a34,0xb020cf53),L_(0xdb72531d), LL(0xdf800c86,0xc73ecdcb),LL(0x1c4cd34a,0x47adcb2b),LL(0x039494e8,0xd23f79c7),L_(0xa21edcfd), + LL(0x849075c6,0x17c4dc71),LL(0x5ce9dcb0,0xe5cdb7bf),LL(0x25936d77,0xebe08b04),L_(0x70fc7789), LL(0x25213d26,0x7f3546f4),LL(0x7563a201,0xc56c68df),LL(0x10bfdb20,0xdb570af0),L_(0xeb5d3434), + LL(0x106086c7,0xef345020),LL(0xd0e87d76,0x18e51b40),LL(0x222e0b10,0xe86e0a88),L_(0x6fa3b5c9), LL(0x410e6c90,0x9da8796a),LL(0xabdf6b2a,0x4fe0bb39),LL(0x7ead1796,0xbbc76905),L_(0x8c71560c), + LL(0x7da50c4d,0x890d8ce1),LL(0xb5cc05b2,0x2a181e5a),LL(0x7de1e29b,0x866cd575),L_(0xfc749a26), LL(0x41ec00b6,0x7eaa0bb5),LL(0x886271b4,0x2328a91e),LL(0xed069b46,0x65bd26ca),L_(0x2f59240d), + LL(0x4df1cae4,0xd954621f),LL(0xcbae178c,0xdf03015e),LL(0xe71c4857,0x70950b7d),L_(0xc8f7ab52), LL(0xc0de4c81,0x52a1a22d),LL(0x24d86b6f,0x23da02f5),LL(0x70b94a09,0xd450bfb1),L_(0x0d983cdd), + LL(0x3e6f9967,0x5619c34a),LL(0x77d8a74f,0x47b1c13c),LL(0x71f4dc89,0xfa0612da),L_(0xb127ccb2), LL(0xcca647d0,0x879fb15f),LL(0x205b4589,0x5e51a2db),LL(0xe8b6c8f4,0xaad791a8),L_(0x1490d5ca), + LL(0x1b020bba,0x32c5bb8f),LL(0x28810885,0xc2383a3f),LL(0x2ceffdff,0x14a9a0ad),L_(0x7970ea64), LL(0xc1169c12,0x8708e80a),LL(0xbbf83ccb,0x5a067425),LL(0x5cc8baac,0x800c8eba),L_(0x98c8b359), + LL(0x8f103cfd,0x22c84664),LL(0xdbd93971,0x993bf6c5),LL(0x697a6de7,0x6f50eaa6),L_(0x87cb9042), LL(0x9314a935,0x5f70b39d),LL(0x2af4f24a,0xc319862f),LL(0xd8a16a0e,0x8f4571e4),L_(0xa7b9ed81), + LL(0xf4ec13c6,0x6e1de980),LL(0x725de358,0x0fffcef7),LL(0x85652ca3,0x50b9f863),L_(0xd5f0033d), LL(0x312ea9c7,0x19c015bb),LL(0x948391b7,0x92417df8),LL(0xaf6e5ed6,0x33c66538),L_(0x96bfa703), + LL(0xab942a96,0x069d03c0),LL(0x47f60293,0xc848a1bd),LL(0xc32780f6,0x3bfeba8e),L_(0xc2e0eb4e), LL(0x4521b895,0x29bd4112),LL(0xc233f03a,0xe259118f),LL(0x574da3b6,0xf4f6a7ac),L_(0xae0e0260), + LL(0x16cf432a,0xbb8f2924),LL(0x5a087018,0x455c67ee),LL(0x8c5485dd,0x351ad0f1),L_(0xf7d7679e), LL(0x318e4ede,0x2bdc4896),LL(0x277bcf81,0x8b26b0fb),LL(0xf147c5b4,0x87185cc7),L_(0xdf48c9e6), + LL(0x196afebb,0xdb75cc0f),LL(0x17222dc7,0x225c2aa1),LL(0x3dd256a1,0x3a012a3b),L_(0x7ec5696a), LL(0x25bf2c5f,0x9381bed5),LL(0x4f4cdfd3,0x0f7e0324),LL(0x7b5b13a4,0xbf5f2e84),L_(0x2639cce3), + LL(0xf2e480ae,0xdb334a58),LL(0x6da64943,0xe10cd107),LL(0xfce96a35,0x6ad5dfad),L_(0xa599a89e), LL(0x26a35a0e,0xeddc6625),LL(0x2ef746d1,0x5f6fe79f),LL(0xa912c3f0,0xd6859ab6),L_(0x1ac3aba5), + LL(0x705906c9,0x0e5c7c8c),LL(0xb050ceeb,0x5f2aa907),LL(0x53e4bd2f,0x710688b3),L_(0x19a215e7), LL(0xb2485b2a,0x9a042c99),LL(0xd22cc140,0xf04c2421),LL(0xb5f7e2eb,0x7f1f59f8),L_(0x154453dd), + LL(0xc0354be0,0x9473341b),LL(0x9f33393e,0xb5b4c257),LL(0xb3dc51a3,0x09fb2116),L_(0x0ee5b84f), LL(0x3e9814a0,0xf91ac675),LL(0x816d6ca6,0x506f79fd),LL(0xc0c5edef,0x0cfcd5ed),L_(0xb71c05e5), + LL(0x400799f5,0x17c9bd2a),LL(0x4c0897b1,0xd4538ae6),LL(0x7a16a169,0x0f41e0dd),L_(0xa88a35f1), LL(0x4037d80f,0x5e5fc718),LL(0x85b8baf5,0xee15dcea),LL(0xd33b70cd,0x5dd4f3b8),L_(0x5ddf49c1), + LL(0x0095bcdb,0x795a3892),LL(0xedb3e7f2,0x6c1761f4),LL(0x33c80575,0x26818153),L_(0xf7414144), LL(0x943d7d31,0x6f311ca0),LL(0x24d6e452,0xe928eca1),LL(0xe7193165,0x66d1a963),L_(0x89fa62a1), + LL(0x8faef020,0x375aa47c),LL(0x70c638b9,0xc033edb6),LL(0xf167acbd,0xddf9d01a),L_(0x364bf41a), LL(0xf8bb63ab,0x7d82f66c),LL(0x8862080b,0xedb30cf7),LL(0x0448ac9c,0x4dbdd638),L_(0x971eaebd), + LL(0xad55ab2a,0x30f6e053),LL(0x4b4739f2,0xa878cbee),LL(0xd62b9aef,0x3bbafe26),L_(0xb84b80ac), LL(0xc8ef692b,0xd03fc3fd),LL(0xb6803cd9,0xe63bac37),LL(0x81afcd95,0x205a14b1),L_(0x7a0ade0e), + LL(0xff50050c,0x2d735063),LL(0x4b1c54dd,0xab849fa5),LL(0xda4629d6,0xe95c14ae),L_(0x6f216326), LL(0x74ba5b90,0x2fae20c6),LL(0x3112101e,0x5e3d1209),LL(0xbce52748,0xfcd465ea),L_(0x53fd0b9b), + LL(0x2d491553,0x068c0d92),LL(0x1f9856a8,0x37cee3f9),LL(0x3408da24,0xc9b9b412),L_(0xc63b4cb7), LL(0x74b30760,0xb7836b39),LL(0x200fd925,0x339dec72),LL(0x55a6ddf0,0x2b7f407e),L_(0x0fb2a9d1), + LL(0xa824468a,0xe36962f5),LL(0xfae22eee,0x0a42cdab),LL(0xfe866c3e,0x8acadf81),L_(0xa3f599f7), LL(0xcdd076d7,0xe8b6b61e),LL(0xb4b13551,0x46d40c51),LL(0xcdd8f22e,0xb9099888),L_(0xe1b8ed2e), + LL(0x60051d31,0x7e8f146e),LL(0x3818b39f,0x4cd2aa83),LL(0x16886905,0xf677bde6),L_(0x022ae138), LL(0x98b45d19,0x5746a458),LL(0xacf62d52,0x70f9fca7),LL(0x63279bc6,0x4ee46705),L_(0xfd285b9c), +}, +/* digit=23 base_pwr=2^161 */ +{ + LL(0xf0d700ae,0xbad78996),LL(0xc931bee3,0x1b3ef234),LL(0xe3c1a2d8,0x7cd507de),L_(0x58fc32e1), LL(0x83e229c6,0x01135f5f),LL(0xfb8b7b35,0x8d5e43a4),LL(0x413750b4,0x2dffb3fe),L_(0x6a265953), + LL(0x8c78030e,0x796bb2af),LL(0x56c1932c,0x7d56b42a),LL(0xf443c29c,0x6ff56bc3),L_(0xbdd22bb5), LL(0x95857542,0x21178547),LL(0x22575d25,0xbdc732f6),LL(0x7ffb29d7,0x4ffc71d3),L_(0xd6ef14c3), + LL(0x3a92228c,0xbc9613ab),LL(0x5920119c,0x304f1f53),LL(0xc3d2b2e2,0x9633ba13),L_(0xe7d31665), LL(0xc95ebff4,0xe622933c),LL(0xec5df7bb,0x9abfde78),LL(0x50f7f156,0x7feb42b1),L_(0xeb17ef8e), + LL(0x25b9b992,0x1a08721f),LL(0xeeddf997,0xbf152cb2),LL(0xb6ad560a,0x3c761d64),L_(0x52dd9f57), LL(0xc0d79f78,0x39b5966e),LL(0xca15f167,0x562740a0),LL(0xea78b847,0xd9642afb),L_(0x37eea476), + LL(0xd57560f1,0xda4b72e5),LL(0xa3072640,0x68c344ac),LL(0x36a057e0,0x4754e3c5),L_(0x4faa9820), LL(0xe786962f,0xf20160bd),LL(0xa80ee38d,0x77ea80d1),LL(0xc8e173dc,0x665dbf2e),L_(0x2aba1be0), + LL(0xb6e8e52b,0x228ed7a9),LL(0x2d501abf,0x5d4bca44),LL(0xe4f5d2f6,0x3dad807b),L_(0x0b4aa28f), LL(0x006c4c0f,0xe672b21f),LL(0x0888b653,0x614c5f64),LL(0xa3e477ae,0x226f83cd),L_(0x0afdcedc), + LL(0xd5a1f5d3,0xe7476654),LL(0x562eb536,0x7a0ad378),LL(0x7f5bedc0,0x2887f756),L_(0x8581b7e9), LL(0x9986fd24,0x160c3504),LL(0xdde60b62,0x06b5ca84),LL(0x8a8ca29b,0xb94770da),L_(0x3ecfeca1), + LL(0x28c355ca,0xb615d5a7),LL(0x8ea9ecfb,0x2c8d2d4f),LL(0x6be624fe,0x7f81ee5d),L_(0x45035a93), LL(0x97611fa5,0xc3d23100),LL(0xe9c562fc,0x3ccce59a),LL(0xf451c22a,0x6ee17a32),L_(0x932ddcfa), + LL(0xfa127ca0,0x6337e908),LL(0xe970ce2d,0x839c1246),LL(0x58d5ae88,0x253afb1a),L_(0x88db9fe4), LL(0x4550894d,0x3d74de43),LL(0xa90b033e,0x46e1ab37),LL(0x132dd86c,0xed163aa4),L_(0xc88fa0d9), + LL(0xf244b8ec,0xa0e34d4b),LL(0xa5b69894,0x9ac53a57),LL(0x495e2cc1,0x987653cd),L_(0x5e895b41), LL(0xf2fd1824,0xf289cc20),LL(0xc41882ef,0x6ce96956),LL(0x3329b3be,0x4acf73b8),L_(0x9efd2db0), + LL(0x3bb5fbc8,0xdd8e91ba),LL(0xf8f6f870,0x8acac0fe),LL(0x728bf4b5,0x93d4067d),L_(0xb9c4891b), LL(0xaef29279,0xf8bf9b95),LL(0x1e5e8506,0xdf786531),LL(0x11821ebf,0x99083ad2),L_(0x7e9d0c84), + LL(0x83dcfec2,0x85547390),LL(0x48ab6926,0xc7ed1aa0),LL(0x1badb306,0xdd5f7613),L_(0x4641e82a), LL(0xf53ff561,0x3d43a296),LL(0x58c32dfb,0x7b2e7561),LL(0x3ffcd8f7,0xc250c17a),L_(0xfacc3394), + LL(0xda8507d4,0x99456e76),LL(0xe9a40562,0x9ba62d11),LL(0xa165351f,0xd0dd1798),L_(0xbabb6293), LL(0x853dcb7c,0xb83a079a),LL(0xa5cd33ea,0x26c9aa1e),LL(0xf120f051,0x6109e625),L_(0x774354d8), + LL(0x14c27c7a,0x8d82b0bc),LL(0xbafdf31f,0xe88bea46),LL(0x7639e998,0xd2a8308f),L_(0x3be638b1), LL(0xc5e7e09d,0x54bc114e),LL(0xeb650088,0xd1baff80),LL(0xebdef489,0x902bb33d),L_(0xc3e46171), + LL(0x45911106,0xc86362a5),LL(0x56cc1bef,0xce560fcf),LL(0x3b4b4fc8,0xf9436f6d),L_(0x370a01fd), LL(0xa8fad83b,0x632c516d),LL(0x916aa8c7,0xae217497),LL(0xe3ad18e9,0xe18b971a),L_(0x72c7e5a4), + LL(0x9b05d089,0xb483969f),LL(0xf8cf3d39,0xcfd11869),LL(0x1d89563e,0xa602a006),L_(0x91cd1395), LL(0x71d49ace,0xdbdaa432),LL(0x0966d0a9,0x1c3623ed),LL(0x166bebe1,0xf1b4da0f),L_(0x433f9113), + LL(0x86d4180d,0x6fb1d5d9),LL(0xd89da757,0x031c30db),LL(0x2939bdad,0xa42134e8),L_(0x6b13fefc), LL(0x1baf8cf7,0xb37f662e),LL(0x413f92df,0xed2b1683),LL(0x985c742a,0x1de586e7),L_(0x5f4904de), + LL(0x01f2ea9b,0x03906d91),LL(0xb13c4090,0x9d1c566b),LL(0xf3affee6,0x6830a295),L_(0xb7fc1017), LL(0xa0a0056c,0x45d24eb6),LL(0xcb2d95c2,0x622219f3),LL(0x8c20bcd3,0x3379e3da),L_(0x6dc4634a), + LL(0x3e8ae0a0,0xaefdff77),LL(0x97e82574,0xe4ee96dc),LL(0x112d1838,0x20061ca4),L_(0x0060cd67), LL(0x20825ad2,0x11499a1e),LL(0x8b6913cb,0x1c239b7a),LL(0x67ff5e87,0x1680aa3a),L_(0xe823c3ec), + LL(0xd6ad0217,0xd3ac8a55),LL(0xd2b4232a,0xf6ddec57),LL(0x1b8abf52,0x2fcb86b4),L_(0x19bc7365), LL(0x1f2c9fd1,0xa051bf99),LL(0x1aec4d94,0x5a88054d),LL(0xfbc4f987,0xa954ceb9),L_(0x05b104cc), + LL(0x4ccd6345,0x40690896),LL(0x4799ff54,0x3b0d618b),LL(0xbc95c20a,0xc0b5b3de),L_(0x290686c2), LL(0x488b692d,0xca443d1d),LL(0x6b65747b,0x823770ad),LL(0xf2987b72,0xf039026d),L_(0xb9b37c0a), + LL(0x8dd8e6e6,0xd360d772),LL(0xd81505e8,0x994209e4),LL(0x40816745,0xc9836e31),L_(0x9dca53d3), LL(0x07ea2add,0xd820c03f),LL(0xb45d3f5d,0x33d13943),LL(0xedf3a3c9,0xe0ce5e44),L_(0x459a2236), + LL(0x3f40e1dd,0x35ffc3c1),LL(0x273c8e42,0x9e607b7b),LL(0xfecec30d,0xda6a8389),L_(0x8e8dc1c3), LL(0xe7f0596c,0x6fe132fd),LL(0xc3565543,0x8889d9c6),LL(0x951334fa,0x770ed88f),L_(0x4d1d476c), + LL(0x542a815c,0x20e67927),LL(0x488e81fb,0x7f7fe269),LL(0xd968afb4,0xb0057d32),L_(0xb8d04c1b), LL(0x85a86d34,0xee01ab82),LL(0xef38d717,0x5e5709d8),LL(0x830deaea,0xfba87128),L_(0xbf20a3cb), + LL(0x9efe5459,0x15750d83),LL(0x8d8facef,0x4d132c09),LL(0xef0f3af1,0xd1a4206b),L_(0x64ebf883), LL(0x5e56d99d,0x5b6d2c3b),LL(0x3c48ef20,0xce5ccc62),LL(0x153580c5,0x534ba912),L_(0xa02f889c), + LL(0x349d9804,0x1f2a638d),LL(0xcb233fe2,0x9d44fa20),LL(0x792d0e00,0x62d03b08),L_(0xd6646af8), LL(0x180ef2f5,0x4a8172cc),LL(0x82199f97,0x401272d5),LL(0xf879213c,0x8048de29),L_(0x51bdf94f), + LL(0x9c9a3b84,0x75f12d75),LL(0x743c75d7,0xaadd691d),LL(0xc13528c2,0x0f8f0fc7),L_(0x969b010e), LL(0x9b8f4442,0x2acbaab0),LL(0xdfc6ee9c,0x4678a76c),LL(0x23ac3530,0xc00ef611),L_(0xdfc77ba6), + LL(0x3fe1d500,0xd561d679),LL(0xac256d68,0x7e3b2fc9),LL(0xb7f44d84,0x1d8cc6f2),L_(0xddd4acc6), LL(0xccec3e4c,0x83d18ef0),LL(0x6b5fbe51,0xf9877554),LL(0xe616da91,0x04e7aef1),L_(0x7079d8b4), + LL(0xacf8b8b3,0xc0f7d69f),LL(0x2242759d,0x0a034401),LL(0x9949d06b,0xde898701),L_(0x8b24618f), LL(0xa9a593b3,0x3976254d),LL(0xb0bd22ab,0x776af23a),LL(0x2125ed2b,0xa5f28d83),L_(0xa1c543ea), + LL(0x4bd74e55,0xe7ee1b28),LL(0xf2a06317,0xec3c82fa),LL(0x784961c1,0xf759d96d),L_(0x5670b8e7), LL(0x2db86f19,0xafedb099),LL(0x8fbfe519,0x19d6aa87),LL(0x7246a16a,0x7503e23c),L_(0x23ac9af3), + LL(0x952bc75c,0x87836e61),LL(0xc31169c7,0x278dc1c5),LL(0x111050a4,0xa7810165),L_(0x446bda32), LL(0x490d4091,0xa876bcf8),LL(0x32692fd4,0x7e43410e),LL(0xd11e3c6f,0xe2aece33),L_(0xd325fefc), + LL(0x209ded15,0x8c31bbf9),LL(0x0e3623bf,0x818a6e0b),LL(0xbb5c266d,0xc5f4aaef),L_(0xa7179b54), LL(0x15383b16,0xed087186),LL(0x375cb291,0xbbe070e0),LL(0x047af636,0x73198d27),L_(0x8f1c0b06), + LL(0x4cc95bc4,0x77e38347),LL(0xc22c3ac3,0x83e4ab76),LL(0xea1c777a,0xab8de890),L_(0x8b9d1132), LL(0x959ae60a,0x2a2fde96),LL(0x78f20c5e,0x6184d8f4),LL(0xa11f0041,0x302a6212),L_(0xba7689dc), + LL(0x376e162d,0x8c2d9e1f),LL(0x50cda9df,0x7ba9efe3),LL(0xa6158df9,0xadbbeaa9),L_(0x4778c93d), LL(0x4165d3a8,0xd1a49e09),LL(0x6eb6664c,0x276faed0),LL(0xb72ac28c,0xb8fb2671),L_(0xfc493918), + LL(0x3f2f65a3,0xe846a0c3),LL(0xc3ffb873,0xb4100b80),LL(0xd98b14c7,0xcd3fcccf),L_(0xec8c8522), LL(0xde06d3b7,0x27093099),LL(0xc435552e,0x5b30c12f),LL(0x731b5ca4,0x6d3545bb),L_(0xcf95c612), + LL(0x06a11b1f,0x67e83944),LL(0x77e47a85,0x840987d9),LL(0x41b4a73c,0xf14ab27f),L_(0x45673153), LL(0x5e20b8e1,0x29737735),LL(0x3a31ee99,0xdcbc9d02),LL(0x89cb9a71,0x3c610ccb),L_(0x92d6ab9e), + LL(0x7dc21a89,0x24dd0312),LL(0xfe3af3cf,0xbd1479bd),LL(0x1fa695ef,0x49e9da9d),L_(0x35d8485e), LL(0x4caa0416,0x1be78d4d),LL(0x9a8e5ede,0x06a82fad),LL(0xdb4e20a3,0xb9902b0e),L_(0x762073cc), + LL(0x26109f70,0xba8c73ab),LL(0xfd983b8b,0xe8f78dd1),LL(0x3921c9f7,0x9c6ee5d8),L_(0x7f5a5177), LL(0x14bbc8be,0xab53e7b2),LL(0x7db07019,0xfff92c17),LL(0x01f9a8fb,0x2588492a),L_(0x9ea1e98d), + LL(0x924eb396,0x31997788),LL(0xa9a0407f,0xbc33538a),LL(0x26fdd725,0xa3068d9c),L_(0x2aba0c14), LL(0xfb38ce06,0x6d07282f),LL(0x2d5ed5c3,0x0f9417d3),LL(0x52b3241d,0xdbd63903),L_(0x11f4b8bc), + LL(0x88a000ac,0x35994c37),LL(0xccc2b918,0x337027af),LL(0xf27570cb,0x870c688b),L_(0x8c91a8af), LL(0x458bc212,0x5459f2f5),LL(0x9157cd7c,0x7d5dcbea),LL(0x021d9845,0xdbbbf2c4),L_(0x3319b3fc), + LL(0x4a370724,0xaeba325e),LL(0xaecdc264,0xaa40e67e),LL(0x9b24587f,0x859c1b6a),L_(0xf7648c83), LL(0x9da1346e,0x7005a3f3),LL(0xdf465967,0x7311149a),LL(0x2411ca0d,0xd689abfe),L_(0x1452c503), + LL(0x2a06a445,0xcc290bf3),LL(0xf34968d7,0x65fd04ce),LL(0x7228d7c1,0x48e02e21),L_(0x17e25b2b), LL(0x20f72ec8,0xa53a154b),LL(0x1035d2b6,0x8b5f77f7),LL(0xe8ab36c6,0xbcf0eb3e),L_(0x63fa883e), + LL(0xa46b410c,0x0b53b1cd),LL(0x2088982f,0x44825888),LL(0x21364c98,0x45bf87a7),L_(0xeeb978c5), LL(0x77561ed1,0x4e53377b),LL(0x093e9c15,0xe15699a0),LL(0xd8249f4f,0x02b46436),L_(0x64f236ee), + LL(0x5bf80f33,0x3940ee66),LL(0x6164d05e,0x98afb7d2),LL(0xb0056da3,0x872642d0),L_(0xa7bd3d66), LL(0xc227063a,0xce37f406),LL(0xd8371363,0x97040937),LL(0x90362154,0x009ca301),L_(0xe59d0b2d), + LL(0x812ca945,0xcf247bdf),LL(0x73b26b51,0xb0c27a65),LL(0xdf33ad4d,0x1f6d55bd),L_(0x4ed64955), LL(0xc4a11d3d,0xa18aee15),LL(0x8ca23082,0xbd802f68),LL(0x20da76f8,0x2453f147),L_(0x4606958c), + LL(0xfa30299c,0x99950738),LL(0x06b63065,0xbe157ae3),LL(0x9853453b,0x88e04e19),L_(0x456e72cc), LL(0xfcb616c9,0xb168037b),LL(0xd3c22f8d,0xce3f0f0a),LL(0xd989f0f6,0x9699fc87),L_(0x273ac5b1), + LL(0x85664bdc,0x273a933d),LL(0x24f52dc3,0xfcf79a03),LL(0x36bc3c34,0x3e91b0cc),L_(0x61de4790), LL(0x58e3b120,0x8f4b6789),LL(0x44bdcadc,0xb12252e0),LL(0x830bedee,0xbb9ac7e8),L_(0xbda5fd11), + LL(0x44861e3d,0x37a97134),LL(0x079c1b7a,0x6798891d),LL(0xeaf11509,0x3012e200),L_(0x61b63246), LL(0xa87c42aa,0xc0246bd8),LL(0xb92f89e2,0xed286525),LL(0x3fff4a96,0x44cf1045),L_(0x6e70ae3f), + LL(0x5068198a,0x6615d342),LL(0xb3a83dd9,0x0e23ae21),LL(0x0c4c00f7,0x6cbcc29c),L_(0x84d9fe95), LL(0x5e865fd0,0x30e1885b),LL(0x82a4da98,0x5fcfdb20),LL(0xbcc23ab7,0x259db6ce),L_(0x306cf56e), + LL(0x3622f7f7,0x08624c7b),LL(0x32c9c5fc,0x01b193ac),LL(0x38295bfb,0xfb49c594),L_(0xdcfe3634), LL(0x103965ff,0x832aec92),LL(0xe5cda111,0xdb928d8e),LL(0x98fc1728,0xac496bac),L_(0x3f057842), + LL(0x69b7107b,0x7bdd13e7),LL(0x189dee30,0x5aa65303),LL(0x01cf6016,0x02a07254),L_(0x7f6053e7), LL(0x409cab89,0xc9d79b48),LL(0xd467929d,0xe17deac4),LL(0x2a4257b0,0x3ed68c3c),L_(0x0209a951), + LL(0xde7ecacc,0xb50fa3a8),LL(0x568377b7,0x7926440e),LL(0x992fa9d6,0xf3995e07),L_(0xf09351e3), LL(0xaf199c40,0xde00386c),LL(0x7c21c039,0xacbcdf7b),LL(0x983e3f5f,0x970f1581),L_(0xfde7bb8d), + LL(0x31d0cb64,0xa5991d35),LL(0x727713ca,0x20cf37ff),LL(0x81781a16,0x946557d9),L_(0x8b2de3fc), LL(0xd1639576,0xb61c0935),LL(0xefe6ec67,0xcaf4adb1),LL(0x0483381f,0xbfd4710c),L_(0x3cb8617a), + LL(0x19bcb25a,0xfb2c55b6),LL(0x83398c47,0xe6a45901),LL(0xfc55bdd9,0xd6aad463),L_(0x733b8806), LL(0x4da45d48,0x600c9640),LL(0x72a69c3e,0x7bb5666e),LL(0x25c8da01,0xa863f55c),L_(0x6b73f445), + LL(0x8ff6b908,0xb5811bfb),LL(0xdea6b1b8,0x3864ff6e),LL(0x6f29d6ab,0x244c832f),L_(0xb885707b), LL(0x6b307d64,0x19847d9d),LL(0xbb7684f6,0x85f7c9ef),LL(0xc0f2d53c,0x20ef5736),L_(0x9e06880d), + LL(0x8c5dd4e2,0x9a3cdc81),LL(0x6fd2d1fe,0x6a09971a),LL(0xfe48b29f,0xa14e785b),L_(0x7afeb3e3), LL(0xb175e3fc,0x15c1eebf),LL(0xdc5c271b,0xc21625b9),LL(0xd51bbdf3,0x4095982d),L_(0xea6a656f), + LL(0x3418b0dc,0x8df3827e),LL(0xad96082c,0xae05d810),LL(0x0c7308d4,0x747332eb),L_(0x3e790568), LL(0xf43bfba5,0xb410e2a5),LL(0xa45bcc07,0x6ea69371),LL(0x34e11697,0x68990fab),L_(0xd0d95f71), + LL(0x46088da2,0x2b70f4c2),LL(0x95663747,0x2b81bbe7),LL(0x18e76686,0x0510d4ae),L_(0xa806fc64), LL(0x718b9269,0x74a39c45),LL(0x11681c31,0x39edf452),LL(0x7f1ee73e,0xd70c5b25),L_(0x145296c5), + LL(0x38adcbf7,0xb90e977c),LL(0xa73e17c7,0x589db798),LL(0x3b0459aa,0xd910c80d),L_(0x4a996387), LL(0xa2014673,0x09cc1462),LL(0x4ff007ad,0x9deb8885),LL(0x6f3bf601,0x9137e94d),L_(0x3d0461a8), + LL(0x49eeb574,0xdf5e90f6),LL(0x03f3fdc3,0xf38e094e),LL(0x2f05f4ba,0xefe1e30f),L_(0x5dab20a6), LL(0x25f380be,0xfbe83a6a),LL(0x815eac18,0x96d53b04),LL(0x2c89a65a,0x13f75b6a),L_(0x326a87f4), + LL(0xb57a062a,0x99fa6314),LL(0x29c19e9b,0x29ce4a74),LL(0x5d5a6ffb,0x84d62c60),L_(0x4e8640e3), LL(0x63890d47,0x01830a3e),LL(0xa50855ff,0x86e7548d),LL(0xb9b3fb5b,0xbf0727b4),L_(0xa1cac932), + LL(0x93078c89,0x08fe3e54),LL(0x7f8ba9f8,0xa70bd6a5),LL(0xfff3b260,0xa621223a),L_(0xda5b8cc0), LL(0x5c0ab9e1,0xb352caaf),LL(0x52e875c5,0x57c08401),LL(0x888687cb,0xd159fefe),L_(0x788f0705), + LL(0x50a69ca4,0xe7ee73d3),LL(0x71f9e278,0x50682750),LL(0x0511bea3,0x147b2ef8),L_(0x908389b2), LL(0x0d09e4fb,0xb9505161),LL(0x7ab5b1bb,0xf8cfd83f),LL(0xa16a5843,0xeb15fb80),L_(0x8840233d), + LL(0xedb21e1a,0xfbf89123),LL(0xb350235f,0xba1f0507),LL(0x601ae7a3,0x487a5b44),L_(0x168cfefc), LL(0x1632dc61,0x27f10999),LL(0x3f7f4917,0x224754d4),LL(0x576cbe66,0x93695572),L_(0x1697b707), +}, +/* digit=24 base_pwr=2^168 */ +{ + LL(0x897a6b26,0x1aa93494),LL(0x07822bd8,0x0eecf33d),LL(0x690a0450,0x411c39e9),L_(0xddfd5a23), LL(0x752f4021,0x5d8da0af),LL(0x74dcec38,0xc6e5a1f0),LL(0xa51c46ee,0x6b77f2b0),L_(0x1367898a), + LL(0x10f0de19,0xd8b1efba),LL(0x5d75e9ca,0x831a06cb),LL(0xdea9f60f,0x7af3d6fd),L_(0x95065ae4), LL(0x379b23db,0xe0c7e800),LL(0x22ced186,0x5d673e52),LL(0xc6698d7f,0x95367fa5),L_(0x9310a67e), + LL(0x288d3aae,0x7322a3b2),LL(0x1b5b3f21,0xbe251fd5),LL(0x1fe6e87b,0xcad2ee21),L_(0xb5fd4e5b), LL(0x91f82182,0xe5ae89d4),LL(0xc2853eeb,0x3262d99e),LL(0x9036d2d0,0xcb159584),L_(0x45627827), + LL(0xf3073506,0xf3411909),LL(0x2470f5ee,0x94dcf7c1),LL(0xf0be84fe,0xade34d2c),L_(0x90636c20), LL(0xb2827737,0x180fa04c),LL(0x2db09b0f,0x26143ea2),LL(0x0a0909aa,0x83bcc559),L_(0x281cf9aa), + LL(0xb3d17628,0x38d6fee0),LL(0x8f859667,0x5ac1dd54),LL(0x86559609,0xca6e59bf),L_(0x6854b4e8), LL(0x61f3a762,0x23a24b87),LL(0x5840ce8a,0x0173425e),LL(0x946b6bbd,0x00fd3d85),L_(0x8a095b45), + LL(0x156ebcbc,0x74d544f1),LL(0x4c355669,0x3127ea7e),LL(0x1ea60e82,0x099e116c),L_(0xd960a70d), LL(0xd32fd59f,0xd60a4a59),LL(0x305faf07,0x3cf7063d),LL(0x61b80d24,0xe6b8e5c2),L_(0xd428e0cc), + LL(0x22dd5fee,0x26d21295),LL(0x5d2ed352,0xd4c31ad3),LL(0x3daf91ee,0x44afa3c4),L_(0xd529e7b2), LL(0x17333fb8,0xd7ed0b47),LL(0xcc9faccf,0xe9201771),LL(0xc98aa7de,0xc9757998),L_(0xce2c4e14), + LL(0x1ae8d0a4,0x098fc503),LL(0x3d3f8532,0x41d10295),LL(0x50253fa1,0x23d266c8),L_(0xb38286d6), LL(0xfb63eee1,0x99903092),LL(0x48a64b98,0x9100a99c),LL(0xbc43643a,0xfe159bd8),L_(0x2f08eae4), + LL(0xb10e23d2,0x5b31f9b6),LL(0xc558d554,0x5c151d19),LL(0xe3c46403,0x19a6f592),L_(0x2211131c), LL(0xbb6a51b6,0x4d8f19e7),LL(0x8c4d4f34,0x551259ee),LL(0x347e8cae,0xad36aa89),L_(0x5a959e3c), + LL(0x9f2c968b,0x178db2dd),LL(0x8674b5e1,0xc9ba55bc),LL(0xce21dcf1,0xd3884f45),L_(0x7664513a), LL(0xbb703b97,0x7a1b3e56),LL(0xc9e299e2,0xc4a588c4),LL(0xe96de0f0,0x8d3f0ffc),L_(0x5aba434e), + LL(0x8685946b,0x3206b59f),LL(0xbf15b53c,0x7e914149),LL(0x2fff52c6,0xfec55940),L_(0xc3791f65), LL(0x78dad0a9,0x394de0c1),LL(0x5c5fed8c,0x72c6919f),LL(0x674b4d59,0x0e57743b),L_(0xd7315975), + LL(0x3dad8066,0x7fa4f80a),LL(0xc574aafe,0x60f2414d),LL(0x25994c45,0x15293405),L_(0xa00f9231), LL(0x0f201926,0xf98edea5),LL(0xd0bcf974,0xaf5b2ad4),LL(0xa3c74ad6,0x448873ac),L_(0x8ff67b2d), + LL(0x75e7728e,0x510cb9a4),LL(0xbaf15c58,0x54d76825),LL(0x234a8fab,0x0023d4d4),L_(0x9d6e89b9), LL(0x2cce0b43,0x374850ff),LL(0x6bf4faef,0x32a29d1b),LL(0xdd164002,0x799cb3ee),L_(0xfc7797a7), + LL(0x26f100f7,0x34261e54),LL(0xcf17aa4d,0x589b344b),LL(0x801ccc35,0x0c7a04fa),L_(0x762b3408), LL(0x9578af64,0x002ca131),LL(0x0870f41d,0xe0fba28a),LL(0x70686201,0x276eaa26),L_(0xf592842e), + LL(0xcf7ff5db,0xb78b71bc),LL(0xa79ca9e4,0x625166ca),LL(0xe7aab4c3,0x095aa863),L_(0x980df4d9), LL(0xfc336519,0x75ac5ed8),LL(0xcc69d2f9,0xf211966f),LL(0x429f98f1,0x10ce7510),L_(0x5736d4ee), + LL(0xf3b848f9,0x8da51d84),LL(0xb2abe419,0xc9d1a3db),LL(0xefa624dc,0xc275c72e),L_(0x8b684df9), LL(0xc682bec0,0x75f1bf6e),LL(0xfc8f823d,0x3f9f9e0a),LL(0xa497166b,0xb3363382),L_(0x3241d9b6), + LL(0xe748b559,0x9a65f283),LL(0x47d0d598,0xeb923237),LL(0x600f8097,0xdfc5d530),L_(0x06c1d340), LL(0x8c26801d,0x409b596b),LL(0xe50cc47b,0x2d0a1f78),LL(0x71018c42,0x4f9b2683),L_(0x2de345b4), + LL(0x2273bfc3,0xd352b3d9),LL(0xfb7e2c37,0xb02762b6),LL(0xfb5035dc,0xc27af22c),L_(0xe5cf5a24), LL(0x304d3903,0xf2e49e03),LL(0x31c42940,0x8e2941f7),LL(0xb3e476b2,0x3130439d),L_(0x729a7e6b), + LL(0x2fb4a6c0,0x554fbba4),LL(0x1228460b,0xcbf47250),LL(0xda7db077,0xf5e0be2b),L_(0x8a989e94), LL(0xbed3a180,0xe15aded1),LL(0x56931b09,0x2a994a1d),LL(0x7fb6175b,0x36407d3f),L_(0x785ad12c), + LL(0x7466b3e3,0x296e5cc4),LL(0x6178d140,0x29e191e4),LL(0x9088d05b,0x1f9dc468),L_(0x631396d3), LL(0x2665b7af,0x86ffc6cd),LL(0x04f52e9a,0x099491df),LL(0x4b21e0a9,0x27375e3f),L_(0x242c7c60), + LL(0x974f4c4b,0x064ca079),LL(0xa02c0b2b,0xba3c8a64),LL(0xa1558503,0x13a69d2a),L_(0x3d67919d), LL(0x148d5568,0x2bfc83bf),LL(0x3a653e93,0x1054b748),LL(0x69ae9c83,0x0a4cc47c),L_(0xf624d321), + LL(0xacbde616,0x9e3e45cf),LL(0xd947c8c7,0x6374e6b2),LL(0xa439634d,0x5b100526),L_(0x7b5067f8), LL(0xb742e1b9,0xfc26652d),LL(0x701d8d64,0x6746e242),LL(0x277501f0,0x262c9dad),L_(0xd3dd957f), + LL(0x09a69072,0x9f4be9d7),LL(0xcf4ac2d6,0x20d76239),LL(0xe04b6795,0xf3257cbf),L_(0xec24a41c), LL(0x48529c37,0xb3eb3d13),LL(0xcc1689ae,0x35003a26),LL(0x8712e8c7,0xafd7b5e3),L_(0x868af920), + LL(0x620ac627,0xd27017eb),LL(0x75a3408c,0x28e8bc69),LL(0x7de280af,0xfab61ca0),L_(0xb76f92d0), LL(0x3e502b01,0x528c5328),LL(0xffede226,0x05139646),LL(0x9bf76d0b,0x4d2256e2),L_(0xdd96e876), + LL(0x4eb92472,0xe00a0552),LL(0x896685cb,0xfa213a96),LL(0xf0d4a520,0xe8f5fdb6),L_(0x0f380c61), LL(0x5288ae5e,0x4d63b6c6),LL(0xca87cdad,0x7d544fbe),LL(0x69639e2d,0x941d9935),L_(0xed61f0e4), + LL(0x31d063fa,0x72ebb946),LL(0x43d0a9df,0xa9c082ad),LL(0x53424334,0x10a77134),L_(0xdf849ce1), LL(0x5a1815d9,0xdf56c5f3),LL(0x58feabaa,0x6bbd3105),LL(0x787ed1e2,0xa802a275),L_(0xa542b49b), + LL(0x70042e82,0x0646662b),LL(0x33793ddc,0x6dfd5259),LL(0x873a716b,0x3ae6c62d),L_(0xd34ef5f0), LL(0xb3489aa8,0x9974e0c1),LL(0x5b201375,0xae617d06),LL(0x25cf8d2d,0xfac50e53),L_(0xf5028682), + LL(0x82900f26,0x20822d19),LL(0xdb5b7f7b,0xc050b43f),LL(0x610bf8a8,0xe4e5ddab),L_(0x59f6f46f), LL(0x0bd3d213,0xd3a9f7a0),LL(0x29ffdb56,0x8a060b79),LL(0x1aed0fa9,0x7cd570f8),L_(0xa1cdd0bb), + LL(0x0f6923b1,0xb9d66a95),LL(0xf9463b5d,0x6b4aafc6),LL(0x9dcb5191,0x7e8de98d),L_(0x3a174474), LL(0xafeb6788,0xd8a6df5b),LL(0xb00ae812,0x1127ce66),LL(0x595dbbd9,0xab858cbb),L_(0x20456b6d), + LL(0xf945d541,0x6e092bb4),LL(0x4b66cf94,0x27b14454),LL(0x7eca44eb,0x6daca5c9),L_(0x44fbdf91), LL(0xfc99b4f1,0xcb39349a),LL(0x9f292892,0x611f2265),LL(0x7d2e1348,0xb91c9d21),L_(0x2511f796), + LL(0x8d960ed0,0xc742fe68),LL(0xaeca436c,0x7a253140),LL(0xa736c99b,0x7053b3ba),L_(0x4df3a08c), LL(0x1c4bf5fd,0x35ff874b),LL(0xa50437d0,0x85d7a155),LL(0xbb3ee79c,0x28f141d8),L_(0xa309d53a), + LL(0x18a6fdae,0x052107c1),LL(0x4c1843ed,0x7a4941a3),LL(0x5b395717,0x10eeec3a),L_(0x41bed6c7), LL(0xc77d493c,0xe3bf062d),LL(0xe2632a75,0x99abf215),LL(0xccb37627,0xf5482813),L_(0x65bddca7), + LL(0x86262170,0x97f943dd),LL(0xe4f707eb,0x28ad9142),LL(0x1fca8434,0xe200b711),L_(0x5979d82e), LL(0x7eeaf938,0xa0864853),LL(0x073f049c,0xdcbbbd26),LL(0xefca8545,0x5e115ac7),L_(0x81b25231), + LL(0xfa392d8c,0x0b83214e),LL(0x6e208498,0x5535cb0e),LL(0xd7f17a61,0x9d606b9f),L_(0xf9ff6be9), LL(0x532d4992,0x78a1ebcc),LL(0x8412a690,0xc80751aa),LL(0x3fc39e84,0xa4039f18),L_(0x16cc85fa), + LL(0xb6b3e3bf,0xb29dd0f5),LL(0x3f372e07,0xfc4475a6),LL(0x1010f76a,0x499ca40b),L_(0xc9d3a123), LL(0x505bb647,0x1a039620),LL(0x898b067c,0x71c1fd31),LL(0x8fba57f2,0x920244b9),L_(0x2f20dfbe), + LL(0x09e67c8d,0xadba6513),LL(0xf1db6fa1,0x004a0b2d),LL(0x780eff8e,0x6530b59a),L_(0xdcf21d60), LL(0x8ecdce5b,0x767463fd),LL(0x65b13c4b,0x70cc1a2b),LL(0xaf0f593a,0xb10b6a9a),L_(0x05c89578), + LL(0x040645b4,0xae7f7819),LL(0x678205ed,0x421d2a1d),LL(0xcc32c0b0,0x2492206a),L_(0x43d4ee55), LL(0xb779e43f,0x2296a21f),LL(0xd485f134,0x5fce8d33),LL(0x7dae00fa,0xd7fd8f23),L_(0x9c357857), + LL(0xc2d0a0f7,0x683b4c52),LL(0xd1014606,0x457af899),LL(0x2de4b78e,0xed10d363),L_(0xe5547d1b), LL(0x84fc2002,0x40cf6189),LL(0x1fa00e4b,0x6b948df1),LL(0x2643b489,0xa90cf1b7),L_(0x862ff071), + LL(0xf9cd4743,0x2c263fe3),LL(0x3df3e881,0x7af59647),LL(0x4a452825,0xf8b4e248),L_(0xff9860e0), LL(0xe9e940c2,0xfaa7bd3a),LL(0x4dcae86a,0x829a4b08),LL(0x0392152c,0x8e97fa6c),L_(0xfc62b042), + LL(0xf8c2cf32,0x38ca48d4),LL(0xc8c6f843,0xb6e872e4),LL(0x1a5701ad,0x8137a142),L_(0x3332d11c), LL(0xd8632d31,0xf65f8d02),LL(0xc57e0eee,0xed896e7d),LL(0x84c4a8f0,0x474d64c3),L_(0x9a310ea4), + LL(0x1facebce,0x90943f9c),LL(0xabcd06f1,0xc0153740),LL(0x4bf7868b,0x2460277b),L_(0xf15a1c15), LL(0xe70f731b,0xbbc3520f),LL(0xa0084581,0x1881feb6),LL(0x8c12c484,0x4461e279),L_(0x2cb0b794), + LL(0x420f669d,0xc6234887),LL(0x1336960e,0x961fd208),LL(0x6690aec7,0x8dcc7988),L_(0xea8d5acf), LL(0x6a7ab2c0,0xad19e666),LL(0x221a923e,0xf44304de),LL(0x3d295ba2,0x840fc746),L_(0xa849c9c6), + LL(0xffb23eb2,0x796ddb67),LL(0x7f2ece47,0xb74ebddd),LL(0xfa40fdb7,0x6e9b86f6),L_(0x3753a599), LL(0x517edc52,0xb0d32903),LL(0x9ce52e22,0x4e38e3ff),LL(0x609b3583,0xbd4bbc16),L_(0x470387a5), + LL(0xc2767e71,0xb5992b12),LL(0x15e316ec,0x722e7585),LL(0x331c37c2,0x4f66d67c),L_(0x5a547f8e), LL(0xad349dda,0x2343f37d),LL(0xc7ecc98e,0x991a0d20),LL(0x2f36fbc0,0x32fd452b),L_(0xbcd300ae), + LL(0xe9e7130e,0x3965b7db),LL(0xfbb383d5,0xa434d230),LL(0xd5ddeb7e,0x5de1379b),L_(0x89236512), LL(0x26844221,0x71474c32),LL(0x359ac63d,0x86ab382d),LL(0x6a2a41ee,0xb41d45a2),L_(0x4543bf02), + LL(0xc03175e0,0xdd45b154),LL(0x51a93461,0xc32e33b5),LL(0x8e87a379,0x6030bb55),L_(0xe0f74319), LL(0x1ee1fb1c,0x180c566b),LL(0xf64d3bd0,0xd5501439),LL(0x117f7e05,0x9fe6d9b3),L_(0x276cdeae), + LL(0x79701f00,0x90d2c4af),LL(0x8ee8e755,0x19fa045c),LL(0x34e9da6a,0xea446558),L_(0x56f8560b), LL(0x22273b04,0xf104982d),LL(0x7a1b70ed,0x0b6be603),LL(0x5df6c184,0xb3a79a2f),L_(0x3e690209), + LL(0xb6a58003,0x506cdad2),LL(0x1cd413bf,0xcc9e3e86),LL(0x5af9735a,0x4669ee32),L_(0xf133df4e), LL(0xa97f8799,0x343b9133),LL(0xd2df1205,0x5ed1097e),LL(0xc0f593eb,0xe1f7e724),L_(0x03f70d2b), + LL(0xfbde70cb,0xe1aefce0),LL(0x28e77727,0x953078f3),LL(0xea43f71f,0xb1afcd33),L_(0xe72f4294), LL(0x42df1489,0x5c609190),LL(0xd85d4064,0x9d39f16a),LL(0xd4734dcb,0xc6a7ed93),L_(0x73444dba), + LL(0xc80deb54,0x786041c5),LL(0x2e784b12,0x73806b24),LL(0xa5394485,0xd5ed489c),L_(0x7c1354f6), LL(0xc98caa8f,0x6f5aed20),LL(0x04a2cef9,0x343f789d),LL(0x6a61903c,0x0943ad59),L_(0x74525b1c), + LL(0xdd468a2b,0x41295f9e),LL(0x79058179,0x767bb930),LL(0x8356b028,0x8630275b),L_(0x7d99704d), LL(0x9677c30c,0xfe64fec1),LL(0xb6ad9348,0x7bfdd07e),LL(0x323c5204,0xff571db9),L_(0xabd2794c), + LL(0xc6bef1c2,0x39e5ddd7),LL(0xa00ac9eb,0x4166efe7),LL(0x08022d52,0xec00b4ca),L_(0x89c305e2), LL(0xdff9eb38,0x2f6d8e32),LL(0x1c9dee31,0x38a38882),LL(0x063cbbc8,0xd453f2df),L_(0x500b8e1b), + LL(0x2a4b58b5,0x0d2d67f5),LL(0x704fba9c,0xb9671248),LL(0x3b601783,0xc86eeaf3),L_(0x3a076280), LL(0xc6604758,0x057735ba),LL(0x3c7fe621,0xa0f157e0),LL(0xba90bf98,0x7ea681b4),L_(0xc697f17d), + LL(0xf242b313,0x2fb2d012),LL(0x8b404954,0x301b1ef8),LL(0x3864cc2c,0xe8bf6c74),L_(0x27928a4a), LL(0x37e42bc1,0x129459b3),LL(0xb079e9c8,0x95c688c9),LL(0xa8c506fd,0x2be35d42),L_(0xabd7b8a2), + LL(0x8d7f03a4,0x0c337ebf),LL(0x9f5cd253,0x2b5789a5),LL(0x3bc3df04,0x808378fd),L_(0x57ce1b57), LL(0x8394999d,0x9173aca8),LL(0x29069aea,0x68a97a68),LL(0x172c5e76,0xd7f9c6b8),L_(0x2bb82de1), + LL(0x26324b43,0x069b62db),LL(0x3a04b8c8,0x2d6c75c6),LL(0xbb1fb7ef,0x8e5dc130),L_(0xdf0fbfa5), LL(0x427bae31,0x8d6bb598),LL(0xae962471,0x4a1d29d8),LL(0x6a0b5064,0x1f5aad58),L_(0xcb90fbe1), + LL(0xf5f95663,0x99c6500a),LL(0x182ee0a7,0xfc57412c),LL(0x9acab63a,0xd7a3f799),L_(0x579dffc7), LL(0xb294554e,0x2e7244f9),LL(0x7a563978,0xc577a189),LL(0xd9421c12,0x4a33607f),L_(0xca35ca5e), + LL(0xcf4cc636,0x476c2c16),LL(0x84dd3019,0x40d43930),LL(0x9f6a9cbb,0x8de41867),L_(0xa0782cd2), LL(0x8e9807cc,0x335870d6),LL(0x873bb057,0x237820cd),LL(0xd2763265,0x817bb568),L_(0xac61e5ba), + LL(0x277845cc,0xb4216df3),LL(0x28a00ba3,0x4ae7f387),LL(0x8d18d55a,0x83dc382b),L_(0x9929ba4b), LL(0xace5a497,0xc4acbdd9),LL(0x1a149f46,0xfee31905),LL(0x03677afd,0x7f382dd5),L_(0x038b4fd6), + LL(0x39c19776,0x7b1469c5),LL(0x770906b4,0xefd3e00c),LL(0x39b76a30,0xb757175c),L_(0x77b00db2), LL(0xea73fb31,0x00e5eb12),LL(0x564b2b5a,0x5dec4d76),LL(0x16670a8b,0x6cedfc0a),L_(0xfc26f8d3), + LL(0x0f967635,0x5f72f035),LL(0x492f15fb,0x9fbbd203),LL(0x8a57f34e,0xca278cc2),L_(0x136d3def), LL(0x3233be88,0x56bc1e3d),LL(0xb7d079d2,0x167694bb),LL(0xb849beca,0xc12eafab),L_(0xf42c9064), + LL(0xc22b8b50,0x35110b41),LL(0x3593434b,0xc03f7534),LL(0x2a685ee5,0x38e7d616),L_(0xd80d02e7), LL(0x681060ac,0x47c0e163),LL(0xa5dbbbef,0xaf6a4e94),LL(0x58c7a38b,0x00b0f102),L_(0xbe4a965e), + LL(0xa0bfdebf,0x00ea08c9),LL(0x0a49eb36,0x19299c59),LL(0x4a5b799c,0x1389e249),L_(0x6bb607bd), LL(0x003f62bb,0x21dd6f70),LL(0x648ac9f7,0x0d28002f),LL(0x46ddb53c,0x123f4a38),L_(0x0b454608), + LL(0x6cd705df,0xdc0b4f05),LL(0xa26650ba,0x584543cc),LL(0x9623d15d,0x52a5b5a4),L_(0x8530847b), LL(0x95176358,0x2b9634a4),LL(0x88564796,0xb76d8613),LL(0x8cd268e3,0xdd09c6f7),L_(0xfbbf3c6c), +}, +/* digit=25 base_pwr=2^175 */ +{ + LL(0x40600691,0x076ba1e2),LL(0xaf9141d1,0x3cbbece3),LL(0x90d62387,0xeae39108),L_(0x39ae931a), LL(0xef1b81b3,0x7c192920),LL(0x51f12b8a,0x07e50a2b),LL(0xf1cfbc63,0xaa15f8c6),L_(0x26ff0bea), + LL(0x9f2ac6a5,0x5ac7dc31),LL(0x5a5fa2f2,0xd3536baf),LL(0x918f6160,0x19e26cfd),L_(0x3d611b68), LL(0x58b92257,0xfbe0d264),LL(0xa4b3a16e,0x6f69ee33),LL(0xab7e3e38,0xfb87b2d4),L_(0xe4cf7a53), + LL(0x4428fe8d,0x28c21968),LL(0xe6afbadf,0x82133ac1),LL(0x08d0d89a,0x79f622a4),L_(0xfc61f01d), LL(0x0f6397fa,0x9afa4014),LL(0x1819a677,0x8b6b825f),LL(0x9e89a4ea,0xf6189381),L_(0x4c306973), + LL(0x16b5009f,0xd0fd8e2d),LL(0x85f596e9,0x75cd4af4),LL(0xc650d10a,0xc0ff364a),L_(0xaa432f6b), LL(0xb9e1fd35,0x60cd9ff6),LL(0xa81387b4,0xf5ffdf83),LL(0x1fb57eae,0x0f74dad8),L_(0x77ebce28), + LL(0xcd3d74f0,0xe7801dbf),LL(0xc705f5f3,0x4cae30c4),LL(0x892e14fd,0xee1d51c0),L_(0x235f35c6), LL(0xcb840ec4,0xc690d3bf),LL(0x496db46c,0x6a393aed),LL(0xf95db976,0xa0c8c115),L_(0x3a2ff859), + LL(0xad9729df,0x21208e24),LL(0x50a51554,0x8a669134),LL(0x76da2c79,0x739f33c4),L_(0x47f7655f), LL(0x3faf5d73,0x6df3da71),LL(0xda653a05,0x23d729bd),LL(0xe0dbba03,0x7efefa7f),L_(0x8546d123), + LL(0xa29be43b,0x63a12c13),LL(0x81bccb17,0x0f3de027),LL(0xb8ad1f49,0x2ee0f6a0),L_(0xeca64ff4), LL(0x2a0247bd,0x6c14c6a4),LL(0x27d0478a,0x15b008e7),LL(0x9dc8b9de,0xe6cf3269),L_(0xde4a727b), + LL(0x8a41f62f,0xe9139920),LL(0xafcf2338,0xe343f86a),LL(0x8626eb92,0x330ff008),L_(0x62bc5b8f), LL(0x770a0f51,0x9ed3ab3d),LL(0xacdfce8c,0x87533c3f),LL(0x198a1e9a,0x36ceed7d),L_(0x366edc66), + LL(0xad4f0edd,0x9eb6ef96),LL(0xa0790c0e,0x8f2b7875),LL(0x8212d5cc,0xa20f0405),L_(0x2e6749a5), LL(0x43c2e598,0x0b5bf3aa),LL(0xf0598778,0x695b110e),LL(0x6b2a153e,0xacd67e10),L_(0x9a4287cf), + LL(0xc9194ba8,0x1df015df),LL(0x5545ae11,0xe5e57b96),LL(0xf86d852d,0x34894ebb),L_(0x61b42915), LL(0xaab92979,0xb25bf6b0),LL(0x2f858332,0xf88d9382),LL(0x082b80c9,0xa64fc13d),L_(0xdb34263a), + LL(0x4fc8d228,0x6c1819aa),LL(0x46f4e2d8,0x790019d3),LL(0x3e6d5835,0x31b5f8c4),L_(0x0471eabf), LL(0xd70615ba,0x7377bdb1),LL(0x4f0b793d,0x3d8b3ee2),LL(0x1e0b124d,0x8be72522),L_(0xcb228953), + LL(0xd05e1a54,0x3d345d79),LL(0x4e4eb11a,0x9b0c74e2),LL(0x4f7cb301,0xd3f60831),L_(0x9b37cac3), LL(0xdd153410,0xd42ac14a),LL(0x4b82b6e1,0x67daba89),LL(0x062085e8,0x293aa2ea),L_(0xdafbaa02), + LL(0x57e823a2,0x66dc4352),LL(0xeee6c065,0x9d1334ad),LL(0xe974bd6b,0x51129e57),L_(0xc6e911b2), LL(0x4a591897,0x143fbbc0),LL(0xe567aa72,0xfcf32ecc),LL(0xd98a2bac,0x985f3567),L_(0x4940de3e), + LL(0x810e3b3c,0xabba8dd8),LL(0xbc9b4d54,0xd024fdfa),LL(0x09ce7445,0x56df5156),L_(0x869985ec), LL(0xa279f60b,0xd64936f1),LL(0xf9f4dd90,0x147e3ae6),LL(0xb0502e71,0x9b4a0a61),L_(0x2b7784e2), + LL(0x10ea913e,0x228b551a),LL(0x132cf6ee,0xd39b6d23),LL(0xb41622ce,0x1166f594),L_(0x4f82b870), LL(0x771a8cbc,0xc4a0434c),LL(0x1f3f48d6,0x4e8415e6),LL(0x185bfa20,0x0549b016),L_(0xab597880), + LL(0x8f78c75c,0x7783a236),LL(0xcacbfea9,0x185d7d80),LL(0x37384da3,0xd767d6c6),L_(0x1b40ecf0), LL(0xb58df6c4,0xcb35e135),LL(0x1bdb10dc,0xdede089c),LL(0x51f82512,0x032e4873),L_(0x86fff3dd), + LL(0xae08ea09,0x993762bf),LL(0x3c5e8711,0xe9e1ac75),LL(0xc3fb58ec,0x5942a654),L_(0xdc39e05c), LL(0x6f00a879,0x4c7c1a57),LL(0x76f183b7,0xb5b4e070),LL(0x02f3f5a3,0xc7635031),L_(0x95a5b1bf), + LL(0x1973898f,0x44aeebfb),LL(0x6fdec0d8,0x22963505),LL(0x4dccb72d,0xc61c3dc6),L_(0x29d98bbe), LL(0xc79d4f82,0x0b4d124c),LL(0x60e1bdf3,0xab61b974),LL(0x60907706,0x502b0c26),L_(0x52fb8b7c), + LL(0x9cc7f5da,0xa0c27cd2),LL(0xa30b225d,0x39c837e6),LL(0x14e5e017,0x90beb6db),L_(0xca71c799), LL(0x3452a918,0x6406e7a2),LL(0x8b449af9,0x44086ede),LL(0xd95a3674,0xe83116d2),L_(0xc945cb30), + LL(0x757467e5,0x195a49f1),LL(0x19bb9a9f,0xbd375548),LL(0xdd63e900,0x32a666f4),L_(0xbb5bae80), LL(0xb06e326c,0x71975390),LL(0x7200d75c,0xa8071dcb),LL(0x9bb8f1f0,0x34371f37),L_(0xd14c5988), + LL(0x9fc79376,0x330b4d2c),LL(0x4766d5c5,0x89487193),LL(0x96d443b1,0xb683d913),L_(0x55cd4038), LL(0x25e49098,0x0b19d841),LL(0x4de7c5cc,0xc211c53d),LL(0xb34627f6,0x63d637ee),L_(0x05d48e84), + LL(0x2fb2a30d,0xa9ab961d),LL(0x587b4227,0x7f2e1082),LL(0x9bacdb2f,0xf2fed11b),L_(0x4327e6c8), LL(0xf5f7ca94,0x59f2340a),LL(0xdae690d8,0xa241b47f),LL(0x20f9a82b,0x04266881),L_(0xb60ba27e), + LL(0xfc8a6083,0x4b7ea198),LL(0xf7bd165d,0x0439da93),LL(0xaff22d2a,0x6e2c180a),L_(0x5fa28be1), LL(0xb5422e8a,0x8d8de819),LL(0x8623fa12,0x1c7aa689),LL(0xf1b013b4,0xecdeea2c),L_(0x1412065d), + LL(0x1b8b14e9,0x949b80c2),LL(0x9e45f562,0x7634d5e7),LL(0xeacf4d90,0x89e47023),L_(0xed86f825), LL(0x2a28ee33,0xa46549d3),LL(0xcc9d20c3,0x91467c95),LL(0xb5aaafc8,0xaad820ff),L_(0xfa9b0b7b), + LL(0x3a989cf8,0x56a19c64),LL(0x694699c4,0x6a9264c2),LL(0x2172bedd,0xfcd8d182),L_(0x71a622e6), LL(0xabaee9e8,0x096356a3),LL(0x5df0e982,0xb3c8273f),LL(0xbff93f13,0x15499fe5),L_(0x035d8bec), + LL(0xe7f02675,0x8665d519),LL(0x0b392d7e,0x3114f640),LL(0xda3e7314,0x27724fe5),L_(0xb0000497), LL(0x9626e05c,0x92676514),LL(0x86de4867,0x70fdd068),LL(0xbfb7e396,0xc9127734),L_(0xec308376), + LL(0x889048c2,0x13351e00),LL(0x9ee3edde,0x8a42c8c0),LL(0xe9b0df44,0x039a9289),L_(0x3ba48603), LL(0x9ba675fa,0x1bc5a4ed),LL(0x28dbe0e2,0xc1c91604),LL(0x33d13a46,0x683a466a),L_(0x6f0e40d3), + LL(0x6f169f5d,0x49b1e542),LL(0x5467b60c,0x6be4fa18),LL(0x63a775bc,0xf10e3ad8),L_(0x555978a5), LL(0x2b932386,0xe2647c33),LL(0x2d98173b,0xc10e0d58),LL(0xa46de613,0xd4d665e8),L_(0x8c2f942e), + LL(0x76f88409,0x83f96957),LL(0x11159549,0xb83760f5),LL(0xa26fb69f,0xada04407),L_(0xe1ffa9b1), LL(0x7985249d,0xf63dc16c),LL(0xb8212592,0xa9f05a9e),LL(0x8f2b706f,0xa84d6d60),L_(0x0952d255), + LL(0x9ef2ca53,0xc1ad35cf),LL(0x8063b121,0xf54f3d1d),LL(0x2c69c142,0x2eddbe16),L_(0xe20cbf04), LL(0x58b9d3e5,0xc6a6486c),LL(0x5f1b4348,0x30a9fb2a),LL(0xaa7af663,0xbd211312),L_(0xf09747fa), + LL(0x6aeadb7e,0xb3384570),LL(0xad4b300b,0x80077723),LL(0x7e0408aa,0x92c8f393),L_(0xcdb75950), LL(0x91a66b10,0x01c71c42),LL(0x98806233,0xd3e2ebf9),LL(0x4ccfab0f,0x2c91c51d),L_(0x0daddeb0), + LL(0xa5136305,0x1656bed1),LL(0xf9475d4e,0xd5bd811f),LL(0x29f67636,0x71b2886d),L_(0xce5ffc47), LL(0xf8ec4c6c,0xed19aefd),LL(0x00fbe22e,0x3a69ea47),LL(0x1a593c52,0xed7635e4),L_(0xfea5dbc5), + LL(0xb6a32091,0x0a800d89),LL(0x3e952cbe,0xe00a730d),LL(0xb849574d,0x95e45fa4),L_(0xd0b6f06f), LL(0xc0b38793,0xca47bacc),LL(0x3aea6517,0xa1c9b9db),LL(0x98d62545,0xf79f468d),L_(0xe75cf2cd), + LL(0x6d96b85c,0x92cfaed8),LL(0x8aa22d94,0xe9d09bd1),LL(0x951cb3e9,0xcdf3e590),L_(0xc6443f1d), LL(0x5ee4705a,0xa83030bd),LL(0x4ab4bb2e,0xbbdfbd6c),LL(0x126bf8d3,0xab5c4613),L_(0x78abc625), + LL(0x1d099c24,0xb14045e9),LL(0x75f0192a,0xc8658ab0),LL(0x66a1d69f,0x488c7367),L_(0x27671771), LL(0x10f50e10,0x41dc68c8),LL(0xdbf2b527,0x56c509b4),LL(0xaa6e15c2,0xc052c755),L_(0x6264eb14), + LL(0xf102c97a,0x361df88d),LL(0x8133758d,0xfcb2818c),LL(0x89d87e0a,0x8313147e),L_(0x156564f5), LL(0x92996691,0x0305e3bb),LL(0xb530ebe1,0x3ea90757),LL(0x75ed509b,0xe3da205e),L_(0x8810aa6d), + LL(0x0936433b,0x66bf33ad),LL(0x5cdc1d84,0x20489613),LL(0x40fff511,0x02569cc0),L_(0xa46f7fa8), LL(0x6bea5f0b,0x03ae7465),LL(0x6ca8cba2,0xa43eaabe),LL(0x9d34e4a3,0x338f15f5),L_(0xf175cde9), + LL(0x37a46a76,0x43fb29da),LL(0x0f113fed,0x1e4e18be),LL(0xc9bc3624,0x0891f557),L_(0xf2de428e), LL(0x6cb133b6,0x72c78284),LL(0x0bca6469,0x920b91e7),LL(0x4ad07f92,0x9aca547e),L_(0x254d2e0c), + LL(0x48b9afe1,0xbedf71c7),LL(0xfdf235a0,0x7b745717),LL(0x707d37d5,0x519516e4),L_(0xb70d0d1d), LL(0xf179e489,0x026e2e98),LL(0x7b9cd555,0xa64e85e9),LL(0x05a8ce5e,0x490bbe2c),L_(0xfd0ac7d5), + LL(0x5ef0e444,0x4518decb),LL(0x5704b61f,0x84b0fcd7),LL(0x05e3a6d4,0x761243d0),L_(0x09fdcc95), LL(0xf157aba1,0x52cc970f),LL(0x2f40eec4,0x545ea4c0),LL(0x3cfbc5b3,0x82e0b382),L_(0xd24582c3), + LL(0x3def68b9,0x91b73501),LL(0x63a13aee,0xf4c2f983),LL(0x64a186c4,0xd30674ab),L_(0xae849f85), LL(0xd639edb0,0xbd3a7bd8),LL(0x4e22b52c,0xcf732f2a),LL(0xdee6130a,0xa16934a2),L_(0xa208c2f8), + LL(0xee275071,0xacddc898),LL(0xdcfd6055,0xc137eacf),LL(0x4226123c,0x22c7c6d7),L_(0x18caa3b3), LL(0x575ea43d,0x27a1cc63),LL(0xcfd9c842,0x1d810b5b),LL(0x435cc5b0,0x98cdd25d),L_(0x03338f7f), + LL(0x1bca7595,0x39e40f6e),LL(0xa3f28110,0xf4b9f93a),LL(0xbb9b4a37,0x6d2f9be1),L_(0xc7ef39dd), LL(0x288d64fa,0xb0848ac0),LL(0x9e6a4583,0xe3934b4d),LL(0x65ecc897,0x7961633c),L_(0x12939ce7), + LL(0x69332018,0xdf1e1997),LL(0x0eb1a850,0x55fdefde),LL(0xcfe64a91,0x6d23295e),L_(0xf848ec96), LL(0x6ccd6db0,0x8a058163),LL(0x4a1159c7,0x8e9a68bc),LL(0x145557f9,0x44e87592),L_(0x744a6684), + LL(0x570af433,0xd2524bb7),LL(0x71ee45c1,0xe5696936),LL(0xae2e62f4,0x90655f39),L_(0x4aa6dd85), LL(0xce93159e,0x170e2563),LL(0xcd77d544,0xb60f5503),LL(0x1637c5b9,0xbbddebd8),L_(0x116cc13c), + LL(0xa79f3d54,0xda76037a),LL(0xe567b31a,0x1b943930),LL(0x81454a32,0x8ce526e9),L_(0xa68887d1), LL(0x7c692a2a,0x5ccae9e7),LL(0x2aa4281c,0xbad7967a),LL(0x27a06af9,0x67a8cecd),L_(0xb0666bd6), + LL(0x18d87b3d,0x3bf215b9),LL(0xf3b762b4,0x13a2d20a),LL(0xc7ded91e,0xf2dddefc),L_(0x491f86d1), LL(0x9175dde7,0x1a33ec16),LL(0xbfa502ae,0xde7bc4a6),LL(0x8b90c1c4,0x011d4fdb),L_(0xae49e966), + LL(0x339d5841,0x30030ab7),LL(0x9b2323cb,0x93a59a0c),LL(0x3dfca103,0x6d677f76),L_(0x615cef3a), LL(0x5b19bbca,0xd8927084),LL(0x143864e2,0x0d3dc8c8),LL(0xe2a53ef5,0x7bfce607),L_(0x98d95aec), + LL(0xbfe42fd9,0xe4480a82),LL(0x674699df,0xa3a3e505),LL(0xf6bd5725,0xa3bae3b5),L_(0x93074bef), LL(0x2f753cf5,0xc278e6d8),LL(0x9075dc94,0x042fbabf),LL(0x66f70e54,0x2912d169),L_(0x77ae21bf), + LL(0x8e1def19,0xba0002d6),LL(0x9d21da23,0xa31c1302),LL(0xc9cb3757,0xc0b1cdab),L_(0x478c82a0), LL(0x8ebd0b98,0x708a627c),LL(0x95df97f0,0x00716397),LL(0x036873bf,0x9b0c552c),L_(0x31bdcc00), + LL(0xa7f4f676,0xdb1b2796),LL(0x16f6892d,0x287bb1a9),LL(0x513e742d,0x83e7ad99),L_(0x5ba54190), LL(0xd83035ed,0x3ab45168),LL(0xd55025e1,0xc5e72511),LL(0x57264580,0xa5b527ed),L_(0x246fe503), + LL(0x2956f5b6,0x9456aa98),LL(0x72287d14,0x2d9d56ca),LL(0xbd98da03,0x783e5a79),L_(0x1c038840), LL(0x278213f9,0x44dd9214),LL(0x8abbd0fb,0x1ae00f0f),LL(0x546492b8,0x9365b6f9),L_(0x638f9886), + LL(0x2772a4d8,0xe568c53d),LL(0x56dd3ec8,0xc5798254),LL(0xad2418b2,0x40cc9856),L_(0x5fe0c00d), LL(0xcc8ceaa1,0x1cdd7255),LL(0x17304349,0xd65e8630),LL(0xe17ec8e0,0xb4dc2a26),L_(0x62eb4c57), + LL(0xb31137c7,0xdb4ee5e0),LL(0xa85ff057,0x79aa7312),LL(0x0fb19074,0xf1a5850d),L_(0xb1c8c20f), LL(0xa65022af,0x407175d5),LL(0xc25f5ad9,0x1debba7b),LL(0x0ed43d07,0xa2739236),L_(0x55db40b6), + LL(0xeb7ff1d6,0xbb91225a),LL(0xb6ec7895,0x7915895d),LL(0xec277689,0x53e7c8ad),L_(0x503d01fd), LL(0x427cc57e,0xff30aa1b),LL(0x789118fb,0x28a3c0f3),LL(0x45dce70e,0x7758b97b),L_(0x2ad05594), + LL(0x79df6beb,0x64d25f38),LL(0x7c1271cb,0x1d5f4772),LL(0xafba6d13,0x24a4fbdb),L_(0x68588b5a), LL(0x9bd0327d,0x4408d2f3),LL(0xb9962d65,0x0c0efa60),LL(0x1adff959,0x3e504930),L_(0x1a7248ef), + LL(0xc91f5609,0x50371a5d),LL(0x10a09de0,0x8f3d868e),LL(0x544a1f87,0x05603976),L_(0x9bb49da7), LL(0x4f023f6b,0x2686c470),LL(0x22de25c7,0xad3c86da),LL(0x7b318aae,0x0e37c5eb),L_(0x6e0af008), + LL(0x9cc2caa4,0xc4103bd9),LL(0x28c77b28,0x2b106e89),LL(0xc44c9d88,0x02942cad),L_(0x045b3511), LL(0xae7c29af,0x985fde90),LL(0x3e691e0d,0x5ea3d69a),LL(0x66200341,0xa7fa897a),L_(0xc1b7c5c2), + LL(0xf2dba47e,0x858b20b7),LL(0x94f1ad67,0x80c7e010),LL(0xcb454e18,0xe75e697e),L_(0xea8630aa), LL(0xa6af9eed,0x7a69400e),LL(0x7d8e1f8e,0x73d9d6c2),LL(0x87354e0a,0xd949265a),L_(0xddd78248), + LL(0xd65411f3,0xbb95794c),LL(0x56456f06,0xdb7b7288),LL(0xb5869a96,0x7de9cd89),L_(0x4dd26048), LL(0x235bdd25,0x6b7c47bf),LL(0x3a2f738b,0xb8e2adff),LL(0x43a158b6,0x6452e6ec),L_(0xb59a08ed), + LL(0x3b9a55fb,0xfd294202),LL(0xd446ba91,0x77e86f2f),LL(0x0bfb0e2c,0xb347e482),L_(0xda5432f0), LL(0x811a63d8,0xda7568f2),LL(0xfd584f1c,0x32ddb6b6),LL(0x3e5b388e,0xfbef98fa),L_(0xa2090daa), + LL(0x29e9bf1c,0xebc6de84),LL(0x8601a460,0x77ae022e),LL(0x0fe7b2f5,0x51718d1f),L_(0x768e05f3), LL(0x8f61fd67,0x107bee81),LL(0xdafff971,0x45e70a15),LL(0x775aa267,0xf5af9168),L_(0xa74ac1aa), + LL(0xcc4ccaf5,0x7770d24c),LL(0x6c27e8ad,0x931b3ac4),LL(0x0c21256b,0x5eb3480c),L_(0x24df253e), LL(0xd60ece5b,0x32c4c950),LL(0x304257df,0xfa6d0590),LL(0x3821aab8,0xcfd48043),L_(0xe9ed7945), + LL(0xa563da21,0xf927be40),LL(0xa625fc39,0x65697957),LL(0xc708ccff,0xc7c05c00),L_(0xe3551df0), LL(0x1db01e2b,0x4a824ae6),LL(0x00aff255,0x21fd2abf),LL(0xdc1865ba,0x489f78db),L_(0xc47678ec), +}, +/* digit=26 base_pwr=2^182 */ +{ + LL(0xa7314858,0x4d52ad98),LL(0x44075a5f,0xa2ad3d54),LL(0x5f264197,0xa5da6ace),L_(0x1bd94b24), LL(0x1046d21c,0xaddbfb84),LL(0x9deccfa5,0xdb3e6a82),LL(0x38b0cd4d,0x0ae5bd28),L_(0xcdefc743), + LL(0x11028ec7,0x9f68d2fc),LL(0xdb80636b,0xda67d396),LL(0x399cbe22,0x058ae3aa),L_(0x9a20e964), LL(0xa4afe4c5,0x0655d446),LL(0x82be9278,0xd1f696f1),LL(0x02a07951,0x78c7207b),L_(0x5473af3e), + LL(0xa430f529,0x48532007),LL(0xe1daa6d9,0xf9c913ed),LL(0x8ffcdbb0,0xbd0a98e5),L_(0x9b2bdde3), LL(0x3eec4099,0x58bd4134),LL(0x4dcdfb2b,0x524e44ad),LL(0x29153443,0x7457124c),L_(0x9190c343), + LL(0x0c51f106,0xe34c46c7),LL(0x1a8f4562,0xa808bcef),LL(0x6cddda1e,0xb1bb8fc9),L_(0xa502ea7e), LL(0x9e63e9ac,0x9421a0eb),LL(0x124b21a9,0x09abe1ac),LL(0x634ea315,0xfc9b2b69),L_(0x4d2d4d06), + LL(0xd47adb81,0xa4e345b6),LL(0x370847bf,0xc8a3f346),LL(0x242713b8,0x10782ff2),L_(0xbc286c44), LL(0x0501d40b,0xbe6fa011),LL(0x72197eb9,0xe784765d),LL(0x23d8a846,0x9c366368),L_(0xe57509e0), + LL(0xb79e2ec1,0x9344ebd4),LL(0x41626ac2,0xee0983f2),LL(0xb61e001b,0x1c404975),L_(0x05deb3c3), LL(0xafd8bc28,0x10e30e79),LL(0x4f5429b7,0x90c587f7),LL(0x0cc2b3bc,0xe64edacb),L_(0x2a32f215), + LL(0x683db291,0xa18ec5e3),LL(0x597408da,0x0cdbd7b0),LL(0xb4dc90d1,0x9e4003b1),L_(0xf0377012), LL(0x99f01d42,0x2f737bf9),LL(0x138b7393,0x2c744a14),LL(0xbf91bb7d,0x4390dd24),L_(0xd3eb9506), + LL(0x7da97135,0x1f5f9c08),LL(0xcbb88522,0x60a61939),LL(0x590e948d,0xd63bd64a),L_(0x45a94350), LL(0xc3f04dfe,0x8b9c0e7f),LL(0xc46a6c09,0x24ed2a64),LL(0xe5cb409f,0x6aa2f787),L_(0x90fd8928), + LL(0xa6f20c60,0xc519de26),LL(0xc3ac3715,0xb2ee7698),LL(0x915329a3,0x0ca44fa0),L_(0xe61346f3), LL(0x22cc3439,0xe4d76c97),LL(0x37f7afee,0x360a76e2),LL(0x6bdfe1ff,0x536926c2),L_(0xc048c2a4), + LL(0x33feb155,0x57103466),LL(0x9e779a72,0xf0ac4da3),LL(0xf0cf02a3,0xc4a65bf2),L_(0xc464a2f5), LL(0x9f49b51f,0x771064c3),LL(0x545f80d2,0xf119551c),LL(0x142107aa,0xc2378a52),L_(0x233fc912), + LL(0xd5e26db1,0x2c57b6eb),LL(0xe31036a2,0x2260e245),LL(0x18945c38,0xa0a4a4f0),L_(0xa70971ce), LL(0x51a8f045,0x78d2269e),LL(0x3f54f031,0x08084351),LL(0xd1a3f072,0xcf63943b),L_(0x1bbad8a4), + LL(0x450a90c1,0x7535f2ce),LL(0x2202c2ff,0x29cb37b3),LL(0xad07269a,0xc576098a),L_(0x233792b7), LL(0x368c1c44,0x5864342f),LL(0x02886c17,0x2dca7f73),LL(0xb1a39467,0x74bab9ee),L_(0xe841def5), + LL(0x8df7b590,0x7f584614),LL(0x0ec350c2,0x3c0acdb3),LL(0x2da65636,0xb160863f),L_(0xe9a43081), LL(0xb0ebfa84,0xce3663d0),LL(0xafbf1395,0x69ff1f32),LL(0xdfa06d4c,0x9aa85d56),L_(0x4f103987), + LL(0xc94ce26f,0x7df657b9),LL(0xf5ac31c8,0x0b939fc2),LL(0x1a02ac9f,0xd28fa99b),L_(0x6407f45c), LL(0x9ca2e994,0x06494081),LL(0xe60e26a5,0x6719d6cd),LL(0x44cfce93,0x4db05d61),L_(0xf189ccff), + LL(0x665df7e1,0x16a82c07),LL(0xd179a921,0x85dffbcc),LL(0x51227dd4,0x1d15b312),L_(0x129de8ff), LL(0x28e8d109,0xe117dd1d),LL(0x3f1a519b,0x3db3c554),LL(0xd473a86f,0xe470bd93),L_(0x48ca17dc), + LL(0x2cd73074,0xce23cb8f),LL(0x025cce52,0x27d52714),LL(0x7128e28f,0x60a7af16),L_(0xee37af3a), LL(0x291c3d92,0x228c1193),LL(0x8bcaaced,0x559ca84b),LL(0x60835724,0x5f7c29e2),L_(0x42db013e), + LL(0x80630e79,0xd7266cd9),LL(0x5ce7356c,0xb190aae6),LL(0xc0bdd70b,0x086e72f7),L_(0x3319c072), LL(0xef5d0e48,0x4ae3ba69),LL(0x086343f9,0x95fb3fdf),LL(0x80bc2aba,0xbf3ecc51),L_(0x37cadfe3), + LL(0xaab46c1f,0x66dd80d9),LL(0x32cd82a6,0xaf92ed97),LL(0x11e8daf7,0x6268a0b4),L_(0xbd297ce7), LL(0x1771ff6b,0x971bdef8),LL(0x8c4eabfb,0x446b9cea),LL(0xcbfa339f,0x83d9f14e),L_(0xbd36bd3d), + LL(0x38641225,0xe3a86074),LL(0xf0d2a9f0,0x46459726),LL(0x68a87b42,0x27fad851),L_(0x465da389), LL(0x6ec9b313,0xa438dfac),LL(0x1024a804,0x2f7651af),LL(0x3e3b2c83,0x9e904c3e),L_(0xb0cbbf5a), + LL(0x3daddf8b,0x4f3cafc7),LL(0x4571821c,0xc7f93897),LL(0xb59cdc87,0x564a1eff),L_(0xd3fc8e11), LL(0x32b8d1f1,0x210954cc),LL(0x485b3e16,0x0b89424a),LL(0xa39a8650,0xc71b961e),L_(0x50587567), + LL(0x938ffcc2,0x3f39a45a),LL(0x39ff9b1a,0x22ec814a),LL(0xe9320257,0x293c62a7),L_(0x30d60ad1), LL(0xfe028f50,0x51699000),LL(0xecc36d79,0x687388c0),LL(0x782811c0,0xb9b6100c),L_(0xd4aec2a8), + LL(0x4fa8529f,0xbf6700b9),LL(0xdb67bc3c,0xb75d6fc1),LL(0x81a05a44,0x2a63deb4),L_(0x2c00d649), LL(0x4f57cb6a,0xc429bae2),LL(0x962a325d,0x45995d71),LL(0xf818f011,0x07c3cde9),L_(0x8b0fa963), + LL(0x1767b13e,0xdcf6f7fc),LL(0xd9cd3792,0x4c91cbcf),LL(0xa1b347a0,0x9daa99a0),L_(0x6b0d6566), LL(0xe85fb43d,0x448838bb),LL(0xe794fda6,0x9a62c0b7),LL(0xb6b1fe7b,0xc0cb0a63),L_(0xd09a8df9), + LL(0x0510ae51,0x4c395b4e),LL(0x8e70bc90,0x49d7c304),LL(0x04b3e853,0x9d40e63f),L_(0x4a2dc6ce), LL(0x6ec0b89c,0x36339e46),LL(0x385a4594,0xb1a1676b),LL(0x455a09ad,0x9304dd88),L_(0xc6ff5bf9), + LL(0x7502d288,0xe085cdf7),LL(0x187b4002,0xa35b991a),LL(0x4c4e720e,0xab690dee),L_(0x86287cf7), LL(0x05d1c250,0xe2c63b4e),LL(0x9adbcac5,0x6b60d74e),LL(0xddd28bc7,0x4248e502),L_(0xef074338), + LL(0xc29777cd,0x896670f3),LL(0x08020d96,0x3a144e73),LL(0xe06c4b5f,0x9786e6bf),L_(0x3da82523), LL(0x4a099ef4,0x4fe3491e),LL(0x99758e3b,0x14bb92ff),LL(0x0c3b82b6,0xbab9fec3),L_(0x45c4baf6), + LL(0xc471a0d6,0xb5a9a1df),LL(0x4e7e7ab2,0x56e0f739),LL(0xce6cd002,0x7c86bb7d),L_(0xcadc14a3), LL(0xf7d0edab,0xbef7e6fa),LL(0xbff62993,0xd5068119),LL(0x0a2850cc,0xc5820cdd),L_(0x92bdfe5c), + LL(0x8c84d27f,0x484daaee),LL(0xd799a2f4,0x612b63b1),LL(0x550daad1,0xef0ac41a),L_(0x8fbf8f4e), LL(0x377207e9,0x84f7e136),LL(0x7bdb6e8f,0xfd5152ed),LL(0xd89df99b,0x395821cc),L_(0x20d76719), + LL(0x3fd703c0,0xae387d38),LL(0x760e59a1,0x28b52395),LL(0x97e8f74f,0xbc044138),L_(0xa21350e4), LL(0x86d227b5,0x02533549),LL(0xe3e6583e,0x2b3bb618),LL(0xc654b122,0xa829cd8c),L_(0x39c79315), + LL(0xe57a78bc,0x040ffaef),LL(0xb7bea5b5,0x0bea9607),LL(0xdb5241af,0x372d1796),L_(0x940a8a94), LL(0x4374d692,0xf1499afd),LL(0xcabac4a3,0x4a7889cd),LL(0xb861df89,0x0cb6a234),L_(0xa96fe9bb), + LL(0x7682c515,0x5eedd184),LL(0x4434f8b6,0x5b6b1177),LL(0x2a8d7505,0x86f300e0),L_(0x0e35d724), LL(0x5d047332,0xfaf29ff0),LL(0x4c814382,0xea27ade1),LL(0x141d9196,0xd1bc5f13),L_(0xade4deb9), + LL(0x5fdcb5d6,0x406ace5b),LL(0x1ea243b0,0x088fb509),LL(0xd6070207,0x22934b39),L_(0xd9cd49e0), LL(0x0d4662d6,0x4d0f3174),LL(0xc755b33c,0x4eef4ac5),LL(0xf8e1a986,0xc50467c8),L_(0x18a3ef6a), + LL(0x747a7c15,0x4607dcec),LL(0x614bcea6,0x5f5ba78f),LL(0x68ae9194,0xf3906a4b),L_(0x42d006dc), LL(0xde1842dc,0x4cb64b0d),LL(0x9dbec629,0xba893ced),LL(0x0bf6a2f2,0x057751b4),L_(0xa9fae60b), + LL(0x6b278395,0x3c12fc74),LL(0x017d14d0,0xc71b3f43),LL(0xc6c97422,0xa51b04ab),L_(0x626c81c6), LL(0x3b8a9b8d,0x238bcf03),LL(0x5e266a2a,0x6ca507ab),LL(0x23764003,0x51803104),L_(0x5f7162bf), + LL(0xfae6aeea,0xa5dd759e),LL(0x522db1a2,0x5e4a05cb),LL(0x09f2baed,0xf85a6ac1),L_(0x323304d4), LL(0x5e98632e,0x1360791b),LL(0x1db93b31,0xb1ea2fc8),LL(0x1b768230,0xabf3fed8),L_(0x87d85b61), + LL(0x79a68d97,0x761f1b7c),LL(0x0b49a02f,0x3ccdde40),LL(0xbb854598,0x77637e3e),L_(0x0022290f), LL(0xdfc1715a,0xf0700420),LL(0xe0410aa3,0x4a5a58b2),LL(0x6d34be85,0xf6bc18c0),L_(0x46cdba65), + LL(0xe6b65986,0xb9773b59),LL(0xc5228020,0xd35b5bb3),LL(0x453c7ac2,0xf11276b9),L_(0x7df2cd47), LL(0xcbee2352,0xf5a8c36a),LL(0x179c274d,0xba21cd5c),LL(0xa5c5bb12,0x11df901f),L_(0xe17d00c3), + LL(0xb8d6f9d0,0x3ed4c79a),LL(0x8883a1db,0x60bbaf97),LL(0xfdd5776e,0x2f54ff27),L_(0xcb776738), LL(0x589db69e,0x9a98b106),LL(0x79c8cc7c,0x9590d453),LL(0xdc2c596a,0xc7ee141c),L_(0x6b47161b), + LL(0xb554abc5,0x543685f9),LL(0xc2395393,0x329ac98a),LL(0x9a0b36b0,0xaae1a4bd),L_(0xd5d4e38e), LL(0x0637ec30,0xc2e17c96),LL(0xd7e5c5cf,0x54bb5d81),LL(0x2bef0600,0x9ed78a64),L_(0x561fb810), + LL(0x4b821219,0x2fe8fb71),LL(0x1a1e9b3b,0x4feedce1),LL(0x3925b0ea,0xbfb2449b),L_(0xb054dfbe), LL(0x29864158,0x28647223),LL(0xa5055940,0xe2ec01b2),LL(0x31e1b9a5,0x23b5a4c4),L_(0x195035c8), + LL(0xbdf1c841,0xcc50c48c),LL(0x209f3af8,0x1489a642),LL(0x181662d4,0x4b465365),L_(0x91296676), LL(0x9e845d49,0x138dff73),LL(0xed5f5189,0xe47473ab),LL(0xc0ded518,0x40df6431),L_(0x723fb4ad), + LL(0x38970226,0x112db661),LL(0x3569136f,0x7a1d7c16),LL(0xb8e53775,0xdd45fe78),L_(0xfaa04e8d), LL(0xb657c7ea,0xf6371e53),LL(0x86806849,0xf29213b2),LL(0xf4779601,0x4c858ecd),L_(0xa4992880), + LL(0x77e1f87e,0x89673149),LL(0x156e1198,0x03cb4796),LL(0x39eb8237,0xecde7783),L_(0x6d9c66e5), LL(0xe3363ded,0xd2f38b47),LL(0x3fba7225,0x0a4d15cb),LL(0x088944d2,0x4f8035c2),L_(0x33f1f404), + LL(0xa1983b54,0x51cb66d5),LL(0x58d0680a,0xaf231cb5),LL(0xcb3e58aa,0x1d2325d3),L_(0x13427358), LL(0x6d4b8ce2,0x7dbb0bf1),LL(0x85f983bc,0x3bec3e74),LL(0x0f88839b,0x8a991a31),L_(0x6b3985b4), + LL(0xf774901f,0xdf68b707),LL(0xde5b911a,0x0c634df4),LL(0xeed92206,0xfc74f9a9),L_(0xbcecf38d), LL(0x3a74e355,0x96e739bf),LL(0x0a6c0e27,0x9a315c68),LL(0x82400ede,0xbba8f32c),L_(0x27d5bb2b), + LL(0xa27fb54f,0x3131df10),LL(0x042f1a03,0xac9267a7),LL(0xe003b634,0xa22e7895),L_(0x6dd1e8b0), LL(0x4591bb21,0x90518b2b),LL(0x5efe4007,0x9c565a5c),LL(0x88b87e5d,0x63c085ee),L_(0xf6155e0f), + LL(0x9a0ef584,0x841e4add),LL(0x1f02ab50,0x7b5e812b),LL(0x67a50be5,0xe6f206a7),L_(0xf1e42056), LL(0x737ad369,0x2febb4c6),LL(0x657a0dc7,0x93b35fb8),LL(0xbbf3484b,0xea4f285b),L_(0x020b154e), + LL(0xa95bc0c4,0xdbbe1817),LL(0x91e4e9f4,0x8583001a),LL(0xbf43b7fd,0xe4667773),L_(0x52865635), LL(0xec4299f7,0x5962ba86),LL(0x28fe8ab8,0x3849c7d1),LL(0xd449338b,0xd636805c),L_(0xa1efd1c9), + LL(0xa552a401,0xdfd448aa),LL(0xb92e258d,0x3f8fa817),LL(0x35b6d45c,0xa42ce187),L_(0x0d195771), LL(0x0ca7ae66,0x672eaebb),LL(0x60feb26e,0x34d002fc),LL(0x94989870,0xb3b0ede4),L_(0xc41d2caa), + LL(0x75191aa5,0x5261f5a6),LL(0x330e8a48,0xce0728c4),LL(0xf61c72d3,0x33ca48fd),L_(0xffeebdb0), LL(0x7973a221,0x7ae9478e),LL(0x73383a11,0x2d6cf157),LL(0xafb5d296,0x689cfdb3),L_(0x32706d23), + LL(0x5b0f9106,0x0bd73e3b),LL(0x62c3a48b,0x2c31c735),LL(0x0c0a3d1e,0xacf48d35),L_(0x48c97582), LL(0x9aa2cb41,0x01b5bb28),LL(0xaf780b08,0xd14e0836),LL(0x2a0a2d75,0xabcaf322),L_(0x13f7f83f), + LL(0xf123c708,0xb992a79f),LL(0x19a0cf96,0xada9dff4),LL(0x98adc9dd,0x6a4d3e6c),L_(0x40f8faf2), LL(0x676dfe03,0xef011d37),LL(0x7e67e105,0xb029aa6b),LL(0x87eb4d53,0x45add6b3),L_(0xce77f195), + LL(0xfdc3ee1e,0x3d9993bb),LL(0x8adaa277,0x9c19b8ff),LL(0x1fca3093,0x7433657e),L_(0x8caf0d86), LL(0xcbbc468c,0x85112380),LL(0x3dbd4315,0xa4c53f5c),LL(0x9d410bfb,0x374f5e1e),L_(0xf2fcce49), + LL(0x058ab78a,0x532b7587),LL(0xd3243b22,0xf94c79ad),LL(0x11c1365e,0xb045d4a7),L_(0xb70776bc), LL(0x35abd993,0xadb798da),LL(0xe29a9f85,0x449472ae),LL(0x04e83977,0x4ef414c2),L_(0xd94bbca4), + LL(0xe9445da7,0x69a5c563),LL(0x189faa10,0x013c6f09),LL(0x11ff4092,0x270bddfa),L_(0x13f400ef), LL(0xab739108,0x54c1f415),LL(0x5467f97f,0xb88a9ecf),LL(0x8a19f9f7,0xd713b7c7),L_(0xbecb24c4), + LL(0xa9451b5f,0xd9e41b7c),LL(0x154bb407,0xb69c99b7),LL(0x9e3d7f55,0x0a7b54c7),L_(0x135359b7), LL(0xd2d3aed3,0x84d9fef7),LL(0x5d18cdb1,0x61c13355),LL(0x142d7261,0x1cba3904),L_(0xac50bd6c), + LL(0xb8b1ee7a,0x709ebb7d),LL(0xec49c916,0xd7ba31a4),LL(0x16f1cdf7,0x9b4ca4e1),L_(0xd65f4a10), LL(0xad080359,0xf98d2d14),LL(0x03b182b1,0xe7c36b4e),LL(0x060b7206,0xb11e15e8),L_(0x6d96c56b), + LL(0xf8d054a3,0x8f77e5e4),LL(0x72cd83b6,0x93dad056),LL(0x25a5b5bf,0x25a2e699),L_(0x6fdd7673), LL(0x0063d830,0x90a6d4fe),LL(0xd576b9f6,0x3c44f207),LL(0xbcb25092,0xad2b4418),L_(0xefb79bc4), + LL(0x01e94264,0xfba463c6),LL(0x67359a81,0xfc726fd8),LL(0x2c69efd6,0x280e7df6),L_(0x74569519), LL(0xa6c619bb,0xe2c76093),LL(0xb18f6667,0x8f9466f6),LL(0xbd326eec,0x88831553),L_(0x87869f49), + LL(0x88c46f2f,0xbaa57c7c),LL(0x3b207add,0x7299422c),LL(0x4ed0514a,0xb373586a),L_(0x309bffcc), LL(0xc5186b1b,0x461b005b),LL(0xc0f53c53,0xbbee060f),LL(0xc558624a,0xcda6e3b1),L_(0x6faa11b6), + LL(0x1f345075,0xf0680ab2),LL(0xcbfa17ef,0xe657cfab),LL(0xcddb5b82,0x41c78a0d),L_(0x20234c33), LL(0x65c9a0da,0xc2fe9028),LL(0xf4f1f1bc,0x14464393),LL(0xfc36a9bc,0x75517f43),L_(0x7adc3f7c), + LL(0xc9f99571,0xba51387a),LL(0x49c36869,0xfba899b3),LL(0x03e373c6,0xb8b4767a),L_(0xb07a3f0c), LL(0x7fc1b90f,0xe85c5039),LL(0x808f702e,0x94d61e9d),LL(0xdddd86d9,0x7011d76f),L_(0xfae1c7e7), + LL(0xf98efe50,0x80fa818b),LL(0x31e4ef2b,0xf1aadc09),LL(0x68d478dc,0xb3f48a44),L_(0x7630fe61), LL(0xaa830a4a,0xf7c94ece),LL(0x9a533238,0x04218b55),LL(0x5e7ce696,0x3b49d1b1),L_(0x52a95933), + LL(0xc81b286e,0x0d20898e),LL(0x9d1c985c,0x3f1398f8),LL(0xc62ed966,0xce3b7676),L_(0xeb820161), LL(0x4dd1834e,0x60f14618),LL(0xe6d270e0,0x16b9b346),LL(0x80e6594b,0x9e645987),L_(0x879e807c), +}, +/* digit=27 base_pwr=2^189 */ +{ + LL(0xe215ad69,0x0703230b),LL(0x9c23cc7f,0xcb4845fc),LL(0xa6d601b3,0x3f2e4ff0),L_(0x8b1b5936), LL(0xd006b554,0xe0882f46),LL(0x402ec9ab,0xd39b85a4),LL(0x47de2c7e,0x555b4d4e),L_(0xeaa5bf4d), + LL(0xc42ab049,0xf78b555a),LL(0x971ade84,0x9092aa3c),LL(0x4e8a2ef9,0x5ad4ea73),L_(0x0425d602), LL(0x2b7e58ba,0x791d9122),LL(0x227d0459,0xf3147395),LL(0xc9d765a1,0xe4f1346c),L_(0x28b65397), + LL(0xceec40d6,0xe7d16970),LL(0x871eef82,0x361a2299),LL(0xb5fdd432,0xb27a33c5),L_(0x1d066c2a), LL(0x1d4d340e,0xe97230fd),LL(0x862b60db,0x5852f3b4),LL(0x593d4118,0x539f6530),L_(0x8436da66), + LL(0xe5f6d86b,0x2425af9e),LL(0xead58c17,0xe4dac347),LL(0xb5cbb50e,0xccdd8dd9),L_(0x86b3650e), LL(0x95382a07,0xdc320634),LL(0xa3e665cd,0xe9848376),LL(0x1e2d319e,0x33fae415),L_(0xeca579c1), + LL(0x372307a3,0x6a9d1fc4),LL(0x89b65031,0x50fbf109),LL(0x23a987e1,0xfde88e7a),L_(0x2dd37a50), LL(0x2947bdb3,0xcebc549b),LL(0x820f3f12,0xab66e245),LL(0xb7bcaa87,0x694bc71e),L_(0x8e6e09c1), + LL(0xc6572555,0xfe8a98ef),LL(0x63ff767a,0x71371393),LL(0xc5f7f4d1,0x0e5335e1),L_(0x7cfba5a2), LL(0x96ccd504,0x77da1976),LL(0x67e59eaa,0x16544c87),LL(0x504acfd5,0x96be881c),L_(0xbeef988a), + LL(0x3faff788,0x82325976),LL(0x575e852e,0x05c5a091),LL(0x2c72e2f2,0xe0cc66d6),L_(0x1927e387), LL(0xb872a6ad,0x4d0b0121),LL(0x7da1cf59,0x5bcfc963),LL(0x26204672,0xdb25f5a7),L_(0x0a8b1258), + LL(0xe8b35d2e,0x7d69f703),LL(0x20d3988b,0xc271bab1),LL(0x6c24b0c1,0xaab259f9),L_(0x5e65b843), LL(0x51fcf150,0x75e5c2e1),LL(0xea29f462,0x8993613b),LL(0x1834a191,0x8c9eae94),L_(0xf967c0ea), + LL(0x41292928,0x7e9bdb88),LL(0x00adf08d,0xc2ca609c),LL(0xa0138acf,0x8dfb7875),L_(0xd93b37ee), LL(0x3666deb2,0x749de945),LL(0xbfd574b2,0xe95a61ae),LL(0xe75abd5a,0x67c65b64),L_(0x13366a27), + LL(0x21088150,0xb08be400),LL(0xe06fcef2,0x86d73e23),LL(0xe0ba359f,0x607c14ca),L_(0x382ff71f), LL(0xe5594024,0x6404f03c),LL(0x3f8a49f7,0x148ec4e9),LL(0x7853702e,0x4887d191),L_(0x04d3f5e6), + LL(0xd2a149ac,0x67857251),LL(0xdb805369,0x8e8ea485),LL(0xd63c66e4,0xa73ff057),L_(0x587ee86f), LL(0xfb2b3fde,0xa10a6759),LL(0xbd927cb1,0xfc748cd7),LL(0xc799e97c,0x6f65ee19),L_(0x4703e5a8), + LL(0xa25a7805,0xaa2f24f7),LL(0xba32a728,0x7fcf37bf),LL(0xb7ad366d,0xa0ac6a50),L_(0xb220a09e), LL(0xcb955e95,0xd5db1087),LL(0x44a3150c,0x65f007ad),LL(0x09784fd1,0xf6f05498),L_(0xc1e6c0b0), + LL(0xc19f511e,0x06b7160e),LL(0xbb692e48,0xc84c111e),LL(0x15498ecf,0xb77b124d),L_(0xbb092f5d), LL(0xc61f57ba,0x74e3ca97),LL(0x7b0969cb,0xd5839144),LL(0xf86a2cf8,0x6d59ed66),L_(0xdeabc883), + LL(0x9f404ee0,0x17174a2e),LL(0x852eae73,0x1bb80f22),LL(0x1e348f1a,0x0374bff3),L_(0xeacc5568), LL(0x8d863212,0x5585a76c),LL(0xa921615e,0x66b5bc71),LL(0x1b74c1fd,0x26867d50),L_(0x52fb37fb), + LL(0xd2f49002,0xb96496b7),LL(0xe996dbee,0xa5746d1f),LL(0x39670ef3,0x8094e471),L_(0x9c159e05), LL(0x286d79a8,0x4226465a),LL(0xb7fd505e,0xbf1aca6c),LL(0xf2355f67,0xe46e77c4),L_(0xe0fbfc8b), + LL(0xea2be4c9,0xc386d032),LL(0x16b11ec1,0x828f0d4f),LL(0x75da11d8,0x94c7a19f),L_(0xfda19041), LL(0xd7f40cb4,0xcd2bfd30),LL(0x0e5ba2ee,0xb240b8db),LL(0x2339d87a,0x375d7766),L_(0x24af1974), + LL(0xd5cb473f,0x0c7c5c0e),LL(0x06c1fc0c,0xb684879d),LL(0x1d94e717,0xb65c6a50),L_(0x0b30821a), LL(0xd7821eb5,0xefa6a39d),LL(0x6459b552,0x727bdd40),LL(0xb7d7e4b6,0x18598afa),L_(0x00b319ed), + LL(0x703608d7,0xf884841c),LL(0x8dad7679,0xe15515c9),LL(0xb306db5d,0xb36c0754),L_(0x5229df9a), LL(0x979aad60,0xaadea0d7),LL(0x970bb22c,0x6c1b7c90),LL(0x298689dc,0xf3b57c01),L_(0x89fe17ad), + LL(0x428e4fe4,0x76218fd4),LL(0xceff7d9d,0x975a4605),LL(0x63a383cf,0x238b4e91),L_(0x99f29b16), LL(0xccf0edbc,0xe11ea485),LL(0x7ae47927,0x93601402),LL(0x7cd7a8d0,0x0d9f157a),L_(0xb3fa2b87), + LL(0x9e4d4311,0xff049f88),LL(0x85fe8244,0xcf8e1ceb),LL(0x5d2ff231,0xef0f5b1a),L_(0x63ccdc00), LL(0x959a2ad3,0x8ba95ec0),LL(0xa8a44260,0xbd876759),LL(0x6ed75cfa,0x3e43af21),L_(0x8464e573), + LL(0x444c26f1,0x65d29b4c),LL(0x1583a353,0xee5f7609),LL(0x43193114,0x92998e59),L_(0x704abb0e), LL(0x99be8a63,0x2a7e16f6),LL(0x4c98adbf,0xe1fc70da),LL(0xa5a25fbe,0x44dc73bb),L_(0x8fc488f7), + LL(0xa3bfed42,0x784f9377),LL(0xe7bcd034,0x07fc5bf3),LL(0x89f2f24f,0xbec66d27),L_(0x4917e013), LL(0x3a3c1ba3,0xf0cde4d7),LL(0x0961875a,0x69bd4eb3),LL(0xa260b4df,0x59faeb8c),L_(0xbeeb1631), + LL(0x21f06827,0x280f3962),LL(0xa088a1e1,0xcf16263e),LL(0x1dbee473,0x1692a131),L_(0x05a158ea), LL(0x283c279b,0x0a11b121),LL(0x0e13e263,0x8ed3889a),LL(0xdee544e9,0x5dcaae12),L_(0x8f1fbb34), + LL(0x87dc2c5f,0x5d2e90ad),LL(0xb9ad99ac,0xed6943f8),LL(0x0c329529,0xaa459865),L_(0x96ffc387), LL(0x34567b61,0x13516c2e),LL(0xa0aa883b,0x39122bf4),LL(0xbfa72fcc,0xa825f1f8),L_(0x97da099b), + LL(0x67b5c14e,0x14bced82),LL(0xf4fdf535,0xff66cf51),LL(0x689f6f0d,0x71814baa),L_(0x5b8ff385), LL(0x247c8262,0x9c1ba7ce),LL(0x907e4e69,0x4b578253),LL(0xff0ce18c,0x2458cb25),L_(0xef5cd0aa), + LL(0xb909db13,0xfee8b8ef),LL(0xa4710d21,0x1107af8a),LL(0x54895b98,0xb541a6b0),L_(0x526b0ca4), LL(0xa7097ce7,0x16947322),LL(0x31b294e5,0x29af3ae4),LL(0x1b4cb2b5,0x98687802),L_(0xfbcc3509), + LL(0x1586105f,0xb150cf07),LL(0xe16b93aa,0xa4bbc36a),LL(0x253eb0dd,0x984df68e),L_(0x87118c04), LL(0xcb079f80,0x480076ec),LL(0x0dcde0c7,0x6b489d1e),LL(0x18bdffd8,0x16aa0174),L_(0x1f265d34), + LL(0x59614a61,0xee0e75bd),LL(0x5617d582,0x52348cb9),LL(0x3e671da9,0x0190d33a),L_(0x3bee1eff), LL(0xaa01d03e,0xe617dbcb),LL(0x09cde5ef,0x84c57057),LL(0x888eade7,0xfc85f3d6),L_(0x359daa39), + LL(0xd48bf3db,0xb342c647),LL(0x4a00b224,0x40b2d73f),LL(0xb894175c,0x58a94e88),L_(0x70cf2206), LL(0xb4cd9a9f,0xf20b5560),LL(0xdca48b9e,0x1a01b8da),LL(0x7707c57e,0xa0a1a133),L_(0xc33e6b03), + LL(0x727e08d2,0x9c8970a1),LL(0x4c16f18a,0xf1b1124a),LL(0x57b552ca,0xda495e0b),L_(0x1a9ee7da), LL(0xe079a731,0x1ad4dac4),LL(0x06f7634a,0xf219d184),LL(0x1b77c61d,0x110655eb),L_(0xd5bf028c), + LL(0xf20ff378,0x21c59244),LL(0x51b05ff6,0x7d017e8f),LL(0xaf13b9eb,0x2b5c0017),L_(0x4aa9bdff), LL(0x2371404f,0x703d8718),LL(0xaf5e4830,0x1cdec0ba),LL(0x52cc936a,0xf59d1925),L_(0x013abb12), + LL(0x0e22d6ca,0x1082f990),LL(0x2c9f0b9e,0xf4898bd8),LL(0x666e06e5,0x703ab683),L_(0x6f1b6308), LL(0x2a6a3005,0xc8b722db),LL(0xd103845d,0x3ef671a0),LL(0x2459c28d,0xd0ce016b),L_(0x60884f40), + LL(0x59320b15,0x7f7ba17f),LL(0xe194bda2,0x52a3af1a),LL(0xbb90a873,0x2977890c),L_(0x3dc3fa11), LL(0x13cb3938,0x1ec1818b),LL(0x2f976a14,0x813e2d47),LL(0x6ef670b1,0x547bbc14),L_(0x7a6392b3), + LL(0xbee8917d,0xfa44c4a2),LL(0xd1652cf8,0x11ddbdcd),LL(0x7b98c98d,0xeeaab800),L_(0xb60ccb53), LL(0x699cc945,0x20bc5779),LL(0x3d3d111a,0x5d171d19),LL(0xf1c69a57,0x4586a0f5),L_(0x047748d6), + LL(0xcb2dd3a2,0xbfb2046d),LL(0x1ba40992,0x4ab9b7d4),LL(0x492e1976,0x8fa3d075),L_(0xb62a0315), LL(0x9faa1011,0x08bcfaa8),LL(0x55ee388d,0xa8201ba0),LL(0xb50f4e76,0x48d16444),L_(0x947b5775), + LL(0x7bb21f8c,0x27f91139),LL(0x9933e071,0x7328102f),LL(0x16712a5a,0xc739c006),L_(0x75d0d886), LL(0x0d97d602,0x95eed6e2),LL(0x5fd2d047,0x4657933f),LL(0xfa45dd80,0x9d54f1fb),L_(0x741b92d0), + LL(0x4b499dd3,0xe5114e38),LL(0xc2275c0e,0xc9aa88b0),LL(0xb4d5582b,0x3e016601),L_(0x25741fdf), LL(0xe08edbac,0x4514d2eb),LL(0xc4ccb66a,0x16e96390),LL(0xb57016cb,0x59b84d70),L_(0xf83fe7e5), + LL(0x697291df,0x7a6974e3),LL(0xcecae61e,0x747d2ddd),LL(0xa93265cf,0x1f1fb9f1),L_(0xf3c9e95d), LL(0x05b8ad90,0x76333190),LL(0x5ef96caf,0x2e7e8b35),LL(0x26204dea,0x66ec9969),L_(0x8ee499c3), + LL(0x7d3e7f82,0x1abe8df3),LL(0x0e4c49fd,0xf5621bfb),LL(0x5430fd38,0x5d968105),L_(0xb2efc301), LL(0x7fa73503,0xde3d9216),LL(0x191e5b5e,0x358512db),LL(0x32722378,0xa0892090),L_(0xd92f1352), + LL(0xfec9743d,0x757c94ef),LL(0x9649e576,0xf290f932),LL(0x79587916,0x8be7ad8d),L_(0xadd01304), LL(0x61f6de8f,0x29445ca0),LL(0x285b7eef,0x1426c1d9),LL(0xae55ea09,0x962f7357),L_(0x983ad1c5), + LL(0xdfa17534,0x40d7b215),LL(0x3897f22b,0x4a689975),LL(0x0a2968a8,0xae9ce773),L_(0x5eeabf53), LL(0xb2a472ce,0xd7707d10),LL(0x3f8692b7,0x11cc24bf),LL(0x416e3f6c,0x0e77bce9),L_(0xf53b376e), + LL(0xe52b9bd7,0x211044a2),LL(0xb9dbcba3,0x32202391),LL(0x7e52000c,0x2961ea4a),L_(0x12e1a413), LL(0xae4f775c,0xc87793ae),LL(0x36916c16,0xadc59df4),LL(0x93f3db96,0xf4f9dbfb),L_(0x2bfda0b4), + LL(0x04c1afdd,0xbbbbeb9c),LL(0xcc1030ad,0x0de0dfaa),LL(0x49bedb1d,0x1869975f),L_(0x6a004fbc), LL(0x0b0e188c,0x51c16437),LL(0x71e53119,0x3e18f624),LL(0x5e139c48,0x60f866a3),L_(0x0784d6e0), + LL(0xc173ecc1,0x97f340e7),LL(0xa61d043f,0x444c1c32),LL(0x4c35a67a,0x8581b05b),L_(0x44182222), LL(0x731fc41e,0x45bca0e4),LL(0x8bf96ead,0x853c2a7b),LL(0x7c86a091,0x2ad3d962),L_(0x7ea985d4), + LL(0x5d1d3e02,0x1ef28f3e),LL(0xcbf512f8,0x72330e86),LL(0xdc8ed43f,0x516f1183),L_(0xec32dc2b), LL(0xd7b96122,0xc45e5ada),LL(0x505156da,0xdf67ff42),LL(0x983acb8c,0x30958d17),L_(0xe6ec6def), + LL(0x569438de,0x1072cfdf),LL(0xaaf2a64f,0xda01f192),LL(0x33454609,0x8b880e7d),L_(0xa7a46a05), LL(0x35cc6b77,0x7354fe07),LL(0xd5821805,0xa74b9400),LL(0x0fc1cbb1,0x0e58985d),L_(0x9f7a9dad), + LL(0x84ac106e,0x9dc30814),LL(0xe1ebef86,0xcd23c165),LL(0x852da87b,0xed5cdd01),L_(0x21bc828d), LL(0xce6842d7,0x5aadc859),LL(0xe522baaa,0xe89ad998),LL(0x608b4b16,0xee466b6e),L_(0x0c33a24a), + LL(0x8243d368,0x4e1fae12),LL(0xf397b191,0xe6b7d57c),LL(0x2940f1c8,0x47dd818f),L_(0x68c77c89), LL(0xe08f2ebf,0xba40969e),LL(0x8ffaa799,0x102a6172),LL(0x712df945,0x997feef1),L_(0x4da4261d), + LL(0x455b0c0c,0x1b0afa51),LL(0x69fea568,0xf8ba1286),LL(0xdfc334a6,0xa2587e32),L_(0xf61b674f), LL(0x060208cb,0x62eb8d23),LL(0xdd643088,0xa1a3f3bd),LL(0x5cde0036,0x53b8bdc0),L_(0x1c5ce488), + LL(0xb0c1bb5a,0xa64a20a5),LL(0xd22b7f88,0xf22948de),LL(0x0628283c,0x75129c40),L_(0x6efeb06e), LL(0x4000d1fd,0x76f8c148),LL(0x181e8b58,0x7b823883),LL(0xc758648b,0x3d1fbafb),L_(0x069f31f1), + LL(0x2b15bfdf,0x78f3872c),LL(0xbba94fb7,0x49778942),LL(0x852ef997,0x48ddef24),L_(0x066ad60c), LL(0x38c99352,0xc1ba735f),LL(0x790d395b,0x30382b1c),LL(0x7616ea81,0x61b408b8),L_(0x9278e0eb), + LL(0xc0f1bdbb,0x15c9a84f),LL(0xae384504,0x1419e604),LL(0xf4814e2b,0x1b8b0e26),L_(0x2b97a1b4), LL(0xf8656f5b,0x260ef3d0),LL(0x5db22d7e,0xc76c3691),LL(0x2e7c6bf8,0x7c1471d1),L_(0x7bdec3b1), + LL(0xa27bd94f,0xb71fd22e),LL(0x7f17f289,0xbbacb216),LL(0xd4564a47,0x34b284d4),L_(0xb75a3120), LL(0x09622583,0x986c8a28),LL(0x197cc0c1,0x208425a0),LL(0xcc3ae593,0x7da4599c),L_(0x54ebd08e), + LL(0x2d21c888,0x9addab68),LL(0xbd0c6b5c,0xcad27cf0),LL(0x7ec99646,0xaa918115),L_(0x8ee2e6eb), LL(0xea74e73d,0xd9db5cd3),LL(0x2d90dfec,0x719d6c8b),LL(0xf48ec5bb,0x7732d381),L_(0x2daa3783), + LL(0xfe67cafa,0x4087fb31),LL(0x64d12677,0xa7b372c6),LL(0xd6d5d34f,0x5aa57c66),L_(0xeebb82c5), LL(0x03f99f8c,0x0b3a848f),LL(0x4bcc334e,0x3076a517),LL(0xafc73b46,0xbdc15e76),L_(0xba3353c1), + LL(0x666e86e2,0x3f49b089),LL(0xe4f392d0,0xb96b8ccf),LL(0xc0bd57d7,0x4a84d25e),L_(0x8b9c3a47), LL(0xdd447511,0x88262b51),LL(0x5d6233aa,0xf3ee05e8),LL(0x1da165fe,0xba39a0b7),L_(0x56569118), + LL(0x4b2743e8,0x55aff6c3),LL(0xffb26779,0x0356100f),LL(0x581f3f38,0xc259ed17),L_(0x0527ce42), LL(0x9f2d81ee,0xeb967185),LL(0x592962f6,0x58f98402),LL(0xbc866b82,0x9a5ee9a2),L_(0xb8fcf82b), + LL(0x50d6a742,0x7b7d67a1),LL(0x6bed8eca,0x0cc91aac),LL(0x74e18696,0x3ac5ac85),L_(0x4777aaae), LL(0xfcd5973d,0xaf83cba8),LL(0x114e541f,0x44df12cc),LL(0x49c90e0c,0x6ffe97ac),L_(0xd22caea2), + LL(0xf6ec95c0,0xad0a7c5c),LL(0xcf68be2d,0xef894370),LL(0x46042df0,0x44cc34c3),L_(0x83baaad0), LL(0xda2df354,0xe5be5f7b),LL(0x232306d2,0x09c99b41),LL(0x30e3711b,0x7b102ef8),L_(0x90ca6b87), + LL(0x9041d06a,0x24f85c28),LL(0x1c82027d,0xc05d59fb),LL(0xc82c4968,0xf294dde1),L_(0xa0dabc8e), LL(0xc334c99a,0xeadad609),LL(0xa2d74908,0x47f2b704),LL(0xea64f015,0x3b0536e3),L_(0x948d4413), + LL(0x90fcfab4,0x4cd8e163),LL(0x7e58fb10,0x9bc6ae53),LL(0x3a2788f9,0xc860f6b9),L_(0x673a28d4), LL(0x95e581f4,0xc5e46853),LL(0x62c5a4e9,0x6d07718f),LL(0x059005a0,0x5a617a3d),L_(0x06267ff2), + LL(0xe3e68d85,0x2af744e2),LL(0xae8162d5,0x16487042),LL(0xbe28dd7a,0x950dd089),L_(0xcd76243e), LL(0x6a083312,0x887a306e),LL(0xc5cf5f6e,0x963509bb),LL(0x6076457e,0xd58cb75e),L_(0xa5a9063c), + LL(0xe722f017,0x6f63cd75),LL(0xa7571852,0x216d76b1),LL(0x2ab669d0,0xd42dd087),L_(0xee6f2f6d), LL(0x097743cb,0x0a7d1e20),LL(0xfa5d28f5,0x21dad6f5),LL(0x9441fb90,0x8466e082),L_(0x4a673507), + LL(0x55c0ab69,0x980ae610),LL(0xc6222c7e,0x5c17d9ff),LL(0x3030f359,0x9b8f42e0),L_(0xbbf50262), LL(0xd0aba7f0,0x469ee64e),LL(0x3ef5e0ea,0x45ce7ede),LL(0x0e1c1519,0xb57bce91),L_(0x806bb655), +}, +/* digit=28 base_pwr=2^196 */ +{ + LL(0x18e92c63,0x4760716e),LL(0x789ae13c,0x03cdfbf4),LL(0xf578d1e6,0xc6470ce9),L_(0xd42e2969), LL(0x4db766a1,0x5f1cf230),LL(0x733456c6,0x1237a68d),LL(0x592bb19a,0x91e21ef9),L_(0x8814ad36), + LL(0x91a052d2,0xbefb4b7b),LL(0x6eb484b6,0xf5d7b19f),LL(0xa01ef7f5,0xdc02e33e),L_(0xf8ac22f1), LL(0x1ee85810,0x0cbad014),LL(0xc2d60cca,0x0ecada19),LL(0x78a4abe0,0x375f545c),L_(0xbb2518b8), + LL(0xe924bce1,0xe94cf6cf),LL(0x721159cc,0x3c03f7b7),LL(0xd001dacc,0xba328884),L_(0x6ca68824), LL(0x99214785,0x857c60d0),LL(0x3e1c1a16,0x6d9d01de),LL(0x4e3a8a4a,0xe5100cc6),L_(0x2fadd498), + LL(0x21699c26,0x96e332c1),LL(0xba35b5c5,0x3b2527e2),LL(0x32a3227e,0x3d572604),L_(0x3287caa4), LL(0xfa83e20a,0xd3d8aa48),LL(0xd8f05722,0x42872cdc),LL(0x211c6e5e,0x789832b8),L_(0xd7095c24), + LL(0x76f2859a,0x8e5e0a08),LL(0xa554b8a7,0x49d7fe17),LL(0x3cfefa32,0xde9b7d08),L_(0xb712685d), LL(0x3f072319,0xd942a6e3),LL(0xc242dae3,0x105e78a0),LL(0x5c2986c1,0xca0a3ad3),L_(0x58984f5a), + LL(0x90f303c9,0xbb8f3d66),LL(0x75e4f894,0x9f67adfd),LL(0x597d61ec,0xc5900752),L_(0xbb05ff05), LL(0x608daf57,0x78265562),LL(0x8bb8c320,0x31f9a44f),LL(0x2afed365,0x9604812d),L_(0x2ecd08d1), + LL(0x254ed32d,0x4f8511c4),LL(0xf62b6ad5,0x437b9a4f),LL(0xe0ab6bd6,0x9849b3b9),L_(0x437c1eb4), LL(0x99f67517,0x60c66156),LL(0xe2da9ad4,0xe7f23f9d),LL(0xc72b155a,0xe933cc08),L_(0xad7576ea), + LL(0x83df61c0,0x019aa18e),LL(0x06661e71,0x159ba381),LL(0x2a662ce2,0xc9069497),L_(0x5ff8cdb7), LL(0x90a3120f,0x22ef5986),LL(0x62a400b7,0x4c18595b),LL(0xb2915c27,0x1fe38da6),L_(0xffba34b0), + LL(0xeb9b1c9e,0x6335b063),LL(0x6133838a,0xfc5ad1b9),LL(0x76c52676,0xce767a1c),L_(0x5e465185), LL(0x3794090c,0x93f4d92f),LL(0xce0d54ed,0x25dd3904),LL(0x742e50dc,0xabc94edc),L_(0x78bba80f), + LL(0x887eb7e0,0x4b83b6cd),LL(0x81b49586,0x1b46a818),LL(0x1fa505f1,0xae4acd7d),L_(0x832817ef), LL(0xc574766b,0xd6ef910f),LL(0xd59182a2,0x2419996a),LL(0x764ce5bd,0x323e203f),L_(0x63236d14), + LL(0xf567e8d7,0x118cb519),LL(0x8ea6b928,0x1c12c8c9),LL(0x56dd68b6,0x395935d8),L_(0xbd116bcd), LL(0xfc1ac8c8,0x5ced4a6b),LL(0xbb4ec3a0,0x5a7b4c09),LL(0x63f4b4af,0x1f24cb4f),L_(0xf9b9d0b1), + LL(0x596243e7,0x18a50232),LL(0x25f531db,0xaa27b297),LL(0x8b3f0f16,0xa0701098),L_(0x6a6aef50), LL(0x60fe4fb4,0xfb3a537d),LL(0x8c64961c,0xd4b3fc26),LL(0x83f4d9b6,0x440a2f19),L_(0x7696fcba), + LL(0xf34d3bad,0xc0f2402b),LL(0x5360ffc2,0x2a9eb601),LL(0x8d33aed5,0x7da3a904),L_(0x4dde5b44), LL(0x3a35e974,0x6c75adbb),LL(0x0c9938a7,0xd7a77183),LL(0xf4ef9f5d,0x30114b89),L_(0xbb08ab61), + LL(0x80255f87,0xdf042ef7),LL(0x481cae35,0x2a918359),LL(0x6fed5580,0x3de7d914),L_(0xa4e1bcfb), LL(0xcd8bcdde,0x85cd2cff),LL(0x4370738c,0x03a1d543),LL(0xf95a95e6,0x9814d14c),L_(0xe0fd8dcb), + LL(0xf9193246,0xc27ed927),LL(0x655d6ffb,0xe4ef91cd),LL(0x43a4fc18,0xde7270a7),L_(0xcba9f29e), LL(0xa246b9ef,0x2e139df8),LL(0x5cc92d84,0xc362ff6d),LL(0xc5a596a0,0x6a5bdae7),L_(0xd5e4697c), + LL(0xdd86fb5f,0xa1aad671),LL(0xf4cdb11a,0x2ccfca9e),LL(0x35eb1979,0x06b67d2c),L_(0xd3a50661), LL(0xcf9afa3f,0x47a43129),LL(0x2c80dfe0,0x0f28c072),LL(0x1e137541,0x2528a436),L_(0x4a30bc3a), + LL(0xb7a122bc,0xa27429d1),LL(0xac029aa3,0x8704523f),LL(0xe555d79c,0xc26117ae),L_(0xeee94c98), LL(0xd57ea228,0xb5866e80),LL(0x3b83d3dc,0x8fda3fab),LL(0x497236f4,0xa896a908),L_(0xc1eba30c), + LL(0xa3fc2c9c,0x0b1d2a88),LL(0x6a45073a,0x8af7a77a),LL(0x72b5edce,0xc7a3b1fe),L_(0xac5268b4), LL(0x018d5428,0x8589b539),LL(0x25c0f37d,0x22f0783b),LL(0x8cf1d803,0x9d3fe1d2),L_(0x15093e7c), + LL(0x8d1c5128,0xac83f07c),LL(0x6eba86a7,0x4ddb4944),LL(0x7769b4f0,0xce08325a),L_(0x306da9b2), LL(0xe9629e93,0x9fd70869),LL(0xbe70ef68,0xa27b01af),LL(0x9649ac38,0x59634c20),L_(0x260c709f), + LL(0x32f4b675,0xe36b9e35),LL(0x56c9f551,0x59ec0560),LL(0x9a19b345,0x7b1c335b),L_(0xb90484f8), LL(0x2e951d15,0x0b5ef9ec),LL(0x8c6e220c,0xac2613e4),LL(0x4ddb2066,0xe5c29249),L_(0x2b898fcc), + LL(0x90d06a78,0xb7d4dedc),LL(0x3cc11efc,0xf6a2c79e),LL(0xc5d55aae,0xebad4304),L_(0x139832e2), LL(0x73f6f29a,0x0089eaff),LL(0xb6ab35e5,0xa043d1f1),LL(0x35af18a2,0x363f80f9),L_(0x8c8c5655), + LL(0x800e76ab,0x3cad20e2),LL(0xd54cc179,0xd6ee4412),LL(0xc7630400,0x8ece816d),L_(0x4347fa49), LL(0x46a3c678,0xe22f3ed6),LL(0x83d72b28,0x5e7ffb13),LL(0xa89efb1f,0xc07fe6b1),L_(0xf679cfd6), + LL(0x1e9baa6c,0xdd4159b6),LL(0x90409ff6,0xc1a8d493),LL(0x0509d403,0x0d946df5),L_(0x46153d5b), LL(0x2b496957,0x1229cd0e),LL(0x25841f04,0xac5550dc),LL(0x163487ec,0x1f5ac4bb),L_(0x7a894b16), + LL(0xf830d51c,0xd0fd0fde),LL(0x95e0fb87,0x393b8885),LL(0x12d4a0db,0xbac4ce3b),L_(0x4aecb41e), LL(0x949595c3,0xf6b866b2),LL(0xc85cee8f,0xadac3140),LL(0x929c773e,0x22cf7705),L_(0xf90855df), + LL(0x05ee1c57,0xb938eeef),LL(0x054fb615,0x87e5ccbf),LL(0x7d1ce423,0x624c46af),L_(0xc8279b91), LL(0x70e96354,0x6fc664fb),LL(0x957fa9ea,0x0ec18ae0),LL(0x86d71f58,0x8216ab5f),L_(0x4fdcf503), + LL(0x6a1895bd,0x99acbf5f),LL(0xea1c792e,0x2f4551f2),LL(0x3d4924d2,0x09c0ca0b),L_(0xb94190da), LL(0xa95f5a84,0xaccafdf8),LL(0x6c1aa007,0x7635c97f),LL(0xe4eccac7,0x07821f33),L_(0x3125c30a), + LL(0x9eee4fb3,0x76375e85),LL(0xa2da6002,0x6d9d3924),LL(0x851663d2,0x903300d5),L_(0xe4304412), LL(0x16885ea2,0xf13f93d6),LL(0xac6cc7e1,0x765c7a8f),LL(0xc233fc52,0xfa884feb),L_(0x889b7699), + LL(0xea0d9656,0x649b4877),LL(0xb98eb821,0x0e89d083),LL(0xda85057d,0xe8c391ca),L_(0xc53a37ec), LL(0x5ab13638,0x3c241d52),LL(0x1c63bf16,0xe5732bef),LL(0xe6f1e93f,0x882db63a),L_(0xacf7e8f4), + LL(0xb0ae8194,0xd18b015c),LL(0xf734d021,0xe38e3807),LL(0x62b04fde,0xea71f882),L_(0xe9f4d402), LL(0xb7452054,0x3d8f7744),LL(0xad9abc71,0x68a3e02d),LL(0x088ad020,0x25dfde15),L_(0x250f4123), + LL(0x5076af69,0xe0e96eac),LL(0x46b7097a,0xd8c13c78),LL(0x19ddbba5,0x3ecd0bd0),L_(0x1b6e3a84), LL(0x74eefc3e,0xfc22adbe),LL(0xbae23185,0xad1b79ab),LL(0x09ed4ed6,0x062eb7e7),L_(0xe1ce81e7), + LL(0x06e9bfca,0x4649146e),LL(0x91702d2f,0xb30f9778),LL(0xbfdaf3cb,0xcf8f23f1),L_(0xd403a4bb), LL(0x13bbb0cb,0x33bbb504),LL(0x5d48ee37,0x17c9202c),LL(0x79ec8717,0x5a9ca7a8),L_(0x76092de3), + LL(0xfe5a3569,0x1f9e6c97),LL(0x17ed7898,0x2727d871),LL(0x668c28ad,0x745aa2ed),L_(0xcf9ce234), LL(0x83f03768,0x4a783728),LL(0xbf5955af,0xadfd1b46),LL(0x92891a4a,0xea43a792),L_(0xc67457a9), + LL(0x44cf762b,0xb8dd47fc),LL(0xff42200e,0xe06ffe0d),LL(0xa70a5da7,0x6cbd4092),L_(0x6d058c66), LL(0x0f9eb44c,0x110c3a99),LL(0x67048ffd,0x990449de),LL(0x82fc6b0f,0x30242c32),L_(0xa769318b), + LL(0x2521a3f3,0xbb708ced),LL(0x7620c902,0xe32cd4d5),LL(0x75832ff6,0x3a89190f),L_(0x4632df7f), LL(0xc48dd2c2,0x48e4bd13),LL(0x62dc813c,0x4888006f),LL(0x9c55f000,0x36a4457f),L_(0xa330078b), + LL(0xb664f023,0xc1f3d1a9),LL(0xe37d1ff2,0xa5e8a776),LL(0xed06b0a5,0xb1d3c33d),L_(0x939c8f4c), LL(0x836de412,0xf53390d9),LL(0x9e6ebae1,0x35e4cf7b),LL(0x28276f39,0xd69e1327),L_(0x19f5bac2), + LL(0x7885d34d,0xafd90d23),LL(0x0e84e363,0xbf4cd59c),LL(0xabb80eda,0x07e40274),L_(0x95656580), LL(0x0b980b41,0xa8a767b0),LL(0xc90af857,0x560c2286),LL(0x465bb86e,0xbf658fdd),L_(0x7eb455a7), + LL(0x49a8618f,0xda192000),LL(0xad70ec43,0xd0fca141),LL(0xcb55de32,0x7d66c4af),L_(0xa952a38d), LL(0x06ab4e56,0xdceb099e),LL(0xed3e0675,0xa87bfc4d),LL(0x635a862a,0xb9f4c7da),L_(0x8d6f1c2a), + LL(0xfd911863,0x6f166fdc),LL(0x0c46e6e4,0x31dda475),LL(0x0813e798,0x9066cff4),L_(0x2f759557), LL(0x65e21d85,0xbd44613d),LL(0x06317041,0x5b7e81b6),LL(0x6879039b,0x6853d2d8),L_(0x80566bad), + LL(0xe802ae6d,0xd83655d3),LL(0x7c1a0bb8,0x58e0b41c),LL(0xa5228881,0x00c5e925),L_(0xe9c51c0b), LL(0x0ab1b93f,0x0234b858),LL(0x9a00e742,0xe3d995c9),LL(0xf774358e,0x0eb69043),L_(0xd46eb256), + LL(0xe178f2a2,0xe386daab),LL(0x9a305482,0xe1fa3e14),LL(0x7957d9d7,0xf3f52055),L_(0xd1a9ce7b), LL(0xd284eae6,0x303b5ae8),LL(0xac2ad67a,0x7abad7f1),LL(0xb7b58e09,0x7dc5f5f1),L_(0x6c99d250), + LL(0xc25aa814,0x29507cf4),LL(0x37c15d8e,0xa6beff47),LL(0x738d9ba6,0xeb76b42c),L_(0x0b91ab81), LL(0x0d166346,0xac1dd9b7),LL(0xb974e300,0x5e6f1484),LL(0xd55b5832,0x9748b154),L_(0xbfcdc886), + LL(0x767c2496,0xaa791b18),LL(0xb611f73f,0x6dd24074),LL(0x6fd43d11,0xde586616),L_(0xff2a1da0), LL(0xd73b7439,0x7da57054),LL(0x5051af3a,0x9ca8008f),LL(0xa7d98205,0x80c04759),L_(0xd4db7bec), + LL(0xaef4ac4b,0x9051c152),LL(0xd36418fc,0x3e90a332),LL(0x2adcc4ec,0xa2d25b82),L_(0xf9315c6a), LL(0x062c149b,0x6fec476a),LL(0xbdd2e105,0xb614d883),LL(0xdc2bc2a6,0xd473b242),L_(0x9d87dbe8), + LL(0xadf54761,0x4031a20d),LL(0x5001f11c,0x908ea1cd),LL(0x86e3fa8d,0xa7c80719),L_(0x36408819), LL(0x4573ac01,0xaf0e8125),LL(0x430d59f8,0xcf05666d),LL(0x5b426c42,0x1c62cb71),L_(0x1a2c3f95), + LL(0x840aabc5,0x42998c5b),LL(0x9b1ab1ad,0x41b44d81),LL(0xe7ad335c,0xd2f18816),L_(0x9b21d6a9), LL(0x31ed5509,0x09dc1280),LL(0xc3771787,0x5bacd41e),LL(0x69b13c81,0x5419b128),L_(0x472d8eda), + LL(0x47816cca,0xc26aa120),LL(0xb9665455,0xd20f8647),LL(0x3bac4cd2,0x4b96b490),L_(0x15632e30), LL(0x82eca1ec,0x03c2ca5f),LL(0xeac8678b,0x8d2f8677),LL(0x5772568c,0xbef62c1e),L_(0x3b53ce81), + LL(0xee48fbfb,0xbe66a12a),LL(0x753cb0aa,0xe362e842),LL(0x38781cdd,0xcf1c41a4),L_(0xab22df0a), LL(0xb208a69c,0x7e4c97e4),LL(0x3869cfcb,0x72ee6ffc),LL(0xb43a4e85,0x6ab97c59),L_(0x35c0f24a), + LL(0x98013b12,0x292f0d68),LL(0x3e2b7aaa,0x42f99c9b),LL(0xe2ba5f7b,0x77ff29a4),L_(0xe21d90f9), LL(0xd9c0cb58,0xbc0b01cf),LL(0xe3677335,0x4d244a26),LL(0x8fee4b54,0x2b1a301a),L_(0x5d1511ed), + LL(0xa5f95387,0xe5e95789),LL(0xd73285ea,0x392f5392),LL(0x2e506fa2,0xecb1a91e),L_(0x79d54bca), LL(0xbdc25307,0x7bf48edd),LL(0x9f75d7b8,0xe8a12662),LL(0x08014deb,0xaf188a86),L_(0xbfada230), + LL(0xd011c31b,0xa63e4dbc),LL(0xe55bd873,0xd7f84005),LL(0xbe01f07e,0x1b09710f),L_(0x6e6cb66a), LL(0x5fc5038d,0x955bdbc5),LL(0xcecf1493,0xca6b6d16),LL(0x861a0878,0x084d6464),L_(0x0d0e35ed), + LL(0x5abc70f8,0xcdca5ed4),LL(0xdb335ec9,0x1e0fc344),LL(0xa8df4bea,0xcaea8450),L_(0xf564ec58), LL(0x361561bd,0xffeefed9),LL(0x193c8457,0xd89a00be),LL(0xddabba77,0xc99c9580),L_(0xa8ec469f), + LL(0x0423395a,0xf3c9cc68),LL(0xa888f556,0x852e0732),LL(0xaaa99a77,0x79ee145a),L_(0x4760cde2), LL(0x7dbf680b,0xd543c51b),LL(0x4d61119d,0xef9d767f),LL(0xb57c70b1,0x6d605846),L_(0x8392010f), + LL(0x4f6c403d,0x79dcc934),LL(0xd988c0b3,0x48395681),LL(0x66a38840,0x4cdffd50),L_(0x026d6f1b), LL(0x0fd100a8,0xd01d8faf),LL(0xc2150428,0x37a79216),LL(0x453f0c92,0xff7a816c),L_(0x84b14e80), + LL(0x8f9b1871,0xa7823318),LL(0xab4b5d45,0xe70810de),LL(0x1d9dae80,0xce1d1836),L_(0x1f8f6e55), LL(0x3fe286a2,0x8d6bd5d6),LL(0xd272957e,0xe02405c8),LL(0xc59f4815,0x15f1375a),L_(0x1dcaf5b0), + LL(0xd83c608e,0x6c316cb3),LL(0x5f19c736,0x28464849),LL(0x9bb6aded,0xadea31b9),L_(0x1090e65a), LL(0x3ad3fb92,0xfec08eda),LL(0xad397892,0xf01270f2),LL(0xb2c262c0,0xe12a0551),L_(0xcd11a65d), + LL(0xd00f765f,0x5db707b5),LL(0xf10d38e7,0x7a2ef5be),LL(0x5bbaa9f8,0x1692548b),L_(0xa34228d1), LL(0x2a6743e2,0x654e5d41),LL(0xf2342e4d,0xbe7af05e),LL(0x53e19ef6,0x254b33f9),L_(0x2609459a), + LL(0x0a41c216,0x7007f545),LL(0xefba2361,0xd7626b6f),LL(0x72c3ffc3,0x600f4b0f),L_(0x68bfc655), LL(0x0f831a13,0x13f14da7),LL(0x6efd514e,0x9b2809ea),LL(0xcc761584,0x949da3c4),L_(0xd3524443), + LL(0xab16d7fb,0xeca99655),LL(0x98853678,0xf946fea2),LL(0x959586ec,0x2b4d5496),L_(0x882a160e), LL(0x7fe0f8f3,0xdf2a1b88),LL(0x52e13aac,0x0f3ca0d4),LL(0x4a5dde81,0xc432a9d8),L_(0xcafd8d25), + LL(0xae4273ff,0x39455fc9),LL(0x0c20053d,0x2cb371b8),LL(0x04a4c142,0x14b7b1d1),L_(0x38e2582a), LL(0xbc61e230,0x20e68183),LL(0xd15dab61,0x5cc3f0a5),LL(0xba541287,0x62766fdb),L_(0x6619e0a4), + LL(0xf1589655,0x3bc818e2),LL(0x746af8f1,0x7a905de6),LL(0xbbe6fd69,0x53589554),L_(0x446a9ab9), LL(0x42c6c834,0xc30fa665),LL(0xf4d47c43,0x8776e4e7),LL(0x431ea6ae,0x9dafb227),L_(0x4ff395ed), + LL(0xe3b10ef9,0x2c57d4e0),LL(0xa4cceb3e,0xf2b56f06),LL(0xff6cbcb5,0x8918c037),L_(0x268bc00c), LL(0x782ed32e,0xa92f2fb0),LL(0x76415814,0x5aed9645),LL(0x6859753a,0x0a6cac9a),L_(0x399773b8), + LL(0x115dc4ae,0x7652379c),LL(0xc59e872e,0xfceb6ec3),LL(0x1fa3657f,0xa77b28fb),L_(0xa7619402), LL(0x26333b61,0xa85f48be),LL(0xe180ae18,0x671848d6),LL(0x68f2c963,0x3145c6a2),L_(0xe4d30712), + LL(0x71516dad,0xb3ff7310),LL(0x00082b09,0xb2aa0684),LL(0x0de3deec,0x33ecf9d4),L_(0xab6cc532), LL(0xeb77b453,0xd8a0ae1b),LL(0x2a618d55,0xab8ca810),LL(0x3993ca70,0x0f2549ab),L_(0xcc02ada7), + LL(0xe8cde28b,0xb5a7fc92),LL(0xec8a7e38,0x130ca680),LL(0x786e2a86,0x882cb9d7),L_(0x103bbacc), LL(0xc662b5a7,0x7395bf15),LL(0x4be2ec80,0x5a3d7841),LL(0xb287766f,0x4eb9dbc0),L_(0x54c7f36b), +}, +/* digit=29 base_pwr=2^203 */ +{ + LL(0x11dfaf3f,0x663a8bf5),LL(0xf1bd6f1b,0x5a8b6d6f),LL(0x2de2cf36,0x4f49967b),L_(0x49fde755), LL(0x1ea11bab,0x9956f1ec),LL(0x69f14061,0xf4e97d30),LL(0xc5c29863,0x516db87b),L_(0xd4456e50), + LL(0x6d6d1495,0x78b8a6b8),LL(0x5019a365,0x9a15b171),LL(0x025726c3,0xd611e283),L_(0x68f7b67f), LL(0x9367aca7,0xb4e7c5cf),LL(0x5ed6c698,0xfb9fa295),LL(0x6ea9c71d,0x2c0dd03d),L_(0x6ad6ab61), + LL(0x74645aca,0xc1c1b8cc),LL(0xa1ea038f,0x6422554f),LL(0x9d8f1f3b,0x5e44aefb),L_(0xbc0d302a), LL(0xd0763fa0,0x47ec400c),LL(0x9049574e,0x3240a34f),LL(0x9778ebf4,0xe0adc015),L_(0xe0348d22), + LL(0x42b0ea67,0x2b432e4e),LL(0x97be15f2,0x51920440),LL(0xfb123348,0x752805f0),L_(0x095f8861), LL(0x428d08f5,0xe4fd179c),LL(0xbfe10a89,0x4c2603c8),LL(0xc1a95174,0xb5e94f0f),L_(0x38bffbb7), + LL(0xfa8bbea8,0x31d5bf10),LL(0x3b628b52,0x747269c9),LL(0x9f017556,0x50b6b23f),L_(0x2051232c), LL(0xf36708ff,0x18b1fb8b),LL(0x58660126,0x731cf4a4),LL(0x85343658,0xcdefb5cb),L_(0xee7c02af), + LL(0x517736b8,0x842b9439),LL(0x79b770df,0x017ab239),LL(0x2ca8d23d,0x29cf116b),L_(0xd89fd3b6), LL(0xa037ebda,0x20362fb9),LL(0x17dff9f9,0xd010fe68),LL(0xb8350a9e,0xcbd44b83),L_(0x3adbe1f1), + LL(0x3d4cc86b,0x171138f4),LL(0xd002fd82,0x7c670c94),LL(0xed715f0d,0x14d543b9),L_(0x770fe818), LL(0x684456e3,0x2c123603),LL(0x3f8f258d,0xba0fe630),LL(0xa3564d33,0x9d216804),L_(0x435f77e5), + LL(0xa6e5dc69,0xabe6aeac),LL(0x5e84d762,0xde0a4f86),LL(0x538499e7,0x1d74cb44),L_(0xbfb3e36a), LL(0xbbcfa69a,0x198dddb7),LL(0xfcbe1984,0x873a7f1f),LL(0xb7b8bbcc,0x80cf7321),L_(0x00ac08de), + LL(0x78482416,0xe874537f),LL(0xbbb4692e,0x221e1e86),LL(0x02f17d20,0xe7057c36),L_(0xec132fea), LL(0x45b4351e,0x89b168f7),LL(0x7f9c9480,0x9d1de6f5),LL(0xf9e94168,0xf34e2409),L_(0x13e022d0), + LL(0xc854b8a6,0x243d2e96),LL(0x358d9af4,0x8938cb33),LL(0xb47467ae,0x81a7eea1),L_(0x470b2dab), LL(0x7063770a,0x3d8103c4),LL(0x9a2755c4,0xafa79d2a),LL(0x9228b07c,0x5b3e4741),L_(0xbc9b37bd), + LL(0xd005caa3,0x8739f51c),LL(0x3e9880c9,0x48357287),LL(0xe30595ad,0x9d3ff1f0),L_(0x4b286b55), LL(0xcc729699,0x89be0141),LL(0xe2576d49,0x27c29e5a),LL(0x00a64de8,0x1e63717f),L_(0x0c898bea), + LL(0x020255bf,0xd89a7c51),LL(0x489bd20b,0x1abc0a1a),LL(0xe7ac0a69,0xf9bbb011),L_(0xbe7ead5c), LL(0xfd43757d,0x5bf6715d),LL(0x2573c5cb,0x74bb4d9c),LL(0x30483451,0x7ccb3fa6),L_(0x0071ca44), + LL(0x4c06b933,0x83211263),LL(0xd01481a8,0x868c1469),LL(0x48c43d0d,0xd5277f78),L_(0x32c5572b), LL(0xd4224814,0xb34b30d1),LL(0x780e2016,0x750a0da3),LL(0x43e3806b,0xd4ad90ce),L_(0x97399684), + LL(0x31b3565e,0xe410e718),LL(0xb76d2e77,0x78c21e10),LL(0x0df8f583,0x6f7bfcdc),L_(0x210d7777), LL(0x740c2b91,0x04ef15f1),LL(0x87836a34,0x03a4d422),LL(0x44c267d1,0x2fbcd218),L_(0xbb549183), + LL(0xe76d9d5d,0xa541189c),LL(0x3029c3b6,0xd7480312),LL(0x277c1d19,0x761b9d9e),L_(0xd09fd429), LL(0x0b301fd2,0x8fe5cbdf),LL(0xf3cbac6c,0x42373c0c),LL(0xcbbe3e5d,0x4dd89b19),L_(0x75e821e8), + LL(0x9cbc0a32,0xf92173f5),LL(0x3b6ebbae,0xca2337d8),LL(0x44ad6eda,0x49918d50),L_(0x71e12775), LL(0xfc7a6248,0x7c6c9ea3),LL(0xa76fc6d0,0xe1e9ac5f),LL(0x95bb0f9e,0x0e5cdfb0),L_(0x3505e20d), + LL(0x1671e3f6,0xc0a26f33),LL(0xf36a466b,0x6a156434),LL(0x769bd94c,0x96b0b77f),L_(0xbaa9cde2), LL(0xec341956,0x961d12a2),LL(0xe383e51b,0xb1257ac5),LL(0xb680bdbd,0xfbf75271),L_(0x1f5805cf), + LL(0xa993566d,0x205ad8c6),LL(0x84ebf013,0x1874691d),LL(0xf24e1951,0x749aace4),L_(0xaaf03f06), LL(0x49610345,0x33c5f5b4),LL(0x378cf813,0x9fbc337a),LL(0x8ac53e4c,0x9b1441f3),L_(0x0113566b), + LL(0x2f5bdd05,0x8145a740),LL(0xd10f5321,0x525fb54c),LL(0xe73eb2a8,0x4da9ccbf),L_(0xf35011e6), LL(0xe92c65ec,0x375da80a),LL(0x3b493784,0xc35b0096),LL(0xe99fa1c1,0xbfcdd019),L_(0xfcd0fffb), + LL(0x3783ad2b,0x44724690),LL(0x6169a512,0xbbc2508f),LL(0xb54ae1fa,0x97426a68),L_(0x84a843c6), LL(0x1439b1e1,0x57ee4649),LL(0x9fa3b852,0x224d6e57),LL(0xcf4188dd,0xb9fbee93),L_(0xc6f126b9), + LL(0xa302b5dc,0xa34571c9),LL(0x48662048,0xb1b8b7f2),LL(0xfa26591d,0xf6f0b8d2),L_(0xc7f51cd9), LL(0x58d23928,0x5bc438ce),LL(0x66200c18,0x69768b41),LL(0xe6f1c95d,0x6844e3a4),L_(0x6e17cd42), + LL(0xcf358165,0xddd1ff7d),LL(0xf244b120,0x73e9c17f),LL(0xe6046902,0x40c26687),L_(0x1691fe67), LL(0x933ab02a,0x6e105d9c),LL(0x9b50eded,0x02d570b4),LL(0x00b6a88f,0xe9b08b55),L_(0x4b82631a), + LL(0xa09f3f50,0x04530c1e),LL(0xf4435606,0x5195314d),LL(0x9b79c17b,0x9f045501),L_(0x839380ae), LL(0x1cbc90d3,0x2d608e31),LL(0x9b83bfd2,0x913acbf4),LL(0x10db4136,0x22cc1949),L_(0x3a2e7962), + LL(0xd3e26092,0xec19a27b),LL(0xe5ad9b86,0xddac453d),LL(0xc7d0db50,0xb0a1794d),L_(0x12340a6f), LL(0x20e8dc2d,0x8bcb0707),LL(0x54438da3,0xe23756c5),LL(0x33597066,0x68e9b910),L_(0x192ee4ac), + LL(0x64c640f8,0x6b3b3e11),LL(0x6076299f,0xe52af985),LL(0x210c46f7,0x11bcefd8),L_(0x4c36d81d), LL(0x5bed674c,0xd49ab218),LL(0xb72e4c99,0x32664eea),LL(0x0e9348eb,0xa88c581f),L_(0xd67c4ee9), + LL(0x2d66a175,0xe899e96c),LL(0xb64bf9de,0x5fb2ed8e),LL(0x43b5e3af,0xa7cc5c88),L_(0x656b15b3), LL(0xdd7873f4,0xc5c4e24a),LL(0x112512c6,0x6324347c),LL(0x26ff024e,0xccd67ae6),L_(0x4f8c1dc8), + LL(0xe0a8b3e4,0x95b13b3d),LL(0x9e553300,0x85e3e1dc),LL(0x7fafd447,0xf8451257),L_(0x6a113d4a), LL(0x65687ab1,0xedde7f06),LL(0xaf37550f,0x901e09b8),LL(0x0b1d7c55,0x69e02d8f),L_(0xcfd6985a), + LL(0xe1246244,0xecd3e1aa),LL(0xbe482681,0xfb799ecf),LL(0x47798831,0x9fac48a4),L_(0xafbbf20e), LL(0x3d17bb47,0xd6f1c5bc),LL(0xfe2c6138,0x1b8d361a),LL(0xd3d7cf0a,0x0fdeccc7),L_(0xa05a2e1d), + LL(0x3a510fa4,0x6b3cd8ac),LL(0xa16dcddd,0xaee76b4c),LL(0x0fdf0fc4,0x3cb7db3e),L_(0x5144175e), LL(0x2669bf74,0xdaa284b0),LL(0xb41e11e0,0xa0a2a2a4),LL(0x334a53c5,0xb6f86d54),L_(0xea0526d6), + LL(0x142fbd3e,0xef18bf5c),LL(0x268ce667,0xc11a8ae3),LL(0x70f67534,0xb9cffb18),L_(0xc3038e8d), LL(0x941d9303,0x29b2c2c3),LL(0x4d1a3a20,0x4205ec1c),LL(0xe2c18665,0x5ab3157b),L_(0x7a60e7b3), + LL(0xf39b6495,0x1c5c3f1d),LL(0x28e72014,0x609a5ec7),LL(0x32068427,0x47c37b5e),L_(0x94c1b6e7), LL(0x39767baa,0x5f5b4e71),LL(0x77567891,0x05496c1a),LL(0x55a4f7f2,0xc6f93ff6),L_(0x8f654b8b), + LL(0xdb846749,0x8fa02752),LL(0xfcbae4b8,0xc21478c8),LL(0xcff78948,0x22632d48),L_(0x70d86a48), LL(0xa8b86e1b,0x00132e4f),LL(0x3721a7ea,0x0457eac2),LL(0xdcc0060a,0x0c2cbce2),L_(0x38fb42c3), + LL(0x802a7a9c,0x5ea4abc7),LL(0x5f3751e3,0x2aa4c525),LL(0xac364b4b,0x495dd3d3),L_(0xa6c1d838), LL(0x89d692f9,0xba68755b),LL(0x465b3970,0x72a043d5),LL(0xfe4e336e,0xfc16f7fe),L_(0x2c759722), + LL(0x3d83071b,0xdd3a0176),LL(0xcb5a7236,0x61809f55),LL(0xa2bbecb2,0x1f553745),L_(0x46d5deeb), LL(0x2602bcbb,0x4c542035),LL(0xa92075fc,0xc39e8592),LL(0xa99e29ec,0x31176d99),L_(0x7ca41c07), + LL(0xb54001ac,0x39ce1948),LL(0xbff084c9,0xe3b4d5e5),LL(0x43984570,0x05ef0ee6),L_(0x0a861937), LL(0xa8b0d940,0x1288a0c3),LL(0x52997092,0x3fa9ab6b),LL(0x24210cbf,0xec238c31),L_(0x9e2872c4), + LL(0x30423d94,0x40fd474a),LL(0x5b8538a3,0xec2ead24),LL(0x0d208047,0x1cb60ff4),L_(0x761298e8), LL(0x213bda9b,0x1b6d44ff),LL(0xbcf74593,0x5ddf4ac8),LL(0x746b2f76,0x679d4dee),L_(0x32233e28), + LL(0x1482adcb,0x0c718f19),LL(0x66cd9b3c,0x03bfd19e),LL(0x3b3a6f85,0xa9ef5cce),L_(0xca539a7c), LL(0xd847f363,0xed44fad7),LL(0x4db83c78,0xe9c8d39e),LL(0x5a9002ed,0x57ef09d7),L_(0x47b485ec), + LL(0x7d5b0115,0xe4537555),LL(0x4cc9c9c5,0x6d1d53a0),LL(0xec59f90b,0x55f05058),L_(0xaa97605c), LL(0xc10edab1,0x66f84f5d),LL(0x8c26f4dc,0x9097a136),LL(0xff1f32e7,0xda9cb1ce),L_(0x92d5f2c4), + LL(0x918a45ed,0x5ec4e734),LL(0xffb048ed,0x861a69b7),LL(0xb9ca6eb1,0x4f8b9e40),L_(0x089b9558), LL(0xd5b022cc,0x43e91e88),LL(0xcca4bf58,0xe5fa6721),LL(0x21a8592f,0x2180b560),L_(0xa9e4e373), + LL(0x73dd816a,0x95e834c2),LL(0x85c32c46,0x52b23dc2),LL(0xad18d469,0xd0c03244),L_(0xf46c7fff), LL(0xdbde0560,0xde2297b5),LL(0x8c32b174,0x8f0937ea),LL(0x28635d97,0x8b431269),L_(0x47d44429), + LL(0xc8d22f39,0x28c78e40),LL(0x93181781,0x61fdf274),LL(0xcb623c4d,0x98d40a79),L_(0xdda4b06c), LL(0xd808e325,0x3b688d0d),LL(0xfe3fd141,0xd51bdb7c),LL(0xedd15e12,0x3a965d08),L_(0xf3097f75), + LL(0xde0a2f39,0xbda9cc54),LL(0xa6bfe2fa,0xa29b11da),LL(0xf02a2f58,0x1b6a851d),L_(0xcb819b77), LL(0x8a57fd87,0x30780f8e),LL(0x9f0d56af,0x2b7ce89d),LL(0xf8a06fed,0x27c807cb),L_(0xb99605d9), + LL(0x46b40b76,0x6786d96f),LL(0x72255ae1,0x80ccf20e),LL(0xc29277a1,0xb9deb14f),L_(0xb01102d2), LL(0xaeb5c04b,0x1c8e7e81),LL(0xaf4a6e1a,0xaec9e1df),LL(0x06c802c5,0x4637b70a),L_(0x21f2e3c1), + LL(0x9ffc5f28,0xb4a7c608),LL(0x3a6619c7,0x4be0db0a),LL(0x5deb2377,0x24dd90ec),L_(0x7e94da01), LL(0x31617c3e,0x3788bd2b),LL(0x80b9433f,0x0c1368f4),LL(0xd49fdafb,0x47c8e8f1),L_(0x2be1c836), + LL(0x5ef0901d,0x7f036146),LL(0xf5f17b3f,0xdca93bb1),LL(0x45a7571b,0x4e68fb20),L_(0x68857bb8), LL(0x62732201,0xa24d2c8c),LL(0x2d13dee5,0xed2b0948),LL(0x88581796,0x118e784d),L_(0xf0493f53), + LL(0xa0e66eb2,0x58d860e3),LL(0x8971f724,0x9083bee2),LL(0xb4f43e7e,0xb5941f8a),L_(0xb537d4a0), LL(0xb975e31d,0xa327b762),LL(0x0a0d438e,0x9be00cfd),LL(0x23118e99,0x2345f880),L_(0xe40f8c30), + LL(0xe9728c3b,0xcbd96a38),LL(0x1197dd6b,0xe0af89d0),LL(0xb47615a0,0x381bf7e1),L_(0x251a8f8f), LL(0x008018e9,0xad578807),LL(0x34717cff,0x2b57e965),LL(0x5fbc83f2,0xa4f9d4fb),L_(0x64b6d093), + LL(0x69093077,0xf183d4de),LL(0xf6acf5ed,0xbaeb9a58),LL(0x0cc30970,0x181d2aa6),L_(0x53b37c4b), LL(0xce44dcb5,0x2ce916ba),LL(0x38654a56,0x59bee1db),LL(0xaba6d625,0x09ca53b2),L_(0xcdfe1968), + LL(0x91d466d2,0x9e6894a8),LL(0xf7c98ba1,0x7219e547),LL(0x8e28cba4,0xe28c0250),L_(0xdc432b47), LL(0x43c25678,0x52a05293),LL(0xff3120b7,0x789cbfc0),LL(0xd4520462,0x3107ac59),L_(0x92fc422a), + LL(0x41f9db9c,0x378db33c),LL(0x1b7e14c2,0x6dc570a1),LL(0x2e5d4aaf,0x536bff4a),L_(0x885f68ab), LL(0xf657a8f3,0xeaff53c3),LL(0x8ea881df,0xe2da9fbd),LL(0xce5d6df3,0x037450b2),L_(0x6899f186), + LL(0xa41f3714,0xe8227713),LL(0xb158373d,0xc20b72bd),LL(0x73e5ad11,0x0d1473e2),L_(0x963a9a91), LL(0xc3c89f91,0x6796b002),LL(0x32ad74a8,0x978ceeca),LL(0xcad42d90,0xa157eba3),L_(0x64fb791b), + LL(0x6d7b048c,0x91b7250b),LL(0xb6fc8bc6,0x1b7c37d7),LL(0x207504d2,0xa04f258e),L_(0x35b71223), LL(0x69296811,0x000189fb),LL(0x8cf221df,0xf5202462),LL(0xe9692259,0xc7e62afa),L_(0x2482c08d), + LL(0xb34b2712,0xa6fa3a44),LL(0xe579dd13,0x77c307c4),LL(0x8643df69,0x57d9dfb8),L_(0x9f01f782), LL(0x5573d8e5,0xb5c2c766),LL(0xbb840380,0xe0195f6a),LL(0x99e8ca50,0x6c698864),L_(0x31b4df34), + LL(0xc836d70e,0xfa87a978),LL(0x6e27995c,0x2d67fe21),LL(0xc806a202,0xc065a04b),L_(0x93cff90b), LL(0x6731446f,0xbb867036),LL(0x9cdfc841,0xe8066219),LL(0x0a935f1c,0xa9d96eb8),L_(0xc15bd4d8), + LL(0x60b97b73,0xd003f54a),LL(0xa4a49d7a,0x63dd3e3b),LL(0xa79d7693,0xa15f7dc6),L_(0xbd4d523d), LL(0x1e02bb50,0x6dbfe7b7),LL(0xcc88e7c3,0xb0a1bb28),LL(0x8264318f,0x5e5fd5b2),L_(0x5f2fa1e2), + LL(0xe76ef3c7,0xec0ebc09),LL(0x1936faf8,0xd297aff2),LL(0xc8c40d08,0x01dd06de),L_(0x54d85f2a), LL(0x2bc0d559,0x8e9d926c),LL(0xcbcea653,0x99783016),LL(0x37d07340,0x0beb45e4),L_(0x3de7fc81), + LL(0x564ba5a3,0x32058809),LL(0x9850a91e,0x3af0e1db),LL(0x5464c544,0x1452f874),L_(0xb9637ddd), LL(0xeb527a99,0x7298be18),LL(0xdf5911d2,0x16aa8506),LL(0xf6dfd8a8,0x8bb252ea),L_(0x839f8d56), + LL(0x89f63fb2,0xc7fe9490),LL(0xa2fc5db9,0x99c60e8f),LL(0x47b2b00a,0x142e5001),L_(0x557178b5), LL(0x081ac8ff,0x7281e45c),LL(0x80053d16,0x92d81832),LL(0x335bfb8b,0x3285f848),L_(0xe9e17269), + LL(0xdbe51600,0xd38c0683),LL(0xd98fa793,0x990e1e00),LL(0x14117a61,0x0d3b9c08),L_(0x3e3d6538), LL(0x7a4ddb24,0x374f82aa),LL(0x538b9bf1,0x0c5bcaf1),LL(0x1cc16fb8,0xe0c1bced),L_(0xc0de70fa), + LL(0xd35b0a61,0x96448e89),LL(0x98c48df5,0x8846f415),LL(0x50f58327,0x4770c98e),L_(0x197ced21), LL(0x5ff2d0d7,0x1af1733c),LL(0x2bca7ffe,0xa04c1cce),LL(0xeac8a367,0x13278f06),L_(0x5dd0431b), + LL(0xd19368fc,0xa83c536e),LL(0x02379421,0x118f2771),LL(0x218d25f6,0x90caeeec),L_(0xf254a0e4), LL(0xc0dc699c,0x44d824bf),LL(0x0bbdc5a2,0x5903a15f),LL(0x21933c71,0xec2c7cff),L_(0x0a756aae), + LL(0x97dfc53d,0xd8438b57),LL(0x1d02bb13,0x42584478),LL(0x25ef520b,0x3dd8667d),L_(0xe3e22c9e), LL(0xc5d6139c,0x0d51408e),LL(0x0227f2bc,0xd3e4dd71),LL(0x090e4a02,0x2fc23bc1),L_(0xf9171e03), + LL(0x11deaeb1,0x8d85409b),LL(0xd1184135,0x617f0460),LL(0xf357dae1,0x7b364c45),L_(0xb9d690d1), LL(0x4d915f34,0x0c5f601c),LL(0xcb872f3a,0xe0fe9ec0),LL(0xbccb4f0f,0xf7bfadef),L_(0x9f017f8b), + LL(0x83cff561,0x15442e1c),LL(0x62cd98e6,0xab89d9c9),LL(0x58dcba94,0x7426c5f8),L_(0x80d0a8cf), LL(0x449af58b,0x52648e23),LL(0x3a628b1d,0xdb61eac8),LL(0xf0844217,0x9c140c5f),L_(0xb3cc7bfa), +}, +/* digit=30 base_pwr=2^210 */ +{ + LL(0xd6e670a1,0xf0426be1),LL(0xb4f44241,0x14838573),LL(0xefc4394a,0x80845fb2),L_(0x90205f0f), LL(0x99c56ec6,0x8a6cba70),LL(0x4345224a,0x6149abf7),LL(0xfc80d60c,0x9c7703f2),L_(0x3d5a7643), + LL(0x730ebcfd,0xe4b299d2),LL(0x79702991,0x6cdbe6f3),LL(0xad58884b,0xff023a31),L_(0x18b5c6ce), LL(0x51fd8a3b,0x433c522f),LL(0x3e4907b4,0xc1579271),LL(0x3c1c0282,0xe4454779),L_(0xb4df9ee5), + LL(0x48953bf7,0x658675d7),LL(0xdf2f4ac2,0x8faf514a),LL(0xdde660e1,0x50213318),L_(0x64826f9f), LL(0x8433d7fa,0xadcc3e45),LL(0xbc26833e,0xcc5ad72a),LL(0xde547aae,0x619251e2),L_(0xf96f73d1), + LL(0x548b77dd,0xba3e8f2e),LL(0x728cadb5,0x21cbbb1a),LL(0x0feebd0a,0xdb740393),L_(0xdcffa1d6), LL(0xb6848db5,0xa742cd6f),LL(0x549a4fca,0x39f63442),LL(0x3d24aeeb,0x426c42e9),L_(0xb67d7f28), + LL(0x408e3341,0x96cc626e),LL(0xd2b3305b,0x6a9bb18a),LL(0x355d47d4,0x33f53993),L_(0x960f4207), LL(0x16218c03,0x329378d3),LL(0x1004c020,0x70de3eb8),LL(0xa9117cf1,0x0e5dbd85),L_(0x73f55c21), + LL(0xb9a63bdd,0x456357c9),LL(0x1b3ded00,0x8e077f94),LL(0x0f0f2d4f,0x51648f45),L_(0xab4e5da6), LL(0x3963e9df,0xe96f0202),LL(0xcb55e2cd,0x3b87f53b),LL(0x7b3cc4d5,0x0090498b),L_(0xdfb4de74), + LL(0x98927a32,0xba64358f),LL(0x757e6fbb,0xccb5e233),LL(0x95336360,0x50d4d37c),L_(0xac96f5a7), LL(0x93c38d36,0x53008bb8),LL(0x33869046,0x5e587a5b),LL(0xfb1262f2,0x1e3e57ec),L_(0x407af3f6), + LL(0xc458ba03,0x9d6ad457),LL(0x78e956e9,0x82640533),LL(0xb94af545,0x32977879),L_(0xd20f35a3), LL(0x6224b353,0x9c65ff21),LL(0x5c8f99f9,0x4a46fdf7),LL(0x4ce9a00e,0x307b3d8d),L_(0x798d2cf9), + LL(0x77b2fb3b,0x3b8e6ef5),LL(0xb6ca3273,0xe32b3b97),LL(0xe260b8d0,0x9c66c2a4),L_(0x05eb84cf), LL(0xaf957b78,0xbe1f3ecf),LL(0x1e35adca,0x9d69d314),LL(0xa175edc2,0x4681a0ae),L_(0xc6715ff6), + LL(0xee864b12,0xe9c88636),LL(0x32143c93,0xf0c1e0f3),LL(0x434818e2,0x25a929c9),L_(0x4c806ec9), LL(0x20527edc,0xfe3e80ac),LL(0xa0372526,0xd5fc27b6),LL(0x5279eb8a,0xee97419a),L_(0x98dbe31d), + LL(0xcbc47797,0xe8bb79d3),LL(0xa6d3938a,0xf225d84a),LL(0x0b2f88a1,0xf279df50),L_(0x59a54b2e), LL(0x8cd34576,0x1390a7ac),LL(0xc9c37e25,0x1130b1eb),LL(0x0777586f,0x3aa2516a),L_(0x42cece95), + LL(0x3964d2a4,0xa47e21c1),LL(0x5f9875c7,0xf918e47e),LL(0x2ecbc86c,0x4ac56066),L_(0x5ea26fc2), LL(0xa5a826e0,0x31e429d6),LL(0x944c3736,0xd7082b19),LL(0xcf2fc959,0x02c3927e),L_(0x3bb33cf9), + LL(0x27da5082,0xcf56d753),LL(0x64f98e4d,0xd40b530a),LL(0x1c7766cc,0xb20cfe07),L_(0xe351283a), LL(0x18e30244,0xdc9dc465),LL(0x3bee62a5,0xcccf7fb0),LL(0xb2e1f645,0xfe54699d),L_(0x18ee6ea1), + LL(0xdf71a0f1,0xd2fe136d),LL(0xe5f658a8,0x5a88e4dd),LL(0xd83b5473,0x94a1a0d2),L_(0x586d7529), LL(0x1d163bf2,0x0748fda6),LL(0xe98d9b7d,0x69880bc0),LL(0xf0def9e5,0xc4e12cd5),L_(0xf40402da), + LL(0x585fff04,0x2956c762),LL(0x257681d9,0x3ba8ab0b),LL(0x696e0262,0xfd84ba5b),L_(0x0675debf), LL(0xa783c064,0xef9b82a5),LL(0xe38868a1,0xdaa2e717),LL(0xcdcdf922,0x292fb9c9),L_(0x269d71d2), + LL(0x477cabed,0xb5fae58e),LL(0x55622b3a,0x6ba8e28a),LL(0xff23a08d,0x56c2b54e),L_(0x866ba05b), LL(0x037720d9,0xa737dae5),LL(0xdc5131fd,0xb93a1d0b),LL(0xbb1176e6,0x860b7b0d),L_(0xf51c76d2), + LL(0x5845cd10,0x6bd647f2),LL(0xe811642e,0x9a4283ed),LL(0x0776f557,0x2c9fb90d),L_(0xb8cebf02), LL(0xb1612f57,0x643fb865),LL(0xc4902b6c,0x88ea78f7),LL(0x74aac273,0xfd2878e7),L_(0x1a76b1b7), + LL(0xb6983e76,0xf609c0af),LL(0x0cfea42c,0x123027ca),LL(0x7a72c3c7,0xb9872dc7),L_(0x1ae5bcf6), LL(0x57a545e2,0x9fe44848),LL(0x2a90f4d7,0x24f2f0f2),LL(0xdcdd6bf2,0x32eb5ac2),L_(0x939219ad), + LL(0x2540ecf0,0x267059b6),LL(0x2669c8c4,0x6d717377),LL(0xb6ba656d,0xc80a4182),L_(0xa67b63ae), LL(0xac5cdbce,0x7acd503b),LL(0x98520b74,0xf8884f71),LL(0x0688ae94,0x47fc66da),L_(0xacf9786e), + LL(0x9b68b84a,0xde210bb3),LL(0x74430a09,0x98b4770a),LL(0xf0e810c7,0x85cefd4f),L_(0xba1cac0a), LL(0xe9e74a58,0x35194ff1),LL(0x6452e158,0x520f3a2b),LL(0x8ed85e25,0xd8535bc9),L_(0x3bc32918), + LL(0xd77f4a13,0xd3467c72),LL(0x3046a69b,0x3a350288),LL(0x996236b5,0xf02f1b90),L_(0xdbce524f), LL(0x3a7711d1,0xdcc369a6),LL(0xa2eda832,0x81b1b5d5),LL(0xcb12490a,0x33444c56),L_(0xb4685d00), + LL(0x69d66472,0x6fe5a181),LL(0xa316d37d,0x11b6d85b),LL(0x75ec9a2b,0x8b98fad0),L_(0x0c5e4465), LL(0xde810eaf,0xbd855a8d),LL(0x7661287b,0x436e58d0),LL(0x9b044bfa,0x0f23ddbb),L_(0x7abd77aa), + LL(0xc57c3228,0xe7d89f58),LL(0x490f55aa,0x2e88de2d),LL(0x3aa72314,0xdea3a145),L_(0x159c120c), LL(0x5f68084a,0x840bb946),LL(0x1516983e,0xdcbbb7b4),LL(0x527c0f73,0x0d4d39e1),L_(0xe3846304), + LL(0x1fed9837,0x56c53d5f),LL(0x8c3176ff,0x752789ed),LL(0x26489626,0x94c341e1),L_(0xa5b14d8d), LL(0x3cb481d4,0xb4f1f612),LL(0x1a2f793c,0x94eb82ca),LL(0x600ebda4,0x6ba99e2b),L_(0xd1571ee0), + LL(0x960bacd7,0x122b6639),LL(0x41a70d6d,0xbaf725d5),LL(0x50c3e831,0x37bdeb4c),L_(0x948ab46f), LL(0x7cc81d1e,0x1a55a55a),LL(0x3b20c1e6,0xa6276eba),LL(0x81e5763b,0x4ddc462f),L_(0xffa876e2), + LL(0x8d1558c7,0x62e4c32c),LL(0x7f945839,0xfc3257d8),LL(0xa1e798f7,0xfe755aec),L_(0x5524084e), LL(0xcebab559,0x02c2a9e2),LL(0xdc627b0e,0xf8a1716c),LL(0x7c827c47,0xbcf0b634),L_(0x677ad978), + LL(0xe3c947ec,0x74af562c),LL(0x8713ce0f,0x1493b8fd),LL(0x6951ea3a,0x23d2ab0f),L_(0x761f3a78), LL(0xb7530283,0x098fb6af),LL(0xeb48e791,0xccff2003),LL(0x160c9294,0xf1992cc6),L_(0x2ed7be4f), + LL(0xff03f6d9,0x02ad20b1),LL(0x30165084,0x17295d20),LL(0xb8bee43f,0x67757e6c),L_(0x0eedd6d6), LL(0x61ba2edd,0x5acf9123),LL(0x8dee8798,0x4442dc9a),LL(0xa4c9925a,0x9970bc21),L_(0x54e339cf), + LL(0x250b4244,0x7f335cdf),LL(0x5019bc77,0xa892b382),LL(0x15ce5f7d,0x4f31c43e),L_(0xfa69d995), LL(0xae609392,0x1d57d52c),LL(0x20dc7c66,0x6e6a7489),LL(0x01a82796,0xa8fa2376),L_(0x861ac30c), + LL(0xadcafee5,0x56acbf11),LL(0x7b20a0c7,0xa63a1d19),LL(0xbec3d69c,0x5e324e3c),L_(0xd3d97b66), LL(0x64b971b3,0xc585e140),LL(0x2b8e18cf,0xd08772ba),LL(0xe0b2513f,0x6e7d6c03),L_(0x7e1a6116), + LL(0x9bc4e6f4,0x23bf2ce8),LL(0x82122cee,0x0b5f01ee),LL(0x318ba204,0xa93e44d4),L_(0xd2180b18), LL(0xd6ef00ac,0xd9d85de8),LL(0xd7d06345,0x3062eb10),LL(0x5bea35e9,0x952d933c),L_(0x72bd41e8), + LL(0x72791646,0x60e9cb5e),LL(0x107232fb,0x644a5b6e),LL(0x004f8684,0xa23e4299),L_(0xefa3c56b), LL(0x964aa750,0xad436903),LL(0xbb576a94,0x041c2a91),LL(0xbd787460,0x3513e3fe),L_(0x4066ae12), + LL(0x09a3d049,0x627d1807),LL(0x7365dc29,0xe00d7aec),LL(0x0dec445a,0x96d5022f),L_(0xf4343e09), LL(0x515a9b8f,0x00d16dd6),LL(0xeeaa6e35,0x97242fab),LL(0x220f953e,0x91ecc397),L_(0xaf714cd0), + LL(0x79198e5a,0x6dd11272),LL(0xce83daf3,0x9169c8b8),LL(0x6ce5dec2,0xce89d045),L_(0xbafd94a5), LL(0x1563c0ce,0x577ca0e0),LL(0x1e36fd91,0x5a836e51),LL(0x060d3970,0x729ba81d),L_(0x8b7e4fbb), + LL(0x16781a85,0xdbaa195f),LL(0xb43de333,0xec92b939),LL(0x2ccf4a4e,0x036ff9cf),L_(0xafc236c4), LL(0x6221e156,0xd7468cbb),LL(0x9a645200,0xffcbfaac),LL(0x5b2deae6,0xb553572e),L_(0xd4efa381), + LL(0xd257958d,0xc9536b4a),LL(0x10222aaa,0x9ab6a135),LL(0x84934c38,0xdf7ca8e3),L_(0x5e5dc148), LL(0x93660c96,0x4d98778c),LL(0xb7c8140c,0xd69648d2),LL(0x69c09498,0xb0ea1d36),L_(0x7e0cd235), + LL(0x62e6becb,0xe9443a72),LL(0x94747e78,0xd494860b),LL(0x2415da80,0xf50cdd9d),L_(0x65d2fa97), LL(0x48dbba8d,0x3ca3104f),LL(0xaf39be74,0xbd695019),LL(0x0441383f,0x393e3175),L_(0x4e5bd1d4), + LL(0xf6d61041,0x70145555),LL(0xc463ec32,0x8fa5018e),LL(0x1e5a98cb,0x59b3adf7),L_(0xeb7c80fc), LL(0x529f5fa8,0x5234550b),LL(0x4eb610a4,0x2ac1098d),LL(0xdc8db9bb,0x52106057),L_(0xbc3a065e), + LL(0x62bb8d08,0x6e360f55),LL(0xee7f7e1c,0xe795bf91),LL(0x7411bd70,0x627a5d54),L_(0xb191903e), LL(0x323d2db5,0x6bb808ed),LL(0x484c498f,0xded0b935),LL(0x3cef8b7c,0xcc379eda),L_(0xe1f0ee29), + LL(0x106cde5b,0x21a0558f),LL(0x2ea40699,0x84c5579e),LL(0x057a8487,0x5834fcc4),L_(0xb35b630b), LL(0x6779af9f,0x2d1d0e72),LL(0x6560d0d6,0xd90a0248),LL(0xeb026a8b,0x4d572129),L_(0x720991c7), + LL(0x22ee217f,0x657c8db7),LL(0x5e4f7ccd,0x12f06bfb),LL(0x8351fe6c,0xa43443b2),L_(0x48c1ec10), LL(0x01489ccd,0xb376d6e2),LL(0xc77d6dfd,0x727638f2),LL(0x0f953513,0xeae9d0a1),L_(0x91a7a744), + LL(0xb85754e4,0x59454f27),LL(0xd2c2004d,0xe70796e3),LL(0xacc344aa,0x775a61d7),L_(0x1c186106), LL(0xa1656bbb,0xdcf00b60),LL(0x4e3f081c,0x1af1503d),LL(0xcaa9cad3,0x1e4e523b),L_(0x816a09ca), + LL(0x0b303a9c,0xd468c096),LL(0x86aa5ecf,0x1910f732),LL(0x26d08884,0xe43dffa8),L_(0x6501f291), LL(0xa597567a,0x88248cd9),LL(0x4711c711,0x5263d7c9),LL(0xa4245344,0xcfde7c18),L_(0x4111cd00), + LL(0x7d78f32c,0x0df3b4b0),LL(0x9cf9b276,0xf4191861),LL(0x9dd92d6a,0x6444967a),L_(0x5f3bbd8f), LL(0xa9ba118e,0x1a4b4820),LL(0x2d798278,0x2aa6dcc4),LL(0xe9681e55,0x0ce3d37c),L_(0x739d5ad5), + LL(0xdd02bc4d,0xcf20f024),LL(0xf4697953,0x90e93ff7),LL(0x789e6af7,0x1050c0c2),L_(0x383c22fa), LL(0x60720c41,0x5cc2adcc),LL(0x2ace7063,0xf57806de),LL(0xf6f6aaf0,0x31c60f20),L_(0x7b350e28), + LL(0xcc10306b,0xf49e5f61),LL(0x1a92eddc,0x58cb04a9),LL(0x365a5041,0xc89ba67e),L_(0xa32fc59f), LL(0x65f980db,0xdb9db26b),LL(0x09322a72,0x93261bc9),LL(0x3fda4217,0xc6c63b34),L_(0xddeb9986), + LL(0x148cbb74,0x77b280f1),LL(0xc18c7395,0x7ee57ca3),LL(0x4cd0382a,0x37436f93),L_(0x66528f8e), LL(0x6e4b7e57,0x2dd6704a),LL(0x83b283eb,0x7f7d547a),LL(0x54972ce8,0xbca75a4c),L_(0xa9f10506), + LL(0x7605a526,0xee53ff2e),LL(0x5105c866,0x09187c91),LL(0x02b0042e,0x1f9de3bb),L_(0x4f9500fd), LL(0x880380ec,0x8cb6dd5c),LL(0xaa04e960,0x2a74a12c),LL(0xa0112a70,0xec0e66ff),L_(0xec305363), + LL(0x4b9381fc,0xa014524b),LL(0x400fd142,0xcb74eb3a),LL(0xf02c64c2,0xd9684416),L_(0x8da612c7), LL(0x06a80727,0x4e54713b),LL(0x7cde46af,0xfe9ec578),LL(0x8380cc14,0x5013084d),L_(0x80963cc8), + LL(0x349f5079,0x6dcbbe48),LL(0x6b6125b6,0xceefada1),LL(0x67b840bc,0x16c79367),L_(0x570c02ad), LL(0x361a38dd,0x8552cefd),LL(0xa8167ca5,0x286e6e20),LL(0x3636e865,0x9c4305d9),L_(0x43d5af7e), + LL(0x7f9a80cd,0x413cd659),LL(0x9c95b00e,0x310a2a72),LL(0x64a28802,0x6ef01bfc),L_(0x858e4b79), LL(0xb1565b88,0x113a4fdb),LL(0xd390a92a,0xc04a736a),LL(0x774c1209,0x63557ff4),L_(0x54e47eaa), + LL(0x940daa93,0x27760c7d),LL(0x9e3147b9,0xd15d7a6a),LL(0xb9bb9711,0x89363a50),L_(0xe852edfe), LL(0x94740e26,0x43ee10a5),LL(0x2d6acef6,0xaa297031),LL(0x03fc12e9,0x20890385),L_(0x2d45580b), + LL(0x87453ec7,0x4101d16f),LL(0xbebb2e76,0xf89a5338),LL(0x533638ee,0x05bdd7f6),L_(0x92a0bd39), LL(0x56483d09,0x4c760096),LL(0x81c6243b,0x37294333),LL(0x5119eebb,0x049c6a06),L_(0xc410f246), + LL(0xd5e79013,0x51167f9a),LL(0x12be7d2f,0x90e63900),LL(0xa41ce391,0x0572301a),L_(0x0a761ede), LL(0x8bb599f2,0xe5dca064),LL(0x3dad4d6c,0x88bd9756),LL(0x82d548d7,0x79a33fd1),L_(0x76d2d77b), + LL(0x801eaf93,0x77f53014),LL(0xa4c2daab,0xd48faf97),LL(0x40a7d787,0x604bbbf3),L_(0xcc41bd91), LL(0x9de9fab8,0x4c6757d4),LL(0xe55da08f,0x9a37321e),LL(0xf50459ca,0xf8d3a03d),L_(0x8252e5cb), + LL(0xc0ad0a5d,0xaaca1be1),LL(0xb4f7cd9f,0x6f65ac78),LL(0xfae3835d,0xf685dd47),L_(0x6c5b897b), LL(0x8e631946,0x03e5ac2e),LL(0x71afc215,0x4dafffd8),LL(0xce6c64d0,0xa1bdc366),L_(0xe59eacca), + LL(0xcde53a1b,0x28e18006),LL(0x428bd7b1,0xec3e3023),LL(0x3bf0eb6c,0x69ce21e0),L_(0xfbe09c69), LL(0xc6ab638b,0xf7c3e582),LL(0x3283bc61,0xe88ef77b),LL(0x20ead3f8,0x1f935f24),L_(0x3b040b57), + LL(0x20008407,0xa23c1b84),LL(0x41a0017c,0xa6773559),LL(0xc09598e5,0xffd68469),L_(0x1da39b13), LL(0x221c586e,0xb19fcd20),LL(0xe71a0fa0,0xa76b6f70),LL(0x83e812db,0xb605b69d),L_(0x0efdfc84), + LL(0x28c66881,0x656d6dd0),LL(0x58a3f7d2,0x7540ad09),LL(0x1396fe6c,0x52fddb17),L_(0x39edad21), LL(0xa2cc1fe5,0xfb57f569),LL(0x1072b4e0,0xddfbbf25),LL(0x94372dad,0x0a26aff2),L_(0x88f38fd7), + LL(0x8fdfcf33,0x6d4895a3),LL(0xff2438e1,0x19da7037),LL(0x67f22bef,0x3cfaa48e),L_(0x6bf9fc2b), LL(0x0a03dd57,0x7b4df30e),LL(0x956e9ec2,0xd632b168),LL(0xe6cb0b1c,0xfc98eaec),L_(0x7dbf18a3), + LL(0x8b21bd41,0xf20afcc8),LL(0x9b4028ed,0x9fb97c45),LL(0x4e476a55,0x6bf05b58),L_(0x2b0c488d), LL(0xec47e2f5,0x01d3ca74),LL(0xc0d6bd28,0x7a3a23e0),LL(0x95f353c8,0x010298ee),L_(0x034e53ca), + LL(0x6fa3f957,0x80ff52bb),LL(0x50412d11,0xf58be861),LL(0xbe2f9f7b,0xe84a58ad),L_(0x83284261), LL(0xa5a7f060,0xc605b12b),LL(0x5da82b44,0x86fa207e),LL(0x837f9c22,0x10d656cc),L_(0x454d1413), + LL(0xb270f308,0xde3d1dc4),LL(0x6619a39c,0xf9fa04cf),LL(0xaaa9723b,0x3910ae67),L_(0x86fdb715), LL(0x1046b659,0x3782c29a),LL(0x5c35eb01,0xd10183f6),LL(0x76b4be96,0xdd570ea6),L_(0xf56b5810), + LL(0x237ac7ff,0x9ec12515),LL(0x3f080f23,0xed07ceb7),LL(0xc3509a49,0x08bba00b),L_(0x07998613), LL(0xd7bef81a,0x5c36ecb2),LL(0x75c6840a,0x9bd57323),LL(0x3e8f5265,0x2c678e3c),L_(0xc7f1cc73), +}, +/* digit=31 base_pwr=2^217 */ +{ + LL(0x0993fc0d,0x2409728c),LL(0x2d5fd352,0x7af57f46),LL(0x28f3cb14,0xee72bd56),L_(0x21c2f494), LL(0xf22e2fef,0xac73118d),LL(0x283e1b56,0x490bb371),LL(0xf00c3802,0xaa6c7b0a),L_(0xee304f23), + LL(0x2619d224,0x5b89b52b),LL(0x433d8d17,0x1df61e0b),LL(0xec27bd08,0x49353c25),L_(0xf18b23f5), LL(0xc525714f,0x2c798eec),LL(0x91011438,0x6bd787c1),LL(0xfaa77bac,0x6e2db36a),L_(0x2a3b730b), + LL(0xf5bd48f0,0xea09f154),LL(0x18c36c89,0x45e66f63),LL(0xff1cf2a0,0x1d069cd7),L_(0x672fde0f), LL(0x07bd9cc6,0xf1f4e5b9),LL(0x86238146,0xa40cdc9d),LL(0xcdc7fd6b,0x00bcd404),L_(0x51a105eb), + LL(0xd8d6b178,0x348659fa),LL(0x0afc95de,0x724963c6),LL(0x478c3432,0x1f2f8d2f),L_(0xaa0e8781), LL(0x49e60cff,0xbf3811fe),LL(0x526d1fb9,0x50f264cd),LL(0xf802b4ae,0xf613e178),L_(0xd3df21bc), + LL(0x45bfb8c0,0x40af4179),LL(0x3b133f03,0x6e8997cf),LL(0x9b5dda1d,0xc2f372b3),L_(0x3948b7d7), LL(0x55f88793,0x99028593),LL(0xbcf7e27f,0x405ba3c5),LL(0x1de8aa5f,0xb2fc7cd3),L_(0xaadbe95a), + LL(0xd00dadb4,0xff2c8be9),LL(0x26721264,0x8d199a29),LL(0xbd6c65e6,0x7f2093a5),L_(0x21e5d488), LL(0x03e74b8c,0xa000a3ec),LL(0x68530945,0xeb8e9efa),LL(0xd1f6fc3c,0x3f15aab2),L_(0x9b5d8cbb), + LL(0xb9fa3849,0x45f74205),LL(0x97a4da7b,0x121a4ca8),LL(0x6d246547,0x819c4b2d),L_(0x8255bc16), LL(0x156c1f2a,0xf9d396e1),LL(0x5502b719,0xe8758b6d),LL(0x8b9b6ca8,0x94965c57),L_(0x3c7a5ffc), + LL(0x75175beb,0xc11d6984),LL(0xbad3d4ba,0x3c966d66),LL(0x67c5bea4,0xb82cec9b),L_(0x93b6322c), LL(0x1f09f72e,0x9661c046),LL(0x769c204a,0x12a03826),LL(0x35880fdc,0xc215f2cd),L_(0x4f5bf77f), + LL(0x4fdd677c,0x8bce7ecf),LL(0x847b9ee6,0x62abc7d2),LL(0x8cb1f02b,0xfffe0edc),L_(0x62c22364), LL(0x48727496,0xdcd7d51d),LL(0xa9cd4c50,0xfe1661b1),LL(0x30edb5dc,0x82ea29e2),L_(0xcc4403d4), + LL(0xc5db9960,0xed02ad3a),LL(0x66e7f904,0x194f3505),LL(0x2877668b,0xe7bcfa22),L_(0x873070fa), LL(0xf3636bf8,0xf013615b),LL(0x2fe9dfb1,0xeec498a2),LL(0x51ff974b,0x56961e83),L_(0xcd87825f), + LL(0xb4807fe6,0x5b989bd9),LL(0x2f915297,0x161be465),LL(0xcf0387e7,0x80e7dd9f),L_(0x2e7c6dd7), LL(0xb221d5c4,0x861c2ffd),LL(0x1df12d2f,0x36db9960),LL(0x048668bc,0x9b862a1d),L_(0x2b64be2a), + LL(0x02a0d3cc,0x644173f7),LL(0xc3611806,0x2103668f),LL(0x78a6d7ae,0xe22b26dd),L_(0x1dd89345), LL(0x39193a47,0xd1587982),LL(0x7692c1c8,0x9106104e),LL(0xecf6b1c4,0x0a975fb0),L_(0x0d6babd5), + LL(0xf550f1a0,0x4222b0ec),LL(0xb5c91a5b,0x78338ceb),LL(0x0ec3e064,0x9d0d2af2),L_(0x660e553e), LL(0x1d9a4d16,0xa632918f),LL(0xff1f387c,0x7bd6a5fc),LL(0xd795c509,0x67e40e1b),L_(0xdada8e3d), + LL(0x7f1d2d28,0x6eb26316),LL(0x4e1593ef,0x93bd25dd),LL(0x8698f986,0x2d0a715c),L_(0x8e8b3499), LL(0x41550af3,0x0b43672a),LL(0x132c1c9a,0xad35a8ca),LL(0xfd973661,0xea97e0b3),L_(0x3b811a56), + LL(0x87e35efe,0x3edb12df),LL(0xf66872e6,0x31c81b67),LL(0xc41cad87,0x0cfa43fd),L_(0x43f4aa49), LL(0x209b0e7b,0xcfa7ab7a),LL(0xafd31516,0xe582ff27),LL(0xc78d7ba1,0xea786f69),L_(0x2090d8fa), + LL(0x073b3ba4,0x7ee5008f),LL(0x28c86852,0xf81d75be),LL(0xd457685a,0xc2987502),L_(0xe596765e), LL(0xd0f9d913,0xfb631b84),LL(0x42aaadd8,0xe5cce6ba),LL(0xf7d72923,0x2257f5b6),L_(0x260ae885), + LL(0x6ca309b3,0xdb84c9b9),LL(0xdbb13e44,0x0f9da41e),LL(0xa7294694,0xc1443c26),L_(0xe02af6c2), LL(0xfe44c5f2,0x293f667a),LL(0x52bbc8cd,0x5f04fc4a),LL(0x7e1de72e,0x6907cdc1),L_(0xf8a9aeab), + LL(0xa3dd55e7,0x3fade182),LL(0xd7401dde,0x03aa0312),LL(0xc85449b7,0xc47549c9),L_(0x6ec51217), LL(0x55e72225,0xae146938),LL(0x4bbe9886,0x7c810544),LL(0xee349410,0x9f320ada),L_(0x4fdf9e41), + LL(0x2f1465cb,0xf506f849),LL(0xf58881e9,0xb38fe4e3),LL(0xd6feb622,0xedfd3c65),L_(0xb163b10b), LL(0x1345454b,0xb8f8a7bb),LL(0xb10b02be,0x11df874d),LL(0x1e5816ca,0x0ad84066),L_(0x9cd4eb66), + LL(0x4f8d4868,0x413431bb),LL(0x9ea4aaec,0xd1ee0d6b),LL(0x135f5f2b,0x9e4519f3),L_(0xd3e6e8f7), LL(0x7803a8e3,0xfa8179f8),LL(0x318fb330,0x53800116),LL(0xe0cb106b,0x95e1ca87),L_(0xfb4ce48c), + LL(0xec15776f,0xf3d74e54),LL(0xb7fb663d,0x4fd65071),LL(0x6706a48f,0x7f8a7cfb),L_(0x52d7353d), LL(0x811679af,0xc42a9ef3),LL(0x5bd192eb,0x5ed520f8),LL(0x2b1ff6c1,0xaa53702a),L_(0x5388fa6f), + LL(0x2813a7fe,0x423e53a3),LL(0x0baae890,0x3d22e91f),LL(0x08234191,0x9bfec6fd),L_(0xc8f4d373), LL(0xfbdbc5c1,0x119e3476),LL(0x19587b51,0xfeed51b0),LL(0x1c276e15,0x951dd485),L_(0x98048932), + LL(0x21e06022,0xbe2d77e9),LL(0x67d742de,0x0c1663f5),LL(0x7983fd13,0x7115e508),L_(0xb9406bb8), LL(0x4eba1a04,0xe45eb297),LL(0xdf3a7991,0x5693a9fc),LL(0x03ba9fe5,0x25d8f862),L_(0x6ea3053f), + LL(0x93e92c0c,0xefbe9105),LL(0x266e3900,0x1cee1814),LL(0xe042e34f,0xbec6b919),L_(0x5cb15a6f), LL(0xd41a287a,0x7568673f),LL(0xfe7b7595,0x4deaa29d),LL(0x3e67572e,0x2d437dfd),L_(0x03299218), + LL(0xfea564c7,0xc2d6c313),LL(0x3d460901,0x3e809da5),LL(0x591fe51a,0xeb85aa17),L_(0x44d65538), LL(0x45dff18c,0xea596f90),LL(0x48a2f0df,0x4708877e),LL(0xc2cabfd0,0x17f7dae8),L_(0x31a2b7ff), + LL(0x21d1b602,0xb389e984),LL(0xb19669c8,0x806cbce5),LL(0x729a4109,0x53062f9c),L_(0x779cbefe), LL(0x3bd3b5fd,0xa29b29ca),LL(0x6119b4c5,0xd386838e),LL(0xeb10107e,0x91c7a000),L_(0xd43d23aa), + LL(0xa2b87da8,0xb2465acd),LL(0x954d5e2a,0x935ce923),LL(0xcc9f7e66,0xa5432c5d),L_(0xd4ffbeb9), LL(0x02a952ef,0x686dc22d),LL(0xb3aa6ec4,0x272f791c),LL(0xf9a59b07,0xd8d7140f),L_(0x94e1344f), + LL(0x63bd0826,0x7dd1f80d),LL(0x241f38ad,0x61111bf9),LL(0x72ffb980,0xff97f61b),L_(0xa3d97bc2), LL(0x29de7bfe,0xd4e6e73a),LL(0xb4b19841,0xf25ef407),LL(0x5fd346b8,0xa14d6ec3),L_(0x366b72b2), + LL(0x700b11e0,0xb6fe5b75),LL(0xe3a6e8d3,0xf936e40c),LL(0x4ed1cc1b,0x9f69a8a4),L_(0xe6f9dbe8), LL(0xf722e6ee,0xa4d858ef),LL(0x00a65edd,0x39a6e0aa),LL(0x9733fc57,0xd4ab5744),L_(0x38f910bb), + LL(0xf3acc856,0x338c1ad4),LL(0x4a664d78,0x9d2c02fd),LL(0xd6ec4c42,0xaee2829b),L_(0xededd826), LL(0xb9a4651d,0x417e0805),LL(0x2c83b6ff,0x1f17faef),LL(0x65e8562a,0x54ffde80),L_(0x8972a5fc), + LL(0x639320ae,0x49c2561a),LL(0xd2b48b48,0x05a4bc45),LL(0x38241d91,0x7dc5afe2),L_(0xc399df03), LL(0x01fe9bae,0xc98d8a83),LL(0x666e5386,0xb711940d),LL(0x33895285,0xb43f7afa),L_(0xb804c149), + LL(0x4d95806d,0xd0681518),LL(0x93423c26,0x537054cb),LL(0xc2eabb37,0xea12b99c),L_(0x9297a177), LL(0x791e1723,0xddfee3a2),LL(0x4de65e15,0x21639c41),LL(0x7f3bad81,0xe43ae1fc),L_(0x7af9ba54), + LL(0x1981ac7c,0x13762e73),LL(0x5afe2e21,0x76679941),LL(0xa21b9a1b,0xd03deaed),L_(0xd498e9f2), LL(0xc88d2fa4,0x6ae69ee8),LL(0x749eed42,0x14c5a38c),LL(0x5f4c63df,0x95aa2f5a),L_(0x890e5e3e), + LL(0xda52bcab,0xdea7e049),LL(0xbf3bb703,0x6a4a37e1),LL(0xc64a209c,0xddb56124),L_(0x6858318b), LL(0xb063f261,0x9e3a0203),LL(0xf0dbd02b,0xf2de2a02),LL(0xa216442b,0x5ebba836),L_(0xc0cbf7a0), + LL(0xc6f7be46,0x9bdf3b81),LL(0xe5bfa2bd,0x471a44c8),LL(0xef3da843,0x7b3913de),L_(0x107bba5b), LL(0xd5502ac9,0xda5362cd),LL(0x81eecff8,0x51e4974d),LL(0x151060a4,0xd9ea79af),L_(0xe89b5db6), + LL(0x176cf8de,0x310d44fc),LL(0x4509feca,0x8d199332),LL(0x25aa7254,0xd40c4268),L_(0xa958ef77), LL(0x4c15f926,0x5dae221b),LL(0x71fc93e9,0xb47feada),LL(0xc3e3cb95,0x0f45640e),L_(0x6b7cb33b), + LL(0x2033fbaf,0xc74fe8dc),LL(0x8c9ef406,0x8a528a60),LL(0xb6fe758c,0x04aa9cd6),L_(0x4f8b3877), LL(0xbbebde8e,0x9f568191),LL(0x65aed657,0x88ffeed7),LL(0x85d0ebdf,0x7968220e),L_(0x04b5f3d4), + LL(0x2004c53a,0x275a3075),LL(0x3a2a339d,0x686f881e),LL(0x8a7f68a5,0x5e03cf06),L_(0xd663fea8), LL(0x5c2f3d32,0x83dd1968),LL(0x6151157a,0xbccda799),LL(0x620979ba,0x6d82e1aa),L_(0x3ac652d1), + LL(0x4bd23c39,0x9f8e323d),LL(0xed2fa4ec,0xaa59f591),LL(0x554cf5e9,0x14820bd1),L_(0xc72e2e81), LL(0x9ce69d25,0xe15fa1bd),LL(0x5f0c938d,0x6c3a0be6),LL(0xdf2d997a,0x33617c0a),L_(0x3087e812), + LL(0x5df1d007,0x7f81aff5),LL(0xd6ea063d,0xbe630797),LL(0x631cf97e,0x46b66ca3),L_(0x3bfc14a4), LL(0xd751c657,0x3a379a18),LL(0xaa5dfd05,0x87fb3d3b),LL(0xf3af9e8f,0x95b8ff29),L_(0x928c3c53), + LL(0xcfeb637a,0xe65a2e0c),LL(0xa11c784f,0xe52dcc1f),LL(0xd33e59b1,0x2f4c50c4),L_(0xe15fe3e7), LL(0x8e0494e2,0xe61d0a02),LL(0xdff24b95,0xa0c563a3),LL(0xd0e84cbb,0xc9c516d5),L_(0xbc6f5dd3), + LL(0x6f4d5a61,0x828d8489),LL(0x55c5a959,0xe0f2a11a),LL(0xed8f7fcb,0x2be30973),L_(0x0b24dc92), LL(0x835ad623,0xc98877f5),LL(0xd599613c,0x62b30316),LL(0x43a0bc0b,0x34b1b90a),L_(0xa8edd1ca), + LL(0x25bb0089,0x126c6d31),LL(0xb92a7bd7,0xb4a63c35),LL(0xd9952b28,0xb5f3accf),L_(0xd21fa7b1), LL(0x470e5d4b,0x6ac1deda),LL(0x7d8ef38b,0x7ddc00b8),LL(0x26d207ac,0xc01bf44b),L_(0x58c8cee8), + LL(0x43e06d65,0x5d2a899b),LL(0x94d31b36,0x787531d0),LL(0x3050256d,0xd6d74149),L_(0xaa4efe0c), LL(0xa5994d61,0x9b7a733e),LL(0x491dc4d2,0xab4d4ef7),LL(0x7540af9a,0x358ed8d5),L_(0x17dc58f7), + LL(0x2cf09935,0xb13a4f1f),LL(0x4d20428f,0xaa77cf17),LL(0xf17077d7,0x27509ac6),L_(0x58852489), LL(0xdeaa5893,0x64ad83cf),LL(0xc965f9f3,0xeb7c2c1c),LL(0x9a3903b0,0x2677b45a),L_(0x45839bea), + LL(0x1b4a3efd,0x6c1eb10c),LL(0x24b30b08,0x30469bc0),LL(0x44f85d04,0x688bbe3b),L_(0xc74ccccb), LL(0x40ec199e,0x94d1ce02),LL(0x14d9a43c,0x1e8a57b2),LL(0xc940698d,0xf3a40155),L_(0xefe2f127), + LL(0x7ced76b7,0x9378bdc8),LL(0xfbd86f1e,0xd3de6697),LL(0x38c7a9cc,0x965226a4),L_(0x434dbb0c), LL(0x1d9de00e,0xb9259728),LL(0xc36bf7c6,0xe5e2b0b4),LL(0x0a12f12b,0xb21834ca),L_(0x0134eeee), + LL(0x2cbf4902,0xad55e742),LL(0xa30d570d,0xceb87a93),LL(0xdaaa178c,0x525f0059),L_(0xf38af4e6), LL(0xf041ead7,0xa001bb9f),LL(0xa05f60e2,0x951d1144),LL(0x701418ea,0xfd38f820),L_(0x27e433c5), + LL(0x4f2b74e1,0xcf1f6417),LL(0x3570870f,0x2d368dff),LL(0x85875e05,0x96e8ab61),L_(0xefc6f006), LL(0x139ced83,0x0206e9aa),LL(0x6f6540da,0x1f1ac51c),LL(0x2a1b80e0,0x9caa872e),L_(0x895193a8), + LL(0xbe7a3924,0x40d4de6c),LL(0x288f99a9,0xa9bbbd0c),LL(0x74a2cf89,0xad2e6432),L_(0x52ddf73d), LL(0x50719c7b,0xef978612),LL(0xbb41a387,0x44b2c373),LL(0x319f4bf7,0x40e92907),L_(0x9884b224), + LL(0xcb2e3230,0xe2fde7bc),LL(0x282179db,0xb286e49e),LL(0x7897f73f,0x14a26c02),L_(0x07ab717c), LL(0x0974c087,0xb3e012dd),LL(0xa65b35a3,0xb211a759),LL(0xb888ddca,0xc0879433),L_(0x8819e222), + LL(0xa98ed0e0,0x32acb172),LL(0x9f902cc1,0x30fb222b),LL(0x2ea4f419,0xec378b71),L_(0xd036aa23), LL(0xfc7c459e,0x0f8fe474),LL(0x3e16c057,0xea879818),LL(0x134bd529,0xee28b685),L_(0x867d1b08), + LL(0x7f52225a,0x49cdbae9),LL(0x5c1f02d0,0x22253ab0),LL(0xd83d7d9e,0x1b0a0f31),L_(0x2c287cd4), LL(0xa7c8e376,0xf27c6df7),LL(0x74534873,0xb5e78384),LL(0x5a696a14,0x8118c158),L_(0xfc06292e), + LL(0xa94b2f8a,0x5e909156),LL(0xbd1a69f7,0x0e89f809),LL(0x703cb9be,0x5a474912),L_(0xcedd645a), LL(0xfc513c30,0xdeeac8ab),LL(0x2ed53772,0x8d081de8),LL(0x5e6b86ab,0xc7b731bc),L_(0x5ce1f8a0), + LL(0xa68e3ae6,0xcc023bb5),LL(0xdf3b4492,0xd2e02e8e),LL(0x87e45b7d,0x0ae46dae),L_(0xe1f7eac7), LL(0xe31f07c6,0xb2f3733a),LL(0x72718cd2,0x58504950),LL(0x85e9b624,0x626b0871),L_(0xc20aa08f), + LL(0x3383e911,0x44129030),LL(0x8b7e591b,0x1fa18b39),LL(0xc7753dad,0xf476b49f),L_(0x2fc4d6c3), LL(0x43355b4f,0xe26efb18),LL(0x4dd384fc,0xb345e6e0),LL(0x478d3761,0xc574295c),L_(0x421851ef), + LL(0xc576e693,0x92253e01),LL(0x5a61d65d,0xa9b92c8f),LL(0xd4924563,0x1e712886),L_(0x2dd1532f), LL(0x502854fc,0xa3f1ded9),LL(0x39cdfbfc,0xb3758c76),LL(0xb32cde92,0x57960c59),L_(0x40ad0b76), + LL(0x7c7eb74a,0x33392cdf),LL(0x52977c85,0x3035bc66),LL(0x147af937,0x6dd913b5),L_(0x4ec4b1c5), LL(0x3cd27826,0x699ab1e8),LL(0x438d69df,0x674459d8),LL(0xdf94b6a0,0xbc60bb02),L_(0x60dae203), + LL(0x98237ec1,0x01cf54f3),LL(0xc3cf28be,0xc86cd9e7),LL(0xfa448ad2,0xf8c1cf70),L_(0x59baadb3), LL(0x6e0c6ab4,0xb0a595fa),LL(0x60b75e6c,0xaef06103),LL(0xb92101cb,0x99833d0b),L_(0x4be8d444), + LL(0x76a063e0,0xedd8ac9f),LL(0xcb571bbf,0xfbb746f6),LL(0x5a13acc6,0x68203714),L_(0x79914deb), LL(0x698725d3,0x5c3bbaeb),LL(0x211e58cd,0x8019d5ed),LL(0xdef6ac5e,0x776cb363),L_(0x93cb4804), + LL(0x2c9e2d1e,0x7d49d5cb),LL(0x55e82125,0xb44a7110),LL(0xa13cee20,0x7e46156a),L_(0x7d7e5719), LL(0x293b532f,0x629c9b2f),LL(0x5fd28ac1,0x4766f7bb),LL(0xdc393f35,0x8ea6f6e3),L_(0x289c68b1), + LL(0xf29e017b,0x63596f57),LL(0x31f49f27,0xb9a200a0),LL(0x7c86680a,0x0b1f95fd),L_(0x02fc0e10), LL(0x61a4ad54,0x8f2843cd),LL(0xca4cab38,0x5392a44e),LL(0xace069cf,0xf56c80ff),L_(0x270d9677), + LL(0x8e969ba3,0x6352c6f5),LL(0x67e09224,0xd3d5c18a),LL(0x5be06f9a,0x14efc35d),L_(0x38c6a0a2), LL(0x0263394a,0xbe2d9aa3),LL(0x6d0b11a9,0x72815d01),LL(0x6c28404f,0x5538c4c6),L_(0x0af532d8), + LL(0x27afc27a,0x588f5f00),LL(0xb4277028,0x7fd91a63),LL(0x2477f29e,0x5fe5d31d),L_(0x68c2db2a), LL(0x3c6bab6a,0x2a5ebb99),LL(0x7b9c3f49,0x8847424d),LL(0xbf734579,0xbf266e3b),L_(0xfe65a7a9), +}, +/* digit=32 base_pwr=2^224 */ +{ + LL(0x390f347f,0xb7478766),LL(0xeaa013ed,0x69774be3),LL(0x1cf3e562,0xe8c99b44),L_(0x142c4251), LL(0x6addc55a,0x5ac149fb),LL(0x9c824754,0x251be20c),LL(0x6b1233b2,0xb9c09eeb),L_(0xc5afba96), + LL(0x772d2e43,0xb7c40ccd),LL(0xbf4d6687,0x4a5ac532),LL(0x88025f52,0x80c953d4),L_(0xe3dc6d83), LL(0xa265b97a,0xb7668ae2),LL(0x50faab59,0xa19f0872),LL(0x235fc5b5,0xac299c46),L_(0x1378e170), + LL(0xe16661d5,0xb43ccb67),LL(0x47792f74,0x13fe8d1f),LL(0x3dbeda1c,0xb688499b),L_(0x5b9b5ea1), LL(0x6ec4def3,0x6f26a9f6),LL(0xc4515542,0x45056f3c),LL(0x8e4ebca1,0x1f557c40),L_(0xd6441259), + LL(0xa24e4556,0xea6ea3c9),LL(0xc6c1108f,0xb4010098),LL(0xcc83bf8e,0x2c30408f),L_(0x7aa17b0c), LL(0x89219d24,0x06aed6af),LL(0xc003650d,0xbe94120f),LL(0x72d853fe,0x66238f2e),L_(0xf46a15a7), + LL(0xf0756f21,0xf16a11ac),LL(0xccc9dc6a,0xc74dff4d),LL(0x71ab7d63,0x54c72750),L_(0x5a001c63), LL(0xcf7f1e0d,0x226cb517),LL(0xb6f26d84,0x09440d66),LL(0x3fc36947,0x9f9ee30a),L_(0xa1549b9d), + LL(0x9d51f064,0x4a2b2246),LL(0xa65e1e3b,0xfddd99d1),LL(0x671cf2f3,0x6b9d08f0),L_(0x162efd24), LL(0x4fa36b3d,0x3316fc3e),LL(0x84ae1987,0xd5adf51b),LL(0x7d0dc886,0xdbe094f5),L_(0xc7b0730d), + LL(0xb0e743f2,0x1bb62b05),LL(0xe8e8411e,0x6fd286a0),LL(0xd192f8c2,0xe45573f0),L_(0xc1f3a5f2), LL(0x96b5af59,0x26419174),LL(0x6c1325f6,0x50d9352d),LL(0x2f81e001,0xa91d228e),L_(0x0ff7a9b2), + LL(0x1a7e3bbc,0x88e39d85),LL(0x42262241,0xc7cb4750),LL(0x3ad07ade,0x918fdfa4),L_(0xcb771a46), LL(0xd39e0dbc,0x18ba36b7),LL(0xeb29d84d,0x3f5d4732),LL(0x1383d57f,0xf57833f6),L_(0xfda5e179), + LL(0x024fa042,0x47af4689),LL(0x6f69c44b,0xe8d6a5c3),LL(0x5e4d4fa0,0xd6821d55),L_(0x46eb8384), LL(0xc59ebc7b,0xcdeb889d),LL(0x871d5809,0xeaf69758),LL(0x1029a1da,0x5cf9245d),L_(0x1bcc2546), + LL(0x491af658,0xfd8a4d1e),LL(0xdb1de26b,0x677407f1),LL(0x8aa6deb0,0x12fee6da),L_(0x1e010832), LL(0xeb1d6fd4,0xd9702487),LL(0x9e928c81,0x0d5e0508),LL(0x79014b3a,0x844adfba),L_(0xa7623853), + LL(0x6333a497,0x2e0124e6),LL(0x2b6dbd37,0xacc3158a),LL(0xcb4fc42b,0xd7cdda0b),L_(0xd5d8f543), LL(0x1a1d4227,0x485584ae),LL(0xdd6eb7f3,0xa25706fd),LL(0x2e41e3f8,0x65859cc9),L_(0xeca30c9e), + LL(0xef4c8e5d,0xd5f8805d),LL(0x66979e6c,0xc2437937),LL(0x1b2ea5ec,0xf8ebb830),L_(0x416a9eb2), LL(0x87c226a3,0x6e8ad20c),LL(0x15ff6e9d,0x784bfc8d),LL(0x0865593f,0xee384b9b),L_(0xe0eeda2c), + LL(0x148e4a75,0x642f55af),LL(0xbfeb8487,0xc162ad9e),LL(0xfa3ad249,0x7145f39b),L_(0xd0f1abf1), LL(0x4053c61b,0x4defbbd5),LL(0x893a0505,0x1244854d),LL(0xf3e2f26d,0xf54c27a7),L_(0x6f12b38f), + LL(0xa1ac9684,0x5d26514a),LL(0x7cad1aa3,0x3b5e65fa),LL(0x74faf86d,0xfeab4314),L_(0x645cca78), LL(0x3c8ed8d4,0x43f00055),LL(0xb72096cb,0x8df847ba),LL(0x7a1950ed,0x071d954a),L_(0x5337fbf2), + LL(0x33140bfb,0xb32000f6),LL(0x5f6dea95,0x27eab135),LL(0xf4446577,0x19091ccf),L_(0x74118cb1), LL(0x765e46b5,0x9210256d),LL(0x9aacef2c,0x3ce74727),LL(0x1ca9b6d8,0xb1ab02c7),L_(0xa870996c), + LL(0x36f1ae02,0x308cbf7f),LL(0x552396fc,0x748ef11b),LL(0x7b2fda3a,0x4146f800),L_(0xaebf97a3), LL(0xb4291275,0x61fb1316),LL(0xf1c9012c,0x894b0da7),LL(0x0142bd07,0x9b191445),L_(0xbab9da1b), + LL(0xe1f1f426,0x5e508e4a),LL(0xfa8ca29d,0x4b94de05),LL(0x27a0f674,0x4cfdd6de),L_(0xee0206bb), LL(0x4b46ce88,0x978ed51e),LL(0xa6bbc41f,0x6bee5364),LL(0xe52c7867,0x006c7072),L_(0x32478f51), + LL(0x5382793a,0x217f1202),LL(0x04851e7d,0x3c689851),LL(0xd7767c09,0x75b3d632),L_(0x4c08238d), LL(0xab3ed287,0x3858c44e),LL(0xcd97e8ec,0x0c103d5b),LL(0x62318c47,0x93355b9b),L_(0xe2405794), + LL(0xe36e5cd7,0xde835c5d),LL(0x92a463c3,0x1eba7f4e),LL(0x3b10b96a,0x4b4acd48),L_(0x5f5bd93b), LL(0xd4e11813,0xf537b955),LL(0xedc9c8f4,0xf31bab45),LL(0xf2362c8d,0xe58a0000),L_(0x970cf697), + LL(0xc005e339,0x8e7b1729),LL(0x882e0b3e,0xf8b77b70),LL(0xbf2d6013,0x490fbccd),L_(0xc0848551), LL(0x5c4916a2,0x73f7a7d8),LL(0x17b850e8,0xde0ad5e0),LL(0xfcc68ded,0x9cbe988b),L_(0x8d09bdd2), + LL(0xf00570c3,0x6266d7a9),LL(0x7f0f6292,0x4291a62a),LL(0x47c40355,0x1cfc61e3),L_(0x7a59e7d3), LL(0x4ce6c101,0xbddfbabb),LL(0x84b7a613,0xe92b59c0),LL(0x89396fd0,0x0f5147ce),L_(0x2b7f355a), + LL(0x9d82dd91,0x8af9951f),LL(0x4d2ceb91,0x78598c17),LL(0xcc451e87,0x6d59d3b5),L_(0xc116fdef), LL(0xd20d8318,0x79ead74d),LL(0x813a5852,0x8ba20d33),LL(0xc12ffcbc,0xcd67731a),L_(0x3fdf17e1), + LL(0x5e3aff2a,0x764582dd),LL(0xefaccc05,0x05c889a6),LL(0x63018ab6,0x6f606211),L_(0x6b706f5e), LL(0xac9cf98f,0x2663583f),LL(0x2a01814c,0xde1ab3e0),LL(0xe29eede6,0x756a8f01),L_(0x10f5f095), + LL(0x363dd819,0xa38d34f9),LL(0x591f8fe0,0xc1ebeead),LL(0xc31b8775,0xae2b4a7d),L_(0x55de9380), LL(0xd500cd22,0x63877737),LL(0x566cf0c3,0x04743650),LL(0x1ad0462c,0x5c1a2a83),L_(0xcfd11aad), + LL(0x4c679d95,0xbc66bc8e),LL(0x9f39feb3,0xd84b8c56),LL(0x21f0dec8,0xd2ff18e6),L_(0x99e3e6c4), LL(0x08b2eebe,0xcce55b7e),LL(0xf3fabc3c,0xb0ed283f),LL(0x5c869424,0x4f7a9f81),L_(0xc660df04), + LL(0xd868008a,0x302b25d6),LL(0xfbdef76c,0xd9a10be6),LL(0x97f6ff80,0xcc0decf0),L_(0x26e1f975), LL(0xbb6408c0,0x4f37e254),LL(0x61eee678,0x9682814e),LL(0x6a0d8fc5,0xd84014df),L_(0xb7e3a2d4), + LL(0xaa81602b,0xde3d73e7),LL(0x3178216e,0xe66f2866),LL(0x2f89efcd,0xdcbe0246),L_(0xb937b3c2), LL(0x72d505b5,0xacaa30cf),LL(0x7f06975e,0x66c190f4),LL(0x714b6102,0xd23f9579),L_(0xc1321142), + LL(0x3c07cee1,0x8271dfd0),LL(0x6e2e57bd,0x85e2b2d2),LL(0xec736ab9,0xbcfb4116),L_(0x1cc014d1), LL(0x57b0096e,0x02178015),LL(0xbcc0c7bd,0xda722aae),LL(0xdf2643e7,0xedc952bb),L_(0x36ab26fe), + LL(0x76041355,0xc562a324),LL(0xccd2af21,0xa955496c),LL(0x8cbd4ea2,0x9f800082),L_(0x7bbad5e0), LL(0x53fd9c07,0xbfd65f54),LL(0xfa87a290,0xd0f0709a),LL(0xd25d631b,0xe063926d),L_(0x0f3d4539), + LL(0xd48fa29e,0x6bf5b024),LL(0x1a63c568,0xea32fa48),LL(0x7bad1c42,0x62df4304),L_(0x38b54b11), LL(0x97d1cfa8,0x94995152),LL(0x7f7b64f5,0x9e00e462),LL(0xd25d1128,0xbdbbd673),L_(0x4703bb51), + LL(0x304a08ec,0x2509ad1a),LL(0x9885b7f4,0x9877088f),LL(0xf70447a7,0x9bfc053a),L_(0x1b77852c), LL(0xaf70aa1f,0x7dc9ba14),LL(0xa43807b7,0xff3589c4),LL(0x02c669a5,0x220e1871),L_(0x991d8093), + LL(0x86209a7e,0xc434b863),LL(0x6c74aee9,0x10b6d0ca),LL(0x072efedb,0x19982c91),L_(0xac91d19f), LL(0xbfb13837,0xbc64a3c6),LL(0xcea49a1b,0xab8a0475),LL(0xe25c1282,0x37f9ab3c),L_(0x57709b27), + LL(0x422a938d,0x63a0bab7),LL(0xbdede240,0x2dffb3e9),LL(0x8ec0a031,0xbbb0cddc),L_(0xf6b96225), LL(0xfa46994b,0x3dff08ca),LL(0xf46b51e6,0xb21694f6),LL(0xfac7173a,0x3f2c5530),L_(0xe5090eda), + LL(0x5e18c4f3,0x585b98ed),LL(0x4cb0c03a,0x2784af9f),LL(0xb6dc9a2d,0x126ee071),L_(0x3331172f), LL(0x56db8dc6,0xba3aa739),LL(0x71ac411d,0x16a260a5),LL(0x816a4f1b,0xa88c4198),L_(0xe409dbda), + LL(0x5fd428ad,0xccb6c49d),LL(0xa20ba7c3,0xe7066136),LL(0x6782fea2,0x83b7b5ce),L_(0xf28c836e), LL(0x4fe78dbc,0x64f82fd1),LL(0x523afba8,0xe21bc6e2),LL(0x024ddda9,0x65d29391),L_(0x4cec16d6), + LL(0xfd04b964,0xaff84dd9),LL(0x1594f62b,0x4eb6067b),LL(0xa8de453c,0xeb65c1b7),L_(0x30a103ff), LL(0x6903c7e6,0xf76afad7),LL(0xa91cd374,0x2f9fd25b),LL(0xa33359fe,0x0b99c0c0),L_(0x9015867f), + LL(0x8a65619a,0xb366fcc7),LL(0xdde98289,0xec03d9a8),LL(0xe45737f9,0x30c2ee70),L_(0xa6bb364a), LL(0x67ffeeef,0xe00c866a),LL(0x8177a004,0xac4b7104),LL(0x49205953,0x0106e5ac),L_(0x5d4c1327), + LL(0x2e836cb7,0xdebcecc7),LL(0xfdc46d5f,0x9b0df65d),LL(0xa125951c,0x5633f2c4),L_(0x04c2afc1), LL(0x89476731,0xacd14df8),LL(0xb5f970f1,0x9f48abbc),LL(0x0973ccd0,0x236b3580),L_(0x02b2c84a), + LL(0x88394318,0xcb13811c),LL(0x0d2dd5a3,0x9bbda12c),LL(0xd8b5d0f6,0x4f399d9a),L_(0xa46887b9), LL(0x2ae73578,0x10165160),LL(0xddf54326,0xc04d1a88),LL(0xf080be9c,0xeee34275),L_(0x047237fe), + LL(0x7aab908a,0x11fceb11),LL(0xfaf4fbdf,0x96bea707),LL(0x66bfd60d,0x442c2a46),L_(0x28cf0358), LL(0x6d51778b,0x4cebfe24),LL(0x6c01d714,0xaf283f0e),LL(0x11d66f5a,0x8cc29c26),L_(0xcf7771ef), + LL(0x76ad7d8f,0xf90d16be),LL(0xb89933d4,0x2a97f299),LL(0xc8f0695f,0x5a6af165),L_(0x82274dbf), LL(0x1826ea5a,0x225b8bb2),LL(0x5e138d61,0x16c62f5f),LL(0x3e2273d1,0x6b42f9f3),L_(0xaa4b331c), + LL(0x5659f40e,0xa9d80f3d),LL(0x6773ac5a,0x6ed38547),LL(0x970b6067,0x78700052),L_(0x2f129047), LL(0x4d7032cd,0x0393eefa),LL(0x04368569,0x98e16738),LL(0x5180fe6f,0x3014d2a0),L_(0x6bd99185), + LL(0x1988a375,0x299011dd),LL(0xe9270a23,0x40dded09),LL(0x54a9528b,0x7af57c36),L_(0x8e613981), LL(0xb388faf2,0xff5fc081),LL(0xd7dd7f17,0x2ccd8f6a),LL(0x421cb6c5,0x590b13ce),L_(0x8f9e6852), + LL(0xe7870572,0x6513f9c8),LL(0xd5bf1a5d,0x2f88c206),LL(0x1942fc9b,0x6ec5f13f),L_(0x70273957), LL(0xf1998de6,0x5bb80bbb),LL(0xcededd82,0x923e0aaa),LL(0xa7c7ee32,0xfa06b05d),L_(0x86a8080f), + LL(0xe016efc2,0x9510adb0),LL(0x8a998afb,0x9a9168be),LL(0xfaf28d30,0x0863e4f3),L_(0xdcf64ed3), LL(0xa0beb82c,0x1ea72a0f),LL(0x78d5ce7b,0x932f17af),LL(0xafe82b0c,0xde25a969),L_(0x1f0a50fe), + LL(0x496b6eb7,0x3d5e21a1),LL(0xddb5610d,0x36b95de4),LL(0x435cd100,0xcc34b698),L_(0xf7218db6), LL(0xf643607b,0xc23a0cb7),LL(0x3b787bdf,0x2299d421),LL(0x71fcc010,0xb0e2a897),L_(0xd6c69c33), + LL(0x0241244a,0x1cd5d5da),LL(0xe94a402b,0xf0ca4eca),LL(0x26944f06,0x737d6c5b),L_(0xd0af2018), LL(0xc1e2ad0c,0x1b523220),LL(0xa8a984b3,0xc8ebc391),LL(0x39663eef,0xbf4b3e0e),L_(0x0297dfd1), + LL(0x0ac5f597,0xc8d20d13),LL(0xf915a4c4,0x00b7e16a),LL(0x2921fb10,0x6b467b73),L_(0x226421aa), LL(0xf2ede5a3,0xf4bdad8c),LL(0xb5512d8c,0xce15cba2),LL(0x2df92000,0x4979cf5a),L_(0xd5159856), + LL(0x7ac29a8f,0xffb25428),LL(0xad4ff433,0x6903617b),LL(0x22c48ae6,0xdcdd2832),L_(0xee468cb2), LL(0x6769e2fd,0xad56cbbc),LL(0x1ac2be8d,0xc8f4bea8),LL(0x11bdfbf2,0x4c080ea0),L_(0xd326fcc1), + LL(0x264b7510,0x3c2a21b2),LL(0x4c900a2e,0x10bf375c),LL(0x540b6e6d,0x40c9b00d),L_(0xf0db08ca), LL(0x1f0e8e4f,0x75ea8b18),LL(0x23e0167d,0x4ae7f6f7),LL(0x3ce76be8,0xc2578bd9),L_(0x274ef866), + LL(0x2d2787bd,0x06e96b07),LL(0xa0cc2074,0x3859e8e4),LL(0x2181d378,0xbc50ac87),L_(0x46b62277), LL(0xc39d3dd1,0x66a073ee),LL(0xbc771f86,0x2bf462de),LL(0x47275780,0xa7216ad9),L_(0xd3c1c667), + LL(0x359b7048,0x37e5b8f4),LL(0xfea02084,0x16126c46),LL(0xe9a5f0b8,0xbf587847),L_(0x53933d2b), LL(0x1c7944f6,0x492b64e3),LL(0x8257b251,0xf2983977),LL(0x8d9ec843,0xfe9686a0),L_(0xeeea3972), + LL(0xac24ef0b,0xb93a8f55),LL(0xf5712604,0xfa720e89),LL(0x6981acc8,0xf73a981c),L_(0x86cc6e25), LL(0xd19071bc,0xa6c75878),LL(0x18cf2a94,0x15519e91),LL(0x5737dd50,0x4c1f9f0e),L_(0x3114afeb), + LL(0xec690133,0xbbc2c60e),LL(0x19e92b39,0x1a4e7efe),LL(0xb3de20d1,0x43d40348),L_(0x693597af), LL(0x9b86b502,0xa7c941f1),LL(0x4568fdc8,0x38d7607d),LL(0x92d2afed,0x50493b74),L_(0xa510adc1), + LL(0x29a4b55c,0x60867e2e),LL(0xab4d38b7,0xc6a2be65),LL(0x0b8a018f,0x2c774449),L_(0xc11883d8), LL(0x3703e4c4,0xa50bce89),LL(0x9bcfd81b,0xc601a985),LL(0xf5811bc5,0x80f1e9c2),L_(0xe9b876e5), + LL(0xb21301b8,0x368a0741),LL(0x7cf8b51b,0x57d524cc),LL(0x85f0eb37,0xb5926249),L_(0x08e70a34), LL(0xdd6d17da,0xe9a1940d),LL(0x4d61fee9,0x905e7d89),LL(0x24af91e7,0xa85a5970),L_(0x41f4907d), + LL(0xa38b5cfe,0x33182545),LL(0x7b2ef570,0x10b61fe2),LL(0x2a780101,0x8d1180df),L_(0xab954cb9), LL(0x8b6a7232,0x88d81ace),LL(0x0d62a965,0xa35ed73b),LL(0xaa16f924,0x2bc7cca4),L_(0x2d99dcc8), + LL(0x3c5881cd,0xb1395709),LL(0x21c56351,0xe2ce53b6),LL(0x41b20f4e,0x067af3fb),L_(0xbd9ab15f), LL(0x2f06e7bc,0x1523f3df),LL(0x9de5e7e4,0xb26c33fb),LL(0xef0cf37f,0x6c9e48d4),L_(0xc9fc2851), + LL(0xcb75ecba,0xb81124f0),LL(0xea0ed825,0xad146b33),LL(0xf4f08526,0x8e19f5c1),L_(0x820a4f79), LL(0xa1d54353,0x6eaedecb),LL(0xefe607bc,0xe5ecca5d),LL(0x75968ada,0x21596aa9),L_(0xcdf9ca14), + LL(0x4cdb6b68,0xdaef28c2),LL(0x2be4550b,0xdd0b6f86),LL(0xee673998,0x8162c783),L_(0x4e1fb36c), LL(0x58a6ed93,0xab7e3385),LL(0xb682150c,0x3081637e),LL(0x02a0814f,0x9c692620),L_(0x3bb22aea), + LL(0x6bc24027,0xa03f91a5),LL(0xd356a2bd,0x0f664f1a),LL(0x04625b5a,0xf350f847),L_(0x2eacfb01), LL(0x4e04629e,0x63af93f4),LL(0xba005c26,0x8ec3c009),LL(0x7964a6f9,0xc56edb4d),L_(0x14b146d7), + LL(0x64910001,0xcae5ffdd),LL(0x51fbb8f7,0x657b3f67),LL(0xa961e968,0xbe43f447),L_(0x2a774b0b), LL(0xbfbe3d28,0x4e174bef),LL(0xb12fde0a,0x11d92d90),LL(0x85b59e4f,0x421eac26),L_(0x4c480a51), + LL(0x83f1230c,0xc3a1a862),LL(0xfe8ceef6,0xbef83f8d),LL(0x0469540a,0xd231a2a4),L_(0x0828f4c0), LL(0xc896c8cb,0x2c034efa),LL(0x1744d906,0xc9a26995),LL(0x2a85a8d6,0x28f5775a),L_(0x58717292), + LL(0x12ab27ce,0xf12dc791),LL(0xbcc5e464,0xec36c787),LL(0xb881c819,0x2bfca9a4),L_(0x7e60531f), LL(0xd8af4418,0x28ea4eef),LL(0x9ef6ada6,0xe0a7044d),LL(0xf56fbcf0,0x534e2057),L_(0x9857c41f), +} +}; +#else +const __ALIGN64 P224_POINT_AFFINE ec_p224r1_precomputed[33][64] = { +/* digit=0 base_pwr=2^0 */ +{ + LL(0xbc905227,0x6018bfaa),LL(0xf22fe220,0xf96bec04),LL(0x6dd3af9b,0xa21b5e60),L_(0x92f5b516), LL(0x2edca1e6,0x05335a6b),LL(0xe8c15513,0x03dfe878),LL(0xaea9c5ae,0x614786f1),L_(0x100c1218), + LL(0x4ca9a1ed,0x5650df9b),LL(0xbe27f7a3,0x2a0f1689),LL(0x10c911f7,0xcafb50f5),L_(0x18dd00ac), LL(0x02aac79c,0xc90ae186),LL(0xe72e600f,0x76cc1019),LL(0x0c84ced0,0x5cabb880),L_(0x6a5db1a2), + LL(0xe37710d8,0xe0b02223),LL(0xab22ca96,0xc3cf1c5a),LL(0x00505bb6,0xaf8b6496),L_(0x799eb566), LL(0x9a73b174,0x04144e62),LL(0xe434f88f,0x357940ab),LL(0xafe10c35,0x6dfa492c),L_(0x8c1b1da3), + LL(0x0a72faf7,0x0860856c),LL(0xc03cd01a,0xcb021705),LL(0x4256e88a,0x89f0c6a0),L_(0x7bf0ef17), LL(0x6743e274,0x98933269),LL(0x64b26855,0x2058f0ba),LL(0x7ee104d3,0x8c87db62),L_(0x73e0d4c7), + LL(0x0b10cc51,0xf8dfc283),LL(0xac3c8962,0x975a451e),LL(0x39211b77,0xf7f4fe95),L_(0x2bd053a8), LL(0x2b552e74,0x61a9e695),LL(0xc350ef91,0x11bcf037),LL(0x42b54f79,0xc858da82),L_(0x9b052162), + LL(0xde577c0e,0x1a8aec73),LL(0xe82df964,0xe0cd01d3),LL(0x1addb3a1,0xf25fe17c),L_(0x21b6fe25), LL(0x5fae6c33,0x0f5c7709),LL(0x327f8848,0xf2d2c41c),LL(0x9b6731a6,0x162f9e1c),L_(0x2383dee4), + LL(0x90462821,0x68646e35),LL(0x5048b145,0x1e2526ba),LL(0xc81df812,0x7ed50814),L_(0x7665453e), LL(0x610f2186,0x1fb32889),LL(0x9d0eba26,0xed8ceeed),LL(0xce5eb25e,0x0e5952e3),L_(0xa229bf84), + LL(0xb4c9d866,0x7bd696a2),LL(0x1de5d041,0xa3bebfe9),LL(0x8e4a5712,0x1b59038c),L_(0x21e8f814), LL(0x1bc1ff37,0x7558734a),LL(0x38a20402,0x0976258f),LL(0xc4e19066,0x2199c364),L_(0xd65a0dfb), + LL(0x1c1ecc78,0x743eda4a),LL(0x9f0b4f26,0x2f993a4f),LL(0x94f5e9ba,0x79d4095a),L_(0x846b2c39), LL(0xe07e55b7,0x909bf3f1),LL(0xbacf5e6a,0x77e1db6c),LL(0xe733d627,0x5114ac9d),L_(0x70889bf7), + LL(0xb808145d,0xb4f2672c),LL(0x328e0ac0,0x3e8abf6a),LL(0xc1f86e2c,0x9d0cdd53),L_(0x5ac34abe), LL(0xccddcf00,0x02883b6f),LL(0x225faa54,0x873d7e4c),LL(0xdbc0f8ee,0x2a687506),L_(0x722981c9), + LL(0xac358ca9,0xe1bdd5ca),LL(0xde5d2c4d,0xd0dee305),LL(0x0f060758,0xd6d89093),L_(0x9397da27), LL(0x649ea143,0x08181c2c),LL(0x79888920,0x6c63713a),LL(0xc1182bf0,0x45e48d79),L_(0x0e48dd82), + LL(0xf6495da2,0xa37b7441),LL(0xed4d0ea3,0x2b0b8e19),LL(0x571d38d9,0x517af941),L_(0x70792626), LL(0x1305b613,0x8cf21835),LL(0x208ffd66,0x5f59deb1),LL(0xbc243af0,0xa71404ae),L_(0x6d228d4b), + LL(0x489c8db3,0xffbe563f),LL(0x83630fe6,0xbc99fb0a),LL(0x68ade63c,0x398eacf0),L_(0xc5e095c8), LL(0x83fdacb6,0x800df824),LL(0xa3ffe74f,0x442ed616),LL(0xb4dc6bf8,0x3f8060b4),L_(0x12ab633c), + LL(0xf4109222,0xcdbd6eec),LL(0x0e9441ba,0x1ef814eb),LL(0xf4550d2e,0xb3a9b5e4),L_(0x47c1182a), LL(0x1544e90f,0x3f51f9f9),LL(0x9ed249e3,0x825b5794),LL(0x6516bbc3,0x3d1419d2),L_(0x92de7285), + LL(0x55bfd51a,0xb16ed1af),LL(0x464afb0a,0xd91345bb),LL(0xc5e26f62,0x64a35d22),L_(0x16880cc7), LL(0x05015843,0x70203aa0),LL(0x11e4ffa1,0x05a83a63),LL(0xdaf3972b,0x9422f10e),L_(0x5dbc0e12), + LL(0x6a4bd1fa,0xb3f66de6),LL(0x2ec3ac7c,0xfdf9a0e9),LL(0x471a2a7f,0xb9c51b5f),L_(0x8c4bc81e), LL(0xfa29681b,0x9c409bd0),LL(0xd979b98e,0x169405c2),LL(0x48be80cb,0xf22d3fc0),L_(0xbba8bdbd), + LL(0x86ade48c,0x79cec3fb),LL(0x3f83b939,0xb9622253),LL(0x5523244b,0x538d543e),L_(0x07ba7ce6), LL(0x12bb38c0,0xb4e20b8c),LL(0x70a52728,0xe6873448),LL(0x2a2e7758,0x2954321a),L_(0x07a8f3fd), + LL(0x3591891f,0x27a72a86),LL(0x845865a8,0x482460f5),LL(0x70114363,0x3b204e52),L_(0xb84ab43b), LL(0xd5f98054,0xc4a1a96b),LL(0xa4e237e3,0x0c0e97bb),LL(0x6bc4b6e9,0xfb212c8c),L_(0x335f11e1), + LL(0x716b6ade,0xea007122),LL(0x2a638e5a,0x08a75055),LL(0x4e35e1f6,0xe28f8fa3),L_(0xe40528c7), LL(0x7393ad17,0x4e27959e),LL(0xd8f5fd90,0x433c708c),LL(0xe5b33144,0x4c003b64),L_(0x6c3deb98), + LL(0xc57bd47b,0x238addb6),LL(0x8d685d17,0x703c82d4),LL(0xa0691b17,0x14a5d58e),L_(0xcd7fb5fc), LL(0xf9cefcb6,0xfd4764c3),LL(0xbb38397b,0x7b23c9fd),LL(0x23cf8d21,0x1d7b0c07),L_(0x7dafc83c), + LL(0x869c61a2,0x8d8b998b),LL(0x6f781395,0x5cc8116d),LL(0x1f271989,0xa8992ade),L_(0x78c70a2c), LL(0xb2334f8f,0x1fadf19a),LL(0x28c19018,0x71d5904e),LL(0x939bd9ec,0x534ec549),L_(0x77091fc9), + LL(0x22be0a45,0xb12549d7),LL(0xca28a29e,0xb64feadc),LL(0xfa13b698,0xd61f5f42),L_(0x5d34326e), LL(0x7ea2feda,0xd797283d),LL(0xc9ceaa43,0x5e1f2529),LL(0x113c4da8,0x4d22be7a),L_(0x12036cda), + LL(0x746daf60,0x210782c4),LL(0x64b5fd56,0x358ace9d),LL(0x1008fd0f,0xaf231c63),L_(0x285dcb4b), LL(0x17203da1,0x3534d554),LL(0xf8542368,0x30962d47),LL(0xf7552ee7,0xa1ab0080),L_(0x11208856), + LL(0x7708ee69,0x981a94f1),LL(0xd9d79e3e,0x50822535),LL(0x2632b017,0x21031b41),L_(0x8876f1a5), LL(0xa43cc165,0xfb24577f),LL(0xcd6f28d7,0x67a5fdbc),LL(0x583ccbb4,0xdf198e9a),L_(0xf49fccdb), + LL(0xec729f5c,0x347bddd0),LL(0x77f89e52,0x4576657a),LL(0x79a42c59,0x3a4d5c62),L_(0xdc6f4cf3), LL(0x72672c8e,0x34edf6de),LL(0xa9cc925d,0xc615c175),LL(0x264b7c52,0x6b6c91f7),L_(0x78e30c64), + LL(0x3c93deb5,0x454fe5e1),LL(0xc107ca82,0xdaedb454),LL(0x895933d6,0xc3d92afc),L_(0xc23f9ad4), LL(0x62afc583,0xb1ad4196),LL(0x9336c30d,0x122626f8),LL(0x56df6aa3,0x88c5db33),L_(0x454135c4), + LL(0xb5862dd9,0xfb7f7ec7),LL(0xfd622a86,0xf9ca08fc),LL(0xf3fb356b,0xa3379ec2),L_(0x3726b3ee), LL(0x5680ab95,0x1f4b780d),LL(0x72773b06,0x181ad0ef),LL(0x9c77acf2,0xc5ed6c19),L_(0xd82c650d), + LL(0x57cdf957,0x2e235ce3),LL(0x919ef962,0x3b66736f),LL(0x02924c24,0x9611c7f6),L_(0x5a87ed36), LL(0x2cd39c41,0x0ca01257),LL(0x8cec5f84,0x36cea89b),LL(0x576c9d6d,0xc0386b9b),L_(0xdf235412), + LL(0x3ff1d621,0xb26ad028),LL(0x0117a093,0x3395a449),LL(0xbb9b7453,0xac8575b5),L_(0x9638e810), LL(0x19b52295,0x61b7e04c),LL(0x917e2f50,0xea595a39),LL(0xb7e04864,0xf641867a),L_(0xf34ee0e8), + LL(0xf9c67813,0x0d5cd555),LL(0xcd9d1329,0x15609313),LL(0x071a1814,0xcec729a1),L_(0x973d765a), LL(0xd82b0a64,0xe9fd6113),LL(0xe16f07d8,0xf4a523e7),LL(0x095fdd3b,0x601388e8),L_(0xd8da4ec4), + LL(0x5d150589,0x6e0c12f8),LL(0xe3db4074,0xcc4b0246),LL(0xe25d613a,0x98b62cd3),L_(0xc7d0dccf), LL(0xca8e4caa,0x1f7fb23f),LL(0x6eff4dea,0x41da9f53),LL(0x44527d49,0x9e1f47c6),L_(0x3f6e7e3d), + LL(0xb17f74ed,0xe460275b),LL(0x3f409a21,0x56dc0e69),LL(0xaef13524,0x055bebff),L_(0x40a3d6e6), LL(0x5a0a6116,0x52aae002),LL(0x5d1a1a61,0x65c91bf7),LL(0x55229ca9,0x837c7497),L_(0x7a1145b5), + LL(0x724287d7,0x86eb5493),LL(0x2b764d91,0x7078641f),LL(0x72b46e9e,0x0bd8afe8),L_(0xfaf8c9a7), LL(0xeea543b7,0x7400cd52),LL(0x409c38a5,0x0a6c2e1b),LL(0x9604a54b,0xe2c2c60a),L_(0xd18594c9), + LL(0x771d897f,0x80cd8b3c),LL(0x4efc7c63,0x3b961927),LL(0x56c6e625,0x8d0c1d5a),L_(0x5533d0e3), LL(0x4b4c8803,0xee49b988),LL(0xc5802272,0x617aabd8),LL(0xb63d84cc,0xb309e31a),L_(0x0fa8942f), + LL(0x6ef3ab06,0xfbca1b97),LL(0xa18bd2bb,0xd8911f8d),LL(0x639e011c,0xb731d607),L_(0xde5b279e), LL(0x0706cd09,0x8e5b6703),LL(0xc24e209d,0x4d443d16),LL(0xdc2ea468,0xccc5dbca),L_(0x9aba875f), + LL(0xf86e4ee7,0xb38b3cb9),LL(0xdc2bb49b,0xd001bff5),LL(0xad920b0c,0x5f833048),L_(0xd67959b4), LL(0x3c687113,0x9cac7077),LL(0x84cba202,0x2bf74e82),LL(0x92b4f4b1,0xfcfe6f73),L_(0x3154d972), + LL(0x9afdccc2,0xf2645ccf),LL(0xe60a2a1d,0x170bc3dd),LL(0xc06a289b,0x8b3174cc),L_(0xb5e5b73e), LL(0xc289c4c6,0xfcbc66cf),LL(0x3f41c58d,0xea32e7fb),LL(0xfdc7bcec,0x2d89d32d),L_(0x34aa50e4), + LL(0x31801b59,0xcf102dd0),LL(0xf19df129,0x28f82869),LL(0x1600629e,0x6d3edc02),L_(0x3c98d6b7), LL(0xf21cabe7,0x31851ac8),LL(0x09f782ac,0x037f5693),LL(0x3f33781d,0xb0a0ce6c),L_(0xe96df7ee), + LL(0x4821f126,0x43b3629d),LL(0x502c3b9c,0x3319053f),LL(0xa3b32151,0x4038845d),L_(0xea3b4ccc), LL(0xedfaa06d,0x00044014),LL(0xd835277d,0xd5eb20d5),LL(0x1a8a108b,0x38cb0a6b),L_(0x30a8f927), + LL(0xb6e363d5,0xa45c3df2),LL(0x96c532cd,0x00c26ab0),LL(0x85098966,0x6d03e325),L_(0xea0d51b3), LL(0x8eec84d2,0x9134e03a),LL(0x04327df4,0xbbc8d92d),LL(0x4ffdb217,0x6e97a4bb),L_(0xc9fa9177), + LL(0x231d20c6,0xa36c8848),LL(0x3fcb294a,0xcb7e8b54),LL(0xc4b2c9f8,0x2f8b6de0),L_(0x49f1993b), LL(0xb8860526,0xfdcf530d),LL(0xb895a0fe,0x49da0f6a),LL(0xefc627b6,0x4e9fb807),L_(0xdbc0d2c1), + LL(0x64eae114,0x23074bd5),LL(0x48d5696a,0xb246633b),LL(0xefdb4759,0xc6af484b),L_(0xde4380e0), LL(0xe5997f29,0x8979b2ec),LL(0x507b7708,0x530f4963),LL(0x55ee3298,0xccf127a1),L_(0x32413bed), + LL(0x687018fd,0x7c723f5d),LL(0x86c9f952,0x2221f721),LL(0x93828714,0x13d720d3),L_(0x45d38213), LL(0xe61c9b06,0xc198db55),LL(0x598140b2,0xbe38af8f),LL(0x621f284e,0x18f09bf6),L_(0x192fd52c), + LL(0x65cd0db7,0xbb33fff7),LL(0xf9ca36b0,0x44ac0a55),LL(0x7a9b66be,0xc9308cae),L_(0x837d663e), LL(0x1b4048cc,0x5b7ecc1b),LL(0x4ee4991e,0x7fa30aad),LL(0x7e09ad23,0xff1f76e3),L_(0xbc48338a), + LL(0xa178a33b,0xdd724842),LL(0x914d8342,0xa5d7a87b),LL(0x30befc70,0x69874db8),L_(0xec19410e), LL(0xcfcdbfcf,0xe3d6346a),LL(0xde9edbbd,0x84c65475),LL(0xde39cef5,0xe4efb3c1),L_(0xc10a727b), + LL(0x493ae0bb,0xbc282cc0),LL(0xe7ebcd87,0xa024a0e9),LL(0xfa3a15cd,0x240e6a89),L_(0xa4a7db8d), LL(0xbba5d02e,0x95b6f1e0),LL(0x86491854,0x15b8172e),LL(0x158b8956,0x5eb7385c),L_(0xa8d22179), + LL(0x92f507ce,0x9137f80a),LL(0xf48fb253,0xcb924115),LL(0x2be56c53,0x0703cdda),L_(0xb4cf511f), LL(0x0b8faa80,0xb73eae3f),LL(0x58a8b927,0xf227ad25),LL(0xdbca2fc4,0x06231927),L_(0x409b12c6), + LL(0x5983f1a8,0xee7ac222),LL(0xf6df6ac6,0x48c69cbf),LL(0xa7dc8e21,0xbd701252),L_(0x6d2da921), LL(0x483083a2,0xc0d30797),LL(0x003aeb7c,0xd4ec2e63),LL(0xbda6fdc0,0x15074c14),L_(0x55d107b3), + LL(0x29e4a336,0x600e1a17),LL(0xd60d3b73,0xa542f637),LL(0x556905e7,0xc7209080),L_(0x3f5144ad), LL(0x0b55b0fe,0xa6cf1f95),LL(0xb8f822b9,0x9d44e115),LL(0x1acfe880,0xe820af09),L_(0x0a84bb85), + LL(0xc0bba383,0x02857752),LL(0x497eb0a9,0x8b30f555),LL(0xf120846d,0xac6401e1),L_(0xf72d6212), LL(0xcb20c291,0x84fbf362),LL(0xa175ab61,0xecddc9da),LL(0xe1839421,0x63952a13),L_(0xbe8f97bb), + LL(0x24f4b136,0xf0524e07),LL(0x8fd9fda0,0x138ee6ed),LL(0x1770de3d,0xedc6985e),L_(0x6c2f8154), LL(0x997ae150,0xb5a50098),LL(0xd64a082c,0x05ed2b30),LL(0xe84702e1,0x4701e013),L_(0xcb1ea5c4), + LL(0xa2d01aa4,0x5c0291c5),LL(0x7cbe7024,0x87f294c6),LL(0xdc575065,0xf7c66733),L_(0x66e96e2a), LL(0x236da943,0xb2bb31c0),LL(0xef556ba5,0xefda656e),LL(0x0d4ba22a,0xd90a6c4a),L_(0xe51714f5), + LL(0xd7b84537,0x6ae52eb4),LL(0x5303af34,0x4989ea91),LL(0x58155fb2,0x51865588),L_(0xff60915f), LL(0xb2f24a3f,0x4a65a557),LL(0x4126452d,0xc5c558ab),LL(0x39d391fe,0x71637238),L_(0x2d627920), + LL(0xc6f1655c,0xda9557a2),LL(0x9f1d5a1c,0xad9724f4),LL(0x02bd8dec,0x286ae065),L_(0xaae19e79), LL(0x66f88629,0x409bbff9),LL(0x02b7c84e,0xa035e3ce),LL(0xfa9ce7d1,0xb11a05b6),L_(0xae613f5a), + LL(0x7d3144d8,0xeeb7e587),LL(0x4daf5c9e,0x63105e87),LL(0x24b31279,0x358ad95d),L_(0x7ca7cd50), LL(0x8b96f130,0xa1f8699a),LL(0x885e7bf5,0x36720d2e),LL(0x19da3cf7,0x0c3e259d),L_(0xbb7ab1a9), + LL(0x3ebef322,0x18a10476),LL(0xd9293d5e,0x93fc7b17),LL(0x2dd5dfa6,0x453f9b19),L_(0xb064e79e), LL(0x2e375ef2,0xde52e14c),LL(0x6557a54b,0xb86d2dac),LL(0xcdf1057b,0x8978f37a),L_(0x60169743), + LL(0x01ae0873,0xf209f351),LL(0xc477ed75,0x1c3d4f63),LL(0xfd2dee37,0x23082359),L_(0x5b85b81c), LL(0x226bc4e5,0xc1dc2c14),LL(0xfac25e9a,0x4a88888c),LL(0xd11ebfc0,0x5bb60dfa),L_(0x6deca182), + LL(0x6d7442e7,0x1a37fe3d),LL(0xceca55fd,0x64b6847e),LL(0x54c71408,0x9321abab),L_(0x42538c89), LL(0x8f6064c2,0x0fe08518),LL(0x0baba2f8,0x48846076),LL(0x18d88b42,0x5027824f),L_(0x9fa6cb27), + LL(0x2fefa18b,0x0a5f6d1a),LL(0x81d4f0b6,0xaeecb1e3),LL(0xe2cc44ed,0x1a3c357a),L_(0xa72825d3), LL(0xe129986d,0x78fb4e9b),LL(0x872596fe,0x6b166a06),LL(0x1f456692,0x55649b1b),L_(0xff6c52c4), + LL(0xbb9f7823,0x0c7e73f7),LL(0x066a275b,0x4c6140bb),LL(0x1d71b6bb,0xc0d4cee9),L_(0x6b9a5b6c), LL(0x8f080a17,0xf403aa8c),LL(0x2c5d59b4,0xb53cebd4),LL(0x45561f5f,0x176891fe),L_(0x7b917d84), + LL(0xffa27853,0x4d6d2499),LL(0xc40d40a1,0xb5248259),LL(0x5a3e6570,0xb7125d0c),L_(0x27a24b64), LL(0x37979bfd,0x785afdfc),LL(0x695b2c23,0x1a4c7732),LL(0xe1b6b02f,0xa84e1866),L_(0x09ceeb7b), + LL(0x11dbcdce,0x4243fd62),LL(0x68c988bf,0x9892ddcf),LL(0xdc0c4d01,0x5638c3c1),L_(0x8f1f8fc8), LL(0xa42313cf,0xfcf531d5),LL(0x5446a19a,0x1417aa25),LL(0x7e0b8323,0x41b2315e),L_(0x8e4521f0), + LL(0xd6c61ac2,0x9d056208),LL(0x63bfbe05,0xa70e6aa2),LL(0x566c60a7,0x483b996a),L_(0x53b69d35), LL(0xa38d32bd,0xc38fefeb),LL(0x27b8ad3b,0x57f15c47),LL(0x689ae7bd,0x3e0d8010),L_(0xfa0d354d), + LL(0x2587df15,0x904f01e9),LL(0x815dc8cc,0xbce7f04f),LL(0x88aa4d08,0xd7f1a354),L_(0x4478e55e), LL(0x5bfae678,0x299925d3),LL(0x661104da,0x6a708cd7),LL(0x74d5a757,0x97968c25),L_(0x8a264b4a), +}, +/* digit=1 base_pwr=2^7 */ +{ + LL(0xd1a6df99,0x084a7e0f),LL(0xce960ef7,0x0b68659c),LL(0xdf5b6590,0xe1cee8dc),L_(0xbb593215), LL(0xf5210072,0xccbda01d),LL(0xb592eea4,0x4c7a3391),LL(0x01bb7a1c,0xa7484df7),L_(0x4b70313b), + LL(0xcb74232c,0xb62e25de),LL(0xb8cba58b,0xc5c8bed0),LL(0xfb8ee6c4,0x4617192d),L_(0x967548ce), LL(0xeaf8b0a4,0xdc2453a5),LL(0x71f81200,0x4d7f2e27),LL(0xf43e697c,0x2e81523e),L_(0x9d94ae94), + LL(0x37eb6d85,0x29543b70),LL(0xc7648cad,0xe67d4661),LL(0xbf221e31,0xde581875),L_(0xded3fb1b), LL(0xc3db2400,0x82bba001),LL(0x656ecd6b,0x60a68af1),LL(0x5e5ccc27,0x619f6149),L_(0xa5bb89fa), + LL(0xe80619b6,0x25f0799b),LL(0xd58585cc,0xeb6dfe0c),LL(0x9bf5f7d0,0x63a366a0),L_(0xff7302b6), LL(0x927212bf,0x02c160de),LL(0xf4586c45,0x09ea0af5),LL(0xfe05e98d,0xbae9ad71),L_(0xc8fc9d86), + LL(0x65b1d33e,0x7f5ed157),LL(0x36faea3e,0x1ed1728d),LL(0x39babd58,0x0137cf58),L_(0x4d9cad56), LL(0x2546cd9c,0xd0dfb090),LL(0x4316891a,0xb59a51ab),LL(0x6476def4,0xc5dce06b),L_(0x7c93323f), + LL(0x2313823a,0x80c0cfc1),LL(0x753f5cf1,0x27734f88),LL(0x587454f5,0xb527a9af),L_(0x315583c4), LL(0x049823f3,0x25a1f7dc),LL(0xd3e3e75f,0xb3ea257d),LL(0x1d11fd76,0xafd0ae51),L_(0x1190d505), + LL(0xc1bc485d,0x407824fd),LL(0x59b8241e,0xa0e87d81),LL(0x71256aba,0x76217f94),L_(0x64a95b3e), LL(0x8b3e508c,0xe501c9b4),LL(0xfbc90f77,0x08a19d4d),LL(0xfb512fa2,0x59ab4cb8),L_(0xc1d78806), + LL(0xe4756697,0xaf2eb7f6),LL(0x05273793,0x5c0e9df7),LL(0xc52db668,0x2b7064bd),L_(0x8eabb5ee), LL(0x43184721,0xd4b9fc59),LL(0x8bf316a9,0x1343040b),LL(0x88ef261a,0xb9864942),L_(0x212e47d7), + LL(0x0f601217,0xc3ce93c3),LL(0x1d875805,0x40d758a5),LL(0xc0aa59c7,0x54ae6327),L_(0x7111e036), LL(0x111b9e3d,0x09cd3a96),LL(0xcc42b514,0x6bcd40ee),LL(0xb04993c8,0xbcda6d48),L_(0x620f7a62), + LL(0xbabee792,0x030c3495),LL(0x74030e51,0xd426f7b4),LL(0x457bca6d,0x81319526),L_(0xf21490eb), LL(0x2498c159,0x0b2bd04a),LL(0x8221d32e,0x12985a92),LL(0xcff74849,0x837d8e0e),L_(0x2ca32bf2), + LL(0x139de2dc,0xf41adf6c),LL(0xef7b6eba,0xb7dc7dc5),LL(0xede2ea87,0x6e2033f0),L_(0x0647be07), LL(0xcd213865,0x287f4ece),LL(0x4d3993a9,0x50f400e9),LL(0x0fc6614f,0x89db6d9e),L_(0x8f95a105), + LL(0xc42a6b84,0x9e9a2ddd),LL(0xeeea986a,0xf3f5ab2d),LL(0xabdd54a4,0xc69f9a88),L_(0xb87d1977), LL(0x8affd663,0xb4602516),LL(0xa2399ce4,0xc75f7a3e),LL(0xfbd56bb7,0xe4aa58aa),L_(0xa1149cb4), + LL(0xbb083ae4,0x96aff3db),LL(0x896df53d,0x2d9b4c87),LL(0x77208838,0x2b1d390f),L_(0x02221513), LL(0x408ee7f5,0xc9a98c1e),LL(0x3460329f,0xb6789b9e),LL(0xf0a9cf2d,0xcc0aecb1),L_(0x14bf88b6), + LL(0xf99e1bc2,0xf9d97ac1),LL(0x32112cbf,0x31242e73),LL(0xe520dfcf,0xba519709),L_(0xb278a649), LL(0xaada88db,0xdf453d98),LL(0x7de50c69,0x48d83f42),LL(0xa35584c3,0x7efe383e),L_(0xafd2f996), + LL(0x18351406,0x987fc6a6),LL(0x33c7e3ed,0xf2b213b9),LL(0x17bc0ecf,0xd5c3d1eb),L_(0xe316f3eb), LL(0xc603801d,0x037728cd),LL(0x0890b2c8,0xc5cf397d),LL(0xde6243a1,0xaf004bfd),L_(0xb5bd010e), + LL(0x81c21907,0x140e4fb4),LL(0x4a85940e,0xb07d29b4),LL(0x65c368fc,0x6829786b),L_(0x11ef4637), LL(0x7c01e8b8,0x5c2ad4ba),LL(0xd4385c66,0x8b692602),LL(0xca528c39,0x09175739),L_(0x8545624f), + LL(0x3e778b36,0x332aa21b),LL(0x69ac9b72,0x7761d7c9),LL(0xf6af5223,0xded7e546),L_(0x2e42f629), LL(0xde909623,0x254a8b2a),LL(0xd20517cf,0x6fa060cc),LL(0x7cccaea0,0x723e3222),L_(0x17b8fcf9), + LL(0x00e54d7f,0x80274626),LL(0xc377f0eb,0xf63104d2),LL(0x448cb4f0,0x86f7822c),L_(0x905f1a28), LL(0x635d1d1a,0x74c9d852),LL(0x03d82ded,0x2f27a18d),LL(0xa7638f56,0x7df0f814),L_(0xa98013e6), + LL(0xe3919fa2,0x11b8e6be),LL(0x71e986fa,0x2038bc38),LL(0x1a7c24cb,0x547f1d30),L_(0xd082b3c8), LL(0x4937319c,0x14498e9f),LL(0x96e06bdf,0xd9e8ae36),LL(0x71231dd7,0x099cc29c),L_(0x52cc6fea), + LL(0xbfa21d6a,0x9934b257),LL(0x1c360412,0x2552b461),LL(0xfad28b1a,0x87244f2d),L_(0xf0f3af7a), LL(0xe091c63c,0x224579d6),LL(0x8b1b9bd5,0x7927774a),LL(0xa4177fae,0x39e5970f),L_(0xfbd55d05), + LL(0xf8a953dc,0x00e1559f),LL(0x1dc11998,0xa70e63b9),LL(0xf8851e82,0x48191bdc),L_(0x7132a05f), LL(0x6d398f42,0xc96682f5),LL(0x3029087f,0xc2927516),LL(0xd7d600fe,0x6435359a),L_(0x103c49c9), + LL(0x43c60bfa,0xbe98cc32),LL(0xdf3dae98,0xe899fd7e),LL(0x6f6a67d4,0x4805ca93),L_(0xb569900e), LL(0x532ddf8e,0xb5f6725b),LL(0x912c5876,0x9818fcef),LL(0xa6151612,0xa9419c51),L_(0xb659688d), + LL(0x544bdd85,0xf568cb9c),LL(0x486d2f59,0xe070e7d2),LL(0xa9680c77,0xf00849de),L_(0xcaecbf55), LL(0xc159fff5,0x9f8b9614),LL(0xff12efb3,0x0503644e),LL(0xf325f0bc,0x58ef4f26),L_(0xc0fb619b), + LL(0xbb5a090c,0x2a905098),LL(0x34e237a4,0xeb89d751),LL(0xb46f0674,0xc23b6e08),L_(0xf504a980), LL(0x84d934a6,0x7446341f),LL(0xfe8d0cb7,0x2d200177),LL(0x428fe6ef,0x663531ec),L_(0x82b91738), + LL(0xf0622227,0xe7d3d99c),LL(0xea4400e4,0x87323665),LL(0x0421f1d4,0xb81113de),L_(0x3c97ee34), LL(0x121ae441,0x14b9bfe5),LL(0xae37e0d9,0xab8ce456),LL(0xfb0b5381,0xeebf8b75),L_(0xd0e53ea8), + LL(0x4ab996fa,0x3bca93de),LL(0x12681ec0,0x9c224825),LL(0x51f0a250,0x4390fa94),L_(0xe12f0e92), LL(0xa17f9bd0,0x87ca94cd),LL(0x707670f5,0x8eb876e3),LL(0x8c10d532,0x6ab8e456),L_(0xad2be3fc), + LL(0x00191032,0xbe58c480),LL(0x20928c0f,0xf0df6b45),LL(0x51d6cd1a,0x3a2b0d2e),L_(0xdae2e721), LL(0x6e0dcc33,0x17ad59a0),LL(0x4848071c,0x24ff8cf7),LL(0xb1342fd7,0x514eda6b),L_(0x8d26ef04), + LL(0xaede35a9,0x43fadb36),LL(0x00e95336,0xda078971),LL(0x89ff847e,0x757ad40d),L_(0xc2d2fb8d), LL(0x063bed05,0xfbec9b9f),LL(0xc9f9426c,0xa2938400),LL(0xed5c65dd,0xd3c116f5),L_(0xcc982965), + LL(0xbf6d76ac,0x4a94b590),LL(0xdd6503fb,0x43e3ebef),LL(0x6d7848f4,0xe0fe0161),L_(0xb547e19d), LL(0x503ad71c,0x14c43727),LL(0xeef3ccce,0xafbcb090),LL(0xbb5f8c82,0x3bf8d893),L_(0x2cff74c5), + LL(0xa2d0d278,0xd8be2204),LL(0x0ccdf937,0xc9fa15ec),LL(0xac17e25a,0x91b10a2f),L_(0x61c0b94e), LL(0x12c8ef80,0x087fa01d),LL(0x1b1a2421,0xb6f728bd),LL(0xd198666a,0xcef80aee),L_(0x76851ee8), + LL(0x6112903d,0x89add916),LL(0xd329af40,0x1c13960f),LL(0xf15aa481,0x48b52034),L_(0x0726b2a7), LL(0x2d1bf701,0xb5d87e2f),LL(0xb0c178da,0xc7e5ba8c),LL(0x6e4a90fb,0xfd04fbda),L_(0xe99dc375), + LL(0xce2f6f2f,0xae50cdc8),LL(0x37383201,0x5c73b50e),LL(0x59a0da00,0x6b833239),L_(0xe9260200), LL(0x54c575fb,0xfdd6dfac),LL(0xea53c24e,0x761a3a3d),LL(0xb3095407,0xca8c1a6e),L_(0x4e4fe43a), + LL(0x56e12b9f,0xa3fc4119),LL(0xfb6450ca,0xf5bdb563),LL(0xe34b58ec,0x178622a7),L_(0xd55ba9dd), LL(0x9e9ffc06,0xfbbaa19e),LL(0x1f5955f9,0x42f09c2b),LL(0xb2d6157b,0x0c83351e),L_(0x325e63cd), + LL(0x623598c2,0xae4a61b7),LL(0xc98d16f5,0x47014b1b),LL(0xddd6a618,0xc1476c47),L_(0x7f025b55), LL(0x7eb38a44,0x31a26c96),LL(0x374db7ac,0xc5ea2b2a),LL(0xa5a136e8,0x13b6043f),L_(0x43d33d2b), + LL(0x07cb2294,0x210180c0),LL(0x173b6efc,0x782987ea),LL(0xab40f28e,0x4a4f4f9e),L_(0xce66440a), LL(0x591b5ef9,0xf6832896),LL(0x49522fda,0x38f56fa2),LL(0x5abb2a56,0x5a65a98b),L_(0x6a059efd), + LL(0xca04bf7c,0x022f4557),LL(0x7a97541a,0x657a19e3),LL(0xc772f98c,0xa19ab123),L_(0x0e388f1a), LL(0x82db0ead,0xd6b0ec00),LL(0x312059b5,0xdeadd9cc),LL(0x9dab24ce,0x1c2e2051),L_(0xc5487b00), + LL(0x7647407b,0x489801b8),LL(0x05173f8e,0x56034023),LL(0x26d5313c,0x29b7a999),L_(0xf53aa77c), LL(0x6f0fc77d,0xfdc7c2f1),LL(0xe7c7d3e1,0x006982c8),LL(0xcb5af1f9,0x9323d960),L_(0x7cb0661a), + LL(0x76954498,0xa4de751e),LL(0xcad7d975,0x6dc87029),LL(0x805eb5f7,0x1cfb5ce3),L_(0x6e5f7742), LL(0x94bbb1b3,0x43b64965),LL(0x3e36f135,0xaf8d3158),LL(0xc4fd0925,0xeb438d66),L_(0x5568e790), + LL(0xc7a170c2,0xb2973b56),LL(0x51fcb9a5,0x9e4dbac9),LL(0x2cb87666,0xcf3bacaf),L_(0xf1a40af8), LL(0xa2effdcf,0x20a60ee8),LL(0x47e2556f,0x60509402),LL(0x676121bd,0x745c0ff2),L_(0x1082eb4f), + LL(0x3407acc6,0xb6d85b13),LL(0xb4a1c32a,0x7e9049e0),LL(0x9fc877ce,0x64c1005f),L_(0x928ec4e2), LL(0xb606ab7c,0xe272e2ce),LL(0xacbb37ae,0x97ac2d38),LL(0x954f3f1f,0xa0e40e6d),L_(0x1630287d), + LL(0xdf2474e8,0x1f05663e),LL(0x7d0f40a6,0x234012a8),LL(0x89989069,0xc2481dbc),L_(0xb865be0e), LL(0xf495b126,0x452948bf),LL(0x27e891c7,0xa43ea9bb),LL(0x280bca9e,0x8a34a29a),L_(0xdb3ca4d6), + LL(0xae198472,0x848b5387),LL(0xa735717b,0xcce778e0),LL(0x4288a820,0x34cf4dad),L_(0xdcd93303), LL(0x9e011fee,0x30c9f97b),LL(0x7df2569f,0xcba3ae4d),LL(0xdb99670d,0x1db11c4c),L_(0xac0ab6d0), + LL(0x9d825f39,0x2980d39c),LL(0x1edad965,0x5a3ddebc),LL(0x9c96ad62,0x7f86e997),L_(0xa6296b18), LL(0xac938f79,0x7a80b2ac),LL(0x8d1c685b,0xe8b1101a),LL(0x055eb004,0x6c8daec0),L_(0xae9ffa1b), + LL(0xcaf9f5b9,0x88ad0652),LL(0x977ba6f8,0x35f6f9f7),LL(0xbced09e3,0x51bdbe8d),L_(0xaf059927), LL(0xd40e66f8,0xa071947a),LL(0xde4b10ab,0x2ee8b51f),LL(0x9757d231,0x4f80de84),L_(0x84da5f77), + LL(0xdf7efaf1,0x3121c652),LL(0xbb4de8a3,0xd7749653),LL(0xb988c422,0x493a2d8b),L_(0x7dd3a0cf), LL(0x98e08451,0x2760de2f),LL(0x05ba148d,0xcf07c50e),LL(0xe5f81246,0x1a00e240),L_(0x89c9fdf9), + LL(0x9d26d17d,0x89b4fc25),LL(0x35eb5721,0xc765d35e),LL(0x45f96752,0xbfefae3b),L_(0x65d4fcd7), LL(0xf9a4089c,0x22ecc154),LL(0x3b66fffb,0x5053964b),LL(0xdcd4205f,0x1e216001),L_(0x9258ce3b), + LL(0x1db56ac0,0x7ae4875c),LL(0xdd9688b4,0x4a976987),LL(0x4fa88f83,0xd67fc048),L_(0xcf37f9f8), LL(0x49b6eedc,0xedcaacaf),LL(0x38ff0886,0xd26b609b),LL(0x29f6f1cc,0xa9ceab78),L_(0x2a5e8359), + LL(0xf6f8a0a2,0x95c6ea4c),LL(0x0080fd45,0x61916ad2),LL(0x8497eb60,0xb0ce6cae),L_(0x75ea62e8), LL(0x50c80949,0xa31d0581),LL(0x9b2a0c1a,0x4cec302b),LL(0x0bfcd85b,0x36484967),L_(0xe5f11201), + LL(0x6e5e0c26,0x6df2fa39),LL(0x59949304,0x248a89e4),LL(0xdcc52026,0x67b816f2),L_(0xc824943c), LL(0xee7ad3ec,0x5bb4e55c),LL(0xfa312fb7,0xdaa70d9e),LL(0xacc63cd9,0x9155d520),L_(0x42aa4030), + LL(0x6abedf03,0x22a83e65),LL(0x2814f9c9,0xb90ac85d),LL(0x088ae4b8,0x65e3e716),L_(0x7975b338), LL(0xf19b0bfc,0xfbebb9d9),LL(0xfd3c7946,0x7355b4aa),LL(0xef331b93,0xfcb3b647),L_(0x9f037b66), + LL(0x4e49fdb8,0x76e1b0d3),LL(0xa7cfa8e6,0x94293ef4),LL(0x588501b4,0xd8359bfc),L_(0x50d236b6), LL(0x3cd45f3b,0xee63deea),LL(0x49004988,0x98af1b2e),LL(0xd1f2806f,0x12a331f8),L_(0xe108a9ab), + LL(0x3577e68c,0x7e2770f6),LL(0x08275599,0xe8913f4b),LL(0x77ead765,0xe027fdcb),L_(0x2d381d9d), LL(0x9a6f6903,0xf459aa5e),LL(0x38d25f4d,0x24bf5d49),LL(0x130316d1,0xd808c825),L_(0x53bfc24b), + LL(0xe048c7d9,0x33437c46),LL(0xce7c4312,0x2ce661a7),LL(0xe77ba9cf,0x46b5165f),L_(0x98d33543), LL(0xaa89fa67,0xd3057aed),LL(0x81116061,0x142332ba),LL(0xb60e8644,0x3eb16842),L_(0x632f4f19), + LL(0x35fc5d9c,0x9f0177bd),LL(0x63fb6870,0x63dcb419),LL(0x4fc638bc,0x0d4a9edd),L_(0x5f4b4b0c), LL(0xd3a1ffcb,0x2be34520),LL(0x4dfd2636,0x6ee7420c),LL(0x449c7ffa,0xca54fb79),L_(0x1723c63b), + LL(0x447c4951,0x07a8fa48),LL(0x5a4bf1f5,0xfe67f509),LL(0x0d70f3ac,0xdbc48919),L_(0xc6e3be9d), LL(0xc5e2455e,0xc9b41ae6),LL(0x1cb83c2d,0x1f329372),LL(0xaab1f6eb,0xa76cbb84),L_(0x13b46361), + LL(0x7f11366d,0xd38d4421),LL(0xd7176b60,0x9815fc31),LL(0x1e6e5fa0,0x42bf5236),L_(0xaf44dc92), LL(0x15c5a009,0x03ac6a2a),LL(0xabc5024d,0x8f3eaace),LL(0x9eb61a43,0x1a7ff3ad),L_(0x67fd7c39), + LL(0xf6ee6118,0x4446a48b),LL(0xd5ff7dbe,0x08776dba),LL(0x62dd1551,0x856288fd),L_(0xf3f982ab), LL(0xb1d08623,0x59e29beb),LL(0x6ab6d270,0xca32b4fc),LL(0x1744a1d3,0xad3745b8),L_(0xd398bc25), + LL(0x3831190b,0xaaeece81),LL(0x93c12ffb,0xd4655489),LL(0x6dc6a194,0xdbb627e1),L_(0xb1087770), LL(0x91571bcc,0xf5fb9008),LL(0x4ad3caf9,0xf421b41e),LL(0xeaac6d91,0x01604108),L_(0xe70c380f), + LL(0xa2f0a872,0x276d7a20),LL(0xdacb5ad6,0xa823fb57),LL(0x507ee8a9,0xe6fef46d),L_(0x8d424c56), LL(0xa385d034,0x2785c811),LL(0x7ba7caa5,0xbcf3686a),LL(0x8ac8344a,0x3bf9ea41),L_(0x63abeaac), + LL(0x8075a441,0xfea1ac4e),LL(0x2be4fe2a,0x0fedea77),LL(0xb3c70edf,0x2398f701),L_(0xbafd9f1f), LL(0xbbbf63ac,0x9c1eb75a),LL(0x06bc9e31,0x9c3a2d57),LL(0xee68eb75,0x4d73d0da),L_(0x5c2caf9e), + LL(0xe663f1d8,0x31f69d33),LL(0x2870e42c,0x46cf68c3),LL(0xc7fa33cc,0xb4e3665a),L_(0x4310b502), LL(0x48f183b7,0x154b0c79),LL(0x225295c9,0x32607f60),LL(0x1e044a43,0xc1030470),L_(0xf5ab45de), + LL(0x25aa9de1,0x88786097),LL(0x8cfe807b,0x045b9699),LL(0x2e64a0a5,0x921c1980),L_(0xaa9a7dc4), LL(0x8b058dee,0xbe46c84c),LL(0x47ac068a,0x5b271860),LL(0xfe16e851,0x48f0acc8),L_(0x76ccb233), + LL(0x31bafd7f,0x7c0c6e0d),LL(0xdade195f,0xe2d4fff8),LL(0x2b1da468,0x95ee5ad6),L_(0xee119e9e), LL(0xfebeb585,0x307f35dd),LL(0xbf55c718,0x406d6f39),LL(0xe2245982,0xd6be6d55),L_(0x7d132e1a), + LL(0x002c4e28,0x4f02beba),LL(0x8d9e9ed0,0xc26d73f0),LL(0x21de614f,0x38b705d3),L_(0x68620b5e), LL(0x23429db7,0xf4967923),LL(0xb79a106a,0x198f9eec),LL(0x9eb6af93,0x6bb259cc),L_(0xec4f9488), +}, +/* digit=2 base_pwr=2^14 */ +{ + LL(0xdcabdf3b,0xd382ee31),LL(0x8d88ff9c,0xeeccf67f),LL(0xe97abee6,0xc6941824),L_(0x90a4b5a6), LL(0x47a31930,0x7f45fc06),LL(0x876a0ffe,0x7b1af995),LL(0x4d9341e8,0x248e916d),L_(0x398b68bc), + LL(0x082378c3,0x4bf3bcae),LL(0x8cfd71b3,0x82c65899),LL(0xb15f6e52,0x5919e8e7),L_(0x2217b0ec), LL(0x6c251213,0xce97fdbe),LL(0xd1051883,0xf8d74874),LL(0x1738524d,0x44e69e4f),L_(0x98671ed4), + LL(0x54e9eb42,0x1bc6447d),LL(0xcb1bcc20,0x65482174),LL(0x63694975,0x72869bff),L_(0x63c463b1), LL(0xf6aea0ba,0xa70d16cd),LL(0x12e9b82e,0xa212780d),LL(0x14730725,0xdc25b909),L_(0x48e6eff8), + LL(0x8d673443,0x01c6af2b),LL(0xce9a229f,0x24e6fbce),LL(0xc6a67d3a,0xbbe5e1ed),L_(0xf65af52d), LL(0x964e75d2,0xb97b0d10),LL(0x8e72295a,0x09007530),LL(0x67a932fe,0xc85f611d),L_(0x7b6da974), + LL(0xfaef942d,0xfc2eb46d),LL(0xc735b7a5,0x51096872),LL(0x75df38bf,0x75ab062b),L_(0x82f7953f), LL(0x1f5151d3,0x587ef818),LL(0x723a33d5,0xf3f93876),LL(0x0eae8822,0x5b5cb708),L_(0x97da4376), + LL(0x36982471,0x165fe970),LL(0x5a5ca19b,0x2ff27508),LL(0x89cd99ad,0x98c66330),L_(0xd0ea6970), LL(0x0397edc8,0xd2a0cfc6),LL(0x49c7caa4,0x7e36813d),LL(0x722feffe,0x829c6b89),L_(0x3cc14e57), + LL(0xe76a6991,0x394ea602),LL(0xb318aa05,0x1348f9f3),LL(0x34e40071,0x3b2e0279),L_(0xa37218ff), LL(0x5da0385d,0x0b77484a),LL(0x487e3215,0xbbb8ff2e),LL(0x7e2ac222,0xf54f007f),L_(0x916f6355), + LL(0x306867f1,0x271f814c),LL(0x45d99845,0x455790c2),LL(0x038f4fe6,0x58008770),L_(0x1a0c7df5), LL(0xa3ffa256,0x2ae5134e),LL(0x791baa3a,0xfa061cdf),LL(0x2bb3a978,0xa4fe1b53),L_(0xd5d1144c), + LL(0x1840dd40,0x24b14e0e),LL(0x8cc62de0,0xc72a3b9b),LL(0x5ca5c5d6,0x8625f308),L_(0x30593cfa), LL(0x6afba385,0x5e8c195d),LL(0xb4f4df35,0x6bdd2d33),LL(0x0975350a,0xeff53d99),L_(0x20a0786d), + LL(0x3fb95973,0xc84c9bda),LL(0xe3fb8d0f,0x172afe39),LL(0xd069e505,0x2d143bfb),L_(0xcaad811f), LL(0xaf5715f5,0x9c2db5f2),LL(0xf0b61247,0x6e7b80b3),LL(0x71ec23d9,0x4f3f9687),L_(0x8dce56a9), + LL(0xf5d591b8,0x0c9c04f2),LL(0x4e3ba616,0x6d31163c),LL(0x530f24db,0x66c42b68),L_(0xefcb7af5), LL(0x1378c67f,0x777097e8),LL(0xf5bcc425,0x3f1062b8),LL(0x9ee5175c,0xb09a9492),L_(0x8ba812ad), + LL(0x12b71604,0x0ee3063d),LL(0xf5bf2d6e,0x22611499),LL(0x853772f7,0x8f349671),L_(0xdf452ffe), LL(0xdd3b39e2,0x3acb7c05),LL(0xa199b0f1,0x34c3e766),LL(0x9381a0b0,0x74081fec),L_(0x613e10dc), + LL(0x49667bc1,0xd57108d0),LL(0x93327b6d,0x6834cfcd),LL(0xf7fb9009,0x9b0baa9c),L_(0x032637ce), LL(0xd797b7f9,0x13c1353f),LL(0x66821e80,0x6f581f7f),LL(0x48a60c42,0x7f9e960b),L_(0xcad0afca), + LL(0x474d3704,0xcfd42398),LL(0xd4011a09,0x1c1a54ac),LL(0x3c4fd36b,0xd3eac930),L_(0xffda655a), LL(0x7c4dc3a4,0xdce7096a),LL(0x75a38366,0x732dfde0),LL(0xa1787513,0xe804b476),L_(0x86101a86), + LL(0x926b6b5d,0xe5f713d5),LL(0x9c17bdda,0x3d019ddd),LL(0xd52c15ec,0x3e7f7e62),L_(0xfaca5ec2), LL(0x6609394f,0x694f2696),LL(0x283219a8,0x69aa76bd),LL(0x07cee438,0x234e3949),L_(0x77c5f3f2), + LL(0x90a49b53,0xf501aeb8),LL(0x8debc123,0xc50fe003),LL(0x8b89e79b,0xa04a6f7a),L_(0x51dcf521), LL(0xff3922de,0xc4ed1911),LL(0xcefa7526,0x0ce76928),LL(0x8af23a3e,0x918974b5),L_(0x3dda3d79), + LL(0xdfd72a80,0xdea21ead),LL(0xbaf90df8,0x8e3f7ba6),LL(0x1f678c3b,0x6df4e74e),L_(0xc2ad132e), LL(0xc5a010e7,0xd7eea56a),LL(0x77bf90b7,0x389d8452),LL(0xc6ed7b9d,0x2e489cfc),L_(0xdd237e98), + LL(0x914f8bfe,0xbcee9591),LL(0x21b5804c,0x7fab5561),LL(0xec5af129,0xd4d265a9),L_(0x7f0919d3), LL(0xc74bfc94,0xd2bab14c),LL(0x1a3e2e12,0xe6b2e9c5),LL(0x77d00d82,0x161eafab),L_(0xccd81779), + LL(0x8fb0ef8d,0x5b2aa48c),LL(0x43630c37,0xe9937cc2),LL(0x00ef4c97,0xb862b86d),L_(0x1ec65b9f), LL(0x97617d09,0xfe0c2928),LL(0x92157b61,0xce44e423),LL(0x625a38ef,0x8594c3af),L_(0xbd404f57), + LL(0x1b735669,0xbbf9ef82),LL(0x29fe66b9,0x754e211b),LL(0xf058688a,0x95864d69),L_(0xdc64d0f5), LL(0x8beee137,0x9150359b),LL(0x0d596fec,0x0ab949fd),LL(0x3f4ac39d,0xf6d5f8ee),L_(0xc65ea695), + LL(0x4aaed3cc,0xb594cda0),LL(0xea397e70,0x0216bf13),LL(0x9e271d0d,0x7efe2b03),L_(0x52fa9e75), LL(0x6a084fdd,0x824376da),LL(0x369bd28d,0x6acd2b86),LL(0x2deabee7,0x55b4713a),L_(0x9706afcf), + LL(0x9b5408d2,0x5ffa275a),LL(0xce057f13,0xfa6d3b63),LL(0x3f3eff6d,0x5bf0abda),L_(0x82fe66a1), LL(0x97b2723a,0xcf4c8232),LL(0x364b69c3,0x170c8de3),LL(0x047d7294,0x43f7fc78),L_(0xec58892f), + LL(0x0b00d0be,0x4d42a1fe),LL(0x3e6453fd,0xc3907782),LL(0xd65498ad,0x716502c0),L_(0x548685f9), LL(0xaa83d0aa,0xd8e4c681),LL(0x30dc30bb,0xb2730e91),LL(0x3b7c8d6f,0xc3d966b3),L_(0x7eb48e5c), + LL(0xfee9984a,0x2e099be3),LL(0x6964c934,0x5f8e6df6),LL(0x31915ae6,0x2ba98b17),L_(0x72598e4b), LL(0xe2f20ab1,0x791aa104),LL(0xe9c53c90,0x2ae23400),LL(0x5c5045ab,0xd646c528),L_(0xf2a49004), + LL(0x50b8339b,0x2874cd90),LL(0x98df361f,0xcb1961a6),LL(0x221ee1b5,0x43f06891),L_(0x6cf88e87), LL(0xf9998039,0xeac42395),LL(0xe0ee202c,0x042a0af4),LL(0x36489fc2,0x5fdad7bc),L_(0xf5be5517), + LL(0x9f01cd49,0xc0ba445f),LL(0x657c4ee9,0x3e5d8712),LL(0xc6107ee1,0xecfad04a),L_(0xc3a9ecc3), LL(0x45f63660,0xeda195e8),LL(0xfd5f4e28,0x2f2caf70),LL(0x4fc9d505,0x0c644ae1),L_(0x49b7af65), + LL(0xbc63e79a,0x658bc0b0),LL(0x98214b9e,0x2713db76),LL(0xabf827dd,0x12824e7f),L_(0xaef5577f), LL(0xb2616195,0xfcdeef4c),LL(0x532de453,0x3fa68909),LL(0xe9395e01,0x42dfc220),L_(0xa3cf44a0), + LL(0xe6bee7b9,0xb1959a03),LL(0xc236089e,0x9b779900),LL(0xe7e57056,0x1c2d553f),L_(0x6e0825c3), LL(0x47995b49,0xab4d653f),LL(0x07e36ef2,0xf8e40f18),LL(0xb70d2132,0xbaae5b9b),L_(0x6545595f), + LL(0x325e475c,0x8ad5b01f),LL(0x55e3fd67,0x8fb55f0b),LL(0xa691b273,0xee832613),L_(0x372805fc), LL(0xb146b4e4,0x748d793c),LL(0xbb8942e4,0x2911a866),LL(0xdb940971,0x24df2ed3),L_(0xc58ee255), + LL(0x25961d65,0x87754320),LL(0xcf82c1fd,0x15d5a2fe),LL(0xee280f81,0xfc4877e3),L_(0xf4f8e2e1), LL(0xd08afabe,0x8dd2825d),LL(0xe83d6d87,0xa36fe3f1),LL(0x20840934,0x398c3272),L_(0xe0445764), + LL(0x51502993,0x5f9bec8a),LL(0x2f38e2ec,0x878f852f),LL(0xbcaa2cbf,0xe99dba0e),L_(0xec7f9cc7), LL(0x7a185dc4,0x7f95d77f),LL(0x0cb71b4c,0x3f419246),LL(0x2b04f2d5,0x7b04c078),L_(0x21ee7511), + LL(0x597ba0a9,0x0eb981ea),LL(0xc63bbafb,0xe6e2623f),LL(0xbfc4dac6,0x35dd20af),L_(0xf02694cb), LL(0xe7b40564,0x8d381cfb),LL(0xb6c06c63,0x4013b9eb),LL(0x354e81ee,0x50a8d84c),L_(0x9e8bae79), + LL(0xccc6dc57,0xe6735e77),LL(0xf38bca55,0x629efebe),LL(0x2ffffa9f,0xc520680b),L_(0xf670ee6a), LL(0x732a53db,0x4d7d1ede),LL(0x920d6d5f,0xa1f8c1a4),LL(0x3dffe6e5,0xc519ac0b),L_(0x927ed8ff), + LL(0xaef46031,0xdd8a57c6),LL(0x3e80aad3,0xb0540672),LL(0x8ef45968,0x7dc6abd9),L_(0x350f562a), LL(0x5aafa6a4,0x60865fae),LL(0x9a3d30f8,0x9c28ce9f),LL(0xd3cb6b76,0xbbb47176),L_(0xcc25a2e4), + LL(0xfe93910d,0x86c3c000),LL(0x6c40ba85,0xe4bdec34),LL(0xa7194bcc,0x308782a6),L_(0x6cf8fd8c), LL(0x710f1669,0x1c395e5f),LL(0x5062f94e,0x2f41d5d8),LL(0xe7981956,0x53f886b7),L_(0x9bf4dd57), + LL(0x6d08786f,0x2cc15260),LL(0xf3a184c4,0x84169281),LL(0xa53ab3f4,0x49111025),L_(0x98552ba1), LL(0x8db73cd0,0x49ee5fc0),LL(0x200da23c,0xeb0a609a),LL(0x26497f34,0x752e2221),L_(0x8eb5b297), + LL(0x23ccf8a8,0xb6e96781),LL(0x4941507a,0x183ac489),LL(0x907018f8,0xf0ef92fc),L_(0x43e3f1dc), LL(0xac3eb093,0x63c56eb0),LL(0x2b3e238d,0xf8deac28),LL(0x95f55a52,0x5ae62b60),L_(0x3671e27c), + LL(0x8575f397,0x643e6356),LL(0x1662edea,0x3b84d8ca),LL(0x87159fb6,0xb1bb3787),L_(0x5efe452b), LL(0x04257ae6,0x76cc23a1),LL(0xb2b2c1b3,0xf70dd339),LL(0xd9298acd,0x91c26db4),L_(0x146fe566), + LL(0xac24fa29,0xa4b011f9),LL(0x59f4d702,0x77f2a9a2),LL(0x897d9044,0x0ed14d52),L_(0x4303ea08), LL(0xa6397973,0xe2308c70),LL(0xa55b4023,0x72030316),LL(0x664a26ca,0x291e9903),L_(0x1def6c61), + LL(0xd3c9cb6b,0x38755541),LL(0xe7d9324b,0x4ad0effa),LL(0x5c237f2b,0x75646bb1),L_(0x298572b4), LL(0xa07ca1d4,0x7f647322),LL(0xdc1aadce,0xe24ec66c),LL(0x819d18b8,0x95c255a0),L_(0xe3f3efff), + LL(0x82098650,0xddb2d835),LL(0x7218b180,0x41d055f3),LL(0x92d42e0f,0xe2f17df9),L_(0x2b71d06b), LL(0xc2c8d2a7,0x061a667a),LL(0xabfafbb4,0xcc790c1c),LL(0x0eb1df27,0x6dc9536e),L_(0x250d65a7), + LL(0xf7ca40a0,0x55a94793),LL(0xe5044ae1,0x6bbc768a),LL(0x1ef210b3,0x166d8931),L_(0xa28dae32), LL(0xa0bcf015,0x621f63a2),LL(0xcd66892d,0xf790fc9c),LL(0x60203dd4,0x667b1fa7),L_(0xdb4559a6), + LL(0x2884e5db,0xc59a883c),LL(0xb096e972,0x47f89d64),LL(0x780aeefd,0xd528d23d),L_(0x8809b35e), LL(0x9052392a,0x6e9ee32f),LL(0x3da64ad7,0xb5fe2f01),LL(0x1d21d327,0x2883e925),L_(0x66ba0553), + LL(0x8d7cbdb7,0x32e5cd9d),LL(0xefaa43d2,0x2385860e),LL(0x43f0f911,0x4b6038f7),L_(0x3c00934e), LL(0x8ece6e05,0x40da7078),LL(0x76e3e8ff,0xa20a6649),LL(0x1a09dbaf,0x709cfd74),L_(0x1f695127), + LL(0xa2ce00d9,0xa63a8b02),LL(0xa213f1e6,0x35cb5edd),LL(0x706b0821,0x4173e4a2),L_(0xb5a5074f), LL(0xaae1c287,0xaa0caca3),LL(0x36399f11,0x3c3b3e24),LL(0x34698533,0xe3b66255),L_(0x8ba6b5aa), + LL(0x4bf57c52,0x164519e7),LL(0x3666d383,0xb55fe2b9),LL(0xd37e29a7,0x68e8549f),L_(0x34e4d0d6), LL(0x9f4b4c9a,0x8f691f6b),LL(0xf4164295,0xa382964d),LL(0x49524978,0x7d3fe7f2),L_(0x64f14bbf), + LL(0xb90d4bd5,0x087b646e),LL(0xe6a8abf6,0x53865f42),LL(0x2d659d4e,0xb9db500d),L_(0xc5eb1e84), LL(0x17597d7b,0x6ae5d43e),LL(0x8df2f6d4,0x3a5e9fe1),LL(0x4204e314,0xd91a271a),L_(0x236e44d2), + LL(0xcfce86ab,0x16c2854c),LL(0x10d4f976,0x3999a9a1),LL(0x5fd18a95,0xa1e55106),L_(0x40835bcf), LL(0xd7192595,0x7c17c721),LL(0x8a484397,0xf6856acd),LL(0xdb82b841,0x39017815),L_(0x2d60844b), + LL(0xf155113e,0xc2b2be30),LL(0xf31e524e,0x1e571838),LL(0xa98b5f59,0x17a7266e),L_(0x169ce88d), LL(0x02e9f20b,0x9c53a705),LL(0x47373259,0x476b9b10),LL(0x144091ad,0x98e56e88),L_(0x2198d3c0), + LL(0x02f6a9c7,0x461d2e37),LL(0xb1b023de,0x6fe1c2f8),LL(0x8835c958,0xaae29f67),L_(0xb8b906ea), LL(0x1573f8b1,0x2aae0885),LL(0x2db13030,0xc9a1230d),LL(0x930776e9,0xd3f32fd7),L_(0x55a855db), + LL(0xdbf89587,0xfeac4726),LL(0x4978e226,0xe907285f),LL(0x084c5773,0xc72ade03),L_(0x25c0e8f3), LL(0x155c7e91,0xe4fb90c2),LL(0x804fb7e0,0x833af299),LL(0x87b3ac91,0xeeb712d2),L_(0xd0aa3910), + LL(0x08fa5176,0xad400dee),LL(0x04db731c,0xdb50d10c),LL(0x8bbf71ad,0x1322cb4e),L_(0xe4d0dedd), LL(0xb7ef36d3,0xa5571f80),LL(0x8b5c198d,0x938aa53f),LL(0x6a106d1c,0xfee7b99f),L_(0xc1056e3d), + LL(0x0d738fbe,0x9340ad37),LL(0xbe6493b7,0x923ef854),LL(0x1a934a84,0xdc9bcb14),L_(0x38e82bd2), LL(0x3ed1d410,0x3e3d69c5),LL(0x2651cbf9,0x8cf302d1),LL(0x60492caf,0xee0ad716),L_(0x0f316e4d), + LL(0x310d3207,0xc40b6b35),LL(0x61d744ad,0xf6b370b9),LL(0xcf2da667,0x4a7e6061),L_(0x06067357), LL(0xd8d48241,0x186ae81f),LL(0x579e8d1a,0x2133c197),LL(0xa42e3f3d,0xf8b9b6ca),L_(0xa88cfea9), + LL(0x9ac56b2b,0x55759656),LL(0xae1ee373,0xd47c70af),LL(0x0f258220,0x06bffc7e),L_(0xe45fbdcd), LL(0x453179da,0x92ffe53d),LL(0x14a22c11,0x2ac0af28),LL(0xb924dbee,0x7afd903c),L_(0x16b2256c), + LL(0x7d2123b4,0x61435520),LL(0x99b497e3,0x2b094144),LL(0xeb4a95b5,0x09b8c6a3),L_(0x83a43e9d), LL(0x54e52455,0x646b89b4),LL(0xa8391c1c,0x2640a632),LL(0x17b930ad,0xa457233d),L_(0xc396f0d7), + LL(0xbfa90c2c,0xfad26494),LL(0xa527afca,0xbfe60268),LL(0xcd907456,0xb3e48eca),L_(0xfa048d0a), LL(0xdf9ab845,0x94fe7551),LL(0x9a3b8a36,0x8ed01907),LL(0x5a97c45a,0x5dbf5641),L_(0x9c8b0d43), + LL(0x9dc2ad9a,0xe9019e0a),LL(0x0599e3f2,0x588acbee),LL(0xfff03f99,0xc48117a2),L_(0x5ebb6635), LL(0x6e713363,0x59d13072),LL(0x42329388,0xc63c7628),LL(0xefa02cb7,0x39d7f934),L_(0xb944b464), + LL(0x5f6d09bb,0x2361ada1),LL(0xfbd96586,0x5419445d),LL(0x6f750612,0xa30fa392),L_(0x9628fd28), LL(0xd59c34bd,0x32785073),LL(0x45fc9890,0x72c99605),LL(0xb431b539,0xeec4de29),L_(0x30b9fd98), + LL(0x97507c27,0x0c5debbb),LL(0x9bcaab04,0x3c42310d),LL(0xa01f4d9e,0x5fdffd07),L_(0x3c6028a2), LL(0x5e2bab3f,0x4ce43e20),LL(0x5cbd1742,0xe71cd8a2),LL(0x0c4858ed,0x8bd12b19),L_(0x2e0122ff), + LL(0xb6380f04,0x59339488),LL(0x3572c4ae,0x920e2111),LL(0x0fbdd870,0xcf927c12),L_(0x44d377fb), LL(0x4ea01a51,0x20ee4123),LL(0xd8592d08,0xeeedf23a),LL(0x65fa7940,0x82b08346),L_(0x605e1aba), + LL(0x34651056,0x1098cb76),LL(0x90c069b0,0xd6a624c1),LL(0x5fd458b6,0xb856ae59),L_(0x95dd191d), LL(0xd687ba3e,0x33a82abc),LL(0xd59abb2e,0xa949d734),LL(0x1de89776,0xc44fd2cc),L_(0x93c93519), + LL(0xc54039f8,0x1ad9c99d),LL(0xcf76892e,0xa0ae59e5),LL(0x5269a308,0x65e4eb8f),L_(0xdc31876c), LL(0xe212c371,0x2c3e1f0e),LL(0x2e889354,0x8869d4e8),LL(0xc88e85fd,0xbc2a9592),L_(0x9c6eb56e), + LL(0x0f4e6e43,0x2bfeb37b),LL(0x46cf4463,0x6bb45b1a),LL(0x53c918ad,0xb8bd0546),L_(0xf78b526a), LL(0x0e33f7bd,0x9e58beb5),LL(0x55fd6204,0x26578e47),LL(0x986f5206,0x90135d8a),L_(0x02f82c46), +}, +/* digit=3 base_pwr=2^21 */ +{ + LL(0x4190a604,0xec5c8b6b),LL(0xaee61346,0x8bca6d9b),LL(0xba9eddb4,0x3ebc0dcc),L_(0xf76451ad), LL(0xa709d0a1,0xc3d3be2f),LL(0x369d1d0a,0xdae0ded6),LL(0xf474acd5,0x965b8988),L_(0x6aaa8ce1), + LL(0xfb05e5ed,0x7df5ec60),LL(0x04a47fd1,0x2d8170fd),LL(0xd08f651c,0xe08fb325),L_(0x56ea78b7), LL(0xa2b2a174,0x20fa30ea),LL(0x02c33226,0xd169c10d),LL(0xc7e20c19,0x366327ee),L_(0xe33001fc), + LL(0xf62aa1be,0xe359773b),LL(0x95f962ee,0x2b782c07),LL(0x31df9439,0x80e166af),L_(0xa27a128a), LL(0xa7a7bd41,0x50ca47c9),LL(0xfce8c109,0x8ed45ebd),LL(0x4a7eebb4,0x03e9fc9d),L_(0x98c4b4f9), + LL(0xaba113fd,0x0e6fb34d),LL(0xf4000e17,0xe2da2a96),LL(0x491a39ca,0x54e2ec96),L_(0xcc021cc8), LL(0x3d0946da,0x6f792b36),LL(0x90fd2002,0xd78ca8ec),LL(0x13094248,0x286898ab),L_(0x477c2d6f), + LL(0xb354ae4d,0xf18c6f77),LL(0xb0291000,0xf6851269),LL(0x7bd039dc,0xfb3809c0),L_(0xf376bad9), LL(0xc99f8945,0x7c71a341),LL(0xf1d99065,0x21f4ed21),LL(0x4c371e86,0xa650cd97),L_(0x1713e005), + LL(0x8e7cfcb1,0xfcae4f5c),LL(0x4c95ab42,0x62196ce2),LL(0xc9a4fa0c,0x83e1de7b),L_(0x9ba799ce), LL(0x8f0bc09b,0xfa78f5d2),LL(0x66a4ac75,0xda32a5ed),LL(0x23d217eb,0xf8df814e),L_(0xb78e1f71), + LL(0x80a1f117,0x670e56db),LL(0xfafc5e68,0xd1c521aa),LL(0xcaa4c48a,0x41b37b7f),L_(0x1023ff48), LL(0xf6421dcf,0xafa1a3e7),LL(0x6e8a0024,0x5c6bc442),LL(0xbc2935cf,0xbe169cc3),L_(0x21e4c2e0), + LL(0xe587615a,0x308bf446),LL(0x71fb47df,0x8047e56a),LL(0xdb6a36f6,0xe8293346),L_(0x5a6836ba), LL(0x93a30fbe,0xcd2f5b41),LL(0x8ffe1e43,0x0b7b259f),LL(0x3d9fe740,0xaafd7816),L_(0xae0c0484), + LL(0xc167cd8e,0x0d3f30f7),LL(0xed56f42b,0x3b140b83),LL(0x6554fb45,0xc105ccc8),L_(0x1a608916), LL(0x69c72228,0xf0450467),LL(0x7e79aa99,0x88e4350a),LL(0x1e0bbd29,0x6f9f2dff),L_(0x8d2ecd98), + LL(0x1e9d015f,0xf4db6e3d),LL(0xc9229b44,0x60df1944),LL(0x91e766b6,0x2499a2cf),L_(0x714cb7d4), LL(0x0468acbf,0x7a3c83da),LL(0x84752dd2,0xfeedfeca),LL(0x95c8fd77,0x9f35aedd),L_(0x1a75bcf8), + LL(0x1d85a175,0xefd882f6),LL(0x0c28af72,0xfcbc0529),LL(0xff17a943,0x6d3cb4e6),L_(0x20b4e7ca), LL(0x1d223312,0x680691e3),LL(0xe8dd8d7e,0x2d7d7412),LL(0x3d6056c3,0xf379f1df),L_(0x82422458), + LL(0x5a2e3386,0x66a60d5f),LL(0x1f8d438d,0x08231783),LL(0x2621fe6f,0x282e92dc),L_(0x8a0f2c3d), LL(0xa53267dd,0x48813229),LL(0xc6a7def2,0x18f63ff8),LL(0x25a10917,0x06fdd515),L_(0x641475d3), + LL(0x866c8a6d,0xb47ff8e8),LL(0xbec1f669,0x360eff22),LL(0x64925640,0x392f59d5),L_(0x2adaf8c7), LL(0x18c35e30,0xe8c753dd),LL(0xb77a997d,0xeef6fc00),LL(0xdf425955,0x3afc5278),L_(0xa7269ca7), + LL(0xedc68cb8,0x69d044d7),LL(0x9eea78e2,0xb3b58151),LL(0x8eb64dda,0x49c378c0),L_(0x9a66415d), LL(0xaf6b16d7,0x47e931cc),LL(0xb92a119a,0xcb7189bc),LL(0xa9051236,0x2aca5249),L_(0x035313dd), + LL(0xad339987,0x9bcc65a5),LL(0x9643d751,0x278964b2),LL(0x7810f1da,0xf8f2be96),L_(0x4bddc049), LL(0x7e983037,0x59fa9f23),LL(0x1f58def7,0xbc86f69d),LL(0x918a9c4a,0xb726eddd),L_(0xdda12d3a), + LL(0xac70d024,0xe4e20984),LL(0x4a8195f8,0x0ffac935),LL(0xcce8d103,0x6efdaa99),L_(0x354553cc), LL(0xea3876da,0xa0b5e29a),LL(0xe8f12ba7,0x65ae7e77),LL(0x9fa645d3,0x93cb4927),L_(0x2a27d9c3), + LL(0xac6cb27c,0x414e945f),LL(0xbed08489,0x73d9e98d),LL(0x4809a03f,0x2d14c85e),L_(0xf63371a1), LL(0x0b05bc28,0x74a3b39b),LL(0x023622e6,0x003b8462),LL(0xed2336e3,0xc6f085ad),L_(0x40dec42a), + LL(0xd9687e2f,0xd82a965c),LL(0xb6bcd68a,0x088401a5),LL(0x068fa075,0x07ba4f19),L_(0xc382c95f), LL(0xf91cec30,0xb6cda991),LL(0xc0579f61,0xa7b1febd),LL(0x50d069b1,0x5d3d8cd6),L_(0xf0f3759a), + LL(0xf8949e75,0x5c71b954),LL(0xcebf4040,0x28006acb),LL(0x6ba62a11,0xe08cb4da),L_(0xef9d9258), LL(0xc4d932f2,0x9059e525),LL(0x9950d8ec,0x72f4258d),LL(0x4cc4c854,0x0a971745),L_(0x1734dd44), + LL(0xdaf48ca3,0xa543ba94),LL(0xfd2fecf6,0xd71707e4),LL(0x809ef386,0xe30d9ba3),L_(0xe5349c3f), LL(0x69eeeae5,0x6910f9ee),LL(0xfcbbeb95,0xe8c90ba5),LL(0x1896385f,0x12c1a610),L_(0xedaaa3d5), + LL(0x2d268c8c,0xd0a57e22),LL(0x90fb174f,0x907afb7b),LL(0x927d9d98,0x4fb8839b),L_(0x60a57b93), LL(0x967900e3,0xa9ff207a),LL(0xb6defcc2,0xee9dbe18),LL(0x1bad27e2,0x37d4f91b),L_(0xababf8bf), + LL(0x6069565c,0x7e61e032),LL(0x6cf9006f,0x1034f977),LL(0xa6921003,0xf41e571f),L_(0x2a44d157), LL(0xeaa8bfa6,0x77a67171),LL(0x3fd9dd59,0xf9f34af4),LL(0xa78106eb,0x7aed89ac),L_(0xa742544d), + LL(0x8c1c931e,0x2befbabd),LL(0xf8ee2ba0,0x7e64aafd),LL(0xb97c5d4d,0xf99509a4),L_(0x621a29df), LL(0x7372eddd,0x47ce48d3),LL(0x35978129,0x28b7f368),LL(0x1fd874f8,0xfdab8a95),L_(0xef2a56ad), + LL(0x7f6337cf,0xe5943606),LL(0x03a9d402,0x21b5b09f),LL(0xa28353d5,0x917b4519),L_(0x77945c21), LL(0xfd5ad005,0x73dbf462),LL(0x9833b4c9,0x7f25c9d4),LL(0x3b50b66b,0x35b3bf78),L_(0x2effa35c), + LL(0x4b668d6c,0xbe5e047a),LL(0x70ba7d5a,0x2bc0144b),LL(0xee508f32,0x9d2116d4),L_(0x64d1a213), LL(0x5d9a5638,0x8ab6e9cc),LL(0x4c4e14fc,0x954172ff),LL(0x7ca23907,0x9fe1053c),L_(0x8104e1b1), + LL(0x711be7f9,0x4c525b55),LL(0x5f99b985,0xac72c62e),LL(0x962efa73,0x92c58056),L_(0xf4fbd970), LL(0xa38c53ee,0x917ebbe6),LL(0xbab078e6,0x6ab0ae35),LL(0x25682606,0x0603c2dc),L_(0x17325a00), + LL(0xb1a2e954,0x22b420c7),LL(0xd56f6402,0xffe309b5),LL(0xda6ba55a,0xdce7859f),L_(0x2da047de), LL(0x96d10a76,0x057f1caa),LL(0x6e8c3c16,0x72ae9f6f),LL(0x8794d9eb,0xab64c20f),L_(0x63b937d3), + LL(0xb289f4d8,0xd9e2a94a),LL(0x0f4cb441,0x4c9fd385),LL(0xecfdb1d1,0x0675dcf1),L_(0x03f2ed02), LL(0xaccd6ef2,0xfad7706e),LL(0xfcfca764,0x0667c6b2),LL(0x7a5f797a,0x2207359c),L_(0x5a7fb514), + LL(0xf3f519e3,0x06628ac7),LL(0x0a0ab6cc,0xe38c99a2),LL(0x86774ade,0x8f775fea),L_(0xfc59c719), LL(0x1bf25ffd,0xf9325842),LL(0xb246d04d,0x34f4502c),LL(0xabcab086,0x064e829c),L_(0x1a866a52), + LL(0x5fefab0c,0x06295d26),LL(0x523fd3f9,0x234674d7),LL(0x9c70b1f8,0x01e05c18),L_(0xd80fb580), LL(0x352749e3,0xee248ea9),LL(0xe2fc4a35,0x3b3c0b4b),LL(0x7556a699,0x4df0080e),L_(0xb60da54c), + LL(0xc8a345b2,0xaf868ebc),LL(0xaaab8ef9,0x9a7fcfbb),LL(0x959c94fa,0xea271ece),L_(0x14c6b171), LL(0x1ea91da9,0x252b7410),LL(0xb76f94ce,0x560e8134),LL(0x8ada4812,0x4d0eaa23),L_(0x48d2e1e9), + LL(0x11e78f97,0x55ff3762),LL(0x52966809,0xa90b8b11),LL(0xb557ab25,0x98bfd175),L_(0x7a838285), LL(0x453f5ec1,0xbd8cc123),LL(0xb8541423,0xe53f50d0),LL(0xaf6d61de,0xb809c27b),L_(0xbc9d881b), + LL(0x22d1e308,0xcafcf6b1),LL(0x3494f12d,0xb5d40ef0),LL(0x602ae873,0xb5bef707),L_(0x8f093526), LL(0x47aa28a1,0xc7e7a44e),LL(0x363aec6d,0x9f424f8c),LL(0x39a88470,0x08171c6a),L_(0xc8475406), + LL(0xd43ff3e7,0x5bfd96fc),LL(0x35837bfc,0xa9d5f891),LL(0xeb92efe6,0x5ac4853d),L_(0x7f7ee96f), LL(0x90443656,0xe8ead180),LL(0xd124bb70,0x84a289e4),LL(0x98bbea4f,0x8bd0878a),L_(0xe0a10a80), + LL(0xd233645b,0x2cec3fe7),LL(0xc6b4e341,0xc6974f7f),LL(0xc7a73902,0x0b481a08),L_(0x8fcafe71), LL(0x9eb32486,0x9b2ec275),LL(0xb1171362,0xdb3bea45),LL(0x0653d374,0xab7f0ce3),L_(0x652219fa), + LL(0x89e007fe,0xa77a41f1),LL(0x14cba611,0x6059351c),LL(0xed2ec819,0x02bfa9e8),L_(0x1e353c31), LL(0xa9065e32,0xd4e23efc),LL(0x243c2ec7,0x9c16b8bc),LL(0x465d2c9d,0xa1b1cda1),L_(0x45e1763a), + LL(0x54df839e,0xb93666fd),LL(0x5820d4be,0x1fed47ac),LL(0x7e8268de,0xbda2efca),L_(0x61307a90), LL(0x428af14d,0x9ee2b07c),LL(0x72170b4f,0xda44c29e),LL(0xfffd22e4,0x296f1761),L_(0xa25a8bfc), + LL(0xb42b17e3,0x6d4c5ddd),LL(0x83cf7f8a,0xabe94b27),LL(0x89be3310,0xde98e510),L_(0x6e879f48), LL(0x9d4ca35f,0x96df2661),LL(0x45c77761,0xb28f2b85),LL(0x96395d9e,0xc8d9fdcf),L_(0xcb9113a6), + LL(0xdbc075c4,0x54d26d6d),LL(0x6dd2d9b4,0x23c171f1),LL(0xca47c96b,0xecedbb06),L_(0xfce5ed3b), LL(0x31539157,0xe9bac065),LL(0x43e31db0,0x552f0920),LL(0xef1cbf3a,0x2cb42440),L_(0x73830d80), + LL(0xed539df2,0xd5c7355c),LL(0x11447b1e,0x5ced43ea),LL(0x10d60144,0x9618c03b),L_(0x6045d427), LL(0x8b9bcb4b,0xe3767fbb),LL(0x4785c2ae,0x2a5c6f77),LL(0x8a009344,0xa23cb6b8),L_(0xaa1eaa93), + LL(0xfd4a0f5a,0x7ea78a85),LL(0x5acc9416,0x258d360d),LL(0x951a2674,0x7b3935b0),L_(0xf2465f38), LL(0x75e447f1,0x80a9bfbf),LL(0x13abd588,0x8cd933d2),LL(0x1c914702,0x8da2914d),L_(0xb5dceca9), + LL(0xbb8f1e8f,0xbb61ba00),LL(0x68589fbd,0x018ee54b),LL(0xf3ed084b,0x9b33e2c5),L_(0x4187a448), LL(0x90c49202,0xdac991d0),LL(0x0adfa0ba,0x06286fbc),LL(0xfa92d2f7,0x8e51716d),L_(0x869552ad), + LL(0xcd0c67a8,0xe3fcc974),LL(0x1fe4efe6,0x4936eae1),LL(0x38aec941,0x49b20db5),L_(0x31b2e360), LL(0x8806a868,0xfd73d3b4),LL(0xf81a57a6,0x8553e67e),LL(0xbb731824,0x5f399b3f),L_(0x063ed190), + LL(0x04dbb079,0xd2ce86a5),LL(0x396d5523,0x636ee6c6),LL(0x5c029f32,0x90ef1fcb),L_(0x8504873f), LL(0x05c18efc,0x812addf4),LL(0x2fdbed7c,0xdf2ff660),LL(0x34c03bb8,0xb1740945),L_(0x56f08b04), + LL(0x18fd7ff8,0x8a416ad5),LL(0xefbbc814,0xb7d2ecb7),LL(0x8226e0af,0xbb5fb220),L_(0x9bdeb90c), LL(0xcaee42bc,0xb76b0d06),LL(0x184432d2,0x31322587),LL(0x678f208f,0xa07b89ea),L_(0x829d1d5b), + LL(0x5899ed85,0x45083585),LL(0x2e43b3ad,0x7217b173),LL(0x04f9e7f1,0x59198da1),L_(0x2293f74e), LL(0x11d6cf0f,0x908d5768),LL(0x06e9468f,0x51f15b45),LL(0x7dd32a0d,0x58045077),L_(0xa2294102), + LL(0x885c6637,0xf7bef8ba),LL(0x74d38792,0x478b8ac8),LL(0x58aa9fce,0x9d21686d),L_(0x503a4ce7), LL(0x541006e4,0xdfc86dcf),LL(0xc2238c04,0xe98f7f2c),LL(0x73e745a7,0xfc620932),L_(0x1dcc2ceb), + LL(0xd49816e3,0x68c388d9),LL(0x697ef53b,0x741de91e),LL(0x3b976a04,0xf24259fb),L_(0x4259cf50), LL(0xcbbf2685,0x56659522),LL(0xe4e3f7fe,0xea31954d),LL(0x43c8b17e,0x3d1de35f),L_(0xa56dc0b1), + LL(0xddd685db,0xa178f2e3),LL(0xf8a89c1c,0x859daa45),LL(0xc3965546,0x65f74f36),L_(0xef984d98), LL(0x31952a51,0xd0b694c6),LL(0x6a5e5ad0,0x4f51b3a2),LL(0x20b2748b,0x54b61458),L_(0xc1437804), + LL(0x91be987f,0x92390774),LL(0x9816ace7,0x31cf372f),LL(0xd8dc53f9,0x490c450f),L_(0xbaf48158), LL(0x604fd793,0xb689243f),LL(0x86f30400,0xb2d8f482),LL(0xdc31ef3b,0x506dd3c6),L_(0x1b18f132), + LL(0x1196c48d,0x6755b53e),LL(0xe39a017a,0x6a0e74ee),LL(0x2da63e64,0xaa8ea0cd),L_(0x7634dff0), LL(0xb013649f,0x1ab0635d),LL(0xbd02ff70,0x9858a79c),LL(0x65938865,0x3acb5b92),L_(0x99e5f181), + LL(0x28b2de14,0x6b120542),LL(0xc4ac30ff,0x5ea493a4),LL(0xbe9b2f46,0xcdd3cbfb),L_(0x0c6270af), LL(0x543265a6,0xf3a605c4),LL(0x83e752d0,0x92bcbe09),LL(0x1d71ae8b,0x4ad0176d),L_(0x6507463d), + LL(0x183493f1,0x3e8ef423),LL(0xc54a293d,0x065effb9),LL(0xa9ccf6fd,0xb0500b2e),L_(0x88529f19), LL(0xfb31da5e,0xe16bd325),LL(0x5014e1e5,0x2bb46b57),LL(0xcdab125c,0x8179b8bc),L_(0xedd40925), + LL(0x54823e46,0x263960f2),LL(0x24b40650,0x21ab8a5f),LL(0x193b2d98,0xb940533d),L_(0x8d0db660), LL(0x7586193d,0x671090ab),LL(0x08db8178,0x216acee5),LL(0x6bb91540,0xcc4fc3a7),L_(0x37f48327), + LL(0x0e69184e,0xe86f6c8e),LL(0xbcf25a98,0xe88688e1),LL(0x9beec95a,0xd008257c),L_(0x3d97d91c), LL(0x35395ef5,0xacddfb2b),LL(0xa6942cd6,0x086efe97),LL(0x8a1a0ad3,0xb4d51fd4),L_(0x59c722f5), + LL(0xd8510369,0xd5dedcbd),LL(0x14ba55bf,0xab06f6a4),LL(0xbd599498,0xc5aa099e),L_(0x69ec087e), LL(0x32f40f7c,0xad93fbe0),LL(0xf1d12a93,0x40b951f1),LL(0xf5751752,0xfc37b6e1),L_(0x0c14e715), + LL(0xaae77beb,0x58b91be1),LL(0x3f7b0006,0x6ad767a4),LL(0x9b47cd71,0x24d37cee),L_(0x3b245f34), LL(0x70ac8b26,0x2da13b60),LL(0x88aafd82,0xbadf18ac),LL(0x62a037c8,0x78224e07),L_(0x8d974011), + LL(0x6a2f2199,0x1578e706),LL(0x8412ecab,0xa273c222),LL(0xe95f128a,0x2b3feceb),L_(0xccb455b8), LL(0x4d1450ab,0x247fea5e),LL(0xc24d8d6f,0xcc56e420),LL(0x32f2d692,0x001d3250),L_(0xf3727734), + LL(0x3d1a2b09,0x1192eb20),LL(0xde270684,0x194cd935),LL(0xcbfbf813,0xe3ba3ce4),L_(0x9c6fa3d5), LL(0x8d6355ce,0x36393a36),LL(0x18945358,0x6d3b3d2c),LL(0x50d98fcf,0x2da52d36),L_(0x23e01cc2), + LL(0xd68d0be3,0x87600722),LL(0xd90cd86e,0x6a898b0e),LL(0x64b1c25c,0xf26df29e),L_(0x5696f04a), LL(0x0562d2ad,0x116fb9cd),LL(0x3440763e,0xadee6e3f),LL(0x19e073f7,0xdcd869ef),L_(0xe748bcdf), + LL(0xd9747843,0x484e4a18),LL(0xd6812dc5,0x2181acfc),LL(0x09d6f90f,0xae25cfad),L_(0xc6209409), LL(0xb6aa9130,0xeb2922c9),LL(0x38ddc5b3,0xbe3728a6),LL(0x3b8ce6be,0x04eb6141),L_(0x3d9bb134), + LL(0x618ad07f,0xc862e8d5),LL(0x69540999,0xca247f9d),LL(0xaa268736,0xbded92f5),L_(0x29192f4b), LL(0x9481e65a,0x42b36d96),LL(0xe3139379,0xabfa2ad1),LL(0xd3f46f02,0x1cbbeccc),L_(0xe59fce03), + LL(0x2b94f0ff,0x608a4b71),LL(0x9cfd64f9,0xdd35cef7),LL(0x236745ba,0xb4d5d169),L_(0xdfa8120d), LL(0x03f0b33b,0x826e0d5f),LL(0xa5992b36,0x31eb9950),LL(0xf2cc0540,0xd954458a),L_(0x66bed66c), + LL(0x2c486c0c,0xa5afc024),LL(0x8a61a33e,0xb063a531),LL(0x38aa1ed3,0x1558aff7),L_(0x09b640f8), LL(0x904e9ed8,0x3494f0b6),LL(0xd2a51717,0x365e3d04),LL(0x0f32df2c,0x5c8f30d5),L_(0xffb044f0), +}, +/* digit=4 base_pwr=2^28 */ +{ + LL(0x6a7ebdd2,0x676d690b),LL(0x5519e7c3,0xf169ac19),LL(0x70ef4a18,0x3fe9e66e),L_(0xd0bcdec7), LL(0x9b515116,0x6b6b0581),LL(0xe52dad64,0xd0e88db1),LL(0x52c4fcf1,0x31c324fa),L_(0x5e6c0907), + LL(0x403677bc,0xe67af8a2),LL(0x3023628a,0x0b4be307),LL(0xdcb51813,0x88ce9bf6),L_(0x9d1c6e56), LL(0x0f788e99,0x8400154f),LL(0xfbca2d36,0x768edb49),LL(0x80428f26,0x277da6c6),L_(0x1b78829a), + LL(0x6933e247,0x8f4bec34),LL(0x96def5c8,0x13b62771),LL(0xc9121c92,0x543662c4),L_(0xf4ea3390), LL(0x1090a59d,0xc62e7c6f),LL(0x4bb3b3cd,0xa38394a7),LL(0x19c57ed5,0x4823e8cf),L_(0xbab65589), + LL(0x61b100d3,0x416f9398),LL(0xaf6ede17,0x8d0f5a47),LL(0x11e5f34b,0xe047da80),L_(0xffc1a9b5), LL(0xc598885b,0x455959c9),LL(0x6cd277fc,0xe69bdd02),LL(0x6bdbe44c,0xd3377883),L_(0x16b926d8), + LL(0xe93c7bab,0x23daaa17),LL(0x8729f3c7,0xcf91e74b),LL(0x64e1431b,0x1ae9e235),L_(0x2e21bb9c), LL(0x2c005910,0x0ba0b46e),LL(0xdccebddd,0x1ee4dc17),LL(0xe61cc51e,0x93805289),L_(0xbc0293c3), + LL(0xa33f496d,0xc65b315e),LL(0x5526e12e,0x25723786),LL(0x2ca2c551,0xc7e943c6),L_(0xea468f5d), LL(0x1845a432,0xb33bb1fe),LL(0x1b868344,0xf5b5a1e6),LL(0xbb41c15a,0xf5bd0a9c),L_(0x65efbcd3), + LL(0xe9864245,0x73ab2381),LL(0xbfa54a0d,0x0ed12b17),LL(0xe3536d47,0x0a040c86),L_(0xfbd5dc3b), LL(0x550f5801,0xa16b2d10),LL(0x96655492,0xd20e3020),LL(0x40239ea3,0x13feb317),L_(0x63a5974e), + LL(0x1220685a,0xd53242c9),LL(0xcca2c5ae,0x1c2921f4),LL(0x765eedc1,0x84f582be),L_(0x7177d048), LL(0xf8de62bf,0xf9eee520),LL(0x22fe2632,0x209f3c9d),LL(0x721633cd,0xc2e9678c),L_(0xada44b67), + LL(0x852acfe9,0x92b626ca),LL(0xa685284e,0xf571ade2),LL(0x918c76fa,0x5b7490f5),L_(0x375ca970), LL(0xdf1662be,0xbb1e6d8b),LL(0x7fd11993,0xd821c744),LL(0x9a06de8a,0xcaec5c2c),L_(0xc7374f40), + LL(0xc1321d2b,0x74538108),LL(0x95469f9e,0x2f1d65e5),LL(0xc8d1c486,0x8ecd7a1e),L_(0x9b0dcc2a), LL(0x3466245b,0x700ed4ab),LL(0xc2bf54a5,0xfa32badf),LL(0xb5a66db1,0xfc4883b2),L_(0x3315b46b), + LL(0x962b479d,0xc4a7efe4),LL(0x6dc8fec6,0xa0d20fc9),LL(0xd0ef4040,0x76eabbd4),L_(0x4ba76aee), LL(0x17a75082,0x4bc6d910),LL(0x75ee3974,0xff0f721d),LL(0x36e896fb,0x6146e5ef),L_(0xc74f818a), + LL(0x6fd30020,0xe508cb13),LL(0x06427fcd,0x80a73390),LL(0xcc02c93c,0x6ad8073c),L_(0x54e00967), LL(0x5f92bcf7,0x8847a6eb),LL(0x9b0cb317,0x758f4aa3),LL(0x40b45a83,0xdb4ada91),L_(0x6bbb2114), + LL(0xfff55c76,0x661c826d),LL(0x3203a031,0xb311e145),LL(0x6bdaa80c,0xcad158e9),L_(0xff7c21ec), LL(0x4502f455,0x239ecee1),LL(0xde88ad58,0x2b344eb2),LL(0x161e1cf8,0x02ce3f80),L_(0x966ad037), + LL(0x0009b91a,0x53d14c44),LL(0xe564666e,0xcbb01019),LL(0xdf767e4c,0xecc4eabb),L_(0x9ba1897c), LL(0x70cf6d6d,0x0ede1159),LL(0x23335d25,0x4162258e),LL(0x5ed679cb,0x5e706cdc),L_(0xf3d53802), + LL(0x33b4ee65,0x78fd4645),LL(0x97cb4549,0xb807ac0a),LL(0x7e760bcf,0x923934bf),L_(0x32edf91e), LL(0xaf2ed833,0x918e957c),LL(0xdbf70f3d,0xdf411692),LL(0xde9c3716,0x2c76553a),L_(0x05dea60b), + LL(0xd3a72d09,0xa5e4f840),LL(0x40f8ed7a,0xd9bcbe82),LL(0xf00706d6,0x986b79c3),L_(0xc2b3b652), LL(0xefb01a69,0xa0bb40d2),LL(0x5b0da5a2,0x2d182973),LL(0x57b4bca1,0x43c566de),L_(0xbed5ccfc), + LL(0x2f1c7c7c,0xfb3c8bb1),LL(0x3a9383ad,0x1addae50),LL(0x29206628,0xa115d8d7),L_(0x4c24fdb4), LL(0xcf3f272d,0xf2a0db00),LL(0x786a47ad,0x1a242dce),LL(0x048ec06d,0xbe72542f),L_(0x384605c4), + LL(0x515f5abb,0x995a0552),LL(0x65750201,0x809459ef),LL(0x2a52fccb,0x4bf08352),L_(0x3914cd0f), LL(0x94c73472,0x507e309b),LL(0xf31b1135,0x006ea75f),LL(0xd3410c08,0xde332d38),L_(0xce581939), + LL(0x0560814a,0x971e9a21),LL(0x69ce7f60,0xcf890a46),LL(0x607c23ca,0x8c2f3995),L_(0x16579c51), LL(0xb8358126,0xc384ccce),LL(0x74485fa6,0x62e3cda6),LL(0x637377fe,0x36145889),L_(0x11e8766a), + LL(0xa7ccaba4,0x66ee0e61),LL(0xd1b8ad50,0x8b52d5b2),LL(0x9b192439,0xe2507e26),L_(0x141991a3), LL(0xf0320f47,0x164afe7b),LL(0xe4e590ae,0xa9cd4bff),LL(0x53be168f,0xbd05717a),L_(0x2141999f), + LL(0x82377242,0x84720302),LL(0x464bed04,0x3e8eab14),LL(0x8131dcb9,0x8dffb402),L_(0x465f6f0a), LL(0xca875370,0x7476d50d),LL(0xb1cc70c5,0x40036a93),LL(0xad40e0b3,0xae5c279a),L_(0x17d24656), + LL(0xce0565d5,0x4590aef4),LL(0x186eb345,0x813e461c),LL(0x4d52d217,0x64b75926),L_(0xdfc17d52), LL(0x51483219,0x4046569c),LL(0x97c10da8,0xbd282e8a),LL(0x394837d5,0x5a6b2c1e),L_(0xe88b3478), + LL(0x58408341,0xa28c5b85),LL(0x1b86b157,0xac9ddd98),LL(0xdf706c18,0x62a72bba),L_(0x814b4431), LL(0x7437fa2d,0xafe0e361),LL(0x79e0c72f,0x29283b53),LL(0x0c099515,0x469c2258),L_(0xe132ed6d), + LL(0x1869604f,0x69cd9356),LL(0x18ebfef9,0x161651d5),LL(0x8a8472c4,0xa23f6947),L_(0x9ee50c4f), LL(0x9ca82d06,0xdd9ffcc5),LL(0x4875708e,0x7c9cfaaf),LL(0x8b227092,0xc52e4c68),L_(0x757b5a90), + LL(0x66b5d594,0x661a0707),LL(0x5bc7f7db,0xf0ed8bfa),LL(0x9f814a89,0x68ff82da),L_(0xcbb5731e), LL(0x54b6136f,0x125a9d0a),LL(0xe936a8d4,0xdbfa6d51),LL(0x0914dd93,0xacae0da7),L_(0x00b1f096), + LL(0x41cc5967,0x8e32f402),LL(0xd57a3568,0x99f4bcb6),LL(0x0ba8496e,0xcbcf93d4),L_(0xcd9a9a5d), LL(0xf2f0154a,0x22877383),LL(0xd00df8ff,0xf270905c),LL(0xfeb2c8a5,0x15d8ffe9),L_(0xa7504f44), + LL(0xc56f7438,0x435cbc99),LL(0x7613f44d,0xedb87fdb),LL(0xc1d19083,0x702066e8),L_(0xeee5d570), LL(0x2c1c4234,0x672ee8b0),LL(0x8a5f8dd8,0xfc342416),LL(0xaa518aba,0x34e2bf87),L_(0x37243c86), + LL(0x3bd53fa0,0x38707713),LL(0x464e00d7,0x6251ddc9),LL(0x4c9a0c1a,0xe7e323fd),L_(0xec2175bf), LL(0xd33612f3,0x93f34e7a),LL(0x840bea94,0x0c42a980),LL(0xdc0b91ab,0x548c92c1),L_(0x978906cc), + LL(0x72ed1d98,0x0de1c184),LL(0xa6f53ca5,0x86904dc7),LL(0x45379ece,0xbe4c089a),L_(0xb3444b1a), LL(0x6d5238af,0xa27b3f88),LL(0x46b2be92,0x21e17b1e),LL(0x122542d6,0xd6e7acc3),L_(0x7853b372), + LL(0x01b419d1,0xdb03ac70),LL(0xd178ee53,0xc5304465),LL(0x5f684940,0x11655587),L_(0x7b9ce700), LL(0x196f70df,0x771ca6aa),LL(0x08b0e5f8,0x4a94022c),LL(0xb69fe2d3,0x076b1dc4),L_(0x70769678), + LL(0xecd18f23,0x225dd231),LL(0x3eefc65b,0x23c21f5b),LL(0x43b63acb,0xf5b2b43c),L_(0x4753caca), LL(0x7ec7a342,0x362a2c90),LL(0x73799a39,0x308c8fcb),LL(0x608d86ad,0x8858eb99),L_(0x64620322), + LL(0x82f0ce09,0x8ffc3544),LL(0xd650c2af,0x1414581c),LL(0xf2cd74da,0x2ab74c29),L_(0x764c59d9), LL(0xd4c61a1c,0x8b9a0584),LL(0x88cdc1bb,0xd7114187),LL(0x7b53c8e6,0x4e91e1e4),L_(0xb898c7df), + LL(0xe472f5b8,0x120a4cb4),LL(0x28eb2abb,0xd29feaef),LL(0xa568a4a5,0x1f0a0815),L_(0x7c56e710), LL(0x52d9147d,0x1dd52d78),LL(0xf3e9a259,0x99e7538b),LL(0x25ad4a06,0x8a35cc7c),L_(0x7ffcc9ae), + LL(0x3daaae36,0x214d493c),LL(0x4a1951b7,0xc0f2cff4),LL(0x5e7107a4,0x46024f37),L_(0xba7c8cd9), LL(0x0f39120c,0xd7f5199d),LL(0x73d5ebf0,0x2dba9f4c),LL(0x2755fd79,0x54915591),L_(0xb315426e), + LL(0xbd4c1fd8,0xbe148855),LL(0x0336e26d,0x7f36ce43),LL(0x41ceba92,0x40c686c2),L_(0x0c8b60cc), LL(0xa2355b30,0xeb937195),LL(0x71e0029a,0xa1f3ab05),LL(0xf7d38672,0x90828e20),L_(0x2f8c50fe), + LL(0x37b7db09,0x4de172b0),LL(0xa962d349,0x3f8b6d31),LL(0x1253fef8,0xc76ebff6),L_(0x83810f07), LL(0xc53e233f,0x9a8ef0b3),LL(0xc3896b5b,0x2d43fb47),LL(0x9a94edf5,0xc8891237),L_(0xabdf75f9), + LL(0xf1704460,0x115d7eb6),LL(0x59c71eb5,0x71666f11),LL(0xdfdf4d27,0xcd09d0bd),L_(0x7aa8753a), LL(0x658ac045,0xe75c6300),LL(0x83c2df96,0xa0de2977),LL(0x5ea75aec,0xd9cc6143),L_(0x8bd0d544), + LL(0x2a662d9e,0xcd959485),LL(0xb69374f3,0x5ee58eff),LL(0x69a1ad5b,0x417599f7),L_(0x81e4947e), LL(0xc5110168,0x471554ab),LL(0x30912d02,0x4e2c7d96),LL(0xa6a4af88,0xe49c13ae),L_(0x0170b29f), + LL(0x0a1a147e,0x23d766f5),LL(0x8108f7f1,0x5446ea57),LL(0x737348f7,0x49f6d997),L_(0xbb23c442), LL(0x17464a0d,0xf5ac023b),LL(0x82c1f632,0x1c89d9f6),LL(0x5a4700e1,0x918b1cd7),L_(0xdb771a09), + LL(0xd9c4d8a6,0xfb136a04),LL(0x7d3d9198,0x627e23b2),LL(0x2a546df5,0x9518b383),L_(0xe5ad50b4), LL(0xe74b275c,0x9e8dd1b3),LL(0x8399ea24,0xb28d50a9),LL(0xc1bf5392,0xf0c4dae3),L_(0x3a885426), + LL(0x60cd114f,0x19f30dd8),LL(0xc7c06793,0x0a04952a),LL(0xa5b0d428,0x9fbf17a2),L_(0xedbaf6ff), LL(0xdc88c750,0x2eaf445d),LL(0xd380c189,0x24d076c7),LL(0xb7cb2b9f,0x4da895c0),L_(0x6670ebaa), + LL(0x296bb0cf,0x1e460fe9),LL(0xb75d30ca,0x0a2b9d5f),LL(0x779ddda6,0x70ad87c8),L_(0xd421a064), LL(0x1216c7eb,0x015e5322),LL(0xd6456e5a,0x08a2c2b4),LL(0x5c4f2da2,0xda0125d8),L_(0x2594bcc2), + LL(0x5cbb2272,0x0e27b02e),LL(0x74c5826f,0x651b4dc0),LL(0xf5b545b4,0x89864ae4),L_(0x81d8295c), LL(0xbfa85081,0x0162c99f),LL(0x1d4b46ac,0xc148b34a),LL(0x39fddb94,0x7de290f5),L_(0x1840e143), + LL(0x8d61f3cb,0xe4f10dce),LL(0x0ac9caca,0xd40a9135),LL(0xdb4a6076,0x869e961b),L_(0x41f3f07b), LL(0xc8ea7a1d,0xaed2c272),LL(0xff4dfc57,0xdd26742f),LL(0x49228bdc,0x196c9b69),L_(0xe6e12a4e), + LL(0x5f4b0d33,0x357d26fc),LL(0x222c05a9,0x8bc3e1fc),LL(0x44a306a1,0x6ebde62d),L_(0x25b23700), LL(0x66c5b1cb,0xc2a2d52f),LL(0x28621234,0x67f40398),LL(0x6a1916f6,0x9eb2b762),L_(0x3b034368), + LL(0x97ae197f,0xe2705942),LL(0x168b8890,0xb717c180),LL(0xfe30cc2e,0x7aa89b4d),L_(0x2c096c3a), LL(0x3d1c1e8c,0x1731a1a7),LL(0x027b4fb0,0x4cdde0b3),LL(0x85e85391,0xab7103b7),L_(0x4bd9bc5f), + LL(0x39f1ba7f,0x21c91792),LL(0xb47bf115,0xd1e29983),LL(0x8102c46e,0xaa0fed15),L_(0x1b4f9e4a), LL(0x1df20d6b,0x659104e2),LL(0x3765c7ea,0xd4f49239),LL(0x3a84be64,0xca582a23),L_(0x0c1bc735), + LL(0xce03f04b,0x827606a4),LL(0x419327d9,0x10bcfdd1),LL(0x277b88b9,0xf2e4c5e4),L_(0xcee78792), LL(0xb9213c6e,0x161c54e8),LL(0xa56d42d6,0x2814c386),LL(0x859e6045,0xac244cf4),L_(0x4647ade8), + LL(0xd6446676,0xab795487),LL(0xe4456665,0x5a242338),LL(0x899b5dad,0x8520bc7b),L_(0x8d6d1eb8), LL(0xa366ee1b,0xc17b463a),LL(0xa3b19828,0xa0af7c16),LL(0xddf53842,0x85b5501f),L_(0x8d208a89), + LL(0xc8d6368a,0xcd7b3fe1),LL(0xd2025e3b,0x124ab94b),LL(0x6971b236,0x961e282f),L_(0xedb43ca7), LL(0xb92ac7ae,0xc76e5b2f),LL(0xdc4501c0,0x8df11605),LL(0x0055901f,0x666181f1),L_(0x80bfc257), + LL(0x96fcbeb9,0x745bf74b),LL(0xd548eabc,0x24675bb4),LL(0x531c5deb,0xff3b4dca),L_(0x0b69ba2e), LL(0xd74deba3,0x97100731),LL(0x585b1341,0xa26f7f21),LL(0xb1b02a6f,0x396cb1a0),L_(0xa9fd8a08), + LL(0xc6165fe1,0xc07709d3),LL(0xe9ce0fc0,0xfdb883bc),LL(0xb1a5a67b,0xfd535ec8),L_(0x8218f85c), LL(0x8234b459,0x45bb7b81),LL(0xc140afa2,0x1f4c2d89),LL(0xa1cee750,0xcf55f516),L_(0xdada8f87), + LL(0x0e3c9ace,0x75c44136),LL(0xf44b034f,0xbe92eabb),LL(0xf4ca1355,0x9e459c32),L_(0xdf9382e9), LL(0x4bc1af9f,0xc06b8bfe),LL(0xe071459b,0xe0b85f75),LL(0x7d557c99,0xff091685),L_(0x408afbfc), + LL(0x90ad8902,0x5513303a),LL(0x21006721,0x9b18a004),LL(0x8286d6c6,0x73c22592),L_(0x41ba22e9), LL(0xf6402ecd,0x2bbee8cd),LL(0xb578647d,0xab06f9cb),LL(0x42089652,0xc4a5e5a0),L_(0xed915521), + LL(0x6925cada,0x6d3b1102),LL(0xc49fffc0,0x2e19a20a),LL(0x1593ba2f,0x8c091708),L_(0x1ec6ed8c), LL(0x711ad8d7,0x1a65086c),LL(0xef09f54a,0xc052325c),LL(0x182117dd,0x865d46f7),L_(0x6f7c505b), + LL(0xfb548314,0x7a6b6eef),LL(0x1dd26e62,0xec447933),LL(0x6a3d4ff3,0x74866913),L_(0x908f5992), LL(0x38316d03,0xb9635b34),LL(0x9f9cb069,0x2db27176),LL(0xac233369,0x567be1c0),L_(0xcafbd98f), + LL(0xcf5edce1,0xa18e3f0f),LL(0xa7d8738b,0x05816a5b),LL(0xa26ca8e1,0x00da101c),L_(0x41646cb9), LL(0xe04bccb8,0xa1f2efbe),LL(0xf3c3eccf,0x67b00a64),LL(0xdd0dbd8b,0xc0be2197),L_(0x534aabd5), + LL(0xe5a39b64,0x0a586a5d),LL(0x398f5b5b,0xe22752b9),LL(0xfc9ab49c,0x72fc1c09),L_(0x8776fb50), LL(0x8b426975,0xd51e5946),LL(0x3da2aa18,0xa79d2b5c),LL(0x8d8169eb,0xe4ca8644),L_(0xcedb668c), + LL(0x923206aa,0xd8894d89),LL(0xc4c6cb55,0x992f9c2f),LL(0xd0679c8b,0x0cde00fd),L_(0x6c539dfb), LL(0xf07baea9,0x3f8e094d),LL(0x5d6f7d74,0x3d3b3315),LL(0xbfbe9e92,0xd269b8cc),L_(0xa2a08cef), + LL(0x88e7f568,0x8dcb5c06),LL(0xb11d1a28,0xc8f6e06b),LL(0xa6e90095,0x905b4a44),L_(0xcca85279), LL(0x63aa3a4f,0x1e51e461),LL(0xe2429d1d,0x5fb4cf42),LL(0x35d139e9,0x8725096b),L_(0x6621d9ac), + LL(0x2005767c,0x95ce3a0f),LL(0x0f37bc40,0xb593a6ec),LL(0x2c5a09aa,0x762f3669),L_(0xfe96bf28), LL(0x52242367,0x88d38ecc),LL(0xe7e051f4,0x759292de),LL(0x8843ff7b,0x041ca14a),L_(0x1e08ac30), + LL(0x0062aa13,0x4bff8a1d),LL(0x40899756,0x28b33bc2),LL(0x966aee37,0xb4b3e7e7),L_(0x7120e524), LL(0x7f31f05b,0xc7be6aaf),LL(0x3e4b1311,0xc2c97f7a),LL(0x7401d050,0x1475b928),L_(0xd263113e), + LL(0x3150b4ab,0xd53ede1b),LL(0xbf73ccc0,0xe427c40d),LL(0xf4e2d20a,0xe5dd8b02),L_(0x2ff70604), LL(0xac4f8ba8,0xaaacf4ec),LL(0xf84c5c8d,0x8e6b2783),LL(0xd9aeb4f2,0x4019cd79),L_(0x4c8f57ec), + LL(0x680b826c,0xc0a2ee02),LL(0xc24d0dcb,0xb25a4e29),LL(0x8b33b701,0x3a2939d0),L_(0xffb21809), LL(0xe6feb01c,0xd1311810),LL(0x603004cf,0xa701c8f2),LL(0xdbcecc81,0xeee95871),L_(0x69de26b2), +}, +/* digit=5 base_pwr=2^35 */ +{ + LL(0xb5c4c69c,0xf507ee25),LL(0xab3e1589,0x454a0f0f),LL(0xf21fe543,0x29d7720a),L_(0xdd2cdf95), LL(0x7c33ec2b,0xb4054a86),LL(0xc7aaaa08,0xdb971594),LL(0x8a74b5b0,0x8850caf4),L_(0xdd2d6666), + LL(0xa04cb663,0xf30acc6a),LL(0x8b4f4dbb,0xd7560ec1),LL(0xb8b6110c,0x4f8fab99),L_(0xc43951e7), LL(0x4308e609,0x2c9d6af5),LL(0x7c9841d9,0xca6486bf),LL(0x64c7b963,0x28ca7953),L_(0x9438ea6a), + LL(0xbcc43cea,0x3055b2de),LL(0xf0288d97,0x68461c5f),LL(0x4caf7c30,0x84c1743c),L_(0x499d304a), LL(0xb5b3a780,0x0e121552),LL(0x50cacc00,0xc38f8d3d),LL(0x6cee1666,0xe3236cdc),L_(0x54f19290), + LL(0x4220e848,0xa785596f),LL(0x4ad330d5,0xfa4258dc),LL(0xf4276ec4,0xfc15e9f1),L_(0x7728478f), LL(0xa58a670e,0x9382335f),LL(0xfea55d50,0x3477647b),LL(0x311f9766,0x4f5dbddc),L_(0xd6c5903a), + LL(0x69933c7c,0x21ea5af6),LL(0x5f27f3cd,0x4b132119),LL(0x12e00aff,0x0ff8497a),L_(0x9216e688), LL(0xb8a8554e,0x7d054f07),LL(0xc58c4de3,0xb896b1fb),LL(0x91dbc7f3,0x6470c920),L_(0xcdac5d1d), + LL(0xc7b39dea,0x93ff36ec),LL(0x7d199372,0xa6f2dcec),LL(0x802aa4a0,0xdbd9b943),L_(0xae6d61fe), LL(0x8c1a0edc,0x7cecf63b),LL(0xfd6604d6,0x54af7ee0),LL(0x78b1b385,0x0498d501),L_(0x57443efb), + LL(0xf34b8647,0xe5ba9123),LL(0x4408fe5c,0xc14ff9dd),LL(0x42cbd03c,0x2891aad9),L_(0x1d5c6bb5), LL(0xf01f7309,0x55808c68),LL(0x50a8d621,0x2ceb5051),LL(0x9d313349,0x2f3b809e),L_(0x28922b09), + LL(0x59881515,0x919ddc75),LL(0xbff9ce4a,0xf87e0baa),LL(0x3037e369,0xa090d36b),L_(0xe32f5ba4), LL(0x579ffa97,0x38e224e4),LL(0x43d0a980,0x2cadda68),LL(0x702d5e89,0x12eff4f2),L_(0x4c7be34a), + LL(0x52239089,0xfabf2649),LL(0x653b3f99,0xe9f6b8df),LL(0xe84977d6,0x3f58519b),L_(0x6dfea3c8), LL(0x4357d0b9,0x6c5351ad),LL(0x2f6d5e66,0x1253d822),LL(0x9524c5cc,0xfbce31d7),L_(0x55b44df1), + LL(0x2e93f071,0xfcee759c),LL(0xe63cea29,0x132e8c42),LL(0x6a329ada,0xec4a7a29),L_(0x3f854cea), LL(0xd5e2b375,0x36a40bf0),LL(0xb7d3f3d0,0x9da99c9f),LL(0x25e16e85,0x7730bda6),L_(0x17380d79), + LL(0x4e15a70e,0x37a376cc),LL(0xcebedfe1,0xad1329bb),LL(0xe14b9f7a,0xac8b6391),L_(0xb426bc37), LL(0xe412693c,0xb1a885d1),LL(0x817a04ef,0xdd7cdcbf),LL(0x06667f09,0xfb4d7bf2),L_(0x9f6db5e5), + LL(0x350411ec,0xc83be0ea),LL(0x021990cf,0xc7612043),LL(0xbb338804,0xfb1e56ce),L_(0x4934fcc4), LL(0x4b75557c,0xfed92eee),LL(0x4d6a1f0e,0xc3259e9d),LL(0x61eea795,0xc28b498a),L_(0x10a35828), + LL(0x7363a98f,0xe21a8af4),LL(0x6f0486ce,0x2ee0c526),LL(0x081ad4e5,0x50ce73f9),L_(0x41a90bde), LL(0xfce29613,0x52824a9e),LL(0xc969520d,0xb689cfc9),LL(0xac6e83b5,0x006e1f4f),L_(0x134a6574), + LL(0xd0a972a9,0xe471b9e4),LL(0x53835085,0xcfd1906d),LL(0xd7accdbc,0x293a4390),L_(0xe0eb82dd), LL(0x86a243d9,0x06c24954),LL(0x443f28e5,0x3453aba8),LL(0x25495810,0x36f891b2),L_(0x86536633), + LL(0xba90affa,0x4d72eeba),LL(0xedf3c972,0x81c31d9c),LL(0xe70f3373,0xa716b2aa),L_(0x26ec6732), LL(0x42ac4253,0x6f657c45),LL(0x636e8421,0x7d074643),LL(0x461026d7,0x0dde683d),L_(0xddec4661), + LL(0x4b805782,0x8dd02690),LL(0xd5dfb670,0x1d04223d),LL(0x83277841,0x7ea96551),L_(0x5fe1e720), LL(0x4c78c04e,0x70dde026),LL(0xa3bb08d2,0xb58c0bf7),LL(0xae645b00,0x3afcf537),L_(0x1f93a66a), + LL(0x708a8084,0x5f7d6f8d),LL(0x3c0562fc,0x141c5fc5),LL(0x61a1cb60,0x78ecbf19),L_(0x8914e2cb), LL(0x4cb03ebd,0x9be1423d),LL(0xad53b1c9,0xd63cb771),LL(0x51acdaa0,0xa32591dd),L_(0x45b4e835), + LL(0x8de09e87,0x76e08928),LL(0xf8aeca8e,0xa653c233),LL(0x59256cc9,0xd8d51507),L_(0x77e6aff7), LL(0x47309995,0x0f159c21),LL(0x9d735f13,0xfe9fe5bb),LL(0x0339c473,0x16784f00),L_(0xa2846dae), + LL(0xf2b37281,0x1cbd8aaf),LL(0x4c543f48,0x8a600a72),LL(0x2324a737,0x644765ee),L_(0xdb3c4503), LL(0xb4dd3d31,0xa23d1c2e),LL(0xe86265b8,0x953388df),LL(0x864110e9,0xe9be065f),L_(0xb52e4596), + LL(0x9c32af39,0x69ee0a06),LL(0x9737e953,0x1fd2a13b),LL(0x38105c26,0xfe12d778),L_(0x620245c1), LL(0xd4a50775,0xd5b96217),LL(0xc8c3c80c,0x0f0582fa),LL(0x9f4dc144,0x87629c9c),L_(0xc6b8bdb2), + LL(0x377dc9b7,0x9e11a84c),LL(0x6df2302d,0x610e3f5b),LL(0x9becf104,0x87cc9a9b),L_(0x9b8f3304), LL(0xbefa56b9,0x89fa53e3),LL(0x7b1cfe6b,0x0b3ed786),LL(0xdb99856d,0x68d6dbd1),L_(0x9d87c161), + LL(0xdd0b49b0,0x7e561904),LL(0xfe908e50,0x3056fb6f),LL(0xab404bf8,0x4911d8a5),L_(0x69dfa8b2), LL(0x91807632,0x606c33cb),LL(0x776c9da6,0x791f57ef),LL(0x525adb32,0x818f0a1f),L_(0xf0db232f), + LL(0x1094f387,0xd1afe50b),LL(0xfdce1f8c,0xab294ef8),LL(0x69998dcc,0xe0d8fa53),L_(0x57be6614), LL(0x6ccabafe,0x64a8c67c),LL(0xf805f10c,0xd4a601a4),LL(0x8082a5ca,0x1c89a181),L_(0x5727dd5a), + LL(0x4855546d,0x6d3c05ab),LL(0x5bddd7e0,0x44996dfa),LL(0x9e6dd61e,0xe5054752),L_(0xb61ac751), LL(0x4c1c7979,0xa7675178),LL(0x4f0ea0a7,0x72b800a3),LL(0x307c1a34,0xafc9ca2a),L_(0xfac51887), + LL(0xf1254532,0x1e52b09c),LL(0x278095e7,0xfc186009),LL(0x579623ed,0x0b5f7250),L_(0x8c426eed), LL(0xa4d79ef8,0xff4c374a),LL(0xe0d3f5d3,0x252cb607),LL(0xf8e76342,0xfcc58682),L_(0xf5c2231c), + LL(0x586d393b,0x480cf461),LL(0x4ae96cad,0x13a215d3),LL(0xe3cf0a14,0x82460da2),L_(0xe34161f4), LL(0x188125b8,0xcd5956cb),LL(0xe0c83545,0x9e63be58),LL(0xbfc652be,0x275a3495),L_(0xc74408a0), + LL(0x965c6e1a,0x0b8b60d7),LL(0xcfd86827,0xd6030340),LL(0xc11f7050,0x4e201a9c),L_(0x3347de21), LL(0xdd1a6a76,0x73f70f06),LL(0xd2c5e44a,0xebc85954),LL(0x11a05e07,0x0cd2e37d),L_(0xa7ea0cdc), + LL(0x7afa362e,0xf7c5f96b),LL(0xce53fd1c,0xfa20c1e3),LL(0x6be4e8e8,0x02b63df7),L_(0xcb0474d1), LL(0x3453fb75,0xf6fdf958),LL(0x30ff85d7,0x502b7475),LL(0xadf32795,0x20557243),L_(0x066b61d1), + LL(0x446cdc62,0x8eb329bf),LL(0xa1b5aa2f,0x369f125e),LL(0x6c894dd3,0x9db7bb62),L_(0x4e250042), LL(0xc4277491,0x31b2dbea),LL(0x4c24c213,0xb650ae0a),LL(0x20fa0957,0x29da364e),L_(0xb22a0abe), + LL(0x3d0c5314,0xaa1e00ca),LL(0x4668311c,0xe23097a6),LL(0x18e7fcd5,0x790fdadc),L_(0x6c84e798), LL(0x00c47191,0xf4068839),LL(0x0925c817,0x5b080743),LL(0x257f4df1,0xd4b97dd1),L_(0x0e714d67), + LL(0x9e9f267a,0x7fa3e72a),LL(0xd9edf53c,0xec547442),LL(0x303f3000,0x74ba5f3a),L_(0x3088e019), LL(0xe130545a,0xf97b4e0b),LL(0xaed0e113,0x7b3e3dc4),LL(0xb3342459,0x849b0cbe),L_(0xa9fbffa7), + LL(0x156bffa4,0xef4fdf72),LL(0x85aca1ed,0x0fb07e25),LL(0x373456aa,0x0c136024),L_(0x3fbc3b2d), LL(0x75063486,0x09872cea),LL(0xed9cc9ba,0x1f75eff6),LL(0x2982d676,0x46918abe),L_(0x638bc02a), + LL(0x62d87802,0x8ac63776),LL(0x0082e1c8,0x3df57bb4),LL(0xfb67a75d,0x872c8f87),L_(0xe6292f25), LL(0x49548c68,0xa4ab6141),LL(0x1ee50877,0x5ae2f1d3),LL(0xccb21fe7,0xcc4c5d51),L_(0x4dd1a7d5), + LL(0x552b1d5b,0xc91b6f0a),LL(0x8cacee59,0x26244e6c),LL(0xebe1c6fb,0xec4bbe5c),L_(0x6a216c08), LL(0x35cfef8d,0x09fa8c0f),LL(0x17b60280,0x77fdabaa),LL(0x4bbc3792,0x94421437),L_(0xb01866d7), + LL(0x820d6809,0x9b55ccda),LL(0x95bebe41,0x1046f37c),LL(0xed3f6fb8,0x07a02e6b),L_(0x5d6bfa33), LL(0x6676830e,0xf82f2f79),LL(0x8ba9b566,0x35492432),LL(0x79d53ab5,0x16a996a8),L_(0x912fe1fa), + LL(0xaa0f7a90,0xe4d29021),LL(0x63d69e6f,0x9b86e6df),LL(0xdb26a514,0xf567f4e0),L_(0xe0d08a4c), LL(0xc1decce6,0x9de6495a),LL(0x037496f1,0xda23fbb5),LL(0x6557417b,0x7d23ff08),L_(0xb80f990a), + LL(0xc3c2aab9,0x2e3254ff),LL(0x0c882e32,0x48576b69),LL(0x2e3a1aff,0x4e27496a),L_(0xa8981080), LL(0x497974dd,0xb0b36f57),LL(0x57530d61,0x1d0467dd),LL(0xbd12d521,0xee061337),L_(0x2a16986e), + LL(0x3a9a2700,0xfa637395),LL(0x08e9776b,0xbc2048f0),LL(0xf8ae9c4f,0x7229d31e),L_(0x5d5cdb49), LL(0xd86d5de3,0xd7e8cbb1),LL(0x7d36c9cc,0xa98c305d),LL(0x382a852f,0x7668edae),L_(0xb66ddb2f), + LL(0x6d33747f,0x6bae97b0),LL(0x258e1f01,0xc0b8420d),LL(0x0d1395e0,0x5d523a37),L_(0x20fd468f), LL(0xf196c7bd,0x940889f2),LL(0x2b93cb81,0xa914e0ea),LL(0xce35713d,0xcb59a392),L_(0x4ff9fc32), + LL(0x984bd24d,0x17b7b322),LL(0xe322bc50,0x33279a91),LL(0xa03ade42,0x3ac696ec),L_(0xa59b7158), LL(0x36f23ecf,0x4faba2b9),LL(0x22a5cf90,0xf70fdee7),LL(0x7bbc4522,0xeb035a5f),L_(0x58a1a1e9), + LL(0x5ddc6b6d,0x150eef02),LL(0x52864a3e,0xdd3f93bb),LL(0x3696d783,0x15012933),L_(0xad3eb1b1), LL(0xff6e69b6,0xc84755b9),LL(0xb7d6ce03,0x118ab913),LL(0x2da1eb13,0x99325702),L_(0x1f352fb1), + LL(0xdfd09cd9,0xd7ddee82),LL(0xfd22eb92,0x93460443),LL(0x87b5bd4a,0x0d5541f4),L_(0xf4336176), LL(0x2e83bcbb,0xd3e283ed),LL(0x6124a0f9,0xe52b803b),LL(0xb28eb570,0x4a8c77d7),L_(0x782341d6), + LL(0x7636205a,0x18df4a69),LL(0xc1129737,0x70a8c000),LL(0xf96be149,0x69d0bc39),L_(0x8091b05b), LL(0x59f09aa5,0x53e28bf7),LL(0xb0469f79,0x3ef52081),LL(0x28ac324e,0xa781c57c),L_(0x10eeaa88), + LL(0xb145dd4e,0xf8de49f2),LL(0xbbb26b41,0x7a359488),LL(0xa5fe71f0,0xea4e5a13),L_(0x0d776617), LL(0x8252a9aa,0x9870bd6d),LL(0x069f0a83,0xd7e75649),LL(0x73608571,0xb0499f6c),L_(0x9dd9cbb8), + LL(0x63626f39,0xee89a981),LL(0xe0aee1b4,0x45b932da),LL(0x25321bbf,0xdd6b2382),L_(0x0c03fd27), LL(0xbd649607,0xdbca6a1d),LL(0x6b8f9b4b,0x1e699d4b),LL(0x940460cb,0x7938144e),L_(0x8c3cb198), + LL(0xa571f0be,0x50d17043),LL(0xcd56a2ea,0x7b24e599),LL(0x15047c8b,0xe06a40d9),L_(0x99dac8b4), LL(0x5bc5aeb7,0xaeb386d4),LL(0xe9f713d9,0x40fabb3f),LL(0x62bdfdf5,0x6aace159),L_(0x8e028ae5), + LL(0xde610cbf,0xf6b35535),LL(0xfe554daa,0xb7d2826a),LL(0xb4edc29d,0x03e1e5e0),L_(0xd38477a9), LL(0x789b199f,0x03331148),LL(0x4a8a863e,0xa966c649),LL(0x0851cfcf,0x1622ec1d),L_(0xfa31d354), + LL(0x888b7c2c,0x07cb3e37),LL(0x660e7e68,0xb017d5a9),LL(0x7f7c933f,0x42119d1d),L_(0x47d400e6), LL(0x5ef83e46,0x0109f6b8),LL(0xbc268020,0x6e139180),LL(0x1b624dea,0xa2bd127f),L_(0xd4e19a66), + LL(0x54ae0fce,0x9a79307a),LL(0x06792812,0x6d342e22),LL(0xb177e174,0x2e3ee119),L_(0x56fea730), LL(0x5e84ad5f,0x08e8f89c),LL(0x12504248,0x425e69c5),LL(0x42b47533,0x0b2e547d),L_(0x0cd14526), + LL(0x041c2bcb,0xd43f468a),LL(0x6c6a96c8,0x497dff29),LL(0x58417877,0x9551c8f0),L_(0x738c007b), LL(0x1e1b3a52,0x7aa44dae),LL(0x4cb0aa92,0x82e3854e),LL(0xe5d6128d,0x2f25adda),L_(0x1565a105), + LL(0x92365b9a,0xdd4e8322),LL(0x332f51e4,0xe083981e),LL(0xbf7947a0,0x5b678747),L_(0x3196029e), LL(0xc98f910a,0x0ca71f21),LL(0x51b450d0,0xe31a901e),LL(0x922f7c66,0xeed6566d),L_(0x1ed09e18), + LL(0x274ea5df,0x52fe1ee1),LL(0xf298e59f,0x7dc25d85),LL(0xd6285877,0xda84cd8b),L_(0x7cf3ae58), LL(0xbb649ac2,0x1e3ba4b5),LL(0xb225291f,0x6f9c137c),LL(0xbd251ef1,0x5e5495d3),L_(0xb7512d10), + LL(0x8392e4dd,0xdc8ee05c),LL(0x9d3efe45,0x4eeeb0da),LL(0x0d91ecaf,0x31e89bf7),L_(0xdc635197), LL(0xeca9047d,0xee9a09ba),LL(0x789d1a79,0x81d64f7e),LL(0xea9e0b50,0xc7b60917),L_(0xb463b663), + LL(0xb48d4d3f,0xb23798db),LL(0xfd959cce,0x4a87382c),LL(0x829e3bb6,0x492aac4a),L_(0x4bf01b8e), LL(0xf58a6464,0x96935f47),LL(0x82b212c6,0x3396e950),LL(0x6d21b8d5,0xe244afe7),L_(0xa642079f), + LL(0x890b338a,0x01b0423f),LL(0x8f95cd41,0x6d8cdb00),LL(0x68218ce9,0x7809fc35),L_(0x1fe627f9), LL(0x446c771b,0xaae3f3c1),LL(0x622ee718,0xfb834953),LL(0x89128662,0xff1c7590),L_(0x2c4b37c2), + LL(0xe1d9d534,0x52f4f30f),LL(0x9215d0dc,0x3a3e06d4),LL(0x6d0d3574,0x9ec8c4a0),L_(0x4abdacfc), LL(0x06630325,0x8d559ed9),LL(0x7bfef54b,0x295b5ede),LL(0xd0a2d07d,0x8d6e5cd0),L_(0xb74df6fa), + LL(0x42c70b2c,0x4f4ae671),LL(0x613fd1c2,0x536e7d64),LL(0xfaa3003d,0x94dc0a21),L_(0xd4e576a1), LL(0x1592950f,0xb4369ba0),LL(0x3179ef67,0x5f932085),LL(0x6309e764,0xe5462315),L_(0x3bcaa778), + LL(0x08fa6e2f,0xec60e2f4),LL(0x2b9b6197,0xa152beaf),LL(0xcc5f8968,0xbce9a53b),L_(0x1ef40db3), LL(0x076de5d9,0x31df6025),LL(0xe8f5bdc2,0xce591c62),LL(0xead89b13,0x52e8a741),L_(0x12358a25), + LL(0x179fcba1,0xc81fc0c7),LL(0x35daa530,0x8eb90798),LL(0xbdb2ded4,0x820ef61d),L_(0x6b0231aa), LL(0x2b8a3cbc,0xf522badb),LL(0x0e482ed6,0x93d7057a),LL(0x0ff17306,0x70098d33),L_(0x63ccaacb), + LL(0x8e019362,0x2f914dd7),LL(0xf4501079,0x2acdf294),LL(0x2848b0db,0x0f138ea9),L_(0xaf353a81), LL(0x4ea1b03a,0x49ae634d),LL(0xd3a8c33b,0x113138a4),LL(0x8ed0557e,0x2b0adbad),L_(0x739b3ce7), + LL(0xb33942af,0x1ecf958e),LL(0x54804872,0xc66d2148),LL(0x1f27ea10,0x011365d7),L_(0x6ce35983), LL(0xc4208dd6,0xb8f4d08e),LL(0x882dc840,0x3e553026),LL(0x29499eb7,0xea028ba1),L_(0xd9859a66), + LL(0xa6f2df8a,0x5bf03737),LL(0x134f5c42,0x924c5819),LL(0xce383c9e,0x910aeacb),L_(0x2daae049), LL(0xbfdc9b59,0xe45c96a9),LL(0x3a5b8130,0x726cada6),LL(0x8177cf88,0xa738df5c),L_(0x9be237a5), + LL(0x21f30c43,0x8cd0ef97),LL(0xfdd4b300,0xe6a63e2d),LL(0xd4df1941,0x57a01e9c),L_(0xf32119bd), LL(0x9c025c37,0xdc96eafb),LL(0x136dc55e,0x95fe2aa2),LL(0xae8f18c9,0x239e5894),L_(0xb671d770), + LL(0x9c68e7cd,0x22874bf5),LL(0xff1a555a,0x64db3073),LL(0x3b1f7071,0xa14665fa),L_(0xad6ab5ea), LL(0x78fc6e69,0x77eb0618),LL(0xa6cf5f0f,0x6d531160),LL(0xf5f0fda1,0x429c094e),L_(0x78109f67), +}, +/* digit=6 base_pwr=2^42 */ +{ + LL(0x570bd76c,0x42617e4e),LL(0x53cb59f9,0xec1fce2c),LL(0xbd807766,0x5098d9d8),L_(0xf56acb00), LL(0x468a4f17,0xada13650),LL(0x8356714e,0xb5aa694d),LL(0x1dce66de,0x5318219a),L_(0x3439525e), + LL(0xd30a5801,0x97115f9f),LL(0x2621a7d4,0xbb10a57f),LL(0xeff5c174,0xadf87884),L_(0x01243ae4), LL(0x162e3c5c,0xa2bc843d),LL(0x6f4a3579,0x25478286),LL(0xd6f94dfb,0x31bb4258),L_(0xa760a06c), + LL(0x6c5c8428,0x46351cc8),LL(0xa082b7e1,0xb964daab),LL(0xdd1d4004,0xc4d0e945),L_(0x3e7f4540), LL(0xb82dabae,0x7d7ac429),LL(0x956426a7,0xeed99c4b),LL(0x7dafa986,0xcbdad15b),L_(0x8d046994), + LL(0xe5e0aad9,0xc7b54b13),LL(0x257de915,0xe9d8af38),LL(0x008e0bc3,0x71a411c5),L_(0x9f3a29a1), LL(0xc482b233,0x07289ac0),LL(0x780d30d5,0x68d763c3),LL(0xa128b566,0xc589940b),L_(0xd834f239), + LL(0x533b21ee,0xb86a434f),LL(0x9be1cf11,0x5f1dadf4),LL(0x5b2f618e,0x35caca2b),L_(0x9da9d5f4), LL(0x74cf7bbd,0xb14dae65),LL(0xa4192a56,0x27c04cc6),LL(0xf5af0266,0xc41f13b5),L_(0x350b4db7), + LL(0x5727ef4c,0x9fcc3538),LL(0x2806003f,0x957682ab),LL(0xe33ed1c3,0x9f83349f),L_(0x02ec97a0), LL(0x5961423d,0x5148178e),LL(0xeab94e36,0x66e2c6e5),LL(0x5986c53b,0xd5c63ac2),L_(0x9b96cae8), + LL(0x9082e762,0x5661867b),LL(0x29ca7816,0x8ece3937),LL(0x30442089,0xeddde973),L_(0xee4b63a6), LL(0x3e2f6790,0xd4f64e91),LL(0xd529fa9b,0x49fdfa45),LL(0x0b104178,0x78b6687a),L_(0x40d1484c), + LL(0x9f724692,0x0ca4bf22),LL(0xb272519e,0x730e0881),LL(0x4bbc9bf5,0x91d1e24a),L_(0xc6e5ce02), LL(0x26de9372,0xd72ae23c),LL(0xc7a7721b,0x6ca80e46),LL(0x75b289b3,0x777ca3a2),L_(0x28a90ef0), + LL(0x7d3fcac7,0xebb7ace4),LL(0xa09641c2,0x48bb32ea),LL(0x0be22833,0x3f76d58e),L_(0x7e7a5d70), LL(0x8e3df32c,0x00f1bf36),LL(0xee4b523d,0x8b204653),LL(0xdaeaa58b,0x3263322d),L_(0xe7975fbd), + LL(0x648b273e,0x2e05e12c),LL(0x0e313d88,0x3a732a3d),LL(0x691aae3b,0x6cd91464),L_(0x8bd52c9b), LL(0x208c6c3c,0xe4c05b8f),LL(0x66fafda4,0xd9e2218d),LL(0x59cd9a06,0x835930df),L_(0xfb01c010), + LL(0xd984c49f,0x888fc640),LL(0x0bbd6a5e,0x9bb00fea),LL(0x4da4ce0e,0xaffa26c5),L_(0xcc14f2af), LL(0x2307b844,0xc6cd9202),LL(0x340af7d9,0x3f8ed768),LL(0x518df91e,0xeadd2347),L_(0x8dc696be), + LL(0xaf9c00ec,0xb90b0248),LL(0x2d2b08b8,0x45a5507d),LL(0x2e1dac5f,0x8db688a8),L_(0x3e2c52bf), LL(0xa655bbf1,0x0fdb8f93),LL(0x95c256dd,0xfdb2b439),LL(0x9751d174,0xaef8bdee),L_(0x54ac1077), + LL(0xb601c37a,0xdaadd78b),LL(0x450b3671,0xb65d4dfa),LL(0xb3e4d63b,0x653e4965),L_(0xf559fd3c), LL(0x8cbfa3ed,0xbaaa17fd),LL(0xcd2af6b1,0xe29a388e),LL(0x50eade40,0x2ec73453),L_(0x0ca27599), + LL(0xbac1435b,0xaa859487),LL(0xc7540522,0xb5b2445b),LL(0xe22a73a2,0xaa1116ed),L_(0x2e7ed6df), LL(0xf9d72a86,0x616cb03b),LL(0x61a4a6e7,0xcdf54ee7),LL(0x4cf9a426,0xe0208859),L_(0xc62b0ac4), + LL(0x5300262d,0x7faf0467),LL(0xc2571f4f,0xff47f116),LL(0x93b14cbc,0x484a66c2),L_(0xf8b658b8), LL(0x654db6ca,0x20220d13),LL(0xfcb07e76,0x1162cfde),LL(0x897e3ec2,0xf29d6809),L_(0x6eee73ae), + LL(0x2c67eef1,0x1a833a88),LL(0x40f7ac97,0x8aa0f2d7),LL(0x5fc1c69f,0x9f06aae9),L_(0x08b7e77d), LL(0x4ecfeaa6,0x17fda35e),LL(0xeac8ca12,0x338ff3df),LL(0x0013029b,0xe6215b79),L_(0xd490a9e5), + LL(0x45530708,0x3d6ab800),LL(0xe7c57017,0x34dca330),LL(0x5aa02638,0x4dc086c8),L_(0xba336129), LL(0x6df633bc,0x90bbb8d8),LL(0x33ca4c46,0xc7c28c06),LL(0x8dcb3bae,0xa0c24314),L_(0xa41cf732), + LL(0x7f227cd0,0x8cd51cfe),LL(0xb7fcfdc2,0x68ea58e2),LL(0x9f7bcc04,0x3003cc72),L_(0xd4b488a1), LL(0x5b982b6b,0xd5ebd68b),LL(0x4020c3e7,0x675b50df),LL(0x9679336c,0xa8fee552),L_(0x24805e44), + LL(0xe51e0a9f,0x75eb0c92),LL(0x760831bb,0x9af943f7),LL(0xa54c748c,0x28a759fb),L_(0xe246d8ba), LL(0x1cea2c6e,0xb0384781),LL(0x2f1568a6,0xf0f73ecd),LL(0xb65d1ecc,0xd26a77df),L_(0xa639e1e0), + LL(0xcb23a542,0x51dd59db),LL(0xac405e3b,0x24f458a4),LL(0x7d0b3dda,0x26b341e1),L_(0x9072bcaf), LL(0x80d2e583,0x9e2fd334),LL(0xd09edd03,0x114215cc),LL(0x794f6e44,0xe3bb54b5),L_(0x4abbf55b), + LL(0x01429cdd,0x7fb2c053),LL(0xaaeddc51,0xee0b3bcc),LL(0xcf236537,0xca6a0781),L_(0xee610794), LL(0x28ee066f,0xe18e5f40),LL(0xb59956df,0xb1e0ccfc),LL(0x2a9ed4ef,0xa109a3af),L_(0x628906f4), + LL(0xad211ace,0xe621f731),LL(0x99a069f0,0x072030c8),LL(0xa71de5a1,0x106812c7),L_(0x606b3758), LL(0xeac94530,0x4ce2ef8a),LL(0x57357177,0xac163954),LL(0x778bc2dd,0x2e5cbdab),L_(0xa82c012b), + LL(0x0baf79a3,0x9c81a4be),LL(0x4fc50828,0x9363638d),LL(0x8a4d69e0,0x687f0dee),L_(0x92f9ddc4), LL(0xa6d3323d,0x202ed4a9),LL(0x123a8f24,0x1c483cb2),LL(0x5bbe8c1a,0x433b79ed),L_(0x23111e9a), + LL(0x3828dbf8,0xb8a0c6af),LL(0x43cd4321,0x9f12e96d),LL(0xdd7d6cea,0xe595dcab),L_(0xd3b3c998), LL(0xad54b44c,0xe8aa36bb),LL(0x142da061,0x11dcc1c4),LL(0xcbe63415,0xeff9d401),L_(0xa4d738ce), + LL(0xfef9f39f,0x1d099df3),LL(0x769dfb43,0xbf9c0426),LL(0xe387610f,0x0cce0331),L_(0xd6feb0f4), LL(0x31cb09c5,0xd62d1375),LL(0x10527627,0x53290cd4),LL(0xa90e25d2,0xf1396054),L_(0x6a0d6915), + LL(0xf603ca7f,0xe57e3a51),LL(0x26b3d6ba,0x547b9c97),LL(0x5234c386,0x44edaf6d),L_(0x7c4a6d8f), LL(0xcea690d7,0xf13946cc),LL(0x4a083ed0,0x6ad6bb4e),LL(0x9374003a,0x36c2bacd),L_(0x90ccdacc), + LL(0x341c30ae,0x7e1c2c9b),LL(0x313b286c,0x17e56dfd),LL(0xb65db139,0x9888703d),L_(0x65b2fe62), LL(0xe07d7043,0x3e36f84d),LL(0x6fc2a51e,0xb4b9c74c),LL(0xbbd020f9,0xed1bc384),L_(0xb260d341), + LL(0x4432f2d0,0xa3a488e9),LL(0xf4caef39,0x9efd00d6),LL(0xd17c4829,0xf5128936),L_(0xf742f93b), LL(0x35e93b49,0xae41e05a),LL(0xa5eb9e98,0xe030a464),LL(0x41a1010a,0x1db1bce3),L_(0x7cabde79), + LL(0x6fb1c1bf,0xfe5cfb57),LL(0x304db3f7,0x401be370),LL(0x29b356b7,0xef6ca52e),L_(0x16934044), LL(0x131c52a9,0x3084c2e8),LL(0xb1a56ede,0x58a36c5a),LL(0x8cfd7a22,0xa076e835),L_(0xe8967b86), + LL(0x74382b8c,0xecf54e6c),LL(0xe7d202a5,0x80a6564e),LL(0x3378864a,0xf29ac0df),L_(0x3af276ae), LL(0x9549b955,0x71036c3c),LL(0x85bde008,0xd3203c72),LL(0x57b0b3ec,0x7ee244a8),L_(0x7831f44b), + LL(0x2a2ae111,0xe00edfa7),LL(0x8b506f33,0x6b68f39e),LL(0x8f5215f7,0x0b132723),L_(0x36ee465c), LL(0xe7d50419,0x85673786),LL(0x4b253aad,0x231c0709),LL(0x487f22bb,0xf5e2a237),L_(0x8af08593), + LL(0x95692ad3,0x0b27bb26),LL(0xa14fcdb7,0x090be6f5),LL(0x8b45b55f,0xea84dd7c),L_(0xd1971d3e), LL(0x928497fb,0x971d5f28),LL(0x9e1b8d56,0x6339a103),LL(0x9ddbd5c4,0x3fe08c7f),L_(0x71eef4d8), + LL(0x987de693,0x1683a455),LL(0xca6c2d90,0xb38e1de8),LL(0xdd0a3684,0xf01af5d9),L_(0x2403e538), LL(0x86ab28a1,0xd4b30cec),LL(0x533a55fa,0x22c7c598),LL(0x01cadc9e,0xb5dd36ac),L_(0xe6dc51f3), + LL(0x8baf2809,0x60cc335f),LL(0x1ed24f3e,0xa633db28),LL(0x732a899d,0x7f9cd569),L_(0x5d8b2c23), LL(0x04bce2d6,0x1f59a0d7),LL(0x6ef81d6c,0x6d10e528),LL(0xcbb166ea,0x7fe10619),L_(0x1eb94857), + LL(0x04d81b40,0x88a7293b),LL(0x3d8447b9,0x1732c2f0),LL(0x057b8676,0x320d8861),L_(0x5a9e4e63), LL(0x5bfbc591,0x3e5d907e),LL(0xa66f2cce,0x2b35a3fe),LL(0x1597d44b,0x7605acfa),L_(0x756ca23d), + LL(0x6112f38f,0xf5ab47b4),LL(0xf577db42,0x5984ae06),LL(0x4fccb7e8,0x81972b3a),L_(0x1d854210), LL(0x5d4f964c,0x905ded37),LL(0xf605f38c,0x6a31fa68),LL(0xfb6ad50f,0xd83dca88),L_(0x6aacdbc7), + LL(0x1b0ac423,0x711dde1a),LL(0x227d6127,0xb5e309b2),LL(0xac0aec46,0x17ee236d),L_(0x70e64775), LL(0x600531c3,0xce093e55),LL(0x458395de,0xec04dd4e),LL(0x59639b52,0xb9e01d45),L_(0xa8e6469e), + LL(0x1b94ace7,0xb0fee2ec),LL(0x08d97fb0,0x5c0b8f2f),LL(0xdfd59ee2,0x6a154d90),L_(0x35f79449), LL(0x8fb75d5a,0xb692d5e7),LL(0x094a540a,0x080f7773),LL(0x1fbd50fb,0xa4f8f9e4),L_(0xb9d1eb94), + LL(0xf63f6da8,0x2aa45a5e),LL(0x711d55a4,0x2d7d0b41),LL(0x29c1bc69,0x8aabebdc),L_(0x8480eff7), LL(0xb0063dbf,0xc38e3c41),LL(0xe276eaf6,0x661ba0dc),LL(0x99541913,0xc19809da),L_(0x50e215d3), + LL(0xff252d36,0xa1530a6c),LL(0x253d53ef,0xc8e9b570),LL(0x9793a371,0x3489107c),L_(0x8abf5783), LL(0xf0b49c6d,0x879022b6),LL(0x12bdba9e,0x069dd468),LL(0x2c618b82,0x64bde16d),L_(0x2306b3be), + LL(0x2d08df95,0xed7d7c9a),LL(0xd4d38ca9,0xe80a7f84),LL(0x357baf28,0xbb968f21),L_(0xbcca4e7e), LL(0x8315da93,0x96e91923),LL(0x4f08013c,0x4e306972),LL(0xc5ed2a12,0x950bbe41),L_(0x434c75ba), + LL(0x39668cb6,0x077167b6),LL(0x7bdd93d3,0xfce255b9),LL(0xb37bc6b8,0xf3809a3f),L_(0x32c36f4d), LL(0x7f61816e,0x44d57109),LL(0x06f0654c,0xb19cf5fd),LL(0xdb202875,0x42100d63),L_(0x79f51335), + LL(0xaf246d2a,0x7f3965a6),LL(0x8caa84d7,0x8d8e3137),LL(0x26bde524,0x11b1a769),L_(0xa5eaa311), LL(0xd87dfaee,0x707808be),LL(0x69540405,0x04bb89d0),LL(0xe52cbe87,0x7ec2d88f),L_(0x994f86d5), + LL(0x5423908f,0x43d7285e),LL(0x85d0501d,0x49363917),LL(0x6a49fc82,0xf5d8d60b),L_(0x2126dd81), LL(0xa0f34346,0x665a2267),LL(0x1fa53f15,0x1ac20d97),LL(0xffa2e5fc,0x4e6d33c1),L_(0xf9840551), + LL(0x9731dc20,0xf1f7a37e),LL(0xc0130bbb,0x69d32743),LL(0xea51b61d,0x05851633),L_(0xaa72b2e1), LL(0x04e8d8bb,0x1c8bd021),LL(0x4806ed2e,0xef76ca45),LL(0xf099b725,0xd230047c),L_(0x5f4e19c3), + LL(0x44613937,0x62171491),LL(0xf583d82e,0xa07a7b6d),LL(0x9015ad40,0x306c9555),L_(0x0b81ff7a), LL(0x18925a9e,0xaac18583),LL(0xa0946d46,0x7d220576),LL(0xbbd4708e,0xf151aaec),L_(0xe99b984f), + LL(0x6e04c9ed,0xb57b3b22),LL(0x3cdad4a9,0xa3593866),LL(0xa01b71df,0xa63bab57),L_(0x97f85c63), LL(0x773f947c,0xefc9bc76),LL(0x1a1f56a4,0xcb328f94),LL(0xe12b780c,0xb068b3ce),L_(0xc8733217), + LL(0xc1916a8a,0x62555b0b),LL(0xe906cdca,0x00203baf),LL(0x82d7d36f,0xd8764be5),L_(0xc4064873), LL(0xf4469260,0xa23e8076),LL(0x7f6613c6,0xb903dea6),LL(0x11610026,0xd83fceb2),L_(0x6e5bcf50), + LL(0x2739ed0b,0xcff10abf),LL(0xcaa11cd8,0xe0b269eb),LL(0x92761d90,0xc1c0ca0e),L_(0x70aa2955), LL(0xab5ce41e,0x62aaaef4),LL(0xc2c0d2af,0x5a3ff0d9),LL(0x3aa250ea,0x8f10b784),L_(0xdbfc2028), + LL(0xe5bbf62d,0xa53f3f3f),LL(0x5bd69524,0x27dd1951),LL(0xeada36e1,0x40a88ecc),L_(0xd04364be), LL(0xca0a31a4,0xd318529b),LL(0x5b3c2cf3,0xefec04d1),LL(0x49d3e4dd,0x55c20eb0),L_(0x7153c7d3), + LL(0xc5f830cf,0xbf310209),LL(0xbaeecda6,0x1372fe7c),LL(0xab6a0dc0,0xaf6004cc),L_(0xabcb8c97), LL(0x8803af61,0xa6088f51),LL(0xf7850d91,0x53f144ac),LL(0xe77f4280,0xd7d06ab8),L_(0x9a5d8d81), + LL(0xc4969a0e,0xdf19cc2f),LL(0x433da709,0x8bd7557f),LL(0x554a3a06,0x8fe8b8dc),L_(0x687d6a23), LL(0xf261d773,0x82e07bc5),LL(0x9e80d756,0xd1d7da5d),LL(0x45447ad3,0x19044418),L_(0xae95e54a), + LL(0x3956922f,0xf66675c8),LL(0xde2dc4b9,0x20469977),LL(0xf5f39465,0x011863a2),L_(0x1e9807dc), LL(0x960a1dab,0x66b9d7bb),LL(0xdbd5a655,0x074a85ef),LL(0xd9f9bf2e,0x7dd3b230),L_(0x50a9e91b), + LL(0xe5be9e2b,0xb5995728),LL(0x0640124a,0xa127aac9),LL(0x23c8cef0,0xf11df0db),L_(0xb1a238e2), LL(0xefcf6a98,0x38cfdd79),LL(0x78034f01,0x899c799e),LL(0x8c265685,0x7cab4cd9),L_(0x1b6b2132), + LL(0xe6160917,0x480a36fd),LL(0xe17a372a,0x3f7c1b1c),LL(0x91f84a8c,0x85381912),L_(0x202d4232), LL(0xb7408da9,0xe263f58c),LL(0xd1c14892,0x8ca85524),LL(0x8c2f9aa1,0x687fc42f),L_(0xebb9ceb8), + LL(0xf5f78e42,0x3455a3e9),LL(0xbbc51854,0x30c1c2c6),LL(0xddf065a7,0x3692f51b),L_(0x5794f41d), LL(0x0f5906c1,0xbfc2af37),LL(0x3df049a3,0x5fa05ed1),LL(0xd9f88327,0xfad22122),L_(0x143991c0), + LL(0xcbee3325,0xe6fcf3f6),LL(0x29ff58e0,0xc3e92fcb),LL(0x7c43be45,0x6def0ea9),L_(0x95d4294c), LL(0x72baebe1,0xabf2c782),LL(0x8f7dbb1b,0xff1c5bf6),LL(0x21f7b8ae,0xcada1649),L_(0x1d86e3b2), + LL(0x2e618a15,0xe44e287b),LL(0xc0ea1a7b,0x9ec08fdf),LL(0x3cbeeaa0,0x41378416),L_(0x5dfb49d7), LL(0x3defdbe4,0xdd8dc723),LL(0x4d7ec600,0x6d23302b),LL(0xf1849878,0x922ac03f),L_(0x9a57ab5a), + LL(0xf4361243,0x5150fa93),LL(0x3ea14eaa,0x1bf1d7e3),LL(0x0106a9fc,0x90827c46),L_(0x7d96ff3f), LL(0xd4951e8a,0x046acfdb),LL(0xa6461b44,0x38e792fd),LL(0x2d8ca425,0x07e228cc),L_(0x4ba3f6b1), + LL(0x889c6c45,0x12d9fd78),LL(0xf4406adb,0xa6aed947),LL(0x46f4af45,0x4de1be3f),L_(0xd4630c28), LL(0x932b6858,0x4b284bc1),LL(0xcbab5b0d,0xe27e82bb),LL(0x9200852f,0xe41775aa),L_(0xdd624131), + LL(0x50a8e4b8,0x9d917e78),LL(0x558110ca,0x4ac6dc2a),LL(0xeb63b95b,0x585b0c05),L_(0x89d42790), LL(0x876d7f22,0xdd6e7325),LL(0xb4e2f682,0x34a78682),LL(0x55d49e0e,0x7de26ccf),L_(0x076914a6), + LL(0xf3430044,0x09139313),LL(0x6fe59fdd,0x32e29482),LL(0x9f0a6227,0x1c584d56),L_(0xffe716b6), LL(0x11cca786,0x53203116),LL(0x82480098,0xb1c8440a),LL(0x0ec04b4a,0xb4fd311a),L_(0xb256043e), + LL(0x4a367cf1,0x82ea7247),LL(0x8fd0c543,0x35a8f8e4),LL(0xee0c8d61,0xeb47eaf0),L_(0xb1c6b367), LL(0x7fc38517,0xb4361b1e),LL(0xa69d501d,0x24e2fb10),LL(0x94b8860e,0x1e4ac934),L_(0x473c4fc7), + LL(0x9c7f7bc3,0x85016181),LL(0xa46af46a,0xae2e08c9),LL(0x1d49bd53,0xd51d0f76),L_(0x47c1a7de), LL(0x574b1fa3,0x275045e3),LL(0x2b7d8406,0xe63f9064),LL(0xb270bab6,0xcbacaf10),L_(0x1fd8fd5b), +}, +/* digit=7 base_pwr=2^49 */ +{ + LL(0x55cc3b94,0x81366649),LL(0xc85bd06b,0x0bb5529d),LL(0x66b83a45,0x020a3d7b),L_(0xd7360ab5), LL(0x6d9c718e,0x4c39583c),LL(0xf32587a2,0x063ccb66),LL(0x8cb7eb45,0x01328b11),L_(0x7b390466), + LL(0x7e06eeca,0x2a100811),LL(0x1e68642b,0xf4997f3a),LL(0xf5c07857,0xad98c158),L_(0x3c2d33de), LL(0xf408207e,0xdb7d94f4),LL(0xe7bd848f,0x6379ef09),LL(0xf3bfa4da,0xde775bfd),L_(0xa5183e31), + LL(0xd08dfe45,0x9b19df92),LL(0xf42316da,0xfd14af8c),LL(0xcc3f5ace,0x9cb59da4),L_(0x2c68ad5c), LL(0x880b584b,0xe1fa2671),LL(0x0ace93c1,0x76bf0449),LL(0x0c870e3e,0x6fb6d427),L_(0x680b653e), + LL(0x70db0533,0x12b3f8dd),LL(0xb2fbdce6,0x167ab18c),LL(0x92abaaa2,0x5e4d357c),L_(0x452fe014), LL(0x1fb69d3e,0x7bd8f0e1),LL(0x58ea7caf,0x4735131f),LL(0x84b31510,0x6a03c6b3),L_(0xbcfb5361), + LL(0x5a2c8036,0x91588e92),LL(0x4e652187,0x1b69af1a),LL(0x451b95e7,0x3517cee5),L_(0x8c64ca67), LL(0x1e09f843,0x90930327),LL(0xbfeeb455,0xefa175df),LL(0x9e48451e,0xedef520b),L_(0x49b070fb), + LL(0xa2492065,0x1e8946dc),LL(0x318fd845,0x9916090e),LL(0x82135311,0x98fdccc1),L_(0xbc4b4f98), LL(0x31f673e4,0xd529f3ea),LL(0xf1c8e5c6,0xb119ca50),LL(0x428145ce,0xcf53a57b),L_(0xba7a2228), + LL(0x38640e72,0xa872dbd5),LL(0xab84c140,0x8f90052d),LL(0xffeb6e87,0x9a484b77),L_(0xc8d71762), LL(0xe045b6cf,0xb3a39b3e),LL(0x68339c4f,0xdbc53298),LL(0x462a1b9b,0x60a23eea),L_(0x47a583f5), + LL(0x8a1c84eb,0xc38a15ff),LL(0xc73cc555,0x5b502e98),LL(0x67d1f3e4,0xf85c7d6e),L_(0xfaf491bb), LL(0x7c34a745,0x94062e0f),LL(0x5e8b71b9,0x9579c983),LL(0x35f7a266,0x63ed902e),L_(0x2cfe7a8e), + LL(0x7e6607ae,0x54fa21c1),LL(0x7c605a96,0x2c3baf13),LL(0x177b275b,0x39001a70),L_(0x402c2a3f), LL(0xf4928a3d,0x9356e91a),LL(0x443518d7,0x30b1d7ca),LL(0x02772478,0x8c5ecfb7),L_(0x8e06bb83), + LL(0xcaa887a4,0x4f7bef32),LL(0x08805f7d,0x5dec8cf9),LL(0x8965403a,0xcc362cae),L_(0xa8e82004), LL(0x37f6ba21,0x35e7cde5),LL(0x1d022cb1,0x0d54e5e6),LL(0x02d7913b,0xf4a0addb),L_(0x4f243fbb), + LL(0xcc348a38,0x58983641),LL(0xf6e35ba7,0x237be718),LL(0xd585a061,0xdb2539f1),L_(0x2217c68f), LL(0x403565cd,0x78351808),LL(0x36c2d842,0x6202e955),LL(0x7f54736b,0x660d4fb3),L_(0xca575f23), + LL(0x6bc9ce98,0xb22311a5),LL(0x84749347,0xe993a41b),LL(0x85025abd,0x432e7fa6),L_(0x6f57ce7b), LL(0x42d19cdd,0x18144346),LL(0x1fc786db,0x2bd73670),LL(0xede484f4,0xa93261b3),L_(0xc5ec96e8), + LL(0x38a7a729,0x2660c15e),LL(0x6845886e,0xcbfe266f),LL(0xe7715ea4,0x803ef610),L_(0x9d5c3a0d), LL(0x6f2eb06c,0x3e73d7f3),LL(0xaff72fe1,0x8fe4228f),LL(0x2703c35b,0x801d9246),L_(0x0008cf4d), + LL(0xa7b3253e,0xb347bed9),LL(0x87c9db13,0x23367d55),LL(0x50ebab37,0xe6bc8d5f),L_(0x30c060ce), LL(0x9df003d2,0xd78ada2c),LL(0xc3892b4e,0x4989b92d),LL(0xf9b3e9f5,0x2dbbe952),L_(0x48275a07), + LL(0xdc3c1c12,0xfb949b08),LL(0xa8296f5d,0x010b8834),LL(0x69483176,0x73825d94),L_(0xe4f380a5), LL(0x7caf444a,0x3cd86c08),LL(0x6ae21350,0x8ba59225),LL(0x6d5a74bd,0x1e6e636f),L_(0x6a2c970e), + LL(0x51eea6cb,0x69de6020),LL(0x15974dd5,0xf383a0c6),LL(0xa7aeef7a,0xec5ab9f7),L_(0x25652e0f), LL(0xb56f14b1,0x7b78f2d9),LL(0xbe7272ca,0xb4e91769),LL(0x0886a015,0xb8c7fe1f),L_(0x1c2ac0a8), + LL(0x7b2d2c1a,0x3f05a7cf),LL(0x7d9a8322,0x8a978fa5),LL(0x2541de9c,0xb09f953f),L_(0x44b3e4d5), LL(0xa6872622,0x0275a86c),LL(0x5ca47329,0xc3073baf),LL(0xe68e138a,0x70608786),L_(0x426bc12e), + LL(0x22def544,0xc1d1a75a),LL(0x054cdb35,0x9d05dd03),LL(0x13908388,0xcac954e6),L_(0x6b377854), LL(0xfb37d06d,0x42873dd7),LL(0xd85fe89f,0x77686ddb),LL(0xbd6e2bfe,0x0a7aae21),L_(0x3116e23a), + LL(0xd00f34a0,0x6d3f30d8),LL(0xb72f2ac8,0x793a6f93),LL(0x8e54b8c8,0xa3ebe240),L_(0x06c18c5d), LL(0xd3635d7f,0x7527fefe),LL(0x848fd8b1,0x7ecf5710),LL(0xe5c193ac,0xc5b359a4),L_(0x10bdedf6), + LL(0x6a69e07f,0x904b6a99),LL(0xf73ecc50,0x74691c89),LL(0x377aafb2,0xf8dfe49e),L_(0x3116ab06), LL(0xbc4bbf14,0x32c37416),LL(0x4f9adb4e,0xa6bc7861),LL(0xb481fbff,0xab667972),L_(0x50e39a47), + LL(0xd40e7122,0x03318c55),LL(0x2b1438a6,0x5cc23063),LL(0x7d21722c,0x353dafc9),L_(0xb52192c7), LL(0x47d32da4,0x8e5c3c5f),LL(0x356f1a48,0x4c620a4a),LL(0xc2e2dc49,0x6ac8f16a),L_(0xa8078172), + LL(0xe0aae2f3,0x27e1fc19),LL(0xcaccad70,0x9a5089ff),LL(0xd066d3ec,0x83ce796d),L_(0x1d39df6c), LL(0x3d7bd85e,0x2109b271),LL(0xd4d73f64,0x032bfaaa),LL(0x4c64918b,0x28f7e9fb),L_(0xead7a6eb), + LL(0x81a3a911,0x1ec15d92),LL(0x072d4b8d,0x0de6b8aa),LL(0x25d33d78,0xdb94f1aa),L_(0x163e5ede), LL(0xcda754fd,0x0aa41bac),LL(0x8bb9f4cc,0x26a33edd),LL(0xaed5fcc3,0x03b383eb),L_(0x7bac84ac), + LL(0x48a08610,0xe3c9d6e7),LL(0xaf9b1185,0x1c9dda6d),LL(0x37a0510c,0x87290f47),L_(0x395d8f39), LL(0xe0b9b993,0x04968462),LL(0x2065fe67,0x79bcdef3),LL(0xba7c23dd,0x3c9dfbbc),L_(0x48b97684), + LL(0x4867362d,0x9b7dcff4),LL(0xa45a3647,0xf6be4fd8),LL(0x9dd99069,0xfc837cc5),L_(0x463e52d6), LL(0xa104db39,0xe73440a8),LL(0x868e9fad,0x557bc30e),LL(0xca38a194,0x0b9107de),L_(0x3b4896a5), + LL(0xf9b3c663,0x0572aa2e),LL(0xb232d67e,0x6cf8f9e9),LL(0xdc3bb3a4,0x578fa229),L_(0xf77deefa), LL(0x7b780979,0xe2e55595),LL(0x3dd1f800,0x1ae5674a),LL(0x7d70eb0e,0x6287e785),L_(0x6ca9df37), + LL(0x6027714a,0x6418b451),LL(0x6b5f95e4,0x008c73fe),LL(0xc7a41b44,0xb52d2690),L_(0x7e9e5e8d), LL(0x07d5913f,0x28af32f1),LL(0xba6493ac,0x70ace436),LL(0x249d5a1f,0xe2634904),L_(0x9de83ae3), + LL(0x91c3f75f,0x7ac06d6f),LL(0x5d22beb3,0x8cbbcc95),LL(0x6114dddf,0xb3f952c3),L_(0xd55713d1), LL(0x50a376be,0x95633a5e),LL(0x402443f5,0xf0277528),LL(0x900af3d2,0x2aac428e),L_(0xe7ecc502), + LL(0xede089c1,0xafe013ca),LL(0x9bc0d189,0x24f9b481),LL(0x7d9ab065,0x7064130f),L_(0xcb09791d), LL(0x88599e66,0x1a8684d1),LL(0xdbafc741,0x5b828087),LL(0xa7784039,0x274e2738),L_(0xa61d0693), + LL(0x04a64675,0xabf54d81),LL(0xdec1f49f,0x46144f05),LL(0xc232e58f,0x30cddd92),L_(0x5b7d5bc7), LL(0xc2b48798,0xfb8c420d),LL(0xb96fbab6,0xb4d97ee3),LL(0x8d97d0d5,0x8101b1e7),L_(0xb545f661), + LL(0x98af21f0,0xcd4340b9),LL(0x955d1c9d,0xda56271c),LL(0xcf1eb8ab,0xcfc8c025),L_(0xc22b3fb1), LL(0x99747eea,0xdff889cf),LL(0x01a895b8,0x16fb6025),LL(0x22ae0ed8,0x56d4b8ed),L_(0xbdaf7eb9), + LL(0x0745f518,0xc7781923),LL(0xb03571c4,0x6cd5a87b),LL(0xe1112158,0x734d96e5),L_(0xc2f71082), LL(0xfbc17aec,0x7d6ae9b9),LL(0xae846849,0x6b340fbb),LL(0x7a81eba8,0xa462be47),L_(0xf156efcb), + LL(0xbad5359f,0xcc17a45f),LL(0xed5f0c48,0x3962aaeb),LL(0x18daea3d,0x24dea50b),L_(0x2620a5cf), LL(0x2db980d5,0x94722851),LL(0xe16c7ff8,0x17a80fd9),LL(0x0cd73f37,0x0ef75926),L_(0xa7a0ce92), + LL(0x2994a0c6,0x0d862f7d),LL(0x66b2e45e,0xa326bcf2),LL(0xb39a090d,0xebe228c9),L_(0x8223ef31), LL(0x4fe39e14,0x292a329c),LL(0xe179a973,0xa1f5d624),LL(0x3f828ea8,0x70ad621c),L_(0xc04d2e0b), + LL(0xb7237a24,0x94de2084),LL(0x91138a01,0x0c93caa4),LL(0xd97e1daa,0xb74fa915),L_(0x668329b9), LL(0x146706c8,0x23c7b716),LL(0x23606a0c,0x54c6c8af),LL(0xbe011ca0,0x2ef6e4a4),L_(0xb2029de6), + LL(0x33851602,0xbee34e4c),LL(0x7b910cad,0x8e7c506d),LL(0x2b92ce18,0x62b378cc),L_(0x5c490234), LL(0x95f16f74,0x661c0891),LL(0xa5073d8e,0xf7df962d),LL(0x6fdadac1,0x8f3ef4f4),L_(0x373f29f4), + LL(0x23206f03,0x235bc318),LL(0x7518fa1b,0xec43017e),LL(0xc5d4dab6,0xf547238a),L_(0x7e5c2b82), LL(0x25d0ebfc,0x8357f1bb),LL(0xfb571aa9,0xe137e308),LL(0x996a92a8,0x629e8aa7),L_(0x90f06a9f), + LL(0x4a17680f,0xa8c7b143),LL(0x84f5aab1,0xb84edf8c),LL(0xf2acd3a2,0x8b3a07a1),L_(0x5d4c58ed), LL(0x2f0d5862,0xcd2f47fd),LL(0x31cb8d1e,0x0d4da406),LL(0x0cd04a92,0x8ec4fcad),L_(0xb261a451), + LL(0x91a38da6,0x76eb9c38),LL(0x12868725,0x0a5b71ac),LL(0x2d328a16,0xf6122daa),L_(0x9232586e), LL(0xa1eceeb7,0xe9555051),LL(0xc35149d8,0x55020182),LL(0xae8df945,0xcee3a23b),L_(0x6020896d), + LL(0x88241ba5,0x6839ac95),LL(0x75c22b80,0xb01a4c72),LL(0x4e73d41b,0xdbce70d9),L_(0xb035070a), LL(0x25457ffb,0x4155bcf6),LL(0x6c514452,0x79b75848),LL(0x3c25b38e,0x8c33a19b),L_(0xb8f49889), + LL(0xb2bc7aad,0xc7b2eb56),LL(0x96c5270f,0x8112c87f),LL(0x15208222,0x612138e3),L_(0xac8a2edb), LL(0xa6392e61,0x7fcbfb2b),LL(0x95cfe395,0xb4541276),LL(0x134be404,0x3de33a7c),L_(0x2cf45438), + LL(0x27c80982,0x6d995371),LL(0x429942a9,0x1e16c2dc),LL(0x0e59bd58,0xfbad79fb),L_(0xd34caf67), LL(0x481ca9cf,0x6cb52c72),LL(0xfc7ee8ab,0x7e9afc85),LL(0xfda3af10,0x41a661b7),L_(0x7e623e57), + LL(0xc421cd21,0xb68b5cf7),LL(0xd94c04b7,0x107cec0c),LL(0x81208f8b,0x383162ed),L_(0x454f3ae7), LL(0x2a8bfdfe,0x0f977619),LL(0xdeb81615,0x382decc1),LL(0x8e5df4ce,0xf0414c84),L_(0x0f3c79aa), + LL(0x15f29277,0x89f9b2a7),LL(0xa1f621ee,0x817b0a57),LL(0x6ee121aa,0xf56eeafc),L_(0xcf7c22d6), LL(0x0bbfabbf,0x52eb19cc),LL(0x018d1b99,0xc2b09e5e),LL(0xa82ae35b,0xd9cf95b8),L_(0x34b4bf9a), + LL(0x017073a5,0x4f731dd0),LL(0x0235def1,0xd07b63f1),LL(0x5e605b0d,0x4769aa10),L_(0x7252588b), LL(0x26cfa7a5,0xfac43672),LL(0x01631f47,0xbb074c5c),LL(0xe02c4623,0xb6536982),L_(0x061acc5a), + LL(0x0bd8823c,0xca3a4054),LL(0x57b5c945,0xbee559f1),LL(0xac8c4662,0x54b8d320),L_(0x8ee0f3f7), LL(0xfc9e1370,0xafaff913),LL(0x7ef2aacc,0x15a9349a),LL(0x81208c2d,0x0d06fd25),L_(0x00df14a3), + LL(0xc4906f44,0x3cb8d3d5),LL(0xc2b5f2bf,0xef621a83),LL(0x42fee26b,0xcb150abc),L_(0xa4673146), LL(0x8c081901,0x2e00a64e),LL(0x4f184ea9,0x4b9fa7b4),LL(0x7c4d0b48,0x60c91243),L_(0x05745a02), + LL(0x36d03f9b,0x7049620b),LL(0x54cfa460,0x29c0b51d),LL(0x50448eab,0x3f0b2dd9),L_(0x9ed5c1c7), LL(0x217c3b7d,0x8eca2e20),LL(0x7e7352b9,0x20883efa),LL(0x3afc2315,0xc23211b5),L_(0x98c20d32), + LL(0x3c1d5c98,0x00ef3bc9),LL(0x03002427,0xf1f069cc),LL(0xc5bebf01,0xf661ac1a),L_(0xb01d642d), LL(0x54219e42,0x25a7fa91),LL(0x199aa1af,0xf217cbfa),LL(0xf0d4c0bf,0x4f2df7eb),L_(0xf7508aca), + LL(0x8f2773db,0xb21c4cfa),LL(0x966e2843,0x52c04b61),LL(0x590a0567,0xf35f4a84),L_(0xf255a433), LL(0xcc2e1149,0x04ae8b1c),LL(0x13360c1c,0x92f66f03),LL(0x4053731c,0x20aba0c9),L_(0x20d59987), + LL(0x0a4bb9e9,0x9ad08ec7),LL(0xa3719263,0x54a4bb3e),LL(0xc9bcb020,0xf5564618),L_(0xaa1b7ba0), LL(0x2f0cb8aa,0x64275534),LL(0xd09acd41,0x3340504f),LL(0xd7e50869,0xaf97cbcd),L_(0xf4025b06), + LL(0x96f1d3cd,0x9f391b14),LL(0xbd5e9744,0x5f6bed3b),LL(0x008d20fd,0xc79a2572),L_(0x905330f0), LL(0x77d13f02,0xb71f049c),LL(0xdb745573,0xe088bd21),LL(0xbd449f9b,0x3a76432d),L_(0x1de7f0bb), + LL(0x12dbb0cf,0x4f53adb9),LL(0x2e4f3aa7,0x13ff9790),LL(0x61674465,0xa80fdde3),L_(0xfbfa4069), LL(0x50e102a9,0x28391810),LL(0x22e32485,0xd119b153),LL(0x47aca745,0xeb9f0102),L_(0x3f33e272), + LL(0x6dc8e416,0xb01555de),LL(0xa53f861f,0xc5669efc),LL(0x14d3a409,0x0c68a597),L_(0x3ad8c86e), LL(0xed8dbd60,0xc5474f33),LL(0xda13401f,0xd21a574f),LL(0xf6b7aed6,0x0d2d58af),L_(0xbe2e5676), + LL(0xb63ca145,0xa81eba20),LL(0xd0d7b9ca,0xf6e2c558),LL(0x4e439115,0xe8f8d978),L_(0xb32fdbaa), LL(0x8cf406a7,0x23a5c7d1),LL(0x7072ea14,0x4e16a729),LL(0x3db551d5,0x88436ae6),L_(0xd7c49d77), + LL(0x32172eb6,0x7d243c47),LL(0x52231b01,0x4a4714a5),LL(0xed3510a8,0xfde7f4a6),L_(0xccb27287), LL(0x1ed13735,0x4721f266),LL(0x5a39f8ef,0xf2889694),LL(0x99b1258f,0x45218687),L_(0x4c27d1de), + LL(0xe91d500c,0xcf933e7f),LL(0xb7d6326c,0x2d1ad273),LL(0x572c7767,0x79dbaa6e),L_(0x27d7d939), LL(0x17390032,0x6c991f09),LL(0xed70ac81,0xbaab3191),LL(0xfc86b907,0x93c673b6),L_(0x94754f34), + LL(0x3543430c,0xc2f96d5f),LL(0x0ea5f8eb,0xac730ca9),LL(0x376185cf,0xfeac300d),L_(0x41737668), LL(0x96df0a25,0xf035d502),LL(0xb519c6a4,0x9ca93571),LL(0xd64df17a,0x01970aaf),L_(0x18c05bf9), + LL(0x471f2099,0xa5fb90b3),LL(0x53b53a2a,0x2aa370a3),LL(0x09b936b9,0xcfb37658),L_(0x83591048), LL(0xb2dfe05c,0x1e8ce8f7),LL(0x70ca625a,0xcf8806b6),LL(0xf062523d,0xa1c74034),L_(0xc4ca8211), + LL(0x02fd45e9,0xf0d5ce43),LL(0xf94539de,0x86092a2c),LL(0xc28ad111,0x0990658e),L_(0xfc301c6d), LL(0x850acec2,0x097c875b),LL(0x333c1a83,0x79052ae9),LL(0x651a1ec4,0xdf5b00e0),L_(0x8f342e96), + LL(0x7c4aacf4,0x12306aef),LL(0x23bf4f82,0x0369e1f2),LL(0x86d193a6,0x3bdb2181),L_(0x2df259e0), LL(0x4d9c6a9b,0x2e587a72),LL(0x420fa4b9,0xe30d76a5),LL(0xaf97ed7f,0x20fbaabf),L_(0x6c230751), + LL(0x044baaf4,0x04047c5f),LL(0x8d43be85,0xe4e3879a),LL(0x436ef7e0,0xed64bf04),L_(0x6b436c30), LL(0xa9c7bdcd,0xc1679bbd),LL(0x12340567,0x38ba10d1),LL(0x83b598cb,0xa99537cc),L_(0xee36e118), + LL(0x18eb43ca,0x5aa367ce),LL(0x94ff23cf,0x2dc9b946),LL(0x066d6c04,0x3a41602a),L_(0xfe10e991), LL(0xfcc4d378,0x3dbf71fc),LL(0xd6bc1c02,0xc9ccbaad),LL(0x0aacb390,0x985954f5),L_(0x468870a9), + LL(0x1f6e5aac,0x4d58f138),LL(0x088912a4,0x72e75855),LL(0x389e7a2b,0x4b5aa1e6),L_(0x5be86bd9), LL(0x29d90bc3,0xb35b2d2a),LL(0x67e8a5ea,0xaca3b9cb),LL(0x832c687a,0xe2911a57),L_(0xed56df90), +}, +/* digit=8 base_pwr=2^56 */ +{ + LL(0x56b1b68d,0xa766a468),LL(0x7a690380,0x7f0a8ccb),LL(0x15b9ce0d,0x8bfb375d),L_(0x0afa00f6), LL(0xe7944b6b,0xe502fd6a),LL(0x0768ebca,0x079ac7ac),LL(0x956dea42,0x3fc3f258),L_(0x78be0f9a), + LL(0x7157aab8,0xde649372),LL(0x1fe5b1db,0xec032ca1),LL(0xdc9ac491,0xaf00af85),L_(0x42bf819d), LL(0x3d3a586d,0xb1b4ed91),LL(0xc6e9377d,0x0da4ec46),LL(0x2fb5dda7,0xe69eb235),L_(0xa5e4f04d), + LL(0x4a718a91,0x6813c47e),LL(0x520eeb02,0xcc1aaacb),LL(0xbae4540e,0xfd6dde20),L_(0x5d3611cc), LL(0xa139969a,0x47c1ef53),LL(0xa02a9c1d,0xff24f01d),LL(0xa1779b32,0xafe60f72),L_(0x0b879e68), + LL(0x6e7fd2ac,0x206a6f14),LL(0x2a13d9f9,0x43a11c7f),LL(0xf31bbf6c,0xe2ff97fb),L_(0x56107ea7), LL(0xb83101aa,0x0b2c24c6),LL(0xdc078721,0xd00a24a9),LL(0xc5ea4420,0x80054822),L_(0x76c2e6e5), + LL(0x5d302b86,0xa3ec0bc2),LL(0x58cdfe77,0x7469b928),LL(0xde033220,0x9823907b),L_(0xce8c1169), LL(0x8d35908e,0x668b8357),LL(0x90f7260f,0xcd96886e),LL(0x388da84d,0x8e759b7d),L_(0xf9144eb3), + LL(0xb0068632,0xb87f256b),LL(0xd3ec8de1,0xb7562058),LL(0x554d7fad,0x68b8bed1),L_(0x7d3ba275), LL(0x5349ed32,0xaa23f48d),LL(0xf8c55847,0xac744602),LL(0x125ee820,0x150da1ca),L_(0x16333e9d), + LL(0x16a4c78e,0x26b49a2d),LL(0x9714ac19,0x69e5dc39),LL(0x50449596,0x2af6b2c1),L_(0xab90cd95), LL(0x07439dc5,0xf5736c67),LL(0x8b3909e8,0xeeb790dd),LL(0x0cc523dd,0xae2f8bbf),L_(0x21833381), + LL(0x1c12b2d6,0x7628ecee),LL(0x6d88b422,0x0692cc8d),LL(0x65398808,0x5048f64f),L_(0x0e3b6999), LL(0xd880351e,0xae75c4fc),LL(0x3ffdb69e,0x3e2ff89d),LL(0x5f9ca860,0x435a454d),L_(0xd321ca44), + LL(0x1dbe11f1,0x805cdd89),LL(0x0b435944,0x0562fe66),LL(0x5d6a45d5,0xc9eb9f56),L_(0x0394c9fd), LL(0x1fbcb273,0x4b25d574),LL(0x8a17364c,0xb78a31bf),LL(0x1466b27f,0x468c47f5),L_(0xfd233c89), + LL(0xb836e3e6,0xf6743281),LL(0xe6b59ce6,0x13f41d4e),LL(0xd77d4340,0x397a6846),L_(0x45693c8c), LL(0xbd735497,0x6560f3f3),LL(0x5f68d915,0x61405ead),LL(0x119c162c,0x9870a7f1),L_(0x8b74bea7), + LL(0x4f81eb06,0xdd4548bd),LL(0x271e6f36,0x49ea4452),LL(0xff17a13f,0x54ebcb9f),L_(0x83b9157a), LL(0x3e7236a9,0x4d44b2ce),LL(0x7f67612f,0xfeac4aaf),LL(0x0d85cf3f,0x7d7cb315),L_(0x7efb9b30), + LL(0xb0f78faa,0x82bbcdcf),LL(0xf1c04007,0x01559ec3),LL(0x7cdb7afd,0x86ff3957),L_(0xb1597d4b), LL(0x6c320e72,0x45b1afec),LL(0x522b415a,0x9d12c298),LL(0x6b93f2de,0x3e21f849),L_(0x3b1a1d95), + LL(0xc12b655e,0x219c2865),LL(0x6265e8e1,0xc6daf8cf),LL(0xf3ec2977,0x10269fa7),L_(0x4486877c), LL(0x8de615f2,0xb448dafb),LL(0xfeda6cb4,0xac03ce69),LL(0x0fd25b12,0xb96afd44),L_(0x0f6ae181), + LL(0x069f14f1,0xc970df0c),LL(0x614fb451,0x8cddee54),LL(0x0475ddb9,0xac228ae6),L_(0x3ef6e29e), LL(0xc73f5027,0xbdeb5cb1),LL(0x2198420f,0x82098200),LL(0x8694111c,0xc11479ab),L_(0x25c29038), + LL(0x0c73fe99,0x4aa502d3),LL(0xdb603ed0,0x6c27909f),LL(0x4f620050,0xf76957c1),L_(0x0e7e084e), LL(0x1565b30b,0xfba38431),LL(0x95ddd9af,0x65c7038d),LL(0x82a6a124,0xe7db47e4),L_(0x1f3b155f), + LL(0xfdab8dbe,0x773fef23),LL(0x70802c0e,0x011efbb8),LL(0x8698e8b3,0xa4f63ad2),L_(0xa90d02ab), LL(0x60003ff7,0x5aebc019),LL(0x586f4378,0x00bcb9de),LL(0x859f79cb,0x6a9d21d2),L_(0x79d6150e), + LL(0x8a59362f,0x21480f43),LL(0x37e25e9c,0x4a37a459),LL(0xe55838d3,0x36aab717),L_(0x85a89af8), LL(0x1d33aa1c,0x86dc1133),LL(0xb94719f6,0x0a1e4a76),LL(0xf9d8ea27,0x7a9f7b61),L_(0x2b10e5f5), + LL(0xf5ab77c2,0xb7c034fa),LL(0x91179a6a,0x89204639),LL(0xd0dd8159,0x4ba725f9),L_(0x39ccdad0), LL(0xa510602e,0xc72415fe),LL(0x85069c0f,0xa6bc7ccc),LL(0x7d585ae6,0xc90e9462),L_(0x51cea642), + LL(0xaed13816,0x84f02e2a),LL(0x24576ca8,0xffdbb5f9),LL(0x3c9269f7,0x5c61743e),L_(0x498dc0e3), LL(0xd5bc6379,0xa9132224),LL(0x10fc37d9,0x12f65fd8),LL(0x817445f8,0xd0c982b8),L_(0x4f79fcf1), + LL(0x40f7fe3e,0xcb5018ef),LL(0x2fdb045f,0xf32b39b6),LL(0x621a4bd4,0xe7c33edc),L_(0x04fd5dd5), LL(0x7972a384,0x586d4bef),LL(0x22c07adc,0x81211aa8),LL(0x8c46dfe1,0x5b192417),L_(0x68b0abc0), + LL(0x30ddafb1,0xe56640a6),LL(0x55279999,0xe01ca53a),LL(0x6b7735ff,0xa4ba7379),L_(0x1c1eab8d), LL(0x184d5073,0x1f2853f9),LL(0x0a5a3694,0x0c65322e),LL(0x77546dfc,0x814529cf),L_(0x0117bc49), + LL(0xe44092c1,0xdcf36025),LL(0x29873c6a,0xa38b0382),LL(0x2beaabb1,0x57dab020),L_(0x4481b257), LL(0x3fc25cda,0x16190d52),LL(0x3dcabd82,0x632514c7),LL(0xc69b7cce,0xcb82be1e),L_(0xcf490cc0), + LL(0x79d31337,0x78cc588c),LL(0xd718033e,0x5d3ca504),LL(0x0951b466,0x82b9278f),L_(0xc14b1f81), LL(0x6c160f3e,0x308d56c2),LL(0x833254cc,0x460ed897),LL(0xcccc63ee,0xd27b55ac),L_(0xdf9c95f1), + LL(0x2097cb5b,0x6fb48059),LL(0xc33f19e6,0x8b4a82f4),LL(0x3bf0f71b,0xda304a5b),L_(0x7efe85d3), LL(0x5d609c9e,0xb32599e2),LL(0x7d311503,0xea5b4566),LL(0xc072d5ff,0xf9f2dc57),L_(0xf9df545e), + LL(0xdacba736,0xbc32b9c9),LL(0x0ab7c149,0x154849a8),LL(0x29e4dd9b,0x11dfc29e),L_(0x41ace68f), LL(0xc8616f74,0x971ba9a3),LL(0xb0be2d45,0x15776b0b),LL(0x8325c930,0x294a4ec5),L_(0x455623af), + LL(0x16f8969c,0x235d5c34),LL(0xb8ca3700,0x57aa97da),LL(0x5c79afae,0x417467c4),L_(0x796984ff), LL(0xe7950969,0x59ca92ce),LL(0xcd03a014,0xb1a7c34d),LL(0x6203a921,0x066a9647),L_(0xd0909f6f), + LL(0xfb7e2684,0x84d55477),LL(0x0c455cdf,0xbdee722d),LL(0x74bfcbf4,0xf514f2bf),L_(0x8a60e5e7), LL(0xba7058a4,0x4713d1ae),LL(0xecf4c404,0xa6c17b5e),LL(0x8941ddd8,0x261d9ac7),L_(0x7f2f5187), + LL(0xac5e5ce7,0x76370390),LL(0x64cf9d29,0xfce7f694),LL(0x7c34fbe8,0xe3b42849),L_(0x6bc43f22), LL(0x8781925b,0xe273ae40),LL(0x49d3a7bb,0x777881ae),LL(0x6fc64556,0xb0f8e854),L_(0xb4b40191), + LL(0x024a93d5,0xa6b66d66),LL(0x4f2aaf4a,0x56c335ae),LL(0x0805e125,0xe1e39e4e),L_(0xb67d8a1b), LL(0x9bb438e7,0x0e8dde7c),LL(0x24683c26,0x06a38af7),LL(0xa234eb6c,0xd74870f8),L_(0xfa606ebd), + LL(0x05b51c3a,0x62d8ac43),LL(0x8c1548ed,0x8eefcd86),LL(0x3b151865,0x3c047e9d),L_(0xf77c0004), LL(0x76ea2961,0x82998a41),LL(0xa3b38ce5,0x110b3b2e),LL(0xbcfe60ba,0x89b19d6d),L_(0xe68575e8), + LL(0x8765ec49,0x36e47913),LL(0xc6e4ab7e,0x767e806a),LL(0x81bc9bd7,0x38103684),L_(0x5490cabd), LL(0xb8107b91,0xbe9c24b5),LL(0x63e98957,0x43623ea0),LL(0xf06a675d,0xcbeab5be),L_(0xf0e9a898), + LL(0xd064269c,0x1ccbf4e2),LL(0xfe422ab3,0xe21fbca6),LL(0x5cee0a5c,0x263913a5),L_(0x71de7df5), LL(0xa3fc8bae,0x047f1949),LL(0x0df59ed6,0xd1a5d116),LL(0x35a0d475,0x9e3419f5),L_(0x75946143), + LL(0x36c9f809,0x631145ae),LL(0xa1db7d1d,0x0c423276),LL(0x4bf70210,0x2f1584d5),L_(0x8bd52dbd), LL(0xa19699d2,0x8015f024),LL(0x37a40f15,0xf9755c59),LL(0x25538f41,0x387ddb31),L_(0x90af00b0), + LL(0xe8509934,0x9e321e61),LL(0xd2850d36,0x7dc68e41),LL(0xf8206e77,0x58e783b2),L_(0xe67a1042), LL(0x906883cd,0x6f41c769),LL(0x51b97936,0xbb9527f5),LL(0x38e2f27d,0xcf17789e),L_(0x5273cd0e), + LL(0x1bf5a9fa,0xae648e71),LL(0xde909240,0x3f5f8275),LL(0x93245241,0x80482c3b),L_(0x0a2260c5), LL(0xef9f19ab,0xaa9e665a),LL(0x1eb26ff8,0xad522975),LL(0xce379c80,0xf8653e32),L_(0x4f190ba9), + LL(0x19d1e571,0x4023698a),LL(0x4e5e5355,0xa19db2ab),LL(0xd2800586,0xaee5fc4c),L_(0x115617f9), LL(0x378c9aca,0xbcb82922),LL(0xfb1dcef8,0xc3b8fa14),LL(0x1b19035c,0x1c3a4f74),L_(0x190f3cd7), + LL(0xf201d617,0x88de2216),LL(0x778baf2f,0x409f13a5),LL(0xfc1697ff,0x1f4ee383),L_(0xf8b34103), LL(0xc70f2c0b,0x596702b2),LL(0x40ecf08e,0xaa0989d3),LL(0xcc60a8c2,0x5ba1fcbb),L_(0xb0f51744), + LL(0x3f768dfe,0xee929e0b),LL(0x5c573a73,0x171719ed),LL(0x91e6ec5d,0x48549719),L_(0xb182f69c), LL(0xa3e2860c,0xa30f11a1),LL(0x5e01688d,0xf804fa42),LL(0x58c435e7,0xd81ff657),L_(0x49e16053), + LL(0x3ee264bb,0x6d74b27a),LL(0xcb2992ae,0x776b6b64),LL(0xa97538c5,0xdc494d53),L_(0x282ddfc8), LL(0xe4acc7fe,0xe5d1ac83),LL(0x81afb14c,0xf0a59879),LL(0x9b5f787d,0x0730aa6b),L_(0x76168d56), + LL(0x5d202341,0x15ca4cef),LL(0x67cf5e49,0x03078647),LL(0x7cbaad05,0x7a152a3a),L_(0x93ecdb65), LL(0x5c759c30,0x1d48b5cf),LL(0x8a07288f,0x954731e0),LL(0x67babf9e,0xb890cb9b),L_(0x3f7cb4f5), + LL(0xfbda2539,0xd314af3a),LL(0x66f79c66,0x333f8c94),LL(0x07bcc285,0xd37bbc81),L_(0x1102429d), LL(0x7ba7309f,0xe5686c59),LL(0x647c832d,0x96218ea6),LL(0xbfb2d48e,0x7bc6fa2a),L_(0x524c8c0d), + LL(0x91273fc1,0x9f330845),LL(0x6f3e8880,0x8d3155c8),LL(0x229515d9,0x4e0d0091),L_(0xa37bad0a), LL(0x3cca8a71,0xc538ea5c),LL(0x34161b0c,0x7f68ef37),LL(0x412db800,0xfc39477c),L_(0x5112c574), + LL(0x202d77d0,0x3f50f1ae),LL(0x56e724be,0xc9d43dee),LL(0x2163dc89,0x82838e71),L_(0x3e07b5b6), LL(0x68b2f3b5,0x028beb58),LL(0xfb4f5d6e,0x2f67136a),LL(0x63c11941,0xf0576d72),L_(0x36e94125), + LL(0x79a7908b,0xfe08b743),LL(0x7ec4b29f,0xe4c9aa01),LL(0x3121fc40,0xe6d6660d),L_(0xa4b98339), LL(0xcc2b942f,0x54acd180),LL(0xc715b5ed,0x4b3ac4d4),LL(0xde75824c,0x91ca23fa),L_(0xe3ff9f1c), + LL(0x46582483,0xa927f3c9),LL(0xbac6e7bf,0xe88a430f),LL(0x1f445777,0x3e33ca59),L_(0xbdfe1920), LL(0x71dd2627,0x54a8126d),LL(0xebc9b59a,0x9c986433),LL(0xbec3ed13,0x71db698f),L_(0xd57f9239), + LL(0x731eb54d,0xa93b1752),LL(0xa3516b9e,0x6f7bfa08),LL(0x873d3a45,0x4f66d06f),L_(0xbdd10c66), LL(0xc97be3e7,0xf6435b79),LL(0xa3533348,0x207b9b6c),LL(0x1c6c74ae,0xb64b6b70),L_(0x9037812e), + LL(0x74ccc35a,0x4be5a10b),LL(0x425bdafc,0xb790b940),LL(0xc32a94b3,0x098b38b5),L_(0x026ae426), LL(0x4ef7c297,0x18feb7b4),LL(0xda85e218,0x6f3016a8),LL(0xad72b80e,0xbdf2db87),L_(0xa853e440), + LL(0x0ca1adfe,0xb13f2859),LL(0xbd4a0fb5,0xd393e869),LL(0x277e9cd1,0x90b15b52),L_(0x30056558), LL(0xf705f505,0x2c2d2fb1),LL(0x2c3d697e,0xf49260e8),LL(0xcb991398,0xfed48a8d),L_(0x6c792dc1), + LL(0x2687c048,0xc3ca0fc2),LL(0x97ff87d8,0xbb9d5b5b),LL(0xe241d7c5,0xef9d2be9),L_(0x4d4d8a9e), LL(0xfe8acc57,0xc5011f32),LL(0x499812d3,0x8545eca9),LL(0xfa8d83be,0xc270dd4a),L_(0x162b0c9f), + LL(0xb3951e09,0x78a942de),LL(0xc228c3e5,0x57fae761),LL(0xbc46ee6e,0x83169c42),L_(0x1aeae5f1), LL(0x340fc522,0x669821ab),LL(0x51be95bf,0x09548071),LL(0x92a505ce,0x83ec59bb),L_(0xd2e39c6b), + LL(0x3ce8adaf,0x41ed0956),LL(0x8981ccd6,0xc0e55576),LL(0xba8eb552,0xa018b90e),L_(0xf5b2000d), LL(0x67316d00,0x5703fba5),LL(0xf9b6f0d3,0x6078c1e3),LL(0x99f221d3,0x047eec80),L_(0x5f0d4326), + LL(0x81739c80,0x6f11b810),LL(0xf085c79b,0x1e31485f),LL(0x9ddc21df,0x18e4d189),L_(0xbe0bc274), LL(0xe78c5638,0x439dc1dd),LL(0x63733145,0xa22acbf7),LL(0x9f469794,0x161460c0),L_(0xaa724fea), + LL(0xcd4a85b1,0x7e2cff5d),LL(0xaf20e779,0x998dca20),LL(0x3d5bd0f7,0xbfc65978),L_(0x4e61a410), LL(0x388987a1,0x9b689b7c),LL(0x04dbc066,0x3ea487dd),LL(0x5b3b94d0,0xa77864e1),L_(0xe89088a7), + LL(0x39413fae,0x796a4cff),LL(0xd204de5d,0x978f94f4),LL(0x2c42836e,0xdabb2333),L_(0x20078e8b), LL(0x683de92d,0x5fe8df50),LL(0x14c267e5,0x705a1d2a),LL(0xf68dcb2c,0x16e49003),L_(0xcbb04811), + LL(0xc7d4f66e,0x02ce592d),LL(0x6c59b80b,0x449fe24e),LL(0x49708f6f,0xe167be96),L_(0xfb3a4da9), LL(0x7456adb9,0xafdb07bc),LL(0xa22499e7,0xfc9ec5ea),LL(0xb52ff28e,0x55b069a6),L_(0xc0fc9e6f), + LL(0x1b0cec1b,0xf4484ca3),LL(0xb1289e3f,0xde798873),LL(0x05b5ac5f,0x87ada89d),L_(0xf204ab3d), LL(0xbc022acb,0x93b1313e),LL(0xd2127d5a,0xa4d90949),LL(0x71a6efc0,0xd897f8e5),L_(0x88e1bcc1), + LL(0x5f811bb3,0x9a725a78),LL(0x9323ffc2,0x1f6cfcf3),LL(0xe1abeb72,0x6d3872f8),L_(0x71c9aa7f), LL(0x90cec4c5,0x400da758),LL(0xdb55d5c9,0x74d4d54c),LL(0x6fed8555,0x4bc5cda0),L_(0x99c927f2), + LL(0xb4a54352,0xa9b678d4),LL(0x21c3a4ea,0xf079b22c),LL(0xcd89ee25,0xec3738e6),L_(0x12db8e7a), LL(0x17543420,0x3b3edf75),LL(0x83b1a5b2,0x46732c66),LL(0xcfead94a,0xa689fe7e),L_(0x658ac775), + LL(0x91cb6be3,0xa3af0a2d),LL(0xaee7da78,0x37e8c56d),LL(0x44cf93c9,0xc55af358),L_(0xb97c6473), LL(0x2941f1b7,0x85525c49),LL(0x9ccf06d0,0x7fe18358),LL(0x32c35949,0xbf4a513e),L_(0x48b1c1e8), + LL(0x8adf6262,0x6619bed5),LL(0xdba7f0af,0x5bb1a013),LL(0xdac22d18,0xe70d7542),L_(0x0c259dfc), LL(0x0bfa91a2,0x195a05ff),LL(0x1ebdd562,0x52bb429e),LL(0x34d0f00f,0x904ed07d),L_(0x9b10bb9a), + LL(0x4331ebb9,0xf071319d),LL(0xa2d5580a,0x05e7c4f1),LL(0x1da3b676,0x7abf809c),L_(0xcca6bf1e), LL(0x9290a718,0xf68174d5),LL(0xc59959c4,0xa93e4d89),LL(0x7f4cbc44,0x93b784f5),L_(0xe5e1c561), + LL(0x7b8373a6,0x1bf6e31b),LL(0x242d1745,0xb8888fa2),LL(0x010ee8af,0xa7cd3971),L_(0xa9bf1772), LL(0x7e33265e,0xc3866a61),LL(0x541559ba,0x9e1bb3ef),LL(0x9be25bb9,0x1c158895),L_(0xd26003c0), + LL(0x187246e5,0xcb1700db),LL(0x97112d61,0x3aa73e7a),LL(0x93e55e2b,0xb2ba60e5),L_(0x34003d9b), LL(0xbcfa0107,0x253d2d5c),LL(0x43a49e2d,0x9fa1e7c7),LL(0x04ad0152,0xd483d4ee),L_(0x05755d69), + LL(0xd95b8ba9,0x1d6eb61c),LL(0xeee8e590,0xc7ef0d05),LL(0x47b3133d,0xf255931e),L_(0x31eb4e09), LL(0x60b98075,0x2344f6b6),LL(0xe87936e7,0x789f9ba9),LL(0x00302e85,0xccdbabda),L_(0xa2a5f4eb), +}, +/* digit=9 base_pwr=2^63 */ +{ + LL(0x228b3a55,0xd1e183b4),LL(0x3a405373,0x4948ad88),LL(0x18288fb9,0xf45f24f0),L_(0x04bd162a), LL(0x37574ace,0xe86d0db8),LL(0x7b9a4c64,0x1ccef3f8),LL(0x451b09b7,0xee7d7495),L_(0xdc28df83), + LL(0xd52a1c1f,0xd08a8ae3),LL(0x7151a72f,0x750d587a),LL(0x03e86cff,0x27e73149),L_(0x59e2acdb), LL(0x527a9878,0x96f4a96b),LL(0x84533bc0,0xe4a1f248),LL(0x038153e5,0xd9998bb3),L_(0xea910931), + LL(0x281ef607,0xee8331e0),LL(0x9f73b6f4,0x71264952),LL(0xa04c6f5a,0xecfe722b),L_(0xd447f9c0), LL(0xbb4ca37b,0x8ff53f4b),LL(0xf450eeb8,0x42ab819f),LL(0xb862be64,0x8ba40994),L_(0x7a08433e), + LL(0x3c280033,0xa9734b49),LL(0xa8e0d0eb,0xb12f26d4),LL(0x3686d2f0,0xe0111b72),L_(0x1e11df0d), LL(0x7a48dabb,0x069099d5),LL(0xefbc8fff,0xaba55aec),LL(0x6cdc05f3,0x4ea7440f),L_(0x7409d1a4), + LL(0x4a159950,0xe99a66d4),LL(0x35d9b8b9,0x4d054e48),LL(0x924ebb42,0x155f0d8c),L_(0xcc5645dd), LL(0xe319a1e4,0xe6d591f2),LL(0xdce90cf4,0xe9b13d21),LL(0x0834867a,0x39cb42c0),L_(0xb288b9e8), + LL(0xb4a0a7e4,0xd28d49c3),LL(0x034fe0f1,0xab2cbbd4),LL(0x4cb47055,0xcd15bd1f),L_(0xb122fe53), LL(0x16ef99e4,0xb349d462),LL(0xaa28568f,0x746c4b40),LL(0x4dce044a,0x56cf52da),L_(0xce7ffcff), + LL(0xb1b9c2ea,0x0196cebe),LL(0x26971900,0xdb961af6),LL(0x51023370,0x68f286d7),L_(0x840c2641), LL(0xad763cfa,0xa07bb401),LL(0x722e5f14,0xab25bfde),LL(0xac71bbe8,0x06a272b6),L_(0x7294da0b), + LL(0xd727b459,0x6aab9dfc),LL(0x67ba95ef,0x695de10f),LL(0x078f906d,0xa987a86e),L_(0xcc5262f1), LL(0xd50493d5,0x3eb5588c),LL(0x1854cdec,0xce9abc84),LL(0x101a021e,0x2ec773ad),L_(0x9a126518), + LL(0x7bdb0ab3,0xb55a240a),LL(0x520772fd,0x06998947),LL(0xf475a7f2,0x04f12f32),L_(0xb6d5b226), LL(0x688143be,0x73021050),LL(0x2e88c6c2,0x325ed38e),LL(0x8114699d,0x467cd567),L_(0x2d56b4f2), + LL(0xcbfe120c,0xe69958d1),LL(0x952433b8,0xa2e1385b),LL(0x9586b08e,0x8d8876e4),L_(0xc1845358), LL(0xe749a358,0x85028619),LL(0x0332f81b,0xf41d9b86),LL(0x8430e884,0xe0fe7436),L_(0x5fa0aa6a), + LL(0xe20a701c,0xbc42935f),LL(0xa8797955,0xaf260957),LL(0xb55c4fab,0xb5f1b7b4),L_(0xbb614800), LL(0x3e2c715b,0x61856276),LL(0xd080dc4d,0x447c4418),LL(0xfaf57b99,0xa5a859f6),L_(0x450d5963), + LL(0xa50c7da0,0xaaf263e7),LL(0x431eb854,0x27bdb99e),LL(0x28bc6b58,0xa291a83e),L_(0x18ba826c), LL(0xe155fe7b,0xa92f7c72),LL(0x42ebeb92,0xfdebd3d9),LL(0x54ea2480,0x1d9b3247),L_(0x01266a3f), + LL(0x3bb14261,0x40205f43),LL(0x83f98334,0x1e5b99d9),LL(0xf68ea4a3,0x99449ff0),L_(0x73052363), LL(0xb4680c54,0x6506393b),LL(0xe10ba403,0x9fa16280),LL(0x355805ad,0xa8869c1e),L_(0x3bd33351), + LL(0x1fe90871,0x60faf2a7),LL(0x26d3ca4b,0x74445704),LL(0x97761552,0xc6294a8b),L_(0x5d498659), LL(0x8306971f,0x79313f41),LL(0x12b7cc1a,0x459c33b3),LL(0x5ff5c231,0xa47413df),L_(0x77510e5a), + LL(0x8931a00a,0x1bb13907),LL(0xb77711c8,0x6e5d6bd8),LL(0xf62b4e1f,0x419ecc6e),L_(0xad753454), LL(0x2e05fe27,0xa9d7ebb7),LL(0x0a367c60,0x46afc58b),LL(0x2c8381f8,0x243d0786),L_(0x4a4e4910), + LL(0xa5a5886b,0xd9832e54),LL(0x5cc2672d,0xc764b06f),LL(0x38ed1085,0x7eb63b3e),L_(0x98a773f2), LL(0x585bdb21,0x8a62c89d),LL(0x4ef6df8a,0x5e301e9c),LL(0x35b91587,0x4e488867),L_(0xee5d5725), + LL(0x9471249d,0xcfed9152),LL(0xfe397a39,0xf83f6a41),LL(0x859a1f08,0x67c5e1b0),L_(0xd1f44f7d), LL(0x40557a71,0xa37c1374),LL(0x124aed5a,0xecd05a92),LL(0x7635cd74,0xccd0d7f7),L_(0x05c36045), + LL(0x02596d64,0x81ed48b5),LL(0xfe6d2634,0x8ec54e24),LL(0xfd23aeec,0xa7513cf4),L_(0xf5285343), LL(0x8f6097de,0x2a1c180b),LL(0xe4bdeb22,0xacdc4130),LL(0x253dbb93,0x9daa0c3e),L_(0x05a30794), + LL(0x5f61e12e,0x3336043a),LL(0x083aa9bd,0xa83c1fea),LL(0x77fedd08,0x64002d5d),L_(0x131fc28d), LL(0x08aaa0aa,0x210df7fe),LL(0xdba1334e,0x6b43ad20),LL(0x98ade169,0x42a1a05e),L_(0x912ea208), + LL(0x6e92d486,0x8a6e2064),LL(0xc3bdcf88,0xc92637e6),LL(0x4ee89ae2,0xa8cbe2a0),L_(0x8fe2ce18), LL(0x688111d3,0x1fafd02e),LL(0x3e63b605,0x315d9dc5),LL(0x0cd0654b,0x02890b79),L_(0x37460a53), + LL(0x131fab04,0x1ac884b5),LL(0xfe807119,0x8a1a27c6),LL(0x66f76b97,0x254a6db2),L_(0xdb8b9dad), LL(0x17450bad,0xd2c4396b),LL(0x24d3388b,0x2691986c),LL(0xd9864a47,0x31c0544b),L_(0x73b25e3f), + LL(0x2f0f74a1,0xf3ddaad3),LL(0x4dc5341d,0xa0f85073),LL(0x55736fde,0x9f28ac0a),L_(0xdb012f27), LL(0x5b08d300,0xaaec443d),LL(0xa313b8a7,0x9fd17e0e),LL(0xb4213134,0xf10e3114),L_(0xf370d32c), + LL(0xd9301e92,0xa6577039),LL(0xdfaeb17e,0x50e6f1b8),LL(0xf5e86ab5,0x58a7b434),L_(0xff883f58), LL(0x2f0371f5,0x1d3d9546),LL(0xcbb7b3e4,0x39c8d544),LL(0x12bf2df1,0x57a123a0),L_(0x1f072a24), + LL(0x817dc76a,0xfd0338c3),LL(0x991da89f,0xb99ba801),LL(0x36e30856,0x1dcffb49),L_(0xc7872aad), LL(0x665a28b5,0xf1016ea1),LL(0x290f4c2a,0x86a30740),LL(0xc3331f1b,0x0205d1d8),L_(0x7458d6c0), + LL(0x86281810,0xd7ae93c1),LL(0xe873393b,0x227716fe),LL(0x736914e5,0x605a49ab),L_(0x963a1b41), LL(0xa3fd7cc5,0x99cf069a),LL(0x4d54cee8,0x93c53857),LL(0xdc5b96cb,0x080bcc0e),L_(0x137d0d8d), + LL(0xf2335efc,0x398d73b0),LL(0xf7bd4b8b,0x00bb4ee4),LL(0x212ffe0a,0x7ab2ea4d),L_(0xa520fad3), LL(0xe2f332a3,0xeeeaddc8),LL(0x55e1d6f7,0x2962d309),LL(0x143e5fcc,0x49553c63),L_(0x65652ccd), + LL(0xc941479e,0x203c4316),LL(0x8084b17f,0x855dab4e),LL(0x717ef700,0x4ed7e83c),L_(0x663074c4), LL(0x494ccfdf,0xe90530ff),LL(0x74fb2c41,0x8222ad9a),LL(0x4430a4d7,0x252e55b2),L_(0x1ff22265), + LL(0x96506aa4,0x3f484d6d),LL(0xa163cb06,0x10e92da8),LL(0xbd490fb2,0xffd03214),L_(0xac263c3b), LL(0x95792258,0x7dfe3775),LL(0x448fd584,0x9120d23f),LL(0xe2412ec1,0x1de70984),L_(0xa0573144), + LL(0xcb479a8a,0xe7cc4f73),LL(0xb248d660,0x1382187b),LL(0x9dc7e0af,0x01c1bd80),L_(0x5c60ddc2), LL(0x7d0256e8,0xbd971088),LL(0x0fcea0be,0x7ac3d22e),LL(0x36e29a92,0xecf0d361),L_(0xacf3f8d3), + LL(0xbb9273ca,0x4c568852),LL(0x92c61bfb,0xf3f274f4),LL(0xb258fcff,0xdee7fe95),L_(0xd7eec021), LL(0xea16df72,0x3f7227a1),LL(0xef526f6f,0x8865b774),LL(0xc554b491,0x10280ec0),L_(0x0822704a), + LL(0x0aaf5933,0x600576a1),LL(0x3b11f02c,0x63228df6),LL(0x7b64399d,0xb12890cc),L_(0xcd17f990), LL(0xf166eabf,0xa3e9c521),LL(0x6f888420,0x01f649df),LL(0x33266d1a,0xb179110c),L_(0x31f1eb1d), + LL(0xaedb7e52,0x2d780714),LL(0x4aebd349,0x76c48d10),LL(0x95533ca3,0xef834a13),L_(0xf261039a), LL(0xbb0ea84d,0x8bf5f008),LL(0x2b897924,0x0c5e907a),LL(0x5462d73b,0x1d6aecec),L_(0x030fd964), + LL(0xb5bf76f0,0x2e91623e),LL(0x3010a109,0x1415663c),LL(0x15fea421,0x1f1c3707),L_(0x7f39476e), LL(0x03dd8ccb,0xb068031e),LL(0xd4fde062,0xd50a50f3),LL(0xbde1692d,0xc8df6655),L_(0x8e9ba050), + LL(0xa49675bf,0x923698c8),LL(0xf646ab4e,0x9ea67a41),LL(0x67f23bf8,0x4444d11f),L_(0x4ac9487a), LL(0x05829e1a,0x18b3a870),LL(0x354f585f,0x8b8364e8),LL(0x917428eb,0xafd1b685),L_(0x4487c137), + LL(0x5cdc38f6,0x8bdefd85),LL(0x9e5f8f08,0x60854dcf),LL(0x5650f1c2,0x3d41abd0),L_(0xc6e01f27), LL(0x338d5d08,0xd45495b6),LL(0x3b1465b1,0x10d2d56f),LL(0x05c210e8,0x2709cd44),L_(0x48f63367), + LL(0x3d513963,0xb778f7da),LL(0xbe45c33d,0x237a1a75),LL(0xb843c30f,0xfe6c9001),L_(0x2fb3bb50), LL(0xb7f1d85f,0xdfc37735),LL(0x9212109f,0x63ca258a),LL(0xbcb01029,0xaad826a1),L_(0xad2340f1), + LL(0xf2d99edd,0x3824eb60),LL(0x40c15d61,0x96cabc5a),LL(0xdb4188d8,0xe5012748),L_(0x8b0b38b2), LL(0x100a7042,0x73b6c9c4),LL(0x51b0b8d4,0x4b2e4ffc),LL(0x3c179919,0x35fd801d),L_(0x4079a1e8), + LL(0xfba7fe0d,0x71487144),LL(0x74cace4b,0xa5684724),LL(0xde29806b,0xaf1f4b15),L_(0xd8b8bd38), LL(0x0418df99,0x3da078d2),LL(0xd3aa1585,0x9504ef03),LL(0xd6fca127,0x68282ff5),L_(0xc362d893), + LL(0xa4371234,0xf24ea908),LL(0x602bf155,0xbf610641),LL(0x751d8536,0x169f256b),L_(0xd3de9a54), LL(0xa7afba2c,0x66d0d844),LL(0xc7851ba8,0x292f0813),LL(0xb9773f57,0xc95c32ed),L_(0x5941d955), + LL(0x931f72a9,0x2d2921a4),LL(0x0d1c3a96,0xaabec6ae),LL(0x93be322a,0xe0530ac5),L_(0x68a2485a), LL(0x21ed4c4b,0x3539fb21),LL(0xbd921069,0x54c5601c),LL(0xff29a4db,0x33f41e0a),L_(0xb292275c), + LL(0x92370d4a,0x2b5f5d00),LL(0xb56aebc7,0x9ce3670d),LL(0xc246a235,0xcfeee196),L_(0x6a801585), LL(0xc688f199,0xc992ad9e),LL(0x47950bdf,0x3828369e),LL(0xc2172706,0x9016c246),L_(0xa7002475), + LL(0xd8546f63,0xf288a32a),LL(0xe74ad8ab,0xd866770d),LL(0xa8e0a566,0x41c5b1a5),L_(0x0d4e95a9), LL(0xc8eca337,0x06f5713c),LL(0x1b3285da,0xaf5394d4),LL(0x1c420a16,0x05bdbb33),L_(0x57e5568c), + LL(0x8e6a5c3f,0xf167321f),LL(0x12138692,0x4806da87),LL(0x98fd2767,0x23e3b0df),L_(0xc297c470), LL(0xde00e6e5,0xa65aaac7),LL(0xb2271037,0x603b9246),LL(0x39526356,0x6098239e),L_(0x06050e2a), + LL(0xdf432ea4,0x989bae16),LL(0x1d1be94f,0xcdff8337),LL(0x1f3f8b31,0xe12f868b),L_(0xeb8efb64), LL(0xc59a6e03,0x3e2e2a6e),LL(0x10ffb8b9,0x35623aa8),LL(0x66bc9d0b,0x419d9c04),L_(0x56947a3b), + LL(0x3e403fae,0x80a49b71),LL(0x85262cb5,0xa16aaa21),LL(0xc0552283,0x941ac341),L_(0x2ed3e3d7), LL(0x78d10e08,0x9e2010ad),LL(0xd7abaaee,0xc3d626db),LL(0xad4eebe0,0x616d3812),L_(0xa0fef6e2), + LL(0xa85af3a0,0xc44a4210),LL(0xb9a6a0f3,0xd700b9c8),LL(0x00fd9c15,0xaaeaf056),L_(0x882ae9df), LL(0xcc55572c,0x58e16fee),LL(0xa9e63498,0x224b2130),LL(0x27e82ee6,0xe838331f),L_(0x34f7549c), + LL(0xf7314698,0xc2d9578c),LL(0x2c89ffc1,0x28f766e4),LL(0xe5782e17,0xdb1237a2),L_(0x38920553), LL(0xdcb19e7c,0x382be491),LL(0x067bb8d7,0xf76d352a),LL(0x0749b544,0x7968d6a8),L_(0x9449b345), + LL(0xceb29d97,0xe4a0f31c),LL(0x468714c4,0x62bd6085),LL(0xc9e0da34,0xd2a91a43),L_(0xf8dfc14b), LL(0xf153aa4b,0x13715918),LL(0xf2418048,0x5c701eb3),LL(0x3cc2d39d,0x21bc78c7),L_(0x14e1c5f8), + LL(0xd3a03cae,0x5f6a9d91),LL(0x6557c299,0x3753e068),LL(0x733ae857,0x58532343),L_(0xb7e14afc), LL(0xc858b2c6,0x423a2e5e),LL(0x1ec4f502,0x03c6d3e7),LL(0xbd4e4f93,0xb99af7a6),L_(0xb21da53e), + LL(0x74665b48,0x00f3a821),LL(0xc7e083f5,0xbb254497),LL(0xae564546,0x09235190),L_(0x0cb8dbf2), LL(0x148c2b65,0x44c9ade6),LL(0x5f11dd78,0x01120f40),LL(0x2095b0d0,0xc2cfde05),L_(0x755f44dd), + LL(0x9096ca16,0x0b95bea1),LL(0x6343ed3d,0x92a3cb12),LL(0x403109c3,0x67ede54a),L_(0x8fa7e763), LL(0xd330e123,0xe94c1522),LL(0x853e450e,0xcf570b75),LL(0x8c60e503,0x7b779c33),L_(0x372ea779), + LL(0xb7a09ebb,0xb7df5c36),LL(0x6843f859,0xda7c16c6),LL(0xea76230d,0xe12538a4),L_(0x3c692e2d), LL(0x35168d67,0x22a3c5aa),LL(0xcf2d3469,0x90a1a3b7),LL(0x488f516d,0x4a17692d),L_(0xa3eac98f), + LL(0x979163b0,0x5261367c),LL(0xa9ef4f26,0x2b64c77e),LL(0x121123d3,0x57b5327b),L_(0xbd85c013), LL(0x60be5341,0x4320f770),LL(0x376a3aff,0x80d5de37),LL(0xbfa35c85,0x804730a8),L_(0xe518a7ff), + LL(0xfec3e94d,0xe49a8416),LL(0x5dc38222,0xebea12bc),LL(0xd561b8e1,0x1b6939a7),L_(0x839a2cae), LL(0x689cf9a5,0x0a52d61b),LL(0x69ef35c2,0x2707af62),LL(0x00163171,0x888d6c2f),L_(0x7f4b170c), + LL(0xcfab7c65,0xd2cae7f9),LL(0x327075e0,0x10e60343),LL(0x1e1d00e2,0xb1b2d682),L_(0x25112a63), LL(0xbc56c4aa,0x87923cc3),LL(0x3318db1b,0x05fa3aa6),LL(0x07ea55ed,0x08a82010),L_(0x759b62e0), + LL(0x5178b724,0x10a520c7),LL(0x4dacb0ee,0xb1fffc0c),LL(0xbbad0a50,0xab6af890),L_(0xdf72a4c4), LL(0xf8407691,0x2bcfca02),LL(0xeabf10ce,0x56944f45),LL(0xaab35dc1,0xa5c174f8),L_(0x319513dc), + LL(0xd8227573,0xfd6ac99b),LL(0x15483af2,0x4081f3db),LL(0x7e998f3d,0x08e76dcc),L_(0x6a8419c8), LL(0x1c7b8f2c,0x0551cbe8),LL(0xb6e1d5cf,0x045398f6),LL(0x93712a63,0x31b716ac),L_(0xc002e0c5), + LL(0xf2590189,0x1074bc24),LL(0xb0d09d0a,0x9d9df6f5),LL(0x1a06a2f7,0xf880ea5b),L_(0x633280dd), LL(0xe509ec7b,0xc7021265),LL(0x2310dabb,0xe9189561),LL(0x09271f5a,0x910bdd23),L_(0x633a882b), + LL(0x258fd823,0x28518543),LL(0xc6491408,0x338b1787),LL(0xc4796256,0x636fbcb3),L_(0xe0ed072a), LL(0x0aff7ca1,0x9306ff92),LL(0x8d3bca69,0x6a18552b),LL(0x5ba5cce7,0x563c7d80),L_(0xb0af6818), + LL(0x0a2d6942,0x682b34d2),LL(0xab8f78d9,0x467f7fe4),LL(0xa7e4e833,0xfb7c4f69),L_(0x0e7bfaa4), LL(0xd5513496,0xc4ac863c),LL(0x9b356137,0x74229c34),LL(0xd5e5f34b,0x729a99ef),L_(0x3111277a), + LL(0xca92d3bf,0xacf9987e),LL(0x5bcd837b,0x84c9d53e),LL(0xa434a89c,0x0ac3dbe1),L_(0xcc196982), LL(0x05a4b105,0xbbe7f4a2),LL(0xdc365bca,0x2222c71f),LL(0x0091add4,0x5e600780),L_(0xc0e95213), + LL(0xe0a35b2a,0x5acac230),LL(0x8afcf7dd,0x5089506e),LL(0x57de9730,0xf05a525b),L_(0x91a6dc15), LL(0x8c76bd09,0x74d9f43b),LL(0xa8508401,0xdca2c05b),LL(0x4bcbac2f,0xdc6633b9),L_(0x369c598b), + LL(0x1bc30bdf,0xa7ddd558),LL(0x2b2504d6,0xe21dd977),LL(0x66baebc4,0xb4d573cb),L_(0x9f9fe00a), LL(0x352d2cef,0x3c8cf82d),LL(0x5dca0593,0x12474a96),LL(0x8d29a0c1,0x55049164),L_(0x861192c4), + LL(0x8806d22e,0x827b3141),LL(0x25ab2860,0xc0bcac45),LL(0x45f1bee6,0xe2c1046e),L_(0xc00e053d), LL(0x0b973abf,0xc60d4681),LL(0xa5d34050,0x05dd04f3),LL(0x1ae3e5bf,0x64bbe857),L_(0xc97287f7), +}, +/* digit=10 base_pwr=2^70 */ +{ + LL(0xa40697e4,0x6dcf122e),LL(0xe4c0167a,0x179d681e),LL(0x177fcb32,0x2e1c6f2e),L_(0x52ddbb12), LL(0xdfe5e465,0x4c70a028),LL(0xd6ef3161,0x6183ea61),LL(0x4d64db2b,0xd053bce3),L_(0x260d4eb6), + LL(0xe6f3cece,0x3c37feb1),LL(0xc31c16a4,0x857f0262),LL(0xbbc0b11e,0x091b8ddb),L_(0x5637b740), LL(0x8541cfa7,0x772c3ab4),LL(0xc2c0eb60,0x71e71bdf),LL(0xcd2d8bee,0xd29d11b6),L_(0xb9533c16), + LL(0x4b9ffc6b,0x8ee5427c),LL(0x10f45410,0x997ab138),LL(0xff6b61a7,0x3b574c18),L_(0x394db51a), LL(0xbeade6c7,0x315f47ce),LL(0x04f8a73c,0xc77bac27),LL(0x39f2d0a5,0x28b192db),L_(0x2934b689), + LL(0xdfa7909f,0x374a8799),LL(0x3c94fdda,0x121579dd),LL(0xb4c9c6a6,0x2b2a2fe1),L_(0x56fd5e90), LL(0x2dec18ba,0x467e925a),LL(0x8c9027cd,0x51b39838),LL(0x33d05082,0x6dd45eec),L_(0xbdecbc09), + LL(0xe4793da1,0xce68ae65),LL(0x1c93f463,0xdf1eb924),LL(0x67519764,0xa6d12a17),L_(0x50dd0b17), LL(0x1e13ad02,0x43f0840b),LL(0x88680227,0x648dece5),LL(0x0b8a73e1,0x9e1099c5),L_(0x6364790c), + LL(0x00e5b508,0x16e7b26d),LL(0x41d10bda,0x2fb5b17c),LL(0xf5f94483,0x964b96f7),L_(0xc11a04d1), LL(0xb577452c,0x8083550f),LL(0x8030936a,0x2f106bf6),LL(0xe11783de,0x0cfd54c0),L_(0x4ef38a16), + LL(0xd3bf6d03,0x6afe9c51),LL(0x54c55faa,0xbb0f1cd8),LL(0xb0816531,0x27df4a0c),L_(0xa678d64c), LL(0x72f5b39c,0xc24bf200),LL(0x0cc6c1f6,0x39525315),LL(0x2d46495a,0x758c33c9),L_(0xd07deb3e), + LL(0x9c7bf29a,0x2cb77bd8),LL(0xb9eef76b,0xb7245ac8),LL(0x0a33b879,0xe50ef27d),L_(0x90583d39), LL(0xb91f697a,0x1ca8036c),LL(0x3704acdb,0xbee50919),LL(0xf053ecd4,0xece7c952),L_(0xc1b42c32), + LL(0x990b1bde,0x0420911c),LL(0x39fa1b1a,0x4138b100),LL(0xafe94d6f,0x24ff9ef4),L_(0x17756d00), LL(0x528a709c,0xa50cb647),LL(0x66685df4,0xbb306764),LL(0x907cbbc6,0x74217b89),L_(0xb3971d2c), + LL(0x0bab31b7,0xcccd2b33),LL(0x73e2182d,0x5f8e6b04),LL(0xb7efd33b,0x4cd501f3),L_(0xf5c8e01b), LL(0x83d20ea3,0xf475121c),LL(0xb4c7ec19,0x47427d10),LL(0x6eb988c6,0x7acc59a0),L_(0xee8542ac), + LL(0x591ee336,0x16ee1f0b),LL(0xbd069c60,0x27985f4f),LL(0x7a51d8bb,0xfe74d9ab),L_(0x699af4f3), LL(0xaa92cd9e,0x9ed60020),LL(0xddd73047,0x2a0eac86),LL(0x18315901,0x8dbf60b9),L_(0xd894e7a4), + LL(0xaaf7ef60,0x3d3b7078),LL(0x1892a0ca,0xf09fd9b4),LL(0x00b8f278,0xf7dd85ca),L_(0x18c0622f), LL(0x54e7df9e,0xe305da30),LL(0xf030f7b0,0x67dd41aa),LL(0x88c0e226,0xc437eba8),L_(0xbebf59f2), + LL(0x938be7f7,0x7f0d8c8a),LL(0x93333f94,0x74f9489c),LL(0xba77e9c0,0x529b3458),L_(0x5132809e), LL(0xfea9f3ed,0x93d7e4fe),LL(0x2d6cdf17,0xce0cdb8b),LL(0x1c3846b0,0xe0a036d5),L_(0x41588868), + LL(0x3b25f1dd,0x640f0895),LL(0xa1b0f513,0x6a36628c),LL(0x0a1631ee,0x22c92416),L_(0x57e72f37), LL(0xb5997003,0xa3c8067e),LL(0x34446f40,0xe28f2eda),LL(0xca50379b,0x0c2d44ef),L_(0x86add37e), + LL(0x1879da9b,0xf1be18cc),LL(0xc9b308fd,0x5708ec7f),LL(0x088da37b,0xa743ca6b),L_(0x8175ae9b), LL(0xd86f2e96,0x9bf8faf1),LL(0x3935839e,0xae7389fc),LL(0x784c1096,0x1aa41e81),L_(0x5ea779cf), + LL(0x2380678f,0xe93e1c94),LL(0x38fe23d2,0x68003338),LL(0xa4954b13,0xfa7015a9),L_(0x065ec161), LL(0x3e892822,0x19b8125b),LL(0xaf0c1860,0xd66a6abf),LL(0x8579e7ef,0x7516c1ae),L_(0x9c6fa132), + LL(0x8051f522,0xfd3b4b27),LL(0xc58d2a96,0x89473859),LL(0x8c0cdac3,0xd7eaeef0),L_(0xdea9f3b9), LL(0x0639dd0d,0x862785aa),LL(0x6333d5f0,0xd61bc685),LL(0xd861b3a5,0x83d3ee5e),L_(0x3b940e97), + LL(0xb145ba08,0x2aaf9072),LL(0xc3f907fa,0x2f3077a0),LL(0xbc426169,0xc99c4c21),L_(0x6cffa60c), LL(0x6ee68684,0x236b0866),LL(0x2624e5dc,0xe6b22138),LL(0xaa9f8c57,0xc8cde662),L_(0x93a86894), + LL(0xcedbdd1b,0x51eec28a),LL(0xd594682e,0xd4deae6d),LL(0x9e368887,0xaded26ea),L_(0x12b6da79), LL(0xf79fe22a,0xa610dbd9),LL(0x29369dca,0xc8e53035),LL(0x8d4db547,0x6d7b9831),L_(0xe42e7214), + LL(0x6d88e9db,0xaf929b4d),LL(0x7078cb4c,0x09f3ff8a),LL(0x2c88cea6,0x95aa1478),L_(0x4aa22af5), LL(0x4d819417,0xe1f9b46c),LL(0xc564232b,0x3fdecbde),LL(0x368fe183,0xce1f9cb4),L_(0x091d0d56), + LL(0xb55224a6,0xc92c61ce),LL(0x12d8c2ac,0xe42fc5df),LL(0xbae2229f,0x3afc274c),L_(0xaa2d85b2), LL(0x85cd8bfd,0x97118ed0),LL(0x3f0f7abe,0x5e361376),LL(0xaaf47545,0xcad4d556),L_(0x5d8ce6e2), + LL(0x18d8ff5d,0x18b51ba2),LL(0x38eb90ea,0xaf0eee18),LL(0xfc1b2868,0xbb73ea5a),L_(0x04dbbb13), LL(0x88d94a30,0x730a7041),LL(0xecbbc48f,0xd183ab02),LL(0x9867c6b2,0xf198711f),L_(0x29700a86), + LL(0xf225ca3b,0xebadfce9),LL(0xac6e8138,0x7b3285ff),LL(0x09069b77,0x1091afcf),L_(0x7e8bc830), LL(0xdcc4bc5e,0xf1844abf),LL(0xcca811f9,0xf655f51b),LL(0x7e393926,0xa022dcd5),L_(0x7d04ce77), + LL(0x32d143d7,0x07002476),LL(0x4ba2c132,0x028e4eab),LL(0xf7ed9d04,0x65e3cb55),L_(0xec4c3928), LL(0x1cab2512,0xdf940bc6),LL(0x738a5390,0x734bf044),LL(0x2643f52d,0x11593f75),L_(0xdd379063), + LL(0x1759457c,0xcee75815),LL(0x09e08c84,0xbfd1d973),LL(0x5f64050d,0x6b3087eb),L_(0x15138f44), LL(0xc0453460,0x776dfb51),LL(0x19bde576,0x07f4f3a2),LL(0xb17003a3,0xb6a6cf4d),L_(0x25dd19eb), + LL(0x53f04643,0x8525ef52),LL(0xa60e3c60,0x9a0888e3),LL(0xde6f955e,0xda028b89),L_(0x6d89bc76), LL(0x2711d947,0xd109e67e),LL(0xb2e621ad,0x19b5f6ef),LL(0x75754b8c,0x85c8d6d5),L_(0xbd6dfaf2), + LL(0x0a0de3ce,0x725322a6),LL(0x6f41138b,0xabf6f3c3),LL(0xeb4d3502,0x6dfedcca),L_(0xcf8c4120), LL(0x08adcae8,0x33d4bd69),LL(0x1e238409,0xcfe1b692),LL(0x659e68b8,0x82c86924),L_(0xed3a9241), + LL(0xd351d4d8,0x141df0b6),LL(0x232a8ee6,0xaaee8341),LL(0x6a97a114,0xaafe4f6c),L_(0x3cb8f982), LL(0x3ce36b0a,0x4106201e),LL(0x96a879e9,0x40106385),LL(0xfda3c463,0x9e471096),L_(0x3f6ed17a), + LL(0x15f6014d,0xb5217d60),LL(0x166ccebf,0x421c3d60),LL(0xcc23ecec,0xc0f18f53),L_(0xde559d43), LL(0x1ae450bb,0xf66ae880),LL(0x7de0a653,0x2e31cf5a),LL(0x1b3a8046,0x916c70ac),L_(0xf73fb01a), + LL(0xd287d1cb,0x3e47dc5d),LL(0x70eb7ae0,0x42cffa9a),LL(0xc6ddb9b7,0xb751c3c5),L_(0xb82e85ea), LL(0x7f3bfdc4,0x8676c057),LL(0x711d3839,0x2f6a12cb),LL(0x4e43cafd,0x273385a8),L_(0x94a8904e), + LL(0x1d472cf9,0xb53ac163),LL(0xc380e03f,0xf1c57e23),LL(0x5f7dfde3,0x53f9522b),L_(0x51e85640), LL(0xf3d29714,0x3c47cbaf),LL(0xc0110dc8,0x561e0754),LL(0xbe952c4a,0x6bcd178c),L_(0xba15112f), + LL(0x24b41db7,0x36a421ac),LL(0x98891ee6,0xdc178876),LL(0x619cb9a4,0xa266089e),L_(0x9c33f043), LL(0x9fd8767e,0xba9a6d1f),LL(0x20bce515,0x8cbdf583),LL(0x46c198f4,0x23f49a00),L_(0x847b2ffd), + LL(0x281cc4ea,0xc3d2804e),LL(0xdb1f2dbb,0x22a16678),LL(0x638a7949,0x7ec2aa7b),L_(0x9a43ca9e), LL(0x187243a6,0xb99ca6d8),LL(0xd4fc6e37,0x2d4040d0),LL(0xbd604fb9,0x8cefbced),L_(0x1938ccf3), + LL(0xfbd9e1c5,0xd10937a9),LL(0x2f42feaf,0x2fa67093),LL(0xe7b103db,0xb57b1840),L_(0xe03cfeaf), LL(0x1368261e,0x719e1d06),LL(0xfa867b56,0x192c11f4),LL(0xa6cd197f,0x9b41db84),L_(0xc6acf157), + LL(0xd68a5a4a,0xddd06586),LL(0xe7a17b19,0x52664a67),LL(0x2621ebc3,0x6f5736f5),L_(0xc8afa211), LL(0xd267f259,0xbacb7a85),LL(0x15b16a2a,0x5c0b0c67),LL(0xaa7c18e3,0x166ad375),L_(0xd0350e8d), + LL(0x9a21bdff,0x8411fa8b),LL(0x6d5b0f02,0xde70b24c),LL(0xf5bd9bc8,0xd7443ed1),L_(0x996aa702), LL(0x2b84584f,0x3ea3f8f2),LL(0x373c9339,0x099fde12),LL(0xbc0565bd,0x8fd32c81),L_(0xf5bc6acd), + LL(0x7e78ce17,0x12d214f6),LL(0x2fbd45bf,0x5f204c64),LL(0xaf966aac,0x62d03da4),L_(0x3d6e4888), LL(0x0fc086e6,0x14494699),LL(0x61bbb841,0xcc71f940),LL(0xec60a687,0x65805046),L_(0x1641dea6), + LL(0x1ee6996c,0x8eb81a1a),LL(0xa5c29c33,0x547ea700),LL(0x71308d2f,0x9126c06e),L_(0xb4936b4c), LL(0x5fecdc41,0x49cacc43),LL(0x7029bedd,0x89bc7a7f),LL(0x6331b109,0x93db1b8f),L_(0x60235335), + LL(0x18292506,0x93a8f9cc),LL(0x0c3b64eb,0x8d658c23),LL(0x33eef875,0xc22c9665),L_(0xd49b8f05), LL(0x2081f85e,0xbea17435),LL(0xc230d670,0x6f44daf4),LL(0x5ff2900d,0xdea368bd),L_(0x4f64c46a), + LL(0xfcb125d6,0x097e443e),LL(0x9f812e98,0x96ff5377),LL(0x0cce254b,0x562b2166),L_(0xddbade23), LL(0x73e36237,0x992060e2),LL(0xbca2c3a8,0xa968d30e),LL(0x139e42f9,0xe5b89488),L_(0x3ba1f60d), + LL(0x3e86ab5c,0xb177e3f9),LL(0x43f6161d,0x95050abc),LL(0x580bb577,0x75270515),L_(0x4a847f51), LL(0x0ce33d23,0xcab8647e),LL(0x9f53c335,0x139f525e),LL(0x9b1f28f4,0xa2e38250),L_(0x5690d259), + LL(0x193d7b51,0xbe57f373),LL(0x22feedbc,0x192cee98),LL(0x66699aba,0x69e0895a),L_(0x0f685742), LL(0xe787c996,0xb3f66818),LL(0xccac8670,0x5ee5c7cc),LL(0xe1645254,0xb7d10719),L_(0xae1e2e81), + LL(0x17022e02,0x2f237075),LL(0x36a4e5e4,0x01453fba),LL(0xba3615f3,0xa703f343),L_(0xd6276794), LL(0xb5aab8d0,0x30b0a7e8),LL(0x3f7b2b8e,0x33588983),LL(0x8a8547cc,0x64a4028c),L_(0x0ac79b94), + LL(0xbbdec908,0x6c0cb953),LL(0x9af46a78,0xf8e40103),LL(0xca5b89e3,0x5e56c8f6),L_(0x7d134424), LL(0x040b1ce5,0x76bd7772),LL(0xff8f0795,0x396f023d),LL(0x6bb455f6,0x4243e710),L_(0xbe0cd278), + LL(0xff00cfcf,0x7a3a34e7),LL(0x8154829a,0x59745afe),LL(0x13dec9bb,0x00460ef4),L_(0x7d4cb470), LL(0x664b5768,0x467ad23b),LL(0xa9eb5d4a,0xd6ac872c),LL(0xc30f32ed,0x7527a687),L_(0x3f334a32), + LL(0xf2877172,0xa577b4e7),LL(0x50267b62,0x7ac6ab0a),LL(0xbbb22aab,0xce8c4374),L_(0xb58c0879), LL(0x5d2444bc,0x1362bb86),LL(0x6236944e,0xa58498c8),LL(0x03a3218e,0x4585f77c),L_(0xd30bdc81), + LL(0xe1c0eb9a,0xf69271e6),LL(0xf6273f18,0x239476a1),LL(0x3701952a,0xb94d2063),L_(0xe82c2b26), LL(0x39ed615b,0x44f390c3),LL(0x90593547,0xa4418ea5),LL(0x1ce9df6c,0x76cd5c55),L_(0xda44a03e), + LL(0xf5dcc901,0x62fd7874),LL(0xcf83396e,0x74f61b6b),LL(0x9d3a5873,0x10ca6b61),L_(0x5ef1b62a), LL(0x848244dd,0x81922f08),LL(0x3ee1023e,0xbc236621),LL(0xb2a50ed6,0x0cbc779a),L_(0xb8fa6a49), + LL(0x4914c41a,0xb8a19540),LL(0x430e8c3d,0x957adbf0),LL(0xd2aac8dc,0x64d6ca82),L_(0xd8b3157d), LL(0xdb015ecb,0x42372196),LL(0x9a5dfacf,0x05451b08),LL(0xe16c7207,0x463bc66d),L_(0x69536281), + LL(0xfc7e9660,0xb8358066),LL(0x918a74d2,0xa1915c8b),LL(0x8d26603b,0xfdddd6dc),L_(0xfe3b2017), LL(0x2166a1d3,0x79912871),LL(0x71bd6832,0xe96aa265),LL(0x98a560dc,0x43b8858b),L_(0x3b4e1e7b), + LL(0x2d3f3786,0x95813382),LL(0x6e6cb17c,0x89e2322f),LL(0x3568d961,0xfbe11c03),L_(0x5e1857de), LL(0xb0011b62,0xb6d17af8),LL(0x76372b21,0xfa35ee62),LL(0x88f30207,0x496e0cfb),L_(0x1eab8b49), + LL(0x233a4ea3,0xed225296),LL(0xfbf21df5,0x18a0b19e),LL(0x49f70f9b,0xfd4a3974),L_(0xff579fa6), LL(0x6935d1e1,0x4ca49b02),LL(0x5398c8d2,0x465d1798),LL(0x83dab70c,0x3925587b),L_(0xc2e61a09), + LL(0xadc74708,0x97b00e28),LL(0x822c5927,0x4ea78214),LL(0x29de362e,0xe0075db8),L_(0xf94b10e6), LL(0xcd9ea163,0x4071fbdd),LL(0xa89e9ce3,0x5861958d),LL(0xb0f23643,0xfb6d160c),L_(0x2e57bc83), + LL(0x1cf01491,0xfd9abd17),LL(0x59f1c903,0xfdabdb45),LL(0xfe106ce3,0x28aec19b),L_(0x1e243bfb), LL(0x5dea7b0f,0x9dd7eace),LL(0x0d4e5c7f,0x7aa326d1),LL(0x58144fba,0x0c0b55eb),L_(0xf89a2bf1), + LL(0xf492b1ff,0x80211303),LL(0x4c3d5f56,0xa0a42f80),LL(0x812c4f34,0x0513dab5),L_(0xb494414d), LL(0x2b196b3e,0x8292825d),LL(0x845aa8d7,0xfccdef2e),LL(0x8309d41f,0xa6c098eb),L_(0x8feed613), + LL(0xe5e506e6,0x34984e59),LL(0xa9dc7b02,0xc78ed700),LL(0x023cf8a4,0x8818f370),L_(0xb1fe8a66), LL(0xedb22c01,0x61bbfcfe),LL(0xcc714f92,0x4b1c28be),LL(0x87de5c20,0x13f66db8),L_(0x523fa798), + LL(0x311b814d,0x18478ee8),LL(0xc54ff99c,0x5fc08d54),LL(0x3d6c85b2,0x2a4234ef),L_(0xd3a39852), LL(0xef0ad115,0xfa0e9b48),LL(0xdfcd9d52,0xded54ff0),LL(0x8d2d8068,0xe6033e94),L_(0x39c4cac0), + LL(0x567ac24b,0x31e079f2),LL(0x595877b2,0x876a06e1),LL(0xd77d6297,0xabca81b2),L_(0x0324c3ba), LL(0x27382ebe,0xbd7f7ff0),LL(0x5a3b92f8,0xb11d9de0),LL(0x31e36d28,0xba48fe41),L_(0x3263cd91), + LL(0x845700c2,0x659750c3),LL(0x265533c8,0xbfa2263b),LL(0x46779254,0x426c9332),L_(0x73f00210), LL(0xc0627927,0x919159e1),LL(0xc1896bdf,0xb8633c7d),LL(0x62049111,0x57a02e38),L_(0x159b1b0e), + LL(0xc0ef9c33,0x87339367),LL(0x80f32c7a,0xd690f5fd),LL(0x4cb8372b,0x74004a4c),L_(0x92a5a93f), LL(0x62fc7f7f,0xc75eca95),LL(0x2883c5ec,0x87991514),LL(0x1a3939cc,0x9eeab34e),L_(0x57489f74), + LL(0x6dda4270,0xe95ee7c3),LL(0x68b80ca0,0x5637d166),LL(0x0cc0434e,0x108326a5),L_(0x11ccac8e), LL(0x66908f8f,0x768a4dc5),LL(0xe0516b2b,0x24086620),LL(0x0390f0a6,0x34cd95fd),L_(0xb3965e3d), + LL(0xa579c74b,0xf4d1cd8b),LL(0x2437af0c,0xa682fc6c),LL(0xcf5b3a95,0x13389ce6),L_(0x0026ae74), LL(0xc24a47b5,0xf37dbb35),LL(0xf06e1e15,0x9d47aaf6),LL(0x196773ed,0xab42affb),L_(0x76bb4374), + LL(0x78457341,0x0583dc84),LL(0x564d9444,0x6a0c10e4),LL(0xbf46a7bb,0xe6adebac),L_(0x93e051d2), LL(0x3e924b68,0xbf5e7329),LL(0x4afb1643,0x9cb33af1),LL(0x5460942b,0x1005e77d),L_(0x483528ac), + LL(0xa399de32,0x867305ce),LL(0x19e34f30,0x2200477c),LL(0xd31a5a81,0x9ac2ed5d),L_(0xa22c4024), LL(0xa5a011ec,0x9cb94041),LL(0xadcbd258,0xf7a7e8fd),LL(0x26131e4c,0xbcae3bd4),L_(0xa63111af), +}, +/* digit=11 base_pwr=2^77 */ +{ + LL(0xe00bdded,0x44e43c61),LL(0x2a3a0320,0xc584d310),LL(0xb6fe7f05,0x951b39e7),L_(0x1c47e70b), LL(0x1eae5fc6,0x518f5441),LL(0x5cd44db2,0xc505aeda),LL(0x216fb5f2,0xbb781e43),L_(0x7e14b38e), + LL(0xf9a62c2b,0xe0d93869),LL(0x60f3a9a1,0x6d3f0de7),LL(0xb3106bbb,0xb0f77641),L_(0xc3071383), LL(0x9b298ebc,0x184aed09),LL(0x0bacf017,0x334394d4),LL(0x79c1d22f,0xd95473d2),L_(0x93838083), + LL(0xcb7cb58a,0xf3fc070e),LL(0xed4652fb,0x60dba7d5),LL(0xe19d60f2,0xe420c869),L_(0x51893296), LL(0x097e4dab,0x707e030c),LL(0xd90a9475,0x0dcda4fc),LL(0x9803c9fe,0x2bbc44ce),L_(0xa6483469), + LL(0xfabcb24a,0x10355326),LL(0xdcb5236b,0x91ed19ce),LL(0x4f316ca1,0xd20efc4e),L_(0xd00bff40), LL(0x1a2e490d,0x2b83ac01),LL(0x96a6cfc6,0x0fd81acf),LL(0x8da694f8,0xfcad677f),L_(0x6bf5e71f), + LL(0x248ec9b4,0x11ef88d2),LL(0xe791a3bc,0xda678712),LL(0x370ca0ea,0x7a293625),L_(0x4e4cd7d7), LL(0xddfa0a4e,0x1f856294),LL(0x180b43b4,0x6bd2e637),LL(0xf728ccb4,0x91baf444),L_(0xe5fa917a), + LL(0x9e4adc26,0x8981411a),LL(0x6d65f85a,0x7eb4dd2d),LL(0x58e82e86,0x97c312ce),L_(0x8bfebe6a), LL(0xd7b84e09,0xc95c8ff2),LL(0x639198d0,0xd7ac3654),LL(0x7546fc40,0x504a9f59),L_(0xeac2deb7), + LL(0x9edb0426,0x476b8958),LL(0xa2e1a729,0xf14565b5),LL(0x4bab1b96,0x3d3ca262),L_(0xc0e1a0ef), LL(0xfd4e9a48,0x0043c8ff),LL(0xba0665c7,0xafaf8910),LL(0x0b1e0a8d,0xcbeb2d8e),L_(0xe710e7f4), + LL(0xebbefd51,0x44fc634e),LL(0x2a776b18,0x9ddeace8),LL(0x08a9107e,0x909eff4c),L_(0xb7934059), LL(0xf816fa9f,0xefd70514),LL(0xfaad7709,0x598db9de),LL(0x0094b64c,0xc6634c12),L_(0x8b95add1), + LL(0xd6fb5d4c,0xadb1a9c2),LL(0xd2967310,0xc8ec0f39),LL(0x1908657d,0xa25710b3),L_(0x19a27dfc), LL(0x311947c4,0xe187cb1c),LL(0x8cbb3e59,0x1bae3f86),LL(0xafb6e7b5,0x44ea2977),L_(0x781e26f2), + LL(0xee18a383,0x60b120f0),LL(0x24a67462,0x91636563),LL(0x29fd66f4,0x6538f012),L_(0x1b5c47df), LL(0xda52b57b,0x9a0bd35b),LL(0xd095d4fe,0xcee45c94),LL(0xafb1fcae,0x042232d0),L_(0xcff50cb9), + LL(0x54397158,0xd06ed8f6),LL(0xd53d3bfe,0x67644c3e),LL(0xaa8e6af8,0xa26425e0),L_(0x1f9101fd), LL(0x6fb8009e,0xdfa5a71f),LL(0x7b2b6229,0xc4fb8d84),LL(0x413c2883,0xa71e26d8),L_(0x442bf1c5), + LL(0xae3de434,0x45610301),LL(0x828b8be8,0x4ec1365e),LL(0xb95b637d,0x41dbce62),L_(0x1ca7419f), LL(0x9637a66f,0xc6d80952),LL(0x5a685913,0x9289159f),LL(0x761cc88a,0x1da0d501),L_(0xf7bfb795), + LL(0x37a59563,0xb4ca54d1),LL(0xe2ccd3ff,0x5128864b),LL(0x8aad87e4,0x559dacdc),L_(0xdf79df2f), LL(0x89718ac8,0xea353321),LL(0xe6676310,0x2012406a),LL(0xc599f213,0x1eb2fe94),L_(0x626c9f76), + LL(0xba62fad3,0xa7fededf),LL(0x6ff23700,0xd58195f8),LL(0xd3d6b5bf,0x1ad1505f),L_(0x97f82bb0), LL(0x6f93c77b,0xaf96dc4e),LL(0x30b1fb3e,0x20624dad),LL(0x4fa52512,0x3481aabe),L_(0x78a7c4e4), + LL(0xba1c1391,0x9fb8e966),LL(0x5ff12f04,0xb9c0dea0),LL(0xd59d1993,0xe8a99c53),L_(0x6cef1107), LL(0xb44f7478,0x97335f28),LL(0x400e3354,0x6f550c81),LL(0x3749bb24,0x67501a21),L_(0xdbc433a0), + LL(0x2ddbdad7,0x5e95c8ee),LL(0xf4a3dab8,0x74fa6bf7),LL(0xf68aac5d,0xe404df62),L_(0xca08be37), LL(0xf3252764,0x934f4963),LL(0x6033be81,0x65ea023b),LL(0x39f3ad81,0xf2ab45c6),L_(0xec27db5a), + LL(0x6c9e60de,0xd82ebdb2),LL(0x3c94302c,0xf4278e99),LL(0xf76b6d5d,0x177bfa74),L_(0xaec52e0d), LL(0x0698c8f8,0x8af11d92),LL(0xdc8707e7,0xdcfeb87c),LL(0xdf966b87,0x790b8b3b),L_(0x25a49666), + LL(0x0d632c2e,0x82b42706),LL(0xca5f0400,0xeb7e23be),LL(0x67d13334,0x6bef8f5e),L_(0x19a89324), LL(0x5b9d7908,0xb943b3e3),LL(0xe69dc86d,0xe1142402),LL(0xe61e180e,0x3186f5ab),L_(0x5490857c), + LL(0xee73f3c9,0x1c021526),LL(0x229aab7e,0xd5f0330f),LL(0xbcd8ae19,0x34f15f5c),L_(0xab3c7439), LL(0x0f2d33f9,0xf2452d93),LL(0x23f2f3b0,0x0c4b0f70),LL(0x322c677d,0x311431b5),L_(0xc0e186a5), + LL(0xd222f30b,0x32c8e92e),LL(0xb855ce32,0xf9051d2c),LL(0x34079f7e,0xca51b73e),L_(0x1cd08eae), LL(0xe4529fd6,0x3ee05e30),LL(0x48ccde21,0x762211ba),LL(0x957b54a8,0x7bd842d1),L_(0x8ad2f209), + LL(0x8578a05b,0xf931c6ff),LL(0x22b12a51,0x9c1a24ae),LL(0xc7ad186b,0x3aae7e80),L_(0x35a3a465), LL(0xcd94ffe9,0x02d60546),LL(0x02a0030e,0x6d8dfc51),LL(0xf70532cd,0xa7523214),L_(0xbfa52fcd), + LL(0x3898ac4a,0x7ef15aa2),LL(0x04a67481,0xc92d3e61),LL(0x6f146470,0x20a65e93),L_(0x091a434a), LL(0x40c2ff2f,0x5f5f4b1f),LL(0x356b539d,0xef24d37f),LL(0xf5eb8c60,0xd0f3625a),L_(0x5662e1cc), + LL(0x04af9658,0x0ddb7f8a),LL(0xd47a8579,0xf5679a0d),LL(0xfa164f3a,0x75f9e25f),L_(0x88f00eb9), LL(0xd2b96f54,0xc75b8bfb),LL(0xc45f8fe0,0x141dae1d),LL(0xf86c989c,0x1a015ae9),L_(0x4cbe108d), + LL(0x7440165c,0x252d3538),LL(0x94ed0d57,0xc7935e6e),LL(0xfc485299,0xafc2b1d4),L_(0x2763b62a), LL(0x8b5606a9,0x8714fa4e),LL(0x44261681,0xd015855f),LL(0x03679b60,0x14d9de29),L_(0x84a487b9), + LL(0x7fd6372b,0x1fc76dfc),LL(0x30d60bc9,0x835ac98b),LL(0xec54c6b2,0x98cbc171),L_(0xa2ec0376), LL(0x7ab5c98b,0x577a8e9f),LL(0x527bb2a1,0x84028fa5),LL(0xed0d2362,0xb28e8d67),L_(0x1a1d5c1c), + LL(0xd8c85ac3,0x8e20b175),LL(0xe895347f,0xbc82df9e),LL(0x763f7da0,0xc32cc81d),L_(0xd889ef15), LL(0xb02c89ce,0xd305dbab),LL(0xb02c2e9f,0x7f0a99ef),LL(0xd40823f0,0xf958bb39),L_(0x7a07c7e4), + LL(0xee84ea34,0xc5f043e5),LL(0xea668125,0x13f7ae2f),LL(0x18438628,0xb6ffaed5),L_(0xf72f62c4), LL(0x9cb2856e,0x9a5c926d),LL(0xade5a9b1,0xad05632a),LL(0x4d11a1b7,0xf0f2c69d),L_(0xbfed0dd6), + LL(0x60b10c55,0x79943f17),LL(0x930f1ce1,0x15056514),LL(0x70dc2681,0xc9a17b25),L_(0x71659c78), LL(0x4d9c29ff,0x5978aada),LL(0x56151f6b,0x6e84fa9d),LL(0xb01cfa96,0xd9553cbf),L_(0x495ffa1c), + LL(0x14645de0,0x50569b84),LL(0x516171cb,0x76814e1a),LL(0x6931ce3c,0xd1694dea),L_(0x18d486a7), LL(0x8fb4ec93,0xcbd84ece),LL(0x694d8a81,0x70c2d8a8),LL(0x6f1a63f5,0x2ca687fd),L_(0x6788bb91), + LL(0x66719259,0x78b1b1f1),LL(0xa8fe9c13,0x372aba7a),LL(0xf72aecd5,0x049424c7),L_(0x8c2a627b), LL(0x12996520,0x7a3d9b21),LL(0xd87c3d81,0xeea3b43c),LL(0x6296bed4,0x965623a1),L_(0xf57dda17), + LL(0x19faf24f,0xee2a75e2),LL(0xe4882684,0x2608d8d7),LL(0x5dbd9aa4,0x8dd2d8bb),L_(0x1861b78b), LL(0x96024e0a,0x59608369),LL(0x2a96c634,0x3109b985),LL(0x4c72f34b,0x073affe7),L_(0x5e2a53e1), + LL(0x7163417d,0xe75a8d8d),LL(0x003ea53c,0x34a71e84),LL(0xf46ea0c1,0xa472172f),L_(0x582a6663), LL(0x5941f31f,0x45ef8849),LL(0xda61e5b5,0x813628f8),LL(0xff6e6be2,0x2f6a6923),L_(0x24a1906a), + LL(0xf2dab7c1,0xee4fac0c),LL(0xcca97a61,0x442b5426),LL(0xd1572489,0x1b63f7f9),L_(0xc808b93c), LL(0x86beafa1,0x931a009d),LL(0x00b9ab37,0xc54e5dcf),LL(0xce454abb,0xe3d31b67),L_(0x8846c2ca), + LL(0x2bfb9673,0x00a46605),LL(0x6d38eb82,0xd0e900f3),LL(0xac391030,0x56b396f4),L_(0xa240554c), LL(0x111e6519,0x0e375257),LL(0x2daa1839,0x0569eb0d),LL(0x6386031f,0xa43939dc),L_(0x29a2482a), + LL(0x79006500,0x19bae3a4),LL(0xeba63f38,0x278b676b),LL(0x2bfc1210,0x5fd221ef),L_(0x18605ab3), LL(0xaddaabbd,0x3e58e744),LL(0x130a0b49,0x86211a5e),LL(0x27a20439,0x06b96609),L_(0x3be9694d), + LL(0x31e7bef9,0x6b521dbf),LL(0xe38156c6,0xbb74a0f2),LL(0x41f4bd94,0x79699c57),L_(0x565687a7), LL(0x6af2b091,0xb9a903de),LL(0x15159b8e,0x250d78d4),LL(0x565e3c29,0x674edb33),L_(0x0294da46), + LL(0xfa502a7a,0x4f9ab886),LL(0xf7c92c2b,0xb5e6f603),LL(0xcd18776f,0xf25cc6e4),L_(0xa7fa7c45), LL(0x9822aa1b,0x2c1f0788),LL(0x250c2373,0xb4be8a38),LL(0x671f0157,0x5b11123b),L_(0x157f5d60), + LL(0x54914217,0x1256ef4e),LL(0x5163b441,0x075b53ab),LL(0x5ecb1889,0x56e20726),L_(0x6494b2b8), LL(0x09f26255,0xd571b91b),LL(0x9475008d,0x328d62f5),LL(0xde6fb9c6,0xe065c610),L_(0x96baaccd), + LL(0x45144de8,0xdc428059),LL(0x71bed82d,0x9aff2cca),LL(0x9af32379,0x7429380e),L_(0x02f22f65), LL(0xe3565604,0xdea72af4),LL(0x2b9bc7b9,0x2bf6a43f),LL(0x5a2add5c,0x6b6ae6f7),L_(0xb69ca1cf), + LL(0x9ab52f77,0x061d7ef8),LL(0x56d68dd3,0x010840bd),LL(0xae0d756d,0x97508e7b),L_(0xecbb738b), LL(0xadc78972,0x4d5b80a4),LL(0x4dce8c80,0xf9aaef5c),LL(0x256809bb,0xa1ec2e50),L_(0x2ea2af7c), + LL(0x74a6dd4a,0x0393f719),LL(0x93285afe,0x4e57160b),LL(0xac7e00ee,0xe124fa0e),L_(0x044bd9af), LL(0x2389bd88,0x80b4129f),LL(0x758d4e27,0x0e8aaba3),LL(0x1051debe,0x4a190f97),L_(0x85906a99), + LL(0x2156959c,0x37e2ab37),LL(0xd889d270,0xfe9b7157),LL(0x540652e0,0xd3215a27),L_(0xb88094e7), LL(0xc0c597f0,0xceb6dd56),LL(0x5dc4dd34,0x09436cb3),LL(0x07e11a08,0xc752116d),L_(0x86890fd2), + LL(0xe154e3b6,0x3dd689de),LL(0xba6c5590,0xecec9673),LL(0xf4ebc67c,0xdf67ffb0),L_(0x094d0483), LL(0x33ce85c7,0x58ffaf1b),LL(0x04f91a5f,0x741f05be),LL(0x2535c38a,0xffe3294f),L_(0xfe3e3780), + LL(0x4c921d99,0x1c787299),LL(0x8c60577e,0xbf10f0e5),LL(0x4f5e9e39,0xdd4daa07),L_(0xcf714fd4), LL(0x92fcbe08,0xf29c8618),LL(0x38ba462f,0x84305704),LL(0x8d7c9a43,0x19694aa8),L_(0xe7510556), + LL(0x680c4325,0xb1ac0982),LL(0x45857f5e,0xa4032ac9),LL(0xdf576261,0x92007d10),L_(0xc5ae1832), LL(0x755cfb3c,0xb9ea853a),LL(0xc129c1e9,0x28725f38),LL(0xa32de41e,0x8310cc56),L_(0xaf03c7c1), + LL(0x4319f045,0x6a31f14c),LL(0xbb313e19,0xf0a68d54),LL(0xa79121b3,0xf8ad5c75),L_(0x1cd6c518), LL(0x1aca6f0e,0xd728ef75),LL(0xf52ecb39,0x66ed37be),LL(0x244f6df5,0x5253f731),L_(0xc37a0916), + LL(0x7e5a8bd3,0x9d03a5af),LL(0x6b5c7a81,0xa4eb71aa),LL(0xa305f15d,0x1dc7fb76),L_(0xbf1adac7), LL(0x20487a26,0x6deaadda),LL(0xf4c07457,0xe0f4999c),LL(0x9bb84678,0xc451bb12),L_(0x975839c9), + LL(0xc977cb31,0xd39b851a),LL(0x6ac13ca0,0xcd656ca8),LL(0xa19898ab,0xafd81332),L_(0xd849f9e4), LL(0x2bb5e6b1,0xc3157a16),LL(0x5b02f0fe,0x8acd8d9d),LL(0x69cbb158,0x6d76f690),L_(0x05ce3883), + LL(0xc8ff4786,0x2bca8c99),LL(0x186098dc,0xa2caaf78),LL(0xf6be10a7,0x2100372c),L_(0x16272c0e), LL(0x761e39c7,0xdb53800d),LL(0x6551227e,0xc599f6e5),LL(0xc995d474,0xcb2cc9b4),L_(0x73233966), + LL(0x39687ef7,0xe8d0ac03),LL(0xc75c74c9,0x921b3cbb),LL(0xc64c35ba,0xa500e776),L_(0xca48727d), LL(0xe79c45b0,0x60ad3863),LL(0x2b1933ee,0x0ce33aaf),LL(0xaee36834,0x4ce8962e),L_(0xb0c0146f), + LL(0xb38c8507,0xc359f6e1),LL(0x8697a59a,0x31837b1c),LL(0x98af1b3a,0xf716fe6e),L_(0x7fdc52e3), LL(0x3830cd5c,0x52930fa5),LL(0x90fd85df,0x0ad62260),LL(0x097dfa97,0xebeb447d),L_(0xcfdb2bce), + LL(0xe409d4ed,0xe80a1b86),LL(0x4d1e34ae,0x1aaed1d3),LL(0x9eac7e33,0x80bb831f),L_(0x0bb854f1), LL(0x986f7e0c,0x005a4dc6),LL(0x6e3f9cf2,0x52827fef),LL(0x293c91f5,0xcdd9e93e),L_(0xd5679946), + LL(0x2c6558f0,0xd0ac15f2),LL(0x4e88fc97,0xa247b120),LL(0x573e9688,0xd0627f7e),L_(0x96e1d752), LL(0x22887f9f,0xd31e0ba7),LL(0xb47ea27c,0x1eea7f54),LL(0x4423561d,0xf9581051),L_(0xc8a211b5), + LL(0x42a54364,0xeb2affe7),LL(0x35c0456c,0x972ce84c),LL(0xaeca0047,0x4d9efd23),L_(0x2bdbc15a), LL(0x173fea09,0x1d890ba5),LL(0x99e5f4ba,0xfea0ad91),LL(0x5f0b0f42,0x5619484b),L_(0xb1f3e2bf), + LL(0x6cdbcee3,0xc11ff4f3),LL(0xe36e32c6,0xa260adc3),LL(0x5bb4d1a2,0x33e0dbb6),L_(0x1b1ed42c), LL(0xcaa2ba15,0x415b8922),LL(0x9bbb4478,0xb1fbde1c),LL(0xe7f63f33,0xb014bfff),L_(0xd8fe72a1), + LL(0xab510c98,0xee7b323c),LL(0x86f8b471,0x37854732),LL(0x23d6cc1a,0xdbbcc348),L_(0x14fb98ce), LL(0x64d5f136,0x6fb100cc),LL(0xa833932c,0xb27f28f5),LL(0x980b1419,0xca8834c2),L_(0x088ce8d5), + LL(0x1b1c75cb,0x8c28e4a6),LL(0x712165a2,0x2cbdb243),LL(0x1341e16e,0x5a726ab8),L_(0xfe09486b), LL(0x32264866,0x4f4a819e),LL(0xf2a151c4,0xba655f21),LL(0x37367c7d,0xd157593e),L_(0x2b7d23ec), + LL(0x46d94f55,0xc9397bab),LL(0xe0636264,0x65e56a08),LL(0xf946f4e0,0xa259e975),L_(0x55c54603), LL(0xb5060784,0xccc37d6a),LL(0x2614d3d3,0xf709343a),LL(0xfb056c2e,0x89705d36),L_(0xc8dec62e), + LL(0xeb841e0f,0x7663ad0e),LL(0xc791db86,0xb1420c30),LL(0x5792fb1a,0xaebf5ced),L_(0x462f4f9f), LL(0xeaecfdd6,0xd5142858),LL(0xc555cb9f,0xe56ab53a),LL(0xdbaff47d,0xb56ba899),L_(0x048149ff), + LL(0xd69321d0,0xa06234ff),LL(0x4da9e4a7,0x98b8cc13),LL(0x4be3eee6,0xadbfb76c),L_(0x9b536e85), LL(0x57f0cdf4,0x01e45447),LL(0x54dd6853,0xb307918b),LL(0xa0a5d2a4,0xf7e16814),L_(0x4f89fd0d), + LL(0x6c92c96c,0x865c8f6c),LL(0xee136cd4,0xf96358e5),LL(0xc0b20165,0x10dbd508),L_(0x785841fa), LL(0x990a46d5,0x1d1a7d70),LL(0xc190e469,0x6c2d572d),LL(0xa87b8f93,0x03a7ba98),L_(0x3e71c1ad), + LL(0x6d9f3e8d,0x2221d899),LL(0xc7dd63bb,0x7c844ad3),LL(0x142035bd,0x11c04771),L_(0x726cc50e), LL(0xf02e1ad2,0x78ab7d85),LL(0xc57c4d2f,0x301c5ced),LL(0x1feac0e2,0x95568955),L_(0xd216f6fc), + LL(0xc5b0ee89,0xf045ebf3),LL(0x6770bb55,0xc1ea77c3),LL(0x4e10f859,0xa44993af),L_(0xb75a3354), LL(0x38d373df,0x5c460d85),LL(0x940bd01e,0x2bdad16c),LL(0x264105cb,0xb8fc3b3e),L_(0x09401ace), + LL(0x2edb771d,0xd4ae16cd),LL(0x2b675fa3,0xc8055a0a),LL(0xdee3f3e2,0xc08d1ca2),L_(0x28ea187c), LL(0x4252a1b9,0x6d687314),LL(0xf6c34d15,0xba935b80),LL(0x3b383652,0x52c1b85d),L_(0xc5e3b405), +}, +/* digit=12 base_pwr=2^84 */ +{ + LL(0xb92b8af0,0x6354d2c9),LL(0x4c952ade,0x5a4d9e35),LL(0x0381968a,0x52d9f13a),L_(0x86c3819e), LL(0xc63b9fe7,0x3743142f),LL(0xa95b1a9a,0x6e186e52),LL(0xaea91efc,0x3de25aa0),L_(0xc91296ab), + LL(0xf8921758,0xced4216d),LL(0x02fb0911,0x28cf0cfe),LL(0xd524c877,0x03bdd0b5),L_(0xb3e00db0), LL(0xd91831de,0xd131c59b),LL(0x77e04707,0x47533185),LL(0xf492c767,0x63c30af1),L_(0x202f0676), + LL(0x79a5ddcc,0x495cdcc7),LL(0x33ab4ae3,0x7f199f52),LL(0x9d2ff41e,0xbe613bf2),L_(0xf3371b23), LL(0x967f0754,0x76c36728),LL(0xfb8be7aa,0x02af3c44),LL(0xee5a166b,0x0bda3025),L_(0xc9427cb8), + LL(0x6371852a,0x473e976c),LL(0xa5e1af16,0x88aeb436),LL(0x368ce839,0x94aabe19),L_(0x00b82580), LL(0xb19a3ec7,0x7d50b7ff),LL(0xd8fc842f,0x81c2bf45),LL(0x8c4eb78c,0xb0a7cd1f),L_(0xf40b4869), + LL(0x56f0849d,0x5739e59e),LL(0x79c44251,0x1fc64939),LL(0x8b91a3a3,0xd401b24b),L_(0xd2207cb0), LL(0x9decd9c6,0x17f178e9),LL(0xcede5d5c,0xfa904473),LL(0x7a09efc0,0xca5c00a9),L_(0x1761360d), + LL(0x6aca6f61,0x4bda0113),LL(0x3b3dcab5,0x12ccd894),LL(0xb4db3fdf,0xb39be04f),L_(0x1c09bf2c), LL(0xb4705566,0xccbe5154),LL(0xe1559ff1,0xd84fec44),LL(0x11964906,0x9ceede3f),L_(0x0e1b9fc5), + LL(0x13111bdc,0x175d3f77),LL(0x17226b06,0xe0971341),LL(0x21244b52,0xd6fa5642),L_(0xfdbb39d3), LL(0x68da34f2,0xb9933942),LL(0x6759bac0,0xd7d7f790),LL(0xee292b1d,0x4382d48d),L_(0xd117a9f3), + LL(0xe877a209,0xaf9b8432),LL(0xa22db654,0x12572249),LL(0xbb09b923,0x1501cefc),L_(0xd5820554), LL(0xe0614220,0xaa863b6a),LL(0x397b3194,0x27337829),LL(0x1a9bf0b7,0x6912b5aa),L_(0xc767a242), + LL(0x38d01225,0xa0259e70),LL(0xf56ef5aa,0x8e85d7ee),LL(0xfd49df96,0x6bb4a230),L_(0x8f3a3f3a), LL(0x58b99e53,0x3c2be4ba),LL(0xcf1ea9c2,0xe49fcb3f),LL(0x4937a76d,0xdc9b73dc),L_(0x5ea4828a), + LL(0xa516c9ee,0x41610138),LL(0x30e01a74,0x71d5b83a),LL(0x8795351f,0x9a45e682),L_(0xa31707a8), LL(0x9edacc8e,0x17bd9c0b),LL(0x80f31bd0,0x871cc69c),LL(0x57013f03,0xb6b7b6e2),L_(0x1073b301), + LL(0x4855e4b2,0x124c67e8),LL(0x6946f5db,0xd4d7f50b),LL(0x494ae118,0xf16b9834),L_(0x0c0fe86e), LL(0x3a6a8d31,0x18d69af9),LL(0x54ebd047,0x7f8ed091),LL(0x9daffa88,0xba6e7c59),L_(0x54652a79), + LL(0x011788f6,0x9c64aa0c),LL(0x8403c566,0xf896522c),LL(0xcc8f1ea7,0x772627b7),L_(0x707ce584), LL(0x72bb812b,0x57f8a759),LL(0x4f42aa8c,0xc7e7a397),LL(0x4c06faeb,0xeea7a14b),L_(0x1edd837a), + LL(0x0c456e85,0xbacf3d29),LL(0xdb2d668c,0x861ef908),LL(0x17a88c53,0x25d01a4d),L_(0xb97ac689), LL(0x0b6e87ff,0x066301cc),LL(0x78d4d6b5,0x51763eac),LL(0xe4c44c3a,0x243f5364),L_(0x55bb2e54), + LL(0x69fe2b3f,0x623f7c06),LL(0x969019fe,0xa81b9914),LL(0xba6dd8e5,0xd33612e7),L_(0x0fd330c2), LL(0xc948497d,0x7184b686),LL(0x065a9367,0xd5e74279),LL(0xfa105e4b,0x4bfe5bea),L_(0x30ca8fc4), + LL(0x03d46275,0xfcb464d3),LL(0x7fb49121,0xdcb4ccb3),LL(0xf6934477,0xbdf563a1),L_(0x4412ba6f), LL(0x7c07acc8,0x4d4331eb),LL(0xe0dfac8b,0x768b2705),LL(0x022d151f,0xf65a4027),L_(0x251db799), + LL(0x5cbe2654,0x32707706),LL(0xbeb70e80,0x9739588e),LL(0x1d1ff202,0xe51c696a),L_(0x000ae445), LL(0xdd740418,0x25fcc7e6),LL(0x646ef912,0xac2dfbc7),LL(0x0d72b69e,0x1ff6f6ad),L_(0x059f3c0c), + LL(0x3ade75e8,0x3ac64888),LL(0x7ee14ba6,0xd957b10f),LL(0x7be01ff5,0x33f835d6),L_(0xc155847f), LL(0x45081b73,0x0c08887d),LL(0xda2f5358,0x4fc01e34),LL(0x84d416f7,0x15716e5d),L_(0x3950aa76), + LL(0xc4cafbc3,0xcd118e18),LL(0x60a29243,0x78d7034b),LL(0x4d198a7c,0x9ba63f25),L_(0x4b83593c), LL(0x81c83f83,0x9eb5e40b),LL(0xb8f7f9c2,0x3b1941e5),LL(0x27b87b90,0xb7229f73),L_(0x247578db), + LL(0x59afdea4,0xf6e5451f),LL(0xbdfb7efd,0x3c7977fc),LL(0x8262d80b,0x0ff79532),L_(0xb6d1bfa5), LL(0x85d46816,0x41a59faa),LL(0xbb16202e,0x815bfdff),LL(0xf25c4d84,0x26a648ca),L_(0x2a115df7), + LL(0xc47be93f,0x8807158f),LL(0x31f52fc1,0x71da23a7),LL(0xf790aca9,0x26957d5b),L_(0xe3199eb7), LL(0x571af638,0x3a3ff68a),LL(0xa6e9167d,0xb02eef0e),LL(0x78555759,0x1609b893),L_(0x1507c2ea), + LL(0xa99f1707,0x7a4aae59),LL(0x4be04c26,0xc84493c1),LL(0xf1b134ce,0x8f2fb8ff),L_(0xc7137a45), LL(0x8f6f7a80,0x73cd2591),LL(0x3b2e141f,0xf68459a2),LL(0xf4b9ddab,0x1d8a7278),L_(0x403dae23), + LL(0xa1fd53d2,0x3ad117de),LL(0x47d15513,0x75e31939),LL(0x58a5dfbd,0x3251c208),L_(0x66a06b3d), LL(0x8436c37d,0x65ab378a),LL(0x5a3fb858,0x3a685733),LL(0xe6172e78,0xfa11b001),L_(0x25e5c930), + LL(0x18e18997,0x48c2dc2c),LL(0x900eb7f7,0x814d21d9),LL(0x198afe84,0xf76ecf91),L_(0xab542184), LL(0xb918572f,0xf53020dd),LL(0x31471cf4,0x365cb3df),LL(0xa6a9ebf4,0xf8ad1ad6),L_(0xf3563966), + LL(0xf1859231,0xcb1be872),LL(0x1ad60df0,0xfaa10d6a),LL(0x11273c1d,0x240336fa),L_(0x4a1692f1), LL(0x6bfe9509,0x88653660),LL(0xf81505d5,0x7b0d301b),LL(0xd427be59,0xbd85a9ef),L_(0x1a92bc3d), + LL(0x72600cd8,0xe9cae1c4),LL(0xab08a5cc,0x3d1457c8),LL(0x96865fe9,0x9a1bc1e7),L_(0x438f5a09), LL(0x7727a4f6,0x09a2a9db),LL(0x33e95a6f,0x028d0a98),LL(0x9656e0c0,0x7f3b01db),L_(0xeae74794), + LL(0xb1f41609,0x060c3752),LL(0xc0ad59f7,0x6efb7250),LL(0x0bd8bd31,0x79016736),L_(0x34d2ef7f), LL(0x8739f62d,0xf14247f9),LL(0x7675377d,0x29b78fc9),LL(0x5b2e893a,0x5f2b1e7f),L_(0x3c1437bc), + LL(0x380ddade,0x014c7c14),LL(0xe37d7ead,0xc2d7837a),LL(0x78897ca3,0x5567dd34),L_(0xfa7b8f4b), LL(0x00642d56,0xf735f752),LL(0x7fd93e92,0x8afd8b87),LL(0x6eb02b4e,0x120f5319),L_(0x1ccf74f8), + LL(0x8fc31699,0x41d511e3),LL(0xb78232db,0xbdc4fb64),LL(0x9841bbd5,0x207bfaea),L_(0xd0f130c1), LL(0x38fcb8da,0x6ba5dcee),LL(0x93e47594,0x21f2d335),LL(0xf78fd1c0,0xc6f634fd),L_(0x717d258b), + LL(0x09a676cb,0x414f2daa),LL(0xa55cfcad,0x4cbc9d7a),LL(0x3b973302,0xb409e95d),L_(0x7da7223a), LL(0xd16a5ca9,0xf92a6554),LL(0xb7642b9f,0x153e5024),LL(0xa08470dd,0xae4dfb2b),L_(0x8fd81632), + LL(0x5b3c44c4,0x45031642),LL(0xc6ae8b05,0x335c400a),LL(0xd4b606ca,0x6b34a475),L_(0x4a823453), LL(0xf3ab94ff,0x746295e2),LL(0x25948202,0x4b9b612a),LL(0x9d957566,0x510e67f8),L_(0x42ddbb08), + LL(0xdc56f265,0x9f7dfe87),LL(0x20005269,0xe3386448),LL(0x104ea5b5,0xac4c4feb),L_(0x6565a54e), LL(0xb2f8650d,0x186e3514),LL(0x50e692b2,0x58c2701b),LL(0x81927ca6,0x81220e7e),L_(0xa69b1829), + LL(0x727a95ea,0xfb0e9702),LL(0xf0b23c9f,0xcca5f38b),LL(0xdb71dd75,0xad31c5dd),L_(0x93004978), LL(0x3ff02652,0x683b0e57),LL(0x2b3b008d,0xc6e32a5b),LL(0x4ab25a51,0x283f183b),L_(0x0cc76852), + LL(0x08ea3768,0xf7f76eae),LL(0xbcdd82ad,0x222dd486),LL(0x5b8927a0,0xdc4a955f),L_(0xb76f9687), LL(0x605b6b78,0xc80579f6),LL(0xb80f7796,0x4cf36f45),LL(0x94402fad,0x0a226060),L_(0x0c94e0f7), + LL(0x77db1b5e,0x4fe36153),LL(0xb19d463f,0xb32ea8ef),LL(0xa09373a7,0xd4f68719),L_(0x1f1869b7), LL(0xd8828434,0x8c0026e9),LL(0x8f8f9197,0x16af4c49),LL(0x8f7a154b,0xd4672394),L_(0xee9c7e14), + LL(0xe432397f,0xd0196172),LL(0x035a30a5,0x114f01ee),LL(0x12143827,0x2f06e8aa),L_(0xcd20f526), LL(0x00bcfcbd,0x76ee873b),LL(0x542f1d14,0x8ff02da4),LL(0x786da906,0x7e7e6c65),L_(0x65c3eef1), + LL(0xc8866c5c,0x0d7b5118),LL(0x5cd6ca19,0xef5be83d),LL(0xb37c41ce,0x4b76c64d),L_(0xf34bee15), LL(0x3d28da12,0x8ef7cb6d),LL(0xf55195fb,0xa3c8a2d4),LL(0x4369465e,0xc8c85ab8),L_(0xe811f166), + LL(0x08818979,0x17f1f556),LL(0xbf0f506b,0x251cc269),LL(0x686c57b8,0x865fb805),L_(0xa325b2ac), LL(0x19c7f60f,0x23566eda),LL(0x40af0c7c,0x47f95f9f),LL(0x867b619f,0x1958dbaa),L_(0x164ce142), + LL(0xfc7b0693,0x44b01395),LL(0xedbed37e,0x0d676d86),LL(0x075607b2,0xe2ae6225),L_(0x9bd85253), LL(0x56d68f4b,0x7e636075),LL(0xb247649e,0x1ffba2be),LL(0x5c1ff02f,0xc237448d),L_(0x063c33cf), + LL(0xc3ea5576,0x52c5f87e),LL(0x2fe94caa,0xf135c024),LL(0x593535ee,0xe0965b4b),L_(0xf93cf6d1), LL(0x54fedc18,0x28d4b11a),LL(0x6c77585d,0xe5297d99),LL(0xb0d741c2,0x39caed90),L_(0xcc98dae9), + LL(0x340fdc0b,0x1b8201fc),LL(0x44198c3e,0x248c333c),LL(0x5f236490,0x12ee4946),L_(0x0789670e), LL(0xcbc6b050,0xee23385a),LL(0x3c3d5d85,0x630f7175),LL(0x0000016a,0x3f6e71ce),L_(0xad6ec3de), + LL(0xc6c3de31,0xf99b211a),LL(0xca35fe5b,0x549ffc41),LL(0x5ffb8aa4,0xbc4e1e52),L_(0x6ac77d41), LL(0xe552f85a,0xf54af34b),LL(0x6e1ccff3,0xd61618d2),LL(0x5bb20166,0x5776aea6),L_(0x7336f961), + LL(0x535ea11e,0x565a5471),LL(0x8f3dcba8,0x4200ddee),LL(0x8ee2aacd,0x2493fe70),L_(0xab90e06d), LL(0x048f6037,0xc2e41220),LL(0x075b150b,0x3193a2ea),LL(0x3c56b2e4,0xc4609ea4),L_(0x2fdd776c), + LL(0x4276f086,0x57d5dc17),LL(0x2aa19794,0xa8a6408a),LL(0xd70d74f7,0x98acac12),L_(0x351c3c6f), LL(0x93cd39aa,0xb4c10685),LL(0x5a2bba35,0x56a7276e),LL(0x2685e5f9,0x3eeed659),L_(0x8a344719), + LL(0x2d701e0e,0x22e61da3),LL(0xbff21920,0x4511147f),LL(0x403fd833,0x6080d968),L_(0xb3b3779a), LL(0x2957083b,0x38eb9ccb),LL(0x2ed5dc9b,0x1be2ace4),LL(0x726301f8,0x002d8909),L_(0xbe322533), + LL(0xd1f9235a,0xf591d929),LL(0x0184d742,0xdc207a6c),LL(0x5e691f61,0x20b8f8e8),L_(0x2095a193), LL(0xaeb5ac0a,0x7eacabb9),LL(0x3b17347c,0x0985df07),LL(0x04b8e782,0xf8047be0),L_(0x5ad11ec9), + LL(0x8492830a,0x4f347f1e),LL(0x841e8dca,0x7d0fc13f),LL(0xf365c6c6,0xa0e662a2),L_(0xded71c8e), LL(0x8d246f45,0x9dcc1860),LL(0x852d364c,0x5bfa3ba2),LL(0xd9a27ded,0xb281e327),L_(0x80b9b9f1), + LL(0x6bab642c,0xbff128cd),LL(0x692c67d2,0x268f3893),LL(0xfc73c613,0xcd24728c),L_(0xa40f87b1), LL(0xd40e3e5a,0xf065f5d7),LL(0x96f653a5,0x15d0cf49),LL(0x4ad712bb,0x70e00818),L_(0x2102b8b4), + LL(0xf1c993cb,0xa2f0722e),LL(0x214a8d8b,0xe7c142ac),LL(0x8af34ce1,0xce47aba3),L_(0x2d01c664), LL(0xb6c1f1ff,0x49dac9d9),LL(0x8e33c0f4,0x1baa4d4c),LL(0xdbfb3b98,0xd927e1a8),L_(0x80f00079), + LL(0xde4413ab,0xad27b0b5),LL(0xa2997bb5,0x0077242e),LL(0x5d3ac419,0xe50d3185),L_(0x185f649f), LL(0x6b5c4eea,0x4a8b1fad),LL(0x210cc7de,0x30ac4e43),LL(0xe0253dce,0xf07461c4),L_(0x4bf807d6), + LL(0x01947899,0x818e99b8),LL(0x9008b9c3,0x05ed8f7b),LL(0x09f78f1a,0x2fb78a6e),L_(0xa5da0ebd), LL(0x4c2f0cce,0xfdd48d7b),LL(0x79765db8,0xe380dbfc),LL(0x9600539c,0xbb3a3bc6),L_(0x3627d4d4), + LL(0xea2bbe08,0x394029b6),LL(0x7ac327d1,0xc8ab0424),LL(0x90339a91,0xda368bdd),L_(0xab696429), LL(0x0fe39b21,0xc20f1366),LL(0x32fec057,0x8075ba9a),LL(0xdc668935,0xa20bdd31),L_(0x2abf3c1d), + LL(0x513bd424,0x02a9ea32),LL(0x7d6a8a11,0xe477baa6),LL(0x52b0bc16,0x4583bc9a),L_(0xba81bd4a), LL(0x287b6485,0xf11b84b0),LL(0xae92c68c,0x7925b118),LL(0x4ab367b2,0x8ff3e481),L_(0x33179660), + LL(0xb6f13800,0x0ffc4673),LL(0x56722205,0xf9b0dfda),LL(0x2bd4103b,0x56acfc3b),L_(0x1ce74c3f), LL(0xa87de434,0x046c39cd),LL(0x41e5b749,0xdebd1720),LL(0x11fa933d,0x3f6c6fec),L_(0x275b834e), + LL(0xae4601fd,0x06034734),LL(0x58ee806e,0x104d85dc),LL(0xca3c77a5,0x8bca0d2e),L_(0xb812f136), LL(0xc78c356f,0x70ae3402),LL(0x7dc1f03f,0x322e72ad),LL(0x07c1fe3f,0xd0253c44),L_(0x511d1058), + LL(0xc2f400a7,0x2291a21d),LL(0x993f820b,0x287eac12),LL(0x1680907d,0x03a00f86),L_(0xd919412a), LL(0xbadddbc3,0xc5e15b28),LL(0x0ba18a53,0xf30664e1),LL(0xb1606671,0xc87b6603),L_(0xb55b207a), + LL(0x1873a8e5,0x5f55c7e2),LL(0x6eaa78ce,0x33f083bf),LL(0x95c9fed8,0xe416fc1c),L_(0x7e0f68c5), LL(0x02778f82,0x07eef5cc),LL(0xdb2aa0a1,0xa94149df),LL(0xc8c44cfd,0x2d270d3f),L_(0xec06f559), + LL(0x7ca9f958,0xdb514ec8),LL(0xd0ace135,0x9718dd09),LL(0xe2bf5a99,0x325fe3bd),L_(0xe5c7ec48), LL(0x4f0b0621,0x78990d74),LL(0x76fa9487,0x5e70b57d),LL(0xdfbda68c,0x6e9d11e2),L_(0xda96d406), + LL(0xd61361bb,0xf97cf589),LL(0xc96cb924,0x7b21af76),LL(0xf64ebc1a,0x03a72d66),L_(0xecbebd56), LL(0x4b021c91,0x39df1cca),LL(0x9863869e,0xb84c6c05),LL(0x9b6a3082,0x62692bf5),L_(0x977cbcb4), + LL(0x94bf9bf5,0x79c2f57b),LL(0xb9be753d,0xe0d00e09),LL(0x49f82bc9,0xb9c9a088),L_(0xcc0f3b6b), LL(0x1f589bd1,0x11cd1677),LL(0xe709a25f,0x8f952cff),LL(0xb5eec978,0x016b826e),L_(0x3188c1dc), + LL(0x8cd03f0b,0xf4d9e042),LL(0x2b04b26f,0xb07e4630),LL(0x361c9252,0x5bbca5ce),L_(0x783f801a), LL(0xe687c320,0x7bd9a485),LL(0x465590b5,0xe200e18a),LL(0x0e254cdc,0x50dce2e6),L_(0x49ff7ac4), + LL(0x4b5e8399,0x284e641b),LL(0x33c76e5a,0x0556fa64),LL(0xa75c4fae,0x4cc4b972),L_(0xc36d03ba), LL(0x90fa7978,0x6a40b04c),LL(0x4fef28c9,0x97401c28),LL(0x613bfe61,0x443845a0),L_(0x5144c164), + LL(0x532b2d68,0x482b7779),LL(0xe955f441,0x123aa16d),LL(0x6886e7e9,0x9147c317),L_(0x1d0658eb), LL(0x1530f2af,0xaed04ac0),LL(0x192c5796,0xa4111d16),LL(0xd86157ff,0x7406594c),L_(0xbf39202c), + LL(0xafae6c6f,0xed33d4b3),LL(0xc5df16a7,0x008ec0c8),LL(0xfe186e4e,0x7e4ed374),L_(0xdca675ee), LL(0x2534f6e8,0x798a9b30),LL(0x82a138f5,0xfcf997cc),LL(0xb58b2b67,0x2f620d55),L_(0x634f7d88), + LL(0xe8013df8,0x5dcfcfb0),LL(0x7b9259dd,0x7a765b72),LL(0xe0fbd7bb,0x4e7374be),L_(0xc2b9a098), LL(0xfe9b1e78,0x375f54f1),LL(0x0f6c804f,0x55dc463e),LL(0x202fbb6d,0x8d697af5),L_(0xe7949513), +}, +/* digit=13 base_pwr=2^91 */ +{ + LL(0xbcdbe445,0xcb4a833d),LL(0x57ca0395,0xb07c1b51),LL(0xef7d725e,0xdc3b82a8),L_(0x1e9a35e2), LL(0xaa5a7ac9,0x01e0c54e),LL(0xc707131f,0x2b2d5196),LL(0xd220aca6,0x736d91c9),L_(0xde0d6eba), + LL(0x74d6d0bb,0x9af89e67),LL(0xfa0e831e,0xada42e0d),LL(0xda49ba83,0x83b9c4ec),L_(0x782ab495), LL(0x12dfdcae,0x7e0a2cb4),LL(0x4c92a243,0xf7e3a9a3),LL(0x6ed0d332,0x1f8c692d),L_(0x5cfab000), + LL(0x634cef8b,0xff5ab01d),LL(0xa43c6f4e,0x6fe3ecd8),LL(0xc1015db1,0xfcc7ca4c),L_(0x9199e5e3), LL(0x795d4927,0xed944d52),LL(0xe4551e73,0xf5f41299),LL(0x8f02bb95,0x4007343b),L_(0x896481e0), + LL(0xab703650,0x6b243c97),LL(0x660da927,0x8b9a0873),LL(0x0ef0ffd5,0xb6e8c899),L_(0x8a32e4b8), LL(0x2e1c11d9,0x154379bf),LL(0x2d0a9ded,0x4382cb75),LL(0xc876e0d3,0xc42ec7d1),L_(0x41c57824), + LL(0x1ce64f94,0xa2eb2c00),LL(0x1eb0301b,0xe1a22445),LL(0x450ed02e,0x4205e69b),L_(0xf1b1c831), LL(0xc76dda33,0x62db01d8),LL(0x168f45ba,0x633bdf3e),LL(0x95c86470,0xf9a6b309),L_(0x9bb55ae5), + LL(0x71aba88f,0x590711c2),LL(0xdea8afde,0x0b6ab427),LL(0x87bfe939,0x1d7e1090),L_(0x54be137d), LL(0x7ebcdd75,0x2a937fd4),LL(0x5fed0f5c,0x3da3197d),LL(0xc4155a57,0xf1cea938),L_(0xe2bb2c19), + LL(0x6a4807e4,0xc0c52eb7),LL(0xfe9c4a62,0x12ba26fb),LL(0xda7a7914,0x001d4756),L_(0x48b356bb), LL(0xce833468,0x3290dbb2),LL(0x432f5193,0x2b0a3429),LL(0x579c91eb,0xc27c2657),L_(0xcb222ca0), + LL(0x6983b44c,0xf27d3e69),LL(0xf7665b89,0xe4a043b5),LL(0x5a97c010,0xc8f8a3c7),L_(0x4ed639e6), LL(0xb9a3137d,0x5655ed33),LL(0x6987fe61,0x0b6dca4a),LL(0xc1a3c9bb,0x630b3026),L_(0x9305872d), + LL(0x082a09e8,0xffe89fc6),LL(0x673fe488,0x9f4f85f6),LL(0x0b13732a,0xba6e1981),L_(0x1f59e6b4), LL(0x26eb2b18,0xa4afe844),LL(0xaddd3cd6,0x91a5d7b7),LL(0xeeac1c21,0xe71f5fe0),L_(0x5d6e8430), + LL(0x3f0bf1e4,0x031e908e),LL(0xecd2060c,0xbc16afc5),LL(0x6e2d4301,0x90a23e58),L_(0xd061f3f4), LL(0x2549fd21,0x52f4ef15),LL(0x4103c527,0x5b4c8dd8),LL(0x1a5fbbc3,0x7457c84e),L_(0x6620daf1), + LL(0xa3742f5a,0xe0a87786),LL(0x20460a16,0xe7aa9e66),LL(0xba62daa3,0x65327959),L_(0x4241a339), LL(0x7da31b21,0x7fbf3e8d),LL(0x352aab9b,0xe1a7865a),LL(0x877c379e,0xea576b89),L_(0x5f93fdc5), + LL(0xaa616a3a,0x662a3f35),LL(0x94cdd0da,0xdd7d18e3),LL(0xee30f511,0xf25f9c46),L_(0x35fff2a0), LL(0x3e51e653,0x81e804d2),LL(0x5957ab11,0xb0aaaab1),LL(0xa52da544,0x89ee1376),L_(0x8411811a), + LL(0x97063083,0xf2d42cdd),LL(0xc2eb7ee5,0x9cb9423b),LL(0x6ed009de,0x746461d4),L_(0xc182c461), LL(0x6b0d52f8,0xe78fe609),LL(0xde14941d,0x6887a9cc),LL(0x8730c3cd,0x0f586a94),L_(0x2cf25d5d), + LL(0xf21a3a3a,0x38aa8cf8),LL(0xd96cec97,0x3dd4e3e4),LL(0xf1f81f18,0xee849585),L_(0x6ec8f23e), LL(0xb30df029,0x1e4514ba),LL(0xaf980c70,0x757e6771),LL(0x3b2305c3,0xa3525eaa),L_(0x755d2034), + LL(0xb94b1aa2,0xebe6c1cc),LL(0xb228a915,0x1c4d6e36),LL(0xfa33270b,0x5d3856a1),L_(0xf1ba0e3a), LL(0x77b4e048,0x63f53943),LL(0x99a1ff3b,0x7466b59d),LL(0xaea29806,0xb1050d57),L_(0x9d139cfa), + LL(0x0fd330ad,0x0f394d0b),LL(0x27d1a20e,0xe4d013eb),LL(0x99b62540,0xaa1c17a9),L_(0xb03afc54), LL(0x8f6f2743,0x7366a618),LL(0x449f676a,0x88e9ae3d),LL(0x8818aa14,0xd2fe924a),L_(0xf5d1f438), + LL(0xfd68c56f,0xd02cd00e),LL(0x94c49868,0x5f260a20),LL(0x9c693311,0x7c1f3266),L_(0xe30a0258), LL(0x5b2551ac,0x9f545dde),LL(0xbdeac1ae,0x19f87afd),LL(0x019bf350,0x58397798),L_(0x787035be), + LL(0x46ea826b,0xc4da4263),LL(0x10febee0,0x43744264),LL(0xbfc3581d,0xcc933457),L_(0xd3126394), LL(0x9cbecb37,0xe0071ca1),LL(0x715c9dad,0xcf0cfc02),LL(0x67a2c9e1,0x2f9bd24a),L_(0x47f400b4), + LL(0xa57e540f,0xd472cc15),LL(0x1ab19257,0x0b60e719),LL(0xcd92d76f,0xca1c9c30),L_(0xb75f7cec), LL(0x8103a773,0xa305744b),LL(0x34bb833a,0x54b38b68),LL(0xd6e1cdb8,0xb252c82f),L_(0xbbc88ea0), + LL(0x935833d7,0xd0e4995f),LL(0x44ed9975,0x87f60eef),LL(0x933733ca,0x379092ba),L_(0x0e167f70), LL(0xd4945af6,0x60eeff1a),LL(0x589a5723,0x1d6d1def),LL(0x4bee290d,0xe87f42f5),L_(0xbdc1579e), + LL(0xb222f294,0x080ba4e8),LL(0x83c59be1,0x8ed3f3c1),LL(0xfdee70ba,0x58ce1c82),L_(0x309979e1), LL(0x8490a253,0x6ed1e744),LL(0xc3fded2d,0x0602a508),LL(0x585e149e,0x7fa41773),L_(0xb933ab59), + LL(0xf0a3e292,0x74dc9c31),LL(0x0f2c66c5,0xc78baacd),LL(0x01ba1bb0,0xe5b73558),L_(0x328bebc7), LL(0x9cf50576,0x5421bd61),LL(0x76efecbb,0x241cb4c6),LL(0xc421cfb2,0x9ba297a8),L_(0xe31b7671), + LL(0xf0b66ab8,0xd87b31c9),LL(0x3aacb600,0xcf835fb0),LL(0x2f9c67c3,0xaee5e6cf),L_(0x6453b1bf), LL(0xd04298de,0x0411e445),LL(0x0d41aa91,0xbec90b82),LL(0xe6adb2bb,0xb30b9c0e),L_(0xdd7ea812), + LL(0xda605906,0x5f01a30b),LL(0xe7f26ffc,0x7c053c40),LL(0x40f7d79d,0xc39feb03),L_(0x232940c5), LL(0x3d73c35a,0xe472e81d),LL(0xe3395fd8,0xdea2bf33),LL(0xed66fd6f,0xddc57518),L_(0x675b9966), + LL(0x7095dc92,0x07f66d27),LL(0xb47087dd,0x9c4ec651),LL(0xfd209ae0,0x35803afd),L_(0x6fb37653), LL(0x51ed5eaf,0xc9356091),LL(0x47c63c1c,0x19e52bee),LL(0x0676dac6,0xd87e6c27),L_(0x6badd30e), + LL(0x83877c71,0x39e61abc),LL(0x58d7a9d9,0xa88e05f0),LL(0x7885ad28,0x53bfba53),L_(0xb10f5fdb), LL(0x8e433b9f,0xd6a78477),LL(0x0f088cf6,0xcefdc6a1),LL(0x3ce9d748,0x27d9ed71),L_(0x648c4c3c), + LL(0xe57aeaa9,0x7dbbbc39),LL(0xe13d51e3,0x5c942e4d),LL(0x7d7aed19,0xe660d0ba),L_(0x25bcdf2d), LL(0xff40dd8a,0xcdf314b5),LL(0x5faa85f3,0xa625d183),LL(0x9d4344c2,0x54f2035c),L_(0x5243fd7f), + LL(0xd120f88e,0xe3275570),LL(0x79d8c8b5,0xe344b66c),LL(0x8d5e8c7f,0x1cdb23ae),L_(0x8c372c12), LL(0xce526de3,0xe64e6355),LL(0x6f104a11,0x5edf30db),LL(0x7e00559a,0x3501ee2b),L_(0xdc58f8f4), + LL(0xa6698b0d,0x91c3e912),LL(0x65e168bf,0xbb24391d),LL(0xe1e797f6,0x0526a13b),L_(0x6c3a27f6), LL(0x41e4a8d4,0x63a82271),LL(0x759095e6,0xf6fd3e09),LL(0xd1ade148,0x85041faa),L_(0x1d102e98), + LL(0x7675921b,0xb30c5e67),LL(0x781de03f,0x9a004497),LL(0xb5aac131,0xcaf13420),L_(0x2a74de9b), LL(0xced86c6a,0xc60fb50a),LL(0x4039952c,0x8cd4dd3d),LL(0xdec98657,0x5092d4ea),L_(0x573afadf), + LL(0x9fb10f98,0x67b904af),LL(0x7233bca1,0xeebc687c),LL(0xdae52182,0x13aadd13),L_(0x7d61309e), LL(0x59a01f8b,0xc7db8080),LL(0xf6d5d889,0x5433ce07),LL(0x8b26a71a,0x0c3a71bb),L_(0x69b8c725), + LL(0xd29e534f,0x215aa2f4),LL(0x7bb1ac35,0x5f00b26b),LL(0x3c54a3cb,0x71ec2585),L_(0xf6cd94c0), LL(0x4c553b93,0xf86c23e2),LL(0x1b7d9626,0xdd6cfc3d),LL(0x0fb40ab0,0x4647a750),L_(0xeb4bdb5e), + LL(0xa51a2b17,0x1bbce0ea),LL(0x6f2f98f1,0x7e78ccf9),LL(0xc6736591,0xbaa1d2e3),L_(0x47e735c5), LL(0x6b8866aa,0xc5b9e6fb),LL(0x8aef23dc,0xc39a4975),LL(0x712336e6,0x86d2dbfa),L_(0x19715594), + LL(0x106db862,0xc495534c),LL(0x83770bdf,0x75c25358),LL(0x05a076ec,0xf2a1fa9b),L_(0xc80d0362), LL(0xbab703f6,0x2b054aeb),LL(0xb0c6f6de,0x9e9ed0de),LL(0x42646b39,0xd95a6334),L_(0xddad98a9), + LL(0xbcf088fe,0x373df018),LL(0x7bd97cea,0x2646d8cd),LL(0xa20f1b7c,0x3bc4db1d),L_(0xdd201f09), LL(0xf3f061d1,0x7fc330d4),LL(0x5f1ec56b,0xd64fa0dc),LL(0x93be4afe,0x239a905a),L_(0x38df95b1), + LL(0x9491ca55,0x3a2368e4),LL(0x4fc84a15,0x840339bb),LL(0x60229ac3,0x0f74bf45),L_(0xb9a147d0), LL(0x6bb23f64,0x6df7548f),LL(0x928eccc2,0xbd4ae1ae),LL(0xa5c7651d,0x85fa24af),L_(0xe98fba58), + LL(0xd9ea606f,0x72e722f7),LL(0xde7a39d8,0x02dae11e),LL(0x6f289df9,0x4f148db1),L_(0x3efce901), LL(0xe8df6027,0x0dc6bfc0),LL(0x25925285,0xc2c848fa),LL(0x726e5bf2,0x0bc8372a),L_(0x4993590e), + LL(0xe513cc61,0xb368f5d3),LL(0x1b56d61c,0x2acdbaae),LL(0x077919ee,0x69129848),L_(0xd6bce458), LL(0x5c03af06,0x6892cac4),LL(0xbf732964,0x13156875),LL(0x86df2c46,0x15d63bed),L_(0xc24215d0), + LL(0x85ef7366,0x7ea45182),LL(0x4f6a510a,0x7d4f7ca5),LL(0x2e79b2d0,0x5befddd6),L_(0xdfce6391), LL(0xeb6e8178,0xd7c05293),LL(0xbb2671e0,0xfebc1ad3),LL(0xc2ef9919,0x5ef848b7),L_(0x3034a5d7), + LL(0xec0fd6a3,0x4834ba2e),LL(0xd4612c03,0x7006255f),LL(0xee7cbb5c,0xa240c28a),L_(0x9d74ecc1), LL(0xe8bfcc1e,0x36f2363a),LL(0xe755acbe,0x2c7fdeac),LL(0xee2e792a,0xfe7f2c9f),L_(0xf3a0b1ef), + LL(0x21f1b866,0x7a207076),LL(0xb0caba72,0xa289580c),LL(0x26c6a4fb,0xac8c38be),L_(0x46fba43d), LL(0x603a204d,0xc96f5b0f),LL(0xaae0bc34,0x07974abf),LL(0xdd93d467,0xd384c831),L_(0xd3e228d7), + LL(0x130cf278,0xb2d57bad),LL(0xa4b0a795,0x2e92cb97),LL(0x0b8251c6,0x04325b28),L_(0xd825f27e), LL(0x630e8e9e,0xcda5985f),LL(0x1bba4e62,0xf6775b8c),LL(0x9cc53c03,0x466530c7),L_(0xca08b958), + LL(0x9b3d46a1,0x6911dd6e),LL(0x64d4f9aa,0xf5c72cad),LL(0xaf7d9d93,0x53a3ea3c),L_(0x99f4337b), LL(0xd879f3e2,0xeaba3d43),LL(0x7bd203ea,0x4c921337),LL(0x3b59edc0,0x7ee8ea79),L_(0xdbb3851a), + LL(0x64c557a1,0x70b7979e),LL(0xb3681c37,0x482d3b99),LL(0x08f6017c,0xecf693f0),L_(0xf2fe6d70), LL(0xe8816978,0x7c8670c4),LL(0x8a10a4b5,0x05b8eed1),LL(0xa8713371,0x0ebddf9f),L_(0x6f3c3b96), + LL(0x50dc1443,0x884f22ec),LL(0xf9400068,0x5f885c0a),LL(0x4da67f62,0x6b4ecc34),L_(0x170f203f), LL(0x0181b70d,0x3f227a67),LL(0x844eab79,0xafb114cc),LL(0x5b40835b,0x2f5bb436),L_(0x7b9f2c6f), + LL(0xdaec76bc,0x6deef6a5),LL(0x338dc21a,0xe70560f2),LL(0x8e70fab8,0xdf0c450d),L_(0xd256f11d), LL(0x916202a8,0x05568dde),LL(0xfb8b1d84,0x89f0919e),LL(0xb5670da6,0x97387cce),L_(0x8943fa2b), + LL(0xc269a34a,0xae4d60ad),LL(0xf62da5da,0x27a5a54f),LL(0x560e5ad7,0xfd2940f8),L_(0xab3236ae), LL(0x86446f6a,0x2f2ee978),LL(0xb56b7c8a,0xa481c719),LL(0xd1f9e2af,0x99fe0c80),L_(0xdc1ec6c4), + LL(0x4c354b32,0x9d2731ca),LL(0x45db9456,0x9583d77a),LL(0x549fab4d,0x20743202),L_(0x133b9227), LL(0xc9afa8d6,0x6ee70bf6),LL(0x61e414f2,0x71181b55),LL(0x34dfc171,0x186cb701),L_(0x05ba8622), + LL(0x784904dd,0xef91aeb1),LL(0x2ce39834,0x466ca745),LL(0x9dba0e94,0x3efbbed0),L_(0x6d055926), LL(0x818f1161,0x238404b3),LL(0xb2aba397,0x7f378101),LL(0x9026e5ef,0xd9ddeb02),L_(0x639d63b6), + LL(0x50f516e3,0x359a1ffe),LL(0xc1a03316,0xea7c7fb8),LL(0x2be043ec,0xe26d37f4),L_(0x34fbd1da), LL(0xd8d23c82,0x97f967f3),LL(0xf3027282,0xf97e68ce),LL(0x0cf39dbb,0x0f3321ca),L_(0xb0d6de7c), + LL(0xd200dfe0,0x890af600),LL(0x905f6954,0x1e4962b1),LL(0xd80a6211,0x78562fa6),L_(0x0e1b429e), LL(0xdb5a70a8,0x68abbf8d),LL(0xdf21b056,0xbffea5c7),LL(0x99114570,0x6e5308c1),L_(0xd35cddc0), + LL(0xdfd77bdd,0x16448af4),LL(0x5bcbe7ef,0xcb49f269),LL(0x9f4c424c,0x7c95aa03),L_(0xb01ff8c8), LL(0xd5948dcf,0x93acdc43),LL(0x2ddd8b66,0x7efb93e4),LL(0xa8e311a0,0xe08e348f),L_(0x7b4dacb7), + LL(0x96c4f395,0xfa10af74),LL(0xd4e05b0b,0x489d01bb),LL(0x1a4d0c16,0x5457e489),L_(0x70983e98), LL(0x1331b4da,0x84eb66f4),LL(0xf38823d2,0x8054cf79),LL(0xb191cf89,0x3e7eb0cb),L_(0xa37a2976), + LL(0x8d0cc3cf,0xa6a534ca),LL(0x7bc55c38,0xa917d88e),LL(0xc01d22dc,0xba878e3c),L_(0x2cf82f89), LL(0x927b4948,0xfb3f69e6),LL(0xfab01485,0x44185c63),LL(0x586c00a9,0x151ed75b),L_(0x923f3333), + LL(0x72a46aaa,0x519ea031),LL(0x8709198a,0x75f6f740),LL(0x1a09f429,0x0df7057c),L_(0xd41f8134), LL(0x0511f342,0xb8c291d5),LL(0xb5c05e96,0xbd814e25),LL(0xa548461b,0xdfcbbc5c),L_(0x17c27bbb), + LL(0xf029f708,0x3ec594c8),LL(0xe348f36b,0xaacdcc44),LL(0x76cebcbf,0x6e7c6e83),L_(0xb7e69828), LL(0xa2a8a4ff,0xb47c1c29),LL(0x994406d9,0xebdc1ae0),LL(0xed49a01f,0x63ac12b4),L_(0x513ab51d), + LL(0xaa4b3b47,0x7e957ed9),LL(0xa7f34b22,0xe3393b0f),LL(0xde08b775,0x8522afee),L_(0xacd12226), LL(0x5f73d8d2,0x39d26476),LL(0xa513c2b3,0x7e58e0f2),LL(0x213036c0,0xa0e99b0f),L_(0xfb5cf2f2), + LL(0x8a626b92,0xbe1888a3),LL(0xbb87a44b,0xc627fd2d),LL(0xd55c88b6,0x75c0ce87),L_(0xe7f3660a), LL(0x96806d02,0xdce56555),LL(0xc75c5771,0xa0f7c5e9),LL(0x286d7145,0xd7ff4a5f),L_(0xebc158a2), + LL(0x9f5793c9,0xf0bdad25),LL(0xb6481975,0xde77b0dd),LL(0x308c67ea,0x506de99f),L_(0x814dd111), LL(0x9d70efea,0xf8b6908d),LL(0x664f6334,0xa22f9208),LL(0xabb48b09,0x305250b9),L_(0x589eade6), + LL(0x04d51ba4,0x69790978),LL(0x61258755,0x55c72c44),LL(0x340e8617,0xcd9b6116),L_(0xabcb82e4), LL(0x80b0004b,0x8b257151),LL(0x337e5242,0xae7d889b),LL(0xd95f9a8b,0xf7035183),L_(0x7680c71b), + LL(0x96f17da3,0x21e71bcf),LL(0x808dcdbd,0x1dde1b45),LL(0x3f515a1d,0x1f620fc3),L_(0x06bbadac), LL(0xd2651946,0xa7f48789),LL(0x0f2c7fea,0xa9e9b7f9),LL(0x048d20b9,0x2efd58e7),L_(0x44fa3d36), + LL(0x095a8f26,0x6fa2edc8),LL(0x653c44b2,0xad7dc081),LL(0xf2b8fbbb,0x88a7b80f),L_(0x02238eac), LL(0x2ca3a2a6,0x00c09332),LL(0x7cf3c758,0x0d577a60),LL(0x54d150a9,0x1837cf45),L_(0x9354157c), + LL(0xec967632,0x281b5c05),LL(0x5e849910,0x580362ba),LL(0x832277a8,0x702cc617),L_(0x1dbc415a), LL(0x7335bdc9,0x26119f95),LL(0x90031be3,0xa27ab673),LL(0xc7055ffb,0x652580b5),L_(0xe35bf60a), + LL(0x8c6adedb,0xebd0f498),LL(0xe1e7a7db,0x993107cf),LL(0xb75b4edf,0x0232f8ad),L_(0xc245bc0d), LL(0xa2592946,0x48d03da4),LL(0xb9489c19,0xda871c00),LL(0x8ff4aaa7,0x746e818b),L_(0x8df35cfe), +}, +/* digit=14 base_pwr=2^98 */ +{ + LL(0x23800fac,0x8b12655a),LL(0xd7cd8acc,0x66a4179c),LL(0x33023ec9,0x1205108d),L_(0xc1d74575), LL(0xa87c96ad,0x402e3b9a),LL(0x744ac2f7,0xd3dbe7de),LL(0x6b327511,0x5723ac5b),L_(0x244ec59a), + LL(0x7ab235be,0xbede2315),LL(0x536bf0cf,0x5f9a6aaf),LL(0xe6d56464,0x357d2d21),L_(0x58f34a26), LL(0x95a60b41,0x49f4f0e2),LL(0xcfbab99d,0xdca3ea61),LL(0x4c739342,0xa16d4457),L_(0xcea92902), + LL(0xdcc5b89b,0x463327ef),LL(0xb7378179,0xadb9328a),LL(0xbfce0e4d,0x0d62b545),L_(0x1e6b72f2), LL(0x349d4bd6,0xa76133fc),LL(0xdf69c044,0xb4284723),LL(0xf5a817aa,0xf7ca3909),L_(0xe22765b5), + LL(0x0655039f,0xbc44db37),LL(0xc2621a65,0x3d4e983d),LL(0x67d89bbe,0x85435ace),L_(0x803ede1a), LL(0xbe29cd2f,0xed39f9e8),LL(0x00ea11c0,0x55cac6ca),LL(0x3d4406d2,0xde2aeda4),L_(0xe7cb91b3), + LL(0x8fabc04b,0xc39c7ee8),LL(0x26cf4402,0xc619fe58),LL(0x0816fbf5,0xf268e46c),L_(0x9482a940), LL(0x3b1bba20,0xce226865),LL(0x080cefa6,0x4f9ab1d9),LL(0xfcbc42d6,0x13efb949),L_(0x1f067e43), + LL(0xf8119c35,0x75d60569),LL(0x73a4c13b,0xb3a82994),LL(0x063e48b4,0xc7bc1aec),L_(0xb0ab7ce1), LL(0xd626619f,0xa7f182b8),LL(0x4c843004,0x533f72eb),LL(0x1d2aa02e,0x69953a2d),L_(0x0de00959), + LL(0x91516c9c,0x0913a322),LL(0x1648e50c,0xa0961a2f),LL(0xa7d10cb1,0x06e4869d),L_(0x9b4a803d), LL(0xdc8793e7,0x9f17e3a2),LL(0x298436f6,0x06fc6243),LL(0xe925133a,0xb7622bbe),L_(0x70b9a2e8), + LL(0x84e7489b,0xc981ba62),LL(0x5e1a4306,0x792db847),LL(0xa77e3ed1,0xf4b6792a),L_(0x87bb9d13), LL(0xed208d23,0xfc7dc4a1),LL(0xcc352f71,0x87c2b023),LL(0x095b0ea7,0x88016bc4),L_(0xc90a59de), + LL(0x59119397,0x4b8755e3),LL(0xfbec0058,0x19aa2513),LL(0x554f2e58,0x1d544a58),L_(0xb7908e98), LL(0x1d4dba6f,0xcc8e5a35),LL(0x4e98597c,0x4036ed62),LL(0xf70ca123,0x0fd793e9),L_(0x5567a702), + LL(0x269da085,0x2503b0ce),LL(0x06549f68,0xb3025472),LL(0xeb061c62,0xfe9ae248),L_(0xd6d45a68), LL(0xd2906b25,0xeaa5d6f9),LL(0x58bbd291,0xaf62d50d),LL(0xc2b140a8,0x4dbbcce0),L_(0x86272e95), + LL(0xa7cf5775,0x3293ff9c),LL(0x7b80a205,0x1f10127d),LL(0x739c1785,0xc01f9711),L_(0xa80be8ca), LL(0x68df6a04,0x50fcfcf4),LL(0x3da76c49,0x340f3712),LL(0x53cced84,0xba988480),L_(0x89efc853), + LL(0xed47d3b8,0x6051e486),LL(0x27be6887,0x2e29a4c3),LL(0xfb40723c,0x77353f89),L_(0x3a8109a1), LL(0xf4eea8fe,0x672e091a),LL(0xaaf35dba,0xef34d20e),LL(0x96e8e707,0x2e5db0f7),L_(0x259349f1), + LL(0x089e80e9,0x259bf71c),LL(0x6e48997b,0xcd52c81c),LL(0x9b67f891,0x106aa44c),L_(0x2b828bad), LL(0x84cca320,0x492159ac),LL(0x860638da,0x99fc08c2),LL(0x9a577c2f,0xde194e1f),L_(0x4c02a837), + LL(0x1a869e95,0xac2dd0d1),LL(0x40fe4710,0x0d98495d),LL(0x926fca45,0x230d5720),L_(0xc0032a5e), LL(0xda97aa70,0xe0f36b43),LL(0xdbdd8384,0xbdee74a8),LL(0x9f4f1e97,0x616d2e77),L_(0x285c8908), + LL(0x4ce190f7,0xe408ab06),LL(0x4de68c46,0xaec26bc8),LL(0xea864657,0xa9c6ad27),L_(0x3b9d1eb1), LL(0xab862944,0x327c78b5),LL(0x2af9ede8,0x751ec6f1),LL(0x18bed81e,0x1c240eff),L_(0x17dc571f), + LL(0x291c3273,0x766b4a0b),LL(0x3a97f292,0x6e56a096),LL(0xafb97a2d,0xe3cbbbe3),L_(0x3a6fadc9), LL(0x2369de35,0x4c78c1e4),LL(0x6d3e5c1b,0x698e8f8f),LL(0x14e1a4eb,0xa87a74e8),L_(0xbb410055), + LL(0x0fb37060,0xcbf75eae),LL(0x323f451f,0x58dee01f),LL(0x1a149a9e,0x67ff1a09),L_(0xb35d39d2), LL(0xf63b9865,0x7e111af7),LL(0x02e5fd5f,0xf6f23814),LL(0xc6967a32,0x682916a3),L_(0x5d411958), + LL(0x53aa5942,0x7dc26ca6),LL(0xec5ae21b,0x3a7f3858),LL(0x4c6d1720,0x87d2b98e),L_(0x6b3c9448), LL(0x379e4e5d,0xc6bec0ca),LL(0x7de28c5b,0x69db9b01),LL(0x4e3aaa80,0x4f1a3e1f),L_(0x31e76223), + LL(0x948cf979,0xe1a7d9ed),LL(0x32518804,0xabd060c2),LL(0xe2c8026f,0xd02845ba),L_(0x60dcd816), LL(0xd2fcd20d,0x9fc8ca8b),LL(0x67ac75bd,0x0b403597),LL(0xfa897231,0xd2ef4cab),L_(0xba971f13), + LL(0xc9608224,0xd3a9e728),LL(0x21de382a,0x9fcf28b0),LL(0x23db313b,0xe226f1f0),L_(0x9e002526), LL(0xdd26c698,0xa3a7479f),LL(0x8170903e,0x4871cbd5),LL(0xc0ad648f,0x30414c9b),L_(0x3c9d51d0), + LL(0xe22f8df5,0xdef38d08),LL(0xd456476e,0xfacc23b0),LL(0x7b3f5e97,0x00fb9a2e),L_(0x6c5a7311), LL(0xa6cbdc37,0x87cb811c),LL(0x82204464,0x6214d9f5),LL(0x06806fd9,0x5ba8f23b),L_(0x6d93ebe3), + LL(0x86e33c2d,0x23423298),LL(0xa44ec00f,0xf6e30434),LL(0xc242970e,0xbf53cf6b),L_(0x3a4d00db), LL(0x84f61eea,0x8140c42b),LL(0x5afabd85,0xe917c409),LL(0x70d1d6e0,0x95de99fd),L_(0xc2091589), + LL(0x396774db,0xc249bd59),LL(0x93024ffe,0x24f81111),LL(0xfe365e8c,0xdcd6e625),L_(0x35c6e777), LL(0xe01d308d,0x65e9e9bf),LL(0x4a7452da,0x5cab99f6),LL(0x441a1645,0xf759c45d),L_(0x97ffca58), + LL(0x1d7326e2,0x52fad307),LL(0x04705ea5,0x3aa389bc),LL(0x9cfd5023,0x3feb82b3),L_(0x7a752b31), LL(0x4a98e745,0x22a198b9),LL(0x303fc591,0x5fd22f5c),LL(0x95dcd425,0xe40ca457),L_(0xfc4759f2), + LL(0x0baec1d1,0x899726d2),LL(0x9ccc4295,0x9503a4a3),LL(0x8faff5a9,0x735bcc96),L_(0x4938831d), LL(0x294b5dc9,0x27659e3e),LL(0xcf8e86f6,0xb11539c4),LL(0xf82de3fd,0xdafe9d6c),L_(0x93a3f5c4), + LL(0xb712f886,0x4ba7d548),LL(0xf1d50673,0x627d0815),LL(0xebc722ce,0x4e21b679),L_(0xd4fcebfe), LL(0x3d488792,0x654d615e),LL(0xa956f701,0x38e30e59),LL(0xbe0c2b4a,0xd07392c8),L_(0xd0eefb50), + LL(0x91582e11,0x245a5f24),LL(0x83071bb8,0x1d74fe95),LL(0x32fc2d8c,0x8d68efa6),L_(0x3c6bb1ba), LL(0xb73ed81b,0xd7c363e3),LL(0x4814d6e2,0x4b18aee3),LL(0x8563cd35,0xf296ee0a),L_(0x39c2c1ab), + LL(0x6115b5df,0x12e07056),LL(0x27e6f810,0xab33f925),LL(0xdadb7f44,0x9611fc0f),L_(0x21632fa8), LL(0x135db674,0x0014d0b2),LL(0x90282284,0x34df44bd),LL(0x498b5aab,0x3b9bf0a3),L_(0x83adadb8), + LL(0x8b47b85a,0x7583945e),LL(0x8ca7a054,0xe1eb9f0d),LL(0xdc6bf5ac,0x7f50540d),L_(0x4929ffcf), LL(0x7dcd0fa3,0xd6e884e8),LL(0x547fd35b,0x0fda3c41),LL(0x42d34849,0x156b5074),L_(0x2b73e553), + LL(0x9db1951e,0x6f1aaef3),LL(0xbba949f2,0xe07d36c2),LL(0xaf67c45f,0x598114fb),L_(0x0be51018), LL(0x18833890,0xb079f72e),LL(0x43a0dd65,0xee4b11df),LL(0x20bd4ca5,0x9dd3598b),L_(0xb6e5a89d), + LL(0x2f515e07,0x0a130c64),LL(0x53781704,0x21d929ed),LL(0xed92766f,0x2370b592),L_(0xb01a78da), LL(0x461ee105,0x11fd327c),LL(0xe5630841,0xaac6ed8a),LL(0xaad362f8,0xe2c21338),L_(0xd9bef88b), + LL(0x7702c305,0x90cc96b4),LL(0x284ea1e9,0xec9108c9),LL(0xea844162,0x79d4d2e8),L_(0x5fc96bdc), LL(0x9b799340,0x9b1c7dfd),LL(0x6ae02844,0xf8a798d5),LL(0x4a0a1e22,0x7439503e),L_(0x0838edf3), + LL(0x25112357,0x41a91a95),LL(0x9794aee7,0x12be9cf2),LL(0xa09c00f4,0x87b213f6),L_(0x588e9419), LL(0x3cc23f7d,0xa44cb6f8),LL(0x9ee73263,0xda068194),LL(0x999bf774,0xe19968dc),L_(0x4f9d93f1), + LL(0xa3bbb5a7,0xe07280d1),LL(0xfdb5f5ba,0xf0df9657),LL(0xbd50dd87,0x4715fc8b),L_(0xdf379027), LL(0xf3d02095,0x64a96f25),LL(0xeb119764,0x661822c2),LL(0x8b298e54,0xd8ec764d),L_(0x6f6ea251), + LL(0xcef9767b,0x3dfab91a),LL(0x7c6e866c,0x85522367),LL(0xdbf3bb79,0x79b36fcb),L_(0xbeb97ad5), LL(0x05fa18c2,0xdbafd1f2),LL(0x64259c17,0xdd549dff),LL(0x83c7657f,0xd82dc78f),L_(0x8cf18ed9), + LL(0xad65298a,0x58b32f27),LL(0x2131395b,0x976af3d3),LL(0x4b505f2e,0x64c1cfc3),L_(0xca10d83b), LL(0x59913cfe,0xbab37657),LL(0x1414d1f4,0xf066519c),LL(0x8887e569,0x3d975cff),L_(0x0c251f08), + LL(0x425ddccc,0xd5368e78),LL(0x93068700,0x12b23c3e),LL(0x33076aa1,0x4e626b1c),L_(0x4e14ead6), LL(0x23f1e78d,0xe86079cd),LL(0xd274538d,0x908738cd),LL(0x244ce9b9,0xe51d741d),L_(0xb4e734ed), + LL(0xb2d63b64,0xf0e2cdd8),LL(0x0675f17a,0x90f8143d),LL(0x75829dd6,0x2d23a43f),L_(0x298a7f8f), LL(0x47ca8668,0xa526104e),LL(0x4e9c496b,0x85467645),LL(0xb0f0e275,0xafc61eeb),L_(0xa1acd83f), + LL(0x7ee79f81,0xf367484b),LL(0xafa3db78,0xbfc8c257),LL(0x7fe068cc,0x7c8262c2),L_(0x784b4654), LL(0xc96069f1,0xd5c6e6e1),LL(0xb65beee4,0xd8f70ecb),LL(0x24073300,0xc55f13ed),L_(0xa8d09ea4), + LL(0x82d22a52,0xe38ffddd),LL(0x7bb613d6,0x03df60ca),LL(0xa964fa9d,0x9bfb3890),L_(0xcc4b84d8), LL(0x96707a64,0xf9f9338a),LL(0x110b10fd,0x5d9e9ac0),LL(0x0acd29e1,0x854609dd),L_(0x11a3136a), + LL(0x84bcad3b,0x9a527f75),LL(0x28be328f,0xc16f294f),LL(0xdaf38a99,0xfdebcc4a),L_(0x4d614ff9), LL(0x8a1d04d1,0x760244a9),LL(0x9fb5fa88,0x97af96d1),LL(0x79ecc48b,0xebf508a7),L_(0x63ac8146), + LL(0xa9338a44,0x4ccbe898),LL(0xe952b759,0xf79ce658),LL(0xa7f6528b,0x0efdd8f6),L_(0x59bff1cf), LL(0x7002f54b,0x859eefc9),LL(0xb915b857,0x3146204d),LL(0x25b282fd,0x9e570ac1),L_(0xf3c10eb4), + LL(0x50227a06,0x96cc2182),LL(0xd7972029,0xed491383),LL(0xa5e9dc33,0xef480409),L_(0x8303786c), LL(0x48fc5413,0x7b84b8b7),LL(0xa8f1bf3e,0x22c04cfa),LL(0xa2e99567,0x5b8dc942),L_(0x089858bd), + LL(0xb80a1877,0x7a0670f5),LL(0xf5def779,0x99c0dcfe),LL(0xceba003e,0x737b1478),L_(0xba188761), LL(0x88ff60f8,0xa8a91790),LL(0xf962acd7,0xd0c75bfc),LL(0x8ebfd0f8,0xaf779b34),L_(0xe9624be3), + LL(0x62fec3e2,0xd969780a),LL(0x9daaebb5,0xafc171d8),LL(0x647f2bf8,0x16ea68f8),L_(0xfbfe21fa), LL(0x83f12d03,0xb63e2588),LL(0xe8295dd0,0xb69cbad8),LL(0x08b7948f,0x1a740ee4),L_(0xbe240b54), + LL(0xd3ad093e,0x97774b8e),LL(0xa107d5a6,0x1438ff5a),LL(0x90acd42f,0x0c55b2c6),L_(0xb1008301), LL(0xf3524dc1,0x1f19f151),LL(0x28b7f8e9,0xc5b4f62c),LL(0xe1899b7c,0x7240d638),L_(0x249ecebf), + LL(0xf9ad69ab,0x9a96c604),LL(0x1e0ee825,0xd851c4f0),LL(0x816d9822,0xf895d68e),L_(0x14e58e83), LL(0x914c2f30,0xdd941975),LL(0xd4525a24,0x1765af51),LL(0x9bdc107c,0x770ebc8f),L_(0x908f73aa), + LL(0xe5bd052d,0x70e13a7c),LL(0xe46ccc09,0x0c4379c5),LL(0x0121f8bc,0x71124e71),L_(0x38ba16ad), LL(0xe43a3a55,0xd9134113),LL(0x23d0c031,0xbbbe7daf),LL(0x3a1c2b24,0x3950f72a),L_(0x78296ab5), + LL(0xa71550b1,0xdab1491f),LL(0xdd5b8553,0x12caa6e8),LL(0xb08da5e0,0xf505a6eb),L_(0x95e70a66), LL(0x79842177,0xcf510150),LL(0x68ab8f60,0xafdce053),LL(0x87a8f6f2,0x8158b7f4),L_(0xc0b75050), + LL(0xe2eb279e,0x6af97a49),LL(0x86441bbb,0xd6c5159d),LL(0xcf78aa46,0x572c05de),L_(0xbc71b5c6), LL(0x0bf9ef5a,0x5b7148bf),LL(0x97df4c43,0xbe4b9631),LL(0x101cdc13,0x7ed1a6b8),L_(0xafbd7e88), + LL(0xb580b670,0xc6feaf5c),LL(0x3ea251a2,0xa5cb0934),LL(0x02cfec16,0xc2e35b6b),L_(0x0fdebc9c), LL(0x6e09365e,0x5b4d0976),LL(0x54866bd4,0x405714ef),LL(0x892fc650,0x75be4796),L_(0x224babf9), + LL(0xe024a3b6,0xffdc8787),LL(0xd74e707e,0xbb8a1baa),LL(0x16ab4631,0xf8688223),L_(0x0cf2f805), LL(0x09c1692c,0x0dd7a6f9),LL(0x42b206a1,0x3a94d17c),LL(0x784e210d,0x58b82c4e),L_(0xd0f07cde), + LL(0x4df5c033,0xa0b09f2e),LL(0x0f337455,0x8ad59d7f),LL(0x92ef323e,0x9dfd994c),L_(0x8a562d14), LL(0x9c294c52,0x0784cc2a),LL(0x06b511cb,0xa7c397c2),LL(0x9d3b5c1e,0x5217298a),L_(0x9becab02), + LL(0x5fffdecc,0x15425166),LL(0xf204b46f,0x3ad2c54a),LL(0x29b56266,0xcbfffc28),L_(0xe2bc5353), LL(0xb4051ff4,0xe925b548),LL(0x91eefed0,0xd94421d2),LL(0x61d3c552,0x0c2f7520),L_(0xd3c090b5), + LL(0x70ed4769,0xfa81c450),LL(0x0d59a33e,0xbb762aa7),LL(0x72d6d774,0x7b08eae2),L_(0x92885c49), LL(0x1432d1e2,0xc28ea0a8),LL(0x4e6a192a,0x57bf54b2),LL(0x163d3f1c,0x996fa17f),L_(0x8242af09), + LL(0xb5fbaaff,0x53d923d7),LL(0x4aa1aad8,0x8aeef401),LL(0xc59594fb,0xd7267ef5),L_(0xba7c28dc), LL(0xede0bedf,0xb019e637),LL(0xbffc5a3c,0x1be98aa8),LL(0xb69b47a8,0xd848486b),L_(0xa1758b69), + LL(0x95f000bf,0xc417bef7),LL(0x3f4d4f15,0xef4c07bd),LL(0x74d082e1,0xc8cd53f2),L_(0x2210c27d), LL(0xfb026021,0xaab1ecd2),LL(0xf4960e70,0x47b08bcf),LL(0xf6ef41f3,0x386f4e41),L_(0x3d017b94), + LL(0x806b3be3,0xd292e87d),LL(0xbe3b4c15,0xb316049f),LL(0x08cba3d2,0x5a2daba1),L_(0xf68b1a9e), LL(0xff881fa0,0x4da3f147),LL(0x91960fa2,0x8b3ea4f6),LL(0x7f14ef98,0x931e6a5d),L_(0x291615c1), + LL(0xe8b9b590,0x3a4e460d),LL(0xc7e1d53c,0x410540d3),LL(0xba715bf7,0x9faf6e57),L_(0xf612eab8), LL(0x13d8563c,0x83a48da3),LL(0x97d11aa7,0xbc420fcd),LL(0x54fecfb6,0x8ea8b02c),L_(0xa72e89c3), + LL(0x9f084f51,0x15421912),LL(0x19ce8a03,0x69e1810f),LL(0xa93a4b35,0x66d7f71e),L_(0x33703d01), LL(0x9b5d9b15,0x85c36f3f),LL(0xfb68061e,0xf837f0c3),LL(0x1acc3982,0x4526c2f7),L_(0x8aadc35e), + LL(0x91bdbe3e,0x5c462b80),LL(0x46606493,0x16f4a53a),LL(0x2452c556,0xf2ad5a49),L_(0x9ecfe87c), LL(0x961c76d7,0x92031bde),LL(0xd10c3c2b,0x43fc1b45),LL(0xba2806c7,0x7ea59306),L_(0x6d69a1f2), + LL(0xaf7fedc2,0x44f370ac),LL(0x575c1c1e,0x7fa03a08),LL(0x366491a9,0x65caef0d),L_(0x1d17b3b8), LL(0x3a469596,0x6dc5f00f),LL(0xf2e830cb,0x87cfba82),LL(0xebb7f488,0x1fc8150f),L_(0x74afed78), + LL(0x70a3e6c6,0xf79b97ae),LL(0xcd939876,0x0c92525a),LL(0xfd612292,0x1f877d7a),L_(0xf5237757), LL(0x2b249ae1,0x78a6422a),LL(0x2855f66b,0x0659a5e3),LL(0x12934c87,0x7a1fca11),L_(0xdfe80915), + LL(0xe4d85de7,0x0b0acedc),LL(0x21b8a084,0x6091bc19),LL(0x04526cb6,0xac1ce3ea),L_(0xd10a9ec7), LL(0xb54024a7,0x57196c24),LL(0xb55d4b5d,0x42e04321),LL(0x1b1d5d2a,0xbb31992a),L_(0xdb403b45), +}, +/* digit=15 base_pwr=2^105 */ +{ + LL(0x5f4861c4,0x9a1b156b),LL(0xa1ffe24b,0x45b930e9),LL(0xbeb5a955,0x29358e8a),L_(0x4b8943f1), LL(0x7923968c,0x127f5983),LL(0xe58cb43e,0xf5aef19b),LL(0x22713358,0x0329be36),L_(0x552fb6a1), + LL(0x842632cf,0xd5e766ad),LL(0x947d9fff,0xbab1e908),LL(0xdb59f894,0xa103e5b9),L_(0x68372195), LL(0xc4c67973,0x2cd3e628),LL(0x146f4450,0xe2690b88),LL(0x787c5ec2,0x153e0f49),L_(0x1f9bb175), + LL(0x01d9f5c7,0x9acab18d),LL(0xbfb21488,0x96fbb4bb),LL(0x43543f3e,0x19e99acc),L_(0x81d541e2), LL(0x4d4f6943,0x2373babd),LL(0x96c02c64,0x0c662932),LL(0xdd524336,0x60570f05),L_(0x6ee3ed44), + LL(0x98a32e88,0x9cbcb156),LL(0xda250980,0x00a72b87),LL(0x85f705a8,0xb60a7751),L_(0xa9f85b0d), LL(0x3ef7d9b7,0xc5078f17),LL(0x1fa64414,0x20110dec),LL(0xb521d60e,0x324aa16f),L_(0xcf1c2382), + LL(0x7053cd15,0xc75d38b5),LL(0x9347ce64,0xff26fb52),LL(0xfa6fa1ea,0x89904b74),L_(0xab858e49), LL(0xb9800606,0x0d1c07cb),LL(0x0b99d5c5,0x7605e424),LL(0xe982130b,0xc4ed9c66),L_(0x45e55121), + LL(0xb3f704f5,0x2c1c043d),LL(0xf3189b49,0x3e68dbd7),LL(0xbe354e8d,0xa6382386),L_(0xf9ee8ba0), LL(0x26378a41,0x04646666),LL(0x709ded81,0x88fbb7ce),LL(0xc603785d,0xb1f815a2),L_(0x5bec9b88), + LL(0x43fe75c9,0x9a29229c),LL(0x052996cb,0x9b1a9170),LL(0x7d4b9fc3,0x133dd4ca),L_(0x1176baf5), LL(0x53ac8647,0x6dcfad8d),LL(0xcfedffdb,0xe4115707),LL(0x6ff11203,0xbc9689c3),L_(0x8c720cdb), + LL(0xd81789dd,0x7b457d74),LL(0x127b60ac,0x5a87275f),LL(0x096f997d,0x5d4f53a6),L_(0x554f3104), LL(0xc86a176b,0x0c5247ba),LL(0xf8205ea1,0xb389c0f4),LL(0x325d67b1,0xdc511ddb),L_(0xc2e38d56), + LL(0xa0dc02ad,0xca8b2e56),LL(0xac05bb21,0x59515fbd),LL(0x152ad207,0xa0ec8c40),L_(0x1f1977c8), LL(0x990ce0e9,0x094829b8),LL(0x110ed8d5,0xdb93ce78),LL(0x53af9d31,0x980f663f),L_(0x50f753ac), + LL(0x0afcea41,0x1410da31),LL(0x91d6a357,0xc4dd3dce),LL(0xefc18ea4,0xc844e9d6),L_(0x599c3b4e), LL(0xb2b1ab81,0x89238185),LL(0x716b0afc,0xc007c089),LL(0xcc238164,0x10df0d3c),L_(0x7d888920), + LL(0xd9103e74,0xec1702f8),LL(0x0184b65e,0x67e64440),LL(0x4b15a941,0xaf4f5d2e),L_(0x334e5ea3), LL(0x82fa4171,0xbc874d39),LL(0x3cf9db40,0x63f968d3),LL(0x7b71afa1,0xc6c567c6),L_(0x74793b87), + LL(0x1b353d4c,0xfbfeb69e),LL(0x001b5f0d,0xaecee209),LL(0xdaae33fb,0xc8d8f5e9),L_(0x76be05c8), LL(0x4e52dc92,0x87741a88),LL(0xd9f61920,0xa5009003),LL(0x72c650f3,0xc505e3b9),L_(0xb83f81c5), + LL(0x2c204e5b,0x9f6af3aa),LL(0xd6b594bc,0x0423f2f8),LL(0xa67cf879,0xe29129c8),L_(0x99f505a1), LL(0x47ebe4a1,0xc220fd1a),LL(0xe7a97ad3,0xa34e4a65),LL(0x1fd2a0dc,0x7c1fb322),L_(0xa75c8fc2), + LL(0xcad10b12,0xca3e4b04),LL(0xdda6763a,0x00ce7df6),LL(0x9c84d269,0xa1f31fb1),L_(0xe404dde9), LL(0x45e72361,0xf80ab752),LL(0x300ca2ea,0x38935f38),LL(0x36966819,0xebd1b2a6),L_(0x758de4e7), + LL(0x28dbeb1e,0x4f41e2ad),LL(0x153889e3,0x1c0782c2),LL(0x60b0cb1e,0xb609d7c2),L_(0x9a85e0f0), LL(0x403c7b8f,0x3d308948),LL(0xe5878d4e,0xd8503865),LL(0x0326fd01,0xd472fecf),L_(0x51c81464), + LL(0x98673101,0x35c900a8),LL(0x2437a09e,0xdb5df675),LL(0x63415d28,0x6b676a73),L_(0x5a4ddb3f), LL(0x3e21afd1,0xa33929eb),LL(0x332e8fd4,0x717e43a3),LL(0x6bf414b5,0x9c7a3c47),L_(0xfb327e1b), + LL(0xf9ea4385,0x6b738d34),LL(0x3b9138c4,0xac317163),LL(0xd197efee,0x2438948a),L_(0x2b8a07e9), LL(0xebcc3f74,0x1fc5f207),LL(0xdfbd84a8,0x9fdbdbb5),LL(0xddc39008,0x663a5b4e),L_(0x9956763f), + LL(0xa670dec5,0xaeb71ef5),LL(0x989b6598,0xa4973f38),LL(0xf981cb1a,0xe1b496e0),L_(0x187e0a5e), LL(0x8185f4e5,0xdcd9014d),LL(0xce47e120,0x01c7c910),LL(0x33bfde50,0x0a1d6d42),L_(0x3059677d), + LL(0x0c74186d,0xf93c6e65),LL(0x2d2eb0b3,0x5c4b9d3f),LL(0x9b7f55b4,0x62194b5f),L_(0xf492843a), LL(0x07af0f2d,0xead94ac6),LL(0x344ba8f7,0xfc351f8f),LL(0x6a02d6dc,0x96c34c95),L_(0x0f698b8c), + LL(0x06eb2a9f,0xb29f094a),LL(0xe4c8cfbd,0x8cc86e57),LL(0x40844e37,0x2f8aef05),L_(0x7588271e), LL(0x864a5330,0x8ade6f97),LL(0xf9d12d2c,0x78f8e22f),LL(0xadc126ed,0x8c70fd00),L_(0xc12911d9), + LL(0xecdb5c0f,0x3c35a3f6),LL(0x0e784293,0xe0f4235e),LL(0x49ad3737,0x162c0cf0),L_(0x9e096565), LL(0x6d262e33,0x3611d970),LL(0x2ea4dc97,0x56f9b02c),LL(0x569e559b,0xd700ade3),L_(0x6445516e), + LL(0xab2f2394,0xf45846e5),LL(0x405bf6e0,0xaa46d2c3),LL(0x1222d842,0xf5f64176),L_(0xedb70efc), LL(0x1b067c96,0xb8990097),LL(0xb7f1c7b7,0xd2f67212),LL(0xe2b50ce3,0x7ac5ce8c),L_(0x05c93598), + LL(0x754843ce,0x77e241fc),LL(0x1ab02032,0xa0d9cbeb),LL(0x55cb5b7b,0x98136c68),L_(0xd09ef0e2), LL(0xef262224,0x523e661c),LL(0x87906701,0x3066156f),LL(0x3e7c7958,0x5ea1ccfb),L_(0xb06fbe44), + LL(0x736e6685,0xa762a548),LL(0x9218789a,0x4ea95a8e),LL(0xa90f1dd7,0xf1a7ecac),L_(0xbc8d97ba), LL(0xdb175bac,0x176f58b3),LL(0x16633eeb,0x66d2f2dc),LL(0xf30e4d83,0x047e3e70),L_(0xfad00adb), + LL(0x2523bfbf,0xdb1ab97e),LL(0x632d110c,0xa7f43904),LL(0xed30067f,0xe0063138),L_(0xcfac77b5), LL(0x0fe3afe1,0xef7cd5f9),LL(0x5720d9f1,0xbceebc62),LL(0x7475ff1c,0x37aef5f2),L_(0xcea553ab), + LL(0x21d9dd89,0xfe303f84),LL(0xe84277eb,0x403a10c6),LL(0xdc250e4e,0xcf99ada4),L_(0x260cc2ff), LL(0x16293017,0xdf77e00f),LL(0x429f4cad,0x017aea98),LL(0xb51701db,0xb9a51f04),L_(0xdf5024fa), + LL(0xaefa0a3f,0x3c6160d1),LL(0x89d2a9d2,0xb6871dd0),LL(0xcb336211,0x30015933),L_(0x4ddcddf5), LL(0x87d9707b,0x18fd4882),LL(0x96d9668a,0xf138a306),LL(0xeeb3123d,0xce3f684e),L_(0x9fa39b7f), + LL(0xdb674dfe,0xd843e892),LL(0xe36b5375,0xedfd1c18),LL(0xc7988b87,0x3bdd2895),L_(0xe52a31a1), LL(0x09bd2af3,0x294ff4e2),LL(0xccd1887d,0x2ad4535e),LL(0x2fc2c26d,0x1ebd13d4),L_(0x647f4987), + LL(0x0bdfba7c,0x94e42bd5),LL(0xae5c8c95,0x512c1be6),LL(0x10be5ddc,0x6bc6d702),L_(0xfd7b27ee), LL(0xd21e8821,0x636561c6),LL(0x170a05c4,0xe4ef4a34),LL(0xda161673,0x3d69a37a),L_(0xf40d8e35), + LL(0xc206f7b1,0xa0f11bca),LL(0x9195ecf0,0x23eeb3f7),LL(0xe6050ed7,0x98889901),L_(0x07aecdeb), LL(0xc76ba3ed,0x4edc3ee6),LL(0x6b353524,0xaab7f4a8),LL(0xc6e7d7b5,0xf6442856),L_(0xd31e141d), + LL(0x5b3638e7,0xd78b290a),LL(0xd885546c,0x77aef8c9),LL(0xfac422d4,0x0feb906d),L_(0x92b8b923), LL(0xdc0b9002,0x331327a2),LL(0x224f79a8,0x5d146cb4),LL(0x6fb5da21,0x3f887bb5),L_(0xc43cfdf7), + LL(0xb80d181d,0x2be6d108),LL(0x3c8b1f59,0xb342ddc9),LL(0x7004d85b,0xbe7db7ec),L_(0xec029270), LL(0xc1d565db,0xb82063a3),LL(0xbb014507,0xaa90beff),LL(0xbf4ed46e,0x3a15f177),L_(0xf45b367f), + LL(0x49eb74f9,0x146e10ea),LL(0x142ba6f4,0x1256c788),LL(0x346ea27d,0x3f761b6a),L_(0x80f4ab84), LL(0x4ec42ca8,0x150f9515),LL(0x566df38a,0xda268f4d),LL(0xe31f64a6,0xe4706b89),L_(0x95e5b78e), + LL(0x0b4a9c49,0xc8ae2f2d),LL(0xd939334e,0x58cc7b3f),LL(0x3e9cb1a4,0x31facd26),L_(0xcce403b2), LL(0x3b34d497,0xd018498f),LL(0xc5995b75,0xcbad3ed8),LL(0xa8867a65,0x0fb7952a),L_(0xef5960a9), + LL(0xe832b584,0xa4bc1e98),LL(0x3d7bc503,0x4ca3a484),LL(0x2da2b54a,0xbbcd02db),L_(0xf9f27d63), LL(0xd59bf324,0x02c44a56),LL(0x3ff8f28b,0x366f36f1),LL(0x0dbb411d,0xf30d32a2),L_(0x31dc1c76), + LL(0x52b2e707,0x19599554),LL(0x83cc5035,0xe133dbc1),LL(0x659be3d5,0x90ee1c4f),L_(0xfff40ea6), LL(0x70cd0820,0xba72fb3b),LL(0xab738bd2,0xff5688ad),LL(0x3ae40925,0xb751810a),L_(0x1c7232bb), + LL(0x071a0770,0x01bb4b01),LL(0xe2be9f54,0xd05cc7e3),LL(0xc3b59e15,0x14e1dc86),L_(0x84ea67cd), LL(0x1ee66ae0,0x743d3ae3),LL(0x9ac0bdc1,0x75487d37),LL(0xb272a24e,0x18ad2d25),L_(0x449e4286), + LL(0x69935fe7,0xefec1497),LL(0xee46b002,0xb3e61f3f),LL(0x1edeaf36,0x2b4abccf),L_(0x1302e388), LL(0x6554f5f3,0x0fe91c0b),LL(0x2afabbc7,0x613c238b),LL(0xf8ba68b0,0x0b0ae516),L_(0x1941e2a5), + LL(0xbf0bc612,0x9c608ede),LL(0x5d82d268,0xc0f244de),LL(0x0b0057eb,0xd601464b),L_(0x12bdb6e6), LL(0x825a361c,0x4ff78a4b),LL(0xb0ac16af,0x6aed48bf),LL(0x1ae6fb3a,0xba8a9318),L_(0x06a955b7), + LL(0x09e379b5,0x9e14e886),LL(0xc7c1e466,0x2fb4d3e2),LL(0x75711ae3,0x036fce90),L_(0x780c6067), LL(0x2542f70f,0xa4a903d2),LL(0x08ed6419,0x95852fe8),LL(0x637b247b,0x012141cc),L_(0x7a1653d5), + LL(0x9af74289,0x41ab3d86),LL(0xaaab79ee,0x29c26aaf),LL(0xcccef77c,0xea5535b0),L_(0xde9a50c2), LL(0xc11f99b2,0xb7f340c5),LL(0x0418cc51,0xfc229594),LL(0x8f5c0876,0xf20b5874),L_(0x79b716cf), + LL(0xba865a56,0x6fae5df7),LL(0x3ec9a3f7,0x2fe39b65),LL(0x8329316a,0xc07b24cb),L_(0xb74832f6), LL(0xd09a26ea,0x1af7d473),LL(0x95d78e89,0x375af56e),LL(0x62993ad0,0x957236cb),L_(0x20cfce9e), + LL(0x3e2c48a4,0x427f5e06),LL(0xe42acf84,0x25837ce1),LL(0xcae65a88,0x966e8b1e),L_(0x2aeaa288), LL(0x9e77f90b,0x306bd895),LL(0x4c375d76,0xd4ed09b8),LL(0xa2585522,0x47a8a9ab),L_(0x875a22b3), + LL(0x1e6ed175,0x8c957fe6),LL(0xcd21f453,0x54818bf4),LL(0x2495a8a9,0xdd977b92),L_(0x2e72900a), LL(0x05a5967e,0x2647b9d2),LL(0x92967328,0x53df8e0b),LL(0x496697af,0x5247a639),L_(0xa66c4dd0), + LL(0x8424e440,0x682c252e),LL(0x6fcf171a,0x0bb6bcff),LL(0x17581f41,0x040d50d1),L_(0x84837a70), LL(0xd070dc3b,0x686cbe64),LL(0x303cfa74,0x3485fd98),LL(0xdfd534bf,0xf0aec283),L_(0x6e4f877a), + LL(0x4458831e,0xbfa63a77),LL(0xa0bbdaf7,0xe6ad6640),LL(0xeffc865e,0xa2922966),L_(0x3c188024), LL(0xae34f5ff,0xec39a211),LL(0xc033bb37,0x5d5de65c),LL(0x1ffcbe06,0xae549602),L_(0x0aa2449f), + LL(0xf7d9eb1b,0x008bf1c7),LL(0x17af13bf,0xdce3e9ef),LL(0x7341adb2,0xe6aba225),L_(0x3fdc83ba), LL(0x52473c29,0x58df1458),LL(0x8e82cc6d,0xcbca23e4),LL(0x64a68f69,0xdc93a2bf),L_(0x402344e1), + LL(0x12f4285e,0x289cd434),LL(0x0df1904f,0x2614d391),LL(0x5132c5c4,0xd24f8d84),L_(0xd529b441), LL(0xb496fb70,0x945d4d6b),LL(0xb4fe1324,0x64d58e9f),LL(0x226ba20e,0x78fadda9),L_(0xf5638c8c), + LL(0x15c734f0,0x71f5ab8b),LL(0x7c5f3148,0xd644b74c),LL(0x8a5a08f7,0x5b9eaf3f),L_(0xcd76894b), LL(0x40783f5a,0xfc35396c),LL(0x5e5b4f20,0xdf4e44eb),LL(0x698ace20,0x85de5335),L_(0x32cc1f5f), + LL(0x4ca6d829,0xc01805fa),LL(0x69a66ff0,0x9b659d62),LL(0xb0bfec7f,0x21ba53df),L_(0x41d8db1a), LL(0x3ca3ffdc,0xaaee16f0),LL(0xac0d7259,0x2829c022),LL(0x8266c12f,0x570a8b63),L_(0x347c6b96), + LL(0x27c7435f,0xc2862c3c),LL(0xd60a5c54,0x577136cb),LL(0xeb454a5e,0x41bf3bc1),L_(0x809d3205), LL(0xd5ff2a5d,0xce2b3472),LL(0x201d2be5,0xabfed878),LL(0xdae3ae7c,0x076254af),L_(0x898c8347), + LL(0xfbee92e1,0x288eeccf),LL(0x6cb1db6b,0x9cfb5c57),LL(0x059c298f,0x53598af7),L_(0x7eda1da0), LL(0x888186af,0x3084705f),LL(0x2f592293,0x86f1d164),LL(0xc59e084e,0xa44fd074),L_(0x3a1664aa), + LL(0xd2343b35,0xa40a6bf8),LL(0xad9e07a1,0x798ceee2),LL(0x27b865e6,0xe4570eb1),L_(0x148877c2), LL(0x8f87d685,0xf0ce5967),LL(0x66f1b39c,0x66b1f5c4),LL(0xcbcc053f,0x2f8869b5),L_(0x9fa693dd), + LL(0x0d7b69df,0x17f0c4dc),LL(0x135a9bac,0xa20e8614),LL(0xabbfb492,0xc7dc6a2c),L_(0x89df23fa), LL(0x2759d406,0xcc56f093),LL(0x7c85ce0d,0x049b5aa5),LL(0x4e1f566b,0xbec87c76),L_(0x19e59ca6), + LL(0x40e840f6,0x1f1c6ca8),LL(0x67aa4467,0x2d21e27a),LL(0x9ca6123e,0x1d5e2c46),L_(0xa8534061), LL(0x5531352f,0x009d49a6),LL(0x161c45f4,0x43dffb38),LL(0xd020075d,0x2f8a3d38),L_(0x25c709ee), + LL(0x5ded0d35,0xd70c960d),LL(0x0bc185ca,0x08b78436),LL(0xbcbebb9e,0x662ab949),L_(0x7643c75a), LL(0x1d7a378d,0xcd6abb3b),LL(0x156ac08f,0x991364b4),LL(0x4dbff4b2,0x6e088465),L_(0x1596ba10), + LL(0xf7fff47a,0x8b812163),LL(0xabc30e47,0xad939c3e),LL(0xd700d28b,0xe8837ac3),L_(0x35b89ca7), LL(0x26712ef3,0xd89edbf6),LL(0x9b73a154,0xeaa73bdc),LL(0x536ef396,0x19770554),L_(0x44c569b4), + LL(0x7f3d2320,0xe904178b),LL(0x58a07ae9,0x7ff297bf),LL(0xad6fdd69,0x6deef5b5),L_(0x4df1d514), LL(0xda650bfe,0xc9fb4677),LL(0xdc9c7db6,0xd99a935e),LL(0x93d1a532,0xbfdd1320),L_(0x939080be), + LL(0x0e07db7e,0xfed77923),LL(0x579fd4db,0x0f05670b),LL(0x112a7ade,0x6e79da66),L_(0xb61b7ee5), LL(0x201e8aa3,0x567d0dd8),LL(0x07f189a5,0x9a0b34dd),LL(0x6a730912,0x5c656713),L_(0xcbc9583d), + LL(0x9522c2ad,0x1cf99a2e),LL(0xfaa41b68,0xbfb9ae12),LL(0x1fe851c4,0x91215bb4),L_(0x7117d94a), LL(0xbf89747d,0x6286cdba),LL(0x9cbfd3ce,0x792aeb4e),LL(0x25b00fa8,0x61f48d5b),L_(0x7160f470), + LL(0xa1ce4a2c,0xbe45da68),LL(0x3b1ab97d,0x63ee425c),LL(0x98b17978,0xd26e1784),L_(0xa89d00db), LL(0x5fcc8cf1,0xecb80c24),LL(0xdcee4667,0x3e21c95c),LL(0x1df7f32b,0xb9cc6c4a),L_(0x77cd2dd5), + LL(0x61766f4d,0xa5778295),LL(0xf4e5e3be,0x34307913),LL(0x57f4defb,0x71c105c8),L_(0x3add4e70), LL(0x2096768f,0xce51bf21),LL(0x8db7461d,0x3117821e),LL(0xf475048e,0x2b073db2),L_(0x1fd13104), + LL(0x70678950,0x4426eaf6),LL(0x4191c57d,0xd1c53be2),LL(0x68f854e3,0xce0157db),L_(0x5837f49f), LL(0x5112a49e,0xa0471f83),LL(0xf2d94f60,0x87e0a2d0),LL(0x9f64998d,0x344b223b),L_(0xa4dc016f), + LL(0x3ad984a8,0x3bd4bc94),LL(0xd8e29806,0xc17ca92f),LL(0x4aee7a72,0xfcd30768),L_(0x488c7f42), LL(0xa1f41f77,0x6fdb5587),LL(0x6983a223,0x3c2763ba),LL(0x0145b604,0x6e18b2a2),L_(0xfd8b9df0), +}, +/* digit=16 base_pwr=2^112 */ +{ + LL(0xa72d343c,0x006aff79),LL(0xd1d45ac5,0x1ec739c8),LL(0x136d5ba5,0xce9ce64b),L_(0x3571e770), LL(0xb11ea221,0x7bfe943d),LL(0x6d87fbd7,0x57dad905),LL(0xac86bd3f,0x5eeba8a6),L_(0x8def8e05), + LL(0xb0b49014,0x31f50c04),LL(0x17dee513,0xd8fab554),LL(0x1d543cec,0x2fb0a285),L_(0x53dbc27d), LL(0x3256cf6a,0xf3bd35dd),LL(0x5fbdcac3,0x8a264f23),LL(0x7dff3a00,0x24830923),L_(0xea9fd27d), + LL(0x3267fa62,0x7dfb1e52),LL(0x423506a7,0x34e86055),LL(0x552bcdda,0x131f5b56),L_(0xbbff309a), LL(0x40256e8d,0x61212776),LL(0xd0114404,0x3807bd7b),LL(0xbba74ace,0xa2ce6ab3),L_(0x9f9afd63), + LL(0xfd519550,0xd0706483),LL(0x53c2383f,0x89215ea5),LL(0x45e8a87b,0xf3cf4080),L_(0x6ab16c59), LL(0x85c8552e,0x09212e02),LL(0xa9576c54,0x10c2badf),LL(0x19c8db39,0x7b08661c),L_(0x32334cfd), + LL(0x9e15fa47,0x148c6f0e),LL(0x64a29d06,0xdb00031e),LL(0x4cc0f2bb,0xc3df9d3a),L_(0x2503f091), LL(0x1ff47bb8,0x620b581f),LL(0xb30724ed,0x5758802f),LL(0x4277ac9d,0xfc122be2),L_(0xcbf16ba0), + LL(0x1336d2bb,0xcd5bed69),LL(0x59c4c7e5,0xedd85962),LL(0xac1f77dc,0xea62aa43),L_(0xa9cecef3), LL(0xf970f802,0x25fc53f3),LL(0xb9f73243,0xf9d1654d),LL(0x9f9753cc,0xb61c3720),L_(0xbf308c03), + LL(0x96dca6f8,0x6461f973),LL(0x48986188,0x1f57a9df),LL(0xb90ca0df,0xb010f7d3),L_(0xe0bff3bf), LL(0x21188095,0xf21e0d4d),LL(0xa1a7cdb0,0x5f3468a5),LL(0xde0d0f24,0x4e99f334),L_(0x2e30f9fb), + LL(0x70d90130,0x7c2bf741),LL(0x95a93740,0x47c7e114),LL(0xb309c07a,0x977513df),L_(0xf1b8dcbf), LL(0xbe1ac3e9,0xb7e947d5),LL(0x92ee6dbc,0x09c3181d),LL(0x41e0c220,0x8b26dfc1),L_(0x897d02ce), + LL(0xfa332321,0xbb0f1fc0),LL(0x69d913b4,0x68034aec),LL(0x784ebda5,0x58fb05ab),L_(0xb3fd4405), LL(0x32cca1fd,0x83e2269c),LL(0x330bdb45,0xea0312b8),LL(0x33fa100f,0x1761c87e),L_(0xf9882c81), + LL(0xf4f00cb6,0x58c8e52b),LL(0x468b810a,0x99489e98),LL(0x5593b9ee,0x7924f02b),L_(0xbf5732d6), LL(0x0334e2ee,0x2a2bf206),LL(0x9c377382,0x41ac277c),LL(0xa7b08209,0x0c5c176d),L_(0xb9b519aa), + LL(0x1aabcb40,0x1450434d),LL(0xce36050b,0xcf82a49e),LL(0x0e6c721c,0xec192907),L_(0xc53a6a39), LL(0xfecc2ffe,0x451e72f3),LL(0x5b5bffe2,0x81bf062a),LL(0xe8852ddc,0xf84db443),L_(0x6d98976f), + LL(0x32a2e887,0x8b6ac261),LL(0xb92b83b0,0x3c40d7a5),LL(0x1ea4ca4c,0xf7653f1d),L_(0x4e2014e8), LL(0x2b234800,0x56338adb),LL(0x3350e80f,0x314b0e58),LL(0x8e442705,0x79527139),L_(0x96bf154a), + LL(0xd63c1701,0x1101a669),LL(0x5972ea79,0xd06390a9),LL(0x501cc189,0x0fb428cd),L_(0x41b24261), LL(0x0a3108ab,0x47aa9e0e),LL(0xff7237f2,0x9c9d8b16),LL(0xe86804cc,0x61ad5482),L_(0x723214f3), + LL(0x44329550,0xebe782ea),LL(0x4bfefe5b,0x11e7dc71),LL(0xf9379d6a,0xfb5f9f2d),L_(0x016789fa), LL(0x98c8071f,0x3aaef819),LL(0x0fa53740,0x3d2ba5a3),LL(0x0cd97039,0x2ceebe22),L_(0x144818d7), + LL(0x6646c444,0xc3bd60cf),LL(0x1bea68cc,0x7c4048fa),LL(0x5e74dbe8,0xd29b1f65),L_(0x08428502), LL(0x91eb1131,0xbb7b08f7),LL(0x64a0eea4,0x3c5f214f),LL(0x6e37d0fb,0x85ea7863),L_(0x3a3aec39), + LL(0x87af66ae,0x26b2f70c),LL(0xb4a6f150,0x5494f6a7),LL(0xf84408cd,0xa24b6575),L_(0xb95400d4), LL(0xfb2baa36,0x7fdee2e4),LL(0xfc9f2464,0x82709ae0),LL(0xd231d3ab,0x8897d9e8),L_(0x4c1905ed), + LL(0x43e3a796,0xc93592ff),LL(0x194deaea,0x9aebe35d),LL(0x3c64092f,0x1d7432bf),L_(0x64ab56a9), LL(0x608467bf,0x61dd14ba),LL(0x64e82c68,0x1c6431e0),LL(0xb4aaeed1,0x53076cd4),L_(0x2ded5d91), + LL(0x064ac09f,0x3e9b39b0),LL(0x087b2333,0xd8bc5839),LL(0xc348b809,0x8ab0974b),L_(0xdf5fc364), LL(0xf886797e,0x9cb572eb),LL(0xba2f94be,0x9a9ce85b),LL(0xa4a7eaf7,0x0e1605e0),L_(0x3070f698), + LL(0xbe868d80,0xeca36ddb),LL(0x52ac8d98,0x86bfcb64),LL(0x9b8656b8,0x2ab84426),L_(0x8306d84c), LL(0x01fe86cc,0x15f75f0e),LL(0xe9502973,0x4c2d70f4),LL(0xf239468a,0x365ce9d3),L_(0x7dd8e76a), + LL(0xd3426970,0xe0e930b3),LL(0x19af19f4,0xcb67c28f),LL(0x163565e8,0x312a9e2c),L_(0x0438b01e), LL(0x54d76d69,0x3b508871),LL(0x2b43ad8e,0x71ee55a1),LL(0x4dfa9969,0x0a025c8a),L_(0xe6813e1d), + LL(0xc327e03a,0xf0b0b5d0),LL(0x46569e00,0xb462d687),LL(0xa70b2c73,0x842f549f),L_(0x18faac6a), LL(0x864e3871,0x0b583d9c),LL(0xf49df9a7,0x1693a8f2),LL(0x5135df56,0xadecd5ac),L_(0xe9e25ac7), + LL(0xe8f3d99b,0x04c439d1),LL(0x811daa39,0x79057450),LL(0xed15dbf2,0x557c06cf),L_(0x3a8a2063), LL(0x09b41a1f,0x0bdc4dcf),LL(0x71cd60e9,0xf085eab4),LL(0xdab0c931,0xb48d6df1),L_(0x8189574b), + LL(0x4a27c44c,0x2e80ab05),LL(0x7a2bc54c,0xdd9e1375),LL(0x242b815a,0xa910e496),L_(0xd9fa2e90), LL(0x3f111c1d,0x0c5343f8),LL(0xb573bc74,0x10be67cd),LL(0xf1c6ef62,0x4ae541a2),L_(0x74676666), + LL(0xc5669a28,0xff0e3e5f),LL(0x6a94d97d,0xe911fae7),LL(0x85cd9be2,0x4150b30c),L_(0x41391baf), LL(0xab441ae7,0x92888eaf),LL(0xd928ee21,0x0dd62a7d),LL(0x1b7b37eb,0xe9e642a8),L_(0x61afa524), + LL(0x15d73eaf,0x70f4b3ee),LL(0x10edc9d7,0x70ff29ef),LL(0xe8787785,0xe5dce106),L_(0x3877e81c), LL(0x36b7ab1b,0x21a140c0),LL(0xf44e41b6,0xffda816e),LL(0x9bd299e0,0x5968ca97),L_(0x9fed2d11), + LL(0xb0d9d8cd,0x6b2cc953),LL(0x12446f29,0x932fd885),LL(0x8fc62139,0xb782380e),L_(0x93475be3), LL(0x83692f15,0xb2525747),LL(0xe5afa458,0x2213b2d5),LL(0xc34bb49f,0x0270ba84),L_(0xa86bdfc6), + LL(0x3b925a7f,0xe649f779),LL(0x72b7e377,0xa3785aef),LL(0x20ce9dd7,0xdc5dbf9f),L_(0xb05950ec), LL(0xa5d677f5,0xcdc8d1f8),LL(0xb5473ac8,0xd8d3e9cc),LL(0xb698eb06,0x5cbd0981),L_(0xc3f31c81), + LL(0x1e752016,0xf090a1eb),LL(0x644b9e14,0xfc5f6857),LL(0x111a1741,0xbaf89d9f),L_(0x01aef06c), LL(0x1b1a1911,0xbfa9551b),LL(0x7145d55b,0x24a70db9),LL(0xc5ef4f06,0x17130b26),L_(0xb33cca3b), + LL(0xfcb5b043,0x911935f4),LL(0x4a8bc08e,0x53ed6fbf),LL(0x90634826,0x4421e383),L_(0xe0140eb6), LL(0xc4143e7b,0x12ccf145),LL(0x94d99fc8,0x5bf8b6c9),LL(0x4f765081,0x73883a6d),L_(0x6767c401), + LL(0x997c821b,0xe367cf8a),LL(0x246522bf,0x3d0455b7),LL(0xe8c7bcd6,0x92643ab4),L_(0x10fbb420), LL(0x547dbeb2,0xb5294491),LL(0x4b5c792e,0x07450e44),LL(0x472a47f1,0x5e3d3784),L_(0xd9dc604f), + LL(0x6372566b,0x4d62de9c),LL(0xbc29741f,0x7a323977),LL(0x78458eba,0x4b2a04ff),L_(0xdc2f1c5d), LL(0x6819a84d,0x0eccbf7e),LL(0x8ee3cc6d,0xb93eec58),LL(0x2a02a086,0x03b76658),L_(0x51df6b51), + LL(0x6aa0f608,0x0cbee27c),LL(0x00ecc369,0xae1d1601),LL(0xbbef8dbb,0xe7149786),L_(0x4036b3f1), LL(0xbe686d04,0x91ce3c48),LL(0x339d0769,0x3d6535b6),LL(0xa72c8f85,0x2c9fb75e),L_(0x0a96cd51), + LL(0x39dd7c75,0xeebc8c18),LL(0x799170f2,0xa2f0fb39),LL(0xba0f68a4,0x1c5cad2c),L_(0xb5b8faf2), LL(0x3f59abfb,0x3a302769),LL(0xcda64ecb,0xed44c055),LL(0xcd7fd1b9,0x2e974b80),L_(0x7a2910b3), + LL(0xc8f85c88,0xf503e456),LL(0x9dc39b84,0xb53e74a2),LL(0x8eb76194,0xc4c55423),L_(0x96e13302), LL(0x153fefbb,0xdbbf4e12),LL(0x77137bad,0x80b3f693),LL(0x65187422,0x9f8c212c),L_(0xdd256603), + LL(0x0fcf5a39,0x6da81ce6),LL(0xb694d211,0xdff4777e),LL(0x4a9fa32b,0x602640c6),L_(0x5470aa01), LL(0xa3b241c7,0x2a545f4f),LL(0x33f7dd48,0x4cf94265),LL(0x8a894ff1,0xb5260296),L_(0xd9db2162), + LL(0x7320211d,0x391c4f62),LL(0xf472fe07,0x6d039385),LL(0xbaa68de6,0xc3a5b977),L_(0xde6b274a), LL(0xbca0bb5d,0x3643a502),LL(0xc554c3be,0xe1f5b66b),LL(0xa430cadc,0xb32e7ba4),L_(0x0c4ee9d5), + LL(0xd36efcf1,0xb6b2c682),LL(0x0eae66b5,0x1024c516),LL(0x8688f303,0x99e728ff),L_(0xb372d118), LL(0x777d2fdf,0xd972734f),LL(0xd9b90e14,0x25251453),LL(0xf34097fd,0x424404cf),L_(0xc047b89b), + LL(0x3079239e,0xb5ff46f3),LL(0x60a4632b,0x36a3735f),LL(0xc9986723,0xd572091f),L_(0x2023152d), LL(0x0dabffd6,0x0da45042),LL(0xb1524e2b,0xf6ffaf4f),LL(0xbffef0a3,0xbc330b9d),L_(0x99ba1268), + LL(0x3b961738,0x8d236f27),LL(0x3ea1c105,0xa5f014ae),LL(0xee0ae6b2,0xe50d6b9c),L_(0x82eb66db), LL(0x2b5f008f,0x36e3aea7),LL(0x4b4fc9df,0x2f8e4e4e),LL(0x5ccc02df,0x9f683f7c),L_(0x3e46d9c7), + LL(0x7caa8791,0x6c2cdda1),LL(0x3274eefe,0xa9cdf2c5),LL(0xb634f00e,0xd5eec010),L_(0xc5ba5f67), LL(0xa6a18c32,0xde736bf0),LL(0x554e6af6,0xc1059185),LL(0xb7a29e2b,0x9b23923d),L_(0xbfed0131), + LL(0x7000c5a3,0xb80400e7),LL(0xf87a8596,0x147ccc0f),LL(0x9e3b070e,0xfef094f1),L_(0x36f53f39), LL(0xefcc38e1,0xf5b77a62),LL(0xb1c0b71d,0x2c523e41),LL(0x02efb3f7,0x210639da),L_(0xf54c9329), + LL(0x330449ca,0x7b06f635),LL(0x0b195c0b,0x9efabcec),LL(0xe879f10c,0x3bd206ba),L_(0x25bef90b), LL(0x0312bdf4,0x88e64b25),LL(0x29334071,0x3a8f90eb),LL(0x3e832f84,0x14f38ac0),L_(0x8a3f7e05), + LL(0x4bafc9dd,0xb6461cb6),LL(0x75d1e03c,0x266ba391),LL(0xf40aa6ef,0x4715355a),L_(0x38cce560), LL(0x1adab378,0x716f224c),LL(0x38711bba,0x32b71c39),LL(0x7e89026d,0xccbd7f4e),L_(0xcc519b13), + LL(0x7de43131,0x0bbfe1df),LL(0xb1b9786c,0x10701821),LL(0x0bc2e01c,0xf68b86eb),L_(0xf6a6da2b), LL(0xad3f4af5,0x67e192cc),LL(0x6b4b9ef1,0xd5fef609),LL(0x52b649b2,0xf428af30),L_(0x7e90cd58), + LL(0xe78176a6,0x0b536242),LL(0xaae8f896,0xdbdd8b1f),LL(0x503b274a,0x51151bfa),L_(0x60ba1217), LL(0x393d4fa6,0xdc4e6926),LL(0xdc183c7c,0xe68387c3),LL(0xaaf0a35e,0x06168aa4),L_(0xb2853fe5), + LL(0x2143ebc1,0x4be98451),LL(0xbb1ddca8,0x10163229),LL(0x5db7c140,0x8229e729),L_(0xca486360), LL(0xc19bdf58,0xc8df52e4),LL(0x560cd486,0x89b605d0),LL(0x2a200cdd,0x0c94090c),L_(0xb1ef5103), + LL(0x4588f760,0x0c217b21),LL(0x92ea3d28,0x90f7c008),LL(0xef37649d,0xddfaf799),L_(0xfddee462), LL(0x3d0294b1,0x510fffdd),LL(0x1bcf376e,0x4a21ba9d),LL(0x6c9d2cfa,0xa7ada78e),L_(0xca77a07b), + LL(0x1db377da,0x63e01d3e),LL(0x5a90911d,0x8c9ad4b9),LL(0x6bb31b8b,0x2b26fec2),L_(0x30e44c9b), LL(0xfeaf9d42,0xe2652b65),LL(0xf5866de8,0x03b48401),LL(0xee2505a9,0x2438e370),L_(0x7a7bb613), + LL(0x900098e6,0xd8a6e8e7),LL(0x830770af,0xd8452a93),LL(0x2e2cd25b,0x569bbc5f),L_(0x61715beb), LL(0x80e8caaa,0x3f76b251),LL(0x6dff2aa3,0xeba89ffa),LL(0x18acb601,0xf763dfc0),L_(0xa2dd440c), + LL(0x24370f96,0xf5f4a36c),LL(0xbf3c4689,0x527f5322),LL(0x43417948,0xe0b12bf9),L_(0x012f5cea), LL(0x6aa04128,0xb3183162),LL(0x2fa9d189,0x1070f9c1),LL(0xe002ccee,0x5b0c2b4a),L_(0x5840d505), + LL(0xfca649ba,0xb223ef3b),LL(0xbe1308ed,0x4aa0b0e6),LL(0x10592f0a,0x3d9b366f),L_(0x3859ac39), LL(0xab61a042,0xaa721fec),LL(0x7c5e75a3,0x0a96a720),LL(0x06be6fd2,0xb3b3cccc),L_(0x9eb8c9a0), + LL(0xfda2de37,0xd8576cc3),LL(0x1c4530fa,0x22b5be7f),LL(0xaad70136,0xc40c9235),L_(0x785c8acf), LL(0x9edbb745,0x501faa81),LL(0xc7fdb942,0x60c831a6),LL(0x6f69ca6e,0x813a83b7),L_(0x04f82291), + LL(0xdbe3e246,0x4f871623),LL(0x56664b79,0xd2c645ee),LL(0x18c48541,0xc037f50e),L_(0x569420f7), LL(0x08342ef8,0x68cc0cb0),LL(0x19c07325,0x0c4fa826),LL(0x186c66ac,0x8f28f4b4),L_(0x34c938cf), + LL(0x837493cf,0xf41079e1),LL(0x433d8394,0x1a861703),LL(0x0e538567,0x03e474ae),L_(0xd2b95886), LL(0xa188c7eb,0x76610ad6),LL(0xe08cbcbd,0x706eb272),LL(0x8025f126,0xb5435d24),L_(0x5ba9b4af), + LL(0xc4faeda3,0xd9ddd6dd),LL(0xf3e74637,0xe2be7abf),LL(0x4eea6f06,0x2e6abaae),L_(0xd79c705f), LL(0x99f8dc46,0xc8502a73),LL(0x2f4fd6ed,0xa0aa0199),LL(0xcbc5ed2a,0x38689f49),L_(0x55280ac3), + LL(0xfbd4773f,0xb3548b14),LL(0xd9714b0f,0x18cea1f4),LL(0xb3c65f79,0x3bc435fc),L_(0x2867e8b0), LL(0x0706d139,0xca5b9f90),LL(0xe62b0a3f,0x14b87e76),LL(0x864343f5,0x8a82e1fd),L_(0xcbeb5bc7), + LL(0x18cc1c83,0x0026972b),LL(0xb17ec512,0x1da022c5),LL(0x2265f4c6,0xb898d6b7),L_(0x5d09e5e7), LL(0xa53eabdd,0xef23c0bd),LL(0x92d24054,0x98e844f5),LL(0x05f9b046,0x7d2e87fd),L_(0x3f161352), + LL(0xe7c2033e,0x1c9b4b3f),LL(0x29dc2f02,0xf61d0ee2),LL(0xdb5834bf,0x28b5943b),L_(0x065b016c), LL(0x5de6ece7,0xf70c350f),LL(0xf1049294,0x1c3bfb3b),LL(0xd4c8be79,0xe75564bc),L_(0xb3dd4c16), + LL(0xab2cc7d8,0xa4b41a9f),LL(0xbc199c52,0x912b7247),LL(0xe6362cfb,0x6a214d2e),L_(0xe939b311), LL(0xc25479e6,0xf1652743),LL(0x06ef2b5a,0x3bcb10e6),LL(0x9be1e8e0,0xbede8ea6),L_(0x638ffa1e), + LL(0x31630309,0x706d16a7),LL(0x3ba8e623,0x9f2f55c9),LL(0xed0db8c6,0xaf0d2787),L_(0xba630c04), LL(0xb9e7f044,0xce8001ee),LL(0x10c159b4,0xb8247bef),LL(0x658e5458,0x78f60572),L_(0x922d3798), + LL(0x5c6843ce,0xacaada1c),LL(0x8794be44,0xcabaa3a1),LL(0x36370b66,0x507a17a3),L_(0x943c28ea), LL(0x481b026e,0x48821bb8),LL(0x3885a0a3,0x5ed8e5b9),LL(0x78f46113,0x6817009d),L_(0x1451c3b3), + LL(0x7560c539,0x75119a00),LL(0x953124bc,0xe00d03ba),LL(0xef71ab39,0xb863df63),L_(0xfd67781c), LL(0xfedea07d,0xddc4f0d5),LL(0x332679da,0xc43e8f08),LL(0x36d08634,0x9efb1f02),L_(0xa0922f94), + LL(0xba05b2e6,0x949d9990),LL(0xd565ff0f,0x76e4e628),LL(0x6c281fa8,0xdccad172),L_(0xcdea3fff), LL(0x6b13eba7,0x70f4d75c),LL(0xd884ce93,0xd34756fc),LL(0x70986d34,0x8f71b0bd),L_(0x430f43f0), + LL(0xe10a7afd,0x0a9b3e69),LL(0x66948394,0xf0719c40),LL(0x046cc610,0x9d1611b2),L_(0xda89278b), LL(0x6b7a1ff1,0x24d9cfba),LL(0x638bd7b1,0x5bd30853),LL(0xa740e80a,0x1159e67f),L_(0xe45ccb34), +}, +/* digit=17 base_pwr=2^119 */ +{ + LL(0x5898fc39,0xa3566efe),LL(0x23845a77,0x24bb02bf),LL(0x91ab4316,0x00e46ee0),L_(0x83d35ab7), LL(0x8c02881d,0x1d20ad25),LL(0x5866ad97,0xb01ce79f),LL(0x40d6ae86,0x40b2c06e),L_(0x613bd7a9), + LL(0x259779c6,0xafc1d993),LL(0x8a7cf112,0x86dcdc82),LL(0x77170d32,0x8441f4f3),L_(0xb0280bfe), LL(0x46144a82,0x58df739b),LL(0x5ee74a89,0x344ddb0c),LL(0x01761502,0xd9bcbeb6),L_(0x676d4eda), + LL(0x66de5a2e,0xbc92b6dd),LL(0x7b186d5d,0x1127f258),LL(0x3d18ed03,0x3982ee92),L_(0x50719e4f), LL(0x0deb6e5c,0xcf198f32),LL(0x09997437,0xdbaf1a5d),LL(0x373c1801,0xf524b8bb),L_(0x73564dbc), + LL(0x1bb01e24,0xdc5e2b63),LL(0xaca4f30b,0x5df11c5a),LL(0x68e597b5,0x44ec319c),L_(0x8b90301e), LL(0x0e27f5b1,0xe5706d53),LL(0x8b76e26f,0xe67b3280),LL(0xa7111b5d,0x8fa5ee02),L_(0x7b4c92ce), + LL(0xe429fedd,0x0035c84e),LL(0x867c82eb,0x2ea6f134),LL(0xa3e74746,0xb9e352c0),L_(0xaebebaa5), LL(0x220bb556,0xb02520ea),LL(0x95bda416,0xe4b94f8e),LL(0x16dbb87f,0xfbff49d6),L_(0x5707ddc9), + LL(0xda91cf49,0xdb33cda7),LL(0x450672ba,0x006b3e24),LL(0x489cbfa3,0x4461fe49),L_(0x3d14a461), LL(0x4d93f4e2,0xfe8e5310),LL(0xb7769c82,0xf3d0d29d),LL(0x94deaf7c,0x3d33e2ad),L_(0x3b18c763), + LL(0x453d6839,0x8f1b71e2),LL(0xf6d77216,0x923024ff),LL(0xf76bc681,0x17e6ad54),L_(0xe16b7b80), LL(0x66dcecfa,0xe97810bb),LL(0x94c6a912,0x13de9f05),LL(0x30c9e7f2,0xf16b622f),L_(0xa2fbcfd3), + LL(0x436d6b95,0x794d49e6),LL(0x3831fe74,0x702ede17),LL(0x5afaacb1,0x3b5ca68c),L_(0x1bea7d66), LL(0x9ca7b740,0x50a1e66e),LL(0x8f17d122,0x762e4f6f),LL(0xc90185b1,0x6705418b),L_(0x36b69158), + LL(0x52070a5f,0x9065afef),LL(0xa326ad21,0xe6df0c7d),LL(0x441709b3,0xa1396423),L_(0xade77b6a), LL(0x66868b27,0x14f84f84),LL(0xd0d002f8,0x1fa90792),LL(0x19c083f9,0xfd8d88ee),L_(0x57fe75c9), + LL(0x5330a97a,0x77ea9ee9),LL(0x00005025,0xce00962f),LL(0xae5f476a,0x05915824),L_(0xdba736d5), LL(0x8cdf0301,0x660a19e4),LL(0xee8a2aed,0xc70e0b10),LL(0xfdd0563d,0x41d445a6),L_(0xede64633), + LL(0x53d54a0b,0xa5729cf8),LL(0xb0c860cb,0xea907454),LL(0xea0c5a91,0x59a871ce),L_(0x9efd302f), LL(0xe025ffb1,0x7e7fb9d6),LL(0xc8022642,0xb7954f0d),LL(0x46ca7ee6,0x010dbf77),L_(0x4bfdff6a), + LL(0x2d7e46ec,0x034b69ed),LL(0x850b3401,0x51bc6b66),LL(0x3c25755a,0xe9bf6717),L_(0x95d70595), LL(0x06b03b20,0xfc3e6ce3),LL(0x384f786b,0x723756d9),LL(0x58938357,0xb4731ae9),L_(0xa8628a04), + LL(0x42ddfb71,0x33bbd9fd),LL(0xc9ef16f4,0xeb7aa22e),LL(0x725fdf2d,0x88eb52e0),L_(0xa3454e1e), LL(0xf1fc16a0,0x297a7972),LL(0xda90c2e0,0xb92c9670),LL(0x9c2b4849,0xa557aadd),L_(0xb719cae7), + LL(0x58182a1a,0x27b6bd7a),LL(0x5c2d70bf,0x1c72e559),LL(0xd4f2f098,0x6b53e542),L_(0xb01de7e4), LL(0xb5e11023,0xa8f7b191),LL(0xb49a6846,0xe62460d3),LL(0xbdf84150,0x74ea071f),L_(0xf4211acf), + LL(0xe03c9ae5,0x54fef132),LL(0x9ad73091,0xa900e0bc),LL(0x1a602274,0x3698f6bb),L_(0xe357c5f7), LL(0x57e365aa,0xbb500025),LL(0xf19ef91f,0xf16e71f5),LL(0xa0dfe4bb,0x9d6899d1),L_(0x399ab86c), + LL(0x8e577079,0x8e833532),LL(0xabea4782,0xfb9fbccf),LL(0xf5ead3c9,0xe31f0c20),L_(0xe2d0d774), LL(0xc462ce8c,0x6466c045),LL(0xb27628d9,0x85e7447b),LL(0x3e1abc03,0xbba798a0),L_(0x0e905d6c), + LL(0x122d19eb,0x158f4312),LL(0x3f408e0e,0x297c5747),LL(0xa7b750fd,0x735aa3f7),L_(0xbe1aa687), LL(0x312d6129,0x31e8bfa3),LL(0x166c2e16,0xcc9cdadc),LL(0x5094cff5,0x9ea9beca),L_(0x2dea9edd), + LL(0x616ee97a,0x78257a1a),LL(0x7f5d51d6,0xec98f98c),LL(0x609d2d2d,0xd1b3e514),L_(0xe1569e97), LL(0x2e59e233,0x8087f780),LL(0x45e2a372,0x0b40b852),LL(0x95c2865d,0x8fa97627),L_(0xeeff0632), + LL(0x1eac58fb,0x7eb748d9),LL(0x9922cc63,0x0d0b673d),LL(0x955ae146,0x72be4e8e),L_(0xd6809bdd), LL(0x22b08d07,0xa4616bf8),LL(0x09a6ef9c,0xc55b68a6),LL(0x42138fbd,0x8fa529ef),L_(0x4dfbed1e), + LL(0xf1766b08,0xdfd72d39),LL(0xe0356587,0xad6b741e),LL(0x59466485,0x5fceaf74),L_(0x24d9205e), LL(0x0c6b2255,0x453c9eae),LL(0xb9402267,0x9c00d050),LL(0xa3817166,0xa87cf553),L_(0x43612515), + LL(0xad146ff6,0x39990110),LL(0x0f98feec,0x3d48a808),LL(0xfe9fea01,0x8c4aac12),L_(0x2443ed5e), LL(0x0ad5c79c,0x0a9a20f7),LL(0xd997c3c0,0x942b38bd),LL(0x9475c474,0x218031b5),L_(0xd3104cb6), + LL(0x4c0dcf95,0xcc4c491e),LL(0x234524ca,0xfa790f55),LL(0x12a6c289,0x37abf7be),L_(0x3e8c0631), LL(0xcb586fd3,0x192166df),LL(0x0c51f097,0x7ee1f233),LL(0xbba9c768,0xb69b46cc),L_(0x2fb2d51d), + LL(0x1443ddcd,0x6345a210),LL(0xecf6729f,0x5f035126),LL(0xc8273e38,0xdafd6230),L_(0x83e19f89), LL(0x4e79437a,0xb3e2d215),LL(0x6090e2e6,0x138aca38),LL(0x4afb3539,0x4ce64d55),L_(0x30dd853f), + LL(0x6b581765,0xc440084f),LL(0xeff7dc6a,0x088767cc),LL(0xf0d004bb,0x94e1ba30),L_(0xb9c2e7e5), LL(0xa1a18413,0xa732c539),LL(0xc31da773,0x896710e9),LL(0x3608f75d,0xa067bb07),L_(0xaa5451e2), + LL(0x98910990,0x65c3a1bc),LL(0xbcc70896,0x3321e865),LL(0xb04ac43d,0x438e5b9d),L_(0x43f7f50c), LL(0xe260d025,0xc4dcba5f),LL(0x79613445,0x5e5f5f4b),LL(0x37b1a0c5,0xd3c102f6),L_(0xa0273b2c), + LL(0x4e2d8d4d,0xe6546299),LL(0xd9309108,0x262d0cb1),LL(0x2df96d37,0x349292cc),L_(0xa711d216), LL(0x2fddedd4,0x452a0a52),LL(0xd9e54125,0x2293a068),LL(0xc6053861,0xfe3040eb),L_(0xfb5c3f2b), + LL(0x02a166d6,0x861af22b),LL(0xc493f8ef,0x449c202b),LL(0xb053bb62,0xbb664aff),L_(0xc5af5837), LL(0xd6f78a71,0x14a57ed2),LL(0x1edcfcb6,0x6c5eedb1),LL(0x591d0e00,0x0f079d9e),L_(0x6f7c473e), + LL(0x4ee2bac7,0xc62978b5),LL(0xeec393c4,0x0ee684b8),LL(0x56d30723,0xb8993d5e),L_(0xdfc904ce), LL(0xfc172756,0xc8622a7c),LL(0x73b2ec23,0x4279a0bc),LL(0x78cfee0e,0x3826553c),L_(0xd011a36e), + LL(0x9060272d,0xe29ecb65),LL(0xa0218247,0xdd976fe9),LL(0x53d3e99f,0x278ebed1),L_(0xbcea3ccb), LL(0x7a33fe7f,0x13d33afb),LL(0x1b5111c1,0x19d16dca),LL(0xe09a47ad,0x6e0ed440),L_(0x80396140), + LL(0xb55799e5,0xf145e8f9),LL(0x443b3b83,0x97e6d3d7),LL(0x4580043d,0x5f599a51),L_(0x2fcd83a6), LL(0x38414d95,0xe09d35b1),LL(0x8975407b,0x31551460),LL(0xce073290,0x1044bab9),L_(0xae80d126), + LL(0x045e586e,0x4d8f2d61),LL(0xc81d17ac,0x1d2b5b31),LL(0xb68b7f81,0x10bccde6),L_(0xdb1468c5), LL(0x0578898d,0x1e460079),LL(0x5fbad189,0x31bad31c),LL(0x3d22477a,0x8c371b62),L_(0x32c1f1f2), + LL(0xbc19767c,0x3ed7db29),LL(0x4ce61a9a,0xcc843c60),LL(0x45c30a38,0x508e3e86),L_(0x5c8b4727), LL(0x1457fcb7,0xd8c78825),LL(0x308a4222,0xc3997703),LL(0x2b88e6d2,0xa1f2140c),L_(0xbcc49242), + LL(0x2a452962,0x62444dca),LL(0xffa24655,0x2d1e976d),LL(0x1d95f51c,0x2bbeb1e0),L_(0x6ac09c41), LL(0x052daa66,0x827170a2),LL(0xbfaa5a51,0xc319b1ba),LL(0x3a8ec55d,0xda0c72a5),L_(0x9d0ac0b5), + LL(0x898419db,0x35d9c257),LL(0xbb8eaa1e,0x55758230),LL(0x4e49ddae,0xd5c3703e),L_(0x8e18cadd), LL(0x28d3f23f,0xa38dc679),LL(0x8ecf56aa,0x749ea0b7),LL(0x15724faa,0x68bb7f4b),L_(0xb2b1249d), + LL(0x0f227df0,0x24af0afb),LL(0x8232f88c,0x1cd97cee),LL(0xa2ad5165,0x678f117f),L_(0xa5067478), LL(0x03432db4,0x5cb93ad2),LL(0x610d26df,0xcc96e20a),LL(0x47d3d48c,0x5f1e84c2),L_(0x31ddfcd2), + LL(0xc4293aef,0x7f0db6e5),LL(0xe03b4f7e,0x9069c8c1),LL(0xa088318f,0xf4210d55),L_(0xe9cec920), LL(0x13bd2b28,0x34f3150f),LL(0x7de6ff6f,0xe152eb4e),LL(0x0fcf1527,0x32e6885b),L_(0x4f27fdfa), + LL(0xed8906ba,0x7eba0ca2),LL(0x615cf3d7,0xc91dbf27),LL(0xcf44aa02,0xf0948c8c),L_(0x346dc0f7), LL(0xbe7b7488,0x88539394),LL(0xe2e6a4a5,0x938fd6cc),LL(0x15b31f6a,0x1a1c6b6b),L_(0xe564ce23), + LL(0xc2869ed1,0x61ab3c9f),LL(0x5693bc05,0x3bb8c353),LL(0xb69781cd,0x95d9d719),L_(0x31986774), LL(0x3de229eb,0xc3895533),LL(0x72ce5e74,0xb6955022),LL(0x5ac0d29e,0x6e2ba9f7),L_(0x12310f9c), + LL(0xfe2a24c4,0xe3406a2f),LL(0x7a0fc1d1,0x67beb235),LL(0xf6867bb0,0x5b2cd123),L_(0xc3378be7), LL(0x2ff473c0,0xed6d88b5),LL(0x5fc0483f,0x046a1f51),LL(0x9b3025d4,0xf4a61d8b),L_(0x8b1ce295), + LL(0x99e19904,0x7e9f5e57),LL(0x5c52c743,0x00b84a5f),LL(0xbabc0faa,0x6de1d67d),L_(0xbf153310), LL(0x04041367,0x06672f11),LL(0x6650a578,0x0dd51cb3),LL(0xd4012aa6,0xd5ce357e),L_(0xf901856b), + LL(0x538e45db,0x5f595162),LL(0xf13e03cd,0xe917805d),LL(0x56a8bf52,0xf639993d),L_(0x043c78de), LL(0xe876b4ea,0x3eaf3c58),LL(0xe2439014,0x0ef45389),LL(0xbd9e742b,0x95404cfd),L_(0xce45f79b), + LL(0x1453ad22,0xf6e400ed),LL(0x0126443e,0xa4ababfc),LL(0xd78f1c8d,0x4ebc5852),L_(0x6149c4a9), LL(0x4e1ef091,0x7415270e),LL(0x84d52256,0x921cd6ee),LL(0x472b1f2f,0x765126f0),L_(0x33de4dc4), + LL(0xade7ead4,0x1b413797),LL(0x12c424fc,0xeabbca1c),LL(0xdb705e2f,0xa1e12595),L_(0xa616017b), LL(0xd199009a,0x79fa6f2e),LL(0x9168da3a,0x4d4d9e09),LL(0xbde588b7,0xc0ac97e6),L_(0x9a16a183), + LL(0x23cf39f8,0xca0fac28),LL(0x40729683,0x4b179b82),LL(0xb04c01ae,0xfaca0fdf),L_(0xdf006531), LL(0x7b3cc76a,0x3d2bf7fd),LL(0x7e32a952,0xf3f2fea3),LL(0x6369c1d9,0x866e7f8e),L_(0x8e8c8b64), + LL(0x6eb23ad9,0x8b9475ca),LL(0xe0c8ef8e,0x22d9fd00),LL(0x4e9a3adc,0x813270cc),L_(0x478c17a1), LL(0x292b8bdd,0x3db7f857),LL(0xe5e27f6a,0xf6ba5b62),LL(0x1b8a570a,0xe20966e7),L_(0x040d840b), + LL(0x19b8e7ad,0xe13277a1),LL(0x9ff0663a,0x6d26fe40),LL(0x8f8b7d68,0x0ce42976),L_(0xfa5a58e5), LL(0x48e32e15,0x0ee98d37),LL(0xbdd62d44,0x3bb6c2c8),LL(0x4db74aeb,0x208cd0d9),L_(0x55f387e3), + LL(0x47ef78f3,0x6c1b0959),LL(0x5e1f13ab,0x2ccd97f7),LL(0xa303c278,0xf9abf5b1),L_(0x3d97a968), LL(0x0276fd8b,0x2be1d396),LL(0x8b026a44,0x726beb4a),LL(0xe09517f7,0x4e0b499e),L_(0x502462c0), + LL(0x5979197b,0x82ffa270),LL(0x23bdd46d,0x52cb678a),LL(0x260e7e25,0xc541f700),L_(0x908eadfa), LL(0x33983072,0xb1d2f59c),LL(0x321cd9fd,0x11eef46a),LL(0xf2b04fe3,0x9fba737f),L_(0xe7bc6bb7), + LL(0xbcadb426,0x4d1d9a21),LL(0xb55fa843,0xe2a1f313),LL(0x2170c6a6,0xc15b9466),L_(0xdebbce2a), LL(0xd605b2bd,0x1578a3c5),LL(0xd1e52a96,0x96b6befb),LL(0xbe0b58ef,0xc4f7e33c),L_(0x93e465b5), + LL(0x3fcf43e2,0xb19e11a6),LL(0x508b2d4a,0x5ad2ed77),LL(0x6ff2a142,0xe587c79f),L_(0xa56c66b2), LL(0xb2ee2d93,0xa6fa95d1),LL(0x260067ec,0xac5ec114),LL(0x67b40c94,0x41c55249),L_(0xdc3ac489), + LL(0x6b551c3d,0xc6c2dc20),LL(0x3452633a,0x1b5233ca),LL(0x0e6faafe,0x4565862d),L_(0xfa2f6235), LL(0x08edb0b2,0xb2d31417),LL(0x6cfe0764,0xa0fa599b),LL(0x77052491,0xb2ac3ec1),L_(0xbcc97485), + LL(0x0302d5f4,0x04cf0b83),LL(0x37cf0d9a,0xbd701c39),LL(0xfb565431,0x5eee993f),L_(0xc5d3f450), LL(0xb5c3681b,0x5d285d50),LL(0xcc0e0a9d,0x8618542a),LL(0x8c1d7c52,0xd80996cc),L_(0xddcf4356), + LL(0xf0fe4630,0x12c3c8ab),LL(0xfb1a997d,0xb6349270),LL(0x2f148e5a,0xc0513e4e),L_(0x7a141833), LL(0xd043e9e1,0x340f7e04),LL(0x52e9cd98,0xb4acedf4),LL(0x5d6d0c14,0xfa56de6f),L_(0x2558f609), + LL(0x435b6bea,0xdb7a9959),LL(0xe6426f07,0x24fae08f),LL(0x3cd0982b,0xcc274d62),L_(0x7f805789), LL(0x02f3dea7,0x2a9bf937),LL(0x774708d8,0xadae4c03),LL(0x61de7f6d,0xba3c48a3),L_(0x8de4f729), + LL(0x2faaf033,0x7e8bb31a),LL(0x31c0bd80,0xd937aa83),LL(0x072c01c9,0xbfe57ab1),L_(0x201af074), LL(0x87c5d209,0x4ce95747),LL(0x5887d49e,0x7ee66e9b),LL(0xf01cc54c,0x48478c99),L_(0xa2993bcb), + LL(0x957c7ee7,0x50dc83b5),LL(0xacc25598,0xf0f63a80),LL(0xb4578760,0x842ee70e),L_(0x4ca0ed74), LL(0xc891093c,0x24bc2867),LL(0xfd1e42bc,0x0f85fa2e),LL(0x47bb6a4a,0x18cd7a2e),L_(0xa82a8343), + LL(0x4077391a,0x798cdd8c),LL(0x381485f3,0x68858372),LL(0xd2c5ede9,0xb240c94c),L_(0x7ec5555b), LL(0x62fa25b8,0x89e94f52),LL(0x25aa6ecf,0x0332347b),LL(0xf207ad5e,0x9e15c0d4),L_(0x865ae0a0), + LL(0x6d748e08,0x298a62fb),LL(0x549ffe32,0x8f7fb37e),LL(0x35f944a9,0x92926ab1),L_(0xe6fbe10e), LL(0xdc488cff,0x3f355ad1),LL(0xd2f1e956,0x84b0d884),LL(0x29d8cb57,0x98d39682),L_(0xae5bcaa6), + LL(0x07c8c79a,0x4957a527),LL(0x3263fbf3,0xec94fd37),LL(0x601424db,0xf147c20f),L_(0x01b4af1e), LL(0x1adc80ef,0x7d9440bd),LL(0x4bfaaf0a,0xee7355aa),LL(0xd1adde68,0xba4d5d2f),L_(0xbdf3e79f), + LL(0x4dc18b13,0xebadf0ef),LL(0x13192c94,0xbc8d4cc9),LL(0x8da37771,0xd555d58a),L_(0xebc86daf), LL(0x42fdc282,0x2e05719e),LL(0x8384340b,0xdc3ed2bf),LL(0x4c1b070d,0x03356492),L_(0x28e4bef9), + LL(0x8da99c00,0xa1e580f6),LL(0xf42f23ce,0xcf6d6c6d),LL(0xfdabb4f2,0x80f7da41),L_(0xf36b6774), LL(0x6a7dfe85,0x36467882),LL(0x8548109a,0x30e07e69),LL(0x943abc5d,0x8ae444d2),L_(0xe5bede0f), + LL(0x2441fb9a,0x0599179e),LL(0x18313f97,0x54669582),LL(0x47767b19,0x6fa66e89),L_(0xd078a973), LL(0x4596bd1c,0x93f0e068),LL(0xb6d46417,0xe4e3af1f),LL(0xff7a84e2,0x951ce962),L_(0xe6d6f09c), + LL(0x466ca296,0x16afa866),LL(0xd1fa4080,0x1f38aad4),LL(0xf4e1677b,0x672e7252),L_(0xf4b620a7), LL(0x3a4e5077,0xfcdff811),LL(0xf5e013a3,0xfe6adcab),LL(0x8a4243ce,0xb04666fc),L_(0xd68cd7b8), + LL(0xc1e32f39,0xd93030cb),LL(0xe0a61e74,0x15f3f603),LL(0x39bbca36,0x90c8c914),L_(0x18b00023), LL(0xa53fc682,0xb0c4475f),LL(0x4cc3825c,0xed39d188),LL(0xa57cf95c,0xd562a81e),L_(0xc8cb3e5f), +}, +/* digit=18 base_pwr=2^126 */ +{ + LL(0x02c6b67a,0xace24453),LL(0xde939bb0,0xa58b929a),LL(0xbe2f8e64,0x9a3a7efb),L_(0xa302408a), LL(0x7fa4b043,0x8ad04303),LL(0x56c2b048,0x426eb7ce),LL(0xa67998c5,0xdb040e40),L_(0x60850b87), + LL(0x3bfd76b3,0x0361658a),LL(0x20f7e490,0xa3b9f1bd),LL(0x66db4cfa,0x4799c86b),L_(0x82aaeafc), LL(0x01c04fa1,0x0689618b),LL(0xd1b3660a,0x5b052123),LL(0x524802bb,0x25748110),L_(0xd8501db2), + LL(0xb542168b,0xf1dd3dc0),LL(0x40fb7327,0x3556cca8),LL(0xf3b21a79,0x0d1838e7),L_(0x0263cade), LL(0xbbfa3610,0x43bd0f7a),LL(0xe6ca4a48,0xfdb15d30),LL(0xa293f5a9,0xf9285692),L_(0xf5e2782f), + LL(0x7da35373,0xefe3b9e7),LL(0x475d95b1,0x08b0fe30),LL(0x9c0da2e3,0xcef9bd5d),L_(0x69f4e019), LL(0x087c3eeb,0x4386ea9f),LL(0x107a9f5d,0x5543dce0),LL(0x057ad0af,0xc5162840),L_(0x036ea919), + LL(0xfeb313d2,0x398fba1b),LL(0x07cbfb75,0x52f53c92),LL(0x98909181,0xfa320278),L_(0x063675b6), LL(0x906d44a1,0x427688fa),LL(0xb499f521,0x1c7282db),LL(0x3901fb30,0xaea2389a),L_(0x14299eff), + LL(0x1424671b,0xd8f3ae0f),LL(0xb631e50c,0x80279f71),LL(0xd5a77238,0x602ce4df),L_(0x988af873), LL(0x12e9402e,0xdd095d1b),LL(0xc30915a3,0x6cb15ad1),LL(0x0927adfc,0xa38c1285),L_(0x3f7c6e76), + LL(0xec392753,0x1ec20f01),LL(0xeb41a6c8,0x5cac318e),LL(0x47520f8c,0x407d9032),L_(0x0390450a), LL(0xe2f3dac2,0x43dd79ec),LL(0xc468403d,0x9236a81c),LL(0x6bccf29e,0x00ac0a42),L_(0x713f0b03), + LL(0x163c66e1,0x849061a0),LL(0xa20a6586,0x70208a9e),LL(0x7bf95257,0x07a10de2),L_(0x3b8aa66f), LL(0x45c2891b,0x15468f60),LL(0xf59044a4,0x73d5ebe8),LL(0x387e234b,0xa32ac9f8),L_(0x4f7ec3db), + LL(0x962ab071,0x570fcd62),LL(0xbcce236d,0xb1757c3e),LL(0x1ad55d1b,0x7fa592e8),L_(0x1700716f), LL(0x76fcda7e,0x73d3b392),LL(0xdd37f1e3,0xd6cf8e98),LL(0xca16fca1,0xac456eec),L_(0x5752c0f7), + LL(0x5f929901,0x240b92b4),LL(0xef15652c,0x0f7d51a0),LL(0x632731fb,0xf12e9902),L_(0x38b796f7), LL(0x47671b3d,0x985803eb),LL(0x5fa89402,0x63e1c219),LL(0x608cf2d5,0x109df03c),L_(0x8ca4e888), + LL(0xb25c71f6,0x0829cedc),LL(0xc7a41eb8,0x483d485c),LL(0x15305f6f,0x60591120),L_(0xf6bc7367), LL(0xbcdea717,0xbeeee884),LL(0xe598a6e7,0x88e6ef42),LL(0xc3d80182,0x4b2dd1d3),L_(0x45cd3855), + LL(0x482c367a,0x6b421466),LL(0x8af36484,0xa58b570e),LL(0x2587649f,0x3e4ee4bd),L_(0xe423c77c), LL(0xf5a1388b,0x98494de5),LL(0x4a294a8e,0x8f25a37c),LL(0x50e9cc72,0x1ce8e0f0),L_(0x10660a4c), + LL(0x119587d4,0x8e668562),LL(0x167d0962,0x0554888c),LL(0xd291f0fd,0xae58bc85),L_(0x8d335bc2), LL(0x48a506c2,0x034f448b),LL(0xd5ffc68b,0x94dd32bb),LL(0x2bd2abc9,0xb9e588a6),L_(0x2eec6265), + LL(0xe1e13902,0x24b16627),LL(0x8bba7a24,0x4f52ca61),LL(0x749a749b,0x8bb11e14),L_(0xb6773c49), LL(0x3dfafe08,0x71bdac04),LL(0x76569dbf,0x40bd5231),LL(0x1587ab8c,0x7d2d4a5e),L_(0xfceafeac), + LL(0xe7ffdae7,0x0f11fde4),LL(0x936bcbdb,0xc78059f3),LL(0xf4713c18,0x2d2f0ff7),L_(0x0dce5266), LL(0x5fe86c6b,0xa381beae),LL(0x9b2c0aaf,0x734b9092),LL(0x77f56796,0x82131b3f),L_(0x02162cc8), + LL(0x612650a2,0xc29b399c),LL(0xad145b7e,0xd673cbc4),LL(0x60425f21,0xece9a8ea),L_(0xc5d38ead), LL(0x394b09de,0x1b69fef5),LL(0x55ee50f6,0x141fc2ff),LL(0xeab0baec,0x85999248),L_(0x02f743df), + LL(0xf4b4d1ac,0xe1f44cb9),LL(0x3931200d,0xd86fbb1f),LL(0x32861f51,0x242811d3),L_(0xa373e731), LL(0x20b39e71,0xc64c1332),LL(0xd9a6709c,0xeb68ae08),LL(0x93c66ba1,0x7aee0eb8),L_(0xfc730fa6), + LL(0xf47a18f8,0x992398ea),LL(0x454ef012,0x7414fdc9),LL(0x325c8a8e,0x42a7e664),L_(0x3b3fca9b), LL(0x9f053cb0,0xc8869d15),LL(0x34539da1,0x1878089e),LL(0x11f347a4,0x19ee0c05),L_(0xf6ff52d9), + LL(0x3e51ac6d,0xb254a9d3),LL(0x5029469e,0xe2b48c1d),LL(0x916d1214,0x3732c0ab),L_(0x07cba5dc), LL(0x0cb5ec91,0x010093ba),LL(0x6f2fe681,0x4a4cfff5),LL(0x4aaf2a44,0x7feda10c),L_(0xafba52c9), + LL(0x58625a1b,0xea895fc8),LL(0x19dbf9ed,0xcac0e4d3),LL(0x53bc48b9,0x1d43df08),L_(0xc8f9dd1d), LL(0xee75a211,0xe1a8bb95),LL(0x718f5878,0x7e7b6ce4),LL(0x4a384a6d,0x26d0db83),L_(0x97ecfe38), + LL(0x2bb95cae,0x9e42faba),LL(0x9254494c,0x0f33c611),LL(0xf67c4703,0x6ef93df3),L_(0xc1445815), LL(0x8ad685b3,0x8655685c),LL(0x6cd1cca8,0xdd2eb2bd),LL(0x5e194627,0x523e0fd5),L_(0x44f05a1f), + LL(0x301fcb43,0x80fb78c3),LL(0xa1be1446,0xfaa1d373),LL(0x3d0a850d,0x051bb6db),L_(0xc2bd0c81), LL(0xc7fc5cae,0x38f0078b),LL(0x6d01ae21,0x91a544af),LL(0x0c9540c5,0x6cae5443),L_(0x41626d37), + LL(0x93ebe56f,0x71946cb7),LL(0x3335ca8b,0xc82366c4),LL(0xbfe6dc77,0x27a97fdc),L_(0xcc4d8fd2), LL(0x80d9e528,0x16d42ed2),LL(0x4d8537e9,0x18abf493),LL(0x0398f8d3,0xa190c5d3),L_(0xd448e52b), + LL(0x495ac576,0x6bf10350),LL(0x3cbf1e78,0x2674f39c),LL(0x90ac41f9,0xaedc843c),L_(0x784b629d), LL(0x60ca4b9d,0x2b7ba708),LL(0x88cff37a,0xc070ae0f),LL(0x2e4b0764,0x0c3aa1f7),L_(0xdf186109), + LL(0x555cdffc,0xd2310fa6),LL(0xac832df3,0x42f99111),LL(0x9d84f404,0x8d7bdc74),L_(0x64936918), LL(0x43e3cff3,0x1c149edf),LL(0x3b328f21,0x442dde2f),LL(0x1288dcc9,0xb3089de8),L_(0xbad0531f), + LL(0x3fe61e60,0x20fa93a3),LL(0x494f3a66,0xb50f92b6),LL(0xbfce0907,0x632ab7a5),L_(0xe52a0324), LL(0xc89531e3,0x6f57401d),LL(0xca680133,0xf7bd6eaf),LL(0x5942987b,0xceffd87f),L_(0x75f1a1f6), + LL(0x74161b3e,0x647fc617),LL(0xe81422f8,0xac7f6cc7),LL(0xc485b2c8,0x0414a8d3),L_(0x5303bc61), LL(0xd7de72d1,0x1d6064d4),LL(0x88c0864e,0x9fd6a734),LL(0x8ee9cb9d,0xf104d7a9),L_(0x963ee732), + LL(0x776f7b1b,0x2b058af6),LL(0xdc25f3b6,0x0d06abe1),LL(0xb0461b36,0x90f6c358),L_(0xdad04c23), LL(0x9a72e439,0x24e0b184),LL(0x2af6be09,0x6facf219),LL(0x5f6753e5,0x5f23a218),L_(0xa362a760), + LL(0x4fd2cc26,0xadb7e3c5),LL(0xdb9150c6,0x4cc67b4d),LL(0x4c80d589,0x6a578011),L_(0x78347a1f), LL(0xc04163a4,0x41978a17),LL(0xa1df6b38,0x4701d625),LL(0x7acb9d4e,0x7040eb22),L_(0xb0f7aae6), + LL(0xb117faf4,0x734cd26d),LL(0xd884a071,0x071d299d),LL(0x1ad0de2a,0x826417df),L_(0xda3b0c06), LL(0x487c15f0,0x5432a8df),LL(0x85244fbd,0xa5c2986f),LL(0xe2f76737,0x89d1f699),L_(0xefb644c7), + LL(0xe366cc7a,0xcd96f1e7),LL(0x2f09e23c,0x19aa24f5),LL(0xbe89a16c,0xea6d5b99),L_(0x1a7565fa), LL(0x1afa0bd5,0x8da4ca09),LL(0x8baaef4d,0xb278c555),LL(0x07697581,0xea0b2435),L_(0xbefa5c32), + LL(0x233ddda5,0xbce451f2),LL(0x6e777818,0xe21ca9dc),LL(0x33fc9573,0x608477ee),L_(0x96d9fe99), LL(0x37f4a8b7,0x2f253ba7),LL(0x11d98950,0xec1f4a44),LL(0x3ae7c4bb,0x1c649260),L_(0x4eeee3f8), + LL(0xf186ffa8,0xca88dd8a),LL(0x25be4bd7,0x2e8c5c07),LL(0xd65638d7,0x8d28689a),L_(0xbe778846), LL(0xa2ef4b72,0xfa5e89e0),LL(0x0b27fb66,0xea4b66ad),LL(0x2a17ea7b,0x5744fef5),L_(0xaf89c334), + LL(0x1c88c916,0xd377e6af),LL(0xa6b3f831,0xbd38c04e),LL(0x43c62134,0x7e11eacb),L_(0xaee51855), LL(0x3aa19a13,0xe65b9643),LL(0x3179a57d,0x0069d240),LL(0x7d7a8265,0xc6fc4f7d),L_(0x8ea00bca), + LL(0xb94a17c9,0x95308d83),LL(0xb3928b84,0x88fbad83),LL(0xd14a8ca8,0xcec36252),L_(0xff246228), LL(0xf71bccbb,0xd90035ec),LL(0x499776eb,0x55a73d3d),LL(0x07f0cdbc,0x6c21797b),L_(0x87db5d7d), + LL(0x5162797f,0xa87e9702),LL(0xaaf9d8c4,0x2ab93ff2),LL(0xe2a31c19,0x4288abf4),L_(0xc4cc32b4), LL(0x80be51bc,0x388680ed),LL(0x2f0c118b,0x8dedbe91),LL(0x7da1a829,0xbffcb08c),L_(0xd681d331), + LL(0xac1e2c01,0x1d6fb3ed),LL(0x470f2923,0x9f1545b9),LL(0xfb03795c,0xbfecfd6e),L_(0x8ff73fcd), LL(0xac0e74d1,0xcbb46331),LL(0xcbbfedab,0x01849564),LL(0x79c317d8,0x9186b0db),L_(0x7c549c1e), + LL(0xe3b9e5c2,0xa44cb510),LL(0x670d9ca9,0xc72bcde9),LL(0xd47f22e3,0x23ac79ad),L_(0xe4bd107b), LL(0x1062a84a,0xd5193e66),LL(0xfb47ce43,0xbb25e4cf),LL(0x600c0353,0x5ab3c0ce),L_(0x005311ba), + LL(0x1f36e1c7,0x2a414d7c),LL(0x9bec3869,0x23202a91),LL(0xbdd27859,0xd77095b4),L_(0x270698b1), LL(0xe5a99ae2,0xd0fffc9e),LL(0x3c7db5f1,0x490dac48),LL(0xacfdbe9c,0x9079d090),L_(0x1bb8021a), + LL(0x550c3e83,0x448361bb),LL(0xa680b478,0xfeab6ae3),LL(0x41969a52,0x74461138),L_(0x9a38f5c6), LL(0x47381f19,0x2be030b8),LL(0x0e86ce09,0xb091abda),LL(0x4da4cd53,0x7df8f1e3),L_(0xc85acbda), + LL(0x9fea92c4,0xab2fcd01),LL(0x85b0423d,0xd804cba2),LL(0xed8709e8,0x19141717),L_(0x49d8b7d7), LL(0x88fb9ada,0xe105506b),LL(0xb90d9a8b,0x8cb2c13d),LL(0x4e6bda23,0xcaba970f),L_(0x7dff6f41), + LL(0x9fcd4872,0x160191eb),LL(0xff541bf6,0x826242d0),LL(0x99634c20,0xdfe7289d),L_(0xa5294071), LL(0x4e07eba9,0xc0910d55),LL(0xd5b6e07a,0x7e042c86),LL(0x6d7f5229,0xd74d6e82),L_(0x42f50e09), + LL(0x6efbc0e3,0xcc3c4e8e),LL(0xd7d09e5f,0xf21ea1ea),LL(0x05c108d8,0x37aae682),L_(0xb8b85f83), LL(0xf2c37fd3,0x9a51bef8),LL(0x5a44835e,0x5ac834a6),LL(0x2a412b12,0xac985dae),L_(0xf2f75363), + LL(0x39bd009a,0xb0ef3648),LL(0x7df070de,0xc1936e74),LL(0x97226670,0x3eb64bfc),L_(0x0a7cc2d5), LL(0xfa065546,0x6cb29cd8),LL(0xd6b2e07d,0x010fbc35),LL(0x6b24b9f4,0x72e08b5c),L_(0xbf7187b5), + LL(0xe0daf351,0x2fb0ee94),LL(0x31222305,0xac57f680),LL(0xb753971e,0x6b0ccf83),L_(0x55a3abe2), LL(0xc3b1be68,0x48b857e0),LL(0xfc1add13,0x81e2b0c5),LL(0x595ebecb,0x7151293e),L_(0x43489ffc), + LL(0xb788bb44,0x65e9c6ef),LL(0x071c9990,0x10062256),LL(0x891a7719,0xe6f377fd),L_(0x65d3a9e3), LL(0x7355204b,0xe23556ab),LL(0xa6502b28,0x188872cd),LL(0x84eba15d,0xea03bca4),L_(0x62a2b9d4), + LL(0x159bb9df,0x08807416),LL(0x78e83862,0x496eba6b),LL(0x5b5b2f5c,0xff4f34bf),L_(0xc8432a05), LL(0x376ec0fd,0xe48904ea),LL(0x386ae415,0x96bf9fdc),LL(0x9b04b325,0x003a468c),L_(0x2a854bd5), + LL(0x3d1be5ac,0x62509881),LL(0x76fd690b,0x65d5f607),LL(0x244caa05,0xc38d7540),L_(0x7a21901b), LL(0x0b239ba7,0x19b8e994),LL(0x14c3955b,0x1f25f647),LL(0xd2cd0a34,0x7f481820),L_(0x3e5eb658), + LL(0xdceb71b6,0xf395c0f2),LL(0xcc104e0d,0x6febd95a),LL(0x01d8efcf,0x0b358ed0),L_(0xcfc537fe), LL(0xdadfd46d,0xdc5d961e),LL(0x8b4f6c1d,0x221b24a2),LL(0x6d21a125,0xc0f16b10),L_(0x36a0bac0), + LL(0x3ece8319,0x54252e96),LL(0x46f2cfb1,0x97edd630),LL(0x020e1e99,0x90685aae),L_(0xfc1bcb8e), LL(0xb7771435,0x6df64310),LL(0x43c4fd8d,0xa647e2b0),LL(0xd6ddd8f3,0x52ff5927),L_(0x5282fbcf), + LL(0x3fd7f727,0xbb02013a),LL(0xb1fc86fe,0x6e17def8),LL(0x5ef8f2c6,0x1b970bb2),L_(0x1b714b57), LL(0x1236fa50,0xf7a7e274),LL(0xcc10de34,0xffa4c977),LL(0x5a8179b1,0x99219388),L_(0x798129b2), + LL(0x4b407e5a,0x844408fa),LL(0x6be5d183,0x742669b3),LL(0x0751f86d,0xb00b365a),L_(0x34786bf6), LL(0x3a8360fc,0x4147cefc),LL(0x88816150,0x0393026d),LL(0x7a7f6231,0x2e39fa34),L_(0x879f391b), + LL(0x149f6c78,0xbbcd3afa),LL(0x8292ce76,0x742b5b3a),LL(0x899f0118,0xb8dc2766),L_(0x18090cc3), LL(0x14c00e4d,0x6924c12d),LL(0x7049ac09,0x4e4111e8),LL(0xdf09100c,0x2987ca68),L_(0xa8c45230), + LL(0x65fad089,0x01cff054),LL(0x821650cf,0x3002769e),LL(0xdb9e3912,0xe9ad46db),L_(0x1a05deca), LL(0x6e7044a0,0x34ab7ab5),LL(0xc61e42fb,0xfda4dfcf),LL(0x91982d5a,0x89e4ea03),L_(0x58b6c30e), + LL(0x132822ea,0x1ad76044),LL(0x50f3dcec,0x3e8a1d95),LL(0xb981b100,0x37f04df1),L_(0x4ca707b5), LL(0x0b17bd0a,0x0b34cd00),LL(0x93aa35e9,0x447278ac),LL(0x07bde57a,0x6e16122f),L_(0xc9b96d1e), + LL(0x3221832d,0x8fc6ce6c),LL(0xe4935439,0x65828539),LL(0x9cdf65d9,0x708e9258),L_(0x5929b880), LL(0x273611ed,0xef43a507),LL(0x9e4d7f82,0x86406043),LL(0x88f84624,0x4396769b),L_(0x09401931), + LL(0x25903256,0x6b8a2c7e),LL(0xe200a555,0x567e51fb),LL(0x06f1e0ff,0xdbcad515),L_(0x435d74d4), LL(0xe7fa9d16,0x627c717b),LL(0xd7c74d13,0x3efff1d7),LL(0xa592bf53,0xe58ce7d7),L_(0x0a6f5daa), + LL(0x6bc49db7,0x2993ad98),LL(0x11f1de05,0xcd8ed14b),LL(0x104437d1,0x419b6d76),L_(0xb80b6ad9), LL(0x5ceae532,0x2692fc9f),LL(0xfc4d4052,0x96208c11),LL(0x5f17f19b,0x34a83cc6),L_(0x7f3e627c), + LL(0x3d238cb2,0x6bb879b5),LL(0xb752157e,0xeaa5d8ca),LL(0x137d82f5,0x769e2a68),L_(0xca3db7bd), LL(0x0ddf279e,0x2ae26372),LL(0xf5d74ea6,0x79e90c0b),LL(0x27e773c7,0x3451edcf),L_(0xe4341440), + LL(0x65ddd79b,0x34988e35),LL(0x25d7df53,0xd93aac10),LL(0x814a0e96,0x76ec0b96),L_(0x74d8980e), LL(0xe6639b9b,0x9143cd9d),LL(0xa3c743dd,0xfb9b096b),LL(0x77b915d6,0x4353464a),L_(0x7fdf9660), + LL(0x103b0149,0x52d3720b),LL(0x668172a6,0x3c3a2d72),LL(0x0ddae18b,0x3543af35),L_(0x36c45c71), LL(0x90846158,0x51cb6d6c),LL(0xb970c6e8,0x5541df27),LL(0x9f0df2e6,0xa96cf0e6),L_(0xef118589), + LL(0x51a147b4,0xbd4c751a),LL(0x9547874d,0x5b132575),LL(0x81839452,0x9b0ddfdb),L_(0x24fa7687), LL(0x2e14f24d,0x54dad63a),LL(0x7d8859c9,0xd7c2228f),LL(0xdcbca7aa,0x77c13acc),L_(0x87a21fd3), + LL(0x4c326ed3,0xe3bb354a),LL(0x3fb8d397,0xa148004d),LL(0xb7008ce3,0x85367d7f),L_(0x177a646b), LL(0xe73d1a40,0x93d871ad),LL(0x4c9f1031,0x7768c968),LL(0xd9c87876,0xb38a617a),L_(0xd2b0142c), + LL(0xaf8f400e,0x2a94dc1a),LL(0x9b975861,0xf8d2e935),LL(0x370e4c06,0xca35bd61),L_(0xf7a243e9), LL(0x2c1682bd,0x00b6e1a8),LL(0x56f9e8d0,0x6d92e1f3),LL(0x9bb1736d,0x3e9b6293),L_(0xf048cf7d), +}, +/* digit=19 base_pwr=2^133 */ +{ + LL(0x6c35afbd,0xf6f30e96),LL(0xdab13683,0x669a8251),LL(0x4dd8bbbe,0xb9b1b311),L_(0xd57a4a34), LL(0xbbcbf46c,0xe4d59c99),LL(0x4923f91d,0x63242254),LL(0x877172dc,0x853f9c6a),L_(0xd7aca10a), + LL(0xd5de71ea,0xe7f42edd),LL(0xf74adcc0,0x0b9c53be),LL(0xc8c4ee60,0x82b1e37e),L_(0xbcd67b5f), LL(0x4a10fc2a,0x7502a8c1),LL(0xba4a9d44,0x2dc30076),LL(0xcd3e0527,0x2e0b7f76),L_(0xaeb6d158), + LL(0xc21b4aac,0x95f715ce),LL(0x5afa8860,0xff0073cc),LL(0xdb4528e5,0xdbc69fc1),L_(0xe9e0b780), LL(0x8bfc4102,0xe2c44c67),LL(0x74449572,0xa7624b30),LL(0x1b45072c,0xebf2704a),L_(0xae9e87a4), + LL(0x540d3efa,0xe3c8a7e4),LL(0xd1ad9579,0x39e285d6),LL(0xccac1ee5,0x8d098605),L_(0x02e00e9d), LL(0xaae2485d,0x424cb101),LL(0x57781fcb,0xd81ebf02),LL(0xe045acd6,0xec4b922d),L_(0xf679db8b), + LL(0xe3ace3b4,0xfb5f0701),LL(0x7920ba12,0x1534cde4),LL(0x25904a79,0xf9421b0d),L_(0xc0438d82), LL(0x0528f7cb,0x8a9c39d8),LL(0xb1c7fc33,0xdd383932),LL(0x458a7c5f,0xbaf06aaf),L_(0x2cd8a058), + LL(0x193b780c,0x5abd6eef),LL(0x6b220286,0xd76b78fc),LL(0x06442ef6,0x0cdf9e58),L_(0xe520f054), LL(0x8a5c2658,0xfae0a442),LL(0x0ae1c405,0x6c2b778b),LL(0xb5adbc98,0xf374d05a),L_(0x010faa2e), + LL(0x0c6f4ae3,0xb657d5f6),LL(0x56b561b2,0xabf1920f),LL(0x70a6775e,0x15909e3c),L_(0x7b91068f), LL(0xe0c73f44,0x563843f3),LL(0xe4066970,0xe15912cf),LL(0x2e80308d,0xd16c8835),L_(0x3121c5b3), + LL(0xb00d284a,0x96ae8e24),LL(0x8746c0ee,0x118cc562),LL(0x4ed443bf,0x9157acc3),L_(0x898cd623), LL(0x14b54c13,0x07d6ccc1),LL(0xc5299940,0x05ec4684),LL(0xf50f2de3,0x1a923061),L_(0x8aaedd52), + LL(0xc692cc03,0xbe07dcf7),LL(0xa61a9062,0x5f04bf17),LL(0x0d99300c,0xa062deba),L_(0xcb7f082f), LL(0xa3451ea6,0x02666d4c),LL(0xd4b6a94d,0x2fc9077f),LL(0x564f33fe,0xa2bd47b9),L_(0x4614aa04), + LL(0x6da6ee28,0xdab5f17d),LL(0x2dfb0cbf,0x36f25faf),LL(0x16a32d1e,0xbe5fdc6d),L_(0x48f97f60), LL(0xd0d79fc9,0x0ee979a6),LL(0xfb77bc6d,0xc670e326),LL(0x6435ef83,0x55425ae2),L_(0x32e4cd8a), + LL(0xcf395e17,0xd8954092),LL(0x1415743a,0x516938e6),LL(0xc57f3db0,0xe85d4e50),L_(0x71f22b3c), LL(0x611ea790,0xfafbe19a),LL(0xdbd2d4ed,0xe5ff8f06),LL(0xe41d9e6e,0x9e9c0da2),L_(0xb15426d4), + LL(0x86e3cdb6,0x8cb41178),LL(0x0f8b5836,0x5615e2b5),LL(0x74d58d7a,0x4e6b82ce),L_(0x3030ebba), LL(0xd4c6f918,0x374bbfd9),LL(0xcfea53e3,0x15aadf45),LL(0x363a6cec,0x30dd6649),L_(0xf3c95a58), + LL(0xe43c3adf,0x120bb9ae),LL(0x98ad19ec,0x73ef55fc),LL(0xbe8a8508,0xbb18ab2b),L_(0x1cda9cb5), LL(0x84facee2,0xb277aa2c),LL(0x29318829,0x742a25ca),LL(0x1cc06cae,0xd0d24d20),L_(0x3eca1b2b), + LL(0x0714ac72,0x2ffe0519),LL(0x522d36e5,0x843d58c8),LL(0x7eddcb6d,0x1e60abe4),L_(0xc55d6e85), LL(0xb10fd185,0xccf9c609),LL(0x5c6153bd,0xa0d4b626),LL(0xe398ed34,0x7b6aa811),L_(0xb4dde424), + LL(0xa960c0c3,0x735c5d62),LL(0x8c6a5476,0x0f8b2a35),LL(0x54dfa089,0x0b186f7a),L_(0x6ab8d48a), LL(0x3e7eb8a2,0xc1614c55),LL(0x7aebdcff,0x8c7e1e1c),LL(0x390e7c1c,0xf9b0b2f9),L_(0x869d2266), + LL(0x11277d6a,0xd9cc122b),LL(0xb86417ff,0x3f2ab21c),LL(0x2cbd87e4,0x494cfb1e),L_(0x991ddd86), LL(0xae7bbb90,0x9977c3ac),LL(0xc3e66760,0x397ea438),LL(0xb224446c,0x7ed9dec7),L_(0x2a81eaf1), + LL(0xc59accd2,0xb5432f0f),LL(0xf2593b02,0x8b88a936),LL(0xc5589b93,0x02d9f9ac),L_(0xc3eb3094), LL(0x8302d1bc,0x43908c95),LL(0x08ef4088,0xb05f1091),LL(0x83349c39,0x5061c674),L_(0xff815654), + LL(0xf0350dc9,0x9371eb01),LL(0x45527229,0xce86a9ba),LL(0x47861a0c,0x9aad7247),L_(0x098cb788), LL(0xa559d5e9,0x979093ec),LL(0xbbd1def2,0x0fe17efb),LL(0xb512322d,0xae7b8c8e),L_(0xe9bbc66c), + LL(0x3ad82961,0x9f1ec261),LL(0xfe0fcc30,0x77a36e95),LL(0x3914c9d2,0x9b1a4daf),L_(0x09933bfc), LL(0xec701059,0xcfa77625),LL(0xea8985c9,0xe60b3f29),LL(0x0c5b4abf,0x138b4b1c),L_(0xdc77bb16), + LL(0x174442e6,0x1802aa28),LL(0xd075865e,0x26f71e88),LL(0x4a4c5681,0x7ae33021),L_(0x5a2cabc9), LL(0x5d0cad44,0x2893d185),LL(0x55811da2,0x7e9947d1),LL(0xc9f7bf57,0xf2c79dea),L_(0x8ef92a2a), + LL(0x3a838f3b,0xfbe1be02),LL(0x57b6d887,0x229d491d),LL(0x207a3044,0x9f62e957),L_(0x9f828177), LL(0x2e1d3c54,0x86b7e5aa),LL(0xf3280135,0x3fbda02e),LL(0x7c96f0d5,0xbda44f9b),L_(0x2f723070), + LL(0xf1aeae6a,0xae77f574),LL(0x7664fba7,0x0423c396),LL(0xf1f4d153,0x47451433),L_(0x91ff938d), LL(0x3a807b63,0xe8e961f8),LL(0x748646da,0xd89c3d2a),LL(0x467e86a7,0x6ac80359),L_(0x32bf2f50), + LL(0xb36ecce7,0x3b7324a4),LL(0x156202b0,0xbb9babf8),LL(0x98c1904a,0xf1e380ce),L_(0x46867bcd), LL(0x01b08225,0x2eeb6052),LL(0x1eab01ca,0xd0d4fef8),LL(0x19322045,0x256a539e),L_(0xfc3623ca), + LL(0xc94659b1,0x2f0568f5),LL(0x16a9b3fa,0xed8a79a8),LL(0x50d8c266,0x757c7d99),L_(0x2300fd6a), LL(0x66c509e5,0xb792533a),LL(0xb5184b40,0x9e0c5aac),LL(0x34caeea9,0xe5b193ca),L_(0xbbba3e14), + LL(0x195602fc,0x701a9997),LL(0xf834de05,0x3992cbc9),LL(0xe1e8b8e7,0x8ec536e5),L_(0x7e773653), LL(0x29c79b3a,0x29bff6f9),LL(0x17984cef,0x596ff95f),LL(0x0a0ae347,0x661b4a6c),L_(0xb2e9719f), + LL(0x8e767a04,0x473ec4b4),LL(0x4c2d676e,0xeb4d7d16),LL(0x98a8ebb4,0xc4e28600),L_(0x2f68f9cd), LL(0x6beb5ab8,0x1ef6cbd4),LL(0x85cd9d62,0x83dde9de),LL(0x1762ff65,0x9ddef882),L_(0xeb7899f5), + LL(0xe6f97e53,0x2cfac1c4),LL(0xf67b6c86,0x94800529),LL(0x61546993,0xcee96715),L_(0xf2a9560b), LL(0xdc612b8b,0x01036872),LL(0x88524fe9,0x9e6d56f4),LL(0x0543bf02,0x40b8eb7d),L_(0xb87c4dba), + LL(0xbb67b974,0xd7ceddb4),LL(0x1d6817da,0x7bd3859f),LL(0xe95a7ea2,0xc85424d7),L_(0x55d605cd), LL(0x5d21362b,0x2591a7e7),LL(0x4aa1f58c,0x52817220),LL(0xc9d8453d,0xa286acd4),L_(0x3bd2f891), + LL(0xaf830116,0x6e587151),LL(0x8a5ecfca,0x3d8da1ef),LL(0x7afbb075,0xf8e3bab4),L_(0x80131b4d), LL(0x3ab8344d,0x6bb840b0),LL(0x9e591ccd,0xae7bd8be),LL(0x73177038,0xed8b2ca2),L_(0x15e01308), + LL(0xadd91144,0x680d82d3),LL(0xd982abff,0x88f503dd),LL(0x37393204,0x746b676f),L_(0xaf201907), LL(0xaf979504,0x5a9be716),LL(0x4001d634,0x0e682183),LL(0x609ab748,0xfe8299c8),L_(0x39a6e2fd), + LL(0x193fd321,0x8682c4c0),LL(0xc2a26123,0xd566872d),LL(0x1407f3c3,0x16439d5d),L_(0x3caca395), LL(0xd6726482,0xb7bc2fb3),LL(0x5cdf7fec,0xb4c2b53d),LL(0xfeaaece6,0x4349edc0),L_(0x53d16a46), + LL(0xae9bd278,0x6c678048),LL(0xa706ff5d,0x52da3781),LL(0xae4a71b1,0x1e1cd89e),L_(0x9b61b4af), LL(0x2f691fc7,0xf69884c3),LL(0x4d64cf90,0x19020a41),LL(0xdda1c483,0x77d02be4),L_(0xb0f86443), + LL(0x8529164d,0x5d80aafd),LL(0x4f432d73,0x1e830c7c),LL(0xa9f0f09e,0xfc3140e2),L_(0x815acd77), LL(0xdc2ab04b,0x59befe02),LL(0xf1e93bd2,0xa44595d5),LL(0xc1e15529,0xf19abe7c),L_(0xaeca237e), + LL(0xa12bccee,0xb9b8ec3a),LL(0xf2dadd08,0xad1e4dce),LL(0xb63e0921,0xfcf65288),L_(0xc5a2a0be), LL(0x863fe95b,0x5cd2ee3e),LL(0x4eb4db06,0x60da7b42),LL(0xb8a5bd09,0xc426f7df),L_(0xf3f19cce), + LL(0x6e5213a2,0xfd69d7c4),LL(0x31496bf2,0xd30cf23b),LL(0x37d44e1e,0x7097147f),L_(0xae15b228), LL(0x12d98f15,0x32dad5dc),LL(0xa8f63aff,0xdfe1a802),LL(0xc8d86798,0xeccd5411),L_(0xcde1acb9), + LL(0x74006fc7,0xd7d803d8),LL(0x4af70159,0xec84dc01),LL(0xae309a70,0xa34e0e32),L_(0x585516f0), LL(0x235ed632,0xcfee558c),LL(0xcab23f91,0xcd97224e),LL(0x1640e7f8,0x92280213),L_(0xbd508662), + LL(0xc7ccf41a,0xd687e53b),LL(0x854bd98e,0x9a24176d),LL(0x01470242,0x01aae44b),L_(0xaca3a958), LL(0x9fcc8bad,0xc772707a),LL(0x1ce44586,0xeeb32998),LL(0x1fd556e7,0x6b1da8b4),L_(0x0ed6e63a), + LL(0x3fe0c228,0x929fe46b),LL(0x01aa7a0a,0x55829b86),LL(0xb0593728,0x46968b3a),L_(0xef6bc08e), LL(0x861c28fb,0x49238b1c),LL(0xbb911c6f,0xe2c2f08e),LL(0xc3669826,0xd635b05a),L_(0x67f9c7c0), + LL(0x1cbd7072,0xac172491),LL(0xa75f68ed,0x70337bd5),LL(0x5806f542,0xba62de08),L_(0xf48e9477), LL(0xc789f89a,0x778e6f6e),LL(0xca466063,0x7cb61f89),LL(0x9e11ca0e,0x54802afa),L_(0x00753b33), + LL(0xa293c1d0,0x98d81eae),LL(0xf1d86049,0x4a05fd64),LL(0xf7776c88,0x31edd830),L_(0x4b09f540), LL(0x6b003742,0x530442c5),LL(0xd29f0459,0x2717506e),LL(0x993316f5,0x977de534),L_(0x8efd4242), + LL(0x1f4b647c,0x63a5f62f),LL(0x1b839fad,0x184bd1bf),LL(0x9eb8bb02,0xd878238c),L_(0x8ce0fbd0), LL(0xee715bb5,0x0c09795f),LL(0x7e0f0890,0x4a0ef110),LL(0x5e80a1d2,0xa6ed70be),L_(0xdd973aaa), + LL(0x02e27464,0x66b5fa13),LL(0x8d608579,0xf1afa0fe),LL(0x262f06e4,0x55328bec),L_(0xe5f137a6), LL(0x359db889,0x56654a88),LL(0xd39b145c,0x31866e5f),LL(0xe08705c4,0x615a5b05),L_(0xe8573a34), + LL(0xd9bae78a,0xa18e5f26),LL(0xbcdd1733,0xfb66461e),LL(0x52d89638,0xf79de1ee),L_(0xd8be4f6d), LL(0x63f080c5,0xf0c1209c),LL(0x4e984349,0x8927edaa),LL(0x1cca87fa,0x0d7779de),L_(0xc4be69f3), + LL(0x4ffe196d,0x591b255f),LL(0xf9220e24,0xf529da70),LL(0xa919472f,0xc172114d),L_(0x12b84dab), LL(0x727df75e,0x0b0bba01),LL(0xc90f5813,0x18ae3fac),LL(0x7de531b5,0x60f6010b),L_(0xb124c090), + LL(0xc8ab7f3a,0x4d9c100a),LL(0x3147c457,0xb3ed07de),LL(0x2ce9d1a7,0xf4f877cc),L_(0x11395ce0), LL(0xe5d2ab31,0x7ebabf38),LL(0x7ffb38b5,0xce3cfa6e),LL(0x85ffa6d1,0xfa3306d0),L_(0x40ce1960), + LL(0x014e69fb,0xe40ec6e7),LL(0x92b1fc6d,0x8f04f43e),LL(0xf3db5670,0xb9883ee7),L_(0x01e865eb), LL(0xeab3fc5e,0xedf8720a),LL(0x639d496f,0xf1d8e66c),LL(0x32cbb568,0xc1af6691),L_(0x0ce5c8b4), + LL(0xb78b642b,0x65a42ec3),LL(0xade40c55,0xdde23958),LL(0x216642c3,0xcdc7c4de),L_(0x6aa44ccd), LL(0x82ee3789,0xec771036),LL(0x1fafeef8,0xefdcbefc),LL(0xe82b1033,0x5b9fd0f1),L_(0xb5931c71), + LL(0xd7c8b169,0x94661f99),LL(0x88be16da,0x968a0513),LL(0x5e1f223f,0xf2936613),L_(0x7a916253), LL(0xf109d748,0x8c1958b8),LL(0xca3eb1cf,0xd1baaa06),LL(0x58b0e5bd,0x04265560),L_(0x6b971198), + LL(0x3c1c78f3,0x73aa994d),LL(0x7b9ba5eb,0x9d13c878),LL(0x5ffbf962,0x329365bf),L_(0xfc14c225), LL(0x151d1b5c,0x9b162681),LL(0x8b322eba,0x67d4adbd),LL(0x2d7119cb,0x112111da),L_(0x14cee07e), + LL(0x8de63a53,0x0bc054e9),LL(0x090ebf32,0xac3474f8),LL(0xd39c2e22,0x8a24d244),L_(0xc5c07687), LL(0x72de574a,0xf7cd28ca),LL(0x00f6286c,0xaba378ca),LL(0xacf3d959,0xc4c178a4),L_(0x1e3ce962), + LL(0x4c196a72,0xeddb3096),LL(0x4807ba1c,0x78d05007),LL(0x4a5cea44,0x4e82cf55),L_(0xbefd7586), LL(0xbc157856,0x42628bb0),LL(0x4d8a1ece,0x3bec9a03),LL(0x414298ee,0x71c76fb4),L_(0x1b7c9c7e), + LL(0x451c89a9,0x0eb5ca81),LL(0xdfcc6c7e,0x85231811),LL(0xdd102f79,0xcfe70175),L_(0xd23927f1), LL(0x74f56210,0xa68f72ba),LL(0x0c268cfa,0x107305c7),LL(0xa0b90dac,0x71b34217),L_(0x86344588), + LL(0xdf55e61f,0x74013fe7),LL(0x0ad28f61,0x384b1c58),LL(0x117f4d5d,0xb417f0cc),L_(0x2ee194ab), LL(0xa5e101d2,0x6a59354e),LL(0xe4650ef7,0xdd613a28),LL(0x3d053f50,0x49698601),L_(0x211980e7), + LL(0xbc28da29,0x350510df),LL(0x729451ca,0xa5492aa2),LL(0x488b3ebb,0x7335580b),L_(0xf1e1f254), LL(0x150fa7dc,0x5080e530),LL(0x021a9cf2,0x41fe0adc),LL(0x151c555b,0xefba1701),L_(0x1be676fd), + LL(0xda6ab8ee,0x01c286ca),LL(0xd11e769f,0x8f870f9f),LL(0x67983e93,0x011517ba),L_(0xb19fd8df), LL(0x2ffdf94a,0xb8ff2b68),LL(0x1becdb97,0x57a7f08b),LL(0x2df51505,0x7f4a552d),L_(0x2e44b40c), + LL(0x8a38d901,0x15f78485),LL(0x94822c36,0x40936fbc),LL(0xdc403083,0x26d9bf80),L_(0x3f811b6c), LL(0x6682494b,0x658eecc3),LL(0x1088923f,0xe2c9cc47),LL(0xf4a7dc3b,0x6742c074),L_(0x7820af17), + LL(0xaf6f6f62,0x1be73cf3),LL(0x0c32e84f,0x74ffb428),LL(0xb29ede9a,0x92e686e4),L_(0xa74c4c0a), LL(0xdfa5e890,0xe1001a22),LL(0xa64183c0,0xf029cdcb),LL(0x55c88ddd,0xd03747ad),L_(0x5bbce6dc), + LL(0x69bc7a5c,0x0a7f469b),LL(0x3d28b7fa,0x8cb61ff4),LL(0x860090cf,0x25e6acf5),L_(0x3ca61e58), LL(0x7c9a5e59,0xd04ea633),LL(0xbdcdff22,0x3dccdf34),LL(0x679b9a9c,0xe0d5601a),L_(0xa518db92), + LL(0x75393ffb,0x503ff7fc),LL(0x29c7bd8b,0xad84a980),LL(0x6a8d3ebd,0xb711116c),L_(0x6093e9c1), LL(0x508cd2db,0x298e39d1),LL(0x88ffc5c3,0xd182ffce),LL(0xf35bcd9e,0x56f60d5a),L_(0x24c2d62a), + LL(0xbb37476d,0x126c6b62),LL(0x1974758e,0x12e961ad),LL(0xc23fa1b7,0x620a85d2),L_(0xe713a9df), LL(0x18d50f3d,0x90b6bf4d),LL(0xd31ffadf,0xcce6cb43),LL(0xb9db57be,0x43232f2b),L_(0xee956b0f), + LL(0x33f18557,0x34242913),LL(0xae8e0c9c,0xecfa2ca9),LL(0xe5298223,0x54c9b89f),L_(0x81ae98bb), LL(0x2bb85e08,0x3640c336),LL(0x187eb8dd,0x23f72dcc),LL(0xe33bfd7f,0x1138d09f),L_(0xa5c71157), + LL(0x9635cec5,0x99decb38),LL(0x120fcaaa,0xb6e94853),LL(0xc90493fd,0xe03d588e),L_(0xdacd4c22), LL(0xbbcd5ead,0x1a5efdc4),LL(0xa034c92d,0xbd3e5f34),LL(0xc0bbee30,0x91354a2e),L_(0x83ce5653), + LL(0xe4fe7dd6,0x40ccc046),LL(0x12391a03,0xef734198),LL(0xe37fbe0f,0xd7d730ce),L_(0xb901d8d1), LL(0xe81b74b5,0xccbbf03c),LL(0xbd70b08b,0xaf1eca00),LL(0xb202f7d1,0x6c3a5371),L_(0x3e4aeff1), + LL(0x3c293b50,0x50ebe82c),LL(0x841f67d5,0xe5c199da),LL(0x9c0164e0,0x09ae68d1),L_(0x48a6a776), LL(0xe57a6b35,0x764aa682),LL(0x4cc646a6,0xd2b124d2),LL(0xc37e19f6,0x0684de69),L_(0x244cccbe), +}, +/* digit=20 base_pwr=2^140 */ +{ + LL(0x3a2fc0af,0x3a0ae219),LL(0xaf66dcd1,0x71d39324),LL(0xeb53d991,0x1b364d9f),L_(0xd1a4ab2f), LL(0xe634fa4c,0xfa979c3b),LL(0x0ae7194f,0x0315e213),LL(0x7a46c430,0x7d3dfc00),L_(0x3dda16de), + LL(0x69c6dab5,0xcad74055),LL(0xb4650e32,0x5f980b30),LL(0xf457bc61,0x08ab6cd4),L_(0x08a5060d), LL(0x5a3b5bba,0x4a0eec60),LL(0xc2d75bdb,0xa4e2c616),LL(0xeb13a610,0xea8ba726),L_(0x5b8d1d6b), + LL(0x8208d95c,0x1d126957),LL(0x852ba024,0x38314570),LL(0x2b3db8bc,0x68f43b86),L_(0x4e0c61da), LL(0x2fa1baa8,0xb3650b8b),LL(0x81c43c97,0x222d46b4),LL(0x5ab609c3,0xe25b7f78),L_(0xf9adc44c), + LL(0x2601c0c3,0x10c7e593),LL(0xa7b8c442,0xd433020b),LL(0x163a4b4f,0x6ea15982),L_(0x0f63c4b6), LL(0x559abf4c,0x168cc6db),LL(0x63fa3ddc,0x5e92d42d),LL(0xcad8eab0,0x0f718cd3),L_(0x285e6621), + LL(0x1eccb2de,0x45eee5ae),LL(0xbeed1a3a,0xf0f806c2),LL(0x93a2e216,0x0b896b9f),L_(0xaddd1455), LL(0x21ac6aa7,0xe860a374),LL(0xbb52f5cc,0xa6d03d5e),LL(0xc336e305,0x5a3ad08c),L_(0xd8ac8cf1), + LL(0x7e65ba98,0x927ff44f),LL(0x6eb2af1f,0xcfc2ebd3),LL(0x8e5841d8,0xcd919ed2),L_(0xfaf6b44a), LL(0xd72479fb,0xf9079703),LL(0x31c04f62,0x11b2590d),LL(0x12b5f89b,0x9e78739c),L_(0x829845bb), + LL(0x6497199a,0xcc5dc1ba),LL(0xc6068072,0xc7f1ee3e),LL(0xbbfeebe2,0xa475b47a),L_(0x497220e3), LL(0xd00f3343,0xaa2739d6),LL(0x7a34054f,0x6bcba57f),LL(0x08421638,0xf50c5c89),L_(0x5194e599), + LL(0x56ea720f,0xc14eb574),LL(0xd278e415,0xe5e93b7f),LL(0x1a141ce9,0xf206da59),L_(0xde031e92), LL(0xff6ca107,0x85cacb30),LL(0x27ecddf0,0xe54c35f6),LL(0x73777848,0xeecb8a06),L_(0xbffbbc51), + LL(0x27c35c5b,0x88bff7c0),LL(0xfa52e107,0x2aa31cbc),LL(0x84083f2a,0xee8007b7),L_(0xd8f67690), LL(0x7c2ce5dc,0x46d1faf9),LL(0x5603d9f9,0x0d96af5f),LL(0x7894ccc6,0x143733e6),L_(0x28c9191f), + LL(0xf8d1a77a,0x74b4b8c0),LL(0x4ced8b77,0xaa226357),LL(0x0f677dc4,0xbce88332),L_(0xc71a3df5), LL(0xa1eff8f9,0x6f4d4505),LL(0xf4ee4e98,0x37fa6b4f),LL(0x12414006,0x078af83c),L_(0x51f9c8c5), + LL(0x6038786b,0x043fa7bc),LL(0x495de129,0x39594053),LL(0xeeffb86e,0xf30130d5),L_(0x5566ee1e), LL(0x8e9bddbc,0x6b822690),LL(0x38bca4e0,0xfd18acdc),LL(0xf63c9ead,0xa29c50ef),L_(0x6ca8c69e), + LL(0x875129d9,0x23706eff),LL(0xbd7972c9,0x343d01b3),LL(0x0f0278ce,0xd1d2ff3f),L_(0x859707ee), LL(0x62e80eee,0x76f993b1),LL(0x46b5aa17,0xd8eb588f),LL(0xc6cb2916,0x3e93bfcf),L_(0x6b823d1c), + LL(0x40dc4150,0x02fc59aa),LL(0xa4e36005,0xf4b7afee),LL(0x2f863790,0xdf1eb2ce),L_(0xdcdc305f), LL(0xc2be20ae,0x05119c9a),LL(0xb011c039,0xa27da69c),LL(0x59ba9dd2,0x50be3370),L_(0xd25c56ab), + LL(0xe5b817ad,0x62b0b4a9),LL(0xe35b330d,0x0b744a17),LL(0x49eb11c0,0xdb8066e2),L_(0x8fd0b5d0), LL(0x60bc3f3c,0x9db8094e),LL(0x0164c3cb,0x1a336970),LL(0xe5728d57,0x98764bb8),L_(0x9ecea57f), + LL(0x471336b1,0xc97e0c25),LL(0x84d2cb69,0x40016340),LL(0xd00385b4,0x8681b939),L_(0x5ce0fa5b), LL(0xfb0504e7,0x190db5e8),LL(0xec267c2d,0x73d7879b),LL(0x5e7d8612,0x8f037a48),L_(0x0c242e37), + LL(0xb920e08a,0x81a11e46),LL(0xab88a28a,0xad7ce91b),LL(0xd873ff38,0x463f5eef),L_(0xb449cd10), LL(0xa997a44b,0x67e4b48b),LL(0x65bf00de,0x76b8b69f),LL(0xc007a35a,0x0ebb0cc7),L_(0x168c4cb8), + LL(0xc6331e3e,0xd49fdc60),LL(0x08277aa2,0x3f1c848b),LL(0xae53c6d2,0xa68e50ed),L_(0xf9940a04), LL(0xa6271229,0x1c7298ae),LL(0xc7e75832,0x34a3a581),LL(0x6666c91c,0x1bb3ce87),L_(0x9b5e6600), + LL(0xabcce7ac,0xfa1beb7d),LL(0xc3326260,0x9cab965d),LL(0x7c86ce27,0x145e311e),L_(0x1c036012), LL(0x3f4563cb,0xbd40ee7b),LL(0x7397bad6,0x077a6fa1),LL(0x6ab23a8b,0xdc9e2258),L_(0x524c8593), + LL(0x27ccdc45,0x8d1e93bc),LL(0x8122aeec,0xac370974),LL(0xf5fc7d6a,0xbb95cf97),L_(0x1eb1cdc1), LL(0x65e7b71f,0x0d3d8936),LL(0xa618f61f,0x6cfc89cc),LL(0xcfd74f9d,0x46c8cccc),L_(0xc39ab4da), + LL(0x82c35cad,0x4b6e4cd2),LL(0xb8deb535,0x5bf959ed),LL(0x77b74152,0x14153467),L_(0x1a6510e5), LL(0xc645fe15,0x44ed7180),LL(0xd0e5fe61,0xb00a2c9e),LL(0x94c68e2e,0xcd40eadd),L_(0x54367d74), + LL(0xdb397bc6,0x0bb14c2d),LL(0x90f3c232,0x2727c32a),LL(0xbdde6547,0xf5120538),L_(0x6f363a14), LL(0xca65a20d,0xb71dac7a),LL(0x0119f615,0x7b4ac903),LL(0xf1e5f556,0x83bcccf1),L_(0x78b79fc1), + LL(0x35177eb9,0xb2a6346d),LL(0xba547ec6,0xcdb1c427),LL(0xc5c607ff,0xf6acdcca),L_(0x272a5f1e), LL(0xf198276f,0x5ddfa215),LL(0x187de1bb,0xc1a3402a),LL(0xc070af80,0x0cf324a8),L_(0x8fba9bd5), + LL(0x4be4c6c1,0xddf4319c),LL(0x90adc217,0xcc96208a),LL(0xd1d87cd0,0xc3e2cfee),L_(0x76bd76ab), LL(0x1fa3dfb2,0x89c310af),LL(0x72ee71cc,0xab955eb2),LL(0x1e3849a6,0x4e9da2a4),L_(0x49c4cd74), + LL(0x4a409b87,0x4a32be16),LL(0x591f45c9,0x5593d29c),LL(0x1f66b255,0xa18a71de),L_(0xa8df9f2a), LL(0x871121b9,0xd5a52a49),LL(0x0518891a,0xdb77ca39),LL(0x35178007,0xe87f7b56),L_(0x236d66c4), + LL(0x7b11bf26,0x78cced92),LL(0x100d41cd,0x568cf974),LL(0x1aebc2f4,0x90437303),L_(0x4f4abe7b), LL(0xe1d4cf61,0x81708f04),LL(0x6b6039fa,0xfcd5e395),LL(0xe8e310f1,0x67164e9b),L_(0xc7f1e7ae), + LL(0x4435eb40,0x309bf5d1),LL(0x2117b17c,0xfa094c38),LL(0x0f0f519b,0xa7712b6a),L_(0xa339b91c), LL(0x13b443e9,0x3fc80f73),LL(0x5eca3c74,0xc2640658),LL(0x6f1b036b,0xcc979824),L_(0xf9c46f37), + LL(0x2c10fe83,0xfd4280b4),LL(0x61dfd82b,0x0f01bc50),LL(0x8ebbdd0a,0xe8a94761),L_(0xea097594), LL(0x3331db36,0xe9e32c12),LL(0xe3fdca90,0xdfdd390f),LL(0x7d9acfa0,0xf46a833d),L_(0x0a373092), + LL(0xb1f7d641,0x50d5c66d),LL(0xc56a23dd,0xd311bbb7),LL(0x2df06964,0x4d4065dd),L_(0xb24fc29d), LL(0xc5f2e7e8,0xa84be5a0),LL(0xee36de4b,0x704fb714),LL(0x72ab97c4,0xe211a72c),L_(0x4e457419), + LL(0x22e459d1,0x262ef282),LL(0x57520b17,0xcdae125f),LL(0x400cd2e5,0xc83f210b),L_(0x219146d8), LL(0xb3e5fe6d,0x87c8591a),LL(0x2cdbe07a,0x83fd93d1),LL(0x072fa5b0,0x7d701f58),L_(0xe67d391b), + LL(0x2dda42fc,0xa6490705),LL(0xbae6af5f,0x5e3ec835),LL(0x974661b3,0x77bc552c),L_(0x51a8bdf3), LL(0xf43d8648,0x351407f7),LL(0xfaacee32,0x93522ec0),LL(0x2effc179,0x49a92d02),L_(0x35e7a36b), + LL(0x1178f9fe,0x88ddd25b),LL(0x29a04cd6,0x4985102d),LL(0x0d3de452,0xf0277afe),L_(0x52955ade), LL(0xf3571745,0x53a80f8c),LL(0xcd0ea6e7,0xa11d84cc),LL(0x8cf3c9a4,0x7920692b),L_(0x66ad31de), + LL(0x3259f508,0x8dd2cea1),LL(0x147b087a,0xb1de59c3),LL(0x964e43aa,0xd9cd158a),L_(0xe62a2c56), LL(0xf7812a2a,0x483a8ca3),LL(0x133f5131,0xf93b681f),LL(0xeafa2982,0x22d0852e),L_(0x47016331), + LL(0x4e3898e8,0x178b364e),LL(0xdedbf0d6,0x1e46cbf2),LL(0x07005a6a,0x017cc249),L_(0x3350187e), LL(0xe3d63871,0x9df8269b),LL(0xa447b67a,0xf32797a4),LL(0x1360de40,0x70a39a32),L_(0xe07cb8f9), + LL(0x58005eff,0xefaea42e),LL(0x83a181cc,0xcdf53ee0),LL(0x8d9f77b8,0x2be3de21),L_(0x199ee148), LL(0x219fc4d7,0xdfbb4054),LL(0x5fd887bc,0x370b0057),LL(0x0df48ecb,0x67c50061),L_(0xceadf6f4), + LL(0x7d760757,0x3cb96c10),LL(0x8354395a,0x87378266),LL(0xc87eb4e6,0xbb1b1686),L_(0x2ca2a9d8), LL(0xf625cc5e,0x7f6f6bfb),LL(0x375d90fd,0xd1702bfc),LL(0xd7459a05,0xb5757892),L_(0x8e648246), + LL(0x2285b7c6,0x77777ad0),LL(0x98514815,0x2a9f4626),LL(0xa5cb03f7,0xbdaaa412),L_(0xf84efd29), LL(0x6fb99eec,0x8aa2c4b5),LL(0x9cd6f177,0xec729246),LL(0x9b9f8d87,0x995887b7),L_(0x2944bcdb), + LL(0x786ccb7d,0xae2ffd66),LL(0x8f43b748,0xb566147f),LL(0x0b294654,0x42cc5e69),L_(0x8eb5ab4c), LL(0x1634f1db,0x21f0caa7),LL(0x3cc24e4e,0x80410ce5),LL(0x6597b349,0xb9e50c3a),L_(0x390fd5fc), + LL(0xbefea27f,0xca541338),LL(0x25552e16,0x0bc9bd26),LL(0xd300e1fc,0xed2c5b84),L_(0x2e61d461), LL(0x731c6b5e,0x10fed3e2),LL(0x3f6fedc9,0x6d0357ab),LL(0x30c7c4b6,0xfdf04b77),L_(0x6ca06d8d), + LL(0x6198d0c0,0x49f0ac87),LL(0x355910d8,0x5908bfa5),LL(0x2b72f231,0x1e0626b8),L_(0xcd397171), LL(0xb70c5fde,0xaffd0d74),LL(0x4feccf28,0x30464156),LL(0xbe7c0da7,0xf7d2ffb9),L_(0x782a9bb8), + LL(0xd71029f7,0x2ac8267b),LL(0x58c7cc85,0x4f8afc3b),LL(0x2d787218,0x10f9a786),L_(0x005ab707), LL(0xf19c8b55,0x5209c3c7),LL(0xa8e944c7,0x0212109d),LL(0x4e16a382,0x3cd77b54),L_(0x4ca72d58), + LL(0x4e89b492,0x77501651),LL(0x30cb8e0d,0xdbb5243c),LL(0xd24a6908,0xc09c5df1),L_(0x10b46bf0), LL(0xeb16d02e,0xb28822ac),LL(0x790e9a17,0xbf46b5ba),LL(0xde8a18e1,0x47566077),L_(0xd3c60b3c), + LL(0xa4a4f17a,0xdff0cf1f),LL(0x0aef91c0,0x34f0c71a),LL(0x3fe5a3b7,0xae1fff30),L_(0xf6c8d6d5), LL(0x95b01983,0x5317a578),LL(0x10e5c9f2,0x8872e118),LL(0x154f511a,0x28ad7f46),L_(0x1f7f1f26), + LL(0x3c4162c4,0x88636424),LL(0xf620131a,0x69f32e24),LL(0xf3a9fd92,0x049b5c0f),L_(0x6242ff79), LL(0xd8613d4e,0xcf94b297),LL(0x0250d486,0xf99f0a15),LL(0xf6d69ab3,0x8d722881),L_(0x726c3c6f), + LL(0x2829f721,0xc6f3cc19),LL(0xf0ba6d0a,0xf0c5f76a),LL(0x0d44b294,0xda7df6fd),L_(0x98458d66), LL(0x35516899,0x4c9c487f),LL(0x541b8cf7,0x312edd80),LL(0x14a64ecf,0x2db649b2),L_(0xfcc41f23), + LL(0x4e65d481,0x17bb4dfc),LL(0xd8c4ee32,0x7b8827b1),LL(0x6ffab99a,0x788afe5d),L_(0xb6cc5390), LL(0x6ed82d81,0x75adf2d6),LL(0x91030ac7,0x99ce22b0),LL(0x957976c9,0xf8f37372),L_(0xeae6d45f), + LL(0x3ad3db42,0xa4617ca1),LL(0x9ea768ee,0x8f228cec),LL(0xc881f9da,0x86ddd01a),L_(0xddfb0090), LL(0xd1467d75,0x41eb80aa),LL(0x1320c7b6,0x875ce5dd),LL(0xaf17e700,0x398d4c2f),L_(0x91ca1818), + LL(0x165575ce,0x7c2f636f),LL(0x94407a75,0x89f1290d),LL(0xeeb00b1e,0x2cccc148),L_(0xc1a654f8), LL(0x7200e28d,0x5409cce2),LL(0xd579aa30,0xf8e920e0),LL(0x3586fdc4,0x11976312),L_(0xd9b1a394), + LL(0x45934ad1,0xa0a5bb1b),LL(0xba2becd7,0xbcff29b1),LL(0x9f749e30,0x2f73a7d6),L_(0x54ae60bd), LL(0x9ed5bbc9,0x111b6548),LL(0xae179cab,0x7b77f3f8),LL(0x9b568171,0xf1563350),L_(0xd57cc18c), + LL(0xbde3fb89,0xaa153efc),LL(0xe84063ac,0xf24dc768),LL(0x91718221,0x3207691f),L_(0x666ced3f), LL(0xa962b884,0x8a1416f3),LL(0x81f644db,0x2992be1f),LL(0x388328fb,0xa7c4a359),L_(0xebba06d7), + LL(0x709d7f63,0xf678058c),LL(0x9bd30551,0x9db19686),LL(0x8d882bae,0x81afb521),L_(0x6d103aa5), LL(0xce4ef2a5,0xc6ecf132),LL(0x681bea3a,0xab9bbd78),LL(0x3831d678,0x4c11a697),L_(0xb9d4a72d), + LL(0x8a72049d,0xc3e5138e),LL(0xa25affde,0x9235657e),LL(0xf870194a,0xeddbefb7),L_(0x2583cc8f), LL(0x85d843d9,0x912ed2a5),LL(0x57849cc4,0xdb45568e),LL(0x453cbf16,0x410259cc),L_(0xcf9369cf), + LL(0x10263ac8,0x69fa2805),LL(0xf7446e45,0x6120289a),LL(0x86d5285d,0xece730a0),L_(0xc0ee3533), LL(0xbf1197bc,0x286f4724),LL(0x878ea926,0x0b0ef425),LL(0xf397181d,0x24bb4f4f),L_(0x542c7100), + LL(0x162331a9,0x536a5ea6),LL(0x0e466905,0x88c29411),LL(0x3da5a14f,0x9cb76097),L_(0x1db565b9), LL(0xef6b8094,0x59e86f3c),LL(0x31eb3c89,0x817764eb),LL(0x58eed1b6,0x8010fc3d),L_(0xaeba318a), + LL(0x9a5b60b8,0x108b3068),LL(0x1ffea59a,0x1312c3e1),LL(0x8dd4b81a,0x0253e9dc),L_(0xf4ed9d4a), LL(0x5eadccd2,0xbd75013a),LL(0xad3eaf49,0xc60daec9),LL(0x20b2a500,0xbea0d3fb),L_(0xdac00c65), + LL(0xfb3fb2a5,0x72db2c7a),LL(0x566902bc,0x864ce0fe),LL(0xe0956799,0x2d7ec9e2),L_(0xdfb7dad8), LL(0xdfc7056f,0x77bfd4dc),LL(0x7e188701,0x0f42ffde),LL(0x110fb370,0xf2040e52),L_(0x7944c336), + LL(0x2100e0bb,0x6fd68890),LL(0xf771684b,0x67978ca7),LL(0xb3a94049,0x279171ad),L_(0xc7c765dd), LL(0x02304590,0x41165c1f),LL(0x0299e83b,0x8eadb8c9),LL(0xe9eaef42,0x2345e24e),L_(0xb4a1f1f3), + LL(0x41f15f2a,0xaa889c09),LL(0x2cb037eb,0x6adf563e),LL(0x559edd7a,0xdd414cb0),L_(0x601d6943), LL(0xad8084b7,0xbef296d5),LL(0xb70a236e,0x0ee342a9),LL(0x074068ff,0xc9f3a4e6),L_(0x7b2eab9c), + LL(0x4db1a4b4,0x983e29af),LL(0x50522f93,0xc26d69a6),LL(0x8e174c76,0x2739f19a),L_(0xc8bc9b25), LL(0x4c902614,0xcd082b89),LL(0xa5f7fd65,0xa0661a39),LL(0xd0352acc,0x6d5bd738),L_(0x15c9e55c), + LL(0x0f46deb3,0xe05a9d3e),LL(0x25c49119,0x68d899f5),LL(0x2fcbd485,0xd426fbf8),L_(0x94e9a11d), LL(0x70b0b8e8,0xbb075381),LL(0x21ac6e8b,0xeb323221),LL(0x933c7339,0x90f21e83),L_(0x03161725), + LL(0x65c3ea70,0x26f5292f),LL(0xbb26aa3c,0x2f10ff0c),LL(0xd0e0dfce,0xe59106aa),L_(0x5bc81388), LL(0x334ef2fc,0xe1d13288),LL(0xe91c8aea,0xb2dd6c7f),LL(0xe941b704,0x428af6d0),L_(0xb82ceae2), + LL(0xf4357fa7,0xe9634ed9),LL(0xea606688,0x92f270b1),LL(0xb8fe4e04,0x4ce90044),L_(0x53c7eb07), LL(0x9137e5a7,0xb5cf536c),LL(0xc36a01d3,0xaa1f4551),LL(0x0024f90b,0x981784b9),L_(0x3bab3f51), + LL(0xb04608df,0x5269aa1a),LL(0x0cf9cff7,0x8ce1ebee),LL(0x3d279465,0xc66381e8),L_(0x61b7ab00), LL(0xc4fa7ed7,0xdeb29303),LL(0x6b7bf135,0x510a3c8c),LL(0x2ce166ff,0xaa227120),L_(0xf4a128bb), + LL(0x992398c7,0xa3c32177),LL(0xb5766593,0xcd19850c),LL(0x70b1a326,0x9505296f),L_(0xe3747c6e), LL(0x41bcad27,0x0d11515a),LL(0x290e370c,0x04883b49),LL(0x5a5bf602,0x72d1e180),L_(0x6a7dda52), + LL(0x5898ed4f,0x63896a2f),LL(0x0341c2af,0x82a0ed92),LL(0x71349776,0x71b9b0ee),L_(0xa68523a0), LL(0xd360630d,0xbd440771),LL(0xc3982209,0xa49907d9),LL(0x28e79328,0xad498ec7),L_(0xde9039d6), +}, +/* digit=21 base_pwr=2^147 */ +{ + LL(0x5e338a95,0x0324aba6),LL(0x3d1721e0,0x970769c2),LL(0x32ea5aeb,0x82c3b64b),L_(0x62043072), LL(0x80346bd2,0x98111ddd),LL(0x629d0540,0x2ef9e31e),LL(0x75819a03,0x6e0e0fd4),L_(0x01541785), + LL(0xd481d37b,0x676ef7cb),LL(0x6e38c822,0x6ddd542a),LL(0x4bcdf56d,0x0807b6ee),L_(0x152d3a20), LL(0x99cfbaa2,0x440a4586),LL(0x1b803f58,0xaab76f82),LL(0x62bde53d,0x31ca293a),L_(0x774dabf4), + LL(0x279a665e,0xc73f3e56),LL(0xd125dbe9,0x51cce02a),LL(0x925d55fe,0xba48033b),L_(0x77120d9c), LL(0x3c3cd000,0x09219d1a),LL(0x9ef0977f,0x7bba7cbf),LL(0xf5c59984,0x1f9a5c08),L_(0x043d05c2), + LL(0x4079f18a,0x150e6d32),LL(0xfb5764ea,0x8b6cc92a),LL(0x2d425400,0x274ec0c2),L_(0xee8a4646), LL(0x6d261b02,0x4ef086f3),LL(0xf2d17e5e,0x29dec1b8),LL(0x7f963f71,0xea97e94e),L_(0x70fad827), + LL(0x9e51713a,0xf9867b99),LL(0x07c55712,0xbd760d63),LL(0xafdf07e9,0xa9b6e9ec),L_(0x2af4df6e), LL(0x6d61dc08,0x93df1b2e),LL(0x743dfab5,0x3387d03a),LL(0x32d7f165,0x759fb34d),L_(0x749bb511), + LL(0x08345265,0x34f3b1e1),LL(0x599ac0d2,0x20468106),LL(0xc8425e31,0xa035ed92),L_(0xbe2663eb), LL(0x15129a67,0xd78546b5),LL(0x6589a68a,0x2a03f31f),LL(0x04faf0ec,0xf88d60ed),L_(0xe6fe6524), + LL(0x99a20d9b,0x31f03c7f),LL(0x93d3e67b,0xbdab0f5d),LL(0xcfc1004b,0xc665b5f0),L_(0x3425e2f4), LL(0x57bee517,0xbaab1072),LL(0xf467b795,0x64de4d9f),LL(0xa4efa899,0xb3c284d3),L_(0xdb30b893), + LL(0xb6ef6e50,0x7bd6b2eb),LL(0xe6144c26,0xb87b9f50),LL(0x3506344d,0xe38cc59c),L_(0x136536d0), LL(0x98614bb5,0x4daab44e),LL(0xb52c6949,0x3957de52),LL(0xc680567a,0xcfa42f96),L_(0xf20dca71), + LL(0xad09cde7,0x89749594),LL(0xf7758fa7,0x686f22a7),LL(0xe7736848,0xd741b21a),L_(0x916557d5), LL(0x08ab4ccf,0xb87c7bef),LL(0x6a38311f,0x982331fd),LL(0x177b6473,0x63140d6e),L_(0x61df2b77), + LL(0x061731b0,0x785b32f1),LL(0x92de60e5,0xc296cafa),LL(0x1de52e72,0xb10ce663),L_(0x4c01f4cd), LL(0xc6a06419,0x50016567),LL(0xdcbb4a2a,0x4502900a),LL(0x2c001a59,0x450b28d5),L_(0x3a580e42), + LL(0xc5913491,0x9e1e0ce7),LL(0xfef505e8,0xc5f9091a),LL(0xe7590273,0x44739896),L_(0x35472c73), LL(0x449da237,0x3913a8d2),LL(0xbf43ca41,0xd9db43d7),LL(0x0204d37f,0xbcc28709),L_(0x11a4ed34), + LL(0x0ba00095,0x00c25d60),LL(0x00ebff6a,0xb0ccb9b4),LL(0xaecf0812,0xebecd41b),L_(0x02599546), LL(0xe2cb6f9d,0xe9f56be8),LL(0xc9f2f401,0xab12b175),LL(0x77e2c3e0,0xa7a303fc),L_(0x6a047f0f), + LL(0x0ca6bd42,0x829f1bab),LL(0x4982f7c1,0xf8aa85f0),LL(0x63a37ba6,0xe64c6ed3),L_(0x55b4405c), LL(0x9ba22c57,0x14c74bf2),LL(0xd812115d,0x57cdabdb),LL(0xa2f36ff4,0x69603458),L_(0x4e2488fd), + LL(0x049ce7fa,0xdda045cf),LL(0x3c6cc5ab,0xae642723),LL(0x444a4ca8,0xc9e8efd0),L_(0x2e1a3d2a), LL(0xee8ff913,0xa5f0a182),LL(0x3921bdba,0xf34c0a0e),LL(0x0bdcdecc,0xf174835c),L_(0x438b3f48), + LL(0x262941ff,0x50dd89e9),LL(0xb28cdfa5,0x52bfaad5),LL(0x3d3651a5,0x42da9f6d),L_(0x52955126), LL(0x72f24772,0xfb5c67bc),LL(0x10d91fd9,0x6f3ebdef),LL(0xa8074e96,0x5d95664e),L_(0xe7fca6c3), + LL(0xfbd1a192,0x14bfa4e1),LL(0x11f314f6,0xf38ce345),LL(0xfcdb2abd,0x45ebd21e),L_(0xc304c980), LL(0xb59f480b,0x3957e934),LL(0xe9a58ad5,0x7c1b8e5a),LL(0xe4604f76,0xc58f2c48),L_(0xe27bf6c2), + LL(0xc8a88d73,0x21606f53),LL(0x6659ed53,0xfba9c9f7),LL(0xfb7d83f0,0xf57dce48),L_(0x2d2bf81f), LL(0xfb80fd50,0xc8023a07),LL(0x2b42338b,0x71a25cd3),LL(0x7a9a0b44,0xb3c48734),L_(0xc8125d7c), + LL(0xf3798b63,0x9856b7c0),LL(0x914f7c82,0x04ab2721),LL(0x471ef449,0xe5f3587d),L_(0x77a7fe57), LL(0x21b96d5d,0xb0c73d23),LL(0x1ec93eff,0xa3366107),LL(0x69323a1b,0x759bf017),L_(0x3f60e38c), + LL(0x80978741,0x4e3559fa),LL(0x56b197e8,0xe7559080),LL(0x56ad6e70,0xdd9852be),L_(0xeeb6dec8), LL(0x0e7594df,0xe4523a8a),LL(0xc67e9c41,0x4a40d4e4),LL(0x8ca5687b,0xfaf902fd),L_(0x731b0560), + LL(0xd2ade9c2,0x683ea003),LL(0x3784426d,0x6bbf8ead),LL(0x26a2a12e,0x65fa7d53),L_(0x7ab25cd8), LL(0x6154c3a9,0xd7f809d8),LL(0xe2d2306a,0x20ed9166),LL(0x24d19af8,0xa692a83a),L_(0xbb68e143), + LL(0xd8e0ff43,0x90e27172),LL(0xf7b7a48d,0xa90bf44c),LL(0x8cb2abd0,0x130d21df),L_(0xd9526e86), LL(0xa3cb4c6b,0x65186f8e),LL(0xe78f59ce,0x0c9c373c),LL(0x3514478f,0x5307b692),L_(0x15b86f33), + LL(0xd8568306,0x34f0a25d),LL(0xea579beb,0x039f50cb),LL(0x0eadf2ee,0x6a3cf739),L_(0x7fffefa0), LL(0x6a9a2a62,0x5c707fb6),LL(0x1f9a5c8a,0x22360abf),LL(0x756df232,0xa7a12676),L_(0x735ae0c5), + LL(0x232b0eaf,0xfac26fa0),LL(0xc57f8429,0x91c0d8b2),LL(0xc85b9885,0xcd36972d),L_(0x5ab11f6e), LL(0x2b28cccc,0xa2197bb4),LL(0xcf7b4e95,0xb00034b6),LL(0x22200d9d,0x398b8d08),L_(0x0660c348), + LL(0x3e7e898b,0x14230734),LL(0x8aa5dec9,0x7bef4c46),LL(0xe3a1abc8,0xb3564180),L_(0xb7f989a9), LL(0xeed81407,0xa7ce6e62),LL(0x98270e80,0xf02dd5b1),LL(0x9d22967d,0xa3ff259d),L_(0x4be6673e), + LL(0x25d2230c,0x2d777318),LL(0xd815968f,0x80147bab),LL(0x6f460f52,0x22279805),L_(0x99635b0b), LL(0x20550e62,0x7880c924),LL(0xf836942d,0x6428d5fb),LL(0x9a19cfda,0xd1beb76b),L_(0x819b5402), + LL(0xe36b37d4,0x5d87cbda),LL(0xe5f505ca,0x404781e7),LL(0xc8f5e0fe,0xa1bc9eda),L_(0x45ef1d15), LL(0x23299059,0x970d11fc),LL(0x54084d2b,0x785e8445),LL(0x8ed8b412,0xbad1345d),L_(0x2975a901), + LL(0xce7aa169,0xd74f6437),LL(0x62092837,0x1f1dc772),LL(0xf497718e,0x36827efd),L_(0x15977763), LL(0x978b617d,0x4cb29feb),LL(0x65c2897d,0x6a64b733),LL(0xe04077a2,0x86b8cf3a),L_(0xac075b5b), + LL(0xed710d98,0x3a39d170),LL(0x1f51c85b,0xef236d5b),LL(0x0683a7e0,0xb9141322),L_(0x7e27c47c), LL(0x05109f74,0xdade7c00),LL(0xe2f07be3,0xf251df44),LL(0xc11452de,0xb48e9a05),L_(0xd6a44d08), + LL(0x4aaf45f5,0x17039bf0),LL(0xbdcaec9d,0xb97b72ed),LL(0x30ce8053,0xa7a81755),L_(0x54e63f42), LL(0xc81db67b,0x80753cae),LL(0xc22c28cc,0xe3019a4c),LL(0xd1c81400,0x80f8c3da),L_(0x5a1e70b6), + LL(0x165046c9,0x030a06a1),LL(0x0ea30c39,0x09c6053c),LL(0x3932bda3,0xcb39f998),L_(0x148a2b68), LL(0x87fe7977,0x0499f81d),LL(0xb74e61d3,0x7ae298cf),LL(0x2959c264,0x1edb6b93),L_(0x6a2188dd), + LL(0x6ab52745,0xdc732139),LL(0xd70d9371,0xbd7da6c2),LL(0x5ac1ef95,0xe2e0217e),L_(0x6e9b8907), LL(0x63f9997b,0x5cecf1c0),LL(0x24552abf,0x5082d35e),LL(0x7b11e2cc,0x04cc9aa3),L_(0x78bdf631), + LL(0xfecff2a7,0x4f8b44cc),LL(0xe83e0eae,0xd783714b),LL(0x9c0232a4,0x90d1c36e),L_(0xb7e1ea84), LL(0xbd5e004e,0xf22c1c14),LL(0x99441dcc,0x77b24ec6),LL(0xc4fac858,0xf15be1db),L_(0x52f8f005), + LL(0xfa634b74,0xc11ab7e2),LL(0x865bc26d,0x39a0b0f1),LL(0xa62656eb,0xdf5e8e89),L_(0x5bbda8fa), LL(0x35af495b,0x059e7fc2),LL(0x996cfbbc,0x5721bb40),LL(0xfcc7a8b4,0x18530aec),L_(0xef18ab08), + LL(0x1efc0865,0x7e7fb20d),LL(0x1432863a,0x319cbee0),LL(0x32603543,0x216780a7),L_(0x3173509d), LL(0x771c5009,0x332b0e8d),LL(0xfb32ae5e,0x27842538),LL(0xfd61d211,0x176fee74),L_(0x8d713db3), + LL(0x6d5e6c04,0x8f5f36df),LL(0xfd53bdb1,0xff86482f),LL(0x04fc8d4b,0x35ae923b),L_(0x9c93970b), LL(0xd03159ae,0x6c4574ce),LL(0x0c5ab9f8,0x95e44ad6),LL(0xecc940e1,0xacd13e60),L_(0x73d20a97), + LL(0x24bc05c8,0x55d8d9f7),LL(0x1be2d507,0xba1700c6),LL(0xb1dbe56e,0x12671c92),L_(0x515125ba), LL(0x7c873619,0x28f4b581),LL(0x00d413e2,0xc27395ee),LL(0x31a82a43,0x84bd2bdf),L_(0x5c6f966e), + LL(0x19e04eb8,0xd6d7b2f3),LL(0x32f059d3,0x7d71e73c),LL(0x5dd55589,0x6b4ddaaf),L_(0x7a059ec2), LL(0xb3874052,0xb395499e),LL(0x4cfc04e5,0x56ce4cbb),LL(0x5961b289,0x2960f1ca),L_(0xa3443d73), + LL(0x739615c6,0x72405988),LL(0xe58efd36,0x03300370),LL(0xe254b688,0x6195855d),L_(0xf750a8c1), LL(0xd298a296,0xf9205a13),LL(0xfa55931d,0x11ffb780),LL(0xe00ca07c,0xd638a7b0),L_(0x1b178a9f), + LL(0x970fe956,0x62d115d2),LL(0xe1985dec,0x4db47694),LL(0xb3652081,0x6565367b),L_(0x61dc5dcc), LL(0x53a17e00,0xbb00a05a),LL(0x390808ef,0xc7f0a623),LL(0x2971e39b,0x091d591c),L_(0xb6590fcd), + LL(0x3c51221d,0xd273fed4),LL(0xb849d7b6,0x7eaf4c1f),LL(0xc3cadac4,0x155b21cd),L_(0x21bf043c), LL(0x824bf43a,0x6d0eb93f),LL(0xa560a086,0x2b5af50c),LL(0xe79201e3,0x50f56685),L_(0x7193da80), + LL(0xaf65e93e,0x64cb8361),LL(0x097bd142,0x2b08724e),LL(0x5d31d941,0x128434b7),L_(0xd3455462), LL(0xd87b7934,0x6349bb61),LL(0x5338e12b,0x8eabd8cd),LL(0x46f9abc2,0xf5ec4590),L_(0xee7ff24e), + LL(0xde41b0d8,0xfcdd55ff),LL(0x390333ed,0x5f8a8d73),LL(0xad3ba05b,0x5c84894a),L_(0xdd94d8c3), LL(0x1041454f,0x7425541f),LL(0x4f6bd774,0xf6c08550),LL(0xbe1bce56,0x48f611af),L_(0x723a0cc8), + LL(0x31e207ab,0x505aaa25),LL(0x6f2c316a,0x9df3ed0d),LL(0x6ff3aa7e,0xecdfa689),L_(0x0c314ac1), LL(0x4660e302,0xaac1b188),LL(0x74b064f0,0xe2709e66),LL(0x1690d7e4,0x9ad43bd5),L_(0x19d6932b), + LL(0x9e4966da,0xe9803ed7),LL(0x5acbb8d9,0x94a51c09),LL(0xc6ffb2f1,0xac8d73e5),L_(0xce57f6f3), LL(0x87ee51c7,0x0f2f29a5),LL(0x1c8a8a14,0xcf07fe4d),LL(0xf7993328,0x11a086fc),L_(0x816c9d1d), + LL(0x482c8577,0x7008cc6c),LL(0x3a6923e0,0x4ea34320),LL(0xea139dd1,0x0db793be),L_(0xc99cf72e), LL(0x2f88c486,0x46daeb88),LL(0xf235e9a5,0xa9529776),LL(0x8fc9a2fe,0x685903b2),L_(0x2e9cc9d4), + LL(0x160fbbb0,0x85dc20a6),LL(0xd5312a9c,0x40ce263b),LL(0x33ee3772,0xf18c955f),L_(0xb8cc18a3), LL(0x7c56f68f,0x629b2ed0),LL(0x381772f9,0xd3751f88),LL(0x9be35b3d,0x87658375),L_(0x438ba3ac), + LL(0x4c71f19e,0x2e6a930b),LL(0xbe1f2bc3,0x6ce370bd),LL(0x9c7c9090,0x9fe1a225),L_(0x9a8e7036), LL(0xdfc19acd,0xbc4095c2),LL(0xbf429df7,0x102343b7),LL(0xa07aed0a,0x75593f8f),L_(0x2c1e8aa6), + LL(0x6e87ec50,0xe6abe1ed),LL(0x6cc4e156,0x32e1d48d),LL(0x9c1aa3b0,0xbc16fee9),L_(0x76d25ea9), LL(0x1a098c5e,0xa549cfc8),LL(0x0fe8dc0a,0xdccd7aa4),LL(0xe2b2179c,0xf0addee3),L_(0x9bcbee00), + LL(0xec32e6a2,0x4e4d505b),LL(0xb0ad3d1f,0xd584c5e1),LL(0x708100e5,0xf05981fa),L_(0x5ca8160b), LL(0xff71c066,0x30b8c50e),LL(0x9b267d28,0xa5cf6579),LL(0x00705bdb,0x003593a1),L_(0x6e7726d9), + LL(0x1e13fade,0xbfa0d04d),LL(0x1fd12275,0xcc81bd28),LL(0x39bdb34f,0x7ba2330c),L_(0x9cb5c2f4), LL(0xb5b45e69,0xf331bfd7),LL(0x9513f2a2,0xdd8ec1b5),LL(0xb3f734e7,0x9064cd97),L_(0x5f64f559), + LL(0x6a4afc60,0x4194a70e),LL(0x63999525,0x523cff27),LL(0x03d08e2c,0xa27e7edc),L_(0xea90ab15), LL(0xfc7c7e87,0x38873c54),LL(0x398075fc,0x805abf3c),LL(0x414e3976,0x93991ee1),L_(0x66b18c5a), + LL(0xa5d47504,0xb3fe93fc),LL(0x9c7a5517,0xb7d168b6),LL(0x4a438e8c,0x581ebd7a),L_(0x30a1a792), LL(0x1e7c3d15,0x7749a42d),LL(0x2279fc57,0x86e109ed),LL(0x853c9b29,0x069c5f0b),L_(0x4438f7b5), + LL(0x2fb4329a,0x2b3c8790),LL(0x99bbcc79,0xc7e0dd07),LL(0x9d9741bf,0x51f341ad),L_(0xaedc9e99), LL(0x6eb8a76e,0x1cd0ce38),LL(0xbf6e3e65,0x2698a77a),LL(0x3099a97a,0x6b8968f5),L_(0x6186d5b8), + LL(0x5f3a4ac3,0xf86e098a),LL(0x6977fcd9,0x3194f379),LL(0xc98e48aa,0x886d05b8),L_(0x23139131), LL(0xbd9be137,0x488df6bf),LL(0x876a25e7,0x9833b00c),LL(0xfb9f6810,0x5fb8ce68),L_(0x1a974ef1), + LL(0x0996c57a,0xe74e34d6),LL(0x95d0e8ba,0xbc8f1cc6),LL(0x49ddbd76,0x3e421dda),L_(0x47780198), LL(0x2fcb399c,0xf599d67b),LL(0xb3cd50fe,0x1de7c3a8),LL(0x60ffdfcb,0x41018e1d),L_(0x2a84e15c), + LL(0x5803c901,0x1834f6f5),LL(0x85be2737,0x0269aa98),LL(0x51b8942e,0xbd9cdd7b),L_(0x2573f804), LL(0xf11dba7f,0x62d1e404),LL(0xa026f5f4,0xa355486e),LL(0x3e7a1a96,0xa6d151c2),L_(0x52ecef89), + LL(0x5bfcbb66,0xd3cf6d12),LL(0x7f3efdcc,0x509c593b),LL(0x36beb1c6,0x50bb9be7),L_(0x87aa0794), LL(0x461e9081,0xf6ebcef8),LL(0xb12b6973,0x53fb4ce0),LL(0x1d3a4bb3,0x84afbe92),L_(0x765f75f5), + LL(0x36690884,0x66809ebd),LL(0x7d2d307f,0x6d7b75a0),LL(0xb7561328,0x7a0b342b),L_(0x25a224f2), LL(0xc6b75bd1,0x8b464a87),LL(0x98a06fcf,0x145091bc),LL(0x33cace99,0x723fe6fa),L_(0x24207bff), + LL(0xfa386bf7,0x5ea0874e),LL(0x470c5915,0xa8387298),LL(0xbc4e6a04,0x02c73e59),L_(0x303d5c07), LL(0xa4b7c2f2,0x2fdbbf21),LL(0x2d418059,0xb933bbc7),LL(0x874e47bf,0x4c88b532),L_(0xbd3a8f84), + LL(0xb79dad0b,0x94b27e28),LL(0x4c3cb167,0x47dac83a),LL(0xb3a5c35a,0xace3bd6b),L_(0x36f5d64a), LL(0x0d91b2e8,0x00f91b6c),LL(0x561c257a,0xd6d7f96c),LL(0x390f1d25,0x92ea6b8b),L_(0xce74a256), + LL(0xa7f8fb96,0xf0e6068b),LL(0xbee4d1a2,0xed56b92a),LL(0x64cd2ac4,0xd3d78730),L_(0xb0ed4379), LL(0xf0d95042,0x8db9e812),LL(0x0e235698,0x6447eba8),LL(0xd45b6fa7,0x3b769af7),L_(0x95403694), + LL(0x26f167ea,0x8aa2617b),LL(0xdb91ef4a,0x52ead3d0),LL(0x41988291,0x5273a693),L_(0xab06325b), LL(0x18ca4d81,0xb1bbd253),LL(0xea972289,0x58878e32),LL(0x64d8aee8,0xe8908e43),L_(0xcdca72ce), + LL(0x5a517623,0x6d83d019),LL(0xd917cb4a,0x969711bf),LL(0xa7fc9902,0x225d81d7),L_(0x9b126599), LL(0x25981f40,0xe02b87e7),LL(0x04539a29,0xaec410d7),LL(0x35630262,0xb1115eaa),L_(0x7461aa8b), + LL(0x83f05d8e,0x4dc83cf2),LL(0xdbbadf9b,0x523db1cc),LL(0x09d299d6,0xe968d6f7),L_(0x46a4ef90), LL(0xa5406082,0xc545ab6a),LL(0x158f1789,0xcf6e254a),LL(0x592bcabb,0x599530e8),L_(0x4ecd132b), +}, +/* digit=22 base_pwr=2^154 */ +{ + LL(0xa46f08d9,0xb9fe1aa5),LL(0x27531029,0x32c26e96),LL(0x1b37edc0,0xcc6e653e),L_(0x25a6777f), LL(0x37d78fa5,0x44dcd6fa),LL(0x44e59684,0x637ed305),LL(0x72834a3e,0x6cab1cc9),L_(0x715fa76f), + LL(0x2efe30a2,0xd9e804cc),LL(0x1c1ded3a,0xb9c440fa),LL(0xb6ba8131,0xd2f7d77a),L_(0xe00e21fe), LL(0x88842b6a,0x96f7f20a),LL(0x616f6de5,0x9a618ffe),LL(0xf13ceac5,0x6577f60c),L_(0x9f1e4b0d), + LL(0xbd482557,0x1653fafd),LL(0x93eea72d,0x39c09ddd),LL(0x10b62eeb,0x79f6d014),L_(0x2f476cfa), LL(0x78d81b01,0xe8963c96),LL(0xc395c6ba,0x8c8f6a4c),LL(0x63b70385,0x551fa28d),L_(0x28bb3fc4), + LL(0x433a33ea,0x2417a7ce),LL(0x5591c910,0x1d9155b1),LL(0x7a6ca298,0xd78e16eb),L_(0x09ee38e9), LL(0x3ab289c2,0xee48adb0),LL(0xd5b76200,0xc998c5c8),LL(0x8dd6fbf2,0x959f66af),L_(0x3ee20307), + LL(0x23a0b23a,0x63c09a4f),LL(0x24b60423,0x021b685f),LL(0x7c69331c,0xb1b0820e),L_(0x7d3c58fb), LL(0x81992f9f,0x370cbdbc),LL(0x05814d87,0xeecaf1b3),LL(0xaeba071a,0x5ebd44d9),L_(0x079434ba), + LL(0x193df859,0x7a008e70),LL(0x9d5dd4b2,0x8d7ec280),LL(0x81377001,0x3062b2db),L_(0xe82fbf45), LL(0x11e91ce4,0xdb62847d),LL(0xf3a5dc38,0x9a492d59),LL(0x42b1e033,0x3c5e448f),L_(0x0028f02b), + LL(0x48abe62f,0xf00a790d),LL(0xf18c3029,0xe9c56ab7),LL(0xcfc6643d,0x7a101d9e),L_(0x69f64af5), LL(0xe901d5c7,0xc8b5cd29),LL(0x8ab0437d,0x5a6c0d3b),LL(0xf3eacfba,0x437c7e62),L_(0x9c1ff745), + LL(0x7ef25e16,0xc3c0b7b4),LL(0x3b2bb23b,0x2d411980),LL(0x9d372287,0xb86b3e9a),L_(0xbd217cd9), LL(0x6682e29c,0xc7ddb5da),LL(0xf55b3a32,0x80ad8745),LL(0xc5272ca9,0x67e42112),L_(0x80ddaf6c), + LL(0x2db21214,0x26d65fa2),LL(0x593699c3,0x27bc92ce),LL(0x780c1507,0x62bfbfb4),L_(0x4e610cd9), LL(0x90f53f42,0x306b2065),LL(0xdcabb7e7,0x70add873),LL(0x97f0fe05,0xef3f82de),L_(0xfd016e06), + LL(0xdb734eee,0x49ac57da),LL(0x33f4851e,0x0575235f),LL(0x12e6b4bb,0x84ba90a7),L_(0x0c414c40), LL(0x2e72cdc9,0xde178a6e),LL(0x5d680978,0x2c8f9b52),LL(0x04316bb2,0x4d9021c4),L_(0xd58bf8cb), + LL(0x4a88c720,0x7c1da7fb),LL(0x2166e51a,0x7be262ad),LL(0x26f4415e,0x8d16df29),L_(0x4308e38d), LL(0x6e047968,0xf7050746),LL(0x5191f59a,0x40048ee7),LL(0x671c053f,0x7e714450),L_(0x41cd1cf2), + LL(0x126208fc,0xc3f913fb),LL(0x77459414,0x81ebbd24),LL(0xfbf9d09c,0x60be0397),L_(0x5762fb0e), LL(0xab78592e,0xee7a8b54),LL(0x11f20203,0x58636e68),LL(0x5cb0cea7,0x56d9eb18),L_(0xaab58567), + LL(0xe2adb3e3,0xe08bf721),LL(0x8b739b3c,0xcf7b1348),LL(0x88809e01,0x9dd0883a),L_(0x0dd14479), LL(0xfb2d7aed,0x9e8ac3f6),LL(0xe9234e02,0x3aa9be66),LL(0x87a97238,0x56210cb7),L_(0xa8cca221), + LL(0x515e6713,0x989ff215),LL(0xd0bcb26c,0x8011a540),LL(0x16b0e517,0xc8b38bfd),L_(0x7706fd39), LL(0x23c98754,0xc13358b8),LL(0x8745e3cc,0x5edc147a),LL(0xae944131,0xbf36526b),L_(0xb43ce39b), + LL(0xbb74f4a3,0xf57da9fb),LL(0xcceacf96,0xe6c34f99),LL(0xae99e920,0x72b76a37),L_(0x7a9e0b84), LL(0xd46a4b67,0x377b9f4d),LL(0xd8d2d70f,0x2dbc27cb),LL(0x1c0d51f0,0x63d0ccbb),L_(0x423d586b), + LL(0x76b14e74,0x5173ac48),LL(0xda87fd76,0xac29f6c0),LL(0xde2af466,0xbf4405a4),L_(0x0df31635), LL(0xe0a96172,0xc72bdfba),LL(0xcbefcc71,0x14f78dd6),LL(0x2c991165,0xf55ecef7),L_(0xa3278f6d), + LL(0x6d393d5f,0x05e626ee),LL(0xb9f01d8a,0x225ed4d6),LL(0x53d41ea6,0x5d5b37e5),L_(0x6b1d2d7a), LL(0x70c4e493,0xf4527dfb),LL(0x465bfaf7,0x1522deea),LL(0x333509e1,0xad711cce),L_(0x988f296f), + LL(0x072fecfb,0xba0c11d9),LL(0x46a2c644,0xa6fa784d),LL(0xa105395f,0x3a914aba),L_(0xa42699f5), LL(0x5b5ec37d,0x6c8c0e8b),LL(0xffef0a76,0x79696f78),LL(0x69daa4dd,0xbf1748ba),L_(0x8134c310), + LL(0x16919c10,0x43014c2d),LL(0xa35261bb,0xd14b6a12),LL(0x6d705b75,0x395d7526),L_(0x58721301), LL(0x4c3aec80,0x3f4fd283),LL(0x2e6f84dc,0x4114b4f9),LL(0x6a63c71b,0xcf962aae),L_(0x58c876bc), + LL(0x085cede4,0x39a2603a),LL(0x21ba7529,0xb203a75d),LL(0xbdc1e809,0xf3640f89),L_(0x6631c30e), LL(0x8d9d5265,0xd480817b),LL(0xad93303a,0x8492b58b),LL(0xe3368ba5,0x24b8bfe6),L_(0x26c2b552), + LL(0x64ef9738,0x6907b7bc),LL(0xd795d794,0xf6e9c508),LL(0x33b42737,0x55ab9706),L_(0xa13daed3), LL(0xf586dde7,0x76b6d961),LL(0x0ff335ce,0xad1c2111),LL(0x13031322,0x1f36dd8c),L_(0x0558b72f), + LL(0xe83a229d,0xe6ae70d0),LL(0xf0770ab8,0x5b9ca0d2),LL(0x06479e73,0x17cdf808),L_(0x5e50cd46), LL(0xd020f0ac,0xca397995),LL(0x3337dcfa,0x70369103),LL(0x3d37e46a,0x67d7bcef),L_(0x5283dec8), + LL(0x920986d4,0x9e2b8580),LL(0x6f80e6f8,0x1a1086a7),LL(0xf038b9fd,0x76630d46),L_(0x4e9ef3ae), LL(0xf2780d64,0x479ba46d),LL(0xfa784b8c,0x41a1c68f),LL(0x953b6640,0xeee5927b),L_(0x9d4a23e4), + LL(0x7a6dcf80,0x2905feea),LL(0x2a632e73,0x5bba1fd8),LL(0xd274195e,0x596a73e6),L_(0xaf3000ac), LL(0xfa2e2813,0x83723184),LL(0x99dae0bb,0x21f09629),LL(0x7ce090e3,0xb6dc2849),L_(0x2bd68086), + LL(0xb254b1d5,0xfee577f0),LL(0x2ce6dbab,0x6accfc0b),LL(0x775f693f,0x483bf6a8),L_(0x829c24e8), LL(0x74582da0,0x6731bfc7),LL(0x8e7f87f0,0xf7d39568),LL(0xa67faa6c,0xc9418804),L_(0xf17ae402), + LL(0xb5eb4f29,0x7a12fe1e),LL(0x862fd52c,0x4a7fc70e),LL(0xd820b6bc,0x319719c3),L_(0x06ce9593), LL(0xbd26f429,0x66648099),LL(0xb9a6a586,0xb662d062),LL(0x7fbe15d7,0xf16d93b4),L_(0xab04debd), + LL(0x951453e5,0x8f99c3ae),LL(0x01cc4900,0x422aab2c),LL(0xec8547fb,0x73be771b),L_(0xfba8505c), LL(0x49e82e72,0x7b15c573),LL(0x238927ca,0xaa7e7ca5),LL(0xfd867aa0,0x58e19238),L_(0x16e92a3b), + LL(0x31e49c5d,0x2d0ee2e6),LL(0xe0a95c2b,0x3aaf91fc),LL(0x271cbc87,0xcb833e79),L_(0xcebcb341), LL(0x36952e47,0xf7a6bc06),LL(0xb8a1069c,0xda163496),LL(0x1f6ccb97,0xb5f271bf),L_(0xadf492cf), + LL(0x606962f9,0x82d18205),LL(0x983a462e,0x4a97cf98),LL(0xde06a403,0x23969744),L_(0x5ada65f3), LL(0xba5e083e,0x22229508),LL(0x7b88f442,0x376f004a),LL(0x1c53a79e,0x8ecb5e4a),L_(0xbc00e747), + LL(0xfde8f5b0,0xdaab7a45),LL(0x6896d43d,0x6cba2e3f),LL(0x0b7de7e1,0x31db2861),L_(0xc8a85283), LL(0xf98e6b35,0xa8d3e811),LL(0x9ed4cc65,0xc4961d3a),LL(0x4bad9114,0x10d64cdd),L_(0x6febb44a), + LL(0xadd9fbd1,0xca3624bb),LL(0x6bfd7f49,0x73a43aec),LL(0x91301b03,0xc382f632),L_(0xe009202a), LL(0x83cfde00,0xf9e8d9a9),LL(0x03cf587c,0xb9fbd92f),LL(0x25a8b014,0x760229d4),L_(0x62d554e5), + LL(0x95ce63b9,0x225d1531),LL(0xe0e28f19,0xf1862d71),LL(0x143d1930,0x89faba0c),L_(0x3d640e67), LL(0x36978269,0x2f3bfacc),LL(0xb38209aa,0x8ecd6947),LL(0x817ba1fc,0xad14a0f1),L_(0x53f8ce9d), + LL(0x9cf6404c,0x0fcc7993),LL(0x455cb661,0x94227ce7),LL(0x36c49b8b,0x1add1a97),L_(0xb4a122ea), LL(0xddf10443,0x478ddbe9),LL(0xd7d28b55,0x598d307a),LL(0xa7d77b58,0x7c2d4285),L_(0xf914cdbb), + LL(0x6ae40efc,0x7296f54f),LL(0x267136bd,0xbb8c6473),LL(0xa3248f4d,0x096d9531),L_(0x9bf47d4a), LL(0x72de2ce7,0x71d14af5),LL(0xe31fd133,0x8bb3132e),LL(0xd5a9918e,0x1ee08068),L_(0x4ad2e489), + LL(0x3a3e84c5,0xdb4c764e),LL(0xc0cdb80f,0x06967d90),LL(0x9a2fe8e7,0x3d0cd72c),L_(0x8105ce51), LL(0x663d93ac,0xd9a81336),LL(0xc772e40c,0xe092d50f),LL(0x7d2f5c90,0x45eb443a),L_(0x7ebd73b6), + LL(0x29b44f03,0x09859196),LL(0xd120b82e,0x274e7581),LL(0x8a9337d3,0xe9cba9ed),L_(0xb7d3254c), LL(0xab1d6ae2,0x1aa44fa6),LL(0xa2bcdd85,0x8201c08a),LL(0x2f9c1703,0xd128d342),L_(0xcf87815f), + LL(0x04dbb489,0x4b130afb),LL(0x8592bf1f,0x010d7b25),LL(0xf211bcec,0xb7f03740),L_(0x2070aa40), LL(0x24a2aec7,0xf116e286),LL(0x3a91fedf,0x4b6af9fa),LL(0x974e5b38,0x19cbf327),L_(0x0cab111f), + LL(0x8816e066,0x52f1347b),LL(0x0dbd4053,0x733b5b5e),LL(0xe33961b0,0x8db340e4),L_(0xc28fe0e4), LL(0x8fdad9c7,0xb3cc8f09),LL(0x66c9d4cd,0x36371a81),LL(0x36432409,0x9dd5d017),L_(0xfd34cf28), + LL(0xc45e9f49,0xce25491e),LL(0x464328d6,0xf8ea793b),LL(0x27eb6c22,0x6371b206),L_(0xc53f6320), LL(0x80e32e79,0x61eaf6df),LL(0x3ece8680,0xdc4f44ca),LL(0x387b7804,0x4b85bf6e),L_(0x4e286cd6), + LL(0x2c7dc475,0x18a6a123),LL(0x87751313,0x53665bc9),LL(0xaa43dbfb,0x8d40fe26),L_(0xe4bce37a), LL(0x6723ea8f,0xbade3e08),LL(0x4b0254d2,0xc7adbb73),LL(0x2f3a4dc9,0xa74a2565),L_(0x829b9555), + LL(0x9ab672f0,0x9cc7bfa2),LL(0x1fad100a,0x2ac7eb07),LL(0x368be66e,0x5665ccf4),L_(0x8056fdf1), LL(0xb250a7ac,0x465d7f98),LL(0x64180c0c,0x8d2b97aa),LL(0x7b979097,0xa50d476d),L_(0x45ad85c0), + LL(0x8b68fb1b,0x542fb7f1),LL(0xf8fc7eb8,0xd96b7a33),LL(0xb020cf53,0xdb72531d),L_(0xb4c510b9), LL(0xc73ecdcc,0x1c4cd34a),LL(0x272dd7b1,0x039494e8),LL(0xd23f79c7,0xa21edcfd),L_(0x207ff37a), + LL(0x17c4dc72,0x5ce9dcb0),LL(0x6a5e2d85,0x25936d77),LL(0xebe08b04,0x70fc7789),L_(0x7b6f8a3a), LL(0x7f3546f5,0x7563a201),LL(0xea8da605,0x10bfdb1f),LL(0xdb570af0,0xeb5d3434),L_(0xdadec2da), + LL(0xef345021,0xd0e87d76),LL(0x2945a207,0x222e0b0f),LL(0xe86e0a88,0x6fa3b5c9),L_(0xef9f7939), LL(0x9da8796b,0xabdf6b2a),LL(0x90ef27c9,0x7ead1795),LL(0xbbc76905,0x8c71560c),L_(0xbef19370), + LL(0x890d8ce2,0xb5cc05b2),LL(0xa7bd2aa7,0x7de1e29a),LL(0x866cd575,0xfc749a26),L_(0x825af3b3), LL(0x7eaa0bb6,0x886271b4),LL(0x6514a9d4,0xed069b45),LL(0x65bd26ca,0x2f59240d),L_(0xbe13ff4a), + LL(0xd9546220,0xcbae178c),LL(0x2cf4cc42,0xe71c4857),LL(0x70950b7d,0xc8f7ab52),L_(0xb20e351c), LL(0x52a1a22e,0x24d86b6f),LL(0xe4b84f76,0x70b94a08),LL(0xd450bfb1,0x0d983cdd),L_(0x3f21b37f), + LL(0x5619c34b,0x77d8a74f),LL(0x86215aa3,0x71f4dc88),LL(0xfa0612da,0xb127ccb2),L_(0xc1906699), LL(0x879fb160,0x205b4589),LL(0x2af7eaab,0xe8b6c8f4),LL(0xaad791a8,0x1490d5ca),L_(0x3359b830), + LL(0x32c5bb90,0x28810885),LL(0xdd3a45f9,0x2ceffdfe),LL(0x14a9a0ad,0x7970ea64),L_(0xe4fdf446), LL(0x8708e80b,0xbbf83ccb),LL(0x1b1d1037,0x5cc8baac),LL(0x800c8eba,0x98c8b359),L_(0x3ee963ee), + LL(0x22c84665,0xdbd93971),LL(0x284c33c2,0x697a6de7),LL(0x6f50eaa6,0x87cb9042),L_(0x70efc303), LL(0x5f70b39e,0x2af4f24a),LL(0x562e2f64,0xd8a16a0e),LL(0x8f4571e4,0xa7b9ed81),L_(0x6ceb56cb), + LL(0x6e1de981,0x725de358),LL(0x04ebe2bd,0x85652ca3),LL(0x50b9f863,0xd5f0033d),L_(0x0b13ec3a), LL(0x19c015bc,0x948391b7),LL(0xc37027bf,0xaf6e5ed5),LL(0x33c66538,0x96bfa703),L_(0xced15639), + LL(0x069d03c1,0x47f60293),LL(0x73dccc53,0xc32780f6),LL(0x3bfeba8e,0xc2e0eb4e),L_(0x546bd56a), LL(0x29bd4113,0xc233f03a),LL(0x277aca24,0x574da3b6),LL(0xf4f6a7ac,0xae0e0260),L_(0xbade476b), + LL(0xbb8f2925,0x5a087018),LL(0x5c2bab18,0x8c5485dc),LL(0x351ad0f1,0xf7d7679e),L_(0xe930bcd6), LL(0x2bdc4897,0x277bcf81),LL(0xbcb4ffd9,0xf147c5b3),LL(0x87185cc7,0xdf48c9e6),L_(0xce71b122), + LL(0xdb75cc10,0x17222dc7),LL(0x3bc7295c,0x3dd256a0),LL(0x3a012a3b,0x7ec5696a),L_(0xe6950145), LL(0x9381bed6,0x4f4cdfd3),LL(0x353d2f83,0x7b5b13a3),LL(0xbf5f2e84,0x2639cce3),L_(0xda40d3a1), + LL(0xdb334a59,0x6da64943),LL(0xd3f151b5,0xfce96a35),LL(0x6ad5dfad,0xa599a89e),L_(0x0d1b7f52), LL(0xeddc6626,0x2ef746d1),LL(0x861341ad,0xa912c3ef),LL(0xd6859ab6,0x1ac3aba5),L_(0xd95ca5f2), + LL(0x0e5c7c8d,0xb050ceeb),LL(0xcf83afd0,0x53e4bd2e),LL(0x710688b3,0x19a215e7),L_(0x8fa6f937), LL(0x9a042c9a,0xd22cc140),LL(0xa2947f4b,0xb5f7e2eb),LL(0x7f1f59f8,0x154453dd),L_(0x4db7a4d6), + LL(0x9473341c,0x9f33393e),LL(0x75ea0e37,0xb3dc51a3),LL(0x09fb2116,0x0ee5b84f),L_(0x3fcab420), LL(0xf91ac676,0x816d6ca6),LL(0x8f078e9d,0xc0c5edee),LL(0x0cfcd5ed,0xb71c05e5),L_(0xc167eb60), + LL(0x17c9bd2b,0x4c0897b1),LL(0x145b24db,0x7a16a169),LL(0x0f41e0dd,0xa88a35f1),L_(0xbff8660b), LL(0x5e5fc719,0x85b8baf5),LL(0x2e4db4f9,0xd33b70cd),LL(0x5dd4f3b8,0x5ddf49c1),L_(0xbfc827f1), + LL(0x795a3893,0xedb3e7f2),LL(0x6cad1ecf,0x33c80574),LL(0x26818153,0xf7414144),L_(0xff6a4325), LL(0x6f311ca1,0x24d6e452),LL(0x7d6669d2,0xe7193165),LL(0x66d1a963,0x89fa62a1),L_(0x6bc282cf), + LL(0x375aa47d,0x70c638b9),LL(0x4fe2ddd6,0xf167acbd),LL(0xddf9d01a,0x364bf41a),L_(0x70510fe0), LL(0x7d82f66d,0x8862080b),LL(0xe66e70a2,0x0448ac9c),LL(0x4dbdd638,0x971eaebd),L_(0x07449c55), + LL(0x30f6e054,0x4b4739f2),LL(0x55ce7718,0xd62b9aef),LL(0x3bbafe26,0xb84b80ac),L_(0x52aa54d6), LL(0xd03fc3fe,0xb6803cd9),LL(0xaf2b1562,0x81afcd95),LL(0x205a14b1,0x7a0ade0e),L_(0x371096d5), + LL(0x2d735064,0x4b1c54dd),LL(0xaad4a4b1,0xda4629d6),LL(0xe95c14ae,0x6f216326),L_(0x00affaf4), LL(0x2fae20c7,0x3112101e),LL(0xd2f76d99,0xbce52747),LL(0xfcd465ea,0x53fd0b9b),L_(0x8b45a470), + LL(0x068c0d93,0x1f9856a8),LL(0x6517f94c,0x3408da23),LL(0xc9b9b412,0xc63b4cb7),L_(0xd2b6eaad), LL(0xb7836b3a,0x200fd925),LL(0xa850f3d2,0x55a6ddef),LL(0x2b7f407e,0x0fb2a9d1),L_(0x8b4cf8a0), + LL(0xe36962f6,0xfae22eee),LL(0xb2671435,0xfe866c3d),LL(0x8acadf81,0xa3f599f7),L_(0x57dbb976), LL(0xe8b6b61f,0xb4b13551),LL(0x14a48328,0xcdd8f22e),LL(0xb9099888,0xe1b8ed2e),L_(0x322f8929), + LL(0x7e8f146f,0x3818b39f),LL(0xacd7c7b4,0x16886904),LL(0xf677bde6,0x022ae138),L_(0x9ffae2cf), LL(0x5746a459,0xacf62d52),LL(0x09ae59c0,0x63279bc6),LL(0x4ee46705,0xfd285b9c),L_(0x674ba2e7), +}, +/* digit=23 base_pwr=2^161 */ +{ + LL(0xbad78997,0xc931bee3),LL(0x0c15f2e2,0xe3c1a2d8),LL(0x7cd507de,0x58fc32e1),L_(0x0f28ff52), LL(0x01135f60,0xfb8b7b35),LL(0x11406d6a,0x413750b4),LL(0x2dffb3fe,0x6a265953),L_(0x7c1dd63a), + LL(0x796bb2b0,0x56c1932c),LL(0x09ceb738,0xf443c29c),LL(0x6ff56bc3,0xbdd22bb5),L_(0x7387fcf2), LL(0x21178548,0x22575d25),LL(0x534ca838,0x7ffb29d7),LL(0x4ffc71d3,0xd6ef14c3),L_(0x6a7a8abe), + LL(0xbc9613ac,0x5920119c),LL(0x6ae141df,0xc3d2b2e1),LL(0x9633ba13,0xe7d31665),L_(0xc56ddd74), LL(0xe622933d,0xec5df7bb),LL(0x641e9e6c,0x50f7f156),LL(0x7feb42b1,0xeb17ef8e),L_(0x36a1400c), + LL(0x1a087220,0xeeddf997),LL(0xe4cee644,0xb6ad5609),LL(0x3c761d64,0x52dd9f57),L_(0xda46466e), LL(0x39b5966f,0xca15f167),LL(0x16fee018,0xea78b847),LL(0xd9642afb,0x37eea476),L_(0x3f286088), + LL(0xda4b72e6,0xa3072640),LL(0x3e38a59d,0x36a057e0),LL(0x4754e3c5,0x4faa9820),L_(0x2a8a9f0f), LL(0xf20160be,0xa80ee38d),LL(0x5f711700,0xc8e173dc),LL(0x665dbf2e,0x2aba1be0),L_(0x187969d1), + LL(0x228ed7aa,0x2d501abf),LL(0x1434af6f,0xe4f5d2f6),LL(0x3dad807b,0x0b4aa28f),L_(0x49171ad5), LL(0xe672b220,0x0888b653),LL(0x61b8ab73,0xa3e477ad),LL(0x226f83cd,0x0afdcedc),L_(0xff93b3f1), + LL(0xe7476655,0x562eb536),LL(0x4facc94b,0x7f5bedc0),LL(0x2887f756,0x8581b7e9),L_(0x2a5e0a2d), LL(0x160c3505,0xdde60b62),LL(0xa03cc7a8,0x8a8ca29a),LL(0xb94770da,0x3ecfeca1),L_(0x667902dc), + LL(0xb615d5a8,0x8ea9ecfb),LL(0x55508319,0x6be624fd),LL(0x7f81ee5d,0x45035a93),L_(0xd73caa36), LL(0xc3d23101,0xe9c562fc),LL(0xd42e053f,0xf451c229),LL(0x6ee17a32,0x932ddcfa),L_(0x689ee05b), + LL(0x6337e909,0xe970ce2d),LL(0x7dae8ee6,0x58d5ae88),LL(0x253afb1a,0x88db9fe4),L_(0x05ed8360), LL(0x3d74de44,0xa90b033e),LL(0x8c323484,0x132dd86b),LL(0xed163aa4,0xc88fa0d9),L_(0xbaaf76b3), + LL(0xa0e34d4c,0xa5b69894),LL(0x8d09f343,0x495e2cc1),LL(0x987653cd,0x5e895b41),L_(0x0dbb4714), LL(0xf289cc21,0xc41882ef),LL(0x5fe6817a,0x3329b3be),LL(0x4acf73b8,0x9efd2db0),L_(0x0d02e7dc), + LL(0xdd8e91bb,0xf8f6f870),LL(0xc680bcc6,0x728bf4b4),LL(0x93d4067d,0xb9c4891b),L_(0xc44a0438), LL(0xf8bf9b96,0x1e5e8506),LL(0x8e6af7aa,0x11821ebf),LL(0x99083ad2,0x7e9d0c84),L_(0x510d6d87), + LL(0x85547391,0x48ab6926),LL(0x4bca1962,0x1badb306),LL(0xdd5f7613,0x4641e82a),L_(0x7c23013e), LL(0x3d43a297,0x58c32dfb),LL(0x706e6ac2,0x3ffcd8f7),LL(0xc250c17a,0xfacc3394),L_(0x0ac00a9f), + LL(0x99456e77,0xe9a40562),LL(0x762b34e5,0xa165351f),LL(0xd0dd1798,0xbabb6293),L_(0x257af82c), LL(0xb83a079b,0xa5cd33ea),LL(0xac07759a,0xf120f050),LL(0x6109e625,0x774354d8),L_(0x7ac23484), + LL(0x8d82b0bd,0xbafdf31f),LL(0xfd4e66c0,0x7639e997),LL(0xd2a8308f,0x3be638b1),L_(0xeb3d8386), LL(0x54bc114f,0xeb650088),LL(0x97a2e01d,0xebdef489),LL(0x902bb33d,0xc3e46171),L_(0x3a181f63), + LL(0xc86362a6,0x56cc1bef),LL(0x13e720d5,0x3b4b4fc8),LL(0xf9436f6d,0x370a01fd),L_(0xba6eeefa), LL(0x632c516e,0x916aa8c7),LL(0x571c4cd2,0xe3ad18e9),LL(0xe18b971a,0x72c7e5a4),L_(0x570527c5), + LL(0xb48396a0,0xf8cf3d39),LL(0x6ad6e8f2,0x1d89563e),LL(0xa602a006,0x91cd1395),L_(0x64fa2f77), LL(0xdbdaa433,0x0966d0a9),LL(0x8e0abebb,0x166bebe0),LL(0xf1b4da0f,0x433f9113),L_(0x8e2b6532), + LL(0x6fb1d5da,0xd89da757),LL(0x89f048e8,0x2939bdac),LL(0xa42134e8,0x6b13fefc),L_(0x792be7f3), LL(0xb37f662f,0x413f92df),LL(0x08daa37a,0x985c742a),LL(0x1de586e7,0x5f4904de),L_(0xe4507309), + LL(0x03906d92,0xb13c4090),LL(0x9f0f4106,0xf3affee5),LL(0x6830a295,0xb7fc1017),L_(0xfe0d1565), LL(0x45d24eb7,0xcb2d95c2),LL(0x02c21f5f,0x8c20bcd3),LL(0x3379e3da,0x6dc4634a),L_(0x5f5ffa94), + LL(0xaefdff78,0x97e82574),LL(0x2379777c,0x112d1838),LL(0x20061ca4,0x0060cd67),L_(0xc1751f60), LL(0x11499a1f,0x8b6913cb),LL(0x3ca5f64c,0x67ff5e86),LL(0x1680aa3a,0xe823c3ec),L_(0xdf7da52e), + LL(0xd3ac8a56,0xd2b4232a),LL(0xcd8aee6e,0x1b8abf52),LL(0x2fcb86b4,0x19bc7365),L_(0x2952fde9), LL(0xa051bf9a,0x1aec4d94),LL(0x79b4a51e,0xfbc4f986),LL(0xa954ceb9,0x05b104cc),L_(0xe0d3602f), + LL(0x40690897,0x4799ff54),LL(0x87dac4d0,0xbc95c209),LL(0xc0b5b3de,0x290686c2),L_(0xb3329cbb), LL(0xca443d1e,0x6b65747b),LL(0xcac2d9da,0xf2987b71),LL(0xf039026d,0xb9b37c0a),L_(0xb77496d3), + LL(0xd360d773,0xd81505e8),LL(0x271af0ca,0x40816745),LL(0xc9836e31,0x9dca53d3),L_(0x7227191a), LL(0xd820c040,0xb45d3f5d),LL(0x3bbb6420,0xedf3a3c8),LL(0xe0ce5e44,0x459a2236),L_(0xf815d523), + LL(0x35ffc3c2,0x273c8e42),LL(0xdda15d58,0xfecec30c),LL(0xda6a8389,0x8e8dc1c3),L_(0xc0bf1e23), LL(0x6fe132fe,0xc3565543),LL(0x707a3332,0x951334fa),LL(0x770ed88f,0x4d1d476c),L_(0x180fa694), + LL(0x20e67928,0x488e81fb),LL(0xd3aa63c5,0xd968afb3),LL(0xb0057d32,0xb8d04c1b),L_(0xabd57ea4), LL(0xee01ab83,0xef38d717),LL(0xe3ff770c,0x830deae9),LL(0xfba87128,0xbf20a3cb),L_(0x7a5792cc), + LL(0x15750d84,0x8d8facef),LL(0xec118062,0xef0f3af0),LL(0xd1a4206b,0x64ebf883),L_(0x6101aba7), LL(0x5b6d2c3c,0x3c48ef20),LL(0x2cb3a5ff,0x153580c5),LL(0x534ba912,0xa02f889c),L_(0xa1a92663), + LL(0x1f2a638e,0xcb233fe2),LL(0xd1e29224,0x792d0dff),LL(0x62d03b08,0xd6646af8),L_(0xcb6267fc), LL(0x4a8172cd,0x82199f97),LL(0x582165ca,0xf879213b),LL(0x8048de29,0x51bdf94f),L_(0xe7f10d0b), + LL(0x75f12d76,0x743c75d7),LL(0x4777a4a1,0xc13528c2),LL(0x0f8f0fc7,0x969b010e),L_(0x6365c47c), LL(0x2acbaab1,0xdfc6ee9c),LL(0xe207ebae,0x23ac352f),LL(0xc00ef611,0xdfc77ba6),L_(0x6470bbbe), + LL(0xd561d67a,0xac256d68),LL(0xbe1d04c9,0xb7f44d83),LL(0x1d8cc6f2,0xddd4acc6),L_(0xc01e2b00), LL(0x83d18ef1,0x6b5fbe51),LL(0xc673b3a0,0xe616da91),LL(0x04e7aef1,0x7079d8b4),L_(0x3313c1b4), + LL(0xc0f7d6a0,0x2242759d),LL(0xb6fbfcb4,0x9949d06a),LL(0xde898701,0x8b24618f),L_(0x5307474d), LL(0x3976254e,0xb0bd22ab),LL(0x211085ed,0x2125ed2b),LL(0xa5f28d83,0xa1c543ea),L_(0x565a6c4d), + LL(0xe7ee1b29,0xf2a06317),LL(0x3813d14f,0x784961c1),LL(0xf759d96d,0x5670b8e7),L_(0xb428b1ab), LL(0xafedb09a,0x8fbfe519),LL(0x478f19a0,0x7246a169),LL(0x7503e23c,0x23ac9af3),L_(0xd24790e7), + LL(0x87836e62,0xc31169c7),LL(0xbcb98921,0x111050a3),LL(0xa7810165,0x446bda32),L_(0x6ad438a4), LL(0xa876bcf9,0x32692fd4),LL(0xc750819f,0xd11e3c6e),LL(0xe2aece33,0xd325fefc),L_(0xb6f2bf6f), + LL(0x8c31bbfa,0x0e3623bf),LL(0xa2285b20,0xbb5c266c),LL(0xc5f4aaef,0xa7179b54),L_(0xdf6212eb), LL(0xed087187,0x375cb291),LL(0xd118abf6,0x047af635),LL(0x73198d27,0x8f1c0b06),L_(0xeac7c4ea), + LL(0x77e38348,0xc22c3ac3),LL(0xd0ae073a,0xea1c7779),LL(0xab8de890,0x8b9d1132),L_(0xb336a43c), LL(0x2a2fde97,0x78f20c5e),LL(0xf71fbefe,0xa11f0040),LL(0x302a6212,0xba7689dc),L_(0x6a6519f6), + LL(0x8c2d9e20,0x50cda9df),LL(0xb3180610,0xa6158df8),LL(0xadbbeaa9,0x4778c93d),L_(0xc891e9d3), LL(0xd1a49e0a,0x6eb6664c),LL(0x68d58278,0xb72ac28b),LL(0xb8fb2671,0xfc493918),L_(0xbe9a2c58), + LL(0xe846a0c4,0xc3ffb873),LL(0xf33f7123,0xd98b14c6),LL(0xcd3fcccf,0xec8c8522),L_(0xc0d09a5d), LL(0x2709309a,0xc435552e),LL(0x393794e6,0x731b5ca4),LL(0x6d3545bb,0xcf95c612),L_(0x21f92c49), + LL(0x67e83945,0x77e47a85),LL(0x8aaaa2f8,0x41b4a73b),LL(0xf14ab27f,0x45673153),L_(0xf95ee4e1), LL(0x29737736,0x3a31ee99),LL(0x3add55e3,0x89cb9a71),LL(0x3c610ccb,0x92d6ab9e),L_(0xa1df471f), + LL(0x24dd0313,0xfe3af3cf),LL(0x3ad69446,0x1fa695ef),LL(0x49e9da9d,0x35d8485e),L_(0x823de577), LL(0x1be78d4e,0x9a8e5ede),LL(0x535233c3,0xdb4e20a2),LL(0xb9902b0e,0x762073cc),L_(0xb355fbea), + LL(0xba8c73ac,0xfd983b8b),LL(0x0f082d41,0x3921c9f7),LL(0x9c6ee5d8,0x7f5a5177),L_(0xd9ef6090), LL(0xab53e7b3,0x7db07019),LL(0x14b4f4d5,0x01f9a8fb),LL(0x2588492a,0x9ea1e98d),L_(0xeb443742), + LL(0x31997789,0xa9a0407f),LL(0x4e820720,0x26fdd725),LL(0xa3068d9c,0x2aba0c14),L_(0x6db14c6a), LL(0x6d072830,0x2d5ed5c3),LL(0x0acce5d9,0x52b3241d),LL(0xdbd63903,0x11f4b8bc),L_(0x04c731fa), + LL(0x35994c38,0xccc2b918),LL(0xbc10285b,0xf27570ca),LL(0x870c688b,0x8c91a8af),L_(0x775fff54), LL(0x5459f2f6,0x9157cd7c),LL(0xc2e98dfc,0x021d9844),LL(0xdbbbf2c4,0x3319b3fc),L_(0xba743dee), + LL(0xaeba325f,0xaecdc264),LL(0xf477eda2,0x9b24587e),LL(0x859c1b6a,0xf7648c83),L_(0xb5c8f8dc), LL(0x7005a3f4,0xdf465967),LL(0x10b24908,0x2411ca0d),LL(0xd689abfe,0x1452c503),L_(0x625ecb92), + LL(0xcc290bf4,0xf34968d7),LL(0x9003a913,0x7228d7c0),LL(0x48e02e21,0x17e25b2b),L_(0xd5f95bbb), LL(0xa53a154c,0x1035d2b6),LL(0xac56a6bf,0xe8ab36c5),LL(0xbcf0eb3e,0x63fa883e),L_(0xdf08d138), + LL(0x0b53b1ce,0x2088982f),LL(0xe8ed9994,0x21364c97),LL(0x45bf87a7,0xeeb978c5),L_(0x5b94bef4), LL(0x4e53377c,0x093e9c15),LL(0x58acb871,0xd8249f4f),LL(0x02b46436,0x64f236ee),L_(0x88a9e12f), + LL(0x3940ee67,0x6164d05e),LL(0xf4a7c705,0xb0056da2),LL(0x872642d0,0xa7bd3d66),L_(0xa407f0cd), LL(0xce37f407,0xd8371363),LL(0x592b0f71,0x90362154),LL(0x009ca301,0xe59d0b2d),L_(0x3dd8f9c6), + LL(0xcf247be0,0x73b26b51),LL(0x31ef23aa,0xdf33ad4d),LL(0x1f6d55bd,0x4ed64955),L_(0x7ed356bb), LL(0xa18aee16,0x8ca23082),LL(0x82214ca5,0x20da76f8),LL(0x2453f147,0x4606958c),L_(0x3b5ee2c3), + LL(0x99950739,0x06b63065),LL(0xb845a47f,0x9853453b),LL(0x88e04e19,0x456e72cc),L_(0x05cfd664), LL(0xb168037c,0xd3c22f8d),LL(0xcaf525d3,0xd989f0f6),LL(0x9699fc87,0x273ac5b1),L_(0x0349e937), + LL(0x273a933e,0x24f52dc3),LL(0x825de5df,0x36bc3c34),LL(0x3e91b0cc,0x61de4790),L_(0x7a99b424), LL(0x8f4b678a,0x44bdcadc),LL(0x0a060400,0x830bedee),LL(0xbb9ac7e8,0xbda5fd11),L_(0xa71c4ee0), + LL(0x37a97135,0x079c1b7a),LL(0xac1ea75a,0xeaf11508),LL(0x3012e200,0x61b63246),L_(0xbb79e1c3), LL(0xc0246bd9,0xb92f89e2),LL(0x95a4a7cf,0x3fff4a96),LL(0x44cf1045,0x6e70ae3f),L_(0x5783bd56), + LL(0x6615d343,0xb3a83dd9),LL(0x5e8bc7ab,0x0c4c00f6),LL(0x6cbcc29c,0x84d9fe95),L_(0xaf97e676), LL(0x30e1885c,0x82a4da98),LL(0xbe563af0,0xbcc23ab6),LL(0x259db6ce,0x306cf56e),L_(0xa179a030), + LL(0x08624c7c,0x32c9c5fc),LL(0x37d48ba3,0x38295bfa),LL(0xfb49c594,0xdcfe3634),L_(0xc9dd0809), LL(0x832aec93,0xe5cda111),LL(0xebcbf38d,0x98fc1727),LL(0xac496bac,0x3f057842),L_(0xefc69a01), + LL(0x7bdd13e8,0x189dee30),LL(0xc45d637e,0x01cf6015),LL(0x02a07254,0x7f6053e7),L_(0x9648ef85), LL(0xc9d79b49,0xd467929d),LL(0x221a964d,0x2a4257b0),LL(0x3ed68c3c,0x0209a951),L_(0xbf635477), + LL(0xb50fa3a9,0x568377b7),LL(0x57a50eda,0x992fa9d6),LL(0xf3995e07,0xf09351e3),L_(0x21813534), LL(0xde00386d,0x7c21c039),LL(0x5bd67bbb,0x983e3f5f),LL(0x970f1581,0xfde7bb8d),L_(0x50e663c0), + LL(0xa5991d36,0x727713ca),LL(0x52a00363,0x81781a15),LL(0x946557d9,0x8b2de3fc),L_(0xce2f349c), LL(0xb61c0936,0xefe6ec67),LL(0x9c584327,0x0483381f),LL(0xbfd4710c,0x3cb8617a),L_(0x2e9c6a8a), + LL(0xfb2c55b7,0x83398c47),LL(0x00610b5b,0xfc55bdd9),LL(0xd6aad463,0x733b8806),L_(0xe6434da6), LL(0x600c9641,0x72a69c3e),LL(0xc959c3b6,0x25c8da00),LL(0xa863f55c,0x6b73f445),L_(0xb25ba2b8), + LL(0xb5811bfc,0xdea6b1b8),LL(0xc85bb876,0x6f29d6aa),LL(0x244c832f,0xb885707b),L_(0x700946f8), LL(0x19847d9e,0xbb7684f6),LL(0xf1284753,0xc0f2d53b),LL(0x20ef5736,0x9e06880d),L_(0x94cf829c), + LL(0x9a3cdc82,0x6fd2d1fe),LL(0xf6676bfc,0xfe48b29e),LL(0xa14e785b,0x7afeb3e3),L_(0x73a22b1e), LL(0x15c1eec0,0xdc5c271b),LL(0x738c09b5,0xd51bbdf3),LL(0x4095982d,0xea6a656f),L_(0x4e8a1c04), + LL(0x8df3827f,0xad96082c),LL(0xe21e88ec,0x0c7308d3),LL(0x747332eb,0x3e790568),L_(0xcbe74f24), LL(0xb410e2a6,0xa45bcc07),LL(0x62e28f16,0x34e11697),LL(0x68990fab,0xd0d95f71),L_(0x0bc4045b), + LL(0x2b70f4c3,0x95663747),LL(0x718a4989,0x18e76685),LL(0x0510d4ae,0xa806fc64),L_(0xb9f7725e), LL(0x74a39c46,0x11681c31),LL(0xab7986bb,0x7f1ee73d),LL(0xd70c5b25,0x145296c5),L_(0x8e746d97), + LL(0xb90e977d,0xa73e17c7),LL(0x914b838f,0x3b0459a9),LL(0xd910c80d,0x4a996387),L_(0xc7523409), LL(0x09cc1463,0x4ff007ad),LL(0x3feccef8,0x6f3bf601),LL(0x9137e94d,0x3d0461a8),L_(0x5dfeb98d), + LL(0xdf5e90f7,0x03f3fdc3),LL(0x3d7cbec2,0x2f05f4ba),LL(0xefe1e30f,0x5dab20a6),L_(0xb6114a8c), LL(0xfbe83a6b,0x815eac18),LL(0xbcc8bbc2,0x2c89a659),LL(0x13f75b6a,0x326a87f4),L_(0xda0c7f42), + LL(0x99fa6315,0x29c19e9b),LL(0xdf48509e,0x5d5a6ffa),LL(0x84d62c60,0x4e8640e3),L_(0x4a85f9d6), LL(0x01830a3f,0xa50855ff),LL(0xea7061d4,0xb9b3fb5a),LL(0xbf0727b4,0xa1cac932),L_(0x9c76f2b9), + LL(0x08fe3e55,0x7f8ba9f8),LL(0x3a13632e,0xfff3b260),LL(0xa621223a,0xda5b8cc0),L_(0x6cf87377), LL(0xb352cab0,0x52e875c5),LL(0xb3cb3de2,0x888687ca),LL(0xd159fefe,0x788f0705),L_(0xa3f5461f), + LL(0xe7ee73d4,0x71f9e278),LL(0xa10ec3f4,0x0511bea2),LL(0x147b2ef8,0x908389b2),L_(0xaf59635c), LL(0xb9505162,0x7ab5b1bb),LL(0x05d9bd3a,0xa16a5843),LL(0xeb15fb80,0x8840233d),L_(0xf2f61b05), + LL(0xfbf89124,0xb350235f),LL(0xa7d12321,0x601ae7a3),LL(0x487a5b44,0x168cfefc),L_(0x124de1e6), LL(0x27f1099a,0x3f7f4917),LL(0x387a3135,0x576cbe65),LL(0x93695572,0x1697b707),L_(0xe9cd239f), +}, +/* digit=24 base_pwr=2^168 */ +{ + LL(0x1aa93495,0x07822bd8),LL(0x98675e63,0x690a044f),LL(0x411c39e9,0xddfd5a23),L_(0x768594da), LL(0x5d8da0b0,0x74dcec38),LL(0x3c14e211,0xa51c46ee),LL(0x6b77f2b0,0x1367898a),L_(0x8ad0bfdf), + LL(0xd8b1efbb,0x5d75e9ca),LL(0x940ae4e4,0xdea9f60e),LL(0x7af3d6fd,0x95065ae4),L_(0xef0f21e7), LL(0xe0c7e801,0x22ced186),LL(0x9502622d,0xc6698d7e),LL(0x95367fa5,0x9310a67e),L_(0xc864dc25), + LL(0x7322a3b3,0x1b5b3f21),LL(0xe6b25a83,0x1fe6e87a),LL(0xcad2ee21,0xb5fd4e5b),L_(0xd772c552), LL(0xe5ae89d5,0xc2853eeb),LL(0xc45afb20,0x9036d2cf),LL(0xcb159584,0x45627827),L_(0x6e07de7e), + LL(0xf341190a,0x2470f5ee),LL(0x87e42cc7,0xf0be84fe),LL(0xade34d2c,0x90636c20),L_(0x0cf8cafa), LL(0x180fa04d,0x2db09b0f),LL(0xd896b5d9,0x0a0909a9),LL(0x83bcc559,0x281cf9aa),L_(0x4d7d88c9), + LL(0x38d6fee1,0x8f859667),LL(0x0e93537c,0x86559609),LL(0xca6e59bf,0x6854b4e8),L_(0x4c2e89d8), LL(0x23a24b88,0x5840ce8a),LL(0x6366e9c0,0x946b6bbc),LL(0x00fd3d85,0x8a095b45),L_(0x9e0c589e), + LL(0x74d544f2,0x4c355669),LL(0x4696a73a,0x1ea60e81),LL(0x099e116c,0xd960a70d),L_(0xea914344), LL(0xd60a4a5a,0x305faf07),LL(0x1026dbdc,0x61b80d24),LL(0xe6b8e5c2,0xd428e0cc),L_(0x2cd02a61), + LL(0x26d21296,0x5d2ed352),LL(0xf7a07ac1,0x3daf91ed),LL(0x44afa3c4,0xd529e7b2),L_(0xdd22a012), LL(0xd7ed0b48,0xcc9faccf),LL(0x00535729,0xc98aa7de),LL(0xc9757998,0xce2c4e14),L_(0xe8ccc048), + LL(0x098fc504,0x3d3f8532),LL(0x5cb9d339,0x50253fa0),LL(0x23d266c8,0xb38286d6),L_(0xe5172f5c), LL(0x99903093,0x48a64b98),LL(0x8c64987d,0xbc43643a),LL(0xfe159bd8,0x2f08eae4),L_(0x049c111f), + LL(0x5b31f9b7,0xc558d554),LL(0x0d2340eb,0xe3c46403),LL(0x19a6f592,0x2211131c),L_(0x4ef1dc2e), LL(0x4d8f19e8,0x8c4d4f34),LL(0x107caba4,0x347e8cae),LL(0xad36aa89,0x5a959e3c),L_(0x4495ae4a), + LL(0x178db2de,0x8674b5e1),LL(0x68e6ec47,0xce21dcf1),LL(0xd3884f45,0x7664513a),L_(0x60d36975), LL(0x7a1b3e57,0xc9e299e2),LL(0x8015c45b,0xe96de0f0),LL(0x8d3f0ffc,0x5aba434e),L_(0x448fc469), + LL(0x3206b5a0,0xbf15b53c),LL(0x0516d5b4,0x2fff52c6),LL(0xfec55940,0xc3791f65),L_(0x797a6b95), LL(0x394de0c2,0x5c5fed8c),LL(0xeba16248,0x674b4d58),LL(0x0e57743b,0xd7315975),L_(0x87252f57), + LL(0x7fa4f80b,0xc574aafe),LL(0x9e9fc1b3,0x25994c44),LL(0x15293405,0xa00f9231),L_(0xc2527f9a), LL(0xf98edea6,0xd0bcf974),LL(0xbe7b43fa,0xa3c74ad5),LL(0x448873ac,0x8ff67b2d),L_(0xf0dfe6da), + LL(0x510cb9a5,0xbaf15c58),LL(0xcabedab3,0x234a8faa),LL(0x0023d4d4,0x9d6e89b9),L_(0x8a188d72), LL(0x37485100,0x6bf4faef),LL(0x5f70a85e,0xdd164001),LL(0x799cb3ee,0xfc7797a7),L_(0xd331f4bd), + LL(0x34261e55,0xcf17aa4d),LL(0x7f8c3542,0x801ccc34),LL(0x0c7a04fa,0x762b3408),L_(0xd90eff09), LL(0x002ca132,0x0870f41d),LL(0x767451ee,0x70686201),LL(0x276eaa26,0xf592842e),L_(0x6a87509c), + LL(0xb78b71bd,0xa79ca9e4),LL(0x31d15ca5,0xe7aab4c3),LL(0x095aa863,0x980df4d9),L_(0x30800a25), LL(0x75ac5ed9,0xcc69d2f9),LL(0xee44fb88,0x429f98f1),LL(0x10ce7510,0x5736d4ee),L_(0x03cc9ae7), + LL(0x8da51d85,0xb2abe419),LL(0xbd89ecd4,0xefa624dc),LL(0xc275c72e,0x8b684df9),L_(0x0c47b707), LL(0x75f1bf6f,0xfc8f823d),LL(0x06225cca,0xa497166b),LL(0xb3363382,0x3241d9b6),L_(0x397d4140), + LL(0x9a65f284,0x47d0d598),LL(0xd2dae790,0x600f8097),LL(0xdfc5d530,0x06c1d340),L_(0x18b74aa7), LL(0x409b596c,0xe50cc47b),LL(0xb9309f95,0x71018c41),LL(0x4f9b2683,0x2de345b4),L_(0x73d97fe3), + LL(0xd352b3da,0xfb7e2c37),LL(0xd29b2279,0xfb5035db),LL(0xc27af22c,0xe5cf5a24),L_(0xdd8c403d), LL(0xf2e49e04,0x31c42940),LL(0xbe767afa,0xb3e476b1),LL(0x3130439d,0x729a7e6b),L_(0xcfb2c6fd), + LL(0x554fbba5,0x1228460b),LL(0xfba91910,0xda7db076),LL(0xf5e0be2b,0x8a989e94),L_(0xd04b5940), LL(0xe15aded2,0x56931b09),LL(0xe96ceb9d,0x7fb6175a),LL(0x36407d3f,0x785ad12c),L_(0x412c5e80), + LL(0x296e5cc5,0x6178d140),LL(0x9e4845c7,0x9088d05a),LL(0x1f9dc468,0x631396d3),L_(0x8b994c1d), LL(0x86ffc6ce,0x04f52e9a),LL(0x2ffa498e,0x4b21e0a8),LL(0x27375e3f,0x242c7c60),L_(0xd99a4851), + LL(0x064ca07a,0xa02c0b2b),LL(0x518bd6af,0xa1558503),LL(0x13a69d2a,0x3d67919d),L_(0x68b0b3b5), LL(0x2bfc83c0,0x3a653e93),LL(0x24e20cb0,0x69ae9c82),LL(0x0a4cc47c,0xf624d321),L_(0xeb72aa98), + LL(0x9e3e45d0,0xd947c8c7),LL(0x1032ccc8,0xa439634d),LL(0x5b100526,0x7b5067f8),L_(0x534219ea), LL(0xfc26652e,0x701d8d64),LL(0x1e89c3fb,0x277501f0),LL(0x262c9dad,0xd3dd957f),L_(0x48bd1e47), + LL(0x9f4be9d8,0xcf4ac2d6),LL(0x2a7df2ab,0xe04b6794),LL(0xf3257cbf,0xec24a41c),L_(0xf6596f8e), LL(0xb3eb3d14,0xcc1689ae),LL(0x7d52d65d,0x8712e8c6),LL(0xafd7b5e3,0x868af920),L_(0xb7ad63c9), + LL(0xd27017ec,0x75a3408c),LL(0x8af38290,0x7de280ae),LL(0xfab61ca0,0xb76f92d0),L_(0x9df539d9), LL(0x528c5329,0xffede226),LL(0x4363c147,0x9bf76d0a),LL(0x4d2256e2,0xdd96e876),L_(0xc1afd4ff), + LL(0xe00a0553,0x896685cb),LL(0x48da5f08,0xf0d4a520),LL(0xe8f5fdb6,0x0f380c61),L_(0xb146db8e), LL(0x4d63b6c7,0xca87cdad),LL(0xcfdcfe1c,0x69639e2c),LL(0x941d9935,0xed61f0e4),L_(0xad7751a2), + LL(0x72ebb947,0x43d0a9df),LL(0xdb90e6a7,0x53424333),LL(0x10a77134,0xdf849ce1),L_(0xce2f9c06), LL(0xdf56c5f4,0x58feabaa),LL(0xc5d546de,0x787ed1e1),LL(0xa802a275,0xa542b49b),L_(0xa5e7ea27), + LL(0x0646662c,0x33793ddc),LL(0xde0180db,0x873a716a),LL(0x3ae6c62d,0xd34ef5f0),L_(0x8ffbd17e), LL(0x9974e0c2,0x5b201375),LL(0x61aa17ae,0x25cf8d2d),LL(0xfac50e53,0xf5028682),L_(0x4cb76558), + LL(0x20822d1a,0xdb5b7f7b),LL(0x42e0c365,0x610bf8a8),LL(0xe4e5ddab,0x59f6f46f),L_(0x7d6ff0da), LL(0xd3a9f7a1,0x29ffdb56),LL(0x95d9dd8c,0x1aed0fa8),LL(0x7cd570f8,0xa1cdd0bb),L_(0xf42c2ded), + LL(0xb9d66a96,0xf9463b5d),LL(0x7ab3d377,0x9dcb5190),LL(0x7e8de98d,0x3a174474),L_(0xf096dc4f), LL(0xd8a6df5c,0xb00ae812),LL(0xc11335ee,0x595dbbd8),LL(0xab858cbb,0x20456b6d),L_(0x50149878), + LL(0x6e092bb5,0x4b66cf94),LL(0x20f71995,0x7eca44eb),LL(0x6daca5c9,0x44fbdf91),L_(0x06ba2abf), LL(0xcb39349b,0x9f292892),LL(0x5db8d756,0x7d2e1348),LL(0xb91c9d21,0x2511f796),L_(0x03664b0f), + LL(0xc742fe69,0xaeca436c),LL(0x07bb4010,0xa736c99b),LL(0x7053b3ba,0x4df3a08c),L_(0x7269f130), LL(0x35ff874c,0xa50437d0),LL(0xa2239752,0xbb3ee79b),LL(0x28f141d8,0xa309d53a),L_(0xe3b40a03), + LL(0x052107c2,0x4c1843ed),LL(0x92f03f51,0x5b395716),LL(0x10eeec3a,0x41bed6c7),L_(0xe7590252), LL(0xe3bf062e,0xe2632a75),LL(0x61293b51,0xccb37627),LL(0xf5482813,0x65bddca7),L_(0x3882b6c4), + LL(0x97f943de,0xe4f707eb),LL(0xaed3b2b2,0x1fca8433),LL(0xe200b711,0x5979d82e),L_(0x79d9de90), LL(0xa0864854,0x073f049c),LL(0x5ba6b65e,0xefca8545),LL(0x5e115ac7,0x81b25231),L_(0x811506c8), + LL(0x0b83214f,0x6e208498),LL(0x4f6ef89a,0xd7f17a61),LL(0x9d606b9f,0xf9ff6be9),L_(0x05c6d274), LL(0x78a1ebcd,0x8412a690),LL(0x1b349b3c,0x3fc39e84),LL(0xa4039f18,0x16cc85fa),L_(0xacd2b66e), + LL(0xb29dd0f6,0x3f372e07),LL(0xb2f85965,0x1010f76a),LL(0x499ca40b,0xc9d3a123),L_(0x494c1c41), LL(0x1a039621,0x898b067c),LL(0xc21db378,0x8fba57f1),LL(0x920244b9,0x2f20dfbe),L_(0xafa449b9), + LL(0xadba6514,0xf1db6fa1),LL(0x0a3087ba,0x780eff8d),LL(0x6530b59a,0xdcf21d60),L_(0xf6198373), LL(0x767463fe,0x65b13c4b),LL(0xff99e886,0xaf0f5939),LL(0xb10b6a9a,0x05c89578),L_(0x713231a5), + LL(0xae7f781a,0x678205ed),LL(0x46236fd1,0xcc32c0af),LL(0x2492206a,0x43d4ee55),L_(0xfbf9ba4c), LL(0x2296a220,0xd485f134),LL(0x17487172,0x7dae00fa),LL(0xd7fd8f23,0x9c357857),L_(0x48861bc1), + LL(0x683b4c53,0xd1014606),LL(0x084b9990,0x2de4b78e),LL(0xed10d363,0xe5547d1b),L_(0x3d2f5f09), LL(0x40cf618a,0x1fa00e4b),LL(0xf090adf3,0x2643b488),LL(0xa90cf1b7,0x862ff071),L_(0x7b03dffe), + LL(0x2c263fe4,0x3df3e881),LL(0x74c2dd8a,0x4a452825),LL(0xf8b4e248,0xff9860e0),L_(0x0632b8bd), LL(0xfaa7bd3b,0x4dcae86a),LL(0x6c838bca,0x0392152c),LL(0x8e97fa6c,0xfc62b042),L_(0x1616bf3e), + LL(0x38ca48d5,0xc8c6f843),LL(0xafab4216,0x1a5701ad),LL(0x8137a142,0x3332d11c),L_(0x073d30ce), LL(0xf65f8d03,0xc57e0eee),LL(0xc5ec9bae,0x84c4a8f0),LL(0x474d64c3,0x9a310ea4),L_(0x279cd2cf), + LL(0x90943f9d,0xabcd06f1),LL(0xdfc2230e,0x4bf7868a),LL(0x2460277b,0xf15a1c15),L_(0xe0531432), LL(0xbbc35210,0xa0084581),LL(0xff9171d1,0x8c12c483),LL(0x4461e279,0x2cb0b794),L_(0x18f08ce5), + LL(0xc6234888,0x1336960e),LL(0xd82f38a5,0x6690aec6),LL(0x8dcc7988,0xea8d5acf),L_(0xbdf09963), LL(0xad19e667,0x221a923e),LL(0x5ebdb79e,0x3d295ba2),LL(0x840fc746,0xa849c9c6),L_(0x95854d40), + LL(0x796ddb68,0x7f2ece47),LL(0xb700fc8f,0xfa40fdb7),LL(0x6e9b86f6,0x3753a599),L_(0x004dc14e), LL(0xb0d32904,0x9ce52e22),LL(0x9fb7c051,0x609b3582),LL(0xbd4bbc16,0x470387a5),L_(0xae8123ae), + LL(0xb5992b13,0x15e316ec),LL(0x34a4f3f6,0x331c37c2),LL(0x4f66d67c,0x5a547f8e),L_(0x3d89818f), LL(0x2343f37e,0xc7ecc98e),LL(0x464eaafa,0x2f36fbc0),LL(0x32fd452b,0xbcd300ae),L_(0x52cb6226), + LL(0x3965b7dc,0xfbb383d5),LL(0x8e1be53e,0xd5ddeb7e),LL(0x5de1379b,0x89236512),L_(0x1618ecf2), LL(0x71474c33,0x359ac63d),LL(0xad2f7a4e,0x6a2a41ed),LL(0xb41d45a2,0x4543bf02),L_(0xd97bbddf), + LL(0xdd45b155,0x51a93461),LL(0x835fa995,0x8e87a379),LL(0x6030bb55,0xe0f74319),L_(0x3fce8a20), LL(0x180c566c,0xf64d3bd0),LL(0xf4320f55,0x117f7e04),LL(0x9fe6d9b3,0x276cdeae),L_(0xe11e04e4), + LL(0x90d2c4b0,0x8ee8e755),LL(0x936a235c,0x34e9da69),LL(0xea446558,0x56f8560b),L_(0x868fe100), LL(0xf104982e,0x7a1b70ed),LL(0x2d932107,0x5df6c183),LL(0xb3a79a2f,0x3e690209),L_(0xddd8c4fc), + LL(0x506cdad3,0x1cd413bf),LL(0x8343be89,0x5af9735a),LL(0x4669ee32,0xf133df4e),L_(0x495a7ffd), LL(0x343b9134,0xd2df1205),LL(0x08509117,0xc0f593eb),LL(0xe1f7e724,0x03f70d2b),L_(0x56807867), + LL(0xe1aefce1,0x28e77727),LL(0x910ee9be,0xea43f71f),LL(0xb1afcd33,0xe72f4294),L_(0x04218f35), LL(0x5c609191,0xd85d4064),LL(0xe01905f3,0xd4734dca),LL(0xc6a7ed93,0x73444dba),L_(0xbd20eb77), + LL(0x786041c6,0x2e784b12),LL(0x3b8e5678,0xa5394485),LL(0xd5ed489c,0x7c1354f6),L_(0x37f214ac), LL(0x6f5aed21,0x04a2cef9),LL(0xfdcc232c,0x6a61903b),LL(0x0943ad59,0x74525b1c),L_(0x36735571), + LL(0x41295f9f,0x79058179),LL(0x53c2435b,0x8356b028),LL(0x8630275b,0x7d99704d),L_(0x22b975d5), LL(0xfe64fec2,0xb6ad9348),LL(0x1275938a,0x323c5204),LL(0xff571db9,0xabd2794c),L_(0x69883cf4), + LL(0x39e5ddd8,0xa00ac9eb),LL(0x0825e1a9,0x08022d52),LL(0xec00b4ca,0x89c305e2),L_(0x39410e3e), LL(0x2f6d8e33,0x1c9dee31),LL(0x189d73ba,0x063cbbc8),LL(0xd453f2df,0x500b8e1b),L_(0x200614c8), + LL(0x0d2d67f6,0x704fba9c),LL(0xe3b26afd,0x3b601782),LL(0xc86eeaf3,0x3a076280),L_(0xd5b4a74b), LL(0x057735bb,0x3c7fe621),LL(0x67519f38,0xba90bf98),LL(0x7ea681b4,0xc697f17d),L_(0x399fb8a8), + LL(0x2fb2d013,0x8b404954),LL(0x225dd20b,0x3864cc2c),LL(0xe8bf6c74,0x27928a4a),L_(0x0dbd4ced), LL(0x129459b4,0xb079e9c8),LL(0xcdaab48a,0xa8c506fc),LL(0x2be35d42,0xabd7b8a2),L_(0xc81bd43f), + LL(0x0c337ec0,0x9f5cd253),LL(0xb8d68d49,0x3bc3df03),LL(0x808378fd,0x57ce1b57),L_(0x7280fc5c), LL(0x9173aca9,0x29069aea),LL(0xec3e1405,0x172c5e75),LL(0xd7f9c6b8,0x2bb82de1),L_(0x7c6b6663), + LL(0x069b62dc,0x3a04b8c8),LL(0x539ec109,0xbb1fb7ee),LL(0x8e5dc130,0xdf0fbfa5),L_(0xd9cdb4bd), LL(0x8d6bb599,0xae962471),LL(0x8c98d809,0x6a0b5063),LL(0x1f5aad58,0xcb90fbe1),L_(0xbd8451cf), + LL(0x99c6500b,0x182ee0a7),LL(0xf250978f,0x9acab63a),LL(0xd7a3f799,0x579dffc7),L_(0x0a06a99d), LL(0x2e7244fa,0x7a563978),LL(0x780bf6d7,0xd9421c12),LL(0x4a33607f,0xca35ca5e),L_(0x4d6baab2), + LL(0x476c2c17,0x84dd3019),LL(0x1020ff66,0x9f6a9cbb),LL(0x8de41867,0xa0782cd2),L_(0x30b339ca), LL(0x335870d7,0x873bb057),LL(0xb2102899,0xd2763264),LL(0x817bb568,0xac61e5ba),L_(0x7167f834), + LL(0xb4216df4,0x28a00ba3),LL(0x72603953,0x8d18d559),LL(0x83dc382b,0x9929ba4b),L_(0xd887ba34), LL(0xc4acbdda,0x1a149f46),LL(0xabc8bd9c,0x03677afd),LL(0x7f382dd5,0x038b4fd6),L_(0x531a5b69), + LL(0x7b1469c6,0x770906b4),LL(0x29957782,0x39b76a30),LL(0xb757175c,0x77b00db2),L_(0xc63e688a), LL(0x00e5eb13,0x564b2b5a),LL(0x486048a7,0x16670a8b),LL(0x6cedfc0a,0xfc26f8d3),L_(0x158c04cf), + LL(0x5f72f036,0x492f15fb),LL(0xaf524838,0x8a57f34d),LL(0xca278cc2,0x136d3def),L_(0xf06989cb), LL(0x56bc1e3e,0xb7d079d2),LL(0x48aa5343,0xb849bec9),LL(0xc12eafab,0xf42c9064),L_(0xcdcc4178), + LL(0x35110b42,0x3593434b),LL(0x826b0084,0x2a685ee5),LL(0x38e7d616,0xd80d02e7),L_(0x3dd474b0), LL(0x47c0e164,0xa5dbbbef),LL(0x177aaf40,0x58c7a38b),LL(0x00b0f102,0xbe4a965e),L_(0x97ef9f54), + LL(0x00ea08ca,0x0a49eb36),LL(0xb9e97b18,0x4a5b799b),LL(0x1389e249,0x6bb607bd),L_(0x5f402141), LL(0x21dd6f71,0x648ac9f7),LL(0x0d6762ea,0x46ddb53b),LL(0x123f4a38,0x0b454608),L_(0xffc09d45), + LL(0xdc0b4f06,0xa26650ba),LL(0xc51c49ab,0x9623d15c),LL(0x52a5b5a4,0x8530847b),L_(0x9328fa21), LL(0x2b9634a5,0x88564796),LL(0x4c84e96b,0x8cd268e3),LL(0xdd09c6f7,0xfbbf3c6c),L_(0x6ae89ca8), +}, +/* digit=25 base_pwr=2^175 */ +{ + LL(0x076ba1e3,0xaf9141d1),LL(0x7d1bf374,0x90d62386),LL(0xeae39108,0x39ae931a),L_(0xbf9ff96f), LL(0x7c192921,0x51f12b8a),LL(0xf7008bde,0xf1cfbc62),LL(0xaa15f8c6,0x26ff0bea),L_(0x10e47e4d), + LL(0x5ac7dc32,0x5a5fa2f2),LL(0x727e3254,0x918f6160),LL(0x19e26cfd,0x3d611b68),L_(0x60d5395b), LL(0xfbe0d265,0xa4b3a16e),LL(0xc823108a,0xab7e3e37),LL(0xfb87b2d4,0xe4cf7a53),L_(0xa746dda9), + LL(0x28c21969,0xe6afbadf),LL(0xc63c394e,0x08d0d899),LL(0x79f622a4,0xfc61f01d),L_(0xbbd70173), LL(0x9afa4015,0x1819a677),LL(0x9acf1a59,0x9e89a4e9),LL(0xf6189381,0x4c306973),L_(0xf09c6806), + LL(0xd0fd8e2e,0x85f596e9),LL(0x8c824b93,0xc650d109),LL(0xc0ff364a,0xaa432f6b),L_(0xe94aff61), LL(0x60cd9ff7,0xa81387b4),LL(0xafe1dcb8,0x1fb57eae),LL(0x0f74dad8,0x77ebce28),L_(0x461e02cb), + LL(0xe7801dc0,0xc705f5f3),LL(0x19eba5b4,0x892e14fd),LL(0xee1d51c0,0x235f35c6),L_(0x32c28b10), LL(0xc690d3c0,0x496db46c),LL(0x35bd49b1,0xf95db976),LL(0xa0c8c115,0x3a2ff859),L_(0x347bf13c), + LL(0x21208e25,0x50a51554),LL(0x37fdbb13,0x76da2c79),LL(0x739f33c4,0x47f7655f),L_(0x5268d621), LL(0x6df3da72,0xda653a05),LL(0x63868730,0xe0dbba02),LL(0x7efefa7f,0x8546d123),L_(0xc050a28d), + LL(0x63a12c14,0x81bccb17),LL(0xb1d9c462,0xb8ad1f48),LL(0x2ee0f6a0,0xeca64ff4),L_(0x5d641bc5), LL(0x6c14c6a5,0x27d0478a),LL(0x3fb250a4,0x9dc8b9dd),LL(0xe6cf3269,0xde4a727b),L_(0xd5fdb843), + LL(0xe9139921,0xafcf2338),LL(0x6d85ee99,0x8626eb92),LL(0x330ff008,0x62bc5b8f),L_(0x75be09d1), LL(0x9ed3ab3e,0xacdfce8c),LL(0xfe5d4b90,0x198a1e99),LL(0x36ceed7d,0x366edc66),L_(0x88f5f0af), + LL(0x9eb6ef97,0xa0790c0e),LL(0x3c7a8752,0x8212d5cc),LL(0xa20f0405,0x2e6749a5),L_(0x52b0f123), LL(0x0b5bf3ab,0xf0598778),LL(0xad1df6a6,0x6b2a153d),LL(0xacd67e10,0x9a4287cf),L_(0xbc3d1a68), + LL(0x1df015e0,0x5545ae11),LL(0xaefec73e,0xf86d852d),LL(0x34894ebb,0x61b42915),L_(0x36e6b458), LL(0xb25bf6b1,0x2f858332),LL(0xa346bcfb,0x082b80c9),LL(0xa64fc13d,0xdb34263a),L_(0x5546d687), + LL(0x6c1819ab,0x46f4e2d8),LL(0xc8c8ebfb,0x3e6d5834),LL(0x31b5f8c4,0x0471eabf),L_(0xb0372dd8), LL(0x7377bdb2,0x4f0b793d),LL(0x1491549c,0x1e0b124d),LL(0x8be72522,0xcb228953),L_(0x28f9ea46), + LL(0x3d345d7a,0x4e4eb11a),LL(0x6b6a8f36,0x4f7cb301),LL(0xd3f60831,0x9b37cac3),L_(0x2fa1e5ac), LL(0xd42ac14b,0x4b82b6e1),LL(0x44efee99,0x062085e8),LL(0x293aa2ea,0xdafbaa02),L_(0x22eacbf0), + LL(0x66dc4353,0xeee6c065),LL(0xf4fb584f,0xe974bd6a),LL(0x51129e57,0xc6e911b2),L_(0xa817dc5e), LL(0x143fbbc1,0xe567aa72),LL(0x474c4763,0xd98a2bac),LL(0x985f3567,0x4940de3e),L_(0xb5a6e769), + LL(0xabba8dd9,0xbc9b4d54),LL(0x51333936,0x09ce7445),LL(0x56df5156,0x869985ec),L_(0x7ef1c4c4), LL(0xd64936f2,0xf9f4dd90),LL(0xb6f830f1,0xb0502e70),LL(0x9b4a0a61,0x2b7784e2),L_(0x5d8609f5), + LL(0x228b551b,0x132cf6ee),LL(0xe485fe61,0xb41622cd),LL(0x1166f594,0x4f82b870),L_(0xef156ec2), LL(0xc4a0434d,0x1f3f48d6),LL(0xc59ea2a2,0x185bfa1f),LL(0x0549b016,0xab597880),L_(0x88e57344), + LL(0x7783a237,0xcacbfea9),LL(0xa7d644dc,0x37384da2),LL(0xd767d6c6,0x1b40ecf0),L_(0x708738a4), LL(0xcb35e136,0x1bdb10dc),LL(0x946bff60,0x51f82512),LL(0x032e4873,0x86fff3dd),L_(0x4a72093c), + LL(0x993762c0,0x3c5e8711),LL(0x97ea967e,0xc3fb58ec),LL(0x5942a654,0xdc39e05c),L_(0x51f715f7), LL(0x4c7c1a58,0x76f183b7),LL(0x24b588e9,0x02f3f5a3),LL(0xc7635031,0x95a5b1bf),L_(0x90ff5787), + LL(0x44aeebfc,0x6fdec0d8),LL(0x3c09be94,0x4dccb72c),LL(0xc61c3dc6,0x29d98bbe),L_(0xe68c7671), LL(0x0b4d124d,0x60e1bdf3),LL(0x72ff08f6,0x60907706),LL(0x502b0c26,0x52fb8b7c),L_(0x3862b07e), + LL(0xa0c27cd3,0xa30b225d),LL(0xd6902dc0,0x14e5e016),LL(0x90beb6db,0xca71c799),L_(0x63380a26), LL(0x6406e7a3,0x8b449af9),LL(0x785b17f6,0xd95a3673),LL(0xe83116d2,0xc945cb30),L_(0xcbad56e8), + LL(0x195a49f2,0x19bb9a9f),LL(0x32abbd2d,0xdd63e900),LL(0x32a666f4,0xbb5bae80),L_(0x8a8b981b), LL(0x71975391,0x7200d75c),LL(0x58755037,0x9bb8f1f0),LL(0x34371f37,0xd14c5988),L_(0x4f91cd94), + LL(0x330b4d2d,0x4766d5c5),LL(0x29100509,0x96d443b1),LL(0xb683d913,0x55cd4038),L_(0x60386c8a), LL(0x0b19d842,0x4de7c5cc),LL(0xe7f655d5,0xb34627f5),LL(0x63d637ee,0x05d48e84),L_(0xda1b6f68), + LL(0xa9ab961e,0x587b4227),LL(0xaee0b38f,0x9bacdb2e),LL(0xf2fed11b,0x4327e6c8),L_(0xd04d5cf3), LL(0x59f2340b,0xdae690d8),LL(0x98397f13,0x20f9a82b),LL(0x04266881,0xb60ba27e),L_(0x0a08356c), + LL(0x4b7ea199,0xf7bd165d),LL(0x00c43b16,0xaff22d2a),LL(0x6e2c180a,0x5fa28be1),L_(0x03759f7d), LL(0x8d8de81a,0x8623fa12),LL(0xd1bcd513,0xf1b013b3),LL(0xecdeea2c,0x1412065d),L_(0x4abdd176), + LL(0x949b80c3,0x9e45f562),LL(0x91bfead0,0xeacf4d8f),LL(0x89e47023,0xed86f825),L_(0xe474eb17), LL(0xa46549d4,0xcc9d20c3),LL(0xbb6f6ac8,0xb5aaafc7),LL(0xaad820ff,0xfa9b0b7b),L_(0xd5d711cd), + LL(0x56a19c65,0x694699c4),LL(0xa52b01ba,0x2172bedc),LL(0xfcd8d182,0x71a622e6),L_(0xc5676308), LL(0x096356a4,0x5df0e982),LL(0x5f771127,0xbff93f13),LL(0x15499fe5,0x035d8bec),L_(0x54511618), + LL(0x8665d51a,0x0b392d7e),LL(0x19051cb5,0xda3e7314),LL(0x27724fe5,0xb0000497),L_(0x180fd98b), LL(0x92676515,0x86de4867),LL(0x0724b0c4,0xbfb7e396),LL(0xc9127734,0xec308376),L_(0x69d91fa4), + LL(0x13351e01,0x9ee3edde),LL(0x12d31182,0xe9b0df44),LL(0x039a9289,0x3ba48603),L_(0x776fb73e), LL(0x1bc5a4ee,0x28dbe0e2),LL(0x5d6f8bfe,0x33d13a46),LL(0x683a466a,0x6f0e40d3),L_(0x64598a06), + LL(0x49b1e543,0x5467b60c),LL(0xdafb9975,0x63a775bb),LL(0xf10e3ad8,0x555978a5),L_(0x90e960a3), LL(0xe2647c34,0x2d98173b),LL(0xeca130de,0xa46de612),LL(0xd4d665e8,0x8c2f942e),L_(0xd46cdc7a), + LL(0x83f96958,0x11159549),LL(0x2f2fe4fe,0xa26fb69f),LL(0xada04407,0xe1ffa9b1),L_(0x89077bf7), LL(0xf63dc16d,0xb8212592),LL(0x23757f3b,0x8f2b706f),LL(0xa84d6d60,0x0952d255),L_(0x867adb63), + LL(0xc1ad35d0,0x8063b121),LL(0x94420770,0x2c69c142),LL(0x2eddbe16,0xe20cbf04),L_(0x610d35ad), LL(0xc6a6486d,0x5f1b4348),LL(0x8963cf0f,0xaa7af662),LL(0xbd211312,0xf09747fa),L_(0xa7462c1b), + LL(0xb3384571,0xad4b300b),LL(0xeaf252a1,0x7e0408a9),LL(0x92c8f393,0xcdb75950),L_(0x95152482), LL(0x01c71c43,0x98806233),LL(0x65895709,0x4ccfab0f),LL(0x2c91c51d,0x0daddeb0),L_(0x6e5994f0), + LL(0x1656bed2,0xf9475d4e),LL(0x7ad0e424,0x29f67636),LL(0x71b2886d,0xce5ffc47),L_(0x5aec9cfb), LL(0xed19aefe,0x00fbe22e),LL(0x335636b3,0x1a593c52),LL(0xed7635e4,0xfea5dbc5),L_(0x0713b394), + LL(0x0a800d8a,0x3e952cbe),LL(0x96ad939e,0xb849574d),LL(0x95e45fa4,0xd0b6f06f),L_(0x495cdf6f), LL(0xca47bacd,0x3aea6517),LL(0x627d416e,0x98d62545),LL(0xf79f468d,0xe75cf2cd),L_(0x3f4c786d), + LL(0x92cfaed9,0x8aa22d94),LL(0x5767542d,0x951cb3e9),LL(0xcdf3e590,0xc6443f1d),L_(0x926947a4), LL(0xa83030be,0x4ab4bb2e),LL(0x1ac42dc6,0x126bf8d3),LL(0xab5c4613,0x78abc625),L_(0xa11b8fa6), + LL(0xb14045ea,0x75f0192a),LL(0xe56f26d4,0x66a1d69e),LL(0x488c7367,0x27671771),L_(0xe2f663dc), LL(0x41dc68c9,0xdbf2b527),LL(0x67ba17c4,0xaa6e15c1),LL(0xc052c755,0x6264eb14),L_(0xef0af1f0), + LL(0x361df88e,0x8133758d),LL(0xedb54b06,0x89d87e0a),LL(0x8313147e,0x156564f5),L_(0x0efd3686), LL(0x0305e3bc,0xb530ebe1),LL(0xd1426de8,0x75ed509a),LL(0xe3da205e,0x8810aa6d),L_(0x6d66996f), + LL(0x66bf33ae,0x5cdc1d84),LL(0x297ed94e,0x40fff510),LL(0x02569cc0,0xa46f7fa8),L_(0xf6c9bcc5), LL(0x03ae7466,0x6ca8cba2),LL(0x102909c9,0x9d34e4a3),LL(0x338f15f5,0xf175cde9),L_(0x9415a0f5), + LL(0x43fb29db,0x0f113fed),LL(0x55f28334,0xc9bc3623),LL(0x0891f557,0xf2de428e),L_(0xc85b958a), LL(0x72c78285,0x0bca6469),LL(0xfebcc59d,0x4ad07f91),LL(0x9aca547e,0x254d2e0c),L_(0x934ecc4a), + LL(0xbedf71c8,0xfdf235a0),LL(0xc42e06f8,0x707d37d4),LL(0x519516e4,0xb70d0d1d),L_(0xb746501f), LL(0x026e2e99,0x7b9cd555),LL(0x97c86a72,0x05a8ce5e),LL(0x490bbe2c,0xfd0ac7d5),L_(0x0e861b77), + LL(0x4518decc,0x5704b61f),LL(0xe3a1e11b,0x05e3a6d3),LL(0x761243d0,0x09fdcc95),L_(0xa10f1bbc), LL(0x52cc9710,0x2f40eec4),LL(0x45b65061,0x3cfbc5b3),LL(0x82e0b382,0xd24582c3),L_(0x0ea8545f), + LL(0x91b73502,0x63a13aee),LL(0x32b2623c,0x64a186c4),LL(0xd30674ab,0xae849f85),L_(0xc2109747), LL(0xbd3a7bd9,0x4e22b52c),LL(0xa5ad1cda,0xdee6130a),LL(0xa16934a2,0xa208c2f8),L_(0x29c61250), + LL(0xacddc899,0xdcfd6055),LL(0xaf5f3b40,0x4226123c),LL(0x22c7c6d7,0x18caa3b3),L_(0x11d8af8f), LL(0x27a1cc64,0xcfd9c842),LL(0x74dfaf98,0x435cc5af),LL(0x98cdd25d,0x03338f7f),L_(0xa8a15bc3), + LL(0x39e40f6f,0xa3f28110),LL(0x10846ecf,0xbb9b4a37),LL(0x6d2f9be1,0xc7ef39dd),L_(0xe4358a6b), LL(0xb0848ac1,0x9e6a4583),LL(0x0c20b047,0x65ecc897),LL(0x7961633c,0x12939ce7),L_(0xd7729b06), + LL(0xdf1e1998,0x0eb1a850),LL(0xbf310ff6,0xcfe64a90),LL(0x6d23295e,0xf848ec96),L_(0x96ccdfe8), LL(0x8a058164,0x4a1159c7),LL(0xfb67d66c,0x145557f8),LL(0x44e87592,0x744a6684),L_(0x93329250), + LL(0xd2524bb8,0x71ee45c1),LL(0x3c745d69,0xae2e62f4),LL(0x90655f39,0x4aa6dd85),L_(0xa8f50bcd), LL(0x170e2564,0xcd77d544),LL(0x84a26aa1,0x1637c5b9),LL(0xbbddebd8,0x116cc13c),L_(0x316cea62), + LL(0xda76037b,0xe567b31a),LL(0xc3337684,0x81454a31),LL(0x8ce526e9,0xa68887d1),L_(0x5860c2ac), LL(0x5ccae9e8,0x2aa4281c),LL(0x3740c0a4,0x27a06af9),LL(0x67a8cecd,0xb0666bd6),L_(0x8396d5d6), + LL(0x3bf215ba,0xf3b762b4),LL(0x2c7b4d47,0xc7ded91d),LL(0xf2dddefc,0x491f86d1),L_(0xe72784c3), LL(0x1a33ec17,0xbfa502ae),LL(0x6ff1a28d,0x8b90c1c4),LL(0x011d4fdb,0xae49e966),L_(0x6e8a2219), + LL(0x30030ab8,0x9b2323cb),LL(0xc742f24d,0x3dfca102),LL(0x6d677f76,0x615cef3a),L_(0xcc62a7bf), LL(0xd8927085,0x143864e2),LL(0x68578492,0xe2a53ef4),LL(0x7bfce607,0x98d95aec),L_(0xa4e64436), + LL(0xe4480a83,0x674699df),LL(0x638814de,0xf6bd5725),LL(0xa3bae3b5,0x93074bef),L_(0x401bd027), LL(0xc278e6d9,0x9075dc94),LL(0x33a4f7b4,0x66f70e53),LL(0x2912d169,0x77ae21bf),L_(0xd08ac30b), + LL(0xba0002d7,0x9d21da23),LL(0x313a021b,0xc9cb3757),LL(0xc0b1cdab,0x478c82a0),L_(0x71e210e7), LL(0x708a627d,0x95df97f0),LL(0x8f2e6f2f,0x036873be),LL(0x9b0c552c,0x31bdcc00),L_(0x7142f468), + LL(0xdb1b2797,0x16f6892d),LL(0xd070a81f,0x513e742c),LL(0x83e7ad99,0x5ba54190),L_(0x580b098a), LL(0x3ab45169,0xd55025e1),LL(0x9e175afe,0x57264580),LL(0xa5b527ed,0x246fe503),L_(0x27cfca13), + LL(0x9456aa99,0x72287d14),LL(0x56f44c80,0xbd98da02),LL(0x783e5a79,0x1c038840),L_(0xd6a90a4a), LL(0x44dd9215,0x8abbd0fb),LL(0x42622308,0x546492b7),LL(0x9365b6f9,0x638f9886),L_(0xd87dec07), + LL(0xe568c53e,0x56dd3ec8),LL(0xecec272c,0xad2418b1),LL(0x40cc9856,0x5fe0c00d),L_(0xd88d5b28), LL(0x1cdd7256,0x17304349),LL(0xa2eb70d1,0xe17ec8e0),LL(0xb4dc2a26,0x62eb4c57),L_(0x3373155f), + LL(0xdb4ee5e1,0xa85ff057),LL(0x2cbbaad9,0x0fb19074),LL(0xf1a5850d,0xb1c8c20f),L_(0x4ceec839), LL(0x407175d6,0xc25f5ad9),LL(0xc43bdd2a,0x0ed43d06),LL(0xa2739236,0x55db40b6),L_(0x59afdd51), + LL(0xbb91225b,0xb6ec7895),LL(0x64957b33,0xec277689),LL(0x53e7c8ad,0x503d01fd),L_(0x14800e2a), LL(0xff30aa1c,0x789118fb),LL(0x6b208671,0x45dce70d),LL(0x7758b97b,0x2ad05594),L_(0xbd833a82), + LL(0x64d25f39,0x7c1271cb),LL(0x973eb35d,0xafba6d12),LL(0x24a4fbdb,0x68588b5a),L_(0x86209415), LL(0x4408d2f4,0xb9962d65),LL(0xa7df2cdd,0x1adff958),LL(0x3e504930,0x1a7248ef),L_(0x642fcd83), + LL(0x50371a5e,0x10a09de0),LL(0x585cdc97,0x544a1f87),LL(0x05603976,0x9bb49da7),L_(0x36e0a9f7), LL(0x2686c471,0x22de25c7),LL(0xfc3ec645,0x7b318aad),LL(0x0e37c5eb,0x6e0af008),L_(0xb0fdc095), + LL(0xc4103bda,0x28c77b28),LL(0xc7d3392d,0xc44c9d87),LL(0x02942cad,0x045b3511),L_(0x633d355c), LL(0x985fde91,0x3e691e0d),LL(0x0d200049,0x66200341),LL(0xa7fa897a,0xc1b7c5c2),L_(0x5183d651), + LL(0x858b20b8,0x94f1ad67),LL(0x73a3848e,0xcb454e18),LL(0xe75e697e,0xea8630aa),L_(0x0d245b82), LL(0x7a69400f,0x7d8e1f8e),LL(0x1a8975af,0x87354e0a),LL(0xd949265a,0xddd78248),L_(0x59506113), + LL(0xbb95794d,0x56456f06),LL(0xb1cf847b,0xb5869a96),LL(0x7de9cd89,0x4dd26048),L_(0x29abee0d), LL(0x6b7c47c0,0x3a2f738b),LL(0xdc3e8b24,0x43a158b5),LL(0x6452e6ec,0xb59a08ed),L_(0xdca422db), + LL(0xfd294203,0xd446ba91),LL(0xb382c52a,0x0bfb0e2b),LL(0xb347e482,0xda5432f0),L_(0xc465aa05), LL(0xda7568f3,0xfd584f1c),LL(0xb3f81a8e,0x3e5b388d),LL(0xfbef98fa,0xa2090daa),L_(0x7ee59c28), + LL(0xebc6de85,0x8601a460),LL(0xa197c14a,0x0fe7b2f4),LL(0x51718d1f,0x768e05f3),L_(0xd61640e4), LL(0x107bee82,0xdafff971),LL(0xd549077c,0x775aa266),LL(0xf5af9168,0xa74ac1aa),L_(0x709e0299), + LL(0x7770d24d,0x6c27e8ad),LL(0x5f6805b9,0x0c21256b),LL(0x5eb3480c,0x24df253e),L_(0x33b3350b), LL(0x32c4c951,0x304257df),LL(0xd07bd3eb,0x3821aab8),LL(0xcfd48043,0xe9ed7945),L_(0x29f131a5), + LL(0xf927be41,0xa625fc39),LL(0x0acd5378,0xc708ccff),LL(0xc7c05c00,0xe3551df0),L_(0x5a9c25df), LL(0x4a824ae7,0x00aff255),LL(0x3fad48ea,0xdc1865b9),LL(0x489f78db,0xc47678ec),L_(0xe24fe1d5), +}, +/* digit=26 base_pwr=2^182 */ +{ + LL(0x4d52ad99,0x44075a5f),LL(0x49de85ac,0x5f264197),LL(0xa5da6ace,0x1bd94b24),L_(0x58ceb7a8), LL(0xaddbfb85,0x9deccfa5),LL(0xeb853c9e,0x38b0cd4c),LL(0x0ae5bd28,0xcdefc743),L_(0xefb92de4), + LL(0x9f68d2fd,0xdb80636b),LL(0xeb6a625d,0x399cbe21),LL(0x058ae3aa,0x9a20e964),L_(0xeefd7139), LL(0x0655d447,0x82be9278),LL(0x76a67bb6,0x02a07951),LL(0x78c7207b,0x5473af3e),L_(0x5b501b3b), + LL(0x48532008,0xe1daa6d9),LL(0x9dfa0916,0x8ffcdbb0),LL(0xbd0a98e5,0x9b2bdde3),L_(0x5bcf0ad7), LL(0x58bd4135,0x4dcdfb2b),LL(0x913a8546,0x29153442),LL(0x7457124c,0x9190c343),L_(0xc113bf67), + LL(0xe34c46c8,0x1a8f4562),LL(0xb45aadf5,0x6cddda1d),LL(0xb1bb8fc9,0xa502ea7e),L_(0xf3ae0efa), LL(0x9421a0ec,0x124b21a9),LL(0xa80fcb58,0x634ea314),LL(0xfc9b2b69,0x4d2d4d06),L_(0x619c1654), + LL(0xa4e345b7,0x370847bf),LL(0x9d1ecec7,0x242713b8),LL(0x10782ff2,0xbc286c44),L_(0x2b85247f), LL(0xbe6fa012,0x72197eb9),LL(0xec864a68,0x23d8a845),LL(0x9c366368,0xe57509e0),L_(0xfafe2bf5), + LL(0x9344ebd5,0x41626ac2),LL(0xa5a7b2b3,0xb61e001b),LL(0x1c404975,0x05deb3c3),L_(0x4861d13f), LL(0x10e30e7a,0x4f5429b7),LL(0x409e441f,0x0cc2b3bc),LL(0xe64edacb,0x2a32f215),L_(0x502743d8), + LL(0xa18ec5e4,0x597408da),LL(0x75198a41,0xb4dc90d0),LL(0x9e4003b1,0xf0377012),L_(0x97c24d6f), LL(0x2f737bfa,0x138b7393),LL(0xc6646756,0xbf91bb7c),LL(0x4390dd24,0xd3eb9506),L_(0x660fe2be), + LL(0x1f5f9c09,0xcbb88522),LL(0xde4f8a6e,0x590e948c),LL(0xd63bd64a,0x45a94350),L_(0x82568ecb), LL(0x8b9c0e80,0xc46a6c09),LL(0xe8dd7862,0xe5cb409e),LL(0x6aa2f787,0x90fd8928),L_(0x3c0fb202), + LL(0xc519de27,0xc3ac3715),LL(0x59e082f8,0x915329a3),LL(0x0ca44fa0,0xe61346f3),L_(0x590df3a0), LL(0xe4d76c98,0x37f7afee),LL(0x58d6ab1b,0x6bdfe1fe),LL(0x536926c2,0xc048c2a4),L_(0xdd33cbc7), + LL(0x57103467,0x9e779a72),LL(0x24aafef8,0xf0cf02a3),LL(0xc4a65bf2,0xc464a2f5),L_(0xcc014eab), LL(0x771064c4,0x545f80d2),LL(0x90630a3b,0x142107aa),LL(0xc2378a52,0x233fc912),L_(0x60b64ae1), + LL(0x2c57b6ec,0xe31036a2),LL(0xf8434ff6,0x18945c37),LL(0xa0a4a4f0,0xa70971ce),L_(0x2a1d924f), LL(0x78d2269f,0x3f54f031),LL(0x59b13396,0xd1a3f071),LL(0xcf63943b,0x1bbad8a4),L_(0xae570fbb), + LL(0x7535f2cf,0x2202c2ff),LL(0x6ed5c874,0xad072699),LL(0xc576098a,0x233792b7),L_(0xbaf56f3f), LL(0x58643430,0x02886c17),LL(0x64569bb7,0xb1a39466),LL(0x74bab9ee,0xe841def5),L_(0xc973e3bc), + LL(0x7f584615,0x0ec350c2),LL(0xca028343,0x2da65635),LL(0xb160863f,0xe9a43081),L_(0x72084a70), LL(0xce3663d1,0xafbf1395),LL(0x1aeb19b6,0xdfa06d4c),LL(0x9aa85d56,0x4f103987),L_(0x4f14057c), + LL(0x7df657ba,0xf5ac31c8),LL(0xd4e08231,0x1a02ac9e),LL(0xd28fa99b,0x6407f45c),L_(0x36b31d91), LL(0x06494082,0xe60e26a5),LL(0x03bcc061,0x44cfce93),LL(0x4db05d61,0xf189ccff),L_(0x635d166c), + LL(0x16a82c08,0xd179a921),LL(0xec3df3ad,0x51227dd3),LL(0x1d15b312,0x129de8ff),L_(0x99a2081f), LL(0xe117dd1e,0x3f1a519b),LL(0x669c965d,0xd473a86e),LL(0xe470bd93,0x48ca17dc),L_(0xd7172ef7), + LL(0xce23cb90,0x025cce52),LL(0x54ac5788,0x7128e28e),LL(0x60a7af16,0xee37af3a),L_(0xd328cf8c), LL(0x228c1194,0x8bcaaced),LL(0x7eb8e5dd,0x60835723),LL(0x5f7c29e2,0x42db013e),L_(0xd6e3c26e), + LL(0xd7266cda,0x5ce7356c),LL(0x31f3b95f,0xc0bdd70b),LL(0x086e72f7,0x3319c072),L_(0x7f9cf187), LL(0x4ae3ba6a,0x086343f9),LL(0x85584e27,0x80bc2aba),LL(0xbf3ecc51,0x37cadfe3),L_(0x10a2f1b8), + LL(0x66dd80da,0x32cd82a6),LL(0x5a4759b6,0x11e8daf7),LL(0x6268a0b4,0xbd297ce7),L_(0x554b93e1), LL(0x971bdef9,0x8c4eabfb),LL(0x5bdd9c55,0xcbfa339e),LL(0x83d9f14e,0xbd36bd3d),L_(0xe88e0095), + LL(0xe3a86075,0xf0d2a9f0),LL(0x7ea9a94b,0x68a87b41),LL(0x27fad851,0x465da389),L_(0xc79beddb), LL(0xa438dfad,0x1024a804),LL(0x9e4004c2,0x3e3b2c82),LL(0x9e904c3e,0xb0cbbf5a),L_(0x91364ced), + LL(0x4f3cafc8,0x4571821c),LL(0x05a71822,0xb59cdc87),LL(0x564a1eff,0xd3fc8e11),L_(0xc2522075), LL(0x210954cd,0x485b3e16),LL(0x3e42143b,0xa39a864f),LL(0xc71b961e,0x50587567),L_(0xcd472e0f), + LL(0x3f39a45b,0x39ff9b1a),LL(0xb67c7e0c,0xe9320256),LL(0x293c62a7,0x30d60ad1),L_(0x6c70033e), LL(0x51699001,0xecc36d79),LL(0x66761810,0x782811c0),LL(0xb9b6100c,0xd4aec2a8),L_(0x01fd70b0), + LL(0xbf6700ba,0xdb67bc3c),LL(0x0705c260,0x81a05a44),LL(0x2a63deb4,0x2c00d649),L_(0xb057ad61), LL(0xc429bae3,0x962a325d),LL(0x94f128db,0xf818f010),LL(0x07c3cde9,0x8b0fa963),L_(0xb0a83496), + LL(0xdcf6f7fd,0xd9cd3792),LL(0x63f97d0d,0xa1b3479f),LL(0x9daa99a0,0x6b0d6566),L_(0xe8984ec2), LL(0x448838bc,0xe794fda6),LL(0x82c274f4,0xb6b1fe7b),LL(0xc0cb0a63,0xd09a8df9),L_(0x17a04bc3), + LL(0x4c395b4f,0x8e70bc90),LL(0x4ee87155,0x04b3e852),LL(0x9d40e63f,0x4a2dc6ce),L_(0xfaef51af), LL(0x36339e47,0x385a4594),LL(0x20622007,0x455a09ad),LL(0x9304dd88,0xc6ff5bf9),L_(0x913f4764), + LL(0xe085cdf8,0x187b4002),LL(0x185e6ba2,0x4c4e720e),LL(0xab690dee,0x86287cf7),L_(0x8afd2d78), LL(0xe2c63b4f,0x9adbcac5),LL(0x7132999e,0xddd28bc6),LL(0x4248e502,0xef074338),L_(0xfa2e3db0), + LL(0x896670f4,0x08020d96),LL(0xfcabc640,0xe06c4b5e),LL(0x9786e6bf,0x3da82523),L_(0x3d688833), LL(0x4fe3491f,0x99758e3b),LL(0x5ec531f3,0x0c3b82b5),LL(0xbab9fec3,0x45c4baf6),L_(0xb5f6610c), + LL(0xb5a9a1e0,0x4e7e7ab2),LL(0x1b52980f,0xce6cd002),LL(0x7c86bb7d,0xcadc14a3),L_(0x3b8e5f2a), LL(0xbef7e6fb,0xbff62993),LL(0xccd76ec4,0x0a2850cc),LL(0xc5820cdd,0x92bdfe5c),L_(0x082f1255), + LL(0x484daaef,0xd799a2f4),LL(0xedb03630,0x550daad0),LL(0xef0ac41a,0x8fbf8f4e),L_(0x737b2d81), LL(0x84f7e137,0x7bdb6e8f),LL(0x34c35ad6,0xd89df99b),LL(0x395821cc,0x20d76719),L_(0xc88df817), + LL(0xae387d39,0x760e59a1),LL(0x688c2755,0x97e8f74e),LL(0xbc044138,0xa21350e4),L_(0xc028fc40), LL(0x0253354a,0xe3e6583e),LL(0xb20dddcd,0xc654b121),LL(0xa829cd8c,0x39c79315),L_(0x792dd84b), + LL(0x040ffaf0,0xb7bea5b5),LL(0xf1650ec3,0xdb5241ae),LL(0x372d1796,0x940a8a94),L_(0x1a858744), LL(0xf1499afe,0xcabac4a3),LL(0x8ded605f,0xb861df88),LL(0x0cb6a234,0xa96fe9bb),L_(0xbc8b296e), + LL(0x5eedd185,0x4434f8b6),LL(0xd1edd68c,0x2a8d7504),LL(0x86f300e0,0x0e35d724),L_(0x897d3aeb), LL(0xfaf29ff1,0x4c814382),LL(0x472c2113,0x141d9196),LL(0xd1bc5f13,0xade4deb9),L_(0xa2fb8cce), + LL(0x406ace5c,0x1ea243b0),LL(0x686c6adf,0xd6070206),LL(0x22934b39,0xd9cd49e0),L_(0xa0234a2a), LL(0x4d0f3175,0xc755b33c),LL(0x5c35ad9b,0xf8e1a985),LL(0xc50467c8,0x18a3ef6a),L_(0xf2b99d2a), + LL(0x4607dced,0x614bcea6),LL(0xd3d623a4,0x68ae9193),LL(0xf3906a4b,0x42d006dc),L_(0x8b8583eb), LL(0x4cb64b0e,0x9dbec629),LL(0x98a17fc9,0x0bf6a2f2),LL(0x057751b4,0xa9fae60b),L_(0x21e7bd24), + LL(0x3c12fc75,0x017d14d0),LL(0x3242c2d8,0xc6c97422),LL(0xa51b04ab,0x626c81c6),L_(0x94d87c6b), LL(0x238bcf04,0x5e266a2a),LL(0xa82fa338,0x23764002),LL(0x51803104,0x5f7162bf),L_(0xc4756473), + LL(0xa5dd759f,0x522db1a2),LL(0x5930b4b5,0x09f2baed),LL(0xf85a6ac1,0x323304d4),L_(0x05195116), LL(0x1360791c,0x1db93b31),LL(0x108292f6,0x1b768230),LL(0xabf3fed8,0x87d85b61),L_(0xa1679cd2), + LL(0x761f1b7d,0x0b49a02f),LL(0xb6746bd7,0xbb854597),LL(0x77637e3e,0x0022290f),L_(0x86597269), LL(0xf0700421,0xe0410aa3),LL(0x2a1bca0c,0x6d34be85),LL(0xf6bc18c0,0x46cdba65),L_(0x203e8ea6), + LL(0xb9773b5a,0xc5228020),LL(0xba11b539,0x453c7ac2),LL(0xf11276b9,0x7df2cd47),L_(0x1949a67a), LL(0xf5a8c36b,0x179c274d),LL(0x860ff0ae,0xa5c5bb12),LL(0x11df901f,0xe17d00c3),L_(0x3411dcae), + LL(0x3ed4c79b,0x8883a1db),LL(0x1992a967,0xfdd5776e),LL(0x2f54ff27,0xcb776738),L_(0x47290630), LL(0x9a98b107,0x79c8cc7c),LL(0xee2e8af1,0xdc2c5969),LL(0xc7ee141c,0x6b47161b),L_(0xa7624962), + LL(0x543685fa,0xc2395393),LL(0xe7ef754f,0x9a0b36af),LL(0xaae1a4bd,0xd5d4e38e),L_(0x4aab543b), LL(0xc2e17c97,0xd7e5c5cf),LL(0x5af349b1,0x2bef05ff),LL(0x9ed78a64,0x561fb810),L_(0xf9c813d0), + LL(0x2fe8fb72,0x1a1e9b3b),LL(0x9b70eefa,0x3925b0e9),LL(0xbfb2449b,0xb054dfbe),L_(0xb47dede7), LL(0x28647224,0xa5055940),LL(0x0c72430a,0x31e1b9a5),LL(0x23b5a4c4,0x195035c8),L_(0xd679bea8), + LL(0xcc50c48d,0x209f3af8),LL(0xd27b6e83,0x181662d3),LL(0x4b465365,0x91296676),L_(0x420e37bf), LL(0x138dff74,0xed5f5189),LL(0x82f8d0f4,0xc0ded518),LL(0x40df6431,0x723fb4ad),L_(0x617ba2b7), + LL(0x112db662,0x3569136f),LL(0xb2b47e3c,0xb8e53774),LL(0xdd45fe78,0xfaa04e8d),L_(0xc768fdda), LL(0xf6371e54,0x86806849),LL(0xa8e9db9c,0xf4779601),LL(0x4c858ecd,0xa4992880),L_(0x49a83816), + LL(0x8967314a,0x156e1198),LL(0x7bad4014,0x39eb8236),LL(0xecde7783,0x6d9c66e5),L_(0x881e0782), LL(0xd2f38b48,0x3fba7225),LL(0xed8353b8,0x088944d1),LL(0x4f8035c2,0x33f1f404),L_(0x1cc9c213), + LL(0x51cb66d6,0x58d0680a),LL(0x50bb5809,0xcb3e58aa),LL(0x1d2325d3,0x13427358),L_(0x5e67c4ac), LL(0x7dbb0bf2,0x85f983bc),LL(0xa937cb56,0x0f88839a),LL(0x8a991a31,0x6b3985b4),L_(0x92b4731e), + LL(0xdf68b708,0xde5b911a),LL(0x03d7de13,0xeed92206),LL(0xfc74f9a9,0xbcecf38d),L_(0x088b6fe1), LL(0x96e739c0,0x0a6c0e27),LL(0xd4a63fbd,0x82400edd),LL(0xbba8f32c,0x27d5bb2b),L_(0xc58b1cab), + LL(0x3131df11,0x042f1a03),LL(0x4f121cf6,0xe003b634),LL(0xa22e7895,0x6dd1e8b0),L_(0x5d804ab1), LL(0x90518b2c,0x5efe4007),LL(0xe1e8157d,0x88b87e5c),LL(0x63c085ee,0xf6155e0f),L_(0xba6e44df), + LL(0x841e4ade,0x1f02ab50),LL(0x156d76af,0x67a50be5),LL(0xe6f206a7,0xf1e42056),L_(0x65f10a7c), LL(0x2febb4c7,0x657a0dc7),LL(0x072e3321,0xbbf3484b),LL(0xea4f285b,0x020b154e),L_(0x8c852c97), + LL(0xdbbe1818,0x91e4e9f4),LL(0x2edec0de,0xbf43b7fd),LL(0xe4667773,0x52865635),L_(0x56a43f3c), LL(0x5962ba87,0x28fe8ab8),LL(0x248c61c8,0xd449338b),LL(0xd636805c,0xa1efd1c9),L_(0x13bd6609), + LL(0xdfd448ab,0xb92e258d),LL(0xe4e24c18,0x35b6d45b),LL(0xa42ce187,0x0d195771),L_(0x5aad5bff), LL(0x672eaebc,0x60feb26e),LL(0x4177b162,0x9498986f),LL(0xb3b0ede4,0xc41d2caa),L_(0xf358519a), + LL(0x5261f5a7,0x330e8a48),LL(0x43204369,0xf61c72d3),LL(0x33ca48fd,0xffeebdb0),L_(0x8ae6e55b), LL(0x7ae9478f,0x73383a11),LL(0xa6e09378,0xafb5d295),LL(0x689cfdb3,0x32706d23),L_(0x868c5ddf), + LL(0x0bd73e3c,0x62c3a48b),LL(0x8741583b,0x0c0a3d1d),LL(0xacf48d35,0x48c97582),L_(0xa4f06efa), LL(0x01b5bb29,0xaf780b08),LL(0x6bf0d377,0x2a0a2d75),LL(0xabcaf322,0x13f7f83f),L_(0x655d34bf), + LL(0xb992a7a0,0x19a0cf96),LL(0x9ecda6fc,0x98adc9dd),LL(0x6a4d3e6c,0x40f8faf2),L_(0x0edc38f8), LL(0xef011d38,0x7e67e105),LL(0x1797a86e,0x87eb4d53),LL(0x45add6b3,0xce77f195),L_(0x989201fd), + LL(0x3d9993bc,0x8adaa277),LL(0x99dda71d,0x1fca3093),LL(0x7433657e,0x8caf0d86),L_(0x023c11e2), LL(0x85112381,0x3dbd4315),LL(0x708185e8,0x9d410bfb),LL(0x374f5e1e,0xf2fcce49),L_(0x3443b974), + LL(0x532b7588,0xd3243b22),LL(0xfed73137,0x11c1365d),LL(0xb045d4a7,0xb70776bc),L_(0xfa754876), LL(0xadb798db,0xe29a9f85),LL(0x7a404c41,0x04e83976),LL(0x4ef414c2,0xd94bbca4),L_(0xca54266d), + LL(0x69a5c564,0x189faa10),LL(0xea80ccb0,0x11ff4091),LL(0x270bddfa,0x13f400ef),L_(0x16bba259), LL(0x54c1f416,0x5467f97f),LL(0x63fe2fd7,0x8a19f9f7),LL(0xd713b7c7,0xbecb24c4),L_(0x548c6ef8), + LL(0xd9e41b7d,0x154bb407),LL(0x5fe1b516,0x9e3d7f55),LL(0x0a7b54c7,0x135359b7),L_(0x56bae4a1), LL(0x84d9fef8,0x5d18cdb1),LL(0x3494e228,0x142d7261),LL(0x1cba3904,0xac50bd6c),L_(0x2d2c512d), + LL(0x709ebb7e,0xec49c916),LL(0x906c201e,0x16f1cdf7),LL(0x9b4ca4e1,0xd65f4a10),L_(0x474e1186), LL(0xf98d2d15,0x03b182b1),LL(0x94cb6ea7,0x060b7206),LL(0xb11e15e8,0x6d96c56b),L_(0x52f7fca7), + LL(0x8f77e5e5,0x72cd83b6),LL(0x8cab24f9,0x25a5b5bf),LL(0x25a2e699,0x6fdd7673),L_(0x072fab5d), LL(0x90a6d4ff,0xd576b9f6),LL(0x3ca8ca37,0xbcb25091),LL(0xad2b4418,0xefb79bc4),L_(0xff9c27d0), + LL(0xfba463c7,0x67359a81),LL(0xfe5bb23c,0x2c69efd5),LL(0x280e7df6,0x74569519),L_(0xfe16bd9c), LL(0xe2c76094,0xb18f6667),LL(0x365a80b1,0xbd326eec),LL(0x88831553,0x87869f49),L_(0x5939e645), + LL(0xbaa57c7d,0x3b207add),LL(0xfb5db15b,0x4ed05149),LL(0xb373586a,0x309bffcc),L_(0x773b90d1), LL(0x461b005c,0xc0f53c53),LL(0x8106712a,0xc558624a),LL(0xcda6e3b1,0x6faa11b6),L_(0x3ae794e5), + LL(0xf0680ab3,0xcbfa17ef),LL(0x058c2020,0xcddb5b82),LL(0x41c78a0d,0x20234c33),L_(0xe0cbaf8b), LL(0xc2fe9029,0xf4f1f1bc),LL(0x7a0fe46d,0xfc36a9bb),LL(0x75517f43,0x7adc3f7c),L_(0x9a365f26), + LL(0xba51387b,0x49c36869),LL(0xc5a22f24,0x03e373c6),LL(0xb8b4767a,0xb07a3f0c),L_(0x36066a8f), LL(0xe85c503a,0x808f702e),LL(0x1497d7ac,0xdddd86d9),LL(0x7011d76f,0xfae1c7e7),L_(0x803e46f1), + LL(0x80fa818c,0x31e4ef2b),LL(0xeb39da59,0x68d478dc),LL(0xb3f48a44,0x7630fe61),L_(0x067101b0), LL(0xf7c94ecf,0x9a533238),LL(0xaea4959f,0x5e7ce695),LL(0x3b49d1b1,0x52a95933),L_(0x557cf5b6), + LL(0x0d20898f,0x9d1c985c),LL(0x072ec166,0xc62ed966),LL(0xce3b7676,0xeb820161),L_(0x37e4d792), LL(0x60f14619,0xe6d270e0),LL(0x648b3694,0x80e6594a),LL(0x9e645987,0x879e807c),L_(0xb22e7cb2), +}, +/* digit=27 base_pwr=2^189 */ +{ + LL(0x0703230c,0x9c23cc7f),LL(0xad5df365,0xa6d601b3),LL(0x3f2e4ff0,0x8b1b5936),L_(0x1dea5297), LL(0xe0882f47,0x402ec9ab),LL(0xa3a23af8,0x47de2c7e),LL(0x555b4d4e,0xeaa5bf4d),L_(0x2ff94aac), + LL(0xf78b555b,0x971ade84),LL(0x54bd5a85,0x4e8a2ef9),LL(0x5ad4ea73,0x0425d602),L_(0x3bd54fb7), LL(0x791d9123,0x227d0459),LL(0x1e92cc4f,0xc9d765a1),LL(0xe4f1346c,0x28b65397),L_(0xd481a746), + LL(0xe7d16971,0x871eef82),LL(0x0506636f,0xb5fdd432),LL(0xb27a33c5,0x1d066c2a),L_(0x3113bf2a), LL(0xe97230fe,0x862b60db),LL(0x75a027c2,0x593d4117),LL(0x539f6530,0x8436da66),L_(0xe2b2cbf2), + LL(0x2425af9f,0xead58c17),LL(0xcad19bb2,0xb5cbb50e),LL(0xccdd8dd9,0x86b3650e),L_(0x1a092795), LL(0xdc320635,0xa3e665cd),LL(0x7ebcad7d,0x1e2d319e),LL(0x33fae415,0xeca579c1),L_(0x6ac7d5f9), + LL(0x6a9d1fc5,0x89b65031),LL(0x881ef8ac,0x23a987e0),LL(0xfde88e7a,0x2dd37a50),L_(0xc8dcf85d), LL(0xcebc549c,0x820f3f12),LL(0xd4ae9ff8,0xb7bcaa86),LL(0x694bc71e,0x8e6e09c1),L_(0xd6b8424d), + LL(0xfe8a98f0,0x63ff767a),LL(0x378e38e8,0xc5f7f4d1),LL(0x0e5335e1,0x7cfba5a2),L_(0x39a8daab), LL(0x77da1977,0x67e59eaa),LL(0xad21218b,0x504acfd4),LL(0x96be881c,0xbeef988a),L_(0x69332afc), + LL(0x82325977,0x575e852e),LL(0x45759819,0x2c72e2f1),LL(0xe0cc66d6,0x1927e387),L_(0xc0500878), LL(0x4d0b0122,0x7da1cf59),LL(0x14427010,0x26204672),LL(0xdb25f5a7,0x0a8b1258),L_(0x478d5953), + LL(0x7d69f704,0x20d3988b),LL(0xab2517df,0x6c24b0c1),LL(0xaab259f9,0x5e65b843),L_(0x174ca2d2), LL(0x75e5c2e2,0xea29f462),LL(0xdb90528b,0x1834a190),LL(0x8c9eae94,0xf967c0ea),L_(0xae030eb0), + LL(0x7e9bdb89,0x00adf08d),LL(0x03f389c4,0xa0138acf),LL(0x8dfb7875,0xd93b37ee),L_(0xbed6d6d8), LL(0x749de946,0xbfd574b2),LL(0x1fc14060,0xe75abd5a),LL(0x67c65b64,0x13366a27),L_(0xc999214e), + LL(0xb08be401,0xe06fcef2),LL(0xa7dfbf73,0xe0ba359e),LL(0x607c14ca,0x382ff71f),L_(0xdef77eb0), LL(0x6404f03d,0x3f8a49f7),LL(0xf9e8050d,0x7853702d),LL(0x4887d191,0x04d3f5e6),L_(0x1aa6bfdc), + LL(0x67857252,0xdb805369),LL(0x612fee31,0xd63c66e4),LL(0xa73ff057,0x587ee86f),L_(0x2d5eb654), LL(0xa10a675a,0xbd927cb1),LL(0xf79fccb5,0xc799e97c),LL(0x6f65ee19,0x4703e5a8),L_(0x04d4c022), + LL(0xaa2f24f8,0xba32a728),LL(0x2229afc4,0xb7ad366d),LL(0xa0ac6a50,0xb220a09e),L_(0x5da587fb), LL(0xd5db1088,0x44a3150c),LL(0x31856642,0x09784fd1),LL(0xf6f05498,0xc1e6c0b0),L_(0x346aa16b), + LL(0x06b7160f,0xbb692e48),LL(0x89eb623c,0x15498ecf),LL(0xb77b124d,0xbb092f5d),L_(0x3e60aee2), LL(0x74e3ca98,0x7b0969cb),LL(0x9ba2e8fe,0xf86a2cf8),LL(0x6d59ed66,0xdeabc883),L_(0x39e0a846), + LL(0x17174a2f,0x852eae73),LL(0xbaf85e02,0x1e348f19),LL(0x0374bff3,0xeacc5568),L_(0x60bfb120), LL(0x5585a76d,0xa921615e),LL(0xf43bee83,0x1b74c1fc),LL(0x26867d50,0x52fb37fb),L_(0x7279cdee), + LL(0xb96496b8,0xe996dbee),LL(0x7868fd21,0x39670ef3),LL(0x8094e471,0x9c159e05),L_(0x2d0b6ffe), LL(0x4226465b,0xb7fd505e),LL(0xe7884414,0xf2355f66),LL(0xe46e77c4,0xe0fbfc8b),L_(0xd7928658), + LL(0xc386d033,0x16b11ec1),LL(0x6cbaf218,0x75da11d8),LL(0x94c7a19f,0xfda19041),L_(0x15d41b37), LL(0xcd2bfd31,0x0e5ba2ee),LL(0x8a34c58f,0x2339d87a),LL(0x375d7766,0x24af1974),L_(0x280bf34c), + LL(0x0c7c5c0f,0x06c1fc0c),LL(0x8c4fcedc,0x1d94e717),LL(0xb65c6a50,0x0b30821a),L_(0x2a34b8c1), LL(0xefa6a39e,0x6459b552),LL(0x49fdfbf5,0xb7d7e4b6),LL(0x18598afa,0x00b319ed),L_(0x287de14b), + LL(0xf884841d,0x8dad7679),LL(0x518b1ea0,0xb306db5d),LL(0xb36c0754,0x5229df9a),L_(0x8fc9f729), LL(0xaadea0d8,0x970bb22c),LL(0x03b629f0,0x298689dc),LL(0xf3b57c01,0x89fe17ad),L_(0x686552a0), + LL(0x76218fd5,0xceff7d9d),LL(0xd9e895e9,0x63a383ce),LL(0x238b4e91,0x99f29b16),L_(0xbd71b01c), LL(0xe11ea486,0x7ae47927),LL(0x605101be,0x7cd7a8d0),LL(0x0d9f157a,0xb3fa2b87),L_(0x330f1244), + LL(0xff049f89,0x85fe8244),LL(0x6ddb5ffc,0x5d2ff231),LL(0xef0f5b1a,0x63ccdc00),L_(0x61b2bcef), LL(0x8ba95ec1,0xa8a44260),LL(0x5321922c,0x6ed75cfa),LL(0x3e43af21,0x8464e573),L_(0x6a65d52d), + LL(0x65d29b4d,0x1583a353),LL(0x32ab9cfa,0x43193114),LL(0x92998e59,0x704abb0e),L_(0xbbb3d90f), LL(0x2a7e16f7,0x4c98adbf),LL(0x7bbafb3d,0xa5a25fbe),LL(0x44dc73bb,0x8fc488f7),L_(0x6641759d), + LL(0x784f9378,0xe7bcd034),LL(0xabbc4935,0x89f2f24e),LL(0xbec66d27,0x4917e013),L_(0x5c4012be), LL(0xf0cde4d8,0x0961875a),LL(0xa3f96a56,0xa260b4de),LL(0x59faeb8c,0xbeeb1631),L_(0xc5c3e45d), + LL(0x280f3963,0xa088a1e1),LL(0xf1068e65,0x1dbee472),LL(0x1692a131,0x05a158ea),L_(0xde0f97d9), LL(0x0a11b122,0x0e13e263),LL(0xb70fb035,0xdee544e8),LL(0x5dcaae12,0x8f1fbb34),L_(0xd7c3d865), + LL(0x5d2e90ae,0xb9ad99ac),LL(0x75457057,0x0c329529),LL(0xaa459865,0x96ffc387),L_(0x7823d3a1), LL(0x13516c2f,0xa0aa883b),LL(0x6d68a755,0xbfa72fcb),LL(0xa825f1f8,0x97da099b),L_(0xcba9849f), + LL(0x14bced83,0xf4fdf535),LL(0x671c909f,0x689f6f0d),LL(0x71814baa,0x5b8ff385),L_(0x984a3eb2), LL(0x9c1ba7cf,0x907e4e69),LL(0x6fd404b5,0xff0ce18b),LL(0x2458cb25,0xef5cd0aa),L_(0xdb837d9e), + LL(0xfee8b8f0,0xa4710d21),LL(0xca118a9d,0x54895b97),LL(0xb541a6b0,0x526b0ca4),L_(0x46f624ed), LL(0x16947323,0x31b294e5),LL(0xd0b8b7cb,0x1b4cb2b4),LL(0x98687802,0xfbcc3509),L_(0x58f68319), + LL(0xb150cf08,0xe16b93aa),LL(0xba41d3c9,0x253eb0dc),LL(0x984df68e,0x87118c04),L_(0xea79efa1), LL(0x480076ed,0x0dcde0c7),LL(0x36503c9e,0x18bdffd8),LL(0x16aa0174,0x1f265d34),L_(0x34f86080), + LL(0xee0e75be,0x5617d582),LL(0xab95d71a,0x3e671da8),LL(0x0190d33a,0x3bee1eff),L_(0xa69eb59f), LL(0xe617dbcc,0x09cde5ef),LL(0x2ec74095,0x888eade7),LL(0xfc85f3d6,0x359daa39),L_(0x55fe2fc2), + LL(0xb342c648,0x4a00b224),LL(0x153ecb1a,0xb894175c),LL(0x58a94e88,0x70cf2206),L_(0x2b740c25), LL(0xf20b5561,0xdca48b9e),LL(0xcecf5379,0x7707c57d),LL(0xa0a1a133,0xc33e6b03),L_(0x4b326561), + LL(0x9c8970a2,0x4c16f18a),LL(0x642f1b1c,0x57b552ca),LL(0xda495e0b,0x1a9ee7da),L_(0x8d81f72e), LL(0x1ad4dac5,0x06f7634a),LL(0xd29378b5,0x1b77c61d),LL(0x110655eb,0xd5bf028c),L_(0x1f8658cf), + LL(0x21c59245,0x51b05ff6),LL(0x6f117207,0xaf13b9eb),LL(0x2b5c0017,0x4aa9bdff),L_(0x0df00c88), LL(0x703d8719,0xaf5e4830),LL(0x40500109,0x52cc9369),LL(0xf59d1925,0x013abb12),L_(0xdc8ebfb1), + LL(0x1082f991,0x2c9f0b9e),LL(0x02ac62a2,0x666e06e5),LL(0x703ab683,0x6f1b6308),L_(0xf1dd2936), LL(0xc8b722dc,0xd103845d),LL(0x6960a1a5,0x2459c28c),LL(0xd0ce016b,0x60884f40),L_(0xd595cffb), + LL(0x7f7ba180,0xe194bda2),LL(0xabd5ba2f,0xbb90a872),LL(0x2977890c,0x3dc3fa11),L_(0xa6cdf4eb), LL(0x1ec1818c,0x2f976a14),LL(0x9509667f,0x6ef670b0),LL(0x547bbc14,0x7a6392b3),L_(0xec34c6c8), + LL(0xfa44c4a3,0xd1652cf8),LL(0xd0c64f4a,0x7b98c98c),LL(0xeeaab800,0xb60ccb53),L_(0x41176e83), LL(0x20bc577a,0x3d3d111a),LL(0xc6b3e65e,0xf1c69a56),LL(0x4586a0f5,0x047748d6),L_(0x966336bb), + LL(0xbfb2046e,0x1ba40992),LL(0x15e78b76,0x492e1976),LL(0x8fa3d075,0xb62a0315),L_(0x34d22c5e), LL(0x08bcfaa9,0x55ee388d),LL(0x47ca2bb1,0xb50f4e76),LL(0x48d16444,0x947b5775),L_(0x6055efef), + LL(0x27f9113a,0x9933e071),LL(0xeeda2fbb,0x16712a59),LL(0xc739c006,0x75d0d886),L_(0x844de074), LL(0x95eed6e3,0x5fd2d047),LL(0x53ef6941,0xfa45dd7f),LL(0x9d54f1fb,0x741b92d0),L_(0xf26829fe), + LL(0xe5114e39,0xc2275c0e),LL(0x14f42683,0xb4d5582b),LL(0x3e016601,0x25741fdf),L_(0xb4b6622d), LL(0x4514d2ec,0xc4ccb66a),LL(0xf7783f3c,0xb57016ca),LL(0x59b84d70,0xf83fe7e5),L_(0x1f712454), + LL(0x7a6974e4,0xcecae61e),LL(0xddefbfbc,0xa93265ce),LL(0x1f1fb9f1,0xf3c9e95d),L_(0x968d6e21), LL(0x76333191,0x5ef96caf),LL(0x343738c5,0x26204de9),LL(0x66ec9969,0x8ee499c3),L_(0xfa475270), + LL(0x1abe8df4,0x0e4c49fd),LL(0x72a09b7d,0x5430fd38),LL(0x5d968105,0xb2efc301),L_(0x82c1807e), LL(0xde3d9217,0x191e5b5e),LL(0xb52c47de,0x32722377),LL(0xa0892090,0xd92f1352),L_(0x8058cafd), + LL(0x757c94f0,0x9649e576),LL(0xf15a6d6f,0x79587916),LL(0x8be7ad8d,0xadd01304),L_(0x01368bc3), LL(0x29445ca1,0x285b7eef),LL(0x761da068,0xae55ea08),LL(0x962f7357,0x983ad1c5),L_(0x9e092171), + LL(0x40d7b216,0x3897f22b),LL(0x2a0a0ea9,0x0a2968a8),LL(0xae9ce773,0x5eeabf53),L_(0x205e8acc), LL(0xd7707d11,0x3f8692b7),LL(0xc470978d,0x416e3f6b),LL(0x0e77bce9,0xf53b376e),L_(0x4d5b8d32), + LL(0x211044a3,0xb9dbcba3),LL(0x174bbf68,0x7e52000c),LL(0x2961ea4a,0x12e1a413),L_(0x1ad46429), LL(0xc87793af,0x36916c16),LL(0x5c151550,0x93f3db96),LL(0xf4f9dbfb,0x2bfda0b4),L_(0x51b088a4), + LL(0xbbbbeb9d,0xcc1030ad),LL(0x12a28f87,0x49bedb1c),LL(0x1869975f,0x6a004fbc),L_(0xfb3e5023), LL(0x51c16438,0x71e53119),LL(0x49270eb0,0x5e139c47),LL(0x60f866a3,0x0784d6e0),L_(0xf4f1e774), + LL(0x97f340e8,0xa61d043f),LL(0x05c008f3,0x4c35a67a),LL(0x8581b05b,0x44182222),L_(0x3e8c133f), LL(0x45bca0e5,0x8bf96ead),LL(0xf85bee99,0x7c86a090),LL(0x2ad3d962,0x7ea985d4),L_(0x8ce03be2), + LL(0x1ef28f3f,0xcbf512f8),LL(0xcf504c88,0xdc8ed43e),LL(0x516f1183,0xec32dc2b),L_(0xa2e2c1fe), LL(0xc45e5adb,0x505156da),LL(0xb7216064,0x983acb8c),LL(0x30958d17,0xe6ec6def),L_(0x28469ede), + LL(0x1072cfe0,0xaaf2a64f),LL(0x30962a70,0x33454609),LL(0x8b880e7d,0xa7a46a05),L_(0xa96bc722), LL(0x7354fe08,0xd5821805),LL(0xdd17ff77,0x0fc1cbb0),LL(0x0e58985d,0x9f7a9dad),L_(0xca339489), + LL(0x9dc30815,0xe1ebef86),LL(0x51cfd1d3,0x852da87b),LL(0xed5cdd01,0x21bc828d),L_(0x7b53ef92), LL(0x5aadc85a,0xe522baaa),LL(0xb7031c6f,0x608b4b16),LL(0xee466b6e,0x0c33a24a),L_(0x3197bd29), + LL(0x4e1fae13,0xf397b191),LL(0x68fba8e4,0x2940f1c8),LL(0x47dd818f,0x68c77c89),L_(0x7dbc2c98), LL(0xba40969f,0x8ffaa799),LL(0xf0b99031,0x712df944),LL(0x997feef1,0x4da4261d),L_(0x1f70d141), + LL(0x1b0afa52,0x69fea568),LL(0x3e151e92,0xdfc334a6),LL(0xa2587e32,0xf61b674f),L_(0xbaa4f3f4), LL(0x62eb8d24,0xdd643088),LL(0xa7a5fc88,0x5cde0035),LL(0x53b8bdc0,0x1c5ce488),L_(0xf9fdf735), + LL(0xa64a20a6,0xd22b7f88),LL(0xa2eb0438,0x0628283c),LL(0x75129c40,0x6efeb06e),L_(0x4f3e44a6), LL(0x76f8c149,0x181e8b58),LL(0xbb830a80,0xc758648a),LL(0x3d1fbafb,0x069f31f1),L_(0xbfff2e03), + LL(0x78f3872d,0xbba94fb7),LL(0x748d4921,0x852ef996),LL(0x48ddef24,0x066ad60c),L_(0xd4ea4021), LL(0xc1ba7360,0x790d395b),LL(0x6901be6e,0x7616ea80),LL(0x61b408b8,0x9278e0eb),L_(0xc7366cae), + LL(0x15c9a850,0xae384504),LL(0xd50ba3bf,0xf4814e2a),LL(0x1b8b0e26,0x2b97a1b4),L_(0x3f0e4245), LL(0x260ef3d1,0x5db22d7e),LL(0xbfd1a5ec,0x2e7c6bf8),LL(0x7c1471d1,0x7bdec3b1),L_(0x079a90a5), + LL(0xb71fd22f,0x7f17f289),LL(0x5e288b65,0xd4564a47),LL(0x34b284d4,0xb75a3120),L_(0x5d8426b1), LL(0x986c8a29,0x197cc0c1),LL(0x29e64b23,0xcc3ae592),LL(0x7da4599c,0x54ebd08e),L_(0xf69dda7d), + LL(0x9addab69,0xbd0c6b5c),LL(0xf7f44578,0x7ec99645),LL(0xaa918115,0x8ee2e6eb),L_(0xd2de3778), LL(0xd9db5cd4,0x2d90dfec),LL(0x5c1253c8,0xf48ec5bb),LL(0x7732d381,0x2daa3783),L_(0x158b18c3), + LL(0x4087fb32,0x64d12677),LL(0xa61b3dc0,0xd6d5d34f),LL(0x5aa57c66,0xeebb82c5),L_(0x01983506), LL(0x0b3a8490,0x4bcc334e),LL(0x347044a3,0xafc73b45),LL(0xbdc15e76,0xba3353c1),L_(0xfc066074), + LL(0x3f49b08a,0xe4f392d0),LL(0x1fda13b1,0xc0bd57d7),LL(0x4a84d25e,0x8b9c3a47),L_(0x9991791e), LL(0x88262b52,0x5d6233aa),LL(0xd1327af9,0x1da165fe),LL(0xba39a0b7,0x56569118),L_(0x22bb8aef), + LL(0x55aff6c4,0xffb26779),LL(0x4e7d53f7,0x581f3f37),LL(0xc259ed17,0x0527ce42),L_(0xb4d8bc18), LL(0xeb967186,0x592962f6),LL(0xf82705f0,0xbc866b81),LL(0x9a5ee9a2,0xb8fcf82b),L_(0x60d27e12), + LL(0x7b7d67a2,0x6bed8eca),LL(0x5d9fc1ee,0x74e18695),LL(0x3ac5ac85,0x4777aaae),L_(0xaf2958be), LL(0xaf83cba9,0x114e541f),LL(0x41b4aa09,0x49c90e0c),LL(0x6ffe97ac,0xd22caea2),L_(0x032a68c3), + LL(0xad0a7c5d,0xcf68be2d),LL(0xe675d930,0x46042df0),LL(0x44cc34c3,0x83baaad0),L_(0x09136a40), LL(0xe5be5f7c,0x232306d2),LL(0xe3f78e95,0x30e3711a),LL(0x7b102ef8,0x90ca6b87),L_(0x25d20cac), + LL(0x24f85c29,0x1c82027d),LL(0x509f2a65,0xc82c4968),LL(0xf294dde1,0xa0dabc8e),L_(0x6fbe2f96), LL(0xeadad60a,0xa2d74908),LL(0x0b27809e,0xea64f015),LL(0x3b0536e3,0x948d4413),L_(0x3ccb3666), + LL(0x4cd8e164,0x7e58fb10),LL(0x2cc3a907,0x3a2788f9),LL(0xc860f6b9,0x673a28d4),L_(0x6f03054c), LL(0xc5e46854,0x62c5a4e9),LL(0x02ecf383,0x059005a0),LL(0x5a617a3d,0x06267ff2),L_(0x6a1a7e0c), + LL(0x2af744e3,0xae8162d5),LL(0xfa2efdc7,0xbe28dd79),LL(0x950dd089,0xcd76243e),L_(0x1c19727b), LL(0x887a306f,0xc5cf5f6e),LL(0x003d3ccd,0x6076457e),LL(0xd58cb75e,0xa5a9063c),L_(0x95f7ccee), + LL(0x6f63cd76,0xa7571852),LL(0x089066c8,0x2ab669d0),LL(0xd42dd087,0xee6f2f6d),L_(0x18dd0fe9), LL(0x0a7d1e21,0xfa5d28f5),LL(0x2b521ac0,0x9441fb8f),LL(0x8466e082,0x4a673507),L_(0xf688bc35), + LL(0x980ae611,0xc6222c7e),LL(0xb1d88568,0x3030f358),LL(0x9b8f42e0,0xbbf50262),L_(0xaa3f5497), LL(0x469ee64f,0x3ef5e0ea),LL(0x167a26ce,0x0e1c1519),LL(0xb57bce91,0x806bb655),L_(0x2f545810), +}, +/* digit=28 base_pwr=2^196 */ +{ + LL(0x4760716f,0x789ae13c),LL(0x1cb72857,0xf578d1e5),LL(0xc6470ce9,0xd42e2969),L_(0xe716d39d), LL(0x5f1cf231,0x733456c6),LL(0x5fef0d2e,0x592bb199),LL(0x91e21ef9,0x8814ad36),L_(0xb248995f), + LL(0xbefb4b7c,0x6eb484b6),LL(0x87780471,0xa01ef7f5),LL(0xdc02e33e,0xf8ac22f1),L_(0x6e5fad2e), LL(0x0cbad015,0xc2d60cca),LL(0x2db33229,0x78a4abdf),LL(0x375f545c,0xbb2518b8),L_(0xe117a7f0), + LL(0xe94cf6d0,0x721159cc),LL(0x2528b498,0xd001dacc),LL(0xba328884,0x6ca68824),L_(0x16db431f), LL(0x857c60d1,0x3e1c1a16),LL(0x06be4963,0x4e3a8a4a),LL(0xe5100cc6,0x2fadd498),L_(0x66deb87b), + LL(0x96e332c2,0xba35b5c5),LL(0x5c8ec408,0x32a3227d),LL(0x3d572604,0x3287caa4),L_(0xde9663da), LL(0xd3d8aa49,0xd8f05722),LL(0x3d0b0ee6,0x211c6e5e),LL(0x789832b8,0xd7095c24),L_(0x057c1df6), + LL(0x8e5e0a09,0xa554b8a7),LL(0xc0ca83b1,0x3cfefa31),LL(0xde9b7d08,0xb712685d),L_(0x890d7a66), LL(0xd942a6e4,0xc242dae3),LL(0x4f659bb9,0x5c2986c0),LL(0xca0a3ad3,0x58984f5a),L_(0xc0f8dce7), + LL(0xbb8f3d67,0x75e4f894),LL(0x305ab1c6,0x597d61ec),LL(0xc5900752,0xbb05ff05),L_(0x6f0cfc37), LL(0x78265563,0x8bb8c320),LL(0x928753a6,0x2afed364),LL(0x9604812d,0x2ecd08d1),L_(0x9f7250a9), + LL(0x4f8511c5,0xf62b6ad5),LL(0x68ca6d7c,0xe0ab6bd5),LL(0x9849b3b9,0x437c1eb4),L_(0xdab12cd3), LL(0x60c66157,0xe2da9ad4),LL(0x81e8b4b4,0xc72b155a),LL(0xe933cc08,0xad7576ea),L_(0x66098ae9), + LL(0x019aa18f,0x06661e71),LL(0x997b0541,0x2a662ce1),LL(0xc9069497,0x5ff8cdb7),L_(0x7c209e40), LL(0x22ef5987,0x62a400b7),LL(0xdcbb6b6a,0xb2915c26),LL(0x1fe38da6,0xffba34b0),L_(0x6f5cedf1), + LL(0x6335b064,0x6133838a),LL(0xe7f5ee57,0x76c52676),LL(0xce767a1c,0x5e465185),L_(0x1464e362), LL(0x93f4d930,0xce0d54ed),LL(0x5d714210,0x742e50db),LL(0xabc94edc,0x78bba80f),L_(0xc86bf6f4), + LL(0x4b83b6ce,0x81b49586),LL(0xa3c55ff8,0x1fa505f0),LL(0xae4acd7d,0x832817ef),L_(0x77814820), LL(0xd6ef9110,0xd59182a2),LL(0xe98e0fd5,0x764ce5bc),LL(0x323e203f,0x63236d14),L_(0x3a8b8995), + LL(0x118cb51a,0x8ea6b928),LL(0x117ab1a0,0x56dd68b6),LL(0x395935d8,0xbd116bcd),L_(0x0a981729), LL(0x5ced4a6c,0xbb4ec3a0),LL(0x569614d1,0x63f4b4af),LL(0x1f24cb4f,0xf9b9d0b1),L_(0x03e53738), + LL(0x18a50233,0x25f531db),LL(0x0389f67e,0x8b3f0f16),LL(0xa0701098,0x6a6aef50),L_(0xa69dbc19), LL(0xfb3a537e,0x8c64961c),LL(0x35b24bda,0x83f4d9b6),LL(0x440a2f19,0x7696fcba),L_(0x9f01b04c), + LL(0xc0f2402c,0x5360ffc2),LL(0x1debf1ae,0x8d33aed5),LL(0x7da3a904,0x4dde5b44),L_(0x0cb2c453), LL(0x6c75adbc,0x0c9938a7),LL(0x11dd5af7,0xf4ef9f5d),LL(0x30114b89,0xbb08ab61),L_(0xc5ca168c), + LL(0xdf042ef8,0x481cae35),LL(0xaab6e2e0,0x6fed557f),LL(0x3de7d914,0xa4e1bcfb),L_(0x7fdaa079), LL(0x85cd2d00,0x4370738c),LL(0xd12da321,0xf95a95e5),LL(0x9814d14c,0xe0fd8dcb),L_(0x32743222), + LL(0xc27ed928,0x655d6ffb),LL(0xde08c413,0x43a4fc18),LL(0xde7270a7,0xcba9f29e),L_(0x06e6cdba), LL(0x2e139df9,0x5cc92d84),LL(0x65a9b95c,0xc5a596a0),LL(0x6a5bdae7,0xd5e4697c),L_(0x5db94611), + LL(0xa1aad672,0xf4cdb11a),LL(0x0a56c5fd,0x35eb1979),LL(0x06b67d2c,0xd3a50661),L_(0x227904a1), LL(0x47a4312a,0x2c80dfe0),LL(0xdec3bab1,0x1e137540),LL(0x2528a436,0x4a30bc3a),L_(0x306505c1), + LL(0xa27429d2,0xac029aa3),LL(0x3ea574fb,0xe555d79c),LL(0xc26117ae,0xeee94c98),L_(0x485edd44), LL(0xb5866e81,0x3b83d3dc),LL(0x6558e1d3,0x497236f4),LL(0xa896a908,0xc1eba30c),L_(0x2a815dd8), + LL(0x0b1d2a89,0x6a45073a),LL(0x2ef3d416,0x72b5edce),LL(0xc7a3b1fe,0xac5268b4),L_(0x5c03d364), LL(0x8589b53a,0x25c0f37d),LL(0x247dcc63,0x8cf1d802),LL(0x9d3fe1d2,0x15093e7c),L_(0xfe72abd8), + LL(0xac83f07d,0x6eba86a7),LL(0xdaf79a6c,0x7769b4ef),LL(0xce08325a,0x306da9b2),L_(0x72e3aed8), LL(0x9fd7086a,0xbe70ef68),LL(0x8bdda042,0x9649ac38),LL(0x59634c20,0x260c709f),L_(0x169d616d), + LL(0xe36b9e36,0x56c9f551),LL(0x8ce0bbd5,0x9a19b344),LL(0x7b1c335b,0xb90484f8),L_(0xcd0b498b), LL(0x0b5ef9ed,0x8c6e220c),LL(0xdabb30f9,0x4ddb2065),LL(0xe5c29249,0x2b898fcc),L_(0xd16ae2eb), + LL(0xb7d4dedd,0x3cc11efc),LL(0x87733216,0xc5d55aae),LL(0xebad4304,0x139832e2),L_(0x6f2f9588), LL(0x0089eb00,0xb6ab35e5),LL(0x143ac48b,0x35af18a2),LL(0x363f80f9,0x8c8c5655),L_(0x8c090d66), + LL(0x3cad20e3,0xd54cc179),LL(0x56fcbabd,0xc7630400),LL(0x8ece816d,0x4347fa49),L_(0x7ff18955), LL(0xe22f3ed7,0x83d72b28),LL(0xa523c18b,0xa89efb1e),LL(0xc07fe6b1,0xf679cfd6),L_(0xb95c3988), + LL(0xdd4159b7,0x90409ff6),LL(0xe0447eff,0x0509d402),LL(0x0d946df5,0x46153d5b),L_(0xe1645594), LL(0x1229cd0f,0x25841f04),LL(0xd79eba33,0x163487eb),LL(0x1f5ac4bb,0x7a894b16),L_(0xd4b696a9), + LL(0xd0fd0fdf,0x95e0fb87),LL(0x316c5da1,0x12d4a0db),LL(0xbac4ce3b,0x4aecb41e),L_(0x07cf2ae4), LL(0xf6b866b3,0xc85cee8f),LL(0x4241c703,0x929c773e),LL(0x22cf7705,0xf90855df),L_(0x6b6a6a3d), + LL(0xb938eef0,0x054fb615),LL(0x8dd3e916,0x7d1ce422),LL(0x624c46af,0xc8279b91),L_(0xfa11e3a9), LL(0x6fc664fc,0x957fa9ea),LL(0x7faaee34,0x86d71f57),LL(0x8216ab5f,0x4fdcf503),L_(0x8f169cac), + LL(0x99acbf60,0xea1c792e),LL(0x995de7af,0x3d4924d1),LL(0x09c0ca0b,0xb94190da),L_(0x95e76a43), LL(0xaccafdf9,0x6c1aa007),LL(0x1f952403,0xe4eccac7),LL(0x07821f33,0x3125c30a),L_(0x56a0a57c), + LL(0x76375e86,0xa2da6002),LL(0x0c8b88d7,0x851663d2),LL(0x903300d5,0xe4304412),L_(0x6111b04d), LL(0xf13f93d7,0xac6cc7e1),LL(0x8ce4d931,0xc233fc51),LL(0xfa884feb,0x889b7699),L_(0xe977a15e), + LL(0x649b4878,0xb98eb821),LL(0xf89766d9,0xda85057c),LL(0xe8c391ca,0xc53a37ec),L_(0x15f269aa), LL(0x3c241d53,0x1c63bf16),LL(0x40246227,0xe6f1e93f),LL(0x882db63a,0xacf7e8f4),L_(0xa54ec9c8), + LL(0xd18b015d,0xf734d021),LL(0x943cb99b,0x62b04fde),LL(0xea71f882,0xe9f4d402),L_(0x4f517e6c), LL(0x3d8f7745,0xad9abc71),LL(0x1fe90081,0x088ad020),LL(0x25dfde15,0x250f4123),L_(0x48badfac), + LL(0xe0e96ead,0x46b7097a),LL(0x2937ebe1,0x19ddbba5),LL(0x3ecd0bd0,0x1b6e3a84),L_(0xaf895097), LL(0xfc22adbf,0xbae23185),LL(0x220a75e9,0x09ed4ed6),LL(0x062eb7e7,0xe1ce81e7),L_(0x8b1103c2), + LL(0x4649146f,0x91702d2f),LL(0xb9f95742,0xbfdaf3ca),LL(0xcf8f23f1,0xd403a4bb),L_(0xf9164036), LL(0x33bbb505,0x5d48ee37),LL(0x2b84d0f7,0x79ec8716),LL(0x5a9ca7a8,0x76092de3),L_(0xec444f35), + LL(0x1f9e6c98,0x17ed7898),LL(0x25820dda,0x668c28ad),LL(0x745aa2ed,0xcf9ce234),L_(0x01a5ca97), LL(0x4a783729,0xbf5955af),LL(0x31ed52ae,0x92891a4a),LL(0xea43a792,0xc67457a9),L_(0x7c0fc898), + LL(0xb8dd47fd,0xff42200e),LL(0x253f7438,0xa70a5da7),LL(0x6cbd4092,0x6d058c66),L_(0xbb3089d5), LL(0x110c3a9a,0x67048ffd),LL(0xa8a2fe2a,0x82fc6b0e),LL(0x30242c32,0xa769318b),L_(0xf0614bb4), + LL(0xbb708cee,0x7620c902),LL(0x084e78c8,0x75832ff6),LL(0x3a89190f,0x4632df7f),L_(0xdade5c0d), LL(0x48e4bd14,0x62dc813c),LL(0x0d15d331,0x9c55f000),LL(0x36a4457f,0xa330078b),L_(0x3b722d3e), + LL(0xc1f3d1aa,0xe37d1ff2),LL(0x5c4d9799,0xed06b0a5),LL(0xb1d3c33d,0x939c8f4c),L_(0x499b0fdd), LL(0xf53390da,0x9e6ebae1),LL(0xb952b38d,0x28276f38),LL(0xd69e1327,0x19f5bac2),L_(0x7c921bee), + LL(0xafd90d24,0x0e84e363),LL(0x37d2a8e9,0xabb80eda),LL(0x07e40274,0x95656580),L_(0x877a2cb3), LL(0xa8a767b1,0xc90af857),LL(0x61a42dc7,0x465bb86d),LL(0xbf658fdd,0x7eb455a7),L_(0xf467f4bf), + LL(0xda192001,0xad70ec43),LL(0x1aa502d0,0xcb55de32),LL(0x7d66c4af,0xa952a38d),L_(0xb6579e71), LL(0xdceb099f,0xed3e0675),LL(0xaf274aa3,0x635a8629),LL(0xb9f4c7da,0x8d6f1c2a),L_(0xf954b1aa), + LL(0x6f166fdd,0x0c46e6e4),LL(0x2f6ebcd8,0x0813e798),LL(0x9066cff4,0x2f759557),L_(0x026ee79d), LL(0xbd44613e,0x06317041),LL(0xc1609f3b,0x6879039a),LL(0x6853d2d8,0x80566bad),L_(0x9a1de27b), + LL(0xd83655d4,0x7c1a0bb8),LL(0x40e36289,0xa5228881),LL(0x00c5e925,0xe9c51c0b),L_(0x17fd5193), LL(0x0234b859,0x9a00e742),LL(0xee8b4f08,0xf774358d),LL(0x0eb69043,0xd46eb256),L_(0xf54e46c1), + LL(0xe386daac,0x9a305482),LL(0xc37330b6,0x7957d9d7),LL(0xf3f52055,0xd1a9ce7b),L_(0x1e870d5e), LL(0x303b5ae9,0xac2ad67a),LL(0x4d3fc2d7,0xb7b58e09),LL(0x7dc5f5f1,0x6c99d250),L_(0x2d7b151a), + LL(0x29507cf5,0x37c15d8e),LL(0x6919a75b,0x738d9ba6),LL(0xeb76b42c,0x0b91ab81),L_(0x3da557ec), LL(0xac1dd9b8,0xb974e300),LL(0x6b8577ca,0xd55b5831),LL(0x9748b154,0xbfcdc886),L_(0xf2e99cba), + LL(0xaa791b19,0xb611f73f),LL(0xe44e650a,0x6fd43d10),LL(0xde586616,0xff2a1da0),L_(0x8983db6a), LL(0x7da57055,0x5051af3a),LL(0x73e374c8,0xa7d98205),LL(0x80c04759,0xd4db7bec),L_(0x28c48bc7), + LL(0x9051c153,0xd36418fc),LL(0xed854f7d,0x2adcc4eb),LL(0xa2d25b82,0xf9315c6a),L_(0x510b53b5), LL(0x6fec476b,0xbdd2e105),LL(0xbc40ed1e,0xdc2bc2a5),LL(0xd473b242,0x9d87dbe8),L_(0xf9d3eb65), + LL(0x4031a20e,0x5001f11c),LL(0x3e83e92e,0x86e3fa8d),LL(0xa7c80719,0x36408819),L_(0x520ab89f), LL(0xaf0e8126,0x430d59f8),LL(0x1479126e,0x5b426c42),LL(0x1c62cb71,0x1a2c3f95),L_(0xba8c53ff), + LL(0x42998c5c,0x9b1ab1ad),LL(0xc5bef946,0xe7ad335b),LL(0xd2f18816,0x9b21d6a9),L_(0x7bf5543b), LL(0x09dc1281,0xc3771787),LL(0x8d9a2927,0x69b13c80),LL(0x5419b128,0x472d8eda),L_(0xce12aaf7), + LL(0xc26aa121,0xb9665455),LL(0x1990f311,0x3bac4cd2),LL(0x4b96b490,0x15632e30),L_(0xb87e9336), LL(0x03c2ca60,0xeac8678b),LL(0x101c2863,0x5772568c),LL(0xbef62c1e,0x3b53ce81),L_(0x7d135e14), + LL(0xbe66a12b,0x753cb0aa),LL(0xd1abe43d,0x38781cdd),LL(0xcf1c41a4,0xab22df0a),L_(0x11b70405), LL(0x7e4c97e5,0x3869cfcb),LL(0x24f71698,0xb43a4e85),LL(0x6ab97c59,0x35c0f24a),L_(0x4df75964), + LL(0x292f0d69,0x3e2b7aaa),LL(0xdafad7ad,0xe2ba5f7a),LL(0x77ff29a4,0xe21d90f9),L_(0x67fec4ee), LL(0xbc0b01d0,0xe3677335),LL(0x26e5157e,0x8fee4b54),LL(0x2b1a301a,0x5d1511ed),L_(0x263f34a8), + LL(0xe5e9578a,0xd73285ea),LL(0xdf28a719,0x2e506fa1),LL(0xecb1a91e,0x79d54bca),L_(0x5a06ac79), LL(0x7bf48ede,0x9f75d7b8),LL(0xa6637969,0x08014deb),LL(0xaf188a86,0xbfada230),L_(0x423dacf9), + LL(0xa63e4dbd,0xe55bd873),LL(0xa80a0320,0xbe01f07e),LL(0x1b09710f,0x6e6cb66a),L_(0x2fee3ce5), LL(0x955bdbc6,0xcecf1493),LL(0x2a3070a3,0x861a0878),LL(0x084d6464,0x0d0e35ed),L_(0xa03afc73), + LL(0xcdca5ed5,0xdb335ec9),LL(0x78cc343c,0xa8df4be9),LL(0xcaea8450,0xf564ec58),L_(0xa5438f08), LL(0xffeefeda,0x193c8457),LL(0x0eaf627b,0xddabba77),LL(0xc99c9580,0xa8ec469f),L_(0xc9ea9e43), + LL(0xf3c9cc69,0xa888f556),LL(0x8951408c,0xaaa99a76),LL(0x79ee145a,0x4760cde2),L_(0xfbdcc6a6), LL(0xd543c51c,0x4d61119d),LL(0x6d5cde8a,0xb57c70b1),LL(0x6d605846,0x8392010f),L_(0x824097f5), + LL(0x79dcc935,0xd988c0b3),LL(0x97a596be,0x66a3883f),LL(0x4cdffd50,0x026d6f1b),L_(0xb093bfc3), LL(0xd01d8fb0,0xc2150428),LL(0x477892be,0x453f0c91),LL(0xff7a816c,0x84b14e80),L_(0xf02eff58), + LL(0xa7823319,0xab4b5d45),LL(0x76a3294f,0x1d9dae80),LL(0xce1d1836,0x1f8f6e55),L_(0x7064e78f), LL(0x8d6bd5d7,0xd272957e),LL(0x20068c6a,0xc59f4815),LL(0x15f1375a,0x1dcaf5b0),L_(0xc01d795e), + LL(0x6c316cb4,0x5f19c736),LL(0x0082a8d7,0x9bb6aded),LL(0xadea31b9,0x1090e65a),L_(0x27c39f72), LL(0xfec08edb,0xad397892),LL(0x2ae66c84,0xb2c262c0),LL(0xe12a0551,0xcd11a65d),L_(0xc52c046e), + LL(0x5db707b6,0xf10d38e7),LL(0x4a3e6c1d,0x5bbaa9f8),LL(0x1692548b,0xa34228d1),L_(0x2ff089a1), LL(0x654e5d42,0xf2342e4d),LL(0xe8e23440,0x53e19ef5),LL(0x254b33f9,0x2609459a),L_(0xd598bc1e), + LL(0x7007f546,0xefba2361),LL(0xe1a42d85,0x72c3ffc2),LL(0x600f4b0f,0x68bfc655),L_(0xf5be3dea), LL(0x13f14da8,0x6efd514e),LL(0xaaab23fd,0xcc761583),LL(0x949da3c4,0xd3524443),L_(0xf07ce5ed), + LL(0xeca99656,0x98853678),LL(0xa45dd69d,0x959586ec),LL(0x2b4d5496,0x882a160e),L_(0x54e92805), LL(0xdf2a1b89,0x52e13aac),LL(0x8f1d99c7,0x4a5dde80),LL(0xc432a9d8,0xcafd8d25),L_(0x801f070d), + LL(0x39455fca,0x0c20053d),LL(0xdaf5e5b7,0x04a4c141),LL(0x14b7b1d1,0x38e2582a),L_(0x51bd8c01), LL(0x20e68184,0xd15dab61),LL(0x1925d2d5,0xba541287),LL(0x62766fdb,0x6619e0a4),L_(0x439e1dd0), + LL(0x3bc818e3,0x746af8f1),LL(0x6be8f43b,0xbbe6fd69),LL(0x53589554,0x446a9ab9),L_(0x0ea769ab), LL(0xc30fa666,0xf4d47c43),LL(0xca3dad1b,0x431ea6ad),LL(0x9dafb227,0x4ff395ed),L_(0xbd3937cc), + LL(0x2c57d4e1,0xa4cceb3e),LL(0xd6667dff,0xff6cbcb5),LL(0x8918c037,0x268bc00c),L_(0x1c4ef107), LL(0xa92f2fb1,0x76415814),LL(0xd31c6973,0x68597539),LL(0x0a6cac9a,0x399773b8),L_(0x87d12cd2), + LL(0x7652379d,0xc59e872e),LL(0x0e493371,0x1fa3657f),LL(0xa77b28fb,0xa7619402),L_(0xeea23b52), LL(0xa85f48bf,0xe180ae18),LL(0x8d4b8437,0x68f2c962),LL(0x3145c6a2,0xe4d30712),L_(0xd9ccc49f), + LL(0xb3ff7311,0x00082b09),LL(0x23fb7431,0x0de3deec),LL(0x33ecf9d4,0xab6cc532),L_(0x8eae9253), LL(0xd8a0ae1c,0x2a618d55),LL(0x97045c63,0x3993ca70),LL(0x0f2549ab,0xcc02ada7),L_(0x14884bad), + LL(0xb5a7fc93,0xec8a7e38),LL(0xfbda890b,0x786e2a85),LL(0x882cb9d7,0x103bbacc),L_(0x17321d75), LL(0x7395bf16,0x4be2ec80),LL(0x20a02de8,0xb287766f),LL(0x4eb9dbc0,0x54c7f36b),L_(0x399d4a59), +}, +/* digit=29 base_pwr=2^203 */ +{ + LL(0x663a8bf6,0xf1bd6f1b),LL(0x6c6b1cae,0x2de2cf35),LL(0x4f49967b,0x49fde755),L_(0xee2050c1), LL(0x9956f1ed,0x69f14061),LL(0x138a98db,0xc5c29863),LL(0x516db87b,0xd4456e50),L_(0xe15ee455), + LL(0x78b8a6b9,0x5019a365),LL(0x0782c606,0x025726c3),LL(0xd611e283,0x68f7b67f),L_(0x9292eb6b), LL(0xb4e7c5d0,0x5ed6c698),LL(0x8f074f3c,0x6ea9c71d),LL(0x2c0dd03d,0x6ad6ab61),L_(0x6c985359), + LL(0xc1c1b8cd,0xa1ea038f),LL(0xd886b019,0x9d8f1f3a),LL(0x5e44aefb,0xbc0d302a),L_(0x8b9ba536), LL(0x47ec400d,0x9049574e),LL(0x02b6e2ef,0x9778ebf4),LL(0xe0adc015,0xe0348d22),L_(0x2f89c060), + LL(0x2b432e4f,0x97be15f2),LL(0x9442eea7,0xfb123347),LL(0x752805f0,0x095f8861),L_(0xbd4f1599), LL(0xe4fd179d,0xbfe10a89),LL(0x8eb30cbd,0xc1a95173),LL(0xb5e94f0f,0x38bffbb7),L_(0xbd72f70b), + LL(0x31d5bf11,0x3b628b52),LL(0x6efe2871,0x9f017556),LL(0x50b6b23f,0x2051232c),L_(0x05744158), LL(0x18b1fb8c,0x58660126),LL(0x6683fda3,0x85343658),LL(0xcdefb5cb,0xee7c02af),L_(0x0c98f701), + LL(0x842b943a,0x79b770df),LL(0x52f1e8f1,0x2ca8d23c),LL(0x29cf116b,0xd89fd3b6),L_(0xae88c948), LL(0x20362fba,0x17dff9f9),LL(0x7048ea42,0xb8350a9e),LL(0xcbd44b83,0x3adbe1f1),L_(0x5fc81426), + LL(0x171138f5,0xd002fd82),LL(0xb9b3d4ff,0xed715f0c),LL(0x14d543b9,0x770fe818),L_(0xc2b33795), LL(0x2c123604,0x3f8f258d),LL(0x22543d13,0xa3564d33),LL(0x9d216804,0x435f77e5),L_(0x97bba91d), + LL(0xabe6aead,0x5e84d762),LL(0x84f02bef,0x538499e7),LL(0x1d74cb44,0xbfb3e36a),L_(0x591a2397), LL(0x198dddb8,0xfcbe1984),LL(0x430a25b9,0xb7b8bbcc),LL(0x80cf7321,0x00ac08de),L_(0x44305966), + LL(0xe8745380,0xbbb4692e),LL(0x9a66429c,0x02f17d1f),LL(0xe7057c36,0xec132fea),L_(0x87b7dbea), LL(0x89b168f8,0x7f9c9480),LL(0xe2d21c13,0xf9e94167),LL(0xf34e2409,0x13e022d0),L_(0xba4bcae2), + LL(0x243d2e97,0x358d9af4),LL(0x518d83d9,0xb47467ae),LL(0x81a7eea1,0x470b2dab),L_(0x37ab475a), LL(0x3d8103c5,0x9a2755c4),LL(0x200b1434,0x9228b07c),LL(0x5b3e4741,0xbc9b37bd),L_(0x8f9c88f6), + LL(0x8739f51d,0x3e9880c9),LL(0x183b3d2a,0xe30595ad),LL(0x9d3ff1f0,0x4b286b55),L_(0x2ffa355d), LL(0x89be0142,0xe2576d49),LL(0xf43534f3,0x00a64de7),LL(0x1e63717f,0x0c898bea),L_(0x338d6967), + LL(0xd89a7c52,0x489bd20b),LL(0x1cbe5fd9,0xe7ac0a68),LL(0xf9bbb011,0xbe7ead5c),L_(0xfdfdaa41), LL(0x5bf6715e,0x2573c5cb),LL(0x71fec319,0x30483451),LL(0x7ccb3fa6,0x0071ca44),L_(0x02bc8a83), + LL(0x83211264,0xd01481a8),LL(0xd292cd9c,0x48c43d0c),LL(0xd5277f78,0x32c5572b),L_(0xb3f946cd), LL(0xb34b30d2,0x780e2016),LL(0x492c55b7,0x43e3806b),LL(0xd4ad90ce,0x97399684),L_(0x2bddb7ec), + LL(0xe410e719,0xb76d2e77),LL(0xaa75746e,0x0df8f582),LL(0x6f7bfcdc,0x210d7777),L_(0xce4ca9a2), LL(0x04ef15f2,0x87836a34),LL(0x77b0ffb3,0x44c267d0),LL(0x2fbcd218,0xbb549183),L_(0x8bf3d46f), + LL(0xa541189d,0x3029c3b6),LL(0xbeb5a06f,0x277c1d19),LL(0x761b9d9e,0xd09fd429),L_(0x189262a3), LL(0x8fe5cbe0,0xf3cbac6c),LL(0x4d675bde,0xcbbe3e5c),LL(0x4dd89b19,0x75e821e8),L_(0xf4cfe02e), + LL(0xf92173f6,0x3b6ebbae),LL(0x66df420a,0x44ad6eda),LL(0x49918d50,0x71e12775),L_(0x6343f5ce), LL(0x7c6c9ea4,0xa76fc6d0),LL(0xde640ea7,0x95bb0f9e),LL(0x0e5cdfb0,0x3505e20d),L_(0x03859db8), + LL(0xc0a26f34,0xf36a466b),LL(0x8087482a,0x769bd94b),LL(0x96b0b77f,0xbaa9cde2),L_(0xe98e1c0a), LL(0x961d12a3,0xe383e51b),LL(0x9d59941b,0xb680bdbd),LL(0xfbf75271,0x1f5805cf),L_(0x13cbe6aa), + LL(0x205ad8c7,0x84ebf013),LL(0xc207bf8a,0xf24e1950),LL(0x749aace4,0xaaf03f06),L_(0x566ca993), LL(0x33c5f5b5,0x378cf813),LL(0xe91d36bf,0x8ac53e4b),LL(0x9b1441f3,0x0113566b),L_(0xb69efcbb), + LL(0x8145a741,0xd10f5321),LL(0x81bb9251,0xe73eb2a7),LL(0x4da9ccbf,0xf35011e6),L_(0xd0a422fb), LL(0x375da80b,0x3b493784),LL(0xac876682,0xe99fa1c1),LL(0xbfcdd019,0xfcd0fffb),L_(0x16d39a14), + LL(0x44724691,0x6169a512),LL(0xf345fdba,0xb54ae1f9),LL(0x97426a68,0x84a843c6),L_(0xc87c52d5), LL(0x57ee464a,0x9fa3b852),LL(0x36872038,0xcf4188dc),LL(0xb9fbee93,0xc6f126b9),L_(0xebc64e1f), + LL(0xa34571ca,0x48662048),LL(0x54bb6dce,0xfa26591d),LL(0xf6f0b8d2,0xc7f51cd9),L_(0x5cfd4a24), LL(0x5bc438cf,0x66200c18),LL(0xc248c469,0xe6f1c95c),LL(0x6844e3a4,0x6e17cd42),L_(0xa72dc6d8), + LL(0xddd1ff7e,0xf244b120),LL(0x431f42e4,0xe6046902),LL(0x40c26687,0x1691fe67),L_(0x30ca7e9b), LL(0x6e105d9d,0x9b50eded),LL(0x961020de,0x00b6a88e),LL(0xe9b08b55,0x4b82631a),L_(0x6cc54fd6), + LL(0x04530c1f,0xf4435606),LL(0xf234709d,0x9b79c17a),LL(0x9f045501,0x839380ae),L_(0x5f60c0b0), LL(0x2d608e32,0x9b83bfd2),LL(0xadf75cc7,0x10db4135),LL(0x22cc1949,0x3a2e7962),L_(0xe3436f2d), + LL(0xec19a27c,0xe5ad9b86),LL(0xb18ea5cf,0xc7d0db50),LL(0xb0a1794d,0x12340a6f),L_(0x2c1d9f6e), LL(0x8bcb0708,0x54438da3),LL(0x032032f2,0x33597066),LL(0x68e9b910,0x192ee4ac),L_(0xdf1723d3), + LL(0x6b3b3e12,0x6076299f),LL(0x49f13a7d,0x210c46f7),LL(0x11bcefd8,0x4c36d81d),L_(0x9b39bf08), LL(0xd49ab219,0xb72e4c99),LL(0x8e53b636,0x0e9348ea),LL(0xa88c581f,0xd67c4ee9),L_(0xa41298b4), + LL(0xe899e96d,0xb64bf9de),LL(0x8d198f03,0x43b5e3ae),LL(0xa7cc5c88,0x656b15b3),L_(0xd2995e8b), LL(0xc5c4e24b,0x112512c6),LL(0x409ca870,0x26ff024e),LL(0xccd67ae6,0x4f8c1dc8),L_(0x22878c0c), + LL(0x95b13b3e,0x9e553300),LL(0x668c95c0,0x7fafd447),LL(0xf8451257,0x6a113d4a),L_(0x1f574c1c), LL(0xedde7f07,0xaf37550f),LL(0xf5868469,0x0b1d7c54),LL(0x69e02d8f,0xcfd6985a),L_(0x9a97854f), + LL(0xecd3e1ab,0xbe482681),LL(0xdc9e0113,0x47798831),LL(0x9fac48a4,0xafbbf20e),L_(0x1edb9dbc), LL(0xd6f1c5bd,0xfe2c6138),LL(0x58a4f161,0xd3d7cf09),LL(0x0fdeccc7,0xa05a2e1d),L_(0xc2e844b9), + LL(0x6b3cd8ad,0xa16dcddd),LL(0xe9387af0,0x0fdf0fc3),LL(0x3cb7db3e,0x5144175e),L_(0xc5aef05c), LL(0xdaa284b1,0xb41e11e0),LL(0xc70c6218,0x334a53c4),LL(0xb6f86d54,0xea0526d6),L_(0xd996408c), + LL(0xef18bf5d,0x268ce667),LL(0xd54a4821,0x70f67533),LL(0xb9cffb18,0xc3038e8d),L_(0xebd042c2), LL(0x29b2c2c4,0x4d1a3a20),LL(0xd6237f1f,0xe2c18664),LL(0x5ab3157b,0x7a60e7b3),L_(0x6be26cfd), + LL(0x1c5c3f1e,0x28e72014),LL(0x5435c35c,0x32068427),LL(0x47c37b5e,0x94c1b6e7),L_(0x0c649b6b), LL(0x5f5b4e72,0x77567891),LL(0x3ebfe7c4,0x55a4f7f1),LL(0xc6f93ff6,0x8f654b8b),L_(0xc6898456), + LL(0x8fa02753,0xfcbae4b8),LL(0x9d98e011,0xcff78948),LL(0x22632d48,0x70d86a48),L_(0x247b98b7), LL(0x00132e50,0x3721a7ea),LL(0xad1058dd,0xdcc00609),LL(0x0c2cbce2,0x38fb42c3),L_(0x574791e5), + LL(0x5ea4abc8,0x5f3751e3),LL(0xaacf3fc1,0xac364b4a),LL(0x495dd3d3,0xa6c1d838),L_(0x7fd58564), LL(0xba68755c,0x465b3970),LL(0xfc76d6ce,0xfe4e336d),LL(0xfc16f7fe,0x2c759722),L_(0x76296d07), + LL(0xdd3a0177,0xcb5a7236),LL(0x9f03a670,0xa2bbecb1),LL(0x1f553745,0x46d5deeb),L_(0xc27cf8e5), LL(0x4c542036,0xa92075fc),LL(0xe9a1424d,0xa99e29eb),LL(0x31176d99,0x7ca41c07),L_(0xd9fd4345), + LL(0x39ce1949,0xbff084c9),LL(0x98f4d791,0x43984570),LL(0x05ef0ee6,0x0a861937),L_(0x4abffe54), LL(0x1288a0c4,0x52997092),LL(0xe85a84ab,0x24210cbe),LL(0xec238c31,0x9e2872c4),L_(0x574f26c0), + LL(0x40fd474b,0x5b8538a3),LL(0x1c70eab8,0x0d208047),LL(0x1cb60ff4,0x761298e8),L_(0xcfbdc26c), LL(0x1b6d4500,0xbcf74593),LL(0x7f1b2563,0x746b2f75),LL(0x679d4dee,0x32233e28),L_(0xdec42565), + LL(0x0c718f1a,0x66cd9b3c),LL(0x18427f69,0x3b3a6f84),LL(0xa9ef5cce,0xca539a7c),L_(0xeb7d5235), LL(0xed44fad8,0x4db83c78),LL(0xc210c701,0x5a9002ed),LL(0x57ef09d7,0x47b485ec),L_(0x27b80c9d), + LL(0xe4537556,0x4cc9c9c5),LL(0xea7854b5,0xec59f90a),LL(0x55f05058,0xaa97605c),L_(0x82a4feeb), LL(0x66f84f5e,0x8c26f4dc),LL(0x51a67be7,0xff1f32e7),LL(0xda9cb1ce,0x92d5f2c4),L_(0x3ef1254f), + LL(0x5ec4e735,0xffb048ed),LL(0x17a4afa4,0xb9ca6eb1),LL(0x4f8b9e40,0x089b9558),L_(0x6e75ba13), LL(0x43e91e89,0xcca4bf58),LL(0xbbaa89ed,0x21a8592f),LL(0x2180b560,0xa9e4e373),L_(0x2a4fdd34), + LL(0x95e834c3,0x85c32c46),LL(0xc68fbf2c,0xad18d468),LL(0xd0c03244,0xf46c7fff),L_(0x8c227e96), LL(0xde2297b6,0x8c32b174),LL(0x6ae73d4a,0x28635d97),LL(0x8b431269,0x47d44429),L_(0x2421faa0), + LL(0x28c78e41,0x93181781),LL(0x2ad021ad,0xcb623c4d),LL(0x98d40a79,0xdda4b06c),L_(0x372dd0c7), LL(0x3b688d0e,0xfe3fd141),LL(0xad24bea1,0xedd15e12),LL(0x3a965d08,0xf3097f75),L_(0x27f71cdb), + LL(0xbda9cc55,0xa6bfe2fa),LL(0x80a54113,0xf02a2f58),LL(0x1b6a851d,0xcb819b77),L_(0x21f5d0c7), LL(0x30780f8f,0x9f0d56af),LL(0xb5d4e624,0xf8a06fec),LL(0x27c807cb,0xb99605d9),L_(0x75a80279), + LL(0x6786d970,0x72255ae1),LL(0xc780fd84,0xc29277a0),LL(0xb9deb14f,0xb01102d2),L_(0xb94bf48a), LL(0x1c8e7e82,0xaf4a6e1a),LL(0x5d7fa22a,0x06c802c5),LL(0x4637b70a,0x21f2e3c1),L_(0x514a3fb5), + LL(0xb4a7c609,0x3a6619c7),LL(0xebdd3a32,0x5deb2376),LL(0x24dd90ec,0x7e94da01),L_(0x6003a0d8), LL(0x3788bd2c,0x80b9433f),LL(0x3d74e532,0xd49fdafa),LL(0x47c8e8f1,0x2be1c836),L_(0xce9e83c2), + LL(0x7f036147,0xf5f17b3f),LL(0x3b99cbce,0x45a7571b),LL(0x4e68fb20,0x68857bb8),L_(0xa10f6fe3), LL(0xa24d2c8d,0x2d13dee5),LL(0x4f9e2b49,0x88581796),LL(0x118e784d,0xf0493f53),L_(0x9d8cddff), + LL(0x58d860e4,0x8971f724),LL(0x316a2d94,0xb4f43e7e),LL(0xb5941f8a,0xb537d4a0),L_(0x5f19914e), LL(0xa327b763,0x0a0d438e),LL(0x5555f01a,0x23118e99),LL(0x2345f880,0xe40f8c30),L_(0x468a1ce3), + LL(0xcbd96a39,0x1197dd6b),LL(0xca22160b,0xb47615a0),LL(0x381bf7e1,0x251a8f8f),L_(0x168d73c5), LL(0xad578808,0x34717cff),LL(0x2bd8024e,0x5fbc83f1),LL(0xa4f9d4fb,0x64b6d093),L_(0xff7fe717), + LL(0xf183d4df,0xf6acf5ed),LL(0x23f4cacf,0x0cc30970),LL(0x181d2aa6,0x53b37c4b),L_(0x96f6cf89), LL(0x2ce916bb,0x38654a56),LL(0x2803be90,0xaba6d625),LL(0x09ca53b2,0xcdfe1968),L_(0x31bb234b), + LL(0x9e6894a9,0xf7c98ba1),LL(0x03ee4c19,0x8e28cba4),LL(0xe28c0250,0xdc432b47),L_(0x6e2b992e), LL(0x52a05294,0xff3120b7),LL(0xbc5f1638,0xd4520461),LL(0x3107ac59,0x92fc422a),L_(0xbc3da988), + LL(0x378db33d,0x1b7e14c2),LL(0xafbf4c3d,0x2e5d4aae),LL(0x536bff4a,0x885f68ab),L_(0xbe062464), LL(0xeaff53c4,0x8ea881df),LL(0xd93248b0,0xce5d6df3),LL(0x037450b2,0x6899f186),L_(0x09a8570d), + LL(0xe8227714,0xb158373d),LL(0x662aa9d1,0x73e5ad11),LL(0x0d1473e2,0x963a9a91),L_(0x5be0c8ec), LL(0x6796b003,0x32ad74a8),LL(0x5b558e5b,0xcad42d90),LL(0xa157eba3,0x64fb791b),L_(0x3c37606f), + LL(0x91b7250c,0xb6fc8bc6),LL(0x88f73c63,0x207504d1),LL(0xa04f258e,0x35b71223),L_(0x9284fb74), LL(0x000189fc,0x8cf221df),LL(0x5e498c73,0xe9692259),LL(0xc7e62afa,0x2482c08d),L_(0x96d697ef), + LL(0xa6fa3a45,0xe579dd13),LL(0x2b0e2ed6,0x8643df69),LL(0x57d9dfb8,0x9f01f782),L_(0x4cb4d8ee), LL(0xb5c2c767,0xbb840380),LL(0x358d384f,0x99e8ca50),LL(0x6c698864,0x31b4df34),L_(0xaa8c271b), + LL(0xfa87a979,0x6e27995c),LL(0xf59ed52f,0xc806a201),LL(0xc065a04b,0x93cff90b),L_(0x37c928f2), LL(0xbb867037,0x9cdfc841),LL(0x4f37a688,0x0a935f1c),LL(0xa9d96eb8,0xc15bd4d8),L_(0x98cebb91), + LL(0xd003f54b,0xa4a49d7a),LL(0xc496b9ae,0xa79d7692),LL(0xa15f7dc6,0xbd4d523d),L_(0x9f46848d), LL(0x6dbfe7b8,0xcc88e7c3),LL(0xcea47678,0x8264318e),LL(0x5e5fd5b2,0x5f2fa1e2),L_(0xe1fd44b0), + LL(0xec0ebc0a,0x1936faf8),LL(0xba06a3b9,0xc8c40d08),LL(0x01dd06de,0x54d85f2a),L_(0x18910c39), LL(0x8e9d926d,0xcbcea653),LL(0xc539056f,0x37d0733f),LL(0x0beb45e4,0x3de7fc81),L_(0xd43f2aa7), + LL(0x3205880a,0x9850a91e),LL(0x913c877e,0x5464c543),LL(0x1452f874,0xb9637ddd),L_(0xa9b45a5d), LL(0x7298be19,0xdf5911d2),LL(0x01fcff9f,0xf6dfd8a8),LL(0x8bb252ea,0x839f8d56),L_(0x14ad8567), + LL(0xc7fe9491,0xa2fc5db9),LL(0x23bc4e41,0x47b2b00a),LL(0x142e5001,0x557178b5),L_(0x7609c04e), LL(0x7281e45d,0x80053d16),LL(0x9af2e131,0x335bfb8a),LL(0x3285f848,0xe9e17269),L_(0xf7e53701), + LL(0xd38c0684,0xd98fa793),LL(0x74f33400,0x14117a61),LL(0x0d3b9c08,0x3e3d6538),L_(0x241aea00), LL(0x374f82ab,0x538b9bf1),LL(0x86a9a615,0x1cc16fb7),LL(0xe0c1bced,0xc0de70fa),L_(0x85b224dc), + LL(0x96448e8a,0x98c48df5),LL(0x5ba1fe76,0x50f58327),LL(0x4770c98e,0x197ced21),L_(0x2ca4f59f), LL(0x1af1733d,0x2bca7ffe),LL(0x003eeda5,0xeac8a367),LL(0x13278f06,0x5dd0431b),L_(0xa00d2f29), + LL(0xa83c536f,0x02379421),LL(0xe322906d,0x218d25f5),LL(0x90caeeec,0xf254a0e4),L_(0x2e6c9704), LL(0x44d824c0,0x0bbdc5a2),LL(0x19e00afb,0x21933c71),LL(0xec2c7cff,0x0a756aae),L_(0x3f239664), + LL(0xd8438b58,0x1d02bb13),LL(0xda3809b5,0x25ef520a),LL(0x3dd8667d,0xe3e22c9e),L_(0x68203ac3), LL(0x0d51408f,0x0227f2bc),LL(0x99baf10d,0x090e4a02),LL(0x2fc23bc1,0xf9171e03),L_(0x3a29ec64), + LL(0x8d85409c,0xd1184135),LL(0x735db311,0xf357dae0),LL(0x7b364c45,0xb9d690d1),L_(0xee21514f), LL(0x0c5f601d,0xcb872f3a),LL(0x2e8ffdf4,0xbccb4f0f),LL(0xf7bfadef,0x9f017f8b),L_(0xb26ea0cc), + LL(0x15442e1d,0x62cd98e6),LL(0x2f59cf2a,0x58dcba94),LL(0x7426c5f8,0x80d0a8cf),L_(0x7c300a9f), LL(0x52648e24,0x3a628b1d),LL(0x1ffce053,0xf0844217),LL(0x9c140c5f,0xb3cc7bfa),L_(0xbb650a75), +}, +/* digit=30 base_pwr=2^210 */ +{ + LL(0xf0426be2,0xb4f44241),LL(0xeb69f614,0xefc43949),LL(0x80845fb2,0x90205f0f),L_(0x29198f5f), LL(0x8a6cba71,0x4345224a),LL(0xfb0f1abd,0xfc80d60b),LL(0x9c7703f2,0x3d5a7643),L_(0x663a913a), + LL(0xe4b299d3,0x79702991),LL(0xdfeaa3f0,0xad58884a),LL(0xff023a31,0x18b5c6ce),L_(0x8cf14303), LL(0x433c5230,0x3e4907b4),LL(0x13551cac,0x3c1c0282),LL(0xe4454779,0xb4df9ee5),L_(0xae0275c5), + LL(0x658675d8,0xdf2f4ac2),LL(0xd8448d41,0xdde660e0),LL(0x50213318,0x64826f9f),L_(0xb76ac409), LL(0xadcc3e46,0xbc26833e),LL(0x508eaf24,0xde547aae),LL(0x619251e2,0xf96f73d1),L_(0x7bcc2806), + LL(0xba3e8f2f,0x728cadb5),LL(0x765732f7,0x0feebd09),LL(0xdb740393,0xdcffa1d6),L_(0xab748823), LL(0xa742cd70,0x549a4fca),LL(0xf07ac1f7,0x3d24aeea),LL(0x426c42e9,0xb67d7f28),L_(0x497b724b), + LL(0x96cc626f,0xd2b3305b),LL(0xab29e4cb,0x355d47d3),LL(0x33f53993,0x960f4207),L_(0xbf71ccbf), LL(0x329378d4,0x1004c020),LL(0x86ffcabb,0xa9117cf0),LL(0x0e5dbd85,0x73f55c21),L_(0xe9de73fd), + LL(0x456357ca,0x1b3ded00),LL(0x47adbb71,0x0f0f2d4f),LL(0x51648f45,0xab4e5da6),L_(0x4659c423), LL(0xe96f0203,0xcb55e2cd),LL(0x74ebdf1a,0x7b3cc4d4),LL(0x0090498b,0xdfb4de74),L_(0xc69c1621), + LL(0xba643590,0x757e6fbb),LL(0x65485c65,0x95336360),LL(0x50d4d37c,0xac96f5a7),L_(0x676d85ce), LL(0x53008bb9,0x33869046),LL(0xf21c0791,0xfb1262f1),LL(0x1e3e57ec,0x407af3f6),L_(0x6c3c72ca), + LL(0x9d6ad458,0x78e956e9),LL(0x46bcbf36,0xb94af545),LL(0x32977879,0xd20f35a3),L_(0x3ba745fd), LL(0x9c65ff22,0x5c8f99f9),LL(0xac6bb14a,0x4ce9a00d),LL(0x307b3d8d,0x798d2cf9),L_(0x9ddb4cad), + LL(0x3b8e6ef6,0xb6ca3273),LL(0x5ade36d2,0xe260b8d0),LL(0x9c66c2a4,0x05eb84cf),L_(0x884d04c5), LL(0xbe1f3ed0,0x1e35adca),LL(0x4cff4e8c,0xa175edc2),LL(0x4681a0ae,0xc6715ff6),L_(0x506a8488), + LL(0xe9c88637,0x32143c93),LL(0xdf482c05,0x434818e2),LL(0x25a929c9,0x4c806ec9),L_(0x1179b4ee), LL(0xfe3e80ad,0xa0372526),LL(0xf64ea692,0x5279eb89),LL(0xee97419a,0x98dbe31d),L_(0xdfad8124), + LL(0xe8bb79d4,0xa6d3938a),LL(0xbdea4fe1,0x0b2f88a1),LL(0xf279df50,0x59a54b2e),L_(0x343b8869), LL(0x1390a7ad,0xc9c37e25),LL(0x9e03f761,0x0777586e),LL(0x3aa2516a,0x42cece95),L_(0x732cba8a), + LL(0xa47e21c2,0x5f9875c7),LL(0x327db722,0x2ecbc86c),LL(0x4ac56066,0x5ea26fc2),L_(0xc69b2d5c), LL(0x31e429d7,0x944c3736),LL(0x7cb051f9,0xcf2fc959),LL(0x02c3927e,0x3bb33cf9),L_(0x5a57d920), + LL(0xcf56d754,0x64f98e4d),LL(0xfbe5a38c,0x1c7766cb),LL(0xb20cfe07,0xe351283a),L_(0xd825af7e), LL(0xdc9dc466,0x3bee62a5),LL(0xe5b281f4,0xb2e1f644),LL(0xfe54699d,0x18ee6ea1),L_(0xe71cfdbc), + LL(0xd2fe136e,0xe5f658a8),LL(0x39fa85ce,0xd83b5473),LL(0x94a1a0d2,0x586d7529),L_(0x208e5f0f), LL(0x0748fda7,0xe98d9b7d),LL(0x869e47b2,0xf0def9e4),LL(0xc4e12cd5,0xf40402da),L_(0xe2e9c40e), + LL(0x2956c763,0x257681d9),LL(0x9408aa0f,0x696e0261),LL(0xfd84ba5b,0x0675debf),L_(0xa7a000fc), LL(0xef9b82a6,0xe38868a1),LL(0x8226a77b,0xcdcdf922),LL(0x292fb9c9,0x269d71d2),L_(0x587c3f9c), + LL(0xb5fae58f,0x55622b3a),LL(0xb3258e77,0xff23a08c),LL(0x56c2b54e,0x866ba05b),L_(0xb8835413), LL(0xa737dae6,0xdc5131fd),LL(0xbcb13de4,0xbb1176e5),LL(0x860b7b0d,0xf51c76d2),L_(0xfc88df27), + LL(0x6bd647f3,0xe811642e),LL(0xf28850fd,0x0776f556),LL(0x2c9fb90d,0xb8cebf02),L_(0xa7ba32f0), LL(0x643fb866,0xc4902b6c),LL(0x3a4ba84e,0x74aac273),LL(0xfd2878e7,0x1a76b1b7),L_(0x4e9ed0a9), + LL(0xf609c0b0,0x0cfea42c),LL(0xc8c86640,0x7a72c3c6),LL(0xb9872dc7,0x1ae5bcf6),L_(0x4967c18a), LL(0x9fe44849,0x2a90f4d7),LL(0x7c9836d4,0xdcdd6bf1),LL(0x32eb5ac2,0x939219ad),L_(0xa85aba1e), + LL(0x267059b7,0x2669c8c4),LL(0x92b26067,0xb6ba656c),LL(0xc80a4182,0xa67b63ae),L_(0xdabf1310), LL(0x7acd503c,0x98520b74),LL(0xa4e52b3f,0x0688ae94),LL(0x47fc66da,0xacf9786e),L_(0x53a32432), + LL(0xde210bb4,0x74430a09),LL(0x341d2f54,0xf0e810c7),LL(0x85cefd4f,0xba1cac0a),L_(0x649747b6), LL(0x35194ff2,0x6452e158),LL(0x3bf68483,0x8ed85e25),LL(0xd8535bc9,0x3bc32918),L_(0x1618b5a8), + LL(0xd3467c73,0x3046a69b),LL(0x11b44c9b,0x996236b5),LL(0xf02f1b90,0xdbce524f),L_(0x2880b5ed), LL(0xdcc369a7,0xa2eda832),LL(0xbc28c7a6,0xcb124909),LL(0x33444c56,0xb4685d00),L_(0xc588ee2f), + LL(0x6fe5a182,0xa316d37d),LL(0x7b8d3ccd,0x75ec9a2a),LL(0x8b98fad0,0x0c5e4465),L_(0x96299b8e), LL(0xbd855a8e,0x7661287b),LL(0x21ef677f,0x9b044bfa),LL(0x0f23ddbb,0x7abd77aa),L_(0x217ef151), + LL(0xe7d89f59,0x490f55aa),LL(0xf4051055,0x3aa72313),LL(0xdea3a145,0x159c120c),L_(0x3a83cdd8), LL(0x840bb947,0x1516983e),LL(0x3c23bffe,0x527c0f73),LL(0x0d4d39e1,0xe3846304),L_(0xa097f7b6), + LL(0x56c53d60,0x8c3176ff),LL(0x95152224,0x26489625),LL(0x94c341e1,0xa5b14d8d),L_(0xe01267c9), LL(0xb4f1f613,0x1a2f793c),LL(0xd1a0049e,0x600ebda3),LL(0x6ba99e2b,0xd1571ee0),L_(0xc34b7e2c), + LL(0x122b663a,0x41a70d6d),LL(0x5102d2ac,0x50c3e831),LL(0x37bdeb4c,0x948ab46f),L_(0x69f45329), LL(0x1a55a55b,0x3b20c1e6),LL(0x22ef8bd8,0x81e5763b),LL(0x4ddc462f,0xffa876e2),L_(0x8337e2e2), + LL(0x62e4c32d,0x7f945839),LL(0x8947b09f,0xa1e798f7),LL(0xfe755aec,0x5524084e),L_(0x72eaa739), LL(0x02c2a9e3,0xdc627b0e),LL(0xc75c26c5,0x7c827c47),LL(0xbcf0b634,0x677ad978),L_(0x31454aa7), + LL(0x74af562d,0x8713ce0f),LL(0xf85d00e9,0x6951ea39),LL(0x23d2ab0f,0x761f3a78),L_(0x1c36b814), LL(0x098fb6b0,0xeb48e791),LL(0x84522286,0x160c9294),LL(0xf1992cc6,0x2ed7be4f),L_(0x48acfd7d), + LL(0x02ad20b2,0x30165084),LL(0x162d53f9,0xb8bee43f),LL(0x67757e6c,0x0eedd6d6),L_(0x00fc0927), LL(0x5acf9124,0x8dee8798),LL(0xa5fd0b77,0xa4c99259),LL(0x9970bc21,0x54e339cf),L_(0x9e45d123), + LL(0x7f335ce0,0x5019bc77),LL(0xcd9df5c6,0x15ce5f7c),LL(0x4f31c43e,0xfa69d995),L_(0xdaf4bdbc), LL(0x1d57d52d,0x20dc7c66),LL(0x1ccb081b,0x01a82796),LL(0xa8fa2376,0x861ac30c),L_(0x519f6c6e), + LL(0x56acbf12,0x7b20a0c7),LL(0x54051bfe,0xbec3d69c),LL(0x5e324e3c,0xd3d97b66),L_(0x5235011b), LL(0xc585e141,0x2b8e18cf),LL(0x3540e46d,0xe0b2513f),LL(0x6e7d6c03,0x7e1a6116),L_(0x9b468e4d), + LL(0x23bf2ce9,0x82122cee),LL(0xa723e8e2,0x318ba203),LL(0xa93e44d4,0xd2180b18),L_(0x643b190c), LL(0xd9d85de9,0xd7d06345),LL(0x0751ebbc,0x5bea35e9),LL(0x952d933c,0x72bd41e8),L_(0x2910ff54), + LL(0x60e9cb5f,0x107232fb),LL(0xd6c371b4,0x004f8683),LL(0xa23e4299,0xefa3c56b),L_(0x8d86e9ba), LL(0xad436904,0xbb576a94),LL(0x9a66d1e1,0xbd78745f),LL(0x3513e3fe,0x4066ae12),L_(0x69b558b0), + LL(0x627d1808,0x7365dc29),LL(0xe9b14b35,0x0dec4459),LL(0x96d5022f,0xf4343e09),L_(0xf65c2fb7), LL(0x00d16dd7,0xeeaa6e35),LL(0xe87ecb3a,0x220f953d),LL(0x91ecc397,0xaf714cd0),L_(0xaea56471), + LL(0x6dd11273,0xce83daf3),LL(0x0a835712,0x6ce5dec2),LL(0xce89d045,0xbafd94a5),L_(0x86e671a6), LL(0x577ca0e1,0x1e36fd91),LL(0x6fe72f1f,0x060d396f),LL(0x729ba81d,0x8b7e4fbb),L_(0xea9c3f32), + LL(0xdbaa1960,0xb43de333),LL(0x030ad3be,0x2ccf4a4e),LL(0x036ff9cf,0xafc236c4),L_(0xe987e57b), LL(0xd7468cbc,0x9a645200),LL(0x61eddc02,0x5b2deae6),LL(0xb553572e,0xd4efa381),L_(0x9dde1eaa), + LL(0xc9536b4b,0x10222aaa),LL(0x6d0e36c2,0x84934c38),LL(0xdf7ca8e3,0x5e5dc148),L_(0x2da86a73), LL(0x4d98778d,0xb7c8140c),LL(0x69fc5568,0x69c09498),LL(0xb0ea1d36,0x7e0cd235),L_(0x6c99f36a), + LL(0xe9443a73,0x94747e78),LL(0x377b44d6,0x2415da80),LL(0xf50cdd9d,0x65d2fa97),L_(0x9d194135), LL(0x3ca31050,0xaf39be74),LL(0x06450aa6,0x0441383f),LL(0x393e3175,0x4e5bd1d4),L_(0xb7244573), + LL(0x70145556,0xc463ec32),LL(0x867b11cf,0x1e5a98cb),LL(0x59b3adf7,0xeb7c80fc),L_(0x0929efbf), LL(0x5234550c,0x4eb610a4),LL(0x7d606935,0xdc8db9ba),LL(0x52106057,0xbc3a065e),L_(0xad60a058), + LL(0x6e360f56,0xee7f7e1c),LL(0x4a514c99,0x7411bd70),LL(0x627a5d54,0xb191903e),L_(0x9d4472f8), LL(0x6bb808ee,0x484c498f),LL(0x110de6ea,0x3cef8b7c),LL(0xcc379eda,0xe1f0ee29),L_(0xcdc2d24b), + LL(0x21a05590,0x2ea40699),LL(0x953235f9,0x057a8486),LL(0x5834fcc4,0xb35b630b),L_(0xef9321a5), LL(0x2d1d0e73,0x6560d0d6),LL(0x4083b1e7,0xeb026a8b),LL(0x4d572129,0x720991c7),L_(0x98865061), + LL(0x657c8db8,0x5e4f7ccd),LL(0x35de8d7a,0x8351fe6b),LL(0xa43443b2,0x48c1ec10),L_(0xdd11de81), LL(0xb376d6e3,0xc77d6dfd),LL(0x73bed5bf,0x0f953512),LL(0xeae9d0a1,0x91a7a744),L_(0xfeb76333), + LL(0x59454f28,0xd2c2004d),LL(0x9f5eebc7,0xacc344aa),LL(0x775a61d7,0x1c186106),L_(0x47a8ab1c), LL(0xdcf00b61,0x4e3f081c),LL(0xbc56bbf8,0xcaa9cad2),LL(0x1e4e523b,0x816a09ca),L_(0x5e9a9445), + LL(0xd468c097,0x86aa5ecf),LL(0x244131ce,0x26d08883),LL(0xe43dffa8,0x6501f291),L_(0xf4cfc564), LL(0x88248cda,0x4711c711),LL(0xf7fb2e43,0xa4245343),LL(0xcfde7c18,0x4111cd00),L_(0x5a68a986), + LL(0x0df3b4b1,0x9cf9b276),LL(0x71920b8d,0x9dd92d6a),LL(0x6444967a,0x5f3bbd8f),L_(0x82870cd4), LL(0x1a4b4821,0x2d798278),LL(0xd460ee52,0xe9681e54),LL(0x0ce3d37c,0x739d5ad5),L_(0x5645ee72), + LL(0xcf20f025,0xf4697953),LL(0x6debfc44,0x789e6af7),LL(0x1050c0c2,0x383c22fa),L_(0x22fd43b3), LL(0x5cc2adcd,0x2ace7063),LL(0x55ea131f,0xf6f6aaf0),LL(0x31c60f20,0x7b350e28),L_(0x9f8df3bf), + LL(0xf49e5f62,0x1a92eddc),LL(0x24db3514,0x365a5041),LL(0xc89ba67e,0xa32fc59f),L_(0x33efcf95), LL(0xdb9db26c,0x09322a72),LL(0xf91f9ca4,0x3fda4216),LL(0xc6c63b34,0xddeb9986),L_(0x9a067f25), + LL(0x77b280f2,0xc18c7395),LL(0x93723817,0x4cd03829),LL(0x37436f93,0x66528f8e),L_(0xeb73448c), LL(0x2dd6704b,0x83b283eb),LL(0xedc8d2d1,0x54972ce7),LL(0xbca75a4c,0xa9f10506),L_(0x91b481a9), + LL(0xee53ff2f,0x5105c866),LL(0x7f1e21b7,0x02b0042d),LL(0x1f9de3bb,0x4f9500fd),L_(0x89fa5ada), LL(0x8cb6dd5d,0xaa04e960),LL(0xb2782218,0xa0112a6f),LL(0xec0e66ff,0xec305363),L_(0x77fc7f14), + LL(0xa014524c,0x400fd142),LL(0x17086d36,0xf02c64c2),LL(0xd9684416,0x8da612c7),L_(0xb46c7e04), LL(0x4e54713c,0x7cde46af),LL(0x0546cc9f,0x8380cc14),LL(0x5013084d,0x80963cc8),L_(0xf957f8d9), + LL(0x6dcbbe49,0x6b6125b6),LL(0x038efe1a,0x67b840bc),LL(0x16c79367,0x570c02ad),L_(0xcb60af87), LL(0x8552cefe,0xa8167ca5),LL(0x5e88a6fd,0x3636e864),LL(0x9c4305d9,0x43d5af7e),L_(0xc9e5c723), + LL(0x413cd65a,0x9c95b00e),LL(0xb0a4ab3f,0x64a28801),LL(0x6ef01bfc,0x858e4b79),L_(0x80657f33), LL(0x113a4fdc,0xd390a92a),LL(0x71a0cef2,0x774c1209),LL(0x63557ff4,0x54e47eaa),L_(0x4ea9a478), + LL(0x27760c7e,0x9e3147b9),LL(0x656b24fd,0xb9bb9711),LL(0x89363a50,0xe852edfe),L_(0x6bf2556d), LL(0x43ee10a6,0x2d6acef6),LL(0x3e9d7e57,0x03fc12e9),LL(0x20890385,0x2d45580b),L_(0x6b8bf1da), + LL(0x4101d170,0xbebb2e76),LL(0x7fdf91ff,0x533638ee),LL(0x05bdd7f6,0x92a0bd39),L_(0x78bac139), LL(0x4c760097,0x81c6243b),LL(0x8d71803c,0x5119eeba),LL(0x049c6a06,0xc410f246),L_(0xa9b7c2f7), + LL(0x51167f9b,0x12be7d2f),LL(0x66cdc913,0xa41ce391),LL(0x0572301a,0x0a761ede),L_(0x2a186fed), LL(0xe5dca065,0x3dad4d6c),LL(0x14733148,0x82d548d7),LL(0x79a33fd1,0x76d2d77b),L_(0x744a660e), + LL(0x77f53015,0xa4c2daab),LL(0x54ae5f2a,0x40a7d787),LL(0x604bbbf3,0xcc41bd91),L_(0x7fe1506d), LL(0x4c6757d5,0xe55da08f),LL(0x38212cd6,0xf50459ca),LL(0xf8d3a03d,0x8252e5cb),L_(0x62160548), + LL(0xaaca1be2,0xb4f7cd9f),LL(0x3012b6d5,0xfae3835d),LL(0xf685dd47,0x6c5b897b),L_(0x3f52f5a3), LL(0x03e5ac2f,0x71afc215),LL(0xdc13191e,0xce6c64cf),LL(0xa1bdc366,0xe59eacca),L_(0x719ce6ba), + LL(0x28e18007,0x428bd7b1),LL(0xba236a3e,0x3bf0eb6c),LL(0x69ce21e0,0xfbe09c69),L_(0x321ac5e5), LL(0xf7c3e583,0x3283bc61),LL(0xaf3a5b06,0x20ead3f8),LL(0x1f935f24,0x3b040b57),L_(0x39549c75), + LL(0xa23c1b85,0x41a0017c),LL(0xc677b960,0xc09598e4),LL(0xffd68469,0x1da39b13),L_(0xdfff7bf9), LL(0xb19fcd21,0xe71a0fa0),LL(0xc987c7de,0x83e812da),LL(0xb605b69d,0x0efdfc84),L_(0xdde3a792), + LL(0x656d6dd1,0x58a3f7d2),LL(0x9e07158a,0x1396fe6b),LL(0x52fddb17,0x39edad21),L_(0xd739977f), LL(0xfb57f56a,0x1072b4e0),LL(0x80c7df0a,0x94372dad),LL(0x0a26aff2,0x88f38fd7),L_(0x5d33e01b), + LL(0x6d4895a4,0xff2438e1),LL(0xa9ba3f6a,0x67f22bee),LL(0x3cfaa48e,0x6bf9fc2b),L_(0x702030cd), LL(0x7b4df30f,0x956e9ec2),LL(0xe0368ebf,0xe6cb0b1b),LL(0xfc98eaec,0x7dbf18a3),L_(0xf5fc22a9), + LL(0xf20afcc9,0x9b4028ed),LL(0x2adb3986,0x4e476a55),LL(0x6bf05b58,0x2b0c488d),L_(0x74de42bf), LL(0x01d3ca75,0xc0d6bd28),LL(0x668206d5,0x95f353c8),LL(0x010298ee,0x034e53ca),L_(0x13b81d0b), + LL(0x80ff52bc,0x50412d11),LL(0x652fe1b8,0xbe2f9f7b),LL(0xe84a58ad,0x83284261),L_(0x905c06a9), LL(0xc605b12c,0x5da82b44),LL(0x2ca210de,0x837f9c22),LL(0x10d656cc,0x454d1413),L_(0x5a580fa0), + LL(0xde3d1dc5,0x6619a39c),LL(0xac6af7d7,0xaaa9723b),LL(0x3910ae67,0x86fdb715),L_(0x4d8f0cf8), LL(0x3782c29b,0x5c35eb01),LL(0xe1483a4f,0x76b4be95),LL(0xdd570ea6,0xf56b5810),L_(0xefb949a7), + LL(0x9ec12516,0x3f080f23),LL(0x108296b6,0xc3509a49),LL(0x08bba00b,0x07998613),L_(0xdc853801), LL(0x5c36ecb3,0x75c6840a),LL(0x73946b3d,0x3e8f5265),LL(0x2c678e3c,0xc7f1cc73),L_(0x284107e6), +}, +/* digit=31 base_pwr=2^217 */ +{ + LL(0x2409728d,0x2d5fd352),LL(0x84897b53,0x28f3cb13),LL(0xee72bd56,0x21c2f494),L_(0xf66c03f3), LL(0xac73118e,0x283e1b56),LL(0x3b39e360,0xf00c3802),LL(0xaa6c7b0a,0xee304f23),L_(0x0dd1d011), + LL(0x5b89b52c,0x433d8d17),LL(0x440ff02f,0xec27bd07),LL(0x49353c25,0xf18b23f5),L_(0xd9e62ddc), LL(0x2c798eed,0x91011438),LL(0x30fcf910,0xfaa77bac),LL(0x6e2db36a,0x2a3b730b),L_(0x3ada8eb1), + LL(0xea09f155,0x18c36c89),LL(0x3ba3b853,0xff1cf2a0),LL(0x1d069cd7,0x672fde0f),L_(0x0a42b710), LL(0xf1f4e5ba,0x86238146),LL(0xabca7963,0xcdc7fd6a),LL(0x00bcd404,0x51a105eb),L_(0xf842633a), + LL(0x348659fb,0x0afc95de),LL(0x4b20153e,0x478c3432),LL(0x1f2f8d2f,0xaa0e8781),L_(0x27294e88), LL(0xbf3811ff,0x526d1fb9),LL(0x9ad871cc,0xf802b4ad),LL(0xf613e178,0xd3df21bc),L_(0xb619f301), + LL(0x40af417a,0x3b133f03),LL(0xb449508f,0x9b5dda1c),LL(0xc2f372b3,0x3948b7d7),L_(0xba404740), LL(0x99028594,0xbcf7e27f),LL(0x96542b58,0x1de8aa5e),LL(0xb2fc7cd3,0xaadbe95a),L_(0xaa07786d), + LL(0xff2c8bea,0x26721264),LL(0x5d2747dd,0xbd6c65e6),LL(0x7f2093a5,0x21e5d488),L_(0x2ff2524c), LL(0xa000a3ed,0x68530945),LL(0xef75ea86,0xd1f6fc3b),LL(0x3f15aab2,0x9b5d8cbb),L_(0xfc18b474), + LL(0x45f74206,0x97a4da7b),LL(0xcc1484f1,0x6d246546),LL(0x819c4b2d,0x8255bc16),L_(0x4605c7b7), LL(0xf9d396e2,0x5502b719),LL(0xfde1aa97,0x8b9b6ca7),LL(0x94965c57,0x3c7a5ffc),L_(0xea93e0d6), + LL(0xc11d6985,0xbad3d4ba),LL(0xb1adc951,0x67c5bea3),LL(0xb82cec9b,0x93b6322c),L_(0x8ae8a415), LL(0x9661c047,0x769c204a),LL(0x31aa2f54,0x35880fdb),LL(0xc215f2cd,0x4f5bf77f),L_(0xe0f608d2), + LL(0x8bce7ed0,0x847b9ee6),LL(0xb2892f4e,0x8cb1f02a),LL(0xfffe0edc,0x62c22364),L_(0xb0229884), LL(0xdcd7d51e,0xa9cd4c50),LL(0x4688d647,0x30edb5dc),LL(0x82ea29e2,0xcc4403d4),L_(0xb78d8b6a), + LL(0xed02ad3b,0x66e7f904),LL(0xdf2ace65,0x2877668a),LL(0xe7bcfa22,0x873070fa),L_(0x3a2466a0), LL(0xf013615c,0x2fe9dfb1),LL(0xe228049a,0x51ff974b),LL(0x56961e83,0xcd87825f),L_(0x0c9c9408), + LL(0x5b989bda,0x2f915297),LL(0xca9c644b,0xcf0387e6),LL(0x80e7dd9f,0x2e7c6dd7),L_(0x4b7f801a), LL(0x861c2ffe,0x1df12d2f),LL(0xe8fd6f24,0x048668bb),LL(0x9b862a1d,0x2b64be2a),L_(0x4dde2a3c), + LL(0x644173f8,0xc3611806),LL(0x23a43a5b,0x78a6d7ad),LL(0xe22b26dd,0x1dd89345),L_(0xfd5f2c34), LL(0xd1587983,0x7692c1c8),LL(0xca1f4a95,0xecf6b1c3),LL(0x0a975fb0,0x0d6babd5),L_(0xc6e6c5b9), + LL(0x4222b0ed,0xb5c91a5b),LL(0x6d847e8b,0x0ec3e064),LL(0x9d0d2af2,0x660e553e),L_(0x0aaf0e60), LL(0xa6329190,0xff1f387c),LL(0x9970f312,0xd795c508),LL(0x67e40e1b,0xdada8e3d),L_(0xe265b2ea), + LL(0x6eb26317,0x4e1593ef),LL(0x12da5305,0x8698f986),LL(0x2d0a715c,0x8e8b3499),L_(0x80e2d2d8), LL(0x0b43672b,0x132c1c9a),LL(0xee8ab3bd,0xfd973660),LL(0xea97e0b3,0x3b811a56),L_(0xbeaaf50d), + LL(0x3edb12e0,0xf66872e6),LL(0xb9ab7a65,0xc41cad86),LL(0x0cfa43fd,0x43f4aa49),L_(0x781ca102), LL(0xcfa7ab7b,0xafd31516),LL(0x061e0da2,0xc78d7ba1),LL(0xea786f69,0x2090d8fa),L_(0xdf64f185), + LL(0x7ee50090,0x28c86852),LL(0xff58b162,0xd4576859),LL(0xc2987502,0xe596765e),L_(0xf8c4c45c), LL(0xfb631b85,0x42aaadd8),LL(0xb6c6bfcd,0xf7d72923),LL(0x2257f5b6,0x260ae885),L_(0x2f0626ed), + LL(0xdb84c9ba,0xdbb13e44),LL(0x7c40add1,0xa7294693),LL(0xc1443c26,0xe02af6c2),L_(0x935cf64d), LL(0x293f667b,0x52bbc8cd),LL(0x5d49c23c,0x7e1de72e),LL(0x6907cdc1,0xf8a9aeab),L_(0x01bb3a0e), + LL(0x3fade183,0xd7401dde),LL(0xa78758f9,0xc85449b6),LL(0xc47549c9,0x6ec51217),L_(0x5c22aa19), LL(0xae146939,0x4bbe9886),LL(0xd2682769,0xee34940f),LL(0x9f320ada,0x4fdf9e41),L_(0xaa18dddb), + LL(0xf506f84a,0xf58881e9),LL(0xe2a44aae,0xd6feb621),LL(0xedfd3c65,0xb163b10b),L_(0xd0eb9a35), LL(0xb8f8a7bc,0xb10b02be),LL(0x2524cc98,0x1e5816c9),LL(0x0ad84066,0x9cd4eb66),L_(0xecbabab5), + LL(0x413431bc,0x9ea4aaec),LL(0x217b55d3,0x135f5f2b),LL(0x9e4519f3,0xd3e6e8f7),L_(0xb072b798), LL(0xfa8179f9,0x318fb330),LL(0xcb83a9f9,0xe0cb106a),LL(0x95e1ca87,0xfb4ce48c),L_(0x87fc571d), + LL(0xf3d74e55,0xb7fb663d),LL(0x3bebc7e0,0x6706a48f),LL(0x7f8a7cfb,0x52d7353d),L_(0x13ea8891), LL(0xc42a9ef4,0x5bd192eb),LL(0xdfeb9aa7,0x2b1ff6c0),LL(0xaa53702a,0x5388fa6f),L_(0x7ee98651), + LL(0x423e53a4,0x0baae890),LL(0x6536911d,0x08234190),LL(0x9bfec6fd,0xc8f4d373),L_(0xd7ec5802), LL(0x119e3477,0x19587b51),LL(0xfac91771,0x1c276e15),LL(0x951dd485,0x98048932),L_(0x04243a3f), + LL(0xbe2d77ea,0x67d742de),LL(0x2df6c417,0x7983fd12),LL(0x7115e508,0xb9406bb8),L_(0xde1f9fde), LL(0xe45eb298,0xdf3a7991),LL(0xa54dc400,0x03ba9fe4),LL(0x25d8f862,0x6ea3053f),L_(0xb145e5fc), + LL(0xefbe9106,0x266e3900),LL(0xb0d74420,0xe042e34e),LL(0xbec6b919,0x5cb15a6f),L_(0x6c16d3f4), LL(0x75686740,0xfe7b7595),LL(0x2204cb17,0x3e67572e),LL(0x2d437dfd,0x03299218),L_(0x2be5d786), + LL(0xc2d6c314,0x3d460901),LL(0x3d26026c,0x591fe51a),LL(0xeb85aa17,0x44d65538),L_(0x015a9b39), LL(0xea596f91,0x48a2f0df),LL(0x8ce8790a,0xc2cabfcf),LL(0x17f7dae8,0x31a2b7ff),L_(0xba200e74), + LL(0xb389e985,0xb19669c8),LL(0xa23e72e7,0x729a4108),LL(0x53062f9c,0x779cbefe),L_(0xde2e49fe), LL(0xa29b29cb,0x6119b4c5),LL(0x0f5a398b,0xeb10107e),LL(0x91c7a000,0xd43d23aa),L_(0xc42c4a03), + LL(0xb2465ace,0x954d5e2a),LL(0x361566cb,0xcc9f7e66),LL(0xa5432c5d,0xd4ffbeb9),L_(0x5d478258), LL(0x686dc22e,0xb3aa6ec4),LL(0x29d8cc0b,0xf9a59b06),LL(0xd8d7140f,0x94e1344f),L_(0xfd56ad11), + LL(0x7dd1f80e,0x241f38ad),LL(0xc4ce241f,0x72ffb97f),LL(0xff97f61b,0xa3d97bc2),L_(0x9c42f7da), LL(0xd4e6e73b,0xb4b19841),LL(0x1c3d7005,0x5fd346b8),LL(0xa14d6ec3,0x366b72b2),L_(0xd6218402), + LL(0xb6fe5b76,0xe3a6e8d3),LL(0x6941f5ec,0x4ed1cc1b),LL(0x9f69a8a4,0xe6f9dbe8),L_(0x8ff4ee20), LL(0xa4d858f0,0x00a65edd),LL(0x30c9c798,0x9733fc57),LL(0xd4ab5744,0x38f910bb),L_(0x08dd1912), + LL(0x338c1ad5,0x4a664d78),LL(0x90d8cb53,0xd6ec4c42),LL(0xaee2829b,0xededd826),L_(0x0c5337aa), LL(0x417e0806,0x2c83b6ff),LL(0xd8bc600c,0x65e85629),LL(0x54ffde80,0x8972a5fc),L_(0x465b9ae3), + LL(0x49c2561b,0xd2b48b48),LL(0x6937dcf3,0x38241d90),LL(0x7dc5afe2,0xc399df03),L_(0x9c6cdf52), LL(0xc98d8a84,0x666e5386),LL(0xb9102fbb,0x33895284),LL(0xb43f7afa,0xb804c149),L_(0xfe016452), + LL(0xd0681519,0x93423c26),LL(0xa105d538,0xc2eabb36),LL(0xea12b99c,0x9297a177),L_(0xb26a7f93), LL(0xddfee3a3,0x4de65e15),LL(0x9a81b364,0x7f3bad80),LL(0xe43ae1fc,0x7af9ba54),L_(0x86e1e8dd), + LL(0x13762e74,0x5afe2e21),LL(0x8fe945bd,0xa21b9a1a),LL(0xd03deaed,0xd498e9f2),L_(0xe67e5384), LL(0x6ae69ee9,0x749eed42),LL(0xdd52d330,0x5f4c63de),LL(0x95aa2f5a,0x890e5e3e),L_(0x3772d05c), + LL(0xdea7e04a,0xbf3bb703),LL(0x449cf48c,0xc64a209c),LL(0xddb56124,0x6858318b),L_(0x25ad4355), LL(0x9e3a0204,0xf0dbd02b),LL(0xa3421c63,0xa216442b),LL(0x5ebba836,0xc0cbf7a0),L_(0x4f9c0d9f), + LL(0x9bdf3b82,0xe5bfa2bd),LL(0x0e12030e,0xef3da843),LL(0x7b3913de,0x107bba5b),L_(0x390841ba), LL(0xda5362ce,0x81eecff8),LL(0x2734c216,0x151060a4),LL(0xd9ea79af,0xe89b5db6),L_(0x2aafd537), + LL(0x310d44fd,0x4509feca),LL(0xa4868c10,0x25aa7253),LL(0xd40c4268,0xa958ef77),L_(0xe8930722), LL(0x5dae221c,0x71fc93e9),LL(0x0095e400,0xc3e3cb95),LL(0x0f45640e,0x6b7cb33b),L_(0xb3ea06da), + LL(0xc74fe8dd,0x8c9ef406),LL(0xaa86860f,0xb6fe758b),LL(0x04aa9cd6,0x4f8b3877),L_(0xdfcc0451), LL(0x9f568192,0x65aed657),LL(0x44ebcd65,0x85d0ebdf),LL(0x7968220e,0x04b5f3d4),L_(0x44142172), + LL(0x275a3076,0x3a2a339d),LL(0x88744d58,0x8a7f68a4),LL(0x5e03cf06,0xd663fea8),L_(0xdffb3ac6), LL(0x83dd1969,0x6151157a),LL(0x18fce4cb,0x620979ba),LL(0x6d82e1aa,0x3ac652d1),L_(0xa3d0c2ce), + LL(0x9f8e323e,0xed2fa4ec),LL(0xf62c31ca,0x554cf5e8),LL(0x14820bd1,0xc72e2e81),L_(0xb42dc3c7), LL(0xe15fa1be,0x5f0c938d),LL(0x0920a90b,0xdf2d997a),LL(0x33617c0a,0x3087e812),L_(0x631962db), + LL(0x7f81aff6,0xd6ea063d),LL(0x1c54d79e,0x631cf97e),LL(0x46b66ca3,0x3bfc14a4),L_(0xa20e2ff9), LL(0x3a379a19,0xaa5dfd05),LL(0x5f4d0392,0xf3af9e8f),LL(0x95b8ff29,0x928c3c53),L_(0x28ae39a9), + LL(0xe65a2e0d,0xa11c784f),LL(0xb5192f99,0xd33e59b1),LL(0x2f4c50c4,0xe15fe3e7),L_(0x30149c86), LL(0xe61d0a03,0xdff24b95),LL(0x2ec9f885,0xd0e84cbb),LL(0xc9c516d5,0xbc6f5dd3),L_(0x71fb6b1e), + LL(0x828d848a,0x55c5a959),LL(0x503ffb7b,0xed8f7fcb),LL(0x2be30973,0x0b24dc92),L_(0x90b2a59f), LL(0xc98877f6,0xd599613c),LL(0xe60dd939,0x43a0bc0a),LL(0x34b1b90a,0xa8edd1ca),L_(0x7ca529dd), + LL(0x126c6d32,0xb92a7bd7),LL(0xda613cbe,0xd9952b27),LL(0xb5f3accf,0xd21fa7b1),L_(0xda44ff77), LL(0x6ac1dedb,0x7d8ef38b),LL(0xc4ea5e03,0x26d207ab),LL(0xc01bf44b,0x58c8cee8),L_(0xb8f1a2b5), + LL(0x5d2a899c,0x94d31b36),LL(0xbc559f35,0x3050256c),LL(0xd6d74149,0xaa4efe0c),L_(0xbc1f929b), LL(0x9b7a733f,0x491dc4d2),LL(0x50e69c58,0x7540af9a),LL(0x358ed8d5,0x17dc58f7),L_(0x5a66b29f), + LL(0xb13a4f20,0x4d20428f),LL(0xd768684c,0xf17077d6),LL(0x27509ac6,0x58852489),L_(0xd30f66cb), LL(0x64ad83d0,0xc965f9f3),LL(0xca2684af,0x9a3903b0),LL(0x2677b45a,0x45839bea),L_(0x2155a76d), + LL(0x6c1eb10d,0x24b30b08),LL(0x4b90dabd,0x44f85d03),LL(0x688bbe3b,0xc74ccccb),L_(0xe4b5c103), LL(0x94d1ce03,0x14d9a43c),LL(0x5f767150,0xc940698c),LL(0xf3a40155,0xefe2f127),L_(0xbf13e662), + LL(0x9378bdc9,0xfbd86f1e),LL(0x50cbdd4e,0x38c7a9cc),LL(0x965226a4,0x434dbb0c),L_(0x83128949), LL(0xb9259729,0xc36bf7c6),LL(0x038090c2,0x0a12f12b),LL(0xb21834ca,0x0134eeee),L_(0xe2621ff2), + LL(0xad55e743,0xa30d570d),LL(0xfb77c395,0xdaaa178b),LL(0x525f0059,0xf38af4e6),L_(0xd340b6fe), LL(0xa001bba0,0xa05f60e2),LL(0x855efc1b,0x701418ea),LL(0xfd38f820,0x27e433c5),L_(0x0fbe1529), + LL(0xcf1f6418,0x3570870f),LL(0x7c6202e0,0x85875e04),LL(0x96e8ab61,0xefc6f006),L_(0xb0d48b1f), LL(0x0206e9ab,0x6f6540da),LL(0x32b7b29f,0x2a1b80df),LL(0x9caa872e,0x895193a8),L_(0xec63127d), + LL(0x40d4de6d,0x288f99a9),LL(0x6835f630,0x74a2cf89),LL(0xad2e6432,0x52ddf73d),L_(0x4185c6dc), LL(0xef978613,0xbb41a387),LL(0x95245fee,0x319f4bf6),LL(0x40e92907,0x9884b224),L_(0xaf8e6385), + LL(0xe2fde7bd,0x282179db),LL(0x7db516ce,0x7897f73f),LL(0x14a26c02,0x07ab717c),L_(0x34d1cdd0), LL(0xb3e012de,0xa65b35a3),LL(0xbb8667e0,0xb888ddc9),LL(0xc0879433,0x8819e222),L_(0xf68b3f79), + LL(0x32acb173,0x9f902cc1),LL(0xda89f30b,0x2ea4f418),LL(0xec378b71,0xd036aa23),L_(0x56712f20), LL(0x0f8fe475,0x3e16c057),LL(0xe703ddb6,0x134bd529),LL(0xee28b685,0x867d1b08),L_(0x0383ba62), + LL(0x49cdbaea,0x5c1f02d0),LL(0xa1775d0a,0xd83d7d9d),LL(0x1b0a0f31,0x2c287cd4),L_(0x80addda6), LL(0xf27c6df8,0x74534873),LL(0x5db066fa,0x5a696a14),LL(0x8118c158,0xfc06292e),L_(0x58371c8a), + LL(0x5e909157,0xbd1a69f7),LL(0xb7d52793,0x703cb9bd),LL(0x5a474912,0xcedd645a),L_(0x56b4d076), LL(0xdeeac8ac,0x2ed53772),LL(0x89595a18,0x5e6b86ab),LL(0xc7b731bc,0x5ce1f8a0),L_(0x03aec3d0), + LL(0xcc023bb6,0xdf3b4492),LL(0x796e6974,0x87e45b7d),LL(0x0ae46dae,0xe1f7eac7),L_(0x5971c51a), LL(0xb2f3733b,0x72718cd2),LL(0x3b6f5116,0x85e9b624),LL(0x626b0871,0xc20aa08f),L_(0x1ce0f83a), + LL(0x44129031,0x8b7e591b),LL(0x5325744a,0xc7753dac),LL(0xf476b49f,0x2fc4d6c3),L_(0xcc7c16ef), LL(0xe26efb19,0x4dd384fc),LL(0xf67b422f,0x478d3760),LL(0xc574295c,0x421851ef),L_(0xbccaa4b1), + LL(0x92253e02,0x5a61d65d),LL(0x6f301322,0xd4924563),LL(0x1e712886,0x2dd1532f),L_(0x3a89196d), LL(0xa3f1deda,0x39cdfbfc),LL(0x039de172,0xb32cde92),LL(0x57960c59,0x40ad0b76),L_(0xafd7ab04), + LL(0x33392ce0,0x52977c85),LL(0xacb473b0,0x147af936),LL(0x6dd913b5,0x4ec4b1c5),L_(0x838148b6), LL(0x699ab1e9,0x438d69df),LL(0xa416d1fe,0xdf94b69f),LL(0xbc60bb02,0x60dae203),L_(0xc32d87da), + LL(0x01cf54f4,0xc3cf28be),LL(0x609058a8,0xfa448ad2),LL(0xf8c1cf70,0x59baadb3),L_(0x67dc813f), LL(0xb0a595fb,0x60b75e6c),LL(0x1cfccbb7,0xb92101cb),LL(0x99833d0b,0x4be8d444),L_(0x91f3954c), + LL(0xedd8aca0,0xcb571bbf),LL(0x7257aad6,0x5a13acc6),LL(0x68203714,0x79914deb),L_(0x895f9c20), LL(0x5c3bbaec,0x211e58cd),LL(0xe9a0fbc0,0xdef6ac5d),LL(0x776cb363,0x93cb4804),L_(0x9678da2d), + LL(0x7d49d5cc,0x55e82125),LL(0xe0e89e2e,0xa13cee1f),LL(0x7e46156a,0x7d7e5719),L_(0xd361d2e2), LL(0x629c9b30,0x5fd28ac1),LL(0x70a24aea,0xdc393f34),LL(0x8ea6f6e3,0x289c68b1),L_(0xd6c4acd1), + LL(0x63596f58,0x31f49f27),LL(0xac40021b,0x7c86680a),LL(0x0b1f95fd,0x02fc0e10),L_(0x0d61fe85), LL(0x8f2843ce,0xca4cab38),LL(0xb53751a2,0xace069ce),LL(0xf56c80ff,0x270d9677),L_(0x9e5b52ac), + LL(0x6352c6f6,0x67e09224),LL(0x626c5d2d,0x5be06f9a),LL(0x14efc35d,0x38c6a0a2),L_(0x7169645d), LL(0xbe2d9aa4,0x6d0b11a9),LL(0x74e4964b,0x6c28404e),LL(0x5538c4c6,0x0af532d8),L_(0xfd9cc6b6), + LL(0x588f5f01,0xb4277028),LL(0xa788dcdd,0x2477f29d),LL(0x5fe5d31d,0x68c2db2a),L_(0xd8503d86), LL(0x2a5ebb9a,0x7b9c3f49),LL(0xc4b2edb7,0xbf734578),LL(0xbf266e3b,0xfe65a7a9),L_(0xc3945496), +}, +/* digit=32 base_pwr=2^224 */ +{ + LL(0xb7478767,0xeaa013ed),LL(0xa2868062,0x1cf3e561),LL(0xe8c99b44,0x142c4251),L_(0xc6f0cb81), LL(0x5ac149fc,0x9c824754),LL(0x8ff9a766,0x6b1233b1),LL(0xb9c09eeb,0xc5afba96),L_(0x95223aa6), + LL(0xb7c40cce,0xbf4d6687),LL(0xc187f375,0x88025f51),LL(0x80c953d4,0xe3dc6d83),L_(0x88d2d1bd), LL(0xb7668ae3,0x50faab59),LL(0x4404c1ec,0x235fc5b5),LL(0xac299c46,0x1378e170),L_(0x5d9a4686), + LL(0xb43ccb68,0x47792f74),LL(0xf564eef4,0x3dbeda1b),LL(0xb688499b,0x5b9b5ea1),L_(0x1e999e2b), LL(0x6f26a9f7,0xc4515542),LL(0xb3ca4e2f,0x8e4ebca0),LL(0x1f557c40,0xd6441259),L_(0x913b210d), + LL(0xea6ea3ca,0xc6c1108f),LL(0x564f45ee,0xcc83bf8e),LL(0x2c30408f,0x7aa17b0c),L_(0x5db1baaa), LL(0x06aed6b0,0xc003650d),LL(0x47b5af33,0x72d853fe),LL(0x66238f2e,0xf46a15a7),L_(0x76de62dc), + LL(0xf16a11ad,0xccc9dc6a),LL(0xb7c36e6e,0x71ab7d63),LL(0x54c72750,0x5a001c63),L_(0x0f8a90df), LL(0x226cb518,0xb6f26d84),LL(0xd8c32b73,0x3fc36946),LL(0x9f9ee30a,0xa1549b9d),L_(0x3080e1f3), + LL(0x4a2b2247,0xa65e1e3b),LL(0x9b2f8a35,0x671cf2f3),LL(0x6b9d08f0,0x162efd24),L_(0x62ae0f9c), LL(0x3316fc3f,0x84ae1987),LL(0x25516058,0x7d0dc886),LL(0xdbe094f5,0xc7b0730d),L_(0xb05c94c3), + LL(0x1bb62b06,0xe8e8411e),LL(0x20b9ca92,0xd192f8c2),LL(0xe45573f0,0xc1f3a5f2),L_(0x4f18bc0e), LL(0x26419175,0x6c1325f6),LL(0xe78ee486,0x2f81e000),LL(0xa91d228e,0x0ff7a9b2),L_(0x694a50a7), + LL(0x88e39d86,0x42262241),LL(0xe249830c,0x3ad07add),LL(0x918fdfa4,0xcb771a46),L_(0xe581c444), LL(0x18ba36b8,0xeb29d84d),LL(0x12fb54ee,0x1383d57f),LL(0xf57833f6,0xfda5e179),L_(0x2c61f244), + LL(0x47af468a,0x6f69c44b),LL(0xeb264605,0x5e4d4f9f),LL(0xd6821d55,0x46eb8384),L_(0xfdb05fbe), LL(0xcdeb889e,0x871d5809),LL(0xb09553d3,0x1029a1da),LL(0x5cf9245d,0x1bcc2546),L_(0x3a614385), + LL(0xfd8a4d1f,0xdb1de26b),LL(0xb08efe49,0x8aa6deaf),LL(0x12fee6da,0x1e010832),L_(0xb6e509a8), LL(0xd9702488,0x9e928c81),LL(0xf87b74dc,0x79014b39),LL(0x844adfba,0xa7623853),L_(0x14e2902c), + LL(0x2e0124e7,0x2b6dbd37),LL(0x0ff6ba21,0xcb4fc42b),LL(0xd7cdda0b,0xd5d8f543),L_(0x9ccc5b69), LL(0x485584af,0xdd6eb7f3),LL(0xbc744924,0x2e41e3f7),LL(0x65859cc9,0xeca30c9e),L_(0xe5e2bdd9), + LL(0xd5f8805e,0x66979e6c),LL(0xb1900794,0x1b2ea5ec),LL(0xf8ebb830,0x416a9eb2),L_(0x10b371a3), LL(0x6e8ad20d,0x15ff6e9d),LL(0x000e2330,0x0865593f),LL(0xee384b9b,0xe0eeda2c),L_(0x783dd95d), + LL(0x642f55b0,0xbfeb8487),LL(0xd5f0f813,0xfa3ad248),LL(0x7145f39b,0xd0f1abf1),L_(0xeb71b58b), LL(0x4defbbd6,0x893a0505),LL(0x52984b68,0xf3e2f26c),LL(0xf54c27a7,0x6f12b38f),L_(0xbfac39e5), + LL(0x5d26514b,0x7cad1aa3),LL(0xdd0afc7e,0x74faf86c),LL(0xfeab4314,0x645cca78),L_(0x5e53697c), LL(0x43f00056,0xb72096cb),LL(0xca87208e,0x7a1950ec),LL(0x071d954a,0x5337fbf2),L_(0xc371272c), + LL(0xb32000f7,0x5f6dea95),LL(0x5afebd30,0xf4446576),LL(0x19091ccf,0x74118cb1),L_(0xccebf405), LL(0x9210256e,0x9aacef2c),LL(0xb3458ddc,0x1ca9b6d7),LL(0xb1ab02c7,0xa870996c),L_(0x89a1b94b), + LL(0x308cbf80,0x552396fc),LL(0xab809f1d,0x7b2fda39),LL(0x4146f800,0xaebf97a3),L_(0xc90e51fe), LL(0x61fb1317,0xf1c9012c),LL(0x3d74201c,0x0142bd07),LL(0x9b191445,0xbab9da1b),L_(0x4bd6ed8b), + LL(0x5e508e4b,0xfa8ca29d),LL(0x2d86d22b,0x27a0f674),LL(0x4cfdd6de,0xee0206bb),L_(0x1e0e0bda), LL(0x978ed51f,0xa6bbc41f),LL(0xb73521ec,0xe52c7866),LL(0x006c7072,0x32478f51),L_(0xb4b93178), + LL(0x217f1203,0x04851e7d),LL(0x8feb118b,0xd7767c08),LL(0x75b3d632,0x4c08238d),L_(0xac7d86c6), LL(0x3858c44f,0xcd97e8ec),LL(0xb74f0fe2,0x62318c46),LL(0x93355b9b,0xe2405794),L_(0x54c12d79), + LL(0xde835c5e,0x92a463c3),LL(0x0228dc25,0x3b10b96a),LL(0x4b4acd48,0x5f5bd93b),L_(0x1c91a329), LL(0xf537b956,0xedc9c8f4),LL(0xc7fcc358,0xf2362c8d),LL(0xe58a0000,0x970cf697),L_(0x2b1ee7ed), + LL(0x8e7b172a,0x882e0b3e),LL(0xb8bd5ea9,0xbf2d6013),LL(0x490fbccd,0xc0848551),L_(0x3ffa1cc7), LL(0x73f7a7d9,0x17b850e8),LL(0x3a53ec82,0xfcc68ded),LL(0x9cbe988b,0x8d09bdd2),L_(0xa3b6e95e), + LL(0x6266d7aa,0x7f0f6292),LL(0x329716ed,0x47c40355),LL(0x1cfc61e3,0x7a59e7d3),L_(0x0ffa8f3d), LL(0xbddfbabc,0x84b7a613),LL(0x36121ac1,0x89396fd0),LL(0x0f5147ce,0x2b7f355a),L_(0xb3193eff), + LL(0x8af99520,0x4d2ceb91),LL(0x15dc69a8,0xcc451e87),LL(0x6d59d3b5,0xc116fdef),L_(0x627d226f), LL(0x79ead74e,0x813a5852),LL(0x5daf904b,0xc12ffcbc),LL(0xcd67731a,0x3fdf17e1),L_(0x2df27ce8), + LL(0x764582de,0xefaccc05),LL(0x640388d0,0x63018ab5),LL(0x6f606211,0x6b706f5e),L_(0xa1c500d6), LL(0x26635840,0x2a01814c),LL(0x8ab7ad6f,0xe29eede6),LL(0x756a8f01,0x10f5f095),L_(0x53630671), + LL(0xa38d34fa,0x591f8fe0),LL(0xf829c6c6,0xc31b8774),LL(0xae2b4a7d,0x55de9380),L_(0xc9c227e7), LL(0x63877738,0x566cf0c3),LL(0xd9750372,0x1ad0462b),LL(0x5c1a2a83,0xcfd11aad),L_(0x2aff32de), + LL(0xbc66bc8f,0x9f39feb3),LL(0x24b329eb,0x21f0dec8),LL(0xd2ff18e6,0x99e3e6c4),L_(0xb398626b), LL(0xcce55b7f,0xf3fabc3c),LL(0xb9a016fd,0x5c869423),LL(0x4f7a9f81,0xc660df04),L_(0xf74d1142), + LL(0x302b25d7,0xfbdef76c),LL(0xb2090c70,0x97f6ff80),LL(0xcc0decf0,0x26e1f975),L_(0x2797ff76), LL(0x4f37e255,0x61eee678),LL(0x51e68a0e,0x6a0d8fc5),LL(0xd84014df,0xb7e3a2d4),L_(0x449bf740), + LL(0xde3d73e8,0x3178216e),LL(0x90f08891,0x2f89efcd),LL(0xdcbe0246,0xb937b3c2),L_(0x557e9fd5), LL(0xacaa30d0,0x7f06975e),LL(0xd99696a9,0x714b6101),LL(0xd23f9579,0xc1321142),L_(0x8d2afa4b), + LL(0x8271dfd1,0x6e2e57bd),LL(0xc1ea81b3,0xec736ab8),LL(0xbcfb4116,0x1cc014d1),L_(0xc3f8311f), LL(0x02178016,0xbcc0c7bd),LL(0x3222341c,0xdf2643e7),LL(0xedc952bb,0x36ab26fe),L_(0xa84ff692), + LL(0xc562a325,0xccd2af21),LL(0x1f595cc1,0x8cbd4ea2),LL(0x9f800082,0x7bbad5e0),L_(0x89fbecab), LL(0xbfd65f55,0xfa87a290),LL(0x24ee0ca1,0xd25d631b),LL(0xe063926d,0x0f3d4539),L_(0xac0263f9), + LL(0x6bf5b025,0x1a63c568),LL(0xbec29ce6,0x7bad1c42),LL(0x62df4304,0x38b54b11),L_(0x2b705d62), LL(0x94995153,0x7f7b64f5),LL(0x35d2b40a,0xd25d1128),LL(0xbdbbd673,0x4703bb51),L_(0x682e3058), + LL(0x2509ad1b,0x9885b7f4),LL(0xc8c1117b,0xf70447a6),LL(0x9bfc053a,0x1b77852c),L_(0xcfb5f714), LL(0x7dc9ba15,0xa43807b7),LL(0xaea633e3,0x02c669a5),LL(0x220e1871,0x991d8093),L_(0x508f55e1), + LL(0xc434b864,0x6c74aee9),LL(0x96d76b48,0x072efeda),LL(0x19982c91,0xac91d19f),L_(0x79df6582), LL(0xbc64a3c7,0xcea49a1b),LL(0x6b3b3cac,0xe25c1282),LL(0x37f9ab3c,0x57709b27),L_(0x404ec7c9), + LL(0x63a0bab8,0xbdede240),LL(0x702a4776,0x8ec0a030),LL(0xbbb0cddc,0xf6b96225),L_(0xbdd56c73), LL(0x3dff08cb,0xf46b51e6),LL(0xac5d2e41,0xfac7173a),LL(0x3f2c5530,0xe5090eda),L_(0x05b966b5), + LL(0x585b98ee,0x4cb0c03a),LL(0x859d7492,0xb6dc9a2c),LL(0x126ee071,0x3331172f),L_(0xa1e73b0d), LL(0xba3aa73a,0x71ac411d),LL(0x6d7dee6b,0x816a4f1a),LL(0xa88c4198,0xe409dbda),L_(0xa924723a), + LL(0xccb6c49e,0xa20ba7c3),LL(0x46da89e3,0x6782fea2),LL(0x83b7b5ce,0xf28c836e),L_(0xa02bd753), LL(0x64f82fd2,0x523afba8),LL(0x3203549e,0x024ddda9),LL(0x65d29391,0x4cec16d6),L_(0xb0187244), + LL(0xaff84dda,0x1594f62b),LL(0x4bbabfdf,0xa8de453c),LL(0xeb65c1b7,0x30a103ff),L_(0x02fb469c), LL(0xf76afad8,0xa91cd374),LL(0x98a39a41,0xa33359fd),LL(0x0b99c0c0,0x9015867f),L_(0x96fc381a), + LL(0xb366fcc8,0xdde98289),LL(0x76693b42,0xe45737f9),LL(0x30c2ee70,0xa6bb364a),L_(0x759a9e66), LL(0xe00c866b,0x8177a004),LL(0x144b5ff3,0x49205953),LL(0x0106e5ac,0x5d4c1327),L_(0x98001111), + LL(0xdebcecc8,0xfdc46d5f),LL(0xc9916314,0xa125951b),LL(0x5633f2c4,0x04c2afc1),L_(0xd17c9349), LL(0xacd14df9,0xb5f970f1),LL(0x289012ed,0x0973ccd0),LL(0x236b3580,0x02b2c84a),L_(0x76b898cf), + LL(0xcb13811d,0x0d2dd5a3),LL(0x23f6e444,0xd8b5d0f6),LL(0x4f399d9a,0xa46887b9),L_(0x77c6bce8), LL(0x10165161,0xddf54326),LL(0xeb345000,0xf080be9b),LL(0xeee34275,0x047237fe),L_(0xd518ca88), + LL(0x11fceb12,0xfaf4fbdf),LL(0x116a3791,0x66bfd60d),LL(0x442c2a46,0x28cf0358),L_(0x85546f76), LL(0x4cebfe25,0x6c01d714),LL(0x1c79b699,0x11d66f5a),LL(0x8cc29c26,0xcf7771ef),L_(0x92ae8875), + LL(0xf90d16bf,0xb89933d4),LL(0xa1457028,0xc8f0695e),LL(0x5a6af165,0x82274dbf),L_(0x89528271), LL(0x225b8bb3,0x5e138d61),LL(0x2eed19b9,0x3e2273d0),LL(0x6b42f9f3,0xaa4b331c),L_(0xe7d915a6), + LL(0xa9d80f3e,0x6773ac5a),LL(0xc52d7955,0x970b6066),LL(0x78700052,0x2f129047),L_(0xa9a60bf2), LL(0x0393eefb,0x04368569),LL(0xe6519a05,0x5180fe6e),LL(0x3014d2a0,0x6bd99185),L_(0xb28fcd33), + LL(0x299011de,0xe9270a23),LL(0x5a66907e,0x54a9528a),LL(0x7af57c36,0x8e613981),L_(0xe6775c8b), LL(0xff5fc082,0xd7dd7f17),LL(0xe0568a5c,0x421cb6c4),LL(0x590b13ce,0x8f9e6852),L_(0x4c77050e), + LL(0x6513f9c9,0xd5bf1a5d),LL(0x170fc778,0x1942fc9b),LL(0x6ec5f13f,0x70273957),L_(0x1878fa8e), LL(0x5bb80bbc,0xcededd82),LL(0x83d79890,0xa7c7ee32),LL(0xfa06b05d,0x86a8080f),L_(0x0e66721a), + LL(0x9510adb1,0x8a998afb),LL(0x7aa85880,0xfaf28d30),LL(0x0863e4f3,0xdcf64ed3),L_(0x1fe9103e), LL(0x1ea72a10,0x78d5ce7b),LL(0x33edcfdb,0xafe82b0c),LL(0xde25a969,0x1f0a50fe),L_(0x5f4147d4), + LL(0x3d5e21a2,0xddb5610d),LL(0x8024cc9b,0x435cd0ff),LL(0xcc34b698,0xf7218db6),L_(0xb6949149), LL(0xc23a0cb8,0x3b787bdf),LL(0x18dd349c,0x71fcc010),LL(0xb0e2a897,0xd6c69c33),L_(0x09bc9f85), + LL(0x1cd5d5db,0xe94a402b),LL(0xf30b7314,0x26944f05),LL(0x737d6c5b,0xd0af2018),L_(0xfdbedbb6), LL(0x1b523221,0xa8a984b3),LL(0x8ace709d,0x39663eef),LL(0xbf4b3e0e,0x0297dfd1),L_(0x3e1d52f4), + LL(0xc8d20d14,0xf915a4c4),LL(0x0b7dd701,0x2921fb0f),LL(0x6b467b73,0x226421aa),L_(0xf53a0a69), LL(0xf4bdad8d,0xb5512d8c),LL(0xc103b145,0x2df92000),LL(0x4979cf5a,0xd5159856),L_(0x0d121a5d), + LL(0xffb25429,0xad4ff433),LL(0xe3c5fc0a,0x22c48ae5),LL(0xdcdd2832,0xee468cb2),L_(0x853d6571), LL(0xad56cbbd,0x1ac2be8d),LL(0x305ea1a5,0x11bdfbf2),LL(0x4c080ea0,0xd326fcc1),L_(0x98961d03), + LL(0x3c2a21b3,0x4c900a2e),LL(0x370aac6c,0x540b6e6c),LL(0x40c9b00d,0xf0db08ca),L_(0xd9b48af0), LL(0x75ea8b19,0x23e0167d),LL(0x69f68546,0x3ce76be7),LL(0xc2578bd9,0x274ef866),L_(0xe0f171b1), + LL(0x06e96b08,0xa0cc2074),LL(0x658170a1,0x2181d377),LL(0xbc50ac87,0x46b62277),L_(0xd2d87843), LL(0x66a073ef,0xbc771f86),LL(0xef91a0af,0x4727577f),LL(0xa7216ad9,0xd3c1c667),L_(0x3c62c22f), + LL(0x37e5b8f5,0xfea02084),LL(0x4baddc8e,0xe9a5f0b7),LL(0xbf587847,0x53933d2b),L_(0xca648fb8), LL(0x492b64e4,0x8257b251),LL(0x0f117e6d,0x8d9ec843),LL(0xfe9686a0,0xeeea3972),L_(0xe386bb0a), + LL(0xb93a8f56,0xf5712604),LL(0xa696fd94,0x6981acc8),LL(0xf73a981c,0x86cc6e25),L_(0x53db10f5), LL(0xa6c75879,0x18cf2a94),LL(0xe6e2104d,0x5737dd4f),LL(0x4c1f9f0e,0x3114afeb),L_(0x2e6f8e44), + LL(0xbbc2c60f,0x19e92b39),LL(0x06b78031,0xb3de20d1),LL(0x43d40348,0x693597af),L_(0x1396fecd), LL(0xa7c941f2,0x4568fdc8),LL(0xd45e157f,0x92d2afec),LL(0x50493b74,0xa510adc1),L_(0x64794afe), + LL(0x60867e2f,0xab4d38b7),LL(0xf04773c1,0x0b8a018e),LL(0x2c774449,0xc11883d8),L_(0xd65b4aa4), LL(0xa50bce8a,0x9bcfd81b),LL(0xfd058e49,0xf5811bc4),LL(0x80f1e9c2,0xe9b876e5),L_(0xc8fc1b3c), + LL(0x368a0742,0x7cf8b51b),LL(0x09e82684,0x85f0eb37),LL(0xb5926249,0x08e70a34),L_(0x4decfe48), LL(0xe9a1940e,0x4d61fee9),LL(0x6dcb9563,0x24af91e7),LL(0xa85a5970,0x41f4907d),L_(0x2292e826), + LL(0x33182546,0x7b2ef570),LL(0xb4417ce0,0x2a780100),LL(0x8d1180df,0xab954cb9),L_(0x5c74a302), LL(0x88d81acf,0x0d62a965),LL(0x2ec9496d,0xaa16f924),LL(0x2bc7cca4,0x2d99dcc8),L_(0x74958dce), + LL(0xb139570a,0x21c56351),LL(0x1f26d583,0x41b20f4e),LL(0x067af3fb,0xbd9ab15f),L_(0xc3a77e33), LL(0x1523f3e0,0x9de5e7e4),LL(0xe1731bb7,0xef0cf37e),LL(0x6c9e48d4,0xc9fc2851),L_(0xd0f91844), + LL(0xb81124f1,0xea0ed825),LL(0x788a57ed,0xf4f08526),LL(0x8e19f5c1,0x820a4f79),L_(0x348a1346), LL(0x6eaedecc,0xefe607bc),LL(0x87c20db0,0x75968ada),LL(0x21596aa9,0xcdf9ca14),L_(0x5e2abcad), + LL(0xdaef28c3,0x2be4550b),LL(0x29e6daee,0xee673998),LL(0x8162c783,0x4e1fb36c),L_(0xb3249498), LL(0xab7e3386,0xb682150c),LL(0x89285111,0x02a0814e),LL(0x9c692620,0x3bb22aea),L_(0xa759126d), + LL(0xa03f91a6,0xd356a2bd),LL(0x7b288f41,0x04625b59),LL(0xf350f847,0x2eacfb01),L_(0x943dbfd9), LL(0x63af93f5,0xba005c26),LL(0xdcc822a7,0x7964a6f8),LL(0xc56edb4d,0x14b146d7),L_(0xb1fb9d62), + LL(0xcae5ffde,0x51fbb8f7),LL(0xca0c3f68,0xa961e967),LL(0xbe43f447,0x2a774b0b),L_(0x9b6effff), LL(0x4e174bf0,0xb12fde0a),LL(0xd1976ab8,0x85b59e4e),LL(0x421eac26,0x4c480a51),L_(0x4041c2d8), + LL(0xc3a1a863,0xfe8ceef6),LL(0x42e96299,0x0469540a),LL(0xd231a2a4,0x0828f4c0),L_(0x7c0edcf4), LL(0x2c034efb,0x1744d906),LL(0x92393260,0x2a85a8d6),LL(0x28f5775a,0x58717292),L_(0x37693735), + LL(0xf12dc792,0xbcc5e464),LL(0xfee1ef55,0xb881c818),LL(0x2bfca9a4,0x7e60531f),L_(0xed54d832), LL(0x28ea4ef0,0x9ef6ada6),LL(0xb9564865,0xf56fbcf0),LL(0x534e2057,0x9857c41f),L_(0x2750bbe8), +} +}; +#endif + +#endif /* _DISABLE_ECP_224R1_HARDCODED_BP_TBL_ */ +#endif /* _IPP_DATA */ + + +IPP_OWN_DEFN (const cpPrecompAP*, gfpec_precom_nistP224r1_fun, (void)) +{ + static cpPrecompAP t = { + /* w */ 7, + /* select function */ p224r1_select_ap_w7, + /* precomputed data */ (BNU_CHUNK_T*)ec_p224r1_precomputed + }; + return &t; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpp256r1precomca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpp256r1precomca.c new file mode 100644 index 0000000..b7fc21b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpp256r1precomca.c @@ -0,0 +1,2540 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (P256r1 precomputed) +// +// +*/ +#include "owncp.h" +#include "pcpgfpecstuff.h" + + +#define OPERAND_BITSIZE (256) +#define LEN_P256 (BITS_BNU_CHUNK(OPERAND_BITSIZE)) + +/* P256 affine point */ +typedef struct{ + BNU_CHUNK_T X[LEN_P256]; + BNU_CHUNK_T Y[LEN_P256]; +} P256_POINT_AFFINE; + +extern const __ALIGN64 P256_POINT_AFFINE ec_p256r1_precomputed[37][64]; + + +#if defined( _IPP_DATA ) + +#if !defined(_DISABLE_ECP_256R1_HARDCODED_BP_TBL_) +/* see ippcp_baseptbl.cpp test for generation details */ + +const __ALIGN64 P256_POINT_AFFINE ec_p256r1_precomputed[37][64] = { +/* digit=0 base_pwr=2^0 */ +{ + LL(0x18a9143c,0x79e730d4),LL(0x5fedb601,0x75ba95fc),LL(0x77622510,0x79fb732b),LL(0xa53755c6,0x18905f76), LL(0xce95560a,0xddf25357),LL(0xba19e45c,0x8b4ab8e4),LL(0xdd21f325,0xd2e88688),LL(0x25885d85,0x8571ff18), + LL(0x10ddd64d,0x850046d4),LL(0xa433827d,0xaa6ae3c1),LL(0x8d1490d9,0x73220503),LL(0x3dcf3a3b,0xf6bb32e4), LL(0x61bee1a5,0x2f3648d3),LL(0xeb236ff8,0x152cd7cb),LL(0x92042dbe,0x19a8fb0e),LL(0x0a5b8a3b,0x78c57751), + LL(0x4eebc127,0xffac3f90),LL(0x087d81fb,0xb027f84a),LL(0x87cbbc98,0x66ad77dd),LL(0xb6ff747e,0x26936a3f), LL(0xc983a7eb,0xb04c5c1f),LL(0x0861fe1a,0x583e47ad),LL(0x1a2ee98e,0x78820831),LL(0xe587cc07,0xd5f06a29), + LL(0x46918dcc,0x74b0b50d),LL(0xc623c173,0x4650a6ed),LL(0xe8100af2,0x0cdaacac),LL(0x41b0176b,0x577362f5), LL(0xe4cbaba6,0x2d96f24c),LL(0xfad6f447,0x17628471),LL(0xe5ddd22e,0x6b6c36de),LL(0x4c5ab863,0x84b14c39), + LL(0xc45c61f5,0xbe1b8aae),LL(0x94b9537d,0x90ec649a),LL(0xd076c20c,0x941cb5aa),LL(0x890523c8,0xc9079605), LL(0xe7ba4f10,0xeb309b4a),LL(0xe5eb882b,0x73c568ef),LL(0x7e7a1f68,0x3540a987),LL(0x2dd1e916,0x73a076bb), + LL(0x3e77664a,0x40394737),LL(0x346cee3e,0x55ae744f),LL(0x5b17a3ad,0xd50a961a),LL(0x54213673,0x13074b59), LL(0xd377e44b,0x93d36220),LL(0xadff14b5,0x299c2b53),LL(0xef639f11,0xf424d44c),LL(0x4a07f75f,0xa4c9916d), + LL(0xa0173b4f,0x0746354e),LL(0xd23c00f7,0x2bd20213),LL(0x0c23bb08,0xf43eaab5),LL(0xc3123e03,0x13ba5119), LL(0x3f5b9d4d,0x2847d030),LL(0x5da67bdd,0x6742f2f2),LL(0x77c94195,0xef933bdc),LL(0x6e240867,0xeaedd915), + LL(0x9499a78f,0x27f14cd1),LL(0x6f9b3455,0x462ab5c5),LL(0xf02cfc6b,0x8f90f02a),LL(0xb265230d,0xb763891e), LL(0x532d4977,0xf59da3a9),LL(0xcf9eba15,0x21e3327d),LL(0xbe60bbf0,0x123c7b84),LL(0x7706df76,0x56ec12f2), + LL(0x264e20e8,0x75c96e8f),LL(0x59a7a841,0xabe6bfed),LL(0x44c8eb00,0x2cc09c04),LL(0xf0c4e16b,0xe05b3080), LL(0xa45f3314,0x1eb7777a),LL(0xce5d45e3,0x56af7bed),LL(0x88b12f1a,0x2b6e019a),LL(0xfd835f9b,0x086659cd), + LL(0x9dc21ec8,0x2c18dbd1),LL(0x0fcf8139,0x98f9868a),LL(0x48250b49,0x737d2cd6),LL(0x24b3428f,0xcc61c947), LL(0x80dd9e76,0x0c2b4078),LL(0x383fbe08,0xc43a8991),LL(0x779be5d2,0x5f7d2d65),LL(0xeb3b4ab5,0x78719a54), + LL(0x6245e404,0xea7d260a),LL(0x6e7fdfe0,0x9de40795),LL(0x8dac1ab5,0x1ff3a415),LL(0x649c9073,0x3e7090f1), LL(0x2b944e88,0x1a768561),LL(0xe57f61c8,0x250f939e),LL(0x1ead643d,0x0c0daa89),LL(0xe125b88e,0x68930023), + LL(0xd2697768,0x04b71aa7),LL(0xca345a33,0xabdedef5),LL(0xee37385e,0x2409d29d),LL(0xcb83e156,0x4ee1df77), LL(0x1cbb5b43,0x0cac12d9),LL(0xca895637,0x170ed2f6),LL(0x8ade6d66,0x28228cfa),LL(0x53238aca,0x7ff57c95), + LL(0x4b2ed709,0xccc42563),LL(0x856fd30d,0x0e356769),LL(0x559e9811,0xbcbcd43f),LL(0x5395b759,0x738477ac), LL(0xc00ee17f,0x35752b90),LL(0x742ed2e3,0x68748390),LL(0xbd1f5bc1,0x7cd06422),LL(0xc9e7b797,0xfbc08769), + LL(0xb0cf664a,0xa242a35b),LL(0x7f9707e3,0x126e48f7),LL(0xc6832660,0x1717bf54),LL(0xfd12c72e,0xfaae7332), LL(0x995d586b,0x27b52db7),LL(0x832237c2,0xbe29569e),LL(0x2a65e7db,0xe8e4193e),LL(0x2eaa1bbb,0x152706dc), + LL(0xbc60055b,0x72bcd8b7),LL(0x56e27e4b,0x03cc23ee),LL(0xe4819370,0xee337424),LL(0x0ad3da09,0xe2aa0e43), LL(0x6383c45d,0x40b8524f),LL(0x42a41b25,0xd7663554),LL(0x778a4797,0x64efa6de),LL(0x7079adf4,0x2042170a), + LL(0x0bc6fb80,0x808b0b65),LL(0x3ffe2e6b,0x5882e075),LL(0x2c83f549,0xd5ef2f7c),LL(0x9103b723,0x54d63c80), LL(0x52a23f9b,0xf2f11bd6),LL(0x4b0b6587,0x3670c319),LL(0xb1580e9e,0x55c4623b),LL(0x01efe220,0x64edf7b2), + LL(0xd53c5c9d,0x97091dcb),LL(0xac0a177b,0xf17624b6),LL(0x2cfe2dff,0xb0f13975),LL(0x6c7a574e,0xc1a35c0a), LL(0x93e79987,0x227d3146),LL(0xe89cb80e,0x0575bf30),LL(0x0d1883bb,0x2f4e247f),LL(0x3274c3d0,0xebd51226), + LL(0x56ada97a,0x5f3e51c8),LL(0x8f8b403e,0x4afc964d),LL(0x412e2979,0xa6f247ab),LL(0x6f80ebda,0x675abd1b), LL(0x5e485a1d,0x66a2bd72),LL(0x8f4f0b3c,0x4b2a5caf),LL(0x1b847bba,0x2626927f),LL(0x0502394d,0x6c6fc7d9), + LL(0xa5659ae8,0xfea912ba),LL(0x25e1a16e,0x68363aba),LL(0x752c41ac,0xb8842277),LL(0x2897c3fc,0xfe545c28), LL(0xdc4c696b,0x2d36e9e7),LL(0xfba977c5,0x5806244a),LL(0xe39508c1,0x85665e9b),LL(0x6d12597b,0xf720ee25), + LL(0xd2337a31,0x8a979129),LL(0x0f862bdc,0x5916868f),LL(0x5dd283ba,0x048099d9),LL(0xfe5bfb4e,0xe2d1eeb6), LL(0x7884005d,0x82ef1c41),LL(0xffffcbae,0xa2d4ec17),LL(0x8aa95e66,0x9161c53f),LL(0xc5fee0d0,0x5ee104e1), + LL(0xc135b208,0x562e4cec),LL(0x4783f47d,0x74e1b265),LL(0x5a3f3b30,0x6d2a506c),LL(0xc16762fc,0xecead9f4), LL(0xe286e5b9,0xf29dd4b2),LL(0x83bb3c61,0x1b0fadc0),LL(0x7fac29a4,0x7a75023e),LL(0xc9477fa3,0xc086d5f1), + LL(0x2f6f3076,0x0fc61135),LL(0xe3912a9a,0xc99ffa23),LL(0xd2f8ba3d,0x6a0b0685),LL(0xe93358a4,0xfdc777e8), LL(0x35415f04,0x94a787bb),LL(0x4d23fea4,0x640c2d6a),LL(0x153a35b5,0x9de917da),LL(0x5d5cd074,0x793e8d07), + LL(0x2de45068,0xf4f87653),LL(0x9e2e1f6e,0x37c7a7e8),LL(0xa3584069,0xd0825fa2),LL(0x1727bf42,0xaf2cea7c), LL(0x9e4785a9,0x0360a4fb),LL(0x27299f4a,0xe5fda49c),LL(0x71ac2f71,0x48068e13),LL(0x9077666f,0x83d0687b), + LL(0x15d02819,0x6d3883b2),LL(0x40dd9a35,0x6d0d7550),LL(0x1d2b469f,0x61d7cbf9),LL(0x2efc3115,0xf97b232f), LL(0xb24bcbc7,0xa551d750),LL(0x88a1e356,0x11ea4949),LL(0x93cb7501,0x7669f031),LL(0xca737b8a,0x595dc55e), + LL(0xd837879f,0xa4a319ac),LL(0xed6b67b0,0x6fc1b49e),LL(0x32f1f3af,0xe3959933),LL(0x65432a2e,0x966742eb), LL(0xb4966228,0x4b8dc9fe),LL(0x43f43950,0x96cc6312),LL(0xc9b731ee,0x12068859),LL(0x56f79968,0x7b948dc3), + LL(0xed1f8008,0x61e4ad32),LL(0xd8b17538,0xe6c9267a),LL(0x857ff6fb,0x1ac7c5eb),LL(0x55f2fb10,0x994baaa8), LL(0x1d248018,0x84cf14e1),LL(0x628ac508,0x5a39898b),LL(0x5fa944f5,0x14fde97b),LL(0xd12e5ac7,0xed178030), + LL(0x97e2feb4,0x042c2af4),LL(0xaebf7313,0xd36a42d7),LL(0x084ffdd7,0x49d2c9eb),LL(0x2ef7c76a,0x9f8aa54b), LL(0x09895e70,0x9200b7ba),LL(0xddb7fb58,0x3bd0c66f),LL(0x78eb4cbb,0x2d97d108),LL(0xd84bde31,0x2d431068), + LL(0x172ccd1f,0x4b523eb7),LL(0x30a6a892,0x7323cb28),LL(0xcfe153eb,0x97082ec0),LL(0xf2aadb97,0xe97f6b6a), LL(0xd1a83da1,0x1d3d393e),LL(0x804b2a68,0xa6a7f9c7),LL(0x2d0cb71e,0x4a688b48),LL(0x40585278,0xa9b4cc5f), + LL(0xcb66e132,0x5e5db46a),LL(0x0d925880,0xf1be963a),LL(0x0317b9e2,0x944a7027),LL(0x48603d48,0xe266f959), LL(0x5c208899,0x98db6673),LL(0xa2fb18a3,0x90472447),LL(0x777c619f,0x8a966939),LL(0x2a3be21b,0x3798142a), + LL(0x3298b343,0xb4241cb1),LL(0xb44f65a1,0xa3a14e49),LL(0x3ac77acd,0xc5f4d6cd),LL(0x52b6fc3c,0xd0288cb5), LL(0x1c040abc,0xd5cc8c2f),LL(0x06bf9b4a,0xb675511e),LL(0x9b3aa441,0xd667da37),LL(0x51601f72,0x460d45ce), + LL(0x6755ff89,0xe2f73c69),LL(0x473017e6,0xdd3cf7e7),LL(0x3cf7600d,0x8ef5689d),LL(0xb1fc87b4,0x948dc4f8), LL(0x4ea53299,0xd9e9fe81),LL(0x98eb6028,0x2d921ca2),LL(0x0c9803fc,0xfaecedfd),LL(0x4d7b4745,0xf38ae891), + LL(0xc5e3a3d8,0xd8c5fccf),LL(0x4079dfbf,0xbefd904c),LL(0xfead0197,0xbc6d6a58),LL(0x695532a4,0x39227077), LL(0xdbef42f5,0x09e23e6d),LL(0x480a9908,0x7e449b64),LL(0xad9a2e40,0x7b969c1a),LL(0x9591c2a4,0x6231d792), + LL(0x0f664534,0x87151456),LL(0x4b68f103,0x85ceae7c),LL(0x65578ab9,0xac09c4ae),LL(0xf044b10c,0x33ec6868), LL(0x3a8ec1f1,0x6ac4832b),LL(0x5847d5ef,0x5509d128),LL(0x763f1574,0xf909604f),LL(0xc32f63c4,0xb16c4303), + LL(0x7ca23cd3,0xb6ab2014),LL(0xa391849d,0xcaa7a5c6),LL(0x75678d94,0x5b0673a3),LL(0xdd303e64,0xc982ddd4), LL(0x5db6f971,0xfd7b000b),LL(0x6f876f92,0xbba2cb1f),LL(0x3c569426,0xc77332a3),LL(0x570d74f8,0xa159100c), + LL(0xdec67ef5,0xfd16847f),LL(0x233e76b7,0x742ee464),LL(0xefc2b4c8,0x0b8e4134),LL(0x42a3e521,0xca640b86), LL(0x8ceb6aa9,0x653a0190),LL(0x547852d5,0x313c300c),LL(0x6b237af7,0x24e4ab12),LL(0x8bb47af8,0x2ba90162), + LL(0xa8219bb7,0x3d5e58d6),LL(0x1b06c57f,0xc691d0bd),LL(0xd257576e,0x0ae4cb10),LL(0xd54a3dc3,0x3569656c), LL(0x94cda03a,0xe5ebaebd),LL(0x162bfe13,0x934e82d3),LL(0xe251a0c6,0x450ac0ba),LL(0xdd6da526,0x480b9e11), + LL(0x8cce08b5,0x00467bc5),LL(0x7f178d55,0xb636458c),LL(0xa677d806,0xc5748bae),LL(0xdfa394eb,0x2763a387), LL(0x7d3cebb6,0xa12b448a),LL(0x6f20d850,0xe7adda3e),LL(0x1558462c,0xf63ebce5),LL(0x620088a8,0x58b36143), + LL(0x4d63c0ee,0x8a2cc3ca),LL(0x0fe948ce,0x51233117),LL(0x222ef33b,0x7463fd85),LL(0x7c603d6c,0xadf0c7dc), LL(0xfe7765e5,0x0ec32d3b),LL(0xbf380409,0xccaab359),LL(0x8e59319c,0xbdaa84d6),LL(0x9c80c34d,0xd9a4c280), + LL(0xa059c142,0xa9d89488),LL(0xff0b9346,0x6f5ae714),LL(0x16fb3664,0x068f237d),LL(0x363186ac,0x5853e4c4), LL(0x63c52f98,0xe2d87d23),LL(0x81828876,0x2ec4a766),LL(0xe14e7b1c,0x47b864fa),LL(0x69192408,0x0c0bc0e5), + LL(0xb82e9f3e,0xe4d7681d),LL(0xdf25e13c,0x83200f0b),LL(0x66f27280,0x8909984c),LL(0x75f73227,0x462d7b00), LL(0xf2651798,0xd90ba188),LL(0x36ab1c34,0x74c6e18c),LL(0x5ef54359,0xab256ea3),LL(0xd1aa702f,0x03466612), + LL(0x2ed22e91,0x624d6049),LL(0x6f072822,0x6fdfe0b5),LL(0x39ce2271,0xeeca1115),LL(0xdb01614f,0x98100a4f), LL(0xa35c628f,0xb6b0daa2),LL(0xc87e9a47,0xb6f94d2e),LL(0x1d57d9ce,0xc6773259),LL(0x03884a7b,0xf70bfeec), + LL(0xed2bad01,0x5fb35ccf),LL(0x1da6a5c7,0xa155cbe3),LL(0x30a92f8f,0xc2e2594c),LL(0x5bfafe43,0x649c89ce), LL(0xe9ff257a,0xd158667d),LL(0xf32c50ae,0x9b359611),LL(0x906014cf,0x4b00b20b),LL(0x89bc7d3d,0xf3a8cfe3), + LL(0x248a7d06,0x4ff23ffd),LL(0x878873fa,0x80c5bfb4),LL(0x05745981,0xb7d9ad90),LL(0x3db01994,0x179c85db), LL(0x61a6966c,0xba41b062),LL(0xeadce5a8,0x4d82d052),LL(0xa5e6a318,0x9e91cd3b),LL(0x95b2dda0,0x47795f4f), + LL(0xd55a897c,0xecfd7c1f),LL(0xb29110fb,0x009194ab),LL(0xe381d3b0,0x5f0e2046),LL(0xa98dd291,0x5f3425f6), LL(0x730d50da,0xbfa06687),LL(0x4b083b7f,0x0423446c),LL(0xd69d3417,0x397a247d),LL(0x387ba42a,0xeb629f90), + LL(0xd5cd79bf,0x1ee426cc),LL(0x946c6e18,0x0032940b),LL(0x57477f58,0x1b1e8ae0),LL(0x6d823278,0xe94f7d34), LL(0x782ba21a,0xc747cb96),LL(0xf72b33a5,0xc5254469),LL(0xc7f80c81,0x772ef6de),LL(0x2cd9e6b5,0xd73acbfe), + LL(0x49ee90d9,0x4075b5b1),LL(0xa06e9eba,0x785c339a),LL(0xabf825e0,0xa1030d5b),LL(0xa42931dc,0xcec684c3), LL(0xc1586e63,0x42ab62c9),LL(0x5ab43f2b,0x45431d66),LL(0x55f7835d,0x57c8b2c0),LL(0xc1b7f865,0x033da338), + LL(0xcaa76097,0x283c7513),LL(0x36c83906,0x0a624fa9),LL(0x715af2c7,0x6b20afec),LL(0xeba78bfd,0x4b969974), LL(0xd921d60e,0x220755cc),LL(0x7baeca13,0x9b944e10),LL(0x5ded93d4,0x04819d51),LL(0x6dddfd27,0x9bbff86e), + LL(0x77adc612,0x6b344130),LL(0xbbd803a0,0xa7496529),LL(0x6d8805bd,0x1a1baaa7),LL(0x470343ad,0xc8403902), LL(0x175adff1,0x39f59f66),LL(0xb7d8c5b7,0x0b26d7fb),LL(0x529d75e3,0xa875f5ce),LL(0x41325cc2,0x85efc7e9), + LL(0x1ff6acd3,0x21950b42),LL(0x53dc6909,0xffe70484),LL(0x28766127,0xff4cd0b2),LL(0x4fb7db2b,0xabdbe608), LL(0x5e1109e8,0x837c9228),LL(0xf4645b5a,0x26147d27),LL(0xf7818ed8,0x4d78f592),LL(0xf247fa36,0xd394077e), + LL(0x488c171a,0x0fb9c2d0),LL(0x13685278,0xa78bfbaa),LL(0xd5b1fa6a,0xedfbe268),LL(0x2b7eaba7,0x0dceb8db), LL(0x9ae2b710,0xbf9e8089),LL(0xa4449c96,0xefde7ae6),LL(0xcc143a46,0x43b7716b),LL(0xc3628c13,0xd7d34194), + LL(0x3b3f64c9,0x508cec1c),LL(0x1e5edf3f,0xe20bc0ba),LL(0x2f4318d4,0xda1deb85),LL(0x5c3fa443,0xd20ebe0d), LL(0x73241ea3,0x370b4ea7),LL(0x5e1a5f65,0x61f1511c),LL(0x82681c62,0x99a5e23d),LL(0xa2f54c2d,0xd731e383), + LL(0x83445904,0x2692f36e),LL(0xaf45f9c0,0x2e0ec469),LL(0xc67528b7,0x905a3201),LL(0xd0e5e542,0x88f77f34), LL(0x5864687c,0xf67a8d29),LL(0x22df3562,0x23b92eae),LL(0x9bbec39e,0x5c27014b),LL(0x9c0f0f8d,0x7ef2f226), + LL(0x546c4d8d,0x97359638),LL(0x92f24679,0x5f9c3fc4),LL(0xa8c8acd9,0x912e8bed),LL(0x306634b0,0xec3a318d), LL(0xc31cb264,0x80167f41),LL(0x522113f2,0x3db82f6f),LL(0xdcafe197,0xb155bcd2),LL(0x43465283,0xfba1da59), + LL(0xb212cf53,0xa0425b8e),LL(0xf8557c5f,0x4f2e512e),LL(0x25c4d56c,0xc1286ff9),LL(0xee26c851,0xbb8a0fea), LL(0xe7d6107e,0xc28f70d2),LL(0xe76265aa,0x7ee0c444),LL(0x1d1936b1,0x3df277a4),LL(0xea9595eb,0x1a556e3f), + LL(0xe7305683,0x258bbbf9),LL(0x07ef5be6,0x31eea5bf),LL(0x46c814c1,0x0deb0e4a),LL(0xa7b730dd,0x5cee8449), LL(0xa0182bde,0xeab495c5),LL(0x9e27a6b4,0xee759f87),LL(0x80e518ca,0xc2cf6a68),LL(0xf14cf3f4,0x25e8013f), + LL(0x7e8d7a14,0x8fc44140),LL(0x9556f36a,0xbb1ff3ca),LL(0x14600044,0x6a844385),LL(0x7451ae63,0xba3f0c4a), LL(0x1f9af32a,0xdfcac25b),LL(0xb1f2214b,0x01e0db86),LL(0xa4b596ac,0x4e9a5bc2),LL(0x026c2c08,0x83927681), + LL(0x7acaca28,0x3ec832e7),LL(0xc7385b29,0x1bfeea57),LL(0xfd1eaf38,0x068212e3),LL(0x6acf8ccc,0xc1329830), LL(0x2aac9e59,0xb909f2db),LL(0xb661782a,0x5748060d),LL(0xc79b7a01,0xc5ab2632),LL(0x00017626,0xda44c6c6), + LL(0xa7ea82f0,0xf26c00e8),LL(0xe4299aaf,0x99cac80d),LL(0x7ed78be1,0xd66fe3b6),LL(0x648d02cd,0x305f725f), LL(0x623fb21b,0x33ed1bc4),LL(0x7a6319ad,0xfa70533e),LL(0xbe5ffb3e,0x17ab562d),LL(0x56674741,0x06374994), + LL(0x5c46aa8e,0x69d44ed6),LL(0xa8d063d1,0x2100d5d3),LL(0xa2d17c36,0xcb9727ea),LL(0x8add53b7,0x4c2bab1b), LL(0x15426704,0xa084e90c),LL(0xa837ebea,0x778afcd3),LL(0x7ce477f8,0x6651f701),LL(0x46fb7a8b,0xa0624998), + LL(0xed8a6e19,0xdc1e6828),LL(0x4189d9c7,0x33fc2336),LL(0x671c39bc,0x026f8fe2),LL(0xbc6f9915,0xd40c4ccd), LL(0xf80e75ca,0xafa135bb),LL(0x22adff2c,0x12c651a0),LL(0x4f51ad96,0xc40a04bd),LL(0xbbe4e832,0x04820109), + LL(0x7f4c04cc,0x3667eb1a),LL(0xa9404f84,0x59556621),LL(0x7eceb50a,0x71cdf653),LL(0x9b8335fa,0x994a44a6), LL(0xdbeb9b69,0xd7faf819),LL(0xeed4350d,0x473c5680),LL(0xda44bba2,0xb6658466),LL(0x872bdbf3,0x0d1bc780), + LL(0xa1962f91,0xe535f175),LL(0xed58f5a7,0x6ed7e061),LL(0x2089a233,0x177aa4c0),LL(0xe539b413,0x0dbcb03a), LL(0xbb32e38e,0xe3dc424e),LL(0x6806701e,0x6472e5ef),LL(0x814be9ee,0xdd47ff98),LL(0x35ace009,0x6b60cfff), + LL(0x9ff91fe5,0xb8d3d931),LL(0xf0518eed,0x039c4800),LL(0x9182cb26,0x95c37632),LL(0x82fc568d,0x0763a434), LL(0x383e76ba,0x707c04d5),LL(0x824e8197,0xac98b930),LL(0x91230de0,0x92bf7c8f),LL(0x40959b70,0x90876a01), + LL(0x05968b80,0xdb6d96f3),LL(0x089f73b9,0x380a0913),LL(0xc2c61e01,0x7da70b83),LL(0x569b38c7,0x95fb8394), LL(0x80edfe2f,0x9a3c6512),LL(0x8faeaf82,0x8f726bb9),LL(0x78424bf8,0x8010a4a0),LL(0x0e844970,0x29672044), +}, +/* digit=1 base_pwr=2^7 */ +{ + LL(0x7a2ad62a,0x63c5cb81),LL(0xac62ff54,0x7ef2b6b9),LL(0xb3ad9db5,0x3749bba4),LL(0x46d5a617,0xad311f2c), LL(0xc2ff3b6d,0xb77a8087),LL(0x367834ff,0xb46feaf3),LL(0x75d6b138,0xf8aa266d),LL(0xec008188,0xfa38d320), + LL(0x696946fc,0x486d8ffa),LL(0xb9cba56d,0x50fbc6d8),LL(0x90f35a15,0x7e3d423e),LL(0xc0dd962c,0x7c3da195), LL(0x3cfd5d8b,0xe673fdb0),LL(0x889dfca5,0x0704b7c2),LL(0xf52305aa,0xf6ce581f),LL(0x914d5e53,0x399d49eb), + LL(0x6ec293cd,0x380a496d),LL(0x8e7051f5,0x733dbda7),LL(0xb849140a,0x037e388d),LL(0x5946dbf6,0xee4b32b0), LL(0xcae368d1,0xb1c4fda9),LL(0xfdb0b2f3,0x5001a7b0),LL(0x2e3ac46e,0x6df59374),LL(0x39b3e656,0x4af675f2), + LL(0x39949296,0x44e38110),LL(0x361db1b5,0x5b63827b),LL(0x206eaff5,0x3e5323ed),LL(0xc21f4290,0x942370d2), LL(0xe0d985a1,0xf2caaf2e),LL(0x7239846d,0x192cc64b),LL(0xae6312f8,0x7c0b8f47),LL(0x96620108,0x7dc61f91), + LL(0xc2da7de9,0xb830fb5b),LL(0x0ff8d3be,0xd0e643df),LL(0x188a9641,0x31ee77ba),LL(0xbcf6d502,0x4e8aa3aa), LL(0x9a49110f,0xf9fb6532),LL(0x2dd6b220,0xd18317f6),LL(0x52c3ea5a,0x7e3ced41),LL(0x7d579c4a,0x0d296a14), + LL(0xed4c3717,0x35d6a53e),LL(0x3d0ed2a3,0x9f8240cf),LL(0xe5543aa5,0x8c0d4d05),LL(0xdd33b4b4,0x45d5bbfb), LL(0x137fd28e,0xfa04cc73),LL(0xc73b3ffd,0x862ac6ef),LL(0x31f51ef2,0x403ff9f5),LL(0xbc73f5a2,0x34d5e0fc), + LL(0x08913f4f,0xf2526820),LL(0xeac93d95,0xea20ed61),LL(0x6ca6b26c,0x51ed38b4),LL(0xea4327b0,0x8662dcbc), LL(0x725d2aaa,0x6daf295c),LL(0x8e52dcda,0xbad2752f),LL(0x0b17dacc,0x2210e721),LL(0xd51e8232,0xa37f7912), + LL(0x44cc3add,0x4f7081e1),LL(0x87be82cf,0xd5ffa1d6),LL(0x0edd6472,0x89890b6c),LL(0x3ed17863,0xada26e1a), LL(0x63483caa,0x276f2715),LL(0x2f6077fd,0xe6924cd9),LL(0x0a466e3c,0x05a7fe98),LL(0xb1902d1f,0xf1c794b0), + LL(0x82a8042c,0xe5213688),LL(0xcd278298,0xd931cfaf),LL(0xf597a740,0x069a0ae0),LL(0xeb59107c,0x0adbb3f3), LL(0x5eaa8eb8,0x983e951e),LL(0x11b48e78,0xe663a8b5),LL(0x8a03f2c5,0x1631cc0d),LL(0x11e271e2,0x7577c11e), + LL(0x08369a90,0x33b2385c),LL(0x190eb4f8,0x2990c59b),LL(0xc68eac80,0x819a6145),LL(0x2ec4a014,0x7a786d62), LL(0x20ac3a8d,0x33faadbe),LL(0x5aba2d30,0x31a21781),LL(0xdba4f565,0x209d2742),LL(0x55aa0fbb,0xdb2ce9e3), + LL(0x168984df,0x8cef334b),LL(0x33879638,0xe81dce17),LL(0x263720f0,0xf6e6949c),LL(0xf593cbec,0x5c56feaf), LL(0xfde58c84,0x8bff5601),LL(0x2eccb314,0x74e24117),LL(0x4c9a8a78,0xbcf01b61),LL(0x544c9868,0xa233e35e), + LL(0x8bd7aff1,0xb3156bf3),LL(0x1d81b146,0x1b5ee4cb),LL(0xd628a915,0x7ba1ac41),LL(0xfd89699e,0x8f3a8f9c), LL(0xa0748be7,0x7329b9c9),LL(0xa92e621f,0x1d391c95),LL(0x4d10a837,0xe51e6b21),LL(0x4947b435,0xd255f53a), + LL(0xf1788ee3,0x07669e04),LL(0xa86938a2,0xc14f27af),LL(0xe93a01c0,0x8b47a334),LL(0xd9366808,0xff627438), LL(0xca2a5965,0x7a0985d8),LL(0xd6e9b9b3,0x3d9a5542),LL(0x4cf972e8,0xc23eb80b),LL(0x4fdf72fd,0x5c1c33bb), + LL(0x74a86108,0x0c4a58d4),LL(0xee4c5d90,0xf8048a8f),LL(0xe86d4c80,0xe3c7c924),LL(0x056a1e60,0x28c889de), LL(0xb214a040,0x57e2662e),LL(0x37e10347,0xe8c48e98),LL(0x80ac748a,0x87742862),LL(0x186b06f2,0xf1c24022), + LL(0x5f74040a,0xac2dd4c3),LL(0xfceac957,0x409aeb71),LL(0x55c4ec23,0x4fbad782),LL(0x8a7b76ec,0xb359ed61), LL(0xed6f4a60,0x12744926),LL(0x4b912de3,0xe21e8d7f),LL(0xfc705a59,0xe2575a59),LL(0xed2dbc0e,0x72f1d4de), + LL(0xeb7926b8,0x3d2b24b9),LL(0xcdbe5509,0xbff88cb3),LL(0xe4dd640b,0xd0f399af),LL(0x2f76ed45,0x3c5fe130), LL(0x3764fb3d,0x6f3562f4),LL(0x3151b62d,0x7b5af318),LL(0xd79ce5f3,0xd5bd0bc7),LL(0xec66890f,0xfdaf6b20), + LL(0x6063540c,0x735c67ec),LL(0xe5f9cb8f,0x50b259c2),LL(0x3f99c6ab,0xb8734f9a),LL(0xa3a7bc85,0xf8cc13d5), LL(0xc5217659,0x80c1b305),LL(0x4ec12a54,0xfe5364d4),LL(0x681345fe,0xbd87045e),LL(0x582f897f,0x7f8efeb1), + LL(0xd5923359,0xe8cbf1e5),LL(0x539b9fb0,0xdb0cea9d),LL(0x49859b98,0x0c5b34cf),LL(0xa4403cc6,0x5e583c56), LL(0xd48185b7,0x11fc1a2d),LL(0x6e521787,0xc93fbc7e),LL(0x05105b8b,0x47e7a058),LL(0xdb8260c8,0x7b4d4d58), + LL(0x46eb842a,0xe33930b0),LL(0x7bdae56d,0x8e844a9a),LL(0x13f7fdfc,0x34ef3a9e),LL(0x636ca176,0xb3768f82), LL(0x4e09e61c,0x2821f4e0),LL(0xa0c7cddc,0x414dc3a1),LL(0x54945fcd,0xd5379437),LL(0xb3555ff1,0x151b6eef), + LL(0x6339c083,0xb31bd613),LL(0xdfb64701,0x39ff8155),LL(0xe29604ab,0x7c3388d2),LL(0xa6b10442,0x1e19084b), LL(0xeccd47ef,0x17cf54c0),LL(0x4a5dfb30,0x89693385),LL(0x47daf9f6,0x69d023fb),LL(0x7d91d959,0x9222840b), + LL(0x803bac62,0x439108f5),LL(0x379bd45f,0x0b7dd91d),LL(0xca63c581,0xd651e827),LL(0x509c104f,0x5c5d75f6), LL(0x1f2dc308,0x7d5fc738),LL(0xd98454be,0x20faa7bf),LL(0xa517b031,0x95374bee),LL(0x642692ac,0xf036b9b1), + LL(0x39842194,0xc5106109),LL(0x49d05295,0xb7e2353e),LL(0xefb42ee0,0xfc8c1d5c),LL(0x08ce811c,0xe04884eb), LL(0x7419f40e,0xf1f75d81),LL(0xa995c241,0x5b0ac162),LL(0xc4c55646,0x120921bb),LL(0x8d33cf97,0x713520c2), + LL(0xe98c5100,0xb4a65a5c),LL(0x2ddd0f5a,0x6cec871d),LL(0x9ba2e78b,0x251f0b7f),LL(0xce3a2a5f,0x224a8434), LL(0x25f5c46f,0x26827f61),LL(0x48545ec0,0x6a22bedc),LL(0xb1bb5cdc,0x25ae5fa0),LL(0xfcb9b98f,0xd693682f), + LL(0x91e5d7d3,0x32027fe8),LL(0x73a07678,0xf14b7d17),LL(0xc0dfdd61,0xf88497b3),LL(0x2a8c4f48,0xf7c2eec0), LL(0x3756e621,0xaa5573f4),LL(0x1825b948,0xc013a240),LL(0x63878572,0x1c03b345),LL(0x653a4184,0xa0472bea), + LL(0x0ac69a80,0xf4222e27),LL(0xf51e54f6,0x34096d25),LL(0x8fffa591,0x00a648cb),LL(0x69b6527f,0x4e87acdc), LL(0xe285ccb4,0x0575e037),LL(0x50ddcf52,0x188089e4),LL(0x870ff719,0xaa96c9a8),LL(0x1fc7e369,0x74a56cd8), + LL(0x1726931a,0x41d04ee2),LL(0x3660ecfd,0x0bbbb2c8),LL(0x24818e18,0xa6ef6de5),LL(0xe7d57887,0xe421cc51), LL(0xbea87be6,0xf127d208),LL(0xb1cdd682,0x16a475d3),LL(0x439b63f7,0x9db1b684),LL(0xf0f113b6,0x5359b3db), + LL(0x8bf06e31,0xdfccf1de),LL(0xdd383901,0x1fdf8f44),LL(0x5017e7d2,0x10775cad),LL(0x58d11eef,0xdfc3a597), LL(0xb1ecff10,0x6ec9c8a0),LL(0x28400549,0xee6ed6cc),LL(0x1b4f8d73,0xb5ad7bae),LL(0xe00aaab9,0x61b4f11d), + LL(0xd4eff2d7,0x7b32d69b),LL(0x4288b60f,0x88ae6771),LL(0x37a1e723,0x159461b4),LL(0x570aae8c,0x1f3d4789), LL(0x7f9871da,0x869118c0),LL(0xf635e278,0x35fbda78),LL(0xe1541dac,0x738f3641),LL(0xc0dae45f,0x6794b13a), + LL(0x09cc0917,0x065064ac),LL(0xc68540fd,0x27c53729),LL(0xef227671,0x0d2d4c8e),LL(0xa1785a04,0xd23a9f80), LL(0x52650359,0x98c59528),LL(0x74a1acad,0xfa09ad01),LL(0x0b55bf5c,0x082d5a29),LL(0x419b8084,0xa40f1c67), + LL(0xdcc18770,0x3a5c752e),LL(0x8825c3a5,0x4baf1f2f),LL(0x21b153ed,0xebd63f74),LL(0xb2f64723,0xa2383e47), LL(0x2646d19a,0xe7bf620a),LL(0x03c83ffd,0x56cb44ec),LL(0x4f6be9f1,0xaf7267c9),LL(0xc06bb5e9,0x8b2dfd7b), + LL(0xa672c5c7,0xb87072f2),LL(0x0d53c5e2,0xeacb11c8),LL(0xff435932,0x22dac29d),LL(0x4408693c,0x37bdb99d), LL(0x2899c20f,0xf6e62fb6),LL(0x447ece24,0x3535d512),LL(0xff577ce3,0xfbdc6b88),LL(0x190575f2,0x726693bd), + LL(0xab4b35a2,0x6772b0e5),LL(0xf5eeaacf,0x1d8b6001),LL(0x795b9580,0x728f7ce4),LL(0x41fb81da,0x4a20ed2a), LL(0x4fec01e6,0x9f685cd4),LL(0xa7ff50ad,0x3ed7ddcc),LL(0x0c2d97fd,0x460fd264),LL(0xeb82f4f9,0x3a241426), + LL(0x6a8ea820,0x17d1df2c),LL(0xf22cc254,0xb2b50d3b),LL(0xb7291426,0x03856cba),LL(0x04f5ee39,0x87fd26ae), LL(0x02bee4ba,0x9cb696cc),LL(0x06820fd6,0x53121804),LL(0x0212e985,0xa5dfc269),LL(0x160f9a09,0x666f7ffa), + LL(0xbccd9617,0xc503cd33),LL(0xba7730a3,0x365dede4),LL(0x5ddb0786,0x798c6355),LL(0xfc9cd3bc,0xa6c3200e), LL(0xe5e35efd,0x060ffb2c),LL(0x5555a1c1,0x99a4e25b),LL(0xf70b3751,0x11d95375),LL(0x160e1bf6,0x0a57354a), + LL(0xf8e4b065,0xecb3ae4b),LL(0x2e53022b,0x07a834c4),LL(0x8692ed96,0x1cd300b3),LL(0x61ee14ec,0x16a6f792), LL(0x6a8649ed,0x8f1063c6),LL(0x869f3e14,0xfbcdfcfe),LL(0x00a7b3ec,0x2cfb97c1),LL(0x7130c2f1,0xcea49b3c), + LL(0xe9d96488,0x462d044f),LL(0x8182a0c1,0x4b53d52e),LL(0x0391e9e9,0x84b6ddd3),LL(0xb1741a09,0x80ab7b48), LL(0x27d3317f,0xec0e15d4),LL(0x1a64671e,0x8dfc1ddb),LL(0xd49c5b92,0x93cc5d5f),LL(0x3674a331,0xc995d53d), + LL(0x090090ae,0x302e41ec),LL(0xedb06830,0x2278a0cc),LL(0xfbc99690,0x1d025932),LL(0xb80d68da,0x0c32fbd2), LL(0xf341a6c1,0xd79146da),LL(0x1bef68a0,0xae0ba139),LL(0x8d774b3a,0xc6b8a563),LL(0x880ba4d7,0x1cf307bd), + LL(0x19803511,0xc033bdc7),LL(0x8888c3be,0xa9f97b3b),LL(0x85c6d05e,0x3d68aebc),LL(0x193919eb,0xc3b88a9d), LL(0xc48b0ee3,0x2d300748),LL(0x07a746c1,0x7506bc7c),LL(0x6e6d57f3,0xfc48437c),LL(0xcfeaa91a,0x5bd71587), + LL(0xc1bc5225,0xa4ed0408),LL(0x2719226d,0xd0b946db),LL(0x758d2d43,0x109ecd62),LL(0x2751759b,0x75c8485a), LL(0x9ce4177a,0xb0b75f49),LL(0x79c10c3d,0x4fa61a1e),LL(0xa167fcd7,0xc062d300),LL(0x750f0fa8,0x4df3874c), + LL(0x83dfedc9,0x29ae2cf9),LL(0x8d87631a,0xf8437134),LL(0x7429c8d2,0xaf571711),LL(0x146d9272,0x18d15867), LL(0x69769bb7,0x83053ecf),LL(0xc479ab82,0xc55eb856),LL(0x21b0f4b2,0x5ef7791c),LL(0x3d491525,0xaa5956ba), + LL(0x9fe20eba,0x407a96c2),LL(0xe52a5ad3,0xf27168bb),LL(0xbf1d9d89,0x43b60ab3),LL(0x710e727a,0xe45c51ef), LL(0x099b4221,0xdfca5276),LL(0x2557a159,0x8dc6407c),LL(0x91035895,0x0ead8335),LL(0x9c55dc32,0x0a9db957), + LL(0xdf61bc76,0xe40736d3),LL(0x3f778cdb,0x13a619c0),LL(0xc56ea28f,0x6dd921a4),LL(0x2fa647b4,0x76a52433), LL(0xac5bdc5d,0x23591891),LL(0xbac7dc01,0xff4a1a72),LL(0x62df8453,0x9905e261),LL(0xe63b265f,0x3ac045df), + LL(0xad53dba7,0x8a3f341b),LL(0x837b625a,0x8ec269cc),LL(0x3ae31189,0xd71a2782),LL(0x55e96120,0x8fb4f9a3), LL(0xff9875cf,0x804af823),LL(0x5d442a9b,0x23224f57),LL(0xecc62679,0x1c4d3b9e),LL(0xa0e7ddb1,0x91da22fb), + LL(0x6c04a661,0xa370324d),LL(0x5e376d17,0x9710d3b6),LL(0x3044e357,0xed8c98f0),LL(0x6422701c,0xc364ebbe), LL(0x7733d61c,0x347f5d51),LL(0xcea826c3,0xd55644b9),LL(0x55a25548,0x80c6e0ad),LL(0x844220a7,0x0aa7641d), + LL(0x31810660,0x1438ec81),LL(0xde4b4043,0x9dfa6507),LL(0xcc3e0273,0x10b515d8),LL(0x28d8cfb2,0x1b6066dd), LL(0x9c9efebd,0xd3b04591),LL(0xa21c1ff4,0x425d4bdf),LL(0xd57607d3,0x5fe5af19),LL(0x54481084,0xbbf773f7), + LL(0x94b03ed1,0x8435bd69),LL(0x634cc546,0xd9ad1de3),LL(0x00e420ca,0x2cf423fc),LL(0xa03096dd,0xeed26d80), LL(0xa4db09d2,0xd7f60be7),LL(0x960622f7,0xf47f569d),LL(0x7296c729,0xe5925fd7),LL(0x26ca2715,0xeff2db26), + LL(0xb913e759,0xa6fcd014),LL(0x8ff4de93,0x53da4786),LL(0xc32068e1,0x14616d79),LL(0xccdf352e,0xb187d664), LL(0x1dc90b59,0xf7afb650),LL(0x7daa1b26,0x8170e943),LL(0x700c0a84,0xc8e3bdd8),LL(0x6482bdfa,0x6e8d345f), + LL(0xc5c5ea50,0x84cfbfa1),LL(0x67960681,0xd3baf14c),LL(0x0dd50942,0x26398403),LL(0x4716a663,0xe4b7839c), LL(0xe7de6dc0,0xd5f1f794),LL(0x622aa7ce,0x5cd0f4d4),LL(0x59acfeec,0x5295f3f1),LL(0x953e0607,0x8d933552), + LL(0x776c5722,0xc7db8ec5),LL(0x2b5f290c,0xdc467e62),LL(0x4ff425a9,0xd4297e70),LL(0x0cf7bb72,0x4be924c1), LL(0xa1892131,0x0d5dc5ae),LL(0xa705c992,0x8bf8a8e3),LL(0x7a305ac5,0x73a0b064),LL(0x9a8c77a8,0x00c9ca4e), + LL(0x83774bdd,0x5dfee80f),LL(0x85734485,0x63131602),LL(0x914a69a9,0xa1b524ae),LL(0xd4e300d7,0xebc2ffaf), LL(0x7cfa46a5,0x52c93db7),LL(0x21653b50,0x71e6161f),LL(0xa4bc580a,0x3574fc57),LL(0xe1bc1253,0xc09015dd), + LL(0xd174d7aa,0x4b7b47b2),LL(0xf3a15d04,0x4072d8e8),LL(0xd6fa07ed,0xeeb7d47f),LL(0xedbdafb1,0x6f2b9ff9), LL(0x3760fe8a,0x18c51615),LL(0xf06c6c13,0x7a96e6bf),LL(0x0ea2d071,0x4d7a0410),LL(0x0be2a5ce,0xa1914e9b), + LL(0xd8a3c5cf,0x5726e357),LL(0x2abb2b13,0x1197ecc3),LL(0x31ae88dd,0x6c0d7f7f),LL(0xfdbb3efe,0x15b20d1a), LL(0x70584039,0xcd06aa26),LL(0xa7dc9747,0x2277c969),LL(0x7855d815,0xbca69587),LL(0x5188b32a,0x899ea238), + LL(0x760c1c9d,0x37d9228b),LL(0x9b5c18da,0xc7efbb11),LL(0x19f6dbc5,0x7f0d1bc8),LL(0x07e6905b,0x4875384b), LL(0x3ba8cd86,0xc7c50baa),LL(0xc2905de0,0xb0ce40fb),LL(0x7a231952,0x70840673),LL(0xcf43de26,0xa912a262), + LL(0xeb5b76c1,0x9c38ddcc),LL(0x26fc0ab4,0x746f5285),LL(0xd62c269f,0x52a63a50),LL(0x99458621,0x60049c55), LL(0x3c2f7c9e,0xe7f48f82),LL(0x917d5cf3,0x6bd99043),LL(0x8701f469,0xeb1317a8),LL(0x9a449fe0,0xbd3fe2ed), + LL(0x12ef3d36,0x421e79ca),LL(0x3e7ea5de,0x9ee3c36c),LL(0xcdff36f7,0xe48198b5),LL(0xc6b82228,0xaff4f967), LL(0xc47adb7e,0x15e19dd0),LL(0x032e7dfa,0x45699b23),LL(0x1fae026a,0x40680c8b),LL(0x550dbf4d,0x5a347a48), + LL(0x3cef0d7d,0xe652533b),LL(0x2bbb4381,0xd94f7b18),LL(0x0e80f500,0x838752be),LL(0x9e9c9bfb,0x8e6e2488), LL(0x16caca6a,0xc9751697),LL(0x38531ad9,0x866c49d8),LL(0x7151ade1,0xc917e239),LL(0x6037c407,0x2d016ec1), + LL(0x00eac3f9,0xa407ccc9),LL(0xe2ed4748,0x835f6280),LL(0x1cc98e0d,0xcc54c347),LL(0xdcb572eb,0x0e969937), LL(0x8f30c9cb,0x1b16c8e8),LL(0x373c4661,0xa606ae75),LL(0x35502cab,0x47aa689b),LL(0x4d9bb64f,0xf89014ae), + LL(0x31c71f7b,0x202f6a9c),LL(0x296ffe5c,0x01f95aa3),LL(0x53cec3a3,0x5fc06014),LL(0x5f498a45,0xeb991237), LL(0x5d91ba87,0xae9a935e),LL(0x0b564a19,0xc6ac6281),LL(0x3bd44e69,0x8a8fe81c),LL(0x9dd11d45,0x7c8b467f), + LL(0xea5b8e69,0xf772251f),LL(0xc5b75fbc,0xaeecb3bd),LL(0x887ff0e5,0x1aca3331),LL(0x19f0a131,0xbe5d49ff), LL(0xe5c8646f,0x582c13aa),LL(0x20e19980,0xdbaa12e8),LL(0xf7abbd94,0x8f40f31a),LL(0x1dfc7663,0x1f13f5a8), + LL(0xaceb4fc0,0x5d81f1ee),LL(0x5e6f0f42,0x36256002),LL(0x751370c8,0x4b67d6d7),LL(0x03e80589,0x2608b698), LL(0x05268301,0xcfc0d2fc),LL(0x40309212,0xa6943d39),LL(0x1fd0e1c2,0x192a90c2),LL(0x37f1dc76,0xb209f113), + LL(0x97bf1298,0xefcc5e06),LL(0x219d639e,0xcbdb6730),LL(0xb81e8c6f,0xd009c116),LL(0x1a7ce2e5,0xa3ffdde3), LL(0xa914d3ba,0xc53fbaaa),LL(0x88df85ee,0x836d500f),LL(0x66ee0751,0xd98dc71b),LL(0x714516fd,0x5a3d7005), + LL(0x39eedbba,0x21d3634d),LL(0x0455a46d,0x35cd2e68),LL(0xf9d7eb0c,0xc8cafe65),LL(0x00cefb3e,0xbda3ce9e), LL(0x2c9cf7a4,0xddc17a60),LL(0x7bcb8773,0x01572ee4),LL(0x8c7548df,0xa92b2b01),LL(0xa84600e3,0x732fd309), + LL(0x16543a40,0xe22109c7),LL(0xfede3c6c,0x9acafd36),LL(0x6824e614,0xfb206852),LL(0xda25dca0,0x2a4544a9), LL(0x91d60b06,0x25985262),LL(0x28753545,0x281b7be9),LL(0x90f13b27,0xec667b1a),LL(0x940e2eb4,0x33a83aff), + LL(0xd5d721d5,0x80009862),LL(0x5bd3a182,0x0c3357a3),LL(0x7aa2cda4,0x27f3a83b),LL(0xf6f83085,0xb58ae74e), LL(0x2e6dad6b,0x2a911a81),LL(0xf43d6c5b,0xde286051),LL(0xf996c4d8,0x4bdccc41),LL(0x0ae1e24e,0xe7312ec0), +}, +/* digit=2 base_pwr=2^14 */ +{ + LL(0x6e6485b3,0xf8d112e7),LL(0x771c52f8,0x4d3e24db),LL(0x684a2f6d,0x48e3ee41),LL(0x21d95551,0x7161957d), LL(0xcdb12a6c,0x19631283),LL(0x2e50e164,0xbf3fa882),LL(0x3166cc73,0xf6254b63),LL(0xaee8cc38,0x3aefa7ae), + LL(0x3b36f9fd,0x79b0fe62),LL(0xfde19fc0,0x26543b23),LL(0x958482ef,0x136e64a0),LL(0x9b095825,0x23f63771), LL(0xb6a1142e,0x14cfd596),LL(0x335aac0b,0x5ea6aac6),LL(0xf3081dd5,0x86a0e8bd),LL(0x003dc12a,0x5fb89d79), + LL(0xf72e34d4,0xf615c33a),LL(0x110eec35,0x0bd9ea40),LL(0xc1dea34e,0x1c12bc5b),LL(0x49ae4699,0x686584c9), LL(0x8c97b942,0x13ad95d3),LL(0x4e5c7562,0x4609561a),LL(0xf2737f89,0x9e94a4ae),LL(0x371c78b6,0xf57594c6), + LL(0xe3779ee3,0x0f0165fc),LL(0xbd495d9e,0xe00e7f9d),LL(0x20284e7a,0x1fa4efa2),LL(0x47ac6219,0x4564bade), LL(0xc4708e8e,0x90e6312a),LL(0xa71e9adf,0x4f5725fb),LL(0x3d684b9f,0xe95f55ae),LL(0x1e94b415,0x47f7ccb1), + LL(0x8d946581,0x7322851b),LL(0xbdf4a012,0xf0d13133),LL(0x6584dae0,0xa3510f69),LL(0x3c9f6c6d,0x03a7c171), LL(0xe475381a,0x5be97f38),LL(0x85823334,0xca1ba422),LL(0x0be17dda,0xf83cc5c7),LL(0x0b918c0f,0x158b1494), + LL(0x522e6b69,0xda3a77e5),LL(0xbbcd6c18,0x69c908c3),LL(0xd924fd56,0x1f1b9e48),LL(0xaa4bb3f7,0x37c64e36), LL(0xee478d7d,0x5a4fdbdf),LL(0x0193f7a0,0xba75c8bc),LL(0x56cd16df,0x84bc1e84),LL(0x46fad151,0x1fb08f08), + LL(0x842e9f30,0x8a7cabf9),LL(0x5eab83af,0xa331d4bf),LL(0x017f2a6a,0xd272cfba),LL(0x83aba0e3,0x27560abc), LL(0x0e3a6b75,0x94b83387),LL(0x6b9f50f5,0x25c6aea2),LL(0xb5fdf6d0,0x803d691d),LL(0xe6333514,0x03b77509), + LL(0x61a341c1,0x36178903),LL(0x0cfd6142,0x3604dc60),LL(0x8533316c,0x022295eb),LL(0x44af2922,0x3dbde4ac), LL(0x1c7eef69,0x898afc5d),LL(0xd14f4fa1,0x58896805),LL(0x203c21ca,0x05002160),LL(0x40ef730b,0x6f0d1f30), + LL(0x196224f8,0x8e8c44d4),LL(0x374d079d,0x75a4ab95),LL(0x7d48f123,0x79085ecc),LL(0x1bf65ad8,0x56f04d31), LL(0xbda602b2,0xe220bf1c),LL(0xf9612c69,0x73ee1742),LL(0x084fd06b,0x76008fc8),LL(0xf11380d1,0x4000ef9f), + LL(0x12cfe297,0x48201b4b),LL(0x292f74e5,0x3eee129c),LL(0xc9e874e8,0xe1fe114e),LL(0x92c5fc41,0x899b055c), LL(0x3a39c8cf,0x4e477a64),LL(0x78963cc9,0x82f09efe),LL(0xd333f863,0x6fd3fd8f),LL(0xdc949c63,0x85132b2a), + LL(0x516eb17b,0x7e06a3ab),LL(0xd2c7372b,0x73bec06f),LL(0xba896da6,0xe4f74f55),LL(0x8e9eb40f,0xbb4afef8), LL(0xe61d66b0,0x2d75bec8),LL(0xef29300b,0x02bda4b4),LL(0x026baa5a,0x8bbaa8de),LL(0xa07f4440,0xff54befd), + LL(0xbe7a2af3,0xbd9b8b1d),LL(0x4fb74a72,0xec51caa9),LL(0x63879697,0xb9937a4b),LL(0xec2687d5,0x7c9a9d20), LL(0x6ef5f014,0x1773e44f),LL(0xe90c6900,0x8abcf412),LL(0x8142161e,0x387bd022),LL(0xfcb6ff2a,0x50393755), + LL(0xed6def63,0x9813fd56),LL(0x7d53106c,0x53cf6482),LL(0x431f7ac1,0x991a35bd),LL(0x63e65faf,0xf1e274dd), LL(0x44cc7880,0xf63ffa3c),LL(0x7c256981,0x411a426b),LL(0x93a420e0,0xb698b9fd),LL(0xae53f8fe,0x89fdddc0), + LL(0x32398baa,0x766e0722),LL(0x5cfca031,0x205fee42),LL(0x7a029cf2,0xa49f5341),LL(0x4023890d,0xa88c68b8), LL(0x7337aaa8,0xbc275041),LL(0x0eb384f4,0x9ed364ad),LL(0x29aba92f,0xe0816f85),LL(0x04e38a88,0x2e9e1941), + LL(0x3dafd2d5,0x57eef44a),LL(0x97ed98d8,0x35d1fae5),LL(0x2307f9b1,0x50628c09),LL(0xd6cba5c6,0x09d84aae), LL(0x88aaa691,0x67071bc7),LL(0xafe6cb03,0x2dea57a9),LL(0x3d78ac01,0xdfe11bb4),LL(0x7fd7aa51,0x7286418c), + LL(0x77f7195a,0xfabf7709),LL(0xadeb838f,0x8ec86167),LL(0xbb4f012d,0xea1285a8),LL(0x9a3eab3f,0xd6883503), LL(0x309004c2,0xee5d24f8),LL(0x13ffe95e,0xa96e4b76),LL(0xbd223ea4,0x0cdffe12),LL(0xb6739a53,0x8f5c2ee5), + LL(0xdd968198,0x5cb4aaa5),LL(0x72413a6c,0xfa131c52),LL(0x9536d903,0x53d46a90),LL(0x48606d8e,0xb270f0d3), LL(0xa053a3bc,0x518c7564),LL(0x1a86caef,0x088254b7),LL(0x0ab5efd0,0xb3ba8cb4),LL(0x4605945d,0x5c59900e), + LL(0xa1887395,0xecace1dd),LL(0x932a65de,0x40960f36),LL(0x3aa95529,0x9611ff5c),LL(0x7c1e5a36,0xc58215b0), LL(0xf0e1a524,0xd48c9b58),LL(0xf590dfb8,0xb406856b),LL(0x9cd95662,0xc7605e04),LL(0xa33ecf82,0x0dd036ee), + LL(0xc33156b3,0xa50171ac),LL(0x4a80172e,0xf09d24ea),LL(0x76dc8eef,0x4e1f72c6),LL(0x5e3d44ee,0xe60caadc), LL(0x979b1d8f,0x006ef8a6),LL(0x97788d26,0x60908a1c),LL(0x266feec0,0x6e08f95b),LL(0x22e8c94e,0x618427c2), + LL(0x59145a65,0x3d613339),LL(0xfa406337,0xcd9bc368),LL(0x2d8a52a0,0x82d11be3),LL(0x97a1c590,0xf6877b27), LL(0xf5cbdb25,0x837a819b),LL(0xde090249,0x2a4fd1d8),LL(0x74990e5f,0x622a7de7),LL(0x7945511b,0x840fa5a0), + LL(0x6558842d,0x30b974be),LL(0x17f3d0a6,0x70df8c64),LL(0x7542e46d,0x7c803520),LL(0xe4ecc823,0x7251fe7f), LL(0x5e9aac9a,0xe59134cb),LL(0xf0045d71,0x11bb0934),LL(0xdbcb1d4e,0x53e5d9b5),LL(0x92defc91,0x8d97a905), + LL(0x7946d3f9,0xfe289327),LL(0x07472273,0xe132bd24),LL(0x1eb6ae86,0xeeeb510c),LL(0xf0595067,0x777708c5), LL(0x1297029e,0x18e2c8cd),LL(0xbbf9305e,0x2c61095c),LL(0x6b85d6d9,0xe466c258),LL(0xda1ea530,0x8ac06c36), + LL(0xa1304668,0xa365dc39),LL(0x07f89606,0xe4a9c885),LL(0xacc7228d,0x65a4898f),LL(0x84ca8303,0x3e2347ff), LL(0xea7d23a3,0xa5f6fb77),LL(0x672a71cd,0x2fac257d),LL(0x7e6a44d3,0x6908bef8),LL(0x891d3d7a,0x8ff87566), + LL(0x6b0cf82e,0xe58e90b3),LL(0x2615b5e7,0x6438d246),LL(0x669c145a,0x07b1f8fc),LL(0x36f1e1cb,0xb0d8b2da), LL(0xd9184c4d,0x54d5dadb),LL(0xf93d9976,0x3dbb18d5),LL(0xd1147d47,0x0a3e0f56),LL(0xa0a48609,0x2afa8c8d), + LL(0xbc36742c,0x275353e8),LL(0xeea0ed90,0x898f427e),LL(0x3e477b00,0x26f4947e),LL(0x308741e3,0x8ad8848a), LL(0xd74a2a46,0x6c703c38),LL(0x9ba17ba2,0x5e3e05a9),LL(0x4ab9a9e4,0xc1fa6f66),LL(0x3841d6ec,0x474a2d9a), + LL(0x653ae326,0x871239ad),LL(0xa74cbb43,0x14bcf72a),LL(0x20d4c083,0x8737650e),LL(0x110ed4af,0x3df86536), LL(0xb53ca555,0xd2d86fe7),LL(0xabd5d538,0x688cb00d),LL(0x1ad38468,0xcf81bda3),LL(0xf01167b6,0x7ccfe3cc), + LL(0x6c4c1fe6,0xcf4f47e0),LL(0x298bbb79,0x557e1f1a),LL(0x30d45a14,0xf93b974f),LL(0x0baf97c4,0x174a1d2d), LL(0xc51fbf53,0x7a003b30),LL(0xee68b225,0xd8940991),LL(0x1c0f4173,0x5b0aa7b7),LL(0xa20a7153,0x975797c9), + LL(0xe3533d77,0x26e08c07),LL(0x2e341c99,0xd7222e6a),LL(0x8d2dc4ed,0x9d60ec3d),LL(0x7c476cf8,0xbdfe0d8f), LL(0x1d056605,0x1fe59ab6),LL(0x86a8551f,0xa9ea9df6),LL(0x47fb8d8c,0x8489941e),LL(0x4a7f1b10,0xfeb874eb), + LL(0x7ee0d98f,0xfe5fea86),LL(0xdbf61864,0x201ad34b),LL(0x37c031d4,0x45d8fe47),LL(0x795f0822,0xd5f49fae), LL(0xc7f4a40c,0xdb0fb291),LL(0x730ddd92,0x2e69d9c1),LL(0x49d76987,0x754e1054),LL(0x7662db87,0x8a24911d), + LL(0x60a71676,0x61fc1810),LL(0xf66a8ad1,0xe852d1a8),LL(0x6417231e,0x172bbd65),LL(0x3babb11f,0x0d6de7bd), LL(0xc8e347f8,0x6fde6f88),LL(0x9bd99cc3,0x1c587547),LL(0x34076950,0x78e54ed0),LL(0x796e83ba,0x97f0f334), + LL(0x4924867a,0xe4dbe1ce),LL(0x60b84917,0xbd5f51b0),LL(0x3cb09a79,0x37530040),LL(0xff1743d8,0xdb3fe0f8), LL(0x556fa9db,0xed7894d8),LL(0x23412fbf,0xfa262169),LL(0xba7b9291,0x563be0db),LL(0x0c9fb234,0x6ca8b8c0), + LL(0xbd763802,0xed406aa9),LL(0x65303da1,0xc21486a0),LL(0xc7e62ec4,0x61ae291e),LL(0xdf99333e,0x622a0492), LL(0xbb7a8ee0,0x7fd80c9d),LL(0x6c01aedb,0xdc2ed3bc),LL(0x08be74ec,0x35c35a12),LL(0x469f671f,0xd540cb1a), + LL(0xcf84f6c7,0xd16ced4e),LL(0x2d090f43,0x8561fb9c),LL(0x6f239db4,0x7e693d79),LL(0x77bd0d94,0xa736f928), LL(0x2c1950ee,0x07b4d929),LL(0x56dc11b3,0xda177543),LL(0x7a6a878e,0xa5dfbbaa),LL(0x4decb08a,0x1c70cb29), + LL(0x6f0f7c50,0xfba28c8b),LL(0x854dcc6d,0xa8eba2b8),LL(0x36b78642,0x5ff8e89a),LL(0xf6873adf,0x070c1c8e), LL(0x6484d2e4,0xbbd3c371),LL(0x0d414129,0xfb78318f),LL(0x6ad93b0b,0x2621a39c),LL(0xa9e917f7,0x979d74c2), + LL(0x61fb0428,0xfc195647),LL(0xbee624d4,0x4d78954a),LL(0xb8ae86fd,0xb94896e0),LL(0xc91c8b13,0x6667ac0c), LL(0x43bcf832,0x9f180512),LL(0xa0010137,0xfbadf8b7),LL(0xb3ba8aa7,0xc69b4089),LL(0xe687ce85,0xfac4bacd), + LL(0x977eab40,0x9164088d),LL(0x2760b390,0x51f4c5b6),LL(0x340dd553,0xd238238f),LL(0xdb1d31c9,0x358566c3), LL(0x5068f5ff,0x3a5ad69e),LL(0xdaff6b06,0xf31435fc),LL(0xd6debff0,0xae549a5b),LL(0x75e01331,0x59e5f0b7), + LL(0x98559acf,0x5d492fb8),LL(0x4db79b50,0x96018c2e),LL(0x609f66aa,0x55f4a48f),LL(0x4900a14f,0x1943b3af), LL(0x15a40d39,0xc22496df),LL(0x4c20f7c5,0xb2a44684),LL(0x3b98404c,0x76a35afa),LL(0xff5d1b77,0xbec75725), + LL(0xbea06444,0xb67aa163),LL(0xf724b6f2,0x27e95bb2),LL(0xd238c8ab,0x3c20e3e9),LL(0xddd6ae17,0x1213754e), LL(0x716e0f74,0x8c431020),LL(0xffc095c2,0x6679c82e),LL(0xd0ac2932,0x2eb3adf4),LL(0x01bb7a76,0x2cc970d3), + LL(0x740f0e66,0x70c71f2f),LL(0x2b6b23cc,0x545c616b),LL(0xb40a8bd7,0x4528cfcb),LL(0x2ab27722,0xff839633), LL(0x025ac99a,0x049127d9),LL(0x2b63e33b,0xd314d4a0),LL(0x28d84519,0xc8c310e7),LL(0xb3bc84ba,0x0fcb8983), + LL(0x38634818,0x2cc52261),LL(0xb44c2e0b,0x501814f4),LL(0x54dfdba3,0xf7e181aa),LL(0xe759718c,0xcfd58ff0), LL(0xd3b507a8,0xf90cdb14),LL(0xc50bdad8,0x57bd478e),LL(0x50e5f9aa,0x29c197e2),LL(0xe40bc855,0x4db6eef8), + LL(0xd1fc0654,0x2cc8f21a),LL(0x81269d73,0xc71cc963),LL(0x077f49f9,0xecfbb204),LL(0xca56b793,0xdde92571), LL(0xf97ad8f7,0x9abed6a3),LL(0x924de3bd,0xe6c19d3f),LL(0xa140a800,0x8dce92f4),LL(0x1337af07,0x85f44d1e), + LL(0x09d64c52,0x5953c08b),LL(0xf5df9749,0xa1b5e49f),LL(0x52735f7d,0x336a8fb8),LL(0x9add676b,0xb332b6db), LL(0xb4511aa4,0x558b88a0),LL(0xdbd5cc55,0x09788752),LL(0xd8cd52bd,0x16b43b9c),LL(0xc2a2696b,0x7f0bc5a0), + LL(0xc11f61ef,0x146e12d4),LL(0x3a83e79e,0x9ce10754),LL(0x6cbfca15,0x08ec73d9),LL(0x5b49653f,0x09ff29ad), LL(0xe7da946e,0xe31b72bd),LL(0xee80a4f2,0xebf9eb3b),LL(0x17598ce4,0xd1aabd08),LL(0x53f37e80,0x18b5fef4), + LL(0x5958cd79,0xd5d5cdd3),LL(0x1d373114,0x3580a1b5),LL(0xfa935726,0xa36e4c91),LL(0xef20d760,0xa38c534d), LL(0x2ff5845b,0x7088e40a),LL(0xbd78177f,0xe5bb40bd),LL(0x857f9920,0x4f06a7a8),LL(0xe968f05d,0xe3cc3e50), + LL(0xe5682d26,0x1d68b7fe),LL(0xaec7f87c,0x5206f76f),LL(0x041951ab,0x41110530),LL(0xd4b5a71a,0x58ec52c1), LL(0x0f75cf9a,0xf3488f99),LL(0xba82d0d5,0xf411951f),LL(0x618895ab,0x27ee75be),LL(0x6d8aab14,0xeae060d4), + LL(0x7fb54dc2,0x9ae1df73),LL(0x25963649,0x1f3e391b),LL(0xfe055081,0x242ec32a),LL(0x8491c9bd,0x5bd450ef), LL(0x981eb389,0x367efc67),LL(0x3a0550d5,0xed7e1928),LL(0xab3ce75c,0x362e776b),LL(0x1f24c523,0xe890e308), + LL(0xfeccef76,0xb961b682),LL(0x8bba6d92,0x8b8e11f5),LL(0x2b2375c4,0x8f2ccc4c),LL(0xe2f86cfa,0x0d7f7a52), LL(0x9efe5633,0xfd94d30a),LL(0x5451f934,0x2d8d246b),LL(0x244e6a00,0x2234c6e3),LL(0xddec8c50,0xde2b5b0d), + LL(0xbf776f5b,0x2ce53c5a),LL(0x60357b05,0x6f724071),LL(0x71bf3f7a,0xb2593717),LL(0x440c4a9f,0x87d2501c), LL(0x87b05340,0x440552e1),LL(0x21624c32,0xb7bf7cc8),LL(0x22facddb,0x4155a6ce),LL(0x889837ef,0x5a4228cb), + LL(0xfd4fd671,0xef87d6d6),LL(0xc2daa10e,0xa233687e),LL(0x03c0eb96,0x75622244),LL(0x8bf19be6,0x7632d184), LL(0x40735ff4,0x05d0f8e9),LL(0xc00931f1,0x3a3e6e13),LL(0xdafe3f18,0x31ccde6a),LL(0xcfe51207,0xf381366a), + LL(0x60167d92,0x24c222a9),LL(0x7529f18c,0x62f9d6f8),LL(0x0353b114,0x412397c0),LL(0xef808043,0x334d89dc), LL(0x2a4383ce,0xd9ec63ba),LL(0x5cf92ba0,0xcec8e937),LL(0xc8be74c0,0xfb8b4288),LL(0x105d4391,0x67d6912f), + LL(0x1b913149,0x7b996c46),LL(0x3a4e02da,0x36aae2ef),LL(0x972de594,0xb68aa003),LL(0x4ec6d545,0x284ec70d), LL(0x61391d54,0xf3d2b2d0),LL(0xfe114e92,0x69c5d5d6),LL(0xb4482dff,0xbe0f00b5),LL(0xf5bf33c5,0xe1596fa5), + LL(0x96a71cba,0x10595b56),LL(0xfdcadeb7,0x944938b2),LL(0xfccd8471,0xa282da4c),LL(0x0d37bfe1,0x98ec05f3), LL(0x0698304a,0xe171ce1b),LL(0x21bdf79b,0x2d691444),LL(0x1b21dec1,0xd0cd3b74),LL(0x16a15f71,0x712ecd8b), + LL(0x00fd56e1,0x8d4c00a7),LL(0xf9527c18,0x02ec9692),LL(0x4a3e42e1,0x21c44937),LL(0x1392ae0a,0x9176fbab), LL(0x44b7b618,0x8726f1ba),LL(0xf1de491c,0xb4d7aae9),LL(0x07b582c0,0xf91df7b9),LL(0xef60aa3a,0x7e116c30), + LL(0x466265d7,0x99270f81),LL(0x4df7adf0,0xb15b6fe2),LL(0xf9738f7f,0xfe33b2d3),LL(0xd6d70f95,0x48553ab9), LL(0xc21e94db,0x2cc72ac8),LL(0xbdc0bbee,0x795ac38d),LL(0x2e40478f,0x0a1be449),LL(0x052bde55,0x81bd3394), + LL(0x56b3c4f2,0x63c8dbe9),LL(0x904177cc,0x017a99cf),LL(0x4d010fc1,0x947bbddb),LL(0xbb2c9b21,0xacf9b00b), LL(0x47173611,0x2970bc8d),LL(0xac7d756f,0x1a4cbe08),LL(0x67d541a2,0x06d9f4aa),LL(0x59c2cf44,0xa3e8b689), + LL(0x4d88f1dd,0xaad066da),LL(0x7ad35dea,0xc604f165),LL(0x4478ca67,0x7edc0720),LL(0xba02ce06,0xa10dfae0), LL(0xaf36f4e4,0xeceb1c76),LL(0xaf3f8f48,0x994b2292),LL(0x77c8a68c,0xbf9ed77b),LL(0x51744c9d,0x74f544ea), + LL(0x8113a757,0x82d05bb9),LL(0x8a9885e4,0x4ef2d2b4),LL(0x1aa7865f,0x1e332be5),LL(0x290d1a52,0x22b76b18), LL(0x44351683,0x308a2310),LL(0xa3f22840,0x9d861896),LL(0x841ed947,0x5959ddcd),LL(0x154b73bf,0x0def0c94), + LL(0x4c7c15e0,0xf0105417),LL(0x3a277c32,0x539bfb02),LL(0xf9dccf5f,0xe699268e),LL(0x0247a3bd,0x9f5796a5), LL(0x4f157269,0x8b839de8),LL(0x7a30196b,0xc825c1e5),LL(0xdc8a5a91,0x6ef0aabc),LL(0x498b7fe6,0xf4a8ce6c), + LL(0x70cbac78,0x1cce35a7),LL(0xf6b23958,0x83488e9b),LL(0xd76cb011,0x0341a070),LL(0xae1b2658,0xda6c9d06), LL(0xdd648c52,0xb701fb30),LL(0x52fb9fd1,0x994ca02c),LL(0x6f563086,0x06933117),LL(0x17856bab,0x3d2b8100), + LL(0x5963a46e,0xe89f48c8),LL(0xa99e61c7,0x658ab875),LL(0x4b8517b4,0x6e296f87),LL(0xfc1bc656,0x36c4fcdc), LL(0xa3906def,0xde5227a1),LL(0x62418945,0x9fe95f57),LL(0xfdd96cde,0x20c91e81),LL(0xda4480de,0x5adbe47e), + LL(0x396de2b6,0xa009370f),LL(0xf0ecc7bd,0x98583d4b),LL(0xe51d0672,0xf44f6b57),LL(0x556b1984,0x03d6b078), LL(0xb0b64912,0x27dbdd93),LL(0x15687b09,0x9b3a3434),LL(0x51ec20a9,0x0dba6461),LL(0xff28187c,0xec93db7f), + LL(0x66e48bdd,0x00ff8c24),LL(0x11ccd78e,0x2514f2f9),LL(0xe1250603,0xeba11f4f),LL(0x243fa156,0x8a22cd41), LL(0xb283e4c6,0xa4e58df4),LL(0x8b39783f,0x78c29859),LL(0xa5259809,0x5235aee2),LL(0x0e0227dd,0xc16284b5), + LL(0x1338830d,0xa5f57916),LL(0xd2123fca,0x6d4b8a6b),LL(0xf9c546f8,0x236ea68a),LL(0xfa608d36,0xc1d36873), LL(0x8d436d13,0xcd76e495),LL(0x8fb080af,0xd4d9c221),LL(0xe8ad3fb5,0x665c1728),LL(0xb3d572e0,0xcf1ebe4d), + LL(0x584c5e20,0xa7a8746a),LL(0xb9dc7035,0x267e4ea1),LL(0xb9548c9b,0x593a15cf),LL(0x4bd012f3,0x5e6e2135), LL(0x8c8f936e,0xdf31cc6a),LL(0xb5c241dc,0x8af84d04),LL(0x345efb86,0x63990a6f),LL(0xb9b962cb,0x6fef4e61), +}, +/* digit=3 base_pwr=2^21 */ +{ + LL(0x25722608,0xf6368f09),LL(0x131cf5c6,0x131260db),LL(0xfab4f7ac,0x40eb353b),LL(0x37eee829,0x85c78880), LL(0xc3bdf24e,0x4c1581ff),LL(0xf5c3c5a8,0x5bff75cb),LL(0xa14e6f40,0x35e8c83f),LL(0x0295e0ca,0xb81d1c0f), + LL(0xf43a730f,0xfcde7cc8),LL(0x33ab590e,0xe89b6f3c),LL(0xad03240b,0xc823f529),LL(0x98bea5db,0x82b79afe), LL(0x962fe5de,0x568f2856),LL(0x60c591f3,0x0c590adb),LL(0x4a28a858,0x1fc74a14),LL(0xb3203f4c,0x3b662498), + LL(0x6c39765a,0x91e3cf0d),LL(0xac3cca0b,0xa2db3acd),LL(0xcb953b50,0x288f2f08),LL(0xcf43cf1a,0x2414582c), LL(0x60eee9a8,0x8dec8bbc),LL(0x729aa042,0x54c79f02),LL(0x6532f5d5,0xd81cd5ec),LL(0xcf82e15f,0xa672303a), + LL(0x719c0563,0x376aafa8),LL(0xbc5fc79f,0xcd8ad2dc),LL(0xcb750cd3,0x303fdb9f),LL(0x4418b08e,0x14ff052f), LL(0x3e2d6520,0xf75084cf),LL(0x144ed509,0x7ebdf0f8),LL(0xd3f25b98,0xf43bf0f2),LL(0xa354d837,0x86ad71cf), + LL(0x26f43572,0xb827fe92),LL(0x5d824758,0xdfd3ab5b),LL(0x539094c1,0x315dd23a),LL(0x66623d68,0x85c0e37a), LL(0x7be19ae0,0x575c7972),LL(0xdf0d36b5,0x616a3396),LL(0x26b1ff7e,0xa1ebb3c8),LL(0x140ad453,0x635b9485), + LL(0xda430c0b,0x92bf3cda),LL(0x3a96dac6,0x4702850e),LL(0x15ac326a,0xc91cf0a5),LL(0xab8c25e4,0x95de4f49), LL(0xe265c17c,0xb01bad09),LL(0x087b3881,0x24e45464),LL(0xe1fac5ca,0xd43e583c),LL(0x6ead97a6,0xe17cb318), + LL(0x74dcec46,0x6cc39243),LL(0x54c2b73f,0x33cfc02d),LL(0xf26cd99c,0x82917844),LL(0xd1773f89,0x8819dd95), LL(0x0871f427,0x09572aa6),LL(0xf6f01c34,0x8e0cf365),LL(0xbff1f5af,0x7fa52988),LL(0xe75e8e50,0x4eb357ea), + LL(0x868af75d,0xd9d0c8c4),LL(0x45c8c7ea,0xd7325cff),LL(0xcc81ecb0,0xab471996),LL(0x611824ed,0xff5d55f3), LL(0x1977a0ee,0xbe314541),LL(0x722038c6,0x5085c4c5),LL(0xf94bb495,0x2d5335bf),LL(0xc8e2a082,0x894ad8a6), + LL(0xada35438,0x5c3e2341),LL(0x049b8c4e,0xf4a9fc89),LL(0x9f17cf34,0xbeeb355a),LL(0x6c91fe10,0x3f311e0e), LL(0x92ab9891,0xc2d20038),LL(0x3e8ce9a9,0x257bdcc1),LL(0x88c53bee,0x1b2d9789),LL(0xcdba143a,0x927ce89a), + LL(0x523db280,0xb0a32cca),LL(0x50d43783,0x5c889f8a),LL(0x4897d16f,0x503e04b3),LL(0x08f5f2e8,0x8cdb6e78), LL(0x179c8e74,0x6ab91cf0),LL(0x48211d60,0xd8874e52),LL(0xea851200,0xf948d4d5),LL(0xe6f9840a,0x4076d41e), + LL(0x47b517ea,0xc20e263c),LL(0x30685e5e,0x79a448fd),LL(0xf90631a0,0xe55f6f78),LL(0xa79e6346,0x88a790b1), LL(0x80969fe8,0x62160c7d),LL(0x41491bb9,0x54f92fd4),LL(0x5c957526,0xa6645c23),LL(0xbea3ce7b,0xf44cc5ae), + LL(0x8b1e68b7,0xf7628327),LL(0x303f29d3,0xc731ad7a),LL(0x57d03ecb,0xfe5a9ca9),LL(0x41bc97a7,0x96c0d50c), LL(0x9b4f7f24,0xc4669fe7),LL(0x3d9967ef,0xfdd781d8),LL(0x5d2c208d,0x7892c7c3),LL(0xae545cb3,0x8bf64f7c), + LL(0x467be912,0xc01f862c),LL(0xc73d30cc,0xf4c85ee9),LL(0x6ab83ec7,0x1fa6f4be),LL(0x4e3e3cf9,0xa07a3c1c), LL(0x0c00beb3,0x87f8ef45),LL(0x000d4c3e,0x30e2c2b3),LL(0xfe08bf5b,0x1aa00b94),LL(0x9224ef52,0x32c133aa), + LL(0x32e5685d,0x38df16bb),LL(0x58e6f544,0x68a9e069),LL(0xcdc5ebc6,0x495aaff7),LL(0x378b135f,0xf894a645), LL(0x09e27ecf,0xf316350a),LL(0x58f7179d,0xeced201e),LL(0xe97861ba,0x2eec273c),LL(0xd693be2e,0x47ec2cae), + LL(0xf68367ce,0xfa4c97c4),LL(0xbe5a5755,0xe4f47d0b),LL(0xb298a979,0x17de815d),LL(0xc177dc7d,0xd7eca659), LL(0x49ded0a3,0x20fdbb71),LL(0xfb34d3c5,0x4cb2aad4),LL(0x60858a33,0x2cf31d28),LL(0xa24aa40f,0x3b6873ef), + LL(0x2c11bb37,0x540234b2),LL(0xed4c74a3,0x2d0366dd),LL(0xeec5f25d,0xf9a968da),LL(0x67b63142,0x36601068), LL(0x68d7b6d4,0x07cd6d2c),LL(0x0c842942,0xa8f74f09),LL(0x7768b1ee,0xe2751404),LL(0xfe62aee4,0x4b5f7e89), + LL(0x89070d26,0xc6a77177),LL(0xdd1c8bc7,0xa1f28e4e),LL(0x469e1f17,0xea5f4f06),LL(0xfbdb78e0,0x78fc242a), LL(0x8b0588f1,0xc9c7c592),LL(0x1535921e,0xb6b7a0fd),LL(0xbde5ae35,0xcc5bdb91),LL(0x12ff1864,0xb42c485e), + LL(0xdbab98aa,0xa1113e13),LL(0xa17b1024,0xde9d469b),LL(0xc0462d3a,0x23f48b37),LL(0x7c5c078d,0x3752e537), LL(0x15544eb9,0xe3a86add),LL(0x80fba279,0xf013aea7),LL(0xf22001b5,0x8b5bb76c),LL(0xf02891ab,0xe617ba14), + LL(0x936219d3,0xd39182a6),LL(0xae51cb19,0x5ce1f194),LL(0xbf07a74c,0xc78f8598),LL(0x22cbf1bc,0x6d7158f2), LL(0xe300ce18,0x3b846b21),LL(0x2d11275d,0x35fba630),LL(0xa0239b9b,0x5fe25c36),LL(0xdf05d940,0xd8beb35d), + LL(0x1f7e320d,0x4db02bb0),LL(0x6da320ea,0x0641c364),LL(0x821389a3,0x6d95fa5d),LL(0x8fcd8e3d,0x92699748), LL(0xceb6c143,0x316fef17),LL(0xd933762b,0x67fcb841),LL(0x118b17f8,0xbb837e35),LL(0x9fd24821,0x4b92552f), + LL(0x46aca793,0xae6bc70e),LL(0xe579311b,0x1cf0b0e4),LL(0x5802f716,0x8dc631be),LL(0xbddbee4d,0x099bdc6f), LL(0x0caf8b05,0xcc352bb2),LL(0x72d63df2,0xf74d505a),LL(0x91c4f408,0xb9876d4b),LL(0x9e229b2d,0x1ce18473), + LL(0x83abdb4a,0x49507597),LL(0xdee84b18,0x850fbcb6),LL(0x609e67dc,0x6325236e),LL(0x9336c6d8,0x04d831d9), LL(0xfa12d45d,0x8deaae3b),LL(0x4746e246,0xe425f8ce),LL(0x24f5f31e,0x8004c175),LL(0xad62c3b7,0xaca16d8f), + LL(0x9152f934,0x0dc15a6a),LL(0xed0e12c1,0xf1235e5d),LL(0xda477dac,0xc33c06ec),LL(0xb2ea0006,0x76be8732), LL(0x0c0cd313,0xcf3f7831),LL(0xa614260d,0x3c524553),LL(0xcab22d15,0x31a756f8),LL(0x77827a20,0x03ee10d1), + LL(0x1994ef20,0xd1e059b2),LL(0x638ae318,0x2a653b69),LL(0x2f699010,0x70d5eb58),LL(0x09f5f84a,0x279739f7), LL(0x8b799336,0x5da4663c),LL(0x203c37eb,0xfdfdf14d),LL(0xa1dbfb2d,0x32d8a9dc),LL(0x77d48f9b,0xab40cff0), + LL(0xd20b42d5,0xc018b383),LL(0x9f78845f,0xf9a810ef),LL(0xbdba9df0,0x40af3753),LL(0x131dfdf9,0xb90bdcfc), LL(0xf01ab782,0x18720591),LL(0x6af12a88,0xc823f211),LL(0x0dc14401,0xa51b80f3),LL(0xfb2dfbe3,0xde248f77), + LL(0x0cafe751,0xef5a44e5),LL(0xd4dcd221,0x73997c9c),LL(0xde854024,0x32fd86d1),LL(0xa09b84bb,0xd5b53adc), LL(0xdcedd8d1,0x008d7a11),LL(0x74b32c84,0x406bd1c8),LL(0x05dde8b1,0x5d4472ff),LL(0xfce2b32f,0x2e25f2cd), + LL(0x29dfc254,0xbec0dd5e),LL(0x2b98b267,0x4455fcf6),LL(0xc72df2ad,0x0b4d43a5),LL(0x48a75397,0xea70e6be), LL(0x5820f3bf,0x2aad6169),LL(0x9e37f68f,0xf410d2dd),LL(0x7be5ac83,0x70fb7dba),LL(0x36ec3eec,0x636bb645), + LL(0x9754e21c,0x27104ea3),LL(0x8d63c373,0xbc87a3e6),LL(0x4109db9a,0x483351d7),LL(0x60134da7,0x0fa724e3), LL(0xb0720b16,0x9ff44c29),LL(0x06aceead,0x2dd0cf13),LL(0xe26929a6,0x5942758c),LL(0xb766a92b,0x96c5db92), + LL(0x5f18395e,0xcec7d4c0),LL(0x1f80d032,0xd3f22744),LL(0xcb86075b,0x7a68b37a),LL(0xafef92db,0x074764dd), LL(0x7bc7f389,0xded1e950),LL(0xb9756460,0xc580c850),LL(0x7da48157,0xaeeec2a4),LL(0x82c587b3,0x3f0b4e7f), + LL(0xa9f19c53,0x231c6de8),LL(0x6974e34e,0x5717bd73),LL(0xf1508fa9,0xd9e1d216),LL(0xdadaa124,0x9f112361), LL(0x823b7348,0x80145e31),LL(0xac634069,0x4dd8f0d5),LL(0x2297c258,0xe3d82fc7),LL(0x9cee7431,0x276fcfee), + LL(0x2bc0aea9,0x8eb61b5e),LL(0xde329431,0x4f668fd5),LL(0x38e4b87e,0x03a32ab1),LL(0x73d0ef0b,0xe1374517), LL(0x853ac983,0x1a46f7e6),LL(0x68e78a57,0xc3bdf42e),LL(0x2ea96dd1,0xacf20785),LL(0xf1638460,0xa10649b9), + LL(0x879fbbed,0xf2369f0b),LL(0xda9d1869,0x0ff0ae86),LL(0x56766f45,0x5251d759),LL(0x2be8d0fc,0x4984d8c0), LL(0xd21008f0,0x7ecc95a6),LL(0x3a1a1c49,0x29bd54a0),LL(0xd26c50f3,0xab9828c5),LL(0x51d0d251,0x32c0087c), + LL(0x0c1cdb26,0x9bac3ce6),LL(0x557ca205,0xcd94d947),LL(0x9db1fdcd,0x1b1bd598),LL(0xa3d8b149,0x0eda0108), LL(0x56152fcc,0x95066610),LL(0xe7192b33,0xc2f037e6),LL(0xc92e05a4,0xdeffb41a),LL(0xc2f6c62e,0x1105f6c2), + LL(0x8733913c,0x68e73500),LL(0x3f3adc40,0xcce86163),LL(0x38a278e9,0xf407a942),LL(0x2ab21292,0xd13c1b9d), LL(0x1c74cf5c,0x93ed7ec7),LL(0xf1a4c1b4,0x8887dc48),LL(0x4b3a11f1,0x3830ff30),LL(0x58937cb6,0x358c5a3c), + LL(0x89022829,0x027dc404),LL(0x3b798f79,0x40e93977),LL(0x38be6ead,0x90ad3337),LL(0xf34c0a5d,0x9c23f6bc), LL(0xfbffd8bb,0xd1711a35),LL(0x1949d3dd,0x60fcfb49),LL(0x7825d93a,0x09c8ef4b),LL(0xa0a8c968,0x24233cff), + LL(0xe6d982af,0x67ade46c),LL(0xe7544d7c,0xebb6bf3e),LL(0x3d8bd087,0xd6b9ba76),LL(0x4dc61280,0x46fe382d), LL(0xb5bdbd75,0xbd39a7e8),LL(0xb8f228fe,0xab381331),LL(0xce1c4300,0x0709a77c),LL(0xf337ceac,0x6a247e56), + LL(0x636288be,0x8f34f21b),LL(0xc8a7c305,0x9dfdca74),LL(0xea919e04,0x6decfd1b),LL(0x8e1991f8,0xcdf2688d), LL(0xd0f8a67e,0xe607df44),LL(0x0b58d010,0xd985df4b),LL(0x0c24f8f4,0x57f834c5),LL(0xa0bf01ae,0xe976ef56), + LL(0xa1c32373,0x536395ac),LL(0x734c0a13,0x351027aa),LL(0x5e6bd5bc,0xd2f1b5d6),LL(0x223debed,0x2b539e24), LL(0x0eaa1d71,0xd4994cec),LL(0x661dcf65,0x2a83381d),LL(0x7b54c740,0x5f1aed2f),LL(0xd6dda5ee,0x0bea3fa5), + LL(0x36cc6134,0x9d4fb684),LL(0xc0a443dd,0x8eb9bbf3),LL(0x383b7d2a,0xfc500e2e),LL(0x5b775257,0x7aad621c), LL(0x0a8f7cc0,0x69284d74),LL(0x07562d65,0xe820c2ce),LL(0x499758ee,0xbf9531b9),LL(0x6ee0cc2d,0x73e95ca5), + LL(0xfbaf50a5,0xf61790ab),LL(0x684e0750,0xdf55e76b),LL(0xf176b005,0xec516da7),LL(0x7a2dddc7,0x575553bb), LL(0x553afa73,0x37c87ca3),LL(0x4d55c251,0x315f3ffc),LL(0xaf3e5d35,0xe846442a),LL(0x6495ff28,0x61b91149), + LL(0xfa326dc3,0x23cc95d3),LL(0x18fc2cea,0x1df4da1f),LL(0xd0a37d59,0x24bf9adc),LL(0x320d6e1e,0xb6710053), LL(0x618344d1,0x96f9667e),LL(0xa06445af,0xcc7ce042),LL(0xd68dbc3a,0xa02d8514),LL(0x280b5a5b,0x4ea109e4), + LL(0xb40961bf,0x5741a7ac),LL(0x6aa56bfa,0x4ada5937),LL(0x02b765d1,0x7feb9145),LL(0xe6ad1582,0x561e97be), LL(0xda3982f5,0xbbc4a5b6),LL(0xb546f468,0x0c2659ed),LL(0x59612d20,0xb8e7e6aa),LL(0xac19e8e0,0xd83dfe20), + LL(0xb835398c,0x8530c45f),LL(0xb38a41c2,0x6106a8bf),LL(0x35f5dcdb,0x21e8f9a6),LL(0xcae498ed,0x39707137), LL(0xd8249f00,0x70c23834),LL(0xab2537a0,0x9f14b58f),LL(0x5f61c0c2,0xd043c365),LL(0x09a194a7,0xdc5926d6), + LL(0x8e77738a,0xddec0339),LL(0xfba46426,0xd07a63ef),LL(0xee7f6e86,0x2e58e79c),LL(0xff32d241,0xe59b0459), LL(0x20fa0338,0xc5ec84e5),LL(0xeaff5ace,0x97939ac8),LL(0xb4a38313,0x0310a4e3),LL(0x8f9d9885,0x9115fba2), + LL(0x5fadf8c3,0x8dd710c2),LL(0xce19c0e2,0x66be38a2),LL(0x4cfe5022,0xd42a279c),LL(0x0e24e1b8,0x597bb530), LL(0xc153ca7f,0x3cde86b7),LL(0x707d63bd,0xa8d30fb3),LL(0xbd60d21e,0xac905f92),LL(0x7b9a54ab,0x98e7ffb6), + LL(0xe9726a30,0xd7147df8),LL(0xafce3533,0xb5e216ff),LL(0x2ff1ec40,0xb550b799),LL(0xa1e953fd,0x6b613b87), LL(0x792d5610,0x87b88dba),LL(0xa190fbe1,0x2ee1270a),LL(0x2ef581da,0x02f4e2dc),LL(0xeff82a95,0x016530e4), + LL(0x8fd6ee89,0xcbb93dfd),LL(0x46848fff,0x16d3d986),LL(0x1da47adf,0x600eff24),LL(0x0ad47a71,0x1b9754a0), LL(0x70c33b98,0x8f9266df),LL(0xdf34186e,0xaadc87ae),LL(0x4ad24132,0x0d2ce8e1),LL(0x19946eba,0x8a47cbfc), + LL(0x62b5f3af,0x47feeb66),LL(0x0abb3734,0xcefab561),LL(0x19f35cb1,0x449de60e),LL(0x157f0eb9,0x39f8db14), LL(0x3c61bfd6,0xffaecc5b),LL(0x41216703,0xa5a4d41d),LL(0x224e1cc2,0x7f8fabed),LL(0x871ad953,0x0d5a8186), + LL(0xd22da9a9,0xf10774f7),LL(0xcc8a9b0d,0x45b8a678),LL(0xbdc32cff,0xd9c2e722),LL(0x337202a5,0xbf71b5f5), LL(0x69fc4db9,0x95c57f2f),LL(0x765d01e1,0xb6dad34c),LL(0xcb904635,0x7e0bd13f),LL(0x763a588c,0x61751253), + LL(0x81af2c2d,0xd85c2997),LL(0x81b9d7da,0xc0f7d9c4),LL(0x08533e8d,0x838a34ae),LL(0x311d8311,0x15c4cb08), LL(0x8e121e14,0x97f83285),LL(0x85000a5f,0xeea7dc1e),LL(0x5d256274,0x0c6059b6),LL(0xb95075c0,0xec9beace), + LL(0x1df97828,0x173daad7),LL(0xa8937877,0xbf851cb5),LL(0x01646f3c,0xb083c594),LL(0x50c6d352,0x3bad30cf), LL(0x496bbcea,0xfeb2b202),LL(0x18a1e8ba,0x3cf9fd4f),LL(0x1c066029,0xd26de7ff),LL(0x4e9ed4f8,0x39c81e9e), + LL(0x7b390d35,0xd8be0cb9),LL(0x964aab27,0x01df2bbd),LL(0xc3ef64f8,0x3e8c1a65),LL(0x716ed1dd,0x567291d1), LL(0x5f5406d3,0x95499c6c),LL(0x5ba8e23f,0x71fdda39),LL(0xd5096ece,0xcfeb320e),LL(0xca66dd16,0xbe7ba92b), + LL(0xc6fb5a7d,0x4608d36b),LL(0x6d2dd0e0,0xe3eea15a),LL(0x8f97a36a,0x75b0a3eb),LL(0x1c83de1e,0xf59814cc), LL(0x1c33c23f,0x56c9c5b0),LL(0x6faa4136,0xa96c1da4),LL(0xde316551,0x46bf2074),LL(0x1f756c8f,0x3b866e7b), + LL(0x1495ed6b,0x727727d8),LL(0xb682dce7,0xb2394243),LL(0x758610f3,0x8ab8454e),LL(0x857d72a4,0xc243ce84), LL(0xdbbf370f,0x7b320d71),LL(0x78e0f7ca,0xff9afa37),LL(0xea7b523f,0x0119d1e0),LL(0x058c7d42,0xb997f8cb), + LL(0x37bbb184,0x285bcd2a),LL(0xa45d1fa6,0x51dcec49),LL(0xe29634cb,0x6ade3b64),LL(0x26b86ef1,0x080c94a7), LL(0x2283fbe3,0xba583db1),LL(0x5a9315ed,0x902bddc8),LL(0x86964bec,0x07c1ccb3),LL(0xb6258301,0x78f4eacf), + LL(0x56f90823,0x4bdf3a49),LL(0x741d777b,0xba0f5080),LL(0xf38bf760,0x091d71c3),LL(0x9b625b02,0x9633d50f), LL(0xb8c9de61,0x03ecb743),LL(0x5de74720,0xb4751254),LL(0x74ce1cb2,0x9f9defc9),LL(0x00bd32ef,0x774a4f6a), + LL(0x73848f22,0xaca385f7),LL(0xf3f8558e,0x53dad716),LL(0x93c471f9,0xab7b34b0),LL(0x19644bc7,0xf530e069), LL(0xdd59d31a,0x3d9fb1ff),LL(0x08daa795,0x4382e0df),LL(0xd5cc88d7,0x165c6f4b),LL(0x4a18c900,0xeaa392d5), + LL(0x648024ee,0x94203c67),LL(0x8c2fabcd,0x188763f2),LL(0xbbaec835,0xa80f87ac),LL(0xf29d8d54,0x632c96e0), LL(0x4c00a95e,0x29b0a60e),LL(0xe011e9fa,0x2ef17f40),LL(0x15b77223,0xf6c0e1d1),LL(0x14b04e32,0xaaec2c62), + LL(0x3d84e58c,0xd35688d8),LL(0x958571db,0x2af5094c),LL(0x760682a6,0x4fff7e19),LL(0xe39a407c,0x4cb27077), LL(0x4ff0e321,0x0f59c547),LL(0x1b34c8ff,0x169f34a6),LL(0x52bc1ba7,0x2bff1096),LL(0x83583544,0xa25423b7), + LL(0x0ac8b782,0x5d55d5d5),LL(0x2db3c892,0xff6622ec),LL(0x6b8bb642,0x48fce741),LL(0x69d7e3dc,0x31d6998c), LL(0xcadcaed0,0xdbaf8004),LL(0xd81d053c,0x801b0142),LL(0x59630ec6,0x94b189fc),LL(0xaf762c8e,0x120e9934), + LL(0xfdc6a404,0x53a29aa4),LL(0xa1909948,0x19d8e01e),LL(0xd7e89681,0x3cfcabf1),LL(0x4e132d37,0x3321a50d), LL(0xe9a86111,0xd0496863),LL(0x06a3bc65,0x8c0cde61),LL(0xfc9f8eef,0xaf866c49),LL(0xff7f5141,0x2066350e), + LL(0xe56ddfbd,0x4f8a4689),LL(0xfe32983a,0xea1b0c07),LL(0x873cb8cb,0x2b317462),LL(0x2d93229f,0x658deddc), LL(0x0f64ef58,0x65efaf4d),LL(0x730cc7a8,0xfe43287d),LL(0x3d047d70,0xaebc0c72),LL(0xd92d26c9,0x92efa539), + LL(0x94b56526,0x06e78457),LL(0x0961002d,0x415cb80f),LL(0x76dcb10f,0x89e5c565),LL(0xff9259fe,0x8bbb6982), LL(0x9abc2668,0x4fe8795b),LL(0x1e678fb1,0xb5d4f534),LL(0x7b7da2b9,0x6601f3be),LL(0xa13d6805,0x98da59e2), + LL(0x01799a52,0x190d8ea6),LL(0xb86d2952,0xa20cec41),LL(0x7fff2a7c,0x3062ffb2),LL(0x79f19d37,0x741b32e5), LL(0x4eb57d47,0xf80d8181),LL(0x16aef06b,0x7a2d0ed4),LL(0x1cecb588,0x09735fb0),LL(0xc6061f5b,0x1641caaa), +}, +/* digit=4 base_pwr=2^28 */ +{ + LL(0x20151427,0x7f99824f),LL(0x92430206,0x206828b6),LL(0xe1112357,0xaa9097d7),LL(0x09e414ec,0xacf9a2f2), LL(0x27915356,0xdbdac9da),LL(0x001efee3,0x7e0734b7),LL(0xd2b288e2,0x54fab5bb),LL(0xf62dd09c,0x4c630fc4), + LL(0x1ac2703b,0x8537107a),LL(0x6bc857b5,0xb49258d8),LL(0xbcdaccd1,0x57df14de),LL(0xc4ae8529,0x24ab68d7), LL(0x734e59d0,0x7ed8b5d4),LL(0xc495cc80,0x5f8740c8),LL(0x291db9b3,0x84aedd5a),LL(0x4fb995be,0x80b360f8), + LL(0x5fa067d1,0xae915f5d),LL(0x9668960c,0x4134b57f),LL(0xa48edaac,0xbd3656d6),LL(0xfc1d7436,0xdac1e3e4), LL(0xd81fbb26,0x674ff869),LL(0xb26c33d4,0x449ed3ec),LL(0xd94203e8,0x85138705),LL(0xbeeb6f4a,0xccde538b), + LL(0xa61a76fa,0x55d5c68d),LL(0xca1554dc,0x598b441d),LL(0x773b279c,0xd39923b9),LL(0x36bf9efc,0x33331d3c), LL(0x298de399,0x2d4c848e),LL(0xa1a27f56,0xcfdb8e77),LL(0x57b8ab70,0x94c855ea),LL(0x6f7879ba,0xdcdb9dae), + LL(0x019f2a59,0x7bdff8c2),LL(0xcb4fbc74,0xb3ce5bb3),LL(0x8a9173dd,0xea907f68),LL(0x95a75439,0x6cd3d0d3), LL(0xefed021c,0x92ecc4d6),LL(0x6a77339a,0x09a9f9b0),LL(0x7188c64a,0x87ca6b15),LL(0x44899158,0x10c29968), + LL(0xed6e82ef,0x5859a229),LL(0x65ebaf4e,0x16f338e3),LL(0x5ead67ae,0x0cd31387),LL(0x54ef0bb4,0x1c73d228), LL(0x74a5c8c7,0x4cb55131),LL(0x7f69ad6a,0x01cd2970),LL(0xe966f87e,0xa04d00dd),LL(0x0b7b0321,0xd96fe447), + LL(0x88fbd381,0x342ac06e),LL(0x5c35a493,0x02cd4a84),LL(0x54f1bbcd,0xe8fa89de),LL(0x2575ed4c,0x341d6367), LL(0xd238202b,0xebe357fb),LL(0xa984ead9,0x600b4d1a),LL(0x52436ea0,0xc35c9f44),LL(0xa370751b,0x96fe0a39), + LL(0x7f636a38,0x4c4f0736),LL(0x0e76d5cb,0x9f943fb7),LL(0xa8b68b8b,0xb03510ba),LL(0x9ed07a1f,0xc246780a), LL(0x6d549fc2,0x3c051415),LL(0x607781ca,0xc2953f31),LL(0xd8d95413,0x955e2c69),LL(0x7bd282e3,0xb300fadc), + LL(0x87e9189f,0x81fe7b50),LL(0xf42dda27,0xdb17375c),LL(0xcf0a5904,0x22f7d896),LL(0xebe348e6,0xa0e57c5a), LL(0xf40e3c80,0xa61011d3),LL(0x8db705c5,0xb1189321),LL(0x50fedec3,0x4ed9309e),LL(0x4d6d5c1d,0xdcf14a10), + LL(0x55691342,0x056c265b),LL(0x91049dc7,0xe8e08504),LL(0xc9bae20a,0x131329f5),LL(0xd9dccdb4,0x96c8b3e8), LL(0xfb4ee6b4,0x8c5ff838),LL(0x41e8ccf0,0xfc5a9aeb),LL(0xfae050c6,0x7417b764),LL(0x00452080,0x0953c3d7), + LL(0x38dfe7e8,0x21372682),LL(0x2bb79d4b,0xea417e15),LL(0x76e7cf2d,0x59641f1c),LL(0xea0bcfcc,0x271e3059), LL(0x7253ecbd,0x624c7dfd),LL(0x4fca6186,0x2f552e25),LL(0x4d866e9c,0xcbf84ecd),LL(0xf68d4610,0x73967709), + LL(0xc27901b4,0xa14b1163),LL(0x899b8bf3,0xfd9236e0),LL(0xcbc6da0a,0x42b091ec),LL(0x5ad1d297,0xbb1dac6f), LL(0xa91cf76e,0x80e61d53),LL(0xd31f1ee7,0x4110a412),LL(0x13efcf77,0x2d87c3ba),LL(0xdf450d76,0x1f374bb4), + LL(0x0d188dab,0x5e78e2f2),LL(0xf4b885ef,0xe3968ed0),LL(0x7314570f,0x46c0568e),LL(0x01170521,0x31616338), LL(0x4f0c8afe,0x18e1e7e2),LL(0xdeea78da,0x4caa75ff),LL(0x7c5d8a51,0x82db67f2),LL(0x6f505370,0x36a44d86), + LL(0x0333974f,0xd72c5bda),LL(0x27a70146,0x5db516ae),LL(0x210ef921,0x34705281),LL(0x0c9c38e5,0xbff17a8f), LL(0x12476da1,0x78f4814e),LL(0x33c16980,0xc1e16613),LL(0x424d4bca,0x9e5b386f),LL(0xc85740de,0x4c274e87), + LL(0x6c2f5226,0xb6a9b88d),LL(0x550d7ca8,0x14d1b944),LL(0x1fc41709,0x580c85fc),LL(0x54c6d519,0xc1da368b), LL(0xd5113cf7,0x2b0785ce),LL(0x5a34708f,0x0670f633),LL(0x15cc3f88,0x46e23767),LL(0x50c72c8f,0x1b480cfa), + LL(0x4147519a,0x20288602),LL(0x26b372f0,0xd0981eac),LL(0xa785ebc8,0xa9d4a7ca),LL(0xdbdf58e9,0xd953c50d), LL(0xfd590f8f,0x9d6361cc),LL(0x44e6c917,0x72e9626b),LL(0x22eb64cf,0x7fd96110),LL(0x9eb288f3,0x863ebb7e), + LL(0x6aca8ee7,0x6e6ab761),LL(0xd7b40358,0x97d10b39),LL(0x1e5feb0d,0x1687d377),LL(0x8265a27a,0xc83e50e4), LL(0xc954b313,0x8f75a9fe),LL(0x310d1f61,0xcc2e8f47),LL(0x6557d0e0,0xf5ba81c5),LL(0x3eaf6207,0x25f9680c), + LL(0x4354080b,0xf95c6609),LL(0x7bf2fe1c,0x5225bfa5),LL(0x5c7d98fa,0xc5c004e2),LL(0x019aaf60,0x3561bf1c), LL(0xba151474,0x5e6f9f17),LL(0xb04f6eca,0xdec2f934),LL(0x269acb1e,0x64e368a1),LL(0x0cdda493,0x1332d9e4), + LL(0xdf23de05,0x60d6cf69),LL(0x009339a0,0x66d17da2),LL(0x0a693923,0x9fcac985),LL(0xed7c6a6d,0xbcf057fc), LL(0xf0b5662c,0xc3c5c8c5),LL(0xdcba4f24,0x25318dd8),LL(0x082b69ff,0x60e8cb75),LL(0x1e728c01,0x7c23b3ee), + LL(0x097e4403,0x15e10a0a),LL(0x19854665,0xcb3d0a86),LL(0xd67d4826,0x88d8e211),LL(0x0b9d2839,0xb39af66e), LL(0xbd475ca8,0xa5f94588),LL(0xc077b80b,0xe06b7966),LL(0xda27c26c,0xfedb1485),LL(0xfe0fd5e0,0xd290d33a), + LL(0xf34fb0fa,0xa40bcc47),LL(0x1fb1ab09,0xb4760cc8),LL(0xa273bfe3,0x8fca0993),LL(0xf70b213c,0x13e4fe07), LL(0xfdb05163,0x3bcdb992),LL(0x0c2b19b6,0x8c484b11),LL(0xaaf2e3e2,0x1acb815f),LL(0xb89ff1b4,0xc6905935), + LL(0x586e74e1,0xb2ad6f9d),LL(0x67b80484,0x488883ad),LL(0x369c3ddb,0x758aa2c7),LL(0x9f9afd31,0x8ab74e69), LL(0x5e21beb1,0x10fc2d28),LL(0x318c42f9,0x3484518a),LL(0x53cf40c3,0x377427dc),LL(0x391bc1d9,0x9de0781a), + LL(0x693807e1,0x8faee858),LL(0x4e81ccc7,0xa3865327),LL(0x6f835b84,0x02c30ff2),LL(0x0d3d38d4,0xb604437b), LL(0x5ca1823d,0xb3fc8a98),LL(0x03be0324,0xb82f7ec9),LL(0xcf684a33,0xee36d761),LL(0x9f29bf7d,0x5a01df0e), + LL(0x1306583d,0x686202f3),LL(0x437c622e,0x05b10da0),LL(0x076a7bc8,0xbf9aaa0f),LL(0x8f8f4e43,0x25e94efb), LL(0xfa3dc26d,0x8a35c9b7),LL(0x96ff03c5,0xe0e5fb93),LL(0xebc394ce,0xa77e3843),LL(0x8361de60,0xcede6595), + LL(0xa1993545,0xd27c22f6),LL(0x24d671ba,0xab01cc36),LL(0xa169c28e,0x63fa2877),LL(0x2eb08376,0x925ef904), LL(0x53aa0b32,0x3b2fa3cf),LL(0x71c49d7a,0xb27beb5b),LL(0xd105e27f,0xb60e1834),LL(0x4f68570d,0xd6089788), + LL(0xd6fbc2ac,0x23094ce0),LL(0x815ff551,0x738037a1),LL(0x6bef119c,0xda73b1bb),LL(0xeef506ba,0xdcf6c430), LL(0xe3ef104a,0x00e4fe7b),LL(0x0a065628,0xebdd9a2c),LL(0x8792043e,0x853a81c3),LL(0xb3b59108,0x22ad6ece), + LL(0x39cd297d,0x9fb813c0),LL(0x05bda5d9,0x8ec7e16e),LL(0x0d104b96,0x2834797c),LL(0x7c511510,0xcc11a2e7), LL(0x96ee6380,0x96ca5a53),LL(0xcea38742,0x054c8655),LL(0xd54dfa7d,0xb5946852),LL(0x1f4ab207,0x97c422e7), + LL(0x0c22b540,0xbf907509),LL(0xb7c267d4,0x2cde42aa),LL(0x5ab0d693,0xba18f9ed),LL(0x6e4660d9,0x3ba62aa6), LL(0xab9ea96a,0xb24bf97b),LL(0xe3b60e32,0x5d039642),LL(0x7c4d9bd5,0x4e6a4506),LL(0x7ed4a6a4,0x666c5b9e), + LL(0x8edbd7cc,0xfa3fdcd9),LL(0xc6ccd753,0x4660bb87),LL(0x21e6b64f,0x9ae90820),LL(0xb36bfb3f,0x8a56a713), LL(0x5726d47f,0xabfce096),LL(0x0b1a9a7f,0x9eed01b2),LL(0x4eb74a37,0x30e9cad4),LL(0x53e9666d,0x7b2524cc), + LL(0x8f4b002f,0x6a29683b),LL(0x41f4fc20,0xc2200d7a),LL(0x3a338acc,0xcf3af47a),LL(0xe7128975,0x6539a4fb), LL(0xc33c7fcf,0xcec31c14),LL(0xc7be322b,0x7eb6799b),LL(0x6646f623,0x119ef4e9),LL(0x54d7299b,0x7b7a26a5), + LL(0x403f46f2,0xcb37f08d),LL(0x1a0ec0c7,0x94b8fc43),LL(0xc332142f,0xbb8514e3),LL(0xe80d2a7a,0xf3ed2c33), LL(0xb639126c,0x8d2080af),LL(0xe3553ade,0xf7b6be60),LL(0x1c7e2b09,0x3950aa9f),LL(0x6410f02b,0x847ff958), + LL(0x678a31b0,0x877b7cf5),LL(0x3998b620,0xd50301ae),LL(0xc00fb396,0x734257c5),LL(0x04e672a6,0xf9fb18a0), LL(0xe8758851,0xff8bd8eb),LL(0x5d99ba44,0x1e64e4c6),LL(0x7dfd93b7,0x4b8eaedf),LL(0x04e76b8c,0xba2f2a98), + LL(0xe8053433,0x7d790cba),LL(0x3d2c9585,0xc8e725a0),LL(0xcdd8f5ed,0x58c5c476),LL(0xefa9fe1d,0xd106b952), LL(0x0eff13a9,0x3c5c775b),LL(0xe057b930,0x242442ba),LL(0xc9b70cbd,0xe9f458d4),LL(0xa3cdb89a,0x69b71448), + LL(0x0e2ed742,0x41ee46f6),LL(0x40067493,0x573f1045),LL(0x9d54c304,0xb1e154ff),LL(0x8d3a7502,0x2ad0436a), LL(0x431a8121,0xee4aaa2d),LL(0x886f11ed,0xcd38b3ab),LL(0x034a0eb7,0x57d49ea6),LL(0xf7e85e58,0xd2b773bd), + LL(0x9b5c1f14,0x4a559ac4),LL(0x3e54df2b,0xc444be1a),LL(0xeda41891,0x13aad704),LL(0x5eb5c788,0xcd927bec), LL(0xe48c8a34,0xeb3c8516),LL(0x4b546669,0x1b7ac812),LL(0x594df8ec,0x1815f896),LL(0x79227865,0x87c6a79c), + LL(0x9b56ddbd,0xae02a2f0),LL(0x8a2f1cf3,0x1339b5ac),LL(0x839dff0d,0xf2b569c7),LL(0xfee9a43d,0xb0b9e864), LL(0x77bb064e,0x4ff8ca41),LL(0xfd249f63,0x145a2812),LL(0xf86f689a,0x3ab7beac),LL(0x01d35f5e,0x9bafec27), + LL(0x4265aa91,0x28054c65),LL(0x035efe42,0xa4b18304),LL(0x9639dec7,0x6887b0e6),LL(0x3d52aea5,0xf4b8f6ad), LL(0x971a8a13,0xfb9293cc),LL(0x4c934d07,0x3f159e5d),LL(0x09acbc29,0x2c50e9b1),LL(0x7154d129,0x08eb65e6), + LL(0x30b75c3e,0x4feff589),LL(0x94491c93,0x0bb82fe2),LL(0x89af62bb,0xd8ac377a),LL(0x9685e49f,0xd7b51490), LL(0x04497f19,0xabca9a7b),LL(0x1a7ad13f,0x1b35ed0a),LL(0x3ec86ed6,0x6b601e21),LL(0xce0c76f1,0xda91fcb9), + LL(0xd7ab27e1,0x9e28507b),LL(0x63945b7b,0x7c19a555),LL(0xaafc9827,0x6b43f0a1),LL(0x3aa55b91,0x443b4fbd), LL(0x6962c88f,0x962b2e65),LL(0xce0db0ca,0x139da8d4),LL(0x1b8d6c4f,0xb93f05dd),LL(0x180b9824,0x779cdff7), + LL(0xae57c7b7,0xbba23fdd),LL(0x1b932522,0x345342f2),LL(0x556d4aa3,0xfd9c80fe),LL(0x6525bb61,0xa03907ba), LL(0xff218933,0x38b010e1),LL(0xaa52117b,0xc066b654),LL(0x94f2e6ea,0x8e141920),LL(0x0d32f2b2,0x66a27dca), + LL(0x048b3717,0x69c7f993),LL(0xb178ae1c,0xbf5a989a),LL(0x564f1d6b,0x49fa9058),LL(0xd31fde4e,0x27ec6e15), LL(0x7276e7fc,0x4cce0373),LL(0x89d6bf02,0x64086d79),LL(0x4ccdd979,0x5a72f046),LL(0x47775631,0x909c3566), + LL(0x75dd7125,0x1c07bc6b),LL(0x87a0428d,0xb4c6bc97),LL(0xfdeb6b9d,0x507ece52),LL(0xb2c95432,0xfca56512), LL(0xd0e8bd06,0x15d97181),LL(0xc6bb46ea,0x384dd317),LL(0x3952b624,0x5441ea20),LL(0x4e7dc2fb,0xbcf70dee), + LL(0x6628e8c3,0x372b016e),LL(0xb60a7522,0x07a0d667),LL(0x0a344ee2,0xcf05751b),LL(0x118bdeec,0x0ec09a48), LL(0xd83dce46,0x6e4b3d4e),LL(0x99d2fc6e,0x43a6316d),LL(0x56cf044c,0xa99d8989),LL(0xae3e5fb7,0x7c7f4454), + LL(0xfbabbe92,0xb2e6b121),LL(0xe1330076,0x281850fb),LL(0x97890015,0x093581ec),LL(0x75ff77f5,0x69b1dded), LL(0xab105105,0x7cf0b18f),LL(0xa89ccfef,0x953ced31),LL(0xeb914009,0x3151f85f),LL(0x88ed48ad,0x3c9f1b87), + LL(0x4a7eadcb,0xc9aba1a1),LL(0x522e71cf,0x928e7501),LL(0x3a2e4f83,0xeaede727),LL(0x1ce3bbd3,0x467e10d1), LL(0xb955dcf0,0xf3442ac3),LL(0xd3d5e527,0xba96307d),LL(0xfd77f474,0xf763a10e),LL(0x6a6e1ff0,0x5d744bd0), + LL(0xa777899e,0xd287282a),LL(0xd03f3cde,0xe20eda8f),LL(0x50b07d31,0x6a7e75bb),LL(0x6f379de4,0x0b7e2a94), LL(0x19f593cf,0x31cb64ad),LL(0x1e76ef1d,0x7b1a9e4f),LL(0xb62d609c,0xe18c9c9d),LL(0xe779a650,0x439bad6d), + LL(0xe032f144,0x219d9066),LL(0xe8b2ec6a,0x1db632b8),LL(0xfda12f78,0xff0d0fd4),LL(0x2a25d265,0x56fb4c2d), LL(0x255a03f1,0x5f4e2ee1),LL(0xe96af176,0x61cd6af2),LL(0xd068bc97,0xe0317ba8),LL(0x264b988e,0x927d6bab), + LL(0xe90fb21e,0xa18f07e0),LL(0xbba7fca1,0x00fd2b80),LL(0x95cd67b5,0x20387f27),LL(0xd39707f7,0x5b89a4e7), LL(0x894407ce,0x8f83ad3f),LL(0x6c226132,0xa0025b94),LL(0xf906c13b,0xc79563c7),LL(0x4e7bb025,0x5f548f31), + LL(0xeac6d113,0x2b4c6b8f),LL(0x0e813c76,0xa67e3f9c),LL(0x3fe1f4b9,0x3982717c),LL(0x26d8050e,0x58865819), LL(0xf7f06f20,0x99f3640c),LL(0x2a66ebc2,0xdc610216),LL(0x767a1e08,0x52f2c175),LL(0x5999871b,0x05660e1a), + LL(0x6d3c4693,0x6b0f1762),LL(0x37ed7bea,0xf0e7d627),LL(0xb75b226d,0xc51758c7),LL(0x1f91613b,0x40a88628), LL(0xbbb38ce0,0x889dbaa7),LL(0xbddcad81,0xe0404b65),LL(0x8bc9671f,0xfebccd3a),LL(0xee1f5375,0xfbf9a357), + LL(0x28f33398,0x5dc169b0),LL(0x72e90f65,0xb07ec11d),LL(0xfaab1eb1,0xae7f3b4a),LL(0x5f17538a,0xd970195e), LL(0x0181e640,0x52b05cbe),LL(0x2643313d,0xf5debd62),LL(0x5df31f82,0x76148154),LL(0x3a9e13c5,0x23e03b33), + LL(0x4fde0c1f,0xff758949),LL(0xe5b6ec20,0xbf8a1abe),LL(0x87e1db6c,0x702278fb),LL(0x35ed658f,0xc447ad7a), LL(0x03d0ccf2,0x48d4aa38),LL(0x819a7c03,0x80acb338),LL(0x6e17cecc,0x9bc7c89e),LL(0x03be1d82,0x46736b8b), + LL(0xc0432f96,0xd65d7b60),LL(0xdeb5442f,0xddebe7a3),LL(0x7dff69a2,0x79a25307),LL(0x02cf3122,0x37a56d94), LL(0xf2350d0a,0x8bab8aed),LL(0x037b0d9a,0x13c3f276),LL(0x44c65cae,0xc664957c),LL(0xc2e71a88,0x88b44089), + LL(0x5cb02664,0xdb88e5a3),LL(0x8686c72e,0x5d4c0bf1),LL(0xa682d53e,0xea3d9b62),LL(0x0b2ad431,0x9b605ef4), LL(0xc69645d0,0x71bac202),LL(0x6a1b66e7,0xa115f03a),LL(0x158f4dc4,0xfe2c563a),LL(0x4d12a78c,0xf715b3a0), + LL(0xd413213a,0x8f7f0a48),LL(0xc04becdb,0x2035806d),LL(0x5d8587f5,0xecd34a99),LL(0x9f6d3a71,0x4d8c3079), LL(0x8d95a8f6,0x1b2a2a67),LL(0xf2110d0d,0xc58c9d7d),LL(0xcf8fba3f,0xdeee81d5),LL(0x0c7cdf68,0xa42be3c0), + LL(0xd43b5eaa,0x2126f742),LL(0xdfa59b85,0x054a0766),LL(0x126bfd45,0x9d0d5e36),LL(0x384f8a8f,0xa1f8fbd7), LL(0xd563fccc,0x317680f5),LL(0xf280a928,0x48ca5055),LL(0x27b578cf,0xe00b81b2),LL(0x2994a514,0x10aad918), + LL(0xb7bdc953,0xd9e07b62),LL(0x5bc086dd,0x9f0f6ff2),LL(0x655eee77,0x09d1ccff),LL(0x5bef7df1,0x45475f79), LL(0x86f702cc,0x3faa28fa),LL(0x0f021f07,0x92e60905),LL(0x7f8fa8c6,0xe9e62968),LL(0xf036ea2c,0xbd71419a), + LL(0x6028da9a,0x171ee1cc),LL(0xc251f573,0x5352fe1a),LL(0x3fa997f4,0xf8ff236e),LL(0xa5749d5f,0xd831b6c9), LL(0xe350e2c2,0x7c872e1d),LL(0x1e0ce403,0xc56240d9),LL(0x6974f5cb,0xf9deb077),LL(0x961c3728,0x7d50ba87), + LL(0x5a3a2518,0xd6f89426),LL(0xc6303d43,0xcf817799),LL(0x619e5696,0x510a0471),LL(0x3a5e307b,0xab049ff6), LL(0xfeb13ec7,0xe4cdf9b0),LL(0x9d8ff90c,0xd5e97117),LL(0x9afa96af,0xf6f64d06),LL(0x9d2012a2,0x00d0bf5e), + LL(0x358bcdc0,0xe63f301f),LL(0x0a9d47f8,0x07689e99),LL(0x4f43d43a,0x1f689e2f),LL(0x90920904,0x4d542a16), LL(0x9ca0a707,0xaea293d5),LL(0x8ac68065,0xd061fe45),LL(0x0090008c,0x1033bf1b),LL(0xc08a6db6,0x29749558), + LL(0xc1d5d034,0x74b5fc59),LL(0x67e215e0,0xf712e9f6),LL(0x860200e6,0xfd520cbd),LL(0x3ea22588,0x0229acb4), LL(0xfff0c82e,0x9cd1e14c),LL(0x59c69e73,0x87684b62),LL(0x96ccb989,0xda85e61c),LL(0xa3d06493,0x2d5dbb02), + LL(0xe86b173c,0xf22ad33a),LL(0xa79ff0e3,0xe8e41ea5),LL(0xdd0d0c10,0x01d2d725),LL(0x032d28f9,0x31f39088), LL(0x7829839e,0x7b3f71e1),LL(0x4502ae58,0x0cf691b4),LL(0xbefc6115,0xef658dbd),LL(0xb3ab5314,0xa5cd6ee5), + LL(0x5f1d2347,0x206c8d7b),LL(0x4cc2253a,0x794645ba),LL(0x58389e08,0xd517d8ff),LL(0x9f847288,0x4fa20dee), LL(0xd797770a,0xeba072d8),LL(0xbf429e26,0x7360c91d),LL(0x80af8279,0x7200a3b3),LL(0x82dadce3,0x6a1c9150), + LL(0xc35d8794,0x0ee6d3a7),LL(0x0356bae5,0x042e6558),LL(0x643322fd,0x9f59698d),LL(0x50a61967,0x9379ae15), LL(0xfcc9981e,0x64b9ae62),LL(0x6d2934c6,0xaed3d631),LL(0x5e4e65eb,0x2454b302),LL(0xf9950428,0xab09f647), +}, +/* digit=5 base_pwr=2^35 */ +{ + LL(0x22248acc,0xb2083a12),LL(0x3264e366,0x1f6ec0ef),LL(0x5afdee28,0x5659b704),LL(0xe6430bb5,0x7a823a40), LL(0xe1900a79,0x24592a04),LL(0xc9ee6576,0xcde09d4a),LL(0x4b5ea54a,0x52b6463f),LL(0xd3ca65a7,0x1efe9ed3), + LL(0x305406dd,0xe27a6dbe),LL(0xdd5d1957,0x8eb7dc7f),LL(0x387d4d8f,0xf54a6876),LL(0xc7762de4,0x9c479409), LL(0x99b30778,0xbe4d5b5d),LL(0x6e793682,0x25380c56),LL(0xdac740e3,0x602d37f3),LL(0x1566e4ae,0x140deabe), + LL(0xafd32acf,0x4481d067),LL(0xe1f71ccf,0xd8f0fcca),LL(0xb596f2da,0xd208dd0c),LL(0x9aad93f9,0xd049d730), LL(0x42ab580e,0xc79f263d),LL(0x23f707b4,0x09411bb1),LL(0x835e0eda,0x8cfde1ff),LL(0x90f03402,0x72707490), + LL(0xc49a861e,0xeaee6126),LL(0xe14f0d06,0x024f3b65),LL(0xc69bfc17,0x51a3f1e8),LL(0xa7686381,0xc3c3a8e9), LL(0xb103d4c8,0x3400752c),LL(0x9218b36b,0x02bc4613),LL(0x7651504a,0xc67f75eb),LL(0xd02aebfa,0xd6848b56), + LL(0xc30fa92b,0xbd9802e6),LL(0x9a552784,0x5a70d96d),LL(0x3f83169b,0x9085c4ea),LL(0x06908228,0xfa9423bb), LL(0xfe97a5b9,0x2ffebe12),LL(0x71b99118,0x85da6049),LL(0x63178846,0x9cbc2f7f),LL(0x9153218e,0xfd96bc70), + LL(0x1782269b,0x958381db),LL(0x2597e550,0xae34bf79),LL(0x5f385153,0xbb5c6064),LL(0xe3088048,0x6f0e96af), LL(0x77884456,0xbf6a0215),LL(0x69310ea7,0xb3b5688c),LL(0x04fad2de,0x17c94295),LL(0x17896d4d,0xe020f0e5), + LL(0x0976505f,0x730ba0ab),LL(0x095e2ec5,0x567f6813),LL(0x6331ab71,0x47062010),LL(0x41d22b9f,0x72cfa977), LL(0x8a2373da,0x33e55ead),LL(0x7ba45a68,0xa8d0d5f4),LL(0x03029d15,0xba1d8f9c),LL(0xfc55b9f3,0x8f34f1cc), + LL(0xbbe5a1a9,0xcca4428d),LL(0x3126bd67,0x8187fd5f),LL(0x48105826,0x0036973a),LL(0xb8bd61a0,0xa39b6663), LL(0x2d65a808,0x6d42deef),LL(0x94636b19,0x4969044f),LL(0xdd5d564c,0xf611ee47),LL(0xd2873077,0x7b2f3a49), + LL(0x300eb294,0x94157d45),LL(0x169c1494,0x2b2a656e),LL(0xd3a47aa9,0xc000dd76),LL(0xa6243ea4,0xa2864e4f), LL(0xdb89842e,0x82716c47),LL(0x61479fb7,0x12dfd7d7),LL(0xe0b2f6dc,0x3b9a2c56),LL(0xd7f85d67,0x46be862a), + LL(0x0f82b214,0x03b0d8dd),LL(0xf103cbc6,0x460c34f9),LL(0x18d79e19,0xf32e5c03),LL(0xa84117f8,0x8b8888ba), LL(0xc0722677,0x8f3c37dc),LL(0x1c1c0f27,0x10d21be9),LL(0xe0f7a0c6,0xd47c8468),LL(0xadecc0e0,0x9bf02213), + LL(0x42b48b99,0x0baa7d12),LL(0x48424096,0x1bcb665d),LL(0xebfb5cfb,0x8b847cd6),LL(0x9ad4d10d,0x87c2ae56), LL(0x0de36726,0xf1cbb122),LL(0x3fdfbd21,0xe7043c68),LL(0x4e79d460,0x4bd0826a),LL(0x4bd1a2cb,0x11f5e598), + LL(0xb7fe7b6e,0x97554160),LL(0x400a3fb2,0x7d16189a),LL(0xe328ca1e,0xd73e9bea),LL(0xe793d8cc,0x0dd04b97), LL(0x506db8cc,0xa9c83c9b),LL(0xcf38814c,0x5cd47aae),LL(0xb64b45e6,0x26fc430d),LL(0xd818ea84,0x079b5499), + LL(0xc1c24a3b,0xebb01102),LL(0x1c161c1a,0xca24e568),LL(0x36f00a4a,0x103eea69),LL(0x76176c7b,0x9ad76ee8), LL(0x538e0ff7,0x97451fc2),LL(0x6604b3b0,0x94f89809),LL(0x3249cfd7,0x6311436e),LL(0x41224f69,0x27b4a7bd), + LL(0xe0ac2941,0x03b5d21a),LL(0xc2d31937,0x279b0254),LL(0xcac992d0,0x3307c052),LL(0xefa8b1f3,0x6aa7cb92), LL(0x0d37c7a5,0x5a182580),LL(0x342d5422,0x13380c37),LL(0xd5d2ef92,0x92ac2d66),LL(0x030c63c6,0x035a70c9), + LL(0x4ce4f152,0xc16025dd),LL(0xf9df7c06,0x1f419a71),LL(0x91e4bb14,0x6d5b2214),LL(0x839fb4ce,0xfc43c6cc), LL(0x925d6b2d,0x49f06591),LL(0x62186598,0x4b37d9d3),LL(0xd01b1629,0x8c54a971),LL(0x51d50e05,0xe1a9c29f), + LL(0x71ba1861,0x5109b785),LL(0xd0c8f93d,0x48b22d5c),LL(0x8633bb93,0xe8fa84a7),LL(0x5aebbd08,0x53fba6ba), LL(0xe5eea7d8,0x7ff27df3),LL(0x68ca7158,0x521c8796),LL(0xce6f1a05,0xb9d5133b),LL(0xfd0ebee4,0x2d50cd53), + LL(0xc5a3ef16,0xc82115d6),LL(0xba079221,0x993eff9d),LL(0x4b5da81c,0xe4da2c5e),LL(0x8033fd85,0x9a89dbdb), LL(0x2b892891,0x60819ebf),LL(0x5d14a4d5,0x53902b21),LL(0xd7fda421,0x6ac35051),LL(0x61c83284,0xcc6ab885), + LL(0xf74cff17,0x14eba133),LL(0xecb813f2,0x240aaa03),LL(0x6f665bee,0xcfbb6540),LL(0xa425ad73,0x084b1fe4), LL(0xd081f6a6,0x009d5d16),LL(0xeef82c90,0x35304fe8),LL(0xaa9eaa22,0xf20346d5),LL(0xac1c91e3,0x0ada9f07), + LL(0x968a6144,0xa6e21678),LL(0x07b31a1e,0x54c1f77c),LL(0x5781fbe1,0xd6bb787e),LL(0xe31f1c4a,0x61bd2ee0), LL(0x781105fc,0xf25aa1e9),LL(0x7b2f8e80,0x9cf2971f),LL(0xcdff919b,0x26d15412),LL(0x34bc896e,0x01db4ebe), + LL(0xb40df1cf,0x7d9b3e23),LL(0x94e971b4,0x59337373),LL(0x669cf921,0xbf57bd14),LL(0x0c1a1064,0x865daedf), LL(0x83279125,0x3eb70bd3),LL(0x34ecdaab,0xbc3d5b9f),LL(0x5f755caf,0x91e3ed7e),LL(0xd41e6f02,0x49699f54), + LL(0xd4a7a15b,0x185770e1),LL(0xeaac87e7,0x08f3587a),LL(0x473133ea,0x352018db),LL(0x04fd30fc,0x674ce719), LL(0x088b3e0e,0x7b8d9835),LL(0x5d0d47a1,0x7a0356a9),LL(0x6474a3c4,0x9d9e7659),LL(0xff66966c,0x61ea48a7), + LL(0x0f3e4834,0x30417758),LL(0x17a9afcb,0xfdbb21c2),LL(0x2f9a67b3,0x756fa17f),LL(0xa245c1a8,0x2a6b2421), LL(0x4af02291,0x64be2794),LL(0x2a5804fe,0xade465c6),LL(0xa6f08fd7,0x8dffbd39),LL(0xaa14403b,0xc4efa84c), + LL(0x442b0f5c,0xa1b91b2a),LL(0xcf997736,0xb748e317),LL(0xcee90e16,0x8d1b62bf),LL(0x0b2078c0,0x907ae271), LL(0x0c9bcddd,0xdf31534b),LL(0x39adce83,0x043fb054),LL(0xd826846a,0x99031043),LL(0xb144f393,0x61a9c0d6), + LL(0x47718427,0xdab48046),LL(0x6e830f8b,0xdf17ff9b),LL(0xe49a1347,0x408d7ee8),LL(0x91c1d4ae,0x6ac71e23), LL(0x1defd73c,0xc8cbb9fd),LL(0xbbbbfec5,0x19840657),LL(0x9e7ef8ea,0x39db1cb5),LL(0x64105f30,0x78aa8296), + LL(0xa3738c29,0xa3d9b7f0),LL(0xbc3250a3,0x0a2f235a),LL(0x445e4caf,0x55e506f6),LL(0x33475f7a,0x0974f73d), LL(0x5ba2f5a8,0xd37dbba3),LL(0x6af40066,0x542c6e63),LL(0xc5d73e2c,0x26d99b53),LL(0x6c3ca33e,0x06060d7d), + LL(0x065fef4a,0xcdbef1c2),LL(0xfd5b92e3,0x77e60f7d),LL(0x26708350,0xd7c549f0),LL(0x34f121bf,0x201b3ad0), LL(0x0334fc14,0x5fcac2a1),LL(0x344552f6,0x8a9a9e09),LL(0x97653082,0x7dd8a1d3),LL(0x79d4f289,0x5fc0738f), + LL(0x17d2d8c3,0x787d244d),LL(0x70830684,0xeffc6345),LL(0xe4f73ae5,0x5ddb96dd),LL(0x172549a5,0x8efb14b1), LL(0x2245ae7a,0x6eb73eee),LL(0xea11f13e,0xbca4061e),LL(0x30b01f5d,0xb577421d),LL(0x782e152c,0xaa688b24), + LL(0xbd3502ba,0x67608e71),LL(0xb4de75a0,0x4ef41f24),LL(0xfd6125e5,0xb08dde5e),LL(0xa409543f,0xde484825), LL(0x65cc2295,0x1f198d98),LL(0x6e0edfa2,0x428a3771),LL(0xadf35fc7,0x4f9697a2),LL(0xf7cac3c7,0x01a43c79), + LL(0x0fd3659a,0xb05d7059),LL(0xbb7f2d9a,0x8927f30c),LL(0x8cf984d3,0x4023d1ac),LL(0x02897a45,0x32125ed3), LL(0x3d414205,0xfb572dad),LL(0xe3fa82a9,0x73000ef2),LL(0xf10a5581,0x4c0868e9),LL(0x6b0b3ca5,0x5b61fc67), + LL(0x7cae440c,0xc1258d5b),LL(0x402b7531,0x21c08b41),LL(0xde932321,0xf61a8955),LL(0x2d1408af,0x3568faf8), LL(0x9ecf965b,0x71b15e99),LL(0xe917276f,0xf14ed248),LL(0x820cf9e2,0xc6f4caa1),LL(0x18d83c7e,0x681b20b2), + LL(0xc6c01120,0x6cde738d),LL(0xae70e0db,0x71db0813),LL(0x74afe18c,0x95fc0644),LL(0x129e2be7,0x34619053), LL(0xdb2a3b15,0x80615cea),LL(0xdb4c7073,0x0a49a19e),LL(0x8fd2d367,0x0e1b84c8),LL(0x033fb8aa,0xd74bf462), + LL(0x533ef217,0x889f6d65),LL(0xc3ca2e87,0x7158c7e4),LL(0xdc2b4167,0xfb670dfb),LL(0x844c257f,0x75910a01), LL(0xcf88577d,0xf336bf07),LL(0xe45e2ace,0x22245250),LL(0x7ca23d85,0x2ed92e8d),LL(0x2b812f58,0x29f8be4c), + LL(0x076fe12b,0xdd9ebaa7),LL(0xae1537f9,0x3f2400cb),LL(0x17bdfb46,0x1aa93528),LL(0x67883b41,0xc0f98430), LL(0x0170911d,0x5590ede1),LL(0x34d4b17f,0x7562f5bb),LL(0x1826b8d2,0xe1fa1df2),LL(0x6bd80d59,0xb40b796a), + LL(0x3467ba92,0xd65bf197),LL(0xf70954b0,0x8c9b46db),LL(0x0e78f15d,0x97c8a0f3),LL(0x85a4c961,0xa8f3a69a), LL(0x61e4ce9b,0x4242660f),LL(0x6ea6790c,0xbf06aab3),LL(0xec986416,0xc6706f8e),LL(0x9a9fc225,0x9e56dec1), + LL(0x9a9898d9,0x527c46f4),LL(0x5633cdef,0xd799e77b),LL(0x7d9e4297,0x24eacc16),LL(0x6b1cb734,0xabb61cea), LL(0xf778443c,0xbee2e8a7),LL(0x29de2fe6,0x3bb42bf1),LL(0x3003bb6f,0xcbed86a1),LL(0xd781cdf6,0xd3918e6c), + LL(0x9a5103f1,0x4bee3271),LL(0xf50eac06,0x5243efc6),LL(0x6adcc119,0xb8e122cb),LL(0xc0b80a08,0x1b7faa84), LL(0x6dfcd08c,0x32c3d1bd),LL(0x0be427de,0x129dec4e),LL(0x1d263c83,0x98ab679c),LL(0xcef64eff,0xafc83cb7), + LL(0x2fa6be76,0x85eb6088),LL(0x1328cbfe,0x892585fb),LL(0xcf618dda,0xc154d3ed),LL(0x3abaf26e,0xc44f601b), LL(0x2be1fdfd,0x7bf57d0b),LL(0x21137fee,0xa833bd2d),LL(0x2db591a8,0x9353af36),LL(0x5562a056,0xc76f26dc), + LL(0x3fdf5a51,0x1d87e47d),LL(0x55c9cab0,0x7afb5f93),LL(0x89e0586e,0x91bbf58f),LL(0x0d843709,0x7c72c018), LL(0x99b5c3dc,0xa9a5aafb),LL(0x3844aeb0,0xa48a0f1d),LL(0xb667e482,0x7178b7dd),LL(0x6e23a59a,0x453985e9), + LL(0x01b25dd8,0x4a54c860),LL(0xfb897c8a,0x0dd37f48),LL(0x0ea90cd9,0x5f8aa610),LL(0x16d5830d,0xc8892c68), LL(0xef514ca5,0xeb4befc0),LL(0xe72c9ee6,0x478eb679),LL(0xdbc40d5f,0x9bca20da),LL(0xdde4f64a,0xf015de21), + LL(0xeaf4b8a5,0xaa6a4de0),LL(0x4bc60e32,0x68cfd9ca),LL(0x7fd15e70,0x668a4b01),LL(0xf27dc09d,0xd9f0694a), LL(0xba708bcd,0xf6c3cad5),LL(0x5bb95c2a,0x5cd2ba69),LL(0x33c0a58f,0xaa28c1d3),LL(0xabc77870,0x23e274e3), + LL(0xdfd20a4a,0x44c3692d),LL(0x81a66653,0x091c5fd3),LL(0x09a0757d,0x6c0bb691),LL(0x667343ea,0x9072e8b9), LL(0x80848bec,0x31d40eb0),LL(0x79fd36cc,0x95bd480a),LL(0x65ed43f5,0x01a77c61),LL(0x2e0d40bf,0xafccd127), + LL(0x1cc1884b,0xeccfc82d),LL(0x5d4753b4,0xc85ac201),LL(0x658e099f,0xc7a6caac),LL(0x04b27390,0xcf46369e), LL(0x506467ea,0xe2e7d049),LL(0x37cdeccc,0x481b63a2),LL(0xed80143a,0x4029abd8),LL(0xbcb00b88,0x28bfe3c7), + LL(0x0643d84a,0x3bec1009),LL(0xabd11041,0x885f3668),LL(0xf83a34d6,0xdb02432c),LL(0x719ceebe,0x32f7b360), LL(0xdad1fe7a,0xf06c7837),LL(0x5441a0b0,0x60a157a9),LL(0xe2d47550,0x704970e9),LL(0x271b9020,0xcd2bd553), + LL(0x33e24a0b,0xff57f82f),LL(0xf2565079,0x9cbee23f),LL(0xeb5f5825,0x16353427),LL(0xe948d662,0x276feec4), LL(0xda10032b,0xd1b62bc6),LL(0xf0e72a53,0x718351dd),LL(0x2420e7ba,0x93452076),LL(0x3a00118d,0x96368fff), + LL(0x150a49e4,0x00ce2d26),LL(0x3f04706b,0x0c28b636),LL(0x58b196d0,0xbad65a46),LL(0xec9f8b7c,0x6c8455fc), LL(0x2d71867e,0xe90c895f),LL(0xedf9f38c,0x5c0be31b),LL(0xd8f6ec04,0x2a37a15e),LL(0x8cd85251,0x239639e7), + LL(0x9c7c4c6b,0xd8975315),LL(0xd7409af7,0x603aa3c0),LL(0x007132fb,0xb8d53d0c),LL(0xa6849238,0x68d12af7), LL(0xbf5d9279,0xbe0607e7),LL(0xaada74ce,0x9aa50055),LL(0xba7e8ccb,0xe81079cb),LL(0xa5f4ff5e,0x610c71d1), + LL(0x5aa07093,0x9e2ee1a7),LL(0xa75da47c,0xca84004b),LL(0x3de75401,0x074d3951),LL(0xbb311592,0xf938f756), LL(0x00a43421,0x96197618),LL(0x07bc78c8,0x39a25362),LL(0x0a171276,0x278f710a),LL(0x8d1a8f08,0xb28446ea), + LL(0xe3b6a661,0x184781bf),LL(0xe6d279f7,0x7751cb1d),LL(0xc59eb662,0xf8ff95d6),LL(0x58d3dea7,0x186d90b7), LL(0xdfb4f754,0x0e4bb6c1),LL(0x2b2801dc,0x5c5cf56b),LL(0x1f54564d,0xc561e452),LL(0xf0dd7f13,0xb4fb8c60), + LL(0x33ff98c7,0xf8849630),LL(0xcf17769c,0x9619fffa),LL(0x1bfdd80a,0xf8090bf6),LL(0x422cfe63,0x14d9a149), LL(0x6f6df9ea,0xb354c360),LL(0x218f17ea,0xdbcf770d),LL(0x79eb3480,0x207db7c8),LL(0x559b6a26,0x213dbda8), + LL(0x29fc81b3,0xac4c200b),LL(0x171d87c1,0xebc3e09f),LL(0x1481aa9e,0x91799530),LL(0x92e114fa,0x051b92e1), LL(0xecb5537f,0xdf8f92e9),LL(0x290c7483,0x44b1b2cc),LL(0x2adeb016,0xa711455a),LL(0x81a10c2c,0x964b6856), + LL(0xcec03623,0x4f159d99),LL(0xef3271ea,0x05532225),LL(0xc5ee4849,0xb231bea3),LL(0x7094f103,0x57a54f50), LL(0x9598b352,0x3e2d421d),LL(0x67412ab4,0xe865a49c),LL(0x1cc3a912,0xd2998a25),LL(0x0c74d65d,0x5d092808), + LL(0x4088567a,0x73f45908),LL(0x1f214a61,0xeb6b280e),LL(0xcaf0c13d,0x8c9adc34),LL(0xf561fb80,0x39d12938), LL(0xbc6edfb4,0xb2dc3a5e),LL(0xfe4d210e,0x7485b1b1),LL(0xe186ae72,0x062e0400),LL(0x6eeb3b88,0x91e32d5c), + LL(0x4be59224,0x6df574d7),LL(0x716d55f3,0xebc88ccc),LL(0xcad6ed33,0x26c2e6d0),LL(0x0d3e8b10,0xc6e21e7d), LL(0x5bcc36bb,0x2cc5840e),LL(0x7da74f69,0x9292445e),LL(0x4e5193a8,0x8be8d321),LL(0x8df06413,0x3ec23629), + LL(0xb134defa,0xc7e9ae85),LL(0x1bb2d475,0x6073b1d0),LL(0x2863c00d,0xb9ad615e),LL(0x525f4ac4,0x9e29493d), LL(0x4e9acf4f,0xc32b1dea),LL(0xa50db88d,0x3e1f01c8),LL(0x04da916c,0xb05d70ea),LL(0xd865803e,0x714b0d0a), + LL(0x9920cb5e,0x4bd493fc),LL(0x92c7a3ac,0x5b44b1f7),LL(0xbcec9235,0xa2a77293),LL(0xcd378553,0x5ee06e87), LL(0xda621607,0xceff8173),LL(0x99f5d290,0x2bb03e4c),LL(0xa6f734ac,0x2945106a),LL(0xd25c4732,0xb5056604), + LL(0xe079afee,0x5945920c),LL(0x6789831f,0x686e17a0),LL(0xb74a5ae5,0x5966bee8),LL(0x1e258d46,0x38a673a2), LL(0x83141c95,0xbd1cc1f2),LL(0x0e96e486,0x3b2ecf4f),LL(0x74e5fc78,0xcd3aa896),LL(0x2482fa7a,0x415ec10c), + LL(0x80503380,0x15234419),LL(0xd314b392,0x513d917a),LL(0x63caecae,0xb0b52f4e),LL(0x2dc7780b,0x07bf22ad), LL(0xe4306839,0xe761e8a1),LL(0x5dd7feaa,0x1b3be962),LL(0x74c778f1,0x4fe728de),LL(0x5e0070f6,0xf1fa0bda), + LL(0x6ec3f510,0x85205a31),LL(0xd2980475,0x2c7e4a14),LL(0x6f30ebfd,0xde3c19c0),LL(0xd4b7e644,0xdb1c1f38), LL(0x5dce364a,0xfe291a75),LL(0x058f5be3,0xb7b22a3c),LL(0x37fea38c,0x2cd2c302),LL(0x2e17be17,0x2930967a), + LL(0x0c061c65,0x87f009de),LL(0xedc6ed44,0xcb014aac),LL(0x3bafb1eb,0x49bd1cb4),LL(0x282d3688,0x81bd8b5c), LL(0xf01a17af,0x1cdab87e),LL(0xe710063b,0x21f37ac4),LL(0x42fc8193,0x5a6c5676),LL(0x56a6015c,0xf4753e70), + LL(0xa15b0a44,0x020f795e),LL(0x8958a958,0x8f37c8d7),LL(0xa4b675b5,0x63b7e89b),LL(0x0fc31aea,0xb4fb0c0c), LL(0xa7ff1f2e,0xed95e639),LL(0x619614fb,0x9880f5a3),LL(0x947151ab,0xdeb6ff02),LL(0xa868dcdb,0x5bc5118c), + LL(0x4c20cea5,0xd8da2055),LL(0x14c4d69a,0xcac2776e),LL(0x622d599b,0xcccb22c1),LL(0x68a9bb50,0xa4ddb653), LL(0x1b4941b4,0x2c4ff151),LL(0x6efba588,0xe1ff19b4),LL(0xc48345e0,0x35034363),LL(0x1e29dfc4,0x45542e3d), + LL(0x349f7aed,0xf197cb91),LL(0x8fca8420,0x3b2b5a00),LL(0x23aaf6d8,0x7c175ee8),LL(0x35af32b6,0x54dcf421), LL(0x27d6561e,0x0ba14307),LL(0xd175b1e2,0x879d5ee4),LL(0x99807db5,0xc7c43673),LL(0x9cd55bcd,0x77a54455), + LL(0x0105c072,0xe6c2ff13),LL(0x8dda7da4,0x18f7a99f),LL(0x0e2d35c1,0x4c301820),LL(0xd9cc6c82,0x06a53ca0), LL(0xf1aa1d9e,0xaa21cc1e),LL(0x4a75b1e8,0x32414334),LL(0x0ebe9fdc,0x2a6d1328),LL(0x98a4755a,0x16bd173f), + LL(0x2133ffd9,0xfbb9b245),LL(0x830f1a20,0x39a8b2f1),LL(0xd5a1f52a,0x484bc97d),LL(0xa40eddf8,0xd6aebf56), LL(0x76ccdac6,0x32257acb),LL(0x1586ff27,0xaf4d36ec),LL(0xf8de7dd1,0x8eaa8863),LL(0x88647c16,0x0045d5cf), +}, +/* digit=6 base_pwr=2^42 */ +{ + LL(0xc005979d,0xa6f3d574),LL(0x6a40e350,0xc2072b42),LL(0x8de2ecf9,0xfca5c156),LL(0xa515344e,0xa8c8bf5b), LL(0x114df14a,0x97aee555),LL(0xfdc5ec6b,0xd4374a4d),LL(0x2ca85418,0x754cc28f),LL(0xd3c41f78,0x71cb9e27), + LL(0x03605c39,0x89105079),LL(0xa142c96c,0xf0843d9e),LL(0x16923684,0xf3744934),LL(0xfa0a2893,0x732caa2f), LL(0x61160170,0xb2e8c270),LL(0x437fbaa3,0xc32788cc),LL(0xa6eda3ac,0x39cd818e),LL(0x9e2b2e07,0xe2e94239), + LL(0x0260e52a,0x6967d39b),LL(0x90653325,0xd42585cc),LL(0x21ca7954,0x0d9bd605),LL(0x81ed57b3,0x4fa20877), LL(0xe34a0bbe,0x60c1eff8),LL(0x84f6ef64,0x56b0040c),LL(0xb1af8483,0x28be2b24),LL(0xf5531614,0xb2278163), + LL(0x5922ac1c,0x8df27545),LL(0xa52b3f63,0xa7b3ef5c),LL(0x71de57c4,0x8e77b214),LL(0x834c008b,0x31682c10), LL(0x4bd55d31,0xc76824f0),LL(0x17b61c71,0xb6d1c086),LL(0xc2a5089d,0x31db0903),LL(0x184e5d3f,0x9c092172), + LL(0xc00cc638,0xdd7ced5b),LL(0x61278fc2,0x1a2015eb),LL(0x6a37f8d6,0x2e8e5288),LL(0xe79933ad,0xc457786f), LL(0x2c51211a,0xb3fe4cce),LL(0x24c20498,0xad9b10b2),LL(0xd28db5e5,0x90d87a4f),LL(0x3aca2fc3,0x698cd105), + LL(0xe91b536d,0x4f112d07),LL(0x9eba09d6,0xceb982f2),LL(0x197c396f,0x3c157b2c),LL(0x7b66eb24,0xe23c2d41), LL(0x3f330d37,0x480c57d9),LL(0x79108deb,0xb3a4c8a1),LL(0xcb199ce5,0x702388de),LL(0xb944a8d4,0x0b019211), + LL(0x840bb336,0x24f2a692),LL(0xa669fa7b,0x7c353bdc),LL(0xdec9c300,0xda20d6fc),LL(0xa13a4f17,0x625fbe2f), LL(0xdbc17328,0xa2b1b61a),LL(0xa9515621,0x008965bf),LL(0xc620ff46,0x49690939),LL(0x8717e91c,0x182dd27d), + LL(0xea6c3997,0x5ace5035),LL(0xc2610bef,0x54259aaa),LL(0x3c80dd39,0xef18bb3f),LL(0x5fc3fa39,0x6910b95b), LL(0x43e09aee,0xfce2f510),LL(0xa7675665,0xced56c9f),LL(0xd872db61,0x10e265ac),LL(0xae9fce69,0x6982812e), + LL(0xce800998,0x29be11c6),LL(0xb90360d9,0x72bb1752),LL(0x5a4ad590,0x2c193197),LL(0x9fc1dbc0,0x2ba2f548), LL(0xe490ebe0,0x7fe4eebb),LL(0x7fae11c0,0x12a0a4cd),LL(0xe903ba37,0x7197cf81),LL(0xde1c6dd8,0xcf7d4aa8), + LL(0x3fd5684c,0x92af6bf4),LL(0x80360aa1,0x2b26eecf),LL(0x00546a82,0xbd960f30),LL(0xf59ad8fe,0x407b3c43), LL(0x249c82ba,0x86cae5fe),LL(0x2463744c,0x9e0faec7),LL(0x94916272,0x87f551e8),LL(0x6ceb0615,0x033f9344), + LL(0x8be82e84,0x1e5eb0d1),LL(0x7a582fef,0x89967f0e),LL(0xa6e921fa,0xbcf687d5),LL(0xd37a09ba,0xdfee4cf3), LL(0xb493c465,0x94f06965),LL(0x7635c030,0x638b9a1c),LL(0x66f05e9f,0x76667864),LL(0xc04da725,0xccaf6808), + LL(0x768fccfc,0xca2eb690),LL(0xb835b362,0xf402d37d),LL(0xe2fdfcce,0x0efac0d0),LL(0xb638d990,0xefc9cdef), LL(0xd1669a8b,0x2af12b72),LL(0x5774ccbd,0x33c536bc),LL(0xfb34870e,0x30b21909),LL(0x7df25aca,0xc38fa2f7), + LL(0xbf81f3f5,0x74c5f02b),LL(0xaf7e4581,0x0525a5ae),LL(0x433c54ae,0x88d2aaba),LL(0x806a56c5,0xed9775db), LL(0xc0edb37d,0xd320738a),LL(0x66cc1f51,0x25fdb6ee),LL(0x10600d76,0xac661d17),LL(0xbdd1ed76,0x931ec1f3), + LL(0x19ee43f1,0x65c11d62),LL(0x60829d97,0x5cd57c3e),LL(0x984be6e8,0xd26c91a3),LL(0x8b0c53bd,0xf08d9309), LL(0xc016e4ea,0x94bc9e5b),LL(0x11d43d2b,0xd3916839),LL(0x73701155,0x886c5ad7),LL(0x20b00715,0xe0377626), + LL(0xaa80ba59,0x7f01c9ec),LL(0x68538e51,0x3083411a),LL(0xe88128af,0x970370f1),LL(0x91dec14b,0x625cc3db), LL(0x01ac3107,0xfef9666c),LL(0xd5057ac3,0xb2a8d577),LL(0x92be5df7,0xb0f26299),LL(0x00353924,0xf579c8e5), + LL(0x1341ed7a,0xb8fa3d93),LL(0xa7b59d49,0x4223272c),LL(0x83b8c4a4,0x3dcb1947),LL(0xed1302e4,0x4e413c01), LL(0xe17e44ce,0x6d999127),LL(0x33b3adfb,0xee86bf75),LL(0x25aa96ca,0xf6902fe6),LL(0xe5aae47d,0xb73540e4), + LL(0x1b4a158c,0x32801d7b),LL(0x27e2a369,0xe571c99e),LL(0x10d9f197,0x40cb76c0),LL(0x3167c0ae,0xc308c289), LL(0xeb7958f2,0xa6ef9dd3),LL(0x300879b1,0xa7226dfc),LL(0x7edf0636,0x6cd0b362),LL(0x7bc37eed,0x4efbce6c), + LL(0x8d699021,0x75f92a05),LL(0x772566e3,0x586d4c79),LL(0x761ad23a,0x378ca5f1),LL(0x1465a8ac,0x650d86fc), LL(0x842ba251,0x7a4ed457),LL(0x42234933,0x6b65e3e6),LL(0x31aad657,0xaf1543b7),LL(0xcbfec369,0xa4cefe98), + LL(0x9f47befb,0xb587da90),LL(0x41312d13,0x6562e9fb),LL(0xeff1cefe,0xa691ea59),LL(0x05fc4cf6,0xcc30477a), LL(0x0b0ffd3d,0xa1632461),LL(0x5b355956,0xa1f16f3b),LL(0x4224ec24,0x5b148d53),LL(0xf977012a,0xdc834e7b), + LL(0xb2c69dbc,0x7bfc5e75),LL(0x03c3da6c,0x3aa77a29),LL(0xca910271,0xde0df03c),LL(0x7806dc55,0xcbd5ca4a), LL(0x6db476cb,0xe1ca5807),LL(0x5f37a31e,0xfde15d62),LL(0xf41af416,0xf49af520),LL(0x7d342db5,0x96c5c5b1), + LL(0xeb4ceb9b,0x155c43b7),LL(0x4e77371a,0x2e993010),LL(0x675d43af,0x1d2987da),LL(0x8599fd72,0xef2bc1c0), LL(0x9342f6b2,0x96894b7b),LL(0x7c8e71f0,0x201eadf2),LL(0x4a1f3efc,0xf3479d9f),LL(0x702a9704,0xe0f8a742), + LL(0xb3eba40c,0xeafd44b6),LL(0xc1c1e0d0,0xf9739f29),LL(0x619d505e,0x0091471a),LL(0x9d7c263e,0xc15f9c96), LL(0x83afbe33,0x5be47285),LL(0x04f1e092,0xa3b6d6af),LL(0x751a9d11,0xe76526b9),LL(0x9a4ae4d2,0x2ec5b26d), + LL(0x02f6fb8d,0xeb66f4d9),LL(0x96912164,0x4063c561),LL(0x80ef3000,0xeb7050c1),LL(0xeaa5b3f0,0x288d1c33), LL(0x07806fd8,0xe87c68d6),LL(0x4bbbf50f,0xb2f7f9d5),LL(0xac8d6627,0x25972f3a),LL(0x10e8c13b,0xf8547774), + LL(0x872b4a60,0xcc50ef6c),LL(0x4613521b,0xab2a34a4),LL(0x983e15d1,0x39c5c190),LL(0x59905512,0x61dde5df), LL(0x9f2275f3,0xe417f621),LL(0x451d894b,0x0750c8b6),LL(0x78b0bdaa,0x75b04ab9),LL(0x458589bd,0x3bfd9fd4), + LL(0xee9120b6,0xf1013e30),LL(0x23a4743e,0x2b51af93),LL(0x48d14d9e,0xea96ffae),LL(0x698a1d32,0x71dc0dbe), LL(0x0180cca4,0x914962d2),LL(0xc3568963,0x1ae60677),LL(0x437bc444,0x8cf227b1),LL(0xc9962c7a,0xc650c83b), + LL(0xfe7ccfc4,0x23c2c7dd),LL(0x1b929d48,0xf925c89d),LL(0x06783c33,0x4460f74b),LL(0xa590475a,0xac2c8d49), LL(0xb807bba0,0xfb40b407),LL(0x69ff8f3a,0x9d1e362d),LL(0xcbef64a4,0xa33e9681),LL(0x332fb4b2,0x67ece5fa), + LL(0x739f10e3,0x6900a99b),LL(0xff525925,0xc3341ca9),LL(0xa9e2d041,0xee18a626),LL(0x29580ddd,0xa5a83685), LL(0x9d7de3cd,0xf3470c81),LL(0x2062cf9c,0xedf02586),LL(0xc010edb0,0xf43522fa),LL(0x13a4b1ae,0x30314135), + LL(0xdb22b94b,0xc792e02a),LL(0xa1eaa45b,0x993d8ae9),LL(0xcd1e1c63,0x8aad6cd3),LL(0xc5ce688a,0x89529ca7), LL(0xe572a253,0x2ccee3aa),LL(0x02a21efb,0xe02b6438),LL(0xc9430358,0xa7091b6e),LL(0x9d7db504,0x06d1b1fa), + LL(0xc4744733,0x58846d32),LL(0x379f9e34,0x40517c71),LL(0x130ef6ca,0x2f65655f),LL(0xf1f3503f,0x526e4488), LL(0x7ee4a976,0x8467bd17),LL(0x921363d1,0x1d9dc913),LL(0xb069e041,0xd8d24c33),LL(0x2cdf7f51,0x5eb5da0a), + LL(0x197b994f,0x1c0f3cb1),LL(0x2843eae9,0x3c95a6c5),LL(0xa6097ea5,0x7766ffc9),LL(0xd723b867,0x7bea4093), LL(0x4db378f9,0xb48e1f73),LL(0xe37b77ac,0x70025b00),LL(0xaf24ad46,0x943dc8e7),LL(0x16d00a85,0xb98a15ac), + LL(0x2743b004,0x3adc38ba),LL(0x334415ee,0xb1c7f4f7),LL(0x1e62d05a,0xea43df8f),LL(0x9d76a3b6,0x32618905), LL(0xa23a0f46,0x2fbd0bb5),LL(0x6a01918c,0x5bc971db),LL(0xb4743f94,0x7801d94a),LL(0x676ae22b,0xb94df65e), + LL(0xaf95894c,0xaafcbfab),LL(0x276b2241,0x7b9bdc07),LL(0x5bdda48b,0xeaf98362),LL(0xa3fcb4df,0x5977faf2), LL(0x052c4b5b,0xbed042ef),LL(0x067591f0,0x9fe87f71),LL(0x22f24ec7,0xc89c73ca),LL(0xe64a9f1b,0x7d37fa9e), + LL(0x15562627,0x2710841a),LL(0xc243b034,0x2c01a613),LL(0x2bc68609,0x1d135c56),LL(0x8b03f1f6,0xc2ca1715), LL(0x3eb81d82,0xc9966c2d),LL(0x8f6df13e,0xc02abf4a),LL(0x8f72b43b,0x77b34bd7),LL(0x360c82b0,0xaff6218f), + LL(0x8d55b9d2,0x0aa5726c),LL(0x99e9bffb,0xdc0adbe9),LL(0xefb9e72a,0x9097549c),LL(0x9dfb3111,0x16755712), LL(0xf26847f9,0xdd8bf984),LL(0xdfb30cb7,0xbcb8e387),LL(0x5171ef9c,0xc1fd32a7),LL(0x389b363f,0x977f3fc7), + LL(0xf4babda0,0x116eaf2b),LL(0xf7113c8e,0xfeab68bd),LL(0xb7def526,0xd1e3f064),LL(0xe0b3fa02,0x1ac30885), LL(0x40142d9d,0x1c5a6e7b),LL(0x30921c0b,0x839b5603),LL(0x36a116a3,0x48f301fa),LL(0xcfd9ee6d,0x380e1107), + LL(0x58854be1,0x7945ead8),LL(0xcbd4d49d,0x4111c12e),LL(0x3a29c2ef,0xece3b1ec),LL(0x8d3616f5,0x6356d404), LL(0x594d320e,0x9f0d6a8f),LL(0xf651ccd2,0x0989316d),LL(0x0f8fdde4,0x6c32117a),LL(0xa26a9bbc,0x9abe5cc5), + LL(0x9723f671,0xcff560fb),LL(0x7f3d593c,0x21b2a12d),LL(0x24ba0696,0xe4cb18da),LL(0xc3543384,0x186e2220), LL(0x88312c29,0x722f64e0),LL(0x17dc7752,0x94282a99),LL(0x5a85ee89,0x62467bbf),LL(0xf10076a0,0xf435c650), + LL(0x43b3a50b,0xc9ff1539),LL(0x1a53efbc,0x7132130c),LL(0xf7b0c5b7,0x31bfe063),LL(0x4ea994cc,0xb0179a7d), LL(0xc85f455b,0x12d064b3),LL(0x8f6e0062,0x47259328),LL(0xb875d6d9,0xf64e590b),LL(0xad92bcc7,0x22dd6225), + LL(0xb9c3bd6d,0xb658038e),LL(0xfbba27c8,0x00cdb0d6),LL(0x1062c45d,0x0c681337),LL(0x2d33407d,0xd8515b8c), LL(0x8cbb5ecf,0xcb8f699e),LL(0xc608d7d8,0x8c4347f8),LL(0xbb3e00db,0x2c11850a),LL(0xecb49d19,0x20a8dafd), + LL(0x45ee2f40,0xbd781480),LL(0x416b60cf,0x75e354af),LL(0x8d49a8c4,0xde0b58a1),LL(0xfa359536,0xe40e94e2), LL(0x62accd76,0xbd4fa59f),LL(0x8c762837,0x05cf466a),LL(0x448c277b,0xb5abda99),LL(0x48b13740,0x5a9e01bf), + LL(0x326aad8d,0x9d457798),LL(0xc396f7e7,0xbdef4954),LL(0xc253e292,0x6fb274a2),LL(0x1cfe53e7,0x2800bf0a), LL(0x44438fd4,0x22426d31),LL(0x5e259f9a,0xef233923),LL(0x03f66264,0x4188503c),LL(0x7f9fdfab,0x9e5e7f13), + LL(0x5fcc1aba,0x565eb76c),LL(0x59b5bff8,0xea632548),LL(0xaab6d3fa,0x5587c087),LL(0x6ce39c1b,0x92b639ea), LL(0x953b135c,0x0706e782),LL(0x425268ef,0x7308912e),LL(0x090e7469,0x599e92c7),LL(0x9bc35e75,0x83b90f52), + LL(0x244975b3,0x4750b3d0),LL(0x11965d72,0xf3a44358),LL(0x9c8dc751,0x179c6774),LL(0xd23d9ff0,0xff18cdfe), LL(0x2028e247,0xc4013833),LL(0xf3bfbc79,0x96e280e2),LL(0xd0880a84,0xf60417bd),LL(0x2a568151,0x263c9f3d), + LL(0x2d2ce811,0x36be15b3),LL(0xf8291d21,0x846dc0c2),LL(0x789fcfdb,0x5cfa0ecb),LL(0xd7535b9a,0x45a0beed), LL(0x96d69af1,0xec8e9f07),LL(0x599ab6dc,0x31a7c5b8),LL(0xf9e2e09f,0xd36d45ef),LL(0xdcee954b,0x3cf49ef1), + LL(0x086cff9b,0x6be34cf3),LL(0x39a3360f,0x88dbd491),LL(0x0dbfbd1d,0x1e96b8cc),LL(0xcb7e2552,0xc1e5f7bf), LL(0x28819d98,0x0547b214),LL(0x7aea9dcb,0xc770dd9c),LL(0x041d68c8,0xaef0d4c7),LL(0x13cb9ba8,0xcc2b9818), + LL(0xfe86c607,0x7fc7bc76),LL(0x502a9a95,0x6b7b9337),LL(0xd14dab63,0x1948dc27),LL(0xdae047be,0x249dd198), LL(0xa981a202,0xe8356584),LL(0x3a893387,0x3531dd18),LL(0xc85c7209,0x1be11f90),LL(0xe2a52b5a,0x93d2fe1e), + LL(0xec6d6b97,0x8225bfe2),LL(0xbd0aa5de,0x9cf6d6f4),LL(0x54779f5f,0x911459cb),LL(0x86aeb1f3,0x5649cddb), LL(0x3f26ce5a,0x32133579),LL(0x550f431e,0xc289a102),LL(0x73b84c6f,0x559dcfda),LL(0xee3ac4d7,0x84973819), + LL(0xf2606a82,0xb51e55e6),LL(0x90f2fb57,0xe25f7061),LL(0xb1a4e37c,0xacef6c2a),LL(0x5dcf2706,0x864e359d), LL(0x7ce57316,0x479e6b18),LL(0x3a96b23d,0x2cab2500),LL(0x8ef16df7,0xed489862),LL(0xef3758b5,0x2056538c), + LL(0xf15d3101,0xa7df865e),LL(0x61b553d7,0x80c5533a),LL(0x4ed14294,0x366e1997),LL(0xb3c0bcd6,0x6620741f), LL(0xedc45418,0x21d1d9c4),LL(0xc1cc4a9d,0x005b859e),LL(0xa1c462f0,0xdf01f630),LL(0xf26820c7,0x15d06cf3), + LL(0x3484be47,0x9f7f24ee),LL(0x4a0c902f,0x2ff33e96),LL(0x5a0bc453,0x00bdf457),LL(0x1aa238db,0x2378dfaf), LL(0x856720f2,0x272420ec),LL(0x96797291,0x2ad9d95b),LL(0x768a1558,0xd1242cc6),LL(0x5cc86aa8,0x2e287f8b), + LL(0x990cecaa,0x796873d0),LL(0x675d4080,0xade55f81),LL(0x21f0cd84,0x2645eea3),LL(0xb4e17d02,0x7a1efa0f), LL(0x037cc061,0xf6858420),LL(0xd5d43e12,0x682e05f0),LL(0x27218710,0x59c36994),LL(0x3f7cd2fc,0x85cbba4d), + LL(0x7a3cd22a,0x726f9729),LL(0x4a628397,0x9f8cd5dc),LL(0xc23165ed,0x17b93ab9),LL(0x122823d4,0xff5f5dbf), LL(0x654a446d,0xc1e4e4b5),LL(0x677257ba,0xd1a9496f),LL(0xde766a56,0x6387ba94),LL(0x521ec74a,0x23608bc8), + LL(0x6688c4d4,0x16a522d7),LL(0x07373abd,0x9d6b4282),LL(0xb42efaa3,0xa62f07ac),LL(0xe3b90180,0xf73e00f7), LL(0x49421c3e,0x36175fec),LL(0x3dcf2678,0xc4e44f9b),LL(0x7220f09f,0x76df436b),LL(0x3aa8b6cf,0x172755fb), + LL(0x446139cc,0xbab89d57),LL(0x5fe0208f,0x0a0a6e02),LL(0x11e5d399,0xcdbb63e2),LL(0xa8977f0b,0x33ecaa12), LL(0xf7c42664,0x59598b21),LL(0xab65d08a,0xb3e91b32),LL(0xf4502526,0x035822ee),LL(0x720a82a9,0x1dcf0176), + LL(0x3d589e02,0x50f8598f),LL(0xb1d63d2c,0xdf0478ff),LL(0x1571cd07,0x8b8068bd),LL(0xd79670cd,0x30c3aa4f), LL(0x941ade7f,0x25e8fd4b),LL(0x32790011,0x3d1debdc),LL(0x3a3f9ff0,0x65b6dcbd),LL(0x793de69c,0x282736a4), + LL(0xd41d3bd3,0xef69a0c3),LL(0x07a26bde,0xb533b8c9),LL(0xdb2edf9f,0xe2801d97),LL(0xe1877af0,0xdc4a8269), LL(0x3d590dbe,0x6c1c5851),LL(0xee4e9357,0x84632f6b),LL(0x79b33374,0xd36d36b7),LL(0x9bbca2e6,0xb46833e3), + LL(0xf7fc0586,0x37893913),LL(0x66bf4719,0x385315f7),LL(0xb31855dc,0x72c56293),LL(0x849061fe,0xd1416d4e), LL(0x51047213,0xbeb3ab78),LL(0xf040c996,0x447f6e61),LL(0x638b1d0c,0xd06d310d),LL(0xbad1522e,0xe28a413f), + LL(0x82003f86,0x685a76cb),LL(0x0bcdbca3,0x610d07f7),LL(0x9ca4c455,0x6ff66021),LL(0xcea10eec,0x7df39b87), LL(0xe22db218,0xb9255f96),LL(0x08a34c44,0x8cc6d9eb),LL(0x859f9276,0xcd4ffb86),LL(0x50d07335,0x8fa15eb2), + LL(0xcf2c24b5,0xdf553845),LL(0x52f9c3ba,0x89f66a9f),LL(0xe4a7ceb3,0x8f22b5b9),LL(0x0e134686,0xaffef809), LL(0x8eb8fac2,0x3e53e1c6),LL(0x28aec98e,0x93c1e4eb),LL(0x32a43bcb,0xb6b91ec5),LL(0xb2d74a51,0x2dbfa947), + LL(0xca84bad7,0xe065d190),LL(0xad58e65c,0xfb13919f),LL(0xf1cb6e31,0x3c41718b),LL(0x06d05c3f,0x688969f0), LL(0x21264d45,0xd4f94ce7),LL(0x7367532b,0xfdfb65e9),LL(0x0945a39d,0x5b1be8b1),LL(0x2b8baf3b,0x229f789c), + LL(0x6f49f15d,0xd8f41f3e),LL(0x907f0792,0x678ce828),LL(0xfca6e867,0xc69ace82),LL(0xd01dcc89,0x106451ae), LL(0x19fc32d2,0x1bb4f7f0),LL(0xb00c52d2,0x64633dfc),LL(0xad9ea445,0x8f13549a),LL(0xfb323705,0x99a3bf50), + LL(0x534d4dbc,0x0c9625a2),LL(0xc2a2fea3,0x45b8f1d1),LL(0xa530fc1a,0x76ec21a1),LL(0x9e5bd734,0x4bac9c2a), LL(0x7b4e3587,0x5996d76a),LL(0x1182d9e3,0x0045cdee),LL(0x1207f13d,0x1aee24b9),LL(0x97345a41,0x66452e97), + LL(0x9f950cd0,0x16e5b054),LL(0xd7fdd075,0x9cc72fb1),LL(0x66249663,0x6edd61e7),LL(0xf043cccb,0xde4caa4d), LL(0x55c7ac17,0x11b1f57a),LL(0x1a85e24d,0x779cbd44),LL(0xe46081e7,0x78030f86),LL(0x8e20f643,0xfd4a6032), + LL(0x0a750c0f,0xcc7a6488),LL(0x4e548e83,0x39bacfe3),LL(0x0c110f05,0x3d418c76),LL(0xb1f11588,0x3e4daa4c), LL(0x5ffc69ff,0x2733e7b5),LL(0x92053127,0x46f147bc),LL(0xd722df94,0x885b2434),LL(0xe6fc6b7c,0x6a444f65), +}, +/* digit=7 base_pwr=2^49 */ +{ + LL(0xc3f16ea8,0x7a1a465a),LL(0xb2f1d11c,0x115a461d),LL(0x6c68a172,0x4767dd95),LL(0xd13a4698,0x3392f2eb), LL(0xe526cdc7,0xc7a99ccd),LL(0x22292b81,0x8e537fdc),LL(0xa6d39198,0x76d8cf69),LL(0x2446852d,0xffc5ff43), + LL(0xa90567e6,0x97b14f7e),LL(0xb6ae5cb7,0x513257b7),LL(0x9f10903d,0x85454a3c),LL(0x69bc3724,0xd8d2c9ad), LL(0x6b29cb44,0x38da9324),LL(0x77c8cbac,0xb540a21d),LL(0x01918e42,0x9bbfe435),LL(0x56c3614e,0xfffa707a), + LL(0xd4e353b7,0x0ce4e3f1),LL(0xef46b0a0,0x062d8a14),LL(0x574b73fd,0x6408d5ab),LL(0xd3273ffd,0xbc41d1c9), LL(0x6be77800,0x3538e1e7),LL(0xc5655031,0x71fe8b37),LL(0x6b9b331a,0x1cd91621),LL(0xbb388f73,0xad825d0b), + LL(0x1cb76219,0x56c2e05b),LL(0x71567e7e,0x0ec0bf91),LL(0x61c4c910,0xe7076f86),LL(0xbabc04d9,0xd67b085b), LL(0x5e93a96a,0x9fb90459),LL(0xfbdc249a,0x7526c1ea),LL(0xecdd0bb7,0x0d44d367),LL(0x9dc0d695,0x95399917), + LL(0x9e240d18,0x61360ee9),LL(0xb4b94466,0x057cdcac),LL(0x2fe5325c,0xe7667cd1),LL(0x21974e3b,0x1fa297b5), LL(0xdb083d76,0xfa4081e7),LL(0xf206bd15,0x31993be6),LL(0x14c19f8c,0x8949269b),LL(0xa9d92357,0x21468d72), + LL(0xa4c506ec,0x2ccbc583),LL(0xd1acfe97,0x957ed188),LL(0x12f1aea2,0x8baed833),LL(0x8325362d,0xef2a6cb4), LL(0x8e195c43,0x130dde42),LL(0x0e6050c6,0xc842025a),LL(0x08686a5d,0x2da972a7),LL(0xe508b4a8,0xb52999a1), + LL(0x10a5a8bd,0xd9f090b9),LL(0x096864da,0xca91d249),LL(0x3f67dbc1,0x8e6a93be),LL(0xf5f4764c,0xacae6fba), LL(0xd21411a0,0x1563c6e0),LL(0xda0a4ad8,0x28fa787f),LL(0x908c8030,0xd524491c),LL(0x4c795f07,0x1257ba0e), + LL(0xceca9754,0x83f49167),LL(0x4b7939a0,0x426d2cf6),LL(0x723fd0bf,0x2555e355),LL(0xc4f144e2,0xa96e6d06), LL(0x87880e61,0x4768a8dd),LL(0xe508e4d5,0x15543815),LL(0xb1b65e15,0x09d7e772),LL(0xac302fa0,0x63439dd6), + LL(0xc14e35c2,0xb93f802f),LL(0x4341333c,0x71735b7c),LL(0x16d4f362,0x03a25104),LL(0xbf433c8e,0x3f4d069b), LL(0xf78f5a7c,0x0d83ae01),LL(0x7c4eed07,0x50a8ffbe),LL(0x76e10f83,0xc74f8906),LL(0x9ddaf8e1,0x7d080966), + LL(0x698e04cc,0xb11df8e1),LL(0x169005c8,0x877be203),LL(0x4f3c6179,0x32749e8c),LL(0x7853fc05,0x2dbc9d0a), LL(0x9454d937,0x187d4f93),LL(0xb4800e1b,0xe682ce9d),LL(0x165e68e8,0xa9129ad8),LL(0xbe7f785b,0x0fe29735), + LL(0x5b9e02b7,0x5303f40c),LL(0x35ee04e8,0xa37c9692),LL(0x34d6632b,0x5f46cc20),LL(0x96ac545b,0x55ef72b2), LL(0x7b91b062,0xabec5c1f),LL(0xbb33e821,0x0a79e1c7),LL(0x3a9f4117,0xbb04b428),LL(0xfd2a475a,0x0de1f28f), + LL(0x3a4434b4,0x31019ccf),LL(0x1a7954dc,0xa3458111),LL(0xe34972a7,0xa9dac80d),LL(0x74f6b8dd,0xb043d054), LL(0x11137b1a,0x021c319e),LL(0xed5cc03f,0x00a754ce),LL(0xcbea5ad4,0x0aa2c794),LL(0x70c015b6,0x093e67f4), + LL(0xc97e3f6b,0x72cdfee9),LL(0xb6da7461,0xc10bcab4),LL(0xb59806b9,0x3b02d2fc),LL(0xa1de6f47,0x85185e89), LL(0x0eb6c4d4,0x39e6931f),LL(0xd4fa5b04,0x4d4440bd),LL(0x34be7eb8,0x5418786e),LL(0x9d7259bc,0x6380e521), + LL(0xd598d710,0x20ac0351),LL(0xcb3a4da4,0x272c4166),LL(0xca71de1f,0xdb82fe1a),LL(0xd8f54b0f,0x746e79f2), LL(0x4b573e9b,0x6e7fc736),LL(0xfd4b5040,0x75d03f46),LL(0x0b98d87b,0x5c1cc36d),LL(0x1f472da1,0x513ba3f1), + LL(0xabb177dd,0x79d0af26),LL(0x7891d564,0xf82ab568),LL(0x72232173,0x2b6768a9),LL(0x8c1f6619,0xefbb3bb0), LL(0xa6d18358,0xb29c11db),LL(0xb0916d3a,0x519e2797),LL(0x9188e290,0xd4dc18f0),LL(0x98b0ca7f,0x648e86e3), + LL(0x983c38b5,0x859d3145),LL(0x637abc8b,0xb14f176c),LL(0xcaff7be6,0x2793fb9d),LL(0x35a66a5a,0xebe5a55f), LL(0x9f87dc59,0x7cec1dcd),LL(0xfbdbf560,0x7c595cd3),LL(0x26eb3257,0x5b543b22),LL(0xc4c935fd,0x69080646), + LL(0x81e9ede3,0x7f2e4403),LL(0xcaf6df0a,0x243c3894),LL(0x1c073b11,0x7c605bb1),LL(0xba6a4a62,0xcd06a541), LL(0x49d4e2e5,0x29168949),LL(0x4af66880,0x33649d07),LL(0xe9a85035,0xbfc0c885),LL(0xfc410f4b,0xb4e52113), + LL(0x78a6513b,0xdca3b706),LL(0x9edb1943,0x92ea4a2a),LL(0xdb6e2dd8,0x02642216),LL(0x9fd57894,0x9b45d0b4), LL(0xc69d11ae,0x114e70db),LL(0x4c57595f,0x1477dd19),LL(0xec77c272,0xbc2208b4),LL(0xdb68f59c,0x95c5b4d7), + LL(0x42e532b7,0xb8c4fc63),LL(0x9ae35290,0x386ba422),LL(0xd201ecbc,0xfb5dda42),LL(0xa0e38fd6,0x2353dc8b), LL(0x68f7e978,0x9a0b85ea),LL(0x2ad6d11f,0x96ec5682),LL(0xe5f6886d,0x5e279d6c),LL(0x3cb1914d,0xd3fe03cd), + LL(0x7ea67c77,0xfe541fa4),LL(0xe3ea810c,0x952bd2af),LL(0x8d01d374,0x791fef56),LL(0x0f11336e,0xa3a1c621), LL(0xc7ec6d79,0x5ad0d5a9),LL(0x3225c342,0xff7038af),LL(0xbc69601b,0x003c6689),LL(0x45e8747d,0x25059bc7), + LL(0xf2086fbf,0xfa4965b2),LL(0x86916078,0xf6840ea6),LL(0x70081d6c,0xd7ac7620),LL(0xb5328645,0xe600da31), LL(0x529b8a80,0x01916f63),LL(0x2d7d6f3e,0xe80e4858),LL(0xd664ca7c,0x29eb0fe8),LL(0xe7b43b0c,0xf017637b), + LL(0x76cb2566,0x9a75c806),LL(0xb24892d9,0x8f76acb1),LL(0x1f08fe45,0x7ae7b9cc),LL(0x6a4907d8,0x19ef7329), LL(0x5f228bf0,0x2db4ab71),LL(0x817032d7,0xf3cdea39),LL(0xdcabe3c0,0x0b1f482e),LL(0xbb86325c,0x3baf76b4), + LL(0x10089465,0xd49065e0),LL(0x8e77c596,0x3bab5d29),LL(0x193dbd95,0x7636c3a6),LL(0xb246e499,0xdef5d294), LL(0x286b2475,0xb22c58b9),LL(0xcd80862b,0xa0b93939),LL(0xf0992388,0x3002c83a),LL(0xeacbe14c,0x6de01f9b), + LL(0xadd70482,0x6aac688e),LL(0x7b4a4e8a,0x708de92a),LL(0x758a6eef,0x75b6dd73),LL(0x725b3c43,0xea4bf352), LL(0x87912868,0x10041f2c),LL(0xef09297a,0xb1b1be95),LL(0xa9f3860a,0x19ae23c5),LL(0x515dcf4b,0xc4f0f839), + LL(0x97f6306a,0x3c7ecca3),LL(0x68a3a4b0,0x744c44ae),LL(0xb3a1d8a2,0x69cd13a0),LL(0x5256b578,0x7cad0a1e), LL(0x33791d9e,0xea653fcd),LL(0x74b2e05f,0x9cc2a05d),LL(0xfd7affa2,0x73b391dc),LL(0xb6b05442,0xddb7091e), + LL(0x8538a5c6,0xc71e27bf),LL(0x89abff17,0x195c63dd),LL(0x1b71e3da,0xfd315285),LL(0xfa680fa0,0x9cbdfda7), LL(0x849d7eab,0x9db876ca),LL(0x3c273271,0xebe2764b),LL(0xf208dcea,0x663357e3),LL(0x565b1b70,0x8c5bd833), + LL(0x9837fc0d,0xccc3b4f5),LL(0xa79cf00f,0x9b641ba8),LL(0xdfdf3990,0x7428243d),LL(0x020786b1,0x83a594c4), LL(0x526c4502,0xb712451a),LL(0x6adb3f93,0x9d39438e),LL(0xe9ff0ccd,0xfdb261e3),LL(0xe07af4c3,0x80344e3c), + LL(0x2fa4f126,0x75900d7c),LL(0x5c99a232,0x08a3b865),LL(0xdb25e0c3,0x2478b6bf),LL(0x71db2edf,0x482cc2c2), LL(0x5f321bb8,0x37df7e64),LL(0x9a8005b4,0x8a93821b),LL(0xcc8c1958,0x3fa2f10c),LL(0x2c269d0a,0x0d332218), + LL(0xe246b0e6,0x20ab8119),LL(0xd349fd17,0xb39781e4),LL(0xb31aa100,0xd293231e),LL(0xbb032168,0x4b779c97), LL(0xc8470500,0x4b3f19e1),LL(0x0c4c869d,0x45b7efe9),LL(0xa1a6bbcc,0xdb84f38a),LL(0xb2fddbc1,0x3b59cb15), + LL(0x3fd165e8,0xba5514df),LL(0x061f8811,0x499fd6a9),LL(0xbfef9f00,0x72cd1fe0),LL(0x79ad7e8a,0x120a4bb9), LL(0x5f4a5ac5,0xf2ffd095),LL(0x95a7a2f0,0xcfd174f1),LL(0x9d17baf1,0xd42301ba),LL(0x77f22089,0xd2fa487a), + LL(0xb1dc77e1,0x9cb09efe),LL(0x21c99682,0xe9566939),LL(0x6c6067bb,0x8c546901),LL(0x61c24456,0xfd378574), LL(0x81796b33,0x2b6a6cbe),LL(0x58e87f8b,0x62d550f6),LL(0x7f1b01b4,0x1b763e1c),LL(0x1b1b5e12,0x4b93cfea), + LL(0x1d531696,0xb9345238),LL(0x88cdde69,0x57201c00),LL(0x9a86afc7,0xdde92251),LL(0xbd35cea8,0xe3043895), LL(0x8555970d,0x7608c1e1),LL(0x2535935e,0x8267dfa9),LL(0x322ea38b,0xd4c60a57),LL(0x804ef8b5,0xe0bf7977), + LL(0xc06fece4,0x1a0dab28),LL(0x94e7b49d,0xd405991e),LL(0x706dab28,0xc542b6d2),LL(0xa91618fb,0xcb228da3), LL(0x107d1cea,0x224e4164),LL(0xd0f5d8f1,0xeb9fdab3),LL(0x0d6e41cd,0xc02ba386),LL(0x9b1f7146,0x676a72c5), + LL(0x4d6cb00b,0xffd6dd98),LL(0xde2e8d7c,0xcef9c5ca),LL(0x641c7936,0xa1bbf5d7),LL(0xee8f772e,0x1b95b230), LL(0xe8ac25b1,0xf765a92e),LL(0x3a18b7c6,0xceb04cfc),LL(0x0acc8966,0x27944cef),LL(0x434c1004,0xcbb3c957), + LL(0xa43ff93c,0x9c9971a1),LL(0xa1e358a9,0x5bc2db17),LL(0xa8d9bc82,0x45b4862e),LL(0x2201e052,0x70ebfbfb), LL(0x92871591,0xafdf64c7),LL(0xb42d0219,0xea5bcae6),LL(0x2ad8f03c,0xde536c55),LL(0xa76aa33c,0xcd6c3f4d), + LL(0x0bca6de3,0xbeb5f623),LL(0xb1e706fd,0xdd20dd99),LL(0xac9059d4,0x90b3ff9d),LL(0x7ccccc4e,0x2d7b2902), LL(0xce98840f,0x8a090a59),LL(0x8410680a,0xa5d947e0),LL(0x923379a5,0x49ae346a),LL(0xb28a3156,0x7dbc84f9), + LL(0x54a1aff2,0xfd40d916),LL(0x3a78fb9b,0xabf318ba),LL(0x3029f95e,0x50152ed8),LL(0xc58ad7fa,0x9fc1dd77), LL(0x13595c17,0x5fa57915),LL(0x8f62b3a9,0xb9504668),LL(0xff3055b0,0x907b5b24),LL(0x9a84f125,0x2e995e35), + LL(0x7e9bbcfb,0x87dacf69),LL(0xe86d96e3,0x95d0c1d6),LL(0x2d95a75c,0x65726e3c),LL(0xacd27f21,0x2c3c9001), LL(0x6c973f57,0x1deab561),LL(0xa5221643,0x108b7e2c),LL(0xc4ef79d4,0x5fee9859),LL(0x40d4b8c6,0xbd62b88a), + LL(0x197c75d6,0xb4dd29c4),LL(0xb7076feb,0x266a6df2),LL(0x4bf2df11,0x9512d0ea),LL(0x6b0cc9ec,0x1320c24f), LL(0x01a59596,0x6bb1e0e1),LL(0xeff9aaac,0x8317c5bb),LL(0x385aa6c9,0x65bb405e),LL(0x8f07988f,0x613439c1), + LL(0x16a66e91,0xd730049f),LL(0xfa1b0e0d,0xe97f2820),LL(0x304c28ea,0x4131e003),LL(0x526bac62,0x820ab732), LL(0x28714423,0xb2ac9ef9),LL(0xadb10cb2,0x54ecfffa),LL(0xf886a4cc,0x8781476e),LL(0xdb2f8d49,0x4b2c87b5), + LL(0x0a44295d,0xe857cd20),LL(0x58c6b044,0x707d7d21),LL(0xf596757c,0xae8521f9),LL(0x67b2b714,0x87448f03), LL(0x5ebcd58d,0x13a9bc45),LL(0x9122d3c1,0x79bcced9),LL(0x9e076642,0x3c644247),LL(0x2df4767d,0x0cf22778), + LL(0x71d444b6,0x5e61aee4),LL(0xc5084a1d,0x211236bf),LL(0x4fd3eaf6,0x7e15bc9a),LL(0xab622bf5,0x68df2c34), LL(0x59bf4f36,0x9e674f0f),LL(0xd7f34d73,0xf883669b),LL(0x31497b1d,0xc48ac1b8),LL(0x5106703b,0x323b925d), + LL(0x74082008,0x22156f42),LL(0xc8482bcb,0xeffc521a),LL(0x12173479,0x5c6831bf),LL(0xc4739490,0xcaa2528f), LL(0x8f1b3c4d,0x84d2102a),LL(0x2d9bec0d,0xcf64dfc1),LL(0x78a546ef,0x433febad),LL(0x7b73cef1,0x1f621ec3), + LL(0x37338615,0x6aecd627),LL(0x01d8edf6,0x162082ab),LL(0x19e86b66,0x833a8119),LL(0xd299b5db,0x6023a251), LL(0xbbf04b89,0xf5bb0c3a),LL(0xae749a44,0x6735eb69),LL(0x4713de3b,0xd0e058c5),LL(0x2c3d4ccd,0xfdf2593e), + LL(0xfdd23667,0x1b8f414e),LL(0xfa2015ee,0xdd52aaca),LL(0xbd9625ff,0x3e31b517),LL(0x8db5918c,0x5ec9322d), LL(0xa96f5294,0xbc73ac85),LL(0x61a0666a,0x82aa5bf3),LL(0xbf08ac42,0x49755810),LL(0x891cedfc,0xd21cdfd5), + LL(0x67f8be10,0x918cb57b),LL(0x56ffa726,0x365d1a7c),LL(0x6532de93,0x2435c504),LL(0x2674cd02,0xc0fc5e10), LL(0x9cbbb142,0x6e51fcf8),LL(0xafc50692,0x1d436e5a),LL(0x3fbcae22,0x766bffff),LL(0xfd55d3b8,0x3148c2fd), + LL(0x233222fa,0x52c7fdc9),LL(0xe419fb6b,0x89ff1092),LL(0x25254977,0x3cd6db99),LL(0x1cf12ca7,0x2e85a161), LL(0xdc810bc9,0xadd2547c),LL(0x9d257c22,0xea3f458f),LL(0x27d6b19b,0x642c1fbe),LL(0x140481a6,0xed07e6b5), + LL(0x86d2e0f8,0x6ada1d42),LL(0x0e8a9fd5,0xe5920122),LL(0x708c1b49,0x02c936af),LL(0x2b4bfaff,0x60f30fee), LL(0x858e6a61,0x6637ad06),LL(0x3fd374d0,0xce4c7767),LL(0x7188defb,0x39d54b2d),LL(0xf56a6b66,0xa8c9d250), + LL(0xb24fe1dc,0x58fc0f5e),LL(0x6b73f24c,0x9eaf9dee),LL(0x33650705,0xa90d588b),LL(0xaf2ec729,0xde5b62c5), LL(0xd3c2b36e,0x5c72cfae),LL(0x034435da,0x868c19d5),LL(0xe17ee145,0x88605f93),LL(0x77a5d5b1,0xaa60c4ee), + LL(0x3b60c472,0xbcf5bfd2),LL(0xeb1d3049,0xaf4ef13c),LL(0xe13895c9,0x373f44fc),LL(0x0cbc9822,0xf29b382f), LL(0x73efaef6,0x1bfcb853),LL(0xa8c96f40,0xcf56ac9c),LL(0x7a191e24,0xd7adf109),LL(0xbf8a8dc2,0x98035f44), + LL(0x1e750c84,0xf40a71b9),LL(0x5dc6c469,0xc57f7b0c),LL(0x6fbc19c1,0x49a0e79c),LL(0xa48ebdb8,0x6b0f5889), LL(0xa07c4e9f,0x5d3fd084),LL(0xab27de14,0xc3830111),LL(0x33e08dcc,0x0e4929fe),LL(0x40bb73a3,0xf4a5ad24), + LL(0x490f97ca,0xde86c2bf),LL(0x67a1ce18,0x288f09c6),LL(0x1844478d,0x364bb886),LL(0xceedb040,0x7840fa42), LL(0x5a631b37,0x1269fdd2),LL(0xa47c8b7d,0x94761f1e),LL(0x481c6266,0xfc0c2e17),LL(0x3daa5fa7,0x85e16ea2), + LL(0x92491048,0xccd86033),LL(0xf4d402d7,0x0c2f6963),LL(0xdf6a865c,0x6336f7df),LL(0xb5c02a87,0x0a2a463c), LL(0xbf2f12ee,0xb0e29be7),LL(0x66bad988,0xf0a22002),LL(0x9123c1d7,0x27f87e03),LL(0x328a8c98,0x21669c55), + LL(0x92f14529,0x186b9803),LL(0x63954df3,0xd3d056cc),LL(0x175a46f6,0x2f03fd58),LL(0x11558558,0x63e34ebe), LL(0x5b80cfa5,0xe13fedee),LL(0xd401dbd1,0xe872a120),LL(0xe8a9d667,0x52657616),LL(0xe08d6693,0xbc8da4b6), + LL(0x1b703e75,0x370fb9bb),LL(0xd4338363,0x6773b186),LL(0xecef7bff,0x18dad378),LL(0x995677da,0xaac787ed), LL(0x0437164b,0x4801ea8b),LL(0x73fe795e,0xf430ad20),LL(0x8ee5eb73,0xb164154d),LL(0x108f7c0e,0x0884ecd8), + LL(0x5f520698,0x0e6ec096),LL(0x44f7b8d9,0x640631fe),LL(0xa35a68b9,0x92fd34fc),LL(0x4d40cf4e,0x9c5a4b66), LL(0x80b6783d,0x949454bf),LL(0x3a320a10,0x80e701fe),LL(0x1a0a39b2,0x8d1a564a),LL(0x320587db,0x1436d53d), + LL(0x6556c362,0xf5096e6d),LL(0xe2455d7e,0xbc23a3c0),LL(0x807230f9,0x3a7aee54),LL(0x22ae82fd,0x9ba1cfa6), LL(0x99c5d706,0x833a057a),LL(0x842315c9,0x8be85f4b),LL(0x66a72f12,0xd083179a),LL(0xcdcc73cd,0x2fc77d5d), + LL(0x5616ee30,0x22b88a80),LL(0xe7ab1083,0xfb09548f),LL(0x511270cd,0x8ad6ab0d),LL(0x6924d9ab,0x61f6c57a), LL(0x90aecb08,0xa0f7bf72),LL(0x0df784a4,0x849f87c9),LL(0xcfaf1d03,0x27c79c15),LL(0xc463face,0xbbf9f675), + LL(0x765ba543,0x91502c65),LL(0x42ea60dd,0x18ce3cac),LL(0x6e43ecb3,0xe5cee6ac),LL(0x68f2aeeb,0x63e4e910), LL(0xc85932ee,0x26234fa3),LL(0x4c90c44d,0x96883e8b),LL(0xa18a50f6,0x29b9e738),LL(0x3f0420df,0xbfc62b2a), + LL(0x6d3e1fa9,0xd22a7d90),LL(0xfe05b8a3,0x17115618),LL(0xbb2b9c01,0x2a0c9926),LL(0xe07e76a2,0xc739fcc6), LL(0x165e439a,0x540e9157),LL(0x6a9063d8,0x06353a62),LL(0x61e927a3,0x84d95594),LL(0xe2e0be7f,0x013b9b26), + LL(0x973497f1,0x4feaec3b),LL(0x093ebc2d,0x15c0f94e),LL(0x33af0583,0x6af5f227),LL(0xc61f3340,0x0c2af206), LL(0x4457397c,0xd25dbdf1),LL(0xcabcbae0,0x2e8ed017),LL(0xc2815306,0xe3010938),LL(0xe8c6cd68,0xbaa99337), + LL(0x3b0ec7de,0x08513182),LL(0x58df05df,0x1e1b822b),LL(0xa5c3b683,0x5c14842f),LL(0x3eba34ce,0x98fe977e), LL(0x0d5e8873,0xfd2316c2),LL(0xbd0d427d,0xe48d839a),LL(0x623fc961,0x495b2218),LL(0xb46fba5e,0x24ee56e7), + LL(0x91e4de58,0x9184a55b),LL(0xdfdea288,0xa7488ca5),LL(0xa8dcc943,0xa723862e),LL(0x849dc0fc,0x92d762b2), LL(0x091ff4a9,0x3c444a12),LL(0x0cada274,0x581113fa),LL(0x30d8eae2,0xb9de0a45),LL(0xdf6b41ea,0x5e0fcd85), + LL(0xc094dbb5,0x6233ea68),LL(0xd968d410,0xb77d062e),LL(0x58b3002d,0x3e719bbc),LL(0x3dc49d58,0x68e7dd3d), LL(0x013a5e58,0x8d825740),LL(0x3c9e3c1b,0x21311747),LL(0x7c99b6ab,0x0cb0a2a7),LL(0xc2f888f2,0x5c48a3b3), +}, +/* digit=8 base_pwr=2^56 */ +{ + LL(0x991724f3,0xc7913e91),LL(0x39cbd686,0x5eda799c),LL(0x63d4fc1e,0xddb595c7),LL(0xac4fed54,0x6b63b80b), LL(0x7e5fb516,0x6ea0fc69),LL(0xd0f1c964,0x737708ba),LL(0x11a92ca5,0x9628745f),LL(0x9a86967a,0x61f37958), + LL(0xaa665072,0x9af39b2c),LL(0xefd324ef,0x78322fa4),LL(0xc327bd31,0x3d153394),LL(0x3129dab0,0x81d5f271), LL(0xf48027f5,0xc72e0c42),LL(0x8536e717,0xaa40cdbc),LL(0x2d369d0f,0xf45a657a),LL(0xea7f74e6,0xb03bbfc4), + LL(0x0d738ded,0x46a8c418),LL(0xe0de5729,0x6f1a5bb0),LL(0x8ba81675,0xf10230b9),LL(0x112b33d4,0x32c6f30c), LL(0xd8fffb62,0x7559129d),LL(0xb459bf05,0x6a281b47),LL(0xfa3b6776,0x77c1bd3a),LL(0x7829973a,0x0709b380), + LL(0xa3326505,0x8c26b232),LL(0xee1d41bf,0x38d69272),LL(0xffe32afa,0x0459453e),LL(0x7cb3ea87,0xce8143ad), LL(0x7e6ab666,0x932ec1fa),LL(0x22286264,0x6cd2d230),LL(0x6736f8ed,0x459a46fe),LL(0x9eca85bb,0x50bf0d00), + LL(0x877a21ec,0x0b825852),LL(0x0f537a94,0x300414a7),LL(0x21a9a6a2,0x3f1cba40),LL(0x76943c00,0x50824eee), LL(0xf83cba5d,0xa0dbfcec),LL(0x93b4f3c0,0xf9538148),LL(0x48f24dd7,0x61744162),LL(0xe4fb09dd,0x5322d64d), + LL(0x3d9325f3,0x57447384),LL(0xf371cb84,0xa9bef2d0),LL(0xa61e36c5,0x77d2188b),LL(0xc602df72,0xbbd6a7d7), LL(0x8f61bc0b,0xba3aa902),LL(0x6ed0b6a1,0xf49085ed),LL(0xae6e8298,0x8bc625d6),LL(0xa2e9c01d,0x832b0b1d), + LL(0xf1f0ced1,0xa337c447),LL(0x9492dd2b,0x800cc793),LL(0xbea08efa,0x4b93151d),LL(0xde0a741e,0x820cf3f8), LL(0x1c0f7d13,0xff1982dc),LL(0x84dde6ca,0xef921960),LL(0x45f96ee3,0x1ad7d972),LL(0x29dea0c7,0x319c8dbe), + LL(0x7b82b99b,0xd3ea3871),LL(0x470eb624,0x75922d4d),LL(0x3b95d466,0x8f66ec54),LL(0xbee1e346,0x66e673cc), LL(0xb5f2b89a,0x6afe67c4),LL(0x290e5cd3,0x3de9c1e6),LL(0x310a2ada,0x8c278bb6),LL(0x0bdb323b,0x420fa384), + LL(0x0eb919b0,0x0ae1d63b),LL(0xa74b9620,0xd74ee51d),LL(0xa674290c,0x395458d0),LL(0x4620a510,0x324c930f), LL(0xfbac27d4,0x2d1f4d19),LL(0x9bedeeac,0x4086e8ca),LL(0x9b679ab8,0x0cdd211b),LL(0x7090fec4,0x5970167d), + LL(0xfaf1fc63,0x3420f2c9),LL(0x328c8bb4,0x616d333a),LL(0x57f1fe4a,0x7d65364c),LL(0x55e5c73a,0x9343e877), LL(0xe970e78c,0x5795176b),LL(0x60533627,0xa36ccebf),LL(0x09cdfc1b,0xfc7c7380),LL(0xb3fec326,0xb39a2afe), + LL(0x6224408a,0xb7ff1ba1),LL(0x247cfc5e,0xcc856e92),LL(0xc18bc493,0x01f102e7),LL(0x2091c727,0x4613ab74), LL(0xc420bf2b,0xaa25e89c),LL(0x90337ec2,0x00a53176),LL(0x7d025fc7,0xd2be9f43),LL(0x6e6fe3dc,0x3316fb85), + LL(0x9ac50814,0x27520af5),LL(0x9a8e4223,0xfdf95e78),LL(0x56bec5a0,0xb7e7df2a),LL(0xdf159e5d,0xf7022f7d), LL(0xcac1fe8f,0x93eeeab1),LL(0x37451168,0x8040188c),LL(0xd967dce6,0x7ee8aa8a),LL(0x3abc9299,0xfa0e79e7), + LL(0x2064cfd1,0x67332cfc),LL(0xb0651934,0x339c31de),LL(0x2a3bcbea,0x719b28d5),LL(0x9d6ae5c6,0xee74c82b), LL(0xbaf28ee6,0x0927d05e),LL(0x9d719028,0x82cecf2c),LL(0xddb30289,0x0b0d353e),LL(0xfddb2e29,0xfe4bb977), + LL(0x640bfd9e,0xbb5bb990),LL(0x82f62108,0xd226e277),LL(0x02ffdd56,0x4bf00985),LL(0x2ca1b1b5,0x7756758a), LL(0x5285fe91,0xc32b62a3),LL(0x8c9cd140,0xedbc546a),LL(0xaf5cb008,0x1e47a013),LL(0x073ce8f2,0xbca7e720), + LL(0x17a91cae,0xe10b2ab8),LL(0x08e27f63,0xb89aab65),LL(0xdba3ddf9,0x7b3074a7),LL(0x330c2972,0x1c20ce09), LL(0x5fcf7e33,0x6b9917b4),LL(0x945ceb42,0xe6793743),LL(0x5c633d19,0x18fc2215),LL(0xc7485474,0xad1adb3c), + LL(0x6424c49b,0x646f9679),LL(0x67c241c9,0xf888dfe8),LL(0x24f68b49,0xe12d4b93),LL(0xa571df20,0x9a6b62d8), LL(0x179483cb,0x81b4b26d),LL(0x9511fae2,0x666f9632),LL(0xd53aa51f,0xd281b3e4),LL(0x7f3dbd16,0x7f96a765), + LL(0x074a30ce,0xa7f8b5bf),LL(0x005a32e6,0xd7f52107),LL(0x50237ed4,0x6f9e0907),LL(0x8096fa2b,0x2f21da47), LL(0xeec863a0,0xf3e19cb4),LL(0x9527620a,0xd18f77fd),LL(0x407c1cf8,0x9505c81c),LL(0x1b6ec284,0x9998db4e), + LL(0xc247d44d,0x7e3389e5),LL(0x3f4f3d80,0x12507141),LL(0x4a78a6c7,0xd4ba0110),LL(0x767720be,0x312874a0), LL(0x75944370,0xded059a6),LL(0x3b2c0bdd,0xd6123d90),LL(0x51c108e3,0xa56b717b),LL(0x070623e9,0x9bb7940e), + LL(0x84ac066c,0x794e2d59),LL(0xe68c69a0,0xf5954a92),LL(0x4fd99dcc,0x28c52458),LL(0xb1012517,0x60e639fc), LL(0x7de79248,0xc2e60125),LL(0xf12fc6d7,0xe9ef6404),LL(0x2a3b5d32,0x4c4f2808),LL(0xc768eb8a,0x865ad32e), + LL(0x13fb70b6,0xac02331b),LL(0x95599b27,0x037b44c1),LL(0x60bd082c,0x1a860fc4),LL(0xc980cd01,0xa2e25745), LL(0x1da0263e,0xee3387a8),LL(0x2d10f3d6,0x931bfb95),LL(0xa1f24a32,0x5b687270),LL(0xca494b86,0xf140e65d), + LL(0xb2f1ac7a,0x4f4ddf91),LL(0x760fee27,0xf99eaabb),LL(0x49c228e5,0x57f4008a),LL(0x1cf713bb,0x090be440), LL(0x5004f022,0xac91fbe4),LL(0x569e1af6,0xd838c2c2),LL(0x0f1daaa5,0xd6c7d20b),LL(0x1bbb02c0,0xaa063ac1), + LL(0x59558a78,0x0938a422),LL(0x8435da2f,0x5343c669),LL(0x034410dc,0x96f67b18),LL(0x84510804,0x7cc1e424), LL(0x16dfbb7d,0x86a1543f),LL(0x5b5bd592,0x921fa942),LL(0xb33dd03c,0x9dcccb6e),LL(0xb843f51e,0x8581ddd9), + LL(0x81d73c9e,0x54935fcb),LL(0x0a5e97ab,0x6d07e979),LL(0xcf3a6bab,0x4dc7b30a),LL(0x170bee11,0x147ab1f3), LL(0x9fafdee4,0x0aaf8e3d),LL(0x538a8b95,0xfab3dbcb),LL(0x6ef13871,0x405df4b3),LL(0x088d5a49,0xf1f4e9cb), + LL(0x66b33f1d,0x9bcd24d3),LL(0x5ce445c0,0x3b97b820),LL(0xba93ff61,0xe2926549),LL(0x4dafe616,0xd9c341ce), LL(0x16efb6f3,0xfb30a76e),LL(0x605b953c,0xdf24b8ca),LL(0xc2fffb9f,0x8bd52afe),LL(0xe19d0b96,0xbbac5ff7), + LL(0x459afccd,0x43c01b87),LL(0xb7432652,0x6bd45143),LL(0x55b5d78e,0x84734530),LL(0x1554ba7d,0x81088fdb), LL(0x1e269375,0xada0a52c),LL(0x2dc5ec10,0xf9f037c4),LL(0x94bfbc11,0xc0660607),LL(0xc9c40d2f,0xc0a630bb), + LL(0xab64c31e,0x5efc797e),LL(0x74507144,0xffdb1dab),LL(0x1ca6790c,0xf6124287),LL(0xe69bf1bf,0xe9609d81), LL(0x00d24fc9,0xdb898595),LL(0xe51fb417,0x9c750333),LL(0xfef7bbde,0x51830a91),LL(0x945f585c,0x0ce67dc8), + LL(0x4763eb50,0x9a730ed4),LL(0xc1ab0d66,0x24a0e221),LL(0x648748f3,0x643b6393),LL(0x6d3c6291,0x1982daa1), LL(0x8bbc5549,0x6f00a9f7),LL(0x7f36384e,0x7a1783e1),LL(0xde977f50,0xe8346323),LL(0xb245502a,0x91ab688d), + LL(0x6d0bdd66,0x331ab6b5),LL(0x64b71229,0x0a6ef32e),LL(0xfe7c352f,0x1028150e),LL(0xce7b39d3,0x27e04350), LL(0xc1070c82,0x2a3c8acd),LL(0x80c9feef,0xfb2034d3),LL(0x709f3729,0x2d729621),LL(0x62cb4549,0x8df290bf), + LL(0xfc2e4326,0x02f99f33),LL(0x5eddf032,0x3b30076d),LL(0x0c652fb5,0xbb21f8cf),LL(0xed91cf7b,0x314fb49e), LL(0x2f700750,0xa013eca5),LL(0x712a4575,0x2b9e3c23),LL(0xaf30fbb0,0xe5355557),LL(0x7c77e771,0x1ada3516), + LL(0x7b135670,0x45f6ecb2),LL(0x7cfc202e,0xe85d19df),LL(0x58d1be9f,0x0f1b50c7),LL(0xead2e344,0x5ebf2c0a), LL(0xabc199c9,0x1531fe4e),LL(0x56bab0ae,0xc7032592),LL(0x6c1fec54,0x16ab2e48),LL(0x04280188,0x0f87fda8), + LL(0x609e4a74,0xdc9f46fc),LL(0xba667f91,0x2a44a143),LL(0xb4d83436,0xbc3d8b95),LL(0xc7bd2958,0xa01e4bd0), LL(0x73483c90,0x7b182932),LL(0xa7c7b598,0xa79c6aa1),LL(0xeaaac07e,0xbf3983c6),LL(0x96e0d4e6,0x8f18181e), + LL(0x051af62b,0x8553d37c),LL(0x0bf94496,0xe9a998eb),LL(0xb0d59aa1,0xe0844f9f),LL(0xe6afb813,0x983fd558), LL(0x65d69804,0x9670c0ca),LL(0x6ea5ff2d,0x732b22de),LL(0x5fd8623b,0xd7640ba9),LL(0xa6351782,0x9f619163), + LL(0xacee5043,0x0bfc27ee),LL(0x2eb10f02,0xae419e73),LL(0x8943fb05,0x19c028d1),LL(0xff13aa2a,0x71f01cf7), LL(0x8887a132,0x7790737e),LL(0x66318410,0x67513309),LL(0x7ddb795e,0x9819e8a3),LL(0xdad100b2,0xfecb8ef5), + LL(0x3021926a,0x59f74a22),LL(0x6f9b4c1c,0xb7c28a49),LL(0x912ad0ab,0xed1a733f),LL(0x01a5659c,0x42a910af), LL(0x7bd68cab,0x3842c6e0),LL(0x76d70ac8,0x2b57fa38),LL(0x3c53aaeb,0x8a6707a8),LL(0x65b4db18,0x62c1c510), + LL(0xb2d09dc7,0x8de2c1fb),LL(0x266bd23b,0xc3dfed12),LL(0xd5b27db6,0x927d039b),LL(0x103243da,0x2fb2f0f1), LL(0x80be7399,0xf855a07b),LL(0x1f9f27a8,0xed9327ce),LL(0x729bdef7,0xa0bd99c7),LL(0x28250d88,0x2b67125e), + LL(0x8670ced7,0x784b26e8),LL(0xc31bd3b4,0xe3dfe41f),LL(0xbcc85cbc,0x9e353a06),LL(0x60178a9d,0x302e2909), LL(0xa6eac16e,0x860abf11),LL(0xaa2b3aac,0x76447000),LL(0x850afdab,0x46ff9d19),LL(0xfdb2d4c1,0x35bdd6a5), + LL(0x7e5c9ce9,0xe82594b0),LL(0x20af346e,0x0f379e53),LL(0xbc65ad4a,0x608b31e3),LL(0x267c4826,0x710c6b12), LL(0x71954cf1,0x51c966f9),LL(0x0d0aa215,0xb1cec793),LL(0x86bd23a8,0x1f155989),LL(0xf9452e86,0xae2ff99c), + LL(0x340ceaa2,0xd8dd953c),LL(0x2e2e9333,0x26355275),LL(0x8586f06d,0x15d4e5f9),LL(0xf7cab546,0xd6bf94a8), LL(0xb76a9af0,0x33c59a0a),LL(0xba095af7,0x52740ab3),LL(0x24389ca0,0xc444de8a),LL(0x706da0cb,0xcc6f9863), + LL(0x6b2515cf,0xb5a741a7),LL(0x9585c749,0x71c41601),LL(0xe683de97,0x78350d4f),LL(0x63d0b5f5,0x31d61524), LL(0xfbce090b,0x7a0cc5e1),LL(0xfbcb2a5b,0xaac927ed),LL(0x20d84c35,0xe920de49),LL(0x22b4de26,0x8c06a0b6), + LL(0xafe7ddf3,0xd34dd58b),LL(0xc1e6e55b,0x55851fed),LL(0x960696e7,0xd1395616),LL(0x5f22705f,0x940304b2), LL(0xb0a2a860,0x6f43f861),LL(0x0e7cc981,0xcf121282),LL(0x0ab64a96,0x12186212),LL(0xb789383c,0x09215b9a), + LL(0x37387c09,0x311eb305),LL(0xf03ee760,0xc5832fce),LL(0x32f7ea19,0x30358f58),LL(0x91d53551,0xe01d3c34), LL(0xda48ea80,0x1ca5ee41),LL(0xcf4fa4c1,0x34e71e8e),LL(0x7af1e1c7,0x312abd25),LL(0x2153f4a5,0xe3afcdeb), + LL(0x00235e9a,0x9d5c84d7),LL(0x8c4c836f,0x0308d3f4),LL(0x89332de5,0xc0a66b04),LL(0x89e566ef,0x610dd399), LL(0xd1ac1635,0xf8eea460),LL(0x20a2c0df,0x84cbb3fb),LL(0xe74a48c5,0x40afb488),LL(0xd326b150,0x29738198), + LL(0xa6d74081,0x2a17747f),LL(0x55a26214,0x60ea4c05),LL(0x1f88c5fe,0x53514bb4),LL(0x7e83426c,0xedd64567), LL(0x96460b25,0xd5d6cbec),LL(0x68dc115e,0xa12fd0ce),LL(0x697840ea,0xc5bc3ed2),LL(0xa6331e31,0x969876a8), + LL(0x472ff580,0x60c36217),LL(0x4ad41393,0xf4229705),LL(0xa03b8b92,0x4bd99ef0),LL(0xc144f4f6,0x501c7317), LL(0x18464945,0x159009b3),LL(0x74c5c6be,0x6d5e594c),LL(0x321a3660,0x2d587011),LL(0x3898d022,0xd1e184b1), + LL(0x4c6a7e04,0x5ba04752),LL(0x45550b65,0x47fa1e2b),LL(0x48c0a9a5,0x9419daf0),LL(0x7c243236,0x66362953), LL(0x5cb12a88,0xcd0744b1),LL(0x2b646188,0x561b6f9a),LL(0x66c2c0c0,0x599415a5),LL(0x0f83f09a,0xbe3f0859), + LL(0xb92041b8,0x9141c5be),LL(0x26477d0d,0x01ae38c7),LL(0xd12c7a94,0xca8b71f3),LL(0x765c70db,0xfab5b31f), LL(0x487443e9,0x76ae7492),LL(0x990d1349,0x8595a310),LL(0x7d460a37,0xf8dbeda8),LL(0x1e45a38f,0x7f7ad082), + LL(0x1059705a,0xed1d4db6),LL(0xe6b9c697,0xa3dd492a),LL(0x6eb38bd5,0x4b92ee3a),LL(0x67cc0bb7,0xbab2609d), LL(0x6e70ee82,0x7fc4fe89),LL(0x13e6b7e3,0xeff2c56e),LL(0x34d26fca,0x9b18959e),LL(0x889d6b45,0x2517ab66), + LL(0xbdefdd4f,0xf167b4e0),LL(0xf366e401,0x69958465),LL(0xa73bbec0,0x5aa368ab),LL(0x7b240c21,0x12148709), LL(0x18969006,0x378c3233),LL(0xe1fe53d1,0xcb4d73ce),LL(0x130c4361,0x5f50a80e),LL(0x7ef5212b,0xd67f5951), + LL(0x9e70c72e,0xf145e21e),LL(0x5566d2fb,0xb2e52e29),LL(0x032397f5,0x44eaba4a),LL(0x7e31a7de,0x5e56937b), LL(0x456c61e1,0x68dcf517),LL(0xa8b0a388,0xbc2e954a),LL(0x60a8b755,0xe3552fa7),LL(0x73ad0cde,0x03442dae), + LL(0xceb26210,0x37ffe747),LL(0x787baef9,0x983545e8),LL(0x86a3de31,0x8b8c8535),LL(0xfacd46db,0xc621dbcb), LL(0x59266fbb,0x82e442e9),LL(0x339d471c,0xa3514c37),LL(0x62cdad96,0x3a11b771),LL(0xecf9bdf0,0xf0cb3b3c), + LL(0x478e2135,0x3fcbdbce),LL(0xbda35342,0x7547b5cf),LL(0x8a677af6,0xa97e81f1),LL(0x28817987,0xc8c2bf83), LL(0x45580985,0xdf07eaaf),LL(0xc93b45cb,0xc68d1f05),LL(0xc77b4cac,0x106aa2fe),LL(0x04a7ae86,0x4c1d8afc), + LL(0x9eb45ab2,0xdb41c3fd),LL(0xd4b22e74,0x5b234b5b),LL(0xf215958a,0xda253dec),LL(0xa04edfa0,0x67e0606e), LL(0xef751b11,0xabbbf070),LL(0xf6f06dce,0xf352f175),LL(0x6839f6b4,0xdfc4b6af),LL(0x9959848e,0x53ddf9a8), + LL(0xc21520b0,0xda49c379),LL(0xdbd5d1b6,0x90864ff0),LL(0x5f49c7f7,0x2f055d23),LL(0xa796b2d8,0xe51e4e6a), LL(0x5c9dc340,0xc361a67f),LL(0xbca7c620,0x5ad53c37),LL(0x32c756d0,0xda1d6588),LL(0x8bb67e13,0xad60d911), + LL(0x0eeec8c6,0xd6c47bdf),LL(0x078a1821,0x4a27fec1),LL(0xc3099524,0x081f7415),LL(0x82cd8060,0x8effdf0b), LL(0x65842df8,0xdb70ec1c),LL(0xd319a901,0x8821b358),LL(0xde42b529,0x72ee56ee),LL(0x236e4286,0x5bb39592), + LL(0xfd6f7140,0xd1183316),LL(0xbd8e81f7,0xf9fadb5b),LL(0x5a02d962,0x701d5e0c),LL(0x1b601324,0xfdee4dbf), LL(0x35d7620e,0xbed17407),LL(0xf48c0012,0x04e3c2c3),LL(0x3455449a,0x9ee29da7),LL(0x91a836c4,0x562cdef4), + LL(0x47701097,0x8f682a5f),LL(0xff88d0c2,0x617125d8),LL(0x57bb86dd,0x948fda24),LL(0x289f7286,0x348abb8f), LL(0x99d94bbd,0xeb10eab5),LL(0x4684d160,0xd51ba28e),LL(0x30c8f41a,0xabe0e51c),LL(0x13254f4a,0x66588b45), + LL(0xfad097a5,0x147ebf01),LL(0x610e815d,0x49883ea8),LL(0x8a11de56,0xe44d60ba),LL(0x827a7a6d,0xa970de6e), LL(0x5e17fc19,0x2be41424),LL(0x01214057,0xd833c657),LL(0x363e723f,0x1375813b),LL(0xe6a52e9b,0x6820bb88), + LL(0xd875d56a,0x7e7f6970),LL(0x51fbf6bf,0xd6a0a9ac),LL(0xa3083c12,0x54ba8790),LL(0x6ae7eb64,0xebaeb23d), LL(0xb99a907a,0xa8685c3a),LL(0x026bf40b,0xf1e74550),LL(0xc802cd9e,0x7b73a027),LL(0x4fef4635,0x9a8a927c), + LL(0x08191224,0xe1b6f60c),LL(0xde4ec091,0xc4126ebb),LL(0x4ae38d84,0xe1dff4dc),LL(0x4f2ef985,0xde3f57db), LL(0xd446a1dd,0x34964337),LL(0x859e77f6,0x7bf217a0),LL(0x8e1d13f5,0x8ff10527),LL(0x74eeae27,0xa304ef03), + LL(0xd19dfa5a,0xfc6f5e47),LL(0x7fad982b,0xdb007de3),LL(0x613715f5,0x28205ad1),LL(0x7889529e,0x251e6729), LL(0x1ae98e78,0x72705184),LL(0x271cac32,0xf818537d),LL(0xb7f410f5,0xc8a15b7e),LL(0x81f62393,0xc474356f), + LL(0xc242316b,0x92dbdc5a),LL(0xdbf4aff5,0xabe060ac),LL(0x909a8ec6,0x6e8c38fe),LL(0x6116cb94,0x43e514e5), LL(0x07d784f9,0x2078fa38),LL(0xf4b5b357,0x1161a880),LL(0x13adea3d,0x5283ce79),LL(0xcc6a910b,0x0756c3e6), + LL(0xaaa79697,0x60bcfe01),LL(0x56391db1,0x04a73b29),LL(0x189b45a0,0xdd8dad47),LL(0x48d5b8d9,0xbfac0dd0), LL(0x7d3d2ec2,0x34ab3af5),LL(0x207bd3af,0x6fa2fc2d),LL(0x66550ded,0x9ff40092),LL(0x1fd5b913,0x719b3e87), + LL(0x6d17fbc7,0xa573a496),LL(0x73d2b24e,0x0cd1a70a),LL(0xb2676937,0x34e2c5ca),LL(0xbf669f21,0xe7050b06), LL(0x1ede9046,0xfbe948b6),LL(0x97662659,0xa0530051),LL(0xf10124c5,0x58cbd4ed),LL(0xdd6c06c8,0xde2646e4), + LL(0x8cad38c0,0x332f8108),LL(0x6bd68ae2,0x471b7e90),LL(0x0d8e27a3,0x56ac3fb2),LL(0x136b4b0d,0xb54660db), LL(0xa6fd8de4,0x123a1e11),LL(0xa37799ef,0x44dbffea),LL(0xce6ac17c,0x4540b977),LL(0xaf60acef,0x495173a8), +}, +/* digit=9 base_pwr=2^63 */ +{ + LL(0x391c2a82,0x9ebb284d),LL(0x158308e8,0xbcdd4863),LL(0x83f1edca,0x006f16ec),LL(0x695dc6c8,0xa13e2c37), LL(0x4a057a87,0x2ab756f0),LL(0xa6b48f98,0xa8765500),LL(0x68651c44,0x4252face),LL(0xe1765e02,0xa52b540b), + LL(0x16a0d2bb,0x4f922fc5),LL(0x1a623499,0x0d5cc16c),LL(0x57c62c8b,0x9241cf3a),LL(0xfd1b667f,0x2f5e6961), LL(0xf5a01797,0x5c15c70b),LL(0x60956192,0x3d20b44d),LL(0x071fdb52,0x04911b37),LL(0x8d6f0f7b,0xf648f916), + LL(0xe60b7cf7,0x6dc1acaf),LL(0x84a9d869,0x25860a50),LL(0xe7ba8ac4,0x56fc6f09),LL(0x6148d29e,0x828c5bd0), LL(0xdc55ae5f,0xac6b435e),LL(0xc0117411,0xa527f56c),LL(0xfd24342c,0x94d5045e),LL(0x70b67c0d,0x2c4c0a35), + LL(0xfac61d9a,0x027cc8b8),LL(0xe3c6fe8a,0x7d25e062),LL(0xe5bff503,0xe08805bf),LL(0x6ff632f7,0x13271e6c), LL(0x232f76a5,0x55dca6c0),LL(0x701ef426,0x8957c32d),LL(0xa10a5178,0xee728bcb),LL(0xb62c5173,0x5ea60411), + LL(0xd0b8892b,0xfc4e964e),LL(0x9301bb74,0x9ea17683),LL(0xfcc48626,0x6265c5ae),LL(0xbb3e9102,0xe60cf82e), LL(0xd4df5531,0x57adf797),LL(0x8deeefe2,0x235b59a1),LL(0x3f306eb1,0x60adcf58),LL(0x3d09492d,0x105c2753), + LL(0xb5def996,0x4090914b),LL(0x233dd1e7,0x1cb69c83),LL(0x9b3d5e76,0xc1e9c1d3),LL(0xfccf6012,0x1f3338ed), LL(0x2f5378a8,0xb1e95d0d),LL(0x2f00cd21,0xacf4c2c7),LL(0xeb5fe290,0x6e984240),LL(0x248088ae,0xd66c038d), + LL(0xf94d70cf,0x804d264a),LL(0x7314bf7e,0xbdb802ef),LL(0x4333ed02,0x8fb54de2),LL(0x285635d9,0x740461e0), LL(0x365e9383,0x4113b2c8),LL(0x3fdef652,0xea762c83),LL(0x47b956c1,0x4eec6e2e),LL(0x65620fa4,0xa3d814be), + LL(0xb4d8bc50,0x9ad5462b),LL(0xa9195770,0x181c0b16),LL(0x78412a68,0xebd4fe1c),LL(0xc0dff48c,0xae0341bc), LL(0x7003e866,0xb6bc45cf),LL(0x8a24a41b,0xf11a6dea),LL(0xd04c24c2,0x5407151a),LL(0xda5b7b68,0x62c9d27d), + LL(0x88cceff6,0x2e964235),LL(0x8b07ed69,0x8594c54f),LL(0xc84d0d0d,0x1578e73c),LL(0xff532868,0x7b4e1055), LL(0xb5ec995a,0xa348c0d5),LL(0x14289a54,0xbf4b9d55),LL(0x58fbd777,0x9ba155a6),LL(0x1a84491d,0x186ed7a8), + LL(0x614c0900,0xd4992b30),LL(0xbd00c24b,0xda98d121),LL(0x7ec4bfa1,0x7f534dc8),LL(0x37dc34bc,0x4a5ff674), LL(0x1d7ea1d7,0x68c196b8),LL(0x80a6d208,0x38cf2893),LL(0xe3cbbd6e,0xfd56cd09),LL(0x4205a5b6,0xec72e27e), + LL(0xa44f77f7,0x15ea68f5),LL(0xb43c52bc,0x7aa5f9fd),LL(0x94f0e609,0x86ff676f),LL(0x2e2d432b,0xa4cde963), LL(0xeee470af,0x8cafa0c0),LL(0x8a3f5ec8,0x84137d0e),LL(0xfaa31231,0xebb40411),LL(0x6f7f7ccf,0xa239c13f), + LL(0xa8afd30b,0x32865719),LL(0x8a826dce,0x86798328),LL(0xc4a8fbe0,0xdf04e891),LL(0xebf56ad3,0xbb6b6e1b), LL(0x471f1ff0,0x0a695b11),LL(0xbe15baf0,0xd76c3389),LL(0xbe96c43e,0x018edb95),LL(0x90794158,0xf2beaaf4), + LL(0xc3076a27,0x152db09e),LL(0xe416545d,0x5e82908e),LL(0x356d6f2e,0xa2c41272),LL(0x31fd74e1,0xdc9c9642), LL(0x519bf615,0x66ceb88d),LL(0x05a2274e,0xe29ecd76),LL(0xbf5e2fa0,0x3a0473c4),LL(0x64284e67,0x6b6eb671), + LL(0xb88756dd,0xe8b97932),LL(0xf17e3e61,0xed4e8652),LL(0x3ee1c4a4,0xc2dd1499),LL(0x597f8c0e,0xc0aaee17), LL(0x6c168af3,0x15c4edb9),LL(0xb39ae875,0x6563c7bf),LL(0x20adb436,0xadfadb6f),LL(0x9a042ac0,0xad55e8c9), + LL(0xb76da1f5,0x975a1ed8),LL(0xa58acb94,0x10dfa466),LL(0xac060282,0x8dd7f7e3),LL(0x572a051e,0x6813e66a), LL(0x350cb901,0xb4ccae1e),LL(0x50cb7822,0xb653d656),LL(0xdfab3b87,0x42484710),LL(0x9b670fd0,0xcd7ee537), + LL(0x523b8bf6,0x0a50b12e),LL(0x8f910c1b,0x8009eb5b),LL(0x4a167588,0xf535af82),LL(0xfb2a2abd,0x0f835f9c), LL(0x2afceb62,0xf59b2931),LL(0x169d383f,0xc797df2a),LL(0x66ac02b0,0xeb3f5fb0),LL(0xdaa2d0ca,0x029d4c6f), + LL(0xafab4bc5,0xd4059bc1),LL(0x56783247,0x833f5c6f),LL(0x8d2d3605,0xb5346630),LL(0xd34d8433,0x83387891), LL(0xadd9419a,0xd973b30f),LL(0xafe3fce8,0xbcca1099),LL(0x0809aac6,0x08178315),LL(0x540f0f11,0x01b7f21a), + LL(0x909523c8,0x65c29219),LL(0xa3a1c741,0xa62f648f),LL(0x60c9e55a,0x88598d4f),LL(0x0e4f347a,0xbce9141b), LL(0x35f9b988,0x9af97d84),LL(0x320475b6,0x0210da62),LL(0x9191476c,0x3c076e22),LL(0x44fc7834,0x7520dbd9), + LL(0xc1ab1bbd,0x6a6b2cfe),LL(0xdc650938,0xef8a65be),LL(0x805d7bc4,0x72855540),LL(0xed11fdfd,0xda389396), LL(0x74660876,0xa9d5bd36),LL(0xb45dff35,0x11d67c54),LL(0xa4f5da94,0x6af7d148),LL(0xc0bbeb31,0xbb8d4c3f), + LL(0xe0a1b12a,0x87a7ebd1),LL(0x770ba95f,0x1e4ef88d),LL(0xdc2ae9cb,0x8c33345c),LL(0x01cc8403,0xcecf1276), LL(0x1b39b80f,0x687c012e),LL(0x35c33ba4,0xfd90d0ad),LL(0x5c9661c2,0xa3ef5a67),LL(0xe017429e,0x368fc88e), + LL(0x196a2fa2,0xd30c6761),LL(0xbd5b312e,0x931b9817),LL(0x72f54a31,0xba01000c),LL(0x66eaa541,0xa203d2c8), LL(0x98939db3,0xf2abdee0),LL(0x3e606c02,0xe37d6c2c),LL(0x521ff643,0xf2921574),LL(0xd7e2fca3,0x2781b3c4), + LL(0x7850ec06,0x664300b0),LL(0x7d3a10cf,0xac5a38b9),LL(0xe34ab39d,0x9233188d),LL(0x5072cbb9,0xe77057e4), LL(0xb59e78df,0xbcf0c042),LL(0x1d97de52,0x4cfc91e8),LL(0x3ee0ca4a,0x4661a26c),LL(0xfb8507bc,0x5620a4c1), + LL(0x049f842c,0x4b44d4aa),LL(0x1540e82b,0xceabc5d5),LL(0x15c6f156,0x306710fd),LL(0x63db1d72,0xbe5ae52b), LL(0x334957f1,0x06f1e7e6),LL(0x31144a70,0x57e388f0),LL(0xdf96447b,0xfb69bb2f),LL(0x73e38a12,0x0f78ebd3), + LL(0x2b7ce542,0xb8222605),LL(0x7472bde1,0xe6d4ce99),LL(0x09d2f4da,0x53e16ebe),LL(0x53b92b2e,0x180ff42e), LL(0x2c34a1c6,0xc59bcc02),LL(0x422c46c2,0x3803d6f9),LL(0x5c14a8a2,0x18aff74f),LL(0x10a08b28,0x55aebf80), + LL(0x7135593f,0x66097d58),LL(0x2be570cd,0x32e6eff7),LL(0x2a8c860d,0x584e6a10),LL(0xa2eb4163,0xcd185890), LL(0x6d97e134,0x7ceae99d),LL(0xdd8447ce,0xd42c6b70),LL(0xb8c50273,0x59ddbb4a),LL(0x3cf34e1e,0x03c612df), + LL(0x04b6c5a0,0x84b9ca15),LL(0x18f0e3a3,0x35216f39),LL(0xbd986c00,0x3ec2d2bc),LL(0xd19228fe,0x8bf546d9), LL(0x4cd623c3,0xd1c655a4),LL(0x502b8e5a,0x366ce718),LL(0xeea0bfe7,0x2cfc84b4),LL(0xcf443e8e,0xe01d5cee), + LL(0x036520f8,0x8ec045d9),LL(0x92d40e98,0xdfb3c3d1),LL(0xcc559a04,0x0bac4cce),LL(0x240ea6b1,0x35eccae5), LL(0xf8a5a0ac,0x180b32db),LL(0xeb699700,0x547972a5),LL(0xca26bca0,0xa3765801),LL(0xa647f25a,0x57e09d0e), + LL(0x2fdd23cc,0xb956970e),LL(0x5682e971,0xb80288bc),LL(0x9ae86ebc,0xe6e6d91e),LL(0x8c9f1939,0x0564c83f), LL(0x39560368,0x551932a2),LL(0x049c28e2,0xe893752b),LL(0xa6a158c3,0x0b03cee5),LL(0x04964263,0xe12d656b), + LL(0x63e3bc1d,0x4b47554e),LL(0x45044ff7,0xc719b6a2),LL(0xe48daa07,0x4f24d30a),LL(0xc8c1edc3,0xa3f37556), LL(0x0700d360,0x9a47bf76),LL(0x822ae4e2,0xbb1a1824),LL(0x89f1fb4c,0x22e275a3),LL(0x9968c5f5,0x72b1aa23), + LL(0xbe063f64,0xa75feaca),LL(0xbce47a09,0x9b392f43),LL(0x1ad07aca,0xd4241509),LL(0x8d26cd0f,0x4b0c591b), LL(0x92f1169a,0x2d42ddfd),LL(0x4cbf2392,0x63aeb1ac),LL(0x0691a2af,0x1de9e877),LL(0xd98021da,0xebe79af7), + LL(0x40e50acf,0xcfdf2a4e),LL(0xaf01d665,0xf0a98ad7),LL(0x1831be1f,0xefb640bf),LL(0x80e9ada0,0x6fe8bd2f), LL(0x6cafbc91,0x94c103a1),LL(0x8308e08c,0x170f8759),LL(0x9780ff4f,0x5de2d2ab),LL(0x45b201f2,0x666466bc), + LL(0xf5b343bc,0x58af2010),LL(0xf2f142fe,0x0f2e400a),LL(0xa85f4bdf,0x3483bfde),LL(0x03bfeaa9,0xf0b1d093), LL(0xc7081603,0x2ea01b95),LL(0x3dba1097,0xe943e4c9),LL(0xb438f3a6,0x47be92ad),LL(0xe5bf6636,0x00bb7742), + LL(0x824297b4,0x136b7083),LL(0x5584455f,0x9d0e5580),LL(0xf1c7d69e,0xab48cedc),LL(0x2a256e76,0x53a9e481), LL(0x65eb2413,0x0402b0e0),LL(0x8fc407a7,0xdadbbb84),LL(0x8d7f5492,0xa65cd5a4),LL(0x74bae294,0x21d44293), + LL(0x3b5f1cc4,0x66917ce6),LL(0xce872e62,0x37ae52ea),LL(0x2905f244,0xbb087b72),LL(0x1e6af74f,0x12077086), LL(0x1058edea,0x4b644e49),LL(0xb638ca1d,0x827510e3),LL(0x6038591c,0x8cf2b704),LL(0xfe635063,0xffc8b47a), + LL(0x1b4d5e63,0x3ae220e6),LL(0x9d961b4b,0xbd864742),LL(0x9bd16bed,0x610c107e),LL(0x1127147b,0x4270352a), LL(0x64cfc50e,0x7d17ffe6),LL(0x1e36cb42,0x50dee01a),LL(0x35dc5f9a,0x068a7622),LL(0xdf53f62c,0x9a08d536), + LL(0x6be5f7de,0x4ed71457),LL(0xc2263c9e,0xd93006f8),LL(0xcacacb36,0xe073694c),LL(0x3ae118ab,0x2ff7a5b4), LL(0xcd871236,0x3cce53f1),LL(0xc2aa6d52,0xf156a39d),LL(0xb198d76d,0x9cc5f271),LL(0x81383d39,0xbc615b6f), + LL(0xde3eee6b,0xa54538e8),LL(0xab910d91,0x58c77538),LL(0x58d278bd,0x31e5bdbc),LL(0xb963acae,0x3cde4adf), LL(0x5302169c,0xb1881fd2),LL(0xa989ed8b,0x8ca60fa0),LL(0xff96a0ee,0xa1999458),LL(0xac6c283d,0xc1141f03), + LL(0x6dfafed3,0x7677408d),LL(0x39661588,0x33a01653),LL(0x0b726fa0,0x3c9c15ec),LL(0x6c9b56da,0x090cfd93), LL(0xa3c40af5,0xe34f4bae),LL(0xd21129f1,0x3469eadb),LL(0x1e207ce8,0xcc51674a),LL(0xc83b1ef9,0x1e293b24), + LL(0x1e6c0bb4,0x17173d13),LL(0x90776d35,0x19004695),LL(0x6de6f922,0xe7980e34),LL(0xf4dd9a22,0x873554cb), LL(0xcbf18a51,0x0316c627),LL(0x3032c081,0x4d93651b),LL(0x3946834d,0x207f2771),LL(0x30cdbf80,0x2c08d7b4), + LL(0x86df2a61,0x137a4fb4),LL(0xecf7b4a2,0xa1ed9c07),LL(0x7bd042ff,0xb2e460e2),LL(0x5f62f5ec,0xb7f5e2fa), LL(0xcc2423b7,0x7aa6ec6b),LL(0xba63eea7,0x75ce0a7f),LL(0xf250a6e1,0x67a45fb1),LL(0xe53cdc9f,0x93bc919c), + LL(0x871942df,0x9271f56f),LL(0x7859ad66,0x2372ff6f),LL(0x33cb1a78,0x5f4c2b96),LL(0x5838aa83,0xe3e29101), LL(0xe4e8110c,0xa7ed1611),LL(0x330198ce,0x2a2d70d5),LL(0x6720efe0,0xbdf132e8),LL(0x66a471bf,0xe61a8962), + LL(0x825808bd,0x796d3a85),LL(0x3fd6e902,0x51dc3cb7),LL(0x916219d1,0x643c768a),LL(0xa2ad7d32,0x36cd7685), LL(0xb22922a4,0xe3db9d05),LL(0xdba29660,0x6494c87e),LL(0xbcd2ebc7,0xf0ac91df),LL(0x45107f8d,0x4deb57a0), + LL(0xc3d12a73,0x42271f59),LL(0xa5c2c51d,0x5f71687c),LL(0x05797bcb,0xcb1f50c6),LL(0xd6d34eb0,0x29ed0ed9), LL(0x4683c2eb,0xe5fe5b47),LL(0x97447c46,0x4956eeb5),LL(0x71207167,0x5b163a43),LL(0x0248c5ef,0x93fa2fed), + LL(0x31f63950,0x67930af2),LL(0x14caa2c9,0xa77797c1),LL(0x27ac7e62,0x526e80ee),LL(0x58b28aec,0xe1e6e626), LL(0xb3c9fef0,0x636178b0),LL(0x6d5f90be,0xaf7752e0),LL(0xeece51cf,0x94ecaf18),LL(0xca806e1f,0x2864d0ed), + LL(0x97c69134,0x6de2e383),LL(0xeb291293,0x5a42c316),LL(0x6a60bae0,0xc7779219),LL(0x6b7599d1,0xa24de346), LL(0xb75d4941,0x49d374aa),LL(0x2d501ff0,0x98900586),LL(0xeb7974cf,0x9f16d40e),LL(0xcdd8c115,0x1033860b), + LL(0x2094cec3,0xb6c69ac8),LL(0x403b770c,0x9976fb88),LL(0x4859590d,0x1dea026c),LL(0x8562d1fd,0xb6acbb46), LL(0x44569d85,0x7cd6c461),LL(0x97f0891d,0xc3190a36),LL(0x48d5a17d,0xc6f53195),LL(0xd749abc8,0x7d919966), + LL(0xdd1c8a20,0x65104837),LL(0x2f683419,0x7e5410c8),LL(0xbe94022e,0x958c3ca8),LL(0x6145dac2,0x605c3197), LL(0x01683d54,0x3fc07501),LL(0x595b1234,0x1d7127c5),LL(0x9481277f,0x10b8f87c),LL(0xe65a1adb,0x677db2a8), + LL(0xddce3345,0xec2fccaa),LL(0x012a4350,0x2a6811b7),LL(0xac598bdc,0x96760ff1),LL(0xd1bf4128,0x054d652a), LL(0x92a21005,0x0a1151d4),LL(0x33110fdf,0xad7f3971),LL(0x1960100f,0x8c95928c),LL(0x7bf03362,0x6c91c825), + LL(0xce309f06,0xc8c8b2a2),LL(0xca27204b,0xfdb27b59),LL(0x0848e32e,0xd223eaa5),LL(0xe7bfaf1e,0xb93e4b2e), LL(0x44aa3ded,0xc5308ae6),LL(0xc015d573,0x317a666a),LL(0x1a979707,0xc888ce23),LL(0x0d5c4958,0xf141c1e6), + LL(0x61906373,0xb53b7de5),LL(0xeb999595,0x858dbade),LL(0xa59e5c36,0x8cbb47b2),LL(0xdcf4e842,0x660318b3), LL(0x12ba4b7a,0xbd161ccd),LL(0xf8c8282a,0xf399daab),LL(0xeeb2130d,0x1587633a),LL(0xda38dd7d,0xa465311a), + LL(0x64d3779b,0x5f75eec8),LL(0xad64c171,0x3c5d0476),LL(0x2a914428,0x87410371),LL(0x90e2fc29,0x8096a891), LL(0x23b3ebc2,0xd3d2ae9d),LL(0xa580cfd6,0x90bdd6db),LL(0xc5b01f6c,0x52dbb7f3),LL(0xe102a2dc,0xe68eded4), + LL(0x99eb6df0,0x17785b77),LL(0x7386b779,0x26c3cc51),LL(0x6417a48e,0x345ed988),LL(0x07d6ef31,0xe990b4e4), LL(0x2586abba,0x0f456b7e),LL(0x59c96e9a,0x239ca6a5),LL(0xe2eb4206,0xe327459c),LL(0xa002b90a,0x3a4c3313), + LL(0xf6a3f6fb,0x2a114806),LL(0x85c251dd,0xad5cad2f),LL(0xf5a784d3,0x92c1f613),LL(0x349766d5,0xec7bfacf), LL(0x3e23cb3b,0x04b3cd33),LL(0xc5a64b2d,0x3979fe84),LL(0x7e589106,0x192e2720),LL(0xa15b527f,0xa60c43d1), + LL(0xbe7cf3a6,0x2dae9082),LL(0xbc967274,0xcc86ba92),LL(0xaea0a8a9,0xf28a2ce8),LL(0x6ee988b3,0x404ca6d9), LL(0x005921b8,0xfd7e9c5d),LL(0x44e79bf9,0xf56297f1),LL(0x0d75ddc2,0xa163b460),LL(0xa1f2be87,0x30b23616), + LL(0xbfe50e2b,0x4b070d21),LL(0xe1bfede1,0x7ef8cfd0),LL(0x2aac4ae0,0xadba0011),LL(0xb9ebd033,0x2a3e7d01), LL(0xe38d9d1c,0x995277ec),LL(0x9c5d2de3,0xb500249e),LL(0xf13ca8c9,0x8912b820),LL(0x877793af,0xc8798114), + LL(0xec3f1dec,0x19e6125d),LL(0x911178da,0x07b1f040),LL(0x904a6738,0xd93ededa),LL(0x0bebedcd,0x55187a5a), LL(0xeb329d41,0xf7d04722),LL(0xf170b391,0xf449099e),LL(0xca99f828,0xfd317a69),LL(0x34a4976d,0x50c3db2b), + LL(0x3757b392,0xe9ba7784),LL(0xaa3ca05a,0x326caefd),LL(0xf1e593d4,0x78e5293b),LL(0x0d98fd13,0x7842a937), LL(0x5f96b10d,0xe694bf96),LL(0x06a8cd05,0x373a9df6),LL(0xe8f0c7fc,0x997d1e51),LL(0x63fd972e,0x1d019790), + LL(0x5499fb32,0x0064d858),LL(0x77a8aeb7,0x7b67bad9),LL(0x2d08eec5,0x1d3eb977),LL(0xcbabae1d,0x5fc047a6), LL(0xe54a64bb,0x0577d159),LL(0xc43497e4,0x8862201b),LL(0x2ce0608d,0xad6b4e28),LL(0x0b167aac,0x8b687b7d), + LL(0x8b2ecfa9,0x6ed4d367),LL(0xa90c3c38,0x24dfe62d),LL(0x3fe5c42b,0xa1862e10),LL(0xd5732a9f,0x1ca73dca), LL(0x76bb87ad,0x35f038b7),LL(0xf242b81f,0x674976ab),LL(0xb0fd90cd,0x4f2bde7e),LL(0xa7fdf092,0x6efc172e), + LL(0x92222f1f,0x3806b69b),LL(0x6cf7ae70,0x5a2459ca),LL(0xa85217ee,0x6789f69c),LL(0xe3dc85ac,0x5f232b5e), LL(0x48e9e516,0x660e3ec5),LL(0x3197eb31,0x124b4e47),LL(0xaafcca23,0x10a0cb13),LL(0x8213224f,0x7bd63ba4), + LL(0x290a7f4f,0xaffad7cc),LL(0x0286b461,0x6b409c9e),LL(0xffa407af,0x58ab809f),LL(0xc68ac073,0xc3122eed), LL(0x4ef24d7e,0x17bf9e50),LL(0x3e2a5811,0x5d929794),LL(0x02902e01,0x519bc867),LL(0x39c8a851,0x76bba5da), + LL(0xda94951e,0xe9f9669c),LL(0x66b8d418,0x4b6af58d),LL(0x17d426a4,0xfa321074),LL(0x9dde6027,0xc78e66a9), LL(0x4a53b964,0x0516c083),LL(0xff602330,0xfc659d38),LL(0x58c5c897,0x0ab55e5c),LL(0x838bc5df,0x985099b2), + LL(0xc52fc238,0x061d9efc),LL(0x6ac1da3f,0x712b2728),LL(0x9283fe08,0xfb658149),LL(0xb8aaa2f7,0x4954ac94), LL(0x7fb2e74f,0x85c0ada4),LL(0xb89926b0,0xee8ba98e),LL(0x23d1af5b,0xe4f9d37d),LL(0xba9b015e,0x14ccdbf9), + LL(0x7bfe7178,0xb674481b),LL(0x65405868,0x4e1debae),LL(0xc48c867d,0x061b2821),LL(0x513b30ea,0x69c15b35), LL(0x36871088,0x3b4a1666),LL(0x1220b1ff,0xe5e29f5d),LL(0x233d9f4d,0x4b82bb35),LL(0x18cdc675,0x4e076333), +}, +/* digit=10 base_pwr=2^70 */ +{ + LL(0xa3e6fced,0x0d53f5c7),LL(0xf45fbdeb,0xe8cbbdd5),LL(0x13339a70,0xf85c01df),LL(0x142ceb81,0x0ff71880), LL(0xbd70437a,0x4c4e8774),LL(0xba0bda6a,0x5fb32891),LL(0xf18bd26e,0x1cdbebd2),LL(0x03a9d522,0x2f9526f1), + LL(0x92c4d684,0x40ce3051),LL(0x7612efcd,0x8b04d725),LL(0x6f9cae20,0xb9dcda36),LL(0xf058856c,0x0edc4d24), LL(0x85427900,0x64f2e6bf),LL(0xdc09dfea,0x3de81295),LL(0x379bf26c,0xd41b4487),LL(0x6df135a9,0x50b62c6d), + LL(0xc72dfe67,0xd4f8e3b4),LL(0x90e19fdf,0xc416b0f6),LL(0x4c13bd35,0x18b9098d),LL(0x15b8cb9e,0xac11118a), LL(0xf0062841,0xf598a318),LL(0x89f356f4,0xbfe0602f),LL(0x30177a0c,0x7ae3637e),LL(0x61136537,0x34097747), + LL(0xd005832a,0x0db2fb5e),LL(0x91042e4f,0x5f5efd3b),LL(0xed70f8ca,0x8c4ffdc6),LL(0xb52da9cc,0xe4645d0b), LL(0xc9001d1f,0x9596f58b),LL(0x4e117205,0x52c8f0bc),LL(0xe398a084,0xfd4aa0d2),LL(0x104f49de,0x815bfe3a), + LL(0x23885e5f,0x97e5443f),LL(0xe8433aab,0xf72f8f99),LL(0xe4d4e604,0xbd00b154),LL(0xe5e173ff,0xd0b35e6a), LL(0x9164722d,0x57b2a048),LL(0x88761ec8,0x3e3c665b),LL(0x3da83832,0x6bdd1397),LL(0x73dafe3b,0x3c8b1a1e), + LL(0x54317cac,0x4497ace6),LL(0x521771b3,0xbe600ab9),LL(0xb0dfe8b8,0xb42e409e),LL(0x3942310f,0x386a67d7), LL(0x4431cc28,0x25548d8d),LL(0x985dc524,0xa7cff142),LL(0x93c4be32,0x4d60f5a1),LL(0xd071c6e1,0x83ebd5c8), + LL(0xb1fd2b0b,0xba3a80a7),LL(0x5bec33e8,0x9b3ad396),LL(0x79743fb3,0xb3868d61),LL(0xfdb462fa,0xcfd169fc), LL(0x9ce0a6af,0xd3b499d7),LL(0xe42d3ff8,0x55dc1cf1),LL(0xc6c3e1b2,0x04fb9e6c),LL(0x6f69a474,0x47e6961d), + LL(0xe548b37b,0x54eb3acc),LL(0x84d40549,0xb38e7542),LL(0x7b341b4f,0x8c3daa51),LL(0x690bf7fa,0x2f6928ec), LL(0x86ce6c41,0x0496b323),LL(0x10adadcd,0x01be1c55),LL(0x4bb5faf9,0xc04e67e7),LL(0xe15c9985,0x3cbaf678), + LL(0x50ca4247,0x8cd12145),LL(0xe7dd30aa,0xba1aa47a),LL(0xe58fee24,0x2f81ddf1),LL(0xeec9b0e8,0x03452936), LL(0x243aea96,0x8bdc3b81),LL(0x15c3d0e5,0x9a2919af),LL(0x10948361,0x9ea640ec),LL(0x6e0bcccf,0x5ac86d5b), + LL(0xc36cf440,0xf892d918),LL(0xc939719c,0xaed3e837),LL(0xc0218b64,0xb07b08d2),LL(0xce9790dd,0x6f1bcbba), LL(0x60919b8e,0x4a84d6ed),LL(0x8ac1f9eb,0xd8900791),LL(0x0dd5daef,0xf84941aa),LL(0x67fd62c5,0xb22fe40a), + LL(0x157f2db3,0x97e15ba2),LL(0x8e28ca9c,0xbda2fc8f),LL(0x37b9f454,0x5d050da4),LL(0x2379d72e,0x3d57eb57), LL(0xfb5ee997,0xe9b5eba2),LL(0xe11538ca,0x01648ca2),LL(0xf6327974,0x32bb76f6),LL(0xff3f4bb7,0x338f14b8), + LL(0xd7ab9a2d,0x524d226a),LL(0x7dfae958,0x9c00090d),LL(0x8751d8c2,0x0ba5f539),LL(0x3ab8262d,0x8afcbcdd), LL(0xe99d043b,0x57392729),LL(0xaebc943a,0xef51263b),LL(0x20862935,0x9feace93),LL(0xb06c817b,0x639efc03), + LL(0x66b4be7a,0x1fe054b3),LL(0x84a37a1e,0x3f25a9de),LL(0x78d75cd9,0xf39ef1ad),LL(0x5062c1b5,0xd7b58f49), LL(0xff563436,0x6f74f9a9),LL(0xe8af51e7,0xf718ff29),LL(0x15e97fec,0x5234d313),LL(0x292f1c0a,0xb6a8e2b1), + LL(0x327720c1,0xa7f53aa8),LL(0xba092cc8,0x956ca322),LL(0x28746c4d,0x8f03d64a),LL(0x66d0d392,0x51fe1782), LL(0x3c832c80,0xd19b34db),LL(0x6da2e3b4,0x60dccc5c),LL(0x0a104ccc,0x245dd62e),LL(0x620b21fd,0xa7ab1de1), + LL(0x3893d123,0xb293ae0b),LL(0xb15ee71c,0xf7b75783),LL(0x42a9468b,0x5aa3c614),LL(0xdb15d744,0xd686123c), LL(0xa7ab4116,0x8c616891),LL(0xa4e6a459,0x6fcd72c8),LL(0x77e5fad7,0xac219110),LL(0x704fa46b,0xfb6a20e7), + LL(0x341d81dc,0xe839be7d),LL(0x32148379,0xcddb6889),LL(0xf7026ead,0xda6211a1),LL(0xf4d1cc5e,0xf3b2575f), LL(0xa7a73ae6,0x40cfc8f6),LL(0x61d5b483,0x83879a5e),LL(0x41a50ebc,0xc5acb1ed),LL(0x3c07d8fa,0x59a60cc8), + LL(0xb1876262,0x1b73bdce),LL(0x12af4ee9,0x2b0d79f0),LL(0xd46e1d07,0x8bcf3b0b),LL(0xe45d152f,0x17d6af9d), LL(0x6d736451,0x73520461),LL(0x56b0bf5a,0x43cbbd97),LL(0xd5999b9d,0xb0833a5b),LL(0xeb72e398,0x702614f0), + LL(0x59c3e9f8,0x0aadf01a),LL(0xce6b3d16,0x40200e77),LL(0xdeddafad,0xda22bdd3),LL(0x310d72e1,0x76dedaf4), LL(0x4bc2e88f,0x49ef807c),LL(0x146dd5a5,0x6ba81291),LL(0x7d8d59e9,0xa1a4077a),LL(0x802db349,0x87b6a2e7), + LL(0x1b4e598e,0xd5679997),LL(0x06fe4b1d,0xf499ef1f),LL(0xfcb267c5,0x3978d3ae),LL(0x235786d0,0xb582b557), LL(0x1715cb07,0x32b3b2ca),LL(0x8480241d,0x4c3de6a2),LL(0xcb571ecd,0x63b5ffed),LL(0xed2fe9a9,0xeaf53900), + LL(0xc3b81990,0xdec98d4a),LL(0x9e0cc8fe,0x1cb83722),LL(0xd2b427b9,0xfe0b0491),LL(0xe983a66c,0x0f2386ac), LL(0xb3291213,0x930c4d1e),LL(0x59a62ae4,0xa2f82b2e),LL(0xf93e89e3,0x77233853),LL(0x11777c7f,0x7f8063ac), + LL(0x59ad2877,0xff0eb567),LL(0x9865c754,0x6f454642),LL(0x236e9a84,0xe6fe701a),LL(0x06e40fc3,0xc586ef16), LL(0x24bafad9,0x3f62b6e0),LL(0x64da906a,0xc8b42bd2),LL(0xda3276a0,0xc98e1eb4),LL(0x06cbf852,0x30d0e5fc), + LL(0xe8b4dfd4,0x1b6b2ae1),LL(0x8301cbac,0xd754d5c7),LL(0x112a39ac,0x66097629),LL(0x93ba4ab9,0xf86b5999), LL(0x99f9d581,0x26c9dea7),LL(0xc2fafeaa,0x0473b1a8),LL(0x3b2505a5,0x1469af55),LL(0xd6a43323,0x227d16d7), + LL(0xad3d97f9,0x3316f73c),LL(0x1f137455,0x52bf3bb5),LL(0x09954e7c,0x953eafeb),LL(0xdd732411,0xa721dfed), LL(0x141d4579,0xb4929821),LL(0xaa3bd435,0x3411321c),LL(0x17fa6015,0xafb355aa),LL(0x18e42f0e,0xb4e7ef4a), + LL(0x59371000,0x604ac97c),LL(0x7f759c18,0xe1c48c70),LL(0xa5db6b65,0x3f62ecc5),LL(0x38a21495,0x0a78b173), LL(0xbcc8ad94,0x6be1819d),LL(0xd89c3400,0x70dc04f6),LL(0xa6b4840a,0x462557b4),LL(0x60bd21c0,0x544c6ade), + LL(0x907a544b,0x6a00f24e),LL(0x313da210,0xa7520dcb),LL(0x11e4994b,0xfe939b75),LL(0xbc275d70,0x918b6ba6), LL(0x644be892,0xd3e5e0fc),LL(0xfdaf6c42,0x707a9816),LL(0xf15c13fe,0x60145567),LL(0xe130a54a,0x4818ebaa), + LL(0x58d2f767,0x28aad3ad),LL(0xd7e7c773,0xdc5267fd),LL(0xc3afcc98,0x4919cc88),LL(0x2db8cd4b,0xaa2e6ab0), LL(0xd0c63eaa,0xd46fec04),LL(0x19ffa832,0xa1cb92c5),LL(0xe43a631f,0x678dd178),LL(0x3dc788b3,0xfb5ae1cd), + LL(0x6e77de04,0x68b4fb90),LL(0xf06dbb97,0x7992bcf0),LL(0xc417c01d,0x896e6a13),LL(0xb956be01,0x8d96332c), LL(0x413aa2b9,0x902fc93a),LL(0xfc98c8a5,0x99a4d915),LL(0x565f1137,0x52c29407),LL(0x21e4f281,0x4072690f), + LL(0x02ff6072,0x36e607cf),LL(0x8ad98cdc,0xa47d2ca9),LL(0xf5f56609,0xbf471d1e),LL(0xf264ada0,0xbcf86623), LL(0xaa9e5cb6,0xb70c0687),LL(0x17401c6c,0xc98124f2),LL(0xd4a61435,0x8189635f),LL(0xa9d98ea6,0xd28fb8af), + LL(0x40c251f8,0xb9a67c2a),LL(0xa2da44be,0x88cd5d87),LL(0xe09b5423,0x437deb96),LL(0x64287dc1,0x150467db), LL(0xcdabb839,0xe161debb),LL(0xf1839a3e,0xa79e9742),LL(0x652d202b,0xbb8dd3c2),LL(0xe9f97d96,0x7b3e67f7), + LL(0xb1cb6ac9,0x5aa5d78f),LL(0xca1d0d45,0xffa13e8e),LL(0x2ba5bf95,0x369295dd),LL(0x39aff05e,0xd68bd1f8), LL(0x26d783f2,0xaf0d86f9),LL(0xfc3aafc1,0x543a59b3),LL(0x7b7da97c,0x3fcf81d2),LL(0xd25dee46,0xc990a056), + LL(0x519cce2c,0x3e6775b8),LL(0xae13d863,0xfc9af71f),LL(0x47c1605c,0x774a4a6f),LL(0x2fd205e8,0x46ba4245), LL(0xd3fd524d,0xa06feea4),LL(0x6de1acc2,0x1e724641),LL(0x334e2b42,0xf53816f1),LL(0x922f0024,0x49e5918e), + LL(0x65c7322d,0x439530b6),LL(0xb3c1b3fb,0xcf12cc01),LL(0x0172f685,0xc70b0186),LL(0x1b58391d,0xb915ee22), LL(0xa317db24,0x9afdf03b),LL(0x17b8ffc4,0x87dec659),LL(0xe4d3d050,0x7f46597b),LL(0x006500e7,0x80a1c1ed), + LL(0x78bf030e,0x84902a96),LL(0x50560148,0xfb5e9c9a),LL(0x63362426,0x6dae0a92),LL(0xa9e30c40,0xdcaeecf4), LL(0x518d0c6b,0xc0d887bb),LL(0xcb985b9d,0x99181152),LL(0xef7bc381,0xad186898),LL(0x9ee46201,0x18168ffb), + LL(0x2502753c,0x9a04cdaa),LL(0x51407c41,0xbb279e26),LL(0xf23564e5,0xeacb03aa),LL(0x71e61016,0x18336582), LL(0xeb809877,0x8684b8c4),LL(0xea0e672e,0xb336e18d),LL(0x34ee5867,0xefb601f0),LL(0x1341cfd1,0x2733edbe), + LL(0x26025c3c,0xb15e809a),LL(0x9350df88,0xe6e981a6),LL(0x8502fd8e,0x92376237),LL(0x0c12be9b,0x4791f216), LL(0x25f02425,0xb7256789),LL(0x7a974443,0xec863194),LL(0xfb41cc52,0x7c0ce882),LL(0xf25c07f2,0xc266ff7e), + LL(0x017025f3,0x3d4da8c3),LL(0xfb9579b4,0xefcf628c),LL(0x1f3716ec,0x5c4d0016),LL(0x6801116e,0x9c27ebc4), LL(0x1da1767e,0x5eba0ea1),LL(0x47004c57,0xfe151452),LL(0x8c2373b7,0x3ace6df6),LL(0x5dbc37ac,0x75c3dffe), + LL(0xddc925fc,0x3dc32a73),LL(0x2f65ee0b,0xb679c841),LL(0x451cbfeb,0x715a3295),LL(0xf76e9a29,0xd9889768), LL(0xb28ad247,0xec20ce7f),LL(0x00894d79,0xe99146c4),LL(0x9f5e3ea7,0x71457d7c),LL(0x38030031,0x097b2662), + LL(0xcf9f82a8,0xdb7f6ae6),LL(0x438f473a,0x319decb9),LL(0x283856c3,0xa63ab386),LL(0xb06a361b,0x13e3172f), LL(0x7d5a006c,0x2959f8dc),LL(0x75fba752,0x2dbc27c6),LL(0x87c22c9e,0xc1227ab2),LL(0x71a268b2,0x06f61f75), + LL(0x04779ce2,0x1b6bb971),LL(0x0aadcb1d,0xaca83812),LL(0xaeaab2d5,0x297ae0bc),LL(0x5bfb9f13,0xa5c14ee7), LL(0xf17a62c7,0xaa00c583),LL(0x173759f6,0x39eb962c),LL(0x86c9a88f,0x1eeba1d4),LL(0xdf016c5e,0x0ab6c37a), + LL(0xa28a0749,0xa2a147db),LL(0xee519165,0x246c20d6),LL(0xd3810715,0x5068d1b1),LL(0x748160b9,0xb1e7018c), LL(0xf380ff62,0x03f5b1fa),LL(0xf3cb2c1e,0xef7fb1dd),LL(0xfc91a7da,0xeab539a8),LL(0xf3f9b561,0x83ddb707), + LL(0xfe7df7a4,0xc550e211),LL(0x063f6f40,0xa7cd07f2),LL(0x2976879c,0xb0de3635),LL(0xe55741da,0xb5f83f85), LL(0xf3d8ac3d,0x4ea9d25e),LL(0x62819f02,0x6fe2066f),LL(0xcef4a564,0x4ab2b9c2),LL(0x5ffa2de3,0x1e155d96), + LL(0xc3a72d00,0x0eb0a19b),LL(0x8513c31b,0x4037665b),LL(0x04c64637,0x2fb2b6bf),LL(0x08cdc639,0x45c34d6e), LL(0xf01fd796,0x56f1e10f),LL(0xfe3667b8,0x4dfb8101),LL(0x9021d0c0,0xe0eda253),LL(0x8a06c6ab,0x7a94e9ff), + LL(0xbb9aa882,0x2d3bb0d9),LL(0xec05fd10,0xea20e4e5),LL(0x1a1ca64e,0xed7eeb5f),LL(0xc6327cbd,0x2fa6b43c), LL(0x3aa91121,0xb577e3cf),LL(0x3a34079b,0x8c6bd5ea),LL(0x60e02fc0,0xd7e5ba39),LL(0x90141bf8,0xf16dd2c3), + LL(0x80101b98,0xb57276d9),LL(0xb82f0f66,0x760883fd),LL(0x4bc3eff3,0x89d7de75),LL(0x5dc2ab40,0x03b60643), LL(0xe05beeac,0xcd6e53df),LL(0xbc3325cd,0xf2f1e862),LL(0x774f03c3,0xdd0f7921),LL(0x4552cc1b,0x97ca7221), + LL(0x1cd19f72,0x5a0d6afe),LL(0xf183fbeb,0xa20915dc),LL(0x832c403c,0x9fda4b40),LL(0xbe425442,0x32738edd), LL(0xb5eccf1a,0x469a1df6),LL(0x28bbe1f0,0x4b5aff42),LL(0x570dfc93,0x31359d7f),LL(0xf0088628,0xa18be235), + LL(0xb00ed3a9,0xa5b30fba),LL(0x73cdf8be,0x34c61374),LL(0xabc56797,0x2c5c5f46),LL(0xb82a8ae2,0x5cecf93d), LL(0xa968fbf0,0x7d3dbe41),LL(0x1a5c7f3d,0xd23d4583),LL(0xc087a9c7,0xf28f69a0),LL(0x474471ca,0xc2d75471), + LL(0x4eb732ec,0x36ec9f4a),LL(0xb1ca6bed,0x6c943bbd),LL(0xf2457892,0xd64535e1),LL(0xf7e2ac06,0x8b84a8ea), LL(0x2499dd5f,0xe0936cd3),LL(0x0ed04e57,0x12053d7e),LL(0xe4305d9d,0x4bdd0076),LL(0x1f67f0a2,0x34a527b9), + LL(0x9cec46ea,0xe79a4af0),LL(0x658b9bc7,0xb15347a1),LL(0x35af2f75,0x6bd2796f),LL(0x4051c435,0xac957990), LL(0xc33a655d,0x2669dda3),LL(0x88514aa3,0x5d503c2e),LL(0x3753dd41,0xdfa11337),LL(0x0b754f78,0x3f054673), + LL(0x496125bd,0xbf185677),LL(0x3775006c,0xfb0023c8),LL(0x3a037899,0xfa0f072f),LL(0x0e4aea57,0x4222b6eb), LL(0x7866d25a,0x3dde5e76),LL(0x4837aa6f,0xb6eb04f8),LL(0x2cf1cdb8,0x5315591a),LL(0x2d4e683c,0x6dfb4f41), + LL(0x48ee1f3a,0x7e923ea4),LL(0x05a2afd5,0x9604d9f7),LL(0x40ea4948,0xbe1d4a33),LL(0xb44cbd2f,0x5b45f1f4), LL(0x4acc757e,0x5faf8376),LL(0x63d68ff7,0xa7cf9ab8),LL(0xdf0e404b,0x8ad62f69),LL(0x12bdafdf,0xd65f33c2), + LL(0xa377b14e,0xc365de15),LL(0x8e39f60c,0x6bf5463b),LL(0x2ce68148,0x62030d2d),LL(0xe6f843a8,0xd95867ef), LL(0xef5ab017,0xd39a0244),LL(0x4ab55d12,0x0bd2d8c1),LL(0x41639169,0xc9503db3),LL(0xf7660c8a,0x2d4e25b0), + LL(0xe224c5d7,0x760cb3b5),LL(0x68616919,0xfa3baf8c),LL(0x8d142552,0x9fbca113),LL(0x7669ebf5,0x1ab18bf1), LL(0x9bdf25dd,0x55e6f53e),LL(0xcb6cd154,0x04cc0bf3),LL(0x95e89080,0x595bef49),LL(0x104a9ac1,0xfe9459a8), + LL(0xcce9bb32,0xad2d89ca),LL(0xf7de8285,0xddea65e1),LL(0xb351bd4b,0x62ed8c35),LL(0x0c0e19a7,0x4150ff36), LL(0x345f4e47,0x86e3c801),LL(0x203a266c,0x3bf21f71),LL(0x855b1f13,0x7ae110d4),LL(0x07262517,0x5d6aaf6a), + LL(0x813d28f1,0x1e0f12e1),LL(0x7ad7a523,0x6000e11d),LL(0xc744a17b,0xc7d8deef),LL(0x14c05a00,0x1e990b48), LL(0x93e976d5,0x68fddaee),LL(0x46610d63,0x696241d1),LL(0x893dda88,0xb204e7c3),LL(0x6a3a6946,0x8bccfa65), + LL(0xc5cd1411,0xb59425b4),LL(0xff3658b1,0x701b4042),LL(0x4784cf93,0xe3e56bca),LL(0x8fe68d60,0x27de5f15), LL(0xf8d53f19,0x4ab9cfce),LL(0xa40a730d,0xddb10311),LL(0x4eee0a8a,0x6fa73cd1),LL(0x5249719d,0xfd548748), + LL(0xa8123ef0,0x49d66316),LL(0xe7f95438,0x73c32db4),LL(0x0d9e7854,0x2e2ed209),LL(0x9d9f0507,0xf98a9329), LL(0x0c6aa20a,0xc5d33cf6),LL(0x75279bb2,0x9a32ba14),LL(0x774a7307,0x7e3202cb),LL(0xe8c42dbd,0x64ed4bc4), + LL(0xd4caed0d,0xc20f1a06),LL(0x171d22b3,0xb8021407),LL(0xd13268d7,0xd426ca04),LL(0x25f4d126,0x92377007), LL(0x71f21a85,0x4204cbc3),LL(0xf82369ba,0x18461b7a),LL(0x3fc858f9,0xc0c07d31),LL(0xe2bab569,0x5deb5a50), + LL(0xd5eea89e,0xd5959d46),LL(0x08437f4b,0xfdff8424),LL(0x3cfe254f,0xf21071e4),LL(0x95468321,0x72417696), LL(0x102cae3e,0x5d8288b9),LL(0xf1965dff,0x2d143e3d),LL(0xa078d847,0x00c9a376),LL(0x26028731,0x6fc0da31), + LL(0xe45083a2,0xa2baeadf),LL(0x5e5b4bcd,0x66bc7218),LL(0xd04b8e7f,0x2c826442),LL(0x6c4b586b,0xc19f5451), LL(0x5b7eeed5,0x60182c49),LL(0x7aa9dfa1,0xd9954ecd),LL(0xc73884ad,0xa403a8ec),LL(0x9bb39041,0x7fb17de2), + LL(0xabb020e8,0x694b64c5),LL(0x19c4eec7,0x3d18c184),LL(0x1c4793e5,0x9c4673ef),LL(0x056092e6,0xc7b8aeb5), LL(0xf0f8c16b,0x3aa1ca43),LL(0xd679b2f6,0x224ed5ec),LL(0x55a205c9,0x0d56eeaf),LL(0x4b8e028b,0xbfe115ba), + LL(0x3927f4fe,0x97e60849),LL(0x759aa7c5,0xf91fbf94),LL(0x6be90a51,0x985af769),LL(0x78ccb823,0xc1277b78), LL(0xe7a75952,0x395b656e),LL(0x928da5f5,0x00df7de0),LL(0x4ca4454f,0x09c23175),LL(0x7aa2d3c1,0x4ec971f4), + LL(0xe75d9ccc,0x45c3c507),LL(0x3dc90306,0x63b7be8a),LL(0x5db44bdc,0x37e09c66),LL(0x6841c6a2,0x50d60da1), LL(0x08df1b12,0x6f9b65ee),LL(0x7ff089df,0x38734879),LL(0x3fe8013d,0x9c331a66),LL(0x5f42fcc8,0x017f5de9), + LL(0xe8e57567,0x43077866),LL(0xf9fcdb18,0xc9f781ce),LL(0x9b12e174,0x38131dda),LL(0x8a03752a,0x25d84aa3), LL(0x4d0c0ce2,0x45e09e09),LL(0x92bebba5,0x1564008b),LL(0xa87284c7,0xf7e8ad31),LL(0x97e7bbaa,0xb7c4b46c), + LL(0x97acf4ec,0x3e22a7b3),LL(0x5ea8b640,0x0426c400),LL(0x4e969285,0x5e3295a6),LL(0xa6a45670,0x22aabc59), LL(0x5f5942bc,0xb929714c),LL(0xfa3182ed,0x9a6168bd),LL(0x104152ba,0x2216a665),LL(0xb6926368,0x46908d03), +}, +/* digit=11 base_pwr=2^77 */ +{ + LL(0x5a1251fb,0xa9f5d874),LL(0xc72725c7,0x967747a8),LL(0x31ffe89e,0x195c33e5),LL(0xe964935e,0x609d210f), LL(0x2fe12227,0xcafd6ca8),LL(0x0426469d,0xaf9b5b96),LL(0x5693183c,0x2e9ee04c),LL(0xc8146fef,0x1084a333), + LL(0xaed1d1f7,0x96649933),LL(0x50563090,0x566eaff3),LL(0xad2e39cf,0x345057f0),LL(0x1f832124,0x148ff65b), LL(0xcf94cf0d,0x042e89d4),LL(0x520c58b3,0x319bec84),LL(0x5361aa0d,0x2a267626),LL(0x8fbc87ad,0xc86fa302), + LL(0x5c8b06d5,0xfc83d2ab),LL(0xfe4eac46,0xb1a785a2),LL(0x846f7779,0xb99315bc),LL(0xef9ea505,0xcf31d816), LL(0x15d7dc85,0x2391fe6a),LL(0xb4016b33,0x2f132b04),LL(0x181cb4c7,0x29547fe3),LL(0x650155a1,0xdb66d8a6), + LL(0xadc1696f,0x6b66d7e1),LL(0x0acd72d0,0x98ebe593),LL(0xcc1b7435,0x65f24550),LL(0xb4b9a5ec,0xce231393), LL(0xdb067df9,0x234a22d4),LL(0xcaff9b00,0x98dda095),LL(0x6100c9c1,0x1bbc75a0),LL(0x939cf695,0x1560a9c8), + LL(0x99e0925f,0xcf006d3e),LL(0x6322375a,0x2dd74a96),LL(0xb56af5ba,0xc58b446a),LL(0xe0b9b4f1,0x50292683), LL(0x1aeaffa3,0xe2c34cb4),LL(0x9b9587c1,0x8b17203f),LL(0xead1350c,0x6d559207),LL(0xfb7f9604,0x2b66a215), + LL(0xfe51bf74,0x0850325e),LL(0x5e460094,0x9c4f579e),LL(0x76da2f25,0x5c87b92a),LL(0x6febef33,0x889de4e0), LL(0x646083ce,0x6900ec06),LL(0xbfe12773,0xbe2a0335),LL(0xc5344110,0xadd1da35),LL(0xb802cd20,0x757568b7), + LL(0x00f7e6c8,0x75559779),LL(0x0facd2f0,0x38e8b94f),LL(0x03fde375,0xfea1f3af),LL(0x75881dfc,0x5e11a1d8), LL(0xc1e2f2ef,0xb3a6b02e),LL(0xc605a6c5,0x193d2bbb),LL(0x339a0b2d,0x325ffeee),LL(0x9e0c8846,0x27b6a724), + LL(0xf1c367ca,0xe4050f1c),LL(0xc90fbc7d,0x9bc85a9b),LL(0xe1a11032,0xa373c4a2),LL(0xad0393a9,0xb64232b7), LL(0x167dad29,0xf5577eb0),LL(0x94b78ab2,0x1604f301),LL(0xe829348b,0x0baa94af),LL(0x41654342,0x77fbd8dd), + LL(0xb964e39a,0xdab50ea5),LL(0xd0d3c76e,0xd4c29e3c),LL(0x56d11964,0x80dae67c),LL(0xe5ffcc2f,0x7307a8bf), LL(0x91708c3b,0x65bbc1aa),LL(0x28bf0eeb,0xa151e62c),LL(0x6fa34db7,0x6cb53381),LL(0xa29403a8,0x5139e05c), + LL(0x94a7cd2e,0x6ff651b4),LL(0x0699336c,0x5671ffd1),LL(0x979a896a,0x6f5fd2cc),LL(0xd8148cef,0x11e893a8), LL(0x65cf7b10,0x988906a1),LL(0xc50d8485,0x81b67178),LL(0x8a35b3de,0x7c0deb35),LL(0xc1d29799,0x423ac855), + LL(0xdac50b74,0xaf580d87),LL(0x5869734c,0x28b2b89f),LL(0x874e28fb,0x99a3b936),LL(0x25f3f73a,0xbb2c9190), LL(0x84a9d5b7,0x199f6918),LL(0x7e770374,0x7ebe2325),LL(0x0738efe2,0xf442e107),LL(0xcf9082d2,0xcf9f3f56), + LL(0x09618708,0x719f69e1),LL(0xc183f9b1,0xcc9e8364),LL(0x366a21af,0xec203a95),LL(0x068b141f,0x6aec5d6d), LL(0x994f04e9,0xee2df78a),LL(0x271245b0,0xb39ccae8),LL(0x97e43f4f,0xb875a4a9),LL(0xdb2cea98,0x507dfe11), + LL(0x489b03e9,0x4fbf81cb),LL(0x6ec414fa,0xdb86ec5b),LL(0xf51b3ae5,0xfad444f9),LL(0x1914e3fe,0xca7d33d6), LL(0x0ae6c4d0,0xa9c32f5c),LL(0x73969568,0xa9ca1d1e),LL(0x1aa7467e,0x98043c31),LL(0xe21b5ac6,0xe832e75c), + LL(0x5232123d,0x314b7aea),LL(0x65ae86db,0x08307c8c),LL(0xaa4668ed,0x06e7165c),LL(0xb4d3ec39,0xb170458b), LL(0xc19bb986,0x4d2e3ec6),LL(0xae0304ed,0xc5f34846),LL(0x6c9f9722,0x917695a0),LL(0x4cab1c0a,0x6c7f7317), + LL(0x9d6d2e8b,0x6295940e),LL(0x549f7c97,0xd318b8c1),LL(0x97713885,0x22453204),LL(0xa8a440fe,0x468d834b), LL(0xbfba796e,0xd81fe5b2),LL(0x6d71f116,0x152364db),LL(0xb5b66e53,0xbb8c7c59),LL(0x2641a192,0x0b12c61b), + LL(0xfcf0a7fd,0x31f14802),LL(0x5488b01e,0x42fd0789),LL(0x9952b498,0x71d78d6d),LL(0x07ac5201,0x8eb572d9), LL(0x4d194a88,0xe0a2a44c),LL(0xba017e66,0xd2b63fd9),LL(0xf888aefc,0x78efc6c8),LL(0x4a881a11,0xb76f6bda), + LL(0xb46c2397,0x187f314b),LL(0x5ded2819,0x004cf566),LL(0x38764d34,0xa9ea5704),LL(0x78084709,0xbba45217), LL(0x1171121e,0x06474571),LL(0xe7c9b671,0xad7b7eb1),LL(0x730f7507,0xdacfbc40),LL(0xc7ad7bd1,0x178cd8c6), + LL(0xb2a67238,0xbf0be101),LL(0xaf9c14f2,0x3556d367),LL(0xa5662075,0x104b7831),LL(0x79d9e60a,0x58ca59bb), LL(0xa569a73b,0x4bc45392),LL(0x5698f6c9,0x517a52e8),LL(0xaeadd755,0x85643da5),LL(0x2a581b84,0x1aed0cd5), + LL(0x80af1372,0xb9b4ff84),LL(0xf1ba5d1f,0x244c3113),LL(0xf5f98d31,0x2a5dacbe),LL(0x4375bc2a,0x2c3323e8), LL(0x5594b1dd,0x17a3ab4a),LL(0xceb4797e,0xa1928bfb),LL(0xe4886a19,0xe83af245),LL(0x72b5a74a,0x8979d546), + LL(0x19f9e967,0xa0f726bc),LL(0xe8fbbf4e,0xd9d03152),LL(0xb7707d40,0xcfd6f51d),LL(0x63f6e6e0,0x633084d9), LL(0x55667eaf,0xedcd9cdc),LL(0x2e44d56f,0x73b7f92b),LL(0x4e962b14,0xfb2e39b6),LL(0xf671fcbf,0x7d408f6e), + LL(0x164a89bb,0xcc634ddc),LL(0x3ef3bd05,0x74a42bb2),LL(0x428decbb,0x1280dbb2),LL(0x402c8596,0x6103f6bb), LL(0x355a5752,0xfa2bf581),LL(0x00946674,0x562f96a8),LL(0x6da0223b,0x4e4ca16d),LL(0x28d3aa25,0xfe47819f), + LL(0xf8dfcf8a,0x9eea3075),LL(0x95669825,0xa284f0aa),LL(0x867d3fd8,0xb3fca250),LL(0x269d691e,0x20757b5f), LL(0x93b8a5de,0xf2c24020),LL(0xebc06da6,0xd3f93359),LL(0xb2739c33,0x1178293e),LL(0xbcd686e5,0xd2a3e770), + LL(0xcd941534,0xa76f49f4),LL(0xe3c71c0e,0x0d37406b),LL(0x3b97f7e3,0x172d9397),LL(0xbd7fd0de,0xec17e239), LL(0x6f496ba2,0xe3290551),LL(0x36ad50e7,0x6a693172),LL(0x83e7eff5,0xc4e539a2),LL(0x18e1b4cf,0x752737e7), + LL(0x68af43ee,0xa2f7932c),LL(0x703d00bd,0x5502468e),LL(0x2fb061f5,0xe5dc978f),LL(0x28c815ad,0xc9a1904a), LL(0x470c56a4,0xd3af538d),LL(0x193d8ced,0x159abc5f),LL(0x20108ef3,0x2a37245f),LL(0x223f7178,0xfa17081e), + LL(0x10c8c0f5,0x27b0fb2b),LL(0x40650547,0x2102c3ea),LL(0x8ac3bfa7,0x594564df),LL(0x509dad96,0x98102033), LL(0xf1d18a13,0x6989643f),LL(0xd7fc5af0,0x35eebd91),LL(0xfaeaafd8,0x078d096a),LL(0xdef3de98,0xb7a89341), + LL(0xecf2a73a,0x2a206e8d),LL(0x8e551994,0x066a6397),LL(0xb98d53a2,0x3a6a088a),LL(0x2d1124aa,0x0ce7c67c), LL(0x759a113c,0x48cec671),LL(0x4f6f67fa,0xe3b373d3),LL(0xfd36727b,0x5455d479),LL(0xa13c0d81,0xe5a428ee), + LL(0x1c86682b,0xb853dbc8),LL(0xb8d02b2a,0xb78d2727),LL(0x8ebc329a,0xaaf69bed),LL(0x293b2148,0xdb6b40b3), LL(0xb8c4961f,0xe42ea77d),LL(0x20e5e0ab,0xb1a12f7c),LL(0x79e8b05e,0xa0ec5274),LL(0xfab60a80,0x68027391), + LL(0x16b1bd5e,0x6bfeea5f),LL(0x4de30ad3,0xf957e420),LL(0x6a353b9e,0xcbaf664e),LL(0x26d14feb,0x5c873312), LL(0xb65f57cb,0x4e87f98c),LL(0x5e0cdd41,0xdb60a621),LL(0xa6881440,0x67c16865),LL(0x46ab52aa,0x1093ef1a), + LL(0x3f4ece64,0xc095afb5),LL(0x7604551a,0x6a6bb02e),LL(0x0b26b8cd,0x55d44b4e),LL(0xf971268a,0xe5f9a999), LL(0x11a7de84,0xc08ec425),LL(0xfda469dd,0x83568095),LL(0x6c6c90a2,0x737bfba1),LL(0xbe229831,0x1cb9c4a0), + LL(0xbb2eec64,0x93bccbba),LL(0xda03adbe,0xa0c23b64),LL(0xe0e86ac4,0x5f7aa00a),LL(0xfc1401e6,0x470b941e), LL(0x9df43574,0x5ad8d679),LL(0x0f65d810,0x4ccfb8a9),LL(0xaa7fbd81,0x1bce80e3),LL(0x9508d20a,0x273291ad), + LL(0x42a92806,0xf5c4b46b),LL(0xa86ab44a,0x810684ec),LL(0xca0bc9f8,0x4591640b),LL(0x5c4b6054,0xb5efcdfc), LL(0x6e9edd12,0x16fc8907),LL(0xd4d792f9,0xe29d0b50),LL(0x9b03116d,0xa45fd01c),LL(0xc81765a4,0x85035235), + LL(0xb4b4b67c,0x1fe2a9b2),LL(0xe8020604,0xc1d10df0),LL(0xbc8058d8,0x9d64abfc),LL(0x712a0fbb,0x8943b9b2), LL(0x3b3def04,0x90eed914),LL(0x4ce775ff,0x85ab3aa2),LL(0x7bbc9040,0x605fd4ca),LL(0xe2c75dfb,0x8b34a564), + LL(0x10358560,0x41ffc94a),LL(0x9e5c28aa,0x2d8a5072),LL(0x4cc7eb15,0xe915a0fc),LL(0x8f6d0f5d,0xe9efab05), LL(0xd19e9b91,0xdbab47a9),LL(0x0276154c,0x8cfed745),LL(0x2cfede0d,0x154357ae),LL(0x19f5a4ef,0x520630df), + LL(0xe382360f,0x25759f7c),LL(0x88bf5857,0xb6db05c9),LL(0x6c58d46c,0x2917d61d),LL(0xfd20cb7a,0x14f8e491), LL(0x11c20340,0xb68a727a),LL(0xaf7ccbb6,0x0386f86f),LL(0xfee09a20,0x5c8bc6cc),LL(0xbb7eea35,0x7d76ff4a), + LL(0xdb15be7a,0xa7bdebe7),LL(0xd89f0302,0x67a08054),LL(0xc1193364,0x56bf0ea9),LL(0x62837ebe,0xc8244467), LL(0x20d841b8,0x32bd8e8b),LL(0xdbb8a54f,0x127a0548),LL(0x63b20236,0x83dd4ca6),LL(0x203491fa,0x87714718), + LL(0xaa8a5288,0x4dabcaaa),LL(0xaf23a1c9,0x91cc0c8a),LL(0x3f220e0c,0x34c72c6a),LL(0x1232144a,0xbcc20bdf), LL(0xa20ede1b,0x6e2f42da),LL(0x74a00515,0xc441f00c),LL(0x734b8c4b,0xbf46a5b6),LL(0x7b56c9a4,0x57409503), + LL(0xe4585d45,0x9f735261),LL(0x6734e642,0x9231faed),LL(0xbe70ee6c,0x1158a176),LL(0x7c3501bf,0x35f1068d), LL(0xa2d26115,0x6beef900),LL(0xef0afee3,0x649406f2),LL(0xbc2420a1,0x3f43a60a),LL(0xd5aee4ac,0x509002a7), + LL(0x3ff3571b,0xb46836a5),LL(0x837927c1,0x24f98b78),LL(0x4533c716,0x6254256a),LL(0xd07ee196,0xf27abb0b), LL(0x5c6d5bfd,0xd7cf64fc),LL(0xf0cd7a77,0x6915c751),LL(0x8798f534,0xd9f59012),LL(0xf81d8b5f,0x772b0da8), + LL(0x2e03fa69,0x1244260c),LL(0x3be1a374,0x36cf0e3a),LL(0xef06b960,0x6e7c1633),LL(0x671f90f6,0xa71a4c55), LL(0x33c673db,0x7a941251),LL(0x73e8c131,0xc0bea510),LL(0xd4f6c734,0x61a8a699),LL(0x341ed001,0x25e78c88), + LL(0x8e2f7d90,0x5c18acf8),LL(0x77be32cd,0xfdbf33d7),LL(0xd2eb5ee9,0x0a085cd7),LL(0xb3201115,0x2d702cfb), LL(0x85c88ce8,0xb6e0ebdb),LL(0x1e01d617,0x23a3ce3c),LL(0x567333ac,0x3041618e),LL(0x157edb6b,0x9dd0fd8f), + LL(0xb57872b8,0x27f74702),LL(0x657d5fe1,0x2ef26b4f),LL(0x57cf3d40,0x95426f0a),LL(0x65a6067a,0x847e2ad1), LL(0x09996a74,0xd474d9a0),LL(0x2a26115c,0x16a56acd),LL(0xd16f4d43,0x02a615c3),LL(0xaadb85b7,0xcc3fc965), + LL(0xce07d1b0,0x386bda73),LL(0x58ad4178,0xd82910c2),LL(0xcd2617f4,0x124f82cf),LL(0xef691770,0xcc2f5e8d), LL(0xb8c30ccc,0x82702550),LL(0x1a8e575a,0x7b856aea),LL(0xb1ab9459,0xbb822fef),LL(0xec24e38e,0x085928bc), + LL(0xba8f4b4d,0x5d0402ec),LL(0x00b4d58b,0xc07cd4ba),LL(0x29227e7a,0x5d8dffd5),LL(0x31bf386f,0x61d44d0c), LL(0x135e6f4d,0xe486dc2b),LL(0xe79410ef,0x680962eb),LL(0xf10088b5,0xa61bd343),LL(0xe2e28686,0x6aa76076), + LL(0x8fb98871,0x80463d11),LL(0xbbc76aff,0xcb26f5c3),LL(0xfbe03614,0xd4ab8edd),LL(0xc0cf2dee,0xc8eb579b), LL(0xc93bae41,0xcc004c15),LL(0x3aeca3b2,0x46fbae5d),LL(0x0f1e9ab1,0x671235cf),LL(0x9ec285c1,0xadfba934), + LL(0xf216c980,0x88ded013),LL(0xf79e0bc1,0xc8ac4fb8),LL(0xfb97a237,0xa29b89c6),LL(0x9922d8e7,0xb697b780), LL(0xddb945b5,0x3142c639),LL(0xe094c3a9,0x447b06c7),LL(0x72266c90,0xcdcb3642),LL(0xa9385046,0x633aad08), + LL(0xb57c6477,0xa36c936b),LL(0xe94dbcc6,0x871f8b64),LL(0xa591a67b,0x28d0fb62),LL(0xc1d926f5,0x9d40e081), LL(0xf2d84b5a,0x3111eaf6),LL(0xa565b644,0x228993f9),LL(0x2c83188b,0x0ccbf592),LL(0x3df3e197,0xf87b30ab), + LL(0x7642bca8,0xb8658b31),LL(0x52800f17,0x1a032d7f),LL(0x79bf9445,0x051dcae5),LL(0x54a2e253,0xeba6b8ee), LL(0xd4485692,0x5c8b9cad),LL(0x8986e9be,0x84bda40e),LL(0x2f0db448,0xd16d16a4),LL(0xa14d4188,0x8ec80050), + LL(0x98fa7aaa,0xb2b26107),LL(0xf073aa4e,0x41209ee4),LL(0xf2d6b19b,0xf1570359),LL(0xfc577caf,0xcbe6868c), LL(0x32c04dd3,0x186c4bdc),LL(0xcfeee397,0xa6c35fae),LL(0xf086c0cf,0xb4a1b312),LL(0xd9461fe2,0xe0a5ccc6), + LL(0x1536189f,0xc32278aa),LL(0xba6df571,0x1126c55f),LL(0xb194560e,0x0f71a602),LL(0x324bd6e1,0x8b2d7405), LL(0x3738be71,0x8481939e),LL(0x1a4d97a9,0xb5090b1a),LL(0xf05ba915,0x116c65a3),LL(0xaae448aa,0x21863ad3), + LL(0xa7aae5d3,0xd24e2679),LL(0x0de5c1c4,0x7076013d),LL(0xbb05b629,0x2d50f8ba),LL(0x6e66efbb,0x73c1abe2), LL(0xf2488af7,0xefd4b422),LL(0x663ba575,0xe4105d02),LL(0x53a69457,0x7eb60a8b),LL(0xc945973b,0x62210008), + LL(0x77a50ec6,0xfb255478),LL(0x0a37a72c,0xbf0392f7),LL(0x4be18e7a,0xa0a7a19c),LL(0x25b1e0af,0x90d8ea16), LL(0xef953f57,0x7582a293),LL(0xbdc5465a,0x90a64d05),LL(0xe2510717,0xca79c497),LL(0x18cb641f,0x560dbb7c), + LL(0x4b66abfb,0x1d8e3286),LL(0x59030900,0xd26f52e5),LL(0x5584941a,0x1ee3f643),LL(0x569f5958,0x6d3b3730), LL(0x4789dba5,0x9ff2a62f),LL(0x72b5c9b7,0x91fcb815),LL(0x6c8f9a0e,0xf446cb7d),LL(0x39b7ecb5,0x48f625c1), + LL(0x1c6219b8,0xbabae801),LL(0x28ac2f23,0xe7a562d9),LL(0x26e20588,0xe1b48732),LL(0x775af051,0x06ee1cad), LL(0xfaff79f7,0xda29ae43),LL(0x652ee9e0,0xc141a412),LL(0x195f4bd0,0x1e127f6f),LL(0x072f34f8,0x29c6ab4f), + LL(0x30448112,0x7b7c1477),LL(0xe4a38656,0x82b51af1),LL(0x2f315010,0x2bf2028a),LL(0x6ea88cd4,0xc9a4a01f), LL(0x257e5818,0xf63e95d8),LL(0xb4519b16,0xdd8efa10),LL(0x0da910bf,0xed8973e0),LL(0x5c0fe4a9,0xed49d077), + LL(0xb7caee1e,0xac3aac5e),LL(0xa7f4da57,0x1033898d),LL(0x5c6669b9,0x42145c0e),LL(0xc1aa2aa0,0x42daa688), LL(0x1a1d885a,0x629cc15c),LL(0xf4b76817,0x25572ec0),LL(0x9c8f8f28,0x8312e435),LL(0x81965490,0x8107f8cd), + LL(0x6fa6110c,0x516ff3a3),LL(0xfb93561f,0x74fb1eb1),LL(0x8457522b,0x6c0c9047),LL(0x6bb8bdc6,0xcfd32104), LL(0xcc80ad57,0x2d6884a2),LL(0x86a9b637,0x7c27fc35),LL(0xadf4e8cd,0x3461baed),LL(0x617242f0,0x1d56251a), + LL(0xc955bef4,0x0b80d209),LL(0x06adb047,0xdf02cad2),LL(0x5ec74fee,0xf0d7cb91),LL(0x1111ba44,0xd2503375), LL(0xdf53cb36,0x9671755e),LL(0x3368551b,0x54dcb612),LL(0xc8a025a4,0x66d69aac),LL(0xe77ef445,0x6be946c6), + LL(0xa995e094,0x719946d1),LL(0xe51e04d8,0x65e848f6),LL(0x6a1e3113,0xe62f3300),LL(0x501de503,0x1541c7c1), LL(0xf4acfade,0x4daac9fa),LL(0x44cd0b71,0x0e585897),LL(0x0a51cd77,0x544fd869),LL(0x0031016d,0x60fc20ed), + LL(0xa4276867,0x58b404ec),LL(0x34f34993,0x46f6c3cc),LL(0xc636e5bd,0x477ca007),LL(0x7c458b47,0x8018f5e5), LL(0xe47b668f,0xa1202270),LL(0xee14f203,0xcef48ccd),LL(0x62ff9b4d,0x23f98bae),LL(0xc589eddd,0x55acc035), + LL(0x64db4444,0x3fe712af),LL(0xbecdd480,0x19e9d634),LL(0xa930978a,0xe08bc047),LL(0xa1280733,0x2dbf24ec), LL(0x2cd706b2,0x3c0ae38c),LL(0x359017b9,0x5b012a5b),LL(0x72e0f5ae,0x3943c38c),LL(0x57176fa3,0x786167ea), + LL(0x594881dc,0xe5f9897d),LL(0xcfb820c1,0x6b5efad8),LL(0xd55018de,0xb2179093),LL(0x0bac56ce,0x39ad7d32), LL(0x2cfc0e81,0xb55122e0),LL(0xf6d89daa,0x117c4661),LL(0xcb64fa09,0x362d01e1),LL(0x3e9c4ddd,0x6a309b4e), + LL(0xabea49b1,0xfa979fb7),LL(0x10e2c6c5,0xb4b1d27d),LL(0x23afde7a,0xbd61c2c4),LL(0x9786d358,0xeb6614f8), LL(0x7f6f7459,0x4a5d816b),LL(0x09360e7b,0xe431a44f),LL(0xc309914c,0x8c27a032),LL(0xcaede3d8,0xcea5d68a), + LL(0x3a0a3f95,0x3668f665),LL(0x7ceba27b,0x89369416),LL(0xe4728fe9,0x89981fad),LL(0x8a093562,0x7102c8a0), LL(0x235d21c8,0xbb80310e),LL(0xbefb7f7b,0x505e55d1),LL(0x12958a67,0xa0a90811),LL(0x4d851fef,0xd67e106a), + LL(0x431dd80e,0xb84011a9),LL(0x73306cd9,0xeb7c7cca),LL(0xd1b3b730,0x20fadd29),LL(0xfe37b3d3,0x83858b5b), LL(0xb6251d5c,0xbf4cd193),LL(0x1352d952,0x1cca1fd3),LL(0x90fbc051,0xc66157a4),LL(0x89b98636,0x7990a638), +}, +/* digit=12 base_pwr=2^84 */ +{ + LL(0x87dec0e1,0xe5aa692a),LL(0xf7b39d00,0x010ded8d),LL(0x54cfa0b5,0x7b1b80c8),LL(0xa0f8ea28,0x66beb876), LL(0x3476cd0e,0x50d7f531),LL(0xb08d3949,0xa63d0e65),LL(0x53479fc6,0x1a09eea9),LL(0xf499e742,0x82ae9891), + LL(0x5ca7d866,0xab58b910),LL(0x3adb3b34,0x582967e2),LL(0xcceac0bc,0x89ae4447),LL(0x7bf56af5,0x919c667c), LL(0x60f5dcd7,0x9aec17b1),LL(0xddcaadbc,0xec697b9f),LL(0x463467f5,0x0b98f341),LL(0xa967132f,0xb187f1f7), + LL(0x214aeb18,0x90fe7a1d),LL(0x741432f7,0x1506af3c),LL(0xe591a0c4,0xbb5565f9),LL(0xb44f1bc3,0x10d41a77), LL(0xa84bde96,0xa09d65e4),LL(0xf20a6a1c,0x42f060d8),LL(0xf27f9ce7,0x652a3bfd),LL(0x3b3d739f,0xb6bdb65c), + LL(0xec7fae9f,0xeb5ddcb6),LL(0xefb66e5a,0x995f2714),LL(0x69445d52,0xdee95d8e),LL(0x09e27620,0x1b6c2d46), LL(0x8129d716,0x32621c31),LL(0x0958c1aa,0xb03909f1),LL(0x1af4af63,0x8c468ef9),LL(0xfba5cdf6,0x162c429f), + LL(0x753b9371,0x2f682343),LL(0x5f1f9cd7,0x29cab45a),LL(0xb245db96,0x571623ab),LL(0x3fd79999,0xc507db09), LL(0xaf036c32,0x4e2ef652),LL(0x05018e5c,0x86f0cc78),LL(0xab8be350,0xc10a73d4),LL(0x7e826327,0x6519b397), + LL(0x9c053df7,0xe8cb5eef),LL(0xb300ea6f,0x8de25b37),LL(0xc849cffb,0xdb03fa92),LL(0xe84169bb,0x242e43a7), LL(0xdd6f958e,0xe4fa51f4),LL(0xf4445a8d,0x6925a77f),LL(0xe90d8949,0xe6e72a50),LL(0x2b1f6390,0xc66648e3), + LL(0x173e460c,0xb2ab1957),LL(0x30704590,0x1bbbce75),LL(0xdb1c7162,0xc0a90dbd),LL(0x15cdd65d,0x505e399e), LL(0x57797ab7,0x68434dcb),LL(0x6a2ca8e8,0x60ad35ba),LL(0xde3336c1,0x4bfdb1e0),LL(0xd8b39015,0xbbef99eb), + LL(0x1711ebec,0x6c3b96f3),LL(0xce98fdc4,0x2da40f1f),LL(0x57b4411f,0xb99774d3),LL(0x15b65bb6,0x87c8bdf4), LL(0xc2eef12d,0xda3a89e3),LL(0x3c7471f3,0xde95bb9b),LL(0xd812c594,0x600f225b),LL(0x2b75a56b,0x54907c5d), + LL(0x8db60e35,0xa93cc5f0),LL(0xfa833319,0x743e3cd6),LL(0xf81683c9,0x7dad5c41),LL(0x9c34107e,0x70c1e7d9), LL(0xa6be0907,0x0edc4a39),LL(0x86d0b7d3,0x36d47035),LL(0x272bfa60,0x8c76da03),LL(0x0f08a414,0x0b4a07ea), + LL(0x45c1dd53,0x699e4d29),LL(0x231debb5,0xcadc5898),LL(0xa77f00e0,0xdf49fcc7),LL(0xa73e5a0e,0x93057bbf), LL(0x027a4cd1,0x2f8b7ecd),LL(0xc614011a,0x114734b3),LL(0x67677c68,0xe7a01db7),LL(0x7e273f4f,0x89d9be5e), + LL(0x089808ef,0xd225cb2e),LL(0xd59e4107,0xf1f7a27d),LL(0x8211b9c9,0x53afc761),LL(0xe6819159,0x0361bc67), LL(0x7f071426,0x2a865d0b),LL(0xe7072567,0x6a3c1810),LL(0x0d6bcabd,0x3e3bca1e),LL(0x408591bc,0xa1b02bc1), + LL(0x31fba239,0xe0deee59),LL(0x98bd91d1,0xf47424d3),LL(0x071a3c1d,0x0f8886f4),LL(0xa819233b,0x3f7d41e8), LL(0xcf6eb998,0x708623c2),LL(0x609a287f,0x86bb49af),LL(0x63c90762,0x942bb249),LL(0x55a9654b,0x0ef6eea5), + LL(0x36f5defe,0x5f6d2d72),LL(0x56f99176,0xfa9922dc),LL(0xf78ce0c7,0x6c8c5ece),LL(0xbe09b55e,0x7b44589d), LL(0x9ea83770,0xe11b3bca),LL(0x2ab71547,0xd7fa2c7f),LL(0x2a1ddcc0,0x2a3dd6fa),LL(0x5a7b7707,0x09acb430), + LL(0x649d4e57,0x4add4a2e),LL(0x1917526e,0xcd53a2b0),LL(0x20b44ac4,0xc5262330),LL(0xbaa2c31d,0x4028746a), LL(0x64291d4c,0x51318390),LL(0xee5ad909,0xbf48f151),LL(0x7b185681,0xcce57f59),LL(0x4854d442,0x7c3ac1b0), + LL(0xc093c171,0x65587dc3),LL(0x24f42b65,0xae7acb24),LL(0x955996cb,0x5a338adb),LL(0x6051f91b,0xc8e65675), LL(0x28b8d0b1,0x66711fba),LL(0xb6c10a90,0x15d74137),LL(0x3a232a80,0x70cdd7eb),LL(0x6191ed24,0xc9e2f07f), + LL(0xf79588c0,0xa80d1db6),LL(0xb55768cc,0xfa52fc69),LL(0x7f54438a,0x0b4df1ae),LL(0xf9b46a4f,0x0cadd1a7), LL(0x1803dd6f,0xb40ea6b3),LL(0x55eaae35,0x488e4fa5),LL(0x382e4e16,0x9f047d55),LL(0x2f6e0c98,0xc9b5b7e0), + LL(0x95762649,0x6b1bd2d3),LL(0xc7aea3f6,0xa9604ee7),LL(0x6dc6f896,0x3646ff27),LL(0x2860bad1,0x9bf0e7f5), LL(0x7cb44b92,0x2d92c821),LL(0xaea9c182,0xa2f5ce63),LL(0x9154a5fd,0xd0a2afb1),LL(0x95801da6,0x482e474c), + LL(0xb611c24b,0xc19972d0),LL(0x60a8f351,0x1d468e65),LL(0x7bcf6421,0xeb758069),LL(0x88fbc491,0xec9dd0ee), LL(0x956c2e32,0x5b59d2bf),LL(0xdcddf94e,0x73dc6864),LL(0xbcee7665,0xfd5e2321),LL(0x5e9a06c4,0xa7b4f8ef), + LL(0x7280f855,0xfba918dd),LL(0x8baec688,0xbbaac260),LL(0x33400f42,0xa3b3f00f),LL(0x66f2e6e4,0x3d2dba29), LL(0x98509375,0xb6f71a94),LL(0xcea423cc,0x8f33031f),LL(0x4807e6fb,0x009b8dd0),LL(0x5cdb954c,0x5163cfe5), + LL(0xcf41c6e8,0x03cc8f17),LL(0x037b925c,0xf1f03c2a),LL(0x66d2427c,0xc39c19cc),LL(0x7b6c18e4,0x823d24ba), LL(0x901f0b4f,0x32ef9013),LL(0xf8941c2e,0x684360f1),LL(0x2c28092e,0x0ebaff52),LL(0x256c932f,0x7891e4e3), + LL(0xac445e3d,0x51264319),LL(0x8ea74381,0x553432e7),LL(0x67e9c50a,0xe6eeaa69),LL(0x62e628c7,0x27ced284), LL(0x7a4afa57,0x3f96d375),LL(0xe484c150,0xde0a14c3),LL(0x38bd9923,0x364a24eb),LL(0xe5177422,0x1df18da0), + LL(0xd8d38a9b,0x174e8f82),LL(0xe7de1391,0x2e97c600),LL(0xa1c175dd,0xc5709850),LL(0x32ae5035,0x969041a0), LL(0x76a2086b,0xcbfd533b),LL(0xd7c2e8fe,0xd6bba71b),LL(0x099dfb67,0xb2d58ee6),LL(0x064a85d9,0x3a8b342d), + LL(0x522f9be3,0x3bc07649),LL(0xdf1f49a8,0x690c075b),LL(0x3854ec42,0x80e1aee8),LL(0x17689dc7,0x2a7dbf44), LL(0x3faf4078,0xc004fc0e),LL(0xdf11862c,0xb2f02e9e),LL(0xa0a1b7b3,0xf10a5e0f),LL(0x8936ec80,0x30aca623), + LL(0x02f40d9a,0xf83cbf05),LL(0x2c318a4d,0x4681c468),LL(0x0e9c2674,0x98575618),LL(0x1847092e,0xbe79d046), LL(0x78bd01e0,0xaf1e480a),LL(0x72a51db9,0x6dd359e4),LL(0xe3afbab6,0x62ce3821),LL(0x17733199,0xc5cee5b6), + LL(0x6ffd9fbb,0xe08b30d4),LL(0x36c610b7,0x6e5bc699),LL(0x9ce262cf,0xf343cff2),LL(0x68b914c1,0xca2e4e35), LL(0x16de36c5,0x011d64c0),LL(0x42e2b829,0xe0b10fdd),LL(0x6685aaf8,0x78942981),LL(0x230ede97,0xe7511708), + LL(0x3b922bf8,0x671ed8fc),LL(0x4c29b133,0xe4d8c0a0),LL(0x3b6e99c4,0x87eb1239),LL(0x8793beba,0xaff3974c), LL(0x2c18df9b,0x03749405),LL(0x91007139,0xc5c3a293),LL(0xe37a0b95,0x6a77234f),LL(0xb661c96b,0x02c29a21), + LL(0x141ecf61,0xc3aaf1d6),LL(0x3bb22f53,0x9195509e),LL(0x22d51357,0x29597404),LL(0x537bed60,0x1b083822), LL(0xe07289f0,0xcd7d6e35),LL(0x6dd86eff,0x1f94c48c),LL(0xeb0f9cfa,0xc8bb1f82),LL(0x1b2eb97d,0x9ee0b7e6), + LL(0x34d74e31,0x5a52fe2e),LL(0x3bf79ab6,0xa352c310),LL(0xabfeeb8f,0x97ff6c5a),LL(0xf5c97305,0xbfbe8fef), LL(0xa7904608,0xd6081ce6),LL(0xc4fca249,0x1f812f3a),LL(0xb9e5e200,0x9b24bc9a),LL(0x38012ee8,0x91022c67), + LL(0x30a713a1,0xe83d9c5d),LL(0x84ef0f93,0x4876e3f0),LL(0xc1fbf928,0xc9777029),LL(0xbce7d2a4,0xef7a6bb3), LL(0xdfa2a659,0xb8067228),LL(0xd877a48f,0xd5cd3398),LL(0x025d0f3f,0xbea4fd8f),LL(0x2eae7c2b,0xd67d2e35), + LL(0xcc5f4394,0x184de7d7),LL(0x4536e142,0xb5551b5c),LL(0xd34aa60a,0x2e89b212),LL(0xf50051d5,0x14a96fea), LL(0x0d12bb0b,0x4e21ef74),LL(0x60b9677e,0xc522f020),LL(0x2df7731d,0x8b12e467),LL(0x7b326d31,0x39f80382), + LL(0x39024a94,0xdfb8630c),LL(0x97319452,0xaacb96a8),LL(0xeda3867c,0xd68a3961),LL(0x77c4ffca,0x0c58e2b0), LL(0x4da919fa,0x3d545d63),LL(0xf15e2289,0xef79b69a),LL(0x808bab10,0x54bc3d3d),LL(0x45f82c37,0xc8ab3007), + LL(0x7c4a658a,0xc12738b6),LL(0x40e72182,0xb3c47639),LL(0x8798e44f,0x3b77be46),LL(0x17a7f85f,0xdc047df2), LL(0x5e59d92d,0x2439d4c5),LL(0xe8e64d8d,0xcedca475),LL(0x87ca9b16,0xa724cd0d),LL(0xa5540dfe,0x35e4fd59), + LL(0xe4bcf6b1,0xf8c1ff18),LL(0x295018fa,0x856d6285),LL(0x3263c949,0x433f665c),LL(0xa1f21409,0xa6a76dd6), LL(0xcc7b4f79,0x17d32334),LL(0x06720e4a,0xa1d03122),LL(0x81d9bed5,0xadb6661d),LL(0x11db15d1,0xf0d6fb02), + LL(0x1fb747d2,0x7fd11ad5),LL(0x3033762b,0xab50f959),LL(0xfbefaf5a,0x2a7e711b),LL(0x3fef2bbf,0xc7393278), LL(0x0df6f9be,0xe29fa244),LL(0x71efd215,0x9092757b),LL(0x4f3d6fd9,0xee60e311),LL(0x0acfb78b,0x338542d4), + LL(0x38961a0f,0x44a23f08),LL(0x986987ca,0x1426eade),LL(0x4a863cc6,0x36e6ee2e),LL(0x628b8b79,0x48059420), LL(0x7396e1de,0x30303ad8),LL(0x38c5aad1,0x5c8bdc48),LL(0x5c8f5066,0x3e40e11f),LL(0x8d246bbd,0xabd6e768), + LL(0x23330a01,0x68aa40bb),LL(0xc34eafa0,0xd23f5ee4),LL(0x5de02c21,0x3bbee315),LL(0xd1d8dd06,0x18dd4397), LL(0x122d7b44,0x3ba1939a),LL(0xa33870d6,0xe6d3b40a),LL(0x1c4fe3f8,0x8e620f70),LL(0xd3a50cbf,0xf6bba1a5), + LL(0xcfc0aee0,0x4a78bde5),LL(0xc08c50bd,0x847edc46),LL(0xad63c9b2,0xbaa2439c),LL(0x10fc2acb,0xceb4a728), LL(0x26da033d,0xa419e40e),LL(0x03e02683,0x6cc3889d),LL(0xfdccf725,0x1cd28559),LL(0x8d13d208,0x0fd7e0f1), + LL(0x1f0df9d4,0x01b9733b),LL(0xa2b5e4f3,0x8cc2c5f3),LL(0x3a304fd4,0x43053bfa),LL(0x0a9f1aa7,0x8e87665c), LL(0xd73dc965,0x087f29ec),LL(0x3e9023db,0x15ace455),LL(0x2bce28b4,0x2370e309),LL(0xb6b1e84a,0xf9723442), + LL(0xb72d9f26,0xbeee662e),LL(0xf0e47109,0xb19396de),LL(0xe13289d0,0x85b1fa73),LL(0x54e58e32,0x436cf77e), LL(0xe990ef77,0x0ec833b3),LL(0x1b11fc25,0x7373e3ed),LL(0x0fc332ce,0xbe0eda87),LL(0x8d7ea856,0xced04970), + LL(0x7e977ca0,0xf85ff785),LL(0xdfdd5d2b,0xb66ee8da),LL(0x905af461,0xf5e37950),LL(0x966d487c,0x587b9090), LL(0x32ba0127,0x6a198a1b),LL(0x141615ac,0xa7720e07),LL(0x996ef2f2,0xa23f3499),LL(0x470bcb3d,0xef5f64b4), + LL(0x92b8c559,0xa526a962),LL(0x69740a0f,0x0c14aac0),LL(0xa6bdc0a5,0x0d41a9e3),LL(0x9c48aef4,0x97d52106), LL(0x3e7c253b,0xcf16bd30),LL(0x47fdedc1,0xcc834b1a),LL(0x373aab2e,0x7362c6e5),LL(0xc5f590ff,0x264ed85e), + LL(0x66d41870,0x7a46d9c0),LL(0x4787ba09,0xa50c20b1),LL(0xe3d44635,0x185e7e51),LL(0x31e2d8dc,0xb3b3e080), LL(0xa179e9d9,0xbed1e558),LL(0x74a76781,0x2daa3f79),LL(0x3a40864f,0x4372baf2),LL(0x4fe75cb5,0x46900c54), + LL(0xf76765d0,0xb95f171e),LL(0x95c87502,0x4ad726d2),LL(0x4d7c99bd,0x2ec769da),LL(0xc36cdfa8,0x5e2ddd19), LL(0xa93e6dea,0xc22117fc),LL(0x93771123,0xe8a2583b),LL(0xfa08a3a2,0xbe2f6089),LL(0x8f0e1112,0x4809d5ed), + LL(0xda7a095e,0x3b414aa3),LL(0x26f5aadd,0x9049acf1),LL(0x6be8b84a,0x78d46a4d),LL(0xb732b9b3,0xd66b1963), LL(0xde6e9555,0x5c2ac2a0),LL(0xb5bd8770,0xcf52d098),LL(0x0fd28921,0x15a15fa6),LL(0x8b27536d,0x56ccb81e), + LL(0x9f4ccbb8,0x0f0d8ab8),LL(0xdb221729,0xed5f44d2),LL(0x00bed10c,0x43141988),LL(0x1d735b8b,0xc94348a4), LL(0x29ef8479,0x79f3e9c4),LL(0x614c693f,0x4c13a4e3),LL(0x8e143a14,0x32c9af56),LL(0xe29ac5c4,0xbc517799), + LL(0x2774856f,0x05e17992),LL(0x6c1bf55f,0x6e52fb05),LL(0xe4f19e16,0xaeda4225),LL(0xaf5ccb26,0x70f4728a), LL(0xb2947f22,0x5d2118d1),LL(0x281d6fb9,0xc827ea16),LL(0x8cf0eabd,0x8412328d),LL(0x03ef9dcf,0x45ee9fb2), + LL(0xbb937d63,0x8e700421),LL(0xcc4b37a6,0xdf8ff2d5),LL(0x5ced7b68,0xa4c0d5b2),LL(0xc7308f59,0x6537c1ef), LL(0x3b37f8e8,0x25ce6a26),LL(0xdeebc6ce,0x170e9a9b),LL(0x8728d72c,0xdd037952),LL(0x850154bc,0x445b0e55), + LL(0x83a7337b,0x4b7d0e06),LL(0xffecf249,0x1e3416d4),LL(0x66a2b71f,0x24840eff),LL(0xb37cc26d,0xd0d9a50a), LL(0x6fe28ef7,0xe2198150),LL(0x23324c7f,0x3cc5ef16),LL(0x769b5263,0x220f3455),LL(0xa10bf475,0xe2ade2f1), + LL(0x458d3671,0x28cd20fa),LL(0x2dc4847b,0x1549722c),LL(0x591941e3,0x6dd01e55),LL(0x27128ccb,0x0e6fbcea), LL(0x3bef0262,0xae1a1e6b),LL(0x8f54e103,0xfa8c472c),LL(0x72c052ec,0x7539c0a8),LL(0x5a3490e9,0xd7b27369), + LL(0x71684349,0x143fe1f1),LL(0x32e19b97,0x36b4722e),LL(0x90980aff,0xdc059227),LL(0x9e13d674,0x175c9c88), LL(0x6e6bfdb1,0xa7de5b22),LL(0xbedb4b46,0x5ea5b7b2),LL(0xd34a6e44,0xd5570191),LL(0xa24ff7e6,0xfcf60d2e), + LL(0x677819e1,0x614a392d),LL(0xaa5a29e8,0x7be74c7e),LL(0x63c85f3f,0xab50fece),LL(0x46cab337,0xaca2e2a9), LL(0x122a6fe3,0x7f700388),LL(0x882a04a8,0xdb69f703),LL(0xcf7aed57,0x9a77935d),LL(0x8d91c86f,0xdf16207c), + LL(0x63ed9998,0x2fca49ab),LL(0xa77ddf96,0xa3125c44),LL(0x24344072,0x05dd8a86),LL(0xfec3fb56,0xa023dda2), LL(0x0c743032,0x421b41fc),LL(0x5e438639,0x4f2120c1),LL(0xc83c1b07,0xfb7cae51),LL(0xcac2171a,0xb2370caa), + LL(0x6cc820fb,0x2eb2d962),LL(0xb85a44bf,0x59feee5c),LL(0x5b6598f0,0x94620fca),LL(0x7e314051,0x6b922cae), LL(0x106bed4e,0xff8745ad),LL(0xdfa1e9ab,0x546e71f5),LL(0x1ec29487,0x935c1e48),LL(0x4d936530,0x9509216c), + LL(0x85c9a2db,0xc7ca3067),LL(0x6be8606f,0xd6ae5152),LL(0xe14c651d,0x09dbcae6),LL(0x9bc32f96,0xc9536e23), LL(0x34521b03,0xa90535a9),LL(0x878756ff,0xf39c526c),LL(0x8aedf03c,0x383172ec),LL(0xefe0c034,0x20a8075e), + LL(0x64026422,0xf22f9c62),LL(0x24b9d076,0x8dd10780),LL(0x3bef2950,0x944c742a),LL(0x88a2b00b,0x55b9502e), LL(0x86a09817,0xa59e14b4),LL(0x47bb4071,0xa39dd3ac),LL(0x3be0592f,0x55137f66),LL(0xc9e63f5b,0x07fcafd4), + LL(0x346eb226,0x963652ee),LL(0xec2facb7,0x7dfab085),LL(0x691add26,0x273bf2b8),LL(0xf2b46c44,0x30d74540), LL(0xf2c2d065,0x05e8e73e),LL(0xd42eeac9,0xff9b8a00),LL(0x97209d22,0x2fcbd205),LL(0xde14ea2c,0xeb740ffa), + LL(0xa8aef518,0xc71ff913),LL(0xfff4cfa2,0x7bfc74bb),LL(0xb6b36048,0x1716680c),LL(0x9ef79af1,0x121b2cce), LL(0xa01eb3d3,0xbff3c836),LL(0x5f79077b,0x50eb1c6a),LL(0xa004bbcf,0xa48c32d6),LL(0x7d64f61d,0x47a59316), + LL(0x93102016,0x6068147f),LL(0x94d12576,0x12c5f654),LL(0xc9bc6b91,0xefb071a7),LL(0x6e23ea95,0x7c2da0c5), LL(0xd4a1dd5d,0xf4fd45b6),LL(0x9122b13c,0x3e7ad9b6),LL(0xe6f57a48,0x342ca118),LL(0x06f8288f,0x1c2e94a7), + LL(0x5a97d231,0x99e68f07),LL(0x4d838758,0x7c80de97),LL(0x05872727,0xbce0f5d0),LL(0x19c4d016,0xbe5d95c2), LL(0x9c2492ee,0x921d5cb1),LL(0x404d6fb3,0x42192dc1),LL(0x32f988d3,0x4c84dcd1),LL(0xa17b8e85,0xde26d61f), + LL(0x137c7408,0xc466dcb6),LL(0x36a266da,0x9a38d7b6),LL(0x83bebf1b,0x7ef5cb06),LL(0x0fd014e3,0xe5cdcbbf), LL(0xf65965a0,0x30aa376d),LL(0xebb3e95e,0x60fe88c2),LL(0x66ee6f20,0x33fd0b61),LL(0x3f41f0a0,0x8827dcdb), + LL(0x0c56c690,0xbf8a9d24),LL(0xddb7641d,0x40265dad),LL(0x3a6b662b,0x522b05bf),LL(0xb1478c9b,0x466d1dfe), LL(0x1484469b,0xaa616962),LL(0x02df8f9f,0x0db60549),LL(0x3cb8bf51,0xc37bca02),LL(0x21371ce8,0x5effe346), + LL(0xff112c32,0xe8f65264),LL(0x7b971fb2,0x8a9c736d),LL(0x7b75080d,0xa4f19470),LL(0x8839c59b,0xfc3f2c5a), LL(0x5aeb49c2,0x1d6c777e),LL(0xda1addfe,0xf3db034d),LL(0x5535affc,0xd76fee5a),LL(0xb92251fd,0x0853ac70), + LL(0x8b2a29d5,0x37e3d594),LL(0x4de00ddb,0x28f1f457),LL(0xf42c328b,0x8083c1b5),LL(0xe493c73b,0xd8ef1d8f), LL(0x41dc61bd,0x96fb6260),LL(0x27ee2f8a,0xf74e8a9d),LL(0x2c946a5d,0x7c605a80),LL(0x3839ccfd,0xeed48d65), + LL(0x3a29467a,0x9894344f),LL(0xc51eba6d,0xde81e949),LL(0xa5e5c2f2,0xdaea066b),LL(0x08c8c7b3,0x3fc8a614), LL(0x06d0de9f,0x7adff88f),LL(0x3b75ce0a,0xbbc11cf5),LL(0xfbbc87d5,0x9fbb7acc),LL(0x7badfde2,0xa1458e26), +}, +/* digit=13 base_pwr=2^91 */ +{ + LL(0xe039c256,0x1cb43668),LL(0x7c17fd5d,0x5f26fb8b),LL(0x79aa062b,0xeee426af),LL(0xd78fbf04,0x072002d0), LL(0xe84fb7e3,0x4c9ca237),LL(0x0c82133d,0xb401d8a1),LL(0x6d7e4181,0xaaa52592),LL(0x73dbb152,0xe9430833), + LL(0xbe24319a,0xf92dda31),LL(0xe095a8e7,0x03f7d28b),LL(0x98782185,0xa52fe840),LL(0x29c24dbc,0x276ddafe), LL(0x1d7a64eb,0x80cd5496),LL(0x7f1dbe42,0xe4360889),LL(0x8438d2d5,0x2f81a877),LL(0x85169036,0x7e4d52a8), + LL(0x1d59715d,0x19e3d5b1),LL(0xd788983e,0xc7eaa762),LL(0xabf1f248,0xe5a730b0),LL(0xfae3fd83,0xfbab8084), LL(0x53765b2f,0x65e50d21),LL(0xfa127f3d,0xbdd4e083),LL(0x397b1b10,0x9cf3c074),LL(0xb1b59fd3,0x59f8090c), + LL(0x615faa8f,0x7b15fd9d),LL(0x968554ed,0x8fa1eb40),LL(0x7aa44882,0x7bb4447e),LL(0x029fff32,0x2bb2d0d1), LL(0x6caa6d2f,0x075e2a64),LL(0x22e7351b,0x8eb879de),LL(0x9a506c62,0xbcd5624e),LL(0xa87e24dc,0x218eaef0), + LL(0x44ddfa35,0x37e56847),LL(0xdab3f747,0x9ccfc5c5),LL(0x1ee96cf4,0x9ac1df3f),LL(0x3b480b8f,0x0c0571a1), LL(0x4b3a7b3c,0x2fbeb3d5),LL(0x5dcdbb99,0x35c03669),LL(0xb2415b3a,0x52a0f5dc),LL(0x4413ed9a,0xd57759b4), + LL(0x3d30a2c5,0x1fe647d8),LL(0xf78a81dc,0x0857f77e),LL(0x131a4a9b,0x11d5a334),LL(0x29d393f5,0xc0a94af9), LL(0xdaa6ec1a,0xbc3a5c0b),LL(0x88d2d7ed,0xba9fe493),LL(0xbb614797,0xbb4335b4),LL(0x72f83533,0x991c4d68), + LL(0xd2f01cb3,0x53258c28),LL(0xd75db0b1,0x93d6eaa3),LL(0xe87d0db4,0x419a2b0d),LL(0xd8fe8493,0xa1e48f03), LL(0xc508b23a,0xf747faf6),LL(0x35d53549,0xf137571a),LL(0xfcf9b838,0x9f5e58e2),LL(0xa7fd3cf5,0xc7186cee), + LL(0xe978a1d3,0x77b868ce),LL(0x7ab92d04,0xe3a68b33),LL(0x87a5b862,0x51029794),LL(0x3a61d41d,0x5f0606c3), LL(0x6f9326f1,0x2814be27),LL(0xc6fe3c2e,0x2f521c14),LL(0xacdf7351,0x17464d7d),LL(0x777f7e44,0x10f5f9d3), + LL(0x269fb37d,0xce8e616b),LL(0x7de62de5,0xaaf73804),LL(0x4fdd4153,0xaba11175),LL(0x3770b49b,0x515759ba), LL(0xaa423a61,0x8b09ebf8),LL(0xcd41fb92,0x592245a1),LL(0x9b4c8936,0x1cba8ec1),LL(0xaf36710e,0xa87e91e3), + LL(0x3d34a2e3,0x1fd84ce4),LL(0xb43b5d61,0xee3759ce),LL(0x619186c7,0x895bc78c),LL(0xcbb9725a,0xf19c3809), LL(0xde744b1f,0xc0be21aa),LL(0x60f8056b,0xa7d222b0),LL(0xb23efe11,0x74be6157),LL(0x0cd68253,0x6fab2b4f), + LL(0x4bf1d725,0xad33ea5f),LL(0x4f6c950f,0x9c1d8ee2),LL(0xa377af06,0x544ee78a),LL(0x94a113e1,0x54f489bb), LL(0x992fb7e8,0x8f11d634),LL(0xa2a44347,0x0169a7aa),LL(0x95020e00,0x1d49d4af),LL(0xe08e120b,0x95945722), + LL(0xa4d32282,0xb6e33878),LL(0x48020ae7,0xe36e029d),LL(0x37a9b750,0xe05847fb),LL(0xb29e3819,0xf876812c), LL(0xd23a17f0,0x84ad138e),LL(0xf0b3950e,0x6d7b4480),LL(0x2fd67ae0,0xdfa8aef4),LL(0x52333af6,0x8d3eea24), + LL(0xb15d5acc,0x0d052075),LL(0xbd815bc4,0xc6d9c79f),LL(0xdfa36cf2,0x8dcafd88),LL(0x38aa9070,0x908ccbe2), LL(0xba35afce,0x638722c4),LL(0xfd6abf0b,0x5a3da8b0),LL(0xc9c335c1,0x2dce252c),LL(0x65aa799b,0x84e7f0de), + LL(0xb99a72cb,0x2101a522),LL(0x87618016,0x06de6e67),LL(0xe6f3653e,0x5ff8c7cd),LL(0xc7a6754a,0x0a821ab5), LL(0x7cb0b5a2,0x7e3fa52b),LL(0xc9048790,0xa7fb121c),LL(0x06ce053a,0x1a725020),LL(0x04e929b0,0xb490a31f), + LL(0x62dd61ad,0xe17be47d),LL(0x6be01371,0x781a961c),LL(0xdae3cbba,0x1063bfd3),LL(0x7f73c9ba,0x35647406), LL(0x2736a129,0xf50e957b),LL(0xed13f256,0xa6313702),LL(0x3a19fcc5,0x9436ee65),LL(0xe7a4c8b6,0xcf2bdb29), + LL(0xc5f95cd8,0xb06b1244),LL(0xf4ab95f4,0xda8c8af0),LL(0xb9e5836d,0x1bae59c2),LL(0x3acffffc,0x07d51e7e), LL(0xc2ccbcda,0x01e15e6a),LL(0x8528c3e0,0x3bc1923f),LL(0xa49fead4,0x43324577),LL(0x2aa7a711,0x61a1b884), + LL(0x700230ef,0xf9a86e08),LL(0xbd19adf8,0x0af585a1),LL(0xf55ad8f2,0x7645f361),LL(0x46c3614c,0x6e676223), LL(0x4e774d3f,0x23cb257c),LL(0xac102d1b,0x82a38513),LL(0x7b126aa5,0x9bcddd88),LL(0xeefd3ee4,0xe716998b), + LL(0xfb167583,0x4239d571),LL(0xd16c8f8a,0xdd011c78),LL(0x69a27519,0x271c2895),LL(0xd2d64b6a,0x9ce0a3b7), LL(0xd5ec6738,0x8c977289),LL(0x8840ef6b,0xa3b49f9a),LL(0x9a453419,0x808c14c9),LL(0x0cf0a2d5,0x5c00295b), + LL(0x1d4bcc76,0x524414fb),LL(0x459a88f1,0xb07691d2),LL(0xf70d110f,0x77f43263),LL(0xb7abf9f3,0x64ada5e0), LL(0x5b544cf5,0xafd0f94e),LL(0xfd2713fe,0xb4a13a15),LL(0x250c74f4,0xb99b7d6e),LL(0x20324e45,0x097f2f73), + LL(0xaffa8208,0x994b37d8),LL(0xdc29aafc,0xc3c31b0b),LL(0x7a3a607f,0x3da74651),LL(0xfe6955d6,0xd8e1b8c1), LL(0xc8418682,0x716e1815),LL(0x7dc91d97,0x541d487f),LL(0xc6996982,0x48a04669),LL(0x83a6502e,0xf39cab15), + LL(0xe68db055,0x025801a0),LL(0xba3338d5,0xf3569758),LL(0xee2afa84,0xb0c8c0aa),LL(0xfb6562d1,0x4f6985d3), LL(0x132ed17a,0x351f1f15),LL(0xc04365fe,0x510ed0b4),LL(0xe5b1f066,0xa3f98138),LL(0x32df03dc,0xbc9d95d6), + LL(0x19abd09e,0xa83ccf6e),LL(0x4ff17edb,0x0b4097c1),LL(0xd64a06ce,0x58a5c478),LL(0x544a58fd,0x2ddcc3fd), LL(0x9e8153b8,0xd449503d),LL(0x7774179b,0x3324fd02),LL(0xdbd9120c,0xaf5d47c8),LL(0x34fa94db,0xeb860162), + LL(0x972f07f4,0x5817bdd1),LL(0xd27bbceb,0xe5579e2e),LL(0x5f11e5a6,0x86847a1f),LL(0x7c3cf048,0xb39ed255), LL(0xa2f62e55,0xe1076417),LL(0x1bcf82a2,0x6b9ab38f),LL(0x7aeb29f9,0x4bb7c319),LL(0x17227a46,0xf6d17da3), + LL(0x0f968c00,0xab53ddbd),LL(0x000c880b,0xa03da7ec),LL(0x6a9ad24d,0x7b239624),LL(0x01ec60d0,0x612c0401), LL(0x109f5df1,0x70d10493),LL(0x80af7550,0xfbda4030),LL(0xc6b9a9b3,0x30b93f95),LL(0x007d9418,0x0c74ec71), + LL(0x6edb951f,0x94175564),LL(0x7f22c282,0x5f4a9d78),LL(0xb38d1196,0xb7870895),LL(0xa228ce7c,0xbc593df3), LL(0x6af3641a,0xc78c5bd4),LL(0x3d9b3dcc,0x7802200b),LL(0x8be33304,0x0dc73f32),LL(0x61ffb79a,0x847ed87d), + LL(0x6d671192,0xf85c974e),LL(0xde16f60f,0x1e14100a),LL(0x95c38797,0x45cb0d5a),LL(0x9b022da4,0x18923bba), LL(0xbbe7e86e,0xef2be899),LL(0x216067bf,0x4a1510ee),LL(0x84d5ce3e,0xd98c8154),LL(0xf92a2b90,0x1af777f0), + LL(0x4ef65724,0x9fbcb400),LL(0x3c0ca6fe,0x3e04a4c9),LL(0x55002994,0xfb3e2cb5),LL(0x5363ecab,0x1f3a93c5), LL(0x3923555b,0x1fe00efe),LL(0x1e1751ea,0x744bedd9),LL(0x6ab69357,0x3fb2db59),LL(0xf5e6618b,0x8dbd7365), + LL(0xdf1ea40e,0x99d53099),LL(0x57d61e64,0xb3f24a0b),LL(0x596eb812,0xd088a198),LL(0x5762940b,0x22c8361b), LL(0xf9c0d95c,0x66f01f97),LL(0x8e43cdae,0x88461172),LL(0xb72b15c3,0x11599a7f),LL(0x420d95cc,0x135a7536), + LL(0x5f7ae2f6,0x2dcdf0f7),LL(0xd7fa6da2,0x15fc6e1d),LL(0xd1d441b6,0x81ca829a),LL(0x04a106b6,0x84c10cf8), LL(0xa73fbbd0,0xa9b26c95),LL(0x4d8f6ee8,0x7f24e0cb),LL(0x1e25a043,0x48b45937),LL(0x036f3dfe,0xf8a74fca), + LL(0xc9f84296,0x1ed46585),LL(0x3bc278b0,0x7fbaa8fb),LL(0x6c4fcbd0,0xa8e96cd4),LL(0x73b60a5f,0x940a1202), LL(0x55a4aec8,0x34aae120),LL(0xdbd742f0,0x550e9a74),LL(0x228c68ab,0x794456d7),LL(0xa4e25ec6,0x492f8868), + LL(0xb2d8f398,0x682915ad),LL(0x5b84c953,0xf13b51cc),LL(0x5bb917d6,0xcda90ab8),LL(0x4ea3dee1,0x4b615560), LL(0x0a52c1c8,0x578b4e85),LL(0x20b75fc4,0xeab1a695),LL(0xaa0bb3c6,0x60c14f3c),LL(0xb8216094,0x220f448a), + LL(0xb0e63d34,0x4fe7ee31),LL(0xa9e54fab,0xf4600572),LL(0xd5e7b5a4,0xc0493334),LL(0x06d54831,0x8589fb92), LL(0x6583553a,0xaa70f5cc),LL(0xe25649e5,0x0879094a),LL(0x10044652,0xcc904507),LL(0x02541c4f,0xebb0696d), + LL(0xb9718710,0x5a171fde),LL(0xf374a9f5,0x38f1bed8),LL(0xba39bdc1,0xc8c582e1),LL(0x908cc0ce,0xfc457b0a), LL(0x883841e2,0x9a187fd4),LL(0x38725381,0x8ec25b39),LL(0x96f84395,0x2553ed05),LL(0x6f6c6897,0x095c7661), + LL(0x4bdc5610,0x917ac85c),LL(0x179eb301,0xb2885fe4),LL(0x8b78bdcc,0x5fc65547),LL(0xe59e4699,0x4a9fc893), LL(0x3ce299af,0xbb7ff0cd),LL(0xadf38b20,0x195be9b3),LL(0xd38ddb8f,0x6a929c87),LL(0xb21a51b9,0x55fcc99c), + LL(0x721a4593,0x2b695b4c),LL(0x768eaac2,0xed1e9a15),LL(0x7489f914,0xfb63d71c),LL(0x78118910,0xf98ba31c), LL(0x9b128eb4,0x80291373),LL(0xd448af4a,0x7801214e),LL(0x55418dd3,0xdbd2e22b),LL(0xd3998242,0xeffb3c0d), + LL(0xc7bf3827,0xdfa6077c),LL(0x47f8238f,0xf2165bcb),LL(0x8564d554,0xfe37cf68),LL(0x0a81fb98,0xe5f825c4), LL(0xffed4d6f,0x43cc4f67),LL(0xb50a34b0,0xbc609578),LL(0x5041faf1,0x8aa8fcf9),LL(0x651773b6,0x5659f053), + LL(0x6044d63b,0xe87582c3),LL(0x0cdb0ca0,0xa6089409),LL(0xbfb2bcf6,0x8c993e0f),LL(0x45985cfc,0xfc64a719), LL(0x83dbedba,0x15c4da80),LL(0x2be67df7,0x804ae112),LL(0xa23defde,0xda4c9658),LL(0x5156e0d3,0x12002ddd), + LL(0x5dd21b96,0xe68eae89),LL(0xcf44624d,0x8b99f28b),LL(0x1ec8897a,0x0ae00808),LL(0x6712f76e,0xdd0a9303), LL(0x4e233de4,0x96237522),LL(0x2b36a8a5,0x192445b1),LL(0x023993d9,0xabf9ff74),LL(0x2aad4a8f,0x21f37bf4), + LL(0xf8bd2bbd,0x340a4349),LL(0x4868195d,0x1d902cd9),LL(0xe5fdb6f1,0x3d27bbf1),LL(0x124f9f1c,0x7a5ab088), LL(0xf7a09e03,0xc466ab06),LL(0x31f2c123,0x2f8a1977),LL(0x041b6657,0xda355dc7),LL(0x8ece2a7c,0xcb840d12), + LL(0x7db32675,0xb600ad9f),LL(0x07a06f1b,0x78fea133),LL(0xb31f6094,0x5d032269),LL(0x83ec37aa,0x07753ef5), LL(0x9c0bea78,0x03485aed),LL(0xbc3f4524,0x41bb3989),LL(0x697f726d,0x09403761),LL(0xdf394820,0x6109beb3), + LL(0x3b6d1145,0x804111ea),LL(0xa8582654,0xb6271ea9),LL(0x24e66562,0x619615e6),LL(0xd7b6ad9c,0xa2554945), LL(0x99bfe35f,0xd9c4985e),LL(0x7b51cdf6,0x9770ccc0),LL(0x92881832,0x7c327013),LL(0x286b26d1,0x8777d45f), + LL(0xd847999d,0x9bbeda22),LL(0xc3525d32,0x03aa33b6),LL(0x28a959a1,0x4b7b96d4),LL(0x31e5d234,0xbb3786e5), LL(0x6961f247,0xaeb5d3ce),LL(0x02f93d3f,0x20aa85af),LL(0xd7a7ae4f,0x9cd1ad3d),LL(0x781adaa8,0xbf6688f0), + LL(0x7469cead,0xb1b40e86),LL(0x309fca48,0x1904c524),LL(0x4b54bbc7,0x9b7312af),LL(0x593affa2,0xbe24bf8f), LL(0xbd98764b,0xbe5e0790),LL(0xa26e299e,0xa0f45f17),LL(0x6b8fe4c7,0x4af0d2c2),LL(0x8ae8a3e6,0xef170db1), + LL(0x29e0ccc1,0x0e8d61a0),LL(0x60ad36ca,0xcd53e87e),LL(0xc8173822,0x328c6623),LL(0xa496be55,0x7ee1767d), LL(0x648945af,0x89f13259),LL(0x25c8009c,0x9e45a5fd),LL(0x1f61ab8c,0xaf2febd9),LL(0x8a275385,0x43f6bc86), + LL(0xf2142e79,0x87792348),LL(0xc6e6238a,0x17d89259),LL(0x4a839d9b,0x7536d2f6),LL(0x76a1fbdc,0x1f428fce), LL(0x0db06dfe,0x1c109601),LL(0x50a3a3cc,0xbfc16bc1),LL(0x9b30f41b,0xf9cbd9ec),LL(0x00138cce,0x5b5da0d6), + LL(0x56ef96a7,0xec1d0a48),LL(0x982bf842,0xb47eb848),LL(0xec3f700d,0x66deae32),LL(0xaa1181e0,0x4e43c42c), LL(0xd1a4aa2a,0xa1d72a31),LL(0xc004f3ce,0x440d4668),LL(0x45fe8a7a,0x0d6a2d3b),LL(0xfb128365,0x820e52e2), + LL(0x25e51b09,0x29ac5fcf),LL(0x2023d159,0x180cd2bf),LL(0xa1ebf90e,0xa9892171),LL(0x7c132181,0xf97c4c87), LL(0xc03dbb7e,0x9f1dc724),LL(0x018cbbe4,0xae043765),LL(0x0767d153,0xfb0b2a36),LL(0x249cbaeb,0xa8e2f4d6), + LL(0xd95ea168,0x172a5247),LL(0x2970764a,0x1758fada),LL(0x1d978169,0xac803a51),LL(0xde77e01b,0x299cfe2e), LL(0xb0a98927,0x652a1e17),LL(0x20014495,0x2e26e1d1),LL(0x7175b56a,0x7ae0af9f),LL(0xd64b9f95,0xc2e22a80), + LL(0xd90a060a,0x4d0ff9fb),LL(0xbaf38085,0x496a27db),LL(0xda776bcf,0x32305401),LL(0x725f209e,0xb8cdcef6), LL(0x436a0bba,0x61ba0f37),LL(0x76860049,0x263fa108),LL(0xda3542cf,0x92beb98e),LL(0xd5849538,0xa2d4d14a), + LL(0x12e9a1bc,0x989b9d68),LL(0x5f6e3268,0x61d9075c),LL(0x99ace638,0x352c6aa9),LL(0x920f43ff,0xde4e4a55), LL(0xd673c017,0xe5e4144a),LL(0x6f6e05ea,0x667417ae),LL(0xdcd1bd56,0x613416ae),LL(0x86693711,0x5eb36201), + LL(0x3a1aa914,0x2d7bc504),LL(0x76dc5975,0x175a1299),LL(0x3fc8125c,0xe900e0f2),LL(0x11198875,0x569ef68c), LL(0x63a113b4,0x9012db63),LL(0x98835766,0xe3bd3f56),LL(0x76412dea,0xa5c94a52),LL(0xaa735e5c,0xad9e2a09), + LL(0x508b65e9,0x405a984c),LL(0x6df1a0d1,0xbde4a1d1),LL(0xdfba80da,0x1a9433a1),LL(0x9440ad2e,0xe9192ff9), LL(0x5099fe92,0x9f649696),LL(0x0b27a54a,0x25ddb65c),LL(0xc590da61,0x178279dd),LL(0xfbde681a,0x5479a999), + LL(0x013fe162,0xd0e84e05),LL(0x632d471b,0xbe11dc92),LL(0xfc0e089f,0xdf0b0c45),LL(0x4c144025,0x04fb15b0), LL(0x13c99927,0xa61d5fc2),LL(0x3de2eb35,0xa033e9e0),LL(0xb8dacbb4,0xf8185d5c),LL(0x8644549d,0x9a88e265), + LL(0x54671ff6,0xf717af62),LL(0x5fa58603,0x4bd4241b),LL(0xe67773c0,0x06fba40b),LL(0x6a2847e9,0xc1d933d2), LL(0x689e2c70,0xf4f5acf3),LL(0x46bafd31,0x92aab0e7),LL(0x3473f6e5,0x798d76aa),LL(0x93141934,0xcc6641db), + LL(0xd31e535e,0xcae27757),LL(0x87c2ee11,0x04cc43b6),LL(0x2e029ffa,0x8d1f9675),LL(0xe4cc7a2c,0xc2150672), LL(0x8d68b013,0x3b03c1e0),LL(0xedf298f3,0xa9d6816f),LL(0xa2804464,0x1bfbb529),LL(0x5db22125,0x95a52fae), + LL(0x0e1cb64e,0x55b32160),LL(0x7e7fc9fe,0x004828f6),LL(0x1bb0fb93,0x13394b82),LL(0x35f1a920,0xb6293a2d), LL(0xd145d2d9,0xde35ef21),LL(0xbb8fa603,0xbe6225b3),LL(0x32cf252d,0x00fc8f6b),LL(0x117cf8c2,0xa28e52e6), + LL(0x4c371e6d,0x9d1dc89b),LL(0x36ef0f28,0xcebe0675),LL(0xa4292f81,0x5de05d09),LL(0x353e3083,0xa8303593), LL(0x7e37a9bb,0xa1715b0a),LL(0x2b8faec3,0x8c56f61e),LL(0x33c9b102,0x52507431),LL(0xa44431f0,0x0130cefc), + LL(0xbd865cfb,0x56039fa0),LL(0xbc5f1dd7,0x4b03e578),LL(0xbabe7224,0x40edf2e4),LL(0x3a1988f6,0xc752496d), LL(0x564beb6b,0xd1572d3b),LL(0x39a1c608,0x0db1d110),LL(0x16f60126,0x568d1934),LL(0xf354af33,0x05ae9668), + LL(0xc92544f2,0x19de6d37),LL(0xa35837d5,0xcc084353),LL(0x1a514ece,0xcbb6869c),LL(0x2e1d1066,0xb633e728), LL(0x936c581c,0xf15dd69f),LL(0x7439c4f9,0x96e7b8ce),LL(0x2e448a5b,0x5e676f48),LL(0xfd916bbb,0xb2ca7d5b), + LL(0xf5024025,0xd55a2541),LL(0xe4c2d937,0x47bc5769),LL(0x0362189f,0x7d31b92a),LL(0xef7816f9,0x83f3086e), LL(0xb587579a,0xf9f46d94),LL(0x30e76c5f,0xec2d22d8),LL(0xb000ffcf,0x27d57461),LL(0x364ffc2c,0xbb7e65f9), + LL(0x6652a220,0x7c7c9477),LL(0xd696c981,0x61618f89),LL(0x89effff3,0x5021701d),LL(0x7c314163,0xf2c8ff8e), LL(0x8efb4d3e,0x2da413ad),LL(0xce176d95,0x937b5adf),LL(0x2a67d51c,0x22867d34),LL(0x18eb3ac9,0x262b9b10), + LL(0xc43ff28b,0x4e314fe4),LL(0x6a664e7a,0x76476627),LL(0xb7a565c2,0x3e90e40b),LL(0xc1acf831,0x8588993a), LL(0x8f938829,0xd7b501d6),LL(0x3edd7d4c,0x996627ee),LL(0x90cd34c7,0x37d44a62),LL(0xf3833e8d,0xa8327499), + LL(0x4bf50353,0x2e18917d),LL(0x556765fb,0x85dd726b),LL(0x93d5ab66,0x54fe65d6),LL(0x915c25fe,0x3ddbaced), LL(0x12f22e85,0xa799d9a4),LL(0x6d06f6bc,0xe2a24867),LL(0x43ca1637,0xf4f1ee56),LL(0x61ece30a,0xfda2828b), + LL(0xa2dee7a6,0x758c1a3e),LL(0x734b2284,0xdcde2f3c),LL(0x4eaba6ad,0xaba445d2),LL(0x76cee0a7,0x35aaf668), LL(0xe5aa049a,0x7e0b04a9),LL(0x91103e84,0xe74083ad),LL(0x40afecc3,0xbeb183ce),LL(0xea043f7a,0x6b89de9f), +}, +/* digit=14 base_pwr=2^98 */ +{ + LL(0xfe67ba66,0x0e299d23),LL(0x93cf2f34,0x91450760),LL(0x97fcf913,0xf45b5ea9),LL(0x8bd7ddda,0x5be00843), LL(0xd53ff04d,0x358c3e05),LL(0x5de91ef7,0xbf7ccdc3),LL(0xb69ec1a0,0xad684dbf),LL(0x801fd997,0x367e7cf2), + LL(0xb0dc8595,0x0ca1f3b7),LL(0x9f1d9f2e,0x27de4608),LL(0xbadd82a7,0x1af3bf39),LL(0x65862448,0x79356a79), LL(0xf5f9a052,0xc0602345),LL(0x139a42f9,0x1a8b0f89),LL(0x844d40fc,0xb53eee42),LL(0x4e5b6368,0x93b0bfe5), + LL(0xc024789c,0x5434dd02),LL(0x41b57bfc,0x90dca9ea),LL(0x243398df,0x8aa898e2),LL(0x894a94bb,0xf607c834), LL(0xc2c99b76,0xbb07be97),LL(0x18c29302,0x6576ba67),LL(0xe703a88c,0x3d79efcc),LL(0xb6a0d106,0xf259ced7), + LL(0xc8de610b,0x0f893a5d),LL(0x67e223ce,0xe8c515fb),LL(0x4ead6dc5,0x7774bfa6),LL(0x925c728f,0x89d20f95), LL(0x098583ce,0x7a1e0966),LL(0x93f2a7d7,0xa2eedb94),LL(0x4c304d4a,0x1b282097),LL(0xc077282d,0x0842e3da), + LL(0x3b9e2d7b,0xe4d972a3),LL(0xc48218ff,0x7cc60b27),LL(0x84149d91,0x8fc70838),LL(0x2f461ecc,0x5c04346f), LL(0x614650a9,0xebe9fdf2),LL(0xc1f666ac,0x5e35b537),LL(0x88babc83,0x645613d1),LL(0xc5e1c93e,0x88cace3a), + LL(0x3de92e23,0x209ca375),LL(0x5fbbb6e3,0xccb03cc8),LL(0xd7b1487e,0xccb90f03),LL(0xc710941f,0xfa9c2a38), LL(0x6724ceed,0x756c3823),LL(0x192d0323,0x3a902258),LL(0xea5e038e,0xb150e519),LL(0xc7427591,0xdcba2865), + LL(0x78890732,0xe549237f),LL(0x53fcb4d9,0xc443bef9),LL(0xeb3480d6,0x9884d8a6),LL(0x3048b186,0x8a35b6a1), LL(0x65e9a90a,0xb4e44716),LL(0x653006c0,0x45bf380d),LL(0x4fe9ae3b,0x8f3f820d),LL(0x979a3b71,0x244a35a0), + LL(0x74cd06ff,0xa1010e9d),LL(0xaca3eeac,0x9c17c7df),LL(0x8063aa2b,0x74c86cd3),LL(0x734614ff,0x8595c4b3), LL(0x990f62cc,0xa3de00ca),LL(0xca0c3be5,0xd9bed213),LL(0xdf8ce9f5,0x7886078a),LL(0x5cd44444,0xddb27ce3), + LL(0x58926ddd,0xed374a66),LL(0x908015b8,0x138b2d49),LL(0xde1f7ab8,0x886c6579),LL(0xc3020b7a,0x888b9aa0), LL(0x3a96e355,0xd3ec034e),LL(0xf30fbe9a,0xba65b0b8),LL(0xff21367a,0x064c8e50),LL(0x0b04b46e,0x1f508ea4), + LL(0x747c866c,0x98561a49),LL(0x0518a062,0xbbb1e5fe),LL(0xecdc3608,0x20ff4e8b),LL(0x20184027,0x7f55cded), LL(0xf38c85f0,0x8d73ec95),LL(0x8bc3b8c3,0x5b589fdf),LL(0x0f12b66f,0xbe95dd98),LL(0x0e338e01,0xf5bd1a09), + LL(0x5e915918,0x65163ae5),LL(0x86f8a46b,0x6158d6d9),LL(0xeeebf99c,0x8466b538),LL(0xbca477ef,0xca8761f6), LL(0x9ebbc601,0xaf3449c2),LL(0xe0c3ae2f,0xef3b0f41),LL(0x5de63752,0xaa6c577d),LL(0x64682a51,0xe9166601), + LL(0xfc15aa1e,0x5a3097be),LL(0xb54b0745,0x40d12548),LL(0x519a5f12,0x5bad4706),LL(0xa439dee6,0xed03f717), LL(0x4a02c499,0x0794bb6c),LL(0xcffe71d2,0xf725083d),LL(0x0f3adcaf,0x2cad7519),LL(0x43729310,0x7f68ea1c), + LL(0xb7ffd977,0xe747c8c7),LL(0x80761a22,0xec104c35),LL(0x5a3ffb83,0x8395ebaf),LL(0xe4b63db7,0xfb3261f4), LL(0xd883e544,0x53544960),LL(0x8cc2eeb8,0x13520d70),LL(0xd3d65f99,0x08f6337b),LL(0x781cf95b,0x83997db2), + LL(0x0dbd2c01,0xce6ff106),LL(0x1f9ce934,0x4f8eea6b),LL(0x0e993921,0x546f7c4b),LL(0x5e753fc7,0x6236a324), LL(0xa16022e9,0x65a41f84),LL(0x43d1dbb2,0x0c18d878),LL(0x2d4cef9c,0x73c55640),LL(0x70444c74,0xa0428108), + LL(0x9afdfb3c,0x68e4f15e),LL(0x5bdfb6df,0x49a56143),LL(0x5f823d97,0xa9bc1bd4),LL(0xea111c2a,0xbceb5970), LL(0xb269bbc4,0x366b455f),LL(0xe9bc5d62,0x7cd85e1e),LL(0x4f18b086,0xc743c41c),LL(0x95294fb9,0xa4b40990), + LL(0x26ee8382,0x9c7c581d),LL(0x359d638e,0xcf17dcc5),LL(0xb728ae3d,0xee8273ab),LL(0xf821f047,0x1d112926), LL(0x50491a74,0x11498477),LL(0xfde0dfb9,0x687fa761),LL(0x7ea435ab,0x2c258022),LL(0x91ce7e3f,0x6b8bdb94), + LL(0x3bf834aa,0x4c5b5dc9),LL(0x4f6c7e4b,0x04371819),LL(0x3736bcad,0xc284e00a),LL(0x21ae8f8d,0x0d881118), LL(0xf48c8e33,0xf9cf0f82),LL(0xa1bf40db,0xa11fd075),LL(0xdc2733e5,0xdceab0de),LL(0x8e986bd7,0xc560a8b5), + LL(0x3929d097,0x48dd1fe2),LL(0x92f188f1,0x3885b290),LL(0xda6fcdac,0x0f2ae613),LL(0xb662a46c,0x9054303e), LL(0x0738042a,0xb6871e44),LL(0xbdaf6449,0x98e6a977),LL(0xd1c9df1b,0xd8bc0650),LL(0x36e098f9,0xef3d6451), + LL(0xb6d72d28,0x03fbae82),LL(0xf5d84080,0x77ca9db1),LL(0xa58efc1c,0x8a112cff),LL(0xc564cb4a,0x518d761c), LL(0xf0d1b5ce,0x69b5740e),LL(0xe9eb1785,0x717039cc),LL(0x22f53382,0x3fe29f90),LL(0x6bc7c95c,0x8e54ba56), + LL(0xf7f91d0f,0x9c806d8a),LL(0xa82a5728,0x3b61b0f1),LL(0x94d76754,0x4640032d),LL(0x47d834c6,0x273eb5de), LL(0x7b4e4d53,0x2988abf7),LL(0xde401777,0xb7ce66bf),LL(0x715071b3,0x9fba6b32),LL(0xad3a1a98,0x82413c24), + LL(0xe0e8ad93,0x5b7fc8c4),LL(0x5fab868d,0xb5679aee),LL(0x2b3946f3,0xb1f9d2fa),LL(0x5685b50a,0x458897dc), LL(0x89d0caf3,0x1e98c930),LL(0x78642e92,0x39564c5f),LL(0x0dbdaf18,0x1b77729a),LL(0x579e82e6,0xf9170722), + LL(0xe4515fa5,0x680c0317),LL(0xfb0c790f,0xf85cff84),LL(0x6d2e0765,0xc7a82aab),LL(0x35c82b32,0x7446bca9), LL(0x6d63184f,0x5de607aa),LL(0x262803a6,0x7c1a46a8),LL(0xaebe8035,0xd218313d),LL(0xc73c51f8,0x92113ffd), + LL(0x12e7e46c,0x4b38e083),LL(0x56126bd5,0x69d0a37a),LL(0x73c07e04,0xfb3f324b),LL(0x8fda7267,0xa0c22f67), LL(0x4d2c7d8f,0x8f2c0051),LL(0xcbe2cae5,0xbc45ced3),LL(0xa8f0f277,0xe1c6cf07),LL(0x1eb99a98,0xbc392312), + LL(0x3cc8ac85,0x75537b7e),LL(0xdd02753b,0x8d725f57),LL(0xb737df2f,0xfd05ff64),LL(0xf6d2531d,0x55fe8712), LL(0x6ab6b01c,0x57ce04a9),LL(0x7cd93724,0x69a02a89),LL(0xcf86699b,0x4f82ac35),LL(0x9cb4b232,0x8242d3ad), + LL(0xd62105e5,0x713d0f65),LL(0x2d29be61,0xbb222bfa),LL(0x6cfbef09,0xf2f9a79e),LL(0xd5d6782f,0xfc24d8d3), LL(0xd4129967,0x5db77085),LL(0xdc3c2a43,0xdb81c3cc),LL(0x05d8d9a3,0x9d655fc0),LL(0x54298026,0x3f5d057a), + LL(0x88c54694,0x1157f56d),LL(0x9b09573e,0xb26baba5),LL(0x22adffd1,0x2cab03b0),LL(0xdd69f383,0x60a412c8), LL(0x54b25039,0xed76e98b),LL(0x687e714d,0xd4ee67d3),LL(0x7b00b594,0x87739648),LL(0xc9ef709b,0xce419775), + LL(0x1c203a40,0x40f76f85),LL(0xeafd8f91,0x30d352d6),LL(0x95578dd2,0xaf196d3d),LL(0x77cc3f3d,0xea4bb3d7), LL(0xb98e782b,0x42a5bd03),LL(0x0624920d,0xac958c40),LL(0xfc56fcc8,0xb838134c),LL(0x89572e5e,0x86ec4ccf), + LL(0x9be47be0,0x69c43526),LL(0xcb28fea1,0x323b7dd8),LL(0x3a6c67e5,0xfa5538ba),LL(0x1d378e46,0xef921d70), LL(0x3c4b880e,0xf92961fc),LL(0x98940a67,0x3f6f914e),LL(0xfef0ff39,0xa990eb0a),LL(0xf0eeff9c,0xa6c2920f), + LL(0x51b8d9a3,0xca804166),LL(0x0ffb0db1,0x42531bc9),LL(0xaa82e7ce,0x72ce4718),LL(0xdf574741,0x6e199913), LL(0xd5d36946,0xd5f1b13d),LL(0xf68f0194,0x8255dc65),LL(0x8710d230,0xdc9df4cd),LL(0x138c1988,0x3453c20f), + LL(0x89a6ef01,0x9af98dc0),LL(0x9857df85,0x4dbcc3f0),LL(0x5c1ad924,0x34805601),LL(0xd0493046,0x40448da5), LL(0x4ee343e2,0xf629926d),LL(0x90e8a301,0x6343f1bd),LL(0x40815b3f,0xefc93491),LL(0xde8f66fb,0xf882a423), + LL(0xe7db9f57,0x3a12d5f4),LL(0x3c384c27,0x7dfba38a),LL(0x6fc660b1,0x7a904bfd),LL(0x2773b21c,0xeb6c5db3), LL(0x1cdfe049,0xc350ee66),LL(0x44540f29,0x9baac0ce),LL(0xa5ec6aad,0xbc57b6ab),LL(0x0a7c1baa,0x167ce8c3), + LL(0x53fb2b56,0xb23a03a5),LL(0x4e057f78,0x6ce141e7),LL(0x89e490d9,0x796525c3),LL(0xa31a7e75,0x0bc95725), LL(0x1220fd06,0x1ec56791),LL(0x408b0bd6,0x716e3a3c),LL(0xe8ebeba9,0x31cd6bf7),LL(0xbee6b670,0xa7326ca6), + LL(0xcd090c43,0x3d9f851c),LL(0xf12c3988,0x561e8f13),LL(0x904b7be4,0x50490b6a),LL(0x0410737b,0x61690ce1), LL(0x0f009052,0x299e9a37),LL(0xf026092e,0x258758f0),LL(0xfdfcdc0f,0x9fa255f3),LL(0xc0e1bcd2,0xdbc9fb1f), + LL(0x24651840,0x35f9dd6e),LL(0xa5c59abc,0xdca45a84),LL(0xecca4938,0x103d396f),LL(0xb97b3f29,0x4532da0a), LL(0x1999a6bf,0xc4135ea5),LL(0x5e6bf2ee,0x3aa9505a),LL(0x3f5be093,0xf77cef06),LL(0xa943152e,0x97d1a0f8), + LL(0x2e1c21dd,0x2cb0ebba),LL(0x2c6797c4,0xf41b29fc),LL(0xb300101f,0xc6e17321),LL(0xd0d79a89,0x4422b0e9), LL(0x92f1bfc4,0x49e4901c),LL(0xe1e10ed9,0x06ab1f8f),LL(0xdb2926b8,0x84d35577),LL(0x356e8ec2,0xca349d39), + LL(0x343bf1a9,0x70b63d32),LL(0x37d1a6b1,0x8fd3bd28),LL(0x316865b4,0x0454879c),LL(0xc458efa2,0xee959ff6), LL(0x9706dc3f,0x0461dcf8),LL(0x164e4b2e,0x737db0e2),LL(0x2f8843c8,0x09262680),LL(0x7745e6f6,0x54498bbc), + LL(0xa29e24af,0x359473fa),LL(0x70aa87a1,0xfcc3c454),LL(0x00573ace,0xfd2c4bf5),LL(0x28dd1965,0xb65b514e), LL(0x2193e393,0xe46ae7cf),LL(0xf5444d97,0x60e9a4e1),LL(0x00ff38ed,0xe7594e96),LL(0x0a0e0f02,0x43d84d2f), + LL(0xee398a21,0x8b6db141),LL(0xe3bcc5be,0xb88a56ae),LL(0x373460ea,0x0a1aa52f),LL(0x160bb19b,0x20da1a56), LL(0x65bf0384,0xfb54999d),LL(0x5d5a180e,0x71a14d24),LL(0x21737b04,0xbc44db7b),LL(0x01dd8e92,0xd84fcb18), + LL(0xfa44b479,0x80de937b),LL(0x5c98fd4f,0x53505499),LL(0x28f08727,0x1edb12ab),LL(0xa5f3ef53,0x4c58b582), LL(0x8327f246,0xbfb236d8),LL(0x4d7df320,0xc3a3bfaa),LL(0xb96024f2,0xecd96c59),LL(0x7f4e0433,0xfc293a53), + LL(0x5acf6e10,0x5341352b),LL(0xafe652c3,0xc50343fd),LL(0x18577a7f,0x4af3792d),LL(0xaf16823d,0xe1a4c617), LL(0x33425d0a,0x9b26d0cd),LL(0x9b7bc47f,0x306399ed),LL(0x706bb20b,0x2a792f33),LL(0x98111055,0x31219614), + LL(0x87f5d28b,0x864ec064),LL(0x962277fd,0x11392d91),LL(0xbb6aed5f,0xb5aa7942),LL(0x47e799d9,0x080094dc), LL(0x208ba19b,0x4afa588c),LL(0x8512f284,0xd3e7570f),LL(0x02f5799a,0xcbae64e6),LL(0x514b9492,0xdeebe7ef), + LL(0xe5c298ff,0x30300f98),LL(0x3678361f,0x17f561be),LL(0x98cb9a16,0xf52ff312),LL(0x5562d490,0x6233c3bc), LL(0x92e3a2cb,0x7bfa15a1),LL(0xe6365119,0x961bcfd1),LL(0x2c8c53b1,0x3bdd29bf),LL(0x822844ba,0x739704df), + LL(0x7e7b754b,0x7dacfb58),LL(0xa806c9b9,0x23360791),LL(0x23504452,0xe7eb88c9),LL(0x852c1783,0x2983e996), LL(0x958d881d,0xdd4ae529),LL(0x262c7b3c,0x026bae03),LL(0x960b52d1,0x3a6f9193),LL(0x92696cfb,0xd0980f90), + LL(0xd5f30851,0x4c1f428c),LL(0x2a4f6630,0x94dfed27),LL(0xfc5d48a4,0x4df53772),LL(0x933260ce,0xdd2d5a2f), LL(0xd44cc7a5,0x574115bd),LL(0xbd12533a,0x4ba6b20d),LL(0x243057c9,0x30e93cb8),LL(0x14de320e,0x794c486a), + LL(0xf21496e4,0xe925d4ce),LL(0xec696331,0xf951d198),LL(0x3e8d812f,0x9810e2de),LL(0x389294ab,0xd0a47259), LL(0x0e3bab66,0x513ba2b5),LL(0xabad306f,0x462caff5),LL(0xaf04c49e,0xe2dc6d59),LL(0xe0b84b0b,0x1aeb8750), + LL(0x2f7d0ca2,0xc034f12f),LL(0xe06acf2f,0x6d2e8128),LL(0x21facc2f,0x801f4f83),LL(0xf40ef607,0xa1170c03), LL(0x7805a99c,0xfe0a1d4f),LL(0xcc26aba5,0xbde56a36),LL(0x35531f40,0x5b1629d0),LL(0x9afa6108,0xac212c2b), + LL(0x15697be5,0x30a06bf3),LL(0x2c63c7c1,0x6f0545dc),LL(0x7ccdadaf,0x5d8cb842),LL(0xac7015bb,0xd52e379b), LL(0xf462c23e,0xc4f56147),LL(0x46bc24b0,0xd44a4298),LL(0xe2856d4f,0xbc73d23a),LL(0x0832bcdf,0x61cedd8c), + LL(0x99f241d7,0x60953556),LL(0x001a349d,0xee4adbd7),LL(0xaa89e491,0x0b35bf6a),LL(0x136f7546,0x7f0076f4), LL(0x9264da3d,0xd19a18ba),LL(0x62a7a28b,0x6eb2d2cd),LL(0x8761c971,0xcdba941f),LL(0xa3be4a5d,0x1550518b), + LL(0x57d0b70c,0xd0e8e2f0),LL(0xcd133ba3,0xeea8612e),LL(0x44416aec,0x814670f0),LL(0x30775061,0x424db6c3), LL(0x16213fd1,0xd96039d1),LL(0x18a3478f,0xc61e7fa5),LL(0xcb0c5021,0xa805bdcc),LL(0x0cc616dd,0xbdd6f3a8), + LL(0x5d97f7e2,0x06009667),LL(0xaf0bf4b6,0x31db0fc1),LL(0x5491627a,0x23680ed4),LL(0x7d741fb1,0xb99a3c66), LL(0x36b1ff92,0xe9bb5f55),LL(0x512b388d,0x29738577),LL(0x50fcf263,0xdb8a2ce7),LL(0x6c4f7b47,0x385346d4), + LL(0x31631f9e,0xbe86c5ef),LL(0x03a57a29,0xbf91da21),LL(0x7b23f821,0xc3b1f796),LL(0x770db354,0x0f7d00d2), LL(0xd8fe79da,0x8ffc6c3b),LL(0xd525c996,0xcc5e8c40),LL(0xcfff632a,0x4640991d),LL(0x67112528,0x64d97e8c), + LL(0x02f1cd1e,0xc232d973),LL(0x1dd212a4,0xce87eacb),LL(0xe69802f7,0x6e4c8c73),LL(0x1fffddbd,0x12ef0290), LL(0x1bcea6e2,0x941ec74e),LL(0x3cb92cbb,0xd0b54024),LL(0x7e8f9d05,0x809fb9d4),LL(0xf2992aae,0x3bf16159), + LL(0xf8a7a838,0xad40f279),LL(0x05615660,0x11aea631),LL(0xa01f6fa1,0xbf52e6f1),LL(0x3dc2aec9,0xef046995), LL(0xd8080711,0x785dbec9),LL(0x9fdedf76,0xe1aec60a),LL(0xfa21c126,0xece797b5),LL(0x05e52732,0xc66e898f), + LL(0x08811fdb,0x39bb69c4),LL(0x2fc7f082,0x8bfe1ef8),LL(0x174f4138,0xc8e7a393),LL(0xd58d1f98,0xfba8ad1d), LL(0xbfd2fd5b,0xbc21d0ce),LL(0x6ee60d61,0x0b839a82),LL(0xafd22253,0xaacf7658),LL(0xaae396b3,0xb526bed8), + LL(0x38564464,0xccc1bbc2),LL(0x8c45bc73,0x9e3ff947),LL(0x58188a78,0xcde9bca3),LL(0xd73bf8f7,0x138b8ee0), LL(0x4123c489,0x5c7e234c),LL(0xfa643297,0x66e69368),LL(0x39a15fa3,0x0629eeee),LL(0xa9e2a927,0x95fab881), + LL(0xeafbb1e1,0xb2497007),LL(0xe75b7a93,0xd75c9ce6),LL(0xefb68d78,0x3558352d),LL(0x223f6396,0xa2f26699), LL(0xe469b17a,0xeb911ecf),LL(0xe72d3ec2,0x62545779),LL(0x82cb113f,0x8ea47de7),LL(0x4e1fa98d,0xebe4b086), + LL(0x8cdfedb1,0xec2d5ed7),LL(0xfe211a74,0xa535c077),LL(0x11d244c5,0x9678109b),LL(0xbe299a76,0xf17c8bfb), LL(0xfb11fbc4,0xb651412e),LL(0x94ab3f65,0xea0b5482),LL(0x0cf78243,0xd8dffd95),LL(0xce0361d4,0x2e719e57), + LL(0x304ddc5b,0x9007f085),LL(0x4daba2ea,0x095e8c6d),LL(0x3f9d28a9,0x5a33cdb4),LL(0xe2283003,0x85b95cd8), LL(0xb9744733,0xbcd6c819),LL(0xfc7f5783,0x29c5f538),LL(0xd59038e4,0x6c49b2fa),LL(0x3bbe1018,0x68349cc1), + LL(0x21830ee5,0xcc490c1d),LL(0xe9bfa297,0x36f9c4ee),LL(0x48de1a94,0x58fd7294),LL(0x4e8f2cdc,0xaadb13a8), LL(0x81313dba,0x515eaaa0),LL(0xc2152dd8,0xc76bb468),LL(0xa653dbf8,0x357f8d75),LL(0xb14ac143,0xe4d8c4d1), + LL(0xb055cb40,0xbdb8e675),LL(0x977b5167,0x898f8e7b),LL(0xb82fb863,0xecc65651),LL(0x6d88f01f,0x56544814), LL(0x263a75a9,0xb0928e95),LL(0x1a22fcda,0xcfb6836f),LL(0x3f3bd37c,0x651d14db),LL(0xb6ad4664,0x1d3837fb), + LL(0xff4f94ab,0x7c5fb538),LL(0x6d7fb8f2,0x7243c712),LL(0xa85c5287,0xef13d60c),LL(0x4bb8dd1b,0x18cfb7c7), LL(0x72908219,0x82f9bfe6),LL(0x9d5144ab,0x35c4592b),LL(0x9cf4b42f,0x52734f37),LL(0x8c60ddc4,0x6bac55e7), + LL(0x94dea0f6,0xb5cd811e),LL(0xe18cc1a3,0x259ecae4),LL(0x15e660f8,0x6a0e836e),LL(0x0e02bff2,0x6c639ea6), LL(0x7e1026fd,0x8721b8cb),LL(0x63261942,0x9e73b50b),LL(0x77f01da3,0xb8c70974),LL(0x8268f57f,0x1839e6a6), + LL(0x5150b805,0x571b9415),LL(0xf92c7097,0x1892389e),LL(0x4a084b95,0x8d69c18e),LL(0xbe5b495c,0x7014c512), LL(0x1b07523c,0x4780db36),LL(0x2c1c64fa,0x2f6219ce),LL(0x602c105a,0xc38b81b0),LL(0x5dc8e360,0xab4f4f20), + LL(0xcf7d62d2,0x20d3c982),LL(0x23ba8150,0x1f36e29d),LL(0x92763f9e,0x48ae0bf0),LL(0x1d3a7007,0x7a527e6b), LL(0x581a85e3,0xb4a89097),LL(0xdc158be5,0x1f1a520f),LL(0x167d726e,0xf98db37d),LL(0x1113e862,0x8802786e), +}, +/* digit=15 base_pwr=2^105 */ +{ + LL(0x36f09ab0,0xefb2149e),LL(0x4a10bb5b,0x03f163ca),LL(0x06e20998,0xd0297045),LL(0x1b5a3bab,0x56f0af00), LL(0x70880e0d,0x7af4cfec),LL(0xbe3d913f,0x7332a66f),LL(0x7eceb4bd,0x32e6c84a),LL(0x9c228f55,0xedc4a79a), + LL(0xc55c4496,0xc37c7dd0),LL(0x25bbabd2,0xa6a96357),LL(0xadd7f363,0x5b7e63f2),LL(0x2e73f1df,0x9dce3782), LL(0xb2b91f71,0xe1e5a16a),LL(0x5ba0163c,0xe4489823),LL(0xf6e515ad,0xf2759c32),LL(0x8615eecf,0xa5e2f1f8), + LL(0xabded551,0x74519be7),LL(0xc8b74410,0x03d358b8),LL(0x0e10d9a9,0x4d00b10b),LL(0x28da52b7,0x6392b0b1), LL(0x0b75c904,0x6744a298),LL(0xa8f7f96c,0xc305b0ae),LL(0x182cf932,0x042e421d),LL(0x9e4636ca,0xf6fc5d50), + LL(0xd64cc78c,0x795847c9),LL(0x9b6cb27b,0x6c50621b),LL(0xdf8022ab,0x07099bf8),LL(0xc04eda1d,0x48f862eb), LL(0xe1603c16,0xd12732ed),LL(0x5c9a9450,0x19a80e0f),LL(0xb429b4fc,0xe2257f54),LL(0x45460515,0x66d3b2c6), + LL(0x822e37be,0x6ca4f87e),LL(0x253bda4e,0x73f237b4),LL(0x41190aeb,0xf747f3a2),LL(0x804cf284,0xf06fa36f), LL(0xfc621c12,0x0a6bbb6e),LL(0x40b80ec6,0x5d624b64),LL(0x7ba556f3,0x4b072425),LL(0x3e2d20a8,0x7fa0c354), + LL(0xe3229d41,0xe921fa31),LL(0x94531bd4,0xa929c652),LL(0xa6d38209,0x84156027),LL(0x6bdb97bd,0xf3d69f73), LL(0x16833631,0x8906d19a),LL(0x03d51be3,0x68a34c2e),LL(0x0e511cd8,0xcb59583b),LL(0xfdc132a8,0x99ce6bfd), + LL(0xffcdb463,0x3facdaaa),LL(0x34a38b08,0x658bbc1a),LL(0xf1a9078d,0x12a801f8),LL(0x6ab855de,0x1567bcf9), LL(0x3572359b,0xe08498e0),LL(0x8659e68b,0xcf0353e5),LL(0x7d23807c,0xbb86e9c8),LL(0x2198e8a2,0xbc08728d), + LL(0x453cadd6,0x8de2b7bc),LL(0xbc0bc1f8,0x203900a7),LL(0xa6abd3af,0xbcd86e47),LL(0x8502effb,0x911cac12), LL(0xec965469,0x2d550242),LL(0x29e0017e,0x0e9f7692),LL(0x65979885,0x633f078f),LL(0x4cf751ef,0xfb87d449), + LL(0xfc25419a,0xe1790e4b),LL(0x4bff3cfd,0x36467203),LL(0x25b6e83f,0xc8db6386),LL(0x6cad6fd2,0x6cc69f23), LL(0x6bc68bb9,0x0219e45a),LL(0x297f7334,0xe43d79b6),LL(0x465dc97c,0x7d445368),LL(0x2a0b949a,0x4b9eea32), + LL(0x6102d021,0x1b96c6ba),LL(0x2f4461ea,0xeaafac78),LL(0xc49f19a8,0xd4b85c41),LL(0xcf538875,0x275c28e4), LL(0xdd2e54e0,0x35451a9d),LL(0x0605618b,0x6991adb5),LL(0x7b36cd24,0x5b8b4bcd),LL(0x56f37216,0x372a4f8c), + LL(0xa6a5da60,0xc890bd73),LL(0xdc4c9ff0,0x6f083da0),LL(0xf0536e57,0xf4e14d94),LL(0xaaec8243,0xf9ee1eda), LL(0x8bdcf8e7,0x571241ec),LL(0x0b041e26,0xa5db8271),LL(0xe3fff040,0x9a0b9a99),LL(0x7c271202,0xcaaf21dd), + LL(0x4f0dd2e8,0xb4e2b2e1),LL(0x0a377ac7,0xe77e7c4f),LL(0x0d7a2198,0x69202c3f),LL(0x28200eb8,0xf759b7ff), LL(0xdcfe314e,0xc87526ed),LL(0x53d5cf99,0xeb84c524),LL(0x515138b6,0xb1b52ace),LL(0x23fca3f4,0x5aa7ff8c), + LL(0xb9791a26,0xff0b13c3),LL(0xcdd58b16,0x960022da),LL(0x57aad2de,0xdbd55c92),LL(0xf30fe619,0x3baaaaa3), LL(0x0d881efd,0x9a4b2346),LL(0x46325e2a,0x506416c0),LL(0x035c18d4,0x91381e76),LL(0xf27817b0,0xb3bb68be), + LL(0x5116f937,0x15bfb8bf),LL(0xc1268943,0x7c64a586),LL(0x8419a2c8,0x71e25cc3),LL(0x8335f463,0x9fd6b0c4), LL(0xe8ee0e0e,0x4bf0ba3c),LL(0x298c21fa,0x6f6fba60),LL(0xae66bee0,0x57d57b39),LL(0x22672544,0x292d5130), + LL(0xbab093b3,0xf451105d),LL(0x02839986,0x012f59b9),LL(0x3474a89c,0x8a915802),LL(0x2de03e97,0x048c919c), LL(0x91071cd5,0xc476a2b5),LL(0x034970a5,0x791ed89a),LL(0xe1b7994b,0x89bd9042),LL(0xa1057ffd,0x8eaf5179), + LL(0xd551ee10,0x6066e2a2),LL(0x727e09a6,0x87a8f1d8),LL(0x2c01148d,0x00d08bab),LL(0x424f33fe,0x6da8e4f1), LL(0xcf9a4e71,0x466d17f0),LL(0x3bf5cb19,0xff502010),LL(0xd062ecc0,0xdccf97d8),LL(0x81d80ac4,0x80c0d9af), + LL(0x033f2876,0xe87771d8),LL(0x7d5cc3db,0xb0186ec6),LL(0x3bc9bc1d,0x58e8bb80),LL(0x6f6ef60e,0x4d1395cc), LL(0x186244a0,0xa73c62d6),LL(0x110a5b53,0x918e5f23),LL(0x741b7eab,0xed4878ca),LL(0xdbe03e51,0x3038d71a), + LL(0xa93c3246,0x840204b7),LL(0xa0b9b4cd,0x21ab6069),LL(0xb1d64218,0xf5fa6e2b),LL(0xf3d56191,0x1de6ad0e), LL(0xff1929c7,0x570aaa88),LL(0x640e87b5,0xc6df4c6b),LL(0xc65f0ccc,0xde8a74f2),LL(0xe6f6cc01,0x8b972fd5), + LL(0x0b846531,0x3fff36b6),LL(0x10a5e475,0xba7e45e6),LL(0x4145b6c5,0x84a1d10e),LL(0x5e046d9d,0xf1f7f91a), LL(0x44de90d7,0x0317a692),LL(0xf199c15e,0x951a1d4a),LL(0xc9d73deb,0x91f78046),LL(0xfab8224f,0x74c82828), + LL(0xe7560b90,0xaa6778fc),LL(0xa7e824ce,0xb4073e61),LL(0xd642eba8,0xff0d693c),LL(0x5dccef38,0x7ce2e57a), LL(0x1df1ad46,0x89c2c789),LL(0x098346fd,0x83a06922),LL(0xda2fc177,0x2d715d72),LL(0x85b6cf1d,0x7b6dd71d), + LL(0x73fa9cb0,0xc60a6d0a),LL(0x328bf5a9,0xedd3992e),LL(0x832c8c82,0xc380ddd0),LL(0xa2a0bf50,0xd182d410), LL(0xd9a528db,0x7d9d7438),LL(0xcaf53994,0xe8b1a0e9),LL(0x0e19987c,0xddd6e5fe),LL(0x190b059d,0xacb8df03), + LL(0x8300129f,0x53703a32),LL(0x68c43bfd,0x1f637662),LL(0x00e54051,0xbcbd1913),LL(0x7bf5a8c5,0x812fcc62), LL(0x29fb85da,0x3f969d5f),LL(0x694759e8,0x72f4e00a),LL(0x790726b7,0x426b6e52),LL(0x3bdbb209,0x617bbc87), + LL(0x97aee317,0x511f8bb9),LL(0xe81536a8,0x812a4096),LL(0x3ac09b9b,0x137dfe59),LL(0xba8c9a7a,0x0682238f), LL(0xaeccb4bd,0x7072ead6),LL(0x692ba633,0x6a34e9aa),LL(0x6fff9d33,0xc82eaec2),LL(0x1d4d2b62,0xfb753512), + LL(0x1d7aadab,0x1a0445ff),LL(0xd5f6a67c,0x65d38260),LL(0x91cfb26f,0x6e62fb08),LL(0x5c7d91d6,0xef1e0fa5), LL(0x33db72cd,0x47e7c7ba),LL(0xfa7c74b2,0x017cbc09),LL(0xf50a503c,0x3c931590),LL(0x616baa42,0xcac54f60), + LL(0xb2369f0f,0x9b6cd380),LL(0x23c76151,0x97d3a70d),LL(0x9862a9c6,0x5f9dd6fc),LL(0x12312f51,0x044c4ab2), LL(0x834a2ddc,0x035ea0fd),LL(0xcc7b826d,0x49e6b862),LL(0x62fce490,0xb03d6883),LL(0xb37e36e9,0x62f2497a), + LL(0xc6458293,0x04b005b6),LL(0xe8d10af7,0x36bb5276),LL(0x8ee617b8,0xacf2dc13),LL(0xb004b3d4,0x470d2d35), LL(0xfeeb1b77,0x06790832),LL(0x85657f9c,0x2bb75c39),LL(0xc0f60004,0xd70bd4ed),LL(0x219b018b,0xfe797ecc), + LL(0x753aebcc,0x9b5bec2a),LL(0xc939eca5,0xdaf9f3dc),LL(0xd095ad09,0xd6bc6833),LL(0xdaa4d2fc,0x98abdd51), LL(0x8d168be5,0xd9840a31),LL(0x2325a23c,0xcf7c10e0),LL(0x7e6ecfaf,0xa5c02aa0),LL(0xb5bfdf18,0x2462e7e6), + LL(0xa0cc3f12,0xab2d8a8b),LL(0xbc672a29,0x68dd485d),LL(0x596f2cd3,0x72039752),LL(0xa0cf3d8d,0x5d3eea67), LL(0xe6602671,0x810a1a81),LL(0x14026c0c,0x8f144a40),LL(0x76b50f85,0xbc753a6d),LL(0x645cd4a4,0xc4dc21e8), + LL(0x521d0378,0xc5262dea),LL(0x05011c6f,0x802b8e0e),LL(0x0b4c19ea,0x1ba19cbb),LL(0xebf0aaec,0x21db64b5), LL(0x70342f9d,0x1f394ee9),LL(0x1bc44a14,0x93a10aee),LL(0x3efd0baa,0xa7eed31b),LL(0x1d154e65,0x6e7c824e), + LL(0x9966e7ee,0xee23fa81),LL(0x05b7920d,0x64ec4aa8),LL(0x2d90aad4,0x2d44462d),LL(0xdf277ad5,0xf44dd195), LL(0xbb46b6a1,0x8d6471f1),LL(0xfd885090,0x1e65d313),LL(0x13a977b4,0x33a800f5),LL(0x0797e1ef,0xaca9d721), + LL(0xfcff6a17,0x9a5a85a0),LL(0x1eca7cee,0x9970a3f3),LL(0xc9504be3,0xbb9f0d6b),LL(0xadd24ee2,0xe0c504be), LL(0x77fcc2f4,0x7e09d956),LL(0x65bb5fc4,0xef1a5227),LL(0x8b9286aa,0x145d4fb1),LL(0x6649028b,0x66fd0c5d), + LL(0x1bf4581c,0x98857ceb),LL(0xaca7b166,0xe635e186),LL(0x659722ac,0x278ddd22),LL(0x1db68007,0xa0903c4c), LL(0x48f21402,0x366e4589),LL(0xb96abda2,0x31b49c14),LL(0xe0403190,0x329c4b09),LL(0xd29f43fe,0x97197ca3), + LL(0x274983d8,0x8073dd1e),LL(0x55717c8f,0xda1a3bde),LL(0x0361f9d1,0xfd3d4da2),LL(0x4c7de1ce,0x1332d081), LL(0xaa6d0e10,0x9b7ef7a3),LL(0xf54f1c4a,0x17db2e73),LL(0x4cd35567,0xaf3dffae),LL(0xe56f4e71,0xaaa2f406), + LL(0x7ace3fc7,0x8966759e),LL(0x45a8d8c6,0x9594eacf),LL(0x91834e0e,0x8de3bd8b),LL(0x548c0421,0xafe4ca53), LL(0xe6ee81c6,0xfdd7e856),LL(0x6b891a3a,0x8f671beb),LL(0xfae63829,0xf7a58f2b),LL(0x9c11ac9f,0x9ab186fb), + LL(0x10b5be76,0x8d6eb369),LL(0xfb040bcd,0x046b7739),LL(0xcb73de88,0xccb4529f),LL(0xcf26be03,0x1df0fefc), LL(0xbcfcd027,0xad7757a6),LL(0xbb3165ca,0xa8786c75),LL(0x7e99a4d9,0xe9db1e34),LL(0xb06c504b,0x99ee86df), + LL(0xc15c9f0a,0x5b7c2ddd),LL(0x4295989e,0xdf87a734),LL(0x03d08fda,0x59ece47c),LL(0xad5fc702,0xb074d3dd), LL(0x51a03776,0x20407903),LL(0x2a608007,0x2bb1f77b),LL(0xe1153185,0x25c58f4f),LL(0x766e6447,0xe6df62f6), + LL(0xed51275a,0xefb3d1be),LL(0x2f0f483f,0x5de47dc7),LL(0x97c2bedf,0x7932d98e),LL(0x0219f8a1,0xd5c11927), LL(0xa73a294e,0x9d751200),LL(0x9dc20172,0x5f88434a),LL(0xa26f506a,0xd28d9fd3),LL(0x9d1dcd48,0xa890cd31), + LL(0x70f4d3b4,0x0aebaec1),LL(0x0ffc8d00,0xfd1a1369),LL(0x57d57838,0xb9d9c240),LL(0x68bac361,0x45929d26), LL(0x25b15ca6,0x5a2cd060),LL(0x6e474446,0x4b3c83e1),LL(0xee1e5134,0x1aac7578),LL(0xc91e2f41,0xa418f5d6), + LL(0x213ed68b,0x6936fc8a),LL(0x510a5224,0x860ae7ed),LL(0xdef09b53,0x63660335),LL(0xcd79c98d,0x641b2897), LL(0x01110f35,0x29bd38e1),LL(0x648b1937,0x79c26f42),LL(0x9d9164f4,0x64dae519),LL(0x0265c273,0xd85a2310), + LL(0x4b07e2b1,0x7173dd5d),LL(0x8d9ea221,0xd144c4cb),LL(0x1105ab14,0xe8b04ea4),LL(0xfe80d8f1,0x92dda542), LL(0xcf03dce6,0xe9982fa8),LL(0x1a22cffc,0x8b5ea965),LL(0x3fad88c4,0xf7f4ea7f),LL(0x6a5ba95c,0x62db773e), + LL(0x93f24567,0xd20f02fb),LL(0x315257ca,0xfd46c69a),LL(0x8bcab987,0x0ac74cc7),LL(0x5ceca2f5,0x46f31c01), LL(0x888b219e,0x40aedb59),LL(0xe1fccd02,0xe50ecc37),LL(0x911f816c,0x1bcd9dad),LL(0x8db9b00c,0x583cc1ec), + LL(0xa483bf11,0xf3cd2e66),LL(0xb1b2c169,0xfa08a6f5),LL(0x4be9fa28,0xf375e245),LL(0x5b6d011f,0x99a7ffec), LL(0xc4ae62da,0x6a3ebddb),LL(0x374aef5d,0x6cea00ae),LL(0x9d4d05bc,0xab5fb98d),LL(0xd560f252,0x7cba1423), + LL(0x208490de,0x49b2cc21),LL(0xbcfb2879,0x1ca66ec3),LL(0x1b6fb16f,0x7f1166b7),LL(0x65fe5db3,0xfff63e08), LL(0x8b2610be,0xb8345abe),LL(0x39de3df4,0xb732ed80),LL(0x211c32b4,0x0e24ed50),LL(0x848ff27d,0xd10d8a69), + LL(0xed4de248,0xc1074398),LL(0x10488927,0xd7cedace),LL(0x85673e13,0xa4aa6bf8),LL(0x6daf30af,0xb46bae91), LL(0xfcef7ad8,0x07088472),LL(0xd4b35e97,0x61151608),LL(0xdde29986,0xbcfe8f26),LL(0xd5a34c79,0xeb84c4c7), + LL(0x164e1214,0xc1eec55c),LL(0xa147bb03,0x891be86d),LL(0x0ba96835,0x9fab4d10),LL(0xa5c1ae9f,0xbf01e9b8), LL(0xb186ebc0,0x6b4de139),LL(0x85b91bca,0xd5c74c26),LL(0xc2d93854,0x5086a99c),LL(0xa7a9dfbc,0xeed62a7b), + LL(0x76b7618a,0x8778ed6f),LL(0x03b66062,0xbff750a5),LL(0xb65186db,0x4cb7be22),LL(0xcc3a6d13,0x369dfbf0), LL(0x7191a321,0xc7dab26c),LL(0x40ed718e,0x9edac3f9),LL(0xd0cfd183,0xbc142b36),LL(0x7c991693,0xc8af82f6), + LL(0x97ce0b2a,0xb3d1e4d8),LL(0xc3a55cdf,0xe6d7c87f),LL(0x68b81afe,0x35846b95),LL(0xd3c239d8,0x018d12af), LL(0x01206e15,0x2b2c6208),LL(0xa3b882c6,0xe0e42453),LL(0xa50162d5,0x854470a3),LL(0x7017a62a,0x08157478), + LL(0x820357c7,0x18bd3fb4),LL(0x6f1458ad,0x992039ae),LL(0x25b44aa1,0x9a1df3c5),LL(0xed3d5281,0x2d780357), LL(0xc77ad4d4,0x58cf7e4d),LL(0xf9df4fc4,0xd49a7998),LL(0x1d71205e,0x4465a8b5),LL(0x649254aa,0xa0ee0ea6), + LL(0xab7bd771,0x4b5eeecf),LL(0x35c262b9,0x6c873073),LL(0x3c9d61e7,0xdc5bd648),LL(0x321460d2,0x233d6d54), LL(0xfc195bcc,0xd20c5626),LL(0x04d78b63,0x25445958),LL(0x17ec8ef3,0xe03fcb3d),LL(0x46b8f781,0x54b690d1), + LL(0x21230646,0x82fa2c8a),LL(0x084f418c,0xf51aabb9),LL(0x1a30ba43,0xff4fbec1),LL(0x743c9df7,0x6a5acf73), LL(0xd635b4d5,0x1da2b357),LL(0xecd5c1da,0xc3de68dd),LL(0xd61af0dd,0xa689080b),LL(0xd665bf99,0xdea5938a), + LL(0xfe637294,0x0231d71a),LL(0xa5a81cd8,0x01968aa6),LL(0x048e63b5,0x11252d50),LL(0x6ca007e9,0xc446bc52), LL(0x96d6134b,0xef8c50a6),LL(0x9e09a05c,0x9361fbf5),LL(0xdca3291a,0xf17f85a6),LL(0xff251a21,0xb178d548), + LL(0xa4df3915,0x87f6374b),LL(0x2fd5d608,0x566ce1bf),LL(0x7de35102,0x425cba4d),LL(0x58c5d5e2,0x6b745f8f), LL(0x63122edf,0x88402af6),LL(0x3b989a89,0x3190f9ed),LL(0xebba3156,0x4ad3d387),LL(0xc7c469a5,0xef385ad9), + LL(0x3f642c29,0xb08281de),LL(0x910ffb88,0x20be0888),LL(0xd5292546,0xf353dd4a),LL(0x8377a262,0x3f1627de), LL(0xeefcd638,0xa5faa013),LL(0x74cc77c3,0x8f3bf626),LL(0xa348f55e,0x32618f65),LL(0x9fefeb9e,0x5787c0dc), + LL(0xd9a23e44,0xf1673aa2),LL(0x4e10690d,0x88dfa993),LL(0x2bf91108,0x1ced1b36),LL(0x3af48649,0x9193ceca), LL(0x2d738fc5,0xfb34327d),LL(0x975fee6c,0x6697b037),LL(0xc04079a5,0x2f485da0),LL(0x2feaa1ac,0x2cdf5735), + LL(0xbd55659e,0x76944420),LL(0x4376090c,0x7973e32b),LL(0x163b591a,0x86bb4fe1),LL(0xc196f0ca,0x10441aed), LL(0x045ad915,0x3b431f4a),LL(0xa4afacb1,0x6c11b437),LL(0x71fdbbd8,0x30b0c7db),LL(0xeda65acd,0xb642931f), + LL(0x9c92b235,0x4baae6e8),LL(0x6b3993a1,0xa73bbd0e),LL(0x693dd031,0xd06d60ec),LL(0x7156881c,0x03cab91b), LL(0x1db3574b,0xd615862f),LL(0x64bb061a,0x485b0185),LL(0xa0181e06,0x27434988),LL(0xc1c0c757,0x2cd61ad4), + LL(0x2ff9f403,0x3effed5a),LL(0x62239029,0x8dc98d8b),LL(0x1f17b70d,0x2206021e),LL(0xbf510015,0xafbec0ca), LL(0x80130dfa,0x9fed7164),LL(0x8a02dcf5,0x306dc2b5),LL(0xfeb10fc0,0x48f06620),LL(0x5a57cf51,0x78d1e1d5), + LL(0x192ef710,0xadef8c5a),LL(0x3b7431f9,0x88afbd4b),LL(0x64250c9e,0x7e1f7407),LL(0xb58bec07,0x6e31318d), LL(0x24f89b4e,0xfd4fc4b8),LL(0x48c36a2a,0x65a5dd88),LL(0xf024baa7,0x4f1eccff),LL(0xcba94650,0x22a21cf2), + LL(0x42a554f7,0x95d29dee),LL(0x002ec4ba,0x828983a5),LL(0x8badb73d,0x8112a1f7),LL(0xa27c1839,0x79ea8897), LL(0xd065fd83,0x8969a5a7),LL(0xb262a0bc,0xf49af791),LL(0xaf2b5127,0xfcdea8b6),LL(0x564c2dbc,0x10e913e1), + LL(0xbc21ef51,0x51239d14),LL(0x4ce57292,0xe51c3ceb),LL(0x47bbcc3b,0x795ff068),LL(0xbd7e11e6,0x86b46e1e), LL(0x80041ef4,0x0ea6ba23),LL(0x6262342e,0xd72fe505),LL(0x31d294d4,0x8abc6dfd),LL(0x1278c2c9,0xbbe017a2), + LL(0xb389328a,0xb1fcfa09),LL(0xd01771b5,0x322fbc62),LL(0x60b045bf,0x04c0d063),LL(0x10e52d01,0xdb652edc), LL(0x03ec6627,0x50ef932c),LL(0xc1ee50e3,0xde1b3b2d),LL(0xdc37a90d,0x5ab7bdc5),LL(0x31e33a96,0xfea67213), + LL(0x4f2999aa,0x6482b5cb),LL(0xb8cbf0dd,0x38476cc6),LL(0x173405bb,0x93ebfacb),LL(0xe52369ec,0x15cdafe7), LL(0xd935b7db,0xd42d5ba4),LL(0x1c99a4cd,0x648b6004),LL(0xa3b5545b,0x785101bd),LL(0x9dd67faf,0x4bf2c38a), + LL(0x4442449c,0xb1aadc63),LL(0x33ad4fb8,0xe0e9921a),LL(0xaa686d82,0x5c552313),LL(0x465d866c,0xdee635fa), LL(0x18ee6e8a,0xbc3c224a),LL(0xed42e02f,0xeed748a6),LL(0xd474cd08,0xe70f930a),LL(0xfff24adf,0x774ea6ec), + LL(0xf3480d4a,0x03e2de1c),LL(0xbc8acf1a,0xf0d8edc7),LL(0x68295a9c,0xf23e3303),LL(0xc546a97d,0xfadd5f68), LL(0x96f8acb1,0x895597ad),LL(0x671bdae2,0xbddd49d5),LL(0x21dd43f4,0x16fcd528),LL(0x6619141a,0xa5a45412), +}, +/* digit=16 base_pwr=2^112 */ +{ + LL(0xc360e25a,0x8ce9b6bf),LL(0x075a1a78,0xe6425195),LL(0x481732f4,0x9dc756a8),LL(0x5432b57a,0x83c0440f), LL(0xd720281f,0xc670b3f1),LL(0xd135e051,0x2205910e),LL(0xdb052be7,0xded14b0e),LL(0xc568ea39,0x697b3d27), + LL(0xfb3ff9ed,0x2e599b9a),LL(0x17f6515c,0x28c2e0ab),LL(0x474da449,0x1cbee4fd),LL(0x4f364452,0x071279a4), LL(0x01fbe855,0x97abff66),LL(0x5fda51c4,0x3ee394e8),LL(0x67597c0b,0x190385f6),LL(0xa27ee34b,0x6e9fccc6), + LL(0x14092ebb,0x0b89de93),LL(0x428e240c,0xf17256bd),LL(0x93d2f064,0xcf89a7f3),LL(0xe1ed3b14,0x4f57841e), LL(0xe708d855,0x4ee14405),LL(0x03f1c3d0,0x856aae72),LL(0xbdd7eed5,0xc8e5424f),LL(0x73ab4270,0x3333e4ef), + LL(0xdda492f8,0x3bc77ade),LL(0x78297205,0xc11a3aea),LL(0x34931b4c,0x5e89a3e7),LL(0x9f5694bb,0x17512e2e), LL(0x177bf8b6,0x5dc349f3),LL(0x08c7ff3e,0x232ea4ba),LL(0xf511145d,0x9c4f9d16),LL(0x33b379c3,0xccf109a3), + LL(0xa1f25897,0xe75e7a88),LL(0xa1b5d4d8,0x7ac6961f),LL(0x08f3ed5c,0xe3e10773),LL(0x0a892dfb,0x208a54ec), LL(0x78660710,0xbe826e19),LL(0x237df2c8,0x0cf70a97),LL(0xed704da5,0x418a7340),LL(0x08ca33fd,0xa3eeb9a9), + LL(0x169bca96,0x49d96233),LL(0x2da6aafb,0x04d286d4),LL(0xa0c2fa94,0xc09606ec),LL(0x23ff0fb3,0x8869d0d5), LL(0xd0150d65,0xa99937e5),LL(0x240c14c9,0xa92e2503),LL(0x108e2d49,0x656bf945),LL(0xa2f59e2b,0x152a733a), + LL(0x8434a920,0xb4323d58),LL(0x622103c5,0xc0af8e93),LL(0x938dbf9a,0x667518ef),LL(0x83a9cdf2,0xa1843073), LL(0x5447ab80,0x350a94aa),LL(0xc75a3d61,0xe5e5a325),LL(0x68411a9e,0x74ba507f),LL(0x594f70c5,0x10581fc1), + LL(0x80eb24a9,0x60e28570),LL(0x488e0cfd,0x7bedfb4d),LL(0xc259cdb8,0x721ebbd7),LL(0xbc6390a9,0x0b0da855), LL(0xde314c70,0x2b4d04db),LL(0x6c32e846,0xcdbf1fbc),LL(0xb162fc9e,0x33833eab),LL(0xb0dd3ab7,0x9939b48b), + LL(0xcb0c9c8c,0x5aaa98a7),LL(0x81c4375c,0x75105f30),LL(0x5ef1c90f,0xceee5057),LL(0xc23a17bf,0xb31e065f), LL(0xd4b6d45a,0x5364d275),LL(0x62ec8996,0xd363f3ad),LL(0x4391c65b,0xb5d21239),LL(0xebb41b47,0x84564765), + LL(0x37107c78,0x20d18ecc),LL(0x570c2a66,0xacff3b6b),LL(0x9bd0d845,0x22f975d9),LL(0xba178fa0,0xef0a0c46), LL(0x76b6028e,0x1a419651),LL(0x248612d4,0xc49ec674),LL(0x7338af55,0x5b6ac4f2),LL(0x7bee5a36,0x06145e62), + LL(0xe75746b5,0x33e95d07),LL(0xc40c78be,0x1c1e1f6d),LL(0x222ff8e2,0x967833ef),LL(0xb49180ad,0x4bedcf6a), LL(0x3d7a4c8a,0x6b37e9c1),LL(0x6ddfe760,0x2748887c),LL(0xaa3a5bbc,0xf7055123),LL(0x7bbb8e74,0x954ff225), + LL(0x97c3dfb9,0xc42b8ab1),LL(0xcf168154,0x55a549b0),LL(0xc1b50692,0xad6748e7),LL(0x6fc5cbcb,0x2775780f), LL(0xe1c9d7c8,0x4eab80b8),LL(0x3fdbcd56,0x8c69dae1),LL(0x9969eace,0x47e6b4fb),LL(0xa705cb5a,0x002f1085), + LL(0x6d3fea55,0x4e23ca44),LL(0xf4810568,0xb4ae9c86),LL(0x2a62f27d,0x47bfb91b),LL(0xd9bac28c,0x60deb4c9), LL(0x7de6c34c,0xa892d894),LL(0x4494587d,0x4ee68259),LL(0x1a3f8a5b,0x914ee14e),LL(0x28700385,0xbb113eaa), + LL(0x2115b4c9,0x81ca03b9),LL(0x8908cad1,0x7c163d38),LL(0xaa18179a,0xc912a118),LL(0x886e3081,0xe09ed750), LL(0x26f516ca,0xa676e3fa),LL(0x8e732f91,0x753cacf7),LL(0x833da8b4,0x51592aea),LL(0x4cbea8aa,0xc626f42f), + LL(0xa7b56eaf,0xef9dc899),LL(0x34ef7316,0x00c0e52c),LL(0xfe818a86,0x5b1e4e24),LL(0xc538be47,0x9d31e20d), LL(0x3ed68974,0x22eb932d),LL(0x7c4e87c4,0xe44bbc08),LL(0x0dde9aef,0x4121086e),LL(0x134f4345,0x8e6b9cff), + LL(0x711b0eb9,0x96892c1f),LL(0x780ab954,0xb905f2c8),LL(0xa20792db,0xace26309),LL(0x0684e126,0xec8ac9b3), LL(0xb40a2447,0x486ad8b6),LL(0x9fe3fb24,0x60121fc1),LL(0x1a8e3b3f,0x5626fccf),LL(0x6ad1f394,0x4e568622), + LL(0x196aa5a1,0xda7aae0d),LL(0x1041b5fb,0xe0df8c77),LL(0x26b318b7,0x451465d9),LL(0x7ab136e9,0xc29b6e55), LL(0x71148463,0x2c2ab48b),LL(0x64454a76,0xb5738de3),LL(0x5a03abe4,0x54ccf9a0),LL(0x0427d58e,0x377c0296), + LL(0x2bb39c1f,0x73f5f0b9),LL(0xe608d8c5,0x14373f2c),LL(0x00fbb805,0xdcbfd314),LL(0x83afdcfb,0xdf18fb20), LL(0x42b3523f,0x81a57f42),LL(0x87f650fb,0xe958532d),LL(0x8b0a7d7c,0xaa8dc8b6),LL(0x150166be,0x1b75dfb7), + LL(0x2d7d1413,0x90e4f7c9),LL(0x9834f597,0x67e2d6b5),LL(0xa808c3e8,0x4fd4f4f9),LL(0xd5281ec1,0xaf8237e0), LL(0x84687cee,0x25ab5fdc),LL(0xa5b26c09,0xc5ded6b1),LL(0xc8ea7650,0x8e4a5aec),LL(0x14cc417f,0x23b73e5c), + LL(0x3037bf52,0x2bfb4318),LL(0x78c725d7,0xb61e6db5),LL(0xbbb3e5d7,0x8efd4060),LL(0xdbac488e,0x2e014701), LL(0x360aa449,0xac75cf9a),LL(0x79634d08,0xb70cfd05),LL(0xfffb15ef,0xa591536d),LL(0xd07c106c,0xb2c37582), + LL(0xf50225f9,0xb4293fdc),LL(0xb0e12b03,0xc52e175c),LL(0xd0a8bf64,0xf649c3ba),LL(0xeb8ae3c6,0x745a8fef), LL(0x58321bc3,0x30d7e5a3),LL(0x0bc4df48,0xb1732be7),LL(0xe9ea5058,0x1f217993),LL(0x3e4fd745,0xf7a71cde), + LL(0x894c5bbb,0x86cc533e),LL(0x69d83082,0x6915c7d9),LL(0x5815c244,0xa6aa2d05),LL(0x49b22ce5,0xaeeee592), LL(0x78135486,0x89e39d13),LL(0x16b76f2f,0x3a275c1f),LL(0xe036e8f5,0xdb6bcc1b),LL(0x5e4709f5,0x4df69b21), + LL(0x2d0f39aa,0xa188b250),LL(0x15a85947,0x622118bb),LL(0xfde0f4fa,0x2ebf520f),LL(0x4860e539,0xa40e9f29), LL(0x22b57f0f,0x7b6a51eb),LL(0x7e80644a,0x849a33b9),LL(0x1cf095fe,0x50e5d16f),LL(0xec55f002,0xd754b54e), + LL(0x236f4a98,0x5cfbbb22),LL(0x066800bb,0x0b0c59e9),LL(0x5a9a7774,0x4ac69a8f),LL(0xd6bec948,0x2b33f804), LL(0x32e6c466,0xb3729295),LL(0x4e599c73,0x68956d0f),LL(0x155c31cc,0xa47a249f),LL(0xe1ce284e,0x24d80f0d), + LL(0x988baf01,0xcd821dfb),LL(0xdbb16647,0xe6331a7d),LL(0x094cb960,0x1eb8ad33),LL(0xc91bbca5,0x593cca38), LL(0x26567456,0x384aac8d),LL(0xc04b6490,0x40fa0309),LL(0xdab6c8f6,0x97834cd6),LL(0x3f91e55f,0x68a7318d), + LL(0xfc4d3157,0xa00fd04e),LL(0x2bf3bdea,0xb56f8ab2),LL(0x4fa57172,0x014f5648),LL(0x450abdb3,0x948c5860), LL(0x0ebd4f08,0x342b5df0),LL(0x0e82938e,0x3e5168cd),LL(0xb0df5dd0,0x7aedc1ce),LL(0xe5732516,0x6bbbc6d9), + LL(0x605daaa6,0xc7bfd486),LL(0xbb9a6c9e,0x46fd72b7),LL(0xa124fb89,0xe4847fb1),LL(0xa2d8ffbc,0x75959cbd), LL(0xc8a588ee,0x42579f65),LL(0xb80b499d,0x368c92e6),LL(0x999a5df1,0xea4ef6cd),LL(0x936fe604,0xaa73bb7f), + LL(0x6457d188,0xf347a70d),LL(0x8b7a388b,0x86eda86b),LL(0x0ccd6013,0xb7cdff06),LL(0xd0053fb2,0xbeb1b6c7), LL(0x99240a9f,0x0b022387),LL(0x776189b2,0x1bbb384f),LL(0x9066193a,0x8695e71e),LL(0x06ffac7e,0x2eb50097), + LL(0x4a7d2caa,0x0654a9c0),LL(0xa5aaa290,0x6f3fb3d1),LL(0xff476e8f,0x835db041),LL(0xc42295e4,0x540b8b0b), LL(0x05e214f5,0xa5c73ac9),LL(0x56a0b638,0x9a74075a),LL(0xce9e680b,0x2e4b1090),LL(0x6b8d9afa,0x57a5b479), + LL(0x26bfe65c,0x0dca48e7),LL(0x7290c307,0x097e391c),LL(0x6669e72e,0x683c462e),LL(0x062559ac,0xf505be1e), LL(0xe3a3035a,0x5fbe3ea1),LL(0x9cd50da8,0x6431ebf6),LL(0x1f6407f2,0xfd169d5c),LL(0x60fce6b8,0x8d838a95), + LL(0x650006f0,0x2a2bfa7f),LL(0x50c0fbb2,0xdfd7dad3),LL(0xccf9ad96,0x92452495),LL(0xd95635f9,0x183bf494), LL(0x4a7bd989,0x02d5df43),LL(0xa5431095,0x505385cc),LL(0xfd43f53e,0xdd98e67d),LL(0x500c34a9,0xd61e1a6c), + LL(0x4a8a3d62,0x5a4b46c6),LL(0x247743d2,0x8469c4d0),LL(0x88f7e433,0x2bb3a13d),LL(0x01be5849,0x62b23a10), LL(0xa63d1a4c,0xe83596b4),LL(0x7d183f3e,0x454e7fea),LL(0x17afb01c,0x643fce61),LL(0x1c4c3638,0x4e65e5e6), + LL(0xef74c45b,0x41d85ea1),LL(0xae328506,0x2cfbfa66),LL(0x3ada7da9,0x98b078f5),LL(0xec752fbb,0xd985fe37), LL(0x5a0148b4,0xeece68fe),LL(0x2d78136d,0x6f9a55c7),LL(0xd2b729ce,0x232dccc4),LL(0x90aafbc4,0xa27e0dfd), + LL(0x12b4603e,0x96474452),LL(0x6b706d14,0xa876c551),LL(0x69a9d412,0xdf145fcf),LL(0x2d479c34,0xe2ab75b7), LL(0x1a23ff97,0x12df9a76),LL(0x5d359d10,0xc6138992),LL(0xfa835f22,0x6e51c7ae),LL(0xc0fcc4d9,0x69a79cb1), + LL(0x594cc7e1,0xf57f350d),LL(0x3350ab79,0x3079ca63),LL(0x9aff594a,0x226fb614),LL(0x6d59a62b,0x35afec02), LL(0x06ed2c6e,0x9bee46f4),LL(0x7d939a57,0x58da1735),LL(0x8fd1797e,0x44c50402),LL(0x5ccea6ca,0xd8853e7c), + LL(0xa35fcd5f,0x4065508d),LL(0x495ccaeb,0x8965df8c),LL(0x12e1a962,0x0f2da850),LL(0xc1cf1cc4,0xee471b94), LL(0x0a08fb75,0xcef19bc8),LL(0x81de3591,0x704958f5),LL(0x3aef4f88,0x2867f8b2),LL(0xea9f9a5f,0x8d749384), + LL(0x8c9049f4,0x1b385537),LL(0x7b92d8b6,0x5be948f3),LL(0xb6e2bd6b,0xd96f725d),LL(0x958c454d,0x37a222bc), LL(0x8809bf61,0xe7c61abb),LL(0x1346f18d,0x46f07fbc),LL(0xe87c0d1c,0xfb567a7a),LL(0x7ef3d07a,0x84a461c8), + LL(0xd9278d98,0x0a5adce6),LL(0x9dfc73e1,0x24d94813),LL(0x054321c3,0x4f3528b6),LL(0x692ea706,0x2e03fdde), LL(0x47b533c0,0x10e60619),LL(0x2ca3c055,0x1a8bc73f),LL(0x1bb62b8f,0xae58d4b2),LL(0x584a24e3,0xb2045a73), + LL(0xbd76e195,0x3ab3d5af),LL(0x6938a810,0x478dd1ad),LL(0x6ee3d5cb,0x6ffab393),LL(0x22b361e4,0xdfb693db), LL(0x51dbf1a7,0xf9694496),LL(0x08a2e762,0xcab4b4ef),LL(0xd39bba9a,0xe8c92f25),LL(0xf1464d96,0x850e61bc), + LL(0xdc09508b,0xb7e830e3),LL(0x74317655,0xfaf6d2cf),LL(0xdf690355,0x72606ceb),LL(0xd0c3ded6,0x48bb92b3), LL(0x5c7cf892,0x65b75484),LL(0xd5d5f01f,0xf6cd7ac9),LL(0x96401d69,0xc2c30a59),LL(0xed921878,0x91268650), + LL(0xb78c558f,0x380bf913),LL(0xc8afdaa9,0x43c0baeb),LL(0x54f169d3,0x377f61d5),LL(0xae5ff20b,0xf8da07e3), LL(0xa8a90ea8,0xb676c49d),LL(0x83a29b21,0x81c1ff2b),LL(0x2ad8d276,0x383297ac),LL(0xba89f982,0x3001122f), + LL(0x6718e448,0xe1d794be),LL(0x7c3e6e13,0x246c1482),LL(0x5d26b5ef,0x56646ef8),LL(0x88069cdd,0x80f5091e), LL(0x724bdd38,0xc5992e2f),LL(0x8471e8c7,0x02e915b4),LL(0x0d0ff2a9,0x96ff320a),LL(0x4384d1a0,0xbf886487), + LL(0xc93f72d6,0xbbe1e6a6),LL(0xcad800ea,0xd5f75d12),LL(0xe7acf117,0xfa40a09f),LL(0x7581a355,0x32c8cdd5), LL(0x7023c499,0x74221992),LL(0x38ec3901,0xa8afe5d7),LL(0xa90e83f0,0x5691afcb),LL(0x0b8f8eac,0x41bcaa03), + LL(0x8d2668d5,0xe38b5ff9),LL(0x7ad81965,0x0715281a),LL(0x03c6ce11,0x1bc8fc7c),LL(0x8b650436,0xcbbee6e2), LL(0x0cdb9808,0x06b00fe8),LL(0xfe3ed315,0x17d6e066),LL(0x4d0b5018,0x2e9d38c6),LL(0x844dcaef,0xab8bfd56), + LL(0x513aed8b,0x42894a59),LL(0x314bd07a,0xf77f3b6d),LL(0x8e42b582,0xbbdecb8f),LL(0xd2390fe6,0xf10e2fa8), LL(0x62a2f201,0xefb95022),LL(0x50ee32b0,0x4d59ea50),LL(0x6da789a8,0xd87f7728),LL(0xf79492c4,0xcf98a2cf), + LL(0x720943c2,0xf9577239),LL(0x3990b9d0,0xba044cf5),LL(0x95f2884a,0x5aa8e823),LL(0x0278a0af,0x834de6ed), LL(0x5f25bd12,0xc8e1ee9a),LL(0x6f7ab271,0x9259ceaa),LL(0x77d00b76,0x7e6d97a2),LL(0xa437832a,0x5c0c6eea), + LL(0x5606b81d,0x5232c20f),LL(0x0d991ee5,0xabd7b375),LL(0x8632d951,0x4d2bfe35),LL(0x98ed9364,0x78f85146), LL(0xf30c3282,0x951873f0),LL(0xa789230b,0x0da8ac80),LL(0x5398967f,0x3ac7789c),LL(0xbdda0fb5,0xa69b8f7f), + LL(0x6add8545,0xe5db7717),LL(0x72c49b66,0x1b71cb66),LL(0x68421d77,0xd8560739),LL(0x83e3afea,0x03840fe8), LL(0x1ec69977,0xb391dad5),LL(0x307f6726,0xae243fb9),LL(0xe8ca160c,0xc88ac87b),LL(0x4ce355f4,0x5174cced), + LL(0xe58ba37d,0x98a35966),LL(0x7817335d,0xfdcc8da2),LL(0x83fbc7bf,0x5b752830),LL(0xd9c96984,0x68e419d4), LL(0x02a40380,0x409a39f4),LL(0x1fe977bc,0x88940faf),LL(0x8f8edea6,0xc640a94b),LL(0xed11547d,0x1e22cd17), + LL(0x59ffc3e2,0xe28568ce),LL(0xc1dee4e7,0x60aa1b55),LL(0x837cb363,0xc67497c8),LL(0x105a2bf2,0x06fb438a), LL(0x500d8e20,0x30357ec4),LL(0x0670db10,0x1ad9095d),LL(0xc73b7cfd,0x7f589a05),LL(0x880d6d28,0xf544607d), + LL(0xa20ef103,0x17ba93b1),LL(0x6ba6577b,0xad859130),LL(0x6fa214a0,0x65c91cf6),LL(0x27990da5,0xd7d49c6c), LL(0x20bb569d,0xecd9ec8d),LL(0xeeffbc33,0xbd4b2502),LL(0x6bed0467,0x2056ca5a),LL(0x5b63728c,0x7916a1f7), + LL(0x53a4f566,0xd4f9497d),LL(0x97b56810,0x89734664),LL(0x0494a621,0xf8e1da74),LL(0x8d011c68,0x82546a93), LL(0xc61ac162,0x1f3acb19),LL(0xabad0d3e,0x52f8fa9c),LL(0xb4b7ea43,0x15356523),LL(0xae608125,0x5a16ad61), + LL(0x4faed184,0xb0bcb87f),LL(0x5029f45f,0x5f236b1d),LL(0x0bc6b1fc,0xd42c7607),LL(0x68aefce3,0xc644324e), LL(0x5c5d8446,0x8e191d59),LL(0x13ae1979,0xc0208077),LL(0x3ba59cc7,0xadcaee55),LL(0xa2cb81ba,0x20ed6d6b), + LL(0xb6efcffc,0x0952ba19),LL(0x97c0b87c,0x60f12d68),LL(0x9caa30bc,0x4ee2c7c4),LL(0x97fbff4e,0x767238b7), LL(0x501b5d92,0xebc73921),LL(0xc2a37737,0x3279e3df),LL(0x6d197543,0x9fc12bc8),LL(0x0a40db4e,0xfa94dc6f), + LL(0x530ccbbd,0x7392b41a),LL(0xea823525,0x87c82146),LL(0x05d98d0c,0xa52f984c),LL(0x5ef6974c,0x2ae57d73), LL(0x3042a6dd,0x9377f7bf),LL(0x19647a64,0xb1a007c0),LL(0x0cca9767,0xfaa9079a),LL(0xf68f72d5,0x3d81a25b), + LL(0xff81578e,0x752067f8),LL(0x9045447d,0x78622150),LL(0x0505aa6f,0xc0c22fcf),LL(0x6bed1c77,0x1030f0a6), LL(0x1f0bd739,0x31f29f15),LL(0xe6debe85,0x2d7989c7),LL(0x8e677e98,0x5c070e72),LL(0x06e81fd5,0x0a817bd3), + LL(0xb0f2ac95,0xc110d830),LL(0xab20e64e,0x48d0995a),LL(0x7729cd9a,0x0f3e00e1),LL(0xdd556946,0x2a570c20), LL(0x4e86214d,0x912dbcfd),LL(0xcf615498,0x2d014ee2),LL(0x3530d76e,0x55e2b1e6),LL(0xfd0fd6d1,0xc5135ae4), + LL(0xd4f3049f,0x0066273a),LL(0xe7087477,0xbb8e9893),LL(0x14c6e5fd,0x2dba1ddb),LL(0x51f57e6c,0xdba37886), LL(0x5a72f2cf,0x5aaee0a6),LL(0x7bea5642,0x1208bfbf),LL(0x67872c37,0xf5c6aa3b),LL(0x43f93224,0xd726e083), + LL(0x061f1658,0x1854daa5),LL(0xdf0cd2b3,0xc0016df1),LL(0x833d50de,0xc2a3f23e),LL(0xbbbd3017,0x73b681d2), LL(0x3ac343c0,0x2f046dc4),LL(0x85716421,0x9c847e7d),LL(0x0917eed4,0xe1e13c91),LL(0x63a1b9c6,0x3fc9eebd), + LL(0x7fe02299,0x0f816a72),LL(0x294f3319,0x6335ccc2),LL(0x4745c5be,0x3820179f),LL(0x922f066e,0xe647b782), LL(0x02cafb8a,0xc22e49de),LL(0xfcc2eccc,0x299bc2ff),LL(0x6e0e8282,0x9a8feea2),LL(0xfe893205,0xa627278b), + LL(0x7933e47b,0xa7e19733),LL(0x2e766402,0xf4ff6b13),LL(0x98440d9f,0xa4d8be0a),LL(0x38938808,0x658f5c2f), LL(0xc95b3b3e,0x90b75677),LL(0x3137b6ff,0xfa044269),LL(0x43c47c29,0x077b039b),LL(0x8a6445b2,0xcca95dd3), + LL(0x2333fc4c,0x0b498ba4),LL(0xf736a1b1,0x274f8e68),LL(0x5f1d4b2e,0x6ca348fd),LL(0xa8f10199,0x24d3be78), LL(0xca14f530,0x8535f858),LL(0x5b982e51,0xa6e7f163),LL(0x36e1bf62,0x847c8512),LL(0x03448418,0xf6a7c58e), + LL(0xf9374ab6,0x583f3703),LL(0x6e564145,0x864f9195),LL(0x22526d50,0x33bc3f48),LL(0x1262a496,0x9f323c80), LL(0x3f046a9a,0xaa97a7ae),LL(0xdf8a039a,0x70da183e),LL(0x52aa0ba6,0x5b68f71c),LL(0x21459c2d,0x9be0fe51), + LL(0xcbc613e5,0xc1e17eb6),LL(0x497ea61c,0x33131d55),LL(0xaf7eded5,0x2f69d39e),LL(0xde6af11b,0x73c2f434), LL(0xa4a375fa,0x4ca52493),LL(0xb833c5c2,0x5f06787c),LL(0x3e6e71cf,0x814e091f),LL(0x8b746666,0x76451f57), +}, +/* digit=17 base_pwr=2^119 */ +{ + LL(0x694db7e0,0x80f9bdef),LL(0xb9fcddc6,0xedca8787),LL(0x03b8dce1,0x51981c34),LL(0x70e10ba1,0x4274dcf1), LL(0x6def6d1a,0xf72743b8),LL(0xebdb1866,0xd25b1670),LL(0x050c6f58,0xc4491e8c),LL(0x87fbd7f5,0x2be2b2ab), + LL(0xd111f8ec,0x3e0e5c9d),LL(0xb7c4e760,0xbcc33f8d),LL(0xbd392a51,0x702f9a91),LL(0xc132e92d,0x7da4a795), LL(0x0bb1151b,0x1a0b0ae3),LL(0x02e32251,0x54febac8),LL(0x694e9e78,0xea3a5082),LL(0xe4fe40b8,0xe58ffec1), + LL(0xd1e0cf9e,0xf85592fc),LL(0xc0e7b2e8,0xdea75f0d),LL(0xc135584e,0xc04215cf),LL(0x2f57092a,0x174fc727), LL(0xeb930bea,0xe7277877),LL(0x5eb02a5a,0x504caccb),LL(0xf5241b9b,0xf9fe08f7),LL(0x8d5ca954,0xe7fb62f4), + LL(0x29c4120b,0xfbb8349d),LL(0xc0d0d915,0x9f94391f),LL(0x5410ba51,0xc4074fa7),LL(0x150a5911,0xa66adbf6), LL(0x34bfca38,0xc164543c),LL(0xb9e1ccfc,0xe0f27560),LL(0xe820219c,0x99da0f53),LL(0xc6b4997a,0xe8234498), + LL(0x9d4c5423,0xcfb88b76),LL(0xb0521c49,0x9e56eb10),LL(0xbe8700a1,0x418e0b5e),LL(0xf93cb58a,0x00cbaad6), LL(0xd92a5e67,0xe923fbde),LL(0x1f347f11,0xca4979ac),LL(0x6bc0585b,0x89162d85),LL(0xac3c70e3,0xdd6254af), + LL(0x516e19e4,0x7b23c513),LL(0xc5c4d593,0x56e2e847),LL(0x5ce71ef6,0x9f727d73),LL(0xf79a44c5,0x5b6304a6), LL(0x3ab7e433,0x6638a736),LL(0xfe742f83,0x1adea470),LL(0x5b7fc19f,0xe054b854),LL(0xba1d0698,0xf935381a), + LL(0x799e9a74,0x546eab2d),LL(0xa949f729,0x96239e0e),LL(0x7090055a,0xca274c6b),LL(0x9020c9b0,0x835142c3), LL(0xa2e8807f,0xa405667a),LL(0x1aa3d39e,0x29f2c085),LL(0x42fc72f5,0xcc555d64),LL(0xfbeacb3c,0xe856e0e7), + LL(0x918e4936,0xb5504f9d),LL(0xb2513982,0x65035ef6),LL(0x6f4d9cb9,0x0553a0c2),LL(0xbea85509,0x6cb10d56), LL(0xa242da11,0x48d957b7),LL(0x672b7268,0x16a4d3dd),LL(0x8502a96b,0x3d7e637c),LL(0x730d463b,0x27c7032b), + LL(0xe4136a14,0xbdc02b18),LL(0x678e32bf,0xbacf969d),LL(0xdd9c3c03,0xc98d89a3),LL(0x23becc4f,0x7b92420a), LL(0xc64d565c,0xd4b41f78),LL(0x10f28295,0x9f969d00),LL(0xb13d051a,0xec7f7f76),LL(0xa92da585,0x08945e1e), + LL(0x5846426f,0x55366b7d),LL(0x247d441d,0xe7d09e89),LL(0x736fbf48,0x510b404d),LL(0xe784bd7d,0x7fa003d0), LL(0x17fd9596,0x25f7614f),LL(0x35cb98db,0x49e0e0a1),LL(0x2e83a76a,0x2c65957b),LL(0xcddbe0f8,0x5d40da8d), + LL(0x050bad24,0xf2b8c405),LL(0xc2aa4823,0x8918426d),LL(0xa38365a7,0x2aeab3dd),LL(0x7c91b690,0x72031717), LL(0x60a94120,0x8b00d699),LL(0xe99eaeec,0x478a255d),LL(0x6f60aafd,0xbf656a5f),LL(0x5dee77b3,0xdfd7cb75), + LL(0xa595939d,0x37f68bb4),LL(0x28740217,0x03556479),LL(0x84ad7612,0x8e740e7c),LL(0x9044695f,0xd89bc843), LL(0x85a9184d,0xf7f3da5d),LL(0x9fc0b074,0x562563bb),LL(0xf88a888e,0x06d2e6aa),LL(0x161fbe7c,0x612d8643), + LL(0xf64085e7,0x465edba7),LL(0x29aa8511,0xb230f304),LL(0xcda2d188,0x53388426),LL(0x4b666649,0x90885735), LL(0x652f54f6,0x6f02ff9a),LL(0x5fae2bf0,0x65c82294),LL(0x62f5eee3,0x7816ade0),LL(0xfcc56d70,0xdcdbdf43), + LL(0x54530bb2,0x9fb3bba3),LL(0xcb0869ea,0xbde3ef77),LL(0x0b431163,0x89bc9046),LL(0xe4819a35,0x4d03d7d2), LL(0x43b6a782,0x33ae4f9e),LL(0x9c88a686,0x216db307),LL(0x00ffedd9,0x91dd88e0),LL(0x12bd4840,0xb280da9f), + LL(0x1635e741,0x32a7cb8a),LL(0x78be02a7,0xfe14008a),LL(0x1b7ae030,0x3fafb334),LL(0x5add0ce9,0x7fd508e7), LL(0xd607ad51,0x72c83219),LL(0x8d40964a,0x0f229c0a),LL(0x1c878da2,0x1be2c336),LL(0xeab2ab86,0xe0c96742), + LL(0x3e538cd7,0x458f8691),LL(0x8e08ad53,0xa7001f6c),LL(0xbf5d15ff,0x52b8c6e6),LL(0x011215dd,0x548234a4), LL(0x3d5b4045,0xff5a9d2d),LL(0x4a904190,0xb0ffeeb6),LL(0x48607f8b,0x55a3aca4),LL(0x30a0672a,0x8cbd665c), + LL(0x42583068,0x87f834e0),LL(0xf3f6e683,0x02da2aeb),LL(0x05c12248,0x6b763e5d),LL(0x65a8aefc,0x7230378f), LL(0x71e8e5ca,0x93bd80b5),LL(0xb3b62524,0x53ab041c),LL(0x6c9c552e,0x1b860513),LL(0xd5524e66,0xe84d402c), + LL(0xf37f5937,0xa37f3573),LL(0xd1e4fca5,0xeb0f6c7d),LL(0xac8ab0fc,0x2965a554),LL(0x274676ac,0x17fbf56c), LL(0xacf7d720,0x2e2f6bd9),LL(0x10224766,0x41fc8f88),LL(0x85d53bef,0x517a14b3),LL(0x7d76a7d1,0xdae327a5), + LL(0xc4818267,0x6ad0a065),LL(0x37c1bbc1,0x33aa189b),LL(0x27392a92,0x64970b52),LL(0x2d1535ea,0x21699a1c), LL(0xc2d7a7fd,0xcd20779c),LL(0x99c83cf2,0xe3186059),LL(0x72c0b8c7,0x9b69440b),LL(0x7b9e0e4d,0xa81497d7), + LL(0x1f5f82dc,0x515d5c89),LL(0x6361079e,0x9a7f67d7),LL(0x11a35330,0xa8da81e3),LL(0x4b18be1b,0xe44990c4), LL(0xaf103e59,0xc7d5ed95),LL(0x8dac9261,0xece8aba7),LL(0x9394b8d3,0xbe82b099),LL(0x16adfe83,0x6830f09a), + LL(0x88172d01,0x250a29b4),LL(0xcaff9e02,0x8b20bd65),LL(0xe8a6329a,0xb8a7661e),LL(0xd3fce920,0x4520304d), LL(0x2b47f7ef,0xae45da1f),LL(0x5bffc540,0xe07f5288),LL(0x3464f874,0xf7997009),LL(0xa6fa1f38,0x2244c2cd), + LL(0x94d7d9b1,0x43c41ac1),LL(0xc82e7f17,0x5bafdd82),LL(0x5fda0fca,0xdf0614c1),LL(0xa8ae37ad,0x74b043a7), LL(0x9e71734c,0x3ba6afa1),LL(0x9c450f2e,0x15d5437e),LL(0x67e242b1,0x4a5883fe),LL(0x2c1953c2,0x5143bdc2), + LL(0xfc5e8920,0x542b8b53),LL(0x9a9cee08,0x363bf9a8),LL(0xc3486e08,0x02375f10),LL(0x8c5e70d2,0x2037543b), LL(0x625640b4,0x7109bccc),LL(0x8bc62c3b,0xcbc1051e),LL(0x803f26ea,0xf8455fed),LL(0xeb372424,0x6badceab), + LL(0x6b53f5f9,0xa2a9ce7c),LL(0x1b176d99,0x64246595),LL(0xb95c081b,0xb1298d36),LL(0x1d9a9ee6,0x53505bb8), LL(0xf2ba70b0,0x3f6f9e61),LL(0x8afad453,0xd07e16c9),LL(0xe7eb4a6a,0x9f1694bb),LL(0x3cb0bc8e,0xdfebced9), + LL(0x53868c8b,0x92d3dcdc),LL(0x386107a6,0x174311a2),LL(0x689b4e64,0x4109e07c),LL(0x2df3dcb6,0x30e4587f), LL(0x0811b3b2,0x841aea31),LL(0x0cce43ea,0x6144d41d),LL(0x2a9a7803,0x464c4581),LL(0x3e158930,0xd03d371f), + LL(0xb1f3390b,0xc676d7f2),LL(0xa5b61272,0x9f7a1b8c),LL(0xc2e127a9,0x4ebebfc9),LL(0x5dd997bf,0x4602500c), LL(0x4711230f,0x7f09771c),LL(0x020f09c1,0x058eb37c),LL(0xfee5e38b,0xab693d4b),LL(0x4653cbc0,0x9289eb1f), + LL(0xd51b9cf5,0xbecf46ab),LL(0x9f0121af,0xd2aa9c02),LL(0xe90dc274,0x36aaf7d2),LL(0x48b95a3c,0x909e4ea0), LL(0x6f32dbdb,0xe6b70496),LL(0x8b030b3e,0x672188a0),LL(0xcfb617e2,0xeeffe5b3),LL(0x7c82709e,0x87e947de), + LL(0x1770f5a7,0xa44d2b39),LL(0x0e44eb82,0xe4d4d791),LL(0x3f69712a,0x42e69d1e),LL(0xac6a820e,0xbf11c4d6), LL(0x42c4224c,0xb5e7f3e5),LL(0x449d941c,0xd6b4e81c),LL(0x5450e878,0x5d72bd16),LL(0xee25ac54,0x6a61e28a), + LL(0xe6f1cd95,0x33272094),LL(0x0d18673f,0x7512f30d),LL(0x5afc1464,0x32f7a4ca),LL(0x6bbb977b,0x2f095656), LL(0xa8226200,0x586f47ca),LL(0x1ac07369,0x02c868ad),LL(0xc613acbe,0x4ef2b845),LL(0x0386054c,0x43d7563e), + LL(0xab952578,0x54da9dc7),LL(0x26e84d0b,0xb5423df2),LL(0x9b872042,0xa8b64eeb),LL(0x5990f6df,0xac205782), LL(0x21f4c77a,0x4ff696eb),LL(0xaab273af,0x1a79c3e4),LL(0x9436b3f1,0x29bc922e),LL(0xd6d9a27a,0xff807ef8), + LL(0x778f22a0,0x82acea3d),LL(0x5b5e7469,0xfb10b2e8),LL(0x2818ee7d,0xc0b16980),LL(0xc91c1a2f,0x011afff4), LL(0xad124418,0x95a6d126),LL(0xe72e295f,0x31c081a5),LL(0xf2f4db75,0x36bb283a),LL(0x7acef462,0xd115540f), + LL(0x33f6746c,0xc7f3a8f8),LL(0xfea990ca,0x21e46f65),LL(0xcaddb0a9,0x915fd5c5),LL(0x78614555,0xbd41f016), LL(0x426ffb58,0x346f4434),LL(0x14dbc204,0x80559436),LL(0x5a969b7f,0xf3dd20fe),LL(0xe899a39a,0x9d59e956), + LL(0x8ad4cf4b,0xf1b0971c),LL(0x2ffb8fb8,0x03448860),LL(0x65340ba4,0xf071ac3c),LL(0xb27fd758,0x408d0596), LL(0x98c364b0,0xe7c78ea4),LL(0x051e8ab5,0xa4aac4a5),LL(0x485d9002,0xb9e1d560),LL(0x88844455,0x9acd518a), + LL(0xd06f56c0,0xe4ca688f),LL(0xdf027972,0xa48af70d),LL(0x5e9a609d,0x691f0f04),LL(0xee61270e,0xa9dd82cd), LL(0xa0ef18d3,0x8903ca63),LL(0x3d6ca3bd,0x9fb7ee35),LL(0xabf47d03,0xa7b4a09c),LL(0x1c67de8e,0x4cdada01), + LL(0x9355a244,0x52003749),LL(0x4f2151a9,0xe77fd2b6),LL(0x66b4efcb,0x695d6cf6),LL(0xda2cfe25,0xc5a0cacf), LL(0xef811865,0x104efe5c),LL(0x9ea5cc3d,0xf52813e8),LL(0x40b58dbc,0x855683dc),LL(0x175fcb11,0x0338ecde), + LL(0x74921592,0xf9a05637),LL(0xb9bb9d31,0xb4f1261d),LL(0x4e9c5459,0x551429b7),LL(0x6ea71f53,0xbe182e6f), LL(0xdfc50573,0xd3a3b07c),LL(0x62be8d44,0x9ba1afda),LL(0x52ab65d3,0x9bcfd2cb),LL(0xa9571802,0xdf11d547), + LL(0x02a2404a,0x099403ee),LL(0x21088a71,0x497406f4),LL(0x5004ae71,0x99479409),LL(0xa812c362,0xbdb42078), LL(0xd8828442,0x2b72a30f),LL(0xfcb5ed1c,0x283add27),LL(0x66a40015,0xf7c0e200),LL(0x08b295ef,0x3e3be641), + LL(0xe038a675,0xac127dc1),LL(0x8c5c6320,0x729deff3),LL(0xa90d2c53,0xb7df8fd4),LL(0x681e7cd3,0x9b74b0ec), LL(0xdab407e5,0x5cb5a623),LL(0x76b340c6,0xcdbd3615),LL(0x7d28392c,0xa184415a),LL(0xe96f7830,0xc184c1d8), + LL(0x81d3a80f,0xc3204f19),LL(0xc8e02432,0xfde0c841),LL(0x8149e0c1,0x78203b3e),LL(0x08053a73,0x5904bdbb), LL(0x101b6805,0x30fc1dd1),LL(0x49aa6d49,0x43c223bc),LL(0x7a174087,0x9ed67141),LL(0xd5997008,0x311469a0), + LL(0x5e43fc61,0xb189b684),LL(0xe0d3ab57,0xf3282375),LL(0xb1181da8,0x4fa34b67),LL(0x99ee52b8,0x621ed0b2), LL(0xad990676,0x9b178de1),LL(0x56d54065,0xd51de67b),LL(0x7538c201,0x2a2c27c4),LL(0x38a40f5c,0x33856ec8), + LL(0xbe6cdcde,0x2522fc15),LL(0x9f0c6f89,0x1e603f33),LL(0x103e30a6,0x7994edc3),LL(0x220c853e,0x033a00db), LL(0xf7bb7fd7,0xd3cfa409),LL(0x462d18f6,0x70f8781e),LL(0x687fe295,0xbbd82980),LL(0x595669f3,0x6eef4c32), + LL(0x2f7e85c3,0x86a9303b),LL(0x71988f9b,0x5fce4621),LL(0xc138acb5,0x5b935bf6),LL(0x25661212,0x30ea7d67), LL(0xe51ab9a2,0xef1eb5f4),LL(0xae067c78,0x0587c98a),LL(0x77ca9ca6,0xb3ce1b3c),LL(0x54b5f057,0x2a553d4d), + LL(0x4da29ec2,0xc7898236),LL(0xb9c57316,0xdbdd5d13),LL(0x2cd80d47,0xc57d6e6b),LL(0xfe9e7391,0x80b460cf), LL(0xf963c31e,0x98648cab),LL(0xcc4d32fd,0x67f9f633),LL(0xfdf7c687,0x0af42a9d),LL(0x0b015ea7,0x55f292a3), + LL(0xcd21ab3d,0x89e468b2),LL(0xc393d392,0xe504f022),LL(0xa5013af9,0xab21e1d4),LL(0xc2c28acb,0xe3283f78), LL(0x226bf99f,0xf38b35f6),LL(0x0e291e69,0xe8354274),LL(0xb20c162d,0x61673a15),LL(0xb04fbdbe,0xc101dc75), + LL(0x255bd617,0x8323b4c2),LL(0x6c2a9154,0x6c969693),LL(0x62679387,0xc6e65860),LL(0xb8c88e23,0x8e01db0c), LL(0x893a5559,0x33c42873),LL(0x47a3e149,0x7630f04b),LL(0xddcf35f8,0xb5d80805),LL(0x77dfe732,0x582ca080), + LL(0x0b1894a0,0x2c7156e1),LL(0xd81c68c0,0x92034001),LL(0xc8b115b5,0xed225d00),LL(0x83b907f2,0x237f9c22), LL(0x4470e2c0,0x0ea2f32f),LL(0x58be4e95,0xb725f7c1),LL(0xb1ae5463,0x0f1dcafa),LL(0x1ba2fc04,0x59ed5187), + LL(0xd0115d4d,0xf6e0f316),LL(0xd3691599,0x5180b12f),LL(0x527f0a41,0x157e32c9),LL(0xa8e0ecc0,0x7b0b081d), LL(0xbf4f0dd0,0x6dbaaa8a),LL(0x4d252696,0x99b289c7),LL(0xdbf864fe,0x79b7755e),LL(0x76cad3ab,0x6974e2b1), + LL(0x06ddd657,0x35dbbee2),LL(0x2ff3a96d,0xe7cbdd11),LL(0x076be758,0x88381968),LL(0x08c91f5d,0x2d737e72), LL(0x86ec3776,0x5f83ab62),LL(0x945fa7a1,0x98aa649d),LL(0x72ef0933,0xf477ec37),LL(0x098c17b1,0x66f52b1e), + LL(0xd803738b,0x9eec58fb),LL(0xe4e86aa4,0x91aaade7),LL(0xa5b51492,0x6b1ae617),LL(0xbbc45974,0x63272121), LL(0x862c5129,0x7e0e28f0),LL(0x3321a4a0,0x0a8f79a9),LL(0x5041c88f,0xe26d1664),LL(0x53233e3a,0x0571b805), + LL(0xc9520711,0xd1b0ccde),LL(0x3c8b84bf,0x55a9e4ed),LL(0xa1fef314,0x9426bd39),LL(0x6eb93f2b,0x4f5f638e), LL(0x2bf9341b,0xba2a1ed3),LL(0x4d42d5a9,0xd63c1321),LL(0x316dc7c5,0xd2964a89),LL(0xca511851,0xd1759606), + LL(0xf9e6ed35,0xd8a9201f),LL(0x6736925a,0xb7b5ee45),LL(0x99581af7,0x0a83fbbc),LL(0x64eeb051,0x3076bc40), LL(0x02dec312,0x5511c98c),LL(0x238dcb78,0x270de898),LL(0x539c08c9,0x2cf4cf9c),LL(0x38d3b06e,0xa70cb65e), + LL(0xcfe57bbd,0xb12ec10e),LL(0x35a0c2b5,0x82c7b656),LL(0x161c67bd,0xddc7d5cd),LL(0xae3a32cc,0xe32e8985), LL(0xd11a5529,0x7aba9444),LL(0x2427fa1a,0xe964ed02),LL(0x24a1770a,0x1528392d),LL(0x12c72fcd,0xa152ce2c), + LL(0x8ec07649,0x714553a4),LL(0x459dd453,0x18b4c290),LL(0x7b64b110,0xea32b714),LL(0x2e6f07a2,0xb871bfa5), LL(0x9e2e3c9b,0xb67112e5),LL(0x44aa90f6,0xfbf250e5),LL(0xbd539006,0xf77aedb8),LL(0xd172a66f,0x3b0cdf9a), + LL(0xf8c51187,0xedf69fea),LL(0x741e4da7,0x05bb67ec),LL(0x08114345,0x47df0f32),LL(0xbb9792b1,0x56facb07), LL(0x8f6229e4,0xf3e007e9),LL(0x526fba0f,0x62d103f4),LL(0xb0339d79,0x4f33bef7),LL(0xb59bfec1,0x9841357b), + LL(0xc34e6705,0xfa8dbb59),LL(0x7fdaa84c,0xc3c7180b),LL(0xa4108537,0xf95872fc),LL(0x932a3e5a,0x8750cc3b), LL(0xb7275d7d,0xb61cc69d),LL(0x2e59b2e9,0xffa0168b),LL(0x6ecbb493,0xca032abc),LL(0x2c9082d8,0x1d86dbd3), + LL(0xe28ef5ba,0xae1e0b67),LL(0xcb18e169,0x2c9a4699),LL(0x1e6bbd20,0x0ecd0e33),LL(0xaf5e81d2,0x571b360e), LL(0x101c1d45,0xcd9fea58),LL(0x18880452,0x6651788e),LL(0x1f8dd446,0xa9972635),LL(0xe37281d0,0x44bed022), + LL(0x33da525d,0x094b2b2d),LL(0x13144fd8,0xf193678e),LL(0xf4c1061d,0xb8ab5ba4),LL(0xdccbe0f4,0x4343b5fa), LL(0x63812713,0xa8702371),LL(0xf7611d93,0x47bf6d2d),LL(0xbd21e1d7,0x46729b8c),LL(0xd629e77d,0x7484d4e0), + LL(0x60dbac1f,0x830e6eea),LL(0xda06a2f7,0x23d8c484),LL(0x50ca535b,0x896714b0),LL(0xebd97a9b,0xdc8d3644), LL(0xb12177b4,0x106ef9fa),LL(0x534d5d9c,0xf79bf464),LL(0xa6ab360b,0x2537a349),LL(0xa00c744f,0xc7c54253), + LL(0xe5911a76,0xb3c7a047),LL(0x647f1ee7,0x61ffa5c8),LL(0x8f56ab42,0x15aed36f),LL(0xa3ff9ac9,0x6a0d41b0), LL(0xcc30d357,0x68f469f5),LL(0x6b72be96,0xbe9adf81),LL(0x903ad461,0x1cd926fe),LL(0xcaca441b,0x7e89e38f), + LL(0xfacf69d4,0xf0f82de5),LL(0x4775344c,0x363b7e76),LL(0xb2e36d04,0x6894f312),LL(0x11d1c9a5,0x3c6cb4fe), LL(0x4008e1f2,0x85d9c339),LL(0x249f326c,0x5e9a85ea),LL(0x678c5e06,0xdc35c60a),LL(0x9f86fba9,0xc08b944f), + LL(0x89f71f0f,0xde40c02c),LL(0xff3da3c0,0xad8f3e31),LL(0x42125ded,0x3ea5096b),LL(0xa7379183,0x13879cbf), LL(0x6b306a0b,0x6f4714a5),LL(0x67646c5e,0x359c2ea6),LL(0x07726368,0xfacf8943),LL(0x65ff431e,0x07a58935), + LL(0x68754ab0,0x24d661d1),LL(0x6f429a76,0x801fce1d),LL(0xa58ce769,0xc068a85f),LL(0x5d5eca2b,0xedc35c54), LL(0xa3f660d1,0xea31276f),LL(0xb8fc7167,0xa0184ebe),LL(0x1d8db0ae,0x0f20f21a),LL(0x56c35e12,0xd96d095f), + LL(0xf8c2a25b,0xedf402b5),LL(0x059204b6,0x1bb772b9),LL(0x19b4e34c,0x50cbeae2),LL(0x3fa0845a,0x93109d80), LL(0x8ef59fb5,0x54f7ccf7),LL(0x88070963,0x3b438fe2),LL(0x31f3ba9b,0x9e28c659),LL(0xead9da92,0x9cc31b46), + LL(0xb733aa5f,0x3c2f0ba9),LL(0xf05af235,0xdece47cb),LL(0xa2ac82a5,0xf8e3f715),LL(0x2203f18a,0xc97ba641), LL(0x09c11060,0xc3af5504),LL(0x46af512d,0x56ea2c05),LL(0xf3f28146,0xfac28daf),LL(0x959ef494,0x87fab43a), +}, +/* digit=18 base_pwr=2^126 */ +{ + LL(0xd4c5105f,0x09891641),LL(0x6d7fbd65,0x1ae80f8e),LL(0xbee6bdb0,0x9d67225f),LL(0x7fc4d860,0x3b433b59), LL(0x93e85638,0x44e66db6),LL(0xe3e9862f,0xf7b59252),LL(0x665c32ec,0xdb785157),LL(0xae362f50,0x702fefd7), + LL(0x0fefb0c3,0x3754475d),LL(0x46d7c35d,0xd48fb56b),LL(0x363798a4,0xa070b633),LL(0x8fdb98e6,0xae89f3d2), LL(0x6363d14c,0x970b89c8),LL(0x67abd27d,0x89817521),LL(0x44d5a021,0x9bf7d474),LL(0xcac72aee,0xb3083baf), + LL(0xbe949a44,0x389741de),LL(0x546a4fa5,0x638e9388),LL(0xa0047bdc,0x3fe6419c),LL(0xaaea57ca,0x7047f648), LL(0x41fbab17,0x54e48a90),LL(0x576bdba2,0xda8e0b28),LL(0xc72afddc,0xe807eebc),LL(0xf42577bf,0x07d3336d), + LL(0xbfe20925,0x62a8c244),LL(0x8fdce867,0x91c19ac3),LL(0xdd387063,0x5a96a5d5),LL(0x21d324f6,0x61d587d4), LL(0xa37173ea,0xe87673a2),LL(0x53778b65,0x23848008),LL(0x05bab43e,0x10f8441e),LL(0x4621efbe,0xfa11fe12), + LL(0x81685d7b,0x047b772e),LL(0xbf34a976,0x23f27d81),LL(0x915f48ef,0xc27608e2),LL(0xa521d5c3,0x3b0b43fa), LL(0x63ca7284,0x7613fb26),LL(0x1d4db837,0x7f5729b4),LL(0x583b526b,0x87b14898),LL(0xbbadd3d1,0x00b732a6), + LL(0x2048e396,0x8e02f426),LL(0x383d9de4,0x436b50b6),LL(0x471e85ad,0xf78d3481),LL(0xd005c8d6,0x8b01ea6a), LL(0x97015c07,0xd3c7afee),LL(0x4e3ba2ae,0x46cdf1a9),LL(0x83d3a1d2,0x7a42e501),LL(0xb541dff4,0xd54b5268), + LL(0x4e23e9bc,0x3f24cf30),LL(0x126e3624,0x4387f816),LL(0x3b0b6d61,0x26a46a03),LL(0x8b2d777c,0xaf1bc845), LL(0x527de79c,0x25c401ba),LL(0x4261bbb6,0x0e1346d4),LL(0x287b4bc7,0x4b96c44b),LL(0x5254562f,0x658493c7), + LL(0xb8a24a20,0x23f949fe),LL(0xf52ca53f,0x17ebfed1),LL(0xbcfb4853,0x9b691bbe),LL(0x6278a05d,0x5617ff6b), LL(0xe3c99ebd,0x241b34c5),LL(0x1784156a,0xfc64242e),LL(0x695d67df,0x4206482f),LL(0xee27c011,0xb967ce0e), + LL(0x21c80b5d,0x65db3751),LL(0xa31ecca0,0x2e7a563c),LL(0x5238a07e,0xe56ffc4e),LL(0x32ced854,0x3d6c2966), LL(0xaf70b885,0xe99d7d1a),LL(0x2d686459,0xafc3bad9),LL(0x0cc8ba5b,0x9c78bf46),LL(0x18955aa3,0x5a439519), + LL(0x5fe4e314,0xf8b517a8),LL(0xfcb8906f,0xe60234d0),LL(0xf2061b23,0xffe542ac),LL(0x6b4cb59c,0x287e191f), LL(0x09d877d8,0x21857ddc),LL(0x14678941,0x1c23478c),LL(0xb6e05ea4,0xbbf0c056),LL(0xb01594fe,0x82da4b53), + LL(0xfadb8608,0xf7526791),LL(0x7b74cdf6,0x049e832d),LL(0xc2b90a34,0xa43581cc),LL(0x9360b10c,0x73639eb8), LL(0xe1e4a71b,0x4fba331f),LL(0x8072f919,0x6ffd6b93),LL(0x65679032,0x6e53271c),LL(0xf14272ce,0x67206444), + LL(0xb2335834,0xc0f734a3),LL(0x90ef6860,0x9526205a),LL(0x04e2bb0d,0xcb8be717),LL(0x02f383fa,0x2418871e), LL(0x4082c157,0xd7177681),LL(0x29c20073,0xcc914ad0),LL(0xe587e728,0xf186c1eb),LL(0x61bcd5fd,0x6fdb3c22), + LL(0xf2f9f8e9,0x30d014a6),LL(0x4fec49d2,0x963ece23),LL(0x9605a8d9,0x862025c5),LL(0x19f8929a,0x39874445), LL(0x12bf476a,0x01b6ff65),LL(0x09cf7d91,0x598a64d8),LL(0x93be56ca,0xd7ec7749),LL(0xcbb33615,0x10899785), + LL(0x02eee3ad,0xb8a092fd),LL(0x30145270,0xa86b3d35),LL(0x8512b675,0x323d98c6),LL(0x62ebb40f,0x4b8bc785), LL(0x413f9cde,0x7d301f54),LL(0x2bab5664,0xa5e4fb4f),LL(0x1cbfec23,0x1d2b252d),LL(0xe177120d,0xfcd576bb), + LL(0x83731a34,0x04427d3e),LL(0xed836e8e,0x2bb9028e),LL(0xb612ca7c,0xb36acff8),LL(0xd3d9c73a,0xb88fe5ef), LL(0xedea4eb3,0xbe2a6bc6),LL(0x488eec77,0x43b93133),LL(0xb17106e1,0xf41ff566),LL(0x654efa32,0x469e9172), + LL(0x41c23fa3,0xb4480f04),LL(0xc1989a2e,0xb4712eb0),LL(0x93a29ca7,0x3ccbba0f),LL(0xd619428c,0x6e205c14), LL(0xb3641686,0x90db7957),LL(0x45ac8b4e,0x0432691d),LL(0xf64e0350,0x07a759ac),LL(0x9c972517,0x0514d89c), + LL(0xa8e67fc3,0x1701147f),LL(0xab2085be,0x9e2e0b8b),LL(0xac284e57,0xd5651824),LL(0x74893664,0x890d4325), LL(0xc55e68a3,0x8a7c5e6e),LL(0x4339c85a,0xbf12e90b),LL(0xf922b655,0x31846b85),LL(0x0bf4d700,0x9a54ce4d), + LL(0xf1a14295,0xd7f4e83a),LL(0xb285d4f9,0x916f955c),LL(0x99ffdaba,0xe57bb0e0),LL(0xeab0d152,0x28a43034), LL(0xb8a9cef8,0x0a36ffa2),LL(0xb9ec051a,0x5517407e),LL(0xea68e672,0x9c796096),LL(0xfb3c77fb,0x853db5fb), + LL(0xe864a51a,0x21474ba9),LL(0x6e8a1b8b,0x6c267699),LL(0x94120a28,0x7c823626),LL(0x8383a5db,0xe61e9a48), LL(0x9f84216d,0x7dd75003),LL(0xad43cd85,0xab020d07),LL(0xda12c659,0x9437ae48),LL(0xe65452ad,0x6449c2eb), + LL(0x2cf9d7c1,0xcc7c4c1c),LL(0xee95e5ab,0x1320886a),LL(0xbeae170c,0xbb7b9056),LL(0xdbc0d662,0xc8a5b250), LL(0xc11d2303,0x4ed81432),LL(0x1f03769f,0x7da66912),LL(0x84539828,0x3ac7a5fd),LL(0x3bccdd02,0x14dada94), + LL(0x7ef6b0d1,0x8b84c321),LL(0x7c933f22,0x52a9477a),LL(0xfd440b82,0x5ef6728a),LL(0x6ce4bd5e,0x5c3bd859), LL(0xf22c2d3e,0x918b80f5),LL(0xb7bb6cc5,0x368d5040),LL(0x2695a11c,0xb66142a1),LL(0xeb19ea70,0x60ac583a), + LL(0x0eab2437,0x317cbb98),LL(0x5e2654c8,0x8cc08c55),LL(0xe6d8307f,0xfe2d6520),LL(0x57428993,0xe9f147f3), LL(0xd2fd6cf1,0x5f9c7d14),LL(0x2d4fcbb0,0xa3ecd064),LL(0x8e7341f7,0xad83fef0),LL(0x3a63115c,0x643f23a0), + LL(0xe65ab743,0xd38a78ab),LL(0x35edc89c,0xbf7c75b1),LL(0x530df568,0x3dd8752e),LL(0xe308c682,0xf85c4a76), LL(0xe68acf37,0x4c9955b2),LL(0xab32af85,0xa544df3d),LL(0xa25cf493,0x4b8ec3f5),LL(0x1a622feb,0x4d8f2764), + LL(0xf0dcbc49,0x7bb4f7aa),LL(0x70bbb45b,0x7de551f9),LL(0x9f2ca2e5,0xcfd0f3e4),LL(0x1f5c76ef,0xece58709), LL(0x167d79ae,0x32920edd),LL(0xfa7d7ec1,0x039df8a2),LL(0xbb30af91,0xf46206c0),LL(0x22676b59,0x1ff5e2f5), + LL(0x6ea51d66,0x11f4a039),LL(0x807d7a26,0x506c1445),LL(0x755a9b24,0x60da5705),LL(0x1f1a319e,0x8fc8cc32), LL(0x9433d67d,0x83642d4d),LL(0x6a7dd296,0x7fa5cb8f),LL(0x9b7bde07,0x576591db),LL(0x419716fb,0x13173d25), + LL(0xd5b340ff,0xea30599d),LL(0xb0fe76c5,0xfc6b5297),LL(0xab8f5adc,0x1c6968c8),LL(0x901c928d,0xf723c7f5), LL(0x9773d402,0x4203c321),LL(0x1b51dd47,0xdf7c6aa3),LL(0x552be23c,0x3d49e37a),LL(0x0b5a6e87,0x57febee8), + LL(0x7bd8e739,0xc5ecbee4),LL(0xae63bf75,0x79d44994),LL(0x38fb8923,0x168bd00f),LL(0xd0533130,0x75d48ee4), LL(0xdb5cdf33,0x554f77aa),LL(0x3c696769,0x3396e896),LL(0xd3fd674e,0x2fdddbf2),LL(0x99d0e3e5,0xbbb8f6ee), + LL(0xcbae2f70,0x51b90651),LL(0x93aaa8eb,0xefc4bc05),LL(0xdd1df499,0x8ecd8689),LL(0x22f367a5,0x1aee99a8), LL(0xae8274c5,0x95d485b9),LL(0x7d30b39c,0x6c14d445),LL(0xbcc1ef81,0xbafea90b),LL(0xa459a2ed,0x7c5f317a), + LL(0x4ef44227,0x01211075),LL(0xdc20f496,0xa17bed6e),LL(0x819853cd,0x0cdfe424),LL(0xf71e2ce7,0x13793298), LL(0xdbbe307b,0x3c1f3078),LL(0x76ee9936,0x6dd1c20e),LL(0x423caa20,0x23ee4b57),LL(0x8efb840e,0x4ac3793b), + LL(0xed1f8ca0,0x934438eb),LL(0x4ebb25a2,0x3e546658),LL(0xc069896f,0xc415af0e),LL(0x9a5aa43d,0xc13eddb0), LL(0xd49eb8f6,0x7a04204f),LL(0xd74f1670,0xd0d5bdfc),LL(0x56fc0558,0x3697e286),LL(0x01cebade,0x10207371), + LL(0x0647a82b,0x5f87e690),LL(0x8f40054f,0x908e0ed4),LL(0x79853803,0xa9f633d4),LL(0x4a28b252,0x8ed13c9a), LL(0x1f460f64,0x3e2ef676),LL(0x36d06336,0x53930b9b),LL(0x8fc4979b,0x347073ac),LL(0x5ecd5597,0x84380e0e), + LL(0xc4fe3c39,0xe3b22c6b),LL(0x6c7bebdf,0xba4a8153),LL(0x25693459,0xf23ab6b7),LL(0x14922b11,0x53bc3770), LL(0x5afc60db,0x4645c8ab),LL(0x20b9f2a3,0xaa022355),LL(0xce0fc507,0x52a2954c),LL(0x7ce1c2e7,0x8c2731bb), + LL(0x18a0339d,0xf39608ab),LL(0x3735436c,0xac7a658d),LL(0xcd992b4f,0xb22c2b07),LL(0xf40dcfd4,0x4e83daec), LL(0x2f39ea3e,0x8a34c7be),LL(0xb0a56d2e,0xef0c005f),LL(0x6edd8038,0x62731f6a),LL(0x4e3cb075,0x5721d740), + LL(0xfbeeee1b,0x1ea41511),LL(0xef1d0c05,0xd1ef5e73),LL(0x73c07d35,0x42feefd1),LL(0x8a329493,0xe530a00a), LL(0xf15ebfb0,0x5d55b7fe),LL(0xd322491a,0x549de03c),LL(0x745b3237,0xf7b5f602),LL(0x1ab6e2b6,0x3632a3a2), + LL(0x0ef59f78,0x0d3bba89),LL(0xc9e52b9a,0x0dfc6443),LL(0x72631447,0x1dc79699),LL(0xb3be20b1,0xef033917), LL(0xb1383948,0x0c92735d),LL(0xc0dd7d7d,0xc1fc29a2),LL(0x403ed068,0x6485b697),LL(0xaac93bdc,0x13bfaab3), + LL(0x0deeaf52,0x410dc6a9),LL(0x4c641c15,0xb003fb02),LL(0x5bc504c4,0x1384978c),LL(0x864a6a77,0x37640487), LL(0x222a77da,0x05991bc6),LL(0x5e47eb11,0x62260a57),LL(0xf21b432c,0xc7af6613),LL(0xab4953e9,0x22f3acc9), + LL(0x8e41d155,0x52934922),LL(0x3ac059ef,0x4d024568),LL(0x4d884411,0xb0201755),LL(0xa59a178f,0xce8055cf), LL(0xf6204549,0xcd77d1af),LL(0xc7066759,0xa0a00a3e),LL(0x0272c229,0x471071ef),LL(0xd3c4b6b0,0x009bcf6b), + LL(0x22305177,0x2a2638a8),LL(0x41645bbf,0xd51d59df),LL(0xc0a7a3c0,0xa81142fd),LL(0x4c7063ee,0xa17eca6d), LL(0x60d9dcec,0x0bb887ed),LL(0x20ad2455,0xd6d28e51),LL(0xa67102ba,0xebed6308),LL(0x8bffa408,0x042c3114), + LL(0x8aa68e30,0xfd099ac5),LL(0x1483513e,0x7a6a3d7c),LL(0xba2d8f0c,0xffcc6b75),LL(0x1e78b954,0x54dacf96), LL(0xa4a9af89,0xf645696f),LL(0x06ac98ec,0x3a411940),LL(0x22a67a20,0x41b8b3f6),LL(0x99dec626,0x2d0b1e0f), + LL(0x40be34e8,0x27c89192),LL(0x91907f35,0xc7162b37),LL(0xa956702b,0x90188ec1),LL(0xdf93769c,0xca132f7d), LL(0x0e2025b4,0x3ece44f9),LL(0x0c62f14c,0x67aaec69),LL(0x22e3cc11,0xad741418),LL(0x7ff9a50e,0xcf9b75c3), + LL(0x4d348272,0x02fa2b16),LL(0x9959d56d,0xbd99d61a),LL(0x18762916,0xbc4f19db),LL(0x49c1ac80,0xcc7cce50), LL(0xd846bd83,0x4d59ebaa),LL(0xa9202849,0x8775a9dc),LL(0x6e1f4ca9,0x07ec4ae1),LL(0xba893f11,0x27eb5875), + LL(0x662cc565,0x00284d51),LL(0x0db4138d,0x82353a6b),LL(0xaa32a594,0xd9c7aaaa),LL(0xa5669c47,0xf5528b5e), LL(0x2f23c5ff,0xf3220231),LL(0x6affa3a1,0xe3e8147a),LL(0x202ddda0,0xfb423d5c),LL(0x6b871bd4,0x3d6414ac), + LL(0xa51a168a,0x586f82e1),LL(0x48ae5448,0xb712c671),LL(0x76233eb8,0x9a2e4bd1),LL(0x78811ca9,0x0188223a), LL(0xf7c18de1,0x553c5e21),LL(0xb27bb286,0x7682e451),LL(0x0e51e929,0x3ed036b3),LL(0xec9cb34f,0xf487211b), + LL(0x0c24efc8,0x0d094277),LL(0xbef737a4,0x0349fd04),LL(0x514cdd28,0x6d1c9dd2),LL(0x30da9521,0x29c135ff), LL(0xf78b0b6f,0xea6e4508),LL(0x678c143c,0x176f5dd2),LL(0x4be21e65,0x08148418),LL(0xe7df38c4,0x27f7525c), + LL(0x748ab1a4,0x1fb70e09),LL(0x5efe4433,0x9cba50a0),LL(0x15f75af2,0x7846c7a6),LL(0x5ee73ea8,0x2a7c2c57), LL(0x3f0a449a,0x42e566a4),LL(0xad90fc3d,0x45474c3b),LL(0x8b61d057,0x7447be3d),LL(0x3a4ec092,0x3e9d1cf1), + LL(0xf380a6e6,0x1603e453),LL(0x9b1437c2,0x0b86e431),LL(0xef29610a,0x7a4173f2),LL(0xf03d57f7,0x8fa729a7), LL(0x6c9c217e,0x3e186f6e),LL(0x91919524,0xbe1d3079),LL(0x153d4fb1,0x92a62a70),LL(0xd68c2f71,0x32ed3e34), + LL(0x9eb1a8b7,0xd785027f),LL(0xc5b22fe8,0xbc37eb77),LL(0xb9d6a191,0x466b34f0),LL(0x9a05f816,0x008a89af), LL(0x7d42c10a,0x19b028fb),LL(0x49b3f6b8,0x7fe8c92f),LL(0xa5a0ade3,0x58907cc0),LL(0x559d1a7c,0xb3154f51), + LL(0xd9790ed6,0x5066efb6),LL(0xa6aa793b,0xa77a0cbc),LL(0x223e042e,0x1a915f3c),LL(0x69c5874b,0x1c5def04), LL(0x73b6c1da,0x0e830078),LL(0xfcd8557a,0x55cf85d2),LL(0x0460f3b1,0x0f7c7c76),LL(0x46e58063,0x87052acb), + LL(0x907eae66,0x09212b80),LL(0x4d721c89,0x3cb068e0),LL(0xdd45ac1c,0xa87941ae),LL(0x0daa0dbb,0xde8d5c0d), LL(0xe3502e6e,0xda421fdc),LL(0x4d89a084,0xc8944201),LL(0xf0c24bfb,0x7307ba5e),LL(0x20bde0ef,0xda212beb), + LL(0xf82ce682,0xea2da24b),LL(0x07f71fe4,0x058d3816),LL(0x5ffad8de,0x35a02462),LL(0xaadcefab,0xcd7b05dc), LL(0x1d9f54ec,0xd442f8ed),LL(0xb2d3b5ca,0x8be3d618),LL(0xe06b2ce2,0xe2220ed0),LL(0x1b0da4c0,0x82699a5f), + LL(0x71c0c3a7,0x3ff106f5),LL(0x0d34180c,0x8f580f5a),LL(0x22d7d375,0x4ebb120e),LL(0xe9513675,0x5e5782cc), LL(0x99c82a70,0x2275580c),LL(0x15ea8c4c,0xe8359fbf),LL(0x7b415e70,0x53b48db8),LL(0x100c6014,0xaacf2240), + LL(0xe4652f1d,0x9faaccf5),LL(0xd56157b2,0xbd6fdd2a),LL(0x6261ec50,0xa4f4fb1f),LL(0x476bcd52,0x244e55ad), LL(0x047d320b,0x881c9305),LL(0x6181263f,0x1ca983d5),LL(0x278fb8ee,0x354e9a44),LL(0x396e4964,0xad2dbc0f), + LL(0x9268b3de,0x723f3aa2),LL(0xe6e0609a,0x0d1ca29a),LL(0x6cf44252,0x794866aa),LL(0x01af87ed,0x0b59f3e3), LL(0x7f4a6c51,0xe234e5ff),LL(0x61dc2f7e,0xa8768fd2),LL(0x0a94d81f,0xdafc7332),LL(0x06938ce1,0xd7f84282), + LL(0x0546063e,0xae0b3c0e),LL(0x5d61abc6,0x7fbadcb2),LL(0x369ac400,0xd5d7a2c9),LL(0xae67d10c,0xa5978d09), LL(0x4f85eaac,0x290f211e),LL(0xfacac681,0xe61e2ad1),LL(0x388384cd,0xae125225),LL(0xccfde30f,0xa7fb68e9), + LL(0x3daed4c2,0x7a59b936),LL(0x2606f789,0x80a9aa40),LL(0xf6a6d90a,0xb40c1ea5),LL(0x514d5885,0x948364d3), LL(0x70985182,0x062ebc60),LL(0x33310895,0xa6db5b0e),LL(0xe329c2f5,0x64a12175),LL(0x90ea237e,0xc5f25bd2), + LL(0x2d0a4c23,0x7915c524),LL(0x6bb3cc52,0xeb5d26e4),LL(0xc09e2c92,0x369a9116),LL(0xcf182cf8,0x0c527f92), LL(0x2aede0ac,0x9e591938),LL(0x6cc34939,0xb2922208),LL(0x99a34361,0x3c9d8962),LL(0xc1905fe6,0x3c81836d), + LL(0xa001ec5a,0x4bfeb57f),LL(0xa0dc5dba,0xe993f5bb),LL(0x724a1380,0x47884109),LL(0x32fe9a04,0x8a0369ab), LL(0x8c927db8,0xea068d60),LL(0x94655741,0xbf5f37cf),LL(0x04b6c7ea,0x47d402a2),LL(0x6af259cb,0x4551c295), + LL(0xed77ee8b,0x698b71e7),LL(0xf309d5c7,0xbddf7bd0),LL(0x34e780ca,0x6201c22c),LL(0x4c295ef4,0xab04f7d8), LL(0x4313a8ce,0x1c947294),LL(0x92ca4cfe,0xe532e4ac),LL(0xd0a7a97a,0x89738f80),LL(0xa580fd5b,0xec088c88), + LL(0x42ce9e51,0x612b1ecc),LL(0xb25fdd2a,0x8f9840fd),LL(0x01e7f839,0x3cda78c0),LL(0xece05480,0x546b3d3a), LL(0x80d30916,0x271719a9),LL(0x584c20c4,0x45497107),LL(0x5bc78608,0xaf8f9478),LL(0x277e2a4c,0x28c7d484), + LL(0x88a2ffe4,0xfce01767),LL(0x28e169a5,0xdc506a35),LL(0x7af9c93a,0x0ea10861),LL(0x03fa0e08,0x1ed24361), LL(0xa3d694e7,0x96eaaa92),LL(0xef50bc74,0xc0f43b4d),LL(0x64114db4,0xce6aa58c),LL(0x7c000fd4,0x8218e8ea), + LL(0x185f8844,0xac815dfb),LL(0x1557abfb,0xcd7e90cb),LL(0xafbfecdf,0x23d16655),LL(0x085cac4a,0x80f3271f), LL(0xd0e62f47,0x7fc39aa7),LL(0x460a48e5,0x88d519d1),LL(0xd28f101e,0x59559ac4),LL(0xca9ae816,0x7981d9e9), + LL(0x9ac38203,0x5c38652c),LL(0x57657fe5,0x86eaf87f),LL(0xe21f5416,0x568fc472),LL(0xe7e597b5,0x2afff39c), LL(0x256d4eab,0x3adbbb07),LL(0x8285ab89,0x22598692),LL(0x041caefe,0x35f8112a),LL(0xa5064c8b,0x95df02e3), + LL(0xc7004bf3,0x4d63356e),LL(0xdb83c7de,0x230a08f4),LL(0x8709a7b7,0xca27b270),LL(0xcb9abd2d,0x0d1c4cc4), LL(0x7550fee8,0x8a0bc66e),LL(0x9cf7247e,0x369cd4c7),LL(0x92b5b7e7,0x75562e84),LL(0x5802af7b,0x8fed0da0), + LL(0xe48fb889,0x6a7091c2),LL(0x7b8a9d06,0x26882c13),LL(0x1b82a0e2,0xa2498663),LL(0x3518152d,0x844ed736), LL(0xd86e27c7,0x282f476f),LL(0x04afefdc,0xa04edaca),LL(0x6119e34d,0x8b256ebc),LL(0x0787d78b,0x56a413e9), +}, +/* digit=19 base_pwr=2^133 */ +{ + LL(0x5a74be50,0x82ee061d),LL(0xdea16ff5,0xe41781c4),LL(0x99bfc8a2,0xe0b0c81e),LL(0x0b547e2d,0x624f4d69), LL(0xbdcc9ae4,0x3a83545d),LL(0x409b1e8e,0x2573dbb6),LL(0xa6c93539,0x482960c4),LL(0x5ae18798,0xf01059ad), + LL(0x3112795f,0x715c9f97),LL(0x984e6ee1,0xe8244437),LL(0xecb66bcd,0x55cb4858),LL(0xabaffbee,0x7c136735), LL(0x5dbec38e,0x54661595),LL(0x388ad153,0x51c0782c),LL(0xc6e0952f,0x9ba4c53a),LL(0x1b21dfa8,0x27e6782a), + LL(0x4ed2dbc2,0x682f903d),LL(0x7c3b2d83,0x0eba59c8),LL(0x9c7e9335,0x8e9dc84d),LL(0x0eb226d7,0x5f9b21b0), LL(0xaf267bae,0xe33bd394),LL(0xbe2e15ae,0xaa86cc25),LL(0x6a8ec500,0x4f0bf67d),LL(0xf9630658,0x5846aa44), + LL(0xe2c2bf15,0xfeb09740),LL(0xa9e99704,0x627a2205),LL(0xc2fbc565,0xec8d73d0),LL(0xc20c8de8,0x223eed8f), LL(0xa8363b49,0x1ee32583),LL(0xc9c2b0a6,0x1a0b6cb9),LL(0x90dbc85c,0x49f7c3d2),LL(0x1ef4c1ac,0xa8dfbb97), + LL(0x65c7c2ab,0xafb34d4c),LL(0xe2c5ea84,0x1d4610e7),LL(0x973c4ab5,0x893f6d1b),LL(0x945ba5c4,0xa3cdd7e9), LL(0x064417ee,0x60514983),LL(0xad6bdf2b,0x1459b23c),LL(0x5cf726c3,0x23b2c341),LL(0x32d6354a,0x3a829635), + LL(0xab192c18,0x294f901f),LL(0x7030164f,0xec5fcbfe),LL(0xe2246ba6,0xe2e2fcb7),LL(0x221a1a0c,0x1e7c88b3), LL(0xc92d88c5,0x72c7dd93),LL(0x1106fb59,0x41c2148e),LL(0xa0f60f14,0x547dd4f5),LL(0x63960f31,0xed9b52b2), + LL(0xb0a5b358,0x6c8349eb),LL(0x9e7e2ed6,0xb154c5c2),LL(0xeda462db,0xcad5eccf),LL(0x2de66b69,0xf2d6dbe4), LL(0x8665e5b2,0x426aedf3),LL(0x7b7f5723,0x488a8513),LL(0x8bcbb386,0x15cc43b3),LL(0xd791d879,0x27ad0af3), + LL(0x846e364f,0xc16c236e),LL(0xdea50ca0,0x7f33527c),LL(0x0926b86d,0xc4810775),LL(0x0598e70c,0x6c2a3609), LL(0xf024e924,0xa6755e52),LL(0x9db4afca,0xe0fa07a4),LL(0x66831790,0x15c3ce7d),LL(0xa6cbb0d6,0x5b4ef350), + LL(0xb6205969,0x2c4aafc4),LL(0xf6c7854f,0x42563f02),LL(0x1d983b48,0x016aced5),LL(0x99949755,0xfeb356d8), LL(0xd1a39bd7,0x8c2a2c81),LL(0xe6934ae9,0x8f44340f),LL(0x447904da,0x148cf91c),LL(0x0f51a926,0x7340185f), + LL(0x7409ab46,0x2f8f00fb),LL(0x80e289b2,0x057e78e6),LL(0xa888e5d1,0x03e5022c),LL(0x9dede4e2,0x3c87111a), LL(0x7809460b,0x5b9b0e1c),LL(0x71c9abc7,0xe751c852),LL(0xc7cc1dc9,0x8b944e28),LL(0x1d3cfa08,0x4f201ffa), + LL(0x3e6721ce,0x02fc905c),LL(0xd0b3674c,0xd52d70da),LL(0x18810da4,0x5dc2e5ca),LL(0x5c69dd99,0xa984b273), LL(0x84de5ca4,0x63b92527),LL(0xc852dec4,0x2f1c9872),LL(0xc2e3de09,0x18b03593),LL(0x9813dc2f,0x19d70b01), + LL(0xa6dc1d29,0x42806b2d),LL(0xf871e144,0xd3030009),LL(0xaaf49276,0xa1feb333),LL(0xc70bc04b,0xb5583b9e), LL(0x95695f20,0x1db0be78),LL(0x89d012b5,0xfc841811),LL(0x05f61643,0x6409f272),LL(0xd5883128,0x40d34174), + LL(0x67419833,0xd79196f5),LL(0x863b7b08,0x6059e252),LL(0x1c56700c,0x84da1817),LL(0xb28d3ec4,0x5758ee56), LL(0x013b0ea6,0x7da2771d),LL(0x54c5e9b9,0xfddf524b),LL(0x24305d80,0x7df4faf8),LL(0x3a97763f,0x58f5c1bf), + LL(0x7c696042,0xa5af37f1),LL(0x4a2538de,0xd4cba22c),LL(0x9ea42600,0x211cb995),LL(0x7b069889,0xcd105f41), LL(0xddb81e74,0xb1e1cf19),LL(0x5157b8ca,0x472f2d89),LL(0xee9db885,0x086fb008),LL(0x0f26d131,0x365cd570), + LL(0xa2be7053,0x284b02bb),LL(0x7ab9a6d6,0xdcbbf7c6),LL(0x20f7a530,0x4425559c),LL(0x188767c8,0x961f2dfa), LL(0x70dc80c4,0xe2fd9435),LL(0xf0784120,0x104d6b63),LL(0x53567122,0x7f592bc1),LL(0xf688ad77,0xf6bc1246), + LL(0x0f15dde9,0x05214c05),LL(0x0d5f2b82,0xa47a76a8),LL(0x62e82b62,0xbb254d30),LL(0x3ec955ee,0x11a05fe0), LL(0x9d529b36,0x7eaff46e),LL(0x8f9e3df6,0x55ab1301),LL(0x99317698,0xc463e371),LL(0xccda47ad,0xfd251438), + LL(0x23d695ea,0xca9c3547),LL(0x16e589b5,0x48ce626e),LL(0xb187d086,0x6b5b64c7),LL(0xb2207948,0xd02e1794), LL(0x7198111d,0x8b58e98f),LL(0xdcf9c3cc,0x90ca6305),LL(0xf34089b0,0x5691fe72),LL(0xfc7c80ff,0x60941af1), + LL(0x22eb51e5,0xa09bc0a2),LL(0xaa9cf09a,0xc0bb7244),LL(0x80159f06,0x36a8077f),LL(0xdddc560e,0x8b5c989e), LL(0x512e1f43,0x19d2f316),LL(0xad08ff62,0x02eac554),LL(0x07d20b4e,0x012ab84c),LL(0xd6d4e4e1,0x37d1e115), + LL(0xab7b19a8,0xb6443e1a),LL(0xdef8cd45,0xf08d067e),LL(0x685e03da,0x63adf3e9),LL(0x4792b916,0xcf15a10e), LL(0xb738a425,0xf44bcce5),LL(0x9636b2fd,0xebe131d5),LL(0x7850d605,0x94068841),LL(0xb40d749d,0x09684eaa), + LL(0x72ba075b,0x8c3c669c),LL(0xba469015,0x89f78b55),LL(0x3e9f8ba8,0x5706aade),LL(0xb32d7ed7,0x6d8bd565), LL(0x805f08d6,0x25f4e63b),LL(0xc3bcc1b5,0x7f48200d),LL(0xb025d847,0x4e801968),LL(0x87cbe0a8,0x74afac04), + LL(0x7e63d690,0x43ed2c2b),LL(0x0223cdb8,0xefb6bbf0),LL(0x2884d3fe,0x4fec3cae),LL(0xd75e25a4,0x065ecce6), LL(0x69f79071,0x6c2294ce),LL(0x044b8666,0x0d9a8e5f),LL(0x17b69d8f,0x5009f238),LL(0xc5dfdaf7,0x3c29f8fe), + LL(0xebae68c4,0x9067528f),LL(0x30c5ba21,0x5b385632),LL(0x1fdd1aec,0x540df119),LL(0xcfba4c78,0xcf37825b), LL(0xbeb11454,0x77eff980),LL(0x60c1b066,0x40a1a991),LL(0xf889a1c7,0xe8018980),LL(0x76c24be0,0xb9c52ae9), + LL(0x45650ef4,0x05fbbcce),LL(0x8aa29ac7,0xae000f10),LL(0x4f04c470,0x884b7172),LL(0x19bb5c25,0x7cd4fde2), LL(0xe8840869,0x6477b22a),LL(0x5fbd0686,0xa8868859),LL(0x1116dfba,0xf23cc02e),LL(0xd87d7776,0x76cd563f), + LL(0xa9d82abf,0xe2a37598),LL(0xe6c170f5,0x5f188ccb),LL(0x5066b087,0x81682200),LL(0xc7155ada,0xda22c212), LL(0xfbddb479,0x151e5d3a),LL(0x6d715b99,0x4b606b84),LL(0xf997cb2e,0x4a73b54b),LL(0x3ecd8b66,0x9a1bfe43), + LL(0x2a67d48a,0x1c312809),LL(0x031fa9e2,0xcd6a671e),LL(0x0e43a34a,0xbec3312a),LL(0x55ef47d3,0x1d935639), LL(0x8fea73ea,0x5ea02489),LL(0xa035afb2,0x8247b364),LL(0x5265b54c,0xb58300a6),LL(0x722c7148,0x3286662f), + LL(0xb4ec4c20,0xb77fd76b),LL(0x0f3fe3fd,0xf0a12fa7),LL(0x41d8c7e8,0xf845bbf5),LL(0x5ec10aa8,0xe4d969ca), LL(0x43e232a3,0x4c0053b7),LL(0x37f8a45a,0xdc7a3fac),LL(0x20d81c8f,0x3c4261c5),LL(0xb00eab00,0xfd4b3453), + LL(0xd36e3062,0x76d48f86),LL(0xa143ff02,0x626c5277),LL(0xaf76f42e,0x538174de),LL(0x6407ceac,0x2267aa86), LL(0x72e572d5,0xfad76351),LL(0xba7330eb,0xab861af7),LL(0x418d8657,0xa0a1c8c7),LL(0x20289a52,0x988821cb), + LL(0xcccc18ad,0x79732522),LL(0xf1a6e027,0xaadf3f8d),LL(0x17c2354d,0xf7382c93),LL(0xd818b689,0x5ce1680c), LL(0xd9ecbee9,0x359ebbfc),LL(0x1cae62ac,0x4330689c),LL(0xc51ac38a,0xb55ce5b4),LL(0xfe238ee8,0x7921dfea), + LL(0x271d1ca5,0x3972bef8),LL(0xe8aabd18,0x3e423bc7),LL(0x44a3e5e3,0x57b09f3f),LL(0x7b444d66,0x5da886ae), LL(0xa9964375,0x68206634),LL(0x699cd0ff,0x356a2fa3),LL(0xdba515e9,0xaf0faa24),LL(0xb321d79a,0x536e1f5c), + LL(0x5c04e4ea,0xd3b9913a),LL(0xd6f11513,0xd549dcfe),LL(0x79fd1d94,0xee227bf5),LL(0xb43f2c67,0x9f35afee), LL(0xf1314f53,0xd2638d24),LL(0xcabcd822,0x62baf948),LL(0x4ef48db0,0x5542de29),LL(0xfc5f6bb2,0xb3eb6a04), + LL(0x1208e16a,0x23c110ae),LL(0xf8363e24,0x1a4d15b5),LL(0x164be00b,0x30716844),LL(0xf6f4690d,0xa8e24824), LL(0x90b170cf,0x548773a2),LL(0x42f191f4,0xa1bef331),LL(0x9247aa97,0x70f418d0),LL(0x48be9147,0xea06028e), + LL(0xdbfb894e,0xe13122f3),LL(0xce274b18,0xbe9b79f6),LL(0xca58aadf,0x85a49de5),LL(0x11487351,0x24957758), LL(0xbb939099,0x111def61),LL(0x26d13694,0x1d6a974a),LL(0xd3fc253b,0x4474b4ce),LL(0x4c5db15e,0x3a1485e6), + LL(0x147c15b4,0xe79667b4),LL(0x7bc61301,0xe34f553b),LL(0x17094381,0x032b80f8),LL(0x723eaa21,0x55d8bafd), LL(0xf1c0e74e,0x5a987995),LL(0xebba289c,0x5a9b292e),LL(0xeb4c8251,0x413cd4b2),LL(0xd162db0a,0x98b5d243), + LL(0x68342520,0xbb47bf66),LL(0xbaa862d1,0x08d68949),LL(0xe906abcd,0x11f349c7),LL(0xed7bf00e,0x454ce985), LL(0xb55b803b,0xacab5c9e),LL(0x31e3c16d,0xb03468ea),LL(0xd273bf12,0x5c24213d),LL(0x71587887,0x211538eb), + LL(0x731dea2d,0x198e4a2f),LL(0x74ed7b2a,0xd5856cf2),LL(0x13a664fe,0x86a632eb),LL(0xbda41291,0x932cd909), LL(0xc0c4ddc0,0x850e95d4),LL(0x347fc2c9,0xc0f422f8),LL(0x86076bcb,0xe68cbec4),LL(0xcd6cd286,0xf9e7c0c0), + LL(0x0f5f27ca,0x65994ddb),LL(0xa80d59ff,0xe85461fb),LL(0x66601023,0xff05481a),LL(0xfc9ebbfb,0xc665427a), LL(0x7587fd52,0xb0571a69),LL(0x8d49efce,0x935289f8),LL(0xea420688,0x61becc60),LL(0x13a786af,0xb22639d9), + LL(0x361ecf90,0x1a8e6220),LL(0x25506463,0x001f23e0),LL(0x0a5c2b79,0xe4ae9b5d),LL(0xd8149db5,0xebc9cdad), LL(0x934aa728,0xb33164a1),LL(0xae9b60f3,0x750eb00e),LL(0x9b9cfbfd,0x5a91615b),LL(0xef45f7f6,0x97015cbf), + LL(0xbf5151df,0xb462c4a5),LL(0xb07118f2,0x21adcc41),LL(0x043fa42c,0xd60c545b),LL(0xe96be1ab,0xfc21aa54), LL(0x4e51ea80,0xe84bc32f),LL(0x259b5d8d,0x3dae45f0),LL(0xc38f1b5e,0xbb73c7eb),LL(0xe8ae617d,0xe405a74a), + LL(0x9f1c56bd,0xbb1ae9c6),LL(0x49f196a4,0x8c176b98),LL(0x6875092b,0xc448f311),LL(0x9f976033,0xb5afe3de), LL(0x145813e5,0xa8dafd49),LL(0xe2b34226,0x687fc4d9),LL(0x4c7ff57f,0xf2dfc92d),LL(0x401f1b46,0x004e3fc1), + LL(0x1430c9ab,0x5afddab6),LL(0x2238e997,0x0bdd41d3),LL(0x418042ae,0xf0947430),LL(0xcdddc4cb,0x71f9adda), LL(0xc52dd907,0x7090c016),LL(0x29e2047f,0xd9bdf44d),LL(0x1b1011a6,0xe6f1fe80),LL(0xd9acdc78,0xb63accbc), + LL(0x1272a95b,0xcfc7e235),LL(0xa6276ac8,0x0c667717),LL(0xe2d7eef7,0x3c0d3709),LL(0x9a685b3e,0x5add2b06), LL(0x14ea5d65,0x363ad32d),LL(0x8d7dd506,0xf8e01f06),LL(0x75b4aac6,0xc9ea2213),LL(0x0d353466,0xed2a2bf9), + LL(0xe9d3a7c3,0x439d79b5),LL(0x81b7f34b,0x8e0ee5a6),LL(0x1dc4ba75,0xcf3dacf5),LL(0xeb3310c7,0x1d3d1773), LL(0x7747ae83,0xa8e67112),LL(0x197d6b40,0x31f43160),LL(0xcd961400,0x0521ccee),LL(0xf6535768,0x67246f11), + LL(0xef0c3133,0x702fcc5a),LL(0x7e16693b,0x247cc45d),LL(0xc729b749,0xfd484e49),LL(0xb218320f,0x522cef7d), LL(0x59ab93b3,0xe56ef405),LL(0x9f181071,0x225fba11),LL(0x15330ed0,0x33bd6595),LL(0x1ddb32f7,0xc4be69d5), + LL(0x0448087c,0x264c7668),LL(0x71432dae,0xac30903f),LL(0x00f9bf47,0x3851b266),LL(0x6cdd6d03,0x400ed311), LL(0xf8fd2424,0x045e79fe),LL(0xfa6da98b,0xfdfd974a),LL(0x0c1e673a,0x45c9f641),LL(0x5b2c5168,0x76f2e733), + LL(0x2a601753,0x1adaebb5),LL(0xc57c2d49,0xb286514c),LL(0x1e0bfd24,0xd8769670),LL(0x04478922,0x950c547e), LL(0xe5d32bfe,0xd1d41969),LL(0x750d6c3e,0x30bc1472),LL(0xe0e27f3a,0x8f3679fe),LL(0xa4a6ee0c,0x8f64a7dc), + LL(0x633dfb1f,0x2fe59937),LL(0x977f2547,0xea82c395),LL(0x661ea646,0xcbdfdf1a),LL(0xb9085451,0xc7ccc591), LL(0x81761e13,0x82177962),LL(0x9196885c,0xda57596f),LL(0x28ffbd70,0xbc17e849),LL(0x2671d36f,0x1e6e0a41), + LL(0x4152fcf5,0x61ae872c),LL(0x9e77e754,0x441c87b0),LL(0xa34dff09,0xd0799dd5),LL(0x88a6b171,0x766b4e44), LL(0x11f1c792,0xdc06a512),LL(0x4be35c3e,0xea02ae93),LL(0xe90c469e,0xe5ca4d6d),LL(0x56e4ff5c,0x4df4368e), + LL(0x4baef62e,0x7817acab),LL(0xa85b91e8,0x9f5a2202),LL(0x6ce57610,0x9666ebe6),LL(0xf73bfe03,0x32ad31f3), LL(0x25bcf4d6,0x628330a4),LL(0x515056e6,0xea950593),LL(0xe1332156,0x59811c89),LL(0x8c11b2d7,0xc89cf1fe), + LL(0x04e60cc0,0x75b63913),LL(0x4625d375,0xce811e8d),LL(0x2d26e562,0x030e43fc),LL(0x608d36a0,0xfbb30b4b), LL(0x48528118,0x634ff82c),LL(0xcd285911,0x7c6fe085),LL(0x99358f28,0x7f2830c0),LL(0x665e6c09,0x2e60a95e), + LL(0x9b785dbf,0x08407d3d),LL(0xa759bce7,0x530889ab),LL(0x52f61239,0xf228e0e6),LL(0x6879be3c,0x2b6d1461), LL(0x51a7bbf7,0xe6902c04),LL(0x76f24a64,0x30ad99f0),LL(0x98bc6da0,0x66d9317a),LL(0xcb596ac0,0xf4f877f3), + LL(0x4c44f119,0xb05ff62d),LL(0xe9b77416,0x4555f536),LL(0x8caed63b,0xc7c0d059),LL(0xc358b2a9,0x0cd2b7ce), LL(0x46945fa3,0x3f33287b),LL(0xd67c8791,0xf8785b20),LL(0x9637bd08,0xc54a7a61),LL(0x18be79d7,0x54d4598c), + LL(0xc46d7ce1,0x889e5acb),LL(0x8b085877,0x9a515bb7),LL(0x0b7a5050,0xfac1a03d),LL(0xf2926035,0x7d3e738a), LL(0x2a6cb0eb,0x861cc2ce),LL(0x8f7adc79,0x6f2e2955),LL(0x33016376,0x61c4d451),LL(0x5ad59090,0xd9fd2c80), + LL(0xb2b836a1,0xe5a83738),LL(0x7c0d6622,0x855b41a0),LL(0x7cc19af1,0x186fe317),LL(0xfdd99acb,0x6465c1ff), LL(0x6974b99e,0x46e5c23f),LL(0xa2717cbe,0x75a7cf8b),LL(0x062be658,0x4d2ebc3f),LL(0x5f209c98,0x094b4447), + LL(0xb940cb5a,0x4af285ed),LL(0x7cc82f10,0x6706d792),LL(0x030526fa,0xc8c8776c),LL(0xa0da9140,0xfa8e6f76), LL(0x591ee4f0,0x77ea9d34),LL(0x40274166,0x5f46e337),LL(0xea671457,0x1bdf98bb),LL(0x862a1fe2,0xd7c08b46), + LL(0x1c08ad63,0x46cc303c),LL(0x4c845e7b,0x99543440),LL(0x48f36bf7,0x1b8fbdb5),LL(0x8c8273a7,0x5b82c392), LL(0x928435d5,0x08f712c4),LL(0x79330380,0x071cf0f1),LL(0xa8da054a,0xc74c2d24),LL(0x43c46b5c,0xcb0e7201), + LL(0xc0b7eff3,0x0ad7337a),LL(0xc5e48b3c,0x8552225e),LL(0x73f13a5f,0xe6f78b0c),LL(0x82349cbe,0x5e70062e), LL(0xe7073969,0x6b8d5048),LL(0xc33cb3d2,0x392d2a29),LL(0x4ecaa20f,0xee4f727c),LL(0x2ccde707,0xa068c99e), + LL(0xb87a2913,0xfcd5651f),LL(0x3cc252f0,0xea3e3c15),LL(0x3b6cd3e4,0x777d92df),LL(0xc5a732e7,0x7a414143), LL(0xa71ff493,0xa895951a),LL(0xbbd37cf6,0xfe980c92),LL(0xdecfeeff,0x45bd5e64),LL(0xa44c43e9,0x910dc2a9), + LL(0xcca9f54d,0xcb403f26),LL(0x9303f6db,0x928bbdfb),LL(0xa9eee67c,0x3c37951e),LL(0xf79961c3,0x3bd61a52), LL(0x395c9a79,0x09a238e6),LL(0x61eb352d,0x6940ca2d),LL(0xc1875631,0x7d1e5c5e),LL(0x1e1b20d1,0x1e19742c), + LL(0x23fc2e6e,0x4633d908),LL(0x08959149,0xa76e29a9),LL(0x84ed7da5,0x61069d9c),LL(0x5dbcad51,0x0baa11cf), LL(0x961849da,0xd01eec64),LL(0xaf3d8c28,0x93b75f1f),LL(0x1ca2ee44,0x57bc4f9f),LL(0x00e00558,0x5a26322d), + LL(0x61a023ef,0x1888d658),LL(0xb9e5246e,0x1d72aab4),LL(0xe5563ec0,0xa9a26348),LL(0xc3439a43,0xa0971963), LL(0xadb9b5b7,0x567dd54b),LL(0xc45a524b,0x73fac1a1),LL(0xfe38e608,0x8fe97ef7),LL(0x3f384f48,0x608748d2), + LL(0xc486094f,0xb0571794),LL(0x8bf3a8d6,0x869254a3),LL(0x310b0e25,0x148a8dd1),LL(0x9aa3f7d8,0x99ab9f3f), LL(0x6706c02e,0x0927c68a),LL(0x69790e6c,0x22b5e76c),LL(0x6c71376c,0x6c325260),LL(0x09ef6657,0x53a57690), + LL(0xedffcf3a,0x8d63f852),LL(0x3c0a6f55,0xb4d2ed04),LL(0x12519b9e,0xdb3aa8de),LL(0x1e0a569a,0x5d38e9c4), LL(0x303747e2,0x871528bf),LL(0xf5b5c18d,0xa208e77c),LL(0xca6bf923,0x9d129c88),LL(0xbf02839f,0xbcbf197f), + LL(0x27323194,0x9b9bf030),LL(0x339ca59d,0x3b055a8b),LL(0x0f669520,0xb46b2312),LL(0x497e5f24,0x19789f1f), LL(0xaaf01801,0x9c499468),LL(0x8b69d59c,0x72ee1190),LL(0xacf4c079,0x8bd39595),LL(0x8e0cd048,0x3ee11ece), + LL(0x1ed66f18,0xebde86ec),LL(0xd61fce43,0x225d906b),LL(0xe8bed74d,0x5cab07d6),LL(0x27855ab7,0x16e4617f), LL(0xb2fbc3dd,0x6568aadd),LL(0x8aeddf5b,0xedb5484f),LL(0x6dcf2fad,0x878f20e8),LL(0x615f5699,0x3516497c), +}, +/* digit=20 base_pwr=2^140 */ +{ + LL(0xfa181e69,0xef0a3fec),LL(0x30d69a98,0x9ea02f81),LL(0x66eab95d,0xb2e9cf8e),LL(0x24720021,0x520f2beb), LL(0x1df84361,0x621c540a),LL(0x71fa6d5d,0x12037721),LL(0x0ff5f6ff,0x6e3c7b51),LL(0xabb2bef3,0x817a069b), + LL(0xb294cda6,0x83572fb6),LL(0xb9039f34,0x6ce9bf75),LL(0x095cbb21,0x20e012f0),LL(0xd063f0da,0xa0aecc1b), LL(0xf02909e5,0x57c21c3a),LL(0x48ce9cdc,0xc7d59ecf),LL(0x8ae336f8,0x2732b844),LL(0x3f4f85f4,0x056e3723), + LL(0x89e800ca,0x8a10b531),LL(0x145208fd,0x50fe0c17),LL(0xb714ba37,0x9e43c0d3),LL(0x34189acc,0x427d200e), LL(0xe616e2c0,0x05dee24f),LL(0xee1854c1,0x9c25f4c8),LL(0x8f342a73,0x4d3222a5),LL(0xa027c952,0x0807804f), + LL(0x4f0d56f3,0xc222653a),LL(0xca28b805,0x961e4047),LL(0x4a73434b,0x2c03f8b0),LL(0xab712a19,0x4c966787), LL(0x864fee42,0xcc196c42),LL(0x5b0ece5c,0xc1be93da),LL(0xc131c159,0xa87d9f22),LL(0xdce45655,0x2bb6d593), + LL(0xb809b7ce,0x22c49ec9),LL(0xe2c72c2c,0x8a41486b),LL(0xfea0bf36,0x813b9420),LL(0xa66dac69,0xb3d36ee9), LL(0x328cc987,0x6fddc08a),LL(0x3a326461,0x0a3bcd2c),LL(0xd810dbba,0x7103c49d),LL(0x4b78a4c4,0xf9d81a28), + LL(0xe4d55941,0x3de865ad),LL(0x30384087,0xdedafa5e),LL(0x4ef18b9b,0x6f414abb),LL(0xfaee5268,0x9ee9ea42), LL(0x37a55a4a,0x260faa16),LL(0x015f93b9,0xeb19a514),LL(0x9e9c3598,0x51d7ebd2),LL(0x1932178e,0x523fc56d), + LL(0xb98fe684,0x501d070c),LL(0x124a1458,0xd60fbe9a),LL(0x92bc6b3f,0xa45761c8),LL(0xfe6f27cb,0xf5384858), LL(0xb59e763b,0x4b0271f7),LL(0x5b5a8e5e,0x3d4606a9),LL(0x05a48292,0x1eda5d9b),LL(0xe6fec446,0xda7731d0), + LL(0x90d45871,0xa3e33693),LL(0x06166d8d,0xe9764040),LL(0x89a90403,0xb5c33682),LL(0x72f1d637,0x4bd17983), LL(0xd5d2c53a,0xa616679e),LL(0xfdcf3b87,0x5ec4bcd8),LL(0xb66a694e,0xae6d7613),LL(0xe3fc27e5,0x7460fc76), + LL(0x95caabee,0x70469b82),LL(0x889501e3,0xde024ca5),LL(0x076ed265,0x6bdadc06),LL(0x5a0ef8b2,0x0cb1236b), LL(0x0972ebf9,0x4065ddbf),LL(0x22aca432,0xf1dd3875),LL(0x744aff76,0xa88b97cf),LL(0xfe8e3d24,0xd1359afd), + LL(0x91502cf3,0x52a3ba2b),LL(0x084db75d,0x2c3832a8),LL(0xde30b1c9,0x04a12ddd),LL(0xe31fd60c,0x7802eabc), LL(0xa37fddab,0x33707327),LL(0xfaafa973,0x65d6f2ab),LL(0x11e6f91a,0x3525c5b8),LL(0x5f46530b,0x76aeb0c9), + LL(0x2f93a675,0xe8815ff6),LL(0x05f48679,0xa6ec9684),LL(0x358ae884,0x6dcbb556),LL(0xe19e3873,0x0af61472), LL(0xa5f696be,0x72334372),LL(0x6f22fb70,0xc65e57ea),LL(0x946cea90,0x268da30c),LL(0x65681b2a,0x136a8a87), + LL(0x0f9f44d4,0xad5e81dc),LL(0x2c46585a,0xf09a6960),LL(0xc447d1b1,0xd1649164),LL(0x879dc8b1,0x3b4b36c8), LL(0x3b6b234c,0x20d4177b),LL(0x1730d9d0,0x096a2505),LL(0xef80531d,0x0611b9b8),LL(0x64bb495d,0xba904b3b), + LL(0x93a3147a,0x1192d9d4),LL(0x9a565545,0x9f30a5dc),LL(0x6ef07212,0x90b1f9cb),LL(0x0d87fc13,0x29958546), LL(0xc17db9ba,0xd3323eff),LL(0xcb1644a8,0xcb18548c),LL(0x4f49ffbc,0x18a306d4),LL(0x4c2e8684,0x28d658f1), + LL(0xa99f8c71,0x44ba60cd),LL(0x4bf742ff,0x67b7abdb),LL(0x914b3f99,0x66310f9c),LL(0xf412c161,0xae430a32), LL(0x88ace52f,0x1e6776d3),LL(0x52d7067d,0x4bc0fa24),LL(0x8f07cd1b,0x03c286aa),LL(0xa985b2c1,0x4cb8f38c), + LL(0x8c3bff36,0x83ccbe80),LL(0x5263e575,0x005a0bd2),LL(0x259bdcd1,0x460d7dda),LL(0xfa5cab6b,0x4a1c5642), LL(0x9fe4fc88,0x2b7bdbb9),LL(0xcc97bbb5,0x09418e28),LL(0xa12321ae,0xd8274fb4),LL(0x5c87b64e,0xb137007d), + LL(0xc63c4962,0x80531fe1),LL(0x981fdb25,0x50541e89),LL(0xfd4c2b6b,0xdc1291a1),LL(0xa6df4fca,0xc0693a17), LL(0x0117f203,0xb2c4604e),LL(0x0a99b8d0,0x245f1963),LL(0xc6212c44,0xaedc20aa),LL(0x520f52a8,0xb1ed4e56), + LL(0xf8547be3,0xfe48f575),LL(0xa9e45f98,0x0a7033cd),LL(0x18c50100,0x4b45d3a9),LL(0xa61d41da,0xb2a6cd6a), LL(0x57933c6b,0x60bbb4f5),LL(0x2b0d7ffc,0xa7538ebd),LL(0x8cd626b6,0x9ea3ab8d),LL(0x3601625a,0x8273a484), + LL(0x0168e508,0x88859845),LL(0x99a94abd,0x8cbc9bb2),LL(0xfab0a671,0x713ac792),LL(0x6c9ebffc,0xa3995b19), LL(0x1239e152,0xe711668e),LL(0xbbb8dff4,0x56892558),LL(0xdbf17963,0x8bfc7dab),LL(0xb3de1253,0x5b59fe5a), + LL(0x34a9f7ae,0x7e3320eb),LL(0xd751efe4,0xe5e8cf72),LL(0xd9be2f37,0x7ea003bc),LL(0xb6c08ef7,0xc0f551a0), LL(0x038f6725,0x56606268),LL(0x6d92d3b6,0x1dd38e35),LL(0xc3cbd686,0x07dfce7c),LL(0x651c5da8,0x4e549e04), + LL(0x08b19340,0x4058f93b),LL(0xcac6d89d,0xc2fae6f4),LL(0x8f159cc7,0x4bad8a8c),LL(0xcb0b601c,0x0ddba4b3), LL(0x1dd95f8c,0xda4fc7b5),LL(0xcea5c255,0x1d163cd7),LL(0x274a8c4c,0x30707d06),LL(0x2802e9ce,0x79d9e008), + LL(0xe6ddd505,0x02a29ebf),LL(0xb50bed1a,0x37064e74),LL(0xa7327d57,0x3f6bae65),LL(0xf83920bc,0x3846f5f1), LL(0x60df1b9b,0x87c37491),LL(0x2d1da29f,0x4cfb2895),LL(0x4ed1743c,0x10a478ca),LL(0x3edd47c6,0x390c6030), + LL(0x8c0a78de,0x8f3e5312),LL(0x1e85df70,0xccd02bda),LL(0xa61b6582,0xd6c75c03),LL(0xfc0eebd1,0x0762921c), LL(0xd85010c0,0xd34d0823),LL(0x0044cf1f,0xd73aaacb),LL(0xa3b5e78a,0xfb4159bb),LL(0xe5826f3f,0x2287c7f7), + LL(0x580b1a01,0x4aeaf742),LL(0x60423b79,0xf080415d),LL(0xa7dea144,0xe12622cd),LL(0x59d62472,0x49ea4996), LL(0x571f3913,0xb42991ef),LL(0xf5b25a8a,0x0610f214),LL(0x30b79e8f,0x47adc585),LL(0x07a065a2,0xf90e3df6), + LL(0x43e2e034,0x5d0a5deb),LL(0x444024aa,0x53fb5a34),LL(0x6b0c9f7f,0xa8628c68),LL(0xac563656,0x9c69c29c), LL(0xbace47b6,0x5a231feb),LL(0x9ea5a2ec,0xbdce0289),LL(0x9463853e,0x05da1fac),LL(0x509e78aa,0x96812c52), + LL(0x57151692,0xd3fb5771),LL(0xd98e1c44,0xeb2721f8),LL(0x32399be1,0xc0506087),LL(0xd979d8b8,0xda5a5511), LL(0xc6f56780,0x737ed55d),LL(0x0dc7a7f4,0xe20d3004),LL(0xf5941a03,0x02ce7301),LL(0xed30f83a,0x91ef5215), + LL(0x4092d85f,0x28727fc1),LL(0x5c49e41a,0x72d223c6),LL(0xba6a4d81,0xa7cf30a2),LL(0xb030d87d,0x7c086209), LL(0xfc588b09,0x04844c7d),LL(0x5874bbb0,0x728cd499),LL(0xe84c0495,0xcc1281ee),LL(0xec31958f,0x0769b5ba), + LL(0xf99c2471,0x665c228b),LL(0x191eb110,0xf2d8a11b),LL(0xd36d7024,0x4594f494),LL(0xcdcb25a1,0x482ded8b), LL(0xdadd4885,0xc958a9d8),LL(0xf1d2b547,0x7004477e),LL(0x2a0af550,0x0a45f6ef),LL(0x2f8d6351,0x4fc739d6), + LL(0x786f08a9,0x75cdaf27),LL(0x42c2737f,0x8700bb26),LL(0x1c4e2670,0x855a7141),LL(0x15076fef,0x810188c1), LL(0xabcd3297,0xc251d0c9),LL(0xf48108eb,0xae4c8967),LL(0x18ceed30,0xbd146de7),LL(0xc986bced,0xf9d4f07a), + LL(0x83fa1e08,0x5ad98ed5),LL(0xbeabd1fb,0x7780d33e),LL(0x903b1196,0xe330513c),LL(0xa47bc8c4,0xba11de9e), LL(0x02c2d064,0x684334da),LL(0xa48de23b,0x7ecf360d),LL(0x0a9089d8,0x57a1b474),LL(0xff36734c,0xf28fa439), + LL(0xea4570b3,0xf2a482cb),LL(0xa5ebcee9,0xee65d68b),LL(0xb9694cd5,0x988d0036),LL(0x37885d32,0x53edd0e9), LL(0xbeb9bc6d,0xe37e3307),LL(0x9f5c6768,0xe9abb907),LL(0x51f2160f,0x4396ccd5),LL(0x47336da6,0x2500888c), + LL(0x926fce43,0x383f9ed9),LL(0x04da2930,0x809dd1c7),LL(0x8a4cb227,0x30f6f596),LL(0x73a56b38,0x0d700c7f), LL(0xab64a065,0x1825ea33),LL(0x1338df80,0xaab9b735),LL(0x9b63f57f,0x1516100d),LL(0x27a6a634,0x2574395a), + LL(0x700a1acd,0xb5560fb6),LL(0xfd999681,0xe823fd73),LL(0x6cb4e1ba,0xda915d1f),LL(0x6ebe00a3,0x0d030118), LL(0x89fca8cd,0x744fb0c9),LL(0xf9da0e0b,0x970d01db),LL(0x7931d76f,0x0ad8c564),LL(0xf659b96a,0xb15737bf), + LL(0xa8b484e7,0xdc9933e8),LL(0x7a26dec7,0xb2fdbdf9),LL(0x9f1f0136,0x2349e9a4),LL(0x70fddddb,0x7860368e), LL(0xf9ad3e18,0xd93d2c1c),LL(0x689f4e79,0x6d6c5f17),LL(0xb24ff1b6,0x7a544d91),LL(0xfe16cd8c,0x3e12a5eb), + LL(0xa56b872f,0x543574e9),LL(0xfcf68ea2,0xa1ad550c),LL(0x3f560ef7,0x689e37d2),LL(0xc9d47a8b,0x8c54b9ca), LL(0x088ac342,0x46d40a4a),LL(0x1576c6d0,0xec450c7c),LL(0x1f9689e9,0xb589e31c),LL(0xb8781718,0xdacf2602), + LL(0xc8cb6b42,0xa89237c6),LL(0xb96ef381,0x1326fc93),LL(0xb5f07825,0x55d56c6d),LL(0x7449e22d,0xacba2eea), LL(0x633c3000,0x74e0887a),LL(0xd7cbcf71,0xcb6cd172),LL(0xc36cf1be,0x309e81de),LL(0x60ae399b,0x07a18a6d), + LL(0x9edce57e,0xb36c2679),LL(0xdf001d41,0x52b892f4),LL(0x16a1f2c6,0xd884ae5d),LL(0xefcc370a,0x9b329424), LL(0xbd2e21df,0x3120daf2),LL(0x02470a99,0x55298d2d),LL(0xa05db32e,0x0b78af6c),LL(0x601f5636,0x5c76a331), + LL(0xf8a4f29c,0xaae861ff),LL(0xd68f8d49,0x70dc9240),LL(0x81b1321c,0x960e649f),LL(0x8792e4ce,0x3d2c801b), LL(0x42521876,0xf479f772),LL(0x416c79b1,0x0bed93bc),LL(0x263e5bc9,0xa67fbc05),LL(0x521db049,0x01e8e630), + LL(0xc6f3431e,0x76f26738),LL(0xe3267541,0xe609cb02),LL(0x818c877c,0xb10cff2d),LL(0x786a13cb,0x1f0e75ce), LL(0x1158544d,0xf4fdca64),LL(0x6cb71ed0,0x5d777e89),LL(0xa9aa4755,0x3c233737),LL(0xe527ab40,0x7b453192), + LL(0x39f05ffe,0xdb59f688),LL(0x6d82574e,0x8f4f4be0),LL(0xee292d1b,0xcce3450c),LL(0x61ccd086,0xaa448a12), LL(0xf7914967,0xabce91b3),LL(0x1908a5ed,0x4537f09b),LL(0xf51042e7,0xa812421e),LL(0xec0b3a34,0xfaf5cebc), + LL(0x4ca6b39a,0x730ffd87),LL(0x02efd342,0x70fb72ed),LL(0xd75c8edb,0xeb4735f9),LL(0xc278aa51,0xc11f2157), LL(0xbf3bfebf,0xc459f635),LL(0x6bd9601f,0x3a1ff0b4),LL(0xc420cb73,0xc9d12823),LL(0x3c2915a3,0x3e9af3e2), + LL(0xb41c3440,0xe0c82c72),LL(0xe3039a5f,0x175239e5),LL(0x558795a3,0xe1084b8a),LL(0xd01e5c60,0x328d0a1d), LL(0xd3788a04,0x0a495f2e),LL(0x66c11a9f,0x25d8ff16),LL(0x9ed692d6,0xf5155f05),LL(0x4f425fe4,0x954fa107), + LL(0xe98aaa99,0xd16aabf2),LL(0x96b0f88a,0x90cd8ba0),LL(0xc154026a,0x957f4782),LL(0x52af56d2,0x54ee0734), LL(0x45b4147a,0xbcf89e54),LL(0x9a52816c,0x3d102f21),LL(0x39b62e77,0x6808517e),LL(0x69169ad8,0x92e25421), + LL(0xbb608558,0xd721d871),LL(0xf6d4ff9b,0x60e4ebae),LL(0x41f2763e,0x0ba10819),LL(0x51ee3247,0xca2e45be), LL(0x2bfd7a5f,0x66d172ec),LL(0x74d0b12d,0x528a8f2f),LL(0xdabe70dc,0xe17f1e38),LL(0x9f93983c,0x1d5d7316), + LL(0xdf423e31,0x51b2184a),LL(0xaedb1a10,0xcb417291),LL(0x625bcab9,0x2054ca93),LL(0xa98998f0,0x54396860), LL(0xa54ae57e,0x4e53f6c4),LL(0xee648e9d,0x0ffeb590),LL(0x6afaf6bc,0xfbbdaadc),LL(0xaa3bfb8a,0xf88ae796), + LL(0xd2359ed9,0x209f1d44),LL(0xf3544ce2,0xac68dd03),LL(0xfd51e569,0xf378da47),LL(0x2cc80097,0xe1abd860), LL(0x343b6e3a,0x23ca18d9),LL(0xb40a1bae,0x480797e8),LL(0x533f3e67,0xd1f0c717),LL(0x06e6cdfc,0x44896970), + LL(0x52a82e8d,0x8ca21055),LL(0x78460cdc,0xb2caf785),LL(0xe9037178,0x4c1b7b62),LL(0xdb514b58,0xefc09d2c), LL(0x9113be5c,0x5f2df9ee),LL(0xb3f9271c,0x2fbda78f),LL(0x8f83fc54,0xe09a81af),LL(0x8afb5141,0x06b13866), + LL(0x43e3865d,0x38f6480f),LL(0x1ddf47d9,0x72dd77a8),LL(0x4c205ff7,0xf2a8e971),LL(0x9d088ad8,0x46d449d8), LL(0x185d706f,0x926619ea),LL(0xc7dd7f62,0xe47e02eb),LL(0x8cbc2031,0xe7f120a7),LL(0x998d4ac9,0xc18bef00), + LL(0x6bdf22da,0x18f37a9c),LL(0x90dc82df,0xefbc432f),LL(0x5d703651,0xc52cef8e),LL(0xd99881a5,0x82887ba0), LL(0xb920ec1d,0x7cec9dda),LL(0xec3e8d3b,0xd0d7e8c3),LL(0x4ca88747,0x445bc395),LL(0x9fd53535,0xedeaa2e0), + LL(0x6cc87475,0x461b1d93),LL(0x6d2383bd,0xd92a52e2),LL(0xd7903546,0xfabccb59),LL(0x3d14b112,0x6111a761), LL(0xb3d5f612,0x0ae584fe),LL(0x60e828ec,0x5ea69b8d),LL(0x54087030,0x6c078985),LL(0xac4821fe,0x649cab04), + LL(0x8bdce214,0x25ecedcf),LL(0x86af7361,0xb5622f72),LL(0x7038b9e2,0x0e1227aa),LL(0xac20fa77,0xd0efb273), LL(0x79df975b,0x817ff88b),LL(0x1999503e,0x856bf286),LL(0x5038ec46,0xb4d5351f),LL(0xfc42af6e,0x740a52c5), + LL(0x2cbb1a3f,0x2e38bb15),LL(0x17a83429,0xc3eb99fe),LL(0xdd66bb74,0xca4fcbf1),LL(0xcde5e8fc,0x880784d6), LL(0xb4e7a0be,0xddc84c1c),LL(0xbd15a72f,0x8780510d),LL(0x81ec30e1,0x44bcf1af),LL(0x0a61073e,0x141e50a8), + LL(0x47be87ae,0x0d955718),LL(0xf76a4372,0x68a61417),LL(0xc607c3d3,0xf57e7e87),LL(0x5252f332,0x043afaf8), LL(0x1552a4d2,0xcc14e121),LL(0xbb4d4ab4,0xb6dee692),LL(0xa03816a4,0xb6ab74c8),LL(0x6f394a29,0x84001ae4), + LL(0xd795fb45,0x5bed8344),LL(0xb79f55a5,0x57326e7d),LL(0x4accdffc,0xc9533ce0),LL(0x3993fa04,0x53473caf), LL(0xa13df4c8,0x7906eb93),LL(0x97cbe46f,0xa73e51f6),LL(0x0ae4ccf8,0xd1ab3ae1),LL(0x8a5b3dbc,0x25614508), + LL(0x11a71b27,0x61eff962),LL(0x6bb7fa39,0xdf71412b),LL(0x2bd7f3ef,0xb31ba6b8),LL(0x69180d29,0xb0b9c415), LL(0x014cdde5,0xeec14552),LL(0x227b4bbb,0x702c624b),LL(0xd3e988f3,0x2b15e8c2),LL(0xa4f7fd04,0xee3bcc6d), + LL(0x42ac6c85,0x9d00822a),LL(0x1df9f2b7,0x2db0cea6),LL(0x42de1e58,0xd7cad2ab),LL(0x2d6fbb61,0x346ed526), LL(0x1a2faf09,0xb3962995),LL(0x7c25612e,0x2fa8a580),LL(0x7cf56490,0x30ae04da),LL(0x0eea3961,0x75662908), + LL(0x3d080847,0x3609f5c5),LL(0x5241d4f6,0xcb081d39),LL(0x77961a63,0xb4fb3810),LL(0x2abb66fc,0xc20c5984), LL(0xf902f245,0x3d40aa7c),LL(0x4e536b1e,0x9cb12736),LL(0x99b3134f,0x5eda24da),LL(0x5cd011af,0xafbd9c69), + LL(0xc7088c7d,0x9a16e30a),LL(0x3207389f,0x5ab65710),LL(0xe7407a53,0x1b09547f),LL(0x4fdc6eab,0x2322f9d7), LL(0x7430de4d,0xc0f2f22d),LL(0xe68ca9a9,0x19382696),LL(0x918e5868,0x17f1eff1),LL(0x586f4204,0xe3b5b635), + LL(0x3fbc4341,0x146ef980),LL(0x5b5eed4e,0x359f2c80),LL(0x7482e41d,0x9f35744e),LL(0xf3b224c2,0x9a9ac3ec), LL(0x91fc50ae,0x9161a6fe),LL(0xc613fa7c,0x89ccc66b),LL(0xc732f15a,0x89268b14),LL(0xb467ed03,0x7cd6f4e2), + LL(0xce56b40e,0xfbf79869),LL(0xc02dde98,0xf93e094c),LL(0xedee2cd7,0xefe0c3a8),LL(0xb268fd42,0x90f3ffc0), LL(0x08241aed,0x81a7fd56),LL(0x00b1afe8,0x95ab7ad8),LL(0x3e310d52,0x40127056),LL(0x09d9fc43,0xd3ffdeb1), + LL(0xd11a8594,0xc8f85c91),LL(0x31cf6db8,0x2e74d258),LL(0x02b5dfd0,0x829c7ca3),LL(0x69143c86,0xe389cfbe), LL(0x941768d8,0xd01b6405),LL(0x03bf825d,0x45103995),LL(0x56cd17e2,0xcc4ee166),LL(0xba037e79,0xbea3c283), + LL(0xd9a47520,0x4e1ac06e),LL(0xaf852404,0xfbfe18aa),LL(0x8087648a,0x5615f8e2),LL(0xb9d150d9,0x7301e47e), LL(0xb299b977,0x79f9f9dd),LL(0xa5b78314,0x76697a7b),LL(0x7d7c90e7,0x10d67468),LL(0x937210b5,0x7afffe03), + LL(0x28c22cee,0x5aef3e4b),LL(0x09fd55ae,0xefb0ecd8),LL(0x0d2a5d6a,0x4cea7132),LL(0x01db6357,0x9cfb5fa1), LL(0xf36e1ac5,0x395e0b57),LL(0x36cafb7d,0x008fa9ad),LL(0x5308c4db,0x8f6cdf70),LL(0x95ed2477,0x51527a37), + LL(0x5bd21311,0xba0dee30),LL(0x909c90d7,0x6ed41b22),LL(0x7c8696d3,0xc5f6b758),LL(0x3ce83a80,0x0db8eaa8), LL(0xb24b4b6f,0xd297fe37),LL(0x522d1f0d,0xfe58afe8),LL(0x8c98dbd9,0x97358736),LL(0x9454a527,0x6bc226ca), + LL(0xce53c2d0,0xa12b384e),LL(0x5e4606da,0x779d897d),LL(0x73ec12b0,0xa53e47b0),LL(0x5756f1ad,0x462dbbba), LL(0xcafe37b6,0x69fe09f2),LL(0xecce2e17,0x273d1ebf),LL(0x3cf607fd,0x8ac1d538),LL(0x12e10c25,0x8035f7ff), +}, +/* digit=21 base_pwr=2^147 */ +{ + LL(0x7e6c5520,0x854d34c7),LL(0xdcb9ea58,0xc27df9ef),LL(0xd686666d,0x405f2369),LL(0x0417aa85,0x29d1febf), LL(0x93470afe,0x9846819e),LL(0xe2a27f9e,0x3e6a9669),LL(0xe31e6504,0x24d008a2),LL(0x9cb7680a,0xdba7cecf), + LL(0x338d6e43,0xecaff541),LL(0x4541d5cc,0x56f7dd73),LL(0x96bc88ca,0xb5d426de),LL(0x9ed3a2c3,0x48d94f6b), LL(0x2ef8279c,0x6354a3bb),LL(0x0b1867f2,0xd575465b),LL(0x95225151,0xef99b0ff),LL(0xf94500d8,0xf3e19d88), + LL(0xe32dd620,0x92a83268),LL(0x627849a2,0x913ec99f),LL(0x2c378882,0xedd8fdfa),LL(0xee6f8cfe,0xaf96f33e), LL(0xdc3fa8a5,0xc06737e5),LL(0xb0b03a1d,0x236bb531),LL(0x89f037b0,0x33e59f29),LL(0xd9a12a53,0x13f9b5a7), + LL(0x51efb310,0x0d0df6ce),LL(0x958df5be,0xcb5b2eb4),LL(0x36158e59,0xd6459e29),LL(0x1466e336,0x82aae2b9), LL(0x411aa636,0xfb658a39),LL(0xd4c0a933,0x7152ecc5),LL(0x49f026b7,0xf10c758a),LL(0xcb09311f,0xf4837f97), + LL(0xc753c45f,0xddfb02c4),LL(0xf9c840fe,0x18ca81b6),LL(0xb0f8a3e6,0x846fd09a),LL(0xe7733dbc,0xb1162add), LL(0x236e3ab6,0x7070ad20),LL(0xb2a56326,0xf88cdaf5),LL(0x997cbc7a,0x05fc8719),LL(0x4b665272,0x442cd452), + LL(0xb71698f5,0x7807f364),LL(0x9f7b605e,0x6ba418d2),LL(0xa03b2cbb,0xfd20b00f),LL(0xda54386f,0x883eca37), LL(0xf3437f24,0xff0be43f),LL(0xa48bb33c,0xe910b432),LL(0x329df765,0x4963a128),LL(0xbe2fe6f7,0xac1dd556), + LL(0x24a0a3fc,0x557610f9),LL(0xe881c3f9,0x38e17bf4),LL(0xed0dac99,0x6ba84faf),LL(0x59eeb918,0xd4a222c3), LL(0x13f542b6,0xc79c1dbe),LL(0xe425d457,0x1fc65e0d),LL(0x1debb779,0xeffb754f),LL(0x9e08af60,0x638d8fd0), + LL(0x626332d5,0x994f523a),LL(0x5561bb44,0x7bc38833),LL(0x3d845ea2,0x005ed4b0),LL(0xc2a1f08a,0xd39d3ee1), LL(0xe7676b0d,0x6561fdd3),LL(0xfb706017,0x620e35ff),LL(0xf264f9a8,0x36ce424f),LL(0xda2681f7,0xc4c3419f), + LL(0x69beb6e8,0xfb6afd2f),LL(0x6d700d03,0x3a50b993),LL(0x0c83a14f,0xc840b2ad),LL(0x54085bef,0x573207be), LL(0x09fe7e5b,0x5af882e3),LL(0x3b40a7e1,0x957678a4),LL(0x543056e2,0x172d4bdd),LL(0x0df13c0a,0x9c1b26b4), + LL(0xf405ff06,0x1c30861c),LL(0x486e828b,0xebac86bd),LL(0x636933fc,0xe791a971),LL(0x7aeee947,0x50e7c2be), LL(0xfa90d767,0xc3d4a095),LL(0xe670ab7b,0xae60eb7b),LL(0x397b056d,0x17633a64),LL(0x105012aa,0x93a21f33), + LL(0xabb88643,0x663c370b),LL(0x22e21599,0x91df36d7),LL(0x8b761671,0x183ba835),LL(0x728f3bf1,0x381eea1d), LL(0x39966e6c,0xb9b2f1ba),LL(0xe7295492,0x7c464a28),LL(0x09b26b7f,0x0fd5f70a),LL(0xfbe009df,0xa9aba1f9), + LL(0x369b87ad,0x857c1f22),LL(0x32fca556,0x3c00e5d9),LL(0x90b06466,0x1ad74cab),LL(0x550faaf2,0xa7112386), LL(0x6d9bd5f5,0x7435e198),LL(0x59c3463f,0x2dcc7e38),LL(0xca7bd4b2,0xdc7df748),LL(0x9dec2f31,0x13cd4c08), + LL(0xe3237710,0x0d3b5df8),LL(0xcbd2f7b0,0x0dadb26e),LL(0xe4aa082b,0x9f5966ab),LL(0x350e966e,0x666ec8de), LL(0xee524216,0x1bfd1ed5),LL(0x41dab0b6,0xcd93c59b),LL(0xd186d6ba,0x658a8435),LL(0x159d1195,0x1b7d34d2), + LL(0x22caf46b,0x5936e460),LL(0x9a96fe4f,0x6a45dd8f),LL(0xb98f474e,0xf7925434),LL(0x0053ef15,0x41410412), LL(0x41de97bf,0x71cf8d12),LL(0xbd80bef4,0xb8547b61),LL(0xc4db0037,0xb47d3970),LL(0xfef20dff,0xf1bcd328), + LL(0x10caad67,0x31a92e09),LL(0x5531a1e1,0x1f591960),LL(0x5f4fc840,0x3bb852e0),LL(0x93a72c6c,0x63e297ca), LL(0x49abad67,0x3c2b0b2e),LL(0xed3db0d9,0x6ec405fc),LL(0x7fef1d40,0xdc14a530),LL(0x280896fc,0xccd19846), + LL(0x9bb81648,0x00f83176),LL(0x653120d0,0xd69eb485),LL(0x4ccabc62,0xd17d75f4),LL(0xb749fcb1,0x34a07f82), LL(0xbbfb5554,0x2c3af787),LL(0x62e283f8,0xb06ed4d0),LL(0xa19213a0,0x5722889f),LL(0xdcf3c7b4,0x162b085e), + LL(0xe0dd3eca,0xbcaecb31),LL(0xe52f13a5,0xc6237fbc),LL(0x27bac297,0xcc2b6b03),LL(0xb917f54a,0x2ae1cac5), LL(0x7845ae4f,0x474807d4),LL(0xce5972e0,0xfec7dd92),LL(0x1d7915bb,0xc3bd2541),LL(0xd94907ca,0x66f85dc4), + LL(0xbdbcf0ca,0xd981b888),LL(0xdf279e9f,0xd75f5da6),LL(0x7054e934,0x128bbf24),LL(0x81db134b,0x3c6ff6e5), LL(0x047d26e4,0x795b7cf4),LL(0x5049ec37,0xf370f7b8),LL(0xced945af,0xc6712d4d),LL(0x095642bc,0xdf30b5ec), + LL(0x4896246e,0x9b034c62),LL(0xee90bbd1,0x5652c016),LL(0x87fedb73,0xeb38636f),LL(0x0135a613,0x5e32f847), LL(0xcf933c83,0x0703b312),LL(0x1a7f47e6,0xd05bb76e),LL(0x949c2415,0x825e4f0c),LL(0x7250d6f8,0x569e5622), + LL(0x6568013e,0xbbe9eb3a),LL(0x22f243fc,0x8dbd203f),LL(0xb342734a,0x9dbd7694),LL(0x46afa984,0x8f6d12f8), LL(0xc9eade29,0xb98610a2),LL(0x47dd0f18,0xbab4f323),LL(0x671c0d46,0x5779737b),LL(0xd3e0a42a,0x10b6a7c6), + LL(0x3035b41c,0xfb19ddf3),LL(0x99c45895,0xd336343f),LL(0x54c857e5,0x61fe4938),LL(0xae4e57d5,0xc4d506be), LL(0xbbc33f75,0x3cd8c8cb),LL(0x9262c77d,0x7281f08a),LL(0xf11a2823,0x083f4ea6),LL(0x9fba2e33,0x8895041e), + LL(0x9c438edf,0xfcdfea49),LL(0x91edba44,0x7678dcc3),LL(0xe2ba50f0,0xf07b3b87),LL(0x43948c1b,0xc13888ef), LL(0x1140af42,0xc2135ad4),LL(0x926ed1a7,0x8e5104f3),LL(0x88f6695f,0xf24430cb),LL(0x6d73c120,0x0ce0637b), + LL(0xfe631e8f,0xb2db01e6),LL(0xd7bdd24b,0x1c5563d7),LL(0x369ad44f,0x8daea3ba),LL(0x8187a9f9,0x000c81b6), LL(0xaae1fd9a,0x5f48a951),LL(0x8d5aed8a,0xe35626c7),LL(0x0498c622,0x20952763),LL(0x773aa504,0x76d17634), + LL(0xeb300f7a,0x36d90dda),LL(0xedb5e801,0x9dcf7dfc),LL(0x74d5244c,0x645cb268),LL(0x348e3aa2,0xa127ee79), LL(0x575f1dbb,0x488acc53),LL(0x80e6161e,0x95037e85),LL(0x292650d0,0x57e59283),LL(0x14938216,0xabe67d99), + LL(0x3f8e1065,0x3c7f944b),LL(0x330e8924,0xed908cb6),LL(0x6f530136,0x08ee8fd5),LL(0xd7ffc169,0x2227b7d5), LL(0xb5cd6dd5,0x4f55c893),LL(0xa62796e8,0x82225e11),LL(0xcb18e12c,0x5c6cead1),LL(0x84f5a51a,0x4381ae0c), + LL(0x7fafa4c8,0x345913d3),LL(0x0491aac0,0x3d918082),LL(0x3e69264c,0x9347871f),LL(0xb4f4f0cd,0xbea9dd3c), LL(0x3eadd3e7,0xbda5d067),LL(0x0573bcd8,0x0033c1b8),LL(0x5da2486c,0x25589379),LL(0x86abbee7,0xcb89ee5b), + LL(0x22532e5d,0x8fe0a8f3),LL(0x727dfc4c,0xb6410ff0),LL(0x226726db,0x619b9d58),LL(0x7a2b2dc7,0x5ec25669), LL(0x4c3beb01,0xaf4d2e06),LL(0x7acea556,0x852123d0),LL(0xf783487a,0x0e9470fa),LL(0x5664b3eb,0x75a7ea04), + LL(0x6798e4ba,0x4ad78f35),LL(0xc7d0e091,0x9214e6e5),LL(0xb1290403,0xc420b488),LL(0xfc295749,0x64049e0a), LL(0x3ae9841f,0x03ef5af1),LL(0xb0b662a6,0xdbe4ca19),LL(0xfa453458,0x46845c5f),LL(0x10b66722,0xf8dabf19), + LL(0xcce2793b,0xb650f0aa),LL(0xc5ec47c1,0x71db851e),LL(0x3b234fa9,0x3eb78f3e),LL(0xfc0106ce,0xb0c60f35), LL(0x774eadbd,0x05427121),LL(0xce323863,0x25367faf),LL(0xcd086976,0x7541b5c9),LL(0xdc507ad1,0x4ff069e2), + LL(0x8776e667,0x74145256),LL(0xb23c6bb5,0x6e76142c),LL(0x1b3a8a87,0xdbf30712),LL(0x98450836,0x60e7363e), LL(0xb7366d80,0x5741450e),LL(0x4837dbdf,0xe4ee14ca),LL(0x69d4316f,0xa765eb9b),LL(0x8ef43825,0x04548dca), + LL(0x5ae888eb,0x9c9f4e4c),LL(0x56e9ac99,0x733abb51),LL(0xba6ac029,0xdaad3c20),LL(0x2ba3e38e,0x9b8dd3d3), LL(0x0bc5d11a,0xa9bb4c92),LL(0x9c5f88a3,0xf20127a7),LL(0x161d3cb8,0x4f52b06e),LL(0x6afaf0a6,0x26c1ff09), + LL(0x7189e71f,0x32670d2f),LL(0x5ecf91e7,0xc6438748),LL(0xdb757a21,0x15758e57),LL(0x290a9ce5,0x427d09f8), LL(0x38384a7a,0x846a308f),LL(0xb0732b99,0xaac3acb4),LL(0x17845819,0x9e941009),LL(0xa7ce5e03,0x95cba111), + LL(0xb00009c4,0x6f3d4f7f),LL(0x8ff28b5f,0xb8396c27),LL(0x1c97975d,0xb1a9ae43),LL(0xe5d9fed5,0x9d7ba8af), LL(0x34f485b6,0x338cf09f),LL(0x64122516,0xbc0ddacc),LL(0x05d471fe,0xa450da12),LL(0x628dd8c9,0x4c3a6250), + LL(0xd1295837,0x69c7d103),LL(0x3807eb2f,0xa2893e50),LL(0xbdb41491,0xd6e1e1de),LL(0x5e138235,0xc630745b), LL(0x48661ae1,0xc892109e),LL(0xea2b2674,0x8d17e7eb),LL(0xc328d6b5,0x00ec0f87),LL(0xf079ff9e,0x6d858645), + LL(0x19115ead,0x6cdf243e),LL(0x4bac4fcf,0x1ce1393e),LL(0x9c29f25b,0x2c960ed0),LL(0x9d388a05,0x59be4d8e), LL(0xd0def72b,0x0d46e06c),LL(0xe0342748,0xb923db5d),LL(0x936d4a3d,0xf7d3aacd),LL(0x0b0b099e,0x558519cc), + LL(0x827097ef,0x3ea8ebf8),LL(0xd054f55d,0x259353db),LL(0x6d2ed089,0x84c89abc),LL(0x8e096a7c,0x5c548b69), LL(0x994b995d,0xd587f616),LL(0xa5845601,0x4d1531f6),LL(0x451fd9f0,0x792ab31e),LL(0x65adf6ca,0xc8b57bb2), + LL(0x1cd5ad73,0x68440fcb),LL(0x6144da4f,0xb9c860e6),LL(0x8462beb8,0x2ab286aa),LL(0xef46797f,0xcc6b8fff), LL(0x20c8a471,0xac820da4),LL(0x77ff7faf,0x69ae05a1),LL(0xbfb5da77,0xb9163f39),LL(0x2c73ab7a,0xbd03e590), + LL(0xb2940d9e,0x7e862b5e),LL(0x4b9af564,0x3c663d86),LL(0xbde3033d,0xd8309031),LL(0xd42c5bc6,0x298231b2), LL(0x552ad093,0x42090d2c),LL(0xff854695,0xa4799d1c),LL(0xd31f0d00,0x0a88b5d6),LL(0xa2f26b46,0xf8b40825), + LL(0xf1bd7218,0xec29b1ed),LL(0x4b24c86e,0xd491c53b),LL(0x3395ea65,0xd2fe588f),LL(0x4456ef15,0x6f3764f7), LL(0xcdc34800,0xdb43116d),LL(0xc1e33955,0xcdbcd456),LL(0x74ab286b,0xefdb5540),LL(0xd18c5d7c,0x948c7a51), + LL(0x7378058e,0xeb81aa37),LL(0x04411154,0x41c746a1),LL(0xfb828ac7,0xa10c73bc),LL(0x9d972b29,0x6439be91), LL(0x43a2fbad,0x4bf3b4b0),LL(0x82b5e840,0x39e6dadf),LL(0x6397bd4c,0x4f716408),LL(0x7f1eeccb,0x0f7de568), + LL(0xd2ffbfc1,0x5865c5a1),LL(0x4ccb6451,0xf74211fa),LL(0xc0b32558,0x66368a88),LL(0x9ad7812e,0x5b539dc2), LL(0x2f3af6f6,0x579483d0),LL(0x99934ece,0x52132078),LL(0xdcc9e983,0x50b9650f),LL(0xaee42b8a,0xca989ec9), + LL(0xd6f62f99,0x6a44c829),LL(0x4c2a7c0c,0x8f06a309),LL(0x98a0cb0a,0x4ea2b3a0),LL(0xbeee8364,0x5c547b70), LL(0x682afe11,0x461d40e1),LL(0x7b41c0a8,0x9e0fc77a),LL(0xe20d5d36,0x79e4aefd),LL(0x32dd9f63,0x2916e520), + LL(0x3f883faf,0xf59e52e8),LL(0x2b868d35,0x396f9639),LL(0x4ca19881,0xc902a9df),LL(0xdb2401a6,0x0fc96822), LL(0x66f1c68d,0x41237587),LL(0xfb476c0d,0x10fc6de3),LL(0x841f5d90,0xf8b6b579),LL(0xfa24f44a,0x2ba8446c), + LL(0xef4a9975,0xa237b920),LL(0x2330435f,0x60bb6004),LL(0xcfb7e7b5,0xd6f4ab5a),LL(0x83435391,0xb2ac5097), LL(0xb0d1ea67,0xf036ee2f),LL(0x74c56230,0xae779a6a),LL(0xab838ae6,0x59bff8c8),LL(0x9b38e6f0,0xcd83ca99), + LL(0xe33deed3,0xbb27bef5),LL(0x001892a8,0xe6356f6f),LL(0x7adfbd3e,0xbf3be6cc),LL(0x33d1ac9d,0xaecbc81c), LL(0xe6e861dc,0xe4feb909),LL(0x53f5f801,0x90a247a4),LL(0x27346e57,0x01c50acb),LL(0x461acc1b,0xce29242e), + LL(0x2f998a91,0x04dd214a),LL(0xd4baf27b,0x271ee9b1),LL(0xe8c26722,0x7e3027d1),LL(0x1820dce5,0x21d1645c), LL(0x7501779c,0x086f242c),LL(0xfa0e8009,0xf0061407),LL(0x60187129,0xf23ce477),LL(0x0fde9bd0,0x05bbdedb), + LL(0x25d98473,0x682f4832),LL(0x5c658427,0xf207fe85),LL(0x4166ffa1,0xb6fdd7ba),LL(0x9eed799d,0x0c314056), LL(0x4107e28f,0x0db8048f),LL(0x41216840,0x74ed3871),LL(0x56a3c06e,0x74489f8f),LL(0x12777134,0x1e1c005b), + LL(0xf37ec3c3,0xdb332a73),LL(0xdd59eba0,0xc65259bd),LL(0xdb4d3257,0x2291709c),LL(0xbd389390,0x9a793b25), LL(0xe43756f0,0xf39fe34b),LL(0x9afb56c9,0x2f76bdce),LL(0x61208b27,0x9f37867a),LL(0x089972c3,0xea1d4307), + LL(0x8bdf623a,0x8c595330),LL(0x8441fb7d,0x5f5accda),LL(0x32ddfd95,0xfafa9418),LL(0x0fde9be7,0x6ad40c5a), LL(0xaeca8709,0x43faba89),LL(0x2c248a9d,0xc64a7cf1),LL(0x72637a76,0x16620252),LL(0x22b8d1bb,0xaee1c791), + LL(0x21a843b2,0xf0f798fd),LL(0x8d005cb1,0x56e4ed4d),LL(0x1f0d8abe,0x355f7780),LL(0x34522326,0x197b04cf), LL(0xfd42c13f,0x41f9b31f),LL(0xb40f933d,0x5ef7feb2),LL(0x5d60bad4,0x27326f42),LL(0x8c92cf89,0x027ecdb2), + LL(0x4e3352fe,0x04aae4d1),LL(0x73591b90,0x08414d2f),LL(0xb7da7d60,0x5ed6124e),LL(0x4d13d4ec,0xb985b931), LL(0x96bf36f9,0xa592d3ab),LL(0xbbdf51df,0x012dbed5),LL(0xdf6c177d,0xa57963c0),LL(0x87ca29cf,0x010ec869), + LL(0xbf926dff,0xba1700f6),LL(0xf4bf6bc2,0x7c9fdbd1),LL(0x64da11f5,0xdc18dc8f),LL(0xd938ae75,0xa6074b7a), LL(0xe84f44a4,0x14270066),LL(0xd27b954e,0x99998d38),LL(0xb4f38e9a,0xc1be8ab2),LL(0x15c01016,0x8bb55bbf), + LL(0x0ea2ab30,0xf73472b4),LL(0xf73d68dd,0xd365a340),LL(0x19c2e1eb,0xc01a7168),LL(0x34061719,0x32f49e37), LL(0x01d8b4d6,0xb73c57f1),LL(0x26b47700,0x03c8423c),LL(0xa4d8826a,0x321d0bc8),LL(0x4bc0e638,0x6004213c), + LL(0xc1c06681,0xf78c64a1),LL(0xef018e50,0x16e0a16f),LL(0xdb42b2b3,0x31cbdf91),LL(0xe0d36f58,0xf8f4ffce), LL(0x4cc5e3e0,0xcdcc71cd),LL(0xa129e3e0,0xd55c7cfa),LL(0x0fb2cbf1,0xccdb6ba0),LL(0xc4bce3cb,0x6aba0005), + LL(0xd232cfc4,0x501cdb30),LL(0xd58a3cef,0x9ddcf12e),LL(0x87e09149,0x02d2cf9c),LL(0x2c976257,0xdc5d7ec7), LL(0x0b50d7dd,0x6447986e),LL(0x807f112a,0x88fdbaf7),LL(0xb00ae9f6,0x58c9822a),LL(0x6d3d27e0,0x6abfb950), + LL(0x8a429f4f,0xd0a74487),LL(0xdb516609,0x0649712b),LL(0xe769b5df,0xb826ba57),LL(0x1fc7aaf2,0x82335df2), LL(0x5c93d995,0x2389f067),LL(0x68677be6,0x59ac367a),LL(0x21d9951b,0xa77985ff),LL(0x85011cce,0x038956fb), + LL(0xbb734e37,0x608e48cb),LL(0x2be5b26f,0xc08c0bf2),LL(0xf9b1a0d9,0x17bbdd3b),LL(0x10483319,0xeac7d898), LL(0xbc1a6dea,0xc95c4baf),LL(0x172aafdb,0xfdd0e2bf),LL(0x8235c41a,0x40373cbc),LL(0xfb6f41d5,0x14303f21), + LL(0x0408f237,0xba063621),LL(0xecd2d1ed,0xcad3b09a),LL(0x52abb6a2,0x4667855a),LL(0xaa8b417b,0xba9157dc), LL(0x4f013efb,0xfe7f3507),LL(0xaa38c4a2,0x1b112c4b),LL(0x9ba64345,0xa1406a60),LL(0x6993c80b,0xe53cba33), + LL(0xded40d23,0x45466063),LL(0x54908e25,0x3d5f1f4d),LL(0x403c3c31,0x9ebefe62),LL(0x0672a624,0x274ea0b5), LL(0x451d1b71,0xff818d99),LL(0x8f79cf79,0x80e82643),LL(0x73ce37f5,0xa165df13),LL(0xfe3a21fd,0xa744ef4f), + LL(0xcf551396,0x73f1e7f5),LL(0x868c676b,0xc616898e),LL(0x8c442c36,0x671c28c7),LL(0x5e0a317d,0xcfe5e558), LL(0x7051f476,0x1242d818),LL(0x14f03442,0x56fad2a6),LL(0x0a44d0f6,0x262068bc),LL(0xce6edf4e,0xdfa2cd6e), + LL(0xd15d1517,0x0f43813a),LL(0x377d44f5,0x61214cb2),LL(0xc639b35f,0xd399aa29),LL(0x54c51c19,0x42136d71), LL(0x08417221,0x9774711b),LL(0x52545a57,0x0a5546b3),LL(0x1150582d,0x80624c41),LL(0xfbc555bc,0x9ec5c418), + LL(0x771849f1,0x2c87dcad),LL(0x01d7bf6f,0xb0c932c5),LL(0x89116eb2,0x6aa5cd3e),LL(0x51ca7bd3,0xd378c25a), LL(0x9e6e3e31,0xc612a0da),LL(0xb68ad5d0,0x0417a54d),LL(0x22c6edb8,0x00451e4a),LL(0xb42827ce,0x9fbfe019), + LL(0xba9384a2,0x2fa92505),LL(0x64ad69c1,0x21b8596e),LL(0x983b35a6,0x8f4fcc49),LL(0x72754672,0xde093760), LL(0xf7bffe6d,0x2f14ccc8),LL(0x5d94263d,0x27566bff),LL(0x2df3ec30,0xb5b4e9c6),LL(0x3e6ea6ba,0x94f1d7d5), + LL(0xaaca5e9b,0x97b7851a),LL(0x56713b97,0x518aa521),LL(0x150a61f6,0x3357e8c7),LL(0xec2c2b69,0x7842e7e2), LL(0x6868a548,0x8dffaf65),LL(0xe068fc81,0xd963bd82),LL(0x65917733,0x64da5c8b),LL(0x7b247328,0x927090ff), +}, +/* digit=22 base_pwr=2^154 */ +{ + LL(0xd298c241,0x214bc9a7),LL(0x56807cfd,0xe3b697ba),LL(0x4564eadb,0xef1c7802),LL(0xb48149c5,0xdde8cdcf), LL(0x5a4d2604,0x946bf0a7),LL(0x6c1538af,0x27154d7f),LL(0xde5b1fcc,0x95cc9230),LL(0x66864f82,0xd88519e9), + LL(0x7cb1282c,0xb828dd1a),LL(0xbe46973a,0xa08d7626),LL(0xe708d6b2,0x6baf8d40),LL(0x4daeb3f3,0x72571fa1), LL(0xf22dfd98,0x85b1732f),LL(0x0087108d,0x87ab01a7),LL(0x5988207a,0xaaaafea8),LL(0x69f00755,0xccc832f8), + LL(0x36ff3bf0,0x964d950e),LL(0xf0b34638,0x8ad20f6f),LL(0xb5d7585f,0x4d9177b3),LL(0xef3f019f,0xcf839760), LL(0x8288c545,0x582fc5b3),LL(0x13116bd1,0x2f8e4e9b),LL(0x332120ef,0xf91e1b2f),LL(0x2a17dd23,0xcf568724), + LL(0xca8d9d1a,0x488f1185),LL(0xd987ded2,0xadf2c77d),LL(0x60c46124,0x5f3039f0),LL(0x71e095f4,0xe5d70b75), LL(0x6260e70f,0x82d58650),LL(0xf750d105,0x39d75ea7),LL(0x75bac364,0x8cf3d0b1),LL(0x21d01329,0xf3a7564d), + LL(0x2f52d2a7,0x182f04cd),LL(0xe2df565a,0x4fde149a),LL(0xa79fb2f7,0xb80c5eec),LL(0x22ddc897,0xab491d7b), LL(0xc6312c7f,0x99d76c18),LL(0x6aa41a57,0xca0d5f3d),LL(0xd15363a0,0x71207325),LL(0xbeb252c2,0xe82aa265), + LL(0xec3128c2,0x94ab4700),LL(0x8e383f49,0x6c76d862),LL(0xc03024eb,0xdc36b150),LL(0x53daac69,0xfb439477), LL(0x8dc79623,0xfc68764a),LL(0xb440fbb2,0x5b86995d),LL(0xccc5ee0d,0xd66879bf),LL(0x95aa8bd3,0x05228942), + LL(0x1e6a75c1,0xb51a40a5),LL(0x0ea7d817,0x24327c76),LL(0x07774597,0x06630182),LL(0x97fa7164,0xd6fdbec3), LL(0x13c90f48,0x20c99dfb),LL(0x686ef263,0xd6ac5273),LL(0xfef64eeb,0xc6a50bdc),LL(0x86fdfc32,0xcd87b281), + LL(0x3fcd3efc,0xb24aa43e),LL(0xb8088e9a,0xdd26c034),LL(0xbd3d46ea,0xa5ef4dc9),LL(0x8a4c6a6f,0xa2f99d58), LL(0x2f1da46c,0xddabd355),LL(0x1afacdd1,0x72c3f8ce),LL(0x92d40578,0xd90c4eee),LL(0xca623b94,0xd28bb41f), + LL(0x745edc11,0x50fc0711),LL(0x3dc87558,0x9dd9ad7d),LL(0xb49d1e64,0xce6931fb),LL(0xc98bd0f9,0x6c77a0a2), LL(0x6baf7cb1,0x62b9a629),LL(0xccf72d22,0xcf065f91),LL(0x79639071,0x7203cce9),LL(0xf9cb732f,0x09ae4885), + LL(0xee8314f3,0x5e7c3bec),LL(0xdbea298f,0x1c068aed),LL(0x7c80acec,0x08d381f1),LL(0xe330495b,0x03b56be8), LL(0x9222882d,0xaeffb8f2),LL(0xc4af8bf7,0x95ff38f6),LL(0x1fc57d8c,0x50e32d35),LL(0x17b444f0,0x6635be52), + LL(0xa5177900,0x04d15276),LL(0xf6858752,0x4e1dbb47),LL(0xc615796c,0x5b475622),LL(0x691867bf,0xa6fa0387), LL(0x2844c6d0,0xed7f5d56),LL(0x03a2477d,0xc633cf9b),LL(0x2d3721d6,0xf6be5c40),LL(0xe9fd68e6,0xaf312eb7), + LL(0xe7417ce1,0x242792d2),LL(0x970ee7f5,0xff42bc71),LL(0x5c67a41e,0x1ff4dc6d),LL(0x20882a58,0x77709b7b), LL(0xbe217f2c,0x3554731d),LL(0x5bb72177,0x2af2a8cd),LL(0x591dd059,0x58eee769),LL(0x4bba6477,0xbb2930c9), + LL(0x7d930cfc,0x863ee047),LL(0x396fd1f4,0x4c262ad1),LL(0x039af7e1,0xf4765bc8),LL(0x5ba104f6,0x2519834b), LL(0xd105f961,0x7cd61b4c),LL(0xd63bca54,0xa5415da5),LL(0x88a1f17c,0x778280a0),LL(0x2329512c,0xc4968949), + LL(0xcecdaa7a,0x174a9126),LL(0x0b13247b,0xfc8c7e0e),LL(0x3484c1c4,0x29c110d2),LL(0x831dfc3b,0xf8eb8757), LL(0xc0067452,0x022f0212),LL(0x7b9b926c,0x3f6f69ee),LL(0xef42daf4,0x09032da0),LL(0x83f80de4,0x79f00ade), + LL(0x81236c97,0x6210db71),LL(0x3ee0781f,0x74f7685b),LL(0xa3e41372,0x4df7da7b),LL(0xb1a1553e,0x2aae38b1), LL(0xf6dd9d1b,0x1688e222),LL(0x5b8b6487,0x57695448),LL(0x4b2edeaa,0x478d2127),LL(0x1e85956a,0xb2818fa5), + LL(0xf176f2c0,0x1e6addda),LL(0xe2572658,0x01ca4604),LL(0x85342ffb,0x0a404ded),LL(0x441838d6,0x8cf60f96), LL(0xc9071c4a,0x9bbc691c),LL(0x34442803,0xfd588744),LL(0x809c0d81,0x97101c85),LL(0x8c456f7f,0xa7fb754c), + LL(0xd51805e1,0xc95f3c5c),LL(0xb299dca8,0xab4ccd39),LL(0x47eaf500,0x3e03d20b),LL(0xd7b80893,0xfa3165c1), LL(0xe160e552,0x005e8b54),LL(0x9019d11f,0xdc4972ba),LL(0x0c9a4a7a,0x21a6972e),LL(0x37840fd7,0xa52c258f), + LL(0xc1e99d81,0xf8559ff4),LL(0xa3c617c0,0x08e1a7d6),LL(0x248c6ba7,0xb398fd43),LL(0xd1283794,0x6ffedd91), LL(0xd629d208,0x8a6a59d2),LL(0x3490530e,0xa9d141d5),LL(0x38505989,0x42f6fc18),LL(0x479d94ee,0x09bf250d), + LL(0xb3822790,0x223ad3b1),LL(0x93b8971c,0x6c5926c0),LL(0x75f7fa62,0x609efc7e),LL(0x1ec2d989,0x45d66a6d), LL(0x987d2792,0x4422d663),LL(0x3eb31d2b,0x4a73caad),LL(0xa32cb9e6,0xf06c2ac1),LL(0x91aeba84,0xd9445c5f), + LL(0xaf71013f,0x6af7a1d5),LL(0x0bedc946,0xe68216e5),LL(0xd27370a0,0xf4cba30b),LL(0x870421cc,0x7981afbf), LL(0x9449f0e1,0x02496a67),LL(0x0a47edae,0x86cfc4be),LL(0xb1feca22,0x3073c936),LL(0x03f8f8fb,0xf5694612), + LL(0x901515ea,0xd063b723),LL(0x749cf038,0x4c6c77a5),LL(0xab9e5059,0x6361e360),LL(0xa76a37c0,0x596cf171), LL(0x6530ae7a,0x800f53fa),LL(0x0792a7a6,0x0f5e631e),LL(0xefdb81c9,0x5cc29c24),LL(0x3f9c40ba,0xa269e868), + LL(0x2cb7191e,0xec14f9e1),LL(0xe5b08ea6,0x78ea1bd8),LL(0x46332bb9,0x3c65aa9b),LL(0xbf80ce25,0x84cc22b3), LL(0xd49d5bf1,0x0098e9e9),LL(0x19087da4,0xcd4ec1c6),LL(0xaef6e357,0x3c9d07c5),LL(0x9f8f64b8,0x839a0268), + LL(0xc6d8607f,0xc5e9eb62),LL(0x6aa995e4,0x759689f5),LL(0xbbb48317,0x70464669),LL(0xe402417d,0x921474bf), LL(0x2a354c8c,0xcabe135b),LL(0x812fa4b5,0xd51e52d2),LL(0x53311fe8,0xec741096),LL(0xb864514b,0x4f774535), + LL(0x5bde48f8,0xbcadd671),LL(0x2189bc7d,0xc9703873),LL(0xc709ee8a,0x5d45299e),LL(0x845aaff8,0xd1287ee2), LL(0xdb1dbf1f,0x7d1f8874),LL(0x990c88d6,0xea46588b),LL(0x84368313,0x60ba649a),LL(0x60d543ae,0xd5fdcbce), + LL(0x810d5ab0,0x90b46d43),LL(0x04d7e5cc,0x6739d8f9),LL(0x0d337c33,0x021c1a58),LL(0x68e67c40,0x00a61162), LL(0x379f0a1f,0x95ef413b),LL(0xe9e2ab95,0xfe126605),LL(0x2f5f199c,0x67578b85),LL(0x2cb84913,0xf5c00329), + LL(0x37577dd8,0xf7956430),LL(0x29c5fe88,0x83b82af4),LL(0xcdbdc132,0x9c1bea26),LL(0x9c04339e,0x589fa086), LL(0xb13799df,0x033e9538),LL(0xd295d034,0x85fa8b21),LL(0xbd9ddcca,0xdf17f73f),LL(0xddb66334,0xf32bd122), + LL(0x858b044c,0x55ef88a7),LL(0x5aa9e397,0x1f0d69c2),LL(0x40d85559,0x55fd9cc3),LL(0x7785ddb2,0xc774df72), LL(0xd3bd2e1c,0x5dcce9f6),LL(0xa85dfed0,0xeb30da20),LL(0xd3ed09c4,0x5ed7f5bb),LL(0x82a9c1bd,0x7d42a35c), + LL(0x9890272d,0xcf3de995),LL(0x3e713a10,0x75f3432a),LL(0xe28227b8,0x5e13479f),LL(0xfefacdc8,0xb8561ea9), LL(0x8332aafd,0xa6a297a0),LL(0x73809b62,0x9b0d8bb5),LL(0x0c63036f,0xd2fa1cfd),LL(0xbd64bda8,0x7a16eb55), + LL(0x78e62ddc,0x3f5cf5f6),LL(0x07fd752b,0x2267c454),LL(0x5e437bbe,0x5e361b6b),LL(0x8354e075,0x95c59501), LL(0xf2b254d9,0xec725f85),LL(0x2cb52b4e,0x844b617d),LL(0xcf425fb5,0xed8554f5),LL(0x2af9f312,0xab67703e), + LL(0x3cf48283,0x4cc34ec1),LL(0x9c8a705e,0xb09daa25),LL(0x5b7d4f84,0xd1e9d0d0),LL(0xdb38929d,0x4df6ef64), LL(0xaa21ba46,0xe16b0763),LL(0xa293f8fb,0xc6b1d178),LL(0xd520aabf,0x0ff5b602),LL(0xc339397a,0x94d671bd), + LL(0x4f5792fa,0x7c7d98cf),LL(0x11215261,0x7c5e0d67),LL(0xa7c5a6d4,0x9b19a631),LL(0x7a45274d,0xc8511a62), LL(0xa5a60d99,0x0c16621c),LL(0xcf5e48cb,0xf7fbab88),LL(0xf7ddee08,0xab1e6ca2),LL(0xe7867f3c,0x83bd08ce), + LL(0x2ac13e27,0xf7e48e8a),LL(0x4eb1a9f5,0x4494f6df),LL(0x981f0a62,0xedbf84eb),LL(0x536438f0,0x49badc32), LL(0x004f7571,0x50bea541),LL(0xdf1c94ee,0xbac67d10),LL(0xb727bc31,0x253d73a1),LL(0x30686e28,0xb3d01cf2), + LL(0x55fd0b8b,0x51b77b1b),LL(0xfeec3173,0xa099d183),LL(0x670e72b7,0x202b1fb7),LL(0xa8e1635f,0xadc88b33), LL(0xf989d905,0x34e8216a),LL(0x29b58d01,0xc2e68d20),LL(0x6fe55a93,0x11f81c92),LL(0x8f296f40,0x15f1462a), + LL(0xea3d62f2,0x1915d375),LL(0x01c8977d,0xa17765a3),LL(0xe47b26f6,0x7559710a),LL(0x535077a5,0xe0bd29c8), LL(0x08d84858,0x615f976d),LL(0x69ced5c1,0x370dfe85),LL(0xa734fa56,0xbbc7503c),LL(0x91ac4574,0xfbb9f1ec), + LL(0x060dd7ef,0x95d7ec53),LL(0x6e657979,0xeef2dacd),LL(0xe2a08235,0x54511af3),LL(0x1f4aea3d,0x1e324aa4), LL(0xe6e67671,0x550e7e71),LL(0xbf52faf7,0xbccd5190),LL(0x223cc62a,0xf880d316),LL(0x2b32eb5d,0x0d402c7e), + LL(0x306a5a3b,0xa40bc039),LL(0x96783a1b,0x4e0a41fd),LL(0x0253cdd4,0xa1e8d39a),LL(0xc7388638,0x6480be26), LL(0x2285f382,0xee365e1d),LL(0xec0b5c36,0x188d8d8f),LL(0x1f0f4d82,0x34ef1a48),LL(0xa487d29a,0x1a8f43e1), + LL(0x77aefb3a,0x8168226d),LL(0x1e72c253,0xf69a751e),LL(0xe9594df1,0x8e04359a),LL(0xd14c0467,0x475ffd7d), LL(0x3844e95c,0xb5a2c2b1),LL(0xdd12ef94,0x85caf647),LL(0xf1063d00,0x1ecd2a9f),LL(0x23843311,0x1dd2e229), + LL(0x73d17244,0x38f0e09d),LL(0x8fc653f1,0x3ede7746),LL(0xdc20e21c,0xae4459f5),LL(0x6a8599ea,0x00db2ffa), LL(0x30cfd905,0x11682c39),LL(0xa5c112a6,0x4934d074),LL(0x568bfe95,0xbdf063c5),LL(0x016c441a,0x779a440a), + LL(0x97d6fbdc,0x0c23f218),LL(0xe0776aac,0xd3a5cd87),LL(0xd712e8db,0xcee37f72),LL(0x26f74e8d,0xfb28c70d), LL(0xb61301a0,0xffe0c728),LL(0xd3724354,0xa6282168),LL(0x768ffedc,0x7ff4cb00),LL(0x03b02de9,0xc51b3088), + LL(0x3902dda5,0xa5a8147c),LL(0xfe6973b4,0x35d2f706),LL(0xc257457e,0x5ac2efcf),LL(0x8700611b,0x933f48d4), LL(0x4912beb2,0xc365af88),LL(0x162edf94,0x7f5a4de6),LL(0x0c32f34b,0xc646ba7c),LL(0xb2091074,0x632c6af3), + LL(0x753e43a9,0x58d4f2e3),LL(0x24d4e23f,0x70e1d217),LL(0xafede6a6,0xb24bf729),LL(0x710c8b60,0x7f4a94d8), LL(0x8d4faa6a,0xaad90a96),LL(0xb066b690,0xd9ed0b32),LL(0x78b6dbfd,0x52fcd37b),LL(0x8bd2b431,0x0b64615e), + LL(0xcfb9fad5,0x228e2048),LL(0x240b76bd,0xbeaa386d),LL(0x90dad7bc,0x2d6681c8),LL(0x06d38f5e,0x3e553fc3), LL(0x9d5f9750,0xf27cdb9b),LL(0xd28c5b0e,0x3e85c52a),LL(0x5247c39b,0x190795af),LL(0xbddd6828,0x547831eb), + LL(0x4a82f424,0xf327a227),LL(0x7e47f89d,0x36919c78),LL(0x43c7392c,0xe4783919),LL(0x2316fefe,0xf101b9aa), LL(0x1c5009d2,0xbcdc9e9c),LL(0x9cd18345,0xfb55ea13),LL(0xa3ce77c7,0xf5b5e231),LL(0xd2f2cb3d,0xde6b4527), + LL(0x9bb26f5f,0x10f6a333),LL(0x044d85b6,0x1e85db8e),LL(0x94197e54,0xc3697a08),LL(0xa7cb4ea8,0x65e18cc0), LL(0xa471fe6e,0xa38c4f50),LL(0x2f13439c,0xf031747a),LL(0xc007318b,0x53c4a6ba),LL(0x1deccb3d,0xa8da3ee5), + LL(0x558216b1,0x0555b31c),LL(0x2f79e6c2,0x90c7810c),LL(0xfe8eed3c,0x9b669f4d),LL(0xe0fac126,0x70398ec8), LL(0xf701b235,0xa96a449e),LL(0xeb94f395,0x0ceecdb3),LL(0xd0cb7431,0x285fc368),LL(0x16a18c64,0x0d37bb52), + LL(0xb880d2dd,0x05110d38),LL(0x65930d57,0xa60f177b),LL(0xf36235f5,0x7da34a67),LL(0x183816b9,0x47f5e17c), LL(0xdb394af4,0xc7664b57),LL(0x7036f789,0x39ba215d),LL(0x2f27b472,0x46d2ca0e),LL(0xf73a84b7,0xc42647ee), + LL(0x64488f1d,0x44bc7545),LL(0xf4cf85d5,0xaa922708),LL(0x53e4df63,0x721a01d5),LL(0x5db46ced,0x649c0c51), LL(0x3cffcb6c,0x6bf0d64e),LL(0x50f71d96,0xe3bf93fe),LL(0xbcc194a0,0x75044558),LL(0x6afdc554,0x16ae3372), + LL(0x5ca48f3f,0xbfc01adf),LL(0xe22a9b84,0x64352f06),LL(0xc1099e4a,0xcee54da1),LL(0xfa1b89c0,0xbbda54e8), LL(0x6f6e55fb,0x166a3df5),LL(0x20176f88,0x1ca44a24),LL(0xdfb7b5ff,0x936afd88),LL(0x8611d4a0,0xe34c2437), + LL(0x86142103,0x7effbb75),LL(0x1f34fc4d,0x6704ba1b),LL(0x10c1b122,0x7c2a468f),LL(0x8c6aace9,0x36b3a610), LL(0x75a0d050,0xabfcc0a7),LL(0x3ce33e32,0x066f9197),LL(0x29fe09be,0xce905ef4),LL(0xa8376351,0x89ee25ba), + LL(0xfd29dc76,0x2a3ede22),LL(0x36f17260,0x7fd32ed9),LL(0x284b4126,0x0cadcf68),LL(0xa7951fc8,0x63422f08), LL(0x0807e199,0x562b24f4),LL(0x22ad4490,0xfe9ce5d1),LL(0x0db2b1b4,0xc2f51b10),LL(0xe4541d0d,0xeb3613ff), + LL(0x2680813b,0xbd2c4a05),LL(0x561b08d6,0x527aa55d),LL(0xa7205558,0xa9f8a40e),LL(0x243d0bec,0xe3eea56f), LL(0xa0ff58b3,0x7b853817),LL(0x1a69e627,0xb67d3f65),LL(0xa869b5d6,0x0b76bbb9),LL(0x546723ed,0xa3afeb82), + LL(0x3e554892,0x5f24416d),LL(0x430e2a45,0x8413b53d),LL(0x9032a2a0,0x99c56aee),LL(0xeec367b1,0x09432bf6), LL(0xdaf0ecc1,0x552850c6),LL(0x5bc92048,0x49ebce55),LL(0x54811307,0xdfb66ba6),LL(0x6f298597,0x1b84f797), + LL(0x8d1d7a0d,0x79590481),LL(0x3a6fa556,0xd9fabe03),LL(0xba9e5d35,0xa40f9c59),LL(0xf6247577,0xcb1771c1), LL(0xe9a6312b,0x542a47ca),LL(0x552dd8c5,0xa34b3560),LL(0x0d794716,0xfdf94de0),LL(0x9c623094,0xd46124a9), + LL(0x68afe8b4,0x56b7435d),LL(0x6c0d8ea1,0x27f20540),LL(0x73186898,0x12b77e14),LL(0x7479490f,0xdbc3dd46), LL(0xc03b0c05,0x951a9842),LL(0x7921bc96,0x8b1b3bb3),LL(0x2b202e0a,0xa573b346),LL(0x47254d56,0x77e4665d), + LL(0xd23e3984,0x08b70dfc),LL(0xebd14236,0xab86e8bc),LL(0x57114ba7,0xaa3e07f8),LL(0xab0ef4f2,0x5ac71689), LL(0x0139d9af,0x88fca384),LL(0x76644af0,0x72733f88),LL(0x65d74f4a,0xf122f72a),LL(0xa5626c7a,0x13931577), + LL(0x70f8d5a4,0xd5b5d9eb),LL(0xd7bbb228,0x375adde7),LL(0x0c1c0b32,0x31e88b86),LL(0x173edbaa,0xd1f568c4), LL(0x5459df02,0x1592fc83),LL(0x0fcd9a7e,0x2beac0fb),LL(0x1b473b0a,0xb0a6fdb8),LL(0x0fe8fc48,0xe3224c6f), + LL(0xe87edf5b,0x680bd00e),LL(0x20e77cf5,0x30385f02),LL(0x4d42d1b2,0xe9ab98c0),LL(0xd3816d77,0x72d191d2), LL(0x0917d9e5,0x1564daca),LL(0x1f8fed7f,0x394eab59),LL(0x7fbb3896,0xa209aa8d),LL(0xbe6ac98e,0x5564f3b9), + LL(0xd73654ef,0xead21d05),LL(0x13d78d74,0x68d1a9c4),LL(0x6d4973a0,0x61e01708),LL(0x46e6d32a,0x83da3500), LL(0x68ae0118,0x6a3dfca4),LL(0xd02da069,0xa1b9a4c9),LL(0xebab8302,0x0b2ff9c7),LL(0x944ba436,0x98af07c3), + LL(0x995f0f9f,0x85997326),LL(0x71b58bc6,0x467fade0),LL(0xbd625a2b,0x47e4495a),LL(0x33c3b8cd,0xfdd2d01d), LL(0xc693f9fa,0x2c38ae28),LL(0x348f7999,0x48622329),LL(0x2161f583,0x97bf738e),LL(0x565e8cc9,0x15ee2fa7), + LL(0x5777e189,0xa1a5c845),LL(0x456f2829,0xcc10bee0),LL(0xda762bd5,0x8ad95c56),LL(0xe9d91da8,0x152e2214), LL(0x7cb23c74,0x975b0e72),LL(0xa90c66df,0xfd5d7670),LL(0x225ffc53,0xb5b5b8ad),LL(0xfaded2ae,0xab6dff73), + LL(0x6f4cbe9d,0xebd56781),LL(0x6a574bd7,0x0ed8b249),LL(0x81a881fa,0x41c246fe),LL(0xc3db9c70,0x91564805), LL(0x5b862809,0xd7c12b08),LL(0x55858d7b,0x1facd1f1),LL(0xaf09e92a,0x7693747c),LL(0x189a425f,0x3b69dcba), + LL(0x967365ef,0x0be28e9f),LL(0xe801f5c9,0x57300eb2),LL(0xd583352f,0x93b8ac6a),LL(0xcd05b2b7,0xa2cf1f89), LL(0x4dcc40cc,0x7c0c9b74),LL(0xada523fb,0xfee38c45),LL(0x1099cc4d,0xb49a4dec),LL(0x69f069c6,0x325c377f), + LL(0x476cc9ff,0xe12458ce),LL(0xc6d4cb63,0x580e0b6c),LL(0x9072289b,0xd561c8b7),LL(0xa619e6da,0x0377f264), LL(0x88e591a5,0x26685362),LL(0x7523ca2b,0xa453a7bd),LL(0xc1df4533,0x8a9536d2),LL(0xbe972f79,0xc8e50f2f), + LL(0x6d3549cf,0xd433e50f),LL(0xfacd665e,0x6f33696f),LL(0xce11fcb4,0x695bfdac),LL(0xaf7c9860,0x810ee252), LL(0x7159bb2c,0x65450fe1),LL(0x758b357b,0xf7dfbebe),LL(0xd69fea72,0x2b057e74),LL(0x92731745,0xd485717a), +}, +/* digit=23 base_pwr=2^161 */ +{ + LL(0xee36860c,0x896c42e8),LL(0x4113c22d,0xdaf04dfd),LL(0x44104213,0x1adbb7b7),LL(0x1fd394ea,0xe5fd5fa1), LL(0x1a4e0551,0x68235d94),LL(0x18d10151,0x6772cfbe),LL(0x09984523,0x276071e3),LL(0x5a56ba98,0xe4e879de), + LL(0x285b9491,0xaaafafb0),LL(0x1e4c705e,0x01a0be88),LL(0x2ad9caab,0xff1d4f5d),LL(0xc37a233f,0x6e349a4a), LL(0x4a1c6a16,0xcf1c1246),LL(0x29383260,0xd99e6b66),LL(0x5f6d5471,0xea3d4366),LL(0xff8cc89b,0x36974d04), + LL(0xcfe89d80,0xc26c49a1),LL(0xda9c8371,0xb42c026d),LL(0xdad066d2,0xca6c013a),LL(0x56a4f3ee,0xfb8f7228), LL(0xd850935b,0x08b579ec),LL(0xd631e1b3,0x34c1a74c),LL(0xac198534,0xcb5fe596),LL(0xe1f24f25,0x39ff21f6), + LL(0x8f929057,0x27f29e14),LL(0xc0c853df,0x7a64ae06),LL(0x58e9c5ce,0x256cd183),LL(0xded092a5,0x9d9cce82), LL(0x6e93b7c7,0xcc6e5979),LL(0x31bb9e27,0xe1e47092),LL(0xaa9e29a0,0xb70b3083),LL(0x3785e644,0xbf181a75), + LL(0x8ead09f7,0xf53f2c65),LL(0x9780d14d,0x1335e1d5),LL(0xcd1b66bc,0x69cc20e0),LL(0xbbe0bfc8,0x9b670a37), LL(0x28efbeed,0xce53dc81),LL(0x8326a6e5,0x0c74e77c),LL(0xb88e9a63,0x3604e0d2),LL(0x13dc2248,0xbab38fca), + LL(0x5c0a3f1e,0x8ed6e8c8),LL(0x7c87c37f,0xbcad2492),LL(0x9ee3b78d,0xfdfb62bb),LL(0xcbceba46,0xeba8e477), LL(0xeeaede4b,0x37d38cb0),LL(0x7976deb6,0x0bc498e8),LL(0x6b6147fb,0xb2944c04),LL(0xf71f9609,0x8b123f35), + LL(0xde79dc24,0xa155dcc7),LL(0x558f69cd,0xf1168a32),LL(0x0d1850df,0xbac21595),LL(0xb204c848,0x15c8295b), LL(0x7d8184ff,0xf661aa36),LL(0x30447bdb,0xc396228e),LL(0xbde4a59e,0x11cd5143),LL(0x6beab5e6,0xe3a26e3b), + LL(0x1402b9d0,0xd3b3a13f),LL(0x2c7bc863,0x573441c3),LL(0x578c3e6e,0x4b301ec4),LL(0x0adaf57e,0xc26fc9c4), LL(0x7493cea3,0x96e71bfd),LL(0x1af81456,0xd05d4b3f),LL(0x6a8c608f,0xdaca2a8a),LL(0x0725b276,0x53ef07f6), + LL(0x7824fc56,0x07a5fbd2),LL(0x13289077,0x34675218),LL(0xe0c48349,0x5bf69fd5),LL(0xb6aa7875,0xa613ddd3), LL(0x5450d866,0x7f78c19c),LL(0x8f84a481,0x46f4409c),LL(0x90fce239,0x9f1d1928),LL(0xb2ce44b9,0x016c4168), + LL(0xc7435978,0xbae023f0),LL(0x20e30e19,0xb152c888),LL(0xe3fa6faf,0x9c241645),LL(0x84823e60,0x735d95c1), LL(0x03955317,0x03197573),LL(0xf03b4995,0x0b4b02a9),LL(0x70274600,0x076bf559),LL(0xaaf57508,0x32c5cc53), + LL(0x60624129,0xe8af6d1f),LL(0x9a5e2b5e,0xb7bc5d64),LL(0x5f082d72,0x3814b048),LL(0xce19677a,0x76f267f2), LL(0xb36eed93,0x626c630f),LL(0x3bf56803,0x55230cd7),LL(0xce2736a0,0x78837949),LL(0xaa6c55f1,0x0d792d60), + LL(0xd5c7c5d2,0x0318dbfd),LL(0x072b342d,0xb38f8da7),LL(0x7b8de38a,0x3569bddc),LL(0xa1c94842,0xf25b5887), LL(0x2946ad60,0xb2d5b284),LL(0xe9d1707e,0x854f29ad),LL(0x2c6a4509,0xaa5159dc),LL(0x57189837,0x899f94c0), + LL(0xf4a55b03,0xcf6adc51),LL(0x35e3b2d5,0x261762de),LL(0x04827b51,0x4cc43012),LL(0xc6021442,0xcd22a113), LL(0x247c9569,0xce2fd61a),LL(0xd152beca,0x59a50973),LL(0x63a716d4,0x6c835a11),LL(0x187dedcf,0xc26455ed), + LL(0x49ce89e7,0x27f536e0),LL(0xcc890cb5,0x18908539),LL(0xd83c2aa1,0x308909ab),LL(0x1ab73bd3,0xecd3142b), LL(0xb3f5ab84,0x6a85bf59),LL(0xf2bea4c6,0x3c320a68),LL(0x6da4541f,0xad8dc538),LL(0xb7c41186,0xeaf34eb0), + LL(0x977c97c4,0x1c780129),LL(0xc57eb9fa,0x5ff9beeb),LL(0xc822c478,0xa24d0524),LL(0x461cd415,0xfd8eec2a), LL(0xf027458c,0xfbde194e),LL(0x1d1be115,0xb4ff5319),LL(0x4866d6f4,0x63f874d9),LL(0xb21ad0c9,0x35c75015), + LL(0x46ac49d2,0xa6b5c9d6),LL(0x83137aa9,0x42c77c0b),LL(0x68225a38,0x24d000fc),LL(0x2fe1e907,0x0f63cfc8), LL(0xc6441f95,0x22d1b01b),LL(0xec8e448f,0x7d38f719),LL(0x787fb1ba,0x9b33fa5f),LL(0x190158df,0x94dcfda1), + LL(0x5f6d4a09,0xc47cb339),LL(0xee52b826,0x6b4f355c),LL(0xf51b930a,0x3d100f5d),LL(0x9f668f69,0xf4512fac), LL(0x206c4c74,0x546781d5),LL(0xcb4d2e48,0xd021d4d4),LL(0xca085c2d,0x494a54c2),LL(0x520850a8,0xf1dbaca4), + LL(0x490a1aca,0x63c79326),LL(0x41526b02,0xcb64dd9c),LL(0xa2979258,0xbb772591),LL(0x48d97846,0x3f582970), LL(0x7c213ba7,0xd66b70d1),LL(0xe8a0ced4,0xc28febb5),LL(0xc10338c1,0x6b911831),LL(0xbf0126f3,0x0d54e389), + LL(0x4af206ee,0x7048d460),LL(0x77e97cb9,0x786c88f6),LL(0xac64802e,0xd4375ae1),LL(0xd53ec11c,0x469bcfe1), LL(0x47062230,0xfc9b340d),LL(0xc5b4a3ac,0xe743bb57),LL(0x59ef45ac,0xfe00b4aa),LL(0x59edf188,0x29a4ef23), + LL(0xb483689b,0x40242efe),LL(0x513ac262,0x2575d3f6),LL(0x0ca6db72,0xf30037c8),LL(0x98864be2,0xc9fcce82), LL(0x0149362d,0x84a112ff),LL(0x1c4ae971,0x95e57582),LL(0x945cf86c,0x1fa4b1a8),LL(0x0b024a2f,0x4525a734), + LL(0x8f338360,0xe76c8b62),LL(0x28edf32b,0x483ff593),LL(0x298b1aec,0x67e8e90a),LL(0x736d9a21,0x9caab338), LL(0x66892709,0x5c09d2fd),LL(0xb55a1d41,0x2496b4dc),LL(0xe24a4394,0x93f5fb1a),LL(0x6fa8f6c1,0x08c75049), + LL(0xc905d85f,0xcaead1c2),LL(0x0733ae57,0xe9d7f790),LL(0xf07cdd94,0x24c9a65c),LL(0xa4b55931,0x7389359c), LL(0x367e45f7,0xf58709b7),LL(0xcb7e7adc,0x1f203067),LL(0xc7b72818,0x82444bff),LL(0xbaac8033,0x07303b35), + LL(0xd13b7ea1,0x1e1ee4e4),LL(0xe0e74180,0xe6489b24),LL(0x7e70ef70,0xa5f2c610),LL(0xbdd10894,0xa1655412), LL(0x7af4194e,0x555ebefb),LL(0x8e89bd9c,0x533c1c3c),LL(0x89895856,0x735b9b57),LL(0x567f5c15,0x15fb3cd2), + LL(0x526f09fd,0x057fed45),LL(0x8128240a,0xe8a4f10c),LL(0xff2bfd8d,0x9332efc4),LL(0xbd35aa31,0x214e77a0), LL(0x14faa40e,0x32896d73),LL(0x01e5f186,0x767867ec),LL(0x17a1813e,0xc9adf8f1),LL(0x54741795,0xcb6cda78), + LL(0x349d51aa,0xb7521b6d),LL(0xe3c7b8e9,0xf56b5a9e),LL(0x32a096df,0xc6f1e5c9),LL(0xa3635024,0x083667c4), LL(0x18087f2f,0x365ea135),LL(0xd136e45d,0xf1b8eaac),LL(0x73aec989,0xc8a0e484),LL(0x142c9259,0xd75a324b), + LL(0x01dae185,0xb7b4d001),LL(0x9b7a94bc,0x45434e0b),LL(0xfbd8cb0b,0xf54339af),LL(0xe98ef49e,0xdcc4569e), LL(0x09a51299,0x7789318a),LL(0xb2b025d8,0x81b4d206),LL(0xfae85792,0xf64aa418),LL(0xacd7baf7,0x3e50258f), + LL(0x2996864b,0xdce84cdb),LL(0x1f485fa4,0xa2e67089),LL(0x534c6a5a,0xb28b2bb6),LL(0xc94b9d39,0x31a7ec6b), LL(0xd6bc20da,0x1d217766),LL(0x86761190,0x4acdb5ec),LL(0x73701063,0x68726328),LL(0x2128c29b,0x4d24ee7c), + LL(0xa19fd868,0xc072ebd3),LL(0xdb8ddd3b,0x612e481c),LL(0x1a64d852,0xb4e1d754),LL(0xc4c6c4ab,0x00ef95ac), LL(0xaa0a6c46,0x1536d2ed),LL(0x43774790,0x61294086),LL(0x343fda10,0x54af25e8),LL(0xfd25d6f2,0x9ff9d98d), + LL(0x468b8835,0x0746af7c),LL(0x730ecea7,0x977a31cb),LL(0xc2cf4a81,0xa5096b80),LL(0x6458c37a,0xaa986833), LL(0xa6bd9d34,0x6af29bf3),LL(0x33c5d854,0x6a62fe9b),LL(0xb7133b5e,0x50e6c304),LL(0x7d6e6848,0x04b60159), + LL(0x5579bea4,0x4cd296df),LL(0x5ceedaf1,0x10e35ac8),LL(0xe3bcc5b1,0x04c4c5fd),LL(0x89412cf9,0x95f9ee8a), LL(0x82b6eb0f,0x2c9459ee),LL(0x95c2aadd,0x2e845765),LL(0xd327fcfe,0x774a84ae),LL(0x0368d476,0xd8c93722), + LL(0xf83e8a3b,0x0dbd5748),LL(0x8d2495f3,0xa579aa96),LL(0xae496e9b,0x535996a0),LL(0xb7f9bcc2,0x07afbfe9), LL(0x5b7bd293,0x3ac1dc6d),LL(0x7022323d,0x3b592cff),LL(0x9c0a3e76,0xba0deb98),LL(0x4b197acb,0x18e78e9f), + LL(0x296c36ef,0x211cde10),LL(0x82c4da77,0x7ee89672),LL(0xa57836da,0xb617d270),LL(0x9cb7560b,0xf0cd9c31), LL(0xe455fe90,0x01fdcbf7),LL(0x7e7334f3,0x3fb53cbb),LL(0x4e7de4ec,0x781e2ea4),LL(0x0b384fd0,0x8adab3ad), + LL(0x53d64829,0x129eee2f),LL(0xa261492b,0x7a471e17),LL(0xe4cb4a2c,0xe4f9adb9),LL(0x97ba2c2d,0x3d359f6f), LL(0x0aacd697,0x346c6786),LL(0x75c2f8a8,0x92b444c3),LL(0xd85df44e,0xc79fa117),LL(0x398ddf31,0x56782372), + LL(0xbbbab3b8,0x60e690f2),LL(0x8b04816b,0x4851f8ae),LL(0x9c92e4d2,0xc72046ab),LL(0x7cf3136b,0x518c74a1), LL(0xf9877d4c,0xff4eb50a),LL(0xa919cabb,0x14578d90),LL(0xac5eb2b6,0x8218f8c4),LL(0x542016e4,0xa3ccc547), + LL(0x327f8349,0x025bf48e),LL(0xf43cb641,0xf3e97346),LL(0x500f1085,0xdc2bafdf),LL(0x2f063055,0x57167876), LL(0x411925a6,0x5bd914b9),LL(0xa1123de5,0x7c078d48),LL(0x182b165d,0xee6bf835),LL(0xba519727,0xb11b5e5b), + LL(0x1eea7b85,0xe33ea76c),LL(0x92d4f85e,0x2352b461),LL(0xafe115bb,0xf101d334),LL(0x889175a3,0xfabc1294), LL(0x5233f925,0x7f6bcdc0),LL(0xe77fec55,0xe0a802db),LL(0x8069b659,0xbdb47b75),LL(0xf98fbd74,0x1c5e12de), + LL(0x4b8457ee,0x869c58c6),LL(0x4f7ea9f7,0xa5360f69),LL(0xf460b38f,0xe576c09f),LL(0x22b7fb36,0x6b70d548), LL(0x3bfae315,0x3fd237f1),LL(0xcbdff369,0x33797852),LL(0x25b516f9,0x97df25f5),LL(0xba38ad2d,0x46f388f2), + LL(0x89d8ddbb,0x656c4658),LL(0x70f38ee8,0x8830b26e),LL(0xde1212b0,0x4320fd5c),LL(0xe4a2edb2,0xc34f30cf), LL(0x56ab64b8,0xabb131a3),LL(0xd99c5d26,0x7f77f0cc),LL(0xbf981d94,0x66856a37),LL(0x738bd76e,0x19e76d09), + LL(0x96238f39,0xe76c8ac3),LL(0xa830b366,0xc0a482be),LL(0x0b4eb499,0xb7b8eaff),LL(0x4bfb4865,0x8ecd83bc), LL(0xa2f3776f,0x971b2cb7),LL(0xf4b88adf,0xb42176a4),LL(0xbe1fa446,0xb9617df5),LL(0xcd031bd2,0x8b32d508), + LL(0x53b618c0,0x1c6bd47d),LL(0x6a227923,0xc424f46c),LL(0xdd92d964,0x7303ffde),LL(0x71b5abf2,0xe9712878), LL(0xf815561d,0x8f48a632),LL(0xd3c055d1,0x85f48ff5),LL(0x7525684f,0x222a1427),LL(0x67360cc3,0xd0d841a0), + LL(0x0b9267c6,0x4245a926),LL(0xcf07f863,0xc78913f1),LL(0x4d0d9e24,0xaa844c8e),LL(0x3d5f9017,0xa42ad522), LL(0xa2c989d5,0xbd371749),LL(0xe1f5e78e,0x928292df),LL(0x0a1ea6da,0x493b383e),LL(0x13aee529,0x5136fd8d), + LL(0xf2c34a99,0x860c44b1),LL(0xbf5855ac,0x3b00aca4),LL(0xfaaf37be,0xabf6aaa0),LL(0x2a53ec08,0x65f43682), LL(0xa11b12e1,0x1d9a5801),LL(0xe20ed475,0x78a7ab2c),LL(0x9a41e0d5,0x0de1067e),LL(0x305023ea,0x30473f5f), + LL(0x169c7d97,0xdd3ae09d),LL(0xcfaef9cd,0x5cd5baa4),LL(0x65a44803,0x5cd7440b),LL(0x47f364de,0xdc13966a), LL(0x2b8357c1,0x077b2be8),LL(0xe9d57c2a,0x0cb1b4c5),LL(0x05ff363e,0x7a4ceb32),LL(0xca35a9ef,0xf310fa4d), + LL(0xf97f68c6,0xdbb7b352),LL(0x0b02cf58,0x0c773b50),LL(0x3c1f96d9,0xea2e4821),LL(0xeee01815,0xffb357b0), LL(0xe0f28039,0xb9c924cd),LL(0x46a3fbe4,0x0b36c95a),LL(0x5e46db6c,0x1faaaea4),LL(0x1928aaff,0xcae575c3), + LL(0xa70dab86,0x7f671302),LL(0x71c58cfc,0xfcbd12a9),LL(0xbee0cb92,0xcbef9acf),LL(0xf8c1b583,0x573da0b9), LL(0x0d41d550,0x4752fcfe),LL(0x2155cffe,0xe7eec0e3),LL(0x545ae248,0x0fc39fcb),LL(0x8065f44e,0x522cb8d1), + LL(0x70cbb96c,0x263c962a),LL(0xbcd124a9,0xe034362a),LL(0x3c2ae58d,0xf120db28),LL(0xfef6d507,0xb9a38d49), LL(0x1ff140fd,0xb1fd2a82),LL(0x20aee7e0,0xbd162f30),LL(0xcb251949,0x4e17a5d4),LL(0x4f7e1c3d,0x2aebcb83), + LL(0x937b0527,0x608eb25f),LL(0xeb7d9997,0xf42e1e47),LL(0xb8a53a29,0xeba699c4),LL(0xe091b536,0x1f921c71), LL(0x5b26bbd5,0xcce29e7b),LL(0x3b61a680,0x7a8ef5ed),LL(0xba1f1c7e,0xe5ef8043),LL(0x18158dda,0x16ea8217), + LL(0x599ff0f9,0x01778a2b),LL(0x8104fc6b,0x68a923d7),LL(0xda694ff3,0x5bfa44df),LL(0xf7667f12,0x4f7199db), LL(0xe46f2a79,0xc06d8ff6),LL(0xe9f8131d,0x08b5dead),LL(0xabb4ce7c,0x02519a59),LL(0xb42aec3e,0xc4f710bc), + LL(0x78bde41a,0x3d77b057),LL(0xb4186b5a,0x6474bf80),LL(0x88c65741,0x048b3f67),LL(0x03c7c154,0xc64519de), LL(0x0edfcc4f,0xdf073846),LL(0x48f1aa6b,0x319aa737),LL(0xca909f77,0x8b9f8a02),LL(0x7580bfef,0x90258139), + LL(0xc0c22719,0xd8bfd3ca),LL(0xc9ca151e,0xc60209e4),LL(0xd9a1a69c,0x7a744ab5),LL(0x14937f8f,0x6de5048b), LL(0xe115ac04,0x171938d8),LL(0x1c6b16d2,0x7df70940),LL(0x7f8e94e7,0xa6aeb663),LL(0x2a2cf094,0xc130388e), + LL(0x77f54e6e,0x1850be84),LL(0x65d60fe5,0x9f258a72),LL(0x6c9146d6,0xff7ff0c0),LL(0xe63a830b,0x039aaf90), LL(0x9460342f,0x38f27a73),LL(0x3f795f8a,0x4703148c),LL(0x9681a97e,0x1bb5467b),LL(0xecaeb594,0x00931ba5), + LL(0x786f337c,0xcdb6719d),LL(0xe704397d,0xd9c01cd2),LL(0x555c2fef,0x0f4a3f20),LL(0x7c0af223,0x00452509), LL(0x84db8e76,0x54a58047),LL(0x93c8aa06,0x3bacf1aa),LL(0xf7919422,0x11ca957c),LL(0x78cdaa40,0x50641053), + LL(0x9f7144ae,0x7a303874),LL(0x43d4acfd,0x170c963f),LL(0x58ddd3ef,0x5e148149),LL(0x9e72dba8,0xa7bde582), LL(0x6fa68750,0x0769da8b),LL(0x572e0249,0xfa64e532),LL(0x2619ad31,0xfcaadf9d),LL(0xa7b349cd,0x87882daa), + LL(0x6c67a775,0x9f6eb731),LL(0xefc5d0b1,0xcb10471a),LL(0xe1b806b2,0xb433750c),LL(0x57b1ae7e,0x19c5714d), LL(0xed03fd3f,0xc0dc8b7b),LL(0x31bc194e,0xdd03344f),LL(0x8c6320b5,0xa66c52a7),LL(0xd0b6fd93,0x8bc82ce3), + LL(0xb35f1341,0xf8e13501),LL(0x25a43e42,0xe53156dd),LL(0x4daeb85c,0xd3adf27e),LL(0xbbeddeb5,0xb81d8379), LL(0x2e435867,0x1b0b546e),LL(0xeba5dd60,0x9020eb94),LL(0x8210cb9d,0x37d91161),LL(0x5c91f1cf,0x4c596b31), + LL(0x0e0b040d,0xb228a90f),LL(0x45ff897f,0xbaf02d82),LL(0x00fa6122,0x2aac79e6),LL(0x8e36f557,0x24828817), LL(0x113ec356,0xb9521d31),LL(0x15eff1f8,0x9e48861e),LL(0xe0d41715,0x2aa1d412),LL(0x53f131b8,0x71f86203), + LL(0x3fd19408,0xf60da8da),LL(0x278d9d99,0x4aa716dc),LL(0xa8c51c90,0x394531f7),LL(0xf59db51c,0xb560b0e8), LL(0xfa34bdad,0xa28fc992),LL(0x9cd4f8bd,0xf024fa14),LL(0x23a9d0d3,0x5cf530f7),LL(0xe28c9b56,0x615ca193), + LL(0x6f73c51e,0x6d2a483d),LL(0xea0dc2dd,0xa4cb2412),LL(0x1eb917ff,0x50663c41),LL(0xeade299e,0x3d3a74cf), LL(0x4a7a9202,0x29b3990f),LL(0xa7b15c3d,0xa9bccf59),LL(0xa5df9208,0x66a3ccdc),LL(0x43f2f929,0x48027c14), + LL(0x40b557f0,0xd385377c),LL(0xcd684660,0xe001c366),LL(0xe2183a27,0x1b18ed6b),LL(0x63210329,0x879738d8), LL(0xbda94882,0xa687c74b),LL(0xa684b299,0xd1bbcc48),LL(0x863b3724,0xaf6f1112),LL(0x2c8ce9f8,0x6943d1b4), + LL(0x098cafb4,0xe044a3bb),LL(0x60d48caf,0x27ed2310),LL(0x3a31b84d,0x542b5675),LL(0xfcddbed7,0xcbf3dd50), LL(0x41b1d830,0x25031f16),LL(0xcb0c1e27,0xa7ec851d),LL(0xb5ae75db,0xac1c8fe0),LL(0x08c52120,0xb24c7557), + LL(0x1d4636c3,0x57f811dc),LL(0x681a9939,0xf8436526),LL(0x9c81adb3,0x1f6bc6d9),LL(0x5b7d80d4,0x840f8ac3), LL(0xf4387f1a,0x731a9811),LL(0xb5156880,0x7c501cd3),LL(0xdfe68867,0xa5ca4a07),LL(0x5fcea120,0xf123d8f0), + LL(0xd607039e,0x1fbb0e71),LL(0xcd3a4546,0x2b70e215),LL(0x53324091,0x32d2f01d),LL(0x180ab19b,0xb796ff08), LL(0x3c57c4aa,0x32d87a86),LL(0xb7c49a27,0x2aed9caf),LL(0x31630d98,0x9fb35eac),LL(0x5c3e20a3,0x338e8cdf), + LL(0x66cde8db,0x80f16182),LL(0x2d72fd36,0x4e159980),LL(0x9b6e5072,0xd7b8f13b),LL(0x3b7b5dc1,0xf5213907), LL(0x8ce4396e,0x4d431f1d),LL(0xa7ed2142,0x37a1a680),LL(0xd01aaf6b,0xbf375696),LL(0xe63aab66,0xaa1c0c54), + LL(0x4ed80940,0x3014368b),LL(0x7a6fcedd,0x67e6d056),LL(0xca97579f,0x7c208c49),LL(0xa23597f6,0xfe3d7a81), LL(0x7e096ae2,0x5e203202),LL(0x24b39366,0xb1f3e1e7),LL(0x2fdcdffc,0x26da26f3),LL(0x6097be83,0x79422f1d), +}, +/* digit=24 base_pwr=2^168 */ +{ + LL(0x9db3b381,0x263a2cfb),LL(0xd4df0a4b,0x9c3a2dee),LL(0x7d04e61f,0x728d06e9),LL(0x42449325,0x8b1adfbc), LL(0x7e053a1b,0x6ec1d939),LL(0x66daf707,0xee2be5c7),LL(0x810ac7ab,0x80ba1e14),LL(0xf530f174,0xdd2ae778), + LL(0x205b9d8b,0x0435d97a),LL(0x056756d4,0x6eb8f064),LL(0xb6f8210e,0xd5e88a8b),LL(0xec9fd9ea,0x070ef12d), LL(0x3bcc876a,0x4d849505),LL(0xa7404ce3,0x12a75338),LL(0xb8a1db5e,0xd22b49e1),LL(0x14bfa5ad,0xec1f2051), + LL(0xb6828f36,0xadbaeb79),LL(0x01bd5b9e,0x9d7a0258),LL(0x1e844b0c,0xeda01e0d),LL(0x887edfc9,0x4b625175), LL(0x9669b621,0x14109fdd),LL(0xf6f87b98,0x88a2ca56),LL(0x170df6bc,0xfe2eb788),LL(0xffa473f9,0x0cea06f4), + LL(0xc4e83d33,0x43ed81b5),LL(0x5efd488b,0xd9f35879),LL(0x9deb4d0f,0x164a620f),LL(0xac6a7394,0xc6927bdb), LL(0x9f9e0f03,0x45c28df7),LL(0xfcd7e1a9,0x2868661e),LL(0xffa348f1,0x7cf4e8d0),LL(0x398538e0,0x6bd4c284), + LL(0x289a8619,0x2618a091),LL(0x6671b173,0xef796e60),LL(0x9090c632,0x664e46e5),LL(0x1e66f8fb,0xa38062d4), LL(0x0573274e,0x6c744a20),LL(0xa9271394,0xd07b67e4),LL(0x6bdc0e20,0x391223b2),LL(0xeb0a05a7,0xbe2d93f1), + LL(0x3f36d141,0xf23e2e53),LL(0x4dfca442,0xe84bb3d4),LL(0x6b7c023a,0xb804a48d),LL(0x76431c3b,0x1e16a8fa), LL(0xddd472e0,0x1b5452ad),LL(0x0d1ee127,0x7d405ee7),LL(0xffa27599,0x50fc6f1d),LL(0xbf391b35,0x351ac53c), + LL(0x4444896b,0x7efa14b8),LL(0xf94027fb,0x64974d2f),LL(0xde84487d,0xefdcd0e8),LL(0x2b48989b,0x8c45b260), LL(0xd8463487,0xa8fcbbc2),LL(0x3fbc476c,0xd1b2b3f7),LL(0xc8f443c0,0x21d005b7),LL(0x40c0139c,0x518f2e67), + LL(0x06d75fc1,0x56036e8c),LL(0x3249a89f,0x2dcf7bb7),LL(0xe245e7dd,0x81dd1d3d),LL(0xebd6e2a7,0xf578dc4b), LL(0xdf2ce7a0,0x4c028903),LL(0x9c39afac,0xaee36288),LL(0x146404ab,0xdc847c31),LL(0xa4e97818,0x6304c0d8), + LL(0xa91f6791,0xae51dca2),LL(0x9baa9efc,0x2abe4190),LL(0x559c7ac1,0xd9d2e2f4),LL(0xfc9f773a,0xe82f4b51), LL(0x4073e81c,0xa7713027),LL(0xfbb596fc,0xc0276fac),LL(0xa684f70c,0x1d819fc9),LL(0xc9f7b1e0,0x29b47fdd), + LL(0x459b1940,0x358de103),LL(0x5b013e93,0xec881c59),LL(0x49532ad3,0x51574c93),LL(0xb37b46de,0x2db1d445), LL(0xdf239fd8,0xc6445b87),LL(0x151d24ee,0xc718af75),LL(0xf43c6259,0xaea1c4a4),LL(0x70be02f7,0x40c0e5d7), + LL(0x721b33f2,0x6a4590f4),LL(0xfedf04ea,0x2124f1fb),LL(0x9745efe7,0xf8e53cde),LL(0x65f046d9,0xe7e10432), LL(0xe4d0c7e6,0xc3fca28e),LL(0x87253b1b,0x847e339a),LL(0x3743e643,0x9b595348),LL(0x4fd12fc5,0xcb6a0a0b), + LL(0x27d02dcc,0xfb6836c3),LL(0x7a68bcc2,0x5ad00982),LL(0x005e912d,0x1b24b44c),LL(0x811fdcfe,0xcc83d20f), LL(0x666fba0c,0x36527ec1),LL(0x14754635,0x69948197),LL(0x556da9c2,0xfcdcb1a8),LL(0x81a732b2,0xa5934267), + LL(0xa714181d,0xec1214ed),LL(0x6067b341,0x609ac13b),LL(0xa545df1f,0xff4b4c97),LL(0x34d2076b,0xa1240501), LL(0x1409ca97,0x6efa0c23),LL(0x20638c43,0x254cc1a8),LL(0xdcfb46cd,0xd4e363af),LL(0x03942a27,0x62c2adc3), + LL(0x56e46483,0xc67b9df0),LL(0x63736356,0xa55abb20),LL(0xc551bc52,0xab93c098),LL(0xb15fe64b,0x382b49f9), LL(0x4dff8d47,0x9ec221ad),LL(0x437df4d6,0x79caf615),LL(0xbb456509,0x5f13dc64),LL(0x191f0714,0xe4c589d9), + LL(0x3fd40e09,0x27b6a8ab),LL(0x77313ea9,0xe455842e),LL(0x1f55988b,0x8b51d1e2),LL(0x062bbbfc,0x5716dd73), LL(0x4e8bf3de,0x633c11e5),LL(0x1b85be3b,0x9a0e77b6),LL(0x0911cca6,0x56510729),LL(0xefa6590f,0x27e76495), + LL(0x070d3aab,0xe4ac8b33),LL(0x9a2cd5e5,0x2643672b),LL(0x1cfc9173,0x52eff79b),LL(0x90a7c13f,0x665ca49b), LL(0xb3efb998,0x5a8dda59),LL(0x052f1341,0x8a5b922d),LL(0x3cf9a530,0xae9ebbab),LL(0xf56da4d7,0x35986e7b), + LL(0xff3513cc,0x3a636b5c),LL(0x3198f7dd,0xbb0cf8ba),LL(0x41f16f86,0xb8d40522),LL(0xde13a7bf,0x760575d8), LL(0x9f7aa181,0x36f74e16),LL(0xf509ed1c,0x163a3ecf),LL(0x3c40a491,0x6aead61f),LL(0xdfe8fcaa,0x158c95fc), + LL(0x13cda46f,0xa3991b6e),LL(0x342faed0,0x79482415),LL(0x666b5970,0xf3ba5bde),LL(0xb26ab6dd,0x1d52e6bc), LL(0x8608dd3d,0x768ba1e7),LL(0xea076586,0x4930db2a),LL(0xe7dc1afa,0xd9575714),LL(0xf7c58817,0x1fc7bf7d), + LL(0xd9eee96c,0x6b47accd),LL(0xe58cec37,0x0ca277fb),LL(0xe702c42a,0x113fe413),LL(0xc47cbe51,0xdd1764ee), LL(0x7b3ed739,0x041e7cde),LL(0x5ce9e1c0,0x50cb7459),LL(0x2925b212,0x35568513),LL(0x001b081c,0x7cff95c4), + LL(0x8088b454,0x63ee4cbd),LL(0x9a9e0c8a,0xdb7f32f7),LL(0x6b2447cb,0xb377d418),LL(0xd370219b,0xe3e982aa), LL(0xc2a2a593,0x06ccc1e4),LL(0x0773f24f,0x72c36865),LL(0x95859423,0xa13b4da7),LL(0x75040c8f,0x8bbf1d33), + LL(0xda50c991,0x726f0973),LL(0x822d6ee2,0x48afcd5b),LL(0x20fd7771,0xe5fc718b),LL(0xfd0807a1,0xb9e8e77d), LL(0x99a7703d,0x7f5e0f44),LL(0x618e36f3,0x6972930e),LL(0x23807bbe,0x2b7c77b8),LL(0xcb27ff50,0xe5b82405), + LL(0xbd379062,0xba8b8be3),LL(0x2dce4a92,0xd64b7a1d),LL(0xb2952e37,0x040a73c5),LL(0xd438aeca,0x0a9e252e), LL(0xc39d3bcb,0xdd43956b),LL(0xb32b2d63,0x1a31ca00),LL(0x5c417a18,0xd67133b8),LL(0x2ef442c8,0xd08e4790), + LL(0x255c0980,0x98cb1ae9),LL(0x2b4a739f,0x4bd86381),LL(0x1e4a45a1,0x5a5c31e1),LL(0x9cb0db2f,0x1e5d55fe), LL(0x8ff5cc29,0x74661b06),LL(0x0eb8a4f4,0x026b389f),LL(0x58848c24,0x536b21a4),LL(0x81dc72b0,0x2e5bf8ec), + LL(0xad886aac,0x03c187d0),LL(0xb771b645,0x5c16878a),LL(0xc74045ab,0xb07dfc6f),LL(0x7800caed,0x2c6360bf), LL(0xb9c972a3,0x24295bb5),LL(0x7c9a6dba,0xc9e6f88e),LL(0x92a79aa6,0x90ffbf24),LL(0x41c26ac2,0xde29d50a), + LL(0xd309cbe6,0x9f0af483),LL(0xe0bced4f,0x5b020d8a),LL(0xb38023e3,0x606e986d),LL(0x1abc6933,0xad8f2c9d), LL(0xe7400e93,0x19292e1d),LL(0x52be5e4d,0xfe3e18a9),LL(0x2e0680bf,0xe8e9771d),LL(0xc54db063,0x8c5bec98), + LL(0x74a55d1f,0x2af9662a),LL(0x046f66d8,0xe3fbf28f),LL(0xd4dc4794,0xa3a72ab4),LL(0x5c7c2dd8,0x09779f45), LL(0xc3d19d8d,0xd893bdaf),LL(0x57d6a6df,0xd5a75094),LL(0x952e6255,0x8cf8fef9),LL(0xda9a8aff,0x3da67cfb), + LL(0x2c160dcd,0x4c23f62a),LL(0x8f90eaef,0x34e6c5e3),LL(0xa9a65d5a,0x35865519),LL(0x8fd38a3d,0x07c48aae), LL(0x50068527,0xb7e7aeda),LL(0x1c90936a,0x2c09ef23),LL(0xe879324c,0x31ecfeb6),LL(0xfb0ec938,0xa0871f6b), + LL(0xd84d835d,0xb1f0fb68),LL(0x861dc1e6,0xc90caf39),LL(0x7594f8d7,0x12e5b046),LL(0x65012b92,0x26897ae2), LL(0xa4d6755d,0xbcf68a08),LL(0x0991fbda,0x403ee41c),LL(0x3bbf17e8,0x733e343e),LL(0x679b3d65,0xd2c7980d), + LL(0xd2e11305,0x33056232),LL(0xf3c07a6f,0x966be492),LL(0xbb15509d,0x6a8878ff),LL(0x0a9b59a4,0xff221101), LL(0xabe30129,0x6c9f564a),LL(0x336e64cf,0xc6f2c940),LL(0x8b0c8022,0x0fe75262),LL(0x6ae8db87,0xbe0267e9), + LL(0x93bc042b,0x22e192f1),LL(0xb237c458,0xf085b534),LL(0x832c4168,0xa0d192bd),LL(0xbdf6271d,0x7a76e9e3), LL(0xb88911b5,0x52a882fa),LL(0xb4db0eb5,0xc85345e4),LL(0x81a7c3ff,0xa3be02a6),LL(0xf0ec0469,0x51889c8c), + LL(0xa5e829e5,0x9d031369),LL(0x1607aa41,0xcbb4c6fc),LL(0x241d84c1,0x75ac59a6),LL(0x8829e0ee,0xc043f2bf), LL(0x8ea5e185,0x82a38f75),LL(0xd87cbd9f,0x8bda40b9),LL(0x2d8fc601,0x9e65e75e),LL(0xa35690b3,0x3d515f74), + LL(0xda79e5ac,0x534acf4f),LL(0x8630215f,0x68b83b3a),LL(0xd085756e,0x5c748b2e),LL(0xe5d37cb2,0xb0317258), LL(0xc5ccc2c4,0x6735841a),LL(0x3d9d5069,0x7d7dc96b),LL(0xfd1754bd,0xa147e410),LL(0xd399ddd5,0x65296e94), + LL(0xbc8fa5bc,0xf6b5b2d0),LL(0x500c277b,0x8a5ead67),LL(0xdfa08a5d,0x214625e6),LL(0x959cf047,0x51fdfedc), LL(0x289fca32,0x6bc9430b),LL(0x9d9bdc3f,0xe36ff0cf),LL(0x58ea0ede,0x2fe187cb),LL(0x5a900b3f,0xed66af20), + LL(0x5fa9f4d6,0x00e0968b),LL(0x37a362e7,0x2d4066ce),LL(0xbd07e772,0xa99a9748),LL(0x06a4f1d0,0x710989c0), LL(0xce40cbd8,0xd5dedf35),LL(0x1743293d,0xab55c5f0),LL(0x8aa24e2c,0x766f1144),LL(0x605fbcb4,0x94d874f8), + LL(0xa518001b,0xa365f0e8),LL(0x9d04ef0f,0xee605eb6),LL(0xba8d4d25,0x5a3915cd),LL(0xb5113472,0x44c0e1b8), LL(0x8b6740dc,0xcbb024e8),LL(0xee1d4f0c,0x89087a53),LL(0x1fc4e372,0xa88fa05c),LL(0xaf8b3af2,0x8bf395cb), + LL(0xdeb8568b,0x1e71c9a1),LL(0x80fb3d32,0xa35daea0),LL(0x2cf8fb81,0xe8b6f266),LL(0x9490696a,0x6d51afe8), LL(0x51803a19,0x81beac6e),LL(0x86219080,0xe3d24b7f),LL(0xdf6f463c,0x727cfd9d),LL(0x72284ee8,0x8c6865ca), + LL(0xb743f4ef,0x32c88b7d),LL(0xe7d11dce,0x3793909b),LL(0x2ff2ebe8,0xd398f922),LL(0xe5e49796,0x2c70ca44), LL(0xcb1131b1,0xdf4d9929),LL(0x25888e79,0x7826f298),LL(0xf1d8740a,0x4d3a112c),LL(0x270afa8b,0x00384cb6), + LL(0x3ab48095,0xcb64125b),LL(0x62d05106,0x3451c256),LL(0xa4955845,0xd73d577d),LL(0xbf9f4433,0x39570c16), LL(0xadecf263,0xd7dfaad3),LL(0xdc76e102,0xf1c3d8d1),LL(0x54c6a836,0x5e774a58),LL(0x3e92d47b,0xdad4b672), + LL(0xf0d796a0,0xbe7e990f),LL(0xdf0e8b02,0x5fc62478),LL(0x030c00ad,0x8aae8bf4),LL(0x9004ba0f,0x3d2db93b), LL(0xd85d5ddc,0xe48c8a79),LL(0x6bb07f34,0xe907caa7),LL(0xa39eaed5,0x58db343a),LL(0xadaf5724,0x0ea6e007), + LL(0xd23233f3,0xe00df169),LL(0x77cb637f,0x3e322796),LL(0x1da0cf6c,0x1f897c0e),LL(0x31d6bbdd,0xa651f5d8), LL(0x1a230c76,0xdd61af19),LL(0xcdaa5e4a,0xbd527272),LL(0xd0abcd7e,0xca753636),LL(0x370bd8dc,0x78bdd37c), + LL(0x17cd93fe,0xc23916c2),LL(0xdadce6e2,0x65b97a4d),LL(0x174e42f8,0xe04ed4eb),LL(0xbb21480a,0x1491ccaa), LL(0x23196332,0x145a8280),LL(0x587b479a,0x3c3862d7),LL(0x01dcd0ed,0x9f4a88a3),LL(0x3ea12f1f,0x4da2b7ef), + LL(0xb126e48e,0xf8e7ae33),LL(0xf494e237,0x404a0b32),LL(0xc55acadb,0x9beac474),LL(0xcbec9fd9,0x4ee5cf3b), LL(0x7df3c8c3,0x336b33b9),LL(0xb76808fd,0xbd905fe3),LL(0xaa45c16a,0x8f436981),LL(0x3dd27b62,0x255c5bfa), + LL(0xc3dd9b4d,0x71965cbf),LL(0xfc068a87,0xce23edbf),LL(0x745b029b,0xb78d4725),LL(0xcefdd9bd,0x74610713), LL(0x1266bf52,0x7116f75f),LL(0x18e49bb6,0x02046722),LL(0x3d6f19e3,0xdf43df9f),LL(0xe685cb2f,0xef1bc7d0), + LL(0x7078c432,0xcddb27c1),LL(0xb77fedb7,0xe1961b9c),LL(0xc2290570,0x1edc2f5c),LL(0x19cbd886,0x2c3fefca), LL(0xc2af389a,0xcf880a36),LL(0xbda71cea,0x96c610fd),LL(0x32aa8463,0xf03977a9),LL(0x8586d90a,0x8eb7763f), + LL(0x2a296e77,0x3f342454),LL(0x42837a35,0xc8718683),LL(0x6a09c731,0x7dc71090),LL(0x51b816db,0x54778ffb), LL(0xaf06defd,0x6b33bfec),LL(0x8592b70b,0xfe3c105f),LL(0x61da6114,0xf937fda4),LL(0x4c266ad7,0x3c13e651), + LL(0x855938e8,0xe363a829),LL(0x9de54b72,0x2eeb5d9e),LL(0x20ccfab9,0xbeb93b0e),LL(0x25e61a25,0x3dffbb5f), LL(0x1acc093d,0x7f655e43),LL(0x3964ce61,0x0cb6cc3d),LL(0xe5e9b460,0x6ab283a1),LL(0xa1c7e72d,0x55d787c5), + LL(0xdeadbf02,0x4d2efd47),LL(0xac459068,0x11e80219),LL(0x71f311f0,0x810c7626),LL(0x4ab6ef53,0xfa17ef8d), LL(0x93e43bff,0xaf47fd25),LL(0x0be40632,0x5cb5ff3f),LL(0x8ee61da3,0x54687106),LL(0xb08afd0f,0x7764196e), + LL(0xf0290a8f,0x831ab3ed),LL(0xcb47c387,0xcae81966),LL(0x184efb4f,0xaad7dece),LL(0x4749110e,0xdcfc53b3), LL(0x4cb632f9,0x6698f23c),LL(0xb91f8067,0xc42a1ad6),LL(0x6284180a,0xb116a81d),LL(0xe901326f,0xebedf5f8), + LL(0x97e3e044,0xf2274c9f),LL(0x11d09fc9,0x42018520),LL(0xd18e6e23,0x56a65f17),LL(0x352b683c,0x2ea61e2a), LL(0x575eaa94,0x27d291bc),LL(0xb8ff522d,0x9e7bc721),LL(0xa7f04d6f,0x5f7268bf),LL(0xaba41748,0x5868c73f), + LL(0x7be0eead,0x9f85c2db),LL(0xff719135,0x511e7842),LL(0xc5ea90d7,0x5a06b1e9),LL(0x26fab631,0x0c19e283), LL(0xe9206c55,0x8af8f0cf),LL(0x3553c06a,0x89389cb4),LL(0xf65f8004,0x39dbed97),LL(0xc508991d,0x0621b037), + LL(0x96e78cc4,0x1c52e635),LL(0x0c06b4a8,0x5385c8b2),LL(0xb0e87d03,0xd84ddfdb),LL(0x934bafad,0xc49dfb66), LL(0x59f70772,0x7071e170),LL(0x3a1db56b,0x3a073a84),LL(0x3b8af190,0x03494903),LL(0xd32920f0,0x7d882de3), + LL(0xb2cf8940,0x91633f0a),LL(0x6f948f51,0x72b0b178),LL(0x782653c8,0x2d28dc30),LL(0xdb903a05,0x88829849), LL(0x6a19d2bb,0xb8095d0c),LL(0x86f782cb,0x4b9e7f0c),LL(0x2d907064,0x7af73988),LL(0x8b32643c,0xd12be0fe), + LL(0x0e165dc3,0x358ed23d),LL(0x4e2378ce,0x3d47ce62),LL(0xfeb8a087,0x7e2bb0b9),LL(0xe29e10b9,0x3246e8ae), LL(0x03ce2b4d,0x459f4ec7),LL(0xbbc077cf,0xe9b4ca1b),LL(0x0e9940c1,0x2613b4f2),LL(0x047d1eb1,0xfc598bb9), + LL(0x45036099,0x9744c62b),LL(0x167c65d8,0xa9dee742),LL(0xdabe1943,0x0c511525),LL(0x93c6c624,0xda110554), LL(0x651a3be2,0xae00a52c),LL(0x884449a6,0xcda5111d),LL(0xff33bed1,0x063c06f4),LL(0x0d3d76b4,0x73baaf9a), + LL(0x7fc63668,0x52fb0c9d),LL(0x0c039cde,0x6886c9dd),LL(0x55b22351,0x602bd599),LL(0x360c7c13,0xb00cab02), LL(0x81b69442,0x8cb616bc),LL(0xb55c3cee,0x41486700),LL(0xf49ba278,0x71093281),LL(0x64a50710,0xad956d9c), + LL(0x638a7e81,0x9561f28b),LL(0x5980ddc3,0x54155cdf),LL(0xd26f247a,0xb2db4a96),LL(0x4787d100,0x9d774e4e), LL(0x078637d2,0x1a9e6e2e),LL(0x5e0ae06a,0x1c363e2d),LL(0xe9cfa354,0x7493483e),LL(0x7f74b98d,0x76843cb3), + LL(0xd4b66947,0xbaca6591),LL(0x04460a8c,0xb452ce98),LL(0x43768f55,0x6830d246),LL(0x7dff12df,0xf4197ed8), LL(0x400dd0f7,0x6521b472),LL(0x4b1e7093,0x59f5ca8f),LL(0x080338ae,0x6feff11b),LL(0xa29ca3c6,0x0ada31f6), + LL(0x94a2c215,0x24794eb6),LL(0x05a57ab4,0xd83a43ab),LL(0x2a6f89fe,0x264a543a),LL(0xdd5ec7c2,0x2c2a3868), LL(0x8439d9b2,0xd3373940),LL(0x0acd1f11,0x715ea672),LL(0xe7e6cc19,0x42c1d235),LL(0xb990585c,0x81ce6e96), + LL(0xd809c7bd,0x04e5dfe0),LL(0x8f1050ab,0xd7b2580c),LL(0xd8a4176f,0x6d91ad78),LL(0x4e2e897c,0x0af556ee), LL(0x921de0ac,0x162a8b73),LL(0x7ea78400,0x52ac9c22),LL(0xefce2174,0xee2a4eea),LL(0x6d637f79,0xbe61844e), + LL(0x789a283b,0x0491f1bc),LL(0x880836f4,0x72d3ac3d),LL(0x88e5402d,0xaa1c5ea3),LL(0xd5cc473d,0x1b192421), LL(0x9dc84cac,0x5c0b9998),LL(0x9c6e75b8,0xb0a8482d),LL(0x3a191ce2,0x639961d0),LL(0x6d837930,0xda3bc865), + LL(0x056e6f8f,0xca990653),LL(0x64d133a7,0x84861c41),LL(0x746abe40,0x8b403276),LL(0xebf8e303,0xb7b4d51a), LL(0x220a255d,0x05b43211),LL(0x02419e6e,0xc997152c),LL(0x630c2fea,0x76ff47b6),LL(0x281fdade,0x50518677), + LL(0xcf902b0b,0x3283b8ba),LL(0x37db303b,0x8d4b4eb5),LL(0x755011bc,0xcc89f42d),LL(0xdd09d19b,0xb43d74bb), LL(0x8adba350,0x65746bc9),LL(0xb51c1927,0x364eaf8c),LL(0x10ad72ec,0x13c76596),LL(0xf8d40c20,0x30045121), + LL(0xea7b979b,0x6d2d99b7),LL(0xe6fb3bcd,0xcd78cd74),LL(0x86cffbfe,0x11e45a9e),LL(0x637024f6,0x78a61cf4), LL(0x3d502295,0xd06bc872),LL(0x458cb288,0xf1376854),LL(0x342f8586,0xb9db26a1),LL(0x4beee09e,0xf33effcf), + LL(0xb30cfb3a,0xd7e0c4cd),LL(0x6c9db4c8,0x6d09b8c1),LL(0x07c8d9df,0x40ba1a42),LL(0x1c52c66d,0x6fd495f7), LL(0x275264da,0xfb0e169f),LL(0xe57d8362,0x80c2b746),LL(0x49ad7222,0xedd987f7),LL(0x4398ec7b,0xfdc229af), +}, +/* digit=25 base_pwr=2^175 */ +{ + LL(0x52666a58,0xb0d1ed84),LL(0xe6a9c3c2,0x4bcb6e00),LL(0x26906408,0x3c57411c),LL(0x13556400,0xcfc20755), LL(0x5294dba3,0xa08b1c50),LL(0x8b7dd31e,0xa30ba286),LL(0x991eca74,0xd70ba90e),LL(0xe762c2b9,0x094e142c), + LL(0x979f3925,0xb81d783e),LL(0xaf4c89a7,0x1efd130a),LL(0xfd1bf7fa,0x525c2144),LL(0x1b265a9e,0x4b296904), LL(0xb9db65b6,0xed8e9634),LL(0x03599d8a,0x35c82e32),LL(0x403563f3,0xdaa7a54f),LL(0x022c38ab,0x9df088ad), + LL(0xbb3fd30a,0xe5cfb066),LL(0xeff0354e,0x429169da),LL(0x3524e36c,0x809cf852),LL(0x0155be1d,0x136f4fb3), LL(0x1fbba712,0x4826af01),LL(0x506ba1a1,0x6ef0f0b4),LL(0x77aea73e,0xd9928b31),LL(0x5eaa244e,0xe2bf6af2), + LL(0x4237b64b,0x8d084f12),LL(0xe3ecfd07,0x688ebe99),LL(0xf6845dd8,0x57b8a70c),LL(0x5da4a325,0x808fc59c), LL(0xa3585862,0xa9032b2b),LL(0xedf29386,0xb66825d5),LL(0x431ec29b,0xb5a5a8db),LL(0x3a1e8dc8,0xbb143a98), + LL(0x12ae381b,0x35ee94ce),LL(0x86ccda90,0x3a7f176c),LL(0x4606eaca,0xc63a657e),LL(0x43cd04df,0x9ae5a380), LL(0xed251b46,0x9bec8d15),LL(0xcaca5e64,0x1f5d6d30),LL(0x9ff20f07,0x347b3b35),LL(0xf7e4b286,0x4d65f034), + LL(0xf111661e,0x9e93ba24),LL(0xb105eb04,0xedced484),LL(0xf424b578,0x96dc9ba1),LL(0xe83e9069,0xbf8f66b7), LL(0xd7ed8216,0x872d4df4),LL(0x8e2cbecf,0xbf07f377),LL(0x98e73754,0x4281d899),LL(0x8aab8708,0xfec85fbb), + LL(0xa5ba5b0b,0x9a3c0dee),LL(0x42d05299,0xe6a116ce),LL(0xe9b02d42,0xae9775fe),LL(0xa1545cb6,0x72b05200), LL(0x31a3b4ea,0xbc506f7d),LL(0x8bbd9b32,0xe5893078),LL(0xe4b12a97,0xc8bc5f37),LL(0x4a73b671,0x6b000c06), + LL(0x765fa7d0,0x13b5bf22),LL(0x1d6a5370,0x59805bf0),LL(0x4280db98,0x67a5e29d),LL(0x776b1ce3,0x4f53916f), LL(0x33ddf626,0x714ff61f),LL(0xa085d103,0x4206238e),LL(0xe5809ee3,0x1c50d4b7),LL(0x85f8eb1d,0x999f450d), + LL(0xe4c79e9b,0x658a6051),LL(0xc66a9fea,0x1394cb73),LL(0xc6be7b23,0x27f31ed5),LL(0x5aa6f8fe,0xf4c88f36), LL(0x4aaa499e,0x0fb0721f),LL(0xe3fb2a6b,0x68b3a7d5),LL(0x3a92851d,0xa788097d),LL(0xe96f4913,0x060e7f8a), + LL(0x1a3a93bc,0x82eebe73),LL(0xa21adc1a,0x42bbf465),LL(0xef030efd,0xc10b6fa4),LL(0x87b097bb,0x247aa4c7), LL(0xf60c77da,0x8b8dc632),LL(0xc223523e,0x6ffbc26a),LL(0x344579cf,0xa4f6ff11),LL(0x980250f6,0x5825653c), + LL(0xbc1aa2b9,0xb2dd097e),LL(0x37a0333a,0x07889393),LL(0x37a0db38,0x1cf55e71),LL(0x792c1613,0x2648487f), LL(0x3fcef261,0xdad01336),LL(0x0eabf129,0x6239c81d),LL(0x9d276be2,0x8ee761de),LL(0x1eda6ad3,0x406a7a34), + LL(0x4a493b31,0x4bf367ba),LL(0x9bf7f026,0x54f20a52),LL(0x9795914b,0xb696e062),LL(0x8bf236ac,0xcddab96d), LL(0xed25ea13,0x4ff2c70a),LL(0x81cbbbe7,0xfa1d09eb),LL(0x468544c5,0x88fc8c87),LL(0x696b3317,0x847a670d), + LL(0x64bcb626,0xf133421e),LL(0x26dee0b5,0xaea638c8),LL(0xb310346c,0xd6e7680b),LL(0xd5d4ced3,0xe06f4097), LL(0x7512a30b,0x09961452),LL(0xe589a59a,0xf3d867fd),LL(0x52d0c180,0x2e73254f),LL(0x333c74ac,0x9063d8a3), + LL(0xd314e7bc,0xeda6c595),LL(0x467899ed,0x2ee7464b),LL(0x0a1ed5d3,0x1cef423c),LL(0x69cc7613,0x217e76ea), LL(0xe7cda917,0x27ccce1f),LL(0x8a893f16,0x12d8016b),LL(0x9fc74f6b,0xbcd6de84),LL(0xf3144e61,0xfa5817e2), + LL(0x0821ee4c,0x1f354164),LL(0x0bc61992,0x1583eab4),LL(0x1d72879f,0x7490caf6),LL(0xf76ae7b2,0x998ad9f3), LL(0xa41157f7,0x1e181950),LL(0xe8da3a7e,0xa9d7e1e6),LL(0x8426b95f,0x963784eb),LL(0x542e2a10,0x0ee4ed6e), + LL(0xac751e7b,0xb79d4cc5),LL(0xfd4211bd,0x93f96472),LL(0xc8de4fc6,0x8c72d3d2),LL(0xdf44f064,0x7b69cbf5), LL(0xf4bf94e1,0x3da90ca2),LL(0xf12894e2,0x1a5325f8),LL(0x7917d60b,0x0a437f6c),LL(0x96c9cb5d,0x9be70486), + LL(0xe1dc5c05,0xb4d880bf),LL(0xeebeeb57,0xd738adda),LL(0xdf0fe6a3,0x6f0119d3),LL(0x66eaaf5a,0x5c686e55), LL(0xdfd0b7ec,0x9cb10b50),LL(0x6a497c21,0xbdd0264b),LL(0x8c546c96,0xfc093514),LL(0x79dbf42a,0x58a947fa), + LL(0x49ccd6d7,0xc0b48d4e),LL(0x88bd5580,0xff8fb02c),LL(0x07d473b2,0xc75235e9),LL(0xa2188af3,0x4fab1ac5), LL(0x97576ec0,0x030fa3bc),LL(0x0b7e7d2f,0xe8c946e8),LL(0x70305600,0x40a5c9cc),LL(0xc8b013b4,0x6d8260a9), + LL(0x70bba85c,0x0368304f),LL(0xa4a0d311,0xad090da1),LL(0x2415eec1,0x7170e870),LL(0x8461ea47,0xbfba35fe), LL(0xc1e91938,0x6279019a),LL(0x1afc415f,0xa47638f3),LL(0xbcba0e0f,0x36c65cbb),LL(0x034e2c48,0x02160efb), + LL(0x615cd9e4,0xe6c51073),LL(0xf1243c06,0x498ec047),LL(0xb17b3d8c,0x3e5a8809),LL(0x0cc565f1,0x5cd99e61), LL(0x7851dafe,0x81e312df),LL(0xa79061e2,0xf156f5ba),LL(0x880c590e,0x80d62b71),LL(0x0a39faa1,0xbec9746f), + LL(0xc8ed1f7a,0x1d98a9c1),LL(0xa81d5ff2,0x09e43bb5),LL(0x0da0794a,0xd5f00f68),LL(0x661aa836,0x412050d9), LL(0x90747e40,0xa89f7c4e),LL(0xb62a3686,0x6dc05ebb),LL(0x308e3353,0xdf4de847),LL(0x9fb53bb9,0x53868fbb), + LL(0xcfdcf7dd,0x2b09d2c3),LL(0x723fcab4,0x41a9fce3),LL(0x07f57ca3,0x73d905f7),LL(0xac8e1555,0x080f9fb1), LL(0x9ba7a531,0x7c088e84),LL(0xed9a147f,0x07d35586),LL(0xaf48c336,0x602846ab),LL(0x0ccf0e79,0x7320fd32), + LL(0xb18bd1ff,0xaa780798),LL(0xafdd2905,0x52c2e300),LL(0x434267cd,0xf27ea3d6),LL(0x15605b5f,0x8b96d16d), LL(0x4b45706b,0x7bb31049),LL(0x743d25f8,0xe7f58b8e),LL(0x87f30076,0xe9b5e45b),LL(0x5d053d5a,0xd19448d6), + LL(0xd3210a04,0x1ecc8cb9),LL(0xdafb5269,0x6bc7d463),LL(0x67c3489f,0x3e59b10a),LL(0x65641e1b,0x1769788c), LL(0xbd6cb838,0x8a53b82d),LL(0x236d5f22,0x7066d6e6),LL(0x6908536e,0x03aa1c61),LL(0x66ae9809,0xc971da0d), + LL(0xc49a2fac,0x01b3a86b),LL(0x3092e77a,0x3b8420c0),LL(0x7d6fb556,0x02057300),LL(0xbff40a87,0x6941b2a1), LL(0x0658ff2a,0x140b6308),LL(0x3424ab36,0x87804363),LL(0x5751e299,0x0253bd51),LL(0x449c3e3a,0xc75bcd76), + LL(0x7f8f875d,0x92eb4090),LL(0x56c26bbf,0x9c9d754e),LL(0x8110bbe7,0x158cea61),LL(0x745f91ea,0x62a6b802), LL(0xc6e7394b,0xa79c41aa),LL(0xad57ef10,0x445b6a83),LL(0x6ea6f40c,0x0c5277eb),LL(0x88633365,0x319fe96b), + LL(0x385f63cb,0x0b0fc61f),LL(0x22bdd127,0x41250c84),LL(0x09e942c2,0x67d153f1),LL(0xc021ad5d,0x60920d08), LL(0x724d81a5,0x229f5746),LL(0x5bba3299,0xb7ffb892),LL(0xde413032,0x518c51a1),LL(0x3c2fd94c,0x2a9bfe77), + LL(0x3191f4fd,0xcbcde239),LL(0xd3d6ada1,0x43093e16),LL(0x58769606,0x184579f3),LL(0xd236625c,0x2c94a8b3), LL(0x5c437d8e,0x6922b9c0),LL(0xd8d9f3c8,0x3d4ae423),LL(0x2e7090a2,0xf72c31c1),LL(0xd76a55bd,0x4ac3f5f3), + LL(0x6b6af991,0x342508fc),LL(0x1b5cebbd,0x0d527100),LL(0xdd440dd7,0xb84740d0),LL(0x780162fd,0x748ef841), LL(0xdfc6fafb,0xa8dbfe0e),LL(0xf7300f27,0xeadfdf05),LL(0xfeba4ec9,0x7d06555f),LL(0x9e25fa97,0x12c56f83), + LL(0xd39b8c34,0x77f84203),LL(0x3125eddb,0xed8b1be6),LL(0xf6e39dc5,0x5bbf2441),LL(0x6a5d678a,0xb00f6ee6), LL(0x57d0ea99,0xba456ecf),LL(0x17e06c43,0xdcae0f58),LL(0x0f5b4baa,0x01643de4),LL(0xd161b9be,0x2c324341), + LL(0xe126d468,0x80177f55),LL(0x76748e09,0xed325f1f),LL(0xcfa9bdc2,0x6116004a),LL(0x3a9fb468,0x2d8607e6), LL(0x6009d660,0x0e573e27),LL(0x8d10c5a1,0x3a525d2e),LL(0x3b9009a0,0xd26cb45c),LL(0xde9d7448,0xb6b0cdc0), + LL(0xe1337c26,0x949c9976),LL(0xd73d68e5,0x6faadebd),LL(0xf1b768d9,0x9e158614),LL(0x9cc4f069,0x22dfa557), LL(0xbe93c6d6,0xccd6da17),LL(0xa504f5b9,0x24866c61),LL(0x8d694da1,0x2121353c),LL(0x0140b8c6,0x1c6ca580), + LL(0xe964021e,0xc245ad8c),LL(0x032b82b3,0xb83bffba),LL(0x47ef9898,0xfaa220c6),LL(0x982c948a,0x7e8d3ac6), LL(0xbc2d124a,0x1faa2091),LL(0x05b15ff4,0xbd54c3dd),LL(0xc87c6fb7,0x386bf3ab),LL(0xfdeb6f66,0xfb2b0563), + LL(0x5b45afb4,0x4e77c557),LL(0xefb8912d,0xe9ded649),LL(0x42f6e557,0x7ec9bbf5),LL(0x62671f00,0x2570dfff), LL(0x88e084bd,0x2b3bfb78),LL(0xf37fe5b4,0xa024b238),LL(0x95649aee,0x44e7dc04),LL(0x5e7ec1d8,0x498ca255), + LL(0xaaa07e86,0x3bc766ea),LL(0xf3608586,0x0db6facb),LL(0xbdc259c8,0xbadd2549),LL(0x041c649f,0x95af3c6e), LL(0x02e30afb,0xb36a928c),LL(0x008a88b8,0x9b5356ad),LL(0xcf1d9e9d,0x4b67a5f1),LL(0xa5d8d8ce,0xc6542e47), + LL(0x7adfb6cc,0x73061fe8),LL(0x98678141,0xcc826fd3),LL(0x3c80515a,0x00e758b1),LL(0x41485083,0x6afe3247), LL(0xb6ae8a75,0x0fcb08b9),LL(0x4acf51e1,0xb8cf388d),LL(0x6961b9d6,0x344a5560),LL(0x6a97fd0c,0x1a6778b8), + LL(0xecc4c7e3,0xd840fdc1),LL(0x16db68cc,0xde9fe47d),LL(0xa3e216aa,0xe95f89de),LL(0x9594a8be,0x84f1a6a4), LL(0x5a7b162b,0x7ddc7d72),LL(0xadc817a3,0xc5cfda19),LL(0x78b58d46,0x80a5d350),LL(0x82978f19,0x93365b13), + LL(0x26a1fc90,0x2e44d225),LL(0x4d70705d,0x0d6d10d2),LL(0xd70c45f4,0xd94b6b10),LL(0xb216c079,0x0f201022), LL(0x658fde41,0xcec966c5),LL(0x7e27601d,0xa8d2bc7d),LL(0xff230be7,0xbfcce3e1),LL(0x0033ffb5,0x3394ff6b), + LL(0x8132c9af,0xd890c509),LL(0x361e7868,0xaac4b0eb),LL(0xe82d15aa,0x5194ded3),LL(0x23ae6b7d,0x4550bd2e), LL(0xea5399d4,0x3fda318e),LL(0x91638b80,0xd989bffa),LL(0xa14aa12d,0x5ea124d0),LL(0x3667b944,0x1fb1b899), + LL(0x44c44d6a,0x95ec7969),LL(0x57e86137,0x91df144a),LL(0x73adac44,0x915fd620),LL(0x59a83801,0x8f01732d), LL(0x3aa0a633,0xec579d25),LL(0xc9d6d59c,0x06de5e7c),LL(0xb1ef8010,0xc132f958),LL(0xe65c1a02,0x29476f96), + LL(0xd34c3565,0x336a77c0),LL(0x1b9f1e9e,0xef1105b2),LL(0xf9e08002,0x63e6d08b),LL(0xc613809e,0x9aff2f21), LL(0x3a80e75d,0xb5754f85),LL(0x6bbda681,0xde71853e),LL(0x8197fd7a,0x86f041df),LL(0x127817fa,0x8b332e08), + LL(0xb9c20cda,0x05d99be8),LL(0xd5cd0c98,0x89f7aad5),LL(0x5bb94183,0x7ef936fe),LL(0xb05cd7f2,0x92ca0753), LL(0x74a1e035,0x9d65db11),LL(0x13eaea92,0x02628cc8),LL(0x49e4fbf2,0xf2d9e242),LL(0xe384f8b7,0x94fdfd9b), + LL(0x63428c6b,0x65f56054),LL(0x90b409a5,0x2f7205b2),LL(0xff45ae11,0xf778bb78),LL(0xc5ee53b2,0xa13045be), LL(0x03ef77fe,0xe00a14ff),LL(0xffef8bef,0x689cd59f),LL(0x1e9ade22,0x3578f0ed),LL(0x6268b6a8,0xe99f3ec0), + LL(0xea1b3c3e,0xa2057d91),LL(0xb8823a4a,0x2d1a7053),LL(0x2cca451e,0xabbb336a),LL(0x2218bb5d,0xcd2466e3), LL(0xc8cb762d,0x3ac1f42f),LL(0x7690211f,0x7e312aae),LL(0x45d07450,0xebb9bd73),LL(0x46c2213f,0x207c4b82), + LL(0x375913ec,0x99d425c1),LL(0x67908220,0x94e45e96),LL(0xcd67dbf6,0xc08f3087),LL(0xc0887056,0xa5670fbe), LL(0x66f5b8fc,0x6717b64a),LL(0x786fec28,0xd5a56aea),LL(0xc0ff4952,0xa8c3f55f),LL(0x457ac49b,0xa77fefae), + LL(0x98379d44,0x29882d7c),LL(0x509edc8a,0xd000bdfb),LL(0xe66fe464,0xc6f95979),LL(0xfa61bde0,0x504a6115), LL(0xeffea31a,0x56b3b871),LL(0xf0c21a54,0x2d3de26d),LL(0x834753bf,0x21dbff31),LL(0x69269d86,0xe67ecf49), + LL(0x151fe690,0x7a176952),LL(0x7f2adb5f,0x03515804),LL(0xd1b62a8d,0xee794b15),LL(0xaae454e6,0xf004ceec), LL(0xf0386fac,0x0897ea7c),LL(0xd1fca751,0x3b62ff12),LL(0x1b7a04ec,0x154181df),LL(0xfb5847ec,0x2008e04a), + LL(0x41dbd772,0xd147148e),LL(0x22942654,0x2b419f73),LL(0xe9c544f7,0x669f30d3),LL(0xc8540149,0x52a2c223), LL(0x634dfb02,0x5da9ee14),LL(0xf47869f3,0x5f074ff0),LL(0xa3933acc,0x74ee878d),LL(0x4fe35ed1,0xe6510651), + LL(0xf1012e7a,0xb3eb9482),LL(0xa8a566ae,0x51013cc0),LL(0x47c00d3b,0xdd5e9243),LL(0x946bb0e5,0x7fde089d), LL(0xc731b4b3,0x030754fe),LL(0x99fda062,0x12a136a4),LL(0x5a1a35bc,0x7c1064b8),LL(0x446c84ef,0xbf1f5763), + LL(0xa16d4b34,0xed29a56d),LL(0xdca21c4f,0x7fba9d09),LL(0x6d8de486,0x66d7ac00),LL(0x73a2a5e1,0x60061987), LL(0x9da28ff0,0x8b400f86),LL(0x43c4599c,0x3133f708),LL(0xee28cb0d,0x9911c9b8),LL(0x8e0af61d,0xcd7e2874), + LL(0x72ed91fc,0x5a85f0f2),LL(0x9cd4a373,0x85214f31),LL(0x1925253c,0x881fe5be),LL(0x91e8bc76,0xd8dc98e0), LL(0x585cc3a2,0x7120affe),LL(0x735bf97a,0x724952ed),LL(0x3eb34581,0x5581e7dc),LL(0xe52ee57d,0x5cbff4f2), + LL(0x87d8cc7b,0x8d320a0e),LL(0xf1d280d0,0x9beaa7f3),LL(0x9beec704,0x7a0b9571),LL(0x5b7f0057,0x9126332e), LL(0x8ed3bd6d,0x01fbc1b4),LL(0xd945eb24,0x35bb2c12),LL(0x9a8ae255,0x6404694e),LL(0x8d6abfb3,0xb6092eec), + LL(0xcc058865,0x4d76143f),LL(0x6e249922,0x7b0a5af2),LL(0x6a50d353,0x8aef9440),LL(0x64f0e07a,0xe11e4bcc), LL(0xa14a90fa,0x4472993a),LL(0xba0c51d4,0x7706e20c),LL(0x1532672d,0xf403292f),LL(0x21829382,0x52573bfa), + LL(0x3b5bdb83,0x6a7bb6a9),LL(0xa4a72318,0x08da65c0),LL(0x63eb065f,0xc58d22aa),LL(0x1b15d685,0x1717596c), LL(0xb266d88b,0x112df0d0),LL(0x5941945a,0xf688ae97),LL(0x7c292cac,0x487386e3),LL(0x57d6985c,0x42f3b50d), + LL(0x6a90fc34,0x6da4f998),LL(0x65ca8a8d,0xc8f257d3),LL(0x6951f762,0xc2feabca),LL(0x74c323ac,0xe1bc81d0), LL(0x251a2a12,0x1bc68f67),LL(0xbe8a70dc,0x10d86587),LL(0xf0f84d2e,0xd648af7f),LL(0x6a43ac92,0xf0aa9ebc), + LL(0x27596893,0x69e3be04),LL(0x45bf452b,0xb6bb02a6),LL(0xf4c698c8,0x0875c11a),LL(0xbece3794,0x6652b5c7), LL(0x4f5c0499,0x7b3755fd),LL(0xb5532b38,0x6ea16558),LL(0xa2e96ef7,0xd1c69889),LL(0x61ed8f48,0x9c773c3a), + LL(0x9b323abc,0x2b653a40),LL(0xf0e1d791,0xe26605e1),LL(0x4a87157a,0x45d41064),LL(0xcbbce616,0x8f9a78b7), LL(0xc407eddd,0xcf1e44aa),LL(0xa35b964f,0x81ddd1d8),LL(0xfd083999,0x473e339e),LL(0x8e796802,0x6c94bdde), + LL(0x8545d185,0x5a304ada),LL(0x738bb8cb,0x82ae44ea),LL(0xdf87e10e,0x628a35e3),LL(0xa15b9fe3,0xd3624f3d), LL(0x14be4254,0xcc44209b),LL(0xbdbc2ea5,0x7d0efcbc),LL(0x04c37bbe,0x1f603362),LL(0x56a5852c,0x21f363f5), + LL(0xa8501550,0xa1503d1c),LL(0xd8ab10bb,0x2251e0e1),LL(0x6961c51c,0xde129c96),LL(0x81910f68,0x1f7246a4), LL(0x5f2591f2,0x2eb744ee),LL(0x5e627157,0x3c47d33f),LL(0x22f3bd68,0x4d6d62c9),LL(0xcb8df856,0x6120a64b), + LL(0x7b5d07df,0x3a9ac6c0),LL(0x7ef39783,0xa92b9558),LL(0xab3a9b4f,0xe128a134),LL(0xb1252f05,0x41c18807), LL(0x80ba9b1c,0xfc7ed089),LL(0xc532a9dd,0xac8dc6de),LL(0x55246809,0xbf829cef),LL(0x5b4ee80f,0x101b784f), + LL(0xb6f11603,0xc09945bb),LL(0x41d2801e,0x57b09dbe),LL(0xa97534a8,0xfba5202f),LL(0xc17b9614,0x7fd8ae5f), LL(0x78308435,0xa50ba666),LL(0xd3868c4d,0x9572f77c),LL(0x2dd7aab0,0x0cef7bfd),LL(0x2c7c79ff,0xe7958e08), + LL(0x25346689,0x81262e42),LL(0xb07c7004,0x716da290),LL(0xb7950ee3,0x35f911ea),LL(0x261d21b5,0x6fd72969), LL(0x08b640d3,0x52389803),LL(0x887f12a1,0x5b0026ee),LL(0x742e9311,0x20e21660),LL(0x5ff77ff7,0x0ef6d541), + LL(0xf9c41135,0x969127f0),LL(0x68a64993,0xf21d60c9),LL(0xe541875c,0x656e5d0c),LL(0xa1d3c233,0xf1e0f84e), LL(0x06002d60,0x9bcca359),LL(0x06191552,0xbe2da60c),LL(0x61181ec3,0x5da8bbae),LL(0x65806f19,0x9f04b823), + LL(0xd4b79bb8,0xf1604a7d),LL(0x52c878c8,0xaee806fb),LL(0x8d47b8e8,0x34144f11),LL(0x949f9054,0x72edf52b), LL(0x2127015a,0xebfca84e),LL(0x9cb7cef3,0x9051d0c0),LL(0x296deec8,0x86e8fe58),LL(0x41010d74,0x33b28188), +}, +/* digit=26 base_pwr=2^182 */ +{ + LL(0x171b445f,0x01079383),LL(0x8131ad4c,0x9bcf21e3),LL(0xc93987e8,0x8cdfe205),LL(0xc92e8c8f,0xe63f4152), LL(0x30add43d,0x729462a9),LL(0xc980f05a,0x62ebb143),LL(0x3b06e968,0x4f3954e5),LL(0x242cf6b1,0xfe1d75ad), + LL(0xaf8685c8,0x5f95c6c7),LL(0x2f8f01aa,0xd4c1c8ce),LL(0x2574692a,0xc44bbe32),LL(0xd4a4a068,0xb8003478), LL(0x2eca3cdb,0x7c8fc6e5),LL(0xec04d399,0xea1db16b),LL(0x8f2bc5cf,0xb05bc82e),LL(0xf44793d2,0x763d517f), + LL(0x08bd98d0,0x4451c1b8),LL(0x6575f240,0x644b1cd4),LL(0x7375d270,0x6907eb33),LL(0xfa2286bd,0x56c8bebd), LL(0xc4632b46,0xc713d2ac),LL(0xafd60242,0x17da427a),LL(0xc95c7546,0x313065b7),LL(0xbf17a3de,0xf8239898), + LL(0x4c830320,0xf3b7963f),LL(0x903203e3,0x842c7aa0),LL(0xe7327afb,0xaf22ca0a),LL(0x967609b6,0x38e13092), LL(0x757558f1,0x73b8fb62),LL(0xf7eca8c1,0x3cc3e831),LL(0xf6331627,0xe4174474),LL(0xc3c40234,0xa77989ca), + LL(0x44a081e0,0xe5fd17a1),LL(0xb70e296a,0xd797fb7d),LL(0x481f719c,0x2b472b30),LL(0xfe6f8c52,0x0e632a98), LL(0xc5f0c284,0x89ccd116),LL(0x2d987c62,0xf51088af),LL(0x4c2de6cf,0x2a2bccda),LL(0xf679f0f9,0x810f9efe), + LL(0x7ffe4b3e,0xb0f394b9),LL(0xe5fa5d21,0x0b691d21),LL(0x9dfbbc75,0xb0bd7747),LL(0xfaf78b00,0xd2830fda), LL(0x52434f57,0xf78c249c),LL(0x98096dab,0x4b1f7545),LL(0x8ff8c0b3,0x73bf6f94),LL(0x454e134c,0x34aef03d), + LL(0xb7ac7ec5,0xf8d151f4),LL(0xe50da7d5,0xd6ceb95a),LL(0xdc3a0eb8,0xa1b492b0),LL(0xb3dd2863,0x75157b69), LL(0xc5413d62,0xe2c4c74e),LL(0xbc5fc4c7,0xbe329ff7),LL(0x60fa9dda,0x835a2aea),LL(0x7445cb87,0xf117f5ad), + LL(0xb0166f7a,0xae8317f4),LL(0xceec74e6,0xfbd3e3f7),LL(0xe0874bfd,0xfdb516ac),LL(0xc681f3a3,0x3d846019), LL(0x7c1620b0,0x0b12ee5c),LL(0x2b63c501,0xba68b4dd),LL(0x6668c51e,0xac03cd32),LL(0x4e0bcb5b,0x2a6279f7), + LL(0x6ae85c10,0x17bd69b0),LL(0x1dfdd3a6,0x72946979),LL(0x2c078bec,0xd9a03268),LL(0xbfd68a52,0x41c6a658), LL(0x0e023900,0xcdea1024),LL(0xb10d144d,0xbaeec121),LL(0x058ab8dc,0x5a600e74),LL(0xbb89ccdd,0x1333af21), + LL(0x3aaba1f1,0xdf25eae0),LL(0x3b7144cf,0x2cada16e),LL(0x71ab98bc,0x657ee27d),LL(0x7a6fc96e,0x99088b4c), LL(0x3549dbd4,0x05d5c0a0),LL(0xf158c3ac,0x42cbdf8f),LL(0x87edd685,0x3fb6b3b0),LL(0x86f064d0,0x22071cf6), + LL(0xff2811e5,0xd2d6721f),LL(0xfe7fae8c,0xdb81b703),LL(0xd3f1f7bb,0x3cfb74ef),LL(0x16cdeb5d,0x0cdbcd76), LL(0x566a808c,0x4f39642a),LL(0x340064d6,0x02b74454),LL(0x0528fa6f,0xfabbadca),LL(0xd3fc0bb6,0xe4c3074c), + LL(0xb796d219,0xb32cb8b0),LL(0x34741dd9,0xc3e95f4f),LL(0x68edf6f5,0x87212125),LL(0xa2b9cb8e,0x7a03aee4), LL(0xf53a89aa,0x0cd3c376),LL(0x948a28dc,0x0d8af9b1),LL(0x902ab04f,0xcf86a3f4),LL(0x7f42002d,0x8aacb62a), + LL(0xf62ffd52,0x106985eb),LL(0x5797bf10,0xe670b54e),LL(0xc5e30aef,0x4b405209),LL(0x4365b5e9,0x12c97a20), LL(0x1fe32093,0x104646ce),LL(0x3907a8c9,0x13cb4ff6),LL(0xd46e726b,0x8b9f30d1),LL(0xaba0f499,0xe1985e21), + LL(0x10a230cd,0xc573dea9),LL(0xcd30f947,0x24f46a93),LL(0xabe2010a,0xf2623fcf),LL(0x73f00e4f,0x3f278cb2), LL(0x50b920eb,0xed55c67d),LL(0x8e760571,0xf1cb9a2d),LL(0x0895b709,0x7c50d109),LL(0x190d4369,0x4207cf07), + LL(0xc4127fe1,0x3b027e81),LL(0x3ae9c566,0xa9f8b9ad),LL(0xacbfbba5,0x5ab10851),LL(0x569556f5,0xa747d648), LL(0x2ba97bf7,0xcc172b5c),LL(0xbcfa3324,0x15e0f77d),LL(0x7686279d,0xa345b797),LL(0xe38003d3,0x5a723480), + LL(0x8f5fcda8,0xfd8e139f),LL(0xbdee5bfd,0xf3e558c4),LL(0xe33f9f77,0xd76cbaf4),LL(0x71771969,0x3a4c97a4), LL(0xf6dce6a7,0xda27e84b),LL(0x13e6c2d1,0xff373d96),LL(0xd759a6e9,0xf115193c),LL(0x63d2262c,0x3f9b7025), + LL(0x317cd062,0xd9764a31),LL(0x199f8332,0x30779d8e),LL(0x16b11b0b,0xd8074106),LL(0x78aeaed8,0x7917ab9f), LL(0x28fb1d8e,0xb67a9cbe),LL(0x136eda33,0x2e313563),LL(0xa371a86c,0x010b7069),LL(0x6744e6b7,0x44d90fa2), + LL(0xd6b3e243,0x68190867),LL(0x59048c48,0x9fe6cd9d),LL(0x95731538,0xb900b028),LL(0x32cae04f,0xa012062f), LL(0x9399d082,0x8107c8bc),LL(0x41df12e2,0x47e8c54a),LL(0xb6ef3f73,0x14ba5117),LL(0x81362f0b,0x22260bea), + LL(0x1a18cc20,0x90ea261e),LL(0x2321d636,0x2192999f),LL(0xe311b6a0,0xef64d314),LL(0x3b54a1f5,0xd7401e4c), LL(0x6fbca2ba,0x19019983),LL(0x8fbffc4b,0x46ad3293),LL(0x3786bf40,0xa142d3f6),LL(0xb67039fc,0xeb5cbc26), + LL(0x252bd479,0x9cb0ae6c),LL(0x12b5848f,0x05e0f88a),LL(0xa5c97663,0x78f6d2b2),LL(0xc162225c,0x6f6e149b), LL(0xde601a89,0xe602235c),LL(0xf373be1f,0xd17bbe98),LL(0xa8471827,0xcaf49a5b),LL(0x18aaa116,0x7e1a0a85), + LL(0x270580c3,0x6c833196),LL(0xf1c98a14,0x1e233839),LL(0xae34e0a5,0x67b2f7b4),LL(0xd8ce7289,0x47ac8745), LL(0x100dd467,0x2b74779a),LL(0x4ee50d09,0x274a4337),LL(0x83608bc9,0x603dcf13),LL(0xc89e8388,0xcd9da6c3), + LL(0x355116ac,0x2660199f),LL(0xb6d18eed,0xcc38bb59),LL(0x2f4bc071,0x3075f31f),LL(0x265dc57e,0x9774457f), LL(0xc6db88bb,0x06a6a9c8),LL(0x4ec98e04,0x6429d07f),LL(0x05ecaa8b,0x8d05e57b),LL(0x7872ea7b,0x20f140b1), + LL(0xca494693,0xdf8c0f09),LL(0xf252e909,0x48d3a020),LL(0x57b14b12,0x4c5c29af),LL(0xbf47ad1c,0x7e6fa37d), LL(0x49a0c938,0x66e7b506),LL(0x6be5f41f,0xb72c0d48),LL(0xb2359412,0x6a6242b8),LL(0x8e859480,0xcd35c774), + LL(0x87baa627,0x12536fea),LL(0xf72aa680,0x58c1fec1),LL(0x601e5dc9,0x6c29b637),LL(0xde9e01b9,0x9e3c3c1c), LL(0x2bcfe0b0,0xefc8127b),LL(0x2a12f50d,0x35107102),LL(0x4879b397,0x6ccd6cb1),LL(0xf8a82f21,0xf792f804), + LL(0xa9b46402,0x509d4804),LL(0xc10f0850,0xedddf85d),LL(0x4b6208aa,0x928410dc),LL(0x391012dc,0xf6229c46), LL(0x7727b9b6,0xc5a7c41e),LL(0xaa444842,0x289e4e4b),LL(0xe9a947ea,0x049ba1d9),LL(0x83c8debc,0x44f9e47f), + LL(0x611f8b8e,0xfa77a1fe),LL(0xf518f427,0xfd2e416a),LL(0x114ebac3,0xc5fffa70),LL(0x5d89697b,0xfe57c4e9), LL(0xb1aaf613,0xfdd053ac),LL(0xea585a45,0x31df210f),LL(0x24985034,0x318cc10e),LL(0x5f1d6130,0x1a38efd1), + LL(0x0b1e9e21,0xbf86f237),LL(0x1dbe88aa,0xb258514d),LL(0x90c1baf9,0x1e38a588),LL(0xbdb9b692,0x2936a01e), LL(0x6dd5b20c,0xd576de98),LL(0x70f98ecf,0xb586bf71),LL(0xc42d2fd7,0xcccf0f12),LL(0xfb35bd7b,0x8717e61c), + LL(0x35e6fc06,0x8b1e5722),LL(0x0b3e13d5,0x3477728f),LL(0xaa8a7372,0x150c294d),LL(0x3bfa528a,0xc0291d43), LL(0xcec5a196,0xc6c8bc67),LL(0x5c2e8a7c,0xdeeb31e4),LL(0xfb6e1c51,0xba93e244),LL(0x2e28e156,0xb9f8b71b), + LL(0x968a2ab9,0xce65a287),LL(0x46bbcb1f,0xe3c5ce69),LL(0xe7ae3f30,0xf8c835b9),LL(0xff72b82b,0x16bbee26), LL(0xfd42cd22,0x665e2017),LL(0xf8b1d2a0,0x1e139970),LL(0x79204932,0x125cda29),LL(0x49c3bee5,0x7aee94a5), + LL(0x89821a66,0x68c70160),LL(0x8f981669,0xf7c37678),LL(0x48cc3645,0xd90829fc),LL(0xd70addfc,0x346af049), LL(0x370bf29c,0x2057b232),LL(0x42e650ee,0xf90c73ce),LL(0xa126ab90,0xe03386ea),LL(0x975a087b,0x0e266e7e), + LL(0x0fca65d9,0x80578eb9),LL(0x16af45b8,0x7e2989ea),LL(0xcac75a4e,0x7438212d),LL(0x4fef36b8,0x38c7ca39), LL(0xd402676a,0x8650c494),LL(0xf72c7c48,0x26ab5a66),LL(0xce3a464e,0x4e6cb426),LL(0x2b72f841,0xf8f99896), + LL(0x1a335cc8,0x8c318491),LL(0x6a5913e4,0x563459ba),LL(0xc7b32919,0x1b920d61),LL(0xa02425ad,0x805ab8b6), LL(0x8d006086,0x2ac512da),LL(0xbcf5c0fd,0x6ca4846a),LL(0xac2138d7,0xafea51d8),LL(0x344cd443,0xcb647545), + LL(0xbd7d9040,0x0429ee8f),LL(0x819b9c96,0xee66a2de),LL(0xdea7d744,0x54f9ec25),LL(0x671721bb,0x2ffea642), LL(0x114344ea,0x4f19dbd1),LL(0xfd0dbc8b,0x04304536),LL(0x29ec7f91,0x014b50aa),LL(0xbb06014d,0xb5fc22fe), + LL(0x1ee682e0,0x60d963a9),LL(0xfe85c727,0xdf48abc0),LL(0x2e707c2d,0x0cadba13),LL(0xa645aeff,0xde608d3a), LL(0xedafd883,0x05f1c28b),LL(0xbd94de1f,0x3c362ede),LL(0x13593e41,0x8dd0629d),LL(0x766d6eaf,0x0a5e736f), + LL(0xf68cf9d1,0xbfa92311),LL(0xc1797556,0xa4f9ef87),LL(0x5601c209,0x10d75a1f),LL(0x09b07361,0x651c374c), LL(0x88b5cead,0x49950b58),LL(0x6fa9dbaa,0x0ef00058),LL(0x4e15f33a,0xf51ddc26),LL(0x2ef46140,0x1f8b5ca6), + LL(0xee9523f0,0x343ac0a3),LL(0x975ea978,0xbb75eab2),LL(0x107387f4,0x1bccf332),LL(0x9ab0062e,0x790f9259), LL(0x1e4f6a5f,0xf1a363ad),LL(0x62519a50,0x06e08b84),LL(0x7265f1ee,0x60915187),LL(0x93ae985e,0x6a80ca34), + LL(0xaaba4864,0x81b29768),LL(0x8d52a7d6,0xb13cabf2),LL(0x8ead03f1,0xb5c36348),LL(0x81c7c1c0,0xc932ad95), LL(0xcae1e27b,0x5452708e),LL(0x1b0df648,0x9dac4269),LL(0xdfcdb8bc,0x233e3f0c),LL(0xec540174,0xe6ceccdf), + LL(0x95081181,0xbd0d845e),LL(0x699355d5,0xcc8a7920),LL(0xc3b375a8,0x111c0f6d),LL(0xfd51e0dc,0xfd95bc6b), LL(0x6888523a,0x4a106a26),LL(0xcb01a06d,0x4d142bd6),LL(0xadb9b397,0x79bfd289),LL(0xe9863914,0x0bdbfb94), + LL(0x1660f6a6,0x29d8a229),LL(0x551c042d,0x7f6abcd6),LL(0x0ac3ffe8,0x13039deb),LL(0xec8523fb,0xa01be628), LL(0x0ca1c328,0x6ea34103),LL(0xb903928e,0xc74114bd),LL(0x9e9144b0,0x8aa4ff4e),LL(0x7f9a4b17,0x7064091f), + LL(0xe447f2c4,0xa3f4f521),LL(0x604291f0,0x81b8da7a),LL(0x7d5926de,0xd680bc46),LL(0x34a1202f,0x84f21fd5), LL(0x4e9df3d8,0x1d1e3181),LL(0x39ab8d34,0x1ca4861a),LL(0x5b19aa4a,0x809ddeec),LL(0x4d329366,0x59f72f7e), + LL(0x386d5087,0xa2f93f41),LL(0xdd67d64f,0x40bf739c),LL(0x66702158,0xb4494205),LL(0x73b1e178,0xc33c65be), LL(0x38ca6153,0xcdcd657c),LL(0xdc791976,0x97f4519a),LL(0xcd6e1f39,0xcc7c7f29),LL(0x7e3c3932,0x38de9cfb), + LL(0x7b793f85,0xe448eba3),LL(0xf067e914,0xe9f8dbf9),LL(0xf114ae87,0xc0390266),LL(0xcd6a8e2a,0x39ed75a7), LL(0x7ffba390,0xadb14848),LL(0x6af9bc09,0x67f8cb8b),LL(0x9c7476db,0x322c3848),LL(0x52a538d6,0xa320fecf), + LL(0xb2aced2b,0xe0493002),LL(0x616bd430,0xdfba1809),LL(0xc331be70,0x531c4644),LL(0x90d2e450,0xbc04d32e), LL(0x0f9f142d,0x1805a0d1),LL(0x47ee5a23,0x2c44a0c5),LL(0x3989b4e3,0x31875a43),LL(0x0c063481,0x6b1949fd), + LL(0xbe0f4492,0x2dfb9e08),LL(0xe9d5e517,0x3ff0da03),LL(0xf79466a8,0x03dbe9a1),LL(0x15ea9932,0x0b87bcd0), LL(0xab1f58ab,0xeb64fc83),LL(0x817edc8a,0x6d9598da),LL(0x1d3b67e5,0x699cff66),LL(0x92635853,0x645c0f29), + LL(0xeabaf21c,0x253cdd82),LL(0x2241659e,0x82b9602a),LL(0x2d9f7091,0x2cae07ec),LL(0x8b48cd9b,0xbe4c720c), LL(0x6f08d6c9,0x6ce5bc03),LL(0xaf10bf40,0x36e8a997),LL(0x3e10ff12,0x83422d21),LL(0xbcc12494,0x7b26d3eb), + LL(0xc9469ad6,0xb240d2d0),LL(0x30afa05b,0xc4a11b4d),LL(0xdd6ba286,0x4b604ace),LL(0x3ee2864c,0x18486600), LL(0x8d9ce5be,0x5869d6ba),LL(0xff4bfb0d,0x0d8f68c5),LL(0x5700cf73,0xb69f210b),LL(0x6d37c135,0x61f6653a), + LL(0x5aff5a48,0xff3d432b),LL(0x72ba3a69,0x0d81c4b9),LL(0xfa1899ef,0xee879ae9),LL(0x2d6acafd,0xbac7e2a0), LL(0x1c664399,0xd6d93f6c),LL(0x5bcb135d,0x4c288de1),LL(0x9dab7cbf,0x83031dab),LL(0x3abbf5f0,0xfe23feb0), + LL(0xcdedca85,0x9f1b2466),LL(0x1a09538c,0x140bb710),LL(0x5e11115d,0xac8ae851),LL(0x6f03f59e,0x0d63ff67), LL(0x7d234afb,0x755e5551),LL(0x7e208fc1,0x61c2db4e),LL(0xf28a4b5d,0xaa9859ce),LL(0x34af030f,0xbdd6d4fc), + LL(0x3be01cb1,0xd1c4a26d),LL(0x243aa07c,0x9ba14ffc),LL(0xb2503502,0xf95cd3a9),LL(0x7d2a93ab,0xe379bc06), LL(0xd4ca8d68,0x3efc18e9),LL(0x80bb412a,0x083558ec),LL(0x9645a968,0xd903b940),LL(0x9ba6054f,0xa499f0b6), + LL(0xb8349abe,0x208b573c),LL(0x30b4fc1c,0x3baab3e5),LL(0xcb524990,0x87e978ba),LL(0xccdf0e80,0x3524194e), LL(0x7d4bcc42,0x62711725),LL(0xb90109ba,0xe90a3d9b),LL(0x1323e1e0,0x3b1bdd57),LL(0x5eae1599,0xb78e9bd5), + LL(0x9e03d278,0x0794b746),LL(0xd70e6297,0x80178605),LL(0x99c97855,0x171792f8),LL(0xf5a86b5c,0x11b393ee), LL(0xd8884f27,0x48ef6582),LL(0xbf19ba5f,0xbd44737a),LL(0xa42062c6,0x8698de4c),LL(0x61ce9c54,0x8975eb80), + LL(0xd7fe71f3,0xd50e57c7),LL(0xbc97ce38,0x15342190),LL(0x4df07b63,0x51bda2de),LL(0x200eb87d,0xba12aeae), LL(0xa9b4f8f6,0xabe135d2),LL(0xfad6d99c,0x04619d65),LL(0x7994937c,0x4a6683a7),LL(0x6f94f09a,0x7a778c8b), + LL(0x20a71b89,0x8c508623),LL(0x1c229165,0x241a2aed),LL(0xaaf83a99,0x352be595),LL(0x1562bac8,0x9fbfee7f), LL(0x5c4017e3,0xeaf658b9),LL(0x15120b86,0x1dc7f9e0),LL(0x4c034d6f,0xd84f13dd),LL(0xeaea3038,0x283dd737), + LL(0xcd85d6a2,0x197f2609),LL(0xfae60177,0x6ebbc345),LL(0x4e12fede,0xb80f031b),LL(0x07a2186b,0xde55d0c2), LL(0x24dcdd5a,0x1fb3e37f),LL(0x7ed191fb,0x8d602da5),LL(0x76023e0d,0x108fb056),LL(0x459c20c0,0x70178c71), + LL(0x3fe54cf0,0xfad5a386),LL(0x02bbb475,0xa4a3ec4f),LL(0x919d94d7,0x1aa5ec20),LL(0xa81e4ab3,0x5d3b63b5), LL(0x5ad3d2af,0x7fa733d8),LL(0xd1ac7a37,0xfbc586dd),LL(0x40779614,0x282925de),LL(0xe74a242a,0xfe0ffffb), + LL(0x906151e5,0x3f39e67f),LL(0x55e10649,0xcea27f5f),LL(0xc17cf7b7,0xdca1d4e1),LL(0x2fe2362d,0x0c326d12), LL(0x7dd35df3,0x05f7ac33),LL(0xc396dbdf,0x0c3b7639),LL(0x03b7db1c,0x0912f5ac),LL(0x5c9ed4a9,0x9dea4b70), + LL(0xaae3f639,0x475e6e53),LL(0xfc278bac,0xfaba0e7c),LL(0x9490375f,0x16f9e221),LL(0xa5a7ed0a,0xaebf9746), LL(0xf41ad5d6,0x45f9af3f),LL(0xb2e99224,0x03c4623c),LL(0xb3cf56aa,0x82c5bb5c),LL(0x34567ed3,0x64311819), + LL(0x8be489ac,0xec57f211),LL(0xb9a1104b,0x2821895d),LL(0x6064e007,0x610dc875),LL(0x5b20d0fe,0x8e526f3f), LL(0x5b645aee,0x6e71ca77),LL(0x800e10ff,0x3d1dcb9f),LL(0x189cf6de,0x36b51162),LL(0x6bb17353,0x2c5a3e30), + LL(0x2a6c6fbf,0xc186cd3e),LL(0x4bf97906,0xa74516fa),LL(0x279d6901,0x5b4b8f4b),LL(0x2b573743,0x0c4e57b4), LL(0xb6e386b6,0x75fdb229),LL(0x99deac27,0xb46793fd),LL(0xcf712629,0xeeec47ea),LL(0xcbc3b2dd,0xe965f3c4), + LL(0x425c6559,0x8dd1fb83),LL(0x0af06fda,0x7fc00ee6),LL(0x33d956df,0xe98c9225),LL(0x4fbdc8a2,0x0f1ef335), LL(0xb79b8ea2,0x2abb5145),LL(0xbdbff288,0x40fd2945),LL(0xd7185db7,0x6a814ac4),LL(0xc084609a,0xc4329d6f), + LL(0xed1be45d,0xc9ba7b52),LL(0xe4cd2c74,0x891dd20d),LL(0x824139b1,0x5a4d4a7f),LL(0xb873c710,0x66c17716), LL(0x2843c4e0,0x5e5bc141),LL(0xb97eb5bf,0xd5ac4817),LL(0x450c95c7,0xc0f8af54),LL(0x318406c5,0xc91b3fa0), + LL(0xab9d97f8,0x360c340a),LL(0x90a2d611,0xfb57bd07),LL(0xa6a6f7e5,0x4339ae3c),LL(0x2feb8a10,0x9c1fcd2a), LL(0xc7ea7432,0x972bcca9),LL(0x308076f6,0x1b0b924c),LL(0x2a5b4ca5,0x80b2814a),LL(0x61ef3b29,0x2f78f55b), + LL(0xc18a414f,0xf838744a),LL(0x903d0a86,0xc611eaae),LL(0x2a453f55,0x94dabc16),LL(0x14efb279,0xe6f2e3da), LL(0x9320dc3c,0x5b7a6017),LL(0x8df6b5a4,0x692e382f),LL(0x2d40fa90,0x3f5e15e0),LL(0x643dd318,0xc87883ae), + LL(0x53544774,0x511053e4),LL(0x3adba2bc,0x834d0ecc),LL(0xbae371f5,0x4215d7f7),LL(0x6c8663bc,0xfcfd57bf), LL(0xd6901b1d,0xded2383d),LL(0xb5587dc3,0x3b49fbb4),LL(0x07625f62,0xfd44a08d),LL(0x9de9b762,0x3ee4d65b), +}, +/* digit=27 base_pwr=2^189 */ +{ + LL(0x0d63d1fa,0x64e5137d),LL(0x02a9d89f,0x658fc052),LL(0x50436309,0x48894874),LL(0xd598da61,0xe9ae30f8), LL(0x818baf91,0x2ed710d1),LL(0x8b6a0c20,0xe27e9e06),LL(0x1c1a6b44,0x1e28dcfb),LL(0xd6ac57dc,0x883acb64), + LL(0xc2c6ff70,0x8735728d),LL(0xc5dc2235,0x79d6122f),LL(0x19e277f9,0x23f5d003),LL(0xdded8cc7,0x7ee84e25), LL(0x63cd880a,0x91a8afb0),LL(0x3574af60,0x3f3ea7c6),LL(0x02de7f42,0x0cfcdc84),LL(0xb31aa152,0x62d0792f), + LL(0x8a5807ce,0x8e1b4e43),LL(0xe4109a7e,0xad283893),LL(0xafd59dda,0xc30cc9cb),LL(0x3d8d8093,0xf65f36c6), LL(0xa60d32b2,0xdf31469e),LL(0x3e8191c8,0xee93df4b),LL(0x355bdeb5,0x9c1017c5),LL(0x8616aa28,0xd2623185), + LL(0xdec31a21,0xb02c83f9),LL(0x6ad9d573,0x988c8b23),LL(0xa57be365,0x53e983ae),LL(0x646f834e,0xe968734d), LL(0x5da6309b,0x9137ea8f),LL(0xc1f1ce16,0x10f3a624),LL(0xca440921,0x782a9ea2),LL(0x5b46f1b5,0xdf94739e), + LL(0xcce85c9b,0x9f9be006),LL(0xa4c7c2d3,0x360e70d6),LL(0xaefa1e60,0x2cd5beea),LL(0x8c3d2b6d,0x64cf63c0), LL(0xe1cf6f90,0xfb107fa3),LL(0xd5e044e6,0xb7e937c6),LL(0xce34db9f,0x74e8ca78),LL(0x3e210bd0,0x4f8b36c1), + LL(0x34a35ea8,0x1df165a4),LL(0x4d4412f6,0x3418e0f7),LL(0x518836c3,0x5af1f8af),LL(0x130e1965,0x42ceef4d), LL(0x543a1957,0x5560ca0b),LL(0x886cb123,0xc33761e5),LL(0xfe98ed30,0x66624b1f),LL(0x1090997d,0xf772f4bf), + LL(0x4885d410,0xf4e540bb),LL(0x9ba5f8d7,0x7287f810),LL(0xde98dfb1,0x22d0d865),LL(0xbcfbb8a3,0x49ff51a1), LL(0x6bc3012e,0xb6b6fa53),LL(0x170d541d,0x3d31fd72),LL(0x4b0f4966,0x8018724f),LL(0x87dbde07,0x79e7399f), + LL(0xf4f8b16a,0x56f8410e),LL(0xc47b266a,0x97241afe),LL(0x6d9c87c1,0x0a406b8e),LL(0xcd42ab1b,0x803f3e02), LL(0x04dbec69,0x7f0309a8),LL(0x3bbad05f,0xa83b85f7),LL(0xad8e197f,0xc6097273),LL(0x5067adc1,0xc097440e), + LL(0x3524ff16,0x730eafb6),LL(0x823fc6ce,0xd7f9b51e),LL(0x443e4ac0,0x27bd0d32),LL(0x4d66f217,0x40c59ad9), LL(0x17c387a4,0x6c33136f),LL(0xeb86804d,0x5043b8d5),LL(0x675a73c9,0x74970312),LL(0xf16669b6,0x838fdb31), + LL(0x418e7ddd,0xc507b6dd),LL(0x472f19d6,0x39888d93),LL(0x0c27eb4d,0x7eae26be),LL(0xfbabb884,0x17b53ed3), LL(0x2b01ae4f,0xfc27021b),LL(0xcf488682,0x88462e87),LL(0x215e2d87,0xbee096ec),LL(0xd242e29b,0xeb2fea9a), + LL(0xb821fc28,0x5d985b5f),LL(0xdc1e2ad2,0x89d2e197),LL(0x9030ba62,0x55b566b8),LL(0x4f41b1c6,0xe3fd41b5), LL(0xb9a96d61,0xb738ac2e),LL(0x369443f4,0x7f8567ca),LL(0xf803a440,0x8698622d),LL(0x8fe2f4dc,0x2b586236), + LL(0x56b95bce,0xbbcc00c7),LL(0x616da680,0x5ec03906),LL(0x72214252,0x79162ee6),LL(0x86a892d2,0x43132b63), LL(0x2f3263bf,0x4bdd3ff2),LL(0x9cd0a142,0xd5b3733c),LL(0x44415ccb,0x592eaa82),LL(0x8d5474ea,0x663e8924), + LL(0x5236344e,0x8058a25e),LL(0xbda76ee6,0x82e8df9d),LL(0x11cc3d22,0xdcf6efd8),LL(0x3b4ab529,0x00089cda), LL(0xbd38a3db,0x91d3a071),LL(0xef72b925,0x4ea97fc0),LL(0xea3edf75,0x0c9fc15b),LL(0xa4348ed3,0x5a6297cd), + LL(0xce7c42d4,0x0d38ab35),LL(0x82feab10,0x9fd493ef),LL(0x82111b45,0x46056b6d),LL(0x73efc5c3,0xda11dae1), LL(0x5545a7fb,0xdc740278),LL(0x40d507e6,0xbdb2601c),LL(0x7066fa58,0x121dfeeb),LL(0x39ae8c2a,0x214369a8), + LL(0x06e0956c,0x195709cb),LL(0x010cd34b,0x4c9d254f),LL(0x0471a532,0xf51e13f7),LL(0x1e73054d,0xe19d6791), LL(0xdb5c7be3,0xf702a628),LL(0xb24dde05,0xc7141218),LL(0xf29b2e2e,0xdc18233c),LL(0x85342dba,0x3a6bd1e8), + LL(0xb311898c,0x3f747fa0),LL(0xcd0eac65,0xe2a272e4),LL(0xf914d0bc,0x4bba5851),LL(0xc4a43ee3,0x7a1a9660), LL(0xa1c8cde9,0xe5a367ce),LL(0x7271abe3,0x9d958ba9),LL(0x3d1615cd,0xf3ff7eb6),LL(0xf5ae20b0,0xa2280dce), + LL(0xcf640147,0x56dba5c1),LL(0x5e83d118,0xea5a2e3d),LL(0xda24c511,0x04cd6b6d),LL(0xe854d214,0x1c0f4671), LL(0x69565381,0x91a6b7a9),LL(0xdecf1f5b,0xdc966240),LL(0xfcf5d009,0x1b22d21c),LL(0x9021dbd5,0x2a05f641), + LL(0xd4312483,0x8c0ed566),LL(0x643e216f,0x5179a95d),LL(0x17044493,0xcc185fec),LL(0x54991a21,0xb3063339), LL(0x0081a726,0xd801ecdb),LL(0x4fa89bbb,0x0149b0c6),LL(0x4391b6b9,0xafe9065a),LL(0xd633f3a3,0xedc92786), + LL(0xae6a8e13,0xe408c24a),LL(0x9f3897ab,0x85833fde),LL(0xd81a0715,0x43800e7e),LL(0xb44ffc5f,0xde08e346), LL(0xcdeff2e0,0x7094184c),LL(0x165eaed1,0x49f9387b),LL(0x777c468a,0x635d6129),LL(0x538c2dd8,0x8c0dcfd1), + LL(0x7a6a308b,0xd6d9d9e3),LL(0x4c2767d3,0x62375830),LL(0xf38cbeb6,0x874a8bc6),LL(0xccb6fd9e,0xd94d3f1a), LL(0xba21f248,0x92a9735b),LL(0x6cd1efb0,0x272ad0e5),LL(0x05b03284,0x7437b69c),LL(0x6948c225,0xe7f04702), + LL(0xcba2ecec,0x8a56c04a),LL(0xe3a73e41,0x0c181270),LL(0x03e93725,0x6cb34e9d),LL(0x496521a9,0xf77c8713), LL(0xfa7f9f90,0x94569183),LL(0x8c9707ad,0xf2e7aa4c),LL(0x26c1c9a3,0xced2c9ba),LL(0x40197507,0x9109fe96), + LL(0xe9adfe1c,0x9ae868a9),LL(0x314e39bb,0x3984403d),LL(0xf2fe378f,0xb5875720),LL(0xba44a628,0x33f901e0), LL(0x3652438c,0xea1125fe),LL(0x9dd1f20b,0xae9ec4e6),LL(0xbebf7fbd,0x1e740d9e),LL(0x42dbe79c,0x6dbd3ddc), + LL(0xedd36776,0x62082aec),LL(0xe9859039,0xf612c478),LL(0x032f7065,0xa493b201),LL(0x4ff9b211,0xebd4d8f2), LL(0xaac4cb32,0x3f23a0aa),LL(0x15ed4005,0xea3aadb7),LL(0xafa27e63,0xacf17ea4),LL(0xc11fd66c,0x56125c1a), + LL(0x3794f8dc,0x266344a4),LL(0x483c5c36,0xdcca923a),LL(0x3f9d10a0,0x2d6b6bbf),LL(0x81d9bdf3,0xb320c5ca), LL(0x47b50a95,0x620e28ff),LL(0xcef03371,0x933e3b01),LL(0x99100153,0xf081bf85),LL(0xc3a8c8d6,0x183be9a0), + LL(0xd6bbe24d,0x4e3ddc5a),LL(0x53843795,0xc6c74630),LL(0x65ec2d4c,0x78193dd7),LL(0xcd3c89b2,0xb8df26cc), LL(0x5a483f8d,0x98dbe399),LL(0x7dd3313a,0x72d8a957),LL(0xab0bd375,0x65087294),LL(0x7c259d16,0xfcd89248), + LL(0x7613aa81,0x8a9443d7),LL(0x85fe6584,0x80100800),LL(0x7fb10288,0x70fc4dbc),LL(0xe86beee8,0xf58280d3), LL(0x7c978c38,0x14fdd82f),LL(0x0de44d7b,0xdf1204c1),LL(0x4160252f,0xa08a1c84),LL(0xc17646a5,0x591554ca), + LL(0xa05bd525,0x214a37d6),LL(0x07957b3c,0x48d5f09b),LL(0xd7109bc9,0x0247cdcb),LL(0x30599ce7,0x40f9e4bb), LL(0xf46ad2ec,0xc325fa03),LL(0xc3e3f9ee,0x00f766cf),LL(0xd43a4577,0xab556668),LL(0x3ee03b93,0x68d30a61), + LL(0x77b46a08,0x7ddc81ea),LL(0xc7480699,0xcf5a6477),LL(0x6633f683,0x43a8cb34),LL(0x92363c60,0x1b867e6b), LL(0x1f60558e,0x43921114),LL(0x2f41450e,0xcdbcdd63),LL(0xcc630e8b,0x7fc04601),LL(0x97038b43,0xea7c66d5), + LL(0x04e99fd8,0x7259b8a5),LL(0x4785549a,0x98a8dd12),LL(0x840552e1,0x0e459a7c),LL(0x4bb0909e,0xcdfcf4d0), LL(0x53758da7,0x34a86db2),LL(0xeac997e1,0xe643bb83),LL(0x530c5b7e,0x96400bd7),LL(0xb41c8b52,0x9f97af87), + LL(0xfbeee3f9,0x34fc8820),LL(0x49091afd,0x93e53490),LL(0x9a31f35c,0x764b9be5),LL(0x57e3d924,0x71f37864), LL(0x943aa75e,0x02fb34e0),LL(0xab8ff6e4,0xa18c9c58),LL(0x33cf0d19,0x080f31b1),LL(0x083518a7,0x5c9682db), + LL(0xb709c3de,0x873d4ca6),LL(0x3575b8f0,0x64a84262),LL(0x020154bb,0x6275da1f),LL(0xd17cf1ab,0x97678caa), LL(0x951a95c3,0x8779795f),LL(0x50fccc08,0xdd35b163),LL(0x33d8f031,0x32709627),LL(0x498dd85c,0x3c5ab10a), + LL(0x41dca566,0xb6c185c3),LL(0xd8622aa3,0x7de7feda),LL(0x901b6dfb,0x99e84d92),LL(0x7c4ad288,0x30a02b0e), LL(0x2fd3cf36,0xc7c81daa),LL(0xdf89e59f,0xd1319547),LL(0xcd496733,0xb2be8184),LL(0x93d3412b,0xd5f449eb), + LL(0x25fe531d,0x7ea41b1b),LL(0x6a1d5646,0xf9797432),LL(0x2bde501a,0x86067f72),LL(0x0c85e89c,0xf91481c0), LL(0xf8b05bc6,0xca8ee465),LL(0x02e83cda,0x1844e1cf),LL(0xb4dbe33b,0xca82114a),LL(0x4eabfde2,0x0f9f8769), + LL(0x38b27fe2,0x4936b1c0),LL(0xaba402df,0x63b6359b),LL(0x656bdbab,0x40c0ea2f),LL(0x6580c39c,0x9c992a89), LL(0x2a60aed1,0x600e8f15),LL(0xe0bf49df,0xeb089ca4),LL(0x2d42d99a,0x9c233d7d),LL(0x4c6bc2fa,0x648d3f95), + LL(0xe1add3f3,0xdcc383a8),LL(0x4f64a348,0xf42c0c6a),LL(0x0030dbdb,0x2abd176f),LL(0x7d6c215e,0x4de501a3), LL(0x4b9a64bc,0x4a107c1f),LL(0x2496cd59,0xa77f0ad3),LL(0x7688dffb,0xfb78ac62),LL(0x67937d8e,0x7025a2ca), + LL(0xd1a8f4e7,0xfde8b2d1),LL(0x7354927c,0xf5b3da47),LL(0xd9205735,0xe48606a3),LL(0xe177b917,0xac477cc6), LL(0xa883239a,0xfb1f73d2),LL(0xcc8b8357,0xe12572f6),LL(0xfb1f4f86,0x9d355e9c),LL(0xd9f3ec6e,0x89b795f8), + LL(0xb54398dc,0x27be56f1),LL(0x3fedeed5,0x1890efd7),LL(0x9c6d0140,0x62f77f1f),LL(0x596f0ee4,0x7ef0e314), LL(0xcc61dab3,0x50ca6631),LL(0xf4866e4f,0x4a39801d),LL(0xae363b39,0x66c8d032),LL(0x2ead66aa,0x22c591e5), + LL(0xde02a53e,0x954ba308),LL(0xd389f357,0x2a6c060f),LL(0xfbf40b66,0xe6cfcde8),LL(0xc6340ce1,0x8e02fc56), LL(0x73adb4ba,0xe4957795),LL(0xa7b03805,0x7b86122c),LL(0x0c8e6fa6,0x63f83512),LL(0x057d7804,0x83660ea0), + LL(0x21ba473c,0xbad79105),LL(0xded5389d,0xb6c50bee),LL(0xaa7c9bc0,0xee2caf4d),LL(0x8c4e98a7,0xd97b8de4), LL(0xab3bbddb,0xa9f63e70),LL(0x2597815a,0x3898aabf),LL(0xac15b3d9,0x7659af89),LL(0x703ce784,0xedf7725b), + LL(0xe085116b,0x25470fab),LL(0x87285310,0x04a43375),LL(0xe2bfd52f,0x4e39187e),LL(0x7d9ebc74,0x36166b44), LL(0xfd4b322c,0x92ad433c),LL(0xba79ab51,0x726aa817),LL(0xc1db15eb,0xf96eacd8),LL(0x0476be63,0xfaf71e91), + LL(0x641fad98,0xdd69a640),LL(0x29622559,0xb7995918),LL(0xde4199dc,0x03c6daa5),LL(0xad545eb4,0x92cadc97), LL(0x256534e4,0x1028238b),LL(0x8595409a,0x73e80ce6),LL(0xd05dc59b,0x690d4c66),LL(0x981dee80,0xc95f7b8f), + LL(0xd856ac25,0xf4337014),LL(0xac524dca,0x441bd9dd),LL(0x5f0499f5,0x640b3d85),LL(0xd5fda182,0x39cf84a9), LL(0xb2aa95a0,0x04e7b055),LL(0x0ddf1860,0x29e33f0a),LL(0x423f6b43,0x082e74b5),LL(0x0aaa2b0f,0x217edeb9), + LL(0x83cbea55,0x58b83f35),LL(0xbc185d70,0xc485ee4d),LL(0x1e5f6992,0x833ff03b),LL(0xcf0c0dd5,0xb5b9b9cc), LL(0x4e9e8a50,0x7caaee8e),LL(0x6269dafd,0x462e907b),LL(0xfbe791c6,0x6ed5cee9),LL(0xed430790,0x68ca3259), + LL(0x13b5ba88,0x2b72bdf2),LL(0x35ef0ac4,0x60294c8a),LL(0x19b99b08,0x9c3230ed),LL(0x6c2589aa,0x560fff17), LL(0xd6770374,0x552b8487),LL(0x9a56f685,0xa373202d),LL(0x45f175d9,0xd3e7f907),LL(0xd080d810,0x3c2f315f), + LL(0x7b9520e8,0x1130e9dd),LL(0x0af037b5,0xc078f9e2),LL(0x1e9c104c,0x38cd2ec7),LL(0xc472fe92,0x0f684368), LL(0x6247e7ef,0xd3f1b5ed),LL(0x396dfe21,0xb32d33a9),LL(0x4a9aa2c2,0x46f59cf4),LL(0xff0f7e41,0x69cd5168), + LL(0x4b3234da,0x3f59da0f),LL(0xb4579ebe,0xcf0b0235),LL(0x6d2476c7,0x6d1cbb25),LL(0x9dc30f08,0x4f0837e6), LL(0x906f6e98,0x9a4075bb),LL(0xc761e7d1,0x253bb434),LL(0x6e73af10,0xde2e645f),LL(0x0c5f131c,0xb89a4060), + LL(0xb8cc037f,0xd12840c5),LL(0x7405bb47,0x3d093a5b),LL(0x206348b8,0x6202c253),LL(0xc55a3ca7,0xbf5d57fc), LL(0x8c3bef48,0x89f6c90c),LL(0x5a0a960a,0x23ac7623),LL(0x552b42ab,0xdfbd3d6b),LL(0x132061f6,0x3ef22458), + LL(0xc97e6516,0xd74e9bda),LL(0xc230f49e,0x88779360),LL(0x1e74ea49,0xa6ec1de3),LL(0x3fb645a2,0x581dcee5), LL(0x8f483f14,0xbaef2391),LL(0xd137d13b,0x6d2dddfc),LL(0xd2743a42,0x54cde50e),LL(0xe4d97e67,0x89a34fc5), + LL(0x12e08ce5,0x13f1f5b3),LL(0xa7f0b2ca,0xa80540b8),LL(0x01982805,0x854bcf77),LL(0x233bea04,0xb8653ffd), LL(0x02b0b4c9,0x8e7b8787),LL(0x9acb170a,0x2675261f),LL(0x930c14e5,0x061a9d90),LL(0xdef0abea,0xb59b30e0), + LL(0x0200ec7d,0x1dc19ea6),LL(0x0bce132b,0xb6f4a3f9),LL(0xf13e27e0,0xb8d5de90),LL(0x1fade16f,0xbaee5ef0), LL(0xe4c6cf38,0x6f406aaa),LL(0xd1369815,0xab4cfe06),LL(0xefd550c6,0x0dcffe87),LL(0x75ff7d39,0x9d4f59c7), + LL(0x51deb6ad,0xb02553b1),LL(0xb1877749,0x812399a4),LL(0xca6006e1,0xce90f71f),LL(0xb02b6e77,0xc32363a6), LL(0xdc36c64d,0x02284fbe),LL(0xa7e1ae61,0x86c81e31),LL(0xb909d94a,0x2576c7e5),LL(0x818b2bb0,0x8b6f7d02), + LL(0x56faa38a,0xeca3ed07),LL(0x9305bb54,0xa3790e6c),LL(0x7bc73061,0xd784eeda),LL(0x6dd50614,0xbd56d369), LL(0x229a8aa9,0xd6575949),LL(0x4595ec28,0xdcca8f47),LL(0x06ab4fe6,0x814305c1),LL(0x24f43f16,0xc8c39768), + LL(0x523f2b36,0xe2a45f36),LL(0x920d93bb,0x995c6493),LL(0x90f1632b,0xf8afdab7),LL(0x1c295954,0x79ebbecd), LL(0x79592f48,0xc7bb3ddb),LL(0x5f88e998,0x67216a7b),LL(0xbc01193e,0xd91f098b),LL(0xb1db83fc,0xf7d928a5), + LL(0xe991f600,0x55e38417),LL(0x2981a934,0x2a91113e),LL(0x06b13bde,0xcbc9d648),LL(0x0755ff44,0xb011b6ac), LL(0x045ec613,0x6f4cb518),LL(0xc2f5930a,0x522d2d31),LL(0x382e65de,0x5acae1af),LL(0x27bc966f,0x57643067), + LL(0x1c7193f0,0x5e12705d),LL(0x3be8858e,0xf0f32f47),LL(0x96c6dfc7,0x785c3d7d),LL(0xbf31795d,0xd75b4a20), LL(0x342659d4,0x91acf17b),LL(0x44f0378f,0xe596ea34),LL(0xce52129d,0x4515708f),LL(0x79f2f585,0x17387e1e), + LL(0x49dee168,0x72cfd2e9),LL(0x3e2af239,0x1ae05223),LL(0x1d94066a,0x009e75be),LL(0x38abf413,0x6cca31c7), LL(0x9bc49908,0xb50bd61d),LL(0xf5e2bc1e,0x4a9b4a8c),LL(0x946f83ac,0xeb6cc5f7),LL(0xebffab28,0x27da93fc), + LL(0x4821c8c5,0xea314c96),LL(0xa83c15f4,0x8de49ded),LL(0x7af33004,0x7a64cf20),LL(0xc9627e10,0x45f1bfeb), LL(0x54b9df60,0x878b0626),LL(0xa95c0b33,0x5e4fdc3c),LL(0xc2035d8e,0xe54a37ca),LL(0x80f20b8c,0x9087cda9), + LL(0x8319ade4,0x36f61c23),LL(0xde8cfdf8,0x766f287a),LL(0x346f3705,0x48821948),LL(0x16e4f4a2,0x49a7b853), LL(0x5cedadfd,0xb9b3f8a7),LL(0x8db2a815,0x8f562815),LL(0x01f68f95,0xc0b7d554),LL(0x688a208e,0x12971e27), + LL(0xd0ff34fc,0xc9f8b696),LL(0x1222718c,0x20824de2),LL(0x0c95284d,0x7213cf9f),LL(0xdc158240,0xe2ad741b), LL(0x54043ccf,0x0ee3a6df),LL(0xd84412b3,0x16ff479b),LL(0xdfc98af0,0xf6c74ee0),LL(0x52fcd2fb,0xa78a169f), + LL(0x99c930e9,0xd8ae8746),LL(0x49e117a5,0x1d33e858),LL(0x6624759f,0x7581fcb4),LL(0x5bedc01d,0xde50644f), LL(0xcaf3155e,0xbeec5d00),LL(0xbc73e75f,0x672d66ac),LL(0x270b01db,0x86b9d8c6),LL(0x50f55b79,0xd249ef83), + LL(0x73978fe3,0x6131d6d4),LL(0x754b00a1,0xcc4e4542),LL(0x57dfcfe9,0x4e05df05),LL(0x51ef6bf0,0x94b29cdd), LL(0x9bc7edf2,0xe4530cff),LL(0xd3da65f3,0x8ac236fd),LL(0xc8eb0b48,0x0faf7d5f),LL(0x660eb039,0x4d2de14c), + LL(0x60430e54,0xc006bba7),LL(0xda3289ab,0x10a2d0d6),LL(0xd7979c59,0x9c037a5d),LL(0xa116d944,0x04d1f3d3), LL(0x8a0983cd,0x9ff22473),LL(0xc883cabb,0x28e25b38),LL(0x47a58995,0xe968dba5),LL(0x774eebdf,0x2c80b505), + LL(0x4a953beb,0xee763b71),LL(0x1642e7f6,0x502e223f),LL(0x61d5e722,0x6fe4b641),LL(0xdbef5316,0x9d37c5b0), LL(0xf8330bc7,0x0115ed70),LL(0x75a72789,0x139850e6),LL(0xffceccc2,0x27d7faec),LL(0x4fd9f7f6,0x3016a860), + LL(0x4cd8f64c,0xc492ec64),LL(0x279d7b51,0x58a2d790),LL(0x1fc75256,0x0ced1fc5),LL(0x8f433017,0x3e658aed), LL(0x05da59eb,0x0b61942e),LL(0x0ddc3722,0xba3d60a3),LL(0x742e7f87,0x7c311cd1),LL(0xf6b01b6e,0x6473ffee), +}, +/* digit=28 base_pwr=2^196 */ +{ + LL(0x692ac542,0x8303604f),LL(0x227b91d3,0xf079ffe1),LL(0x15aaf9bd,0x19f63e63),LL(0xf1f344fb,0xf99ee565), LL(0xd6219199,0x8a1d661f),LL(0xd48ce41c,0x8c883bc6),LL(0x3c74d904,0x1065118f),LL(0x0faf8b1b,0x713889ee), + LL(0x81a1b3be,0x972b3f8f),LL(0xce2764a0,0x4f3ce145),LL(0x28c4f5f7,0xe2d0f1cc),LL(0xc7f3985b,0xdeee0c0d), LL(0xd39e25c3,0x7df4adc0),LL(0xc467a080,0x40619820),LL(0x61cf5a58,0x440ebc93),LL(0x422ad600,0x527729a6), + LL(0xb1b76ba6,0xca6c0937),LL(0x4d2026dc,0x1a2eab85),LL(0x19d9ae0a,0xb1715e15),LL(0xbac4a026,0xf1ad9199), LL(0x07ea7b0e,0x35b3dfb8),LL(0x3ed9eb89,0xedf5496f),LL(0x2d6d08ab,0x8932e5ff),LL(0x25bd2731,0xf314874e), + LL(0x3f73f449,0xefb26a75),LL(0x8d44fc79,0x1d1c94f8),LL(0x3bc0dc4d,0x49f0fbc5),LL(0x3698a0d0,0xb747ea0b), LL(0x228d291e,0x5218c3fe),LL(0x43c129d6,0x35b804b5),LL(0xd1acc516,0xfac859b8),LL(0x95d6e668,0x6c10697d), + LL(0x0876fd4e,0xc38e438f),LL(0x83d2f383,0x45f0c307),LL(0xb10934cb,0x203cc2ec),LL(0x2c9d46ee,0x6a8f2439), LL(0x65ccde7b,0xf16b431b),LL(0x27e76a6f,0x41e2cd18),LL(0x4e3484d7,0xb9c8cf8f),LL(0x8315244a,0x64426efd), + LL(0xfc94dea3,0x1c0a8e44),LL(0xdad6a0b0,0x34c8cdbf),LL(0x04113cef,0x919c3840),LL(0x15490ffa,0xfd32fba4), LL(0x795dcfb7,0x58d190f6),LL(0x83588baf,0xfef01b03),LL(0xca1fc1c0,0x9e6d1d63),LL(0xf0a41ac9,0x53173f96), + LL(0xba16f73b,0x2b1d402a),LL(0x8cf9b9fc,0x2fb31014),LL(0x446ef7bf,0x2d51e60e),LL(0xb91e1745,0xc731021b), LL(0x4fee99d4,0x9d3b4724),LL(0xfac5c1ea,0x4bca48b6),LL(0xbbea9af7,0x70f5f514),LL(0x974c283a,0x751f55a5), + LL(0xcb452fdb,0x6e30251a),LL(0x50f30650,0x31ee6965),LL(0x933548d9,0xb0b3e508),LL(0xf4b0ef5b,0xb8949a4f), LL(0x3c88f3bd,0x208b8326),LL(0xdb1d9989,0xab147c30),LL(0x44d4df03,0xed6515fd),LL(0xe72eb0c5,0x17a12f75), + LL(0x36cf69db,0x3b59796d),LL(0x56670c18,0x1219eee9),LL(0x7a070d8e,0xfe3341f7),LL(0xa327f90c,0x9b70130b), LL(0x0ae18e0e,0x36a32462),LL(0x46c0a638,0x2021a623),LL(0xc62eb0d4,0x251b5817),LL(0x4c762293,0x87bfbcdf), + LL(0xcdd61d64,0xf78ab505),LL(0xc8c18857,0x8c7a53fc),LL(0x16147515,0xa653ce6f),LL(0xea7d52d5,0x9c923aa5), LL(0x5c18871f,0xc24709cb),LL(0x73b3cc74,0x7d53bec8),LL(0xfdd1d4c4,0x59264aff),LL(0x240da582,0x5555917e), + LL(0x548f5a0e,0xcae8bbda),LL(0x3bbfbbe1,0x1910eaba),LL(0x7677afc3,0xae579685),LL(0x73ff0b5c,0x49ea61f1), LL(0x4f7c3922,0x78655478),LL(0x20c68eef,0x95d337cd),LL(0xdf779ab9,0x68f1e1e5),LL(0xb5cf69a8,0x14b491b0), + LL(0x28e3fe89,0x7a6cbbe0),LL(0xc5aac0eb,0xe7e1fee4),LL(0x697e5140,0x7f47eda5),LL(0xb454921f,0x4f450137), LL(0x95cd8185,0xdb625f84),LL(0xcdb2e583,0x74be0ba1),LL(0xdd5e6de4,0xaee4fd7c),LL(0xe8101739,0x4251437d), + LL(0xac620366,0x686d72a0),LL(0xb6d59344,0x4be3fb9c),LL(0xa1eb75b9,0x6e8b44e7),LL(0x91a5c10c,0x84e39da3), LL(0xb38f0409,0x37cc1490),LL(0x2c2ade82,0x02951943),LL(0x1190a2d8,0x9b688783),LL(0x231182ba,0x25627d14), + LL(0x658a6d87,0x6eb550aa),LL(0xcf9c7325,0x1405aaa7),LL(0x5c8748c9,0xd147142e),LL(0x53ede0e0,0x7f637e4f), LL(0x14ffad2c,0xf8ca2776),LL(0xbafb6791,0xe58fb1bd),LL(0xbf8f93fc,0x17158c23),LL(0x0a4a4655,0x7f15b373), + LL(0xd842ca72,0x39d4add2),LL(0x3ed96305,0xa71e4391),LL(0x6700be14,0x5bb09cbe),LL(0xd8befcf6,0x68d69d54), LL(0x37183bcf,0xa45f5367),LL(0x3370dff7,0x7152b7bb),LL(0xbf12525b,0xcf887baa),LL(0xd6d1e3cd,0xe7ac7bdd), + LL(0x81fdad90,0x25914f78),LL(0x0d2cf6ab,0xcf638f56),LL(0xcc054de5,0xb90bc03f),LL(0x18b06350,0x932811a7), LL(0x9bbd11ff,0x2f00b330),LL(0xb4044974,0x76108a6f),LL(0xa851d266,0x801bb9e0),LL(0xbf8990c1,0x0dd099be), + LL(0xabe32986,0x58c5aaaa),LL(0x50d59c27,0x0fe9dd2a),LL(0x8d307305,0x84951ff4),LL(0x86529b78,0x6c23f829), LL(0x0b136a79,0x50bb2218),LL(0x77a20996,0x7e2174de),LL(0xc0bb4da6,0x6f00a4b9),LL(0xefdde8da,0x89a25a17), + LL(0xc11ee01d,0xf728a27e),LL(0xe5f10dfb,0xf900553a),LL(0x02ec893c,0x189a83c8),LL(0x23f66d77,0x3ca5bdc1), LL(0x97eada9f,0x98781537),LL(0x10256230,0x59c50ab3),LL(0x323c69b3,0x346042d9),LL(0x2c460449,0x1b715a6d), + LL(0x6ae06e0b,0xa41dd476),LL(0x9d42e25f,0xcdd7888e),LL(0x56b25a20,0x0f395f74),LL(0x8700e27e,0xeadfe0ae), LL(0x69950093,0xb09d52a9),LL(0x327f8d40,0x3525d9cb),LL(0x67df886a,0xb8235a94),LL(0x035faec2,0x77e4b0dd), + LL(0x517d7061,0x115eb20a),LL(0x6c2df683,0x77fe3433),LL(0xcdc6fc67,0x6870ddc7),LL(0x0b87de83,0xb1610588), LL(0xd9c4ddbe,0x343584ca),LL(0x3d754be2,0xb3164f1c),LL(0xc1e6c894,0x0731ed3a),LL(0x4f6b904c,0x26327dec), + LL(0x97b5cd32,0x9d49c6de),LL(0xb5eceecd,0x40835dae),LL(0xd9ded7fe,0xc66350ed),LL(0x7a678804,0x8aeebb5c), LL(0x5b8ee9ec,0x51d42fb7),LL(0x8e3ca118,0xd7a17bdd),LL(0x2ef4400e,0x40d7511a),LL(0x875a66f4,0xc48990ac), + LL(0x2199e347,0x8de07d2a),LL(0x2a39e051,0xbee75556),LL(0x916e51dc,0x56918786),LL(0x4a2d89ec,0xeb191313), LL(0x37d341ed,0x6679610d),LL(0x56d51c2b,0x434fbb41),LL(0xd7492dba,0xe54b7ee7),LL(0x59021493,0xaa33a79a), + LL(0xe4bd6d3d,0x49fc5054),LL(0x5ab551d0,0x09540f04),LL(0x4942d3a6,0x8acc9085),LL(0x2d28323b,0x231af02f), LL(0x0992c163,0x93458cac),LL(0x888e3bb4,0x1fef8e71),LL(0xbe8c268c,0x27578da5),LL(0xe805ec00,0xcc8be792), + LL(0xc61c3855,0x29267bae),LL(0x58c1fd3b,0xebff429d),LL(0x8c0b93b8,0x22d886c0),LL(0x2ddb8953,0xca5e00b2), LL(0xc3fed8b7,0xcf330117),LL(0x819c01f6,0xd49ac6fa),LL(0x3c0fbd54,0x6ddaa6bd),LL(0x8049a2cf,0x91743068), + LL(0xaff2ef81,0xd67f981e),LL(0x2818ae80,0xc3654d35),LL(0x1b2aa892,0x81d05044),LL(0x3d099328,0x2db067bf), LL(0x703dcc97,0xe7c79e86),LL(0xe133e215,0xe66f9b37),LL(0xe39a7a5c,0xcdf119a6),LL(0x876f1b61,0x47c60de3), + LL(0xd860f1b2,0x6e405939),LL(0xf5ed4d4a,0x3e9a1dbc),LL(0xc9b6bcbd,0x3f23619e),LL(0x734e4497,0x5ee790cf), LL(0x5bdaf9bb,0xf0a834b1),LL(0x4ca295f0,0x02cedda7),LL(0xcb8e378c,0x4619aa2b),LL(0xcc987ea4,0xe5613244), + LL(0x76b23a50,0x0bc022cc),LL(0x0a6c21ce,0x4a2793ad),LL(0x89cac3f5,0x38328780),LL(0xcba26d56,0x29176f1b), LL(0x4f6f59eb,0x06296187),LL(0x8bdc658e,0x86e9bca9),LL(0x57e30402,0x2ca9c4d3),LL(0x516a09bb,0x5438b216), + LL(0x7672765a,0x0a6a063c),LL(0x0547b9bf,0x37a3ce64),LL(0x98b1a633,0x42c099c8),LL(0x05ee6961,0xb5ab800d), LL(0x11a5acd6,0xf1963f59),LL(0x46201063,0xbaee6157),LL(0xa596210a,0x36d9a649),LL(0x1ba7138c,0xaed04363), + LL(0xa4a82b76,0xcf817d1c),LL(0xf3806be9,0x5586960e),LL(0x09dc6bb5,0x7ab67c89),LL(0x114fe7eb,0x52ace7a0), LL(0xcbbc9b70,0xcd987618),LL(0x604ca5e1,0x4f06fd5a),LL(0x6dbde133,0x90af14ca),LL(0x948a3264,0x1afe4322), + LL(0xc44b2c6c,0xa70d2ca6),LL(0x0ef87dfe,0xab726799),LL(0x2e696377,0x310f64dc),LL(0x4c8126a0,0x49b42e68), LL(0xcea0b176,0x0ea444c3),LL(0xcb269182,0x53a8ddf7),LL(0xbbba9dcb,0xf3e674eb),LL(0xd8669d33,0x0d2878a8), + LL(0xd019b6a3,0x04b935d5),LL(0x406f1e46,0xbb5cf88e),LL(0x5b57c111,0xa1912d16),LL(0x19ebfd78,0x9803fc21), LL(0xc07764a9,0x4f231c9e),LL(0xb75bd055,0xd93286ee),LL(0x8ee6c9de,0x83a9457d),LL(0x6087ec90,0x04695915), + LL(0x58d6cd46,0x14c6dd8a),LL(0x8e6634d2,0x9cb633b5),LL(0xf81bc328,0xc1305047),LL(0x26a177e5,0x12ede0e2), LL(0x065a6f4f,0x332cca62),LL(0x67be487b,0xc3a47ecd),LL(0x0f47ed1c,0x741eb187),LL(0xe7598b14,0x99e66e58), + LL(0x63d0ff12,0x6f0544ca),LL(0xb610a05f,0xe5efc784),LL(0x7cad7b47,0xf72917b1),LL(0xf2cac0c0,0x3ff6ea20), LL(0xf21db8b7,0xcc23791b),LL(0xd7d93565,0x7dac70b1),LL(0x694bdaad,0x682cda1d),LL(0x1023516d,0xeb88bb8c), + LL(0xdfdbeb1b,0xc4c634b4),LL(0xb4ee4dea,0x22f5ca72),LL(0xe6524821,0x1045a368),LL(0x052b18b2,0xed9e8a3f), LL(0xb961f49a,0x9b7f2cb1),LL(0x7b009670,0x7fee2ec1),LL(0x22507a6d,0x350d8754),LL(0x4db55f1d,0x561bd711), + LL(0x320bbcaf,0x4c189ccc),LL(0xdf1de48c,0x568434cf),LL(0x0fa8f128,0x6af1b00e),LL(0x8907583c,0xf0ba9d02), LL(0x32ff9f60,0x735a4004),LL(0xc25dcf33,0x3dd8e4b6),LL(0x42c74cef,0xf2230f16),LL(0x013fa8ad,0xd8117623), + LL(0xf51fe76e,0x36822876),LL(0x11d62589,0x8a6811cc),LL(0x46225718,0xc3fc7e65),LL(0xc82fdbcd,0xb7df2c9f), LL(0xdd7b205b,0x3b1d4e52),LL(0x47a2e414,0xb6959478),LL(0xefa91148,0x05e4d793),LL(0xfd2e9675,0xb47ed446), + LL(0x04c9d9bf,0x1a7098b9),LL(0x1b793048,0x661e2881),LL(0xb01ee461,0xb1a16966),LL(0x2954746f,0xbc521308), LL(0x2477de50,0xc909a0fc),LL(0x7dbd51ef,0xd80bb41c),LL(0x53294905,0xa85be7ec),LL(0x83958f97,0x6d465b18), + LL(0xfb6840fd,0x16f6f330),LL(0x3401e6c8,0xfaaeb214),LL(0xccb5b4f8,0xaf83d30f),LL(0x266dec4b,0x22885739), LL(0x7bc467df,0x51b4367c),LL(0xd842d27a,0x926562e3),LL(0x0fea14a6,0xdfcb6614),LL(0xf2734cd9,0xeb394dae), + LL(0x11c0be98,0x3eeae5d2),LL(0x814e8165,0xb1e6ed11),LL(0xe52bce1c,0x191086bc),LL(0xa75a04da,0x14b74cc6), LL(0x8c060985,0x63cf1186),LL(0x2dbd7f7c,0x071047de),LL(0xce0942ca,0x4e433b8b),LL(0xd8fec61d,0xecbac447), + LL(0xebf3232f,0x8f0ed0e2),LL(0xc52a2edd,0xfff80f9e),LL(0x75b55fdb,0xad9ab433),LL(0xe42e0c11,0x73ca7820), LL(0xe6251b46,0x6dace0a0),LL(0x4c0d932d,0x89bc6b5c),LL(0x095da19a,0x3438cd77),LL(0x8d48bdfb,0x2f24a939), + LL(0x766561b7,0x99b47e46),LL(0x0ed0322a,0x736600e6),LL(0x638e1865,0x06a47cb1),LL(0xcb136000,0x927c1c2d), LL(0x0cc5df69,0x29542337),LL(0x09d649a9,0x99b37c02),LL(0x6aefdb27,0xc5f0043c),LL(0x1be95c27,0x6cdd9987), + LL(0x390420d2,0x69850931),LL(0x0983efa4,0x299c40ac),LL(0xaf39aead,0x3a05e778),LL(0x43a45193,0x84274408), LL(0x91a711a0,0x6bcd0fb9),LL(0x9f52ab17,0x461592c8),LL(0xda3c6ed6,0xb49302b4),LL(0x330d7067,0xc51fddc7), + LL(0xda50d531,0x94babeb6),LL(0xa6a7b9da,0x521b840d),LL(0x404bdc89,0x5305151e),LL(0xd0d07449,0x1bcde201), LL(0x3b76a59a,0xf427a78b),LL(0x07791a1b,0xf84841ce),LL(0xbf91ed1c,0xebd314be),LL(0xbf172943,0x8e61d34c), + LL(0x5541b892,0x1d5dc451),LL(0xfc9d9e54,0xb186ee41),LL(0xd5bf610d,0x9d9f345e),LL(0xf6acca9f,0x3e7ba65d), LL(0xa8369486,0x9dda787a),LL(0x8eb5ba53,0x09f9dab7),LL(0xd6481bc3,0x5afb2033),LL(0xafa62104,0x76f4ce30), + LL(0xf4f066b5,0xa8fa00cf),LL(0x461dafc2,0x89ab5143),LL(0xa3389998,0x44339ed7),LL(0xbc214903,0x2ff862f1), LL(0xb05556e3,0x2c88f985),LL(0x3467081e,0xcd96058e),LL(0xedc637ea,0x7d6a4176),LL(0x36a5acdc,0xe1743d09), + LL(0x7eb37726,0x66fd72e2),LL(0x1481a037,0xf7fa264e),LL(0x45f4aa79,0x9fbd3bde),LL(0x767c3e22,0xed1e0147), LL(0x82e7abe2,0x7621f979),LL(0x45f633f8,0x19eedc72),LL(0x6137bf3a,0xe69b155e),LL(0x414ee94e,0xa0ad13ce), + LL(0x1c0e651a,0x93e3d524),LL(0x02ce227e,0xab1a6e2a),LL(0x4ab27eca,0xe7af1797),LL(0xbd444f39,0x245446de), LL(0x56c07613,0x59e22a21),LL(0xf4275498,0x43deafce),LL(0x67fd0946,0x10834ccb),LL(0x47406edf,0xa75841e5), + LL(0x7b0ac93d,0xebd6a677),LL(0x78f5e0d7,0xa6e37b0d),LL(0x76f5492b,0x2516c096),LL(0x9ac05f3a,0x1e4bf888), LL(0x4df0ba2b,0xcdb42ce0),LL(0x5062341b,0x935d5cfd),LL(0x82acac20,0x8a303333),LL(0x5198b00e,0x429438c4), + LL(0x049d33fa,0x1d083bc9),LL(0x946f67ff,0x58b82dda),LL(0x67a1d6a3,0xac3e2db8),LL(0x1798aac8,0x62e6bead), LL(0xde46c58c,0xfc85980f),LL(0x69c8d7be,0xa7f69379),LL(0x837b35ec,0x23557927),LL(0xe0790c0c,0x06a933d8), + LL(0x077ff55d,0x827c0e9b),LL(0xbb26e680,0x53977798),LL(0x1d9cb54f,0x59530874),LL(0x4aac53ef,0xcca3f449), LL(0xa07eda0f,0x11dc5c87),LL(0xfd6400c8,0xc138bccf),LL(0x13e5da72,0x549680d3),LL(0x4540617e,0xc93eed82), + LL(0x4d0b75c0,0xfd3db157),LL(0x6386075b,0x9716eb42),LL(0x817b2c16,0x0639605c),LL(0xf1e4f201,0x09915109), LL(0x5cca6c3b,0x35c9a928),LL(0x3505c900,0xb25f7d1a),LL(0x630480c4,0xeb9f7d20),LL(0x2a1a501c,0xc3c7b8c6), + LL(0x5a1f8e24,0x3f99183c),LL(0x9dd255f0,0xfdb118fa),LL(0xc27f62a6,0xb9b18b90),LL(0x396ec191,0xe8f732f7), LL(0x0be786ab,0x524a2d91),LL(0x0ac5a0f5,0x5d32adef),LL(0x9725f694,0x9b53d4d6),LL(0x0510ba89,0x032a76c6), + LL(0xebeb1544,0x840391a3),LL(0x3ed73ac3,0x44b7b88c),LL(0x256cb8b3,0xd24bae7a),LL(0xe394cb12,0x7ceb151a), LL(0x5bc1e6a8,0xbd6b66d0),LL(0x090f07bf,0xec70cecb),LL(0x7d937589,0x270644ed),LL(0x5f1dccfe,0xee9e1a3d), + LL(0x745b98d2,0xb0d40a84),LL(0x2556ed40,0xda429a21),LL(0x85148cb9,0xf676eced),LL(0xded18936,0x5a22d40c), LL(0x70e8a4ce,0x3bc4b9e5),LL(0x9eae0379,0xbfd1445b),LL(0x1a0bd47e,0xf23f2c0c),LL(0xe1845531,0xa9c0bb31), + LL(0x0a4c3f6b,0x9ddc4d60),LL(0x2c15ef44,0xbdfaad79),LL(0x7f484acc,0xce55a236),LL(0x055b1f15,0x08653ca7), LL(0x538873a3,0x2efa8724),LL(0xace1c7e7,0x09299e5d),LL(0xade332ba,0x07afab66),LL(0x92dd71b7,0x9be1fdf6), + LL(0x5758b11c,0xa49b5d59),LL(0xc8654f40,0x0b852893),LL(0x52379447,0xb63ef6f4),LL(0x105e690c,0xd4957d29), LL(0x646559b0,0x7d484363),LL(0x49788a8e,0xf4a8273c),LL(0x34ce54a9,0xee406cb8),LL(0xf86fda9b,0x1e1c260f), + LL(0xcf6a4a81,0xe150e228),LL(0x1b488772,0x1fa3b6a3),LL(0xc5a9c15b,0x1e6ff110),LL(0x8ad6aa47,0xc6133b91), LL(0x9dffa978,0x8ac5d55c),LL(0x5f3965f2,0xba1d1c1d),LL(0x7732b52f,0xf969f4e0),LL(0xa5172a07,0xfceecdb5), + LL(0x10f2b8f5,0xb0120a5f),LL(0x5c4c2f63,0xc83a6cdf),LL(0xf8f9c213,0x4d47a491),LL(0xd3f1bbd5,0xd9e1cce5), LL(0xaba7e372,0x0d91bc7c),LL(0xdfd1a2db,0xfcdc74c8),LL(0x374618e5,0x05efa800),LL(0x15a7925e,0x11216969), + LL(0xf6021c5d,0xd4c89823),LL(0xeff14423,0x880d5e84),LL(0x6dcd1396,0x6523bc5a),LL(0x113c978b,0xd1acfdfc), LL(0xbbb66840,0xb0c164e8),LL(0x72b58459,0xf7f4301e),LL(0xa638e8ec,0xc29ad4a6),LL(0x46b78699,0xf5ab8961), + LL(0x0e954750,0x9dbd7974),LL(0x64f9d2c6,0x0121de88),LL(0xd985232e,0x2e597b42),LL(0x53451777,0x55b6c3c5), LL(0x519cb9fb,0xbb53e547),LL(0x8428600d,0xf134019f),LL(0xe081791a,0x5a473176),LL(0x35fb0c08,0x2f3e2263), + LL(0x73d273b0,0xb28c3017),LL(0x7721ef9a,0xccd21076),LL(0xb650dc39,0x054cc292),LL(0x6188045e,0x662246de), LL(0x6b83c0d1,0x904b52fa),LL(0x97e9cd46,0xa72df267),LL(0x899725e4,0x886b43cd),LL(0xd849ff22,0x2b651688), + LL(0x02f34533,0x60479b79),LL(0x0c77c148,0x5e354c14),LL(0xa8537c78,0xb4bb7581),LL(0xefe1495f,0x188043d7), LL(0x8c1d5026,0x9ba12f42),LL(0x93d4aaab,0x2e0c8a26),LL(0xaa57c450,0xbdba7b8b),LL(0x9bbdafef,0x140c9ad6), + LL(0x25ac0f18,0x2067aa42),LL(0x04d1fbf3,0xf7b1295b),LL(0xa4b04824,0x14829111),LL(0x33bd5e91,0x2ce3f192), LL(0x8f2e1b72,0x9c7a1d55),LL(0x302aa243,0xfe932286),LL(0xd4be9554,0x497ca7b4),LL(0xe0547a6e,0xb8e821b8), + LL(0x67e573e0,0xfb2838be),LL(0x4084c44b,0x05891db9),LL(0x96c1c2c5,0x91311373),LL(0xd958444b,0x6aebfa3f), LL(0xe56e55c1,0xac9cdce9),LL(0x2caa46d0,0x7148ced3),LL(0xb61fe8eb,0x2e10c7ef),LL(0xff97cf4d,0x9fd835da), +}, +/* digit=29 base_pwr=2^203 */ +{ + LL(0x081e9387,0xa36da109),LL(0x8c935828,0xfb9780d7),LL(0xe540b015,0xd5940332),LL(0xe0f466fa,0xc9d7b51b), LL(0xd6d9f671,0xfaadcd41),LL(0xb1a2ac17,0xba6c1e28),LL(0xed201e5f,0x066a7833),LL(0xf90f462b,0x19d99719), + LL(0x060b5f61,0xf431f462),LL(0x7bd057c2,0xa56f46b4),LL(0x47e1bf65,0x348dca6c),LL(0x41bcf1ff,0x9a38783e), LL(0xda710718,0x7a5d33a9),LL(0x2e0aeaf6,0x5a779987),LL(0x2d29d187,0xca87314d),LL(0xc687d733,0xfa0edc3e), + LL(0x6a31e09b,0x9df33621),LL(0xc1350e35,0xde89e44d),LL(0x4ca0cf52,0x29214871),LL(0x0b88a538,0xdf379672), LL(0x2591d61b,0xc92a510a),LL(0x585b447b,0x79aa87d7),LL(0xe5287f77,0xf67db604),LL(0x5efe7a80,0x1697c8bf), + LL(0xcb198ac7,0x1c894849),LL(0x0f264665,0xa884a93d),LL(0x9b200678,0x2da964ef),LL(0x009834e6,0x3c351b87), LL(0xe2c4b44b,0xafb2ef9f),LL(0x3326790c,0x580f6c47),LL(0x0b02264a,0xb8480521),LL(0x42a194e2,0x8ba6f9e2), + LL(0x8fb54738,0xfc87975f),LL(0x27c3ead3,0x35160788),LL(0xb74a085a,0x834116d2),LL(0xa62fe996,0x53c99a73), LL(0x5b81c51b,0x87585be0),LL(0xbe0852b7,0x925bafa8),LL(0xa84d19a7,0x76a4fafd),LL(0x585206d4,0x39a45982), + LL(0x5eb03c0e,0x499b6ab6),LL(0x72bc3fde,0xf19b7954),LL(0x6e3a80d2,0xa86b5b9c),LL(0x6d42819f,0xe4377508), LL(0xbb3ee8a3,0xc1663650),LL(0xb132075f,0x75eb14fc),LL(0x7ad834f6,0xa8ccc906),LL(0xe6e92ffd,0xea6a2474), + LL(0x0f8d6758,0x9d72fd95),LL(0x408c07dd,0xcb84e101),LL(0xa5e23221,0xb9114bfd),LL(0xe94e742c,0x358b5fe2), LL(0x95f40e75,0x1c0577ec),LL(0x3d73f3d6,0xf0155451),LL(0xbd1b9b66,0x9d55cd67),LL(0xaf8d63c7,0x63e86e78), + LL(0xd3c095f1,0x39d934ab),LL(0xe4b76d71,0x04b261be),LL(0xe73e6984,0x1d2e6970),LL(0x5e5fcb11,0x879fb23b), LL(0xdfd75490,0x11506c72),LL(0x61bcf1c1,0x3a97d085),LL(0xbf5e7007,0x43201d82),LL(0x798232a7,0x7f0ac52f), + LL(0x6eb564d4,0x2715cbc4),LL(0x9e570e29,0x8d6c752c),LL(0x9ef5fd5d,0xf80247c8),LL(0xd53eb514,0xc3c66b46), LL(0x0f87de56,0x9666b401),LL(0xc6c603b5,0xce62c06f),LL(0x7e4fc942,0xae7b4c60),LL(0x663a9c19,0x38ac0b77), + LL(0x4b049136,0xcb4d20ee),LL(0x356a4613,0x8b63bf12),LL(0x70e08128,0x1221aef6),LL(0x4acb6b16,0xe62d8c51), LL(0x379e7896,0x71f64a67),LL(0xcafd7fa5,0xb25237a2),LL(0x3841ba6a,0xf077bd98),LL(0x3cd16e7e,0xc4ac0244), + LL(0x21fea4ca,0x548ba869),LL(0xf3dfdac1,0xd36d0817),LL(0xf4685faf,0x09d8d71f),LL(0xc52c459a,0x8eff66be), LL(0x0b57235e,0x182faee7),LL(0x0106712b,0xee3c39b1),LL(0xc0fcdcb0,0x5107331f),LL(0xa51054ba,0x669fb9dc), + LL(0x319d7682,0xb25101fb),LL(0x0a982fee,0xb0293129),LL(0x0261b344,0x51c1c9b9),LL(0xbfd371fa,0x0e008c5b), LL(0x0278ca33,0xd866dd1c),LL(0xe5aa53b1,0x666f76a6),LL(0x6013a2cf,0xe5cfb779),LL(0xa3521836,0x1d3a1aad), + LL(0x73faa485,0xcedd2531),LL(0xc0a76878,0xc8ee6c4f),LL(0x2a11667d,0xddbccfc9),LL(0x1c2f695a,0x1a418ea9), LL(0x51f73971,0xdb11bd92),LL(0xda2ed89f,0x3e4b3c82),LL(0xe73e0319,0x9a44f3f4),LL(0x303431af,0xd1e3de0f), + LL(0x50f75f9c,0x3c5604ff),LL(0x7e752b22,0x1d8eddf3),LL(0x3c9a1118,0x0ef074dd),LL(0xccb86d7b,0xd0ffc172), LL(0x037d90f2,0xabd1ece3),LL(0x6055856c,0xe3f307d6),LL(0x7e4c6daf,0x422f9328),LL(0x334879a0,0x902aac66), + LL(0x94cdfade,0xb6a1e7bf),LL(0x7fc6d634,0x6c97e1ed),LL(0xa2fb63f8,0x662ad24d),LL(0xa5928405,0xf81be1b9), LL(0xd14b4206,0x86d765e4),LL(0x8fa0db65,0xbecc2e0e),LL(0xb17fc76c,0xa28838e0),LL(0xe37cf24e,0xe49a602a), + LL(0x567193ec,0x76b4131a),LL(0xe5f6e70b,0xaf3c305a),LL(0x031eebdd,0x9587bd39),LL(0x71bbe831,0x5709def8), LL(0x0eb2b669,0x57059983),LL(0x875b7029,0x4d80ce1b),LL(0x0364ac16,0x838a7da8),LL(0xbe1c83ab,0x2f431d23), + LL(0xf9294dd3,0xe56812a6),LL(0x9b4b0d77,0xb448d01f),LL(0x04e8305c,0xf3ae6061),LL(0x94d8c63e,0x2bead645), LL(0x84fd8b07,0x0a85434d),LL(0xf7a9dee5,0x537b983f),LL(0xef55bd85,0xedcc5f18),LL(0x21c6cf8b,0x2041af62), + LL(0xb940c71e,0x8e52874c),LL(0xdb5f4b3a,0x211935a9),LL(0x301b1dc3,0x94350492),LL(0x29958620,0x33d2646d), LL(0xef911404,0x16b0d64b),LL(0x9a3c5ef4,0x9d1f25ea),LL(0x4a352c78,0x20f200eb),LL(0x4bd0b428,0x43929f2c), + LL(0xc7196e29,0xa5656667),LL(0x9391be48,0x7992c2f0),LL(0x9ee0cd6e,0xaaa97cbd),LL(0x3dc8c9bf,0x51b0310c), LL(0xdd9f22cb,0x237f8acf),LL(0xb585d584,0xbb1d81a1),LL(0x8c416388,0x8d5d85f5),LL(0x42fe474f,0x0d6e5a5a), + LL(0x38235d4e,0xe7812766),LL(0x496e3298,0x1c62bd67),LL(0x3f175bc8,0x8378660c),LL(0x17afdd4d,0x4d04e189), LL(0x85a8068c,0x32a81601),LL(0x92b29a85,0xdb58e4e1),LL(0xc70d8a3b,0xe8a65b86),LL(0x98a0403b,0x5f0e6f4e), + LL(0x69ed2370,0x08129684),LL(0x0871ee26,0x34dc30bd),LL(0x7c9c5b05,0x3a5ce948),LL(0x43a90c87,0x7d487b80), LL(0xdd0e7179,0x4089ba37),LL(0xb4041811,0x45f80191),LL(0x98747ba5,0x1c3e1058),LL(0x6e1ae592,0x98c4e13a), + LL(0xe82c9f9e,0xd44636e6),LL(0xc33a1043,0x711db87c),LL(0xaa8aec05,0x6f431263),LL(0x2744a4aa,0x43ff120d), LL(0xae77779b,0xd3bd892f),LL(0x8cdc9f82,0xf0fe0cc9),LL(0xf1c5b1bc,0xca5f7fe6),LL(0x44929a72,0xcc63a682), + LL(0x09dbe19a,0xc7eaba0c),LL(0x6b5c73c2,0x2f3585ad),LL(0x0ae50c30,0x8ab8924b),LL(0x638b30ba,0x17fcd27a), LL(0x10b3d5a5,0xaf414d34),LL(0x2a9accf1,0x09c107d2),LL(0x946a6242,0x15dac49f),LL(0xd707d642,0xaec3df2a), + LL(0x3f894ae0,0x2c2492b7),LL(0xb75f18ce,0xf59df3e5),LL(0x8f53cad0,0x7cb740d2),LL(0xc4f01294,0x3eb585fb), LL(0x32c7f717,0x17da0c86),LL(0xaf943f4c,0xeb8c795b),LL(0xf67c51d2,0x4ee23fb5),LL(0x68889949,0xef187575), + LL(0x0389168b,0xa6b4bdb2),LL(0xea577d03,0xc4ecd258),LL(0x55743082,0x3a63782b),LL(0xc72f08cd,0x6f678f4c), LL(0x65e58dd8,0x553511cf),LL(0xd402c0cd,0xd53b4e3e),LL(0xa037c14c,0x37de3e29),LL(0xc05712aa,0x86b6c516), + LL(0xb38dff6f,0x2834da3e),LL(0xea636be8,0xbe012c52),LL(0x61dd37f8,0x292d238c),LL(0x8f8142db,0x0e54523f), LL(0x036a05d8,0xe31eb436),LL(0x1e93c0ff,0x83e3cdff),LL(0x50821ddf,0x3fd2fe0f),LL(0xff9eb33b,0xc8e19b0d), + LL(0xb569a5fe,0xc8cc943f),LL(0xd4342d75,0xad0090d4),LL(0xcaeca000,0x82090b4b),LL(0x1bd410eb,0xca39687f), LL(0x65959d77,0xe7bb0df7),LL(0x9c964999,0x39d78218),LL(0xb2415451,0xd87f62e8),LL(0xbed76108,0xe5efb774), + LL(0xe822f0d0,0x3ea011a4),LL(0x5a8704f8,0xbc647ad1),LL(0x50c6820f,0xbb315b35),LL(0xb7e76bec,0x863dec3d), LL(0xf017bfc7,0x01ff5d3a),LL(0x976b8229,0x20054439),LL(0x0bbd0d3b,0x067fca37),LL(0x7f5e3d0f,0xf63dde64), + LL(0x2a4c94e9,0x22dbefb3),LL(0x96f8278a,0xafbff0fe),LL(0x3503793d,0x80aea0b1),LL(0x5f06cd29,0xb2238029), LL(0x8ec3feca,0x65703e57),LL(0x393e7053,0x06c38314),LL(0x7c6734c4,0xa0b751eb),LL(0xc59f0f1e,0xd2e8a435), + LL(0x5e9ca895,0x147d9052),LL(0x972072df,0x2f4dd31e),LL(0xe6c6755c,0xa16fda8e),LL(0xcf196558,0xc66826ff), LL(0x0cf43895,0x1f1a76a3),LL(0x83c3097b,0xa9d604e0),LL(0x66390e0e,0xe1908309),LL(0xb3c85eff,0xa50bf753), + LL(0xf6a70251,0x0696bdde),LL(0x3c6ab16a,0x548b801b),LL(0xa4d08762,0x37fcf704),LL(0xdff76c4e,0x090b3def), LL(0x69cb9158,0x87e8cb89),LL(0x995ece43,0x44a90744),LL(0x0ad9fbf5,0xf85395f4),LL(0x4fb0c82d,0x49b0f6c5), + LL(0xadf7cccf,0x75d9bc15),LL(0xdfa1e1b0,0x81a3e5d6),LL(0x249bc17e,0x8c39e444),LL(0x8ea7fd43,0xf37dccb2), LL(0x907fba12,0xda654873),LL(0x4a372904,0x35daa6da),LL(0x6283a6c5,0x0564cfc6),LL(0x4a9395bf,0xd09fa4f6), + LL(0xaeb19a36,0x688e9ec9),LL(0xc7bfbfb4,0xd913f1ce),LL(0x61c2faa6,0x797b9a3c),LL(0x6a0a9c12,0x2f979bec), LL(0x359679ec,0xb5969d0f),LL(0x079b0460,0xebcf523d),LL(0x10fab870,0xfd6b0008),LL(0x9373a39c,0x3f2edcda), + LL(0x6f568431,0x0d64f9a7),LL(0x02f8898c,0xf848c27c),LL(0x260b5bd5,0xf418ade1),LL(0x6973dee8,0xc1f3e323), LL(0x26c185dd,0x46e9319c),LL(0x546f0ac4,0x6d85b7d8),LL(0x247f9d57,0x427965f2),LL(0xb0035f48,0xb519b636), + LL(0xab87d59c,0x6b6163a9),LL(0x39caaa11,0xff9f58c3),LL(0x3177387b,0x4ac39cde),LL(0x873e77f9,0x5f6557c2), LL(0x36a83041,0x67504006),LL(0x75ef196c,0x9b1c96ca),LL(0xb08c7940,0xf34283de),LL(0x1128c316,0x7ea09644), + LL(0x6aa39dff,0xb510b3b5),LL(0x9f8e4d8c,0x59b43da2),LL(0x9e4c4b9f,0xa8ce31fd),LL(0xc1303c01,0x0e20be26), LL(0xe8ee47c9,0x18187182),LL(0x7db98101,0xd9687cdb),LL(0xa1e14ff6,0x7a520e4d),LL(0x8836d572,0x429808ba), + LL(0x4944b663,0xa37ca60d),LL(0xa3f91ae5,0xf901f7a9),LL(0x9e36e3b1,0xe4e3e76e),LL(0x29d93250,0x9aa219cf), LL(0x056a2512,0x347fe275),LL(0xde65d95c,0xa4d643d9),LL(0x699fc3ed,0x9669d396),LL(0xcf8c6bbe,0xb598dee2), + LL(0xdda9e5c6,0x682ac1e5),LL(0xcaa9fc95,0x4e0d3c72),LL(0x772bea44,0x17faaade),LL(0xab0009c8,0x5ef8428c), LL(0x460ff016,0xcc4ce47a),LL(0x725281cb,0xda6d12bf),LL(0x0223aad2,0x44c67848),LL(0x36256e28,0x6e342afa), + LL(0x93a37c04,0x1400bb0b),LL(0xdd10bd96,0x62b1bc9b),LL(0x0dac46b7,0x7251adeb),LL(0x7be4ef51,0x7d33b92e), LL(0xe61fa29a,0x28b2a94b),LL(0x06422233,0x4b2be13f),LL(0x330d8d37,0x36d6d062),LL(0xb28ca005,0x5ef80e1e), + LL(0x6d16768e,0x174d4699),LL(0x628bf217,0x9fc4ff6a),LL(0x154e490d,0x77705a94),LL(0x8d2d997a,0x9d96dd28), LL(0xce5d72c4,0x77e2d9d8),LL(0xc11c714f,0x9d06c5a4),LL(0x79e4a03e,0x02aa5136),LL(0x030ff28b,0x1386b3c2), + LL(0xfb283f61,0xfe82e8a6),LL(0xf3abc3fb,0x7df203e5),LL(0x3a4d3622,0xeec7c351),LL(0xdf762761,0xf7d17dbf), LL(0x522055f0,0xc3956e44),LL(0x8fa748db,0xde3012db),LL(0xbf1dcc14,0xca9fcb63),LL(0xbe4e2f3a,0xa56d9dcf), + LL(0x8bcec9c2,0xb86186b6),LL(0x680b9f06,0x7cf24df9),LL(0xc0d29281,0xc46b45ea),LL(0x07b10e12,0xfff42bc5), LL(0x4d289427,0x12263c40),LL(0xb4848ec4,0x3d5f1899),LL(0xd040800c,0x11f97010),LL(0x300feb20,0xb4c5f529), + LL(0xde94fdcb,0xcc543f8f),LL(0xc7c2f05e,0xe96af739),LL(0x882692e1,0xaa5e0036),LL(0x950d4ae9,0x09c75b68), LL(0xb5932a7a,0x62f63df2),LL(0xde0979ad,0x2658252e),LL(0xb5e69631,0x2a19343f),LL(0x525b666b,0x718c7501), + LL(0xea40dc3a,0x26a42d69),LL(0xaecc018f,0xdc84ad22),LL(0x3270f04a,0x25c36c7b),LL(0x50fa72ed,0x46ba6d47), LL(0x93e58a8e,0x6c37d1c5),LL(0x120c088c,0xa2394731),LL(0xcb6e86da,0xc3be4263),LL(0x7126d038,0x2c417d36), + LL(0x8b6f8efa,0x5b70f9c5),LL(0x37718536,0x671a2faa),LL(0xb539c92b,0xd3ced3c6),LL(0xa31203c2,0xe56f1bd9), LL(0x9ff3c8eb,0x8b096ec4),LL(0x43491cea,0x2deae432),LL(0x17943794,0x2465c6eb),LL(0x20586843,0x5d267e66), + LL(0xb07159d0,0x9d3d116d),LL(0xc1896210,0xae07a67f),LL(0xbb961579,0x8fc84d87),LL(0x1c1f8dd6,0x30009e49), LL(0xe3132819,0x8a8caf22),LL(0xf23ab4ff,0xcffa197c),LL(0x205dd687,0x58103a44),LL(0x0ded67a2,0x57b796c3), + LL(0xa1779ad7,0x0b9c3a6c),LL(0x357c09c5,0xa33cfe2e),LL(0x3db4a57e,0x2ea29315),LL(0x8ebeb52e,0x91959695), LL(0xe546c879,0x118db9a6),LL(0x6295c8d6,0x8e996df4),LL(0x55ec806b,0xdd990484),LL(0x165c1035,0x24f291ca), + LL(0x440e2229,0xcca523bb),LL(0x73ef4d04,0x324673a2),LL(0x3e11ec39,0xaf3adf34),LL(0xdc5968d3,0x6136d7f1), LL(0xb053a927,0x7a7b2899),LL(0xae067ecd,0x3eaa2661),LL(0x02779cd9,0x8549b9c8),LL(0xc53385ea,0x061d7940), + LL(0xf06d18bd,0x3e0ba883),LL(0xb2700843,0x4ba6de53),LL(0x591a9e4d,0xb966b668),LL(0x7f4fa0ed,0x93f67567), LL(0x4347237b,0x5a02711b),LL(0xe794608e,0xbc041e2f),LL(0x70f73d8c,0x55af10f5),LL(0xbb7564f7,0xd2d4d4f7), + LL(0xb3e93ce7,0xd7d27a89),LL(0x5d3a2c1b,0xf7b5a875),LL(0x255b218a,0xb29e68a0),LL(0x8af76754,0xb533837e), LL(0x579fab2e,0xd1b05a73),LL(0xecd74385,0xb41055a1),LL(0x445e9115,0xb2369274),LL(0xf520274e,0x2972a7c4), + LL(0xf678e68a,0x6c08334e),LL(0x99b057ed,0x4e4160f0),LL(0x52ccb69a,0x3cfe11b8),LL(0x21c8f772,0x2fd1823a), LL(0x3298f055,0xdf7f072f),LL(0xfec74a6e,0x8c0566f9),LL(0x5bb4d041,0xe549e019),LL(0x9208d850,0x7c3930ba), + LL(0xaaa2902b,0xe07141fc),LL(0xe4f69ad3,0x539ad799),LL(0x813f9ffd,0xa6453f94),LL(0x375bc2f7,0xc58d3c48), LL(0x5dc64e96,0xb3326fad),LL(0xb240e354,0x3aafcaa9),LL(0xaca1e7a9,0x1d1b0903),LL(0x1211b8a0,0x4ceb9767), + LL(0xe32a858e,0xeca83e49),LL(0xae907bad,0x4c32892e),LL(0x2eb9b494,0xd5b42ab6),LL(0x1eabae1b,0x7fde3ee2), LL(0xcaf54957,0x13b5ab09),LL(0xe5f5d5d5,0xbfb028be),LL(0x2003e2c0,0x928a0650),LL(0x67476843,0x90793aac), + LL(0xc81710a0,0x5e942e79),LL(0x27ccadd4,0x557e4a36),LL(0x4bcf6d0c,0x72a2bc56),LL(0x26d7b80c,0x09ee5f43), LL(0xd4292f19,0x6b70dbe9),LL(0x63f16b18,0x56f74c26),LL(0x35fbb42a,0xc23db0f7),LL(0x6ae10040,0xb606bdf6), + LL(0x044573ac,0x1eb15d4d),LL(0x556b0ba4,0x7dc3cf86),LL(0xc60df6f7,0x97af9a33),LL(0xa716ce8c,0x0b1ef85c), LL(0xc96958be,0x2922f884),LL(0x35690963,0x7c32fa94),LL(0xeaa00061,0x2d7f667c),LL(0x3547365c,0xeaaf7c17), + LL(0x87032d58,0x1eb4de46),LL(0x5e2c79e0,0xc54f3d83),LL(0x5d04ef23,0x07818df4),LL(0x673d41b4,0x55faa9c8), LL(0x89b95355,0xced64f6f),LL(0xb7415c84,0x4860d2ea),LL(0x050ebad3,0x5fdb9bd2),LL(0x6685a5bf,0xdb53e0cc), + LL(0x9feb6593,0xb830c031),LL(0x6accff17,0xdd87f310),LL(0x9f555c10,0x2303ebab),LL(0x287e7065,0x94603695), LL(0x2e83358c,0xf88311c3),LL(0xeefb0178,0x508dd9b4),LL(0x2dba8652,0x7ca23706),LL(0x0047abe5,0x62aac5a3), + LL(0x8b1ea7b3,0x9a61d2a0),LL(0xae8b1485,0xd495ab63),LL(0x87052f99,0x38740f84),LL(0xb2974eea,0x178ebe5b), LL(0x5b36d17f,0x030bbcca),LL(0xaaf86eea,0xb5e4cce3),LL(0x68f8e9e0,0xb51a0220),LL(0x09eb3e75,0xa4348796), + LL(0xeef1a752,0xbe592309),LL(0x6f2aa1ed,0x5d7162d7),LL(0x0f007dd2,0xaebfb5ed),LL(0xc89edd22,0x255e14b2), LL(0x0303b697,0xba85e072),LL(0xf05720ff,0xc5d17e25),LL(0x5128ebb6,0x02b58d6e),LL(0xd754e113,0x2c80242d), + LL(0xabfae1ca,0x919fca5f),LL(0x1a21459b,0x937afaac),LL(0x1f66a4d2,0x9e0ca91c),LL(0x23ec1331,0x194cc7f3), LL(0x8aa11690,0xad25143a),LL(0x09b59e08,0xbe40ad8d),LL(0xe750860a,0x37d60d9b),LL(0xc6bf434c,0x6c53b008), + LL(0x1356eb80,0xb572415d),LL(0x9578ded8,0xb8bf9da3),LL(0x5e8fb38b,0x22658e36),LL(0x5af8cb22,0x9b70ce22), LL(0x829a8180,0x7c00018a),LL(0xb81ed295,0x84329f93),LL(0x5f3cea83,0x7c343ea2),LL(0x67586536,0x38f8655f), + LL(0x1d3ec517,0xa661a0d0),LL(0x512321ae,0x98744652),LL(0xeca92598,0x084ca591),LL(0x1dcb3feb,0xa9bb9dc9), LL(0x78b4c240,0x14c54355),LL(0x610cafdc,0x5ed62a3b),LL(0x1b38846b,0x07512f37),LL(0xb0e38161,0x571bb70a), + LL(0x2da705d2,0xb556b95b),LL(0xb1a08f98,0x3ef8ada6),LL(0xddecfbe5,0x85302ca7),LL(0x943105cd,0x0e530573), LL(0x21a9255d,0x60554d55),LL(0xf2f3802a,0x63a32fa1),LL(0xcd477875,0x35c8c5b0),LL(0x6ad42da1,0x97f458ea), + LL(0xeb6b242d,0x832d7080),LL(0x3b71e246,0xd30bd023),LL(0xbe31139d,0x7027991b),LL(0x462e4e53,0x68797e91), LL(0x6b4e185a,0x423fe20a),LL(0x42d9b707,0x82f2c67e),LL(0x4cf7811b,0x25c81768),LL(0x045bb95d,0xbd53005e), +}, +/* digit=30 base_pwr=2^210 */ +{ + LL(0x9d8e68fd,0xe5f649be),LL(0x1b044320,0xdb0f0533),LL(0xe0c33398,0xf6fde9b3),LL(0x66c8cfae,0x92f4209b), LL(0x1a739d4b,0xe9d1afcc),LL(0xa28ab8de,0x09aea75f),LL(0xeac6f1d0,0x14375fb5),LL(0x708f7aa5,0x6420b560), + LL(0x6254dc41,0x9eae499c),LL(0x7a837e7e,0x7e293924),LL(0x090524a7,0x74aec08c),LL(0x8d6f55f2,0xf82b9219), LL(0x1402cec5,0x493c962e),LL(0xfa2f30e7,0x9f17ca17),LL(0xe9b879cb,0xbcd783e8),LL(0x5a6f145f,0xea3d8c14), + LL(0x5e0dee6e,0xdede15e7),LL(0xdc628aa2,0x74f24872),LL(0x7861bb93,0xd3e9c4fe),LL(0x6187b2e0,0x56d4822a), LL(0xc59826f9,0xb66417cf),LL(0x2408169e,0xca260969),LL(0xc79ef885,0xedf69d06),LL(0xdc7d138f,0x00031f8a), + LL(0x0ebcf726,0x103c46e6),LL(0x6231470e,0x4482b831),LL(0x487c2109,0x6f6dfaca),LL(0x62e666ef,0x2e0ace97), LL(0x1f8d1f42,0x3246a9d3),LL(0x574944d2,0x1b1e83f1),LL(0xa57f334b,0x13dfa63a),LL(0x9f025d81,0x0cf8daed), + LL(0x00ee11c1,0x30d78ea8),LL(0xb5e3dd75,0xeb053cd4),LL(0xd58c43c5,0x9b65b13e),LL(0xbd151663,0xc3ad49bd), LL(0xb6427990,0x99fd8e41),LL(0x707eae1e,0x12cf15bd),LL(0x1aabb71e,0x29ad4f1b),LL(0x07545d0e,0x5143e74d), + LL(0xc88bdee1,0x30266336),LL(0x5876767c,0x25f29306),LL(0xc6731996,0x9c078571),LL(0xed552951,0xc88690b2), LL(0x852705b4,0x274f2c2d),LL(0x4e09552d,0xb0bf8d44),LL(0x986575d1,0x7628beeb),LL(0x7f864651,0x407be238), + LL(0xa639fc6b,0x0e5e3049),LL(0x86003625,0xe75c35d9),LL(0x5dcc1646,0x0cf35bd8),LL(0x6c26273a,0x8bcaced2), LL(0xb5536742,0xe22ecf1d),LL(0x1a9e068b,0x013dd897),LL(0x8a7909c5,0x17f411cb),LL(0x861dd506,0x5757ac98), + LL(0x1e935abb,0x85de1f0d),LL(0x154de37a,0xdefd10b4),LL(0x369cebb5,0xb8d9e392),LL(0x761324be,0x54d5ef9b), LL(0x74f17e26,0x4d6341ba),LL(0x78c1dde4,0xc0a0e3c8),LL(0x87d918fd,0xa6d77581),LL(0x02ca3a13,0x66876015), + LL(0xf36658f0,0xc7313e9c),LL(0x71f8057e,0xc433ef1c),LL(0x1b6a835a,0x85326246),LL(0x7c86394c,0xc8f05398), LL(0xe983c4a1,0xff398cdf),LL(0x03b7b931,0xbf5e8162),LL(0xb7b9045b,0x93193c46),LL(0xa4a6e46b,0x1e4ebf5d), + LL(0x43a24fe7,0xf9942a60),LL(0xffb3492b,0x29c1191e),LL(0x902fde05,0x9f662449),LL(0x6713c32d,0xc792a7ac), LL(0xb737982c,0x2fd88ad8),LL(0xa21e60e3,0x7e3a0319),LL(0x7383591a,0x09b0de44),LL(0x8310a456,0x6df141ee), + LL(0xe6d6f471,0xaec1a039),LL(0x1198d12e,0x14b2ba0f),LL(0x3aeee5ac,0xebc1a160),LL(0xe0b964ce,0x401f4836), LL(0x4fd03f66,0x2ee43796),LL(0xdd8f3f12,0x3fdb4e49),LL(0x29380f18,0x6ef267f6),LL(0x8da64d16,0x3e8e9670), + LL(0x207674f1,0xbc19180c),LL(0x33ae8fdb,0x112e09a7),LL(0x6aaeb71e,0x99667554),LL(0xe101b1c7,0x79432af1), LL(0xde2ddec6,0xd5eb558f),LL(0x5357753f,0x81392d1f),LL(0x3ae1158a,0xa7a76b97),LL(0x4a899991,0x416fbbff), + LL(0x0d4a9dcf,0x9e65fdfd),LL(0x944ddf12,0x7bc29e48),LL(0x3c856866,0xbc1a92d9),LL(0x6e98dfe2,0x273c6905), LL(0xcdfaa6b8,0x69fce418),LL(0x5061c69f,0x606bd823),LL(0x6af75e27,0x42d495a0),LL(0x6d873a1f,0x8ed3d505), + LL(0x6ab25b6a,0xaf552841),LL(0x2b1a4523,0xc6c0ffc7),LL(0x21c99e03,0xab18827b),LL(0x9034691b,0x060e8648), LL(0x93c7f398,0x5207f90f),LL(0x82f8d10b,0x9f4a96cb),LL(0x3ad0f9e3,0xdd71cd79),LL(0xfc3a54f5,0x84f435d2), + LL(0x8e33787f,0x4b03c55b),LL(0xa6384673,0xef42f975),LL(0x5051b9f0,0xff7304f7),LL(0x741c87c2,0x18aca1dc), LL(0x2d4bfe80,0x56f120a7),LL(0x053e732c,0xfd823b3d),LL(0x7537ca16,0x11bccfe4),LL(0x1b5a996b,0xdf6c9c74), + LL(0x904fc3fa,0xee7332c7),LL(0xc7e3636a,0x14a23f45),LL(0xf091d9aa,0xc38659c3),LL(0xb12d8540,0x4a995e5d), LL(0xf3a5598a,0x20a53bec),LL(0xb1eaa995,0x56534b17),LL(0xbf04e03c,0x9ed3dca4),LL(0xd8d56268,0x716c563a), + LL(0x1d6178e7,0x27ba77a4),LL(0x68a1ff8e,0xe4c80c40),LL(0x0a13f63d,0x75011099),LL(0xa61d46f3,0x7bf33521), LL(0x10b365bb,0x0aff218e),LL(0x0fd7ea75,0x81021804),LL(0xa4b3a925,0x05a3fd8a),LL(0x9b3db4e6,0xb829e75f), + LL(0x4d53e5fb,0x6bdc75a5),LL(0xd52717e3,0x04a5dc02),LL(0xe9a42ec2,0x86af502f),LL(0x2630e382,0x8867e8fb), LL(0xbec9889b,0xbf845c6e),LL(0xcb47c98d,0x54f491f2),LL(0x790c2a12,0xa3091fba),LL(0xc20f708b,0xd7f6fd78), + LL(0xacde5e17,0xa569ac30),LL(0x6852b4d7,0xd0f996d0),LL(0x4609ae54,0xe51d4bb5),LL(0x0daed061,0x3fa37d17), LL(0x34b8fb41,0x62a88684),LL(0x9efb64f1,0x99a2acbd),LL(0x6448e1f2,0xb75c1a5e),LL(0x42b5a069,0xfa99951a), + LL(0x2f3b26e7,0x6d956e89),LL(0xda875247,0xf4709860),LL(0x2482dda3,0x3ad15179),LL(0x017d82f0,0xd64110e3), LL(0xfad414e4,0x14928d2c),LL(0x2ed02b24,0x2b155f58),LL(0xcb821bf1,0x481a141b),LL(0x4f81f5da,0x12e3c770), + LL(0x9fff8381,0xe49c5de5),LL(0x5bbec894,0x11053232),LL(0x454d88c4,0xa0d051cc),LL(0x1f8e531b,0x4f6db89c), LL(0xca563a44,0x34fe3fd6),LL(0x58da8ab9,0x7f5c2215),LL(0x9474f0a1,0x8445016d),LL(0xcb7d8a0a,0x17d34d61), + LL(0x1c474019,0x8e9d3910),LL(0xd52ceefb,0xcaff2629),LL(0xc1622c2b,0xf9cf3e32),LL(0xe9071a05,0xd4b95e3c), LL(0x1594438c,0xfbbca61f),LL(0x04aadedf,0x1eb6e6a6),LL(0x68e14940,0x853027f4),LL(0xdfabda9c,0x221d322a), + LL(0xb7cb179a,0xed8ea9f6),LL(0xb7934dcc,0xdc7b764d),LL(0x5e09180d,0xfcb13940),LL(0xb47dc2dd,0x6629a6bf), LL(0x9f5a915e,0xbfc55e4e),LL(0x6204441e,0xb1db9d37),LL(0x930c5f53,0xf82d68cf),LL(0xcbb605b1,0x17d3a142), + LL(0x308780f2,0xdd5944ea),LL(0x3845f5e4,0xdc8de761),LL(0x7624d7a3,0x6beaba7d),LL(0x304df11e,0x1e709afd), LL(0x02170456,0x95364376),LL(0xc8f94b64,0xbf204b3a),LL(0x5680ca68,0x4e53af7c),LL(0xe0c67574,0x0526074a), + LL(0xecd92af6,0x95d8cef8),LL(0x6cd1745a,0xe6b9fa7a),LL(0xa325c3e4,0x3d546d3d),LL(0x9ae93aae,0x1f57691d), LL(0x9d2e1a33,0xe891f3fe),LL(0xac063d35,0xd430093f),LL(0x5513a327,0xeda59b12),LL(0x5536f18f,0xdc2134f3), + LL(0x5c210286,0xaa51fe2c),LL(0x1cab658c,0x3f68aaee),LL(0xf9357292,0x5a23a00b),LL(0x7efdabed,0x9a626f39), LL(0x199d78e3,0xfe2b3bf3),LL(0x71bbc345,0xb7a2af77),LL(0x1e59802c,0x3d19827a),LL(0xb487a51c,0x823bbc15), + LL(0x99d0a422,0x856139f2),LL(0xf456c6fb,0x9ac3df65),LL(0x701f8bd6,0xaddf65c6),LL(0x3758df87,0x149f321e), LL(0x721b7eba,0xb1ecf714),LL(0x31a3312a,0xe17df098),LL(0xd5c4d581,0xdb2fd6ec),LL(0x8fcea1b3,0xfd02996f), + LL(0x7882f14f,0xe29fa63e),LL(0x07c6cadc,0xc9f6dc35),LL(0xb882bed0,0x46f22d6f),LL(0xd118e52c,0x1a45755b), LL(0x7c4608cf,0x9f2c7c27),LL(0x568012c2,0x7ccbdf32),LL(0x61729b0e,0xfcb0aedd),LL(0xf7d75dbf,0x7ca2ca9e), + LL(0x6f640f62,0xf58fecb1),LL(0x39f51946,0xe274b92b),LL(0x6288af44,0x7f4dfc04),LL(0xeac329e5,0x0a91f32a), LL(0xd6aaba31,0x43ad274b),LL(0x0f6884f9,0x719a1640),LL(0xdaf91e20,0x685d29f6),LL(0x27e49d52,0x5ec1cc33), + LL(0x3b54a059,0x38f4de96),LL(0xefbcfdb3,0x0e0015e5),LL(0x4dbb8da6,0x177d23d9),LL(0x97a617ad,0x98724aa2), LL(0xfdb6558e,0x30f0885b),LL(0xc7899a96,0xf9f7a28a),LL(0x872dc112,0xd2ae8ac8),LL(0x73c3c459,0xfa0642ca), + LL(0xe7dfc8d6,0x15296981),LL(0x1fb5b94a,0x67cd4450),LL(0x0eddfd37,0x0ec71cf1),LL(0x9a8eddc7,0xc7e5eeb3), LL(0x81d95028,0x02ac8e3d),LL(0x70b0e35d,0x0088f172),LL(0xe1881fe3,0xec041fab),LL(0xd99e7faa,0x62cf71b8), + LL(0xe0f222c2,0x5043dea7),LL(0x72e65142,0x309d42ac),LL(0x9216cd30,0x94fe9ddd),LL(0x0f87feec,0xd6539c7d), LL(0x432ac7d7,0x03c5a57c),LL(0x327fda10,0x72692cf0),LL(0x280698de,0xec28c85f),LL(0x7ec283b1,0x2331fb46), + LL(0x2867e633,0xd34bfa32),LL(0x0a9cc815,0x78709a82),LL(0x875e2fa5,0xb7fe6964),LL(0x9e98bfb5,0x25cc064f), LL(0x493a65c5,0x9eb0151c),LL(0x53182464,0x5fb5d941),LL(0xf04618e2,0x69e6f130),LL(0xf89c8ab6,0xa8ecec22), + LL(0xb96209bd,0xcd6ac88b),LL(0xb3e1c9e0,0x65fa8cdb),LL(0x4a8d8eac,0xa47d22f5),LL(0x8d33f963,0x83895cdf), LL(0xb56cd3d1,0xa8adca59),LL(0xdaf38232,0x10c8350b),LL(0xa5080a9f,0x2b161fb3),LL(0x3af65b3a,0xbe7f5c64), + LL(0x97403a11,0x2c754039),LL(0x121b96af,0x94626cf7),LL(0x6a983ec2,0x431de7c4),LL(0x52cc3df7,0x3780dd3a), LL(0x2baf8e3b,0xe28a0e46),LL(0x51d299ae,0xabe68aad),LL(0x647a2408,0x603eb8f9),LL(0x5c750981,0x14c61ed6), + LL(0xc53352e7,0x88b34414),LL(0x1337d46e,0x5a34889c),LL(0xf95f2bc8,0x612c1560),LL(0xd4807a3a,0x8a3f8441), LL(0x5224da68,0x680d9e97),LL(0xc3eb00e9,0x60cd6e88),LL(0x9a6bc375,0x3875a98e),LL(0x4fd554c2,0xdc80f924), + LL(0x6ac77407,0x6c4b3415),LL(0x25420681,0xa1e5ea8f),LL(0x4607a458,0x541bfa14),LL(0x96d7fbf9,0x5dbc7e7a), LL(0x31590a47,0x646a851b),LL(0x15ee6df8,0x039e85ba),LL(0xd7b43fc0,0xd19fa231),LL(0x299a0e04,0x84bc8be8), + LL(0xf20df03a,0x2b9d2936),LL(0x8608d472,0x24054382),LL(0x9149202a,0x76b6ba04),LL(0x3670e7b7,0xb21c3831), LL(0xd6fdee10,0xddd93059),LL(0x78488e71,0x9da47ad3),LL(0xa0fcfb25,0x99cc1dfd),LL(0x64696954,0x42abde10), + LL(0x17eab9fe,0x14cc15fc),LL(0xd3e70972,0xd6e863e4),LL(0x6432112c,0x29a7765c),LL(0x5b0774d8,0x88660001), LL(0x2c088eae,0x3729175a),LL(0x8230b8d4,0x13afbcae),LL(0x915f4379,0x44768151),LL(0xd8d22812,0xf086431a), + LL(0xc298b974,0x37461955),LL(0xf8711e04,0x905fb5f0),LL(0xfe969d18,0x787abf3a),LL(0x6f6a494e,0x392167c2), LL(0x28c511da,0xfc7a0d2d),LL(0xb66a262d,0xf127c7dc),LL(0xfd63fdf0,0xf9c4bb95),LL(0x3913ef46,0x90016589), + LL(0x11aa600d,0x74d2a73c),LL(0x9fb5ab52,0x2f5379bd),LL(0x7fb70068,0xe49e53a4),LL(0x404aa9a7,0x68dd39e5), LL(0x2ecaa9c3,0xb9b0cf57),LL(0xe824826b,0xba0e103b),LL(0x4631a3c4,0x60c2198b),LL(0xfa8966a2,0xc5ff84ab), + LL(0xac95aff8,0x2d6ebe22),LL(0xb5a46d09,0x1c9bb6db),LL(0x53ee4f8d,0x419062da),LL(0xbb97efef,0x7b9042d0), LL(0x830cf6bd,0x0f87f080),LL(0x6ec8a6c6,0x4861d19a),LL(0x202f01aa,0xd3a0daa1),LL(0xf25afbd5,0xb0111674), + LL(0x1afb20d9,0x6d00d6cf),LL(0x40671bc5,0x13695000),LL(0x2485ea9b,0x913ab0dc),LL(0x9eef61ac,0x1f2bed06), LL(0x6d799e20,0x850c8217),LL(0x3271c2de,0x93415f37),LL(0x6c4f5910,0x5afb06e9),LL(0xc4e9e421,0x688a52df), + LL(0xe2a9a6db,0x30495ba3),LL(0x58f9268b,0x4601303d),LL(0x7eb0f04f,0xbe3b0dad),LL(0x4456936d,0x4ea47250), LL(0xd33fd3e7,0x8caf8798),LL(0xeb433708,0x1ccd8a89),LL(0x87fd50ad,0x9effe3e8),LL(0x6b29c4df,0xbe240a56), + LL(0xca0e7ebd,0xec4ffd98),LL(0xe748616e,0xf586783a),LL(0xc77baa99,0xa5b00d8f),LL(0xb4f34c9c,0x0acada29), LL(0x0fe723ac,0x36dad67d),LL(0x39c36c1e,0x1d8e53a5),LL(0x1f4bea41,0xe4dd342d),LL(0xebc9e4e0,0x64fd5e35), + LL(0x57908805,0x96f01f90),LL(0x5ed480dd,0xb5b9ea3d),LL(0x3efd2dd0,0x366c5dc2),LL(0x6e9dfa27,0xed2fe305), LL(0x6e9197e2,0x4575e892),LL(0xab502a5d,0x11719c09),LL(0xe81f213f,0x264c7bec),LL(0x55f5c457,0x741b9241), + LL(0x49a5f4f4,0x78ac7b68),LL(0x9fc45b7d,0xf91d70a2),LL(0xb0f5f355,0x39b05544),LL(0xeef930d9,0x11f06bce), LL(0x038d05e1,0xdb84d25d),LL(0xbacc1d51,0x04838ee5),LL(0x9e8ee00b,0x9da3ce86),LL(0xc36eda1f,0xc3412057), + LL(0x64d9c2f4,0xae80b913),LL(0xa010a8ff,0x7468bac3),LL(0x37359d41,0xdfd20037),LL(0x15efeacc,0x1a0f5ab8), LL(0x659d0ce0,0x7c25ad2f),LL(0x6785cff1,0x4011bcbb),LL(0x7e2192c7,0x128b9912),LL(0x13ccb0e8,0xa549d8e1), + LL(0xc85438b1,0x805588d8),LL(0xbc25cb27,0x5680332d),LL(0x1a4bfdf4,0xdcd1bc96),LL(0x706f6566,0x779ff428), LL(0xf059987a,0x8bbee998),LL(0xcc686de7,0xf6ce8cf2),LL(0x953cfdb2,0xf8ad3c4a),LL(0x2205da36,0xd1d426d9), + LL(0xc781a241,0xb3c0f13f),LL(0xd75362a8,0x3e89360e),LL(0xc8a91184,0xccd05863),LL(0xefa8a7f4,0x9bd0c9b7), LL(0x8a912a4b,0x97ee4d53),LL(0xbcf518fd,0xde5e15f8),LL(0xc467e1e0,0x6a055bf8),LL(0x1587e256,0x10be4b4b), + LL(0x668621c9,0xd90c14f2),LL(0xab9c92c1,0xd5518f51),LL(0xd6d47b3c,0x8e6a0100),LL(0x66716175,0xcbe980dd), LL(0xddd83683,0x500d3f10),LL(0x99cac73c,0x3b6cb35d),LL(0x6083d550,0x53730c8b),LL(0xdf0a1987,0xcf159767), + LL(0x43ad73b3,0x84bfcf53),LL(0x4f035a94,0x1b528c20),LL(0x33eeac69,0x4294edf7),LL(0x817f3240,0xb6283e83), LL(0x0a5f25b1,0xc3fdc959),LL(0x5844ee22,0xefaf8aa5),LL(0xdbdde4de,0xde269ba5),LL(0xc56133bf,0xe3347160), + LL(0x8d9ea9f8,0xc1184219),LL(0xf3fc1ab5,0x090de5db),LL(0x0bf22cda,0x404c37b1),LL(0xf5618894,0x7de20ec8), LL(0xecdaecab,0x754c588e),LL(0x88342743,0x6ca4b0ed),LL(0xf4a938ec,0x76f08bdd),LL(0x91493ccb,0xd182de89), + LL(0xc8a4186a,0xd652c53e),LL(0x946d8e33,0xb3e878db),LL(0x5f37663c,0x088453c0),LL(0xb407748b,0x5cd9daaa), LL(0x586d5e72,0xa1f5197f),LL(0xc443ca59,0x47500be8),LL(0xe2652424,0x78ef35b2),LL(0x6dd7767d,0x09c5d26f), + LL(0xa74d3f7b,0x7175a79a),LL(0xcf5ea459,0x0428fd8d),LL(0xa5d1746d,0x511cb97c),LL(0xe71d1278,0x36363939), LL(0x10350bf4,0xcf2df955),LL(0x60aae782,0xb3817439),LL(0x3e688809,0xa748c0e4),LL(0xd7a5a006,0x98021fbf), + LL(0x0e367a98,0x9076a70c),LL(0x0f62b7c2,0xbea1bc15),LL(0x30fe0343,0x2645a68c),LL(0x699dc14f,0xacaffa78), LL(0x457bf9c4,0xf4469964),LL(0x0d2ead83,0x0db6407b),LL(0xb2c6f3eb,0x68d56cad),LL(0xf376356c,0x3b512e73), + LL(0xfce10408,0xe43b0e1f),LL(0x5a5e257d,0x89ddc003),LL(0x0362e5b3,0xb0ae0d12),LL(0xb0519161,0x07f983c7), LL(0x5d5231e7,0xc2e94d15),LL(0x0b4f9513,0xcff22aed),LL(0x6ad0b0b5,0xb02588dd),LL(0x11d0dcd5,0xb967d1ac), + LL(0xcf777b6c,0x8dac6bc6),LL(0x4c6d1959,0x0062bdbd),LL(0x0ef5cc85,0x53da71b5),LL(0x4006f14f,0x07012c7d), LL(0xac47800d,0x4617f962),LL(0xc102ed75,0x53365f2b),LL(0x4ab8c9d3,0xb422efcb),LL(0x34af31c9,0x195cb26b), + LL(0x05f2c4ce,0x3a926e29),LL(0x9856966c,0xbd2bdecb),LL(0x85527015,0x5d16ab3a),LL(0x4486c231,0x9f81609e), LL(0xda350002,0xd8b96b2c),LL(0xfa1b7d36,0xbd054690),LL(0xe71d79bc,0xdc90ebf5),LL(0x08964e4e,0xf241b6f9), + LL(0x2fe3cd4c,0x7c838643),LL(0xb4bc633c,0xe0f33acb),LL(0x3d139f1f,0xb4a9ecec),LL(0xdc4a1f49,0x05ce69cd), LL(0xf5f98aaf,0xa19d1b16),LL(0x6f23e0ef,0x45bb71d6),LL(0x46cdfdd3,0x33789fcd),LL(0xcee040ca,0x9b8e2978), + LL(0xae0a6828,0x9c69b246),LL(0x7078d5aa,0xba533d24),LL(0x7bb4fbdb,0x7a2e42c0),LL(0x7035385c,0xcfb4879a), LL(0x3281705b,0x8c3dd30b),LL(0x404fe081,0x7e361c6c),LL(0x3f604edf,0x7b21649c),LL(0xe52ffe47,0x5dbf6a3f), + LL(0x4b54d9bf,0xc41b7c23),LL(0x3511c3d9,0x1374e681),LL(0xc1b2b758,0x1863bf16),LL(0x1e9e6a96,0x90e78507), LL(0x5d86f174,0xab4bf98d),LL(0x85e96fe4,0xd74e0bd3),LL(0xcac5d344,0x8afde39f),LL(0xbd91b847,0x90946dbc), + LL(0xfe1a838c,0xf5b42358),LL(0x620ac9d8,0x05aae6c5),LL(0xa1ce5a0b,0x8e193bd8),LL(0x4dabfd72,0x8f710571), LL(0x182caaac,0x8d8fdd48),LL(0x040745cf,0x8c4aeefa),LL(0xf3b93e6d,0x73c6c30a),LL(0x16f42011,0x991241f3), + LL(0xe457a477,0xa0158eea),LL(0xee6ddc05,0xd19857db),LL(0x18c41671,0xb3265224),LL(0x3c2c0d58,0x3ffdfc7e), LL(0x26ee7cda,0x3a3a5254),LL(0xdf02c3a8,0x341b0869),LL(0x723bbfc8,0xa023bf42),LL(0x14452691,0x3d15002a), +}, +/* digit=31 base_pwr=2^217 */ +{ + LL(0x85edfa30,0x5ef7324c),LL(0x87d4f3da,0x25976554),LL(0xdcb50c86,0x352f5bc0),LL(0x4832a96c,0x8f6927b0), LL(0x55f2f94c,0xd08ee1ba),LL(0x344b45fa,0x6a996f99),LL(0xa8aa455d,0xe133cb8d),LL(0x758dc1f7,0x5d0721ec), + LL(0x79e5fb67,0x6ba7a920),LL(0x70aa725e,0xe1331feb),LL(0x7df5d837,0x5080ccf5),LL(0x7ff72e21,0xe4cae01d), LL(0x0412a77d,0xd9243ee6),LL(0xdf449025,0x06ff7cac),LL(0x23ef5a31,0xbe75f7cd),LL(0x0ddef7a8,0xbc957822), + LL(0xb0ce1c55,0x8cf7230c),LL(0x0bbfb607,0x5b534d05),LL(0x0e16363b,0xee1ef113),LL(0xb4999e82,0x27e0aa7a), LL(0x79362c41,0xce1dac2d),LL(0x91bb6cb0,0x67920c90),LL(0x2223df24,0x1e648d63),LL(0xe32e8f28,0x0f7d9eef), + LL(0xfa833834,0x6943f39a),LL(0xa6328562,0x22951722),LL(0x4170fc10,0x81d63dd5),LL(0xaecc2e6d,0x9f5fa58f), LL(0xe77d9a3b,0xb66c8725),LL(0x6384ebe0,0x11235cea),LL(0x5845e24a,0x06a8c118),LL(0xebd093b1,0x0137b286), + LL(0x44ace150,0xc589e1ce),LL(0x4381e97c,0xe0f8d3d9),LL(0x62c5a4b8,0x59e99b11),LL(0xfd0ec9f9,0x90d262f7), LL(0x283e13c9,0xfbc854c9),LL(0xaedc7085,0x2d04fde7),LL(0x47dcbecb,0x057d7765),LL(0x9a76fa5f,0x8dbdf591), + LL(0x0de1e578,0xd0150695),LL(0xe9f72bc6,0x2e1463e7),LL(0x1b39eca5,0xffa68441),LL(0x7c037f2f,0x673c8530), LL(0x747f91da,0xd0d6a600),LL(0xc9cb78e9,0xb08d43e1),LL(0x27b5cef5,0x0fc0c644),LL(0xa60a2fd6,0x5c1d160a), + LL(0x28c8e13b,0xf98cae53),LL(0xb2eddcd1,0x375f10c4),LL(0x5cce06ad,0xd4eb8b7f),LL(0x80a2e1ef,0xb4669f45), LL(0x5bbd8699,0xd593f9d0),LL(0xe7976d13,0x5528a4c9),LL(0x1c7e28d3,0x3923e095),LL(0x3f6bb577,0xb9293790), + LL(0xc42bd6d2,0xdb567d6a),LL(0xbb1f96ae,0x6df86468),LL(0x4843b28e,0x0efe5b1a),LL(0x6379b240,0x961bbb05), LL(0x70a6a26b,0xb6caf5f0),LL(0x328e6e39,0x70686c0d),LL(0x895fc8d3,0x80da06cf),LL(0xb363fdc9,0x804d8810), + LL(0x207f1670,0xbe22877b),LL(0x4e615291,0x9b0dd188),LL(0x97a3c2bf,0x625ae8dc),LL(0x439b86e8,0x08584ef7), LL(0xdcd898ff,0xde7190a5),LL(0x2058ee3d,0x26286c40),LL(0x5f87b1c1,0x3db0b217),LL(0x102a6db5,0xcc334771), + LL(0x2f770fb1,0xd99de954),LL(0x4cd7535e,0x97c1c620),LL(0x3f09cefc,0xd3b6c448),LL(0x5a63b4f8,0xd725af15), LL(0xc01e20ec,0x0c95d24f),LL(0x9ae7121f,0xdfd37494),LL(0xec77b7ec,0x7d6ddb72),LL(0x0353a4ae,0xfe079d3b), + LL(0x2e6ac8d2,0x3066e70a),LL(0x106e5c05,0x9c6b5a43),LL(0xede59b8c,0x52d3c6f5),LL(0xfccec9ae,0x30d6a5c3), LL(0x4fc0a9ef,0xedec7c22),LL(0x95c16ced,0x190ff083),LL(0x94de0fde,0xbe12ec8f),LL(0x852d3433,0x0d131ab8), + LL(0x85701291,0x42ace07e),LL(0x194061a8,0x94793ed9),LL(0xd7f4a485,0x30e83ed6),LL(0xf9eeff4d,0x9eec7269), LL(0x0c9d8005,0x90acba59),LL(0x1e79b9d1,0x5feca458),LL(0x1d506a1e,0x8fbe5427),LL(0x2439cfa7,0xa32b2c8e), + LL(0x73dd0b4e,0x1671c173),LL(0x44a054c6,0x37a28214),LL(0x4e8b53f1,0x81760a1b),LL(0xf9f93b9e,0xa6c04224), LL(0xcf671e3c,0x18784b34),LL(0xcda9b994,0x81bbecd2),LL(0xb2ab3848,0x38831979),LL(0xf2e03c2d,0xef54feb7), + LL(0xfb8088fa,0xcf197ca7),LL(0x4ddc96c5,0x01427247),LL(0x30777176,0xa2d2550a),LL(0x4d0cf71d,0x53469898), LL(0x3a2aaac6,0x6ce937b8),LL(0x5af38d9b,0xe9f91dc3),LL(0xc8bf2899,0x2598ad83),LL(0xb5536c16,0x8e706ac9), + LL(0xf688dc98,0x40dc7495),LL(0x124c4afc,0x26490cd7),LL(0x1f18775c,0xe651ec84),LL(0xb4fdaf4a,0x393ea6c3), LL(0x7f338e0d,0x1e1f3343),LL(0x6053e7b5,0x39fb832b),LL(0x619e14d5,0x46e702da),LL(0xcdeef6e0,0x859cacd1), + LL(0x4462007d,0x63b99ce7),LL(0x4cb5f5b7,0xb8ab48a5),LL(0xf55edde7,0x9ec673d2),LL(0x8cfaefda,0xd1567f74), LL(0x0887bcec,0x46381b6b),LL(0xe178f3c2,0x694497ce),LL(0x1e6266cb,0x5e6525e3),LL(0x697d6413,0x5931de26), + LL(0x0e58d493,0x87f8df7c),LL(0x58b73f12,0xb1ae5ed0),LL(0xdea0c34d,0xc368f784),LL(0x859a91a0,0x9bd0a120), LL(0xcc863c68,0xb00d88b7),LL(0x3d1f4d65,0x3a1cc11e),LL(0x0aa85593,0xea38e0e7),LL(0x7dc4aee8,0x37f13e98), + LL(0xbc947bad,0x10d38667),LL(0x2a36ee2e,0x738e07ce),LL(0xc577fcac,0xc93470cd),LL(0x2782470d,0xdee1b616), LL(0x2e793d12,0x36a25e67),LL(0xe0f186da,0xd6aa6cae),LL(0x80e07af7,0x474d0fd9),LL(0xba8a5cd4,0xf7cdc47d), + LL(0xab15247f,0x28af6d9d),LL(0x493a537f,0x7c789c10),LL(0x23a334e7,0x7ac9b110),LL(0x12c9c277,0x0236ac09), LL(0x1d7a5144,0xa7e5bd25),LL(0xf13ec4ec,0x098b9c2a),LL(0xd3f0abca,0x3639daca),LL(0xa23960f9,0x642da81a), + LL(0x4f7269b1,0x7d2e5c05),LL(0xe287c385,0xfcf30777),LL(0xf2a46f21,0x10edc84f),LL(0x4f43fa36,0x35441757), LL(0xfd703431,0xf1327899),LL(0x16dd587a,0xa438d7a6),LL(0xe9c8352d,0x65c34c57),LL(0x5cc5a24e,0xa728edab), + LL(0x42531689,0xaed78abc),LL(0x010963ef,0x0a51a0e8),LL(0xd717d9b3,0x5776fa0a),LL(0x7dd3428b,0xf356c239), LL(0x8d3a3dac,0x29903fff),LL(0x3d94491f,0x409597fa),LL(0xbf4a56a4,0x4cd7a5ff),LL(0x8adab462,0xe5096474), + LL(0x5c3427b0,0xa97b5126),LL(0xd282c9bd,0x6401405c),LL(0x222c5c45,0x3629f8d7),LL(0xe8d50aed,0xb1c02c16), LL(0xd9635bc9,0xbea2ed75),LL(0x6e24552f,0x226790c7),LL(0x65f1d066,0x3c33f2a3),LL(0x6dfccc2e,0x2a43463e), + LL(0xdb483761,0x8cc3453a),LL(0x65d5672b,0xe7cc6085),LL(0xde3efc87,0x277ed6cb),LL(0x69234eaf,0x19f2f368), LL(0x5c0b800b,0x9aaf4317),LL(0x8b6da6e2,0x1f1e7c89),LL(0xb94ec75e,0x6cfb4715),LL(0x453118c2,0xd590dd5f), + LL(0x1f17a34c,0x14e49da1),LL(0x235a1456,0x5420ab39),LL(0x2f50363b,0xb7637241),LL(0xc3fabb6e,0x7b15d623), LL(0xe274e49c,0xa0ef40b1),LL(0x96b1860a,0x5cf50744),LL(0x66afe5a4,0xd6583fbf),LL(0xf47e3e9a,0x44240510), + LL(0x11b2d595,0x99254343),LL(0xeec8df57,0xf1367499),LL(0x3e73dd05,0x3cb12c61),LL(0x7dac102a,0xd248c033), LL(0xa77739f5,0xcf154f13),LL(0x23d2af42,0xbf4288cb),LL(0x32e4a1cf,0xaa64c9b6),LL(0xc8a208f3,0xee8c07a8), + LL(0x6fe8393f,0xe10d4999),LL(0xe91f3a32,0x0f809a3f),LL(0x802f63c8,0x61096d1c),LL(0x57750d3d,0x289e1462), LL(0x9889feea,0xed06167e),LL(0xe0993909,0xd5c9c0e2),LL(0x56508ac6,0x46fca0d8),LL(0x4f1b8e83,0x91826047), + LL(0x9a4a2751,0x4f2c877a),LL(0xcae6fead,0x71bd0072),LL(0x06aa1941,0x38df8dcc),LL(0x63beeaa8,0x5a074b4c), LL(0xc1cec8ed,0xd6d65934),LL(0xaabc03bd,0xa6ecb49e),LL(0xde8a8415,0xaade91c2),LL(0x691136e0,0xcfb0efdf), + LL(0x23ab3495,0x11af45ee),LL(0x0b77463d,0xa132df88),LL(0x815d06f4,0x8923c15c),LL(0x0d61a436,0xc3ceb3f5), LL(0xe88fb1da,0xaf52291d),LL(0x1da12179,0xea057974),LL(0xd2fef720,0xb0d7218c),LL(0x8e1d8845,0x6c0899c9), + LL(0x752ddad7,0x98157504),LL(0xa1a68a97,0xd60bd74f),LL(0xf658fb99,0x7047a3a9),LL(0x5f8511e4,0x1f5d86d6), LL(0x4b5a6d88,0xb8a4bc42),LL(0x1abefa7d,0x69eb2c33),LL(0x13c9c510,0x95bf39e8),LL(0xd48aab43,0xf571960a), + LL(0x704e23c6,0x7e8cfbcf),LL(0x28aaa65b,0xc71b7d22),LL(0x245e3c83,0xa041b2bd),LL(0xd21854ff,0x69b98834), LL(0x963bfeec,0x89d227a3),LL(0xde7da7cb,0x99947aaa),LL(0xee68a9b1,0x1d9ee9db),LL(0x698ec368,0x0a08f003), + LL(0x78ef2487,0xe9ea4094),LL(0x02cfec26,0xc8d2d415),LL(0xb7dcf328,0xc52f9a6e),LL(0x85b6a937,0x0ed489e3), LL(0xbef3366e,0x9b94986b),LL(0xedddddb8,0x0de59c70),LL(0xeadddbe2,0xffdb748c),LL(0x8266ea40,0x9b9784bb), + LL(0x1a93507a,0x142b5502),LL(0x8d3c06cf,0xb4cd1187),LL(0x91ec3f40,0xdf70e76a),LL(0x4e7553c2,0x484e81ad), LL(0x272e9d6e,0x830f87b5),LL(0xc6ff514a,0xea1c93e5),LL(0xc4192a8e,0x67cc2adc),LL(0x42f4535a,0xc77e27e2), + LL(0xd2b713c5,0x9cdbab36),LL(0xcf7b0cd3,0x86274ea0),LL(0x09af826b,0x784680f3),LL(0x0c72dea3,0xbfcc837a), LL(0xd6529b73,0xa8bdfe9d),LL(0x63a88002,0x708aa228),LL(0xc91d45b9,0x6c7a9a54),LL(0xfd004f56,0xdf1a38bb), + LL(0xb8bad853,0x2e8c9a26),LL(0x3723eae7,0x2d52cea3),LL(0x56ca2830,0x054d6d81),LL(0x9a8dc411,0xa3317d14), LL(0xfd4ddeda,0xa08662fe),LL(0xb55d792b,0xed2a153a),LL(0xbfc6e944,0x7035c16a),LL(0x00171cf3,0xb6bc5834), + LL(0x83d102b6,0xe27152b3),LL(0x0646b848,0xfe695a47),LL(0x916e6d37,0xa5bb09d8),LL(0x0d17015e,0xb4269d64), LL(0x0a1d2285,0x8d8156a1),LL(0x46d26d72,0xfeef6c51),LL(0x4c5434a7,0x9dac57c8),LL(0x59d39e31,0x0282e5be), + LL(0x721c486d,0xedfff181),LL(0xbc58824e,0x301baf10),LL(0x00570031,0x8136a6aa),LL(0x1cddde68,0x55aaf78c), LL(0x59c63952,0x26829371),LL(0x8bc25baf,0x3a3bd274),LL(0xb7e52dc3,0xecdf8657),LL(0xfd78e6c8,0x2dd8c087), + LL(0xf5531461,0x20553274),LL(0x5d95499b,0x8b4a1281),LL(0x1a80f9d2,0xe2c8763a),LL(0x4ddec758,0xd1dbe32b), LL(0x30c34169,0xaf12210d),LL(0x78baa533,0xba74a953),LL(0xa438f254,0x3d133c6e),LL(0x201bef5b,0xa431531a), + LL(0xf669d7ec,0x15295e22),LL(0x357fb515,0xca374f64),LL(0xeaa3fdb3,0x8a8406ff),LL(0xdf3f2da8,0x106ae448), LL(0x33c8e9a1,0x8f9b0a90),LL(0x71ad5885,0x234645e2),LL(0x1c0aed14,0x3d083224),LL(0x7a942d46,0xf10a7d3e), + LL(0x40d5c9be,0x7c11deee),LL(0xba84ed98,0xb2bae7ff),LL(0xaad58ddd,0x93e97139),LL(0x3f6d1fa3,0x3d872796), LL(0x8569ff13,0x483aca81),LL(0x9a600f72,0x8b89a5fb),LL(0xc06f2b86,0x4cbc27c3),LL(0x63ad9c0b,0x22130713), + LL(0x48ac2840,0xb5358b1e),LL(0xecba9477,0x18311294),LL(0xa6946b43,0xda58f990),LL(0x9ab41819,0x3098baf9), LL(0x4198da52,0x66c4c158),LL(0x146bfd1b,0xab4fc17c),LL(0xbf36a908,0x2f0a4c3c),LL(0x58cf7838,0x2ae9e34b), + LL(0x3fa11b1f,0xf411529e),LL(0x974af2b4,0x21e43677),LL(0xc230793b,0x7c20958e),LL(0x16e840f3,0x710ea885), LL(0xc5dc67cf,0xfc0b21fc),LL(0x88405718,0x08d51647),LL(0xcfe49eb7,0xd955c21f),LL(0x56dd4a1f,0x9722a5d5), + LL(0xc861baa5,0xc9ef50e2),LL(0x9505ac3e,0xc0c21a5d),LL(0x8b7c063f,0xaf6b9a33),LL(0x2f4779c1,0xc6370339), LL(0x638167c3,0x22df99c7),LL(0x795db30c,0xfe6ffe76),LL(0xa4854989,0x2b822d33),LL(0x30563aa5,0xfef031dd), + LL(0xd57c667f,0x16b09f82),LL(0xcc0b76f1,0xc70312ce),LL(0xc9118aec,0xbf04a9e6),LL(0x3409d133,0x82fcb419), LL(0xab45d44d,0x1a8ab385),LL(0x617b83a3,0xfba07222),LL(0x58e81b52,0xb05f50dd),LL(0x21ce5aff,0x1d8db553), + LL(0xe344a873,0x3097b8d4),LL(0xfe36d53e,0x7d8d116d),LL(0x7875e750,0x6db22f58),LL(0x43e144ea,0x2dc5e373), LL(0xe799eb95,0xc05f32e6),LL(0x6899e6ec,0xe9e5f4df),LL(0x1fab23d5,0xbdc3bd68),LL(0x73af60e6,0xb72b8ab7), + LL(0x2cecc84a,0x8db27ae0),LL(0x7bdb871c,0x600016d8),LL(0xd7c46f58,0x42a44b13),LL(0xc3a77d39,0xb8919727), LL(0xdafd6088,0xcfc6bbbd),LL(0x6bd20d39,0x1a740146),LL(0x98c41072,0x8c747abd),LL(0xbdf68ea1,0x4c91e765), + LL(0x08819a78,0x7c95e5ca),LL(0xc9587921,0xcf48b729),LL(0xdebbcc7d,0x091c7c5f),LL(0xf0e05149,0x6f287404), LL(0x26cd44ec,0xf83b5ac2),LL(0xcfea250e,0x88ae32a6),LL(0x1d06ebc5,0x6ac5047a),LL(0xd434f781,0xc7e550b4), + LL(0x5c727bd2,0x61ab1cf2),LL(0x1cf915b0,0x2e4badb1),LL(0xf69d3920,0x1b4dadec),LL(0xf14c1dfe,0xe61b1ca6), LL(0xbd6bd51f,0x90b479cc),LL(0x8045ec30,0x8024e401),LL(0x25ef0e62,0xcab29ca3),LL(0x49e4ebc0,0x4f2e9416), + LL(0x0ccced58,0x45eb40ec),LL(0x0da44f98,0x25cd4b9c),LL(0x871812c6,0x43e06458),LL(0x16cef651,0x99f80d55), LL(0xce6dc153,0x571340c9),LL(0xd8665521,0x138d5117),LL(0x4e07014d,0xacdb45bc),LL(0x84b60b91,0x2f34bb38), + LL(0x2ae8921e,0xf44a4fd2),LL(0x892ba1e2,0xb039288e),LL(0xb1c180b2,0x9da50174),LL(0x1693dc87,0x6b70ab66), LL(0xe7057481,0x7e9babc9),LL(0x9c80dc41,0x4581ddef),LL(0x51294682,0x0c890da9),LL(0x3f4736e5,0x0b5629d3), + LL(0xb06f5b41,0x2340c79e),LL(0x4e243469,0xa42e84ce),LL(0x045a71a9,0xf9a20135),LL(0xd27b6fb6,0xefbfb415), LL(0x9d33cd6f,0x25ebea23),LL(0xaa6c0af8,0x9caedb88),LL(0xd9ce6f96,0x53dc7e9a),LL(0x51e0b15a,0x3897f9fd), + LL(0x8e5d788e,0xf51cb1f8),LL(0xe1d490ee,0x1aec7ba8),LL(0xcc58cb3c,0x265991e0),LL(0x9fc3ad31,0x9f306e8c), LL(0x5040a0ac,0x5fed006e),LL(0xfb476f2e,0xca9d5043),LL(0xbeea7a23,0xa19c06e8),LL(0x0edabb63,0xd2865801), + LL(0x6967469a,0xdb92293f),LL(0x8d8a8ed8,0x2894d839),LL(0xbbc77122,0x87c9e406),LL(0x2ea3a26a,0x8671c6f1), LL(0xd7de9853,0xe42df8d6),LL(0xb1f2bcc7,0x2e3ce346),LL(0x899d50cf,0xda601dfc),LL(0xfb1b598f,0xbfc913de), + LL(0xe61f7908,0x81c4909f),LL(0x9bbc7b29,0x192e304f),LL(0xc104b338,0xc3ed8738),LL(0x783f5d61,0xedbe9e47), LL(0x2db30660,0x0c06e9be),LL(0xc0eb7d8e,0xda3e613f),LL(0x322e096e,0xd8fa3e97),LL(0xd336e247,0xfebd91e8), + LL(0xdf655a49,0x8f13ccc4),LL(0x5eb20210,0xa9e00dfc),LL(0xc656b6ea,0x84631d0f),LL(0xd8c0d947,0x93a058cd), LL(0x67bd3448,0x6846904a),LL(0xf394fd5c,0x4a3d4e1a),LL(0xdb225f52,0xc102c1a5),LL(0xfc4f5e9a,0xe3455bba), + LL(0x4b9ad1ce,0x6b36985b),LL(0x5bb7f793,0xa9818536),LL(0x48b1a416,0x6c25e1d0),LL(0x3c81bee7,0x1381dd53), LL(0x7a4a7620,0xd2a30d61),LL(0x39b8944c,0xc8412926),LL(0x7a97c33a,0x3c1c6fbe),LL(0x938664e7,0x941e541d), + LL(0x4a34f239,0x417499e8),LL(0xb90402d5,0x15fdb83c),LL(0x433aa832,0xb75f46bf),LL(0x63215db1,0xb61e15af), LL(0xa127f89a,0xaabe59d4),LL(0x07e816da,0x5d541e0c),LL(0xa618b692,0xaaba0659),LL(0x17266026,0x55327733), + LL(0x95f57552,0xaf53a0fc),LL(0x6cacb0c9,0x32947650),LL(0xc821be01,0x253ff58d),LL(0xa06f1146,0xb0309531), LL(0x05c2e54d,0x59bbbdf5),LL(0x26e8dd22,0x158f27ad),LL(0x397e1e53,0xcc5b7ffb),LL(0x7fc1e50d,0xae03f65b), + LL(0x9c95f0f9,0xa9784ebd),LL(0x24640771,0x5ed9deb2),LL(0x035561c4,0x31244af7),LL(0x7ee857de,0x87332f3a), LL(0x2b9e0d88,0x09e16e9e),LL(0x56a06049,0x52d910f4),LL(0xa9592f48,0x507ed477),LL(0x2365d678,0x85cb917b), + LL(0x4c8998d1,0xf8511c93),LL(0x730ea58f,0x2186a3f1),LL(0xb2029db0,0x50189626),LL(0x02ceb75a,0x9137a6d9), LL(0x748bc82c,0x2fe17f37),LL(0x80469f8c,0x87c2e931),LL(0xbf891aa2,0x850f71cd),LL(0x75ec3d8d,0x0ca1b89b), + LL(0x5e1cd3cd,0x516c43aa),LL(0x9a887c28,0x89397808),LL(0xddea1f9f,0x0059c699),LL(0x8e6868f7,0x7737d6fa), LL(0x60f1524b,0x6d93746a),LL(0xba052aa7,0x36985e55),LL(0xed923ea5,0x41b1d322),LL(0x25852a11,0x3429759f), + LL(0x092e9f41,0xbeca6ec3),LL(0x62256bbd,0x3a238c66),LL(0x70ad487d,0xd82958ea),LL(0x65610d93,0x4ac8aaf9), LL(0x5e4ccab0,0x3fa101b1),LL(0x9de14bfb,0x9bf430f2),LL(0x6531899d,0xa10f5cc6),LL(0xea8ce17d,0x590005fb), + LL(0x24544cb6,0xc437912f),LL(0xd79ac2e3,0x9987b71a),LL(0xc058a212,0x13e3d9dd),LL(0xd2de9606,0x00075aac), LL(0x6cac8369,0x80ab508b),LL(0xf54f6c89,0x87842be7),LL(0x6bc532a4,0xa7ad663d),LL(0x78a91bc8,0x67813de7), + LL(0xc3427239,0x5dcb61ce),LL(0xc56934d9,0x5f3c7cf0),LL(0xe3191591,0xc079e0fb),LL(0xb01aada7,0xe40896bd), LL(0x0492d25f,0x8d466791),LL(0xe7408276,0x8aeb30c9),LL(0x9287aacc,0xe9437495),LL(0x79fe03d4,0x23d4708d), + LL(0xd0c05199,0x8cda9cf2),LL(0xfae78454,0x502fbc22),LL(0xf572a182,0xc0bda9df),LL(0x6158b372,0x5f9b71b8), LL(0x2b82dd07,0xe0f33a59),LL(0x9523032e,0x76302735),LL(0xc4505a32,0x7fe1a721),LL(0xf796409f,0x7b6e3e82), +}, +/* digit=32 base_pwr=2^224 */ +{ + LL(0x35d0b34a,0xe3417bc0),LL(0x8327c0a7,0x440b386b),LL(0xac0362d1,0x8fb7262d),LL(0xe0cdf943,0x2c41114c), LL(0xad95a0b1,0x2ba5cef1),LL(0x67d54362,0xc09b37a8),LL(0x01e486c9,0x26d6cdd2),LL(0x42ff9297,0x20477abf), + LL(0x292a9287,0xa004dcb3),LL(0x77b092c7,0xddc15cf6),LL(0x806c0605,0x083a8464),LL(0x3db997b0,0x4a68df70), LL(0x05bf7dd0,0x9c134e45),LL(0x8ccf7f8c,0xa4e63d39),LL(0x41b5f8af,0xa6e6517f),LL(0xad7bc1cc,0xaa8b9342), + LL(0x1e706ad9,0x126f35b5),LL(0xc3a9ebdf,0xb99cebb4),LL(0xbf608d90,0xa75389af),LL(0xc6c89858,0x76113c4f), LL(0x97e2b5aa,0x80de8eb0),LL(0x63b91304,0x7e1022cc),LL(0x6ccc066c,0x3bdab605),LL(0xb2edf900,0x33cbb144), + LL(0x7af715d2,0xc4176471),LL(0xd0134a96,0xe2f7f594),LL(0xa41ec956,0x2c1873ef),LL(0x77821304,0xe4e7b4f6), LL(0x88d5374a,0xe5c8ff97),LL(0x80823d5b,0x2b915e63),LL(0xb2ee8fe2,0xea6bc755),LL(0xe7112651,0x6657624c), + LL(0xdace5aca,0x157af101),LL(0x11a6a267,0xc4fdbcf2),LL(0xc49c8609,0xdaddf340),LL(0xe9604a65,0x97e49f52), LL(0x937e2ad5,0x9be8e790),LL(0x326e17f1,0x846e2508),LL(0x0bbbc0dc,0x3f38007a),LL(0xb11e16d6,0xcf03603f), + LL(0x7442f1d5,0xd6f800e0),LL(0x66e0e3ab,0x475607d1),LL(0xb7c64047,0x82807f16),LL(0xa749883d,0x8858e1e3), LL(0x8231ee10,0x5859120b),LL(0x638a1ece,0x1b80e7eb),LL(0xc6aa73a4,0xcb72525a),LL(0x844423ac,0xa7cdea3d), + LL(0xf8ae7c38,0x5ed0c007),LL(0x3d740192,0x6db07a5c),LL(0x5fe36db3,0xbe5e9c2a),LL(0x76e95046,0xd5b9d57a), LL(0x8eba20f2,0x54ac32e7),LL(0x71b9a352,0xef11ca8f),LL(0xff98a658,0x305e373e),LL(0x823eb667,0xffe5a100), + LL(0xe51732d2,0x57477b11),LL(0x2538fc0e,0xdfd6eb28),LL(0x3b39eec5,0x5c43b0cc),LL(0xcb36cc57,0x6af12778), LL(0x06c425ae,0x70b0852d),LL(0x5c221b9b,0x6df92f8c),LL(0xce826d9c,0x6c8d4f9e),LL(0xb49359c3,0xf59aba7b), + LL(0xda64309d,0x5c8ed8d5),LL(0x91b30704,0x61a6de56),LL(0x2f9b5808,0xd6b52f6a),LL(0x98c958a7,0x0eee4194), LL(0x771e4caa,0xcddd9aab),LL(0x78bc21be,0x83965dfd),LL(0xb3b504f5,0x02affce3),LL(0x561c8291,0x30847a21), + LL(0x52bfda05,0xd2eb2cf1),LL(0x6197b98c,0xe0e4c4e9),LL(0xf8a1726f,0x1d35076c),LL(0x2db11e3d,0x6c06085b), LL(0x4463ba14,0x15c0c4d7),LL(0x0030238c,0x9d292f83),LL(0x3727536d,0x1311ee8b),LL(0xbeaedc1e,0xfeea86ef), + LL(0x66131e2e,0xb9d18cd3),LL(0x80fe2682,0xf31d974f),LL(0xe4160289,0xb6e49e0f),LL(0x08e92799,0x7c48ec0b), LL(0xd1989aa7,0x818111d8),LL(0xebf926f9,0xb34fa0aa),LL(0xa245474a,0xdb5fe2f5),LL(0x3c7ca756,0xf80a6ebb), + LL(0xafa05dd8,0xa7f96054),LL(0xfcaf119e,0x26dfcf21),LL(0x0564bb59,0xe20ef2e3),LL(0x61cb02b8,0xef4dca50), LL(0x65d30672,0xcda7838a),LL(0xfd657e86,0x8b08d534),LL(0x46d595c8,0x4c5b4395),LL(0x425cb836,0x39b58725), + LL(0x3de9abe3,0x8ea61059),LL(0x9cdc03be,0x40434881),LL(0xcfedce8c,0x9b261245),LL(0xcf5234a1,0x78c318b4), LL(0xfde24c99,0x510bcf16),LL(0xa2c2ff5d,0x2a77cb75),LL(0x27960fb4,0x9c895c2b),LL(0xb0eda42b,0xd30ce975), + LL(0x1a62cc26,0xfda85393),LL(0x50c0e052,0x23c69b96),LL(0xbfc633f3,0xa227df15),LL(0x1bae7d48,0x2ac78848), LL(0x187d073d,0x487878f9),LL(0x967f807d,0x6c2be919),LL(0x336e6d8f,0x765861d8),LL(0xce528a43,0x88b8974c), + LL(0xff57d051,0x09521177),LL(0xfb6a1961,0x2ff38037),LL(0xa3d76ad4,0xfc0aba74),LL(0x25a7ec17,0x7c764803), LL(0x48879bc8,0x7532d75f),LL(0x58ce6bc1,0xea7eacc0),LL(0x8e896c16,0xc82176b4),LL(0x2c750fed,0x9a30e0b2), + LL(0x421d3aa4,0xc37e2c2e),LL(0xe84fa840,0xf926407c),LL(0x1454e41c,0x18abc03d),LL(0x3f7af644,0x26605ecd), LL(0xd6a5eabf,0x242341a6),LL(0x216b668e,0x1edb84f4),LL(0x04010102,0xd836edb8),LL(0x945e1d8c,0x5b337ce7), + LL(0xc055dc14,0xd2075c77),LL(0x81d89cdf,0x2a0ffa25),LL(0x6ffdcbaf,0x8ce815ea),LL(0xfb648867,0xa3428878), LL(0x884655fb,0x277699cf),LL(0x364d3e41,0xfa5b5bd6),LL(0x441e1cb7,0x01f680c6),LL(0xb70a7d67,0x3fd61e66), + LL(0xcc78cf66,0x666ba2dc),LL(0x6fdbff77,0xb3018174),LL(0x168d4668,0x8d4dd0db),LL(0x1dab3a2a,0x259455d0), LL(0xcde3acec,0xf58564c5),LL(0x13adb276,0x77141925),LL(0x8a303f65,0x527d725d),LL(0xe6f38f7b,0x55deb6c9), + LL(0xb1fa70fb,0xfd5bb657),LL(0xd8073a00,0xfa07f50f),LL(0xbca02500,0xf72e3aa7),LL(0x9975740d,0xf68f895d), LL(0x5cae2a6a,0x30112060),LL(0x02874842,0x01bd7218),LL(0x7ce47bd3,0x3d423891),LL(0x789544f6,0xa66663c1), + LL(0x3272d838,0x864d05d7),LL(0xfa6295c5,0xe22924f9),LL(0x6c2fda32,0x8189593f),LL(0xb184b544,0x330d7189), LL(0xbde1f714,0x79efa62c),LL(0xe5cb1a63,0x35771c94),LL(0x641c8332,0x2f4826b8),LL(0xc8cee854,0x00a894fb), + LL(0x36194d40,0xb4b9a39b),LL(0x77612601,0xe857a7c5),LL(0x4ecf2f58,0xf4209dd2),LL(0x5a033487,0x82b9e66d), LL(0xe4e8b9dd,0xc1e36934),LL(0xa42377d7,0xd2372c9d),LL(0x0e3ae43b,0x51dc94c7),LL(0x04474f6f,0x4c57761e), + LL(0x1058a318,0xdcdacd0a),LL(0x78053a9a,0x369cf3f5),LL(0x31c68de2,0xc6c3de50),LL(0x3c4b6d9f,0x4653a576), LL(0xaa4e5c97,0x1688dd5a),LL(0xb7ab3c74,0x5be80aa1),LL(0xbc65c283,0x70cefe7c),LL(0x06867091,0x57f95f13), + LL(0x4415503b,0xa39114e2),LL(0x4cbb17e9,0xc08ff7c6),LL(0xd7dec966,0x1eff674d),LL(0x53376f63,0x6d4690af), LL(0xea74237b,0xff6fe32e),LL(0xcd57508e,0xc436d17e),LL(0xedcc40fe,0x15aa28e1),LL(0x581bbb44,0x0d769c04), + LL(0x34eaacda,0xc240b6de),LL(0x2ba0f1de,0xd9e116e8),LL(0x79438e55,0xcbe45ec7),LL(0x96f752d7,0x91787c9d), LL(0xf129ac2f,0x897f532b),LL(0x5a36e22c,0xd307b7c8),LL(0x749fb8f3,0x91940675),LL(0x157fdb28,0xd14f95d0), + LL(0x6ae55043,0xfe51d029),LL(0x44a87de1,0x8931e98f),LL(0x09e4fee2,0xe57f1cc6),LL(0x4e072d92,0x0d063b67), LL(0xed0e4316,0x70a998b9),LL(0x306aca46,0xe74a736b),LL(0x4fda97c7,0xecf0fbf2),LL(0x3e178d93,0xa40f65cb), + LL(0x16df4285,0x16253604),LL(0xd0c56ae2,0xb0c9babb),LL(0xcfc5cfc3,0x73032b19),LL(0x09752056,0xe497e5c3), LL(0x164bda96,0x12096bb4),LL(0xa0b74da1,0x1ee42419),LL(0x403826ba,0x8fc36243),LL(0xdc09e660,0x0c8f0069), + LL(0xc27253c9,0x8667e981),LL(0x92b36a45,0x05a6aefb),LL(0x9cb7bb46,0xa62c4b36),LL(0x11f7027b,0x8394f375), LL(0x5f109d0f,0x747bc79c),LL(0x5b8cc60a,0xcad88a76),LL(0x58f09e68,0x80c5a66b),LL(0xf6127eac,0xe753d451), + LL(0x5b0ec6f5,0xc44b74a1),LL(0x5289b2b8,0x47989fe4),LL(0x58d6fc73,0x745f8484),LL(0xf61c70ab,0xec362a6f), LL(0xb3a8ad41,0x070c98a7),LL(0x7b63db51,0x73a20fc0),LL(0xf44c35f4,0xed2c2173),LL(0x9acc9dca,0x8a56149d), + LL(0x9ac6e0f4,0x98f17881),LL(0xa413b5ed,0x360fdeaf),LL(0xa300b0fd,0x0625b8f4),LL(0x5b3222d3,0xf1f4d76a), LL(0x587f76b8,0x9d6f5109),LL(0x2317fdb5,0x8b4ee08d),LL(0x8c68b095,0x88089bb7),LL(0x5808d9b9,0x95570e9a), + LL(0x35d33ae7,0xa395c36f),LL(0x50bb5a94,0x200ea123),LL(0x0bafe84b,0x20c789bd),LL(0x0919276a,0x243ef52d), LL(0xe23ae233,0x3934c577),LL(0xa460d1ec,0xb93807af),LL(0xf8fa76a4,0xb72a53b1),LL(0xc3ca4491,0xd8914cb0), + LL(0x3fb42622,0x2e128494),LL(0x500907d5,0x3b2700ac),LL(0x1a95ec63,0xf370fb09),LL(0x31b6dfbd,0xf8f30be2), LL(0x69e55f15,0xf2b2f8d2),LL(0xcc1323e9,0x1fead851),LL(0xd9e5eef6,0xfa366010),LL(0xe316107e,0x64d487b0), + LL(0xd23ddc82,0x4c076b86),LL(0x7e0143f0,0x03fd344c),LL(0x317af2c5,0xa95362ff),LL(0xe18b7a4f,0x0add3db7), LL(0x8260e01b,0x9c673e3f),LL(0x54a1cc91,0xfbeb49e5),LL(0x92f2e433,0x91351bf2),LL(0x851141eb,0xc755e7ec), + LL(0x29607745,0xc9a95139),LL(0xa26f2b28,0x0ca07420),LL(0x4bc6f9dd,0xcb2790e7),LL(0xadcaffc0,0x345bbb58), LL(0xbe0f27a2,0xc65ea38c),LL(0x641fcb56,0x67c24d7c),LL(0xa9e2c757,0x2c25f0a7),LL(0x16f16c49,0x93f5cdb0), + LL(0xc5ee30a1,0x2ca5a9d7),LL(0xb909b729,0xd1593635),LL(0xdadeff48,0x804ce9f3),LL(0xb07c30c3,0xec464751), LL(0x9e49af6a,0x89d65ff3),LL(0x6f3d01bc,0xf2d6238a),LL(0x0bced843,0x1095561e),LL(0xc8a13fd8,0x51789e12), + LL(0x763231df,0xd633f929),LL(0xe7cbddef,0x46df9f7d),LL(0xcb265da8,0x01c889c0),LL(0xaf4336d2,0xfce1ad10), LL(0xfc6a0a7e,0x8d110df6),LL(0x6da425dc,0xdd431b98),LL(0x1834aabe,0xcdc4aeab),LL(0x8439b7fc,0x84deb124), + LL(0x3c2a5998,0x8796f169),LL(0x7947190d,0x9b9247b4),LL(0x11597014,0x55b9d9a5),LL(0x7b1566ee,0x7e9dd70d), LL(0xcbcd5e64,0x94ad78f7),LL(0x9bd4c032,0x0359ac17),LL(0x7cc222ae,0x3b11baaf),LL(0xba78e812,0xa6a6e284), + LL(0x24cea1a0,0x8392053f),LL(0x33621491,0xc97bce4a),LL(0x35399ee9,0x7eb1db34),LL(0xece81ad1,0x473f78ef), LL(0xf63d3d0d,0x41d72fe0),LL(0xafab62fc,0xe620b880),LL(0x93158383,0x92096bc9),LL(0x8f896f6c,0x41a21357), + LL(0xc7dcfcab,0x1b5ee2fa),LL(0x9546e007,0x650acfde),LL(0xb1b02e07,0xc081b749),LL(0xf9eca03d,0xda9e41a0), LL(0x175a54ab,0x013ba727),LL(0xea5d8d10,0xca0cd190),LL(0x95fd96a9,0x85ea52c0),LL(0xbc5c3940,0x2c591b9f), + LL(0x2bad4d5f,0x6fb4d4e4),LL(0xfef0059b,0xfa4c3590),LL(0xf5122294,0x6a10218a),LL(0xa85751d1,0x9a78a81a), LL(0xa98e84e7,0x04f20579),LL(0x4997e5b5,0xfe1242c0),LL(0xca21e1e4,0xe77a273b),LL(0x9411939d,0xfcc8b1ef), + LL(0x92d0487a,0xe20ea302),LL(0x294b91fe,0x1442dbec),LL(0xbb6b0e8f,0x1f7a4afe),LL(0x6889c318,0x1700ef74), LL(0x70f1fc62,0xf5bbffc3),LL(0x69c79cca,0x3b31d4b6),LL(0xa7f6340d,0xe8bc2aab),LL(0xa725e10a,0xb0b08ab4), + LL(0xae340050,0x44f05701),LL(0x1cf0c569,0xba4b3016),LL(0xfbe19a51,0x5aa29f83),LL(0xb71d752e,0x1b9ed428), LL(0xeb4819f5,0x1666e54e),LL(0x9e18b75b,0x616cdfed),LL(0x3ee27b0b,0x112ed5be),LL(0x44c7de4d,0xfbf28319), + LL(0xe0e60d84,0xd685ec85),LL(0x1db7ee78,0x68037e30),LL(0x003c4d6e,0x5b65bdcd),LL(0x93e29a6a,0x33e7363a), LL(0x08d0756c,0x995b3a61),LL(0x2faf134b,0xd727f85c),LL(0x1d337823,0xfac6edf7),LL(0x0439b8b4,0x99b9aa50), + LL(0xe2b4e075,0x722eb104),LL(0x437c4926,0x49987295),LL(0x46a9b82d,0xb1e4c0e4),LL(0x57a006f5,0xd0cb3197), LL(0xd7808c56,0xf3de0f7d),LL(0x51f89772,0xb5c54d8f),LL(0xadbd31aa,0x500a114a),LL(0x295f6cab,0x9afaaaa6), + LL(0x04cf667a,0x94705e21),LL(0x9d3935d7,0xfc2a811b),LL(0x6d09267c,0x560b0280),LL(0xf780e53b,0xf19ed119), LL(0x067b6269,0xf0227c09),LL(0x5caef599,0x967b8533),LL(0x68efeebc,0x155b9243),LL(0xc497bae6,0xcd6d34f5), + LL(0x6cceb370,0x1dd8d5d3),LL(0xa78d7bf9,0x2aeac579),LL(0x70b67a62,0x5d65017d),LL(0x17c53f67,0x70c8e44f), LL(0x86a34d09,0xd1fc0950),LL(0xe7134907,0xe0fca256),LL(0x80fdd315,0xe24fa29c),LL(0xd87499ad,0x2c4acd03), + LL(0x3b5a9ba6,0xbaaf7517),LL(0x12e51a51,0xb9cbe1f6),LL(0x5e154897,0xd88edae3),LL(0x77b66ca0,0xe4309c3c), LL(0xf67f3746,0xf5555805),LL(0xa36401ff,0x85fc37ba),LL(0xd9499a53,0xdf86e2ca),LL(0xecbc955b,0x6270b2a3), + LL(0x974ad33b,0xafae64f5),LL(0xfe7b2df1,0x04d85977),LL(0x4ab03f73,0x2a3db3ff),LL(0x8702740a,0x0b87878a), LL(0x5a061732,0x6d263f01),LL(0xa32a1901,0xc25430ce),LL(0xdb155018,0xf7ebab3d),LL(0x63a9b78e,0x3a86f693), + LL(0xda9f3804,0x349ae368),LL(0xa164349c,0x470f07fe),LL(0x8562baa5,0xd52f4cc9),LL(0x2b290df3,0xc74a9e86), LL(0x43471a24,0xd3a1aa35),LL(0xb8194511,0x239446be),LL(0x81dcd44d,0xbec2dd00),LL(0xc42ac82d,0xca3d7f0f), + LL(0xfdaf4520,0x1f3db085),LL(0x4549daf2,0xbb6d3e80),LL(0x19ad5c42,0xf5969d8a),LL(0xdbfd1511,0x7052b13d), LL(0x682b9060,0x11890d1b),LL(0xac34452c,0xa71d3883),LL(0x783805b4,0xa438055b),LL(0x4725b23e,0x43241277), + LL(0x4901bbed,0xf20cf96e),LL(0xf432a2bb,0x6419c710),LL(0xdfa9cd7d,0x57a0fbb9),LL(0x00daa249,0x589111e4), LL(0x7b60554e,0x19809a33),LL(0xede283a4,0xea5f8887),LL(0x503bfd35,0x2d713802),LL(0x585d2a53,0x151bb0af), + LL(0x43b30ca8,0x40b08f74),LL(0xd9934583,0xe10b5bba),LL(0xb51110ad,0xe8a546d6),LL(0x28e0b6c5,0x1dd50e66), LL(0xcff2b821,0x292e9d54),LL(0x47281760,0x3882555d),LL(0x3724d6e3,0x134838f8),LL(0x22ddcda1,0xf2c679e0), + LL(0x6d2a5768,0x40ee8815),LL(0x1c1e7e2d,0x7f227bd2),LL(0xd04ff443,0x487ba134),LL(0xc614e54b,0x76e2ff3d), LL(0xa3177ec7,0x36b88d6f),LL(0x2328fff5,0xbf731d51),LL(0x49ba158e,0x758caea2),LL(0x02938188,0x5ab8ff4c), + LL(0x35edc56d,0x33e16056),LL(0x7e940d79,0x5a69d349),LL(0x03866dcb,0x6c4fd001),LL(0x4893cdef,0x20a38f57), LL(0xfac3a15b,0xfbf3e790),LL(0x7a4f8e6b,0x6ed7ea2e),LL(0xbc3aca86,0xa663eb4f),LL(0x080d53f7,0x22061ea5), + LL(0xf546783f,0x2480dfe6),LL(0x5a0a641e,0xd38bc6da),LL(0x2ede8965,0xfb093cd1),LL(0xacb455cf,0x89654db4), LL(0x26e1adee,0x413cbf9a),LL(0x373294d4,0x291f3764),LL(0x648083fe,0x00797257),LL(0x208cc341,0x25f504d3), + LL(0xc3a0ee43,0x635a8e5e),LL(0x679898ff,0x70aaebca),LL(0x5dc63d56,0x9ee9f547),LL(0xffb34d00,0xce987966), LL(0x5e26310a,0xf9f86b19),LL(0x382a8ca8,0x9e435484),LL(0xc2352fe4,0x253bcb81),LL(0x4474b571,0xa4eac8b0), + LL(0xc1ad8cf8,0xc1b97512),LL(0x99e0b697,0x193b4e9e),LL(0x01e85df0,0x939d2716),LL(0xcd44eafd,0x4fb265b3), LL(0xe51e1ae2,0x321e7dcd),LL(0xe3d8b096,0x8e3a8ca6),LL(0x52604998,0x8de46cb0),LL(0x39072aa7,0x91099ad8), + LL(0x93aa96b8,0x2617f91c),LL(0x7fca2e13,0x0fc8716b),LL(0x95328723,0xa7106f5e),LL(0x262e6522,0xd1c9c40b), LL(0x42b7c094,0xb9bafe86),LL(0x1543c021,0x1873439d),LL(0x5cbefd5d,0xe1baa5de),LL(0x521e8aff,0xa363fc5e), + LL(0xf862eaac,0xefe6320d),LL(0x22c647dc,0x14419c63),LL(0x4e46d428,0x0e06707c),LL(0x4a178f8f,0xcb6c834f), LL(0xd30f917c,0x0f993a45),LL(0x9879afee,0xd4c4b049),LL(0x70500063,0xb6142a1e),LL(0xa5d9d605,0x7c9b41c3), + LL(0x2f8ba2c7,0xbc00fc2f),LL(0x7c67aa28,0x0966eb2f),LL(0x5a786972,0x13f7b516),LL(0x8a2fbba0,0x3bfb7557), LL(0x5a2b9620,0x131c4f23),LL(0x6faf46be,0xbff3ed27),LL(0x7e172323,0x9b4473d1),LL(0x339f6246,0x421e8878), + LL(0x25a41632,0x0fa8587a),LL(0xa35b6c93,0xc0814124),LL(0x59ebb8db,0x2b18a9f5),LL(0x76edb29c,0x264e3357), LL(0xc87c51e2,0xaf245ccd),LL(0x501e6214,0x16b3015b),LL(0x0a3882ce,0xbb31c560),LL(0xfec11e04,0x6961bb94), + LL(0xeff7a3a0,0x3b825b8d),LL(0xb1df7326,0xbec33738),LL(0x99604a1f,0x68ad747c),LL(0x9a3bd499,0xd154c934), LL(0x1cc7a906,0xac33506f),LL(0x6c560e8f,0x73bb5392),LL(0x263e3944,0x6428fcbe),LL(0x1c387434,0xc11828d5), + LL(0x3e4b12ff,0x3cd04be1),LL(0x2d88667c,0xc3aad9f9),LL(0x248120cf,0xc52ddcf8),LL(0x2a389532,0x985a892e), LL(0x3bb85fa0,0xfbb4b21b),LL(0x8dfc6269,0xf95375e0),LL(0x7ee2acea,0xfb4fb06c),LL(0x309c4d1f,0x6785426e), + LL(0xd8ceb147,0x659b17c8),LL(0xb70a5554,0x9b649eee),LL(0xac6bc634,0x6b7fa0b5),LL(0x1d6e732f,0xd99fe2c7), LL(0x8d3abba2,0x30e6e762),LL(0xa797b799,0x18fee6e7),LL(0xc696464d,0x5c9d360d),LL(0x27bfde12,0xe3baeb48), + LL(0xf23206d5,0x2bf5db47),LL(0x1d260152,0x2f6d3420),LL(0x3f8ff89a,0x17b87653),LL(0x378fa458,0x5157c30c), LL(0x2d4fb936,0x7517c5c5),LL(0xe6518cdc,0xef22f7ac),LL(0xbf847a64,0xdeb483e6),LL(0x92e0fa89,0xf5084558), +}, +/* digit=33 base_pwr=2^231 */ +{ + LL(0xdf7304d4,0xab9659d8),LL(0xff210e8e,0xb71bcf1b),LL(0xd73fbd60,0xa9a2438b),LL(0x5d11b4de,0x4595cd1f), LL(0x4835859d,0x9c0d329a),LL(0x7dbb6e56,0x4a0f0d2d),LL(0xdf928a4e,0xc6038e5e),LL(0x8f5ad154,0xc9429621), + LL(0xf23f2d92,0x91213462),LL(0x60b94078,0x6cab71bd),LL(0x176cde20,0x6bdd0a63),LL(0xee4d54bc,0x54c9b20c), LL(0x9f2ac02f,0x3cd2d8aa),LL(0x206eedb0,0x03f8e617),LL(0x93086434,0xc7f68e16),LL(0x92dd3db9,0x831469c5), + LL(0x8f981354,0x8521df24),LL(0x3588a259,0x587e23ec),LL(0xd7a0992c,0xcbedf281),LL(0x38961407,0x06930a55), LL(0xbe5bbe21,0x09320deb),LL(0x2491817f,0xa7ffa5b5),LL(0x09065160,0xe6c8b4d9),LL(0xfff6d2a9,0xac4f3992), + LL(0x3ae9c1bd,0x7aa7a158),LL(0xe37ce240,0xe0af6d98),LL(0x28ab38b4,0xe54342d9),LL(0x0a1c98ca,0xe8b75007), LL(0xe02358f2,0xefce86af),LL(0xea921228,0x31b8b856),LL(0x0a1c67fc,0x052a1912),LL(0xe3aead59,0xb4069ea4), + LL(0x7fa03cb3,0x3232d6e2),LL(0x0fdd7d88,0xdb938e5b),LL(0x2ccbfc5d,0x04c1d2cd),LL(0xaf3a580f,0xd2f45c12), LL(0x7883e614,0x592620b5),LL(0xbe7c5f26,0x5fd27e68),LL(0x1567e1e3,0x139e45a9),LL(0x44d8aaaf,0x2cc71d2d), + LL(0xe36d0757,0x4a9090cd),LL(0xd9a29382,0xf722d7b1),LL(0x04b48ddf,0xfb7fb04c),LL(0xebe16f43,0x628ad2a7), LL(0x20226040,0xcd3fbfb5),LL(0x5104b6c4,0x6c34ecb1),LL(0xc903c188,0x30c0754e),LL(0x2d23cab0,0xec336b08), + LL(0x1e206ee5,0x473d62a2),LL(0x8c49a633,0xf1e27480),LL(0xe9f6b2c3,0x87ab956c),LL(0x62b606ea,0x61830b48), LL(0xe78e815f,0x67cd6846),LL(0x4c02082a,0xfe40139f),LL(0x952ec365,0x52bbbfcb),LL(0x6b9836ab,0x74c11642), + LL(0x558df019,0x9f51439e),LL(0xac712b27,0x230da4ba),LL(0x55185a24,0x518919e3),LL(0x84b78f50,0x4dcefcdd), LL(0xa47d4c5a,0xa7d90fb2),LL(0xb30e009e,0x55ac9abf),LL(0x74eed273,0xfd2fc359),LL(0xdbea8faf,0xb72d824c), + LL(0x4513e2ca,0xce721a74),LL(0x38240b2c,0x0b418612),LL(0xd5baa450,0x05199968),LL(0x2b0e8c25,0xeb1757ed), LL(0x3dfac6d5,0x6ebc3e28),LL(0x48a237f5,0xb2431e2e),LL(0x52f61499,0x2acb5e23),LL(0xe06c936b,0x5558a2a7), + LL(0xcbb13d1b,0xd213f923),LL(0x5bfb9bfe,0x98799f42),LL(0x701144a9,0x1ae8ddc9),LL(0x4c5595ee,0x0b8b3bb6), LL(0x3ecebb21,0x0ea9ef2e),LL(0x3671f9a7,0x17cb6c4b),LL(0x726f1d1f,0x47ef464f),LL(0x6943a276,0x171b9484), + LL(0x7ef0329c,0x51a4ae2d),LL(0x91c4402a,0x08509222),LL(0xafd45bbc,0x64a61d35),LL(0x3035a851,0x38f096fe), LL(0xa1dec027,0xc7468b74),LL(0x4fc7dcba,0xe8cf10e7),LL(0xf4a06353,0xea35ff40),LL(0x8b77dd66,0x0b4c0dfa), + LL(0xde7e5c19,0x779b8552),LL(0xc1c0256c,0xfab28609),LL(0xabd4743d,0x64f58eee),LL(0x7b6cc93b,0x4e8ef838), LL(0x4cb1bf3d,0xee650d26),LL(0x73dedf61,0x4c1f9d09),LL(0xbfb70ced,0xaef7c9d7),LL(0x1641de1e,0x1ec0507e), + LL(0xcde45079,0xcd7e5cc7),LL(0x516ac9e4,0xde173c9a),LL(0xc170315c,0x517a8494),LL(0x91d8e8fb,0x438fd905), LL(0xc7d9630b,0x5145c506),LL(0xf47d4d75,0x6457a87b),LL(0x0d9a80e8,0xd31646bf),LL(0xcef3aabe,0x453add2b), + LL(0xa607419d,0xc9941109),LL(0xbb6bca80,0xfaa71e62),LL(0x07c431f3,0x34158c13),LL(0x992bc47a,0x594abebc), LL(0xeb78399f,0x6dfea691),LL(0x3f42cba4,0x48aafb35),LL(0x077c04f0,0xedcd65af),LL(0xe884491a,0x1a29a366), + LL(0x1c21f2bf,0x023a40e5),LL(0xa5057aee,0xf99a513c),LL(0xbcab072e,0xa3fe7e25),LL(0x40e32bcf,0x8568d2e1), LL(0xd3f69d9f,0x904594eb),LL(0x07affab1,0x181a9733),LL(0xb6e330f4,0xe4d68d76),LL(0xc75a7fc1,0x87a6dafb), + LL(0xef7d9289,0x549db2b5),LL(0x197f015a,0x2480d4a8),LL(0xc40493b6,0x61d5590b),LL(0x6f780331,0x3a55b52e), LL(0x309eadb0,0x40eb8115),LL(0x92e5c625,0xdea7de5a),LL(0xcc6a3d5a,0x64d631f0),LL(0x93e8dd61,0x9d5e9d7c), + LL(0x206d3ffc,0xf297bef5),LL(0x7d808bd4,0x23d5e033),LL(0xd24cf5ba,0x4a4f6912),LL(0x09cdaa8a,0xe4d8163b), LL(0xd3082e8e,0x0e0de9ef),LL(0x0192f360,0x4fe1246c),LL(0x4b8eee0a,0x1f900150),LL(0xf1da391b,0x5219da81), + LL(0xf7ea25aa,0x7bf6a5c1),LL(0xfbb07d5f,0xd165e6bf),LL(0x89e78671,0xe3539361),LL(0x2bac4219,0xa3fcac89), LL(0xf0baa8ab,0xdfab6fd4),LL(0xe2c1c2e5,0x5a4adac1),LL(0x40d85849,0x6cd75e31),LL(0x19b39181,0xce263fea), + LL(0x07032c72,0xcb6803d3),LL(0x790968c8,0x7f40d5ce),LL(0xdce978f0,0xa6de86bd),LL(0x368f751c,0x25547c4f), LL(0x65fb2a9e,0xb1e685fd),LL(0x1eb9179c,0xce69336f),LL(0x12504442,0xb15d1c27),LL(0xb911a06b,0xb7df465c), + LL(0x315980cd,0xb8d804a3),LL(0xfa3bebf7,0x693bc492),LL(0x2253c504,0x3578aeee),LL(0xcd2474a2,0x158de498), LL(0xcfda8368,0x1331f5c7),LL(0x78d7177e,0xd2d7bbb3),LL(0xf3c1e46e,0xdf61133a),LL(0xd30e7be8,0x5836ce7d), + LL(0x94f834cb,0x83084f19),LL(0x429ed782,0xd35653d4),LL(0x59e58243,0xa542f16f),LL(0x0470a22d,0xc2b52f65), LL(0x18f23d96,0xe3b6221b),LL(0x3f5252b4,0xcb05abac),LL(0x87d61402,0xca00938b),LL(0x411933e4,0x2f186cdd), + LL(0x9a29a5c5,0xe042ece5),LL(0x3b6c8402,0xb19b3c07),LL(0x19d92684,0xc97667c7),LL(0xebc66372,0xb5624622), LL(0x3c04fa02,0x0cb96e65),LL(0x8eaa39aa,0x83a7176c),LL(0xeaa1633f,0x2033561d),LL(0x4533df73,0x45a9d086), + LL(0x3dc090bc,0xe0542c1d),LL(0xaa59c167,0x82c996ef),LL(0x0ee7fc4d,0xe3f735e8),LL(0x7c35db79,0x7b179393), LL(0xf8c5dbfd,0xb6419e25),LL(0x1f327b04,0x4d9d7a1e),LL(0x298dfca8,0x979f6f9b),LL(0x8de9366a,0xc7c5dff1), + LL(0x04c82bdd,0x1b7a588d),LL(0xf8319dfd,0x68005534),LL(0xd8eb9580,0xde8a55b5),LL(0x8d5bca81,0x5ea886da), LL(0x252a0b4d,0xe8530a01),LL(0x35eaa0a1,0x1bffb4fe),LL(0xd8e99563,0x2ad828b1),LL(0x95f9cd87,0x7de96ef5), + LL(0xd77d970c,0x4abb2d0c),LL(0xd33ef9cb,0x03cfb933),LL(0x8b211fe9,0xb0547c01),LL(0xa56ed1c6,0x2fe64809), LL(0xc2ac98cc,0xcb7d5624),LL(0x1a393e33,0x2a1372c0),LL(0x29660521,0xc8d1ec1c),LL(0xb37ac3e9,0xf3d31b04), + LL(0x5ece6e7c,0xa29ae9df),LL(0x0facfb55,0x0603ac8f),LL(0xdda233a5,0xcfe85b7a),LL(0xbd75f0b8,0xe618919f), LL(0x99bf1603,0xf555a3d2),LL(0xf184255a,0x1f43afc9),LL(0x319a3e02,0xdcdaf341),LL(0x03903a39,0xd3b117ef), + LL(0x65d1d131,0xe095da13),LL(0xc37ad03e,0x86f16367),LL(0x462cd8dd,0x5f37389e),LL(0xd67a60e6,0xc103fa04), LL(0xf4b478f0,0x57c34344),LL(0xe117c98d,0xce91edd8),LL(0x231fc12e,0x001777b0),LL(0xb207bccb,0x11ae47f2), + LL(0x20f8a242,0xd983cf8d),LL(0xf22e1ad8,0x7aff5b1d),LL(0x7fc4feb3,0x68fd11d0),LL(0xb0f1c3e1,0x5d53ae90), LL(0xec041803,0x50fb7905),LL(0x14404888,0x85e3c977),LL(0xac628d8f,0x0e67faed),LL(0x6668532c,0x2e865150), + LL(0x6a67a6b0,0x15acaaa4),LL(0xb25cec41,0xf4cdee25),LL(0xe4c6701e,0x49ee565a),LL(0xfc7d63d8,0x2a04ca66), LL(0xef0543fb,0xeb105018),LL(0xd1b0d81d,0xf709a4f5),LL(0x2915d333,0x5b906ee6),LL(0x96f1f0ab,0xf4a87412), + LL(0x4d82f4c2,0xb6b82fa7),LL(0x6804efb3,0x90725a60),LL(0xadc3425e,0xbc82ec46),LL(0x2787843e,0xb7b80581), LL(0xdd1fc74c,0xdf46d91c),LL(0xe783a6c4,0xdc1c62cb),LL(0x1a04cbba,0x59d1b9f3),LL(0x95e40764,0xd87f6f72), + LL(0x317f4a76,0x02b4cfc1),LL(0x91036bce,0x8d2703eb),LL(0xa5e72a56,0x98206cc6),LL(0xcf53fb0f,0x57be9ed1), LL(0xef0b17ac,0x09374571),LL(0xd9181b38,0x74b2655e),LL(0x89935d0e,0xc8f80ea8),LL(0x91529936,0xc0d9e942), + LL(0x1e84e0e5,0x19686041),LL(0xaea34c93,0xa5db84d3),LL(0x7073a732,0xf9d5bb19),LL(0x6bcfd7c0,0xb8d2fe56), LL(0xf3eb82fa,0x45775f36),LL(0xfdff8b58,0x8cb20ccc),LL(0x8374c110,0x1659b65f),LL(0x330c789a,0xb8b4a422), + LL(0x6fe8208b,0x75e3c3ea),LL(0x286e78fe,0xbd74b9e4),LL(0xd7d93a1a,0x0be2e81b),LL(0xdd0a5aae,0x7ed06e27), LL(0x6be8b800,0x721f5a58),LL(0xd846db28,0x428299d1),LL(0x5be88ed3,0x95cb8e6b),LL(0x1c034e11,0xc3186b23), + LL(0x8977d99b,0xa6312c9e),LL(0x83f531e7,0xbe944331),LL(0x18d3b1d4,0x8232c0c2),LL(0xe1247b73,0x617aae8b), LL(0x282aec3b,0x40153fc4),LL(0xf7b8f823,0xc6063d2f),LL(0x3304f94c,0x68f10e58),LL(0xee676346,0x31efae74), + LL(0x40a9b97c,0xbadb6c6d),LL(0x4f666256,0x14702c63),LL(0x5184b2e3,0xdeb954f1),LL(0x94b6ca40,0x5184a526), LL(0x003c32ea,0xfff05337),LL(0x205974c7,0x5aa374dd),LL(0x4b0dd71a,0x9a763854),LL(0xdeb947ec,0x459cd27f), + LL(0x459c2b92,0xa6e28161),LL(0x75ee8ef5,0x2f020fa8),LL(0x30b06310,0xb132ec2d),LL(0xbc6a4530,0xc3e15899), LL(0xaa3f451a,0xdc5f53fe),LL(0xc2d9acac,0x3a3c7f23),LL(0x6b27e58b,0x2ec2f892),LL(0xd742799f,0x68466ee7), + LL(0x1fa26613,0x98324dd4),LL(0xbdc29d63,0xa2dc6dab),LL(0xd712d657,0xf9675faa),LL(0x21fd8d15,0x813994be), LL(0xfd4f7553,0x5ccbb722),LL(0xf3a36b20,0x5135ff8b),LL(0x69559df5,0x44be28af),LL(0x9d41bf30,0x40b65bed), + LL(0x3734e520,0xd98bf2a4),LL(0x209bdcba,0x5e3abbe3),LL(0xbc945b35,0x77c76553),LL(0xc6ef14aa,0x5331c093), LL(0x76b60c80,0x518ffe29),LL(0x7ace16f8,0x2285593b),LL(0xbe2b9784,0xab1f64cc),LL(0xab2421b6,0xe8f2c0d9), + LL(0xc1df065c,0x617d7174),LL(0x5f6578fa,0xafeeb5ab),LL(0x263b54a8,0x16ff1329),LL(0xc990dce3,0x45c55808), LL(0xecc8c177,0x42eab6c0),LL(0x5982ecaa,0x799ea9b5),LL(0xb607ef8e,0xf65da244),LL(0x32a3fc2c,0x8ab226ce), + LL(0x7ea973dc,0x745741e5),LL(0x20888f2e,0x5c00ca70),LL(0x45fd9cf1,0x7cdce3cf),LL(0x5507f872,0x8a741ef1), LL(0x196b4cec,0x47c51c2f),LL(0xc97ea618,0x70d08e43),LL(0x15b18a2b,0x930da15c),LL(0x2f610514,0x33b6c678), + LL(0x07ac9794,0xc662e4f8),LL(0xba06cb79,0x1eccf050),LL(0xe7d954e5,0x1ff08623),LL(0x24cf71c3,0x6ef2c5fb), LL(0x67978453,0xb2c063d2),LL(0x1d654af8,0xa0cf3796),LL(0x7ebdaa37,0x7cb242ea),LL(0xb86747e0,0x206e0b10), + LL(0xd5ecfefc,0x481dae5f),LL(0xc2bff8fc,0x07084fd8),LL(0xea324596,0x8040a01a),LL(0xd4de4036,0x4c646980), LL(0xd65abfc3,0x9eb8ab4e),LL(0x13541ec7,0xe01cb91f),LL(0xfd695012,0x8f029adb),LL(0x3c7569ec,0x9ae28483), + LL(0xa66d80a1,0xa5614c9e),LL(0x75f5f911,0x680a3e44),LL(0xceba4fc1,0x0c07b14d),LL(0xa13071c1,0x891c285b), LL(0x799ece3c,0xcac67ceb),LL(0x41e07e27,0x29b910a9),LL(0xf2e43123,0x66bdb409),LL(0x7ac9ecbe,0x06f8b137), + LL(0x38547090,0x5981fafd),LL(0x85e3415d,0x19ab8b9f),LL(0xc7e31b27,0xfc28c194),LL(0x6fbcbb42,0x843be0aa), LL(0xa6db836c,0xf3b1ed43),LL(0x01a45c05,0x2a1330e4),LL(0x95c1a377,0x4f19f3c5),LL(0x44b5ee33,0xa85f39d0), + LL(0x4ae52834,0x3da18e6d),LL(0x7423dcb0,0x5a403b39),LL(0xf2374aef,0xbb555e0a),LL(0x1e8ca111,0x2ad599c4), LL(0x014b3bf8,0x1b3a2fb9),LL(0xf66d5007,0x73092684),LL(0xc4340102,0x079f1426),LL(0x8fddf4de,0x1827cf81), + LL(0xf10ff927,0xc83605f6),LL(0x23739fc6,0xd3871451),LL(0xcac1c2cc,0x6d163450),LL(0xa2ec1ac5,0x6b521296), LL(0x6e3cb4a5,0x0606c4f9),LL(0x778abff7,0xe47d3f41),LL(0xbe8e3a45,0x425a8d5e),LL(0xa6102160,0x53ea9e97), + LL(0x39cbb688,0x477a106e),LL(0xf3386d32,0x532401d2),LL(0xb1b9b421,0x8e564f64),LL(0x81dad33f,0xca9b8388), LL(0x2093913e,0xb1422b4e),LL(0x69bc8112,0x533d2f92),LL(0xebe7b2c7,0x3fa017be),LL(0xcaf197c6,0xb2767c4a), + LL(0xaedbae9f,0xc925ff87),LL(0x36880a54,0x7daf0eb9),LL(0x9c4d0e71,0x9284ddf5),LL(0x316f8cf5,0x1581cf93), LL(0x3ac1f452,0x3eeca887),LL(0xfb6aeffe,0xb417fce9),LL(0xeefb8dc3,0xa5918046),LL(0x02209400,0x73d318ac), + LL(0x728693e5,0xe800400f),LL(0x339927ed,0xe87d814b),LL(0x57ea9910,0x93e94d3b),LL(0x2245fb69,0xff8a35b6), LL(0x7f200d34,0x043853d7),LL(0x0f653ce1,0x470f1e68),LL(0x59a06379,0x81ac05bd),LL(0x03930c29,0xa14052c2), + LL(0x26bc2797,0x6b72fab5),LL(0x99f16771,0x13670d16),LL(0x1e3e48d1,0x00170052),LL(0xb7adf678,0x978fe401), LL(0xd41c5dd4,0x55ecfb92),LL(0xc7b27da5,0x5ff8e247),LL(0x013fb606,0xe7518272),LL(0x2f547a3c,0x5768d7e5), + LL(0x60017a5f,0xbb24eaa3),LL(0x9c64ce9b,0x6b18e6e4),LL(0x103dde07,0xc225c655),LL(0x7592f7ea,0xfc3672ae), LL(0xd06283a1,0x9606ad77),LL(0xe4d59d99,0x542fc650),LL(0x2a40e7c2,0xabb57c49),LL(0xa8db9f55,0xac948f13), + LL(0xb04465c3,0x6d4c9682),LL(0x6468bd15,0xe3d062fa),LL(0x5f318d7e,0xa51729ac),LL(0x9eb6fc95,0x1fc87df6), LL(0x0591f652,0x63d146a8),LL(0x589621aa,0xa861b8f7),LL(0xce31348c,0x59f5f15a),LL(0x440da6da,0x8f663391), + LL(0xb591ffa3,0xcfa778ac),LL(0x4cdfebce,0x027ca9c5),LL(0x444ea6b3,0xbe8e05a5),LL(0xa78d8254,0x8aab4e69), LL(0xb474d6b8,0x2437f04f),LL(0x045b3855,0x6597ffd4),LL(0xca47ecaa,0xbb0aea4e),LL(0x85c7ebfc,0x568aae83), + LL(0xc73b2383,0x0e966e64),LL(0xd17d8762,0x49eb3447),LL(0x8da05dab,0xde107821),LL(0x016b7236,0x443d8baa), LL(0xea7610d6,0x163b63a5),LL(0xce1ca979,0xe47e4185),LL(0x80baa132,0xae648b65),LL(0x0e0d5b64,0xebf53de2), + LL(0xd3c8c1ca,0x8d3bfcb4),LL(0x5d04b309,0x0d914ef3),LL(0x3de7d395,0x55ef6415),LL(0x26b850e8,0xbde1666f), LL(0xd449ab19,0xdbe1ca6e),LL(0xe89a2672,0x8902b322),LL(0xdacb7a53,0xb1674b7e),LL(0xf52523ff,0x8e9faf6e), + LL(0x9a85788b,0x6ba535da),LL(0xbd0626d4,0xd21f03ae),LL(0xe873dc64,0x099f8c47),LL(0x018ec97e,0xcda8564d), LL(0xde92c68c,0x3e8d7a5c),LL(0x73323cc4,0x78e035a1),LL(0xf880ff7c,0x3ef26275),LL(0x273eedaa,0xa4ee3dff), + LL(0xaf4e18f8,0x58823507),LL(0x0672f328,0x967ec9b5),LL(0x559d3186,0x9ded19d9),LL(0x6cdce39c,0x5e2ab3de), LL(0x11c226df,0xabad6e4d),LL(0x87723014,0xf9783f43),LL(0x1a885719,0x9a49a0cf),LL(0x90da9dbf,0xfc0c1a5a), + LL(0x571d92ac,0x8bbaec49),LL(0x4692517f,0x569e85fe),LL(0xa14ea4af,0x8333b014),LL(0x12e5c5ad,0x32f2a62f), LL(0x06d89b85,0x98c2ce3a),LL(0x2ff77a08,0xb90741aa),LL(0x01f795a2,0x2530defc),LL(0x84b3c199,0xd6e5ba0b), + LL(0x12e4c936,0x7d8e8451),LL(0xbd0be17b,0xae419f7d),LL(0x22262bc9,0xa583fc8c),LL(0x91bfe2bd,0x6b842ac7), LL(0x440d6827,0x33cef4e9),LL(0xef81fb14,0x5f69f4de),LL(0x234fbb92,0xf16cf6f6),LL(0xd9e7e158,0x76ae3fc3), + LL(0xe9740b33,0x4e89f6c2),LL(0x4962d6a1,0x677bc85d),LL(0x68d10d15,0x6c6d8a7f),LL(0x0257b1cd,0x5f9a7224), LL(0x4ad85961,0x7096b916),LL(0xe657ab4a,0x5f8c47f7),LL(0xf7461d7e,0xde57d7d0),LL(0x80ce5ee2,0x7eb6094d), + LL(0x34190547,0x0b1e1dfd),LL(0xf05dd150,0x8a394f43),LL(0x97df44e6,0x0a9eb24d),LL(0x87675719,0x78ca06bf), LL(0x6ffeec22,0x6f0b3462),LL(0x36cdd8fb,0x9d91bcea),LL(0xa105be47,0xac83363c),LL(0x069710e3,0x81ba76c1), + LL(0x28c682c6,0x3d1b24cb),LL(0x8612575b,0x27f25228),LL(0xe8e66e98,0xb587c779),LL(0x405eb1fe,0x7b0c03e9), LL(0x15b548e7,0xfdf0d030),LL(0x38b36af7,0xa8be76e0),LL(0x4f310c40,0x4cdab04a),LL(0xf47ecaec,0x6287223e), + LL(0x8b399320,0x678e6055),LL(0xc01e4646,0x61fe3fa6),LL(0x03261a5e,0xc482866b),LL(0x5c2f244a,0xdfcf45b8), LL(0x2f684b43,0x8fab9a51),LL(0xc7220a66,0xf796c654),LL(0xf5afa58f,0x1d90707e),LL(0x4fdbe0de,0x2c421d97), + LL(0xaf2ebc2f,0xc4f4cda3),LL(0xcb4efe24,0xa0af843d),LL(0x9ccd10b1,0x53b857c1),LL(0x914d3e04,0xddc9d1eb), LL(0x62771deb,0x7bdec8bb),LL(0x91c5aa81,0x829277aa),LL(0x832391ae,0x7af18dd6),LL(0xc71a84ca,0x1740f316), +}, +/* digit=34 base_pwr=2^238 */ +{ + LL(0xeeaf8c49,0x8928e99a),LL(0x6e24d728,0xee7aa73d),LL(0xe72b156c,0x4c5007c2),LL(0xed408a1d,0x5fcf57c5), LL(0xb6057604,0x9f719e39),LL(0xc2868bbf,0x7d343c01),LL(0x7e103e2d,0x2cca254b),LL(0xf131bea2,0xe6eb38a9), + LL(0x8be762b4,0xb33e624f),LL(0x058e3413,0x2a9ee4d1),LL(0x67d805fa,0x968e6369),LL(0x7db8bfd7,0x9848949b), LL(0xd23a8417,0x5308d7e5),LL(0xf3e29da5,0x892f3b1d),LL(0x3dee471f,0xc95c139e),LL(0xd757e089,0x8631594d), + LL(0xde918dcc,0xe0c82a3c),LL(0x26fdcf4b,0x2e7b5994),LL(0x32cb1b2d,0x82c50249),LL(0x7657ae07,0xea613a9d), LL(0xf1fdc9f7,0xc2eb5f6c),LL(0x879fe682,0xb6eae8b8),LL(0x591cbc7f,0x253dfee0),LL(0x3e1290e6,0x000da713), + LL(0x1f095615,0x1083e2ea),LL(0x14e68c33,0x0a28ad77),LL(0x3d8818be,0x6bfc0252),LL(0xf35850cd,0xb585113a), LL(0x30df8aa1,0x7d935f0b),LL(0x4ab7e3ac,0xaddda07c),LL(0x552f00cb,0x92c34299),LL(0x2909df6c,0xc33ed1de), + LL(0x80e87766,0x22c2195d),LL(0x9ddf4ac0,0x9e99e6d8),LL(0x65e74934,0x09642e4e),LL(0xff1ff241,0x2610ffa2), LL(0x751c8159,0x4d1d47d4),LL(0xaf3a9363,0x697b4985),LL(0x87477c33,0x0318ca46),LL(0x9441eff3,0xa90cb565), + LL(0x36f024cb,0x58bb3848),LL(0x36016168,0x85be1f77),LL(0xdc7e07f1,0x6c59587c),LL(0xaf1d8f02,0x191be071), LL(0xcca5e55c,0xbf169fa5),LL(0xf7d04eac,0x3864ba3c),LL(0x8d7d05db,0x915e367f),LL(0xa6549e5d,0xb48a876d), + LL(0x580e40a2,0xef89c656),LL(0x728068bc,0xf194ed8c),LL(0xa47990c9,0x74528045),LL(0x5e1a4649,0xf53fc7d7), LL(0x78593e7d,0xbec5ae9b),LL(0x41db65d7,0x2cac4ee3),LL(0x04a3d39b,0xa8c1eb24),LL(0x03f8f3ef,0x53b7d634), + LL(0x3e07113c,0x2dc40d48),LL(0x7d8b63ae,0x6e4a5d39),LL(0x79684c2b,0x5582a94b),LL(0x622da26c,0x932b33d4), LL(0x0dbbf08d,0xf534f651),LL(0x64c23a52,0x211d07c9),LL(0xee5bdc9b,0x0eeece0f),LL(0xf7015558,0xdf178168), + LL(0x0a712229,0xd4294635),LL(0x09273f8c,0x93cbe448),LL(0x8f13bc83,0x00b095ef),LL(0x8798978c,0xbb741972), LL(0x56dbe6e7,0x9d7309a2),LL(0x5a5d39ec,0xe578ec56),LL(0x851f9a31,0x3961151b),LL(0xe5709eb4,0x2da7715d), + LL(0x53dfabf0,0x867f3017),LL(0xb8e39259,0x728d2078),LL(0x815d9958,0x5c75a0cd),LL(0x16603be1,0xf84867a6), LL(0x70e35b1c,0xc865b13d),LL(0x19b03e2c,0x02414468),LL(0xac1f3121,0xe46041da),LL(0x6f028a7c,0x7c9017ad), + LL(0x0a482873,0xabc96de9),LL(0xb77e54d4,0x4265d6b1),LL(0xa57d88e7,0x68c38e79),LL(0x9ce82de3,0xd461d766), LL(0x64a7e489,0x817a9ec5),LL(0xa0def5f2,0xcc5675cd),LL(0x985d494e,0x9a00e785),LL(0x1b03514a,0xc626833f), + LL(0x83cdd60e,0xabe7905a),LL(0xa1170184,0x50602fb5),LL(0xb023642a,0x689886cd),LL(0xa6e1fb00,0xd568d090), LL(0x0259217f,0x5b1922c7),LL(0xc43141e4,0x93831cd9),LL(0x0c95f86e,0xdfca3587),LL(0x568ae828,0xdec2057a), + LL(0xf98a759a,0xc44ea599),LL(0xf7c23c1d,0x55a0a7a2),LL(0x94c4f687,0xd5ffb6e6),LL(0x12848478,0x3563cce2), LL(0xe7b1fbe1,0x812b3517),LL(0x4f7338e0,0x8a7dc979),LL(0x52d048db,0x211ecee9),LL(0xc86ea3b8,0x2eea4056), + LL(0xba772b34,0xd8cb68a7),LL(0x5f4e2541,0xe16ed341),LL(0x0fec14db,0x9b32f6a6),LL(0x391698be,0xeee376f7), LL(0x83674c02,0xe9a7aa17),LL(0x5843022a,0x65832f97),LL(0x5ba4990f,0x29f3a8da),LL(0xfb8e3216,0x79a59c3a), + LL(0xbd19bb16,0x9cdc4d2e),LL(0xb3262d86,0xc6c7cfd0),LL(0x969c0b47,0xd4ce14d0),LL(0x13e56128,0x1fa352b7), LL(0x973db6d3,0x383d55b8),LL(0xe8e5b7bf,0x71836850),LL(0xe6bb571f,0xc7714596),LL(0x2d5b2dd2,0x259df31f), + LL(0x913cc16d,0x568f8925),LL(0xe1a26f5a,0x18bc5b6d),LL(0xf5f499ae,0xdfa413be),LL(0xc3f0ae84,0xf8835dec), LL(0x65a40ab0,0xb6e60bd8),LL(0x194b377e,0x65596439),LL(0x92084a69,0xbcd85625),LL(0x4f23ede0,0x5ce433b9), + LL(0x6ad65143,0xe8e8f04f),LL(0xd6e14af6,0x11511827),LL(0x8295c0c7,0x3d390a10),LL(0x621eba16,0x71e29ee4), LL(0x63717b46,0xa588fc09),LL(0xe06ad4a2,0x02be02fe),LL(0x04c22b22,0x931558c6),LL(0x12f3c849,0xbb4d4bd6), + LL(0x20efd662,0x54a4f496),LL(0xc5952d14,0x92ba6d20),LL(0xcc9784c2,0x2db8ea1e),LL(0x4b353644,0x81cc10ca), LL(0x4b4d7f6c,0x40b570ad),LL(0x84a1dcd2,0x5c9f1d96),LL(0x3147e797,0x01379f81),LL(0x2bd499f5,0xe5c6097b), + LL(0x328e5e20,0x40dcafa6),LL(0x54815550,0xf7b5244a),LL(0x47bfc978,0xb9a4f118),LL(0xd25825b1,0x0ea0e79f), LL(0x646c7ecf,0xa50f96eb),LL(0x446dea9d,0xeb811493),LL(0xdfabcf69,0x2af04677),LL(0xc713f6e8,0xbe3a068f), + LL(0x42e06189,0x860d523d),LL(0x4e3aff13,0xbf077941),LL(0xc1b20650,0x0b616dca),LL(0x2131300d,0xe66dd6d1), LL(0xff99abde,0xd4a0fd67),LL(0xc7aac50d,0xc9903550),LL(0x7c46b2d7,0x022ecf8b),LL(0x3abf92af,0x3333b1e8), + LL(0x6c491c14,0x11cc113c),LL(0x80dd3f88,0x05976688),LL(0x29d932ed,0xf5b4d9e7),LL(0xa2c38b6d,0xe982aad8), LL(0x8be0dcf0,0x6f925347),LL(0x65ca53f2,0x700080ae),LL(0x443ca77f,0xd8131156),LL(0xec51f984,0xe92d6942), + LL(0x85dfe9ae,0xd2a08af8),LL(0x4d2a86ca,0xd825d9a5),LL(0x39dff020,0x2c53988d),LL(0x430cdc40,0xf38b135a), LL(0x62a7150b,0x0c918ae0),LL(0x0c340e9b,0xf31fd8de),LL(0x4dbbf02e,0xafa0e7ae),LL(0x5eba6239,0x5847fb2a), + LL(0xdccbac8b,0x6b1647dc),LL(0x06f485c8,0xb642aa78),LL(0x7038ecdf,0x873f3765),LL(0xfa49d3fe,0x2ce5e865), LL(0xc98c4400,0xea223788),LL(0xf1fa5279,0x8104a8cd),LL(0x06becfd7,0xbcf7cc7a),LL(0xc8f974ae,0x49424316), + LL(0x84d6365d,0xc0da65e7),LL(0x8f759fb8,0xbcb7443f),LL(0x7ae81930,0x35c712b1),LL(0x4c6e08ab,0x80428dff), LL(0xa4faf843,0xf19dafef),LL(0xffa9855f,0xced8538d),LL(0xbe3ac7ce,0x20ac409c),LL(0x882da71e,0x358c1fb6), + LL(0xfd349961,0xafa9c0e5),LL(0x8421c2fc,0x2b2cfa51),LL(0xf3a28d38,0x2a80db17),LL(0x5d138e7e,0xa8aba539), LL(0x6e96eb8d,0x52012d1d),LL(0xcbaf9622,0x65d8dea0),LL(0xb264f56c,0x57735447),LL(0x1b6c8da2,0xbeebef3f), + LL(0xce785254,0xfc346d98),LL(0xbb64a161,0xd50e8d72),LL(0x49794add,0xc03567c7),LL(0x752c7ef6,0x15a76065), LL(0x961f23d6,0x59f3a222),LL(0x73ecc0b0,0x378e4438),LL(0x5a82fde4,0xc74be434),LL(0xd8b9cf34,0xae509af2), + LL(0x577f44a1,0x4a61ee46),LL(0xb611deeb,0xe09b748c),LL(0xf5f7b884,0xc0481b2c),LL(0x61acfa6b,0x35626678), LL(0xbf8d21e6,0x37f4c518),LL(0xb205a76d,0x22d96531),LL(0x954073c0,0x37fb85e1),LL(0x65b3a567,0xbceafe4f), + LL(0xbe42a582,0xefecdef7),LL(0x65046be6,0xd3fc6080),LL(0x09e8dba9,0xc9af13c8),LL(0x641491ff,0x1e6c9847), LL(0xd30c31f7,0x3b574925),LL(0xac2a2122,0xb7eb72ba),LL(0xef0859e7,0x776a0dac),LL(0x21900942,0x06fec314), + LL(0xf8c22049,0x2464bc10),LL(0x875ebf69,0x9bfbcce7),LL(0x4336326b,0xd7a88e2a),LL(0x5bc2acfa,0xda05261c), LL(0xeba7efc8,0xc29f5bdc),LL(0x25dbbf2e,0x471237ca),LL(0x2975f127,0xa72773f2),LL(0x04d0b326,0xdc744e8e), + LL(0xa56edb73,0x38a7ed16),LL(0x2c007e70,0x64357e37),LL(0x5080b400,0xa167d15b),LL(0x23de4be1,0x07b41164), LL(0x74c89883,0xb2d91e32),LL(0x2882e7ed,0x3c162821),LL(0x7503e482,0xad6b36ba),LL(0x0ea34331,0x48434e8e), + LL(0x2c7ae0b9,0x79f4f24f),LL(0x1939b44a,0xc46fbf81),LL(0x56595eb1,0x76fefae8),LL(0xcd5f29c7,0x417b66ab), LL(0xc5ceec20,0x5f2332b2),LL(0xe1a1cae2,0xd69661ff),LL(0x9b0286e6,0x5ede7e52),LL(0xe276b993,0x9d062529), + LL(0x7e50122b,0x324794b0),LL(0x4af07ca5,0xdd744f8b),LL(0xd63fc97b,0x30a12f08),LL(0x76626d9d,0x39650f1a), LL(0x1fa38477,0x101b47f7),LL(0xd4dc124f,0x3d815f19),LL(0xb26eb58a,0x1569ae95),LL(0x95fb1887,0xc3cde188), + LL(0xf9539a48,0x54e9f37b),LL(0x7408c1a5,0xb0100e06),LL(0xea580cbb,0x821d9811),LL(0x86e50c56,0x8af52d35), LL(0xdbbf698b,0xdfbd9d47),LL(0x03dc1c73,0x2961a1ea),LL(0xe76a5df8,0x203d38f8),LL(0x6def707a,0x08a53a68), + LL(0x1bee45d4,0x26eefb48),LL(0x3c688036,0xb3cee346),LL(0xc42f2469,0x463c5315),LL(0x81378162,0x19d84d2e), LL(0x1c4d349f,0x22d7c3c5),LL(0x163d59c5,0x65965844),LL(0xb8abceae,0xcf198c56),LL(0x628559d5,0x6fb1fb1b), + LL(0x07bf8fe3,0x8bbffd06),LL(0x3467734b,0x46259c58),LL(0x35f7f0d3,0xd8953cea),LL(0xd65b0ff1,0x1f0bece2), LL(0xf3c72914,0xf7d5b4b3),LL(0x3cb53389,0x29e8ea95),LL(0x836b6d46,0x4a365626),LL(0xea174fde,0xe849f910), + LL(0xf4737f21,0x7ec62fbb),LL(0x6209f5ac,0xd8dba5ab),LL(0xa5f9adbe,0x24b5d7a9),LL(0xa61dc768,0x707d28f7), LL(0xcaa999ea,0x7711460b),LL(0x1c92e4cc,0xba7b174d),LL(0x18d4bf2d,0x3c4bab66),LL(0xeb8bd279,0xb8f0c980), + LL(0x324b4737,0x024bea9a),LL(0x32a83bca,0xfba9e423),LL(0xa232dced,0x6e635643),LL(0x2571c8ba,0x99619367), LL(0x54b7032b,0xe8c9f357),LL(0x2442d54a,0xf936b3ba),LL(0x8290c65a,0x2263f0f0),LL(0xee2c7fdb,0x48989780), + LL(0x13d4f95e,0xadc5d55a),LL(0xad9b8500,0x737cff85),LL(0x8a73f43d,0x271c557b),LL(0xe18bc476,0xbed617a4), LL(0x7dfd8ab2,0x66245401),LL(0x3a2870aa,0xae7b89ae),LL(0x23a7e545,0x1b555f53),LL(0xbe057e4c,0x6791e247), + LL(0x324fa34d,0x860136ad),LL(0x4cbeae28,0xea111447),LL(0xbedd3299,0x023a4270),LL(0xc1c35c34,0x3d5c3a7f), LL(0x8d0412d2,0xb0f6db67),LL(0xfcdc6b9a,0xd92625e2),LL(0x4e28a982,0x92ae5ccc),LL(0x47a3ce7e,0xea251c36), + LL(0x790691bf,0x9d658932),LL(0x06b736ae,0xed610589),LL(0xc0d63b6e,0x712c2f04),LL(0xc63d488f,0x5cf06fd5), LL(0xd9588e41,0x97363fac),LL(0x2b93257e,0x1f9bf762),LL(0x667acace,0xa9d1ffc4),LL(0x0a061ecf,0x1cf4a1aa), + LL(0xdc1818d0,0x40e48a49),LL(0xa3621ab0,0x0643ff39),LL(0xe39ef639,0x5768640c),LL(0x04d86854,0x1fc099ea), LL(0xeccd28fd,0x9130b9c3),LL(0x7eec54ab,0xd743cbd2),LL(0xe5b475b6,0x052b146f),LL(0x900a7d1f,0x058d9a82), + LL(0x91262b72,0x65e02292),LL(0xbb0edf03,0x96f924f9),LL(0xfe206842,0x5cfa59c8),LL(0x5eafa720,0xf6037004), LL(0x18d7dd96,0x5f30699e),LL(0xcbab2495,0x381e8782),LL(0xdd8be949,0x91669b46),LL(0x26aae8ef,0xb40606f5), + LL(0xfc6751a4,0x2812b839),LL(0xfba800ef,0x16196214),LL(0x4c1a2875,0x4398d5ca),LL(0x653d8349,0x720c00ee), LL(0xd820007c,0xc2699eb0),LL(0xa39b5825,0x880ee660),LL(0x471f6984,0x70694694),LL(0xe3dda99a,0xf7d16ea8), + LL(0xc0519a23,0x28d675b2),LL(0x4f6952e3,0x9ebf94fe),LL(0xa2294a8a,0xf28bb767),LL(0xfe0af3f5,0x85512b4d), LL(0x99b16a0d,0x18958ba8),LL(0xba7548a7,0x95c2430c),LL(0xa16be615,0xb30d1b10),LL(0x85bfb74c,0xe3ebbb97), + LL(0x18549fdb,0xa3273cfe),LL(0x4fcdb792,0xf6e200bf),LL(0x83aba56c,0x54a76e18),LL(0x89ef6aa2,0x73ec66f6), LL(0xd1b9a305,0x8d17add7),LL(0xb7ae1b9d,0xa959c5b9),LL(0x6bcc094a,0x88643522),LL(0xd7d429b9,0xcc5616c4), + LL(0xe6a33f7c,0xa6dada01),LL(0x9d4e70ad,0xc6217a07),LL(0x09c15b7c,0xd619a818),LL(0x0e80c854,0xea06b329), LL(0xa5f5e7b9,0x174811ce),LL(0x787c65f4,0x66dfc310),LL(0x3316ab54,0x4ea7bd69),LL(0x1dcc0f70,0xc12c4acb), + LL(0x1e407dd9,0xe4308d1a),LL(0x91afa997,0xe8a3587c),LL(0xab77b7a5,0xea296c12),LL(0x673c0d52,0xb5ad49e4), LL(0x7006085a,0x40f9b2b2),LL(0x87bf6ec2,0xa88ff340),LL(0x4e3066a6,0x978603b1),LL(0xb5e486e2,0xb3f99fc2), + LL(0xb2e63645,0x07b53f5e),LL(0x84c84232,0xbe57e547),LL(0x7214d5cf,0xd779c216),LL(0x029a3aca,0x617969cd), LL(0x8a7017a0,0xd17668cd),LL(0xbe9b7ee8,0x77b4d19a),LL(0x9c161776,0x58fd0e93),LL(0xd5968a72,0xa8c4f4ef), + LL(0x67b3de77,0x296071cc),LL(0x634f7905,0xae3c0b8e),LL(0x8a7100c9,0x67e440c2),LL(0xeb4b9b42,0xbb8c3c1b), LL(0xc51b3583,0x6d71e8ea),LL(0x9525e642,0x7591f5af),LL(0x13f509f3,0xf73a2f7b),LL(0x5619ac9b,0x618487aa), + LL(0x9d61718a,0x3a72e5f7),LL(0x7592d28c,0x00413bcc),LL(0x963c35cf,0x7d9b11d3),LL(0xb90a46ed,0x77623bcf), LL(0xdcdd2a50,0xdeef273b),LL(0x0601846e,0x4a741f9b),LL(0x0ec6e929,0x33b89e51),LL(0x8b7f22cd,0xcb02319f), + LL(0x084bae24,0xbbe1500d),LL(0x343d2693,0x2f0ae8d7),LL(0x7cdef811,0xacffb5f2),LL(0x263fb94f,0xaa0c030a), LL(0xa0f442de,0x6eef0d61),LL(0x27b139d3,0xf92e1817),LL(0x0ad8bc28,0x1ae6deb7),LL(0xc0514130,0xa89e38dc), + LL(0xd2fdca23,0x81eeb865),LL(0xcc8ef895,0x5a15ee08),LL(0x01905614,0x768fa10a),LL(0x880ee19b,0xeff5b8ef), LL(0xcb1c8a0e,0xf0c0cabb),LL(0xb8c838f9,0x2e1ee9cd),LL(0x8a4a14c0,0x0587d8b8),LL(0x2ff698e5,0xf6f27896), + LL(0x89ee6256,0xed38ef1c),LL(0x6b353b45,0xf44ee1fe),LL(0x70e903b3,0x9115c0c7),LL(0x818f31df,0xc78ec0a1), LL(0xb7dccbc6,0x6c003324),LL(0x163bbc25,0xd96dd1f3),LL(0x5cedd805,0x33aa82dd),LL(0x7f7eb2f1,0x123aae4f), + LL(0xa26262cd,0x1723fcf5),LL(0x0060ebd5,0x1f7f4d5d),LL(0xb2eaa3af,0xf19c5c01),LL(0x9790accf,0x2ccb9b14), LL(0x52324aa6,0x1f9c1cad),LL(0x7247df54,0x63200526),LL(0xbac96f82,0x5732fe42),LL(0x01a1c384,0x52fe771f), + LL(0xb1001684,0x546ca13d),LL(0xa1709f75,0xb56b4eee),LL(0xd5db8672,0x266545a9),LL(0x1e8f3cfb,0xed971c90), LL(0xe3a07b29,0x4e7d8691),LL(0xe4b696b9,0x7570d9ec),LL(0x7bc7e9ae,0xdc5fa067),LL(0xc82c4844,0x68b44caf), + LL(0xbf44da80,0x519d34b3),LL(0x5ab32e66,0x283834f9),LL(0x6278a000,0x6e608797),LL(0x627312f6,0x1e62960e), LL(0xe6901c55,0x9b87b27b),LL(0x24fdbc1f,0x80e78538),LL(0x2facc27d,0xbbbc0951),LL(0xac143b5a,0x06394239), + LL(0x376c1944,0x35bb4a40),LL(0x63da1511,0x7cb62694),LL(0xb7148a3b,0xafd29161),LL(0x4e2ea2ee,0xa6f9d9ed), LL(0x880dd212,0x15dc2ca2),LL(0xa61139a9,0x903c3813),LL(0x6c0f8785,0x2aa7b46d),LL(0x901c60ff,0x36ce2871), + LL(0xe10d9c12,0xc683b028),LL(0x032f33d3,0x7573baa2),LL(0x67a31b58,0x87a9b1f6),LL(0xf4ffae12,0xfd3ed11a), LL(0x0cb2748e,0x83dcaa9a),LL(0x5d6fdf16,0x8239f018),LL(0x72753941,0xba67b49c),LL(0xc321cb36,0x2beec455), + LL(0x3f8b84ce,0x88015606),LL(0x8d38c86f,0x76417083),LL(0x598953dd,0x054f1ca7),LL(0x4e8e7429,0xc939e110), LL(0x5a914f2f,0x9b1ac2b3),LL(0xe74b8f9c,0x39e35ed3),LL(0x781b2fb0,0xd0debdb2),LL(0x2d997ba2,0x1585638f), + LL(0x9e2fce99,0x9c4b646e),LL(0x1e80857f,0x68a21081),LL(0x3643b52a,0x06d54e44),LL(0x0d8eb843,0xde8d6d63), LL(0x42146a0a,0x70321563),LL(0x5eaa3622,0x8ba826f2),LL(0x86138787,0x227a58bd),LL(0x10281d37,0x43b6c03c), + LL(0xb54dde39,0x6326afbb),LL(0xdb6f2d5f,0x744e5e8a),LL(0xcff158e1,0x48b2a99a),LL(0xef87918f,0xa93c8fa0), LL(0xde058c5c,0x2182f956),LL(0x936f9e7a,0x216235d2),LL(0xd2e31e67,0xace0c0db),LL(0xf23ac3e7,0xc96449bf), + LL(0x170693bd,0x7e9a2874),LL(0xa45e6335,0xa28e14fd),LL(0x56427344,0x5757f6b3),LL(0xacf8edf9,0x822e4556), LL(0xe6a285cd,0x2b7a6ee2),LL(0xa9df3af0,0x5866f211),LL(0xf845b844,0x40dde2dd),LL(0x110e5e49,0x986c3726), + LL(0xf7172277,0x73680c2a),LL(0x0cccb244,0x57b94f0f),LL(0x2d438ca7,0xbdff7267),LL(0xcf4663fd,0xbad1ce11), LL(0xd8f71cae,0x9813ed9d),LL(0x961fdaa6,0xf43272a6),LL(0xbd6d1637,0xbeff0119),LL(0x30361978,0xfebc4f91), + LL(0x2f41deff,0x02b37a95),LL(0xe63b89b7,0x0e44a59a),LL(0x143ff951,0x673257dc),LL(0xd752baf4,0x19c02205), LL(0xc4b7d692,0x46c23069),LL(0xfd1502ac,0x2e6392c3),LL(0x1b220846,0x6057b1a2),LL(0x0c1b5b63,0xe51ff946), +}, +/* digit=35 base_pwr=2^245 */ +{ + LL(0x566c5c43,0x6e85cb51),LL(0x3597f046,0xcff9c919),LL(0x4994d94a,0x9354e90c),LL(0x2147927d,0xe0a39332), LL(0x0dc1eb2b,0x8427fac1),LL(0x2ff319fa,0x88cfd8c2),LL(0x01965274,0xe2d4e684),LL(0x67aaa746,0xfa2e067d), + LL(0x3e5f9f11,0xb6d92a7f),LL(0xd6cb3b8e,0x9afe153a),LL(0xddf800bd,0x4d1a6dd7),LL(0xcaf17e19,0xf6c13cc0), LL(0x325fc3ee,0x15f6c58e),LL(0xa31dc3b2,0x71095400),LL(0xafa3d3e7,0x168e7c07),LL(0x94c7ae2d,0x3f8417a1), + LL(0x813b230d,0xec234772),LL(0x17344427,0x634d0f5f),LL(0xd77fc56a,0x11548ab1),LL(0xce06af77,0x7fab1750), LL(0x4f7c4f83,0xb62c10a7),LL(0x220a67d9,0xa7d2edc4),LL(0x921209a0,0x1c404170),LL(0xface59f0,0x0b9815a0), + LL(0x319540c3,0x2842589b),LL(0xa283d6f8,0x18490f59),LL(0xdaae9fcb,0xa2731f84),LL(0xc3683ba0,0x3db6d960), LL(0x14611069,0xc85c63bb),LL(0x0788bf05,0xb19436af),LL(0x347460d2,0x905459df),LL(0xe11a7db1,0x73f6e094), + LL(0xb6357f37,0xdc7f938e),LL(0x2bd8aa62,0xc5d00f79),LL(0x2ca979fc,0xc878dcb9),LL(0xeb023a99,0x37e83ed9), LL(0x1560bf3d,0x6b23e273),LL(0x1d0fae61,0x1086e459),LL(0x9a9414bd,0x78248316),LL(0xf0ea9ea1,0x1b956bc0), + LL(0xc31b9c38,0x7b85bb91),LL(0x48ef57b5,0x0c5aa90b),LL(0xaf3bab6f,0xdedeb169),LL(0x2d373685,0xe610ad73), LL(0x02ba8e15,0xf13870df),LL(0x8ca7f771,0x0337edb6),LL(0xb62c036c,0xe4acf747),LL(0xb6b94e81,0xd921d576), + LL(0x2c422f7a,0xdbc86439),LL(0xed348898,0xfb635362),LL(0xc45bfcd1,0x83084668),LL(0x2b315e11,0xc357c9e3), LL(0x5b2e5b8c,0xb173b540),LL(0xe102b9a4,0x7e946931),LL(0x7b0fb199,0x17c890eb),LL(0xd61b662b,0xec225a83), + LL(0xee3c76cb,0xf306a3c8),LL(0xd32a1f6e,0x3cf11623),LL(0x6863e956,0xe6d5ab64),LL(0x5c005c26,0x3b8a4cbe), LL(0x9ce6bb27,0xdcd529a5),LL(0x04d4b16f,0xc4afaa52),LL(0x7923798d,0xb0624a26),LL(0x6b307fab,0x85e56df6), + LL(0x2bf29698,0x0281893c),LL(0xd7ce7603,0x91fc19a4),LL(0xad9a558f,0x75a5dca3),LL(0x4d50bf77,0x40ceb3fa), LL(0xbc9ba369,0x1baf6060),LL(0x597888c2,0x927e1037),LL(0x86a34c07,0xd936bf19),LL(0xc34ae980,0xd4cf10c1), + LL(0x859dd614,0x3a3e5334),LL(0x18d0c8ee,0x9c475b5b),LL(0x07cd51d5,0x63080d1f),LL(0xb88b4326,0xc9c0d0a6), LL(0xc234296f,0x1ac98691),LL(0x94887fb6,0x2a0a83a4),LL(0x0cea9cf2,0x56511427),LL(0xa24802f5,0x5230a6e8), + LL(0x72e3d5c1,0xf7a2bf0f),LL(0x4f21439e,0x37717446),LL(0x9ce30334,0xfedcbf25),LL(0x7ce202f9,0xe0030a78), LL(0x1202e9ca,0x6f2d9ebf),LL(0x75e6e591,0xe79dde6c),LL(0xf1dac4f8,0xf52072af),LL(0xbb9b404d,0x6c8d087e), + LL(0xbce913af,0xad0fc73d),LL(0x458a07cb,0x909e587b),LL(0xd4f00c8a,0x1300da84),LL(0xb54466ac,0x425cd048), LL(0x90e9d8bf,0xb59cb9be),LL(0x3e431b0e,0x991616db),LL(0x531aecff,0xd3aa117a),LL(0x59f4dc3b,0x91af92d3), + LL(0xe93fda29,0x9b1ec292),LL(0xe97d91bc,0x76bb6c17),LL(0xaface1e6,0x7509d95f),LL(0xbe855ae3,0x3653fe47), LL(0x0f680e75,0x73180b28),LL(0xeeb6c26c,0x75eefd1b),LL(0xb66d4236,0xa4cdf29f),LL(0x6b5821d8,0x2d70a997), + LL(0x20445c36,0x7a3ee207),LL(0x59877174,0x71d1ac82),LL(0x949f73e9,0x0fc539f7),LL(0x982e3081,0xd05cf3d7), LL(0x7b1c7129,0x8758e20b),LL(0x569e61f2,0xffadcc20),LL(0x59544c2d,0xb05d3a2f),LL(0x9fff5e53,0xbe16f5c1), + LL(0xaad58135,0x73cf65b8),LL(0x037aa5be,0x622c2119),LL(0x646fd6a0,0x79373b3f),LL(0x0d3978cf,0x0e029db5), LL(0x94fba037,0x8bdfc437),LL(0x620797a6,0xaefbd687),LL(0xbd30d38e,0x3fa5382b),LL(0x585d7464,0x7627cfbf), + LL(0x4e4ca463,0xb2330fef),LL(0x3566cc63,0xbcef7287),LL(0xcf780900,0xd161d2ca),LL(0x5b54827d,0x135dc539), LL(0x27bf1bc6,0x638f052e),LL(0x07dfa06c,0x10a224f0),LL(0x6d3321da,0xe973586d),LL(0x26152c8f,0x8b0c5738), + LL(0x34606074,0x07ef4f2a),LL(0xa0f7047a,0x80fe7fe8),LL(0xe1a0e306,0x3d1a8152),LL(0x88da5222,0x32cf43d8), LL(0x5f02ffe6,0xbf89a95f),LL(0x806ad3ea,0x3d9eb9a4),LL(0x79c8e55e,0x012c17bb),LL(0x99c81dac,0xfdcd1a74), + LL(0xb9556098,0x7043178b),LL(0x801c3886,0x4090a1df),LL(0x9b67b912,0x759800ff),LL(0x232620c8,0x3e5c0304), LL(0x70dceeca,0x4b9d3c4b),LL(0x181f648e,0xbb2d3c15),LL(0x6e33345c,0xf981d837),LL(0x0cf2297a,0xb626289b), + LL(0x8baebdcf,0x766ac659),LL(0x75df01e5,0x1a28ae09),LL(0x375876d8,0xb71283da),LL(0x607b9800,0x4865a96d), LL(0x237936b2,0x25dd1bcd),LL(0x60417494,0x332f4f4b),LL(0x370a2147,0xd0923d68),LL(0xdc842203,0x497f5dfb), + LL(0x32be5e0f,0x9dc74cbd),LL(0x17a01375,0x7475bcb7),LL(0x50d872b1,0x438477c9),LL(0xffe1d63d,0xcec67879), LL(0xd8578c70,0x9b006014),LL(0x78bb6b8b,0xc9ad99a8),LL(0x11fb3806,0x6799008e),LL(0xcd44cab3,0xcfe81435), + LL(0x2f4fb344,0xa2ee1582),LL(0x483fa6eb,0xb8823450),LL(0x652c7749,0x622d323d),LL(0xbeb0a15b,0xd8474a98), LL(0x5d1c00d0,0xe43c154d),LL(0x0e3e7aac,0x7fd581d9),LL(0x2525ddf8,0x2b44c619),LL(0xb8ae9739,0x67a033eb), + LL(0x9ef2d2e4,0x113ffec1),LL(0xd5a0ea7f,0x1bf6767e),LL(0x03714c0a,0x57fff75e),LL(0x0a23e9ee,0xa23c422e), LL(0x540f83af,0xdd5f6b2d),LL(0x55ea46a7,0xc2c2c27e),LL(0x672a1208,0xeb6b4246),LL(0xae634f7a,0xd13599f7), + LL(0xd7b32c6e,0xcf914b5c),LL(0xeaf61814,0x61a5a640),LL(0x208a1bbb,0x8dc3df8b),LL(0xb6d79aa5,0xef627fd6), LL(0xc4c86bc8,0x44232ffc),LL(0x061539fe,0xe6f9231b),LL(0x958b9533,0x1d04f25a),LL(0x49e8c885,0x180cf934), + LL(0x9884aaf7,0x89689595),LL(0x07b348a6,0xb1959be3),LL(0x3c147c87,0x96250e57),LL(0xdd0c61f8,0xae0efb3a), LL(0xca8c325e,0xed00745e),LL(0xecff3f70,0x3c911696),LL(0x319ad41d,0x73acbc65),LL(0xf0b1c7ef,0x7b01a020), + LL(0x63a1483f,0xea32b293),LL(0x7a248f96,0x89eabe71),LL(0x343157e5,0x9c6231d3),LL(0xdf3c546d,0x93a375e5), LL(0x6a2afe69,0xe76e9343),LL(0xe166c88e,0xc4f89100),LL(0x4f872093,0x248efd0d),LL(0x8fe0ea61,0xae0eb3ea), + LL(0x9d79046e,0xaf89790d),LL(0x6cee0976,0x4d650f2d),LL(0x43071eca,0xa3935d9a),LL(0x283b0bfe,0x66fcd2c9), LL(0x696605f1,0x0e665eb5),LL(0xa54cd38d,0xe77e5d07),LL(0x43d950cf,0x90ee050a),LL(0xd32e69b5,0x86ddebda), + LL(0xfddf7415,0x6ad94a3d),LL(0x3f6e8d5a,0xf7fa1309),LL(0xe9957f75,0xc4831d1d),LL(0xd5817447,0x7de28501), LL(0x9e2aeb6b,0x6f1d7078),LL(0xf67a53c2,0xba2b9ff4),LL(0xdf9defc3,0x36963767),LL(0x0d38022c,0x479deed3), + LL(0x3a8631e8,0xd2edb89b),LL(0x7a213746,0x8de855de),LL(0xb00c5f11,0xb2056cb7),LL(0x2c9b85e4,0xdeaefbd0), LL(0xd150892d,0x03f39a8d),LL(0x218b7985,0x37b84686),LL(0xb7375f1a,0x36296dd8),LL(0xb78e898e,0x472cd4b1), + LL(0xe9f05de9,0x15dff651),LL(0x2ce98ba9,0xd4045069),LL(0x9b38024c,0x8466a7ae),LL(0xe5a6b5ef,0xb910e700), LL(0xb3aa8f0d,0xae1c56ea),LL(0x7eee74a6,0xbab2a507),LL(0x4b4c4620,0x0dca11e2),LL(0x4c47d1f4,0xfd896e2e), + LL(0x308fbd93,0xeb45ae53),LL(0x02c36fda,0x46cd5a2e),LL(0xbaa48385,0x6a3d4e90),LL(0x9dbe9960,0xdd55e62e), LL(0x2a81ede7,0xa1406aa0),LL(0xf9274ea7,0x6860dd14),LL(0x80414f86,0xcfdcb0c2),LL(0x22f94327,0xff410b10), + LL(0x49ad467b,0x5a33cc38),LL(0x0a7335f1,0xefb48b6c),LL(0xb153a360,0x14fb54a4),LL(0xb52469cc,0x604aa9d2), LL(0x754e48e9,0x5e9dc486),LL(0x37471e8e,0x693cb455),LL(0x8d3b37b6,0xfb2fd7cd),LL(0xcf09ff07,0x63345e16), + LL(0x23a5d896,0x9910ba6b),LL(0x7fe4364e,0x1fe19e35),LL(0x9a33c677,0x6e1da8c3),LL(0x29fd9fd0,0x15b4488b), LL(0x1a1f22bf,0x1f439254),LL(0xab8163e8,0x920a8a70),LL(0x07e5658e,0x3fd1b249),LL(0xb6ec839b,0xf2c4f79c), + LL(0x4aa38d1b,0x1abbc3d0),LL(0xb5d9510e,0x3b0db35c),LL(0x3e60dec0,0x1754ac78),LL(0xea099b33,0x53272fd7), LL(0x07a8e107,0x5fb0494f),LL(0x6a8191fa,0x4a89e137),LL(0x3c4ad544,0xa113b7f6),LL(0x6cb9897b,0x88a2e909), + LL(0xb44a3f84,0x17d55de3),LL(0x17c6c690,0xacb2f344),LL(0x10232390,0x32088168),LL(0x6c733bf7,0xf2e8a61f), LL(0x9c2d7652,0xa774aab6),LL(0xed95c5bc,0xfb5307e3),LL(0x4981f110,0xa05c73c2),LL(0xa39458c9,0x1baae31c), + LL(0xcbea62e7,0x1def185b),LL(0xeaf63059,0xe8ac9eae),LL(0x9921851c,0x098a8cfd),LL(0x3abe2f5b,0xd959c3f1), LL(0x20e40ae5,0xa4f19525),LL(0x07a24aa1,0x320789e3),LL(0x7392b2bc,0x259e6927),LL(0x1918668b,0x58f6c667), + LL(0xc55d2d8b,0xce1db2bb),LL(0xf4f6ca56,0x41d58bb7),LL(0x8f877614,0x7650b680),LL(0xf4c349ed,0x905e16ba), LL(0xf661acac,0xed415140),LL(0xcb2270af,0x3b8784f0),LL(0x8a402cba,0x3bc280ac),LL(0x0937921a,0xd53f7146), + LL(0xe5681e83,0xc03c8ee5),LL(0xf6ac9e4a,0x62126105),LL(0x936b1a38,0x9503a53f),LL(0x782fecbd,0x3d45e2d4), LL(0x76e8ae98,0x69a5c439),LL(0xbfb4b00e,0xb53b2eeb),LL(0x72386c89,0xf1674712),LL(0x4268bce4,0x30ca34a2), + LL(0x78341730,0x7f1ed86c),LL(0xb525e248,0x8ef5beb8),LL(0xb74fbf38,0xbbc489fd),LL(0x91a0b382,0x38a92a0e), LL(0x22433ccf,0x7a77ba3f),LL(0xa29f05a9,0xde8362d6),LL(0x61189afc,0x7f6a30ea),LL(0x59ef114f,0x693b5505), + LL(0xcd1797a1,0x50266bc0),LL(0xf4b7af2d,0xea17b47e),LL(0x3df9483e,0xd6c4025c),LL(0xa37b18c9,0x8cbb9d9f), LL(0x4d8424cf,0x91cbfd9c),LL(0xab1c3506,0xdb7048f1),LL(0x028206a3,0x9eaf641f),LL(0x25bdf6ce,0xf986f3f9), + LL(0x224c08dc,0x262143b5),LL(0x81b50c91,0x2bbb09b4),LL(0xaca8c84f,0xc16ed709),LL(0xb2850ca8,0xa6210d9d), LL(0x09cb54d6,0x6d8df67a),LL(0x500919a4,0x91eef6e0),LL(0x0f132857,0x90f61381),LL(0xf8d5028b,0x9acede47), + LL(0x90b771c3,0x844d1b71),LL(0xba6426be,0x563b71e4),LL(0xbdb802ff,0x2efa2e83),LL(0xab5b4a41,0x3410cbab), LL(0x30da84dd,0x555b2d26),LL(0xee1cc29a,0xd0711ae9),LL(0x2f547792,0xcf3e8c60),LL(0xdc678b35,0x03d7d5de), + LL(0xced806b8,0x071a2fa8),LL(0x697f1478,0x222e6134),LL(0xabfcdbbf,0xdc16fd5d),LL(0x121b53b8,0x44912ebf), LL(0x2496c27c,0xac943674),LL(0x1ffc26b0,0x8ea3176c),LL(0x13debf2c,0xb6e224ac),LL(0xf372a832,0x524cc235), + LL(0x9f6f1b18,0xd706e1d8),LL(0x44cce35b,0x2552f005),LL(0xa88e31fc,0x8c8326c2),LL(0xf9552047,0xb5468b2c), LL(0x3ff90f2b,0xce683e88),LL(0x2f0a5423,0x77947bdf),LL(0xed56e328,0xd0a1b28b),LL(0xc20134ac,0xaee35253), + LL(0x3567962f,0x7e98367d),LL(0x8188bffb,0x379ed61f),LL(0xfaf130a1,0x73bba348),LL(0x904ed734,0x6c1f75e1), LL(0x3b4a79fc,0x18956642),LL(0x54ef4493,0xf20bc83d),LL(0x9111eca1,0x836d425d),LL(0x009a8dcf,0xe5b5c318), + LL(0x13221bc5,0x3360b25d),LL(0x6b3eeaf7,0x707baad2),LL(0x743a95a1,0xd7279ed8),LL(0x969e809f,0x7450a875), LL(0xe5d0338f,0x32b6bd53),LL(0x2b883bbc,0x1e77f7af),LL(0x1063ecd0,0x90da12cc),LL(0xc315be47,0xe2697b58), + LL(0xda85d534,0x2771a5bd),LL(0xff980eea,0x53e78c1f),LL(0x900385e7,0xadf1cf84),LL(0xc9387b62,0x7d3b14f6), LL(0xcb8f2bd2,0x170e74b0),LL(0x827fa993,0x2d50b486),LL(0xf6f32bab,0xcdbe8c9a),LL(0xc3b93ab8,0x55e906b0), + LL(0x8fe280d1,0x747f22fc),LL(0xb2e114ab,0xcd8e0de5),LL(0xe10b68b0,0x5ab7dbeb),LL(0xa480d4b2,0x9dc63a9c), LL(0x4be1495f,0x78d4bc3b),LL(0x9359122d,0x25eb3db8),LL(0x0809cbdc,0x3f8ac05b),LL(0xd37c702f,0xbf4187bb), + LL(0x1416a6a5,0x84cea069),LL(0x43ef881c,0x8f860c79),LL(0x38038a5d,0x41311f8a),LL(0xfc612067,0xe78c2ec0), LL(0x5ad73581,0x494d2e81),LL(0x59604097,0xb4cc9e00),LL(0xf3612cba,0xff558aec),LL(0x9e36c39e,0x35beef7a), + LL(0xdbcf41b9,0x1845c7cf),LL(0xaea997c0,0x5703662a),LL(0xe402f6d8,0x8b925afe),LL(0x4dd72162,0xd0a1b1ae), LL(0x03c41c4b,0x9f47b375),LL(0x0391d042,0xa023829b),LL(0x503b8b0a,0x5f5045c3),LL(0x98c010e5,0x123c2688), + LL(0x36ba06ee,0x324ec0cc),LL(0x3dd2cc0c,0xface3115),LL(0xf333e91f,0xb364f3be),LL(0x28e832b0,0xef8aff73), LL(0x2d05841b,0x1e9bad04),LL(0x356a21e2,0x42f0e3df),LL(0x4add627e,0xa3270bcb),LL(0xd322e711,0xb09a8158), + LL(0x0fee104a,0x86e326a1),LL(0x3703f65d,0xad7788f8),LL(0x47bc4833,0x7e765430),LL(0x2b9b893a,0x6cee582b), LL(0xe8f55a7b,0x9cd2a167),LL(0xd9e4190d,0xefbee3c6),LL(0xd40c2e9d,0x33ee7185),LL(0xa380b548,0x844cc9c5), + LL(0x66926e04,0x323f8ecd),LL(0x8110c1ba,0x0001e38f),LL(0xfc6a7f07,0x8dbcac12),LL(0x0cec0827,0xd65e1d58), LL(0xbe76ca2d,0xd2cd4141),LL(0xe892f33a,0x7895cf5c),LL(0x367139d2,0x956d230d),LL(0xd012c4c1,0xa91abd3e), + LL(0x87eb36bf,0x34fa4883),LL(0x914b8fb4,0xc5f07102),LL(0xadb9c95f,0x90f0e579),LL(0x28888195,0xfe6ea8cb), LL(0xedfa9284,0x7b9b5065),LL(0x2b8c8d65,0x6c510bd2),LL(0xcbe8aafd,0xd7b8ebef),LL(0x96b1da07,0xedb3af98), + LL(0x6295d426,0x28ff779d),LL(0x3fa3ad7b,0x0c4f6ac7),LL(0x8b8e2604,0xec44d054),LL(0x8b0050e1,0x9b32a66d), LL(0xf0476ce2,0x1f943366),LL(0xa602c7b4,0x7554d953),LL(0x524f2809,0xbe35aca6),LL(0xfd4edbea,0xb6881229), + LL(0x508efb63,0xe8cd0c8f),LL(0x6abcefc7,0x9eb5b5c8),LL(0xb441ab4f,0xf5621f5f),LL(0xb76a2b22,0x79e6c046), LL(0xe37a1f69,0x74a4792c),LL(0x03542b60,0xcbd252cb),LL(0xb3c20bd3,0x785f65d5),LL(0x4fabc60c,0x8dea6143), + LL(0xde673629,0x45e21446),LL(0x703c2d21,0x57f7aa1e),LL(0x98c868c7,0xa0e99b7f),LL(0x8b641676,0x4e42f66d), LL(0x91077896,0x602884dc),LL(0xc2c9885b,0xa0d690cf),LL(0x3b9a5187,0xfeb4da33),LL(0x153c87ee,0x5f789598), + LL(0x52b16dba,0x2192dd47),LL(0x3524c1b1,0xdeefc0e6),LL(0xe4383693,0x465ea76e),LL(0x361b8d98,0x79401711), LL(0xf21a15cb,0xa5f9ace9),LL(0xefee9aeb,0x73d26163),LL(0xe677016c,0xcca844b3),LL(0x57eaee06,0x6c122b07), + LL(0x15f09690,0xb782dce7),LL(0x2dfc0fc9,0x508b9b12),LL(0x65d89fc6,0x9015ab4b),LL(0xd6d5bb0f,0x5e79dab7), LL(0x6c775aa2,0x64f021f0),LL(0x37c7eca1,0xdf09d8cc),LL(0xef2fa506,0x9a761367),LL(0x5b81eec6,0xed4ca476), + LL(0x10bbb8b5,0x262ede36),LL(0x0641ada3,0x0737ce83),LL(0xe9831ccc,0x4c94288a),LL(0x8065e635,0x487fc1ce), LL(0xb8bb3659,0xb13d7ab3),LL(0x855e4120,0xdea5df3e),LL(0x85eb0244,0xb9a18573),LL(0xa7cfe0a3,0x1a1b8ea3), + LL(0x67b0867c,0x3b837119),LL(0x9d364520,0x8d5e0d08),LL(0xd930f0e3,0x52dccc1e),LL(0xbf20bbaf,0xefbbcec7), LL(0x0263ad10,0x99cffcab),LL(0xfcd18f8a,0xd8199e6d),LL(0xe9f10617,0x64e2773f),LL(0x08704848,0x0079e8e1), + LL(0x8a342283,0x1169989f),LL(0xa83012e6,0x8097799c),LL(0x8a6a9001,0xece966cb),LL(0x072ac7fc,0x93b3afef), LL(0x2db3d5ba,0xe6893a2a),LL(0x89bf4fdc,0x263dc462),LL(0xe0396673,0x8852dfc9),LL(0x3af362b6,0x7ac70895), + LL(0x5c2f342b,0xbb9cce4d),LL(0xb52d7aae,0xbf80907a),LL(0x2161bcd0,0x97f3d3cd),LL(0x0962744d,0xb25b0834), LL(0x6c3a1dda,0xc5b18ea5),LL(0x06c92317,0xfe4ec7eb),LL(0xad1c4afe,0xb787b890),LL(0x0ede801a,0xdccd9a92), + LL(0xdb58da1f,0x9ac6ddda),LL(0xb8cae6ee,0x22bbc12f),LL(0x815c4a43,0xc6f8bced),LL(0xf96480c7,0x8105a92c), LL(0x7a859d51,0x0dc3dbf3),LL(0x3041196b,0xe3ec7ce6),LL(0x0d1067c9,0xd9f64b25),LL(0x3d1f8dd8,0xf2321321), + LL(0x76497ee8,0x8b5c619c),LL(0xc717370e,0x5d2b0ac6),LL(0x4fcf68e1,0x98204cb6),LL(0x62bc6792,0x0bdec211), LL(0xa63b1011,0x6973ccef),LL(0xe0de1ac5,0xf9e3fa97),LL(0x3d0e0c8b,0x5efb693e),LL(0xd2d4fcb4,0x037248e9), +}, +/* digit=36 base_pwr=2^252 */ +{ + LL(0x1ec34f9e,0x80802dc9),LL(0x33810603,0xd8772d35),LL(0x530cb4f3,0x3f06d66c),LL(0xc475c129,0x7be5ed0d), LL(0x31e82b10,0xcb9e3c19),LL(0xc9ff6b4c,0xc63d2857),LL(0x92a1b45e,0xb92118c6),LL(0x7285bbca,0x0aec4414), + LL(0x1e29a3ef,0xfc189ae7),LL(0x4c93302e,0xcbe906f0),LL(0xceaae10e,0xd0107914),LL(0xb68e19f8,0xb7a23f34), LL(0xefd2119d,0xe9d875c2),LL(0xfcadc9c8,0x03198c6e),LL(0x4da17113,0x65591bf6),LL(0x3d443038,0x3cf0bbf8), + LL(0x2b724759,0xae485bb7),LL(0xb2d4c63a,0x945353e1),LL(0xde7d6f2c,0x82159d07),LL(0x4ec5b109,0x389caef3), LL(0xdb65ef14,0x4a8ebb53),LL(0xdd99de43,0x2dc2cb7e),LL(0x83f2405f,0x816fa3ed),LL(0xc14208a3,0x73429bb9), + LL(0xb01e6e27,0xb618d590),LL(0xe180b2dc,0x047e2ccd),LL(0x04aea4a9,0xd1b299b5),LL(0x9fa403a4,0x412c9e1e), LL(0x79407552,0x88d28a36),LL(0xf332b8e3,0x49c50136),LL(0xe668de19,0x3a1b6fcc),LL(0x75122b97,0x178851bc), + LL(0xfb85fa4c,0xb1e13752),LL(0x383c8ce9,0xd61257ce),LL(0xd2f74dae,0xd43da670),LL(0xbf846bbb,0xa35aa23f), LL(0x4421fc83,0x5e74235d),LL(0xc363473b,0xf6df8ee0),LL(0x3c4aa158,0x34d7f52a),LL(0x9bc6d22e,0x50d05aab), + LL(0xa64785f4,0x8c56e735),LL(0x5f29cd07,0xbc56637b),LL(0x3ee35067,0x53b2bb80),LL(0xdc919270,0x50235a0f), LL(0xf2c4aa65,0x191ab6d8),LL(0x8396023b,0xc3475831),LL(0xf0f805ba,0x80400ba5),LL(0x5ec0f80f,0x8881065b), + LL(0xcc1b5e83,0xc370e522),LL(0x860b8bfb,0xde2d4ad1),LL(0x67b256df,0xad364df0),LL(0xe0138997,0x8f12502e), LL(0x7783920a,0x503fa0dc),LL(0xc0bc866a,0xe80014ad),LL(0xd3064ba6,0x3f89b744),LL(0xcba5dba5,0x03511dcd), + LL(0x95a7b1a2,0x197dd46d),LL(0x3c6341fb,0x9c4e7ad6),LL(0x484c2ece,0x426eca29),LL(0xde7f4f8a,0x9211e489), LL(0xc78ef1f4,0x14997f6e),LL(0x06574586,0x2b2c0910),LL(0x1c3eede8,0x17286a6e),LL(0x0f60e018,0x25f92e47), + LL(0x31890a36,0x805c5646),LL(0x57feea5b,0x703ef600),LL(0xaf3c3030,0x389f747c),LL(0x54dd3739,0xe0e5daeb), LL(0xc9c9f155,0xfe24a4c3),LL(0xb5393962,0x7e4bf176),LL(0xaf20bf29,0x37183de2),LL(0xf95a8c3b,0x4a1bd7b5), + LL(0x46191d3d,0xa83b9699),LL(0x7b87f257,0x281fc8dd),LL(0x54107588,0xb18e2c13),LL(0x9b2bafe8,0x6372def7), LL(0x0d8972ca,0xdaf4bb48),LL(0x56167a3f,0x3f2dd4b7),LL(0x84310cf4,0x1eace32d),LL(0xe42700aa,0xe3bcefaf), + LL(0xd785e73d,0x5fe5691e),LL(0x2ea60467,0xa5db5ab6),LL(0xdfc6514a,0x02e23d41),LL(0xe03c3665,0x35e8048e), LL(0x1adaa0f8,0x3f8b118f),LL(0x84ce1a5a,0x28ec3b45),LL(0x2c6646b8,0xe8cacc6e),LL(0xdbd0e40f,0x1343d185), + LL(0xcaaa358c,0xe5d7f844),LL(0x9924182a,0x1a1db7e4),LL(0x9c875d9a,0xd64cd42d),LL(0x042eeec8,0xb37b515f), LL(0x7b165fbe,0x4d4dd409),LL(0xe206eff3,0xfc322ed9),LL(0x59b7e17e,0x7dee4102),LL(0x8236ca00,0x55a481c0), + LL(0xc23fc975,0x8c885312),LL(0x05d6297b,0x15715806),LL(0xf78edd39,0xa078868e),LL(0x03c45e52,0x956b31e0), LL(0xff7b33a6,0x470275d5),LL(0x0c7e673f,0xc8d5dc3a),LL(0x7e2f2598,0x419227b4),LL(0x4c14a975,0x8b37b634), + LL(0x8b11888c,0xd0667ed6),LL(0x803e25dc,0x5e0e8c3e),LL(0xb987a24a,0x34e5d0dc),LL(0xae920323,0x9f40ac3b), LL(0x34e0f63a,0x5463de95),LL(0x6b6328f9,0xa128bf92),LL(0xda64f1b7,0x491ccd7c),LL(0xc47bde35,0x7ef1ec27), + LL(0xa36a2737,0xa857240f),LL(0x63621bc1,0x35dc1366),LL(0xd4fb6897,0x7a3a6453),LL(0xc929319d,0x80f1a439), LL(0xf8cb0ba0,0xfc18274b),LL(0x8078c5eb,0xb0b53766),LL(0x1e01d0ef,0xfb0d4924),LL(0x372ab09c,0x50d7c67d), + LL(0x3aeac968,0xb4e370af),LL(0xc4b63266,0xe4f7fee9),LL(0xe3ac5664,0xb4acd4c2),LL(0xceb38cbf,0xf8910bd2), LL(0xc9c0726e,0x1c3ae50c),LL(0xd97b40bf,0x15309569),LL(0xfd5a5a1b,0x70884b7f),LL(0xef8314cd,0x3890896a), + LL(0xa5618c93,0x58e1515c),LL(0x77d942d1,0xe665432b),LL(0xb6f767a8,0xb32181bf),LL(0x3a604110,0x753794e8), LL(0xe8c0dbcc,0x09afeb7c),LL(0x598673a3,0x31e02613),LL(0x7d46db00,0x5d98e557),LL(0x9d985b28,0xfc21fb8c), + LL(0xb0843e0b,0xc9040116),LL(0x69b04531,0x53b1b3a8),LL(0x85d7d830,0xdd1649f0),LL(0xcb7427e8,0xbb3bcc87), LL(0xc93dce83,0x77261100),LL(0xa1922a2a,0x7e79da61),LL(0xf3149ce8,0x587a2b02),LL(0xde92ec83,0x147e1384), + LL(0xaf077f30,0x484c83d3),LL(0x0658b53a,0xea78f844),LL(0x027aec53,0x912076c2),LL(0x93c8177d,0xf34714e3), LL(0xc2376c84,0x37ef5d15),LL(0x3d1aa783,0x8315b659),LL(0xef852a90,0x3a75c484),LL(0x16086bd4,0x0ba0c58a), + LL(0x529a6d48,0x29688d7a),LL(0xc2f19203,0x9c7f250d),LL(0x682e2df9,0x123042fb),LL(0xad8121bc,0x2b7587e7), LL(0xe0182a65,0x30fc0233),LL(0xe3e1128a,0xb82ecf87),LL(0x93fb098f,0x71682861),LL(0x85e9e6a7,0x043e21ae), + LL(0x66c834ea,0xab5b49d6),LL(0x47414287,0x3be43e18),LL(0x219a2a47,0xf40fb859),LL(0xcc58df3c,0x0e6559e9), LL(0x0c6615b4,0xfe1dfe8e),LL(0x56459d70,0x14abc8fd),LL(0x05de0386,0x7be0fa8e),LL(0xe9035c7c,0x8e63ef68), + LL(0x53b31e91,0x116401b4),LL(0x4436b4d8,0x0cba7ad4),LL(0x107afd66,0x9151f9a0),LL(0x1f0ee4c4,0xafaca8d0), LL(0x9ee9761c,0x75fe5c1d),LL(0xf0c0588f,0x3497a16b),LL(0x0304804c,0x3ee2bebd),LL(0xc2c990b9,0xa8fb9a60), + LL(0x39251114,0xd14d32fe),LL(0xcac73366,0x36bf25bc),LL(0xdba7495c,0xc9562c66),LL(0x46ad348b,0x324d301b), LL(0xd670407e,0x9f46620c),LL(0xe3733a01,0x0ea8d4f1),LL(0xb0c324e0,0xd396d532),LL(0x03c317cd,0x5b211a0e), + LL(0x5ffe7b37,0x090d7d20),LL(0x1747d2da,0x3b7f3efb),LL(0xb54fc519,0xa2cb525f),LL(0xf66a971e,0x6e220932), LL(0xb486d440,0xddc160df),LL(0x3fe13465,0x7fcfec46),LL(0x76e4c151,0x83da7e4e),LL(0xd8d302b5,0xd6fa48a1), + LL(0x5872cd88,0xc6304f26),LL(0x278b90a1,0x806c1d3c),LL(0xcaf0bc1c,0x3553e725),LL(0xbb9d8d5c,0xff59e603), LL(0x7a0b85dd,0xa4550f32),LL(0x93ecc217,0xdec5720a),LL(0x69d62213,0x0b88b741),LL(0x5b365955,0x7212f245), + LL(0xb5cae787,0x20764111),LL(0x1dfd3124,0x13cb7f58),LL(0x1175aefb,0x2dca77da),LL(0xffaae775,0xeb75466b), LL(0xdb6cff32,0x74d76f3b),LL(0x61fcda9a,0x7440f37a),LL(0xb525028b,0x1bb3ac92),LL(0xa1975f29,0x20fbf8f7), + LL(0xdf83097f,0x982692e1),LL(0x554b0800,0x28738f6c),LL(0xa2ce2f2f,0xdc703717),LL(0x40814194,0x7913b93c), LL(0x1fe89636,0x04924593),LL(0xf78834a6,0x7b98443f),LL(0x5114a5a1,0x11c6ab01),LL(0xffba5f4c,0x60deb383), + LL(0x01a982e6,0x4caa54c6),LL(0x3491cd26,0x1dd35e11),LL(0x7cbd6b05,0x973c315f),LL(0x52494724,0xcab00775), LL(0x6565e15a,0x04659b1f),LL(0x8c8fb026,0xbf30f529),LL(0xa8a0de37,0xfc21641b),LL(0xfa5e5114,0xe9c7a366), + LL(0x52f03ad8,0xdb849ca5),LL(0x024e35c0,0xc7e8dbe9),LL(0xcfc3c789,0xa1a2bbac),LL(0x9c26f262,0xbf733e7d), LL(0xb8444823,0x882ffbf5),LL(0x6bf8483b,0xb7224e88),LL(0x65bef640,0x53023b8b),LL(0xd4d5f8cd,0xaabfec91), + LL(0x079ea1bd,0xa40e1510),LL(0xd05d5d26,0x1ad9addc),LL(0x13e68d4f,0xdb3f2eab),LL(0x640f803f,0x1cff1ae2), LL(0xd4cee117,0xe0e7b749),LL(0x4036d909,0x8e9f275b),LL(0x8f4d4c38,0xce34e31d),LL(0xd75130fc,0x22b37f69), + LL(0xb4014604,0x83e0f1fd),LL(0x89415078,0xa8ce9919),LL(0x41792efe,0x82375b75),LL(0x97d4515b,0x4f59bf5c), LL(0x923a277d,0xac4f324f),LL(0x650f3406,0xd9bc9b7d),LL(0x8a39bc51,0xc6fa87d1),LL(0x5ccc108f,0x82588530), + LL(0x82e4c634,0x5ced3c9f),LL(0x3a4464f8,0x8efb8314),LL(0x7a1dca25,0xe706381b),LL(0x5a2a412b,0x6cd15a3c), LL(0xbfcd8fb5,0x9347a8fd),LL(0x6e54cd22,0x31db2eef),LL(0xf8d8932f,0xc4aeb11e),LL(0x344411af,0x11e7c1ed), + LL(0xdc9a151e,0x2653050c),LL(0x3bb0a859,0x9edbfc08),LL(0xfd5691e7,0x926c81c7),LL(0x6f39019a,0x9c1b2342), LL(0x7f8474b9,0x64a81c8b),LL(0x01761819,0x90657c07),LL(0x55e0375a,0x390b3331),LL(0xb6ebc47d,0xc676c626), + LL(0xb7d6dee8,0x51623247),LL(0x79659313,0x0948d927),LL(0xe9ab35ed,0x99700161),LL(0x8ddde408,0x06cc32b4), LL(0x061ef338,0x6f2fd664),LL(0xc202e9ed,0x1606fa02),LL(0x929ba99b,0x55388bc1),LL(0x1e81df69,0xc4428c5e), + LL(0xf91b0b2a,0xce2028ae),LL(0xf03dfd3f,0xce870a23),LL(0x0affe8ed,0x66ec2c87),LL(0x284d0c00,0xb205fb46), LL(0x44cefa48,0xbf5dffe7),LL(0xa19876d7,0xb6fc37a8),LL(0x08b72863,0xbecfa84c),LL(0x2576374f,0xd7205ff5), + LL(0x8887de41,0x80330d32),LL(0x869ea534,0x5de0df0c),LL(0x3c56ea17,0x13f42753),LL(0x452b1a78,0xeb1f6069), LL(0xe30ea15c,0x50474396),LL(0xc1494125,0x575816a1),LL(0xfe6bb38f,0xbe1ce55b),LL(0x96ae30f7,0xb901a948), + LL(0xd8fc3548,0xe5af0f08),LL(0xd73bfd08,0x5010b5d0),LL(0x53fe655a,0x993d2880),LL(0x1c1309fd,0x99f2630b), LL(0xb4e3b76f,0xd8677baf),LL(0xb840784b,0x14e51ddc),LL(0xbf0092ce,0x326c750c),LL(0xf528320f,0xc83d306b), + LL(0x77d4715c,0xc4456715),LL(0x6b703235,0xd30019f9),LL(0xd669e986,0x207ccb2e),LL(0xf6dbfc28,0x57c824af), LL(0xd8f92a23,0xf0eb532f),LL(0x9bb98fd2,0x4a557fd4),LL(0xc1e6199a,0xa57acea7),LL(0x8b94b1ed,0x0c663820), + LL(0xf83a9266,0x9b42be8f),LL(0x0101bd45,0xc7741c97),LL(0x07bd9ceb,0x95770c11),LL(0x8b2e0744,0x1f50250a), LL(0x1477b654,0xf762eec8),LL(0x15efe59a,0xc65b900e),LL(0x9546a897,0x88c96148),LL(0xc30b4d7c,0x7e8025b3), + LL(0x12045cf9,0xae4065ef),LL(0x9ccce8bd,0x6fcb2caf),LL(0xf2cf6525,0x1fa0ba4e),LL(0xcb72c312,0xf683125d), LL(0xe312410e,0xa01da4ea),LL(0x6cd8e830,0x67e28677),LL(0x98fb3f07,0xabd95752),LL(0xeef649a5,0x05f11e11), + LL(0x9d3472c2,0xba47faef),LL(0xc77d1345,0x3adff697),LL(0xdd15afee,0x4761fa04),LL(0xb9e69462,0x64f1f61a), LL(0x9bfb9093,0xfa691fab),LL(0xa1133dfe,0x3df8ae8f),LL(0x58cc710d,0xcd5f8967),LL(0x16c7fe79,0xfbb88d50), + LL(0xe88c50d1,0x8e011b4c),LL(0xa8771c4f,0x7532e807),LL(0xe2278ee4,0x64c78a48),LL(0x3845072a,0x0b283e83), LL(0x49e69274,0x98a6f291),LL(0x1868b21c,0xb96e9668),LL(0xb1a8908e,0x38f0adc2),LL(0x1feb829d,0x90afcff7), + LL(0x210b0856,0x9915a383),LL(0xdef04889,0xa5a80602),LL(0x7c64d509,0x800e9af9),LL(0xb8996f6f,0x81382d0b), LL(0x81927e27,0x490eba53),LL(0x4af50182,0x46c63b32),LL(0xd3ad62ce,0x784c5fd9),LL(0xf8ae8736,0xe4fa1870), + LL(0xd7466b25,0x4ec9d0bc),LL(0xdb235c65,0x84ddbe1a),LL(0x163c1688,0x5e2645ee),LL(0x00eba747,0x570bd00e), LL(0x128bfa0f,0xfa51b629),LL(0x6c1d3b68,0x92fce1bd),LL(0xb66778b1,0x3e7361dc),LL(0x5561d2bb,0x9c7d249d), + LL(0x0bbc6229,0xa40b28bf),LL(0xdfd91497,0x1c83c05e),LL(0xf083df05,0x5f9f5154),LL(0xeee66c9d,0xbac38b3c), LL(0xec0dfcfd,0xf71db7e3),LL(0x8b0a8416,0xf2ecda8e),LL(0x7812aa66,0x52fddd86),LL(0x4e6f4272,0x2896ef10), + LL(0x0fe9a745,0xff27186a),LL(0x49ca70db,0x08249fcd),LL(0x441cac49,0x7425a2e6),LL(0xece5ff57,0xf4a0885a), LL(0x7d7ead58,0x6e2cb731),LL(0x1898d104,0xf96cf7d6),LL(0x4f2c9a89,0xafe67c9d),LL(0x1c7bf5bc,0x89895a50), + LL(0x573cecfa,0xdc7cb8e5),LL(0xd15f03e6,0x66497eae),LL(0x3f084420,0x6bc0de69),LL(0xacd532b0,0x323b9b36), LL(0x0115a3c1,0xcfed390a),LL(0x2d65ca0e,0x9414c40b),LL(0x2f530c78,0x641406bd),LL(0x833438f2,0x29369a44), + LL(0x903fa271,0x996884f5),LL(0xb9da921e,0xe6da0fd2),LL(0x5db01e54,0xa6f2f269),LL(0x6876214e,0x1ee3e9bd), LL(0xe27a9497,0xa26e181c),LL(0x8e215e04,0x36d254e4),LL(0x252cabca,0x42f32a6c),LL(0x80b57614,0x99481487), + LL(0x40d9cae1,0x4c4dfe69),LL(0x11a10f09,0x05869580),LL(0x3491b64b,0xca287b57),LL(0x3fd4a53b,0x77862d5d), LL(0x50349126,0xbf94856e),LL(0x71c5268f,0x2be30bd1),LL(0xcbb650a6,0x10393f19),LL(0x778cf9fd,0x639531fe), + LL(0xb2935359,0x02556a11),LL(0xaf8c126e,0xda38aa96),LL(0x0960167f,0x47dbe6c2),LL(0x501901cd,0x37bbabb6), LL(0x2c947778,0xb6e979e0),LL(0x7a1a1dc6,0xd69a5175),LL(0x9d9faf0c,0xc3ed5095),LL(0x1d5fa5f0,0x4dd9c096), + LL(0x64f16ea8,0xa0c4304d),LL(0x7e718623,0x8b1cac16),LL(0x7c67f03e,0x0b576546),LL(0xcbd88c01,0x559cf5ad), LL(0x0e2af19a,0x074877bb),LL(0xa1228c92,0x1f717ec1),LL(0x326e8920,0x70bcb800),LL(0x4f312804,0xec6e2c5c), + LL(0x3fca4752,0x426aea7d),LL(0x2211f62a,0xf12c0949),LL(0x7be7b6b5,0x24beecd8),LL(0x36d7a27d,0xb77eaf4c), LL(0xfda78fd3,0x154c2781),LL(0x264eeabe,0x848a83b0),LL(0x4ffe2bc4,0x81287ef0),LL(0xb6b6fc2a,0x7b6d88c6), + LL(0xce417d99,0x805fb947),LL(0x8b916cc4,0x4b93dcc3),LL(0x21273323,0x72e65bb3),LL(0x6ea9886e,0xbcc1badd), LL(0x4bc5ee85,0x0e223011),LL(0xc18ee1e4,0xa561be74),LL(0xa6bcf1f1,0x762fd2d4),LL(0x95231489,0x50e6a5a4), + LL(0xa00b500b,0xca96001f),LL(0x5d7dcdf5,0x5c098cfc),LL(0x8c446a85,0xa64e2d2e),LL(0x971f3c62,0xbae9bcf1), LL(0x8435a2c5,0x4ec22683),LL(0x4bad4643,0x8ceaed6c),LL(0xccccf4e3,0xe9f8fb47),LL(0x1ce3b21e,0xbd4f3fa4), + LL(0xa3db3292,0xd79fb110),LL(0xb536c66a,0xe28a37da),LL(0x8e49e6a9,0x279ce87b),LL(0xfdcec8e3,0x70ccfe8d), LL(0x3ba464b2,0x2193e4e0),LL(0xaca9a398,0x0f39d60e),LL(0xf82c12ab,0x7d7932af),LL(0x91e7e0f7,0xd8ff50ed), + LL(0xfa28a7e0,0xea961058),LL(0x0bf5ec74,0xc726cf25),LL(0xdb229666,0xe74d55c8),LL(0xa57f5799,0x0bd9abbf), LL(0x4dfc47b3,0x7479ef07),LL(0x0c52f91d,0xd9c65fc3),LL(0x36a8bde2,0x8e0283fe),LL(0x7d4b7280,0xa32a8b5e), + LL(0x12e83233,0x6a677c61),LL(0xdcc9bf28,0x0fbb3512),LL(0x0d780f61,0x562e8ea5),LL(0x1dc4e89c,0x0db8b22b), LL(0x89be0144,0x0a6fd1fb),LL(0xca57113b,0x8c77d246),LL(0xff09c91c,0x4639075d),LL(0x5060824c,0x5b47b17f), + LL(0x16287b52,0x58aea2b0),LL(0xd0cd8eb0,0xa1343520),LL(0xc5d58573,0x6148b4d0),LL(0x291c68ae,0xdd2b6170), LL(0x1da3b3b7,0xa61b3929),LL(0x08c4ac10,0x5f946d79),LL(0x7217d583,0x4105d4a5),LL(0x25e6de5e,0x5061da3d), + LL(0xec1b4991,0x3113940d),LL(0x36f485ae,0xf12195e1),LL(0x731a2ee0,0xa7507fb2),LL(0x6e9e196e,0x95057a8e), LL(0x2e130136,0xa3c2c911),LL(0x33c60d15,0x97dfbb36),LL(0xb300ee2b,0xcaf3c581),LL(0xf4bac8b8,0x77f25d90), + LL(0x6d840cd6,0xdb1c4f98),LL(0xe634288c,0x471d62c0),LL(0xcec8a161,0x8ec2f85e),LL(0xfa6f4ae2,0x41f37cbc), LL(0x4b709985,0x6793a20f),LL(0xefa8985b,0x7a7bd33b),LL(0x938e6446,0x2c6a3fbd),LL(0x2a8d47c1,0x19042619), + LL(0xcc36975f,0x16848667),LL(0x9d5f1dfb,0x02acf168),LL(0x613baa94,0x62d41ad4),LL(0x9f684670,0xb56fbb92), LL(0xe9e40569,0xce610d0d),LL(0x35489fef,0x7b99c65f),LL(0x3df18b97,0x0c88ad1b),LL(0x5d0e9edb,0x81b7d9be), + LL(0xc716cc0a,0xd85218c0),LL(0x85691c49,0xf4b5ff90),LL(0xce356ac6,0xa4fd666b),LL(0x4b327a7a,0x17c72895), LL(0xda6be7de,0xf93d5085),LL(0x3301d34e,0xff71530e),LL(0xd8f448e8,0x4cd96442),LL(0x2ed18ffa,0x9283d331), + LL(0x2a849870,0x4d33dd99),LL(0x41576335,0xa716964b),LL(0x179be0e5,0xff5e3a9b),LL(0x83b13632,0x5b9d6b1b), LL(0xa52f313b,0x3b8bd7d4),LL(0x637a4660,0xc9dd95a0),LL(0x0b3e218f,0x30035962),LL(0xc7b28a3c,0xce1481a3), + LL(0x43228d83,0xab41b43a),LL(0x4ad63f99,0x24ae1c30),LL(0x46a51229,0x8e525f1a),LL(0xcd26d2b4,0x14af860f), LL(0x3f714aa1,0xd6baef61),LL(0xeb78795e,0xf51865ad),LL(0xe6a9d694,0xd3e21fce),LL(0x8a37b527,0x82ceb1dd), +} +}; +#endif /* _DISABLE_ECP_256R1_HARDCODED_BP_TBL_ */ +#endif /* _IPP_DATA */ + + +IPP_OWN_DEFN (const cpPrecompAP*, gfpec_precom_nistP256r1_fun, (void)) +{ + static cpPrecompAP t = { + /* w */ 7, + /* select function */ p256r1_select_ap_w7, + /* precomputed data */ (BNU_CHUNK_T*)ec_p256r1_precomputed + }; + return &t; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpp384r1precomca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpp384r1precomca.c new file mode 100644 index 0000000..a0644d0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpp384r1precomca.c @@ -0,0 +1,1524 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (P384r1 precomputed) +// +// +*/ +#include "owncp.h" +#include "pcpgfpecstuff.h" + + +#define OPERAND_BITSIZE (384) +#define LEN_P384 (BITS_BNU_CHUNK(OPERAND_BITSIZE)) + +/* P384 affine point */ +typedef struct{ + BNU_CHUNK_T X[LEN_P384]; + BNU_CHUNK_T Y[LEN_P384]; +} P384_POINT_AFFINE; + +extern const __ALIGN64 P384_POINT_AFFINE ec_p384r1_precomputed[77][16]; + + +#if defined ( _IPP_DATA ) + +#if !defined(_DISABLE_ECP_384R1_HARDCODED_BP_TBL_) +/* see ippcp_baseptbl.cpp test for generation details */ + +const __ALIGN64 P384_POINT_AFFINE ec_p384r1_precomputed[77][16] = { +/* digit=0 base_pwr=2^0 */ +{ + LL(0x49c0b528,0x3dd07566),LL(0xa0d6ce38,0x20e378e2),LL(0x541b4d6e,0x879c3afc),LL(0x59a30eff,0x64548684),LL(0x614ede2b,0x812ff723),LL(0x299e1513,0x4d3aadc2), LL(0x4b03a4fe,0x23043dad),LL(0x7bb4a9ac,0xa1bfa8bf),LL(0x2e83b050,0x8bade756),LL(0x68f4ffd9,0xc6c35219),LL(0x3969a840,0xdd800226),LL(0x5a15c5e9,0x2b78abc2), + LL(0x783dde91,0xc8229e55),LL(0x022b53f0,0x8e6c8f2e),LL(0xff9d48a1,0x3504e6f0),LL(0xf0687f50,0xda821495),LL(0x2de4b506,0x9c90a4fd),LL(0x427460c3,0xdb93b776), LL(0x3140bfda,0x42ea8463),LL(0xc2aaccd8,0xe8e8e4a8),LL(0xdc588258,0x15e4f18b),LL(0x5172bad9,0x09f1fe41),LL(0x00b0e684,0x070d4309),LL(0x123df0c2,0xe34947f7), + LL(0xc1dc4073,0x05e4dbe6),LL(0xf04f779c,0xc54ea9ff),LL(0xa170ccf0,0x6b2034e9),LL(0xd51c6c3e,0x3a48d732),LL(0x263aa470,0xe36f7e2d),LL(0xe7c1c3ac,0xd283fe68), LL(0xc04ee157,0x7e284821),LL(0x7ae0e36d,0x92d789a7),LL(0x4ef67446,0x132663c0),LL(0xd2e1d0b4,0x68012d5a),LL(0x5102b339,0xf6db68b1),LL(0x983292af,0x465465fc), + LL(0xebb68f2c,0x0aae8477),LL(0xee0421e3,0x30594ccb),LL(0x0aecac46,0x2e4f153b),LL(0x736400ad,0x078358d4),LL(0xd685d979,0xfb40f647),LL(0x34179228,0xcfeee6dd), LL(0x9b3a03b2,0x54f3e8e7),LL(0x7bfec97e,0xe74bb7f1),LL(0x4c542ad1,0x8e3e61a3),LL(0x0418c693,0x147162d3),LL(0x3820017d,0xe607b9e3),LL(0x303df319,0x50946875), + LL(0x68f1f0df,0xbb595eba),LL(0xcc873466,0xc185c0cb),LL(0x293c703b,0x7f1eb1b5),LL(0xaacc05e6,0x60db2cf5),LL(0xe2e8e4c6,0xc676b987),LL(0x1d178ffb,0xe1bb26b1), LL(0x7073fa21,0x2b694ba0),LL(0x72f34566,0x22c16e2e),LL(0x01c35b99,0x80b61b31),LL(0x982c0411,0x4b237faf),LL(0x24de236d,0xe6c59440),LL(0xe209e4a3,0x4db1c9d6), + LL(0x7d56dad8,0x7eb5c931),LL(0x39d3413a,0xcb2454b3),LL(0x580d57f2,0xec52930f),LL(0x1bdf6015,0x2a33f666),LL(0x2b02d33b,0x4f0f6a96),LL(0xf0430c40,0xc482e189), LL(0xa7b08203,0x3f62b16e),LL(0x5b3d4dce,0x739ac69d),LL(0xb79e33b0,0x8bd4bffc),LL(0x1b546f05,0x93c9e5f6),LL(0xdf21559a,0x586d8ede),LL(0xaf2a9eba,0xc9962152), + LL(0x7d69222b,0xdf13b9d1),LL(0x874774b1,0x4ce6415f),LL(0x211faa95,0x731edcf8),LL(0x659753ed,0x5f4215d1),LL(0x9db2df55,0xf893db58),LL(0x1c89025b,0x932c9f81), LL(0x7706a61e,0x0996b220),LL(0xa8641c79,0x135349d5),LL(0x50130844,0x65aad76f),LL(0x01fff780,0x0ff37c04),LL(0x693b0706,0xf57f238e),LL(0xaf6c9b3e,0xd90a16b6), + LL(0xdd9bcbba,0x23f60a05),LL(0xae9b587a,0x9e336de5),LL(0x93d7e30f,0x1c5c2e71),LL(0x4f3ddb37,0x1d9aebd6),LL(0x16b66423,0x1c7b5fe1),LL(0x349cd9b1,0x5db4f184), LL(0xe6655a44,0x0d2cfe83),LL(0xb7e55e87,0x836dbb36),LL(0x7d8686e4,0x701754bf),LL(0xa42dbba2,0xe9923263),LL(0xc48ecf0e,0x7008d943),LL(0x0d27ef61,0x3c0c6dd7), + LL(0x2353b92f,0x2f5d200e),LL(0x3fd7e4f9,0xe35d8729),LL(0xa96d745d,0x26094833),LL(0x3cbfff3f,0xdc351dc1),LL(0xdad54d6a,0x26d464c6),LL(0x53636c6a,0x5cab1d1d), LL(0xb18ec0b0,0xf2813072),LL(0xd742aa2f,0x3777e270),LL(0x033ca7c2,0x27f061c7),LL(0x68ead0d8,0xa6ecaccc),LL(0xee69a754,0x7d9429f4),LL(0x31e8f5c6,0xe7706334), + LL(0xc8d99c02,0x845539d3),LL(0xe58d6787,0x2a15a9a6),LL(0xab225fa3,0xe9f6368e),LL(0xeb32cabe,0x54a612d7),LL(0x5c4845ec,0xc2f64602),LL(0xdb1c212e,0xa91a5280), LL(0xe67b5fce,0xbb971f78),LL(0x13b9e85c,0x03a530eb),LL(0x794eabfd,0x592ac0ba),LL(0xcfd7fd1d,0x81961b8c),LL(0x47a9b8aa,0x3e03370a),LL(0xc80174e8,0x6eb995be), + LL(0xb68b8c7d,0xc7708b19),LL(0x44377aba,0x4532077c),LL(0x6cdad64f,0x0dcc6770),LL(0x147b6602,0x01b8bf56),LL(0xf0561d79,0xf8d89885),LL(0x7ba9c437,0x9c19e9fc), LL(0xbdc4ba25,0x764eb146),LL(0xac144b83,0x604fe46b),LL(0x8a77e780,0x3ce81329),LL(0xfe9e682e,0x2e070f36),LL(0x3a53287a,0x41821d0c),LL(0x3533f918,0x9aa62f9f), + LL(0x70313de0,0x3db84772),LL(0x5d970420,0xd4258cc5),LL(0xc8edfee1,0x03aced26),LL(0x35d77d83,0xf67eb422),LL(0xcf9ab45c,0x523c40db),LL(0x9c35b26d,0x627b415f), LL(0x8be55ed8,0xfacc45e4),LL(0x27aa651a,0x80d60af6),LL(0xd0e102ac,0x8c79848f),LL(0x66bed5af,0x40c64a4e),LL(0xf7942f0e,0x0329eab1),LL(0xf9c4af3d,0x0c6e430e), + LL(0x75ccbdfb,0x9b7aeb7e),LL(0xf6749a95,0xb25e28c5),LL(0x33b7d4ae,0x8a7a8e46),LL(0xd9c1bd56,0xdb5203a8),LL(0xed22df97,0xd2657265),LL(0x8cf23c94,0xb51c56e1), LL(0x6c3d812d,0xf4d39459),LL(0x87cae0c2,0xd8e88f1a),LL(0xcf4d0fe3,0x789a2a48),LL(0xfec38d60,0xb7feac2d),LL(0x3b490ec3,0x81fdbd1c),LL(0xcc6979e1,0x4617adb7), + LL(0x8f75244c,0x5865e501),LL(0x01ec909f,0xd02225fb),LL(0xb1f85c2a,0xca6b1af8),LL(0x88957166,0x44ce05ff),LL(0x5710c0c9,0x8058994c),LL(0x32f6b1ba,0x46d227c4), LL(0x03cb68e5,0xbe4b4a90),LL(0x730a99d1,0x540b8b82),LL(0xe11dbbbf,0x1ecc8585),LL(0xd9c3b691,0x72445345),LL(0x13690a74,0x647d24db),LL(0xdefbadf5,0x4429839d), + LL(0x4709f4a9,0x446ad888),LL(0xec3dabd8,0x2b7210e2),LL(0x50e07b34,0x83ccf195),LL(0x789b3075,0x59500917),LL(0xeb085993,0x0fc01fd4),LL(0x4903026b,0xfb62d26f), LL(0x6fe989bb,0x2309cc9d),LL(0x144bd586,0x61609cbd),LL(0xde06610c,0x4b23d3a0),LL(0xd898f470,0xdddc2866),LL(0x400c5797,0x8733fc41),LL(0xd0bc2716,0x5a68c6fe), + LL(0x7c33ed91,0xda6e8a7f),LL(0x0ecdd2d8,0x992afb5b),LL(0x7917652a,0x37cf6551),LL(0x2887d5ff,0x317b63ea),LL(0x13bdc3fa,0x37065f53),LL(0x435abaa1,0xa10896aa), LL(0xefabca26,0x9b21615f),LL(0x230cf00d,0xeb07ddea),LL(0x154d410f,0x914871dc),LL(0xc88ee148,0xb333bdfb),LL(0xa72d1967,0x51c305c6),LL(0x81ef2513,0x659db481), +}, +/* digit=1 base_pwr=2^5 */ +{ + LL(0x8c0a64a4,0xad852b87),LL(0x08f779d5,0x0d784cf2),LL(0xc651b1dd,0x1896b9fc),LL(0x12e8dc87,0xba8953d6),LL(0xa631cfb0,0x3a9865ba),LL(0x626b3d79,0x5dd2a4a0), LL(0x687c20bb,0x1148bc72),LL(0xf2a52bfd,0xa372dfc2),LL(0x9448fd08,0x77315f9e),LL(0x0a2377b6,0x4bcb06f0),LL(0xb35b4ff0,0x73b42725),LL(0xaabca99f,0xc510ad93), + LL(0x8e89b258,0xed4de4f8),LL(0x297a9a37,0x957c980f),LL(0xf8a0580d,0xe04b3d30),LL(0xca57b7bd,0xa309199d),LL(0x3be44d56,0xfc8e87cf),LL(0xd1b30e5c,0x4f5d5ab6), LL(0x30a9325b,0xb213c6a0),LL(0xf091bc01,0x0fd1c52d),LL(0x1090fede,0xfe51bbbf),LL(0x301fe259,0x6d97cabc),LL(0x5ecd3fe8,0x3ee12789),LL(0x9404ca51,0x888b708b), + LL(0x1d15bba7,0xdc529385),LL(0x148840d4,0x8fd61e31),LL(0x52ce08f6,0xee21597e),LL(0x81998af2,0x26de4c65),LL(0x9741eb42,0xcb4aa43c),LL(0x39c18b96,0x8bf3dccc), LL(0xd872984e,0x66e1d5da),LL(0x7dd87c39,0x507e2405),LL(0x305ee4f0,0x8545396f),LL(0xc831254b,0xf9d19ea0),LL(0x5d5680b8,0xcee0842f),LL(0x5a4434a4,0xb257fb6c), + LL(0x8258030f,0xe484fd9f),LL(0xf21af80a,0x0f4fa5ef),LL(0x7c1c3984,0xc0dd449e),LL(0x18eb5195,0xf3133891),LL(0x777a16d2,0x0336aab8),LL(0xc241720a,0xa6661cc4), LL(0xa7efacd0,0x678db970),LL(0x6461e382,0x22896865),LL(0xa022c7a4,0x5d85a0e4),LL(0x34a02a6b,0xb01f1e04),LL(0xa5791ce3,0x2657eedd),LL(0xa277ac5b,0x239dcab2), + LL(0x0b3db49f,0x7f74672d),LL(0xadc8418d,0x026275e9),LL(0xfe7cc4ee,0x18866523),LL(0xbbf36aca,0x3546317b),LL(0x80caf049,0x282313fa),LL(0x9a49d91c,0x08275c9c), LL(0xab71c2a8,0xe41f92ec),LL(0x1dd95aae,0xb5674698),LL(0xa8c4dc57,0x82347685),LL(0xa847fd1e,0xf78b23d6),LL(0xb692f868,0x8194f070),LL(0xa5f2ad03,0xfe0dee06), + LL(0x22abee22,0x61361c06),LL(0x2f42a470,0xe82123bc),LL(0x3c6ceff4,0x2c97dda1),LL(0xadcb0453,0xc2eea21f),LL(0xc78ec9ff,0xdf2a4407),LL(0x9625c735,0x54557c55), LL(0x7c440c0a,0xc693c985),LL(0xe56622c1,0x918e7324),LL(0xa5f0900f,0x8f990d2b),LL(0xc6aeeaef,0xa5b7d193),LL(0x450456e3,0x9f053d95),LL(0xad418ef1,0xdf783b1c), + LL(0xbe6c3221,0x08d2ce13),LL(0x9c0624ea,0x0f53824c),LL(0x5da25412,0xad3ca744),LL(0x93fea7cc,0xecb38a41),LL(0xcba623c4,0x8ae838d8),LL(0x41d85222,0xa75c8da5), LL(0x290807c8,0x9f48b828),LL(0x22f4ab6a,0x4df75f09),LL(0x7cab82a9,0x02511a3e),LL(0x21da354e,0xcc052dc3),LL(0x03ca73e9,0x7db6c1f3),LL(0xcb99244e,0xca1d4279), + LL(0x6ae54da1,0x9a7a5b42),LL(0x7040b022,0xcfcadaab),LL(0x3d9f0e61,0x7539438e),LL(0xe328c2e2,0x013c6719),LL(0xcccbf891,0x7f4a706c),LL(0x735a2d28,0xa335ab82), LL(0x3d984124,0x46694ef0),LL(0xc166b337,0x0e0bdfab),LL(0x423d47e4,0x9d54ed8b),LL(0xf44c9180,0x8075a8cb),LL(0xabe9b384,0xd4f5b184),LL(0x41abdc75,0x424dd00b), + LL(0xf2432ae5,0x36137ae5),LL(0xad5f443d,0x2d941abc),LL(0xee013f80,0xe31e96f2),LL(0x26385266,0x24127d20),LL(0xeae97f8b,0xd4004cf3),LL(0x62527c9e,0xbf3b201a), LL(0xcb8a2a2f,0x0cd0fee1),LL(0x8fdceb43,0x48894469),LL(0x2250a356,0xe67565ba),LL(0x2ccfe7d1,0xbbb5f8ca),LL(0xa350fe88,0x431a268f),LL(0x4e5b64d2,0xd2eb0373), + LL(0xf5d0255f,0x2ad7ae0d),LL(0xd0836f8b,0x4c04dee0),LL(0x4ec33606,0x15e3493e),LL(0x5f5eea30,0x590aefe6),LL(0x2e49ba54,0x09ef5a2e),LL(0x69a0d232,0xf56d447f), LL(0xe2510f5e,0x045f26b6),LL(0x9941083a,0xf90e1888),LL(0xe66f6b9b,0x6e6a3bd6),LL(0x251f82ae,0x637ad390),LL(0x43cdf91a,0xc40374b6),LL(0x2d6c6d88,0x62f5f8ad), + LL(0x11ff6e09,0x3da03590),LL(0xa685c0e8,0x41194047),LL(0x72b9c9ce,0x7c5323ff),LL(0x4f599daf,0xf84ce505),LL(0x5882ef19,0xd0dec10e),LL(0xde1e3a06,0x1ca3449d), LL(0xb09af84a,0x317e4e7f),LL(0x1a46bc75,0x0ebaa2c9),LL(0xc103f200,0x42b00a72),LL(0xc3719155,0x1c30ada5),LL(0x5c1cb58c,0x565bacfa),LL(0x35fa4670,0x374f516c), + LL(0x400f1d87,0x99a710b1),LL(0xee47be8b,0xec3ca386),LL(0x37168fee,0x3a00dcad),LL(0x1765a0df,0x1a69d674),LL(0x917c4909,0x85233afe),LL(0xe9bd6210,0xa3aa97a4), LL(0x310885f3,0x01010526),LL(0xb5007b08,0x21c5de3f),LL(0xee8ddabd,0x5df0c6ef),LL(0x0f6dfdef,0x5d17d45a),LL(0x251c9f79,0xb73d831a),LL(0xcb1df19d,0x397779e6), + LL(0xaf897b90,0xe97f7bcf),LL(0xe05af5ab,0xbaeecdda),LL(0x1b81fd5b,0x3c80856e),LL(0xeca4f983,0xea309bd0),LL(0x22ab76a2,0x534d8950),LL(0x6d620cfe,0xa8d63627), LL(0x6b0cfcfc,0xa13d622c),LL(0x84d9147e,0x584a9d2e),LL(0xd200ea9c,0xf23b9565),LL(0x193d5fe0,0x5f44b61a),LL(0x64b4c78d,0x430a3d3a),LL(0xc91c2b41,0x045dc592), + LL(0xbe13c1e4,0xafd6bf05),LL(0xe2ea8c26,0x256e2b98),LL(0x903983f4,0x06711695),LL(0x217c1577,0xbefbfbcb),LL(0x7324c85c,0xa214b26a),LL(0xdd98b968,0xfa2d4711), LL(0xf5d9cc93,0xed722297),LL(0xd45f8635,0x65cec50f),LL(0xc792e6b3,0x41627e4a),LL(0x228dbb39,0xc254a08c),LL(0xdf6fb26b,0x5b28fd3f),LL(0x4bdc6969,0x1471d353), + LL(0x7c20f79d,0x8600c734),LL(0xca3052ba,0x2bef7d87),LL(0x7b082112,0xc52d0bdf),LL(0x2bf757ca,0x0e649a4b),LL(0xb13eeb84,0x0c1f41bd),LL(0x8ecf9ace,0xd7115e8f), LL(0x1698ba68,0xa62f5e31),LL(0xc862e751,0x6a06d7be),LL(0xf95eee3f,0xd875c1a4),LL(0xa0b2b331,0xb25f8da3),LL(0x1827c037,0x4e52fd11),LL(0x9d72ba3c,0xacd38c2a), + LL(0xd7801edf,0x5e2cb07f),LL(0x3adc065a,0xf9fa2c0b),LL(0xd4de1f25,0xa296c53f),LL(0x838f7169,0xd408060c),LL(0x2e8a6ce7,0x68e19d7b),LL(0x94b58671,0x2cc6e06c), LL(0xc1cb6151,0x93d02a07),LL(0x35003126,0xa10fb4cf),LL(0x1aa3bc4e,0x6aa069f5),LL(0xdd09b142,0x0e44fbf0),LL(0x832e5945,0xe264f343),LL(0x1fc166ac,0x5dca2adc), +}, +/* digit=2 base_pwr=2^10 */ +{ + LL(0xa0eb176a,0x56ad7582),LL(0x6e19aa00,0x085b5a36),LL(0xe2c8b036,0x85f2c6ff),LL(0xfcd7336b,0x55c6d357),LL(0x22a46acd,0xb1ecc56d),LL(0x36277ac7,0x8e0f9767), LL(0x01878921,0xa4ed11e9),LL(0x7f4fb650,0xd3835566),LL(0x266158aa,0x5fdaba45),LL(0x89e0dff0,0x6f0b27fd),LL(0xeb6b02d0,0x32ef7ae2),LL(0xcc1b46e1,0x2f145871), + LL(0xf6e9f82c,0xe65245ce),LL(0xf6da7b5a,0x9e234dff),LL(0xbad2c806,0x5677c121),LL(0x060fcf24,0xc52dec32),LL(0x5d78ccca,0x78d07067),LL(0x1bc8b6fb,0x630002ea), LL(0xeb2e99ae,0xc5cb86be),LL(0x8551d16f,0xf13981e7),LL(0xc92a70d7,0xfbb7cdf2),LL(0xf53cd2a1,0x5a9ff1f1),LL(0x984f1139,0xfdbe6b7a),LL(0xa470a9f1,0x4403d046), + LL(0x8d040c69,0xb5b16cd4),LL(0xf95a2dc4,0xfdaba315),LL(0x61ce4704,0xc9fef349),LL(0xdbb53ed6,0x5fe87a0d),LL(0x73d70f93,0x3f0ccc79),LL(0x46724a1a,0x4601d1bc), LL(0x24f08565,0x5c4a15ae),LL(0xaa577320,0x5eda1e8a),LL(0xbb32d307,0xe31ebb35),LL(0xdc770a0e,0xcdc6f13b),LL(0xc434c2f8,0xbe3ae514),LL(0x3a0ef0d1,0x57c7fdf6), + LL(0x0f547eba,0xeb06571c),LL(0x6246c0dc,0xf292c38d),LL(0x26eed224,0xa1859667),LL(0x6100e387,0x8d9e56e4),LL(0xdc6298d1,0x470506b9),LL(0xf3350ad0,0xb19e084c), LL(0x12abd898,0x83eb62a8),LL(0x2222342c,0x70f152cc),LL(0xe1bd4a82,0xb089e880),LL(0xcaf3b3fc,0xd4d1e70f),LL(0x95ffd65c,0xd0b1ec63),LL(0x9b184ebb,0x79f27f3a), + LL(0xb92a9a13,0xc2467f9a),LL(0x6add349a,0x0c3ee8eb),LL(0x45e99644,0x59250eda),LL(0xc8a2df27,0x22ce0635),LL(0x312e8698,0xec7b643e),LL(0xaebd1587,0x334ccf2e), LL(0x6bca2900,0x0c1c6873),LL(0x09826cb1,0x00beb4c2),LL(0xcde6b725,0x12dbb586),LL(0x66dfed41,0x8d7cee6c),LL(0x014de4c8,0xc38deba2),LL(0x6248442d,0xa3ba6ec7), + LL(0xd42aeb95,0x3c686079),LL(0x85e3ea0f,0xc162e5e9),LL(0x1bbb2455,0x34cf5861),LL(0x8773b064,0x7650de1d),LL(0xd2bab35d,0x3b7562c6),LL(0x33d0741b,0x83191b44), LL(0x3a6bd9cf,0x4b604db0),LL(0x074aed21,0x87cd84db),LL(0xd4f91f9c,0x02a042d2),LL(0xe42c2a67,0x5a5d52e5),LL(0x1d5f216a,0x31291acd),LL(0xcd6203c8,0x9c3971bc), + LL(0xaa1ef488,0x92e4c851),LL(0x846528a9,0x4f91fc22),LL(0x5dc13a84,0xaa2f5d2d),LL(0xf1072d4d,0xba06aa68),LL(0x5e3a2ba9,0xf4f3b17c),LL(0x5e4dde77,0xff36a535), LL(0xae17dddc,0xe8ef143a),LL(0xa1fcd4ec,0xcc82631b),LL(0xc7d3963b,0x97db807e),LL(0x21d85ce2,0xe4aff045),LL(0x74667392,0x2d6480e6),LL(0xa55d0b3f,0xc8ce97f3), + LL(0x4c14ee0e,0xd4277731),LL(0x5e881c3f,0xda8146d1),LL(0x99f1867c,0x6b0746b0),LL(0x602dd4cc,0x1ec73d72),LL(0x38081120,0x27fae515),LL(0x2f8b2f2d,0x6a677bdc), LL(0xd60544e9,0xb924af64),LL(0xdcfc6b16,0x1439e183),LL(0x068565ac,0x4e88e9ae),LL(0xa9a4f146,0x8a3dbd25),LL(0x3f93f734,0xdb4a3e48),LL(0x8f1d33bc,0xb1971c05), + LL(0x56e8d1d1,0x059a5f98),LL(0x5c228e89,0x93a2d2ac),LL(0xfea48feb,0x0bc2c885),LL(0x7b7e188c,0x27fdf3be),LL(0xd399ba6e,0xec579682),LL(0xb3fdb5d0,0x01b12e12), LL(0x742f66b8,0x8e00941b),LL(0x478d8ee4,0xd79bc58f),LL(0x9ad50729,0xefdcd053),LL(0x3bf7bfea,0x8738666f),LL(0xd7afaff1,0x20192810),LL(0xd233e892,0x6c13e29e), + LL(0xff9ae345,0xf1f5df24),LL(0x3dd75336,0x5b8038b9),LL(0x7d2c0058,0x4746c5da),LL(0x51ddd827,0x667b07bd),LL(0x8cfdfd25,0x1b3c67c2),LL(0x06dac0d6,0x058dea06), LL(0x846c847d,0xd19040fd),LL(0x07d00ec6,0x4e5cbe59),LL(0xf1930296,0x2dc442fe),LL(0xfdb583be,0x12f89640),LL(0x5606f66b,0x0694f44f),LL(0xbbef8446,0xa9fcca6c), + LL(0x4b5cb12f,0x0c229fb9),LL(0xe930ff3e,0x0486ba2d),LL(0xb80c37f6,0x00797293),LL(0xb43335e0,0x71be9538),LL(0xda0d9cb7,0x0a604693),LL(0x745cc02e,0xb248f30d), LL(0xd9a3013b,0xfa889343),LL(0x53d6a4d4,0xd8642bc0),LL(0xe216af4b,0x9f89c21a),LL(0x41947320,0x86272900),LL(0xa269c49a,0x06f5f8f7),LL(0x727495ba,0xb3bc6d62), + LL(0xd1066e5a,0xd1de964b),LL(0x749909df,0x26ccad6b),LL(0x0de9eb07,0xbc4b55af),LL(0xa60289bf,0x9f11b995),LL(0x52392c61,0x3e59000c),LL(0x5ccff4fc,0x2128c46a), LL(0xea1e0d5e,0xb28b306b),LL(0xf5cfa47d,0x5f6ef5d5),LL(0xe3f824fa,0x59b0542c),LL(0x2f52b71c,0x70f097b2),LL(0x04c6d263,0x4483b726),LL(0x34d93e5b,0x48720be7), + LL(0x95ad5fe3,0x864bbbb7),LL(0xe8113888,0x24f15dcb),LL(0x6edb61fe,0x968fcf60),LL(0xc7e1aff9,0xbf266d1c),LL(0x3f4a5d91,0xd85a1555),LL(0xf8416ead,0xd5ff238e), LL(0x30d18538,0xcc0e75da),LL(0xb2b91d71,0xcf8a5435),LL(0x027a7e72,0xfae90226),LL(0x627c27f7,0x8e39918c),LL(0x18d454b0,0x5b8235c9),LL(0x44987803,0xef92e643), + LL(0x41ff08a5,0xb39b33ce),LL(0x32450c03,0xaf6dfb28),LL(0x7b45c3f2,0xe21947c5),LL(0xbc35bfdf,0x21b529f6),LL(0x4c5be366,0xc0a26928),LL(0xd366b21e,0x73856f3a), LL(0x3df444f7,0x53e26b66),LL(0x8b9c7ac1,0xb18a295e),LL(0x301f1f11,0x6b9df497),LL(0x7607f372,0x6e77d0e2),LL(0xc311fc01,0xd2e6b296),LL(0x7a396a6c,0x18a224e0), + LL(0x9b97dff2,0x8507c707),LL(0x983d26af,0xacc0b6a5),LL(0x2b62fdb2,0xdcc30fd6),LL(0xa68ab541,0xe9b03755),LL(0x13b6c3a0,0x270bc073),LL(0x649051fc,0x952ec4b9), LL(0xe70ceb83,0x4814c2c2),LL(0xc090e636,0x07632da5),LL(0x11fcdb0b,0x1c340686),LL(0x113f2ce9,0x7b4985d7),LL(0x16da3a7d,0xc3072347),LL(0x0d7d36ab,0xf0575aef), + LL(0x3a7f43c3,0x721bb836),LL(0xc12dca0f,0xab5b3108),LL(0x34853870,0x9cc9a789),LL(0xaf598c4f,0x78b604a7),LL(0x476f27c8,0xd370375e),LL(0x0b15cba5,0x9f0415b8), LL(0xd400dc1f,0x2bcfd9a4),LL(0x4bc62ddd,0x2a6fe03b),LL(0xa211b19f,0xb05a6464),LL(0x9990b504,0xce059d41),LL(0xd94951d3,0x011c5f87),LL(0x00d9c7b0,0x13cec089), +}, +/* digit=3 base_pwr=2^15 */ +{ + LL(0x9b280fd9,0xf365419f),LL(0x46365672,0x13e3b127),LL(0xc41880aa,0x8a91c165),LL(0xf9712fbd,0x3eb27a97),LL(0x76c55678,0xa6587aec),LL(0x02cd79cf,0x7c3a04b7), LL(0xda712eb8,0xfc878f9c),LL(0x0fca3e02,0x076e6117),LL(0xcaf6df5e,0x09a184d8),LL(0xaabcde75,0xd32bf232),LL(0x03de597a,0xf601d0de),LL(0xc5da2858,0x85d2b5fc), + LL(0xae899afa,0xb6029961),LL(0x681b6e1b,0x8eedd66e),LL(0x4df3e5b1,0x82db693b),LL(0xa0c3e357,0xb5131488),LL(0xccb2f577,0xbfb01ff3),LL(0x27a72cc3,0x28ea9470), LL(0xe39e325e,0x26170928),LL(0x84f80188,0x42d4876b),LL(0x4c872d76,0x0bec6a63),LL(0xb14d9c9f,0xa0a7cc90),LL(0x4f6c7778,0x8a32d2c4),LL(0x3b889a4c,0xe7cd346e), + LL(0x437f2efb,0xea95b5c9),LL(0x8e52ce8e,0x67d3e9b5),LL(0x88ff5455,0xfb331024),LL(0xc3101ded,0x883ec9fc),LL(0x0ca5f1f9,0x056218e2),LL(0xbccc65d8,0x4c9b2483), LL(0x123db2ec,0x13199b26),LL(0x9aff90db,0xbc1d8247),LL(0x367516f6,0xc4a9311d),LL(0xcb6e90d6,0x3b40c867),LL(0x4034415c,0xff674ff8),LL(0xa1821e7e,0x2787db45), + LL(0x609683ac,0x860ef794),LL(0x9af1c522,0xf0631ad3),LL(0x2ee7f522,0x322c8366),LL(0x54122af0,0x58ccd95f),LL(0x2bbb2d80,0x7454880c),LL(0x86d8d577,0xea173d82), LL(0xea1cc801,0xd5a3057c),LL(0xbbb2a189,0xfd08d482),LL(0xc3c512f8,0x26aac99e),LL(0x556d891e,0xc3eac036),LL(0xbf9f6112,0x866c3aa7),LL(0xc144f7e9,0x7c4c8fb9), + LL(0x8343f5dd,0xc0c1eb0f),LL(0x0126fa2d,0xa205e66a),LL(0x44fd8be4,0x37530a32),LL(0x96ab64cf,0x3c7af6f6),LL(0x76657202,0x1f043050),LL(0x828f3fb1,0x2c59d31c), LL(0xe7f5926e,0xa61cba51),LL(0x91ebeb81,0xb2de4273),LL(0xbb855476,0x976f2c34),LL(0x269e4f86,0x0c6c02c3),LL(0xe2e01fff,0xc69bed8f),LL(0x480bf7b4,0x19aa421c), + LL(0xb617206b,0xad0d24c7),LL(0x9dd13f59,0xb8be483f),LL(0x7655aa29,0xa55134ff),LL(0x7a5e217e,0xa3d10385),LL(0x3a21b295,0xfbb9eeb5),LL(0xc84a136d,0x1de71555), LL(0x0b7487b3,0x7579398e),LL(0xa14dce87,0xc6ca9575),LL(0x46452257,0x99b32e1e),LL(0x479d8f2c,0x99d54955),LL(0x930b80ef,0xc2a8a6a1),LL(0x656c850b,0x58d9db81), + LL(0xa682508c,0x6c85a0ed),LL(0xedcce6c8,0x8350b515),LL(0x6f95898d,0xa84a6765),LL(0x1e419847,0x55b0ae97),LL(0xdd6a885d,0x11115c6e),LL(0x4fb174a4,0x6f0beaf6), LL(0x815af3af,0xaaae44b2),LL(0x927a2c1c,0xcf0697b9),LL(0xd7d645ee,0x37639d62),LL(0x2effec37,0x157b7eda),LL(0x9b9c66e6,0xb55e5075),LL(0x85f597ec,0x33a66a1f), + LL(0x8f7782b6,0x171898aa),LL(0x499b3a81,0x8b7a706b),LL(0xfdb2c1ba,0xbc0e835f),LL(0x591f5aaf,0x4ee30281),LL(0xcc272c6a,0xfd71de3b),LL(0x6e93f68a,0x532800c4), LL(0x8365c576,0x35ee0804),LL(0xcd4c0221,0x6c2bcc94),LL(0x957b2ff6,0x49f37ff5),LL(0x5ec029c0,0x315d8e7e),LL(0xef324c12,0x33230602),LL(0x966b2578,0xf5847f9b), + LL(0x1453b1e1,0xc29b123e),LL(0x20059b44,0xefb07788),LL(0x9291671e,0x15554ade),LL(0x429dea37,0xeb5a1980),LL(0x6c4b867d,0xf96dacbb),LL(0xabab4d68,0x4f5563d6), LL(0xcbe76297,0xb5b0ecff),LL(0x51d6bd43,0x5a22996a),LL(0xb7e5cfc6,0x0088ec95),LL(0xfe373e05,0x4863a5a1),LL(0xc244d93d,0x42b7925b),LL(0x40117113,0x85bad135), + LL(0xf9daa551,0x86283e21),LL(0x1f696f1b,0x47fd23f8),LL(0xb9784a9a,0x7d029b1b),LL(0xa0c0acb5,0x7c7798be),LL(0x6d7c682b,0x41241c71),LL(0x1d33c2b0,0x11c6c113), LL(0x3565cf32,0x5d469ca2),LL(0xbad4bdbf,0xa949f022),LL(0xa13cf4cd,0x3d054cc2),LL(0x9e3ce279,0x13bd2166),LL(0x8a4beafc,0x01bc70e6),LL(0x8aba087e,0xb39e351d), + LL(0x82357232,0x26072a5d),LL(0x9f0fd2f1,0x3762764e),LL(0x0c16733e,0x9c5813f6),LL(0x718951d4,0xea2e0e03),LL(0x69e63818,0xae195bd4),LL(0xfa2f9a6e,0x241a4afc), LL(0x9165d59a,0x0e97519f),LL(0x58e5af1b,0x416bd373),LL(0x8197b7ec,0xc4e81128),LL(0x9c6ba0d0,0x4145be2c),LL(0x7d40b98a,0xc82cb2a1),LL(0xbccfa8b8,0xc3c28487), + LL(0xa4ee10a1,0x84aa863e),LL(0x87919ccd,0x24d805a6),LL(0xb5c399b2,0x553f3206),LL(0x3cc109bd,0x775b9217),LL(0xfe384088,0x25c01263),LL(0xd5f743cf,0xa3c4418b), LL(0xdf91f1f0,0x3d69705d),LL(0x9ebddad1,0x547d4626),LL(0x2626cebd,0x0198ab1a),LL(0x85b1afe8,0xaf8320f2),LL(0xe17e6efd,0xb9c0968c),LL(0x90215bb6,0xfedc75c2), + LL(0x143ff74d,0x8061d340),LL(0xa23aa7b3,0x59e94fc6),LL(0x914c3b81,0xf7c79a0b),LL(0x702c6ae7,0x5a836211),LL(0x718123c3,0x2570d63c),LL(0xc9f5ce3c,0x7e86d11e), LL(0x17bcce3b,0x80f03f97),LL(0x465a7446,0x073975b1),LL(0xcb357ace,0x29f66de7),LL(0x04894fb0,0xe87bd12c),LL(0x51a0b5ae,0xfc501a26),LL(0x02207a3b,0x20a3170c), + LL(0x5fe63bd4,0x3ee52d8f),LL(0x7a7da77c,0x09f8405c),LL(0x1881a757,0x35ce95c6),LL(0xc13e3707,0x0a8cf9d9),LL(0x48d2d3f6,0xe71258d5),LL(0x0bbe7c0d,0xcf4fd691), LL(0xfb9479f1,0xbd6496e1),LL(0xab8cb3a2,0x711c669a),LL(0x58cfdfb4,0xcbe85013),LL(0x59275b4a,0x655c902b),LL(0x20f722ba,0x7e0ff05b),LL(0x42b17aad,0xcb00031d), + LL(0xd54bf1a4,0x3360ff11),LL(0x5c79494b,0xbab994cb),LL(0x757d7771,0x953ad553),LL(0x68b58ed5,0xf17f14f0),LL(0x7523c422,0x22361531),LL(0x5ebf0d49,0xf0f05f96), LL(0x49182267,0x33866765),LL(0xad71c3eb,0xf87eccc1),LL(0x913d8dca,0xd7708e18),LL(0xb193eef9,0x27fe27e1),LL(0xcc45e65d,0x33376365),LL(0xd700ac20,0x599b4778), + LL(0xed068028,0xda643272),LL(0xa91fb87f,0x86b52135),LL(0x35b43943,0x23865a7c),LL(0x4606bbf2,0x6ac01588),LL(0x1559fb9a,0x9660ab72),LL(0x3ce2f1a5,0x1fcb09e7), LL(0x793d2f0c,0x62af29ab),LL(0x3aee7efc,0xad5aaef5),LL(0x44c11037,0xee9f29b7),LL(0xd36c2571,0xb2a19cf1),LL(0x65b552b7,0xb87d88e2),LL(0xbeb253d4,0xd8b4f172), +}, +/* digit=4 base_pwr=2^20 */ +{ + LL(0x72876ae8,0x31bdb483),LL(0x961ed1bf,0xe3325d98),LL(0x9b6fc64d,0x18c04246),LL(0x15786b8c,0x0dcc15fa),LL(0x8e63da4a,0x81acdb06),LL(0xdada70fb,0xd3a4b643), LL(0xdea424eb,0x46361afe),LL(0x89b92970,0xdc2d2cae),LL(0x615694e6,0xf389b61b),LL(0x872951d2,0x7036def1),LL(0xd93badc7,0x40fd3bda),LL(0x380a68d3,0x45ab6321), + LL(0xce0b5b72,0xc5cf8997),LL(0x9d7154ba,0x350adde1),LL(0x307b254a,0x8139681e),LL(0x75cd94d7,0xcc87fb57),LL(0x78684954,0x90e70274),LL(0x95ceb991,0xc4fdf4c0), LL(0x8762c84c,0x91bbc0ab),LL(0xce09e8ad,0x5e09e226),LL(0x4b93d45f,0x1cb83d70),LL(0xf541da1f,0xe2299024),LL(0x4b7ffd10,0x3eef7ce1),LL(0xb3fc1b9a,0x53ee63bb), + LL(0x81a2703a,0x23c1f744),LL(0xb9859136,0x1a5d075c),LL(0x5afd1bfd,0xa4f82c9d),LL(0xf89d76fe,0xa3d1e9a4),LL(0x75702f80,0x964f7050),LL(0xf56c089d,0x182bf349), LL(0xbe0da6e1,0xe205fa8f),LL(0x0a40f8f3,0x32905eb9),LL(0x356d4395,0x331a1004),LL(0xfdbbdfde,0x58b78901),LL(0x9ba00e71,0xa52a1597),LL(0x55497a30,0xe0092e1f), + LL(0x03682f59,0xe5004e80),LL(0xf642ac0f,0xccdb9cb7),LL(0xbd869f77,0x405f50d1),LL(0xe7ebea2c,0xecffa54d),LL(0xd87620ba,0x3354dc22),LL(0xb1c01ff4,0x01bb2988), LL(0xe16477fd,0xd9370076),LL(0x2e71ba4b,0x45303d2a),LL(0x3291e5c5,0xc0de7627),LL(0xf0a7ca55,0x5cfebd87),LL(0x9e592a30,0xde116280),LL(0xa78ebce4,0xdd26e577), + LL(0x70ee8f39,0x5562a856),LL(0x64e52a9c,0x86b0c117),LL(0x09c75b8c,0xc19f3174),LL(0x24923f80,0x21c7cc31),LL(0x8f5b291e,0xe63fe47f),LL(0x0dc08b05,0x3d6d3c05), LL(0xee0c39a1,0x58ae455e),LL(0x0ad97942,0x78bea431),LL(0x3ee3989c,0x42c7c97f),LL(0xf38759ae,0xc1b03af5),LL(0xbcf46899,0x1a673c75),LL(0x8d508c7d,0x4831b7d3), + LL(0xff1735a8,0x1e9b23b9),LL(0x2b0e4b7b,0xc3bf3d5b),LL(0x59b7721c,0xd4cc00fe),LL(0x9e2f4ceb,0xd5c36f9c),LL(0xc90af70e,0xdeca06ba),LL(0x416ee799,0x42676f12), LL(0x6f748c6f,0x0d7afe1b),LL(0x39c39d55,0x0b7a6de5),LL(0xe6eaed18,0x11e43d6e),LL(0x496087e0,0x5baf8602),LL(0xb1a3a66e,0xf833634f),LL(0x79398677,0x25098c8a), + LL(0xc552e354,0x76512d1b),LL(0x273020fd,0x2b7eb6df),LL(0x025a5f25,0xd1c73aa8),LL(0x5cbd2a40,0x2aba1929),LL(0xc88d61c6,0xb53cadc3),LL(0x098290f3,0x7e66a95e), LL(0xaf4c5073,0x72800ecb),LL(0x9dc63faf,0x81f2725e),LL(0x282ba9d1,0x14bf92a7),LL(0xbd5f1bb2,0x90629672),LL(0xa97c6c96,0x362f68eb),LL(0x7ea9d601,0xb1d3bb8b), + LL(0xd4720770,0xe141e763),LL(0xddb3b450,0xb9739e70),LL(0x96131446,0x46e6cde4),LL(0xcb6c2ef7,0x0458a5d5),LL(0x532f9fd8,0xb7747634),LL(0x16544457,0xf62d3721), LL(0xd3100854,0xbfacb4de),LL(0xb39d3f62,0x70788a31),LL(0xf22d92e4,0x9b543220),LL(0x55723258,0xaa4590f6),LL(0x01ddb8bc,0xc7b6730e),LL(0x69e1e7bd,0xae252cf8), + LL(0xa9c94429,0x73878f7f),LL(0x456ca6d8,0xb35c3bc8),LL(0xf721923a,0xd96f0b3c),LL(0xe6d44fa1,0x28d8f06c),LL(0xd5cd671a,0x94efdcdc),LL(0x3f97d481,0x0299ab93), LL(0x2fd1d324,0xb7ced6ea),LL(0x7e932ec2,0xbd683208),LL(0xcb755a6e,0x24ed31fb),LL(0xe48781d2,0xa636098e),LL(0xf0a4f297,0x8687c63c),LL(0x07478526,0xbb523440), + LL(0xdd4b8d8d,0xc618cf0d),LL(0x40dcfbfb,0x471cda86),LL(0x08882ce6,0xba0dd7ac),LL(0x6cd336e7,0x58e5d2f5),LL(0xaf096540,0xcdda8301),LL(0x3cf31600,0xf6d26846), LL(0x2197efd5,0x6150cd98),LL(0x55fb0877,0x4440fbfa),LL(0x90757f1d,0xca31871c),LL(0xbdd756c8,0xc4a1faac),LL(0xcbb8421e,0xc9d4ac1b),LL(0xb17c43be,0x3c0c2914), + LL(0x34124b56,0x2e5f7419),LL(0x4b3f02ca,0x1f223ae1),LL(0xe8336c7e,0x6345b427),LL(0xf5d0e3d0,0x92123e16),LL(0x45e79f3a,0xdaf0d14d),LL(0x6f3bd0c6,0x6aca6765), LL(0x403813f4,0xf6169fab),LL(0x334a4c59,0x31dc39c0),LL(0xd589866d,0x74c46753),LL(0x984c6a5d,0x5741511d),LL(0x97fed2d3,0xf2631287),LL(0x11614886,0x5687ca1b), + LL(0xaa6fe9ea,0x46fdb65c),LL(0x05494cd9,0xe0d48e5e),LL(0x4afbf837,0x5adef570),LL(0x1c9e2cad,0xc96ba4b9),LL(0x054a158c,0x1e8158f7),LL(0x9e38b88d,0x47be7320), LL(0x6d2993ec,0x9b99971e),LL(0xdf980ecc,0xac9b0bfa),LL(0xd96ca391,0x9da09642),LL(0x9bf4305c,0xd6710536),LL(0xa0dfafae,0x40cc1adf),LL(0xa209699b,0xe27e32f8), + LL(0x33836d4b,0x076d902a),LL(0x24afb557,0xec6c5c43),LL(0xa0516a0f,0xa0fe2d1c),LL(0x00d22ecc,0x6fb8d737),LL(0xdaf1d7b3,0xf1de9077),LL(0xd4c0c1eb,0xe4695f77), LL(0xb4375573,0x5f0fd8a8),LL(0x5e50944f,0x76238359),LL(0x635cd76f,0x65ea2f28),LL(0x25fde7b0,0x08547769),LL(0x51944304,0xb2345a2e),LL(0xa16c980d,0x86efa2f7), + LL(0x36e87d82,0xeaaddeb8),LL(0x1ffd7210,0xc12587a7),LL(0x731f6838,0xf93d2f5c),LL(0xf7097a65,0xb96594e8),LL(0xb016e8d3,0x08d6717a),LL(0x1984d825,0x9c378de8), LL(0xcb2a0c26,0x627d41e7),LL(0xc697ceb1,0x1f447501),LL(0xc760550b,0x8dc40831),LL(0x7fac97b0,0x70ad4870),LL(0x7021c170,0x5ac7f22e),LL(0x929d5931,0xa6f730e4), + LL(0xbf4d1d63,0x4ccbe2d0),LL(0x397366d5,0x32e33401),LL(0x71bda2ce,0xc83afdde),LL(0x478ed9e6,0x8dace2ac),LL(0x763fdd9e,0x3ac6a559),LL(0xb398558f,0x0ffdb04c), LL(0xafb9d6b8,0x6c1b99b2),LL(0x27f815dd,0x572ba39c),LL(0x0dbcf842,0x9de73ee7),LL(0x29267b88,0x2a3ed589),LL(0x15ebbbb3,0xd46a7fd3),LL(0xe29400c7,0xd1d01863), + LL(0xd186cb09,0x7d27d71f),LL(0x3bc213c7,0x67cb7f4e),LL(0x6075b2cf,0x418cafeb),LL(0xd93a06f7,0xc0d691e6),LL(0x9dd001b9,0xc16a9525),LL(0x026f17b9,0xa0583230), LL(0x7845900b,0x4c1041b0),LL(0x47a22aae,0x28740791),LL(0x2c1758e9,0x8d08efd6),LL(0xe6c3229a,0x9cc6f207),LL(0x082d8924,0xec69e902),LL(0xf331dfe7,0x9cfa1dea), +}, +/* digit=5 base_pwr=2^25 */ +{ + LL(0xabe7c60e,0x8ec4335f),LL(0x0a6a9fb5,0x01f198c1),LL(0x01141ab6,0x3ff96de0),LL(0x2eca98a1,0xb21acc2c),LL(0x10fdf648,0x61548490),LL(0xd1403e8b,0x2c01a99c), LL(0x6fa509d7,0xf1a35f30),LL(0xe3f08e9f,0xf7715fe3),LL(0x7fc9a752,0x89c26c07),LL(0x420d48a0,0x8d2535fe),LL(0x52fe2e73,0x80ec5ddd),LL(0x71704f39,0x041b8df0), + LL(0xc65fafce,0x498808ed),LL(0x4e806bc3,0x3676a7cc),LL(0x76c6b964,0x796e25f1),LL(0xac474261,0x1aced64b),LL(0x29a460c1,0xa62470fc),LL(0x5e751e48,0x77501dce), LL(0x6d9e3641,0xcc00053b),LL(0x9a3f5a0b,0x2b5bc4ae),LL(0x3f9ca178,0xddaccc2e),LL(0x0b80d1b0,0xad33f34a),LL(0x64642225,0x6a76df93),LL(0x778e761b,0xc145f36f), + LL(0xc10b43ae,0xdffb03ae),LL(0x9433a54b,0x39b1266e),LL(0xb19fe0db,0x4c262521),LL(0x3d5c7fee,0x0ec1e54f),LL(0x05e68e1e,0x2856510b),LL(0xdc80b8a6,0x49382c1e), LL(0x2471bdd5,0x80a50931),LL(0x81974aa9,0xe8cde185),LL(0x28235c52,0xca6112ee),LL(0x301f9653,0xd28a0eb8),LL(0xe11fcdf8,0x22b11e26),LL(0xe4d735f3,0x97e6fc5d), + LL(0xecc5f6da,0x9c66a32b),LL(0x1719ba2c,0xe4ff4043),LL(0x21e716ef,0x8c6cfab7),LL(0x96ed74e6,0x32c8fccb),LL(0x0b110c83,0x475890dd),LL(0x5cb4eefe,0xdfada95f), LL(0x93240fe1,0x9d7b89a6),LL(0x210b776f,0x6afdb2d0),LL(0xca7a7d52,0xc3f0b55b),LL(0x55d04585,0xa6e56a06),LL(0x4257acc5,0x818e221c),LL(0xfcb8d39e,0x05207b63), + LL(0xac993097,0x00fc2cfe),LL(0x76b4508c,0x4d3532b1),LL(0xa24c9add,0x0887cb17),LL(0x1a9972dd,0x2698a3e0),LL(0x26352346,0xa6f0258b),LL(0x3132792c,0xf6791599), LL(0x92f39495,0x2c3d2ebb),LL(0x3f9e744c,0xf8405894),LL(0x54833968,0xba220301),LL(0xb7936d27,0xa0c5454e),LL(0xb0ca30ec,0xdd9078e1),LL(0x5a80ec29,0xfa5e78ae), + LL(0xe68b9c0c,0xc948333b),LL(0x265cb56b,0x96e30042),LL(0x3c12bd22,0x957516bd),LL(0x0033e495,0x0d328473),LL(0xc9e2117d,0x23ff9704),LL(0xe3d28a72,0x81693f98), LL(0xf8066b56,0x7a0ba6c9),LL(0x332bdbe8,0x0e2e2242),LL(0xe9f4ce83,0x885d3c00),LL(0x8ae96eae,0x3145e5ec),LL(0x91e3d14c,0x8d4b26d4),LL(0xa9ad6fe2,0x17b5782a), + LL(0xcabd9b94,0x39062094),LL(0xc68dc31e,0x20b3910b),LL(0x0b708450,0x7d78a043),LL(0x522bd0d5,0xe113da29),LL(0xf7f27be8,0xc46fa10a),LL(0x295393a0,0x8258b2ce), LL(0x28d958d8,0xf69dc622),LL(0x991261e9,0x97b796c0),LL(0x36482f74,0xaedea55e),LL(0xeec962cd,0x6e4f2e3c),LL(0x2a06b626,0x782f495a),LL(0xfe23dd4b,0x41e32a4d), + LL(0xd8ada6cc,0x0a74da82),LL(0xbea55457,0xc6b98a3c),LL(0x57c2f5ac,0x896c26bb),LL(0x845d45e4,0x981e2f72),LL(0x7e9a7d36,0xca152b87),LL(0x7b582e8f,0x49666d45), LL(0x49fc20b9,0xea3b9bda),LL(0x7c71f153,0x5bcbc84a),LL(0x5748a654,0xd346fc5d),LL(0x622665ee,0x7ac2f217),LL(0xb6f16e27,0xbb5efe7f),LL(0x644c9dc8,0xb1810a70), + LL(0x2e4b40f5,0xe0c7946c),LL(0x7dd0f6de,0x7ef21fe5),LL(0xf81f21f3,0xc2a8cdd8),LL(0xc7ebe7a3,0x6ad73e42),LL(0x5d42a7be,0x3e8099c3),LL(0x6df16d22,0x151c6de8), LL(0x5293c104,0x214a649b),LL(0x56b4d1d3,0x769763d1),LL(0xe3bebab4,0x78b33355),LL(0xc78a804e,0xe10a70ce),LL(0x211f735e,0x985a16fe),LL(0x56605fc0,0xfd50c868), + LL(0x8f646cdb,0x16bcf31c),LL(0x059f65cc,0xe9b4b8a9),LL(0xf8971dbd,0x34a47964),LL(0xcfe021f8,0xd76f597f),LL(0x573891a6,0x23bab19c),LL(0x2645a244,0xdf670285), LL(0x4347674f,0xcc338b2c),LL(0x4c07ab3e,0x1ddd83da),LL(0x31bf757d,0xee6bb102),LL(0x032ec673,0x3f2828c5),LL(0x0cd7c8f8,0xcf0daf5f),LL(0x56eba0e0,0x1b5d1126), + LL(0xdcf6b460,0xd9b864d9),LL(0x5141107c,0xf453164e),LL(0xb019575e,0xac36ccb9),LL(0xa0040dea,0x996c97bb),LL(0x86b7899e,0xf684e539),LL(0x6b00d106,0xb11bb533), LL(0xbe831360,0x86af64aa),LL(0xb21650cc,0xbda68250),LL(0xffd7b0c6,0xc080ad14),LL(0x1a3844b9,0xa7cd582f),LL(0x4531ed01,0x3656ad08),LL(0x0af8481d,0xe12b761e), + LL(0x8532b8e4,0x94a35f64),LL(0x2227ce87,0xc2cf81f8),LL(0x7f8ce7c5,0x2c0a163f),LL(0xba5e1907,0xe9de5d0f),LL(0xc1c14999,0x0a58d572),LL(0xd934e441,0x406a3ecc), LL(0xc315d861,0xf9ef9a75),LL(0x864a28bd,0xa18abeae),LL(0x60831100,0x9e65f3c3),LL(0x2d4749cd,0xf757107b),LL(0xe8d782bb,0x8fec38eb),LL(0xbe94b2ae,0x74b4d444), + LL(0x59480d9b,0x796fc73b),LL(0xe0555912,0x9f33407a),LL(0xbcd9a4bb,0xbc428d52),LL(0x82ac0b57,0xffe3b15b),LL(0xab58a714,0x0f4cbd0a),LL(0x353b82bf,0xe3b57ece), LL(0xf77aa72a,0x67e16b95),LL(0x112a81cc,0x7b5d3eda),LL(0x8d749470,0x6795b775),LL(0xfe097d37,0x334bb4fc),LL(0x64deebc3,0x0489c0ce),LL(0x85092a86,0x0cf28e29), + LL(0x29843096,0x0cc8d80b),LL(0x6aaf8c7c,0x1f5107d7),LL(0xd1647601,0x6fb26e1a),LL(0x2e2ca8c0,0x66da08d5),LL(0x34d9e92a,0xac0a6fbb),LL(0x52203763,0x596cc78b), LL(0x8e88a471,0x145521ab),LL(0x64bde8f6,0x9bdbd482),LL(0x01b4d2ea,0x5a4d25bd),LL(0x6e7152ae,0xef1fc362),LL(0xfba5462d,0xd188389a),LL(0xf01d497c,0x951d4667), + LL(0xbcc5466f,0x294045fb),LL(0x039461a1,0x145c2078),LL(0x2a620ace,0x21c890c8),LL(0x896d0458,0x1ecc5928),LL(0xd3f9fb46,0x14130b9c),LL(0x461506dc,0xf001b3c2), LL(0x97a1b5c2,0xd90a95e2),LL(0xdf6487be,0x9e37948e),LL(0x3b165a95,0x00ce787c),LL(0xe49c468d,0x10d4f2a6),LL(0xacd25333,0xf0f26eb8),LL(0x63f3570b,0x25b513b0), + LL(0x9bc5917b,0x98d08981),LL(0x187fac5d,0x9f90885d),LL(0x7cfc13db,0x651b1828),LL(0x8655a658,0x2d606e4c),LL(0x63c91b71,0xba64d3c5),LL(0xb82a5090,0x36c7d7d8), LL(0xcab1d598,0x2d1dff02),LL(0xbe78f90d,0xa95788d7),LL(0x0ea1fe01,0x1ac2ee6b),LL(0xd5c1273c,0xc100b60c),LL(0xeae603e7,0x4496084c),LL(0x77c2fdfb,0x7fcaaf5f), +}, +/* digit=6 base_pwr=2^30 */ +{ + LL(0x6abdd842,0xe8e3225a),LL(0x3b367b02,0x8c85f18f),LL(0x9f42edb9,0xf147a421),LL(0x0d411d4d,0x6d4bc00d),LL(0x70014bb4,0xa1a13a27),LL(0xfa10166f,0xb896d97b), LL(0x0c302c6d,0xb2a1dfa7),LL(0x808a63a4,0x0a24bd5d),LL(0xf88c7359,0x8409a3a2),LL(0x347726a0,0x071f3838),LL(0x27507bb9,0xd18a551c),LL(0xb359b167,0xe0c4cc34), + LL(0x80e9d9f3,0xa44f6ef8),LL(0x1d14d618,0xaa7621e9),LL(0x0eaf6671,0xcb0e4ed8),LL(0x181514a2,0x2bf485f8),LL(0x0a2927ea,0x74670e18),LL(0x12c14645,0xe1b54616), LL(0x2a67ed61,0x4068c074),LL(0x739063ca,0xd10c7a57),LL(0x698b2816,0x391b651d),LL(0x6da14fa8,0xf310d169),LL(0xd8a578b1,0xa089be6b),LL(0x44389ad7,0xa314b3a8), + LL(0x68ed0760,0x664ff053),LL(0x38fae9fe,0xc3cdc991),LL(0xf1f30a86,0x3fe057aa),LL(0x2d08c72a,0xbae99022),LL(0x6f09e13f,0x4f5faf3f),LL(0x13d26b29,0x44461a44), LL(0xf95418ed,0xc2504c1b),LL(0xdb3ff26c,0x12766ea7),LL(0x07a22399,0x2f956e95),LL(0x5a00cdd3,0x2716e70f),LL(0x0e9fba99,0x80c02014),LL(0x0519875d,0xbe587ac3), + LL(0x70128295,0xba86aec1),LL(0xc12f35ce,0x83a09b65),LL(0x89df2f80,0x8978ff07),LL(0x97a773d5,0x85750cfd),LL(0xfc3f35f6,0x806bb730),LL(0xfed868c9,0x04503422), LL(0x86ffdbae,0xdc0fcde0),LL(0x1860f43b,0x8f4297e1),LL(0x8d3ad6cd,0xfefb7d02),LL(0x97293550,0x5c652b59),LL(0xed5cfbba,0x32e12942),LL(0x98800d22,0x06192aaf), + LL(0x3c9eb0ca,0xccf0fb42),LL(0x4aa03b40,0x8703c669),LL(0x4001af07,0x44c735a7),LL(0x2e874ed1,0x9616dd93),LL(0x474ba621,0x5c2e8520),LL(0xfa93d8b4,0xddf13cd3), LL(0x75df1b67,0xd68c9b45),LL(0x8f80d389,0x4cd24228),LL(0xc09f47fc,0x0f1a16bc),LL(0x9cd4842a,0xc414dc6a),LL(0x1f353c6d,0xbb0fa94f),LL(0x1950d073,0x40512455), + LL(0x4d2edd5f,0x935b9e24),LL(0xefb287b8,0xfeb46fb5),LL(0xf5018b92,0xa51700a3),LL(0x23864e2a,0xc328beba),LL(0x995f70c1,0x113b5c9a),LL(0xda1b5d51,0xc0b11c22), LL(0xf4a360cb,0x9b99b907),LL(0xadf0b094,0xf4ee9995),LL(0xf94b3f0e,0xf67c7cf2),LL(0xdcaf10cc,0x664a51a1),LL(0xe937a669,0xa3709ccc),LL(0x4862f098,0xea97bace), + LL(0xc942a3f5,0x51758334),LL(0x32182ba6,0x7cc01e88),LL(0x74de4fe6,0x772af257),LL(0xe9667bf8,0xb1b3c448),LL(0x8079caf6,0x71cb2738),LL(0x1d823a40,0x48890c64), LL(0x0e9edbda,0x47a5887b),LL(0xbe089e5a,0x916dfb0c),LL(0x1eb42ddf,0x3185090e),LL(0xb7f3af26,0x3c7eaa13),LL(0x9e9963b2,0x940ed8c7),LL(0x3426ac10,0xd85e77db), + LL(0x2efffd95,0x162cdf34),LL(0xd59086e8,0x92111fda),LL(0x454eb977,0x4478d114),LL(0xdea38a67,0x8ce403d8),LL(0x7435728a,0xd459633b),LL(0xa63b0504,0x3a7be4e3), LL(0x335dba3e,0x0c74066b),LL(0xc6ea6ee5,0x4e8fb1d7),LL(0xa99690ed,0x3398b588),LL(0x3ad77562,0x4949517c),LL(0xcbbb60ee,0xf9824f09),LL(0x85660bec,0x9fdcafdf), + LL(0x8875d9e8,0xf26bd861),LL(0xbea9c273,0x22e2380d),LL(0x91995508,0x5f151837),LL(0x648aa1c6,0xb97f40a6),LL(0x3977d848,0x7478f5f8),LL(0x35b57de6,0x21e876ae), LL(0xa93fc7f6,0xf620b180),LL(0x1b148996,0xf49bd07e),LL(0x1c4f60e1,0xfb085726),LL(0x7ad6b84d,0x6a6653af),LL(0x2e05b686,0x913a2d02),LL(0x407dda9a,0x94746629), + LL(0xda684157,0xb8465d16),LL(0xb238faae,0xdadde1ab),LL(0xc6b9bea8,0xe2cd45e7),LL(0x5cf413d5,0x7251d4a1),LL(0xaae1765b,0x615cea8b),LL(0x13f36885,0x75aa8318), LL(0xb8767cc1,0x7d5b0bf7),LL(0x8022968c,0xec38a8ff),LL(0x2a07faeb,0x034805b6),LL(0x33b7321e,0x916f9eb0),LL(0xc0c577ce,0x34963633),LL(0xabb8d3ce,0x8ee07efd), + LL(0xa8a29345,0xa0bd6c0d),LL(0x5d7f5ef9,0xc676b6c5),LL(0x20ad7259,0x303b6d7c),LL(0xd8fe09a7,0x06542a19),LL(0xa959014a,0x5a06653c),LL(0x5bcfe0cb,0xf45fd79a), LL(0x4e583468,0x29058d98),LL(0x0cd7afc0,0xf1bd25e6),LL(0xf7dbe54c,0x2a88246e),LL(0x35e0ef3d,0x680eaff8),LL(0x726e59b9,0x5942c97f),LL(0x8d5c0825,0x43e97139), + LL(0xa406d4d7,0x2daddb11),LL(0xa2a33d81,0xb02b5da5),LL(0x21a6aa89,0xb73ce827),LL(0x467506de,0x10919587),LL(0x428d8daa,0x0927724c),LL(0x7c17adfd,0x0ede991f), LL(0xbf7ddb3d,0x8518dab1),LL(0x2a54e1b8,0x04b091c4),LL(0x89e7a398,0x5943c37f),LL(0xe273f6f3,0x8e63f5e8),LL(0x83143d22,0xc6d0352b),LL(0xebd1628e,0x30e43182), + LL(0x9991167e,0x606658a9),LL(0x72c4b43d,0xb8773e15),LL(0xe025abce,0x6cb364cd),LL(0x0c5a653e,0xafa58e9b),LL(0x134a68bf,0xa7e35a54),LL(0xba4d9db6,0xcb831d42), LL(0xde83ef97,0xae37348e),LL(0x62ddd553,0x4ac64a6a),LL(0x715bb6b4,0x5feb5e0d),LL(0x043424b2,0xf876efae),LL(0xad91a9ef,0x7b56a291),LL(0x356f3ade,0x817c7053), + LL(0x8dff7dff,0x3d6e8d6d),LL(0xd5be4ad1,0x6b6c194a),LL(0xb6fcd08b,0x57b93f2d),LL(0xf3761f23,0x99f09948),LL(0xac8b018f,0x4062f3d6),LL(0xa27af72c,0x4b58ac05), LL(0x04d0cdfd,0x4abcc815),LL(0xbda4b02f,0xa50043e0),LL(0x27a9c083,0xe11297e5),LL(0x9779c5b3,0x2b2d8d52),LL(0xdfdecfed,0x3de3d330),LL(0xae7fc522,0xfe2487ca), + LL(0x8f7a83e5,0x1c2fc508),LL(0xb9970c92,0xa7c56233),LL(0x8bafa66f,0x949c7173),LL(0x5bbb0490,0x1e299b2d),LL(0x18fcb9e8,0xb9a79e7c),LL(0x9cb5cc50,0xe6372ce6), LL(0xf465c6aa,0x114fc628),LL(0x8cb797f6,0xc5539520),LL(0xa73ad211,0x7df94ed7),LL(0x8e0cd008,0x41eb8e1f),LL(0x004cbb0d,0xb028725a),LL(0x372c1656,0x1340186d), + LL(0x978029bb,0x4074ee27),LL(0xbae0d0c0,0xa9394bda),LL(0x72cecb4b,0xaa01d539),LL(0x9a7dd9c4,0x4b0cf127),LL(0x5bc787cf,0x3e3e3f16),LL(0x942de53f,0xdf48f7e1), LL(0x567b9d0e,0x0cc69719),LL(0x8d0d2750,0x631e3315),LL(0x92314a09,0x9fedc1e2),LL(0x14a1adcb,0x7547d226),LL(0x8662b86a,0x405561a4),LL(0xf5480b7d,0x149fa2b1), +}, +/* digit=7 base_pwr=2^35 */ +{ + LL(0xbda4aaa7,0x923d0b44),LL(0xfee29f7b,0xced14ce4),LL(0x9cf5b87d,0x1656be00),LL(0x1d61103d,0x13a37d0d),LL(0xfb652393,0x1d705880),LL(0xed712ed8,0x870a31bb), LL(0xad7c21e3,0x15ad02e6),LL(0xc36c2831,0xf004e447),LL(0xba2b3ffd,0x56aa376c),LL(0x9745443c,0xc3be2b2f),LL(0xeb903660,0x47c8a870),LL(0x6c6c192d,0x976c303e), + LL(0xf4fb80d4,0x148bd39c),LL(0xfff04e65,0x469b208c),LL(0xce548415,0xf397fbe2),LL(0x87fdde9f,0x441e5c2c),LL(0xfee9c179,0x6366b49f),LL(0x2938dc71,0x38d02bd3), LL(0xc49c5444,0x26d450fa),LL(0x2b23d3d7,0x4569f95d),LL(0x298fd876,0x5f68bf4d),LL(0x544768b6,0xe86df047),LL(0xf8491267,0x40b69a32),LL(0xf917c71a,0xcbf3adf9), + LL(0x8125489c,0x32498d4d),LL(0xa5a46ae0,0x965e8d07),LL(0xe96a7e29,0x6cea5e47),LL(0x668039ff,0xf78293a4),LL(0xf63edd32,0x62548a96),LL(0xa83e8256,0xe8e6af95), LL(0x0db6263b,0x76e60c3b),LL(0x21b3d668,0xa1ee4b06),LL(0x9e49b0b5,0xa17dbf8b),LL(0x7eb366fd,0x4b29ba12),LL(0xd29b565f,0x5e0ed781),LL(0x199b36f9,0x8cb50d53), + LL(0x29aa3150,0xa66c7035),LL(0x479e61fc,0xd038a5ab),LL(0xb5ab5410,0xdee33e96),LL(0x7c57d123,0xd068929c),LL(0xf1d6ad37,0x0839a208),LL(0x123f8178,0x8f523dab), LL(0xa67d3840,0xb3e5e524),LL(0x52eb59df,0x88bda75e),LL(0x389f2dd3,0x513a0ab7),LL(0x890bba6f,0x3197a145),LL(0x6f66bf09,0x61add75b),LL(0x4eef1722,0x5c9dfc15), + LL(0x07769b1b,0x66dc2850),LL(0x4d71fac4,0xe07fb741),LL(0xc2abbe60,0x5ae688a6),LL(0xdcbfd296,0x08ae92fa),LL(0xb43044d1,0xbc291256),LL(0x9fcdf213,0x0e1d71ed), LL(0x02485685,0xf0c5b281),LL(0xe3f68f42,0x5d3f9302),LL(0xffe4f036,0xbbbfac50),LL(0x74fdba44,0xb5b5f261),LL(0x0d746760,0x4ebe1d07),LL(0xc37f04b7,0xbb0f7812), + LL(0x6df1199d,0x810b6ab3),LL(0xb4f293b7,0xc229308b),LL(0x89897750,0x3cf838dd),LL(0x7a336c9a,0x3e391e4e),LL(0x176f89c0,0x70148337),LL(0xbc4f1e22,0x54b15bac), LL(0x2c0f2885,0x32b104f9),LL(0x67034f2a,0x2c39cefa),LL(0xbf178ac1,0xb8310437),LL(0xc99370d5,0x722299f5),LL(0x332b93a8,0x0a493cf0),LL(0xa420f719,0x00e0ab41), + LL(0x4cab24da,0xf2592543),LL(0xe7c3b9c5,0x52be9bbc),LL(0x4660d1a0,0xaab7a8b6),LL(0x9a9600f4,0x09738b81),LL(0x37de9e3c,0x58f0c866),LL(0x5db31f4f,0x0aea5cc1), LL(0x499868be,0xe480406f),LL(0xf6913a44,0x0d8fc7f0),LL(0x35f2e14e,0x72823644),LL(0x45e37a93,0xb147b310),LL(0xf15c1af7,0xb1e7aa5b),LL(0xb03e7713,0xa8685068), + LL(0x21c34c2b,0x21feb7fc),LL(0xddb0140e,0xab6a553a),LL(0x24b04e6f,0x03a65576),LL(0x342cb0ad,0x2531f186),LL(0xa24f6426,0x088c4d54),LL(0x06a873ea,0x9a0ee15c), LL(0xd33bc748,0xdbe0253f),LL(0x5db8ac9e,0xdad3339f),LL(0x73e65901,0xeaaf3681),LL(0xccbfa504,0x71f1fab2),LL(0x4b0e163e,0xb7b84522),LL(0x3c779f3b,0xe0fca837), + LL(0x46baf373,0x710988eb),LL(0xb57d5018,0x8cceb935),LL(0xa45fdf17,0x1864603f),LL(0xef48e6d0,0x3dcaae73),LL(0x590322c5,0xadd9420b),LL(0x9b135f67,0x947783e3), LL(0x8bf5049f,0xfde76368),LL(0x2caa4023,0xf00e4c18),LL(0xd355b3d6,0x4d3b0f23),LL(0x20d5799d,0x181fabcc),LL(0xab2ad0af,0x29499b40),LL(0xf9a938aa,0xf6e66328), + LL(0xbcbe922d,0xcd7b3c42),LL(0x95dd1a5c,0x2fe02b3b),LL(0x24ef5c38,0xeb66bcbd),LL(0xe579c309,0x7edcc21c),LL(0x16f6c900,0x7b19d491),LL(0xb6317c2c,0x36019ecd), LL(0x91d9001c,0x554ba553),LL(0x14f31e44,0xa5e30b98),LL(0xffda4032,0x3d1fe33b),LL(0x2306675c,0x5dfec478),LL(0x000c91e7,0xbe59305e),LL(0x25a6b879,0x3c4e52a3), + LL(0xc5ea88ac,0x02fcc14e),LL(0x56d093b6,0xca29bb6d),LL(0x0e6fe94d,0x876aeda9),LL(0xd7225a9d,0xfa11a142),LL(0x3d03fed8,0xfea3ca05),LL(0xc54d5962,0x435854c6), LL(0x54a6dfd5,0xd7707374),LL(0xa3e55d02,0xb8960017),LL(0x04d65c3e,0xd4015a0c),LL(0xe98a1204,0x397f93d1),LL(0x5f3ed850,0xb0efa2e5),LL(0x8a3ec67b,0x18f24469), + LL(0xd62cd9f9,0xa35802f5),LL(0x4148436e,0x0ca9c15d),LL(0x472b9d21,0x261a991d),LL(0xa2f8e875,0xd81a1ed6),LL(0x699b6d63,0x942f213a),LL(0x0ae57758,0x041a12fc), LL(0xbd70aabb,0x61191c82),LL(0xee4c23b3,0x3776eb8b),LL(0x52511222,0xabe23e86),LL(0x30dabb91,0x66dd967d),LL(0x7ed27424,0x77650c59),LL(0xab25a050,0x08ea2ebd), + LL(0xb6cb5a02,0xa410ba3a),LL(0xd07c5c6b,0x6eb40d15),LL(0x07dcc811,0x0de81e91),LL(0x2631b7af,0x996f46eb),LL(0x5b7a22f3,0x5a350ba7),LL(0x634159af,0xf42b24e7), LL(0xc30952fd,0x07bae0ab),LL(0xd644e0b0,0x3488cda2),LL(0xe2111e12,0x23ae40d0),LL(0xc80cdb56,0x650af54e),LL(0x7d4aa2a8,0x0f33a30b),LL(0x442a00e8,0x4e8d3e98), + LL(0x59a8bc95,0xa624ab37),LL(0x1c971228,0x4b7e3fa6),LL(0x73aa694b,0xe8229c42),LL(0x779288ab,0x0cc31029),LL(0x57575e0e,0xf8eff30f),LL(0x7d52803e,0xee5e0194), LL(0x8a78f632,0x32d87e55),LL(0xe454904e,0x48a06031),LL(0x16c6e626,0xaa2cb8dd),LL(0x2c140452,0xadd098ac),LL(0x2d3031b1,0xd25f285d),LL(0x75b59543,0xfb5fbbe1), + LL(0xd7a21503,0x2297041f),LL(0x657f03f0,0xfe7738c2),LL(0x168fa34a,0x994a8deb),LL(0xa53c4fdb,0x0c772e02),LL(0x50124cd3,0x67f835d1),LL(0x6993cbbe,0x0e0d2635), LL(0x5257f11d,0x9857ed84),LL(0xac556942,0xdc23a728),LL(0xdeb32a7f,0xf0e1bb29),LL(0xee0d70f4,0xb8c3c43f),LL(0xc60ad214,0xc294b0ef),LL(0x679067ca,0xa4d438dc), + LL(0x05c755e4,0x520b0bb9),LL(0xf89f0048,0xa2c2c59b),LL(0x73c23975,0x85c1c73a),LL(0x783aabba,0x6e4dec49),LL(0xb0463155,0x69f0c69b),LL(0x9c97b17b,0x61a42b94), LL(0x45d331a3,0x55af24a9),LL(0xf5fe81fd,0x4b0e63f8),LL(0x708671c4,0x4034283d),LL(0x5fd9001a,0x200ddab3),LL(0x342eaf3b,0xe45f28e4),LL(0x1ba936c4,0x3e8375b4), +}, +/* digit=8 base_pwr=2^40 */ +{ + LL(0xec074aea,0xb083ba6a),LL(0x7f0b505b,0x46fac5ef),LL(0xfc82dc03,0x95367a21),LL(0x9d3679d8,0x227be26a),LL(0x7e9724c0,0xc70f6d6c),LL(0xf9ebec0f,0xcd68c757), LL(0x8ff321b2,0x29dde03e),LL(0x031939dc,0xf84ad7bb),LL(0x0f602f4b,0xdaf590c9),LL(0x49722bc4,0x17c52888),LL(0x089b22b6,0xa8df99f0),LL(0xe59b9b90,0xc21bc5d4), + LL(0x51c2bb65,0x6f577529),LL(0x4b874bdb,0x4a0c1c28),LL(0x78b96c6d,0x19a18427),LL(0x2f593505,0xa674f992),LL(0x6b7209d6,0x5abeeec4),LL(0x47cf5fff,0x42d15d01), LL(0xb49e3b4e,0xe24509b7),LL(0x639ee6e8,0x81be939c),LL(0x5761e8e3,0x7f7daf59),LL(0xd420a288,0xed5cfcb8),LL(0x7a0ff696,0x365b29eb),LL(0x99a1ac8f,0x7d146805), + LL(0x8a31973f,0x4936c6a0),LL(0x83b8c205,0x54d442fa),LL(0x5714f2c6,0x03aee8b4),LL(0x3f5ac25a,0x139bd692),LL(0xb5b33794,0x6a2e42ba),LL(0x3ff7bba9,0x50fa1164), LL(0xf7e2c099,0xb61d8643),LL(0xbd5c6637,0x2366c993),LL(0x72eb77fa,0x62110e14),LL(0x3b99c635,0x3d5b96f1),LL(0xf674c9f2,0x956ecf64),LL(0xef2ba250,0xc56f7e51), + LL(0xadcdaa68,0x9ee1ec3a),LL(0xdcbb6548,0xd98c498f),LL(0x88102ac0,0x32b97375),LL(0xc08527f4,0xdd296cf9),LL(0xfae3dfbe,0xb74f8145),LL(0x6cd7cc4f,0x84131eb9), LL(0x927ff15b,0xa0f2fe7a),LL(0xeee1a4b4,0x6b0ade4d),LL(0x0eeb90a7,0x6e7df2d4),LL(0xbe4de684,0xe2f46e20),LL(0x3fdd06bc,0xcd28feba),LL(0xe6d6d9f6,0x8e4205ae), + LL(0xff602c1b,0x246ffcb6),LL(0x6e1258e0,0x1e1a1d74),LL(0x250e6676,0xb4b43ae2),LL(0x924ce5fa,0x95c1b5f0),LL(0xebd8c776,0x2555795b),LL(0xacd9d9d0,0x4c1e03dc), LL(0x9ce90c61,0xe1d74aa6),LL(0xa9c4b9f9,0xa88c0769),LL(0x95af56de,0xdf74df27),LL(0xb331b6f4,0x24b10c5f),LL(0x6559e137,0xb0a6df9a),LL(0xc06637f2,0x6acc1b8f), + LL(0xf4a59550,0x9ad73168),LL(0x8409d7af,0x2a488d69),LL(0x30b3a5f4,0x9e946c2d),LL(0xc89723d5,0x7bc4ced7),LL(0xbd31f607,0x79f514a1),LL(0x3a274341,0x7d493f59), LL(0xd45d55d1,0x7eb01027),LL(0x14735d1e,0x63f76909),LL(0x35be21cb,0x718d760f),LL(0x7c793331,0xba58160b),LL(0x8d30d29e,0xfbc0ce1f),LL(0xc6f4b03a,0x4645c0c2), + LL(0x34b4e381,0xbd8c0868),LL(0x30dff271,0x278cacc7),LL(0x02459389,0x87ed12de),LL(0xdef840b6,0x3f7d98ff),LL(0x5f0b56e1,0x71eee0cb),LL(0xd8d9be87,0x462b5c9b), LL(0x98094c0f,0xe6b50b5a),LL(0x508c67ce,0x26f3b274),LL(0x7cb1f992,0x418b1bd1),LL(0x4ff11827,0x607818ed),LL(0x9b042c63,0xe630d93a),LL(0x8c779ae3,0x38b9eff3), + LL(0xf78d33ef,0x35d47426),LL(0x8440c42c,0x4af25db2),LL(0x2e91bf5e,0xbd6a15e2),LL(0xc08b6b1a,0xe366a84c),LL(0x55b97de8,0x759c122f),LL(0x08a03f29,0xecec558f), LL(0xea9d2060,0xdcc9fca2),LL(0x9f361fe1,0xb3e49b8e),LL(0x9b59cd04,0xdeae3902),LL(0x6f5e5bd4,0xf532ede0),LL(0x36099f4d,0x84fbeeb9),LL(0x088d2052,0x73576b1f), + LL(0x729c5431,0xe8767d36),LL(0xbb94642c,0xa8bd07c0),LL(0x58f2e5b2,0x0c11fc8e),LL(0x547533fe,0xd8912d48),LL(0x230d91fb,0xaae14f5e),LL(0x676dfba0,0xc122051a), LL(0x5ea93078,0x9ed4501f),LL(0xbd4bee0a,0x2758515c),LL(0x94d21f52,0x97733c6c),LL(0x4ad306a2,0x139bcd6d),LL(0x298123cc,0x0aaecbdc),LL(0x1cb7c7c9,0x102b8a31), + LL(0x139a9f83,0x2b85dd03),LL(0xf55cbff9,0x02740ec2),LL(0xdc3bd5f6,0xdca3616a),LL(0xa5da9b2c,0x52843de5),LL(0x1e22c1cb,0x7ee7890a),LL(0x4aee5c76,0xd5cf1511), LL(0xc7234c24,0x86d70af0),LL(0x6b6a709f,0x1735378b),LL(0x423f32e5,0x9989ed21),LL(0xe739b1fd,0x4675a4ff),LL(0x304ce529,0x873edb62),LL(0x7f1db13e,0x60b6624a), + LL(0xfaf46675,0x22a28e59),LL(0x10a31e7d,0x10757308),LL(0x2b4c2f4f,0xc7eeac84),LL(0xb5ef5184,0xba370148),LL(0x8732e055,0x4a5a2866),LL(0xb887c36f,0x14b8dcdc), LL(0x433f093d,0xdba8c85c),LL(0x1c9a201c,0x73df549d),LL(0x70f927d8,0x69aa0d7b),LL(0xd7d2493a,0xfa3a8685),LL(0x0a7f4013,0x6f48a255),LL(0xdd393067,0xd20c8bf9), + LL(0x6b512bc6,0xee43828c),LL(0x50b91e60,0xf73dc9f5),LL(0xf5dbde6b,0x68f23f30),LL(0xddd15e00,0xaf2fe9e3),LL(0x86578d49,0xfbf34dae),LL(0x6c130010,0x68979655), LL(0x09942897,0x137a5fc3),LL(0x9959f06d,0xff1f0bfe),LL(0xbd7ee14b,0x2dd0a04a),LL(0xe54e2161,0x59c46072),LL(0xea7518ad,0xf470bdae),LL(0x40c471cd,0xce556e43), + LL(0x81625e78,0x4ec874ea),LL(0x3fbe9267,0x8b8d8b5a),LL(0x9421ec2f,0xa3d9d164),LL(0x880ea295,0x490e92d9),LL(0xd8f3b6da,0x745d1edc),LL(0x8f18ba03,0x0116628b), LL(0x834eadce,0x0ff6bce0),LL(0x000827f7,0x464697f2),LL(0x498d724e,0x08dccf84),LL(0x1e88304c,0x7896d365),LL(0x135e3622,0xe63ebcce),LL(0xdc007521,0xfb942e8e), + LL(0x829ee0c7,0x6b97dae4),LL(0xf274bc7d,0xcaaad7e3),LL(0x512f7d86,0x87bee424),LL(0x3f7b0dde,0xb5a03e5b),LL(0xf9fca5a8,0xc29db9c1),LL(0xede64af8,0x538a6f8b), LL(0x8b3c715f,0x0e518ba8),LL(0x3cb5b861,0xc1cb4c31),LL(0xc76dbfa0,0x5cd61604),LL(0xffdac22e,0xc557e50d),LL(0xf67b53e8,0x7892fc51),LL(0xea7c141c,0xa42f85c1), + LL(0xa3688621,0xbb155a66),LL(0xf91b52a3,0xed2fd7cd),LL(0xea20cb88,0x52798f5d),LL(0x373f7dd8,0x069ce105),LL(0x8ca78f6b,0xf9392ec7),LL(0x6b335169,0xb3013e25), LL(0x6b11715c,0x1d92f800),LL(0xff9dc464,0xadd4050e),LL(0x8465b84a,0x2ac22659),LL(0x465b2bd6,0x2729d646),LL(0xe4eff9dd,0x6202344a),LL(0xcd9b90b9,0x51f3198f), + LL(0x22b25f2e,0x2ba8c790),LL(0x1af0f4a8,0xf4670a51),LL(0x2fc2451e,0x6842f36a),LL(0xbb91e1e3,0xfc5c9558),LL(0xc3ead762,0x035d1dfc),LL(0x031e5556,0x3d0721cb), LL(0x3af18a2e,0x3af0cc81),LL(0xbd11a363,0x7888cee2),LL(0x6ade1d12,0x80c3de0a),LL(0x93b2dcb5,0xe8c3a5bd),LL(0x90a2214d,0xe3adbd7c),LL(0x1192948d,0xfe8646d5), +}, +/* digit=9 base_pwr=2^45 */ +{ + LL(0xba958101,0x65026297),LL(0x8eef151d,0xbeb4adf9),LL(0x60c8bbf7,0x623763a4),LL(0x8b2a7120,0xfa8f5ad7),LL(0x085497e7,0xfd744bdf),LL(0x2ba35618,0xf9b6f97e), LL(0xf8a15e86,0x0cebfe9d),LL(0x29576088,0x47a6d013),LL(0xfcf19627,0x655817a3),LL(0xc2f11261,0x30ab44f7),LL(0xddf2c850,0xbb001c9d),LL(0x073260c3,0xb45c7eff), + LL(0xe2319f38,0xfa7d1236),LL(0xa551d3fe,0x9ba1a1c0),LL(0xbeb1282b,0x9ea27288),LL(0x07fee8a9,0x1c069efa),LL(0x5870fee9,0x5749c7b5),LL(0xafcec6fa,0xbedca76f), LL(0x4c63c5e2,0xa3f8f1b1),LL(0x94758ac3,0xaa1bb156),LL(0xb59dc06e,0x753329a9),LL(0x98a92c38,0xfa8e5f5b),LL(0x3c2b4662,0x6b6f46fd),LL(0xec04c6c6,0x716f41a1), + LL(0x59b11c7a,0x0430af77),LL(0xd4f47aca,0xf71cc5b1),LL(0x12e9190f,0xe1a7905f),LL(0x12db9e14,0x1c689b70),LL(0x0abaeeac,0x6bdd3dc9),LL(0x504f0319,0x97f1c244), LL(0xa7a54b51,0x874afd61),LL(0xe3d979b8,0xd4604ecb),LL(0xebf4aab1,0x0d33eee1),LL(0x1aa49fe6,0xa3631cac),LL(0xf2217cfd,0x0d8340fb),LL(0x423b7e77,0xf6373284), + LL(0x9563e3bb,0x78a53b5a),LL(0x86af355c,0x19c75eb2),LL(0x019a6f8e,0x3520f427),LL(0xdc3ad0ba,0xde6fcad6),LL(0x79745b7c,0xfec96e4f),LL(0xb133f2dc,0x5e566bbd), LL(0x26561be7,0x50088a2b),LL(0xc5fddfc7,0x16275b4c),LL(0x23ae4b9d,0xf21332ff),LL(0x85246712,0x8cbc659e),LL(0xf50b515e,0x27fa9c8d),LL(0x494ac8b7,0x25ecf745), + LL(0x641e3bd1,0x437b7224),LL(0x80a58460,0x84e39f79),LL(0x09759523,0x68e52927),LL(0xe77f5904,0x0176a3ac),LL(0xe151e242,0xde92fb15),LL(0xeb1438d4,0x79965c9a), LL(0x596700b9,0x318a810a),LL(0xc2198cbe,0xa8a6ec57),LL(0xbf030fd2,0xd7709aaa),LL(0x72f5d326,0xb4320234),LL(0xb03bce50,0xc9945214),LL(0xd4ecba09,0x0bc06d9b), + LL(0x96fffb94,0xa1972c21),LL(0x99d7633b,0xbe040930),LL(0x7e23d66e,0xb116ff40),LL(0x949a19f1,0xcb12b2bb),LL(0x79e49e91,0x75df10ee),LL(0x4890bcf4,0xa3bf9076), LL(0x09a30252,0xcbaa76a6),LL(0x0ee5728e,0x17c224a9),LL(0xf4f3f4cb,0xcbc56e5c),LL(0x4fe868a5,0x8a07110f),LL(0x25e110a2,0x23289f21),LL(0xd7693c45,0x0289c12b), + LL(0xf184c91f,0x06ba6db1),LL(0x45fd0382,0x3c0a348a),LL(0x4434b527,0x0d535b6e),LL(0x692bae0b,0x7bbfa2c8),LL(0xbe7fe51c,0x5c59a08e),LL(0x36e80cb8,0xbaa7d2be), LL(0xbed3cae8,0x8a42d8d1),LL(0x15ff4962,0xd9e0bc0d),LL(0x644c75ff,0xe51fce93),LL(0xb9392d63,0x40222561),LL(0x023b4787,0x8ab1d286),LL(0xa1b3190d,0xfa85c220), + LL(0x2b0c535b,0x29864753),LL(0x70506296,0x90dd6953),LL(0x216ab9ac,0x038cd6b4),LL(0xbe12d76a,0x3df9b7b7),LL(0x5f347bdb,0x13f4d978),LL(0x13e94489,0x222c5c9c), LL(0x2680dc64,0x5f8e796f),LL(0x58352417,0x120e7cb7),LL(0xd10740b8,0x254b5d8a),LL(0x5337dee6,0xc38b8efb),LL(0x94f02247,0xf688c2e1),LL(0x6c25bc4c,0x7b5c75f3), + LL(0x938e2480,0x1213a2fc),LL(0xed7b85a1,0x2dff0ea8),LL(0x0cc9619f,0x4f280c74),LL(0x66678817,0x3a215bc5),LL(0x327985a8,0xa51602ca),LL(0x10f02615,0xd6916920), LL(0x3dc4f2b6,0xd01274aa),LL(0x3ff804d1,0x40feccd7),LL(0xa2100fa4,0xe508a27c),LL(0xd90cc776,0x37cc757e),LL(0xb74827c4,0xaec4884b),LL(0x4ac34a27,0x0c904582), + LL(0xf761baaa,0x6ad3f9d3),LL(0xa5ae6088,0x3936d0d9),LL(0x83a0dd41,0x54d5bff0),LL(0x70cfa7fe,0x418c4b20),LL(0x892317fc,0xdc4d4742),LL(0x265359a1,0x1248e2c0), LL(0x2120ad82,0xa14ee83c),LL(0x89260c26,0x048b7b27),LL(0xb5fe2775,0x637e71d5),LL(0xf3e08f62,0x849213d5),LL(0x5a9faeb1,0x8a8a2da8),LL(0xef96612f,0x57047761), + LL(0x027a3ba4,0x7608efa7),LL(0xcf450422,0x14cb67ad),LL(0x5330ef70,0xc55a7aae),LL(0x64901358,0x3f18a78d),LL(0xfeb668df,0x264d3a5c),LL(0xbcf7c23b,0xdc0569c0), LL(0x6ae18907,0x93af8a9b),LL(0x24b8e961,0x55dd3e3a),LL(0x36171692,0x1c3fcd34),LL(0x2f1f7ca5,0xfc0ab80c),LL(0x564878a7,0x0dadc166),LL(0x000d81f3,0x7d20d34b), + LL(0xb19231b4,0x0e8d03f8),LL(0xdcda852b,0xe4498dbd),LL(0x31a8dddb,0x9963a6bb),LL(0xd949c6cf,0x80bfba33),LL(0x1b6e1ab7,0x72e48395),LL(0x82d7301d,0x35a2a802), LL(0x01edd784,0xcfce9b91),LL(0xcd9bcb9c,0x794ab50e),LL(0xf465c3ca,0xe1f90e0c),LL(0x902b1b57,0xb7ded4b5),LL(0x8a8c27b4,0xe593dafc),LL(0x6f940ca0,0xa80faad4), + LL(0x421e65d3,0x5396fd32),LL(0x15e91d37,0x6c826fe2),LL(0x3cc3d9cb,0x14c80f4a),LL(0x43028c7f,0xdb760a13),LL(0xce1a3e42,0xd7a541bd),LL(0xb81193f1,0x4726a5b7), LL(0xf004d745,0x5c96916d),LL(0xad031ec3,0xc86961c0),LL(0xb8d28a4f,0xb5a73c58),LL(0x7ed16163,0x9a7bcb0e),LL(0x2fbd18f5,0x3f6871f3),LL(0x42155e84,0xebef020a), + LL(0x4cdd5d8c,0xce6c810c),LL(0x078c9bc1,0x9fc290a4),LL(0x96d47e78,0x84debffd),LL(0x143590fe,0xf291d301),LL(0x69111f3a,0x7cc5c36d),LL(0x46311370,0x3a2cdc42), LL(0x9e6f14a7,0xd3d8bb98),LL(0x481e22db,0xfeb97d41),LL(0x462748fe,0xe1e20cfb),LL(0xa2eaf3c7,0xcf8f5dd0),LL(0x302a52d4,0xb4f9acd4),LL(0x8d715c96,0x1c154dcd), + LL(0xb13ddc9c,0x5f492236),LL(0xbabe466c,0xc930fb0e),LL(0x5c017d52,0x21daa0b9),LL(0xea9a6c9b,0xc182a975),LL(0x4fd839af,0x45d3d6c0),LL(0x3d89b744,0x9e4ce10a), LL(0x3a947f16,0x906af540),LL(0xa377f4f3,0xa0f8e1b7),LL(0xe343d7c1,0x268360cc),LL(0xbddbeb1a,0x66fe9eb7),LL(0xc3bafefc,0xf5c80368),LL(0x6664d04e,0x0e183ee0), + LL(0xfda8520e,0x36c9dbbe),LL(0x6ae3ea98,0x573507ce),LL(0x96a8f9f1,0x1ab38db6),LL(0x6b01e6bc,0xe031d235),LL(0x8afc4ada,0x10466ae6),LL(0xed9c44e4,0x3b35df41), LL(0xc7bd99e8,0x61272c12),LL(0x805afd79,0x6a4ae7b4),LL(0x0ecc49eb,0xf4c47a91),LL(0xcbe84d5c,0xeb95dfec),LL(0x8ee497d7,0x43f3b71c),LL(0x4c6fece4,0x2547af52), +}, +/* digit=10 base_pwr=2^50 */ +{ + LL(0xced45039,0xe323ed0c),LL(0xa90aa713,0x04ce0b67),LL(0xe8d68e4e,0x9c092f06),LL(0xd0742e5d,0xd8f5555a),LL(0x00d3df92,0xe2d175bf),LL(0x4f71aeab,0x8ca55f15), LL(0x642d391d,0xd1762d72),LL(0xaec466bd,0x0dfdd3c2),LL(0x6281f2a7,0x2caacb4c),LL(0x3603e53a,0x635ba470),LL(0x49fecf29,0x94a9811d),LL(0x466bf361,0x3a42cf09), + LL(0x11bccf2f,0x0f05710b),LL(0x7aec1bc6,0x7113085a),LL(0x46b8d0e2,0x137da67a),LL(0x698b78cc,0x454b89fc),LL(0x258a9393,0xf2a6e1de),LL(0x16488e69,0x5f1804e7), LL(0x15b3bf35,0x7c6c5502),LL(0xb05c2ec1,0x3b0e09a5),LL(0x92f15247,0x4b9de30e),LL(0x27e70a0a,0x09d4ca93),LL(0x0d149363,0x9c8b1634),LL(0xce642137,0x54a8287c), + LL(0x346a764e,0x49ecd6f3),LL(0x4105e657,0xe46847f1),LL(0x7550f608,0xce9cb2b5),LL(0xf4cf062e,0x45f1a1f7),LL(0x2c27d38a,0xcdb19a11),LL(0x84e50b19,0x36d375b2), LL(0x0dba6405,0xf4369154),LL(0x040354dc,0x4c9dc863),LL(0xa24d09cf,0x7229e70e),LL(0x7cf6831b,0xe72aa86c),LL(0x25392838,0x487fb684),LL(0x430b9b47,0xe88bed04), + LL(0x666fa8a8,0xf8a8499b),LL(0x71bba84a,0xd0f94015),LL(0x515e1328,0xb85e1b1d),LL(0xa941e788,0x88a2636b),LL(0x2b5dd8d8,0xa045241d),LL(0x332f0350,0x161be476), LL(0xa18fac6b,0x96c4b205),LL(0x73fc5337,0x5cbe8d5e),LL(0xd00b6029,0x6fc33fc6),LL(0x89aa3b79,0x07a914ee),LL(0xa4d4dd00,0x35353eb7),LL(0x673e8956,0xc026bdc0), + LL(0x67103852,0xccda3721),LL(0x0c54de53,0xf78d2247),LL(0xafa44aa5,0xebd16036),LL(0x64a24ab9,0x7b880248),LL(0x1c2bc78e,0x86b38e96),LL(0x8d63b295,0xd0aa0d05), LL(0xc62fcbf1,0x24912955),LL(0xb9ac435c,0x77a68156),LL(0x1b360b26,0x432401c7),LL(0x4c58ef8c,0x091f19f3),LL(0x83d46c9d,0x3a4a61f4),LL(0xe8d616cd,0xad0e5c72), + LL(0xcdaa6831,0x33b029bc),LL(0x4c1f9ced,0x2548552d),LL(0xdece1c8c,0x35f1a002),LL(0xacc23aa5,0xc6b87fd7),LL(0xbab029a4,0x0b8bb275),LL(0x30bfb42d,0xf07bc067), LL(0x1f69ce9d,0x1688ff5d),LL(0xdb10585e,0xeedb7b5e),LL(0xd432c197,0xb7a88cf0),LL(0x015a350a,0x20731bdd),LL(0x63223f5c,0x5fa18354),LL(0x8024693f,0xe392e131), + LL(0x0154731e,0xdfaeea0a),LL(0xe15a0388,0x9e53419d),LL(0x25a992c8,0x2ad6a83c),LL(0xe125501a,0xa2ba020f),LL(0xd4dd04dc,0x894ebaf8),LL(0x50765559,0xd48cb958), LL(0x80dec92b,0xf9b58d09),LL(0x9da299d7,0x2a0e1165),LL(0xefe9cb11,0x3c081853),LL(0xa511c5e0,0xb9f3b702),LL(0x70486180,0xa8f7a25f),LL(0x591b3e2f,0xc0358b25), + LL(0xeaddf274,0xf36e8398),LL(0x6a5e4ddb,0xe41553a1),LL(0x4efc5b0c,0x36ab0746),LL(0xd316c434,0xb211e59a),LL(0x16ccf839,0x2515ec9f),LL(0x03dc6a07,0x6ecb7465), LL(0xc65c1b07,0x842b7275),LL(0x35750ab6,0xf7ceeec5),LL(0xcef5255d,0x967d711c),LL(0x5108cb92,0xcd3bfb07),LL(0xec1b9740,0xe50c0d8a),LL(0x1a9e6308,0x9e8d5661), + LL(0xccb8a36a,0x84f8ea13),LL(0xa05709a2,0x5f7aeeff),LL(0x60574f37,0x4942d04e),LL(0xe048b400,0x855b13e9),LL(0xa6b59c09,0x747e4067),LL(0x074d3990,0xc349fb05), LL(0xec2c7e03,0x398e6afa),LL(0xec2d5a4c,0xce361865),LL(0xb6f57d22,0xfc04bf8e),LL(0x759ce6c6,0xf0e0b84c),LL(0x5ee7e528,0xb6514123),LL(0xf5c0f9b9,0x8ca144bd), + LL(0xf6a58536,0xc1370dae),LL(0xc56b0ae9,0x6f2e5b37),LL(0x92f6b6a3,0x5511d682),LL(0xae575249,0x2e9e5034),LL(0x1d14bed7,0x3e5a32f8),LL(0x75efd17a,0xa346a86f), LL(0x0309fd7a,0x4f2510a6),LL(0xd0b1425e,0x689ecd74),LL(0x9f771e24,0x9e9bebe1),LL(0xadc5b48c,0x20188045),LL(0xb845230b,0xe49811b6),LL(0x5a8687f8,0x420855ec), + LL(0xbae1b94d,0xe650e49a),LL(0xe3199794,0xb6b162e6),LL(0xb4ec0480,0xdc706859),LL(0x4b1a06ca,0x28b618c2),LL(0x403acdc2,0x0929a001),LL(0x2da3aefd,0x796dfd97), LL(0xef4c1673,0x16389072),LL(0xfc94a4f5,0x600d8bd7),LL(0xe5f386a9,0xf003214d),LL(0xe62cbb48,0xa7af0499),LL(0xde82bad5,0x750a3b00),LL(0x8e7dc8ee,0x6c615b83), + LL(0xd71543a6,0x283eec26),LL(0xa7627841,0x98fa08be),LL(0x27ad302d,0x269a83b8),LL(0xbde3fdd0,0x225f2f12),LL(0x0130b3a6,0x046fcf38),LL(0xc3ed9043,0xea733c1a), LL(0x70aa08d1,0xf870f14d),LL(0x34391e0b,0x643d18b8),LL(0x847be772,0xf3e1d5f4),LL(0xd0ed73a0,0xa9498223),LL(0x14b3babb,0x6933ccf0),LL(0x37f08f70,0xc2439ae4), + LL(0xf88d049b,0xb643f4e0),LL(0x12682fcf,0x5e0ac1fb),LL(0x9f981c8f,0xeaf7874d),LL(0xb1af779d,0x9c2adfd2),LL(0xdaa8c275,0x9a7abead),LL(0x24cacec4,0x09ad5521), LL(0x0ead1646,0x069cd5c4),LL(0x0a6157d1,0x5186bf19),LL(0x96503506,0xcc222a93),LL(0xbd29686e,0xfeaa7bde),LL(0xa7257c8d,0xb0d65b0d),LL(0x98aa227f,0xc31c0a88), + LL(0x5be1d45b,0xb4b7651f),LL(0x7f0cf680,0x0425200a),LL(0x8960be95,0x200d12b4),LL(0x4945b193,0x02fdd1a1),LL(0x27d046d8,0xedd70e3e),LL(0x83f14e12,0xc1cc086a), LL(0x2629396e,0x1580e72b),LL(0xf9ed73c1,0xc87439db),LL(0xa90c5128,0x5debdf30),LL(0x9fbe14ef,0x0b6c020e),LL(0x0149a0b0,0x168da56a),LL(0x79c58ac6,0xc66a4dbd), + LL(0x330b8e2e,0x800aec84),LL(0x335837bf,0x1a2c033e),LL(0xfe6f6dd9,0xf1a91551),LL(0x1de7360a,0x326c42b2),LL(0x7b66f9d5,0x300e740b),LL(0x68ce95d4,0x53bcc700), LL(0x3d80f228,0xc9e225ac),LL(0x6977dfdb,0x64b2ad4e),LL(0x01f23221,0xac863b08),LL(0xdf11e5f7,0x0517a648),LL(0x68d11050,0xbf7aedcb),LL(0x77b3029c,0x2607e337), + LL(0x206add5d,0x1afa6aa6),LL(0x150ea4c9,0x66cfbbae),LL(0x5d36da4f,0x07fb920b),LL(0x291e774f,0x144d51f9),LL(0xf40d87a8,0x26c2c134),LL(0xa932f1a0,0xc8cf3524), LL(0x5aeb0bde,0x35bb2a42),LL(0xc4be960a,0x5cfcc1da),LL(0xaa1838ed,0x5c40cabf),LL(0xe2855f1f,0xea0c05ff),LL(0xfd525934,0x931ebb02),LL(0x16246fd4,0x31a7b78f), +}, +/* digit=11 base_pwr=2^55 */ +{ + LL(0x6d6ae962,0xcd92906c),LL(0x9807d881,0x62835615),LL(0x1fdc1915,0x0d692978),LL(0x269d611e,0x45d01a8c),LL(0x9665b00a,0xe7bd1e70),LL(0x9bcaa388,0x08638534), LL(0x2dd24299,0x8f189e88),LL(0xb82fb270,0x5f643392),LL(0xc633b111,0xca65bf16),LL(0xd6f1dac8,0xc6adc9c9),LL(0xa3c3381d,0x0df2c293),LL(0x8388cd12,0xdd6ae97d), + LL(0x3863db02,0xffdbd0eb),LL(0x2f57e10f,0x8b825683),LL(0x35e7a3a2,0xc11acead),LL(0x67833028,0x4998cf8c),LL(0x844c7976,0x8f3a346b),LL(0xdb9b1a1c,0x0a9d872c), LL(0xb98d445d,0x8735dabc),LL(0x305fa0a9,0x93790d80),LL(0xd267a01a,0x7c0add49),LL(0xffa20d11,0x2b46c913),LL(0xd8ab2d4a,0xf2acef26),LL(0x3d926080,0x71b701b9), + LL(0xfbf536a1,0x7cae584d),LL(0x5b52faa9,0x0af06fc2),LL(0x2827b872,0x807706dc),LL(0x59903742,0xf029478f),LL(0x85cd7aed,0x0e8c393c),LL(0xa331bee7,0x2e1e6ade), LL(0xcdb7f97a,0xf5d13dad),LL(0x8577c3cc,0xdb22fed1),LL(0x81e39043,0x6f453180),LL(0xa00a806c,0x41d4bb25),LL(0xe28670a7,0xe6064b92),LL(0xa7c63f88,0x733a31c8), + LL(0x5133de8e,0xe9d2a98a),LL(0xb81b8b00,0x37083b60),LL(0xceaf86ae,0xf399325d),LL(0x8f161525,0x03b17c88),LL(0x84211b9d,0xd8ac35c9),LL(0x9050ca48,0x22083784), LL(0xc9fab832,0xa818c44b),LL(0xe5aea7da,0x8882bcce),LL(0xf8715b04,0x633aaf35),LL(0x9d8829a9,0x5463e1b9),LL(0x84a820f1,0xb18df52d),LL(0xd096675d,0x9d5ef891), + LL(0xc2b47dd2,0x394174d0),LL(0x30a87997,0x06b117a8),LL(0x894731e5,0xcfd81d41),LL(0xc84e7d05,0x6476b1af),LL(0x743b8218,0x37069fe2),LL(0xa36173d8,0x93f21550), LL(0xedadef86,0xa8ae70f4),LL(0x76366073,0xdac5d021),LL(0x1c982076,0x4063ba57),LL(0x2d24d61e,0xc3471b5f),LL(0xae4d820f,0x57b69c41),LL(0xc463a391,0xf73e14a4), + LL(0xac60496d,0xd54e2c7c),LL(0x04cd50a4,0xc06d5e5d),LL(0xe60f7f59,0xcb4105e8),LL(0x427483ad,0x705db308),LL(0xf2bff383,0xf73ba98b),LL(0x0220e6e9,0xa945611a), LL(0xd957e12b,0xc01c46b8),LL(0xacb1f371,0x458897b7),LL(0xfa3403e6,0xf738dc0b),LL(0xd2202896,0x098bc687),LL(0x5f882e5e,0xec0c217a),LL(0xa1f4eb13,0x8f25af77), + LL(0x195736e9,0x86e6a8fc),LL(0x1a20620f,0x88099bf3),LL(0x9cdc1db9,0xf3d9998b),LL(0x47042834,0xf36d583e),LL(0x25a0d589,0xcd4519d8),LL(0x1e04a91d,0x78c435ab), LL(0xe44d9e31,0x0428e76e),LL(0xce821aab,0x63994993),LL(0x2625cfce,0xd167ceb7),LL(0x0ddf2b89,0x9f242245),LL(0x717ffb5e,0x182e33d6),LL(0x26de205c,0x3800290e), + LL(0x59ee4124,0x2615c782),LL(0x76532b4b,0x4dc2824c),LL(0x1c84a04b,0x9c3b1d77),LL(0xcb9f9e34,0xb6fc203f),LL(0xc64f7846,0xbed65464),LL(0xeb004248,0x04f520a2), LL(0x4a58fd22,0x5c017727),LL(0xc10d9472,0x25958482),LL(0xaceb0e3a,0xb78c6666),LL(0xfc046f0a,0x18d3c188),LL(0x1baa9595,0x7f3e2f30),LL(0x8a2844e8,0xa574f8cd), + LL(0x4bcc5a81,0x9fd2e524),LL(0xd3f2f80f,0xc73e2598),LL(0x15dc157c,0xff838e9c),LL(0xc522319e,0x0b2491d2),LL(0x909592d6,0x2b65f042),LL(0x849f5dee,0xf113045c), LL(0x9b287c3d,0x84fe31be),LL(0x80097bd0,0xd0481262),LL(0x63660f87,0xaf203cba),LL(0x0ea001b3,0x5621c3d8),LL(0x346ea29b,0x125a5eaa),LL(0x3ca79649,0x3dcec22c), + LL(0x81c2d81f,0xc899eba3),LL(0xf3f0a431,0xb27267d6),LL(0xda55568e,0x607c8629),LL(0x2b1dc1d9,0x6b547228),LL(0xc9c001ff,0x23232311),LL(0x488f8b85,0x207a2eb2), LL(0xdac37a28,0x3867ac9a),LL(0x2584a5f0,0xa36d14d3),LL(0xa74488ff,0x7398c647),LL(0xbe087640,0xf6ed920f),LL(0x6319a571,0x72beddc7),LL(0x6a244aeb,0x55c2cd82), + LL(0x9ede92f1,0x42df5630),LL(0x76503aa7,0xe6957bc4),LL(0xfbec9577,0xe7abfe12),LL(0x274b7f6c,0x9660e762),LL(0x35f12ea2,0xf78af22c),LL(0xb51bec2c,0x73ab1bbb), LL(0x1641f424,0x5e8e40e6),LL(0x0905e4be,0x1af751d9),LL(0x1c033198,0xb5130ade),LL(0xa22f5f19,0x96d47ab2),LL(0x55340d4c,0x5aff3e66),LL(0x1bed65fa,0xab4aa5a7), + LL(0x25f8a53b,0xb7da79c6),LL(0xd331ad8e,0x6b950bdf),LL(0x4aa36d18,0x3481b7b5),LL(0xed0e3091,0x6efeaf88),LL(0xc993074c,0xeb017bdd),LL(0x529dd654,0x8431a6d6), LL(0xbd069585,0xf5177231),LL(0x3ce85096,0x6d753b10),LL(0x4ca26741,0x194d82d3),LL(0xadcd1650,0xeaeffe15),LL(0xaf7758b7,0x4dcec3d9),LL(0x4cc2c819,0xf5fdf666), + LL(0xae850261,0x7e9ad6e8),LL(0x11960968,0xbeb25ca0),LL(0xad4cfa42,0x60c5f0ee),LL(0xeca5c508,0x81714b3f),LL(0xb78a290f,0x810e71d7),LL(0x3260114c,0x03536cec), LL(0x6a16cdf6,0x24d29f0f),LL(0x4a4c964c,0xec45a58a),LL(0xea344b44,0xbaaa5fb3),LL(0xfeb783f6,0xf1b7a68c),LL(0x3ddd7062,0x533c4b72),LL(0x1598b5f7,0x6a33a0dc), + LL(0xa9810744,0x94bdc7f4),LL(0xf045d859,0x464195da),LL(0xd654cb57,0x27e2dab0),LL(0x7a491956,0x1d4e1e53),LL(0x31d5d099,0xa1ef570c),LL(0x295f3de7,0x01cd21ee), LL(0xb8249038,0x8681b00d),LL(0x93781b71,0x17c31bce),LL(0x6a1b5748,0x4324e90c),LL(0x1222e554,0x44f9324c),LL(0xffd53dd0,0xe30ba10f),LL(0xb48eeef0,0x2e5817a8), + LL(0x24b3a9a7,0x6b0ee060),LL(0xd5f6545c,0xcf78c3fe),LL(0xfe0f2ec6,0x050bedaa),LL(0xb3e6b4f8,0x0b7774b8),LL(0x2b4a853e,0x096b9e19),LL(0x0698f7ca,0xa7463661), LL(0xf090c97b,0xe1d1d1be),LL(0x55517a2f,0xbc1e09d9),LL(0x5868ee3d,0xb3e24dd5),LL(0xd7b83619,0xd6103494),LL(0x3b0129b3,0xaf739559),LL(0x3ded9fb3,0xef42bb60), + LL(0xc8fccaa9,0xa349bb66),LL(0x31a53ee7,0x7888755f),LL(0xc18d3750,0xa6e1d891),LL(0xae8d2bfb,0x9985aa4d),LL(0x31b33078,0x8baec9ae),LL(0x98750e94,0xee68295a), LL(0xd6ddf305,0x0d834bf8),LL(0x9762126c,0xab33dff3),LL(0x0c51d098,0x0c22faaa),LL(0xb887a10f,0x32404042),LL(0x248bed32,0x31f6a614),LL(0x1ce0d662,0x311f8630), +}, +/* digit=12 base_pwr=2^60 */ +{ + LL(0xedc9ce62,0xac6dbdf6),LL(0x0f9c006e,0xa58f5b44),LL(0xdc28e1b0,0x16694de3),LL(0xa6647711,0x2d039cf2),LL(0xc5b08b4b,0xa13bbe6f),LL(0x10ebd8ce,0xe44da930), LL(0x19649a16,0xcd472087),LL(0x683e5df1,0xe18f4e44),LL(0x929bfa28,0xb3f66303),LL(0x818249bf,0x7c378e43),LL(0x847f7cd9,0x76068c80),LL(0x987eba16,0xee3db6d1), + LL(0xae658422,0x1238e097),LL(0x568df55f,0xb4631ddb),LL(0xf74c5c50,0x451254e7),LL(0x8805813b,0x238b16d2),LL(0x925e7a6e,0x23987b28),LL(0x2a1a10bc,0x93b72e2d), LL(0x05e44b7f,0x944c784d),LL(0x8c8e3120,0x7d70fd09),LL(0xead45716,0x6bf1ab2a),LL(0x31c04205,0xd5f8f0e6),LL(0xa10b8881,0xac062526),LL(0xfe5505a8,0xa1a83cf0), + LL(0xc42a2f52,0xcbbd8576),LL(0x9d2b06bb,0x9acc6f70),LL(0x2e6b72a4,0xe5cb5620),LL(0x7c024443,0x5738ea0e),LL(0xb55368f3,0x8ed06170),LL(0x1aeed44f,0xe54c99bb), LL(0xe2e0d8b2,0x3d90a6b2),LL(0xcf7b2856,0x21718977),LL(0xc5612aec,0x089093dc),LL(0x99c1bacc,0xc272ef6f),LL(0xdc43eaad,0x47db3b43),LL(0x0832d891,0x730f30e4), + LL(0x37d83369,0xac92ee15),LL(0xfecec65c,0xc968c187),LL(0x6e7a3265,0x29a7ca87),LL(0x8456c9af,0x0f2b7e7a),LL(0x9754326f,0x7471824e),LL(0x364d2ec8,0x498687bf), LL(0x3c6ee351,0x86d8aacd),LL(0xf6f41e85,0x01ee6823),LL(0x1d79f7eb,0x9805fc88),LL(0x0040547d,0x377ac3a4),LL(0x61b4e90b,0xd39215d4),LL(0x4c5fd81b,0x2547416e), + LL(0x0c7fecdb,0x9ffe5563),LL(0xf88101e5,0x55cc67b6),LL(0xcbefa3c7,0x3039f981),LL(0x667bfd64,0x2ab06883),LL(0x4340e3df,0x9007a257),LL(0x5a3a49ca,0x1ac3f3fa), LL(0xc97e20fd,0x9c7be629),LL(0xa3dae003,0xf61823d3),LL(0xe7380dba,0xffe7ff39),LL(0x9facc3b8,0x620bb9b5),LL(0x31ae422c,0x2ddcb8cd),LL(0xd12c3c43,0x1de3bcfa), + LL(0xf6e10789,0x03621d34),LL(0x410c4004,0xc72143b8),LL(0x6ca2933a,0x886d00f6),LL(0xde1494a2,0xd0a22382),LL(0x0bdcdf82,0xb8656c4d),LL(0xc4a58832,0x5199d9d3), LL(0xe1c0f449,0xa982c1aa),LL(0x5a922bdb,0xe9cefb90),LL(0x31390c6d,0xfc268de4),LL(0x8c4fe595,0xea4160e1),LL(0x6790d82f,0x6c427a63),LL(0xfc519907,0xaceb0154), + LL(0xd6e0f9a9,0x8c074946),LL(0x51c3b05b,0x662fa995),LL(0x04bb2048,0x6cdae969),LL(0xd6dc8b60,0x6dec9594),LL(0x54438bbc,0x8d265869),LL(0x1b0e95a5,0x88e983e3), LL(0x60cbf838,0x8189f114),LL(0x771dc46b,0x77190697),LL(0x27f8ec1a,0x775775a2),LL(0x607e3739,0x7a125240),LL(0x4f793e4e,0xafae84e7),LL(0x5bf5baf4,0x44fa17f3), + LL(0x07a6fbf8,0xa10292b9),LL(0x3fa6235b,0x292c1a20),LL(0x73ad7a1f,0x7a36f18f),LL(0x5897b11f,0x8b2c7b0c),LL(0xcb664c61,0xf7b9a272),LL(0x8f81e22c,0xb6d366af), LL(0x8e342bba,0x3b99b211),LL(0xb06ced2b,0x03ce158b),LL(0x001db74b,0x3af1175d),LL(0x7159cb8a,0x526f0846),LL(0xebde4601,0x6a3c6e1f),LL(0x8c232eac,0xfad5963b), + LL(0xd03ac439,0xa21e69a5),LL(0x88aa8094,0x2069c5fc),LL(0x8c08f206,0xb041eea7),LL(0x3d65b8ed,0x55b9d461),LL(0xd392c7c4,0x951ea25c),LL(0x9d166232,0x4b9a1cec), LL(0xfcf931a4,0xc184fcd8),LL(0x063ad374,0xba59ad44),LL(0x1aa9796f,0x1868ad2a),LL(0xdff29832,0x38a34018),LL(0x03df8070,0x01fc8801),LL(0x48dd334a,0x1282cce0), + LL(0x42a993b1,0x21387e8a),LL(0x68b1f2a0,0xbcb65533),LL(0xa5f7e4b9,0x3161d87a),LL(0x1bce0a84,0x927b4130),LL(0x87be0ae9,0x3aabd327),LL(0x0aed169d,0x0163c6aa), LL(0x11584051,0xad1e0f0e),LL(0x74093225,0x1ba46302),LL(0x0a6a1f01,0xed045618),LL(0x6da060ed,0x15d07c63),LL(0xb7e003ae,0xa4584f72),LL(0xa02ec136,0xf6e6b7c4), + LL(0x26d8503c,0x76aa9557),LL(0x6bc3e3d0,0xbe962b63),LL(0x97de8841,0xf5ca93e5),LL(0xaf3f2c16,0x1561b05e),LL(0xd34bff98,0x34be00aa),LL(0xd23d2925,0xea21e6e9), LL(0x394c3afb,0x55713230),LL(0xd6c8beca,0xeaf0529b),LL(0x202b9a11,0xff38a743),LL(0x6d3a398b,0xa13e39fc),LL(0x86e2615a,0x8cbd644b),LL(0x191057ec,0x92063988), + LL(0x74962b2e,0x71503219),LL(0xe1db7ec7,0x3beeaf81),LL(0x445c4e88,0x702784cc),LL(0x9baad7ed,0x7aa58315),LL(0xd044b38a,0x72b76e2f),LL(0x6a7f2101,0x35a2bb8f), LL(0xa14c2bf2,0x5e6ae263),LL(0x34bb719d,0x8fb614c0),LL(0x5dcd7c65,0x7949eeca),LL(0x7670aeb1,0xb198be58),LL(0x93cabf25,0xb2238dfc),LL(0x1a422b95,0x6a6d4555), + LL(0x13f89146,0x787835ce),LL(0x69446c3f,0x7fcd42cc),LL(0x840e679d,0x0da2aa98),LL(0x18779a1b,0x44f20523),LL(0xefbf5935,0xe3a3b34f),LL(0xb9947b70,0xa5d2cfd0), LL(0x27f4e16f,0xae2af4ef),LL(0xb9d21322,0xa7fa70d2),LL(0xb3fd566b,0x68084919),LL(0xd7aad6ab,0xf04d71c8),LL(0x10bc4260,0xdbea21e4),LL(0x8d949b42,0xaa7dc665), + LL(0xf7848707,0x3ddea6d9),LL(0x6dbcf13a,0xc11260d0),LL(0xa16508f0,0x62c5ee60),LL(0xcb56535c,0x4bdfa38f),LL(0x4fa5a6b0,0xd391e786),LL(0x2d4cd296,0x173c3d70), LL(0x5fa3f53c,0x9c5c596f),LL(0x1ddd3732,0x77d39514),LL(0xa5bf4edc,0xca1bb91d),LL(0x1cd98b09,0xb23afd38),LL(0x60bdfa0f,0x198248ed),LL(0xfb69a4b1,0x559dd99c), + LL(0x6ccb8213,0xd8e958a0),LL(0x91900b54,0x118d9db9),LL(0x85e8ced6,0x09bb9d49),LL(0x24019281,0x410e9fb5),LL(0x6d74c86e,0x3b31b4e1),LL(0x020bb77d,0x52bc0252), LL(0x27092ce4,0x5616a26f),LL(0xa08f65cd,0x67774dbc),LL(0xc08bd569,0x560ad494),LL(0xad498783,0xbe26da36),LL(0x7f019c91,0x0276c8ab),LL(0x5248266e,0x09843ada), + LL(0xb1b43eef,0xa54768da),LL(0xe14fda22,0x13e41f47),LL(0xfaef6863,0x774df203),LL(0xbd7471b3,0xf795a034),LL(0xb47de2e9,0xf0958718),LL(0xe1160cff,0xc92f7888), LL(0x0146c790,0x86ded97b),LL(0x480a4b7b,0x015918f5),LL(0x424e8459,0x05588920),LL(0xeecf8b2b,0x37455914),LL(0xb968a6fa,0xe7d3df1f),LL(0xbad0719f,0x07a0ffd6), +}, +/* digit=13 base_pwr=2^65 */ +{ + LL(0x98d23f6f,0x2566021f),LL(0x34ca97ca,0xfb883e12),LL(0xd9f51b69,0x34e047a5),LL(0xf8efa646,0x0b50d91d),LL(0x971f584f,0xc2bbcbb2),LL(0x0907c91c,0x4136f0e4), LL(0xe735cc48,0xa7ebeb0d),LL(0xe113c8fa,0xa7d1bedc),LL(0x3f5c962a,0xc04d9a07),LL(0x3ff74a2f,0x95c155e5),LL(0x3df0749d,0x923c65a5),LL(0x27ae35d0,0x10d5f812), + LL(0xecee6e87,0xd5469c7b),LL(0x33a4c917,0x056180bc),LL(0xa16caa7a,0xf881ca21),LL(0xe6cc7f39,0x221de182),LL(0x31378723,0x10d61ab5),LL(0x520c9660,0xfb763bd9), LL(0x0d6b1541,0x145214cd),LL(0xd70223e7,0xd9f7ff2d),LL(0x0cb1fe69,0x9fce59e3),LL(0x3e299fe7,0x2e6e77fa),LL(0xd5af78cf,0x3a0cf652),LL(0x3e852159,0x50cc42c5), + LL(0xdf764716,0x0791dfa1),LL(0x8c66da07,0x31bf6876),LL(0xccedf4f3,0x49f25b77),LL(0x5d965c05,0x05170ccd),LL(0xd49e6727,0x37d9521b),LL(0x86a00176,0x15482512), LL(0x6c00eb48,0xdab44493),LL(0xe00c5c5d,0x102c6b95),LL(0x4c2506ba,0x43660c3e),LL(0x5ec6f132,0xb2fb2616),LL(0x99ac7691,0xccc4221a),LL(0xa576deb5,0x05b29758), + LL(0xad8c49b7,0x4a873076),LL(0x7146575f,0x891598ce),LL(0x427ea198,0xc1d3042f),LL(0xed259219,0xdc592111),LL(0x234850ca,0x0abdbd16),LL(0x43b6fe8d,0x26b94126), LL(0x36a1cfe9,0xd3c79d17),LL(0x9a2b3baf,0x57638621),LL(0x5a98bf65,0xa736535d),LL(0xab2cdb2b,0xacb3b7dd),LL(0xdaaf89ee,0x37d3743a),LL(0x0b348532,0xf19d9aba), + LL(0xd93c54b2,0xafad01a0),LL(0x95536c49,0x659bff96),LL(0xb9734c15,0x7b91aac2),LL(0x24e02f59,0x55c7f082),LL(0x3a26e551,0xebcb71e7),LL(0x6c6343eb,0x5b7225f7), LL(0x17d5e775,0x021c48fb),LL(0xbd859c87,0x57536a42),LL(0xae2b63e9,0x24852cc0),LL(0xc0ce0ef1,0x21515020),LL(0x8fed825e,0x2ac7336c),LL(0xa0152819,0x4bc87fee), + LL(0x4654712c,0xe7c396d6),LL(0x7a26e994,0x1fa5ea50),LL(0x09012b83,0xaa987687),LL(0xe9b17e98,0xf1ef9792),LL(0x6dc2db10,0x2c22bcb0),LL(0xed4be80e,0xae42ddf6), LL(0x2e743405,0x672080f6),LL(0x5b7821d1,0xa15a7f97),LL(0x47adbf07,0x0cd912f2),LL(0xb6c4ae8f,0x6919c0dc),LL(0x14c6253c,0x62b13edd),LL(0xf8032287,0x66f35919), + LL(0x9d628e1d,0x2713b58c),LL(0xfaa8ba1e,0x3729960a),LL(0xab53bc93,0xc3438130),LL(0xe9c165e7,0xcef9eda5),LL(0xfd02650b,0x9bacd1c3),LL(0x540ece72,0xbb300334), LL(0xd9c4f0ed,0x21f5a5d3),LL(0xf7c19269,0x1ba32e4f),LL(0x2cf320fb,0x8f073beb),LL(0x5599646f,0xb49766c4),LL(0x49e4f200,0x68180d66),LL(0xeaeddb36,0x8203d8ae), + LL(0x2caf088b,0xa68295da),LL(0x5c8709fe,0x23d6439a),LL(0xfe0c3df0,0x8deba0cf),LL(0x3cd00a1a,0x5b4d037b),LL(0xaa0f9088,0xe9edc429),LL(0x5847def7,0x6f5827e3), LL(0x306ad966,0x9739d03d),LL(0xaed51d04,0x7c6b18af),LL(0x1759060a,0xdc3d34ff),LL(0xa7e94dbb,0x029e9aa9),LL(0xf7e8b7f3,0x2a3cdfa0),LL(0xbbd8f6f4,0x42f87bf0), + LL(0xed210253,0x516f138b),LL(0x4433461a,0x5ec2fa32),LL(0xcdaf1280,0x0dbe2c66),LL(0x9fbf3318,0x086b91e5),LL(0x399a1ca6,0xfb0223ee),LL(0x0db5b20f,0xd6f86d9b), LL(0x5752d618,0xec02bca2),LL(0xaf69f3f1,0x952fafca),LL(0x33c4d294,0xf304cb75),LL(0xdac65608,0x78085727),LL(0x840a4466,0x22f302ef),LL(0xc371c31f,0x33fb889d), + LL(0x4f890542,0xaa4711f5),LL(0xeba822c4,0x862421d9),LL(0x848280fc,0x2f667179),LL(0xc201ed75,0x4de16d87),LL(0xc5e61b5d,0xd20e1399),LL(0x9ed67ec7,0x3f7114b4), LL(0x9b5a88f9,0x561fd497),LL(0xd84db2c1,0xb202eb86),LL(0xc8637d3d,0x67d8fb90),LL(0x032b1853,0x3d1d78a1),LL(0xef1af9ac,0xe07bf775),LL(0xa57d6ada,0x691e1dee), + LL(0x3801c65d,0x834701da),LL(0x955aa27d,0x5bb35c48),LL(0xdb7ad387,0x0ef0f375),LL(0x06cd1d53,0xd25e337f),LL(0x90cd91de,0x757a1f9d),LL(0xd61bbd60,0x1604f153), LL(0x8bb95dc4,0x6a01e8cf),LL(0x75bbdb13,0x34b7be62),LL(0x21e9b029,0x0a96b3a1),LL(0x2946df44,0x25615c3b),LL(0x19d04842,0x5eda7d19),LL(0xfba84668,0x08317975), + LL(0xa55a3a4d,0x12474bd8),LL(0x9e471af4,0xe326aaf1),LL(0x8caadaa6,0xf201a930),LL(0x35304341,0x546821f8),LL(0x088353e1,0x7fe452c3),LL(0xfc82566a,0x8f1ff628), LL(0xd99f8967,0x49526f46),LL(0xa4009690,0xb19c80c0),LL(0x1cbc0716,0xeccf9759),LL(0xaf4cbc8b,0x2e13ae2c),LL(0xf32e29ad,0x12b0df13),LL(0x0b1565f0,0xa2005d6e), + LL(0xd06014e0,0xdb891eed),LL(0x03e9970a,0x69685d61),LL(0x02838113,0x3a612db4),LL(0xdcdef0b0,0xc1cd7b3a),LL(0xa41d6c1e,0x612b299d),LL(0x0ed386a0,0x982161ed), LL(0x3ea1bf1e,0xb36bbe2f),LL(0xceb2a5ec,0x0d8c3752),LL(0xec03bddd,0xc02cd7f6),LL(0x52631d9e,0xa87977c1),LL(0x7e398d7e,0x7b546cc3),LL(0x04845671,0x5b1218a8), + LL(0x6cb173d1,0xfa3e43e5),LL(0x4591b5a5,0x2502258d),LL(0x8ca9682a,0xae8c4b55),LL(0xec81a288,0x8cb1ffb4),LL(0xbfc84fd1,0xd11ae888),LL(0xd774577c,0xa3b083a0), LL(0x1da9afc5,0x119b41c1),LL(0x2934e22b,0x44bc7762),LL(0x7c639d6c,0xa04694f3),LL(0x32c5b8ea,0xd5e1ce57),LL(0x9749e8b6,0xd507c39b),LL(0x55255b63,0x16cc0b57), + LL(0x614f6f37,0x71e5df78),LL(0xa0b80bee,0x5cf0e08d),LL(0x1e32051f,0x1f8dae17),LL(0x83bc233b,0x54ae365d),LL(0x1b84aaa6,0x97ea005b),LL(0x64c75139,0xf4766d92), LL(0x41215701,0x9b93bbf2),LL(0x8cf8a865,0xb18f042d),LL(0x0867556f,0x5dfb96dd),LL(0x597fd6a1,0xe9fafbb8),LL(0xfe48bbc6,0x729b2f50),LL(0x7f37ff9b,0x2cf85f6b), + LL(0x378ef62b,0xcd8c2ec9),LL(0x91a7c4b9,0x7f4a3c54),LL(0xdb83e1ab,0xe12386d4),LL(0xbb549bb5,0x9a792032),LL(0x2807c0f1,0xaf81cba6),LL(0xabf2008e,0xd4ad7d87), LL(0x7e9ad6cd,0x9d7a7230),LL(0x8b517b2a,0xb30636b0),LL(0xec900516,0x47c324da),LL(0x7193eb30,0x408cd0d3),LL(0xd315c655,0x8f0bcce2),LL(0x869d6c22,0x540ad4e0), +}, +/* digit=14 base_pwr=2^70 */ +{ + LL(0xdae0ff8b,0x4b5e753d),LL(0xda3d97b5,0xe55c83e4),LL(0x42fa905b,0x4034d75f),LL(0xc33e462b,0x89b85eda),LL(0x058de3bb,0x31f413c3),LL(0x6ba75391,0x66c01c80), LL(0x373e28de,0x3f500202),LL(0x4b9be739,0x5090b33b),LL(0xefa2addd,0x7297aa10),LL(0x1a6566aa,0x3e8ccdbe),LL(0xd7b4f214,0x4dfda07f),LL(0x7cb1cae7,0xa659bd1e), + LL(0x87d11691,0x31796c23),LL(0xa9de506c,0x02991ea2),LL(0x6cb0c301,0x4ff0cb71),LL(0xd1702ca9,0xdd0cdbd7),LL(0xe1a02a90,0x470a26c8),LL(0xd7054625,0xb705b7ba), LL(0xd4a1a268,0xfadc2e86),LL(0x68e9f923,0x0fd97646),LL(0x2951a8fe,0x042b5ebb),LL(0x64197a76,0xe4af9d03),LL(0x249c1b5d,0xdd2c6bb3),LL(0xf01932b1,0x60af89bd), + LL(0x41afcd64,0x4374145d),LL(0x49d21198,0x98b72d60),LL(0xdfde8a41,0xc0ff394b),LL(0xee1ff7a5,0xed1112e5),LL(0xcb5036fc,0x87a920e8),LL(0x2deb225e,0x437123f6), LL(0xb9ad8c58,0x37e527af),LL(0xabfaef38,0x3e3c9998),LL(0x50b2b4e2,0xb656bcc5),LL(0x3bf5699d,0xfacc8a19),LL(0xe616307a,0x98cec74f),LL(0xd3ef8bab,0x34af333d), + LL(0x744ec273,0x9ded9b0d),LL(0x2e79e4d5,0x5bdfe547),LL(0x39393728,0x94f3aaf5),LL(0xa438413e,0x22136862),LL(0x373c7de3,0x449286da),LL(0x29aa1540,0xa709d85d), LL(0x0284a4f6,0x1bd13e41),LL(0xf0799c8b,0x37b54d69),LL(0x6bcd0cb2,0xd43b558f),LL(0x9e610369,0xf5757c0e),LL(0xc16e0651,0x15c80b23),LL(0x001820aa,0xacb2cf64), + LL(0xeec37f48,0x0629e4dd),LL(0xda7de716,0x3f7556a2),LL(0x661662bc,0xcacd8f27),LL(0xeaf01690,0x65d8bc2c),LL(0x7c39c893,0x83ac6647),LL(0xf59440e1,0x353f60db), LL(0x852c575a,0x2597b0d6),LL(0x3a40c2b3,0x410885f7),LL(0x2fcc2488,0x953ab347),LL(0xef4cc6d8,0x9f753e5e),LL(0x97f69e63,0xfc32bb4d),LL(0x87e8c264,0x461c1b0c), + LL(0xac4b62f5,0x2bea7e75),LL(0xc6297871,0xcf255dcd),LL(0x81b25c72,0xdd88db87),LL(0xb617dc04,0x77ad90b3),LL(0x0ed4a7d6,0x65ee1382),LL(0xd9644c8f,0x4c08df9c), LL(0x525a023f,0x072d3784),LL(0xfce399fb,0x1aef69ce),LL(0xd7f29044,0xb07fd78e),LL(0x43043fa7,0xa3754e14),LL(0xff9fe4d9,0x97bdae92),LL(0x700fe6b8,0xad63ba6a), + LL(0xa1ca17db,0xa571929b),LL(0x45e146b8,0xce7a12f5),LL(0xd9eb426d,0x39df1446),LL(0xdc27f268,0x1e48b3f8),LL(0xff548455,0xa2d7dfa1),LL(0x6ef1cc82,0x750068b8), LL(0x667fce62,0x4d699306),LL(0x131c5412,0x98540b9d),LL(0x47c580b0,0xf8a62cd3),LL(0x73795005,0x2b55460f),LL(0x7b8db337,0x3206c025),LL(0xd0dda5a2,0x2280934b), + LL(0x2352478c,0x10a8aabd),LL(0x1364c40f,0x599d9dfa),LL(0x076945a2,0xa009df1a),LL(0x03861f02,0xf869152c),LL(0x9f866a3c,0xc405226e),LL(0x8b41ecbb,0x93bd737d), LL(0x33901eed,0xb5c9ed10),LL(0x99312b80,0x90e4ce8f),LL(0x57589279,0x1a9ef22e),LL(0x7fe2d6aa,0x83ef607d),LL(0x3473dbfa,0xf2da8454),LL(0x57879066,0x14f36d3a), + LL(0x0ea3ce34,0x2c780f22),LL(0x4b8aac72,0x240a211e),LL(0xc2625a99,0x7a266e5d),LL(0x3b30c878,0x1cb15d3e),LL(0x0e1b21d3,0x8cd8ccab),LL(0xadc1a6b3,0x53c64279), LL(0x60bf708d,0xe60d15b9),LL(0x0cb5ad4b,0x6e431c1b),LL(0x82033111,0xec874c3e),LL(0x9141eae5,0x88054a1d),LL(0xddf53a28,0x98438a5a),LL(0xfa12c657,0x168f0b0f), + LL(0x629d7a57,0xd621ce26),LL(0xcc1f8af1,0xbf571de7),LL(0x304adaa3,0x2c5cfaf9),LL(0x3f283b49,0x950addbd),LL(0xee6d1cd1,0x622dc27c),LL(0x1f0863fb,0x26d92004), LL(0xa41ec585,0xe243b2a2),LL(0x2bea6235,0x0dbd9adf),LL(0x6f0820f5,0xcb083c4d),LL(0xd5493931,0x809ecbaf),LL(0x14b7ffa8,0x96470674),LL(0x53fc2224,0x2cdfe22e), + LL(0x7cd74a06,0xc8b1333c),LL(0x271006bf,0x7d5ac4dc),LL(0xc14e0e56,0xe9377d9f),LL(0x7a92ab1f,0xaa8651db),LL(0x5fc11fb5,0x77cee814),LL(0x27870b14,0x1e7c5ca0), LL(0x6f959698,0xdeef4b3f),LL(0x5daa9bda,0x7c59f26e),LL(0xfd312368,0x4d0d5aec),LL(0xb738de68,0x5247a6f3),LL(0x90c04807,0x1c4e8ba9),LL(0xcce126ca,0x0554b41b), + LL(0xa81cc26a,0x3df98ea9),LL(0x83b2c6f3,0x982ed568),LL(0xea6d6976,0xc9cbd1b5),LL(0x7e25ffbc,0x3f9f2319),LL(0x7da6280e,0xbca8e056),LL(0x35cda713,0x7abd3166), LL(0x740ae011,0x46ef321d),LL(0xdb214a33,0xb17f6c75),LL(0x51de4044,0x37b73b4b),LL(0xccd9ba8d,0x5bccf3cc),LL(0xd0f7045b,0xa2ca080d),LL(0x68cf4dcc,0x79caf906), + LL(0xa7b07d22,0xcc3605a9),LL(0xb4ebe4e7,0x4370eb18),LL(0x248867c1,0xbe393039),LL(0xcb1a75ff,0xc8e4851e),LL(0x39cb6da4,0x215f3fbe),LL(0xe41f9a34,0x6f2102ee), LL(0xdfae7c27,0x61d484ab),LL(0x6f1260fc,0xf5143bd2),LL(0x7514bccc,0xa70b6c06),LL(0xd23506f5,0xe71ca833),LL(0xeae03a8e,0xe2f50ba8),LL(0x83c33359,0x2ac3b508), + LL(0x3655cabd,0xe94b930f),LL(0xa342443c,0x6ef6aac4),LL(0xbae255da,0x2feb8005),LL(0xac6e2095,0x4625a15c),LL(0x0ec76c1e,0x75c6311d),LL(0x9b81c6f2,0x896a0740), LL(0x073378bb,0xbad3e2f9),LL(0x29266ec0,0x2984a106),LL(0xa788010a,0xa68a5351),LL(0x017cd052,0x321aa113),LL(0x2f34db5c,0xecfb6175),LL(0xca2b51df,0xfe080ced), + LL(0x03360c88,0xba296908),LL(0x36311812,0x3fb087c0),LL(0xd9ff6a48,0xec5fb10a),LL(0x207dd8e1,0x52f7077f),LL(0x44c02fbc,0x8e65cfb8),LL(0xfbf4bfd5,0x4f4fcde2), LL(0x6cc74320,0xd1ff5415),LL(0xf68b036a,0xf989d544),LL(0x973bd9be,0xafedc2db),LL(0x4785e26a,0x7bdc3569),LL(0x751ae9ba,0x0df36796),LL(0xccdabd95,0xfebde691), + LL(0xc77488cc,0x3499b44b),LL(0xaa8f3c10,0xde000e01),LL(0xc1f517d7,0x30140406),LL(0x82a174e8,0xd4b3c8eb),LL(0xaf2c9b3a,0xc8835b13),LL(0x443716f4,0x2cd7626e), LL(0x9c22de71,0x336c4f4b),LL(0xf2529f60,0x9dd2b277),LL(0x828c34d3,0x6ffba2dc),LL(0x34d0d1f7,0x257a345c),LL(0x6ca4fbce,0xbc1eff05),LL(0x0768349f,0x05b33562), +}, +/* digit=15 base_pwr=2^75 */ +{ + LL(0x10fa53ce,0x5d7afe9a),LL(0x1ff49021,0xe2415b50),LL(0xc6523492,0xe39a067d),LL(0x27557f5d,0x34585275),LL(0x930e9f9d,0x756b8d86),LL(0x040d52d8,0x88df6219), LL(0x5362b045,0x606eb60b),LL(0x9e383cbe,0xd179818c),LL(0xe068d293,0xa6215748),LL(0x2fce158d,0x73fbdca2),LL(0xcb183c8f,0x9cfee07d),LL(0xa5e03c98,0xc0bf2beb), + LL(0xf14a99d9,0x86230934),LL(0x97c1c092,0x1cf9c66e),LL(0x6f595ed3,0x01e186ba),LL(0xe2284a58,0xd3291c3d),LL(0x1b9e5e25,0x03dee231),LL(0x15cc9f53,0xf2e9b4ad), LL(0x9770c29d,0x4fba1567),LL(0x50c4ae2f,0xbf7d6736),LL(0x2532d015,0x86901eb9),LL(0x4e7455de,0x4396fd78),LL(0xbcf811c9,0x2fbcea8f),LL(0xae952b37,0x3981ad15), + LL(0x4caea411,0x5563ba50),LL(0x59297ddc,0x6d06f7ff),LL(0xb2fb7a6e,0x244c00ac),LL(0x35df586e,0x25aac3d4),LL(0xd67be5f9,0x6c6777e4),LL(0x11c24c8a,0x476cbbf1), LL(0x83b496fc,0x60832359),LL(0x09a871a0,0xcc13c191),LL(0x56f78170,0x7c0e049f),LL(0x6eafe939,0x19bda64b),LL(0x09aa41e3,0x04381fb6),LL(0x0b8c4500,0xe4d4dcbb), + LL(0x4cb89afa,0x6bd2c54d),LL(0x36527751,0xe78c8bfa),LL(0xe3eee747,0x27f52654),LL(0x9598d907,0x56f20583),LL(0x27cb3712,0x5f91c2d0),LL(0xa3e33c5b,0xc501819f), LL(0x4eded738,0x248490aa),LL(0x27789065,0xde7ac944),LL(0x74f7d38b,0x20138b3d),LL(0x2fb60214,0xae791f60),LL(0xbd033d4e,0x6b4fb300),LL(0xbdfd1f17,0xc69c25d9), + LL(0x92717482,0x8a5cb7ae),LL(0xc49d53e9,0x3dc97730),LL(0x5302d584,0x07b1e2cc),LL(0x397b46cb,0x06617e8e),LL(0xe8e9f451,0x73426657),LL(0xdd517335,0x82dcb228), LL(0x5cad6d10,0x39cde297),LL(0x2e4b0ebd,0x76991f56),LL(0x5020cfc6,0xf3e3b3b2),LL(0xe51bbaca,0x35a63790),LL(0x4a23f2ec,0x47b21e6f),LL(0x620e0b06,0x964c6fb5), + LL(0xbe832a0c,0x9bd2b68d),LL(0xf1e49e28,0x0efa6b60),LL(0x89586da1,0x67e7f359),LL(0xfd0944ca,0xbc04c8aa),LL(0xf0569b5a,0xb8db5424),LL(0x0cc5546e,0xc27dd125), LL(0xad67b577,0x8f0ef9df),LL(0x9824cbc2,0x1b5d4068),LL(0x4e7329b8,0x2ec1eb64),LL(0xe6896b04,0x3e0e7f37),LL(0xc118b6bf,0xad2bff11),LL(0x40f93fa6,0xe17ec8dc), + LL(0x98692833,0xc7e68075),LL(0xe1257bf1,0x311dd043),LL(0xe6258442,0xd4356992),LL(0x05ddf6d2,0x7868deca),LL(0x95a0951b,0x61b3a961),LL(0x61c6fc20,0x0cda07bf), LL(0x2d061312,0xac9aa65d),LL(0xb65f1a23,0x4d9c4f20),LL(0xb06611a8,0x862e4927),LL(0x6cccfba0,0x0d1c906e),LL(0x6a49234c,0x0c82ac34),LL(0xaadbaea7,0x70f3c70b), + LL(0xbe21a890,0x0c3d2056),LL(0x8fcfba99,0x1c1ffbfb),LL(0x1fbf56ca,0x1b68a98b),LL(0x396e31cd,0x56fd85ff),LL(0x4382c03b,0xd2ca5844),LL(0x7d3ef917,0xc442030a), LL(0x426afafa,0x4129a731),LL(0x5eaae9c6,0xacff17ff),LL(0x653f3b23,0x9e854180),LL(0x9ee066bd,0xe65a1a14),LL(0x362ea5fe,0x3420084e),LL(0xc7911e2e,0x6fe58801), + LL(0x02a97a2b,0x4dd38c42),LL(0xfdb98b51,0x2a8d49c2),LL(0x5cb2e0ca,0x7034c753),LL(0xc4b59d32,0x87e0d6d3),LL(0x35514758,0x9544bb07),LL(0x4ff27eb0,0x9ed743da), LL(0xa0163883,0x1d21a1ad),LL(0x38c57492,0xd6442498),LL(0xf2fd1b7f,0xf46e8acc),LL(0xc8005cfe,0xec9932c5),LL(0xb802c109,0x5fe24f52),LL(0xe3ee9ba7,0x5539c03b), + LL(0x05a95afb,0x4c3f4f49),LL(0xf5db9d6b,0xd53a1d3a),LL(0x1844be86,0x0fbd9bad),LL(0x272afc14,0x6b3b80ee),LL(0xaa40a738,0xd727b0b6),LL(0x4d809a37,0x8a21bf45), LL(0x70767232,0x09fb0640),LL(0x45bf5d89,0xa671a2cc),LL(0xd73a211b,0xa7e5ff8a),LL(0xd9a6b97d,0xccdbf621),LL(0xc89e8f56,0x38448e2c),LL(0x2126da09,0x07e4e9c3), + LL(0x7adcc8b8,0x762e6584),LL(0x7924eafd,0xedef40a7),LL(0x5913e468,0xe1990688),LL(0x289bc581,0x1882db47),LL(0xe6084637,0x680cd99b),LL(0x1adacd71,0x5249f00e), LL(0x933bf6cb,0x0d3b583f),LL(0x9b02a63d,0x51c86774),LL(0x1dc57813,0x0be0ada7),LL(0x1e2c5f9f,0xd4b63964),LL(0x2c77c4e5,0x810b38b0),LL(0x862013cb,0x9f06b31d), + LL(0x6c4be6a0,0x43f0ae67),LL(0x504bffeb,0xabc6a17a),LL(0xdbb4492e,0xd5be6c25),LL(0x84bff97f,0x7efc9ee8),LL(0x062da2e2,0x54fbd9d7),LL(0xc6d2ac32,0x1befeb61), LL(0xcbafef5b,0x14cf6dc0),LL(0x71d12192,0x8e640e47),LL(0xd9a16800,0xd0566543),LL(0xbeb1e28d,0x9cc2ade9),LL(0x38e65833,0xcbfeb450),LL(0xd0f5acb2,0x3852eaac), + LL(0x3b5a25dc,0x8bbac220),LL(0x7deb0149,0x6e216404),LL(0x7ae10837,0x3d6eecea),LL(0xeb944203,0x2ab3cd3d),LL(0x5114d51a,0x8fdcb595),LL(0x92471196,0xfbf0f726), LL(0xb20cdb3f,0x6523ea38),LL(0x5796c8f8,0xc08ad84a),LL(0x71b564cb,0xd7977bc2),LL(0x5d66093e,0x16a9e700),LL(0x144cd814,0x67fb9a6f),LL(0xeecab0d4,0xbf2094bb), + LL(0x6c50f6a1,0x4f10299a),LL(0x69a1caf7,0xa51c6774),LL(0x0cbede41,0x024b18bc),LL(0x6b03ac7d,0x7883c01e),LL(0x1c5213be,0xa7b99e74),LL(0x85a97540,0x529fad7a), LL(0x03125409,0x0b6fe59d),LL(0x76275b90,0xf1581679),LL(0x6ad18919,0xcaa0969d),LL(0xee7761dc,0xc47afe13),LL(0xdfb12539,0xcae681fc),LL(0x02ea65f4,0x58dabc76), + LL(0x35a505bf,0x3f19ec08),LL(0xb0032e38,0x83186122),LL(0x2b3126ec,0xe5c2eac4),LL(0xf752524f,0x4368683f),LL(0x3ab28694,0x03de9cdd),LL(0x61d0253b,0xfc5e70df), LL(0x789a16d4,0x7991742a),LL(0x226ee928,0xefdb8003),LL(0xa4e62a4f,0x0468fe4c),LL(0xd07dd729,0x3ce92ccb),LL(0xfe688c1e,0x447bed93),LL(0xcae39d89,0xd5ff355d), + LL(0xc4650b85,0x5e930d65),LL(0x6350da54,0xbe96b2ae),LL(0xfa08bd49,0xcfac4f7e),LL(0xa6e10f64,0x277e8456),LL(0x407ac162,0x41be3067),LL(0x52a9b68b,0xcfd1d032), LL(0x9c337e0b,0x8d8d216a),LL(0x4e1b9cf5,0xace044dc),LL(0xc60d54c3,0xad9a4102),LL(0x28815187,0xb09420f0),LL(0x0b3b8e59,0x881179c6),LL(0x5b09aba1,0x872685ed), +}, +/* digit=16 base_pwr=2^80 */ +{ + LL(0x5852b59b,0x22313dee),LL(0xb6a0b37f,0x6f56c8e8),LL(0xa76ec380,0x43d6eeae),LL(0x0275ad36,0xa1655136),LL(0xdf095bda,0xe5c1b65a),LL(0x367c44b0,0xbd1ffa8d), LL(0x6b48af2b,0xe2b419c2),LL(0x3da194c8,0x57bbbd97),LL(0xa2baff05,0xb5fbe51f),LL(0x6269b5d0,0xa0594d70),LL(0x23e8d667,0x0b07b705),LL(0x63e016e7,0xae1976b5), + LL(0x236b71dc,0x022aa09d),LL(0xa65a7640,0xb1ce6a0e),LL(0xb38b417a,0x317344c5),LL(0x436451ec,0x29a74cdb),LL(0xa8b1c876,0xd898eb6c),LL(0xb74eeffd,0xf0134f99), LL(0x225d71f7,0x0d9eab64),LL(0xceb3cc2d,0x9679b453),LL(0x14dbff2f,0x37c894ce),LL(0x27065280,0x3704d349),LL(0xba29a0cd,0x9ee435d8),LL(0x09c11c4f,0x675bea14), + LL(0xfbecaaae,0x2fde4893),LL(0x30332229,0x444346de),LL(0x09456ed5,0x157b8a5b),LL(0x25797c6c,0x73606a79),LL(0x33c14c06,0xa9d0f47c),LL(0xfaf971ca,0x7bc8962c), LL(0x65909dfd,0x6e763c51),LL(0x14a9bf42,0x1bbbe41b),LL(0xc49e9efc,0xd95b7ecb),LL(0xb38f2b59,0x0c317927),LL(0xb3c397db,0x97912b53),LL(0x45c7abc7,0xcb3879aa), + LL(0x714a10e8,0x8b3269a2),LL(0xa4a2727e,0x64cef040),LL(0xe428865c,0xbc5ac714),LL(0xfdaba094,0x531dd17f),LL(0x18d657f2,0x86d24057),LL(0x2f99dbbf,0xe807b0d9), LL(0x6848ef88,0xc428a80f),LL(0xd0b73ce5,0xb3ef0709),LL(0x22a5d255,0xa7526919),LL(0x18a18586,0xbfe63923),LL(0xfcf633b3,0x28a0c772),LL(0x3f3c5298,0xad22b4ec), + LL(0x24359b81,0xcd81bdcf),LL(0xdb4c321c,0x6fd326e2),LL(0xf8ebe39c,0x4cb0228b),LL(0xb2cdd852,0x496a9dce),LL(0xd0e9b3af,0x0f115a1a),LL(0xd8eeef8a,0xaa08bf36), LL(0x06e5e739,0x5232a515),LL(0x8407a551,0x21fae9d5),LL(0x8994b4e8,0x289d18b0),LL(0x09097a52,0xb4e346a8),LL(0x324621d0,0xc641510f),LL(0x95a41ab8,0xc567fd4a), + LL(0x7176dd90,0xc8c9b0ae),LL(0x2917d487,0xa9560454),LL(0xe62c508e,0xb03b7946),LL(0xe9fe2321,0x60425926),LL(0x80c1d136,0x73b10bba),LL(0x9d218c9c,0xc30a847d), LL(0x2073859f,0x6ed0c8ef),LL(0x432dd97f,0xa176eabf),LL(0xb9e96167,0x3078096a),LL(0xc473e377,0xb28f0e6c),LL(0x683a3bc8,0xb44e4995),LL(0xd3523796,0x483512ee), + LL(0xd57c8de9,0x261578c7),LL(0x3836c5c8,0xb9bc491f),LL(0x14c8038f,0x993266b4),LL(0xfaa7cc39,0xbacad755),LL(0xd69b7e27,0x418c4def),LL(0xae751533,0x53fdc5cd), LL(0xc3eea63a,0x6f3bd329),LL(0xe53dd29e,0xa7a22091),LL(0xdc4c54ec,0xb7164f73),LL(0x44d3d74e,0xca66290d),LL(0x4c9ea511,0xf77c6242),LL(0x1f714c49,0x34337f55), + LL(0xfa43699c,0x93a62e7c),LL(0x1bc422d9,0xdad73890),LL(0x10cc9544,0x265e3cbb),LL(0x2f37154c,0x28cceb06),LL(0x3bf2e08b,0x6b79b071),LL(0x3ab39091,0x88e025df), LL(0x126522bd,0x50a8d04d),LL(0xb779bacf,0xeabbc1b7),LL(0xc21cc62e,0x3db4336a),LL(0x6fc00450,0x4747f0a3),LL(0x544b2d95,0x067cbf1c),LL(0xfd2be7a7,0x2480b7d8), + LL(0xa64b6c4b,0x5ed2b216),LL(0x3aae640d,0x1c38794f),LL(0x8905794f,0x30bbaee0),LL(0xc8699cfb,0x0d9ee41e),LL(0xcf7b7c29,0xaf38daf2),LL(0x43e53513,0x0d6a05ca), LL(0x2606ab56,0xbe96c644),LL(0xe9eb9734,0x13e7a072),LL(0x5ff50cd7,0xf9669445),LL(0x47da6f1d,0x68ef26b5),LL(0x23687cb7,0xf0028738),LL(0x6217c1ce,0x5ed9c876), + LL(0xb6ba9a0d,0x21965f08),LL(0x06cab532,0xa744f068),LL(0x5d4b9a92,0xd15d5fa4),LL(0x955ee1f4,0xc79fc89b),LL(0x7b2775f6,0x6367e8b2),LL(0xe2294382,0x2076cd8b), LL(0xc9089d2e,0xe5c20ced),LL(0xd76d6424,0xe6097605),LL(0x02b41ca6,0xa69f79eb),LL(0x63e07498,0x9738d39d),LL(0x92974776,0x35c14a6a),LL(0x7270fd02,0x3a000a51), + LL(0x0a3a9691,0x423ba513),LL(0xb3179296,0xf421b1e7),LL(0x1a871e1b,0x6b51bcdb),LL(0x464e4300,0x6e3bb5b5),LL(0xfc6c54cc,0x24171e2e),LL(0xd3e58dc2,0xa9dfa947), LL(0x9de9cfa7,0x175b3309),LL(0x2d1015da,0x707b2529),LL(0x993ea65a,0xcbb95f17),LL(0x0447450d,0x93515063),LL(0x1b2753c9,0x0f47b205),LL(0xe7d427cf,0x4a0bab14), + LL(0x58e996c5,0x2724c535),LL(0xd7453876,0x24010322),LL(0x74818d0b,0x3532b79c),LL(0xb27b07e9,0x1984146e),LL(0x471646df,0x948b0c4f),LL(0x32ef8e08,0x551b1c61), LL(0x25f892bb,0x22c08434),LL(0x4853c594,0x8fd873a8),LL(0xffdf802e,0xff0bdf02),LL(0x8239b779,0x527d6993),LL(0x27ace140,0xed66f25c),LL(0xa1a846a0,0x7389d738), + LL(0xb5aa7ca1,0xa39def39),LL(0xc47c33df,0x591cb173),LL(0x6bbab872,0xa09dac79),LL(0x7208ba2f,0x3ef9d7cf),LL(0x7a0a34fc,0x3cc18931),LL(0xbcc3380f,0xae31c62b), LL(0x0287c0b4,0xd72a6794),LL(0x68e334f1,0x3373382c),LL(0xbd20c6a6,0xd0310ca8),LL(0x42c033fd,0xa2734b87),LL(0x8dce4509,0xa5d390f1),LL(0x3e1afcb5,0xfc84e74b), + LL(0x12e471f2,0xd605097e),LL(0x2bb4aa64,0x8cff86ab),LL(0xea65a755,0x4a878412),LL(0x2f2b55fb,0xfa790b77),LL(0x5098a08f,0xd13fb9b3),LL(0xb359117d,0x39e474d7), LL(0xe8e5a3d5,0xc37499c1),LL(0xc7586167,0x5abf6859),LL(0x310fec8a,0x13143ba8),LL(0x75918fc2,0x3543979f),LL(0xcbb0bde9,0xf55a9b62),LL(0x29359b93,0xd6c1d000), + LL(0xf2cd8a9c,0xb028334d),LL(0x570f76f6,0xb8719291),LL(0x01065a2d,0x662a386e),LL(0x53d940ae,0xdf1634cb),LL(0x8f5b41f9,0x625a7b83),LL(0xee6aa1b4,0xa033e4fe), LL(0x1e42babb,0x51e9d463),LL(0x0d388468,0x660bc2e4),LL(0xfcbb114a,0x3f702189),LL(0xb414ca78,0x6b46fe35),LL(0x4a57316b,0x328f6cf2),LL(0x381ad156,0x917423b5), + LL(0xd52eb122,0x0233e423),LL(0x1154b0c9,0xc2848352),LL(0x6349e35b,0x2ca09cef),LL(0xded2ec54,0x3b70afc3),LL(0x52dded3d,0xc813474d),LL(0x12f00ee0,0x2d3f21bf), LL(0x92f215c6,0xa0908f76),LL(0x4e9c0440,0xb97d60e9),LL(0x34b6a8e0,0x84ad10c1),LL(0x6e7c163e,0x6f37fd95),LL(0xd057e0c3,0x7caae8c8),LL(0x553721a2,0x534f52c2), +}, +/* digit=17 base_pwr=2^85 */ +{ + LL(0x72a041b2,0xa354c1de),LL(0x29d1330a,0xe83df259),LL(0x9d532bbd,0x67661099),LL(0x52011751,0xb7c2f4cf),LL(0xf659e35e,0x6945d34f),LL(0xa1303b7b,0x6217d20b), LL(0x25751bad,0xa200ddba),LL(0x01d3566d,0xa74a7290),LL(0xaa82b46f,0x3018445f),LL(0xfccedc1b,0xc3e6a3ac),LL(0x3353e29f,0xe86ae870),LL(0xfd7e8547,0x1c8085bb), + LL(0xb4dcea78,0x306b63e3),LL(0x13636935,0x4b102092),LL(0x8bdeddea,0x36bb6889),LL(0x67a329ac,0x9331655d),LL(0xba92ccce,0x14c7fe26),LL(0xbe0519b4,0x4e7d6929), LL(0x164d50e2,0x0dc39dbd),LL(0xb1679cc5,0xd4c430a0),LL(0xfa8682ba,0xc7f78818),LL(0xb60aad97,0x43396ead),LL(0xff2c64cc,0x751784d7),LL(0x866af43e,0xd37928be), + LL(0x5a91c9aa,0xf0a69e68),LL(0x05c13197,0x0304d201),LL(0xcd14af1d,0x773a3ab7),LL(0x558d555d,0xc0b88edd),LL(0xd2e63dd6,0xeb12d197),LL(0xbcd9cdb3,0x4a8e849f), LL(0x965eaa14,0x06432985),LL(0x1a5a6f43,0x453d9386),LL(0x4171b9bc,0xbd28f616),LL(0xbbfcf90a,0x37781639),LL(0x3a36084d,0x1f93898f),LL(0xdd00ca75,0x1fefd8b7), + LL(0x2b2f1bcc,0xb9207dbc),LL(0xa3e83ef7,0x6afd6871),LL(0x34ba150b,0x49924e55),LL(0xdfec9972,0x2935ebf1),LL(0xb76f870d,0x34bf5e94),LL(0x4c20385c,0x22d0f32b), LL(0x8ccc8e72,0xc78ac172),LL(0x0ccecb0a,0x7b45b822),LL(0xcfb4b8ba,0x76c67ee4),LL(0xcd8724b6,0xecfaefb2),LL(0x340bc1ef,0xe9bc3d67),LL(0xca5541b5,0xed40b2b9), + LL(0x069e6432,0x9b8a3eb3),LL(0x721397f7,0x43aaa7fc),LL(0xa7e83a71,0x46e23c6c),LL(0x93fa3c25,0x71b261d5),LL(0x0f523a72,0x4a47a105),LL(0x8dcad752,0x31919e89), LL(0x0c5dd2ad,0x4c8b06e7),LL(0x8bdc55e7,0x677ec5f3),LL(0xcb1b5828,0x4372d55d),LL(0xf04dd321,0x7bf054c1),LL(0x2e44584e,0x4e8c1a99),LL(0x51d35d78,0x68078037), + LL(0x81d311b3,0x25ae3499),LL(0x3b16037b,0x8640f52a),LL(0x1d947065,0xac083994),LL(0xe2e693d2,0x3723c75e),LL(0xb66f429e,0x65040a51),LL(0x035a3a53,0x7f582b0b), LL(0x0a166da6,0x20eca9e1),LL(0x2c4cc565,0x45b37e20),LL(0x7a8a96e3,0xeab88295),LL(0xb60a1a1b,0x99e771da),LL(0x23b03965,0x2cdd778c),LL(0x91052478,0x8d4d7a72), + LL(0x7e8f73b5,0xa52d3c2d),LL(0x51842657,0x86d80633),LL(0xb3949eba,0x58f01253),LL(0xe79367d0,0x97689f15),LL(0x0d820328,0x918bf9a3),LL(0x41c959df,0x2d4bc994), LL(0x8c16ee54,0x37392f6e),LL(0xe6f0849b,0x9f726d58),LL(0xb8208f08,0x497de1e4),LL(0xd51a29b3,0x60c51233),LL(0xc9e1d465,0x0f61fb03),LL(0xfbe2613c,0x09494bd0), + LL(0x154d0f99,0x4f293478),LL(0xd07a24b3,0x1b82320d),LL(0x64d55f6f,0x1bf7c94f),LL(0x725c5125,0x4489b57d),LL(0xb1b6a091,0x3aa4d43a),LL(0xcf7a60fa,0x054842bd), LL(0x2aeb4cb6,0xaa918a4d),LL(0xac7d317b,0xcbdaff99),LL(0x6812a03c,0xed0e00a1),LL(0x0b0a1e4b,0xb09acf27),LL(0xac28386b,0xc73a41f7),LL(0xf4cd1321,0x43134dbd), + LL(0xd8c33924,0x08336ffe),LL(0x5140b253,0x15b56cbf),LL(0x306caedb,0x38dcd310),LL(0x47944afd,0x04ecd496),LL(0x68a48f95,0x1280d23f),LL(0x34363c6e,0xf4142204), LL(0xcaa8717f,0xd0a397ea),LL(0xc3994b80,0xb51a1669),LL(0x6c56808b,0xa02eed91),LL(0x83545c3c,0xc3ab55c5),LL(0xfd26114a,0x8b835820),LL(0xffff324c,0xe0cfa4a6), + LL(0x51c0d95e,0xcf63d279),LL(0xac86a014,0x3b170a0b),LL(0x881095e1,0xc21eaaa9),LL(0x6069a3eb,0xed2fda11),LL(0xbd2f1c5a,0x536264b9),LL(0xde312c2c,0x819e1cff), LL(0xdfd6ce38,0x6c30f983),LL(0x980b439e,0x2f32cc4c),LL(0x3b9c03b2,0x9fab10b6),LL(0x011ab74a,0xdfebe34e),LL(0xb80963f6,0x587360e3),LL(0x8692e352,0x3db1f610), + LL(0xfe3d070b,0x63a094c5),LL(0x88515eb1,0xf769b919),LL(0x50d1131d,0xafe86e14),LL(0x6774d3d4,0x6bf27788),LL(0xffd805d0,0x7231d699),LL(0x6304116e,0x05132e5b), LL(0xe34ce5bc,0x3d5e255b),LL(0xc95e3089,0xfd9c3bd0),LL(0xb83cbac9,0x22a24023),LL(0xb0b3b98a,0xfb6d2b6f),LL(0xf7e36fcd,0x74af1115),LL(0xf9da3bf0,0xcfe15eaf), + LL(0xbb2fcdae,0x266d2c1c),LL(0x52be93f2,0xb538d4a2),LL(0x73bd0094,0x774c88ba),LL(0x81a7e042,0x65283a9b),LL(0xd0381625,0xe1438bbf),LL(0x4d0db206,0x450e1f64), LL(0x4e60fc4a,0xb38ae9ef),LL(0x12719817,0x14ce87e1),LL(0x570303f0,0x831d41ec),LL(0x28850444,0x71729170),LL(0xccd609f5,0x2077ea32),LL(0xcd273fdc,0x091d1166), + LL(0x4b2ca517,0xdf216dd8),LL(0xc6b74c4d,0xb3eec4b9),LL(0x1c14e77b,0xf564e6c8),LL(0x2c2c9395,0xcde25f1c),LL(0x049fcc83,0x7e31f7a5),LL(0x9284c753,0x6913707b), LL(0x58e6eb5f,0xb92a6f24),LL(0x95148292,0x85b0cab5),LL(0x7449be92,0xeaad036d),LL(0xeb94a702,0x2f6a2888),LL(0x47d59fb0,0xd7d8773d),LL(0x03c0bf25,0x612d2573), + LL(0x84d2c3bb,0xb872a105),LL(0x39196026,0x44bca571),LL(0x4e352e5d,0x857327d8),LL(0xd925f99f,0xa6c6004a),LL(0xbab79ead,0x48aaf266),LL(0xadab2a3f,0x213ad923), LL(0xf371cc48,0x3be29b6d),LL(0x2385c9f4,0xe732b906),LL(0x23f0a84e,0x562e0be1),LL(0x28c4b0db,0xbb6b0172),LL(0xf4c6d8be,0x71a93ae5),LL(0x551f1fe9,0x76b8bb16), + LL(0xc39926a8,0xbbfaaa94),LL(0x60a138aa,0xb9a59fdb),LL(0x947e30e9,0x217a1aa2),LL(0xc52c9fff,0xcac988c9),LL(0x3bae3c39,0x5676473a),LL(0x857f04c9,0x7d84b353), LL(0xdd324e24,0xdeded30c),LL(0x9c242899,0xf07c678a),LL(0x8cb64f3b,0x956d0553),LL(0x502cb2b0,0x9d34e2f5),LL(0x51dd03b3,0x99e1054a),LL(0x4d60a593,0x86b8bfa5), + LL(0x754d15e9,0x10a53b90),LL(0x5f4c7218,0x6cde9a0c),LL(0xabef2b96,0x740d513f),LL(0xd3f802fd,0xff6cc47c),LL(0xeb0627af,0x1be6825b),LL(0x5886c2dc,0xdb21ede5), LL(0xf5daaed7,0xb6cfb2c6),LL(0xfae29a9c,0x68b61aa8),LL(0x3a5a485b,0x7a1e16f5),LL(0xe7b2223e,0x16b60b92),LL(0x36a13a9b,0x332f33d8),LL(0x876cd1a2,0x4567c313), +}, +/* digit=18 base_pwr=2^90 */ +{ + LL(0x4aea3fa2,0xa896d28e),LL(0x6db06ee9,0xc6137a45),LL(0x06fb15cc,0x1bbafe8c),LL(0x1cdffdad,0x2daab296),LL(0xe1119b3a,0x984defc8),LL(0xde2a25a3,0x9cd44c3c), LL(0x54ed6d73,0xa7f54ece),LL(0x50907054,0xd283017f),LL(0x6a3b9442,0x69130efc),LL(0x6785163b,0x5d17f127),LL(0x172b1d0a,0xc019911b),LL(0x7e3e093c,0xa19c745f), + LL(0xab83d932,0xe185bdc2),LL(0xd7c4e754,0x0a75845d),LL(0xc3fe5695,0x1f6f3397),LL(0x61f6a04f,0x6c9f3a5f),LL(0xb390a92b,0x3c0f9d4b),LL(0x4793b454,0x9e3336b7), LL(0x1472f06b,0x91ad0c34),LL(0x892cbdd7,0x4110047a),LL(0x65d53c83,0xfa24d905),LL(0x4176007d,0xd63e5833),LL(0x2cd1623c,0x741089fd),LL(0x2685d345,0x6b3d9202), + LL(0x3586c802,0x1ac37f31),LL(0xa9cb8662,0x70daca37),LL(0x8d9c7bf6,0x6e57d6a7),LL(0xe97eec9b,0x27069e2c),LL(0x4e877a82,0x16284ceb),LL(0x774f6288,0x30856e26), LL(0xfa33820f,0x88e53ea7),LL(0x715ced07,0xbfe3b89f),LL(0x743b11c6,0x6cf55589),LL(0x2edd201e,0x020f8a09),LL(0xb15c27e0,0xc80afc2c),LL(0xcc53751a,0x56557371), + LL(0xc9cb7f6d,0x1d510157),LL(0x46ab7372,0x532a0773),LL(0xc6dde9e2,0x2ea07e2f),LL(0x37d5bb1f,0xceed9ad9),LL(0x98cc6e28,0x3121994b),LL(0x67ad8fc4,0x67d2fbb5), LL(0xdc9f195d,0x34707fb3),LL(0x1fd5a013,0x6a601f48),LL(0x81ef6cb5,0xfe939b8d),LL(0x1223a9a1,0x5c51e8ab),LL(0xdb74cf37,0x8f6d7993),LL(0x972808e1,0x0b81c5b7), + LL(0xe26e1212,0xcc54c384),LL(0xfee6836b,0xf4c6a3cb),LL(0x91cccc5d,0x62589405),LL(0x135a7e68,0x5ed3f3ce),LL(0xe1994768,0x11cc139f),LL(0x77c07147,0x6386c5c4), LL(0x75fb9b2d,0x444230ae),LL(0x8029bfeb,0xa5972fd9),LL(0xdb5f8291,0x46e687ec),LL(0x0de6d5f7,0xf00bf689),LL(0x36af6be3,0xbf946c50),LL(0xd39dee1d,0xd6d9aaef), + LL(0x3bf921af,0xcb4e8512),LL(0x532e81d3,0x28fc6332),LL(0xf69f907d,0x682d8637),LL(0x5f759a16,0xbd9fa8f4),LL(0x51f03716,0x091ea9fa),LL(0x32c630e9,0xd685a141), LL(0x3d249cf4,0x7600c9ac),LL(0x002cd2b5,0x687e2022),LL(0x55334058,0x7ec205ab),LL(0x3ecf1368,0x9d0d86b1),LL(0xfc7baf6d,0xb3fc17a7),LL(0x361c91cd,0x57939961), + LL(0x37318b6d,0xeb74751b),LL(0x5abe7213,0xb967a3cf),LL(0x02ab79cc,0x80741539),LL(0xafbc3e08,0x11b647ae),LL(0x9e949616,0x3e34458e),LL(0x72591fad,0x6d714d62), LL(0x131b7659,0x813ba91c),LL(0xfd0e295e,0x7f836d36),LL(0xc5d7be99,0x192531f8),LL(0x901e480e,0xce83f561),LL(0xf1bba4dd,0x7b187da4),LL(0xf5c82a33,0x090754d0), + LL(0x010c0754,0x0db33228),LL(0x8eca7c59,0x10635ffa),LL(0x0e8a38fa,0x6efd8538),LL(0x769360d8,0xc1812ea5),LL(0x76f27ef5,0x505723dc),LL(0xf35af2e8,0xd0358e02), LL(0xd99419ee,0x9f7bb7fe),LL(0x430a0e2d,0x87c66e83),LL(0x773eaf7f,0x01187549),LL(0x89d51bda,0x05bbbba4),LL(0x640ccde6,0x52cabb06),LL(0xe7ff387d,0x0d5cb557), + LL(0x526f59f3,0xa338a9f7),LL(0x216004f8,0x9b866285),LL(0xeb6fcc58,0xe94cfd05),LL(0x2432ba24,0xc60b12ea),LL(0xef1227f5,0x3bb7acf1),LL(0x5ec503dd,0x176e0ef7), LL(0x236f9707,0x9c2337f7),LL(0xe029aa27,0xdc2b0f77),LL(0xa5f85372,0xb795424a),LL(0x1485af98,0x2eb6d54b),LL(0x7e3f46c7,0xd2ce87dd),LL(0x830f743c,0xc5eaf6ad), + LL(0x10e06f1a,0x709d61ca),LL(0x78eba75c,0xaa1e9fc5),LL(0x914b2cfc,0xf85d062f),LL(0x9089d85c,0xe73b3baf),LL(0xc4a284b9,0x4ac05fea),LL(0x3acb7268,0x92c78a43), LL(0xee45bb4d,0x7b5586f8),LL(0x6ac0a9e7,0xc39a0d0e),LL(0x4d6f9ab8,0xe4bbe3d5),LL(0x1fd46a08,0x1489463f),LL(0x29dba364,0x3ba31825),LL(0x8138511b,0x94f000d6), + LL(0xee9b6d83,0x9f8eafec),LL(0xf4eac676,0x069d5ad9),LL(0xb5687ebb,0xbf9ccc6e),LL(0x9d213f1d,0x5d0f44be),LL(0x0d0d6e70,0x372b5296),LL(0xf8bb90f3,0xef1466ef), LL(0x949789b2,0x2ec1daae),LL(0xb40be288,0x5c7d9b6d),LL(0xc351e87f,0x48319460),LL(0x5db0fde9,0xfeaa721c),LL(0xd16dc699,0xb33d58e9),LL(0x3ca8d3d6,0xe3f296f2), + LL(0xc39c1cef,0x70187dfb),LL(0x0c50c71e,0xa785216e),LL(0x6a6c0d60,0x30188b81),LL(0x6a27e97b,0xeaeda67d),LL(0x6ba389aa,0x4a519282),LL(0xa2bf1273,0xb96c7c7e), LL(0x267fe714,0x8ff10657),LL(0x996d91b7,0xdff4a271),LL(0x1dc7aed4,0xe34ba3e1),LL(0x38853d61,0xc457048b),LL(0x1ccbf658,0xe89825db),LL(0x6b255edd,0x68c7b455), + LL(0x49d38208,0x2f6f6a0f),LL(0x11ee2340,0x38fbcb5b),LL(0x865a4304,0x06e43b73),LL(0x06d728ad,0x08c696e6),LL(0x961c650b,0xdb82930d),LL(0xc20c5b68,0xf1de8cb5), LL(0xf8e1f9b9,0xd8e3cf7b),LL(0xf3a304ed,0xd6e081ab),LL(0xcd3cb8bb,0xf08d8624),LL(0xb26aa9cb,0xa3fe7742),LL(0x0de295f6,0xa3a9d220),LL(0xb9579b7f,0xfd465046), + LL(0x74871e18,0xdc14cb2a),LL(0xfcb8974a,0x017b1340),LL(0x6e93c20f,0xea5cb054),LL(0xa9e2ad1f,0xa7c078ad),LL(0xbeb26838,0xa37207d4),LL(0xde7ee8ee,0xcd8b3b25), LL(0x2801a7ff,0xdca6606a),LL(0x0f8af3fa,0xad2fedcf),LL(0x9b530c05,0xf27d30b4),LL(0x6b2a4613,0x071fc1c3),LL(0xb72cea9f,0x363aaa99),LL(0x3d350374,0x7a33ed8f), + LL(0xf2f3e9ea,0xac02833f),LL(0x956fb86e,0x43bcbde0),LL(0x95735c62,0xf888696b),LL(0xe7b2fd9e,0xefa03aae),LL(0xe75b2684,0x0e563e92),LL(0xd7f335bd,0xa49d2f31), LL(0x8ac31404,0x08664171),LL(0x07ccf31f,0x35866eab),LL(0x04122373,0x6bca6111),LL(0xcb21398b,0x730c92e5),LL(0xb93102d5,0x283791a8),LL(0x50dc1b38,0x4b41d94d), + LL(0xbb20fabf,0xc377b373),LL(0xf986b847,0x68d3aa52),LL(0xf39b6894,0xd9c2f2ad),LL(0xbd6da22e,0x1bbff106),LL(0x7e09678e,0x3f7e5b8e),LL(0x7ed3ee78,0xad6a8789), LL(0xaf9807b9,0x689e6b31),LL(0xbd1f6ef2,0xeca87778),LL(0xdda78c54,0x17d3277e),LL(0xefb65cb7,0xe686cacc),LL(0x19a30f0c,0x758aa1ab),LL(0xb11f071e,0xb40df97f), +}, +/* digit=19 base_pwr=2^95 */ +{ + LL(0x953b9223,0x16f3708b),LL(0x770e7cf3,0x0d3780f8),LL(0x27bb71a8,0x97a615b2),LL(0x162f8b55,0xa8b9a864),LL(0xd91e3fb9,0x80ee8362),LL(0xf83a4ff6,0xb2009a09), LL(0xc1696281,0x07a7873a),LL(0x23095ddc,0x17ff00c2),LL(0x860d60ba,0x427f683d),LL(0x1f87d32a,0xea995927),LL(0xa050319d,0xb2ac69fa),LL(0xd2d0b9ce,0x30c362b9), + LL(0xf26feef9,0x24480c57),LL(0x3a0e1240,0xc31a2694),LL(0x273e2bc7,0x735002c3),LL(0x3ef1ed4c,0x8c42e9c5),LL(0x7f4948e8,0x028babf6),LL(0x8a978632,0x6a502f43), LL(0xb74536fe,0xf5f13a46),LL(0xd8a9f0eb,0x1d218bab),LL(0x37232768,0x30f36bcc),LL(0x576e8c18,0xc5317b31),LL(0x9bbcb766,0xef1d57a6),LL(0xb3e3d4dc,0x917c4930), + LL(0xaae4bfd4,0xff41d51f),LL(0x17c44fac,0xcf50b141),LL(0x657a1ea4,0x078b808e),LL(0x93c00c55,0xc5aac1a8),LL(0xcc4d1c0f,0xcb99cfd0),LL(0x3fa123a6,0x1d104893), LL(0x023ca92f,0x49646059),LL(0xf3982134,0x5833e326),LL(0xc5781cdd,0x2e0d4bc9),LL(0x8d5e75f5,0x5f7f84ed),LL(0xe1e8a383,0xb6655f1f),LL(0x296e4943,0xcc18514c), + LL(0xb475d8f3,0x53ebbaae),LL(0xff76beda,0x3d6ea31c),LL(0x340986b4,0x3c15f25d),LL(0x3365312a,0xc5925d2e),LL(0x51641f96,0xc35d3ee2),LL(0x984128e4,0x11eb2f75), LL(0x3d04bc99,0xb41a21a8),LL(0x6436c3d0,0xf2d28600),LL(0xfaf5663c,0x4ffcf4c0),LL(0x0a62c9dc,0x889d285a),LL(0xcb2d60c5,0x0908665a),LL(0x0a131be5,0xe2f19c59), + LL(0xb1e46617,0xa93a7dbc),LL(0xd77195a7,0xd0ad4a47),LL(0xe5948165,0x020d8467),LL(0x25267b60,0xa3375bd9),LL(0x8c44e226,0x8cf54ab2),LL(0xf2bf5bfe,0xc4f7f467), LL(0xc414077a,0xde336f92),LL(0xc92d7219,0x92656952),LL(0xc09ee1d9,0x31645a70),LL(0x292630eb,0x8cfe567d),LL(0x7c16c0a8,0xb835edb9),LL(0x48dee1be,0x8ac88e9c), + LL(0x22319bfb,0x318feb4c),LL(0xa1ee9625,0xfd0a1331),LL(0x5b238661,0x1e4a786d),LL(0xa722c591,0x88e04305),LL(0xf406cb01,0x38eb062a),LL(0xe7216364,0x21caa381), LL(0xf0e1f665,0x450c1d29),LL(0x207a1320,0x369af7bf),LL(0x6f6c0680,0xfe46a53a),LL(0x25eac032,0x4553199a),LL(0xffc49722,0x41fa659a),LL(0xbbcb7a29,0xfb9e0c73), + LL(0x7e09207e,0xca53bd85),LL(0xfa171b01,0xa304cf7d),LL(0x881beffa,0xa93499d7),LL(0x95cf1295,0xc0b04ee5),LL(0x90cd1e30,0xe0cf548e),LL(0x821b4efc,0xfa82436e), LL(0x979ab01d,0xbb4d7958),LL(0xd48ca82b,0x4a1c815b),LL(0x627640fd,0xbacf6a1b),LL(0xabd9a758,0x31150946),LL(0x906bea56,0x2c9d7d14),LL(0x17e06ed1,0xd2450bcf), + LL(0x091354ff,0xc0de60f5),LL(0xa1bd1975,0xd7cc38bb),LL(0xe734e2df,0xf4122aa8),LL(0xef773db6,0x08f40f63),LL(0xce2d71c5,0x0a7e9484),LL(0x78a3f825,0xcc791733), LL(0xb47beec9,0x0cac7a5b),LL(0xa3f7b5b6,0x1cbea0e4),LL(0xd3e18145,0xecf19a90),LL(0x0aadf689,0x0d1b062a),LL(0xf3f0acf7,0x42299f1f),LL(0x5ac252b9,0x63a64539), + LL(0x8db83d7d,0xfda26247),LL(0x2ad24ac5,0x3a2d1c29),LL(0xa01d0daa,0x586b6219),LL(0x949dbc96,0xa92f773b),LL(0x5dc355d0,0x89c9668d),LL(0xb5e40672,0x0aaeecdf), LL(0x2c701014,0x5945a47f),LL(0xc5590e89,0x50e4a494),LL(0x7fd21edd,0x307cbd9f),LL(0xb85543b7,0x96005378),LL(0xba214861,0x9f87fddf),LL(0xf9d9d2c7,0xcdd615c5), + LL(0x5bd11a42,0xa4bfe151),LL(0x0ea6729b,0x38920da2),LL(0xa0ee708f,0x41e28260),LL(0xabc9d5f5,0xff4fdff4),LL(0xffaae99e,0x6ed92241),LL(0xc04fe4d9,0x6075ce0d), LL(0x5db066f5,0xf10a173e),LL(0xe75ef129,0xa2edee12),LL(0x8ed02e85,0xd2a0823f),LL(0x2e522dc1,0xffa78cf4),LL(0x00c939fd,0x07041e46),LL(0x3a9a8bba,0x3369357f), + LL(0x1a4e68b1,0x3b202533),LL(0xf7504686,0xa2ead2d7),LL(0xa569cb36,0xb0bb1808),LL(0xc015c68f,0x953f539c),LL(0x9a6fefe1,0x45c876dc),LL(0xbede60ec,0xe87ce8ac), LL(0xf3c69642,0xecfc84dd),LL(0x3a90826b,0xabe549ed),LL(0x57cca429,0x03a29df4),LL(0xb8e72b83,0x08d4e9a1),LL(0x79e7ef57,0x88d9c51b),LL(0xdc075e38,0x48102f0e), + LL(0xba264fa8,0xa1fc20f7),LL(0x4105e64a,0x0f625415),LL(0x8e2e8e16,0x2e5581c2),LL(0x4c456234,0x912c80fa),LL(0xe9e31ad4,0x7fd61fb9),LL(0x1c19e811,0xa5de4e7d), LL(0x2961b64d,0x6a7ab2cc),LL(0x5bb2da03,0xbf06db25),LL(0xab06af4d,0x33f76a85),LL(0x4866378d,0xda387e54),LL(0x56826f8c,0xbb71deb8),LL(0xeb64ecb6,0xbbe3ce38), + LL(0xbb63bdd0,0x2a6f3d84),LL(0xc2792d41,0x8b5becec),LL(0xec9fa26b,0xb0412202),LL(0xbc706607,0xbdaad566),LL(0xd8d361d4,0xd34e3ef1),LL(0xeb3c3f14,0xb4fe020e), LL(0x82656dd4,0xc6bac95d),LL(0x88ffb328,0x8cf56ba0),LL(0xdc84969a,0x33f04dda),LL(0xa5f1d0eb,0x642d93d6),LL(0x93ce88dd,0x122e35f6),LL(0x02c0e82a,0xbcffc369), + LL(0xfd5d084c,0x0e935934),LL(0x9121a6e0,0x7cd4992a),LL(0x8e15d863,0xab773dba),LL(0xcab64644,0x9cea4a51),LL(0x2efff061,0x516754d7),LL(0xcd3a36a4,0xd8af89da), LL(0x4615774e,0xc7d352ac),LL(0x21ae0d27,0xd1bb914b),LL(0x9199938e,0x8a8aed97),LL(0xcd6f3495,0xeb06789a),LL(0x775f93ee,0xc51d7766),LL(0x0a8af851,0x7eb6909f), + LL(0x6a82fd92,0xf6708309),LL(0xfffaccf2,0x08652b2b),LL(0xf0da4c5c,0x8ac0e67b),LL(0xb0fabc5d,0x26c5a4be),LL(0xb1dcad44,0x3e5ad350),LL(0x3aa30664,0xb539e10e), LL(0xa41fef81,0x266f4163),LL(0x78ef116f,0x860401ec),LL(0x3563575b,0xb0fc5cc3),LL(0xd4c9e03a,0xc28a5cb5),LL(0xcc9bbe6c,0x93399eff),LL(0x9c9a84a2,0x50a48cb2), + LL(0x5c1d4586,0x44bfd166),LL(0x8d1d86d6,0x46434e19),LL(0xc3936683,0xe50fcf81),LL(0x8b08680f,0xc9b4eb06),LL(0x2832aab0,0xf90882c5),LL(0xecbf5dda,0x42823cef), LL(0x44ae08f0,0xfd4d51c7),LL(0xbbd21c1c,0xb54a08f1),LL(0xfb187c34,0xb72953db),LL(0xf8ed037f,0x166f7f26),LL(0x097bad45,0xd2b1077a),LL(0x790dd808,0x47794cdc), +}, +/* digit=20 base_pwr=2^100 */ +{ + LL(0xd1d1b007,0xa05c751c),LL(0x0213e478,0x016c213b),LL(0xf4c98fee,0x9c56e26c),LL(0xe7b3a7c7,0x6084f8b9),LL(0xdecc1646,0xa0b042f6),LL(0xfbf3a0bc,0x4a6f3c1a), LL(0x51c9f909,0x94524c2c),LL(0x3a6d3748,0xf3b3ad40),LL(0x7ce1f9f5,0x18792d6e),LL(0xfc0c34fa,0x8ebc2fd7),LL(0x780a1693,0x032a9f41),LL(0x56a60019,0x34f9801e), + LL(0x62c36887,0xa4b957d2),LL(0xfc24cff8,0xaf15a485),LL(0xa271d9e0,0x11575e80),LL(0x4b9367e5,0x0fff68d4),LL(0x2279779f,0xf55ba673),LL(0xd4d68f68,0x9d72cca6), LL(0x590ffe4c,0x01474ab1),LL(0x074d634b,0xd20f44e1),LL(0x36111d25,0x63903a83),LL(0xab531cef,0x37342a5f),LL(0x702ed867,0xd3c93fe7),LL(0x6279f7e1,0x05d14369), + LL(0xf0db3751,0xb398290c),LL(0xba42c976,0x01170580),LL(0x56560b89,0x3e71aa29),LL(0x50e6647b,0x80817aac),LL(0xa0be42da,0x35c833ad),LL(0xf1baba4e,0xfa3c6148), LL(0xcd8f6253,0xc57be645),LL(0xc657ad0d,0x77cee46b),LL(0x0defd908,0x83007731),LL(0x899cba56,0x92fe9bce),LL(0xbceffb5a,0x48450ec4),LL(0xf2f5f4bf,0xe615148d), + LL(0xcbaf4685,0xdccf68bc),LL(0x270a2bcc,0xb333e464),LL(0x254dd3e3,0xe43ae199),LL(0xddce5c84,0xe8526e26),LL(0xea0b4258,0x52bad815),LL(0x094574c4,0x67c12c1b), LL(0x861545b7,0xa5362fcb),LL(0xc2b2eb62,0x3e904c35),LL(0xeeffc2cd,0x0f9312b5),LL(0x14de4e5b,0x5475657b),LL(0xf0233fa5,0x746e67d4),LL(0x35471ec2,0xb5157d7f), + LL(0x90b86166,0xf55edabb),LL(0x075430a2,0x27f7d784),LL(0x9bf17161,0xf53e822b),LL(0xafe808dc,0x4a5b3b93),LL(0xd7272f55,0x590bbbde),LL(0xeaea79a1,0x233d63fa), LL(0xfe1eba07,0xd7042bea),LL(0x10750d7e,0xd2b9aea0),LL(0x31078aa5,0xd8d1e690),LL(0x7e37bc8b,0x9e837f18),LL(0x85008975,0x9558ff4f),LL(0x421fe867,0x93edb837), + LL(0x1f23a0d8,0x3503d937),LL(0xc321dde0,0x64c598a8),LL(0x5b52e0f0,0x67f101ef),LL(0xf955b5fa,0xb6b5b4c2),LL(0x880e0569,0xb5f03d53),LL(0xc99393ef,0x121c3ac1), LL(0x57330666,0x90755bd6),LL(0xd4d71d3d,0x70ae5793),LL(0x9e9ce792,0x326ffd51),LL(0x96ccfa14,0x1b772d73),LL(0x874a22de,0x652710f4),LL(0xdb210342,0x72768469), + LL(0x83d55b5a,0xaa6489df),LL(0x86bf27f7,0xea092e49),LL(0x5fa2efec,0x4d8943a9),LL(0x720e1a8c,0xc9baae53),LL(0x95a4f8a3,0xc055444b),LL(0xa7c1206b,0x93bd01e8), LL(0x714a27df,0xd97765b6),LL(0x193f1b16,0xd622d954),LL(0xf1503b15,0x115cc35a),LL(0xa9fa21f8,0x1dd5359f),LL(0x6dfed1f1,0x197c3299),LL(0xf77f2679,0xdee8b7c9), + LL(0x171c1439,0x58d6998e),LL(0x01feedec,0xfd4a98f4),LL(0x65739fce,0x420b2a01),LL(0x22f7a073,0x5c5db308),LL(0x05042f00,0x016c5478),LL(0xa12413d9,0x5fc73ce2), LL(0xe932aa17,0x8ceb2d70),LL(0x0537afaf,0xb4d66b67),LL(0x339c146b,0x2638d012),LL(0x28ac0555,0x02fbb7b6),LL(0x62d46e63,0x7fcb0c81),LL(0x066d088e,0xeaa9ff4f), + LL(0x394fd855,0x5405179f),LL(0x49fdfb33,0xc9d6e244),LL(0xbd903393,0x70ebcab4),LL(0xa2c56780,0x0d3a3899),LL(0x683d1a0a,0x012c7256),LL(0x80a48f3b,0xc688fc88), LL(0x6f7df527,0x18095754),LL(0x71315d16,0x9e339b4b),LL(0xa956bb12,0x90560c28),LL(0xd42eee8d,0x2becea60),LL(0x50632653,0x82aeb9a7),LL(0xdfa5cd6a,0xed34353e), + LL(0x2d189057,0x8273db99),LL(0xe1b5f8cc,0x4d1b05fc),LL(0x0a7c32d1,0x5fec7c83),LL(0xea9b4d45,0x28ddaf28),LL(0xa2fc58be,0xb6bb62ac),LL(0x4a41852d,0xfc65b7aa), LL(0x1c9e6045,0x6e765194),LL(0xfc116257,0x3acabf28),LL(0x4b5a4ba8,0xc9d5e805),LL(0xcbdcf1eb,0x9a072259),LL(0x439fc8fc,0xc67cf643),LL(0xb4333aa8,0x917ef6f8), + LL(0x91aecce4,0x82154d2c),LL(0x5041887f,0x312c6070),LL(0xfb9fbd71,0xecf589f3),LL(0xb524bde4,0x67660a7d),LL(0x724acf23,0xe99b029d),LL(0x6d1cd891,0xdf06e4af), LL(0x80ee304d,0x07806cb5),LL(0x7443a8f8,0x0c70bb9f),LL(0x08b0830a,0x01ec3414),LL(0x5a81510b,0xfd7b63c3),LL(0x453b5f93,0xe90a0a39),LL(0x9bc71725,0xab700f8f), + LL(0xfc37efed,0x311ebba2),LL(0x60cfd6bc,0x8a6a42d6),LL(0xf2a4871e,0xb4051b3a),LL(0xc2f0ebf0,0x66ce77b8),LL(0x0ad28477,0x84abc948),LL(0x63d9d11a,0xc82e5c62), LL(0x007dcf93,0x99ffc70c),LL(0xd964c822,0x5e974edf),LL(0x513085e3,0x0fee3572),LL(0x46ce8444,0xbe67a880),LL(0x06d17129,0x136ceeb8),LL(0x662d86fd,0x0da512ae), + LL(0xb9f00793,0x9401aec2),LL(0xb997f0bf,0x064ec4f4),LL(0x849240c8,0xdc0cc1fd),LL(0xb6e92d72,0x39a75f37),LL(0x0224a4ab,0xaa43ca5d),LL(0x54614c47,0x9c4d6325), LL(0xc6709da3,0x1767366f),LL(0x23479232,0xa6b482d1),LL(0x84d63e85,0x54dc6ddc),LL(0xc99d3b9e,0x0accb5ad),LL(0xe8aa3abf,0x211716bb),LL(0x69ec6406,0xd0fe25ad), + LL(0x57e0105d,0xe7acaa84),LL(0x3851fd57,0x3c06d3bd),LL(0x2a9c631b,0x23cf3c61),LL(0x33863bf8,0x13888aaa),LL(0x717783ee,0xf2396355),LL(0x36b300e1,0xf21e1a48), LL(0x9d27b4cb,0xa734cb3b),LL(0x796e34b6,0x0a7effed),LL(0x3615cc7a,0xfc586477),LL(0x88844a21,0x1f98ed77),LL(0x7ad4c7bd,0xd6e28940),LL(0xe9331c7e,0xa00d64eb), + LL(0xdf85c705,0x0d5c1769),LL(0xa409dcd1,0x7086c93d),LL(0x0e8d75d8,0x9710839d),LL(0xebdd4177,0x17b7db75),LL(0xf649a809,0xaf69eb58),LL(0x8a84e220,0x6ef19ea2), LL(0x65c278b2,0x36eb5c66),LL(0x81ea9d65,0xd2a15128),LL(0x769300ad,0x4fcba840),LL(0xc8e536e5,0xc2052ccd),LL(0xac263b8f,0x9caee014),LL(0xf9239663,0x56f7ed7a), + LL(0x713b8541,0x17e3d0b8),LL(0xc6b5e839,0xf372b048),LL(0xd0bb1848,0xf8ef0261),LL(0xc71a3bbe,0x9b804cee),LL(0x542a88ae,0x00b7d171),LL(0xe9097b9e,0xf2b8ed10), LL(0x2c0a009a,0xdbad9f12),LL(0x205fb1bf,0x245fc1e9),LL(0xb83debf5,0xa8a4834f),LL(0x637e449b,0xc3ee226d),LL(0xcab82664,0xe3070d93),LL(0xb37320e8,0x24b8094d), +}, +/* digit=21 base_pwr=2^105 */ +{ + LL(0x25059699,0xb506b7b9),LL(0x01ab02e5,0x349fd83f),LL(0x3789281e,0x64b729ad),LL(0xf9af4561,0x69ae8f81),LL(0x56f91860,0x007befe1),LL(0xedc250fb,0xb578c566), LL(0x67ae4801,0x1c16d75a),LL(0xa1d3f592,0x04c35a7e),LL(0x36881f89,0x5dc97da9),LL(0x03a5b1ea,0xaddb6031),LL(0x3e153a0f,0x7eb515b1),LL(0x2b4a1ee2,0xdc3a9219), + LL(0x512cc92a,0xe7365f9e),LL(0x5172a654,0x9efdcf8b),LL(0x8e611fc3,0xbfb389ac),LL(0x699c227d,0xce778fd5),LL(0x3ff2ef17,0xdc1f47b6),LL(0x02672ed4,0x2ae0f683), LL(0xa8e879cb,0x51c63806),LL(0x3909f526,0xd3dfecf0),LL(0xe00e12a2,0x375b3d13),LL(0xbf8df325,0x91f9f750),LL(0x1df5f21a,0xf1ea0e42),LL(0x32c60584,0xaed73e7f), + LL(0xe0728e6d,0x03b07fb5),LL(0x1012234e,0x9e046920),LL(0x35637644,0x272e6449),LL(0xa55bcadf,0x2b6ad1c2),LL(0x5c71c6af,0x86c52776),LL(0xc1678806,0xa25bd60b), LL(0x76bb32f8,0x0cae8294),LL(0x33e03cb2,0x389ce4e6),LL(0x7513dfb6,0x504df833),LL(0x1b351dda,0x4260ee8e),LL(0xdbaf7cd0,0xa473c5d9),LL(0x71e390fd,0x22cb7cb4), + LL(0x26caebd8,0x1d9aa9fa),LL(0x26b7a673,0x6b646869),LL(0x7f167b47,0x7ebed6a1),LL(0x5bd9153d,0x324c13f8),LL(0x4c682ba6,0xe9ea5b73),LL(0x7e3ff6e2,0x2961da7d), LL(0x01a83dad,0x1ed2b050),LL(0xc4a2f60a,0xb232951d),LL(0xd68b8ec6,0xafcea5d3),LL(0x1c6ce0d2,0x21dc058d),LL(0xe719410c,0x0043de75),LL(0xe15cf534,0x4edd792c), + LL(0x3babe09e,0x0f45245f),LL(0xa9f2fac5,0x0959326f),LL(0x5cc136e5,0x7629e7fb),LL(0xe48b7eb0,0x208bd5a6),LL(0xb75a85cb,0x637891d6),LL(0x9f27b57e,0xf0ad9d8d), LL(0xe0454b05,0x437b6944),LL(0x02ed3592,0x022c51d7),LL(0x0dc0a769,0x0f79e2bd),LL(0xd9b81f9b,0x54ace1fd),LL(0xf95ea8dc,0x38611d66),LL(0xf0e6147b,0x52443ca8), + LL(0x8bc272d0,0x857d6855),LL(0xb5be2485,0x4583eeee),LL(0xfe0152ec,0xb83586da),LL(0xe830294a,0x8b0eb223),LL(0xa5b0e880,0x757582b6),LL(0x9cca7fff,0x5140c016), LL(0xe9228f12,0x07a00782),LL(0xd4973080,0xb96e2b5d),LL(0xe88efbe6,0x3cceb9a6),LL(0x73fcdd25,0x9955b630),LL(0x2805d470,0x04f26ab0),LL(0x424da086,0x90b38299), + LL(0xf4f6c5b4,0x73f1ae48),LL(0x4a477f01,0xee5af13d),LL(0xddb93d52,0x274614a2),LL(0xc320aaf5,0x90b0c563),LL(0xef990b0b,0xee2303c8),LL(0x3061f140,0x00d028e7), LL(0xcb3d8eaf,0xff705011),LL(0x62594f4c,0xae1d9908),LL(0xdafea438,0x22a27cec),LL(0xc5962ea9,0xa78e12d5),LL(0x8e65f9cf,0x5bbe9d87),LL(0xf47cefa6,0xa222580c), + LL(0x959abb9e,0xf7aaa732),LL(0x2ebf80b9,0x1222ad0a),LL(0x2e0c286e,0xa1a41737),LL(0x5da3472d,0x3b668502),LL(0x7576f2a2,0xbc0d116b),LL(0xa36a27d4,0xfdbcad95), LL(0x9d54f7ee,0xcdb3f474),LL(0x8a5643a3,0xe2e0f5f9),LL(0x69d4f171,0xc70d11b9),LL(0x6cca4ef7,0xdf96d136),LL(0x2fc6afdf,0x570693db),LL(0x567504da,0x5059e67b), + LL(0x7fe632a2,0x2c8107d4),LL(0xede7bff8,0xfc46c745),LL(0x4650025b,0x2d3b1286),LL(0xe74cd65f,0x815ef3cb),LL(0xa256f01c,0x5431b01b),LL(0x39915cfa,0xe832ff11), LL(0x07d7af84,0x2c106de6),LL(0x6d4753e7,0x67303b78),LL(0x6d75c8de,0x5f886ffa),LL(0x967131cf,0x932a6c20),LL(0x70aebbb0,0x5bc94a91),LL(0xfd56e06d,0xa85b3044), + LL(0xe7eba799,0xc904558a),LL(0xb2fa7331,0x46b6031b),LL(0x0653675c,0x6620e2b5),LL(0x7d2218f7,0xd1373a35),LL(0xaf55a5e7,0x0f4b3ca3),LL(0x714e70c2,0x50774160), LL(0x69188455,0xacc63d14),LL(0x043b8b30,0x89a795fe),LL(0xe1e4b9cf,0xac2fd66c),LL(0x1bf67f26,0xac792702),LL(0x1143d437,0xb9513f0d),LL(0x811f2931,0x02198050), + LL(0x7b480776,0x6d4acdba),LL(0x66dffeb5,0x8b518cd4),LL(0x51918859,0x8826c994),LL(0x38fad835,0xd2b6a7a0),LL(0x6929a870,0xd315417a),LL(0xc5a769e1,0x05d85252), LL(0xec0d091e,0x2fa06335),LL(0x87768c88,0xb0cc3372),LL(0x58a2eb9d,0xacbda5ba),LL(0x76b7b057,0x2a404fc9),LL(0x838c6135,0x073abb71),LL(0x5cfc4f3c,0xbdf89b13), + LL(0x3508675f,0xd00eb9c5),LL(0xa117dc95,0x92ec76a4),LL(0x334ca15c,0xf58d6f85),LL(0x9cee0544,0xeeb52216),LL(0xf21457c2,0x3eb9847f),LL(0x5524c60d,0x547908bc), LL(0x5198709b,0xb5b49d22),LL(0x324abc67,0x718abce6),LL(0x4abd54ba,0xdab8ff2f),LL(0x7184d444,0x98be59e6),LL(0xbabeb4b0,0x45b74b54),LL(0xff71a5ac,0xd8d8bb30), + LL(0x8ec13e6f,0x8aedf7e2),LL(0xd950792d,0x8b952620),LL(0x04918f59,0x36e9dac2),LL(0xd3dd47ed,0x5e49a5a2),LL(0xe863c2bd,0xb17455be),LL(0x326a0d66,0x8caac6a9), LL(0x27bb72e1,0xb6c3f5e4),LL(0xee5fe09b,0x17566c9d),LL(0x5e3db64d,0xfd6bbcc2),LL(0x3189319c,0xd437d07a),LL(0xcd3166a5,0xad00dfc4),LL(0x0bd63003,0xab75927b), + LL(0xafc43be8,0xa7672a39),LL(0xc72f97aa,0xefc49015),LL(0x0e48f2ed,0x81c63c05),LL(0x833a22cc,0x62f39f32),LL(0x72c0c0c4,0xf7a34801),LL(0xa4158538,0x4711cd41), LL(0x1d15f2f3,0xa3c99a4d),LL(0x7bee1b47,0x4b82c1c1),LL(0x9d199f10,0xc7d60b48),LL(0x5f16fa95,0xd1d1f03d),LL(0x32fbeaf2,0x96c780c9),LL(0x376ff106,0x0662e250), + LL(0x78571c8f,0x728e3346),LL(0x6cb339d3,0xd0a886b5),LL(0x0a5671ba,0xf4ea3338),LL(0xa64850a4,0x43823401),LL(0x33117b9d,0xa7729cd5),LL(0x2b78cffb,0x4dd45760), LL(0x5a67d812,0xbe057111),LL(0x5105a3fc,0x7ec6cf92),LL(0x0ccafeec,0x5dbcb4bc),LL(0x803092f8,0xa7587f15),LL(0xa884efad,0x67ee61d5),LL(0xca47d9ca,0xd4ced554), + LL(0x3b03dcbc,0x02c6b608),LL(0x3b9d868a,0x2b20149e),LL(0x4f57eb0c,0xaf5ab01d),LL(0xd750e515,0x59935b94),LL(0x8f89ad68,0x32721b40),LL(0xa7e3ceff,0x673bd755), LL(0xef3b3393,0xbd462fd8),LL(0x0e59a120,0x99142264),LL(0x9263fa61,0x4162da61),LL(0xb6488eb3,0x2ed1f2de),LL(0x725680c4,0xb0bd37a8),LL(0x29ec27b0,0x17218bf0), +}, +/* digit=22 base_pwr=2^110 */ +{ + LL(0x4bd044ef,0xfb09e756),LL(0xbb964fb3,0xdbc9fcdf),LL(0xcdb1f4f5,0x451c5b01),LL(0xf1dd1cf0,0xb02f9068),LL(0x0c687e41,0xd4765e7c),LL(0xd1967bd3,0x89b64981), LL(0xf0439d65,0x06a0e4ec),LL(0xa5abbcec,0x564c387d),LL(0xc651d806,0xc1e9d01a),LL(0x0618a96c,0x5e6ebd83),LL(0xc54ad8ce,0x9ce1aace),LL(0x9953f90f,0xe5248a08), + LL(0xbb296c27,0xda2b0725),LL(0xd341171b,0x1f22ffa4),LL(0x5b132756,0xc721e35a),LL(0xfadb6907,0xe5695e84),LL(0xc283f546,0xbc5a3bf4),LL(0xdde128ae,0x9182cb3e), LL(0x6592e05e,0x179c7fa6),LL(0xf38e8586,0x1e604790),LL(0xa16bad55,0xaf7e83be),LL(0x9137ecd8,0x6f41231e),LL(0x8f30d1ab,0xac87543d),LL(0xb1ee0ee8,0x630a9d87), + LL(0x0850b471,0xd8ccf550),LL(0xf7ebfcd8,0xea8f73d2),LL(0x7f138136,0xfb374f99),LL(0x6cd70e73,0x6b3a1ab8),LL(0x42f40008,0xc4577c13),LL(0xa0517e75,0xb23800f9), LL(0x7088a19b,0x79deaa45),LL(0x702183ed,0xa8410f51),LL(0x2c9f6594,0xe2761fa1),LL(0x52868276,0xed1d0112),LL(0xc22157a4,0x9e9805d7),LL(0xac7653e9,0xbe4aa213), + LL(0x5e4f1914,0x359cbfa0),LL(0xd922176a,0x6992cd48),LL(0x630cbcb5,0x2fd5e90f),LL(0x6ddbf238,0x2b0f6130),LL(0x3af753e6,0x5070970a),LL(0x41727016,0x433d0094), LL(0x9dca97e2,0x2eb15b3b),LL(0x00d96875,0x3a7379cd),LL(0xe930a75c,0x0d4374ae),LL(0x849e7727,0x3cd36046),LL(0x336a19ff,0x9ac3117e),LL(0x433ddc04,0x2d1373c9), + LL(0x13a2a0f2,0xa60026f0),LL(0xd6d8a91c,0x173b08a8),LL(0x2b0ff7fb,0xfa6b3eea),LL(0x4201b05d,0xd5417c7e),LL(0x598f4c56,0x91dacda7),LL(0x0a1513a5,0x00323d9a), LL(0x38bf2619,0x079a4ef1),LL(0xee3512cc,0xfe91faae),LL(0xbc03468f,0xbf9aabdf),LL(0x01d3ca1f,0x0d46d3a2),LL(0x3e677e61,0x6bfad511),LL(0x6dabf925,0x219a70aa), + LL(0x8faa3eca,0x8e145869),LL(0x511c5e09,0x811c0ab1),LL(0xc39fd20b,0x08a52f04),LL(0x5e5f514e,0xa7c22263),LL(0x8507e5c6,0x365701af),LL(0x495177cc,0xba1c1ce2), LL(0xbd3d7f93,0xed128050),LL(0x05af7fa4,0xf5b3a508),LL(0x7eb1e384,0xe27a017c),LL(0x1fbca0a7,0xaa95d99a),LL(0xbaa36562,0x61a459cf),LL(0xb7f845c1,0xc1ee4d7b), + LL(0xa4945d9f,0x164a4fa1),LL(0x8ce7f161,0xf9fb0327),LL(0x9891eb36,0x5284c704),LL(0x9bd23713,0x9ca1ee6b),LL(0xbdc043e0,0xb364fe43),LL(0xae82eb35,0xcf9c9e70), LL(0xbcde9b87,0xa375b9d5),LL(0x6d5ada37,0x55f437d7),LL(0x85bf9126,0x49fc8d72),LL(0xbd83c1bc,0xf13cbab7),LL(0xf847972a,0xc3d75306),LL(0xb1da55aa,0x8c27482a), + LL(0xe4362d67,0x985dcb43),LL(0xa939bea4,0xecb860c2),LL(0x55fbf1d5,0x40597f30),LL(0x43fcd98a,0xb6d166bf),LL(0x15ec99ca,0x59325709),LL(0xc5bdd370,0xe05ae3b9), LL(0x0c7b943b,0xc18f7827),LL(0x4dd572cc,0x84bde9c6),LL(0xf478e56b,0x5d50a89d),LL(0x64d29053,0x242c2f48),LL(0x61cf7e0e,0xcda12c61),LL(0xac8d1d40,0xf8b6890e), + LL(0x37d03584,0x3f2c311e),LL(0xbb91574c,0xb387b3cc),LL(0x8d7aa7af,0xa64914d1),LL(0xea0fb673,0x7586999a),LL(0x14e5014b,0x25ee7a70),LL(0x557fce0f,0x1015b142), LL(0x10cc9d92,0xbb839712),LL(0xabbb3b2a,0x31118426),LL(0xa29866e5,0xc3a4fa6f),LL(0x66310a76,0xbdb3a495),LL(0xa11055e7,0xd397ceb6),LL(0x0240bd0d,0xc8d47e10), + LL(0xfc8d5f6c,0x76624aad),LL(0x0144a0ff,0xb7dbb41a),LL(0x30df5bea,0xd7be190f),LL(0x9ed58b6a,0xb61e3e97),LL(0x14bf09f6,0xce192eb3),LL(0x27ad3fc4,0x3d6632af), LL(0xcdd7c93b,0x51f180e8),LL(0x932c89f0,0x257df031),LL(0xb680f621,0xed1ffbcb),LL(0x34ef3db2,0x6d8be601),LL(0x044b7f8a,0xc32869e3),LL(0x09958ada,0x8db40995), + LL(0x7ebfc0d7,0x004888a9),LL(0x0f90df8a,0x5546c1b1),LL(0xddfb6a92,0x9c73aa4e),LL(0x2782e2a4,0xb492a723),LL(0x2b9b9390,0x5e79ce87),LL(0x2564d95a,0xb8165f79), LL(0xa8f207cb,0x235665e2),LL(0x1087fea6,0xdef16762),LL(0x7b9d891a,0xf4275ea2),LL(0x1f98a6d9,0x70b2662a),LL(0x5fd15926,0x2990c521),LL(0xc30089d5,0x2734975d), + LL(0xb6b11af8,0x7a9a9dd7),LL(0x16a42f8f,0xac4c50dd),LL(0x6726c324,0x1550267b),LL(0x7f841afb,0x12cfb2fd),LL(0x2f046430,0xa41b1905),LL(0xb7d786cb,0x8f5f5373), LL(0x729e09fb,0x7b61b39c),LL(0x2626da4c,0x07a5415f),LL(0x4a348905,0x65d6efb8),LL(0xf2e864b9,0xf377862c),LL(0xab96cdb2,0x3c94000d),LL(0x8efef0f6,0x2176321a), + LL(0xaffedcf4,0xfbd2489f),LL(0x41feb03e,0xe41dfcac),LL(0xe9a86cd1,0xf13f6579),LL(0xb7ff732f,0xfe3c2a91),LL(0x82f008fc,0xd447c728),LL(0x0e1924b1,0xe4e9c054), LL(0x048bb350,0xe0d6bfc3),LL(0x2fb5fc5b,0x5fe1c204),LL(0xaa50972b,0x70799d49),LL(0xd2c42b50,0x7c79c24e),LL(0x5405cd06,0x4ffa2c53),LL(0xc7ba6f8b,0xd890bea7), + LL(0x94747d5c,0xee68456d),LL(0xef40112f,0x5c0312fe),LL(0x0fcb2565,0x57a480bc),LL(0x7c529c68,0x549b31ad),LL(0x402bdb03,0xcfdc5d33),LL(0xbf45407e,0x955bfd5d), LL(0x1fdb20fd,0x8381a1ec),LL(0x4fe0b38e,0x89dafbdc),LL(0x68f59249,0xf33c8706),LL(0x68be67d2,0xb6cc7618),LL(0xea4845b8,0xa7897c7c),LL(0x93f2f9a9,0x6773f08b), + LL(0x14a03fc5,0x642405c1),LL(0x4cf34763,0x5b973584),LL(0x3331c03d,0x9f24c2cc),LL(0x6f9bf4f5,0x56428a37),LL(0x703e5e73,0x2e83d070),LL(0x573c3cf4,0xb8e4e61a), LL(0x8f3e3efe,0x7a18ec58),LL(0xb812e5a7,0xbab02660),LL(0x8289de63,0xee826e36),LL(0xc1a8eb16,0x86d263f9),LL(0xd7b85103,0xee8f3e72),LL(0xe567c787,0xa3627d9b), + LL(0xc1c1bc68,0x8c558000),LL(0x83fd6ca6,0x9e48a67a),LL(0xeb7a35cb,0xacf0d75c),LL(0xf0a93110,0x0fbdce4c),LL(0x9cc50c85,0x82b2d13a),LL(0xcef70d6f,0x696fd259), LL(0x457b88c2,0x1cc9be2e),LL(0x1f04c0bc,0x0d58b34b),LL(0x195a532b,0x52bd479f),LL(0x1ab3605c,0x769fe6ca),LL(0x8a24c1e0,0xba6a63e4),LL(0x99da5d7a,0x86dea462), +}, +/* digit=23 base_pwr=2^115 */ +{ + LL(0x65682820,0x509abccb),LL(0x4ff86137,0xfbfa1d09),LL(0x640bc2f6,0x1ae371bd),LL(0x8f546c68,0xa155c297),LL(0xc08b8cbf,0x8858cadc),LL(0x1d96948b,0xafac5b0a), LL(0x82e25016,0x919cb226),LL(0xb064ffc3,0xd147df4a),LL(0xb4abe560,0x25dd0f1a),LL(0x9cb75bd1,0xc6bbe636),LL(0x47a778e4,0xbb367cf9),LL(0xbde524b3,0x5714aa4d), + LL(0x166915df,0xc6307399),LL(0x5da8a26e,0xb35545bc),LL(0xe3a99321,0x8e0126aa),LL(0xda9308d1,0x0fbfdf76),LL(0x6168e505,0x2163ed6b),LL(0x7500d8bd,0x71f3d008), LL(0x5ac13f65,0xf5715960),LL(0x55172d5b,0xc1cd9a67),LL(0x53d84c65,0x6b225f7e),LL(0x025029da,0x9c031269),LL(0x17d89aed,0x54c1edfa),LL(0x6b435150,0x5b023878), + LL(0x1e94d949,0x1e7ae160),LL(0xe78e6221,0x177dc53e),LL(0x7aeb9882,0x8af29d8f),LL(0x9e3f3906,0x2d9a60fd),LL(0xdf962156,0x6979fcd6),LL(0x7e1e54b8,0xdd2fe588), LL(0x76643453,0x9cccf310),LL(0x4e0643d0,0x94ece1a8),LL(0xc111d8cf,0x745449cb),LL(0xe6cfbd97,0x872afa4a),LL(0x10dfb34e,0x5c27b7ca),LL(0x533480fe,0x505e62bc), + LL(0xae238fa1,0x009ef80a),LL(0x486af6b5,0xb41d9b24),LL(0x685add95,0xab4455ed),LL(0x72c7dac8,0x18f323f6),LL(0xe372f168,0xe7009790),LL(0x067bea99,0x4d5bcba6), LL(0x51a2a9a9,0xf15bdbcc),LL(0x2fac9169,0xde7e4f74),LL(0x4bdbec36,0x2fd62c30),LL(0x1b3ac6c7,0x8b3ea93d),LL(0x9c293889,0xce1c8e5c),LL(0x11564f8b,0x19664dda), + LL(0x175418b2,0x5f266f13),LL(0x03a626f9,0x6bd7a869),LL(0x4a6f11ca,0xc7b53230),LL(0xbfc8cced,0xa216b056),LL(0xb288cb7c,0xa274d5d8),LL(0x924897ad,0x6fc4a35d), LL(0x81fc6940,0x1ea532eb),LL(0x2acbbc45,0x2fcd817e),LL(0x67814fa3,0x45eee93f),LL(0x1229e035,0x3b3da48c),LL(0xefd8e3d7,0xd049a976),LL(0xbf81f314,0x8087dff7), + LL(0xbd366155,0x77faac22),LL(0x282f11b5,0x13cc4038),LL(0x5fbd35ab,0x31ad1dd4),LL(0x45d6d40e,0x7e0de9da),LL(0x39749ef6,0xa16c5f19),LL(0x85691cf2,0x761cd6cf), LL(0x4d59b802,0x156536ad),LL(0x87c4b11d,0xee98dc41),LL(0xd35088fc,0x165a1eac),LL(0x38fb995c,0xce8a7335),LL(0x3293b3a5,0x34d0d331),LL(0x8b570e79,0xfcf548ca), + LL(0xa85af34a,0x7be5946e),LL(0xda6fb0e0,0x420593c9),LL(0x987f9246,0x40b83c00),LL(0xa15d192b,0xac35f4e9),LL(0x776a678c,0x1979bd33),LL(0x8f6068d3,0x0a7d973e), LL(0x7e6298fe,0x71d322e8),LL(0x36af9b65,0xbb23a299),LL(0x6644c50c,0x14e2b970),LL(0x73570bd3,0x5f7f2073),LL(0x9055538b,0x40215c56),LL(0x365500c9,0x91372e64), + LL(0x9af0a75c,0x6a3a2327),LL(0x9f1f250d,0xf832a815),LL(0x22a82d3f,0x17030c33),LL(0x14cbc835,0x24bf18ea),LL(0xb2da2727,0x319dc4ca),LL(0x6d020d4a,0x481df360), LL(0x7fc22ba5,0xaeebdd8a),LL(0xa91e28ab,0xbd0515c6),LL(0x595f361d,0xfc8a2978),LL(0x1ae8fa3c,0xe60dd96c),LL(0xa5341575,0x19c2109a),LL(0x06a0ee48,0xfd6e92bb), + LL(0x77b5c7c5,0x650008f9),LL(0x02a6d087,0x4bcf6002),LL(0x82234273,0x391ebfa3),LL(0x86cd884e,0x9dcb05d3),LL(0x8753b3fa,0x1b5e7afa),LL(0x2d1e513c,0xd453e9b4), LL(0x4b3a74b4,0x3b1a0dea),LL(0x5bebd592,0xf8989aac),LL(0x61dc640b,0xac3ec9f5),LL(0x6c4b301b,0x4ba9dd0a),LL(0x04e48df6,0xf686a5fe),LL(0x0631d1e6,0x8b0d2d76), + LL(0x03c87a20,0x443deaab),LL(0xbb817740,0x57dbd224),LL(0xd51d6acd,0xf88918de),LL(0x7d4ec9be,0x48c9b2f1),LL(0xdfc48a69,0x78b41104),LL(0x78827f54,0x44a81443), LL(0x16a22495,0x74ed7949),LL(0xea92d3e6,0x6b9e9128),LL(0x0d9fe252,0x6fe6449b),LL(0xc14e825a,0x31743d06),LL(0x08690f00,0xd9d4ad8c),LL(0x0e65f748,0x562e2f60), + LL(0xb1befb0a,0xe9e01117),LL(0xd25cb1ae,0x7ce74721),LL(0x2d4437ad,0x946216c6),LL(0xea8b2d85,0x83b03131),LL(0x2c4fa895,0x3614b15d),LL(0xc23c6bf7,0x393317d9), LL(0xdcd80eb4,0xda73495a),LL(0x6338db53,0x850dfb46),LL(0xc9b8943a,0x3329c498),LL(0x0765d94a,0x1ee615b5),LL(0xd67c17b3,0x2d16a6cf),LL(0x6f8a9ed5,0x050d6bfe), + LL(0x8b7f8cb2,0x92501ff8),LL(0x578a4cfd,0xcdaa6c93),LL(0xe5303846,0x95b99a05),LL(0x2543d1a9,0x30a72fdc),LL(0x11808771,0x6648126b),LL(0xeefb8145,0x6980dd99), LL(0xf1949aad,0xd9a87f83),LL(0xf7b8ab95,0x3b4f208d),LL(0xc40122e4,0x3341e118),LL(0x853b291e,0xe513567f),LL(0x3b565bad,0xd4f1bbb7),LL(0xa7658f77,0x86207b43), + LL(0xf87cd72c,0xe5737dce),LL(0x7a1c9f7f,0xdf41230a),LL(0x34e0d519,0x7fe5cfcf),LL(0x26033c9d,0x83cc1da5),LL(0xe16dffd0,0x4195ef1a),LL(0x8af91671,0x7c388f6e), LL(0x62283e40,0xeb1002b8),LL(0xa96be04c,0x08122e57),LL(0xdf9fac7c,0x48cb2d0c),LL(0x41f0946d,0xa3c007ae),LL(0xcec27088,0x4d6fa5c5),LL(0xdcd5fcfa,0xe1fbc03e), + LL(0x9bece059,0xbab37a63),LL(0x7fa28fc3,0x5a60ec87),LL(0x1d4a8f5b,0x69e1086c),LL(0x65814282,0x3a873844),LL(0x2676cc7d,0xb642dc16),LL(0x97e423a0,0x8d152bb9), LL(0x160ff40b,0x66ac1b37),LL(0xf657547a,0x7410ba67),LL(0xadcc6823,0x8f47debb),LL(0xb1536fa1,0x44e9bf64),LL(0x939e5461,0x1cfdd504),LL(0x43bb3b90,0xba5729e1), + LL(0xdd63f6d4,0xb5c26108),LL(0x60229177,0xefb8dfc7),LL(0x23e4cc7d,0x9d980889),LL(0xc211c92b,0x59579ee8),LL(0x35fe17b4,0x6f146300),LL(0x0ca61e4a,0x6c1e7075), LL(0x19d582e1,0x8671156d),LL(0x07230605,0x36a3d858),LL(0xa352dcdc,0xd9314e61),LL(0x4c6bc2b1,0x7a9d0de7),LL(0xc894ab77,0xb57d3f71),LL(0xd4e5a04e,0xb3564bc7), + LL(0xcde5e785,0x2e32f896),LL(0xb9db8f31,0xcd55ae7a),LL(0x8f832885,0x278db1ad),LL(0xadcbd933,0x271d9078),LL(0x4a64f863,0x2208fae3),LL(0x39c89365,0x974046e0), LL(0xb3cd0cd3,0xcb46f272),LL(0x74e59edc,0x31f34e1a),LL(0xedd50418,0x3421d316),LL(0xcabe36ed,0xb1d8a064),LL(0x362efcda,0xdb13e560),LL(0xc71eb3ee,0x567c2b6c), +}, +/* digit=24 base_pwr=2^120 */ +{ + LL(0x70d4d7bc,0x2af8ed81),LL(0xb632435c,0xabc3e15f),LL(0x78219356,0x4c0e726f),LL(0xb87254c4,0x8c1962a1),LL(0xc9e7691a,0x30796a71),LL(0xa75a12ee,0xd453ef19), LL(0x13ae4964,0x535f42c2),LL(0x0da9586a,0x86831c3c),LL(0xe39a7a58,0xb7f1ef35),LL(0xd459b91a,0xa2789ae2),LL(0x02fd429d,0xeadbca7f),LL(0x65290f57,0x94f215d4), + LL(0x4b950889,0xc0855002),LL(0x8ce24da0,0xee99dbfe),LL(0x4318e860,0xdda71d96),LL(0x04fe9b85,0x01d3d396),LL(0xe25e7e20,0xda4bc065),LL(0xe076c81c,0xd3a50b87), LL(0x31e5f494,0x5b9f8219),LL(0xa6a1b821,0x6a140527),LL(0xd8dd159b,0xf52683e4),LL(0x20b18043,0xca9c8887),LL(0x08a0d8f5,0x73c040fa),LL(0x179525c4,0x92e482e8), + LL(0x1cfb79ac,0x37ed2be5),LL(0xe7af84c3,0x801946f3),LL(0xe77c2f00,0xb061ad8a),LL(0x44de16a8,0xe87e1a9a),LL(0x7ee490ff,0xdf4f57c8),LL(0x005993ed,0x4e793b49), LL(0xbccb593f,0xe1036387),LL(0x95e09b80,0xf1749411),LL(0x5ab42f91,0x59cb20d1),LL(0xac0ff033,0xa738a18d),LL(0x2ac1e7f4,0xda501a2e),LL(0x84d8a6e0,0x1b67eda0), + LL(0xa4d2313e,0x3a828904),LL(0x92e66888,0xbf4946b1),LL(0xe5fa19d2,0xc574898a),LL(0x5e1c5fa4,0x0b13dbb6),LL(0x7c390fc2,0xf11343ba),LL(0xd7d32187,0x35b1418f), LL(0x83e7fe7b,0xc92cb1bb),LL(0xd78365c4,0x0b969455),LL(0x672f2af7,0xda69dfe5),LL(0x30932441,0x9c62d7b4),LL(0x94af02d6,0x165672ad),LL(0xcde81c22,0xd2cc734d), + LL(0x1080e90b,0x1d27efce),LL(0x3fd01dc6,0xa2815246),LL(0xcaa26d18,0x99a3fb83),LL(0xb82babbe,0xd27e6133),LL(0xd783dd60,0x61030dfd),LL(0x73c78cb8,0x295a2913), LL(0x68be6a92,0x8707a2cf),LL(0xeeb3474a,0xc9c2fb98),LL(0xa2b176b8,0x7c3fd412),LL(0xc7202101,0xd5b52e2f),LL(0xf0a6d536,0x24a63030),LL(0x04648ec0,0x05842de3), + LL(0x33f4d416,0xd45e3501),LL(0x4bf9131e,0xbb40233a),LL(0xe302483a,0x1532a088),LL(0x2c2485c0,0x3475e8b8),LL(0x969cdbe6,0x08f9ea56),LL(0x253cd738,0x31928645), LL(0xac9836be,0x1cf323a4),LL(0x02b6e4de,0xdf647ccf),LL(0xc06f3d09,0x9a31e84f),LL(0x39efe6d9,0xd326b86e),LL(0x14ac4dec,0x77e3e1df),LL(0xf3e0c582,0xf2d5917a), + LL(0x30577ac9,0x67477cdc),LL(0x244f92a8,0x51dd9775),LL(0x917eec66,0x31fd60b9),LL(0xd66c5c1d,0xacd95bd4),LL(0xbf9508ba,0x2e0551f3),LL(0x688cb243,0x121168e1), LL(0x4540d230,0x8c039740),LL(0x009ecdf9,0xc4ed3cf6),LL(0x44db62af,0x191825e1),LL(0xc4a030da,0x3ee8acab),LL(0x94081504,0x8ab154a8),LL(0x486c9cd0,0x1fe09e4b), + LL(0x02cf37fd,0xe92b56c0),LL(0xf71b34de,0xa75bbcb0),LL(0x50f5c482,0x7754d0ef),LL(0x11fa89fe,0x850a9ef6),LL(0xba4ea7d8,0x97d74b1b),LL(0xaab7ba2e,0xfc757c25), LL(0xf2a67fdd,0x06f30ab0),LL(0x12e72af8,0xb10aba14),LL(0x7a2e053d,0x47580bca),LL(0xdcf0e14c,0x85795598),LL(0xd6f55310,0xc3596781),LL(0x4c9b7e18,0x8ab251b7), + LL(0xd113450b,0x512f82f9),LL(0x2dbc9197,0x5878c901),LL(0xe13f355b,0xdb87412b),LL(0x935b8a5e,0x0a0a4a9b),LL(0xf25a5351,0x818587bd),LL(0x31e3d9c7,0xe8079310), LL(0x611bc1b1,0x8b1d47c7),LL(0x72a823f2,0x51722b58),LL(0x53b36b3e,0x6f97ee8a),LL(0x946dd453,0x6e085aac),LL(0xe65e6533,0x2ec5057d),LL(0x4bb18801,0xf82d9d71), + LL(0x4dbb8798,0x4ab13850),LL(0x72d04cd2,0x0e7980d7),LL(0x0b3271c6,0x1755c566),LL(0x9d9d1468,0x8414efb0),LL(0x1795ce66,0x61a58630),LL(0x232924a1,0xb6a8b393), LL(0xae031bd6,0xa992f0ce),LL(0x2915acc1,0x6747fb5f),LL(0x93e9c0d2,0x03daa266),LL(0x5400d554,0xc18fa364),LL(0x9497e895,0xaf04ff8d),LL(0x50b6b339,0x86c3cfc2), + LL(0x8ba5aa8e,0xad81fa93),LL(0x8f7aa69e,0x723e628e),LL(0xef35937c,0x0ba7c2de),LL(0x6decfb40,0x83a43ec5),LL(0xe60c4f2d,0xf520f849),LL(0x457e3b5e,0x8260e8ae), LL(0xbf1d9ed7,0x7ce874f0),LL(0x7f1a5466,0x5fde3553),LL(0x0c162dbb,0x5a63777c),LL(0xdad87289,0x0fd04f8c),LL(0x640761d5,0xca2d9e0e),LL(0x38501adb,0x4615cff8), + LL(0x04e1e6e3,0x376b2a7f),LL(0xa31774b4,0xea0dcb70),LL(0x5cbdec2e,0xfc7fe4cc),LL(0xf03f459e,0x8568499d),LL(0x8b78900e,0xe9fd8fb2),LL(0xe431bf97,0xd33c6e30), LL(0xc896e766,0xd904b8f5),LL(0x82748cef,0xa8f577cf),LL(0x87e044b3,0x93dd921b),LL(0xf76eebe9,0x23d79837),LL(0xe569feeb,0x5e0a7493),LL(0x414dddb6,0xd0797549), + LL(0x110b4a25,0x9422789b),LL(0x70ad8cc1,0x5c26779f),LL(0xec4f1e14,0x4ee6a748),LL(0x5c7ab5e0,0xfb584a0d),LL(0xfb21ee66,0xed1dcb0b),LL(0x11c6863c,0xdbed1f00), LL(0xb1b1d187,0xd2969269),LL(0xafe964e6,0xf7d0c3f2),LL(0x12bb865e,0xe05ee93f),LL(0xed79118e,0x1afb7bee),LL(0x0f0fe453,0x220af138),LL(0x52782ab9,0x1463aa1a), + LL(0xbfe5b1a7,0xfd9e3542),LL(0x75938cea,0xb42d2a41),LL(0x3befb760,0x74688a15),LL(0x2e33dbe7,0x8daeeaa2),LL(0x3e677801,0xc9c1ea08),LL(0x34effe1e,0x68ecf4e4), LL(0xd294c321,0x927700cc),LL(0xe940afc5,0x9e2e723d),LL(0x7cf6cd43,0xbcfac07a),LL(0xd1006bc3,0xa009ef94),LL(0x373d13e3,0xa02016b0),LL(0xabae5822,0x4e097adb), + LL(0xd7dbe5f9,0x7c139d56),LL(0x0b83685b,0xfc16e611),LL(0x9018463c,0xfa723c02),LL(0x840bf5d7,0xc472458c),LL(0x0af07591,0x4d809359),LL(0x3308dfd9,0x418d8830), LL(0x0c365ae3,0x9b381e04),LL(0xf8190fd1,0x3780bf33),LL(0xdd03e854,0x45397418),LL(0x4e51e491,0xa95d030f),LL(0xe3286cea,0x87c8c686),LL(0x900b5f83,0x01c773bf), + LL(0xc898b8bc,0x8db8b78c),LL(0x502940cd,0x686896da),LL(0x2dde2e3c,0x67e50f02),LL(0x8cbf406c,0x2e2461f3),LL(0xe1f7ff60,0x32182781),LL(0xe30e2688,0x26934b05), LL(0xfc4494f6,0x95adc204),LL(0x161b7499,0x4c7f30c5),LL(0xb7341737,0xd5caf060),LL(0xd128d46c,0xed93187f),LL(0x20fc1e04,0x3f2819cb),LL(0x2b7f70a1,0x48c4086f), +}, +/* digit=25 base_pwr=2^125 */ +{ + LL(0x0ece13ae,0xd74d09c1),LL(0x57a6bd95,0x5e59d9e0),LL(0xe132b940,0xdb1ccfdc),LL(0x843d3c66,0xa0e5309c),LL(0xf9cb3ef4,0x1fbd03a5),LL(0x00ea5177,0xcdc9ef0a), LL(0xcb784a6b,0x1ebf5a15),LL(0x8a0d109a,0xa67382af),LL(0xa0d34d15,0x3256c37a),LL(0x0fca43af,0xee40efa5),LL(0xb9841bde,0xc299bbd4),LL(0x3bef4a0b,0x6df68f60), + LL(0xd9d7c50a,0xe01295fd),LL(0x67f8ef0d,0xaf31b4ea),LL(0x9eaf8eb7,0x2ec9689f),LL(0xc622acc5,0x327b96c5),LL(0xb2757f2a,0xae918f81),LL(0x4fd6606e,0x74927d68), LL(0x18574215,0x09bb7fce),LL(0xe8e68b72,0xfea383bc),LL(0x5fb47511,0xdf2a6f12),LL(0x8e399520,0xbe88faa1),LL(0x3fb1c3a2,0x0166d57e),LL(0xe525f81f,0x5907ef2f), + LL(0x57797e5c,0xf24fab9b),LL(0x3add682c,0xd8500987),LL(0x33ac4fb5,0x75e370e7),LL(0xc8a69765,0xd8d61642),LL(0x5e30aa0e,0xf057f10a),LL(0x4f1e8637,0xaf953087), LL(0x701cd1a0,0x887f06b3),LL(0x8a200082,0xd7c6a9c6),LL(0x6e6483a0,0x4319f7eb),LL(0x475d4bfd,0x44f1aaff),LL(0x0f811fc7,0xa8ee73b5),LL(0x82916f8b,0xbb67bcdc), + LL(0x8a37f660,0xdefe3a7b),LL(0x858f5765,0x7898db8c),LL(0x73d1f9b4,0x7366c26a),LL(0x237ae8b7,0x35d5d718),LL(0xb4478259,0x3efb20fe),LL(0xaa545ee3,0xccd0fed7), LL(0xed22d152,0x750edd05),LL(0xee20d4c6,0x4f8020f9),LL(0x0a9e29dc,0x16e60f37),LL(0xbfbec7f6,0x9cf0a136),LL(0x2e47e143,0xb430a34b),LL(0xc6cdd1a9,0x2e2560bb), + LL(0x2a9af85a,0x2cb5625c),LL(0xfea96b60,0x9ba19601),LL(0xd7549809,0x25dd6aca),LL(0x5b4c31e9,0x9c02e613),LL(0xe09dbb63,0x40c4baea),LL(0xf49beece,0x38455db7), LL(0x5b848716,0xe9846d0d),LL(0xd2ebce35,0x9c63043d),LL(0xbc26a79e,0xedee8e86),LL(0x0869c85f,0x70c46f94),LL(0x7bc49e34,0x34b05934),LL(0xa80d7d8e,0x737fe693), + LL(0xe4161a65,0x799352da),LL(0x56253ce6,0xe5cf7ad8),LL(0x6de32775,0xf606bf79),LL(0x57fce8db,0xddc0f3a3),LL(0x16cf4a47,0x1075fc23),LL(0xb27c5ad8,0x078f0e04), LL(0x3f7100aa,0x9fc47795),LL(0x4673ffa2,0x3ac48925),LL(0xf9cd8348,0xb8263f42),LL(0x68cc92d5,0x5bdfde30),LL(0x1ac37f9c,0x2250927b),LL(0xb33da359,0x26ec8328), + LL(0x236eaa28,0x76923635),LL(0x657d0640,0x5acc650b),LL(0xc20a58ab,0x2652ec7e),LL(0x1a5459f9,0x93385f9f),LL(0x306e2091,0x45952d39),LL(0x34cef2f2,0xb31f5a42), LL(0x1f3c055f,0x0d9b0484),LL(0xae3b1f4d,0x6c5ef8ee),LL(0xbd24becd,0xd3fdb464),LL(0xa2210148,0x49b11a10),LL(0x1472893a,0x73de2855),LL(0x6ee531f7,0x99435b57), + LL(0xc88d568a,0xf186d6bc),LL(0x528535dd,0x872bc4c7),LL(0xdfe64dc3,0xc9e7432e),LL(0xd795ea57,0xd9fc4832),LL(0xc845af2b,0xf4ffdb81),LL(0x2b670517,0x66d7e788), LL(0xd7b7a1c6,0xa7c1be04),LL(0xd5b2a249,0xbed88479),LL(0x03f2ef6d,0x62ff8aba),LL(0x20dc701d,0x60ecaac4),LL(0x4ff10119,0x9f4b559f),LL(0x3cd54fd0,0x0582c931), + LL(0x3940d5dd,0x271f5d70),LL(0x192fb2b6,0x7707d109),LL(0xda210560,0xc6131828),LL(0x0bc534c0,0x21750048),LL(0x23487a61,0xa96593d1),LL(0x0b9c6e92,0xdab1698b), LL(0x1f7abe02,0x8a7951a5),LL(0xecd49e72,0xa9394a4b),LL(0x2f6f53d5,0x95ba141d),LL(0x6cdb90cc,0x03809379),LL(0x0586a3ef,0xa5e0ebf3),LL(0x70606681,0x75f006dd), + LL(0x12bbaeb6,0xea9da8f0),LL(0x8c9f8360,0x3fba06b1),LL(0xb28c0ac3,0xc11bd7ab),LL(0xaa8a01bd,0x1e05af2f),LL(0xf000b1c3,0xae1e99c5),LL(0x53d79930,0x93ee8064), LL(0x4c4f5513,0x5728089e),LL(0xb1f70b76,0x755351f3),LL(0x675f77ef,0x187ac651),LL(0x53067d84,0x5cf7bfb5),LL(0x8174b5c0,0x62929083),LL(0x8d5be74d,0x720e2079), + LL(0x15c26385,0xcdb0e974),LL(0x40c9a381,0x7110dd68),LL(0x8180d82b,0x1fd85657),LL(0xb97cf516,0xa42cad51),LL(0xc8e096e2,0xd0e97506),LL(0xac077c1c,0x9372ac5c), LL(0x528c6d23,0x080cac77),LL(0xc192bee8,0xa62b7b39),LL(0x596c03b4,0xf1b2be68),LL(0x65cdde1a,0x9e367c67),LL(0xa97f58bb,0xdb0a51db),LL(0xc29dbc0d,0xa9b52c4d), + LL(0xf5eb9ad1,0xab8cc09f),LL(0x132edbfe,0x97a4de76),LL(0x8baf6347,0xa2e11c54),LL(0x683cfcf6,0xcee54229),LL(0xdcfc6555,0xe1e993b8),LL(0xbe9df066,0x333bf16a), LL(0x060d62df,0x5207e093),LL(0x69b0f5fa,0xfa32324d),LL(0xd3243d2d,0xef16fbcf),LL(0xf04f8e45,0x540a2e59),LL(0x48317bba,0xb5e70f9c),LL(0x5b35baa1,0x00dbe9b2), + LL(0x04a6bdf1,0xa7c2e951),LL(0x45310414,0xe7b50010),LL(0x250deed3,0x0ad7ac85),LL(0x07fd3b65,0xeb7ad465),LL(0x6bdc321a,0xcac35e8c),LL(0xe992ce8e,0xf773a3e9), LL(0x0f682437,0x99cbc651),LL(0x7ff5f4b3,0x62f7b15d),LL(0x6d131441,0xf3c08dfd),LL(0xfbb0fd4b,0x8b998754),LL(0xe061d5b7,0x88dae889),LL(0x2a346488,0x9f6e8dca), + LL(0xebb512ee,0x0eaed675),LL(0x058efbd5,0x347e0756),LL(0x296d3d47,0xadf792ca),LL(0x4654d012,0x57f00c0a),LL(0xbccc5803,0xa1e08a04),LL(0x5b2f11d1,0x610677f0), LL(0xb81acfd2,0x0d9393d7),LL(0x0587c219,0xb258e157),LL(0xb4ceba47,0x372a1857),LL(0x3ecc1c5d,0xe1ce8bb5),LL(0x922cecd0,0x7efdf301),LL(0x0d8aa653,0xcab8cb17), + LL(0xd85124dd,0xc60f118a),LL(0xa77febb9,0x4a045272),LL(0x45555221,0xbae0edf5),LL(0xab65b900,0x1fb21695),LL(0xd56b9882,0x9179d546),LL(0x006bc680,0xead5f78e), LL(0xed9d8095,0x6c3ef1c1),LL(0xd024f8cd,0xe22832c1),LL(0x157c5363,0x8c783b9d),LL(0x000d3603,0x25c1bd7e),LL(0xe8ff7a81,0x4e76a723),LL(0x4a955196,0xa55cfcb4), + LL(0x79d05497,0x31954a56),LL(0xfe76d4d8,0xc12520b6),LL(0xe37ef1d2,0x8c433ec5),LL(0x75bc3b66,0xcd0f2035),LL(0x249cd98b,0x3723f145),LL(0xea3b42a3,0x1356e0d2), LL(0xf174c7b5,0xf607fee0),LL(0x0127be39,0x318afc5e),LL(0xcea5417f,0xd47b5d74),LL(0x10fca22b,0x6891940a),LL(0x2b635e8b,0x5cea4133),LL(0xb5934fef,0x93db2ed6), +}, +/* digit=26 base_pwr=2^130 */ +{ + LL(0x41b959b5,0xb1f4fead),LL(0xe71890c0,0x6edb53a9),LL(0x2e28aa2a,0x48b47efe),LL(0xb3151d67,0x70dad2e9),LL(0x436a3460,0x87a8178b),LL(0x801f7af7,0x0f86f9f5), LL(0xa982fc14,0xfab462e3),LL(0xcb03e978,0xe29126ba),LL(0xe6681282,0xb4696b3f),LL(0x6a3fdc1d,0x3bd9910a),LL(0x49e37dac,0x44091284),LL(0xcf605fb3,0x3b4bfabc), + LL(0xdf9a9f18,0x57edf71e),LL(0x627a0b79,0xbf834240),LL(0xa6934160,0xb37aba1a),LL(0x5e639a54,0xd45b3d2c),LL(0x70bce957,0x62c6b9ad),LL(0x5d7e87f3,0x16bc35a7), LL(0x66b4a982,0xb0216982),LL(0x0e51c9bc,0xb56050dd),LL(0x478e4b91,0x15aa692b),LL(0xbe3fe25a,0xdd67cf29),LL(0x06bdd4a8,0xf1ef75b0),LL(0x41df627a,0xf71a285b), + LL(0xb7a3ce87,0x3176a43a),LL(0x5f130e73,0x9fa09e97),LL(0x9368e156,0x971cc37b),LL(0xb8981792,0x2cabf535),LL(0x4d0f0bc0,0xaec2862e),LL(0x3ce8c100,0xa1a48c18), LL(0x4af2eae9,0x288f4e69),LL(0x1f9339bd,0x778845f2),LL(0x17dfaa6a,0x1ef5fdfd),LL(0x3483a6fc,0xc784117e),LL(0xf3c5c19e,0xe8c82f05),LL(0x1da87ab6,0xf39b3c1d), + LL(0x8a541be6,0xa58a27c5),LL(0x54fd7683,0xaf669499),LL(0x00079a25,0x24318266),LL(0x2606caf5,0x113f6fcf),LL(0x16cb28c8,0xf6ff2be3),LL(0x3c17caa6,0x8f7fc60e), LL(0x7d35e26c,0x8ea577e0),LL(0xf0628903,0xc3e744c0),LL(0x592a57ee,0x4b28eff4),LL(0x5e3f67b2,0x76e1f87c),LL(0xfb008902,0x40d7a676),LL(0x4b6e6b7e,0x68a9dc76), + LL(0xe4fbe1d8,0xfc14716f),LL(0x88d5fc05,0x711c4f60),LL(0x8ab2552f,0x53df2711),LL(0xb3039434,0xeb4a3587),LL(0x825eba03,0x84223085),LL(0xf69569cf,0x1aff30b7), LL(0x30d38055,0x752a2b25),LL(0x7dc7727e,0x8fb8ef5e),LL(0x679f84b9,0x8f0b8bea),LL(0x004815a9,0x076e7b05),LL(0x65debb2f,0x93275037),LL(0x83a686b0,0x24f92c79), + LL(0xdff082f7,0xf9597802),LL(0xeb44042c,0xb47fcfee),LL(0x8ac1e9a9,0x0e2570c9),LL(0x9168391b,0x144f5763),LL(0xfe49b3b2,0x7c5fe9e6),LL(0xe1202056,0x30dd5a41), LL(0xe5132272,0xb0866f7c),LL(0x199d029c,0xaa2d7c60),LL(0x2bd29ff9,0xb37c3232),LL(0x29bc5c36,0x6c0e511b),LL(0xbaa69a42,0xb64e16b6),LL(0x28545caf,0x9fd7d79b), + LL(0x0c3576d3,0x0620003c),LL(0x31b24876,0xa9608ecd),LL(0x2d83f362,0x7dcb576a),LL(0x7df98941,0x88a3a38a),LL(0xb3596afe,0xdd182e8a),LL(0xe02a5357,0x9c740d01), LL(0xb75d5d3b,0x2fed2fcc),LL(0x8d34ca34,0xd0ea6d50),LL(0x8f4bd1dd,0xfe4b8743),LL(0xb130c5c5,0x73c5acd3),LL(0x2dadbb20,0x9f8b0817),LL(0xc5a59dfc,0x34b74294), + LL(0x12575913,0x76e2751a),LL(0x4a5f8c4a,0x2c605991),LL(0x71fba662,0x58322dfb),LL(0x5e0886af,0x228aec08),LL(0x6aee544c,0x8d83b627),LL(0xe29f9639,0x338f5fb6), LL(0xbf5e19fa,0x1ba4cfe0),LL(0xb9e4f8f6,0x2eea84c5),LL(0xcee95d92,0x7e0eed58),LL(0xbe535540,0x2d29282a),LL(0x07a9a1f4,0x866638b6),LL(0x6ab8dc82,0x91599977), + LL(0xa2144cce,0x6741d444),LL(0x771c7c6c,0x06ce4c04),LL(0xae0ff352,0x3d5bc2d2),LL(0x0e05ca8a,0xa0ae03ba),LL(0x833183d8,0xbf718851),LL(0xccdc956f,0x7095f4d4), LL(0xb3ab0c80,0x23268e4a),LL(0xc16c6dad,0x984cf1eb),LL(0x67120ec8,0xc2333145),LL(0x9cd9a03f,0xc615f60e),LL(0xb3d16871,0xe9e35fd3),LL(0x15e76b83,0xc9b9853d), + LL(0xbb7fefd2,0xd8e314ab),LL(0xaa61ea98,0x74d946ba),LL(0x00fd45fb,0x10cfd8b5),LL(0x357f8e42,0x87dfda88),LL(0x27bf7a3a,0x6dab4a2f),LL(0x03b613f5,0x4dd49a15), LL(0x74f2c1a8,0x1f5491dc),LL(0xcc8ee02e,0x98d17370),LL(0x3f141b03,0x01ef39fe),LL(0x6984d4c3,0xa5b04673),LL(0x4a926cdb,0x10c40183),LL(0xe2af1758,0x3e7bbe18), + LL(0xc2faa336,0xda178c95),LL(0x68d20d12,0xb4ae744b),LL(0x5e066620,0x5b75f4e9),LL(0x41e95800,0x4138315f),LL(0x010c2c6a,0xf5c97da9),LL(0xe07bbf8d,0xbc4ff40f), LL(0x21709050,0xbdd6bc9f),LL(0xc5f7bb78,0xf3f4dcf2),LL(0x9b1da924,0x584ce030),LL(0xec3d9507,0x44b81c48),LL(0x73e1961e,0x1dfab021),LL(0xa77458a8,0x9212120e), + LL(0xf9612455,0x599ece96),LL(0xc725cba4,0xe1c6eb81),LL(0x6373ccee,0x2afc5ba8),LL(0xc64f7261,0xe4556833),LL(0x414ce1ab,0xf29ad540),LL(0x3ee82a77,0x7094e067), LL(0xdc9716fa,0xfa26eaa0),LL(0x9ceb36af,0x340da4be),LL(0x4e1c94bd,0x9570ade5),LL(0x20b0ddbe,0xf4a86d8e),LL(0xdc6eff6f,0x967ab653),LL(0xe8bab7a4,0x48890be5), + LL(0xc96437e8,0x74ee2fe8),LL(0x0bfd2bb3,0xeafe095d),LL(0x103d0a33,0xc75f0e53),LL(0x70f13700,0x22190801),LL(0x101d133f,0x96684d5b),LL(0xf21b224b,0x6c98178f), LL(0x1d59dd67,0xe3003316),LL(0x734e7699,0x933de68f),LL(0xa305ea84,0xa3883631),LL(0x6d66c90f,0x21afb4d0),LL(0x330568ce,0xebeecc72),LL(0x62b5a3ae,0x6b65b757), + LL(0xf637841c,0x4069700f),LL(0xb1c94076,0x6ee84605),LL(0x829e1bc4,0xcd64ecd7),LL(0x45cdda86,0x606944a8),LL(0x58f036ae,0xb0f2d5fe),LL(0xec18a23f,0x0391590a), LL(0x1a82e6c2,0xe29e3891),LL(0x6bd4a55b,0x5489a628),LL(0x7ae6380c,0xf3e7a351),LL(0x6ae0ae07,0xb3272f23),LL(0xe66cd800,0xb605a039),LL(0x2aac1b66,0xd4e522ea), + LL(0x8b153f87,0xbb9d0b1c),LL(0x518a921d,0xa3d52cbe),LL(0x2110cd42,0x232a29d2),LL(0x8a40d59b,0x9083e225),LL(0xaa50f55d,0x3b00382e),LL(0xc02cf900,0x4ad64680), LL(0x36cad5d1,0x2f25bd26),LL(0x64eb9c45,0x9c693e85),LL(0xccf60708,0xe4ea93e5),LL(0xf9270541,0xb932b958),LL(0x5eb454ce,0xee93f51f),LL(0x38840e07,0x1e82e354), + LL(0xaf4d260b,0xa16c79cf),LL(0xfab3c3c8,0xfe853f6c),LL(0xc2f47e68,0xb8bd6aa0),LL(0x2c9b4914,0x277d590f),LL(0x097242a8,0xb6d1c810),LL(0x45f75512,0xcf2f3d8e), LL(0x74a20c3b,0x2176162b),LL(0x2b2bcdda,0xeee8bcb8),LL(0xa503aee7,0xfcf8c0d1),LL(0x7af4dd78,0x5d1f94a5),LL(0x2ab43be4,0x8f0bc1a6),LL(0xba9e071b,0xd22dbf16), +}, +/* digit=27 base_pwr=2^135 */ +{ + LL(0xb4e814b3,0x705bfe37),LL(0x702013c6,0x22f0de61),LL(0xbc456797,0x811e77a9),LL(0x17081a2f,0x4f52c4e6),LL(0x9fe1640e,0x87405d81),LL(0x707711d7,0x53fa82b7), LL(0x0ee4aea6,0xdc6fff83),LL(0xfd60373d,0x8413e22f),LL(0xa9cf3ead,0x0ecb66be),LL(0x87139b8b,0x7418372e),LL(0x5e42b4d7,0x6aaccf29),LL(0x31fc932e,0xb6dc5925), + LL(0xb88ee8f9,0xfa3b4c8e),LL(0xb521ab57,0x1f288e60),LL(0x2e8c4d8c,0x06aa3956),LL(0xcf89935b,0x4981c3e5),LL(0x45fa071e,0xbdbd0c47),LL(0x496073be,0xa78f831c), LL(0xa4e5c001,0x09a72986),LL(0x709cb728,0xac527731),LL(0x988f2781,0x9a64b5b3),LL(0x73b1719d,0x6ac9440d),LL(0xe3d2e807,0x58ad54c7),LL(0x8f06742b,0x1c157448), + LL(0x75c953b2,0x6b5b7b2a),LL(0xa7f1cd5b,0x927ed77c),LL(0x4cba0e5e,0x2e8c5399),LL(0x3f4a941b,0x03aeb14a),LL(0xa1385c8a,0xedbad9a0),LL(0x67fd2258,0x925a49c1), LL(0x3365ffed,0xe7e368ee),LL(0xd106eb87,0xcc4aad2d),LL(0xa980b53b,0x4ce908da),LL(0x16929ac8,0xd3f49540),LL(0xd5c05c32,0x613c804d),LL(0xd7973344,0xa42290cc), + LL(0xa98cf218,0x33952177),LL(0x579ee53a,0x841d9e1f),LL(0x0a285bd5,0x1084d61e),LL(0x71171b1c,0x3935a84e),LL(0xf29b29f9,0x8ac2433c),LL(0x6dd1e9bd,0x5dd868b5), LL(0x8d102390,0x88da0478),LL(0x657400d1,0x1140735a),LL(0x9d5b19e1,0xa792a25f),LL(0x6a27fa79,0x9ee015cb),LL(0x7ba16a8e,0xea3bf8b5),LL(0xc15fde67,0xc5f0cc26), + LL(0xee2c3290,0x2e152d95),LL(0x4a9ceda4,0x8437df2e),LL(0x3c7ebfd1,0x4151754e),LL(0x88f80aea,0x556c59a8),LL(0x8de44dbc,0x8d099c5d),LL(0x77abeecc,0x9ecce7fc), LL(0x3aa311cf,0x5e0a0f38),LL(0xb8f2bff5,0x99ff1eec),LL(0xb5dcf488,0x5ae0b483),LL(0x91483a02,0x11212c45),LL(0x312134a1,0x99fe0738),LL(0xa72745ef,0x3b855db0), + LL(0x892c8eec,0x37f50e43),LL(0x7d85a7e2,0xf06a2f04),LL(0xe1d11150,0x3916af85),LL(0x6785ae1c,0xf56e852f),LL(0xae6ada8c,0xbf8c72ad),LL(0xe13285b2,0x1fcd53e3), LL(0xbd56d348,0x5327920c),LL(0x445658a8,0x82a394fb),LL(0x3caf3792,0xa7132857),LL(0x550ffe1c,0xb15ab34b),LL(0x6a5d4e4f,0x81898066),LL(0x2f854f9d,0x0bda153b), + LL(0x722730fe,0x77a31009),LL(0xd5cdd297,0x93707ac4),LL(0xd3811e8c,0xa290be39),LL(0x92a5cdb7,0x831a9b95),LL(0xe7342270,0xc74cda84),LL(0x3f48affc,0x96466190), LL(0x5520b0f0,0xb0496cca),LL(0xbae930ff,0xc8742cd9),LL(0xeaea703a,0x3a30737a),LL(0xfb758854,0x0a8e6fb7),LL(0x6796f4d1,0x9ab9523e),LL(0xfdf7140f,0x36e6c05d), + LL(0x64ef6a95,0x3b623150),LL(0xaaa5b792,0x97645381),LL(0x56471100,0x4bc2c31c),LL(0x1bae8d2a,0x4a0e73bb),LL(0x8df1f76a,0xbfc0770a),LL(0xa7bb16ca,0x5089916f), LL(0xf31fe82e,0x2afe5b1c),LL(0xf0119977,0x0b06831d),LL(0xa1af2a82,0x97caa333),LL(0xdafed6cd,0x93cb92c5),LL(0x92c3b2e3,0x09553e7e),LL(0x61af2956,0x3d9c4b7d), + LL(0x08f84746,0xd83f574a),LL(0xca07f5f8,0x48fc9715),LL(0xdcc51638,0xb3d5d0d2),LL(0x6153bdcd,0xc2a5e335),LL(0x8aa4ef74,0x8242cd9a),LL(0x0bdaa0d0,0xe71ba25b), LL(0xa4ff172d,0x4342d4bb),LL(0xfc1341a2,0x81db10df),LL(0x7dacb140,0xdd93dd87),LL(0xd12d347f,0x6f8a4e81),LL(0x1bc369be,0x0d4e7e46),LL(0x1fafd0c5,0x3ce10a77), + LL(0xe67145b6,0x5559dd31),LL(0x5b2427e7,0xf2d905b4),LL(0xcaf57d0c,0x0d840fab),LL(0x78742ab6,0x96258665),LL(0x409c1c8e,0xc85482ad),LL(0xadaa6167,0xdca2a058), LL(0x0c8885fd,0xec26ad9a),LL(0x2a600cb2,0x1b93b8a2),LL(0x2539986b,0x340aa7fc),LL(0xa23dee41,0xd7674876),LL(0x2e1a9837,0xa948a929),LL(0x71438da9,0x9ae67d2a), + LL(0xd56bdf1f,0xeac6f447),LL(0xc2b502ff,0xb22e8425),LL(0xfca5a501,0xe1cc9d3d),LL(0xb64baf39,0x8192bc29),LL(0x52ce849e,0xeb2c901a),LL(0x1dd506f1,0x7f5f38b1), LL(0x0f0a1d68,0xfb3684b1),LL(0xe9240ff8,0x16c4aacd),LL(0x5a4d8995,0xffa68243),LL(0x54e4c95d,0x27264ab5),LL(0x4f34ffaa,0x9aa40cdc),LL(0x5fd818ee,0xcb8a30a3), + LL(0xf7f35053,0x39038863),LL(0x328787d2,0x421a17f3),LL(0xf3d8310f,0x38aa682e),LL(0xf4123153,0xb52d41e8),LL(0x7026310b,0x4fbef3dd),LL(0xf6ff5692,0x0c6bd7ad), LL(0xa9be5d0c,0x3831c6b2),LL(0xe8d328b8,0xb5c9ae85),LL(0x6516bba4,0x76d26abc),LL(0x446d35a8,0xc237f9a5),LL(0xf012a8d0,0xb2b16c0f),LL(0x0ee0315b,0xddf2b7fe), + LL(0x056ad6c2,0xbb85b640),LL(0xac074372,0x7c51ef96),LL(0xf10b43fc,0x1c7ce31c),LL(0x26f4d3a4,0x08e4101b),LL(0x3968459f,0xd18511c4),LL(0xd6d07839,0x00e20c3f), LL(0xe4fcdc11,0xd5bcd598),LL(0xc877f6a2,0x99e9a4d0),LL(0xbd491646,0x9c5dd9d0),LL(0x9bfd7a1a,0x83918f60),LL(0x7e2b95a3,0x4bc130cd),LL(0xfbc31c83,0x668825fb), + LL(0x06a9ad54,0x817d77b1),LL(0x89a25eca,0x3a999d7d),LL(0xda68b768,0xd3ac4107),LL(0xbebc4c4d,0x6904bcdd),LL(0xa53d39e9,0xb0d2103c),LL(0x30a5e950,0xdba86bd2), LL(0x4f52208e,0xb0925680),LL(0x28495b2c,0x37c3156a),LL(0xc15855ae,0x2389ab34),LL(0x3017194f,0xc14dfd96),LL(0x1146b838,0x420e0719),LL(0x8fb4b6fc,0x1a9f909b), + LL(0xdd63404d,0x2926ef17),LL(0x1399cc68,0x0e89c4d4),LL(0xf7ec20b8,0x6507fede),LL(0x88c751d6,0x1ac084ff),LL(0xdefe29e6,0x31bc08be),LL(0x4f0692c5,0xd4219971), LL(0x36069bc0,0x4d6ee742),LL(0xff80f3d7,0x3868ef6a),LL(0x5a9c6f4b,0x6df02d7c),LL(0x101abf69,0x2c3096bb),LL(0x8eaacaeb,0x0c2b01ec),LL(0xeb2e687a,0x65914c20), + LL(0x79ed523a,0x13722ab0),LL(0x249d5624,0x33b29bec),LL(0xf76fdaf7,0xd3d0f467),LL(0x12ddfd9a,0x7ce072f9),LL(0x47bdefd3,0xce918a57),LL(0x750e5315,0x14d38ab4), LL(0x3346f647,0x08bbb20e),LL(0x05b26894,0x428b917f),LL(0xca865ba6,0xc8fb5c21),LL(0x2e6e8e6f,0xee6e41e0),LL(0x4c608b60,0xd00ae621),LL(0x6ff685cd,0x65975639), +}, +/* digit=28 base_pwr=2^140 */ +{ + LL(0xa368eff6,0xbbccce39),LL(0x8ceb5c43,0xd8caabdf),LL(0xd2252fda,0x9eae35a5),LL(0x54e7dd49,0xa8f4f209),LL(0x295100fd,0xa56d72a6),LL(0x56767727,0x20fc1fe8), LL(0x0bbaa5ab,0xbf60b248),LL(0x313911f2,0xa4f3ce5a),LL(0xb93dab9c,0xc2a67ad4),LL(0x22d71f39,0x18cd0ed0),LL(0x5f304db2,0x04380c42),LL(0x6729c821,0x26420cbb), + LL(0x0eb008c8,0xca07923c),LL(0x9985912e,0xab79402d),LL(0x3cb02510,0x41e379e8),LL(0xbeb383ef,0xfabac005),LL(0x1076dd0d,0x24d12d9a),LL(0xb208f127,0x95afd46f), LL(0xb1031e46,0x9cc38a60),LL(0x7009f6bc,0x93e21e97),LL(0x8ac219ef,0x6f6360d9),LL(0xaf284c80,0x1edaab3f),LL(0x019e366a,0x9c3b5281),LL(0xbc9e9726,0x6475c579), + LL(0xbdfbcae8,0x26bd07d6),LL(0xdf01a80a,0x10b5173f),LL(0x6798b96c,0xd831c546),LL(0x1d3f3859,0x1d6b4108),LL(0x991b9ec7,0x501d38ec),LL(0xd78431a9,0x26319283), LL(0x118b343c,0x8b85baf7),LL(0x58def7d0,0x4696cddd),LL(0x7acdcf58,0xefc7c110),LL(0x848d5842,0xd9af415c),LL(0x0ac7fdac,0x6b5a06bc),LL(0xa344319b,0x7d623e0d), + LL(0x8d85a25a,0x8410d829),LL(0x4af81a14,0x48ee0135),LL(0x18c25348,0xae460d0d),LL(0x7eb035a3,0x5d0279a0),LL(0x9a114414,0x87e7c128),LL(0xc0744f79,0x17c08a8e), LL(0x025cdbe3,0xb7b2b4f1),LL(0x82d1af60,0x9a74f15d),LL(0xb51ee685,0x124a7395),LL(0xf6122422,0xf2937c4b),LL(0x07f1a7ff,0xb4ec1332),LL(0xf886032e,0xad801112), + LL(0x0c9d3547,0x4c0d7806),LL(0xcf2aed47,0x993f048d),LL(0xe4b57e22,0x5217c453),LL(0xf4172b28,0xb4669e35),LL(0x49f999f8,0x509a3cd0),LL(0x87c69d41,0xd19f8632), LL(0x4c8fded0,0xe14d01e8),LL(0xeafd9e1c,0x342880fd),LL(0x70dc2bf0,0x0e17bff2),LL(0xc0186400,0x46560b7b),LL(0x49a4dd34,0xe28c7b9c),LL(0x0f325d06,0x18211916), + LL(0x7bb5346e,0xdd4eb3d0),LL(0x382e7db7,0x9a46ad01),LL(0xdc1973c7,0x1200285d),LL(0xa0046b98,0xfd342bea),LL(0x1219a7fc,0xd1917349),LL(0xb7caffe5,0x5383d319), LL(0x2e0fa118,0xea5a0c4e),LL(0xa5457b28,0x1cc2de3c),LL(0x6046eeea,0x5b2a16dc),LL(0xcc8e64b1,0x1755e1fe),LL(0x9e7fadda,0x51e4946e),LL(0xfcbf4ec2,0xf805422f), + LL(0xd7e02e18,0x46d70888),LL(0xd9f11fd9,0x7c806954),LL(0x4fbea271,0xe4948fca),LL(0xbd80a9df,0x7d6c7765),LL(0xf3871c71,0x1b470ea6),LL(0x8330a570,0xd62de244), LL(0xc659c3a7,0xdaecddc1),LL(0x077f7afc,0x8621e513),LL(0xcaeeef13,0x56c7cd84),LL(0xc685a356,0xc60c910f),LL(0x9dd93ddc,0xe68bc5c5),LL(0xfeb64895,0xd904e89f), + LL(0xbd08ffaf,0xf877e8c6),LL(0xaf23012f,0x24718fef),LL(0x2b004cfe,0x19ff269f),LL(0x95450f8b,0x8adc5d77),LL(0xe2a7d458,0x688ce8bc),LL(0x97bd7fdc,0x74d7445b), LL(0x41e6abad,0x1b9f4ad6),LL(0xf00e4bf5,0x6652ed05),LL(0x71d83d86,0xabee1f7e),LL(0x25ffc219,0xe693c76d),LL(0xc873f553,0x1c9a84af),LL(0x66d77a55,0x84d27187), + LL(0x8ba7917a,0x75d874fb),LL(0xfd043bd4,0x18fa7f53),LL(0x1fc3979e,0x212a0ad7),LL(0x5d6eac0e,0x5703a7d9),LL(0x017dead5,0x222f7188),LL(0x0f6c1817,0x1ec687b7), LL(0x238bacb6,0x23412fc3),LL(0x54ced154,0xb85d70e9),LL(0xbda674d0,0xd4e06722),LL(0x36f5a0c2,0x3ea5f178),LL(0xf5c6d2ca,0x7e7d79cf),LL(0x3dbb3c73,0x1fff9464), + LL(0x7e5f7121,0xe566dc05),LL(0x2ed07bc3,0xccac74e2),LL(0xc70401b4,0xaabfdfcd),LL(0x6254e0db,0xac9fc449),LL(0x11c7de05,0x358d885f),LL(0xd60772b4,0xb8e6a4a9), LL(0xcfe917ce,0x884272a5),LL(0x9a3d347a,0xdfbe9868),LL(0xc9d1bacc,0x06b90848),LL(0xdb8c6288,0xc4ccedb6),LL(0x79e5683e,0x892878b9),LL(0x243273e3,0x1b521829), + LL(0xf163e4a8,0x916e19d0),LL(0x1489df17,0x1e6740e7),LL(0x339f3a47,0x1eaf9723),LL(0x124b8dad,0x22f0ed1a),LL(0x49c3dd04,0x39c9166c),LL(0xce1e9acc,0x628e7fd4), LL(0x40031676,0x124ddf27),LL(0x1eddb9be,0x00256939),LL(0xd360b0da,0xd39e25e7),LL(0x4aa6c4c9,0x6e3015a8),LL(0x623eda09,0xc6a2f643),LL(0x50aa99fb,0xbeff2d12), + LL(0xbf0c6fbe,0x099369c4),LL(0xfe7d5727,0x976f78b2),LL(0xd18267a9,0x32feb503),LL(0x1a7dd0fe,0x162c4150),LL(0x26b8e969,0x3141e377),LL(0x3b53a94a,0x50497a64), LL(0x607b4cfc,0x96159f41),LL(0x2f111bab,0x1999b704),LL(0x760f2eae,0x3254987c),LL(0x841014fa,0x5308075b),LL(0x4e7adad8,0xc634127e),LL(0x59ffbfe6,0x32a70a60), + LL(0x93ee8089,0x1feef7ce),LL(0x252dd7bd,0xc6b180bc),LL(0x1788f051,0xa16fb20b),LL(0xe046ed39,0xd86fd392),LL(0x9378ce1d,0xda0a3611),LL(0xa5f7a61d,0x121ef3e7), LL(0x92d13cae,0x94d22061),LL(0x77c72e08,0x5076046a),LL(0x7d2308b9,0xf18bc233),LL(0x17f977b1,0x004db3c5),LL(0x0471c11d,0xd05ae399),LL(0x85cd1726,0x86a2a557), + LL(0xa1f857e6,0x7279c369),LL(0x27fb373a,0x029d30ef),LL(0x6827358b,0xe82cbc80),LL(0xa18f57ab,0x2bfe09aa),LL(0xe5503492,0x63bf3145),LL(0xfb28ee43,0x7ea15bea), LL(0x5eec91b8,0x8e6d428f),LL(0x611b1799,0x215e03e9),LL(0x61d476de,0xb9957371),LL(0xe76726a5,0x2320c764),LL(0x8e5e26f5,0xc5de8817),LL(0x9161e0b7,0x24aae069), + LL(0x72107804,0xb8d9b286),LL(0x3303b79b,0xb5a7c413),LL(0x5fa37ded,0x927eef78),LL(0xad67daba,0xa1c5cf1e),LL(0x7360e7c7,0xaa5e3fb2),LL(0x0a0c0993,0x8354e61a), LL(0x7f5458cc,0x2ec73af9),LL(0x48474325,0xde4cb488),LL(0x7209bc69,0x2dd134c7),LL(0x451a2abe,0xb70c5567),LL(0x8e293018,0x2cd1b200),LL(0xd33c0d72,0x15f8da7a), + LL(0x893b9a2d,0x5584cbb3),LL(0x00850c5d,0x820c660b),LL(0x7df2d43d,0x4126d826),LL(0x0109e801,0xdd5bbbf0),LL(0x38172f1c,0x85b92ee3),LL(0xf31430d9,0x609d4f93), LL(0xeadaf9d6,0x1e059a07),LL(0x0f125fb0,0x70e6536c),LL(0x560f20e7,0xd6220751),LL(0x7aaf3a9a,0xa59489ae),LL(0x64bae14e,0x7b70e2f6),LL(0x76d08249,0x0dd03701), +}, +/* digit=29 base_pwr=2^145 */ +{ + LL(0x31cb94c9,0xaff47822),LL(0x803c1af4,0xf1b5a0b7),LL(0x2ef696a9,0xbeb85f8d),LL(0x4fa94fca,0x8ce5baab),LL(0x00d41a43,0x0a32f962),LL(0x74f6e772,0x0f69ad57), LL(0x6ccb5157,0xbe0221af),LL(0x2a4f91ff,0xcb83969a),LL(0xa7e49f39,0x78ff85d6),LL(0xcb5d3c63,0x63006589),LL(0x96eb65f5,0xe8e43835),LL(0xff8adbdf,0x79f59da9), + LL(0x10eeed24,0x082ea61d),LL(0x143fd59d,0x7c9d5ade),LL(0x2e54f5cf,0x7d33df96),LL(0xe39dc6ab,0x340b0d36),LL(0x8d179b13,0xd97a8b84),LL(0x288d388c,0x88184bb0), LL(0xe116ae6d,0x2237e507),LL(0x211b2cf0,0x3e97b063),LL(0x42be7459,0x645f8bcb),LL(0xde2176b6,0xce2b0f54),LL(0xd1e2f09c,0xaf570a09),LL(0x57fdc001,0x110adf56), + LL(0x842e4246,0x9d21c740),LL(0x4ab098a5,0x30f474c4),LL(0xaae5d701,0x57f8b1a3),LL(0x91978d15,0x477e4f88),LL(0x0fb85b1e,0x2913ffb4),LL(0x58489fb4,0x80aedb22), LL(0x0912d86a,0x0e1ab267),LL(0x82933f3a,0xea5e6a41),LL(0x57ab8d86,0xf578ccb6),LL(0x547f64bd,0x339fd796),LL(0x3f3e497e,0x90469394),LL(0x1d864706,0x2cde596c), + LL(0xbcb6db29,0x158bfe27),LL(0x0054d963,0x96721241),LL(0x8e71aca1,0xf07b153b),LL(0x71b11643,0x5e676981),LL(0xd04e2f90,0x77b7dd7d),LL(0xf0dcf109,0x07814aa6), LL(0xfe1d0b1e,0xd3bab2a4),LL(0xbe69e691,0x50abba31),LL(0xc6f53cd9,0x54fe99af),LL(0x628039e4,0x071f2a4f),LL(0xb183aa16,0xf1f44181),LL(0x5010f6f9,0xdf0138e0), + LL(0xaff45947,0x5737f903),LL(0x62c5dfd1,0xe576f8ff),LL(0xe1e379a2,0x2a7d4f18),LL(0xf2f26f8c,0x8596ffaf),LL(0xfab86608,0x12f49653),LL(0x3aab1eaf,0xcc618497), LL(0x865d9611,0x7b5d029f),LL(0xbf6861f5,0x87af2d58),LL(0x31b2d7cf,0xb7653646),LL(0x8be46ef9,0x60f47e05),LL(0xb1c175ce,0xa21106a9),LL(0xdeb065b6,0x21cb0e27), + LL(0xb340f4eb,0xb287e26d),LL(0xb34c2948,0xa397cfd1),LL(0x9a7ead15,0x0d91d763),LL(0x66e88f14,0xd71901b9),LL(0xb4884ac2,0x02447ad3),LL(0xd006f448,0x5e6f7545), LL(0xec5744e3,0x2af6bfee),LL(0xf55cd6ab,0x643f46b1),LL(0x87b2c127,0x13400bc4),LL(0x4629859b,0xd1dddff4),LL(0x20feecb9,0x9710cae0),LL(0xd02d4300,0x594bc27c), + LL(0xaa331700,0xb0233c4d),LL(0x30b3a854,0x3bf63a61),LL(0x49b806e2,0xc269d68b),LL(0xda8ddca6,0x4a079274),LL(0x0db04d76,0x256bc009),LL(0x1f47dfee,0x7395c11c), LL(0x13886ec2,0x51bbcc95),LL(0x244d1f0d,0x9d5377eb),LL(0xff9e6b3a,0xa6e0f054),LL(0x204e5258,0x307f9e93),LL(0x8d34d97b,0x8069c01d),LL(0x43c24997,0x30f785ce), + LL(0x9c77ca60,0xb1e36501),LL(0x01018e14,0xfe084a23),LL(0xa4bfdcec,0xbf451d2c),LL(0xb29cdcfe,0xd210892f),LL(0x94514871,0x5b12bcd8),LL(0x1809b1e3,0xd03ca18c), LL(0x5858e4ea,0x09b24311),LL(0xe57524b4,0x37b30d50),LL(0x5de334b5,0xcef0a16b),LL(0x0b116076,0xfe0bd1e2),LL(0x89ae2bf4,0x54e4b482),LL(0x68c8a937,0xfbcc5e1a), + LL(0x91ae19fb,0xf7473902),LL(0x1bed3f8d,0xc1c228d3),LL(0x6552154e,0x763ba8ee),LL(0xe2063ce7,0xb7b60248),LL(0x178a5184,0xafaf01b8),LL(0x7901d21a,0xe193b834), LL(0x260eb30f,0x1d29f7d6),LL(0xc23b4b94,0x030516c6),LL(0x7dc4a09a,0x30a8046a),LL(0xb5ee1147,0x8414133a),LL(0x4453f1a6,0x7ec8dccf),LL(0x59da8e8d,0xe380e69c), + LL(0x22c77f4b,0x28fdf72f),LL(0x32485220,0xfa077b15),LL(0xd6325081,0xc3801faf),LL(0x891ff5df,0x8dd4e3c9),LL(0xaa73c827,0x6726c5ec),LL(0xe9b9b128,0x12af9707), LL(0xff25cf07,0x5e6ae3cb),LL(0xd7cb7adc,0x0613bef8),LL(0xcef4cd47,0xfd426caa),LL(0xced0e435,0xe14ed3ef),LL(0x6000ec80,0x8323a861),LL(0x941f8071,0x58221041), + LL(0xa83eb17c,0x6f766821),LL(0xbd7ff851,0x91fe58c1),LL(0x9063af8a,0x7ce09f04),LL(0x6c109e02,0x5230b3c6),LL(0x36274fb9,0x501adb33),LL(0x90547af3,0xcfb34bec), LL(0x40f61ec9,0x622d1387),LL(0x0e3e98d1,0xdf26e4c6),LL(0x7c676ee3,0x6b3b3d62),LL(0xf0244737,0x9f841097),LL(0x4fc8dd58,0xcfaf6ead),LL(0xcd534ab4,0x4f5d463b), + LL(0x64be0f56,0xa5023e13),LL(0x0046f45c,0x6a7310e0),LL(0xec8700d3,0xe0af09ae),LL(0xeb2d38f0,0xdea5fb7c),LL(0x859852e6,0xc038eae6),LL(0x8c34f04c,0xd515fb4c), LL(0x1488c207,0x546b778e),LL(0x6258d8ba,0x8cf4f114),LL(0x5182c96c,0x474e60d8),LL(0x3dbde757,0xcd038730),LL(0x76ab01ff,0x387232f8),LL(0x28231392,0x277614f6), + LL(0x80b86bc6,0x930c3c59),LL(0xa06d10a8,0xebc8c144),LL(0x81e55432,0x20743ddd),LL(0x006d3073,0x059521d4),LL(0x12dbc785,0x8d4fe303),LL(0xdf4901d6,0x15a4debc), LL(0xb37b2d8b,0xc9b226b1),LL(0x0fe9bdbf,0x385ac3c9),LL(0xfeaa5bff,0x64df89f7),LL(0x81de42ba,0x67de8e03),LL(0x5bf63f6f,0x12bd8322),LL(0x32b4ec13,0xb499f8ff), + LL(0x9608c827,0xba0800b8),LL(0x47dfcd84,0x560c31f5),LL(0x520dba6e,0xd52e8fdc),LL(0x854fa4ee,0x66a917c2),LL(0xc6e5d664,0xfc543b79),LL(0x32b530aa,0x5b643692), LL(0x563a3933,0x427dc1ef),LL(0x88d5902f,0x4e47c929),LL(0x93eca4fb,0x9eff3a40),LL(0x59260a69,0xc5b396ee),LL(0x48e70137,0x936a5062),LL(0x21bba959,0x69eb44f6), + LL(0xab9f58a2,0x0af34aa7),LL(0x7d150dcb,0x0ba16d98),LL(0xb7c119b4,0x444d5495),LL(0x5ecc3210,0x317a55e5),LL(0xb1aa90c0,0x8fa3b3ce),LL(0x0b2e9392,0x8e7c7539), LL(0xe8921afa,0x5ee5f3e8),LL(0xa3100dd3,0x3934ea8d),LL(0x4c8ae6ce,0x955045b1),LL(0x649897fc,0xa54acb5f),LL(0xe7081246,0x4204fab6),LL(0x6dce3a55,0xec3dc968), + LL(0x3ef5a413,0x309b1eb3),LL(0xa81f43fb,0xa7607981),LL(0xbf8a894c,0x87c2b81e),LL(0x0d293293,0x27a40bce),LL(0xe4bf3714,0x7f4c315b),LL(0x01236895,0x03fdc14e), LL(0xdff053fe,0x319c88f8),LL(0xea3fa121,0x146bb448),LL(0xf0dd1380,0xfcc2a05d),LL(0x4acba9fa,0xc8d55b02),LL(0x5927313e,0x871358de),LL(0x17ce294a,0xfd1d81d3), +}, +/* digit=30 base_pwr=2^150 */ +{ + LL(0xff63ae69,0xd3797831),LL(0xce4c7eaf,0xa753de02),LL(0x11a4e339,0x2ff7a6a6),LL(0x5328043a,0x904f86f0),LL(0x12e9f7dd,0xe29d31c0),LL(0xc0a51904,0x8825a639), LL(0xebfc2cc7,0x070c2696),LL(0xc5f7a943,0xc03ce643),LL(0x12c8a1f5,0x5b970d0c),LL(0xab352a83,0x572aaaa1),LL(0x0c5eb0c7,0x63df45a9),LL(0xd4977599,0x95c951e1), + LL(0xbca07a42,0x2c5b0e42),LL(0x7a0dffa1,0xbe57f359),LL(0x9aa90727,0xace48595),LL(0xf658699b,0x32be886a),LL(0xda3b18e6,0xce75d6c6),LL(0x69caf667,0x9d563e4f), LL(0x065eb772,0xc17c66cf),LL(0x4df9f6ef,0xfbe12381),LL(0x623db4ef,0xceb80041),LL(0xc74762e1,0xe75615b2),LL(0x8671c52f,0xade8a543),LL(0xcacaf2ec,0xb713c401), + LL(0x467eb167,0xe6d039ef),LL(0x74696cf9,0xa7e0959d),LL(0x7078d8a0,0xf3a19b9d),LL(0x07cdc6f6,0x5d4ec99c),LL(0x8386eed8,0x4842d0f9),LL(0x545fc0d5,0x48f5ab80), LL(0x6d39c2f7,0x8906fc62),LL(0x1bf5366a,0x1c050d69),LL(0x9f54d0d6,0xac506c57),LL(0xf9e4b94c,0x9a356a6e),LL(0x08a75e61,0x62632c51),LL(0xc6951dc2,0xfc1b9fa5), + LL(0xa4886619,0xa933aaf7),LL(0x4af13c7f,0x9ec1915f),LL(0x854de496,0x25a9dff8),LL(0x247bec15,0xa8b31d9b),LL(0x4661e58d,0x468a25c8),LL(0x786a0707,0x8989c046), LL(0xbb66922e,0x282db8ca),LL(0x45ca29ff,0x73bf240d),LL(0xeaeda06e,0xa2c40faa),LL(0xadd94b47,0x69632929),LL(0xb0069076,0xc72354f6),LL(0x7878e92c,0x8d197fbf), + LL(0xa83bbb88,0x69a9ebd8),LL(0x29f98875,0xcbab0b5a),LL(0x4e7611f0,0x325e487e),LL(0xd955cc3b,0x90aa24b1),LL(0x3c264d53,0x840e70a1),LL(0xad7f4f81,0x15bcf88b), LL(0x2cf0df0a,0xe47552cc),LL(0x79205ea9,0xcb999733),LL(0x10d5ca45,0x25dc58bd),LL(0x1228b978,0x0947d715),LL(0x4f2c7c4a,0x9a0204da),LL(0x4690052c,0x4377ea4a), + LL(0xb8e79179,0x015c325e),LL(0x5b57dce6,0xf4fc6133),LL(0x78d6858f,0x27a51e5d),LL(0x4dd5f180,0x13babcab),LL(0x847e499e,0xfaa19cb1),LL(0x08aaea61,0xe2688ae6), LL(0xe86100d5,0xe20d7edc),LL(0xed2fedac,0xa9b0d46b),LL(0x1d357ded,0x5e99cc0c),LL(0x723cac89,0x4c1263ab),LL(0xf15e22f4,0xad5f3e6f),LL(0xd77dae65,0xf25f3950), + LL(0x37e8e6b2,0x3c0e2b97),LL(0x575da8b7,0xa2037913),LL(0xb925cbb2,0xeedf0a75),LL(0xc561b405,0x4f28ec1b),LL(0x2901931c,0x368fb274),LL(0x2f26221f,0x52b54eee), LL(0x247812a9,0x381845b6),LL(0x9115a0df,0xf9bcc961),LL(0xcb84d25b,0xef127dfe),LL(0xfa10e0a7,0x4256afe5),LL(0x353a15eb,0x0c08a532),LL(0x6a91e61e,0xbbd15b17), + LL(0x6150771a,0x854b0584),LL(0xd9ca9868,0x35fdd9b4),LL(0x4c32fc71,0xec829389),LL(0x9ec8f90d,0x882fad4c),LL(0xc6c7b9c0,0x2d39990d),LL(0xd71a25e5,0x7fbc201b), LL(0x5166da7d,0x6b852e65),LL(0x3d8c6e36,0xc6bde23a),LL(0x5857f048,0x37001154),LL(0x1ccb9bc8,0x746621fc),LL(0x612bb853,0x97e44e63),LL(0x758da4ed,0xabc3b450), + LL(0xbb44db8a,0x856463a8),LL(0x4baf2f64,0x18ec5abe),LL(0x92980518,0x75efa67d),LL(0x94b03911,0xabd300f8),LL(0xc1cd1cf7,0x4ea44bd4),LL(0x124ef41a,0x525c3583), LL(0x6b3a701c,0x03c18c0a),LL(0xcc7b6885,0x5ef82731),LL(0xa00d0089,0xb3716386),LL(0x741c0ea9,0xed9af50c),LL(0x8a36f03f,0x31f0e49a),LL(0xca724a16,0xce993757), + LL(0x9a90c6b9,0x2b6bf5e2),LL(0xa2c0b35f,0x9b134fc8),LL(0xa6717af8,0xd64157f8),LL(0x45dd16f0,0x7228156c),LL(0xdaa99226,0x0a9ab894),LL(0xbccb07e5,0xa7b130f7), LL(0x4d9243d5,0x2b7ab41f),LL(0x88568fa4,0x570b4ed0),LL(0xc9c84d91,0x666b98be),LL(0x02124f7a,0x2a75793d),LL(0x0aa16891,0x0fb9b82e),LL(0x3cc7f2d7,0x71e94c93), + LL(0xf47211c0,0xb7c13bfb),LL(0xe70929ae,0x738fe4a9),LL(0x423db6f3,0x4cdac99b),LL(0xe4255a4d,0x3883201a),LL(0x9730d749,0x7ffe82d3),LL(0xafbaaabb,0x033ba755), LL(0xf836d62b,0xa302ba65),LL(0x143a88bb,0xc626f604),LL(0x31c16b3f,0xe1d189aa),LL(0x7d68ab17,0x4664c050),LL(0xc02eb56b,0xcffc651c),LL(0x6ab49c24,0x6c52f3c4), + LL(0x8fb415b7,0x58775857),LL(0x9e88dca3,0xab580f15),LL(0xa62265d2,0x86a12e3d),LL(0x72d98b08,0x8ec42786),LL(0xf61e9c85,0xb9da8016),LL(0x800994ce,0xa895aedb), LL(0xe38ef526,0x63d0878c),LL(0x1efb6575,0xa081d714),LL(0x6a1c1efa,0x780b9e12),LL(0xebd0497a,0xeed68d0e),LL(0x9265231e,0xbfeee3d2),LL(0x80e03127,0x46f751da), + LL(0x03e4074c,0x806dce0b),LL(0x6973bc6d,0xd4054700),LL(0xa2897b68,0x8c4d393e),LL(0xc7f9af16,0x7d592d04),LL(0x4f895dbf,0x0625826a),LL(0x37dfc8a1,0x038bed29), LL(0x4799d78a,0x981862da),LL(0x6675c1b3,0xf34c1895),LL(0x8706afd3,0x64de9a5b),LL(0xf74e6ca3,0xc80ff68e),LL(0x83ed8cf2,0x26a90422),LL(0x77a47011,0xc1f5ce53), + LL(0x4aabf727,0x0bd15783),LL(0x54fce3ad,0xfe79d2dd),LL(0x1a77ce78,0x242b1806),LL(0xda7e489c,0x30cf6c32),LL(0xb2966f38,0x854e43f8),LL(0xa2dd4ea3,0x6e5fb045), LL(0xde3c9b7a,0xb2a48aec),LL(0x3625ebd6,0x66ac77e6),LL(0x0bcbabe2,0x332a969b),LL(0x5be51225,0x19de2701),LL(0x5b9b80e6,0xf26c73fc),LL(0x91025007,0x98c9cbf7), + LL(0xbc098b8e,0x1e41874e),LL(0x8a234773,0x758ccd5f),LL(0xc1bc847f,0x47ab76ca),LL(0xc540ceaf,0x377f32e9),LL(0x69c2df21,0xba8897c3),LL(0x233c3a02,0x6afacb01), LL(0xfb54ffb1,0xf1609b45),LL(0xb8f9c150,0xb7a98e5a),LL(0x08b1977e,0x607478b0),LL(0xd48b7a90,0xa9500582),LL(0x32fa7597,0xacd841e3),LL(0xf9333957,0xdf53373d), + LL(0x04926a41,0xd25f6508),LL(0x514045da,0x7236b475),LL(0x08b9b08b,0x0b360311),LL(0x3fe92e91,0x16477aff),LL(0x03189ddc,0x6e5f6cb1),LL(0xc698a38f,0x81ff008e), LL(0xc93adb23,0x02a09218),LL(0x445d8fae,0x71fcecd3),LL(0x8fd6b76c,0x55a15eac),LL(0x11ef96b4,0x1e37ec36),LL(0x30e433b5,0xd1b3b3fc),LL(0x51d174c3,0x49518733), +}, +/* digit=31 base_pwr=2^155 */ +{ + LL(0xb8c9f82e,0x7914213d),LL(0xfc038e90,0x7a3e4e38),LL(0x26a34238,0x6edae5a1),LL(0x701ce8c7,0xe566bf50),LL(0x55656e02,0x3562e875),LL(0xb4e8efbf,0x48325ebf), LL(0x66505ec3,0x5f10a504),LL(0x8da78aec,0xd8b9834b),LL(0xcc2f2e40,0x49d1fc25),LL(0xaf5718c1,0xe973bb1c),LL(0xd2d6b890,0x9b8825da),LL(0xe2f00f12,0x7de7885e), + LL(0x7ef79898,0xe37211be),LL(0x21344d16,0xa8103877),LL(0xa1b9f8b4,0xfdcd7e26),LL(0x7d7f72d5,0x5641e45d),LL(0xc449c920,0x5377c1be),LL(0xefc7b2a1,0xd3edcb0c), LL(0xe14b42fc,0xc657a9ff),LL(0x00831b07,0xc8f858c8),LL(0xd020eaa8,0x6bfcd1bc),LL(0x3f6860c7,0x17534b0a),LL(0x84c7c806,0x8ce57222),LL(0x2bd7456b,0xa1d40eaf), + LL(0xc9aa57ee,0xe0c93007),LL(0x8895a604,0xebb2d47b),LL(0xc4fd6ffe,0xb8aebc49),LL(0x73f300b6,0x2c06e1e5),LL(0x81628b8b,0xa019070d),LL(0xbaf8c1ea,0x2db1690b), LL(0xcc94ccd2,0xb3fce6c8),LL(0x85bcdf4f,0xf3014638),LL(0xe2f82c32,0xb1e62616),LL(0x68295a54,0x85581e24),LL(0xbf51f8fa,0x0f2e2ff5),LL(0x155c1f6f,0x940716f1), + LL(0x4e623856,0xaed02b6b),LL(0x3e1d74cb,0x7a6d2bef),LL(0x654e7c30,0x82226ec4),LL(0xe7034bfd,0x008ac003),LL(0x7fd6b555,0xe343c540),LL(0x1b429d44,0xca1b2907), LL(0x9c3ceea2,0xe0702a33),LL(0x732694c3,0x48079aa9),LL(0xd4652401,0x7e6d72f6),LL(0x35f60043,0xd92655ed),LL(0x273e8cc4,0xa0dbaac6),LL(0x3c3ffb40,0x0bb8f0f9), + LL(0xc95cd23b,0xb41b87b6),LL(0x55e371a4,0xb99714ba),LL(0x6f571ceb,0xb138ee8f),LL(0x80146ec7,0x09c42be4),LL(0xee9aa125,0x275ee21e),LL(0x3a878b59,0x0cef4d6f), LL(0xa801068d,0xd436eb1c),LL(0x762b8a80,0xe2c5448c),LL(0xf3640eca,0x243beee1),LL(0x32bbba7a,0xf979458b),LL(0xa63407d3,0x6bc26cfe),LL(0x392dd1d3,0xd3b6e132), + LL(0x3de4ba2e,0xbc06ecab),LL(0x9e491bcd,0xf51ca063),LL(0x453c94be,0xa6fc6fa0),LL(0xed1a6731,0x5460f943),LL(0x4ec3f1fb,0xeb11656a),LL(0xff1e7d4e,0x2fcb2cab), LL(0x8fea2286,0x59526467),LL(0x4e0bee38,0x838117a3),LL(0x24fd2ce5,0x7bdf5888),LL(0x9f2c2925,0x13df0c83),LL(0xdee97f30,0x1bf621e6),LL(0xebea6641,0xb43b2558), + LL(0x8e729329,0x246c8660),LL(0xd693dac8,0x39fcc41d),LL(0xc062a6c0,0x48a65b54),LL(0x6a5a3101,0x368a5770),LL(0x47ed1988,0xd143600f),LL(0xa764ce3d,0x48466d92), LL(0x5a22cb6b,0xb0500613),LL(0xedea070c,0xf1d77247),LL(0x617f2464,0xb1ddd151),LL(0x28b83fd9,0x79050698),LL(0xd70bf93e,0x021abb26),LL(0xab5a5e1e,0x590b3c42), + LL(0x1cfb991f,0x5906a35c),LL(0x740a7744,0xb62a4f80),LL(0x36f84763,0x65c8ac91),LL(0xbe0f1dd3,0xf73b3deb),LL(0xa2d26c21,0x40358868),LL(0x76792ae7,0xd907e90a), LL(0x668c3d5f,0x3ecea167),LL(0x6754b49c,0x731068f2),LL(0x0e006243,0x6db89109),LL(0xdd94681b,0xd29106e6),LL(0xa85a3de2,0xb40b8694),LL(0x936b86cc,0xc80c7bf1), + LL(0x11913575,0x003d45d0),LL(0x87e1186b,0x866cb2dd),LL(0x46b69a22,0x692f6301),LL(0x8174c1d1,0xd296a55c),LL(0x9f17af00,0x77ef6fbe),LL(0x3aa922e1,0x6b588be9), LL(0x033e6dd7,0x99ecb44f),LL(0x1d22b7cd,0x32edea2c),LL(0xba7006f3,0x3122b027),LL(0xbb6ebc5c,0x8950054b),LL(0x82dab805,0x4f6d6061),LL(0x1bae5f1b,0xc1205518), + LL(0xe08c180f,0x28d33e79),LL(0xf6aec9ce,0x768c7794),LL(0xce683c5b,0x5a749f3b),LL(0x8371fe75,0x717629d9),LL(0x57712c1d,0x5e828fc0),LL(0x7e4c61aa,0xb46c6ed1), LL(0x5bccf95c,0x5d927bad),LL(0xd72f68ec,0x55d6fc80),LL(0x98591dc2,0x560a99a3),LL(0x4836664c,0xc885fe8a),LL(0x26d79298,0xd18acd42),LL(0x185df1d7,0x05e4cd17), + LL(0xfcc83355,0xebc60e21),LL(0xd9119b77,0xc94dbc02),LL(0x2f18ae9a,0xceb05a31),LL(0xb8f69016,0xa8462962),LL(0x8f67b5f4,0x58dde5a4),LL(0xaf3c234d,0xb8bdf9c9), LL(0x80e85df8,0xe95c069f),LL(0xab3aa0e5,0x9d525e1b),LL(0x76276d8b,0x73c8a92f),LL(0x163530ef,0x7feb4abd),LL(0x5ef5ad73,0x8ca949b3),LL(0x2e3d057b,0xe129431e), + LL(0x60dec308,0x9d8a925b),LL(0x6b3ea363,0xb72e3efa),LL(0xdfb534b8,0x4f53ca6d),LL(0x6dd78a32,0x4e64874c),LL(0xc2a146d5,0x336e5b46),LL(0x98395201,0x07c76d63), LL(0x8fe3e815,0xa4c09522),LL(0x3221cc26,0x887e659d),LL(0xc36286ec,0x0ff92f64),LL(0xc3ebb08c,0x57b1b903),LL(0x65f00c30,0xc6bdc9b6),LL(0x9a46d36e,0x82624226), + LL(0x1a0b619a,0xc782c16c),LL(0xbe316086,0x8643d42b),LL(0xc0daa421,0x49d2966b),LL(0xb7b487e0,0x080b1caf),LL(0x144de273,0x1d33bb53),LL(0x6faf7ed9,0x8bafce2d), LL(0x408d4636,0xdafbe3cf),LL(0x7ee8835b,0xf10527df),LL(0xe2e75522,0xe1123f3e),LL(0xebe27d60,0xb388c64b),LL(0xe3f1f55e,0x2cb38dc1),LL(0xe34524d8,0x57ff8e43), + LL(0x85653dc8,0xd67dc92b),LL(0x0bc93ab9,0x8e0970af),LL(0x8b87c0af,0xb6f09baa),LL(0x52760ef4,0x5a8a9030),LL(0x1047bf85,0x2e2ae756),LL(0x85bd4e74,0xd049078f), LL(0x3729f708,0xced11ff8),LL(0xd91068a6,0xdd21cbeb),LL(0x24b3e911,0x83d488ff),LL(0x1afd2196,0x6e166fda),LL(0x4f0d2128,0x66a91211),LL(0x05c9f39c,0xd11078ed), + LL(0xbdbdf0cf,0xd87003d3),LL(0x56c298f1,0xe9750b5b),LL(0xb73ad05d,0xc256c3a2),LL(0x2ee94279,0xe0779a19),LL(0x279626a3,0x31d8b3c6),LL(0x90163bc8,0x469056bb), LL(0x23755853,0xe6aeabc6),LL(0x896a6f4c,0x9fffdfe2),LL(0xa36cf41b,0x15c1ce78),LL(0xeee41941,0xd4c8c025),LL(0x7653be9d,0xf7a917ee),LL(0x59d52222,0xfa3cba96), + LL(0x5f8ab132,0x913f9207),LL(0x5c14080f,0xd5b6792c),LL(0x787c3594,0xefab4e2c),LL(0xe7b7b7dd,0xa55d465f),LL(0x34e28e6a,0x921aaad8),LL(0x12d6a7bc,0xc4f3a35e), LL(0x6115a5ae,0x109803c4),LL(0xe709f9a1,0xc023098c),LL(0x99c5bb66,0x1a8c8bdb),LL(0xbc7c2da7,0x1cd1c2b6),LL(0x5f927eef,0x50189c97),LL(0x229f9410,0x493823d1), +}, +/* digit=32 base_pwr=2^160 */ +{ + LL(0xf0ce2df4,0x11a8fde5),LL(0xfa8d26df,0xbc70ca3e),LL(0xc74dfe82,0x6818c275),LL(0x38373a50,0x2b0294ac),LL(0xe8e5f88f,0x584c4061),LL(0x7342383a,0x1c05c1ca), LL(0x911430ec,0x263895b3),LL(0xa5171453,0xef9b0032),LL(0x84da7f0c,0x144359da),LL(0x924a09f2,0x76e3095a),LL(0xd69ad835,0x612986e3),LL(0x392122af,0x70e03ada), + LL(0x6754f492,0x3ee0a31c),LL(0x96769ff5,0x02636c6b),LL(0xf0fbfa96,0x90a64f4f),LL(0xfafea65a,0x513f054e),LL(0x9cf4b9f9,0x796ba747),LL(0x932a9590,0x3198c068), LL(0x549ee095,0x93af8a65),LL(0xa212760f,0xb8b6f72c),LL(0xc1a46c8f,0x23bc71e9),LL(0x4c9bca72,0x000643af),LL(0x848cea30,0xb6d967c7),LL(0x73312ec2,0xe06b6b4e), + LL(0x67aad17b,0xfeb707ee),LL(0x83042995,0xbb21b287),LL(0x9a0d32ba,0x26de1645),LL(0x1ffb9266,0x9a2ff38a),LL(0x8f578b4a,0x4e5ad96d),LL(0x883e7443,0x26cc0655), LL(0x2ee9367a,0x1d8eecab),LL(0x881de2f8,0x42b84337),LL(0xd758ae41,0xe49b2fae),LL(0x4a85d867,0x6a9a2290),LL(0xe68cba86,0x2fb89dce),LL(0x7f09a982,0xbc252635), + LL(0x1d85a725,0x52ec9956),LL(0xf3208012,0x0f9be000),LL(0x6dcc7816,0xe881337c),LL(0x791f7cf1,0xe4e7b6d9),LL(0x59885a42,0xfaa717aa),LL(0xf9c01e41,0xb1bbb5c7), LL(0xa0361880,0xcf208d58),LL(0x20afa350,0x24426e40),LL(0x264ce04a,0x7261871b),LL(0xcd42026a,0x66be4a86),LL(0x829f99fe,0xc5397b77),LL(0x24578e2b,0xffe4a6bc), + LL(0x8c61aaac,0xadc79436),LL(0x5e926563,0x24c7fd13),LL(0x0406c129,0xef9faaa4),LL(0x8b658d3c,0xf4e6388c),LL(0x1e435baf,0x7262beb4),LL(0xfdaeac99,0x3bf622cc), LL(0x4e1aeddc,0xd359f7d8),LL(0xd78c17b7,0x05dc4f8c),LL(0x29498ba5,0xb18cf032),LL(0x85bf35ad,0xc67388ca),LL(0x62aa4bc8,0x8a7a6aa2),LL(0x72f4627a,0x0b8f458e), + LL(0xf822d5f9,0x0733667a),LL(0x18339700,0xd7f81b9e),LL(0xa7bc265f,0x7ca29b27),LL(0xeb4f0c7a,0x9fefa698),LL(0x01f27630,0x7b6f3513),LL(0xfcfb1133,0x72f0f152), LL(0x5c81eb14,0x9928d9d0),LL(0xed8ff6cb,0xa16ac36b),LL(0xe041bef3,0x7fbd1acb),LL(0xf8d99854,0x7d25159a),LL(0xdb5a0dc5,0x2ec3a7d8),LL(0x87e3e933,0xd86fc4cc), + LL(0xc68e4488,0x3fb812ee),LL(0x60ef7281,0x53c5eaa4),LL(0x8fbefbe4,0xe5724183),LL(0xa4b24a05,0x2b7d49f4),LL(0x710c0a43,0x23b138d0),LL(0xa85ec1db,0x16a5b4c1), LL(0x305feb02,0x7cc1f3d7),LL(0x5b6c1b54,0x52f7947d),LL(0x8f56981c,0x1bda2312),LL(0xb4080a01,0x68663eae),LL(0x9f999b7f,0x8dd7ba7e),LL(0xb686580c,0xd8768d19), + LL(0x4c20e15f,0xba8418f3),LL(0xfb54404e,0x7eed2494),LL(0xbce1e82d,0x4e6438d7),LL(0xb397915b,0x9e489b3e),LL(0xfb4cf659,0xa9baea9f),LL(0x42ef4aff,0x8bc5b2ba), LL(0x7e62a188,0xae3fb533),LL(0x496e8e35,0xcd648493),LL(0xdefe047b,0x89728e28),LL(0xd24e60fe,0x63a8c679),LL(0x470f710c,0xadacbf92),LL(0x5e198d3c,0xd470aeb9), + LL(0x7afdda94,0xbcd0e0ad),LL(0x34a30687,0x95a0dbbe),LL(0x8c5e2665,0xbbe3c3df),LL(0xebf2bc16,0x742becd8),LL(0x3fa163a6,0x300ceb48),LL(0x4663354b,0x0c5d02ee), LL(0xb5e606a4,0xe4fb9ad6),LL(0xcf49ff95,0x93f507b8),LL(0x585c193b,0x9406a90c),LL(0x4ecf9517,0xad1440c1),LL(0x9cea53f1,0x184cb475),LL(0x8ef11302,0x6855c474), + LL(0x7a3e874a,0x8e3807dd),LL(0x89ac3a99,0xc4edb45b),LL(0x4bfd77d2,0x9ba9cdaf),LL(0xb540fffc,0x31d33f59),LL(0x0c60028b,0x404c8779),LL(0x89688c81,0x7f89da71), LL(0x504b862b,0xdd3390e5),LL(0xe937efe3,0xdf1e721b),LL(0x63e6036f,0x5833d0df),LL(0x385fbab4,0x7712527a),LL(0xd210c0d4,0x6347236b),LL(0x8d238e2d,0x12d7733c), + LL(0xedcafa52,0x00ecb523),LL(0x086f69d3,0x0da0ae0e),LL(0xc242f347,0xc384de15),LL(0x848c12b7,0xfb050e6e),LL(0x64e015ce,0x22f67654),LL(0x7ca122f2,0xcbdc2a48), LL(0x445fb02c,0xa940d973),LL(0x3767d89d,0x00f31e78),LL(0x613dabdd,0x2b65a237),LL(0xc875ae09,0x2be0ab05),LL(0xba204f8e,0xb22e54fd),LL(0x0f7687b9,0x65e2029d), + LL(0x302e943f,0x0ecb0723),LL(0x4a443e78,0xd180ca1e),LL(0x23dd2c9e,0x39e78911),LL(0x01fe50bb,0xfa2a4404),LL(0x154d39d1,0x4678e7ed),LL(0xaf513e01,0x64ddaee1), LL(0x634904da,0x6d4c615a),LL(0xba5c900c,0x937c6326),LL(0xeb6c8582,0x70658f5f),LL(0xf3d65166,0x2a04fd51),LL(0xb676eb47,0xcefe7472),LL(0xf597d887,0xd3565a71), + LL(0x1855a71c,0xffd82538),LL(0x438bd8d8,0x26a330b3),LL(0xf9d8c5f9,0x89628311),LL(0x953738a0,0x8d5fb9cf),LL(0xedfcd4e5,0xcb7159c9),LL(0x2064c7c2,0xd64e5230), LL(0x689f3cfe,0xf858ed80),LL(0x56128b67,0x4830e309),LL(0xe0e90688,0x2e1692da),LL(0xca9cc232,0xab818913),LL(0xa5d229a6,0xe2e30c23),LL(0x0e740e23,0xa544e8b1), + LL(0xe5dcba80,0x299520f4),LL(0x2b758045,0x522ad4b5),LL(0x193b36d4,0x54eabe27),LL(0x45e9e442,0xda4d3bff),LL(0x637311f3,0x44cb9252),LL(0x71338ebf,0x4cd620a9), LL(0xcc9524fb,0xec908157),LL(0xa8c955d7,0x2731a11b),LL(0x5cb94009,0x72a5e054),LL(0x9126cfe8,0x7eee8f3b),LL(0x3dd5d5ce,0xc71e2920),LL(0x22069494,0xe886f91a), + LL(0xdc61e6cc,0x1c15e569),LL(0x58fc7800,0x8fd72967),LL(0x37a9dfc5,0xe61e7db7),LL(0x5afd7822,0x3f34a9c6),LL(0x19e80773,0x0a112742),LL(0x4760fc58,0xa353460c), LL(0xb3124c71,0x2fb7deeb),LL(0x2d4009cc,0x48463627),LL(0xc3a10370,0x399d1933),LL(0x54388dbd,0x7eb19450),LL(0x7c2a006a,0x8ecce639),LL(0x55c932a0,0x3d565daf), + LL(0x0db962c0,0x294d2955),LL(0x6d523ab0,0xd6994ef4),LL(0x58f95037,0xfa1a7f91),LL(0x64420c94,0xb1379811),LL(0x093caea8,0x2b686e1e),LL(0xf9e1c340,0xdef10944), LL(0x611d9bf5,0xcd1beecf),LL(0xa1b5267b,0x34696c50),LL(0x2dfc2b16,0xcecbc719),LL(0xcee7e854,0x2cdb955d),LL(0xf2635cc8,0x9fefc321),LL(0x2936f7d3,0x276d2e4f), +}, +/* digit=33 base_pwr=2^165 */ +{ + LL(0x67141724,0x4962c021),LL(0xabe7762f,0x5f81eabe),LL(0xdd189c3f,0x78549a79),LL(0x6ce517a7,0x47675cdd),LL(0x32d6bb97,0x5102294e),LL(0x6ed1a029,0xb19500c6), LL(0xb16a206c,0x3efb54e8),LL(0x0dc135b8,0x7dbdcc25),LL(0x8967fb04,0x955bc294),LL(0xbe04e909,0x373615c9),LL(0x111efad6,0xf1fcf820),LL(0x6fd2e97a,0x8530f97d), + LL(0xb4805410,0x3f2b5bd4),LL(0xf96c5ee7,0x201ca7a9),LL(0x94256fe1,0x532ef2db),LL(0x318ddb03,0xacbfc459),LL(0x5f24c8e1,0x2375f9fd),LL(0x370783db,0xd27c479b), LL(0x56541ae6,0x1bd461e8),LL(0x7f7ea49a,0x78f054a7),LL(0x8845f315,0xc9f8777d),LL(0x97fc92c7,0x81aed296),LL(0x49929540,0x9f2f8d79),LL(0xff5ebfe0,0x7531e78b), + LL(0x543b3e41,0xbd9a66d6),LL(0x2ae73774,0x2948c0a6),LL(0xef38e9b3,0xa75151df),LL(0x754fb3fb,0xa3348ae5),LL(0x13069b72,0x1218fa8f),LL(0x0835dfaf,0x532bb051), LL(0xdf2be3c6,0x2121a98e),LL(0x9e5199bc,0x85980de6),LL(0x1a1eb6ee,0x1b23a4be),LL(0xadeb3ae5,0xb5c48b92),LL(0xedea2b45,0xeebd305d),LL(0xc37198ea,0x20543f04), + LL(0x0fab968c,0xd0960bd8),LL(0xae028db0,0x6899e4fa),LL(0xa9850916,0x975ccc77),LL(0xe5f81554,0xb41bd531),LL(0xc8cff2c8,0xbdf8ab57),LL(0xf5822be3,0xea306a01), LL(0xbefbdbbe,0x1f0ac0e7),LL(0x60519f87,0x72f4b0e9),LL(0xe3cc86ab,0x22bd8b82),LL(0x2b2beaee,0xc43bde8d),LL(0x412617ff,0x8168781e),LL(0xb7ee7096,0xc5610627), + LL(0x40671bd2,0xc4085538),LL(0x599f8eba,0xccb74902),LL(0x8d65e832,0x477a4a09),LL(0xf1241626,0xc620c314),LL(0xe7344054,0x05f2f152),LL(0x6d83320d,0x2483a2d9), LL(0x0e344da8,0x167439d7),LL(0x1002fb36,0xa0c02b1b),LL(0x30afbda2,0x46cb4a70),LL(0x229d4efe,0xe74e3488),LL(0x1cef3aec,0xd371dce2),LL(0x7c3f2521,0x96e0592c), + LL(0xcc5f4a6e,0xdec0e091),LL(0x201108d0,0x055ff295),LL(0xdc202800,0x2b371998),LL(0x0d5f5f9d,0x1b650d83),LL(0xa0226262,0x29be5503),LL(0x185bf3f1,0x0f3681ac), LL(0x8c3e2c4f,0x4f5c44b4),LL(0x717814fd,0xcfe74a51),LL(0x7b52a561,0xb0ce4183),LL(0xee634895,0xde143e3a),LL(0x48d46b2f,0x08f83100),LL(0x6386486d,0x34b79d55), + LL(0xee1388ba,0xa60659c4),LL(0x7eca29dd,0x475a4d06),LL(0x66470e08,0xf1ef88c1),LL(0x687c716a,0x71cac87a),LL(0x86f43fff,0x994fb1ee),LL(0x43384658,0x53ded2ef), LL(0x56c41587,0x317d9024),LL(0x9dea0f26,0x6807f0ad),LL(0xa0b0d53b,0xfee8beaa),LL(0xb0288c94,0x15c06a1f),LL(0x47028bce,0x4b9eab03),LL(0x1446bc6e,0x6d214435), + LL(0x01a8eb44,0x0869457a),LL(0x7a7bedd7,0x52223985),LL(0x00057505,0x2c04b0c6),LL(0x0b09adeb,0x468be6e8),LL(0x6f81474f,0x2f3bf32b),LL(0xa712ccce,0xf54f949d), LL(0x4cdd8f2a,0x292cee42),LL(0x9c221be1,0x3d9fdf6b),LL(0x56f47b2a,0xe54da661),LL(0x840b5d1b,0x2ca76835),LL(0x8a6e8cf6,0xb52adb6a),LL(0xdade153e,0x8b416a6b), + LL(0x3fd55732,0xb8c3a138),LL(0x2fd63498,0x4b348243),LL(0xf52c44d3,0x60cc7a0f),LL(0x9422c87e,0x9787f356),LL(0x60ad9f57,0x0fae6177),LL(0x82e307c4,0xb9ac12d8), LL(0x79416b16,0x52316618),LL(0x91611196,0xe603cb9a),LL(0xe34c3a39,0x0ce22a84),LL(0x5b2dca8b,0xcb191237),LL(0x9e5b750a,0xcb87c227),LL(0x13a02f00,0x7feccba0), + LL(0xa3cd6034,0xd3a66685),LL(0x3a283c3a,0x74ad77ad),LL(0x66644cd0,0x0c903afb),LL(0xc1f90202,0x34e18018),LL(0x0008aa04,0x8fa168e7),LL(0xbe8e35ae,0x871573df), LL(0xf4561f77,0x8841591e),LL(0xc6cd0f01,0x55e3033d),LL(0xfe1fa6ff,0x44327c97),LL(0xdddfcba8,0xed4e69d6),LL(0x52d907ec,0xe82e1e42),LL(0x79064b8d,0x8a5d9a71), + LL(0xc26f1000,0xa47a884c),LL(0x44a4b7cb,0x9f047634),LL(0x89813d5f,0xa5a2bf4e),LL(0xd8e9318d,0x83ce02c9),LL(0x2c86c874,0xf2731254),LL(0x02a0813f,0xb76a3096), LL(0x0c8b505d,0x1fabaa2d),LL(0x4895425b,0xad693de0),LL(0xa55ff754,0xaf7f5848),LL(0xeadfa222,0x098a328f),LL(0x470c5898,0x4c20adfb),LL(0x5d199733,0xe4b23ead), + LL(0x9a605b1a,0x37b29ca7),LL(0x30caed34,0x2ca364f6),LL(0x3c2ff02f,0x1e538aae),LL(0x42c6a320,0xc10c493c),LL(0xa2b9e130,0x92f18f28),LL(0xdd0ed08f,0xaf249c41), LL(0x70285f2f,0x42039a52),LL(0xcfd3ebb0,0x4e502f2e),LL(0x6e39b30f,0x42821914),LL(0x70e820ca,0xdbc809f1),LL(0xd2781e34,0xbccb5567),LL(0xbe3d05dd,0x8c1e3bfc), + LL(0x43eeed0a,0xa29f0b65),LL(0xfd8b3697,0x0b96b6a1),LL(0x88f8755a,0xc2454ada),LL(0xaf85d4ac,0x57202337),LL(0xa7aaef40,0xaddfc388),LL(0xb156a5fc,0xc1495163), LL(0x1f1a8775,0x67abbbcb),LL(0x5958bb8b,0xb83e3dd6),LL(0x36dbf23c,0x17119d02),LL(0xb954337a,0xeb336460),LL(0x6f46dca7,0x9b101001),LL(0xbc991dbe,0xef862ae8), + LL(0x66befae3,0xaa878db5),LL(0x05110c6f,0x28bb9c9e),LL(0x4caa069a,0xc3a57a8c),LL(0x594a2753,0xbb5b550e),LL(0x5187afb8,0x01ab8056),LL(0xb9255f65,0x1f7c9ed3), LL(0x146f6635,0xcd669e2b),LL(0xcb9457a3,0xa8f2d4b6),LL(0x849dba46,0x0f0541aa),LL(0x2cc9c7e9,0x2537bf02),LL(0xd79ac77e,0x872b4f59),LL(0xe0a44aa9,0x705c2219), + LL(0x3849051d,0x03b42dbb),LL(0xf27a63a6,0xe2efcfe3),LL(0xceb478c2,0xf709a5ff),LL(0xc4f7feda,0x2cc86b82),LL(0x066a1c08,0xefa834e4),LL(0x309fd644,0x153b64ef), LL(0xe62168ff,0x8cff4eb0),LL(0x095d9f3b,0x0d7781db),LL(0x10bce338,0x01f8e1af),LL(0x139d8f2b,0x14aa9a02),LL(0x259ec819,0x1985d844),LL(0x3a072e8e,0xd7758b21), + LL(0x8565afc9,0x65f7d2c1),LL(0x70fa7b82,0x764c8971),LL(0x986436f2,0xe268634c),LL(0x33356165,0x6334d8d1),LL(0x9ec7957d,0xf1716426),LL(0xb8093983,0xae834331), LL(0xd2dfcce7,0xedb1fe5c),LL(0x68463e5c,0x6195b863),LL(0xa691b665,0x746e5f4d),LL(0xe1e2727e,0x61171291),LL(0x6f27b029,0xbb4aa8f1),LL(0x7f42c197,0x1037657d), +}, +/* digit=34 base_pwr=2^170 */ +{ + LL(0x0d5b855a,0xfe19901b),LL(0x2f745022,0x5facb955),LL(0x56c4ce5c,0x92fd0125),LL(0x938c89ab,0x23172d65),LL(0xaaa587b1,0xa71f8a33),LL(0xb55c9c50,0x511a3745), LL(0x7185086e,0xec005f6a),LL(0xf894c6ab,0x6dfc2761),LL(0x9e26361f,0x98a4d67f),LL(0x21389c25,0x7f0a2b23),LL(0x95ffbcee,0xd1588207),LL(0x9f36a888,0x4d6b29ab), + LL(0xd0a701a5,0xeccb421e),LL(0xb60cd286,0xad4cb9a5),LL(0x05a53972,0xd344da9e),LL(0x7bc99fea,0x3a8035e0),LL(0xc0f77bf5,0xe0214485),LL(0xe54df78a,0x50ada30e), LL(0x4ec2d576,0xdef45af6),LL(0x5f9a8678,0xa05d6184),LL(0xc337e017,0xa9b17db1),LL(0xb84671d5,0x026a4f66),LL(0x3b7d696d,0x60614234),LL(0x81cfd22a,0x71ed9aaf), + LL(0x439ada39,0x1b76a3c5),LL(0x89236ae5,0x818829cf),LL(0x750f8129,0x2277cb7a),LL(0x4d46502b,0x44aa462a),LL(0x64f06dc8,0x7a12e1e1),LL(0xba5630cf,0xb9a3300d), LL(0x55b05f4b,0xd2cc8d9c),LL(0xa700be7a,0x6d0b0b88),LL(0x9617500c,0xa7be9969),LL(0xc03f8a50,0x2b5b8dea),LL(0x785b3dfd,0x712f703e),LL(0xccf93950,0x96a5a60a), + LL(0xee828fba,0xfeb984b3),LL(0xe2bd188c,0x8273f830),LL(0x3ca0a99f,0x177ef97e),LL(0xacc000ac,0x76d4796d),LL(0xb140f51a,0xbad0fa6e),LL(0x06ebc810,0xb2756567), LL(0xa18cb32f,0xf89eb78f),LL(0xa65285b0,0xcfc37eae),LL(0xb25e9d1b,0xe2b29cfb),LL(0xb4e7aef3,0x9388ea8f),LL(0xe267e845,0xee606c12),LL(0x9f5806d7,0x6b103c54), + LL(0x3766f2ae,0xf418e3f6),LL(0x053ef1c1,0x4a3ad3c8),LL(0x560db262,0xd01e5b5b),LL(0xc02bf4c3,0xa583edc7),LL(0x52f318d3,0x7c9f7060),LL(0x1f5e1ffe,0x0852556f), LL(0xfeb0e63c,0xe1c70aa7),LL(0x89a8c058,0x59f0a3f9),LL(0x1ffc0ade,0x4aa4cf02),LL(0x38a78632,0xbb880e41),LL(0x6f28f096,0x35b0f759),LL(0xd5757d7d,0xf9c4fe17), + LL(0xfd6376eb,0x78b8879c),LL(0xc01e1edb,0x22a76461),LL(0x369cf0c4,0x6a44be39),LL(0x5ae54539,0x6653670d),LL(0x6fb43ad0,0x257bd751),LL(0x12baffdd,0xb3ac3715), LL(0x7548eabb,0x48659d61),LL(0x0cd468cb,0xd8f931f8),LL(0x49e3b531,0x98f02415),LL(0x70df011f,0x90b0d716),LL(0xab98f066,0x26d73c54),LL(0x88475d5e,0x06591ec9), + LL(0x412b84c3,0x627f6328),LL(0xa04545d2,0xd427e977),LL(0x104f25c2,0x5b0145bc),LL(0x2ac7ad62,0xa6931c4f),LL(0x1f8d42f5,0x40761143),LL(0xe7f8a0b3,0xfda5a76b), LL(0xfe0946b9,0x4f1ca5cf),LL(0xbeb2d427,0x6def7b9f),LL(0xc9a0d136,0x984bd4bb),LL(0x5b3af1c1,0xb9a77823),LL(0x38ac2087,0x04ee66ae),LL(0x26d9dbb9,0x63374ed9), + LL(0x44ba39e6,0xcf947c06),LL(0xedfe78d7,0xf5d5216c),LL(0x5f1835c9,0xd00115c0),LL(0xd8c79d90,0xdf084152),LL(0x6db5f791,0xc0c3a684),LL(0x749b18cd,0x40514451), LL(0x734df3f1,0xd314b7d5),LL(0x7f541415,0xbccdd3f0),LL(0x6855a942,0x97ed5af0),LL(0xe9d02ab9,0xea84ae9e),LL(0x3238a5d0,0xb87e9034),LL(0x650a0eab,0xd12d25c3), + LL(0x86a515a9,0x1473b55b),LL(0x3b337c64,0xa9e3230a),LL(0x9db668f1,0x7e8bf904),LL(0xf27f9fc9,0x1db2c25e),LL(0x2d9e467e,0x0c108607),LL(0xa3f00d52,0x4505579a), LL(0x240400a7,0xe2ad661b),LL(0x11af4874,0x8022294c),LL(0x78bba8e8,0x29e90370),LL(0xf6baca04,0xbf0fbf08),LL(0x4101fab0,0x2e46d2b7),LL(0xc61089e6,0x66065490), + LL(0x09f8a1cb,0x2131ce5b),LL(0x8ab129e2,0x7b373ed2),LL(0x77c1292a,0x463cc8d6),LL(0x94ffe9c5,0xa9b7cf65),LL(0xb99bfc4f,0x129125ce),LL(0x9820d323,0x819b4284), LL(0x76541a41,0x3f709763),LL(0xe32c7a7b,0xfd679ae5),LL(0xf65b6b3c,0xc39a208d),LL(0x50002745,0x1c22ebc0),LL(0xe2bcd202,0x268f19dd),LL(0x9c3d4266,0xfeac809c), + LL(0x87dcdbdb,0x07467837),LL(0x320493d8,0x4cce33be),LL(0x713c7746,0x9ab08cbe),LL(0x9c6dc5cb,0xd6f0c1de),LL(0x2ac03761,0x19400538),LL(0xd0547be5,0x3fc11f38), LL(0x819fe3fd,0x66b378ce),LL(0x3700fe7e,0x6a590acc),LL(0x8924b396,0x4c976a72),LL(0x70b9b250,0xa5006d8d),LL(0x12b85f9c,0x2fdce1b2),LL(0x495f8f1c,0x5858f7ce), + LL(0x903ff177,0x357540ab),LL(0x276af514,0x225280b8),LL(0x14d7fed3,0x33d273ac),LL(0xd186ee3d,0xfef6b9ff),LL(0x01a7b1d9,0xa94c2071),LL(0x50bc8bc2,0x4ea36274), LL(0xfa98a918,0xc68959c9),LL(0xc7bdc262,0x8f5eccee),LL(0xe6861310,0x7a73a4fc),LL(0xc828330f,0x19bcac90),LL(0x7ef74fdb,0x73e3b66f),LL(0x52d8f2f4,0x60f76983), + LL(0x7e03a14e,0xab357804),LL(0x8caf673f,0x0f4f2868),LL(0x66530425,0x919e661e),LL(0x91ba47c5,0x28da445c),LL(0x66c394fe,0xd6d05375),LL(0x02e8ae91,0xfe1864a3), LL(0xa753aec4,0xd34baca2),LL(0xa2c8d292,0x43b7ffe7),LL(0x04efb8f1,0x496659eb),LL(0xe0252dfd,0x310ec2a9),LL(0x9168a80e,0x98173d2f),LL(0x31497255,0xa3e018d6), + LL(0xd9d9284a,0x3266c887),LL(0x73646ab7,0x690f818b),LL(0xaf7fc33b,0x67315ec6),LL(0xc30b1ccb,0x181e61ab),LL(0x105a9e1c,0x1b81e6cd),LL(0x5078b9bb,0x62a15daf), LL(0x6fa8cc65,0x74f9840f),LL(0x43388573,0x356b7774),LL(0x06b3fd46,0xba0f7d05),LL(0x92b4fdad,0xb0ac864c),LL(0xef192cde,0xcdeac253),LL(0xc313b4a7,0x0c24810b), + LL(0x1adb09b7,0xef8c40bf),LL(0x0b74992a,0x2efeb49c),LL(0x2b79957f,0x3f0f8a41),LL(0x87a06873,0x08927bfe),LL(0x9288cb9a,0x1f63a410),LL(0xdf2b373a,0x8c66fb70), LL(0x980facae,0x98da4712),LL(0xd819d026,0x15ce5b17),LL(0x749a671a,0x097571a5),LL(0x894dd269,0x85a40804),LL(0x34cb6797,0x3e89c13c),LL(0xd07119a4,0x2d19d5e4), + LL(0xecbafb80,0x58225208),LL(0xaa73d6de,0x4f212035),LL(0x62fe86db,0x1224e455),LL(0x2dc5b2f1,0xa8c8a478),LL(0xc3096555,0x8a957b8d),LL(0xb1591452,0x6a3248b0), LL(0xcb604c18,0x1e563c58),LL(0x9bf1045e,0x32808cb5),LL(0x9462e7a2,0xf8f62de9),LL(0xc2489214,0x6b3dfe91),LL(0x2174639c,0x6c1d8fc4),LL(0xef88d4b5,0xdfca11b8), +}, +/* digit=35 base_pwr=2^175 */ +{ + LL(0x18690ad0,0x5a4a5ce4),LL(0xfe27f51a,0xd0f788e0),LL(0x4efe9a30,0xd459388e),LL(0xef9d074b,0x3a45c11a),LL(0x93ab9cb0,0xf68ab50b),LL(0xecd9a566,0x62fbc397), LL(0xcc587a7e,0xbfb79b7f),LL(0x92870bae,0xfcf4d66f),LL(0x877390f0,0x4f31aa21),LL(0xe314cfb5,0x2de0c645),LL(0x238eab12,0x56d904f6),LL(0xccb4d4f6,0x4d104a42), + LL(0x29358cd3,0x3eb83a87),LL(0xb9c6d430,0xad741295),LL(0x53abe4e9,0x57b8c77a),LL(0xbb9feb82,0x0a14673e),LL(0xf26f922e,0xc0a6cbf7),LL(0xa32e526c,0x213de299), LL(0x7b6ca858,0xca417e67),LL(0xfc2e0900,0x8d6ae0f7),LL(0x62e135dd,0x2bae0e7a),LL(0xa7ee82c7,0x962bdcae),LL(0xe5776c74,0x573d7f6a),LL(0x6ffbefeb,0x9c4de649), + LL(0x09335d38,0x8c962fc8),LL(0xeb38d176,0x26d1bc81),LL(0xc47711ed,0xe1aeb295),LL(0x6cbe3e4e,0x0812b992),LL(0x0ab9805d,0xeecacaf9),LL(0x3521a0ad,0x82fefbaa), LL(0xe2c31b9d,0x3a6948c0),LL(0xe82daf2b,0xb7d3905b),LL(0x25a34c37,0xbd3ac90e),LL(0x61453063,0x55afd99b),LL(0x90b99303,0x56d87cd1),LL(0x97ddb0a3,0xc9bf82dd), + LL(0x68916917,0xcbc0bb19),LL(0x1094bf88,0x0bbb9f92),LL(0xd3806442,0xf62cb350),LL(0x397a7602,0xe4d2f1cc),LL(0x43987d82,0xa54bd48e),LL(0x4f0a19fa,0x77b6f831), LL(0x6e766443,0xfa0c9a45),LL(0xf51ba70b,0x995ae0ff),LL(0x9cbd8d33,0x8e242c5b),LL(0x13d97956,0x1671eb08),LL(0x40da55fa,0xccae388f),LL(0xf376dce5,0x97cc48fa), + LL(0xe8c91718,0x1c2919bb),LL(0x5097bde3,0x9dbb727a),LL(0xf8ea2fb2,0x23f87ae7),LL(0xba310121,0xe1bfffdc),LL(0x75329669,0x5938c50c),LL(0x0549855c,0x716c63e0), LL(0x654814f0,0xe091b0c9),LL(0x0e43daee,0xa20535d1),LL(0x593ddd04,0x16ce68b2),LL(0xf59900bd,0x7813a49a),LL(0xd3e5d232,0xef0d3eec),LL(0x0ee3fd4d,0xe7d12cc4), + LL(0xef01fc5f,0xe54d92cd),LL(0xdda2e25a,0xc46c2ab8),LL(0x849f6142,0x7c907fd2),LL(0xacd0202d,0xbb11dd2d),LL(0x1d92d19a,0xa4913a70),LL(0xcf610677,0xe9a26ae0), LL(0x538943c5,0xfff1e1d5),LL(0xa47b2204,0x5943dcc4),LL(0x92cabf71,0xcafcf33a),LL(0xe329d1ad,0xd571e13c),LL(0x7a9a0e4c,0x7626ad23),LL(0x130d7f86,0xf0aa0d9f), + LL(0x19e6aa7e,0x09df3a44),LL(0x5841b1cd,0xe27ad047),LL(0xbde75934,0x02d2a69f),LL(0xfd9ba435,0xb0e05e53),LL(0xe008c16e,0x4732d88a),LL(0xea72110d,0xdebc4777), LL(0x2e3143ad,0xccb7d993),LL(0xea8cd06a,0x674f3753),LL(0x051562cf,0x56012a7a),LL(0x25f74cd6,0x961df684),LL(0x214d8a95,0x26630e71),LL(0x65d92f84,0x584e8d63), + LL(0xebc5557c,0x8a89daef),LL(0x275e1649,0x7ca71403),LL(0x5b80bb4a,0x48d92377),LL(0xa45b3626,0x0a587c52),LL(0xc75bfe91,0xdaff503c),LL(0x116d07d7,0xd845d3e6), LL(0xa51eeca2,0x6b5a4715),LL(0x74481991,0x34ac02bd),LL(0x595abf8d,0x8f076cfc),LL(0xed0391ce,0xc9de4ce9),LL(0xe1fcabd3,0xaaaad03a),LL(0x87b199ed,0x8d48ec00), + LL(0xae5dd482,0xbd0f2653),LL(0x060032dc,0x59f968dd),LL(0x67283310,0x6bea33e0),LL(0x012aa50c,0xccce88cc),LL(0x66838f46,0xbb6d7f2f),LL(0x05ec9bcb,0xb764c95f), LL(0x51477cca,0xd097b604),LL(0x82b20a85,0xc2fbda7b),LL(0x24e9ca8d,0x75fe07a4),LL(0x0cc40d01,0xfc4fa824),LL(0x0c0e95f7,0x0b17d5f9),LL(0x6e1e46dc,0x285e6e8a), + LL(0xbc9b2654,0xb0641d09),LL(0x8aa8fa35,0xf9fcc2e6),LL(0x00d5ec6e,0xd12a5b4b),LL(0x5569d89a,0x9be1a111),LL(0xffac7208,0x9c0566de),LL(0x7034edf1,0x7a9fd4ff), LL(0x9571c375,0x636aeb6b),LL(0x55cdf187,0x60d05aec),LL(0x734e9d2f,0xf4e2f898),LL(0x5ccdc6bc,0xdaf74219),LL(0x608a4f28,0x9d39249f),LL(0x8820e2c3,0xb5f1bb5e), + LL(0xd9589548,0xd02e9936),LL(0x5341402f,0x8f1bf575),LL(0x057300aa,0x1535a443),LL(0x65d29324,0x3062478e),LL(0xc656a3f3,0x4203351f),LL(0x6569c4ff,0xbeb21b51), LL(0xe1f0f263,0x8113ce70),LL(0x03f9320c,0x59d12939),LL(0xd08f8936,0x95061255),LL(0x97d4b705,0x8be3c0f9),LL(0x827837c2,0x0259742e),LL(0x95c65cda,0xf55ea28d), + LL(0x603dc3dc,0x62024812),LL(0xefd67b57,0x25dc5337),LL(0xd7f033fd,0x86b3eb38),LL(0x32646d6f,0xee3226b2),LL(0xf1dae596,0x8c4825f6),LL(0xa5bcb8e5,0xd2303055), LL(0x3c0baa76,0x904a5349),LL(0xe08646a7,0xe60f6125),LL(0x21d45f89,0xaf6a329f),LL(0x06605546,0xf20ad88a),LL(0x19a93d14,0xcf7a0e96),LL(0x91c97174,0xf1eabcc8), + LL(0x8f02af51,0x72b76e9e),LL(0xd580f95a,0xac94cbf1),LL(0x01d854a4,0x2e9cd748),LL(0x1f08a1bc,0x4ed4e906),LL(0x9d2bd936,0x0a2b2841),LL(0x51c89dda,0xbf863500), LL(0xe3f00bf5,0x9407b0e7),LL(0x28b57ac1,0x6b1f71ff),LL(0xcd28801f,0xc1dfe03f),LL(0xafa55309,0xf3d83d64),LL(0x8af8f76f,0x47aafba2),LL(0x6604b2e9,0x54eed45f), + LL(0x0f3e541f,0x59edd264),LL(0x82b76ba8,0x318674b5),LL(0x4e7f0716,0xbf4a0d30),LL(0x19b88200,0x36fc0e41),LL(0x40da801e,0x91db5602),LL(0x2c72c2c7,0x638371ad), LL(0xd5822da7,0xfe960c25),LL(0x4a7415e1,0x7a7571d1),LL(0xbccc1576,0x5a6480fe),LL(0xc3c88f47,0x72f4e5e5),LL(0x9a7bd8ec,0x224e7e74),LL(0x7631455e,0x3ebbf52c), + LL(0x8608ab37,0xae3c2bc0),LL(0x39f336b6,0x35e3da8c),LL(0x81f44511,0x74136642),LL(0x1d8506e7,0x21ce7c51),LL(0x846165f1,0x9b6718b3),LL(0xf5cabf6a,0x9e455007), LL(0x02611073,0xec582a0e),LL(0x83bf042e,0x269aa18d),LL(0x86306757,0x7c54fb7c),LL(0x1b948faf,0x45333602),LL(0xb7025d73,0xd3a5c508),LL(0x428471e4,0xcd6e555b), + LL(0x11a224e6,0x42c9fad5),LL(0x69b2ac26,0x6b6aeb8b),LL(0xb149854b,0x0cf4c7fd),LL(0x2fc359eb,0x4a7d9000),LL(0x29ec8603,0x9ff0c3ea),LL(0x9b24ee14,0x157ae785), LL(0x8979e9bb,0x638c809a),LL(0x7869d8c5,0x347dfb2e),LL(0xa07ea547,0x2fb1e0f8),LL(0xaecdec3f,0x1e580d32),LL(0x0f74025b,0xbbf89573),LL(0xdd529164,0xeb94d71b), +}, +/* digit=36 base_pwr=2^180 */ +{ + LL(0x8d688e31,0xfa2db51a),LL(0xa09c88d4,0x225b696c),LL(0x6059171f,0x9f88af1d),LL(0x782a0993,0x1c5fea5e),LL(0x4ec710d3,0xe0fb1588),LL(0xd32ce365,0xfaf372e5), LL(0x26506f45,0xd9f896ab),LL(0x8373c724,0x8d350338),LL(0xca6e7342,0x1b76992d),LL(0x6fd0c08b,0x76338fca),LL(0xa00f5c23,0xc3ea4c65),LL(0xb316b35b,0xdfab29b3), + LL(0x04a8313d,0x14f962e4),LL(0x5f1f5a26,0xc6e3e7c4),LL(0x79e777be,0x2c0e11c0),LL(0x4657c31b,0xa1705efb),LL(0x3c494de3,0x02688fd2),LL(0x412a8718,0x75664a84), LL(0x7a422f8a,0x878fc7ad),LL(0x7419bd0a,0xe5d581df),LL(0x704b70c0,0x7c813c4c),LL(0x7323c008,0x98553da8),LL(0x63089f1a,0x4f63cec6),LL(0x9655d291,0x9626d6fa), + LL(0x483aebf9,0x84e5541f),LL(0x49165772,0x8adff7dc),LL(0x9beaad3c,0xe0a43ad6),LL(0xf51c2714,0x97dd1820),LL(0x57ea5b0c,0xac2b4cb4),LL(0xd11767ca,0x87dbd011), LL(0xbfc7957a,0x18ccf36c),LL(0x1bc79227,0xd4a08841),LL(0xd8d292a8,0x9811ce43),LL(0xd58c4ee7,0x72c5fc68),LL(0xd35c65a7,0x5bc0f0be),LL(0xcbbf9669,0x0b446dbc), + LL(0x507f8b27,0x10586ea7),LL(0xa261f7d7,0x1510deb9),LL(0xdfbfa352,0xa42fc4d7),LL(0x1e1c2291,0xbf38c382),LL(0x0e11760a,0x46e40ef6),LL(0xdcb974d7,0xc24f6061), LL(0xa7619027,0x755b105b),LL(0xb8ffa759,0x8004bf09),LL(0x0945db60,0xa630d0b0),LL(0xf2809e1c,0xa160ac9c),LL(0xdc6c95c5,0x38fc1113),LL(0x5d52574f,0x01f54098), + LL(0x9cee9bce,0x7eba3da6),LL(0xd5377750,0x3e2c1248),LL(0x2b93d8b2,0x8c917d98),LL(0x7cad1f75,0xca8fc6ac),LL(0xa0ff150a,0x5f581f19),LL(0xe08327fa,0x872cc14a), LL(0xe9333188,0xc774f187),LL(0x497af7e8,0x528ed4ac),LL(0x8ad72b10,0xce036e9b),LL(0x917986cf,0x463f9ebb),LL(0x1325cf9b,0xbe516328),LL(0xdd7e5fea,0xd28d5c50), + LL(0x0b237b7b,0xa4e9f29f),LL(0x4270ee2d,0x8ed65b09),LL(0x2993359d,0x0e2184e4),LL(0x224d5aa3,0x4f96ce7f),LL(0x3a132c48,0x0862e200),LL(0x0f015f5f,0x5bbc6ad8), LL(0xd7162f5c,0xab9d5149),LL(0x1267e5d2,0xfe657729),LL(0xc1fd96e2,0x4865e671),LL(0x71a703b2,0x7baf4dbe),LL(0x142add10,0x83dd6cf9),LL(0x98461d30,0xab4fc1aa), + LL(0xdd58bbe3,0x714c1d1b),LL(0x039afd0f,0x85ba01ae),LL(0x6951ac80,0x7f23ea3a),LL(0xac00c837,0x5c599290),LL(0xbf24cc1b,0xf6efa2b3),LL(0x1e84462b,0x393d8e42), LL(0xf8b89453,0x9bda627d),LL(0xb23e0d1b,0xe66fff2e),LL(0xc3b94ec2,0xd1ee7089),LL(0x3031699a,0xf75dba6e),LL(0x242b2453,0x8ff75f79),LL(0x289bfed4,0xe721edeb), + LL(0x698ee21c,0xcda68a7e),LL(0x6a5e725e,0xc7414d19),LL(0xdce20b91,0x483be2da),LL(0xfc69dca6,0x7de1601c),LL(0xac4f9891,0x4bec17aa),LL(0x8d479a56,0xe8741dd1), LL(0xac23a286,0xc623cb8d),LL(0x166133f0,0xe20a96b5),LL(0x30dcde61,0xda9bb7c0),LL(0x3a1733fd,0xf84ea327),LL(0xe82fac31,0xd7afb6c3),LL(0xd3897449,0x37ea7d35), + LL(0xc1390fa8,0x083215a1),LL(0x6dce8ce0,0x901d686a),LL(0x837073ff,0x4ab1ba62),LL(0x34beaba5,0x10c287aa),LL(0x46985239,0xb4931af4),LL(0xb053c4dc,0x07639899), LL(0xe721eecd,0x29e7f44d),LL(0x57b3ff48,0x65817182),LL(0x5054e2e0,0x198542e2),LL(0x84616de8,0x923c9e15),LL(0xad465bb9,0x2a9c15e1),LL(0x16319245,0xd8d4efc7), + LL(0xfd028642,0xed85257e),LL(0xb96a2068,0x93657f45),LL(0xa13ac381,0xfef64eda),LL(0x56c557a3,0x108f6ff2),LL(0x9204e3f7,0xe690d92b),LL(0x03ef8640,0x902a3e38), LL(0x6416f50e,0xaefd4922),LL(0xb7eae8f0,0x9b272152),LL(0x29d93d8d,0xa911921f),LL(0x5eeeea56,0x7c6bc499),LL(0x3ca7c720,0xbd3439d8),LL(0xc39b208d,0x2f8cf2e3), + LL(0x9961a674,0x72dc7943),LL(0xa0e13668,0x839a0a52),LL(0x334945ea,0xd7a53fa9),LL(0xe7aa25db,0xdb21db77),LL(0x66e96da3,0xb6675a7d),LL(0xe66f33c0,0x2c31c406), LL(0x6ec7b9cb,0x45020b62),LL(0x0391f267,0xff46e9cd),LL(0x0fa2f221,0x7dabd744),LL(0x9d4a2a3e,0x9a32364b),LL(0x52d2e47a,0xf0f84ae8),LL(0x888f488a,0xd0b872bb), + LL(0x0370327b,0x120649b2),LL(0xcd48cdc6,0x0e76555a),LL(0xca01db03,0x4ed54dec),LL(0xac601d22,0x7be21319),LL(0x01b6576e,0xf7116619),LL(0x4e73537f,0x7839fa06), LL(0xe46e860a,0x169d43ac),LL(0x3078eed9,0xde6d658c),LL(0x5032142b,0x8df73139),LL(0x9b3c76c7,0x6be199b0),LL(0xf8bbffe5,0xc2f385f6),LL(0xd5ffd28c,0x848df7f3), + LL(0xc9790eef,0x531e4cef),LL(0x2b8d1a58,0xf7b5735e),LL(0xef568511,0xb8882f1e),LL(0x86a86db3,0xafb08d1c),LL(0xf54de8c7,0x88cb9df2),LL(0x9a683282,0xa44234f1), LL(0xa6e9ab2e,0xbc1b3d3a),LL(0x87fc99ee,0xefa071fb),LL(0xa102dc0f,0xfa3c737d),LL(0xd6a0cbd2,0xdf3248a6),LL(0x1ecc1bf4,0x6e62a4ff),LL(0xc8f1bc17,0xf718f940), + LL(0xf874628f,0xa342bf65),LL(0xc71a57bd,0x18de3f8a),LL(0x71fc321b,0xb4d12a17),LL(0xa25ebf10,0x96716602),LL(0x1a286d80,0x744f6820),LL(0xe3cf63b8,0xd60bad1d), LL(0x4368da09,0x0c0b1ac4),LL(0x73a6d3be,0x53afeae3),LL(0xa90af331,0x4d2e6ce2),LL(0x88bc0638,0xd797224f),LL(0x9396d893,0x10c60b2d),LL(0xfe45e1a8,0xae3b0c11), + LL(0x4f63f026,0x2c8b0aad),LL(0x50b253cc,0x2aff6238),LL(0x10c4d122,0xcab3e942),LL(0x07cd2816,0x52b59f04),LL(0x982c41fc,0x22322803),LL(0x8cf50b19,0x38844e66), LL(0xbe3264cd,0x42a959f7),LL(0x6c983524,0xbddc24bd),LL(0x462b8640,0xa489eb0c),LL(0x98029be7,0xb7c05092),LL(0xa1addc64,0xd5546b5f),LL(0xa0c655af,0xe7cac1fc), + LL(0xc6a6d6d1,0xa189f30f),LL(0x69665ab8,0xdd674d36),LL(0x7d8da76d,0x307c9ec3),LL(0xc1ea7c10,0xb3e1d006),LL(0xb88c62d4,0xc15e20b3),LL(0x0bff3b3a,0xb0000ec5), LL(0x9ff9aa5c,0x9e330eb1),LL(0xdf578877,0x8663f9fd),LL(0x02e1eb2a,0x157d3cb0),LL(0xf525e4d4,0x638f297b),LL(0x34a3dff1,0xa20f8332),LL(0x45a9c051,0x748ea86b), +}, +/* digit=37 base_pwr=2^185 */ +{ + LL(0xbc266ee3,0xfe9fdde8),LL(0xba18e6c7,0x91668688),LL(0xddde6f6e,0xa65349ac),LL(0x7e54356c,0xc53c29c9),LL(0x5709f73c,0xee15ad94),LL(0xe5429277,0x033b3940), LL(0xd0c3475a,0xf52035cd),LL(0x93f1f1f0,0x9c5bef4d),LL(0xca319bd4,0x26e0b0ce),LL(0x6951fd8d,0x4e7eb67b),LL(0x95c34d6f,0xac3a6f43),LL(0x00f60b59,0x1f2769e6), + LL(0xfb787270,0xbd10b8bf),LL(0xe43aaab6,0x4f0b1566),LL(0xc0c90781,0x9a18be5e),LL(0x1ad167ce,0x3677f4c7),LL(0xa68c1c56,0xccb254e2),LL(0xe2c4d275,0x392493e6), LL(0xd5b63617,0x44958cb1),LL(0x4caa4e7c,0x178f141a),LL(0xa2ffdbd5,0x7445a767),LL(0xb0b6c22d,0x0e789c99),LL(0x5dc92b2e,0x3ff8b656),LL(0xeca98782,0x1623e5c3), + LL(0x78207cef,0xfadf9be9),LL(0x9cb5718e,0x97d5ba56),LL(0x2f995393,0xcbad24ec),LL(0x61203303,0x6236a268),LL(0x6589a4be,0xe4bafc33),LL(0x5e23fa82,0x6cba7718), LL(0x4583e65a,0x8ccbc577),LL(0x4bc2f415,0xe5d88bca),LL(0x41df8dd1,0xe6bc2d58),LL(0x14d31fcd,0xec24e1d9),LL(0xfc26010b,0xacaaf13e),LL(0xe01b92f3,0x7e1da447), + LL(0x899ef333,0x6f6a6104),LL(0x39067165,0x95496f6d),LL(0xb51989e5,0x42fd9a6a),LL(0x68f5b168,0x1b60ce0f),LL(0x56f7fe67,0x97324d87),LL(0x676815a8,0x443812f7), LL(0x685a7260,0x265ee994),LL(0x6c6515f0,0x342c7b2f),LL(0x34b4adb0,0xe9092323),LL(0x1e5a8d18,0xddcd233e),LL(0x5f4f6456,0x3dc5b27c),LL(0x7f421d9b,0x9664533a), + LL(0xff7543c0,0xc48bc829),LL(0x4d72bfaa,0xc0bda14c),LL(0x03be0af1,0x2f470ec7),LL(0x92d37eb4,0xc70f1e8e),LL(0x418f410f,0x08abdd98),LL(0x35386176,0xe38c74ab), LL(0x8c00426c,0x9c07cfdd),LL(0xa998f1ad,0xba74c310),LL(0xb7d2dda8,0x76b45140),LL(0x4948330e,0xa52b5e58),LL(0x8d8efb26,0x9b733234),LL(0x5d19a312,0x5d176373), + LL(0x987cf64a,0x43b58def),LL(0x3d4bcd4e,0xc95b16c6),LL(0xbcd9b923,0x5d1b1373),LL(0x522e052b,0xaf560542),LL(0x83800352,0xc2ff8f75),LL(0x7fe2a4ea,0x11723aa1), LL(0xe94bd9bd,0x28de7668),LL(0x874018a5,0x0ce80e0f),LL(0x8d43e726,0x0fe3755d),LL(0xf9b075c5,0xa78296ac),LL(0x82207423,0x76d58d98),LL(0x1db99205,0x5c5bc697), + LL(0xe3b7e746,0x583ee7df),LL(0xa4fab3a8,0x0b6659e4),LL(0x1946db5d,0x34ee0275),LL(0x1a12eeca,0x5ae3c0ba),LL(0x4ccb83e7,0x36756ed4),LL(0xa80eaf3a,0x973b0861), LL(0x6982ca90,0x969e38f4),LL(0x018d01fc,0x9a9bcd10),LL(0x3272476a,0xb540e953),LL(0x75ab7002,0xcf91dd0a),LL(0x39ceb983,0x2c7d363f),LL(0x974747c7,0x4369c221), + LL(0xf893a2ca,0xbfc40c30),LL(0x0623bad1,0xdf96980e),LL(0xc027511c,0x4fd7b54d),LL(0xcf3484ce,0xf4799284),LL(0x069beea5,0x655ab811),LL(0x7392e280,0x52588bc8), LL(0x4f0c17cb,0x522e7b40),LL(0xc705e9b2,0xc0d88aca),LL(0x77f3913f,0x9cf1b958),LL(0x7dd52514,0x3e06b926),LL(0x2908cbca,0x992e920e),LL(0x6d6ed529,0x13baced2), + LL(0x95ab4944,0x41c59b9c),LL(0xf32dcf4b,0xfacdd4fa),LL(0xef361bc3,0x6401bcb6),LL(0x697706ff,0x8d1fbbf7),LL(0x12fafc26,0xa9dcd2cd),LL(0xced1b64c,0xc1fce537), LL(0x06433b06,0x760b3eb2),LL(0xf6f894d2,0x53a27b08),LL(0x851c8bce,0xb50135cd),LL(0x0e058bf5,0x9bf9a243),LL(0xcc0a78ef,0x231624a6),LL(0xff090623,0x8200be42), + LL(0x67a470ce,0x5a0f6f6b),LL(0x7ccca885,0x7b8c2a88),LL(0x25f812da,0x4421fe13),LL(0xe0833478,0xc7a9c622),LL(0x94829d7c,0x6aff42db),LL(0x37d888fe,0x6fc2f23b), LL(0x64b75c84,0x9cfb8a14),LL(0x46139fee,0x56e1b7d4),LL(0xfe72fca5,0xcc6943b8),LL(0xc3d621cd,0x12e757e1),LL(0x72c1571a,0x6d9d63b0),LL(0x7b300fc6,0xa1fc3db1), + LL(0x393e37ca,0x7d58f7e6),LL(0x5e47e4c4,0x30290d9b),LL(0xa69dc1eb,0x831e5039),LL(0x5a758799,0xe2f42725),LL(0xc8f86525,0xe8166ff7),LL(0xaa85cc41,0x28e08f58), LL(0xe5409138,0x66be9bb7),LL(0xdcbb88ca,0x7d0f8807),LL(0xe7803e98,0x870cd794),LL(0xd59d39c6,0x78fd1eae),LL(0xcc0e56c0,0x1aabe0af),LL(0x5bf0f272,0x7f8de733), + LL(0xfdf88aba,0x36aa63d9),LL(0xcdf43217,0x87198fdc),LL(0xa9a923c9,0xd8fe6f62),LL(0xee2ae4ba,0x85c81a0e),LL(0x32dbcb0b,0x7c20dea0),LL(0x129a31c6,0x08baa938), LL(0xb1d60f99,0xa47b3003),LL(0x6905192e,0x31459993),LL(0x32cf2c7c,0x67e22899),LL(0x3b3c32bc,0x4c5f4375),LL(0xd499aa00,0x8de5d14d),LL(0xa311dac5,0x54875c3f), + LL(0x898dc5c7,0xb6475210),LL(0x080e62b4,0x0f709811),LL(0xbb3bffd9,0x8a0016a7),LL(0xafb97dc7,0xfc56d337),LL(0xca1b43d9,0xff911c89),LL(0x9187747a,0x9f8d40b7), LL(0x5700a9ea,0xf6ba5214),LL(0x8c04b9b7,0x7bc1c0e3),LL(0xa2c924d2,0x2dd3de09),LL(0x717b13cf,0x12378655),LL(0x6e9d0e85,0x090f2556),LL(0x88f728c3,0xf66a337d), + LL(0xe5140e1e,0x1f90b9b6),LL(0xea2d44e5,0x7ab77ce1),LL(0xf4878fcf,0xc8a3343e),LL(0xc544e407,0xbeb73f7c),LL(0xfec32a61,0xdb3a8266),LL(0x38db88f8,0xbe30a82e), LL(0x525080cc,0xf33ad1df),LL(0x1a553e27,0xf66aa44d),LL(0xa9b7b198,0x3194733f),LL(0xb4b9b4c1,0xd87c8145),LL(0x68883c51,0xca7cd392),LL(0xca49b152,0x2d12a779), + LL(0x2c4ffe71,0x407842b5),LL(0xc3f5593b,0xfa9a9143),LL(0x69f25d01,0x97f9f32a),LL(0x4571d150,0x78e3d5fd),LL(0x89878e86,0x8cbd1078),LL(0x633ed774,0xd4a4be53), LL(0xd65d0ef1,0x2e6cf7f0),LL(0x18a2e243,0x5cefa892),LL(0x130c9ba1,0xb2e8ccb6),LL(0xde1ec2f9,0xda209fc8),LL(0xb3448d09,0x64845a36),LL(0x92896b12,0xbde9ae0a), + LL(0xbfc6c637,0x8a46c911),LL(0xe4fa4fe6,0x8dc3d699),LL(0x5cf8e4c9,0xd4ba64f1),LL(0x01cb3488,0x01b3908a),LL(0x38bd7ded,0x69b1fa5d),LL(0x18b2eae1,0x92ad4838), LL(0xb33955b6,0x619324be),LL(0x5c8a6df7,0xc7f37335),LL(0x925b3f69,0xa397f42e),LL(0x5f7e4d11,0x32169a49),LL(0x8d0d9f01,0xc0fa9a54),LL(0x89d8f2e5,0xf52a1f22), +}, +/* digit=38 base_pwr=2^190 */ +{ + LL(0xba8e0a52,0x9c0d5231),LL(0x93e465d7,0x94d0509e),LL(0x98515454,0x67df90dd),LL(0x8dbfb46a,0x223e8b9c),LL(0x6d757ce3,0xf39529a3),LL(0xb4648296,0xffec9175), LL(0xf78aae7b,0x330749e8),LL(0x45f93cc3,0x19e55496),LL(0x94083aa8,0x8c320b34),LL(0x21e321c6,0x1161f5a3),LL(0xde3e7892,0x0980deed),LL(0x6ad76ccc,0x605aa919), + LL(0x180660f7,0x73fa3508),LL(0x2d24936f,0x4cae013e),LL(0x58493d98,0xf64a549f),LL(0xdc79f602,0xd9ceae0a),LL(0xd1512b84,0x6569e37b),LL(0x151c9151,0x11e4c022), LL(0xb55c5813,0x075678c2),LL(0x09d3cb16,0xb26cdb58),LL(0xa57fb969,0x6334dca3),LL(0x223dc3ce,0x0ed90820),LL(0xbd11e277,0x74f9c3ae),LL(0x79c0b8e2,0xaeefed36), + LL(0x2a24c385,0x0e2fc74c),LL(0x34679278,0x836a4740),LL(0x817e2c41,0x25518f16),LL(0xb4b7d3c1,0x8b573a8e),LL(0x4ab56adf,0x012797f9),LL(0xfa2ab690,0x9e0e56d0), LL(0x1c9f6f08,0x009ba1ee),LL(0x2f412e9e,0x8ebf4aac),LL(0x1cfb4e02,0xb143122a),LL(0xcbf2b783,0x988cf0ec),LL(0x57f5be97,0x44a7ed96),LL(0x51804147,0xbdcad872), + LL(0x2f9fbe67,0x378205de),LL(0x7f728e44,0xc4afcb83),LL(0x682e00f1,0xdbcec06c),LL(0x114d5423,0xf2a145c3),LL(0x7a52463e,0xa01d9874),LL(0x7d717b0a,0xfc0935b1), LL(0xd4d01f95,0x9653bc4f),LL(0x9560ad34,0x9aa83ea8),LL(0xaf8e3f3f,0xf77943dc),LL(0xe86fe16e,0x70774a10),LL(0xbf9ffdcf,0x6b62e6f1),LL(0x588745c9,0x8a72f39e), + LL(0xb1bafbc4,0xa96955b9),LL(0x646ece39,0x8dcb55a7),LL(0xeb00e541,0x2b62784f),LL(0x2693249b,0x462f9d7d),LL(0x794c189d,0x8b264697),LL(0x63354e69,0xded6ff55), LL(0xeed1089f,0x7c8ea441),LL(0x1462f461,0xe355f75c),LL(0x1210fd5b,0x87b691f6),LL(0x6983cb27,0x7291bffb),LL(0x92800095,0x9ed83afc),LL(0x1f24d923,0x307a3dc8), + LL(0xd7804b2f,0x7cec60ea),LL(0x45c11441,0x00644643),LL(0x769cd685,0x3c6de88b),LL(0xc7f01232,0x34709186),LL(0xedd2bd0d,0xd9eef41e),LL(0xe427faa9,0x3bafcccd), LL(0xc07e701a,0x33e5350e),LL(0xa87c1fd1,0x9cb2eb47),LL(0x0d5f5b28,0x9fa9a779),LL(0x07ea2e53,0xa2e7076b),LL(0x5c169cf4,0x72f4da32),LL(0x7e751588,0xb7f19294), + LL(0x32dd7a30,0x47eb1335),LL(0xa9db654d,0x9d058169),LL(0x6e7a2b1a,0x375c59df),LL(0x7a35f29f,0x55d37c67),LL(0x493c4cde,0xc78a3678),LL(0x8d83e31b,0xe5f0e2d6), LL(0xe9777bf9,0xf7927002),LL(0xa5afdfc7,0xdd559324),LL(0xb81c08cc,0x077c6c48),LL(0xaa2ef694,0xba1c98cc),LL(0x4c02dd46,0x06c6c954),LL(0x7dd3145e,0x211e50f3), + LL(0x06616d0d,0x2a5f8ecf),LL(0xc7deb373,0xca9b1cb8),LL(0xc59c4301,0x9de31ced),LL(0x0111d998,0x1e0f40b1),LL(0x960d5b95,0xd29d229f),LL(0xd1dabab8,0x10563249), LL(0xa05ecac9,0x7b225cc9),LL(0x78f3b8a0,0xb02e6896),LL(0xf5fb06b2,0x009b52a1),LL(0x842b9081,0x8a575d3f),LL(0xe9272512,0xfddb48af),LL(0x0b452cb7,0xd39b8f1d), + LL(0x67e09987,0x0d6b9c7c),LL(0x0761ad52,0x261a564d),LL(0x9f60925b,0xec462174),LL(0x18529b03,0x83ee0c12),LL(0xfbcfff74,0x72972467),LL(0x6abc4bfb,0x37fc074c), LL(0x54e65e89,0x8b6015bd),LL(0x991583cb,0xde8583eb),LL(0xb4d2c62a,0x379548e1),LL(0x9b24a5e5,0x88024a9a),LL(0xfc03abfc,0x633aa869),LL(0x8fa35283,0xa27657b9), + LL(0x61d9e770,0xde9703b4),LL(0xef4653ef,0x02d4091d),LL(0x576eb5e1,0xefd229aa),LL(0xf77eb987,0xc0b0b243),LL(0xeefe8f71,0xb11309b2),LL(0x68478044,0xfeeacf2f), LL(0x43ac3dd7,0x8dfd8e86),LL(0xb07f95c3,0xc0a24181),LL(0x24be161c,0x551ca096),LL(0xb098cdc3,0x6cb2c1d4),LL(0xe74f84f8,0xbfc74e9b),LL(0x067e3388,0xe58e14d9), + LL(0xf025baa6,0x9eca6f94),LL(0xbcf9c741,0xb2db0741),LL(0x90bb8f56,0xf8e2aab5),LL(0x08762829,0x47729032),LL(0xe2a266c8,0x067a0c5a),LL(0x71b7d7d0,0x22b104c7), LL(0x53e406db,0x4a48cd69),LL(0x24f0070b,0xb85e44d5),LL(0xe10133ff,0x6168262f),LL(0xe4874e8f,0xdfc02315),LL(0xca317e3b,0x20dba2d7),LL(0xe1d2c0c3,0x441c56d2), + LL(0x808dc4b1,0xae10069e),LL(0x8fb3ba73,0x64df30e1),LL(0x7ebaad0b,0xbbe4caf2),LL(0x3dd6119c,0x5907bf37),LL(0x9dfceefe,0x0a723dff),LL(0xf7cffc7e,0x59bff4dd), LL(0x6a6f43c2,0x7bc95fa2),LL(0x3ca0e2b3,0x9001d1d5),LL(0x27b3335b,0x316a7ecd),LL(0x7b8d7d49,0xbf08e672),LL(0xc619058f,0x4b209f93),LL(0x59d8f9ea,0x4c0ca01e), + LL(0xcae69c3b,0x18c452c4),LL(0xef0f00fa,0xf45690ac),LL(0x4f66a5cc,0x3b363aa0),LL(0x47718c52,0x9dd41c0a),LL(0x7e5cd370,0xfa219d7e),LL(0xb2196dfb,0x5d384db7), LL(0x90b4d46b,0x5e14749b),LL(0xd9db9481,0x55796656),LL(0xc8cf353e,0x3bf13d0a),LL(0xa95c485a,0xb89a28a6),LL(0x5da29783,0x568fa3d0),LL(0xd182b1a4,0x4aa008ee), + LL(0xb09fa8f3,0xf7e1ed3b),LL(0x1da5be9e,0xbb4fe6f7),LL(0xf4d1ba21,0xcbab0e01),LL(0x76a5f326,0xb7327410),LL(0x206092af,0xd94d2349),LL(0x728e0e4d,0x739f3cd0), LL(0xf81fd823,0x568644aa),LL(0x6110e2f6,0x510cff6b),LL(0x566c3598,0xef4cf1ac),LL(0x62aae69b,0x2c26f171),LL(0x8964a2a5,0x1e436046),LL(0x3e472c50,0x83c0bbf6), + LL(0xaabd965a,0x79c04804),LL(0x43d0b660,0x9581aab4),LL(0x5ba71d23,0x59bff003),LL(0xb6a0cd80,0x212ecd58),LL(0xbf1ea5d6,0x29bdcd33),LL(0x77a002e1,0x59fd2ff4), LL(0x8d9cd247,0x3c9d2130),LL(0xb1786da3,0x790e9dbc),LL(0x14464d04,0x967ee5e7),LL(0x2b5373af,0xd6f7ebbd),LL(0x39768d40,0x1c0b22d5),LL(0x913f6cc3,0xdfb54983), + LL(0x51b3f1ce,0x167ec88a),LL(0x420024c8,0x19756ee0),LL(0x3877e634,0x10f2e244),LL(0x03462cb3,0x6321bf26),LL(0x9d3afcee,0x1dbd10ee),LL(0x2ca17dcd,0x0726f5f2), LL(0x0bacf018,0x09465266),LL(0xe1feb969,0xc92a9f2d),LL(0x5e1c5912,0x0043b0f9),LL(0x757d3a63,0xa09b94d1),LL(0x9fdef1e0,0x53395652),LL(0xd4fedd41,0x9826886c), +}, +/* digit=39 base_pwr=2^195 */ +{ + LL(0xec65b53e,0x2e75a26e),LL(0x70552fb3,0xfeb630b2),LL(0xee7d8e4a,0x53dfd057),LL(0x8994f449,0xb959110d),LL(0xbb538367,0xb4a16596),LL(0xef82f29c,0xa70917bd), LL(0x43bba6ae,0x5a764300),LL(0xcfbc194a,0xee207476),LL(0x03a4184b,0xc7eab238),LL(0x0f7fcd62,0x60c67ef2),LL(0xdfa8a0c6,0x41e05799),LL(0x04d352b0,0x5d7d05e6), + LL(0x436b59f5,0xc97c01eb),LL(0xef1848ab,0x1d15aca7),LL(0x7fa7d3c2,0xdba1ce80),LL(0x81060874,0x69e6f96e),LL(0x4d7eeead,0x6e3e0df7),LL(0xb0c87f3d,0x8a3b5e85), LL(0xc8a23914,0xc55ae3db),LL(0xbf6d27d9,0x5cdc2a92),LL(0x1de7860a,0xa6008c59),LL(0x6e546731,0x8202f8b6),LL(0xaa1e45d4,0x652f2d07),LL(0x6df5415a,0x146f214f), + LL(0x651ed62d,0x4b01246a),LL(0x0b9eb006,0x9f6824a3),LL(0xba95697f,0x763ae8fe),LL(0xefe2182e,0xeff4f88d),LL(0x74ba79b4,0x2c2ef50c),LL(0x1d23d649,0x319df6c9), LL(0xf6c273a8,0x481f9faf),LL(0x9706dc07,0xe7156457),LL(0x424d5cb9,0x06f0617a),LL(0xa8ad220a,0x6d6b5d8b),LL(0xa2bd8c40,0xa485ca14),LL(0x54cb54f6,0x7a7a15bf), + LL(0xce5fb4d9,0x041c706c),LL(0xb22a79a7,0xddc78cb3),LL(0x839e9d5a,0x7dc4cd27),LL(0xbf3c4c06,0xdfc9db83),LL(0x38b7bd22,0x85b80941),LL(0xd0f4c2da,0x1007dea2), LL(0xc633fba0,0xd7b34006),LL(0x4476e55f,0xa8880acf),LL(0x75236685,0xa340b2c2),LL(0x0113a85f,0x5ddd0551),LL(0x9cb32704,0x7dfc7ab2),LL(0xdabf22ff,0x9a334a33), + LL(0x7e546950,0xa8261c31),LL(0x26706dc1,0x89d19cae),LL(0xedc9af36,0xf8dbf6ca),LL(0x7e446207,0xda79fe4e),LL(0x81cee38d,0x8bbc3195),LL(0xb5be9577,0x9d121e9c), LL(0xcddb5a61,0xdb3fff88),LL(0x3ee86665,0x751cad15),LL(0x3d07abad,0x5c0986df),LL(0x83fe8451,0xf77489b3),LL(0xf90d3e94,0x3546c5a9),LL(0xd8a694b7,0x1ec54bd1), + LL(0xd7d1146b,0x2fbce9be),LL(0xb3980bd2,0xcec9e5d8),LL(0x9f4cbaf0,0x48ea4593),LL(0x2574a3bd,0x56c54009),LL(0xe792c39a,0x84a39630),LL(0xeef81343,0xe5c690f8), LL(0x17655bc9,0xf996760f),LL(0x6c0c191c,0x6009c215),LL(0x966d7027,0xa0ca12e6),LL(0x2e6f7c63,0x92a6d5d5),LL(0x9bd13ead,0x46809d26),LL(0x67aac681,0x3c11fa79), + LL(0xea8edc01,0xca2876f9),LL(0x28d411c8,0x85c1a4ac),LL(0xfb0299a8,0x4d5fca4c),LL(0x17fe8b37,0xa1df6c5a),LL(0xcc062556,0x2ffb3570),LL(0x465ef78f,0x4c59773e), LL(0xb3601ca5,0xea3e39ae),LL(0xb7d30864,0x9806345f),LL(0xe6600e5d,0x428fe41f),LL(0x067a59c0,0x19bd665d),LL(0x908cdb52,0xbd6b8272),LL(0x0b2707d8,0xb6025028), + LL(0xabe2cc87,0x3fc9b3cf),LL(0x514e77fe,0xfd8d64e3),LL(0xfe1ad535,0x2003a58a),LL(0xcb39149f,0xcec4be38),LL(0xbdedf470,0x4d578c99),LL(0x3a356519,0xcd35d7a3), LL(0x8b078d6b,0x7a762f27),LL(0x31ae2701,0x3b6891ed),LL(0x270c508e,0xdc0e817f),LL(0x9fdb29c8,0x5a7be204),LL(0xcb2711d3,0xfa1a0be3),LL(0x3786a0c2,0x5865f55f), + LL(0x23bd1613,0x5f5db9af),LL(0xcfd5bc16,0x0071d2c4),LL(0x81adfd03,0x273d7ad1),LL(0xa0570ffa,0x683508f4),LL(0x611a75fc,0xa49f5c08),LL(0x8fbcfbb1,0xfca5bd12), LL(0x2a4fc49c,0x7eabb339),LL(0xad6e6d32,0xdc5fae69),LL(0xe0b03dce,0x2e599c43),LL(0x7ca7250c,0xf4ad8d3e),LL(0xedeee0fe,0x626c1f7e),LL(0xaab0b3f0,0x2296376f), + LL(0x2f641cff,0x1d0af6b5),LL(0x3648c4a0,0xa932adeb),LL(0xb1ea8fc4,0x67641951),LL(0xb1fae482,0xc0b90064),LL(0x6623477b,0x7012642f),LL(0x5bf019ce,0x1cddc024), LL(0xc2c32737,0xca1f4675),LL(0x97d6b18c,0x11525a5e),LL(0xd3868de9,0x9c034ef2),LL(0x044e0c18,0x0533d921),LL(0xcb5e38c3,0xba6cf14e),LL(0x509d7053,0x438309f3), + LL(0x790b3f68,0x0b82b506),LL(0x12c1f59d,0x75b38ef1),LL(0x4dbdd80b,0x6bc1e007),LL(0xd13bb11c,0xf81480db),LL(0x17259091,0x25131887),LL(0xc61dde4e,0xc5c8823e), LL(0x9b2736d1,0x0ddb06a0),LL(0xa785e570,0x64dfb1fe),LL(0xcc593359,0x6f8f9945),LL(0xd186352c,0xe8e457a9),LL(0x7c342abc,0x389479b6),LL(0x4cc71b3b,0xc73ddfee), + LL(0xc4ff9f0b,0xe164268e),LL(0xe6c36e63,0x6c8e9349),LL(0x78ab17f3,0x734f9794),LL(0x0179ed0e,0x46d468de),LL(0xdfa26867,0x7e68f006),LL(0xe3d0485d,0xe4d4a85d), LL(0xf84c0f8f,0x0913a1d7),LL(0x25a9c9cd,0x4095c8c0),LL(0x49eadd08,0xeeb1a79e),LL(0x7dd8f954,0x433f5e41),LL(0x30bb26d0,0x70a62814),LL(0xff5e8e29,0xad94d8f6), + LL(0x5652a3c0,0x0b614a64),LL(0x34597010,0x89279185),LL(0x810a812f,0x06c97f68),LL(0xbe358e91,0x566120f4),LL(0xc044ff3c,0xcb1f8b75),LL(0x7d1a468b,0x77b3b0b0), LL(0x7d49aad9,0x37a72862),LL(0x8646efd4,0x3ada117b),LL(0x21bac6d0,0xd8626c0d),LL(0xdd2c980c,0x2263d74c),LL(0xfd1b6bda,0x8afd14c0),LL(0xcae64c0b,0x693742cb), + LL(0x9e32c0a4,0x6a352b57),LL(0x77ec7a40,0x5274a082),LL(0x240e6dc5,0xee1f7c7a),LL(0xd313b4a9,0x85d5be62),LL(0x5c01a405,0x1522c5d2),LL(0x960afd5a,0xcfa08aab), LL(0x8e8a93dc,0xa3cb77f3),LL(0x6d1c98c7,0xaacb1676),LL(0x3b93fa9d,0x84090c7d),LL(0x3c0383ad,0xc77f1ee1),LL(0x76f7a220,0x461c93b7),LL(0x04ac0bfc,0x66d63a1d), + LL(0xd2343e0b,0x3d1904c6),LL(0x775f877f,0x8ce038f7),LL(0xd797f231,0xa9d4dce1),LL(0x16c08c2f,0xb6712aa7),LL(0x335ad61a,0x5045b87a),LL(0x7115bb4b,0x44a251fb), LL(0xe66511bc,0x2363cf68),LL(0x81cc48f3,0xed8ab553),LL(0x725c6bae,0x8bf71687),LL(0xc23ab12c,0x16e0d015),LL(0x21f333c0,0xfbdcc064),LL(0x62c9f01d,0xe73df709), + LL(0x4751207f,0x3c7d6b64),LL(0xe440c1a2,0x65e1f96a),LL(0xaa0eaa1e,0x8ed15d20),LL(0xc0eab490,0xe944ad2f),LL(0xf6d9f260,0x71525aa1),LL(0x16146ba3,0x5cd14c88), LL(0x14a41275,0xf9401908),LL(0x2288618d,0x3bb7ea74),LL(0xcab1060a,0x6a4e1c37),LL(0xc8cac96f,0x357fe4d0),LL(0x6a2466ec,0x97a8b8ab),LL(0x9c01be70,0xb6e83fdb), +}, +/* digit=40 base_pwr=2^200 */ +{ + LL(0x2a7aeced,0x5b0b5d69),LL(0x01dc545f,0x4c03450c),LL(0x404a3458,0x72ad0a4a),LL(0x9f467b60,0x1de8e255),LL(0x90634809,0xa4b35705),LL(0x706f0178,0x76f30205), LL(0x4454f0e5,0x588d21ab),LL(0x64134928,0xd22df549),LL(0x241bcd90,0xf4e7e73d),LL(0x2facc7cc,0xb8d8a1d2),LL(0x1d25d2a0,0x483c35a7),LL(0x1ef9f608,0x7f8d2545), + LL(0x22ec7edd,0xa7040160),LL(0xcc9c8ee8,0x19124972),LL(0x2ccb9417,0x697f301f),LL(0x6f00d8aa,0x3ee87764),LL(0x8138a017,0x2b5afaf8),LL(0x832d7543,0xf152b14c), LL(0x383052f9,0x27c27ce2),LL(0xe1dae11b,0x4746c5b5),LL(0x5b752008,0x92dc5ac7),LL(0xe84fe5f1,0xcf382e01),LL(0x7d5929ce,0x90e03419),LL(0x15ca3ffa,0xafee3abb), + LL(0x54ebc926,0xcb51f039),LL(0xb8d4a7bb,0xe235d356),LL(0xb41fe1a6,0x93c8fafa),LL(0xa719f254,0x6297701d),LL(0x644f5cde,0x6e9165bc),LL(0x0c11c542,0x6506329d), LL(0xa92b4250,0xa2564809),LL(0x889c2e3e,0x0e9ac173),LL(0x22b1d1be,0x286a5926),LL(0x6ecdd041,0x86a3d752),LL(0x649f9524,0x4b867e0a),LL(0x0629cb0f,0x1fe7d95a), + LL(0x1c6d03b0,0x11dd860e),LL(0x09eec660,0x30c17008),LL(0x35c0192f,0xd4f8aff6),LL(0xe3a4a900,0x96a727b1),LL(0xde78c8ba,0x1426daff),LL(0x8d1527c4,0xfacaa9bd), LL(0xcd072989,0x0c0d5234),LL(0x918550b5,0x1936c20d),LL(0x3d914fb3,0x4828bee4),LL(0xf3ba26a6,0x8324ea38),LL(0xa94eb26f,0x027590f3),LL(0xacd957bf,0xfd354295), + LL(0xca5baf54,0xf4f66843),LL(0xefe7db78,0x298db357),LL(0x7365712f,0xf607e86e),LL(0x8a822bc0,0xd5882298),LL(0xc61299b3,0x2cfbd63a),LL(0x67167b1a,0x6f713d9b), LL(0xde0b077a,0x750f673f),LL(0xee2178da,0x07482708),LL(0x69123c75,0x5e6d5bd1),LL(0xeab99b37,0x6a93d1b6),LL(0x8caec6a3,0x6ef4f7e6),LL(0xcf3ed818,0x7be411d6), + LL(0x3dfb423c,0x959353be),LL(0x36d41cc8,0x8458e858),LL(0x99d7a4f4,0x6a4826f7),LL(0x52fe4b65,0xab146ece),LL(0x35038573,0x94fc21d7),LL(0xf4d56e84,0x26f50135), LL(0x3162d92d,0xe15ca04d),LL(0x34ed4e84,0x8f652fcd),LL(0x4f21a910,0xef7e7924),LL(0xdcc76132,0x2eecb7e9),LL(0xdbe89048,0x5b484745),LL(0x6e43a2c5,0xbf8c490f), + LL(0x63a0a7d2,0xf92b3073),LL(0x881dc8cf,0x32da431c),LL(0xc578e3a3,0xe51bd5ed),LL(0x9587fa22,0xefda70d2),LL(0x9b2eba85,0xcfec1708),LL(0xaf7ba530,0x6ab51a4b), LL(0x98174812,0x5ac155ae),LL(0xccb076e3,0xcaf07a71),LL(0xc38718a7,0x280e86c2),LL(0xd63745b7,0x9d12de73),LL(0xbf8a79aa,0x0e8ea855),LL(0xbd705bf7,0x5eb2bed8), + LL(0xda756624,0x3ede2484),LL(0x73b13062,0xb22da2ab),LL(0x962a667b,0x56069e93),LL(0x130f2cea,0xc931266b),LL(0xa7366a66,0x4bd6a6fc),LL(0xaa5ac3b1,0x23f30563), LL(0xd7c2b26f,0xa025d0ef),LL(0x62129bc7,0x597ce7d8),LL(0x2b3057f2,0x4809927f),LL(0x1499f884,0xb001c10a),LL(0x30b9a653,0x309d141c),LL(0xbf659d05,0xadddce7d), + LL(0xae16de53,0x33fe9578),LL(0x10bec902,0x3ae85eb5),LL(0x44af850e,0xc4f49658),LL(0x087dd658,0x6ea222b3),LL(0xa51f1447,0xb255e6fd),LL(0x117e3f48,0xb35e4997), LL(0x05616ca1,0x562e813b),LL(0x8a61e156,0xdf5925d6),LL(0x571c728b,0xb2fa8125),LL(0xa2f2d1cf,0x00864805),LL(0x1bccb6ff,0x2dc26f41),LL(0x63ae37dd,0xebd5e093), + LL(0x9303b5b0,0xe448127c),LL(0x7a74ec27,0x6ac65681),LL(0x86f0e7ec,0x05128a52),LL(0x7603a73c,0x9d9f32c1),LL(0x556c51ea,0xb5a799bf),LL(0xa9a9a416,0xea94f169), LL(0xaebde511,0xebb3e549),LL(0xf9cccd3f,0x9037046a),LL(0xf08a8254,0x2a9343a8),LL(0xd40c7f26,0xc5cc43f6),LL(0xb39677d2,0xe1146cdc),LL(0xbe66d4b6,0x6cbfec8b), + LL(0x0a285611,0xd2d68bb3),LL(0xdc8378f2,0x3eae7596),LL(0x6cc688a3,0x2dc6ccc6),LL(0x011f5dfb,0xc45e5713),LL(0x62d34487,0x6b9c4f6c),LL(0x1fc65551,0xfad6f077), LL(0x62b23b52,0x5e3266e0),LL(0xe98f4715,0xf1daf319),LL(0x3ed0ae83,0x064d12ea),LL(0x564125cb,0x5ccf9326),LL(0xc63c1e9f,0x09057022),LL(0xdc9b5d2e,0x7171972c), + LL(0xb0a219fd,0x10867369),LL(0x5ab56581,0xe7efac97),LL(0x7813f6f5,0xde3372e8),LL(0x1d4b8ed3,0x3bbe5977),LL(0x53376573,0xf17f61e5),LL(0x97964b90,0xf719d06d), LL(0xb24d2e4a,0xb1e8c483),LL(0x184379eb,0x53709647),LL(0x8a8fcb83,0xb739eb82),LL(0x60163017,0x5d2f3a4d),LL(0xf823b4af,0xccedf4f4),LL(0xff315eae,0xa6e166a1), + LL(0xeabd21b2,0x2364fd9a),LL(0x9174ad6d,0x3ce5f4bb),LL(0xb38688c0,0xa4d6d5d0),LL(0x6d87fd7d,0x2292a2d2),LL(0x4ca02e54,0x2a7d1b53),LL(0xb4185715,0x7bee6e7e), LL(0x8fc63acd,0x73e54609),LL(0x4064e09d,0xf4d93a12),LL(0x2b92daa5,0xd20e157a),LL(0xc4b81a00,0x90d125db),LL(0x7682de13,0xcb951c9e),LL(0x27987545,0x1abe58f4), + LL(0xe1c01fab,0x76452ac1),LL(0x6bd32f0c,0x167d7326),LL(0x4a283a42,0x72d209c1),LL(0x48ea0ba9,0xd26859b6),LL(0x6369309b,0x7b3e5c46),LL(0x93c9a0b9,0x474a9625), LL(0xc76e25dc,0x676ea3e4),LL(0x71400c71,0xb0c9ccf7),LL(0x36f83518,0xc4295870),LL(0x267f0c25,0xb86c1b39),LL(0x28884a80,0x8a3e3524),LL(0x43ba8e28,0xea182c45), + LL(0x30c70c8d,0x6d351640),LL(0xce2361b8,0x8047d811),LL(0xdf8e2c81,0x3f8b3d4f),LL(0x33fa1f6c,0x5d595477),LL(0xe29b8a91,0xf769fe5a),LL(0xd737b2a2,0x26f0e606), LL(0xb8b31c6a,0x70cbfa5d),LL(0x863d3aea,0x0f883b4a),LL(0xe386ae2f,0x156a4479),LL(0xade8a684,0xa17a2fcd),LL(0xe2a7e335,0x78bdf958),LL(0x3b9e3041,0xd1b4e673), + LL(0xaf2825c1,0xa6c32900),LL(0xd223a04e,0xb37c46c1),LL(0x063de7ea,0x691e7d39),LL(0x10daf9bd,0x998df4e7),LL(0x718b5d7a,0xc7085b9e),LL(0x16b3d4b8,0xd41abcc8), LL(0xf9bc4041,0x4dfce693),LL(0x659ec7a8,0x383677ed),LL(0x4491fb34,0x2c1904bf),LL(0x4552451c,0x7c1bf111),LL(0x3c5e5e40,0x6562cc2c),LL(0xfe0e4372,0x1ecaa2a1), +}, +/* digit=41 base_pwr=2^205 */ +{ + LL(0xf3087bc8,0x84c3630b),LL(0x74be6e26,0x152691e6),LL(0xf61af001,0x5abd125f),LL(0xbfea3525,0x69bca56f),LL(0x00e0cb6e,0x384af199),LL(0xd00475a6,0xb0b13cfe), LL(0x5e394049,0xedafde49),LL(0xdaf2add6,0xd988b558),LL(0x6c8ffcc9,0xf14cf97a),LL(0xe5a9cc5c,0x4d6cec23),LL(0x8a104e05,0xb0d678f8),LL(0x9fb527c1,0x80a7fcba), + LL(0xc22137d2,0x0d6cadbf),LL(0x628a3298,0xb5db59d3),LL(0x4ab19507,0x3b433c73),LL(0x660086b3,0x4fc53405),LL(0xa1eb0f43,0x770ae903),LL(0xf6b5b58d,0x31b5857e), LL(0xd392868e,0xe206e141),LL(0x4b31de04,0x8be6956f),LL(0x47449e07,0xcfbfca2f),LL(0x39fef8e1,0xebaef256),LL(0xc16fc80c,0x959e37b8),LL(0xe911d61b,0x8bb4bdd2), + LL(0x4343bbf8,0xc483d4b2),LL(0x8a0fc95e,0x42aca2e1),LL(0xcbab1fb0,0x5165df6f),LL(0xf6cdfc0c,0xeb284370),LL(0x994320fc,0xab565c00),LL(0x62133e80,0xc0d157fd), LL(0x5b69644a,0x7850cda5),LL(0x806ec8b8,0xe37ae76a),LL(0xc2c82edd,0xd14b805c),LL(0xcf244539,0xcb5468b6),LL(0x25dbe92d,0x97d43ee8),LL(0x89fb8f1e,0x14422436), + LL(0xa0a85236,0xd7bf2ac6),LL(0x7194c46d,0x2921b55c),LL(0x9afa9762,0x162fabaa),LL(0xb62b36ab,0x7b7f1664),LL(0x296a84e9,0x77b9f797),LL(0x7dbd843d,0xfcc1ad65), LL(0xcd77b7f6,0xc6e9c1e1),LL(0x917067c9,0x9cf0e272),LL(0x3bfa90bf,0xfa7fa93d),LL(0xd050e46a,0x55846fe9),LL(0x35c56256,0x473b9a0d),LL(0x2b656a65,0xadd29e33), + LL(0xebc69b0b,0x926c2552),LL(0xd4c7432e,0x953a850f),LL(0xb9359035,0x0ee85e14),LL(0xbde090a5,0x8b10b01a),LL(0xec423943,0xb2878dca),LL(0xf70bde20,0x2571a178), LL(0xf5ebeee5,0x24ed159a),LL(0x043f6539,0x60c202af),LL(0xc8d4ffc3,0xdaaa76f4),LL(0x06eda10f,0x2fc1f1ba),LL(0x88ded556,0xddf159ee),LL(0xe67b1ec4,0xcfa71782), + LL(0xd5d826b0,0x31521f66),LL(0x40787844,0x0a636952),LL(0x9c8f934e,0xc0a3bd05),LL(0x2f0ce835,0x12c57dd4),LL(0x67064213,0x847f6a99),LL(0xa88bd71a,0x1c9e1a7a), LL(0x171e8407,0xc4060eb2),LL(0xed106780,0xdf78d8df),LL(0x0d704729,0xa3d28ceb),LL(0x46ca3912,0x4f8e5232),LL(0x017791f4,0x09e9f852),LL(0x1e6ea97d,0x59400663), + LL(0x83939224,0x547b0d95),LL(0x1e026769,0x3a0823ff),LL(0x25bd43ac,0x60166715),LL(0x18ba5f64,0xb6cf475e),LL(0xc8b6d09d,0xa22f9c92),LL(0x3ccf50ab,0x73055368), LL(0xee6deefe,0xa6de248e),LL(0xacc3ca20,0x32aaf8b2),LL(0xad44e674,0x0e254c5b),LL(0x35f95f98,0x8aa73e65),LL(0x60a2dc1e,0xe6226001),LL(0x9109020a,0xdf948210), + LL(0x2893f2f7,0x372798f0),LL(0x9e5030ca,0x4f62bfac),LL(0x8a1e2567,0x5e64f9a9),LL(0xe70391c8,0x5870254c),LL(0x41f02458,0x2def81a3),LL(0x1d087bed,0x25d4e4dc), LL(0x4fe24a13,0x3557d07d),LL(0xdc3112bc,0x6da49186),LL(0x5f73ba50,0x08c8c567),LL(0x9c7c6706,0x5309050b),LL(0xbd985072,0x2ab67da3),LL(0xe5df4e96,0x9bafa8b1), + LL(0x5c02f173,0x3ac66289),LL(0x76d566e5,0x6a110e38),LL(0xb9577e26,0xd9cc14e2),LL(0xfdfe617e,0x6f3d5df9),LL(0x352bb2ca,0x8fac740f),LL(0xc28e6310,0x50bc8a0c), LL(0x77ac93f7,0x6e572fc4),LL(0x605bb8e9,0x56277377),LL(0x402b8c55,0xad6d0637),LL(0x4509eda7,0xdab37791),LL(0x0854e91b,0xae770abc),LL(0x742b3de8,0x523bd278), + LL(0x17fecb90,0xc2cbd644),LL(0xb32dffdb,0x61616eb3),LL(0x9f5d2095,0xdc4485a2),LL(0x6553371b,0xf7891124),LL(0xbf9b20af,0x4f06ba18),LL(0x1a2c4df1,0x136d4f29), LL(0xfb8b685f,0xc04aca34),LL(0xf2b657bb,0xeec83c20),LL(0x5925a36a,0x4da5d70a),LL(0x72ff2965,0x80608741),LL(0x9f352620,0x2e0dd9ff),LL(0x46d1a7a8,0x5f0afa67), + LL(0xf46aace5,0xa995a95b),LL(0x9eaa630a,0x44ede537),LL(0x00336e3b,0x421f3b35),LL(0xcf47c9ed,0xbf897478),LL(0x259e0827,0xf360ae32),LL(0x2e6a9f6b,0x04e0e3e8), LL(0xa9136702,0xb26eae5f),LL(0x853674b4,0xd6cb15a1),LL(0x748bcbc9,0xf81276e2),LL(0x0a4ca1d7,0x7fc02e22),LL(0xcd82f330,0xf650f48e),LL(0xabaa8859,0xf4ea7c1d), + LL(0x95c746ec,0x1ccd44ff),LL(0x10405763,0xe18914b5),LL(0x21a3a927,0x50ed6443),LL(0x43ef8e8f,0x4f96a1b1),LL(0x77952bf8,0x7f5645e5),LL(0x66dbdf15,0x4bc5c7ab), LL(0x23930a08,0xacc16126),LL(0x504cf9b6,0xbf5ed482),LL(0xd71ecbd7,0xdeb7a798),LL(0x4a4dd859,0xf62e63b1),LL(0xdaf714d9,0x668809a7),LL(0xf3a4329e,0xdd836382), + LL(0xc48f3ad5,0x894bdbd9),LL(0x09e167f6,0x687ff8de),LL(0x30371c43,0xf06104a9),LL(0xce84dd10,0x82fd34b7),LL(0x66ce5abd,0xae122deb),LL(0xfc4a90b2,0x31f041d2), LL(0x9a01c607,0x2589535c),LL(0x695bd7ab,0x231bcc85),LL(0x62e3a31d,0xc67c3062),LL(0x7af3e186,0x31be4475),LL(0x88efa7f1,0x1a2077a3),LL(0x815fad1a,0xffe53e22), + LL(0x66229776,0xdd155913),LL(0xf7882064,0x84093730),LL(0xe50ee337,0x6dddcb14),LL(0x7a1f7e81,0xa8e6ec59),LL(0xf3738a6a,0x8467f998),LL(0xad3f1840,0x70fcc6bc), LL(0x723b3f4b,0xf82eb4be),LL(0x06beec1b,0xf0f39354),LL(0x7ddcb539,0x1b181ea3),LL(0xad6a81b9,0x9c82c4fa),LL(0x5c612c2b,0xcc5ea543),LL(0xbb258d6f,0x63ce7571), + LL(0x0b96547f,0x1b588855),LL(0x65d1a59d,0x4539c9c0),LL(0x26e15084,0xd6c95fea),LL(0x86b96242,0xf84ad9e2),LL(0x451a5486,0x92f57d6d),LL(0x06a9e87e,0x0215cfcb), LL(0xf66e46f3,0xe05b10ea),LL(0x655a0642,0xe7b0e72f),LL(0x7b117f43,0x03503267),LL(0x779ea4a1,0xf5b78105),LL(0xa4adac77,0x28ee00fa),LL(0x6a93a2b1,0x1ea67d71), + LL(0x26882c6c,0x7be81fb1),LL(0xecd25498,0xe2d5a251),LL(0x7a8d1678,0xbb3d40e2),LL(0xd520811e,0x1806c67a),LL(0x86f65d23,0xadd4bb66),LL(0xe20e23d7,0x3a62b1b3), LL(0x6548b3eb,0x208b4700),LL(0xb7ec2809,0x0497f09a),LL(0x121c37e2,0xbd3964f8),LL(0xa598efbb,0xd35ef301),LL(0xc5eef966,0xbd76a276),LL(0x0af64e46,0x64700a7f), +}, +/* digit=42 base_pwr=2^210 */ +{ + LL(0xd3812087,0x169474a2),LL(0x6698ca7a,0x9de300da),LL(0x2ede425b,0x8589de92),LL(0x6df8a890,0x50e03fea),LL(0x4ba8b8e3,0x0d8a5c1c),LL(0x3fffb91a,0xf273aa67), LL(0x75fc8236,0x21cf0544),LL(0x9799c242,0x6ceafacf),LL(0xd0962c81,0xc3237eae),LL(0x213f6004,0x43d6ac34),LL(0xd4148b6b,0x45e619b2),LL(0xea5fb80a,0xfafa18b5), + LL(0x2f063b51,0x9a8580aa),LL(0x1c216613,0xa83c8ff7),LL(0xbe07f781,0xb4da0970),LL(0x712f7b7c,0x0ac2a260),LL(0x436a7b97,0xc9b8ee84),LL(0x11fb2f62,0xd758c20d), LL(0xf170b799,0x5daabed9),LL(0xc46bc387,0x018d2fdd),LL(0xd96cfb8e,0x82d6b5b7),LL(0x44d9e843,0x4d7d0d93),LL(0x91e7da3c,0xfa2a9ea9),LL(0xd531b253,0x8230c1a3), + LL(0x5ec31754,0x82412f52),LL(0x9d32e890,0x42f462c8),LL(0xce897ff2,0x1e7b58ce),LL(0x41164628,0xcfef7852),LL(0xd8bb22ef,0x34ee0422),LL(0x7d32f01b,0x6e0d44ac), LL(0x5a3cc196,0x96825165),LL(0x99eb23d3,0xa26724dc),LL(0xa75f7252,0xdb575faf),LL(0x62a3e5b1,0x778e3330),LL(0x84cccc80,0x8689884e),LL(0xb645502d,0x9883cd19), + LL(0x34220e26,0x4cc41f28),LL(0xa49749c4,0xb5937c6d),LL(0x0fa1ca24,0x70536664),LL(0x91e5edaf,0xeeb40f3b),LL(0xf1d3de14,0xcdf98235),LL(0xff018c43,0xa65e5b7e), LL(0xacee3a6e,0xaa3228e7),LL(0xe08f4ff1,0xb63a6289),LL(0x650b2daa,0x90e90425),LL(0x6d875f17,0xe4a8cad2),LL(0x9ce8a46e,0xc212029c),LL(0x5ed7cfb5,0xce051283), + LL(0x59b79436,0xb0df2261),LL(0xa195be26,0x82bd0daf),LL(0x3398c317,0xbc99a94b),LL(0x3c96ee31,0xbeb44c90),LL(0x664d2e46,0x3c39ad81),LL(0x0a3e0585,0x08178752), LL(0x413e269a,0x9a054b6b),LL(0x98c3b62e,0xbe58891d),LL(0xe5734974,0xe7fa4c4d),LL(0xd0a846a8,0x8ac535f4),LL(0xa651339c,0xea0f95f1),LL(0xd96aa239,0xa255274f), + LL(0x9534047e,0xe23b7b22),LL(0x3a3bd625,0xbd70aea8),LL(0x238db60b,0xf44b05fe),LL(0x0293abcf,0x9c46fb14),LL(0xbfd8875b,0x12cab5d3),LL(0x12dd0c65,0x1f38d4aa), LL(0x2adf9805,0x4bed4157),LL(0x8a56609d,0x3f87da92),LL(0xda02c903,0x10b93363),LL(0x21ce4786,0x7ecc7266),LL(0x1e3da5bf,0x8ae36685),LL(0xd3edee12,0x196040ff), + LL(0xe81508e8,0x4805841f),LL(0xa4808642,0xe2a578d3),LL(0xcd0b2555,0x6bbf10ac),LL(0xaf5cde28,0xc5071eff),LL(0x9a7124a9,0x665e7543),LL(0xc1437981,0x157c11ed), LL(0x7aeddd8b,0x2019367d),LL(0x386e3b8a,0x74a1e104),LL(0xfbe09a42,0xe72d429b),LL(0x061b862e,0xaca96fd9),LL(0x122595f8,0xbb2d2bc8),LL(0xc509d644,0xc90c6503), + LL(0xcff05ada,0xadb5966f),LL(0x5c57284e,0x8ed26c02),LL(0x44693a95,0xa76e73e2),LL(0x5982bbd3,0x14da7435),LL(0x5d2ca132,0x46e982cd),LL(0x24938e76,0x8f390740), LL(0x0a89b09a,0x749206b3),LL(0x93b4a1e5,0x429653c7),LL(0x7025bb7c,0xbee3d156),LL(0x19555c9e,0xe23f0e1e),LL(0x751639ba,0x0dec3837),LL(0x05d43bd0,0xb36cb844), + LL(0x74f90b6e,0xae76a96e),LL(0x24c6789c,0x5fa8e948),LL(0x03abbb81,0x2b3584bb),LL(0x5c451f72,0xe19ce47c),LL(0xd619ac7a,0x35792fba),LL(0x50059bf4,0xfa0282a2), LL(0xdabe692f,0x562bfd14),LL(0x47eeb6c2,0x1aaf542c),LL(0x045d0360,0x392d5bba),LL(0xd80fe998,0x4e7bb31b),LL(0x1111e14d,0x08f62ef3),LL(0x4e9ee1b8,0x4de917b0), + LL(0x67166271,0x8b9d2d58),LL(0x142bab7c,0x658db4ea),LL(0xa4ad2849,0xdf84932f),LL(0x5f6f86a7,0x04b11335),LL(0x50cfcea7,0x2de6b29c),LL(0x9be6a3a3,0x46d8f68a), LL(0xaf0204af,0xfb88cda7),LL(0x26029d72,0x3ece4491),LL(0x3f946dfd,0x69fef1e2),LL(0x01ef7bb5,0x708532fb),LL(0xeb3795a2,0x78d5053d),LL(0x6b36d57b,0x819a6320), + LL(0xe509d19e,0xca07e0c1),LL(0x9f6281b1,0x6c7e42c3),LL(0x77b66728,0x0e2ff439),LL(0x80e76251,0x1d740e78),LL(0x31a0eb23,0x6bfae4c6),LL(0xaa9b0b3b,0xd78ca917), LL(0x991e1781,0xe140c662),LL(0x0dd3cfee,0x6e396b5f),LL(0x6ce7f6c7,0xf0a1d197),LL(0xd5b01564,0xbe10f8ef),LL(0x101a5194,0x865cbd54),LL(0x66861ded,0xf6658852), + LL(0x5b28f7da,0xe4e52e86),LL(0x9a58683f,0xeb43a680),LL(0xb49f2b38,0x73b951bf),LL(0x3f8097cf,0x7b6cb7db),LL(0x328fbf05,0x9dfb8d0b),LL(0xebce6211,0x491635a5), LL(0x90fdd577,0xa31a1523),LL(0x1cd2f39c,0x334120df),LL(0x6b563876,0x1d22834e),LL(0x10ee5a28,0xfd91b30d),LL(0x59aee4ea,0x3d7a282d),LL(0x73300a76,0x36814c6b), + LL(0x6621c251,0x7b584add),LL(0x4233aba3,0x98da669d),LL(0x33aa2065,0x4d652b79),LL(0xdf7b4ed4,0x901bcfb8),LL(0x48012f81,0xb2ce5879),LL(0x3cb71b88,0xc18e2cd6), LL(0xff86279d,0xadb0f2bd),LL(0x5bd15866,0x46d9e5d6),LL(0xc635a4c0,0x11b1fb3e),LL(0x01b1006a,0x8bcd0ad2),LL(0xcbab210b,0x0f6f7502),LL(0x0d6b3995,0xd6cc3e56), + LL(0x137264c5,0xa54a6420),LL(0xf9c2e45e,0xa6ef0e78),LL(0xd58d850c,0xba8b5a73),LL(0x6ef6fc3e,0xc0209ed8),LL(0x91f7518a,0xe39dd0f3),LL(0x42b3eda6,0x74697b89), LL(0xabfc9150,0x2dccac36),LL(0x98b2f5a5,0x80e4fba2),LL(0x771018d5,0xe0e56fd2),LL(0x4c22bb94,0xa31fd168),LL(0x1a66ef21,0x8b0998f7),LL(0xb5a53ddb,0xed483e55), + LL(0xf23978eb,0x95db1c0e),LL(0xf04011f4,0x80ad1612),LL(0x4d7ae83d,0xe76bd182),LL(0x8fc3bd60,0x841d6e66),LL(0x6875e2d0,0xb68e8079),LL(0xd5d9dee7,0xe3965efc), LL(0x58930931,0xc488bb7e),LL(0xa907aa24,0x52f4de19),LL(0x321cc197,0x39aebbdd),LL(0x67de5c66,0xd2f5b1f9),LL(0x8efe3e76,0x60f1a8c2),LL(0xaf988831,0xf40604a0), + LL(0x0acb5935,0x78b5c14c),LL(0x4311d3be,0xd9ec715c),LL(0x09e1759e,0xffa22ab2),LL(0xb4b2f68c,0x5a86263d),LL(0x6b5be7f4,0x71e77c51),LL(0x19844f6d,0xfb5bea3a), LL(0x0890ffab,0x2519d006),LL(0xf0329ef0,0x426a03f0),LL(0x85b3c2a9,0x2c6d74a6),LL(0xc294f449,0x9306f68f),LL(0x2c69fb46,0x552e77c2),LL(0x10bb9886,0x7c7337ad), +}, +/* digit=43 base_pwr=2^215 */ +{ + LL(0x44b133d9,0x3313a9d5),LL(0x2da910dd,0xdb85c25d),LL(0x5e4dd5cd,0xc0fdef91),LL(0xc565dd67,0x902a2a93),LL(0x7fed05ac,0xd8eba4dc),LL(0xe157dae9,0xd453995c), LL(0xf250cb55,0xd655d0b3),LL(0x86119222,0x4194a09e),LL(0x0652872b,0x5b7e525a),LL(0xe68c0ddb,0xaf7968ef),LL(0xf51cb31c,0x2ec02930),LL(0xf2be071e,0x237f3ae4), + LL(0x94d0864f,0x9b23ab4e),LL(0x009c9fc1,0x46356266),LL(0xe798edf9,0xdbe99e51),LL(0x307675c7,0x38547449),LL(0x628c0fb6,0x23ffaf55),LL(0x1698c372,0x56ccd2a3), LL(0x8347ce95,0x39f45a57),LL(0x4f2c6118,0xe0aaec74),LL(0x4af138fc,0x2a89079e),LL(0x2ee4ecc0,0xb86371ea),LL(0x06bbf92f,0x076d256a),LL(0xae3c4c51,0x9073adb8), + LL(0x4c99252c,0xdaa77b43),LL(0x326cb0e1,0x59e38731),LL(0x03ca6c85,0x281a38cc),LL(0x433835a0,0x83565666),LL(0x30a928fd,0x3654ec9e),LL(0x7cb281f8,0x1c82abca), LL(0x13fafa6f,0xbeba0fe5),LL(0x99440e63,0x67432292),LL(0x0034d0cf,0xd62777af),LL(0x9cde52a0,0xd42b95fe),LL(0x6a23630f,0xb5b891bc),LL(0x64594976,0x8e4d2984), + LL(0x743c15d9,0xba2e9543),LL(0x1c99c984,0x7d5812db),LL(0x45bdc19e,0xf94db951),LL(0x382e77bd,0x951d00ae),LL(0xb220b29a,0x9940a5fb),LL(0x58fc91f1,0x6908d50e), LL(0xdd0940fe,0x682e42ea),LL(0xa1d32009,0x2124e23a),LL(0x16294d05,0xbe158100),LL(0x2e326d68,0xaea13fe3),LL(0x15e64fce,0xc0dfe1ef),LL(0xb8237a8a,0x32dbc0b5), + LL(0x663771f8,0xea6f1448),LL(0x9a0906cb,0xbf11f126),LL(0xd1a6a6a5,0x8c08219d),LL(0xdd56f277,0xf2af6e04),LL(0xd569188a,0x8ad26705),LL(0x6e071c9b,0xf0547631), LL(0xfd4aa6ec,0x945f40bd),LL(0x8486987c,0xbd6a8a8d),LL(0xb947b6f2,0x4f577728),LL(0x7aab6bc1,0xe2754a8e),LL(0xfb48a9ab,0x918d02cd),LL(0xbf904200,0x7e3ddaea), + LL(0x0bc28725,0xe59b8002),LL(0x149f8991,0xc6f8fa54),LL(0x5af5b47e,0x8799172a),LL(0x160d7e8d,0xf72c9780),LL(0xb2f9a549,0x1d1ce972),LL(0x4857b44a,0x8fce3f16), LL(0x1ed5e0e5,0x2545bdd7),LL(0xc259176a,0x222c33a7),LL(0x4e23c064,0x5a60343a),LL(0x1d1fd9cd,0x986779f9),LL(0x3570b5b3,0x5bd5611b),LL(0xf9d765a3,0x2758caea), + LL(0x3a16e352,0xd8135f00),LL(0x55d4e996,0xbfc784b2),LL(0x0a6874d0,0x5da46321),LL(0xe8e1e4d0,0xf1635286),LL(0xa0e9c4de,0xe71332b3),LL(0x60fc995e,0x5f076117), LL(0xc19ebb59,0x1c1305d1),LL(0x4603baa2,0x1d5987f2),LL(0xea7c9f9c,0x6b7885b0),LL(0xfe9ba1fc,0x362734e8),LL(0xb3892110,0x49a3ff32),LL(0xd4997512,0x0e5b2166), + LL(0xc36d3f25,0x6ee65a08),LL(0xe393e4d4,0x7b6c811f),LL(0x2876e523,0xc4a2cc38),LL(0xd3bf53aa,0xab7aba26),LL(0xdb7f290c,0x5bf00871),LL(0x1ee6d5bd,0x3cb1cd13), LL(0xde998ada,0x4cafb218),LL(0xf6319101,0xa1ecf36a),LL(0x20b281cc,0xa1fe7855),LL(0x64d9c65e,0xe457198e),LL(0xc5a0e67b,0xa3d1a6d0),LL(0x90cc468a,0x69ddbc32), + LL(0x53a38094,0x3f3cb2a5),LL(0x3b19448a,0xc9308152),LL(0x925ba579,0x9cf32819),LL(0x1801e686,0x44b9590a),LL(0xdf04be40,0xab6b284c),LL(0x2c216ae7,0xc40a58e6), LL(0x018a60e8,0xa6ced619),LL(0x83e5cc65,0x83d4cde1),LL(0xb5a91945,0xe2559f51),LL(0xef0f53ee,0x53d9122b),LL(0x32e40a83,0x474c281b),LL(0x944dfe65,0x332324a7), + LL(0xa1fd057d,0xcb0a3657),LL(0xb4aa013b,0xad79ae60),LL(0x97ed887a,0x0b852109),LL(0x30d9b297,0x5fec3e1f),LL(0x10fb9c74,0x4ce4149f),LL(0x8cbff785,0x4e08b4d9), LL(0x2f07cbae,0x81b0d7b4),LL(0x495230ae,0x08bc321e),LL(0x2b841eb8,0xaec221de),LL(0xd83c22e6,0x62c7d86a),LL(0x85affe91,0x5504dcf8),LL(0xf445481d,0x785a06f1), + LL(0x18670cf4,0x8b68acc4),LL(0xcba49dca,0x64073a95),LL(0x9f508534,0x633757b0),LL(0x916f3a09,0x78ad9df7),LL(0x08617468,0x46187c92),LL(0xc5f77b94,0x48f37eb6), LL(0xd058bb27,0x333224f0),LL(0xd8852abb,0x7f28ace7),LL(0xa2e62327,0x8c9f634f),LL(0xc4116c1b,0x75212283),LL(0xcc0c0851,0xb7723ad0),LL(0x5b72e5aa,0xc8a4cff1), + LL(0x6dadc46f,0xd4ee3f7f),LL(0x5d7febd6,0xa1f3dc92),LL(0x63ebab5b,0x4c0bee13),LL(0x005ec237,0x70e32d77),LL(0xc52fb006,0x302fc73d),LL(0x8f159899,0x1af84c0a), LL(0x0686232a,0x42a5478f),LL(0x8a308687,0xb4fc5634),LL(0xc8378f0d,0x042c4970),LL(0x8e2c86c5,0x70c19575),LL(0x84c7c767,0x61a95e68),LL(0xd6fb43a8,0xd96a8216), + LL(0x1a2db746,0x67f8fa3e),LL(0xba267cad,0x2f041d4a),LL(0x65bafbee,0xdf8126ea),LL(0xfad7b234,0xf63fc68f),LL(0x702db8ab,0x4280e1e1),LL(0x02468fc2,0xbe6122f2), LL(0xc89b74de,0x06b076d0),LL(0x4bbdb7cc,0xc2515543),LL(0x50eb6108,0xae82a08f),LL(0x2f0f3f5e,0xf49da234),LL(0xeca6448e,0x7d1923b2),LL(0x70b144ab,0x9e01d58c), + LL(0xee49a1bf,0xeaf237a8),LL(0x4ea1da86,0x78bf04bf),LL(0x3f251ad5,0x5fbffa47),LL(0xd828578e,0xc40570f6),LL(0xd4e118ad,0x5cc65c0d),LL(0x5da48548,0x9e18ff96), LL(0xef7e714b,0xe27fc2e7),LL(0x13df7524,0x19ff3f7c),LL(0x5e27fb12,0x35a32fa5),LL(0x10003fae,0x7fcfd728),LL(0xf49800c1,0xc74b50a3),LL(0xdbafb2e6,0xbf0732eb), + LL(0xc1238c95,0x1d1b02ab),LL(0xee8c4d2b,0x85b3878a),LL(0xc761afd0,0x23ba366c),LL(0x4f023bb7,0x47324d03),LL(0x388c8e5b,0x75902ccc),LL(0x86e2e6b6,0x374484b9), LL(0x9f548719,0x38360f84),LL(0x41cbc7ef,0xdf999916),LL(0x5091ed27,0xa9cbe298),LL(0xc5f2cb5d,0xcf5a1440),LL(0xd413500d,0x5bdff729),LL(0x373f8b2d,0x55530d56), + LL(0x543c1255,0x0c62fd2d),LL(0xef361a27,0x71ea9c6f),LL(0xcef3f9e3,0x76b0933d),LL(0x9889ffa2,0x51b1ec2d),LL(0x9a3c88d2,0x9e84b2ba),LL(0x1913e52f,0xc8996b96), LL(0xcee43e36,0xbafc5e94),LL(0x70c658b7,0xd9898d24),LL(0xbed17108,0x4e9bcc41),LL(0x6c7a41c8,0x0db5b733),LL(0x795369cd,0xd4be07a7),LL(0x7bd3a934,0xb899f92f), +}, +/* digit=44 base_pwr=2^220 */ +{ + LL(0xde1e4e55,0x3f2eff53),LL(0xe4d3ecc4,0x6b749943),LL(0x0dde190d,0xaf10b18a),LL(0xa26b0409,0xf491b98d),LL(0xa2b1d944,0x66080782),LL(0x97e8c541,0x59277dc6), LL(0x006f18aa,0xfdbfc5f6),LL(0xfadd8be1,0x435d165b),LL(0x57645ef4,0x8e5d2638),LL(0xa0258363,0x31bcfda6),LL(0xd35d2503,0xf5330ab8),LL(0xc7cab285,0xb71369f0), + LL(0x14c5969b,0xf16938f5),LL(0x944b2271,0xde2e3cf0),LL(0x0b6490d6,0x2d509553),LL(0xa28a296a,0x8432fef1),LL(0x8d26415c,0x6f254dd0),LL(0xd50c2865,0x3780eead), LL(0x665b8794,0x4f5bc455),LL(0x56cb7018,0xef31fb9e),LL(0x65e59340,0xbab8dd6e),LL(0xa56dc2ea,0x676baca2),LL(0xeaa90e05,0x38eea06b),LL(0x174bada0,0x26e64224), + LL(0x40acc5a8,0xe6a19dcc),LL(0xdbc6dbf8,0x1c3a1ff1),LL(0xc6455613,0xb4d89b9f),LL(0xa7390d0e,0x6cb0fe44),LL(0x59ea135a,0xade197a4),LL(0x20680982,0xda6aa865), LL(0x5a442c1b,0x03db9be9),LL(0x2bfb93f2,0x221a2d73),LL(0x753c196c,0x44dee8d4),LL(0x0b7c6ff5,0x59adcc70),LL(0x4ca1b142,0xc6260ec2),LL(0x46cbd4f2,0x4c3cb5c6), + LL(0x2cb40964,0xa35e411c),LL(0xc331a3d6,0xdd7d4f4c),LL(0x89a66f2b,0x7c7c859e),LL(0x0def8ecd,0x9908c37e),LL(0x344947b7,0x8274124e),LL(0x568b0ce8,0x0d279f7b), LL(0x866091ec,0xe5291961),LL(0x3a08acc7,0xb056e3bf),LL(0x56bd3a7d,0x60fb39e1),LL(0x268f8562,0xe56a34d6),LL(0x13fd8293,0xb3a1fe16),LL(0x67537fcb,0x6a41e1a9), + LL(0xa417111f,0x8a15d6fe),LL(0x71d93fcc,0xfe4a16bd),LL(0x55bbe732,0x7a7ee38c),LL(0x1ff94a9d,0xeff146a5),LL(0xdd585ab5,0xe572d13e),LL(0x06491a5d,0xd879790e), LL(0x2a58cb2e,0x9c84e1c5),LL(0x6c938630,0xd79d1374),LL(0x385f06c7,0xdb12cd9b),LL(0x7a7759c3,0x0c93eb97),LL(0x683bd706,0xf1f5b0fe),LL(0x85ec3d50,0x541e4f72), + LL(0xc6bb5e5f,0x89e48d8b),LL(0x1ea95a10,0x0880ede0),LL(0x302c0daa,0x60f033d7),LL(0x048eefe3,0x15e4578a),LL(0xb0a72244,0xfd6dec89),LL(0x309489cd,0x1f7cd75e), LL(0xe9aba7fd,0x7cdcc2a0),LL(0xf28ba00f,0xd18dc5c7),LL(0x5812b55f,0xa6300a45),LL(0x2ca31d8c,0x8fa5c415),LL(0x4f3a5b5a,0x36aa3c23),LL(0xc86cf4e0,0xd128739e), + LL(0x81833608,0x9a0e1535),LL(0x6e2833ac,0x5cce871e),LL(0xfb29777c,0xc17059ea),LL(0xe354cafd,0x7e40e5fa),LL(0x4d07c371,0x9cf59405),LL(0xa71c3945,0x64ce36b2), LL(0x56caf487,0x69309e96),LL(0x1ae3454b,0x3d719e9f),LL(0xe25823b6,0xf2164070),LL(0x0bc27359,0xead851bd),LL(0xb0925094,0x3d21bfe8),LL(0x34a97f4e,0xa783b1e9), + LL(0x13575004,0xd9272830),LL(0x20b2275e,0x01a330d6),LL(0x450db713,0x58b9207f),LL(0x23e16d95,0xae953384),LL(0xe60e349c,0x4f10c6d4),LL(0xfeb122bc,0x541d03ec), LL(0x2c648211,0x22548cd2),LL(0xd01354f5,0x5c2dc84c),LL(0xb6167b3c,0xa1c6f912),LL(0x7902d2ba,0x6967bab2),LL(0x36de34ba,0xebbe0b08),LL(0x4b79625e,0x6985b33a), + LL(0x9546491a,0x406b0c26),LL(0xf293c4e5,0x9e5e15e2),LL(0x15b164db,0xc60d6413),LL(0x0c75a78e,0x0da46f53),LL(0xea0c656b,0x7c599bb7),LL(0x1b1a8122,0x0f07a512), LL(0x15172686,0x14c7204a),LL(0x5165625d,0x8faedff8),LL(0x37aede40,0x20f260ce),LL(0x8f357ffe,0xc81f771e),LL(0xb0912557,0x25499197),LL(0x4c739c74,0x736197dc), + LL(0xaefdd5e1,0xf2d015bd),LL(0x4bf9edae,0x33b4e21c),LL(0x64b35fcc,0x4860aadc),LL(0xf13f8112,0x77b657e9),LL(0x90dc84fe,0x65f28625),LL(0xd66be036,0x4eabfd22), LL(0x6ff05dd0,0xb0213123),LL(0xa9fe5d11,0xa104d4aa),LL(0x9b8ae390,0xdc7efe3b),LL(0xeb87fb5c,0x46918b54),LL(0x72b7172a,0xf5771d71),LL(0x13587bf0,0x41e49e47), + LL(0x381b3462,0x6151bab1),LL(0x43dbd344,0x27e5a078),LL(0xa1c3e9fb,0x2cb05bd6),LL(0x27cf2a11,0x2a759760),LL(0xff43e702,0x0adcf9db),LL(0x1f484146,0x4bbf03e2), LL(0x55b6521a,0x0e74997f),LL(0xade17086,0x15629231),LL(0x7493fc58,0x7f143e86),LL(0xaf8b9670,0x60869095),LL(0x7e524869,0x482cfcd7),LL(0x1d454756,0x9e8060c3), + LL(0x9004b845,0x3ec55ddc),LL(0xbe7b06eb,0x5101127c),LL(0xfc4176e9,0xdddafd57),LL(0xe4a31ddc,0xd8cb31c0),LL(0xd42feabb,0x94e83a89),LL(0x2f74ec68,0xd4401def), LL(0x5adb654d,0x9c9defb6),LL(0x5053eeda,0x9a3513f0),LL(0x7cdb455e,0xceab2dec),LL(0x59d24f59,0x99542808),LL(0x31d30504,0x22ead452),LL(0x0521a229,0xe9df48f9), + LL(0xc88b4d3b,0xe495747a),LL(0xae8a948f,0xb7559835),LL(0xdeb56853,0x67eef3a9),LL(0x9dee5adf,0x0e20e269),LL(0x61f0a1aa,0x9031af67),LL(0x683402bc,0x76669d32), LL(0x06718b16,0x90bd2313),LL(0x864efdac,0xe1b22a21),LL(0x6620089f,0xe4ffe909),LL(0x3428e2d9,0xb84c842e),LL(0xfe3871fc,0x0e28c880),LL(0x3f21c200,0x8932f698), + LL(0x441d0806,0x36dd28ea),LL(0x21518207,0x6680c72c),LL(0x0a484dbc,0xc5d40e28),LL(0xa3a2ba6e,0xdb1170c6),LL(0x40a91c7d,0x07290fd1),LL(0x95ee9ca2,0xdd125716), LL(0x07876188,0x595dad63),LL(0x499d4827,0x6fcf18c0),LL(0x206e617d,0xdcd946a3),LL(0xe7bceaef,0x6cf08f51),LL(0xb19a06ac,0x7a85c02d),LL(0x7140a7df,0xf1365fc6), + LL(0x6c90ea5d,0x603f00ce),LL(0x40a2f693,0x64739307),LL(0x2174e517,0xaf65148b),LL(0xf784ae74,0x162fc2ca),LL(0x4d5f6458,0x0d9a8825),LL(0x43aace93,0x0c2d5861), LL(0x9f73cbfc,0xbf1eadde),LL(0x9c68bbca,0xde9c34c0),LL(0x67ef8a1a,0x6d95602d),LL(0xa791b241,0x0af2581b),LL(0x12cad604,0x14f77361),LL(0xe2acd1ad,0x19f2354d), + LL(0x771560ab,0x9e857f33),LL(0x250f109d,0x4ae1ba22),LL(0xff4f6566,0xf8538d68),LL(0xac339148,0x35380f15),LL(0x5ddfc12f,0xfef0bfdd),LL(0x1387d93e,0xf706c6bf), LL(0x5357e131,0x618ce77d),LL(0x236478c4,0xf0921744),LL(0x00dc0da5,0x24eaf06e),LL(0x07603cc7,0x049113be),LL(0x8f6963c7,0x5cf48908),LL(0xede4a300,0xbe5eb9e6), +}, +/* digit=45 base_pwr=2^225 */ +{ + LL(0x5d066c15,0x77e486f8),LL(0x4ed5307d,0x0c05b6c2),LL(0x7df36628,0x322b28ab),LL(0x6704dcd6,0x2d14d131),LL(0xf29a3567,0xd359977a),LL(0xec96d3b6,0xc29bb132), LL(0xe6bfa701,0xfd6e400a),LL(0x4c7e5101,0x03db9924),LL(0x9b8533af,0x62d81c7d),LL(0x8de66eb8,0xefa638c2),LL(0xe86784ee,0x7405a9d7),LL(0xa6c22223,0xafaa74ef), + LL(0xb9d36e91,0xf9b2dba4),LL(0xfda9b2c4,0x5fb4f6ce),LL(0x3b8104ee,0x7692a4f3),LL(0xe4e1896e,0x5da885b0),LL(0x73d2aa36,0xc2a30fec),LL(0x86f60bca,0x7d06e6af), LL(0x87287887,0xbc8bf16d),LL(0x3d701bec,0x6c3dd86a),LL(0x7e35610a,0x8e79e2f3),LL(0x82f9d71c,0x981139f4),LL(0x24e62733,0xf8997ec4),LL(0xa3518061,0x330d989a), + LL(0xca89fbad,0x4e6ef410),LL(0x53933b78,0xe0fc53ba),LL(0xfd41d143,0xa4f03403),LL(0xe0774c37,0x3a507177),LL(0x8ec7484a,0x078e8c56),LL(0xfbb3f66b,0xfb73c6b6), LL(0x3bfbdff6,0x169c9475),LL(0x0a232243,0x44d28606),LL(0x08303114,0x3e8e9685),LL(0xfad0def2,0x7a9797b8),LL(0xefc1c8da,0x0ad14404),LL(0x21ced721,0x6daae4e9), + LL(0x88dd2dc2,0xc7e9ddef),LL(0x19a0c0b5,0x2c21a998),LL(0xb239bb82,0x6bc0746d),LL(0x28ea1341,0xc811a8eb),LL(0x1d1309b0,0x5f714ca7),LL(0xd4eb9b34,0x79eabd20), LL(0xdf0fb30f,0xe0e5afdc),LL(0x8c0814c6,0x1b01a16d),LL(0x84334366,0x670e1e7b),LL(0x0eed1116,0xc8c38f9a),LL(0x619bbd50,0xf914fae2),LL(0x51c1995a,0x1ed062cb), + LL(0xcb583422,0xd4e60e15),LL(0x320f296f,0xc6b1ef90),LL(0xd9bfc834,0x0714bad0),LL(0x9050e2c2,0x5ee2ca8c),LL(0x24f7cf1d,0x074a8ca8),LL(0x10df8516,0xb9750249), LL(0xc2636d2c,0xecee8ab7),LL(0x3b4b7bbd,0x308e5af1),LL(0xee2ae021,0xfed4f27e),LL(0x2065253b,0x7cd4bb19),LL(0x4de525b4,0x6b21a3f8),LL(0xac27fddb,0x0f10e7bd), + LL(0x870e29cb,0xd5068487),LL(0xfc52d5cb,0xf9420b85),LL(0x496d000d,0x50c3265a),LL(0x166bd6b4,0xe605414a),LL(0xc62b2a6c,0x4de8d724),LL(0xa1a11048,0x16af06f2), LL(0x45f43c4c,0x5406bde9),LL(0x751ad18e,0x5e15bf6c),LL(0xb6a59587,0xa846e665),LL(0x1816ac55,0xcdb28a7d),LL(0x819b73f8,0x899b3551),LL(0xbc848d08,0x2d46297b), + LL(0x299127be,0xdc4cc720),LL(0xfaab8165,0x5b34e762),LL(0xb39c120d,0x2289b2f7),LL(0x6e52b913,0x687a78d0),LL(0x2a3ea6a5,0xd2a091dd),LL(0x38eab329,0xc61eced6), LL(0x7887ff2b,0x652231ea),LL(0x0479db4e,0x77a56875),LL(0xd43c5722,0x1ef471c8),LL(0xf3764c34,0xf82bf436),LL(0x0445cafe,0x962af405),LL(0x5ff47259,0xed8b227f), + LL(0xd89594ab,0xde849cd1),LL(0x0ec4fb3a,0x00e2d2b1),LL(0xabe92fba,0x3fbd9e3d),LL(0x3324900a,0x785414d4),LL(0xde20904e,0xdaead1ab),LL(0xaa5f1ba8,0xb493e121), LL(0x6eaea0dc,0xd60a4f2d),LL(0x6fca8596,0x394746b5),LL(0x34efa243,0x163dc789),LL(0x216a8d8c,0x3067dccf),LL(0xa901617b,0x116b6534),LL(0xbbabe51e,0x8c4bd099), + LL(0xac3a082c,0xc8c2df45),LL(0xc8d4c40e,0xc353d074),LL(0x5a3c2de7,0xb214f9c0),LL(0xf86b0214,0x504bc42c),LL(0xd1922a58,0xc82df5cb),LL(0xa5bc3267,0x40887948), LL(0x88ba8bb2,0x04bcd217),LL(0x046fd401,0xe21b3e7f),LL(0x616af5cf,0x8419c338),LL(0xaedfce9d,0x7f24760b),LL(0xddbd519a,0xded8035b),LL(0x1693faab,0x1f1fb0d7), + LL(0xd02ffcf6,0xbb067b49),LL(0x3e657299,0x7cedf8f9),LL(0x406bbfe3,0xc3829961),LL(0x37c12472,0xefe4b5aa),LL(0xfec7dee8,0x7dc01cf9),LL(0x89472f50,0x70a9db23), LL(0xb31bf737,0x29c269f8),LL(0xae3fa7db,0xa26deac3),LL(0x33caca41,0x0046e912),LL(0xb6e78b55,0x3bf4bc8a),LL(0xd9eb5ef1,0xca83bc6c),LL(0xc0c5deff,0x73f25c62), + LL(0x44b4aae8,0x697dc47d),LL(0x782c331c,0xb3525cc0),LL(0x0bd7c78c,0xff71cca4),LL(0x10c0ab69,0x5f3d7766),LL(0xe2ba07e3,0xbdc10267),LL(0xe6373f6e,0xc656f75c), LL(0xb5607b62,0x9e2938b4),LL(0x10b0a0f7,0xa65017d4),LL(0x5cc6ac25,0x8dad3119),LL(0x8ba5d1e6,0x00f8f2d1),LL(0x43305aef,0x608137bc),LL(0xdcb81cb1,0xddad34bb), + LL(0xb1f82ca5,0xe133d941),LL(0xfdf115bf,0x2af8b98b),LL(0x57aaa6f3,0xdc6179c8),LL(0x130ade06,0xabaa83e9),LL(0x0e8bffd1,0x7836b6fb),LL(0xfa103703,0xc479751f), LL(0x9c89963b,0x0ff3c129),LL(0x0b84c24f,0xe6407256),LL(0xf34f6bc9,0xa92a4ea2),LL(0x3197989b,0xba45b305),LL(0x99243aab,0xd12b5a01),LL(0x442af625,0x3015772c), + LL(0xe6f065c5,0xd95fca81),LL(0xfc8655de,0x45e886d5),LL(0x27cff79e,0x35809577),LL(0x625877d9,0x92a39a34),LL(0xdfee17ee,0xdda02684),LL(0x986f635b,0x6354f871), LL(0xd409c182,0xb3a6e9ed),LL(0xc4fbbb3a,0xf0b1c8d9),LL(0x9b77aded,0x28721c01),LL(0xbf94f028,0x3c356df1),LL(0x29a81f1a,0xff221bd2),LL(0x56b20b0d,0x20edf2e8), + LL(0x835fda9e,0x97fff124),LL(0x0bc68512,0xa79ceb2f),LL(0xa2fc3995,0x70ba93d1),LL(0x9e51c5ee,0x62bd28ab),LL(0xd5bbbaa9,0xb95fa624),LL(0x8c1f571e,0x0654dc45), LL(0x65a45ed6,0xb9a4edc6),LL(0x21ad0612,0xbf5ed1bc),LL(0xb1a3551b,0x74adc1a1),LL(0xdbbd6cef,0x3dfa3dc8),LL(0x2fa3afd2,0xce5dd40b),LL(0x30a746ca,0x14894e0f), + LL(0xb8ca2a2c,0x7e729c58),LL(0xcaac04af,0x0f32ea1e),LL(0xbdd549e3,0x47267f13),LL(0x90be3b50,0x35b94406),LL(0x4b27f670,0xad0f2bb1),LL(0x92341803,0xd7e5874e), LL(0x1f9ec462,0x7dc841cf),LL(0x512b2a42,0xebeff994),LL(0x320dc858,0x22998a7f),LL(0x19946f59,0xf08eb5c7),LL(0xa68ea75e,0x228c8dcd),LL(0x7b20dee5,0x40dc6dc3), + LL(0xb3952db4,0x929454f6),LL(0x4d3f69f5,0x412142ec),LL(0xee25c0b0,0xf5b0a7c5),LL(0x2e752295,0x7d3372ff),LL(0x6eacac68,0xd6dadc7d),LL(0xa96a8e3c,0x5f0076cc), LL(0x71725b3a,0xea831db6),LL(0xc29ab454,0x4a286c89),LL(0x72e3c00c,0x5ff817e5),LL(0x2a5fb6ba,0xb022e25d),LL(0xbb392476,0xb611c5bc),LL(0x190485a0,0x062c14dc), +}, +/* digit=46 base_pwr=2^230 */ +{ + LL(0xc419b0aa,0x0372678d),LL(0xc13fdf17,0xf95031d8),LL(0xb79594c3,0xebaebca4),LL(0xaf3b75cf,0xe587850b),LL(0x2c1e09c6,0x534183ac),LL(0xc08204cd,0x3f5b0bfd), LL(0xe297cc77,0xdac2cf06),LL(0xd0487084,0x5e47d9c6),LL(0x90b0f6c2,0xf6f509f4),LL(0xc2c62207,0x3ffc3cd6),LL(0x32ff1887,0xbb21eb11),LL(0xe62ccc6f,0x2116a023), + LL(0x16960728,0x406a7e21),LL(0x5597d8c4,0xd03923f8),LL(0x020748ee,0xd4402eff),LL(0xf39b58db,0x7827442a),LL(0x8d8cfb04,0x77e3f276),LL(0xe45a978f,0xf6eb49c8), LL(0x49247f6a,0x9db08299),LL(0x06669fe5,0xce71a747),LL(0xb82775f5,0xe434ce47),LL(0x63910016,0xe84995ef),LL(0x1e47792f,0xa35e8b97),LL(0x7c6aaeb9,0xc779cb3d), + LL(0xff381db7,0x66428800),LL(0x55574ac6,0xa9b9d019),LL(0xbdf4a86c,0x30cdc21f),LL(0x741c4a26,0x2ec38d35),LL(0x0b6be057,0x35496c23),LL(0x01656b1f,0xaecc67e6), LL(0xf7d70324,0x781af00f),LL(0x5d7ee71e,0xac0e6579),LL(0xa6b14e3a,0x60a35c6c),LL(0x0e6c1c3a,0xacd6813b),LL(0x1faeef73,0xd7f77024),LL(0x23eddf05,0xd2254b8f), + LL(0xaf31ea1d,0x1fa064cf),LL(0x48e8d974,0x2a9547a8),LL(0xfa9d9453,0xda8102a1),LL(0xdc6bd7ea,0x786aecab),LL(0xca2f6044,0xcaf91e3b),LL(0x8573f208,0x67d86ea7), LL(0xc505ae24,0xd309fce9),LL(0x7f86eb8e,0x67ddc5b1),LL(0xf3d53056,0x57791ae0),LL(0x0d1fd61e,0x26b053f0),LL(0x045ebfa6,0x91c962c0),LL(0x076ed979,0xe95246de), + LL(0x5f6e9ea4,0xc49c9989),LL(0xe16ec8e4,0x4a91578b),LL(0x0aeb5ac5,0xa1c54e89),LL(0xee09b9a0,0xa9094b07),LL(0x09a74b27,0x3587752f),LL(0x44bbfed0,0x973bf8b0), LL(0x5636a52e,0x91c26f23),LL(0xeb7e3b41,0x8ac948b8),LL(0xfc457d56,0x14234675),LL(0xc76398a3,0xde98e4fa),LL(0x0f4a46e0,0xa80f7311),LL(0x22b66fa3,0xcba089bd), + LL(0x0746d174,0x156eaf57),LL(0xcda35250,0xa2d4a83d),LL(0x0290fa02,0x60a9f48c),LL(0x5c33b4ac,0x9855d26d),LL(0x97eb1c30,0x06e379c6),LL(0x6e219664,0x4f2e2dbe), LL(0x29006065,0x6b7448f8),LL(0x115062a9,0x237a1f31),LL(0xad92cb24,0x5c635a90),LL(0x2eed977e,0x2e857f8c),LL(0x856dc88a,0x3d512df7),LL(0xe597a27b,0xbde85263), + LL(0x10a98e42,0x0b114aea),LL(0xae19dd14,0x6133aa52),LL(0xa99eb2c5,0x0c235df2),LL(0x7f59582c,0x0085a619),LL(0x8cf7feba,0xf9002bba),LL(0xaf6a3261,0x275742d7), LL(0x249e8e9e,0x302b4823),LL(0xce696f91,0xa142aba9),LL(0x64c37b14,0xdeb28c44),LL(0x0766002f,0x14bb8f23),LL(0xc52fe891,0xafeff88b),LL(0x46faeb2d,0xb9d493ae), + LL(0xe7c03ce4,0x49f24994),LL(0x2aed9ba3,0x274a8c13),LL(0xd5e91bc0,0x897b9103),LL(0xcb404f68,0x63db1efb),LL(0x42f7fc02,0x70efd9d8),LL(0xc6a230af,0xd6e02921), LL(0x11ae0a56,0x8d5b199f),LL(0xce33da6a,0xc98287de),LL(0x504dd889,0xde583d34),LL(0xf823686f,0x03756001),LL(0x95fc73dc,0xf19ab86f),LL(0x93f12f42,0x300406c6), + LL(0xa427d89c,0x68fdb78a),LL(0xa3944c0d,0x84e9ba49),LL(0xc1833422,0x1c3569b1),LL(0x1a01f4b9,0x30773fc0),LL(0x7da01321,0x18b8f17d),LL(0x7198c85e,0x8370fb0f), LL(0x99a898e3,0xda12c8d4),LL(0x8ba82ded,0x7667b46a),LL(0x77e1e31f,0x2aab259b),LL(0xbe71c9bc,0xd03f7708),LL(0x8e43eb38,0x9f784cce),LL(0x21c1208c,0x7ddedc8b), + LL(0x7759701c,0x2f73595f),LL(0x6fe0e0d9,0x8dc2069a),LL(0xc286a65d,0xb7de7114),LL(0x84c0e487,0xfecc429e),LL(0x14344c07,0x51061a2c),LL(0x96869e37,0x4d709725), LL(0x2be9403b,0x8b02781f),LL(0xde3ab5d9,0x6cb6aa02),LL(0xff6bdc9a,0xb013508e),LL(0xe5438c58,0x568d2e84),LL(0xe4206c3b,0x7b35a979),LL(0xb17a8bc7,0x0bb793c0), + LL(0x57ed2360,0x41248052),LL(0x6ba9bd95,0xfc0cb1b6),LL(0x2337a8fd,0x342f16db),LL(0x88099007,0xe1417411),LL(0x8cd74752,0xc96c29ee),LL(0xedf5fb4a,0x376047cc), LL(0x439546c1,0x5f40ce08),LL(0x1a235de8,0x14d2c666),LL(0x98e355f0,0x9b66892d),LL(0xa4bb19e9,0x8a65f6dd),LL(0x046a2581,0xf72848f8),LL(0x8373b2b6,0xfed74b3b), + LL(0xfa4dd561,0x3f896ca9),LL(0xd2de2ecb,0x4b9a98ab),LL(0x600e4e2a,0xd0741632),LL(0x69e702d5,0x87c7db5f),LL(0x53e0df2b,0x1f5a3b80),LL(0xf443dfba,0xe1e24b49), LL(0x5eef3a1d,0xeb90e230),LL(0xd38f73fc,0x8f3fc8a6),LL(0xa5aa335a,0xfb1e8299),LL(0x4197b32a,0xd78504cd),LL(0x6755918e,0x0e7a79cc),LL(0x883b1c72,0xc7c98ae2), + LL(0x969088c1,0x4b74fcbd),LL(0x361a8c96,0x4d16f895),LL(0x760d61fa,0xefcb6ced),LL(0xcc3e8808,0x3f14a7cb),LL(0x664ea335,0x51f5fd2c),LL(0x3a65d305,0xe0cad090), LL(0x031a6911,0x86409de9),LL(0xe5f9715b,0x23ea4aed),LL(0x1f3532c6,0x6e5b8cea),LL(0x11271ead,0x33fc873e),LL(0x5b8131d7,0x842b59a6),LL(0x61b7bf60,0xbd95818a), + LL(0x03d2becc,0x1a12727c),LL(0xc6741372,0x810a37df),LL(0xb7049f39,0x44ac483f),LL(0xa36fc614,0xab73e5e7),LL(0xeeff8aeb,0x298d453f),LL(0x7e1b586b,0x2127dd16), LL(0xe07bd60c,0xeadc5c54),LL(0xf5e2d2e2,0x67cdae00),LL(0xc9d2f10a,0x03fe0446),LL(0x95e38ed2,0x07840987),LL(0xe1a6306e,0x5d348a7c),LL(0x562f5463,0x4903f1b6), + LL(0x4a3862e8,0x906ab8a8),LL(0xb2f5c878,0x8fc76114),LL(0x2035287f,0x2dac3952),LL(0x18af4378,0xaa8372f3),LL(0xdbf64476,0x915050c9),LL(0xe992d0c8,0x896f734d), LL(0x3a35846c,0x5c3e36da),LL(0xac8f4fc3,0xfe774b4a),LL(0xaadd8a59,0x66347050),LL(0x2cd12be8,0xea94ebda),LL(0xdab94de0,0x45b1e7e2),LL(0x264b508c,0x539d580e), + LL(0xb44b1d0c,0xbf66baa5),LL(0xa44f8eda,0xbbed18ed),LL(0xeaaa466c,0x80bc32ab),LL(0xe5f2733b,0x605b7897),LL(0xa2531afa,0xe9e7e3a1),LL(0x3deb8369,0x25d66db3), LL(0xb2f25d10,0x36212ea3),LL(0xa08d303e,0x52d6b3f4),LL(0x444e9e9f,0xefa54b31),LL(0x69530c1b,0x9c2229a1),LL(0x4b79bdd1,0x68feb985),LL(0x8b984cc3,0xd570e84f), +}, +/* digit=47 base_pwr=2^235 */ +{ + LL(0x845f26e4,0xd6a4d25c),LL(0x1b039dff,0x71e554ce),LL(0x1cdedfc0,0x94205973),LL(0x03d6502f,0x0c4e3856),LL(0xe15ce8c8,0x981a4fc5),LL(0x7aca30b7,0x85d1b0f1), LL(0x77bb9e43,0xf2037ef7),LL(0xe87ae187,0xc52804f4),LL(0x71f3e4e3,0x9c98a23c),LL(0xf47b504b,0xa73c8b89),LL(0x023233aa,0xb9e33f54),LL(0xf92c9f68,0xf2bcfc17), + LL(0x7b3b336d,0xba03ba3b),LL(0x28c9c55d,0xe57ce509),LL(0x4f0f60b2,0xf96b8cfe),LL(0x6fcccd96,0xb908d77e),LL(0xe79dd17a,0x7208ef7d),LL(0x3ec3d048,0x73909533), LL(0x1163fe78,0x9c5ad2da),LL(0xcd4a15c2,0x4e2a8685),LL(0x470eb938,0xac999449),LL(0xee7d772f,0xfaaf27fb),LL(0xd0b7ad09,0xfbe402ab),LL(0x57db00a9,0x704d4f0e), + LL(0xe12b4e64,0xe93ee31a),LL(0x662d17f4,0x2ab8e378),LL(0x69516582,0x2544bd99),LL(0x2e1e5485,0x7bf80e4b),LL(0x729d9361,0xf30f0b14),LL(0x8268d40f,0xb3ffb5d1), LL(0xac193a63,0x34605055),LL(0xf8e04d69,0x9e5ca9a9),LL(0x085ecbb2,0xcbbeebc1),LL(0xf340eac4,0xda03b75b),LL(0x84436462,0x3bf9468a),LL(0x0f26f20c,0xdfa8b4c8), + LL(0x3fc14a85,0x10c082a6),LL(0x1c0b14c4,0x59389ebc),LL(0x4cb291a7,0x785d935b),LL(0x13e9ce08,0xfc2ae153),LL(0x4df6f1c4,0x3146fabf),LL(0xc87dd24c,0xa2a4a457), LL(0x1deb49bb,0x85fdd877),LL(0x9b055934,0x2b784370),LL(0x3e7e0297,0xc81d0501),LL(0xb92df904,0xb56ddd1f),LL(0x295ddccb,0x4612df9f),LL(0x0e27cf1d,0xc24bd4cf), + LL(0x422ea2c0,0x5564875b),LL(0x8285b03f,0xabc2e7de),LL(0x733e9f1f,0xfd662091),LL(0x68465da3,0x68f16745),LL(0x38fa6f63,0x965a0a05),LL(0xdfae710d,0x0ed70fee), LL(0x153b24da,0x56c6227f),LL(0x01470f6e,0xf1dcf574),LL(0xf51771cf,0x9992caa8),LL(0x14a9b029,0xa884b481),LL(0x7b9a4062,0xad11cbaa),LL(0xe55533e0,0x60ec99d8), + LL(0x18674f49,0x86d9b060),LL(0x97bb1a7d,0x81b06486),LL(0x27d9d64c,0x9b6e8e7c),LL(0xba04e6ad,0x79bd66ba),LL(0x828abbbe,0x77e4d0b0),LL(0xcc540d04,0xae7548ac), LL(0x869cbebf,0xf5d8a46f),LL(0x38a6cc83,0x99fb1a63),LL(0x563fe6b4,0xb93bb852),LL(0x97cdd04f,0x06bb3ae9),LL(0x8b7de47d,0x07f011d4),LL(0xde78f61c,0x8d90e2e4), + LL(0xd33cb6e2,0x731c6dd6),LL(0x8cce0290,0xa3ea317e),LL(0x1c42206e,0xdca9b2f9),LL(0xd1e5dfd9,0x6acbbce9),LL(0x2fc948cd,0x40745846),LL(0xa82f9cec,0xc7a50d91), LL(0x4c1aa161,0xb906d69e),LL(0x0ebe948e,0x3a9b14be),LL(0xb63aeb70,0x11a9f12c),LL(0xead745f2,0x0365b4cc),LL(0xf9f16c17,0x5f6c2bbd),LL(0xa03e558e,0x89131238), + LL(0xd1944d1b,0x7830460f),LL(0x84350af2,0xc56f08e7),LL(0x307d9c78,0x73bee2aa),LL(0x5aad8b6c,0x1b02af1b),LL(0x03848db5,0x5e318827),LL(0xf230f476,0x4785958b), LL(0x4f80e25d,0x4ea6535d),LL(0xd23c7f72,0x9958c9c7),LL(0x2fd33cab,0x4c197b33),LL(0xc566914f,0x24c7b0b1),LL(0x71952d3b,0x956ce3c3),LL(0xfabae5f2,0x8735694b), + LL(0x40ec913c,0xde37ae28),LL(0x056685b3,0xdc915f83),LL(0xf66a4501,0xf7bc3488),LL(0x6a900e5f,0x30e61042),LL(0xca3cf645,0x505525c1),LL(0xb1f3ed40,0x35338c53), LL(0xd70b7c41,0x6823159a),LL(0x7384ba7c,0x660f518a),LL(0x2482056b,0xc6cf6a4b),LL(0x1df15990,0xb308b215),LL(0xfb5c130b,0xba63b2e3),LL(0x277b7515,0x1c660db8), + LL(0xfddc9fdd,0x8a95e5c1),LL(0x5adab0c3,0x679d4e0f),LL(0xcda40bc0,0x1859df6a),LL(0x8234471d,0xf9097aa5),LL(0x783c0100,0xaddc0c9b),LL(0xfeb7067c,0x55388dc8), LL(0x80a2eac7,0xe3805fd1),LL(0xec886879,0xf800a75c),LL(0x1943a0a0,0xa4599992),LL(0x1dfe627e,0xb47f0619),LL(0xda06515f,0x313d4f09),LL(0x1f54a73e,0xde26052d), + LL(0x6827365d,0x85a1b879),LL(0x667debe8,0x595a6915),LL(0x93a3d50f,0x214670fb),LL(0xb37de08e,0x9dfb028e),LL(0x9c6cf2a9,0xdd077e2f),LL(0xc9b96e8e,0x96897d8f), LL(0xefd39543,0xe6e93c07),LL(0x4454e73d,0x19dcdaa4),LL(0xd7b8c758,0x4a67424f),LL(0xb1e91e2a,0x03d4de0d),LL(0xe887b6b6,0x7c843988),LL(0x6ffdcfff,0x7db4f3da), + LL(0xd8bb43ec,0x0db87bc3),LL(0x13a7b669,0xbf71d27c),LL(0xa3fd2a60,0xea81e9c4),LL(0xc9f017e5,0x190c9c71),LL(0xbcc75768,0x21864180),LL(0x43dbcde6,0x137bd615), LL(0x5d468ff5,0x5b090c71),LL(0x126c6bc6,0x3a622b60),LL(0xa918bf24,0xfef3d268),LL(0xae204f49,0xe10c52c8),LL(0x86d7c356,0xf4be898d),LL(0x2fdb5a17,0x8276da18), + LL(0x7f2ad562,0x89c992af),LL(0xafc83ad3,0x7e1459d9),LL(0x9278dd04,0x4c0d9681),LL(0xd8eebe36,0x4496d9a7),LL(0xb8d4b1a2,0x7c037261),LL(0xbad3d6d7,0x827c49a5), LL(0x836926ae,0xf4d94deb),LL(0x4064af58,0x65417bf6),LL(0xfcdafc9f,0xa79471ac),LL(0x8123312d,0x3f85ccb4),LL(0xa3360be4,0x4d374cad),LL(0xee325a2b,0x56b476d6), + LL(0x41af8c08,0x0079c69d),LL(0xacbe515e,0x7dcfa4f4),LL(0x01396859,0xb8d18666),LL(0x946fbedd,0x08590ca4),LL(0x641aace4,0x7fecd9b9),LL(0x2936a1b6,0xaad5cc44), LL(0xf92c5958,0x925b6235),LL(0x82d6231e,0x7b1442f1),LL(0x8c6fb34a,0x971e663d),LL(0x2fc1c10d,0x543146dc),LL(0x0642b822,0x6e4053c7),LL(0x492e524d,0x4a49f247), + LL(0x17ba53f1,0x1b51f7b4),LL(0x3d5c43bf,0x170ff1eb),LL(0x681f7ee7,0xc2f160f8),LL(0x47814310,0x4c0a54d0),LL(0xa83d061c,0xfc689a13),LL(0x7ff6333d,0x1cbc99b4), LL(0x6581cd16,0xe19fd790),LL(0x9ca37b0c,0x67da79c7),LL(0x63bd0b5c,0x2507d167),LL(0x1befb82b,0x4449985b),LL(0x914699ec,0x6bea3969),LL(0xef202abb,0x9f606dd4), + LL(0x597bd10e,0xca9872e1),LL(0x4aed951f,0x6725cc9a),LL(0x4e05b280,0x96b17cb8),LL(0xfa234d45,0x97987146),LL(0xbb35a7d8,0xba78949e),LL(0x6fc59384,0xb82e9b9f), LL(0x70f165c7,0xa303e54a),LL(0xb9c2cad9,0xfd6bb0dc),LL(0xee722045,0xe57e2de8),LL(0x63e27035,0xa05c1065),LL(0x02d2fe6f,0xaa38e866),LL(0xee2f6aad,0x78e02fa8), +}, +/* digit=48 base_pwr=2^240 */ +{ + LL(0x5e3c647b,0xc0426b77),LL(0x8cf05348,0xbfcbd939),LL(0x172c0d3d,0x31d312e3),LL(0xee754737,0x5f49fde6),LL(0x6da7ee61,0x895530f0),LL(0xe8b3a5fb,0xcf281b0a), LL(0x41b8a543,0xfd149735),LL(0x3080dd30,0x41a625a7),LL(0x653908cf,0xe2baae07),LL(0xba02a278,0xc3d01436),LL(0x7b21b8f8,0xa0d0222e),LL(0xd7ec1297,0xfdc270e9), + LL(0xe2a07891,0x4f120aa7),LL(0xa25d3225,0x9158bab3),LL(0xcfe5f7a8,0xc96bac5e),LL(0xbbf3cec6,0xd4e73d59),LL(0x60361cd5,0xed8d2335),LL(0x562f444c,0x9b1a252c), LL(0xc70f23c2,0xbd37d3cf),LL(0xa52ea19e,0xf13b3b6e),LL(0x3d2f41ed,0x7e35535a),LL(0xe8b1743e,0x0353b52e),LL(0x7b5a2765,0x31d89dfd),LL(0x8d9ea8b8,0x2b7ac684), + LL(0x9f101e64,0x06a67bd2),LL(0xe1733a4a,0xcb6e0ac7),LL(0x97bc62d2,0xee0b5d51),LL(0x24c51874,0x52b17039),LL(0x82a1a0d5,0xfed1f423),LL(0xdb6270ac,0x55d90569), LL(0x5d73d533,0x36be4a9c),LL(0x976ed4d5,0xbe9266d6),LL(0xb8f8074b,0xc17436d3),LL(0x718545c6,0x3bb4d399),LL(0x5c757d21,0x8e1ea355),LL(0x8c474366,0xf7edbc97), + LL(0xc46db855,0x73457010),LL(0xdd579fb8,0xccb68c43),LL(0x9c25fe5b,0x705b0e8c),LL(0x82dd0485,0x40f36ea1),LL(0x27ac2805,0x3d55bc85),LL(0xad921b92,0x15177c6f), LL(0x5ab18cab,0x51586cd5),LL(0xcbb4488c,0xf51b5296),LL(0x84f0abca,0xbb4e605e),LL(0x772dd0da,0x354ef8e3),LL(0x5e4e1d41,0x7f1a8f79),LL(0xde5d8491,0x93461f09), + LL(0x6ea83242,0xec72c650),LL(0x1b2d237f,0xf7de7be5),LL(0x1819efb0,0x3c5e2200),LL(0x8cdde870,0xdf5ab6d6),LL(0x92a87aee,0x75a44e9d),LL(0xbcf77f19,0xbddc46f4), LL(0x669b674d,0x8191efbd),LL(0xed71768f,0x52884df9),LL(0x65cf242c,0xe62be582),LL(0x80b1d17b,0xae99a3b1),LL(0x92de59a9,0x48cbb446),LL(0x2dcb3ce2,0xd3c226cf), + LL(0x9311182c,0xf3899558),LL(0xb657a7b7,0x1bee4c4b),LL(0x2df8d1a7,0x0b1c4fd3),LL(0x76d3fbbf,0xf16bcc23),LL(0xf4fd52bc,0xd5888916),LL(0xd5cde1f0,0x3de6cfb4), LL(0xd4a07dfd,0x764ffffd),LL(0xe2642182,0x5e674426),LL(0xccd57b85,0x34f64762),LL(0x29351062,0x2233a4c3),LL(0xd9c642f3,0xdf076095),LL(0x59f0df34,0xac917a2c), + LL(0x9fd94ec4,0x9580cdfb),LL(0x28631ad9,0xed273a6c),LL(0xc327f3e7,0x5d3d5f77),LL(0x35353c5f,0x05d5339c),LL(0x5c258eb1,0xc56fb5fe),LL(0xedce1f79,0xeff8425e), LL(0xcf83cf9c,0xab7aa141),LL(0x207d6d4f,0xbd2a690a),LL(0x458d9e52,0xe1241491),LL(0xaa7f0f31,0xdd2448cc),LL(0xf0fda7ab,0xec58d3c7),LL(0xc91bba4d,0x7b6e122d), + LL(0x775f516f,0x3bd258d8),LL(0xc715927f,0x4bedebd5),LL(0xe3f966a0,0x5b432512),LL(0x709d0c2d,0x338bfca7),LL(0x49658259,0xd142cc10),LL(0x636b8023,0xfabc6138), LL(0x4d4ef14d,0xa9ef9401),LL(0xc54c570c,0xd5917ac1),LL(0x5cb64487,0xfd2f63c5),LL(0x1cea475b,0xbae949b1),LL(0x1e67a25f,0xa4544603),LL(0xdc6a7a6a,0xa547abc1), + LL(0xb1b48156,0x2a2dedaf),LL(0xbb93db87,0xa0a2c63a),LL(0x08acd99e,0xc6559078),LL(0xfe4ac331,0x03ea42af),LL(0xeb180ed6,0x43d2c14a),LL(0xb1156a1a,0xc2f293dd), LL(0xa9d81249,0x1fafabf5),LL(0x9a8eee87,0x39addead),LL(0x119e2e92,0x21e206f2),LL(0xd74dceb6,0xbc5dcc2e),LL(0x0a73a358,0x86647fa3),LL(0x2f53f642,0xead8bea4), + LL(0xb12c4bb1,0x158d814d),LL(0x2f0cf4fa,0xe52f75d2),LL(0x6141b59c,0xf106023e),LL(0xbeb9d941,0x5eb8b8eb),LL(0x90cf579c,0x1dd39729),LL(0x69ee6efa,0xb273252e), LL(0x3e9947a0,0xe43a3c59),LL(0x6c19dd01,0xd605124f),LL(0x05c578b0,0x8090fdbd),LL(0x622ff18c,0x8e6c535a),LL(0x57d12071,0x3600b0c2),LL(0x78d001d7,0x6d026e5c), + LL(0x91c09091,0x636225f5),LL(0x71bdcfdf,0xccf5070a),LL(0xb9668ee2,0x0ef8d625),LL(0xb5e04e4f,0x57bdf6cd),LL(0x7c75ea43,0xfc6ab0a6),LL(0xf7fd6ef3,0xeb6b8afb), LL(0x2a3df404,0x5b2aeef0),LL(0xb9823197,0x31fd3b48),LL(0x83a7eb23,0x56226db6),LL(0x5bb1ed2f,0x3772c21e),LL(0xcd1aba6a,0x3e833624),LL(0xac672dad,0xbae58ffa), + LL(0xdaff1807,0x00e0a003),LL(0x92c94fd0,0xcb9d1559),LL(0xcebbf905,0x3c2b5c3d),LL(0xd338afa9,0x9c799ec7),LL(0x4e2cfccc,0x60b9908c),LL(0xae3c6f92,0x4bfe1a57), LL(0xfb116150,0x480d310e),LL(0xe3e7888e,0xa1ed6c31),LL(0x720b5196,0x841a11d9),LL(0x8adff37d,0xcc337d17),LL(0x5faa86c5,0x08c66826),LL(0x9dfcc7ad,0x945c90d4), + LL(0x31ba1705,0xce92224d),LL(0xf0197f63,0x022c6ed2),LL(0xa4dc1113,0x21f18d99),LL(0x03616bf1,0x5cd04de8),LL(0x9ff12e08,0x6f900679),LL(0x48e61ddf,0xf59a3315), LL(0xb51bd024,0x9474d42c),LL(0x9051e49d,0x11a0a413),LL(0xdce70edb,0x79c92705),LL(0x34198426,0x113ce278),LL(0xea8616d2,0x8978396f),LL(0xea894c36,0x9a2a14d0), + LL(0x8e2941a6,0x9f9ac960),LL(0x2fc4fe1e,0x43e7ff90),LL(0x6033e041,0x5ec41359),LL(0x6f6ff0f3,0x5ce791c4),LL(0x9d907343,0x8d134b89),LL(0x86304df2,0x7bd15c77), LL(0x77c4a913,0x2cd2ebc7),LL(0x45f07153,0xcd86a39d),LL(0x88bc423b,0xe7e12d2e),LL(0x0b3163f4,0x478e814b),LL(0xbe8ec766,0x78bd9c8a),LL(0x7709ce48,0x6a5763e8), + LL(0x604f6e4a,0x4f1e1254),LL(0x0187d585,0x4513b088),LL(0x19e0f482,0x9022f257),LL(0xe2239dbf,0x51fb2a80),LL(0x998ed9d5,0x49940d9e),LL(0x6c932c5d,0x0583d241), LL(0xf25b73f7,0x1188cec8),LL(0x3b3d06cd,0xa28788cb),LL(0xa083db5a,0xdea194ec),LL(0x22df4272,0xd93a4f7e),LL(0x6a009c49,0x8d84e4bf),LL(0x3e3e4a9e,0x893d8dd9), + LL(0x8d095606,0xd699ea2d),LL(0x1e0ddd3a,0x3cd080c5),LL(0x66a8b35b,0x46604bad),LL(0x4233fccb,0x0c779b62),LL(0xbfd3cf0c,0x578458ac),LL(0x96bf57af,0x6820f665), LL(0xbf1f302c,0xa9724245),LL(0x277a6c3e,0xbbde24da),LL(0xc6be8c14,0x0980a5b8),LL(0x774d62c4,0x6230e3ec),LL(0x4fbde24b,0xda1467d8),LL(0xcc862204,0xd9d68d07), +}, +/* digit=49 base_pwr=2^245 */ +{ + LL(0x7378f90d,0x67c51634),LL(0x66647082,0xbc201a79),LL(0x9ee450cf,0x77fcc8dc),LL(0xb41a3e2f,0x8dd2b318),LL(0x93bf0689,0xdf6a935e),LL(0xa92e5464,0x75edabf3), LL(0x604d208a,0x49afcd9f),LL(0xd465ca48,0x372f0ea7),LL(0xc7ea7810,0xcdbd8ad2),LL(0x550822b2,0xfe61571e),LL(0x86606adc,0x744a4f93),LL(0xd9d4e110,0x6beb3c9c), + LL(0xe700b9f2,0x1fef389c),LL(0x425bc8ab,0x63029466),LL(0x37f04a33,0xbd770a14),LL(0xd0169369,0xc7438e29),LL(0xe2377cc3,0x6b265742),LL(0xc369fa4f,0xdf24bf96), LL(0x0ad94e08,0xdfdbcf47),LL(0x7f75a7dc,0xd101b861),LL(0x2a9c483c,0x5574a0b8),LL(0x2de43228,0x0563fe94),LL(0xead1fabe,0x58ca0e8a),LL(0x66023966,0xdc3d9a84), + LL(0xc3fd20e5,0x383bda07),LL(0x5c29449b,0x9619b1df),LL(0x369f39bf,0x6f3c717d),LL(0x1a5a3900,0x1bb593d1),LL(0x2aec6c2b,0xd0f07ecc),LL(0x4240b202,0x9d72eb2a), LL(0xc50e4a0c,0x35342f6c),LL(0x6b93bf61,0x701b4662),LL(0xccb6a888,0xfcd6eb09),LL(0x85aa42c5,0xabb7a6f7),LL(0xaa4e5895,0x952f8824),LL(0x5c406582,0x49860db8), + LL(0x3955812b,0x3667a720),LL(0x284d1dac,0x0d73483b),LL(0xfc62f791,0xe084535e),LL(0x389faf7f,0x5bc1652b),LL(0x3a71b7f6,0x40cf5168),LL(0xd4f39703,0x8a4b19fa), LL(0x2a8eff13,0x823e754a),LL(0xbffa5afc,0xf01b2021),LL(0x7225b319,0x5639ee02),LL(0xfc282f16,0x7533bc86),LL(0xc69f61ae,0x710009d2),LL(0xbf65e803,0xe30c499d), + LL(0x734b4ec3,0x0da7ac1b),LL(0x12a2afbe,0xf47fc1d0),LL(0x87dce4a2,0xbbbc99be),LL(0xdd5c6378,0xf7264b4e),LL(0xf618ffdc,0xe9409305),LL(0xd1846ac1,0xafadda9b), LL(0xa21850d4,0xe734f9d0),LL(0x8722a316,0x199cb44f),LL(0x38cae89f,0xcfe8704b),LL(0x6b151b57,0x2db1e56b),LL(0x69ce7b2c,0x116ca5cf),LL(0x57de97c8,0xe9b8625f), + LL(0xaf247c49,0x18811bd5),LL(0xe124dbda,0xbc180793),LL(0x21234fc4,0xed978d3a),LL(0x0616ae15,0x516dd9a7),LL(0x74e430b8,0x8f806777),LL(0x06e8fc49,0x90942569), LL(0xa4e61235,0x4ca03fb5),LL(0xb617f361,0xb91de709),LL(0x0ed08bc3,0x0898d82d),LL(0x8cb08146,0x2bd71236),LL(0xe213176d,0x45b92d45),LL(0xf2bf5b9c,0x05894791), + LL(0x2695ea2b,0x0d79cb89),LL(0xc88e538a,0x2cb0f8df),LL(0xa80f36fe,0xc1b8dc3d),LL(0x84f00cc2,0xd756fa66),LL(0x9cb9efb2,0xa6f1cdec),LL(0xa6a21818,0x5c3f15a8), LL(0x6995d09f,0x9a7ee351),LL(0xd70434bf,0x88885463),LL(0x4f7d5d33,0x18cecc6d),LL(0x6b353bd1,0x3f013886),LL(0x0d9ad368,0x53bf798b),LL(0x28dbc3ee,0xeffd465a), + LL(0xb5d98ac1,0xeb29e44c),LL(0x0e227a4f,0xe47e57f8),LL(0x3d2bf688,0xd09c0494),LL(0x47428dd2,0x3ab7799a),LL(0xe9aafac8,0xdc558d6b),LL(0x87f9f6e0,0xc042c4cd), LL(0x89fb4693,0x93842bcd),LL(0x7068fbf7,0x62dbc82f),LL(0x7e6d47b5,0x16455268),LL(0x4c37eeee,0xab304b7a),LL(0x3fc412ce,0xdbb3d4e1),LL(0xa726a2c8,0x4f65dad0), + LL(0x605cdaee,0xb25e01b2),LL(0xbc57969d,0x74abec55),LL(0xcdd9d41a,0x9c57bfab),LL(0x4a9e32a3,0xa3330e3f),LL(0xe5792fd8,0x5929a0d8),LL(0x71ea2cde,0x830b4ea2), LL(0xfd06d246,0x80065ac1),LL(0x32e64a25,0xa2b416e6),LL(0xc0c927a9,0x3950bde7),LL(0x679d9b8c,0x9951f3bd),LL(0x651b6855,0xc235a274),LL(0x5ad97bc1,0xbfe5e08e), + LL(0x744ae145,0x4409a5b6),LL(0x7f620908,0x5e83fa0b),LL(0x2e140aa0,0xfc489bec),LL(0xe3cae337,0x5805a462),LL(0xc2211c21,0xe56e9ff7),LL(0x0c955362,0xb722f2b4), LL(0x41371f33,0xb098a32f),LL(0xbb4923d6,0xe6ccecea),LL(0xd82a311c,0x1cfbe2b3),LL(0x6b98f917,0xcf942893),LL(0x92ef848c,0xd60dc624),LL(0x5adb5228,0x34af446e), + LL(0x796ce1ca,0x0eb7e743),LL(0xd851377c,0x138653e5),LL(0x2b11c8e0,0x69c7c86f),LL(0xcdf2b205,0x878ec1de),LL(0xae0e8562,0x03e6688a),LL(0x935a36a8,0x20810666), LL(0x26635c50,0xc8ab7c7f),LL(0x744a21db,0xe75cdb06),LL(0xd720e198,0x4e26f32f),LL(0xd8cded81,0xa1c6395a),LL(0x6ce4fc04,0xb75dc6ea),LL(0x004623b5,0x71750b33), + LL(0x7e60c447,0xbdef8407),LL(0x2a65acca,0x88570f71),LL(0x0bb6aa79,0xef3d4a40),LL(0x60212976,0x5c9d1890),LL(0x1d96c43c,0x80179ea2),LL(0x53d2948e,0x3f002e6d), LL(0x49d78183,0x14b2cc91),LL(0xb496c279,0x7a549c71),LL(0x44995f6d,0xf4beac3f),LL(0x00bc78fe,0x5a342398),LL(0x60e42da0,0xa874dc1b),LL(0xcf5824d5,0x3a984010), + LL(0xdfb9760c,0xe514ee06),LL(0x77b8951f,0xb8862d75),LL(0xf8ee1141,0x0144676e),LL(0x02eb3e82,0x49561a30),LL(0x4ff9f897,0xb3541c15),LL(0xa7a99791,0x1670edf0), LL(0x64aea7f9,0xd41d6035),LL(0x2b3463b4,0xf66ffd09),LL(0xc3b26fb6,0x0784e015),LL(0xec46f8c8,0x88edce33),LL(0xb6381011,0x1b1e25a3),LL(0xff95ab97,0xbfaadc03), + LL(0x0c7be4e1,0x727a59fe),LL(0xf58ced15,0x75a7d5e3),LL(0x90f569e7,0x146fc0d9),LL(0xb7f1dc54,0x94dbccd2),LL(0xb75bf232,0x0df1ef90),LL(0xa2568190,0x2943a082), LL(0x67837b06,0x75f2f80d),LL(0x24b44b6e,0x07e3506f),LL(0xd0d2231b,0x7c30829a),LL(0x93277abf,0x9ce577ca),LL(0xb17549ec,0xa19d1868),LL(0x25e8c4d7,0x0ad6ff55), + LL(0x1c24d075,0x16b38dfe),LL(0x992959f6,0x3acd4c36),LL(0xac2da7ab,0xdaf2fe88),LL(0x89644935,0x76e8ff0e),LL(0xe85f7076,0xb8547c26),LL(0x1cdea7ce,0x9f149faa), LL(0x9e125d84,0x181a6072),LL(0x18751ce6,0xc4aef9fa),LL(0x0e00f00a,0x451c8466),LL(0xc4e3e6b8,0x662b3e7a),LL(0xc6b64507,0x57b7114e),LL(0x0b37fb70,0x07aeb198), + LL(0x4516234a,0x79d88e00),LL(0x31f9ceda,0x98dd3cb9),LL(0xce7d606e,0xb528000f),LL(0x2fa27fd3,0xc773557e),LL(0xe19436af,0x55b53dd3),LL(0xe10b64c7,0x675084b3), LL(0x56d56374,0xe5832665),LL(0x307e2e60,0xf8f7fd2a),LL(0x7af3e3dc,0x7b93bf53),LL(0xf47d298c,0x94fafa2c),LL(0x21121369,0x94c2ff9a),LL(0x33468ff6,0xa41de95f), +}, +/* digit=50 base_pwr=2^250 */ +{ + LL(0x1b270599,0xbe45d81a),LL(0x97d6c603,0x50696e7d),LL(0xb078ea89,0x63c5a516),LL(0xb4464764,0x9f3efe41),LL(0x101e5232,0x84580e24),LL(0xc8ae8220,0x00850a1a), LL(0xed55c404,0xbff4077d),LL(0xf2e7bf50,0xd74de734),LL(0x07e1c03d,0x4df4eef2),LL(0x6e654d58,0x4ab3d039),LL(0x086f1596,0xb20056cd),LL(0x8acd7cd5,0xe4d08a27), + LL(0xc90b13f5,0x8cd6c9f7),LL(0x52a9d469,0xec0c187d),LL(0x89b8ad2b,0x9c0db0f5),LL(0x0d9c999d,0x692a8db7),LL(0xc9f84ab4,0xa407fd03),LL(0xcc9a906c,0xa5742fd1), LL(0xc8e72867,0x4813a765),LL(0xe2e9a10f,0x9c65943d),LL(0x4fa0a23e,0xca6bf293),LL(0xcb1f8d7a,0x1dfa3af7),LL(0x98d10c53,0x28036f54),LL(0x0e012c13,0x7bfbcaf2), + LL(0x893e8032,0x2d9513a9),LL(0xf4688db4,0x49257f7a),LL(0x3af4d9ac,0x73d8b12c),LL(0x48a13c4d,0x903dc9fa),LL(0x60709433,0x6190753e),LL(0x49387d24,0xa093364e), LL(0x3b261e16,0x0436949e),LL(0x4a3055b6,0x96db3b27),LL(0xe85dfe23,0x514eacc7),LL(0x5d8805c9,0x1538b25c),LL(0x664a20f6,0xd4c6b75b),LL(0x4753292d,0xd1984f21), + LL(0x4ec177f6,0xa53f1a10),LL(0x3faa5ca8,0x4a2ef9aa),LL(0x32976d13,0x30efed85),LL(0x5ee692d1,0xcf5ada16),LL(0x259e7cc1,0x3ceda69d),LL(0x9baab472,0x2aae29e9), LL(0x737cc8bc,0x7ee5baef),LL(0x7fe68ded,0x1717af74),LL(0xcfdaff63,0x9e5c8dda),LL(0xcec07693,0x575c8db9),LL(0xfdfb509d,0x9afc8ae0),LL(0x85651697,0x27836d36), + LL(0x85e79c26,0x9d152839),LL(0xd36ab6f7,0x8d87faad),LL(0xa87afe15,0x08d4fe09),LL(0xfab81fd4,0x1e10b10f),LL(0x93c98a84,0x3cbd17d3),LL(0x246ceed3,0xbd40a4e8), LL(0x9a09d134,0x582d9de0),LL(0x4ee37dec,0x120440c2),LL(0x2f1073e4,0x4d4ff934),LL(0xf45e648d,0x7da76757),LL(0xebcbb80b,0x4427a608),LL(0x30e661a7,0xc6bccb54), + LL(0x135e2a30,0xb7b7628f),LL(0xb526fa7b,0x863552c4),LL(0xe0e30451,0x6ec18d05),LL(0x5769db60,0x1b36c93a),LL(0x170c236c,0xf0fe0007),LL(0x66130046,0xeceb540c), LL(0x3fc4bdbe,0x86a7a74d),LL(0xfbae3320,0x066b097b),LL(0x6e5c21ae,0x78fb5247),LL(0xe1adf398,0x3e19e9fc),LL(0x1a32a745,0x429b9cbd),LL(0x36d1a2b8,0xedd2c40f), + LL(0xceeaeed7,0x993ca8a9),LL(0x62545429,0xb2d28681),LL(0x91cf32d4,0x24003737),LL(0xd88bd4f0,0xda9ef96f),LL(0xe1b2d52a,0x916a4947),LL(0xf31b107c,0x5ef7f9d0), LL(0x62d3d3cb,0x3c424ba8),LL(0x7c3c3bba,0xab155cc2),LL(0x4bae8070,0x6a0404b2),LL(0xdf36677c,0x5f3d0592),LL(0x9a0e4800,0x7c9f2bef),LL(0x97959a09,0x6babbcb4), + LL(0x17eb9264,0xa7342f95),LL(0x0a8a6eef,0x9264a6a0),LL(0x7471c384,0x50e48bf0),LL(0x30827f34,0x729e5ab1),LL(0xea779c23,0x17199191),LL(0x9fa9fd58,0xd13ab853), LL(0x3b1d773e,0x7d579937),LL(0xd196c3df,0x65f8e7c6),LL(0xe8541725,0x253f7d51),LL(0xec720355,0x107a793d),LL(0x6aa16268,0x1c14d056),LL(0x8bbb231b,0x9dc5fca3), + LL(0x897a05a4,0x69935431),LL(0x51eaa290,0x34397b68),LL(0xf58fb7a5,0x90ec1a37),LL(0x50c4d76b,0xf10d0783),LL(0xcc47f990,0xeb9db48c),LL(0x0cea5865,0xef0b97a2), LL(0xd9f94396,0x04708d6f),LL(0x41c21452,0x82ff5771),LL(0xdbb65bdd,0x772d8493),LL(0xddf73c8c,0x561abc8b),LL(0x9830ff05,0x98a56463),LL(0x9f0d4cad,0x73e28296), + LL(0x589cb234,0x7fa7064e),LL(0x65ca4f3c,0xdfa4e846),LL(0x792d5254,0x476b6618),LL(0x583bdaeb,0xc0ce93bc),LL(0xe4ab5dc9,0x30b11dac),LL(0xf5c89e2f,0x237a64e5), LL(0x7dba60b7,0x54339fad),LL(0x084b09ed,0x0072505a),LL(0xb140717a,0x5e89c81f),LL(0xb407595d,0xd56de3a2),LL(0x0aab0d25,0x9f3a6c42),LL(0xa685543d,0x6c2aa69d), + LL(0x6cd00d89,0xda5fcddc),LL(0xac9ef99d,0x0b8c3feb),LL(0xa19298e3,0x16569ca8),LL(0x2783ac1d,0x5d998d56),LL(0xc18ecbdf,0x298c681c),LL(0xe6e13de4,0x209323d1), LL(0xbd75118d,0xe064255e),LL(0x43f945ef,0x78f69fd3),LL(0x95e6ff8f,0x9a4d591f),LL(0x3b378848,0xfa621d46),LL(0xe4d12bb8,0x4c951f31),LL(0x00ebfea6,0xefe8d01b), + LL(0x12b09f53,0xf5689c5e),LL(0x9e87ff7d,0xc1da32e1),LL(0x12eaa533,0x1af879d0),LL(0xd9271e94,0xdba775e6),LL(0x10e63c34,0x60f85073),LL(0xa686a24b,0x445f3e21), LL(0x15bc277e,0xed5ca8fa),LL(0x364ab7ab,0x9839198a),LL(0x6d90a7d4,0xe2ee3942),LL(0xccd37e76,0xe5b3e4cb),LL(0xf1412e0f,0x9013bd08),LL(0xce999048,0x82f5c532), + LL(0xf4bbf123,0xdd7ff816),LL(0xde3e9923,0xbf1a5fd0),LL(0x68d10f94,0x8c388f8c),LL(0x45057388,0x4e9cf0c4),LL(0x8d010855,0xcff64aa3),LL(0x7d8f55fc,0x4b639596), LL(0xbddc00c5,0xca314522),LL(0x3da89cfa,0x95482d72),LL(0x9eb9a710,0xb9bf18b8),LL(0x4651cc5d,0x0936a88e),LL(0xf59d0f45,0xec20ac01),LL(0x8ba74374,0x011a4868), + LL(0xf9c4caf8,0x07252272),LL(0x290ab63f,0x9b016799),LL(0x558d649d,0xeaa616bb),LL(0xa66d8089,0x00f2ef38),LL(0xf72863ae,0x284b0146),LL(0x1968cf45,0x9a207d77), LL(0xbdcbb689,0x33d7bac8),LL(0xe5348dae,0x393f34d5),LL(0x6f524620,0xeb86c8f1),LL(0x610689f8,0x62500c62),LL(0x6b7fa65e,0x66febc05),LL(0xfb836b3e,0x39c8a70b), + LL(0x5e4d0351,0xb91977b5),LL(0x9e8dddf7,0xd8ed39aa),LL(0x9d1b25c3,0x9ae994c2),LL(0x6ca7b19b,0x7369e189),LL(0xec0d7c2c,0x33deb695),LL(0xede6435b,0xddcc6250), LL(0x145a654d,0x44b7ba23),LL(0xd280567e,0x653ee81a),LL(0x0a39d324,0x7694c972),LL(0xe97e1710,0xf0af25b1),LL(0x3ee1a076,0x6e154646),LL(0x7ccfde8c,0x062ce983), + LL(0xf3fe3441,0x61d0e01b),LL(0x2af47609,0x674e5233),LL(0xb362902d,0xd4a4e224),LL(0x9e0a5d16,0x45923c12),LL(0x95e580e9,0x4fc2bdd4),LL(0xa8c3d954,0x6d1d974c), LL(0xd0bbeaaa,0xaeff1135),LL(0x1baafc9e,0x013ab5b3),LL(0xab8f9f31,0x80907d3e),LL(0x6d566c15,0xaf2c1216),LL(0x952e6fa7,0x0082daba),LL(0x2df9e03a,0xa4671003), +}, +/* digit=51 base_pwr=2^255 */ +{ + LL(0x1fd64063,0x5f79f0df),LL(0x81e118ec,0xd2d39dd3),LL(0x11571c5b,0xd631a68e),LL(0x2474faf7,0x6d072b4e),LL(0x862a924a,0x5e043a6d),LL(0xb0fc8d7a,0xcae58bd8), LL(0xb1351f28,0xf54bb7f3),LL(0x0413275e,0x4588b628),LL(0x5909ec04,0x81459f4c),LL(0xabd16460,0xd28cda25),LL(0x8db1c69e,0xbb676d01),LL(0xac5036f4,0xc0056e2d), + LL(0x44ce3ad8,0xdbe04c30),LL(0x4ce8aad5,0x995fbb1b),LL(0x70911457,0xdbf8b546),LL(0x3f7a1757,0x9e683b5b),LL(0x9c7bd62c,0x7b89a08a),LL(0x0b3fc97e,0x448865a4), LL(0x3bb01e94,0x0ac9abfc),LL(0x1e756124,0xa0776042),LL(0xd9deed97,0x0aa6c335),LL(0x72603e08,0xe270580f),LL(0x6c783bb2,0x70857a94),LL(0xcaa929ae,0xa0047774), + LL(0xb25c4d4a,0x292f8874),LL(0x7e79f526,0x54961fd8),LL(0x008c6ec9,0x949a1fae),LL(0x525524fd,0x6ae82f0d),LL(0x2edbcb1a,0xd1f6f4ef),LL(0x977ddffb,0x41617a6d), LL(0x1baf0668,0x6ae38fb7),LL(0xd538ab3c,0xa79ea228),LL(0xfc44e273,0x70babb05),LL(0xbca85910,0x247384fb),LL(0x6a564959,0xdc0e069b),LL(0x1a7438ad,0x37a9c552), + LL(0x8fa06859,0xf071c987),LL(0x1a52390b,0x0083e531),LL(0x61483bc2,0x845eb12a),LL(0x1caf6dd6,0x17471d80),LL(0xddc21b92,0x7b603616),LL(0xb992536d,0xd38fe0f6), LL(0x297c25a4,0x433f0652),LL(0xb1c4bf41,0x03d4d8fc),LL(0xa9adf49b,0xdf617386),LL(0x2cb2944f,0x4bfeb399),LL(0xb3d9c076,0xbf288427),LL(0x965b4576,0x17818c3e), + LL(0x689f43ef,0x2bb798ff),LL(0x5f26ec54,0x5813e441),LL(0x5005c929,0x51f64c49),LL(0x4b42e417,0x60e213a5),LL(0x62cc3734,0xc1528442),LL(0x09d994e1,0x6ecd6c3b), LL(0x83dd047f,0xa6e72f71),LL(0xb0019803,0x3836f663),LL(0x257493cb,0xbcf1265b),LL(0x9e62d78b,0x59b15ff0),LL(0x6cb92ecf,0xaac5ed5b),LL(0x9662651c,0x37e6ad7d), + LL(0xa17e560c,0xce23a19a),LL(0x62550e2b,0x6491b95f),LL(0x1d15a005,0xc7200012),LL(0xf4355a1f,0x15fde735),LL(0x607f7807,0x3849761f),LL(0x18204691,0xcbe322d0), LL(0xa95e8e91,0x75756e4e),LL(0x817a9b8e,0x365959fe),LL(0x3d4ce3dc,0x63123276),LL(0xf1d66e00,0xa769d2fe),LL(0xc28829e6,0x8624ddba),LL(0xd2df06ef,0x03274297), + LL(0x16f01d7b,0x52fced31),LL(0xdaf046aa,0x88c6b172),LL(0xfe7a338e,0x1a189403),LL(0x61798b1e,0x39741ecd),LL(0x2934b879,0x6a47b071),LL(0x828d1e9d,0x3b1a5dd1), LL(0x7f35a7ef,0xd4bd4848),LL(0xc1eebaf8,0x71774b5b),LL(0xd55344ba,0xa86471e5),LL(0x7b8a483a,0xfbf145f1),LL(0xaa53802c,0x70f9b214),LL(0x10b066e1,0x995af930), + LL(0x27a28f9d,0x9ec11597),LL(0xb847cd83,0x96f2c44b),LL(0x31fca111,0xacf794e1),LL(0x96076f45,0x438b9178),LL(0x51732588,0xad71035b),LL(0xa5d910da,0x2db32f32), LL(0xfe1cc184,0xefaad0e8),LL(0x2e00bbed,0x6f0360b5),LL(0x474ce326,0x99402426),LL(0x2aa270da,0xd53b687a),LL(0xd78fa6eb,0x96c8bb78),LL(0x6e699411,0xd07f3bba), + LL(0x361dae4c,0x07276886),LL(0x84c57896,0xb26f2579),LL(0x5f0c2bf4,0xe8b59ef5),LL(0xdd64aaac,0xf544c145),LL(0x9fdcb039,0xa9dc03bb),LL(0x58c44b72,0xeb1e9aeb), LL(0x1d6bf26e,0x39a4fe8e),LL(0xeaf8e241,0x2c4285cc),LL(0x14770f31,0x647ce584),LL(0x062f0912,0xe5bfeb70),LL(0xe8314467,0x7031db80),LL(0xd658d2cd,0xac970937), + LL(0xefe1757b,0x2dfc39e6),LL(0xb5d2cb93,0xab52fbf3),LL(0x313aa477,0x1cf12123),LL(0x9c6acbd1,0x785b025b),LL(0xe4d54177,0xb4aeb5b8),LL(0xd943c1ea,0xde3d28f8), LL(0x5b0b1921,0x7892db85),LL(0x58caff2d,0xc09ff903),LL(0x1cbd3231,0xbdee13c6),LL(0x00b6c34e,0x873e0a77),LL(0xe23de32b,0xb279505a),LL(0xc5a03302,0xf056ffdc), + LL(0x6ba563ee,0x0ac76959),LL(0xb9a51868,0x2053a5fb),LL(0x06178e3c,0x60e2555f),LL(0xc0933775,0x1bc99e72),LL(0xb7d5160e,0x4a2d31b2),LL(0x895d1db8,0x7cc03ff3), LL(0xee53e79f,0x98fb331e),LL(0x8583b893,0xc2f0a93f),LL(0x49802eb4,0x8fa79ed8),LL(0x55ce8dcd,0xe9548175),LL(0x1d4d44f1,0xb5155a21),LL(0x89570a82,0xbb044d28), + LL(0xe1f833e0,0x430b669f),LL(0x6d8127bd,0x1797ac3a),LL(0x4c33493e,0x01ad730c),LL(0xca00ed39,0x8c882c1f),LL(0xd24a5516,0xab2e9c89),LL(0xf5d0327d,0x21a49e0a), LL(0x58a280d1,0x46488bd8),LL(0x772ed759,0xfc4a1e8a),LL(0xaecfac7c,0xf9f60e90),LL(0xac6a9e8c,0x4afd1f5f),LL(0xc98bcda8,0x57a20bf8),LL(0x8b46b998,0xa1107ea0), + LL(0x45fc08c4,0xe931c21b),LL(0x3aca653f,0x13fad41f),LL(0xec1fe395,0xf28a3515),LL(0xcf55cc91,0x191a4f2d),LL(0x90535f59,0x46fb07b7),LL(0xd69686a9,0x76c60cc1), LL(0xc8122c04,0x1e7dec15),LL(0x1b72798c,0x0743fcf0),LL(0x1e5939fd,0xb10c96bd),LL(0x6e8338e5,0x261e6c0a),LL(0xe5cdaa43,0x56148d3c),LL(0xa031239a,0x3f004371), + LL(0x0db0c9c0,0x1f2f6c1f),LL(0xac770b5f,0xca4e1964),LL(0x569c090b,0x20dce4a6),LL(0xdb3986d0,0xe4f7401a),LL(0x85f553ea,0x13897994),LL(0x09026bff,0x119ad50d), LL(0xe265a0a9,0x88067aa7),LL(0x7f749167,0x209c62b4),LL(0x459bf269,0x84165019),LL(0x8b6e76f6,0xec125bd5),LL(0x1f0d2434,0xd636f932),LL(0x8e05e0b0,0xf6023a4c), + LL(0x65ea2b23,0x84c14e31),LL(0x19e93301,0xb68121be),LL(0xb25d9a83,0x9c1a873f),LL(0xd13773f0,0xafb9d04c),LL(0x51c32d57,0xd05014da),LL(0x0904efdc,0xff2e350b), LL(0xa0a01069,0x20bd7d22),LL(0x6c3b3fd8,0x3e74eb0f),LL(0xb743b72c,0x643ea531),LL(0x8e3ae785,0xa2fe7414),LL(0x92e8d320,0xb8fe89bf),LL(0x495f6d28,0x58985dc1), + LL(0x389283ba,0xfe1f11ad),LL(0x0cd91b22,0xc87e20b6),LL(0x3c5babf8,0x99d0015a),LL(0x5929ea0a,0x7e795b4d),LL(0x1dfb7b7e,0xc9cf6833),LL(0xa64992e8,0xc1c07346), LL(0x9889746d,0x0b7e0dd8),LL(0x1c43ea4a,0xa89d7b46),LL(0x34f02b96,0x64023cf0),LL(0x5662f0c8,0xf7dd410a),LL(0xa1058cca,0xa3bb6088),LL(0x4e7801ed,0xedb25dc3), +}, +/* digit=52 base_pwr=2^260 */ +{ + LL(0xdd93d50a,0x140a0f9f),LL(0x83b7abac,0x4799ffde),LL(0x04a1f742,0x78ff7c23),LL(0x195ba34e,0xc0568f51),LL(0x3b7f78b4,0xe9718360),LL(0xf9efaa53,0x9cfd1ff1), LL(0xbb06022e,0xe924d2c5),LL(0xfaa2af6d,0x9987fa86),LL(0x6ee37e0f,0x4b12e73f),LL(0x5e5a1dde,0x1836fdfa),LL(0x9dcd6416,0x7f1b9225),LL(0x677544d8,0xcb2c1b4d), + LL(0xda1c29ab,0x279fd119),LL(0x2b30d40c,0xbd068802),LL(0xda44105d,0xd8f57da4),LL(0x28223fe1,0xb1814b7a),LL(0xe06f2d2e,0xcf2fd241),LL(0x01dfde06,0x99003a02), LL(0xfded7e4b,0x876a31af),LL(0x2f725094,0x1efaf827),LL(0x493a6a0a,0x5117d608),LL(0xa88c03e7,0xdcec8088),LL(0xea916897,0xeae1d352),LL(0x6e8b2c57,0x8cdc2810), + LL(0x9c213d95,0x0254486d),LL(0xcb2f6e94,0x68a9db56),LL(0x000f5491,0xfb5858ba),LL(0x34009fb6,0x1315bdd9),LL(0xc42bde30,0xb18a8e0a),LL(0xf1070358,0xfdcf93d1), LL(0x3022937e,0xbeb1db75),LL(0xcac20db4,0x9b9eca7a),LL(0xe4122b20,0x152214d4),LL(0xaabccc7b,0xd3e673f2),LL(0xaed07571,0x94c50f64),LL(0xe66b4f17,0xd767059a), + LL(0x54e93c1e,0x09f8bb06),LL(0xad81e27c,0xb0045884),LL(0x076e13eb,0x26ebc7b6),LL(0x5d5ac07f,0xbda0b553),LL(0x48ab69e6,0xbcb81322),LL(0x1c0f21fa,0xd3847d2e), LL(0xc834d740,0x7a466528),LL(0xe0823ff2,0x6c67a79a),LL(0x4c1d7cb8,0x85dd1186),LL(0x2d081301,0x096f849f),LL(0x8a5ea0f0,0xb4f503dd),LL(0xd1bf69b2,0x71ee0889), + LL(0xdcd6d14b,0x40336b12),LL(0xe3b4919c,0xf6bcff5d),LL(0x9c841f0c,0xc337048d),LL(0x1d617f50,0x4ce6d025),LL(0x8117d379,0x00fef219),LL(0xf95be243,0x18b7c4e9), LL(0x38df08ff,0x98de119e),LL(0x8d772d20,0xdfd803bd),LL(0x0f9678bd,0x94125b72),LL(0x334ace30,0xfc5b57cd),LL(0xb7e86e04,0x09486527),LL(0x6e552039,0xfe9f8bcc), + LL(0x02a2d1e6,0x4ab7a22c),LL(0x1371d5a4,0x967e19a3),LL(0x078de336,0x20f59f95),LL(0xf7869245,0xfd28fa36),LL(0xcbf1d96f,0x1de42581),LL(0x366e1f0f,0x2e0127d7), LL(0x2258c741,0xbc65fa9d),LL(0xdd6d65f8,0x1f2f3356),LL(0x4a0822a9,0x06384f3a),LL(0xfd05a0aa,0x1c81332b),LL(0xd95ee3ce,0xbfb12361),LL(0x42016d00,0x180aaf06), + LL(0xd6f5a10e,0x3b75c45b),LL(0xc1c35f38,0xfd4680f4),LL(0xf8e0a113,0x5450227d),LL(0x73ddba24,0x5e69f1ae),LL(0x57f24645,0x2007b80e),LL(0x3d159741,0xc63695dc), LL(0x4530f623,0xcbe54d29),LL(0x2869586b,0x986ad573),LL(0x4cc39f73,0xe19f7059),LL(0x2b1b8da9,0x80f00ab3),LL(0x73f68d26,0xb765aaf9),LL(0xe993f829,0xbc79a394), + LL(0x95b3287d,0x0a159f62),LL(0x48cecad0,0xb18f8759),LL(0x1661a23f,0x6d1ab8ee),LL(0xc95c41b3,0xcae7f40e),LL(0x7c51eb56,0xbc3d2040),LL(0xe8754250,0xa7527283), LL(0x1f9e668a,0x81561056),LL(0x900f5912,0xb8aa7296),LL(0x6af2a00c,0xabdbc1bf),LL(0x2d0a56c0,0xe9a94254),LL(0x7bc8959e,0x4774a7b7),LL(0x19cef2f3,0x0a837ff0), + LL(0xf310d2a0,0x9c441043),LL(0xdc5eb106,0x2865ee58),LL(0x9cb8065c,0x71a95922),LL(0xa052af0f,0x8eb3a733),LL(0xb09d716e,0x56009f42),LL(0xabcbe6ad,0xa7f923c5), LL(0xfa375c01,0x263b7669),LL(0x21ef27a2,0x641c47e5),LL(0xb08ffd25,0xa89b474e),LL(0xf0a239f3,0x5be8ec3f),LL(0x242a6c5a,0x0e79957a),LL(0x0c6c75f5,0x1dfb26d0), + LL(0xa084fae6,0x36f3a3d8),LL(0x9a9b0d95,0x75983589),LL(0xcc80fcb6,0x70722186),LL(0x96d84c04,0xf28ed0c7),LL(0xffb63f90,0x95a32263),LL(0x98766034,0xdd7d60a0), LL(0x1d5c387c,0xe193a31f),LL(0xb8310f8b,0x6c5eca7e),LL(0xc083ff47,0xfe61d523),LL(0xcb2944e9,0x90c832db),LL(0x593334b7,0xa9f3f293),LL(0x2d7d1c33,0xe6cde2e1), + LL(0x9dfbf22a,0x2fd97b9b),LL(0x5643532d,0xdec16cc8),LL(0x60fee7c3,0xdf0e6e39),LL(0x545860c8,0xd09ad7b6),LL(0x73fc3b7c,0xcc16e984),LL(0x0d4e1555,0x6ce734c1), LL(0x4b5f6032,0xc6efe68b),LL(0x14f54073,0x3a64f34c),LL(0xac44dc95,0x25da689c),LL(0x5358ad8a,0x990c477e),LL(0xf36da7de,0x00e958a5),LL(0xc9b6f161,0x902b7360), + LL(0xf144b6cc,0xbd079cf1),LL(0xb4f4a764,0x7f86e29b),LL(0xf21f9cbf,0x5b08b290),LL(0x75e3aeb9,0xada0c85b),LL(0x6666c2df,0xd0789f8b),LL(0xd71ec2ec,0xcf5d8a8c), LL(0xe7e4364b,0x6f7780c3),LL(0x85d2eb75,0xdd9a6529),LL(0xd952a38e,0x8222f66b),LL(0x27260a29,0x9dd5f7eb),LL(0x57947178,0xce49b344),LL(0xcdda7e39,0xaa215f82), + LL(0x9347b90a,0x454ab42c),LL(0xa698b02b,0xcaebe64a),LL(0xfb86fa40,0x119cdc69),LL(0xc3109281,0x2e5cb7ad),LL(0xcd0c3d00,0x67bb1ec5),LL(0x83f25bbf,0x5d430bc7), LL(0x5cde0abb,0x69fd84a8),LL(0x9816b688,0x69da263e),LL(0x0e53cbb8,0xe52d93df),LL(0xadd2d5a7,0x42cf6f25),LL(0xc87ca88f,0x227ba59d),LL(0xda738554,0x7a1ca876), + LL(0x3004db31,0xaa44b286),LL(0xd43e4430,0x86f43d7a),LL(0xb0b0240d,0xdc4874cd),LL(0xadc45a06,0x79986a23),LL(0x3cee4631,0xbb275b44),LL(0x63a217aa,0x21daee8a), LL(0xd7b25c02,0x1e7c5397),LL(0xc5e668fa,0xe677d3cb),LL(0xed51b4bf,0xc7c84e28),LL(0x923e5408,0x7ca19e99),LL(0xc3f832e7,0xc6f8a595),LL(0x5fb049a3,0x2d0a789c), + LL(0x1cac82c4,0x3fa5c105),LL(0x8a78c9be,0x23c76087),LL(0x1c5cfa42,0xe98cdad6),LL(0x0a6c0421,0x09c30252),LL(0x42fc61b9,0x149bac7c),LL(0x3004a3e2,0x3a1c22ac), LL(0x202c7fed,0xde6b0d6e),LL(0xe7e63052,0xb2457377),LL(0x3706b3ef,0x31725fd4),LL(0x2b1afdbf,0xe16a347d),LL(0x8c29cf66,0xbe4850c4),LL(0x2939f23c,0x8f51cc4d), + LL(0x44922386,0x114a25c8),LL(0x6d4e8b57,0xdd084d44),LL(0x1e7bd7de,0xc49b6841),LL(0xd6da54db,0x5b0359fa),LL(0x3f0da321,0xa6e6e5f9),LL(0xd640a87e,0xb65ec55c), LL(0xae64020e,0xc1a4f6ce),LL(0x088e1337,0x91e29cd2),LL(0x3c0a631c,0xf44ceb8e),LL(0xb756445f,0x0205b11d),LL(0x5bc8880e,0x04844e84),LL(0xb85e00d3,0xb630ddc0), +}, +/* digit=53 base_pwr=2^265 */ +{ + LL(0xae7ce296,0xe76894c3),LL(0xa6cafc34,0x87737ee2),LL(0xe55cd1e6,0x566dfcfb),LL(0x3a7ad5b9,0x5421a9f2),LL(0x4687a4ef,0xa005838a),LL(0x23a2c423,0x3837219a), LL(0x8a82cd1b,0x4b780012),LL(0xc728b588,0x401c07be),LL(0x37ced8f3,0x2b5f69e9),LL(0x8c1e1eaa,0x306b621d),LL(0xd389cc4d,0x8acbbe71),LL(0xf4ab7774,0x922fa665), + LL(0xd35c2d80,0x2df6f242),LL(0x3493ce97,0xf65a99a9),LL(0x372bcc87,0x9e80232b),LL(0x6e428cc5,0x26ba13b8),LL(0x13a1b763,0x2526ef1f),LL(0xdc97c5f3,0xcef3edcd), LL(0xbde16b73,0x4954867f),LL(0x368ff6cb,0x9817813d),LL(0xbe143027,0x7e39fa69),LL(0xcf54f28b,0x12329463),LL(0x7597c2da,0xcf0991dc),LL(0x52e07099,0x0cda3969), + LL(0xee993749,0xdc9daeef),LL(0x7ff3d775,0xd40c8a54),LL(0xca53d53d,0x206d2271),LL(0x4d1aa50b,0xb0546335),LL(0x88001e99,0x52bca910),LL(0x17fb7aa4,0x25c117c2), LL(0x42685945,0xfff3af95),LL(0x8f4ce0fb,0x083fd4de),LL(0x24753989,0xaac004be),LL(0xe90950c6,0x4a5de2b6),LL(0x738efe5b,0xb46af0ab),LL(0xdb4459f9,0xcf80a17f), + LL(0xf303955d,0x412f64a3),LL(0xbd692593,0xe92bdca9),LL(0xc2e964e0,0xfbe6cdc2),LL(0x0011cb01,0xe9a3b1fd),LL(0xcf228f23,0x6c30762d),LL(0xbe9199a1,0x1270b84a), LL(0xe3c9cbb1,0x732711df),LL(0xd91d9513,0xa3aabe37),LL(0xc6eceba7,0x8ee08ba0),LL(0xf3c3d31d,0xb1711531),LL(0x3c716948,0x65060b63),LL(0x2ff2cadd,0x046b4ea1), + LL(0x961719fe,0xdeb7a462),LL(0x76ba6ed7,0x5e22796d),LL(0x25d22208,0x5907daf2),LL(0x2c21c04c,0xd98260a7),LL(0xe090f349,0x56b24923),LL(0xa31a8f95,0xb5960ad9), LL(0xf8b1da62,0xb6ac57ce),LL(0x9f14f70f,0xe47d0995),LL(0x04dba20b,0x0ba6eb01),LL(0x46b60e27,0x41ca71c8),LL(0x6bf5eeef,0x2bbfdb30),LL(0xf075b238,0x58e16788), + LL(0xbab220c6,0x25d1124f),LL(0x61524e3e,0xcd1423c8),LL(0x0434fb51,0x75e4f45f),LL(0x5180ab2b,0xb5180a8f),LL(0x5b22e388,0x144e214e),LL(0x92263054,0x6b16dad1), LL(0x40863566,0x3ea75907),LL(0xdada3b46,0x372d5abd),LL(0x893d210f,0xb3ff5a3a),LL(0x5e29f3dc,0x39f8d1ce),LL(0x68200e82,0x559186ce),LL(0x1202cb66,0xf4876454), + LL(0xeaf4f2a1,0x699c2db9),LL(0x61c0c17c,0x8cd33227),LL(0x64f16a56,0x971b50d4),LL(0x102bbe10,0xcd00d42b),LL(0xb05f3cac,0x928d0ae2),LL(0x14bcf472,0x245dbe38), LL(0x947c0184,0x43d29526),LL(0x4612a4ea,0x24089968),LL(0x6c2b4541,0xbc763fcd),LL(0xf82448fa,0xb4e7ae0f),LL(0xb02b6459,0x94f1fa15),LL(0x67d39bbd,0xbcafa1ec), + LL(0x8b540904,0x6f178dbf),LL(0x8720472a,0x0264bccd),LL(0x59b46611,0xa6e8b4b4),LL(0xc72b4a58,0xafce8267),LL(0xa45985ad,0x21142175),LL(0xe649d733,0xd23401df), LL(0x85dc7720,0x6bf42fe0),LL(0x40e3f2f5,0xc5c8ab94),LL(0xcd029197,0xb0c8a58a),LL(0x215492e1,0xa73ff329),LL(0xb1b5a5f0,0x895c545e),LL(0x6fcaf49b,0x6dbc2445), + LL(0x44cc852a,0x705ad3d8),LL(0xb80518e0,0x10bb5add),LL(0x0de9f160,0xa34905bb),LL(0x94b1aacd,0x7d45aae3),LL(0x7fd5de7a,0xd30411d0),LL(0x4d0167cd,0xe2fc6206), LL(0x10ce71cf,0xd3b19612),LL(0xb28225fd,0x8e8096b6),LL(0x64a1c849,0x4b46fa80),LL(0xa51364e9,0x160479ac),LL(0xfaa3f0f2,0x9ebb6fdb),LL(0xf1511754,0x9ce029b9), + LL(0x0f2d76a3,0x25ef32d6),LL(0xaf4a7d46,0x540650b9),LL(0xd991d7f4,0x8979a4b8),LL(0x99202400,0xdaa706c2),LL(0xf19d281d,0x8a729680),LL(0x4ec44de2,0xde25bdc4), LL(0xc2054496,0x0fc50832),LL(0x0aaf2941,0xfee72fb6),LL(0xb82ed4f0,0xc8263e64),LL(0x6f49055c,0x91a8cb73),LL(0xf2bb515c,0xb7585458),LL(0xb855e6c6,0x03d2b23a), + LL(0xa0879d68,0x8e11e8e8),LL(0xbf7a84ac,0x2bea77a3),LL(0xa74b45d0,0x98140930),LL(0x810e587c,0x1ce28654),LL(0x8869daac,0x0a30756b),LL(0x39d2fe12,0xbf5e8245), LL(0xcac16a87,0xe6414992),LL(0x437aff7b,0xd2fa182d),LL(0xb6146094,0x4e61412d),LL(0x2f31bb4b,0x30a949ec),LL(0x22dc8ac6,0xf254c71f),LL(0x1ab2a0bd,0x1d9ed85c), + LL(0xcfd3f182,0x09ec1e3d),LL(0xadae7af9,0x1f1c30b5),LL(0x6b454164,0xf3a33f7c),LL(0x94647c4f,0x0318926f),LL(0x87db14ec,0x8e37bdd7),LL(0x2ab364d3,0x811cbd34), LL(0x7c2b369d,0x1dd1e507),LL(0xa28056bd,0x7a57bc46),LL(0x089efe44,0xfca5be4b),LL(0x6dc1290e,0xb3bd84d7),LL(0x8793e6ae,0x40d7af09),LL(0xa3723942,0x4e08e11f), + LL(0xeb73ec7d,0x649eeabb),LL(0x26a9aaf9,0x10983304),LL(0x4e296235,0xf22e4514),LL(0x7b85f801,0x695c8df4),LL(0x3ae7caaa,0xd4553344),LL(0xffbba90e,0x3e35bd47), LL(0x5d13b9ec,0xdd04f7c9),LL(0xe259f70d,0xe39a5d12),LL(0x201ae17c,0x39073063),LL(0x6a85435f,0xdba3eda0),LL(0xe948924a,0x6df48093),LL(0x00e3394f,0x9ffc4dcf), + LL(0x899ffebb,0xca3709ad),LL(0x77c00602,0x1a873778),LL(0xa99b4af0,0x5ff40c2a),LL(0xa80e870c,0x680464e5),LL(0x94e10b1d,0xd2f7f044),LL(0x4e9aa1a7,0xee9b206c), LL(0x96cbe950,0xb536d675),LL(0x9e8305f3,0x84185689),LL(0x369fa255,0xae1b669c),LL(0x7233e1ea,0x62e26026),LL(0x6aa60c24,0xac05c513),LL(0xd2691677,0xdfc6814f), + LL(0x397147d6,0xbe414528),LL(0x42592203,0x74851314),LL(0x0364b0eb,0x9084d330),LL(0x6ad70814,0xf8e5d6d5),LL(0xffb4ac5f,0xfcd4e0e3),LL(0x1fbf8899,0xf652417a), LL(0xccbd7eaa,0xb1165da7),LL(0x6e2d4e8c,0xf5dbd11a),LL(0x32ddcea8,0x5dab120a),LL(0x9892f728,0x30aaa56f),LL(0xd3d73838,0x71c2412d),LL(0xd2e2becb,0xbc0253d1), + LL(0x0a02b0fb,0x8baef5df),LL(0xc2b92b02,0x58a2b06b),LL(0x54c8267a,0x268558d7),LL(0xccf70393,0xf924f795),LL(0xf68ee021,0xe3763f30),LL(0x5c01ba4b,0xc1e856f0), LL(0x722b6bff,0xcc01a3e9),LL(0xed5b3b02,0xd2be4623),LL(0x6c45e33f,0x1ab3512e),LL(0x4ef433f6,0xa978fe48),LL(0x8e21f5af,0x23e2ea01),LL(0x11524a40,0x49647d88), +}, +/* digit=54 base_pwr=2^270 */ +{ + LL(0xd087b788,0xe1d42d94),LL(0xba0e176a,0xfbfb221a),LL(0x83686966,0x5f6698e7),LL(0x74a30dbf,0xbb5e1594),LL(0xcfd20230,0xef86bb5b),LL(0x403b8f8b,0xf055a1c5), LL(0xd9d85ea7,0xf249aac8),LL(0x3d200198,0x7318f7bc),LL(0xefca9a90,0x3b80960c),LL(0x8f449c4b,0xf28e3388),LL(0xf0cfe09e,0x0cdfc61b),LL(0x8b22cd26,0x3b169c63), + LL(0x203bb368,0x923e948c),LL(0x231a80e0,0x58e37a2b),LL(0x6df27deb,0x345a011a),LL(0xd57f4ca2,0xba6784c1),LL(0x114196e9,0xf01b3703),LL(0x1aab426d,0x981a63eb), LL(0x51770c1c,0x2ffdc978),LL(0xefa722fc,0xddd19da6),LL(0x16f09c1e,0x5ca1c012),LL(0x5b9cc0b6,0x612021de),LL(0x5e150569,0x910e10e9),LL(0xe2ab93ea,0xacace9dc), + LL(0x5070e0a9,0x0cdd8372),LL(0xec550783,0x7c5ad562),LL(0x4f3b8d2b,0x9652b847),LL(0xe6e98d73,0xfdd60d93),LL(0xa3479d0b,0xd51cae2c),LL(0xee05c006,0x11b93b6d), LL(0x8a3b40d5,0x9d72b82d),LL(0xa7d24855,0xc6e996fe),LL(0x398603de,0x420672f7),LL(0x9a1af2ce,0xd551b34a),LL(0x13bdce0c,0xdeb8c1d9),LL(0xebbeba7a,0x56ca926d), + LL(0x9db9ca19,0xac58c9e0),LL(0x390054d0,0xd308ea5d),LL(0x2cc42529,0x32ef4afc),LL(0x97c2bdf9,0x08bd48b3),LL(0xa849e19a,0xac8a7803),LL(0x75c31496,0xcd51c0da), LL(0xf0e2d49f,0x733dc7de),LL(0xb44b8cc5,0x7c9caad1),LL(0x47be21a8,0x6d9c5b08),LL(0x5ebf426f,0xfab0fdc5),LL(0xf94e9e5b,0xd60748ca),LL(0x69366089,0x3072e592), + LL(0x007d3d4d,0x8f24cfd9),LL(0xcb27ae66,0x2f4f7195),LL(0x9c1ed35e,0x8166162e),LL(0x4f4bdfd8,0xfdcf7666),LL(0x3b5373ca,0x6dc993bc),LL(0xa3ec42d5,0x2faa34d6), LL(0xc6b6109e,0x1cce84cb),LL(0x43632b7e,0x7a4178ad),LL(0xb2aab55b,0x6899adb5),LL(0x683c1397,0x080e42bd),LL(0x10960bfe,0x34d5e192),LL(0x7223406a,0x65415d21), + LL(0x6254954d,0x8acd0a3e),LL(0x1d8c2442,0x708f9359),LL(0x6dab2ff9,0x85f59135),LL(0x33a9d96a,0x9df92007),LL(0xcaa797ad,0xac9849a7),LL(0xfe95aa38,0xcea2067f), LL(0x995cb879,0x5a3c21b4),LL(0x69d4f07d,0x5c981a5d),LL(0xf0dc9aa6,0x028cf95d),LL(0xc68fd96a,0xcc5cee93),LL(0x0b69676f,0x8a5e73ee),LL(0x832d230b,0x190a7229), + LL(0xd40c5d9f,0xc0ab244b),LL(0xf3ef03ed,0xc05a11cf),LL(0x650ea714,0x1593d8ad),LL(0xfc6a1235,0x1edb6cbd),LL(0x4e451d0b,0xead76e1e),LL(0xae7ee558,0x4759e3c0), LL(0x7c8145e2,0x480627be),LL(0x88a339a2,0x82cec2c7),LL(0x752b0f58,0x17e887b6),LL(0xaa6c9df1,0x91c866d0),LL(0x29b3b1d3,0xd9a54848),LL(0xf641bbc1,0x17ae47ae), + LL(0xf0ef77c5,0xb06c17a3),LL(0x6df6bf59,0xc144e784),LL(0x0038aeb2,0x2440ae99),LL(0x58b402ca,0x83bf711b),LL(0xb577732a,0xb8763e00),LL(0xf651a932,0x509e91ef), LL(0x00ac109e,0xbe02ab9d),LL(0x8dfd78f1,0xfbcb426c),LL(0x4283f80f,0x7ed272f6),LL(0x2365da5e,0x098cf057),LL(0x05dc6beb,0xd90e6f18),LL(0xcf7b9d72,0x09ef177f), + LL(0x199785d5,0xebee13d1),LL(0x92f8f141,0x3d370648),LL(0x15e3a026,0x56595c83),LL(0xe6e5c659,0xa0f4960e),LL(0xc052f5d6,0x27d5bc54),LL(0x8677ef24,0x9c5437dc), LL(0xdb9b78fc,0x28ae53a3),LL(0xd91113f9,0x025b81a7),LL(0x6ac445f9,0xfb423fb3),LL(0x575649cf,0xb555a8b5),LL(0x06ae0cb6,0x3ac93b08),LL(0x8d2e3be1,0x7a706bb5), + LL(0x97c0cb32,0xcabd7301),LL(0x65f83e1d,0x1698cfd3),LL(0x62043a8b,0x44833254),LL(0x3b4c8bac,0x43d1641f),LL(0xe344a05a,0xcc394a5b),LL(0x853c1f6d,0x9e085b71), LL(0xcf627733,0x35a67867),LL(0xeab971dc,0x9d668c2b),LL(0x46c5f3bd,0x458a801d),LL(0x446c5e3b,0xb7b08696),LL(0xc15fc828,0x468ae44a),LL(0xab2761ff,0x9503fd49), + LL(0xce8d4427,0x7278f12f),LL(0xeb15f81f,0x036aef18),LL(0xa58e95ea,0xcbe64b86),LL(0x59b2428c,0xf933a850),LL(0x52f1b0fb,0x8d117bfc),LL(0x0bed33e2,0xba9dbc6f), LL(0x6ece7dc5,0x3e152fb5),LL(0xeb80d7ff,0x90e88871),LL(0xddf09489,0x0aa265aa),LL(0x2476414f,0xa17584cc),LL(0xcb0418ea,0x2d241d33),LL(0xc3a7529b,0x1c40e835), + LL(0xe3e1aee9,0x89b2edba),LL(0xe55e4aca,0x33533137),LL(0x8943fce4,0x1dded9ce),LL(0x5a2ff996,0xaaf07ff4),LL(0xc96e87fe,0x69e60f92),LL(0xb9ee808c,0xc2e5c1d5), LL(0xbe466616,0x79d6d8c6),LL(0xdffe4e28,0x897f6c2f),LL(0x350b7fc9,0xc8a65267),LL(0x8a2abab0,0xe9fcb46d),LL(0x2c4faf7c,0x57c3bfc6),LL(0xa8207c8a,0x41a8cc2a), + LL(0xa74858fb,0xb60e8daa),LL(0x50c7c2f8,0xaa4c8ca5),LL(0x31a39837,0x9eb37096),LL(0x69f44aa5,0x5742fb32),LL(0x51e793a0,0x086aa479),LL(0x55e7c373,0x9bcaf657), LL(0x1b13101d,0x63cee90c),LL(0xe7d42d91,0xbc410452),LL(0x913aa0fb,0x3041007f),LL(0x502be876,0x9a00ad14),LL(0xb8677568,0xa137a443),LL(0x46ca4efa,0x5b26b995), + LL(0xaed60a78,0xe33585b7),LL(0x3cbb2e24,0xd208deea),LL(0x1aca34e8,0xff32cccb),LL(0x7b64d250,0x52ebce07),LL(0x84d8b907,0x93a4d0a1),LL(0x77de8d42,0xd759bef0), LL(0x5d07487d,0xbd60c87a),LL(0xc74f79df,0x63620986),LL(0x4a52f837,0x54f5f2c8),LL(0xf3cec619,0x340a1f53),LL(0xfc85bf85,0x1e922bc4),LL(0xc3bfdf09,0xc92e7b85), + LL(0x0eb9e6bc,0xeb1a37c3),LL(0x4fbf1cb1,0x8c393d89),LL(0xd1f10389,0x6ac9427e),LL(0x7c8caefc,0x77b8cfc1),LL(0x1a61bdd5,0x554ab4af),LL(0xad75e46b,0x661a99f1), LL(0x439a0456,0x11fc01bb),LL(0x8d0046c9,0x5b7fedd2),LL(0xfbece9e3,0xcfded2ef),LL(0x11440124,0x4e4dcb0b),LL(0xdee23c7d,0x2a5fd0fa),LL(0xf662402c,0xaa5f345a), + LL(0xeeb791c4,0x6f101762),LL(0xdf261eff,0x0d942184),LL(0xac1dc827,0x2c58e2aa),LL(0xf835a1b6,0x51410e89),LL(0x629915a4,0x981333a7),LL(0x0c14148d,0x371891b6), LL(0xc0904446,0x4d20b3d3),LL(0x949776d8,0xdda7ecc8),LL(0x2a2645f7,0xa664b68c),LL(0xadd082ea,0x7a6bc857),LL(0x3e5ff206,0xe7467dc6),LL(0x04e2dfcc,0x40a6c340), +}, +/* digit=55 base_pwr=2^275 */ +{ + LL(0x03744726,0xa2354569),LL(0xd8d275ac,0xd2169e6d),LL(0x132c5689,0xab0c247b),LL(0xcc4760bb,0x129a5c9d),LL(0x26ae821b,0x03eba467),LL(0x3df1cf83,0x67a33fda), LL(0xb8421b7a,0x010813cf),LL(0x98cd6d76,0x7b0f5070),LL(0x1fe4b600,0x907320b3),LL(0x98dd3239,0xda3bfeb3),LL(0x41abb34c,0x23f1ed16),LL(0x946f85f1,0x01b30f29), + LL(0x9666f8ea,0x28524830),LL(0xeff6502b,0xd579b3df),LL(0x00e4f78a,0x3a66fa99),LL(0x54a3f7a3,0xfd8a65bb),LL(0x1965a79a,0x505d3f63),LL(0x1891469a,0x9524972c), LL(0x3354da3a,0x78367cbc),LL(0xe4941c6e,0xbfe1fe3d),LL(0xf5af173e,0xe41bb3f6),LL(0x5ca36597,0x57cb03ca),LL(0x7b99f795,0x27f86cb8),LL(0xad4dcef1,0x5cae6514), + LL(0xd87a86c9,0x4952a350),LL(0x034f45f9,0x08ed7da7),LL(0x2bd716d0,0x1e9ff827),LL(0xf1d9331f,0x2471fd15),LL(0xd7370b75,0x0c708362),LL(0xfc1a1051,0xaddedde6), LL(0xdb27b186,0xf4475288),LL(0x3760bc11,0x5be4d46b),LL(0x06d47ee1,0xe44435d9),LL(0xd0b7c8a2,0x865cf7c8),LL(0x8d31a252,0xdb412be0),LL(0x2f24d71c,0x4b90a932), + LL(0x55f0ad9f,0x38a49bf5),LL(0xf3618639,0x1a84c6b8),LL(0x01b2f7c2,0x5f709eca),LL(0x5be8359f,0xc479a650),LL(0xd6646b3f,0x6b6a22bf),LL(0xcc5b711b,0xcce78878), LL(0xb446cc63,0x8e7dbc63),LL(0x218f800f,0x231bd027),LL(0x030271eb,0x2d3a7e04),LL(0xe22fb3c7,0xb08b5978),LL(0x9be0d46c,0x860d6278),LL(0x1d49a915,0x253a31c2), + LL(0x376bdc3f,0xbad6b669),LL(0x23a9ff38,0xc4a8e7bc),LL(0x555fb0a3,0x3f54d8c4),LL(0x2b23db1b,0xfb3d5e1d),LL(0xf0d7eba9,0x6379f78a),LL(0xfa0beffa,0x36004feb), LL(0x334ff01a,0xdf0a373c),LL(0xdff12a1c,0x10314749),LL(0xf184c1b3,0x1d52ddc7),LL(0xab02d404,0x79431663),LL(0x7f4d3795,0x1a6488c1),LL(0x7cca9102,0x3363660f), + LL(0x04dc109c,0x56d65156),LL(0x09c1d307,0xcd740cc8),LL(0x10dfaead,0xef9e049f),LL(0xe30b70b8,0x19750b3a),LL(0x15c6a562,0x11ed8600),LL(0x12097026,0x53bdf97e), LL(0x6c0d908f,0x79559d05),LL(0xb506d527,0x8f1d75ba),LL(0xae8fb3c4,0xd6fd7323),LL(0xa4111f88,0x834639c9),LL(0xa310a683,0xfc69a029),LL(0x255f2e9a,0xa4467bbb), + LL(0xa3526fec,0x5f0a58a8),LL(0x02f028c1,0x849c171b),LL(0x56a5d3b5,0x34d77ce8),LL(0x54d5a92b,0x97016217),LL(0x2cc5b70b,0x0cce35c1),LL(0xe83f1f4a,0xd9d5a00a), LL(0x26a0368b,0x064223f8),LL(0x40e16452,0x328a9f69),LL(0xd305ad2c,0x3a6ac093),LL(0xbeba7c44,0x759d9a16),LL(0x637ce7c8,0x86021de7),LL(0xcc80c1cc,0x276bed61), + LL(0xbbe4bdda,0x66797f56),LL(0xda51b1a2,0xb92a369e),LL(0x31adb034,0x18eef4a5),LL(0x5d185cfc,0xcf1cb5ee),LL(0xbd53c27b,0xf596a59b),LL(0x69002569,0x1e1dd6f5), LL(0x7687e48a,0xd9433e79),LL(0x0cbcb9ce,0x7d8d24c2),LL(0x65d68ecd,0x233cd7ed),LL(0xfb2aded8,0x201bbe09),LL(0xac9b750e,0x987f4975),LL(0x337f7f25,0x949da385), + LL(0x9c7de99f,0x00c2ee1c),LL(0x15e50391,0x28a7461d),LL(0xa1c77952,0x1bdc0e32),LL(0xd53d640d,0xe98242c4),LL(0xcf153c7d,0x1a4724d2),LL(0xba477d46,0x194e5dcc), LL(0x3a0d4ccd,0x871c8cfe),LL(0x9af451fb,0x62010af0),LL(0x6ddec75c,0x9b354f9e),LL(0x680e3511,0xe5db0a5d),LL(0xd247745b,0x183d1270),LL(0xeecf52dc,0x9910867a), + LL(0x0410ca0b,0x6fb6b7bc),LL(0xb3c13935,0x0e16eed2),LL(0x316ff531,0x98ad89d8),LL(0x9894d65f,0x4800ee17),LL(0x48280170,0x034ea3c4),LL(0xc30be537,0x8126d12d), LL(0x5120e525,0x43c2d27e),LL(0xee65df90,0x96a5d498),LL(0x5eaef29a,0x65454010),LL(0x7b678fc7,0x1d8f07a1),LL(0x7b301270,0x54bc6f73),LL(0xe9473365,0xe58a8102), + LL(0x4ac6cf02,0x460c2990),LL(0x420a35b7,0x01482cfa),LL(0x34680972,0xf793933a),LL(0x17e2367b,0x2cd1f500),LL(0x3944f060,0x2411c352),LL(0x11c06b05,0x3d58b974), LL(0xcddebb3b,0x4552e369),LL(0x009aeab9,0xe1c38aec),LL(0x353b6e4f,0x9d34737c),LL(0xb16d7b0c,0xf2c99e2c),LL(0x7bbba6a2,0x57029fa4),LL(0xd13ef64d,0x0565d1bc), + LL(0xa329deb1,0x3561b3fe),LL(0xf1c3c3e4,0xfdca0e34),LL(0x47fb79d6,0x43748313),LL(0xc48002ed,0xa7f497e1),LL(0x2c44dcb0,0x86221cce),LL(0x43785e06,0x65e3f046), LL(0xdf4cf461,0x9ee9061f),LL(0xf022d27a,0xc7479e8c),LL(0x76f7f52b,0x1d8de85b),LL(0x0fd6c65f,0x39a713c9),LL(0x711f8a39,0xf74ca067),LL(0x8ebc640a,0xad1119ad), + LL(0x8ba85dff,0x59753bf6),LL(0xd1c89bce,0xec8b82ef),LL(0xb8b6a683,0xd7f1a651),LL(0x6f84416d,0x9c329bf3),LL(0xe68db225,0xaecbf4b9),LL(0x5a614d23,0x94ec3b0f), LL(0x93a9543d,0xbcb66725),LL(0xf19132ed,0x90c46c46),LL(0x950b080f,0x4767c73c),LL(0x971fd9e5,0x0b9b143e),LL(0x8ec8c68d,0xbce6886f),LL(0xd47f512e,0x167b0f8a), + LL(0xcd74009d,0x1942c2ff),LL(0xe9c286a4,0x71c4d5f5),LL(0x771a5972,0xf3c152b5),LL(0x363c2048,0x4cfb1e74),LL(0x9ddb8da2,0xcd2ce824),LL(0xa5ee443d,0x5d97c8e0), LL(0x68d7b3d5,0x6fa84b3d),LL(0x9ce14ec3,0x97eaa76d),LL(0x8e13869d,0x2e457136),LL(0x96f0f8a1,0x39ac6a0c),LL(0x42d93dc0,0xe24458ac),LL(0x5f60bec9,0x7eb3689d), + LL(0x588360ec,0x29544fe8),LL(0xffb550ea,0xa1aa9b9f),LL(0x4af4d28d,0xe1f6cf99),LL(0x0c6fd477,0x723d48b0),LL(0x5c81b252,0xf51f1b2f),LL(0x4f5a33ee,0x88ec11c0), LL(0x2cd72de4,0x7747f043),LL(0xd71c92c1,0xcca69b0a),LL(0x4e8cc763,0x9455d86e),LL(0xc08444e0,0xc9e0aa1b),LL(0xe8fffa63,0x93803b68),LL(0x2d781b7d,0xc296af29), + LL(0xb08d2d0e,0x514cc1dc),LL(0x30e93536,0x4e6b379e),LL(0x2fc9230f,0xf0e422ac),LL(0x92e23e21,0xaa50a1ad),LL(0x676d1ac0,0x70ac46d8),LL(0xf9f54493,0x698b9991), LL(0x8649519f,0x59a6b86a),LL(0xe3511da4,0xc1f11ad6),LL(0x3192968c,0xd3d9cff1),LL(0x0b342dd8,0x13e700b4),LL(0x3b1da441,0xfd5dc7bb),LL(0x2c883760,0x02426e7c), +}, +/* digit=56 base_pwr=2^280 */ +{ + LL(0x1c4c9d90,0x9e9af315),LL(0xd12e0a89,0x8665c5a9),LL(0x58286493,0x204abd92),LL(0xb2e09205,0x79959889),LL(0xfe56b101,0x0c727a3d),LL(0x8b657f26,0xf366244c), LL(0xcca65be2,0xde35d954),LL(0xb0fd41ce,0x52ee1230),LL(0x36019fee,0xfa03261f),LL(0x66511d8f,0xafda42d9),LL(0x821148b9,0xf63211dd),LL(0x6f13a3e1,0x7b56af7e), + LL(0xcc8998d6,0x3997900e),LL(0xbaa60da1,0x8fa564b7),LL(0x661f3c57,0x71bf5b0a),LL(0xaab1292b,0x44b13388),LL(0xd4d993f2,0xcbe80cb9),LL(0x2203f966,0x0b19b4c9), LL(0x0080f259,0xbc82a652),LL(0xad96dea3,0x870ebc08),LL(0x502f0003,0xa388c7e7),LL(0x56a38f73,0x9c704ef0),LL(0x3487d9b0,0x93cde8a7),LL(0xec11a1f3,0x5e9148b0), + LL(0x5913e184,0x47fe4799),LL(0x82145900,0x5bbe584c),LL(0x9a867173,0xb76cfa8b),LL(0x514bf471,0x9bc87bf0),LL(0x71dcf1fc,0x37392dce),LL(0x3ad1efa8,0xec3efae0), LL(0x14876451,0xbbea5a34),LL(0x6217090f,0x96e5f543),LL(0x9b1665a9,0x5b3d4ecd),LL(0xe329df22,0xe7b0df26),LL(0x0baa808d,0x18fb438e),LL(0xdd516faf,0x90757ebf), + LL(0xa748b8f5,0x63f27a25),LL(0x2cd246c4,0x68c8f3ec),LL(0x65f9ce38,0x5d317cd9),LL(0x635ba300,0x162c92e0),LL(0xfe343662,0x5259f64f),LL(0x8e614ac8,0x4a6b2b66), LL(0x01177c3b,0x97fb55bb),LL(0xa705cb01,0xfb586c21),LL(0x78061824,0xa57e7325),LL(0x6c1e6306,0x892f6b38),LL(0x2367b14c,0xf12e4c07),LL(0xc83a48c5,0x580d5fe2), + LL(0xd5a98d68,0x1e6f9a95),LL(0x849da828,0x759ea7df),LL(0x6e8b4198,0x365d5625),LL(0x7a4a53f9,0xe1b9c53b),LL(0xe32b9b16,0x55dc1d50),LL(0xbb6d5701,0xa4657ebb), LL(0xeacc76e2,0x4c270249),LL(0x162b1cc7,0xbe49ec75),LL(0x0689902b,0x19a95b61),LL(0xa4cfc5a8,0xdd5706bf),LL(0x14e5b424,0xd33bdb73),LL(0xe69eba87,0x21311bd1), + LL(0xea2bafb3,0x6897401c),LL(0x15c56fe4,0x7b96ecc2),LL(0x39e2b43b,0xe511b329),LL(0xbf809331,0x39522861),LL(0xc958f8f4,0x815f6c1d),LL(0xc213e727,0x2abbdf6b), LL(0xc39bc01f,0xeb09ae59),LL(0x676b56a5,0xffe3b831),LL(0xa20f86c6,0x8f4815a2),LL(0x9aa30807,0x748a1766),LL(0x1b758878,0xf1f46a21),LL(0x6f6fc3d7,0xbd421fe7), + LL(0x72a21acc,0x75ba2f9b),LL(0xa28edb4c,0x356688d4),LL(0x610d080f,0x3c339e0b),LL(0x33a99c2f,0x614ac293),LL(0xaa580aff,0xa5e23af2),LL(0xe1fdba3a,0xa6bcb860), LL(0xb43f9425,0xaa603365),LL(0xf7ee4635,0xae8d7126),LL(0x56330a32,0xa2b25244),LL(0x9e025aa3,0xc396b5bb),LL(0xf8a0d5cf,0xabbf77fa),LL(0xea31c83b,0xb322ee30), + LL(0x0d6ded89,0x300b0484),LL(0xc3ab55ed,0x0b1092cb),LL(0x0cc10a74,0x17d9c542),LL(0xeff9d010,0x7f637e84),LL(0x27aa1285,0xd732aa1e),LL(0xe2a77114,0xedb97340), LL(0x5ef4dfb0,0x62acf158),LL(0xba1d7b81,0x1e94fc6e),LL(0x2e6eb2db,0x88bec5d2),LL(0x8d18263d,0xaec27202),LL(0xe4bbd6ac,0x4b687353),LL(0x0ff7e4c0,0x031be351), + LL(0x7890e234,0x04881384),LL(0x672e70c6,0x387f1159),LL(0x7b307f75,0x1468a614),LL(0xed85ec96,0x56335b52),LL(0xd45bcae9,0xda1bb60f),LL(0xf9faeadd,0x4d94f3f0), LL(0xfc78d86b,0x6c6a7183),LL(0x3018dec6,0xa425b5c7),LL(0x2d877399,0xb1549c33),LL(0x92b2bc37,0x6c41c50c),LL(0x83ee0ddb,0x3a9f380c),LL(0xc4599e73,0xded5feb6), + LL(0xf086d06c,0x6c00f388),LL(0x5add0cf4,0x17ee4503),LL(0x07caf89c,0xf96984c7),LL(0x648ed5e9,0x9d49d667),LL(0xa0164881,0x3ef95015),LL(0x7d9c651f,0x39e28e44), LL(0x59f37780,0xb13ad240),LL(0xb9522225,0x08cee349),LL(0x2ba1b214,0x9245ee6f),LL(0xa886d8d2,0x12bedaa9),LL(0xfcb8186f,0xe139ae08),LL(0xfc2ef864,0x99203fb6), + LL(0x0b7f8354,0x14d34c21),LL(0x9177ce45,0x1475a1cd),LL(0x9b926e4b,0x9f5f764a),LL(0x05dd21fe,0x77260d1e),LL(0xc4b937f7,0x3c882480),LL(0x722372f2,0xc92dcd39), LL(0xec6f657e,0xf636a1be),LL(0x1d30dd35,0xb0e6c312),LL(0xe4654efe,0xfe4b0528),LL(0x21d230d2,0x1c4a6820),LL(0x98fa45ab,0x615d2e48),LL(0x01fdbabf,0x1f35d6d8), + LL(0x64c9323d,0x3c292847),LL(0x0491f77d,0x40115a89),LL(0x2d7a05f5,0xec141ade),LL(0x222a5f9f,0x0c35e4d9),LL(0x442a3e9b,0x5ea51791),LL(0xe51b841e,0x17e68ece), LL(0xd6ae9174,0x415c0f6c),LL(0x9ffd7595,0xe6df85f8),LL(0x8dedf59c,0x65fc694f),LL(0xfee92718,0xc609503e),LL(0x97d565ae,0x57d2592e),LL(0x7e20862b,0xb761bf15), + LL(0x3a7b10d1,0xa636eeb8),LL(0xf4a29e73,0x4e1ae352),LL(0xe6bb1ec7,0x01704f5f),LL(0x0ef020ae,0x75c04f72),LL(0x5a31e6a6,0x448d8cee),LL(0x208f994b,0xe40a9c29), LL(0xfd8f9d5d,0x69e09a30),LL(0x449bab7e,0xe6a5f7eb),LL(0x2aa1768b,0xf25bc18a),LL(0x3c841234,0x9449e404),LL(0x016a7bef,0x7a3bf43e),LL(0x2a150b60,0xf25803e8), + LL(0x82376117,0xd443b265),LL(0x1a1beb0d,0xb91087c1),LL(0x45cc5951,0x3fe62a65),LL(0xe6e472d5,0x49c754bc),LL(0x77c424eb,0x7e60bb81),LL(0x830cbb97,0xbcd4088e), LL(0xba26df7b,0x3da5c94e),LL(0xf72b4338,0x508b4f55),LL(0x69ad7784,0x409c5c74),LL(0xfdf44d6a,0x82e5f1b0),LL(0xeed2766f,0x10654a1c),LL(0xa6e83f4a,0xef1e65fa), + LL(0xb215f9e0,0xe44a2a57),LL(0x19066f0a,0x38b34dce),LL(0x40bb1bfb,0x8bb91dad),LL(0xe67735fc,0x64c9f775),LL(0x88d613cd,0xde142417),LL(0x1901d88d,0xc5014ff5), LL(0xf38116b0,0xa250341d),LL(0x9d6cbcb2,0xf96b9dd4),LL(0x76b3fac2,0x15ec6c72),LL(0x8124c1e9,0x88f1952f),LL(0x975be4f5,0x6b72f8ea),LL(0x061f7530,0x23d288ff), + LL(0x5f56dc3c,0xa6e19d0a),LL(0x0b88326a,0xe387e269),LL(0x0ee527a4,0xef738095),LL(0x7c4278a6,0x78b7174b),LL(0xe70798ff,0xc133d867),LL(0x9e9230ca,0x9d0fef75), LL(0x1a955ab9,0x7431eef0),LL(0x8868d922,0x3772e703),LL(0x8d6af3f7,0xf7a4306a),LL(0xbbec076a,0x633bb5a0),LL(0x7a257ca3,0x6d07623e),LL(0x21c00663,0xffb5e165), +}, +/* digit=57 base_pwr=2^285 */ +{ + LL(0xbb71e9b0,0x35ae891b),LL(0x522b77f0,0x1f6ce6ca),LL(0xe63745c4,0xc2dab3ca),LL(0xf218d139,0x55b8c185),LL(0x89f3b0e2,0x6ab039c8),LL(0xc644c3fa,0xd9e25bfd), LL(0x3e2ed47b,0xc8496f20),LL(0x8d67e17c,0xc395ec02),LL(0x92114918,0x5c678392),LL(0xef73f345,0xe962e52f),LL(0x54fcfb22,0x3818baf3),LL(0x9d4bc911,0x4d75d65d), + LL(0x022ea83a,0x7f1f985c),LL(0xa7584e7f,0x90a22662),LL(0x5188fcf6,0xb40a930a),LL(0xa3a82904,0x3fad79ab),LL(0xf3151027,0x7bee8d22),LL(0xc2c3e17b,0x79a1a838), LL(0x33cc3509,0x1fbe06e9),LL(0x9abd5cca,0x629c56aa),LL(0x2d9cf7a5,0xfff290ec),LL(0x9bd062c5,0x5d0dedaa),LL(0xd7d35381,0x080344ab),LL(0xf5cf9eda,0x0848373a), + LL(0xa1b31832,0x170098de),LL(0x2cd8c540,0xc6876bf9),LL(0x8660d773,0x35b1b04b),LL(0x5152ad1e,0x8b4b6c4b),LL(0x4a8e92aa,0x0b0bd1c2),LL(0x88172f4e,0xd814f6f4), LL(0x41ac83d1,0x13a76899),LL(0xf1d80357,0x355933b4),LL(0x3dc17e72,0x4b394f97),LL(0x0a7124b2,0x1b0cd166),LL(0xce8a372c,0x7549880c),LL(0xe712603b,0x79f53f7e), + LL(0x7a0c0bc0,0x31d31f7a),LL(0xb251d2bf,0x7a37a84a),LL(0x52f04d67,0x1793362e),LL(0x21c7b651,0x5808e709),LL(0xed6f47f6,0x33fe9123),LL(0x58f71405,0xdeb1dde9), LL(0xae56b472,0x821d3045),LL(0xe02043ad,0x9f61f761),LL(0x5b2048a9,0x932ddb14),LL(0xd7811330,0x17d989fe),LL(0x128fd85f,0x032ae4cb),LL(0x7d1ef434,0x8f1956b4), + LL(0xce1d819c,0x30cc44e0),LL(0x15a3a414,0x3c2c4f04),LL(0x665b634b,0x7f06d2c2),LL(0x7df0bd69,0xf609fe64),LL(0x6a0dbf94,0x4b0c8c36),LL(0x46e9b487,0xd0a3a752), LL(0x9578a83f,0x7407e157),LL(0xfe0f975f,0xc46ffa4f),LL(0x1efaeccf,0xb086f199),LL(0x64796c18,0xf984b69d),LL(0x8ebfb97d,0x14de44fa),LL(0xf5245ba6,0x47bcb675), + LL(0x0dbafc49,0xed58c1ab),LL(0x6fa8b473,0x41995a5a),LL(0x85450ccb,0x3af3853d),LL(0x601fe529,0x18d1a7be),LL(0xdab2d926,0xd196520f),LL(0x56213ae9,0xd1f2c1e9), LL(0x77ded01d,0x37218255),LL(0xa7905433,0x366c552d),LL(0x7d9430d1,0x023d0373),LL(0x9ddf69ea,0x65d70cb6),LL(0xd4d3fa24,0x6321df7a),LL(0xb91084c3,0x77b1c2bf), + LL(0x0aef0ff7,0x2434dbdf),LL(0x9cc3d2ba,0xca74413e),LL(0x3dcbaeb9,0x0aee0d65),LL(0x4184a72e,0x89568076),LL(0xbe7d4e0f,0x3d7e4f61),LL(0xae441b4c,0x284a8608), LL(0x6a67283c,0x7274fd92),LL(0xc06e0b84,0xa222be15),LL(0xe4623e88,0xa0713cd7),LL(0x4710aa33,0x0ff4a317),LL(0x044dcecd,0xc9a35a65),LL(0xde77a24b,0x7bae1eca), + LL(0x16973cf4,0x070d34e1),LL(0x7e4f34f7,0x20aee08b),LL(0x5eb8ad29,0x269af9b9),LL(0xa6a45dda,0xdde0a036),LL(0x63df41e0,0xa18b528e),LL(0xa260df2a,0x03cc71b2), LL(0xa06b1dd7,0x24a6770a),LL(0x9d2675d3,0x5bfa9c11),LL(0x96844432,0x73c1e2a1),LL(0x131a6cf0,0x3660558d),LL(0x2ee79454,0xb0289c83),LL(0xc6d8ddcd,0xa6aefb01), + LL(0x81069df0,0xda10f552),LL(0x560ea55a,0x70088f4e),LL(0xda8cbaa5,0xdcee31af),LL(0xbe16a7b7,0x1213b76a),LL(0x781114d1,0x4fffa388),LL(0x904479d2,0x2cc19aa9), LL(0x85eec3cd,0xc4ebf9be),LL(0xbaff7431,0x48ea527a),LL(0x52d3ce22,0xa5b6fdca),LL(0x7e139d7a,0xbcb07835),LL(0x5de5de75,0xc7f17551),LL(0x15bcf7aa,0x005ab4d9), + LL(0x41383866,0x041511b0),LL(0xdb9c2b4c,0x7c90692a),LL(0x61deb106,0x736cc6af),LL(0xb5b84abe,0x57d428cb),LL(0x35dbcca9,0x14bc4981),LL(0x37f3ef2e,0xece90041), LL(0x85066bd0,0xe2c33711),LL(0xabef34c2,0x335bd50b),LL(0xb017a337,0x755f513d),LL(0x284445ad,0x28d263e2),LL(0xecb8436e,0x83402ba9),LL(0xf31d28d4,0x66357324), + LL(0xbce3cea0,0x05fb9811),LL(0xdf2b07df,0x74adb5b5),LL(0x0c4e5e2a,0x870e63b8),LL(0xe5fec889,0xe873cba7),LL(0x19cd8a3e,0x28b0e005),LL(0x48112c8c,0xbb16491b), LL(0x5df42faa,0xddb8cfb5),LL(0x2ab1a097,0x35f952ae),LL(0xd2dfa18f,0x35e885e1),LL(0x5b4a0277,0xc3ec2325),LL(0xc5aecee0,0xdbe2e40c),LL(0x0690080a,0x133383c9), + LL(0x7521f457,0xe5d473dc),LL(0xa00be577,0xe9ef09bd),LL(0xb6eaa640,0xf6d0965f),LL(0x75726560,0xeb494868),LL(0x28817302,0x452116d5),LL(0xfbde3597,0xf0424fdb), LL(0xbb454915,0xd6096da3),LL(0x41422141,0xde482808),LL(0x2d19fac0,0x7a135197),LL(0x21393f6f,0xdc9a5ec4),LL(0xeb2c8ada,0xcabcc1e3),LL(0x42d8c4f2,0xd4366431), + LL(0x34246834,0x46c7b438),LL(0x2eb9f9bc,0xa0a570c3),LL(0x586a0fdd,0x3e01db21),LL(0x9d21192c,0x15732b07),LL(0xfd747dd8,0xe544b5c9),LL(0x7ed374fb,0xeee7e104), LL(0x3b75a586,0xfa619086),LL(0xe8a4cfd4,0x9d0909f4),LL(0xccd93fa4,0x30e0e476),LL(0xa14595d4,0x5bdc5c02),LL(0xe245400f,0xf6f8ab6a),LL(0xa4198419,0x7bd194ce), + LL(0x68e190a6,0x713a654c),LL(0x3037a8e6,0x6317f6ab),LL(0x6b6bafef,0x005f412a),LL(0xb53eb41a,0xf67f8201),LL(0xf51ccf93,0x96585fcb),LL(0x045104e0,0x68ed706e), LL(0x40db03d9,0x535dc2a7),LL(0xda583d56,0xb9f0b69d),LL(0x414dc76a,0xeed71cb1),LL(0xc5e7698e,0x7368aabd),LL(0x4689da93,0x3eb6a4f1),LL(0xfcb73b42,0x753fa65a), + LL(0x9b860b0f,0xf5131aad),LL(0xd9bcf9c9,0x19af93ce),LL(0x307ca1ca,0x69b9765d),LL(0x87936c0b,0x0b735808),LL(0x1107f1cb,0x10eaf7ae),LL(0x16dfe03b,0x15211ad5), LL(0xb11c8af6,0x9bde40d5),LL(0xed1fc8be,0x513ea01a),LL(0x5412ab86,0x2f04be27),LL(0x9de582fc,0x2a1adc4a),LL(0x00de0043,0xb0c43235),LL(0x006efb33,0x689e15e4), + LL(0x89e4e449,0x0ed1082f),LL(0x833f2378,0xdb1fb471),LL(0xece77352,0xa35fef0e),LL(0x4bf0c426,0x76adaa46),LL(0xa011b2fb,0xfbab929a),LL(0x9d8cc4d3,0x6f475d5b), LL(0x74351480,0xbe6d7f21),LL(0x93e4a7ae,0x2d1362d1),LL(0x106ceaab,0xc7e2cba5),LL(0x45258697,0xfe94528a),LL(0x075945b0,0x7109b17d),LL(0xcae17f7a,0xfd395b2c), +}, +/* digit=58 base_pwr=2^290 */ +{ + LL(0x310404a8,0xfc1faedc),LL(0xd3bcb128,0xea339148),LL(0x6416defd,0xf0048545),LL(0xc58653e7,0x75de7770),LL(0xe2f6f99e,0xdd2dcbeb),LL(0xd159ac07,0xa4380ef4), LL(0xe4173608,0x45dd713c),LL(0x446a6789,0x44919b61),LL(0x6b962b38,0x3f73756b),LL(0xbffd3f0c,0x3cb9f53b),LL(0x7f08ebae,0xd723c40b),LL(0x0c3cddba,0x998a9b17), + LL(0xf6ba1469,0xf6377e3b),LL(0x09c832d3,0xc334fb6c),LL(0xc21c0cf1,0x7f85ac42),LL(0x857d8edb,0x7a3e31c9),LL(0x27b77ed6,0x2eb10763),LL(0x38dae10b,0x2bfbbdbc), LL(0x7bae3b4f,0xed7c6fb1),LL(0x36d04e6f,0xc5911d9f),LL(0x4569e72f,0x4dc43550),LL(0xbedae3ab,0xaa82fb97),LL(0x4f27e463,0x06d37bef),LL(0xf0c35a11,0xd0dbce6d), + LL(0x661019ec,0x43c78835),LL(0x24e66d29,0x68e916b1),LL(0x24094671,0x02c0f3a2),LL(0xd0f17d86,0xab6f1c05),LL(0xa22d4264,0x6d3bac72),LL(0xf6e5fafe,0xd7b8f152), LL(0x39447eb3,0x95627c63),LL(0x79e1ff93,0xfd159018),LL(0x5ad80806,0x39277c83),LL(0x0d7c7b74,0x758aafc9),LL(0x4cb8bec9,0x605ad8ca),LL(0x5741828a,0x6a90085c), + LL(0xff8b1fbe,0x6edf5561),LL(0xf6eac0c0,0x614b788e),LL(0xd8d66d29,0x7699ae56),LL(0xe9d58eb2,0x5f81602a),LL(0xfaf9176d,0xd0c04874),LL(0x523153b1,0x4b3a0046), LL(0xf6315883,0x9690930f),LL(0xa60ca92d,0xa81c0b44),LL(0x73bcba90,0x2d0e7258),LL(0xe902e329,0x57efe72d),LL(0x76bc27b9,0x3fcd5986),LL(0x93940c09,0x492adf03), + LL(0xf820c195,0x6895dbe2),LL(0x3f6c7b40,0x3787a500),LL(0xac1e90f3,0xdc718243),LL(0xba5d0870,0x352f8c91),LL(0xec0112b5,0xf3d1c53e),LL(0x6b84f64a,0x08a0782f), LL(0x8eedd5d4,0xd659e635),LL(0x29537276,0xfc30df6c),LL(0xa1755ce0,0xbfb09978),LL(0xaa2b4187,0x227f7b12),LL(0x226539d2,0x828730b9),LL(0xb2472c95,0x9051a37c), + LL(0xb39317b8,0x7de743e2),LL(0x2d372acf,0x9205d447),LL(0x3eeb0012,0x8226fc30),LL(0x2af74be6,0xab2a3e05),LL(0x4af91ac0,0xbe476780),LL(0x0ca36bf4,0x98497c71), LL(0x8d6dedb4,0x74fdf7cd),LL(0xa0fc5919,0xb50778ee),LL(0x2fcd7c63,0x5d5ec33f),LL(0x7f33cde0,0x667b8193),LL(0x38364d44,0xce48ae4b),LL(0x223ed67e,0xb8578963), + LL(0x9567f5f4,0x4b20d182),LL(0x18f02b34,0xde7e8149),LL(0xecff9dd7,0xc9a4be7b),LL(0x9812fd3f,0xe2f70bbe),LL(0x9c889263,0x471bf90c),LL(0x3e61f5bf,0xb60d01b5), LL(0xd22d855b,0x258c7f89),LL(0xb75a7d4f,0x35ef5c15),LL(0xb247f27d,0x26d8e1da),LL(0x8d0f7757,0xcf136199),LL(0x3f8e894d,0x31244780),LL(0xe1a3d47d,0x8d2a20ba), + LL(0xc795a2c7,0xc68b641e),LL(0x5a4d6647,0x4fe559b1),LL(0xd89ce668,0xeda98cba),LL(0x6c269d8e,0x15f84dc0),LL(0xcbf34023,0xf0eb685e),LL(0xc032634a,0x3668c530), LL(0xe4531f59,0x2e3d7fff),LL(0x85494d06,0xe6270306),LL(0xa3e050df,0xf02cabcf),LL(0xc001dcd9,0xccd2da67),LL(0x066d2d52,0x50aa3723),LL(0x7224a41f,0xdb075650), + LL(0x43c1e526,0xeaf0b05d),LL(0x5c673e17,0x7eddb25a),LL(0xc1f9bda7,0xfce700de),LL(0x646a1550,0x4e6f4c70),LL(0x8ba60c35,0xae06c3e5),LL(0xc0c89c50,0x1a57101f), LL(0x60a36192,0xdb35b1f3),LL(0x265c3ed1,0xf740a467),LL(0x6175e771,0xeab65a88),LL(0xcbe28ed4,0xd94c8cc6),LL(0x465dd7a8,0xac347e7b),LL(0xca9b899d,0xca927337), + LL(0xdd045c9e,0x3499422e),LL(0x727f1f25,0x5119c675),LL(0xd76e7077,0xf44a89ae),LL(0x8eb17f39,0x9d6967b5),LL(0x2bbed398,0x49b39b81),LL(0x07deb5e9,0x05b3a455), LL(0x62039ba0,0x8fa1ebf3),LL(0x0b650544,0x52c3d14b),LL(0x37d3fd20,0x2603aaf7),LL(0xec70bfc0,0xb6fe1148),LL(0xfeb74df8,0xe68b8475),LL(0xe1f4aba2,0x56fafdcb), + LL(0xa05b2adf,0xd061bce6),LL(0x8b7cf4c6,0xdb6e9324),LL(0x189f7ce1,0x6344c3b6),LL(0x3cfe678d,0x06993dc9),LL(0xb6e22f1b,0xb00b5227),LL(0x780137c1,0x8f836e4c), LL(0x1e9ad145,0xaf822953),LL(0xdfaaa159,0xb438997c),LL(0x904396e6,0x03e27459),LL(0xc8259d86,0xd06a0fdb),LL(0xe040907b,0x8afe766b),LL(0x038fbdba,0x0f5ff265), + LL(0xe1aa5953,0x4b1f8698),LL(0xbf0a531c,0x94af5c0c),LL(0x6536e156,0xba1c49f5),LL(0x09c95a75,0x31c31631),LL(0x0a6d9af1,0x268a308f),LL(0xfa9ca8b7,0x185b9762), LL(0x65035840,0xc3b1cbcd),LL(0xb429c82a,0x17b276c6),LL(0x34396850,0x1b4ec7b3),LL(0x048a6943,0x42f21d5a),LL(0x5b5820bf,0x0bf948f5),LL(0xb48dca58,0xac944487), + LL(0xac575242,0xefa91b2e),LL(0xcdaca77a,0x58280b99),LL(0xcf39d117,0x5c3f8382),LL(0x70e5d17c,0x4d460dba),LL(0x12f6d6ad,0x440b4db8),LL(0x7f3a6bb8,0x14b9b335), LL(0x837833cf,0xfbb3746a),LL(0xda482df0,0xbbf4735b),LL(0x9fbf146f,0x4efe32c0),LL(0x346e3b9e,0x65e5d7c2),LL(0x434ce5d5,0x742e81ce),LL(0x3bb80bd5,0x87e2b772), + LL(0xe02f8540,0x18d0983b),LL(0xe8058c66,0x708654c7),LL(0x92894720,0x0b137f68),LL(0x4345444d,0x0b8b62e9),LL(0xbfd043d0,0x426807d3),LL(0x2a1ac390,0x17ac2e64), LL(0xf2076839,0xd037e79c),LL(0x3ba4d3f2,0x991d4ff0),LL(0xd3724bfa,0x3937247e),LL(0x218413d1,0xb5b50d34),LL(0x28157b81,0x2accc172),LL(0xc472e8b2,0xd07f18b7), + LL(0xca12be15,0x0df64ab2),LL(0xc2f0d0f9,0xdbac68f8),LL(0xa2107d3e,0x5772ba8b),LL(0x76724cfe,0x32c43ac9),LL(0x166229b7,0x6f4a367e),LL(0x230b3527,0x9c8bdb42), LL(0xe71f261d,0x343d644d),LL(0x26283547,0x45c849bf),LL(0xd8e4c7b0,0x3a42b9b8),LL(0x9f8af5c7,0x0aff84ad),LL(0x418cfee8,0x453c3a30),LL(0xa241633a,0x6be57cdb), + LL(0x65b55050,0x04418b59),LL(0xd324aa67,0xa8a797c3),LL(0x7c65a6d9,0x5f87e22c),LL(0x1dbeffe4,0xaac71065),LL(0xbd3cc05c,0xff619d64),LL(0xe65c92c4,0x9a29c966), LL(0xdad7fcbd,0x23af2b21),LL(0x153b817f,0x4950a767),LL(0xc6478c55,0xc34a7efa),LL(0xf6cd140e,0x57cde95a),LL(0xf5a0db2e,0x64b74575),LL(0x75d7fb76,0xd4b5ea52), +}, +/* digit=59 base_pwr=2^295 */ +{ + LL(0x8e72aafb,0x28405062),LL(0x8ea8bf00,0x655bf353),LL(0x05547f7b,0x789d9444),LL(0x3441e472,0x7fa445ed),LL(0x4a44ce87,0xfeb19825),LL(0x129aed14,0xccb5f12c), LL(0xaf94fb34,0x22b05de3),LL(0xd3f03199,0x7422a040),LL(0xa83f7f08,0xfba252ca),LL(0xcefaa757,0x0f6ad6e6),LL(0x6517d806,0xe1ad1871),LL(0x8e9d97ad,0xd16dc8ed), + LL(0xa3736eb6,0xcbb99194),LL(0x36dcf470,0xdd5161cd),LL(0xb6ab6c03,0xd50b24aa),LL(0xbc41f4b7,0x419d2810),LL(0x295496cf,0xe2e88d7a),LL(0xf2457ac0,0x350713f2), LL(0x0427e2ae,0x838e4a36),LL(0x4d974e5a,0x7631472a),LL(0x7a5c5fdc,0x9fa3ab1c),LL(0xde34cb8d,0x324798cd),LL(0x889105fe,0xbfa5a9d0),LL(0xfd0df249,0xd05dad34), + LL(0xcb419fc3,0x536e5657),LL(0x1d271dd1,0xe8c208bc),LL(0x22d2b9ad,0x6a3713bd),LL(0x471d808d,0xa4c761a7),LL(0x7e6dca35,0xd93aafb6),LL(0x8f55ca32,0xc46c0ae3), LL(0xa78bfca0,0x55dc0de7),LL(0x3407d0ca,0xe9cfb301),LL(0xb3256c14,0x777e2a60),LL(0x6d8fee02,0x32b2238c),LL(0x46e43ee8,0xe8b35396),LL(0x247985dd,0x310bc1ba), + LL(0x581f9d3c,0x9974759c),LL(0xe5cb1973,0x9e76a970),LL(0xc64941ca,0x8afec58e),LL(0x01d05575,0x2d7c57fa),LL(0x5c448db5,0xc07c74cc),LL(0x01bb1440,0xa52474ce), LL(0x00115bbb,0x93162d97),LL(0xfd7480f9,0x483b6147),LL(0x6af18ded,0x4f28c57e),LL(0x174a3089,0x36faed8f),LL(0xa3dd6265,0x702dbd64),LL(0x6adc0d7b,0x86a9c43f), + LL(0xb9de7b63,0xa3510710),LL(0x4019c9df,0x9f364ad1),LL(0x9b5bdce3,0x5b66a5d7),LL(0x78b1b385,0x2b2f6951),LL(0x3cfa9f99,0x3e4783d3),LL(0x6bd6bcf4,0x1af51750), LL(0x81d8d7ef,0xf9c0686a),LL(0x37c068d3,0xdc0f22ec),LL(0x93545faf,0xe1b86653),LL(0xa8a52881,0x37ca8501),LL(0x5603e359,0x07ac5c8a),LL(0x542cc937,0x98fb2bab), + LL(0x0c5bbd3e,0x4981be69),LL(0x185fdb55,0xb047df0a),LL(0x74cff00c,0x3168e050),LL(0xb52c7f9c,0x111150a1),LL(0xa51c7986,0x0db2ed84),LL(0xe61272ad,0x7d991630), LL(0x28de14dd,0x7443d936),LL(0xa5daed5f,0xfdf31f41),LL(0x866b5e40,0x71e0ef4e),LL(0xb965a670,0x05c57a45),LL(0x70e1aa77,0x85bdb58c),LL(0x9df3ce32,0xe4d1fe2a), + LL(0x2f722563,0x5d461898),LL(0x567db14c,0x11d22b39),LL(0x6779cd40,0x9a8f004e),LL(0x5473ecd5,0x0812ae3d),LL(0x4e6c296a,0x4ed82862),LL(0x064ee61f,0x2d9ce40c), LL(0xd8a9eb1e,0x4856d586),LL(0x5d1b5e3a,0x2ddd6b12),LL(0x382fba3c,0x0ab5eec0),LL(0xfcf4a9c8,0x302018df),LL(0xab3cdedb,0x7b4e6fd2),LL(0x8f64cb1d,0x266c246a), + LL(0xf5c3af59,0xc41e4aed),LL(0x6de9a78a,0xa0284ad0),LL(0x8ed812d2,0xf5eaab7b),LL(0x1afb58b8,0x7801fbb8),LL(0x71efcc3a,0xbe5cdba6),LL(0xcd10cb91,0xe31a0e3c), LL(0x85dc0bc6,0x882e821e),LL(0xbb32e506,0xd3ad070f),LL(0xd8a0f038,0x3afede2b),LL(0x857fd3a0,0xe20a117c),LL(0x3060f767,0xebaa2aa4),LL(0x2b9d1da1,0x6524aa0d), + LL(0x88cffe33,0x9cc5847c),LL(0x6e8eb6c1,0xff726b0f),LL(0x1bc45d8e,0x9bb2ca16),LL(0xa6d8a5a6,0xe7903009),LL(0x47db2201,0x4f089cc0),LL(0xe6b5928a,0x4135884d), LL(0xe5c017cf,0xb1a86a0a),LL(0xb0a393df,0xb1d9bf6d),LL(0x28bb3277,0x33d9c1c6),LL(0x45b582ce,0xcb05b67b),LL(0xf33792c5,0xa72585fc),LL(0xa7d1ed84,0x78b7c5e8), + LL(0x9346df25,0x70e478fb),LL(0xb4a4ada5,0x01dc0c2e),LL(0x5be36ea7,0xaec82b00),LL(0x6717e06a,0x82618b8f),LL(0x008f1977,0x2db1f6d4),LL(0x16b390d1,0x4e493f3b), LL(0x990a75eb,0xfe86fd4d),LL(0x783f6076,0xa1cf7f99),LL(0x0c049158,0x6cbb23e7),LL(0xed456235,0xd05be7e5),LL(0x9bd836ef,0x60137406),LL(0x32e5f604,0x94ec9644), + LL(0x3d87bfa8,0x3361e1ce),LL(0x8dcca4f0,0x92f235e7),LL(0xbe323fd1,0xc8084cb4),LL(0xc24c6d16,0x3fd481a5),LL(0x2cea81ba,0x9b1bd940),LL(0x0c5aa59f,0xf5091191), LL(0xf81d5e2a,0x4cd8c9ef),LL(0x1550bff4,0x5ad00013),LL(0x8cc32e55,0x29d47b9f),LL(0x11694ece,0x66e3e6f1),LL(0x7950dd7e,0xd5edf701),LL(0x0f6350c6,0x9ccb1096), + LL(0x18f2470c,0x09db138d),LL(0xf613658f,0x63bd2290),LL(0x4feebab9,0x0bb64779),LL(0x7fdb1e71,0xfce4aee1),LL(0xa7f1f65d,0x7d5c0c61),LL(0x8d02d6cd,0x46405b61), LL(0x6fdcb0d0,0x7cac0485),LL(0x2f8ec5af,0x85224c4b),LL(0xdb0aa864,0xb5879a59),LL(0xff94f8b5,0x75f391b8),LL(0x49c97f8e,0xa6c994ae),LL(0xd690b232,0x4d968fad), + LL(0x67e0b4e7,0xf5cd290a),LL(0x7c1594b6,0xaa6fa680),LL(0xb63270be,0xebedfbd7),LL(0xa369bfee,0x574b410b),LL(0x020ea888,0x431cba5a),LL(0x56c71d47,0xd3a3102f), LL(0xa90a853a,0x4894bfe0),LL(0x5f9c4b6b,0xd78bd98b),LL(0xd900c5c1,0x9b1324f6),LL(0x718c2147,0xc65c944d),LL(0xa987f634,0xf661de6b),LL(0x172628d8,0x0315e69f), + LL(0x22ea5f1c,0x07c60c75),LL(0xa36bee4f,0x35beae34),LL(0xdcba8997,0xa8b00a09),LL(0x802ce50c,0xa77f1f3a),LL(0x2a2144b0,0x6c4050df),LL(0xab1b10db,0xf79bfa96), LL(0x433a9b1c,0x9025d470),LL(0x90d9eec8,0xaf3e3917),LL(0x9ae2d535,0xbcad2d62),LL(0xeff0f6a9,0x7a152829),LL(0x925fa5a0,0xe87345cd),LL(0x0e84039c,0x6ce00720), + LL(0x07f6a05a,0xdae449c0),LL(0x5bf26c9e,0xbc1b84f5),LL(0xb1c13820,0xe3b3f9ed),LL(0x4090598b,0x5442ad5b),LL(0x13749e4d,0x794ef656),LL(0x948b71c5,0xde809180), LL(0xe203c5b5,0x4c72dc7d),LL(0x1b349fc4,0x8902b097),LL(0x225a1569,0xa899bedb),LL(0xe6ff3f53,0xeb7da73d),LL(0x7c0be37b,0x6ee8e160),LL(0xa31bf943,0x9ee667d2), + LL(0xdb81146d,0x5017e145),LL(0x45c54db8,0xc7d2086d),LL(0xfa98234a,0x2541059d),LL(0x9985af98,0x4bf344d9),LL(0x7b5b7b1c,0x39737ed6),LL(0x87c411ad,0x8e246919), LL(0xb877a75f,0x2fad8ced),LL(0x17e60ee2,0xe42352df),LL(0x404043f7,0x1a53d856),LL(0x863927a1,0x6c1f07a5),LL(0xb6892121,0x38d3a4f4),LL(0x01976c8f,0xf4c10920), +}, +/* digit=60 base_pwr=2^300 */ +{ + LL(0xc2ec3731,0xbcc88422),LL(0x10dc4ec2,0x78a3e4d4),LL(0x2571d6b1,0x745da1ef),LL(0x739a956e,0xf01c2921),LL(0xe4bffc16,0xeffd8065),LL(0xf36fe72c,0x6efe62a1), LL(0x0f4629a4,0xf49e90d2),LL(0x8ce646f4,0xadd1dcc7),LL(0xb7240d91,0xcb78b583),LL(0x03f8387f,0x2e1a7c3c),LL(0x3200f2d9,0x16566c22),LL(0xaaf80a84,0x2361b14b), + LL(0x246dc690,0x7a1a522b),LL(0x4b61ab70,0xb563cbe1),LL(0x3d4ac4ab,0x41bb4abe),LL(0x37f996e8,0xc52950b3),LL(0x79727761,0x01d991e6),LL(0x978fd7d2,0x35de93bd), LL(0x5706d336,0x86bad5e6),LL(0xe7f26c20,0x10844155),LL(0x05757453,0x58ffeb77),LL(0x3939df77,0xbb186129),LL(0x6a78ea0f,0xbfdd394a),LL(0x6e33e1d3,0x907ff054), + LL(0xb5733309,0xdb1cffd2),LL(0x0f9dd939,0x24bc250b),LL(0xa3c1db85,0xa4181e5a),LL(0xac55d391,0xe5183e51),LL(0xefd270d0,0x2793d5ef),LL(0xc0631546,0x7d56f63d), LL(0x0c1ee59d,0xecb40a59),LL(0xbb5bfa2c,0xe613a9e4),LL(0x6c5830f9,0xa89b14ab),LL(0xa03f201e,0x4dc477dc),LL(0xc88c54f6,0x5604f5da),LL(0x2acfc66e,0xd49264dc), + LL(0x0df93b34,0xa7f29532),LL(0x5c14df30,0x855934f2),LL(0xefae348c,0xd2f54ce9),LL(0xac52758d,0x5acb931c),LL(0xd22961a4,0x287b3e18),LL(0x748f8fe1,0x42a5516d), LL(0x877224ca,0x1b62b341),LL(0xd30a4aa7,0xaff58db3),LL(0xbe8da847,0xbad78dad),LL(0x54f18276,0x85fa7109),LL(0x7c4bfdad,0xe2cc9d28),LL(0x2c75f237,0xbb131f76), + LL(0x1c4dfa95,0x283dd7f0),LL(0x62c0b160,0xb898cc2c),LL(0x870282aa,0xba08c095),LL(0xf4e36324,0xb02b00d8),LL(0x604cecf2,0x53aaddc0),LL(0x84ddd24e,0xf1f927d3), LL(0xe2abc9e1,0x34bc00a0),LL(0x60289f88,0x2da1227d),LL(0xcef68f74,0x5228eaaa),LL(0x3c029351,0x40a790d2),LL(0x8442e3b7,0xe0e9af5c),LL(0xa9f141e0,0xa3214142), + LL(0x03844670,0xcdcdd7d7),LL(0xb4a23f91,0x79ec59af),LL(0xc00ce5c3,0x5923c569),LL(0xc589d0c7,0x099c17ff),LL(0x89fa6fe6,0x0335eeea),LL(0xa4e868c4,0x916bcaca), LL(0xfb687bd5,0xb7037325),LL(0x9853b564,0x57d6bca7),LL(0xd5e26d28,0xdf3132ef),LL(0xde919cbe,0x7ed994b8),LL(0x6fbbb18d,0x12df67cd),LL(0x6baff508,0x516e07c0), + LL(0xf9a58e3d,0x72f4949e),LL(0xa48660a6,0x738c700b),LL(0x092a5805,0x71b04726),LL(0x0f5cdb72,0xad5c3c11),LL(0x554bfc49,0xd4951f9e),LL(0x6131ebe7,0xee594ee5), LL(0x3c1af0a9,0x37da59f3),LL(0xcb040a63,0xd7afc73b),LL(0x4d89fa65,0xd020962a),LL(0x71d824f5,0x2610c61e),LL(0x3c050e31,0x9c917da7),LL(0xe6e7ebfb,0x3840f92f), + LL(0x97e833e4,0xf2ec9ef5),LL(0x34ec7e41,0x97bdef97),LL(0x7d2ac6e3,0x90e2b238),LL(0x0318a3b7,0xcf682b12),LL(0xea84a7a0,0x7fe76089),LL(0x16546d05,0x85c489f9), LL(0x6abdda05,0xf987118f),LL(0xaa4b95fc,0x675cf998),LL(0x888a7e8c,0x544c7774),LL(0x63ec5831,0xbd2647ba),LL(0xfd2fe985,0xb479cea3),LL(0x28d163e8,0xa0421345), + LL(0x8d8b8ced,0x50fbd7fe),LL(0x47d240ae,0xc7282f75),LL(0x1930ff73,0x79646a47),LL(0x2f7f5a77,0x2e0bac4e),LL(0x26127e0b,0x0ee44fa5),LL(0x82bc2aa7,0x678881b7), LL(0x67f5f497,0xb9e5d384),LL(0xa9b7106b,0x8f94a7d4),LL(0x9d329f68,0xbf7e0b07),LL(0x45d192fb,0x169b93ea),LL(0x20dbe8c0,0xccaa9467),LL(0x938f9574,0xd4513a50), + LL(0xe5947c6f,0xd93506a4),LL(0x39b81d08,0x4340d76a),LL(0x17930d30,0x741aee59),LL(0x18fdb81c,0xfea3d99a),LL(0x289bcb07,0x1088ff6b),LL(0xb7c082c6,0xc6b45602), LL(0x453d8d69,0x50e2baab),LL(0xe893e183,0xda9bf561),LL(0xb29a284d,0x0af25f86),LL(0x73e01380,0x0e92e674),LL(0x2be00e59,0xe173a0e3),LL(0xada8954a,0x402d2f3d), + LL(0x054cb874,0x841c96b4),LL(0xa3c26834,0xd75b1af1),LL(0xee6575f0,0x7237169d),LL(0x0322aadc,0xd71fc7e5),LL(0x949e3a8e,0xd7a23f1e),LL(0xdd31d8c7,0x77e2d102), LL(0xd10f5a1f,0x5ad69d09),LL(0xb99d9a0b,0x526c9cb4),LL(0x972b237d,0x521bb10b),LL(0xa326f342,0x1e4cd42f),LL(0xf0f126ca,0x5bb6db27),LL(0xa4a515ad,0x587af22c), + LL(0x0399721d,0xca9cb389),LL(0xa3291479,0x03ad9f4a),LL(0x6dee003d,0xd85b5df5),LL(0x64a4f83a,0xe1fa7b02),LL(0xb73f7324,0x01c4cbfd),LL(0x5cf2ddf4,0x707010d4), LL(0xb12e02f8,0x3c6df430),LL(0x85531489,0x921a2901),LL(0x91d1022c,0x302fc77c),LL(0x342d8f3f,0xc3733ec0),LL(0x6195a665,0xb83bc75f),LL(0xa79f8027,0x4a14b9e7), + LL(0xb12e542f,0x1123a531),LL(0xb9eb2811,0x1d01a64d),LL(0xf2d70f87,0xa4a3515b),LL(0xb4bd0270,0xfa205234),LL(0x5eda26b9,0x74b81830),LL(0x56578e75,0x9305d6e6), LL(0x9f11be19,0xf38e69de),LL(0x44dbe89f,0x1e2a5c23),LL(0xfd286654,0x1077e7bc),LL(0x0fca4741,0xd3669894),LL(0x278f8497,0x893bf904),LL(0xeb3e14f4,0xd6ac5f83), + LL(0xe2a57359,0x9f0e5428),LL(0x14998c16,0xc690a3c7),LL(0xde37e07e,0xd73c3ca2),LL(0xdba0bc0d,0x2ddf91b8),LL(0x7570ae71,0x69d834b2),LL(0x735195a6,0x2ac8bed4), LL(0x3b1fcc5c,0xcd8c51ff),LL(0x1ba6863f,0x7aa8cf4e),LL(0xae70f428,0xebb69e72),LL(0xaa9e936d,0xa29409df),LL(0x5a332b9b,0x43f6ee80),LL(0xc2eab0a9,0x0de49efa), + LL(0x488f5f74,0x327b9dab),LL(0xcab7364f,0x2b44f4b8),LL(0x19b6c6bd,0xb4a6d22d),LL(0xfc77cd3e,0xa087e613),LL(0xb0b49bc7,0x4558e327),LL(0xcd835d35,0x188805be), LL(0xc1dc1007,0x592f293c),LL(0x6af02b44,0xfaee660f),LL(0x904035f2,0x5bfbb3bf),LL(0x79c07e70,0xd7c9ae60),LL(0x234896c2,0xc5287dd4),LL(0xcb0e4121,0xc4ce4523), + LL(0x2310333e,0x04baa176),LL(0x7b9bad46,0xdc75e35f),LL(0xc6cd6108,0xc4a6031d),LL(0x30bf87a5,0xba2534d0),LL(0x31e497cc,0x7ebc6e21),LL(0x851fd665,0x8a2a82b4), LL(0x6d5faf40,0x9ecae011),LL(0x96956ecb,0xfa3a6d7f),LL(0x2fa52782,0x39e8a9c2),LL(0x236d442e,0x74c93801),LL(0xb1c289ce,0x8b21ba23),LL(0x25c769cf,0x7f3e221b), +}, +/* digit=61 base_pwr=2^305 */ +{ + LL(0xca114c4a,0x761e10e2),LL(0x894301b3,0xe39d121d),LL(0x3dbc6fca,0xa0870ff4),LL(0xcbe0ba8a,0x97651286),LL(0xc0f1ff6a,0x47d46075),LL(0x3abeb5b6,0x18669c84), LL(0xad8d9309,0x1234c80e),LL(0x1f6f97ff,0x1ccbe4d5),LL(0xd82ab780,0x399a2d41),LL(0xde426e50,0x8a03afaf),LL(0xca6dde77,0xa2bcb109),LL(0x0618f5ec,0x840e13b0), + LL(0xec645a62,0x15d47e52),LL(0x8d6d4423,0xabe0ddb3),LL(0x70cddb11,0x51226a30),LL(0x2b5a8db7,0x63a253d3),LL(0xbef37d65,0xe8be4d1f),LL(0xc0920b91,0x41e625d9), LL(0xd9d040ec,0x08b713a8),LL(0xc450cdba,0x467fb08d),LL(0x917ee393,0xa8975877),LL(0x1528cd12,0x294792e9),LL(0x37daf6aa,0x4512dc8c),LL(0x197a99b9,0xa83becc9), + LL(0x538d92d8,0x3b21dc1f),LL(0xc005aa86,0xc80b22b3),LL(0x0da87d65,0xf536e5d3),LL(0x0cd999a0,0x4ce10edf),LL(0x50e08f5d,0x89491814),LL(0x526647e6,0x77fd8f2e), LL(0x250099fd,0xcb207ee9),LL(0xfd6aa078,0x03c7d1ab),LL(0x25e0cf15,0x7d4940d2),LL(0x067fa052,0xb688b311),LL(0xa98b2e21,0x89308326),LL(0x72311eab,0x3ee4cc2b), + LL(0x68d7dfcf,0x37be5d3f),LL(0xb945e6f2,0x97bdbd49),LL(0x9d1569e7,0x165a24b5),LL(0xb4e293ab,0x254aaf59),LL(0x6fb7c0a4,0x3c751fbd),LL(0x5018cb18,0x14eda4ba), LL(0x1b5f6aed,0xacb3b897),LL(0x1e4b6b78,0x6d10be44),LL(0x621df6d7,0x245d7258),LL(0x185f0e2a,0x2af0e283),LL(0x8fddbd81,0x1e7edc81),LL(0xc538d02a,0xbd1e6c72), + LL(0xaa006a4f,0x12014812),LL(0x83374604,0xf84aa1a2),LL(0x0ee4a8ae,0x9f8475d9),LL(0x135811df,0x37a1b21d),LL(0x1166af52,0x34143171),LL(0xf5a33016,0x204dd449), LL(0x372b6edd,0xc838d3d5),LL(0x3987611c,0x314f3053),LL(0xd112605c,0x819adbe0),LL(0xf6b32c5c,0x45da01ae),LL(0x7e3b13a9,0x3deb3018),LL(0x79058926,0x0e78a3a2), + LL(0x42adface,0xaab5f0c7),LL(0xcb580132,0x50d9f53f),LL(0xb5fd6ebf,0x68a3c689),LL(0x9cde184a,0xdea2f2ee),LL(0x13a98466,0x8c174c44),LL(0xcb4d921b,0x8c4e2ae1), LL(0x2d4c6d5a,0xba973c6c),LL(0x61d2ec5f,0xc6150714),LL(0x666d8bf9,0x2aba1375),LL(0xd41272cd,0x2fa2768f),LL(0xa0bc34bf,0x49f3b8d7),LL(0xca45e5fc,0x61118166), + LL(0x2665ae2e,0x92f565ea),LL(0x1aefd472,0xaaa91acf),LL(0xfbb062aa,0x3878c718),LL(0x7de3c64d,0xde46e7d8),LL(0xbd506a76,0xff9900a4),LL(0x3daa73b9,0x4e30ed72), LL(0x7cbaff42,0xd6be9446),LL(0x5dd691f4,0x26fe6305),LL(0x6d393800,0x64ef093c),LL(0x7bb8f155,0x2448c67a),LL(0xe5d732d7,0x9da6e75f),LL(0x50b080e0,0xe837a602), + LL(0x844626a2,0x6858b674),LL(0x0cbba6a6,0x610cd40f),LL(0x29d9194d,0x324e674e),LL(0xdcb30a51,0x2dc6fdf6),LL(0x528aa549,0x3f3ecb77),LL(0x23ffaa92,0x0721f8f9), LL(0x27a77538,0xd8efcbd6),LL(0xd6162c9c,0xf4e642bf),LL(0x4cf4a16f,0x04f2b0b7),LL(0xbbf335fd,0xbc0bb49f),LL(0x5a928c36,0xc6b6e5bd),LL(0xd893dd45,0x981b01f4), + LL(0x518f01dc,0x2836b977),LL(0xa06c7362,0x117e833e),LL(0x31152b22,0xcda89f2c),LL(0x34be0102,0x2084c5c4),LL(0x2bfac8c2,0x6478de75),LL(0xebda5ede,0x9f7e901a), LL(0x5aa25b6f,0x5f43adfc),LL(0xcdd0eee3,0xfb719dca),LL(0xf9b16d84,0x14431b2d),LL(0x97f04b2b,0x846261af),LL(0x1edd7d4d,0xcc6100b8),LL(0x87cde5f4,0x6197c87f), + LL(0xa01cb6d9,0x4a9e6281),LL(0x87065307,0x699c0719),LL(0xdffa58be,0xa8ca4971),LL(0x89efaadd,0x8adc304f),LL(0x1f3c79d2,0xeef0af15),LL(0x581587e8,0xb3be9c6c), LL(0xda0be326,0x79010ad2),LL(0x4be00f8b,0x4f361e1e),LL(0xc53f3c74,0x180e66ba),LL(0xb2521c2d,0xa668c3f3),LL(0xabb73a09,0x60bc2fa6),LL(0x4392692f,0x0bda0ff6), + LL(0x9aa349d0,0x2e88f308),LL(0xa7cf751f,0x9f19df26),LL(0x4885be75,0x1e0229c7),LL(0xa32fce2e,0x6770eee2),LL(0x562d99d7,0x448366e4),LL(0x8bc7484a,0x7670bd68), LL(0x92d83c6b,0x15374837),LL(0x14f7c403,0xe0f499f8),LL(0x8a6b78f6,0x4cf02671),LL(0x9849e689,0x75f6e30a),LL(0xcad5065a,0xaf6fe2a5),LL(0x43ba98aa,0x6378401b), + LL(0xd2446552,0x26d6d225),LL(0xd2600e42,0x9b74929e),LL(0x447126d2,0xcef3a052),LL(0x4d1c7e0a,0xcba2f70c),LL(0x020d33a3,0x0250a96f),LL(0xa5e587d9,0x9c946f94), LL(0x86653ae6,0xc7d4343f),LL(0x8884e9c6,0x9c832859),LL(0x1c234f88,0xa44fa8c5),LL(0x987f04dc,0x7193e6db),LL(0x6a25ff37,0x24efebac),LL(0x953b3db8,0xe23f0a14), + LL(0x491d9ba9,0xb13db9bb),LL(0x5556de42,0x7105da91),LL(0xbd48b3d9,0xafa75d3b),LL(0xed6c519c,0x10246797),LL(0x1aa866f5,0x83b27882),LL(0x54f64e96,0x66f4ceaa), LL(0x43c07b18,0xf4f03faf),LL(0xdd18ddec,0x97eed374),LL(0x6c454f23,0x43b702f1),LL(0x3cb61e31,0x3c53810c),LL(0xedcaea49,0x8a50cfe9),LL(0x89a6acd9,0xfb70772b), + LL(0xa01edb12,0xedc09655),LL(0x7ad80675,0xdaa9f823),LL(0xf6a1052d,0xb7f23b6c),LL(0x697dc18b,0x22dc809f),LL(0x6c8bcd69,0x0453593f),LL(0xc9b43cf0,0x80f76d8b), LL(0x8c5b3ba9,0x8e781e17),LL(0x66a0b318,0x66544c7c),LL(0x9d95b620,0xedd99cbd),LL(0xbdf0933e,0xc36c4334),LL(0x8b59acf3,0x3d550b68),LL(0x6ca6a2b7,0xcd7d1701), + LL(0xc3d5ad9a,0x94457602),LL(0x3a7a1abc,0xa1a9608e),LL(0x31a107cb,0x16eb2310),LL(0x987bf106,0x5d921026),LL(0xd2ccc296,0xec2e5789),LL(0xb8f2fbd4,0x87b86d1b), LL(0x6da88d7d,0x939b5802),LL(0xe19707e2,0x22fa6ef8),LL(0xc547ce83,0x8b95bd5d),LL(0xd16fb119,0x91268688),LL(0x6e3627f2,0xbf199148),LL(0xc31ab346,0xd072bdf2), + LL(0xf810465b,0xfb083c2a),LL(0x02ce0dee,0xb66a8de9),LL(0x47a81b95,0x6e4130e7),LL(0x58a98737,0xcd704dc6),LL(0x592829c9,0x842ae329),LL(0xbe20dd63,0x99bedc34), LL(0xd53b2df4,0xabee8e55),LL(0x6010b37c,0x6ce65758),LL(0x467112b9,0x781f39b2),LL(0xbe341038,0x6f06058f),LL(0x12a2f8be,0x5effdca5),LL(0xaf34466e,0xaa9bdad7), +}, +/* digit=62 base_pwr=2^310 */ +{ + LL(0x9c216137,0xab86639e),LL(0x82b18d64,0x45a12fb8),LL(0xd763f0bb,0xb5734418),LL(0x11a9802c,0xd2cc3322),LL(0x81269b8a,0xe41d7db8),LL(0x2ecfa355,0x91072fc1), LL(0x04ce306f,0x59d69125),LL(0xa131b86d,0x916d9d4d),LL(0x8a739738,0x84478b6b),LL(0x1cc83ae3,0xe86ad7d9),LL(0x797ccd97,0xbc9b2084),LL(0x694944c6,0xc1e94af4), + LL(0x895c0318,0x585edee3),LL(0x45e8205b,0x775e142f),LL(0xd85ad31f,0x3bd7924f),LL(0x9124bffa,0x2e7d8f91),LL(0x44c62868,0x885397c0),LL(0x7fda9f5d,0xc0c2dff4), LL(0xc14e693d,0xd302582e),LL(0x6cec31ba,0x53d6e33a),LL(0x63653c06,0xb0216b5b),LL(0x9c70dad4,0x8f08a1ad),LL(0xffbba93d,0xccf014aa),LL(0xa33f12b7,0x900b0d2c), + LL(0x9b8cfa41,0x0dd2395e),LL(0xd4f92a44,0x50e203ab),LL(0x6630023b,0x7280aff1),LL(0x07de820e,0xfcce59dc),LL(0xbc8189ad,0xa686be05),LL(0xaac70b7e,0xac4b59bf), LL(0x7a3c71ac,0xd2c0070e),LL(0x35ac1c47,0x1d550add),LL(0xfb881c1b,0xd42b6389),LL(0xd0dafd42,0x57ca3fcc),LL(0xbe26ccc9,0x909e8284),LL(0xa002235c,0x1abe7595), + LL(0xa6a1913d,0x1e34781a),LL(0x7d0adc38,0x9a8f3228),LL(0x28af85ba,0xfc185ccc),LL(0x3ae9ba11,0xc923d78b),LL(0xa7bdb313,0x7d494d7e),LL(0xaf8f8b87,0xf774dfa5), LL(0x16e863b8,0xc178ccc1),LL(0xa8899691,0x2d472f2a),LL(0x80a50372,0x608747cc),LL(0xe6f90197,0x8147aa90),LL(0x78c2f216,0x4683d4c9),LL(0x552f3b51,0x8323652c), + LL(0xcc2c9a2a,0xa5c08e8b),LL(0x8baaf0fc,0x70e1b405),LL(0x9e36e50c,0xf29e1e5c),LL(0x80f258c5,0xa3d90800),LL(0xecad4498,0xc9ceac25),LL(0xca32f3fe,0xcb73130f), LL(0x48b3863d,0x2dbe620c),LL(0x14ff53bf,0x8c52727f),LL(0x6b45e9b8,0xb60b22a8),LL(0xf6483c5d,0x81e05bc0),LL(0xcd542972,0x217caa6b),LL(0xfa780778,0xffab716a), + LL(0x0d7410d6,0xe5d3e0d8),LL(0x2be432c9,0xcfa9ed74),LL(0xa85a0686,0x60044434),LL(0x6ad6918a,0x93b35716),LL(0x051762be,0x1a3c3e6d),LL(0xb0ab32d6,0x80813589), LL(0xaad403fd,0x64214b92),LL(0x4d3fb746,0x684befc1),LL(0x79515046,0xaca5a514),LL(0x72e84485,0xacdba034),LL(0x287d9e97,0x61aa2834),LL(0xcad222e7,0x07a515a5), + LL(0x8af19670,0xb0309306),LL(0x34c6bf0f,0xd784125a),LL(0x255a8396,0x0b425ee0),LL(0xfb541162,0x91076433),LL(0x86f47a0e,0xc4d81885),LL(0xfd7bc7c1,0x3b767d54), LL(0xbee196e9,0x98b405d3),LL(0xedaccf4a,0x4ef9c511),LL(0x03f4f1a6,0x5a6deb65),LL(0x1b4c5104,0x4a22ca64),LL(0x9145ce41,0x2cce3667),LL(0x3206810d,0xd0518752), + LL(0x037bebad,0x29d81538),LL(0xd9e0b78c,0x76e52c73),LL(0x8783d1fd,0xaa4ace6e),LL(0xf0e3c126,0x9c14ebdd),LL(0x6eca4b71,0x0eb1c08d),LL(0x1c91df35,0xd10c6b96), LL(0xe81bb84a,0xdb8119bb),LL(0x17e3ceef,0xf784d3c1),LL(0x35436f81,0x053c9168),LL(0x9b18d212,0xeb41ccbb),LL(0xb1bc3497,0x93b3fb43),LL(0x8c1ced81,0xd85a7c75), + LL(0x811af84a,0x004105c3),LL(0xa7934a0f,0x01307934),LL(0x9b3226a1,0x179fd49b),LL(0xde6834b4,0x195d9e5c),LL(0x0e6051bd,0xfbb79dc0),LL(0x367f4487,0x354273ed), LL(0x74fb892d,0x4afa9d45),LL(0xa1b7f3bb,0x03ae905e),LL(0x592f6122,0xea32cd5d),LL(0xf1103301,0xa758eed2),LL(0xc59d1cc8,0x9dde4238),LL(0x51022a42,0xe2760bcc), + LL(0xd377d7b0,0x54f84d70),LL(0x3344bc4e,0xb745d190),LL(0x8f33aa53,0x1c693ed0),LL(0x8bfbee7f,0x990ed45f),LL(0xe9b258fb,0xad620c9f),LL(0x1a54bf46,0x465ccb10), LL(0xebc40951,0x5330a0d3),LL(0xa405da61,0x34423e8c),LL(0xb83043b6,0xeef1ce78),LL(0xac06d182,0x99678f22),LL(0x1802f14c,0x9213f57d),LL(0xadf11fda,0xf8549616), + LL(0xb6e392e1,0xf31796d2),LL(0x93b3395e,0x199d6248),LL(0x12f9b763,0xef14c7c2),LL(0x43edb7a5,0x721ebf21),LL(0x5e96f3ba,0xa40b8894),LL(0x4cff8394,0x8770608c), LL(0x8d0def0e,0x990c99ae),LL(0xa15a5649,0x292b26df),LL(0x91ca89d7,0xa98fda2c),LL(0x973e5f5f,0x916cb1b4),LL(0xa72de0bb,0xa2823f13),LL(0x8cd3219d,0x415f7bd2), + LL(0x3ed03d5f,0x521f4af7),LL(0xeaf9064c,0xe3461f66),LL(0xae03777f,0xad099ab7),LL(0xb65f73ff,0x541cadcd),LL(0xa86059b9,0x53430463),LL(0x043e9f82,0x8ff88fe5), LL(0xe42cde45,0xd515f4c7),LL(0xf41c3269,0xf7f3dec3),LL(0x7ef1b8ff,0x7bed5356),LL(0x1295b5fe,0x8782b45f),LL(0x03917627,0xab54ebaa),LL(0x8787ed9f,0x8516beb2), + LL(0x65b68624,0xba7df5a9),LL(0x6e7d58bb,0x30b4d6ed),LL(0xdbb81762,0x67e52341),LL(0x0deeac1d,0xd697ab1b),LL(0x5577ea92,0x01d15e8e),LL(0x98fb38da,0xbb12d724), LL(0x4e04908d,0x302faa6d),LL(0x09b90a9c,0x66cf6cb9),LL(0x98d96736,0xcd665dbd),LL(0xb86f3af4,0xf7d3c528),LL(0x1d8b07f4,0x4844c754),LL(0x1eaf7dc0,0x2a77d7b9), + LL(0x1d70eb73,0x53e59f25),LL(0xe69d0525,0x8aed17af),LL(0x64413768,0x26ddc178),LL(0x5e48c349,0xa7c8d40f),LL(0x87ff01fb,0x29ad92d1),LL(0x965b2de1,0x8f4e1b3b), LL(0x1446eca2,0xb83cfadf),LL(0xe609d416,0x7432bda4),LL(0xf1c7de69,0xcf97e8a3),LL(0x32f55f07,0x45899bd8),LL(0x51175738,0x41a68117),LL(0xb8efff21,0x89eeb115), + LL(0x936c3eba,0x54a01e60),LL(0xec01b12c,0xf326fe96),LL(0x66e4de2e,0xcdfaf003),LL(0x392fd0a8,0xc53dba07),LL(0x6ec46004,0x00d9b80f),LL(0x3ba63f8d,0x84d59be8), LL(0x9dea6062,0xbac4ea43),LL(0xadd568ca,0xb4b4845b),LL(0xd225e2d0,0xa6ca3d34),LL(0xb50070a2,0xce72955d),LL(0x21c78b68,0x56e5c913),LL(0x999488a3,0x888eb198), + LL(0x11c92f34,0x5255508c),LL(0xa294d382,0x9a346cf3),LL(0x3095205b,0xd9765eea),LL(0x2c470ef7,0xfea2ed70),LL(0x9c40bf0a,0xf5e8a0fc),LL(0xe4137a16,0xb572390e), LL(0x2bf2f545,0xb9175371),LL(0x58cd9cc7,0x2c2d0f4c),LL(0x02385486,0xbea6bce9),LL(0xa8bc3a94,0x46208408),LL(0x3ac45044,0x64a87a2a),LL(0x7df70151,0xe40da33c), +}, +/* digit=63 base_pwr=2^315 */ +{ + LL(0x39161b8a,0xee9e25d9),LL(0xe2eead91,0x8763f2a2),LL(0xd2d91300,0xd2fc1157),LL(0xffcbe50f,0xe7597e2f),LL(0xe11d376e,0x4be3814f),LL(0xdbf14562,0x1eab3d7e), LL(0xc0ad183a,0x38a107c0),LL(0x7c753bbf,0x82976626),LL(0xcaebd481,0x18014e09),LL(0xf9ace60d,0xb28c331b),LL(0x211cb8e7,0xe8fba04f),LL(0xe42dc65e,0x41c4b797), + LL(0xc3e88580,0x009dc2f4),LL(0x99db1fb5,0x4a405be8),LL(0xec5d91fe,0xc89bfaa2),LL(0xf160afcb,0x461be9a0),LL(0x7d7566b5,0xfdd084bb),LL(0xe48099a2,0x795275e8), LL(0xfe9815db,0x1b461fc9),LL(0x73627bbc,0x576214cd),LL(0x9f09a206,0x3246332d),LL(0x6941d6ef,0xbde4c0c3),LL(0xf387f5f6,0x44ef03fd),LL(0x57b63400,0x99c8ac01), + LL(0x2f6e4301,0xaa512f20),LL(0xbf94a1cc,0xef668a5f),LL(0x15861b88,0x08713c30),LL(0xc99bb2b6,0x49d47551),LL(0xe2f0258e,0x6db5f812),LL(0x998d7435,0x70c9b299), LL(0x5d176ae0,0x46168e1c),LL(0xf730ec30,0xec3306e4),LL(0xab69c15d,0x49439df3),LL(0xea0143e4,0x1040408b),LL(0xbc549b0a,0xb48ab8ea),LL(0x10f89223,0x4aa38bbf), + LL(0x9598f49a,0x7e485159),LL(0x9629305b,0xbdac3d5e),LL(0xa6fbabfb,0x20de0daf),LL(0x8f09fff7,0x04f01583),LL(0x6a06994f,0x5a056297),LL(0x6e3ccd33,0xf51dac8f), LL(0x3af507b8,0xc087ef9c),LL(0x6a5c6663,0x525ab76e),LL(0xd916ee93,0x4fc04814),LL(0xd23d140b,0x3369c978),LL(0x1662028f,0xb0fcd70f),LL(0xe1e28adb,0x2ca77de2), + LL(0xc512bc71,0x838acd1b),LL(0xdc18afd0,0xac06d6bd),LL(0x9ec45f4b,0xc991c1e3),LL(0xcc27c68e,0x667c5e89),LL(0xed07f829,0x0e059b04),LL(0xcec4b3a7,0xceccf1d4), LL(0xb953f9a1,0x3d9c2dc9),LL(0x2d599b16,0x4be2f7e7),LL(0x97256c26,0x1a2054b1),LL(0x8b4fdfeb,0xcf66fa47),LL(0x8134d7ef,0x896cc1b3),LL(0xd41dadbc,0xa17264ca), + LL(0x37627e56,0xe3ccfe8e),LL(0x7b6b21a6,0x00733a86),LL(0xb605c427,0x3f13e2cb),LL(0xb0d80992,0x5ee12395),LL(0xb9991381,0x4dcaea94),LL(0x8c4c4b6e,0x4cfed7ee), LL(0x7f7f45df,0xd7aad54b),LL(0xb3809bf8,0x2229407f),LL(0x68048fd9,0x6eb31eee),LL(0xd57225fc,0x693842df),LL(0xa88dfd3f,0x3e62cd85),LL(0xd5462cf1,0xc6307d53), + LL(0xf344f5fd,0x2d15615e),LL(0xa7f23989,0xe0ba6a8a),LL(0x1c84e3f2,0xbbfc5804),LL(0x6f4ba826,0x22ffeaae),LL(0x94292682,0x1e9bf274),LL(0x46c02af9,0xc768f891), LL(0x177cdafa,0x894127d6),LL(0x2acdc791,0x8d0523da),LL(0xdc78c3c4,0x71ada9ae),LL(0x2c532a01,0xf21dbbb9),LL(0xacb20fda,0x0c797d5e),LL(0x16cf57b0,0x1ff99d76), + LL(0x493c1d64,0x99b5f150),LL(0xfb74075e,0x3422b656),LL(0xff19bf24,0xe7493900),LL(0x260925ed,0xc82e5b80),LL(0xc0ea1eaf,0x3398d340),LL(0x1287121e,0xe7de2ba1), LL(0x87847031,0xea6dfb0b),LL(0x566af2f2,0x73bed0a1),LL(0x12012999,0xe26678bf),LL(0x32e5cebc,0xb5369e4d),LL(0x6d181e32,0x2304eac8),LL(0x3d364add,0xafdbd954), + LL(0x75da4189,0x5b1a53ca),LL(0x2eb4862b,0xa9048580),LL(0x2783ad6a,0x31942409),LL(0x1a9e025e,0x15a4c5e1),LL(0x13837199,0x841bc533),LL(0xe642954a,0x6e9d3e14), LL(0xd436ec5c,0xf4a02bbd),LL(0xc6d6ad53,0x62fe177b),LL(0xac86425a,0xedbf1e4e),LL(0xd9f752f5,0xff9359c8),LL(0x2d7ad656,0x79c685d9),LL(0xfdde9052,0x8d82c0c4), + LL(0x702f640d,0xf55f868e),LL(0x1dedda11,0xe459aa9b),LL(0xbb5ba193,0xbec0ff9b),LL(0x57724703,0xf7325c49),LL(0x23e0e4fb,0x5ab8f063),LL(0xecb0fd7c,0xfbf02e91), LL(0xa2e5fa31,0xcc72e8da),LL(0x32cb53cf,0x47de2528),LL(0x4252763c,0xbfa646e6),LL(0xb8d81de3,0x7a769efe),LL(0x1e772f00,0xf5ec7003),LL(0x2729aa5e,0x049bea9a), + LL(0x759090d6,0xe987ba54),LL(0x619ef08b,0x904d6901),LL(0x2024a6fc,0x9e16d138),LL(0xa9f3b7e4,0xb6f0459b),LL(0x17ee069a,0x1f2a5308),LL(0x2be31049,0x99403b2e), LL(0xbfb2f288,0xba1663c6),LL(0xc7a92b41,0xf829195c),LL(0x8ae621b1,0x89b915ee),LL(0x50f8ea92,0x3fbbb1e1),LL(0x8c901ddc,0xb1fe7f97),LL(0xbbc69ca4,0x16d1f62c), + LL(0xfda072db,0x51f19bb3),LL(0xe3f7e0a2,0xa815459f),LL(0x987112ca,0x5f7cde2f),LL(0x759de2cb,0xdc51d948),LL(0xed49bd98,0x9d05c410),LL(0x364341fd,0xf063ab99), LL(0xd1aa0a11,0xd7869d68),LL(0x5d862d01,0xc2029106),LL(0xc2591073,0x7f258180),LL(0x6ebc4ebc,0x7b90fc7a),LL(0x3dda1d68,0x5565390f),LL(0xa44e4493,0xae77fca8), + LL(0x47c49ee8,0x97564e48),LL(0xab4ebef5,0xc56bb5a9),LL(0x7b4f86bc,0x80d96941),LL(0x41026cf0,0xa594b4e5),LL(0x5a89ece9,0xd56c8996),LL(0x6a0f922b,0xbcf60931), LL(0x1103475c,0x70259616),LL(0x8a2a2abb,0xb1224fb5),LL(0x715cd61b,0x0a437a03),LL(0x739921ed,0xcbe2d2b2),LL(0x385541c4,0xf3b1b5e9),LL(0xae010520,0x5d0984f4), + LL(0xcfd9295e,0xb4a2742d),LL(0xae929cd0,0x9cd36774),LL(0xdd7fcf4f,0xb15fadcc),LL(0x37d4fcc6,0x0b1fa2b3),LL(0xf01c7ab7,0x242c7b26),LL(0x50574cc9,0x2be8131b), LL(0xbd89a03c,0x6ee50f42),LL(0x005e7765,0xc7f6ff8f),LL(0x8420501b,0x04d13af1),LL(0x1b6e7d2a,0xc22e092b),LL(0xe9516f80,0xa393be7e),LL(0xb80bb5b5,0xa2593652), + LL(0x8b23bebb,0x5caa5da6),LL(0x1fdbbdf4,0xa1ad33e8),LL(0x4e5c1de0,0x18dc93cf),LL(0x5bd9e178,0xc3e6addb),LL(0x7cb8cd03,0xf30d517e),LL(0xf1abc417,0xbb84ce54), LL(0x67699659,0x0774b64c),LL(0xb7d4a401,0x228005b9),LL(0x80b2d3d2,0xd8c2ec5b),LL(0x3450ba7f,0x419c4cd9),LL(0x789286a6,0x520ae681),LL(0xaa8bcfbb,0x24b67ea9), + LL(0x0f74808f,0x9e41b9b7),LL(0x0c061bdb,0x2d835dae),LL(0xf272346c,0x67e50c8c),LL(0xdef57493,0xc98a5ef5),LL(0xa02676fc,0xc2dea8af),LL(0x6ace4659,0x59508de2), LL(0xda6cd733,0xc2b707aa),LL(0x4be7bfb9,0x6c1f226a),LL(0xa778c20b,0x5b580fa2),LL(0x57af166b,0x272c3a1d),LL(0xca78ce62,0xe47a64a9),LL(0x71d35087,0xd12db7d7), +}, +/* digit=64 base_pwr=2^320 */ +{ + LL(0x1a42e5e7,0xc20fb911),LL(0x81d12863,0x075a678b),LL(0x5cc0aa89,0x12bcbc6a),LL(0x4fb9f01e,0x5279c6ab),LL(0x11ae1b89,0xbc8e1789),LL(0xc290003c,0xae74a706), LL(0x79df3f45,0x9949d6ec),LL(0x96c8d37f,0xba18e262),LL(0xdd2275bf,0x68de6ee2),LL(0xc419f1d5,0xa9e4fff8),LL(0xa52b5a40,0xbc759ca4),LL(0x63b0996d,0xff18cbd8), + LL(0x844eefc5,0xf6827150),LL(0x4515ef68,0x002e82c4),LL(0xc51916c4,0xa46c8f55),LL(0x61ee081f,0x98c3524b),LL(0xad64872a,0x5ab7f2c2),LL(0x7e555faa,0x0b503ff0), LL(0xb4c58d29,0x802e0d23),LL(0x2fd917fe,0x12289040),LL(0x7af20d26,0xb56d1908),LL(0x6be50784,0x8d619e21),LL(0x1372b851,0x10fdbb72),LL(0x4935576e,0xf2c1673e), + LL(0xd7dd47e5,0x73c57fde),LL(0xd49a7f5d,0xb0fe5479),LL(0xcfb9821e,0xd25c71f1),LL(0xcf6a1d68,0x9427e209),LL(0xacd24e64,0xbf3c3916),LL(0xbda7b8b5,0x7e9f5583), LL(0xcf971e11,0xe7c5f7c8),LL(0x3c7f035e,0xec16d5d7),LL(0xe66b277c,0x818dc472),LL(0xb2816f1e,0x4413fd47),LL(0x48383c6d,0x40f262af),LL(0x4f190537,0xfb057584), + LL(0x51a135f6,0xd97a9b14),LL(0x97b4df14,0x6d16aaf5),LL(0x54818818,0xc57160c2),LL(0x1d59be44,0x4dbdeab6),LL(0x81f2b247,0xb93a9dad),LL(0xecbcab33,0xe2868cf5), LL(0x83a86711,0x5e1ce828),LL(0x29c55428,0x29a9ca2f),LL(0x2d82b0df,0xe716273a),LL(0xac8ff52f,0xb017f5f6),LL(0x70ea7ccd,0x7563e799),LL(0x3f0e674b,0x5fedf0a6), + LL(0x08962f6b,0x487edc07),LL(0x190a7e55,0x6002f1e7),LL(0x10fdba0c,0x7fc62bea),LL(0x2c3dbf33,0xc836bbc5),LL(0x4f7d2a46,0x4fdfb5c3),LL(0xdca0df71,0x824654de), LL(0x0c23902b,0x30a07676),LL(0x77fbbf37,0x7f1ebb93),LL(0xfacc13db,0xd307d49d),LL(0xae1a261a,0x148d673a),LL(0x52d98650,0xe008f95b),LL(0x9f558fde,0xc7614440), + LL(0xaf907da2,0xd084564b),LL(0x51d4997a,0x5b2ae487),LL(0x3bc7206d,0x24bd4bf6),LL(0xfc3d5772,0xdd37b4ef),LL(0x35c4924f,0x8156d6f6),LL(0x1d1d396e,0x21e067c3), LL(0xd40c7db8,0x977b3b39),LL(0xf5ad63bc,0x7ea4ecb4),LL(0xae811d70,0xe581f9c4),LL(0xa06c7f0d,0xe5441d5c),LL(0x1949d87f,0x0275c92b),LL(0x780469bb,0x511fd3e1), + LL(0x9cb16650,0x17cd6af6),LL(0x69f4eebe,0x86cc27c1),LL(0x78822432,0x7e495b1d),LL(0x1b974525,0xfed338e3),LL(0x86f3ce21,0x527743d3),LL(0xb515c896,0x87948ad3), LL(0xb17f2fb8,0x9fde7039),LL(0xd9b89d96,0xa2fa9a5f),LL(0x36ff74dc,0x5d46600b),LL(0x8302c3c9,0x8ea74b04),LL(0xf744b5eb,0xd560f570),LL(0xfe762402,0xc921023b), + LL(0x88d7b3fb,0xa7f85014),LL(0xec78386e,0x3b5ec513),LL(0x2ad5053d,0xc6586b8a),LL(0xfbcebe43,0x88c09a43),LL(0x20054f16,0xde7f2a4a),LL(0xbbbb147f,0x63daba80), LL(0x7d352b55,0x087e48f3),LL(0x8317ab79,0x997e32a0),LL(0x7f27cac7,0x8ae802ff),LL(0x37b1f6e1,0xb01a131c),LL(0x9a6d1dea,0x3f0d4c2e),LL(0xe7ceef80,0xe06114fc), + LL(0xfff4c8ed,0xa35ab657),LL(0x8a5fabd7,0x017c6124),LL(0x09acda28,0x56463025),LL(0x14cf238a,0x6038d361),LL(0xaf1b9f07,0x1428b1b6),LL(0x7482e95c,0x5827ff44), LL(0x780ff362,0xcb997e18),LL(0xe0bcac1e,0x2b89d702),LL(0xa837ddc8,0xc632a0b5),LL(0x59762647,0xf3efcf1f),LL(0x38b0d60a,0xe9ba309a),LL(0x20b5fb37,0x05deabdd), + LL(0xe5ea795b,0x1e2a4fb1),LL(0x89ef6c3d,0x862616a2),LL(0xf69e2f1d,0x24617f4f),LL(0xffa0eb28,0xe0be24fe),LL(0x4b76a8ad,0x0ffb092f),LL(0x3a0dc9e8,0x21549090), LL(0xe9080e04,0xf255fe06),LL(0x39228e7f,0xd270d25e),LL(0x5d6c6f1c,0x198e45db),LL(0x4373044c,0x6c9060ad),LL(0x61a8cc25,0x3af93464),LL(0xf22bbaa3,0x1945bf59), + LL(0xcb8af047,0xd44e5dba),LL(0x943cfe82,0x15400cb4),LL(0x9df88b67,0xdbd69575),LL(0xb2405a7d,0x8299db2b),LL(0x0b1d80cd,0x46e3bf77),LL(0xe82ba3d9,0xc50cf66c), LL(0xf2f747a9,0xb2910a07),LL(0x5adc89c1,0xf6b669db),LL(0x9052b081,0x3b5ef1a0),LL(0xb594ace2,0x0f5d5ed3),LL(0xd5f01320,0xda30b8d5),LL(0xaafcd58f,0x0d688c5e), + LL(0x359590bf,0x311df84c),LL(0xdf6ca4b4,0xf907d69d),LL(0x82f22c64,0x876fd367),LL(0x9713e68c,0x64c4d14d),LL(0x6b07f539,0xd431858d),LL(0x84990283,0x39dfea33), LL(0x80cf6498,0x6afb8cf0),LL(0xde060e9e,0x327056bc),LL(0x49a71086,0x5103ce4a),LL(0xcdf853ab,0xfc94be75),LL(0x8ca579cd,0x2bfb105f),LL(0x50454b41,0x02d19c3a), + LL(0x2a161074,0x5eee3a31),LL(0xefe2be37,0x6baaae56),LL(0xe3d78698,0xf9787f61),LL(0x50630a30,0xc6836b26),LL(0x1445def1,0x7445b85d),LL(0xd568a6a5,0xd72016a2), LL(0xe355614f,0x9dd6f533),LL(0x91e04588,0x637e7e5f),LL(0xb9fb1391,0x42e142f3),LL(0x41afe5da,0x0d07c05c),LL(0x1394edf1,0xd7cd25c8),LL(0xb99288ee,0xebe6a0fc), + LL(0x7d4867b7,0xd9e2c5ee),LL(0x87873152,0x2c5602e0),LL(0x2c06b73a,0xb3358aa6),LL(0x09063076,0x967aec39),LL(0xd2f654fc,0x94dec534),LL(0xd69f485e,0x119aa4ed), LL(0x35bc085d,0xb7c597b8),LL(0xbdbe0d0c,0x8781396d),LL(0x22f92ef5,0xba688929),LL(0xd438f5ec,0xeece3d4e),LL(0x44faac8b,0x4ead06f8),LL(0x9ef7c5f1,0x86a01ba9), + LL(0xbabbad86,0xb8e63b7b),LL(0x90d66766,0x63226a9f),LL(0x5cf26666,0x26381836),LL(0x4cadd0bf,0xccbd142d),LL(0x9ac29470,0xa070965e),LL(0x25ff23ed,0x6bdca260), LL(0x87dca7b3,0xd4e00fd4),LL(0x9e0e8734,0xa5097833),LL(0x048173a4,0xf73f162e),LL(0x9c3c2fa2,0xd23f9196),LL(0xe4ac397a,0x9ab98b45),LL(0x543f2d4b,0x2baa0300), + LL(0xcde121c7,0xaa03b474),LL(0x55e52c76,0x74a648cb),LL(0xf37b57bc,0xb286ef86),LL(0x2a6371d2,0x95b797eb),LL(0x4077ccbd,0xa489ef89),LL(0x8e99ca6d,0xf46ade04), LL(0x23242d03,0x5cf9e237),LL(0xcb708390,0x33c7d32a),LL(0xba7ba477,0x329523b6),LL(0x57de30bf,0xd406ab87),LL(0x1536ca01,0xaa10e4a2),LL(0xdfa7aac5,0xdcec94f4), +}, +/* digit=65 base_pwr=2^325 */ +{ + LL(0x1f5a9609,0x762a5eec),LL(0x765b337f,0xfe4f5f6a),LL(0xaa4f964a,0x0fd534ae),LL(0xd6526f01,0xcf46648e),LL(0x18d71d72,0xbc62a54a),LL(0x4f8488ea,0x48d94f2a), LL(0xa0c72a86,0x62c40de7),LL(0x725dd2ef,0xd73ac51a),LL(0x6ab19096,0x3a51d746),LL(0x2dd1ad3d,0xf07bea4b),LL(0x2ef88078,0x2a0ec467),LL(0x664e435d,0x92598cb3), + LL(0xb515fff5,0xee6e7006),LL(0x13258ed5,0xaedf6e39),LL(0xfc45111b,0x373adf7d),LL(0x875c23c8,0x0c7535b1),LL(0x97039d49,0x2a7e04f8),LL(0x9afd1a06,0xd76787ae), LL(0x91b6dc89,0x049dd385),LL(0x932dab78,0x8f0c8ad0),LL(0x925498c2,0xdce057b9),LL(0xda25daa3,0x7b9c9bd2),LL(0xd4decb7d,0x6d0b70a3),LL(0x03df76ef,0x099a2183), + LL(0xd8948f95,0x779905b6),LL(0x91cd0206,0x3c7085b5),LL(0x679096db,0xce9af0aa),LL(0xf558913c,0xfdf04f10),LL(0x6f24a2e2,0x05300cb0),LL(0x5d581b35,0xf9d9a2f2), LL(0x6a713751,0x855c8de9),LL(0x0e0c0dfb,0xc9ac24bc),LL(0x97740d65,0x67612a41),LL(0x44c9360d,0x7588a527),LL(0x325cc880,0x928ac910),LL(0xacdd3188,0xa74abdaf), + LL(0xb9d926c2,0x3c6c5618),LL(0x4a9099f9,0x7e14c3ae),LL(0xae2fb830,0xb3259c90),LL(0xec31a504,0xf7cc6e43),LL(0x126230bd,0x83bb13c6),LL(0xff1dae3a,0x5a1f4313), LL(0x49b0b65b,0x0cc6c1a5),LL(0x274a84c2,0x67fa836a),LL(0xe604a58d,0xd454c75f),LL(0x2491f872,0xceadfd91),LL(0x9ce116a5,0x6c5575da),LL(0xb24a4481,0xfaa4903f), + LL(0x5a4703aa,0x7a8a898d),LL(0x1cd6f9d6,0xc59933ea),LL(0xd28124cd,0x703265f5),LL(0x0178d1fe,0xe1c1bee1),LL(0x241262e9,0x9ff31cd4),LL(0xa3c9f80f,0x9174a939), LL(0xbc2a62ee,0x0f7a3d2d),LL(0x62f1b3ac,0x0454051c),LL(0xa2421254,0x83502c9e),LL(0xb684199b,0xb4fa51fc),LL(0xc5e36a44,0x257e9e2b),LL(0x97d8647f,0x14efeed5), + LL(0x5cecb21f,0x6e96a819),LL(0xd8beecae,0x3a58d8b2),LL(0xc0c715a8,0x93c3cbb0),LL(0x541759b7,0xfb06f977),LL(0x771c3d2c,0xf25ba095),LL(0xa3bfd322,0x7560446e), LL(0xa015cb4f,0x7cd99f35),LL(0x0786f235,0xa0e54196),LL(0x8b8e291a,0x0f868f76),LL(0x2f95050b,0xc8260b0b),LL(0xf4c0a462,0xaf38376e),LL(0x98a3395d,0x2b3c0f3b), + LL(0xb9d0bdf0,0x0952b888),LL(0x8ce32fb7,0x3973763c),LL(0x6dd860c7,0x221f0ba5),LL(0xb16ac501,0xbb7a27fe),LL(0xbc8fe58f,0xf113b194),LL(0x65839ffb,0x18f3297a), LL(0x8dc30003,0xa2d4eb7c),LL(0x8e334479,0x3fb4b487),LL(0x1a8310e3,0xa4f32c65),LL(0xf78f46ac,0x944cd644),LL(0xf96fb91f,0x14e40c4a),LL(0x4ddf6e72,0xc31402bd), + LL(0xb45a8002,0xc5ad791e),LL(0xba2d7a40,0x4a23fd68),LL(0x98544bc4,0x673b9e49),LL(0xd273c360,0x934d8f55),LL(0x68a75a8c,0x7fb48d07),LL(0x5e0fac97,0x2e620105), LL(0xf10ed580,0xbe01655f),LL(0x9e96731f,0xd21d52ae),LL(0x53325138,0x74f830de),LL(0xde9f3fc5,0xa7240331),LL(0xa7e01fa5,0x96b25206),LL(0x07eda4b4,0x3fcfedee), + LL(0x9336b8bb,0xf1b08a42),LL(0x9a768ca6,0xbaaa5571),LL(0x2c0a2938,0x70180b4c),LL(0x92dd3c07,0x8e735436),LL(0xe3fd5b1c,0x2dd3af0b),LL(0x1f1af8b8,0x3cf3d179), LL(0xd558c174,0xaff210c6),LL(0xe2560d5e,0x1007938a),LL(0x8f99a78f,0xa625558c),LL(0xc1b91dea,0xa61d5edf),LL(0xdab80815,0xa86a4e5b),LL(0x78283ea3,0x88944bfe), + LL(0x0ee6d492,0xeec9b118),LL(0xb7ef00fc,0xf2bd388f),LL(0x3191c902,0x5c6c0bbf),LL(0xd42366de,0x6796929a),LL(0x3285710d,0xeecb5b3f),LL(0x41bad8a3,0x58d6773d), LL(0xd0f05005,0x7cdbade6),LL(0x0e25444b,0xb117e1ba),LL(0xfe4dc071,0xa52b4926),LL(0xe0669f10,0xce8a1b69),LL(0x0db21d46,0x5e765439),LL(0xd929bfab,0xc5dcdea4), + LL(0xa36aea84,0x67832f48),LL(0xfc78df1d,0x25256118),LL(0x03b8f04d,0x5a085d70),LL(0x108969c2,0x19a25d9c),LL(0xb70b14b7,0xb6fe713b),LL(0xfa6b89d8,0xba23ac4a), LL(0xc2684b8c,0x6a88e4e7),LL(0x6ab59297,0x3e816609),LL(0x30151aaf,0xdb7927fe),LL(0xaa7d95da,0x0fd67681),LL(0x17432b4c,0xc60e5dd5),LL(0xed48ccda,0x3192dc27), + LL(0x8af859e2,0x2116a017),LL(0xbd9f8800,0x8a77a7a2),LL(0xf1f2034d,0xc78836fb),LL(0x3864566b,0x8fd4299a),LL(0x29f9deca,0x0386eead),LL(0x2f1a677c,0x2042ef18), LL(0x2af95cc8,0xed4511c0),LL(0xc93dfbc5,0xdb0a334d),LL(0xb64ab345,0x0d788ac9),LL(0xb20638f2,0xd8410402),LL(0x592448e6,0x8aca5131),LL(0x000de69b,0x3ac508d2), + LL(0x4f9b2400,0x1ee6d3b5),LL(0x00c9c182,0x4c71999a),LL(0x35fc481e,0xd6b1c470),LL(0xf8b5d59c,0xf41ef454),LL(0x2824a13d,0x7edbc3c9),LL(0xb7aa0ade,0xa3baba91), LL(0x2b97b392,0x75c77e71),LL(0x9cc2d53c,0x8ec107d6),LL(0x29322233,0x652146fe),LL(0xe679e990,0x710578df),LL(0x260547db,0x47f838ba),LL(0x23a78365,0xa4e57bec), + LL(0xefb058db,0xe4320313),LL(0x3d04e752,0xaad2377c),LL(0x9865c63e,0x3f8cbca9),LL(0x3009e55d,0xe89238a5),LL(0x12519936,0x58fad5fe),LL(0xb024b695,0x03b16a00), LL(0xf8f3b8c7,0xa556d096),LL(0xd8a599c4,0xce6df197),LL(0x6a13b031,0x45ac8a25),LL(0xca6819e8,0x31af2dcc),LL(0x390418e9,0x7a4dce86),LL(0x4600d7f4,0xdd24b0bd), + LL(0xb96e667f,0x88a8aa87),LL(0x4e704eaa,0xb9a76c18),LL(0x72c924b7,0x25d4edc1),LL(0x16b67f80,0xe56872a8),LL(0xc464e4ff,0xda0dbb11),LL(0xc8cec410,0x0435f391), LL(0x0983fd7a,0x8d9b4043),LL(0xe7ff343c,0x51ec5bc1),LL(0x85994bc9,0x8bc85fc4),LL(0x8806c150,0x69c78834),LL(0x3db3665b,0x542cec89),LL(0xfd720bde,0x931bc4b0), + LL(0x8d5c039f,0x3e6e9381),LL(0x8a8d2cc9,0x80949422),LL(0xb843ec06,0xf2d7c8b4),LL(0xaf8a23f1,0x0055d882),LL(0xd3792335,0xe848010e),LL(0x55e08e74,0x9b41a55f), LL(0x5de83059,0x956ea8e9),LL(0x3263678e,0xf159a997),LL(0xcca1b548,0x5f7b9271),LL(0xf1d0b7f1,0xd41d2281),LL(0x5c9963fb,0xb187047b),LL(0x02536cd8,0x213ff6af), +}, +/* digit=66 base_pwr=2^330 */ +{ + LL(0x0d0fa76c,0xe51a9570),LL(0x4d2e9c8e,0x67c7890e),LL(0xf974d2cb,0xc6160fa2),LL(0x4c6a78de,0xe00474f7),LL(0x0ac89d11,0xee916e51),LL(0xf826f133,0x1adad97a), LL(0x8d2d77f3,0x3fc65d3f),LL(0x0ba6c300,0xda942075),LL(0x0b9196b1,0x5237a82e),LL(0xa572b6f5,0x4975e680),LL(0xb9bed2bc,0x41ea8b92),LL(0x9826825e,0xbe0ad710), + LL(0x9fcaba39,0x8a390dca),LL(0x278d22b3,0x3879f0b4),LL(0xbc5e82f1,0x77bbea69),LL(0x4628d6f1,0x71f02e2d),LL(0xf968e240,0x6260790c),LL(0x665270ee,0x1c7f3df5), LL(0x1a87b1c8,0x33639545),LL(0x4ffd9fb8,0x2011fd21),LL(0x7807ed55,0x69060f86),LL(0x9dfa452c,0x1b0ac011),LL(0x06d27c0d,0xbbdb25fe),LL(0xa60ef90c,0x5c25d23a), + LL(0x734b2e9c,0xd3eb69e4),LL(0xc35ff1b3,0x1c2754e2),LL(0x9f3e8c51,0xa26681e6),LL(0xa2cae737,0x7892ad11),LL(0xcbd8bda6,0x88b1da43),LL(0x419d21c4,0x8a576942), LL(0xc90f4545,0x7c124343),LL(0x26453bae,0xa5a8d93b),LL(0x76ae72e8,0x9a4c08fd),LL(0x7b064e94,0xa08b82d9),LL(0x83725330,0x4f803ba0),LL(0x865235f3,0x33672455), + LL(0xc8110f1f,0x1877dfd0),LL(0x18db27c2,0xea88f59d),LL(0xc78e295e,0x9d089536),LL(0x74a04cc5,0xcbb5d553),LL(0x827f75ed,0xe3666006),LL(0x61e7378c,0x8557b81a), LL(0xed223f48,0x74170170),LL(0xd86ee829,0x84197a6e),LL(0xac1c4a0f,0xd75a30f8),LL(0x3cd92824,0xd7e7be0d),LL(0x1b5e86d4,0x5ea0abdb),LL(0xb3b615ef,0x41146ae1), + LL(0x1ae5e9da,0x570deceb),LL(0xb73ead01,0x5c079c70),LL(0xd2ce6639,0x522a30a6),LL(0xf4056ac9,0x71dc5c3f),LL(0xbaac149f,0xd93c7a2d),LL(0xf1844ceb,0x5c3298b8), LL(0x8c23c0dc,0x282adf40),LL(0x9b646f86,0xbe498189),LL(0x628da9e5,0xe77c1950),LL(0xa1fd5a18,0x38cc27ba),LL(0xaacdca52,0xb5579728),LL(0xc8e1ecbd,0x8d34fdb4), + LL(0xadea7d6a,0x323f2102),LL(0xb694b253,0x035b354e),LL(0x5b8a36c2,0x66dc4e4a),LL(0x71795ca4,0xb6092224),LL(0xd300d80f,0xd8c6d7ee),LL(0xb3b94954,0xf31f258d), LL(0x277ced5b,0x0f2eb679),LL(0xeba40e3f,0x0b16efa3),LL(0x0dca4f36,0x40003507),LL(0x59a9a3a1,0xd34c91cb),LL(0x86da6408,0x5e8fea32),LL(0x03f31728,0xf237959f), + LL(0xbbaaedc6,0x1b5bc630),LL(0xb49cbb3b,0xe7d25088),LL(0x0deb8cf0,0x5622cbf7),LL(0xd309c3ba,0x3b20803c),LL(0xff45e2fc,0x64c2e7de),LL(0x9aab84a5,0xfa730ffb), LL(0x4edfb52e,0xba83cc51),LL(0x748bbd69,0xe05c0140),LL(0x2254ec43,0x27bbb5f5),LL(0x324c8c40,0xca740989),LL(0xd26491b4,0xa21488b1),LL(0x69d8626b,0xe2753a1f), + LL(0x8956146c,0x618ca06b),LL(0x552cdecf,0xd51f1e6f),LL(0xa3b6ce7d,0x981372cc),LL(0x5f14bb57,0xb44a68ee),LL(0x6373abbb,0xfc1167e9),LL(0x767d4c0a,0x3d621f8b), LL(0xf6ecc778,0xc6dcdfeb),LL(0x82d1fbdd,0xddda9262),LL(0xbfcbf2f7,0x477501aa),LL(0x67aa8277,0x0be7228a),LL(0x1daab9cd,0x5de7b833),LL(0x262feb4a,0xb88a4f9a), + LL(0x936fb33f,0x28f18586),LL(0x381bf7bb,0x9809b2ab),LL(0xeac3c252,0xf6e16931),LL(0x5e151187,0x366d1833),LL(0x7a3b6460,0xe5b4c235),LL(0x0a68bc91,0x693a9fa5), LL(0x6a7f8b6e,0xa35f104a),LL(0x688676c4,0x3e5d6981),LL(0x0651a609,0xc0c081b1),LL(0xd77057c9,0x6df5da2d),LL(0xc4602847,0x8bb271bb),LL(0xc4bd07d8,0x322547b3), + LL(0x233d39e4,0x8c283529),LL(0xc6092096,0x96300796),LL(0x5dde766c,0x2c549de5),LL(0xb4151002,0x27e0b444),LL(0xf2f88f1b,0x802e5fc3),LL(0x8ba1956d,0x2af579c2), LL(0xd68196c7,0x52edd04e),LL(0x74a202b0,0x2e22e714),LL(0x8bf66459,0x33894824),LL(0x9e39df55,0x8f0d8c25),LL(0x6c5276d9,0xee4f109e),LL(0xc5dc0bf0,0xc0c893f0), + LL(0xf8482849,0x06054c76),LL(0x5fcca65d,0xc24b4a6a),LL(0xa17ebda3,0x71c27e01),LL(0x1be9dfb8,0x1ffce028),LL(0xebc43854,0x3784c950),LL(0xd5086510,0xcf0ecc2d), LL(0xbe24d8eb,0x86d0fc3c),LL(0x1f21788e,0x5bad0191),LL(0xc49b3a12,0xe2c3bcb9),LL(0xf7d5992d,0x66f82433),LL(0x13969246,0xf7cc5eb9),LL(0x8660a6da,0xe52defd4), + LL(0x53fd1a04,0x7a9623b6),LL(0x3a3b8500,0x13bd35bf),LL(0xe0f8e530,0xf8a5dec9),LL(0x1d65dcd4,0x88bcbe29),LL(0x6739541c,0x09fe3803),LL(0xe716a97a,0xebd04b7f), LL(0x1e5ef7cb,0xbd8e34df),LL(0xd7c4fd6c,0xddfc4243),LL(0x3519411f,0x0183d905),LL(0xf7a3c483,0x63450996),LL(0x01355739,0x18283cea),LL(0x9aaa72f7,0x8c1d72cf), + LL(0x8dc72468,0x551e1b4e),LL(0xa7b2f1ac,0x8a926cb2),LL(0x0fd12fad,0xb873e83b),LL(0xa4e7fb13,0xb6cde14f),LL(0x5befc256,0x81ae4141),LL(0xb4c7631c,0xffb0c636), LL(0x8a2478fe,0x80f1408f),LL(0x44fa7605,0xde6d051d),LL(0x4d44a1e4,0x5a15b1f8),LL(0xa0daafe3,0x1a0031c5),LL(0x597652a7,0x304338dd),LL(0xf257f17a,0x6830dcc7), + LL(0x193aabbc,0x4a67ec76),LL(0xd74761f9,0x3da6dec6),LL(0x0b35bb70,0x751720c9),LL(0x8d9e0f8b,0xe5e04905),LL(0x0858f29c,0x3cd37c84),LL(0xb881733e,0x7ff1abfb), LL(0x0c4f7694,0xa0c2698b),LL(0x96b95e4e,0xc7364192),LL(0x37ece651,0xcfa55c55),LL(0x7cb1e9e1,0xa2bbd6ae),LL(0xa0eb0e8a,0xcd2292b9),LL(0x8d5030d0,0x8aba99e1), + LL(0xdc4a1d3e,0xb7c74c58),LL(0x0331ea39,0xe3ec3016),LL(0x023c8712,0x83afb271),LL(0xc9c82680,0xc2670d56),LL(0xfeca1061,0xd426f350),LL(0xba6edc01,0xe8aee692), LL(0x46e801d9,0xc916fbe5),LL(0x7097286e,0xcb001c37),LL(0x78ee1328,0xfcf79d26),LL(0xb6a4afb3,0xb05b0634),LL(0x306da14f,0x2ab327bb),LL(0xba5ff534,0xc11a0294), + LL(0xe19763dd,0x7b7da028),LL(0x8b98ff78,0x662f54df),LL(0x51f3dbd9,0xc056d83c),LL(0xa91d085a,0xe2f4d46f),LL(0xeb35262d,0x31759c9c),LL(0x0c9dd29e,0x624d0cf2), LL(0x1624b02d,0x108cf9bb),LL(0x345531d6,0xa241444e),LL(0x73d372b2,0xf69816b2),LL(0xd5415e53,0x126575a7),LL(0x306b8b0e,0x546bb4c1),LL(0x4d54ea5e,0x82bb0c12), +}, +/* digit=67 base_pwr=2^335 */ +{ + LL(0xf9474bb7,0xdaa7fcc9),LL(0xafa5db2a,0x3c82e74b),LL(0x9894edce,0xfbf918c5),LL(0xa9ac29a7,0x470c45ed),LL(0xbc372f2c,0xdfd44f6f),LL(0xa1e38d3f,0x73a4790a), LL(0xa9674837,0x23d2400b),LL(0x136a92da,0x3dad71bc),LL(0x48baa4ab,0xc76a4881),LL(0xbc26e6b0,0x73227e4e),LL(0xe8ef5662,0xe732edcf),LL(0x0c5662bb,0xfe96aa5f), + LL(0x139b3239,0x87c7dd7d),LL(0x4d833bae,0x8b57824e),LL(0x9fff0015,0xbcbc4878),LL(0x909eaf1a,0x8ffcef8b),LL(0xf1443a78,0x9905f4ee),LL(0xe15cbfed,0x020dd4a2), LL(0xa306d695,0xca2969ec),LL(0xb93caf60,0xdf940cad),LL(0x87ea6e39,0x67f7fab7),LL(0xf98c4fe5,0x0d0ee10f),LL(0xc19cb91e,0xc646879a),LL(0x7d1d7ab4,0x4b4ea50c), + LL(0xe2fdc88b,0x7e047d9c),LL(0x715be007,0x4f6166d9),LL(0xd0debb0a,0xd9661068),LL(0xc3dafce2,0x82f02fd3),LL(0x00f8df79,0x41a6b644),LL(0x6cedd3a8,0xccd5a798), LL(0xf11431b2,0xb6617354),LL(0x8a677e83,0x116d0274),LL(0x89ef485a,0x2f399390),LL(0x5e2270d2,0x3ee06862),LL(0x06d6c72f,0x8c9a70de),LL(0x4e4497e3,0xd7e69177), + LL(0x7db62b5a,0xcfbcbc4a),LL(0x4ab45dde,0x2919bf51),LL(0x22322f91,0x735de056),LL(0x7662ae23,0xd2590bda),LL(0xd82be7a6,0x63d468fe),LL(0x695ea172,0xc84d0435), LL(0x20a6fccd,0xc50f4941),LL(0x620f44f1,0x2d613990),LL(0x1fd25778,0x680ccd04),LL(0x4a3d0808,0x25ddac44),LL(0xc4684cba,0x41d8b738),LL(0x53963888,0x2611645f), + LL(0xe9f43747,0xffe6bb22),LL(0x22f6cd09,0xf387957b),LL(0x607a4892,0x3eb09302),LL(0xf3d2f552,0x52c733a8),LL(0x741bd215,0xcc935b2e),LL(0x1ae0b28e,0x5fff37e3), LL(0xc2e9bc20,0x4234e33c),LL(0x39ea2555,0x4ee1488e),LL(0x17156a8a,0xb8821daf),LL(0x1af16ade,0xc7b45844),LL(0x5b4fa74d,0xc1009ec7),LL(0x5d7cf8bd,0xe0262e65), + LL(0xb0279be5,0xb05cb834),LL(0xf08c5f93,0x2de7d0eb),LL(0xefa9e4f0,0xf023b5aa),LL(0x9bd075ec,0xb8061e5d),LL(0x1aa41bfb,0x7d2ba50f),LL(0x90865d96,0x8963f3e3), LL(0x4713ec7a,0x7f221a79),LL(0x8500b4c3,0xc83bc517),LL(0xf6ab1540,0x085feb6a),LL(0xdc87cd93,0xfd141024),LL(0x3239dbf6,0x3e196fdb),LL(0xdbcd5364,0xb7cf3e16), + LL(0x41a64252,0x72544edb),LL(0xa6fe493d,0x5f3d376f),LL(0xd635df1e,0x17ae424f),LL(0xdf598c63,0x69cb55a0),LL(0x6ce0f1d5,0x31297f4a),LL(0x1bd11b61,0x4573bb7d), LL(0x45a1e918,0x1d8a65c1),LL(0xe5dc2e63,0x2729ab25),LL(0x3ecc307b,0xa3dd0df0),LL(0x952019dd,0x4856546f),LL(0xc784e4fe,0xf8d39888),LL(0x0043b09e,0xdc6732c7), + LL(0xe03a2fb4,0x1466c9f5),LL(0x862a58a2,0xb866c006),LL(0xb5865550,0x291e8c75),LL(0xe65862cc,0x1ddb7154),LL(0x2b997167,0x285153bc),LL(0x954b6c19,0xe2fce0e7), LL(0x16dc2937,0x985d4506),LL(0xee41d9c3,0xf7f14216),LL(0xfa5fe5e5,0x39e098da),LL(0xf90f231d,0x3fc26046),LL(0x32afd0b5,0xde5d5ced),LL(0x60c09c18,0xad688b1d), + LL(0x8f84e987,0xefd9aed0),LL(0xae8c8308,0x5ee0c707),LL(0x2aafc403,0x4c8a7653),LL(0xa232436a,0xfafb60ce),LL(0x9934f053,0xc641294d),LL(0x30310529,0xc673e5b2), LL(0x9066469d,0x3c8fa99a),LL(0x7c09af37,0x5626038b),LL(0xabd66748,0x6ffd8f9b),LL(0xcea58a67,0x21ced048),LL(0x1496d048,0x31071213),LL(0xa9c28d0d,0xfa575242), + LL(0x0f806b59,0x3720b172),LL(0xf224597b,0x1f696d47),LL(0x5b54eefc,0x03c46e31),LL(0x72b0edaa,0x387e4664),LL(0xee77476e,0xfc59b03d),LL(0x607a7259,0x86854e54), LL(0x3e9320dc,0x1478bcee),LL(0x8c9d87e4,0x4aa825a8),LL(0xcf272ee0,0x71272f72),LL(0x8bd885cd,0x19e3a4a3),LL(0x376ba31c,0x9af6415b),LL(0x807b2b36,0x6394b5a7), + LL(0x77460193,0x1bf6c56b),LL(0x5666ae6d,0xd6f7fabf),LL(0xe3e839d1,0xdf677909),LL(0x08cb9984,0x9124d341),LL(0x86a0c7c3,0xbb6b591d),LL(0x8f527a6a,0x4bf33423), LL(0x26941bfe,0x7d137790),LL(0xcf6e4481,0x15a0b541),LL(0x124d5b9b,0xdf833b87),LL(0xa7fdf95d,0x85827dc5),LL(0x49e520af,0xf05a2c45),LL(0x91e0645a,0xfb506d53), + LL(0xe572e06d,0xdbfcfa75),LL(0x8b7d5653,0xafa019d0),LL(0x67a19b60,0xcc6c851d),LL(0x31ae1a67,0xace88bf4),LL(0x93d1e135,0x74554a61),LL(0x4211890a,0x51ba2cdd), LL(0x9e8d1f02,0x7cb32689),LL(0x8b66ab99,0x29a6b825),LL(0x766e72f3,0x0a672c21),LL(0x880642e3,0x24bb718a),LL(0x184d2b36,0x425dc41d),LL(0x891024ab,0x96a1468e), + LL(0x665fe173,0xeff22b64),LL(0xef2eabea,0x38efdef6),LL(0x21a309df,0x8a1f3791),LL(0x2431e2ed,0xa9cf02cf),LL(0x1d939394,0xf38507dc),LL(0x82fc3178,0xf116b085), LL(0xc7571366,0x4c5460dc),LL(0x978495fd,0x99efd9dd),LL(0xf26e347d,0x5159dd41),LL(0xe97ee9f1,0x692962ce),LL(0x6a288815,0x1e2f3af2),LL(0xa71ade78,0x03512344), + LL(0x26df7050,0x3180789c),LL(0x96cdfd31,0xe375a43e),LL(0xe99e922d,0x7951b895),LL(0x3d0bbe80,0x987ea250),LL(0xe2fe79c0,0x6d2f49f0),LL(0xc2b18d2a,0xc9c2c636), LL(0xd8c8620c,0x707798f3),LL(0xd5c6a0ee,0xc2d603da),LL(0xbc447940,0x46cf1e32),LL(0x38a845f3,0x4dfc1459),LL(0x455e5d92,0x210083fe),LL(0xa1fedc3f,0x6be989ea), + LL(0x9f019162,0x55d4fdc3),LL(0xf1ec4585,0xa8222d08),LL(0x3a0ae4f9,0xd987e3eb),LL(0xa9c7a693,0x07deda59),LL(0xf04ee53f,0xc06567d9),LL(0x71364c4d,0x93945788), LL(0xbaa5bc16,0xbb2a2422),LL(0xbfa3931a,0x89574a5d),LL(0xf300f081,0xf09b331e),LL(0x1a0ff82b,0x644de9b7),LL(0xa5ecdf9b,0xae023ce4),LL(0xc1907aac,0x5b67cf8b), + LL(0xdacc038c,0x72fc8198),LL(0xf1077bbd,0x5fdae1d9),LL(0xd99e3036,0x369198bb),LL(0x0efddfca,0x6b68390a),LL(0xf0914741,0x8c35f3e4),LL(0xca7d7807,0xd2bc54ec), LL(0x3a8695d1,0x564d991e),LL(0x1b0d937d,0x5e1e14c8),LL(0x5d635893,0x51f30dab),LL(0xf944e49a,0x0427e346),LL(0x6a233bc0,0x1e0bf1b5),LL(0x617bf93e,0x75b0ee6c), +}, +/* digit=68 base_pwr=2^340 */ +{ + LL(0xd2e15a8c,0xd23658c8),LL(0x16ba28ca,0x23f93df7),LL(0x082210f1,0x6dab10ec),LL(0xbfc36490,0xfb1add91),LL(0x9a4f2d14,0xeda8b02f),LL(0x56560443,0x9060318c), LL(0x64711ab2,0x6c01479e),LL(0xe337eb85,0x41446fc7),LL(0x71888397,0x4dcf3c1d),LL(0x13c34fd2,0x87a9c04e),LL(0x510c15ac,0xfe0e08ec),LL(0xc0f495d2,0xfc0d0413), + LL(0xc4e81268,0xb097b2c5),LL(0x1d50ca8c,0x7ef17552),LL(0x42099644,0x638266e9),LL(0xff729073,0x43d059de),LL(0x148c3940,0xeebb5fe1),LL(0xdaa8e925,0xb82e73d1), LL(0x254380fd,0xf43c78d8),LL(0xfce37fa0,0x2beabc58),LL(0x6b636357,0xcdd5a7d6),LL(0xe096a954,0x8b70a2eb),LL(0xd0afa2fc,0x011d5419),LL(0x04fb095a,0x3e49eb67), + LL(0x156636c2,0xeb05c516),LL(0x090e93fc,0x2f613aba),LL(0x489576f5,0xcfd573cd),LL(0x535a8d57,0xe6535380),LL(0x671436c4,0x13947314),LL(0x5f0a122d,0x1172fb0c), LL(0xc12f58f6,0xaecc7ec1),LL(0x8e41afd2,0xfe42f957),LL(0x3d4221aa,0xdf96f652),LL(0x2851996b,0xfef5649f),LL(0xd5cfb67e,0x46fb9f26),LL(0xef5c4052,0xb047bfc7), + LL(0x168d5e60,0x88e7ac8e),LL(0x6188a98f,0x53abd569),LL(0x18be419a,0x3b96d529),LL(0xc057621c,0x7e75e354),LL(0x5ce57e59,0xcb1b709f),LL(0x844f2463,0xe78befa2), LL(0x3276d4a0,0x53608199),LL(0x157f2024,0x92636ade),LL(0xe0411414,0x6dd0d348),LL(0x4d73eeae,0x5b28e950),LL(0x690ed85e,0x08439232),LL(0x6da14b58,0xdde1a349), + LL(0xf4484374,0x5cbdc442),LL(0xf92452ef,0x6b156957),LL(0xc118d02a,0x58a26886),LL(0x75aaf276,0x87ff74e6),LL(0xf65f6ec1,0xb133be95),LL(0x4b1b8d32,0xa89b6284), LL(0x09c81004,0xdd8a8ef3),LL(0x0cf21991,0x7f8225db),LL(0x26623faf,0xd525a6db),LL(0xbae15453,0xf2368d40),LL(0x84f89fc9,0x55d6a84d),LL(0x86021a3e,0xaf38358a), + LL(0x33c12f53,0x10520b27),LL(0x9286a1f5,0x97d4ef43),LL(0x8948f78b,0x12468ef8),LL(0x50ad452e,0x40a9d275),LL(0xc1c48470,0x5e382347),LL(0x33e73ace,0x5cd739fd), LL(0x1041f8f3,0x7d83d02f),LL(0xe314ad92,0x32642eb0),LL(0x885679e6,0x6716d448),LL(0xfc95919c,0x0e12bdd5),LL(0x92c2194a,0x7da44204),LL(0x15ffcd2d,0x78956db1), + LL(0xff52e280,0xbd048bdc),LL(0x526a1795,0x8a51d0b2),LL(0xa985ac0f,0x40aaa758),LL(0xf2c7ace9,0x6039bcdc),LL(0x6aec347d,0x712092cc),LL(0x6b5acab7,0x7976d090), LL(0x6eed9617,0x1ebcf80d),LL(0xb0f404a4,0xb3a63149),LL(0xd0b610ef,0x3fdd3d1a),LL(0x98c28ac7,0xdd3f6f94),LL(0x3a59750f,0x650b7794),LL(0x2d3991ac,0xec59bab1), + LL(0x39ed9ec9,0x6cbd2757),LL(0xfe5d4aa8,0x5db68a68),LL(0xe4c58c7b,0x177eaa0b),LL(0x0e488784,0x603551ef),LL(0xdf916b0f,0xc8eba131),LL(0x159732e2,0xd0dbceda), LL(0xb0834afa,0x55acca84),LL(0xb59ffbf5,0xdbe98440),LL(0x3bd3b202,0x162a2c70),LL(0x6ddd8eba,0x4c5e5d25),LL(0x77b1d93d,0x66e7844a),LL(0x110b9dcf,0x1292bc0e), + LL(0x2e552766,0x01f40e88),LL(0x66f5354f,0x1fe3d509),LL(0xb3a8ea7f,0x0e46d006),LL(0xf831cd6a,0xf75ab629),LL(0x91465119,0xdad808d7),LL(0x17ef9b10,0x442405af), LL(0x672bdfcb,0xd5fe0a96),LL(0x355dbdec,0xa9dfa422),LL(0x79b25636,0xfdb79aa1),LL(0xeece8aec,0xe7f26ffd),LL(0x7edd5aa2,0xb5925550),LL(0x8eb3a6c2,0x2c8f6ff0), + LL(0x9341a2e0,0x34a4632c),LL(0xc525bc5a,0x05ca421b),LL(0x4ae3284a,0x441cf2eb),LL(0x146012ab,0x1a57684e),LL(0x30acfd0e,0x23a52ee3),LL(0x7d29e389,0xc3f4d94a), LL(0xb4154640,0xfbd4d48e),LL(0xaf9ec396,0x2352e791),LL(0x7327caee,0x45813e8e),LL(0xd9db7e81,0x977f7a08),LL(0x5f53d15d,0xbe55b630),LL(0x6a23f0dc,0xee182ac6), + LL(0x757d6136,0x88887756),LL(0x88b92e72,0xad9ac183),LL(0x8785d3eb,0x92cb2fc4),LL(0x9319764b,0xd1a542fe),LL(0x626a62f8,0xaf4cc78f),LL(0x26bffaae,0x7f3f5fc9), LL(0x40ae2231,0x0a203d43),LL(0x387898e8,0xa8bfd9e0),LL(0x474b7ddd,0x1a0c379c),LL(0x34fd49ea,0x03855e0a),LL(0xb3ef4ae1,0x02b26223),LL(0xe399e0a3,0x804bd8cf), + LL(0x7cdc2211,0x22723fa3),LL(0x31ddb2bd,0x1d339232),LL(0x46626fe6,0x63f354c1),LL(0xf67a4257,0xd0f68526),LL(0x8d3d00b6,0x79aa889b),LL(0x0de4c413,0xca5fc8a7), LL(0xd2879266,0xe311a966),LL(0x5f21e1dd,0xc8d982fe),LL(0xcbb159ff,0xc51f1604),LL(0x092d83ce,0xb449deb8),LL(0x644fd0be,0x4a5f68f8),LL(0xbffb0088,0xeef3fa4d), + LL(0xde865713,0x11a9f3d0),LL(0xbde98821,0x81e36b6b),LL(0x6aa891d0,0x324996c8),LL(0x395682b5,0x7b95bdc1),LL(0xc1600563,0x47bf2219),LL(0x643e38b4,0x7a473f50), LL(0xf5738288,0x0911f50a),LL(0x6f9c415b,0xdf947a70),LL(0x67a067f6,0xbdb994f2),LL(0x88be96cd,0x3f4bec1b),LL(0xe56dd6d9,0x9820e931),LL(0x0a80f419,0xb138f14f), + LL(0x3b32932e,0x2b4056a4),LL(0xbd8a1cb8,0x1c74deb6),LL(0xb181b5a0,0x09601843),LL(0xdc6a92d8,0x50a92353),LL(0x2d6f4331,0xe86c022c),LL(0x3a3ae821,0x0d9671dc), LL(0xc8228d82,0x3d618a20),LL(0xa5292169,0x20e809c1),LL(0x3803f840,0x3b2fe5e7),LL(0x416eb670,0x1f2978e9),LL(0xc35b795c,0x44dcc410),LL(0x503ce975,0xbf5065c0), + LL(0x0429077a,0xa11a1a8f),LL(0x10351c68,0x2bb1e33d),LL(0x89459a27,0x3c25abfe),LL(0x6b8ac774,0x2d0091b8),LL(0x3b2415d9,0xdafc7853),LL(0x9201680d,0xde713cf1), LL(0x68889d57,0x8e5f445d),LL(0x60eabf5b,0x608b209c),LL(0xf9cfa408,0x10ec0acc),LL(0x4d1ee754,0xd5256b9d),LL(0x0aa6c18d,0xff866bab),LL(0xacb90a45,0x9d196db8), + LL(0xf9e89bea,0x1b66faab),LL(0x3a441284,0xc81c5ddc),LL(0xa675f7c8,0x1a82f3a0),LL(0x30313a71,0x82884a2f),LL(0x58aea9e6,0x7ac5d7b0),LL(0xcd5ff05d,0x1954f075), LL(0x6178d270,0x7c29638d),LL(0x19381929,0x6af7f8ba),LL(0xa17ae3a7,0xe85e3c47),LL(0x7009e38a,0x91b107c7),LL(0xf1f9c52e,0xf3b777d8),LL(0x11b688a0,0x5b7b74ff), +}, +/* digit=69 base_pwr=2^345 */ +{ + LL(0xc0385874,0x4ae3d232),LL(0xcbf96d2a,0x83bda9e6),LL(0xec62fd6a,0xba73c769),LL(0x62a4720c,0xd586ba7f),LL(0x0cc1f491,0x6497cd14),LL(0x7b2ac571,0x8b012b70), LL(0x268fd705,0xa65eabb6),LL(0x1431873d,0x8caf100a),LL(0x231457d7,0x25b31b84),LL(0x901645c5,0xcab62f75),LL(0xb2f7b656,0x2377d74d),LL(0x2d33c95c,0x4008277c), + LL(0xed001c43,0xaae2f448),LL(0xcf4be493,0x08ad1d9b),LL(0x82c1f372,0x3262b2f4),LL(0x351a5f7f,0x5521febd),LL(0x916c75a8,0xf8ec9190),LL(0x2728dfb8,0xf3c258c7), LL(0x8af19574,0x5dd4ff4f),LL(0x5d076b1c,0xefddf579),LL(0xba8b777a,0x318b5b98),LL(0xfb7f8409,0xd971d426),LL(0xb0fd31db,0xed1465e8),LL(0x00f66347,0x80d24d43), + LL(0x1ae586a2,0x5ba5288c),LL(0x1b715821,0x044f1fc6),LL(0x602f3c65,0xc1a9a997),LL(0xe08c0223,0xc5c7512f),LL(0x367e6f1d,0x48a19c3c),LL(0xfb241597,0xa9f2195d), LL(0xb5ba32a6,0x9f674a5f),LL(0x0a312742,0x275a060f),LL(0x03d6f13e,0x5aeb8c43),LL(0x917433fb,0x0fed575d),LL(0x59f53490,0xe4a5ef9a),LL(0xf315e616,0xa9f86145), + LL(0x7e5a59f4,0x315ad7a4),LL(0x543c8b00,0x1c615bfc),LL(0xbaa56473,0xe12f97a8),LL(0x46edcfcb,0xf263db44),LL(0x3c1a968e,0x47cf91d5),LL(0xc15db875,0x1a1165b4), LL(0x3479616a,0x5d35e53a),LL(0x5c59958f,0x649f87b4),LL(0x246da3d3,0x5d3d11ea),LL(0x53f06820,0xc1ddfcc6),LL(0x6610c00f,0x8169d711),LL(0x4bddc8c7,0x15f16ba5), + LL(0x1e548ef0,0xcf19fb2a),LL(0xcc694171,0x8bb6dfa0),LL(0x5c5e390f,0xeb1668ca),LL(0xe1975263,0xf5a3485b),LL(0x442cc850,0x4edfc596),LL(0xf9627d74,0x9901f447), LL(0x84d0413a,0x3a6b85c9),LL(0x67de639c,0x14663661),LL(0x11705bbb,0x9fc9fdcf),LL(0xbff2cf80,0x6d066e2b),LL(0xdc3026fd,0x38dedc2f),LL(0x1b828538,0xad533a98), + LL(0xb7bfa29e,0x6c75bc93),LL(0x18ef6d69,0xf86f22b2),LL(0x36dcadf2,0x90ce6a15),LL(0x7ce50921,0xf11f711c),LL(0x38a479e3,0x0739ceda),LL(0x6ec3dbc6,0x840b825e), LL(0x9fa23481,0x7c36c0a5),LL(0x70cb186d,0xceb61fd1),LL(0x26e4754d,0xac6f7d3c),LL(0xf317b385,0x4076d3b5),LL(0x3fd9e9c7,0x52f1bd72),LL(0xbf316043,0x6649d8b6), + LL(0x243cfbd5,0xdc2870f8),LL(0x1ab496f3,0x000b71b3),LL(0x708f4507,0x53511a3f),LL(0x1949d835,0xbd7bd038),LL(0x938f4db6,0x723a007d),LL(0x2d04e9fd,0x5bc8679e), LL(0x76ec7fc4,0x51ca5fd1),LL(0x988f354e,0x86c4205c),LL(0x2a0a4a90,0x9042e76b),LL(0x4ad44d2d,0x368f52a8),LL(0x912edfb7,0xddc2cab8),LL(0xcde80199,0xde74ccf5), + LL(0x3e455a61,0x8002f458),LL(0x5bea205a,0xafbafd37),LL(0xfb93f735,0xa8ced112),LL(0x196e3084,0x27cb6292),LL(0x77e8c744,0x72395bdd),LL(0xee71f5ff,0x02e018d8), LL(0xc1337a1d,0x7cfc14d9),LL(0xd7b4d86e,0x94e14c0a),LL(0xd213738e,0x66e50129),LL(0xbc0b5ea3,0x7a905d91),LL(0xfca06700,0x92cb630a),LL(0xbf3a0821,0x65e06d5c), + LL(0x55c2369d,0xa81e4a4e),LL(0x60a0f544,0x394de01b),LL(0xa8906e17,0x22acfd07),LL(0xcc9bc4d0,0xf59b37a6),LL(0x7ffec12f,0xdd16a22c),LL(0xd5976455,0x07decc2a), LL(0xabe1d122,0xc5019463),LL(0xe318c92c,0x2bf0ac0c),LL(0xb2bfc47b,0xfa50280a),LL(0xc7cf8bff,0x53354fc5),LL(0xe20ca341,0xaea1d293),LL(0x8b626244,0xec25ecda), + LL(0xfd8ba33e,0x313b66ca),LL(0xfabe27dd,0x10bdb130),LL(0x125e2b8c,0x1181334c),LL(0xdb6f94ba,0x0f4f198f),LL(0xac3f5de9,0xf7000076),LL(0x9d6402ae,0x1a78813d), LL(0xc8a9e758,0x3427f75d),LL(0xb01f791f,0xcdac8b34),LL(0x2a9ebaf5,0x922c36d1),LL(0xb0487cc4,0x195ea05f),LL(0xa808baec,0xe33de901),LL(0x57291d89,0x15e1d5ac), + LL(0xc21cdd1f,0x4e2a05c1),LL(0xdd46e76a,0x8a232097),LL(0xd871b1d6,0x8b55313c),LL(0xaf396bc4,0x976ce5f6),LL(0xafd381b1,0xeb91527d),LL(0x14455ee2,0x6cfd4490), LL(0x1f274d1e,0x8723be9e),LL(0x1999fa9f,0x1c63fd01),LL(0x8049b6f8,0x5f172625),LL(0x99a51b4d,0xe18a3ecd),LL(0xb13d4e65,0x329fc2c1),LL(0x0f18f300,0x94da252b), + LL(0xe28fd10d,0x7b151b98),LL(0x1dd884cf,0x8fc01ce8),LL(0x98d56c2c,0x1f0ffb50),LL(0xb084606d,0xf9df1fa2),LL(0xdc7d2008,0xf86232bf),LL(0xd8751699,0xeae5cb8f), LL(0x83ed54fd,0x70f02298),LL(0x86087697,0xb575283a),LL(0x0302e2c3,0xad219135),LL(0xc4b57e01,0x1c09a0d6),LL(0xc541b9fb,0x0f65e1e1),LL(0xf4fe76c0,0x85493d9b), + LL(0x191c21cc,0x353718ce),LL(0x4ad6bd18,0x08e6edf6),LL(0x4dc5b572,0xc2bb0d6e),LL(0x88193daa,0x328e19df),LL(0x7211c958,0xccc9f6ab),LL(0x58aae5c5,0x377d99ef), LL(0x1c823442,0x40e2ecc9),LL(0x8b0d36ab,0x036d6d52),LL(0xda4d0ad3,0x2fe0cd7e),LL(0xfc8af791,0xb8fc3c7f),LL(0x2b201b20,0xdb7e44a4),LL(0xebcf527d,0xa5176004), + LL(0xfa5256d2,0xe19b7576),LL(0xdb3f8bfd,0x418d5425),LL(0x951a1719,0x00424869),LL(0x533b69b0,0x2383c7a8),LL(0xe67a86fd,0x166a38e2),LL(0x5876c435,0xa6baa01c), LL(0x84a208f5,0x574ddc45),LL(0x26b18dbb,0x8cee30b8),LL(0xe9f6b30d,0xeced99c1),LL(0xa7d34bea,0xb638d88d),LL(0x069adedf,0xa4836806),LL(0x7a07c593,0x62beb7ee), + LL(0x724fb94d,0x5093950b),LL(0x8117ff50,0x10782271),LL(0x9f5961d7,0xdc9e34b5),LL(0x2351a33e,0xfaa2fc01),LL(0xd5fc462e,0xb9e0f1d9),LL(0xdd9c6914,0x276a5b3b), LL(0x75365ca5,0xe6136d17),LL(0xa91eed68,0x228b77e2),LL(0x411e4770,0x5cd6a269),LL(0x17590390,0xd8857b0e),LL(0xa0d45faf,0xe7094f3a),LL(0xf40693e4,0xe52d11dc), + LL(0xe5f5b545,0xe96c4aeb),LL(0x2d4c43b0,0x10a85a00),LL(0x32f9151d,0xf86ad2f6),LL(0x302b99e2,0x05daf874),LL(0x14fd3171,0x4299dbfa),LL(0x812cfc62,0x27cbedd6), LL(0xb8772164,0x42e61536),LL(0x6a5423ef,0x52eecef7),LL(0x548fffa3,0xc34c6c70),LL(0x7b6db825,0x1fbed777),LL(0x4ef2989e,0x850bded4),LL(0x815463ee,0x3b8a542c), +}, +/* digit=70 base_pwr=2^350 */ +{ + LL(0x3079fe2f,0x9decf217),LL(0x7c817f6d,0xc32ec570),LL(0xaeb36b92,0xd5649ce8),LL(0x58fb4dc8,0xab9f77d1),LL(0xb52d60cf,0x66b11fb6),LL(0xeaaa4619,0xe217941d), LL(0x1607c412,0xf3522a9a),LL(0xc2a3d8c9,0xea2eba4f),LL(0x25e38722,0x267997c6),LL(0x2d4595ee,0xed5047b7),LL(0x55e5456c,0xaaa41e5f),LL(0x78cfc6fe,0x891e3d12), + LL(0xd7357a51,0xa438634e),LL(0x18c04d59,0x918f14cd),LL(0xac40dd56,0x2ab4dedf),LL(0x4956a5de,0x758e95ee),LL(0x5113f84d,0xfc11e394),LL(0x6059f16c,0x6d71b673), LL(0xfb357c3f,0xfd8e2236),LL(0x32dca873,0xd7c86812),LL(0x8ea13b44,0x02aeb153),LL(0x013d3827,0xde1275d3),LL(0x659ca201,0x470a7b7e),LL(0x5c77b351,0x862c83c5), + LL(0x05084cfb,0xfc9b800f),LL(0x496f23fc,0x1c4d4510),LL(0xc1d08465,0xfea0003c),LL(0x9af48a41,0xf0281da0),LL(0x44d32eed,0xa5c0d971),LL(0x023a2e31,0x2613b73e), LL(0x7dc8ac1a,0x455013c8),LL(0x5958b3da,0x581b1319),LL(0x2290aaea,0xd293f2f2),LL(0x96f6223b,0xa0682564),LL(0x69410ef6,0x38fd18fa),LL(0x2b2cf629,0x74eaf35f), + LL(0xc7ff5b50,0x281f6e58),LL(0xcf9cd114,0xbc67791e),LL(0xfd89abd8,0xe29fa41a),LL(0x7984feef,0xfcb0b0b0),LL(0xd9d20a64,0x0b0928a6),LL(0x6979ccd5,0x2fd385c4), LL(0x1fbe72e4,0xce9c34c8),LL(0xaad0135f,0x69364344),LL(0x50946a5b,0xd4646352),LL(0xf39f53b9,0xb09a97c6),LL(0xdcbc8b64,0x1d47bc20),LL(0xd458b0d6,0xcda5c7bd), + LL(0x87eff3b3,0xad5b8c28),LL(0x9937833a,0xa8a3917d),LL(0x200c3b49,0xbafdc493),LL(0x972c6fbf,0x9e27aac5),LL(0x0518c97d,0xfd292bb2),LL(0x33515a63,0xa62126db), LL(0x1bcfc875,0x9892a8bb),LL(0xe0b674d1,0x93b066b7),LL(0x7fd3d080,0xcde9b008),LL(0x59401ae8,0x1e285a88),LL(0x82cfea96,0x4679e329),LL(0x23e615d3,0x52406ea0), + LL(0x8b6e9462,0x27de6113),LL(0x473464bf,0xb8ade1dc),LL(0x94dacc08,0x911ad493),LL(0x44252cb1,0xd036f28e),LL(0xd13dc20d,0x3865abf6),LL(0xd528f0ba,0xcea487cd), LL(0x4fc290fe,0x14d77eaf),LL(0xc5084101,0x5106533b),LL(0xcda9eccd,0x11001dc7),LL(0x49fc4a78,0xb79ad4bc),LL(0x4567f8a9,0x4f6369f5),LL(0xdf7ab817,0x64050aa2), + LL(0xde07f615,0xffe057aa),LL(0x342700bd,0xf3f91b55),LL(0x27a839f9,0x294761e1),LL(0x80eafe1c,0x6411a2b4),LL(0x0737b80a,0x4900eb12),LL(0xbb73264c,0xa1134d10), LL(0x0ddbf7f1,0x0ebfad73),LL(0xcd1f73ec,0x57bbe692),LL(0xa20f8944,0x675931fc),LL(0xfad2ad19,0x1950eeff),LL(0x9cdf88a0,0x60d30402),LL(0x33fd2c6e,0x121af89e), + LL(0x295c4db2,0x763e3664),LL(0xdbbaa92d,0x632fd676),LL(0xc66b40e9,0x62ab11a8),LL(0xf384b708,0x06244698),LL(0x69197876,0xe7cdf3bd),LL(0x064f8837,0x9cc79c48), LL(0x9486589e,0x95900a22),LL(0x2ff01639,0x7953f6e7),LL(0xdd3e6e46,0x3f65fbbd),LL(0xbaa2e2a0,0x84f52e06),LL(0xe3852824,0x1dc462a8),LL(0x7e4c032c,0x9be69c3f), + LL(0x70977e01,0xa40afc36),LL(0xa6c3b289,0x965f110d),LL(0x805a8491,0xc4952f87),LL(0x0b65e2d3,0xb36804b8),LL(0xe8cf2b2b,0xd3f6f5ac),LL(0xa4b71938,0x0f37a79d), LL(0x489ef671,0xb2f810d9),LL(0x2df23cd8,0x1feae026),LL(0x21a14e4f,0x7412eee3),LL(0x179d51fa,0x1458b8ad),LL(0xe201509c,0x2156a90e),LL(0x72605867,0x39f29fca), + LL(0xb2e066e3,0x231f70ad),LL(0xbb477a19,0xf09db880),LL(0x907e5c63,0xdfa0e503),LL(0xf97022ad,0x12fe09f4),LL(0x20bce7dd,0xdbf06f36),LL(0xf1371cba,0x0140e197), LL(0x64b0b4b0,0x917b6da4),LL(0x20fe3320,0x9a6f4d9b),LL(0xd66bdf87,0x0981d60e),LL(0x62d3487c,0xb430e4e0),LL(0x34dc4a94,0xc3440fb9),LL(0x09a5e3c9,0xe7972dda), + LL(0x93f47052,0x29d63940),LL(0x847e5937,0xadf04e70),LL(0x731bab6f,0xa0ef4fee),LL(0x6ee7d7bd,0x21de3195),LL(0xbd716777,0x99af4a8d),LL(0xdf4c569e,0x9e15c983), LL(0xe94401ea,0x2ec7bc0c),LL(0x85727722,0xda1835ad),LL(0x5dad81da,0x2b5862ce),LL(0x88dddc2e,0xb2be5081),LL(0x1414286b,0xa0248210),LL(0x8ea33f3f,0xc52c436d), + LL(0x3b24e776,0xcc580ea7),LL(0x9d721d6e,0x0f3a8b18),LL(0xb23480cf,0x8665604f),LL(0x34414689,0x95787cba),LL(0x4d10a945,0x425d7c6f),LL(0xb2f1cc78,0xb5ec2626), LL(0x8658de6b,0x55da8885),LL(0xe9aba03e,0xb50919d1),LL(0xd99e417e,0xc64881d7),LL(0xbf28fba2,0x1eeba5aa),LL(0x504eff80,0x20feb7b3),LL(0x50debfb7,0x9f5f9db6), + LL(0x230923db,0x4eb94584),LL(0x7b3a6929,0xba861128),LL(0xab1d6b31,0x5aa7faa3),LL(0x16ae0966,0x95c1e239),LL(0xa2fe2297,0x98674fd3),LL(0x3c42d488,0xa8da0ee5), LL(0xe0740db0,0x103cabac),LL(0x5bf16882,0xf0b860d4),LL(0x289e48ce,0x03cb0cdc),LL(0x9e52c7d5,0x3c15d375),LL(0x98103ca2,0x524f7319),LL(0xc609ffeb,0x828ed65c), + LL(0x83dfb993,0x518f231b),LL(0x37c0826c,0x4b0987db),LL(0xd5177ead,0x0c34961c),LL(0x452c92da,0x9d882d3e),LL(0x8765bced,0xbfeaf558),LL(0xb9962295,0x83957b62), LL(0x7bb084cf,0x2d1d0175),LL(0xe8cffcfc,0x04c4cfcd),LL(0x8d4536c1,0x2f35e33d),LL(0xd83124cf,0xbebb31cb),LL(0xabb29019,0xe342bed2),LL(0x2692a0d3,0x2af0fcde), + LL(0xc7e3b29f,0xece5d865),LL(0x622839dd,0xe58106a4),LL(0xf2969d76,0xf5272d43),LL(0x2a1a240f,0x90c72c1b),LL(0xaf15e14f,0x1e2aa0ac),LL(0xf1b6b5a0,0xfa2f1c7b), LL(0x880224a5,0xfb5d343d),LL(0xf91881c5,0x47b88a84),LL(0xdd142fe7,0x140f5ee9),LL(0x24b37c44,0x4e76982e),LL(0x578b482b,0x6aaf61e9),LL(0x765bc4e2,0x01950e22), + LL(0xe8a2e8f0,0x20ebf79c),LL(0xaca418a2,0xec040d0d),LL(0x8d630d2a,0x016c07e7),LL(0xfa605dcb,0x20021d57),LL(0x42d04705,0x6190f3e9),LL(0x8974b7e6,0x4e000df5), LL(0x5abcedac,0x6710da6c),LL(0x5f95d37c,0xf31aa496),LL(0xa5830899,0x192c4b8b),LL(0xea7dbcdd,0x171ab8c4),LL(0x8cdf1097,0x715f6081),LL(0x205d10ed,0x0e0135bf), +}, +/* digit=71 base_pwr=2^355 */ +{ + LL(0xc8d2bf7b,0xd2db4d35),LL(0x81571d06,0x52105d09),LL(0x723a57bf,0x447565cc),LL(0xd8ded62c,0xd98c3597),LL(0xde2f1a9e,0x0aeac6d9),LL(0x0a98d3b2,0xd363b0b7), LL(0x02ad9933,0xd9708f07),LL(0x64f5809d,0x93346775),LL(0x49cda010,0x499332cf),LL(0x858467e2,0x546df74a),LL(0x93748e8e,0x8b84a550),LL(0x06f09073,0x9e88ef97), + LL(0x52133095,0xc0a40cac),LL(0x93c162bb,0xfe1b22fd),LL(0x34018741,0x8625898c),LL(0x36d9e57a,0x69c9f3f6),LL(0x378aa211,0x69d9d7f3),LL(0xe7dca168,0x6b03f897), LL(0xf997a48f,0x24d49aeb),LL(0xc149ac40,0x1d984c67),LL(0x576f533f,0x667c1d01),LL(0x9ef82ece,0x372eee19),LL(0xc207c14d,0x577723c0),LL(0x0eed37f6,0x4225907a), + LL(0xc5a5a001,0xb8623f36),LL(0x86878b74,0x21847b80),LL(0xd05ac443,0xd19f57f6),LL(0xb9f8acec,0x9bd22882),LL(0x2b41b60a,0x128186d8),LL(0x71980fd1,0x1772e6b0), LL(0x812dfff5,0x22c5ee68),LL(0x7f952796,0xccbd2fe2),LL(0x7da6d35a,0x0d49bfde),LL(0xc249f319,0x348b48db),LL(0xc16a8c0f,0xdb376657),LL(0x002cf8b4,0x28ef362a), + LL(0xbc0e0903,0xc61db977),LL(0x645c32fb,0xbaf6e4da),LL(0x060b1adb,0xce89b8ca),LL(0x88e2c178,0x41db4481),LL(0x923bdd3c,0xba6339f3),LL(0xd29db42c,0xff25b818), LL(0xe6d6b35d,0x3521116e),LL(0xb22f16ac,0x4e1bd283),LL(0xbd79fe5d,0x9357c984),LL(0x9d45eee4,0x2eda73be),LL(0x6288e01f,0x1a50c59f),LL(0x75018873,0x37baf649), + LL(0x751f66f6,0x431ea808),LL(0xf1b8e577,0x06feeefa),LL(0x488a8eee,0xd109d9f0),LL(0xc69843c5,0xeb826b96),LL(0x8b42da29,0x972272e9),LL(0xa137ee9c,0xa9ea9ad1), LL(0x25163be5,0x0385aa95),LL(0x34c32f2b,0x963e9640),LL(0x4dfd935a,0x9067aa89),LL(0xbc3f5f3c,0x6ab23131),LL(0xf302e0e1,0xb96c3406),LL(0xd65f811a,0xc8caad4e), + LL(0x750261dc,0xf323953b),LL(0x38552a8c,0xb8563bf1),LL(0xbd32cb8c,0x2937dfd9),LL(0x7ecf3538,0x07c4e563),LL(0xb6399415,0xb573960e),LL(0xdd1a4a06,0xc1ced6c1), LL(0x787ddf7a,0xc625f400),LL(0x1ce6778f,0xd998c28b),LL(0x6220f24d,0x66c51e5b),LL(0x27d01c6f,0x9f97d758),LL(0x0412372d,0x60b15724),LL(0x04d55048,0x2253abc1), + LL(0xfad177ed,0x15952eb8),LL(0x05058d9f,0x57aaf91d),LL(0x8ab1b9d9,0x268ba730),LL(0xd5b8f86a,0x7decfc47),LL(0x0879ab02,0x596353e7),LL(0x9a68d5a4,0xa3ff2311), LL(0x7534fb5a,0x257c68a1),LL(0x160ec5ea,0x84f7c9de),LL(0x6754185f,0x1b2b770c),LL(0xa74562f7,0xf321ac71),LL(0x264a1961,0x28bf0a15),LL(0x86a033ae,0x0093db73), + LL(0x097f322f,0xc552c6c6),LL(0x8bc06287,0xdf59a302),LL(0x19610b0c,0xc9ed375c),LL(0xb051dad5,0xf0e7b4eb),LL(0xc6556643,0x7267a304),LL(0xc96dc1d8,0x0044f6d8), LL(0xf4fc3725,0xf0ed5f9a),LL(0x9de8e1ff,0xbbaf9f2c),LL(0xaf5a4b4b,0xef5d66f4),LL(0x20644cf2,0x0b5bed3d),LL(0x75ae23c0,0xf7e4543a),LL(0x41325b66,0x696f60dc), + LL(0xcfc0dd38,0x4fd0b582),LL(0x55b7b68a,0x47fd984f),LL(0x699460ec,0x2722a0cb),LL(0xfa26d4c4,0x81b4afd8),LL(0x941c86e4,0xb921d0d5),LL(0xef4db114,0x7208f094), LL(0x17ddadf8,0x2997f43c),LL(0x21bc290d,0xd1aabdea),LL(0x86182be1,0x64e20e00),LL(0xcbddb8eb,0x9bd11568),LL(0x3ba0e6b5,0x639db1d4),LL(0x429a6b4f,0xb99b11fb), + LL(0xaaa48cd8,0x04ef7ad5),LL(0x8a8ac319,0xb97a6501),LL(0x47591d88,0x9ae38a6a),LL(0x6902edb4,0x27d91254),LL(0x812b143a,0x5dae3d83),LL(0x93a2fdf6,0x02ee1353), LL(0x72410377,0x07a00389),LL(0x56e10c82,0xa2fbd343),LL(0x72b1bcb9,0x3fd6c171),LL(0x2d0033c7,0xa8d70f93),LL(0x2916c28a,0x9ea9eea3),LL(0x423edad7,0xecb7e486), + LL(0xc515bacf,0x5525fbb0),LL(0xa2aa22ba,0x48a72394),LL(0x0b9e3fe3,0x9bcd64c9),LL(0x1975aa86,0xe9e11d17),LL(0xc9dbdaac,0xa435bbf0),LL(0xe30c8911,0xe8451f6c), LL(0x5bc2d12f,0xa1706d89),LL(0x406d4883,0xfe73ff43),LL(0x49d5739c,0xb713efc6),LL(0x05c1ec9a,0xdfd0bbf8),LL(0x5e6b3bd0,0xde17c52c),LL(0x5be196af,0x57e06034), + LL(0x2c20f868,0x9949b33d),LL(0xb5706250,0xdb3aa790),LL(0x88e17f2b,0x88ce71e7),LL(0xda9c0881,0xd851baf2),LL(0x86d8c9e9,0xe869c5ba),LL(0xa01425b6,0x1af68d65), LL(0x9bbd3963,0xeae8b1c6),LL(0xec087425,0xf34900b1),LL(0xc374bb96,0x14942910),LL(0x05487483,0x3e13c457),LL(0x35bc6ee1,0xe0e6fad4),LL(0xb54d247b,0xc7c38dc7), + LL(0x8bd92789,0x420dd8f6),LL(0x4ce541dd,0x3831b0e9),LL(0x31ed7f7f,0xbde477e4),LL(0x29c5557b,0xb46eb7f2),LL(0xc56940e0,0x00a07499),LL(0xabacf00e,0xabb9d567), LL(0xc5ba9d0c,0x9f63b87d),LL(0x708d4f4c,0xf4c5c4a3),LL(0xde7fcf63,0xdbde9879),LL(0x616cf5d6,0xbe88f949),LL(0xd1aa38be,0x17560674),LL(0x9c436175,0x160ec365), + LL(0xb9bd0afb,0x9c85e50a),LL(0xd26ac425,0x2b66a1c0),LL(0x807f33db,0xcb78ce81),LL(0x3db81e06,0x9d337d8a),LL(0x72638d70,0xe223eae4),LL(0x6bf2ebab,0x7f9ea217), LL(0x9b634059,0x2804b59c),LL(0xf3dc8d46,0x1043fbf4),LL(0x7bc6949c,0x321eca1e),LL(0x8f051155,0xbf2906c0),LL(0x9c539f40,0xb802a328),LL(0x073cd808,0x08bcca20), + LL(0xcea8bf63,0x08cb6315),LL(0xb59ee7fa,0x7ac2699d),LL(0x8d1601e0,0x4cd2c8a9),LL(0x7b90d9c3,0x6d7188e3),LL(0x6bfe73d3,0x63486716),LL(0xcf9f30b0,0x49ed7faa), LL(0xc9515fd6,0xf7edf5d6),LL(0x8a8ae607,0xbb3ab848),LL(0x21c5c388,0xe6e6b209),LL(0x5b1e03bc,0x5dca0a1d),LL(0x94a8b174,0x15670f59),LL(0x68c27a97,0x6f79e381), + LL(0xab463fa6,0x6d34bdf6),LL(0x0093b9cb,0x7bb127b6),LL(0x5a3bfdd0,0x61d05113),LL(0xf1296bdd,0x4abab575),LL(0x4d2e9a7c,0x72da6849),LL(0x8d11f03d,0x90267bca), LL(0x3e9b310b,0x47811122),LL(0x1b1920cd,0x8ffe91d3),LL(0x7521898e,0xec293ec6),LL(0x96c1da75,0xf0cf0269),LL(0x80f2c7b3,0xb0dbd4c3),LL(0x34e4baf8,0xe5281755), +}, +/* digit=72 base_pwr=2^360 */ +{ + LL(0xdd14d47e,0xc32730e8),LL(0xc0f01e0f,0xcdc1fd42),LL(0x3f5cd846,0x2bacfdbf),LL(0x7272d4dd,0x45f36416),LL(0x5eb75776,0xdd813a79),LL(0x50997be2,0xb57885e4), LL(0xdb8c9829,0xda054e2b),LL(0xaab5a594,0x4161d820),LL(0x026116a3,0x4c428f31),LL(0xdcd85e91,0x372af9a0),LL(0x673adc2d,0xfda6e903),LL(0xa8db59e6,0x4526b8ac), + LL(0xcecb916f,0x51a033a5),LL(0x8d7de61c,0x2ac62f63),LL(0xa42a266e,0x92eece49),LL(0x82c4d11e,0x87e037db),LL(0x6fbae08a,0x875be141),LL(0xc539478c,0xf348fe26), LL(0xff94c01e,0x51f8b907),LL(0x19695a9d,0xc46cc0e0),LL(0x6c51b9c2,0x2c74bd66),LL(0xee565de8,0x635d3d24),LL(0x8982c8c3,0x6bd65663),LL(0xdaf6a93c,0x5c345b79), + LL(0xe23a8472,0x68fe359d),LL(0x4ce3c101,0x43eb12bd),LL(0xfc704935,0x0ec652c3),LL(0x52e4e22d,0x1eeff1f9),LL(0x083e3ada,0xba6777cb),LL(0x8befc871,0xab52d7dc), LL(0x497cbd59,0x4ede689f),LL(0x27577dd9,0xc8ae42b9),LL(0x7ab83c27,0xe0f08051),LL(0x2c8c1f48,0x1f3d5f25),LL(0xaf241aac,0x57991607),LL(0xb8a337e0,0xc4458b0a), + LL(0x210c3144,0x179c59cf),LL(0x33eebbc4,0xfb613c57),LL(0xba0cf384,0xdda75cfd),LL(0x3a8fbafa,0x94081a5b),LL(0x33384e0b,0xb91de90a),LL(0x27aa2a45,0x7d1f8f40), LL(0x62031148,0x0747bcc1),LL(0xf324160b,0xd2db8e39),LL(0x722484f0,0x9c1ce3e9),LL(0xa62d1dda,0x13a7ee5d),LL(0x3a963bce,0x77fd7934),LL(0x83d2f21b,0xcd3d8717), + LL(0x51dd1ba9,0x3dbb3fa6),LL(0x545e960b,0xe53c1c4d),LL(0x793ce803,0x35ac6574),LL(0x83dbce4f,0xb2697dc7),LL(0xe13cf6b0,0xe35c5bf2),LL(0xb0c4a164,0x35034280), LL(0xd9c0d3c1,0xaa490908),LL(0xcb4d2e90,0x2cce614d),LL(0x54d504e4,0xf646e96c),LL(0xb73310a3,0xd74e7541),LL(0x18bde5da,0xead71596),LL(0xaa09aef7,0x96e7f4a8), + LL(0x2d8bcd6e,0x609deb16),LL(0x2591750d,0xe42f23a9),LL(0xb378305c,0x4a9f3132),LL(0x69275f5e,0xf1017998),LL(0x61b089b5,0x14be7467),LL(0x0c81b0c5,0x05f620d2), LL(0x6cb8412e,0xca90a9c0),LL(0x15b1b0d5,0xfe0f6a89),LL(0x20c71988,0x1b25ac96),LL(0x390aedd0,0xb971b61a),LL(0x79d8cd39,0x995214d7),LL(0x65c6e11a,0xd7fa135b), + LL(0x5d6e5f48,0xa8393a24),LL(0xf9175ce8,0x2c8d7ea2),LL(0x55a20268,0xd8824e02),LL(0xa446bcc6,0x9dd9a272),LL(0x5351499b,0xc929cded),LL(0xcfe76535,0xea5ad9ec), LL(0xdc32d001,0x26f3d7d9),LL(0x43eb9689,0x51c3be83),LL(0x759e6ddb,0x91fdcc06),LL(0xe302b891,0xac2e1904),LL(0xc207e1f7,0xad25c645),LL(0xab3deb4a,0x28a70f0d), + LL(0x0f3ff12d,0xa13f19b4),LL(0x019564aa,0x57ee08b1),LL(0x7044a6f4,0x00ec0c99),LL(0xdca1075c,0xaf5665f8),LL(0x0620ab0c,0xded5ca3f),LL(0xa896deff,0x9b2cb8c7), LL(0x07df2345,0x032ab2b3),LL(0xf1da3f88,0x964d109e),LL(0x25133304,0x2286b6f7),LL(0x977a4567,0x0d16d531),LL(0xf1abae4f,0x00a66036),LL(0x95f0103b,0x5debab1d), + LL(0x03bea8f1,0x922d7f97),LL(0x584570be,0x3ad820d4),LL(0x3cd46b43,0x0ce0a850),LL(0xae66743d,0x4c07911f),LL(0xfda60023,0x66519eb9),LL(0xec2acd9c,0x7f83004b), LL(0xc3117ead,0x001e0b80),LL(0x0722ba25,0xbb72d541),LL(0x6e9a5078,0x3af7db96),LL(0x701b6b4c,0x86c5774e),LL(0x37824db5,0xbd2c0e8e),LL(0xbfac286d,0x3ae3028c), + LL(0x5fb7c5a8,0xfd1b413f),LL(0x0206cba1,0x6152b9de),LL(0xd8f51960,0x487f8e3a),LL(0x033ca1bc,0xac34a23c),LL(0x60258d55,0x90bba98f),LL(0xbd9098f2,0x30421acf), LL(0x89c0ce44,0xd9c601f9),LL(0x2f2f1af1,0x621bda83),LL(0x38c45441,0x14fa7ef6),LL(0xe47faa31,0xbd5dc10f),LL(0x74eeb6a1,0x9dce0dcb),LL(0x06346849,0x2cca3e66), + LL(0xa33e071b,0x83d4d4a8),LL(0x61444bb5,0x881c0a92),LL(0x520e3bc3,0xeea1e292),LL(0x2aaab729,0x5a5f4c3c),LL(0xe63c7c94,0x0b766c5e),LL(0xbb2cc79c,0x62bb8a9f), LL(0xaa5dc49d,0x97adc7d2),LL(0x31718681,0x30cc26b3),LL(0x56e86ede,0xac86e6ff),LL(0xcd52f7f2,0x37bca7a2),LL(0x9ce6d87f,0x734d2c94),LL(0xc2f7e0ca,0x06a71d71), + LL(0x20066faf,0xcdb67ea9),LL(0xc7fb7154,0x929b4d2a),LL(0x7fdeb411,0x5842d968),LL(0xafe55cb9,0x2ddf764a),LL(0x47df3cf4,0x608bf76f),LL(0x5984e339,0x1862463b), LL(0x944d22a2,0x7feea86f),LL(0x281f2b84,0xf8562c30),LL(0xbd358ea4,0x332a54d2),LL(0xe7fe1ede,0xa54dec9f),LL(0x932264d4,0x9c8e52a4),LL(0x89817f5b,0x428acd1a), + LL(0xc6357d33,0x559dcf75),LL(0x652517de,0x4616d940),LL(0x1ccf207b,0x3d576b98),LL(0x1979f631,0x51e2d1ef),LL(0x06ae8296,0x57517ddd),LL(0xd6e7151f,0x309a3d7f), LL(0x0e3a6fe5,0xba2a23e6),LL(0xd28b22c3,0x76cf674a),LL(0xf8b808c3,0xd235ad07),LL(0x6b71213a,0x7bbf4c58),LL(0x93271ebb,0x0676792e),LL(0x05b1fc31,0x2cfd2c76), + LL(0xeda2fa8f,0x42aeebc4),LL(0xe77a9c5b,0xda91ada3),LL(0xc585a572,0x29b9d55f),LL(0xa256353d,0xb0e52414),LL(0x29adbd21,0x1d0e7d5f),LL(0xd057d175,0x7ee5ff9c), LL(0xc9097bf9,0x0bf76fcf),LL(0xfe09f5b3,0x023170f8),LL(0x0799f989,0x8a67c124),LL(0xc6a20819,0x4ce28eeb),LL(0x79502d13,0xfc1d7c91),LL(0xde43f895,0x7922d2d9), + LL(0x37a450f5,0x4258e5c0),LL(0x52d2b118,0xc3245f1b),LL(0x82bc5963,0x6df7b484),LL(0x9c273d1e,0xe520da4d),LL(0x2c3010e5,0xed78e012),LL(0x3c1d4c05,0x11222948), LL(0xc692b490,0xe3dae5af),LL(0xc197f793,0x3272bd10),LL(0xe709acaa,0xf7eae411),LL(0x778270a6,0x00b0c95f),LL(0x220d4350,0x4da76ee1),LL(0xab71e308,0x521e1461), + LL(0x96230a58,0xf2cdae31),LL(0xf304e1ea,0x47cf36b4),LL(0xd750b29b,0x9d14f25a),LL(0xdba15f3e,0x931b9c65),LL(0xbf9fe2dd,0x34db169e),LL(0x52663433,0x8a89e47c), LL(0x026ec31f,0x8859a4f8),LL(0xa913ceea,0xeeb703ea),LL(0x67ac4db7,0x74638d6a),LL(0xbe25d755,0x5c8ea7b2),LL(0x38db10ee,0x8a0f0a87),LL(0xe890bcd3,0x96a26bac), +}, +/* digit=73 base_pwr=2^365 */ +{ + LL(0x64a5e869,0x883533af),LL(0x6973ec23,0xaaa778c2),LL(0x46d0fcf3,0x8f0b5fb5),LL(0x4ab05da7,0x7e6b0a0a),LL(0xc67b6614,0xcd91a869),LL(0x6c6f7cf2,0x7de9f2ff), LL(0xd1ec14c3,0xc072a106),LL(0x42a128ee,0x3f4b9606),LL(0x8f0ce946,0x7378192c),LL(0xd1149441,0xdf2e7b9f),LL(0x14ccf45a,0x4fa17cb6),LL(0x45f03568,0x575680e9), + LL(0x8f458c68,0x8b70e94f),LL(0x4160ecc7,0x29272654),LL(0x4d3ef22f,0xe22219ba),LL(0x1999f948,0x7f8a712a),LL(0xabfe7302,0x25575e96),LL(0x564a1af0,0x21c6ffc6), LL(0x7e8500da,0x045e9c66),LL(0x04ef8ea6,0xef7c3cf7),LL(0xc3db161a,0xdd23b825),LL(0xba33a906,0x05fb173a),LL(0x870e41f2,0x9a8b5ecb),LL(0xccc30d1d,0xf3d9db0b), + LL(0xe873ff0f,0xd9243926),LL(0xf20b0e49,0x2e2a5ab6),LL(0x0e35f201,0xa1bcfeee),LL(0x196f23f3,0xd25be5f3),LL(0xffc1d068,0x298c67f2),LL(0x0c3d950b,0x77dae55c), LL(0x8822c996,0x5e15ab99),LL(0x83f60a98,0x52de2e6d),LL(0x47a7e269,0xa9f82ec9),LL(0x2ac22e49,0xf02af9a2),LL(0xa706f090,0xdfb3103f),LL(0x3cf8dcb0,0x12559962), + LL(0x88ad12b2,0xb7b85625),LL(0x1e44b254,0x81f5958b),LL(0xc91b8ffd,0xb4ebddd5),LL(0x55d38511,0xef815ae1),LL(0x1b0da525,0x98587d55),LL(0x34a9ebbd,0x1d418177), LL(0x1e6057d7,0x844811fb),LL(0x76e5976d,0x0c169771),LL(0xf623789b,0x4b268bb4),LL(0x40498058,0xb26ae5be),LL(0x3c2b435a,0xb47a5ded),LL(0x8fceceb3,0xe15a684b), + LL(0xd09dc3c8,0x122a3eed),LL(0xaefe0819,0x6a19907f),LL(0xda325339,0x057aafa1),LL(0xd42a5345,0x138033bd),LL(0x1a065ebe,0x8ac206af),LL(0x25c31ed6,0x0a46f5ae), LL(0xd7e379db,0x7fc577a9),LL(0x69dcee54,0xc6df6943),LL(0xa8336bc1,0x4c167ba2),LL(0xf3a1339c,0x0fbd9708),LL(0x226f612f,0xc6b8c01f),LL(0xd51332e1,0x5d4ed789), + LL(0x5a1abcd8,0x26aa2c2e),LL(0x9609d9d8,0x2b16a12e),LL(0xa2bee00c,0xe485a551),LL(0xf4f2292e,0xfa28c30b),LL(0xb7358f1b,0x99abef78),LL(0x10a276a1,0xda6b3cdf), LL(0x47c03f71,0xbd3858b7),LL(0xb22d05d1,0x4f0bf5f0),LL(0x8250f760,0x2d80f5d2),LL(0x8cd9666c,0x060f9b27),LL(0xb1b014a9,0x6a6c40b0),LL(0x8c440a9e,0x44537af3), + LL(0x76faaca5,0xb564cfd6),LL(0x920dd223,0x8a6e3925),LL(0xa590a383,0xee59a140),LL(0xa1922ad9,0x9e29b552),LL(0x60a0da63,0x604367de),LL(0x92c35fd0,0xc498aca5), LL(0x250ed8a0,0x74135082),LL(0x6c7c3e77,0x5d109d1a),LL(0xc63dff94,0xf9e2d84d),LL(0xf7aa2b0e,0xca50f5e4),LL(0xd543d389,0x7cba9e87),LL(0xd8fd1292,0xaf5fbbef), + LL(0x4fc11c3a,0x70765683),LL(0x66aac4d1,0x53a94031),LL(0xa6db6169,0x2a935ef0),LL(0x2032d599,0x00292761),LL(0x3a6f1316,0xb5babb2d),LL(0xdb26af51,0x601a7dfa), LL(0x1322d983,0x00c34013),LL(0x2bb507c5,0x45b062ec),LL(0x0f9b3656,0xa1bbe2ed),LL(0x34031d18,0xe17a5d49),LL(0xf8fe1224,0xe3661047),LL(0x623c6cf5,0x0e4f3b3d), + LL(0xca45cccf,0xffaac084),LL(0x061ffe3c,0xaea5cc3d),LL(0xb355f387,0x7c5d7c60),LL(0x99cba92d,0x4bbb2a0c),LL(0x2f7768d6,0x6b4ba3ef),LL(0xcc5f9c40,0xc7484ed2), LL(0x52b57a7e,0x5d4e92fc),LL(0xca2c200b,0xba9f16c4),LL(0x3797ccba,0xebe02a8a),LL(0x38c4133a,0xb6b3f421),LL(0x8153d033,0xad5d85b6),LL(0x5714f269,0x782d6ee8), + LL(0x9c0cf752,0x7845b696),LL(0x5a732acf,0xb82d052b),LL(0x1262877b,0x7760564c),LL(0x8ecc7aa5,0x29b3c57a),LL(0xdf1ebbed,0xb58eccb0),LL(0x3c3a3303,0x86fc1544), LL(0x13060f0e,0x44761ddf),LL(0x7371a5a8,0x5a3dacfd),LL(0xf7cbc2bc,0x846f6faa),LL(0x368caabf,0xf5e098b0),LL(0x10c08769,0xe23ea107),LL(0x1563fcda,0xbc5df1db), + LL(0x142d8955,0x65c3a54e),LL(0xe7814f25,0x5c6583cc),LL(0xd924dc7d,0xbd5a07d8),LL(0xc28f6e8e,0x9f717bd9),LL(0x3b6540a7,0xa0c4ec4e),LL(0x142b3283,0x3153bb2b), LL(0x9b296946,0x53bf403c),LL(0xb1cdb6d2,0x659a828a),LL(0x1369af19,0xe9517d81),LL(0x8990e7a0,0xd8c7a099),LL(0xe535cd04,0xbaa9f59d),LL(0x0f269551,0xbb0cc68e), + LL(0x6a78c6e5,0x3c00ac52),LL(0xdefaa52c,0x9c61aca6),LL(0x39794a09,0x00341289),LL(0x41cd7c0a,0xe08910d1),LL(0xa732e3bc,0xa58ffbb6),LL(0x91fe8fd8,0x87bf51ab), LL(0x4a59e2be,0xc4f4f267),LL(0x438071c8,0xdeb512c7),LL(0xe9cd290b,0xddf82842),LL(0x6ae85fe0,0x3e17311d),LL(0xb41be881,0x6e9236a9),LL(0x53555ebf,0xbb9ddf98), + LL(0x09f3f0be,0xccc163ea),LL(0x6a5b0a63,0x9932b56f),LL(0x9c69668e,0xf89fae91),LL(0x5ce13021,0x555f9821),LL(0x37037aa9,0x4b02693f),LL(0xbde50f35,0xc4afee79), LL(0x02aa6c7a,0x4b0919c2),LL(0x991e15e9,0x3166de2a),LL(0x7077fb38,0x284baa3e),LL(0xa116ddec,0xbb7a6416),LL(0xb7636772,0xe8c89547),LL(0x0ef92c54,0xff940362), + LL(0xe2ce6008,0xd5d81275),LL(0x0b3b9d10,0xc45bdf25),LL(0x6cbc83e2,0x15ab5da3),LL(0xc52a66cb,0x85a18cf8),LL(0xb042c047,0x77e202b8),LL(0xe7e7997e,0xc4dc3de2), LL(0x995fa67a,0xfe9335b1),LL(0x75b96a00,0x809e161d),LL(0xa0c3baea,0xfb03c2a5),LL(0x888c2f77,0x5c7e0523),LL(0x87ad10e2,0xa8fda1c8),LL(0x858a3577,0x90484f78), + LL(0xf9fde126,0x49e41f0a),LL(0x3613d3c2,0xec960044),LL(0x10421d3b,0x2c62a49d),LL(0x8131a0d8,0xe2402464),LL(0xbdf794fc,0x8a7ce188),LL(0x4665b1b6,0x704dea7d), LL(0x4d57c6ba,0xbdb9c18e),LL(0xf669b3c0,0x5288a053),LL(0x78a5e252,0xbf7d01b8),LL(0x26b9cb7d,0xb26cccf9),LL(0x84326c47,0x14191a32),LL(0x91f8425b,0x460ff747), + LL(0xbd27be7b,0x59367582),LL(0x1ab2c596,0x92bf5bbc),LL(0xf6a27741,0x5d96351a),LL(0x7f929e0d,0xeab94db8),LL(0x043f1afb,0x865ba011),LL(0x5fb631dd,0x43acea12), LL(0xb2fd1436,0x192e0652),LL(0x7b38d121,0x44f22ff1),LL(0xb7cae5f6,0x7bcc228d),LL(0x6a828b03,0x02eaeccd),LL(0x91f301aa,0x7c48a2ea),LL(0xf5eb1a07,0x1e090717), +}, +/* digit=74 base_pwr=2^370 */ +{ + LL(0x941948e3,0xdf0ae8df),LL(0x1d010bcd,0x123fee90),LL(0x1dd28691,0xde3717ca),LL(0x709b678e,0x0c1db879),LL(0x400acdc6,0x0288959a),LL(0x5ca2d03a,0x66c69181), LL(0xdbbb75de,0xe52534b3),LL(0x3de927cf,0xe914938c),LL(0x73eece30,0x1a9a34f8),LL(0x642a6799,0x0fb0c7bd),LL(0xeaa7e8a8,0x375cc0cf),LL(0xd00ec238,0x75fb9eb5), + LL(0xb72958eb,0x9ca8cc9d),LL(0x1014f562,0x3c8cd0db),LL(0x059b2bba,0x72115d53),LL(0x730e5dc3,0x8fe7ac30),LL(0x841d8998,0x4e67ef69),LL(0xc8ed37a5,0xfb6439ff), LL(0x26df84c4,0x48164b3e),LL(0x365bc99e,0x37d492ad),LL(0xbeed38ce,0xb7fd4643),LL(0xa3e30b3d,0x993cfa9f),LL(0x01ddd484,0xdcc5e7af),LL(0x6840175d,0x5edf3ac0), + LL(0x0c19c625,0x3ee87e54),LL(0xf4a10f9a,0xe4ae611f),LL(0x0aec21e5,0x27d65512),LL(0x8bea1b16,0x737578f5),LL(0x5fb7a3a2,0x6e2b6bf7),LL(0x14f65000,0xecc59251), LL(0xb8dd544c,0x53e1167c),LL(0xc5862fdc,0xee60e60f),LL(0x6cccabeb,0xc86582cf),LL(0x8b4d37c6,0x849fc15d),LL(0x8f4a87bc,0xaa7960ad),LL(0x3b7f0a6d,0x17fe1082), + LL(0x51d33c11,0x79768e9f),LL(0xa4b24889,0xeec34505),LL(0xbe0c67d7,0xc194821b),LL(0x6909fdfc,0x537a6a4a),LL(0x95ccdda7,0xae6d7051),LL(0x92b3926c,0xed4b7222), LL(0x6b24a3d4,0x2c5dd6af),LL(0xe4386095,0x9282ec39),LL(0x397a3bd3,0xdd3c7388),LL(0x8baf59c2,0x9d176c6a),LL(0x380ec958,0xd5c6219e),LL(0x54e8e315,0x194fc116), + LL(0x79650de1,0x3c67c65c),LL(0x00bfb2ac,0x8e2a220b),LL(0x42f02a3d,0x59d225dc),LL(0x5d60e54f,0xc52ce4b2),LL(0x3f306112,0x8894e3cb),LL(0x50d85aec,0xb78a037b), LL(0x2d85f328,0xeeeb3b40),LL(0xe0406ddd,0x3d391b37),LL(0x0502141b,0x5273ebe9),LL(0xe092bfeb,0xd17023e7),LL(0x282f5aed,0x04564385),LL(0x55d82356,0x7e55a4e2), + LL(0x01ec1432,0x0b7576f9),LL(0xabc5f603,0x84b30eec),LL(0xf4a84b7a,0xdaaf7ba9),LL(0x3bb37a99,0x9e3a5daa),LL(0x80378cff,0x56bd9648),LL(0x8e6ed856,0x2fdeeeb7), LL(0x3c81ac34,0x079014a7),LL(0xb4211c27,0xf8de4004),LL(0x7fe4391e,0x0cee3df9),LL(0x2fd2fc38,0x441aa7fb),LL(0x4d1b575a,0xeba7be86),LL(0x231c2d01,0xca2fb5b7), + LL(0xd205869c,0x8307bc81),LL(0x9388e917,0xdd282809),LL(0xbb572a87,0xdd4cb2de),LL(0x14f0db39,0x5a41595d),LL(0xf4e5c71e,0xd5db5f59),LL(0x4a87f672,0xcc9c9f6f), LL(0x012e65ce,0x4ddce8c3),LL(0xab3a94d3,0xaae28eaf),LL(0x0b333aba,0xd4b95011),LL(0x914eb476,0x3e171506),LL(0x69bbfa63,0x109ee9ab),LL(0x5dff5471,0xbddba098), + LL(0x463a8251,0x0683349e),LL(0x5103e72c,0x97dc4f47),LL(0x50663b9e,0x47c71810),LL(0x9733dac6,0xf327d149),LL(0x292137c5,0x03f55e4a),LL(0xda59e1c7,0xccc6232a), LL(0xadc59cb8,0xbaa8b4dd),LL(0xfe7486e4,0x45370d35),LL(0xb0322df9,0x99a88c1d),LL(0xfd69954f,0x394440a7),LL(0xa5a29889,0x9060473d),LL(0x2f04864a,0xc8ca43e3), + LL(0x9451d422,0xf08c8efc),LL(0xba846ac3,0xc8beae41),LL(0x807df062,0x0157f46c),LL(0xdeaada6a,0x080eac20),LL(0xa3bfddd0,0xdf146f3d),LL(0x617d19d7,0x8ed7c2c6), LL(0x40439488,0x314ea3e7),LL(0xb3e6806f,0x544132c7),LL(0xa215a3a2,0x9b9bb477),LL(0xe33f80b0,0x9310fee9),LL(0xe71769af,0xf277895c),LL(0x7d99d5b9,0xe8218e1a), + LL(0xdd8b9644,0xd5c7241a),LL(0x993116d2,0x45a5f2d1),LL(0x823048df,0xbacacd4a),LL(0x2568547a,0xa278fa04),LL(0xbff72820,0x3a4f2482),LL(0x13e4314a,0x1305d1a7), LL(0x34ba28e3,0x9d84c333),LL(0x6a32fb41,0x9995b5bb),LL(0x520946d8,0xb0f75f3c),LL(0xde98aa63,0xd7c4b8b7),LL(0xba856b6b,0xee5efcf3),LL(0x3324ed66,0x36af3368), + LL(0xbb2af5fb,0x29c741c3),LL(0x89e6241e,0xb9c92da6),LL(0x474b7c0f,0x07dace3c),LL(0xd996b6a8,0xd0b3f9bc),LL(0x07c662de,0xe97e3847),LL(0x6c851030,0xbf6d1e05), LL(0x99f1aade,0x150c5e93),LL(0x9bd848d5,0x9c1e2351),LL(0x09cab3db,0xeb808a55),LL(0x9a49916f,0x1bfbe08d),LL(0x64ab0e4e,0xc6a70ea7),LL(0x7823f505,0x77d189f2), + LL(0x5827fe2c,0x90762c1f),LL(0xeaffda88,0x20160f7a),LL(0x5c47c645,0x7420849f),LL(0x6d72e748,0xb0823195),LL(0x8ee11773,0xaeac683b),LL(0xfb5c550e,0x8c2a0a79), LL(0x6c07cc1e,0x6d986d69),LL(0xba8398b9,0x57269140),LL(0xd94d5223,0xd13e136f),LL(0xed5b01c6,0x1aa75419),LL(0x408fcdcc,0x7c2014b1),LL(0xcffde5ee,0x0680a985), + LL(0x23133885,0xfbe88fce),LL(0x7c5a5c4b,0x3c2e3669),LL(0xbbacc6e1,0xe36cf261),LL(0x96ae3cad,0xa72c7bb1),LL(0x4cb1a375,0x08e37103),LL(0x2a02baee,0x5521f445), LL(0xc157e471,0xda9329ba),LL(0x68470808,0x3f90cad1),LL(0xa657de60,0x24182208),LL(0x67c10d1b,0x17c082b7),LL(0x9928d6fd,0x9e222648),LL(0x36ced38f,0x1578c895), + LL(0x42c5a7ee,0x92fc9a33),LL(0xf9f0ed71,0x8768614a),LL(0x87ebfb66,0x1ea5f7ed),LL(0xd361069c,0x296852de),LL(0x0192498e,0x1cec6f1a),LL(0xa9cca3aa,0xbfd4858f), LL(0x2ef240e8,0xfba98c24),LL(0xab635d9f,0xc8b500e4),LL(0x913a3edd,0x9f49c572),LL(0xd42b2d4d,0xe6181f93),LL(0x6aa77fa3,0xf96b5db2),LL(0xe43558d8,0xdfb2241f), + LL(0x7481cd31,0x8f8a1964),LL(0x17b37aa7,0xd5b8197e),LL(0x7ac0dbf6,0x7cfbcd19),LL(0x93662f46,0x4ecd8954),LL(0x0501365f,0x104a090d),LL(0x7f097083,0x828694cd), LL(0x6dc105dd,0x60b865f3),LL(0x85cd4ed7,0x00549f1b),LL(0xcc197cc9,0xd262e38b),LL(0x9a262b4f,0x5d3271de),LL(0xa953d539,0xc7df47e9),LL(0x5b9a86a6,0xab8f1c8a), + LL(0xedee15a5,0xc7d0abb2),LL(0x228cc4a1,0x72dc0105),LL(0xa80767de,0xeb67defc),LL(0x71820908,0x6fa174d8),LL(0x5674d19a,0x3215df48),LL(0x960a081a,0xf944531a), LL(0xef2cce62,0x93ed7180),LL(0xc8bcfc0d,0xb318edbf),LL(0xfe787e58,0x0909d56e),LL(0x8fe8b96f,0x5ae74fc9),LL(0x35ab6811,0x8fc342c4),LL(0x0b991e0c,0x6fc6cc5c), +}, +/* digit=75 base_pwr=2^375 */ +{ + LL(0x542f4e90,0x55671129),LL(0x0623d4cd,0x43bedccf),LL(0xe99ca16b,0x7e21207c),LL(0x7c7a26b9,0x785fa105),LL(0xc2c3ab00,0x33c28658),LL(0xd79cd59f,0xcce42a48), LL(0xb8c3bc75,0x9a674db4),LL(0x6904e3fe,0xea701d15),LL(0x66bf2c6c,0x990e7221),LL(0xbd4c3791,0xba29affa),LL(0x20696ee1,0xd98510cf),LL(0xf93d26a5,0x722ed471), + LL(0xbc579793,0xe7cf5bac),LL(0xd73f881c,0x11db7ddf),LL(0x04fa8473,0x9c1a531d),LL(0x5780efda,0x399e8484),LL(0x4f62cb5a,0x6e9c12be),LL(0x94a5df3b,0xf21bdc49), LL(0x11da2a4f,0x3c15fe12),LL(0x23e631d1,0xdea123bb),LL(0xbe294c90,0x3ef76da4),LL(0xa99b8398,0x5cf21d5a),LL(0x751b9f6a,0x50679cf8),LL(0x54d0b7bf,0x4b3f3b9c), + LL(0x75a46271,0x6c8d97e0),LL(0x9dbed39f,0x0fa0c4cd),LL(0xde74ac6c,0xfb6da5e2),LL(0xc17c1ec5,0x041ce886),LL(0xd7419105,0xb42941a8),LL(0x002fdfd5,0x79768eee), LL(0x88c8111f,0x64849afd),LL(0x814192d6,0xf425fe14),LL(0x0448fd7e,0xe916e864),LL(0x72ed351f,0x31e224ad),LL(0x7c0183c1,0x73e6e6ac),LL(0x21bf7ceb,0x375657c6), + LL(0x03bdf6d6,0xc84f9172),LL(0x69f60e03,0xcfc47187),LL(0xa05068ea,0xcdc4753b),LL(0x077777ef,0xa177ad14),LL(0x7e4cf44a,0x0b7f54eb),LL(0x1860144e,0x4ee443f9), LL(0x42bb6a93,0x1279ed4d),LL(0x436c1b54,0x511137d7),LL(0xb8cdb6ce,0xebc958fa),LL(0xa0c7614a,0xbc4f93f4),LL(0x7b2c6d8e,0xc5bd6cde),LL(0x8d65f38a,0xecff7dd7), + LL(0xe45543d5,0x9a4af7af),LL(0x6a04c87b,0xb7478fe0),LL(0x66f72454,0x974eebba),LL(0x5901d1ec,0x682578fa),LL(0xe82b048b,0xc3595199),LL(0xbbc19ba2,0x83da52fb), LL(0x90450b02,0x40f337af),LL(0x439c46d7,0xbd1ea60b),LL(0x00d0ed85,0x4f3e4818),LL(0x766d9e20,0x59d0a0a9),LL(0x56a16718,0xe81dc4d6),LL(0xdf3d3d98,0xac872a21), + LL(0xbbaae09b,0x4da7d63a),LL(0x0783fab2,0xcae05f37),LL(0x68841d1e,0x1e8c0016),LL(0xb10366f6,0x0688f485),LL(0x05b121e9,0x38ee34b0),LL(0x14e0dc1e,0x2779f009), LL(0xdbff60ce,0x83c1d44e),LL(0x4105c8c2,0x63fbcf82),LL(0x53715349,0x6b732744),LL(0xc5ca18f6,0x5065bdcd),LL(0x2def86e3,0x677313cf),LL(0x33ebff5d,0x6c54d224), + LL(0x88b3fc7b,0x35361c91),LL(0xc22a219e,0x706cf8c5),LL(0x886348e9,0x545bb34b),LL(0x4594a530,0xf25eef25),LL(0x73843cbb,0x59427ed0),LL(0xbeea8c4d,0x39638f20), LL(0x960e3c28,0x4e7d445f),LL(0xd383df18,0x989abc64),LL(0x04694ba7,0xb4238e02),LL(0x0203d67f,0xa3753137),LL(0x86d01fc1,0x404bc421),LL(0xb162392d,0x1af37190), + LL(0xf65de0f5,0xffec6674),LL(0xd23ad193,0x4043079c),LL(0xee61bc95,0x31811365),LL(0x8948b6e2,0x358bbd6e),LL(0xe31644be,0x1cd9c342),LL(0x60a8a7a7,0xbab3aa8c), LL(0xa375beb6,0xe065519f),LL(0x4439990c,0xf7d0b041),LL(0x8517ae8a,0x8957c03b),LL(0x73750d6e,0xc96a0401),LL(0xb2aee6d7,0x4eb2e364),LL(0xed099114,0x813054fe), + LL(0xec3c19fe,0x79d98e89),LL(0x429341e7,0x860ec5a6),LL(0x25dd60c0,0x80ab8568),LL(0xe47973db,0x7d0b3f3f),LL(0xf7899dfb,0x654435bd),LL(0xe6542b1d,0x54c59689), LL(0xbed69ab6,0x171c6842),LL(0x188a3126,0x82b8024d),LL(0x9c37538d,0x31bf057d),LL(0xcb0be742,0xe079326a),LL(0x29ace4e3,0x0232e981),LL(0xce8596ab,0x7b06fb86), + LL(0xcaada268,0x71714896),LL(0xfd0e302c,0xbb3d05dc),LL(0xfe56d08f,0xb0785f33),LL(0x38a1b2ef,0xdd43e0f6),LL(0x360fc15a,0x2df35cfb),LL(0x90b3ed36,0x97173f0f), LL(0xb720544e,0xd4970bdd),LL(0x94a01944,0xb6075f76),LL(0x4a43c4f3,0xc99e8a3e),LL(0xd9cb4808,0x8013609f),LL(0xecc3d094,0xf3fef0ea),LL(0x8642d223,0x3829fac7), + LL(0x0402156b,0xb7613dab),LL(0x0b20ec7a,0x4ad70f1c),LL(0x9c46dd4d,0xd9189b20),LL(0x4bd5235a,0x4b22485f),LL(0x88822a0d,0x6e972031),LL(0x2c136807,0x3cf8d823), LL(0x0884e550,0x5997fa64),LL(0x293aedb8,0x73110b25),LL(0x35319a22,0x7e820168),LL(0xa6c668ba,0x2222c809),LL(0x31e0bfb8,0x2f316be2),LL(0x0d832198,0x86cf3a2d), + LL(0x684456b1,0xffe3104b),LL(0xf0f49278,0x37ba0db6),LL(0xcca2f150,0x15aaed42),LL(0xc421c694,0x8618aa02),LL(0xfab87b36,0x4d6a091d),LL(0x0e786d5f,0x304eaea0), LL(0x2c114074,0x0be97747),LL(0xad387a8d,0xf57e3a19),LL(0x7a70d421,0x6094823b),LL(0x09de860f,0x59287918),LL(0x6dcf6020,0x7f7fca49),LL(0x5f46086e,0x57580c61), + LL(0x5d89002f,0xeab34616),LL(0x72a01a68,0xb3115628),LL(0xdce0c191,0x2f0ced58),LL(0x1a895760,0xec08b09a),LL(0xae62153c,0x206faa7f),LL(0xcf2895bf,0xc31e3815), LL(0x9ac88636,0xd57fbf57),LL(0xce91affd,0x966f5a84),LL(0x63620a73,0x092458b9),LL(0x50805fc1,0xda7b4910),LL(0x5c561649,0x1fc60a25),LL(0x4f899e20,0x8110a1a9), + LL(0xf0cb1370,0xb509f702),LL(0xc658441f,0xbdfcf4a0),LL(0x7f07f328,0x853d832a),LL(0x8fbdcb83,0x074fdecd),LL(0xd6a4650d,0x80ed8de9),LL(0xa5d68720,0x61c39ce8), LL(0x3177feb4,0xe66666d2),LL(0xafacf38b,0xdbf3fc57),LL(0x0da620fb,0x7e2d9951),LL(0xdf866f77,0x901145ff),LL(0xdb045beb,0x442a37e5),LL(0xbeb1b008,0x0cb0600f), + LL(0x54d7a6af,0x1e6604d3),LL(0x07c97f80,0xba6ae4d0),LL(0x77c527fb,0x5e3d978b),LL(0xe93a0d78,0xd0642c72),LL(0xb3c3c215,0x06d8ae5c),LL(0x4eb9a4c9,0x8bf36e5a), LL(0x4d505a53,0x1ca6403c),LL(0xd2f5c7a3,0x0187be5a),LL(0x9f850eac,0x68cce2ba),LL(0x5805353b,0x81055e4a),LL(0x89b4eb85,0x3c242c1d),LL(0xdda42eb8,0xee4a6691), + LL(0x7c34f095,0xbb39a17a),LL(0x22fbbe61,0x7be330a8),LL(0xb91f1482,0x6be6abe3),LL(0xbd39a2bc,0xf972804f),LL(0xf91d813e,0x06737e54),LL(0x1a87cd4a,0xbd606668), LL(0xf538d56e,0xbf88b2e5),LL(0x34afd68f,0xb8206a81),LL(0xa58af042,0x7a93aedf),LL(0xac0511b0,0x8853cdf6),LL(0x067e2c19,0x9d7f416d),LL(0xf9671d8a,0x5d0bc923), +}, +/* digit=76 base_pwr=2^380 */ +{ + LL(0x204be028,0x2e7d0a16),LL(0xd0e41851,0x4f1d082e),LL(0x3eb317f9,0x15f1ddc6),LL(0x5adf71d7,0xf0275071),LL(0xee858bc3,0x2ce33c2e),LL(0xda73b71a,0xa24c76d1), LL(0x6c70c483,0x9ef6a70a),LL(0x05cf9612,0xefcf1705),LL(0x7502de64,0x9f5bf5a6),LL(0xa4701973,0xd11122a1),LL(0xa2ea7b24,0x82cfaac2),LL(0x0a4582e1,0x6cad67cc), + LL(0x51e4de5e,0x96a1e74f),LL(0xe37f5006,0x72913696),LL(0xbe35109c,0x12449c4f),LL(0x4521d7e6,0x1fad8b30),LL(0x57d00293,0xc85eb23d),LL(0x35f68229,0x4ebd334b), LL(0x2df5acf1,0x7c5b8668),LL(0x5463de2e,0xc2b4da6e),LL(0x757cd570,0x067b0456),LL(0x3a1c866b,0xeaab81be),LL(0xbbba88c0,0x72a6af75),LL(0x0ef567dc,0xaed4dbde), + LL(0xb4dc8600,0x597a26ff),LL(0xf9288555,0x264a09f3),LL(0x5c27f5f6,0x0b06aff6),LL(0xd8d544e6,0xce5ab665),LL(0x99275c32,0x92f031be),LL(0xf42e0e7c,0xaf51c5bb), LL(0x1e37b36d,0x5bb28b06),LL(0x8473543a,0x583fba6a),LL(0xf93fb7dc,0xe73fd299),LL(0x6e2ccad9,0xfcd999a8),LL(0x334d4f57,0xb8c8a6df),LL(0x9a2acc9b,0x5adb28dd), + LL(0x7d221ab6,0x3afdee27),LL(0x47bb619e,0xecf10abc),LL(0xba4a3301,0x340c8ee3),LL(0x2a883b7f,0x1a6ea51a),LL(0xd5d7412b,0x64f27976),LL(0x91251b6e,0x7fcf0ecc), LL(0x365b18b7,0x5f3f8f41),LL(0xe2e13e58,0x38e48b96),LL(0xad61b2cb,0xde3b73d6),LL(0xd542676d,0xf08398d5),LL(0x8e7d712b,0xd373931e),LL(0x7f96e023,0x89325d7a), + LL(0x111792b9,0x5adf3d9a),LL(0x4f1e0d09,0x1c77a305),LL(0xa82d3736,0xf9fbce33),LL(0x718c8aa3,0xf307823e),LL(0x416ccf69,0x860578cf),LL(0x1ef8465b,0xb942add8), LL(0xcd9472e1,0x9ee0cf97),LL(0xb01528a8,0xe6792eef),LL(0xc09da90b,0xf99b9a8d),LL(0xcbf3ccb8,0x1f521c2d),LL(0x91a62632,0x6bf66948),LL(0x854fe9da,0xcc7a9ceb), + LL(0x4b3759cf,0xe44e3f86),LL(0x9d74e3f6,0x90cab0eb),LL(0x01c4e171,0x10042545),LL(0xce52defb,0xc12df68c),LL(0xf363100a,0xb1fae2fb),LL(0x3573235f,0x5016c853), LL(0x1d922e9b,0x8d4deb66),LL(0x17f84ef2,0x8a20d423),LL(0x5a4e118e,0x32498583),LL(0x308772e9,0x5abfa961),LL(0xf54e4876,0x41c7611f),LL(0x1f5867b2,0xc1da40d3), + LL(0x491ccb92,0x46303171),LL(0x2771235b,0xa80a8c0d),LL(0xf172c7cf,0xd8e497ff),LL(0x35b193cf,0x7f7009d7),LL(0xf19df4bc,0x6b9fd3f7),LL(0xb46f1e37,0xada548c3), LL(0xc7a20270,0x87c6eaa9),LL(0xae78ef99,0xef2245d6),LL(0x539eab95,0x2a121042),LL(0x79b8f5cc,0x29a6d5d7),LL(0xb77840dc,0x33803a10),LL(0x11a6a30f,0xfedd3a70), + LL(0x85adde98,0x3c90a59f),LL(0xe5269140,0x35414174),LL(0x1a0d58e2,0x9aca885c),LL(0x6816b009,0x77b9b6dd),LL(0x9ee4718f,0x8e5c1213),LL(0x4e4eac45,0x60ad991e), LL(0x4d71f624,0xc00c3569),LL(0x5bc5fd2a,0xacbf4eb2),LL(0x5eaf3eaa,0xcba1ffc7),LL(0x42a87e32,0x5f99092d),LL(0x6f7a882f,0x2e7b49c7),LL(0x29040512,0x5e9bfc5c), + LL(0x142403d1,0xfa070e22),LL(0x15c6f7f5,0x68ff3160),LL(0x223a0ce8,0xe09f04e6),LL(0x53e14183,0x22bbd018),LL(0xcf45b75b,0x35d9fafc),LL(0x7eceec88,0x3a34819d), LL(0xd33262d2,0xd9cf7568),LL(0x841d1505,0x431036d5),LL(0x9eb2a79a,0x0c800565),LL(0x5f7edc6a,0x8e77d9f0),LL(0x65e800aa,0x19e12d05),LL(0xb7784e7c,0x335c8d36), + LL(0x30a3ada1,0x75cba9d5),LL(0xf8ae9565,0xb69e308b),LL(0xca7b8369,0x990e3425),LL(0xe0a7ad0b,0x9f67567f),LL(0x18bd01b7,0x76ed6fe7),LL(0x2ff95cfe,0x282358aa), LL(0x410f8841,0x28d2ea41),LL(0xccd67c81,0x89d1533f),LL(0xb6a7b8f9,0x969bb272),LL(0x26330782,0x54f8664c),LL(0x1dcd9164,0xb89f3ae8),LL(0x3d962c14,0x54d845b9), + LL(0x6484fd40,0x8b2fc4e9),LL(0xa35d24ea,0xee702764),LL(0xb871c3f3,0x15b28ac7),LL(0xe097047f,0x805b4048),LL(0x647cad2f,0xd6f1b8df),LL(0xdc7dd67f,0xf1d5b458), LL(0x25148803,0x324c529c),LL(0x21274faf,0xf6185ebe),LL(0x95148b55,0xaf14751e),LL(0x28f284f4,0x283ed89d),LL(0x4cbebf1a,0x93ad20e7),LL(0x882935e1,0x5f6ec65d), + LL(0x6c0f3509,0xb3984b17),LL(0xd8b4d6bc,0xf9fa4483),LL(0x7dec20d2,0xf4ac2b67),LL(0xb3dbe034,0x67ef024e),LL(0x0f94f4d7,0x2dcc5118),LL(0x74a51393,0x024cdcfd), LL(0x20e7abcb,0xf1c0fead),LL(0xd3a7414f,0xffc18f81),LL(0x7062cb0b,0xb00ce556),LL(0x817bc8d1,0xeccb0521),LL(0x40411c15,0xa0c0fe60),LL(0x1defbe00,0x05311322), + LL(0xa4dcefe9,0xe222eba4),LL(0xec1ceb74,0x63ad235f),LL(0xe05b18e7,0x2e0bf749),LL(0xb48bdd87,0x547bd050),LL(0xf5aa2fc4,0x0490c970),LL(0x2b431390,0xced5e4cf), LL(0x51d2898e,0x07d82704),LL(0x083b57d4,0x44b72442),LL(0x5037fce8,0xa4ada230),LL(0x50510da6,0x55f7905e),LL(0x8d890a98,0xd8ee724f),LL(0x11b85640,0x925a8e7c), + LL(0x56467257,0xda828fe5),LL(0xd640c2a1,0x5e9abf67),LL(0xc25c696a,0x0eed233c),LL(0xb3e1d84f,0x72483dc5),LL(0x4f114abc,0x30bf1ee3),LL(0xd1f9bce8,0xf58b321e), LL(0x97524f33,0xcb26564c),LL(0x1e453229,0xdc2f105e),LL(0x72a982dd,0x9da43ceb),LL(0xfeef8862,0xecf5649d),LL(0x1fa2f06d,0xd8afda34),LL(0x55035432,0xf0d0ced3), + LL(0x1ca459ed,0x5bfa10cd),LL(0x6dcf56bf,0x593f085a),LL(0xc0579c3e,0xe6f0ad9b),LL(0x2527c1ad,0xc11c95a2),LL(0xcf1cb8b3,0x7cfa71e1),LL(0x1d6dc79d,0xedcff833), LL(0x432521c9,0x581c4bbe),LL(0x144e11a0,0xbf620096),LL(0xbe3a107b,0x54c38b71),LL(0xe2606ec0,0xed555e37),LL(0xd721d034,0x3fb148b8),LL(0x0091bc90,0x79d53dad), + LL(0x08d1be5d,0xcf17f9dc),LL(0xafdfeb23,0xb55de4c8),LL(0xe437b29c,0xa69454ff),LL(0xe27ee9e2,0x6628d789),LL(0xee3af03b,0x56e3b975),LL(0x2f532d62,0x0083fe9c), LL(0xe63e7511,0xcae15213),LL(0x86ed849c,0xdb5384f3),LL(0xfa4d825f,0x902ba959),LL(0x5ae17566,0xbad700d5),LL(0x14c82eb4,0x16b2c5dc),LL(0x36708ea7,0xa4b057a7), +} +}; +#endif /* _DISABLE_ECP_384R1_HARDCODED_BP_TBL_ */ +#endif /* _IPP_DATA */ + + +IPP_OWN_DEFN (const cpPrecompAP*, gfpec_precom_nistP384r1_fun, (void)) +{ + static cpPrecompAP t = { + /* w */ 5, + /* select function */ p384r1_select_ap_w5, + /* precomputed data */ (BNU_CHUNK_T*)ec_p384r1_precomputed + }; + return &t; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpp521r1precomca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpp521r1precomca.c new file mode 100644 index 0000000..8bb4b55 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpp521r1precomca.c @@ -0,0 +1,4057 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (P521r1 precomputed) +// +// +*/ +#include "owncp.h" +#include "pcpgfpecstuff.h" + + +#define OPERAND_BITSIZE (521) +#define LEN_P521 (BITS_BNU_CHUNK(OPERAND_BITSIZE)) + +/* P521 affine point */ +typedef struct{ + BNU_CHUNK_T X[LEN_P521]; + BNU_CHUNK_T Y[LEN_P521]; +} P521_POINT_AFFINE; + +extern const __ALIGN64 P521_POINT_AFFINE ec_p521r1_precomputed[105][16]; + + +#if defined ( _IPP_DATA ) + +#if !defined(_DISABLE_ECP_521R1_HARDCODED_BP_TBL_) +/* see ippcp_baseptbl.cpp test for generation details */ + +#if ((_IPP_ARCH == _IPP_ARCH_EM64T) || (_IPP_ARCH == _IPP_ARCH_LP64) || (_IPP_ARCH == _IPP_ARCH_LRB) || (_IPP_ARCH == _IPP_ARCH_LRB2)) +const __ALIGN64 P521_POINT_AFFINE ec_p521r1_precomputed[105][16] = { +/* digit=0 base_pwr=2^0 */ +{ + LL(0x81adc101,0xb331a163),LL(0x18e172de,0x4dfcbf3f),LL(0xe0c2b521,0x6f19a459),LL(0x93d17fd4,0x947f0ee0),LL(0x3bf7f3ac,0xdd50a5af),LL(0xb035a69e,0x90fc1457),LL(0x9c829fda,0x214e3240),LL(0xb311cada,0xe6cf1f65),L_(0x00000074), LL(0x5a9e268e,0x28460e4a),LL(0x3b4fe8b3,0x20445f4a),LL(0x43513961,0xb09a9e38),LL(0x809fd683,0x2062a85c),LL(0x4caf7a13,0x164bf739),LL(0x8b939f33,0x340bd7de),LL(0x24abcda2,0xeccc7aa2),LL(0xda163e8d,0x022e452f),L_(0x000001e0), + LL(0x640909df,0x1e90cf08),LL(0x99dd36bc,0xb3fa1f1c),LL(0xb26b07ec,0xa0e797d1),LL(0x1d1ae2d7,0x83d50825),LL(0x6d377aaa,0x4bd9d902),LL(0x82ebb4df,0x1a96372a),LL(0xcd8e6603,0x3a3a0193),LL(0x40a46141,0x3417e594),L_(0x0000013f), LL(0x31fe1b6c,0x813d2ee3),LL(0x6b30fa0d,0x7b8df1ab),LL(0x4af6e07a,0x7a757e5f),LL(0xd4cd1924,0xb5c9c9bf),LL(0xef4f928f,0xddd9f1bb),LL(0x4c836216,0xa05590d1),LL(0x3e26d4bb,0x5ae35a88),LL(0x777769f8,0x8053f9f6),L_(0x00000133), + LL(0x4910f78a,0xbee9cf4d),LL(0x976f1bd6,0x02d2c8ce),LL(0x43161975,0x0dd75a48),LL(0x8b5acff1,0x028ed35e),LL(0x251d2419,0xe8d69f8b),LL(0x0896bd46,0x5cf2d6bd),LL(0x2d891ecd,0x3cda9537),LL(0x325acaca,0xaeec8eb5),L_(0x0000008c), LL(0xee5f7e98,0x72cfa6c0),LL(0x50f74360,0x212fac46),LL(0xde49d2c8,0x867882e4),LL(0x68ef61e3,0xd816ad67),LL(0x67c6e2ba,0x761716ea),LL(0x8fd1aae7,0x8be97c55),LL(0xd4154e81,0x7978aabf),LL(0x63655c0a,0xeccbcfc3),L_(0x0000016e), + LL(0xd92b8ab0,0x798d6d77),LL(0x09438c81,0xf17de43a),LL(0x2d8472d2,0x3350ea81),LL(0x4f83c578,0xa8745c47),LL(0x257f1e83,0x56432cf1),LL(0xaaa0e9e7,0x7e0362ea),LL(0x8e2ff9cd,0x66e30e44),LL(0xa43e4838,0x61aa5a41),L_(0x00000102), LL(0xe109849e,0x86a0825b),LL(0xfe1a3726,0xecf10fa3),LL(0x78234ce8,0xf75dbfd7),LL(0xa854adc2,0xa029127b),LL(0xf2a5d1c2,0xf93cf941),LL(0xf178cc83,0x0dad731f),LL(0x7b737197,0xdb2a90d7),LL(0xc7585a55,0x5b39f00b),L_(0x000000e9), + LL(0xf14a49e9,0x3c194afc),LL(0x4b764798,0x9c6ad5a8),LL(0xf36c498b,0xd194ebf0),LL(0x5789bf3c,0x11b8897f),LL(0x36af180a,0x721c1e06),LL(0x5c78bbd6,0x926781ed),LL(0x7eda9f86,0x5fbd2cb7),LL(0xc8e02758,0x639ede19),L_(0x00000019), LL(0xc6f75980,0x65d6f9bb),LL(0xf46f5848,0xfc0b9e61),LL(0x92b9aa7b,0xbce8f803),LL(0x108e7aff,0xba188aa0),LL(0xe4839679,0x43ddb44b),LL(0xe4d01a38,0x28f6ec0b),LL(0x47439700,0x488e6c7f),LL(0x88a54089,0x764515b9),L_(0x000000eb), + LL(0x75b36d64,0x5cfb915a),LL(0xf6fbc903,0x5711b98d),LL(0xab2bf9c0,0x4617b374),LL(0x11ca98df,0xca70393d),LL(0x0b0a9fb9,0xa92fde65),LL(0x56f25580,0x79cc0a83),LL(0x4bbfeb8e,0xcab11e98),LL(0x7ca24068,0xa9977f9a),L_(0x0000010c), LL(0x6b433193,0x8bf78095),LL(0xbc2c6a27,0x6f0f5666),LL(0x5aae506d,0x101ee3dc),LL(0x26f13a79,0x4efcb64c),LL(0x872b3246,0x4b655b96),LL(0x93100d45,0x47392054),LL(0xb9ed2d40,0x889555dd),LL(0x82a371d8,0x35716e93),L_(0x000001ac), + LL(0x766756df,0x6a15b574),LL(0xc4140b76,0xcd00e756),LL(0xa87ee130,0xe237ca9f),LL(0x986e71dd,0x6c64d36f),LL(0x855fe34c,0x2ec61846),LL(0x617b88a6,0x14780c69),LL(0x747aa419,0x062f9170),LL(0xed05839d,0xa3775b2f),L_(0x000001b1), LL(0xdf66eaa8,0x8d8f4b46),LL(0xe4829292,0x3dae35c5),LL(0x952eef7e,0x2fcf3b38),LL(0xa2c8e70d,0x15ca91d1),LL(0x49e6f64f,0x2ab5e879),LL(0xc51365ef,0x6eb8edec),LL(0x68141278,0x3c5ae2c1),LL(0xbd1ceb42,0x8868ec18),L_(0x00000150), + LL(0x03ed8c07,0x340208b1),LL(0x2a553c67,0x02c37cf5),LL(0xdad37a02,0x0d5ab144),LL(0x0de46bcf,0xf845acc6),LL(0xdc2bcfa4,0xc7adff52),LL(0x82fc1314,0x0545c51d),LL(0xc54d801f,0x2dea714e),LL(0xcb580871,0x31541a41),L_(0x000001b9), LL(0x475550bf,0x0e58cc64),LL(0x788f8bc0,0xa9c56b21),LL(0xa004a389,0x34cf9dd4),LL(0xf832e2bc,0x5ff85d06),LL(0x552c88da,0x78c4f4e0),LL(0x30833bd4,0xada841ef),LL(0xf4f16038,0xcd227c76),LL(0xb73c429d,0x10247ed5),L_(0x00000024), + LL(0x78d67878,0x03d614e2),LL(0x3cb3e5f1,0x330fa2b1),LL(0x5ec2e694,0xc7a7a85d),LL(0x6fb92d18,0x1af9e2ab),LL(0x9cb09a6d,0x32ba14f0),LL(0xa2dc635b,0x4c962558),LL(0x0dcc43a3,0x44769a2a),LL(0x8ab8ab6a,0x13517adf),L_(0x0000010c), LL(0x6326a2bb,0x270a8b98),LL(0x435cd695,0x9a1d5075),LL(0x74944407,0x3eb9b615),LL(0x67a55fec,0x4207fab7),LL(0xbab02bd6,0x3706b4f2),LL(0x131eeda2,0xdb6412dd),LL(0x2a770e75,0xc7184453),LL(0xbd13d749,0xcf85aaaa),L_(0x000000f9), + LL(0xc8af6987,0x90643ae3),LL(0xd0279799,0xcd43ff84),LL(0x8f8d4740,0xf9be1720),LL(0xd4c21049,0x94ced526),LL(0xbc7e6131,0x885163e0),LL(0x588d6d44,0xe1a54923),LL(0xcd604d63,0x66c80ec3),LL(0xdb390f62,0x0efe7f3d),L_(0x00000194), LL(0xae124585,0xacfad765),LL(0x4c09af63,0x0b94aa5e),LL(0x1cb9cfd5,0xf7f9b44b),LL(0xb1ee2bf5,0x5da9b7b0),LL(0xea26cc32,0x695fa9a4),LL(0xc53177b1,0x05d4bfeb),LL(0x0a2128d9,0xb0617759),LL(0xd86515d3,0x4edd7559),L_(0x0000005d), + LL(0x04967f7b,0xcd229d61),LL(0x16ed066d,0x81766070),LL(0x27d264d4,0x4280ae01),LL(0x75f18c88,0x0de8cd8d),LL(0x999331ed,0x2979ede2),LL(0x2a794c8b,0x4aa1f796),LL(0xf6be0bc2,0xe7f6aee3),LL(0xab9da18a,0xaa378d1c),L_(0x000000ff), LL(0x0ff2ee88,0x0425becc),LL(0xc9672464,0xaeac43a7),LL(0x71fa40cd,0x9b6e5640),LL(0x8c8a54a9,0x559c4919),LL(0x8745a152,0x158de454),LL(0xea705cdc,0x49f6974a),LL(0x31085e82,0x149d6eab),LL(0x3b82a7d9,0xc24e8654),L_(0x00000094), + LL(0x3b24fe08,0xc6703677),LL(0xb8dfc216,0x58bdf34a),LL(0xae298494,0x009bba63),LL(0x4d30749c,0x98a3bff7),LL(0xbe1fdc0d,0xd3227485),LL(0xd2cb3b89,0xdb083e7b),LL(0xabcac22b,0x0f40c3a0),LL(0xba4e3fef,0x2ef27d74),L_(0x0000010e), LL(0x651a9249,0xf850122d),LL(0xd1940d8b,0x363cf3d8),LL(0x91127ad7,0x184425d3),LL(0x7ca8dcdb,0xdb660853),LL(0x21ec37e6,0x4beb68c4),LL(0x5fb50be0,0xd22f2025),LL(0xd5b8a4a6,0x23b1ff32),LL(0x81d34165,0x7f1e70e8),L_(0x000001e0), + LL(0x3e6130e5,0xd39f8fa6),LL(0x46997de6,0xfc0c43c2),LL(0x80559c77,0x74a5f61d),LL(0xf3cd5c47,0xb51aa852),LL(0x84701e4c,0x3099622c),LL(0x5f57adc3,0x1c2776e9),LL(0x66f0da61,0x0d49fb9b),LL(0x95a49243,0xce6bc32e),L_(0x00000156), LL(0x3adb5e07,0xe4c23b96),LL(0xfd811538,0xfb948d00),LL(0x57c88bf4,0xe1b0ccf5),LL(0x9a8e5fdd,0x1f936fee),LL(0xac1c5e3b,0x9560deae),LL(0xd72e0f10,0xe34e3d33),LL(0x1c36aa10,0x04676a85),LL(0x6d51f6ad,0xd48d0c93),L_(0x000001f6), + LL(0xf71546c6,0xba61d6f1),LL(0x16e9a196,0xae964f34),LL(0x5c0977df,0x5533b3fb),LL(0x25bde3a2,0x16bcef9f),LL(0x645c4b91,0xccba7e03),LL(0xd5f0598c,0x17ea7d85),LL(0x0a4a08b5,0x68cac5a0),LL(0x7d57c26f,0xd4f0dc66),L_(0x000001b2), LL(0x8d5d76b2,0xa5172924),LL(0xc293806a,0x5f20b34f),LL(0xa9d43e42,0x06adb487),LL(0xf8e899ee,0x2608f44d),LL(0xd8da79ac,0xb1683bc0),LL(0x4dc36bf6,0x350423e7),LL(0x15c728c1,0xfdc23809),LL(0x5dd5da5e,0xe96b3148),L_(0x000001ad), + LL(0x26af2e49,0xaa9adab6),LL(0x9bde5c6d,0x5ef4d7f1),LL(0x4c0f1fc9,0x8ecdc6cb),LL(0x8e47e019,0xff3c3ade),LL(0x13ede807,0x08dc8e67),LL(0x996f8947,0x296b4bda),LL(0x185a0504,0x07dc7de6),LL(0xe2a36a18,0xf820aac7),L_(0x00000032), LL(0x89c55c4e,0x32ed1a36),LL(0x4050a3aa,0xecab1a1a),LL(0x6622355e,0xc9237ec8),LL(0x11964b64,0x4010a471),LL(0x644ca385,0x6abf4831),LL(0x34cba42f,0x5d25b108),LL(0x54dd6906,0xb1ef824b),LL(0x9199f6df,0xb53e7326),L_(0x0000008e), + LL(0x362a2722,0x17f45f44),LL(0x47c0d420,0x299d3628),LL(0x4003ee6e,0x6f86dfae),LL(0xd3cc2b18,0x5072cb2e),LL(0xc430e500,0x294a1ff8),LL(0x69058c45,0x9eeb197c),LL(0x30c97e9f,0x859543fb),LL(0x13563a1d,0x2eed4bed),L_(0x00000168), LL(0x0f160b5d,0x5f1e8dcd),LL(0x7c4de39d,0x99a97139),LL(0x69468c31,0x294f1802),LL(0xd64eccc4,0xd505983a),LL(0x9cc6daa3,0x60e0c170),LL(0x64a5b5c1,0x8763e518),LL(0x61dc006a,0xb9099af2),LL(0xc69e9f34,0x0fe38a58),L_(0x00000180), +}, +/* digit=1 base_pwr=2^5 */ +{ + LL(0x65b4828e,0x4f59ae8d),LL(0x5c3b72b5,0xc123c8ad),LL(0xae6b62e3,0xca4e7ba3),LL(0xeb5d2e8f,0x7633eb4d),LL(0xaee39acb,0xeb750251),LL(0x77dcf2f5,0x1c6dd3f6),LL(0x32a70340,0x9a1f1f0b),LL(0x7ca5d0ad,0x4a21b83d),L_(0x00000123), LL(0xfd346502,0x5511dec3),LL(0x4c8a50ab,0x1b4ae54d),LL(0xb2d5cb0b,0x6fe6cc64),LL(0xe3f5079b,0x5cdba6f0),LL(0xdc0c66eb,0xbef10266),LL(0xe32e16eb,0xb3e0ef80),LL(0x5faff80c,0x347fbdec),LL(0xaff9f041,0x088b1af3),L_(0x0000015c), + LL(0x5f738444,0xa16ae6a8),LL(0xc671907b,0xa0da9ee1),LL(0x014d027e,0x0efad55c),LL(0x0b01f380,0xb9d3e016),LL(0xe2f7ed1b,0x938df5e9),LL(0xe67c4396,0xd65b0a5c),LL(0xd40c305d,0x533f5edd),LL(0x01c97f61,0x68a79ebe),L_(0x00000176), LL(0x3ae123a4,0xad9d235c),LL(0x2c52fc6f,0xea2d78b8),LL(0xedd0329e,0x21e0e0d2),LL(0xf1d7cac0,0x887a53dc),LL(0xd0a846a5,0xc5d60b2d),LL(0x1b4f43d5,0x0e8631dd),LL(0x4597d8ee,0x36cdd8e6),LL(0xbe2bdcac,0xb9d50810),L_(0x00000102), + LL(0x7955b135,0x571ccaa7),LL(0xcc45066e,0x3f9b1980),LL(0x05f6f02b,0x6757639e),LL(0xe84f381f,0x4e03775c),LL(0x5ef348d1,0x4770c353),LL(0x3845ff7f,0x5fb50c57),LL(0xedbd5036,0x5b16a317),LL(0xe0ae9613,0xbda8a1ca),L_(0x0000011c), LL(0xac6debc0,0x2efb5af8),LL(0xce499e42,0xdea5dcca),LL(0xf85f9a34,0x8fc76f8d),LL(0x07899ffe,0x0f62f621),LL(0xbdb94b70,0x648a20af),LL(0xfe99ecf5,0x436c353f),LL(0xd5421253,0x3fcac929),LL(0xe3d53ffd,0xa9413337),L_(0x00000092), + LL(0x967c9f8e,0xf05f3504),LL(0x72b9b7d4,0xdabb5813),LL(0x90711a09,0xa79713f3),LL(0x40b52fc0,0xbea8efb8),LL(0xb43b138e,0xcba724a4),LL(0x7bae703e,0x96698925),LL(0x5de16b6e,0xd41e4b4c),LL(0x80a9811f,0xc1b18e03),L_(0x0000000f), LL(0x7d41aa0a,0x989bdb97),LL(0x0964efd8,0x725e184a),LL(0x049a3954,0x09871a1a),LL(0x92325673,0x462734e1),LL(0x586e4cd6,0x54d24ffa),LL(0xe1d8d7ce,0xb30da7d5),LL(0x69f3efbc,0x416e700f),LL(0xef7de3d3,0xe729987f),L_(0x0000001e), + LL(0x4f9e94f1,0x08a54bd7),LL(0x32ffe09a,0x456c0723),LL(0xd6fc852f,0x40d62750),LL(0x2ffa7f72,0x4aeb61e4),LL(0xe7b77ac3,0x502b124f),LL(0xfe47dfb6,0x4dd6a90a),LL(0x06cd1ac4,0xa6600862),LL(0x47e8bdf2,0x9f7f4801),L_(0x000001f1), LL(0x059ad258,0x0fde861b),LL(0xc7487c32,0x94ff60ca),LL(0x23970cbd,0x72b3e644),LL(0x6984aedb,0xcc2f8476),LL(0x43e3b1aa,0x4e288fca),LL(0xd6b84507,0x5c070a30),LL(0xcde70c2f,0x806889c8),LL(0x9397e29b,0x4c71559f),L_(0x000001e0), + LL(0xb013de1e,0x85707b44),LL(0xfaa21460,0xdf0df8fc),LL(0x6496b635,0xa66cdf1c),LL(0x72a3871c,0x9e220e51),LL(0x55171f57,0x76519fbf),LL(0x2bc7ff1e,0x1fe67c09),LL(0x4f8bd386,0x55ed0240),LL(0xc5765e29,0x1c77281d),L_(0x000001f5), LL(0x85021f4e,0x9e78f5c3),LL(0x38d57586,0x6a14b857),LL(0xa24ce77d,0x956a40cd),LL(0x6eeb21f0,0x384b0097),LL(0x30d4fd92,0x3f99bf29),LL(0xce9aade0,0x0b162be5),LL(0xc168443c,0x056730f0),LL(0x8b3af3cd,0x86e7a481),L_(0x0000012a), + LL(0x5c84256a,0xd9b7e5ae),LL(0xd2292194,0xc11a98a6),LL(0x0b648125,0x59e37b44),LL(0xaf635b08,0x25aea6af),LL(0x19039f0b,0xd7528475),LL(0xd304853b,0x17b80f08),LL(0x08f86bd4,0x16cad388),LL(0x4ba43b52,0xd0f1e285),L_(0x0000005e), LL(0x4e262ae5,0x7719f6ae),LL(0x0e419d0c,0x808c65ad),LL(0x5ed42353,0x2e40948b),LL(0xc831c79b,0x95dfdfbe),LL(0x1f8615c2,0x19810fc6),LL(0xd5083188,0xc73c4dd3),LL(0x9df9fc10,0xb9ee4c0f),LL(0xb094cd65,0x75870f78),L_(0x000000a4), + LL(0x83c0ad6b,0x56757d02),LL(0xc834df00,0xc0d9c745),LL(0xe5caf285,0x91f23599),LL(0x620faea3,0x2d4e48a9),LL(0xb7461523,0x99bdc7a7),LL(0xf47934e5,0xd4dc2fd4),LL(0x4f65ada3,0x4c81e39c),LL(0x3c079897,0x64a2c57e),L_(0x00000010), LL(0xea9cd04f,0x3fa38d40),LL(0x22a435ef,0xc247d609),LL(0xa826a53b,0x3d8a1866),LL(0xcede94d3,0x75ac695c),LL(0x2b9c71de,0x8bcb2e7b),LL(0xd52b9aa7,0x1a8316e1),LL(0x40f2da2b,0xe2a07695),LL(0x49881db4,0x7e4c0ddd),L_(0x00000021), + LL(0x14f3b7ed,0x7118ea7a),LL(0xe663ce23,0xd6a550c8),LL(0x67612dfd,0x45b8de7d),LL(0xd0a752b6,0x1b9bd789),LL(0x60ad3301,0x023b6c29),LL(0xcc26ecce,0xa078b41a),LL(0x61239a1a,0xee942cd7),LL(0x6922505f,0x5e08263e),L_(0x00000187), LL(0xe0703b39,0x1108fef0),LL(0xe9adb593,0xe4610492),LL(0x509096b8,0x26279733),LL(0x4c917c92,0xc7a80802),LL(0x7516cc5c,0x8edbea9c),LL(0x131d3769,0x1db92a9d),LL(0xe32f86b9,0xc3bfb615),LL(0x16237fcf,0xdaad00e7),L_(0x00000105), + LL(0xf41dbbe2,0x726d1da3),LL(0xd0f5eba7,0x5afe82c9),LL(0x00a2bafd,0xa7f8f99a),LL(0x8c282afe,0x5344cf5a),LL(0xa7ab3e18,0xb4f699ab),LL(0x2626fca8,0x345363ae),LL(0xc44f5f11,0x9cac1c3a),LL(0xa135f6b3,0x2cc9c6d3),L_(0x00000083), LL(0xff61db3e,0x90784ad0),LL(0x4491c85e,0xc87a8f35),LL(0x23793bcb,0x9606baed),LL(0xcd6ee91d,0xaa42a14a),LL(0x54d429b3,0x40a29e37),LL(0x89ff244a,0xd4a2c066),LL(0x0bb505cb,0xdc545060),LL(0xfc93a903,0xad7e26a4),L_(0x0000006e), + LL(0x2c4f1dd9,0x8ceb07a2),LL(0x96bcc3d3,0x99d9281c),LL(0x3db83972,0x3ff2e9a3),LL(0x16268498,0x00d03fc3),LL(0xf0d72767,0x974db3bc),LL(0x52e2c15d,0xcfc51b17),LL(0xe4156324,0x10aa8cfe),LL(0x989f0141,0x8e68c302),L_(0x00000148), LL(0xb8c928ab,0xb1ff4858),LL(0x798b01e9,0xb7bcaeeb),LL(0xb107b933,0x0bdcd04d),LL(0x5499a0b1,0x26fd1d2e),LL(0xacddcbd8,0x56837ddc),LL(0xa9081a22,0x3bdf1491),LL(0x05c3276e,0xc07890c9),LL(0x91891ac9,0xa184d413),L_(0x000001ab), + LL(0xfb3d55c4,0x8bff7233),LL(0x91b8350b,0xf62b4383),LL(0xb265f67b,0xc46f7226),LL(0x21d7036a,0xef90907e),LL(0x8034aa28,0xdabc0434),LL(0xd005b709,0xb12cb388),LL(0x06bb608b,0xe65c7159),LL(0xeb7b8a18,0x11e0f987),L_(0x0000019b), LL(0xc295c43f,0x8d53586a),LL(0x0ab2f2c0,0xe3db9e6a),LL(0x80aa8220,0xb7b44599),LL(0x2bea87eb,0xa54e5ad3),LL(0x6c5ac479,0x93b927af),LL(0x83fb3fac,0x62a4775c),LL(0x9c4bd501,0x657b8d9a),LL(0x88136dc2,0x31811cf2),L_(0x00000073), + LL(0x67baa023,0x4d4e2e15),LL(0xa3ad1cf1,0xf792f378),LL(0x7aef7449,0x4d833ce0),LL(0x5394ba78,0x06fcfedb),LL(0xf33fd365,0x76965949),LL(0x9c4ccb42,0x4e9fbd73),LL(0x61aaa0a9,0x9fa1995c),LL(0x3ba114e8,0x462ee846),L_(0x0000008d), LL(0xed8f7990,0x0442839b),LL(0x46e8546b,0x4cfa345f),LL(0x0d411f16,0xc1e9119e),LL(0xf8d99149,0x0deb6f34),LL(0xb98975e2,0x6508c235),LL(0x6e32684a,0x741c5884),LL(0x99583d46,0xacaecb2f),LL(0xd61998e0,0xdc28ccee),L_(0x000001fc), + LL(0x1b2220c8,0x22a3dc2c),LL(0xbc8dbffe,0xf713e616),LL(0xbe6a57a2,0xbe89cc5f),LL(0x5dfb0ead,0xb5bd5287),LL(0x5dba909b,0xff87fb08),LL(0x124b1f29,0xd39afe41),LL(0x8ad8951f,0x0e13a626),LL(0x2f09f744,0x4826695e),L_(0x00000020), LL(0xe7dc7a13,0x89f11d49),LL(0xd8b689b1,0x42cf8f40),LL(0x8f4bb929,0x1093f58a),LL(0x41b6334a,0x5f1b0229),LL(0xcbfc9d3f,0xfa09f9c8),LL(0x4f838812,0x4ae0b40b),LL(0x114194e2,0x6d9844d6),LL(0x69722fe6,0x15e4c6d7),L_(0x0000004f), + LL(0xa12d6b0f,0x2f86d0f6),LL(0xf27fea27,0xb102e317),LL(0xf76070d1,0xb05afc5b),LL(0x1c9d3a3b,0x5dd0f5d9),LL(0x00e4d9fc,0xee4d6689),LL(0x65f0f1c6,0x2a86ba85),LL(0xde562216,0x3e6bfc0d),LL(0xdbfc35a2,0x9af0f242),L_(0x000000da), LL(0xae71ea00,0x941bae5d),LL(0x2b9df6f5,0x5be1e379),LL(0x818b63c5,0x35a1da29),LL(0x7c374ecf,0x81936096),LL(0x91cdc4c0,0x32597a76),LL(0x72e4e5df,0x3e8a2fa3),LL(0x5b7351e8,0x916e7f8d),LL(0x19372aca,0xabd62e9d),L_(0x0000016a), + LL(0x4a1a9b59,0xc98396a0),LL(0x1d4dea3f,0x2852471e),LL(0xf1b1b604,0x9e270a42),LL(0xbff87527,0xe46c1327),LL(0xfe022231,0xfc05c823),LL(0xe4c1b07e,0xa4581988),LL(0x46e86dbf,0xc3803e03),LL(0xf3ea14d7,0x8c2f4163),L_(0x00000069), LL(0x1c64b3b6,0x474df73f),LL(0x3f77cba0,0x82f0ebae),LL(0x9fac52f4,0xeabe2a5c),LL(0x4d046303,0x5a86c777),LL(0xd8716f60,0x16157561),LL(0x76cfe4cf,0x564b6dae),LL(0xf10528e0,0x9113bb26),LL(0x878d8ad6,0x933ccc8b),L_(0x0000002f), +}, +/* digit=2 base_pwr=2^10 */ +{ + LL(0xff6fab57,0x7c6312ff),LL(0xb394c36d,0xd8c526b5),LL(0xae9f8123,0x6b7fb3e1),LL(0x7287a461,0x2d9f22f9),LL(0xd21b31a9,0x895d4a0f),LL(0xd7cbfded,0x81ff2d23),LL(0x5c105748,0xe830bd0b),LL(0x4fe2bd04,0x9dfeb777),L_(0x000001d6), LL(0xf68f023b,0x83b243fc),LL(0x7e7441cd,0xa23e166b),LL(0x5c91b009,0x85f70865),LL(0x122f85c7,0x22e7768c),LL(0x6db40321,0x2fb75185),LL(0xd6df94b8,0x80b31836),LL(0x98df3edc,0xeea7ce80),LL(0x05298e9a,0x048ecb96),L_(0x000000e6), + LL(0xa74584c1,0x8ec6fc14),LL(0x292021e4,0xa9680402),LL(0x9500ecd0,0xed719b16),LL(0x41202339,0xb81e8a19),LL(0xb85440eb,0xd40e8e4d),LL(0x3f6a53c2,0x84a12a31),LL(0x2796c5c6,0x497c0088),LL(0x91636765,0x751837b7),L_(0x0000000c), LL(0xb89a9335,0xd4740897),LL(0xfeb6c7cf,0x05fd0f39),LL(0x66755043,0x24da0165),LL(0x915708d7,0xcde5846c),LL(0xc7bb1c3f,0x0cbcc847),LL(0x5d5c58a4,0xd0093587),LL(0x531dd999,0x178ab52f),LL(0x88ff3f98,0x4485d318),L_(0x0000007c), + LL(0x4bebd902,0x7f523b68),LL(0x91acdac6,0xe5501216),LL(0x656f99d2,0x9d6ec374),LL(0xe158465f,0xf67a8845),LL(0x15ed0b99,0x0ea75aec),LL(0x01226fd6,0xc000f5ba),LL(0x0a951866,0x2eb378e5),LL(0x185feb1f,0x746f4b9e),L_(0x0000008c), LL(0x8a0bff22,0xae887bf0),LL(0xc9deb828,0x2d928546),LL(0x4d8afcb8,0x7759681c),LL(0x47a77426,0x1f2422bc),LL(0x9941fb7f,0xc9c44935),LL(0x3b4f41a6,0x50ea43ef),LL(0x708dbefd,0x5c9f2544),LL(0xcef3425f,0x8d085b3a),L_(0x0000003d), + LL(0xe2354e89,0x4dbc092f),LL(0xa2f27fd6,0xfff03850),LL(0x2ad51407,0x2ffc14aa),LL(0xc4b80840,0xbe516b67),LL(0x4499107f,0x0f027098),LL(0x715688b4,0x5e2c9af3),LL(0xbddce779,0x26ec8f7d),LL(0xcc8a5dc6,0xcc9e1305),L_(0x0000012a), LL(0x6e30ed0c,0xcd14a595),LL(0xce664e13,0x678ff921),LL(0xb7485d5a,0xed6fe685),LL(0xdd61d65f,0x2b7d0453),LL(0xa066d915,0x81e48dc7),LL(0x0c3395f0,0xc1cb1256),LL(0x6053e587,0x630f2cdd),LL(0xc776afca,0xf0d70553),L_(0x00000014), + LL(0xfa927e34,0x71ac09f5),LL(0xd012b2e5,0x9190907b),LL(0xc03bb972,0xab45bb80),LL(0x8ed0d272,0x3b41e8eb),LL(0xaa3449d8,0xd2d64ef1),LL(0x4e6b21d4,0x9f7e0342),LL(0x9eb72363,0xb6336622),LL(0x69f35a65,0x9114adb9),L_(0x0000017e), LL(0x75b2adb4,0x18b88dd7),LL(0x489c82e7,0x1d050011),LL(0x5e1bdb72,0x80ac7d35),LL(0x3a785f6c,0x6bb1ceb8),LL(0x4d0595c0,0x47ba8e65),LL(0xf29ab5dc,0xfba4c7c5),LL(0x768427d3,0xf250f0c9),LL(0x38fed5ff,0x60390918),L_(0x0000018f), + LL(0x41d4c16a,0xd8129c74),LL(0x33a20918,0x56ec57a8),LL(0x44da27b8,0xfe03052c),LL(0x5c69a6e2,0xb8645b34),LL(0x61e0489c,0xedf7eb89),LL(0x0d9cee51,0xb459ccf4),LL(0x4bbdc11a,0x2e3c7f1a),LL(0x22591a2d,0xab74c4c7),L_(0x000001f8), LL(0x97907b5b,0xffdc8f5b),LL(0x9755e96d,0x00d903b0),LL(0x73fc3336,0xa3ed2567),LL(0xa44f5c0a,0x78da9c2e),LL(0x130585a8,0x5d2a5778),LL(0xf488bddc,0x203a9db6),LL(0x0d642fb8,0x49bb8671),LL(0x86aadd4d,0xc216425a),L_(0x000000a8), + LL(0x0f4d050a,0x106b0907),LL(0x6c59b6a4,0x77bee1fa),LL(0x082792c1,0x39609b3e),LL(0x4e300675,0x9586b280),LL(0x41820c34,0xf4b318a9),LL(0x568da4bf,0x504b9f0d),LL(0x18b54e1d,0x7cd449b1),LL(0xea63bc73,0x35d4426b),L_(0x000001d7), LL(0xa3569b00,0x0b6fffaa),LL(0x5b9ffa5e,0xc584b1b6),LL(0xb1ee386e,0x00bfc921),LL(0x2e48b6f0,0xc1a25580),LL(0x90b9e7af,0x232ccaec),LL(0x60d7386e,0xbcde0a94),LL(0x27832dfe,0x20ca19ad),LL(0xa34dad1d,0x2a628682),L_(0x000001ca), + LL(0x86b1f786,0x61a19c36),LL(0x5540d3da,0xde90b954),LL(0xfed5fc9d,0x08cbe546),LL(0x6579be89,0x931292ec),LL(0x31c8bf2b,0xde0b2215),LL(0x64709233,0xf0e33dcf),LL(0xa91e2913,0x99299206),LL(0x933880d8,0xab37b024),L_(0x00000107), LL(0x081c0df1,0x6eb1d587),LL(0x5f29f3ee,0x6f46862b),LL(0x13755e24,0xe2652ae3),LL(0x952c2e51,0xba6a65e2),LL(0x013b9446,0x3fd1b792),LL(0x5e7bffb4,0x96a14917),LL(0x66af7dd8,0x68a41011),LL(0x553d0d5f,0x4ff29cf9),L_(0x000001a3), + LL(0x1c733b58,0x1d7e25bf),LL(0x707d2643,0xb62058b6),LL(0x3eddf1f7,0xcf147bf5),LL(0x09f87dab,0x11a1e31b),LL(0x9b643ba2,0x4287faad),LL(0x31ecf4ec,0xfdf5220a),LL(0xa4f09336,0x8916b869),LL(0xd2c73095,0xe07b7112),L_(0x000001a5), LL(0x46af9424,0xea00c98e),LL(0x31798ea9,0xee9f1bb9),LL(0xa0db3168,0x33aa5ab3),LL(0x5107a1fa,0xbb110cf5),LL(0xccdd22ec,0xedd17aae),LL(0x8bb0cd07,0x610d689f),LL(0xcf178778,0xcca4e56f),LL(0x95d696e3,0xaef30431),L_(0x00000088), + LL(0xf37c6266,0x7352fa9e),LL(0x4590e4bc,0x951e01ab),LL(0x42b51fb7,0x3643ff6f),LL(0x1a3be50c,0xdad9a3a4),LL(0x5c6479b6,0xb0a91741),LL(0x5f9d3ca9,0x841c9d52),LL(0xbed2f70d,0xdc8331dc),LL(0x3fce8436,0x0a312707),L_(0x000000b2), LL(0xa224f65a,0x4d9d7ef9),LL(0xaec9953a,0x62242fd1),LL(0x04665dd7,0x49b9eb5e),LL(0x7d7f1a35,0x6a03ee74),LL(0xcabc639f,0x22cc5c02),LL(0xf26d2603,0xbb312bf5),LL(0x05ee7955,0x10cf1634),LL(0x00c226f0,0x3baa95d6),L_(0x000000bc), + LL(0x56832121,0x968950c6),LL(0xa826a58f,0xe858945d),LL(0x3a7fc7e7,0xd63d6714),LL(0x63d3c677,0xc319d1ba),LL(0x349e7bde,0xb4155a1d),LL(0x03a4c66e,0x3ddc0044),LL(0x77aa278e,0xccce8941),LL(0xd867d113,0x4e46021e),L_(0x00000105), LL(0x2cac2d26,0x6dd54385),LL(0xd8308ab7,0x2e1458d6),LL(0x0d0a4aaf,0x924e3bd4),LL(0x309fb2fb,0x2f7cd47f),LL(0x5161e4da,0xbc75672b),LL(0x27fa09f8,0x0e420bf8),LL(0x6bf78336,0x83d1b09e),LL(0x3c3d3117,0x89323d7e),L_(0x00000197), + LL(0xe5de2792,0xe8b9e5f2),LL(0x9e4c557e,0xa63316be),LL(0xc510883d,0xfba63955),LL(0x58616eed,0x5eba66cb),LL(0x1f901bb5,0x7d93dd07),LL(0xe4c33f46,0xd7520d11),LL(0x9c2288bd,0x3c9b7282),LL(0xa3f22d4f,0xf979cce9),L_(0x00000016), LL(0x6b3408c8,0x69f91fa6),LL(0x1780ab39,0x9f2b3904),LL(0x1e17f9e9,0x0408a22e),LL(0xf102825a,0xe814b39a),LL(0x4077db13,0x717c70c1),LL(0x116e8d04,0x1642fd91),LL(0x5157bba1,0x072760c2),LL(0x223d53fd,0xf596860d),L_(0x00000130), + LL(0x0b6e1126,0xd03914a2),LL(0x1f8fa1cb,0xbc0f726e),LL(0xc55472bc,0x9dcf7393),LL(0xcc596835,0x86ab65ea),LL(0x0c9b7622,0x90362f16),LL(0x8c0ca08c,0xe8de2a3c),LL(0xec48a671,0xbde41568),LL(0x0286ac32,0xd27da64f),L_(0x00000038), LL(0xa6fda916,0xf6c82cd6),LL(0x53a87083,0x3e753ee6),LL(0xab548bed,0x07afab6b),LL(0xc34ddb60,0xc0dc2ddc),LL(0x378f8e85,0x399c4261),LL(0x5087e698,0x6f7e49f2),LL(0x07f39938,0x6345ae4d),LL(0xc730c9c6,0xb6c2765f),L_(0x000001f3), + LL(0x62619340,0xe4292c6d),LL(0xf4cf1a41,0xfb9a8b65),LL(0xf774c321,0x5046d341),LL(0x7b28d6b2,0xfe598075),LL(0xb06becbe,0xc3187f95),LL(0xd220a206,0xc278703d),LL(0x54ba06d2,0xb514e8c6),LL(0xda1d824d,0xc959300e),L_(0x000000df), LL(0xbb2a1c22,0x7fbd13f4),LL(0xec877f9e,0xd0e494f0),LL(0x209c6b0a,0x529b0f0c),LL(0xc6b1073b,0x50fb2f00),LL(0xd17f2e67,0x80cd82a4),LL(0x62378ddb,0x9f57c57d),LL(0x0162b312,0xc234e4cb),LL(0x8483d5e6,0x501d8ec9),L_(0x0000013f), + LL(0x4bb68070,0x0d037502),LL(0xd53d7a18,0x424ed14d),LL(0xd13f986d,0x29de6753),LL(0x3e4dbff0,0x6d33dc1f),LL(0xf6b77dc2,0x87ad5722),LL(0xbf6050c2,0xaea8f254),LL(0x83742064,0xb17406b4),LL(0x7d90e061,0x13b29245),L_(0x00000002), LL(0xb43c52b6,0x7f20e8bb),LL(0x5bf160f5,0x8562b323),LL(0x1d2d2e90,0x4b31d400),LL(0xea7b242c,0x4a1acb5c),LL(0x229d7510,0xc93f9b92),LL(0x3eba408b,0xb068a0e7),LL(0xb0525ab0,0xb376d6b0),LL(0xd96dff43,0xf1b03f82),L_(0x000001b4), + LL(0xd1e5c64c,0x77ddddf5),LL(0x631d2365,0xc4b6db39),LL(0x5fc5e812,0xd1cccab0),LL(0xc38ec807,0x8729f1a1),LL(0x1629e92c,0xc999e406),LL(0x6b4c00d1,0x781d88f5),LL(0x3cac8f29,0xcce3380c),LL(0x16b02141,0xc7e0e0cc),L_(0x00000120), LL(0xd88382b9,0x76234580),LL(0xd02da7d0,0xe2d27b0a),LL(0xcc82cf5a,0x3adad7f2),LL(0x2c08a15c,0x7009305d),LL(0x55fa7b4d,0xde9e632a),LL(0x0b55b693,0x2a821156),LL(0xb565732e,0x3788cf98),LL(0x89f0adb6,0x2d1f6054),L_(0x0000018e), +}, +/* digit=3 base_pwr=2^15 */ +{ + LL(0x5b21bde5,0xb0c6a7b6),LL(0x23a29c73,0x9c3eafc7),LL(0x392643c3,0xf81be3c4),LL(0x88c0b213,0xec734fa3),LL(0x33b98ae3,0x9b26d37a),LL(0x23074268,0x687a332e),LL(0x28354ec1,0x6935b64e),LL(0xf60d4b7e,0x9d55aecf),L_(0x000001a7), LL(0x910afa18,0xd6073362),LL(0x8bcd336b,0x5b5f67fb),LL(0xb6c7a784,0x5633e845),LL(0xdf601730,0xa907be72),LL(0x2814a576,0xfe65734d),LL(0xc7084b86,0x0758f113),LL(0xd7bad9f2,0x5030c22c),LL(0x3ef6af2a,0x7ff1cabc),L_(0x00000164), + LL(0xeb37269e,0x6184cce9),LL(0xac65525f,0x5051a406),LL(0xc9acc4f2,0x651c4a44),LL(0xb637bdd2,0x571fa6bd),LL(0x2ae9ce59,0x4cf1489d),LL(0xf56bdf32,0x61b0a821),LL(0xe5fa827f,0x9dcea620),LL(0x4b46a244,0x7027c9ed),L_(0x00000094), LL(0x0d4d4505,0x0495f1c5),LL(0x27a410cd,0xee6432c2),LL(0xbc9ba135,0x73536858),LL(0x53142570,0x7e39c350),LL(0xd0616e0b,0x316eeb65),LL(0xa694a069,0x55bbe949),LL(0x9aba0dc4,0x1f9d7b76),LL(0x32d36d72,0x1dcb7a1d),L_(0x00000004), + LL(0x4d5e5081,0x0fce6d79),LL(0x49c3fb55,0x3a2f9da5),LL(0x3a8e9a7e,0x44e158ff),LL(0xd771a67e,0x7de21bd3),LL(0xa6180b0e,0x5cf6b900),LL(0x349f9cad,0x53ff2b3f),LL(0x783786f1,0xe350b1ce),LL(0xec23cb86,0x58690faa),L_(0x000001ef), LL(0x09eb4774,0xe8902691),LL(0x4d7ea0cd,0xdfaca68b),LL(0x13648702,0x595a974f),LL(0x5bd316f2,0xbf226a22),LL(0xbb11b239,0xeaee978b),LL(0x2ab1e433,0xc7607b51),LL(0x870c9a0f,0x43795a95),LL(0xe00a29c5,0x53d7cad7),L_(0x00000060), + LL(0xfc15e51c,0x9b30d330),LL(0x8312448f,0x499ca6a8),LL(0x27c12fd1,0xaf5a132e),LL(0xc3fb765e,0x01b2d2a5),LL(0x07951a8d,0xce3517c8),LL(0x97c68ed6,0xe67d936a),LL(0x8cdd161c,0xad5eb28f),LL(0x795d9876,0x6496ac4a),L_(0x00000197), LL(0x4de7c0ea,0x7fd91252),LL(0x6e4dff62,0xe44601e6),LL(0xa96a9194,0x84a673b1),LL(0xf81ccae8,0x06054966),LL(0x2eba8c5d,0x53226945),LL(0x77e70b53,0x17deba76),LL(0x98891e5c,0x2fe55a92),LL(0xccf9a70e,0x8b39032d),L_(0x000000d4), + LL(0x4b1d8796,0x2c87d9f4),LL(0xce45ab56,0x0de1dc21),LL(0xa16d3789,0x72ace7c2),LL(0xe08192c8,0xe7012d3c),LL(0x4840d465,0x2d9fcc09),LL(0xd2d9e7c8,0xb83abe6f),LL(0x4dc89aa4,0x57f505dd),LL(0x58ef6f90,0xc12ca416),L_(0x000000e4), LL(0x0a635439,0x9e8dd733),LL(0x4f047388,0x1231cdd3),LL(0x536cd1c8,0x45523810),LL(0xd1e5a85f,0x4bcff7cb),LL(0x3fceb99e,0x86ad3d2f),LL(0x00ae1467,0xddf93ca7),LL(0xab6574df,0x4160edd9),LL(0x611238b6,0x0bbbbc9e),L_(0x000000eb), + LL(0x2f4ff50a,0x4b9dc9a7),LL(0x4e86b3f7,0xd56a4df5),LL(0xb7fc672c,0xc91daa4c),LL(0x047ac313,0xd8b04fac),LL(0x71df8b53,0xd047ffb7),LL(0x48cf7c44,0xe196a8ad),LL(0xbf663542,0xea4fed68),LL(0x45aa68b0,0xdbd49e0b),L_(0x00000083), LL(0x389e5cb0,0xd77d603e),LL(0x33664de2,0x5ef7dee2),LL(0x994f9685,0xc8ab10b1),LL(0x5e3c5bf8,0xf5ab3d23),LL(0xff2ae5c2,0xdbff37af),LL(0x9d0fd0f4,0x50db50de),LL(0xa6d91d52,0xe2c950fc),LL(0xa742da0b,0x0ec3836f),L_(0x000001c7), + LL(0xccb5796a,0xea3797f2),LL(0x00f8c37d,0x0b3e1166),LL(0xce0936fa,0xb532c55c),LL(0x204a444f,0xeef2ac73),LL(0xa6b09c79,0x31515d9e),LL(0xac9e3e09,0xdd05ab36),LL(0xe9cef435,0x319eb710),LL(0xfa2d9fd3,0x1d7ac545),L_(0x000000bc), LL(0x588c66b6,0x595b4001),LL(0x2f76c04a,0x0f70018c),LL(0x74e5849f,0xa9c62272),LL(0xb2abd908,0xaecd915f),LL(0x5ffbaabb,0x9fa73bfe),LL(0x111c8c5f,0x35b0554e),LL(0x77c9c2a7,0xcc8177e6),LL(0xe83b44a5,0x3bc6ae04),L_(0x000001ec), + LL(0x53c1578f,0x229b222a),LL(0xb1bb114a,0xff59f733),LL(0x887f6c13,0x2679cded),LL(0xbbad5dfb,0xd35dec8b),LL(0xea94d41f,0x90930770),LL(0xd4f0a601,0x2ad07da8),LL(0x2142901c,0x48f142ed),LL(0x692aaa86,0x252e4559),L_(0x00000142), LL(0x47539509,0x9b4f335e),LL(0x78c42f0d,0xc2716105),LL(0xfda89975,0x2c49b195),LL(0x35776137,0x3ac76051),LL(0x4de0d058,0xfcd0c4d5),LL(0x47ffa549,0xe11bc35f),LL(0x31f21817,0x3f57a567),LL(0x46ac2b10,0xcde0cd71),L_(0x00000084), + LL(0x3acc431b,0xaecaf4a5),LL(0xcd468ef8,0x60b977fb),LL(0xbcb8a438,0x3938f4bc),LL(0xcfcf5c2b,0x2c7337c9),LL(0x7bb844f3,0x23c47750),LL(0xdea5e248,0xf126971b),LL(0x47ee8dea,0x6f1d664c),LL(0xd5392932,0x3efa21b6),L_(0x0000018b), LL(0x3e152528,0x5940abfb),LL(0x28ef7f36,0x3e9bee76),LL(0x8f415722,0x360759cd),LL(0x11a30e1c,0x3c8733e8),LL(0x78196a73,0xc43394c7),LL(0xf3a60c7e,0xac3864e9),LL(0x776e1d00,0x0c19158c),LL(0x2e4681b7,0x517321cc),L_(0x00000040), + LL(0xcba05043,0x69bb2a3b),LL(0x59d22ba1,0x18bc1523),LL(0xee4d727c,0xbabfd9ca),LL(0x4c8338aa,0xe3550512),LL(0xa9cc3cca,0xe599b6e8),LL(0x15386807,0xc5ab3c64),LL(0x3919da2f,0xd2ee43d4),LL(0x801a4c6f,0x38ead934),L_(0x000001be), LL(0x64a97d4d,0x8b8c66b5),LL(0x7834d44e,0x74807217),LL(0x690ef307,0x926feb1c),LL(0x54c7151d,0xbe2f1f34),LL(0x456bd03f,0xc48ce8e6),LL(0x04a6964d,0xafec270c),LL(0xe8febbc7,0x483b3a5f),LL(0xd30f159a,0x96cb139a),L_(0x000000ca), + LL(0x77df0935,0x0e87f867),LL(0xf99ad667,0x75faf57c),LL(0x011dcb9c,0x6c05cb53),LL(0x4f1f75a2,0x3556cade),LL(0x2dea9ad0,0x3f87760d),LL(0xb590f7b4,0xe73b9512),LL(0xc497a74f,0x5a5a684b),LL(0x8d18f07d,0x8e2fa89c),L_(0x00000050), LL(0xb516cc59,0xc3adce30),LL(0x12408706,0x4d73c59c),LL(0xcce1c5bc,0x2ddcd22c),LL(0x381eb1ab,0x0b77c42b),LL(0x43827dd9,0xaee2e20f),LL(0x0ecadad8,0x4d7ed6ba),LL(0x141b0bef,0x69fa3aa0),LL(0x9ae275eb,0x3d138706),L_(0x00000114), + LL(0x3d8a013e,0x7fc0e976),LL(0x65d7b1d3,0xc8c06baa),LL(0x608a4b87,0x2e527b8c),LL(0xa2d8c259,0xcc19bb3a),LL(0xb09308aa,0x4ce5b0ad),LL(0x2458761d,0x7a6ee0f4),LL(0xd73d4f70,0xd791c442),LL(0x0d3867f8,0x3ba7a1a6),L_(0x00000094), LL(0xe51b0763,0x0e7ffca3),LL(0x467af3d9,0x60c44d23),LL(0x9427b9fa,0xe4a16358),LL(0xaff54ce0,0x55e4129a),LL(0x275c2816,0xcbefd5ea),LL(0x7c03c7fc,0xb7160ce2),LL(0xc97ca421,0x84bb35f0),LL(0xea69ee6f,0x35e0436e),L_(0x000001ec), + LL(0xfe162d02,0xf585af17),LL(0xbac45c7f,0xf7251745),LL(0xd6aa93a1,0x8a56414c),LL(0x8fa35248,0xf6e64410),LL(0x1720b12e,0x81f59ca8),LL(0x6cb0f80a,0x232a9916),LL(0x205cfe62,0x872efe0b),LL(0xdcba9616,0xa3d26e5c),L_(0x00000021), LL(0x9ac2f018,0x06a36051),LL(0x478ec567,0x7d42157a),LL(0xa110b6a7,0x0c863ff6),LL(0xb1e77441,0xa6979407),LL(0x7c13c78a,0x6a0ad3b6),LL(0x08c47fd0,0x34e0edd0),LL(0xcd2ed5cb,0x8df0c73d),LL(0x41a8e1a2,0x73883967),L_(0x0000012e), + LL(0x94304215,0x7d33b8c9),LL(0xa6572311,0x3fceee3a),LL(0x1482e2ca,0x52560262),LL(0x6d96dfdb,0xa105a9eb),LL(0xbdc41e36,0x8c0fd8b7),LL(0xa2f2edd5,0xb271c58b),LL(0x050043d8,0x4a51907c),LL(0xa79966a3,0x0fa52e13),L_(0x000000ee), LL(0x6d5fc916,0xdac2d706),LL(0x62accbe2,0x0b78e0d4),LL(0x8397028d,0x2c9d107f),LL(0x711b525e,0xfedd5666),LL(0x0c96203d,0x88395725),LL(0x2be09463,0xf9856d0f),LL(0x6dd96c8f,0x9c7a6702),LL(0x4398fe82,0xfc430b6d),L_(0x000001ac), + LL(0x41758c46,0xaa02764f),LL(0x7d06225f,0x36596aaf),LL(0x23dab345,0x0047b230),LL(0x1f940005,0x1c2f1ccf),LL(0xb4fb0f0c,0x82a82a8c),LL(0x589309ef,0xc66190cb),LL(0x19fbd0a3,0x839f41c1),LL(0x0fe2846b,0xcc1c9536),L_(0x0000019c), LL(0x917c26bb,0x729f81c7),LL(0x27782d0b,0x55359881),LL(0x76e1016b,0xcaad48a7),LL(0x26d82543,0xc89767f1),LL(0xcf1f4470,0xd4acb529),LL(0xe5b4bfed,0x7b75fd29),LL(0xae8ee068,0xc3d34db9),LL(0x3b3ffbcb,0x9c535467),L_(0x000001d7), + LL(0xe3f00489,0x9faba8ba),LL(0x5f421abd,0xe82276fc),LL(0x94ac402c,0x91f2efc8),LL(0x7d55bead,0x8241f32e),LL(0xcc1090d2,0xe8bce170),LL(0x19f59df3,0xe27350cb),LL(0x4ac35c2d,0x3e6cfc43),LL(0xd13cf90c,0x84bc2847),L_(0x000000a7), LL(0x54f1aa33,0xfd3f87f7),LL(0x2713cbe9,0x4fd8d338),LL(0x34163c33,0x46cada61),LL(0x7214cbe3,0x6aa94a54),LL(0x30a042dd,0xf7b92358),LL(0xe120acf2,0x09be500b),LL(0x30c3e8d0,0x51dc7f0d),LL(0x6f225e27,0xb7edd06e),L_(0x00000114), +}, +/* digit=4 base_pwr=2^20 */ +{ + LL(0xe47d13c6,0x20c1256f),LL(0x2fd11810,0x5aa78701),LL(0xc4a46931,0xea26a86c),LL(0x056b1163,0xbe00b905),LL(0xa0ac68e4,0x52f1dad4),LL(0xc19c5769,0xc6fde2d8),LL(0xbbc11dae,0x6293f810),LL(0x3a3baf9c,0x5056fba0),L_(0x00000137), LL(0xbfc9af73,0x5973f08b),LL(0x4cc716b5,0x8efce6c1),LL(0xb5b613b1,0x64d3ad94),LL(0x248f005d,0xba83b800),LL(0xa375eb34,0xc9ee4cf2),LL(0x413af2a4,0x68a27d29),LL(0x25ea8722,0x8d12fde5),LL(0xc9c082bd,0x2d233189),L_(0x000000fa), + LL(0xc1b123bb,0x85f1bef2),LL(0xa73fb5cd,0x111a8c9c),LL(0x1a80d76a,0x8d3b7461),LL(0x2e325f88,0x7765b87f),LL(0xc8ad9e3f,0x92e36012),LL(0x2c7cf6c4,0xbf5a9dc4),LL(0x7d5db366,0x6228a81d),LL(0x915359f9,0x725123cd),L_(0x00000172), LL(0x2cfcba5e,0x8b6c7a0e),LL(0xa38cc5da,0xee14f97f),LL(0xb43bb38e,0x770c4afd),LL(0xaa0f15c0,0x138850f3),LL(0x3953b993,0x2658cf7e),LL(0xb70f0779,0x1d447c8b),LL(0xd78fd38c,0x681177a0),LL(0x8e23ebe4,0x704ca751),L_(0x0000009d), + LL(0x527a9d3e,0xba8fa7e4),LL(0x4e9fda74,0x334944db),LL(0x404855f4,0x65201753),LL(0x31df130f,0x19a9846d),LL(0x661cb9d7,0xbc651ab9),LL(0xc04c2995,0x91c2b653),LL(0x1b2c3fb5,0x1b65fb33),LL(0xc90b91d2,0x9233b624),L_(0x0000001f), LL(0x061f9eb8,0xfceac108),LL(0x86d9cc5a,0x4cdd0a2e),LL(0x0e2ec8f9,0x309b7d38),LL(0xf2c40675,0x0d2223f6),LL(0xc1e34e32,0xa3be480d),LL(0xb364f62b,0xec527b72),LL(0x3595753d,0xf6639f06),LL(0x4e283d90,0x67ed0c35),L_(0x000001d5), + LL(0xd4783247,0x5667e2e3),LL(0x2b33e937,0x711cfb9d),LL(0x5cc9c7d0,0xedf0adb9),LL(0xc5aaa7c2,0x610b704f),LL(0x770150b6,0x1107368e),LL(0xf9af2a47,0x06e6cc4e),LL(0xfe1e566d,0x814dd0ca),LL(0x7ca67146,0x6c67f663),L_(0x000001d3), LL(0x3ca6a46e,0xecb744b3),LL(0xd960d19d,0xc0bcfa2a),LL(0x99ff41db,0x933b28a6),LL(0xb97977ca,0x951faf63),LL(0xca3752a7,0x15168f23),LL(0x01e0f16b,0x4ea397d9),LL(0x05f55f96,0x3b374a51),LL(0x813c0d40,0xe408ed3a),L_(0x00000143), + LL(0x0a55f862,0x937586c5),LL(0x83c230d0,0x61062265),LL(0x9c8f1eaf,0x10419f67),LL(0x2c698769,0x8d67dbad),LL(0x4407836e,0x4c3c184d),LL(0x99fd2f81,0x52a37538),LL(0x7825fefa,0x45a721e3),LL(0xfff07585,0xa4b823d5),L_(0x00000098), LL(0xd4ed2584,0x96e376eb),LL(0x6a23fbe4,0x5f76504d),LL(0xb69ec350,0x545afc26),LL(0xfb28fe6b,0x87ed2073),LL(0xaf95f20e,0xa6145047),LL(0x4d27cd1b,0xc4cc53f8),LL(0xa35d865d,0x9ee96b7f),LL(0xb07b711a,0x430aefde),L_(0x000000ec), + LL(0x69146afc,0xc7354ba1),LL(0x9fdb88ca,0xdc64a8c7),LL(0x9f85e2ef,0x7f3a69d0),LL(0x5631012d,0xd2bed232),LL(0xfd4d1f17,0x04dfd89c),LL(0xe64d46be,0xd5598288),LL(0x9f8bf20f,0x1f269d18),LL(0xc11d0864,0x333e29ff),L_(0x000001fc), LL(0xb8320599,0x9cc7dab1),LL(0xbbef8e94,0x5c714223),LL(0x7f10fed7,0xbb61d266),LL(0x09c647b0,0xe823dbf3),LL(0xf58db2b8,0x4601c5a1),LL(0x3a71fa3e,0x344f9c02),LL(0x0b5cbdd6,0x77b11f1a),LL(0xf8df6a65,0x6eb12db5),L_(0x0000007e), + LL(0x2c64c2a6,0x0db94e9f),LL(0x9c4cc346,0x646b9dff),LL(0x339e03c0,0x7ae26f18),LL(0x64dca76c,0x2ba1712f),LL(0x2c804061,0x16950e5f),LL(0xb5bf0fae,0x13d1569e),LL(0x185858b6,0x5b35ba86),LL(0x6880b124,0x3c937406),L_(0x0000019a), LL(0xe1c9646a,0xee5f1c44),LL(0x96f83044,0x10924610),LL(0xe69176fe,0x5cfb2614),LL(0x324a7887,0x825516a8),LL(0xfbad9007,0xc065d69c),LL(0x3d71727b,0x06621f87),LL(0x85c81c53,0xe21856f1),LL(0x3ac1471a,0x68582e4e),L_(0x0000007e), + LL(0xb68f07c0,0x9ace1c67),LL(0x5469124c,0x24f3ddfa),LL(0xccd6db20,0xaadd52b4),LL(0x74a2fc6f,0x24af0765),LL(0x17b27151,0xb5105915),LL(0x118a106b,0x7e240081),LL(0xcffda2d6,0xc6925ec7),LL(0x88b3b39f,0x37b374e2),L_(0x00000120), LL(0x541ec518,0xefd91b81),LL(0x3a683e17,0xa72b7c41),LL(0x952a60ed,0x0495c130),LL(0x9c4b61d8,0xbf06a574),LL(0x872c4bf6,0x0c7cbd39),LL(0xe01cb7ce,0x989f1a82),LL(0x726d7547,0x44906b41),LL(0x52742de9,0x2e02ff37),L_(0x0000009f), + LL(0xeedd6da3,0x8dd4b66d),LL(0x73240fb0,0xb6e39bb1),LL(0x1303b771,0x195468c6),LL(0x7bd7b8e4,0x9c3d4d09),LL(0xa8684e6d,0x724f9017),LL(0x31c9bec6,0x2fd691e3),LL(0x727ff44a,0x5f3d4db7),LL(0x89060924,0x984ffa88),L_(0x000001b0), LL(0xcf998e88,0x0c1bfdeb),LL(0x6d85f7e8,0x4b0cbc59),LL(0xccccc632,0xe1faf1ce),LL(0xc7a0620b,0xf6e95c18),LL(0x0aa71a3a,0x8fa50a9c),LL(0x49a07249,0x8cf3e64b),LL(0xaed36f6a,0x94bd6377),LL(0xa4cf33ed,0xeb0073d7),L_(0x0000013f), + LL(0xadc752d7,0x87c3614b),LL(0x53d1d5ef,0xaa78183f),LL(0x8ea4ef11,0x5c12de26),LL(0x84a5c0b0,0x3315c75e),LL(0x4f1d31e4,0x0d7a1bdb),LL(0x97ef83ed,0x8d2ce325),LL(0x91dd29d3,0x3340be93),LL(0x184b8ada,0x0f8f6087),L_(0x000001c1), LL(0x9fad91c5,0x8ce9dccb),LL(0x96d5b0b9,0x641d29f6),LL(0xb48f1d66,0x50813cfd),LL(0x8881842c,0x7ff13a32),LL(0x3fa7e856,0x78eaa08d),LL(0x7984e336,0xfbc798ef),LL(0x67c2e064,0xb3fc5a0a),LL(0x23e92b2d,0xe38a9115),L_(0x00000008), + LL(0xb65bfdd1,0x3691b2b4),LL(0x9b215aee,0x5cf3f209),LL(0xa1d61cff,0x4a0f639e),LL(0x2783ba69,0x62cddd5b),LL(0x1490682d,0x5648b7c2),LL(0x074ffd3c,0xeb8c7a90),LL(0x0e9bddbd,0xf3751b7e),LL(0xbce879c7,0xa3bfa1d5),L_(0x000000d9), LL(0x8d0510d3,0x20a30aeb),LL(0xe42922b6,0x3498a06c),LL(0xa129b145,0x26cc79e5),LL(0x9d307ed3,0x1c156f64),LL(0x2baf6215,0x14f481b2),LL(0x9b79cac0,0x2eb38c02),LL(0xc564e629,0xde0cf55a),LL(0x24a1582a,0xad5a0f7d),L_(0x000001d5), + LL(0xea9885d3,0xa68ccc39),LL(0x6997bdf9,0x05de1b77),LL(0x553e670c,0xee7e6f6d),LL(0xadae5917,0x199db8cd),LL(0xecf2088a,0xc33572de),LL(0xa41c0fa0,0x7b432d00),LL(0x1dc922e3,0xc58b8529),LL(0xb5615888,0xb9f1df40),L_(0x0000017e), LL(0x8b5496a0,0xe11804a4),LL(0x9626517b,0xb50d5ff0),LL(0x34399d8f,0xeb302429),LL(0x18d0e9c5,0x47c13487),LL(0x18ec316e,0x851d2afc),LL(0x1b701f2c,0x167d9f22),LL(0x883a9116,0xa8f932f6),LL(0xc3a9d8fd,0x0da0e796),L_(0x00000102), + LL(0xd19a69ad,0x029cd0ab),LL(0x5d899070,0xb5278a7b),LL(0xe7ecc032,0xabde08a8),LL(0x9fa79438,0xd3d2c019),LL(0x80363047,0x66ae4725),LL(0x141bc210,0xeab542c4),LL(0xdf1d8696,0xd1c060a0),LL(0x3908c1f0,0xfa788627),L_(0x000001d3), LL(0x31ffefe1,0x677d387d),LL(0x76fb5476,0x05f7c0d8),LL(0x3a298203,0x2097dc46),LL(0xe5119f75,0x1f8bd7b9),LL(0x5996aa0e,0x61982b25),LL(0x92c77df7,0x119a9371),LL(0xac8b008e,0x7f9b0675),LL(0x346bbe9d,0x8f2407d2),L_(0x00000037), + LL(0x68bd6724,0xab5b8457),LL(0x52f7bee1,0xbc2db536),LL(0x07aae0d0,0x58623dce),LL(0xa3e03da9,0xf444f5c3),LL(0xe7b9b75d,0xdb04bbba),LL(0x8fa837de,0xb065c230),LL(0xed346416,0x51909adb),LL(0x16d6222a,0x3c084209),L_(0x000001f1), LL(0xe28ca284,0xdf2f5e19),LL(0xbbb390fd,0x38ba1ac8),LL(0xc03099bd,0xa1e828bc),LL(0x6f1d5124,0xb83c59a2),LL(0xfd863005,0xc17a0d0d),LL(0x97857c15,0x2d92fc6c),LL(0xd53a6942,0xb70e5498),LL(0x36d4d6e3,0x368524fa),L_(0x00000187), + LL(0xb1d0aee2,0xc5143f3d),LL(0xcda50c31,0x34bd47c0),LL(0x5f0ed71c,0x1dee87b7),LL(0x24882cc9,0x7e3bbffb),LL(0xd1c7d1fb,0x1e403329),LL(0x92474fcc,0x53d4ee41),LL(0xbcade2fb,0xbd25257b),LL(0x48035309,0x1ff4afd1),L_(0x000000f3), LL(0x35e3b8a0,0xf76b6465),LL(0x54f67634,0x6c3176f4),LL(0x76970368,0x7bbaa2a0),LL(0xe3ebf308,0xc5082d82),LL(0x65ccfb3f,0xec84bb5f),LL(0xf3b5795a,0x4999273e),LL(0xa51dd238,0x7a764cec),LL(0xe6317658,0xd0cd1c12),L_(0x000000d5), + LL(0xcba7983d,0x440872f1),LL(0x28a369cd,0xf5dafa9c),LL(0x07ce63b6,0xc12c54cd),LL(0x4f79c12d,0x0a67eec8),LL(0xee7a34a0,0xdbc7a4ed),LL(0xff4e7f63,0x40f7ea82),LL(0x50ab4929,0xdc3353f9),LL(0xcd953027,0x1b37a3c6),L_(0x000001e2), LL(0xb77d6e77,0x4c5f758a),LL(0x416ce18f,0xa1adfbab),LL(0xb9e45e70,0xa53a6ae6),LL(0xf8e38074,0x19bbc5ea),LL(0x639d3359,0x66580a40),LL(0x3c3446f8,0xabe333f8),LL(0xf20b5de7,0x703c0639),LL(0xd4a42c48,0x59dbbfad),L_(0x000000d6), +}, +/* digit=5 base_pwr=2^25 */ +{ + LL(0xc7d07442,0xd025b49c),LL(0xa66a4107,0x4a0b9d5c),LL(0x09a27870,0x383562d7),LL(0xb8b27e26,0x8bfb3e1f),LL(0xcd156bc2,0x06132452),LL(0x49ca82ac,0x92d3d9a2),LL(0xa22d3a0e,0xd34655b5),LL(0xa2b19aaf,0x36ff20de),L_(0x0000008d), LL(0x5835a5bc,0x86f6b302),LL(0x1143727c,0x8192a853),LL(0xc5c00385,0x52b2fd4b),LL(0xdaf1da93,0x5e79d62b),LL(0xde7c27d2,0x23c74903),LL(0x18fc4730,0xd6b3ad47),LL(0x3e48286e,0xe1fb2466),LL(0xde5e2dbe,0x558feb16),L_(0x00000057), + LL(0xf7f37af3,0xf33aa997),LL(0x399f83a0,0xac6eebd0),LL(0xf152344e,0x1e36ba58),LL(0xb233cc5d,0xbcb7b01b),LL(0xb95ccb58,0x0e6e01c3),LL(0xbd70b347,0x28b73f3c),LL(0x471f1429,0x192a8fa5),LL(0x72f33fe8,0x51b82a42),L_(0x00000077), LL(0x1a1be232,0x91ea1f80),LL(0x29d67e20,0x9219d0e0),LL(0x355df0fe,0x86128187),LL(0xb88069ec,0x196ceb57),LL(0xc5d6f3e9,0xcb831fc0),LL(0x61a90e72,0x76ccb185),LL(0x875cf514,0x775daca6),LL(0x5e9d0fdc,0xf11fb0d6),L_(0x00000156), + LL(0xc6a99de8,0x4f6f0528),LL(0x8365a540,0xd607f5f7),LL(0xa6130da7,0x85d457a1),LL(0x282ce8f3,0x7a71db8c),LL(0xf22e8e12,0x675b1780),LL(0x1ed981d1,0x54455f27),LL(0x879b7161,0xd4c8d2d0),LL(0xe53f2395,0xfbb885ea),L_(0x0000002f), LL(0x9830814f,0xdd4e8a69),LL(0xc1a29f03,0xd15d6f60),LL(0x620f1843,0x51aa6497),LL(0xb9253703,0x3bdc3d85),LL(0xb15aa83f,0xd5713630),LL(0x38bafd76,0x46f5b250),LL(0xa324c450,0xc7ef0e3a),LL(0x428ff854,0x3074cd9a),L_(0x00000187), + LL(0xda6a189a,0xdb38150e),LL(0x98edde21,0x02c3ed54),LL(0x10e8c4a8,0x92441512),LL(0x5d3d5708,0xf9b009f9),LL(0x785d69a6,0xac80e954),LL(0xa197859e,0x166d8dbf),LL(0x9a954ad6,0x5cac81cf),LL(0xd5536054,0x01aaa83b),L_(0x0000002d), LL(0xc362ba43,0x578a0342),LL(0x2cf62469,0xb8dd8820),LL(0x8c4987d0,0xc2b48e0f),LL(0x3b1becfe,0xd5c621a6),LL(0xa90343a0,0xf668f2b6),LL(0xfe098700,0x9c088bd4),LL(0xbd8fa882,0x3f0936a8),LL(0x67794851,0xae92e720),L_(0x00000133), + LL(0x96b1f8f9,0x45dfbf2f),LL(0x3cd0f06a,0x39d3c154),LL(0xf941b6e2,0x350febe9),LL(0x2d076edd,0x4d560b52),LL(0x55928e0b,0x2b184b05),LL(0x9f1196dc,0x85b25794),LL(0x6b4f0570,0x942eaf69),LL(0xe7c1e0be,0xd7c08589),L_(0x00000017), LL(0xe3a9890c,0x7bc564b2),LL(0x3dbde8c2,0x8d95012a),LL(0xeff3d407,0xa7b28d48),LL(0x08aae60c,0x1b0df773),LL(0xe7d844e8,0xf5cd9389),LL(0xe907a4ac,0x0fa28183),LL(0x6e40930b,0x381dd63b),LL(0x20fcef80,0xa89662a0),L_(0x0000003a), + LL(0xbd48b590,0x33be7ac9),LL(0xf700265a,0xf01390c9),LL(0x82de52e5,0x6ffa798e),LL(0xd0dd94ce,0xd994953e),LL(0x2a40701e,0xdc206da1),LL(0xf9937624,0x9641d8d6),LL(0x68b09fde,0xfb08d3aa),LL(0x515609bd,0xc7a5c685),L_(0x00000107), LL(0x81a858ff,0x34c11eaf),LL(0x007f99c6,0x3e931cb9),LL(0xfb91b432,0x97268272),LL(0xb16a5f9f,0x14f51fca),LL(0x22fdf379,0x448325ce),LL(0xa5ef4116,0x36a8c292),LL(0xb6fe6cbe,0xee4ed4ed),LL(0xd30012d0,0x5ebc5cb5),L_(0x00000061), + LL(0x4947635b,0x6edf3f2a),LL(0xc68eb226,0x00200346),LL(0x880aea4d,0x8d31f029),LL(0x006895c8,0xf085775e),LL(0x89fcfc5e,0xd25cf0d0),LL(0xb2a191ee,0x4508a2eb),LL(0xc9428552,0x90ca4e90),LL(0x22abade9,0x985e1eac),L_(0x000001a8), LL(0x90df5f6c,0x31e5ce6f),LL(0x96705235,0xdb11d250),LL(0xee984de1,0xfa51e382),LL(0x06a31aa7,0x47482dcf),LL(0xb9587e74,0x988048ce),LL(0x51593c99,0xca987f12),LL(0x6fe30df1,0xffd978f6),LL(0xfa604f59,0x8e6925df),L_(0x00000032), + LL(0x4c96e5ee,0x24f8517a),LL(0x039e54c1,0x2549eb0e),LL(0x1ec83d52,0xeb02627a),LL(0x18a8ead8,0x95771e96),LL(0x497c2cef,0xd4f3a6b4),LL(0x1c4917c7,0x080b55a4),LL(0xe8babd80,0x90585b7b),LL(0x6c548c79,0xdabe54a0),L_(0x00000064), LL(0x978d3828,0x2b269bac),LL(0xdf9b98b0,0xc4ed8c6b),LL(0x0247e075,0xd8e1a299),LL(0xbc830716,0x019c94ed),LL(0x3458f33a,0x9c1665fb),LL(0xcd4d82fa,0xff63e700),LL(0x2326e507,0x41c9bbfa),LL(0x5b452b83,0xc9214a3f),L_(0x000000df), + LL(0x23971495,0x3a39f3de),LL(0x6207af25,0x48efebf2),LL(0xe105123f,0xc14c4d56),LL(0xe2f0d038,0xe91d26ba),LL(0x4b6de581,0x4ec88307),LL(0x8f223417,0x0cd6e3f2),LL(0x6ecd94a5,0x986d1a76),LL(0x75765814,0x6e247362),L_(0x000000a2), LL(0xb07a61db,0x8b83a718),LL(0x6e918a1b,0xc07b54ba),LL(0x67b07959,0x22adb8dc),LL(0x96a7a38b,0x2810db83),LL(0xd80ec3cd,0x512687fb),LL(0x3b7485fc,0x6a994329),LL(0x746aec5a,0xd193ecb8),LL(0x6e8d782e,0x72720264),L_(0x000001e3), + LL(0xd528a6e2,0x18efe68e),LL(0x7f9ec076,0xd1c7c3de),LL(0x2cb36021,0x39cd1c50),LL(0x4584b14d,0x997b5332),LL(0x4275cf71,0x8ac5db4a),LL(0x47f829e4,0x0bc80acc),LL(0x41dd84b4,0x93021863),LL(0x498e9e29,0x6c2be4f1),L_(0x000000f7), LL(0x3b0142c7,0xa59f6acf),LL(0x50030fe6,0xbe6f68fb),LL(0x21480f26,0x217fd94a),LL(0x7a75421a,0xfc5b6dc5),LL(0x7f8cd384,0x19e9f3af),LL(0x674a2419,0xd009b54a),LL(0x454fa1ef,0x6bd82b7f),LL(0xf5bea7db,0x31688d56),L_(0x000000fb), + LL(0x5eccd42c,0x24d7621f),LL(0x92168834,0x1fc413da),LL(0x47855bbd,0x13a9e461),LL(0x17199daf,0x4e3536bc),LL(0x42c2a13c,0x2b4a64a6),LL(0xa8670cf2,0x38d86ada),LL(0x6c221d69,0x68ee0d8d),LL(0x91def21c,0xa5126dec),L_(0x000001f3), LL(0x87485e27,0x03f14fd3),LL(0x378bafe9,0xef0ab60c),LL(0x11be6eda,0xe18440ad),LL(0x59ce941a,0xead8d861),LL(0x59cf0928,0xd51985bf),LL(0x608dbb83,0x54f87d88),LL(0xb568f14f,0x15b84493),LL(0xb574903f,0xbc432c12),L_(0x0000006d), + LL(0xe7e4f826,0x79d38b74),LL(0x7eaa6aae,0xf84e0117),LL(0xdb942db5,0x77a0b212),LL(0xd22fb9bf,0xc56ffa35),LL(0x3c3e0310,0xdf48ef3e),LL(0x9f901558,0x372e6e7b),LL(0xa69d9dbe,0x0da5fd74),LL(0xeaa231af,0x1a06775c),L_(0x000001b6), LL(0x0771a3ac,0xd09d3577),LL(0x7802613d,0x61b4d69e),LL(0x19ba2ee8,0xa78fefe1),LL(0x46a7d09b,0x7be0e7b7),LL(0xedbd7a6a,0xed289bee),LL(0x894b4148,0x613e6a64),LL(0x4ced52a0,0xf2a487de),LL(0x58c856b6,0x802d7250),L_(0x00000032), + LL(0xffe70196,0xd1a6df86),LL(0x365ca885,0xca64af53),LL(0xfb62f8cc,0xbd654db8),LL(0x7be4b7f7,0x029300b0),LL(0x8e5d1bd5,0xc892e13f),LL(0x2c67f87f,0xcc4777a4),LL(0x8b3a46c0,0xba6516c2),LL(0x7400b2a4,0x6cbe8178),L_(0x0000003d), LL(0x43d5f5dd,0x24153eca),LL(0x540f05b7,0x19221aa5),LL(0x97f95061,0x09e289a2),LL(0xbba5984c,0xf2867a5f),LL(0xa0f6739b,0x0f075e42),LL(0xa0718647,0xab768457),LL(0xa0b31eb8,0xae89b1c6),LL(0xf7e66ea8,0x127d85c9),L_(0x000000df), + LL(0x49e4dae9,0x6d5a6301),LL(0x8cd94792,0x3dd861e6),LL(0x97b8baf4,0xfe513cef),LL(0x04f520d2,0x71773dad),LL(0x05832bb5,0xe9f256e1),LL(0x644c2754,0x40df1966),LL(0x700981f0,0x9215b453),LL(0x5dacb6f1,0xc1eb3c97),L_(0x000001dd), LL(0xd29b37a5,0xbd504fe7),LL(0x365fddbf,0x8fd2242a),LL(0xc2c35f7a,0x4bda285f),LL(0xc1adbae0,0xe964cfd7),LL(0x9f4dbf4e,0x29f5bb5b),LL(0xebe1db2e,0xaab3521e),LL(0xeb26c679,0x17b1e0d2),LL(0x460499ac,0x5a9383dc),L_(0x0000000a), + LL(0xbe71a28d,0x13ec332c),LL(0xb2a67fff,0xb50e03f5),LL(0x50971210,0xfca3bc03),LL(0xa67e12aa,0x4ad52b09),LL(0xbb90dbda,0xff71aa12),LL(0x0edb116e,0x6d43afd2),LL(0x0827d59a,0x97de7056),LL(0x29d216bb,0x7a131b7f),L_(0x00000160), LL(0x6cd8ac84,0xb922ac22),LL(0xfa7cdb03,0xf832d6ed),LL(0x58c5a459,0xd373c013),LL(0x7af850a7,0xec4706eb),LL(0x2c0a7082,0x67642828),LL(0x10a909d7,0xae5e13c9),LL(0xab317113,0x5697d646),LL(0x62e5e289,0x83fffcd8),L_(0x00000122), + LL(0xf34910df,0x331109fa),LL(0x16fb0987,0x7dc5b06e),LL(0xf23db224,0x62c76a0d),LL(0x3f2f53ef,0xf6843163),LL(0x41397218,0x63cbe6a9),LL(0x6d05ffc2,0xb14a253b),LL(0x7ed103a8,0x8ee0b454),LL(0xf389d693,0x3310d7c6),L_(0x0000014f), LL(0xe61f0fe2,0xe3ae5ea5),LL(0xeba63d9e,0x281cc803),LL(0xff5d7d44,0x5a289454),LL(0xfa26da52,0x8b5bf3e7),LL(0x8b9a0868,0xf6c91ab8),LL(0x565a2877,0x10a39907),LL(0x737c0212,0x1223d6a1),LL(0xe131db55,0x19f277aa),L_(0x00000014), +}, +/* digit=6 base_pwr=2^30 */ +{ + LL(0x77485e2c,0x205e28d0),LL(0x97a1a0b2,0x662c120e),LL(0x6679b5d8,0xd256806d),LL(0x56f0da8b,0x5177882d),LL(0xb3a0c292,0x1fd447d9),LL(0x15eb4fc8,0xd44b077e),LL(0xf15d98b7,0x44ac5bb0),LL(0x2721ac99,0x38b1a539),L_(0x00000164), LL(0xb3161c22,0x2615ec8d),LL(0x7c193099,0x6014ef39),LL(0xde0771a4,0x7929bb3e),LL(0x2d975e02,0xcd3fad00),LL(0xd387bd9d,0x38c634c3),LL(0x4ebf2479,0x08079c3f),LL(0x2c2da2b5,0xe2b2209a),LL(0xd2e85151,0xbb80ad2e),L_(0x000001f4), + LL(0x3d0b9c70,0x8f90c95c),LL(0xb8f686b4,0x17d91844),LL(0xa7467794,0xb498e1e2),LL(0x354bd2f6,0xdf2e61e9),LL(0xaea1ed2a,0xb0db08d1),LL(0xf347f08d,0x02cba497),LL(0x78713784,0x5de0850c),LL(0x51753c73,0xed5b7079),L_(0x000001f9), LL(0x5f819b71,0xd4d045e4),LL(0x91e83186,0xd4180245),LL(0x2f07b1ad,0x059bab68),LL(0x1d467074,0x2b0f371c),LL(0xb0e391ea,0x80e5f219),LL(0xf22e68c2,0xb058f844),LL(0x79b7f960,0x6ca98996),LL(0xd1250203,0x1f63acdb),L_(0x0000008a), + LL(0x8ce35c47,0x6f5a9e54),LL(0xbda97f65,0xaf7af291),LL(0x3726d93c,0x49f05287),LL(0x2f95260a,0x41ba40c2),LL(0x77d547dd,0x957f51b5),LL(0x05f74755,0x8ef3d171),LL(0xabb35056,0x131fa675),LL(0x73e0ad7e,0xf2cdf14a),L_(0x0000000e), LL(0x411ad210,0xe74e4c04),LL(0x5dd32692,0x56454c03),LL(0xbec75c70,0xcf373902),LL(0xf969fbf4,0xcc237598),LL(0xc6be1f8c,0x25a52045),LL(0x1317acb0,0xbd2ef98d),LL(0x4c09bec9,0x87e896e1),LL(0xa4033c40,0x16b064d0),L_(0x000000c5), + LL(0x9acb75bc,0x2223b656),LL(0xf580463d,0x7a55f8bf),LL(0xbd3d896f,0x19d6f3c8),LL(0xdc1bc358,0xf8dfdd32),LL(0xa2786cfc,0x57477a57),LL(0xae5589b6,0x9f47a3b0),LL(0x06de5eae,0x93f21788),LL(0x2c782fcd,0xd9479ee3),L_(0x0000016d), LL(0xbeebc5d3,0x0929435c),LL(0x52a35de2,0x5b11dd83),LL(0x4ce809af,0x080ab8c3),LL(0xe25a76cf,0x50478b1c),LL(0x216bfb22,0x103b3ff7),LL(0xd87f4762,0x99b34133),LL(0x41327480,0x95f627aa),LL(0xa729b689,0x62b3cf30),L_(0x000000fb), + LL(0x2f95c649,0x721c2b47),LL(0x251bc403,0xb04b42b0),LL(0x43b1fedb,0x73ae072d),LL(0xe91e26a3,0x0d23d9bf),LL(0xbe237693,0x40c88f4f),LL(0x3424beb1,0x6a55a156),LL(0xa0854744,0x26b2b7ac),LL(0xd61d14b6,0x4ee7f1ff),L_(0x0000000f), LL(0x0da4f895,0xa08faeb0),LL(0x9b46600b,0xfac400c9),LL(0x541f8756,0xfa469aa7),LL(0x961867a9,0x9f699a76),LL(0xaf3ea90e,0x6fa24828),LL(0x87c1ec18,0xb4eae9ea),LL(0x13538c6c,0xd16f2fd6),LL(0x3f2706c2,0x1ba2b83f),L_(0x00000147), + LL(0x792b76d7,0xbdd4d4a0),LL(0x305ae6fd,0x1d7bb497),LL(0xfd52a449,0x143fec43),LL(0xf6e16ac4,0xebf21b11),LL(0x8a70f179,0xdc6c0238),LL(0xb6be355d,0xd156ead0),LL(0x45178899,0x91cfd7f1),LL(0xdeb4b443,0x63af6554),L_(0x0000001a), LL(0x3a841a01,0xc755678e),LL(0x78e21693,0x747e0d46),LL(0x04110693,0x8c5bb7f8),LL(0x878fdb74,0xb933a4bb),LL(0x90576198,0x4a53f37a),LL(0xb6123d17,0x9e11e3a2),LL(0x6fc61c48,0xf4f2778d),LL(0x0bd5422a,0x78936064),L_(0x0000002a), + LL(0x889e1c3e,0x1cdfc255),LL(0x0e89d4d5,0x620f831a),LL(0x353a0af0,0xb276e069),LL(0x687747f6,0x848dd0c6),LL(0xf27dfd62,0x0bb6c7d1),LL(0x05846ed2,0x561ca5d7),LL(0x17ec1af8,0x39a2756b),LL(0x4280d8ec,0xf4bc9db5),L_(0x0000002c), LL(0xe51a0c90,0x827b9727),LL(0x96e34ce9,0x3fe216b6),LL(0x79c77aa0,0x6720f347),LL(0xfad412f9,0x695ff127),LL(0x74dee9fa,0x5f1b0845),LL(0xe65f23d5,0xbfbfe9e6),LL(0x3e545ad7,0xe1c9f0ed),LL(0x92d8678d,0x9914fd4c),L_(0x00000088), + LL(0x3d108b40,0x479267d4),LL(0x0396f4d9,0x88692b5b),LL(0xb1e5f7f0,0xfd9437e3),LL(0x9509c3c1,0xb9b9b7fa),LL(0x15a75e6c,0xba37b6c1),LL(0x5b42f650,0xdbbc62d0),LL(0x5b3cf510,0x6d5fe62d),LL(0xc6a49bf0,0x824e6593),L_(0x00000049), LL(0x6b1fb538,0xcc7f70f3),LL(0xeb94fc53,0x6a2b34a3),LL(0x82c0e60f,0xea4bc9d4),LL(0x8f42888a,0x7ab9ed31),LL(0xc0fe3ed3,0x1a505ae0),LL(0xe94fad3b,0x545382c9),LL(0xfa0f3128,0x9ae4fb8e),LL(0x44bd4be6,0xa516fe88),L_(0x00000003), + LL(0xb0f06a74,0x5455aac0),LL(0x2a8889b3,0xc9b0d66d),LL(0x1c5bf83e,0x736f64af),LL(0xeb5d7b9e,0xb0411171),LL(0xfd19cd82,0x22d46926),LL(0xdc65529f,0x1a902efa),LL(0x2603fa26,0xa829e75c),LL(0x4e40cf1f,0x25ea27fc),L_(0x0000007d), LL(0x5762e183,0x4b68fce9),LL(0xe6bccc3f,0x83ed8b75),LL(0x047d0099,0x864c3567),LL(0x85060d1b,0x1ddc1dd9),LL(0x3ee4ae76,0x974e9669),LL(0x389206d2,0x6126092a),LL(0x91ac46ae,0xe4c7cd87),LL(0x450710f2,0x9e581f06),L_(0x00000040), + LL(0xa9671b83,0x291b3bd9),LL(0x42264245,0x08f40ce6),LL(0x9254ae8e,0x244beb5d),LL(0x6f14d800,0x0b345a89),LL(0xe0141c82,0xb5b0de4a),LL(0xf52d10c3,0x2ca8fb1f),LL(0xebf183aa,0xdb884d75),LL(0x8611998a,0xd2bb9afb),L_(0x0000019a), LL(0xf19b2de4,0xa6d267bb),LL(0xe6bc257b,0xf0568557),LL(0x2160d875,0x6b9e2dd7),LL(0xb1fad31a,0x9c709714),LL(0xed3d24a8,0xf7d05de5),LL(0x61431dc5,0xdc526178),LL(0x0501b15f,0xe40e89ee),LL(0xb2aa5088,0xf3025f1d),L_(0x000001f8), + LL(0x7bfa3bb6,0x859759f0),LL(0xb249eb45,0xff4d25bf),LL(0xb8020268,0xd9737355),LL(0x9d4a831f,0x84343cef),LL(0xba7e139e,0x5bb5557e),LL(0xb24bfdbf,0x3ab8fb19),LL(0x31a1b161,0x3fa57146),LL(0x4ec60d24,0x01fac245),L_(0x00000125), LL(0x4ddf36d2,0x3e8c0bac),LL(0x5b116d34,0x0f307133),LL(0x62db28ba,0x9a0533b5),LL(0x0939b6eb,0xacf1cf00),LL(0x89d1c401,0x693e7c92),LL(0x4b14c89d,0x0945b630),LL(0x4ed7b53a,0x4b162dc1),LL(0x79a7fa4b,0x3e6b7dc4),L_(0x000001c0), + LL(0x591d8c23,0xb35e8088),LL(0x24bb4a7a,0x9e47ec6e),LL(0x968fd370,0xe7d95582),LL(0x474d55e3,0x783f2538),LL(0x9d35c0f8,0x83b454e1),LL(0x4578fbe5,0x114b724c),LL(0x98f51326,0x97ee546b),LL(0xf190d99b,0x801229b8),L_(0x00000091), LL(0xaa86765c,0x7ee4a0a2),LL(0x8b19b4b6,0xd4f5faa0),LL(0xc220cb11,0x6ad0862b),LL(0xce1c7d42,0xd7a63508),LL(0x3236a72d,0xeaae4777),LL(0x36cda4ac,0xbe3fcce9),LL(0x808ba7a0,0x972c3840),LL(0x79c14a5b,0xcda4655c),L_(0x00000087), + LL(0x4283d37a,0xf72e10a1),LL(0x23f718f8,0x07515abf),LL(0x820e2eb6,0xc81af50b),LL(0xea92c2b4,0x06af11fd),LL(0x71048468,0x755ab069),LL(0x6ada8b6e,0x5f2e0402),LL(0xe7b549c5,0xa45c27e8),LL(0x13b56922,0x1760f8ae),L_(0x00000001), LL(0xc7aa6c54,0xacb0965f),LL(0xc6437d60,0x30e79f87),LL(0x93a0e68a,0xcb0f4dcf),LL(0x3613f9b0,0x02e2383c),LL(0x32355c62,0x3ec75821),LL(0x4a835dde,0x4891bfda),LL(0xa38a3a87,0xd0206185),LL(0x9de0b9b9,0xa9cedb8c),L_(0x000001f4), + LL(0x2fe65202,0x8b5ddb45),LL(0xdc46fb16,0x8166a790),LL(0x41a5e4a7,0xc854c900),LL(0x342b51ba,0xeb4a908e),LL(0x116e0741,0xb76499d7),LL(0xc1551311,0x6cc7399a),LL(0xccaf37b4,0xf2094cc8),LL(0x9c29f265,0xc4a1fb65),L_(0x000000c7), LL(0x0d15a232,0xa4cf272a),LL(0xcfc8b9dd,0x9fd1f1c4),LL(0xb8e8cb4f,0x8ddfade3),LL(0xfd72dadc,0xf084903f),LL(0x764ba410,0x9094e8a7),LL(0xf43466f6,0xb9869fee),LL(0x1564c4ac,0x82a6cc83),LL(0x7742c943,0x89c1b557),L_(0x0000016c), + LL(0x6d728fc3,0x7e25b95d),LL(0x08c1d55c,0x50011066),LL(0x2de90a9b,0x63db375d),LL(0x46d6ba3b,0x2d3fa9bd),LL(0x9ae8aa53,0x9fb12874),LL(0xabf67a35,0xd1f1f45f),LL(0x05d8099d,0xdc479fc0),LL(0xdf2cdfde,0x779d8b52),L_(0x000001a9), LL(0x9e4fbdda,0x8f62ac2a),LL(0x4ab94029,0x56a0908f),LL(0xbb03c1b4,0x9ce0943b),LL(0x6319d2ed,0xe2d743f7),LL(0x910c2363,0x871683b7),LL(0x78ddf2fb,0x43f08ff4),LL(0xb7c5e230,0x2ebe58e7),LL(0x1c175b19,0xc02e1520),L_(0x00000185), + LL(0x39096d41,0xec7786aa),LL(0x70017bcc,0x26576a40),LL(0x926b1ae0,0x5e810d17),LL(0x6c002685,0xdef24d30),LL(0xd826c439,0x78ac55c7),LL(0xe40f66cc,0xfbf9b521),LL(0xe26697c9,0x01aaaf8a),LL(0x46a75caf,0xd094aec3),L_(0x000000a9), LL(0xc1b67b75,0x99a161f4),LL(0xcc3c7e5f,0x88977e72),LL(0xfefa749b,0xd6e24cf4),LL(0x56863737,0x4607a910),LL(0x4dcfe4ed,0x01768685),LL(0xac7bc09c,0x9ec211ac),LL(0xc731394d,0xac76909c),LL(0xd59e4dd8,0x246fb612),L_(0x000001c7), +}, +/* digit=7 base_pwr=2^35 */ +{ + LL(0x31c9da36,0x5592f380),LL(0x0dd54d04,0x44d05f6f),LL(0x298fe241,0xe2d191b2),LL(0xad46274a,0x9113fc1e),LL(0xcc54c590,0x0dea702c),LL(0x763ea8c1,0x74f208c5),LL(0x35441e72,0x578c5736),LL(0x3706b2f2,0x81343695),L_(0x0000003b), LL(0xa85cfcd7,0x88ada464),LL(0x4c2605a5,0xd5464606),LL(0x004bbf31,0xca04e18d),LL(0x03210805,0x998cccf9),LL(0x89627867,0x48398a6f),LL(0x8d2faed1,0xca85dacf),LL(0xde01ea7e,0x92784742),LL(0x0feb7d82,0x9fe5859b),L_(0x000000b8), + LL(0x5f01aa9d,0x05a73aaf),LL(0xe5e08d95,0x18e919ab),LL(0x9fd8bcd3,0x5b386f06),LL(0xd6891589,0x0b5abd07),LL(0x07bc244a,0x339901b0),LL(0xc3693691,0x684cff05),LL(0x179dcef7,0xe69b7993),LL(0xb0125196,0x23db7308),L_(0x00000028), LL(0x6d0d0635,0xf88e8361),LL(0x0c66e267,0x2f9f93bf),LL(0x6d375b82,0x09656c0d),LL(0x6d5dc021,0x0c1fc16b),LL(0xba81db53,0x86bf6541),LL(0xb4caf94b,0x1a6d443c),LL(0x87f59d3c,0x0e10fe1a),LL(0xb72afcd6,0x8d7e1995),L_(0x00000008), + LL(0xcb74d13d,0x3c2640b0),LL(0xef7a7df4,0xf6850378),LL(0x2a55c175,0xed04fb24),LL(0x8b6f667a,0xa7e743ff),LL(0x93061f72,0xdf46d114),LL(0xdaa59d80,0x9d4ed766),LL(0x49c1e1f8,0x4b991a82),LL(0x2fe74922,0x2e41804d),L_(0x00000122), LL(0xa863fded,0xf37b7a2c),LL(0x140444d6,0x8f14ff32),LL(0xf54bdbb6,0x34a9f316),LL(0xde42c24e,0x9515de8e),LL(0x96a88eec,0xf1959aa3),LL(0xdcd09006,0x30671815),LL(0x058c4643,0xb6c6d2af),LL(0x65dac0ec,0xd1a5f578),L_(0x000001f4), + LL(0x50517e7c,0x0a5b8daa),LL(0xd4667616,0x8dcdbdbe),LL(0x16e7a7e7,0x1b0f5e9c),LL(0x9178e1fc,0x3cb9503e),LL(0xf2294c93,0xe6f78b36),LL(0x420442aa,0xfb2d6ec3),LL(0x01e17c7f,0x7d30fddc),LL(0x9c7293b8,0x36bed473),L_(0x00000128), LL(0x2ebfd049,0x4975943e),LL(0x84cf37b8,0xab2857b7),LL(0x645eaa02,0xe12a0d65),LL(0x3d30b0ac,0x267c93aa),LL(0x30be3eff,0x6fa69542),LL(0x903aa7ed,0x2cdb01e1),LL(0x311ae0c8,0xc5056d4c),LL(0x275f0bd3,0x94d54a98),L_(0x00000020), + LL(0x8cee0a0b,0x8c942dd7),LL(0x62d7fcf1,0xc893077d),LL(0x643ea78e,0xda471b28),LL(0x4e9d9fdb,0xfb673df7),LL(0xe64cf76a,0x51536a0a),LL(0xfccba343,0x5fc020fd),LL(0x279bce99,0xdee7d155),LL(0xb7f5fd72,0x68c90549),L_(0x0000012d), LL(0xd5aaeab9,0x4ecec5bb),LL(0xe6b0baf4,0x6a4bc82c),LL(0x18f9ed0f,0x93eb1856),LL(0xdc69912e,0x6f327c7d),LL(0x2be0e2b3,0x52fbc6f3),LL(0xe7e8bfdb,0x24b3acf0),LL(0xd81de1d4,0x62a84326),LL(0xe49b2810,0xc3912b42),L_(0x00000136), + LL(0xa4f2bbd9,0x551c0660),LL(0x2fe061f4,0x91d40c3d),LL(0xa2b6dd70,0x7cc4883d),LL(0xbd2bf743,0x58011dda),LL(0xaee852f2,0x71bdaba7),LL(0x39f463ce,0xbd7c195b),LL(0xfb33b881,0xede085fe),LL(0xbf7d695a,0xf8fa2471),L_(0x00000180), LL(0x83554b40,0xc96eeab9),LL(0x8a6f5a6d,0x07f71f13),LL(0xb29f154f,0x242d7a9e),LL(0xb707d1c7,0xf3c7f1af),LL(0x44aee842,0x0af2896b),LL(0x01f56999,0xc5c5b556),LL(0x1e36ed07,0xcca26786),LL(0x245430fa,0xa78eb214),L_(0x0000016e), + LL(0xa3dd17dc,0x0f4d9fba),LL(0xa4eddc14,0x9ebcb886),LL(0x420f1510,0x3c114b65),LL(0x4e6c23f6,0xba3fc70d),LL(0x5f6bd766,0x718ac792),LL(0x532bdc38,0x5c2b8f46),LL(0x03efcc7f,0x72c3a472),LL(0x5c2b9a45,0x9dc110c9),L_(0x000001f4), LL(0x58ea447d,0x1a6af2f7),LL(0xb5c3ea1c,0x8052b6c2),LL(0xab27ec8a,0xa3cff07f),LL(0x4c92575d,0x00fa9778),LL(0x4b43265d,0xff1e188a),LL(0x6c53a129,0xedc9e95e),LL(0xfcb7ddb8,0xfb7de911),LL(0xc1dd8dd6,0x6e53bbb5),L_(0x00000072), + LL(0xe68e2079,0x7daeaaab),LL(0x6eaee317,0x6e35398b),LL(0xf5c1267a,0xd01bbbbb),LL(0xa2b92ae2,0x3af1a4f4),LL(0x5cdfa4bf,0x5532088e),LL(0x638a58f5,0x2a0ec8ee),LL(0xbce5e7aa,0x79c3f50b),LL(0x46a16448,0x378387c2),L_(0x0000005e), LL(0x84e88e18,0x6769ee88),LL(0x8ec2984b,0x62fea90f),LL(0x1f9e47bd,0xe7522665),LL(0x3d9f9867,0x916cb049),LL(0x5be2cccb,0x1106987f),LL(0x71156e1c,0x7bad869c),LL(0x011576bd,0x945007a5),LL(0x38b76a6e,0xc18de41f),L_(0x000000ae), + LL(0xa84cca21,0x6ee5a7d6),LL(0x2831843a,0x58bef9ba),LL(0xe4f07223,0xd00147a3),LL(0xe1c40e6b,0xd10e53e0),LL(0xe7025e87,0x5d5f797a),LL(0xf0dd838b,0xe3b65b5f),LL(0x16931c80,0xd1ff0eb8),LL(0x42704c55,0xffc29bf2),L_(0x000000d4), LL(0xe3300a5d,0x279cdc4f),LL(0xed03963d,0x51c15682),LL(0x4a0c613e,0xc3d504ac),LL(0x6ef1db42,0xba61ce4e),LL(0xf8742eaf,0x2e74a6d4),LL(0x3efa3cf5,0xd53943f5),LL(0x201c12e1,0x253da54a),LL(0xa468664c,0xa35c5fd4),L_(0x00000012), + LL(0x9c6f67ac,0x687c5994),LL(0x7a229d23,0x8aa53c89),LL(0x118e3e78,0x080e973b),LL(0x15c794e3,0x5a91046f),LL(0x30c0f281,0x5ac04ed5),LL(0xfe35eb92,0x0c5478a4),LL(0xfe18e707,0x7d9c57a1),LL(0x4bf409ce,0xfcd17edd),L_(0x000000aa), LL(0x8174557a,0x3d2850fe),LL(0x23469475,0x2c45c37e),LL(0xc528ec7d,0xf346792a),LL(0xa36bdd41,0xca964be2),LL(0x11d67932,0x3fef3e79),LL(0xc72673a0,0xf0bc0101),LL(0x53d6358e,0xb9caaba6),LL(0x2bf874e2,0xbbcc3188),L_(0x0000005e), + LL(0x5993ad4f,0xf22d90c1),LL(0x24df4deb,0x449aa491),LL(0x37a9290e,0xd1657f10),LL(0xc6bbdf04,0x4ba3ee4b),LL(0x4075fc0e,0xe3e07500),LL(0xf134a6c8,0x7661f086),LL(0x2b3b9213,0xfd4c08de),LL(0x6f51f169,0xcae8fa7f),L_(0x00000104), LL(0xb2f528e4,0x9303da25),LL(0x1e8df3c5,0x6a92d1ac),LL(0xb326f2a6,0xafc6ed8c),LL(0x46ef258d,0xe7d3b364),LL(0x53fe6ba5,0x7b2d91d1),LL(0x669d23c2,0x150f6968),LL(0xc05b6015,0x230a0105),LL(0x376b2356,0x3df6108a),L_(0x00000172), + LL(0x05384e00,0x5490baa5),LL(0x148d7d84,0xae379993),LL(0xd948464c,0xac4f8ace),LL(0xddaf46fe,0x68d1b7c8),LL(0x6c39b672,0x10d4770e),LL(0x5967ac57,0x30a2c195),LL(0x2098b748,0x19047735),LL(0x858d17fe,0xe5071ced),L_(0x00000191), LL(0xb56551f3,0x44b4f003),LL(0x2a7e4193,0x767f8087),LL(0x1262c90a,0x91770fa4),LL(0x25315a97,0x4bc5606e),LL(0x696dd31a,0x6a665041),LL(0x7ec13746,0xbd01da6d),LL(0xac07c562,0x59de1e2c),LL(0x4bc4a92e,0x8f7d2999),L_(0x00000106), + LL(0x5f2d1510,0x37e7ccad),LL(0x0f93acce,0x909c5616),LL(0x3802c7e4,0x50e428ab),LL(0x3b424aa7,0xdee4af73),LL(0xc7e0291d,0x5bbec3a0),LL(0x3889c563,0x7854dc66),LL(0xe5ea7c0c,0x7701185f),LL(0x9cdf7704,0x0b88a64a),L_(0x000001ad), LL(0x6c0c71ab,0xe5481dba),LL(0xc168583c,0xbb4cbf39),LL(0xec84a971,0x16e6876e),LL(0xdde2891b,0x4940610a),LL(0x9e03560a,0x5406bed1),LL(0x992e1390,0x475b7757),LL(0xc2108310,0x7cc85b3f),LL(0xffbf072b,0x82170451),L_(0x000001eb), + LL(0xb05a9bf6,0x9d172698),LL(0x0dde3c4b,0xb8a86057),LL(0x4dad7e2a,0x376d08bb),LL(0x709cc4fe,0x2d396e50),LL(0xb38f91df,0x78138716),LL(0x6a45d957,0x7a064e9a),LL(0xeda47781,0xf79bf83f),LL(0x037062a4,0x2aff8e01),L_(0x000000a1), LL(0xbd0ea3b4,0x49748577),LL(0xa23c2d1e,0xdb9b468c),LL(0x79fc968b,0x9d5d5807),LL(0xb4f35908,0xae7478af),LL(0x7ffb5a37,0x6f6ac1a1),LL(0x0ad6d095,0x35d076fb),LL(0xcc73da49,0x4e896c83),LL(0xf0b38ddd,0xcd942654),L_(0x0000011f), + LL(0xcfb2b4bb,0xf362300f),LL(0x940a7fee,0x0c8996fe),LL(0xff7970e2,0x434f05d2),LL(0x3ed8edff,0xc3ed10ba),LL(0x5ebc5312,0xdee87d6d),LL(0xa445169b,0x12a8674e),LL(0xc4cceb87,0xd8da7725),LL(0x51c6dc7d,0xea8956f4),L_(0x000001af), LL(0x82d880a3,0x7d3585a5),LL(0xa5517351,0x13fe8bb4),LL(0x74f18cb5,0xe68912ee),LL(0x7950c8df,0x1512d84f),LL(0x2fbcbaca,0x7df13019),LL(0x0de4c1a1,0xe29f312a),LL(0xabb943b7,0x656e2b95),LL(0x7ae2e506,0xfc56ddff),L_(0x000000ab), + LL(0x8e947b1a,0xccd06bf8),LL(0xa717a0ac,0x6ceb8a0c),LL(0xb428946c,0x2bd2594e),LL(0x5c199265,0x80688236),LL(0x3a824141,0x74f2f352),LL(0xb634a60c,0x83ce2e27),LL(0xf4680f2b,0x33426806),LL(0x485e159f,0x18c76ca1),L_(0x00000137), LL(0xe0c9dd7b,0x1f63dec4),LL(0xd74bb74e,0x6f95ccb7),LL(0x4a5f0944,0xc75d0662),LL(0xd36f4555,0xd6e7583f),LL(0x23cc2b28,0x783fa303),LL(0x2c0c49bb,0x4f771001),LL(0x907cd3d6,0xac90f899),LL(0xe2c79e69,0x2cb352bd),L_(0x000000c1), +}, +/* digit=8 base_pwr=2^40 */ +{ + LL(0xe05ab80b,0x93bc2a7f),LL(0x46f49f7a,0xcc0b41a2),LL(0x91072db7,0xd30ac7ec),LL(0x53711f97,0x9de685e7),LL(0xef0acc16,0xb8a2ae9a),LL(0x6bded4c6,0x4497e3c4),LL(0x04789cdc,0xb2b2ea26),LL(0xfea082fe,0xe890cba4),L_(0x0000018f), LL(0x50768242,0x66edbc5c),LL(0xa472aca4,0x5ad15298),LL(0xa8066050,0x1b16ca97),LL(0x38b0c1cc,0xa2937bbf),LL(0x4d56c3dd,0x72545fac),LL(0x7e35494b,0x4b5790f0),LL(0x903e9ca7,0xb0c8bc40),LL(0xc4b43111,0x5923f9e9),L_(0x00000146), + LL(0xb73ff4e5,0xc18b52bf),LL(0x6410d489,0x28530a4d),LL(0xae318bb5,0x3f7d7028),LL(0xb534b71f,0x0b21b686),LL(0x1f599597,0xf01ea275),LL(0x663379dd,0x800a1322),LL(0x77c11915,0x6db4beae),LL(0xe3261c47,0xe89c22a1),L_(0x000000e8), LL(0xa7b1dfa3,0xfce16965),LL(0xd6426762,0x2e53d9af),LL(0x532ec13f,0x1e801ed5),LL(0xe9efe413,0x6963e2f9),LL(0x8aecc3c8,0x5f46e509),LL(0xb47801a3,0xc3d8faa2),LL(0x3b6f3406,0x349f37b4),LL(0xadb57971,0x49ef39f4),L_(0x000001d7), + LL(0x126ca585,0xf9dcaceb),LL(0x5a73dec3,0xd887946c),LL(0x0e0ffb30,0xe7e62831),LL(0x84126935,0xe074c83b),LL(0x158a7df5,0x18b04291),LL(0x08eaada2,0xfa20f72a),LL(0x40d05438,0x9aa8c4aa),LL(0x8405e6ac,0xb7559284),L_(0x00000192), LL(0x74e90dac,0x7981326c),LL(0xf1a037a8,0x13ab2cdb),LL(0x9887e290,0x98bd3d86),LL(0xfecffd65,0x3c803a95),LL(0x2fd8393b,0x3e4c5072),LL(0x129b699c,0x5e3c4e70),LL(0xfa72cdd6,0x65f24da0),LL(0x6c0ccbba,0x5325682f),L_(0x00000097), + LL(0xd21b98be,0x728d8231),LL(0xeede07ff,0x52ba3f46),LL(0xc57dc8bc,0xbbd28782),LL(0x49d96a93,0x9e0a7a0e),LL(0x49576560,0xe9fbe4aa),LL(0x79dbfb8a,0xcccb4c5f),LL(0xe1789960,0xd25ebfd5),LL(0x09b74da3,0x56df642b),L_(0x0000002e), LL(0xbe83a30e,0xd17057a2),LL(0xa759ce4d,0xab9d226b),LL(0x57744ef0,0xb7115a63),LL(0xdddc9ee2,0xd77f24c2),LL(0xdfee8900,0x2142ea1a),LL(0xa9d5346a,0x6d500f3f),LL(0xa84ecd7e,0x7a1527e7),LL(0xae35caeb,0x10e6262d),L_(0x00000081), + LL(0xaea7c3e5,0xcd457989),LL(0xbd6196a6,0xdd85ca16),LL(0x6f76c2cb,0xcfd847e9),LL(0xaa25840d,0x8ea001b3),LL(0x444e27ec,0xa898be24),LL(0x8a0c53dd,0xb3e4397d),LL(0xfa5f98a5,0x64ea9863),LL(0xbe8e1973,0x922c6bbf),L_(0x000001bd), LL(0x5084e379,0x0664b7db),LL(0x54735a19,0x990568c9),LL(0x7371d65a,0x52b4c902),LL(0x600bdaf0,0xc2cc9668),LL(0x4697299e,0xbadac668),LL(0xeb4949cf,0x33272f7c),LL(0xeda14ca0,0x3989fbe1),LL(0xd9927092,0x0f1714c9),L_(0x000000a8), + LL(0xd2d2e356,0x00da9fad),LL(0x39d9c06e,0x69b9bebe),LL(0x7878b1c2,0x50e16aa4),LL(0xa0545c03,0x04f7fb31),LL(0x5d57a4d6,0xd233dc43),LL(0x629977c5,0x87e54a59),LL(0xaa747e53,0x0cca577c),LL(0x80698068,0x3aa24734),L_(0x0000007f), LL(0xff177c1c,0xf46ecf72),LL(0x87d84398,0x5b5e3ea0),LL(0xf8e3dbad,0xc29bdf29),LL(0xc86793b7,0x8b4ad4a2),LL(0x337e0dd4,0x34cf9d25),LL(0xc858ea72,0xb282be01),LL(0xe90a676b,0x7590c7bf),LL(0x7d306f50,0x155053c4),L_(0x00000115), + LL(0xbb970cd1,0xfa311b42),LL(0xa08e6727,0x609fc56f),LL(0x6285b2f7,0x07ce1a3e),LL(0xe94807c3,0xc9c1df6d),LL(0x19a317d8,0xd70b9796),LL(0x052a3379,0x870efdde),LL(0xaa7d20b3,0x8f7406db),LL(0xbb6e443d,0x511beafe),L_(0x00000155), LL(0x0634dd62,0xd62e82c9),LL(0x9d51e499,0x9995a224),LL(0x615ca9d8,0xd99162d9),LL(0xab897ac7,0x51034000),LL(0x24f35e95,0xb70ca9d9),LL(0x853be7df,0xff11b526),LL(0x38dc8c8a,0x463b8a66),LL(0x3331fb01,0xb55e7404),L_(0x000000e7), + LL(0xa03485a9,0x508d4f13),LL(0xdd3ccf18,0xf25524c5),LL(0xd8ab1776,0xe0bf0c9e),LL(0x017a54a5,0x1226e24a),LL(0xb9626be8,0xb0e5b1ec),LL(0x8b7b3bc5,0xf24c6acb),LL(0x14da0130,0x46736054),LL(0x7db2f1d9,0x73af8b9c),L_(0x000001ae), LL(0x806025a6,0xb11a4baf),LL(0xf7b99ef6,0x1e9bb68c),LL(0xb7054990,0xa5ca0071),LL(0xe57cc5d4,0x55009f7b),LL(0x501abab5,0x08dbaab6),LL(0x2d17b21b,0xcde35c58),LL(0x9921c7ba,0x9991c48d),LL(0x3f13fb4b,0xc6f664c9),L_(0x00000172), + LL(0x63f229e4,0xaf8f0fbd),LL(0x3fffd972,0x513637b7),LL(0xa381ede6,0x3a907a7b),LL(0xef4d7386,0xdec53a87),LL(0xfaf3ac39,0x6072c595),LL(0x7077416d,0x25742340),LL(0x8d4d4598,0x0272fbab),LL(0xc3dce550,0x44d3c246),L_(0x0000013a), LL(0x95e9dda0,0x8a45d7f7),LL(0xf5989e00,0xa25dc323),LL(0x31cb7128,0xd19e79bc),LL(0x87f0b5cb,0xf782a69b),LL(0x62e18e62,0x4a3bc664),LL(0xacc62f0f,0x8a21efa3),LL(0x855aaaab,0x5dc442d8),LL(0x895a7f3a,0x7fccf9f6),L_(0x0000017f), + LL(0x6c85234b,0xf6a194a7),LL(0x21638407,0x8d081ace),LL(0xe465a985,0x1e6d4f3f),LL(0x596aa1dd,0x800bb059),LL(0xf63247cf,0x88ecdd17),LL(0xd80d0066,0xd6196a9e),LL(0x359a8606,0x6d1c0b4e),LL(0xf12ac0e0,0x1f003c05),L_(0x000000e8), LL(0xfd00b6d4,0xe591e392),LL(0x88389649,0x09f83a93),LL(0x2b4134e0,0x9d2fd6ac),LL(0x3ada50c0,0xd488e638),LL(0xa2f5e7c7,0x6ae6d5dd),LL(0xece41bdd,0x626ed9b1),LL(0x83fc37eb,0x0ec94ba6),LL(0x390a5c6f,0xd316539d),L_(0x00000034), + LL(0x42827af1,0xf8cf81f2),LL(0xdd03c701,0xce67a0a5),LL(0x957637c9,0x4af6b68e),LL(0x49c7193a,0x2b716eb7),LL(0xa9f1106e,0x04a50d86),LL(0x5cdc8e58,0x29fe3e8a),LL(0x404173b6,0x2217e337),LL(0x8d0fe7b4,0x41f85927),L_(0x000000ee), LL(0x6c97abbd,0xf0033298),LL(0x7982223c,0xeed36f1b),LL(0xc078a101,0xc8a52b8f),LL(0x54b52769,0xfd843c12),LL(0x0c71b06d,0xdaa31445),LL(0xd139607d,0x996c457f),LL(0x3373eded,0x0d6abc25),LL(0x616b57db,0x27a4f9a0),L_(0x00000005), + LL(0x1be04a5c,0x24d46da3),LL(0xc28ba5d6,0x84ca0be2),LL(0x69aff480,0xb7d623cf),LL(0xeee1ba2a,0xc4a065a8),LL(0xe236787f,0xd893b3f3),LL(0xaa351426,0x2106fcf4),LL(0xc4d98be5,0xf2dfc4d7),LL(0x534e82e2,0x4f43180c),L_(0x000000bf), LL(0x0a626913,0xea92fe7d),LL(0x3cd0971d,0xb9b4d4bf),LL(0x5fa502b8,0x56e42bc0),LL(0x2f95fd57,0x9a55a6ac),LL(0xefd75261,0x9c01cac4),LL(0xc54d4200,0x8b9c411f),LL(0x9a2d86c0,0x84f22245),LL(0x0123f4e9,0x924fe183),L_(0x00000190), + LL(0x59d5704e,0x5adfc431),LL(0xed2fd11e,0x1a785308),LL(0xb3ad4ad2,0x534b1813),LL(0x19e08445,0x77328159),LL(0x557af465,0xcd28509e),LL(0x114e6813,0x908aacef),LL(0xdd6f9e0a,0xea30d82c),LL(0x5aec37e4,0x56efd94a),L_(0x00000184), LL(0x9ccb322c,0x9a808c1f),LL(0x8215d192,0x00e65251),LL(0x2e216a64,0x8be89e79),LL(0xa21b58aa,0x1bae586d),LL(0xde6dc431,0x6074af45),LL(0xd9ffe269,0x144f7409),LL(0x7968f9ca,0x4c70bef4),LL(0x057ee0b0,0x464dfc55),L_(0x00000158), + LL(0x42555bd7,0xda8f0d55),LL(0xbc3cfc53,0xf7a0b6a9),LL(0xc3851a8d,0xd221e3bc),LL(0x3b6631e9,0x73e218ec),LL(0xd802d5a9,0xbb393674),LL(0x357ad609,0x17e839e5),LL(0x26a2911a,0xfd4ff33d),LL(0xa9163042,0x40c85178),L_(0x00000088), LL(0xd26e52f2,0xebbb0dce),LL(0x628d1685,0xc4b138ed),LL(0xad6058b9,0x1ab4e65c),LL(0xd77f3507,0xa315e387),LL(0x01e25773,0xc1c7fc22),LL(0x5f337f59,0x9dd402d9),LL(0xc4922f4d,0x8947a84e),LL(0x52e76d6f,0x83ef2457),L_(0x0000008b), + LL(0xd7a5547f,0x67dd4533),LL(0xed953e34,0x0ffa9336),LL(0x4fc44042,0xb44d3a69),LL(0x0c1288b1,0x7f745c6a),LL(0x0c5f14a6,0x345f8ac2),LL(0x765ee067,0xcfed50e8),LL(0x659b1874,0x5ef0443b),LL(0x26abda6a,0x894afeee),L_(0x00000163), LL(0x6f34576d,0xabe2ed4e),LL(0x46dcead8,0x196272e0),LL(0x64053114,0x13a8b18e),LL(0xbcb0e703,0xf9b6c7a1),LL(0xaecaa246,0xb17e245a),LL(0xd0c5c4d7,0xce6786b6),LL(0x01f4866b,0x12c94128),LL(0xea713e45,0x75975359),L_(0x00000004), + LL(0x15f3e5c9,0xb900e09c),LL(0xf45b409f,0x7837bf97),LL(0xff4a0108,0x2bcafcbc),LL(0x6b8d204b,0x0da165ec),LL(0x8423a60a,0xb1171697),LL(0xf8295351,0x3eb1f2f7),LL(0x1f58e2d1,0x2b669228),LL(0xbbed8459,0x5f9819ae),L_(0x0000005f), LL(0xd668278e,0x7b7ea077),LL(0x53ee2ff7,0x5b359b96),LL(0x98e8334b,0x87baabe1),LL(0x85a52104,0x95a5886c),LL(0xc237881a,0x809649f4),LL(0x7f95c6f6,0xd3395612),LL(0xed6c6419,0x657d29fa),LL(0xa5be49aa,0x7ae0b376),L_(0x000001a6), +}, +/* digit=9 base_pwr=2^45 */ +{ + LL(0xcbbde736,0x45fb32ba),LL(0x8a721e35,0x5c4674f0),LL(0x584020b4,0x84a774fc),LL(0xadafd3a2,0x477afffe),LL(0x266e1004,0xd6a2c4ec),LL(0x326c6652,0x428066dc),LL(0x0b3a65b9,0x4c7d5c77),LL(0xe355b810,0x4b6af237),L_(0x0000018c), LL(0xca1cffd4,0x1c0b97b7),LL(0xde135822,0xcc7ac435),LL(0x876cab38,0x8f30b09e),LL(0xec654cdc,0xcb3a4f5a),LL(0x26a9da0c,0xb2ac30ca),LL(0x8e2a6fa3,0x77ee1103),LL(0x545c20a5,0xf50fb144),LL(0x97bff8e2,0x58359a6d),L_(0x00000131), + LL(0xcf89d246,0xb2c8ba9c),LL(0xddf8d1b9,0x7f24e874),LL(0x27291ffd,0x563287c7),LL(0xd028bd9d,0xd01bdb48),LL(0x3b0c1265,0x71b99b97),LL(0x618319b9,0xf686050d),LL(0x8420d531,0xc411c3a3),LL(0xaed7c201,0x468eb84c),L_(0x00000197), LL(0x5bf609a2,0xf6eb2fc0),LL(0x1dab9da1,0xc275b73e),LL(0x49847c3a,0x54d322f9),LL(0xf0578805,0xdd0cd2b7),LL(0x4958eafe,0x185bb3e7),LL(0xd9061a48,0x5c6dfcd8),LL(0xf9ac370d,0xa0217866),LL(0xf54cb188,0xa132c3b5),L_(0x0000017a), + LL(0x9e0b2bbd,0xf197825d),LL(0x1340276a,0x4bbcc96d),LL(0xd82fe632,0xcad6233b),LL(0xc290475e,0x0cd8d04a),LL(0x738cce9a,0x8e8e067d),LL(0xaa038ad0,0xd83e4317),LL(0xa7ce55aa,0xd5e91f49),LL(0x856a1887,0x5efeae92),L_(0x00000013), LL(0x035b085a,0x9bfa0b6a),LL(0x3d153ead,0xef7bc585),LL(0xca7f6fb4,0x0b798e2a),LL(0xf8abfbb3,0x53595cf1),LL(0x79182066,0x1774e7d1),LL(0x862d3928,0x8b4548df),LL(0xdb1e4086,0x6e38fc52),LL(0x72153b33,0xe2e4b80e),L_(0x0000012c), + LL(0xdc232332,0xf5595043),LL(0xe1b752f3,0x1b9318d9),LL(0xf6e2b364,0x5c02bb70),LL(0x38d64e0f,0x9d8f2870),LL(0x07542416,0xa62f3a1b),LL(0x3b8c6755,0xd59701bf),LL(0x2b642127,0x20fbe8ba),LL(0xfac17f0c,0x3410177d),L_(0x00000126), LL(0x0c65efbc,0x2b08cc56),LL(0x70680750,0xe532cef6),LL(0x7a1e8980,0x29a4a8b8),LL(0x3b679637,0xcb3a4f19),LL(0x0043db7c,0x92e07ae8),LL(0x346fea83,0x0da35be0),LL(0xef33f7a0,0xcb41f4e9),LL(0x271ea778,0xbb760e77),L_(0x0000016e), + LL(0xe8812fbf,0x120e5ac8),LL(0xcad14e90,0xd45b7941),LL(0x130b3936,0x78bbd634),LL(0x3839fe90,0x8f94ae22),LL(0xfb2c2b29,0xbd4d9761),LL(0xb2caaa91,0xf6e513d3),LL(0x37bd3dff,0xa0f24baa),LL(0x9dd2846a,0x1d27a8db),L_(0x000000ce), LL(0x62a47784,0xd4e2cdab),LL(0x8296eec9,0xee13214a),LL(0xce1e6780,0x6fed4902),LL(0x8ec28ea6,0x28576525),LL(0xa9bf0652,0x0afbfe7d),LL(0x0c66edcd,0x9e743eb7),LL(0xc8ec4a8a,0x64589360),LL(0x09bf2d23,0x7a6453a2),L_(0x0000017a), + LL(0x91c1326b,0x4d44bd26),LL(0x5e7c9a8b,0xfa441738),LL(0x3f4fd525,0x8cdf278d),LL(0x5b1fa4df,0x60600772),LL(0xb7e79779,0x15388443),LL(0x6b7719f4,0xd7a3aeca),LL(0x17dd158d,0x02441c0d),LL(0x3d070ec1,0xd5eb5d02),L_(0x000000c9), LL(0x84252a63,0x0ab898cb),LL(0x01117e64,0xee325365),LL(0x6f680374,0xbc1ae420),LL(0xdaebee10,0x98a23bbf),LL(0xfec8e51b,0xb59efdf3),LL(0xbbf08b12,0xa18137ff),LL(0x1532459f,0x04b7fdbe),LL(0x60238519,0x37b3447b),L_(0x0000000e), + LL(0x2c315da2,0x6b53a82a),LL(0xad2c8749,0x5a5a47d3),LL(0x75f76d03,0x60558c44),LL(0x6ecf38ff,0x957fd8a0),LL(0x7695311e,0xcd47da64),LL(0x215ee9fe,0x35b22e22),LL(0x4796f4b7,0x949a56db),LL(0xf62c912e,0x74debc0c),L_(0x00000191), LL(0x55d8aab4,0x9bd8df8c),LL(0x203b317a,0x637e055c),LL(0x03c45bfe,0x90fbadef),LL(0x1132b50f,0xaf36e7bf),LL(0x20a98c58,0x4f36088b),LL(0xdebbd429,0xcbb98ba8),LL(0x391e4230,0x3091f3e1),LL(0xb3356938,0xd86355bd),L_(0x00000072), + LL(0xc1f9460e,0xf79ba658),LL(0xb48e1df3,0x3eb15b18),LL(0x5fc03a10,0x3bed592a),LL(0x3591ad26,0x127b78a3),LL(0x07e9d80a,0xc0337c7b),LL(0x349dd74f,0x364ed2a0),LL(0xb1a807c5,0x588d4203),LL(0xecd92cca,0x772a1716),L_(0x0000019e), LL(0xf66f295e,0xf6fc1df3),LL(0x42d25980,0x8922f157),LL(0x36f0fdb0,0xa583206a),LL(0x8cc1fe47,0xc73f8816),LL(0x1d279801,0xe1b77767),LL(0x7ac8979c,0x3dba6831),LL(0xa98b4836,0x60d40152),LL(0xc7f36b74,0xc3d46c62),L_(0x000001a1), + LL(0x3a2954f2,0xed4a0395),LL(0x99b5cc1c,0x4cddc23e),LL(0x1d30267a,0x16bee440),LL(0xcd4130db,0x553abd41),LL(0x6652be0e,0x6e659595),LL(0x22061ff2,0xf0c20235),LL(0x72c720f6,0x077f6daf),LL(0x8079b1de,0x1ad9ac77),L_(0x00000151), LL(0x3b6a7cb2,0x6701fea0),LL(0xdd8e0cb1,0x5849b249),LL(0xf395a61b,0xb92466c3),LL(0xc2b702c7,0x77432a31),LL(0xbd7899d3,0x28b4ebc6),LL(0x307f0a10,0x0b06f919),LL(0x5c8246fb,0x7154af20),LL(0x8f032be2,0xc88de5c5),L_(0x00000009), + LL(0x30655c0a,0x344eafa5),LL(0x0a16f77b,0x724f29ca),LL(0xdbafe962,0x94bbb419),LL(0x30985479,0x2b2c87d2),LL(0x3775b2ba,0xe0e3814b),LL(0xbd366c77,0x1130e80c),LL(0x7b644025,0xf10ea539),LL(0xe1da2161,0xf66677b2),L_(0x00000015), LL(0x08673ef7,0x11454e50),LL(0xd8ab26fa,0x45948446),LL(0x2a4b8bd4,0x35518731),LL(0x34c59cba,0xcc005baf),LL(0xbd4d3f49,0x06c483a0),LL(0xa3e5d238,0xd77da187),LL(0xc4657e79,0xa31fff1d),LL(0x33918629,0x0e898785),L_(0x000001c6), + LL(0x48a7aa5d,0xd39844b5),LL(0xc0ae95fc,0x0cd04d32),LL(0x608fd1bf,0x2b33bcf9),LL(0x8e195302,0x3567e13c),LL(0xb9784d4d,0x6f12914d),LL(0xf39a6a6c,0xf4d361ba),LL(0xcf170781,0x366e62a5),LL(0x70b10e90,0xa3bce706),L_(0x000001dc), LL(0x4172b25a,0x65a7fa0b),LL(0xdf710618,0x93abe742),LL(0x7805a257,0x738295fd),LL(0x76e1d4b4,0xcb5b0f15),LL(0xc121708d,0x716ee171),LL(0x14725b57,0xd2227241),LL(0x2e484d37,0x34400369),LL(0xef0bb7f5,0xebdf59e5),L_(0x00000105), + LL(0xf6fa11c3,0xf35d7da4),LL(0x0ee635c0,0x6c91936e),LL(0xdd72a103,0xa9f8eae7),LL(0x2a073b1f,0xff539491),LL(0x6c35942b,0x0a881a03),LL(0x35498b7c,0x67e4af9a),LL(0x59bde411,0xf903d1e5),LL(0x517835ca,0xf0b93b5b),L_(0x00000064), LL(0x3a244f62,0x079d614c),LL(0x550a47c5,0x1c8515d3),LL(0x1f9595ac,0x1557c55d),LL(0xf301c894,0xb5548c2e),LL(0xecc6608b,0x6ed92475),LL(0xf17244f1,0x9b9d35aa),LL(0x9b6083ca,0x82abcca4),LL(0x902eead4,0x45a99fbc),L_(0x000001a2), + LL(0x73c00a9e,0xc92a2f72),LL(0xb4d59736,0x46a97747),LL(0xec9ee773,0x92e9e427),LL(0x54eed174,0xe62769e0),LL(0xb25c6252,0x26eca3d7),LL(0xb5598a2e,0x72728c2b),LL(0x73ee8036,0x6cefdf35),LL(0x4ee8ce4c,0x700d3d8f),L_(0x000001d7), LL(0x35089629,0x2ffb5bc1),LL(0xb8175b8e,0xd9c451df),LL(0xbdb5cc88,0x9445c144),LL(0x846b2eaf,0x92957da0),LL(0x5ff2e582,0x2da50816),LL(0xe7cc1a15,0x4dc70abf),LL(0xe4999b07,0x24220cc0),LL(0x1b3556b4,0xb4413c1d),L_(0x000000e6), + LL(0x8929ee7c,0xb5ee8957),LL(0x45878f54,0x112fbfa1),LL(0x1879f721,0xe9f0dfae),LL(0xb007f421,0xf113817d),LL(0xeb000fbc,0x35d8e979),LL(0x206151d8,0x0bf9caf2),LL(0x258ab3b7,0x6e8e8e3a),LL(0x92a042db,0x7a4dc496),L_(0x000000da), LL(0x28821f85,0x3d64ea94),LL(0x7360c36c,0xbf2b13c7),LL(0xfb77c37c,0x73884c74),LL(0x65a78a55,0x5d8600a0),LL(0x888762bf,0x77475414),LL(0xc8ba0daf,0x975e6be1),LL(0x59f8b668,0x14cf6707),LL(0x185c7c67,0xfef650be),L_(0x00000023), + LL(0xa52fd88f,0x14dc97d9),LL(0xa3e0a482,0xe962fe1d),LL(0x44364f6d,0x19480b73),LL(0x9ffa10f7,0x28fc88ac),LL(0x7993eaa2,0x8a5db808),LL(0xd4bb9db5,0x4464dfad),LL(0x9088a081,0x903605db),LL(0x86f98ca4,0x87bd4fc7),L_(0x000001cc), LL(0x6477a8aa,0xba5ec771),LL(0xec2c3e51,0x3078a6cd),LL(0x1ad83e79,0x66717c17),LL(0xad871d3e,0x8530527c),LL(0x0e3f9442,0x92315ca1),LL(0x49c67cb7,0x2fc5cd79),LL(0x4eb1ba39,0x256788a6),LL(0x10b0e6f6,0xb9cd18a5),L_(0x0000008b), + LL(0xe7fab2a1,0xd61fb046),LL(0x90213473,0x4f9db0e1),LL(0xcbb6e9b8,0x36fcff78),LL(0x6aa8fb8a,0x7cd5e9d1),LL(0x337a00c4,0x2c2601e9),LL(0xfe8445d7,0xbbab713e),LL(0x0681fd15,0x0b2dd233),LL(0x2151cff9,0x00ab444b),L_(0x00000049), LL(0x0ca8289c,0x06de9a88),LL(0xcb8ede52,0x209abe3b),LL(0xe1369e32,0xb711e224),LL(0x53136516,0x533569db),LL(0x59d96525,0x5419656e),LL(0xf2d68025,0x326eee21),LL(0xd59bb004,0x073cca71),LL(0x1cbb722c,0xaa784f93),L_(0x00000066), +}, +/* digit=10 base_pwr=2^50 */ +{ + LL(0x3b1d2404,0x51931359),LL(0x672b4b0a,0x14acc3b3),LL(0x78ea42e6,0x22fe0a9a),LL(0xe72784cd,0xc20faf43),LL(0x8f9c3ea6,0x5e49f303),LL(0x4c50987c,0x12d1fb91),LL(0x0c76e9b9,0x96a89b90),LL(0x74dc2b7b,0x238b29a0),L_(0x000000de), LL(0x7031f728,0xca68ea37),LL(0x06adb168,0x5ecbae96),LL(0xe58dde88,0x4d422e92),LL(0xeba17742,0xa609937c),LL(0x1451998a,0x8f30fc81),LL(0x9eba807b,0xa724c9f9),LL(0x200db6e7,0x651c126e),LL(0xc9db2dc7,0xb58e38f0),L_(0x0000009c), + LL(0x81dfb8c7,0x71e8870b),LL(0x8ea654a6,0xa23dd690),LL(0xc3eb3660,0x673dbdf6),LL(0xa5ddaf70,0x9bbf5d38),LL(0x1e7af5c1,0x0fe1371d),LL(0xcc1eff61,0x1572e30b),LL(0x1308bdd3,0x20ce33cf),LL(0xc60db70b,0x6ab6b3ed),L_(0x00000003), LL(0xd4f22a67,0xae357b86),LL(0x3ce6e16b,0x94e06b89),LL(0xa3849b8d,0xb6058ad8),LL(0xacee1675,0x6add0f99),LL(0x39df12ed,0x43cd380c),LL(0x5c645ff1,0x0481e233),LL(0x94a0f618,0xc84b4bf9),LL(0x805a52a4,0x49a710f4),L_(0x00000098), + LL(0x577d0472,0x99d73698),LL(0x68ebe9cf,0x7fcd4216),LL(0xc625e525,0x4922e8d6),LL(0xe579cc68,0xe272485c),LL(0x58eef2df,0x1aedb9d5),LL(0x6bba0e47,0xf69dbcc2),LL(0x6afac0cf,0xd8f85c14),LL(0x4dfdd56a,0xc7e717e4),L_(0x000000f2), LL(0xe505ba10,0x7d4e9483),LL(0x056cb0ab,0x8a2580c9),LL(0xee676f9d,0x031109c0),LL(0x0e2ecd89,0x784c6d24),LL(0xedf27261,0xeac131cc),LL(0xdb6b9edd,0x9428ee22),LL(0xf59f93aa,0x90347b1c),LL(0xd59691aa,0xcb3849d1),L_(0x0000014c), + LL(0x6fba378b,0xc42ea299),LL(0x33ae1a32,0xd0ddacf9),LL(0xb43b79b9,0x30561bd9),LL(0x0ad2636c,0x12241370),LL(0xd830def9,0x85a779a0),LL(0xda5f6561,0x28b8580f),LL(0x7e785d86,0x8bafa8c6),LL(0x48ce8b18,0xc75df63d),L_(0x000000d2), LL(0xd7e01b7a,0x5a90afd2),LL(0xc72b304c,0x1b4b2e57),LL(0x40d7dec2,0xe0f45d07),LL(0x3eb94cfd,0xaabbfa71),LL(0xae1b3f10,0x37fa8b4f),LL(0xb080d24d,0x6f6447d2),LL(0x142abdb3,0x20453501),LL(0xfd470df7,0x76e433f8),L_(0x0000007e), + LL(0xb1a06043,0x1f809e00),LL(0x9eb8b4e3,0x91e1d4a1),LL(0xa399e369,0x9b1aa8fe),LL(0xf15f9651,0x80a83b4c),LL(0xea343c7b,0x1c2fb2b2),LL(0xc40680cb,0x4d003567),LL(0xe7a338fc,0x65bc46fe),LL(0x4519127e,0x3a269638),L_(0x00000030), LL(0x000ab675,0x8811cc38),LL(0xcd6861c0,0xd2e2abb7),LL(0xccb0e7ea,0xce5461a0),LL(0x8c05450d,0x28a458ea),LL(0xeff9ba00,0x51ce8e58),LL(0x3e543072,0x41ebfad1),LL(0xa43fc5d6,0x2acf8a4f),LL(0xc0d63fae,0xd16efc25),L_(0x00000094), + LL(0x33854823,0x0c31be44),LL(0x25c21bb4,0x171af22e),LL(0x91e02b25,0xa0756859),LL(0x87db4292,0xd07cf03c),LL(0xd52aff6a,0xf2199b54),LL(0x476b6c0b,0x4c50edf5),LL(0x1bd465b7,0xdeb36507),LL(0x9e6301c1,0x957f58a4),L_(0x00000154), LL(0x8e331516,0xf73b742f),LL(0x5e9d3550,0x6b92e894),LL(0x419be8da,0xd785e55e),LL(0x95d412eb,0x6018e5bc),LL(0xadbd35ab,0x079447ae),LL(0x5f3359bd,0x21b9bd0a),LL(0xc4db3315,0x5774802e),LL(0x4978d805,0x18a2368a),L_(0x00000167), + LL(0x6256586d,0xd53479d6),LL(0x5c26f234,0x2d429971),LL(0x107f7e92,0x8c689924),LL(0x334d8841,0x2a5fa3eb),LL(0xe5ebe430,0xc519e325),LL(0xe8291ba7,0xf2242ce5),LL(0x8a0c19be,0x20419cdc),LL(0x804a91ee,0x70dcad32),L_(0x0000017f), LL(0x96b0c9d5,0x78a46e2e),LL(0x18b297a6,0x21fc2dc8),LL(0x3ba036e4,0x30517e2f),LL(0x7a021835,0x49f89605),LL(0x19710681,0x84156ac1),LL(0xd61e5109,0x05c42243),LL(0x31ade9f9,0x7b661ab8),LL(0x83c25735,0x22eb398b),L_(0x000001df), + LL(0x682dd914,0xad081cbd),LL(0x1433b543,0x88d8cd2c),LL(0x94641d24,0x2da0394f),LL(0xd8e36e70,0x48288ca4),LL(0x461fe782,0xa112c8a6),LL(0x6f063613,0xb8624a48),LL(0x77efb66b,0x511d90ff),LL(0x016e8d41,0xce809694),L_(0x000001a1), LL(0x1fc39355,0x36feced9),LL(0x3ecdac71,0x921f42e8),LL(0xee8e2857,0xe82b293d),LL(0x2c3ef9bb,0x182b25ab),LL(0xac32f4bd,0x297ad819),LL(0x74b598de,0xdd15916b),LL(0xd5e666a5,0x51456a24),LL(0x447be0b1,0x4dc25c5c),L_(0x00000179), + LL(0xf9b2eade,0xb1762839),LL(0xba507049,0xed038901),LL(0xdcef710b,0x4b349ec8),LL(0x2489f866,0x37b4ec4e),LL(0x991460d8,0x94e1cecc),LL(0xbf2a63d2,0x33d105a8),LL(0xc7e7415f,0xbf883b5c),LL(0x268241cd,0x2f565fda),L_(0x00000135), LL(0x38741429,0x8bf3904a),LL(0x8e823f54,0x0c2d77f6),LL(0x06de5eb9,0x2c00d580),LL(0x89b51b4d,0x41ce4b94),LL(0x794caf3e,0x177cd9b2),LL(0x7c62716f,0xfe0ae88a),LL(0xb7e50074,0x4d023907),LL(0x49a489fb,0x545f8faa),L_(0x000000b9), + LL(0xbcdfb42e,0x05813dea),LL(0x62545e9a,0xc84039c9),LL(0x70606ec0,0xcdf6907b),LL(0xec7e8e9b,0x2e4e87c0),LL(0x9d6e053f,0xffa08764),LL(0x22a2e351,0xe5b305f1),LL(0x95345fe3,0x65c90711),LL(0x4f24c950,0x139d472c),L_(0x00000144), LL(0xa58aa810,0x9eeab46c),LL(0xac8b67e8,0x267bcc83),LL(0x702c21e5,0x0dac9b29),LL(0xce390fe4,0x60429071),LL(0x6ef71376,0x4a80e0d7),LL(0x47100322,0x0ef6a473),LL(0xdc625a85,0x759024e7),LL(0xea01db5d,0x1e4722a2),L_(0x000000a2), + LL(0x4965f4fd,0x5678bfca),LL(0xb048b1b3,0x5bacba68),LL(0xf518ba8c,0x1626088b),LL(0x7054f024,0xa686c886),LL(0x933a9118,0xbb623954),LL(0x1c3c471e,0xc4da98f0),LL(0xf1b8c9b1,0xa0619dd5),LL(0xaeebf226,0x24b28dc5),L_(0x00000108), LL(0xad3fede9,0x47814012),LL(0x49c3a34f,0x1cf06d59),LL(0xfcdcc300,0x6d4a798d),LL(0xe86df54b,0xa1a4dd57),LL(0x1534b80c,0xaf606d64),LL(0xbdfde769,0xde1cbaa0),LL(0x649c3a2a,0xbf6c9950),LL(0x763574e1,0xaaf6f737),L_(0x00000139), + LL(0xc23c5aa2,0xf687c377),LL(0xca314119,0x3eacfd33),LL(0x2512d094,0x9c0e1850),LL(0xe55f9fd6,0xc3c6ea7e),LL(0xc20685b7,0x66291556),LL(0x4868b07c,0xb5895337),LL(0xf9f339d7,0x9238a109),LL(0x75d6855b,0xac6af37f),L_(0x00000091), LL(0xf49812cf,0x6eb5d5b2),LL(0xe7603bff,0xf7552855),LL(0x8f73b087,0xc19b7320),LL(0xe8f5c0ad,0x55df5442),LL(0xb6aeabd3,0x3a4b8876),LL(0x8dc2b22b,0xf8bca737),LL(0x26f89265,0x3dbb040c),LL(0xfb6645f2,0xb09ab1bb),L_(0x00000138), + LL(0xe9c4fede,0x9e2dc755),LL(0xea8b03a8,0x5618c490),LL(0xd5d01455,0x01a7a348),LL(0x9622ab8a,0xa6b5c4df),LL(0x9adea853,0x303519de),LL(0xa9b99058,0xb3d0934b),LL(0x0fbd9ea2,0x2cdee030),LL(0xe856d6fd,0xa351d2a2),L_(0x0000008b), LL(0x6ac4ec77,0xea40ff5a),LL(0x4859e663,0x906f2d7a),LL(0x71904b77,0x411180a4),LL(0xbd7ebd35,0xe50b9460),LL(0x0ec190c2,0x2e7f4d73),LL(0x4c9e4aac,0x76a98ae4),LL(0x4323017a,0x7f0e29c1),LL(0x22ea8f39,0x31c71758),L_(0x000001e3), + LL(0x0402f8c8,0xcf98bee6),LL(0x54f3df55,0x672d0a69),LL(0xe9759866,0x1addc9d4),LL(0x9c17b622,0x6c819f7f),LL(0xc42650ec,0x6b1209c4),LL(0x6a1aa1b9,0x2b341fef),LL(0xdbcf91bf,0xc99d2b99),LL(0xed76cdf2,0x27467cd2),L_(0x000000ee), LL(0x6ea222ca,0x64754f19),LL(0xc40615cf,0xb8f1e46f),LL(0x9a8d5587,0x540f1fa5),LL(0x804f7dd7,0x21752096),LL(0x2c95388c,0x9444e15c),LL(0x133319bb,0xb1d5a817),LL(0x29552f4e,0x79fc1cba),LL(0x93730e70,0xcae8a131),L_(0x0000011a), + LL(0x2b9c7277,0x88863b50),LL(0xbfdb676e,0xb9545954),LL(0xfcfc0194,0xe74bedd7),LL(0x888694d2,0xe59a14c4),LL(0x236680d1,0x4cd674c0),LL(0xacdf13c1,0x52151e94),LL(0xcdbecfcb,0x6a28bc34),LL(0x641d77e2,0x6293af48),L_(0x000000e3), LL(0xa8fee6f0,0x2e8f361b),LL(0x67004141,0x39634681),LL(0x5db1f02f,0xf975c602),LL(0xe645bd3a,0x8b39a53a),LL(0xfafccb60,0xa58e37f9),LL(0x33ab2637,0xcf611fd4),LL(0x8b8cc6bb,0xe7f89e7f),LL(0x28eb10f6,0x5f527820),L_(0x000001d1), + LL(0x8fcc2459,0x9411ca0d),LL(0x92267e14,0xb385c8ea),LL(0xbbfcc2ab,0xbfd56d29),LL(0x34b29656,0x5f2180a7),LL(0x06f72807,0x6dc34000),LL(0x02310437,0x854af754),LL(0x1bae73e6,0xbc753242),LL(0x06a8d2dd,0x11770a34),L_(0x000001d7), LL(0xb8d63658,0x14476594),LL(0xeb8cb497,0x6ba99aed),LL(0xc86324ad,0xc49863ca),LL(0x8a316428,0x2e5cfc3d),LL(0xcb62d82a,0x79adc3e0),LL(0x9e5f3fda,0xcaddeff7),LL(0xb4f990b6,0xae15a98e),LL(0xb9b0e410,0xedf394c7),L_(0x0000000a), +}, +/* digit=11 base_pwr=2^55 */ +{ + LL(0xf2938b13,0x20c391c2),LL(0x96d1c5f4,0xeaef76b3),LL(0x6bb17f5e,0x7feb16a1),LL(0x3f16a57b,0xcc801552),LL(0x4aadf126,0xcded6e6d),LL(0xe23393c9,0x6848f602),LL(0x2c8dbcb3,0x49f3a9ae),LL(0xf811e23c,0xc0c1ebfa),L_(0x00000130), LL(0xd5da561b,0xaf1b88cd),LL(0xb4c22029,0xaa7f22fc),LL(0x9624d6d5,0xbb120735),LL(0x416db935,0xa8308449),LL(0x85fd3219,0xc467f9f1),LL(0xb4d3e00f,0xa69d57d8),LL(0x187052a8,0x0528c91e),LL(0xb79e6638,0x2a603bc9),L_(0x000001a4), + LL(0x474fe094,0xca26efe1),LL(0xa3ad38a0,0xf5cd529d),LL(0xec34abea,0x94808b1e),LL(0x27c847ac,0x87ade961),LL(0xfa6df215,0x6a43fa8c),LL(0xbcfdb5ad,0x947fbb39),LL(0xdd4d0c9f,0xbca687c5),LL(0xe8772a4e,0x7d79e215),L_(0x00000122), LL(0x81cb032e,0xbf926e1c),LL(0xb04fbc5a,0xb9c12ffd),LL(0x34707ba5,0x4ee8c89b),LL(0x81aa347c,0x367a152d),LL(0x4cd56572,0x74511a3a),LL(0xa6642939,0xd0e3b8f1),LL(0x60ea13e9,0xee14ab42),LL(0x81a19a28,0xea76ba4c),L_(0x0000015d), + LL(0x04a0af27,0x6b0c75c4),LL(0x23a4b0aa,0xdb181c23),LL(0xdc940ab7,0x7b70983d),LL(0x328a76b8,0xd5b473a0),LL(0xadcb9bcd,0x3863dc05),LL(0x646b4949,0xe5090fd0),LL(0x0b996e3d,0xd0261360),LL(0x7c332b20,0xcc6b2f86),L_(0x000001c7), LL(0x3ac008c8,0x28cd5819),LL(0xf08cb2c4,0xfbdf661a),LL(0x9b1c2455,0x2be7d7be),LL(0x38fbe0c1,0xbd91e037),LL(0x84e69e29,0x1cdba496),LL(0xc6f94abf,0xa8445728),LL(0x8e9508e3,0x4a144f07),LL(0x8ee0e340,0xc5b72f6a),L_(0x00000066), + LL(0xfe2f1a3a,0xaff1e1b3),LL(0x87421ab9,0xbbfcc6da),LL(0xa3305ebd,0x4b75a8e1),LL(0x8f4cb778,0x4410056f),LL(0xb5abdc6a,0x4ff65612),LL(0xd83f32f5,0x21c44b1c),LL(0xb989d251,0x80a7bb1f),LL(0x5214abed,0x8f200e11),L_(0x000001ba), LL(0xd63a07ff,0xf4fb8525),LL(0xf5f23c02,0xaa8e02a4),LL(0x405911d9,0x45abb8b1),LL(0x7a6dae03,0x4834d14e),LL(0x4621957b,0xeb31fdc5),LL(0x7cbf9b75,0x26ee5dca),LL(0xee84304e,0x37349cc1),LL(0xcc6a2c7d,0x5a34c3af),L_(0x000001c8), + LL(0x3c425b85,0x9f122c9e),LL(0x71cfc92d,0x9c6ec42c),LL(0xb86c84b7,0xc8d12bf6),LL(0x1c821b85,0xe8432cc5),LL(0x197e0f04,0x4258bc34),LL(0xc4f03c70,0x60ae518c),LL(0x811512ff,0xf050c9b8),LL(0xe8038335,0x0b215595),L_(0x000000f0), LL(0x9423e29c,0x423ac4f7),LL(0xb15c3155,0x2fd13662),LL(0x7684c454,0x5cf8078d),LL(0x1a7bfb14,0x2b928e98),LL(0x1d05b843,0x3bbf2a85),LL(0xeeb1e658,0x356da90c),LL(0x179bc7a9,0x11d26c87),LL(0xf524843e,0xf4159e0d),L_(0x000000ec), + LL(0xc58a5d93,0x08cefac8),LL(0xb1885068,0xe8422939),LL(0x5985dd6e,0xab14cf0e),LL(0xcda94a64,0xc27af983),LL(0xd127851a,0xf24f6eaf),LL(0xbab20f8f,0xda3b25d8),LL(0xa549d9c6,0xed810bd4),LL(0x5bf18f37,0xf630e4c9),L_(0x00000055), LL(0x51ad76d0,0xae18594e),LL(0x52697460,0xf8de9d89),LL(0xaec56660,0x294777cd),LL(0xe3a93a39,0xf7dc98fd),LL(0x63fcc0bd,0xc0c53dc3),LL(0x5d2c2708,0x55da9198),LL(0x3692d050,0xebcde249),LL(0xcdc4d312,0x8d0017f5),L_(0x000000c1), + LL(0x045c6797,0x1d9471ec),LL(0x737ba42a,0xd3401c6a),LL(0x33fd5fca,0x3c2758a5),LL(0xb016b83f,0x79b1c693),LL(0x3409d0d4,0x5f80d853),LL(0x4f2a7005,0x4d1634ee),LL(0x799e106c,0x1e92ef70),LL(0x632dcd5e,0x86232632),L_(0x00000087), LL(0x06622996,0x1acab1e2),LL(0x92c31a31,0x91455009),LL(0x740223dd,0x15a695ed),LL(0xa95f341b,0xe601b98b),LL(0x17db74b3,0x19ccbb77),LL(0xd916a466,0x44573d27),LL(0xc31a7a19,0x093c0363),LL(0x1bb20e06,0x6715c5f0),L_(0x000001e7), + LL(0x5f8496f0,0xa1a3f86e),LL(0x2df7ec8e,0xf1f8f7f3),LL(0xd8551991,0xb16ec397),LL(0xbc80f4ee,0xebe5be1a),LL(0xa1e6cbf5,0xaf8233b8),LL(0x5c403702,0x41483767),LL(0xbf97ecb0,0x2899a5cc),LL(0x58655568,0x0720d399),L_(0x00000028), LL(0xdc27af93,0x88312054),LL(0xd550df72,0xf87e274d),LL(0x193eb1e2,0xa715c43f),LL(0x97773656,0xcb67dce2),LL(0x8a585c6e,0x0aacb5db),LL(0x6332fcd1,0x4f16d92a),LL(0xdeebccba,0x2b8001ac),LL(0x8936c8da,0x7b627657),L_(0x00000181), + LL(0x24e7c452,0x8d63a794),LL(0x35fd304e,0xdd225fc9),LL(0xa6aae952,0xc40c9b7f),LL(0xd5054f16,0x42316d8c),LL(0xf663b3dc,0xb3d7abe3),LL(0x13c94097,0x7aa82bbf),LL(0x78263190,0x2a622ce8),LL(0x819c0b14,0x2b1dba5c),L_(0x0000001a), LL(0x3c259ac8,0xe69bb850),LL(0xdf7bd8ef,0x985f2ed6),LL(0xc76c2599,0x44f156c5),LL(0x46e2c0c0,0x7cfc49ae),LL(0xf5fb07c3,0x6f59a7a0),LL(0x2f48e451,0x1b89eefd),LL(0x88119cbb,0x1c41ec61),LL(0xa18666db,0x53014a3d),L_(0x000001f0), + LL(0x4e03d590,0x792d6d08),LL(0x4d84ecf1,0xe0110c24),LL(0xc93fa7f1,0xc72b1bb4),LL(0x908f695c,0x1730f1b2),LL(0x8d0bc692,0xdb0b36b3),LL(0xe4bf469c,0xa1db29c0),LL(0x1d41428d,0x7a577f2c),LL(0x2cd1253d,0x23b65522),L_(0x0000006a), LL(0x7ebe31ef,0xa4ba5fbd),LL(0x9808ec8a,0xa5383520),LL(0x49718327,0x2c210a5f),LL(0xdc5bb249,0xef53e1db),LL(0x7e38e02e,0xc9d3c171),LL(0x7b41e983,0x3a07d487),LL(0x2d8aedea,0x6c0e3ba1),LL(0xa17e058b,0x22c8be6d),L_(0x000001d0), + LL(0x51500886,0x6a5713b6),LL(0xac6235d1,0x19855a0f),LL(0x32d1869e,0x093a8212),LL(0x8afdb213,0x89861196),LL(0x3402ba32,0xb3676c48),LL(0x5e54b89e,0x53597329),LL(0xbdde3064,0x94cdc873),LL(0xc3d273b6,0xfd911ed5),L_(0x000001f1), LL(0x65cf5cfb,0x0d98f860),LL(0xf6cf3683,0xa681e586),LL(0x6f5c1e3f,0xc6905825),LL(0x7d626d06,0x571b75e4),LL(0x00a44322,0xf9fe1aa4),LL(0x34ece73d,0x3975b815),LL(0x38add31a,0xfa3db092),LL(0x499ecb33,0x2ce86fab),L_(0x000001bf), + LL(0x8aeb123d,0xf5870ab9),LL(0xa353002c,0xa12da044),LL(0x6150f34f,0x0086b83e),LL(0x69e6eea2,0xa2cdf131),LL(0x5e80e0a0,0x528616b2),LL(0x2d13e0cc,0x4a67c598),LL(0x9702e01a,0x83d6e661),LL(0x15b60ef1,0x6f9172f8),L_(0x0000001f), LL(0x6386476a,0x7b2b5776),LL(0xe6acc547,0x055811a0),LL(0xba422b24,0xa9873020),LL(0x8c990991,0x310acf2c),LL(0x96459d45,0x78701ea7),LL(0x917c30ec,0xd1688c83),LL(0xdb51be44,0xb42ce9e9),LL(0x0b514c3b,0x0b03fd87),L_(0x0000008d), + LL(0xb09c0812,0xbcc82868),LL(0x69816459,0x580f7a11),LL(0x9b94ac07,0x11b4de1d),LL(0x120451de,0x8f21a7aa),LL(0xc048b454,0x0f6b490e),LL(0xca8d647e,0x5d0f4e1d),LL(0xf1f7c090,0x3e12d889),LL(0x1ad27c80,0x5b341256),L_(0x000000e5), LL(0x5e7c3d96,0x35f1970a),LL(0x4366eed0,0x1134e984),LL(0x55c0352d,0x7ea259fe),LL(0xfad7d83b,0xda4dcbce),LL(0xdd5f6008,0xb2924c78),LL(0x01b25214,0xac404086),LL(0xf325f997,0x2b613948),LL(0xf37e21a0,0x26e31be0),L_(0x000000b4), + LL(0x40c9bb67,0x017edbd6),LL(0xf483d72a,0xb08491c6),LL(0x58a225c5,0x568a7e71),LL(0x7fde8697,0x821bf73d),LL(0xef4bc022,0xec765e3a),LL(0x8d1daf2f,0xb59a1d2e),LL(0x72d486e7,0x1edfc037),LL(0x2a595f95,0xf1683f88),L_(0x000001ff), LL(0x4c4cc13e,0x55fc5381),LL(0x96f30cc5,0xc6ce2141),LL(0x76a3af64,0x339f5668),LL(0x5449bfff,0xe438adb5),LL(0xf3c48dff,0x1aa59ae8),LL(0xce59b544,0xc0fd6c57),LL(0xb7bdc7b7,0x8e51d10e),LL(0x973b8e1d,0x6427d578),L_(0x0000016a), + LL(0xf61dae1f,0xe76cf424),LL(0x4eac44ab,0x559e7a5a),LL(0x0ddf44f2,0xc58d75d8),LL(0xfb0d499a,0xab62039f),LL(0x6cf6c677,0xd4e76825),LL(0x2e427953,0xa955fdca),LL(0xe1d73f88,0x049f7f5d),LL(0x89dc4a2e,0xd5493485),L_(0x0000015a), LL(0xadae9a0f,0xa5dc86bb),LL(0x2a75769b,0x606d9e57),LL(0x550fb22b,0x260bcabb),LL(0x7bccdd84,0x2e3ee7a3),LL(0xc4b6b979,0x03bd7f7d),LL(0xfc3349bd,0x122b5333),LL(0x95f84290,0x4bdf7095),LL(0x3057b4f5,0x6af3cf31),L_(0x00000022), + LL(0x64c8b352,0x1d055192),LL(0xf272a08c,0x343f766f),LL(0x142d545c,0xb8bd86e9),LL(0x860ef117,0x60c69c66),LL(0xb6de931c,0x1b54e53c),LL(0x9924f2f5,0x878c0c9b),LL(0x0b949095,0xfba7e2a3),LL(0x6916f5f1,0x7da79c3a),L_(0x00000181), LL(0xbd559979,0xe06ad6ba),LL(0xd551de11,0x3b3cbbe6),LL(0x6c45d4c3,0xcc4aa553),LL(0xe3c9e3df,0x1bb5c238),LL(0x05a1e382,0x8dfc012d),LL(0x84d8d463,0x3b856506),LL(0x05b7e241,0xcdcfd8e8),LL(0x27718949,0xc1a85e66),L_(0x000000ef), +}, +/* digit=12 base_pwr=2^60 */ +{ + LL(0x09d8e58c,0x0af6a9bd),LL(0xdca0a6cc,0xfe5f904b),LL(0xd9e6d336,0xd87d0339),LL(0x3b8c9d8b,0x4d463bab),LL(0xfb629c3f,0xc203e46d),LL(0x4ea62ed4,0x998a0ef3),LL(0x64035458,0x62783285),LL(0x7769592c,0x3c56ebb3),L_(0x0000001b), LL(0xb96cc870,0x259a17aa),LL(0xd51ce441,0x8666df8e),LL(0xc62b1c65,0x437c7966),LL(0x74db6999,0x0fecb364),LL(0x7c60998f,0x1f725b1f),LL(0x71fdafc2,0x5b56396c),LL(0xa547fb5a,0x9d888686),LL(0x0f566ae7,0x130033ff),L_(0x000000f4), + LL(0x41a4fd11,0x12a6c73f),LL(0x66164319,0xfe4c8bf4),LL(0x9c6ffbd2,0x42f313ec),LL(0x2869e4fd,0xf8b100ba),LL(0xbae712b9,0x0e18229b),LL(0x61a1f1da,0xffe55501),LL(0x032c80f2,0x3bfaa0e0),LL(0x48f0b1d5,0xb83c7607),L_(0x0000015f), LL(0x3cf7a1fa,0xa0ed3335),LL(0x8b3c031f,0xc141575b),LL(0x53c30e33,0xfa62217c),LL(0xf9f945a8,0x8b667de4),LL(0x889399aa,0x7c4952fb),LL(0xb711abc7,0xabedb6e3),LL(0x59e7e12f,0x5a1b2cb9),LL(0x1857ebfe,0x4206e243),L_(0x00000134), + LL(0x301de7d2,0xa95f9c5a),LL(0x0d937115,0x2ee0eb80),LL(0x4b1412c8,0xdf5a5904),LL(0xe6f39cf3,0xcd50327e),LL(0x9a796b16,0x0841dfd1),LL(0xc493ac5c,0x19d15d79),LL(0x7275eb23,0x4b9d4479),LL(0x1a3b6feb,0xe1eb10df),L_(0x00000142), LL(0x6d17f389,0x7551bac5),LL(0x7a907c78,0x232dc783),LL(0x82e7d67e,0x5acaf222),LL(0x5ebc3c22,0xe17100c9),LL(0x62250256,0x3198b234),LL(0x4beb3ba2,0x16986b8a),LL(0x492d3035,0x973e4135),LL(0xfcc0dd28,0x2e1155d6),L_(0x0000011b), + LL(0x4df6981f,0x1f14d7bc),LL(0xe951da15,0x3d397c45),LL(0x3964143f,0x24be6549),LL(0x2e556c9c,0xe1293e25),LL(0x3aed330a,0x4bfda40e),LL(0xdf82159a,0x3b13e72c),LL(0x514f7b17,0xa5b859ff),LL(0xe20684bf,0x90812f67),L_(0x000001ce), LL(0xe9a0c258,0xbca9abf7),LL(0x3b0b3a0f,0x72194a82),LL(0x11d27090,0x17f5564f),LL(0x9bbb7a7f,0x87f0af99),LL(0x96c01479,0x69d62017),LL(0x45cce25a,0x0c43d35c),LL(0x26584337,0xcbff6e89),LL(0x19a55401,0xb503e2ea),L_(0x000000ea), + LL(0xb5cd512f,0xaecaabc4),LL(0xdeccde50,0x9ffdf34c),LL(0x395d2404,0x25068e1b),LL(0x40559189,0x93fb9ea4),LL(0xd141ad3f,0x2a60ba95),LL(0xc42f76f0,0x414a5981),LL(0x946bf800,0x138c47b5),LL(0x38435023,0xf314147e),L_(0x0000013c), LL(0x9ae19e3e,0xe38bdcc6),LL(0x3cebd917,0x0966bac7),LL(0xc3533788,0x2718c3e8),LL(0x33ee6ede,0x10236ae8),LL(0x4f5b88fd,0x44797bb1),LL(0x485e76bd,0xb2b31296),LL(0x68194c12,0xe45112ba),LL(0x0cb75dc1,0x8574000b),L_(0x000001d8), + LL(0xeea24bcd,0x8f37d315),LL(0xf77a65b3,0x37731160),LL(0xde279622,0x6f06ae65),LL(0x87ebd334,0x25b38b15),LL(0x2a1d2c7a,0xa55c6b9f),LL(0xb1687394,0x0ccf2f34),LL(0x4f27c66a,0xecf3de75),LL(0xa9866c84,0xa4a0f4aa),L_(0x0000014d), LL(0x8dffeb86,0x0366dd8b),LL(0xbfeaeff0,0xfe941121),LL(0xa80b5c3b,0xc3fed2fa),LL(0x18a5b6a4,0x23dfdf47),LL(0x2ef007c6,0xdb0791d2),LL(0xcec61c6b,0x6d79949c),LL(0xe328d9cc,0x0d03e696),LL(0xaa14a153,0xfdb36710),L_(0x00000145), + LL(0x1a8d3cd0,0x736dcfa6),LL(0x77e26493,0x6af49ff9),LL(0x089ee4ac,0x1720bd71),LL(0x2f3b86d6,0x48d2c5df),LL(0xbcc66a78,0xd78e07af),LL(0x1f230a9e,0x077a7ceb),LL(0xd2f61bf5,0xfbf99e70),LL(0x92770c3f,0x7ae5f084),L_(0x00000148), LL(0x64c29961,0xa6ee44c4),LL(0x990f4f03,0x4aacbd8f),LL(0x45377427,0x0ef447b6),LL(0x55b5c873,0xe02e661f),LL(0x11e65ae5,0x99f13f10),LL(0xfe17d3ed,0x393cf4c8),LL(0xdbeb35dd,0x23277110),LL(0x65a7d1cd,0x444802cd),L_(0x000001fb), + LL(0xf621fd74,0xea71a842),LL(0x4c057a1d,0xfc8fb859),LL(0xe1689c80,0xadc9a8e1),LL(0x09c22f52,0xc47b8163),LL(0x0a960c99,0x90c495f0),LL(0x0a0f356d,0x88242e20),LL(0x87494b79,0xb7f9ca6a),LL(0x6fdcd587,0xd76d2c39),L_(0x000001c4), LL(0xd6ecf158,0x1e35970d),LL(0xaee47a26,0x8df13449),LL(0xadfd394c,0x67553f2c),LL(0x71cdfbec,0xa43c6154),LL(0xf09db2ac,0x4606556e),LL(0xf2e04011,0x12eca225),LL(0x9dfb28da,0x87a4c839),LL(0x28812bc5,0x8cba8984),L_(0x000001ab), + LL(0x49041a38,0xcb554ab1),LL(0x3446834e,0x21810284),LL(0x2ab359a6,0xf95fa59b),LL(0xf33f9ef0,0x16db657d),LL(0x0f8d940a,0x38fe2897),LL(0x39b668bf,0xdeba7f4c),LL(0xc6452278,0x7471cf19),LL(0xb96dd1e3,0x732f77c2),L_(0x0000011b), LL(0xfa410fb4,0x745c3f1b),LL(0xbca782f9,0x8bd5ef13),LL(0xc4e21488,0x8211733f),LL(0x8f6c1b78,0x50b780cb),LL(0x4b628b50,0x1a1a0206),LL(0x78e4de6a,0x44975c37),LL(0xf9f51865,0x6ef7e616),LL(0xbadf032d,0x3882a9ad),L_(0x00000084), + LL(0xe4d0aa5a,0xe84ad756),LL(0xf2f7ceab,0x6545847a),LL(0xfb04aded,0xdd4cb1ba),LL(0x87083ecc,0xf4c8bbb0),LL(0x452249b7,0x6531b732),LL(0x868536ed,0x6968e15d),LL(0x1d0209ca,0xf0285aff),LL(0xfaefc194,0xbed23705),L_(0x000000bd), LL(0x6489b527,0x3ea47ce4),LL(0x69374c35,0x2a6d8757),LL(0xdc6375ee,0xc6f768ea),LL(0xaeba5bab,0x327c743e),LL(0xda6790e0,0x9a01ae4a),LL(0x1a9de4f9,0x3ae6cb85),LL(0x9ac5b7b3,0x6d32a174),LL(0xf134b615,0xdf38a0f3),L_(0x000001ec), + LL(0x7d9c19cd,0x688d2325),LL(0x79db6c85,0x5359ff24),LL(0x764f954a,0xc7801c4a),LL(0xfa78e8b8,0x098ede82),LL(0xb52cd1ab,0xd34f03a8),LL(0x66adb2da,0xcfcfe244),LL(0xfc69d130,0xb5e52304),LL(0xf88483bc,0xab73db68),L_(0x000001c7), LL(0x228f1338,0x077d01af),LL(0x7eb9fb07,0x8abf2d7a),LL(0xcb62a0d5,0x37a4ecbb),LL(0x1a28e347,0x06b68356),LL(0x35c05ae1,0xadaac01c),LL(0x2f3d8c6e,0x712aa1f3),LL(0x9ee5907c,0x69606236),LL(0xc9bdbb2c,0x7b2e6894),L_(0x00000011), + LL(0x0c5ccb9d,0x09d420b2),LL(0xccc993bc,0xa3ad7d2a),LL(0xd8b3ee97,0x7986ac14),LL(0x59fa9e76,0x95dc5774),LL(0x9477b42b,0xfbe8e9d7),LL(0x89d7ab26,0x79b03712),LL(0x017b7f94,0x77f9bdea),LL(0xbd8dcef1,0x7a238609),L_(0x000001fe), LL(0x1b9ffcc9,0xaee002b6),LL(0xcdfc127c,0x23640ec5),LL(0xad2abcbc,0xc6dc5bd5),LL(0x05982646,0x20400061),LL(0x3c1c6b9f,0x6ee16a76),LL(0xc943d1fd,0xd619a75d),LL(0xd16a85f5,0xa278715e),LL(0xd8747be6,0x34ec8668),L_(0x000001c0), + LL(0xdaf270b9,0xcec9be5b),LL(0x6f5e16ec,0xfd62380c),LL(0x192223b1,0x27bda6fd),LL(0xe1e75d7c,0x0df8a788),LL(0xd01bccf4,0xbeed1a6b),LL(0x6611a8ed,0x01402436),LL(0x17838dc7,0x7f189fd3),LL(0x615a507f,0x760bd862),L_(0x00000152), LL(0x090f0135,0x791dac6d),LL(0x2bb5aa65,0x0457b859),LL(0xece0f798,0x1a8af3a8),LL(0xde75b69f,0x6625db63),LL(0xcf064060,0xeeda55fc),LL(0x0d8f8c69,0x05536430),LL(0x27c6a431,0xb1dc58b1),LL(0x56c1ac3a,0x0c1a1dab),L_(0x000000ae), + LL(0x18e8414b,0xfb73e1e4),LL(0x0f973c1b,0x6977f355),LL(0xca40d04d,0x2797e4c7),LL(0x01c089b0,0x1b05804b),LL(0x0064b701,0x7b76fc1d),LL(0x9677da0f,0xb0b47105),LL(0xd02ba9e2,0x4fb9b758),LL(0x6b2435ea,0x2fd704c2),L_(0x000001a6), LL(0x2578e31f,0xdc19942f),LL(0x4734e848,0x366f685b),LL(0x6b9e935f,0xb3827f30),LL(0x81e91d77,0x4cce7910),LL(0x36ada690,0xcb031a95),LL(0x227eb763,0x6ac3a0f0),LL(0x527fc0d3,0x7b60ac80),LL(0x404eb0f3,0x2c62b4f1),L_(0x000000c9), + LL(0xfa5ed043,0x95cf6b06),LL(0x69fd4f1d,0xbdef736a),LL(0x9044b8ce,0xd7e44ee1),LL(0x546a5d1c,0xe3ac270c),LL(0xbe0ace78,0xe59e1538),LL(0x27b93218,0xe51fc4ac),LL(0xf26796fb,0x71f9328b),LL(0x9137cac1,0x07a55147),L_(0x00000057), LL(0xedbb99a1,0x9ce1532c),LL(0x37d59c98,0x2c0e5b9e),LL(0x62f632e6,0x55146f87),LL(0x29fd2249,0x402150dd),LL(0x12ea0f69,0xf442153e),LL(0xfa397b38,0x9b5cefd9),LL(0xc5ad174d,0x8cae5294),LL(0xb46b9f16,0xcd8b0a60),L_(0x000001a7), + LL(0x4276a1bd,0x1e614695),LL(0x71344edf,0x2b4a50c1),LL(0xb3013081,0x4896c770),LL(0x2cf314a1,0x96a68659),LL(0x90053fe7,0xd79226d8),LL(0x5847ac79,0x3ada869a),LL(0xf60993a8,0x7d156a5c),LL(0x67e4b5fe,0x7850cdf6),L_(0x000000dd), LL(0x5fb3dea4,0xf35bcbb3),LL(0x4e2d6021,0x9877f0a3),LL(0x90be9398,0x4d6435bb),LL(0x86130340,0xe5919257),LL(0x2710c007,0xcc99d199),LL(0x87d3586c,0xc1451c79),LL(0xe8681c58,0xfa896da8),LL(0x6659a487,0xb1a9e543),L_(0x000001a8), +}, +/* digit=13 base_pwr=2^65 */ +{ + LL(0x73f3ddc5,0xfa295332),LL(0x0b259ba7,0x94f8c958),LL(0xa4092fea,0xbe9d56f6),LL(0x622efd38,0x0f2ba425),LL(0xa4d25a72,0x57c0adb2),LL(0x2498a9ea,0x11f11875),LL(0x893bbb4d,0x195ec41d),LL(0x2f56b02f,0x2ad72c4b),L_(0x000001ca), LL(0x1ab7060a,0x0fa4013f),LL(0x521f983a,0xebae7f17),LL(0x5292b2f1,0xdebce289),LL(0xd6d75002,0xb6cd203a),LL(0x93bfe503,0x3c3592c9),LL(0xa40b351b,0x180f5400),LL(0x9b6bafed,0x291283ae),LL(0xd4d6a9f0,0x036cf95d),L_(0x00000163), + LL(0x5db76801,0xa8cfa5a9),LL(0x33878665,0xa5401cc3),LL(0x809b2a4b,0x6cdc3f0e),LL(0x90d9594f,0x9bbfac67),LL(0xd551d6e9,0xfd836074),LL(0xe874e847,0x13f89d9a),LL(0x264b3b0b,0x7a6ec5fa),LL(0x0a3ac51f,0x6dd250c6),L_(0x0000001d), LL(0xa7747bde,0xd1e14aba),LL(0x7c3196ca,0x1495ef12),LL(0x78a62924,0x0cbcf8af),LL(0x1f4ded5d,0x83d56ec3),LL(0xfa54b15b,0xcc6ef029),LL(0x6f0a12c6,0xae62cc51),LL(0xce830e11,0x964fd2d0),LL(0x88747fe9,0x56076a32),L_(0x00000067), + LL(0x8f89c374,0xb73d3d92),LL(0xc668cfa4,0xda69c4d9),LL(0x97ee2907,0xbf4c3402),LL(0xbf5fb743,0x4034c59c),LL(0xd60ae9cd,0x99bc4b73),LL(0xda82be72,0xda1f7664),LL(0xe3800a84,0xfb007b67),LL(0xb7700f12,0xb546161e),L_(0x00000082), LL(0xd0f66b94,0x4be150bb),LL(0xfc5d0def,0x660c9122),LL(0x1ba0f43d,0x3a5b4550),LL(0x7224e926,0x33c24e5b),LL(0xba92b4ef,0xd249e1b7),LL(0x2b1856c8,0xb2c9aa15),LL(0x5fe68108,0x6e540179),LL(0x2fe766ae,0xa379f58c),L_(0x00000103), + LL(0x32f3278c,0x4d5341f2),LL(0xdff5ad0b,0xbb141c66),LL(0x6270a82e,0x7912e413),LL(0xfc62897d,0x6b16ad87),LL(0x348f2e6b,0x0fe7c18f),LL(0xae57af6d,0x2f22a03b),LL(0x6d2d6ab0,0xefa7a28a),LL(0xd717c3e7,0x73423958),L_(0x000001c6), LL(0xe49ed5e2,0x0b4f0f2c),LL(0x8c6c9219,0xa884b55b),LL(0xaff1be7f,0xde74b331),LL(0x8882c375,0x7a676c7d),LL(0x57c355f3,0x71190b6b),LL(0x180dbbfa,0x599b9c95),LL(0xd7dc77b1,0x8f766481),LL(0x227eba11,0x840229ee),L_(0x000000aa), + LL(0xa6fa0735,0x5d79f380),LL(0xaa1f8835,0xe2b5d59c),LL(0xf1a96e2b,0x281ece14),LL(0x146a87f9,0xb65f19cf),LL(0x24f845c5,0x2f123e45),LL(0x9418bddf,0x90bcee98),LL(0xb6ea99d9,0x64b9ae2d),LL(0xd147e8b2,0xda89ba61),L_(0x0000018f), LL(0x36b11956,0x53c9380f),LL(0x31e35563,0x7d832bdb),LL(0x7117919e,0x5906fec2),LL(0xbdd97ce8,0x069087de),LL(0x015486f6,0x866bf3e5),LL(0x909fd1d8,0x1d61be88),LL(0x770d7f9e,0xa28b013b),LL(0xe7653682,0xddec12fa),L_(0x00000060), + LL(0x37b31659,0x38819383),LL(0x0ad9906f,0x9f83225e),LL(0x089577d6,0x8e97694b),LL(0x264428ce,0x0c33baa6),LL(0x1bf2f80c,0x0823ef0c),LL(0xbf31819a,0x2e4750ee),LL(0x66c7b596,0xd160d5df),LL(0x886455a6,0x575835ea),L_(0x00000070), LL(0x2fd22a9f,0x3c424c86),LL(0x240ffedb,0xd1be1d89),LL(0x3c874c88,0xed147fb3),LL(0xccd38c51,0xbbd7bef5),LL(0x10af1392,0x101c8dcb),LL(0xd85a000f,0x018793c4),LL(0x0b435263,0x24305a2a),LL(0x989d12cc,0x9c061ca0),L_(0x0000013e), + LL(0xef0eb1ed,0x9a069590),LL(0x8dfc3414,0x1912aeb6),LL(0x0b37df44,0x1e30b3df),LL(0xcbe1989a,0xaf9f97dc),LL(0x415de022,0x4e406185),LL(0x01cb0277,0x2eb58373),LL(0xbb545160,0x0ab0a19c),LL(0x1e53639f,0xeecdb49f),L_(0x00000169), LL(0xcd8e8ac0,0x2a5d2526),LL(0x8e8397e0,0xa679b16d),LL(0x44c0a296,0x7dcafbf3),LL(0xee3f4022,0x668a1c15),LL(0x2ca74425,0xea786663),LL(0x4906d981,0xda4fa2cc),LL(0x0bbbb1fb,0x21224a70),LL(0x8859a117,0x8ec6a6f4),L_(0x000001ec), + LL(0x3de6fe08,0xf1507b4b),LL(0x4574b533,0xb56c3906),LL(0x76eaa707,0xa9532376),LL(0x16e5e98a,0x611c9b67),LL(0x12d9a934,0x9eb6a261),LL(0x8430b478,0x3fab6e06),LL(0x50fd9610,0xd215cdde),LL(0x1d509b62,0xc4da786d),L_(0x000000b7), LL(0x6e4be0a7,0x752a0af8),LL(0x3ebf635e,0x680f5838),LL(0x9175f3f7,0x9f1a0d87),LL(0x861b999d,0x04cce1e2),LL(0x75ef231f,0xe86e6afd),LL(0xaf7240e6,0x04734476),LL(0xe887f56c,0x2837e095),LL(0xc194ba35,0x9e3dc524),L_(0x00000146), + LL(0xcb011cc2,0x9a9802d3),LL(0x4d7d0f39,0x0345d8d7),LL(0x5e5fc037,0x6923910c),LL(0x0187070a,0x5e15ce92),LL(0x3e13ac42,0x456e81c2),LL(0xcc3c7cbf,0xfc527716),LL(0xcd65b4f8,0x5fd7c9e9),LL(0x305c5c1a,0x4af9f3b6),L_(0x000000a4), LL(0x3a78773a,0x1b64c99c),LL(0xc01b599b,0x38f68420),LL(0x53fbc68b,0x1b40d8c5),LL(0x91c4f402,0xe87107e9),LL(0x0e78fd77,0x96fc64fc),LL(0xf13016cf,0x438fbd69),LL(0x59324677,0xa8e5ff95),LL(0x50792eee,0xb6a73a20),L_(0x0000015c), + LL(0xb5b9e0e4,0xb1979059),LL(0xaa4d1038,0xf13725c9),LL(0x5751fd0c,0x22f3e709),LL(0xe1b98f3c,0x131dbcec),LL(0x8e66976f,0x64116cc1),LL(0x2dc36526,0xf955db38),LL(0x6f94eedc,0x6bee8490),LL(0x2dc08c31,0x65ce87a2),L_(0x000000c0), LL(0x96bd706c,0xf8a59cff),LL(0x9035a740,0x328c1388),LL(0x61a4c1b8,0x1625b5b4),LL(0x8b1931b4,0xf35b9bbd),LL(0x075bce63,0xfb1707dd),LL(0x7a601744,0x232aecf6),LL(0x50241612,0xa974c9fd),LL(0xf1cae8d5,0x7f4ba9ea),L_(0x00000036), + LL(0xbe01288f,0xe7c06e65),LL(0xa5756aa5,0xd57f322c),LL(0xc48c9811,0xd934c78f),LL(0xf0128bf0,0x15f53184),LL(0xd880ad31,0xadaedd19),LL(0x967dfa08,0x374ad4e4),LL(0x94c0e608,0x389863cb),LL(0xcecf2255,0xe582d71f),L_(0x000000ed), LL(0x45e5581f,0x5b089de0),LL(0xef23aed0,0x89f1eb4b),LL(0xb93d3851,0x8967136f),LL(0x4daacc4f,0x2482bd87),LL(0xd7b99516,0x8a382f79),LL(0x404615c9,0x9daadb4f),LL(0x114144be,0xd38bfc82),LL(0x173ad4ab,0x7f91ad71),L_(0x00000042), + LL(0x4b16b686,0xa28a3c93),LL(0x299c2c3f,0xf2024c1b),LL(0x63be4df6,0xcbefbff9),LL(0x39510e86,0xe4d80ac6),LL(0xf1d2f2e9,0x8653ba64),LL(0x0165880a,0xd4fac73e),LL(0x73ce1252,0x67da578f),LL(0xeb225590,0xc1817050),L_(0x000001bb), LL(0x235f9feb,0xdc671b44),LL(0xa93c7bda,0x82c05dfc),LL(0xcb77f18a,0xaea605e9),LL(0x1df6368b,0x295e78ce),LL(0x67c3474a,0x93547410),LL(0x18ce6948,0x7dc8108a),LL(0x7c67eccb,0x8d3a8c61),LL(0x67a75d98,0xbd7b3b73),L_(0x0000002e), + LL(0x4238f2fc,0xaca4c347),LL(0x9fc92c8d,0x994543ca),LL(0xc6ad228f,0xb218a1c0),LL(0xc58614eb,0x527974b7),LL(0x6ca62054,0x85ffca1f),LL(0xc2129ab9,0xc6c14b3f),LL(0x019b6e5f,0x2ae678d2),LL(0x1e22f90d,0x5b3ee46c),L_(0x00000046), LL(0x8e533ac3,0xd3c4dd26),LL(0x4e855850,0x74af7741),LL(0xf226ab4a,0xe0d7e588),LL(0x332581f0,0xe826c3d6),LL(0x4284a728,0xcdf777fd),LL(0x6bfcaba8,0x5a83f0b7),LL(0x6ddf35c7,0x2fbd194d),LL(0x212fa0d3,0x072b793f),L_(0x00000049), + LL(0x0ef3997d,0x9d02c3bc),LL(0xb7ec87d2,0xe0887e3d),LL(0xb3caff01,0x4d1f3674),LL(0x567cdbdc,0x39e61184),LL(0x3d19e2c3,0xbe0de4f9),LL(0x0c3139a2,0x500e0978),LL(0x7ba6031f,0x6f3470b9),LL(0xf8e9a69a,0x65297bad),L_(0x00000159), LL(0xb53b49bc,0xc7bd9625),LL(0x69eb8288,0xe4ec65aa),LL(0x281d3a84,0x2c8f25ab),LL(0x27426301,0x97e61a91),LL(0x3672bc0b,0x95476b11),LL(0xbfd7d2dd,0xbff37ddc),LL(0x918c4eae,0x82eea309),LL(0xbd19084d,0x26d27fb7),L_(0x000000af), + LL(0x97ab40fc,0xe6cbd721),LL(0x4a4ba674,0xc27ec19d),LL(0xf1234a47,0x16a6532d),LL(0x3cf8bf88,0xe2dbe535),LL(0xbc4dc6be,0x948616ba),LL(0x46216f90,0x76d1a242),LL(0x02af6244,0xd5de4770),LL(0xd85e1029,0x7fda32cb),L_(0x000001f1), LL(0xa684ba28,0x34f944fd),LL(0x944544cd,0x0193b124),LL(0x72b4685a,0xe7601697),LL(0x6ec14591,0x9572360e),LL(0x2184e096,0xffc7295c),LL(0x9127a0f3,0x844306ef),LL(0x818c91ad,0xf603be79),LL(0x81b486bc,0xc1bd26da),L_(0x0000014f), + LL(0x4a982652,0x3c726e7d),LL(0x7d1e874b,0xa513da39),LL(0x8c78c755,0x92677915),LL(0xe62fef13,0xe9e24f3a),LL(0x3d1cf9e1,0xd96cf621),LL(0xf503d4fc,0x0e1204b1),LL(0xf07e39bb,0x71958180),LL(0x7a406c60,0x3b7b9a61),L_(0x0000006d), LL(0x35ac9c86,0xcdb43a8c),LL(0xf37f3857,0xed377a92),LL(0x8ae49b6b,0x0827d789),LL(0xbd50e338,0xdeff6865),LL(0xfdb287e2,0xe758e466),LL(0xa0c560a0,0x54321f3b),LL(0xb418a264,0xd44767fd),LL(0xfaaa26f5,0xcc7b7f8b),L_(0x00000136), +}, +/* digit=14 base_pwr=2^70 */ +{ + LL(0xf16a4fbf,0x580dde02),LL(0x814d149b,0xeaa3b1ca),LL(0x3cc8c599,0x43a45440),LL(0xc98d833d,0xdba29de3),LL(0x6e31f2af,0xab2ff205),LL(0xf81e95cb,0xb530ab3b),LL(0x49419f19,0x8a6e1bb6),LL(0xd0585b64,0x1dfbba1e),L_(0x000001f1), LL(0x6bc60cfa,0x2b473469),LL(0xc0250d4a,0xb9f4e199),LL(0xd4759758,0x326d4e2a),LL(0xc32f68bc,0xa78113ab),LL(0x840b01ce,0xcd248f92),LL(0xbd87644a,0xa8d8d61e),LL(0xe9a32d38,0x58a69c2a),LL(0x6a0c706b,0xf4b942e1),L_(0x00000161), + LL(0xb0a85bf9,0x538c7fcc),LL(0xff198eec,0x4ec043bd),LL(0x29ee8af8,0x125b846f),LL(0xd01572ea,0x280cfc9e),LL(0x4ba80325,0x3f73f265),LL(0x57e3b7be,0x6bcaffbf),LL(0xf83701a2,0x1a2d3724),LL(0x19d20a25,0x410f80b3),L_(0x000001ec), LL(0xdc0e5194,0x3b961197),LL(0xc136f93b,0xc26463d8),LL(0x000ba8d6,0x8d99824e),LL(0x0e084f84,0xcfbb42b2),LL(0x81fef33f,0x138715f7),LL(0x48ed1078,0xca7dbdd0),LL(0x42869724,0x3c66b900),LL(0xcfde2c20,0x89fab2c4),L_(0x00000104), + LL(0xf7cac6a0,0xaf4c0100),LL(0xca7c1c9b,0xd05a1cfa),LL(0x096b7d5f,0x5e939f07),LL(0xc34c35b1,0x11a408f8),LL(0xef94d03f,0x9c1a3053),LL(0xa610576b,0x89fbfdb7),LL(0xce4bec40,0xabf93595),LL(0x66023f5e,0xc5d43f87),L_(0x000000aa), LL(0x2f2e5545,0xa58413e1),LL(0xcbfb3671,0x1874038a),LL(0xd3ca207c,0xb2e8a04a),LL(0xccca2442,0x3073c925),LL(0x3c9baa99,0x554b9664),LL(0x6d9e1787,0x70e99ee9),LL(0x874df9a7,0x312bf341),LL(0x1b8e89dd,0xfdf17994),L_(0x000001a7), + LL(0x7aa46ca1,0x5838bb0c),LL(0xad2e37a1,0x5a28cd2c),LL(0x54d33ad6,0xd4f1caa9),LL(0x44b04b20,0x80e4c9d2),LL(0x8c65ceb1,0x370a13f5),LL(0xecff016c,0xee758816),LL(0x6ad260ae,0x95c36fe1),LL(0xbbdbb7b5,0x4d06dfe8),L_(0x00000094), LL(0x78399219,0xf5c325d6),LL(0x955c2a22,0x79a376a5),LL(0x16640925,0xf8e9390b),LL(0x36b3aac4,0x2a8dbf22),LL(0x034f2b72,0x77b02d94),LL(0xd5de86b9,0x729cfcd8),LL(0xa8bc9f80,0xbe296bcf),LL(0x1dbcbc03,0xec1469f1),L_(0x00000164), + LL(0x6a57e8a6,0x4141a618),LL(0xa7081b57,0x51abc1ef),LL(0x738c30ac,0xb0cb69e4),LL(0x731cd9ef,0x9b0b0c3b),LL(0xf0dc5cc9,0x40e54a92),LL(0xfba2bbb5,0x0dee7871),LL(0x5ee7b5d3,0x540e5eaf),LL(0xad1a2eba,0x7c44af5d),L_(0x000001da), LL(0x235257d1,0x451af808),LL(0x1bf41212,0xacac98b3),LL(0xd6076452,0xed40fc8f),LL(0x868d93c4,0xb7246c52),LL(0xab7c14c6,0x36bab138),LL(0x789fa296,0x330e1a06),LL(0x88e8110a,0x5dbf4ce1),LL(0xff782421,0x9cabebe8),L_(0x0000018f), + LL(0x3aa4041a,0xbf3ef154),LL(0x0cb92f46,0x47026a02),LL(0xe21fd797,0x1dec53c2),LL(0x0b5a2b41,0xcfbf686e),LL(0xdb7c6dc9,0xd6d5c0b4),LL(0xb4a8866b,0xf8283374),LL(0x14d9e7f9,0x4dd48282),LL(0xffba2822,0xf9de17db),L_(0x00000094), LL(0x19ce594f,0x1814c604),LL(0xbe3bf885,0x12bae7dc),LL(0xdf04c3eb,0xe8ee1061),LL(0xc658c3bb,0x8d34f043),LL(0x47642843,0x2916bbb3),LL(0xedee7c23,0x70f93acd),LL(0xc93b4f5c,0x187bf7ea),LL(0x75348b4d,0xffda6e9d),L_(0x000000eb), + LL(0x9c49db1a,0xc855134e),LL(0xdd4ae89b,0x30b7a1ad),LL(0xd9dc0b8c,0xbee0416e),LL(0x67e1dcbc,0x66147ae2),LL(0x7907e5e7,0xb9dea373),LL(0x537d932f,0xf4450461),LL(0x8d5aa671,0x7b0644e6),LL(0x33269776,0x835848af),L_(0x00000089), LL(0xfbbaf49b,0xfdca2fb9),LL(0x400c0893,0xe36e56e3),LL(0x095f6119,0xaaeab6ef),LL(0x07fda371,0xfdbbf61e),LL(0x51034096,0x65d823d3),LL(0xc5284f49,0xcec7f701),LL(0x00cca32d,0x443cce6e),LL(0x13b673ca,0x7c7c1332),L_(0x00000150), + LL(0xaa83f580,0x5f48e2fc),LL(0xfafcc610,0x2c5d9495),LL(0xd34073fc,0x321d1a08),LL(0x30442510,0x3427742e),LL(0x8068ffd8,0x2ec5f97b),LL(0xa7faa8a5,0xac14530d),LL(0x5d010e52,0xf277a140),LL(0x3edf5701,0xcdd53228),L_(0x000000ea), LL(0xb172daf2,0xf8f566cb),LL(0x51771845,0xa7b0e50c),LL(0x66aafeee,0x81cf4ee6),LL(0x8fd52580,0x1bc2c6ec),LL(0x232a19c7,0x6790d250),LL(0xd4c06ab2,0xdc4411be),LL(0xacd06e0b,0xe0fd2a20),LL(0x19734273,0xc9fb738a),L_(0x000001c1), + LL(0x4d908e20,0x2c36a49c),LL(0x346bc599,0xb17c0c24),LL(0x4349ec6d,0x1994a52e),LL(0xc574a60f,0x479b18ee),LL(0x2cf1156f,0x00b04364),LL(0x587d6a8b,0xe20999e4),LL(0xb840bbc5,0xdfbaad24),LL(0x1056fcbe,0xb84e8539),L_(0x00000015), LL(0x182e14f7,0x34b3807b),LL(0xc83300dc,0xcbae472d),LL(0xff64a1e1,0x60ef1e86),LL(0x1368c7ee,0xfa9cecf9),LL(0x1a548595,0x12c62bf5),LL(0xbab6253e,0xc9541d0a),LL(0x2d9ef7be,0x1e1f27f7),LL(0x956cff19,0xba2813eb),L_(0x000000af), + LL(0x7ab859d4,0xdb558004),LL(0xce8d95f8,0x9dc5e59e),LL(0xf1893cb5,0xa7cb9fd4),LL(0x77041349,0x53461897),LL(0xb8f3b00e,0x4b8c1719),LL(0x3acc8d5e,0xe436769f),LL(0x2b51a3ea,0x15adc570),LL(0x5fa4c1f5,0x93a4046f),L_(0x00000081), LL(0x605a315b,0xc6ca2c5f),LL(0x8e92a20f,0xe70ae728),LL(0x9e74b9b9,0x6d7cbd1a),LL(0x8837144f,0x797825f3),LL(0xa10a4e2b,0xd3f2260a),LL(0xbce98f37,0x3d85debd),LL(0x39cb6f79,0xa763a30e),LL(0xb1f85a09,0x15a4a3f3),L_(0x000001b8), + LL(0xb955d85a,0x4df25f56),LL(0xf82561fb,0x9e75f649),LL(0x8c808470,0xd7751a0f),LL(0x0c1dcee3,0x4322644e),LL(0xf53e90ce,0xd68db21e),LL(0x8f8d9278,0x72d07c10),LL(0x94db3061,0x821b176c),LL(0x2c56677c,0x51fa088e),L_(0x000001b6), LL(0xa05bed3e,0xbd496ef4),LL(0x2f3203c7,0x702f0af9),LL(0xf4ee87af,0xdeead7f4),LL(0x1e240ba2,0x6ba4a666),LL(0x45c6b9bc,0x5b558a22),LL(0x69826d07,0x82f16b78),LL(0x1d474e7f,0x1b099f67),LL(0x6f9ab8f0,0x79936b07),L_(0x000000d3), + LL(0xab96e691,0xdcb6ef22),LL(0x8cd18dc5,0x4111e26b),LL(0x563a07bc,0x3482455e),LL(0x0e2f7391,0xcb5ec4ad),LL(0x21483bed,0xc0caacaf),LL(0xa5a48441,0xc9e80f16),LL(0xbffbf280,0x90242b85),LL(0x91f37a76,0xd9544186),L_(0x000000ee), LL(0x1428479a,0xa766d6e3),LL(0xfc4b8794,0x38293f47),LL(0xa81360ec,0x31e9f867),LL(0xbe34d77d,0xdb92af31),LL(0x882df842,0xcd799976),LL(0xd34a906c,0xabb505dd),LL(0x961ddfb3,0xa3a37b0b),LL(0x4fbbb326,0xf7af85a7),L_(0x000001d4), + LL(0x1e66664b,0xbb0940d9),LL(0x3d70435a,0xff491b63),LL(0x1eb2a685,0x22d3c808),LL(0xa380de6e,0x17e44c8f),LL(0xd9df636d,0x2a35379f),LL(0xa903bf9f,0x0f809249),LL(0x387b8a0a,0xabe12175),LL(0x44dbe0cd,0xfd759d00),L_(0x00000123), LL(0x80918078,0x98196ddd),LL(0x019c1122,0x84c5a37e),LL(0x254adbe9,0x3981a4ac),LL(0xbfc928a2,0x62436eb9),LL(0x7ad29c64,0xc99f2914),LL(0x2cc9ceca,0x2f0c2529),LL(0x2d8109c3,0xbda5dcc9),LL(0xe65ae3c9,0x27c8461f),L_(0x0000014c), + LL(0xd774b2bc,0x7f2f4a50),LL(0xdb205fa6,0x977d6dff),LL(0xb6a346a4,0x39b1aa2b),LL(0xc02f5c26,0x6d520bed),LL(0x7e9b3df6,0x9fe2d1ad),LL(0x8060eb41,0xffe8b3d8),LL(0x8de43158,0x6b78c0bc),LL(0xe900b6ef,0xba5607f9),L_(0x00000027), LL(0x1979ffd7,0x656a0930),LL(0xe37eeb37,0xf82e5547),LL(0x817b9a2b,0xebe57826),LL(0x966c6b06,0xd17239c5),LL(0x0d566764,0xb6e7e211),LL(0xb736c18d,0x67f60fe4),LL(0x7f6de467,0x91c330d3),LL(0xdf003076,0x6fe1ff1d),L_(0x00000038), + LL(0xd1c84dca,0xa38e4c83),LL(0xee75b3b5,0x42633219),LL(0x96b76b44,0xa702e22d),LL(0xce624bec,0x45df636e),LL(0x087dc34c,0x00b8ab39),LL(0x7c3b41a0,0xa4c92149),LL(0xbe1f412c,0xc186c0a7),LL(0xa0e6b72f,0x6f9b9c73),L_(0x0000015c), LL(0x5b6e29ff,0x6d958194),LL(0xfceeca76,0xc25f90a1),LL(0xd246f978,0xf7a79529),LL(0x5bdce3f4,0x202fdba0),LL(0x4516ada4,0x9ccb5769),LL(0x22fea769,0x35cd1aca),LL(0x9d5e791d,0x72d93ad7),LL(0x89481217,0xd00224e0),L_(0x000001d6), + LL(0x4b1bfc3c,0x0b6d183b),LL(0xcdd1f50d,0x88770143),LL(0x721cf9d0,0xd247118a),LL(0x5a8338fa,0xfa498ee5),LL(0x33ff454e,0xa8d98087),LL(0x2107a954,0x4eaefaa7),LL(0x39298606,0xc385af5a),LL(0x3e0c503b,0xfc7e0cec),L_(0x0000015c), LL(0x3ea50970,0xa2f6c113),LL(0xbf161ebf,0x48b5f685),LL(0xbb087e9c,0x58eb481e),LL(0x3b7987c9,0xa465a54c),LL(0x6e92e01e,0xa8194344),LL(0x1e66d88b,0xb0c7a894),LL(0x40dc6c71,0x690cafad),LL(0x057f59a3,0xf02679ac),L_(0x00000130), +}, +/* digit=15 base_pwr=2^75 */ +{ + LL(0x9682d5c5,0x3a007995),LL(0x19e3233b,0xcd545767),LL(0xc78c2194,0xc744ff86),LL(0x789e51d3,0xafacd6dd),LL(0x7a5cd253,0x398cb1ba),LL(0x18b56085,0x273c4fb9),LL(0xff1bce38,0x0ba240c9),LL(0xbca7efa9,0x3bb2e372),L_(0x000001a7), LL(0x353d398f,0x563a114b),LL(0xf4adbd1d,0x90284d2d),LL(0xe9ad940e,0xe3af63ef),LL(0x61ffca7f,0x96feaa4e),LL(0xba0669de,0xdbf94ff4),LL(0xd7b8471a,0x696c5279),LL(0x1dda976e,0x0a229117),LL(0x1566b880,0xba44b588),L_(0x0000015f), + LL(0x317311c8,0x29b896b5),LL(0x025efc3b,0x60e34ccb),LL(0xbcd9f85c,0xae29c1d9),LL(0x1e85f821,0x3dcc6356),LL(0x27219e29,0x3f95824c),LL(0x9e01039f,0xd3a9843c),LL(0x8ef0f79f,0xbba44b84),LL(0x9cddb5a3,0xf0a7f537),L_(0x00000011), LL(0x97dedd3c,0xd40be315),LL(0x0d73669d,0x0706daaa),LL(0x90c605d6,0x262a826a),LL(0x2e67d62f,0x90997b0e),LL(0x5ac29c5e,0xa4dc7322),LL(0x9728fe4c,0x46c28be3),LL(0x8656b7a7,0xcf46a3d3),LL(0x318bd5a3,0x0c58ac70),L_(0x0000000a), + LL(0xc8e39791,0xd012d32d),LL(0xb1701460,0x36257f7f),LL(0x712c0989,0xbc1511ff),LL(0x948635e7,0x929c254b),LL(0x950b6b9e,0x88fae008),LL(0xc1ebc649,0xb35e21c2),LL(0x69de590b,0x0c8fd948),LL(0xc534704d,0x03df94d1),L_(0x0000009f), LL(0x7d20c84c,0x417e1367),LL(0xbe538962,0xbaa7a81a),LL(0x6d9ba3b4,0x085df8a2),LL(0x72446606,0xc1a4f077),LL(0x313aa0fe,0x443486a7),LL(0x310facd1,0x4bb29bc8),LL(0xe7424659,0x70dbb24c),LL(0x7a208944,0x2ddd11bb),L_(0x0000016b), + LL(0xe96cd1b5,0x5370e2b4),LL(0xa39d68ac,0x0b26e23e),LL(0xf98a9904,0xbe557ba1),LL(0xcef362a9,0x202765cc),LL(0xa7731e3c,0xa726d7b5),LL(0xcd815e2b,0x1c25faf8),LL(0xa6579cd9,0xdafb2e8b),LL(0xc5ec8fb3,0x648049fa),L_(0x000000f4), LL(0x452046ec,0xa04b6251),LL(0x96110b89,0xd1ddccb2),LL(0x551f88c9,0x0f26d015),LL(0x9b8bbb0b,0xb5bd39d3),LL(0x9d52ffc2,0x2dc18ef7),LL(0xab6d006e,0x142fb527),LL(0x804f61d0,0x391511fa),LL(0x9dbe5992,0xa3e717ea),L_(0x00000015), + LL(0xc1cd48d7,0x9289c640),LL(0x6b7a800c,0xc76b2795),LL(0x581d42f0,0x8ca1e81a),LL(0x4472a2fa,0xbf4b6019),LL(0x6715dfdd,0xb304a936),LL(0xf9366e36,0x391be48c),LL(0x81f74b90,0x6151e36b),LL(0xd32b6b20,0xc48b79a5),L_(0x000000a9), LL(0x366ec558,0x831e89ed),LL(0xfa6f04fa,0x5ebcfa5b),LL(0xec4ecc3d,0x6b2117a0),LL(0x7b376a81,0x90bf1080),LL(0xba3f0166,0x0993a607),LL(0xaf14e2f1,0x8cf72c90),LL(0xef21c633,0xd173a6d3),LL(0x187451c2,0x13ec5f08),L_(0x00000011), + LL(0xc5533330,0x0f29eadd),LL(0x246b16c2,0x9b833212),LL(0x9da31a63,0x297cf150),LL(0x4995a63d,0x90a2a3a8),LL(0x26054531,0x22ca8af6),LL(0x0cdf918d,0x62531849),LL(0xa5ed4b64,0x7ca9de65),LL(0xfa4fec38,0x72b35acf),L_(0x0000010b), LL(0x23d78ecb,0xf96d539b),LL(0x221e3646,0x256f3a4b),LL(0xb6bf83af,0xc408a90f),LL(0x7abd62f0,0xaefff14a),LL(0x4069cdfd,0xae41ce0c),LL(0x29824953,0xb47d1cba),LL(0xa382ab7e,0x9eb440b7),LL(0x957f6459,0x4c148b60),L_(0x0000014a), + LL(0x6b05018a,0x1086c5f8),LL(0x26130fe0,0x7b4b2e70),LL(0x68d46ff5,0x0c407c0a),LL(0x3b8c39a0,0x88577dcf),LL(0x6dc35106,0x5dafbff6),LL(0x66c847b2,0x2d675f1f),LL(0x17ebe229,0x834c9c2b),LL(0x7dd924e2,0xd5b6edf6),L_(0x0000008b), LL(0x62687b75,0xf96321d7),LL(0x6f67acfa,0x9437a2b2),LL(0xf66029f2,0xe7b46c71),LL(0x329167d4,0xb24e796b),LL(0xcf0f34c3,0x9d6e95b5),LL(0x4f9e7abb,0xcb817cb5),LL(0xb5258968,0xe5382677),LL(0xb89d3951,0xb2e2fb7f),L_(0x00000008), + LL(0x85caa712,0xe65a160c),LL(0x79edff6b,0x1ae3af0e),LL(0x7704970c,0x8b957c42),LL(0xb8aa395f,0x5f0f181e),LL(0xbdf3d6c8,0xe7d8f529),LL(0x4e626c58,0xd83fd353),LL(0x770dabfa,0xe5ada98c),LL(0xac3e65d7,0x6430730b),L_(0x00000167), LL(0x73fcec9a,0x09d0476d),LL(0x4dcf97c3,0xab9b5d71),LL(0xa56a3252,0x9648c08c),LL(0xff297fc7,0x897ba609),LL(0xff84306b,0x4c446a06),LL(0xddb64374,0xbc202d01),LL(0x97ee1218,0xf9b0f80f),LL(0xf048db33,0xb5f54bb8),L_(0x000001e8), + LL(0xcc962077,0xa010dba8),LL(0xa2fb38a6,0x31feab78),LL(0x36899fbf,0xacceb7c2),LL(0x5bdd898c,0x14de8e04),LL(0xe42d5076,0x459ea861),LL(0xe51c21c6,0xfbda1b0c),LL(0xdc95445c,0xa7d4e38e),LL(0x7947f71f,0xadcd66a5),L_(0x00000160), LL(0xcc6a9c26,0xdcdf5f99),LL(0xf1467b2f,0x2de2bfcc),LL(0x9c1ae772,0xd43f04fe),LL(0x41471183,0x043333f6),LL(0xc9a3cf00,0xf1a6e8cb),LL(0x7cc8a0c7,0xa8c3f924),LL(0x74a1cf04,0x828052df),LL(0x8be5596b,0x50a45820),L_(0x0000000f), + LL(0xfbb80d1a,0xe9851bfc),LL(0x51c40077,0xbfe0982b),LL(0x87cd565d,0xd372a1cb),LL(0xccd954bd,0xbbff7b4d),LL(0x294b36f6,0x8237c51d),LL(0x4ce0f879,0x261403d6),LL(0x569d6e3c,0xb79e0e60),LL(0xeba6224f,0xc33dd3b5),L_(0x000000d7), LL(0x5a9cfa17,0xaaf054ad),LL(0x5f93dace,0x160bbbee),LL(0x8aa260aa,0xa9f4b722),LL(0xb1b5025d,0x817d1e67),LL(0x81308a04,0xfe002797),LL(0x4afd2f00,0x680cc208),LL(0x154f68e6,0xd4b7eccc),LL(0x4cc6b0f1,0xb8976118),L_(0x000001e2), + LL(0xc8be5e5b,0x7283e8a9),LL(0x8ab94a45,0x0bb733f0),LL(0x642a6645,0x41a534e8),LL(0x57ae42c1,0xd4abc981),LL(0x8be6ea3e,0x7e62f50a),LL(0xb3f01b7e,0x98a38cc8),LL(0x8865f98b,0x7862605c),LL(0xf1a738cf,0xde02b3ce),L_(0x0000000e), LL(0xbe15c345,0xbb74e488),LL(0xc6aa6ad4,0xf6d314f3),LL(0x1691860c,0x97214287),LL(0x625fd28f,0xab878ce4),LL(0xd6ea61fb,0x8ebed709),LL(0x5a22486f,0x6db8fc6f),LL(0x0b71449e,0x46c06119),LL(0x0e8cd622,0xe44026c8),L_(0x000001c7), + LL(0x2b9ea684,0xab7fbeff),LL(0x954c6cef,0x7291b35c),LL(0x062277d6,0x7553137c),LL(0xaf482063,0xb75730d8),LL(0x710c68e8,0xb68d2250),LL(0xbae3e7c1,0xff637a2f),LL(0x6b643e1a,0x1ef002bf),LL(0xdcdc4699,0x38ab4582),L_(0x0000016e), LL(0x0f3758df,0x92bea0d8),LL(0x899ee626,0x4d7631d6),LL(0xcdd2a79a,0x2ee3aea9),LL(0x8162f50f,0x1476eea0),LL(0xc4d433fb,0x27c81475),LL(0xe8214237,0xdc969d9f),LL(0x56c76934,0x6d918c87),LL(0x7533eb08,0x56d2a891),L_(0x00000030), + LL(0xadf83954,0x1b47da4b),LL(0x8a8331e9,0x49ed1362),LL(0x59fa2b27,0x418a7189),LL(0xe8454c56,0xf46859d8),LL(0xb777077d,0xdf895326),LL(0xa1ecfc76,0x7c0bff3a),LL(0xa0d40120,0x4dc72f6f),LL(0x863ee5b0,0x6d7d35d2),L_(0x000001db), LL(0xbc4e28c1,0x51536aa9),LL(0x797cc189,0x992a786b),LL(0x424d6c36,0x0bb1db2f),LL(0x7b4a72f0,0x99596f48),LL(0xd38d470a,0x7d9d8119),LL(0xed3220cb,0x7fe52443),LL(0xd0f7efd8,0xf334c76a),LL(0x5b52a8c0,0x289cf254),L_(0x00000182), + LL(0x496fb553,0x50ef0de9),LL(0x06fa762c,0xf226f768),LL(0x770bcaf6,0x47cff6b0),LL(0xec1a0d3e,0xb780ef8c),LL(0x49872a4e,0x434df874),LL(0x82a505c8,0xfc298963),LL(0x6b56a94d,0xc037f2d4),LL(0x1721c7a5,0xef98ff94),L_(0x00000059), LL(0x5289072e,0xce001c93),LL(0x229e4010,0xb7710f4a),LL(0xb1be023a,0x4f780b68),LL(0x37373fb4,0x44ddc611),LL(0xb6129029,0xeba09ab8),LL(0x3d4bb157,0x52d1be00),LL(0x7557730f,0x5d184bff),LL(0x79506c27,0xabea8b49),L_(0x00000057), + LL(0x59ac2268,0x475e54a3),LL(0xe76ea216,0x641b111e),LL(0xe7b2f367,0x11e8b8cf),LL(0xaa1845af,0x3e44ec0e),LL(0x634d35d6,0xd9e05b86),LL(0x9fef6a5e,0xa66acc79),LL(0x6151a902,0xc7fadad6),LL(0x95ee0cab,0x4f71151e),L_(0x000000c7), LL(0xa387fffe,0xc47ffd5b),LL(0xd5a7577a,0xa66a5bd1),LL(0x1bbdd68c,0x1a4070bb),LL(0xafa4a6f8,0xb0f9b28c),LL(0x67075086,0x966afc2e),LL(0x63512dbd,0xed5912ae),LL(0x8f26597d,0xd092a281),LL(0xa415f451,0xc2ef31e4),L_(0x0000003e), + LL(0xfadaa485,0x4e004498),LL(0xbea3c389,0xaa39f9d0),LL(0xa8f46458,0x0b3654a0),LL(0xf830008b,0x4cd7392b),LL(0xa46a22ca,0xb12eb97a),LL(0x80e1d7af,0xd888b9cb),LL(0xf74c8adc,0xbb6e179c),LL(0x73b51d04,0x968eb224),L_(0x00000150), LL(0xd8317561,0x51f96d03),LL(0x9de9e100,0xfade3749),LL(0xecddcd4f,0xfc72771e),LL(0x7aa9dbdf,0x5e1bb964),LL(0xdc24156b,0xbaddb508),LL(0x0de78eea,0x6facddf4),LL(0xb1c48894,0x4a964d6a),LL(0x49c82382,0x6f258c87),L_(0x000001b1), +}, +/* digit=16 base_pwr=2^80 */ +{ + LL(0x53fa611d,0x32cc508d),LL(0xcd408945,0x60b1057f),LL(0xf3eb54e9,0x77d231ff),LL(0xbd6ea408,0xe5110313),LL(0xb9ee8343,0x85209f6e),LL(0x64924e77,0xe7fa5897),LL(0x32e258b2,0x618a6eb3),LL(0x11e2e038,0x96067c35),L_(0x000000af), LL(0x15c8ff41,0x0f221560),LL(0xef974e44,0xa1b7a3a0),LL(0x8ea1f931,0x72932b48),LL(0x720e4174,0xbb75d745),LL(0x1bf9c803,0x996758e5),LL(0x7f0b3909,0x8d83f97c),LL(0x39d56a48,0x1fac932b),LL(0x55fe1ded,0xaaf43ccf),L_(0x00000199), + LL(0x4f73774f,0x6ffb7834),LL(0xad3e7387,0xd3158a49),LL(0x8771e37a,0xb98ec469),LL(0x6f103f2c,0x1bd53110),LL(0xf8325af3,0xa434959f),LL(0xe47f875b,0xa10264ee),LL(0xcf224bc0,0x9ccf2f61),LL(0x337d33a1,0x86ce6031),L_(0x00000121), LL(0x28697730,0x6706e91f),LL(0x9179c5ea,0x79ba3023),LL(0x9aa4ed38,0x7e239f26),LL(0xaa83eb97,0xef091443),LL(0x82853a90,0x336fc4d2),LL(0x2b260d34,0x56b3a0bb),LL(0x119fbd07,0xe0f16198),LL(0x3453a3a6,0xa2af0802),L_(0x00000016), + LL(0xd1fcc92a,0x8000185b),LL(0x02ebe1f2,0x3ebcda00),LL(0x30d3e5f2,0x75cccaba),LL(0x8ea9d40f,0x108edd48),LL(0x152a6563,0x6028024e),LL(0x732e422c,0x17618296),LL(0x142e6cc1,0x9dea7266),LL(0x5d4e4488,0xb05325e9),L_(0x000000d1), LL(0x40638703,0xf3270a2e),LL(0xc29b5dca,0xbad984d9),LL(0xd2f759d7,0xad7bc046),LL(0x347ff7c2,0xa4e4f59d),LL(0xc16d4c0f,0x0a06be29),LL(0x872d14ff,0x2390bb31),LL(0xb7a5b6ec,0x66be2ce2),LL(0x408ae4cc,0x6b9b1fe0),L_(0x00000100), + LL(0x2cfafd04,0x9cc4cee5),LL(0xa99628d5,0x4764e916),LL(0x4417813a,0x9a05da16),LL(0xe423f0c4,0x2babb644),LL(0x24dca899,0xd179a66e),LL(0x894f6883,0xc157cbef),LL(0xed7756c7,0x44c30131),LL(0xcdff08e1,0x78b0a3e9),L_(0x000000ac), LL(0x3963ba6c,0x43dabd75),LL(0xbe7ba3ec,0x93626426),LL(0xd17b8f8b,0xbfcd2a78),LL(0x486d7ac0,0xaeda53c9),LL(0xfc3c49c2,0xbc99eeae),LL(0x49fb4a9c,0x12ab3d09),LL(0xdb075628,0xc0f863b5),LL(0x3d5da4c6,0x8ec31fe4),L_(0x000001e0), + LL(0xf45825d6,0x098d00b0),LL(0x4acb7a91,0xf4f8175f),LL(0xfe317cf8,0xf8155d16),LL(0x2bc9a77c,0xac3ddeef),LL(0x17520bd7,0x0aeae3c4),LL(0x44ee6fbc,0x1aaae6ff),LL(0xd8c23852,0xf47bc828),LL(0x553f42c9,0xc09b26d0),L_(0x0000012f), LL(0x6897ed6a,0x9a32ac7c),LL(0xc1e669bb,0xc8aca498),LL(0x697322f4,0x43042d46),LL(0x334625a5,0xdf16aa69),LL(0x67bda03c,0xab4b67c2),LL(0x205d341f,0x83a55d6f),LL(0x05daa2bd,0xcdfd94e0),LL(0x9ac9573f,0x1cb76afe),L_(0x000000e8), + LL(0xe51930fc,0xf074fced),LL(0x97863b91,0xc43281c9),LL(0x92d449a3,0x7a68c2d7),LL(0x063c9119,0x3b2de0b3),LL(0x55e1666a,0xf3e7d825),LL(0xf70b4227,0x6aacf427),LL(0x6c04e18d,0xc2b9b616),LL(0xaa4c82c2,0x376fa210),L_(0x0000015a), LL(0x3a29f55c,0x27de0f4a),LL(0x63844f17,0x70941232),LL(0xfd0bec77,0x3b5f4e85),LL(0x68fe79f4,0x5cbc9a57),LL(0x826a7303,0x65bb2328),LL(0xda7d2209,0x40788a77),LL(0x6978fe42,0x14c7cf99),LL(0xc2ae1a05,0x77943ce3),L_(0x0000005e), + LL(0x94afc854,0xdb0d1b45),LL(0x25e9937f,0x14c566a9),LL(0xd1cf3988,0xcd250848),LL(0xcc300694,0xbfd82b6c),LL(0x135bc75f,0xa47db4ae),LL(0x8639e63f,0x2295c1f1),LL(0x30a5e5b3,0x61f91b27),LL(0x2841bb1c,0x1335383b),L_(0x00000145), LL(0xf2dca32f,0xe1df27e3),LL(0x0fb695c7,0x721a9ee4),LL(0xc8c313d1,0x267e9801),LL(0xb288bc93,0x9aafbe12),LL(0x80d7a36a,0x5e34c2b1),LL(0x5e8b79ae,0x22efe6b6),LL(0x7da7f03b,0x552e9134),LL(0xee94e563,0xab16538c),L_(0x0000014a), + LL(0x9e50fadd,0x336db42e),LL(0x49f7546b,0xf6900421),LL(0x57093c06,0x8e00d7d3),LL(0x8d2dbce8,0xd9ede742),LL(0xd004dc4a,0x01940521),LL(0x0d3be2ce,0xdc4ae497),LL(0x1bb5cf60,0x0a6ccb03),LL(0x803df567,0xdf04605d),L_(0x00000164), LL(0xc7d8a77c,0xa05f0fb0),LL(0x1f8ad28a,0x39d0a695),LL(0xe5b908dd,0xd67e92ff),LL(0x4165f76f,0xf281077f),LL(0xc8d52980,0xc9ee2db2),LL(0x9ff0b841,0xe792e9a0),LL(0x9a5850f2,0xd74d1fff),LL(0xdbe9887c,0x468c4978),L_(0x000001aa), + LL(0x1bc8e2f6,0x17cfe85b),LL(0x645a5369,0x96452c88),LL(0xfe80ed90,0xd777343c),LL(0x53d07352,0x57827fb3),LL(0xa6c3e43b,0x80b39b9b),LL(0x2a8365b0,0xaf5a3de0),LL(0xac5e9239,0x181d4337),LL(0xea4d87d6,0x51fe7247),L_(0x000000e4), LL(0xfcc2be5a,0x0dfb5a0c),LL(0x7a7e75fb,0x15428231),LL(0x5c94ba5c,0xdf444166),LL(0xc521d0ff,0x0f639c5d),LL(0x74de2e37,0xdc1e397f),LL(0xc9356e9e,0x93e3e959),LL(0x0c9e3464,0x39201886),LL(0x6a78ad71,0x2c0f3246),L_(0x00000013), + LL(0x41b934a0,0x9207f1ee),LL(0xc358b92f,0xcc5ec219),LL(0x085c3194,0x8df4ef1a),LL(0x71e157f1,0xb527e0ab),LL(0xdd0a2a0d,0xae481b48),LL(0x0e0efe51,0x27b717b7),LL(0x5828b573,0xc2a98225),LL(0x3d2c0ff1,0x099e23ff),L_(0x0000017e), LL(0x7c722916,0xf9a97523),LL(0x11db2059,0x5fbc6f68),LL(0x162229b6,0x08c87928),LL(0xc7000da2,0xa03b55dc),LL(0x3f37b2b0,0x03e27759),LL(0x2abf57b8,0x6d3e3a66),LL(0x3d925762,0xa5335174),LL(0xec43569d,0xb9e4cda0),L_(0x000000a8), + LL(0x0c0bf0ca,0x991bdfe4),LL(0x46da618a,0x0642fbf7),LL(0x2827ab9a,0xcb3c80d7),LL(0xa4c02a4a,0x24ffe39b),LL(0xdd1a385c,0x81fb1f95),LL(0x4fe48911,0x7ed78cb0),LL(0x42514e36,0x12838990),LL(0x2822adf0,0xcd6245c9),L_(0x000001af), LL(0xc2dbb25f,0x682326f4),LL(0xeadeab18,0xa6714b92),LL(0x8194d4d8,0xec34ca02),LL(0xaf1a5a6c,0x18aa0b61),LL(0x21f3603a,0xd56cb67c),LL(0x9e65c6ef,0x64df4053),LL(0x9af9ff17,0x6e1b769a),LL(0x333abb94,0xd74c3bd6),L_(0x00000157), + LL(0xe34d8e25,0x6b395c0b),LL(0xb5b87b52,0x48432467),LL(0x7efdcd9b,0x9d4e858e),LL(0x7c0c8922,0x30d29dcc),LL(0xd7b3a015,0xe0639cac),LL(0x02eb2112,0xa2a88900),LL(0x99cb32dd,0x5ee2bf55),LL(0xe98b88fa,0xa3323b39),L_(0x000000b7), LL(0x51e66e87,0xe5c08a49),LL(0x2cf028a4,0xf8ca8809),LL(0x62209ac3,0xfaac3cb2),LL(0x91bff5ca,0x2cd5c8b2),LL(0x2f2a80df,0x2244abea),LL(0xd2c4cfb9,0x1181848d),LL(0x345b1ca4,0xc35b7b60),LL(0x3093dc03,0x815a0c2b),L_(0x00000134), + LL(0x4da0913f,0x3f060a04),LL(0xc9440b7b,0xf892eb31),LL(0x8c0aafa8,0xc25c561f),LL(0xebcb8c35,0x12770520),LL(0xe63e94d2,0x6d5c16b3),LL(0x1c828d52,0x9bde42a4),LL(0xcafff93a,0x2b7abe51),LL(0xc4851b92,0xa9fcd2b9),L_(0x00000000), LL(0x573073fe,0x24e44af4),LL(0xebda81a0,0xa695bf6c),LL(0xdee85d40,0x76c43176),LL(0xf9ff0671,0x8106e3cf),LL(0x7e8060f0,0x4ce1b2a1),LL(0xa7e23360,0x81d1ed8c),LL(0xbfa62d39,0x47f7b51a),LL(0x4292ca9e,0x9139a437),L_(0x000001ba), + LL(0xfabf78b3,0xa65d9111),LL(0xb4e47850,0x4221510a),LL(0x7528def6,0xeca10d8d),LL(0x12fd7625,0x4560fa05),LL(0x7883ec3b,0x5c64949f),LL(0x2a961e63,0x4df0271f),LL(0x4eacd5e7,0x9af4d45b),LL(0x62f25ceb,0x9ab3bb2a),L_(0x000001e5), LL(0x198ddde1,0x92cdd0ea),LL(0x6860d474,0xae51b5ad),LL(0xc69002e2,0xccd9fe59),LL(0xe138bd08,0x00304cd8),LL(0x9ec62f95,0x329feb5b),LL(0x0928d5e4,0x8a27990e),LL(0x6d9656bc,0x0f0e6792),LL(0xb7989c9a,0x3f1c862c),L_(0x0000010e), + LL(0x9c0279ea,0xa5a66086),LL(0xf622c1ac,0x7ccf9bc1),LL(0x3beaf465,0xa68726c3),LL(0x900cd78d,0x2a903911),LL(0x2be1c512,0x8832203c),LL(0xa8466703,0xbf042dcb),LL(0x1301ba91,0xa8235209),LL(0xfc06ed9e,0xe1aef50a),L_(0x000000d5), LL(0xf153a82e,0x47a0b87c),LL(0x55a0038d,0x9826631d),LL(0x6ef49646,0xe8d54ad8),LL(0x53644562,0xa200feff),LL(0x773835a0,0x5c4408b1),LL(0xb6c0a0e2,0x3199aac7),LL(0x2d23ae2f,0x4f2c5a36),LL(0xdfbd8171,0xccacb296),L_(0x000000c8), + LL(0x568d9b42,0xcf45bc39),LL(0xbca99c74,0x16aa0dee),LL(0xae132b38,0xb713a9d8),LL(0xf8d3fb57,0x139b0131),LL(0xc38156e4,0xc42164bb),LL(0x3293d5ff,0x74da5a53),LL(0x28a54d0e,0x970fceff),LL(0x56246758,0x3ef8fd2b),L_(0x000001cb), LL(0xb3941bc5,0xa0e59886),LL(0xe1a4b217,0xaa3f69ad),LL(0x23117719,0xb7b4e45e),LL(0x118c88c1,0xf1294233),LL(0xd7dfa5c3,0xf1ed8c9d),LL(0xfa2104f8,0x22213ffa),LL(0x89ed7138,0x3ea0a97e),LL(0x2cd0f857,0xf09db9f8),L_(0x00000081), +}, +/* digit=17 base_pwr=2^85 */ +{ + LL(0xb1d8c85f,0x7d77d542),LL(0x6c76648a,0xa16759eb),LL(0xae9936fc,0xd9556eeb),LL(0x239c8db3,0x1fc23af7),LL(0x3df02c8f,0xd956a664),LL(0xfec894e1,0x2a0d5264),LL(0xaaa92f80,0x14a90b90),LL(0x93d756f0,0x53d197ff),L_(0x00000007), LL(0xf62d4dfe,0x06dc80b7),LL(0x026f8974,0xdd1ea5de),LL(0xbf46ad6a,0x1c416858),LL(0x8f0f1c5d,0x6b3d82ce),LL(0xf587ce4d,0xa459159d),LL(0x3b92c19a,0xb6baca47),LL(0xda5bec10,0x600af3c6),LL(0x40e0b4be,0xc81e2b9b),L_(0x000000f6), + LL(0x66942d96,0x7d27dadf),LL(0x34d4f088,0x4ff582d9),LL(0x24ab4a3b,0x151ee4b7),LL(0x58c8094d,0xdf116aee),LL(0x1ceee559,0x6c8ad814),LL(0xf6cc0cd1,0x3d13c277),LL(0x9d41dd2c,0x75e0cd1b),LL(0xf318d63c,0xbb0767f3),L_(0x00000006), LL(0x9ef6b69f,0x1f3f3f36),LL(0x8f1170f9,0x44906780),LL(0x6338ef63,0x7495e6b8),LL(0x66dcb459,0x3f524ab7),LL(0x7ec63428,0xfd773112),LL(0x86373e0a,0x549575be),LL(0xd2af0e3a,0x9dfc2be8),LL(0x6314c681,0xffa7af5c),L_(0x0000017a), + LL(0x894f83b3,0xb01c752e),LL(0x2db47294,0x8755f110),LL(0xefd03152,0x233b9d7e),LL(0x00f8e7a5,0x69c62c75),LL(0x1ee3b5dc,0x551f4471),LL(0xa5e280f0,0xc7dd9d94),LL(0x11042cd8,0x0c2167eb),LL(0xaf8b1437,0x7f4636e4),L_(0x00000180), LL(0xd4828172,0x77598691),LL(0x12f599a0,0xcf61cb84),LL(0x0459d6b4,0x6f27cc0b),LL(0x126405e2,0xbc7fdaf5),LL(0x4a3026dd,0x0cdbba7c),LL(0x658e4a3b,0x25d0b262),LL(0xf2e795bb,0xeec95e90),LL(0xc8766509,0x52259c52),L_(0x000001a4), + LL(0x3b0becbc,0x35410967),LL(0x88f9eb94,0x20da6297),LL(0xae5dfb3d,0x527623e5),LL(0x17557d88,0xc844e99c),LL(0xb4031115,0xb6a57ec3),LL(0x4aff5aa0,0xbe7d0b59),LL(0xea2e84ad,0x1e84a37c),LL(0x947fcbaf,0x048c2935),L_(0x000000e3), LL(0xe275de5d,0x93f58bed),LL(0x503171b0,0x7aaa2b21),LL(0x7b8e1c73,0x6261f263),LL(0x620dfceb,0x1e8e8701),LL(0xdb241dd2,0xce453d37),LL(0x74e79c85,0x7db88257),LL(0x6f92bc71,0x5a2566e6),LL(0x6fb9d9ff,0x4ae0bd7b),L_(0x000001fd), + LL(0x8f215ed1,0xed12288f),LL(0xe57e6348,0x585fa9eb),LL(0xa7abfcec,0xb1b5c7dc),LL(0x12939a1c,0x76c09203),LL(0x48eb6b41,0xc4cc679c),LL(0x2a08cff6,0x4778574b),LL(0x8519401e,0xa530fac0),LL(0x93672b0d,0xe3ed0e37),L_(0x000001c5), LL(0xd47e0778,0xb789e1a5),LL(0x479a986c,0x85a2af38),LL(0x47001e80,0x5a9c69c9),LL(0x623b25fe,0x05944f83),LL(0xecb76557,0x1f6667a7),LL(0x0cb584d5,0xc3bb24b7),LL(0xdaf97923,0x7f09e6d8),LL(0x00ac1be5,0x6d082075),L_(0x000000db), + LL(0x8445a8cc,0xcbbf6efa),LL(0xd82adf79,0x6f9a1c1d),LL(0x1d21df42,0x47e0a609),LL(0x1902bf5b,0x6c507d72),LL(0xd92481e1,0x732e3603),LL(0x4e995e7f,0x0a9f3c6f),LL(0x57c9de5d,0x171e968b),LL(0xace10341,0x27d7d96d),L_(0x00000173), LL(0x415a7d4e,0x093b8c99),LL(0x68806375,0x999003af),LL(0x0227279e,0x62158e85),LL(0xd41629b8,0x3479df9d),LL(0xa2d25c33,0x3570b3aa),LL(0x61de636e,0x4eb04a19),LL(0x71b3bbeb,0x0ce04d19),LL(0x4c30f9b9,0x15492470),L_(0x000000b3), + LL(0x820638ac,0xcc212f6f),LL(0xf1e3cf43,0x2c575c09),LL(0xdd247286,0x0980e757),LL(0x812b4ec0,0x19cba5ed),LL(0x883e3e28,0x4641e487),LL(0x0bb816e7,0x6c96b70b),LL(0x461f03f5,0x26aaf764),LL(0x918835cd,0xe1cfcddb),L_(0x000001a8), LL(0x2da8f566,0xb11711e0),LL(0x3f7282c1,0xed552895),LL(0x44826000,0x25594479),LL(0x065389a8,0x70867768),LL(0x96f127c7,0x52ffb2b6),LL(0xaa1f5abe,0x0831bd74),LL(0x9739a178,0xf0a510a0),LL(0x6154e726,0x36902e78),L_(0x00000177), + LL(0xf7a9efa0,0x42cd0f9e),LL(0x7bbdb010,0xa3fbb175),LL(0x1996a380,0xf39db731),LL(0xbb69e651,0xf3f08146),LL(0xec6679cd,0x8679b0b6),LL(0x60847478,0x90d1ae26),LL(0x883e5a59,0xa5e209ed),LL(0xba61924f,0x3c755c0b),L_(0x000001de), LL(0xebae92ff,0x847c1f82),LL(0x8ace9c6a,0xa1434ccf),LL(0x857d9026,0xad864d4c),LL(0x0ee5b0b7,0xf613e221),LL(0x86a35718,0x91165b2c),LL(0x55984d67,0x080d19fb),LL(0x15401901,0x3389eccf),LL(0xd99a0e8b,0x8b509b98),L_(0x00000060), + LL(0xfa05d78b,0x7c660e1a),LL(0xfd68d650,0x85aa25df),LL(0xe307472b,0x3713d00e),LL(0x3afed55d,0xc091f93d),LL(0xecc3137e,0xa9f9d1a2),LL(0xa3d44f8a,0x32a1cdec),LL(0x4344089c,0xc8d64b46),LL(0xe3575142,0x11bd5244),L_(0x00000102), LL(0xfd1a1cdc,0xe4cb9635),LL(0x8fa8648f,0x3dc52f80),LL(0xdf8e13dd,0x058fc1b8),LL(0xc1ab282d,0x3abf2a6c),LL(0xf290d505,0xfb0841a1),LL(0xea29f4f4,0x3d94894a),LL(0xb691fb1a,0xf7a1cc29),LL(0x0da25d00,0xfc4326f1),L_(0x0000018c), + LL(0x9155b117,0x1b3c8c62),LL(0x1ffd1435,0x58c3116e),LL(0x54a96e6c,0x90a8d92e),LL(0x623a9ece,0x891efe6b),LL(0x66715556,0xe72489c2),LL(0xd3bfc0b3,0x5b00b58c),LL(0x8cf3d04d,0x06c601e4),LL(0x71f460fb,0x26e3ef6c),L_(0x000001d1), LL(0xe91afddb,0x626e2af2),LL(0xa51ff90c,0xec49ef66),LL(0xd3f82493,0x704277da),LL(0x9f0e6e8a,0xca17ce54),LL(0x80cb9b26,0x62bbba3d),LL(0x39fb568b,0x7ff82aae),LL(0x978a0c13,0x9cbfd867),LL(0x6bd90fa9,0x07be1717),L_(0x00000151), + LL(0xc31dd728,0xd809dd1b),LL(0x12107a4a,0xc16d6592),LL(0xdc70467d,0xf8d5c83c),LL(0x273b2243,0x596e052c),LL(0xeab68bd9,0x8aaa0dde),LL(0x11cdb329,0x39baf3e4),LL(0x65459e78,0x004f292b),LL(0xc654b1e0,0x4da9e734),L_(0x0000014a), LL(0x58fc53ab,0x1ee009d1),LL(0x45bc8ab1,0x24563a0d),LL(0xde252f6e,0x04322137),LL(0xd1d577ae,0x29c5297d),LL(0x554880c2,0xf29e1a9e),LL(0xfc0d4b7e,0xa08f14c2),LL(0xe648399e,0xfd1007c8),LL(0x23bed899,0x2a7303ca),L_(0x00000010), + LL(0xa4c512fb,0x2e60230d),LL(0x883ed27b,0xf8aee1ce),LL(0x57a9715e,0x9d84b9da),LL(0xa58cae2e,0x59971acb),LL(0x62ef042b,0x5b1190ec),LL(0xa8c70cf9,0x1ecd90c6),LL(0x1e5cf5d1,0x0a20a7a4),LL(0x3aec3e16,0x7baf5a6c),L_(0x0000006e), LL(0xc8d347ac,0xf18a3f41),LL(0x0debc30a,0xc54674cb),LL(0x2dbc3b83,0xc265a6e5),LL(0xa8033fd7,0xf450415c),LL(0x2a50f527,0x6adf277c),LL(0x81475ec6,0xff0d3a36),LL(0x5f2c676b,0xdffe6c53),LL(0x0c1f159c,0xf5ad7f78),L_(0x00000106), + LL(0x4f0af995,0x00058fc2),LL(0x8bb729ca,0x48246fa7),LL(0xff563f60,0xb23219d5),LL(0x8c64a3a5,0xf34e49ac),LL(0xe82036c1,0x9d2397f6),LL(0xe392c964,0x58216601),LL(0xbeda885c,0x3d7f9573),LL(0xe289f5e8,0xb1917dc6),L_(0x00000169), LL(0x30057807,0x6d9791a6),LL(0x37eb92d1,0x066237a9),LL(0x7995f34e,0x764ae778),LL(0x8d994f01,0xde5ca0e3),LL(0xaff07e24,0x199bd467),LL(0x544454f0,0x4bf2e809),LL(0x603eb80f,0xe5054850),LL(0xc95e16ef,0xa4a6c672),L_(0x000001a9), + LL(0x8c50d780,0xc8db0700),LL(0x32a1b788,0x45d0d169),LL(0xc842373b,0x87531f66),LL(0x674b4407,0xdbe71b0e),LL(0x8189664d,0x42dd8323),LL(0x7cdb6aec,0x8df2c5a6),LL(0x86c397bc,0x86388fa4),LL(0x110db0bc,0xf6d18eb8),L_(0x00000086), LL(0x5a491f30,0x23e50391),LL(0xa2dcf957,0x6ae2419f),LL(0x1c7362d5,0xac9caab7),LL(0xd238a731,0xbef3c44d),LL(0x28f6d7ae,0x5c3ea7d4),LL(0xdacef1fe,0xd654307e),LL(0xb31d909e,0x01625227),LL(0x6d2db310,0xb2421d90),L_(0x000000f9), + LL(0xb66d61ea,0x4083d41b),LL(0xf3bbcb07,0xe8c0bfc8),LL(0x91274928,0x9c0b763c),LL(0xc81b7765,0xbe1076e4),LL(0xbf368625,0x9568943e),LL(0x8c2112ae,0xb0cd4c71),LL(0xf70e5fb5,0x8fe2ec7e),LL(0x76a5c64b,0x867a5527),L_(0x000000b5), LL(0xd3886d7d,0x3faf5b44),LL(0x41c29ed5,0x50ffd8e6),LL(0xcb155068,0xa9b2855c),LL(0x28ae527b,0x9d8e8d01),LL(0x2a092960,0x5cbf1edf),LL(0x66ffe099,0xb212b2a8),LL(0xd340e610,0xfc0600f9),LL(0xa2f5aba0,0xb08ba7b8),L_(0x0000005b), + LL(0x604c6624,0x0ca526cf),LL(0xa9ed0f7a,0x125b33af),LL(0x1dd685ac,0x516f5290),LL(0x17e7ff8f,0x927c416e),LL(0x720475ad,0x0fc77cc9),LL(0x67e1e919,0x3aec0717),LL(0x6652fcb3,0xcb2653df),LL(0xd80f0d48,0x8bf16720),L_(0x0000006c), LL(0x0590fcd7,0x08404c44),LL(0x7f43e4e4,0x9c42e337),LL(0xdefb2272,0xc3b37e10),LL(0xdaf241ae,0xc795c866),LL(0x0a07c892,0xbf4d3079),LL(0xb2425f5f,0xa5db075b),LL(0xb7cb5830,0x0875f161),LL(0x93c95089,0x91cad664),L_(0x00000065), +}, +/* digit=18 base_pwr=2^90 */ +{ + LL(0x7dddacd3,0xc3797559),LL(0x266f6975,0x22dfb9d9),LL(0x599f544c,0xdb081480),LL(0x839c2be6,0xeb8ec462),LL(0xb5cdf12a,0xd9d49cd3),LL(0xa917fb29,0x96146a8b),LL(0x233b216f,0xd936c0b8),LL(0xf0abf1a9,0x8c8a45a2),L_(0x00000187), LL(0x0356029a,0x0f5dd64a),LL(0xd1625aef,0x056528c2),LL(0x5ff56fc7,0x9b293d67),LL(0xa5c323cb,0x02b295cc),LL(0x4d697cbd,0xbc712910),LL(0xe4eb4b02,0xc1e4d83f),LL(0xa4e9327c,0x9c23cdd4),LL(0x5af46cdc,0x94640699),L_(0x0000013f), + LL(0xc9de104f,0x6147414f),LL(0xaed9435d,0xbd16db82),LL(0x62ff16a9,0x3b07e71a),LL(0x52dcaf4a,0xf9456ee7),LL(0xea0d3e3f,0x1d78dd65),LL(0x64901fef,0x31145bf8),LL(0xbc9f4225,0x366fd367),LL(0x81cb13ee,0x290083f4),L_(0x000001e6), LL(0x6e77ebe4,0x5232088e),LL(0xc5e887c8,0xcc7b3f38),LL(0xf005e149,0xe1bede78),LL(0xe8c89874,0x2dfeaf32),LL(0xcb4e28c7,0x3cb0a4d9),LL(0xda48c711,0xaf7daba5),LL(0x1fe289a0,0x0d3633ab),LL(0xbd5d0dc0,0xc0b05c86),L_(0x00000001), + LL(0xd5e4e9c6,0x85849a1c),LL(0x74861653,0xe1eb1a35),LL(0xaf98abe0,0x53b40a6e),LL(0xf083ec36,0x74acbc0b),LL(0x0b5a921d,0x28a65b06),LL(0x5764e30a,0x5588eb5e),LL(0x62277d5e,0xc8da671e),LL(0x39cae462,0x53bbf492),L_(0x000000a9), LL(0x0635b866,0xa6baa014),LL(0xd98a134c,0x31e23fcb),LL(0xdbe32368,0xa8c7a352),LL(0xef82abaa,0xa98ba793),LL(0xcbb55844,0x1a07e161),LL(0xaf3169a7,0x4991ee4a),LL(0x5cfbe290,0x0c980dd6),LL(0x56a21524,0x1c07cd7d),L_(0x000000cd), + LL(0x06de0083,0x07567a7a),LL(0x045155d8,0x351697cf),LL(0xce8bb246,0xcba64633),LL(0xdd5c2900,0x24297174),LL(0xe7f044c7,0x0c3a3851),LL(0x48c830bf,0xd35ff595),LL(0x817a26f0,0x7d923f53),LL(0xd93d8b02,0x2b3dd7ca),L_(0x0000002c), LL(0x924bf9f6,0xcc94cbf6),LL(0x986d299b,0xdf6f5c09),LL(0xf89ccb5a,0x5aee26f4),LL(0x18699f82,0xc1b545bb),LL(0x6595e656,0xb0d22aa5),LL(0xa3953fae,0x6abcba5e),LL(0xa9580b4b,0xd4e240d3),LL(0xef465246,0x13b6fdf7),L_(0x00000186), + LL(0x69351015,0xbd979035),LL(0xf355f70f,0xc5a87f52),LL(0x1b7365b1,0x18c500a7),LL(0x2dd3210c,0xf547e418),LL(0xc57a734c,0x391d8bfc),LL(0x928abb19,0x5ec8dcf1),LL(0x3e6991b5,0x7de9b3d1),LL(0x9e25eac7,0x6ebd2b96),L_(0x00000022), LL(0x77d1be34,0x7d70fd72),LL(0xae0eb8d2,0xf2114ba3),LL(0xb57b0ed4,0xe573b783),LL(0xc65906cc,0x0b2b3d21),LL(0xfac74740,0x9dbb25b7),LL(0x6157bc5c,0x6859f85a),LL(0x21a21340,0x0df614a7),LL(0xbfacee18,0xe4709b5a),L_(0x000001b2), + LL(0x96a37abe,0x15b4091f),LL(0x31ad054c,0xdccd6f0a),LL(0x57e2c372,0x27698a20),LL(0x9a3a4ffe,0xcc03de8d),LL(0xc754fb8f,0xcd17cdf6),LL(0xa956ebe7,0x7cb9ef2e),LL(0x9d18d38c,0x190daaea),LL(0xdc66f2ca,0x7d783383),L_(0x00000030), LL(0xcbc69492,0x727ea062),LL(0xfefa898f,0x773b6e5f),LL(0xfbea71e8,0x7460be92),LL(0xc9254b56,0x318caed7),LL(0xc982d8e5,0x7f1a16b2),LL(0x9a4da06d,0x38233ae7),LL(0xe149d876,0x3b8c2af7),LL(0x70fc3c7a,0xec4a5565),L_(0x00000154), + LL(0x73053e0b,0x5f08f3c6),LL(0xd02192ba,0x05cbe08a),LL(0xbb7b43e6,0x4a339a92),LL(0x2b4034e6,0x1c6eea28),LL(0x0145dd56,0xb3481662),LL(0x819f74ea,0x5fdab086),LL(0x2d08d669,0x401f4d8b),LL(0x87d855a8,0x71a3977d),L_(0x0000007c), LL(0xf6c94b12,0x787e1654),LL(0x7f231760,0x1dcce655),LL(0x2908ba05,0x10acadce),LL(0xe5a3ffa8,0x60f31016),LL(0xe1211553,0x34c2a1c2),LL(0xfe1b8dad,0xa81d35d3),LL(0xe7230bfc,0x37a78d41),LL(0xcfeaf774,0x6e60ae9c),L_(0x000001f5), + LL(0x725af512,0x898a64cd),LL(0xda668233,0xc8516d6d),LL(0x1f40d7d1,0x5fb1f564),LL(0xd5a115e0,0x7906c2d8),LL(0x496ac4ad,0x6f4efe00),LL(0x3643f707,0x6c892d97),LL(0x414f5838,0xe34b14c4),LL(0x7c2d83c2,0xc00d08bc),L_(0x00000037), LL(0x6bebe71b,0xca445165),LL(0x219e2e5b,0xcbf3a3f2),LL(0x118227ea,0xf1a84019),LL(0x6017e2ce,0x58a5f9de),LL(0x1922122d,0xf1ecfa6e),LL(0x98696eb0,0xd3df6fb1),LL(0x54826be2,0x8fefc088),LL(0x02dea006,0x77c29791),L_(0x0000014e), + LL(0x368b75f9,0xe65ce508),LL(0xa5fec659,0x0b177612),LL(0xdd1bcbef,0xfc84b3bf),LL(0xe4569388,0x984ce0ec),LL(0x163caf98,0x9cc1f201),LL(0x3fa88dda,0x39bac4dd),LL(0x6c254803,0x70562e22),LL(0x75a93dd1,0x2cbfcecc),L_(0x00000095), LL(0xfd7cf39c,0x0ce600b2),LL(0xe3dc17db,0x5b060f2c),LL(0x5399ddf1,0xd74988f6),LL(0x83e38dff,0x2b96b9f4),LL(0x87e221d8,0x2c298bc9),LL(0x0753b765,0xd6bd45f4),LL(0x27e5b1a1,0x18ca1da9),LL(0x41853811,0x7849f1e1),L_(0x000001bd), + LL(0x90c22eba,0x226b8a15),LL(0xdb333954,0x4e3975a7),LL(0xaa52c0ba,0x74176c01),LL(0x1fd9d014,0xa1a2b6b0),LL(0xf61b81e1,0x28415db7),LL(0xd3614b3a,0xa0f6000e),LL(0x9e00e5a5,0x5cf34986),LL(0x9efe446d,0x0b69d383),L_(0x0000001b), LL(0x245ecf44,0x2d0c1e46),LL(0x1f77c4cb,0x553d358a),LL(0xe9f129ca,0x071ebad1),LL(0x088b2769,0x8eb8c2d8),LL(0xf3219a51,0x877b3a25),LL(0x17c5431a,0x9c4b8adf),LL(0xeac2ff93,0xffc69d68),LL(0x23d158f5,0x2e0840d4),L_(0x000001fa), + LL(0xb9d98a1d,0x6d2a4e6a),LL(0xb45f9376,0x6e78c290),LL(0xbff42b2f,0x5fb9af74),LL(0x2af23d44,0xaebdf547),LL(0x7deaa238,0x93a6597a),LL(0x25df5576,0x5dcbd040),LL(0x6f6b456f,0xa9249467),LL(0xb63da0b0,0x7f719d63),L_(0x000000c8), LL(0x15e8e55f,0xda677478),LL(0xcb218f05,0xb119afe4),LL(0x3e0b4052,0x816ad70e),LL(0x342ea279,0x14498c92),LL(0x7acbb776,0xc4e09c12),LL(0xbf8e66de,0x0f290835),LL(0x3edfdd90,0x8ca19f41),LL(0x794255ee,0xa1bde671),L_(0x000001cb), + LL(0x5a5b22a9,0x4848eb20),LL(0x80f21ad1,0xde0fb7d6),LL(0x8e50bcbd,0x4c1119fd),LL(0x83d6f607,0xf9f2e435),LL(0xd9961c79,0x765361f1),LL(0x625f26bb,0x008a463e),LL(0x2b47c8db,0x87cd134f),LL(0x328c3977,0x36eea7ef),L_(0x0000009b), LL(0x3894ce29,0x3d664722),LL(0xdfe036fb,0x206e887a),LL(0x67daf1eb,0x72f017c4),LL(0xf7db19b3,0xad33a99e),LL(0x06ea7ba8,0xf55c0da8),LL(0x14bd637e,0x9b12c024),LL(0x59864973,0xd282f3bb),LL(0x55feed3d,0xce69b372),L_(0x000001ec), + LL(0x79ad5e39,0xdf10c34e),LL(0xb43675e3,0xb8ad4110),LL(0x19590538,0x801f28aa),LL(0x239ed388,0xc602d7b2),LL(0x3a5a6ad4,0xbdc9fad5),LL(0x62b5ae49,0x135d222c),LL(0x5042d74a,0xc3f94c1d),LL(0x28ba3dd9,0xe0a1ec48),L_(0x00000110), LL(0x6cd1e311,0xceb5a088),LL(0x77f171c2,0x8f737348),LL(0x31242fc1,0xdbca643a),LL(0xe44bcbd8,0xcd573afd),LL(0xbad62d6a,0xc4c9f268),LL(0x5d49ae75,0x98fad2cf),LL(0x9b8f817e,0xd8431494),LL(0x96bb2753,0xc49e4432),L_(0x000001a8), + LL(0x87036cde,0xa4f9f22d),LL(0x2ccfc8e8,0x4cd46e2b),LL(0xa48136e4,0xa95491dd),LL(0x8818327f,0x6393487e),LL(0x7a069a28,0x98ab19e5),LL(0x5c879344,0x02491e0c),LL(0x82e60002,0x2abf1999),LL(0xc0e10b00,0x82724899),L_(0x000001ed), LL(0x9fd75e21,0x4897df79),LL(0x688c2193,0x43479e7f),LL(0xc705deaf,0x5b79a04e),LL(0x068f9205,0xb2c04a79),LL(0x3a42239d,0x0245715d),LL(0xa99eca86,0x55284faa),LL(0x9f983742,0x07d9bbaa),LL(0x5fe3bb74,0x5a19431d),L_(0x0000014c), + LL(0x0f3f56ca,0x99e63ac2),LL(0xac6175c0,0xcb3bdf62),LL(0x0d4bf222,0xd9e5f622),LL(0x83e1cd77,0x4ca3e4a6),LL(0xde0dce61,0xf80cd49b),LL(0xeac1e293,0xf190c10e),LL(0x901aa7ce,0x30eda1d3),LL(0x25f2f0a6,0xa509dab1),L_(0x00000199), LL(0xe9fb14f7,0x0ab974ef),LL(0x9b6b41ad,0xcc8fc9d8),LL(0x20269236,0x5472afda),LL(0x81034020,0xb7eb7c83),LL(0x26dba78a,0x7c59479d),LL(0x81e829eb,0xb0ba6216),LL(0x1549f5f2,0x843a5c45),LL(0x18302134,0xa2709e3b),L_(0x0000014c), + LL(0xc1370929,0xbde5f81a),LL(0x8f81e9a3,0x7f4da051),LL(0x9720014a,0xa02bf073),LL(0x9ff1f457,0xe074553e),LL(0x3a46ea6b,0x64351eaa),LL(0x7e32f0dd,0xd22b22b2),LL(0xb488462f,0xcafc2c23),LL(0x566dddda,0xfb75908f),L_(0x0000015b), LL(0xb5b9a118,0xe8987332),LL(0xfe94dfd9,0x4f63b44f),LL(0xf9b91bc6,0x0dbd772b),LL(0xecb77f43,0xd580392a),LL(0x2fb67ddf,0x1ddc69fb),LL(0x2fdb69c9,0x9ea2314d),LL(0x754b9b9f,0xe624f23e),LL(0xf3f2e9c2,0xc6e677e1),L_(0x000000a3), +}, +/* digit=19 base_pwr=2^95 */ +{ + LL(0xb1001608,0x14f438f4),LL(0xd13ff0c3,0xb8e45f22),LL(0xd55796ec,0x3dd2e2bb),LL(0xf830ab87,0x9eb71d33),LL(0x14a36478,0x83167e0b),LL(0x46513aa4,0x793502c2),LL(0x03e86d3a,0xb0fe98cb),LL(0x4eb2db2b,0x0404a0ec),L_(0x000000d7), LL(0xc6b60bcc,0x8f4384c5),LL(0x0cd19a5c,0xc19b3257),LL(0x1c33b468,0x9210942f),LL(0x29ccbac3,0x36048a2a),LL(0xd4ffa97d,0x4f69ef5f),LL(0xcd6b0a67,0x82d0ece5),LL(0x13229739,0x4bce1b8b),LL(0x491493bd,0x4d6596bf),L_(0x0000003d), + LL(0x13acae0c,0x560db5e8),LL(0x1117f6d4,0x8e19b583),LL(0x1106059c,0xe8232c57),LL(0xc78f908c,0xd0f09782),LL(0x4a24aa92,0x8bd0fcb6),LL(0xd766becf,0xf59977e3),LL(0x155f53d2,0xfa9a727f),LL(0x49389ae2,0xff877e92),L_(0x00000120), LL(0xd2d44588,0x9985d510),LL(0x3b4e5204,0xe4788dc7),LL(0xaa68342b,0x8a0ca8e6),LL(0x7b14f89c,0x66874892),LL(0xf19eb3e8,0xe17375dd),LL(0x5e5f8b7c,0x652a41d1),LL(0x2912af54,0xa86a7275),LL(0x5ab9a777,0xbaf706d8),L_(0x0000003b), + LL(0x0d6ee2a7,0x8ca944bc),LL(0x4852fbcb,0xbefff557),LL(0x0369a497,0x3e3736fb),LL(0xe2bb7551,0xd21615dc),LL(0xe6d3b727,0xe319eb4c),LL(0xe8de5f7d,0x48fe4856),LL(0x592a3396,0x7516380a),LL(0x80dc9aef,0x51f52af0),L_(0x00000151), LL(0x7aa71f54,0x98c6421f),LL(0x38523c52,0xec51f4f2),LL(0xf58cc95f,0xbc6c8082),LL(0x36ef370c,0x7bc605a3),LL(0x8ac270e3,0x83d78da4),LL(0x0412498a,0x6de54abd),LL(0x66b38131,0xdb62d8c6),LL(0xe06d3c3f,0xf5e45a86),L_(0x00000129), + LL(0x42a7b358,0x4517d395),LL(0x53d2cbb1,0x5b733d69),LL(0x44a3ef5b,0x472126ff),LL(0xee076565,0xa4a1334d),LL(0xb26c37b2,0x12573d17),LL(0xb5b29517,0x129c2c7a),LL(0xd328148c,0xa2c72b08),LL(0x08907f5a,0x1d10e103),L_(0x0000018b), LL(0x154b57d1,0x5e159666),LL(0x9359d888,0x4827d5dd),LL(0x0281b6f1,0xa475f3a4),LL(0x6b19bc4b,0x2eef4469),LL(0xdc6dfbc3,0x782b50db),LL(0x9ef4383e,0x0583236a),LL(0xd7320845,0x7767db3f),LL(0x0dd190b0,0x3c0278a0),L_(0x00000115), + LL(0xc54f57bf,0xb03b1675),LL(0x97e68374,0x1c499646),LL(0xae2f3284,0x74508785),LL(0x0255084b,0xf1921ad5),LL(0x6ea40714,0x2aabd8cb),LL(0x516433de,0xd1c8abdf),LL(0x5d2d8ded,0x8f5d7b6f),LL(0x421e5a19,0x2d6ae9c5),L_(0x0000007b), LL(0x99149c91,0xd9a89463),LL(0x4310dcf0,0x0122d1b0),LL(0xd2489f91,0x4e7b58d3),LL(0x655fdd5d,0x40c4379c),LL(0xc7862d42,0x6da55b4a),LL(0x975cc64c,0x1e1d3862),LL(0x84484f68,0x301cfa3f),LL(0x2a16eb0a,0x20ebbbdb),L_(0x000000e4), + LL(0x5456251b,0xc0ba905f),LL(0x5b3d8d39,0x62a268ee),LL(0xea2a0a44,0x094457cb),LL(0x80f032f3,0xab36ceff),LL(0x8790739b,0x21b0fdf3),LL(0x40209bce,0x38249dc8),LL(0xf0c1c8e0,0x213ecb4d),LL(0x70c51d81,0x2b025e0d),L_(0x0000003c), LL(0x4b899f83,0x493bb32c),LL(0x798bfbf2,0xf622fccb),LL(0x28838277,0xc2594827),LL(0x5cbe5b67,0x2c07c4dd),LL(0x2c4c703c,0x1c19526a),LL(0xdcd0df4c,0xed390177),LL(0x57a743a1,0x3a4c5274),LL(0xac32bea6,0x1c302e78),L_(0x0000015d), + LL(0x0eb650df,0x18bc0976),LL(0xa3094433,0x7cfa690a),LL(0xca509300,0x660a165b),LL(0x9645aafc,0xdcef6d7c),LL(0xd90c6f35,0x07a41bbc),LL(0xad0dd448,0x702e476c),LL(0x5c881f7f,0x2185a821),LL(0x9c91260c,0x0622b914),L_(0x00000063), LL(0xcb1b5333,0xfe64e60a),LL(0x59c3b83d,0x192bd9cf),LL(0x5bdb24df,0x69659379),LL(0xa9cbc111,0x437b31e8),LL(0x2954601b,0xb7b5589c),LL(0x13c392f3,0x97064b9e),LL(0xf1845fb5,0x8d803336),LL(0x7ea9c980,0xea2d2221),L_(0x000001ca), + LL(0x828ff0ba,0x23fc5003),LL(0xa9841f43,0x62407436),LL(0xc6f35f8a,0xfc260a1f),LL(0xf5e9286e,0xe74c4b2d),LL(0x504bfafb,0x7cb3568b),LL(0x3548e504,0x1af9dbcf),LL(0xd92aaad7,0x85e423c5),LL(0x2d182410,0x94d1d884),L_(0x000000f8), LL(0xf29b75d7,0x9975b2a3),LL(0x55f78348,0x0939eec5),LL(0x92b31a41,0x23255263),LL(0x4a1b7bc2,0x65a25c26),LL(0x283464db,0x08fc1aed),LL(0xecd1a9b7,0xea335c70),LL(0x90a7a2a0,0x9f14ffdd),LL(0x7fd21f2e,0x9566dadd),L_(0x000001b4), + LL(0x553ea74d,0x578b8f88),LL(0x2bbf7009,0x8c4ed5a7),LL(0x30e7b9aa,0x30aeacb4),LL(0xe07a93f1,0x58d02f17),LL(0xbcf5fa42,0x7c6f83ed),LL(0x228f3e47,0x91d5ba0c),LL(0x815eef51,0xe37c3b79),LL(0x0528cf82,0x507b5dce),L_(0x000001f2), LL(0x9a97a830,0x5a682d56),LL(0x14eac39b,0x72ce45fa),LL(0x90b36bc5,0x5f62d8bb),LL(0xadbc91ec,0x9d5e2385),LL(0xfae2e6dd,0xef20a2d0),LL(0x380bdc5e,0x9c9037ea),LL(0x2d48d188,0xdd58b76d),LL(0x6a4ce00a,0xb6e3d5d7),L_(0x00000090), + LL(0x545376f5,0xfffa5757),LL(0x64cbfd55,0x1f1c3ae1),LL(0x8a854545,0xd0be9705),LL(0x7272e007,0x2a8f4c49),LL(0x254138ed,0xc97ed736),LL(0x15e864c7,0x83df5162),LL(0xb624fc1b,0xf4114fdb),LL(0xcc0313aa,0x8c7f0423),L_(0x0000007a), LL(0xc8d276d9,0xa4ed76ab),LL(0x3e74f599,0x8b381bfe),LL(0x25d1f92d,0x9e406956),LL(0x06a5a359,0x869bdf5e),LL(0x5afaf671,0xaec86f62),LL(0xbbcc12cd,0x0dd1d724),LL(0xda751689,0x21630603),LL(0x409b6925,0x1f15a18b),L_(0x00000054), + LL(0xcba2ca51,0xc7b9233c),LL(0xff6a30c8,0x6ecefcff),LL(0xf3d40688,0xfad25722),LL(0xc51d4aa6,0xf9c6f8fa),LL(0x3fae03f2,0xc7864470),LL(0x5ca7328e,0x4201ca64),LL(0x4b3ff739,0x08cad454),LL(0xb2721cee,0x43bf4523),L_(0x000001d1), LL(0x2952e7ef,0xaf0e573e),LL(0x2a8f8fba,0x3f5e67f8),LL(0xae4ce46a,0x616a72e1),LL(0x906994c7,0x3a7f4aec),LL(0x055b94c2,0x8bb69b90),LL(0x67e39f74,0x4eead34a),LL(0x7a7cada3,0xf92b70d7),LL(0x9b22abe2,0xca616691),L_(0x00000092), + LL(0x32e5d612,0xd5e1c4e6),LL(0x7ad659b7,0xe2f2a298),LL(0x3f7f338d,0xaaeb06f1),LL(0x6d9b55a5,0xa60e84f2),LL(0x30c6f8b9,0x9d105631),LL(0xd017d58e,0xfa41e760),LL(0x9e20b973,0xb2f4acf3),LL(0x840eaafd,0xec9c6ab5),L_(0x00000101), LL(0xa5a6302a,0x22f4549b),LL(0xb140b897,0xc2510a98),LL(0x3e099225,0x9117bbe6),LL(0x18af31b1,0xba7147bd),LL(0x68bb5c46,0xf0f540e3),LL(0x29d33114,0x1c9aeacf),LL(0xe59588a0,0xeb2d0e67),LL(0x29ef0e25,0x4bb1b8d0),L_(0x0000018e), + LL(0x3f8475bf,0xfaf6783d),LL(0xd0223b63,0x4dcebc43),LL(0x6472ecb7,0x6ff8245b),LL(0x1d3fdb09,0x3439a621),LL(0xa67bac9a,0xd009a2d7),LL(0x94c65387,0xf330caae),LL(0x19a02e17,0xa6f7e5f9),LL(0xb2396a55,0x5fef60ca),L_(0x00000067), LL(0x92275ac6,0xefd6a160),LL(0xefa0d1c9,0xbd7c0837),LL(0x2ede1f4e,0xfcadb9c0),LL(0xf4756366,0x47882726),LL(0xcd42f3e5,0x5c040ce2),LL(0xa61b16f9,0xdb84713f),LL(0xc4ef07e5,0xe43320a0),LL(0xecb2b8ce,0xac70be15),L_(0x00000090), + LL(0xb5440b5f,0x0457915c),LL(0x89a3e1eb,0xe08cc88e),LL(0xed12c670,0x89133ab9),LL(0x1faeab1d,0x15d9bc0c),LL(0x3c4250f6,0x881504d6),LL(0x084c8e8f,0x3ead62cd),LL(0xaf76dbe5,0x49cfac6a),LL(0x85bf1dcc,0x007ea0b8),L_(0x000001c7), LL(0xfc505153,0x47472352),LL(0x123835c7,0x80692fa2),LL(0x67bab29e,0x8379c2a8),LL(0xc2ecca00,0x9065aafb),LL(0x32da9779,0xda605d2e),LL(0x421bbbfb,0x12432283),LL(0xbdc2e115,0x9c126b9d),LL(0x437a9d89,0x7ce3f8d6),L_(0x00000146), + LL(0xa8e68c25,0x5b6dacf5),LL(0x7ae17558,0x1b0157e6),LL(0x314ffaf0,0x03819c6a),LL(0x9d2d7013,0x0e14b9b1),LL(0xc8c2b439,0x91c83635),LL(0x9f636f0b,0x98fed93b),LL(0xe579e47a,0x23b6808c),LL(0xaeb0f547,0x8d044288),L_(0x00000166), LL(0x8b1f49d3,0x891f32da),LL(0xf965270d,0x655e3634),LL(0xe11f64ad,0xacfd673c),LL(0x1b496453,0x68fb4507),LL(0xdf15820b,0x64f7f419),LL(0xd816e8bb,0xfdaf2edc),LL(0x0858d605,0xd9f619c9),LL(0xbbe07451,0xb9e75198),L_(0x00000002), + LL(0x4a1ee705,0xe000c97f),LL(0x5fa6cdb3,0x595ed0a5),LL(0xfd5fcd60,0xa02a23c6),LL(0x76e522bd,0x61844a1d),LL(0xbaf8c003,0x0c6c179e),LL(0xd0a47af4,0x6aa1a6cc),LL(0x71e2a115,0x4eb00620),LL(0x2ca1b0fc,0xc1c5314a),L_(0x000001b4), LL(0x6702b16a,0xcdc04837),LL(0x5b4e8123,0x242a32ef),LL(0xa7d67834,0xccb0fead),LL(0xc2a3bc3a,0xe65ed32f),LL(0xe7119407,0xab8b44e6),LL(0xaeb1712a,0x5a5977e9),LL(0x39ce4f89,0x708cfeb0),LL(0xa19d43ed,0x4957cd1c),L_(0x00000025), +}, +/* digit=20 base_pwr=2^100 */ +{ + LL(0x2f03e91a,0xc29a3fb6),LL(0xcfe3b9a1,0xca033fc3),LL(0x8c5a528b,0xb4bc3e3f),LL(0x23396cd7,0x4bd134e2),LL(0x39c3eb8c,0x565224c7),LL(0x25548c0a,0xf01471ec),LL(0xb0fb17f6,0x5fc65797),LL(0x18aee101,0xc4d69b6d),L_(0x00000048), LL(0x8f600f91,0x0074be70),LL(0xa14b550a,0x52425042),LL(0xf8e6b95a,0x3f438c42),LL(0x0174e981,0x81004aa1),LL(0x34ae4351,0x510cd9a8),LL(0xb1b67e29,0x8cdf2105),LL(0x41438bad,0x8ec5ba91),LL(0xc64d1130,0x00e8c08d),L_(0x00000143), + LL(0x1480c24d,0xb2625d11),LL(0xdcf3505f,0x9962934c),LL(0xc306874b,0x981e8fcd),LL(0x02704410,0x2bd0a650),LL(0x249eb349,0x6a534a84),LL(0x6b6bb40b,0x0c32e132),LL(0xbe5d2914,0xcb2ca52e),LL(0x029956b2,0x77c72251),L_(0x0000018c), LL(0x7c5c4dde,0xfe2b4e07),LL(0x6314442e,0x4431d884),LL(0x66618e79,0xcd3eeea2),LL(0x6a5a933f,0x64415965),LL(0xbda24f06,0x522dc52f),LL(0x82f45dda,0xd412542f),LL(0x0e5075c9,0xff34a66e),LL(0x9a2aba0f,0x512c4a1d),L_(0x00000169), + LL(0x9f6aa554,0xa3dc5b94),LL(0x9872016b,0x2889ccb7),LL(0x1df5e18d,0xe0129254),LL(0xdbcaebf5,0x4cd20b4c),LL(0x63d6c33a,0x1f301089),LL(0xc46a1dad,0x755fc0db),LL(0x2c0e39b6,0x6ef9e694),LL(0xac07fa12,0xd500d36f),L_(0x00000005), LL(0xd19e5fb5,0xf90b5e7b),LL(0x65e8dbbf,0xeb8ecb37),LL(0xe491cc2d,0xd314c068),LL(0xd31ab995,0xe810513a),LL(0xcca181b4,0x9b50dc0f),LL(0x0c1e0526,0x89302958),LL(0xb6453c85,0xde5a7d26),LL(0x3c9a98b2,0x6f7a7718),L_(0x00000138), + LL(0x542368a9,0xd7d861fc),LL(0x7b3c184c,0x5c425a73),LL(0x14a6e3b9,0x85d4a651),LL(0x532f514e,0xb665bb45),LL(0x8b87e598,0xa66a39b0),LL(0xdbcbbabb,0xa46208db),LL(0xa64b561f,0x9520864b),LL(0x05569250,0xde8b31e2),L_(0x00000081), LL(0x4473c211,0xdd1bb6a7),LL(0xe76a8c5d,0xc0f66932),LL(0xc6ee633c,0xc546bb80),LL(0x911f68d0,0x828f4e0c),LL(0xc213a206,0xeb2a4276),LL(0x04a16b2c,0xcbe98cb2),LL(0x38c09aa1,0xbcc1671d),LL(0xa6f3ebee,0x7a684ba9),L_(0x0000012c), + LL(0x89cd7620,0x2ca34639),LL(0x14b160b2,0xe3cc4351),LL(0x57f2d520,0x8707011b),LL(0x1d30ff78,0xb1a346a6),LL(0xd69eda68,0xa84618b8),LL(0x4008115f,0x06f520c0),LL(0xfeeecaa8,0x6a14e30d),LL(0xcc7e0843,0x68bc839c),L_(0x0000005f), LL(0xc998f3f5,0xb3ae58e3),LL(0xd35d5af6,0xf29bb951),LL(0x38625415,0x552cd755),LL(0xb49fd087,0x87ef7e8a),LL(0xde9ebd20,0x76b067b5),LL(0x110309c1,0x1ece1e74),LL(0xb224505a,0x91a5a2d7),LL(0x03ba9629,0x79263dad),L_(0x00000188), + LL(0x89fcda69,0xa71b7e01),LL(0x5ba885f2,0x915b6877),LL(0xb98305b9,0x53769a90),LL(0x10c19b27,0x8d87ac0d),LL(0x8f7acf63,0x583c77c1),LL(0x64d02af2,0x6e04d239),LL(0xe5be9202,0x8f85294d),LL(0x3a5a3099,0xbb22f880),L_(0x000000d2), LL(0x9876e935,0x150daae0),LL(0xf1b9b104,0x500967b9),LL(0xe48eb13c,0xec3e5c4d),LL(0x895cf9cc,0xb6158629),LL(0x7d90f3f7,0x8aa201ea),LL(0xc0cda29f,0x0c9a6e88),LL(0x4c0d7015,0x97d1c62f),LL(0x56ee70bc,0x4fd0f68d),L_(0x0000018e), + LL(0x44d6c8a7,0x413edbb8),LL(0x792ccd3b,0x1564f76a),LL(0x72527a7c,0x2778d6f2),LL(0x45b55b68,0x67ba3cee),LL(0x6e138f21,0x5d96d43a),LL(0x38c932f1,0x63638065),LL(0x4892afee,0x6ed7c45d),LL(0xb82b82f0,0x287b4614),L_(0x000001ee), LL(0xc1bb9d4e,0xb0953f4f),LL(0x5150d18c,0x107a5e99),LL(0x67e23c2e,0x71a733f6),LL(0x2e8bfba0,0xca46066c),LL(0x1d6c6108,0x8cfb4987),LL(0x39bb5a64,0x48164ece),LL(0x0cf34f51,0x336996f4),LL(0xe99b9250,0xd6e08146),L_(0x00000069), + LL(0x921a752c,0x4008e517),LL(0x7a6c13d1,0xc5d69ab8),LL(0xc4597b07,0xc66db12e),LL(0xdbd74a68,0xca40dec9),LL(0x4579d719,0x5a617fff),LL(0x13172539,0xe3835876),LL(0x09e3b946,0x2478942d),LL(0x42d20c85,0x82ecbef7),L_(0x00000079), LL(0xda1602b7,0x5f589886),LL(0xc9ae2bbd,0x8ba22c3f),LL(0x2126ee97,0xe212b5ab),LL(0xff875595,0x389b739e),LL(0x9d6707a2,0x263595af),LL(0x87d12dd7,0x92670f97),LL(0xb014c330,0x00678803),LL(0x692f506a,0x060764da),L_(0x00000167), + LL(0x6fa9c3c0,0xa7daee3e),LL(0x99e57328,0x3e8ff84f),LL(0x8d930fda,0x140c8ff0),LL(0x29def859,0x514bf396),LL(0x6dc6556a,0x917c0e6f),LL(0x1b3aa174,0x2fd52f94),LL(0xbc6b4711,0xec0486f8),LL(0x10164218,0xfc4d6fdb),L_(0x00000087), LL(0x20fc6faa,0x143ce621),LL(0xb9fa1174,0xb63c7e44),LL(0x5ae8a4f3,0xa5424b58),LL(0x8d2b2e47,0x1058f848),LL(0xc653cf99,0x5d72c078),LL(0x9ab7f092,0x5a469db8),LL(0x0eb39c94,0xb34488b0),LL(0xd4f2904e,0xff8c9e4f),L_(0x0000001b), + LL(0x4cffe2f1,0x5b0f1fb8),LL(0x0aa30019,0xcd7cdff0),LL(0xb19b5360,0x2a32083a),LL(0x1a886b29,0x6fd83895),LL(0x3e954ca3,0xfc9e4bd7),LL(0xf6dfbbe3,0xd49474b0),LL(0xbe5e429b,0x6cfc9bf8),LL(0xb282ba89,0xd4f6b8c7),L_(0x000000b0), LL(0x5a179753,0xf0239d41),LL(0x54cf1a20,0xf833aa3b),LL(0xbe4f75ce,0x667c55f3),LL(0x210d085a,0xf20673c4),LL(0x90bc983c,0xe6d37945),LL(0x2b933ece,0x7400e31b),LL(0xa73815a0,0x0016f173),LL(0x2d12b0bb,0x04943106),L_(0x000000cc), + LL(0x258975c7,0xf8f8e68e),LL(0x93b485b5,0xbbae7e17),LL(0x21f99dd4,0xd3ee3730),LL(0x8f41688d,0x869cee4c),LL(0xa7149163,0x7ffdc064),LL(0x1dd2e6f5,0x3e9c9ee6),LL(0x3e5343f8,0x1bea26ce),LL(0xd599de2a,0xc95e92f5),L_(0x00000021), LL(0xce2962a0,0xc38ccf06),LL(0x17914783,0x5b85620d),LL(0x3066f430,0x8a55dcb5),LL(0x73026026,0x8691669e),LL(0xe37b2f3d,0x6c8abb34),LL(0xb0b3d5e7,0xe607781f),LL(0x1d40a38b,0xbd4e966b),LL(0xa4bb9c3f,0xfa1cc8e1),L_(0x00000053), + LL(0x2fc2f08c,0x61cb5b4c),LL(0xa0ed42bb,0xb44afbfd),LL(0x8be8a8dd,0x0621480b),LL(0xe9c3bbe4,0x3bbd013b),LL(0x3dad214a,0xf63413c0),LL(0xf44a0f7a,0xfedcc70e),LL(0xababaa32,0x0bd6ea84),LL(0xd282fada,0x556a1b66),L_(0x00000155), LL(0x4628c72c,0x2eb9a417),LL(0xd244439d,0xe9ea5bcc),LL(0x6a6b06e0,0xc7910063),LL(0x95bae48e,0x065bc250),LL(0x8f84ff6a,0x70b64554),LL(0x1fab9066,0xe0a46dab),LL(0xe07a85ee,0x8bdd86f2),LL(0x832625e0,0x012550fd),L_(0x0000006d), + LL(0x972aea5c,0xafed2f9f),LL(0x24dc0f96,0xe142d2cd),LL(0xa6689657,0xea2d5cc5),LL(0xae481b79,0xd0bdf4f2),LL(0x8a560bee,0x85db3226),LL(0xa5d11f99,0xfd86b657),LL(0x77ab3c4c,0x9ecb073c),LL(0x1fa5f6fb,0x59db760e),L_(0x000000e3), LL(0x246cd3c8,0x09f0d045),LL(0x3533500a,0x2990659d),LL(0x4eb8eebc,0xe68fe462),LL(0x60e87a33,0x48e37752),LL(0xcd62216e,0x703956e1),LL(0x0dcfead1,0x3a09a4d6),LL(0x491340a6,0x49d2c6bb),LL(0x0b013428,0x50df11ed),L_(0x0000013c), + LL(0x46d493a7,0xd8099e2f),LL(0x2aad5a31,0x5920a298),LL(0xb4992465,0x07aa2d85),LL(0x6aeecdc9,0xe712f629),LL(0x2a16e6d8,0x8639dfb4),LL(0x36815fc1,0x2a477d95),LL(0x0b2f5359,0x7c3ca55a),LL(0x896f9530,0x05522116),L_(0x000000ad), LL(0x41d24f09,0x208b956c),LL(0x65da1fcf,0x1c1f9a0d),LL(0x1534c8eb,0x772539f2),LL(0xb39694db,0xbc2cb67e),LL(0xf14a06ea,0xf6e48c27),LL(0x4aa51441,0xe7141d18),LL(0xbd52c5e3,0x7983136e),LL(0x77a0099a,0xd7f96b63),L_(0x000000a5), + LL(0xfe4e08d0,0x0a99ca48),LL(0xd8a3e48d,0xb1ee6ff6),LL(0xa6f4001b,0x2ec4e0b2),LL(0x04f03a29,0xe977781e),LL(0x0605bcbc,0x0f8d3aa3),LL(0xa1ff6ad7,0x052c4409),LL(0x13eedc9a,0x211fe699),LL(0x4cc42df8,0xd70f26db),L_(0x00000057), LL(0x9eae5eec,0x00258026),LL(0x08667174,0x7f49a77b),LL(0xa5dfdb47,0x0446d9f9),LL(0x6b646fc8,0x24ebc0c9),LL(0xb269fd06,0x244a494a),LL(0xd5fd906c,0x9c16866d),LL(0x27e6983d,0x545f39fb),LL(0x798c184b,0x5fdefa1a),L_(0x0000019b), + LL(0xb90f9d0d,0x1e365108),LL(0x765f67fb,0x1aae9f80),LL(0xb1b38d14,0x7a9407e4),LL(0xc8424d69,0x9693e7cc),LL(0x91d93e3f,0xba50e7d2),LL(0x385c13c5,0xd94ecd34),LL(0x6fcf73c9,0x80eb0bb6),LL(0x0d598f4a,0x1c4d4c29),L_(0x00000172), LL(0xbbeb3c79,0x3b0fb9a3),LL(0x326d546e,0x4c6d27ba),LL(0xa848cf09,0x09d2dc18),LL(0x46c0e416,0x6f0069ca),LL(0x9231b926,0x1c4aef79),LL(0xbdbead08,0x9d4dabac),LL(0x272ba195,0x8e216ceb),LL(0xcc49b720,0x83cc03ec),L_(0x000000ba), +}, +/* digit=21 base_pwr=2^105 */ +{ + LL(0xb5250d8d,0xd64c0998),LL(0x7c43b599,0x85a2ec86),LL(0x4c9f6ac7,0x59f29f0d),LL(0xfaa4ec8b,0xf16ae8c0),LL(0x2017604d,0xab8d8f78),LL(0xbb59089d,0x2772bc38),LL(0x38403982,0x6e88e817),LL(0xf3d1571c,0x7d4e8e0c),L_(0x0000007b), LL(0xea0f9192,0x477bc572),LL(0x39b5eb10,0x8d22a645),LL(0x77d71bc8,0xd769223e),LL(0x9734dc62,0xfe2b562c),LL(0x41cdb0ad,0x3173fab2),LL(0x70ddf3ff,0x997d6033),LL(0x70dbbbbd,0x6d59561f),LL(0x998a88a5,0x64aafc32),L_(0x000000be), + LL(0xdeb9f4b8,0x449a64c8),LL(0x674c0fe9,0x81603532),LL(0x01e88fe6,0x595c6e13),LL(0x913b8697,0xf6f513d4),LL(0x3b6d478c,0x88c1e320),LL(0xb2857351,0x90cfb68d),LL(0xdfb9fd43,0x6c4bb93b),LL(0xcc660149,0x3f388af7),L_(0x00000063), LL(0x23175237,0x9a5258fb),LL(0xacae0a8b,0xb741a40d),LL(0xba0560ab,0x795d005e),LL(0x3978bc6a,0xcaa47999),LL(0x1c0b2496,0x1ff04fef),LL(0x6ddcefe7,0x47c3b092),LL(0xf281ff39,0xcc93f3d8),LL(0x23027cc7,0x773c9a3f),L_(0x00000078), + LL(0x01d1f8d5,0x21f6a156),LL(0xc4591dc9,0xf7c15406),LL(0xb36c8aaa,0x3b0d0813),LL(0x287834fd,0x44ef9e76),LL(0x94a18ce5,0x52fb6092),LL(0xbd019877,0xb24d08d9),LL(0xd4816092,0x39d2d32c),LL(0x188b097d,0x5b9f00f2),L_(0x0000013a), LL(0x0979e9da,0xccda9d6f),LL(0x41dad104,0x637d2807),LL(0x4ee619b7,0x0f5a9cc8),LL(0x4f3d7156,0x97bb554b),LL(0x367054b8,0xe890a210),LL(0x3f1f61c3,0x0784aff6),LL(0xb92963c2,0xc9acc43c),LL(0x309317af,0xdadb0d3e),L_(0x000001c1), + LL(0xd52ab832,0x6d5e67b8),LL(0x049665d8,0x19993eb2),LL(0xb56e1ced,0x7a62ba87),LL(0xdfb9c1fc,0x6fc5cf75),LL(0x4712b627,0x554f5dad),LL(0xe0548bd1,0xecba89fb),LL(0x1ee24125,0xfa18f5ad),LL(0x7e176a53,0x8796b526),L_(0x000000e1), LL(0x7a9eb450,0xada0f1a1),LL(0xe4e5f968,0x89107584),LL(0x8e12a3e0,0xd6a2ba69),LL(0x1ee9c73c,0xe23b2a1f),LL(0x43a76e02,0x428e9adc),LL(0xe3d7526f,0xf09d62c6),LL(0x0557ab8a,0x37cd537d),LL(0x2758b1d3,0xe54434b8),L_(0x00000000), + LL(0x2fd63eb2,0x2d008a6b),LL(0xd28295a8,0x6d2db388),LL(0x2d6dd882,0x7d1d9977),LL(0x1f0a2196,0xa51d2cc4),LL(0x5e445be3,0x52abdb6b),LL(0x3146aac1,0x1f1b1ee1),LL(0xfac49408,0x92df369d),LL(0x84b1d832,0xf67ffdf5),L_(0x000000da), LL(0x35058c28,0xba8a7d18),LL(0xfa0e70ac,0xf53c34b6),LL(0x3063c19b,0x4954b990),LL(0xcac557d4,0x2d1447f0),LL(0xc89953df,0xc7aef4e6),LL(0x66df6476,0xcb978dd6),LL(0xc6d8f86c,0xf9c4098f),LL(0x024e891d,0x25468ae2),L_(0x000000aa), + LL(0xcf9cd67b,0x9893947c),LL(0xe7519a11,0xc44aa926),LL(0x9787209d,0xf096efcd),LL(0x743501fd,0xce9a1706),LL(0x1f7551cc,0x684716f6),LL(0x8dae5405,0x149e678c),LL(0x4cc025ce,0xa47c9f5e),LL(0xb9f91415,0x9acbec1a),L_(0x00000058), LL(0xb6fa6da8,0x1acf3849),LL(0x7e3ea9f8,0x0e3c4dae),LL(0x4d104dc4,0xadaff9d7),LL(0x5ec06e69,0x2c84d798),LL(0x553ccd90,0x93c28668),LL(0x32f13211,0x585b3068),LL(0x2907a89a,0xcdceca12),LL(0xb9c0d594,0x128fec44),L_(0x0000015d), + LL(0x7592747c,0x1698ff12),LL(0x50452a3f,0x34e63149),LL(0x70c02b2e,0xa04d3b54),LL(0x3afdb0b0,0x61ed2692),LL(0x91c5ad02,0xd6e4e6a2),LL(0x602aa43c,0x134189db),LL(0x4a9101c1,0xf2cd081d),LL(0x4ee8352b,0xf7a72eda),L_(0x00000199), LL(0xb3ee6207,0x04ba1ce3),LL(0xf4179875,0xa4294fcb),LL(0xe4a0b700,0x05833496),LL(0xb77b6e65,0x8229a64f),LL(0x204eabae,0xe9137cfc),LL(0x4ba1e0a9,0x66c9fb36),LL(0x5ece8d91,0xabbbb589),LL(0x725a0003,0x62522294),L_(0x000001a3), + LL(0xabc70424,0xfe36c3a6),LL(0x11953f80,0xd15f704a),LL(0x0b4cc57c,0x3d3a8bb5),LL(0x347df44d,0xb5099398),LL(0x3789e215,0xd81f3a55),LL(0x5f2bce30,0x853ed011),LL(0x0b7f91f0,0x20d53ac9),LL(0x8d7ec293,0x63e7bfbe),L_(0x0000005b), LL(0x232c6da4,0x56cdcd80),LL(0xc636cf5e,0x21241d8f),LL(0x6e4c3d96,0x6812f9d5),LL(0x81fb84a8,0x7741d3de),LL(0xb50a7928,0xbab3d77e),LL(0x7cc80386,0x8f374862),LL(0x1901afee,0xbf5ceb2a),LL(0xdd95591f,0xed0c8140),L_(0x0000000a), + LL(0xbfe596a3,0xf412c36c),LL(0x0ee07a01,0x426518b8),LL(0x54499ba5,0x89e701b7),LL(0x380c3953,0xf3f8a9a9),LL(0xd8749088,0x809a3666),LL(0xc559f6c7,0xe768213d),LL(0x64aff50b,0xaad0b2e7),LL(0x0535ff88,0x68771b34),L_(0x000000cd), LL(0xa5e475dc,0x7ec42d0f),LL(0x4a3e7bea,0xf9c7931c),LL(0xe0127a41,0xe2d8d114),LL(0x88a09cfa,0xc08a0d23),LL(0xa27150fc,0x052224fb),LL(0x1880e3c3,0xc80a285d),LL(0xc9c2bfb6,0xd52dcb46),LL(0xfd0b235d,0xfb31d609),L_(0x000000df), + LL(0x43934d7a,0x058ee09e),LL(0xf551f7ab,0xd68828a3),LL(0x85ee0c11,0x6925306d),LL(0x45a463ec,0x9d6c7188),LL(0xf851554d,0x962b1441),LL(0x66783d33,0x32aca09e),LL(0x856ca966,0x9146adcd),LL(0x5f7a7608,0xfe631a24),L_(0x0000011f), LL(0xe52c21b8,0x28d6e11e),LL(0x0d442988,0xc65a878d),LL(0x0f8c7ce6,0x332a94e8),LL(0xf933213e,0x0a2bf942),LL(0x055f0bf3,0x0e64c7cf),LL(0x371deac5,0x34d3b9fd),LL(0x384367de,0x6f42cc1a),LL(0x15ed6027,0x8f2a65d1),L_(0x00000198), + LL(0xe7f29199,0x1243edd3),LL(0x7356cd93,0xc68eb0e6),LL(0xfc213d5b,0xa963c442),LL(0xd8a42be8,0x426acfcc),LL(0x4e52c125,0x3019a35c),LL(0xcccbb098,0xcbc21858),LL(0xd433019c,0x687acf0a),LL(0x47525d0f,0xce5e2216),L_(0x0000002f), LL(0x4634f680,0x88d6b26d),LL(0x6dad247f,0xa7f32d3f),LL(0x70a43312,0x340a2808),LL(0xe2da73d8,0xed020b20),LL(0x477f5bf7,0x752a7c57),LL(0x84e84209,0xaf283680),LL(0xdce8d8d1,0xf2a576b8),LL(0x4e1b6410,0x6cfe6e79),L_(0x00000095), + LL(0x24f5d0cd,0xf28c9f4c),LL(0xd2aab695,0x7916e1ca),LL(0x8e97f41e,0xe9e93d40),LL(0x40703872,0xd7c8afdf),LL(0x78640530,0x8fe1f1c7),LL(0xfb5cc433,0x2060da94),LL(0x7302096f,0x953a5bfe),LL(0x62d3763e,0x4947fc61),L_(0x00000033), LL(0xc1869802,0x92108fd5),LL(0x8477b34f,0xe9b6b7e8),LL(0x31f306a1,0x306db354),LL(0xd16d3be6,0xdf966d91),LL(0x0deb9d15,0x62b5d760),LL(0x613eaff2,0x22a569a2),LL(0xae4f8efb,0x0eeb67c4),LL(0x9bca80a4,0xac4e55dc),L_(0x0000015e), + LL(0x0805ca08,0xe7b9391b),LL(0xb559bea1,0xd94e135b),LL(0xbbc93e2a,0xca27f88b),LL(0xda4f9fae,0x9fac28c9),LL(0xe1b8ef16,0x5f1ec2d6),LL(0xfd385151,0x49bb68ca),LL(0xfb07a8a5,0x55e48d66),LL(0x146bc523,0xe1490dad),L_(0x000001ce), LL(0xc95026ef,0xd3c160fa),LL(0xec69a98d,0xf736bcb4),LL(0xb1767df2,0x0bfa04f5),LL(0x48e96045,0x87d98ff0),LL(0x977f6553,0x6992858f),LL(0x17332805,0x22fe39cc),LL(0xe8ffd592,0xc326d64f),LL(0x6551c98c,0x12a83f56),L_(0x00000129), + LL(0xfb1fe590,0xdef70c89),LL(0x5d8bd28a,0xb3dcf1dc),LL(0xe969cb6b,0x157c6b46),LL(0x846f656e,0x30bef44a),LL(0x3e3ab4ad,0x14cc18c8),LL(0x63d1dbea,0x10262f96),LL(0xfe27704f,0xa567503f),LL(0x1ff786bf,0xff184f96),L_(0x000001d0), LL(0xa08df1ab,0x92281a63),LL(0x41b9f732,0x4f7c669f),LL(0x4f4c7a3a,0xd9aab809),LL(0x6fecf938,0xece45010),LL(0xbed97dd7,0x20855b33),LL(0x41c58421,0xda8500b4),LL(0x6bc3ec2e,0xf0a33322),LL(0xa1fd8aed,0xe01eb188),L_(0x000001d3), + LL(0xc846a806,0x1799b47a),LL(0x205f8a25,0x5c7dce04),LL(0x6bc7753c,0x918feeda),LL(0xb0bd3460,0xba66aed0),LL(0xfae70886,0x5d2bdd64),LL(0x37b93501,0xa85e194e),LL(0x12025c5a,0x44f97270),LL(0xb54faac3,0x98c400c5),L_(0x00000094), LL(0x3c93f521,0xabe22aa5),LL(0x3c01512e,0x99aa80de),LL(0xca5be145,0x312f55d9),LL(0x8dbfded4,0x1ca51916),LL(0x3f318a7b,0xa0b3f9fb),LL(0x42abfb1f,0x2303713b),LL(0x195a32bb,0xb6968fd2),LL(0x069ca809,0x52819c4b),L_(0x0000018b), + LL(0x7efc38cc,0x662fcdd0),LL(0xd8b1dbd1,0x66c06c9a),LL(0xaf6b6e15,0x28998a9b),LL(0xca45c4ad,0x2d2005db),LL(0xfcd947b1,0x609acb17),LL(0x6bf7b35f,0x25ebaf2e),LL(0xb8a8aba3,0x599df520),LL(0xe4302e3f,0x2bf9b973),L_(0x0000000d), LL(0x2a3c0c12,0x25aebd11),LL(0x8868630c,0xa5529c40),LL(0xaf7c4f6b,0xf5657b1a),LL(0xc0fd49e0,0x3fa70b84),LL(0x4d86ecda,0x39f53004),LL(0xc59dce6d,0x39513f7e),LL(0xbdaf7343,0x822c2924),LL(0xce22dd61,0xacb0786e),L_(0x00000066), +}, +/* digit=22 base_pwr=2^110 */ +{ + LL(0xf20e8c4b,0x55dedb27),LL(0x5a0fc338,0x03e8af53),LL(0x788ccd88,0xe10cabd0),LL(0xa1f1f7d6,0x5f889d7f),LL(0x0487ee35,0xa583e303),LL(0x1885d800,0x2fc9f3dd),LL(0x09ae9a4a,0x2887b5bf),LL(0xa554fc30,0xd91181d3),L_(0x00000078), LL(0xcdca6316,0x53b146d6),LL(0x52f280d5,0xdfaac466),LL(0xe0b73d63,0x0d77869d),LL(0xa8a399cd,0xba5ffe6a),LL(0x5c61b757,0xaffc1da6),LL(0x71cf6c9e,0x34d27387),LL(0x20ae1248,0xf184b956),LL(0x7f6504de,0x1c974cb4),L_(0x00000176), + LL(0xc1efab45,0x659b9b53),LL(0xd37156ff,0xf8338bab),LL(0xa115d2c7,0x9d1175b5),LL(0xa927371c,0x53c22d6a),LL(0xe5b07da3,0xb79ee37b),LL(0x3585421c,0x8ac92029),LL(0xbe2b0a93,0xd489e47a),LL(0x363622f3,0xccd5811b),L_(0x000000c9), LL(0xf506ac3d,0x41cb54f0),LL(0xbf83fb74,0x527b4fee),LL(0x7d9fa2d5,0x6d4a3597),LL(0xf8ab4037,0xe4619c87),LL(0xd590e945,0xab913b27),LL(0xe8861075,0x389b1da0),LL(0xd8fb707f,0xe0beb49d),LL(0x35140b6f,0x392dd172),L_(0x000001f7), + LL(0xe7bedc7f,0xbf4176d0),LL(0xfe15067e,0x0120bf23),LL(0xc1d45172,0x0cb82143),LL(0xfbe2cf59,0xb0e80076),LL(0xd69fd57c,0xbd9b2caa),LL(0x78503bca,0x99823d72),LL(0x2730e435,0x31cc7be0),LL(0x4b145b24,0x10f8d6e3),L_(0x000000a2), LL(0xad5f8d1e,0x4fc47dab),LL(0x7a72c91b,0x564e5846),LL(0x5530b4bc,0x1837936f),LL(0x9913d2c3,0xdf60105f),LL(0x5f1a5851,0x839ef0be),LL(0x3d6d7b8d,0x05890a65),LL(0x48845fee,0x57eb20fa),LL(0xcc1288ca,0xf7b7e05b),L_(0x000000ac), + LL(0xdf12457e,0x7082a01c),LL(0x50e34426,0x91616bf5),LL(0x27cfd7b1,0x426bd9ae),LL(0xc299bf54,0x5f468d0e),LL(0x0487ca37,0x695e6354),LL(0xb93aa7dc,0x9322f558),LL(0x8f48edec,0x818f0592),LL(0x957ee742,0xaca5b088),L_(0x00000123), LL(0x85008e45,0x11189720),LL(0x45a01307,0x348cb9e4),LL(0x5bf246e5,0xf5c183c6),LL(0x3fd8ccf1,0xe9a40aeb),LL(0x0fbda6f2,0x087abdef),LL(0xdaf09cee,0x90c450f5),LL(0xe33344ee,0x3abe1073),LL(0xa3404424,0x02a065d1),L_(0x0000018f), + LL(0x9dc26b1a,0x5faa9290),LL(0xbd275c4d,0x69ab8c9c),LL(0xebf0d548,0x7a6bc4a9),LL(0xe5ed16cd,0x31faf28e),LL(0x3e681735,0xcf90331c),LL(0xc624be8b,0xfb66de1c),LL(0x42603696,0x2a65b006),LL(0xc07466ec,0x9d7f9688),L_(0x000000f3), LL(0xd25f7e3b,0x44e0d6d8),LL(0x3db25f9e,0xe7ca860d),LL(0xa581c150,0x1c6481b3),LL(0x49e5b0b8,0x62060736),LL(0x82bc7eb1,0x54daac9c),LL(0x376c43d1,0x68353454),LL(0xfb293af5,0x2dde1795),LL(0x7e2ec37f,0xed4ef8f0),L_(0x0000019b), + LL(0x9a308462,0x4adfcff9),LL(0x2b928ce2,0xb2e63ba3),LL(0xd02f461c,0x73336d0f),LL(0x04fd6cc5,0x48e88a3a),LL(0xc377597d,0x56c730e8),LL(0x5ac92cdd,0xaf7486b7),LL(0x0317d853,0xe151b910),LL(0x0978da40,0x86c35051),L_(0x0000000e), LL(0xe9b441de,0xb333edf0),LL(0x1ff77a8a,0xd24f0df5),LL(0xed7f23b9,0xdfae6385),LL(0xaa4f024f,0x2a1af93e),LL(0x2ffef5f6,0x6f45d8a5),LL(0x9aa11e63,0x853bb088),LL(0x2271f40e,0x8ed5445f),LL(0x3ccb38c1,0x24afc179),L_(0x00000027), + LL(0x6a440850,0x57906522),LL(0x85f7ceb0,0xce569864),LL(0x059b3177,0x45f8df95),LL(0x60efada7,0x2ee7a947),LL(0x87aafc1c,0xdce2c588),LL(0x17fd804d,0xdcf2f1ac),LL(0x10e82f62,0x4b1309c3),LL(0xe852efd4,0x0ef30c42),L_(0x0000012f), LL(0x62278fee,0x95ed2fc9),LL(0x55cd2f37,0x5d0e9c83),LL(0x107085fd,0x479d2ac2),LL(0x85ff5e6a,0x7bd3eb35),LL(0xd6ed74ec,0xc8a77d96),LL(0x366d8e34,0x3be40939),LL(0xf0ec3c90,0x11212f04),LL(0xc317d540,0x5743bb20),L_(0x00000152), + LL(0x8145236d,0xffb66d41),LL(0x6ea70c3f,0xc5491789),LL(0xcb17d54f,0x1eaf6e4d),LL(0x7c642a64,0xb15be10c),LL(0x99328296,0x611efe5f),LL(0x30829e9c,0x3cdec049),LL(0x5f18e861,0x1a7c38da),LL(0xab7985a5,0x3536d908),L_(0x0000018f), LL(0x47f989e7,0x8ece50b4),LL(0x5435f6e4,0x13d04672),LL(0x60505d74,0x07efc4e5),LL(0x8a551fa9,0xcc601ad2),LL(0x8fed3391,0x9eeaf4b1),LL(0x4338a854,0x72c52e1a),LL(0x61868d33,0xce70bb6b),LL(0x9c3a511b,0x5c8d75eb),L_(0x0000001f), + LL(0x8d15261e,0x69ee64f3),LL(0x508ada39,0xdd8c7134),LL(0xda598cbd,0x36c545ec),LL(0x05133eca,0x0c3f5caf),LL(0x2df79eab,0x71cd5c2a),LL(0x920bc258,0xfa67ae73),LL(0x1fd1e4a4,0x2ed2d89b),LL(0x48726f90,0xd5a8956c),L_(0x000001ab), LL(0x02aa7b62,0x6f23ba5d),LL(0xb20e4dfc,0x4bab12af),LL(0x5fcc6747,0x0006c88a),LL(0xaa6ea9a8,0xe8e4646f),LL(0x9db86f5a,0x8f1f8a76),LL(0xec7745fa,0x455291dc),LL(0xe15a4259,0x155de9ad),LL(0x6c3a6e93,0xf9fdf6e0),L_(0x000000df), + LL(0xab1b0548,0x3dc752ef),LL(0x14921ae6,0xdad5624e),LL(0xf829dec8,0x1d1460ab),LL(0x5833de5b,0x9c1ff203),LL(0xd5ded33e,0xfac09cf2),LL(0xbb7c2ad4,0x82d6f980),LL(0xf32dbe91,0xad650ca0),LL(0x82ca75ed,0x6c58e180),L_(0x00000033), LL(0x00ff7cd0,0xea9d2225),LL(0xd3ca85cc,0x434cfef1),LL(0x3bb6dff7,0xce357f60),LL(0x56328ba9,0x0e4a179e),LL(0x4a4804e3,0xe56b9eac),LL(0x254a5b01,0x72a81cd7),LL(0x8b35d349,0xe054875e),LL(0xc8c87ac4,0xe8c11607),L_(0x00000015), + LL(0xc31fc004,0x1c6cc43f),LL(0x6f96d167,0x794436fa),LL(0x586761cb,0xd20a52b3),LL(0x3ae24479,0xc24cf7e8),LL(0x5d299550,0x7aabd2b7),LL(0x9cea4b13,0x09feb305),LL(0xd75ffb4e,0x5b6a28d7),LL(0x5d3cd9c7,0x4d85737b),L_(0x00000063), LL(0x1e01bfde,0x743b9f5d),LL(0xe042f5f5,0xe03225a4),LL(0xc7dc3d4b,0xdfce41ae),LL(0x51cdf46c,0xd3c05da2),LL(0x69bf3a35,0xc18cbac7),LL(0xc0889d43,0x51fc0084),LL(0xf694d481,0xbfa4cfd4),LL(0x05438a3a,0xcb44f4a9),L_(0x000001cc), + LL(0x3e7e05df,0x22d68d2a),LL(0x6f65a633,0xd77949bf),LL(0x368db479,0x738f46ed),LL(0xe52e22f5,0x7212d465),LL(0x4758d194,0x8bb783e2),LL(0xd677a59c,0x1b239d33),LL(0x60904604,0x9c2f2775),LL(0x8df6497a,0x9be5339a),L_(0x0000007a), LL(0x08475032,0xb5804d78),LL(0xbd5cd190,0xbfbebfb6),LL(0xd58769b6,0x66d25685),LL(0xf9ea5b23,0x206ac283),LL(0x09d14a84,0x845e93a9),LL(0xe03b612f,0xc6807818),LL(0x061fa312,0xeb980705),LL(0x6b501efd,0xfa3670b6),L_(0x0000003c), + LL(0x6257d98f,0x509702e1),LL(0xaf686102,0xc4201f85),LL(0x800e44cf,0x7e6e7641),LL(0x35e4ed58,0xa3fd838b),LL(0x5af78edd,0x728e86da),LL(0x0effaedc,0x6fd05e38),LL(0xfd668fa4,0x4ebcbbb8),LL(0xe45dbf60,0x21bf82d8),L_(0x0000016c), LL(0x8050d329,0xf066dcdf),LL(0x1ca3b2b7,0x73890298),LL(0x96fc550c,0xcc4f19d8),LL(0xfc004a6a,0xc9fae54f),LL(0xcdd730f4,0xa5e22c5e),LL(0x2754c10a,0xb60dac89),LL(0x49c473e4,0xd5465b8b),LL(0x30fa2831,0x14688f19),L_(0x00000175), + LL(0x4fd6e5bc,0x8f88f135),LL(0x7034290c,0x2f4cd77e),LL(0x61556d3d,0xe5aea948),LL(0xee182466,0xf7fd60b4),LL(0xfaab2132,0x2107919a),LL(0xb164b7c6,0x4909986e),LL(0xb4de2fa4,0xe1076a94),LL(0x331fc36a,0x8fd3234a),L_(0x000000d4), LL(0xee34ec46,0x07524382),LL(0x3f9109b1,0xec9a0d40),LL(0x11cb9eb3,0x7e1084f2),LL(0x1e06d740,0x00717031),LL(0x2928ea89,0x4bde88cb),LL(0xd69d113b,0x20f91a03),LL(0x2ff2dbf3,0x7a3884fc),LL(0xdf24441f,0x198806cf),L_(0x0000012a), + LL(0xf9b6b90b,0x076f7438),LL(0x1544e142,0x15274247),LL(0x022e14c5,0x96077c3c),LL(0x30862489,0x789e3935),LL(0x50c53e4e,0x1141fcf3),LL(0x3047c405,0x2cd7f2a5),LL(0x57a60daf,0x9bc52e3e),LL(0xcd0ce692,0x21830d42),L_(0x000001a9), LL(0x37349784,0x77ac72bb),LL(0x403e48dc,0xad70038c),LL(0xc084214e,0x170491ff),LL(0xf7b0b5fe,0xd8cf2d1c),LL(0x4af0ed5d,0xd8301c30),LL(0x00208a5b,0x33f56a54),LL(0x18e018d4,0xfa224eea),LL(0xef56c21e,0xaa9262c2),L_(0x0000002d), + LL(0x0c9bed5e,0x328296f3),LL(0x4bb11c1f,0x6050199a),LL(0x15b40849,0x53a7ca7c),LL(0x81bc50ce,0x0a2c1da2),LL(0x682873f5,0x52c0e34f),LL(0x44102170,0x9c5ef21f),LL(0x9f354fbc,0x0bba954e),LL(0x6cd7990a,0x02432a32),L_(0x000000a4), LL(0x976d76d3,0x2be6dddd),LL(0xe55cac7b,0x8a3b8a57),LL(0xda37392c,0xec1dc93e),LL(0x2e3fecd4,0xcf4f78c9),LL(0xfedf3f09,0x1ff689fe),LL(0x03374052,0x092dabd5),LL(0xdf4087ca,0xb9e4e110),LL(0x9d02763e,0xf3f329b7),L_(0x00000089), +}, +/* digit=23 base_pwr=2^115 */ +{ + LL(0x016c8d9b,0xbd54382d),LL(0x826f7b17,0x32c36ec7),LL(0xdce64f28,0x22a16680),LL(0xab2193ae,0xaf6a85c2),LL(0x52cc0a0a,0x2f202702),LL(0xcc1335b3,0x2afbf317),LL(0x3743776e,0x0deb4740),LL(0xf9a19900,0x61591f25),L_(0x000000bc), LL(0x2800729e,0x1d84eebf),LL(0xb06a4eb6,0x8bb72608),LL(0xb23e7396,0x2e886104),LL(0x992a3ae8,0x7c8605d2),LL(0x418a91d2,0xe33bec6e),LL(0x5f2b49e6,0xa9d829d4),LL(0xd1f4a3f4,0xeb2f044b),LL(0x28bc4cea,0xb1ef09fa),L_(0x00000063), + LL(0x611f97e2,0x04826845),LL(0x6b1ee54a,0x9240015b),LL(0x608b1dc0,0x25698b8a),LL(0xa1390509,0xb5e532ad),LL(0xdf4acb31,0x30c41c46),LL(0x16e05bee,0x07d190c1),LL(0x642c1273,0xb566eca9),LL(0x1b365a48,0x5c3cffa2),L_(0x0000006d), LL(0x9754189b,0x21b8836b),LL(0xa0057686,0x520f579e),LL(0x7bf51510,0xa43d38cb),LL(0x0a4bbc0c,0xfe21891a),LL(0x687446c9,0x1242b093),LL(0x8feab881,0xcacb6d61),LL(0x7a921f31,0xcf611aa4),LL(0xc5cb09d3,0xfef9a8ef),L_(0x000001d8), + LL(0x304ccf54,0x4375da6c),LL(0x13287748,0x4fda63b3),LL(0xfa3518ec,0xbc16ce7a),LL(0x996ffb04,0xf3ffb5fd),LL(0xf0a2d30e,0x6da8bb6d),LL(0xa23e83a8,0x08e806aa),LL(0x0642e4da,0x0286c1dc),LL(0x84837dcb,0x8196eb23),L_(0x000001ac), LL(0x584182dd,0x1c8d1ec4),LL(0x9c108506,0x77ca438c),LL(0xdc8649fb,0xdbfc198f),LL(0xaf929bb8,0xa5fbf701),LL(0x60078f43,0xe25fdf9d),LL(0x3f03bcb4,0xca36812b),LL(0x53c1eaae,0xb394d3a9),LL(0x1f8445ff,0x78a7b4ab),L_(0x000001eb), + LL(0xf50749b9,0x6d42c81a),LL(0x40e01936,0x7487e815),LL(0xd6072e7b,0x32da913c),LL(0xe7b4156c,0xf1e87478),LL(0xb217423d,0x4880f5cc),LL(0xca344dd5,0x90182347),LL(0x15da2c26,0x87d2337d),LL(0x8d993e28,0x604cc23f),L_(0x00000179), LL(0xc2ec9c0a,0xf9778d40),LL(0xc9dd1808,0x4345027e),LL(0xdcd7b63f,0x198a63ab),LL(0x03bcf65f,0xa7a4c388),LL(0x1130c2d3,0xb476f99f),LL(0xc1ea5019,0x991ad6b9),LL(0x4f67377e,0xa9f5ad13),LL(0xd99047df,0x80641e2f),L_(0x000001dc), + LL(0xdc84c719,0x2a4af296),LL(0xf45a67c1,0x963ea378),LL(0x3fc32889,0xe19e2266),LL(0x2477017e,0x3e1c3af5),LL(0x1bbfecd7,0x9c6aea32),LL(0x03afdf5f,0xdd92f5ef),LL(0xd2ffd177,0xcff66e71),LL(0x22d56579,0xca369a53),L_(0x00000122), LL(0x5689a8bb,0x3ecebaa8),LL(0x87cbd3db,0xfef4d6ca),LL(0x92b7d8a0,0xf81b8c47),LL(0x4e50ecbb,0xdd6768bf),LL(0x916361ed,0xf3c09bf3),LL(0x6b31d1bf,0x54e2879b),LL(0x17c7f544,0x44b470e7),LL(0xb6fa811a,0x32df7372),L_(0x000001e9), + LL(0xfa7af75f,0x83de0d9d),LL(0x7ea57102,0x4652ceba),LL(0xdd543523,0xb8a36856),LL(0xb586f821,0x19e00261),LL(0x6ce309c3,0x1ed079e5),LL(0xe0f75ac3,0x51ff1099),LL(0x2442020d,0x0c077aee),LL(0x248b83fa,0xc85e1f87),L_(0x0000006f), LL(0x8445d10a,0xaf872b79),LL(0x311d3108,0x2ca2a32b),LL(0x5040c97d,0x4fa4c2f0),LL(0x7615703d,0x80d5eb27),LL(0x36c8c169,0xb5f074a5),LL(0x95daa1e3,0xbda91813),LL(0x672dad89,0x395bd4fb),LL(0xf61f3d94,0xc4a2c81e),L_(0x000000b4), + LL(0xa1b80edf,0x04e91ed9),LL(0x861a2094,0x2bca8d77),LL(0xca4ca01a,0x03590793),LL(0xf16fc210,0xc8ad1877),LL(0x48a85346,0x89666be4),LL(0xf0fc0cfc,0x6adba857),LL(0xcd27d0e4,0xdc6000be),LL(0x66de6f3c,0x410cd2f9),L_(0x000001a3), LL(0x97174f70,0x27a30017),LL(0x3566e721,0x523a0305),LL(0x793773f0,0x1ee9afff),LL(0x7fd66850,0xdbc711c7),LL(0x047ae5ad,0x3acae945),LL(0x203bb8df,0xb932a42c),LL(0xfe2439ff,0xd51dba9f),LL(0xe4630688,0x268de595),L_(0x00000036), + LL(0x56ff8468,0xcb924e78),LL(0xf61f664c,0x2e404ee4),LL(0x5ac67cb0,0x6b002de5),LL(0x87550da7,0x537e3c3c),LL(0xb6b43fc4,0xcc36c052),LL(0xb2d5ce01,0x0c5bb204),LL(0x8e7f6d0e,0xf930fde8),LL(0x09c188bb,0x056f87d9),L_(0x00000168), LL(0x68bd3a01,0xbd1106b6),LL(0xe76203aa,0x0182e8dc),LL(0x02fff311,0x1307d3fa),LL(0x96bf7d1e,0x71013392),LL(0x6ed2ca34,0x24a22e45),LL(0x1b668eed,0xe4102d01),LL(0x79cf95e5,0x681d10cb),LL(0x6c6693b0,0xf94e08ac),L_(0x00000155), + LL(0xec2c327b,0x4ea63be4),LL(0x7d917da9,0x5de2c336),LL(0xadecf30f,0x0955c66b),LL(0x81dabb28,0x6e473865),LL(0x19491209,0x1d9702d7),LL(0xa1f53ee2,0xa4bef482),LL(0x26877597,0x0315b12f),LL(0x18b70eae,0xbca15f03),L_(0x00000095), LL(0x28dc543a,0xe0496daa),LL(0x1afe44da,0x518d4571),LL(0x824106af,0xf72d3376),LL(0xa6b1f64c,0x1149d145),LL(0xbe64f34b,0x8c71ca30),LL(0xd7b0b0f7,0x3acfa7a5),LL(0x58a3ea6e,0xcc0bc394),LL(0x5e42d97e,0xe6ba0355),L_(0x0000009f), + LL(0xb9413a50,0xcc323242),LL(0x25b57cdb,0x0afacd89),LL(0x4d31e696,0x075e88b1),LL(0x80ecc1c8,0x4d853d58),LL(0xe213393a,0x5c2d17b4),LL(0x1d02b340,0x29f6c35a),LL(0x7f4eb22a,0x763f945f),LL(0x991b6570,0x38d9e916),L_(0x0000008a), LL(0x8faf74b7,0x919e262a),LL(0xcdb707d0,0x7b3ccd89),LL(0xc28362e2,0xe31adec3),LL(0xd970a8d2,0x2e5340b0),LL(0x11f1ff4f,0xb76d44ac),LL(0x42bd388a,0x528cddee),LL(0x2165e718,0x7cb055f5),LL(0xa8c2384a,0xd81cae87),L_(0x0000003b), + LL(0xca0a6c60,0x95c8885f),LL(0x504d777f,0x6a9d125d),LL(0x327807d9,0xd624f272),LL(0x0e53c679,0x95ea222f),LL(0xe1387ac7,0xb1597752),LL(0x656acd9a,0x352cac8c),LL(0x6190d15d,0x61bf8856),LL(0x01af7e20,0xc719ce43),L_(0x0000007b), LL(0x3f4af185,0x83f36452),LL(0xfdd73cd6,0x314de5cf),LL(0x46e7aa4c,0x50ce899f),LL(0x7424d707,0x38e875fc),LL(0x3221445c,0x9f3a7a99),LL(0x974ba6c9,0xe06eb667),LL(0xcc871495,0xffe48ed2),LL(0xda22c28b,0x34965180),L_(0x000001bd), + LL(0x067ec5b4,0x8ee0ef29),LL(0xfb175926,0x1c82db3e),LL(0x93d33d24,0xb912da50),LL(0xbb15ebc6,0xcea7d557),LL(0x53132904,0x2a95c0c2),LL(0x1728bce5,0xd703338d),LL(0x3e934774,0x9ff62322),LL(0xa4bdaa17,0x5a25267e),L_(0x000001c0), LL(0x490cbf77,0x4db3f1bf),LL(0x49cf21d2,0xa18c0ec0),LL(0x1567c730,0xf359d391),LL(0x8f78c3e0,0xa1bf7eca),LL(0x9f9aa64e,0x4252d4d8),LL(0x5b2ffd6d,0x3cf77a2e),LL(0x70d5197d,0x420f1fde),LL(0xc4ac046e,0xfbaabfd6),L_(0x00000182), + LL(0x114c23ed,0x6f164190),LL(0xacca9818,0xdc3afb13),LL(0x75acc7ba,0xf8e7944e),LL(0xaec6267e,0x266c89d3),LL(0xc0d5983e,0x7a35b6ea),LL(0xc7025ed3,0xcc6f8ba4),LL(0xb3a78dfe,0xe18e7845),LL(0x43ed79a0,0xd3e423c7),L_(0x0000001e), LL(0xf7626a80,0x7ad878cd),LL(0xf2ddd3e7,0x7ad2ccc8),LL(0xdd3af93f,0xe988a2b1),LL(0xc20e3266,0xf31777a1),LL(0xc508e478,0x5f61decf),LL(0x6ca64937,0x38c983d0),LL(0x63f7f656,0xffbb003b),LL(0xff837029,0x32104839),L_(0x00000105), + LL(0xbf81b286,0x0f23d833),LL(0x787960d2,0x1a82d064),LL(0x1e23da2c,0xf31fd1ab),LL(0x632fca0d,0x67beaa32),LL(0x48f5480d,0xb9e45d26),LL(0xbb162f9b,0xd434d563),LL(0x10e02089,0xa10eef01),LL(0x647082d3,0xb7735d1d),L_(0x000000b7), LL(0x701e6ecf,0x40d95b89),LL(0xe61d29d9,0x68b4d3bb),LL(0x1c7d5b4e,0xd78df4bc),LL(0xbd612a5a,0xd83302ca),LL(0x80982747,0x511140b2),LL(0x754f6279,0x16e7211a),LL(0x1d43610e,0xf0dec954),LL(0x3999e665,0xbaca9f0f),L_(0x000000e6), + LL(0x61d2f78d,0x38877144),LL(0x1ee1b012,0xed46bb99),LL(0xf5e943b8,0x376d8b60),LL(0x1bc6b4d7,0x4b6cb863),LL(0x7dc297da,0x216481d0),LL(0x123486d4,0xc754e76d),LL(0xb1c0a1e8,0xdbcf937b),LL(0xadf930fc,0xdaf06ef4),L_(0x0000013b), LL(0x843220dc,0x03210a22),LL(0x979dbfb5,0xd444dbbf),LL(0xd232212a,0xef6c2520),LL(0x35e7b199,0xee0108b0),LL(0x21bd8267,0xe6dba29f),LL(0xc9483241,0x8b0b6ada),LL(0x304a26d8,0xebc36edd),LL(0x88b58b4d,0x0b4cd577),L_(0x0000009d), + LL(0xb2d92526,0xee657257),LL(0x208861aa,0x5d4bf915),LL(0x8adfc02b,0x8b2a8792),LL(0x51bf7839,0xd1929e39),LL(0x6ac2d82c,0x51878fc6),LL(0x2453f26a,0x67c6a197),LL(0x0ebd963c,0x29e6f9cc),LL(0xab6feb88,0x6a8aecc7),L_(0x00000098), LL(0x8df74f15,0x9f30636d),LL(0xe6a5beb0,0x5f6af11d),LL(0x247b3767,0xa04301fc),LL(0x7893d122,0x577167d7),LL(0x4d974f3f,0xea69addd),LL(0x983fc60d,0xd35bf8be),LL(0x627055a8,0x95c80a83),LL(0x51c83aaf,0xa21f06b1),L_(0x0000009a), +}, +/* digit=24 base_pwr=2^120 */ +{ + LL(0x664d27c9,0x1dc1e136),LL(0x3cf04eac,0x1c4f5e85),LL(0x599f9890,0xcbc44867),LL(0x34ff0e3e,0x5a12a7f8),LL(0x851c12ee,0xf066c152),LL(0x7ca61be6,0x73832df9),LL(0x7153da2c,0x14acdbe2),LL(0xb73e882e,0x87567338),L_(0x0000010b), LL(0xde5b00ae,0xd1fe8148),LL(0x05fd56d3,0x49ee53a4),LL(0xe986a7db,0x11101981),LL(0x695cf7bc,0x750760e2),LL(0xb6aca2a9,0x4815cb90),LL(0x9f5ace2a,0xc3dc9f29),LL(0x6b06b61b,0x3b28698d),LL(0x6b2e5c22,0x5687880a),L_(0x0000000b), + LL(0xe5d59cd4,0x1af552c0),LL(0xaaadcddf,0x160c329a),LL(0xf071e91a,0x77f33e93),LL(0xf9cbbaf7,0x6e836178),LL(0x74f3bc69,0x430ecc6d),LL(0x349ec647,0x9e682571),LL(0xbbec63ff,0x3f624e0d),LL(0x64eff8b4,0x0d19e23a),L_(0x00000000), LL(0x84cb54f8,0xce60d534),LL(0x3eff3832,0xe89d00d8),LL(0x12f600da,0xf8745dbd),LL(0x3eb89d2d,0x48217cd8),LL(0xe79b868a,0xc5ce0f8a),LL(0xc2c4ae44,0x0fe94021),LL(0xa980ca2b,0x5ab9482e),LL(0xf0414674,0xcffa33fc),L_(0x000001c2), + LL(0xfd99bf9e,0xb176fd51),LL(0xa2b01fa7,0xbebf27e3),LL(0x1a17875c,0xca98073a),LL(0x1a08df20,0x73873253),LL(0xcea9581c,0xdc360b05),LL(0xbad316bf,0xb8a68986),LL(0x9591db5f,0x6941db20),LL(0x838ce851,0x0df495ad),L_(0x000000d5), LL(0xa5b29337,0xb5d46b24),LL(0xf0d09b27,0xf2b04a4a),LL(0xe34ef392,0xc4e0cb50),LL(0x9c028d0c,0xbe127061),LL(0x60b8995b,0x202d9276),LL(0xa9beaf92,0x3a61c444),LL(0x686effea,0x7cc238c3),LL(0xce321e42,0x09075147),L_(0x000001e6), + LL(0x4c62b53e,0x16e9dd16),LL(0x8a3599a2,0x05317187),LL(0x0821091d,0x24ef2697),LL(0x6ed2cda3,0x4950f2f1),LL(0xeaefd2e9,0xc815b553),LL(0x9f00612d,0x47c1f801),LL(0x930eacc5,0xfd1730a1),LL(0x136fc4a1,0x8252d52d),L_(0x000001b8), LL(0x6c4bee68,0x6b77522a),LL(0x2deb3842,0xaea9f6b1),LL(0xca869197,0x3823d16a),LL(0x187c4319,0xf12c9d38),LL(0xd5cc9828,0xe31f43da),LL(0x436529c3,0x0781728a),LL(0x63d40c6f,0xbfbb0978),LL(0x94da1798,0x7a196933),L_(0x000001e1), + LL(0x820f8b6c,0x95a20633),LL(0x4ce60573,0x98b05888),LL(0xb9e9ac42,0x9f28e7bc),LL(0x770f80c7,0xabb15751),LL(0xd0147212,0xcce75763),LL(0x67296f82,0x8034afbe),LL(0xa2950d9f,0x11791412),LL(0x9731ca6f,0x87c616f9),L_(0x000000bb), LL(0x004bd5df,0xc7f27dc8),LL(0x5fa5d017,0x95bcc4fc),LL(0x9fdb4deb,0x39917e40),LL(0x30051c1e,0xbefa777d),LL(0x3f36dfcf,0x26ebd51f),LL(0xd9696a85,0x16cc089e),LL(0x58a6c0bc,0x6723f03c),LL(0x3193efb5,0xe4f7a675),L_(0x00000177), + LL(0x63cd0fae,0x0082edbf),LL(0x7ff0d41a,0x1522ffb6),LL(0x76aa53cf,0x453dcda7),LL(0x3ac99dda,0x634bcd8a),LL(0xca31a6bf,0xdf09af12),LL(0xda6aee65,0xb96045d3),LL(0xb2e1c131,0x6f3c7e70),LL(0x72188816,0xcb58f8b9),L_(0x00000121), LL(0x1dcdf91d,0x51f3e032),LL(0xd4da7b11,0xcafbe9a8),LL(0xe3a95788,0x39c010af),LL(0x8c87071e,0x05cb3faf),LL(0xafcfc04b,0x08a702fb),LL(0x42c775b7,0x5b3b6187),LL(0x8aab53d6,0xb84f9386),LL(0x8bb27ffb,0x08491b70),L_(0x00000075), + LL(0x4b15dd19,0x2f20328d),LL(0xb581eaa6,0xd269e274),LL(0x8fb2a285,0x604b1779),LL(0x5b26ea89,0x3aa53ad7),LL(0xd5119e93,0x9fa62691),LL(0x3e002a94,0x8ba167e0),LL(0x62921501,0x195dffb0),LL(0xe4ae2796,0xdc1f93ea),L_(0x00000082), LL(0x1f7743f2,0x980977c6),LL(0x654950f7,0x422ba8f7),LL(0x9f0fcf77,0xb7dc1d4c),LL(0x6b970562,0x0b2f7617),LL(0x2de6068f,0x894ad6c1),LL(0xd3457950,0xe569d53d),LL(0xfc63f78f,0x14981ae1),LL(0xf2a90b52,0x902dadf9),L_(0x000001ab), + LL(0xd7474a9e,0x5406fc86),LL(0xc759885f,0x5bd2a491),LL(0x2d4cddc5,0xc35aa122),LL(0xb5461045,0x2154985e),LL(0x188b457a,0xff0dcbe4),LL(0x235148df,0xd70c6a7e),LL(0xa2535a30,0x7d8e9016),LL(0x6be2be33,0x99a19ee9),L_(0x000000f8), LL(0x747030fd,0xa1dc1860),LL(0x519771ba,0xea4c9a1d),LL(0xe6bf7f8d,0xc44825c6),LL(0xfd4b88d5,0x48270d80),LL(0x619d7b16,0xcd7c088d),LL(0x50ac4887,0xcc2ce67f),LL(0xd1ac72f9,0xaafa6b89),LL(0xafdce091,0xb9365de8),L_(0x000000c9), + LL(0xb743d17c,0x7862482b),LL(0x264a6de7,0xcc327c76),LL(0xf0e8036a,0xdd64bd2f),LL(0x58aff74c,0xd63e620b),LL(0x0d34c089,0x792f3b2b),LL(0x3cffd08d,0xb8e40e30),LL(0x2466d774,0x5be025a8),LL(0x83e235d8,0xe9d3cf63),L_(0x000000e7), LL(0xb1498ace,0x93020cb2),LL(0xfef8d504,0x75b3c700),LL(0xa8404df1,0x1f7b3ca4),LL(0x25cfc4f3,0x3dca1055),LL(0x21fc5f18,0x73402205),LL(0x096d5dd9,0xf8afba2b),LL(0xe13c530e,0x23634751),LL(0x7b6b3f2e,0xca1be461),L_(0x00000076), + LL(0x544eb7eb,0x662154b2),LL(0xf439d6d1,0xc1e155ac),LL(0x0ae5f642,0xa2dd72ce),LL(0x55e79db1,0x71c8da23),LL(0x7905f65a,0x29ffbd0d),LL(0x21383ef6,0x39515d47),LL(0x28c8f708,0x739d692e),LL(0x67130a0a,0xe8283125),L_(0x00000098), LL(0x8eb8a388,0xe6e2797b),LL(0x7f499c43,0x57f047a7),LL(0x8571cbc2,0xe0447784),LL(0x8f068f3b,0x85efc6b1),LL(0x9497bb27,0x4b52e9d2),LL(0x5f954c9e,0x896bc179),LL(0x299e982a,0xfe335eac),LL(0x2fe557d1,0x15ed5037),L_(0x000000c8), + LL(0xc3bc83c7,0x45022f8e),LL(0xe4a9bc90,0xcb58c8df),LL(0xd48d6951,0xf9950f95),LL(0x3a67fa88,0xcfc52411),LL(0x8aad81c0,0xea907dba),LL(0x193feef4,0xbcf6e329),LL(0x847c4744,0x51539dcb),LL(0xbfaf49f8,0xe1705ff1),L_(0x00000189), LL(0xe91285a5,0x7e920920),LL(0x93070144,0xf4a966bf),LL(0xebc39792,0x26b6e21a),LL(0x5380b22b,0xe6fd22df),LL(0xf5ce884f,0x866ea333),LL(0xbbd94169,0x3e0f11de),LL(0x3a3c3087,0x2fd9dd8d),LL(0xefe676ed,0x227a4771),L_(0x00000016), + LL(0x429e8a22,0xe1817fcc),LL(0x7ff2aece,0xe4758b83),LL(0x15a3e785,0x768947ca),LL(0x54660e77,0x2c352eca),LL(0x1486538c,0xaac39b78),LL(0x86e8fec5,0xaa608004),LL(0x414e550f,0x32acb85c),LL(0xa6493364,0x2fd7f2f3),L_(0x00000099), LL(0x2b5bf57f,0x92784c4d),LL(0xcdd72f6c,0x18258546),LL(0x7a0d4685,0xabc0f043),LL(0xa00c87a4,0xfab6104e),LL(0xd492feca,0x0edfb4e6),LL(0xd10ae319,0xa0ad3d18),LL(0x74331002,0xb3e27cea),LL(0x1c928b0f,0x43d33a89),L_(0x000001cb), + LL(0x4af6b119,0x7f32173f),LL(0x5c89c248,0x82306719),LL(0xa569342d,0x5318cae6),LL(0xaf11c888,0xb3871b59),LL(0x43ea3a04,0xccb1894e),LL(0x7d6e369f,0x584ca2b8),LL(0x4a0018e5,0x01476d73),LL(0xc7bd79b4,0x6328258d),L_(0x0000017f), LL(0xfc19d3b7,0x607af994),LL(0x27381ef0,0x686aee98),LL(0x28efc122,0xe773f07b),LL(0x1e1202b4,0xb162dfca),LL(0xd4141270,0xfcd08076),LL(0xdb1eed20,0xd4dbdbd9),LL(0xec8f2a2a,0x31a47c86),LL(0x153d8ef2,0x74c6410f),L_(0x00000122), + LL(0x726b7b76,0x0b272d33),LL(0xb7894299,0x2ade0047),LL(0x7215a462,0x8525f896),LL(0xe97dd7d4,0x8faa7fcd),LL(0xf320c207,0x2aaff4f3),LL(0xce32f0aa,0xf98216e5),LL(0xe5a62be1,0x4be7ec71),LL(0x057e6071,0xe8262bc9),L_(0x000000c9), LL(0x43441bd4,0xe0348118),LL(0x1e1c1702,0x67b5b771),LL(0x61dc410d,0x540bfa59),LL(0x52daedcf,0x722428bd),LL(0xa96118aa,0xced4360f),LL(0xaa07a68e,0xd1ae09f8),LL(0x4870992b,0x98c1f34f),LL(0xf97358de,0xea267e80),L_(0x000001e0), + LL(0x7daaa5fc,0xb75d5e00),LL(0xba7367a8,0xcb6a4c5d),LL(0x698ec043,0xc4a8a172),LL(0x58a0e780,0x1c52f090),LL(0x45a0c118,0x7ba85810),LL(0x41f652a4,0x261486ed),LL(0x14a0dead,0xe61b0bd4),LL(0x4a38be55,0x881f7207),L_(0x0000004d), LL(0x46fc26a5,0xb678cf5d),LL(0x8cf0f2b1,0x1c805e0f),LL(0x50bc855f,0x4c9f70d9),LL(0xab5b49a8,0x82a11ee6),LL(0xbf5c0c4e,0xecca8fd8),LL(0x30c1e91a,0x40104321),LL(0x3bccd5ea,0xf20e8305),LL(0x7cc38a3b,0xa89c9d80),L_(0x00000144), + LL(0x18b1bd58,0x6dc98840),LL(0x33535047,0xa105e17e),LL(0xf240ea34,0x0ca7c1ed),LL(0xe0a7225c,0x60ee9bcd),LL(0x8d5abc2e,0xd1b7a04f),LL(0xed201196,0x421fd636),LL(0xee08dcde,0x3a41da5f),LL(0x4d648f1c,0x37a2b18a),L_(0x0000014b), LL(0xd13216ae,0xe2574ca3),LL(0x8f4aa46c,0x42b5000c),LL(0x5e6cb8b1,0x2cc007b3),LL(0xd18aeecc,0x139d4602),LL(0xfad62b8b,0x4857b6e6),LL(0x0b803515,0xaaf5703a),LL(0x7dfe5be4,0x5b88d9b4),LL(0xc7e255f1,0xb42f23b0),L_(0x000000ee), +}, +/* digit=25 base_pwr=2^125 */ +{ + LL(0x035c3bef,0x9827bf41),LL(0xd6c228d6,0x53bd6003),LL(0xac8482db,0xedd6d84e),LL(0x199f6c6c,0x554b59c1),LL(0xc80a2555,0xbb3dd0d5),LL(0x9a255d70,0xb61698fd),LL(0x8ce8ece5,0x01602388),LL(0x0910e4ff,0x21f2b5b4),L_(0x0000008b), LL(0x3956f121,0xba9be6e9),LL(0xe014bad7,0xd6c8e28b),LL(0x7941a6f1,0x983d3be4),LL(0x93e374aa,0xb03efe8a),LL(0x7787501a,0x2ecc1517),LL(0x3863f010,0x8ce1a07f),LL(0x2339ade0,0xb1181652),LL(0x142e138f,0xed660839),L_(0x0000017d), + LL(0x542c405f,0xb7c246de),LL(0xbed2f33b,0xc7b5006a),LL(0xd46decde,0x50c509c6),LL(0x83eafeed,0x09502cf6),LL(0x6c8d2171,0x6fa7b091),LL(0xe284eb82,0x6ef3971c),LL(0x5478a9a0,0x7e812b4b),LL(0xbedbb05d,0xbdf3afd0),L_(0x000001a9), LL(0x519aab50,0x17c0e4a6),LL(0xde9fb976,0x510f0d79),LL(0x2d46f889,0x5085caf9),LL(0x57625cb7,0x63379f4c),LL(0x7679eef9,0x202dc487),LL(0x61e8da06,0xd95a7481),LL(0x933c7094,0x6f198e77),LL(0x7e527ab9,0x3cef9bb6),L_(0x000000a2), + LL(0x1843b506,0x587ef556),LL(0xadb4b17e,0xe6db7725),LL(0x223554b9,0x298840a9),LL(0x8ea40d6a,0xb9987d3e),LL(0x088f1989,0x8c544359),LL(0x98c4e679,0x26877124),LL(0xd4955574,0xaeb47579),LL(0x42531911,0xedd6bd8c),L_(0x00000025), LL(0x84ee90bd,0x17da2be3),LL(0x578452ef,0xf3506ed1),LL(0x26ec7e64,0x400c530b),LL(0x0a9d93fd,0x42c14bcb),LL(0xeec28064,0xdbc44330),LL(0x21d894ab,0x1784b7a3),LL(0x83284ca2,0xbd2fe673),LL(0xbdaabf2c,0x333a314b),L_(0x0000000b), + LL(0x003a62b3,0x6110cba8),LL(0x3dead375,0x261b1296),LL(0x24e572ee,0x4f710c53),LL(0xa4d924c1,0x3234879d),LL(0x2bb72d3a,0xf0242c6b),LL(0x5319d73b,0x56b72596),LL(0x5d438ac3,0x9c1467ec),LL(0xe4eb1ea6,0x40556d55),L_(0x000000ea), LL(0xbfbdc6bd,0x113bb0cf),LL(0xf755482f,0x1fdd8292),LL(0xb750229b,0x36eb56b3),LL(0x8756dd9d,0xd65055f0),LL(0xad24bc9f,0x305fbea1),LL(0x29626eb1,0xfcecb5ba),LL(0xc9855409,0xf6273264),LL(0x81000d0a,0x9d561b22),L_(0x00000124), + LL(0x714f53f7,0xd531bd0b),LL(0x1f33fdc4,0x1e83cdaa),LL(0x527f8e3e,0x867d160e),LL(0xf198e03e,0x1f8e836d),LL(0x319f12f2,0xe5494da5),LL(0x312ddaeb,0x8aa887f2),LL(0x0cacf5c7,0xab111707),LL(0x0ac8def9,0xbe88c645),L_(0x000000aa), LL(0x65f59d0b,0xaae9a35d),LL(0xecaceba5,0x4a0a292e),LL(0xbb26ecc4,0x686acc28),LL(0x1e45b0f2,0x2a87d12e),LL(0x3a62004b,0x0c521e1f),LL(0x1147391e,0x2c697526),LL(0x4d3ecffc,0x940dd92a),LL(0x45f78060,0x3a2ded9e),L_(0x000001cc), + LL(0x33a3e9dc,0xf734ebf4),LL(0x6012408c,0x62256296),LL(0xf1399678,0x6234e097),LL(0x152b073d,0xd0a76b3e),LL(0xbf3c9a35,0x1dc1794f),LL(0xca7a4461,0x0ba3b03d),LL(0xc31edda4,0x3859cdff),LL(0x8b3288b3,0xf848ef1a),L_(0x000001b9), LL(0x178320aa,0x8f1d82f1),LL(0x25e150c8,0x41cecb20),LL(0x4d109c13,0xc9d21d04),LL(0x7441f09e,0x7778b13b),LL(0xe84ff4c4,0xa32c0c4b),LL(0xc5ccc687,0x309d686e),LL(0x4ddb0a19,0x9203c78d),LL(0xba0868c4,0x53181ea2),L_(0x000000d0), + LL(0x65fb7f46,0xe3ff2a58),LL(0x31973b19,0x2a26ad18),LL(0x67b2d91a,0x5fcc6c2f),LL(0xe2db81e3,0x0637d795),LL(0x74742bd8,0x1ed4fcac),LL(0x26659e88,0x30b9bfbf),LL(0x232b6d3c,0xae535c11),LL(0x97bb1796,0x32eef414),L_(0x0000004f), LL(0x356e33eb,0x4a8e4230),LL(0xc9a735e1,0x8c58bcc2),LL(0xa2ae4a3b,0x1cf20755),LL(0xcba626bb,0x30e29d2d),LL(0xc537d49e,0xa170a87e),LL(0x2ce7cb6e,0x6a6c16d0),LL(0x5f03a6b1,0xa45e1673),LL(0xe7f13685,0x5d8c9454),L_(0x00000148), + LL(0xda756c31,0xaf97f8c2),LL(0x16b51e78,0xbb4d7657),LL(0x4d4e4ac9,0x12ece85a),LL(0x2a2be63f,0x2c2556ca),LL(0x191c3b7f,0x12341b0c),LL(0x6c15ecee,0xdf666379),LL(0x2e302dd7,0xce9cb829),LL(0x76d162a4,0xa7f8ba92),L_(0x000001df), LL(0x587aa554,0xd8403973),LL(0x56dae839,0xd9d38a99),LL(0xd9da7dcb,0xb69b8acf),LL(0xd93d0fff,0x4e0adb2a),LL(0xf74f0454,0xbb2ad644),LL(0xb5de013b,0xd489e7d5),LL(0x944ef674,0xa2d2bd3f),LL(0x0ae01d0e,0xdd32d1ec),L_(0x000000ae), + LL(0x1e606163,0xe4705f69),LL(0xcf1b879b,0xacfbcaf7),LL(0x1be6ac8d,0x7318370c),LL(0xdc61b734,0x68c96561),LL(0x0073d96f,0xb94c34d8),LL(0xc1901cf0,0xf081cf45),LL(0xe5c4c386,0xf0fb0845),LL(0xcbb72560,0x26daccc8),L_(0x00000139), LL(0x76c20a0d,0x649de0ae),LL(0x12fe8c98,0x69621218),LL(0xdfb8607d,0x8791c2b5),LL(0xe9d74f1a,0x3844e43f),LL(0x58b63a6e,0xa8d06c72),LL(0x30e1aac8,0xb6d9b103),LL(0xc4264540,0x3d6167d2),LL(0x08191333,0x68c04430),L_(0x00000167), + LL(0xc3017b9d,0x1aa56828),LL(0x0367e359,0x3cdae245),LL(0xb804f8f2,0x72553e1f),LL(0x67c4cfca,0xb65b5da8),LL(0xd9a5c285,0x9a1f0411),LL(0x613cad66,0xba23bbe1),LL(0xf8b4e4e2,0x1cef34cc),LL(0x8c65734b,0x932e9f3f),L_(0x0000015b), LL(0x97a55485,0x0351d0a0),LL(0x59bb05ec,0x31868efa),LL(0x8bafab58,0x0873b1c0),LL(0xec7f2fe3,0x3643b183),LL(0xc1a9b522,0x8e06d826),LL(0x5a21bdcb,0x8e78107b),LL(0xd0770856,0xf66af571),LL(0xb9c9076a,0xc46c020c),L_(0x00000162), + LL(0xce1c46ca,0x5dc8f4b8),LL(0xa33c20bc,0x7ec83bd7),LL(0xd911b15d,0xd15a6121),LL(0x4a74a6f1,0x7ded1664),LL(0x38816e97,0x011743f8),LL(0x3193fcc0,0x2ce300dc),LL(0xda43c181,0x4a353b8c),LL(0x15a04d1c,0xc667d3b1),L_(0x00000185), LL(0x7f19da91,0x0bdf93dd),LL(0xe0cceb86,0x0ead0ff6),LL(0x54678a08,0x869bbb72),LL(0xb18f6184,0x7bd575cd),LL(0x64b65577,0xa032d6e4),LL(0x7c7dc54a,0x322afc12),LL(0x30a518c1,0xb73e6fec),LL(0x94b0be46,0xfb67de43),L_(0x0000016d), + LL(0xe164535f,0x0f962f35),LL(0x9c6f091e,0x29586d09),LL(0xc7324d43,0xf0870634),LL(0xfc9e4d8f,0xa54095ca),LL(0xb869d9b8,0x750af3db),LL(0x6d2001cc,0x1a6baecc),LL(0x24533332,0x4d43331a),LL(0x73cd1354,0xe8c54106),L_(0x00000197), LL(0xadd82371,0x24bca00e),LL(0xe99a79ea,0xbfa3857b),LL(0x6523d1b7,0xf152a797),LL(0x74d5c2b9,0x7c8d0d7e),LL(0xca070e93,0x0c8c05da),LL(0x90e17c3e,0x6e856e17),LL(0xc09e9cee,0x45014958),LL(0x157a95c9,0x8be88b6b),L_(0x0000001d), + LL(0x25cac8a2,0xcdd92148),LL(0x641d359f,0x2502c5ea),LL(0x6f35d51a,0x3893c7d7),LL(0x20bf4812,0x2ac899fa),LL(0xea66bfcf,0x3dd9d780),LL(0x6686f753,0x5853eeec),LL(0x471826dd,0x3f6607f4),LL(0x63551e77,0xab0845ea),L_(0x0000017e), LL(0xa629b455,0xc75e008e),LL(0x1eb5093e,0xf1fc3d61),LL(0x48e575c1,0x02888aae),LL(0x04ab23f0,0xe87f1ead),LL(0xae16fee2,0xb7f7d076),LL(0x3eebdb5c,0x94d4a8d3),LL(0x1d42f789,0x32f711dd),LL(0xb65c5dfe,0xffe8ae2e),L_(0x0000012c), + LL(0x532fb033,0xc71e34a7),LL(0x960dca9f,0xb04fd5f9),LL(0xf94be13b,0xcb350c8c),LL(0xad91afd4,0x507fe2c4),LL(0x64d4307d,0x965e3503),LL(0x5c7ae781,0x150451f8),LL(0x6bf2a6c3,0x730da87b),LL(0x2d1774ae,0x075f7ca9),L_(0x00000198), LL(0x065c4c40,0x7d9d82a2),LL(0xdf7b7ba4,0x9f994c7a),LL(0x02b12659,0xbc50a3bf),LL(0x9dfdd3b8,0x383c8539),LL(0xfd8d4292,0x17ae38e3),LL(0xf28f2f03,0x882096f8),LL(0x5cc24a79,0x4e0ef573),LL(0xf15428a2,0x57f145e4),L_(0x000000e7), + LL(0x73fadf09,0x457824fc),LL(0x185ab84d,0x1253397d),LL(0x8d154628,0x387df8c9),LL(0x6bebdcd0,0x9150bff8),LL(0x556713ef,0xe0119e69),LL(0x47194289,0xea336304),LL(0xaea5316a,0xfcab6f8a),LL(0x32095630,0x9256e44a),L_(0x0000017f), LL(0x78b228b3,0x91535ac7),LL(0x1d1ebf3f,0x100cda53),LL(0x2af14479,0xebfd994d),LL(0x0287bad1,0x075babf0),LL(0x868eb0f1,0x4f27433f),LL(0x59c4864e,0xb3ca6bbe),LL(0x042e0b78,0x36fc642d),LL(0xc718e121,0x457b51e4),L_(0x0000008a), + LL(0xa6b3bf80,0x8e3c3743),LL(0x2cea274d,0x79b2083b),LL(0xf6accb4a,0xf7eff159),LL(0x1a2ac9cf,0xbd1a458b),LL(0xc30597c5,0xdaf5afd8),LL(0x67ad0a34,0xad0ce95f),LL(0xfcb5f547,0xf492633f),LL(0xd42c927e,0xd70d201b),L_(0x00000118), LL(0x14dfd7c4,0x7325271d),LL(0x511be774,0x532d9f83),LL(0xe33f2540,0x0e1e6624),LL(0x6202d9c5,0xf8f4394e),LL(0x9c8fa1b9,0xf8528991),LL(0x2359d3b9,0xd88ed641),LL(0x4c00c9ea,0x054c125a),LL(0xbd626daa,0xe0db1f33),L_(0x00000053), +}, +/* digit=26 base_pwr=2^130 */ +{ + LL(0x658f6179,0xd73be466),LL(0x565e43ad,0x8ce3b9fd),LL(0x4a046e43,0xef2d69e6),LL(0xb337e9ed,0x7f11d4e7),LL(0xb4d2646c,0x09fce23d),LL(0x9cfe36cf,0xf8577ee6),LL(0xd497797f,0x1e1b23f9),LL(0xba0fa9f7,0x813fdfce),L_(0x000000d2), LL(0x4f0db76d,0xbc5801d3),LL(0xba1d6ad8,0xfa8c88b9),LL(0x38f8437e,0x58d2c493),LL(0xdf5755dc,0xa5d4147a),LL(0x9f31388e,0x2454e0d1),LL(0xd880f0ef,0xed7c5174),LL(0xf4ab4400,0x2972f596),LL(0x422f97c0,0xfd1f05bd),L_(0x000001b7), + LL(0xefc5f8ca,0xe5308733),LL(0xcdb37e83,0x48081b75),LL(0x60b5bfda,0x38365296),LL(0x9f69f061,0x88a8974b),LL(0x5fb9ec96,0x75444cc0),LL(0xf252002f,0x899c5a67),LL(0x664675a1,0x11db7cc9),LL(0xc6b6d7be,0xe5e85617),L_(0x00000149), LL(0x4ec0d894,0x650536e0),LL(0x7897a846,0x57bdeceb),LL(0xb8acad39,0x39f416b8),LL(0xbb4ba894,0xde12e814),LL(0x45c679cf,0xfa77e0ef),LL(0xbfcd091b,0xae92f35b),LL(0xf3ea6cc5,0xff4f9db9),LL(0x15f66583,0x67f0fed3),L_(0x000001a8), + LL(0x589109fb,0x647774c8),LL(0xd728f100,0x8216c030),LL(0x7565d29a,0x38976a5d),LL(0x0e8d40b2,0xdebd4cac),LL(0xe6c701d8,0x3dc8c008),LL(0x71a01dd8,0x54f5f816),LL(0x85aadb00,0xe571a7d2),LL(0x66dfeb71,0x0d64dc32),L_(0x000001ff), LL(0xdb3c541c,0x8476568e),LL(0x70c9c24c,0x81e7d6f2),LL(0xdfa45074,0x8ce07818),LL(0xc75e724b,0x17be95c5),LL(0xf85a8c49,0x56216aaf),LL(0x71eb7f6d,0xf60fc3e9),LL(0x4afdaffe,0xb5697356),LL(0x598d1d44,0x2dfe785a),L_(0x00000074), + LL(0x649ccede,0x70487d30),LL(0xa5efc98a,0x56482796),LL(0x86f3d005,0x81ed5742),LL(0x41ac177d,0x693c9188),LL(0x41f63ff3,0x544078e1),LL(0xcb0cceba,0xf396ad9c),LL(0xcd9ca803,0x1f2f8905),LL(0xb9a3b9f8,0x4318691b),L_(0x0000005b), LL(0x5e41a528,0xf876e309),LL(0xb6fd45a8,0xf87881ff),LL(0xa8a0715e,0xb8d73d7d),LL(0x074192a0,0xca88981c),LL(0xdc66d086,0x00f41a80),LL(0x8f279d46,0x34882bbb),LL(0xb5564038,0x10c7a90c),LL(0x5552b11c,0xf89b04d8),L_(0x00000053), + LL(0xa21b2d84,0x3f7dbd38),LL(0xce9c88b4,0xf194c13b),LL(0xdc04befb,0xdd6c7f32),LL(0xd71b8746,0xc7a2d3eb),LL(0xb71fb09e,0x497484b9),LL(0x73e11c5b,0x1fc70d7c),LL(0x9831a6ee,0x15940a74),LL(0xc9a49067,0xe36e9b20),L_(0x00000185), LL(0xa93d18eb,0x54606829),LL(0x7dd8cbe9,0x0d9bdc27),LL(0xc774aae0,0x36955f4f),LL(0xcfe0f91f,0x72271ae4),LL(0x1d88d914,0xc0f2a388),LL(0xe1f3ebda,0x63cec6da),LL(0xf2b86354,0xe4a5ad95),LL(0xed0252cb,0xedde22e8),L_(0x000001c3), + LL(0xa6a3dd08,0x840ba74d),LL(0xab9733bd,0x35000794),LL(0xc171b7dd,0x7a0a699e),LL(0x370bb4fe,0xed68a491),LL(0xfb486be6,0xf15b9567),LL(0x86467e73,0x5a72e34b),LL(0x007fbbba,0x4fc2fd9c),LL(0x07f9990e,0xf83d0453),L_(0x00000009), LL(0xc9c8d9c1,0x3ed4cc4e),LL(0xd0aa85e5,0xfa4eda85),LL(0xcc6de111,0xab8aa3e2),LL(0xd8d585dc,0x43bc8ccf),LL(0x69adf3a0,0x9f03e827),LL(0x2ce58643,0x4e3d11d6),LL(0xf05e13fa,0x2820b6d0),LL(0x7af921ff,0x94e1a5fd),L_(0x000001f3), + LL(0x3e1d6ea7,0x8c47f3dc),LL(0xfd8a756c,0xca9eb3fd),LL(0x31799e67,0x5933facf),LL(0x70399eb0,0xe0504d9d),LL(0xdc761184,0x469e7106),LL(0x8ef17d6d,0xcd5f283f),LL(0xb55ec3df,0xdaa7f2c8),LL(0x7711b93f,0xa9a6a6b5),L_(0x0000000a), LL(0x63eb36f0,0xc75a128f),LL(0x9a94b1bb,0xa3a9b3b6),LL(0x99889957,0xd56e141d),LL(0xc45c74e4,0x969c754a),LL(0x455c4484,0xf069f686),LL(0x7584cae6,0xbd579d45),LL(0x441fc298,0x29bfd918),LL(0xea727ee8,0xb0624772),L_(0x00000027), + LL(0x88792dc3,0x7b2e1e8d),LL(0x3ae58142,0x2e71222e),LL(0x90f869db,0x9d393376),LL(0xb1ce0668,0x2d537bfd),LL(0x1a9bff70,0x47346bbf),LL(0x4aeeb846,0x8a0e90f5),LL(0x73c9dd46,0x54e3afa2),LL(0xb6c871a6,0x5945d8c3),L_(0x00000146), LL(0xf4a5960f,0x100e770a),LL(0xac70e87a,0x1c87dbe4),LL(0x797d6d91,0x961a5c5e),LL(0xc5b533fb,0x548c0001),LL(0xb560cfb8,0xa9d47191),LL(0xa65c8463,0x37d39eec),LL(0xcad37d21,0x716bab4e),LL(0x7b0514ad,0x89ad5bc2),L_(0x00000187), + LL(0x5ac51d4a,0x7c1ff897),LL(0x4bd5aa83,0x73534a22),LL(0xb8d76f5e,0x26abe76b),LL(0x8f3282b7,0x76978114),LL(0x14a5cb17,0x1bff40a3),LL(0xb7375a3c,0xb7209f08),LL(0x91b36a89,0xb4553b1e),LL(0xcebaa86d,0x73824616),L_(0x000001bd), LL(0xc027549b,0xbcb95506),LL(0x36ce8449,0x45813245),LL(0x0d1e1b38,0xd6d0eea3),LL(0x7fd0d6a4,0x14a3ad4b),LL(0x14bcb34d,0x4fc99703),LL(0xf4772d1d,0xe5d8c8d7),LL(0x1d59825b,0x6cc8f63c),LL(0x8d26276f,0xba00b77b),L_(0x000000fb), + LL(0xc8a3d5f2,0x06031f54),LL(0x960c67a1,0x32f38594),LL(0x09357fe4,0x3b745f59),LL(0x2a14d637,0x8fa90857),LL(0x653eeaba,0x65744c6c),LL(0xfa37b71c,0xf85872c8),LL(0x3238cb4d,0x9700049f),LL(0xbb9a7dcb,0x4c8ed8c4),L_(0x0000001a), LL(0xed276d40,0x49db6e5a),LL(0x58e268fc,0x4b45feb2),LL(0x84cf99d5,0x2045f9c6),LL(0x80f4779c,0xf44869f9),LL(0xa220c8fc,0x058ad702),LL(0x7e09b470,0x948098cc),LL(0x5bc02559,0x495b8c3f),LL(0x33da20c4,0x197459a3),L_(0x000000f6), + LL(0x6a04a321,0xd2a33c4e),LL(0xb8c0a18c,0x4edc42cb),LL(0x42d862b6,0xd775f940),LL(0x1e91d30a,0x6703500a),LL(0xba0ea3ff,0xa7531dda),LL(0x2773ec8b,0x39b7bed4),LL(0x2d04e32b,0xb4d1689d),LL(0x9117e556,0xd20ddefd),L_(0x0000019e), LL(0x3a4ba55b,0xa10a2f30),LL(0x55b7005a,0x2bae1e82),LL(0x53323c22,0x2ff6304e),LL(0x397190c6,0xd9f087fd),LL(0xa7a8b69d,0xb68e3037),LL(0x12602cd7,0x25d350ef),LL(0x22bf670f,0x86cdc0d1),LL(0x8a47dde9,0x8ee7e2f2),L_(0x0000009b), + LL(0x693bf308,0xc6dbc583),LL(0xd24b6766,0xd31b0ef8),LL(0x95890706,0xe3a35296),LL(0xc90c51cc,0xb8ed7618),LL(0x7cff3a80,0x4973ebf1),LL(0xd473b1c4,0x3a129c68),LL(0x098525e4,0x5036c9f1),LL(0xc374031f,0x3955ea92),L_(0x000001d3), LL(0x4ce3a370,0xf46f1c31),LL(0x64ddf24c,0x9e1fda40),LL(0x70db5256,0x5ea2c55e),LL(0xf8940530,0xf14297ac),LL(0x034f59d5,0xa46ea96e),LL(0x42888331,0x7dc4622a),LL(0x102ad134,0xe007741a),LL(0xfe88a514,0x1db8ec7c),L_(0x00000046), + LL(0x6b484938,0x514ddbd3),LL(0x40394ddf,0xc9c65dd0),LL(0x539c2910,0x679067ad),LL(0x449575b2,0x3e4b50f3),LL(0x3cba3f07,0x3ae8deca),LL(0xdb855b46,0xde55385b),LL(0x16ac2f4b,0xcf4ed383),LL(0x1d879d47,0x1e8113f4),L_(0x00000036), LL(0xdc02e925,0xce4c202b),LL(0x3d4593d6,0x973ac87d),LL(0xf0b4acfe,0x01434726),LL(0x6764442d,0xfe9274d4),LL(0x6b582005,0xe308fe9e),LL(0xf520a500,0xaae35487),LL(0x99c31e18,0xcdca5ee1),LL(0x1d99ed71,0x0f6491b7),L_(0x00000198), + LL(0xb383837e,0x46f862a6),LL(0x65cb077c,0x631e9559),LL(0x9b939d55,0x25138071),LL(0xf40d4552,0x952fcfe6),LL(0xb308097f,0x0a6a5375),LL(0x2e65e8e6,0x3e9edb80),LL(0x1310ce7e,0x9008e712),LL(0x36b60d2d,0xef767e69),L_(0x0000009a), LL(0x40188cf2,0xfa1dc587),LL(0x32ad8729,0x4064ce9d),LL(0xe1763571,0x56c0be29),LL(0x7963b458,0xde3b2135),LL(0x95d575d7,0x66e40952),LL(0xa842ef1e,0x444bd560),LL(0x5e446834,0x9e4dbf26),LL(0xf024c8aa,0xf4d25cc1),L_(0x000000bb), + LL(0x24d33325,0x31f1b543),LL(0x5b8d2482,0x0ec252b0),LL(0x19b88e25,0x0818329c),LL(0xcd8bbb1b,0xdb10a837),LL(0x02e4893e,0x81192510),LL(0x84cd1c11,0xbe980656),LL(0x6c489430,0x0f675008),LL(0x346cc643,0x57e72ed9),L_(0x00000152), LL(0x09725c08,0x8c9e3525),LL(0xc2a2b6e2,0x50c3fc67),LL(0xaf377b60,0x018ff455),LL(0xd7f347bb,0xcd5a7fd3),LL(0x820f28df,0x7a766a20),LL(0xbaa35047,0x2e3e3c08),LL(0xea0d932e,0x620422f9),LL(0x561b15cb,0x78d9ad76),L_(0x000000ff), + LL(0x8cf07187,0x0f23847d),LL(0xcb2c301d,0x46a3121c),LL(0x24b1883c,0x64fb5faa),LL(0x43263cce,0xc10bc090),LL(0x731fce3c,0xe510506a),LL(0x134986c0,0xd2899a05),LL(0xaa30a907,0xd8592433),LL(0x6671f165,0xa5074a40),L_(0x000000ea), LL(0xf3b369eb,0x39b1d8c9),LL(0x4f03f7bd,0xed9a2887),LL(0x4a870054,0xbd121753),LL(0x510756ad,0x9a0d0a37),LL(0x85faa51a,0x35296053),LL(0xdf5c089f,0x15a5c2ed),LL(0x130a237e,0xbd316fba),LL(0x3774ff2c,0x2c9d3ce1),L_(0x000001ee), +}, +/* digit=27 base_pwr=2^135 */ +{ + LL(0x10f99602,0x45c384b6),LL(0x2d4100e2,0x7187b9e4),LL(0xc5264e57,0x2477a817),LL(0xcb20ec20,0xc146fbb4),LL(0xa5dd079d,0x6c49fc51),LL(0xb66b540f,0xa207dd34),LL(0x18cb3114,0xfc85f364),LL(0x79042a4a,0xa886f4d4),L_(0x00000192), LL(0x9d642bee,0x3c62b595),LL(0x7df28ef3,0x09a83c10),LL(0xc98bc18d,0x61720266),LL(0xe8b908cb,0xbfa40c64),LL(0x3266ed34,0xc5f7d00d),LL(0x785d5c5a,0xed6e6699),LL(0x0fda50cd,0x0528d637),LL(0x9fa7129a,0x226a0134),L_(0x000001f6), + LL(0x020b6b5c,0x854a83b5),LL(0xa6b72500,0x82b8a64e),LL(0xf5cc5dee,0xa44f4310),LL(0x82f7e307,0xa979f999),LL(0x26038361,0x36271c95),LL(0x9d4a6e7e,0x2c2e87bd),LL(0x83121a68,0x801461a1),LL(0xdda0c42c,0xc46dd1bb),L_(0x000001ef), LL(0xc8adce87,0xf5ff9d53),LL(0xac7e6d6f,0xba6044cb),LL(0x8a2a18c9,0x4e0b1c61),LL(0x47645723,0x538c1881),LL(0xff1d071b,0x0d20849f),LL(0x3d943038,0x033ae333),LL(0xd1326f05,0xe89c6421),LL(0x504a49c4,0x0c637164),L_(0x00000064), + LL(0x161f0a4b,0xc4db51e8),LL(0x0ee6ce1d,0x76a9fbe6),LL(0x471be04f,0xaee80fe4),LL(0x63fea5d4,0x13ed56ca),LL(0xbb7b1989,0xff53dd5a),LL(0xbdd30335,0x5aa48cba),LL(0x8830cbd1,0xced46a92),LL(0x6ec07f47,0x4d0d3e16),L_(0x00000103), LL(0xbf5f1b1f,0x85d83aa8),LL(0x030c528b,0x3981ba7e),LL(0xf6347818,0x51c072a5),LL(0x8851b9e3,0x6bc6f46b),LL(0x908af12a,0xab612e82),LL(0x11ae86d1,0x194bfdad),LL(0x855184ce,0x3ed70ec9),LL(0xbc5ba81b,0x36a51b16),L_(0x00000121), + LL(0x7e0c514f,0x89a7c665),LL(0x4ba50604,0xf92c410e),LL(0x03183bca,0x325bb838),LL(0xde751063,0x4a227afa),LL(0x61ce2f62,0x8d611fad),LL(0xe1c057fe,0x63741f27),LL(0x26a80815,0xcc3f4944),LL(0xdc51e188,0x1fb19202),L_(0x00000060), LL(0x35ecd6ea,0x23f5c4ba),LL(0x8b90f284,0xeac00c83),LL(0xecc8f9f7,0xc63ca5b1),LL(0xabd4ae3b,0x61f4eb49),LL(0x5868250a,0xde5e94c7),LL(0x8aa62e59,0x2e205082),LL(0xa27ce17d,0x4d94b7ec),LL(0x3cf7dcd2,0x84ff72ff),L_(0x000000ed), + LL(0x85eddbf9,0xd6250a4d),LL(0xc4c48937,0xb7e17582),LL(0x30cd4a1c,0x663cb672),LL(0x4ecce3f6,0x51a07652),LL(0xe3e24952,0x971076ab),LL(0xb2837d4a,0xcfa04494),LL(0xae48378d,0x2f234848),LL(0x35aa4670,0x5204cd94),L_(0x000000c1), LL(0xc684c134,0xce99c049),LL(0x189c18e6,0x1251a582),LL(0xe65b23d2,0x1ea8f76e),LL(0x50f4154e,0xde65bbaa),LL(0x55d8a624,0xd1acdeb6),LL(0x9745647b,0xdbc7b696),LL(0xa1a36741,0xc3af97df),LL(0x0e06b475,0xcec9f674),L_(0x00000035), + LL(0x13fe9d4a,0x9edae224),LL(0xc93ceb2c,0xc40b8881),LL(0x376b68f1,0x493ec443),LL(0x2fe4d107,0x2613f055),LL(0x2adbc0de,0xc264177a),LL(0x6850f4d4,0x999b4445),LL(0x024b1759,0xb5528e8c),LL(0xa532c490,0xfe9cb25f),L_(0x000000e4), LL(0x7dabddfd,0xea2401de),LL(0x29f2c840,0xae4f0565),LL(0x6004e218,0x9745c833),LL(0x45a26d7d,0x1aa8e8c7),LL(0x2e1e3abc,0xf254366c),LL(0xd176c592,0x5dba9a65),LL(0x75f2ce2f,0xcb70eda5),LL(0xef390121,0xdf3bd7c9),L_(0x000000d4), + LL(0x92a0df5e,0xf2fb4c5a),LL(0x612c5e22,0x90ec0ad8),LL(0x19eef2ca,0xf648d0a0),LL(0xb08c2818,0x56957806),LL(0xc6fa4d71,0xe858889f),LL(0xd381edae,0x0d311c34),LL(0x51c58427,0xc7d13fa9),LL(0x223f6153,0xe7ffd714),L_(0x00000046), LL(0x3ef9a256,0xa9aa9baf),LL(0xd185738b,0xa46d0a8f),LL(0xb3308a45,0x74e9630a),LL(0x50e76c6b,0xa8af0eb5),LL(0xe6d664ef,0xb4263c27),LL(0xd6ff5afe,0x0d5ab8af),LL(0xa29e25ea,0x35f45527),LL(0x9641d328,0x8c614ae7),L_(0x000000e6), + LL(0x338228bd,0xe352f406),LL(0x44f05c5b,0x36069000),LL(0x3a7061d3,0xe7fd3e15),LL(0xd82371f2,0xb123a32e),LL(0xc0c29bdd,0x1a15e8ee),LL(0x0938b2d1,0x9bba46b8),LL(0xa2ae38c1,0x66a69b9b),LL(0x470c4e74,0xe7a0607a),L_(0x000001a8), LL(0x4d513c26,0x04e250e3),LL(0x0d3d6116,0x99aa8990),LL(0x2850e69a,0xe87aacf0),LL(0x0f5ea018,0xa9b70f5d),LL(0xe629958e,0xc9dfec50),LL(0x67ad0ad8,0xa19fef72),LL(0xfbbc4dd8,0x4e913349),LL(0x44ef73af,0x36506a6e),L_(0x00000106), + LL(0x4259e2ef,0x4fc61403),LL(0xbe686c91,0x3b319ce9),LL(0xcaf2c252,0x48b002e1),LL(0xd7142b37,0xfd368034),LL(0x4805818a,0x24a14bab),LL(0xfef62905,0x0f3cee8f),LL(0x8e05459e,0xdd641c3a),LL(0xf8a79ba8,0xda069476),L_(0x00000043), LL(0x796a282e,0x39168c60),LL(0x3f0a7260,0x3e3e3f10),LL(0xefe9a0b6,0x98f31f24),LL(0x029755ad,0x68a3bd55),LL(0x56d48cfc,0x08db9e00),LL(0x180b09d6,0xf41fc496),LL(0x43518b4d,0x0a026b4c),LL(0x53fa9a78,0x49c51679),L_(0x00000006), + LL(0xa0eb14a2,0x04533f2b),LL(0xc6b20735,0xf37cab9f),LL(0x59889c71,0x2957243a),LL(0xee6d3e3b,0xcd4ef031),LL(0xc82e2f33,0xbe1fa792),LL(0x9431aaa2,0xd5df936a),LL(0x5897dee2,0x69038db7),LL(0x3c5c1a27,0x49337ba9),L_(0x000000c1), LL(0x9595fa29,0xdaff077c),LL(0x92632965,0xd489db4e),LL(0x73090129,0x940397cb),LL(0x3ab24c2f,0x08747c46),LL(0xa7844d0f,0x4063f57e),LL(0xde4ab15b,0xdfb6a687),LL(0x7bdc8db9,0xc4b7272d),LL(0x670393c5,0xcc129fac),L_(0x000000b3), + LL(0xcad13624,0xd3620658),LL(0xd490ca01,0x480b6735),LL(0xf6b97c19,0x5a38261b),LL(0xeb8077db,0x3a0d7cfc),LL(0x6cb95b1f,0xd822b66c),LL(0x027f3439,0x204c12b4),LL(0x5b1121d8,0xd1662f63),LL(0x50df8b79,0x5a06b5c4),L_(0x00000003), LL(0x20c440aa,0xdd45df26),LL(0x7d35a477,0xdf45caba),LL(0x37ca8bf1,0xb9d5153b),LL(0xa163bc4d,0x2a09a7c7),LL(0x79721ae6,0xf16f3745),LL(0x4901566b,0x8b7edc54),LL(0xda6d915a,0x2073fe4c),LL(0x59c5233d,0x0e719f5b),L_(0x0000017c), + LL(0x986f86b0,0x34d85352),LL(0xb5e1a9d1,0x04b6fb3b),LL(0x96ed674c,0xc5869197),LL(0x2201eaeb,0xc13b24f0),LL(0x43fe141e,0xd5acb880),LL(0x77717702,0xb4c36b2f),LL(0xf913c28e,0xbd9e8fe1),LL(0xbb8bc0cb,0x871dc376),L_(0x00000014), LL(0x2919d227,0xb3b18239),LL(0x9062a004,0x0d96f561),LL(0x84b9c0aa,0xd38134a4),LL(0x384e6a14,0x62e9b9dd),LL(0x434945b9,0x0d2a3f87),LL(0x26111d5b,0x0558e17c),LL(0xca088afb,0x7e83601c),LL(0x5f4109b6,0xf3372d86),L_(0x000000be), + LL(0x0f6c0054,0x2e8b93c7),LL(0x08562f0b,0xbd858543),LL(0x4642e58a,0xf3108a95),LL(0xe72a8e55,0xc3b6dcd3),LL(0x48efed30,0xbf3f1b76),LL(0xeef47f99,0xbe7c393d),LL(0x7808a959,0xb13004f3),LL(0xab865ef6,0x937fdeea),L_(0x0000015c), LL(0x22a644cb,0xde622870),LL(0x5b036454,0x1435996c),LL(0x253cdb02,0xc939a75b),LL(0x00181ca2,0x497b4076),LL(0xc885fd30,0x89ffccfd),LL(0x5be5d64a,0xd221db4b),LL(0xf3ff67a9,0x7c1814ff),LL(0xe534c2d3,0xdff1b3c3),L_(0x000000de), + LL(0x6b17ac7f,0xc9c5a035),LL(0x6fb80668,0xd3037f61),LL(0xf7001431,0x61783bd7),LL(0x7eb67860,0xa8db044a),LL(0x687c5be2,0xbd63e80e),LL(0x72619e19,0x79bd6dba),LL(0x3f54433d,0xd3da5abb),LL(0x53179eab,0xbeded885),L_(0x000001ce), LL(0xd2236048,0x8c1156c4),LL(0x0e48c339,0x6a8706fa),LL(0xd70c895f,0xdef1e5d8),LL(0x74e0aa32,0x628036e7),LL(0xb31a93a3,0xa6fa3b42),LL(0xe7bb3f2a,0x91ab3f15),LL(0xd667e0a4,0x1d5276ef),LL(0x172f04b6,0xac2e330e),L_(0x000001fd), + LL(0x5905e1da,0xcaeed330),LL(0x3479c8d8,0x774a0f8d),LL(0x2da43aae,0x89fab1e0),LL(0x5a52588d,0x22017d07),LL(0xf2088700,0x3344f84f),LL(0xf666f8bc,0xcded1b00),LL(0x98c10e11,0x385b1f15),LL(0x4a35267e,0x4cb957d9),L_(0x000001db), LL(0xa8120217,0xb7f4f85a),LL(0x70aae220,0x50f81138),LL(0xd0547dcc,0x320d34eb),LL(0xaa86f5d7,0x4627a90c),LL(0x313d3af2,0x0d86c9fe),LL(0x9d1708c0,0x93baaabc),LL(0x4bb0c611,0x5e3713af),LL(0x8c78d7cf,0x23abcabf),L_(0x00000180), + LL(0x1c368201,0x8d760d23),LL(0x2a50426c,0x2fd748bd),LL(0xd4451d72,0x84a5084a),LL(0x7d518774,0x395bd1ac),LL(0x41ad7719,0x5dc03d65),LL(0x54b40eaa,0xf42c68a2),LL(0xc699a962,0x481b2b4d),LL(0x78f2ecdd,0xd9badbf1),L_(0x00000145), LL(0x35684fe0,0x6890c940),LL(0x7a9849bb,0xe8615e51),LL(0x5822be91,0xe3c3e516),LL(0x9ed67ca7,0x5ebee67a),LL(0x5438f44c,0xbf03236f),LL(0xf9e45ec0,0x29c5029e),LL(0x412d0011,0x4fd4f4e3),LL(0x09bad0b6,0x5f591e3c),L_(0x000000e1), +}, +/* digit=28 base_pwr=2^140 */ +{ + LL(0xdd9afd40,0x6867ca62),LL(0xb2e8cc83,0x2abfd678),LL(0xd7d6c96a,0xbb6c702c),LL(0xb7b75f62,0x8eb9ab34),LL(0x2a8eb698,0x67b38227),LL(0xee1d1728,0xbff15e40),LL(0x6f600751,0x4ec3001b),LL(0x30ff996b,0x7fb8efdf),L_(0x0000015d), LL(0x29a2746d,0xfc62d76a),LL(0x1c80dd81,0x4a2f2f09),LL(0xc1a9825d,0x4ae9b61a),LL(0xb05a4fb5,0x71a812fc),LL(0xa7baf2db,0x8bb96eaa),LL(0xcc434e4e,0x53c2dfd9),LL(0x8fce5672,0xceeb8e7b),LL(0xd6b948ee,0xc787b7e9),L_(0x000000ba), + LL(0x87a8f7ef,0x44566d20),LL(0x816dab3c,0xa555ef8d),LL(0x68ad0a5e,0x93fa3eae),LL(0xb45ab760,0xad51a41f),LL(0x14a732bc,0x3c784a11),LL(0xcd96f357,0x7e912d99),LL(0x7808bc95,0x547dff3f),LL(0xd022a461,0xd3f93d98),L_(0x0000009d), LL(0x3bed20dc,0xdcf5792b),LL(0x9e50e443,0x1c5d0319),LL(0xab35921f,0xce7e3777),LL(0x61acb763,0xc69a2c80),LL(0xd5a1f19e,0xd4921d8b),LL(0x86d49b86,0x3effd3f1),LL(0xd287849a,0x969ee2c3),LL(0x2319a1d3,0x7987e8d9),L_(0x0000017e), + LL(0x4a3f3c42,0x66e6b355),LL(0x48d7c646,0x494cec8b),LL(0x4319bb26,0x3c15f132),LL(0xa4923bd5,0xb25b7340),LL(0xe36296a4,0xd2c82187),LL(0x62a70b23,0x3a2676cb),LL(0x3ce0a44b,0x15ada951),LL(0x93e13762,0xcdd5bfa0),L_(0x000000bd), LL(0xc34a522a,0xe16f0577),LL(0xeb1d23f2,0x563bc2d6),LL(0x74b1ae5a,0x22ce417d),LL(0xf0676c19,0x8b56e586),LL(0x64665c8d,0xd3d21118),LL(0x4a9d1f08,0xb5b57a1f),LL(0x9ad18a2e,0x121b1440),LL(0x31f16f69,0xd3dba51f),L_(0x0000015a), + LL(0x6c14c349,0x14a0990f),LL(0x1571f4bd,0x8a12a2ae),LL(0xa7e98142,0x64ea4bd5),LL(0xf548a570,0xc2f56d89),LL(0x3a99f05e,0x24fcfb51),LL(0xb029c28b,0x468881de),LL(0x16eb364a,0x54a22d8c),LL(0x9df6df67,0x8e7ba7c2),L_(0x0000019a), LL(0x0875f9db,0x6d585b84),LL(0x8b87eab6,0xf8f2e668),LL(0x61b8a4ae,0x1b210ab1),LL(0xcd5968d0,0x38c32d9f),LL(0x9469f27a,0x2170203f),LL(0x7e65bf26,0xdf5327ba),LL(0x268e8f3d,0x0d743f23),LL(0xbbd5d6a5,0x6866dcf3),L_(0x000000f7), + LL(0x9ee406f6,0xfe75ceb3),LL(0xc2dbf93c,0x7d044fdd),LL(0x05aa3d0f,0x3459ab15),LL(0x1e4c0404,0xbea051fd),LL(0xeeca2cbf,0xa5c86723),LL(0x428637a5,0x81d9dd90),LL(0xd3aca9d5,0xf6461276),LL(0x78277709,0x5fdc5888),L_(0x0000019e), LL(0xee7c5a7b,0x105fdead),LL(0xb799ae3c,0xc919db59),LL(0x5e3595ac,0x2aa1f7f7),LL(0x4e9b6f6b,0x519dab32),LL(0x1054eecc,0xd70aa0c8),LL(0xdab1fa02,0x45046840),LL(0xe8162c46,0x382d8fac),LL(0x3f7fc117,0xc63a2e34),L_(0x0000019c), + LL(0xca65cbda,0x40f45643),LL(0x5e42072e,0xb22b4730),LL(0x6980bc47,0x0c0959ae),LL(0xd0091f48,0x17382117),LL(0xe76ce6df,0x6fb6755f),LL(0x083b1371,0x8e338195),LL(0x3ce92877,0x57844465),LL(0x22eadd23,0x88650fd1),L_(0x000001f2), LL(0x66b7e9c4,0xb832d4f9),LL(0x40795011,0x2f5eb6ec),LL(0x56106a16,0x439d72fa),LL(0x7a360472,0x9a695980),LL(0x77c4b5ed,0xbd3315f1),LL(0xcd83808f,0xc773b196),LL(0x21f3f41d,0xdcca40dc),LL(0x42518607,0xd975bf10),L_(0x00000120), + LL(0xa0b7f265,0x7643d0a4),LL(0xca61488d,0xc9a4ec9b),LL(0x78d40864,0x08ac32aa),LL(0xd1f91912,0xe2c33dbb),LL(0x4ce17265,0xa6b041d8),LL(0xc73e5e84,0x130222f6),LL(0xcaf07f55,0xbc20bdd0),LL(0x2fe0bc76,0x482195b2),L_(0x00000043), LL(0x45c6a126,0x37f04c87),LL(0xbdd6ee14,0x601822b2),LL(0xb9431fd2,0xf10879b1),LL(0xebee54b7,0xb8d5c027),LL(0x530c61a6,0x52358509),LL(0x3b953e07,0xc05d71ee),LL(0xd055e247,0xfc120f31),LL(0x51f78c21,0xb71a77f5),L_(0x00000040), + LL(0xdd01fc40,0xdcca1655),LL(0xfcdcd83f,0x6f13723c),LL(0x6fe01dad,0x48fc724e),LL(0x10ea92f4,0xe9506b35),LL(0xbacd3171,0x32c50316),LL(0x5b9c3826,0xb525812e),LL(0xb06a2041,0x6095bd35),LL(0x29d1e51d,0x8c9f2aff),L_(0x0000018f), LL(0x9f6b1c54,0xf5e8440d),LL(0x5815a76f,0x4652dd8f),LL(0x0ba6e7eb,0xa2d772d1),LL(0x67d0cfa7,0x2c2c10a3),LL(0x9134fbe1,0xe122408a),LL(0x4d3fc999,0x98396be7),LL(0xf158ed72,0xf1736447),LL(0x2e445a86,0x3589b3e7),L_(0x00000010), + LL(0x1acdbb17,0xaa39db8a),LL(0xcd1bfa5a,0x3f690830),LL(0xf20580fd,0x47516625),LL(0xc02a443b,0x72df9c02),LL(0x37c50805,0x1f658c86),LL(0x70ba4df8,0xb9b7c15e),LL(0x7863af7e,0x4f048a5e),LL(0xac437359,0x985ed711),L_(0x000000f5), LL(0xe24f4c27,0x31deb67a),LL(0xf7ff8403,0x277a75a7),LL(0x9efc9dd1,0x9e038122),LL(0x72ab76fd,0x380f72e2),LL(0xa5bd7ec4,0x55ee2df7),LL(0xe6e012fa,0x8dba5f73),LL(0x3daacbbb,0x7d57b1b9),LL(0x706e065a,0x2a1528ff),L_(0x00000115), + LL(0xac6b647c,0x15c609d6),LL(0xe5366bdb,0xba4c8f5f),LL(0xab004e8e,0xa55c2b8f),LL(0xbe220e5c,0x9b0a693e),LL(0x328cf3bb,0xf0a01098),LL(0x93659056,0xba4d555d),LL(0xa9299fb7,0x705141f5),LL(0xac2b6ea4,0x44c2570f),L_(0x0000008d), LL(0xcb330456,0x4159e7f0),LL(0xda0acb04,0xd0b0f9e2),LL(0x72227853,0x9c81b6a3),LL(0xfca5d947,0xe37b62d0),LL(0x89f8e6a5,0xa2b087c1),LL(0x397e6f2d,0x79ab8dd4),LL(0x0c2f8337,0xe811e1ad),LL(0xaed2062f,0x41fc3c1b),L_(0x0000006e), + LL(0xbb22cb43,0x62da0bcd),LL(0x66e8ec0f,0xa2436a22),LL(0xb2614d9b,0xc4f2fabe),LL(0xd37ba7ca,0x91730356),LL(0xd6947b5c,0x74afd26f),LL(0xf62dae98,0x24fc84c9),LL(0xa5d82a0e,0x01183e91),LL(0x6d7bad82,0x9ae00850),L_(0x00000098), LL(0x11153170,0xf94e5ea9),LL(0x6a5a8c8b,0x370f5efd),LL(0x4a208fd5,0x0abfbfb6),LL(0xd3eba761,0xb4577a64),LL(0xaea020f7,0x9d9fbff8),LL(0xee185b5a,0x7590eb6e),LL(0xde37c8c6,0x110f6564),LL(0x087e5b3d,0xf182e709),L_(0x00000074), + LL(0x6e7e0a27,0x35933656),LL(0x11881664,0x57d6289e),LL(0xb5dfe85d,0xb19a5774),LL(0x03f55586,0x84a3823a),LL(0x83e66aba,0x819d0f7f),LL(0xe6540e46,0x8229f91b),LL(0xf8e60b64,0x0ebba171),LL(0x3cb7174c,0x13a992ea),L_(0x00000132), LL(0x367ca9d0,0x66b10914),LL(0xaf137af3,0x22188a39),LL(0xa99be2ef,0xc9e8bf06),LL(0x9f80153c,0xb82d6f97),LL(0xb70bb797,0x713e0f8f),LL(0x08001bac,0x2900ebf9),LL(0xc349df5c,0x2dc5150c),LL(0xcda05b0f,0x705ef690),L_(0x0000013e), + LL(0x1c93b8d6,0x2275d0c6),LL(0x43c2cbe5,0xb77f7c23),LL(0x426913e3,0xa4d09bcb),LL(0x193a8beb,0x1c330bb2),LL(0x9694aec2,0xf90a1043),LL(0x466c8910,0x47794b4c),LL(0x013120f4,0x92db08ad),LL(0x27504b4e,0xfd2c4ee7),L_(0x0000008c), LL(0xed071468,0x3983eaf2),LL(0x1520fd40,0x43f9f523),LL(0x10ab4804,0xbbc7abbe),LL(0x4c94f219,0xb3da18c6),LL(0x0653b434,0x34410d29),LL(0xa49aa62c,0x475b1588),LL(0x3fb54eff,0x1efe3b74),LL(0xe35ee322,0xb5457582),L_(0x000001fe), + LL(0xc7c8837b,0xdfafafb9),LL(0xd5ac6ec8,0x3e035e11),LL(0xf1bfe6e7,0x0cdfda44),LL(0x99f86b4b,0xd82854c9),LL(0x51eb2ba8,0xe9378d3c),LL(0xfc70edec,0x0488564d),LL(0x78099daa,0x4df1eac2),LL(0x106d93e9,0xfcd2965d),L_(0x00000092), LL(0x7ad3cd26,0xeb73c32f),LL(0x65c6a4f0,0x12e6dfd1),LL(0x613a95bd,0xc4753f02),LL(0x64c8ae6d,0x6ee36689),LL(0x82594975,0x85faeab2),LL(0xff936e47,0xfd968563),LL(0x16aa8dfb,0xfae37039),LL(0x6a6051eb,0x090bfcd6),L_(0x00000077), + LL(0x75c314c8,0x1283d38d),LL(0xab80a4e1,0xab4695b9),LL(0xb05894a6,0x37378243),LL(0x7f2984bd,0x1227f75f),LL(0xdf654236,0xe2ef58d5),LL(0x290dd3fb,0xdf64907f),LL(0x38ba14af,0xf1d428ec),LL(0x0c38bf2f,0xc2c54bbc),L_(0x00000078), LL(0xb37cd388,0x07837c73),LL(0x31dfd968,0xda6621ef),LL(0x28b820a5,0xe6fe2937),LL(0xb934b778,0x2622aba5),LL(0xe627cb53,0xdff94dc8),LL(0xa81ea0cd,0x560bd971),LL(0x9c8b6e45,0x2209f943),LL(0xdbaad845,0x6e9d457d),L_(0x000001fc), + LL(0x64a50e99,0x8402ef56),LL(0xe6626b55,0x5c34e569),LL(0xbb9dc4c8,0x009d6dab),LL(0x6746cac4,0xcf68656c),LL(0x3336b947,0xfe65ab97),LL(0xe266a898,0x0371ecf3),LL(0x5830a2ee,0x1d57e75b),LL(0xc9710982,0x3e097669),L_(0x00000064), LL(0xfec81877,0x78e2ad77),LL(0xddfb754e,0x284311de),LL(0x4aaa3d53,0xac9d56ca),LL(0xfe5f5938,0x19e9ec29),LL(0x24185a04,0xe89e92d3),LL(0x746f628d,0xfd0968c4),LL(0x6959a461,0x2cc1b198),LL(0x7f39e175,0x5c4efa86),L_(0x00000168), +}, +/* digit=29 base_pwr=2^145 */ +{ + LL(0xd3daa6ec,0x15578941),LL(0x1a86314a,0x6a7421e8),LL(0xe2ec4906,0xe975bc97),LL(0xa7485f37,0xd59fd20a),LL(0xe5e712ab,0x5b001318),LL(0x951133a1,0x1259bdca),LL(0x057f57ee,0xcbd3b2c6),LL(0x33dad04a,0xef3153ef),L_(0x000001e7), LL(0x8c6263d5,0x2ed37d50),LL(0xa4e81e7b,0xf8f36d87),LL(0x5a01a3ef,0x0288c3e4),LL(0x8b372673,0x846f5208),LL(0xa991189b,0x6f560651),LL(0x71db52e5,0x431caeef),LL(0x58e36c06,0xa3f98d5e),LL(0xd8d03f83,0x020099b8),L_(0x000000dc), + LL(0xd73f8b8c,0x52ab1b79),LL(0x7e2040bd,0x95a122c6),LL(0x89ab0660,0xf1cb78af),LL(0x01a20058,0xc77cb751),LL(0x31375e35,0x5e133615),LL(0xea159ba6,0x524c75ea),LL(0x7ecbfca3,0xab8ae0fa),LL(0x5719d039,0x623ac91c),L_(0x0000019c), LL(0x49d36dfe,0x6b1430a2),LL(0x8450eb5d,0xc47b9efe),LL(0xafb92b30,0xa9991147),LL(0xf6824bee,0xe1752c3f),LL(0x2b160b39,0x7fd6a625),LL(0x6256f4b4,0x574646e7),LL(0x076f7bff,0xe5bbdfa9),LL(0xcc3f350c,0x4642b5db),L_(0x000001a8), + LL(0x7f151743,0xfa21d74b),LL(0x37719209,0x8cfe5b17),LL(0x00c8bba2,0x1c2878b2),LL(0xa620523f,0x170331c9),LL(0xa5843ac0,0x8cd83b50),LL(0x0381135b,0xb047131d),LL(0xa643b75e,0xd2ab54c3),LL(0xc5ef1464,0x62ed0e42),L_(0x000001f3), LL(0xad15614e,0x91bb20fb),LL(0x78f86132,0x7805c40a),LL(0x895f7e0d,0xa2a8624a),LL(0x3ce4b54c,0x6579a871),LL(0x1b0cde0e,0xd626e2cc),LL(0x6377df41,0x045193c6),LL(0xcd6454de,0x1c3ca349),LL(0x4909db1f,0xb047b0a1),L_(0x00000191), + LL(0xb6bf0f8a,0xf432b93e),LL(0x4a6f35d7,0x611248d4),LL(0x62f74f5a,0xff45509d),LL(0xef98d968,0xf78b11dc),LL(0x540d2d90,0x8e0fdb4e),LL(0xf1948691,0xf839178d),LL(0x775c9c48,0x1546952b),LL(0x2da4516e,0xb05a9a42),L_(0x00000148), LL(0xe7052400,0x5a0e6542),LL(0x5c40801a,0xc9bfcea8),LL(0x8cf4381f,0xecff5ed1),LL(0x04226551,0xe3765708),LL(0x3addaf06,0xbf10bb39),LL(0xe6d6327d,0xa7a94c0b),LL(0xde98dcbd,0xc9cc265a),LL(0x9445d1d2,0x39198006),L_(0x000001d2), + LL(0x8785f128,0x6fd53bcb),LL(0x11b88070,0x89212039),LL(0x0fd4310b,0x7c570d88),LL(0xfb34d160,0xe29cc2db),LL(0x8d8b6c1d,0x98ac6419),LL(0x633a2387,0x48918f6b),LL(0x3767a8fb,0xc7f5fff0),LL(0x1de5bf8a,0x517008cf),L_(0x00000167), LL(0x1a4a980d,0xc8a802dc),LL(0x31a9aa05,0x3f45d1a4),LL(0x955dbbaa,0x019bc5a3),LL(0xf987ec6f,0x7819e65f),LL(0xa521ab1b,0x6a8b4966),LL(0x9db12d33,0x1c418ebe),LL(0x5c25c25e,0xd371d986),LL(0x05758d98,0xcdb745fe),L_(0x000001d1), + LL(0x24e96217,0x82dda7a2),LL(0x8cb7272f,0x285a44b5),LL(0xd0fa019c,0x772202b8),LL(0x256b2dc1,0xf7a1827e),LL(0x70cc578d,0xf561fd03),LL(0xf01369b8,0x4b48b6ea),LL(0xb34eeab3,0xf869dc36),LL(0xf55466de,0x10fbfa49),L_(0x000000d5), LL(0x9e31568f,0xa79b35c8),LL(0x72243fd6,0x48942459),LL(0x6f4d4b6f,0xbe3c7cfe),LL(0x4b050256,0x273326f3),LL(0xccad925d,0xcfe66f8b),LL(0x63feb094,0xd430d816),LL(0xe74dd574,0xf5ea27b5),LL(0x45e6d69a,0xe57442d8),L_(0x0000006a), + LL(0x8867f7e0,0x91a18dca),LL(0x52fb15ad,0x64cc9794),LL(0x889fc872,0x76b7b4b5),LL(0x516a4447,0x7f78f44e),LL(0xe0dc9367,0x03435817),LL(0x6c0ef141,0x3e179290),LL(0xdcc3815b,0xa243fcb3),LL(0x57d2c5d1,0x33e3e4cd),L_(0x000001bf), LL(0x8a5e2af7,0x373d3db8),LL(0x567532fe,0xa4edcdd2),LL(0xe2cdd2ad,0x313da102),LL(0x7dc4c171,0x9b6477b7),LL(0x10610301,0xd6614ed9),LL(0xe5dbb13f,0x093e9d03),LL(0xc78d8181,0x34692c91),LL(0xd1998555,0xfad9c4a4),L_(0x0000013f), + LL(0xdf7c0d81,0x22136d3a),LL(0x5150ed1e,0xf12f4a61),LL(0x48b602d1,0x58c86ca8),LL(0x8f3a438f,0x2ad94dbc),LL(0xfd28616c,0xa1741520),LL(0xfc8f344f,0x97e96926),LL(0xa2867b76,0x3f74f49a),LL(0xc963769f,0x9eafe4ec),L_(0x00000138), LL(0x880c04c5,0x8d3271ab),LL(0xeb904c8b,0x361247ec),LL(0xcf0e8b6b,0x9dc846a9),LL(0xf58b8dfe,0x1bd5a3dc),LL(0x46766ec7,0xabb872ef),LL(0x7028f76a,0x5976ea25),LL(0x7d56cad7,0xa7a4c1e3),LL(0x50e6e410,0xd9ef6dff),L_(0x000000a4), + LL(0xee967d04,0x54ca4d62),LL(0xa4adf367,0x2f1d9120),LL(0x9de3bdf9,0xa199c49a),LL(0x911112e9,0x918e1ab3),LL(0x51c4e324,0x1ab9377d),LL(0xdecbb2fc,0x089f9423),LL(0xfbdc7272,0x61643ec7),LL(0x297b6a31,0x8eafbdcf),L_(0x00000099), LL(0x380cfd3e,0xb9b29381),LL(0x9618730d,0x5c79e6e7),LL(0x984e3379,0x9a017cdf),LL(0x6a46a60e,0xb44ef6fe),LL(0x6fd9e713,0x8cf5836b),LL(0x2e3b6ebb,0x29b6614b),LL(0x741582d0,0xa7c94b36),LL(0xb93abf5a,0xc0822faa),L_(0x000001a6), + LL(0x7dad6b6f,0x6d40ef9f),LL(0x75d98fac,0x52ee8497),LL(0x4f994b00,0xb0754aa8),LL(0xae60032c,0x19b6eb82),LL(0xb89fa32d,0x3aea1e12),LL(0xd3d62cba,0xa47b84ef),LL(0x7b3e3f24,0x3738323f),LL(0xa1811a10,0xa83238ea),L_(0x00000101), LL(0x5a4fc143,0xe600e837),LL(0x2ba5692b,0x25fadbb6),LL(0x8c4ff4f6,0xad437e54),LL(0xfa9d42cd,0x14c8f3b0),LL(0x79e73eb8,0xa0355c3e),LL(0xee8fbd21,0xefee74e8),LL(0xb4ebba9f,0x0e987b86),LL(0x0e79123c,0xa0018bb1),L_(0x00000009), + LL(0xd08fa2ed,0x68def816),LL(0x3f12ff36,0x4b57900c),LL(0x7fffe102,0xd2939213),LL(0x70f61f2a,0x4ecb6d5f),LL(0x351a0540,0xca3d4a8e),LL(0x51a7737d,0x887af4be),LL(0xbc6bf04e,0xfca084af),LL(0xafb6ef2e,0x80de41d0),L_(0x0000010a), LL(0x9d26a31d,0xa72b2fa9),LL(0x9b7182ad,0xa970074e),LL(0x18bf55a2,0x056574a9),LL(0xb8d1ebac,0xeba9a5ac),LL(0x4bbdf7b3,0xd324a4b9),LL(0x20cc2ce0,0x56572fe4),LL(0x1a2b2538,0xf24f0245),LL(0xef07dd5e,0x5ab8b3cb),L_(0x000001cb), + LL(0x120fa71d,0x41cdedbf),LL(0xcbb3dcf6,0x8c3fb216),LL(0x19500d09,0xe213167c),LL(0xe814428b,0xac93cb34),LL(0x1a28a2b6,0x861cf475),LL(0xbc74e6e7,0xcc2d45ad),LL(0x0f8c1d18,0xbd9bdb71),LL(0x6d7baa7c,0x43067701),L_(0x0000017d), LL(0x41e3705a,0x8a8c2d8f),LL(0xc8929c33,0xc43c1d40),LL(0x819f1cba,0xdda7d3c4),LL(0x598c12f3,0xe612ee48),LL(0xaa092a4b,0x97324657),LL(0xd55e9103,0x1b8a4a06),LL(0xd7a8f2d0,0x010537d8),LL(0xf7a0ab83,0x9ae31bf0),L_(0x0000002d), + LL(0xb8878e45,0xf56f7c26),LL(0xac1feb24,0x4df5d838),LL(0x15563b3a,0x1ca4e8a0),LL(0xbf968a88,0x62060557),LL(0x3ca8c519,0x46507367),LL(0x743fec64,0x374e7834),LL(0xd6eda8e3,0xe0db390d),LL(0x64260f14,0x96c53e95),L_(0x0000010a), LL(0x929af276,0xb4ebbd85),LL(0x2786a497,0x2343b68c),LL(0xbc5660f7,0x3871cff1),LL(0xa03e99a5,0x32a3116c),LL(0x91a2e2b2,0x39a66a33),LL(0xf1e21170,0xb4a691a2),LL(0x0b59581c,0x760bf647),LL(0xbf35d6e4,0xccdb4699),L_(0x0000001b), + LL(0x3d62a61e,0x8ea4e81f),LL(0x6f0c46da,0x6349f724),LL(0xa1f6221f,0x3d1cb710),LL(0x801a6d7d,0x9a8daaa8),LL(0xdff7216c,0xaabb78f1),LL(0x0d054787,0x6a1b8dee),LL(0x9342cf54,0x7426ffaf),LL(0x8839548a,0x7e189575),L_(0x00000021), LL(0xbcb9c78b,0x645473ec),LL(0xf45138ac,0xcb977455),LL(0x51f3e82d,0x23de028c),LL(0xcb2096e6,0x1236013a),LL(0xd60fa53f,0x790031f5),LL(0x590da1dc,0x41383462),LL(0xd75ce15d,0xaac7003b),LL(0x5c3cf3c2,0xe97d1507),L_(0x00000100), + LL(0x37acbeda,0x552d3a3e),LL(0xf4eca93b,0xae1c95c5),LL(0xdc45bd28,0x0c12e32d),LL(0x5dd7eb7c,0x50ac538b),LL(0x692eb87e,0xc65147a8),LL(0xa055973b,0x8ff87281),LL(0x23507ab9,0x63636873),LL(0x1c85fb4c,0x794d2027),L_(0x0000017e), LL(0x8016a521,0xfe67d871),LL(0x0c89c0f4,0x7e7fa083),LL(0x3b0ba9a3,0x25bac099),LL(0x1416b2e6,0xcbc2cc9b),LL(0xbbcf2943,0xc5a1f7b3),LL(0x354fa11e,0xb195e363),LL(0x61adb945,0xcce31308),LL(0xfde526a9,0x5e8055b0),L_(0x000000f0), + LL(0x68b19904,0x7e3f21a0),LL(0x83926d83,0x58ff928b),LL(0x46424f50,0x50a21088),LL(0x656540b1,0xd69839e2),LL(0x2172157b,0xc836bb43),LL(0x34535e3b,0x1f818f5d),LL(0x61ec6b27,0xf4cd40aa),LL(0x8714bd57,0xfdb8302a),L_(0x000001c8), LL(0x553a3a34,0x0a6b22f2),LL(0x8b7033af,0xdbf4f3b5),LL(0x213a07cd,0xd71e271e),LL(0xfa9434d1,0xc069f3af),LL(0xd5d23e3a,0xc4ccd448),LL(0xe785990c,0xdd215a3d),LL(0x500536e9,0x43168909),LL(0xe45a1f48,0x9f92d8e2),L_(0x000000d3), +}, +/* digit=30 base_pwr=2^150 */ +{ + LL(0x0d6ad654,0xf661dbbf),LL(0x43121ba6,0x2325e379),LL(0x176edfca,0xf0cef68c),LL(0xa3861e28,0x617ac6ed),LL(0xa77e7f84,0x57535e8c),LL(0xd31f498d,0xf36e23d1),LL(0x546d78b2,0x2c3f8810),LL(0xcfc7d55e,0x156a1cb9),L_(0x00000091), LL(0xf8c0b462,0x974ce76b),LL(0x894a4c0a,0xc178af73),LL(0xe4d65f8f,0x5d4f42d7),LL(0xf71cb940,0xf73dac29),LL(0x1d35c689,0x32814192),LL(0xe3cb66f4,0x753255de),LL(0xaf9effca,0xa9814253),LL(0xd34e3d9e,0x22e23b71),L_(0x00000020), + LL(0x61e9684a,0xaa0bda51),LL(0x62c59939,0x9d4f255c),LL(0x1e39fae8,0x74c49bbe),LL(0x09372aef,0x180fc9e6),LL(0xde724860,0x163da12a),LL(0xfa823f50,0xa72a28de),LL(0x965a30e8,0x3c600eca),LL(0x905cf108,0x9f8b9687),L_(0x000000af), LL(0xd936a7a3,0x26afd7d7),LL(0x13810cfd,0x986aa03b),LL(0x37d1ddbf,0xeede05c2),LL(0x2715d035,0xb7ae0b88),LL(0x95ef9e71,0x08124878),LL(0xe5042346,0x9f87f170),LL(0x3054f163,0xebc09360),LL(0xce2e674e,0x593b42f2),L_(0x00000054), + LL(0x123b05cf,0x673811ec),LL(0x60b858de,0xeb464fae),LL(0x677b9369,0xd5f16b47),LL(0x26383f92,0xc119870e),LL(0x3f8c6fe9,0x5da1cbb2),LL(0xf7124d37,0xf6c7c1d8),LL(0xdb2b9c75,0x96be948d),LL(0x93746dbd,0x9988eb57),L_(0x000000c6), LL(0xd9a7bbb0,0x03e8f45c),LL(0x3d8c21e4,0x9b0b40cd),LL(0xc1186513,0xb44deee2),LL(0xf970a928,0x2d95e66b),LL(0xa6ac8009,0x8387cee5),LL(0xbddad6f7,0xfec87180),LL(0x0d3ded17,0x2404e11c),LL(0x41ea3e64,0x725101e4),L_(0x000001e0), + LL(0xcbc282c9,0xc6a2f3a1),LL(0xa09feeb1,0x180b5e19),LL(0xc54628c6,0xe8c61be2),LL(0x773cefba,0x054eeedc),LL(0x90648d31,0x0005e411),LL(0x36489351,0x44b74925),LL(0x54e90646,0x573a22ca),LL(0xd626639e,0xa6074dac),L_(0x000001b5), LL(0xcb4398c6,0x9e2e1f28),LL(0x11161ac9,0x4f328fba),LL(0xaaf012b0,0xb74a91c1),LL(0xadb60a6c,0x0cf3c48d),LL(0x1b818269,0xf7c4e07d),LL(0x9eb0dacb,0x2e6fbed1),LL(0xaba09048,0x9ea1ef81),LL(0xaab8c6fb,0x4b567809),L_(0x0000016d), + LL(0x492f635a,0xdee1b8d1),LL(0xeadd7be3,0x42ed487b),LL(0xcb4bb355,0x508d338c),LL(0x1d927c01,0x671a9478),LL(0xd1e3ea8e,0x6482584e),LL(0x83bdc72c,0xb63d17da),LL(0xe52363b8,0x49266941),LL(0x4b78813a,0xcb9e3414),L_(0x000001cf), LL(0xfef1ce8d,0x1f691526),LL(0xb7f97367,0x8a234b55),LL(0xb87b73bd,0x107f953d),LL(0x2944bffc,0x7c0ce6fb),LL(0x6166fb64,0xe784fca9),LL(0x0a71a69e,0x864d9dbf),LL(0xa770d1de,0x1d767a82),LL(0x641a01bc,0xcb0ce972),L_(0x0000019b), + LL(0xab3cbf9f,0xc878d60b),LL(0xeb346a1c,0x6b8a06cb),LL(0x38f8292a,0x28e10a9b),LL(0xa02441ad,0x110ae3e5),LL(0x374d8f2e,0x9df680d7),LL(0x622d31b8,0x0be1994b),LL(0x98b8d29a,0x35da2573),LL(0xcf273b8e,0x5a38591a),L_(0x00000099), LL(0x9b98c33f,0x49364b7a),LL(0xd85cd33a,0x18db5402),LL(0x71a1b4e6,0x7ccbb0bd),LL(0xda26853d,0xc76e0476),LL(0x1360631f,0x888e44f3),LL(0xf6b0ad63,0x2c3af0f8),LL(0xbec71f59,0xdbf01e8d),LL(0x723b0fd6,0x92661703),L_(0x0000000b), + LL(0x1651d7c1,0x28ddccde),LL(0x38aeac57,0x4e993e85),LL(0x0db5dd87,0x38abc090),LL(0xb465add4,0x0c1739c3),LL(0x43d0e74a,0x70bd3e21),LL(0x0b277d58,0xac3af0a3),LL(0x7b2c5422,0x770a41ce),LL(0x08580ab5,0x9864e070),L_(0x00000019), LL(0x9dfe51ae,0xe88e2a8f),LL(0x6783af82,0x37cfc10e),LL(0x6261464c,0x8e8c7510),LL(0xcc9c836e,0x503598b9),LL(0x560d6425,0xcf7c6100),LL(0x4d90b834,0xa8db43b7),LL(0x2444a629,0xd2cb3f5e),LL(0xcabe2a81,0x64aa2ef3),L_(0x00000068), + LL(0xb8ee9ddd,0x8b6de757),LL(0xa9eb572b,0xe5a924f4),LL(0x650813f9,0x4cddfbbc),LL(0x0f808102,0x750529ae),LL(0x8dbdc23e,0xfc407a67),LL(0x3db36c6d,0x549e5c64),LL(0xadee9ab1,0x55d46bd9),LL(0xacadd1f8,0xf68182d8),L_(0x000000ed), LL(0xeef3f12f,0x66e2fb66),LL(0x24a72828,0xe75f104b),LL(0x50b3c877,0xe38bb301),LL(0x2f8590fa,0xb7b5535d),LL(0xf87c6208,0xe1b50eae),LL(0x41ba355d,0x3f0d0c45),LL(0x8bfe9602,0xdc159699),LL(0x3cdcf2ea,0xab8c033f),L_(0x00000141), + LL(0x7513c344,0x29dba96e),LL(0xc3f8209e,0xb1d945a2),LL(0x08e3fd6a,0x6511a3da),LL(0x9263ef8c,0x2562d483),LL(0xd579038d,0xc4c88945),LL(0x5094d203,0x75e4003b),LL(0x54ec1258,0xbe6102b0),LL(0xe7874a8c,0x8d34a4cc),L_(0x00000039), LL(0x777964b9,0x018d87b3),LL(0x19a05999,0xaacd7c73),LL(0xd3cb884f,0xe794b313),LL(0x5a8d6b1c,0x598893b1),LL(0x47ab4f51,0x7e862cb6),LL(0xcd145d37,0x58ebff95),LL(0xa0ddf0aa,0xcb716afe),LL(0xa0791796,0xc7f724f2),L_(0x0000009e), + LL(0x69c35453,0x5cadd4b7),LL(0x080a8ec1,0xb97d34b1),LL(0x7709d928,0x24150cc5),LL(0xfda92711,0x9f76ea4a),LL(0xa5dd93a6,0x4e4f7b83),LL(0x826a2138,0xb1f097c8),LL(0xdb8dc9be,0x877e5a70),LL(0xf1a434cb,0xdd51585b),L_(0x00000174), LL(0xdd348044,0x7f07e424),LL(0x166eed95,0x061bf4f5),LL(0x469e3126,0x06b67307),LL(0xacfcc07d,0x87971f8e),LL(0x96d964d3,0xe5aebd3f),LL(0xa4d18cb2,0x5d286291),LL(0x3fbd829f,0x7560bbae),LL(0x277863a6,0x6e83e561),L_(0x000001b3), + LL(0x4a51a459,0x6a2ec0c8),LL(0x6902a948,0x82ea938a),LL(0x1db5acbd,0x9ed742a7),LL(0x26e981b2,0x64d1ad56),LL(0x075f4b10,0x30adf93f),LL(0xfd5008eb,0xdc51091f),LL(0x7f4f1467,0x907912b5),LL(0x0fb17ba1,0x567270fd),L_(0x0000014f), LL(0xf60b44cf,0xbf940606),LL(0x18337830,0xabbf9925),LL(0x7019fd78,0xbb5e1175),LL(0xc937b351,0x1359c463),LL(0xd19eff42,0xfe68dfe8),LL(0xa8892734,0xed1005b7),LL(0x7cc639ba,0x780e1feb),LL(0xb6ff755e,0x1f0082fa),L_(0x000000da), + LL(0x35763b77,0x191bfbe8),LL(0x87367459,0x58859da0),LL(0xd000c521,0x4373d9cc),LL(0x560dbafd,0xfeee235e),LL(0x8d303a3e,0x1fe980f9),LL(0x2a6082ad,0xb5244f01),LL(0x567ed43e,0x8306748e),LL(0xa7eddca6,0x4e531e38),L_(0x00000195), LL(0x65ee7784,0x100101b4),LL(0x4e959563,0x7ab1df8f),LL(0xbcb6c605,0x0218cd6f),LL(0x3a152b14,0x217b7b09),LL(0x9b32670a,0x7924c99c),LL(0x8550cfd6,0xf9af0b38),LL(0xda396f8c,0x27557bfe),LL(0x01351543,0xf74a0d9f),L_(0x0000018c), + LL(0xadf39ad8,0xafc3d641),LL(0x7e899074,0xeac59b4a),LL(0xab6f7e5f,0x9036a3e5),LL(0xd5685de7,0x32b71856),LL(0x6c3ebc40,0xbe82a80b),LL(0x46fa8ac4,0x8d567d33),LL(0x8f1ba3f5,0xe3d61024),LL(0x9622947c,0x175ff060),L_(0x0000006d), LL(0xbeeb648d,0x79460a8f),LL(0x28338621,0xa409e48e),LL(0x6cee22e8,0x04f98fee),LL(0x448a258d,0xb3e86ccf),LL(0xedd8a07d,0x94212741),LL(0xe7c10493,0x0eae65b6),LL(0x72b816f3,0x3c05e156),LL(0xaf3b8cf2,0xbeed59dd),L_(0x00000111), + LL(0x01f1e3a2,0x09ba81aa),LL(0x3d989afa,0xf91b3d7c),LL(0x24d1650d,0xff5c31f0),LL(0xdd5b9bae,0x20976038),LL(0xf21ca860,0x119240c2),LL(0xcea2f4f0,0x317b48b0),LL(0x894a28c8,0x18cdb521),LL(0x70a13f92,0x613d3aff),L_(0x00000022), LL(0xc0c32ed0,0xe8dd6883),LL(0x18e6d135,0xeb2f0e32),LL(0x2b49078e,0xf0a3dbd6),LL(0x80094f8b,0xad97e7ba),LL(0x7ec56e28,0x9e31d818),LL(0x1a28c019,0x2e151983),LL(0x924258ad,0x40e237bd),LL(0x4e48eb49,0x7b03fcb6),L_(0x0000013c), + LL(0x07f62c3d,0x023dd329),LL(0x3dc56c97,0x2622fb40),LL(0x2ec06f52,0xca023e51),LL(0x70809db6,0x4a297188),LL(0x1e390795,0x4a3bacea),LL(0xc30dbed7,0x0824bcd6),LL(0x3a9a9a8c,0x3e051eda),LL(0x98211435,0xbf2fd10b),L_(0x00000065), LL(0x532b9691,0x6a685349),LL(0x272a03cd,0xecc543f9),LL(0x54d18fe3,0xdf8cb8ba),LL(0x9b7c5d19,0x5f3f336d),LL(0x9e4ff288,0x5e647a61),LL(0x2dc0aea7,0xbdca4466),LL(0x24b8d191,0x33cd397e),LL(0x4bd62cf4,0xf66b542f),L_(0x00000030), + LL(0xdea01906,0x6acd9335),LL(0xcbf85583,0x52fb515f),LL(0x7808f963,0x374fc6d2),LL(0x5865c5cb,0xebbbb50f),LL(0xa4d0c81e,0x47f3a5b9),LL(0x29ed702e,0x97b4bc63),LL(0x64f0fada,0xf73be9b2),LL(0xe65e3bbc,0x42f9f14d),L_(0x000001d4), LL(0xd6414f5f,0x4729cdbc),LL(0x47ce590b,0xf363cdb2),LL(0x5be836dd,0x6a8da968),LL(0x5fd32e4b,0x49bdb981),LL(0x076e41c0,0x8d8f7528),LL(0x097db4cb,0x7fc1d50b),LL(0xf829470b,0x75b1cc67),LL(0xd2b6caef,0xd55324b1),L_(0x000000d3), +}, +/* digit=31 base_pwr=2^155 */ +{ + LL(0xcb47b0cc,0xf69ff50f),LL(0xfc1e2456,0x02b84001),LL(0x1c124be7,0xa6c9b545),LL(0x2857f671,0x07337c72),LL(0xb0a89f0e,0x7d3661d0),LL(0xaf022308,0x61f17db2),LL(0xd9b173b2,0xd0457b51),LL(0xf8c65404,0xefc1cd30),L_(0x0000006e), LL(0xfbb3972b,0xfffafb34),LL(0xc5bd6770,0xfc7a7db4),LL(0x0de59815,0x342e8ca8),LL(0x843b5602,0x0e1c9e4f),LL(0x3bfe9122,0x8b0b7c5b),LL(0xa1e2826c,0xe442b313),LL(0x88ce465c,0xf2ef9e99),LL(0x77217ce5,0xa10ff590),L_(0x00000082), + LL(0x3361b6ed,0x9441390c),LL(0x054f8022,0x7143ab58),LL(0x9b74e159,0x8a901ba0),LL(0x116652a4,0x9b4f3635),LL(0x0afb314b,0x45e2ee30),LL(0xd4622886,0x4d2f79f7),LL(0xb66e6167,0x298ff3c1),LL(0x2505aad2,0x27d64009),L_(0x000001d3), LL(0x43f093dc,0x0eb20dd5),LL(0x4b51c2ba,0x74c9cdac),LL(0xbf1d3648,0x10d4063e),LL(0x6b726013,0xc8c6fbaa),LL(0xf8b94ac6,0x6ce6639f),LL(0x91f488ec,0xf454066c),LL(0x24c600b8,0xf37706e5),LL(0x1cff656e,0x434286c2),L_(0x000000a0), + LL(0x6256aa55,0x35d5b009),LL(0x7857cb4d,0x2bf04d1e),LL(0xc85eb560,0xaf5c9697),LL(0x3e426a2c,0x140d9785),LL(0xe234a765,0xfae3a667),LL(0x6a198191,0x6a2fce6c),LL(0x3a779c8f,0x217e7e57),LL(0xb35dd0c7,0xb20040f7),L_(0x000000e8), LL(0x919a22a2,0x36df1d99),LL(0x46ee68e7,0x884f54d7),LL(0x2f9e3760,0x70670755),LL(0x1a8bd746,0xd3fc19b5),LL(0xc34c78ba,0xaf9a102e),LL(0x9b57be0c,0xfe21514a),LL(0x9e7c2e6d,0xde90d865),LL(0xd44207f4,0x19f36d3f),L_(0x000000a7), + LL(0xb1fd44f9,0xd877e284),LL(0x37f60445,0xca4191dc),LL(0x69f0b4df,0x358c7759),LL(0x12aaa285,0x72cf55e1),LL(0x7f71ae31,0x0ffea4f4),LL(0xfc352eb3,0x5b8d412a),LL(0xc7ffc3d9,0xabdbf74b),LL(0x239ccbac,0xd4b6acd1),L_(0x0000010d), LL(0x26f819d4,0xa6870d63),LL(0xd1598751,0xc6b0d1f9),LL(0xc925f0b6,0xa890fd44),LL(0x106c309b,0xcedd20fe),LL(0xc46673f1,0x2408588d),LL(0xebfbcf6f,0xb54153cb),LL(0xa52fed53,0x7b4aaced),LL(0x672bbf3f,0x84a22a21),L_(0x000000d4), + LL(0xe193ed1c,0x2197649c),LL(0xa6098bec,0x132b7114),LL(0xe879a5ea,0xbf33520a),LL(0x5ec11946,0xf7eb2f05),LL(0x76724ae5,0x8b00135e),LL(0xa281ab75,0xe322da16),LL(0x75ecccf9,0xf00478b6),LL(0x5f741662,0x77d420d9),L_(0x000000cf), LL(0x4fbd26f8,0x8deebf19),LL(0x38cf892d,0xdd1ae54e),LL(0x4a486822,0x8e9572bf),LL(0x15d5deb4,0x83965350),LL(0xa31f170a,0x098efd39),LL(0x225bfe44,0x3effcffa),LL(0xd17f63bf,0xf8e3659d),LL(0x72dce9ce,0x561fceba),L_(0x000001b0), + LL(0x6b2523f7,0xaed633fc),LL(0x573eaf11,0xc05113c8),LL(0x5f254d2b,0x283764aa),LL(0xee71c7fd,0x70135776),LL(0x88759ff7,0x33df5ba9),LL(0x84205188,0xd52265da),LL(0x809c0705,0x912507fb),LL(0x641067f4,0x28d91a94),L_(0x000001b6), LL(0x33e3aace,0xc5e6e2ac),LL(0x0000ebfa,0xa6c0565c),LL(0xced796bd,0x6c90c0d4),LL(0x100a3283,0xee187fc8),LL(0x82bcb3d2,0x8d7848e9),LL(0x290e6b62,0x4a59be08),LL(0x5ab586db,0xb9a00808),LL(0xf4b07e2f,0x210d8de2),L_(0x00000156), + LL(0xe2fa8bdb,0x224b3264),LL(0x7213c7fe,0x43204c94),LL(0x13a1a9d8,0xf7f1cdee),LL(0x68201c17,0xd60991d9),LL(0xbe9464cd,0x4334ef4d),LL(0x715fe2ea,0x590e3478),LL(0x7284a69e,0xe07f24de),LL(0x7c088851,0x5ce9bed5),L_(0x0000002c), LL(0x36040931,0x3aeb8798),LL(0x222d178c,0xa01cdb6d),LL(0xe0c1815b,0x29424615),LL(0xec65cc42,0xfd885c8b),LL(0xfd5df228,0xd9564da3),LL(0x9775d121,0xb060eb3c),LL(0xdc43087b,0xbf975586),LL(0x0c723af3,0x941c0856),L_(0x000000ab), + LL(0xfd05258f,0xbf0ac7a2),LL(0x744e57f4,0x8834334c),LL(0x2edb448a,0x4c1f9523),LL(0xc8e4d56a,0x85d4cde6),LL(0x0bd23e3b,0x83063d71),LL(0x45b52f37,0x14ca833d),LL(0x2012d08a,0xff85aaed),LL(0x02ccbe55,0x9fa9b95c),L_(0x00000091), LL(0x646e2555,0x04999b76),LL(0xf355b09a,0x9309a1f5),LL(0x00d64b66,0xb2bd55ad),LL(0x57889605,0x6b121bac),LL(0x20d91b65,0xed693b72),LL(0x1faab888,0x344453ea),LL(0x45d07a30,0x75e36d67),LL(0xf7e7a52e,0x86433618),L_(0x00000079), + LL(0x0a388dcc,0x9079f10e),LL(0x7efb5f88,0x2c050909),LL(0x1cc662d3,0x7e0de0a5),LL(0x5ee0da97,0xb01a8aa9),LL(0x4922eaaa,0xbf868cba),LL(0x64bbc9e2,0xba2129b9),LL(0x0afac1c6,0x38f86242),LL(0xc8be3270,0x7520a9f0),L_(0x00000139), LL(0x304a400c,0x0cf7a18c),LL(0x5d48ee16,0xc85d4499),LL(0x04528b19,0x2452bab5),LL(0x65b2b9c3,0xbcfb2531),LL(0xd43a545f,0xc03362e0),LL(0x07cc670e,0x5d9aafa1),LL(0x58f98004,0xd816f41a),LL(0x324a8340,0x352c0783),L_(0x000001c3), + LL(0xef15b603,0x4b2484fc),LL(0xfc2dc91c,0x3b5be485),LL(0xd7e9f840,0x42217cb5),LL(0x8585ec85,0x3deede9d),LL(0xa1f0053f,0x48c56ddd),LL(0x845902ce,0xb2e99028),LL(0xdbb111fe,0x8f6659bc),LL(0xf1537c2b,0x89960f5b),L_(0x0000015d), LL(0x9799e891,0x88a9e85b),LL(0x39c6986f,0x69af11de),LL(0x0fa555ee,0x0c555b9d),LL(0x411f3b27,0x62266b30),LL(0x0b0e864c,0x784a1194),LL(0x112da824,0xb7ec5b26),LL(0xc56950bf,0x8a57ba0f),LL(0x3866d81f,0x72e0aa00),L_(0x000001b7), + LL(0x74fcfd7a,0xf2f274e7),LL(0x9f7aad66,0xd84871e2),LL(0x040c2554,0x2a4885e0),LL(0x8ddb8ec0,0xbfd317bb),LL(0x2a407fab,0xaa27b70d),LL(0x17f03cf3,0x1bed7718),LL(0x8c3de6bc,0x34f5d378),LL(0x0e550353,0xfef609bb),L_(0x0000014a), LL(0x0902c90e,0xc5275edd),LL(0xd325a149,0x33824d71),LL(0x14d92534,0xba4131f2),LL(0xf74f4dc1,0x81fdb0c7),LL(0xd354ab8c,0xc33be6cd),LL(0x96e68610,0x7d362d2c),LL(0x967ca304,0xad3a9c9c),LL(0x90a06f8b,0xaf6da5b3),L_(0x0000007a), + LL(0xddf9e139,0xc1710f55),LL(0x67ff0e8d,0x7ef6718f),LL(0x601481b6,0xb39b462d),LL(0x57d09ffa,0xad90ba10),LL(0xf83bbbb5,0x918d94f2),LL(0xed4c7a16,0x2bee8d2e),LL(0x9ddb61a1,0xdadd0291),LL(0xde96ab74,0x2e5753e9),L_(0x00000190), LL(0x7de034dc,0xa3926dcd),LL(0x5af3e375,0x827a6eab),LL(0xeb250dce,0x08623cdc),LL(0x52408bd1,0x9a7d0e9c),LL(0x236fdad4,0xf66e3019),LL(0x55ed033a,0x55cf40ab),LL(0x67077bc7,0x33b49be6),LL(0x3d6972e6,0x34396ea4),L_(0x00000153), + LL(0x8bb989ec,0xa4c22061),LL(0x30374f9c,0x83feabc1),LL(0x5043f74b,0xf24a71b6),LL(0x0f58d08b,0x5ac8cda0),LL(0xe7084b9c,0x20120c5a),LL(0xc241d2a2,0xa700c2dd),LL(0x246b4a2f,0xe50e9154),LL(0xe1b127ec,0x240be13a),L_(0x000000aa), LL(0xbc8ef89e,0x0178da07),LL(0x4ac8c26e,0xa3abe616),LL(0xeaa2008e,0x58d98d73),LL(0x16a0bdbf,0xf5f03b56),LL(0x0dd5224a,0xafd2d956),LL(0x656cc265,0xe30a653c),LL(0x896d53dd,0xc8ac8028),LL(0x8038e832,0xe07a2ee0),L_(0x0000005a), + LL(0x20009e51,0xf7671c0c),LL(0xdb94fffe,0x0361d956),LL(0xbca8fdc3,0x860aa7a6),LL(0xff4ebfa5,0xca2b724b),LL(0xd506fbfe,0xe572f34f),LL(0x2e88a7d1,0x430c48ff),LL(0x74822e19,0xeb20b178),LL(0x623c0129,0x07cc6f01),L_(0x000001db), LL(0xa60b4906,0xe9244f5d),LL(0x8954a885,0x2bf3bfbd),LL(0xf3969954,0xa7e331fd),LL(0x80dc93a6,0x16b29c51),LL(0xe85d8098,0xfad960c8),LL(0x7931b35d,0x74ab3a3a),LL(0x2e570f29,0xf4422349),LL(0x54904daa,0x5e1f7007),L_(0x000000f2), + LL(0xa473f03a,0x11b4e5eb),LL(0x5620232e,0x8a138aa8),LL(0x46f706eb,0xa03d24cf),LL(0x6e11ca59,0x7337f5d0),LL(0xad37149e,0xfa3336f2),LL(0xb68bf40b,0xcb9ee77e),LL(0xa7c9f76e,0x8719bf3b),LL(0xf9bd4330,0x45e4e081),L_(0x00000196), LL(0xabe87083,0x4d1bc133),LL(0x8d32cc0d,0x1bd3eff3),LL(0xa11a2038,0xde1eb1a9),LL(0xa2e7f299,0xb382b9cb),LL(0xac50dfdc,0x62fa8c40),LL(0xe2272381,0xa696bb54),LL(0xf025e3e7,0x68bf08ed),LL(0x608f07d7,0x91eb5365),L_(0x0000002b), + LL(0xae8ee138,0xea56e1b7),LL(0x5f8128c4,0x028409b6),LL(0xe5e0d92d,0x8ed0e1c4),LL(0xf3b74f68,0xc55f66d6),LL(0x35d3f9f9,0xdeb2ab80),LL(0x3bde4296,0xa7cb6b64),LL(0x25e29f7e,0x9b9d057f),LL(0x087f5f23,0x17e3fac2),L_(0x000001af), LL(0x23c7d215,0xd463cbb3),LL(0xd926fd3e,0x014b12b6),LL(0xab9ee679,0x3a1bcb9a),LL(0x1f47e609,0x17170593),LL(0xf44f73dc,0x3b0a4387),LL(0xd2a12e51,0x3ce5c7cd),LL(0x473ec3b7,0x7f341e3a),LL(0x6aef1796,0x09a474c8),L_(0x00000038), +}, +/* digit=32 base_pwr=2^160 */ +{ + LL(0x1f0b504d,0xc2dc9808),LL(0x7f1bc655,0xb688a237),LL(0x67de245f,0xd7a61e34),LL(0x30b260cd,0x9aaf28a3),LL(0x9aeae5d8,0xd4e07803),LL(0x53d349d8,0xd7aea422),LL(0x38cabcfe,0x3728bd24),LL(0x25a9960f,0x58af5683),L_(0x000000f6), LL(0x17d640b6,0x816e52ab),LL(0x1bc21ee4,0x31a5819d),LL(0x26613d4c,0x2a5969b3),LL(0x1a8c1407,0xabfa75ee),LL(0xd357015b,0x7c563bc4),LL(0xd2086ecb,0xa4a80425),LL(0x9b8fafb1,0xc2661a2c),LL(0x547ef737,0xe7afb2d6),L_(0x00000020), + LL(0x9838a5e0,0x65726f32),LL(0xb7a9942b,0xa33e2204),LL(0x4a26b80f,0xbbf82a56),LL(0x73c6f40a,0x970dfcc9),LL(0xf9548526,0xf1c38e96),LL(0xd2bbae55,0x2ecb19ab),LL(0x1edd71d6,0x6d97496c),LL(0x2e20adf2,0x17e1cf32),L_(0x000000d9), LL(0xc3991164,0x76aaf44a),LL(0xb67e29ba,0x1031c67b),LL(0x3d1213c2,0xe37fdfde),LL(0xb4f3b345,0xb46f2bbe),LL(0xef5d5bda,0x53442227),LL(0xbdace910,0x75a65c11),LL(0x0e12dac1,0x99010c36),LL(0x58cdb1cf,0x06f25026),L_(0x00000071), + LL(0x49a4961a,0xee441882),LL(0xdeb1c61c,0xf8ff5eb8),LL(0x6080b71c,0x7b2ccc29),LL(0x214b75b5,0xffb3c6aa),LL(0xe80f53b9,0x90a50e70),LL(0xfeb156be,0x0211fd2f),LL(0xa94620e8,0x15422e55),LL(0x085db41e,0x20305265),L_(0x0000015d), LL(0xe6193074,0x139e1933),LL(0x50841313,0x976e986b),LL(0xb6d55898,0x36a0866b),LL(0xa443f795,0xe06bc0b2),LL(0x63ba00b8,0x734e5428),LL(0xdd7a73a3,0x213440e4),LL(0xb2efa382,0xb0905af8),LL(0xe95312ec,0xb084f884),L_(0x000000ef), + LL(0xc6e6f324,0x3f172c5e),LL(0xd8ddaafa,0xeb6e8784),LL(0x785f2ae4,0xf77d65ef),LL(0x4e5db162,0xdec5c58d),LL(0x2375c785,0x4a30bffa),LL(0xc92e0f7f,0x0c920bb7),LL(0x294b17a0,0x26f93d72),LL(0x0a9107e0,0xce9dc095),L_(0x00000111), LL(0x66f1f498,0x2b841c67),LL(0x72452329,0xb0490079),LL(0x0e7ddb4c,0x55646515),LL(0xc3ad47f9,0x4b2a0877),LL(0xd8708db4,0xa4c3de4b),LL(0xb4a9131f,0x938e9d24),LL(0x85e650ae,0x80176c45),LL(0x60bb2e49,0x0248559a),L_(0x00000182), + LL(0x9a9281b2,0x28cef71c),LL(0x2e3e2609,0x5311578b),LL(0xb15a4e84,0x66031c77),LL(0xc30c76cc,0xf2c06ffc),LL(0xc352a0e2,0xea471db8),LL(0x9a687b94,0x2e1e184b),LL(0xb1979864,0x08e1a1c9),LL(0x7d1d84cf,0xa36c823a),L_(0x00000062), LL(0x1fedfb4a,0x47b77555),LL(0x1b0d298e,0xe7833c92),LL(0x071e1319,0xe5e5ae43),LL(0xbf6e6f4c,0x48ff7cbd),LL(0x44726013,0xec042f31),LL(0x861a992a,0x820461f1),LL(0x0e5f80d4,0x5b728532),LL(0x588846b7,0x4edf14c0),L_(0x000001ef), + LL(0x9277436f,0x0f51608c),LL(0x41c6cf4e,0x15b1b366),LL(0x263e7b75,0x6eb6d459),LL(0x041a5063,0x53679a56),LL(0x6ef1d0df,0x9b4abcaa),LL(0xb47a0301,0xae975077),LL(0xd2d427ef,0x62f30c49),LL(0x5a3dfa91,0xc801e565),L_(0x0000018b), LL(0xac347e0b,0x68202783),LL(0x26d59f48,0x4e17501a),LL(0x3895e666,0x202e3866),LL(0xaa8031f4,0xd2af7613),LL(0x8ddf2869,0xa21cc1e5),LL(0xe13d84ff,0x5da3159e),LL(0x8f6eb59a,0xb87bbc9c),LL(0xdc5df9b7,0xb8b6006c),L_(0x00000171), + LL(0x86ea29f3,0x3e6aa5bb),LL(0x9e7a21c0,0xee3c40e2),LL(0x9e430844,0x91ca8307),LL(0x420584b0,0xfb05a033),LL(0x5dc3546a,0x515d7ef6),LL(0xdfae44d0,0x8e97acb0),LL(0xad35608c,0x1c181a0b),LL(0x85a78e5d,0xd8ba90d8),L_(0x000001ca), LL(0x26e7f38c,0x4b1cba50),LL(0x3d89eff9,0x1828d959),LL(0xb8883419,0x9cd1acbe),LL(0xe7788137,0xd9c16250),LL(0xf51b1fc4,0x2f4d66db),LL(0xbf985d68,0xe78a703c),LL(0x98e4fae0,0x8125e5c9),LL(0x9fe12466,0x7096d179),L_(0x0000015c), + LL(0xe90f79af,0x96c267db),LL(0x2de8af3a,0xd7a0da68),LL(0xfc2373c7,0x5ae71058),LL(0x00846c04,0xb05a94e6),LL(0x87910867,0x49ec9a78),LL(0x0df20f65,0xaecf973f),LL(0xd4a6c168,0x30604ed3),LL(0x0b50f6bc,0x2722d421),L_(0x00000134), LL(0x8348ffda,0x3c89badc),LL(0x32767a9d,0x76ac95ea),LL(0xdc1a4baa,0x3eced60d),LL(0x114219cd,0x2d3cddf3),LL(0x557cfa7d,0x4c14e1ea),LL(0xd40b6e23,0x77a3c466),LL(0x24ae1830,0x9bfca752),LL(0x8ee59e15,0x0d62fa0c),L_(0x000000c3), + LL(0xd968b8f1,0x98c15e86),LL(0xdbdbd0e0,0x13fe0c31),LL(0x3f1495d3,0x611ba4e9),LL(0xcaa1f174,0xd93815b9),LL(0xec434016,0x3bdec28e),LL(0xb9edffc7,0x7d039312),LL(0x995ffc03,0x340b94bb),LL(0xf4d0bdad,0xd62628f0),L_(0x00000155), LL(0xe589b818,0x874bb93e),LL(0xd381244e,0xb9a019ce),LL(0x2710057d,0xa746e7f7),LL(0x5f04bc77,0xc973f2cc),LL(0x16b90cee,0x45b7cdcc),LL(0x3bf24131,0xf860483e),LL(0xa97598d3,0xd873a041),LL(0x5da07fc7,0x13ee03df),L_(0x0000007d), + LL(0x4b42de59,0x04785e61),LL(0x6896551c,0xc769142d),LL(0x1f7de113,0xdc5c38a3),LL(0x6f6444e4,0x619b0fe5),LL(0x442a0f4e,0xc1f930e3),LL(0x0e3d13dc,0xc2166fc3),LL(0x16566439,0x1264bd78),LL(0x043b1c6c,0x55bd407b),L_(0x0000011d), LL(0xdab614fa,0x51809e05),LL(0x37cc449e,0xcbcd15b1),LL(0xc268f122,0xde98d3eb),LL(0xd1094f76,0x2f691855),LL(0x38e9385f,0x940e99ca),LL(0xebc0ca85,0x7a41361e),LL(0x633585a2,0xe77d0dba),LL(0xf4c9fedd,0xffae9098),L_(0x0000015e), + LL(0xaabe6909,0xa2f1a549),LL(0xf13eb703,0xc846b81d),LL(0xcf6235f8,0xba752969),LL(0x28bf7176,0xa83689cf),LL(0x4f491b5c,0xac203f35),LL(0xa5c72127,0x17a19c66),LL(0xc5180b7d,0xe3fefda7),LL(0x2a895472,0xbc0194d4),L_(0x000000ca), LL(0x23d607c3,0x854caa47),LL(0x822cdfe0,0x7cac3eb5),LL(0x3c7db833,0x13d80239),LL(0x03909920,0xd8e93f09),LL(0xde83b6de,0xb075d1a2),LL(0x53e966c9,0x372a1d5e),LL(0x5b917dec,0x60dd5294),LL(0x8284dac3,0xff014a15),L_(0x0000013e), + LL(0x9489dc63,0x73b6bc0c),LL(0xf1bfa63a,0x10296f8e),LL(0xabe3e152,0x9dd0aaa4),LL(0xd3d4285b,0x0c15dad8),LL(0x5f828ba3,0x3be85ac5),LL(0x343ddcee,0x2204e02a),LL(0x532735c7,0x7ba86652),LL(0x2a530b1b,0x262994c6),L_(0x000001c1), LL(0xe1b6b90a,0x391a6b29),LL(0x05dfaa4f,0x6b8878ef),LL(0x3e5666ae,0xb1b8a9fa),LL(0xa4a12d5f,0x9ace0b8d),LL(0xc27561ea,0x7b4c8164),LL(0xef8504c5,0xd8cc29d3),LL(0x16570313,0x483fc408),LL(0x3b7b5ec3,0x2f18c762),L_(0x00000183), + LL(0xa9c74e4c,0xb6ecbf65),LL(0x8116e2fc,0x5e8f5e16),LL(0x59e26819,0x5609fad9),LL(0x8fafa607,0x02dca647),LL(0x7fb0c319,0x1e28746a),LL(0x62d45955,0x6e8dafba),LL(0x53e7625a,0x83169dc2),LL(0xe60b1042,0xebf6fde7),L_(0x00000127), LL(0x801f0ca0,0x7b2d8bde),LL(0x5592a1b4,0x1e6bd0f2),LL(0xcdd5271e,0x566eb6fd),LL(0x7f5033b8,0x4ca0b581),LL(0xd99ab0fb,0xb6096f1e),LL(0x3953fc59,0xcf65a6f4),LL(0x1d2ec4b4,0x1920c542),LL(0xd24e43ac,0xbc37795d),L_(0x000000bf), + LL(0x5e44325f,0x154c2ad3),LL(0x9ab5e4b3,0xa83af5c6),LL(0xaf86c5e5,0x17feec16),LL(0xa5cec56c,0x98ec6557),LL(0x84e83213,0x0f7fdcf1),LL(0x4c26d215,0xffda8a76),LL(0x453ea210,0x9ecd3b2e),LL(0x5f3f4d74,0x45856be9),L_(0x000001e9), LL(0x634817f2,0xa68371b9),LL(0x1b7d95b0,0xe827845c),LL(0xee539828,0x8d12cb9b),LL(0xe4618579,0x7d751e1d),LL(0x49a508f4,0xf62e7726),LL(0x71d8ff6b,0x1aa5f1f4),LL(0x1b002961,0xb185989f),LL(0xc7af8411,0x436bb002),L_(0x00000174), + LL(0x8c5980e4,0x83b10389),LL(0x5487e28e,0x6c59c4b1),LL(0xdbe03ec9,0x5812b87f),LL(0x800f9a8d,0xa69e4288),LL(0x0042610f,0xa98baf31),LL(0xa41914ae,0x04c78aca),LL(0xad52d4dd,0x200e6b24),LL(0xe64f0db1,0xef061a8b),L_(0x000001f3), LL(0x93332071,0x9064a4d4),LL(0xf05bb7bd,0xdb0e1035),LL(0x95a8d7b1,0xbed0afed),LL(0x5aa18c8f,0x1db27276),LL(0xbed5ae9a,0x21d6647d),LL(0x87ff9181,0xd1b9171c),LL(0x25ddbbf6,0x6afd3974),LL(0x58651838,0xd5394b3e),L_(0x000001ca), + LL(0xd9ab3528,0x97acef3f),LL(0xdda16fb0,0x1fbdca04),LL(0xe90de335,0xff197a3e),LL(0x7011f9ba,0x10909fc0),LL(0xbf835536,0xa3d538e2),LL(0xd3c214c4,0xd1adfbd9),LL(0x4b2db047,0xa7800e16),LL(0xe30b9e3f,0x3ba0bb0c),L_(0x00000103), LL(0xf015a843,0x3ccb2552),LL(0xb20301de,0xc8c0dcda),LL(0xaf7c3af2,0x06c79c8a),LL(0x7eefe996,0x38fb5284),LL(0x7cb586d6,0x59bf5673),LL(0xec4f260f,0x36f200b1),LL(0x62ff887d,0x39132913),LL(0xe5ed3b69,0xc40f0d7a),L_(0x000000e3), +}, +/* digit=33 base_pwr=2^165 */ +{ + LL(0x5648dadf,0x9d8a516e),LL(0x02a3fb8a,0x5fedd472),LL(0xecb3edff,0x7fb9838f),LL(0x762220d1,0x8b9ac40f),LL(0x23ad98dc,0x59a8311e),LL(0xfb615d6c,0x86c784ed),LL(0xe6c85dc4,0xee5f8f84),LL(0x6bbf81a7,0x58d5bb86),L_(0x000000cc), LL(0x76fcfa36,0xa1d41ef1),LL(0x007acce1,0x20e9778f),LL(0xd8b8126b,0x438944de),LL(0x437a71b3,0x4e76c73c),LL(0x9a1b4b13,0x14a56abd),LL(0xb7385f9b,0x29b4de8d),LL(0x3115d582,0x91b40784),LL(0x15347258,0xba8c32f8),L_(0x00000055), + LL(0xd1af8588,0x99943818),LL(0x2684f683,0x9d27b5d4),LL(0x68a5f913,0xa3ed9c84),LL(0x9f9d03a1,0xe699de7f),LL(0xe3117424,0x6ddd7e41),LL(0x967769d8,0xf6fd89cc),LL(0x0e9e00b5,0x4a6926ea),LL(0x3d7b6393,0x5b068a8b),L_(0x0000011c), LL(0xa86aa414,0x297d21cf),LL(0x9a2aecb4,0xcf147f52),LL(0x251f8677,0xd2a35774),LL(0xf0bbad3b,0xbedc57bb),LL(0xfe5b3790,0xaa31f1db),LL(0xb3cb7422,0x01bb1e75),LL(0x476bcd99,0xb31cdbf6),LL(0x8c278bd8,0x6fb17125),L_(0x0000002a), + LL(0xc3ec92dd,0x19f12734),LL(0x3d48fbed,0xf69ad2bb),LL(0x49bdd26f,0x985b989b),LL(0x61bfbf26,0x451c21eb),LL(0x35f12cad,0xf237a30e),LL(0x680a082d,0x2751a3b3),LL(0x88ebe4c9,0xc7316941),LL(0x0887a8fb,0xa8bdfe94),L_(0x00000103), LL(0xe0d58839,0x356f89f4),LL(0xde19c8b9,0xeab9cd80),LL(0x77afef27,0xf941390d),LL(0x16f538f8,0x8c79f62a),LL(0x9a2c1a2b,0x84a907ee),LL(0xb7aa5d96,0x9877951e),LL(0xfe7d75aa,0x59fbafe1),LL(0xc17b983b,0xb437db42),L_(0x000000ba), + LL(0x577ceafe,0x3a57f7ec),LL(0x4ce56b58,0x1306d958),LL(0xce15377e,0x1e23a49b),LL(0xbad5b26b,0x2d98c317),LL(0xae8b11f4,0xdc523283),LL(0xf50073f0,0xe7af81dd),LL(0xab516099,0x519277c4),LL(0x6a29299e,0x8cb7cfdd),L_(0x0000000d), LL(0x4a1c8223,0xad29a85f),LL(0x9213cb42,0x37030b7e),LL(0x364e5e4a,0xf8a54d03),LL(0x771a3941,0xb7d507ec),LL(0xd6f8ad50,0xddb1def4),LL(0xbd493bf4,0xc65eeab3),LL(0x716822a9,0x7e2f6019),LL(0x1d5d463b,0x062fa75d),L_(0x00000128), + LL(0x635f0819,0x6ebc6aeb),LL(0x755f610d,0x7e269fb2),LL(0xbc7a68af,0x9a7e6748),LL(0x576c91d2,0xaa653529),LL(0x8b42e1e9,0xe03c250b),LL(0x9e921ac8,0xf313cd04),LL(0x500a0736,0x48b57315),LL(0xfbe580a3,0xd15496dd),L_(0x00000070), LL(0xa35133c9,0xe43286f3),LL(0x42537712,0xfad38cc2),LL(0x5ba8dd4a,0xe8e53c49),LL(0x940cf7d8,0x88cb201e),LL(0xe105c906,0x0310db91),LL(0x14eb5137,0xbdf5c752),LL(0x04b87caa,0x73be9996),LL(0x32ce177d,0x545383f4),L_(0x00000129), + LL(0x43d9b9e5,0x181fa26f),LL(0xe91dfab4,0x8f94d28e),LL(0x500e4763,0x031df707),LL(0x2cdd284c,0xfc76fd9e),LL(0xb532df91,0xdbbb6032),LL(0x95140af2,0x0796b18c),LL(0x1a08045a,0xf970af5c),LL(0xb920694c,0x325b81c7),L_(0x000001e7), LL(0xbe6d4fc7,0x6128ccb5),LL(0x68880de0,0x2e3ad7bc),LL(0xb9bdf74f,0x769e9e60),LL(0x43ac2084,0x0eb7035f),LL(0x71aa1b0a,0x443fc7a8),LL(0x9b9cb064,0x8d6eb3bc),LL(0xcda0b792,0x35030dfe),LL(0xcc362ccd,0x29fff962),L_(0x00000029), + LL(0x4b4a6810,0x01f40601),LL(0x78cfd6f2,0x993f944b),LL(0x042c6de6,0xe197b472),LL(0x7bbfb051,0xe877f763),LL(0xa554df58,0x82d5094c),LL(0x5d801ed2,0xd75061e2),LL(0x89c183e2,0x060481cf),LL(0xe8a754c6,0x43706037),L_(0x000000ef), LL(0x5f0ea03b,0xc842ce7d),LL(0x1ecea7b9,0x0c3c295e),LL(0xd56a995a,0x2352f8cc),LL(0x2d519fed,0x7a9172aa),LL(0x95d8bcc3,0x546f4f90),LL(0x6b8cea31,0xaeee4bb3),LL(0x3f188de2,0xb3d9fe63),LL(0x63e62bb8,0x9f32b579),L_(0x000000c1), + LL(0xad0e3d68,0xa42ee214),LL(0xde6a66c4,0x6c6c7d51),LL(0xc1ce9444,0xd2eee21b),LL(0xeaac0d5d,0x8f8f4a8d),LL(0x5914a3e8,0x755296fd),LL(0x6c394520,0xe647dd87),LL(0x3798ebb4,0x696a7a68),LL(0xc9fd6484,0x66ec9d8e),L_(0x000001d8), LL(0x20495f98,0xf4ca34e1),LL(0x01e46446,0x0fc27507),LL(0x4431e6d9,0x0b310d40),LL(0x9766b761,0xe5199614),LL(0x04e26686,0xae7e80f7),LL(0x4f7efe74,0x9829aa76),LL(0xd9535c6d,0x5702e183),LL(0x755a23c2,0x457bd92a),L_(0x000000a0), + LL(0x92e45342,0xf84e8b92),LL(0xa379a575,0xfe2e00bb),LL(0xf17caafe,0xf713e2fe),LL(0xfca28897,0x56bf2a80),LL(0x484598f6,0x6ed19617),LL(0xda495469,0x3fe63788),LL(0x9a48ed8c,0x32c6923d),LL(0xea4ef749,0xdd905e15),L_(0x0000016d), LL(0x948913ad,0xa7aa5cfa),LL(0xcd183286,0xb2b44bda),LL(0xf345a8f6,0xb4add52f),LL(0xfc8c57af,0x001d629f),LL(0x9943972a,0x1f5f64d0),LL(0xee2dc970,0xc7523ceb),LL(0x078e50ed,0xa1b8fb92),LL(0x98c1c85f,0x69d2866f),L_(0x000001f9), + LL(0xc2e67acb,0x225d5458),LL(0xae1049c6,0x1f4012ec),LL(0xe45c8c16,0x89c61650),LL(0xe63f6f74,0x97d5ef6b),LL(0x13c990c2,0x7718d93d),LL(0x3b2af534,0x388b9ecf),LL(0xb4a19bd4,0x53cfc179),LL(0xc67cc8de,0xea4e62f5),L_(0x0000003e), LL(0xa71fbd0c,0xd67d72c3),LL(0xa67e3213,0xb4e77b7d),LL(0x2c9d3a3a,0x77d06f89),LL(0x5e517015,0x66b06c1c),LL(0x79e0be47,0x25f78836),LL(0xacaba839,0xf6cdc997),LL(0x60da7988,0x18ee069c),LL(0x75dac3e5,0x028ed009),L_(0x0000015d), + LL(0xac3a25eb,0x77706fdb),LL(0xce2d42d6,0x121b5db1),LL(0xbfb3bb0d,0xc38da042),LL(0xcc50a951,0x690091d1),LL(0xe0527354,0xad28eb90),LL(0x6d30c1cf,0x4621b3fa),LL(0x72f783b8,0xecc35c39),LL(0x05168f18,0x7e7054f1),L_(0x0000015d), LL(0x1e8042a4,0x437313ab),LL(0xe9455bf8,0xe4a7314e),LL(0x131fad1a,0xb8e1f53e),LL(0xd17720a7,0xbbc2ae3a),LL(0x41e60518,0xde92e4d6),LL(0x53833db7,0xd91c6976),LL(0xa3c1ec67,0x002fce6d),LL(0x353e4fb8,0xf35678d9),L_(0x000000f1), + LL(0x23d51660,0xe8b6d7a1),LL(0xfb16d9c1,0x2d6fbafd),LL(0x4603abda,0xc1c342fe),LL(0xc3174b01,0xf763e29a),LL(0x6ef24fdc,0x081c9b95),LL(0x6ff0c881,0xadc9659b),LL(0xc6c4ce5a,0xb4df1c4c),LL(0x1e8123e9,0x9a4d9154),L_(0x00000087), LL(0x7529564d,0x08513662),LL(0x1126c683,0x7efb0353),LL(0xe6d3af24,0x506609c7),LL(0x31b758c2,0x8f74142c),LL(0x5c5f34e5,0x0a3e4fdd),LL(0x3f4b19a6,0xe42c81b5),LL(0xf8223898,0x57c04ea3),LL(0x21041b37,0x44625a29),L_(0x00000184), + LL(0x46eb3995,0xafb45817),LL(0x273a1b81,0xe782c707),LL(0x122cfd64,0xf60341eb),LL(0x55516c45,0x19dc551b),LL(0x646dbbd6,0x33015a2f),LL(0xc8d289cb,0x1e2ea096),LL(0x01dbc5f5,0x04e60127),LL(0xb40b7fdc,0xaa434764),L_(0x0000016e), LL(0xb59acf0d,0x3cf9666a),LL(0x4a435c55,0xd7f50159),LL(0xa122c995,0xfe948450),LL(0x52defdfc,0xb5bd3afb),LL(0x272ef1f4,0x702b6fc2),LL(0x86c9c7cf,0x578c41a5),LL(0xce279630,0xafedf374),LL(0x57fd35a9,0xdd29b0d6),L_(0x000001c5), + LL(0xa856b3f9,0x8c313c96),LL(0x81430d8f,0xb3ef9728),LL(0x10f97c7d,0x57ddefb8),LL(0x07066ad1,0xca00506e),LL(0x586f421c,0xdaf65ab2),LL(0x2bf4f170,0x2f754fdd),LL(0xf2415152,0x88f0654f),LL(0x18776438,0x5393b3f2),L_(0x000000ff), LL(0x61d716b0,0x560341d5),LL(0xd6250f20,0xce99e680),LL(0x4c66a708,0xd56bd29f),LL(0x440c3774,0x8b248864),LL(0x9f32acf5,0xad54b8ec),LL(0x99d6dad5,0x45a78e51),LL(0xedbfc6a1,0xa505dcaa),LL(0x28bf41f3,0x8a48ad87),L_(0x0000010a), + LL(0x5a6ac292,0x548d3329),LL(0x8505c5bc,0x76050884),LL(0x4dfc2894,0x6b686f2c),LL(0x1e02d4a1,0x9d97a4c0),LL(0x2a447f5a,0x5475b435),LL(0xc828b6cc,0xd2791aa0),LL(0xe6e9d956,0x3b328dfd),LL(0x69ef6cbe,0x988497f9),L_(0x000001b0), LL(0xdf0efa6a,0x628f0906),LL(0x503dd445,0xc52d4b69),LL(0xdd2f1758,0x438796f6),LL(0x16280d32,0xce2abc46),LL(0x2fb22aa7,0xbdfd0070),LL(0xa5833469,0xd5120c6e),LL(0x80d303f0,0x047308c9),LL(0xd719acb8,0x2a731dfc),L_(0x00000036), + LL(0x024c2f1a,0xfb9cf085),LL(0xff178cc9,0x717cdc0a),LL(0xcd1f6670,0x8870fa8c),LL(0xc4d58854,0x99c44c6b),LL(0xd627431a,0xd7a4c31e),LL(0x552f232d,0x85daf88c),LL(0x940140f0,0x1d886818),LL(0x9aa8211a,0x16e4c1b0),L_(0x00000012), LL(0xe6c31591,0x4b6cac59),LL(0x3374279c,0xc878a0ba),LL(0x8991eda2,0x84ea0b3f),LL(0x32e3b4cf,0x5e729a39),LL(0xcc5f3102,0xd47222c0),LL(0xb4346c5b,0xc5c9ba94),LL(0x2995032e,0x41a4babe),LL(0x7ddb493f,0x7b6e042b),L_(0x0000004e), +}, +/* digit=34 base_pwr=2^170 */ +{ + LL(0x982798bf,0xfe921c0f),LL(0x079475b7,0x410ea1a2),LL(0xea0fd52e,0x77d4bbcb),LL(0x212e44af,0x260a54b0),LL(0xc66a7d1f,0x4269af2e),LL(0x4993bda8,0xd04f3479),LL(0x0b15e358,0x0bdfadc5),LL(0x1c67a4d3,0x2250ea3d),L_(0x00000091), LL(0xe7bebfaa,0x2783de4f),LL(0xfb63579e,0xd5ac84fd),LL(0x1abe0cba,0x4b8a145c),LL(0x84082001,0x5d987c51),LL(0xcfadaba8,0xd9eba9aa),LL(0xf5fccfd5,0x82de291a),LL(0x85e551a9,0x372c4557),LL(0x5e2bcee4,0x9d89842d),L_(0x0000009c), + LL(0x088ef390,0xd17b0f39),LL(0x5ae74e03,0xb17b1a43),LL(0xfbdcdaf3,0xe5084910),LL(0x63c90868,0x9102285b),LL(0xd8e63c01,0xd5454d88),LL(0x80d185fe,0x50f99e23),LL(0xf9e19dfe,0xce8d3eba),LL(0x728e09d7,0x51277498),L_(0x00000015), LL(0x031ef4b2,0x777fda1b),LL(0x7188feeb,0xf597fdfd),LL(0x6801e0f6,0x29652f82),LL(0x252e9d17,0x58dec034),LL(0xc6aa0c9d,0x43cc68d0),LL(0x6779b37e,0x9c62a4e7),LL(0x8d509f56,0x0558ca70),LL(0xd90c4133,0x56b5657b),L_(0x00000169), + LL(0x88fe9cc9,0x9ea07210),LL(0xa60d5a9b,0xe0116982),LL(0xb1b1d6d6,0x275ea7bc),LL(0x7f932848,0x607da14c),LL(0x3af5ede1,0x931400f1),LL(0x03040c84,0x3c889175),LL(0x6c5973c9,0xe7a0f614),LL(0x293b333f,0x5267024f),L_(0x000001bc), LL(0x034cc6c9,0x81f51a89),LL(0x99b337fa,0xc1ab5f24),LL(0x9bde774a,0x5f059cb6),LL(0x070a9fc0,0xdd8da34f),LL(0x8499182e,0x7e9ea166),LL(0x829389a7,0xaf460691),LL(0x5d843a97,0x6edc8515),LL(0xf45adbcb,0x1fe7439d),L_(0x000000a0), + LL(0x56937c27,0x8e9008a5),LL(0x241f1037,0x92193f76),LL(0x4e7ecf00,0xf8905d70),LL(0x28097f48,0x6b4870ad),LL(0x389acac8,0x2f86eb6f),LL(0xc3b9a313,0xfe5a3ffb),LL(0x9c6b9598,0x14fb463a),LL(0x2f429f10,0x40890855),L_(0x00000006), LL(0x41024dec,0x530c94ae),LL(0xd0399afa,0xac70ca6d),LL(0x7da5ef17,0x854eb299),LL(0xe2c80b49,0x4afd62b1),LL(0x77d7cf10,0xf0b13757),LL(0xb8dbecfe,0xdfbd794d),LL(0xf21b1b05,0x47404dbf),LL(0x96f1e68e,0x28abdaf2),L_(0x00000009), + LL(0xfb12ce5f,0xe27f7207),LL(0xf97a3f2a,0x787c8ad9),LL(0xd2383086,0x5dd8b1e6),LL(0xfa851816,0x13c110cb),LL(0x0056cac9,0xffc6bc3c),LL(0xca2b8de4,0x9e086187),LL(0xd596553e,0x4495145a),LL(0xa323bbac,0x799ae6f9),L_(0x00000064), LL(0x0aadfb75,0x53c95598),LL(0x813fad43,0x8941d128),LL(0xee24f158,0x81fe1387),LL(0x7ca3f8b2,0xec9a8f90),LL(0xf4bc106e,0x14a7e155),LL(0x9b049dce,0xacb41c88),LL(0xf8e36863,0x1985dcc1),LL(0xc3075358,0xa78ad338),L_(0x00000117), + LL(0x2801b8c1,0x764f13da),LL(0x1663cb94,0xef025b32),LL(0xdbea0296,0xc7c10036),LL(0x846d4ce1,0xa4ebe01c),LL(0xdcd331ba,0x36bdd387),LL(0x165ca514,0xbf0ef724),LL(0x1500e9b6,0xf3d31456),LL(0x2e001ed2,0x0726f097),L_(0x00000169), LL(0xc4ef20e6,0x8451ea7f),LL(0x6ab1aeac,0x04486599),LL(0x2c2e44e3,0x2f540159),LL(0x22a4d1df,0x2edd9124),LL(0x1b2aa406,0x2d29f8fe),LL(0x3d860387,0xaed1f58f),LL(0x2d78d5df,0xbde871a5),LL(0x65a5c46d,0x6c0a2f54),L_(0x000001f5), + LL(0x41cc93dc,0xbc829d92),LL(0xb48b90b7,0x41e85ef8),LL(0x52345df2,0xdc154eba),LL(0x0472e3d9,0xf8b4b2db),LL(0x59486c6e,0xd8c1f468),LL(0x2a84dd3b,0x1d593d50),LL(0xdf33e197,0x775fa504),LL(0x0cadf964,0xa92cc156),L_(0x0000005b), LL(0xe34e3a5e,0x6a0487ca),LL(0xfc3c5102,0x03084458),LL(0x9fa6a745,0xcfa05014),LL(0x51d7c6b7,0x8f18942a),LL(0x8c314152,0x2258ef9c),LL(0x789bb4fe,0x5dcef195),LL(0x1bcd3685,0x2b822a02),LL(0x4e898c2f,0xbded6e36),L_(0x000000f9), + LL(0x8bf4d11d,0xaa163689),LL(0x5ab01bff,0x58bf7fb7),LL(0xba4b1f3e,0x50bc67e3),LL(0xe8b59cab,0xcea4689c),LL(0x0d30cf8a,0xaf1932a3),LL(0xb3d1d8ed,0x92c06e5c),LL(0xfa7949e4,0xb16d8b25),LL(0x6f41db2d,0x10d851b9),L_(0x00000106), LL(0x66676913,0xb154a2b3),LL(0x96d36fe2,0x09d47ca1),LL(0x766e2a61,0xf1863dba),LL(0xa8ef9263,0xe92b5a5b),LL(0xa5da163b,0xcaad9918),LL(0x520c8298,0xf5e79189),LL(0x0a27963a,0x4ab05f91),LL(0x20b8c3b8,0xfd0103c4),L_(0x000001ba), + LL(0x342eb328,0xdd063123),LL(0xa94faf37,0x027545cc),LL(0x751ad636,0x5633398c),LL(0xe2af69a4,0xdb3a42f8),LL(0x9980ca80,0x3ef70c7f),LL(0x2f9f4827,0x03c8083d),LL(0x4bd7694f,0x2297619a),LL(0xcba6106f,0xb857e944),L_(0x00000130), LL(0x5e9d95f6,0xed99704a),LL(0x9b54a475,0xd04dd3a0),LL(0x45640d66,0xe5bc76a8),LL(0xb7f7aff6,0xffcfd663),LL(0x7c51db6e,0x1b1b7d77),LL(0x4ab9daa0,0x312b124a),LL(0xd1e4a043,0xa2044f37),LL(0x1147cf83,0xc70b0257),L_(0x00000184), + LL(0xf3d71f50,0x240110f2),LL(0x7cc2f02a,0xebf0be7a),LL(0x2ac9b5c2,0x24af14f6),LL(0x9a211862,0x782fa4b4),LL(0x50410353,0xf137e0ec),LL(0x1b26ae96,0x241ccb89),LL(0x2cdf1d3c,0x2b213449),LL(0xac6249df,0x063b93cd),L_(0x00000116), LL(0x1cd6f1cd,0x243cd7a1),LL(0xf821c5c8,0xd3358ef9),LL(0x1dea9bcc,0x06149e77),LL(0x744a2dd7,0x76a25a6f),LL(0x510fe3ad,0x126f991b),LL(0x2bff6928,0x9ff56b6f),LL(0x26743e80,0xb7342a0d),LL(0x75a539a6,0x1e395f1d),L_(0x0000018f), + LL(0x6b793a6c,0xc5b17046),LL(0x6c3972f8,0x30453d81),LL(0xa45c6e8c,0xf914e1ed),LL(0x3fd2d983,0x8df9d87a),LL(0x465d7bda,0x0b35e4f5),LL(0x6fe2ce33,0xd6b328dc),LL(0x54ed3799,0xfe08ef94),LL(0x7a45c9eb,0x18e443cd),L_(0x0000001a), LL(0xebd4b819,0x785e8d35),LL(0x203318e2,0x5c35d7f0),LL(0x7b61a2b1,0x652767b2),LL(0x861e2602,0x830e75ba),LL(0x44dc9f10,0x31d6ffa6),LL(0x2ba2cf1f,0x61cf1408),LL(0x6ea5aa79,0x8a5f9d9e),LL(0xc5f5c401,0x983aa3d8),L_(0x0000011d), + LL(0x8d73683d,0xaae3e45f),LL(0xebb6d11c,0x08fdeeb8),LL(0x0f274ee5,0x562c576b),LL(0xee620c83,0x10e47bae),LL(0x88c57185,0x279b8105),LL(0x919ff42e,0x927c894a),LL(0x7edf259b,0x23100a00),LL(0x169d16a2,0x2acb9ccb),L_(0x0000005b), LL(0x9179d06d,0xfee8415a),LL(0x94d74f07,0xcacca4b5),LL(0x4fb6e0e5,0x08cc549b),LL(0x08e788b7,0xc62edae5),LL(0xc2847dc0,0x9b9ef886),LL(0x64c8eee6,0x9deee406),LL(0xed24b57a,0xa5474323),LL(0xca2b9d44,0x16f12261),L_(0x0000006f), + LL(0x5e48d299,0x7ec4af3a),LL(0x1798ff0a,0x06583190),LL(0xce45de42,0xa85d801e),LL(0x661ec8ee,0xe6f87169),LL(0x12391657,0xa304ed8e),LL(0x70c4e172,0x8bed9dc1),LL(0xd437a386,0xa1738ce7),LL(0xd75a62c8,0xfe484c38),L_(0x00000079), LL(0x374f474b,0x9f0ace8e),LL(0xb886c429,0x274adb1a),LL(0x61800342,0x733bc7ba),LL(0x05688d04,0xe49410a8),LL(0x9d06b25e,0x1965c8e7),LL(0x2b710949,0xad70d74f),LL(0x3d588ffa,0x3d3fbe49),LL(0x7220b560,0xe246db1c),L_(0x00000016), + LL(0xff3c0838,0x8d34890c),LL(0x59239eaf,0x15030568),LL(0x6578691d,0xd8dd39f6),LL(0xcb5a2489,0x121c7f85),LL(0xf808f7b5,0x952a8bc8),LL(0x99daec7f,0x84b94629),LL(0xd8fb611d,0xbad11517),LL(0xcd32215c,0x967dfc54),L_(0x000000b7), LL(0xbbc3c9c3,0x4a7b1ca7),LL(0x3e19d6b8,0x4aecc72b),LL(0xc71b44ff,0xa89c3596),LL(0xc6ae9705,0x99c2157d),LL(0x4697a093,0x95d8264e),LL(0xd6e1d1c0,0x6704d656),LL(0x1e3c6190,0xc7b65104),LL(0x91499ef5,0x466ffc24),L_(0x000001d1), + LL(0x540a9b74,0x9b81bb10),LL(0x3ae92664,0x35d0ee3c),LL(0xc1f7e1a2,0x20c5e62f),LL(0x8f740935,0xb7936d32),LL(0x451ab7a1,0xc573c20c),LL(0x126bba09,0x3152bb2a),LL(0xa66454b3,0xb17e342e),LL(0x784051fe,0x3681b560),L_(0x00000027), LL(0x8fd9a547,0x8f5b18c5),LL(0xcc09f617,0x6e34c103),LL(0x22747cab,0xf8a96755),LL(0xde36110e,0x4dcfc108),LL(0x67ee6834,0x342676fc),LL(0x8502fdf2,0x5b48ae34),LL(0x3b4a1019,0x85dba2fb),LL(0x75c0d58b,0xda298efd),L_(0x00000088), + LL(0x42e6e512,0x335496bc),LL(0xd3205850,0x88067d33),LL(0xcf402bd3,0x0074be0b),LL(0x2913e673,0xe8db4e94),LL(0x62a0cb43,0xcf7beef4),LL(0x8c2ead81,0xf06de58a),LL(0x97eccd5d,0x501f23bb),LL(0x1d5954e3,0xa8b8e4e1),L_(0x000000b4), LL(0x025da1b4,0x1d598c1e),LL(0xbf9648fc,0x9987cb09),LL(0xd224f8ad,0xd88fba1f),LL(0x60665a60,0xd86a1d9f),LL(0xf1e7f754,0x26c4ad1d),LL(0x4acf77f7,0x7713e1da),LL(0x938971a2,0x0f78da10),LL(0x3b7fc94e,0x92811d7d),L_(0x00000039), +}, +/* digit=35 base_pwr=2^175 */ +{ + LL(0xbfdb04a0,0x5c665e2b),LL(0x4232c5cb,0x9a24517e),LL(0x26232f5f,0x5981cd79),LL(0x0a27a027,0xe253d4d8),LL(0x783b1d5a,0xd6c00bb7),LL(0x89b5ab0b,0x40ea4c25),LL(0x6c48caf7,0xfc5351cf),LL(0xa482e177,0x2b0e714b),L_(0x000000fd), LL(0x6ac73dd9,0x8552b5ad),LL(0x11881ba7,0xfb2067a3),LL(0x4ccac10c,0x5d449097),LL(0x9013dcbe,0x8873accd),LL(0xb2cf2a8b,0x380d70e5),LL(0x2c281733,0x4631440b),LL(0x7c3b711d,0x3747bc66),LL(0xd6b99662,0x423c70b2),L_(0x00000183), + LL(0x89dc8694,0xa8b11d17),LL(0x2eed227f,0x7fd9a16c),LL(0x916842cb,0xe12a5d02),LL(0x6758564c,0xe59ed474),LL(0x4b48f9be,0x8e675f35),LL(0xece126be,0xf7c75d69),LL(0x8ce3aca2,0x00f88d21),LL(0x9a768d60,0x26ea6ff2),L_(0x0000010f), LL(0xe69e2709,0x5d96ef4c),LL(0xa0efb2f0,0xac3a2f2d),LL(0xa99dc276,0x757c443d),LL(0x23ce0342,0x390d2a5e),LL(0x9b674e3b,0x7e7ea78e),LL(0x32e72b98,0xdca485e1),LL(0xb6c21856,0xda17d0d6),LL(0xee5bed8c,0x220788bd),L_(0x00000037), + LL(0x221d84fe,0x2333e9ce),LL(0x1a6d9a86,0xaadfe3f4),LL(0x56277e4f,0x389e2b48),LL(0xc71ed641,0x5717d1e3),LL(0x5f0642b9,0x56053a56),LL(0x042b6345,0xafc491b1),LL(0x5ee182db,0x9a47e510),LL(0x6ccadf49,0xf6da1632),L_(0x0000010e), LL(0x3d3d8156,0x05064c02),LL(0x708a5ae7,0xd681e121),LL(0x3b41c0e8,0xcbc4f74a),LL(0xe5968e75,0x47ee9e3e),LL(0x6b8f739b,0xe232789d),LL(0xcd2e53fe,0xd5500a30),LL(0xde9e78f3,0x2cad3174),LL(0x4522aa08,0x295cf494),L_(0x000001f4), + LL(0x6ca25a67,0x0e236499),LL(0x8d70cd44,0x61afc7ec),LL(0xd8467c51,0x4c9aa882),LL(0xbc140872,0x62a215cb),LL(0xad6d3cb9,0x6c4986c1),LL(0x912aaf7a,0x83691332),LL(0x6db2702d,0x7d4a1ecd),LL(0x3fa17e01,0xbf2405e9),L_(0x0000012d), LL(0x8090c5e3,0x8a064116),LL(0xe42ae3e6,0x395b06fc),LL(0x39938713,0x5bb1098f),LL(0xca3394a1,0x97734c1b),LL(0xe8c0bedb,0x1edfc62a),LL(0x9b0452cb,0xc661bc2f),LL(0x04c79c90,0xe6253323),LL(0x2a0dada4,0x2e4ae434),L_(0x000000fc), + LL(0x5105e93f,0xd1cc1e71),LL(0x144c819c,0x6629502b),LL(0xbd39770d,0x8c2f7831),LL(0xe5075a30,0x4ec45cef),LL(0x641b65dd,0xa56294ef),LL(0xe6aa4eb3,0x75854d3b),LL(0xbdb743c7,0x3fdd169d),LL(0x5176a409,0x181ac2e1),L_(0x0000003a), LL(0x9af21ff0,0x0ec035b9),LL(0x3f057fe2,0xa4b87bdc),LL(0x3feadb94,0xf7f87024),LL(0x5db56992,0x6a6cdc10),LL(0x57cd02da,0xdd69ecb6),LL(0xc54a8f0e,0xe1da3c2e),LL(0x10cf592f,0x8fe5cc2e),LL(0x3b4989f2,0x37d88e4b),L_(0x0000000f), + LL(0x156252b6,0xb01bd7fc),LL(0xb25b337f,0xe1e660dd),LL(0xac5d025a,0x6a73c379),LL(0xb69c2605,0xdede6af2),LL(0xb7e81b95,0xa1ae9121),LL(0x6cd030d2,0x5f7f754f),LL(0xb47d1e9a,0xc9b7c0f8),LL(0xe925d238,0xd6fa902c),L_(0x00000132), LL(0x3d941846,0xf95ace42),LL(0x6ab6a655,0x709a356f),LL(0x8fa78d47,0x9d32f258),LL(0xc583f5d3,0xab0a90b8),LL(0x5d68b6be,0x8bc51799),LL(0xcae9d65d,0x4a80ea4a),LL(0xd569fb10,0x2cb3b12f),LL(0x67b09db0,0x4b3e1e5f),L_(0x00000062), + LL(0x453b0260,0xf5df84a6),LL(0xcdd9e65f,0xf9cc5322),LL(0x1ae57f11,0x997b8e3d),LL(0x4ec38b17,0x52bddbee),LL(0x99728cf4,0x3daa3db0),LL(0xc592cfa0,0xe99b31ff),LL(0x0d3ad893,0x71adf8f6),LL(0x21a14a01,0x748065a6),L_(0x0000000e), LL(0x85220921,0x313c6b68),LL(0x45073a51,0x4026740f),LL(0xe3c1a79a,0x03ea00aa),LL(0x7f5aaba8,0xae0bdab5),LL(0x1d349675,0x81ecbbfd),LL(0x611cda3e,0x62377d6e),LL(0xc226bbb5,0x4eee5f15),LL(0xf8b5d257,0x8afee162),L_(0x00000002), + LL(0x6dca0b60,0x1e93cbca),LL(0x14655922,0x200324a2),LL(0x6c357ebc,0xfbe29569),LL(0x537e73da,0xf1c77b70),LL(0x59d41573,0x12b0a8e9),LL(0xc50a71dc,0xd18455d9),LL(0x7c9b3656,0x7fcbc173),LL(0xd9283b61,0x6acf8093),L_(0x00000097), LL(0x15b4734e,0xfa8b5737),LL(0xf0027024,0x3b38a173),LL(0x386bfccf,0xbbc99c54),LL(0xbdb95480,0x8bfbf241),LL(0xcc88d566,0x8353dffb),LL(0x7968e885,0xaa2a216b),LL(0xe22f661f,0xf0cc373d),LL(0x0c189437,0x5601679c),L_(0x0000001f), + LL(0x95c17af7,0xb1e74cee),LL(0x969661a7,0x9719c192),LL(0x390ae167,0x76ffd55a),LL(0xe6fc4921,0x17827dab),LL(0xb57cd8f2,0x4435c383),LL(0x16123417,0x1dcf73d3),LL(0x8d4cffd7,0x0c91ecfd),LL(0xe6e70928,0x8412502a),L_(0x00000171), LL(0x066d3430,0x019a0e1b),LL(0xecf807dc,0xfb488f96),LL(0x6c466766,0xedde1e48),LL(0x756a682c,0x37d6152f),LL(0x2661ee25,0x5b6f467e),LL(0x96e2b2d4,0xb97a8d49),LL(0x1c1589d1,0xbcf05602),LL(0xdf83ce24,0x04b7cee0),L_(0x00000197), + LL(0x182c5012,0xfca86967),LL(0x34e40148,0xfa3981f6),LL(0x1c864ffb,0x42879632),LL(0x5079d6d1,0x3e4b6047),LL(0x9eef5744,0x6e1e5a87),LL(0xfd7f7f13,0x38d5d2b8),LL(0x19b63788,0x1c2726dd),LL(0xc17815ed,0xf17abcb4),L_(0x0000002e), LL(0x9895b25d,0x7bb9a599),LL(0x40e55822,0x3d146be1),LL(0x7f28ae92,0x8852f582),LL(0x8cdd00a5,0x60ada16c),LL(0x7def110e,0x8158a85a),LL(0x1d1152d2,0xa55ae5c6),LL(0x4be61bf1,0x0a31606d),LL(0x8fcf413c,0xd625cdfd),L_(0x0000013c), + LL(0x16a0ecdc,0x6c3d008c),LL(0xd11d6fc4,0x0786f8b9),LL(0x26066afc,0x6f28cd76),LL(0x6a57afc2,0x9d41e208),LL(0x2ed8fbfb,0x32ce6027),LL(0xda94edc4,0xf08d764b),LL(0x94110774,0xd4093a46),LL(0x8526b334,0x084fdb6f),L_(0x000000a0), LL(0xe6b29d32,0x652dbbe1),LL(0x75164543,0x605ecb71),LL(0x6687cd0a,0x3962a1b6),LL(0xfe7869a3,0x347a147b),LL(0x1cab34bd,0x634a95b8),LL(0xbb85dfab,0xbfeffee8),LL(0x4995b282,0xf245a753),LL(0x69b18723,0x5d6e7794),L_(0x000001a4), + LL(0xcce2027d,0x643f2f8c),LL(0xcd45c4a5,0x24a0afd5),LL(0xa6b24112,0x2258c4c8),LL(0xfa87a5ca,0x8f855fed),LL(0x3975cb67,0xc55199f4),LL(0x9edc6298,0x5a48e9a3),LL(0x7312684e,0xf55daba0),LL(0xbfadaeb9,0xc9f5f377),L_(0x00000139), LL(0x382a7ced,0x5a3e0968),LL(0xc70ffd11,0xfcccb869),LL(0xba001f2a,0xfe8068fd),LL(0x124107bd,0x06868f7c),LL(0x28b9fe02,0x3821a909),LL(0x33728dac,0xac94afc5),LL(0x3e9edff0,0x7f67565b),LL(0x0bd10c69,0x250773ba),L_(0x000001ea), + LL(0xb6d26941,0x70ff7fb7),LL(0xad0ad081,0x7b1f1709),LL(0x88afd9fd,0x2c52599e),LL(0xa4b49d9d,0x58896d4a),LL(0x6df73899,0x001961cd),LL(0x1c7f535c,0xd4c3ed4d),LL(0x75c903a7,0x4c699591),LL(0xab8339d4,0x939fc682),L_(0x00000026), LL(0xf58af501,0x921bc00d),LL(0xe64a70d3,0x0644b2c5),LL(0xb7245016,0x4fad690f),LL(0x8e863833,0x52268bfe),LL(0xcab84fe2,0xc76f784d),LL(0x75b08768,0xbd5df903),LL(0x97114157,0x49a7a2a6),LL(0x5dbab306,0xb4ae419a),L_(0x00000120), + LL(0x8f91400d,0x24275a2c),LL(0xc241f782,0xba60fbb4),LL(0xc4fd93b4,0x616268c1),LL(0x0872941b,0x107f7964),LL(0x25e04f20,0x831b4388),LL(0x7786625f,0xc5f61924),LL(0x8de20083,0x791c6d52),LL(0xb0abde39,0x75c25ecf),L_(0x0000013b), LL(0x7b9d8c31,0xb58e09f4),LL(0x4c6bc5ce,0xb3112937),LL(0x38e27941,0xcee2666e),LL(0xca0e3235,0x8dbee896),LL(0x9b498dea,0x53066000),LL(0x0f289764,0x58ff5f8f),LL(0xfb5ee444,0xa7b5e140),LL(0xf9fb559a,0xac85e138),L_(0x00000072), + LL(0xc0135106,0x0d7ebeca),LL(0x20feaf54,0x19cc13ae),LL(0x3c5c75ec,0x38ba79ba),LL(0x78f3f1f0,0xf2dc8803),LL(0x911501c4,0x8fb64807),LL(0x6448cf01,0xa8bff66a),LL(0x206b2cb4,0xce9b312c),LL(0x195342ba,0x219fa1d7),L_(0x000000cd), LL(0xa18154da,0xe91053ed),LL(0x41af0398,0xadc91c1f),LL(0x6166dc26,0x5d9c3eef),LL(0x055887cd,0x90ea6dfd),LL(0x3d270166,0xa4280b95),LL(0x206854af,0x7b358dc6),LL(0xa6ae166a,0x03623eb4),LL(0x34af3892,0xe4258201),L_(0x0000013b), + LL(0x93905a5e,0xbb9f0c61),LL(0x99256667,0x3489213e),LL(0x27fdbbfc,0x1218ca33),LL(0x5630d2c7,0x5a83f00e),LL(0xbdc8df91,0x0d628331),LL(0x28ee96b8,0xbfe73e81),LL(0x6a5f7e06,0x2a7cd331),LL(0xdd16364a,0x8cd2a08b),L_(0x00000074), LL(0x51d38008,0x221d90fa),LL(0x814ecb88,0x8b5df20c),LL(0x00fc7920,0x76343a10),LL(0xc99f2520,0x14b68032),LL(0x71413b8a,0x654fe0dc),LL(0x9a173cb4,0xa9acd97c),LL(0x85a386e9,0x14a40bfc),LL(0x87bf160a,0x849e9970),L_(0x00000032), +}, +/* digit=36 base_pwr=2^180 */ +{ + LL(0xc2ffbb23,0x77384b0d),LL(0x16c289b4,0xf9601e0c),LL(0x9eabe48c,0x71ddca51),LL(0xb3f199d6,0x3fce7863),LL(0xa3ecba6f,0x2e01be3e),LL(0x67c58c7d,0xfbf4b701),LL(0x4893679a,0x2cb78d1f),LL(0xe019a436,0x15a3d7fe),L_(0x00000015), LL(0x746e7221,0x25f2840f),LL(0x160c51fc,0x516e72ef),LL(0x97156a16,0xd9625db3),LL(0xbf6e8398,0x3f5b2c0e),LL(0x651404d6,0xfc5b6523),LL(0xd10c4d87,0x8eef476d),LL(0xf40ffa31,0xe5d39771),LL(0xe3788025,0x98fa2547),L_(0x000001c2), + LL(0x658a6253,0xb2523e81),LL(0x8e050759,0x42659aef),LL(0xb0377d50,0x2b36823c),LL(0x419b9ae7,0xff957169),LL(0xf46fc17e,0x59705ceb),LL(0xb61ce7ad,0x2fffbd18),LL(0xa7135b60,0xfe9192a7),LL(0x96f2e092,0x30a3a8e5),L_(0x0000014d), LL(0xad967512,0x667c895e),LL(0x3da48897,0xeb732652),LL(0x467afe86,0x5b7a7cf8),LL(0x393a5ee2,0xf2568e46),LL(0xb15dd000,0xb79a3304),LL(0x203f1569,0xd91a36bd),LL(0xa5e938c0,0x1a346459),LL(0x521da127,0x88c575bf),L_(0x000000c6), + LL(0x633bf04b,0xc8c62a6b),LL(0xeaef0121,0x98cc53c0),LL(0x58d73540,0x925273a9),LL(0xc04448cc,0x73c56bf4),LL(0xc52be46f,0x542b800b),LL(0x39147d47,0x30298d6b),LL(0x44cb5cfe,0xb2312e04),LL(0x9ed4247f,0x4c4d89dd),L_(0x00000105), LL(0x97a9163d,0x460edd6a),LL(0xed4f4d5b,0x9206a582),LL(0xb9ca6130,0x3e18c6dd),LL(0xa3efafa9,0xa68f9bb8),LL(0xb2d783bd,0xb70a52c8),LL(0xc0dda564,0xdbe47728),LL(0x0dc789e7,0xe8a6481c),LL(0xe4119aa3,0x27f421a4),L_(0x000001ed), + LL(0x34050818,0xf301ee13),LL(0x055dadf3,0xd3d6ab94),LL(0x8803374b,0xa078817f),LL(0xc730e431,0xe1298465),LL(0xaae8170a,0xba08da98),LL(0x8b779119,0xf12876bc),LL(0x1b8f7410,0xbe46247c),LL(0x67bc98dc,0x18059808),L_(0x00000176), LL(0xab5cae23,0xf59de67d),LL(0xd0125b70,0x1682d3d2),LL(0x8c5ad3a0,0x9c7c1b26),LL(0x62fcf59a,0xa095cf63),LL(0x6482c8ad,0x5b79b1ed),LL(0xc253c84e,0xd6952b3b),LL(0x56917d1d,0xdfad9c37),LL(0x5c8f439f,0xa63232aa),L_(0x00000051), + LL(0x75e1f132,0x1f3a0552),LL(0x620ca4b5,0x48133bce),LL(0x765c9fcc,0x710e23a7),LL(0x7a6387e5,0xd9c29479),LL(0xa6621b41,0x9fe4eedd),LL(0x3bf9d9ac,0x5df19f73),LL(0x4cb8a304,0x4f51d70e),LL(0x45d5c96c,0x25c50ad2),L_(0x0000001a), LL(0x87a04f68,0xb3acd866),LL(0x4b6a5c45,0x9f7d19bc),LL(0x3f85a2b0,0x0758494b),LL(0xd50f6942,0x554c9337),LL(0x40c2407e,0x0ccb9c2f),LL(0xc5dfc1a6,0x11e7e482),LL(0x6ad44e8b,0x0fea5311),LL(0xedd080e6,0x9fd549f4),L_(0x00000188), + LL(0xef5cdd9d,0x7e7e29c5),LL(0x2b2e558b,0x3e6bc46b),LL(0x4702314f,0x56eeaa30),LL(0x06726fae,0x5ca44a1b),LL(0x2ee6f214,0x0ea8da79),LL(0x829cf968,0x141e7e4c),LL(0x723cb279,0x45b326cd),LL(0xdac514c6,0x5e8e8931),L_(0x0000013d), LL(0x62bd48fc,0x61e5ed08),LL(0x4a34e74e,0x83644940),LL(0xe1d4a984,0x1f65c56b),LL(0x3e5f4500,0x062ee718),LL(0xaa764b8e,0x6a39ef75),LL(0x9012ed64,0xbddef450),LL(0x42837f0e,0xfaa786a7),LL(0xf89ab588,0x474accf0),L_(0x000000d7), + LL(0xadd26e0e,0x1ef31aa7),LL(0x70683b34,0x5ed33b5f),LL(0x2190eb5f,0xf3278604),LL(0x8f6e3b2b,0xdb29e400),LL(0xc911a62c,0xd42f0700),LL(0x688f5189,0x7efff5e3),LL(0xc2de5c25,0xe2d46677),LL(0x6189c193,0x5de47c98),L_(0x000000ca), LL(0x0383ccaa,0xa7dddac1),LL(0xcaddccac,0xcf555803),LL(0x0778df17,0x5faf93e7),LL(0xb029278c,0x7cfbb523),LL(0xa7546c0e,0x33ef004b),LL(0xd52d052d,0xc8957290),LL(0x54a34c36,0xcc555faf),LL(0xa3e1b89d,0x77136cbc),L_(0x00000187), + LL(0x6c20e825,0x4760b5ef),LL(0x30fe1ead,0xd1479bf4),LL(0x3a480e70,0xba684ec7),LL(0x54c97c0a,0x99909719),LL(0xd306cb54,0xea1c5645),LL(0xcc5c264c,0x3d9ecc85),LL(0x39efac32,0x465cbfa7),LL(0xe63b20c4,0xe9cad749),L_(0x0000014e), LL(0x808827e7,0x13242934),LL(0x860bc182,0x41b298a9),LL(0x7a452bdb,0x51ceda44),LL(0x0786c3f6,0x53ca2965),LL(0x7845a5f1,0x7e0cd8cc),LL(0x5913baf8,0x060bc9cd),LL(0x0312de2e,0x79bfb315),LL(0xf7a14442,0xa16f8265),L_(0x000001f8), + LL(0xa3daf9d3,0x350ffc55),LL(0xd2d93315,0xecd8b90d),LL(0x88f5e22f,0x9eccd42a),LL(0xdc1f662e,0xc4f29c7e),LL(0x42b4d8d9,0x4f6798fc),LL(0xe485f1a4,0x46c699bd),LL(0x6c52567f,0xf81e6fde),LL(0xccefcbe0,0xcd5234c4),L_(0x0000015d), LL(0x821ab350,0xc73f9043),LL(0x8ce6bb52,0xbdeaccaa),LL(0x746080b0,0x3424a5b8),LL(0x0eee571d,0x785554a3),LL(0x4bc343de,0x6aadb674),LL(0x44652a58,0x2ff3c998),LL(0x5fd0a875,0x84f6f7fc),LL(0xba89cfbf,0xd08e7a6f),L_(0x00000002), + LL(0xacd3ee5f,0x533b8a60),LL(0x3e2d62d2,0xda0545e5),LL(0x476d9e76,0xb53693d1),LL(0x8749ddb7,0x78864741),LL(0x6623b715,0x737a1960),LL(0xb1899ac7,0xf216ba69),LL(0x057f8862,0xb25babc9),LL(0xcb288274,0x927aa4d0),L_(0x00000028), LL(0x000f8d74,0xd9a6f518),LL(0xfa70c9cf,0xfefa0627),LL(0xcdcaaa25,0xceb9750a),LL(0x15a2f18a,0x9cc57e80),LL(0xba45323a,0x0cadb63b),LL(0xe3f19ccd,0xa55c80d9),LL(0x1e511bb0,0x3bb4df11),LL(0x164359dc,0x3e271d06),L_(0x00000100), + LL(0xd00dafc7,0xaa6cb262),LL(0xb186d04e,0xe56a357b),LL(0x750898af,0xb3fa3a15),LL(0x4d60c192,0xc07d177e),LL(0x9679fa78,0xf75650e4),LL(0x3ad024b4,0xfc2fc8b2),LL(0x0bddcaf6,0x559b0ced),LL(0x604f3f34,0x995261e0),L_(0x0000009d), LL(0x5bdeacb3,0xfb2e6335),LL(0xc5822803,0x102a3be3),LL(0xc4f23418,0x2683d799),LL(0x446dc4f9,0x87d5a82b),LL(0x82fb7bba,0xba06b349),LL(0x859d405f,0xdacb2e84),LL(0xf7fdeed1,0xa51f1588),LL(0x8b67135b,0xc2217c58),L_(0x000000c7), + LL(0xbf16a7c1,0x8ed9d71b),LL(0xf4c69057,0x0ee9b6ca),LL(0xb90a3ad0,0x690215b5),LL(0xe1a72991,0x9dc86f3e),LL(0x4e4042ba,0x076b900e),LL(0x7d9520d4,0xf559233d),LL(0xa6fe5f79,0xd16f05cc),LL(0x6290cb9a,0x2c55a35f),L_(0x00000145), LL(0xa1a2502b,0x02fbcf5d),LL(0xc7fdf1f3,0x78d6c024),LL(0x3c5ac58d,0x180724dc),LL(0xeafba33f,0x0f2d4859),LL(0xa9ec392b,0x9adb7f75),LL(0x10b122b3,0xa1699e54),LL(0x8be6fae5,0xcfb1317b),LL(0x3a96cd81,0x9a5bff09),L_(0x000001c8), + LL(0x4c7da590,0xf6d8c638),LL(0xd287d869,0x6aabc1f2),LL(0x26b0f715,0xbdea2e8a),LL(0x8e33c1dc,0x689a9c3d),LL(0x8c56f036,0xa841ff6c),LL(0x527eaefd,0x10032f78),LL(0x0c199e97,0x6215f00a),LL(0xd8293042,0x0262f60a),L_(0x00000139), LL(0x116acfb2,0x0105c4ab),LL(0xa09207d0,0x2a3ccda5),LL(0x7549d228,0x67ad8625),LL(0x0483ecfd,0x12b83a0c),LL(0x0eee9667,0xe653fd39),LL(0x14bf0bf5,0xdd617912),LL(0x5b9e1025,0x58e59489),LL(0xb42fb14a,0x6b6fe3f4),L_(0x000000f0), + LL(0x332de310,0x82f2b927),LL(0xfe39c03a,0x595b30e1),LL(0xa1ad263a,0xcc294836),LL(0xe59896e0,0x55678ebc),LL(0xdab6cc2a,0x3b48be12),LL(0xf27aff9b,0x1525c60c),LL(0x72f22657,0xbfa65ac2),LL(0xe179fdb5,0x957d9762),L_(0x000001c4), LL(0xbe080757,0xaabb8ddd),LL(0x2e567a04,0xfc24eb81),LL(0xcd0abafe,0x3ab9ba57),LL(0x7ece5b5f,0x94233802),LL(0xfa49f2c9,0x192ad8a9),LL(0x7c6c9e7b,0xd9733712),LL(0x97c62d5b,0x608ec02f),LL(0x3b573c6e,0x90c6dba5),L_(0x000001a7), + LL(0xf559a7d9,0x41a926ac),LL(0xea7b4b5d,0x6a5e3301),LL(0x595fce21,0xee8aa9a4),LL(0x4300c92a,0xeb1b3325),LL(0xfb3d0ddc,0xe7231d36),LL(0xdd2028ea,0x0407b0dc),LL(0xb99d20da,0xf0f51dbb),LL(0xe418d5a9,0x31d74a02),L_(0x00000025), LL(0x6b8d0453,0xf7fd4389),LL(0xbfaf4600,0xe8d861a3),LL(0xf167dda3,0x92cf759f),LL(0xc46df950,0x32d3e4f2),LL(0xb7815d1f,0x91ed2fcf),LL(0x6d421190,0x12864b88),LL(0xf9dfcf39,0x04988ed3),LL(0x74a6a7f4,0x44aba25d),L_(0x00000120), + LL(0x924eb552,0x4033bcc8),LL(0xd518ffb7,0x2da2c2e9),LL(0x1ae0cd73,0x9a4290d6),LL(0x357cfbc1,0x784c1f06),LL(0x3ca1aed8,0x3fe20989),LL(0x85a8dedb,0x2f87969a),LL(0xc8eb2e93,0x550ff529),LL(0xfbbcc740,0x54bf85aa),L_(0x000000f5), LL(0xd7f84381,0xd5bc6372),LL(0x557f4f2e,0xc2efbdc0),LL(0x9d0c30f3,0x262ac2fc),LL(0xd5dddabb,0xa05b87d4),LL(0x769d1cf7,0xfc91e745),LL(0x4d0a4907,0xdcd38c99),LL(0x89250072,0xc453a288),LL(0x0dffae1a,0xe7245800),L_(0x000000a8), +}, +/* digit=37 base_pwr=2^185 */ +{ + LL(0xbc76a5c9,0x06d9177f),LL(0x5e3cbf74,0x39ed5397),LL(0x2f09def0,0x0caf736f),LL(0xbc534da8,0xfcc790fe),LL(0x46448c8e,0xb0ad47e7),LL(0x36b92fa7,0xc7671ca3),LL(0x90e92c64,0x637080ab),LL(0x52dfd8d6,0x5711517b),L_(0x0000012f), LL(0xf4a15818,0x56a0257f),LL(0x24412386,0x77234bc0),LL(0x364971ed,0x9b2d316f),LL(0x0cc8b1d0,0xdf4ae5e0),LL(0x307856bf,0x1468fa8d),LL(0xe3791c04,0xfc69805b),LL(0xfa589236,0x0c1fe733),LL(0x89a33762,0x37b57609),L_(0x000001e7), + LL(0x1a7b432a,0x8a01e6a2),LL(0x71b4886b,0x83120c39),LL(0xdae7cb78,0xf3efe6ce),LL(0xfd659d28,0xe1699713),LL(0x75625028,0x0252af65),LL(0xcd7c4a21,0x81fe2a3a),LL(0x7efc9c5a,0x2fab4ecf),LL(0x8ae2a5f8,0x92444155),L_(0x000001c0), LL(0x93563c95,0x95495dd4),LL(0xdf9b0e72,0xbb0facca),LL(0xab5a8f70,0x01ed29d2),LL(0x6b65b325,0xf439adfe),LL(0x2e9c2436,0x40a6c720),LL(0xbcd403e2,0x97776531),LL(0x4526a2b6,0x90cd1256),LL(0xa61dc2d5,0x170acdcf),L_(0x00000164), + LL(0xbcef9f83,0x8be8d883),LL(0x770de7cc,0x95107be3),LL(0x07c65e3e,0x780e3eca),LL(0xcf6ac96f,0x3d615089),LL(0x2549b641,0xf585b5b2),LL(0xacd5da79,0x4c0d8b5f),LL(0x3c8b5c5a,0x970b49ff),LL(0xadd6dfaa,0xc025c0e7),L_(0x0000005c), LL(0x34154da5,0x0b3c64dd),LL(0xc797b7cd,0xbc308343),LL(0x1f367813,0x138ae118),LL(0xe7bfbf3f,0x1f8c6302),LL(0xe3cc546f,0xef35ea2e),LL(0x904ac34e,0x852c3a0b),LL(0x2596f106,0xb1310ec5),LL(0x1e6e533a,0x763b1938),L_(0x000000bf), + LL(0xdd556b63,0x29b5e462),LL(0xab5e9c2f,0xc87a1f3a),LL(0x40c3ae00,0x8fdfc7cb),LL(0xf72aade9,0xf671ec86),LL(0xaa376ff2,0x369dd7b2),LL(0x0c4b0748,0x1a9eb6f9),LL(0xe5c39e83,0xb8bdb31a),LL(0xc9ef6929,0x5a4c5224),L_(0x00000112), LL(0x8d10b8d5,0x3d80ab90),LL(0x8a32a994,0x0edbbb7c),LL(0x51b7d4fd,0x9eb83ad0),LL(0xbe08eaa8,0x343de0eb),LL(0xc33cc9fb,0xad3c4d0c),LL(0x24b0953f,0x9c30b151),LL(0x582773fc,0x3a021a47),LL(0x75ab2c19,0xddfb8816),L_(0x00000173), + LL(0x334a8fdc,0x8da3d9ef),LL(0x644b8138,0x80531565),LL(0xf0f2d302,0x3bfd457e),LL(0x64c28e98,0x93b685b7),LL(0x24eeda6c,0x2b149454),LL(0x97f74e2b,0x1420398e),LL(0xce3c2017,0x93fa9e0c),LL(0xa9df8bc6,0x0fc6b820),L_(0x000001ae), LL(0x3c5470c1,0x52bf8c38),LL(0x06c9ae45,0x500fd912),LL(0x2fbca6d5,0x5e0fd35b),LL(0xe9e18d3d,0x39985525),LL(0x3bfa858d,0x3a3dab8f),LL(0x0bc682dc,0x51f2882f),LL(0x5632ba53,0xd2912672),LL(0xa5d16cfd,0xefb27960),L_(0x00000061), + LL(0xb762a667,0xf235a5ba),LL(0x39db43f4,0xdd5bbd91),LL(0x4d8ac038,0xc1e864b1),LL(0xbb5ec32b,0x9c3d8682),LL(0x0da419a0,0x7fa3e54d),LL(0xbbcc85f7,0x4911605b),LL(0x16bf46df,0x459ed701),LL(0x42b3919e,0x4a6f67ab),L_(0x0000014e), LL(0x89475032,0x761f44af),LL(0x78dc3aaf,0x7ec577f5),LL(0x443c49b1,0xcedfe95e),LL(0x4ca71a23,0x80d161de),LL(0x88a46fa9,0xa3a812c1),LL(0x8060703c,0x5d69c965),LL(0x52f25061,0xebe46263),LL(0xf14ae427,0x2518ad4d),L_(0x0000018a), + LL(0xce592fac,0xb29db4b3),LL(0x3eb4e951,0x73934c0c),LL(0xd205a31b,0xcde75602),LL(0xf7d9ceca,0x652846c3),LL(0xa5604560,0xf53ed6dc),LL(0xfcef8ee2,0xa3dda8b0),LL(0x73763d47,0x5dcfc88c),LL(0x3f72bc6b,0x61afbead),L_(0x00000062), LL(0x8b3b90f0,0x706c2fef),LL(0xb896e8ba,0x91189666),LL(0xa5f8d9e7,0x6dc25f9d),LL(0x98f8493f,0x29210ade),LL(0x77e1557d,0xc803167f),LL(0x80aaf764,0x746e916c),LL(0x9a02bf22,0x6f8c70cb),LL(0x692f9669,0xe6efe144),L_(0x000001e7), + LL(0x29ca6496,0xd6f014ad),LL(0x10e7e9c3,0x91edf75a),LL(0x042dda6a,0xfbe9047f),LL(0x6df69276,0x497f9141),LL(0xfce4035a,0xab982ab7),LL(0x1e6adadf,0xd973b8b6),LL(0x218a9fd9,0xe2c23f1d),LL(0x9e1c8c04,0x2274d47d),L_(0x000000b1), LL(0x397b98af,0x090ec3de),LL(0x272cecd7,0x6d724d9a),LL(0x50e492db,0xe32d2f19),LL(0x68f82a50,0x6bf40e9c),LL(0x0678afdb,0x4b25727f),LL(0xe6ae7819,0x06b77a36),LL(0xbb096d18,0xeedfa35c),LL(0xf41afd3f,0xc17d9b9f),L_(0x00000057), + LL(0x3437e67a,0xaac42698),LL(0x9e517311,0x90d691a0),LL(0xbbe7b23f,0x3efcc598),LL(0xc5b3ba4a,0xf044505e),LL(0x8a70a012,0x818530c1),LL(0xb73eeaec,0xd4496b66),LL(0x25f453bb,0xabda0862),LL(0x6dcb9832,0x76d60bb1),L_(0x00000060), LL(0x3a78a95d,0xa348f0b1),LL(0xdb8646ed,0x1af6f002),LL(0x451f5839,0x3087f4a7),LL(0xa66aaeeb,0x47adc893),LL(0xbec9934e,0x6e6950e9),LL(0xb35294f1,0x31d5e186),LL(0x7bf79296,0x590c3c8b),LL(0xc1942a2c,0x1b804ef3),L_(0x00000051), + LL(0x6a948f72,0xd23e1c8a),LL(0x01715db9,0x41d8f90c),LL(0xbbc2c6b8,0x4e56a842),LL(0x83c0fc75,0x986646d8),LL(0x4fe3bcf5,0x5fdbc4e4),LL(0x2ada0ebd,0x0e534106),LL(0x8c28b66c,0xb1f981f9),LL(0x56f04488,0x79a1b1cb),L_(0x00000058), LL(0x544a66a2,0x98d4263b),LL(0xb7ee4dad,0xb8425937),LL(0x9ce9983c,0xaff51ac5),LL(0x798d12f4,0x772fa5da),LL(0xb536f2fe,0x9c00b0c8),LL(0xb35431b3,0xf4789358),LL(0x2ee8e687,0x664cbdfe),LL(0x29120ae4,0xfa9435a4),L_(0x000000dc), + LL(0x1cd7048c,0xde30af0b),LL(0x64ca10f1,0x7af2cf68),LL(0xf23265e4,0xcdd4b45d),LL(0x4fc85e40,0x3c687440),LL(0x3ef2a535,0x6a698fc3),LL(0x9efabe8e,0x63e4d298),LL(0x18de82a4,0xb24c2816),LL(0x775b0ff1,0xf09e7eee),L_(0x00000129), LL(0x7c587948,0x1cbca914),LL(0x7dbc5bb9,0xcc9dfa8d),LL(0x432844d1,0xb35c10a9),LL(0xdb4db17a,0xf5e1db87),LL(0xf9910dba,0x86ff1ebc),LL(0xb2c9c01b,0x189bbc27),LL(0xa7d616b4,0x5df3f754),LL(0xe6cc2fbf,0x274e1d3a),L_(0x0000016e), + LL(0xac9d5c5c,0xcbf1d173),LL(0xd9410d43,0xb76d4376),LL(0x656599eb,0x900d071f),LL(0x2fb9b595,0x5fbadcc3),LL(0xe781b5f4,0xc0a2440b),LL(0x50f63654,0xbfcd2d0c),LL(0x1e522100,0x2f21286a),LL(0x4f742889,0x482b198e),L_(0x0000008a), LL(0x54f76137,0xd5622874),LL(0x47efa194,0x1f58794f),LL(0xbd93f7de,0xc2129e69),LL(0x5496a993,0x1b271db6),LL(0x8f7ac06b,0x5b18ae06),LL(0x78e56286,0x6111cab0),LL(0xbf1dc2cb,0x641b9597),LL(0x9c602e3b,0x6826b02e),L_(0x000000e4), + LL(0x391d1890,0xcc35919a),LL(0x2497cb5e,0x94f52d11),LL(0x4ef3c830,0xdf80522a),LL(0x94fd85cb,0xf72be2d7),LL(0x29671043,0x75499b11),LL(0x540e521b,0x0bd6a835),LL(0x5b01c741,0xc2f40e1e),LL(0x4828498e,0xb6d6e72b),L_(0x000000eb), LL(0x68b34fe4,0x87e9147b),LL(0x4ff41c4c,0x7947091e),LL(0x283731b8,0x31294652),LL(0xf259b874,0x0f36636f),LL(0x8ee00f38,0xfc2118ab),LL(0x8017118b,0x5f13103b),LL(0xc3d2d9af,0xc3feb59d),LL(0xca5c4199,0x39888318),L_(0x00000068), + LL(0x6a30f59e,0x5d7424e4),LL(0x59ff2a43,0x6be7810b),LL(0x30694fb4,0x78ec13fc),LL(0x92716d06,0x1d9c5aae),LL(0x8fbb9bb8,0x416a4a81),LL(0x1881c6a4,0x15a0a324),LL(0x489236d7,0x23235b6b),LL(0x685caeab,0xa5c2734b),L_(0x00000066), LL(0xa48020b3,0x383cc04c),LL(0x2d3601e9,0xd66a0119),LL(0xa6e151c6,0x9e61fd22),LL(0x8339ddd5,0x91be32a7),LL(0x235b6f9c,0x7155449c),LL(0x322c55d0,0xa7e5e410),LL(0xbe0a861c,0xce4ac258),LL(0x0323587c,0xd78b88ca),L_(0x00000146), + LL(0xadf63e55,0xb1cdbf59),LL(0x59991ed2,0x646f1d97),LL(0xae034cb5,0xcf9f8f62),LL(0xa6cfbf1e,0x9a35acdb),LL(0xb02eab43,0x0993f86d),LL(0x172ffcec,0xc65c756e),LL(0x1b44bc51,0x5ec6620c),LL(0x19230f70,0xc9e7a1a7),L_(0x00000196), LL(0x3da4f4f7,0xf7bed1a1),LL(0x28d9a36c,0x391142c7),LL(0x3d4288e4,0x0485a093),LL(0xf59f8fae,0x0209a097),LL(0x94df4e25,0x5fdf8f3e),LL(0xbc0be074,0xb3140419),LL(0xba7e0344,0x5cbb3260),LL(0x95c0673b,0x536a91bd),L_(0x000001a6), + LL(0xb05c5499,0x9f78e57a),LL(0x123d2b21,0xa3cf981a),LL(0xecb0183c,0x1eddfd07),LL(0x6998ed9f,0x8f90e3c6),LL(0x0e05152f,0xfad41bb2),LL(0x7dab5c5e,0x939419c7),LL(0xca783006,0xde605b32),LL(0x98ae5cd1,0x3d6039cc),L_(0x00000193), LL(0x5bb2b74c,0x590ae5b0),LL(0x68c4bf82,0xfdf4f711),LL(0x01a66f3e,0xa65b0015),LL(0x241e1da4,0x0665dbdf),LL(0x4c3387ba,0xf15f360d),LL(0xc88fe301,0x8acf4e85),LL(0x061a8e04,0x9ca9957c),LL(0x51bcc011,0x8585dfcf),L_(0x000001ea), +}, +/* digit=38 base_pwr=2^190 */ +{ + LL(0xa59f069c,0x37a0e4fa),LL(0xd96e52e4,0x3aebdd33),LL(0x29f2632f,0x4fd15682),LL(0x2c70c85e,0x4f3be789),LL(0x0a1634de,0xc7d9fb18),LL(0x638b44c2,0x3e6cb175),LL(0xe33499b5,0x0b60dc32),LL(0x2ecdad29,0xcf1fcbab),L_(0x000001d1), LL(0xfb1da9fe,0x5d1854ce),LL(0xb5b7539f,0xb9d47257),LL(0x96df1240,0x561ffc72),LL(0x5e9f9e9a,0x6d945271),LL(0x9f0df30d,0x25aea910),LL(0x1e814b45,0x4c475d52),LL(0x7037d6e7,0x2239acac),LL(0x6b60afbb,0x3a178a1e),L_(0x000001fe), + LL(0x7bee904e,0x3a760e88),LL(0x1c8cff61,0x662259f9),LL(0x3af1d337,0x798ee44c),LL(0x04c2f55a,0x7171b763),LL(0x6b42022c,0x451b89de),LL(0xe995dc45,0x0166754c),LL(0x5d7e90f4,0x45f5e9ea),LL(0x1437fe2c,0x5f81a1be),L_(0x00000183), LL(0xafbcd8ec,0x34c04a7d),LL(0x2e0b1aaf,0x62ecd7d2),LL(0xad928156,0x3ef4d947),LL(0xdc8b88ed,0x90778ccc),LL(0xd0a75501,0x1ea32bf7),LL(0x615d6df4,0x370394ed),LL(0x6cdce7de,0xa5a2d856),LL(0x5b5d94b6,0xb1500a75),L_(0x00000069), + LL(0x3dbc68d8,0x49071d49),LL(0x08ad1120,0x6075e725),LL(0x64ba748d,0xec7f443e),LL(0xf8b1338b,0xe9769df7),LL(0xe04bfcf3,0x276b48ae),LL(0x8c536f3f,0x51362d75),LL(0x91347181,0x7270c649),LL(0x8771d27e,0xd7277846),L_(0x000000b0), LL(0xbe64e850,0xf80d5fba),LL(0xb1ff417a,0x8984b71a),LL(0x7e8990b5,0x052a6765),LL(0xa10e9e6f,0xa3a8ec04),LL(0x68613043,0xf8edeb0d),LL(0x94eee364,0x38d79bc5),LL(0xb9b0283c,0x1f04d202),LL(0x240928e2,0xef3aafdb),L_(0x000001bb), + LL(0xcb0ca3e9,0x7b556d5f),LL(0x0eb6f5de,0xe1fefbb4),LL(0x92b00751,0xadf10d77),LL(0x245d985b,0xa78c0fd8),LL(0x1ec6c5bd,0x197cec62),LL(0x6f653476,0xf59e9de3),LL(0x29578b20,0x48b6a349),LL(0xdd081291,0x858df1e4),L_(0x00000060), LL(0x0140bb6d,0x4a2df7b8),LL(0x72cf54b6,0x702ccf08),LL(0x5ba02c9e,0x4fa2136f),LL(0x4316a469,0x62ca46c9),LL(0x2a601fae,0x6a69d6c7),LL(0xf210ce68,0x3ca9ff0a),LL(0x108647e2,0x7301dc8a),LL(0xbc72d54b,0xc0d011e4),L_(0x0000014f), + LL(0xb93b5399,0xaff92b49),LL(0x3a47424a,0x1313f0d7),LL(0xd74e27bd,0xc984d57a),LL(0xa310f0f4,0x7f2762aa),LL(0xf81b869f,0xc1c0028a),LL(0x73626037,0x16619502),LL(0xb6eafd5a,0x02aa41a3),LL(0x7a26f16a,0xee8393c6),L_(0x00000009), LL(0xa03a118c,0xc3e1533a),LL(0x770bf892,0xe55dcb14),LL(0x37abc785,0x49d52ff8),LL(0x88ea32f9,0xca2e3d46),LL(0xf41fb729,0x28df94aa),LL(0xd1d7fe42,0x6b931662),LL(0x453917fd,0xf0e1ad47),LL(0x15504e62,0xac5c9f2e),L_(0x00000008), + LL(0x320b74eb,0x1643c1a4),LL(0x253c03de,0x2114e9cd),LL(0xd7e1536e,0x1b41ae52),LL(0xc0d640bc,0xe9135dab),LL(0xb1a92fcb,0x5a9ef7aa),LL(0xf491bd34,0xd3e367c8),LL(0x6cfcfac3,0x6970f4aa),LL(0x28093242,0x12bc2a52),L_(0x0000011c), LL(0x174f8a3c,0x68fd0341),LL(0x14138a14,0x1a1ea358),LL(0xd7ebb375,0x0313c60f),LL(0xb31aaf76,0x7f4e2cd8),LL(0x63d1b78f,0x376b2b87),LL(0xea4746f2,0x9adb2628),LL(0x7159cd2f,0x3fee262c),LL(0x45cb3634,0x258e2340),L_(0x0000007d), + LL(0x8e763c10,0x1f614296),LL(0x5dcc4da5,0xa12ada20),LL(0x6dbe5329,0xd7a52ac6),LL(0x2fb42468,0x898121b3),LL(0xe9f5b08e,0x86a37006),LL(0xf5a42f83,0xbe3e6de0),LL(0x271bec98,0xc405b595),LL(0x486c9095,0x9adc363e),L_(0x000001b2), LL(0xc4ad2398,0x3c99889c),LL(0x5b663d6b,0x0c06893d),LL(0xa00328d8,0x8fc3f4f3),LL(0x2578283f,0x3d264389),LL(0x57571710,0x28e44b9a),LL(0xd62bb6bf,0x87dd3c9c),LL(0xd7a2f5d0,0xf55cade8),LL(0xcb792986,0xfa60b3a6),L_(0x000000dc), + LL(0x2139ecb6,0x04c2c927),LL(0x8890056b,0xc5944824),LL(0x319a82e4,0x5d37d95b),LL(0x5a8bd6a5,0xd80dfb73),LL(0xa7edec74,0xdb368732),LL(0x60fac47d,0x4f46dbb9),LL(0xb7d14924,0xe4ae15b9),LL(0x255c8153,0xd6f56370),L_(0x000001d7), LL(0x37f59faa,0x8323077c),LL(0xc814ef11,0x2b965e01),LL(0xd2dfe1b5,0xcad600e8),LL(0x2cdd66c5,0xcd44f8d0),LL(0xad1f4964,0xbb170f04),LL(0x6b03da74,0x09f8b95d),LL(0x721ac428,0xc3ee7059),LL(0xab3fd08d,0x69cd062a),L_(0x000001bd), + LL(0x23bf9a71,0x02685d92),LL(0xb24b23c8,0xbb2912fa),LL(0x700eb07a,0x547f3fb5),LL(0x51442fab,0xf8090af2),LL(0x91ae8f36,0xd9f38784),LL(0x7db330a6,0x213e5f98),LL(0xc4904ecd,0xd61a36f0),LL(0x18124e05,0xdf7f8676),L_(0x0000003c), LL(0xccdc9361,0x2a682aca),LL(0xa540909b,0x1f256aed),LL(0x28d1d810,0x643a464f),LL(0xc1d65b95,0xc56ce322),LL(0xe242b555,0xf79c9363),LL(0x165401c2,0x90b17574),LL(0xb89e030b,0xd9ba6bec),LL(0xad9d3eed,0x3cf323eb),L_(0x00000191), + LL(0xfce984aa,0x68365daf),LL(0x6bf21ffc,0x9e0da99b),LL(0x2a11bae7,0x4b632c36),LL(0xd82e9b91,0x8aba8d4a),LL(0xd3edcc69,0x7bcaa8bf),LL(0x0780abc1,0x4b5bb38e),LL(0xf449e1ff,0xb3a33e0f),LL(0x8bf9427c,0x5f153607),L_(0x00000101), LL(0x2be62f53,0xc25e9667),LL(0xa71cb23b,0xf1192121),LL(0xb5c951ff,0xe5267dca),LL(0xde9bed29,0x45f5f5dc),LL(0x62c0dc77,0x5c58640e),LL(0xb410973e,0xab71fb6e),LL(0x2ca60e2a,0xbd3de2bc),LL(0x91e919a4,0xb16029e8),L_(0x0000002b), + LL(0xf6897f93,0x920ffb1b),LL(0x2f766525,0x5063e19e),LL(0x0c6dbf49,0x2d7c8225),LL(0xc1e5d3a2,0x5919b3b0),LL(0xc8eabb36,0x9bd4d72c),LL(0x7daca33a,0xe43be366),LL(0x6a2d3407,0xd8bf85b2),LL(0xcb065c13,0x74ca1514),L_(0x0000012a), LL(0x69fde12b,0x240e3231),LL(0xdfa8142f,0xbfc4ea2d),LL(0xe35ddb6b,0xac61b3dd),LL(0xf38f22ff,0x0b6750d6),LL(0x6e04a783,0x0ae7817b),LL(0x43182e3c,0x5fc3f142),LL(0x70dd88d4,0x8958a84f),LL(0xe8d996f8,0x988beb73),L_(0x00000076), + LL(0x45a0ae0a,0x2fb883d8),LL(0xae353f2a,0xb61aaafb),LL(0x473d0ade,0x890f51bd),LL(0xd1b37ae3,0x0f0c4103),LL(0xeca49348,0x0087e22d),LL(0xc669a58f,0x3462ae96),LL(0xd7ec27b9,0x71fe3af0),LL(0x345f63a7,0x5d6f6927),L_(0x00000003), LL(0xb6063a3a,0x2047ce82),LL(0x2d78ca1a,0xbf2a2a03),LL(0xffe80d92,0x8144148d),LL(0x41e35712,0xf4375651),LL(0x70453a65,0x4044794e),LL(0x74d6e72f,0xc3b6ed9d),LL(0xc9dec888,0x03c9efa2),LL(0x01d35b17,0x4a8b5ee1),L_(0x00000156), + LL(0x1a111cdf,0xfedb9ea3),LL(0x89f7c11a,0xcc1f90fd),LL(0x930eb52d,0x56a442b3),LL(0xb33c2951,0x6f35d3db),LL(0x189b9ef0,0x5065c93b),LL(0x03375bf3,0xb5e57110),LL(0x9efd6440,0xacf2c750),LL(0xada5967b,0x09a6e279),L_(0x000001b7), LL(0xc686c489,0xdb8bd33e),LL(0x5a1a6302,0x0c057175),LL(0xd07e6a63,0x12ebc219),LL(0xd7415a35,0x5c53acb6),LL(0xffce8b04,0xc61aee58),LL(0xb8a197ba,0x3531c053),LL(0x76809753,0xd0b9df9b),LL(0x5e6fa51d,0xdfc91e09),L_(0x0000004e), + LL(0x3c210d2a,0x8909747e),LL(0x7284d674,0x9a35c4f1),LL(0xaebd8339,0x80afa728),LL(0xcd763811,0xe7b55292),LL(0xbfb44242,0x2f4e7b9f),LL(0xa0832cc1,0x4b2452f9),LL(0xeb50e9df,0xfb6f4f77),LL(0x557d53b4,0x081a5219),L_(0x000001ef), LL(0x5652cac0,0xda08f2ab),LL(0x71ddd646,0xdc6cd83b),LL(0x827b9770,0x2342c2c2),LL(0x40af5e14,0x1b228d0c),LL(0xc507fb1d,0x01646580),LL(0x8f89f75f,0x38a92b52),LL(0xe00b0563,0x154282c5),LL(0x27686a53,0x4f688875),L_(0x0000019d), + LL(0xce950363,0x5b121bb1),LL(0xfd31a6ee,0x6472e541),LL(0x1922fc41,0xcfaa3052),LL(0xffa68d9b,0x6480d380),LL(0xb9a18b55,0xe83a1c00),LL(0xb0e4740a,0x0caf0d03),LL(0x6f130693,0x4ea7894d),LL(0x36b54495,0x3a9ed5ca),L_(0x0000017e), LL(0x3b1453bb,0x6328956b),LL(0x8cad3c5b,0x903ec0c4),LL(0x38be6a97,0x4788849b),LL(0x6a8af4cc,0xb82169df),LL(0x753b96f7,0xf48e2aee),LL(0xbe19a762,0x3765ed66),LL(0xc53900a8,0x5283437e),LL(0x77012317,0x86b0a458),L_(0x00000027), + LL(0xf420ccb1,0xb5408c0e),LL(0x1466bddd,0x7d090375),LL(0x68631831,0x1d77faab),LL(0xfd6f5c35,0xe1c56990),LL(0xed7bc7e6,0x544f54fd),LL(0x65874640,0x9b1f7a03),LL(0xcb87ac9e,0xd060b45a),LL(0x33e1a951,0xd46b22b1),L_(0x000000d4), LL(0x9e1f9ddb,0x7cb30863),LL(0xb9f34068,0xa3dfc88f),LL(0x545e0d0d,0x7b5897e6),LL(0xbbf7c012,0x8cdc1322),LL(0x5bfb3570,0x13ce8bdc),LL(0xe13ad999,0xe1589aef),LL(0xb6cb7333,0x92265f86),LL(0x5f5d1b9d,0x9dfffba1),L_(0x0000005e), +}, +/* digit=39 base_pwr=2^195 */ +{ + LL(0x3521aefd,0x60d3d920),LL(0xe5091b5f,0xae3143dd),LL(0xec304735,0xe360b755),LL(0xf78afe69,0x119298c9),LL(0xe3ed2ff3,0x5c6a7738),LL(0x24d64036,0x06b1298a),LL(0x8b486bf0,0x3448a967),LL(0x81e9050b,0x6d50f02b),L_(0x000000c4), LL(0xa1d8699d,0xf1b1ce68),LL(0x59ff13a9,0xfc1bfb85),LL(0x23011f5e,0x1d2b17a5),LL(0xbec4e57b,0xfdcb9ac6),LL(0x53d5a58e,0x109c3a11),LL(0x4b16461a,0xe6c06b2a),LL(0xa5edc709,0xc93e99c5),LL(0x4ed62c80,0x18529aa9),L_(0x00000102), + LL(0x5ddb9f3a,0xbce7cc65),LL(0x49c78f7a,0xe90f1135),LL(0x6489c7f6,0x7145775f),LL(0xbe8e5262,0xc353f1ce),LL(0x36a4b927,0xda2f29fc),LL(0x3ef5bace,0x3d4c0acc),LL(0xb8074e6a,0x43a9c64f),LL(0x638d3fd6,0x70fffe4c),L_(0x000000c0), LL(0xf57826f1,0x5efbd2cd),LL(0x9bf19e6e,0xe86655bd),LL(0x7942ab0c,0xe82b0e8c),LL(0xb1c0c790,0x2f2b552c),LL(0x9dab8e1e,0x390a098c),LL(0xa67eba46,0x9b4d9810),LL(0x6a4756fc,0xc97785ec),LL(0x8fe8cb25,0xf5f5b6c1),L_(0x00000015), + LL(0x3550c471,0x3e59c5ce),LL(0x3ea8b15a,0xda44d978),LL(0xd1bea64a,0xe489d3d2),LL(0x8d887f59,0xcaf7d8fe),LL(0xf6f90986,0x76e4ba07),LL(0xeeb4dfe6,0x19aece18),LL(0xaf8390c7,0x4b163792),LL(0x8ecf88f9,0xfe44fa1c),L_(0x000001dd), LL(0x787acd3b,0x631704b8),LL(0xceaf9552,0xd06dc6ba),LL(0x319c43cf,0x86d813d4),LL(0xc141f1cb,0x1dcaf56d),LL(0x59594026,0xe9fcecb7),LL(0x7334a724,0xda2f8a7f),LL(0xd2a3a54d,0x40d320ca),LL(0x376b3d8c,0xee9740bc),L_(0x00000190), + LL(0x6b067f95,0xf72bf06c),LL(0x3dcbcaa8,0xcedc2a2b),LL(0x559f9fa4,0x707cbdc6),LL(0x65301ff4,0xb59f1a1d),LL(0xfc409d5e,0xbb9620b3),LL(0x6c53a5fe,0x48b591f7),LL(0x766a3eea,0xc3fc458e),LL(0x1913597e,0xb4cf309e),L_(0x0000005e), LL(0xae5ce891,0x8bb24162),LL(0xdda1da6f,0xcd895e57),LL(0x92393366,0x02de8414),LL(0x59ad0cbc,0x65ce8f07),LL(0x893b6573,0xbcfa2564),LL(0x73186b40,0x4fbea748),LL(0xd0156cb0,0x512a03d4),LL(0x0e490f66,0x328165e7),L_(0x0000016b), + LL(0xcf63e9fb,0xff9d55c9),LL(0xf62fa8e3,0x4867369d),LL(0xa6218d53,0x3d4df374),LL(0x462df770,0x0e4446f3),LL(0x5002dbe2,0xc6146393),LL(0xee0caf51,0x1bc9af4e),LL(0x615e075e,0x2e28e88d),LL(0x750b8016,0xc58c8ffa),L_(0x000001ff), LL(0x98241c37,0x91860ab2),LL(0x88c92592,0x4340d28a),LL(0xa23735bf,0x72c0db10),LL(0x9e762765,0xb8f03780),LL(0xf1edfe96,0x7d6eee09),LL(0x395f70ab,0x2e30082e),LL(0xa5b4a747,0xa42e66ca),LL(0x42b9fa7f,0x3482cf5e),L_(0x000001bb), + LL(0xe7982b9a,0xd1d63dbd),LL(0xe4cd39f5,0x09f98c29),LL(0xaa873723,0x561fec44),LL(0xdc9951ea,0xfd07b42d),LL(0x6c46651a,0x34575586),LL(0x7bf78c6a,0xa447b2e0),LL(0x9284f87c,0x9cdea2ee),LL(0x7235d419,0x677e1753),L_(0x0000009a), LL(0x5010f5fb,0x14229ff9),LL(0xc8f306c8,0xa79e93cb),LL(0x1a7861e7,0x05616521),LL(0x7842d63c,0x95f90f64),LL(0xcf737ed9,0xc0e16cd8),LL(0xff0413e3,0xedcf1408),LL(0xc3a4f30d,0x43a170a9),LL(0x11a0a6b4,0xcc49b5c7),L_(0x000000e0), + LL(0xf95a857d,0xd2ecdcea),LL(0xd4a0eaa9,0x7d01a093),LL(0xc04512bd,0x177d0211),LL(0xa88e2872,0xe8ab8a12),LL(0xf6040bce,0xec0d3b20),LL(0xbaded143,0xef133b8e),LL(0xc0271e57,0x57beaa78),LL(0xa57bec42,0x6cf36a58),L_(0x00000018), LL(0x6a0ef1d6,0x2b962ada),LL(0x54a163f7,0xdc75f9c2),LL(0x9214028b,0xf894f882),LL(0x29ad8172,0x454f24fe),LL(0x0197a015,0xb080990f),LL(0x1b117443,0x1d17b86f),LL(0x15991c9a,0x66059551),LL(0x66f5d53f,0x4e2d70c8),L_(0x000001c7), + LL(0x74d834c4,0x6bac308c),LL(0xd825f740,0x34b0da8d),LL(0x0fbb496f,0xde870fb5),LL(0x365075e1,0x7841bcf2),LL(0xd3c98322,0x3b8e05ff),LL(0xc39c86d8,0x74cbe33d),LL(0xf0fd6d0e,0x8904ae19),LL(0x6ed62a5a,0x2b1e2805),L_(0x000000c3), LL(0x1b67c792,0x97d2a267),LL(0x4c1cde55,0x05031449),LL(0xc0326e91,0x50606033),LL(0x423b1ee1,0xf18317b0),LL(0x398c9c61,0x5cc474ed),LL(0x96a97237,0x7eb4df47),LL(0x96b52ef0,0x1372ae46),LL(0xb1ed9607,0xabf9d1fe),L_(0x000001ff), + LL(0xfbbc04b2,0x31528630),LL(0xd54e33b3,0x4ada5d83),LL(0x74c3835a,0x9759009e),LL(0xdb44c86f,0x44543433),LL(0xb0ab6930,0x244966ca),LL(0x9d760ccb,0xc7b02622),LL(0x48a60f21,0xe78f542a),LL(0xb0eae43b,0x57db1786),L_(0x0000002e), LL(0x41c0d91e,0xabc5cfbf),LL(0x88276635,0x5fba9138),LL(0xcf1d1a57,0x568aba80),LL(0x955f0fe2,0x9cc142b7),LL(0x00c55c57,0x897ffd8e),LL(0x15579a99,0xb910a1ec),LL(0x58a539a3,0x68c6e345),LL(0xa76c02ea,0x37272cb2),L_(0x0000015b), + LL(0x8f19059c,0x9045b8e2),LL(0x054435bd,0xe377c286),LL(0x12ddcca1,0xd747b1a8),LL(0x3eb9f510,0x775c0ea6),LL(0x4fcce9d3,0x5865c783),LL(0xd37d19f7,0x2eb67bba),LL(0xbbc7cb40,0x53271117),LL(0xd5530a0f,0x600a1a8b),L_(0x00000108), LL(0x02132c61,0xddabbe5e),LL(0xb10fe3c6,0x3587db17),LL(0x11b65599,0x1c163208),LL(0xad78aa4f,0xe7539751),LL(0x51c18792,0xc229bfb7),LL(0x5719f77d,0xf84f03ce),LL(0xdd5c3eed,0xbb9c60b9),LL(0xc4b8c257,0xe60da1b9),L_(0x00000145), + LL(0xf591a021,0xe70fa9ae),LL(0x80aea17a,0x3cc6237a),LL(0xa1a05142,0xbfbb8572),LL(0x82ef4062,0x1a092a36),LL(0x584063eb,0x083d9b48),LL(0x0ca19b36,0xd64fde39),LL(0x92047044,0x84e4f4c7),LL(0x3a0049a4,0x025a777c),L_(0x000000e3), LL(0x1e4d1a64,0xa8f92448),LL(0x1e5063b2,0xfb85d333),LL(0x72d2c93b,0x0f374579),LL(0x6c2c1440,0x599d4bd1),LL(0xf99fc78f,0xf8d879ab),LL(0x7157a6f4,0x24350117),LL(0x6511ce35,0xf5039be6),LL(0x3e9cc395,0xa82c44a2),L_(0x00000156), + LL(0x35779ea3,0xe6e7d409),LL(0x83ea2a70,0x4c9ba2b5),LL(0x137328e5,0xd4654390),LL(0xb93501ec,0xd733683a),LL(0x118e98e5,0x89f374dd),LL(0x00d407bd,0x5295b907),LL(0x13b0afb6,0x57db6bfc),LL(0x6b480958,0x95fc47c6),L_(0x00000108), LL(0x42737627,0xaa37df30),LL(0x84543a49,0x5c127536),LL(0xaf148309,0xbf08a1c2),LL(0xcb7176db,0xbab267dc),LL(0xa7bbd2fb,0xabd6efdf),LL(0x8aeeb27e,0xe86eabfc),LL(0xc902ad03,0x4e44e718),LL(0xf09e682a,0x064991f1),L_(0x000001e0), + LL(0x22921213,0xc501c914),LL(0xa64386af,0x86838cd0),LL(0x4dd63878,0xe353d214),LL(0x2b6e52eb,0x298c7007),LL(0xb94c5abb,0x4bbcef96),LL(0x3cdd0d98,0xfb73d97d),LL(0xe31b50a6,0xd4d6c5d5),LL(0x63019a2f,0xaac04770),L_(0x0000008c), LL(0xd0908913,0x1496527b),LL(0x959a4aac,0xe7cd0ef4),LL(0x5fb6b5e8,0xcf4b2051),LL(0x77e30f99,0x1b0c7952),LL(0x70b054fe,0x38ba1d97),LL(0x5de49575,0x947a5a05),LL(0x8fe1e2d6,0x4246cd4c),LL(0x3254f07e,0x238ade18),L_(0x00000133), + LL(0x4d8e1514,0x99aed77f),LL(0xf6fa1112,0x9c04ed64),LL(0xd4feb2e7,0x7a120999),LL(0x5d57b4f9,0x70550af6),LL(0xd07357d3,0x41340660),LL(0xe4afb7c6,0x05ac084c),LL(0x0826e572,0xae197ca4),LL(0x3ea7fc0b,0x8f07d680),L_(0x00000023), LL(0xcb353a2f,0xaf454a02),LL(0xb5cdf6d6,0xb32bd0de),LL(0xf3bb89c8,0x1bd8c3f6),LL(0x5deb355a,0x3db355ab),LL(0x2f043ae6,0xd5c6b398),LL(0x0e90987d,0xabe8910f),LL(0x380521ad,0x4bf6a241),LL(0x3dfa044a,0xfb752ed2),L_(0x000001d8), + LL(0xd3c55232,0xaad132cf),LL(0x696e2831,0x49f240e0),LL(0x0d4e57f0,0x3025b776),LL(0xf18f53bc,0x0b5878b5),LL(0x56b2575b,0x576025b0),LL(0x452417b5,0x51986dad),LL(0xa57a7837,0x5444a7c0),LL(0x9f4452b9,0x9f945ebb),L_(0x00000132), LL(0x83e66b0c,0xca0455ee),LL(0xd553d885,0x83b12fcc),LL(0xd68fe49a,0x3da8d9a1),LL(0xb71fad5d,0xa984d589),LL(0x1f435980,0x5db787bb),LL(0x659a3f24,0xa908e510),LL(0xdd95c91e,0xbe7d501b),LL(0x4a9245db,0xdaa920fc),L_(0x000000d3), + LL(0x70b49262,0x58fdd1be),LL(0x26f6dba6,0x1bc799a8),LL(0xcb4e9512,0xf00f6eae),LL(0x56df676a,0xe75a521d),LL(0x29d333ce,0x4eca7d77),LL(0x27fb68ac,0x206d2e50),LL(0xa49aec5f,0xaa272aa4),LL(0x1b6a988f,0x341efc69),L_(0x0000007f), LL(0x8415461c,0xe6df0f07),LL(0x3afd9193,0x782686ca),LL(0xe7785c7d,0x3c2a9148),LL(0x4c330f1e,0xa49f1fa5),LL(0x82ded4aa,0xde962297),LL(0xb845da08,0x79a993b6),LL(0x0729c991,0xf8fef022),LL(0xad904b0d,0xe6016c6a),L_(0x000001ce), +}, +/* digit=40 base_pwr=2^200 */ +{ + LL(0x0cbfbe4b,0x938c22f8),LL(0x943a3471,0x837e8130),LL(0xe2773aac,0x4a3c4f46),LL(0xf24010c6,0x2b750229),LL(0x138446be,0x007131ff),LL(0x731813b9,0xc2c90ce7),LL(0xe94672d6,0xdd149a00),LL(0x69dcb075,0x7531381b),L_(0x000000d0), LL(0xe8e6de08,0xb6b38c7b),LL(0x39ced7c6,0xb63d5a97),LL(0xa61fbc4f,0x8f6b6bae),LL(0x075fe4d1,0x6ae1dbab),LL(0xc1ebedbd,0x12c3dbf8),LL(0x6dce109a,0xc087b051),LL(0x4a2962c4,0xa1e1733e),LL(0xf40db685,0x9f800e79),L_(0x000001ad), + LL(0x8bb9ff0e,0x663feea7),LL(0x4fde5cbe,0xdd02746e),LL(0x3f440437,0x33232942),LL(0x8cb2a089,0x21f2f603),LL(0x50b4f0e4,0xa8d7b95a),LL(0x18c0b0f4,0x35a473ae),LL(0xc9451cc0,0x8955b22b),LL(0x9c5154ba,0x9d1fd085),L_(0x00000134), LL(0xc1a8bcfe,0xaff04652),LL(0xe19db868,0x961dcb73),LL(0xeea574f2,0xf8c3e1f9),LL(0xf4327664,0x9b512b73),LL(0xb683e483,0xb02a0ec5),LL(0x0fb615a0,0x7991b38a),LL(0xb1e55bb8,0x3f719551),LL(0x19309417,0x0ba8f164),L_(0x0000013c), + LL(0x63247bdc,0x20ed0fcb),LL(0xe26950f9,0xa5916c61),LL(0x8ac76960,0x2ae5b02a),LL(0x17149cfc,0xda5eb1a5),LL(0x5c8f4a8c,0x9118595b),LL(0x0004518e,0x9e0cc88e),LL(0x9dcbce69,0xb0b05838),LL(0x5edca7cd,0x8f7a0d45),L_(0x00000167), LL(0xd452c748,0x773e3080),LL(0xd9ebc5ab,0x32f8567c),LL(0x92748ad1,0x2f890896),LL(0xd6e0eb81,0x83d0c649),LL(0xdc173290,0xed13cd26),LL(0x6815ffda,0x775e539d),LL(0x727168b4,0x09166ff7),LL(0xbfad2565,0x7a36c1d3),L_(0x00000174), + LL(0xa559258c,0x46e6e936),LL(0xf2627126,0x96d6c787),LL(0x7ee8f552,0x6540e78b),LL(0x53eb4432,0x2fb88504),LL(0x073a9cef,0x0e4739b6),LL(0x9bbfb39a,0x5b6532c1),LL(0xbec805ba,0x4331c495),LL(0xedb74df4,0x002e8ec8),L_(0x000001d3), LL(0x48685b39,0xcbd7ba6c),LL(0x4bae18ce,0x3a66d73d),LL(0xa9e818b4,0x5a439da7),LL(0x422e109d,0xe2bd60c3),LL(0x71574884,0x482785ad),LL(0xf6bd330b,0x8c0c9a5b),LL(0x6c8383da,0x0007cc56),LL(0x2a9a00bf,0xa489783e),L_(0x00000056), + LL(0x7a2ccd93,0x9d4a8e40),LL(0xd221fba2,0xd46ad86a),LL(0xfddda1e9,0x14fcb5bf),LL(0xf9686431,0x60db0e24),LL(0xb0468c5a,0xa659be98),LL(0x5c91bca8,0xcabd0c78),LL(0x1e072204,0xd9453dfb),LL(0x50ebbe04,0x8aef77cf),L_(0x00000168), LL(0x9b426c53,0xb8b62e34),LL(0x7872d194,0x43334446),LL(0xddbd4e1c,0xaad0f260),LL(0x42324117,0x7d8cfb9c),LL(0xbd6c92ea,0x883e18f4),LL(0x68768225,0x72898dd5),LL(0x59ac483a,0x25923bd3),LL(0xffec7082,0x48de2e57),L_(0x00000021), + LL(0x8796d389,0xf64d07f7),LL(0x07a33d42,0x4d434a03),LL(0x8948a2a4,0x03ccc6f0),LL(0xbb8f90db,0x6ff7592e),LL(0x69af4969,0x87ff2ae9),LL(0xec7fca3c,0x687414fc),LL(0xd6cec6f5,0xb3255410),LL(0x9a9ae9c6,0x961c9823),L_(0x000001ea), LL(0x1f7e4e99,0x4284bbf9),LL(0x472d0237,0x86ea89ac),LL(0xd63ca5a6,0x6cd552ac),LL(0xe6161434,0x2fb24ab8),LL(0xdc07d107,0x880d3677),LL(0xd1833f7c,0xd225c8c0),LL(0x17c50635,0x2bf84ae7),LL(0xc1f8a219,0x2e83c678),L_(0x0000007f), + LL(0xc0d1be3d,0xbf7ea965),LL(0xe0762dd1,0x7e003dff),LL(0xd60aa791,0x62c54da8),LL(0xfa92fc72,0x21eaa7ed),LL(0xff6dc244,0x62c86ea7),LL(0x29f82d5e,0x68214737),LL(0x535c6df4,0xb69bc00d),LL(0x494bda6b,0xf34e2601),L_(0x00000133), LL(0x3563dbad,0x1096ecec),LL(0x12da1692,0xcc5db4fd),LL(0x2f945903,0xc4f95586),LL(0x15014cb4,0xf70f6fbb),LL(0x3d80e47a,0x0a6967e2),LL(0xadb489f2,0x65a13ebe),LL(0xa0094906,0xa28958e6),LL(0x9fafaa96,0xda82221f),L_(0x00000031), + LL(0xf4cc713b,0xbab5dfd8),LL(0x59f2c453,0xe3fa69de),LL(0x0c9b2cc1,0xb6318b0b),LL(0x56b33c17,0x2d5d399a),LL(0x6dc3c1e9,0xf08f8a6f),LL(0x9e28633f,0x51ffb7f3),LL(0xf1fbcd43,0x77388eef),LL(0x9d013e8c,0x953e5ebf),L_(0x0000006a), LL(0x19a3879a,0xbdbfd2f4),LL(0x95e5a481,0x017a31d1),LL(0x1ef3e1ae,0xb5b37267),LL(0x8988b706,0x748e8ba9),LL(0x3916983f,0x0e9de7d3),LL(0x6e3e3c93,0xe7e37cde),LL(0xee4ca324,0xae3cdd99),LL(0x2fb6a772,0x5fda48d8),L_(0x00000023), + LL(0xb1b36ee9,0xa88f1006),LL(0x32101e9d,0xec995155),LL(0xbc724136,0x50786c7c),LL(0x718458f0,0x942d96ed),LL(0xd4b44c20,0xa25702b9),LL(0xf245f9d1,0x060c2adf),LL(0xb57e7b0b,0xb9d35bf5),LL(0x7ec560d8,0x06f41b22),L_(0x00000056), LL(0x330450ba,0xe70f3385),LL(0xbb3cf7d0,0xaf72af3d),LL(0xf5c80879,0x24797994),LL(0xcff4c536,0x60ff916d),LL(0x9be0e09a,0xab9b1069),LL(0x069c4c8f,0x4df20300),LL(0x1f6d9a1c,0xe23cffcc),LL(0xe8dd8a18,0xf91c6b69),L_(0x000001c4), + LL(0xd0d2f55b,0x95fbe896),LL(0x77feee72,0x5cc61767),LL(0x8880fcfd,0x37faec22),LL(0x520b4347,0xb528d647),LL(0x08c7efa3,0xd89b3eae),LL(0x7f34be1f,0x0837f588),LL(0x08e5ae1a,0x1bd21d07),LL(0xdd6b478f,0x2fe84dcc),L_(0x0000011c), LL(0x39fd2b7b,0x834a2481),LL(0xe505f7b4,0x2410091c),LL(0x6e308dff,0xc904e585),LL(0x881cb270,0x8e50edd4),LL(0xb510da38,0x1a9f6193),LL(0xdd2ee182,0x09d449a4),LL(0x27fbd7e1,0x8ae922fd),LL(0x099fa1c5,0xd2acc7bd),L_(0x00000151), + LL(0x54d97245,0x0a20a859),LL(0x0116816e,0x589d2f96),LL(0x148078e7,0xaaa24c2b),LL(0x28f3d3c2,0x171935f6),LL(0x48826cdd,0x84d3a584),LL(0x4e4018a0,0x3aa25c41),LL(0x20c105fb,0xfbf31507),LL(0x27ff55a5,0x109df084),L_(0x00000126), LL(0xebf39f9e,0x7739d21b),LL(0xd2b7193f,0xacf34cf4),LL(0x3d27ea07,0xab2591de),LL(0x55176728,0x3ceb2fdd),LL(0x2cd960ff,0x716fc560),LL(0xb6983fee,0x90fd8f68),LL(0xb59a98f6,0xe1bb13d6),LL(0x476cf07b,0x119c8087),L_(0x00000019), + LL(0xbdb26400,0x09bcac55),LL(0xcc9f7ef1,0xf3382f38),LL(0x8ea6fbed,0x08f8a371),LL(0x93651490,0x97354cca),LL(0xa5d779cd,0xd6ff00ea),LL(0x17e28bc6,0xcb936676),LL(0xf2c7be3b,0x4d0cca52),LL(0xe6fcf731,0xfe1b3242),L_(0x000001b7), LL(0x195399b2,0x998790dc),LL(0x13a64152,0x616ab4fc),LL(0x4787beee,0x0f38f16c),LL(0x8c14e216,0x13dc0561),LL(0x5144326b,0xab46b249),LL(0x49417c95,0x0e55d9c5),LL(0x87bf06de,0x4c1e541e),LL(0xc0681d93,0x2c9b452b),L_(0x000000e9), + LL(0x5003f6a6,0x8a76574f),LL(0x406e1518,0xe6346b0d),LL(0xb53c0598,0xbdfc6afa),LL(0x9ca435a5,0x50f31dc3),LL(0x9cc07001,0x1695c15e),LL(0x1fd27db3,0x80905ca0),LL(0x1aadf518,0xad7428a8),LL(0x926949ce,0xff033643),L_(0x0000007c), LL(0x248272b0,0x44dec074),LL(0x2345ae4a,0xe0173b35),LL(0x3909da4f,0x56f4144f),LL(0xe4ed2bfd,0x95f4401f),LL(0x198f03a8,0x1227b3e8),LL(0x49d6d509,0x249ec281),LL(0xe4d920ef,0xc12324e5),LL(0x0d321781,0x00c158a5),L_(0x00000106), + LL(0x62d3fa5e,0xe0633553),LL(0x3677eb41,0xa7b03785),LL(0x9c729924,0x166ddf87),LL(0xac9990ff,0x9eba58ed),LL(0x2f6b0f44,0x29db9988),LL(0x2b93f534,0x7f4a4f14),LL(0x92b08207,0x695177a9),LL(0xc1d4f27c,0x1d65e607),L_(0x000001bf), LL(0x7f241b64,0x1902ce34),LL(0xc2625795,0x99c8b2c8),LL(0xb763d2c5,0x4d39eb77),LL(0xa75ff101,0xd8865e52),LL(0x2e31245d,0x51ed16bc),LL(0xfc608d17,0xd782bd3f),LL(0x441d7032,0x8f51fc25),LL(0x47fab287,0xe871d582),L_(0x0000006e), + LL(0xd66fc4f8,0xec4b364a),LL(0xfbb65219,0xd42017be),LL(0x359e7f30,0x3fc15863),LL(0x5c218315,0xb1a3700b),LL(0xf2cbaf1a,0x040dad16),LL(0xb6cd3ff6,0xbf23d44a),LL(0xd045f02a,0x83befb28),LL(0x4160599e,0x467f747b),L_(0x0000001d), LL(0xe7de34cd,0xc40b618b),LL(0xf743241b,0x64d1d40f),LL(0x576ba83f,0x5ece3029),LL(0x4dc64148,0x47769772),LL(0x6d3057fb,0xd175fe83),LL(0x0884e64d,0x33875e4e),LL(0x859df923,0x481b7714),LL(0x655fbae2,0xdef5f5e2),L_(0x00000044), + LL(0x1c7765f1,0x7dce56e1),LL(0xcfdef637,0x3df3dfe4),LL(0x35b39936,0xa715e9e9),LL(0xffaca630,0x1011f820),LL(0x33f64da2,0x222d3bc6),LL(0xc5987552,0x523adab0),LL(0xc95736a8,0x787c715d),LL(0x058fef5b,0x66393c63),L_(0x0000012a), LL(0x97f6f489,0x5434ecf8),LL(0x891f4ace,0x7708ad40),LL(0x51c5f6bf,0xfe89ee25),LL(0xc377ca62,0xe6011a07),LL(0xd24cd2ea,0xd028c949),LL(0xc094a1a4,0xcf84ab99),LL(0x1fed19d9,0x036f7f03),LL(0x0551c154,0x37b50c32),L_(0x000001d4), +}, +/* digit=41 base_pwr=2^205 */ +{ + LL(0xf5fef4c6,0x8936aa0b),LL(0xb069d26c,0xd3718b4b),LL(0x2e3dd1d0,0xeaaab400),LL(0x26ebf3da,0x315a34c4),LL(0x04c67676,0x1eb5f386),LL(0x97e2eb1b,0xbc318051),LL(0x7226db1a,0xf5e17eb3),LL(0x109ad73d,0x97d098f5),L_(0x00000137), LL(0x82acea04,0x5982cf08),LL(0x48806f5a,0x5f09406b),LL(0xabd27505,0x4db94328),LL(0xb3c49a50,0x38e43c40),LL(0x08d386ec,0x582b99e6),LL(0x3b07fe47,0x9089cf44),LL(0x7186cac2,0xac474a1b),LL(0x22d982cf,0x7b0368d4),L_(0x000001aa), + LL(0x51b31a96,0xba6cb60e),LL(0xf6d82cb3,0x989499c2),LL(0xf86e04fd,0xcb66fb0c),LL(0x35487bb6,0x7e6257cf),LL(0xdbe642cb,0x7caa5a38),LL(0x0132dd97,0xcf0c7ff7),LL(0x9fe7cec8,0x8b24a15a),LL(0x54f2a9f9,0x52eb7ce9),L_(0x00000075), LL(0x42f8ae5a,0x47aec9e8),LL(0x766fc554,0xced0e96d),LL(0x99065768,0x493166bd),LL(0x2b9adad9,0x28be045e),LL(0x222a08c3,0xae70d305),LL(0x4f554727,0x61d8ec1d),LL(0xfed1873d,0x41e23d82),LL(0x9f1c46d5,0xf348b3f1),L_(0x0000002e), + LL(0x482e6f9e,0x82d58220),LL(0x488e8bdb,0xaabd7c77),LL(0xba1084c7,0x67c7272c),LL(0xca6ae765,0xab82a151),LL(0x1ca84a79,0xa826c75c),LL(0x06316ad0,0xf3c64348),LL(0x0dd7f329,0xde3d6a04),LL(0x3f33bd2d,0x689c3e45),L_(0x00000043), LL(0xe98af372,0x8c61be6e),LL(0xb8361cc4,0x624c3291),LL(0xb8a7b622,0x4c3e4e24),LL(0x3c10547d,0x37a21d5f),LL(0xfa09bfc9,0x407a153f),LL(0x54fa325a,0x5c7fa63d),LL(0x8d13bea7,0xfb7c45a4),LL(0x2fe1e55b,0x1a8b8531),L_(0x0000015a), + LL(0xa0b12ed6,0x0246d25c),LL(0xaf11f361,0x0ca563e7),LL(0xd246f5c0,0x3dbda22c),LL(0xc1f5f271,0x1517fe92),LL(0x5b82f357,0xae558aff),LL(0xc7e4811a,0xfe93852e),LL(0x0a7d7a0c,0x80c69efa),LL(0x5a9a571e,0xcf0c8dc8),L_(0x0000015e), LL(0x62ea61b3,0xfa629e03),LL(0x70463933,0x97e91195),LL(0x4d8bdb29,0x1670ff63),LL(0x3758d16c,0xac0e352c),LL(0xc3148942,0xebf5c218),LL(0x789d4077,0x68df9269),LL(0xedd91114,0xc949bb1e),LL(0x1c9e59bd,0xa97ced00),L_(0x00000094), + LL(0xab334809,0xa992abd6),LL(0xe820b690,0x598bc6c2),LL(0x5618c97b,0xdba62f4c),LL(0x4cfa34df,0x13d15fa5),LL(0xe73cd8e4,0xf2867cb6),LL(0x0a135ff7,0xc48f860e),LL(0x8e8f76ce,0xfdd988f5),LL(0x3ad07476,0xcb3a07b0),L_(0x0000009d), LL(0x9756e016,0x59630aad),LL(0xf07162de,0x6a1c1e29),LL(0x47eefc5f,0xd695eb4b),LL(0x69323852,0xf81fbfef),LL(0xeb840ba2,0xe9a13161),LL(0x99e6fb6f,0x82ed26e9),LL(0x0282e82e,0xe498cc70),LL(0x4b2c80ab,0x6264860e),L_(0x00000002), + LL(0xc9751c23,0x9f6dc20b),LL(0x5c25edfa,0x51ff9b10),LL(0x2267d84b,0x77cd497c),LL(0xecebc8bb,0xd41da61c),LL(0x2e90633d,0x0c2e11a1),LL(0x61552b8c,0xe7151d1b),LL(0xfb4a699d,0xa5898fdf),LL(0x74297e23,0x1a45323e),L_(0x000000f7), LL(0xcf0213ad,0xfe2e5183),LL(0xf83ccca0,0x0905c1e0),LL(0xc44aa9ba,0x0b1cf7ee),LL(0xfedaabc5,0xc2ab4a81),LL(0x7f856296,0xb2c2c3b9),LL(0x7c377576,0x110f594f),LL(0x248077d8,0x0920b595),LL(0x7dcc7073,0xa5e1a393),L_(0x000001a3), + LL(0x5b8dd11b,0x003d8cbe),LL(0xf2309e12,0x1dd14f55),LL(0x6aafea42,0xd6b6d08d),LL(0x9f501238,0xd63623cd),LL(0xe37ddae1,0x079b78ad),LL(0xc658ecac,0x62a7d933),LL(0xffca3243,0xf37ff209),LL(0xefd095aa,0x8e2c83df),L_(0x000000dd), LL(0x08483b62,0x5e0a6e24),LL(0x2d2f9a76,0x1a69b899),LL(0xe6914b5b,0xf621c0e1),LL(0x8472fe93,0xdd02cec9),LL(0x92c73abd,0x8f7b6745),LL(0x3d6f438d,0xeceedfc7),LL(0x290f69f2,0x98f33394),LL(0x0ed2ebb0,0x2536905b),L_(0x00000123), + LL(0x63653aa4,0x5e0e658f),LL(0xa4e263b1,0xb787e362),LL(0xf1a72f5c,0xade21c8a),LL(0x3468dd85,0x51d6c477),LL(0xd69f8f93,0x4ea4254f),LL(0xae15e0af,0xdb86c982),LL(0xdd836935,0x98d3a2d5),LL(0xdc232783,0x5ffb0769),L_(0x0000002c), LL(0xd27779d1,0x4c2ae6ec),LL(0x043db3d9,0x09230456),LL(0x73642e7c,0x6f9dd795),LL(0x2a8692df,0x2cd98576),LL(0xa83e7242,0x16ac0a49),LL(0xa2e9e20f,0xd3fe59cf),LL(0xd0093708,0x10a46920),LL(0x10c84d0b,0xa5bda12a),L_(0x0000015f), + LL(0xedf1bf7e,0x8efb0b81),LL(0x586ad55f,0x8d830110),LL(0xef747d77,0xb3bf603e),LL(0x0f99e447,0xb48e1874),LL(0x9da3060c,0x389d594d),LL(0x10ea78c4,0xc7644272),LL(0xd0f0eafb,0xaf1ebb75),LL(0x7afab007,0xcc569782),L_(0x00000154), LL(0x0ef592de,0xebfccb68),LL(0xf0400a4a,0x474a550e),LL(0x9ca24b23,0xbf9aef56),LL(0x5613d5ac,0xab9d9c2b),LL(0xb8267455,0xbf868bc0),LL(0xdb52868c,0x909956ec),LL(0x50005f46,0x49928d00),LL(0xac611000,0x904632ee),L_(0x00000162), + LL(0x8607bdea,0xe148f242),LL(0x2dd2babc,0xa0bb05f4),LL(0xd775106b,0xbff23d31),LL(0xf2919560,0xe97c018c),LL(0x44a63043,0x4871d249),LL(0x87f10683,0x189ec6a3),LL(0x0d74d0f1,0x257dbb86),LL(0xd0bbd041,0xcc6bf0e5),L_(0x000001b9), LL(0x94b22c9b,0xad1a6c62),LL(0x9dacf60b,0xde935e97),LL(0x77ba8de8,0x73dfb2df),LL(0xeb7f0da2,0x64121541),LL(0x18d0ee67,0x024e0b69),LL(0x7b37359f,0x84be521c),LL(0xea621f3f,0xcd285848),LL(0xf996c437,0x6391d449),L_(0x00000030), + LL(0xf3eb9640,0x6d04608b),LL(0x807ec468,0x80a83900),LL(0xb77f8649,0x5eb64b76),LL(0x07caea9c,0xc62e7f72),LL(0x0ec672b0,0x465c546e),LL(0x378afe4c,0x4b627b99),LL(0xcc8adc18,0x929649b3),LL(0x8be7d42e,0x967ccf92),L_(0x00000057), LL(0x7aaa87b5,0xa6efca22),LL(0x4f172127,0xf3a643dc),LL(0x9bf5975f,0x4fe72fa7),LL(0x90cb1c95,0x4c4df518),LL(0xba142ada,0x799862f3),LL(0xca2d035c,0x68d4ff7c),LL(0x98b9e83b,0x7d49b932),LL(0x13c2887b,0x21991bff),L_(0x00000051), + LL(0xecf8f1af,0x881d7401),LL(0x5a044cc4,0xe24e7a25),LL(0x768d2e23,0xfaed9fe1),LL(0x8f7705f4,0x383ba961),LL(0xd4a1b2c2,0x435ea5ef),LL(0xa2b9797e,0x3b37f12c),LL(0xe7f7c215,0xb091bf0d),LL(0xf2391054,0xb7ba542b),L_(0x0000004c), LL(0x7896f156,0x701310b7),LL(0x9eb25738,0x04099f3f),LL(0x6dcb2223,0x7798cb36),LL(0x4d56786a,0x95c47f22),LL(0x919da3c8,0xdaeee06f),LL(0xb6abb8e6,0xa660ceaa),LL(0xa641a64e,0xa1891535),LL(0xc989455e,0x627a7ec8),L_(0x0000019f), + LL(0x8abf9f25,0x834e7f6b),LL(0xa4f8a8ed,0x8d0ab54c),LL(0x7e835385,0x86915b58),LL(0x726730be,0xba996ac1),LL(0xe84c9776,0x0ac27d99),LL(0x59603bec,0xbcd0e3fb),LL(0xc7989ede,0x1f1942c9),LL(0xf303b7a8,0x9f4547f6),L_(0x000000e4), LL(0x681044e0,0x53ff8c21),LL(0x01ac096d,0xdb4b7aa1),LL(0x69d169b0,0x9723cdd3),LL(0x802b82f6,0x2051bc9d),LL(0x256afff2,0x3d4a1ea2),LL(0x321bcf00,0x7da3a724),LL(0x26d669c5,0x4ccffc5e),LL(0x72d30cad,0x212847aa),L_(0x000000bc), + LL(0x8217dbfd,0x7dca0317),LL(0xad5ed722,0xe8f1e786),LL(0x8f7b7ae7,0x30b8f677),LL(0x9bd08de3,0x03bfd2ff),LL(0x1955b540,0x599706c8),LL(0xfa5ccf69,0x66c7232f),LL(0xa98d3152,0xa531f734),LL(0x35d69728,0xa518ffd1),L_(0x000001dd), LL(0x28dfd590,0xc9f5e869),LL(0x5ce90640,0x7f70dba5),LL(0x4d6b1828,0xa187b391),LL(0x636739ff,0xc3f757a7),LL(0x6788a6b6,0xca8804a7),LL(0xc2a0400e,0x6dfa8acb),LL(0x499f91eb,0x8bd0499a),LL(0x28af9210,0x45056091),L_(0x000001ea), + LL(0xdd2aeb14,0x35af5ce4),LL(0x33f86de1,0x24f47c0f),LL(0x237c0920,0x0133d426),LL(0x76b14d92,0xdebe095d),LL(0x989c257d,0x7569cd5b),LL(0x8d5f30b4,0xeebd5dcb),LL(0x7daee4b5,0x381f623b),LL(0x5a27d2f9,0xc7fab47d),L_(0x000000c0), LL(0xb30042d8,0x87c8f748),LL(0x04d7a15d,0x0ac1e9b0),LL(0x96216dea,0xb1634ac6),LL(0x3c5fec65,0xf61cf904),LL(0xab711a92,0x3d592940),LL(0x9bf5392f,0x2eefb59a),LL(0x96b616e2,0x6a36ed7e),LL(0x5c5c3417,0xb15b4b78),L_(0x0000008c), + LL(0xfb9aeab3,0x109924ed),LL(0x46d2968a,0x9147a28a),LL(0xfe84ed7a,0x49744c91),LL(0x88e478aa,0xfd889651),LL(0x65a34f30,0x8dc8d99e),LL(0x21fd955c,0x740206f2),LL(0x7ea7cd99,0xdedce892),LL(0xd4f83ab9,0xa7c26d23),L_(0x000000e8), LL(0x2c8cb8bc,0x91728e18),LL(0x0fa5f320,0x3ad9d78b),LL(0x760a4e2a,0x65aca369),LL(0x6812b50f,0x46ee027e),LL(0xdb993f3d,0xe5a7e2b8),LL(0x2acac076,0x60290375),LL(0xa179054a,0xddbfa0d3),LL(0xf87bff0f,0xee0dfeef),L_(0x000000fe), +}, +/* digit=42 base_pwr=2^210 */ +{ + LL(0xbd161967,0xbda60a63),LL(0x090aae19,0xcb3f577f),LL(0xe7638c32,0xf59abf93),LL(0x34b2a6dd,0x35486136),LL(0x3c50db1d,0x91b5e651),LL(0x49476ec8,0xf4bbb5bd),LL(0x83b636d2,0x3dd95f7a),LL(0xd5071e3a,0x77c02f69),L_(0x000000f9), LL(0x9531d83c,0xc6860379),LL(0x9c9ad3cf,0x35f1eec4),LL(0x4e50cb96,0x26b39588),LL(0xd703ca9d,0x3bd6a0e5),LL(0x3fe9036f,0x08ef03a9),LL(0x605b0ecc,0x070faad2),LL(0x6abd3a9b,0xf3494eab),LL(0x7fa81977,0x164f95f6),L_(0x00000031), + LL(0x676b493c,0x57205d81),LL(0xe8f546e2,0x442871cf),LL(0x87afe8b6,0xd5e346c5),LL(0x748676ba,0xa964afa3),LL(0xca39baf4,0xe1422f71),LL(0x0e9e0a58,0xd62c328b),LL(0xd31cca18,0x07714d71),LL(0x75787f65,0x810168e3),L_(0x000001ad), LL(0xa1440c8d,0xb730f78a),LL(0x509d6354,0xec14ff7e),LL(0xbeae80e0,0x9793053f),LL(0x019f7cc0,0xb6b1fd1b),LL(0xe4fca025,0x44558d48),LL(0x7ed4a037,0x86992aae),LL(0x0e2db1c4,0xf0333757),LL(0x557f4b02,0x30117649),L_(0x000000f4), + LL(0x194c0cc8,0x23e0df8d),LL(0x58c8cbc9,0x1732c3e0),LL(0x3466783b,0xf47836e0),LL(0x4713b9a8,0x79e1d15c),LL(0xa517b03d,0xefa174a4),LL(0x63c15938,0x49e8d766),LL(0xea4a3245,0xd01d6313),LL(0xbc5db16f,0x83758c05),L_(0x00000196), LL(0x7797400c,0xbf271b78),LL(0x3af2e11a,0x60042746),LL(0xa3ab648d,0x79d1c019),LL(0x5000aba1,0x253b9712),LL(0xd9239c9e,0x0e930854),LL(0x85a1b532,0xab5ac676),LL(0xe00c287d,0x57eaaede),LL(0xe2d767a0,0x43b264bf),L_(0x00000180), + LL(0x2d438c77,0xd715e37a),LL(0x436c808c,0x8615d6bb),LL(0x78232591,0x58c6e6b2),LL(0xca6d68ce,0xa40e8f75),LL(0xc4c37875,0xc01da381),LL(0xbe962879,0x58a155d9),LL(0x5dd3d4cf,0x847d5de7),LL(0xee99fd85,0x8f7f76b4),L_(0x00000015), LL(0x4cec18db,0xd4c9f66a),LL(0x5302a76b,0x647086e4),LL(0x1b679cdf,0x93b84a7e),LL(0xd412c242,0x92243bc4),LL(0x519ccba0,0xd5c3c375),LL(0x585371f2,0x8ba3d06b),LL(0x90f4c0f2,0x1daa7685),LL(0xf573b409,0x6342e78b),L_(0x0000013c), + LL(0x0b8749aa,0xf1f9c0d3),LL(0xf2f4b2d5,0xbf3b998b),LL(0x6f543cfb,0x0d744317),LL(0x25e2b138,0xda23008b),LL(0xc81ea703,0xa6df5808),LL(0x322e0c8c,0xdabad20e),LL(0x654aa6ef,0x3a6c3719),LL(0x64c8d439,0xf75f2e54),L_(0x00000167), LL(0x90408e7d,0x50778fd6),LL(0xbd5f5197,0xcbe15e1b),LL(0x16df4a37,0x36cef226),LL(0xb2ae2273,0xfaaea71f),LL(0xb8561402,0x40dd546c),LL(0xed28c500,0xa3b837f1),LL(0xe3cde0fb,0x7315fb7b),LL(0x893aced1,0x828da346),L_(0x0000010b), + LL(0x9ff41dd8,0x136ca413),LL(0x7447323e,0xc3f1d660),LL(0xfd177667,0x0d79142a),LL(0xeab4ffc2,0x1d798fb8),LL(0x8d6c7790,0x65d4f135),LL(0x6db0f7cc,0x3bd9feb2),LL(0xeb9db217,0xf714d78c),LL(0x48ba0ac8,0xa3ca23a0),L_(0x000000ce), LL(0xb4e50378,0xe69b95fe),LL(0x3047872d,0x2f7264f0),LL(0x968482d0,0xedcd1afb),LL(0x4afd067b,0x21685d4f),LL(0xd2b0a788,0x8c222e50),LL(0xee57ad29,0x86dde86a),LL(0xb70520ec,0xb933bc16),LL(0x0ddb5005,0x16594b1c),L_(0x0000019c), + LL(0x2b0b1b51,0xbbca315f),LL(0x5bb1b6c3,0x0036d456),LL(0x7469a0dc,0xd48baaad),LL(0x017a26b6,0x27d8ab41),LL(0x02fa6e32,0x15045224),LL(0x0c7090d4,0xdb62af96),LL(0x85ab46be,0x89adad56),LL(0x45b4363b,0x449f71e2),L_(0x00000079), LL(0x96000d89,0xf8377adc),LL(0x4bc2c5ac,0xa5878a73),LL(0x58a681e7,0x18006b8a),LL(0x8426f06f,0xacb84fbb),LL(0xf862f722,0x32b8466b),LL(0x041f4d43,0xe59c6ec4),LL(0xac4fc1fd,0xda852fdc),LL(0xa52e3ee8,0xf6e11234),L_(0x00000176), + LL(0x9fca3c39,0x9ddcb7fa),LL(0x44679a2a,0x8069c471),LL(0x3be369b3,0x0de84dbc),LL(0x82ca8262,0xd5e1f28d),LL(0x89e87798,0xa205de89),LL(0x84051f10,0x5c22abac),LL(0xc26a5b9c,0x99bba5fa),LL(0xdd74ecbb,0x359fa6c2),L_(0x00000053), LL(0x82c210ab,0x2e2ac09a),LL(0x2ac08c57,0x71535b42),LL(0xc5c720e0,0xa7bd1c3d),LL(0x18f95966,0x0a4b0c9e),LL(0x49a62e3a,0x2faa6c64),LL(0x4f85595c,0x529cbdd4),LL(0x7bb9f75f,0x93955cf4),LL(0x50f46a64,0xdefa5af6),L_(0x00000022), + LL(0x1a200a95,0xedc6f59f),LL(0xa294c7f6,0xa2e1643a),LL(0x5eabc120,0x41a385d8),LL(0x0ff38f95,0x6b429a90),LL(0xa608b840,0x5f0ae2e2),LL(0xf1d02f3b,0x9b8946a7),LL(0x2525fc8f,0xc76a7386),LL(0x49cc1359,0xca2a7f4d),L_(0x0000010a), LL(0x8186c176,0xbe8aed8e),LL(0x60d4bdb6,0x86d41413),LL(0x937209e4,0x3c716d10),LL(0xf538d0a6,0xa096c4bf),LL(0xd3c035a2,0x04283ffd),LL(0xb280d9ae,0x4964e73c),LL(0x3893b1c0,0x75d67682),LL(0xc2768753,0x005e5f85),L_(0x00000022), + LL(0x364ca3fa,0x8d283446),LL(0x498b2996,0x86abf4d3),LL(0xfde09c12,0x2dfa3c50),LL(0x0e695616,0x29f9b0d2),LL(0xb697398e,0x48792e33),LL(0x568f5e3a,0x7493cab0),LL(0x6ee081c2,0x657411b7),LL(0xb996a914,0x6bba20f2),L_(0x000001c4), LL(0x21ec8a20,0xcc21ceb0),LL(0xf3a30195,0x04798bbc),LL(0x8327746a,0xa982d5b1),LL(0x2b585e77,0xd3a733ef),LL(0x8fc21ff1,0x683b1710),LL(0xf43ba5ab,0xca115f83),LL(0xeb98616c,0xa31d56f2),LL(0x89236402,0x2fd3f97b),L_(0x000001e5), + LL(0xfdd8cece,0x71daa8fc),LL(0xcd75aec1,0x1ba03e2e),LL(0x3c07bc51,0x57185304),LL(0xca3e327c,0xb1122c0c),LL(0xfb82f00a,0x4ab98ef9),LL(0x143ac664,0xed517312),LL(0xb67dba09,0x3be9088b),LL(0xf6425b41,0x9abf3748),L_(0x000001ad), LL(0xa0f7c32b,0x58516a7c),LL(0x937668fa,0x1a5b0042),LL(0xee40f983,0x70389a31),LL(0x7bddb2f1,0xc5b229d2),LL(0x5e0553f6,0x5a0d088b),LL(0xeae9e1a5,0xf99b1b9a),LL(0xcda92dcb,0x5ef8199d),LL(0x7e285536,0x50942301),L_(0x000000fb), + LL(0xe8acd015,0x276df335),LL(0x996f91a7,0xbba5b9b6),LL(0x8be8a5f6,0xa13311be),LL(0x1f26ef24,0xfe2a95d5),LL(0xee11bab7,0x684a38b4),LL(0x3233a4f7,0x2ff2caf9),LL(0x3771e476,0x50176c17),LL(0x5e24adb0,0x7e6a96ea),L_(0x0000009e), LL(0xcee15413,0xd1540fbe),LL(0x9f444fde,0xc5cef3f3),LL(0x6ce01534,0x25f5b460),LL(0x1d8a8861,0x37e2199b),LL(0xa77157b4,0xea994fb5),LL(0x599e6b65,0xa91dfcae),LL(0x599cf24d,0x48964c92),LL(0x5d9f0d72,0x171d2191),L_(0x000001ac), + LL(0xb07ac833,0xc7cc5161),LL(0x5e82226c,0x71009ee1),LL(0x4bc92633,0xa4a6f458),LL(0x4b11e8ce,0x27cf64fc),LL(0xa35c3f83,0x247f343b),LL(0x08c548d2,0x5b9231c1),LL(0x8238f13a,0x98e33dc3),LL(0xb0046a52,0x4e6220c7),L_(0x0000010d), LL(0x2464e468,0x60df1456),LL(0xc43a8ffe,0xb03d69e6),LL(0xf8867110,0x760c6b46),LL(0x1cbd5f52,0xbfe67f56),LL(0xee1342df,0xf514ecfe),LL(0x7fa3d377,0xeacf358c),LL(0xb7abe871,0xf91db13e),LL(0xf63ff9a1,0xfd2c6720),L_(0x00000118), + LL(0xbcc7e27c,0xfcd4da3a),LL(0x01aa7fce,0x5590f28d),LL(0x891c86bc,0x3fde019f),LL(0xc74e219a,0x47bfd522),LL(0x17dfc33e,0xcd902b91),LL(0x1e1f0327,0x19b7adb8),LL(0x46fb1987,0x4aa71b67),LL(0x7c133f73,0xc87d00cc),L_(0x0000009a), LL(0x24c3c4f4,0x15897652),LL(0x484cb85c,0x6d5a49dd),LL(0x08cbe8b3,0x0398f1d3),LL(0xc2570a5f,0xcf0588fd),LL(0x75aff023,0x9a086ca6),LL(0x7d708565,0xf0ab637b),LL(0x478e9220,0x4da37998),LL(0x79f64eda,0x9f09ba72),L_(0x00000129), + LL(0xf12325c2,0x3cf6f674),LL(0x4d0bf9f0,0xe45ebcfb),LL(0xf49cc75a,0xf02c0b22),LL(0x198a7df8,0xce88b254),LL(0x798e3ccc,0x452bc12b),LL(0xcb1f3272,0xf7e1eeab),LL(0x204a39c3,0x03001038),LL(0xbf6a035b,0x8bb9d05a),L_(0x00000106), LL(0xbee3c686,0x519b1e12),LL(0xff47d24d,0xecef22c6),LL(0x1e6f218f,0xb9743d3d),LL(0x7fe5e0b0,0x0de37bec),LL(0xb84cd289,0x2f608290),LL(0x640acc09,0x0325466e),LL(0xf1f489de,0x523a4381),LL(0x5a2fd923,0x78483089),L_(0x0000012b), + LL(0x55dde0b9,0x51ca4b89),LL(0x0c9b5cc2,0xcb500f85),LL(0xb99572d6,0xb82356c8),LL(0x9eef8952,0x575ebe20),LL(0x607df288,0x389253a4),LL(0xa421a6b3,0xd513ff9b),LL(0x99f0eb4d,0xc84b56b7),LL(0x253ed789,0xe6a5c4d4),L_(0x00000093), LL(0x0f18530f,0x98ff217c),LL(0x6930ae01,0x8abc5898),LL(0x12fd1e77,0x94fad4d8),LL(0xf2ff5095,0x6733dcce),LL(0xf10fab65,0x36afad83),LL(0x17d75c59,0xd99578f7),LL(0x8a1a313c,0x16027411),LL(0x0bcd7387,0x46078ec4),L_(0x00000154), +}, +/* digit=43 base_pwr=2^215 */ +{ + LL(0xcb23bf8a,0x011cfb71),LL(0x25479ac0,0x785ff67b),LL(0x2f48bcdb,0xc454fd25),LL(0xf62da267,0x4e460f65),LL(0xac759a15,0xe6fa21e0),LL(0xdb56d239,0xfa07770b),LL(0xb91d7cce,0xfe275a8d),LL(0x68f07f46,0xa1529273),L_(0x00000127), LL(0x74d64a14,0xd7969a0d),LL(0x36f066b2,0xd81a759c),LL(0x3a664061,0x929c3e1c),LL(0x5ba8c41d,0x033ad63b),LL(0xfdb5a8a4,0xf387b665),LL(0x2b433c84,0x1e742e2c),LL(0x4f2d4d93,0xb030d2de),LL(0x8d631141,0xe3845c24),L_(0x0000006d), + LL(0x020c1dd2,0x650c039d),LL(0xab7b6907,0xb4f64d44),LL(0x26b42700,0x0576d051),LL(0x2c72c1e2,0x70d2ad71),LL(0x4322ae9e,0x315f4631),LL(0x89904b57,0xef02dfb1),LL(0x24905a45,0x8a7b4701),LL(0xde26cce6,0x5f0db2ca),L_(0x000000ac), LL(0x55247780,0x3a42cd20),LL(0x6fcfeb99,0xee9920d8),LL(0x1cc7f2d2,0x46bdb299),LL(0xb71c2095,0x1516a6ea),LL(0x99e62c53,0x95f1492f),LL(0x197ae770,0xe95c2cf5),LL(0x013e12e8,0x2aad7be2),LL(0x60b78cb7,0x0e70b967),L_(0x00000032), + LL(0x5d9cf3d8,0x7b152b81),LL(0x2adc31d1,0x1cf4d989),LL(0xf817919d,0xea745ea9),LL(0x86caaaf3,0x035cfaca),LL(0xbd62e874,0x533bc33d),LL(0xb1acfb8f,0x2cc2ce8d),LL(0xb212b5b7,0x5b9ba7f0),LL(0x350192b7,0xce04c178),L_(0x000001a3), LL(0xdb4255a9,0x88563e49),LL(0xaa946b95,0xe4c90142),LL(0x0e515aee,0x5062c2f0),LL(0x99e87538,0xcd39192d),LL(0x52eb943e,0x0c893238),LL(0x0201b73c,0xfbac7e1c),LL(0x1ab36a78,0xc6d833b4),LL(0x58d01a7c,0xe359c01f),L_(0x00000008), + LL(0x97851e87,0x7511fae2),LL(0x9afd1135,0x428434f6),LL(0xab322fa2,0xcbe5e3de),LL(0xd89f361e,0xc1b08880),LL(0xc1fbd2b7,0x6a50aa80),LL(0x9e40537e,0x7fdf104d),LL(0xd4f51df5,0xe707164e),LL(0xb78a6cfb,0xd887e3d0),L_(0x00000136), LL(0x9a5983b8,0x96365e7f),LL(0x129d87d9,0x64aad3c6),LL(0x2952186a,0xf8224d3c),LL(0x45209284,0xc689c1d4),LL(0x2c194d7b,0x03f44aec),LL(0x0e7c6b2d,0xf18a57e0),LL(0xe28c9eb3,0xfac4981d),LL(0x4b8b7de6,0x55215906),L_(0x00000009), + LL(0x7754239c,0x3a3a13a5),LL(0xbeace249,0xc22b9755),LL(0xd063181b,0x27e6ebf5),LL(0x80b61753,0x0eb1c7f3),LL(0x8b95d0ba,0x791e9667),LL(0x5d4d0f0f,0x5f5189e4),LL(0xdd28c3e4,0x9162d716),LL(0x2eca4da1,0x40913d2a),L_(0x000001de), LL(0x21d602fa,0x85fea8d0),LL(0x3c004f10,0xa6620dad),LL(0xb3680a4e,0x31990981),LL(0x8a6964de,0x3b4c2f57),LL(0xd1986197,0x3b4fdf64),LL(0x5bc5cf00,0xea0f7010),LL(0xbf1dee68,0x0d263581),LL(0x3edd2000,0x8ba6db98),L_(0x00000012), + LL(0xc5f1e841,0x79147285),LL(0xbe273f4c,0x71c0654b),LL(0xec66bd4d,0xdd0505e9),LL(0xed4771b4,0x4c891619),LL(0x316ff12d,0x821a0542),LL(0x0c65ede4,0xa23cffae),LL(0xb678c0c5,0xbf05e02d),LL(0x32bbf48a,0x91ff4f9e),L_(0x00000126), LL(0xf495f39d,0x944ec586),LL(0x5c351b2d,0x00e74766),LL(0x3939c0c8,0xfe310474),LL(0xb780a9b2,0x5e63ba55),LL(0x11cfcec0,0xc11119c9),LL(0x322972c9,0x0c8e0043),LL(0x35f5ba3b,0xa0946bc2),LL(0x222b78da,0x622257dc),L_(0x00000082), + LL(0xba9d4a6b,0x2b9dae76),LL(0xe9b24e64,0x1b491ac4),LL(0x9716bae3,0x162c28a6),LL(0x192c7196,0x00752a8a),LL(0x10883cfa,0xf0af2cfc),LL(0x77a5388d,0x5f1ffb0c),LL(0xebf78534,0x49abf2a7),LL(0xe9343bc9,0x54bcef89),L_(0x00000090), LL(0xd39aabbd,0xa864f2a7),LL(0x1bdde3a5,0x0362ffa7),LL(0x45782cba,0x113c94f8),LL(0xb94efa60,0xb5969326),LL(0xd2b826b0,0x92ab6bc9),LL(0x228e2d7d,0x1f6dc09e),LL(0xe5f07f40,0x3371efd7),LL(0x87e6028d,0x90fcb8fa),L_(0x000001be), + LL(0x3b22c3be,0xcf53147b),LL(0x97a83e73,0x0fba4048),LL(0xd11d0f51,0xfcf9128e),LL(0x6dfde4ab,0x095c92d5),LL(0xbcfa2a47,0xff3cd334),LL(0xa5b6f4d4,0xa2490038),LL(0x5df73169,0x663bcbc9),LL(0x41aba80e,0xa47769e8),L_(0x00000064), LL(0x62c4cf6f,0x2104365a),LL(0xa582938f,0xc439a531),LL(0x43321e6d,0xf7387146),LL(0xee861253,0x52424fe3),LL(0x13590283,0x0a676302),LL(0xe71ab2ae,0xa837702f),LL(0xf460396c,0xf7bce515),LL(0x01882e04,0xd9362399),L_(0x00000193), + LL(0x1e3dd431,0x9bd2b2b9),LL(0xc33456a5,0x6dcdccc4),LL(0x97a89ed3,0x865bca41),LL(0xbdb83821,0xf50a59bc),LL(0x46a8f43a,0x47c10299),LL(0xbaa69204,0x03139274),LL(0x1da8242a,0xa899da5a),LL(0x49b0bcc8,0x2caf68d5),L_(0x000000a8), LL(0xb2c24e74,0x4dc88c23),LL(0xc1c2ff36,0x896e10e9),LL(0x10c6c1db,0x7a488fa2),LL(0x58161012,0x6b4ceefa),LL(0x5188acf7,0xea1e2f11),LL(0x72073c50,0xefaa1151),LL(0xe696e16a,0xd65f38a5),LL(0xe4dea3d4,0x6710e2d4),L_(0x00000011), + LL(0x84a8ceca,0x6cdc17de),LL(0x2c374c6c,0xe3454080),LL(0xbd6a5b20,0x337b0f58),LL(0x40246c40,0x31d7aff9),LL(0x49292d34,0x01dba6a8),LL(0x37486a26,0x41a0f90b),LL(0x12d61cbe,0x782067dc),LL(0x3033f828,0xebc39fcd),L_(0x00000086), LL(0x93cb68aa,0x026818da),LL(0xf42673be,0xf2e2739e),LL(0x40e906d3,0xd8ecf6f0),LL(0x37b0e1f1,0x5c024d38),LL(0x6708c065,0x3417cf59),LL(0xc5788208,0x17020946),LL(0x8942a103,0x7086a405),LL(0x279752bc,0xbd8d65b0),L_(0x00000074), + LL(0xd8f6e509,0x9abc3d40),LL(0xf756f14e,0x748874bf),LL(0xf9192ae1,0x50f7dc14),LL(0xfb847314,0x424a6e96),LL(0xd7141cc9,0xe0539e22),LL(0xefedb9cd,0x7f5ec1a8),LL(0x7abbcc13,0xe6446e18),LL(0x0050521f,0x3706445a),L_(0x000001d0), LL(0x4ed46ff1,0x2f62e709),LL(0xa5e744f0,0x8279d454),LL(0x8aad355f,0x03801f2d),LL(0x88a334e7,0x30342602),LL(0x34cb8228,0xed180e75),LL(0x6d4243df,0x5bbbe349),LL(0xd0752a72,0x0977e643),LL(0xd5658e43,0xf968b621),L_(0x000000a7), + LL(0x2137affe,0x3757bccc),LL(0xce9d3ef4,0xb10a83cd),LL(0x021aa7d7,0x804bf03e),LL(0x79a14071,0x09822fa4),LL(0x1adecf50,0xec4d2fe9),LL(0x8fc3b061,0x479fbad1),LL(0xffe82ea4,0xb70c5762),LL(0x25b0bed7,0x8a667da4),L_(0x0000015c), LL(0x574750b9,0xd836c22b),LL(0x9beeade4,0xd28e01a7),LL(0xd7c41634,0x124715f6),LL(0x83d30d7e,0x8e33c4f9),LL(0x2ee2bdb2,0x5442a068),LL(0xa18cfc1f,0x68390a48),LL(0xa6cb1637,0x44abd789),LL(0xa8ecc588,0xbd6682cb),L_(0x00000183), + LL(0xe26454b3,0xd8199c9b),LL(0x86b3c132,0x77f229f5),LL(0x1a49a2e0,0x6f74d75e),LL(0x15ed7662,0x8dd2368b),LL(0x7d799783,0x313152ab),LL(0xa636aa73,0x925319e7),LL(0x2947bd12,0xacf4c559),LL(0xb2b023d8,0x0322c16a),L_(0x000001d9), LL(0x6d2236bf,0x8a09b0a3),LL(0xb1025eae,0x8ae5f654),LL(0xf817133f,0x081d6d59),LL(0xea1aa5d9,0xe9090ae2),LL(0xe61c2d54,0x0784905d),LL(0xf542221b,0x5789e25c),LL(0xa9b09c19,0x1b8dadc2),LL(0x7d4f5221,0xabe01efb),L_(0x000000be), + LL(0xa224339b,0x8fc36944),LL(0x1a86a356,0x88a55222),LL(0xa9332303,0xfc1f186c),LL(0xf8c2ca0c,0x1f24d4d5),LL(0xa82a6905,0x753ac024),LL(0xb6761f2a,0x8a0fe4e0),LL(0xb4a03fa9,0xd1d20586),LL(0xcb862d5f,0x2bcb9949),L_(0x00000092), LL(0x50da47cc,0xa285574f),LL(0xb1192295,0x23eb5f8c),LL(0x2436f1d4,0x3f4febd2),LL(0x842f6ae2,0x0f1f266f),LL(0x8091b264,0xee94d349),LL(0xf4561f25,0x6f4a16f1),LL(0x6e303b52,0x80ed7ffc),LL(0x08bbc14e,0x6f957c19),L_(0x000001ac), + LL(0x34993096,0x73d10fa8),LL(0x1b164c81,0xc855b452),LL(0xdd4617a8,0x9be0c270),LL(0x94a57ba1,0xb28b6ed5),LL(0xc1383246,0xc74bc4a3),LL(0x17910967,0x2a2a0e34),LL(0x8aab9202,0x93b8d150),LL(0x122babf8,0x1dffa251),L_(0x00000073), LL(0xc68b787b,0x5fd7acd3),LL(0x906979fc,0xf242bd9e),LL(0xc7fe62d6,0x59507727),LL(0xa94a1beb,0x966af710),LL(0xe7344f5a,0xd397803f),LL(0x5b835cef,0xcd196fb1),LL(0x1ca9e8ac,0xe701eb7e),LL(0x09bdf0a9,0x521354ea),L_(0x000001c2), + LL(0xd206da83,0xc389a398),LL(0x92326c61,0xc80b1e6a),LL(0x0b658149,0x23067f4a),LL(0xd5cf4f64,0xc96735ae),LL(0x1cf4dd22,0x43cb53c3),LL(0x4b478ba1,0xb353c721),LL(0x3b1bbb1e,0x46f2e84d),LL(0x7b5aa79b,0x4a4fa3d6),L_(0x000001d4), LL(0xcf711e2d,0x5ad31f07),LL(0x5402e45d,0x19f0b31e),LL(0x55650578,0x214cdf81),LL(0x3fd50ebe,0x3efd8e06),LL(0xd808dee6,0x0c82d63a),LL(0x39ccec30,0x39dfd0ee),LL(0x5249be7d,0xb6c788e8),LL(0xebe9271a,0x7ffa3cad),L_(0x00000072), +}, +/* digit=44 base_pwr=2^220 */ +{ + LL(0xe3ee97b9,0xfd61361b),LL(0xf3125658,0xd69b66e1),LL(0x96b636a1,0x0c7ac9e9),LL(0xe69b9e47,0xf9bb3617),LL(0x1b1e895b,0x12050a8c),LL(0xfa5a11a5,0xa2492213),LL(0xc2919aff,0x08d55c3b),LL(0xc4be1b10,0x6dcf2c08),L_(0x00000192), LL(0xa93efc6a,0xb4fb57fd),LL(0x6ce4aac2,0x6292f827),LL(0x9277cab1,0xc90518a9),LL(0x144e677d,0x0432d015),LL(0xea4408ab,0x35d9214e),LL(0x49b20eb2,0x2560b8a6),LL(0x48a45d8a,0x37dd269b),LL(0xcecf7d1d,0x71a47616),L_(0x0000009d), + LL(0x189072b9,0xa734d86d),LL(0x475ac257,0x324330b9),LL(0x3a9d12f1,0xcecaa5dc),LL(0x048adf08,0x33641cc3),LL(0xbfcc4fdc,0xe40352a6),LL(0x76f01bad,0x6e93dac8),LL(0x5bd7dfb0,0xc0e1ca53),LL(0xb21d4494,0xb51965b8),L_(0x00000019), LL(0xe7f6998f,0x46f90e9b),LL(0x76cb857a,0x1b517407),LL(0x31907caf,0x38843e17),LL(0x37770400,0xb3c14ab5),LL(0xb47cfe14,0x58c99d12),LL(0xd18daa18,0xd4551590),LL(0x84db2817,0xc4d8f7ef),LL(0x5a8544bb,0x752b595c),L_(0x000000e1), + LL(0x3382876f,0xa8718c9c),LL(0xd016412e,0x3d459798),LL(0xc7059eeb,0x07afd251),LL(0x7f6e9107,0x6ae603f5),LL(0x7c29b336,0x1f1d424b),LL(0x1f08b6d1,0x06f8f459),LL(0xfa0b1884,0x6dfa46f4),LL(0x22a09ce4,0x1ee6193a),L_(0x0000013e), LL(0xcfd96283,0x59ef37aa),LL(0xbc282a7b,0x0490844a),LL(0x16eca5fb,0xfa414af6),LL(0xdd42a4a0,0x748c915e),LL(0x6f1ab810,0xa3d4af5a),LL(0x31ef86bb,0xe768aff3),LL(0xd8ffc35c,0xb6c4e536),LL(0x6e278d36,0x8f61ddd5),L_(0x00000023), + LL(0x1fe994f7,0x108204ec),LL(0x1febacf8,0x0a0c3a40),LL(0xa1fbe66e,0x2570b727),LL(0x343654b7,0x5299d8ae),LL(0x83e0647b,0x158b323a),LL(0x3d86ee0d,0x758534d1),LL(0xe946224b,0xcb77d0eb),LL(0xe9a38321,0xb845ec39),L_(0x000000a9), LL(0xcd07b3e0,0x4421327c),LL(0x13edfc50,0x0c4b9b9c),LL(0xefee69b8,0xce07a452),LL(0x110b9736,0x2779de28),LL(0x810433de,0x19cb506b),LL(0x1468d237,0xd2930983),LL(0xb54615ee,0x895d360f),LL(0xa6a72f30,0x2a939f9f),L_(0x00000082), + LL(0xbad2c784,0x613613a0),LL(0x41479bab,0x64b3ba28),LL(0xae853dfe,0xd7d5f8f3),LL(0xeddc5d69,0x3c023c98),LL(0xc2af1c91,0x5e51c064),LL(0x3e811beb,0x3e28caf2),LL(0x297f73a1,0xb2c63f7a),LL(0xe92c2db1,0x272783c6),L_(0x00000069), LL(0xc2b0e8b8,0x4d9e33e7),LL(0x2930859a,0xf96d2a6e),LL(0x21c82319,0x234b3a37),LL(0x99962855,0xe1e952c7),LL(0x834c3fcf,0xa9fff526),LL(0x2d66290a,0xc1aa8293),LL(0xd0618b6f,0x65d795be),LL(0xd8f51b17,0x7ad3a784),L_(0x000000fa), + LL(0x345605ef,0x6fb99ec5),LL(0x5b03ab3b,0xdf673d70),LL(0xd514df02,0x97926d51),LL(0x68bd9794,0x7f3cad99),LL(0xa7bbd732,0x807b8edf),LL(0x1dee6527,0x9072fc5c),LL(0x6db8f170,0x8a088d03),LL(0xfad430f3,0xf3373c9b),L_(0x0000019d), LL(0x6523f7ac,0x562932b2),LL(0x98ce826d,0x15d43a46),LL(0xd64992b9,0x0e1471fe),LL(0x12cf137e,0x11a1c256),LL(0x19907c68,0x97e5e746),LL(0x5dcff6a5,0xb4d10f45),LL(0xfe503afc,0x2daf8e96),LL(0xbfaf4738,0x51a1e9e2),L_(0x0000005e), + LL(0x52c7ebf5,0x57c899ed),LL(0x309ddc9f,0x000b8805),LL(0xf9ec0561,0x61be65ad),LL(0xdb755990,0xc6ac2e8b),LL(0xf8f392c7,0xc546a9f7),LL(0x709f90fb,0xaa4eb38f),LL(0xb81ee256,0xe3bb73d5),LL(0x920cd9ff,0xe54f7913),L_(0x000000d1), LL(0xecfeb4fd,0x6fe432ea),LL(0x7d99d437,0x42314efd),LL(0xecbf2570,0x0d11bf19),LL(0x1a26524d,0xc070e881),LL(0x80db7170,0x69bb46ac),LL(0xed697625,0x6e7f5dec),LL(0xf5d4f199,0x35c855b1),LL(0x63c6d1bc,0xcfaf131b),L_(0x0000017f), + LL(0xdff5e6d9,0x4c7faf89),LL(0xe289504c,0x7c67c701),LL(0xc21c143d,0x51104808),LL(0x429b8b10,0x8547ea3f),LL(0x643f8b1a,0x442d1597),LL(0x8e30463a,0x1322c20d),LL(0x9700b9ca,0x12313e31),LL(0x53c7c741,0x429e582d),L_(0x0000001d), LL(0xc25e3208,0xa6df174d),LL(0xf30a9c65,0x734a8421),LL(0xcca7bd70,0x12f441c1),LL(0x42e970e9,0xda35c856),LL(0x990f29b0,0x7fbc6108),LL(0x201a5ca8,0x006704f9),LL(0xb4ba5b8a,0xd79c4200),LL(0x5332f15d,0xaa2c5720),L_(0x000001dc), + LL(0x0f472f30,0xa1d7fa9f),LL(0x366d01e5,0x0b24e344),LL(0x05318d76,0x9ed7c092),LL(0xd4cb6907,0xdf8af7fc),LL(0xe97d11bc,0xcad57852),LL(0x4e593cff,0x966648ef),LL(0xeb5229ac,0x0513e9bd),LL(0xc0b6887d,0xc4457c4d),L_(0x00000129), LL(0x1a11be6c,0xe56e3c4a),LL(0x33909c43,0x6b6ecd98),LL(0xab630f3e,0x58c96e3b),LL(0x604359ea,0xa6fb1a48),LL(0x4c23bc41,0x01509a8e),LL(0x735b6fd3,0x55538a0d),LL(0xc1e6f88e,0xca10627f),LL(0x43b7d098,0x7a0bef75),L_(0x000001ab), + LL(0x009970af,0x382d4ce3),LL(0xd57fa9f7,0x39f2057e),LL(0x5a8fe9d0,0x819adb6c),LL(0xf4dd28c4,0x26c97625),LL(0xa0db7a3e,0x6c864ae6),LL(0x8bee872f,0xfd35c90d),LL(0x42a5de0c,0xf6b8c643),LL(0xa0d68766,0x613a35a1),L_(0x00000002), LL(0xcdee18cd,0x7201d2d4),LL(0xeb515a90,0xb3006a97),LL(0x293640a4,0xdf1363d1),LL(0x4e898435,0xc4282ef1),LL(0xf5eec35c,0xf5395189),LL(0xf9d8e87e,0xecf465a2),LL(0x621a8997,0x2d4680db),LL(0xfe197810,0xf2c35d16),L_(0x00000113), + LL(0xbfba8722,0x35c78619),LL(0x976fd128,0x4fea3432),LL(0x9d455cba,0x94c9bc3b),LL(0xb7bb73c4,0x0a3d4425),LL(0x90c2ec3a,0xc1f93e1c),LL(0xa95d87c6,0xe60757b3),LL(0x60d5d399,0xd3d27995),LL(0xce2c84d1,0x81c8808b),L_(0x00000126), LL(0xecc27275,0x32b7091e),LL(0x76361e05,0xe202d497),LL(0x074baca9,0xd7df203d),LL(0x18d9c96d,0x60965442),LL(0x6777a2e9,0xcf542875),LL(0xcea9162e,0xd6cf9057),LL(0x0e08540f,0x118843d2),LL(0x12ce0e32,0x64680618),L_(0x000000bb), + LL(0x4dba9d58,0x3de4c30b),LL(0x1640f2bb,0xe1cd2dab),LL(0x67a49e8e,0x68415129),LL(0xce1a4189,0x5f6655b9),LL(0xf8e55d14,0x58251e9d),LL(0x2a896a11,0x2990bcd1),LL(0x5ee2f029,0xc1b51e11),LL(0xe59128b6,0x7f1b211c),L_(0x000001e4), LL(0xedda1e14,0x0e4c3270),LL(0x75ae7bca,0x0ff0470f),LL(0xe26fb687,0xdde851ff),LL(0x6353acb2,0x661a72f5),LL(0xa014b448,0x6da88895),LL(0x20ab0fca,0x41dc7a9c),LL(0x90420ccc,0x97a7f272),LL(0xf726e07e,0x23c2dfa2),L_(0x000000ae), + LL(0x9697fa66,0x19987e83),LL(0xc4bea825,0xb4d6f37a),LL(0x86bba8ac,0x1e145dc1),LL(0xa3d3860d,0x9d377493),LL(0x538d1baa,0x9ce67c55),LL(0x507bf776,0x0148462a),LL(0x563563c0,0xc2891330),LL(0xba0e7843,0xc76b64b0),L_(0x000000b3), LL(0xac531f86,0xca889e53),LL(0x632d2ce5,0xe7e3d3fe),LL(0xeabe8bf3,0xb7126e82),LL(0xcc6bda06,0xf58c6361),LL(0x69480574,0x6a99c0d0),LL(0x660e6906,0xd1fdf14a),LL(0x465778f0,0xddbb43b6),LL(0xb843d29c,0x575fd92f),L_(0x000001cd), + LL(0x311db7b4,0x2cd4fd82),LL(0x0e9a73c7,0x52c71a96),LL(0x4d3a4557,0x067ccd35),LL(0x3e412b8b,0x7bd77ec2),LL(0x951e31d9,0x4100bac7),LL(0x2ac6482b,0xccccb053),LL(0x6361906b,0x79655211),LL(0x269a7fe0,0x756f8ab0),L_(0x000000fb), LL(0xad9f10f9,0x8ce6630b),LL(0xcbecac0c,0x4a29a7b7),LL(0xf7ba2d3b,0x47e9ba07),LL(0x2e6c073a,0x5aa6a94b),LL(0x73cf6f20,0xa8a42977),LL(0x50805433,0xf9945386),LL(0x74ac62ed,0xc8f04c45),LL(0x14e4baea,0xe37de45f),L_(0x000001c5), + LL(0x860bbc4f,0x73f77d7e),LL(0x6ff96082,0x58430006),LL(0x328b95a9,0x82931188),LL(0x968102c5,0xbb40cb55),LL(0x6514614a,0x370bf205),LL(0xe5de48d5,0xb3d94790),LL(0x73c104b4,0xa5683b4f),LL(0xfb96fb1a,0x36a9677e),L_(0x00000050), LL(0x0c67ecca,0x32bfd837),LL(0x4594f7a0,0x3fb584bb),LL(0x9be3e3c1,0xc86bb1e7),LL(0xef4aecee,0xfa93fc70),LL(0x9de5dfe2,0xbfd3b8de),LL(0xe7670296,0xe1fa5638),LL(0x228cc6d9,0xd230d44e),LL(0xf7797644,0x82d4c20c),L_(0x0000002c), + LL(0xef42daff,0xe19f88e8),LL(0x828b99d9,0xc2beefa3),LL(0x51fa512e,0xd33e3c3c),LL(0xbe14d684,0x34a6c37a),LL(0x5b5936ae,0x89d4bea5),LL(0x2802583c,0xd938e649),LL(0x98da605b,0x1f045420),LL(0x55288cfc,0x659c47e4),L_(0x000000d0), LL(0x046a68ec,0xb9ddaea5),LL(0x22472a49,0x0aefe3e4),LL(0xc2da9569,0x6e21cee3),LL(0x0c7dcef3,0xb14f0abf),LL(0x98c3c9ea,0xb64941e1),LL(0x4819eee0,0x433cfcf6),LL(0xbfe77fa8,0x2f7686dd),LL(0x03c3b28a,0xdbfd2334),L_(0x000001c3), +}, +/* digit=45 base_pwr=2^225 */ +{ + LL(0x522fbbd1,0xc3b7107c),LL(0xe8887082,0xd218cc42),LL(0xcd304c29,0x7a96d44b),LL(0xf1c4d847,0xee7f483f),LL(0x530d4bec,0xc951d19c),LL(0x4d6bf1fd,0x71d2d68d),LL(0xe03d009b,0x4d3bd1df),LL(0xc4553769,0xdb4cb1a2),L_(0x000001c3), LL(0xfa15f331,0x5e86f0e3),LL(0xcaf5fb4c,0x715c3388),LL(0xb2f14ba7,0x81191db4),LL(0x38426103,0xf68a08e3),LL(0xb75d25e0,0xd342059c),LL(0x92f767fc,0x4dec2bd2),LL(0x7a696b41,0x16057d6a),LL(0x8adfb670,0x99c277b1),L_(0x00000159), + LL(0x733860ad,0xd6f16b10),LL(0x088e20bf,0xb30d007e),LL(0x859fdd39,0xd4c40b6e),LL(0xc9196072,0x0a59d2a4),LL(0x3c4f607d,0xaf5b531a),LL(0x5c546c30,0xfdc40588),LL(0xdc1d5df2,0x2971b1ad),LL(0xfb26f4df,0x7cb15104),L_(0x000001b6), LL(0xd17eec91,0x9ff74646),LL(0xe3bee8f3,0x96143e3d),LL(0x560fc63e,0x2e0395d9),LL(0xfd9d7aab,0x099cc808),LL(0x422f153f,0x4e3c3dca),LL(0xefabb0d7,0xed2c2c61),LL(0xd736943a,0x78f87c18),LL(0x96d74e41,0xb76afadf),L_(0x000001d1), + LL(0x6c4ef83b,0x204ae17c),LL(0xab5b4b2b,0x780e0409),LL(0xc4c863d0,0xf73d00d9),LL(0xd243a003,0xd17b97ed),LL(0xca6e6ef7,0xd335ec2e),LL(0x3f246d97,0xd6c07def),LL(0x0e2518ce,0xb8b0595f),LL(0xfdf728fd,0xc1ccb10b),L_(0x000000dc), LL(0xd02f5048,0x869f012d),LL(0x0a767d8e,0xbe6fa9d5),LL(0x1bbb0510,0xdeeebbfe),LL(0x8f4cdab7,0x8332cdf9),LL(0x75d651f0,0x54687821),LL(0xdd0fc83f,0xa428610a),LL(0x965277af,0x38635dc4),LL(0x9dff5c34,0x0961df5b),L_(0x000001fc), + LL(0xeb6cb2dd,0xe4290556),LL(0xea12a072,0x762385fe),LL(0xd082bac9,0x1be16424),LL(0x8697c433,0xf06c7a59),LL(0xa812dd21,0x7de72b68),LL(0x02f90069,0xa2e56525),LL(0x9acaec02,0xa89c7efd),LL(0x4f9120c4,0xdf713a32),L_(0x0000001b), LL(0x1027a34d,0x3367220e),LL(0xc9641453,0x9b048cab),LL(0x29a9fac9,0x0e9e757a),LL(0xcb21c285,0xde9cc170),LL(0xbef96957,0x7d017c03),LL(0x0f3534a3,0xa9c18cbf),LL(0x5a627fd6,0xed78ac58),LL(0xb9c29da8,0xef092aca),L_(0x000000e4), + LL(0x27ba92a3,0x1cb1ec3d),LL(0x669f05f8,0x03d201d9),LL(0x6bb4d70b,0x898f998e),LL(0x96c9a0f0,0xe22c0440),LL(0xfa56577c,0x7b9dee1b),LL(0x00502b66,0x47777bc8),LL(0x483cf935,0xffb71b74),LL(0x0b52d0b2,0x39112d39),L_(0x000000a7), LL(0x6e9174c3,0x75c45aee),LL(0x6b3c757b,0xf1bef81e),LL(0x7e3d1f7a,0x21a74b7e),LL(0x2bfe9dfd,0xb0da2bc4),LL(0x90c8361c,0x0c5783ca),LL(0xf50913fe,0x5dd47036),LL(0x34045f12,0xd0f87837),LL(0x89dc259f,0x560578ab),L_(0x000001bd), + LL(0xdd7b63bd,0x1a521d30),LL(0x8270aad1,0xd7dd0f9e),LL(0x5bd17ae9,0x2341942e),LL(0xc840cb6d,0xdc090118),LL(0x1c49ccd4,0x60e02fa4),LL(0x28d04bf0,0x3570b020),LL(0x1dc79e5a,0xbd1e244b),LL(0x5f5e1042,0xeebf73ef),L_(0x00000024), LL(0xa728463d,0x9d82fe22),LL(0xf2a4d116,0xfbb0437b),LL(0x538ffa89,0xac67b2fd),LL(0x70b0e36d,0xcd72f925),LL(0x3b843326,0x26042e6e),LL(0xd58ef907,0xa54711c2),LL(0x40c00366,0xa62c4885),LL(0x33e0a1f1,0x758c2c5e),L_(0x000001ec), + LL(0x49abc1ac,0xf11f835f),LL(0xd4f6e5fa,0x4a99daab),LL(0xbfa7d98f,0x3f46a163),LL(0xe712c72d,0x869f8036),LL(0x194ee078,0x1283d5ab),LL(0xe9c40094,0xf4b53cf8),LL(0x8eccbfb3,0x865de4fb),LL(0x27d82426,0xc4399d77),L_(0x0000011a), LL(0xc3188564,0xa59eaa03),LL(0xb1a23e95,0x99d72c23),LL(0x48055f02,0xf5f1e1c0),LL(0x04a35336,0x448cee4d),LL(0xcb7b9a8c,0x5862f10a),LL(0xb9974831,0x93b3f7d9),LL(0x93c6f79e,0xb09c629c),LL(0x2a3760fa,0x5d47957f),L_(0x00000001), + LL(0x1bb374f7,0xf060edf1),LL(0x83a0baf3,0x26de7078),LL(0xeb8cb2a3,0xf5a8d631),LL(0xcfa95554,0x6ba14fb5),LL(0x9d950c23,0xc53769e2),LL(0xc3b6e6d4,0x857d43ea),LL(0x00e396b6,0x28cc2c64),LL(0x79b3b40a,0x42ca52de),L_(0x00000101), LL(0x744d95ae,0x4e536d43),LL(0xbb696fec,0x60fe980d),LL(0x86879c73,0x6a564e15),LL(0x6796d473,0x62e7ce9d),LL(0xee75c812,0xd4cced89),LL(0xdd4f732c,0xa3a75fe2),LL(0x4a4d97d2,0x62def6c0),LL(0xfce046cc,0x0bfe781a),L_(0x00000059), + LL(0x46604a6b,0xecebff30),LL(0x5ddecacd,0x66805c77),LL(0xc732fa10,0x62e2a037),LL(0x9f8318aa,0xf535306d),LL(0xcb188ede,0xf3ae9b72),LL(0xd4215242,0x76515ea4),LL(0x19ee9251,0x04b30ff0),LL(0xfdb4add0,0xc2006bc5),L_(0x000001f3), LL(0x3b010d29,0x08f431a5),LL(0x9a41c273,0x4ef9856a),LL(0x29336f92,0x89a37aaf),LL(0xb4f719c9,0xb6ae5b4c),LL(0xcb8b83d7,0x0b93351c),LL(0x579d32e9,0x95ac6c1d),LL(0xed14de1a,0xc9a3144d),LL(0x59d39bac,0x4809d3c0),L_(0x00000192), + LL(0xb4a2a13e,0x46b4f926),LL(0x31967f76,0xedbec7a4),LL(0x9540a500,0x7d4cba0a),LL(0x3664934a,0x44eee05e),LL(0xacaf15ab,0x7e273dff),LL(0x0d40978b,0xb55d54b7),LL(0x7a7b2c2d,0xb2400231),LL(0xcc267a21,0x31dec63e),L_(0x00000134), LL(0x05754bc0,0x9ea998cb),LL(0x5e543e59,0x22a0befc),LL(0x0dee7b14,0x1aa7e66e),LL(0x052bb589,0x51c3c220),LL(0xe7133b54,0x64b84789),LL(0x86fee1bb,0xacd3a9a6),LL(0x6d321f88,0xf3dee610),LL(0x17435c7d,0x64e0790f),L_(0x00000193), + LL(0xfde9e64e,0x824c77d2),LL(0x1d5087de,0x3c424cb9),LL(0xce8e9c97,0xb3b1c7ef),LL(0x65ef8961,0x36bdf7df),LL(0x936db078,0xe1a9152e),LL(0x9ff36717,0xbde711fb),LL(0x1b59074d,0xc2a17a75),LL(0x9070200e,0xa0657ad1),L_(0x0000006b), LL(0xc32bca7d,0xf471b9b4),LL(0x63dca49d,0x23fe450a),LL(0x7aa88a47,0x569744e7),LL(0xbac5c025,0x557519ba),LL(0xc31b16cd,0x90672c5b),LL(0x7609de28,0xeb879e6d),LL(0x06a4f8d7,0xa50f3b9e),LL(0x45a7792d,0x5ef4893d),L_(0x00000138), + LL(0xa91734a3,0xa8c609cd),LL(0x3fc0a251,0xe916d838),LL(0x09bae516,0xdc8442b3),LL(0x70eda9b4,0xdbbd2829),LL(0xe66c09a8,0x44077102),LL(0x46f0d061,0x0c8b9a97),LL(0xe7c4e4e6,0x96dd9b50),LL(0xd93b4696,0xf2abcbd4),L_(0x00000050), LL(0x1b9aac0c,0xdcc67ed7),LL(0x34c98a81,0x1a864f2a),LL(0xd79eca11,0x46960243),LL(0xaf4ad720,0x02db8ea4),LL(0xe8d429d1,0xd9dae172),LL(0x9cac2c01,0xf704ba3a),LL(0x052267c9,0x0ee381b0),LL(0x768c8223,0xf95050b5),L_(0x0000019f), + LL(0x12097ddc,0xdbee340c),LL(0x7596d517,0x7a4d90d8),LL(0xe88332d5,0x27312835),LL(0xadc174b8,0xbf7deda3),LL(0x9fa8589b,0x30a04404),LL(0x50ccb83e,0x9ea9cbf7),LL(0xb73afec5,0x8a699ab5),LL(0x0382ff44,0x682b0e1c),L_(0x000001a1), LL(0xd2f8a7c3,0xa48bc771),LL(0x219fed54,0x20bed549),LL(0xd7a8f53a,0x9f897a30),LL(0xccad31ab,0xf4ab61be),LL(0xe4253812,0xd02eb63b),LL(0x02c0bd98,0xf369795f),LL(0x940fa395,0x6661d80d),LL(0x4fc30a92,0xdb54dfb5),L_(0x00000052), + LL(0xa8b475dc,0x7f4fad25),LL(0x917eadce,0x33c5c41f),LL(0xfac932b3,0x8175e0ba),LL(0x0c6ce437,0xd6df4caa),LL(0x628f6bd2,0xd8cef71f),LL(0xbbd0808b,0x1bf4e3d8),LL(0x82a3b927,0xf2573f56),LL(0x18b7f7d3,0x20a5d5d4),L_(0x0000013b), LL(0x23ee963f,0x6296df1a),LL(0x511300a6,0x48f698fe),LL(0x0a451118,0xe69d9b6b),LL(0xbda90f27,0x6b99560f),LL(0x00ad6b22,0x576639a4),LL(0x70498a28,0xe3fb0685),LL(0x766cc9eb,0x910ed9d6),LL(0xa13d4e5f,0x45079f4a),L_(0x0000016f), + LL(0xfcc87b80,0xb3a0df18),LL(0xcdee46b0,0x0f7b0b8f),LL(0x0dee6d65,0xdc8df7a6),LL(0xca19127a,0x6d034f50),LL(0xd6d74c77,0xcd8b7301),LL(0xd01a93fc,0x7b8e12a8),LL(0x77799926,0xc0b3bdfe),LL(0x157d532e,0x5444b9cf),L_(0x000000d1), LL(0xe1636c3d,0x2776ac60),LL(0xbab425f8,0x947e525b),LL(0x66085567,0x6d095956),LL(0xbcb7adb8,0x4d3075ac),LL(0x99a0d6ce,0xc684b9ba),LL(0x0e134c5c,0x4c65fec0),LL(0x30477674,0x5db48af9),LL(0xf3744581,0xfcc9963e),L_(0x0000004c), + LL(0x3d53729d,0xb51e39da),LL(0x258dcf68,0x1c50cc68),LL(0xb1289a02,0x54112229),LL(0x48928ef6,0xc73b83c7),LL(0xf0df33f4,0x6a0ebdbd),LL(0x5f166393,0x09883324),LL(0xbdfac3bf,0x21bceec6),LL(0x871bcc9c,0x64a15de9),L_(0x000001cb), LL(0xcba002c5,0x7efb2f27),LL(0xce8d66f9,0x6a48e2f0),LL(0xc4f49e10,0x0d05177f),LL(0x04e720d6,0x9e354273),LL(0x746f841b,0xe08c355b),LL(0x7b7b7cd5,0x81226157),LL(0x586eb9a8,0x38ce3838),LL(0x8c03b21c,0x02872c5a),L_(0x000000ec), +}, +/* digit=46 base_pwr=2^230 */ +{ + LL(0x7f032bde,0x8a4b6723),LL(0xa08c6852,0x79a5b3fd),LL(0xc7195714,0xd8054fed),LL(0x7c611c75,0x503c3580),LL(0xe88e5dbf,0xb37b9ea9),LL(0x2b4c4521,0x75f1c942),LL(0x950eb17e,0x79508472),LL(0x238eca42,0x845c91f9),L_(0x0000000f), LL(0xa8c10103,0xa0c1e80e),LL(0x978ae396,0x14b242e1),LL(0x2c0d00f9,0xb47bf7bf),LL(0xe944e43f,0xb416e50a),LL(0x906c3634,0xe7c8d114),LL(0x347f03a3,0x65b00ad0),LL(0xa6eba251,0x53a14e26),LL(0xf521e9dc,0x11eb83f4),L_(0x00000169), + LL(0x9d3bde43,0xcf0b0ab2),LL(0x1ea0d8e5,0x30d499c7),LL(0x4b6204df,0x8f748b9e),LL(0xc4dd3f54,0xba95c754),LL(0xc443876e,0xd7ad6cc9),LL(0x08e10896,0x544e9e2d),LL(0xa9428b4a,0x1a2d3e1e),LL(0xb6fe7189,0x05c7660e),L_(0x000000b9), LL(0x41251790,0xb366e641),LL(0x77106f8f,0x42b36e28),LL(0x2d9d82f5,0x63743c2e),LL(0xaf2b4eca,0xc887146a),LL(0xb28d08cf,0x4ed669e0),LL(0xc7391371,0xda28c885),LL(0x14ee1f56,0x84424da9),LL(0x42d4479d,0x33646227),L_(0x0000015e), + LL(0x23998889,0x12e89265),LL(0x1eeb47d8,0xcbb7e582),LL(0x5b702942,0x0f041cf4),LL(0x64d44f59,0x50d39295),LL(0x5d20f47c,0x0a8b51ba),LL(0x15a076aa,0x79ae768a),LL(0xa2141ba3,0xcf638bb5),LL(0xe3c47d36,0x582394d5),L_(0x0000019a), LL(0x1e394199,0xde6f722f),LL(0x7bcb9faa,0xc544456b),LL(0x7edbbe33,0xc58f127c),LL(0x178e1289,0xcd6856a6),LL(0x29c72942,0x7d0ce889),LL(0xdca59772,0x951589f5),LL(0x6908ef3f,0x4f00ce63),LL(0x110a84b5,0xc1a89443),L_(0x00000018), + LL(0xedea6906,0x1fbd25b5),LL(0xc3abc5bd,0xa3138347),LL(0x78a11d29,0xf2283223),LL(0xadac6d62,0x4af4ece3),LL(0x72b0dc7b,0xf1c75e43),LL(0xa7308e28,0x99139560),LL(0x0ea7127d,0x9cb3c31d),LL(0xee0172da,0xc69386a7),L_(0x00000036), LL(0xcae0b566,0x293be2a2),LL(0x82933139,0xaf8d3077),LL(0x41cdee07,0x4118b415),LL(0xd6d0895f,0xb3a9502a),LL(0x242767b9,0x404e1d44),LL(0x3924f383,0xe7a91a84),LL(0x3c5c40dc,0x0f30db5a),LL(0x2d443e9e,0x38df60b6),L_(0x00000053), + LL(0x06beb164,0x57eacd01),LL(0x8b237781,0x47bc0a58),LL(0x21bf6b08,0x72f947bb),LL(0x5c6b0c6d,0x0c58bea7),LL(0xc78326d2,0x8a6feb8c),LL(0xf3157ca1,0xe147ad97),LL(0x4be255e4,0xa6917b35),LL(0x7006cd50,0x1ceacc56),L_(0x000000eb), LL(0xce77913f,0xc4957a8d),LL(0x5e969282,0x3e3eb59d),LL(0x1216fdc0,0x2f2b0b06),LL(0x51a13162,0x5b88e211),LL(0xff4a6d02,0x6b68e6ae),LL(0xc7c80a3f,0xa7fca940),LL(0x2128145a,0x3a205f85),LL(0x459b75e6,0x48874b7e),L_(0x00000189), + LL(0xb951af1f,0xf1c2d135),LL(0x363e7de4,0xede249f0),LL(0x2374169d,0x92b91c52),LL(0x34ca05e7,0x42f9c460),LL(0xe96d13d2,0xb8dc141b),LL(0x9c04a0eb,0x11349888),LL(0x86f45f6f,0xa3c9d21f),LL(0x34d9dd7d,0x6359abc3),L_(0x000000ae), LL(0xd4c46ca9,0x906d8bdb),LL(0x82617ad1,0x5f4fb81e),LL(0xd70d26c6,0x68367c91),LL(0xe835f648,0x4d712331),LL(0x5dda13b7,0x06ca4385),LL(0x97f662ae,0xcbeb485a),LL(0x211b18b3,0xfe5f6ad4),LL(0x178f31f1,0xbfc76ef2),L_(0x000001e4), + LL(0x1da9f06e,0xab40e045),LL(0xcdbdf2d4,0xbc7f59a3),LL(0x25655817,0x7d6adfae),LL(0x5b05af23,0x3b9e8819),LL(0x1f7f265d,0x4d41011d),LL(0xa02a10b1,0xc4403b75),LL(0x4598d47a,0x21678b80),LL(0x72bcfd2d,0x0a91ddc2),L_(0x0000001a), LL(0x8f87df0f,0x5f9a8c36),LL(0x38d6c310,0x22226823),LL(0x4b065228,0xdf4b4ac7),LL(0x99e6867b,0x1562f4fb),LL(0xc9c2e12f,0xe5bf9e6b),LL(0xef6dd2fe,0xa02f573e),LL(0x99a366b2,0xf15b202c),LL(0x46de40b9,0x8fa6fe0d),L_(0x00000076), + LL(0x070b39a4,0x95d88264),LL(0x5a265bdb,0x771ed60f),LL(0x717063ef,0x0dd813de),LL(0xd974ca00,0x4d348196),LL(0x9d26915a,0x3def1612),LL(0xfcdfb352,0x1ecabcbb),LL(0x7290d698,0x44e08c75),LL(0x5c0c5d24,0x65a063c6),L_(0x0000011c), LL(0xc9b7cd37,0x751941d0),LL(0xa5845e0e,0x90c04cc0),LL(0xc51c09cc,0xf11f3a2b),LL(0x536ce8af,0x3944ca09),LL(0xd614a230,0xe5cb9d10),LL(0x4cf73a00,0x11f36267),LL(0x39b52629,0xe681d436),LL(0x27379f49,0x7634bb09),L_(0x000000a0), + LL(0x83402336,0x3b3162c4),LL(0xf5ba50b2,0x27a53ca3),LL(0x5b2f88a0,0x4d9f7313),LL(0xff015375,0x4100d075),LL(0xefc66c49,0xaa939a6b),LL(0x25334ce7,0x4d836d2e),LL(0x0ff09c49,0x9aa3f59b),LL(0xce13f150,0x6bd297a9),L_(0x000001e2), LL(0x138230ce,0x3c8984bd),LL(0x8be1ce28,0x7a30734a),LL(0x83fbedb8,0x56e66999),LL(0xc37e9e22,0xeb69a4db),LL(0x8b3de542,0x15192947),LL(0x4a7280d3,0x67515315),LL(0x5c05b359,0x0b9a8604),LL(0x5ec92a80,0x973deb80),L_(0x00000068), + LL(0xa6ca6db6,0x1256d99a),LL(0xa13ac401,0x9e017800),LL(0xed3810cb,0x84b7702c),LL(0xb6d9eff1,0xcdc98f94),LL(0xf4a42e06,0xa3cf6c58),LL(0x658f00c3,0x17b79fe0),LL(0x4db4bd0a,0xc0cd6ebd),LL(0x79f4a662,0xb6716eec),L_(0x00000069), LL(0x3c927a1c,0x6f9c5845),LL(0x5d9256ec,0x540f768a),LL(0xe910b5c3,0x4e7e7b8d),LL(0x3c907d51,0xf387c3d5),LL(0xeecfe723,0x213d5d27),LL(0xd78500a0,0xaa244815),LL(0xb117a5b8,0x7049f488),LL(0xfb72f9ed,0x3651c83c),L_(0x000000bb), + LL(0x33e7dea5,0xed918816),LL(0x14555dbb,0x178d8d22),LL(0x4a72d14b,0x1ebffe3d),LL(0xa3a172eb,0x476c82fd),LL(0x4c45f724,0x700c837a),LL(0xd80825d0,0x16888e37),LL(0x104ac32f,0xfaf93105),LL(0xed3ecf8b,0x503fb78e),L_(0x000001b3), LL(0x030d3211,0x3c014136),LL(0x4a1d6c14,0x3ae0a56f),LL(0xa38d42d7,0x765e846a),LL(0xdc241d4a,0x087fcd52),LL(0x0a7a4e2e,0xf088c4c3),LL(0x81043b53,0x0b74dc81),LL(0x030c4d6d,0xf465c63a),LL(0xb63f1e30,0xa7aa0a71),L_(0x0000018f), + LL(0x77408152,0xe2aea3b4),LL(0x22e76e20,0x6eca1ede),LL(0x7e03afde,0x08d1c44e),LL(0x4edf70d6,0x88d58544),LL(0xd8ec390b,0x474d298c),LL(0xc4c3f675,0x9ce21146),LL(0x3274110e,0xb12ae7e7),LL(0xce9cf1f4,0x67e9f4b4),L_(0x000000fa), LL(0x8bf7948c,0x102ef931),LL(0x3efdaff6,0x7a8b94cb),LL(0x59bd3104,0x4b8b1235),LL(0x25afad7c,0x24884827),LL(0xdd939da0,0x43b462ba),LL(0x45cb99ab,0x6a4b9f89),LL(0x0f6b65c6,0xbddb4b1c),LL(0x39e97dd0,0x5a1f7976),L_(0x000001d7), + LL(0x4ee368e9,0xcf01fbe1),LL(0xc5bd929e,0x47c26018),LL(0x2bbeb492,0x050e6a5c),LL(0x04741b6f,0x0f665d42),LL(0x4c0e1ab0,0x15137e0c),LL(0x3ee524ae,0x88980ffb),LL(0x86e225d0,0xa5de3190),LL(0xde9d18c0,0xe6bcf986),L_(0x000000db), LL(0xe5ef1407,0xc1e09545),LL(0xa8ab4958,0x7b416dfc),LL(0x05e9411d,0xb5a5de50),LL(0x916238cc,0x7da6c853),LL(0x0e933ce2,0xcff26d33),LL(0x7fab03f6,0xc2410d1b),LL(0x70a12092,0x130ded07),LL(0xa7ff89e1,0xf34872a0),L_(0x000000ea), + LL(0x7391f663,0x6107960e),LL(0x08dca3fc,0x9f978b8d),LL(0x2367f8bf,0x41ab4ac8),LL(0xbfb304bc,0xc14127da),LL(0xb2161643,0x89ee83de),LL(0x91433f74,0x76ad8bee),LL(0x3abcb595,0xa74d6d8b),LL(0xa8b1935f,0x314e698a),L_(0x0000001e), LL(0xee0dcf96,0xbd5f1508),LL(0x1fda31cb,0x4401991d),LL(0x67b33d1f,0x5dcaee66),LL(0x80ef50d3,0x58f1d026),LL(0xc6160acf,0x00a13e57),LL(0x3be539f3,0x88dd96e9),LL(0xa5a0a5ae,0x97419f0c),LL(0xcf6a8f10,0x44b2ca4c),L_(0x00000018), + LL(0x02cd85a8,0x5c77a891),LL(0x745da1cc,0xa4b913df),LL(0xa1006271,0xec1779e4),LL(0xb3fe3fca,0xcceebf8a),LL(0x7b0d3f86,0x82fd16d4),LL(0xeb20fdbf,0x4e29270b),LL(0x450edccc,0xa783c064),LL(0x2c637dd3,0x7337d5ce),L_(0x000000ee), LL(0x94d66859,0x819b276f),LL(0x71e05cf6,0x45bd1439),LL(0x748c488f,0x280a8add),LL(0xbb099ca8,0xe8e6e69d),LL(0x19429d88,0x4f4c80b6),LL(0x2d2698e8,0x0e4ab44b),LL(0x1eb3fc49,0xcd46fe76),LL(0x02d1a2ca,0x15543eb8),L_(0x00000116), + LL(0xa9d0f944,0x700ec0ac),LL(0x13ee9845,0xe55b42fb),LL(0xe5ad047b,0x45bc6993),LL(0xe8f73a08,0xee41f2ae),LL(0x1ac680ff,0xd3c83204),LL(0x740b12fc,0x5e36bb4a),LL(0x21ebc164,0x35a45c95),LL(0xb8dfcc77,0x92b0fbda),L_(0x000000ef), LL(0xe45e0eb2,0xb5805ed4),LL(0x829d754d,0x6c810584),LL(0x36cc488a,0xc9632468),LL(0x39f60f1e,0xf2ebc30d),LL(0x390502c0,0xde960758),LL(0x6d1feec9,0x63adf462),LL(0xf944bca3,0xdeea2824),LL(0x9c375dbf,0xa887d095),L_(0x0000010b), +}, +/* digit=47 base_pwr=2^235 */ +{ + LL(0xe3366deb,0x2fcf8b0a),LL(0x07415442,0xe5dadc06),LL(0x07a17d21,0xc237ee85),LL(0x83c01e78,0x08f5fd21),LL(0xada49ad1,0x998eec2d),LL(0x3e35f765,0x5a121c30),LL(0x1f207544,0xf5fdddb9),LL(0x537426d8,0x1dd7a92c),L_(0x000000af), LL(0xf6589369,0x9ac31da2),LL(0x22850494,0xd3189e72),LL(0x7646877e,0xe04fe426),LL(0xee0cbac9,0xf8802494),LL(0xd9fdf793,0xa975ea85),LL(0x0c9c6045,0x18cf38fb),LL(0x02b88cb1,0xa12c8778),LL(0x9757e39e,0x8e571e06),L_(0x00000054), + LL(0xd9933844,0xd91f89ee),LL(0xd77e7f7e,0x2cf0f860),LL(0xe9a24986,0x705ade19),LL(0xa4b26963,0x0f929eaf),LL(0xf6f18b5d,0x1a12f7ee),LL(0x0ff1861d,0x4ef59d59),LL(0x6b67d295,0x18bcbd0d),LL(0x70540dd4,0x83b51e51),L_(0x0000016e), LL(0x6f33ef54,0x0b0a67de),LL(0x090cb0d5,0x89c8fd0f),LL(0xb8b3eeba,0x289c0d96),LL(0xa8a26dc5,0xe6dd7431),LL(0xafde18e0,0xf0a660c8),LL(0x2cc76374,0x397bcfa0),LL(0x7654494d,0xd958a15b),LL(0x24476b8f,0x53a314e3),L_(0x0000017d), + LL(0x9d90faa8,0x74ca8553),LL(0x099db6d0,0x79fa68c5),LL(0x4c5dd75b,0x75880817),LL(0xccf37a0a,0x92dc167c),LL(0xf900a103,0xc92d1684),LL(0x8386aa09,0xfba8f79b),LL(0x53d25b65,0xb1822202),LL(0x183fbc1b,0xfccc5a8e),L_(0x000000ce), LL(0xa07c1dfc,0xf2f85858),LL(0x4925f513,0x1b5dd268),LL(0x3efe01ed,0x65fdd3e5),LL(0x47d317de,0x4621cca2),LL(0xdf49fdd8,0x60e7bc31),LL(0x66f9ab90,0xe320caa6),LL(0x2ca76c6f,0x41361bf3),LL(0x602d539d,0x267e9ebe),L_(0x000000f0), + LL(0x3938d7d9,0x925d2d91),LL(0xd471ad5a,0xe6ad36d9),LL(0x18a88bdc,0x599734aa),LL(0xb20b90be,0xaa6c0fb7),LL(0x33c88799,0xceda14b2),LL(0x7dad41f1,0x3bb924ee),LL(0xb7c643d1,0xa63db6fc),LL(0xcfe7d84e,0x46de4666),L_(0x000001a8), LL(0x81e8c45b,0xebe6128e),LL(0x4a9235f4,0x780d1146),LL(0x2db34ea2,0xb3ecdbc0),LL(0x31fad3e8,0xcdbe1207),LL(0x7ab5ebf8,0xdf431809),LL(0xbf1d4990,0x0f8e79eb),LL(0xe93c1583,0xfba03ee7),LL(0xefc40a1c,0xf76de664),L_(0x0000008d), + LL(0xb488c3a9,0xa3f4bcf5),LL(0xf02dfbba,0xf379ed8d),LL(0x7857a0d6,0x6c580cb5),LL(0x7b4a59a6,0x6d6738fc),LL(0x1654de01,0x7c102c44),LL(0x8d6ededc,0xb8a37f11),LL(0xe92baa08,0x6ecce15f),LL(0x1b1b1c96,0xf542a7b0),L_(0x000000af), LL(0x8b1ba942,0xcf9be3be),LL(0x02fd3092,0x22a60f7a),LL(0xd4c8209a,0xdeeb8950),LL(0x8636e6d7,0x29511c76),LL(0x09037c4f,0x419f1652),LL(0x4196d645,0x935dc02e),LL(0xeaef1b05,0xcec8a6ec),LL(0x2f5c9bf8,0xb432cd40),L_(0x000001bc), + LL(0x458a4ebb,0x7f267eeb),LL(0x2a1279f1,0xf11c0e49),LL(0xc76f729a,0x92b8f2a3),LL(0x5437339e,0xe00c8ca3),LL(0x41f96b14,0x1dbcc016),LL(0x449dde57,0x5fa3755e),LL(0x0a0df11e,0x42dc646a),LL(0x5d317707,0x3610d1c6),L_(0x000000f5), LL(0x553c4383,0x76e5b808),LL(0xc99fe831,0x61f75499),LL(0x19eef1a0,0xd4c21a60),LL(0xcc67deb8,0x192fdd7e),LL(0xec37ce33,0x13250ef2),LL(0xf6ed9344,0xac4baff8),LL(0x8d1e9777,0x0d67d5c5),LL(0x7183407f,0xd1b52871),L_(0x0000019f), + LL(0x078b4225,0x64430a84),LL(0x5637bc1f,0xc9bb1131),LL(0x102f522c,0x38d318b3),LL(0xb1dfff39,0x0967edf8),LL(0xc54708cf,0xff7bf052),LL(0xb7dea363,0x5d9deef1),LL(0x44846b8a,0x5e3fdc0e),LL(0x2ccf785a,0xdf1138dc),L_(0x0000010a), LL(0xe715aa67,0x3b1beaf4),LL(0x5076af3c,0xac3bbbf4),LL(0xdda27ca5,0xdc76f92a),LL(0x7c5f4d64,0x67fb8aff),LL(0x98258450,0xfa7eea13),LL(0x183f4c0d,0xc38dcddf),LL(0xc7ca9f82,0x0789054d),LL(0xa0ad28e2,0xf484ac51),L_(0x00000118), + LL(0xb9c90b3e,0x0d9d6152),LL(0x28f43810,0x4ecd5aff),LL(0x0a2146e6,0x7e0c0df1),LL(0x0d19ad5b,0x6775181b),LL(0x151d7c3f,0x43c7a1b8),LL(0x38a3ef9f,0x2dd235c3),LL(0x1fdd8171,0x1f2597ee),LL(0x4734726a,0x083971de),L_(0x000000b3), LL(0x0d9bef58,0xfac04120),LL(0x85ff590d,0x952b767c),LL(0x8b72aaea,0x843ae9c1),LL(0x004265e7,0x307d7542),LL(0x395a8932,0xc2dea503),LL(0xc7f73a8f,0x3399e1ee),LL(0xb86c7eb5,0x2926b2c4),LL(0x0d21cf86,0x47a4f082),L_(0x00000193), + LL(0x41c76045,0x685e8e4b),LL(0xd88c27c3,0x0240c0b1),LL(0xeaa149a2,0x79b4d2d7),LL(0x18bd3d71,0x368319a1),LL(0xdb5d6ca6,0x44a8c42f),LL(0x587ba79d,0x77c41337),LL(0xdb57f6f2,0xea1f7c4f),LL(0x8ae39899,0x77750728),L_(0x00000085), LL(0xc0c436b8,0x22355d19),LL(0xa2946cf3,0x598a6616),LL(0x4d6a2c7a,0x4bb8cec6),LL(0xfe918a85,0x94e93f01),LL(0x833d8970,0x64d5ba4a),LL(0xa972b798,0x05a95b5c),LL(0x636ab756,0xca11412a),LL(0x2b50cab7,0xeca1dc55),L_(0x00000097), + LL(0x4cd7297b,0xcb4f7b38),LL(0x1a4787e2,0xea069acc),LL(0xa5590739,0xa99d3f7a),LL(0x601be73f,0xcf6cca23),LL(0xaa83e6e7,0x3960cc53),LL(0x5920b117,0x20e0b86b),LL(0xa7ac9dbe,0x6b3ef99f),LL(0xf468bb6f,0x60ac6c56),L_(0x00000040), LL(0x9f97eeb6,0xdc626c18),LL(0xb7dd82a1,0x008f892e),LL(0x142f8425,0x705cbf37),LL(0x01a4b241,0x4c8483d1),LL(0x275951c0,0x82075cfd),LL(0xa7dae45a,0xa9d9f282),LL(0x63c98e81,0x30bf2902),LL(0xefc331b1,0xc5f55add),L_(0x00000095), + LL(0x0bea3751,0x4a6e860f),LL(0xaa1272c0,0x85afdbd7),LL(0x53123c17,0x796e466f),LL(0x327374bf,0x19701fb1),LL(0x8c2b076b,0x755659d8),LL(0x8fee24c8,0xc6001497),LL(0x45d95463,0x144b9a21),LL(0x98a62be8,0x9e1d51ac),L_(0x000000e1), LL(0xfa89441a,0xff126e8b),LL(0x87dd796d,0x5e933815),LL(0xad6f752f,0x0f756584),LL(0xa0d6329c,0xc8afd335),LL(0xef95a2f8,0xa429ff1c),LL(0xe9d51af6,0x06c04336),LL(0xbb9ac481,0x0e389129),LL(0x80d2d2dd,0xb75b239e),L_(0x000000d2), + LL(0x36e3c242,0x04ac9a1c),LL(0x9fb461cd,0x2abd08e0),LL(0x7bca251b,0xfa6e8384),LL(0xf5d98ad8,0x7e953f04),LL(0xca3dcc08,0xfd9f6c57),LL(0x679c5992,0xfac8f179),LL(0xc9cf93cd,0x0e29622c),LL(0x050d1a32,0x703c650a),L_(0x00000180), LL(0xd397fdc1,0xc1188e58),LL(0x60a160b5,0x6d729218),LL(0xbe9a3f42,0x311911d3),LL(0x864f8747,0x8a79eb1a),LL(0xe2ff5eb7,0x66b881de),LL(0xef83107b,0x8784fc87),LL(0xd139997a,0xd894e5ee),LL(0x9a80e84f,0x1a2f4197),L_(0x000001aa), + LL(0x82c0cc02,0x6619c168),LL(0xbefe84df,0x7fe74db9),LL(0x7efd2da4,0xc983adbf),LL(0x0bbc28a9,0x6535bd9b),LL(0x169b3680,0xcb39ad32),LL(0xd2112121,0x3f60a9f2),LL(0xa91386ce,0x3a1b138f),LL(0xb1ef230d,0x36ea6d68),L_(0x0000003e), LL(0x9f4b61ea,0x1ef3660b),LL(0x58bee126,0x3a1e13eb),LL(0x84fe2098,0x1044e049),LL(0xd3d31ea9,0x6e975e84),LL(0xbdc58274,0xcbeca3ca),LL(0x4c27aa9f,0xe9036bd4),LL(0xcc717651,0x982f94af),LL(0x20d99920,0x0268a0da),L_(0x000000be), + LL(0x60bd53c9,0x8e317075),LL(0x01712fc5,0x00607c95),LL(0x10e30ced,0x77b023a2),LL(0x53b465e6,0x620a31c6),LL(0xb10af467,0xa8ea62e5),LL(0x48d08ca5,0x31ddd990),LL(0x65af8778,0x5be8899d),LL(0x2b29f5ab,0x21d38a08),L_(0x00000083), LL(0x78c6e380,0xd56bb423),LL(0xba4bc4d5,0x28b7afb8),LL(0xa4ef2e4b,0xc314822c),LL(0xae9b41f7,0x882d9a51),LL(0xf2b327a2,0x1e6c2280),LL(0x7ce965ab,0xa49d1969),LL(0x5708dae4,0xea346d97),LL(0xcff1175e,0x572d2fbc),L_(0x000000e9), + LL(0x6ad30fb7,0x971c75d0),LL(0x3dd39efc,0x11aab543),LL(0xe47dc537,0xf6290b96),LL(0x38ce4cde,0xe7a164cd),LL(0xd9dcb339,0xebee41ba),LL(0xbc0e8d41,0x18ca0dd5),LL(0xd2a8a073,0x92054e4f),LL(0x366d13cf,0x188fdcbb),L_(0x000001e4), LL(0xa68c541a,0x587afc9e),LL(0xf03b4067,0x4ac1fdd7),LL(0x43baf1b7,0x900c0863),LL(0x35bd9902,0xe71367f8),LL(0x3f6d6815,0xfd613341),LL(0xc12cdb6c,0xa8569f79),LL(0x6e58f3c3,0xd9ca8493),LL(0xd420b011,0x6d1ebe67),L_(0x00000179), + LL(0x98392473,0xbd7d4bad),LL(0x4af0bab5,0xade61274),LL(0xcd1e04b0,0xe21c3a53),LL(0x4ec80a7f,0x4b987124),LL(0x28f48386,0x23f6fd14),LL(0x63b180bf,0xe2b4889d),LL(0xacf748b0,0x2996dea6),LL(0xf3a06107,0x7cdec9d5),L_(0x00000060), LL(0x242fc0a6,0xc2a9858e),LL(0x2d38ce4f,0xc709b1fa),LL(0x87521e69,0xf5996fd9),LL(0x4d05bae8,0x131ac99f),LL(0xc3bad75e,0x62578fe3),LL(0xc9d13920,0x3ada2279),LL(0xb6fbac90,0x696da364),LL(0x6c292885,0xd02ce135),L_(0x000001db), +}, +/* digit=48 base_pwr=2^240 */ +{ + LL(0xee1b4c29,0x3d1074e0),LL(0xff10eeb7,0xe087c1a0),LL(0xc77549f2,0x5e2e0837),LL(0x74d6808a,0x48c7156c),LL(0x11f82ce9,0xbc13bf7c),LL(0x72ee287e,0x06f6a514),LL(0x28c4e6f9,0x165038cb),LL(0x320fef0b,0xb6f1c9d9),L_(0x000000d8), LL(0x2cf5a19d,0x57d310ce),LL(0xadd6b6df,0xcd825c08),LL(0xdcd4cb28,0x48bebf85),LL(0x644e1cbe,0xf1d5aa6a),LL(0xbebbd351,0x908ba85c),LL(0x4d8a2aa1,0x518b1bc8),LL(0x343d2d77,0x29b988ed),LL(0x7ea90982,0x940fc8d0),L_(0x0000009f), + LL(0x73a5d8ba,0x8f96ebc1),LL(0x54a67c95,0x984cd0ed),LL(0xef67809a,0x8dd8453d),LL(0xde0abb72,0x4fe5f363),LL(0x4b73609f,0x8e4fc461),LL(0xaab1b83c,0xb989fee4),LL(0x06d2158c,0xfe56f7d6),LL(0x52096597,0x70734a0c),L_(0x00000172), LL(0xa18dccda,0x094ef503),LL(0x32b2f44d,0xebd6d9b7),LL(0x2c29898d,0xe0ef3ff7),LL(0xe5d5ffd4,0x30b99ae2),LL(0xe1c94a38,0x2dd5fca5),LL(0x15b084de,0x6d08e970),LL(0xe94504be,0x90fe0fe0),LL(0xdb79eaed,0xafa2897d),L_(0x0000011e), + LL(0xee4255c9,0x45dd470c),LL(0x907208e4,0x551b38ea),LL(0x1157ba3e,0x1b72b693),LL(0xcf9f94c5,0x83c616c9),LL(0xaf1c59b1,0x2fe84fca),LL(0x7ed67f1d,0xf1bd77c9),LL(0x1d1e1a09,0x51550daa),LL(0xbac2f477,0x58d345e7),L_(0x0000018d), LL(0x854d3f83,0xa5d95b5f),LL(0xa404c99b,0x29f414da),LL(0x5a1bac7d,0x81c9d673),LL(0x56fda469,0x2c1bc499),LL(0x66bdcd65,0xfe505f2a),LL(0x9783eab5,0x92378106),LL(0x6f9996ab,0xa7330e63),LL(0xa6238170,0xfa70e33d),L_(0x00000070), + LL(0x386cf1e1,0xfee86a58),LL(0xf15be04b,0x32b87572),LL(0xe35f663e,0x94b48632),LL(0x165f5c52,0x62fb4267),LL(0x7d3b9413,0x407dadeb),LL(0x189d86c3,0x5689012e),LL(0x63e5f780,0x61c7907d),LL(0xa2b7e335,0xc0dc085f),L_(0x000001c0), LL(0xb4f6916e,0xa9cbe4f2),LL(0x0cfb081d,0x5ac2d2f3),LL(0x11fe0a52,0x32662679),LL(0x8fcbbd46,0x3e344c2d),LL(0xb1ff4122,0xa0a08757),LL(0xa48229e9,0x6f7101c0),LL(0x56b8c92c,0x4af0e804),LL(0xbb9a086c,0xc21360d8),L_(0x00000174), + LL(0x64d30caf,0x5a00ce71),LL(0xb34ef3a4,0xac5c1fed),LL(0x800620f8,0xba6237c0),LL(0x97dc7c79,0xff56f449),LL(0x5563d588,0xe8ab9474),LL(0xb2b00a9a,0xb8d1df21),LL(0x6a286295,0x4cf9b378),LL(0x00426dbb,0x30b70043),L_(0x000000c8), LL(0xf792db10,0x1fa97f98),LL(0x5a4e4b25,0xd3a62f9d),LL(0x61b4d385,0xad987701),LL(0xb90b4cda,0x41727a73),LL(0x84f90823,0xec4abf7e),LL(0xd814c4a3,0xed3df0de),LL(0xc453671b,0x0c8945f2),LL(0xdda88869,0x94087855),L_(0x0000019a), + LL(0x7fb07527,0x0ce5f28e),LL(0x705e5e78,0x3af745ca),LL(0x93ca952d,0x7351cb28),LL(0xfedeccdc,0xc12e9837),LL(0x7e7bfbbf,0xd14b9356),LL(0x47999f34,0xc6295462),LL(0x3f729887,0x5692b0da),LL(0xc96edd28,0x812e383c),L_(0x000001c9), LL(0xd72f1105,0xef1bc941),LL(0x70199860,0x493b99ba),LL(0x2d090c33,0x279e0c37),LL(0xbe1503ff,0x3fbe286b),LL(0x80f6465c,0x06d81c3e),LL(0x0a9257bf,0x2a448d3a),LL(0x402ee72a,0x6a5669fe),LL(0xe592b91c,0x5315497c),L_(0x000000d9), + LL(0xc0b3f5e5,0x93b397c9),LL(0x3318572b,0xa3327857),LL(0x0e667f17,0x7af5bff7),LL(0x2e60b913,0xfb65a96b),LL(0x9e25ef17,0x9188dfee),LL(0xdd117ab2,0x830a2c9e),LL(0x5472d03d,0x063aa4e1),LL(0xd8512ca1,0x9593f42b),L_(0x000000df), LL(0x5ba796ee,0xa143be47),LL(0xbbccf7b7,0x40a8123d),LL(0x80d4a4b1,0x8562ea6a),LL(0xe8424774,0x53e144c7),LL(0xa39d882f,0x49ac8b6f),LL(0xb451fe15,0xffe4b1ea),LL(0xc4538f34,0x0527aede),LL(0x1076cc29,0xbcd8efdc),L_(0x00000183), + LL(0xcdaef57e,0x670c5c5b),LL(0x046d9801,0xf7b5e720),LL(0x0601ff33,0xa07084cc),LL(0x900e6b0a,0x791af83b),LL(0xd2391e07,0x2dd856fd),LL(0x0430654d,0x63408d88),LL(0x82606889,0xc176e8bf),LL(0x2fa4b443,0x18c663e6),L_(0x0000004e), LL(0x41a598d0,0x48d8ae04),LL(0x950e8cec,0x11b8065c),LL(0x1316d53d,0x3a5ccc8c),LL(0xfeecd686,0xed41a668),LL(0xbb41648c,0xf76ba771),LL(0x9a4bb4b6,0xa9b2b49a),LL(0x370974d5,0xf9f130ae),LL(0x200485de,0x4c25f49d),L_(0x000001b8), + LL(0x92cea4ef,0x24ffe291),LL(0x23a38bed,0x41132058),LL(0x4bf85483,0xba359818),LL(0xbdd23ab5,0x3571fdd9),LL(0xd8f1fbf5,0x740422c0),LL(0x46b3d29b,0xdeca158b),LL(0xf7c74726,0x69a6765b),LL(0xe058b8b2,0x3a247a5b),L_(0x000001e5), LL(0x4e26d5da,0x49bfcbe2),LL(0x37b93476,0x3090fa48),LL(0x219565ef,0x30eb0e12),LL(0x996b7d11,0xe2ef23ae),LL(0x935017a2,0xfedd570a),LL(0x59810960,0x510b5963),LL(0xf4feef53,0x4eeb57ef),LL(0xa6aed7bc,0x500fc7dc),L_(0x00000023), + LL(0xd3c23c96,0x502210f9),LL(0x2f6be4ac,0x36ffcfaf),LL(0x97a1a521,0x874cac2d),LL(0xd4b6d8f6,0x4e03da3a),LL(0xebd4b9c9,0x058f53fa),LL(0x05bdde0c,0x54da0035),LL(0xea492dfd,0xf1d437ba),LL(0x083ae453,0x837cc36f),L_(0x00000173), LL(0x653383a8,0x7ab16b91),LL(0x08283456,0xe627feae),LL(0x5b031ea9,0x5ab7febf),LL(0xbc9c1a1e,0xdf744ce8),LL(0x0a851efb,0x944a28d0),LL(0x92d1258a,0xa2c52c0a),LL(0x8db3d01a,0x3272efed),LL(0xc3e55528,0xdd38ae95),L_(0x00000092), + LL(0x20da6312,0x6c8b9288),LL(0x201a8b72,0x55d5044c),LL(0xcd7dd04a,0x5fefdd8a),LL(0xf135da4f,0x326e5c4a),LL(0x93ab679e,0x0ebd3dab),LL(0x9e1c63de,0x3da9430d),LL(0xb6139a96,0x1c3e1a0b),LL(0x0bbe99d7,0x31ad9e61),L_(0x000000a5), LL(0xd0083331,0xd69aa80c),LL(0x312fa8a7,0x10699493),LL(0xb830105a,0x432ef74c),LL(0x427ed742,0x21457fc6),LL(0x32a8f306,0xed7ee077),LL(0xde91b340,0x7b2b2143),LL(0x97655415,0x2c36d1f0),LL(0x3e70f7df,0x8cb68be3),L_(0x00000162), + LL(0xb5a77e18,0x1c2c5d0a),LL(0xbdac0199,0xf48304d4),LL(0x58ea0344,0x7badcb3c),LL(0x61cc3620,0x7fa76693),LL(0xee100174,0xcbbd3041),LL(0xb845bb4b,0x2ab9fe8e),LL(0xbc927037,0x7e87d3ef),LL(0x7c6a4dcf,0x87fd392a),L_(0x0000011a), LL(0xd9fd8527,0xf99d4987),LL(0x3518a503,0x093c711b),LL(0x5ed52438,0x8de87657),LL(0x1283fea5,0xa57d89d4),LL(0x1d760642,0x37377224),LL(0xbf397e80,0x0c073eb0),LL(0x2d948da0,0x19066ff6),LL(0x02fb3665,0x6f15cfd1),L_(0x00000090), + LL(0x33168ab8,0x0703434d),LL(0x36c93ab6,0x08a36b13),LL(0x54182f0b,0x6018b27b),LL(0x1eb09d80,0x52b2eff5),LL(0x5cab8c14,0x54baf54d),LL(0x09c5f439,0x9128c26e),LL(0xa0c91a3e,0x3c462ce0),LL(0xc8a5d523,0x0dcb5998),L_(0x000000bd), LL(0x2a3d44a3,0x168afb3c),LL(0x28b7e59a,0x51e02e1e),LL(0x7db3beb7,0x95d9b53f),LL(0xd954033d,0x4280b408),LL(0xfca4117d,0xd87fffcb),LL(0xa8a2c41d,0xedfe3235),LL(0xa4e146b0,0xc9ab6206),LL(0x23f56e8a,0x6cc8df44),L_(0x000000c5), + LL(0x4d499b60,0xe30728ab),LL(0xeda148e9,0x09a1b090),LL(0xb3c8e802,0x4fed9972),LL(0x6feab8c2,0xbd0bf024),LL(0x684e8ba5,0xb6e56fc0),LL(0xc679d0b7,0x3a2827e6),LL(0x05aad3cf,0x9605e502),LL(0x154eec73,0xc7d72f2b),L_(0x000001e2), LL(0x4a30f9e9,0x04181bc3),LL(0x17c5f246,0x0c346607),LL(0xf9d4abe6,0x9d5267c4),LL(0x53390747,0x76bf72c0),LL(0xe2e74911,0xd4ea29a3),LL(0x25cb2342,0x5c9021dd),LL(0x7e8d6c56,0xac452ce1),LL(0x1fd1d0fd,0x7f5955ad),L_(0x000001df), + LL(0xd68445e9,0x7a984abf),LL(0x46f086dc,0xd299e678),LL(0x3ae95a65,0xb7f240f5),LL(0xadf74e6d,0xdd46db9e),LL(0xc95ae291,0x3dae88ec),LL(0xbc688dc9,0x4474ebcc),LL(0xe0bd1aae,0x3a30df47),LL(0xd32d0317,0x222e18e3),L_(0x00000054), LL(0x13d89ccf,0xda1caa7c),LL(0x3b9e4f98,0x584fa9c3),LL(0xe3bb36fe,0xa33fbb40),LL(0xece5a0db,0x990ebace),LL(0x2d1efc59,0x1c0c2167),LL(0x0ea367aa,0x18eb3285),LL(0xaa6c72bd,0x4a9d7d68),LL(0x69e3bcf6,0xcd396faf),L_(0x0000018f), + LL(0xdfb404ce,0x20f43bc2),LL(0x14169ebb,0x776ec8b3),LL(0x4d9ce016,0x40cc814f),LL(0x64db6f3c,0xde6701fd),LL(0xbc27b375,0x1be16687),LL(0x41b2641f,0xaaf1c8ca),LL(0x5c7ebdd8,0x667e429e),LL(0x9221918a,0x84d2d4cd),L_(0x00000193), LL(0x8bc8b351,0x65966739),LL(0x502a6f1f,0xe772626c),LL(0x612fc65a,0xd1717c45),LL(0x06b47588,0xcd0bd273),LL(0xfd9dbcb0,0xf7b68702),LL(0x5beaed3a,0x60d2a43a),LL(0xd9e3bfb5,0x618c6158),LL(0xeec14b9b,0x1dad1537),L_(0x00000042), +}, +/* digit=49 base_pwr=2^245 */ +{ + LL(0xbf8b6eac,0x7b9e7889),LL(0xe957ebe2,0x3ae1f292),LL(0x2bd715ca,0x6ae08fea),LL(0x3b2ea475,0xcab67aaf),LL(0xf247c9f5,0xd47caa37),LL(0x53af0925,0x409f1b89),LL(0x57e7bd1f,0x4ee5b8e5),LL(0x0b979eb1,0x0e289030),L_(0x000000bb), LL(0x1ea9467a,0x1d78d6b7),LL(0x3dfcb1d0,0x595db0c6),LL(0x8e6aaf05,0x9217ec90),LL(0x45106ff4,0xab12df36),LL(0x8489adb8,0xc4207aff),LL(0x257b835b,0xabbbb85d),LL(0x706e08a1,0x71ad10a3),LL(0x6d2a5b7d,0xe224792f),L_(0x0000016a), + LL(0x50fd71f2,0x8295b8b6),LL(0x65b39aaa,0x0690f149),LL(0x1270c951,0x763e6fce),LL(0xbafb3a9f,0x3f839143),LL(0x0dc990be,0x3866c189),LL(0xa0d6a0e7,0x087b74c8),LL(0xb520d476,0xd8910a14),LL(0xbd81006e,0x16e6fb91),L_(0x0000012b), LL(0x63f73546,0x9757756a),LL(0xbe4b13d0,0xc54fba1e),LL(0x3f1884ce,0x4519ff97),LL(0x68950392,0xfb9e4f42),LL(0x2d309b59,0xf2ce5e20),LL(0x004b85f0,0x35d898e0),LL(0x05c20b17,0xd4f54e0b),LL(0x34add1fb,0x178e2a7f),L_(0x000001cb), + LL(0x2d50f2ad,0x9fa52c89),LL(0x8fc48ea8,0x6a680dfd),LL(0x4b09bae8,0x78d67917),LL(0x7cea1e12,0x0f37ae3b),LL(0x8383d337,0x9f51107a),LL(0x157913dc,0xbbd05c8e),LL(0xc347e479,0xe7f7f024),LL(0xc27dfb63,0xa32c2410),L_(0x000000c4), LL(0x275b47a6,0xa0c5983d),LL(0xbc8a42a9,0x31f03f3e),LL(0xdb0533ac,0x07b4440f),LL(0x1b5cb9b7,0x522041e9),LL(0x18816d64,0xa0763672),LL(0x78c44489,0xa7d823be),LL(0x0289668f,0xa033e066),LL(0x14b7bda9,0x1bf9880e),L_(0x00000004), + LL(0xb8d3d78f,0x992b024d),LL(0xcc47fd44,0x301e6aa5),LL(0x4ca3c2ae,0xa239d460),LL(0xb59f6635,0x72a93968),LL(0x93da741e,0x6f3e7cb4),LL(0xe451c847,0x958457a0),LL(0x0539a4ae,0x0ccc6f49),LL(0x70df123a,0x4b36ee4a),L_(0x0000013e), LL(0xaec0226e,0xa5bf5964),LL(0xd54e934e,0xa4f9d8d0),LL(0x838881f3,0x5759057f),LL(0xd231904c,0xf74d21e3),LL(0x65fa2854,0x09110e09),LL(0x3e3fbb9d,0x73f82547),LL(0x66595687,0xc3213d46),LL(0x4ee05953,0xc6c9fbf7),L_(0x000001e0), + LL(0x978a2d88,0xf19f1768),LL(0xfbd4f466,0xccc78e3b),LL(0x4ab17eab,0xe0f582bd),LL(0x42edf70d,0x32c21454),LL(0xe1c56694,0x7f57c601),LL(0x01c830d3,0xe9eae160),LL(0xe56900b5,0xca26d56c),LL(0x36688674,0xb2fb4c7c),L_(0x0000007e), LL(0x369579ff,0x53c6182e),LL(0x47ff90cc,0xaaf18b16),LL(0x8fc84257,0x96b0582e),LL(0x9e3a6661,0x4532767f),LL(0x0d14fb71,0x29d6ef11),LL(0xdc2f950b,0x54eb6cc6),LL(0x85acbd0a,0x525b30dd),LL(0x5c05fb17,0x67dd5268),L_(0x00000175), + LL(0x66c24804,0xaabdb0b2),LL(0x1475f80e,0x6b7bb07c),LL(0xd90a1e1c,0x92ecf09d),LL(0xc193105d,0x5feeb4d3),LL(0x322cd2b8,0xd3b68b08),LL(0xb3acd3e0,0xb0ed276b),LL(0x50511672,0x512d83e9),LL(0x5830b5d3,0xda968b0a),L_(0x000001db), LL(0x1320700b,0xa9aed6cf),LL(0xadb30375,0xb42997c1),LL(0xc6687e52,0x1d88f275),LL(0x5c1d5e8e,0x5d9e895e),LL(0xdbf775d5,0x1f149b28),LL(0xc29aed12,0xe2724e7a),LL(0x220a70ba,0x7e781bbf),LL(0xcf9cd146,0xfb0950fc),L_(0x00000166), + LL(0x8275d24f,0x26492a48),LL(0x8f120fe7,0xb833386c),LL(0xbca86762,0x7d77fbcb),LL(0x2f67d175,0x5165ed7e),LL(0xf29932da,0x40520604),LL(0x607db461,0x88627d90),LL(0x74dd9734,0x9d6e8589),LL(0xff8795e0,0x0898a1bd),L_(0x00000157), LL(0xa3d097c4,0xf0c19be8),LL(0x3e449e91,0xc086fd4a),LL(0xce081f35,0x60f6bfc7),LL(0x6b980172,0xf116eb17),LL(0x438fccb9,0xb036eed0),LL(0x3b9d80a6,0x355bcf69),LL(0x17f28db4,0x1d897ded),LL(0x5b488d87,0xb2564e1a),L_(0x000000be), + LL(0x823eac8a,0x6223dd4a),LL(0x84db5faf,0x0b0f5e29),LL(0xd39d495d,0x0a14e52d),LL(0x723841ce,0x365ed8de),LL(0xb4ef0fcb,0xdbcb1fe9),LL(0x3aa7d5d2,0xb19047ee),LL(0x2d33c7e0,0xe2978f53),LL(0x2ad3b9dd,0x9dbc97d1),L_(0x00000132), LL(0x6c00ac28,0x02d16555),LL(0xb3172c7a,0x8804b57f),LL(0xbeeb32c4,0xad774958),LL(0x59e99dad,0x34b2bc96),LL(0x90ab3c79,0x33fd281f),LL(0xe477effe,0xfaa713ab),LL(0x78b329a3,0xf3df2353),LL(0xac36cbb9,0x62e824d0),L_(0x00000145), + LL(0x54e642eb,0xd8f323fb),LL(0x81a85944,0x3dd3e0be),LL(0x51a21fab,0xd871d4d4),LL(0xb561f31e,0xb4ce4cde),LL(0x4449b15f,0xce67b526),LL(0x64493f22,0x82ddd4ad),LL(0x546ec9b8,0x0adc07a9),LL(0x4dba63e9,0x82628c7e),L_(0x00000128), LL(0xe1f0172a,0x5e900de4),LL(0x358c4d59,0x8391c4fa),LL(0xd7be91ac,0x82f89ceb),LL(0xac49480b,0x0dcd6532),LL(0x48237726,0xd2dee6e8),LL(0xaeda17a0,0x13850532),LL(0x2d6729ba,0x201426a9),LL(0xf52c6ebc,0x188c6ec6),L_(0x000000ae), + LL(0xc2fccc93,0x9b37123b),LL(0x274b93dc,0x0d11cfe3),LL(0x01caef2f,0xbe8ef001),LL(0xef0d218a,0x2810cd03),LL(0xace6e761,0x5c17a13d),LL(0x65a61b64,0xcfaaba81),LL(0x669bf078,0xd429ba49),LL(0xd9a6abec,0x0f71e96f),L_(0x0000016c), LL(0xee363303,0x592f7894),LL(0xe42d0a9c,0xfcc98aed),LL(0x3730d520,0xbb02c8f7),LL(0xe10a8edd,0x4886062f),LL(0x57fa6238,0xdab28f83),LL(0x308d0fc8,0x13c4d161),LL(0x8db6b346,0xf5b7f11e),LL(0x70d7617e,0xd7fc6e4f),L_(0x000000b4), + LL(0x0e9d00ce,0xe3c773fc),LL(0x30cb34f5,0x854f6660),LL(0xd3093680,0x5abdfb30),LL(0x7939c865,0x3d739567),LL(0xa46ffe9f,0x3a7253f9),LL(0x8fd8f096,0x151f1baa),LL(0xa1443d09,0x9a29f4f0),LL(0xb2ed7af8,0x0ad8104e),L_(0x000001fe), LL(0x58f41327,0x76fc8041),LL(0x0868fcae,0xf81d0998),LL(0xdba2642e,0x7f6f6367),LL(0x57d4f243,0x6a189847),LL(0x92fa4eaf,0xc022a76b),LL(0x4e736ea2,0xdd251f4b),LL(0x06ebdd88,0x39b7f55b),LL(0xb9f83aec,0xf34e682b),L_(0x00000021), + LL(0xfb89a61a,0xba33a960),LL(0x502d83da,0x377dd454),LL(0x04a7d732,0x60c9b2c8),LL(0xfa223630,0xe06fc03b),LL(0x7e497e85,0x145bb405),LL(0x456567db,0x39898314),LL(0x2d3a76ad,0xeb33a535),LL(0xd36b4686,0xebbad130),L_(0x00000079), LL(0x519e8255,0x8a5778a7),LL(0x10340c30,0xf7b160b7),LL(0x1f4c9e0a,0x729201ef),LL(0x09fe7ed4,0xe28d29ca),LL(0x57eb2d3f,0x3e4bfbfc),LL(0xfe99fffb,0xe7397e68),LL(0x62215a66,0x18c1dc25),LL(0x16278a5c,0x1045ab03),L_(0x000000ca), + LL(0xd14c0d2f,0xb45b8788),LL(0xc3b9c902,0xe8e7e9b5),LL(0xe1deb5a2,0x6f0ccb02),LL(0xcc118df4,0x931e8bf3),LL(0x44d4d935,0x8d49c80c),LL(0xae880f5f,0x294917c6),LL(0xdaf4ba46,0xf7ee5bdb),LL(0xe09285b3,0x11c3f0c5),L_(0x000000fe), LL(0xff95a4d2,0x6f1e6d94),LL(0x150c77e6,0xc23026fa),LL(0x8e7cd3f5,0x567d2494),LL(0xfba4ae31,0x380b42d3),LL(0x7135a34e,0xdf2c5139),LL(0xf390e611,0xfd3ff544),LL(0x1f3b21ae,0x4254bcbc),LL(0x67fca024,0x68f4c9e3),L_(0x0000003b), + LL(0x1f4a16c0,0x68ed5662),LL(0x0fa1e08c,0x31f2c30a),LL(0xb65f32e1,0x2d06bb12),LL(0xfec7dae7,0x95c78209),LL(0xb2e0fbb5,0x7d228e9d),LL(0x84a73eeb,0x0a7cd176),LL(0x4968f085,0xb3a9c57a),LL(0xf1034b1b,0x12dc73c6),L_(0x0000001d), LL(0xc3858dc2,0xd74b5636),LL(0x90895a11,0x8864d82d),LL(0x8f322a4c,0x1db6f8b1),LL(0xef6cb619,0xdc8f7532),LL(0x95fe8c47,0x6bb6ba07),LL(0xb376849e,0x6839da29),LL(0x55ee699f,0x9e4c3d09),LL(0x013177ca,0x06ac03b0),L_(0x00000198), + LL(0x1730ff3d,0x68b8e4b1),LL(0x556e1c75,0xb2cfde0a),LL(0xbf8d1b12,0x77dcfbef),LL(0xbfb5199d,0x8803a28c),LL(0xb3b33660,0x9ab9ee66),LL(0xffcfec92,0x75479f96),LL(0x66780a53,0x9e3f38c0),LL(0xe21a1107,0x8d2c8147),L_(0x000001a1), LL(0xec7cd093,0xf14b8da9),LL(0xf175c60b,0xc9c58bd9),LL(0x94928d32,0xf4158167),LL(0xa1977e7b,0x3720a8c9),LL(0x50e84c76,0xb29acf00),LL(0xa73b8ebd,0xdf12fc49),LL(0x8d8a4296,0xf75a8ea8),LL(0x1abde921,0xaa9b61ff),L_(0x000001a6), + LL(0xeb196620,0xd278de5a),LL(0x458b071b,0x5b584d1f),LL(0x0313ed9b,0x5f476cd6),LL(0xc54ce5fe,0x007678c1),LL(0x6d145a77,0xe258964a),LL(0x0c62f8fe,0xfa68420c),LL(0x595f7056,0x63621f28),LL(0x2d891192,0x5d8cf9f8),L_(0x0000019c), LL(0xe6cb7a7c,0x130243d2),LL(0x8d5a87bb,0xbcf908d7),LL(0xc4e517b3,0xd0ef67bc),LL(0xba70f65d,0x30e5dd35),LL(0x41af2fd9,0xaeb9fb07),LL(0x448314ee,0xf71f75eb),LL(0x3702fdd3,0xbac48350),LL(0x1d2c0a91,0x1ebd11ec),L_(0x000000e9), +}, +/* digit=50 base_pwr=2^250 */ +{ + LL(0x8057734b,0xf4bf04df),LL(0x31cb2cfc,0x0c3adb10),LL(0x13e25b6e,0x3dd2ab40),LL(0xff8fa4cf,0x758e2edf),LL(0xfeffd307,0xf31ad907),LL(0x4baf111b,0xeba1b456),LL(0x3c4f6a12,0x13a81607),LL(0x94fad755,0xe7fc43bd),L_(0x0000014d), LL(0x593810b9,0xb8d44eee),LL(0x369ffff5,0x5334df1e),LL(0x64f19da6,0x5c2b9ceb),LL(0x01321d0f,0xc5ca3390),LL(0x02b87e91,0x45689acf),LL(0x3a49c8b5,0x049dbf7c),LL(0x93f7ed7c,0x8d277840),LL(0x73a0a1d5,0x726f20ba),L_(0x000001d7), + LL(0x57fe6e74,0xe3d95d4b),LL(0xe45eed99,0x8fe19237),LL(0xddc0cb97,0x7eb46e14),LL(0x4df73f68,0x57bdaf6e),LL(0x8670ac6f,0x847741a1),LL(0xa46fbe2b,0x02454925),LL(0x82f9632d,0xc15a10d2),LL(0xaf2e144f,0xc55aed10),L_(0x00000015), LL(0x59c9bac7,0xc44dce06),LL(0xa506cebb,0x03aaab25),LL(0x48b6559b,0x933863a2),LL(0xc348048a,0xd37a9de4),LL(0x26cd5e20,0xc20f4402),LL(0xee95db69,0xff1c74e9),LL(0x2f425e1e,0xd820bb88),LL(0x1933a6f8,0x6f95cad2),L_(0x000000f0), + LL(0x909d4b32,0x6d40379a),LL(0x87db0c0b,0x3e1edc80),LL(0x4c5fed50,0x8c80df24),LL(0x0d788315,0x5caf06ac),LL(0x12556a93,0x95b47183),LL(0x76d86da1,0xc5714cff),LL(0xcec43480,0xc2f30fbd),LL(0x4e4b2ab3,0xc0cb91c6),L_(0x000001b8), LL(0x122dfa3e,0xcbc11bf0),LL(0x75b252f6,0x87c8dce3),LL(0x501537b7,0x84253d3b),LL(0x6a40795c,0xa38fd372),LL(0xb45a08cc,0x22234e1d),LL(0x09918d4e,0x76319208),LL(0xea70d97b,0x3ef6521d),LL(0x0bdcb67c,0xae2d874b),L_(0x0000018d), + LL(0x61d5097c,0x8fe4e5c5),LL(0x4aedde8b,0xc0d36b58),LL(0xaba27830,0xc0f869dc),LL(0x32bc7d59,0x14a35cbc),LL(0x22d71ab7,0x04bed4bc),LL(0x00680d9e,0xdf25061f),LL(0x3bf0836a,0xb3d768c3),LL(0x3b0d7fed,0x616b984e),L_(0x0000015c), LL(0x16b36592,0x3c8c5d5c),LL(0xbd923c28,0x9a32e9b2),LL(0xfbaf0321,0xf0d8e95a),LL(0xed15bdb0,0xd3039b5f),LL(0xd8942727,0xcc59ce26),LL(0xa1dec9a4,0xf0b3676f),LL(0x21992696,0xb8c7cfb1),LL(0x27260c98,0xc1c97929),L_(0x00000141), + LL(0x06497092,0x97538cbd),LL(0xee612332,0xd25447d7),LL(0xa1040800,0x9c9bdc12),LL(0x06fb815b,0x191fed4b),LL(0xae49fbdb,0xd9407747),LL(0x3d19e592,0x9715df76),LL(0x4613ac78,0x1e9e20a7),LL(0x6c932530,0xe2bfff7a),L_(0x000000fd), LL(0xe1fd9066,0x6bfd0423),LL(0x13390bb3,0xb87dda88),LL(0x40017ffb,0x979fe6b8),LL(0x635bb57d,0xb6fc9a61),LL(0x8ce87e55,0x535b6b63),LL(0xea1ec56c,0xf567cddc),LL(0x06b927fc,0x72c516ee),LL(0x6fb0868c,0x2ebdac5a),L_(0x000001ff), + LL(0x0de7eaeb,0x76e19265),LL(0x53bc630e,0x506faf0e),LL(0x88127211,0x64c166ff),LL(0xad3fc9e2,0x6308dc18),LL(0xb271bc9b,0xec631a3e),LL(0x23be699b,0x2e23525b),LL(0xbfded0b1,0x2391574a),LL(0x69f0d2b6,0xa8ede972),L_(0x0000007b), LL(0x5c84ab62,0x194cc299),LL(0xf244f4b2,0x911e4585),LL(0x7871cfc3,0x52af7b51),LL(0x331dbf96,0xd41147d5),LL(0x7a399291,0x48e46193),LL(0xb0e20d54,0xd985a24f),LL(0x98e92da0,0x7266525c),LL(0xe9b74352,0xe84bc9e9),L_(0x00000043), + LL(0x7a9ef311,0xe9d37b18),LL(0x456032b1,0x30dc5e77),LL(0xd6168724,0x47f55c35),LL(0x18a17037,0x154ea414),LL(0x86d54c7d,0xe14c43c8),LL(0x8d092542,0x78f9b9e8),LL(0x986d7498,0x98519065),LL(0xcc71fd0e,0x4d22c2b8),L_(0x00000060), LL(0x41eec301,0x91d1b267),LL(0x4de89064,0x601dee13),LL(0x91ac8ed6,0xd375837e),LL(0x8587c0dc,0xa03d56de),LL(0xd2c2524e,0xa4331dd9),LL(0xf36ec517,0x0f8bb8e6),LL(0xb100599c,0x0dc65f7a),LL(0x8ada0049,0xa298259e),L_(0x000000f3), + LL(0xc52854f8,0x1b2f821d),LL(0x01e839c8,0xcb7e709f),LL(0xf9520b0a,0xc4857ab9),LL(0x3a85ef6e,0x09f9eda6),LL(0x51f47572,0x96daca66),LL(0x6e717337,0xccecd697),LL(0x40b37bfa,0x29f4ce02),LL(0xcbb44372,0x0fe8a0ff),L_(0x00000191), LL(0x2adce4eb,0xa3781970),LL(0xcf2ed75e,0xf948f559),LL(0x524a7d80,0x3e1fceef),LL(0x058a1573,0x25fd5510),LL(0x5865318e,0x14c29ed3),LL(0xcb7d9fa1,0xcaa64e51),LL(0xf171b487,0x25580546),LL(0x006163b8,0xde740000),L_(0x00000086), + LL(0x09489108,0x87d0f1ad),LL(0x4cf7363c,0xd4fe9fd3),LL(0x9bd13abd,0x4b7d7e77),LL(0x66face9e,0x42746d44),LL(0x0edf9d57,0xd5f51826),LL(0x888b45bf,0x37b7e3fa),LL(0xde49e8c9,0x0262004d),LL(0x8fd87627,0xdc4da423),L_(0x0000008a), LL(0x5a095bda,0xce31cec6),LL(0x2990a670,0x1be9607f),LL(0xf2081d18,0x8855d0c8),LL(0x11fb1c34,0xc4c2574d),LL(0xf1b8ff1c,0x3e444ec2),LL(0x4404e3fc,0x2db84189),LL(0xb7726488,0x0dd78e74),LL(0x7de996b1,0x7da11b57),L_(0x0000001c), + LL(0xedc7667f,0x6a5bd2b7),LL(0x343d29b0,0xd33329b0),LL(0x82fbc88b,0xd3fc973d),LL(0xe1e7bcdd,0x111c0001),LL(0x1c56ee4a,0x0cb45e7f),LL(0x65818c84,0xaccf98e4),LL(0x69029f68,0x6bbf8831),LL(0x53ac7e98,0xe2fa2c45),L_(0x00000060), LL(0x05a0028e,0xb7950225),LL(0x0e5094fb,0xf11a656a),LL(0x3eed5459,0xd3afccdd),LL(0xe6e4111f,0xc0d31cdb),LL(0x822775ae,0xfb39d140),LL(0x04034f9a,0x5954dd7b),LL(0x8adace51,0xc58c7b83),LL(0xeef24d4c,0xe9d767e5),L_(0x0000004d), + LL(0x0d2c0dba,0x7f21ed73),LL(0xfb8a9c16,0x300cbdfb),LL(0x12e137b8,0x22e8279d),LL(0xefc00fd0,0x173a4228),LL(0xe30fee24,0xaf4fb8a2),LL(0xaa67fa02,0x6171abf8),LL(0xda82a49d,0x418d47f2),LL(0x3ea61949,0x572fdfa4),L_(0x00000170), LL(0x96484020,0xbfe14768),LL(0xceb46b56,0x36fbf6b3),LL(0x2855bf4d,0xd4e1ce80),LL(0xeeceaddf,0x5130ec7d),LL(0xeb1ca189,0x57123316),LL(0xabed8057,0xb7e8b4c8),LL(0xcb8de9f3,0xa878fb40),LL(0x81b143a5,0x2fa96496),L_(0x0000008b), + LL(0xff728615,0x22b150de),LL(0x9a87c082,0xad76f636),LL(0x222ff210,0xf2177234),LL(0x874b4d66,0xfb6d673f),LL(0x7a63aa6f,0x559b847a),LL(0x1fb601b8,0x7f528818),LL(0x1d5a56e1,0x2dceae56),LL(0x159cad3b,0x64799ea5),L_(0x00000170), LL(0x78fbf962,0xc6717776),LL(0xfb840953,0xe6943ee3),LL(0x6c82ee1f,0x45986586),LL(0xa8804763,0xb2c01a1d),LL(0xd2e62027,0x81dd9ed1),LL(0x7ac9ecf9,0xa86a93b5),LL(0xea3ed52c,0xab42d43a),LL(0x9783c732,0x4badd572),L_(0x00000127), + LL(0x251ac5fc,0x4e7db852),LL(0x1ee36133,0x5d2ce89b),LL(0x7925fde7,0x344442cb),LL(0x7be13983,0x933fd989),LL(0xf818aa84,0x868cf674),LL(0xcf763eff,0x970119fa),LL(0x1161eea1,0x91b1cf2b),LL(0xf803b198,0x47f45bbc),L_(0x000001d3), LL(0xb92b375d,0x2871ba24),LL(0x03c6e820,0x946bbdda),LL(0x1e7f5b10,0x6d786c0b),LL(0x901e63e0,0xf905444d),LL(0x61291f2a,0xa07d991d),LL(0x26f8514d,0x5dd4a768),LL(0x8caa8bed,0xba23453d),LL(0x625c627a,0x55ae73dc),L_(0x00000114), + LL(0xb896a822,0x7616ee6c),LL(0x76716c8f,0x9e16dd77),LL(0xd9dc6964,0x424e43ef),LL(0x4f1ab6f7,0x3307372b),LL(0x853acdd0,0xb131b10b),LL(0xd0481561,0x6c779030),LL(0x8833d896,0xb43c81fb),LL(0xf49c69e7,0x013b71e6),L_(0x000000d9), LL(0x89994c4d,0xf6938c6c),LL(0xcb8f8364,0x7d7772d3),LL(0xb55fa4df,0x5c5bb6fc),LL(0xe309036f,0x87518233),LL(0x00458dd9,0x5ae0cb46),LL(0xab6a628e,0x80a93940),LL(0xd00141d6,0xe42dc460),LL(0xe62c337f,0xf594561c),L_(0x000001d5), + LL(0x89aca927,0xe8ff16d6),LL(0xd0f00eb5,0x060c9ece),LL(0x76dc2763,0x8d24bcf1),LL(0xe04a4e63,0x8049d5a2),LL(0x1f378724,0xad86dce1),LL(0xee568d6b,0xbd4ecf75),LL(0x064ed8ea,0x23b4afb4),LL(0x3066bb9b,0xe8ba5019),L_(0x00000062), LL(0xb55d56a1,0x16a5e07c),LL(0x0317cfe4,0xb05c4eb0),LL(0xe263fd3a,0x87619f5b),LL(0xe43b9d32,0x04548fa9),LL(0x5fe60636,0x1e3bb4ee),LL(0x177080d6,0x80dd88dd),LL(0x6b920ffa,0x50a4adb9),LL(0x6cf839eb,0x579a402c),L_(0x00000026), + LL(0xc3463c36,0xd3860f86),LL(0x51b92975,0xd751ffa4),LL(0xe9c89ace,0x0fff3b8f),LL(0x22e82df0,0x44ceed1d),LL(0x9ef4bd4d,0x322e7d38),LL(0xec43e5b6,0x5dafe91d),LL(0x3ac6cd72,0x385f22e0),LL(0xc23c7139,0xecc87ca1),L_(0x000001ee), LL(0x21e0fc81,0xd6515802),LL(0xfbf97dbf,0x72372941),LL(0x689ac9e8,0x4611974e),LL(0xce7740f8,0xe04ba0c6),LL(0x7a8f9746,0x7419caa0),LL(0x05b0cdaf,0x30755659),LL(0xcd257003,0x5af7e403),LL(0x8e3b2c01,0x7d54b47d),L_(0x00000128), +}, +/* digit=51 base_pwr=2^255 */ +{ + LL(0xe0e5a47f,0x06aeb844),LL(0x171b7252,0x2f67f278),LL(0x3ef95a8f,0x411d7c3d),LL(0x1341fdfb,0xbc9db5d5),LL(0x9c831f2c,0x64cd3d49),LL(0x5fa0db40,0xb8bb90a7),LL(0x2c8d72cf,0x050fdef7),LL(0x9770a986,0x584d26e8),L_(0x0000012a), LL(0xd8cc5f72,0x8a357b6c),LL(0x75fe114c,0xe1fc26b3),LL(0xaa2296d0,0xe2fe623c),LL(0xe037cba1,0xca73315c),LL(0x36843eb8,0xb7e86db2),LL(0xb5b70ddf,0x4b155e04),LL(0x20198f9d,0x06921394),LL(0x51535cfa,0xaa06d437),L_(0x000001a2), + LL(0x3bc9f5b9,0x89cc4566),LL(0x68f57feb,0xa2543b28),LL(0x4bd3cbd6,0x0bf63c0e),LL(0x66da5e56,0x648f4a56),LL(0xb7d9cc0e,0x7591427c),LL(0xab848b1a,0xe85c5977),LL(0xf4656829,0x4025667a),LL(0xcdae8f7a,0xab876527),L_(0x000001b4), LL(0x40ffbcdc,0x204ed818),LL(0x30db96c4,0x1b3e5e48),LL(0x26c352dd,0x497308c9),LL(0x54703369,0x3370174e),LL(0xa9534502,0x7c6d8497),LL(0xae86058c,0xae7aecbf),LL(0xa32e4cdc,0x67daf0b8),LL(0x3a4e9eb5,0xaf8dd7df),L_(0x0000005e), + LL(0x715a5a95,0x5b9e36c3),LL(0xe50c2a6e,0x316c41f6),LL(0x6af25999,0xe48ac795),LL(0x813a1e7e,0x65d44dd3),LL(0x7fc2f7f1,0x4d3b130b),LL(0x08cc4038,0x7c00e333),LL(0x4484ccd4,0x8e7636fc),LL(0xf9a80322,0x1688e5f3),L_(0x0000008f), LL(0x05247531,0x0987f80d),LL(0x2cd48e4d,0x9fe4562b),LL(0xaa48e7e6,0xf168a311),LL(0x7fdc1a14,0xdf4018fc),LL(0xc463e403,0x6c8979b5),LL(0xd6d0bb4b,0x62cddf39),LL(0xdf09f24f,0x9b318fce),LL(0xca7e6578,0xcab54343),L_(0x0000018d), + LL(0xe7511a46,0xd7deae24),LL(0xcb23734e,0x23939762),LL(0x66bcd84d,0x989a46bd),LL(0x85ec037c,0x65439883),LL(0xcc808ec0,0xa3f08c8a),LL(0x680dc66c,0xa76800e7),LL(0x4c3c5332,0xcc98ee9e),LL(0x8663204a,0xa0ef46de),L_(0x000001b0), LL(0x7fff2898,0x05b4a4e2),LL(0xb14e22b3,0x930a37ee),LL(0xe9d3141e,0x35f5cd09),LL(0x3364f154,0xf55ccf3f),LL(0x55f31352,0xf6c93770),LL(0x0c74549e,0x4bf80f61),LL(0x8d0207da,0xb1c2b15c),LL(0xafb6ee97,0x0992fd2c),L_(0x00000092), + LL(0xa33e3956,0xea69c80c),LL(0xc95caa93,0xdeca4025),LL(0x95bc2026,0xc8c86ea9),LL(0xe3161b91,0x73fc72d8),LL(0x46b441b7,0x033e25a4),LL(0x6b7c1805,0x2d4e9335),LL(0x5a4b1a06,0xd30b7dc3),LL(0x992637db,0xdaac9a90),L_(0x000001d5), LL(0x67f8c589,0x4f4c9063),LL(0xc14619bd,0xfbc662e7),LL(0x3c65896e,0x8176a953),LL(0x1c5790f3,0x4f51c6bd),LL(0x0ef460cd,0xc6fa754c),LL(0xee3cd226,0x5e872735),LL(0x05291b65,0x79e3b5c0),LL(0x734e1b22,0xfa256432),L_(0x00000194), + LL(0x5d2ebb4f,0x0643f252),LL(0xedb2cca1,0x00e32811),LL(0xc996f279,0x6f6af92c),LL(0xbf992edb,0xbdef8275),LL(0x3384462d,0xa4dd3d26),LL(0x818a7ff9,0x8e214401),LL(0x60e7694d,0xa7aec62d),LL(0x9d54e87f,0x8bdd2244),L_(0x00000198), LL(0xe4e67752,0xfb63c9fb),LL(0x7e7ff11a,0x7eec026d),LL(0xc6b3e18f,0xe08b80f1),LL(0x84b5c983,0x5d6b5a4c),LL(0x4b0fd4b7,0x85f99e3a),LL(0xfc4904ce,0x7afd5a7c),LL(0xc336a99a,0xba1e62f6),LL(0x24e4a736,0xbe20ba29),L_(0x00000195), + LL(0x8229817c,0xbb592469),LL(0x29ecccb0,0x89ef0925),LL(0xce6e29e1,0xf98f60f1),LL(0x36216c3c,0x0848c8bd),LL(0x63a73874,0x085409b9),LL(0xd4abc07d,0x2319eb0d),LL(0x0e39c02a,0x6bda97a9),LL(0x393de5dc,0x0140ddd0),L_(0x00000115), LL(0x6a8c37e9,0xa2f22a24),LL(0x54381101,0x423788d2),LL(0xb694bdb7,0xc151a89e),LL(0xd0ef2b67,0xb01ee242),LL(0x01c2b082,0xc07af292),LL(0x10fd1158,0x3639401a),LL(0x8e3f86e2,0x1ed8f101),LL(0xcf21ea60,0x83b3b62f),L_(0x000001fb), + LL(0x10fc7810,0x718c92f0),LL(0x3423f6e8,0xdae8d246),LL(0x5f129e35,0xaeff7db0),LL(0xdbad59f4,0x963932f5),LL(0x3cf82c0c,0xf5e468db),LL(0x6b7d10e1,0x10e6e23c),LL(0x6e085959,0xc76fb1b0),LL(0x538880e8,0xe8c12594),L_(0x00000134), LL(0xd87b6710,0x89506649),LL(0x272ea4f0,0x9dd1a14c),LL(0xaa274066,0xa6cc0d62),LL(0x191622f8,0x92244f6f),LL(0xd28338e3,0xc3dcbd9d),LL(0xa8dd7166,0xa39c0c61),LL(0x4930a90c,0xb979b8ce),LL(0x6cd41296,0xaa5c88b7),L_(0x00000037), + LL(0xe9bd2031,0x8ef889ab),LL(0x6a8258a9,0x87f34cfa),LL(0x6e977272,0x538468d6),LL(0x198bf996,0xe9cb2903),LL(0xea7ac40d,0x389f9bff),LL(0x50fd922d,0x88f4717c),LL(0xcb0c2bca,0xb57f0298),LL(0x5d670088,0x812c3767),L_(0x000000fa), LL(0xadbacbad,0xd95da33d),LL(0x89860058,0x74f4e9a6),LL(0x9df658db,0x46a06ddb),LL(0x8faf5c15,0x36b96ffe),LL(0xe9bbc867,0x1a07dce2),LL(0x19a59e1c,0x9536a09f),LL(0x1683c160,0x7fabb0f1),LL(0x183d2bdd,0x06b7a416),L_(0x000001d0), + LL(0x31cd955a,0x5451d16f),LL(0x720fff5e,0xeacd93fd),LL(0x6c62e42c,0x6f74fc83),LL(0xff9b7285,0x8a51db93),LL(0x7b6bb42b,0x66ca983e),LL(0x8fd893a3,0x08eee06f),LL(0x491c6c89,0xe1230942),LL(0x638e9f64,0x4984e580),L_(0x00000176), LL(0x979f347c,0x0bad9aba),LL(0x7b9d835c,0x84846555),LL(0x89b78779,0xc6bb325d),LL(0x88fce8c3,0x0fb571c3),LL(0x237c5f2e,0x27185f17),LL(0x37bcf483,0x53b0ac57),LL(0xf037df6f,0x34a972e5),LL(0x73b6f7ae,0xf685c7b2),L_(0x00000021), + LL(0x0e793769,0xfab07625),LL(0x52bebe14,0xd1fbd06a),LL(0x7d25c686,0xe5149dc2),LL(0x20b2f012,0x6707c1b6),LL(0xe4fdb06a,0x5cf7e0a4),LL(0x124b0592,0xdadcb97c),LL(0x9ef54e16,0x97f26141),LL(0x6b91bf50,0x689c475b),L_(0x00000165), LL(0x22cd2270,0x31ac5e9e),LL(0x95772aee,0xf333125d),LL(0xfb3bbb8c,0x906a459b),LL(0xda033a3e,0xd9a3800f),LL(0x7aebdf94,0xea08c76a),LL(0x4600cde7,0xecd96496),LL(0x1b4f8404,0xb9fdf8c7),LL(0x58389c23,0xd186fc48),L_(0x00000137), + LL(0xc0dd4ad7,0xace5575b),LL(0xabf66053,0x6dc5328e),LL(0x54861cbd,0xea9fdaff),LL(0x8555b123,0x0ecf823c),LL(0x09e411c0,0xa5d8934d),LL(0x0ae97a01,0x170ceb09),LL(0x91dabc9c,0x73c40a75),LL(0xd8f751f2,0x52861011),L_(0x0000010f), LL(0x3075cd88,0x352bc9a3),LL(0x79de4fde,0xf0130bb7),LL(0x2eb1b199,0x3c4457b6),LL(0xa95e2900,0xf04878d3),LL(0xc1a9dc9f,0xe04ebfee),LL(0x097a6545,0xf5aa7d0d),LL(0x673c7b41,0x3c5c4ce8),LL(0xa67894e6,0x385d1700),L_(0x0000005b), + LL(0xf3612ddf,0x16680fb2),LL(0x370df675,0xbeb0847a),LL(0xbefb427d,0xbbe54c19),LL(0x9a4770e1,0x44a1916f),LL(0x7f5945d3,0xc14ef507),LL(0x731b2da3,0x17aa92af),LL(0x07208217,0xf69f649d),LL(0xa27c5c7b,0xabd89463),L_(0x000001f1), LL(0x6897edfe,0xe551752e),LL(0x9733c080,0x98e86236),LL(0xc5bfef7f,0x936a2ebe),LL(0xcc36a721,0x25c227b9),LL(0x11dd6248,0xf8d96ae8),LL(0x83440604,0x3b2dca5f),LL(0xc74d7e75,0x3d8a998d),LL(0x3c210303,0xdcf4cf75),L_(0x0000019c), + LL(0xcbdeae01,0xe646d7ef),LL(0x2f349cfb,0xfd187fbb),LL(0x22f14a9c,0x7fb5a2ff),LL(0x781ef46f,0x084df701),LL(0xada115d8,0xab2e7da6),LL(0x37b36285,0x21432735),LL(0x779e5cbe,0x42159b5d),LL(0x987b1bb3,0x182d17ef),L_(0x00000121), LL(0x95d5c1bd,0x0c974b93),LL(0x3e904667,0x4f31ca20),LL(0xc9fa51be,0xed87df23),LL(0x5530167f,0x7ab1aee3),LL(0x34d6716d,0x16c8a7b3),LL(0xb3f82160,0xf3eb37b8),LL(0x77ee013d,0x13ff1326),LL(0xa57a3a10,0xd7d1a2e9),L_(0x0000007e), + LL(0x57c07bef,0xfafe3733),LL(0xb917d893,0x17024a0a),LL(0xfd27b406,0x89eda4ec),LL(0xcd3182c3,0x4e2244fc),LL(0xcec915fc,0x083e32ec),LL(0xbc2fe85f,0x26668631),LL(0x3458ec27,0x23dec845),LL(0x2e647e96,0x35986103),L_(0x000001e3), LL(0x26887044,0x9c1dd0c7),LL(0xc0f6c814,0xdb6594dc),LL(0x59eee455,0x2db7ed2b),LL(0xc7b946f5,0xc94ac2ca),LL(0x45521872,0x1f918bfa),LL(0xe23366a4,0x3439b349),LL(0x50d8220c,0x347cd4a8),LL(0xc2e30ec6,0x9274e0c9),L_(0x0000009c), + LL(0x28064668,0x18b3fd00),LL(0x605cfac8,0x11efdfd4),LL(0x85d2f0b8,0x5bb41efb),LL(0xee216714,0x3c03cac7),LL(0xade36a6e,0x485c4b2d),LL(0xcd3725a1,0x50bc220d),LL(0x2cf525a5,0xb11c84f3),LL(0xe314db66,0x664e47ac),L_(0x00000013), LL(0xe7d464c1,0xa7a48858),LL(0x6f7bbfd1,0x7d04c227),LL(0xe24ada56,0xadced466),LL(0x03a6a941,0x70addbb1),LL(0xf14e02c2,0xc761ca82),LL(0x94b62798,0x03264d07),LL(0xa0bec3f9,0x966e8d47),LL(0xe6caf618,0x1f211c02),L_(0x0000001a), +}, +/* digit=52 base_pwr=2^260 */ +{ + LL(0xb5b4bc19,0x26c24408),LL(0x1e48c2aa,0x2fb6cd86),LL(0x8746f93b,0x515690c4),LL(0x71e5f018,0x76a3c1b7),LL(0x99fbb28d,0x993035c8),LL(0xc338e004,0xa3d8d18a),LL(0xb4e7f02f,0x804c0351),LL(0x09fabf9b,0x3e6175e3),L_(0x000000f8), LL(0x485a0549,0xf6830680),LL(0xc1a8a622,0x50d94962),LL(0xf94a3f34,0x0a44d62d),LL(0x8057a83e,0x05319e21),LL(0xd2bed201,0x3a4a1ebd),LL(0x3d6076c1,0x7368f486),LL(0x4672ca13,0xef4b1a43),LL(0xf96135e4,0x6692537f),L_(0x0000000b), + LL(0x08fe30e3,0xb81b7a5e),LL(0xb048815a,0x11e1229e),LL(0x2e0a161f,0xecb84207),LL(0xf8e1801b,0x5b394a58),LL(0x890edfad,0x37512807),LL(0xb3e4e477,0x5d81f675),LL(0xc9984105,0x1050ce18),LL(0xf43ed35c,0x17bd56ac),L_(0x000000c2), LL(0x5a1e0055,0xcf0d6c8c),LL(0xaf53db5d,0x6ee72ddb),LL(0xc6ab4e4f,0x32c8481a),LL(0x1b4e6860,0x5c545af6),LL(0x0e7c0e41,0xc3595ad6),LL(0x261ffe75,0xbf47f59b),LL(0xf66fa7cf,0xfa1aaf6c),LL(0x212e7097,0x86b7977f),L_(0x00000083), + LL(0x619e46fa,0x4b0029a7),LL(0xf5c33307,0x1eec5f29),LL(0x9e45f3bb,0xf8396133),LL(0x17635aad,0x25d3e2a3),LL(0xbdaba508,0xc34ef799),LL(0x574f4d09,0x78d47f38),LL(0x085e8086,0x3db03879),LL(0x1c4a5748,0x65ae9f6d),L_(0x00000144), LL(0xcaebc4bf,0xb52fb74f),LL(0xb901e46e,0x1868eef4),LL(0x68ec4a86,0x7bab1199),LL(0x9f2f51a0,0x8f19df10),LL(0x6a75a074,0x2d75da4f),LL(0x61385965,0x59f7f255),LL(0x60c80677,0x6b7b569e),LL(0x40b66382,0x0533f4d4),L_(0x00000145), + LL(0x4c4911e4,0x3b8e6670),LL(0xd20de07d,0x7aafab5a),LL(0xdab27e9b,0x3fb66eb5),LL(0x5ac52dbe,0x7ce85634),LL(0xdf84c8cf,0x9025496c),LL(0x95b8e1e2,0x776182f0),LL(0x8e6db4cd,0x21aaa54e),LL(0x3bb0faa3,0xe73fef00),L_(0x0000008a), LL(0x4fac454d,0x16d643fd),LL(0x70e138d6,0x5612fc48),LL(0x69ca59b8,0x0889a9e6),LL(0xfb3a26a0,0x93e3dad6),LL(0x43df1bc0,0xe6ce66bb),LL(0xae036271,0xfcd4244d),LL(0x05182a82,0x958ca2ac),LL(0x102559d8,0x26838c85),L_(0x00000075), + LL(0xcc2f3836,0x0184b954),LL(0x6cd88b38,0xdcf3ba77),LL(0x70c99422,0x86f66f43),LL(0xbc4bd450,0x7b81c0e8),LL(0x93575c5f,0x704cad24),LL(0x4091825d,0x4b9f70ce),LL(0xf1ff4bbf,0xac2a0a24),LL(0xb5d28bd9,0xe5ebf7a3),L_(0x00000023), LL(0xbdf9c155,0xeb270e7e),LL(0xeb783548,0xfd0b1050),LL(0x81562bc5,0x96b8a59a),LL(0xedb5f688,0xbc130375),LL(0x8ab1fc73,0xa1c5bd93),LL(0x89b28fea,0x7f18c19c),LL(0x8d6e4b1c,0xe98a494f),LL(0x409f7384,0x55131bf6),L_(0x0000003f), + LL(0xd9331dcc,0x25a27923),LL(0xffb6351b,0xc9bc2f04),LL(0x7f29f1e1,0x91e80528),LL(0x37069b7d,0x8a56cb26),LL(0xff75d6d4,0x7d9a9a20),LL(0x3f52dd39,0x52270b39),LL(0x703dee3c,0x67288a63),LL(0x13f9c1c2,0x49651d47),L_(0x00000038), LL(0x3c2dd2e1,0x1cd56c85),LL(0xbc1a8d52,0x46598a93),LL(0x87351736,0x4685de4b),LL(0x418967ee,0xf35701ef),LL(0x6dbbce4c,0x380b116b),LL(0xc5acf7cf,0x35416b03),LL(0xe839b424,0x8d1a9cff),LL(0x15841fbd,0xe1730d1f),L_(0x000001a6), + LL(0x9958b964,0xc0fdbea9),LL(0xf76bf65a,0xf573be01),LL(0x0c6778ad,0xffd85a6f),LL(0x927d0f51,0xfe98c72d),LL(0x1738874a,0x187e8ec8),LL(0x032ae57d,0x00c6d76d),LL(0xdf95e888,0xdee55d14),LL(0x0dec4042,0xcd5760c3),L_(0x000000e8), LL(0x4c10f002,0x14eac108),LL(0x7bdb463a,0xcbf771c3),LL(0x76281603,0xd48543b5),LL(0xfc634037,0xb965ac3e),LL(0x6e5426f3,0x49a7be5b),LL(0x87fba366,0x2e203d0f),LL(0xddb5ca9f,0xabbc3174),LL(0x55052649,0x2eb60836),L_(0x000000b7), + LL(0x1438294a,0x5c1bc4f6),LL(0x64a43b5d,0x1c634029),LL(0x5d7c2617,0x93c0fb82),LL(0x1b7967cc,0x96145dca),LL(0xd068364b,0x3b5c4ddf),LL(0xec5bd3c7,0xd5007f0e),LL(0x2d7bf8f0,0x771d6fd8),LL(0x215b93c7,0xf222990f),L_(0x00000075), LL(0x47223677,0x1d26e01b),LL(0x340c9a0a,0x04b5b926),LL(0x7edb2bec,0x0417ca25),LL(0xefd5d17d,0xb41c7280),LL(0x70df3372,0x93c942f6),LL(0xfbfcef99,0x7e3d7910),LL(0x37ef3a57,0x39005c54),LL(0x8c4d4c90,0xdb0ceb3a),L_(0x000000dd), + LL(0x7626a28b,0x65b80a9e),LL(0xb3de9aec,0x47955751),LL(0xebd70107,0xb9795325),LL(0x326b6e2a,0xb7ebfc01),LL(0xb10f9b62,0x6ec48711),LL(0x145049a1,0xb7dced78),LL(0xd8c85f83,0x6736770f),LL(0xf3878209,0x41fd70b3),L_(0x000001ea), LL(0xc5d35978,0x1245bc60),LL(0x228424d3,0x6d611151),LL(0xee80416f,0x92f6a019),LL(0x18ef86e2,0x2e88fbc7),LL(0x9f5d9f5c,0xb51a1205),LL(0x3ee14394,0xd989aea0),LL(0xe73a0ff8,0x81623fdf),LL(0x10ed321a,0x3d71a6e8),L_(0x000001dd), + LL(0xe8302688,0x98f6bd7a),LL(0xef684d8d,0xaac1e35c),LL(0x1c71e036,0x19611929),LL(0x9428ed8d,0x24f7251a),LL(0xf90f6e8a,0xcd34ddd1),LL(0x9742ae40,0xf7d22290),LL(0x9b5b15a7,0x5d805418),LL(0x91f1f6ec,0xe50e28f5),L_(0x000001c6), LL(0xd9e75a63,0x2c60d848),LL(0x5ad92240,0xf18911d8),LL(0xcfb4c90b,0x3c5b71d6),LL(0x1a2c26ff,0x5d53d732),LL(0x9fada03e,0x8bfbe9ec),LL(0xd69c81a8,0x443458b3),LL(0xce7f11ce,0xca59b490),LL(0x2489ddd2,0xdaf9ecde),L_(0x0000011d), + LL(0x75cf30c3,0xd052454d),LL(0x6844d9d2,0x667d1be2),LL(0xec3032ef,0xefd476cd),LL(0x67f7c660,0x47628345),LL(0x80c64c50,0x57751538),LL(0xbb8da5d7,0xb8ef3bee),LL(0xb395bca7,0x3bc2ad45),LL(0xc2e7e012,0x610d67aa),L_(0x000001ca), LL(0x2180bc90,0x3dc2b1ef),LL(0xf5d2b364,0x4ee3fb91),LL(0x38966853,0xe0446916),LL(0xc5fb0623,0x2a6bfc98),LL(0x6bce11c9,0x65c6b297),LL(0xeb6233b0,0xce8c355a),LL(0x6ac473c3,0xd97c1dd8),LL(0x56091541,0x0514bb6d),L_(0x00000023), + LL(0xb9480e26,0x92033f14),LL(0x17a6ef25,0x89a53d65),LL(0xaa5a50af,0xd872b73d),LL(0xa475ab82,0x53f65ce3),LL(0xe34acbbd,0x1d33affd),LL(0x33a80ee5,0xe7690066),LL(0x969e21f0,0xcc415a0a),LL(0xca2e0920,0xb0325f4a),L_(0x00000150), LL(0xa1f546f1,0x0f43e61e),LL(0x40fc3a90,0xa48abf41),LL(0x601d8bcd,0xe374c532),LL(0x786a5e70,0x7446fbbf),LL(0x484d4fec,0x0ec5b3b2),LL(0xdc5db3b9,0xeeb6c43c),LL(0xe60cf40d,0xca2ac048),LL(0x1ff63739,0x4475d66f),L_(0x000001b9), + LL(0x21d05128,0x9adfc451),LL(0xe4847519,0x294337f9),LL(0xbe446976,0x4e7dac1b),LL(0xf00a5441,0x94ab8aec),LL(0xc582caec,0x5ac9bb3b),LL(0xeacdc76b,0x3fe0e66a),LL(0x6e00689b,0x246c86c1),LL(0x6c266a0e,0xbf0ade72),L_(0x0000015c), LL(0xe7ec8261,0x70658a73),LL(0xff30554a,0xc9871f1a),LL(0x4b2448f2,0x8dd50a2b),LL(0x87a01756,0x3ea8f62c),LL(0xb6b9a2ea,0x7311e04c),LL(0x0d165122,0xfc6c9f2d),LL(0xb7efc2eb,0x07406007),LL(0x92c33ebb,0xbd8e5282),L_(0x0000001b), + LL(0xd9c24f49,0xe495d18f),LL(0x9b0f5222,0xac5ca929),LL(0x126086f0,0xf4d4507f),LL(0x41908fc8,0x3355d26d),LL(0x65791c82,0x31183a9c),LL(0xf00a9e80,0x018f0189),LL(0x644ffd95,0xe447af71),LL(0xbffa0975,0x7424f93b),L_(0x00000145), LL(0xfa695a18,0x0d6949a4),LL(0x78519510,0xb79c2b5f),LL(0xa59828a5,0x9ae92003),LL(0x6c54e38e,0xfb93be38),LL(0x67bd521a,0x4c71aeb4),LL(0x04b0340d,0xa3451e4d),LL(0xcd2e92f4,0xa9a77ad0),LL(0xfe218b65,0x656db073),L_(0x0000000f), + LL(0x30e20be8,0xa3b27cf0),LL(0x160f039b,0x82ec5f83),LL(0xcff71736,0xba9364b6),LL(0xebc485a6,0xddbcec8c),LL(0xdc80329f,0x3bda1715),LL(0xcc71e664,0x46fc4c3b),LL(0x592819ef,0xd1da3eb7),LL(0x2cb62fe8,0xb29cfca6),L_(0x00000055), LL(0x566fd9f2,0x8c802541),LL(0xfc158c58,0xe84b30eb),LL(0xf6625ae1,0xd137d022),LL(0x441de79b,0x42e42c6f),LL(0x7d99126c,0xf2ec0ec8),LL(0xfbbd41bb,0xb8f928d4),LL(0x2851ec63,0xf3ff5c1d),LL(0x5e5a9ca7,0xd3429e7d),L_(0x000000f5), + LL(0x4a3ce076,0x56a3a063),LL(0x8d143249,0x627718f3),LL(0x2232fc35,0x5a0479ef),LL(0x6a6d389e,0x82744b80),LL(0xf4d435b8,0xb0bd687a),LL(0x2792b960,0x4cecd317),LL(0xf792e60e,0x063be911),LL(0xb09dcb17,0x02f6ffb4),L_(0x00000059), LL(0x215ba3da,0xb04fbf6f),LL(0xe7c66f8f,0x07b90918),LL(0x95b38bbb,0x01c5b207),LL(0xc67022d2,0x4fdf3937),LL(0xee01b834,0xe5a11142),LL(0xc7b97506,0x11b8cb5f),LL(0x2ae40433,0x2450b7bc),LL(0xe3e1937f,0xa26a70cf),L_(0x0000005f), +}, +/* digit=53 base_pwr=2^265 */ +{ + LL(0x6bfa5396,0xf2fdc439),LL(0x7edbcb88,0xdb91292b),LL(0x19d35421,0xd5dee79a),LL(0xa420a538,0x035e9ea2),LL(0x9cf14f3b,0xe21709fe),LL(0x49703f94,0x690ca5b7),LL(0x495b47e8,0x4deb7af2),LL(0xcc2ef057,0xb09d6324),L_(0x000001d4), LL(0x9fe6e0b5,0x7ff7df3b),LL(0x25c764e6,0x1593ef9e),LL(0xb9153d85,0xef6d9489),LL(0x117822d7,0x238e5449),LL(0x1e34e4c9,0xbbd3333b),LL(0x58cc8198,0x416c6cfb),LL(0x7b487650,0xa8085b4b),LL(0xb3068c07,0x5e20cc8e),L_(0x0000017b), + LL(0x4bef6871,0xf98b837d),LL(0xe15922b2,0x62c29919),LL(0xc8afde9c,0x95a1a3c5),LL(0xffe9534c,0x604b1043),LL(0xfa2f638f,0x27a01a13),LL(0x04cd8a8d,0x2660393b),LL(0xe26fd0c2,0x72545d96),LL(0xcf0808a0,0x1dd10699),L_(0x00000187), LL(0x4ea56b71,0x037dfe3a),LL(0xc38223ef,0xd36e2094),LL(0xe8b66c87,0xe28405ae),LL(0xe3e2766e,0xa065b535),LL(0xed4b87f0,0x084a317d),LL(0x3f53ac0b,0x0ca5866d),LL(0xa0ee5586,0x82b21bd7),LL(0x1fa70803,0xff1d58cf),L_(0x000000f3), + LL(0xa39a68fa,0x905b2c93),LL(0xef13c9b6,0xd34ff12d),LL(0xa5eaf60e,0x46115d13),LL(0xad4eed45,0xc0704820),LL(0x0761b0ac,0xf0c499c6),LL(0x5dd51e45,0x4abd13af),LL(0xa978e552,0xb1ec09b7),LL(0xa79f811d,0x0dab7d3a),L_(0x00000190), LL(0x490481e8,0xa75f21e6),LL(0xea3d3b19,0x0364a9c0),LL(0x68df5edf,0x5e1d6b4a),LL(0xb44c93c9,0x33e2dcc0),LL(0x07832283,0xea8fc7be),LL(0x37cb9512,0xe9e13504),LL(0xc965c20f,0x887068c5),LL(0x62d3176a,0xe870a541),L_(0x00000004), + LL(0xc2984a5a,0xf1dd3a67),LL(0x7824703a,0xb39c772c),LL(0xbf0c69c5,0x46f942bc),LL(0x31d0901a,0x9e0174be),LL(0xd38bdff8,0xab6326f7),LL(0x91bfcc1e,0x8787eadb),LL(0x541868bb,0xa0385662),LL(0x6ba48a8b,0x6d878761),L_(0x00000060), LL(0xd5f8e883,0x69e290f5),LL(0x1d33d545,0x1ba52eb4),LL(0x662b634f,0x9dfd1d1b),LL(0x876fc504,0xbe51c909),LL(0x93e42059,0xcb7406ba),LL(0x49355b9d,0x2651475a),LL(0x8963ea18,0xbcf76704),LL(0x08985cee,0xaa85c805),L_(0x0000002e), + LL(0x2c70c50c,0x6c2616bb),LL(0xeb31ab0e,0x01e38aad),LL(0xbcd43f2e,0xda068909),LL(0x7f990c18,0x9fb2c072),LL(0xa82ff220,0x757bff88),LL(0x81327a89,0x28c2afd8),LL(0x1d3a1126,0x0c2079b4),LL(0x95685773,0xa957db38),L_(0x000000bb), LL(0x4a7cdb09,0x45f5c72a),LL(0x53fe6703,0x42ce353e),LL(0xbe22096d,0x4a3251c0),LL(0x601d33ed,0xaaaf17c5),LL(0xfe2c8cbc,0x3d4b4185),LL(0x242a9581,0xb32328dc),LL(0xbc79d78a,0x03bf4442),LL(0xa103c8f5,0x64e28853),L_(0x0000010c), + LL(0x1346edbb,0xb447a9f7),LL(0x2f9482b0,0x31ede472),LL(0xc7c55120,0x00d8bc4d),LL(0x627457bc,0x5c471ca7),LL(0x4a9f36d6,0x14a28cac),LL(0x436c70c1,0x38a173b4),LL(0x011c4897,0x96f4df0f),LL(0xdde3c9d7,0x587a661d),L_(0x00000193), LL(0x711723c4,0x143023ce),LL(0xbe0156de,0x68012aa1),LL(0xeabfa04c,0x3ed6803f),LL(0x204765fc,0x762dc13e),LL(0x2e5fcd9a,0x5b5cd65c),LL(0x04a542e1,0xd6b6a2d7),LL(0x3dcadeb8,0x57f74a74),LL(0x0da1060f,0xe953f87b),L_(0x00000148), + LL(0xfb1dd3f1,0x326c0546),LL(0x63e8f854,0x35eb9eac),LL(0x39f46433,0xf4944efa),LL(0xc8688704,0x91ff1606),LL(0xfeaa7186,0x99316708),LL(0xa92605cc,0x3fbb0f25),LL(0x2252affe,0xa90598c4),LL(0xcbb64aaa,0xb34934f1),L_(0x00000044), LL(0x58a7e6fc,0x763915ed),LL(0xc814b6b0,0xe697e570),LL(0x69866f7d,0x63fc73af),LL(0xb1f0f7a2,0xb634f283),LL(0x17533e2f,0x423d910b),LL(0xe17bdbad,0xfbcd888f),LL(0x778dac12,0x4c46f8f4),LL(0xfb0bef09,0x72d4d626),L_(0x0000017f), + LL(0x1eb22917,0x13b5648e),LL(0x34c2f6e4,0x202b7ba7),LL(0x8535398b,0xebd7f177),LL(0x7eea8b23,0xd3b0fc5e),LL(0xc7a0f19f,0xa3df55dd),LL(0x577641e2,0xcb9f261d),LL(0xf496646e,0x112454e8),LL(0xdaf2be9c,0xc23da6a9),L_(0x00000002), LL(0x2fed679f,0xe59d35c5),LL(0x1efd66d8,0x1b401767),LL(0x904f29b9,0x084f27ae),LL(0x129c352b,0x23e88566),LL(0xa3263601,0xd229faae),LL(0xc04620cf,0xc91c87f4),LL(0x535f695e,0x27c58545),LL(0x3cfc2a21,0xc94873fa),L_(0x000001a1), + LL(0xde0ea503,0xb13cb473),LL(0x88b9f2da,0x2dda613b),LL(0x3a43eba4,0xedbf11f5),LL(0x9cddb3f6,0xdf7a9b8a),LL(0x1e00c8ae,0x7e0c7ceb),LL(0x84c5eb6c,0x03d1dbf7),LL(0xc6747572,0x80122fe1),LL(0x5d6814d2,0x639c5254),L_(0x00000002), LL(0xde3b211d,0xec1fdac5),LL(0xa4651a82,0x690ed4f8),LL(0x4ef3c551,0xe2f0cf8b),LL(0x20e94507,0x61b6144a),LL(0xeb258124,0x11bde361),LL(0x51d9d605,0x541da730),LL(0x397e8ce6,0x06c00c29),LL(0xb4a5d672,0x43c098bc),L_(0x00000086), + LL(0x344f5276,0xf6656606),LL(0x82cb8136,0xd96edef6),LL(0x84c50ccd,0x9f0978ff),LL(0x37e0a146,0x17bb0d3a),LL(0xbf780900,0xb2dca4ae),LL(0x1d528632,0xb6bd3e16),LL(0x1bee4b87,0x8c609327),LL(0x16432d3a,0x4aa7829a),L_(0x000000d7), LL(0x09d85506,0x3f9c377c),LL(0x5046622f,0xa18dda52),LL(0xd98abc09,0x722fdd39),LL(0x23fb42b0,0xa78f3825),LL(0xf2a75675,0x13487db2),LL(0x33200560,0x244aa1c4),LL(0xc0bf37b5,0x86de25a6),LL(0x57b73e86,0x079d95dd),L_(0x000000bc), + LL(0x0ca7835c,0x17341b4f),LL(0x1e7f52de,0x52ebce6c),LL(0x882af4b0,0x673d8b9f),LL(0xbbd95fb1,0xd64ea8ef),LL(0x28e628db,0x7889079e),LL(0x54b7908d,0x26e0abe9),LL(0x3df2e6e8,0xc20813db),LL(0xfde7f3a9,0x978bffe2),L_(0x00000098), LL(0x280276f0,0xead10e9e),LL(0xad34ad6f,0x1076d303),LL(0x8df8f495,0x8819ee4a),LL(0xc0d57db8,0x70fb03b5),LL(0xb14472ea,0xbc0a100a),LL(0x18cc104f,0x7fbf87ac),LL(0x45839e8e,0x64d66536),LL(0x58fe7198,0x5bfbac43),L_(0x00000156), + LL(0x13d8a69c,0x1836614b),LL(0xcb97f199,0xc897ce78),LL(0x6d967571,0x940b810e),LL(0xd145156b,0x850c5939),LL(0x4b73e9e6,0x04a9944a),LL(0x6e833bad,0x2f7df8e9),LL(0x2cd53823,0x3b222e7b),LL(0xf7a26c91,0x1034f78b),L_(0x00000054), LL(0x40490586,0xfe320dfb),LL(0x5c8c95a5,0xcb9240ce),LL(0xfffec63d,0xc515192c),LL(0x000718b0,0x4259ce4f),LL(0xdbfc0155,0x6f7a6ff2),LL(0xb1ff6013,0x312bdeae),LL(0x4cc245a6,0x79a65a6f),LL(0x29aa5006,0xd3b4632f),L_(0x000001b1), + LL(0xba9ed328,0x356fabd3),LL(0x9eb2fa3e,0x32e2213b),LL(0x296648f1,0x5464f17a),LL(0xa0bf8f36,0xf19ca8ef),LL(0x4c8a5c7a,0x876e29a8),LL(0xa58d7e8a,0xd8f86aee),LL(0x0ec00506,0x04f4e1a3),LL(0x05b4072d,0x1462f8a1),L_(0x000000c1), LL(0x5e326e7e,0x564895eb),LL(0x5b19fee9,0x8c62d05b),LL(0x7809d8ab,0x0cb573dc),LL(0x4ad42bc1,0x1884e984),LL(0x9b3e8a9c,0x6b2d6773),LL(0x2d81f0f3,0x072385dc),LL(0x2f91b4d1,0xca372aa3),LL(0xe4038277,0xd16c3a45),L_(0x000000fa), + LL(0x5060d8be,0x9757a335),LL(0x1143084a,0x38952f06),LL(0xa4710659,0x025fc38c),LL(0x1698caaa,0xcf127f48),LL(0x7f55805b,0x39cb3c58),LL(0x621feb96,0x58068b85),LL(0x3a91b62b,0xa4e48dd8),LL(0xa7ba8220,0xbd22ff75),L_(0x0000005b), LL(0xfc09c649,0x05196c43),LL(0x142e4222,0xaa56e765),LL(0xeee6393d,0x8f13ec6c),LL(0xa88f8eb7,0x536554a7),LL(0x6720e144,0x66972f38),LL(0xadb6408b,0x9d95e37f),LL(0x67ab92ba,0xe96c2792),LL(0xa2d1345c,0x3fb8e9b5),L_(0x000001a7), + LL(0x945df86e,0x61c11852),LL(0xf484baad,0x1e71dab0),LL(0xad2e9168,0xe0ea71a3),LL(0x6e1a90b0,0x2b244009),LL(0xfb37ada7,0x0bd3281f),LL(0x38140203,0x1599d34d),LL(0xf278746c,0x3790a7db),LL(0x17f577dc,0x483a5cb9),L_(0x00000049), LL(0x22a84857,0x8f5c56dd),LL(0x6ec17c1a,0x767ebf21),LL(0xf0a141c4,0xf1091ff1),LL(0x051b5811,0x3ac1c024),LL(0x396942d9,0x692ece19),LL(0x5725cecf,0xf1e6de73),LL(0x75b56339,0x2f629ac2),LL(0x45030754,0x6207c855),L_(0x0000005d), + LL(0x8f44cc57,0xd14b028b),LL(0x8c73e470,0x684d5fa1),LL(0x46af781c,0xeb44feae),LL(0x3e3aadf0,0x4610320e),LL(0xfd9c5960,0xd8fffa44),LL(0x70d9d9d3,0xebbc9082),LL(0x8a6283f3,0xfcec5348),LL(0xdd60d649,0x44a603fa),L_(0x0000001d), LL(0x3023df31,0xce740ae3),LL(0x5f6a91eb,0x2000b013),LL(0x3780772b,0xe7ea71ec),LL(0x0e54747a,0x6f03b13d),LL(0xfc299d7f,0xd6603e33),LL(0xb6e9df68,0x86040d28),LL(0x2043747f,0x3aeee37a),LL(0xe4608968,0x9926fb8d),L_(0x0000004e), +}, +/* digit=54 base_pwr=2^270 */ +{ + LL(0x70b9e18f,0xabb9ad39),LL(0x6af8e430,0x523480bf),LL(0xf59d55e6,0x4bc56b8b),LL(0xc072bd61,0x3df0a6ec),LL(0x25c98f18,0xbee1786d),LL(0xbcc84059,0xf26f3fea),LL(0xb20a09a6,0x79a2dfb7),LL(0x93d600ce,0xcf2e6f03),L_(0x00000139), LL(0x3507cb80,0x9b72a39f),LL(0x2b1470af,0x4804a704),LL(0x7da313b0,0xe67c9622),LL(0xc290e590,0xbec90ccc),LL(0x796f29ca,0xf5e76e6a),LL(0xcadb620b,0x8ec01637),LL(0x15b03af3,0x4087520d),LL(0xd8dcf763,0x6c0ca6b7),L_(0x00000190), + LL(0xc34630d4,0x4f37e57a),LL(0xc030f2d5,0x649effc2),LL(0xb84aa880,0x3ad19d77),LL(0x1cab55a1,0x91bd296d),LL(0x8c081620,0xc8f7f0b2),LL(0x0c469726,0xb847d758),LL(0x1840b8cf,0xc59a8b12),LL(0xc8fd3c1c,0x0e2778fd),L_(0x000000b9), LL(0x2f98900f,0x446cc1f4),LL(0x3bf2f826,0x4fc8626c),LL(0x8d5f8bb7,0x7df08423),LL(0x3a877c74,0x41c77ea1),LL(0x471d935d,0x5556d8fe),LL(0xe98cde5b,0x279a8287),LL(0x068f4d40,0xe400538c),LL(0xdb305a88,0xc091d74b),L_(0x00000077), + LL(0xe7143fd6,0x766c809c),LL(0xdd78e4e1,0x81bcdd43),LL(0xa933555b,0xac8729aa),LL(0xb8964c85,0x4f18e8ec),LL(0x87096359,0x580f05ae),LL(0xffab1de5,0xe800a6a9),LL(0x797b2184,0xb6212cc3),LL(0xea98c5ce,0xc923afe4),L_(0x000000f1), LL(0x9f968c46,0x66353c22),LL(0xfe0b78e5,0x0345fac6),LL(0x27358467,0x180e49dd),LL(0x1ccff0c2,0x92bcdaf0),LL(0xea3ef331,0xf7e4fb3d),LL(0x99b89e87,0x092ef793),LL(0x9bdcca2a,0x1418dec5),LL(0xf9f9dccb,0x314595b4),L_(0x00000067), + LL(0x12cf8643,0x6e190a6f),LL(0xeee766f8,0x360709e0),LL(0x5b775cd5,0xd4566a98),LL(0xe4057c69,0x45df1e07),LL(0xc0672257,0x947733f9),LL(0x1b1c2a5a,0x4bcd6e2b),LL(0xc80987a2,0xe7293fbf),LL(0x89f4061a,0x11f7042b),L_(0x000001d7), LL(0x41e791cb,0xc53c1b03),LL(0xdaedd9c1,0xc48bf537),LL(0x495a12d7,0x2c8c9765),LL(0xe7c2d4a3,0x662fe9df),LL(0xfaed525a,0x27c6bad9),LL(0x5c4df70a,0x24dd660c),LL(0xba7fb076,0x21abac8b),LL(0x4a91b1d6,0xb618ce5d),L_(0x00000171), + LL(0x3e132643,0x7875f26a),LL(0x3212d16b,0x3953e4c1),LL(0x80a99a23,0x42f909d9),LL(0x457c9b9d,0x68f18c26),LL(0x62cfee59,0x0b1b0fbd),LL(0xd6e74f93,0x99d73ca6),LL(0x898ff611,0xecc60074),LL(0x0a0cf8cf,0x0e4b48e0),L_(0x00000082), LL(0xd0fa33dc,0xd086fac9),LL(0x30dd79af,0xe0e34e51),LL(0x0cb837fd,0x052d2441),LL(0xaefa0933,0xbdea4988),LL(0x44aec8de,0xdfac83ea),LL(0x46cb2469,0xfad769d2),LL(0x7cb77050,0x18dd28c7),LL(0x8001a60c,0x8fe3d888),L_(0x00000122), + LL(0xbc9cb5db,0x5b24df92),LL(0x2affa8b5,0x207215e0),LL(0x3c816de9,0x199ff528),LL(0xe11ab159,0xfcc61eda),LL(0xc8d67190,0x661fbf7d),LL(0xdda50129,0x76defd37),LL(0xd466e3a4,0xf14fb3e4),LL(0xc11ac280,0x7620efce),L_(0x000000d8), LL(0xd730b74c,0xb257b1f0),LL(0x4204720c,0xa9d04719),LL(0x620bab0b,0xcf599cb7),LL(0x4b89783a,0x4c38e784),LL(0x96aa7914,0xa4374ec1),LL(0xa42b74a4,0x1d57fa44),LL(0x5d9da37f,0x9e98081b),LL(0x907073d1,0xd1274d9a),L_(0x00000007), + LL(0xcbfb13a3,0x3e1d7c3a),LL(0xde4b8ce4,0x0940b9c2),LL(0x3515847d,0x594f371d),LL(0xc37d20b4,0x0d44e03d),LL(0x4b2281a5,0xa133895a),LL(0x03246afc,0x69dc40a2),LL(0x1243d0ed,0x6acc7d98),LL(0xc664eb78,0xa9ddc8a1),L_(0x00000041), LL(0xfe1862dd,0x1a66ad76),LL(0x3359c96c,0xebb34cc8),LL(0xe69f9794,0xd1662749),LL(0xf9f1455f,0xb162a274),LL(0x40fc34f6,0x9d860e20),LL(0x4fb62774,0xa70d36c2),LL(0x6f971a18,0x990d79fe),LL(0x19225101,0xdd30d9b3),L_(0x00000102), + LL(0x0a7a3999,0x51c91503),LL(0x673999f7,0xfe14668b),LL(0x08c22b2a,0xfc300d5b),LL(0xecabf6a5,0xe178c0bc),LL(0x020b90b3,0x79d38258),LL(0x81c171fe,0xa2f11763),LL(0x86f32623,0xf3a66cde),LL(0x6ff64b9b,0x5668f5ac),L_(0x000000c5), LL(0x6e735d95,0xac181251),LL(0x3ea58c5d,0xde279aea),LL(0x356a5f10,0x7e9f7153),LL(0xd08295b0,0x86ce9eb4),LL(0xf7c783f5,0x4daab1a3),LL(0x030b2d7c,0x603300a0),LL(0x2198f316,0xd0c0475c),LL(0xba184aa9,0xa6fe88ca),L_(0x00000111), + LL(0xdf1983b7,0xade6d9e8),LL(0xaf408fd2,0xfdbd3baf),LL(0xf2f9bf1e,0x437c785f),LL(0x502fb232,0xeb964445),LL(0xeaf5771f,0xb08bd2ce),LL(0x4c23d40b,0x71ed9783),LL(0xa9fbdb0a,0xf481a53f),LL(0x3a79f04f,0x8241d897),L_(0x0000006c), LL(0x2c38aa60,0xdbc73cb0),LL(0x6c1463d2,0x900c1c58),LL(0x38e9d58f,0x9115aabb),LL(0xc0a95554,0xd544f068),LL(0xb7066734,0x00d18a77),LL(0x10cc9da3,0x482d67a5),LL(0xf9d64b4f,0xfe8b6e80),LL(0xff8c0ceb,0xa32070d7),L_(0x000001c3), + LL(0x8777fa77,0x1fdd6bf9),LL(0xbe1106f7,0xc44a87ae),LL(0x43749975,0x3d0c3b77),LL(0xb45ea397,0xf0fbd03e),LL(0xd44cf903,0xb4c5c47c),LL(0xaa4a5ca8,0x38153aea),LL(0x2980014f,0x58f964fd),LL(0x7baccd46,0x2458a3d8),L_(0x00000059), LL(0x9360b528,0x9c8ef0d9),LL(0x8ec22b8f,0x99582fc5),LL(0x2b8aa8c6,0xd629663b),LL(0xcebbdb32,0x2ff405c2),LL(0xbd666f05,0x8989d659),LL(0x6c986174,0x2eb13e6b),LL(0x7fa4c2f5,0x5660e9a9),LL(0x30206c27,0xcd2e9b5f),L_(0x0000017c), + LL(0x970525d5,0x1b2c97bc),LL(0x3e8c6c5f,0x97ea6bef),LL(0xe6975580,0x1fa05de0),LL(0x45dcf226,0x28ce5e22),LL(0x6b3296e1,0xbc5ea09c),LL(0x355e867e,0x171e0d4c),LL(0x93a02b45,0xef953fca),LL(0x72562a6a,0x283e85d5),L_(0x00000050), LL(0x8028e2f9,0x00384cef),LL(0xc61925fd,0xb15e2b9b),LL(0x98c42ea4,0xe51f203e),LL(0xa9da7eaf,0x80b7c7d8),LL(0x00d05b8b,0xd0e8cb9d),LL(0x5b984aa6,0xe2223126),LL(0x97eb783a,0xcbd154b4),LL(0x60eeeb46,0x77d65106),L_(0x00000025), + LL(0x0193b1f2,0xecba5c8d),LL(0x4c52e9d2,0x1a2c0764),LL(0x084d971b,0x04071452),LL(0x5b00ba29,0x420810c9),LL(0xe37ace16,0x8726c12e),LL(0xfb3b3465,0x76e95cdb),LL(0xddb8f121,0x5a782ea1),LL(0xe1266546,0x9e91fb9e),L_(0x00000198), LL(0x5245f9b1,0xde0c16d5),LL(0xb01d1b1a,0xcdfa5a5a),LL(0x186cc016,0x907f643d),LL(0x951f20c1,0x19ce2692),LL(0x499758b8,0xa1e463db),LL(0xae173a15,0x1a60551b),LL(0x9960164a,0xa7db4dec),LL(0x5c5b509d,0xd9cec887),L_(0x00000054), + LL(0x21eaa3b3,0x16cce787),LL(0xd4a2fc01,0xa425fa18),LL(0xf155769e,0xaf00539b),LL(0x60eb1b90,0x688fdaf6),LL(0xfa6d7481,0xf34ab7ee),LL(0x4f289d5b,0x07dbb72e),LL(0x1e391abd,0xbc2da7ee),LL(0x95c48dba,0x566bd167),L_(0x0000003d), LL(0x43c9b589,0x9ce2304f),LL(0xd19a287e,0x13ab6992),LL(0x4405fdb4,0x96fa0864),LL(0x4139a060,0x41a760d2),LL(0xbccd999e,0x5ba64bc7),LL(0xb10009b9,0xd8deab9a),LL(0x984258d2,0x79776c54),LL(0x221ad688,0x10e8fea6),L_(0x0000004d), + LL(0xbd1cacd0,0x5b9e9b6c),LL(0x0d488b76,0x718de1b3),LL(0x479d6241,0xc5e08581),LL(0x835e01af,0x2980a85e),LL(0x5861c30b,0xb6dbc1d7),LL(0xa410ea56,0x2db982c5),LL(0xe2d1be8c,0x9bd416d2),LL(0x8b2c0849,0xff9c097a),L_(0x0000007c), LL(0x5bd1e957,0xbf3507bb),LL(0x84f97fa2,0x638f765c),LL(0xc3635cb6,0xddc089f6),LL(0xadab3335,0x03a11712),LL(0x45c1fc02,0x6d411e20),LL(0x92e35990,0x10bb8db9),LL(0x9c8fd9b8,0xa138660a),LL(0x9289131a,0x37f6b49c),L_(0x0000004c), + LL(0x5bc20fd8,0xa1146df7),LL(0xfdfcfffc,0x81511cf3),LL(0x23bc0f93,0x2b2a7c87),LL(0xdc82a234,0x03b40d7d),LL(0xa08a7dd1,0x710d66e6),LL(0x21695339,0xb2de413b),LL(0x70d88f9e,0xa639823f),LL(0x35b8b90e,0x023e9ff4),L_(0x00000110), LL(0x0c573ac4,0x67b34d07),LL(0x68b9df98,0xae385a70),LL(0x9e1b7b1b,0x69f1b285),LL(0xf3ca9831,0xacc09537),LL(0xd4b56b4f,0x9d566211),LL(0x5fe60450,0x8cabfe34),LL(0xfe548cc5,0xb714db58),LL(0x6edccbce,0x1b3de9cb),L_(0x000000f9), + LL(0xc323d062,0x3fb44e11),LL(0xd6d98611,0xf902dfd5),LL(0x894e506c,0xfd0b1d00),LL(0x85a52247,0x782247b1),LL(0xee7a96a2,0x73bd1827),LL(0x817a81ab,0xb5a675cc),LL(0x58e21da1,0x96f3b0ad),LL(0xba6b1f8c,0x0b4feab1),L_(0x000001eb), LL(0x2721b756,0x94e1e70f),LL(0xa2caaa6a,0xe4a5a160),LL(0xc595ff3d,0x4e2aab67),LL(0xc4ca75c8,0x55f145b7),LL(0x7731bee5,0xfc6003a4),LL(0xe7fe03b5,0x0bfb8f07),LL(0xc95ac06b,0x062bb217),LL(0x970ec8f9,0xa73aafef),L_(0x00000158), +}, +/* digit=55 base_pwr=2^275 */ +{ + LL(0x0e2b9a13,0x10edee70),LL(0x5fd3e47e,0x3597fca0),LL(0x77535436,0xd14d9e5f),LL(0x3e8b8ab9,0x09ae6cb6),LL(0x74096598,0xc8a4dd84),LL(0x9f1a5c96,0xcb6edd24),LL(0xd2f79af0,0x61d2b7a4),LL(0x0e166e53,0xfe3d22a6),L_(0x00000092), LL(0x995a329a,0xf72fa6f8),LL(0x96d7a363,0x488ad6e3),LL(0xd92f57cc,0x8510a286),LL(0xf0a9d195,0xb888aa8b),LL(0x317136c0,0xf42decec),LL(0xdf9fc71b,0xb6cc8b9b),LL(0xc0298d41,0x49e5d99a),LL(0x109ecbb2,0x314b57f8),L_(0x000000db), + LL(0x68b73573,0xb2d708c6),LL(0xce839038,0xce28ed96),LL(0x20e1ea62,0x8763eab5),LL(0xed2713c5,0xc4523fd6),LL(0x71027fe2,0x7eae1cb2),LL(0x9c4b8cf6,0x24a95e4f),LL(0x601ad020,0x07164949),LL(0xdd7d73ac,0x37442ffc),L_(0x00000043), LL(0x3ec84bf8,0xb77851b2),LL(0xdb8574d7,0xb645bbee),LL(0x286ebfe9,0x0c8710d8),LL(0x766e45ce,0xa79aecb4),LL(0xc2d31256,0xbf379f83),LL(0x340ea164,0x164bbbc5),LL(0xf851521a,0x1ac3081d),LL(0x7e9d5d5e,0xb205779b),L_(0x00000081), + LL(0xd2e1d746,0xde9114db),LL(0xe63af665,0x818c463b),LL(0x6295501a,0x35a127bf),LL(0xdce47ef1,0x007d2c0c),LL(0xcdab36d7,0xccb851cf),LL(0xfdd117a8,0xf238753e),LL(0x0f305c31,0x8e2817b4),LL(0x7fa2c0d7,0xf487e902),L_(0x00000153), LL(0x1fcbcf0a,0x170a9d8a),LL(0xfdab89fd,0xd0296988),LL(0xf7158579,0x9d9469d7),LL(0xe8a6f604,0x10415652),LL(0x2b54a37b,0xcc4eb51c),LL(0x0c719573,0xae48f5a3),LL(0xa83c1dff,0x30b12c01),LL(0x72dbb726,0x57308088),L_(0x00000172), + LL(0xc489ffe4,0x313e4b56),LL(0x08231734,0x6717f045),LL(0x479ae527,0xe3d436d8),LL(0xd02cb05c,0xf2257834),LL(0x00a63fdd,0xa7cf8043),LL(0x55acde6a,0x457f48d5),LL(0x3233d0de,0xde5db66b),LL(0x81aa55b4,0x0379d9ac),L_(0x000000a6), LL(0x067058f6,0x13e90717),LL(0xc47bead6,0xf2111132),LL(0x90e8a449,0xb92dfa6c),LL(0x2861c278,0x0e5052e4),LL(0xbb21a8a2,0xdd62ef7f),LL(0xda29cea2,0x06ce5d03),LL(0xb1054057,0x321921a2),LL(0x2bdda27b,0xa8070a21),L_(0x00000113), + LL(0x00c74bbc,0xb57fe1e1),LL(0x13d1fd3d,0xaf39e976),LL(0x9d300d5d,0x3c4bd73d),LL(0x394a792d,0x22ea164b),LL(0xdb8ad2a0,0x94ca8b71),LL(0x46b5c44e,0x29573de6),LL(0x4faada81,0xa68e6f0b),LL(0x9bb3e293,0x2829705e),L_(0x000001a5), LL(0xff57d0ed,0x562f24c2),LL(0x10d7ee7c,0xebc4c9ff),LL(0x1bfc2a5b,0x9e849995),LL(0x9ab67877,0x29bf2cd5),LL(0xebbab48f,0x1c14b040),LL(0xc34becb0,0x0f56d5be),LL(0xa06f84dd,0x74ea8bd3),LL(0x16998590,0x240441e5),L_(0x000000b5), + LL(0x5c45926a,0x99d23cf5),LL(0x8778f5f2,0xc8b025b9),LL(0x1705a5c8,0x3919b71b),LL(0x0d5b88f4,0xcb92372e),LL(0x60fa371f,0x943296e1),LL(0x0a89cc71,0x5fe1a497),LL(0x34a3ae69,0x5dec2f93),LL(0x1251e4b9,0x275a5942),L_(0x000000d6), LL(0xd859c11c,0xff47e08d),LL(0xaa12f1b2,0xad152f2f),LL(0xffb55ea0,0x2d49016e),LL(0x9565927a,0xe898a743),LL(0x6cdbde63,0x47e768ee),LL(0x9201bbe7,0x0a069ce7),LL(0x64e8832a,0x4d3af5cb),LL(0x22cff077,0x58cc25a5),L_(0x0000001d), + LL(0xdae36c68,0xb0b126b6),LL(0xaeabb8d7,0xc38359f0),LL(0x58505b26,0xb9091af1),LL(0xe930c10c,0x1baa2a57),LL(0x4ceb63b6,0xf34a1cb3),LL(0x2ce30eb5,0xe695563a),LL(0xb46502e9,0xbc4a4498),LL(0x3de11285,0x7d46bb82),L_(0x00000037), LL(0x31c841ea,0xd0a4132d),LL(0x4b0d3d99,0x19c65f2c),LL(0xbdec1fb8,0xe96bad16),LL(0xf98a7a22,0xfc740b98),LL(0x4ac9e432,0x327482c4),LL(0x19b02fa9,0xee365754),LL(0x2b71db69,0x8c4b6fcb),LL(0x3b059127,0x1ff3d7d8),L_(0x00000109), + LL(0xb46fd007,0x5b27f0e1),LL(0xc1b9a1af,0xc491a3f1),LL(0x3c2c754f,0x5cbaab1f),LL(0x2a77d316,0x0310665c),LL(0x760e6436,0xda6d64bd),LL(0xabfa1968,0xfb5f4ce1),LL(0x2b0e1701,0xb466c4ed),LL(0x7ed3c4d7,0x2ebf2125),L_(0x00000191), LL(0x2ef47a92,0x50f6e44d),LL(0xddd0d096,0x8ca37cde),LL(0xacd3234c,0xb7244def),LL(0xd39cba5f,0xacca56c2),LL(0x42e4fef3,0x04d3ff0e),LL(0xd03959e1,0xe6513498),LL(0x101ed923,0x40deadab),LL(0x3a0842e2,0x40cf65c5),L_(0x00000071), + LL(0x6ea80244,0x24abeced),LL(0x8027d5fa,0x54bd40b9),LL(0x6c207959,0xaeb9dac1),LL(0xd464b86a,0x419d1ea2),LL(0x6820d398,0x5e25c94d),LL(0x7cb4e131,0x65e1ca01),LL(0x407cd9e9,0xe5ede0b5),LL(0x9cc6a7e8,0xccfcd5ef),L_(0x0000012e), LL(0x8a46a1bf,0x38ae86f8),LL(0x198931f5,0xff746c8b),LL(0xd54d7f12,0x309a79bf),LL(0x1246b150,0x42f00081),LL(0x77449920,0xc47ea560),LL(0x3c1ca128,0x7f0d691e),LL(0x4a7cd82a,0x389f0267),LL(0x3325d3c5,0xd3d69318),L_(0x000000e6), + LL(0xa3cca92f,0xdc420de7),LL(0x40b5d961,0x0c56a78c),LL(0x0669c065,0x20ea2fcd),LL(0xd7d6512b,0x0cfdc1ce),LL(0x793f28c8,0x12dc4c42),LL(0x2a2c66b6,0x65ef14af),LL(0x8712d0f4,0x498de283),LL(0xaba3e10f,0xd43378b1),L_(0x000000ef), LL(0xaf2de227,0x29182339),LL(0x2743e625,0x75c8a0f5),LL(0x7cb967f9,0x3b942e5d),LL(0xa6fa495f,0x2c93c4c7),LL(0xaf911e44,0xe5ea4e81),LL(0x61393032,0x453b1c33),LL(0x6ad975cf,0xbd844374),LL(0x598fb85f,0x19bef583),L_(0x0000009f), + LL(0x9b9466a8,0xca0bf18d),LL(0xa3d7dbe9,0x4163c3c8),LL(0xfea7d95a,0xc8c760db),LL(0xbedeb961,0xbe4aaf54),LL(0x6366da72,0x184e2e0f),LL(0x4b391d6f,0xc176d3d7),LL(0xda402a6d,0x58e13d8b),LL(0x35c88b87,0x9e868f1c),L_(0x000001ed), LL(0x9fd8fa1b,0x060d87ba),LL(0x2fb088fa,0x7bb887ef),LL(0xd9fecef3,0x5f6918e8),LL(0xd8d0ab29,0x584e5e50),LL(0xa68549e0,0x1a0e8dad),LL(0xccee2619,0x6b94fb63),LL(0x2fe6d355,0x41620a75),LL(0x2f9f5687,0x3bbe2240),L_(0x00000024), + LL(0xa7583732,0x7b1c8a03),LL(0x5d3c5d98,0x9b0532ff),LL(0x8d28755f,0xa6811aec),LL(0x526696c9,0x1a05c762),LL(0x85c6fe67,0x9b509178),LL(0xb1cd9732,0x9ce1e435),LL(0xd6424b12,0xd25c0017),LL(0x7dfb69b5,0x8fd3449e),L_(0x000001bf), LL(0x45aa0fc3,0xdb3b3b47),LL(0x4f13caa8,0x8c65da30),LL(0xc42581d3,0xf1637449),LL(0x31ba26e3,0x086fb178),LL(0x1f20a375,0xe70bccf5),LL(0x794f66bb,0x8fb9ec67),LL(0x8759114e,0xba15c3e0),LL(0xbae55fef,0x6274e840),L_(0x000000b0), + LL(0x4b17f564,0x9940c688),LL(0xd17c3809,0x1ad4bb7a),LL(0x02bbb8e4,0x62901ee1),LL(0xdef212fe,0xf758e2a0),LL(0x902e165b,0x41df4c90),LL(0x813c93c1,0x2d980715),LL(0xdfe446b0,0x16925c07),LL(0x26180355,0x6a7e91aa),L_(0x000001e3), LL(0x97ef4ef8,0xa45e236d),LL(0xf8e5447a,0x116cff34),LL(0xe8d1be37,0x05f97032),LL(0xb0f21f66,0x2f2c027f),LL(0xa11df664,0xa89a55e5),LL(0x02fec70b,0x0cbe911b),LL(0x281dbb5c,0xf7515075),LL(0xff0a3fb5,0x5eb63dce),L_(0x0000017e), + LL(0x5eee65be,0xa91392a8),LL(0xbdba8a73,0x3162337e),LL(0x57c70feb,0xfe6064d0),LL(0x1cbf9841,0xb8ea5857),LL(0x0f9265ab,0x8f4d78e1),LL(0x4ea34ee6,0x86c61019),LL(0xe932bccc,0xa4d88afe),LL(0x9518b05b,0x1b666c9e),L_(0x0000007f), LL(0x59e8fd1b,0xf8bfc49e),LL(0x05208c75,0x9df54ae4),LL(0xd373c5a9,0x79933e71),LL(0x5b8a9772,0x64aa0b3f),LL(0xa96b6ada,0x636d2c0d),LL(0x5227ca04,0xc4142eed),LL(0x6e0ffb64,0x11d0f26b),LL(0x03baa051,0x6cce8a32),L_(0x00000049), + LL(0xf1fc721e,0x1b22c75f),LL(0xf889ab10,0x198462a9),LL(0xdc726c9f,0x4488dc01),LL(0x03497dbf,0x6173d9f9),LL(0x44668664,0xa2ec1a91),LL(0x97d57504,0xc515fce6),LL(0xe38e8b7b,0xc94145f8),LL(0x9a57e3c8,0x7f491462),L_(0x000001f6), LL(0xe3b58019,0xfab74bc2),LL(0x21aa76a0,0x5cde4de2),LL(0xcabcd328,0x58febacf),LL(0xfc5df376,0x164a8b43),LL(0x53cf2abb,0xd59cb6ca),LL(0x4bdbc3e7,0x5fa72448),LL(0x61888f39,0xec291663),LL(0xa63d2680,0x8efa4a04),L_(0x00000137), + LL(0x3aef952e,0xbfc44034),LL(0xda15115c,0x7be6519e),LL(0x4c0ca17c,0x1e404697),LL(0xd39b899e,0xd56cb968),LL(0xafaffc6b,0x977f25af),LL(0x0c48baa8,0x33c5e846),LL(0xb8ddcae7,0x8db3dd6c),LL(0xea4e7b4d,0x30c42ef0),L_(0x00000147), LL(0x6e0e9969,0x3bded433),LL(0x1c36485b,0x60a85776),LL(0x3a9ef9b4,0xc163b2e2),LL(0x9caea119,0x699c32d3),LL(0x6fbf2af8,0xc9afc21d),LL(0xb2f30acb,0x5cd46105),LL(0x2782e179,0xc5de1ebd),LL(0x94bd0296,0x40db331e),L_(0x000001f5), +}, +/* digit=56 base_pwr=2^280 */ +{ + LL(0xd75c1b45,0xb3566c4d),LL(0x70856265,0xffa63a05),LL(0xcbace31a,0x64645336),LL(0xd792b4ed,0xe49945b2),LL(0xcdc41c6d,0x4ffedb2c),LL(0xfc3fec1e,0xfb381239),LL(0x6c094341,0xb5868f95),LL(0xa828185b,0xf680572d),L_(0x000000da), LL(0xa2d876a6,0xbfe0585a),LL(0x480f8f0f,0x30bd3b95),LL(0x5be334d5,0xc2d3c86e),LL(0xd762f278,0x676d6c82),LL(0x1488b56b,0x539dec8e),LL(0x756194ec,0xc0fc3e4b),LL(0x4e5ad8a2,0xe01cce49),LL(0x1e1d4129,0x9cb7e94c),L_(0x00000045), + LL(0x1314c572,0x2ee172f7),LL(0x76d70712,0xcc86b737),LL(0x07937b43,0xc42e1bf4),LL(0x0a15775a,0xe0abab13),LL(0x8a6f4155,0x8a4930e5),LL(0x7af5f75c,0xe4d25a23),LL(0xf7ffdb8e,0x47745ba4),LL(0x1c8fe7af,0xda1f09c1),L_(0x00000067), LL(0xb90a6b8d,0x58ab833a),LL(0xbed40193,0x74f45669),LL(0x07b7fe4d,0x00a5516c),LL(0x8fed834f,0x35a36d31),LL(0x46f3d5f0,0xcf8a31bb),LL(0x46e1df3b,0x86c99729),LL(0xef24a7e8,0x383ab5e1),LL(0x3285864c,0x85a50c0d),L_(0x0000005b), + LL(0xd82688c4,0x82d65b66),LL(0x2c2925e3,0xc79d0387),LL(0x80fde81a,0xa2432027),LL(0xbc3a2b38,0xa34422b6),LL(0x24a6595f,0xb948d55e),LL(0x7ed8f149,0xfac14efa),LL(0x1011867c,0xe578bbe7),LL(0x01bd1e94,0xefa02765),L_(0x00000003), LL(0x88ed0039,0xc9f7389f),LL(0x4f58dfa2,0xd3bff99b),LL(0x27af2024,0xe0ca7fb1),LL(0xa09a0ca5,0x782dbf7b),LL(0x01098d83,0x19fa5d61),LL(0xabdbaa4b,0xddb85231),LL(0xc03b0d70,0xbb859499),LL(0x26cba60c,0x8d596ce3),L_(0x000001ba), + LL(0x11830fd3,0x8aec0422),LL(0xb67d56f7,0x85f0af78),LL(0xd0dd5f00,0xd1879493),LL(0xede77ef6,0xd412929c),LL(0x6254c309,0x2f8f47fc),LL(0x6cfbb6a1,0x4af63813),LL(0x0a561c30,0x70283e09),LL(0xb1640127,0xc77dd363),L_(0x000000e8), LL(0x1689c1a7,0x65973023),LL(0x29101040,0x7036678a),LL(0x6adb22ec,0x5919dc9a),LL(0xb9607eaa,0xff75ea6b),LL(0x2f103f0e,0xa1b15402),LL(0xc4300dcf,0xfdcf2e9d),LL(0x3ebbf1ab,0x1be47bf2),LL(0x3bc7610a,0xe48e43b2),L_(0x000000e8), + LL(0x079ff207,0xb1366222),LL(0xc255282f,0xcf45bdd6),LL(0xe5f65eaf,0x15c02959),LL(0x0d59b305,0xd5074c2f),LL(0x69d34def,0xf460bf31),LL(0x98c4daf0,0xcd4dd881),LL(0x2d35aae9,0xa3e2b924),LL(0xcd6f8fb3,0x7ee19179),L_(0x0000005b), LL(0xbdd26b48,0x3015c0c8),LL(0x24998853,0x9b18e9fc),LL(0xc27ee4ed,0x01ee8c44),LL(0x961e30d9,0x38c4d057),LL(0x4e4722a4,0x27a847d7),LL(0xbd34c3ce,0xfc175e9e),LL(0x9ec1f371,0x28ad6264),LL(0x9962cfa6,0x8cb8ba21),L_(0x00000025), + LL(0x61746f87,0x48414eef),LL(0x4b16d635,0x466a22f4),LL(0x91c53690,0xb9b4826e),LL(0xdfa700b4,0xba41d6fd),LL(0xa9d1c269,0x3f48b184),LL(0x9a1ae562,0x1d66af1a),LL(0x88445e2f,0x16e76216),LL(0x5528a0ef,0xa509e874),L_(0x000000a0), LL(0xf9d1f30e,0x4d16c4eb),LL(0x405c88d6,0xcb159002),LL(0x21995ea2,0x9206340d),LL(0xc1b476f5,0xbdb47138),LL(0x73c4a87f,0xf1fc51a6),LL(0xd81d7d81,0x68d2c132),LL(0x3035e2c5,0xc2e86c33),LL(0xdd1981e0,0x25fcaa15),L_(0x0000009f), + LL(0xa5dfb812,0x4ab2a49c),LL(0x70aadb7c,0x983438d3),LL(0x9f4ebf13,0xd25c9ac8),LL(0xd8d4d610,0x9c7f0f75),LL(0xca14e0b4,0xdcfed5c3),LL(0x4d6f7590,0x36f5cd7f),LL(0xd93cbfaa,0x65cb3d17),LL(0x1ddce79a,0xbd97f101),L_(0x000000b4), LL(0x87dd6cdb,0x8b012070),LL(0x77a85a51,0x279a4494),LL(0x7671964d,0xff88af2c),LL(0xf271d11a,0xdbb6c2c5),LL(0x82395ca1,0x85ba326b),LL(0x98f43101,0x0cb73c28),LL(0x63dc513c,0x6b203054),LL(0x4469f278,0xc5c18db9),L_(0x0000001f), + LL(0x36061c1a,0x3d80adc6),LL(0x4e403a26,0xab320624),LL(0xad04e1de,0xdcbb6130),LL(0x2a259720,0x6e850532),LL(0x231b1ad6,0xa60fb3f5),LL(0xcd79b6e2,0x663d49e1),LL(0x179c366e,0x01277eb6),LL(0xe6c6ea0d,0x883c4ffd),L_(0x00000073), LL(0xca5210dd,0x7e40167f),LL(0xaec71f68,0x9a231d95),LL(0x1c63bde5,0x4af79a44),LL(0xdec74dfa,0xfd79e68d),LL(0xa1952760,0x0b613ae2),LL(0x08e61ca7,0x9e73036b),LL(0xe30d2b54,0xd922e0f3),LL(0x6d8fb383,0x28c14621),L_(0x000000a0), + LL(0x6da7d675,0x1137f8e6),LL(0x66cf0839,0x9b11d642),LL(0xa909855d,0xc2008fb3),LL(0xcfac98d0,0x7141e8cb),LL(0xf021a4df,0xf143c405),LL(0xab358375,0x67bc2904),LL(0x17ae0177,0xcc509637),LL(0x3e96013d,0xa1e7d9dc),L_(0x00000049), LL(0xefccd8ff,0x957910b6),LL(0x497bb1a3,0x0139fb02),LL(0xbd519647,0xb1e83186),LL(0x3a05bf5b,0x19b27ed1),LL(0xcae014aa,0x82e975ee),LL(0x2c7c3285,0x88e9df86),LL(0x55efa48e,0xeb606052),LL(0x325f6177,0xa47eec1a),L_(0x00000161), + LL(0x6a4317c6,0x3a878798),LL(0xb073983a,0xb57577f3),LL(0x911ef229,0x88b3000a),LL(0xb6a2c941,0xfe60609c),LL(0x2014f532,0x505cb96c),LL(0x56c1e07d,0x4b65f80c),LL(0xacfa88b0,0x6d867481),LL(0xe4164f16,0x5599d374),L_(0x00000151), LL(0x92b10575,0x0c1e1e98),LL(0xfcb0b84f,0x19e6efb8),LL(0xb9207b90,0x51572182),LL(0xa8d8dbea,0x3bef29fc),LL(0x48664299,0xf1204f85),LL(0x601bc4d3,0x0e9a0fd0),LL(0x04ed7b5b,0x79ce5a54),LL(0x2efbde9a,0xa61da12c),L_(0x000001bd), + LL(0x452d878f,0x50fecf46),LL(0x92f8cb5b,0x920174e1),LL(0xf0d81ba1,0x1067e00e),LL(0xdf0090d3,0x0f944d92),LL(0x7cf84daa,0x29760c45),LL(0xe2fe5c35,0xdcd64aa5),LL(0x0fc3d3ab,0x926c41d9),LL(0xd093a0e6,0xf508ca71),L_(0x0000017a), LL(0x1dd9cd99,0x2ec2b366),LL(0x41b5fe84,0x68d89f52),LL(0x6a8252ee,0x7f37105c),LL(0x12bfbd7a,0x28e7afdd),LL(0xa8c8a6ce,0xc9ac312b),LL(0x64f59820,0xfc0802de),LL(0x91abcca1,0xdd3daa77),LL(0x3663e8e4,0xcba87568),L_(0x00000067), + LL(0x98711310,0x167f86e9),LL(0x77ecbc5f,0xb273c517),LL(0x1e1c4580,0x7b0cac28),LL(0x7d9e9af5,0x8231d2eb),LL(0x6aedfd99,0xaaffb728),LL(0xf1c98f12,0xe925ce44),LL(0x74f1520c,0xc049eb63),LL(0x494b788d,0xd9d95935),L_(0x0000009f), LL(0xc317b32d,0xccc2d2a6),LL(0x0b127e1a,0x3b751841),LL(0xcd70c11d,0x64418006),LL(0x1b4ae955,0xbadb0a24),LL(0x604cb635,0x9f330b20),LL(0xcfd3da79,0x73a90953),LL(0xef2d17df,0x1b20635f),LL(0x4743671d,0xd4c67374),L_(0x000001ab), + LL(0x12cdd2a6,0x9ec2635a),LL(0xbc72074c,0x1fe4598a),LL(0x2c820e92,0xe36ea0ca),LL(0x2497dd7b,0x4a8623f2),LL(0xe23eb8d9,0x60085b98),LL(0x40f9b504,0x1c20ee53),LL(0x201ec927,0x00761ac1),LL(0xd442c9fb,0x4f448cc8),L_(0x00000026), LL(0x820146fb,0x6e43f9a4),LL(0xccd45383,0xdd223c39),LL(0xe5a9e554,0xb7183f2b),LL(0xbabb5193,0xcaf50569),LL(0xd8970128,0x1fd8dbce),LL(0xa8d54145,0x334b381f),LL(0x13a9b729,0x728ba78e),LL(0x437dd328,0xd0616bed),L_(0x000000a9), + LL(0x5533acf3,0xaa17f0bc),LL(0xd2eb9fb6,0xcdb03b75),LL(0x6da41301,0xff96e100),LL(0xd003b0fd,0x0f341e2c),LL(0xeffd7580,0x00a92b40),LL(0xb4e94e73,0x2f420bab),LL(0x7e020897,0x3a5a980e),LL(0x0e9d7689,0x44d12101),L_(0x00000198), LL(0x4d880aed,0x2631776c),LL(0xf32bcc90,0x4363aa54),LL(0x81eb128d,0x290c3760),LL(0xc99f8366,0x24d51de3),LL(0xa9c8e087,0x2212897d),LL(0x63ef80d8,0xe1731f84),LL(0x4ba53d2d,0x71a09c6c),LL(0xf3c92c58,0x1de91d0d),L_(0x000001ba), + LL(0x1f13ee5a,0x007f0db0),LL(0xfc29c9c2,0xc5a04df7),LL(0x7ce23069,0x889ac9f2),LL(0xabf8339c,0x48b685ae),LL(0x57639632,0x3111646e),LL(0xf8b9f075,0x94f37131),LL(0xb2897670,0xf80a60f0),LL(0x834d23a7,0xeca2d9a7),L_(0x00000028), LL(0x3b936a84,0xc94b1130),LL(0x12c12cdf,0x7f1285c7),LL(0x53433711,0xa3fdb413),LL(0x6c9aab37,0xd50e87db),LL(0xc28229bf,0x6e31c080),LL(0x9878eb5e,0x647c7885),LL(0x6255df90,0x3c367034),LL(0x64a27e7b,0x359e7554),L_(0x0000010f), + LL(0x90855e35,0xc3090e22),LL(0x856faf70,0xcf9c3c63),LL(0xa537fff7,0xd0317a7a),LL(0x61c02007,0xdc853b32),LL(0x61687510,0x36ccdf2c),LL(0xd6f188d5,0x26fd385c),LL(0x2955fa1d,0xf2d7d6ea),LL(0x3087bdae,0x3173148e),L_(0x0000010f), LL(0x54216ede,0xd784c9a2),LL(0xc0770de6,0x4bf8c47c),LL(0x35e4c8fa,0xaece660d),LL(0x8910f637,0xedb7b99e),LL(0x5ced4fad,0xa82ce72b),LL(0x0fa07446,0x15701d4d),LL(0x94600c85,0x4152f301),LL(0xf34ffcea,0xf31c15ed),L_(0x00000096), +}, +/* digit=57 base_pwr=2^285 */ +{ + LL(0x31bdcc84,0x3f5b1c78),LL(0x7bafc7d2,0xbe5d1d02),LL(0x50c19efd,0x3cdce225),LL(0x5357a753,0x3af279d2),LL(0x14412057,0x76015a5e),LL(0x57141209,0xc91c803b),LL(0xb203384a,0x12ba72de),LL(0x2bedd680,0x825c3d8d),L_(0x0000011d), LL(0x39f8385e,0x6423553a),LL(0xeaf27fe1,0xef9335b8),LL(0xc4539fb7,0xfa5830e5),LL(0x66badc9f,0x0a5e5034),LL(0x1dcbb895,0xd3a2a96a),LL(0xa62dca0d,0x8a881a89),LL(0xd5f98db4,0x06e0a311),LL(0xe2554b95,0x69efeec8),L_(0x000001cd), + LL(0x8352945a,0xb00f4f23),LL(0xd445a023,0xdd54cf07),LL(0x551441ad,0xe62fb5bf),LL(0x33408f85,0xd275f3aa),LL(0x1f4e87ff,0x701eb4ed),LL(0xb08c2f8d,0xcbec0af1),LL(0x3b5987fa,0xf0bc3119),LL(0x37542336,0x219a8c12),L_(0x00000187), LL(0xae724bcb,0x5d52f04d),LL(0x1a68b0a5,0xd5fa5e22),LL(0x5bdff6f9,0xda24e831),LL(0x9487e3f2,0x8b43c649),LL(0x8cc54962,0xca393f54),LL(0x934d621c,0x52e8acee),LL(0x0ce12d1b,0xf0e6025e),LL(0xcb0b93c4,0x16663ffd),L_(0x00000171), + LL(0xb7086292,0x5de71627),LL(0xf54385d8,0x8b05712b),LL(0x7bb4ab47,0xcc6b5489),LL(0x5a9a3b2c,0x7fa6f6a0),LL(0x3bfef8f6,0x773cd523),LL(0xf505a80c,0x5c9cc4c4),LL(0xa3fe8c18,0xf37ae336),LL(0x6259e110,0xb440dc7a),L_(0x0000014c), LL(0xc7b0e1ec,0x1e50c98b),LL(0x4b379333,0x4a8aa5a0),LL(0x3b8d3103,0xc900df01),LL(0xc99c95da,0x52027ee1),LL(0xb891a1fd,0x4b5be3ae),LL(0xb6857422,0x5ba842f3),LL(0xe00bb37d,0x3f36375a),LL(0x13a7f31f,0x04743a28),L_(0x000000c3), + LL(0x37e819c9,0x69c14cb5),LL(0xb3bf6b75,0xa0bac5f2),LL(0xcc275187,0x6c2d559b),LL(0x0eb925ef,0xa5453338),LL(0xd0382a25,0x119f0d6c),LL(0xbbf74a02,0xaf681433),LL(0xbd994d4c,0x53999a9b),LL(0x8d27772c,0x249226a3),L_(0x000001af), LL(0xb493127b,0x796a46b5),LL(0x887c8f77,0xabbf7bee),LL(0x45dd063e,0xa0a8e117),LL(0xea429199,0x8d22e28a),LL(0x7e69a991,0xb0ecebe7),LL(0xb0ed2dea,0x0edf7cec),LL(0x37aa98d2,0x0e528886),LL(0x0ca19479,0x73078fbd),L_(0x0000000c), + LL(0xca9fd702,0x1809c965),LL(0x4cbc62e0,0x5c062be1),LL(0xeaea9560,0x1162dcba),LL(0x07340ec9,0xcf90fad2),LL(0x6f6f4573,0x8b6c2347),LL(0x41df3d0b,0x4bc4f7c5),LL(0x6aba94c9,0x9ce77c5e),LL(0x25055fe2,0xb9dd3d1f),L_(0x00000193), LL(0xedcc2ca4,0x6e430120),LL(0x31a5bf9b,0x31ef29c8),LL(0x4ea14b33,0x1451c355),LL(0x3d7a4759,0x515170ed),LL(0xc3011219,0x21413d55),LL(0xb375db75,0xacf6ce68),LL(0x846f6627,0x2facb2b3),LL(0xdee7b99f,0x3ead4f94),L_(0x0000004f), + LL(0xcc95dfa0,0x4f0117ae),LL(0xd22f1814,0xe3424149),LL(0xad5ce6fa,0x58b2d2e6),LL(0x70e2e877,0x5de9d268),LL(0x6eb2a9a7,0x6da54502),LL(0xddee04cc,0x3079a618),LL(0xaa37f095,0xef9c41ec),LL(0x0ab072b7,0x4f475d8e),L_(0x00000182), LL(0xb095050c,0x139a7b97),LL(0x482e5296,0x56474127),LL(0xc87e2a1d,0x0fe9253d),LL(0x4757efac,0xbdc55bb0),LL(0xaa65a406,0xfb5b47e5),LL(0xfa5b9027,0xd99ad5b0),LL(0x1f6adf92,0x3a4363de),LL(0x6339348e,0x6d9cf0af),L_(0x000001fa), + LL(0x3a106eb8,0x926c08a5),LL(0x58be047c,0x00018285),LL(0x4ede436f,0xb7c0f8da),LL(0x267d40d4,0x754e0583),LL(0xf821bcca,0x29b55f1c),LL(0x356838ff,0xef10fd05),LL(0xd3bb98d7,0xbf4cd160),LL(0xcf20597a,0xfe3d8718),L_(0x00000039), LL(0xd50f20a2,0xbe1a541a),LL(0x2ab231ad,0x833de73e),LL(0x3217ac39,0xde117a87),LL(0xe86047c6,0xa460f6e7),LL(0xbf61ad63,0x3e4086cc),LL(0x10646884,0xeedbe45c),LL(0x97bd568e,0x9c11e90d),LL(0x23655180,0xb96c7748),L_(0x000000ca), + LL(0xa8697eda,0x469af391),LL(0x5bb73205,0x3f5f97fe),LL(0x80d05ad0,0x73b1a3ca),LL(0x00af9b79,0xb52add98),LL(0xcc82c533,0x93cc487c),LL(0x0da2ae06,0xe46cf71a),LL(0x060c7047,0xaeb64abc),LL(0x3aa21503,0x0075b1d3),L_(0x000000ed), LL(0x12985515,0x29a2bed9),LL(0x9a79f6b1,0xda4630c4),LL(0x22374a19,0x62f001a1),LL(0x90a13059,0xa3cc4dcc),LL(0x026cefa3,0xb188b4cc),LL(0x0fbb1d3f,0xad092ff3),LL(0x36e3761c,0x6354b93c),LL(0xf3dbdbdc,0xb73317cf),L_(0x0000012a), + LL(0xda4d56d5,0x478cc5ff),LL(0x39f6b453,0x4e6379ca),LL(0x063068cd,0xe4f74a40),LL(0xdb751bed,0x94052b38),LL(0xb0c9cab3,0x8ee4c1e6),LL(0xf5c6aa29,0x3ca0dbb3),LL(0x4a2497c4,0x79c6ee9a),LL(0x9ea63ecb,0x00e2354a),L_(0x00000188), LL(0xe4a480ce,0x3a6196f9),LL(0x62b157a8,0xfa37ba58),LL(0x59648d99,0xe7f1758b),LL(0x1b51e49d,0x2e82eb6d),LL(0xbcb59206,0xc0686a28),LL(0x337d156a,0x6197f5d2),LL(0x69bbd81d,0x9b64ab0e),LL(0x45283587,0x8e4d1a8a),L_(0x000001e8), + LL(0x8fd25f3f,0xed1f56a5),LL(0xe00396a8,0xc29bf98d),LL(0x671ac7e0,0x61a8021c),LL(0xa76a8082,0x40244556),LL(0x85eaf05e,0xd4493c4b),LL(0xacc79ffa,0xf3dd2e24),LL(0xa065de83,0xc2899229),LL(0x616b4043,0x26ec1649),L_(0x0000015b), LL(0x8dee69e1,0x177b6d5f),LL(0x5bdfa715,0xbb1b8e86),LL(0x1c8a38c6,0x673dccd1),LL(0xb8eb3119,0x26507bdf),LL(0x5e9ab066,0x52444c41),LL(0x63e06aca,0xc41d72f8),LL(0x6233cfed,0xf0a4cf0f),LL(0xa4204c93,0x2e510168),L_(0x0000002c), + LL(0x02f56d78,0xb9ae2c20),LL(0xa8e73672,0x60eb4910),LL(0x76d9acc7,0xef341ec9),LL(0xf8c3ff90,0x71631b58),LL(0xc5d46ddd,0xf784f3a3),LL(0xaca2efa5,0x9fd6ccc9),LL(0xb432ab0f,0xe0c19b00),LL(0xe9336780,0x8e4851bc),L_(0x00000077), LL(0x763901d3,0xeb8a4214),LL(0x1ec45488,0xab894d10),LL(0xb3592330,0xfe84be74),LL(0xea5cac9d,0xd6ce52df),LL(0xd6fc5829,0x52b83766),LL(0xada3c1ed,0x6e5007f8),LL(0x07f9020f,0x9dc77e51),LL(0xf5261fb1,0x3d52bde6),L_(0x00000176), + LL(0x61e0b43b,0x9c19cb9b),LL(0xa33466a1,0x59900a19),LL(0x2eeef601,0xd1a35b61),LL(0x70079dee,0x043f6bdc),LL(0xcd5ea5fd,0xc0119db9),LL(0x3a3b272f,0x5b2eb1b8),LL(0xe45f974f,0xba8b6e51),LL(0x5f770445,0xd0419f04),L_(0x000001b4), LL(0x8f1c3e6a,0x4b8a6a2f),LL(0x21c82a7e,0x356eb5d6),LL(0x1f805802,0xc510787f),LL(0xb2d54598,0x678fa9fa),LL(0x14375d0b,0x27fba413),LL(0x6ed82aca,0xce44cfc3),LL(0xbe259313,0x826f662b),LL(0xdd1eec94,0xc3f7e810),L_(0x0000010e), + LL(0x451f3fda,0xafd7f180),LL(0x784a36b9,0xa71f98dd),LL(0x55359374,0xb80637d2),LL(0xeb356304,0x319eb954),LL(0xe36b826f,0x6caacbc8),LL(0x5436ed41,0xc8f9a6b4),LL(0x1ad27bfc,0x05822de6),LL(0x66b9e6c5,0x6448fc9c),L_(0x00000061), LL(0x1be571ae,0xd306ac81),LL(0x92fe9fac,0xdda81241),LL(0x90144259,0x08d31ec6),LL(0x70a69700,0x59532bbb),LL(0x548a4797,0x25db5e1c),LL(0xfe84a6c9,0xdb376141),LL(0xd82e648e,0xaf5e43aa),LL(0xd86080c6,0xa3c129c0),L_(0x00000047), + LL(0x16e20146,0x2343be89),LL(0xd0ffaccb,0x1cb1ffd2),LL(0x3a03a0c7,0x899f4ff5),LL(0x6266f542,0x5f5c983d),LL(0xbcc25c41,0xcccfd128),LL(0xdfd7dc3b,0x971841bb),LL(0xb315e6d3,0xab458be6),LL(0xf423c907,0x18de71ba),L_(0x000000e5), LL(0xe2e17f1c,0x8643db82),LL(0xdf24bfc8,0xae140a96),LL(0x7d249f93,0xcc1b0809),LL(0x4a944e10,0xbb9f2bb4),LL(0x2cf2ab30,0x2a9df9a6),LL(0x3e7a3348,0x8877de2f),LL(0xc1bae4c1,0x6e777963),LL(0x95df6e0c,0x12289ec3),L_(0x00000076), + LL(0x1a1f2092,0x84f63e6e),LL(0xb92016c3,0x9065995b),LL(0xcfc7edd3,0x7e853e34),LL(0xe921ec35,0xe5b9d192),LL(0x48df779c,0x22c1257c),LL(0x2377e36c,0xc67f15b2),LL(0x2dd7559c,0x56741ee4),LL(0x8133583f,0x266292d1),L_(0x00000077), LL(0x28b3ff2a,0xcd5b0dfd),LL(0xd6ec8a55,0x0e18cbd7),LL(0xa02b3661,0xd0b4c82b),LL(0x54051775,0x2328d0e9),LL(0xa2bff3fb,0xbab00086),LL(0x8724078f,0xf6183452),LL(0xe2d3f99f,0x1d9f7aa2),LL(0x3419a97e,0x6878b1f4),L_(0x00000013), + LL(0x5635e4e9,0x6c03f366),LL(0x7fd85da2,0xd5f694e1),LL(0x52fd006e,0x2f043a61),LL(0x51032d25,0x8bc9cc74),LL(0x9348d55c,0x6f5370ca),LL(0x56333c4c,0x3610540b),LL(0xc9a5ca53,0x716d25cb),LL(0x39d8071b,0x7337f70a),L_(0x00000036), LL(0x387c11b0,0x97db6fc5),LL(0x3251b143,0xdb755cfd),LL(0xd84aa2bf,0x0cc3e62a),LL(0x9e9e3810,0x6071f1f8),LL(0xe47fb104,0x3e7012d9),LL(0x97ec5c7c,0xf6c7e6ad),LL(0x98bc4de4,0xa4e7cef6),LL(0x240c6a07,0xa03a3a12),L_(0x00000198), +}, +/* digit=58 base_pwr=2^290 */ +{ + LL(0xdddd94b7,0x7663b161),LL(0xc93f0cc5,0x071af3d7),LL(0x22d6ac11,0xb9149bdc),LL(0xe6312d84,0xe44e4632),LL(0xc50d4c88,0xc448cc8e),LL(0x6c85277a,0xbfe4f89a),LL(0x128700ea,0xa38e5f2e),LL(0xb742928e,0x3e261880),L_(0x00000029), LL(0xc113b689,0xfa51028c),LL(0x9b6a14e2,0x09549191),LL(0x82dfd5da,0xe13022a3),LL(0x233ca662,0x96fafc24),LL(0x505fe429,0xa18dea4f),LL(0x96182166,0x15ea5a2d),LL(0x199ba558,0x22a4ac80),LL(0x33772326,0xb13c3b81),L_(0x00000111), + LL(0x46be575a,0x215bf15f),LL(0x61cb09c8,0xe5912a10),LL(0xae2de789,0xd84851c6),LL(0xd74ceccc,0xb95ccd21),LL(0x6a285101,0xd32dddf2),LL(0x0122d3f6,0xdb554921),LL(0x02c5d952,0x96a4aa1f),LL(0xb24be997,0x8cde88aa),L_(0x00000007), LL(0x0855f9f8,0x2cd753b8),LL(0x88aff9b9,0xcc49d782),LL(0xf7cdce61,0xdac4e445),LL(0xfad48cc3,0x0ac2a937),LL(0x956fdfcb,0x98c5bdda),LL(0x81841ce2,0x9f12bb3e),LL(0x170e6c81,0xcab58ad5),LL(0x30efd73e,0x76a3a481),L_(0x000000f4), + LL(0x17b9123c,0x361c2c61),LL(0xb9e4b6ab,0x810de58e),LL(0x3d0db7a6,0xe085a8d1),LL(0x45a31ec7,0xa7bb9df2),LL(0x35378c99,0xd8933f30),LL(0x186da525,0x1033d24d),LL(0xddb7a3b3,0x8af19819),LL(0xb5c9012c,0x57d17203),L_(0x000000c1), LL(0xe7a75fd6,0xce3ccda4),LL(0x41697bbe,0x5c6d7a27),LL(0xb8be57fb,0x1fba9fbf),LL(0xc562ecca,0x9a3fed12),LL(0x1bcd8090,0xf597a3fc),LL(0x74a6954d,0xce4e5ded),LL(0xb4910fbc,0xa5ed9cf8),LL(0x79902452,0x9c77346d),L_(0x00000177), + LL(0xc67a3c57,0x3a4f332d),LL(0x3d82a438,0x56af6321),LL(0x0fc06213,0xca05acfd),LL(0x8864ca32,0xe80103ea),LL(0xc2e9c7c1,0x99dd0ff4),LL(0x4c64b758,0xc9889d76),LL(0x80c128a8,0x881a1256),LL(0x77fdb7cd,0xf09f58ad),L_(0x000000c5), LL(0xcda86d61,0xc6ecd4f9),LL(0x41a633a8,0xecb1fc4c),LL(0x847c2f58,0x92dae51a),LL(0xa7d7b295,0x268f50e3),LL(0x3b5eef6e,0x3a27de2a),LL(0x0d7a599e,0x14916d54),LL(0xd01f9b57,0x204fbca0),LL(0x675e52e5,0xeb48615c),L_(0x000001d9), + LL(0x76250214,0x0a639042),LL(0x0fd5737f,0x31d2eb63),LL(0x801bd86d,0x85ffa7ea),LL(0x1011c35d,0x8d043e51),LL(0x1ef5b87b,0xcb405068),LL(0xec30dbd9,0xc20daf68),LL(0xe48310c1,0xcee24a41),LL(0xa65b8aca,0x119d1da9),L_(0x0000008e), LL(0x58c134d6,0x716e9def),LL(0x8b2825b1,0x5d82926b),LL(0xb4cd0082,0xe1a8a7c8),LL(0x6474e309,0x12620e3a),LL(0xb0da6f90,0x2d673d4a),LL(0x574adf3d,0x628b88b6),LL(0xb210b971,0x9d1b96a3),LL(0x6b2d573e,0xedcd56fc),L_(0x00000081), + LL(0x9d453d01,0xe07f6e9e),LL(0x0f0b6a27,0xa4a6f307),LL(0xddeaaa37,0x9c430a1d),LL(0xeacad6b5,0x9620fd47),LL(0x9a8128c0,0xf279790d),LL(0x3bf5952b,0xfb97ad6f),LL(0xe0561485,0x0fe7692a),LL(0x482c591f,0xe268f3cb),L_(0x0000017f), LL(0x0807b886,0xdb21bfbb),LL(0xde3dc674,0xdbf154f3),LL(0xf4401caa,0x32e63083),LL(0x462197d4,0xb9452cf6),LL(0x46240ddd,0x10344368),LL(0x1c6dbfe6,0xa986f17f),LL(0x94ccbb69,0x4632a20c),LL(0x3f6277d7,0x33029382),L_(0x00000197), + LL(0x0b3dd856,0x2b718a28),LL(0xd8752e23,0x65e87e31),LL(0xa158249b,0x88c3f123),LL(0xff7b1118,0xd9121432),LL(0xbe4461fe,0xa0850e4f),LL(0x9ef5bc2d,0x6350e71c),LL(0xf28780a7,0xc6dabd80),LL(0xfc8f574b,0xc3c266b1),L_(0x00000143), LL(0x007b740b,0x13bf6ff9),LL(0x7db218bc,0x9d000699),LL(0x0dec75e5,0x7cf7628c),LL(0xb95f2df4,0x2fa7aacc),LL(0xe8ff9a7d,0x96555722),LL(0x1076129b,0xa11a1984),LL(0x57afac1e,0xfd1e9ec4),LL(0x16d64a31,0x008b70a7),L_(0x00000032), + LL(0x67420c4b,0xb0d9eaa8),LL(0xf5caa2dd,0xddc322f7),LL(0xc31038c0,0x08fb4b57),LL(0xa651596a,0xd9ca6980),LL(0xab32e2a6,0xc95c78a8),LL(0xe5808eea,0x5a32ba78),LL(0xf5f9923d,0x3bbece34),LL(0x26ad1c8d,0x8f8b8459),L_(0x00000109), LL(0xbeef787c,0x16843645),LL(0x8875d753,0xa90e9fa2),LL(0xe13608c5,0xaf90c364),LL(0x57e5556e,0xcc40e058),LL(0x9e332dda,0x9c5012b5),LL(0x2b76768f,0x8a76230b),LL(0x2932d53c,0x573bdff3),LL(0x14999fbd,0xeee93001),L_(0x00000040), + LL(0x9092892f,0x1a3f60e5),LL(0x7e88fc70,0x29625c0d),LL(0x396b1851,0x610e5833),LL(0x947ef062,0xd8dd1f5d),LL(0x47f1a571,0xa0f65294),LL(0x7850d950,0x49f087e9),LL(0x22c8e733,0x18807434),LL(0xce5508b2,0xd0fc8fca),L_(0x000000a0), LL(0xc40adf60,0xc3d1360f),LL(0x69072d8a,0x8b9d1e81),LL(0x8cba6305,0x01ed34a6),LL(0x7a1a3844,0xbc37f296),LL(0x20c61572,0xd409e84e),LL(0xd54640e7,0xeb6c948c),LL(0xc9243fc3,0xc754fba2),LL(0xb39c9166,0xc28f4f28),L_(0x00000182), + LL(0xfb5d6f5a,0x32c7f33f),LL(0xa0e6751c,0x47474b7e),LL(0x7f265069,0xe1e4a2ce),LL(0x0460d889,0xeab5839d),LL(0xb51b9a7f,0xe7cad388),LL(0xa5032a25,0x46ee855a),LL(0x6621ee7b,0x16fbfccf),LL(0x5a6f1501,0x73af1329),L_(0x00000175), LL(0x070dcac9,0x694e09ae),LL(0x0646542a,0xaceb179f),LL(0x27a867a4,0xeb30df16),LL(0x14cf3975,0xf9d85fe3),LL(0xa18e96b2,0x37e6f97d),LL(0x7781a0f2,0xb09bd1ce),LL(0xa8de0b13,0x278a1089),LL(0xc3a91cf5,0x02296583),L_(0x000001f9), + LL(0xbdb5de67,0xeca2c791),LL(0x86fe1661,0xd1c18fea),LL(0x4aedcbc7,0x3f879f46),LL(0x0545d544,0x4cb96993),LL(0x95120a10,0x595026f4),LL(0x1d335198,0xc959c824),LL(0xa814ec2a,0x8fbede3e),LL(0x0e062b6d,0x3832b5b2),L_(0x00000086), LL(0x78dc0113,0xfad2bbc3),LL(0xcf89249c,0xf01f3803),LL(0x63266c42,0xb86898e0),LL(0x685db0a5,0x3becca8e),LL(0x7e4eaa63,0x3ec31fe1),LL(0x90fcf86c,0x394a64f7),LL(0x967f0628,0xc6f81bcd),LL(0x635c81b7,0xec462896),L_(0x00000158), + LL(0x3f6901a6,0x67236236),LL(0x67484461,0xdb36fbe6),LL(0x5f5c47c0,0x9fb54024),LL(0x71f2fbb3,0x44525a78),LL(0xbedf63e3,0xe71375bd),LL(0x9f085fb8,0x47d50bd8),LL(0x1c59b6e6,0x2f2ea430),LL(0x578031fa,0x58012b66),L_(0x0000019b), LL(0x48eca8dc,0x473015fd),LL(0xfde2151d,0x9616e82c),LL(0xe4d908c9,0xd4ec3b2f),LL(0x04a04977,0x13df9ad2),LL(0x3ee923f5,0x2b66641e),LL(0x175bb5d9,0x0fcd9df2),LL(0xcdb5c3c9,0xfac57254),LL(0x7fde8809,0xc6981a62),L_(0x00000185), + LL(0x89022a23,0xb19296e9),LL(0x9d659d99,0x6d5aa5b9),LL(0x489a28d9,0x5422e69c),LL(0x5cf35829,0x06993c4a),LL(0xed0dce41,0x4bccea69),LL(0x21d11ff6,0xa2e82c7c),LL(0x44f73388,0xda3168f8),LL(0xf6117d7c,0xdff018c7),L_(0x0000007f), LL(0xc2227980,0xeb022661),LL(0xe965ba34,0xfeff852b),LL(0x1a68518b,0xbe9a9ee4),LL(0x53aa84d0,0x31f46a2c),LL(0x112327ea,0x6855b874),LL(0x06311411,0x43d26e75),LL(0x7348f329,0x65628948),LL(0x0582ac08,0xe3244339),L_(0x0000001a), + LL(0xdb77778a,0x5a842868),LL(0x12b97327,0xd4a2fefb),LL(0x8f7a410f,0x84c0e584),LL(0x98e19862,0x3fbe93da),LL(0xe566e4f8,0xe44a9540),LL(0xec1d03a0,0xa377131e),LL(0x1b99313e,0x27336d2c),LL(0xc15f4f38,0x8c27d958),L_(0x0000009e), LL(0x7c38c847,0xe75811d3),LL(0xc54aa0e8,0xe1cbcf6b),LL(0xc195aaf4,0x5596682f),LL(0xe98a5845,0xe0cf229c),LL(0x70256db2,0x5a921b26),LL(0x10d830e7,0x37fc26fd),LL(0xe3def649,0x16a810c8),LL(0xdb834a77,0x613433c1),L_(0x00000034), + LL(0xd8bd0a2c,0x7624b24b),LL(0x7db58054,0xabebbd07),LL(0x20618f15,0x3a5e2752),LL(0x72097df2,0x4f72e3fd),LL(0xf79b4cba,0x6f03686a),LL(0x3cfd9643,0x2d89778e),LL(0x683c4a14,0xeade01a4),LL(0x7cca2771,0x533fd14a),L_(0x00000168), LL(0x182d4838,0x5f84ba35),LL(0xb97b68e1,0x4e0b9bd1),LL(0x6e47fe3d,0xcdba9cfa),LL(0x3026f026,0x6e415889),LL(0x057de03a,0x7c12c31f),LL(0xa54231cf,0xcdfae481),LL(0x68a6cb37,0x3908080c),LL(0x259ee9d4,0xa3c797b1),L_(0x0000009c), + LL(0x25864119,0x021a0d91),LL(0xd49fbe57,0xde5b21b0),LL(0xbb57b277,0x7291e7e7),LL(0x1e6a4b2d,0x16da29ce),LL(0x4426f88c,0x68f8b71f),LL(0x6a6ebaff,0x9995fbf7),LL(0xab510adb,0x6ec18d2c),LL(0x8d4b996a,0x3ce11f1f),L_(0x000001bc), LL(0x1321f3ca,0x8e04c405),LL(0x34703d79,0x6b0a33af),LL(0xdd55e68b,0xb14161e8),LL(0x4737f09c,0x57558d9c),LL(0x90c00b53,0x9d9a485a),LL(0x508e73fc,0xbb09dac2),LL(0xd252e5f5,0x4ba2132e),LL(0x33b1efcb,0xc58bf239),L_(0x00000193), +}, +/* digit=59 base_pwr=2^295 */ +{ + LL(0xaba36d21,0x5b781a84),LL(0x584291d5,0x6ea73ad7),LL(0x992c0a26,0x20e9954c),LL(0x169e02af,0x4d73d175),LL(0x2718e0ca,0xe1612ee1),LL(0xed50926d,0xc638cf1d),LL(0xc1060d91,0xb5998df8),LL(0x4b7dc332,0x4eb7dc88),L_(0x00000062), LL(0xaadf4bca,0xd78eae21),LL(0xa9f4bf2c,0x372725c2),LL(0x86c74c6b,0xb5b5158f),LL(0x736a4de4,0xba4800d6),LL(0x451f4693,0x5138590e),LL(0xd2239cb9,0x6f5d263e),LL(0x45bdc4c5,0xc0f8acf5),LL(0xd06676d4,0x8bbd0743),L_(0x00000003), + LL(0x04c169d8,0x714fe80d),LL(0x2da244cb,0x06f04145),LL(0xcda8b722,0x84ee9fa6),LL(0xc3d58870,0x0e111da7),LL(0x1c267392,0x53bb35ef),LL(0x906e57c4,0x6a858e61),LL(0x3eebaa20,0xf4582387),LL(0x20dd3b5b,0xcf71c4a7),L_(0x0000015d), LL(0x81e2955e,0x91605cdd),LL(0xe98756fb,0xcda7aac0),LL(0xf0286c4c,0xb4372718),LL(0xa4017819,0xdae0a5d8),LL(0x21935131,0x0720f8cb),LL(0x261dafa4,0x40e03217),LL(0x6fb18c8c,0x34851940),LL(0xcd3c7d48,0xe02770ca),L_(0x00000146), + LL(0x3aea03d8,0x826415dc),LL(0xaee30325,0xe70ecca1),LL(0x8395cad3,0x053fd2fe),LL(0x148ed662,0x3520779d),LL(0xad7a6345,0xc9cad78e),LL(0x02a99616,0xb2b3d15b),LL(0xa5bc3102,0x9cfe5a4d),LL(0x4cc19d74,0x59fc0a6f),L_(0x00000106), LL(0x3f02d2b6,0x3a1ba1c9),LL(0x487fa3ac,0xb4dfc9f7),LL(0xe0d152d8,0xf699a6f5),LL(0x8f345525,0x20633fdf),LL(0x76ba6839,0xca7d8a08),LL(0x2bd4f59f,0x88585003),LL(0x05078df0,0xcface355),LL(0x01da05b9,0xfd2eb0a9),L_(0x000000a7), + LL(0x5676f0c9,0x52b16ebe),LL(0xf47e5b46,0xe6ae36e7),LL(0x71c81701,0xe3a4cc33),LL(0x1ae7f8d6,0xb29431c3),LL(0xaeb29c67,0xb869125f),LL(0xe16be2af,0x052cfd62),LL(0x934ce6d0,0x10638824),LL(0xe1cd7490,0x2b021800),L_(0x000000e6), LL(0x23a9de59,0x53a39e35),LL(0x21f34b32,0xaa3b5761),LL(0xd9c36b4a,0x2e0442ed),LL(0x419b2399,0xd2725144),LL(0x4d374723,0x0ced8d6e),LL(0x6d58a708,0x1b1f1118),LL(0x0cd63ed1,0xd7a4d0b5),LL(0xf4c6faa6,0x9c897561),L_(0x00000055), + LL(0xca7de8bc,0x0cb3fdd3),LL(0x8e8db5d1,0xaee5321b),LL(0x87aea4fe,0x3857736e),LL(0x16711165,0x0c1ddf85),LL(0x0fefea55,0x5866facc),LL(0xf4b819a9,0x33eac999),LL(0xde7464eb,0x2a31e0a7),LL(0x4f70e413,0x152ce312),L_(0x0000015a), LL(0xab29e0e3,0xbf9b85c5),LL(0xba27db9c,0x153c54c7),LL(0x65beb177,0x0c1db955),LL(0x38a15bc3,0x12ad15c1),LL(0x1bbf7edd,0x99f39d44),LL(0x0cddc300,0x017014dd),LL(0x7ea43a2c,0x2d23d878),LL(0xbec5a12d,0x5eda0b7d),L_(0x00000155), + LL(0x64653256,0x06657d80),LL(0xd29c3627,0x8365a95f),LL(0xe70b9e02,0x084d6b18),LL(0xc68fa40d,0x80bbbcfe),LL(0xae56df9d,0x8a06728a),LL(0xb6253373,0xbd4a361e),LL(0xc77e1c92,0xd3c11b1d),LL(0xa94fbeef,0xa52dffaf),L_(0x000000fb), LL(0x00a47f4f,0xf19acb1f),LL(0xe668d3f2,0x23cbf024),LL(0x095d2d5d,0x7f105b84),LL(0x89f76b69,0x5b550d74),LL(0xe73345da,0xbc9d3a15),LL(0xd2b26a8f,0xd6a293e0),LL(0xe4494adf,0xe0387451),LL(0x818e6417,0xb0518331),L_(0x000001b0), + LL(0x9bcbace6,0x4d2a42df),LL(0x808fa6d8,0x68651170),LL(0x445f0d4b,0xec0d410c),LL(0xc6980698,0x1ad2d890),LL(0x005f7ee7,0xbcda7089),LL(0xa7d283e2,0xf5b48062),LL(0xcee64fee,0x0051a180),LL(0x73e72ad4,0x848f7b7c),L_(0x000001be), LL(0xfa1706be,0x690a81c3),LL(0x3ce8b5c2,0xac33d774),LL(0x15e254f0,0xa423baea),LL(0x948fbc87,0x2fe89ca2),LL(0x80cdde65,0x9a165eb6),LL(0xb1b05690,0x1c84102d),LL(0xc135f5d4,0x73f34a94),LL(0xc61329b0,0x8ced1268),L_(0x00000189), + LL(0xee3f1678,0x9713685f),LL(0x93a123f3,0x6ef5a591),LL(0xf28bae61,0x67b33050),LL(0xda7d2c65,0xae5b6596),LL(0x59fc9e4d,0x0481adbd),LL(0x18dde4a4,0x2fe92c16),LL(0xf2a19468,0x5adc0431),LL(0xeac05bc1,0xdb30fcce),L_(0x0000013e), LL(0x900c6eaf,0xee47d7a2),LL(0x4cf35b8d,0xc425ee95),LL(0x74908606,0xcdac359e),LL(0x22c94a88,0xef2c8586),LL(0x10d20bfb,0x8d0f7458),LL(0x1ca77f65,0x426741d7),LL(0x82f59f1e,0x640314c7),LL(0x294af36c,0xec0709c8),L_(0x00000035), + LL(0xcbc694b2,0x1e52ac8f),LL(0x0894401e,0x36abe923),LL(0x9f482e3b,0x81ce378a),LL(0x32d1efaf,0xae954687),LL(0x62ff86e1,0xd9afd8e0),LL(0x085da5ec,0x5871a105),LL(0xd026254f,0xe7a32717),LL(0x83b4a648,0x40288d2e),L_(0x0000004b), LL(0x3c782f28,0xb0b6dc10),LL(0xa8898440,0x699ececa),LL(0xe53262d1,0xdf9ba3e4),LL(0xdb180197,0xa82e89ff),LL(0x786042aa,0x97af8b17),LL(0x663faa6b,0x00b45e0a),LL(0xf346c12e,0x03c76f2d),LL(0xe8ec3f00,0xbda1be57),L_(0x000001a3), + LL(0xf094c184,0x55c5c739),LL(0xfcd89a8d,0xc0dcdb75),LL(0x71a4d047,0xd498ccfb),LL(0x8baef9a3,0x669e7edb),LL(0x20d8ae8f,0x98d6b13c),LL(0xd16a48c7,0x3f4ca564),LL(0x50d24170,0x509f9dc2),LL(0x8c680ef9,0xb2c17a38),L_(0x00000170), LL(0x942861e3,0x23778808),LL(0xe8d8d33c,0x9dd269ab),LL(0x9bd2fb18,0x1e8769b1),LL(0x62b11258,0x657bacd6),LL(0x8521c19d,0x584fbcaa),LL(0x46adc05d,0xe93891c5),LL(0x68fad3f4,0x8617aebc),LL(0x0857bce7,0xe39a4226),L_(0x00000101), + LL(0xfc938e83,0x17cc51bf),LL(0x6d70f113,0x546beb5c),LL(0xc94a2150,0x62ee8e73),LL(0x3d11d590,0x255999c7),LL(0x8c21f26b,0x42e22a6e),LL(0x3732f418,0xb9e01e7a),LL(0x910608f5,0xea10cdc0),LL(0xef7f9669,0xf54f17b3),L_(0x000000b5), LL(0x83ab9767,0xe994f9cf),LL(0x5a87a958,0xe54c87e2),LL(0x6e7d0b5e,0xa8cc0493),LL(0xb0873928,0x837b814a),LL(0x92c0806c,0x1e804d39),LL(0x8cb9df2c,0x1d8a5f76),LL(0x2377456b,0x33c3155b),LL(0xe6d63e09,0x55cb5f9c),L_(0x0000011e), + LL(0x4f8eb1a8,0xcfd79f62),LL(0xc5d103b9,0x295548e9),LL(0x71b65a58,0xb8462f3d),LL(0x40934991,0x1453c2b2),LL(0x41c087b5,0x1cf62fb3),LL(0x37eea50d,0x02ab6cdb),LL(0x7ada571b,0x1b49b692),LL(0xda9e677d,0xb52dc4d6),L_(0x00000004), LL(0xb839d5a9,0x2e42f171),LL(0xb6cceda3,0xdbcc6765),LL(0x09960889,0x2e336d9f),LL(0xe5440e8e,0x34cefd7d),LL(0xb410a81f,0xd8bc6b5a),LL(0x91782d60,0x46d80cbb),LL(0x95e22f30,0xc99291ea),LL(0x9e775cf7,0xca473be0),L_(0x00000082), + LL(0xcc9e1b6f,0x06ac0186),LL(0xbd98b833,0xdc7e7944),LL(0xae0a1564,0x0b94fb31),LL(0x8b85aba8,0xfd0db6b7),LL(0x3fa21f5d,0xbb92a7c4),LL(0x7133a3ce,0xfc2b2cf0),LL(0xd39f3731,0xb63fc2cb),LL(0x376e5f1f,0x4d89e9a6),L_(0x000000ce), LL(0x194cc828,0xe98c5f25),LL(0x2178e890,0x51c9207d),LL(0x1be32aa0,0x8d2cb6c8),LL(0x629881ee,0xdb210410),LL(0xe3099a6e,0xfd24a488),LL(0x62aff70d,0x6e705a1e),LL(0xb843d997,0x41319b69),LL(0x8bfa95c3,0xd9b9376a),L_(0x0000010b), + LL(0xa3e1ec65,0xc2b6a03b),LL(0xfefa851e,0x0a7bf6b7),LL(0x3a92f668,0xbf4905c0),LL(0xe8a75dea,0xff483a6a),LL(0x78467396,0xdc163b2b),LL(0x8999b6fb,0x8968be09),LL(0xc4a53538,0x419a12c9),LL(0x40b8e919,0xd87c8896),L_(0x0000010b), LL(0x81b9e47c,0x68f69e36),LL(0x5971b3c8,0x53ea08fc),LL(0xbd601db4,0x0ff01a96),LL(0xa72aee96,0x347158b6),LL(0xef1dc3a0,0xc8994151),LL(0xb70d9ea4,0x39937de4),LL(0xa2906842,0x73a17885),LL(0x1ae4276c,0x34f8bee7),L_(0x000000a8), + LL(0xfe5a5236,0x7fdb683d),LL(0x5075fbb5,0x35997d35),LL(0x4f7513bc,0xb65dda5b),LL(0xcca089f1,0x0e8d30ed),LL(0xf394427e,0x66ecf608),LL(0x7394ebe5,0x80d0cb61),LL(0x2babf408,0x9903b671),LL(0xb8208316,0xe416cdcf),L_(0x000000b6), LL(0xf1d8de96,0x26090ace),LL(0x026550ab,0xf1bd714b),LL(0xceedda36,0xd83c3071),LL(0x6a6fe427,0x5704c9b9),LL(0x9e328311,0xa3fbc241),LL(0xfceb37ff,0xf54b88ee),LL(0x2f82304a,0xb6315a8d),LL(0x77a230c7,0xeda8682e),L_(0x00000177), + LL(0x70a0b8b3,0x5759366b),LL(0xabadb724,0x75b84d60),LL(0x87caf5b9,0x983f793d),LL(0x1cecc3f1,0x94d8de54),LL(0xd8885c2e,0xf90b687c),LL(0xd952f2ac,0xd5046b6c),LL(0x16cc05b6,0x266e5bb0),LL(0x52f97cdb,0x5959c784),L_(0x000001c9), LL(0xefd1f634,0x931e70e0),LL(0xc8989bfc,0x36663528),LL(0xc244410e,0xf14f2667),LL(0xbca645d5,0xf8059cc8),LL(0x1c94e08c,0x1d2134f7),LL(0x1d1a84e9,0x3489ba7b),LL(0x21c35b98,0xdddbab1e),LL(0xa303bfc5,0x86156a8f),L_(0x00000198), +}, +/* digit=60 base_pwr=2^300 */ +{ + LL(0x6b1a72ba,0xec83fd79),LL(0x9640b910,0x5166dc54),LL(0x1f3c076c,0x553ff932),LL(0x6d2b7e4e,0x7e2f7e67),LL(0x097c11e9,0xaa2cdad5),LL(0xc5cedff5,0x397f4bc0),LL(0x57d09eef,0x8d95f667),LL(0xe7743495,0xe32eada9),L_(0x000000ee), LL(0x3f068368,0xe641f85b),LL(0xa0e3496c,0xa6b773ab),LL(0x7931548e,0x3aecbf5b),LL(0x2ef14927,0x58c42c5d),LL(0x1528b818,0xfddc70d9),LL(0x1c157c1e,0x0a328d34),LL(0x690e10df,0x0760fd8a),LL(0xa364f5d3,0x4ec3b44e),L_(0x000000d1), + LL(0x572bb6c8,0xb187f011),LL(0x26f1e48d,0xb62010d1),LL(0x0d05c4b7,0xe0fbe5e8),LL(0x3f0a2ba1,0x24d802f1),LL(0x189e9602,0xde0b8698),LL(0xc0b8af43,0x28591b4b),LL(0x706b742b,0x36bf7aec),LL(0x63d91963,0xec0d2fd3),L_(0x0000007f), LL(0xe7179b1f,0x440b0b6c),LL(0xe982e3e2,0xd99d67cd),LL(0xd814a6fa,0x7edfbb4d),LL(0x4ac7d349,0x46c6afc8),LL(0x63bbd77a,0x84cd907a),LL(0x18bcc3d6,0xf098909a),LL(0x756b5193,0xd6e0581d),LL(0x02f37ab5,0x06adf4d1),L_(0x000001fb), + LL(0xf4b1ce28,0xe90415e4),LL(0x6bd8544c,0xc664d2f1),LL(0x3e7600d6,0x01912f05),LL(0x1e8d9aa4,0x3e0c268e),LL(0x43dcdadd,0x2f140134),LL(0xbf936e21,0x252cf59c),LL(0xb8aaec39,0x00b8cef4),LL(0x7ed652c2,0xc64c11e4),L_(0x000000aa), LL(0x7a2dfe0a,0xa7c08e2e),LL(0x59176893,0xafe8a484),LL(0xc7c5a8f0,0x6f043d0d),LL(0xe33e999b,0xb3c3cfd6),LL(0x2a496c00,0xcb4fbc5e),LL(0xe2690de5,0x1dfaf5ee),LL(0x3d2db451,0x743e9277),LL(0x8bfecdd2,0xade6a194),L_(0x000000f4), + LL(0xcadf2116,0x8d039609),LL(0x488fdf25,0x1e037339),LL(0x0e39945a,0x4d16fa60),LL(0xf539a844,0x54408c23),LL(0xf7f8ccc4,0xbef729fb),LL(0x69b8abc6,0x76661fe3),LL(0x3dbd87a4,0xa8903415),LL(0x790f6266,0xc7b4bcd2),L_(0x000001e3), LL(0xec9aa47b,0xa406e205),LL(0xce1af477,0x72e7763c),LL(0xf7fcf645,0x43d00999),LL(0x1f7c9317,0x02adccdc),LL(0x8b87a139,0x541be26f),LL(0x9ffcb96f,0xbb590677),LL(0xc0636264,0xc0db1d61),LL(0x19484331,0x5585c8ae),L_(0x0000016e), + LL(0x77cc15c1,0xe85899b3),LL(0x5fff92e2,0xb4a44f31),LL(0xa8cf599e,0xca1a3e87),LL(0x17a2e4cf,0x205f34e2),LL(0xe4f28f3f,0xe9eb1362),LL(0x4aa7c205,0x6ace61c8),LL(0xfa76515b,0x7e550586),LL(0xec83aca3,0x35e870f8),L_(0x00000104), LL(0xd113de7a,0x2e02203e),LL(0x318327d4,0x004d9655),LL(0x5d903904,0x7cd1d2f1),LL(0x9d294adb,0xca1242c8),LL(0x3b5bb4eb,0x4af2a142),LL(0x93818e82,0x15e366f3),LL(0x9a304441,0xd6a53de7),LL(0x49183b2c,0xfd324d82),L_(0x0000007b), + LL(0xdb2cf29c,0x4d1ee196),LL(0xa1d04903,0xe53d6718),LL(0xacaf386a,0x6605e4e6),LL(0xcc306d74,0x458e136f),LL(0x77dc40c4,0xab7a1ac6),LL(0xbb331955,0xd47f6ee4),LL(0xb95c386e,0x43841037),LL(0x526640fb,0x00a37bb7),L_(0x00000010), LL(0x8f1bcdb4,0x8b8cc55b),LL(0x71d84cd7,0xe7e03251),LL(0x1b8eb12e,0x64c45f59),LL(0x2d7bd8c5,0xc2283df1),LL(0xb03bfc76,0x28a36461),LL(0xafd6fc81,0xa0580c8c),LL(0xe72b6275,0x511c376a),LL(0xce438282,0x0ca3213f),L_(0x00000078), + LL(0x353cadbb,0x1e2c0f6f),LL(0x17179cb8,0x0810156c),LL(0xaf140db0,0x63cd1fa8),LL(0x2171bb5e,0x729533d8),LL(0x544c77b0,0xf6676827),LL(0x0143af56,0x599efe1b),LL(0x5a759df2,0x3accffd4),LL(0x55962b59,0xc61321e5),L_(0x000001a3), LL(0xa5358dc7,0x66e58a6a),LL(0xbb4d42d5,0xe0cfd739),LL(0xe6ef3760,0x0620ef46),LL(0x6804fb37,0x253e0f9e),LL(0xf4e9cdc2,0xcb5c8c64),LL(0xec6f6658,0xa80d08da),LL(0x04153037,0xcc959be5),LL(0x3d215b47,0x7aaaa865),L_(0x0000004e), + LL(0x166381e4,0xf52bc233),LL(0x9700029a,0x2a837019),LL(0x3201dddf,0xd02e2c74),LL(0xd8885cda,0xaea6cd36),LL(0x7e35126d,0xbc3f784e),LL(0xfa40cea6,0x5130e882),LL(0xeec16a3a,0xace2f121),LL(0x13c43706,0xf3c3a16a),L_(0x000001cc), LL(0x9e7a6c25,0xc964823f),LL(0xe8e4729a,0xeed8d7ba),LL(0x3fe54edf,0xcfba42bd),LL(0xec6a4e7b,0xe917bf88),LL(0x06be2039,0x604163c6),LL(0x17d5a63a,0x6bdf09b0),LL(0x276869bd,0xf021410c),LL(0x4b94144f,0xf038cdd9),L_(0x00000013), + LL(0xf09a7d9b,0xeb8a6d24),LL(0x724db1ba,0x545ff43f),LL(0x850f42b2,0xfcaf8079),LL(0x5c1fdccc,0xde18c209),LL(0x57404da7,0x83097de7),LL(0x267842f8,0x8706015d),LL(0xab9a893d,0xe62c08dd),LL(0xe2b0c7a0,0x736bf358),L_(0x00000142), LL(0x254c1866,0x535f7766),LL(0x4779dd4d,0x10a98c32),LL(0xc095243e,0x0e7bc245),LL(0x3cd82df8,0x5fac69de),LL(0x59efca16,0xa2f0af19),LL(0x692e1ddd,0x40c91226),LL(0x9b21f9d2,0xa682e04c),LL(0xa51cafc2,0x831c0e79),L_(0x00000049), + LL(0x6101ea3e,0x94e4798e),LL(0x00af5508,0x9afdccb8),LL(0x987426d2,0x0f5f64d2),LL(0x4aaeb57e,0x76899b88),LL(0x9a8859b4,0xcf38ab59),LL(0x31a64817,0x8dc36916),LL(0x9b757c1f,0x28b27539),LL(0xfb6a189c,0xc63802b8),L_(0x000001ba), LL(0x997a98e3,0x225f33e4),LL(0x9d6bb39a,0x880025e5),LL(0xffb62ebd,0xb7d05691),LL(0x6aacd544,0x3e434b4f),LL(0xba8454f9,0x4bc06244),LL(0xbe5d3fe0,0x941bd419),LL(0x6732d1a1,0x6794ada7),LL(0x6efab77e,0x1058a767),L_(0x0000008b), + LL(0x7d41eaad,0xe58d6aac),LL(0x5f7430b9,0xb9ab109f),LL(0xedf696b8,0x252205ff),LL(0x65163bdb,0xb3a47496),LL(0xfd8eca02,0x915f0458),LL(0x6d6b0ec4,0xb0096e8b),LL(0xddddf89f,0x874c9e5e),LL(0xab9c669c,0x156cfc6b),L_(0x0000019a), LL(0x80d21354,0x8c6752e9),LL(0xc103803a,0x0196650f),LL(0x653fb161,0xe597cf0d),LL(0xa211c0cf,0x66902c1d),LL(0x126bc8cd,0xff4436dc),LL(0x502c8df4,0xcf0ef89e),LL(0x30d9c137,0xadabc266),LL(0x509eb349,0xcbdaa030),L_(0x000000ae), + LL(0x80ee5e16,0x9d067e75),LL(0x0846b23b,0xb348d35d),LL(0x6044d60f,0x4e0c74fc),LL(0x4437fb47,0x2124f846),LL(0xc898c89a,0x30abdb7d),LL(0xa1ecdf1d,0x496e747c),LL(0x09be44bd,0x9381d368),LL(0x6a34a28e,0x2cd1e7c2),L_(0x0000002c), LL(0x21c47e8f,0xece95ae5),LL(0x705df30c,0x08cdb28f),LL(0x8f6f90ca,0x15b2caca),LL(0xdb9f29f0,0xf8e2597f),LL(0x537c5e8b,0x91e195ad),LL(0xa54bf828,0x1ef3bc47),LL(0x5f5b1233,0x6c98e99d),LL(0x096c5cfe,0x0acf39b3),L_(0x00000155), + LL(0xe7e060d2,0x5e49ea9e),LL(0x8278d27c,0x1462a5cf),LL(0xb6b188b1,0xd1c3e723),LL(0x7adfe4be,0x9f4ab12b),LL(0x4f74656e,0x86f9e399),LL(0x15f9fd6f,0x647aabe4),LL(0x356f176c,0xdc05df56),LL(0x037f39e8,0x6f8c9681),L_(0x000000ab), LL(0x7fabb630,0xfd586565),LL(0x4f9b62dd,0x9a07decc),LL(0x07d1665c,0xb14d451c),LL(0xb6af032b,0x2d75e1e7),LL(0x7c7044cc,0xac2df8ec),LL(0xcac8a69f,0x0906e20d),LL(0x20c70582,0xa4167466),LL(0x1ca47052,0x6fe2b8c1),L_(0x0000002e), + LL(0xc222cb55,0xfb8bf3d9),LL(0xcaac45d1,0x39bf904a),LL(0x4d2bf532,0xa7f1203e),LL(0xa3246448,0x9ecdf560),LL(0x6d3940b2,0xa6d3b3a3),LL(0x89bdcf57,0x40dcc76f),LL(0xdcc95c73,0x9fa5791b),LL(0xa442ea24,0xc94b40b3),L_(0x000000fd), LL(0xbfbd8ae2,0x8473e7d8),LL(0xbc97c4e6,0xe17e4cb3),LL(0x8daa65ee,0x1d315fc6),LL(0x20c6ff94,0xeaec9cae),LL(0x74f3ffd6,0x1b61c251),LL(0x015f6f88,0x6a1cf873),LL(0xee694708,0xd187e0b9),LL(0x37da53ba,0x332ba61a),L_(0x000000b9), + LL(0x44bad1e8,0x409292b4),LL(0x27a8575f,0x23606aaf),LL(0xeb1afe84,0x6df6bd82),LL(0x546ebac4,0xd77cd802),LL(0xd19a6a0e,0x9ac38a98),LL(0x701bcf92,0xae1c0504),LL(0xd6247bc1,0x90bada7a),LL(0x7a406e08,0xcc3c49ab),L_(0x000001ac), LL(0x1a859f6a,0x5dcf7e44),LL(0x5f7352bf,0x2b674d19),LL(0x0ee31d4e,0x35a3e68d),LL(0x92916c56,0x5e153a5d),LL(0x1bbba324,0xa3d55e06),LL(0x27d3691c,0x74f6e553),LL(0xa9153a98,0x2c8a7473),LL(0x6e737b68,0x545acbac),L_(0x00000114), + LL(0x741a9412,0x892f8bd1),LL(0x31776371,0x6c0b1d4f),LL(0x60a2c8b1,0x9a688a19),LL(0xa0d12ea4,0x56eef3ec),LL(0x4746c345,0x90c83381),LL(0x842db71e,0x3fdb8d8b),LL(0x1bf4ae9e,0xbc576b9b),LL(0x9e0ee706,0x85a8de64),L_(0x0000014e), LL(0x9f19edfc,0x3c32799b),LL(0x25980592,0xc4760975),LL(0x76f95241,0xd6c8d637),LL(0xdd3d3b18,0x677dee0d),LL(0xe5bceafa,0x12ad9334),LL(0x7ca46478,0x3990ceb2),LL(0x94d56dac,0x39d6e555),LL(0x8e338deb,0x7a9d83d7),L_(0x0000016c), +}, +/* digit=61 base_pwr=2^305 */ +{ + LL(0x14b21476,0x52d7b490),LL(0x4215e0ae,0x3a781f13),LL(0x84df8cb3,0xaf858985),LL(0xc3ab10bf,0x449e13a9),LL(0xea1a4d65,0xb859368b),LL(0x2c8134df,0xb51fd78c),LL(0xa7d94439,0x4b1e6f5b),LL(0x003f1c75,0x4cf5d47b),L_(0x00000146), LL(0x3cdb3bab,0x6dc28640),LL(0xd6597a8e,0xe3d2ac15),LL(0xe5e853fb,0x8bd192da),LL(0xf804c989,0x76032d13),LL(0xdb66ee3c,0x738d5e8b),LL(0x349bcb16,0xe16c7ea1),LL(0xe4001679,0xd22ec201),LL(0x82f1f584,0xd9f317a9),L_(0x000001b7), + LL(0x92fe026b,0xec0a67f0),LL(0x2efbca10,0x3767ee14),LL(0xcc433a34,0x68131944),LL(0x370f6b9e,0x36fcc884),LL(0xb4d1b5e8,0x85328231),LL(0x85b956e6,0x3e4b895d),LL(0x17afd7ce,0x23cd96a1),LL(0xfc28a48e,0x6cbf1cc4),L_(0x00000175), LL(0x8d0ddeea,0x5cc45bb9),LL(0x4201b856,0xbec2c277),LL(0xb4f52020,0xcd76ab62),LL(0x3aabda65,0x2fb221ac),LL(0xd348e9ae,0xb7ba962f),LL(0x81d5e875,0xacdde7c3),LL(0x8ae119ed,0x7186eb96),LL(0xd5b495d5,0xdf795bd0),L_(0x0000016f), + LL(0xa9d725f4,0xabe42367),LL(0x83477d80,0xae655393),LL(0x9bf84781,0xf1389d3f),LL(0x50d527ea,0x02ffac63),LL(0x30005241,0x5a3b0583),LL(0x332af83e,0x30a51cf3),LL(0xd633aac0,0x7e87b5f6),LL(0x6133508c,0x54cf5544),L_(0x0000014e), LL(0xddfa61bb,0x79f352eb),LL(0x3d5e304a,0xc8a8525e),LL(0x673478be,0xcd082890),LL(0x65272acf,0x6528a7ef),LL(0x07746b44,0x6120a7a6),LL(0xaa126f2d,0xed8dc8aa),LL(0x0714c411,0x242ecc1e),LL(0x09219322,0x4dd29b99),L_(0x0000016f), + LL(0x2f54f5ef,0x9d1e3e79),LL(0x6e1c349e,0x7cb862d6),LL(0x411c782a,0x774f6f73),LL(0xf914b067,0xd88b7029),LL(0xbe145ff3,0x68ac9342),LL(0x0730a2fc,0x77dcfba1),LL(0x7ace014c,0xe34f0621),LL(0x876ebecf,0xb7a85b90),L_(0x00000104), LL(0xd99da4a6,0x2be45d39),LL(0x2af68cfe,0xeda14612),LL(0xbeab553a,0xc8cf47bb),LL(0x185338ec,0x9f26575d),LL(0xbcf5707a,0x2dafc93e),LL(0x9b4f2615,0x85006b56),LL(0x1c517096,0x58e4408a),LL(0x2759575a,0xa451b6b3),L_(0x00000143), + LL(0x03077a78,0x64081168),LL(0x9fca1732,0x9e2e68b9),LL(0xc2ec6027,0xbd01c53b),LL(0xcf8e3aa9,0x3c299cf0),LL(0xc31ff566,0x69a934da),LL(0x3a869b7d,0xee4c3bf3),LL(0x17fb711a,0xe353eab9),LL(0xb5b7fc05,0xd300c851),L_(0x000001da), LL(0xb6667f8d,0x7326c782),LL(0xf868e4b0,0x7616e981),LL(0x9cbec832,0x0d0b19fb),LL(0x0355a1b8,0x7504ef78),LL(0x9b3d9f50,0x75e429da),LL(0x0924def0,0x130ecd97),LL(0x07187605,0x844d6f96),LL(0x7c14ae9f,0x8921d3a7),L_(0x0000001b), + LL(0x6ef420ae,0x9415e0cc),LL(0xdd321662,0x7be013aa),LL(0xad261af9,0x46c47707),LL(0xedc263d9,0xabc20130),LL(0x2f265a39,0xe142f5b7),LL(0xcfeec142,0x90dba064),LL(0xa08536a7,0x488a0175),LL(0x6d419631,0xcf748207),L_(0x0000003f), LL(0x59c2a2ed,0x6def509f),LL(0x7ce6774c,0x4af30ccf),LL(0x09bc7469,0x3bf7cbb3),LL(0x12464f68,0x648dc8e6),LL(0x93c3b96f,0xb39085a1),LL(0x1000d207,0xe4a3e7cf),LL(0xcb93a762,0x39e62f3d),LL(0x08cde0be,0xd6284f5b),L_(0x00000092), + LL(0xcef51b82,0x4fd9fb1f),LL(0x0a484ffb,0x4a91b446),LL(0x5c5120d6,0xcfe786b3),LL(0x521a227a,0x1f347861),LL(0xb4d52b1b,0x005afae4),LL(0x28a0b22b,0xaed64316),LL(0x96565ad0,0xcf8e1f9b),LL(0x3bd3818c,0xd717a7fd),L_(0x000001d7), LL(0xf05313af,0x17132d83),LL(0x4d75e2a9,0xf9bc8f55),LL(0xbaf9947b,0x91f937e7),LL(0xcb9bb75f,0x232e92ab),LL(0x4c1c8f93,0x92cbf962),LL(0x90cd09cd,0xf2dcee1e),LL(0x07dbfe55,0xb89e680d),LL(0xc41cd340,0x51007568),L_(0x000001b3), + LL(0x652aa77d,0xb831eea8),LL(0x9b33eda2,0xdd1ac75a),LL(0xccb42fd2,0xe55769fc),LL(0xa865ab8f,0x2ea2c9e4),LL(0xc60208ee,0x28effb93),LL(0x321dff3f,0x2d6b1522),LL(0x5124fb78,0x6e29dc3e),LL(0x4cf961e9,0x2f39193c),L_(0x0000006d), LL(0xe3174790,0x035e63b7),LL(0x67da851f,0x7e9f39d6),LL(0x7183aa79,0xe886c75f),LL(0x4aa59f9f,0x6d9e6857),LL(0x706045d9,0x2ee25277),LL(0xb18ceb8f,0xe4bcaa94),LL(0x5bc3971a,0x57fc8f0d),LL(0x7b7b6081,0xdd642848),L_(0x00000150), + LL(0xff257250,0x44c8a327),LL(0x1f4b8713,0xa7ee221a),LL(0x5258be1d,0xe41af48e),LL(0xa737a8cd,0x83aeee1c),LL(0x3f320ac3,0xbb3a2bdf),LL(0x8abc18a8,0x2bfe7f09),LL(0x9da43962,0x5ab55046),LL(0x1318b08d,0xc7f6a7c8),L_(0x000000c2), LL(0xe5844c3c,0x072464c5),LL(0xe109557b,0x52cb5223),LL(0x2e18e586,0xbb44cf23),LL(0x58cf033e,0x8273746f),LL(0x9cb3f3f2,0x15027b4d),LL(0x0badc23f,0x03dd0534),LL(0xebde0563,0x94ef00b2),LL(0xf31a2a6f,0x88f86782),L_(0x000001e2), + LL(0x56b13d17,0x5644a483),LL(0x012e151c,0xec46d8f5),LL(0xcb60c92d,0x61f8e693),LL(0xf704143e,0x53579b44),LL(0x06b44ecd,0x98807645),LL(0xdda8c89f,0x17c64951),LL(0xfaab400d,0x2e39e25a),LL(0x16b86130,0xfef9c912),L_(0x0000004f), LL(0x25210623,0xd2367ca5),LL(0xf18ebddc,0xbcf685ba),LL(0x59f6d4cf,0xe159807e),LL(0x8c3c8195,0x2bb8d624),LL(0x02e20259,0x3ad24a15),LL(0x5eef3266,0xe9a952f5),LL(0xc8d0e08e,0x37d37845),LL(0x4cf4addb,0x778df76b),L_(0x0000002e), + LL(0x7a1441af,0x55047df4),LL(0xa605ea07,0x579060a2),LL(0x728d81cf,0x3b8900fa),LL(0x1ccc1e61,0xf49f8c7a),LL(0xd633ce10,0x4957105f),LL(0xe467c698,0xc3e2024e),LL(0xa14d0dd3,0xe7cc2be3),LL(0xc90176ba,0x00090c73),L_(0x000000f8), LL(0xf9360056,0x39d8bbe5),LL(0x229fde87,0x1982808c),LL(0x9d5d0ade,0xb83a93a2),LL(0x56715396,0x4130f493),LL(0xf5d0d1b5,0xc39ee248),LL(0xcbf57700,0x0662cf56),LL(0xf41ee620,0x121da851),LL(0x2397e72c,0x1ab413bf),L_(0x0000017e), + LL(0xdd8cd85b,0xbdd73dc1),LL(0x1f7a793b,0x7252dfce),LL(0xb3c777ad,0x1ad25f35),LL(0xd306f90f,0x314c1227),LL(0x4538596b,0x28e31145),LL(0x15f73822,0x1808f8a9),LL(0x5e4847ef,0x6eb8175d),LL(0x9d57409f,0x2ae642ae),L_(0x0000011d), LL(0x6debc205,0x3f28e667),LL(0xa45a1d7d,0x84954816),LL(0xe5f147ca,0xccfb8bd1),LL(0xcd78f915,0x9a693642),LL(0xb02b310f,0xb6cb5362),LL(0x1f01047b,0x7529c74d),LL(0x81d1fb13,0xdf4ae21d),LL(0x80b9dd94,0x6a1afeec),L_(0x000001bc), + LL(0x7bfa4703,0xc0abf15f),LL(0x81d957db,0x65ae7f67),LL(0xe0fe8725,0x40e4566b),LL(0x7c42febe,0x6340b7ff),LL(0xcf060fa6,0xea1d2782),LL(0x9a689bd9,0xb66eed98),LL(0xc45b992d,0x8f5646a0),LL(0x969dc412,0xd272048a),L_(0x000000d2), LL(0xc0243059,0x50900d7f),LL(0x701c6e38,0xaad1803d),LL(0x92065d64,0xf9668fa4),LL(0x361ef75c,0xfcc216b4),LL(0x962eb248,0x575be56a),LL(0x89b9828a,0xfde9ba30),LL(0x202a575f,0xc435f2ce),LL(0xba3890aa,0xf83734f6),L_(0x0000019a), + LL(0xdd8bb48a,0x3ede16d4),LL(0xa59cdc00,0xd0de9f29),LL(0x9f3b7991,0xda7b6269),LL(0x832ec0d2,0xf2e16e2b),LL(0xa8e7c828,0xd0c41727),LL(0x7f0878f9,0xc4546447),LL(0x7356692c,0xc4af90b2),LL(0x5fbe130e,0xaa2e9ec4),L_(0x00000190), LL(0xa8a02409,0x0074a183),LL(0x16d0ccbe,0x351544e0),LL(0xf7b675ff,0xf37d43c1),LL(0x87055e7e,0xc371f0a0),LL(0x3e668989,0xefcfca1d),LL(0x8323227b,0x36507d20),LL(0x38f76084,0x25498782),LL(0x75a23d95,0xacb8cb75),L_(0x00000154), + LL(0x1d79b659,0x20886bbb),LL(0x9a6dec74,0x296f5cd9),LL(0xfd24a18e,0x6092fe28),LL(0xbb4a7907,0xdeab539d),LL(0x869a8ccb,0x67f524d1),LL(0x61521c17,0xbbe3aaa1),LL(0x5f79a2c2,0x8be17e72),LL(0x7d8ce0cb,0x647da5af),L_(0x000000d8), LL(0xd52aec82,0x0f2ab363),LL(0x6d93da70,0xdfde9d3e),LL(0xa76d10f8,0xe72ce040),LL(0x17308d11,0x075467a8),LL(0xed2aabb3,0x9aa69a1d),LL(0xc10f78d3,0x4caac399),LL(0x13d4d378,0x7e54c473),LL(0xda4d8f8d,0x911cb804),L_(0x000001b0), + LL(0x9aa77765,0x253f45b8),LL(0x2af99e2d,0x112b491f),LL(0x76414d1e,0x4a8e2b12),LL(0x1c380001,0xa17691a7),LL(0xb0f6f9ff,0x4bd4233a),LL(0xcf4e764a,0xccb8bd49),LL(0x012735f0,0xc7fc0714),LL(0xf6037d3d,0x3da811dd),L_(0x000000e0), LL(0x60d3228c,0xf24a8fb9),LL(0xb39e1a42,0xa3c4048e),LL(0x07ea05b5,0xd581bb93),LL(0xfa0b7bd9,0xeeb6ce36),LL(0x8b7b8d8f,0xa0329bb2),LL(0xfcc8cab6,0x5c44e608),LL(0x04c03b08,0x3f2c4c95),LL(0x51593cce,0xb2ef5a3d),L_(0x0000013b), +}, +/* digit=62 base_pwr=2^310 */ +{ + LL(0x2fe5424a,0x052a5d7a),LL(0xbd77fdcc,0x91e67c5a),LL(0x7ae80845,0x30c17511),LL(0x4fb64810,0x92e55b53),LL(0x6c21b31f,0x96bd1eba),LL(0x4d056aef,0x3bd89651),LL(0xf597cfd9,0x39ce9f5d),LL(0x10c4de29,0x6737cbaf),L_(0x000000ef), LL(0xf5b4aa6e,0x726ccf17),LL(0xa070d7b2,0xef8a249b),LL(0x33084cb0,0xfece71d7),LL(0xdbf18fa7,0xf11b328d),LL(0x8e7f8fe7,0xdb8c5b89),LL(0xc5842d33,0x38ef699b),LL(0x44b71419,0x619477d9),LL(0xcd39a13d,0x28db36f5),L_(0x00000152), + LL(0x95d239ec,0xae5a71e3),LL(0x160db974,0xb267126a),LL(0x4df55ba5,0xf2bfd214),LL(0xcf291fe0,0xe4215d39),LL(0x3dc0a627,0x8849498b),LL(0xfec311ed,0x5b220c7d),LL(0x9fbb5099,0xa3d83cc2),LL(0xc55f9ca4,0x32f62dd6),L_(0x00000053), LL(0x69ec48f2,0xe73278db),LL(0xcaebf5d4,0x38b01c56),LL(0xe4ab979b,0x7e210f66),LL(0xfe305e1e,0x00e35bf7),LL(0xbbd247e8,0xf41625bd),LL(0x64eabbca,0xb3c01407),LL(0xe49d3fb6,0xc31a840a),LL(0x6ebed09d,0x6c67185e),L_(0x0000004a), + LL(0xc455f76d,0xeea3bb5d),LL(0xa7efe273,0x382ccbad),LL(0x1d1fd154,0x321aecf3),LL(0xba4c80f8,0x3a3eb329),LL(0x44874ee5,0xfc744e55),LL(0xc89ec973,0xd83775b1),LL(0x9ac52665,0x7c8cecd7),LL(0xe149472e,0xffa02e1a),L_(0x00000119), LL(0x10c59504,0x4863bc6f),LL(0x5e342dcd,0x30b568ee),LL(0xba377da7,0x61a3cd5a),LL(0xdb7394c9,0x7e13d011),LL(0x655ca62a,0x531b03ef),LL(0x687df8b5,0xa07d97a8),LL(0xc1cc63e0,0xc3579f84),LL(0x4f51c0a2,0x1f68d107),L_(0x00000159), + LL(0xcedb78e8,0x73976185),LL(0x5ac29ab9,0x1049200d),LL(0x5376ef50,0xb7cabe96),LL(0xb29fcfde,0x2ebeaa6e),LL(0x849702e8,0x9856863e),LL(0xd5820c1d,0xadb32b7d),LL(0x0b85b8b6,0xcb2a1da8),LL(0x4ecc2beb,0x911240a2),L_(0x000001cb), LL(0x6471f428,0x8e9339d6),LL(0x65738f28,0xc9389868),LL(0xec9ab31a,0xb78b477c),LL(0xb756dfc0,0x2531d4c9),LL(0x2bea7bd2,0xa957b1f7),LL(0x19668750,0xb7acf908),LL(0x23544082,0xfa97aa90),LL(0xd310dd35,0x7c9376d4),L_(0x00000065), + LL(0x03c1f949,0xe690991e),LL(0xb53d0f3e,0x7a8ed401),LL(0xe7688a48,0xc975d343),LL(0x0a163c7f,0xbbe320a1),LL(0x3c38f3b4,0x637ba641),LL(0x0cc94ae6,0x2fa7a438),LL(0xf036a2cd,0x85cf8f8c),LL(0xdd8d3d2d,0xf3dec45f),L_(0x000001f7), LL(0x0257d9f6,0x0fc0fe23),LL(0xf9f35894,0x22cf1f3c),LL(0x5367a382,0x0a85bdb7),LL(0xa486155c,0x43d0dc60),LL(0xa045fb49,0x28e031ef),LL(0x239e6d10,0xbc646dab),LL(0xf3c58cd8,0x37a252ad),LL(0xa190c29a,0x729ace13),L_(0x000001de), + LL(0xaa492173,0x2c2d00f5),LL(0x309124ad,0x92f17a73),LL(0x90896f6d,0xd107aa3f),LL(0x6655b0bf,0x28ed385a),LL(0x4393b8b0,0x64efc785),LL(0xa72dcb01,0xdc6d4959),LL(0xedcf6e0d,0xad09fb16),LL(0xa138cb63,0x5a264a29),L_(0x00000038), LL(0x11888849,0x23fa857e),LL(0xa4afca8f,0x362d992a),LL(0x4718e360,0xe0e7ef99),LL(0x51da204d,0x0a263a3e),LL(0x76d92100,0xc54159bd),LL(0xb90bd792,0x6992a7d6),LL(0x2d4d5792,0x34429060),LL(0xea9796c5,0x2d91640d),L_(0x00000104), + LL(0xd8036123,0x69a4e57c),LL(0xd8256cea,0x1bf79944),LL(0x4d134e77,0x4e8b215e),LL(0x63a4641d,0x83621b34),LL(0x8da5f102,0x530939c0),LL(0x9d6baa6f,0x78356025),LL(0x0a919eb7,0x9cebfe30),LL(0x523c04c9,0xba70fc3b),L_(0x00000060), LL(0x8a6eb39c,0xb404acda),LL(0xbeff381e,0xf36c4399),LL(0xc6bfdda5,0x193ff430),LL(0xddaf4961,0x43e642a9),LL(0x86bb6b08,0x4ebe4623),LL(0xd3326377,0x8dc4af24),LL(0x33ce6709,0xb168c749),LL(0x3757e6ab,0x451bf0a9),L_(0x000000f7), + LL(0xe12c9a10,0x95d393a7),LL(0x15af1e76,0x09f6c873),LL(0x5dad48c9,0x168b010a),LL(0x03c65a7e,0xd86fdc56),LL(0x73f51c26,0x88f52d53),LL(0x697c8b7d,0xbc64a497),LL(0x670982de,0xaf7a0676),LL(0x809f942a,0xb15cc57a),L_(0x000000be), LL(0xfd456a48,0x71728397),LL(0xf5563ef9,0x305f3c8f),LL(0x4e73a2dd,0xa80ae4a1),LL(0x828cc516,0x2258160b),LL(0xe74db735,0x108533e6),LL(0x14ad6801,0x3b320283),LL(0x541598a0,0x763ab107),LL(0x56c3d815,0xf632644f),L_(0x00000089), + LL(0x633a1213,0x3fb5de8b),LL(0x8bc4deb0,0x8d93c4e8),LL(0x1e8e7ab9,0x3dd24d9d),LL(0x201baf56,0xcada68d7),LL(0x0a384ece,0x503d4f19),LL(0x5dcc59f7,0x6763d7ad),LL(0x7849c18f,0xc66f3753),LL(0x6951161c,0xfc052118),L_(0x00000104), LL(0x0fd16654,0x90fd23ab),LL(0xfbb20d46,0xd8a0eeac),LL(0xd979406d,0x508b0789),LL(0xeb2d48ad,0x8cad1e65),LL(0x2f16458c,0x7615ee48),LL(0x8941144f,0x2d4a611a),LL(0x57baf847,0x706729a1),LL(0x04864e43,0x13b7d8ff),L_(0x000001c9), + LL(0x49e14cac,0x6b13d691),LL(0x15c5966e,0x5adf4806),LL(0xe79886a9,0xb44e7b28),LL(0xc0149ae3,0x14ea5297),LL(0x3f2176d8,0xd637170e),LL(0x3d5f7f20,0xfd66e46a),LL(0x5f76d12c,0x998ccf72),LL(0x8fbfc2d6,0xc2738301),L_(0x0000007b), LL(0x790af7d8,0x48202d24),LL(0x5d1ab080,0x22169c9a),LL(0xf44f3ef3,0x4f9cc0b4),LL(0x5a6ea1fc,0xd8a38b0c),LL(0x1e3f8f7c,0xfc2a4b0f),LL(0x2f6b7ea1,0x85236ace),LL(0x7c4797d9,0x507ad976),LL(0x30db4704,0x70b62118),L_(0x000000d0), + LL(0x27eeb11a,0xa8e006d6),LL(0xa8350ceb,0x0b5d40f6),LL(0xd0476f3e,0x7d6beb64),LL(0x8a7277db,0x9a1052c1),LL(0xb78ba330,0x6fb67a25),LL(0xa921f295,0x937d5f7f),LL(0x58e2fb78,0xb3c5ee8b),LL(0x224a8a6c,0x3ba51856),L_(0x00000158), LL(0x10433f3b,0x472c8eee),LL(0x46bd4fc5,0xd6bbe5d7),LL(0xbccb9c2d,0x8704f8a7),LL(0xd4145962,0xf0c09b77),LL(0xe9ce9fc8,0xe24e89e3),LL(0x091189c9,0x34dfd23c),LL(0xa0008822,0xddeaf170),LL(0x43b08954,0xe569f253),L_(0x000000b2), + LL(0x20ee092e,0x21969535),LL(0xd200f675,0x1aa95306),LL(0x8a20dfb4,0x450070d5),LL(0xe56ecbdc,0xa73c2aa2),LL(0x93697944,0x8cf15e09),LL(0x2bf1cc5f,0xb81e3982),LL(0xa98dee98,0x39d2614b),LL(0x4249763f,0x88bf80d0),L_(0x000001ce), LL(0x4194f3d6,0xea90be49),LL(0xeb5f7526,0x9d76e09e),LL(0x42892f62,0x665e7661),LL(0xdd2de6b9,0xdb45bef0),LL(0xe66edde4,0x0f0c29ed),LL(0xd947a3fe,0x39bccdcb),LL(0xdc0bb667,0x97600929),LL(0xeeaa185f,0xf355b62f),L_(0x00000198), + LL(0x05622c5d,0xc2d3d21b),LL(0xbe07feb4,0x9e1d3138),LL(0x90b99ecf,0x358fe997),LL(0x2ec0a3d5,0x95008edf),LL(0x7f72a6c8,0xc6e6cc84),LL(0x2b8ec523,0x351d40b6),LL(0x10aa3646,0x30961dc3),LL(0xdff38b1a,0x6d776cf7),L_(0x00000047), LL(0xafa6b1f7,0xfce248cf),LL(0xad217997,0xe663a7a1),LL(0x423b10cc,0x8d65dc51),LL(0x0215b195,0x0f10bc35),LL(0x0f4e07a2,0x23278029),LL(0x19d23499,0x304b98d9),LL(0x6127a2da,0x9fb0c81f),LL(0xed0c0943,0xcd486835),L_(0x000000c4), + LL(0xaf631c28,0xd0ac0ecd),LL(0xbaefab89,0x0a9db571),LL(0xe4775843,0x6c283a9c),LL(0xfebcf91f,0xd37751b1),LL(0xe02d1251,0x4c69aef4),LL(0x93ca62f3,0x756b2ab1),LL(0x017751a3,0x9921e7da),LL(0xf0df5f26,0x31fbf868),L_(0x00000112), LL(0x1547e61c,0xc24912f9),LL(0x6ce422f4,0xe5bc3bbe),LL(0x0c518f2e,0x00e5a237),LL(0x51ed5f7e,0xb6da428f),LL(0x1a77cbdb,0x248c6951),LL(0x086ad3a0,0xfd285428),LL(0x00d65807,0x460a5bc1),LL(0xc6265db2,0x728547e4),L_(0x000001fe), + LL(0x378f0a8d,0xcdbb56b7),LL(0xea484f2e,0xebecf09a),LL(0xbe1705e2,0xdc0d7050),LL(0xe83a83b6,0x5c8fdff7),LL(0x1cd41a57,0x58f038ee),LL(0x975aeb28,0x858f75ce),LL(0xddbb66ee,0x7455106f),LL(0x7e1bcafd,0x54e1961f),L_(0x00000128), LL(0xd2d34020,0xc329e633),LL(0x32cfb8ca,0x596dc91e),LL(0xe8fb4aa3,0x19c60dcd),LL(0x0c27fe63,0x9c2411d9),LL(0x49228e82,0xf4420f99),LL(0x5075f5a2,0x38a95326),LL(0xadb26b0e,0x7345059b),LL(0x67709e35,0x428212fb),L_(0x00000139), + LL(0xd8fc8db1,0x3a95d178),LL(0xb909e614,0x1860388a),LL(0x89b7600c,0x942112c1),LL(0xa080f4aa,0x5a1967f7),LL(0x5057c08b,0x13543a0e),LL(0xf9ac78fd,0x1598cafb),LL(0x9408a20a,0xfa7974b4),LL(0x8fb58bcd,0x17ad4e19),L_(0x00000162), LL(0xd603cb6a,0x57138c5b),LL(0xf8960264,0x185f172a),LL(0xea9d78b2,0x8652917b),LL(0x62148231,0x9b757159),LL(0xb7470a8b,0x4f2c7ae3),LL(0x532d7747,0xc96fd10f),LL(0x6b40b8bf,0x77081dbd),LL(0xa54da232,0x2cd44f13),L_(0x00000119), +}, +/* digit=63 base_pwr=2^315 */ +{ + LL(0x44140a9f,0xc1b5f874),LL(0x37761e89,0xc768e709),LL(0x052402d8,0xa7063fcc),LL(0x437e0d8f,0x5032ca28),LL(0xd7049706,0xe0560b81),LL(0xfcc5af72,0xdac1a63b),LL(0xabb68cfd,0xe89f3917),LL(0x257b3b85,0x80d7454a),L_(0x00000038), LL(0xbc9cca5e,0x001d4cbe),LL(0xe9651818,0x66b1014f),LL(0x64d65a97,0x511b3639),LL(0xd56646ec,0x26e7c4e0),LL(0xdfae8dcc,0xa94ae11a),LL(0x86e8d406,0x6e9a1a68),LL(0x47bbf4ad,0x3004a685),LL(0x13e8901b,0x5981c480),L_(0x0000008d), + LL(0xb7f0034c,0xc63223ff),LL(0xeed01f7e,0xb10c656e),LL(0xa95759d3,0xc8ceacdd),LL(0x5f68fe9f,0xb6ab8ec3),LL(0x50e97936,0x28c0b215),LL(0xe4ccc3b9,0x92be8e1d),LL(0xfc6be17e,0x20828c77),LL(0xb81bdb63,0x13352a78),L_(0x000001d2), LL(0xcfa9e131,0x3d9b3c1e),LL(0x0b5daa42,0x94567b3a),LL(0x6bf95aa5,0xa3d149d4),LL(0x8d0fbfb5,0x4e997958),LL(0x5e636b3d,0xc1e08ca4),LL(0x8fa3b11b,0x73645c35),LL(0x552e11b1,0x931ab993),LL(0x0db67bc8,0xce614c5f),L_(0x000000d3), + LL(0x78c98029,0xf005a100),LL(0x9be2e7f2,0x06f26644),LL(0x47f29a13,0x7b580ed9),LL(0x3f3b60f9,0x22198889),LL(0x7e6ec70b,0x9f87a7be),LL(0xfc2d715c,0xb2ebc47c),LL(0xfc003ea3,0xfa7b2218),LL(0x79438acf,0xbfd9d6c5),L_(0x000001c2), LL(0xd7ce55f5,0xdac555ca),LL(0x80f3c546,0xe5a4dad1),LL(0xa25a6ba7,0x2d4bd9fe),LL(0xa68cfbbf,0x35faf13a),LL(0x4e3df8bd,0xcfc847de),LL(0x8434c00b,0x4dfdf245),LL(0x40669463,0x6e619d42),LL(0x5f688c19,0x13d3a517),L_(0x0000004b), + LL(0x44f3544b,0x3aed7148),LL(0xbb901084,0x6321bcc4),LL(0x996ac002,0x53c74ad5),LL(0xcb634535,0x741982c7),LL(0xdc48a041,0x1196d8fe),LL(0x44c9f092,0xfabc20d2),LL(0x0a8fce97,0x32828d27),LL(0x62a6d447,0xcab0c775),L_(0x0000000b), LL(0x43969d2b,0x3a61da39),LL(0x4b749d7b,0xfb5b6d67),LL(0x9df6dae1,0xec275083),LL(0x4fd05c30,0x7da1a928),LL(0xb7bd6dae,0xec82a28e),LL(0x66cd19ba,0xa08ca71d),LL(0xd599b2c6,0x6c312c52),LL(0x5bfaa154,0x6795e306),L_(0x000001f0), + LL(0xd73110d9,0xe9c779ff),LL(0x99bf4200,0x2e4558d4),LL(0xb9ba6e9a,0x636dc521),LL(0x836fe297,0xa1c7e0bd),LL(0x461d465e,0xa229d229),LL(0x287fba32,0xe43c8f80),LL(0x5fa34491,0x76cbe0ad),LL(0x0e6b8f16,0x7a25d2a9),L_(0x000001d0), LL(0xdc2e36c7,0xd420ce9f),LL(0x59654147,0x2c11cbe4),LL(0x582dde44,0x73168c78),LL(0xcfbe66e7,0x5f455763),LL(0x55778942,0xd782c483),LL(0x9b69f069,0xff95fe3e),LL(0xaa1addcb,0x00a4bd0b),LL(0x47541c1d,0x8ad93857),L_(0x0000012a), + LL(0x8aa7d8c0,0x0afff918),LL(0x80ad064a,0x89af5deb),LL(0x7114ab96,0x4dbde778),LL(0x099fad0b,0xfd29cd3c),LL(0x525d6055,0xbd379d42),LL(0x4df50e85,0xdfc116d0),LL(0x3602e006,0x374d96b5),LL(0x2ee6c63f,0x6509a7f3),L_(0x000000f2), LL(0x6aa97902,0xee822c17),LL(0xfc039cb1,0x6c2fdf58),LL(0x5872cad5,0xea665324),LL(0x3b9e8ae0,0xaf2e64bf),LL(0xb8314c4d,0xa8f96bb4),LL(0x63c57f41,0x3df990e0),LL(0x5149d306,0x1d5f9b0d),LL(0x08ba6128,0x3d6cc9c6),L_(0x000000ba), + LL(0x5b4bef3c,0x84b34c8d),LL(0x0908a3fa,0x06e343a5),LL(0x8e41dac7,0xca844102),LL(0x83411f49,0x0712aa99),LL(0xd4bcaa5f,0xf85d2ba8),LL(0x0278367b,0xbdc302b7),LL(0x5016082b,0x54ed82be),LL(0xfea00712,0x1e47617b),L_(0x000001a6), LL(0xbea2cdee,0x5025ca72),LL(0xa8a5db48,0xd3c98c1a),LL(0xeb113cc6,0x259b9a28),LL(0x1b35c6d2,0x49923a55),LL(0x266d75d3,0x644a3ecc),LL(0x9590fb6b,0x221e1f1d),LL(0xa7f663c5,0x9c9bd811),LL(0x30cacfb5,0x8f25a4f3),L_(0x000001ad), + LL(0xda490054,0x5c5d4a76),LL(0x224e9112,0x74621c3b),LL(0x62ab184a,0x17406495),LL(0xedfb682a,0xc3f7c8cd),LL(0x16ae2053,0xd8e38d44),LL(0xdf044060,0x39ed9c28),LL(0x86143e57,0xf327b97f),LL(0x8b95f9f7,0x53853147),L_(0x00000072), LL(0x81550101,0x43b98e46),LL(0x05661b39,0xc1bcc1fc),LL(0x9ee23198,0x64ff1647),LL(0x115744fc,0x0f20d871),LL(0xcdf5ac56,0x92c9feea),LL(0x63cba9c3,0xa72f70b2),LL(0xadbac8fd,0x365c71db),LL(0x171aad35,0x9d51687d),L_(0x000000f9), + LL(0x7cba337f,0x7242ca8b),LL(0x45faaf9a,0xc08d85f8),LL(0x550ef4dc,0xb82ff28e),LL(0x814b8cba,0xbbe121da),LL(0x1eb4cd63,0x081656e6),LL(0x82eece40,0xf4405b11),LL(0xe9889d6a,0xf6c9d001),LL(0xbc4f3c1e,0xe85dc906),L_(0x0000001b), LL(0xd4907a17,0xbcfa56fc),LL(0xb8894301,0xa60a71ba),LL(0xc8290de3,0x5b4cf893),LL(0xfa8203e9,0xa8602943),LL(0xb0d9fec8,0x6b75b5c6),LL(0xacaeb1bd,0x40f20d5b),LL(0x228fdb83,0xe7477d37),LL(0x967812d8,0x3271b8d9),L_(0x000001d0), + LL(0x9302acf0,0x69acd4ec),LL(0x76812a69,0xd47ef468),LL(0x62f921ab,0xc8ee3434),LL(0xb7930834,0xc08c033b),LL(0x369c3e87,0xee51d0a2),LL(0xd98cac8f,0xc675c1fb),LL(0xa309b704,0x3fcbb3c6),LL(0x69a173a4,0x32c49495),L_(0x00000094), LL(0x2ef36de7,0x2b5e781f),LL(0x79bd3a70,0x68837e34),LL(0xd74e86eb,0x22881aa5),LL(0xecb38496,0x91b89a84),LL(0xdd2964ba,0x7caeee87),LL(0xb0230b75,0x83a10f40),LL(0x7853cadc,0x465657ae),LL(0xe45f5ad1,0x100e5033),L_(0x00000075), + LL(0x5443e17b,0x27034a2f),LL(0xae458db9,0xe02cc805),LL(0x361c4604,0xc6c6e812),LL(0xf53dab3f,0xe1de7819),LL(0xa93944c6,0x77575b10),LL(0x7d127be6,0x4580ec67),LL(0x18920ad0,0x6451a6a6),LL(0x595f7341,0xe3b018ad),L_(0x0000002f), LL(0x7a6f7a6b,0x73fafabe),LL(0xacea82d0,0xb8e018e3),LL(0xb66d3c1a,0xf0a068d2),LL(0xa0a76281,0x2960ab23),LL(0xd3310f1e,0x1ade815c),LL(0x5df5a459,0x4830c68e),LL(0x9bc40618,0x506f8ded),LL(0xa5b32181,0xb64aea9e),L_(0x00000055), + LL(0x11651e46,0xd2e44f39),LL(0x7f22b492,0xf166288f),LL(0x72f850db,0x45a14853),LL(0x6743ab2a,0x480b82ee),LL(0x235a84e1,0xbca609c8),LL(0x422668b9,0x2f4e85d8),LL(0x5d6f0bf0,0x792321da),LL(0x61afb880,0x2c095f02),L_(0x000000b8), LL(0x25cea9cf,0xe6bc2f57),LL(0x43f99381,0x4a832e1f),LL(0xe6089c84,0x51ad7011),LL(0x65600aa9,0x2a695207),LL(0x11447728,0xa07e689c),LL(0x7bb9a4c6,0xcd7b0e53),LL(0xdf06eaf0,0x78952329),LL(0x777c474d,0x10d2b00b),L_(0x00000107), + LL(0x42f9d45d,0x20322841),LL(0x736265c0,0x91a20b69),LL(0x7c956777,0x530024a3),LL(0xbcd4358f,0x2cfdf5c4),LL(0xe32fe9e3,0x69c3c240),LL(0x7dd472b9,0x16947a8a),LL(0xeeaaeb78,0x03c5cf25),LL(0x9aa3433d,0x521f3b6b),L_(0x00000018), LL(0x96f4132f,0x88ddc3b1),LL(0x00a93570,0xc5b29c7d),LL(0x5d8a1581,0xb3793bd2),LL(0x1877f26e,0x4435f13a),LL(0x4fc7c6c4,0x5b6c76af),LL(0xb032ee18,0x5465338c),LL(0xd7e32969,0x5e8d0f72),LL(0x82259fe6,0x38f0d401),L_(0x00000082), + LL(0xaebae92a,0x8093c8e0),LL(0x248ef981,0x84b971a4),LL(0x5353d713,0x8ab8dd10),LL(0xf3f56422,0x9e95615c),LL(0x163427ee,0x1dbed91f),LL(0x7cd8d83e,0xa05bff8c),LL(0x2a117b26,0x9094b7c0),LL(0x28d65130,0x8d73f3a7),L_(0x000001fa), LL(0x3fc6c29b,0xfb45cf4f),LL(0x5dd01b3b,0xa983b69a),LL(0xe3b24278,0xdda15e64),LL(0x0beba6e8,0xeabdafeb),LL(0xce3cbe7c,0xe28dd1f4),LL(0x03c3a01e,0x315483c0),LL(0x286b68f0,0x44cc13c4),LL(0x653661bd,0xa5a2b18a),L_(0x000000d7), + LL(0x837c224d,0xf83ea93b),LL(0xc7a428c9,0x308d0ccb),LL(0xdeef06fc,0xe456dad3),LL(0xd27b9dce,0x04dd575c),LL(0x8c1bbcfc,0x1c63319a),LL(0x479f6f73,0xed4daaeb),LL(0x7cb52d7f,0x9fe5930e),LL(0xd171cdbc,0xcd65b54d),L_(0x00000100), LL(0xf9b94ca6,0xbb8079a0),LL(0x2ea98b08,0xda724133),LL(0x24b7505e,0xedf1d97e),LL(0x5aed5e6f,0x283c1e51),LL(0xc39ad307,0xf64812a4),LL(0x76820a6c,0x13b5c88a),LL(0xb32f91ce,0xb8954a33),LL(0x211cbd9c,0x31a311e0),L_(0x000000ae), + LL(0xf859a830,0x3ef4f2d2),LL(0xcf466ff0,0x47044584),LL(0x78dc82b8,0xb52d320e),LL(0x8b5110dc,0xdfe140d7),LL(0xe07b117a,0x0b45fd46),LL(0x39af6581,0xbda19439),LL(0x26b6d5c5,0x8309f53f),LL(0x8091095a,0xaad23c7d),L_(0x000000d4), LL(0xa9e7ca16,0x90dd82bc),LL(0x1ee78b60,0x839c5155),LL(0x453fc776,0x2966b875),LL(0x6bf1d026,0x9e2a2996),LL(0x825d3c72,0x48c49cfc),LL(0x1345ab1d,0xad600996),LL(0x7e6049f3,0xf1b39850),LL(0x0b007da4,0x2c8f36cb),L_(0x00000038), +}, +/* digit=64 base_pwr=2^320 */ +{ + LL(0x90646dde,0xeba284c6),LL(0xdcd5cc91,0x292fa3ee),LL(0xea471fd6,0x5841cc32),LL(0x9fb23a12,0x35810a74),LL(0x4e18eb2e,0xd6133648),LL(0x5228a2bf,0x52cab6f6),LL(0x07542e74,0x40c74692),LL(0x62526cd9,0x36b9b329),L_(0x00000129), LL(0x063ef2a7,0x17573e4f),LL(0xca996c2a,0xc3d42418),LL(0x33e1f9c1,0x970fbd47),LL(0x246b3cbf,0x8c0b1561),LL(0xf0853508,0x16e93234),LL(0x8ff90188,0x74d99d7f),LL(0xaa556f85,0xd3b1d290),LL(0xdda5d989,0xab78218e),L_(0x000000d0), + LL(0x57077d65,0x87233d65),LL(0x32cea9ff,0xb1454f2b),LL(0x6963e65d,0xd5f2627e),LL(0xd15b05b7,0x68cad15b),LL(0xf2c9215a,0xf9679cfe),LL(0x982da4ec,0xdd21cc1f),LL(0x73910763,0x3925aff6),LL(0xce110fdc,0xad0858b1),L_(0x0000003c), LL(0xdb7b5667,0x160c51bf),LL(0x88f58f75,0x9e39ee8b),LL(0x713a7cf3,0x5af813ed),LL(0x8ac4ac36,0xa788e43e),LL(0xe789c040,0xc10b5e01),LL(0x3d0cb49e,0x26fdddf3),LL(0x3f2d9bd7,0x504e525a),LL(0x776e3c30,0xa456acdf),L_(0x00000051), + LL(0xa6499d4a,0x23707d7d),LL(0x4b6d85bc,0x2372ec00),LL(0x4b483dd7,0x838f63c9),LL(0x869b15c9,0x40b6584e),LL(0x291644dd,0x05bb5ad6),LL(0x693ec1c9,0xc10969be),LL(0xb5c6018d,0xb81150c7),LL(0x04c9c113,0xbd460de8),L_(0x0000000b), LL(0xb81757db,0x5ad558a1),LL(0xa356589b,0xf88e046d),LL(0xf093ea9c,0xede9de0f),LL(0x39acd54e,0x19ec3f88),LL(0xfcbf451f,0x44ec243f),LL(0xf8b02c0e,0x981fd0d1),LL(0x42d2cc07,0xf4701bca),LL(0x3f363b43,0x30f2e9e4),L_(0x000001f9), + LL(0xd9f5845f,0x006b0772),LL(0xa8c7c3d7,0x1ba3ff28),LL(0xc1d96b23,0xc17a4f5f),LL(0xda50f432,0xca88f653),LL(0xfce5ef14,0x31ac5da9),LL(0xd10257bb,0x18d3105d),LL(0x06b910de,0x4f950082),LL(0x8ed121d6,0x748b9a29),L_(0x00000083), LL(0x0ad3e4ad,0xfedc9456),LL(0xcda193af,0x30addb34),LL(0xf39dff50,0xa3a58a0d),LL(0x586d72c1,0xd7c02e84),LL(0x7190f71a,0xb6dddddd),LL(0xd7f7815a,0x93fd431d),LL(0xb059af28,0xdb90a301),LL(0x626d66eb,0xb55b2545),L_(0x000000c5), + LL(0x55a2cc5b,0x02a70327),LL(0x60173b4c,0x700e187b),LL(0x853a0c8b,0xebfa5d41),LL(0xa74d3fcb,0x6636a248),LL(0x7f152910,0xcd439df1),LL(0x433bf866,0x3d361a48),LL(0xd52b92a9,0x96508fce),LL(0xcdde5dbf,0x08fb48db),L_(0x0000004c), LL(0x57d607dc,0x41e6d707),LL(0xa287bad7,0xc1d0199a),LL(0xaca83d8b,0x3248272c),LL(0xdeea6deb,0x81490886),LL(0xd5830e62,0x803b3e7e),LL(0x0b551501,0x329bb8f5),LL(0xe61ae410,0x1b1ec67e),LL(0x2add209b,0x9d8f057d),L_(0x000001f7), + LL(0x4aeeb4f7,0x159240ac),LL(0x701cba0d,0xbe49e9de),LL(0x1e2030d5,0xa8d80ea6),LL(0x891f5b9f,0x389aa0a7),LL(0x281d5c9e,0xbf08f46a),LL(0x42c2a6a9,0x30133d89),LL(0xcae5c626,0x26d80fbd),LL(0x976ed6f2,0xd7445273),L_(0x0000007c), LL(0xebe5a160,0x4b64112e),LL(0x1ba10f05,0x3c715556),LL(0x076de398,0x051c721c),LL(0x1b6338a0,0xed93ec2b),LL(0x0b18e617,0xe40d08e3),LL(0x2546a805,0x39d986d0),LL(0x289546bf,0x87fe36cb),LL(0xcb29a40d,0x28ca6d96),L_(0x00000038), + LL(0xf08b61b9,0xb8888aaa),LL(0xeb89b8a3,0x504b24ba),LL(0x13c31ce0,0x1577d88f),LL(0x1d308489,0x01541da0),LL(0xc31edb15,0xfbe18906),LL(0xcb88a0c0,0xb123cf8a),LL(0xe0a54814,0xce17eb8d),LL(0x12d30b10,0x5435ad11),L_(0x000001c6), LL(0xd6e0b2ed,0x7a3c3081),LL(0x198cbd6e,0x18481bd1),LL(0x9feff602,0x8a4e33b7),LL(0x4dc9559a,0x242155d3),LL(0x49b265ae,0x0458dbdb),LL(0x66003375,0x19c33688),LL(0x53753ede,0xac09e0c8),LL(0x0eb6969a,0x25b27567),L_(0x00000052), + LL(0x13db9105,0xfd4c030e),LL(0x4bb182d8,0x45ba7b8e),LL(0x24d5733c,0x9bbae322),LL(0x857e0992,0xe18395c5),LL(0x7a4b7ec5,0xbbeb3431),LL(0x9d2ffacf,0x70996597),LL(0x0dac7ff4,0x634b33c0),LL(0xd22ac181,0x5a113dab),L_(0x00000132), LL(0x4a184515,0x1af6d0b4),LL(0x4b60b5e3,0x60067ebd),LL(0x7c6c236a,0xccf47b3d),LL(0x199b1be8,0x1dbd1cc7),LL(0xe888eba2,0xb4932466),LL(0x034c21f8,0x19ff1dee),LL(0xf9da1696,0xe040c95f),LL(0xee7e95c7,0x9dbe56ef),L_(0x0000015a), + LL(0xf23e08f9,0x33a41f31),LL(0xb89596d6,0xde8f6d08),LL(0x36e37e25,0x09152867),LL(0x911d84ea,0x4f3476b2),LL(0xba9def4c,0x44e0d519),LL(0x12065979,0xdfef8c30),LL(0x91c87d28,0xa45cf33c),LL(0x6b8dd103,0x5a48975d),L_(0x000001b7), LL(0x3141bb8f,0xe73885a7),LL(0x36da50e9,0xdc731cf5),LL(0xb97f8cf8,0x67bb07a8),LL(0x922b0be5,0xfb414a3b),LL(0xe9cbd504,0xd391785e),LL(0x2631b899,0x9eb65672),LL(0x50f31f7b,0x4ee45215),LL(0x0d4d0798,0x23e25b24),L_(0x00000039), + LL(0x24f12ded,0x6178bac5),LL(0x9eabc3c5,0xbfa39955),LL(0x503d57c5,0xda006222),LL(0x8e465ace,0xe390a3d7),LL(0x363ca671,0x7ca51f49),LL(0xe0376d27,0xab1c9afc),LL(0x325dbeb1,0xf303951f),LL(0x2ac46079,0x4fcc04e4),L_(0x000000b1), LL(0xa05b906b,0x1e1f126b),LL(0x7a1f14f0,0x0b9e64fc),LL(0x49fb5176,0x394e56b2),LL(0xc1fff51a,0x0b50d33b),LL(0xf135bc4a,0x41e4f563),LL(0xf07911e8,0x0008c3a8),LL(0xfd1855b2,0xd0455066),LL(0xb1f8cdf1,0x3e4e10b9),L_(0x0000006c), + LL(0x03ffd8c0,0x47688773),LL(0x75ef0188,0x81134439),LL(0x6a21abe0,0x3ff532d6),LL(0x9e0177ad,0x3c27f56d),LL(0xe284df24,0xd99892e8),LL(0xfcaf2cd5,0x668c2ac2),LL(0x72c31d05,0x450ea985),LL(0x617df772,0xd1386608),L_(0x000001fb), LL(0xa7faa0ca,0xa81bbd89),LL(0xb7cb40e7,0x6545e4d7),LL(0x4e799290,0x48c0ef0c),LL(0x129414b6,0xf9bc6b77),LL(0x75cac719,0xcf3cf61a),LL(0x7090a084,0x5de671da),LL(0x573167fe,0x53c2428e),LL(0x9be66bcd,0x581cfd76),L_(0x00000105), + LL(0x9912480b,0x868b1c0c),LL(0x6fc274a1,0x83263833),LL(0x7470faca,0x28bbd5e9),LL(0x10a9a5ed,0x92bc266f),LL(0x1a2df530,0xc1420bb6),LL(0x26088825,0x6de27806),LL(0xb843fbcd,0x96eddc77),LL(0xfaacb0c0,0xe58f23ff),L_(0x000000a2), LL(0x265f30c8,0x304fead2),LL(0x46f8b4da,0xcdc2767a),LL(0x030d0ccb,0x4ecb91a4),LL(0xa6cdee79,0x546f1657),LL(0x2f10c656,0xa2c85665),LL(0xdaca38a8,0xb2b32405),LL(0xa84dd381,0x29386bb0),LL(0x4d4926a6,0x3ed722f3),L_(0x000000e5), + LL(0x6732b4e2,0xb158c617),LL(0x70c2bdde,0x1e929730),LL(0x03b67c3d,0x83aa10cb),LL(0x207b6554,0xf8bd1f2f),LL(0x65897412,0x52c6a1bf),LL(0xd2b9e2bf,0x4072e449),LL(0x4573028a,0x51728cdb),LL(0x08f548f4,0xa5fb2b4e),L_(0x000000cb), LL(0x7e81850d,0xc30af1b8),LL(0xf64d0544,0x91fac057),LL(0xe6a44ca8,0x97b402f5),LL(0xf3758797,0x7a48f50b),LL(0x9ec9c59e,0xa20d052f),LL(0x7e0c3edb,0x50d02201),LL(0xba6cf070,0xc4603d10),LL(0x0fd79a40,0x04379719),L_(0x00000047), + LL(0xaa79abd0,0x33d05c0c),LL(0xf4e66422,0x28d54dca),LL(0x3f1a0e0a,0x1b90591f),LL(0x8319bf69,0x50c92b63),LL(0x45f8cdc5,0x2b172382),LL(0xc8908923,0xce47651d),LL(0x282da333,0xe6f22c70),LL(0x6dc02842,0x73a13e20),L_(0x00000052), LL(0x2709e7a5,0x9bb811ca),LL(0x2aa27cb4,0xff020d4c),LL(0x8f138cd3,0x181a1cec),LL(0x35013750,0x47863f93),LL(0x37481122,0xe6028031),LL(0x18f58c65,0xf01c48b5),LL(0x565f6657,0x9f20924b),LL(0xa7b0ed3f,0x643987ef),L_(0x000000eb), + LL(0x25a6701b,0x151f4865),LL(0x1b42c497,0xf30bdc50),LL(0x055325f0,0x144e0aa2),LL(0xf8e98fe1,0xa165a395),LL(0x2e0f9b5a,0x25afa523),LL(0x3ceadf0d,0x70ed634b),LL(0x55dbb9b8,0x1b25f855),LL(0xe2ddb61f,0x8a54708c),L_(0x00000000), LL(0xd7b55067,0x74847fca),LL(0xf91dd3a7,0x92445716),LL(0xe74dda4f,0xfe51e6c5),LL(0xd2ebe9fa,0xe3bfd67c),LL(0xec65184e,0x51f3767f),LL(0xc26dcf5e,0x6092d164),LL(0x7562e715,0x48053ca5),LL(0xc341746a,0xfe264b56),L_(0x000000c8), + LL(0x1486b225,0x688f0816),LL(0x55ab4efe,0x012b1be7),LL(0x48d9609c,0xdb068e78),LL(0x0fb98843,0x958488ad),LL(0xff5eda2d,0x83f6d23c),LL(0x3ec7372a,0xb176c41d),LL(0x5d185ec0,0x925e4903),LL(0x476314a5,0xe4ff9579),L_(0x0000003d), LL(0xc1b43aa4,0xd529cc94),LL(0xd2ad417d,0x2dfe7d43),LL(0x360ab4fa,0x52cc454d),LL(0xfb2e9eb5,0xa4732c24),LL(0xcf82a235,0x68d1e843),LL(0x67bb7a40,0xbca2ab8e),LL(0x91877eaf,0x99566cd1),LL(0x62574ab0,0x22f9872f),L_(0x000001f9), +}, +/* digit=65 base_pwr=2^325 */ +{ + LL(0x8bbe4fe2,0x2d85f820),LL(0x287db7bb,0x702fbecc),LL(0xe568667e,0xa157f36d),LL(0xf4ecbeb7,0x484f3352),LL(0x941bbfbf,0x558da014),LL(0x3d5fe38d,0x7b22586c),LL(0x8a8ef1b3,0x7a9e7fea),LL(0xba594962,0x0c422ebe),L_(0x00000074), LL(0xe63724c0,0x34c2ec0f),LL(0xeb882690,0x8c7ffbe0),LL(0x16f607ed,0x0a729f09),LL(0x9cab235b,0xfb783d21),LL(0xe85a3bb2,0x7a1f91a0),LL(0xaf1659ef,0x067ef36e),LL(0x3c3d4be9,0x2b43e992),LL(0x3b5e5bd9,0xe81391aa),L_(0x000001e3), + LL(0x6902a59d,0x58cf4fb6),LL(0x2d970cf0,0x6108b652),LL(0x8db98564,0x1c524ec7),LL(0xc375bd09,0x8ded01ba),LL(0xeaf41a1c,0xca7571d5),LL(0x1513bf75,0x83433ae4),LL(0x831a58ce,0xb1cbad60),LL(0xd4b5c1d0,0x7e3558b4),L_(0x00000012), LL(0x2423577d,0x5fc5bcaa),LL(0xcd90416c,0x1fd11e95),LL(0xf9cd3e85,0x77429d71),LL(0x4b143cec,0x818263e7),LL(0xb694e333,0x7b0bed2a),LL(0x078fef20,0x900d9d3b),LL(0x22c62d90,0xb2dcc393),LL(0xf713057f,0xee2cd8c7),L_(0x0000010a), + LL(0x62e274a4,0xc6a3697b),LL(0x36666b6c,0x771114f0),LL(0x4615de0a,0x4656bc00),LL(0x27ed1a54,0xfe1b0ffd),LL(0x7a367a4c,0xa7b011fe),LL(0x9395287b,0x1539028f),LL(0xe474d2cd,0x67ab6630),LL(0x50df81c9,0xd416f7d8),L_(0x00000009), LL(0x15224116,0xcd4ff017),LL(0x14e9eb99,0x68ce9cb6),LL(0xefe50131,0x878690dc),LL(0xf0500068,0xa58b25b4),LL(0xfe708a3e,0x1697bfbe),LL(0x4cbc1887,0x109e6148),LL(0xd61b572d,0xea6e538f),LL(0xcd507a67,0x8cf0642b),L_(0x00000010), + LL(0x44c4f316,0x254f2817),LL(0x74f04275,0xe534cfb9),LL(0x22ee390d,0xf368e25e),LL(0xbc4caed9,0x0d3f5a1e),LL(0xd7010447,0x26bb7427),LL(0x91f02404,0x7f0d8308),LL(0x993cd5f3,0x4ebe5786),LL(0x1ba9d89f,0x2549a02c),L_(0x0000007b), LL(0xef3c1601,0x575527d2),LL(0x33afd2d8,0x6240eaa7),LL(0x200435a3,0x0df72a8e),LL(0x104dbed5,0xe9f3dcc7),LL(0x1f5c3464,0x98404140),LL(0x791b398c,0x1581b281),LL(0xd77cd49e,0xa203aa2a),LL(0x2329530c,0x70040738),L_(0x0000004a), + LL(0x4c2c9776,0x8715b292),LL(0x5fca8e16,0x0659f3f5),LL(0x904a8960,0x7cdfccac),LL(0xd46df8f8,0xe8078ecb),LL(0xae2184c0,0xc1352930),LL(0x904b839e,0xf3fd8786),LL(0x36602186,0xc3ec21de),LL(0x2a20030f,0xe08de817),L_(0x00000046), LL(0x36bb6226,0xd7e4cb2f),LL(0x812cd124,0x5ac9609a),LL(0x83d9653b,0x690acbe2),LL(0xf981cac9,0xc894c3a8),LL(0xd274538e,0x286285d9),LL(0xc202f8f4,0x24269d02),LL(0x51bb2579,0xe768b7b8),LL(0x121f910a,0x8baea845),L_(0x00000158), + LL(0x41280631,0xf6a81b1e),LL(0xcf66e145,0x2a3bda3a),LL(0xa88833db,0x44164a42),LL(0x4e0df1ec,0x754bd187),LL(0x89a6c53a,0xff8dc770),LL(0x961d8b4b,0xa3761531),LL(0x87d46b93,0x31b05601),LL(0xf7105b06,0x4ef74177),L_(0x00000156), LL(0x5c72ab48,0xbc131ab6),LL(0x4688d4bb,0x6977d5eb),LL(0x82e94cb0,0x8706473c),LL(0xb785ac18,0x7362c724),LL(0xae704972,0x3b45de5c),LL(0x2e6bdb68,0x2c67f7d5),LL(0x99b0063e,0xa06ed86c),LL(0x4969a5c7,0xc4bf63b2),L_(0x000001fa), + LL(0x181e12e7,0x32f70f29),LL(0x4140b7bb,0xbcda177c),LL(0x0fdc8422,0xad59d7db),LL(0xd2a3a29c,0x9c1893f4),LL(0x8f80936a,0xe60fd330),LL(0xa903804b,0x6b3cc7d2),LL(0x0e38278b,0xfcb7a0c8),LL(0x31deb3a4,0xb47a9458),L_(0x000001ff), LL(0x2d8fb441,0xc70a541d),LL(0xcbbeaa75,0x6f004b75),LL(0xd7f127d6,0x1d4ef334),LL(0x15636fa0,0x758ac159),LL(0xa2548921,0xb047a7de),LL(0x60705693,0x128b4b7e),LL(0xa5696c87,0xf499a64e),LL(0xa206ac49,0xf272bcca),L_(0x000001bd), + LL(0xe83f7b24,0x7a4b896d),LL(0xb2e00072,0x47a68dd1),LL(0xd43d9655,0x1ce79a50),LL(0x6abc1f1e,0xec87252c),LL(0xe7160fe5,0xb60d6a5a),LL(0xdcf45caa,0xd235985f),LL(0x1b3180d9,0xf8982569),LL(0xdc646ab8,0x446d6798),L_(0x000001f0), LL(0x8387586a,0x6eec719b),LL(0x5f9db663,0xcc9431de),LL(0x38f4a187,0xef383b03),LL(0x3dbac366,0x2fa36674),LL(0xcc0b0d02,0x5760fac1),LL(0x67948b1b,0xe8d42650),LL(0x93934495,0x4f889216),LL(0xd3c64b8f,0x1cd8ec2e),L_(0x00000128), + LL(0x354d871a,0x107d6bfc),LL(0x5816dc43,0xb8b5b662),LL(0x1fe1463b,0x973f88f0),LL(0x4370119d,0xc84691e3),LL(0x153d37fa,0xd0059b51),LL(0x9a4e583e,0xe99b060f),LL(0x24c8671a,0x8d5b7c8c),LL(0xfdf410b1,0x0db2233f),L_(0x0000002f), LL(0x3e3f1b42,0x08c95cdd),LL(0x07dbde25,0xf5273466),LL(0xf0969049,0x76e98baa),LL(0x8cba001d,0xae173b76),LL(0xcde8bcc2,0xf10bd659),LL(0x09ae5065,0xc7bec674),LL(0x5cecdf22,0x4bdaed4b),LL(0x41bc9eb9,0xc459b9e5),L_(0x00000146), + LL(0x6121d2de,0xd08a3672),LL(0xbcb161c1,0x52ff0a6b),LL(0x0593dede,0x4b01e845),LL(0xb5c016b2,0x421052e7),LL(0x65c310a3,0x6c1dd249),LL(0xf97c5a2d,0x5a4e53fc),LL(0xe551417d,0x0e004126),LL(0xbf9b31fa,0xc101bc8b),L_(0x00000125), LL(0xf17fd0e4,0x2811e723),LL(0xeaad08c2,0xcc586f7d),LL(0xc771ce2f,0x8c0556ea),LL(0xef8166e5,0x32556e9b),LL(0x2d11bd0c,0x11e66977),LL(0xfe7d92c3,0x646a8dfa),LL(0x58b69181,0x8a624576),LL(0x5de6bd35,0xdeb3cceb),L_(0x00000027), + LL(0x81d3dab7,0x88f96f05),LL(0xc1d3ecbd,0x3593555a),LL(0x3073315c,0xb56c336e),LL(0xaec91693,0x5eea4cf3),LL(0x2eade86a,0x90e05846),LL(0xe134f505,0x9189d76c),LL(0x8d429f7e,0x0feade8a),LL(0xf0013b30,0xe083daf3),L_(0x00000097), LL(0xec2945ab,0x817e33d7),LL(0x9c2537c7,0xa44bf13f),LL(0x9a6a317a,0xda31eee4),LL(0x35ae34c0,0xa0379d97),LL(0xd66b27c5,0x76a48571),LL(0x1ae6d028,0x5d83028d),LL(0xbb8dadac,0x1e4ebd89),LL(0x8aaff54e,0x67e3f97d),L_(0x00000083), + LL(0x0bc690f6,0x9a7f8ba8),LL(0x2992a59b,0xa9efff68),LL(0x1a328627,0x3c1d097f),LL(0x555f21eb,0xf75afbb3),LL(0x48ff742c,0x4bf1016a),LL(0xa10b236e,0x5c770a94),LL(0xff3e57b4,0x5915a516),LL(0xc042020f,0x9df7440a),L_(0x00000186), LL(0x18a6102c,0xecaea7dc),LL(0xbf14c4c4,0xd0035f1c),LL(0x8343dde4,0x19dfc08d),LL(0x3a483722,0x5b130cf1),LL(0xc107a176,0x30fc7b7f),LL(0x6717617b,0xf8cab932),LL(0x626ffb5f,0x59269de8),LL(0x74af7c16,0xbf37b9d1),L_(0x0000000d), + LL(0x81146610,0xd31224fc),LL(0xe9393683,0x52cacad0),LL(0xd4fad3f5,0xd36819ae),LL(0x14ee7de5,0xb8c9f302),LL(0xdb882d76,0x103a3f52),LL(0xb3362378,0x1728ce43),LL(0x0f7553db,0x29c76302),LL(0x56ba2d84,0xc2f44114),L_(0x0000009f), LL(0x8ea45de7,0x3c639715),LL(0xd8b8276f,0xacdd7488),LL(0xac8ea8c0,0x912aac3a),LL(0x8de7940b,0x3dabab2d),LL(0xd5bb2c71,0xbc3f4a45),LL(0xbcc2e33c,0x2bf9f840),LL(0x9d0edeba,0x4b00c80e),LL(0xcbe852a6,0xbe4c368d),L_(0x00000191), + LL(0xc0428b2a,0xc2974837),LL(0x06284fef,0xe24d882e),LL(0x7041703c,0x6f37e52f),LL(0x5e1e37fd,0x7853375d),LL(0x145a0690,0xcc75898e),LL(0x270c5225,0x61f33577),LL(0x82c5658c,0x777ab969),LL(0x47024eca,0x315f136f),L_(0x00000012), LL(0x6ae06c18,0x8c1cda05),LL(0xb7db1fe6,0xedf31bed),LL(0x7f07ba36,0x8079b5f2),LL(0xcdbc2d84,0x22d262c8),LL(0x4bb1ece6,0xb457935a),LL(0x79793249,0x8614cc62),LL(0xe89f8430,0x258e4fa5),LL(0x836da5fd,0x759ca7b4),L_(0x00000009), + LL(0x8d1df13f,0xfa6cf668),LL(0xce181709,0x4b169dbb),LL(0x5bee0010,0x618d280d),LL(0x28a88cdb,0xa9c8042e),LL(0xd8239424,0x8ce15b95),LL(0xcfce331d,0xda740deb),LL(0x6f83c378,0x3c616a52),LL(0xfc05f1b1,0xda236e18),L_(0x0000013b), LL(0xc1e90721,0x8647ebee),LL(0x64ec0125,0x0db33259),LL(0x91422332,0xf49a7d94),LL(0xa2819732,0xcccef356),LL(0x9b90c693,0x87f18954),LL(0x805deff2,0x22aa64c0),LL(0x294b5e96,0x352be09c),LL(0xa610a1bc,0x8368526f),L_(0x0000017e), + LL(0x1fdc38a3,0x8a8479d8),LL(0x893e0d1c,0x045ea96a),LL(0x972cecee,0x37445b26),LL(0xb250ed0b,0x1c0a16a9),LL(0x08e477bf,0x7509c768),LL(0xc826b683,0xb74870c4),LL(0x008a3f9f,0x4d580408),LL(0x4b88d0f7,0x18d474fc),L_(0x0000003d), LL(0x2178ecd9,0x2fa143fb),LL(0x726dcabd,0x36d39ee2),LL(0xe017d3cb,0xd8d9e011),LL(0x43bd77e2,0x332c8650),LL(0xc8965069,0x3231a13f),LL(0xf7a775c4,0x1e3de078),LL(0xe93c91cf,0xe0f7c892),LL(0x50d48604,0x27097492),L_(0x0000011d), +}, +/* digit=66 base_pwr=2^330 */ +{ + LL(0x3a3e42bf,0x9e603471),LL(0xfb50e447,0x2536308c),LL(0xf617634a,0x9549272f),LL(0x302ba17f,0x3e264556),LL(0x0f6ed916,0xddb056ef),LL(0xc67d2e92,0xf04b9449),LL(0x7bf608be,0xe0fd2d62),LL(0x5ba41494,0xfaa0f9ad),L_(0x000000ca), LL(0x08a03740,0xac0394a9),LL(0x9ed3e6c1,0x9017e273),LL(0x2bf950e4,0x441c9b0d),LL(0x9856ca69,0x717b7978),LL(0x3315b53c,0x66f1bf12),LL(0xc5bce131,0xdc85d5ff),LL(0xf7b0dacb,0x32365700),LL(0xf85306a7,0x22ce0f19),L_(0x000000fc), + LL(0x9bda8667,0x4ae474b2),LL(0x3a2640ed,0x3684de2b),LL(0x21f91da9,0x424bab62),LL(0x9955df67,0x50d60209),LL(0x5193b4b5,0x196c99d5),LL(0x8e9f2748,0x7e7f3c74),LL(0x31e6b3fe,0x257248f6),LL(0x73f47f4e,0x56db9ba3),L_(0x0000018f), LL(0xbc4dd12f,0x48bc357e),LL(0x3556892c,0xfa4353a3),LL(0xb72124cf,0x6ea40bee),LL(0x1dbe3505,0xb37aa3c7),LL(0x2e951ca1,0xaddc96c7),LL(0x71de6fca,0x58ae291c),LL(0xf88244eb,0x96fd42af),LL(0x7e89a7bf,0x1c5d8ae9),L_(0x000000b4), + LL(0x263c0965,0x5894fe38),LL(0xa61f78f7,0x1b7d597b),LL(0x4cc9003f,0x94b8d7ad),LL(0x0c50139b,0x899c26b5),LL(0xd4d5af57,0x5801efbb),LL(0x7d0705ee,0xc45f009b),LL(0x345f5d52,0xc52fa5f2),LL(0xda142009,0x8d175fe7),L_(0x00000018), LL(0xbfaa39a3,0x3b59142d),LL(0xc3639f82,0x8ec202a8),LL(0x87394d85,0xb2496e6e),LL(0xa4035f4b,0x7b3e291d),LL(0xcab52bdd,0xda4a9abd),LL(0x41430674,0x23a5aab4),LL(0x7c18c413,0x57ee045c),LL(0xe39c61ef,0x6008e4d9),L_(0x00000190), + LL(0x1fc12350,0xc78ac751),LL(0x51e9a589,0x6a5f85af),LL(0xc09d63e5,0x8ce24a89),LL(0x9b655b51,0x5441652a),LL(0x9b445da7,0xf4ffab48),LL(0x9523b0e9,0x23e77128),LL(0x90aaf7cb,0x99234af2),LL(0xb5c9bc78,0xa65dc198),L_(0x000001dd), LL(0xbd5a6f7c,0xfaeeacca),LL(0x00e72c44,0xa2bbbd59),LL(0x47a63782,0xf531aecb),LL(0x67c393bb,0xc7dda450),LL(0x31630b09,0xe719fa6f),LL(0xc95e46b2,0xf849f3a0),LL(0xdeaf5d70,0x68299654),LL(0x827dd5d4,0x286bc1f0),L_(0x0000016c), + LL(0x70c9336e,0x9eae7cc9),LL(0x62e9226e,0x3410c389),LL(0x2f9c24f5,0x71f68cfe),LL(0x090966e9,0xa4f25f7a),LL(0xbb4733af,0x45a0c4d6),LL(0x6303208f,0x0dc0c0a0),LL(0xc45e9f18,0x9a589e6d),LL(0x5c94f082,0xb9c34b5b),L_(0x0000011d), LL(0x0284760d,0x9fc32695),LL(0xaccb375d,0x15ea0e6d),LL(0x5d3b353b,0xc2dc172e),LL(0xdcb147d5,0x96265816),LL(0x7b5ea6b9,0x7188496d),LL(0x6c199f62,0x067983d5),LL(0x8be2b6ee,0x804ab5f0),LL(0xb8433a5a,0xfb9701f7),L_(0x0000017f), + LL(0x9106f208,0xf346db2e),LL(0x75d8b8a7,0x6d441b3f),LL(0x33c9dd4c,0x49bf3101),LL(0x43c1a96f,0x586195d3),LL(0xc64bca08,0x797aa157),LL(0x35872dbe,0xf8494dd7),LL(0x155f257f,0x370900d2),LL(0x42b380b7,0x2ec6eba8),L_(0x0000006f), LL(0x1c0aaf14,0xd54c0210),LL(0x9ba3b710,0xdf76d347),LL(0x79738efe,0x168b7e7a),LL(0x59395338,0x5be21cb3),LL(0x786b578f,0x6c93997c),LL(0xb42b9419,0xda95deb9),LL(0x1b5aa55d,0x1941c038),LL(0x6282d548,0xdc8f9a51),L_(0x0000014b), + LL(0x8b6edce5,0xc54fa3d4),LL(0xc9331956,0x37673a44),LL(0x95ef7146,0x8699a77e),LL(0x35d322ef,0xe5aa4366),LL(0xb0a5fb37,0x1c2f8160),LL(0xdaf11474,0x6c679654),LL(0x4ae2c0a4,0x25f5bd9f),LL(0x0cd1a20c,0x2812f915),L_(0x000001b7), LL(0xead818f2,0x6ba35704),LL(0xddf43f70,0x66ebac5d),LL(0xbd74a353,0x5836be03),LL(0x8be404aa,0x606cb997),LL(0xa72b6949,0xe5570eb3),LL(0xe6f5f53a,0x1550fdd3),LL(0xd841cc12,0x83299eb2),LL(0xcf113a07,0x76117e23),L_(0x0000014d), + LL(0x8ca6af16,0x0412d583),LL(0xe9241a8a,0xac6aa964),LL(0x24077ab2,0xe1536c1d),LL(0xad189da3,0xdf56af4a),LL(0xfbfb6e01,0x46ef9e57),LL(0x8ca8e624,0x7c9eb17f),LL(0xcbcb351a,0xf3eda4f9),LL(0x0985fb54,0x69708739),L_(0x00000029), LL(0x2e5fdd15,0x4656d8d3),LL(0x4d7980be,0x8d7ed681),LL(0x65dbc6a6,0xe9b6528d),LL(0x433cebeb,0x9dfcc27d),LL(0x25c88d26,0x273aec82),LL(0x43f7caaf,0x8662d906),LL(0x1c78327e,0xe4a09eb4),LL(0x04763fda,0xad9bd2f6),L_(0x0000010e), + LL(0x4dc5df22,0x56bd282f),LL(0xc5002426,0x8905b0bd),LL(0xc776fda9,0x5fecb366),LL(0x691702ab,0x8c3c6d77),LL(0x3eafba03,0x7b9bb860),LL(0xa5a38b66,0x288652df),LL(0x9d4dccfc,0x8b876824),LL(0xf937bd4d,0xedcfeccb),L_(0x000001b7), LL(0x6481cd26,0x4223e6a4),LL(0xffdb1d9f,0x44384141),LL(0x4395593e,0x68a0f97c),LL(0x4b9f43be,0xa0157cb8),LL(0x99290f7c,0xcccef6ba),LL(0xef0777fc,0x4349cd23),LL(0xc43d71c0,0x17318622),LL(0xa3bb9fb0,0x5a9a1343),L_(0x0000009d), + LL(0xf567f47a,0xd92ddb9e),LL(0x22ba5c1f,0x65cc6bf6),LL(0xa333c9c6,0xb76d024f),LL(0x521f0218,0x51355f8c),LL(0xb277d241,0x061005e2),LL(0x36014e7f,0x322c2c56),LL(0x52d2dee0,0x8916c0e7),LL(0xf40b1b75,0x0e73d5bd),L_(0x00000126), LL(0xc2ed535a,0x2bc1b23d),LL(0x34905a27,0x31577092),LL(0xac41b5aa,0xf6758a71),LL(0x0e95917e,0xeccda7b1),LL(0x35f458d2,0x52d35c5f),LL(0x61af1ed0,0x42e21d1f),LL(0x99ea1f96,0x1b6c4f11),LL(0x1c9ff42a,0x5a729a64),L_(0x00000031), + LL(0x5f04894e,0xa860137a),LL(0x2bb38059,0x7d4fcbe1),LL(0xaebc7ba3,0x3de80141),LL(0xabc6d3af,0x0ce79f66),LL(0xecaf11f9,0x9743b455),LL(0x23367c7f,0xcf9fcf33),LL(0x458f9b06,0xaf18324a),LL(0x1dd894da,0xc4c95fe7),L_(0x0000010b), LL(0xd8ec1140,0xe5b4edaf),LL(0xff94f2f7,0xdb54a58e),LL(0xc8f912e0,0x655ce3bf),LL(0xdffa0710,0xb0757830),LL(0x0f2d4402,0xb963f905),LL(0x95d5e868,0x098d24ba),LL(0xcdbb826b,0xba591cbd),LL(0xb02b1feb,0x6a8df836),L_(0x000001ce), + LL(0xc29bd6ff,0x108f42a8),LL(0x9cf21db6,0xddcfd187),LL(0x555cfe2b,0xaf269c11),LL(0xe7b4d452,0xc4011856),LL(0x3cb7c3fc,0xfe6e4559),LL(0x415957c0,0x8996b215),LL(0x3b983b2c,0xbe5cf31d),LL(0x0312f953,0x78abc3a0),L_(0x00000170), LL(0x39b80aba,0xd52c0d6b),LL(0x847e724e,0x42bcb7b2),LL(0xcfdfc839,0x2ddac314),LL(0x2aac1c0a,0xa690e67e),LL(0x60736a1d,0xe310507c),LL(0x15f2f407,0xa30b8b85),LL(0x03447dd4,0xf3ddc7c2),LL(0x87208fe5,0x482e1135),L_(0x00000070), + LL(0x93eecb93,0x4b696b0d),LL(0x98400d78,0x671aa2b1),LL(0x05e6f78c,0x19c7b31c),LL(0x2f26896d,0x537e98fd),LL(0xc925cf6a,0x31498b2c),LL(0xe43ae0bb,0x5b628896),LL(0x2cc9c889,0xf81936b3),LL(0xd5a79df1,0xaa558d67),L_(0x000000a6), LL(0xc651825a,0x947e26f1),LL(0xd8102ba2,0xcba2e206),LL(0xf57819ca,0xe1b53333),LL(0xb14e41e4,0x89b722a1),LL(0xa87036c9,0x43d9c2e8),LL(0x8c5e594a,0x8a9f09ad),LL(0xfc198885,0x96afae31),LL(0xd164a447,0xf6ad8705),L_(0x00000015), + LL(0x4517c109,0x51de1f32),LL(0xfe892ec2,0xde0ad941),LL(0xa5e0c485,0x252b5759),LL(0x04504e2c,0xe069dceb),LL(0xceea659d,0x80061659),LL(0xb3fe1e63,0xee236d87),LL(0x846da87b,0xf97ae613),LL(0xa90f8433,0xe7abcaf3),L_(0x000001d9), LL(0x89e00052,0x583a7dc4),LL(0x6598f335,0x8097f299),LL(0x89f7734d,0xff15633b),LL(0x8aebbf6c,0xb01c7b6a),LL(0xb5108c62,0xc7f93ae2),LL(0xf807ee31,0xf990d4e3),LL(0x34992a71,0x9962859e),LL(0x282fca85,0x8047dde1),L_(0x00000172), + LL(0x5435ca99,0x5945bc53),LL(0xcf237d07,0xe31f2468),LL(0x1ef4bb2f,0x641e2901),LL(0x2c562b14,0x06773e1d),LL(0x2ff0373c,0xca66e36e),LL(0xc519e2a9,0x716d0497),LL(0xbfb75cdb,0x9a1fa9e2),LL(0x0dddfb32,0x1e517999),L_(0x00000085), LL(0x85c6aefd,0x9daa8f05),LL(0xe87b5c36,0xba374fa1),LL(0xcd0b7e7d,0xf9ec22c4),LL(0xd20b7cb2,0xdd4d581d),LL(0x516610e0,0x74ddb0da),LL(0x126aa3e2,0x4e09fb27),LL(0x35d95ce7,0x0c242711),LL(0x40d02e9e,0x926ede13),L_(0x0000004d), + LL(0xb5538d3b,0xb7d7682e),LL(0xd8734ac8,0xd322358b),LL(0x75fcdec6,0x56d9d86e),LL(0xe72ffc35,0x6c363b61),LL(0xd1066f3a,0x3d03c2ea),LL(0x7095dbd7,0xcd1674fd),LL(0xb42f9972,0xacc3e682),LL(0x9370acc2,0x80c71149),L_(0x000000fd), LL(0x89c235f7,0x5a49c095),LL(0xe1070948,0xd0d2a294),LL(0xe55bcaed,0xa073e38c),LL(0xd2884da3,0x25346561),LL(0x95d34747,0xbed8195e),LL(0x990b2c19,0xefe701ec),LL(0x17664aec,0x4c59ce88),LL(0xcb5f1246,0xa23cc1c6),L_(0x00000185), +}, +/* digit=67 base_pwr=2^335 */ +{ + LL(0xf7ab9a02,0x72a8d985),LL(0x39a70c2f,0xea7d5b7d),LL(0xf6f8acab,0x73f642dc),LL(0xdeb95d02,0x8a54bc56),LL(0x3f3cc37d,0x5a63f188),LL(0x1ef1bba0,0x8c074b83),LL(0xe112f14c,0x17f937a8),LL(0x29897379,0x4d335ca2),L_(0x00000087), LL(0x70af11f6,0x998374f7),LL(0x8ee96d7e,0x7d432cad),LL(0xbff9e11a,0xe6665366),LL(0xf6d56384,0x9b692423),LL(0xfc7e344a,0xf75f044e),LL(0xee1b3ddd,0xa827ee60),LL(0x9cd00df7,0xb2612c93),LL(0x3d529eb5,0xc4ffa6a1),L_(0x000000d7), + LL(0xa6774f15,0x39c16671),LL(0x3753ce58,0x7356faf0),LL(0x54c5f8c0,0x1165a356),LL(0xf5c1afc7,0xd6adf86c),LL(0x903b89f9,0xba6b4966),LL(0xbff86c3f,0xb1519f4e),LL(0xa87b0151,0xbe4f95b4),LL(0x85efd27b,0x0513d263),L_(0x00000104), LL(0xfa5d90d2,0xd4622a63),LL(0x2aca99c7,0xa6efd8d9),LL(0x1d6acf3a,0x7e55d6dd),LL(0x95a1b738,0xdb119c22),LL(0xd9703d10,0xd11a67da),LL(0x427c0f52,0xe412eedb),LL(0xe055192f,0x174c7a31),LL(0x404a5758,0xfd4b1dde),L_(0x0000015c), + LL(0x86c17046,0xc51693d1),LL(0x57ada379,0x7b463243),LL(0x9dea1156,0x65ad7ad1),LL(0x3431ffb8,0x22995c21),LL(0xbfb6e47d,0xa47b2e96),LL(0xecc87ed5,0xb5d8ce10),LL(0xab93da78,0xe414756b),LL(0x0b5319cc,0x327dee6e),L_(0x00000168), LL(0x0d0b7234,0x68a2cd80),LL(0xd1a1a9cc,0x62b72c52),LL(0x2c285a2a,0x23638d39),LL(0x1736a146,0x90e668b9),LL(0x99d14c12,0xe73e0b34),LL(0x0861672c,0x88955dcb),LL(0x7dc3cdab,0x06284ae9),LL(0x4fad41f5,0xfe7ca883),L_(0x000000d7), + LL(0x7fe98530,0x370d119f),LL(0x9990254b,0x764d3fe4),LL(0x8a86cb40,0x39be0e2c),LL(0x4b9820af,0x458321b0),LL(0x29475227,0x2c2ba583),LL(0xa07a5c7f,0x52e9ae89),LL(0xfa6d5206,0xbdc0eee8),LL(0x435604c9,0x6f2e4842),L_(0x000000c0), LL(0x2bfa81c0,0x1714a30a),LL(0x66400030,0xa3f37cbd),LL(0xab87a938,0xf3132874),LL(0xecbe1c91,0x7e9d7ac6),LL(0x1734fae5,0x7e33fb88),LL(0x600765b0,0xedfc073b),LL(0x428cbfb5,0x85e9a209),LL(0xbd290285,0x77ef7692),L_(0x000001e0), + LL(0xa928bd97,0xec2103fa),LL(0xa28d165e,0x31bc9c0a),LL(0x9e3c2272,0x1db480d5),LL(0xeefa29bf,0x5fe970c7),LL(0x0625c44f,0xa6473b7b),LL(0xd9e52858,0x66b89d6b),LL(0xea4cd7c9,0xc3e3b579),LL(0x3b6547a2,0xa8bdce16),L_(0x0000004e), LL(0x4065d81f,0x66f8ad71),LL(0x8a1901b7,0x08802270),LL(0x0ceae5cf,0x853e6e60),LL(0x63ae6ec1,0xd89e54bb),LL(0x1f365d84,0x9e03d94f),LL(0x49df44dd,0xddc9e1b3),LL(0xf70096c2,0xba5865a7),LL(0x6cc7a69e,0x01800fb8),L_(0x000001ad), + LL(0xd6f6ffe5,0x91cf34a0),LL(0xc22b2802,0xfd975370),LL(0xe87a26b1,0x662b3666),LL(0x46d8088a,0xa7466010),LL(0x0988f2ee,0xd8edbdfb),LL(0x1f7fc1eb,0xf4b2f213),LL(0x266b6d41,0x00896949),LL(0xc83c1c02,0x7c849de8),L_(0x000001e2), LL(0xf3a594e4,0x90bcda37),LL(0x480ec74a,0xdde9d726),LL(0x26216e2d,0xbef16495),LL(0xa5c64b02,0xfba3c749),LL(0xc3f630aa,0xea872930),LL(0x8695df3b,0xb21d654a),LL(0xb5372491,0xa2f3f6fc),LL(0xe917f3b3,0x13fe01cf),L_(0x00000146), + LL(0xd606cedc,0x12faa2d3),LL(0x93a86247,0x67da197c),LL(0xd4612a0b,0xfd72cd4e),LL(0x145bbdec,0x8483b822),LL(0xb4dba8a8,0xf56d58c6),LL(0x81ec1ed7,0x708f26ec),LL(0x631a5032,0xf4782ede),LL(0xb4b04fc8,0x34a251ec),L_(0x0000008d), LL(0xd85c10f7,0x1b8eedbc),LL(0xadbb93b8,0x1b68a64f),LL(0x2308b6f8,0xc37e6bd6),LL(0x392cec36,0x7b419db4),LL(0x0aee0e63,0x1777f3d5),LL(0xff167cd9,0x706d5278),LL(0xede81ea9,0x2ecab8c0),LL(0x63b7e96c,0xb7b3cb43),L_(0x0000005a), + LL(0x7b39090f,0xec8937b4),LL(0xbb7112fd,0x3ed5a415),LL(0x66e9e19d,0x01eab0fa),LL(0xbbe65978,0x740c409b),LL(0xba92675e,0xa050b19b),LL(0x3e8b56da,0xe6eedf2c),LL(0x2fbcf099,0xa55e0691),LL(0x80195262,0xf2c7d1e9),L_(0x000000e2), LL(0x013e53cc,0xeec1384c),LL(0x51ffea5b,0xeaca6749),LL(0xd0ad477d,0x45756473),LL(0x1fd4ee32,0x80864216),LL(0xa3069430,0x850c8b97),LL(0x405f653b,0x9de4340f),LL(0x5a543cae,0x347d550d),LL(0xa331ca24,0x75f4312e),L_(0x0000012a), + LL(0xf7fa7404,0x6f04ee1f),LL(0x5987ecf0,0xe39114e7),LL(0x8c92a999,0x734b40d0),LL(0x3eb0ff94,0x35b97b9a),LL(0xf3a7b34c,0xd5b35118),LL(0xd35276ae,0xfa36fe0d),LL(0x9e933110,0x3c37067d),LL(0xbe64ccaa,0x9629f86f),L_(0x00000157), LL(0x1cfc72eb,0x2f066fb6),LL(0x69fb1dea,0xd4e489f4),LL(0x65ba5821,0x3abd59aa),LL(0xf0068abc,0xf152d51c),LL(0xfa26b25b,0x4c7900c0),LL(0x929fd963,0x22beebe6),LL(0xd19508e6,0x6c8e147f),LL(0x84fd88ff,0x8a8eae8c),L_(0x00000146), + LL(0x746ef6f8,0x58c8162c),LL(0x15dbf9ea,0x7dd87f67),LL(0x5523d821,0x2bc5b0b4),LL(0x8780b2c5,0xb8903ecd),LL(0xf92e785d,0x54296f75),LL(0x6397e404,0xcce33c6e),LL(0x84bad1cc,0xd3c5f54e),LL(0x95b82162,0x33f935ae),L_(0x00000093), LL(0x3e26a2e6,0xf54ff1c3),LL(0xb4ec10e1,0xc2886785),LL(0xa1634274,0xe5822d49),LL(0xbfab5d5e,0xbe9122e0),LL(0x955a062f,0xf03c2cc2),LL(0x579ad9b7,0xf2e5e08e),LL(0xdd6ee255,0x1b65e701),LL(0x934f08f7,0xcd7d23cb),L_(0x0000009d), + LL(0xaddd73ec,0xaeacb5da),LL(0x9e604f26,0xf00003d7),LL(0xb32a37b8,0x5e05fada),LL(0x771ab3ac,0x03aa3a60),LL(0xe17eba9f,0x31442064),LL(0x7239319d,0x021c13e4),LL(0xa35b4712,0x3f7e400f),LL(0xc6e5283a,0x5888abe0),L_(0x00000110), LL(0x45cb211a,0x0081a506),LL(0x59fee06e,0x6e2f4f1a),LL(0x372aea8b,0x11014792),LL(0x2416f852,0x35841e3f),LL(0x71de69f4,0x710be3f1),LL(0xfa1b9018,0xd8151855),LL(0x92a94717,0x52addc4d),LL(0xff148def,0xea65eaa7),L_(0x0000003b), + LL(0xd85c3892,0x01927819),LL(0xd986cf50,0xab17488d),LL(0xeb6ebddd,0x8628281f),LL(0x92fa1e38,0x4511bc03),LL(0xe1691b01,0x0b79c2a5),LL(0xb842f844,0x8805d866),LL(0x343c71a4,0xd5d795d0),LL(0x042ac5c5,0xedd85588),L_(0x000000aa), LL(0x875f110a,0xfa896067),LL(0x0d43dab8,0xd6f1580b),LL(0x9a3104ec,0x840b3b59),LL(0x76717c31,0x41201091),LL(0x2243eb78,0x88de871b),LL(0xe2323a38,0x53ac3f7b),LL(0x764799c3,0xdaf476ff),LL(0x999d244e,0xc595b87c),L_(0x00000113), + LL(0x7b2cee87,0x8985e912),LL(0x3a4ec3d9,0x639aa554),LL(0x9515b9e8,0x2b90b395),LL(0xe330201b,0x5d9f9a07),LL(0x95f372a0,0x83696835),LL(0x0d679c0c,0x8c8132ef),LL(0xbc4509be,0x5efb5013),LL(0x1a4c14f9,0x7a8442ff),L_(0x0000016b), LL(0x27796676,0x1d220323),LL(0x607be429,0x7b931a60),LL(0x3b156b2a,0x38e10514),LL(0x36aabd76,0x8ff10073),LL(0x384c71ad,0x3603a7b3),LL(0x1c1f643f,0xe305de49),LL(0x08714206,0x63b53241),LL(0x57add901,0xfba33a9a),L_(0x00000151), + LL(0xe6fdb7c1,0x03800832),LL(0xceb77120,0xc3398df2),LL(0x625cbf7e,0x4c74b442),LL(0xdf67b9eb,0x424f2515),LL(0x6b977890,0x2abc1051),LL(0x2462b490,0x7c8f5df8),LL(0x6d60d980,0x1bb838ca),LL(0xff606aee,0x1e7a2ff1),L_(0x0000010e), LL(0x8ce853f5,0x238fce9b),LL(0x49a4c209,0x73f7bb70),LL(0xae2b39f7,0xcfbf4b1e),LL(0x53e2f55b,0x7309ae96),LL(0x026775b0,0xfcf869d6),LL(0x1b0e83da,0x72ed99fe),LL(0x02f8a21d,0xe81bf7a2),LL(0xd737619d,0x2bf238b7),L_(0x0000017b), + LL(0xd12397f2,0x38b832fc),LL(0xb158ca8a,0xb03ae8bb),LL(0x7420634b,0xac8efa9e),LL(0xaa8dadd3,0x259a7c31),LL(0x525d1488,0xf727fdfe),LL(0x8b687898,0x48ffe9fc),LL(0xfbbb696d,0xb00d2cb4),LL(0xf4a8353d,0x9be76892),L_(0x000000b6), LL(0x99e21f84,0xf73fea91),LL(0x0e2a8788,0xfce9c9d4),LL(0x18c2b050,0x3412a938),LL(0x1844f4f0,0xc14eff85),LL(0x4ad5ef54,0x0ad47ee3),LL(0x01446b7d,0x4c160f3e),LL(0xc22336c4,0xc43f42cf),LL(0x64f51a4f,0x34833c47),L_(0x00000155), + LL(0x54298a18,0x7f8ca9dd),LL(0x308270c4,0x433780e1),LL(0x2be3ff97,0xb186cdf8),LL(0xeb6471b0,0x747ce696),LL(0x05f60ab1,0x815fe310),LL(0x5a457da8,0x6bb9ac1b),LL(0x4901af0f,0x925bd14f),LL(0x652f972c,0xacd8b58a),L_(0x00000186), LL(0x72184c64,0x3e0f3553),LL(0xf7ba0c69,0x6dbda5e1),LL(0x5b11db3d,0xe46fae1c),LL(0xf1f51b89,0x7c0b46b0),LL(0xa48d5ec9,0x1cc037ca),LL(0xbdcc7599,0xe5f40355),LL(0x28784dc0,0xd1a3fa4f),LL(0x0b837ead,0xb5a1926d),L_(0x000001ee), +}, +/* digit=68 base_pwr=2^340 */ +{ + LL(0x97e26dd0,0xde847fc3),LL(0x2cc0ff17,0x8ee8bde0),LL(0x63e6fe38,0x774123d0),LL(0x97b1e73a,0xf8cb9f55),LL(0xee20c8da,0x8b938535),LL(0xa8b1bfac,0x3df10d7d),LL(0xe2a81936,0xe7d42738),LL(0x9979861a,0xaf0a6779),L_(0x0000003e), LL(0xde00c8a6,0xbb611141),LL(0xaefc5b58,0x9b3d91d2),LL(0x33633ce2,0x0d5e3306),LL(0x2544bfef,0x8956c10a),LL(0xbeaa2dd7,0xcfa84101),LL(0x8ba22402,0x96a5f958),LL(0x0df8f462,0x34ace12e),LL(0xc6c70187,0xe0e00d25),L_(0x000000a8), + LL(0x28cda662,0x8cc26d9a),LL(0x4788e280,0xddb52ac3),LL(0xa2e3c895,0x305bc55d),LL(0x6e792cc3,0xee807637),LL(0x5f9d9986,0x0da2d9cb),LL(0xaea185a3,0x56f8f0d8),LL(0x8ef462b9,0x096bebd6),LL(0x300f61c2,0x7be11b59),L_(0x0000004e), LL(0x15fdd133,0x9dea58bb),LL(0x2585fb77,0x8271cd5b),LL(0xd75c3d97,0x5ac1a4b5),LL(0x4eee827a,0x6fe4d480),LL(0x0147df6c,0x7f66c09f),LL(0x3f6a4c21,0x92c3b320),LL(0x1eeb9502,0x17768418),LL(0xebda3f44,0x12f688be),L_(0x00000117), + LL(0x8027c8dd,0xea4586de),LL(0xdc25b073,0xf24773b7),LL(0x49b36fcc,0x8466e794),LL(0x3cc03245,0x36f85404),LL(0x02d71d6f,0x251e24c9),LL(0x1a81ea4c,0x9e39ec68),LL(0x0e710d11,0x0e27e696),LL(0x44dfa8e5,0x6885ae0f),L_(0x000001c9), LL(0xe3e54e0a,0x7872cc7e),LL(0xb8224b0f,0xb5696e54),LL(0xe5d4bd3d,0x64ead37d),LL(0xcbad27cc,0x5b2f36d2),LL(0xa45e529b,0x521210e2),LL(0x788fbf74,0xe1bd2c8d),LL(0x40c5440b,0x2b990184),LL(0xbc157b39,0xeeecc510),L_(0x0000005d), + LL(0x1941c1e9,0x241f3444),LL(0xaf242ed2,0x73f71786),LL(0x82ffb7bd,0xee8f684b),LL(0x0b940040,0x0e0a7766),LL(0x3cb9180b,0x05509077),LL(0x41ff934d,0xd7e63b43),LL(0xfe154397,0x597b1f2e),LL(0x73566086,0x115dad8d),L_(0x0000018e), LL(0xe97eed76,0xa0320279),LL(0x44ad3c59,0x95d0b92b),LL(0x9546c02d,0x17644016),LL(0x0cf317d6,0xb1278de8),LL(0x0cc035b3,0xa150eec2),LL(0xa454911c,0x40f34047),LL(0xb15350f1,0x64a854ce),LL(0x24297dd6,0x45fd389a),L_(0x00000015), + LL(0x2f4ddeb6,0x1b54c6d4),LL(0xd46442b3,0xc94971a5),LL(0x8eb97dd3,0x9e3e1ff6),LL(0xe2df3525,0x058db0b2),LL(0xe0b69449,0x46968077),LL(0x4ea6ec9f,0xba76456e),LL(0x8457796a,0x77183369),LL(0x255412cc,0xb4e4306f),L_(0x0000005e), LL(0xf2060dc0,0xcef4a91b),LL(0x57dadc33,0xb934de9a),LL(0x51f56ade,0x2e8fe341),LL(0x49ab29d2,0x85723b5e),LL(0xfabf520f,0x1ee66b41),LL(0xbffe6761,0x0a605253),LL(0x50202f55,0x9e49468c),LL(0xef6d250b,0x956ea13f),L_(0x000001a2), + LL(0xea2b5246,0xe30dc097),LL(0xb5323240,0x3dd49c5e),LL(0x082d33c5,0x17c692b6),LL(0x12bc5d69,0x7e9d695a),LL(0x02c7fd33,0xb78372d6),LL(0x85e92117,0xefa0ef29),LL(0xceacebcb,0x9b4e8e18),LL(0x3d68cc3e,0xe3e50ee1),L_(0x000001b3), LL(0x0d2f5e36,0x5f980bec),LL(0x9b062f58,0x03edaeaa),LL(0x83ee3b51,0x4f1f8028),LL(0xe4a60553,0xd292d29e),LL(0xf95f8dad,0x7f9ba28d),LL(0xfb5785f5,0x54b8d134),LL(0x0fe162fe,0x2bd8287f),LL(0x2247f090,0x769ce1a1),L_(0x000001e3), + LL(0x8c32c911,0x416e3d53),LL(0x74341e71,0xa2b4eeb7),LL(0x9e6b225b,0xff742236),LL(0xd3c824dc,0x5e3b9c67),LL(0xd547224d,0xc6e4276d),LL(0xcc105d46,0x8a001d2d),LL(0x92da4b3a,0x3b252483),LL(0x67e2f395,0xcbb5174b),L_(0x000000fe), LL(0x508826b7,0x2308d720),LL(0x4e2e8071,0x9562db9b),LL(0x1bf6b116,0xb14e6841),LL(0x45f6d2ac,0xbfd32572),LL(0x8cdf458c,0xe189ac1c),LL(0x3f894fd3,0xb5d6ee49),LL(0xa59cbf0a,0x672a0c8f),LL(0xd8476d8e,0xfbb78753),L_(0x000000d9), + LL(0x1cd9788c,0x4e39334c),LL(0x0560e3e7,0xc62b58ab),LL(0xfa2ff6e2,0x25b0cfee),LL(0xe3dacd7d,0x56f469ae),LL(0x6476d0c4,0xaa862fad),LL(0x0d8d2340,0x468a688f),LL(0x648a9494,0x9d4d2feb),LL(0xe9f4ed20,0x3e7c5890),L_(0x000000a9), LL(0x3f607213,0x12cdd07e),LL(0x60784376,0x8d6d1246),LL(0xea183586,0x85400753),LL(0x5c35ba6a,0xf252808d),LL(0x57ba4df7,0x07b45d68),LL(0x048ddde7,0x8c60d683),LL(0x759393e3,0x919b183c),LL(0x760c630e,0x90171725),L_(0x00000120), + LL(0xe24eabd7,0xc6d9113b),LL(0xdfe5709e,0x1b269289),LL(0x74793368,0x0c1477a3),LL(0x3c2fbb12,0x544d389c),LL(0xbce04baa,0x4fcb6058),LL(0x52137546,0x62503bea),LL(0x306eaabe,0xb0495976),LL(0x49afd268,0xa62777c0),L_(0x00000171), LL(0xd8dbfcd4,0x5a2c2301),LL(0x122ed53b,0x23b67f05),LL(0x68235f24,0x0b4f5601),LL(0x850a3bcc,0xfc63cfca),LL(0xfb6987d1,0x7f3a86aa),LL(0xeb70f694,0x726395f3),LL(0xe2648fe0,0xfc883eec),LL(0x04f6906b,0x551243f7),L_(0x000001a2), + LL(0x1ea7b394,0x1329d8d9),LL(0x09b5385e,0xe206cc29),LL(0x8ad6ff5e,0xaa658f94),LL(0x8f946f98,0x428d98ec),LL(0x2b10de6d,0x3ec6baf6),LL(0x7efb3f61,0x21982753),LL(0x47b7f15b,0x578e65b3),LL(0x63e00597,0x5b3f1a2b),L_(0x0000010b), LL(0xc87fe251,0xec6244fb),LL(0xb103454a,0xbf7ff50a),LL(0xc09eeea3,0x5b8194a6),LL(0x117b25e8,0xb8d95b3b),LL(0x4719134d,0x98e3eda5),LL(0xfb408475,0x184c4131),LL(0xaae4a703,0x433cd4bd),LL(0xee1c89f5,0xec3f5308),L_(0x0000005f), + LL(0x15497e40,0x0a7c8b60),LL(0x2ac3e5be,0x35df434c),LL(0x73674292,0x218ab0b7),LL(0x377ea177,0x971e7bf8),LL(0x979969d0,0xa6fc69b3),LL(0xf051da86,0x0f199014),LL(0x50a5af5c,0x0bd76f2a),LL(0x68112265,0x6ef71ee6),L_(0x000001c9), LL(0xb6a11a9c,0x2c92d1d2),LL(0xa2718b70,0x8bc153fa),LL(0xefc700c8,0x7d63139e),LL(0x11bfab90,0x63a1a94a),LL(0x1c81d0a8,0x0af04b86),LL(0x5010811a,0x65f6b2d4),LL(0xd230f7b9,0x346b9acb),LL(0x92db7ff2,0x791ad571),L_(0x000001f4), + LL(0x731fb0e3,0x0bfa96fe),LL(0x4b219ff8,0xe186bb8d),LL(0x34f0b2a5,0x4bbcffce),LL(0x74e4771d,0x4fa8ab52),LL(0xa8af538f,0x0bd9aa17),LL(0x55b8ec20,0x24b82cd8),LL(0x55a1e2c4,0xf54b7fae),LL(0x34a02ce9,0x011a367d),L_(0x00000056), LL(0x49d91d2b,0xb4418223),LL(0xf5b325cc,0x48062883),LL(0xae0ea746,0x3f6511a6),LL(0xa6b89032,0x057c30cb),LL(0xe93a6d10,0x00156e39),LL(0x0c08d7cc,0x7ef1eeb2),LL(0x713dea1e,0xeb905073),LL(0x1c6abbcd,0x4e60ee01),L_(0x0000019c), + LL(0x0253f26d,0x1db48f4d),LL(0x239e10c2,0x3cbc9af9),LL(0xde4c09f2,0xabdb2bfd),LL(0xed72b749,0xf64b181a),LL(0x928d6d6b,0xc85d2a91),LL(0x9af06cb2,0x32a985b4),LL(0x67685c7b,0x4cf68328),LL(0x607e3b33,0x96bdc7c4),L_(0x00000164), LL(0x47c82fba,0xc31b32d2),LL(0xd33b72d3,0xf9f3e673),LL(0xd3e9eee6,0x42fb631c),LL(0x43164033,0x6eedae95),LL(0xaab1e76b,0x986360ea),LL(0x9e6c1afa,0x30e60546),LL(0xffd16939,0x06fbdf1b),LL(0x989aed4f,0x7870903a),L_(0x00000135), + LL(0x02ec16f2,0x1065b996),LL(0x62ecea90,0x92968f64),LL(0x48a2356f,0xe74f9d69),LL(0xc38162fc,0x799bc2e4),LL(0x109958a5,0xc2cf37fd),LL(0xc511c5bf,0xb810f3aa),LL(0xba774e2d,0x04423cab),LL(0x7df651dd,0xfa41efc5),L_(0x000000be), LL(0xc60869cc,0x843bf3e9),LL(0xca2ddefb,0x07001aa8),LL(0x2325dc6b,0xfb77a925),LL(0x19ebdc37,0x984cbf76),LL(0x991269aa,0x0ea96690),LL(0x5a99df72,0xf5a76c72),LL(0x261ea01a,0x11ed492c),LL(0xec2261aa,0x511c25de),L_(0x000000dd), + LL(0x2208733f,0x79d76ece),LL(0xdf3265ef,0xdd97cbec),LL(0x1201ac02,0x82ed80a6),LL(0x3e075507,0x2e6d2b22),LL(0x16c3d5bb,0x9b31411b),LL(0x89a53e5c,0x4570ab97),LL(0x5d2d34d3,0x76b65f73),LL(0xd36a94bd,0x3b4c1384),L_(0x00000038), LL(0x7939931b,0x91c17d4a),LL(0x850a6308,0x5b75be0c),LL(0xa8eb3e41,0xc4b1a960),LL(0x5176e7e2,0xef74a564),LL(0xf493e8d5,0xd77a5683),LL(0x2824a377,0xdc2b6e93),LL(0x3d90456b,0xf50c072e),LL(0xfe49afc1,0x3768ae59),L_(0x0000003a), + LL(0x0b81885e,0xc160ead5),LL(0xf73cb6f0,0xe2cd2bce),LL(0x53fb7eb3,0x9bb7bc05),LL(0xb6b62b31,0x8a471706),LL(0xdaf298a3,0x846c6b42),LL(0x9d404cf9,0x085e05e5),LL(0x8fac2e73,0xc53893b4),LL(0xa8ff0af6,0xe3d8eea7),L_(0x000001e9), LL(0xcd5b055a,0xe934470a),LL(0xc1af94ed,0x89b3f0c4),LL(0x4fba6b38,0x9af80496),LL(0x3e423ee4,0xd89fd53a),LL(0x793fadf0,0x353cc302),LL(0xcd463b61,0x51fbb36d),LL(0x782e102e,0xd6d1c6c3),LL(0xbbc63732,0xc97f604b),L_(0x0000019d), +}, +/* digit=69 base_pwr=2^345 */ +{ + LL(0x4a7e1773,0x5fdb3148),LL(0x0a1bc3a0,0xbda07174),LL(0xc265e95e,0x43b1d3c4),LL(0xedc546d6,0x611b9709),LL(0x807b04c2,0xe15784fc),LL(0x473ceec4,0x58afe5bd),LL(0xc97fb33e,0x7b94dc53),LL(0x39d6d532,0x914fc6dc),L_(0x0000002a), LL(0xd880ddb6,0x67cbcf73),LL(0xdf80627a,0x67d3e029),LL(0x91ccf95a,0xfd91b52d),LL(0xa53ec7ae,0xa855273c),LL(0x1131573a,0x07213a95),LL(0x49db550c,0xfa5fb98a),LL(0x470643af,0x0fa67f5b),LL(0x6bf0628a,0xf906186e),L_(0x0000012c), + LL(0xe3e2f37e,0x7f1be067),LL(0x7f152d2c,0x8659f773),LL(0x242f8dec,0x8df47d63),LL(0xcbac7d95,0x91c63b0a),LL(0xf620881b,0x8c2c6ad3),LL(0xda54c002,0x701999ad),LL(0x3012f693,0x99516f6a),LL(0x3c14f4c4,0x68d44cb7),L_(0x000001a0), LL(0x8d1b414e,0x1d209ec5),LL(0x76dcc740,0x51dc58e8),LL(0x93231067,0x4e14fe98),LL(0x5a4f75f2,0x88f5086e),LL(0x4ccd6aac,0xf294dbdb),LL(0xedf86543,0x30e3be99),LL(0x767f48ab,0x22dcd0b3),LL(0xd87667c6,0xd6615681),L_(0x0000017f), + LL(0xb6dcc9cf,0x0d5b110b),LL(0x1e6c13e6,0xa0842e9d),LL(0x58585159,0x443356f4),LL(0x265b46fe,0x3b25c086),LL(0x87729185,0x0ebbff20),LL(0xc71c4d1d,0x79c2136e),LL(0x3ebeca5e,0x3a96ed20),LL(0x32a0af3b,0x012e330a),L_(0x00000193), LL(0xdacd4b61,0x7d17ed67),LL(0xde583a56,0xe5703cea),LL(0x316840b4,0x03d396ce),LL(0x0cfc4143,0x6970ea48),LL(0xb1974ced,0xa63761b9),LL(0x3de4383d,0xe400788f),LL(0xd07d6726,0xa545993a),LL(0xa33056bd,0xd3fe822e),L_(0x00000185), + LL(0x04e40c0b,0x2f073480),LL(0xf78fb060,0xb2604448),LL(0xc2ac8aeb,0x277ca03f),LL(0x27864b78,0x2c6f473d),LL(0xeb1664eb,0x7cb793a9),LL(0x58eee9a3,0x9c2ce2b3),LL(0x94f18cbc,0xc87a3dc1),LL(0x2df6078b,0x0e93cd11),L_(0x00000022), LL(0x96d6d2f1,0x3fd1ed5d),LL(0xbe10752f,0x0c38fb72),LL(0xd1e47666,0x9b093c35),LL(0x73ab1d6d,0xd898dff7),LL(0x00e4cf26,0xc4b445df),LL(0x422c1136,0x9ee0d1ce),LL(0xb6e821b5,0xe82511ed),LL(0x1ade6252,0x81c804e4),L_(0x000000f4), + LL(0xd903ff9c,0x47ebbcfa),LL(0x75ed251a,0xc45b660c),LL(0xef50c145,0x733af648),LL(0x00ba4ec8,0x5119e848),LL(0xb32a226b,0x3e048abf),LL(0x1f648014,0x7a1eccf1),LL(0xfab712cd,0x2badbf53),LL(0x1ebfb8b6,0xca11e7be),L_(0x000000dd), LL(0xa034de60,0x789ae7e1),LL(0x42374e6e,0xa641227e),LL(0xb2f0016d,0xa82cadf5),LL(0x8ca0d43e,0x552e57fb),LL(0x5a71e67c,0xa8bcdb55),LL(0xc3a4e9f5,0x6e0e2bb2),LL(0x124b38d4,0xc1f40c89),LL(0x60b28fe4,0xabb2620f),L_(0x000001f8), + LL(0x54c77c89,0xfaecca00),LL(0x028ae659,0x3a3c5350),LL(0x656b887c,0x5b3bb83d),LL(0x0c6fbb7d,0xc2d4fb00),LL(0x8f5877ba,0xea3d0289),LL(0x01e01a3d,0x943983b2),LL(0xdcb878b4,0x382b3bfc),LL(0x7e566dbe,0xe238f00d),L_(0x0000010d), LL(0x312dd2eb,0xcf1a3019),LL(0x6043d5c3,0x3f970442),LL(0xd72b4978,0x47e77b41),LL(0x51765a88,0xd9f49bc6),LL(0xc2232c0f,0xf1592cf2),LL(0xba6cd5b1,0x7bdf89de),LL(0x33024471,0xa56dd8d4),LL(0x4e3c8e30,0x6718468d),L_(0x000001f6), + LL(0xb8ab0296,0x1c158b81),LL(0x3cf9ac17,0xe81b84be),LL(0xb9565ca7,0xcf197d97),LL(0x30adec74,0x0f74f924),LL(0xbbec4f93,0xc6810de9),LL(0xdd1aa6b4,0x68e13e67),LL(0x31b1e868,0x3b5ec800),LL(0xb287ff46,0x970d17f9),L_(0x00000148), LL(0x4e217778,0xfc348975),LL(0x3990d639,0x2c996afc),LL(0xbf8e72bb,0xc43038ef),LL(0xf3f9e817,0x673d9367),LL(0xb8d625be,0x33a4c44f),LL(0xb72553a3,0x6135473a),LL(0x7365b95f,0x91b434c7),LL(0x70995374,0x2e7887e0),L_(0x00000182), + LL(0xf2b44edf,0xf8d7989a),LL(0x1b1a9086,0xb30f16f9),LL(0x2b4d5672,0x919c3dae),LL(0x64f72009,0x3b6ec0a9),LL(0x64ce5600,0xc12b7f4f),LL(0xe0f0d4fb,0x144476bf),LL(0x3eb40f82,0x0332a8a4),LL(0x80cbb448,0xf3755660),L_(0x0000013f), LL(0x5e5d7124,0xe518caf3),LL(0xa53d6591,0xe170d9ab),LL(0x4f1b1b50,0x3e56ed3e),LL(0x7c4f9eca,0x8dfe4cc6),LL(0xfa0dd028,0xa59c7726),LL(0x0a01f234,0x6c8fa066),LL(0x704007db,0x8366767a),LL(0x4570b32e,0x94810fb8),L_(0x00000169), + LL(0xa5e76230,0xce3ac7a6),LL(0x5ab23199,0x54772e62),LL(0xd7eb0723,0x8caedc24),LL(0x76dd866b,0xa41bb763),LL(0xba5b9e92,0xe6d92de4),LL(0x28e72bc3,0xce269dcb),LL(0x2b7dc535,0x7e64fab4),LL(0xe7cccbfd,0x02a03896),L_(0x000001c3), LL(0x278c892b,0x3dc101fd),LL(0x079c3e36,0x2332d512),LL(0xf51726ad,0xdbc6dd2c),LL(0x81e5e3b6,0xbd26d73b),LL(0xb2e70917,0x24b0b54b),LL(0x23cdf2f9,0x0353dd40),LL(0x5e2acffb,0x2e871e61),LL(0x07f5e7c3,0x5b299ee3),L_(0x000001c5), + LL(0x66643187,0x3ec2ad9c),LL(0x115859c9,0x1f8e247e),LL(0x17b4d84b,0x2585ef39),LL(0x657d1198,0xa3f20465),LL(0x3619c497,0x6901431a),LL(0xe9a26e53,0xa5276fd0),LL(0x496e3f3c,0x1f276a6e),LL(0x348de17d,0xf8c3af85),L_(0x000001ca), LL(0x2d6d1037,0xb1e10393),LL(0x7b5ce8d7,0x6deb4f48),LL(0xc7dc292e,0xc43c502b),LL(0x6b03f2ee,0xb1a1182c),LL(0x39273259,0x7622b369),LL(0x4498e583,0xfe7d43c5),LL(0xa50df54f,0x61447fad),LL(0xc4339095,0x3a9a1f2a),L_(0x000000b7), + LL(0xcefa0f75,0x4af9a66e),LL(0x310c7a00,0x8db46828),LL(0x9223ef88,0x2cf042cc),LL(0x6fa5a04f,0x7c559200),LL(0x6daaab2b,0x91afc691),LL(0x98a36fb2,0x00b03676),LL(0x188b128b,0x51264914),LL(0xb9803ddb,0x75459c1a),L_(0x00000062), LL(0xbedae6b1,0xe3aaa000),LL(0x7a690ecd,0xceb2c1c6),LL(0x8880b402,0x45703d2c),LL(0xc0613e4c,0x8b8c9eb5),LL(0xc5db208f,0x4c846b8c),LL(0xa9987715,0x6bd91493),LL(0x42b48717,0x9a26dd34),LL(0x3e788715,0xd9b2c58c),L_(0x0000011e), + LL(0x2a0a9afe,0x361ab1b5),LL(0xccf7e9f1,0x24eb3b91),LL(0x5a00b663,0x3a047d69),LL(0x70c649d7,0xff025b44),LL(0xbfc42749,0xf2772669),LL(0xfa1fa0b0,0x1d856fe2),LL(0x59b50aeb,0xdd9c5baa),LL(0x07db1f1d,0x63853e29),L_(0x000001cc), LL(0xa8c851af,0x924ac747),LL(0xaa197f48,0x269fd103),LL(0x40db5d37,0x4cdd9698),LL(0x1838b760,0x491094a5),LL(0xfe6931b4,0x6e498775),LL(0xf5d608e3,0x0107968f),LL(0x1ebcbfad,0x4aa5111c),LL(0x9da743a4,0xcba06022),L_(0x000001c8), + LL(0x92933f27,0x47445d4b),LL(0x06b03d8a,0x4a0fee6b),LL(0x3277857c,0x1fa95d1d),LL(0x87c700bf,0x48c9ebf0),LL(0x2a50e2b3,0x43aa2c31),LL(0x9a26e1ce,0xfdbdd0e9),LL(0x52ca3b19,0xfcedcc05),LL(0x2999c847,0xbff002c4),L_(0x00000000), LL(0x3f6f21d0,0x64d4906f),LL(0x556e52bc,0x42f805ae),LL(0x90f2f1b6,0xda19f3db),LL(0xd869ad98,0x21709755),LL(0xf5bb8103,0x573f3fb7),LL(0x593f7e34,0x2bac9d40),LL(0x02d84c5f,0xa4e8ad8e),LL(0xea53406e,0x80797c51),L_(0x000001d4), + LL(0x1cc4de54,0x02db6ff9),LL(0x9a077100,0x442b39ab),LL(0xa4a99831,0x658c33f0),LL(0xd154641a,0x6b185008),LL(0xbb725f43,0x1c42ee9d),LL(0xc802ceb4,0xd4f04052),LL(0xde1c27e0,0xaa13dd25),LL(0x5e0fdafb,0x59845ec9),L_(0x0000003c), LL(0x0ff8a42e,0xd7b6958f),LL(0xc632359e,0x00693302),LL(0xd897df30,0x3a9976fe),LL(0x07f17ce0,0xb3320f5d),LL(0x0634d694,0x9281633f),LL(0xf481d736,0xa1f17a34),LL(0xc568493c,0x7973f20c),LL(0xc81657ab,0xe331f1c3),L_(0x0000010e), + LL(0x4294bb48,0x02479c06),LL(0xf0c74cf8,0x82afef4d),LL(0x3fe27e63,0xc6ed0fe5),LL(0x2473be19,0x88c3bb5a),LL(0xf0f01c24,0x8b895708),LL(0xff99c560,0xda940b97),LL(0x19fe223c,0x3a3e9972),LL(0xfce2e466,0x48aaf92b),L_(0x0000002b), LL(0x2fe2dabd,0x1f0040f3),LL(0xc569b506,0x6e592ad0),LL(0xa59e0b36,0xe1a1ad5f),LL(0x506ba908,0xdd28fbcb),LL(0x35a90e9a,0x5dfd21f8),LL(0xae82baef,0x5c23fd74),LL(0x03b13133,0xe27cfbd6),LL(0xa41bf476,0x7af7e895),L_(0x00000028), + LL(0x7b4c0b5b,0xdf92a189),LL(0x7fe35612,0x9f97d902),LL(0xa5105c7f,0xba021326),LL(0x9c521853,0xd2cadb21),LL(0x9b3ed45d,0xb3ab9d25),LL(0xddadc1eb,0x7b4a857f),LL(0xaddab060,0xbbc92a34),LL(0x7dff916f,0xee7e6c52),L_(0x00000121), LL(0xc6f5958e,0xa3ba1dab),LL(0x7b0e564a,0x27f6c008),LL(0xb9c963dd,0x3c030970),LL(0x76828eca,0x3cd7a457),LL(0x0834d3c2,0x417b833d),LL(0xf44c50d5,0xf0d8be25),LL(0x153d4a6b,0xa49590ea),LL(0x47e9c71d,0xf3a30dc2),L_(0x00000143), +}, +/* digit=70 base_pwr=2^350 */ +{ + LL(0x5891669d,0x995d0979),LL(0xe0c6b8f7,0xbaad255c),LL(0xcdfa7a67,0x2a84102f),LL(0x2ed9a546,0x5fff322b),LL(0x238b4cca,0x316895f0),LL(0x27a1fc8f,0x4369e1fc),LL(0x399300db,0x708ed717),LL(0x5f16f718,0x931503fe),L_(0x000000a4), LL(0x1ff0c6b2,0xa040da9e),LL(0xf9967269,0x8694422a),LL(0x5871908b,0x8350fa73),LL(0x61c801c8,0x80cd1b5d),LL(0x26cc63f6,0xaf0b4158),LL(0x3474f5f7,0x1e938b36),LL(0x7800e540,0x2f20f7f2),LL(0x56230526,0xd27bb44c),L_(0x000001e6), + LL(0x7e54e734,0x474f3cfa),LL(0x7075c330,0xb3ee576a),LL(0xb446bdd4,0x1b7efe90),LL(0xa143371a,0x826a3c98),LL(0x45fcf113,0xa412cdba),LL(0x4b5601ca,0x614348a4),LL(0x06ebe3f7,0x9f111b62),LL(0xd750e443,0x1f6bb4a9),L_(0x00000045), LL(0x5f2d48b9,0x63c0d59b),LL(0xcb29ae28,0xe216b29b),LL(0x3a9a3c78,0x2465c5b5),LL(0x9567856c,0x36d155fd),LL(0xdfbfe047,0xa1ce6b07),LL(0xc3a4fa43,0xf19b8361),LL(0x0e9f03c0,0xb21f548d),LL(0x9de803f9,0x85460ccb),L_(0x000000f8), + LL(0x1358b22f,0x28dc3d9e),LL(0xb1947d50,0x707f8201),LL(0xa08b9636,0x1277e4a9),LL(0x253ea9d3,0x38720146),LL(0x7e71d50f,0x62240fd9),LL(0x79e96c95,0x99a338fd),LL(0x14656a4b,0x3386ac0d),LL(0xaed2f11c,0x56e7d58c),L_(0x0000008f), LL(0xa0e5c79e,0xd77da8a6),LL(0xb38b4a7f,0xdb95d39e),LL(0xb6a3eb09,0xc15d78b6),LL(0x97f12510,0xc3aba47d),LL(0x56c5ceaa,0x1799be9b),LL(0x71c67e7c,0x8203dc96),LL(0xc27e5165,0xaf4cd822),LL(0x0bd2ed9a,0xb4f47f2a),L_(0x000000a9), + LL(0xbed671f0,0xd26b287c),LL(0x978130d6,0xd433a71c),LL(0x7aadd881,0xb4ad7bb1),LL(0x52e4f45f,0xb1940d6b),LL(0x722e2bd7,0x72d44569),LL(0xf91dc84e,0x6d3fde70),LL(0xd4254643,0xbd1bc41e),LL(0xb847e41a,0x544a7be2),L_(0x0000010f), LL(0x325818a6,0xe2e82545),LL(0x3d8e5d2b,0x1c9863cf),LL(0x5f30317d,0x98fb8ec2),LL(0x81f15ce0,0x8947db85),LL(0xf3e6c315,0x055d8793),LL(0x43a7feb5,0x8417f508),LL(0xac153d3d,0x248bbc38),LL(0x664329e7,0xfcfbcb03),L_(0x0000012f), + LL(0x8dd55839,0x0ffb3aeb),LL(0xf0f73b2b,0xb5b42e5a),LL(0xfb4aa83f,0x6fe3f768),LL(0x166957fb,0x1777755a),LL(0x9be2e29b,0xb21920be),LL(0xd39b15af,0x9d78ee52),LL(0x5cfe70fd,0x4ed3196b),LL(0x55c6cafd,0x34efaf79),L_(0x00000009), LL(0x60602ec9,0x9f0e1288),LL(0xc5b10817,0xef0c0487),LL(0x2d0a8114,0xd48fc96d),LL(0x7cfa2d81,0x641e3931),LL(0x6424b3ef,0xe6f897a1),LL(0x3828d18c,0xf36345b8),LL(0x162c7445,0xa9afd810),LL(0x86dd67b2,0x34f52c33),L_(0x0000011e), + LL(0x71f0b29b,0xbf4033d7),LL(0x282d7d2f,0x4816fc56),LL(0xbba4acf6,0x0d36fbc1),LL(0xed8c95f6,0xc3d935ee),LL(0xf5c04db6,0x0f8e468f),LL(0xf1e487eb,0xe95fc738),LL(0x07ce9d5f,0xd25bbce8),LL(0x3ea06aeb,0xd6e4d92f),L_(0x00000016), LL(0x8ee0b487,0xd1ebfa2e),LL(0x8c6285b5,0xff9b06cd),LL(0x7f8da651,0x19efacac),LL(0xfb702fe7,0xf38839db),LL(0xfd2c9b61,0xb8c0cb9c),LL(0x674adc03,0xd42ebdf8),LL(0x326b91f3,0x8993821a),LL(0x0fbb4e81,0xbc7b32f8),L_(0x0000012a), + LL(0x79b5c2f2,0x2fd676b6),LL(0x892e4204,0x41e60d2e),LL(0xdf88f03d,0x05344e65),LL(0x322deae7,0xabc0c748),LL(0xf3bee9da,0x7dbffa8f),LL(0xe2cbe049,0xf2922597),LL(0xb0e73ee1,0xa1ab5a4f),LL(0xd18b70e5,0xf7b0232d),L_(0x00000143), LL(0xbad81855,0x7a93eb9f),LL(0x998f9412,0x6520852d),LL(0x9fffa5de,0xa83ca014),LL(0x9492e738,0x29961bc9),LL(0x38356eab,0xbaa587e6),LL(0x9aad6aa8,0x5bde3fd3),LL(0x15f7c437,0xc663a8aa),LL(0x2b85ba68,0x922641a9),L_(0x00000017), + LL(0x872c802b,0x90b91e88),LL(0x9329a6f3,0x5e0fd985),LL(0x332091e8,0xc7233994),LL(0x069d0a1f,0x7172741e),LL(0x953488c0,0x1870fafc),LL(0x3b040fb9,0xbb2fd807),LL(0x9e841e1b,0x16872728),LL(0x48f58216,0xa72dc0f5),L_(0x000001bf), LL(0x4695d527,0x4ec2f404),LL(0xc898f3ae,0x893dfe9f),LL(0xd6d16346,0xcfc2a2d6),LL(0xdcfcc356,0x9780e14a),LL(0x52a0f58f,0xe40c34a9),LL(0x1f74017f,0x9637bf1f),LL(0xe85cc7e4,0x7db8273a),LL(0xf7240054,0xd4e119d7),L_(0x000000af), + LL(0xf3b66cc1,0x8bec2bfe),LL(0xe147d2d0,0x0f381def),LL(0x63d396f3,0xd719925a),LL(0xa70c4ee8,0x48335ffe),LL(0x70cbdf17,0xba86e989),LL(0x70acf1a0,0xa8e07900),LL(0x3fdcbfe3,0xf724a710),LL(0xd0dd93f1,0xcce8d0db),L_(0x000000dc), LL(0x2a8a4b44,0x312db409),LL(0xe842388e,0x51c17662),LL(0x5cfc4c86,0x2819cf9f),LL(0x3d2f2e78,0xcb5f9278),LL(0xc84b5c07,0x6807319b),LL(0x91966c94,0xd459389d),LL(0xf68df64d,0x22ed9565),LL(0x30d2c264,0x2d12cd4a),L_(0x0000015c), + LL(0x5d109ca1,0xa8198a4a),LL(0x7fd3cc79,0xcf8b16bd),LL(0x973e67ac,0x27c827e8),LL(0xc9207496,0x6d1709a1),LL(0x9157f587,0xc23a1cd6),LL(0xbedd9d1c,0x8b1088ef),LL(0xb005d24b,0xc08bed38),LL(0x6477a806,0x0681e2ab),L_(0x000001da), LL(0x34165bef,0x2de2bdb8),LL(0xe996e18b,0xf9e30d5c),LL(0x2c137e56,0x5c521393),LL(0x6041c771,0x8cdc666b),LL(0x4ee56b76,0x2695a1d6),LL(0x518a5638,0x41e2f039),LL(0xdb59944d,0xaf0a4010),LL(0x5b0e9f3f,0x829ed1fe),L_(0x0000001f), + LL(0xe83fbb80,0x3b382180),LL(0x93922bbc,0x1503eb8a),LL(0xd83a3aec,0x4f056b98),LL(0x0f661abc,0x1dc0c0de),LL(0x20cd5ab5,0x25a3a29e),LL(0x9409a1df,0x2ec3d724),LL(0x2311b333,0x9cff8f0d),LL(0xd299c4b1,0x30ec4ca0),L_(0x00000084), LL(0x05faf0db,0x674d075d),LL(0x79f5e3c1,0x3e4d547c),LL(0xaceff143,0xa985c66e),LL(0xceb1d3f0,0xc8735e90),LL(0x46311b8a,0xecdbba45),LL(0xd47b3dd0,0x8237affa),LL(0xc21e6384,0x6b4f82bd),LL(0x851d53b8,0xff59ca4e),L_(0x0000000e), + LL(0xfe08a2e5,0x29a5355a),LL(0xa2f29baf,0x3e11c687),LL(0x653058a2,0xab5abb63),LL(0x533110b2,0xead1d1b9),LL(0x254324e4,0x65d1b7b6),LL(0x9ad5a861,0xd1007405),LL(0x0712ab62,0x6f88f2a9),LL(0x78e9d501,0xefd62c6b),L_(0x000000fe), LL(0xe0d173ba,0x62d2d42c),LL(0x8d15289e,0x5d68b919),LL(0x4baf7b53,0x4a9af773),LL(0xbfd8566e,0x2c278158),LL(0x0f0f5b40,0xa603f631),LL(0x366d639e,0xed79331a),LL(0x457655be,0x5b5f4bc7),LL(0x744b4617,0x6ced0122),L_(0x0000008f), + LL(0x94abf4c2,0x0a6157b9),LL(0x62225ddd,0x63f4338a),LL(0x3ff075f1,0x19a505d3),LL(0xa170cf14,0xdfa6bdb1),LL(0x35bb45bd,0x3014f03b),LL(0x01eadcfb,0xdf884f38),LL(0xed9ce5a3,0xeb67d796),LL(0xbe4fe92b,0x01a5f419),L_(0x00000011), LL(0x1ff93f99,0x7ddc4629),LL(0x8bd2744c,0x40075673),LL(0xa1d1f6a9,0x6ac9b5bb),LL(0xbff10759,0x7709ddbb),LL(0xd617233f,0x04b71ff3),LL(0x91a5fcd2,0x45458de2),LL(0x699b54de,0x5147375c),LL(0x4e0307cc,0x2fe5917c),L_(0x000001bc), + LL(0xfc7b0e16,0x4252a836),LL(0xfbc6e97a,0xbffdbb20),LL(0xee422c57,0x5769cb47),LL(0x501f912b,0x3924d571),LL(0xae0c25a9,0x239c3442),LL(0x7ed84b21,0xd601103f),LL(0x74478136,0xebe1703d),LL(0xc5087e65,0x67b021e4),L_(0x00000156), LL(0xf2a22ea9,0x8ce094d8),LL(0xaa167fe1,0x70c73af6),LL(0x79b93fdc,0x0e41e095),LL(0x3eab3290,0x4d79fd87),LL(0xb7162ca3,0x66141590),LL(0x583d1391,0xe0bd38e8),LL(0xa393b806,0x169f55ab),LL(0x4bdc04a4,0xd1df6260),L_(0x00000012), + LL(0x7e5ec530,0x6ef19cc6),LL(0x8fb90054,0x349f0b5e),LL(0x7a588763,0xd381418c),LL(0xe35ce1af,0xec0fd49b),LL(0xecb4e203,0x7786513b),LL(0xa276ad19,0xfe701187),LL(0x99f4ee10,0x47d026fa),LL(0x58423dc0,0xa22bbaf1),L_(0x00000065), LL(0x5096d030,0x9aa1efcf),LL(0x5af392c9,0x8ba7989e),LL(0x7c5174d3,0xd1616f2e),LL(0xd20d0632,0xa5636d7c),LL(0xd761cf0d,0xce718261),LL(0x701e7d69,0x08d4d0bc),LL(0x66e92aa8,0x819aee8a),LL(0x0d6fcd90,0xc96d5138),L_(0x0000018d), + LL(0xb706d0d4,0x81b20efd),LL(0x117c40b0,0xec008c40),LL(0x0a6d9c2a,0x693270e3),LL(0x6114d3e0,0x4266a5ea),LL(0x44a6af67,0x81ebf621),LL(0xee3917e3,0x7fca3d45),LL(0xc35ff5d6,0xa3526048),LL(0x0f6e79db,0x7e7bfed4),L_(0x0000001a), LL(0xc6923808,0x0c06eb8d),LL(0x3343c5a2,0x3418cfe3),LL(0x3e67d0a5,0x15eb001a),LL(0x7e48959e,0x0ead5e7c),LL(0x0962e6ac,0x32e4162f),LL(0x3e28513c,0xfb8117bb),LL(0x317568fa,0x3a2e3034),LL(0x0c912ceb,0x55938174),L_(0x00000102), +}, +/* digit=71 base_pwr=2^355 */ +{ + LL(0x4b2fef73,0xfec70f6d),LL(0x92ef06ac,0xc38aac0c),LL(0x790f3344,0x53c30edf),LL(0x1b40fed7,0xfc800650),LL(0xf6da803d,0xfdf722f2),LL(0x284a42e2,0x5cecc340),LL(0xa0f15400,0xb36ac652),LL(0xef82f0df,0xd1506b21),L_(0x000001be), LL(0x59060616,0x84d76b78),LL(0xa1c3d7b8,0xc6050edc),LL(0x307e9a89,0x1519baa6),LL(0x8c7e0ccc,0x3495eff8),LL(0x22e91666,0xea17475b),LL(0x69639f1c,0x8c53c39e),LL(0x1f0e827f,0xede8121b),LL(0xeb066355,0x91249281),L_(0x0000015b), + LL(0x5a372292,0x3accccf3),LL(0x16751720,0xa2ac7465),LL(0x1bf938ea,0x83a983dd),LL(0xb73d73d6,0x598a6f1d),LL(0x630b4f7f,0x6235f9ed),LL(0xdb784cb5,0x52bdf332),LL(0xb330540f,0x221e5e83),LL(0xc25843b0,0x09499b4e),L_(0x0000004a), LL(0xb0a1b02a,0x010fb3ca),LL(0x8b6e52dc,0x60a24896),LL(0x22c046dc,0xae1e187a),LL(0x482695be,0x006acf49),LL(0x35934bd3,0xb960f105),LL(0x1e1d0143,0x4cfddf01),LL(0x5de371d8,0x56c439e8),LL(0x64828414,0x582ff3b5),L_(0x00000185), + LL(0x4315ba36,0x3578ec4a),LL(0x83ac7330,0x74417a7c),LL(0xb971e045,0xbee09246),LL(0xc391f3f4,0xb2cfe806),LL(0xf8fb9526,0xe574f401),LL(0x2d57d573,0x9ef64156),LL(0xf2047705,0x2e13ebeb),LL(0xe2e05adf,0x97b783be),L_(0x00000025), LL(0x81cc47cb,0xac801f25),LL(0xbb184270,0x1d0026d9),LL(0x772cb73c,0xc5ae7636),LL(0xf37dccd7,0x5106f975),LL(0x276aad31,0x2b41e6c0),LL(0x0c355836,0x7d420351),LL(0x169e0a27,0x25cdd7da),LL(0xe67ef6ec,0xdd1ddbf0),L_(0x000000fe), + LL(0xfbf17fb3,0xbce94ac6),LL(0x89803b61,0xe8afe442),LL(0x2e798f31,0x9a42f37b),LL(0x9475e43d,0x77aef7a7),LL(0xa8f685f3,0x203a6947),LL(0x969c3b8c,0xdf0dc1b4),LL(0x9c542cdb,0xc76bc8fb),LL(0xff501682,0x2a768660),L_(0x00000097), LL(0x5f3009bb,0xc8f9daec),LL(0xc4a46652,0x89b1c325),LL(0xb09499ac,0x721c0cae),LL(0x5e63ccd5,0xda46e344),LL(0xaca0b998,0x32db691c),LL(0x793a1fc7,0x049e845a),LL(0xd927f614,0x7aea310a),LL(0x80024bf0,0x5359be8b),L_(0x00000124), + LL(0x7a3137a8,0x57afd605),LL(0xb3d701e7,0x608eeeaf),LL(0xa1e2923b,0x3a52d1fe),LL(0xb5ba2517,0xf6c570be),LL(0x6f830092,0xa2b946db),LL(0x63e9fd66,0x2b96de58),LL(0xd1292d2e,0x89c80a56),LL(0xf418bcf0,0x02901646),L_(0x00000025), LL(0xbb16a820,0x9f28725f),LL(0xb36ad666,0x73de9bb2),LL(0x25e09cb1,0xf102152a),LL(0x3474fc24,0xb0389a9f),LL(0x9e0b3083,0x245ecf47),LL(0xefc86ff8,0xedc1e824),LL(0xd024fd72,0x022e7528),LL(0x0e37477f,0xa7fdb14f),L_(0x00000191), + LL(0x6cc9e802,0x130db159),LL(0x31d0f49f,0xbd9c79a2),LL(0xf1d499d6,0x7ea9b7cf),LL(0x367757ac,0x05a4d545),LL(0x0a42a453,0x1f2c8548),LL(0xd461a5b5,0xb8ed29ef),LL(0xe691c9e6,0xa549541f),LL(0x6490ea1c,0x09c0153e),L_(0x0000018f), LL(0x3f9a0edf,0x7f525f59),LL(0xc140a1f6,0xa98aaedb),LL(0xf5bef166,0x750be5c2),LL(0xd457a559,0x8cba58b2),LL(0xc5d96cfb,0x514d93d0),LL(0x70bfa2f9,0xb86d7234),LL(0xf6b79058,0xa8a78584),LL(0x18d58f11,0x2b2d0ad4),L_(0x000001e3), + LL(0x55bea4be,0x147202dd),LL(0x3c8a33ae,0xd39c4b9d),LL(0x1d455649,0x3f56141d),LL(0xc0ac51ab,0xa5a57669),LL(0xfc529732,0x1ee307ea),LL(0x1861bed9,0x5f1e4bec),LL(0xc7a796b8,0x5b0d925f),LL(0x06f456f4,0x2257c76d),L_(0x00000117), LL(0xfe075b8e,0x4360118e),LL(0x218cb0cb,0xaa1c1dca),LL(0xf6db0e7a,0x2be1c968),LL(0x5a3744ed,0x46bb2acc),LL(0xaf262fce,0x7e16d340),LL(0xaec37ad7,0xe6df41bd),LL(0x1060715f,0xee38cd22),LL(0x87b94898,0x34109b20),L_(0x000000a2), + LL(0xa0d42ca1,0xb416f6d1),LL(0xc2c062af,0x3dacb0a7),LL(0x1630676c,0xd74ee6b3),LL(0x95fc297a,0x8f736e49),LL(0x48a7a2d1,0xd64edc25),LL(0x6b5d5f53,0x83303159),LL(0x0e945b2c,0xdfaa52c4),LL(0xb0587c06,0x462a8f05),L_(0x0000017c), LL(0x4f0d870e,0x2fcd636b),LL(0x0835ddc0,0x86482b2b),LL(0xd2333470,0xc7f1c7b2),LL(0xb32bf92b,0xd5f30c92),LL(0x49153950,0x9ce136c0),LL(0x29288cec,0x4ac8254d),LL(0x34eb3849,0x0b3117ac),LL(0x14ba2a1b,0x3a85376a),L_(0x00000047), + LL(0x9b2fca5a,0xb3ec3510),LL(0xc014692a,0x988a8cb2),LL(0x70309a41,0xb83a1155),LL(0x92367194,0x22c65f09),LL(0x558d49bd,0x17ac8e14),LL(0xd539b194,0x281a7ecc),LL(0xa19213fa,0xc69fe80b),LL(0x1ef427d6,0x744a4f4d),L_(0x000001db), LL(0x64c7d0d7,0xf07169a2),LL(0x4ff0070e,0xe17c5e0d),LL(0x42362609,0xd97efa2b),LL(0xab4374f3,0x59d17f1f),LL(0xc3027acb,0x8cb9e348),LL(0xc305c872,0x320eb648),LL(0x861bfbe3,0xf68b129b),LL(0x2a98fbd9,0xc35aa741),L_(0x0000004d), + LL(0x9b675376,0xba2a6162),LL(0xfef26d3a,0xaf22bedc),LL(0x4bac42f2,0x32bd0514),LL(0xaf54da8b,0x474d59e6),LL(0x846ca3be,0xac190f17),LL(0x7e7c79bf,0xaea0f3e1),LL(0x13543ecb,0x0ff996ba),LL(0xabe74acd,0xe27a5f5a),L_(0x000000bd), LL(0xfeccff02,0xa8ccc73f),LL(0x1e746179,0x62af882b),LL(0xb19b717d,0x4e0895be),LL(0xb255045a,0x8f194a8b),LL(0x0b37366b,0x089f1cd5),LL(0x7b3da3e1,0x68b1f2a5),LL(0x91e4f674,0x9c4602f6),LL(0x698976ca,0x3ed98ad9),L_(0x000001e5), + LL(0x2100e990,0xfe476990),LL(0x532b6850,0x345630b7),LL(0x7ff00f29,0xecbc85e6),LL(0x9c3d2e4c,0x5f322ba8),LL(0x6869142c,0x9e246c23),LL(0xf5d1ef76,0x0affd2a0),LL(0x2e6d871e,0xb40893c1),LL(0x66c72704,0x1665fed8),L_(0x000001d9), LL(0x91fa7d18,0x06507414),LL(0x9d9489fd,0xa1379411),LL(0x7d5c53f0,0x7e6d3de2),LL(0x097595be,0xc486fb28),LL(0x85e5a09c,0xa9fd8f9e),LL(0x2996fa66,0xd71e16d0),LL(0x040664e3,0x7c75e965),LL(0x14b60428,0x9d686380),L_(0x00000032), + LL(0x23ecdd94,0x86f79bf5),LL(0xbf36d744,0xbf45f97e),LL(0x0fe48147,0x6235b3ae),LL(0xd40868d4,0xa9b13d93),LL(0x45fa9173,0x34e9264c),LL(0xb5705f4c,0xd0d58c79),LL(0xfd4b166f,0xaf4ff870),LL(0x7aaca2ed,0xb68a488f),L_(0x00000155), LL(0x9951fb26,0xf5f3e0b1),LL(0x747dd972,0xf6fc0c26),LL(0x92d84bc8,0x088102b5),LL(0x201255f7,0x84970893),LL(0x7f6288c9,0x6a679170),LL(0x9309b54e,0xd5327276),LL(0x389f4da5,0xc48b5de6),LL(0x8fc1eb23,0xac794b85),L_(0x0000011b), + LL(0xd3fa1c16,0xb408df01),LL(0x66056a5d,0x5f945157),LL(0xc30a058e,0xf0071848),LL(0xb615e360,0xe6a8d838),LL(0x132d49ae,0x1bb7f3ca),LL(0x092ee873,0x398ab7d7),LL(0xbf83bd98,0xba362639),LL(0x73208c1b,0x49dd5ba3),L_(0x000000cd), LL(0x081c78e2,0x0d07ce4b),LL(0x724cd057,0xf7baff54),LL(0x31245cf8,0xff518822),LL(0xff5f1211,0xea1813a7),LL(0x89f90332,0xfa74413f),LL(0x9e68455b,0x9e49a7a6),LL(0x182fad31,0x30e8a2ef),LL(0x233ce0d5,0xbd55ab52),L_(0x00000199), + LL(0xd2d5a87a,0x28e2d8df),LL(0x708c9183,0x1dbe8e5a),LL(0x9b9fb00f,0xa3695cb8),LL(0xaeafe9c7,0x9205b4ca),LL(0xd6ec0b74,0x756f204b),LL(0xa9e0254c,0xd51d1a73),LL(0x152441cf,0x0ca91564),LL(0x370d2b8b,0xe3cdd9e9),L_(0x000000de), LL(0x4dec1468,0x9dab1375),LL(0x5322d78e,0x78b8ab5d),LL(0x0adbfc55,0xa2b97f9c),LL(0xd5d0ce27,0x9cd573f2),LL(0xe94e39b4,0x06ee23d2),LL(0x213bd15a,0x708d61ea),LL(0x561b9d34,0x6271f59e),LL(0x41fb576c,0x9ae94507),L_(0x00000166), + LL(0x312474f6,0x6669c984),LL(0x46ae7251,0x8c23b4d3),LL(0x3738a807,0xad75f8b4),LL(0x3ef8e3cc,0xc8e8bbf5),LL(0x029e586a,0xa7111c4e),LL(0xa4d1beb9,0x238f36b4),LL(0xc4da1680,0xbe34bad6),LL(0x9409b124,0x6ab824c2),L_(0x00000002), LL(0x3b1b0723,0x0666df3f),LL(0xa0015f27,0xd7f90fcc),LL(0x3e7eb3a2,0xcd91c4ca),LL(0xa8bd0fd8,0x56907857),LL(0xc0b60059,0x3559db95),LL(0xc6ed4fe7,0x0c8beff5),LL(0x2ba51007,0x5bc2116c),LL(0x409f6b4e,0xa6198a28),L_(0x000000d8), + LL(0x6c54f37b,0x231a6637),LL(0xb888102e,0xaecdfa06),LL(0x430b0efa,0xd888793c),LL(0xb123b1e3,0x8beed2db),LL(0x8887df4f,0x3ea5e72a),LL(0x2425e985,0xccc7d4aa),LL(0xd98e93f3,0xe9181719),LL(0x07cba97f,0x8ea6eef3),L_(0x000000b0), LL(0xf51c3444,0x5719b171),LL(0x5d9f40be,0xe16ecd5e),LL(0xfea96313,0xfe1e359f),LL(0xd9f1461e,0xde9904f3),LL(0x38f6d943,0x881bb7a0),LL(0x2c5787d5,0xa74eed55),LL(0x67fc2cd9,0x7ccb483a),LL(0x2c643f37,0x7070b576),L_(0x000001db), +}, +/* digit=72 base_pwr=2^360 */ +{ + LL(0xb53942b1,0xca188556),LL(0xbd7c7672,0x20ced736),LL(0xa4667058,0xb83d6897),LL(0xe3f39ba4,0x174ecbf7),LL(0x34188faf,0x203dc58b),LL(0xdb5dba0b,0x5206b453),LL(0xf54df32d,0x52fcf51e),LL(0x08d08e3c,0x2f551f34),L_(0x00000173), LL(0x92603b2e,0x6b937ade),LL(0x7f7f5dfd,0x632c9b6a),LL(0x3151876b,0x3ee4a789),LL(0x7a59040d,0x41b009fd),LL(0xdfc2d274,0x68b427fe),LL(0x1c0ceded,0x07d57f92),LL(0x220fc8f2,0x83c79a42),LL(0x7b0f6753,0x410a2e83),L_(0x0000016f), + LL(0x2ff9c0f0,0x01fb8b79),LL(0x82addc43,0xfdb0062d),LL(0xb9cdf1d9,0xb1cf25ad),LL(0xb5a42255,0xaaea42eb),LL(0x1990669d,0x1dffd105),LL(0x88f20764,0x613001d6),LL(0xda7769bd,0xa275aa11),LL(0xe04ea507,0xea612e43),L_(0x00000073), LL(0x24386fbb,0x21f18b4b),LL(0x268a5e08,0xa7554f72),LL(0xad126436,0x714fe1c3),LL(0x5b5ba02f,0xb7c7cde4),LL(0xf2da3519,0xfc576f09),LL(0x4fb328e0,0x185faef3),LL(0xa0386e0f,0x2adc73d1),LL(0xc97a6bb3,0xda21be9a),L_(0x00000133), + LL(0xd542b802,0x6a23d540),LL(0x500040b2,0xf87554cc),LL(0xe6a09fa7,0x548aea96),LL(0x0cf27fbb,0xfa1d8c06),LL(0x1a618765,0x1943cfee),LL(0xe6a8c7ea,0x20bf61bc),LL(0x99730b0b,0x744528dd),LL(0x42eac170,0xd049742c),L_(0x00000023), LL(0x6da345a9,0x1e2bba63),LL(0xe601cd80,0xcbeefa62),LL(0xc9e240a6,0x8106469f),LL(0xda8103af,0xc7109e54),LL(0xdcc44907,0xdb9a3ec3),LL(0xe44b6df8,0xe34b7788),LL(0x0e67c93e,0x4a58495d),LL(0x63e8347b,0x23b5096e),L_(0x00000172), + LL(0x5b970b5a,0xbf417e03),LL(0xb60364c1,0xdda37ca1),LL(0xea847f52,0x28527f5c),LL(0x98d517fb,0xa1e399f7),LL(0xff102f07,0xc452c79f),LL(0x87dfab3c,0xc5aa688a),LL(0x490b0295,0x0dbc6056),LL(0x3dd17acd,0xb4f6972c),L_(0x000001ac), LL(0x90426354,0x23fa3555),LL(0x3a903ffa,0x262814e3),LL(0xc46f6e35,0x1cec4214),LL(0x7267bc6b,0xca2e1dc8),LL(0x20b7474b,0xeb500457),LL(0xf394811d,0x4304c697),LL(0xa5001f3d,0x0f7a5e2b),LL(0x4c9ea7fd,0xead3d012),L_(0x0000010d), + LL(0xd19de00c,0xb7e47c23),LL(0xd475bc3c,0xff4599af),LL(0x1acc6490,0x39b1950f),LL(0xee09f5dc,0xd14540f1),LL(0xe51c9564,0xf1b75050),LL(0xbebd088f,0x17895647),LL(0x240dba4c,0x097400cf),LL(0x559b95e8,0x5d4b8420),L_(0x0000005b), LL(0xd06fbfb7,0x85986a76),LL(0xc2ffb653,0xa478ed7f),LL(0x18e264c5,0x4841d184),LL(0x17a5a278,0x21d9e8a0),LL(0x297fe2fe,0x1bf52154),LL(0x072d6d91,0x88327dad),LL(0xae77c8ea,0x02d1fcbe),LL(0x13786b6a,0x25554500),L_(0x00000068), + LL(0x9f48a0af,0x17de731f),LL(0x7753cff6,0x972b7a35),LL(0x73655403,0x28d73a10),LL(0x0c70484d,0x46d46c14),LL(0x516a9dc8,0x455b7ef1),LL(0x0b552594,0xb8161489),LL(0xf418ade1,0x64a91645),LL(0x76a465f2,0x7693e9a1),L_(0x000000d3), LL(0xdcbca2f5,0xe5e5c3bf),LL(0xdc135bc4,0xca946121),LL(0x3d39b5c7,0x55877498),LL(0x318be468,0x9fb5d801),LL(0xe5cb6287,0x3afd92b1),LL(0xcd7f8034,0x34ed24ae),LL(0xa3835c84,0x6aa7d954),LL(0xd325764c,0x41780668),L_(0x00000002), + LL(0xf2280c1c,0x34c0928c),LL(0x37933b17,0x74a56f2c),LL(0x6bae2a29,0xb1d26ac8),LL(0x6bd1e8bd,0x84c336cb),LL(0xcdaa1b9c,0xaca41014),LL(0x7838c44f,0xe2ce24b8),LL(0x525239ca,0x515f204f),LL(0x8bcb0507,0xdbd0e0a5),L_(0x00000193), LL(0xc3926ced,0x631411bd),LL(0x3e15aa53,0x8672c87f),LL(0xade47bf6,0x93da50d5),LL(0x148028e4,0x0048f8cd),LL(0x03c75612,0xb5ecfaeb),LL(0x7b7867aa,0xafcde134),LL(0xa0208953,0xe2411e3b),LL(0x24be9b23,0x848d40b4),L_(0x0000002e), + LL(0x8d4ad28a,0x16583ec0),LL(0x7b7ba7d9,0x2bbb4768),LL(0x2b3f0b4e,0xe0e4b3fb),LL(0x9172caac,0xe6fb63a6),LL(0x22aab4b0,0xa00520c8),LL(0x7930e37a,0x4dcbf41f),LL(0x6bfa91da,0xf521a694),LL(0xb88bd604,0xa707c1f0),L_(0x0000000f), LL(0x0d23952f,0xd8520b88),LL(0x22333018,0xa6bc2bb8),LL(0xaa6a00bc,0x011553af),LL(0x5def3469,0x0ed5fc0a),LL(0xbcfec7c2,0xdee0e8c5),LL(0x2f464224,0x8adb476e),LL(0xd844542e,0xd3c1bdb2),LL(0xa709924f,0xac98d161),L_(0x0000012f), + LL(0x715ddc65,0x5054b047),LL(0xa4a3faff,0xd966b478),LL(0x3d33573c,0x1081c0ed),LL(0x928c644f,0x0d2ce409),LL(0xb6d01835,0x9e6a2193),LL(0x52176b02,0xa876fcb1),LL(0xa48ba61b,0x717040ec),LL(0xa24784a2,0x063597d5),L_(0x0000008d), LL(0x2c7389fc,0x24d496ff),LL(0xb9c0170d,0xf08c120b),LL(0xee06f00f,0x24a5f375),LL(0x9e3d247d,0x21f556fe),LL(0x23ca02e0,0x9baf2fa5),LL(0xc33aa42b,0x87dfcff1),LL(0x165eef36,0xf9dd5e4c),LL(0x54afa097,0xfe4014f0),L_(0x0000009f), + LL(0x52348ca8,0x690b2052),LL(0x836e06e2,0x3509fc01),LL(0x7b24d732,0xc3a11a6f),LL(0xf970953d,0xb1e661c4),LL(0x19774ec5,0x1a995696),LL(0xa05e5145,0x777347df),LL(0x1c3c2550,0x77882ae3),LL(0x5a7928a8,0x77fbd07d),L_(0x00000185), LL(0x09fd7d9d,0x41e2d40c),LL(0x16368545,0x29dc25b0),LL(0xfa97fb70,0xb7b7f0ff),LL(0xdec5a377,0x2c841e96),LL(0x205a5df4,0x01390c3a),LL(0xe68d053a,0xa1116c1b),LL(0x83274721,0x62015852),LL(0xe92e4364,0x6a3178e8),L_(0x00000016), + LL(0x84a707a5,0xf2f16fe9),LL(0x75049d40,0x2503f2a9),LL(0x67afd1db,0xe86bcd13),LL(0x1004d640,0x5322f07a),LL(0xf5a688a9,0xb4accb02),LL(0x5af14887,0xe07764b0),LL(0xd65fb0f3,0x03224e24),LL(0x5db913de,0xb8433f0c),L_(0x00000011), LL(0x1ac50b18,0xf3884513),LL(0xe8ba41a9,0xaded3528),LL(0x8ea44980,0x1a82f302),LL(0x31375544,0x0614f686),LL(0x4f3dc64f,0x8ad34274),LL(0x7f906c7a,0xc479a89a),LL(0xe50987f1,0x17709633),LL(0xaa307609,0x33922a61),L_(0x0000009d), + LL(0xbf75fd24,0x7b97be7b),LL(0xd004b0c4,0xd609efba),LL(0x341b0b8f,0x20475d65),LL(0x373000c9,0x00101859),LL(0x5bcb5ae6,0x518d7514),LL(0xfb3c86f2,0xf5d314f2),LL(0x2c8aceec,0x2307be1e),LL(0x778a0fa6,0x5d168daa),L_(0x0000011d), LL(0xd7b35cc7,0xa440b9a3),LL(0xa70ba700,0xa47a36dd),LL(0xef6fc566,0x17be0829),LL(0x8e295843,0x63809dc9),LL(0xa317bc2f,0x1a787c34),LL(0xbce26a9e,0x943ad796),LL(0xc651a487,0xe010f911),LL(0x636a6efc,0xcd31e255),L_(0x00000188), + LL(0xe2c5ac41,0xed5ce1f5),LL(0x8d6c9e3b,0x8f008e9a),LL(0x2a13d48e,0xa44f1754),LL(0x32cde8f7,0xaa90e24c),LL(0x8660b8f9,0x3052b86b),LL(0xd73a03c1,0x79244ef9),LL(0xe4f8a628,0xc2475432),LL(0x35d52164,0x9c11d0d4),L_(0x00000059), LL(0x3d8335f4,0xb6ab40a4),LL(0x3d6aa080,0x9ca82551),LL(0x92a4dc90,0x8044f304),LL(0x5989211e,0x4878d275),LL(0xc33afe23,0x50bb5ea4),LL(0x2b031b41,0x889e9545),LL(0x4a4f6a74,0xdcce463e),LL(0x6a9c23ca,0x260ab0c8),L_(0x0000007c), + LL(0xe02b88ed,0x2372213b),LL(0x84ab40ec,0xdaad8de9),LL(0xcb476943,0x2dbf8cf6),LL(0x9149cbb6,0x72626b77),LL(0x4935f2cb,0xae5d765c),LL(0x79eb7a36,0x65f4be84),LL(0x44c54fc0,0x2049ba34),LL(0xcbfa4bf0,0x9c904608),L_(0x00000083), LL(0x3dbc5a64,0xa5d29aec),LL(0x3010853b,0xdf573f6e),LL(0xcdc13fa1,0x9737e298),LL(0x24add1ef,0xa7f64bf7),LL(0x2431c698,0xb2b280e2),LL(0xbdff9a1e,0x93c22a36),LL(0xd70876a8,0x0c7227cd),LL(0xaf483376,0x04873b2d),L_(0x0000009e), + LL(0x86436bf8,0xecb1f89d),LL(0x21b6a936,0xf18c9695),LL(0x805badb2,0x45242b9a),LL(0x2c7430dc,0x7856a265),LL(0x5aacc16c,0x281a24a8),LL(0xbe56330d,0xd8f8608d),LL(0x50d0b225,0x62852160),LL(0x4403a1c3,0x7038362e),L_(0x00000075), LL(0x2d9d6d70,0x7dc02e97),LL(0x816727a7,0xca536177),LL(0xc5451dd1,0x304d66d8),LL(0xb89ef533,0x360da6a3),LL(0xe6b58c96,0x3f234bec),LL(0xc18619f4,0xb4bfa580),LL(0x891d516d,0x40c1bed3),LL(0x04f1453b,0x7060a227),L_(0x000001e8), + LL(0x367407cb,0x30c9a655),LL(0xcf48b04d,0xb68ea01a),LL(0x04344830,0x174d6fa7),LL(0x1ac58a53,0x9044eeb3),LL(0x0524d6f5,0xf87d51a6),LL(0xfb882d4d,0xaa0ba344),LL(0x1ed41d08,0xa85fb93d),LL(0x6086b6ae,0x7fa57f48),L_(0x000001f2), LL(0xb7febd79,0x596f6fa7),LL(0x2aae9562,0x04813ef9),LL(0xabc183c4,0xded30d2a),LL(0x6a011be4,0x20b7ae96),LL(0x88e77be2,0xbc3e6cfc),LL(0x77d5e0ca,0xac06a92b),LL(0xd7f99c6d,0xc76c3023),LL(0xa80a6be4,0x1d55150d),L_(0x0000012d), +}, +/* digit=73 base_pwr=2^365 */ +{ + LL(0x300cf42b,0xea0d37f0),LL(0xd1a1ebfa,0xb07ea564),LL(0xbce4cf04,0x4677d784),LL(0x74184f2b,0x14a4f867),LL(0x3b8741db,0x00b95ce9),LL(0x35b5960b,0xfae4a317),LL(0xd2c80a76,0x23107ec3),LL(0x0522c4d1,0x8678a9d7),L_(0x000000cd), LL(0xaec13d90,0x304b58dc),LL(0x88c3d5f2,0x45f5267d),LL(0xf847248f,0x28ef4e85),LL(0xc0f3da26,0x7945a7b9),LL(0xea2c17e3,0x8a2da387),LL(0xe84de988,0x211a8e98),LL(0x8290c88f,0x75574343),LL(0x6b4ce366,0xca4612f5),L_(0x00000040), + LL(0x7975b736,0x6c7a73bd),LL(0x1e3ef4b5,0x104fc4ba),LL(0x835871e0,0x4759b57a),LL(0x9b4ed462,0xd3c95d4d),LL(0x648a71ee,0x829d8353),LL(0xece81ad2,0xb2a56bed),LL(0x452c12f2,0xb67ec3e1),LL(0x35ab19b8,0x3f8f88bf),L_(0x000000cd), LL(0x5c7f0b66,0x76062e0d),LL(0x4abff696,0x6641cdd3),LL(0x9b89962a,0x0add12e1),LL(0x9f42be1c,0xa078191a),LL(0x72d9da14,0x7c488cf9),LL(0x607f65fc,0xa7e790e9),LL(0x5cdadd7d,0x83b3584d),LL(0x381ca37f,0x3c6df02d),L_(0x000001d4), + LL(0xded49058,0x3a1e6367),LL(0xfd8b4117,0xcdcf0fd6),LL(0x0c717ef3,0xb3300d01),LL(0x07e608fd,0x527d7c8e),LL(0xe69c0a0d,0xd39cd9ec),LL(0x11bdaf48,0xa5576772),LL(0x5d520c7f,0x92f3c61b),LL(0x1fbf8426,0x814bffe3),L_(0x00000055), LL(0xa0e0f49e,0x3dc94502),LL(0x8193bb95,0xb1a23052),LL(0xd7bdda5a,0x19650b25),LL(0x4c67c4a2,0x78abb7ba),LL(0xb9dbe10f,0x3eb157bd),LL(0xb3d0ff94,0x1a32ace6),LL(0x8180c4dc,0x9e9b36a4),LL(0xeb0124b6,0xfee72796),L_(0x000001a7), + LL(0x7bd3b0c9,0xedaed08c),LL(0x46195fa3,0xbcb9e4e5),LL(0xc31c13ef,0xb2cc8a6a),LL(0x483f2eae,0x912ca200),LL(0x5ee60fa1,0xff0ff27a),LL(0x9e9c56cf,0xdac70a7b),LL(0x4977503c,0xbda5a3e4),LL(0xa564deab,0xfe3a9fcb),L_(0x0000018e), LL(0x784eeb75,0x7e821113),LL(0x560a5e57,0x4ec38a12),LL(0xae4b9aaf,0x8358d926),LL(0x1cb5c9a3,0x7b69c24b),LL(0x0e546449,0xc0748541),LL(0x660a2d50,0xa4c426fb),LL(0x87263ee5,0xc286e0b9),LL(0xf7b3ba20,0x4bed6c50),L_(0x000000e5), + LL(0x1f5a397a,0x9263865c),LL(0x4434e163,0xf3a80e48),LL(0x9ca69373,0x9608a668),LL(0xc4b09404,0xb3964738),LL(0xbe4c6ca8,0xe169bce3),LL(0x7c62a7e9,0x4efa6e4a),LL(0xb46b0f85,0xe2e5aeb2),LL(0xe0111694,0x6babf49a),L_(0x000001f5), LL(0xbb845644,0x2589af29),LL(0x286fb826,0xc48651eb),LL(0x97cf3fde,0x6f9d0884),LL(0x78bdc9d0,0x16ca5665),LL(0x7e3d1e7d,0x5a2f1773),LL(0xc2f14e0d,0x5e7f3258),LL(0x6606eb12,0x77a28311),LL(0x67442ff0,0xc80cdb6d),L_(0x0000010b), + LL(0x009d2841,0x6f4c68a5),LL(0xfaab85ab,0x18a863e0),LL(0x8722f321,0x96627616),LL(0xe3cfe440,0x1b986ba1),LL(0xc819b534,0x03e0ab51),LL(0x7155ab76,0x9cf682fc),LL(0xc9e37547,0x4f4f98e5),LL(0x137f31be,0x3cda736a),L_(0x00000140), LL(0x7723495a,0x91d6868c),LL(0x2e86f052,0x70f48703),LL(0x14f3d533,0x6c353990),LL(0x099af7f9,0x0b2a6c71),LL(0x3d4612fb,0x9b98b62b),LL(0x406a6c59,0xe6249353),LL(0x3b1b8dd5,0xc92d7a1b),LL(0x1f751af1,0xf81ef140),L_(0x000000e0), + LL(0x3770cf24,0x2c9d1106),LL(0xdc34aa30,0xa7bfe1cd),LL(0x28ee1801,0xd18bd43a),LL(0x6d6e3e49,0x52b35eba),LL(0x25fc7059,0x7bfa8888),LL(0xe767a889,0x487fe05c),LL(0xbcac8ca5,0x09996d23),LL(0x52fe2328,0xe81bfb43),L_(0x0000010d), LL(0x0fdc93f1,0x121f3bab),LL(0xa0be654b,0xd4307448),LL(0x230e8622,0xa9492744),LL(0xcf01fc0b,0xe7b1abae),LL(0xcc98dab2,0x4d9cffdc),LL(0xf6504381,0xe9697cb5),LL(0x506e8f37,0xc3dfae33),LL(0xc253ed02,0x141d1dda),L_(0x0000011d), + LL(0xbe8f2ee3,0xd4e2418e),LL(0x60bac026,0xe3153f45),LL(0x08c6a85e,0x8e05a0fb),LL(0xabec7d7d,0x5867d053),LL(0xca6918d3,0x1ebaaa06),LL(0x7d8627f0,0x6ea92220),LL(0xdfe74b9c,0x27dc332f),LL(0x54478deb,0x33ddba7b),L_(0x000000b6), LL(0xda8ae443,0x2feb3b84),LL(0xd254321c,0xbd92cdce),LL(0xae0be12c,0x91edd7e2),LL(0x65ab5fae,0xacef4485),LL(0x607c22ad,0xe3f288c1),LL(0x01e22b70,0xa3baea8d),LL(0xe3598c73,0xc24e3c94),LL(0xf89cd9f6,0x595791d3),L_(0x000001a5), + LL(0x9ea2f400,0xa048c822),LL(0xbacfe4cd,0xc1680ce0),LL(0xd2bf80ae,0x4efcb3dd),LL(0x3024028c,0x029e5c0d),LL(0x63d006d1,0x5acb256c),LL(0x1d3229cd,0xee644462),LL(0x6e4f2a9d,0x4aa18f75),LL(0xd4fdba43,0x437e2a93),L_(0x000000b4), LL(0xffc70ce4,0x7de729a4),LL(0xcdd3e499,0xf4a84a14),LL(0x9fba5f0d,0x87d56bc9),LL(0xa92225d1,0x637de402),LL(0xa9f81afe,0x142558b2),LL(0x2061f42e,0xd09b2789),LL(0x1e15d846,0x8753411a),LL(0xf0c0c378,0x83c3fa31),L_(0x0000012c), + LL(0x2af88f08,0xda11421c),LL(0x8eaf7d82,0x6c3eef0e),LL(0x9224cdcb,0xd8359bd5),LL(0x34b2e501,0x734d08d8),LL(0xd9f7f27c,0x60136559),LL(0xa91fd047,0x7da1c7f8),LL(0xcb1bc103,0xf3f7e7c7),LL(0x65c241e1,0x7555ce39),L_(0x000000f9), LL(0x324c47c2,0x6b91fc57),LL(0x5484eacb,0x8a825cda),LL(0x888b470b,0xf0a2ebdb),LL(0x81202cfc,0x3ac37a5b),LL(0x05c01038,0xfe8f11c9),LL(0x5bf196e7,0x76123e92),LL(0x19cd94ed,0x353febf7),LL(0x9c972db2,0x65d70280),L_(0x0000019d), + LL(0xf016362a,0x217d03c5),LL(0x56db67a5,0xf3d76a7f),LL(0x992b8bdf,0x4fb50dfa),LL(0xf9702a82,0x5593b5d5),LL(0x9088a3ea,0x0ef00d4d),LL(0xbf26b47f,0xeb497149),LL(0x0793417a,0x58262023),LL(0x465f75db,0x4abc908a),L_(0x0000009e), LL(0xfa362b63,0x1667b32f),LL(0x4ef57c25,0x0a4bdcb0),LL(0x6e8c095e,0x696391d7),LL(0xabbc605b,0x1dd44220),LL(0xc3e47f6e,0x38adb47e),LL(0x8f35f645,0xd3c084d2),LL(0xf253b25e,0xfaa3b241),LL(0x0b53ca2e,0x2e3d4cbe),L_(0x00000189), + LL(0x3b140c85,0xf26ba24d),LL(0x0d68d639,0xbc6bcbf3),LL(0x76bdcf5c,0xd445a425),LL(0x4ce8f583,0x036223e7),LL(0x12c0333e,0x6b6c4cf4),LL(0x411cd547,0xfa6d4a89),LL(0x6d3fb3c8,0x7e41166b),LL(0x906f6895,0xd5a83001),L_(0x0000006a), LL(0x04968a38,0x33fa874d),LL(0xdb9b9bf4,0x6efa8bd8),LL(0xc1467c44,0x78067572),LL(0x41957d49,0x68a286e7),LL(0x4563827a,0x3562fe94),LL(0xd87962e4,0x5b2ba1f2),LL(0xbce9e3b5,0xeb40dfc9),LL(0xedcbc4f7,0x6ddd5a2c),L_(0x0000004c), + LL(0xc41935fa,0x70bab965),LL(0x738ba18f,0x90ebea6a),LL(0x3526ec84,0x544312d2),LL(0x606e765e,0x4f7ce18b),LL(0xe015cac8,0x0b6de72b),LL(0xea01d5b4,0x2ff4bd72),LL(0x0c7eba91,0x90d594c1),LL(0x8a32c97a,0x881a1a5b),L_(0x000001cb), LL(0x2511b170,0x8272c1c5),LL(0x72a5ff2c,0x0ff0a2ad),LL(0x4bb6c7a4,0x640ae3bc),LL(0x87804672,0x00da0040),LL(0x94aaf22e,0x2ebbcebb),LL(0xd8f3e9bf,0x8646dab7),LL(0x40c90d99,0xd08b3434),LL(0x8f9d970b,0xd1f0de73),L_(0x000001e3), + LL(0x7fee2d24,0x893123ae),LL(0xc8b47342,0xedaef283),LL(0xde3df2dd,0x303757c2),LL(0xc96f2592,0x089845fe),LL(0xa0d3f290,0x18b0b508),LL(0x4bf7b214,0x93c88975),LL(0x3cd67758,0xed218a4a),LL(0x81a61b63,0xfa78dcc6),L_(0x000001e5), LL(0xb7b5abd8,0x90ce8d10),LL(0x6c24429c,0xe40a36fc),LL(0xc3ad8e81,0x6178bcf5),LL(0x16d9b177,0x488d2cf8),LL(0xc063b1e0,0x57f41dcf),LL(0x1cbca7c3,0x200bb41d),LL(0xec7a80d6,0xd6366c22),LL(0x84ceffb4,0x66439f2f),L_(0x00000092), + LL(0x487aacbc,0x869f06cd),LL(0x97f9882a,0xaa57537b),LL(0xe3b1c07d,0x917cf4ff),LL(0xf6d1aa12,0x2683a59f),LL(0x73bca1f9,0x613d785e),LL(0x496708d8,0x66ede999),LL(0xf45fff24,0x4e9727c1),LL(0xf71c4572,0x39995099),L_(0x000000fb), LL(0xf3181f1f,0x60222373),LL(0xf249fc50,0xdb62572f),LL(0x95b4f7eb,0x4efd7ca7),LL(0xb4994b20,0x0762c1c5),LL(0x99292d14,0x4d4c1985),LL(0x140a608e,0x7f0ba7f9),LL(0x489023fc,0x77e472d8),LL(0xac039583,0xca8aeb86),L_(0x00000176), + LL(0x3a9026ef,0xa1cd049d),LL(0x859af0b3,0x32b70dbe),LL(0xd9aa6b96,0x83656cba),LL(0xa5229dc4,0x02bc7ba1),LL(0x574b487d,0xffc68a06),LL(0x9518ff35,0xad36470a),LL(0xaf20c720,0xcf8b908a),LL(0xee3bb49e,0xf8b9d88a),L_(0x00000166), LL(0xaca41ffd,0xc04ae92a),LL(0xff799aa5,0x352ca9e2),LL(0x48de6d0a,0xeb0f3051),LL(0xe2b8f5f2,0xa98f1062),LL(0xdfe726fe,0xc285eca4),LL(0x22419400,0x527244d3),LL(0x441ba1f9,0x3ec0c841),LL(0x9ac0f611,0xf7b09376),L_(0x00000167), +}, +/* digit=74 base_pwr=2^370 */ +{ + LL(0xde7c70ef,0xce2d58e6),LL(0x4fd2b399,0xe56a0a18),LL(0xd46ffafd,0x43a772e3),LL(0x61832664,0x5e99ec73),LL(0x5a652a9e,0x068acc97),LL(0xda22ced1,0x829eb99d),LL(0x17534159,0xc94c616f),LL(0xf6ab0176,0xa334609d),L_(0x000001bd), LL(0xac6018e9,0xf0e586eb),LL(0x03144a03,0xdf49ef2f),LL(0x70d82d13,0xf054795a),LL(0x3d4fad35,0xca4e83c9),LL(0x7178dcbf,0xdccd2e81),LL(0x06f96d5d,0x059906d9),LL(0x99860a4c,0xb0cc8989),LL(0x0b7c4473,0xc7a2422f),L_(0x000000a9), + LL(0x7dbd2185,0x990d40ea),LL(0xfd292d5f,0xfe2aa0bf),LL(0x0b3c033e,0x350ffa07),LL(0x7093caf5,0xcba18d05),LL(0x8e77aa62,0x5de1ef34),LL(0x8dcafce9,0x8d305062),LL(0x54c13b97,0xa2184206),LL(0x024b7581,0xc1eed7a3),L_(0x000000eb), LL(0xff7787c9,0xd2467c3c),LL(0x5919f6e7,0x6f3a2cc6),LL(0xe4ef4ee6,0xd95dc335),LL(0x8b15339d,0x53862418),LL(0xc47f7183,0xdc9f6ee9),LL(0x0164075a,0x8fc3c2d0),LL(0xfb8c9b9b,0x82f15ec0),LL(0x2cab4250,0x6da80b24),L_(0x00000170), + LL(0x470b4573,0xce799013),LL(0x9e5e77aa,0xdc0e8efd),LL(0x74901979,0x87335bd4),LL(0x8d25ae87,0x7663b155),LL(0x30b14eb3,0x42427def),LL(0x9f7acb63,0x504e9e47),LL(0x8f787f03,0xb68c9ee0),LL(0x9fb2d8ed,0x3fafed1d),L_(0x00000087), LL(0xded73ba3,0x84c837fd),LL(0x1b05f526,0x361ad6ff),LL(0xa178f8a8,0xba6a96b6),LL(0x0afa0765,0xf1a53f48),LL(0xee02b40e,0x455203e1),LL(0x280a052e,0xa80a8929),LL(0xcc11be29,0x6815682d),LL(0x4811eb83,0xd7ede303),L_(0x000000c4), + LL(0xaaf54dad,0x33981c54),LL(0xcebb5e69,0x32546345),LL(0x544b1b16,0x84cafbc6),LL(0x7981b01f,0x5cddc181),LL(0xaa139311,0x1378ad86),LL(0x68cbb494,0x7e2675fe),LL(0x588ce3ac,0x7f2694e9),LL(0xab708d62,0xeda381dd),L_(0x0000009d), LL(0xe3c020fa,0x31d5f56f),LL(0x1a13df6f,0xcdb4564c),LL(0x02f2a54c,0x586ae362),LL(0x19118f47,0xdb9ebb1f),LL(0x7fa3e3f6,0x0b71b651),LL(0x82c695b0,0x82ecf8c2),LL(0x58306aa8,0xc8a72bb7),LL(0x24bb71fa,0x1671f4f9),L_(0x00000135), + LL(0x1f561cbc,0xccfdb09e),LL(0x9138999d,0xfc40b806),LL(0x190da0c1,0x248d01f8),LL(0x660fe973,0x04db0124),LL(0xd2a3e26a,0xae1441c8),LL(0x3b5b69d9,0x542d784a),LL(0xe47e9fb1,0xc8706904),LL(0x07fd3e18,0x7b0252dc),L_(0x00000097), LL(0x8560fa2e,0x9bf565f8),LL(0x7a07a372,0xb2a08e69),LL(0x88b9b9fe,0x23737883),LL(0x4af5f0ac,0x3ee2589f),LL(0x4d74831e,0x0c99ecfe),LL(0xe461011a,0xccb75730),LL(0x5f6ac945,0x44dfe861),LL(0xde67a0c4,0x0a4190dc),L_(0x00000007), + LL(0x3171ae42,0x8eb6ed93),LL(0x8217bc59,0x480c8b3d),LL(0xf9f73e7b,0xed85e1cf),LL(0xa742114e,0xff0dd45c),LL(0x5d90782b,0x6499236f),LL(0xa8a56eb6,0x81a46542),LL(0xc0a1d718,0xb645bf88),LL(0x5ad3645d,0xda5b3451),L_(0x000000b5), LL(0xf266cd0a,0x8177efc2),LL(0x91a87849,0xd64e2a8c),LL(0xb25866c7,0x7fcea597),LL(0x7ba86329,0xf0a84157),LL(0x4ae784c7,0x6d45e6a0),LL(0xe8a4e8e8,0xbad02a45),LL(0x2b8a78de,0xcb445d9a),LL(0x1a096a8f,0x1c606af7),L_(0x0000006c), + LL(0xeb0a1bdc,0x9ad71dbf),LL(0x013860f9,0xefc2f3af),LL(0x0bf5cd2e,0xcc51df06),LL(0x366cecf5,0x8b6d4774),LL(0xff5f3234,0xb320cd40),LL(0xa48903e0,0x331e170d),LL(0xf463c308,0x7b602dac),LL(0x3d097dcb,0xca3b41f6),L_(0x00000081), LL(0xa54e089e,0x1f06f98d),LL(0x9812b78b,0x0835cbf9),LL(0xa1b46e69,0x96985a6e),LL(0xc756e3cb,0x8c83dd55),LL(0xe0033ed2,0xc71730ab),LL(0x7a46dc00,0x333b6de9),LL(0xe8045912,0x8f656577),LL(0x4a453d9c,0x385fe0c5),L_(0x000000a9), + LL(0x0691a0c5,0xcb831835),LL(0x9d0103c9,0x47a18b26),LL(0xbca486ca,0x2d151fe9),LL(0x8a31af06,0xc2bcd6c3),LL(0x4c38dae7,0x6a2dd494),LL(0xd0cab5f5,0x0e8dd5e2),LL(0xaaffb6b8,0xf1aa0451),LL(0xd1b4ab87,0xd7ec926b),L_(0x000001aa), LL(0x0b242143,0x61149962),LL(0xd2761793,0x4dbe67f0),LL(0x5f42f5e1,0xbdaa4d19),LL(0x9aea3745,0x2770c1ff),LL(0x336a10b0,0x730746fe),LL(0x965986ef,0x500d5a19),LL(0x98b61ae7,0x2c56a5b5),LL(0xfbc2b7c9,0xf20ece26),L_(0x000001c0), + LL(0xcdc9325a,0x47a76498),LL(0xc197a42e,0xa29d1821),LL(0x8a2704b7,0x54a27b18),LL(0x6d7a65e1,0x382392a3),LL(0x6e25e555,0x583ca3d8),LL(0xba1298fb,0x92305de9),LL(0x8762634a,0xc416a724),LL(0xe3560751,0x52b73064),L_(0x00000044), LL(0x9052c067,0xd4b7caca),LL(0x4fc9c7a4,0x0d61d52c),LL(0x88ca3d7d,0x39c96801),LL(0x9d85c914,0x072c2d52),LL(0xeae1af66,0xdc7e4834),LL(0x4567d964,0xcf69cc8b),LL(0x225a1435,0xec05edf7),LL(0xcf0fd41b,0x16674dc5),L_(0x000001ec), + LL(0xea4af642,0xc929fb7d),LL(0x666eee50,0xa82ad2d3),LL(0x943d9f3f,0xe959c5db),LL(0x01361c6c,0x413dcd10),LL(0x810dc990,0xba8e95a8),LL(0x111bfce1,0x144ccf37),LL(0x37942ccc,0xbba23cc8),LL(0x6250c86e,0x64797a98),L_(0x00000124), LL(0xe58dd600,0xa61fd6b9),LL(0x324caf26,0x208fb38d),LL(0x802296a3,0x923005fe),LL(0x1cab4d64,0x545d2ffc),LL(0x7edf08f3,0xfd85bdf3),LL(0xbbb0b3e1,0x9feb12a7),LL(0xab0ed8c1,0xeb4e517d),LL(0x45179c50,0x5c75791e),L_(0x00000194), + LL(0xe472b5a7,0xebddb001),LL(0x809051bf,0x932eff69),LL(0xb46ff016,0x5ce81f11),LL(0xb49261d6,0x39183971),LL(0x75ef3047,0x65753518),LL(0xefad1e5a,0x7887db59),LL(0x147b2e1c,0x6c93b47e),LL(0x239259f0,0xb5f34e30),L_(0x000001c3), LL(0xd6dbc9aa,0x905217df),LL(0x861e1dc6,0x9fada5a7),LL(0x3986b470,0xf9e88cd0),LL(0xcaab1d92,0x839c290b),LL(0x02e99a54,0x39b3ffa9),LL(0x910523a5,0x655e6f7c),LL(0x42d47f30,0xb367bd8a),LL(0x1e0a7f1b,0xe25d7561),L_(0x000001b2), + LL(0x72ded5be,0x4987e69d),LL(0xd0493c78,0xba0a5cbe),LL(0x59557a83,0x75bbdb17),LL(0xf2acabc4,0x65e4a623),LL(0x8bf3c53d,0xd71fb7df),LL(0xc8eb2466,0x7545f576),LL(0xc6d8140f,0x620a0123),LL(0x9f02bd4e,0x67837a46),L_(0x000000b9), LL(0x16cfc43b,0x8b871f92),LL(0xc2538248,0x4b3a3a4d),LL(0xef93af7d,0xdbf730c3),LL(0xbfd3ed77,0xf764526e),LL(0x07f935a6,0xcb6152b4),LL(0x1c016476,0x2bf32571),LL(0xdee3ad5a,0x88aaec73),LL(0xfb09ed56,0x5a614fc9),L_(0x000001cb), + LL(0xb19e9875,0x7c127a9f),LL(0x26712a11,0x642533c6),LL(0xf98c4d57,0xc60c2ae3),LL(0x65b1d46e,0x444527b6),LL(0x71dbb3f8,0xe33842c0),LL(0x4381ace7,0x9e839852),LL(0x6a3b078e,0x2c3ba212),LL(0x4de7e214,0xed5d5463),L_(0x00000071), LL(0xde560bf6,0x2312ab46),LL(0x8e3acd07,0x631b2001),LL(0xb498759a,0xcc66ba39),LL(0x22f04bfe,0xd634fae4),LL(0x0f57a006,0x6fb05b0c),LL(0xd36ff867,0x74e2535a),LL(0x85b24cf7,0xe1ec7865),LL(0x1a8674c2,0x8caeb36c),L_(0x000000ae), + LL(0xb79af8c7,0x41ce000d),LL(0xbbc4d762,0x3f8e4f4a),LL(0x850dfa23,0x315d2f3b),LL(0xc39228da,0x146ba937),LL(0xe938195d,0x05ba80b7),LL(0xa6946ca0,0x5996d5e9),LL(0x8582cc92,0x20830fe7),LL(0x9fbffd5e,0x49bd8864),L_(0x0000013a), LL(0x84fe2ca4,0x52a9ca5f),LL(0x342678f8,0x0fde0f1b),LL(0x03e8057f,0xb03f731a),LL(0x68041b2c,0xc1b65ef3),LL(0x5e348390,0x812a39aa),LL(0x6058d643,0x85301a5f),LL(0xf291dfe0,0x3f43fec2),LL(0x4f9f3872,0x0ea19231),L_(0x000001c4), + LL(0x5896d88c,0x2071dbfe),LL(0x4b89bb6a,0x243135cc),LL(0x0ff935bd,0x7a1a0770),LL(0xe0c7a9ac,0x227a5593),LL(0x6dd5bad8,0x3493e1c5),LL(0x8ba3715d,0x35c53b09),LL(0xc6b271ea,0x744d2bd3),LL(0x2e9feb1a,0x214dd692),L_(0x000001cb), LL(0x59ccbf72,0xeec6e175),LL(0x1f9d847f,0x63ead5ad),LL(0x5419e32c,0xd6156a57),LL(0xe233586a,0xe66a5622),LL(0x39029648,0x4092e8b6),LL(0xbf26b933,0x1e5f719a),LL(0xcd5ad746,0x4f5fc4b9),LL(0xf552f21c,0xd86c96b0),L_(0x00000191), + LL(0xb57fb012,0x9ad2983e),LL(0x090fc8d4,0x3313e6f8),LL(0x55cd8ed4,0x8651d168),LL(0x6a906312,0x55315e35),LL(0x591a919d,0x83731bdb),LL(0x5172884d,0x209c90f3),LL(0xe2ac6b9b,0xfbd125cc),LL(0x92e9decc,0x7d19839d),L_(0x000000c6), LL(0x081749ac,0xed7835fb),LL(0x29318405,0x7858c38c),LL(0xc88969fe,0x39c13839),LL(0x9e3d9e6b,0x193b8588),LL(0xa0e5421a,0xe0167b73),LL(0xc6ebad78,0x3cbbe3a6),LL(0xea506121,0xa8cbcf0f),LL(0x20f99af7,0xdbdf82d3),L_(0x00000134), +}, +/* digit=75 base_pwr=2^375 */ +{ + LL(0xbc40ed79,0xa97c0378),LL(0x851811ca,0x042ad325),LL(0x0425fd5a,0xae223e37),LL(0x93b83181,0xf721e5a1),LL(0x90e949dc,0x0e3457f0),LL(0x3df6a852,0xb654bab8),LL(0x5f22447c,0x720ad354),LL(0xec0dfbcc,0x64eddb51),L_(0x00000100), LL(0xa4ebf78a,0xa97e2d48),LL(0x5a22dda9,0xa660386d),LL(0x4ad7ed63,0xf4eac86c),LL(0x0e9bdfd6,0xa0b3cfe9),LL(0xf3d0576e,0xb9746e43),LL(0x38598ede,0x5a4f247c),LL(0x6c63f53c,0x9110d7d6),LL(0x937de7ce,0xedc5628f),L_(0x00000180), + LL(0x39240f69,0x07ceb75a),LL(0x22d17ed3,0x1d003cd4),LL(0x816d46db,0xacb6fef8),LL(0xb000a452,0x9bbe93ac),LL(0xa33425cb,0x8e7044e1),LL(0xd94105ed,0xbf04fc32),LL(0x7b448d72,0x8a8006d7),LL(0x77527b27,0xce1c27af),L_(0x000001f3), LL(0x04fea417,0xede21a04),LL(0xbea8a562,0xfe905ba2),LL(0x7dcaa390,0x8bd01814),LL(0x8c2e3be5,0xad37906a),LL(0x4bd6ba24,0xe4147c93),LL(0xab35993e,0x54b18700),LL(0xeb32c196,0xc4b62833),LL(0xfde9fb6d,0xefce4982),L_(0x000001ff), + LL(0xac68f4e5,0x326389a5),LL(0x8c87ee07,0xc8d06b64),LL(0xf1eb33ca,0x9f6c9bde),LL(0x905394b5,0xe7cff087),LL(0x9daef572,0x15defa0d),LL(0xdcb4d146,0x48372dac),LL(0x9d9bc2a2,0x558be40e),LL(0x918fbab0,0x093092b7),L_(0x00000105), LL(0x420c8419,0x80e77a4e),LL(0x1e399561,0x7faf6193),LL(0xe636d3ab,0xe0b54eab),LL(0x0991ea6a,0x12db09ef),LL(0xe300cb6a,0x2c4871c5),LL(0x5b2c3ec2,0x74b476ca),LL(0x01ab0f81,0x571997b2),LL(0xe7647206,0x7f0593e9),L_(0x00000120), + LL(0x9d919a09,0xcb8ddce8),LL(0x53953842,0xc94eac86),LL(0x510be22e,0x18af52b2),LL(0x5204fc68,0x6cd384c6),LL(0xb08bd4d3,0x40918e38),LL(0xbbca8f66,0xc2ac8cd3),LL(0x9b3d5866,0x2e4fdaef),LL(0xbd15b5a2,0xcebfa696),L_(0x0000008f), LL(0xf5dbeff0,0x8986becf),LL(0xa9f4f0a0,0xb5bde2b0),LL(0x2781857a,0xf623a384),LL(0xf48f7d34,0x2fc32d5d),LL(0x5aed2eab,0x3357b29a),LL(0x85d8000c,0xd7a02a4f),LL(0x47c091a8,0x83e2289a),LL(0x748c8758,0x78f3991d),L_(0x0000004c), + LL(0xa0a58e79,0xe09895cd),LL(0x8dc487b8,0x1ee06c73),LL(0x4cd52925,0x3615a586),LL(0xb40cf5e8,0x67c6302f),LL(0xe2444c40,0xad0f9fd1),LL(0xf79a9138,0x21560ea8),LL(0x53b1f139,0xdfcbf2e3),LL(0x42f5c15e,0x937e18f2),L_(0x000000d1), LL(0xb23bfb76,0x2b5822b9),LL(0x4289e6fd,0xcce11b47),LL(0x6b3fbfe6,0xfc6c35ad),LL(0x42db4435,0xf40269b9),LL(0xd9e6571d,0x052ecbf9),LL(0x43b34e97,0x85c17cbf),LL(0xc1ac2947,0x2cf45704),LL(0xb8e4df72,0xccda58a9),L_(0x00000190), + LL(0x70c63945,0x736929bd),LL(0x2018c224,0x183d085f),LL(0x4be72094,0xb8504d5a),LL(0xa1b86c43,0xb6f18e21),LL(0x4ff46986,0x6ad297dd),LL(0x8a9142ac,0xaf090f36),LL(0x09fe86be,0x3b6921ae),LL(0x3c8c552b,0x953006e9),L_(0x000001ce), LL(0x0006309b,0xaf72ac6e),LL(0x80956c6b,0xfa741157),LL(0xc4c2c5f7,0x5e9870c9),LL(0x7ec1eba5,0xf57cfca1),LL(0xc10b4b60,0xa6e490cc),LL(0xb0618cc6,0x39e98a9d),LL(0x31fe5f00,0xf0f53611),LL(0xb1970dbc,0xa442baa6),L_(0x00000100), + LL(0xa7015016,0x90a308db),LL(0x61cca596,0xcbaa218f),LL(0x6743511c,0x6574da13),LL(0x71c34c1b,0x50cf99a2),LL(0x570b3140,0x8ae9a0ed),LL(0x8fffc78b,0x816fb1cc),LL(0xe8d131b4,0x052abb5d),LL(0x64b0d1f3,0xe564075c),L_(0x0000006f), LL(0x20ebfdef,0x5fb1b653),LL(0x57fab65e,0x72bc2ab8),LL(0xcd37e51f,0x6c8bb1d5),LL(0x57d81547,0x75b37fd5),LL(0x572bd385,0x1441e8d8),LL(0x0eb239d2,0xd5a6c392),LL(0x7cc7ae14,0x16b857f6),LL(0x7141c32a,0x931901c6),L_(0x0000016f), + LL(0x981eb231,0xe53ec842),LL(0x16678799,0xeb3f78fb),LL(0xdb8c26f2,0x1091298b),LL(0xe09307e4,0x265d3b22),LL(0x79682bc2,0xf829161a),LL(0x8a62536f,0xdea99410),LL(0x002fb6de,0x3d369ec2),LL(0xca79fdbc,0xf58b2f20),L_(0x000000ba), LL(0xa6987577,0xf756d9fe),LL(0x61419646,0x9934641a),LL(0x308b017c,0x5de627fc),LL(0xbea2b76a,0x23ee7d29),LL(0xba8603cb,0x6ab47900),LL(0x85c79476,0x004b96bb),LL(0xf41684cc,0x94d547ed),LL(0x8b7656ed,0x2003142b),L_(0x00000030), + LL(0xacf59bef,0x8759d864),LL(0xfb67c7fb,0x0407a03f),LL(0xfb9982ce,0x020231ea),LL(0x55103874,0x9d3dc0fe),LL(0x9a32c3ea,0xc54c5166),LL(0x8e76b967,0x7422e59f),LL(0x538d7969,0x1567215f),LL(0xc1772e51,0x1bee3ac0),L_(0x00000159), LL(0xbc5e3fec,0x44e31ef8),LL(0xeef4a1e2,0xf4d1de52),LL(0x78709a3f,0x144880f0),LL(0x90e1bf50,0xc5a2e2e7),LL(0x6576ad05,0x963afdb1),LL(0x858a5053,0xfb62a6ae),LL(0x720e44be,0xf7d3d903),LL(0x85ea2a35,0x1ce3e300),L_(0x000000b2), + LL(0x22a69bc6,0xc85cdf02),LL(0xe346fabb,0x2b0945e1),LL(0xe07629de,0x76a1e2b5),LL(0x45e5724f,0x43bc885c),LL(0x6f8c506b,0xbd1f5350),LL(0xc4a247ae,0x9759458e),LL(0xe8c49a8e,0xad9f81fe),LL(0xef961f24,0x0789ce81),L_(0x00000078), LL(0x536c8acc,0xbdac3a5a),LL(0x0d120ebb,0x2f38cfe3),LL(0x29a29c91,0x470f8673),LL(0xa93d27e5,0x85f54b6a),LL(0x347ce7f7,0x869bc2c6),LL(0x681c6e83,0x240291c1),LL(0x5f895132,0xd778a681),LL(0x9354c132,0x35657182),L_(0x00000113), + LL(0x600c6b7d,0xdc0c7615),LL(0xd08ada52,0xf64fa06b),LL(0x6fe343d4,0x0f9f191b),LL(0x269d74ba,0x2b582fb8),LL(0x3e1302f6,0x4f3fa209),LL(0x3dd58666,0xbaec4e8c),LL(0x2346df80,0x2addc663),LL(0x961745b5,0x6358e5f3),L_(0x000001c0), LL(0xd43fe3d8,0xbca3dd73),LL(0x0f473bff,0x97d4d8fb),LL(0xcbcc7f23,0x592d62f3),LL(0x8c21a728,0x2d18d7d2),LL(0x08669251,0x5acddad3),LL(0x10138815,0x5eb1d5da),LL(0xaf710391,0xf88b7078),LL(0x200a8738,0x9614df01),L_(0x00000078), + LL(0x3d24acd4,0x39f8e71a),LL(0xf8ccaa40,0x89f9fde7),LL(0x4a565eb3,0x3a88c7c8),LL(0x2241445b,0xb88e20b6),LL(0x0479b1b1,0xe22d8db0),LL(0x96695cdc,0x02fe3690),LL(0x48a70132,0xaba6a66d),LL(0xafe3713b,0x5be868e0),L_(0x000001b6), LL(0x6375d71e,0xc518718a),LL(0x3e38b8c6,0x00a613fe),LL(0xee16d3bd,0x8bab2dac),LL(0x5f51a73a,0xecd0dde7),LL(0x5d598b1d,0xf9a19d5d),LL(0xcc2ed8e1,0x74ed2f5d),LL(0xb66c7686,0x1a036457),LL(0x45717b78,0xeb14d9fd),L_(0x000000d5), + LL(0x8eeb7cce,0x42bb5d7b),LL(0x69ebb0a9,0xf6f6e0b1),LL(0x24c217db,0x751a1bfd),LL(0xd4eab425,0x1570cf87),LL(0x46afeada,0x55c17749),LL(0x84f69779,0x72264346),LL(0x0e0b6e91,0x43f9c928),LL(0x2a080641,0x5face2cd),L_(0x000000e6), LL(0x24d003a4,0xb97b52aa),LL(0xb84b4c35,0x079d267b),LL(0xea1ade5e,0xfd3d3470),LL(0x64da3bef,0x603b5d99),LL(0x4e2ca35b,0xc90c1bef),LL(0x267ee929,0xa4dcd6fa),LL(0x2e371559,0xeadf09d2),LL(0xdc90620c,0x7ae7bd5e),L_(0x00000131), + LL(0xa33f00cd,0xe19a4c40),LL(0x3bef2c63,0x57c68d2d),LL(0x922215ae,0x03e85348),LL(0xb54763ee,0x7a4a0d2c),LL(0x4381fb33,0x747d2320),LL(0x3d971222,0xc828be44),LL(0xd96627f6,0x6d1199b7),LL(0xabfb6b5f,0x3d2170b3),L_(0x00000043), LL(0x971bb69f,0xbb1d3366),LL(0x8c946a2e,0xa111d7a3),LL(0xb29fb103,0x75997def),LL(0xa9647d36,0x82824e10),LL(0xa45fdefd,0xf29d6d05),LL(0xa9b94f37,0xe35c500f),LL(0x317e08c1,0x4c601022),LL(0x2a6ed921,0xa2afcd4a),L_(0x000000e0), + LL(0x6053b527,0x14a651b3),LL(0x6443bb77,0xc4e092bf),LL(0x3d523243,0xb725b204),LL(0x563f7657,0xf0d19ab6),LL(0x6dd80a2f,0x0c09a035),LL(0xa3a7805c,0x72bfb218),LL(0x767659a8,0x5001304f),LL(0x06ad0ad0,0xae6cf2cf),L_(0x00000048), LL(0x12dbf627,0x73b1275f),LL(0x58294610,0xf2619fd4),LL(0x12455781,0xc2991198),LL(0x822a98ac,0xc52b1be7),LL(0x6f92e55b,0x85c5dde4),LL(0x6f912a88,0x71070200),LL(0xc6ff80dc,0xed86ff4f),LL(0x5fb4c0fb,0xf6cd415c),L_(0x000001eb), + LL(0x2ce62ff7,0xe13291f0),LL(0xa731176a,0xc2f095b8),LL(0x53e5b4c8,0x22d8f01b),LL(0x9b8d5a23,0xf09c9053),LL(0x6cfad192,0xac4c2264),LL(0x016016f3,0xc2d48df5),LL(0x500c56f4,0xbed57312),LL(0x206618c9,0x249d3807),L_(0x00000152), LL(0x64b93c61,0xa752bb21),LL(0xa854f0db,0xad82109b),LL(0x2bd9fbff,0x39d0e928),LL(0xe612cee3,0x5cfc63fe),LL(0x3aca9e51,0x18541bf3),LL(0x0fd5f823,0x1df11f0f),LL(0xdccc44f5,0xe5d7f0f8),LL(0xc8e26d92,0xdc204c43),L_(0x00000134), +}, +/* digit=76 base_pwr=2^380 */ +{ + LL(0x4cea698e,0x90df775a),LL(0xa877dfb5,0x0e8c657f),LL(0xf628f95d,0xc58775b2),LL(0x1f94b622,0x55966c52),LL(0xd94ba3e7,0x2b826bca),LL(0x536e1836,0x1429585e),LL(0x6a8bf64e,0xab9cff45),LL(0xfdc0d065,0x7ad254f1),L_(0x0000006b), LL(0x3e6824f3,0x10be2241),LL(0xa869cd60,0xb5cc49f5),LL(0x399cde94,0x029dfb84),LL(0x53bc96f6,0xc7d08220),LL(0xf3d33d68,0xb6cb5f4b),LL(0xd70bd72f,0x81e790ba),LL(0x5f85b782,0xd6b87ddc),LL(0xd6fbd3aa,0x9bddab2f),L_(0x0000007e), + LL(0x658551e0,0xcff0963f),LL(0x1215b91a,0x7ce3e2ea),LL(0x276c4b8c,0xae4d76fd),LL(0x27c2c599,0xcf3a3b9f),LL(0x985a8106,0x706667b3),LL(0x3dd5545f,0xe5bf95ab),LL(0x9ae8ea63,0x5a494d9e),LL(0xac8eb301,0x36df8e2a),L_(0x0000014b), LL(0xf96eb433,0x241e0605),LL(0xec384ae0,0x19fc3d54),LL(0xbfb3e2ee,0xce0a2d7c),LL(0xa5ac041b,0x7e0aa0d1),LL(0x22b978b5,0xcf6adf10),LL(0x50508726,0x13804525),LL(0x77d6d81e,0x02fbac9b),LL(0x34536c98,0x6666d2c2),L_(0x000001d1), + LL(0xc31ca580,0x0fd75964),LL(0x2c167fba,0x79fb34e2),LL(0x68658968,0x3d2ac14b),LL(0x6bb85f11,0xf9265032),LL(0xe567c4fc,0x09815c6a),LL(0x30478f2d,0xb2da7033),LL(0x2d4e045b,0x7450186c),LL(0xeb491702,0x3d6ff5bb),L_(0x000001cb), LL(0xd6230d65,0xa96aee5e),LL(0x7fa7f974,0x54b866f5),LL(0xd445b199,0x7edd540d),LL(0xda3cc41b,0x6672b9ea),LL(0x3c302e2f,0xf5adb45c),LL(0x5ea3de1b,0x201f8535),LL(0x70efd3fa,0x9bc11d2a),LL(0x3d2e2804,0x4d97e055),L_(0x0000009e), + LL(0xe0b71938,0xdb5aaa83),LL(0x020dd38c,0xf16c4ef4),LL(0x0a2db89c,0xa5cd426b),LL(0x43e8b727,0xf5617c8e),LL(0x23ddc0ba,0xb43d6e58),LL(0x259e17f2,0xc826180e),LL(0x06737413,0x55f63ef5),LL(0x434e7412,0x23e6163c),L_(0x00000096), LL(0x0c64c884,0x9695e5ae),LL(0x47505a19,0x6e1e36e5),LL(0x74ec16e2,0x43d8b0e2),LL(0x4831814a,0x037ed439),LL(0x5b1a104c,0x375672e8),LL(0x5bc4b456,0x9fdc64a0),LL(0xf4e8604b,0xdb5b0994),LL(0xa1d8d54c,0x035e5850),L_(0x000000d7), + LL(0xc1e1dd97,0x6b6358ca),LL(0xae97ec9d,0xf89cd326),LL(0x96931bf3,0x0db33ff8),LL(0x1728a8a2,0x5df6988b),LL(0x8b413bf1,0xc9cd5efc),LL(0x7876052f,0x980dbb18),LL(0x662d8014,0x7d44d414),LL(0x56d9235f,0xf0c89214),L_(0x000000df), LL(0x6a6bdb67,0x7d2553d4),LL(0xd43349dd,0xc275fa25),LL(0x98c5095d,0x7e9d6a23),LL(0x5dae8169,0x48607095),LL(0x004d6221,0x0de66e5e),LL(0x88753853,0x407e61fc),LL(0x0cbeddeb,0xb1e576ab),LL(0x85968acb,0x6df65046),L_(0x000000e3), + LL(0xdf2834dc,0x1d1376b4),LL(0x2c927fc0,0xf4b1b912),LL(0xcb3e3200,0xd6a633a1),LL(0x54477db8,0x4c991410),LL(0xbf0c1c32,0xa9a4d4dd),LL(0xda008df0,0x30c04f89),LL(0xf68e5507,0x1a10f51d),LL(0x5ce5c51e,0x41031547),L_(0x000000da), LL(0x15811373,0xdec76b03),LL(0x4ca12b9b,0x53a8bf3f),LL(0x4e3a3297,0x6ef86a89),LL(0xc5a499c8,0x38a372fc),LL(0x97d666c8,0x6ec44e4a),LL(0x41b99123,0x95600ea2),LL(0x650c8dbf,0x4eb71cc1),LL(0x4c7627fd,0x54f79c84),L_(0x00000186), + LL(0xc38fca05,0x69ce7225),LL(0xf1a6e969,0x77785cf6),LL(0x9cc6268e,0xc7d8303c),LL(0x2b8308ef,0x6b0e5276),LL(0xcba9dc8a,0xa4bf9968),LL(0x416fd26f,0x8c4cdb7c),LL(0xe7d932fa,0xde7df0b0),LL(0x472063b5,0xd8e36d94),L_(0x0000001b), LL(0xd88f945e,0x852f11b9),LL(0x528d0c6d,0xe34ebb6a),LL(0x0491c222,0x572cf3b4),LL(0x3235246f,0x6a507a97),LL(0x5419c482,0x151b4954),LL(0xa45d1468,0x1e9ff246),LL(0x555cac59,0x74cf9098),LL(0x3f67c66c,0x4d10852a),L_(0x00000081), + LL(0x109aae5c,0x4d4d6495),LL(0x86a81e7f,0x54df2c4b),LL(0x4316eb10,0x19b90005),LL(0x41b6877c,0x63ac12d9),LL(0xdbe38379,0x7bdc46a9),LL(0xe68280f8,0x04afbefb),LL(0x1d97d1dc,0x64f8fe97),LL(0xfdaabbcc,0x3ef9d7ec),L_(0x00000054), LL(0x1998e321,0x4256ebb2),LL(0x5f744a3f,0x462bbab0),LL(0xc6587a4d,0x8de305cc),LL(0xf7cc14a1,0xd2c0e8c5),LL(0x1c2fa456,0xd7f552fd),LL(0x93096db0,0xdf5d165b),LL(0xeef9e935,0x0a7d4ef3),LL(0x8313440e,0x2ecbc3b6),L_(0x0000000c), + LL(0x86173dd6,0x4ec080d5),LL(0xc4429668,0x00e4c47a),LL(0x0c5790c3,0x2bb3e90a),LL(0x76060854,0xb06d2fa8),LL(0x51871594,0xb8a8220c),LL(0x35fa01c4,0x96cfa275),LL(0x351722c6,0xd2b5aea0),LL(0xcac00f2f,0x9a7e1203),L_(0x00000023), LL(0x30706067,0xa79d695c),LL(0xe8bbd2be,0xc13d4a47),LL(0xdd17ddc3,0x3a2ef1d7),LL(0x835e7fe3,0x3a0d7223),LL(0xdaab1fae,0xeadcb841),LL(0x2baa3375,0x25d48c28),LL(0x3675311d,0x1cfce1d1),LL(0x8cca0828,0x6d648baa),L_(0x0000001e), + LL(0xeb6ee1f4,0x39772678),LL(0x4730fdee,0x4814bf38),LL(0x7da6e5cf,0x717ace32),LL(0x2440a79c,0x1fa530fc),LL(0xee76a431,0xf0840ed5),LL(0x64a9d867,0x9b3e52c9),LL(0x01e024e8,0x388a3167),LL(0xbc5a3de8,0xb45ab215),L_(0x00000127), LL(0xff0e20a7,0xdb989f10),LL(0xb0c72279,0x88321c3c),LL(0x461d5212,0xe2e0c887),LL(0x22583d6f,0x0422ef3c),LL(0x5319c021,0xf26dbc88),LL(0x3aba5f48,0x62bbe876),LL(0x7e742165,0x411f00ae),LL(0xd32aa7f6,0x6608e197),L_(0x000000e2), + LL(0xe82a5867,0xb2aed406),LL(0x5c51d66d,0xe49e1c0a),LL(0x341d6090,0xeca16754),LL(0x50aaa76d,0x3d4ae66e),LL(0x23dd6ea7,0xc264093c),LL(0xb964aff3,0x124124d1),LL(0x6a903309,0x320f2ccc),LL(0x040a80c9,0xa2fc450d),L_(0x000000f1), LL(0x442fcd61,0x660ce624),LL(0x876d5eb7,0xb113de73),LL(0xd6a08a24,0x8fdedfa3),LL(0x7e981617,0xccca4ec9),LL(0xbf6d63e9,0x1cbf7303),LL(0xd5b865e9,0x06258e51),LL(0x8b4e0432,0xae5f01e7),LL(0xed7485d8,0x1fb3ec8e),L_(0x0000007f), + LL(0x9cc6138d,0xcd3a614c),LL(0xaac40dbd,0x933beab8),LL(0x23a3080d,0x4ca0b1cd),LL(0x3c101e4c,0x2d0376e4),LL(0xcb246c76,0x8ef2560e),LL(0x3a96d882,0x54be7604),LL(0x792be430,0x4ed7cfd6),LL(0xe67d1eea,0x924a5689),L_(0x0000002c), LL(0x64c2420d,0x63647d6b),LL(0x9642188f,0x514cab56),LL(0xbcdfd904,0x529ea4a2),LL(0x876c6668,0xcbd5305d),LL(0xfa8c20c0,0x58b69ea6),LL(0x1ac42596,0xf62b1c30),LL(0x80232775,0xee3d0824),LL(0xc5b975d6,0xc483c2ea),L_(0x000000b2), + LL(0xd2eb9667,0x8bc6d688),LL(0x6c7bd269,0x7652c729),LL(0xc96e37bd,0x405791df),LL(0x410e2904,0x5ab8090e),LL(0xf9bde0c1,0x1a7b424c),LL(0x37d8159f,0x26876e27),LL(0x6e7212ab,0x5e21bc5d),LL(0x2ff3af58,0xa29dba58),L_(0x000000a6), LL(0xb27af4fd,0x7dd665a0),LL(0xe048ab97,0x19984a71),LL(0x17b7a849,0x8f61b833),LL(0x9bdf5b57,0xeb63a0f7),LL(0x32adf9dd,0x8eaf0eb1),LL(0xe30814ac,0x799c8225),LL(0x35be0b92,0xa082ff80),LL(0x7e52495d,0x196f3154),L_(0x00000054), + LL(0xe45aefc6,0x3b3fccf2),LL(0xf8b435c7,0x7f599023),LL(0x4b07f56c,0xc6614964),LL(0x1f9e8dd5,0x7bfaa97f),LL(0x57a41fcb,0x056d5124),LL(0x69dc4c19,0x33956d10),LL(0x2e70770f,0xac607019),LL(0x17397aa9,0xbda3b4ee),L_(0x00000082), LL(0x8bf00dd1,0x11800cef),LL(0x9f451540,0x5de0b39d),LL(0x9cd9ec18,0xf09ca421),LL(0x7bff70d3,0x3fc8f958),LL(0x6af7560e,0x348716d4),LL(0x2601495a,0x4ed53bb1),LL(0x39eaa8c2,0x97052bec),LL(0x87acae02,0xb4363c7a),L_(0x00000069), + LL(0xfcf4623b,0x010e4074),LL(0x23ceb817,0x68de1fa6),LL(0xb6c3610b,0x79b5037c),LL(0x794616e3,0x38ca34e7),LL(0x6964c6c9,0x64ba9eb2),LL(0x9d828a84,0x713e3f60),LL(0xc6cafea4,0x69bdca04),LL(0x0035da7c,0xc53921ac),L_(0x00000170), LL(0x66f97d4d,0xbe97815a),LL(0x678d3502,0x09bb25d5),LL(0x417b9931,0xadbec401),LL(0xa021e930,0xef1be11c),LL(0xb53777ab,0xfb3f04c1),LL(0x2e6bc85d,0xab9fbf13),LL(0x22a4d27b,0xb988012f),LL(0xddee5ad9,0xfda8aea8),L_(0x0000007a), + LL(0x3ed86b54,0xc43ac524),LL(0x9805e79d,0x95a2175f),LL(0x2bee2dfe,0x6125c31c),LL(0x2b6284b0,0x10319508),LL(0x2264eba0,0x8cedfa4a),LL(0x25bc143e,0x3199afa3),LL(0xe3ae2485,0x63067c6a),LL(0xebebe969,0x54a7cecd),L_(0x0000016c), LL(0x9434e363,0x6de3a522),LL(0x3a1a5044,0xf721555f),LL(0x644f2db4,0x6dc38924),LL(0xb72ad43f,0x39beb126),LL(0xe7dd7722,0xd840de05),LL(0xd6caacd0,0xc67a2862),LL(0xce6fa639,0xba53021b),LL(0x71087602,0xec9b5982),L_(0x00000079), +}, +/* digit=77 base_pwr=2^385 */ +{ + LL(0xb9a7e9fc,0xb75ed4da),LL(0x7aaab2f5,0x1ee37679),LL(0x30159305,0x0b02f44a),LL(0x021962a3,0xd622cf13),LL(0x55a3eea1,0x9a7dfa05),LL(0x4fcd685c,0x7a2a6aca),LL(0xd2e75077,0xbd4c914a),LL(0x1e6aa905,0xeec52d7b),L_(0x000000ff), LL(0x3f6fa1e0,0xfa95204a),LL(0x539f85e4,0x36eeec34),LL(0xe8ddc16e,0x74599d1c),LL(0x50244a9e,0xb343c6c5),LL(0x714c017a,0xb07951ae),LL(0x4503f92d,0x44d15c8c),LL(0x830499e5,0x94680ff6),LL(0xe7188a7b,0xd6c4809f),L_(0x00000147), + LL(0x9a2546bd,0x1bf14718),LL(0xf89fcfd4,0xc079070b),LL(0xa403ed89,0xa107b324),LL(0x18c3f861,0x4dade6e3),LL(0x665e9f9b,0x332b6327),LL(0xb408e3b3,0xf62f16ec),LL(0x11ee2181,0x67bbd1bc),LL(0xd0ba5904,0x4b5440b9),L_(0x0000000f), LL(0xde86660f,0x9cb1aa7a),LL(0x8a32a33d,0xedb96d1a),LL(0x9ae722d5,0x0654bd1b),LL(0x1664c777,0x03d0e5a5),LL(0x6a4a631d,0xfb01ee81),LL(0x3d1d9344,0xc7691584),LL(0x1e1821b3,0xbe2d285c),LL(0xafc22520,0xe0834d6f),L_(0x000001e7), + LL(0x256a6798,0x08d12f4d),LL(0xf2407064,0xd7255fb6),LL(0x1c6799f2,0xad7d86d9),LL(0x8c32b1c7,0x259fb289),LL(0x172083c6,0xcdc9f2eb),LL(0xc85a5a26,0xdca9f61e),LL(0x303eee79,0x82cff2b2),LL(0x283cc245,0xaea38a1c),L_(0x00000018), LL(0xac3447b7,0xf7fee514),LL(0xb0b385f6,0x0a48204d),LL(0x785b0f88,0xaff858dd),LL(0xae66256a,0x24f69e65),LL(0x92f5e352,0xe99c8d90),LL(0xb2e1ccee,0x40cfa4d5),LL(0x5201f6ee,0xfefbb836),LL(0xbcabc908,0x61f21689),L_(0x00000144), + LL(0x5183d337,0xd756d8ef),LL(0xaef07505,0xdd4c26d4),LL(0xfd1d5c09,0xb43bb4e7),LL(0x53c26645,0x6a817eb2),LL(0xf92f5487,0x6209c32a),LL(0x3e205a08,0xb9b883a6),LL(0x802502f8,0xa6f9cb8d),LL(0x2169b5a7,0x87089ec1),L_(0x0000008d), LL(0x5c3f78b6,0x4cc2981d),LL(0xc0b6dcff,0x39135fd7),LL(0xe2051e1d,0x0f800ca1),LL(0x76456f99,0xae12c766),LL(0x987a86bb,0xdfc5fcbf),LL(0xcbf344cf,0xa853db02),LL(0x65a4f55e,0x24b115f9),LL(0xf4d8cf4b,0x28cffa2b),L_(0x000001db), + LL(0x8518ed22,0x9698f500),LL(0x46e1cf5e,0xa37c3c3b),LL(0xb10d9d35,0xb6d8d81a),LL(0x6814e15a,0x65739419),LL(0x9282b5ea,0xf890c6b6),LL(0x80d764c7,0x7d309653),LL(0xda043f0c,0xd070b1e7),LL(0xc9e15f63,0xb143ef10),L_(0x00000056), LL(0xc6e28862,0x47fd9b69),LL(0x525c5453,0xc9876e94),LL(0xe52b9bcb,0x3078ffdd),LL(0xab8f2cfa,0x7e1a8a45),LL(0x338e4367,0x3382f009),LL(0x5b0092de,0x06454df0),LL(0xdb5d1cbb,0xbbf3f2d3),LL(0x17b40f75,0xb1b7961b),L_(0x000000b1), + LL(0x3c3b17c4,0x7aa9d1e5),LL(0x71e12980,0x69777935),LL(0xdfdae6a9,0x2f45dffd),LL(0xf2900457,0xabe3441f),LL(0x6471580e,0x5aece4d0),LL(0xd983618d,0x85cd6571),LL(0xf5a1d861,0xc2bd978c),LL(0x955894c4,0x81cae98a),L_(0x0000017f), LL(0x4b6d344e,0x3176598c),LL(0x4b7790dd,0xc91def7d),LL(0xb21152ba,0x279d992a),LL(0xd92aafdd,0x7dab9fb5),LL(0x59dd70d0,0x7039956b),LL(0x14d13b0f,0x7cfe20de),LL(0x7bd5d4c8,0x45b10bc2),LL(0x59724543,0x906a5d0d),L_(0x000000cd), + LL(0x2093c6ef,0x80b9235f),LL(0x380affbc,0x0b4024a8),LL(0xcd02b098,0xc98a3c0b),LL(0x36545f0a,0xe62cf7ea),LL(0x3b089c99,0xdade8fc7),LL(0xf9b4c955,0xf404f355),LL(0x7650e822,0xae64ac11),LL(0xcc0fb628,0x593a1b6a),L_(0x000001df), LL(0x4231a24f,0xa51f1936),LL(0x091d5493,0xeabe8135),LL(0x5285ec41,0x6bc185fe),LL(0x9a7df8de,0x25f54dac),LL(0xc1b5836c,0x4e5fc638),LL(0xde102ff5,0x81a60442),LL(0x56a67f9d,0xaeaff6ca),LL(0x7cc5f40e,0xdaecfbdc),L_(0x000001d0), + LL(0x01bbcb15,0xcd4c9f1b),LL(0x64ba5d43,0x262898c5),LL(0xb6d345ac,0x0f11b91d),LL(0xf8015622,0xfa53c395),LL(0xeb32e3dc,0x4e8f0647),LL(0x125c49a2,0x537471de),LL(0x5c783701,0x7a9741e3),LL(0x00bc6a87,0x1c5d3aa6),L_(0x0000003e), LL(0x5cbf2fb6,0x18eae31d),LL(0x1336f732,0x7555dff2),LL(0x63097ec4,0x8f16a8d5),LL(0x928e41ac,0xc063790a),LL(0x72cf7210,0x2842e2a8),LL(0xbed4668e,0xe7f0d214),LL(0x7b5aab91,0xbd94783e),LL(0x472089cb,0x55df6f3c),L_(0x00000177), + LL(0x5f960853,0x6ba36bea),LL(0xd6462023,0x51b0110e),LL(0x0299f400,0x5ad94e5d),LL(0x1c56f2f6,0xd8d6e619),LL(0x0b4ea27d,0xe73e18fa),LL(0xdcfdac26,0x61a026ce),LL(0x27dab320,0x4ebfdadb),LL(0xb6af0729,0xe9561c2a),L_(0x000000e9), LL(0xc786fe34,0x1418a240),LL(0x7a5020b9,0x2b5125f4),LL(0x7dac2ce9,0x6985bdc7),LL(0x7a36a07a,0x6d385362),LL(0xc0a58550,0x940163b2),LL(0xd28cbf38,0xfb9a9d22),LL(0x8eddbcd2,0x1cfcdeb4),LL(0xad40ff84,0x41da1441),L_(0x000001a8), + LL(0xf50794a5,0xf1ab6a88),LL(0x32d8d898,0xb4c956c4),LL(0xbd9881d2,0xbc516c73),LL(0x5116a36c,0x91e840e1),LL(0xff4abf28,0x14bf8bab),LL(0x2bc617a5,0xb012c75f),LL(0xba5d811c,0xf333effb),LL(0x37bcddc4,0x771a4567),L_(0x000001ff), LL(0x4a68eb29,0xa48d6dfa),LL(0xf2542b71,0x495f434b),LL(0xe3f39bde,0x9f969883),LL(0x179f2c63,0xa68cdccf),LL(0x44e28315,0x7408c1bf),LL(0xeb7b9849,0xf6615345),LL(0x823ede15,0xdf405d5f),LL(0x17f01e94,0x0efd64e2),L_(0x00000156), + LL(0xb7d5223a,0x6794bc7e),LL(0xfa3914f9,0x044623b2),LL(0x3e94e3f2,0xc7e42b96),LL(0x85cc2a9d,0xe0fbde7f),LL(0x2e0e1f42,0xde5ec740),LL(0x0a1b4e4e,0x99d96c6e),LL(0xa3d7e876,0x9b31d8b9),LL(0xcce96a38,0x4fd9fc85),L_(0x000001dc), LL(0x05826168,0x5b7a322f),LL(0x4317247b,0x01f0266b),LL(0xfd3d2a1d,0xa9fbb760),LL(0x75fc993e,0x60905a87),LL(0x51ce6740,0x3c6c984d),LL(0x16580d6e,0xe79ccbef),LL(0x585522ad,0x57fa547e),LL(0x42f80ccf,0x8ad71acb),L_(0x00000188), + LL(0xb6d705e0,0xb16ee0ab),LL(0x28366fe7,0x0365fed1),LL(0xc78aeb84,0xf329889f),LL(0xee74063c,0x16267a5a),LL(0x9c5a9197,0x040a619f),LL(0xc54b40cc,0x16e7345f),LL(0x9d5f609d,0x957fc0b1),LL(0xaf8fbbc8,0xb1cb4d02),L_(0x00000117), LL(0x099b7338,0xd023f32f),LL(0x1deda80b,0xacb49a95),LL(0x4c11c95e,0x00e1f672),LL(0xde8db891,0x19ee52cc),LL(0x27a5b5d8,0x6176a0d3),LL(0xd6752e2d,0xdfccd883),LL(0xe819161e,0xe1282202),LL(0xe71376c0,0xb92fddf6),L_(0x00000047), + LL(0xf4c8ed53,0xfecc3203),LL(0x7168d714,0x1726e41f),LL(0xea5d7e70,0x69dd518d),LL(0xe7aec797,0x2cfd3a6e),LL(0x9ab3823a,0xb90a5bea),LL(0x43831c4a,0xdbb35cc6),LL(0xd5853b0b,0x95c10ae1),LL(0x55834ab5,0xd668a3cb),L_(0x000000d5), LL(0xaa33d813,0xc0f7662f),LL(0x98346aa6,0x173a0008),LL(0xabe91f50,0x0508c118),LL(0x97cdd826,0x7255d2e1),LL(0x01d1e340,0xeb07ccd4),LL(0xf2f7ac53,0xc6829327),LL(0x966981ba,0x0fb3f4fa),LL(0x1ee6ae0f,0xea212110),L_(0x00000091), + LL(0xb4ca9aa1,0x3c713d51),LL(0x946494a7,0x1e6130d8),LL(0xbd5ab69c,0x6cbf341e),LL(0xcf57622a,0x94d1578b),LL(0xc7327897,0xb1db7d17),LL(0x2874559e,0xf5607998),LL(0xd15eab48,0x18199bc9),LL(0xfe1d9b44,0xd0aedc11),L_(0x00000056), LL(0x931fafb3,0x50c646ed),LL(0xb75609c0,0x9721b326),LL(0x4a7a787a,0xed16025f),LL(0xbf8835a3,0x78dc9b8f),LL(0xee0c8b9a,0xf388850b),LL(0x5a1458a1,0xc5cee39e),LL(0xb7bfeb06,0xecfd0e49),LL(0x8795b039,0x70bdfc80),L_(0x00000111), + LL(0xaa736cd0,0xa3c85c78),LL(0xbd49ba23,0xe512fde9),LL(0xdb860d3f,0xbf99d4e9),LL(0x3cb574a7,0xbc4c4fcb),LL(0x9af33dfa,0x88710836),LL(0xfd7ca4de,0x4b83fe29),LL(0xa2306095,0x03d40abe),LL(0x5f9ae75f,0x5e83ab1a),L_(0x0000007e), LL(0x64e02741,0x97d0048f),LL(0xf861736c,0xde0ffdb3),LL(0x5e0c4806,0x335dce14),LL(0x76e125cd,0xcc1ff27f),LL(0x16822cb3,0x8414749b),LL(0x44ae46e0,0x68085d31),LL(0xe89d0a2c,0x9c734630),LL(0x569b26c1,0x2d89e8f0),L_(0x0000004b), + LL(0x0c5ceff9,0x1dc45be8),LL(0xf3a736f8,0xb7b948b5),LL(0xf684853d,0xa1d8a120),LL(0x7de7af37,0xe3760299),LL(0xadb401c1,0xa1ba866c),LL(0x24e3e6ca,0xcd94496a),LL(0xc3eae9d6,0x06f82a82),LL(0x4a74e14e,0x069d9983),L_(0x00000143), LL(0x25e3e5c6,0x2a60baf5),LL(0x3855bae4,0xfbb6aadc),LL(0x20017824,0x950ab7cd),LL(0x9dfd2439,0x96b97104),LL(0x65cd1e13,0x4925d16b),LL(0xf2ec4dd1,0xd82c0815),LL(0x9059e2e1,0x3ba03a79),LL(0x2f723163,0xf35a3c60),L_(0x00000017), +}, +/* digit=78 base_pwr=2^390 */ +{ + LL(0xaa6fc06d,0x05eac10d),LL(0x1d0d93fa,0x3fb80a45),LL(0x30dbe366,0x69a67246),LL(0x1373df55,0x3a7b7740),LL(0x3b688d58,0x9a1bf576),LL(0x6f4ce0d1,0xc2030797),LL(0x72348a80,0xd874e749),LL(0x24296b9d,0x26fc8724),L_(0x000000d7), LL(0xacca6490,0xa4626a33),LL(0x0b206a34,0x661fc19c),LL(0x597e1cfe,0x1f98c11b),LL(0x010915e6,0x00c41adb),LL(0xbdcb2d64,0xfed21859),LL(0x07de82c2,0xeffa2475),LL(0xba1295da,0x73a4f297),LL(0x570842d6,0x1fc083b4),L_(0x00000172), + LL(0x3f4e6262,0x05474c79),LL(0xf27eccc5,0x9bbd81b9),LL(0x4061bf3c,0x1385eed6),LL(0xe5477666,0x2d5cbcfe),LL(0xa0f415ca,0xab0a8618),LL(0x18675f88,0x47e7c454),LL(0x0d5df7ae,0x75324e3e),LL(0xe2f88082,0xe818ccf0),L_(0x000001bf), LL(0xacbfbda3,0xc2d8f452),LL(0xef224de6,0x9ce12031),LL(0x70dcb7f7,0xd958660b),LL(0xc4597b18,0x4dfd1c89),LL(0xda0fa522,0x2d69ce10),LL(0x989afa82,0x46e81b65),LL(0x4e1b3fa1,0xdcac5039),LL(0x832ae217,0x7d3eef41),L_(0x00000032), + LL(0x95a0d4c5,0x5daa1b89),LL(0x542982e7,0x98f2622c),LL(0x5b7e32c5,0xa09068bf),LL(0x6931e543,0xeb5500e2),LL(0x9089e794,0x2a396afe),LL(0xe5f72142,0x68403342),LL(0xf625b926,0x14e9fc8c),LL(0x6171d647,0xe3e5f19c),L_(0x0000006f), LL(0x3951af89,0x9dfea1fb),LL(0xaf413d88,0x0a1d61f5),LL(0xb4928484,0xc08e23bd),LL(0xe1c147cc,0x3ba7bae4),LL(0x0b6e0358,0x7c12896b),LL(0xa84a4aed,0x0195f08d),LL(0x7bcd8649,0xeba99c7c),LL(0x799d1ac2,0x0193b278),L_(0x0000008c), + LL(0xd9c64878,0x91f424e1),LL(0xb2d18470,0x482b31fa),LL(0xc18abe11,0x4548d244),LL(0xc9e02a20,0x901117bf),LL(0xb1dc1cd2,0x286c34a4),LL(0x8ee98a6d,0x04c325a4),LL(0xb76b3ae6,0xdb518aec),LL(0x981c8298,0xb7194b24),L_(0x0000008f), LL(0x6a5cca1c,0xfd055d97),LL(0xc7061a3b,0x8a107a2c),LL(0x5a667bf8,0xaf0a63ac),LL(0x21028dc3,0x3e8641fb),LL(0xa80d9487,0xd40a558d),LL(0x63f8d8c7,0x3accfc09),LL(0xccc46547,0xfbd53079),LL(0xe85ce362,0x15006f15),L_(0x0000001d), + LL(0xb205016d,0x913326ba),LL(0xe7e20db3,0xe0e58c40),LL(0x4dd3a381,0x4f08867b),LL(0x8c0db229,0x1f57cf8e),LL(0x862604e5,0x9951f866),LL(0x588c2473,0xdd04385b),LL(0x2a4f4278,0x13235969),LL(0x334e1277,0xe14a7afb),L_(0x0000007c), LL(0x3623f8c9,0xce866486),LL(0x4c0e101a,0xfa8ba6d4),LL(0x8dee1016,0x1cd14dc2),LL(0x5e8be2aa,0xcc56a3c9),LL(0x5bdaa677,0x26fe9020),LL(0x787cbbf4,0x6059a9eb),LL(0xf27c2ccd,0x52b29d1e),LL(0x431ad6f9,0xe9273bbd),L_(0x000001d1), + LL(0x9927e056,0x531296c8),LL(0xa4ea86fa,0xe8dd3548),LL(0x107a45cd,0xd8b7e446),LL(0x1bfd985a,0x280a4e8e),LL(0x4fc2dc66,0x4b39f03c),LL(0x86dfa832,0xc44021ef),LL(0x55d3ccb1,0x210fe263),LL(0x65b0f1db,0x68364a82),L_(0x000000b2), LL(0x6b668b89,0x2aec7e7e),LL(0x7906edd3,0x941505c5),LL(0x5f4fddad,0x657d2929),LL(0x320073ac,0xefaffd81),LL(0x7ab9edd6,0xeb40eb69),LL(0x23d38d94,0xc8df60f9),LL(0x1840540c,0xbfea59c1),LL(0x3958b571,0x2f7620df),L_(0x000000e7), + LL(0x47515b25,0xa8d54597),LL(0x3ace77d1,0x23823b9e),LL(0xb05ab9ad,0x7d6803d7),LL(0x36f8db67,0x8ed91e87),LL(0x8fbc69ec,0xe0042a07),LL(0x6a246110,0x5131b89e),LL(0xf857a2db,0x4c1861d6),LL(0x311746a3,0x84c00e5b),L_(0x000000b3), LL(0x28bacfad,0x95ff285c),LL(0xdd895ec5,0x9a709b4c),LL(0xadcb32cb,0x148a78f6),LL(0x7c40e4b0,0xf62708ee),LL(0xd37d48a3,0x68232a30),LL(0x382492cc,0x79b622f3),LL(0x92d657a0,0x09992500),LL(0x6a420a8f,0x501d8de9),L_(0x00000062), + LL(0xc3287fc0,0xc122141c),LL(0x66149a76,0xc602d382),LL(0xfdcc4f6e,0x16b23844),LL(0xea0673e3,0x7ff0591c),LL(0x53082d2f,0x82264497),LL(0xafeaf6b5,0xca528c9c),LL(0x01a84f6b,0xe2403638),LL(0x43738672,0x54c4174c),L_(0x000000fb), LL(0xab8f5585,0x4205f340),LL(0x0b6de62a,0xa84853c5),LL(0xc67ea2a5,0x92e0d583),LL(0x296584f2,0x6bb7a441),LL(0xa9f0cd97,0xce6eb31f),LL(0xd0d33dfa,0x850b5886),LL(0xca9b5dbb,0xe75a3fb5),LL(0x143bbd95,0xfc35ee42),L_(0x000000a9), + LL(0x34b49dba,0xfe747fdd),LL(0x0c47e2c5,0xf9ae16bd),LL(0xa07b48d2,0x01c347f0),LL(0x2cf4c727,0x747fa6ef),LL(0x850ead03,0xff312b98),LL(0x8726545d,0x5218270e),LL(0xa6f367cc,0x7680a110),LL(0x37efad3c,0xe34b12a3),L_(0x000001a8), LL(0x5db6b25b,0xe1fe8e14),LL(0x5767922c,0xe0242975),LL(0x20a012f1,0xba21fc56),LL(0x49ff848f,0xbd4ab80a),LL(0xde1b4b0f,0x24104f37),LL(0x314917b2,0x221548c9),LL(0xde9c2909,0x6f74cf12),LL(0x75ca1868,0x44d62839),L_(0x000001c2), + LL(0x719fc99a,0x3109d76a),LL(0x044307f0,0x7372da8e),LL(0x9ba083bb,0xd35f32f6),LL(0xd25b7e53,0x0a6c99ed),LL(0x30794ede,0x1523c414),LL(0xed97a2a7,0x7dbb6798),LL(0x6d897d7a,0xb996ba83),LL(0x3c0356d5,0xad5ef539),L_(0x00000049), LL(0xf8000ae0,0x37a40ab6),LL(0xfbb2d425,0x468766c5),LL(0xd03d4766,0xe6a77a37),LL(0x09eb18e1,0x9d7f1644),LL(0xbdb7ff41,0xeeb2f1b2),LL(0xc98d5a19,0x8377c08e),LL(0xda52ef67,0x1078c0d1),LL(0xbd2d3d66,0x9181c3b8),L_(0x000000fd), + LL(0x2445d3d3,0x4f7f8d77),LL(0x663aa882,0x9e806c61),LL(0xcd6d9ab0,0x434a485b),LL(0x575aafa5,0x0477ae83),LL(0x1fb377fe,0x7b98c782),LL(0x4ac83912,0xe2f16d17),LL(0xd098d31f,0xeaff3b56),LL(0xfba7077f,0x13382e53),L_(0x000000cb), LL(0x128eb20a,0x4aad91ac),LL(0xce1a9e84,0xc8fc10e4),LL(0xb01bef42,0x6afd2473),LL(0xb1a03e1c,0x507c30ff),LL(0x3760275a,0x162b2e12),LL(0xabb38d7f,0x558877e8),LL(0x88916de2,0x9f519917),LL(0x2fdccf2d,0xf70b4695),L_(0x00000020), + LL(0xf946f824,0x4848296d),LL(0xe0f71789,0xf3835d1a),LL(0x9fea01e8,0xaf761a04),LL(0xc7d32c83,0x1060dc76),LL(0x6c5ff4c8,0xc3751915),LL(0xec6a8a45,0x6f1dbd4a),LL(0xdf6166cd,0xa64bcecc),LL(0xa942edd4,0x53c4d352),L_(0x00000073), LL(0x0fac6740,0x957c5d10),LL(0xe757c12a,0x60819db9),LL(0x5aae5302,0xfbc64c70),LL(0x72a99d6e,0xb00bbd49),LL(0x3bf68d69,0xf0b755c1),LL(0x125fdc71,0x4502c9bf),LL(0xe7e83089,0x7e20c9f2),LL(0xaefe0993,0x5fa7bc0d),L_(0x0000002b), + LL(0x1c0e7e3e,0x596ae14c),LL(0xc255b673,0x1bb0ac11),LL(0x19c21f40,0x04f2091c),LL(0x1838e69c,0x3e9cb56d),LL(0x51439522,0xc1cecfc9),LL(0x52a9b2f6,0xc564467e),LL(0x3576c345,0xeb0fdcd3),LL(0xd04ad0d7,0x3c3aa2c6),L_(0x000001ef), LL(0xba0860cf,0x54b2e727),LL(0x044d2e1b,0x8701d001),LL(0x794b02f7,0xd8153787),LL(0x4f463b8e,0xa0dfa5ea),LL(0x253e324d,0x8fc87618),LL(0x19c78239,0x6165c650),LL(0x97b77e54,0x134c2cb2),LL(0x5e23fc18,0x5ade1fce),L_(0x0000012a), + LL(0x17c9180d,0xaad903e8),LL(0xb407a7f0,0xa64c2d6e),LL(0xc577e3d9,0xd68527c3),LL(0x398177a5,0xdf38bff9),LL(0xde6b9d9e,0x746c3309),LL(0x1d7385a5,0x98512661),LL(0x3b8d9701,0xe82c0d22),LL(0xe2b7edd9,0x16ce165c),L_(0x000000e2), LL(0x4980d6c2,0x491b562d),LL(0xd2049870,0x1de2def6),LL(0xcf01bb60,0xaff2152d),LL(0x20717620,0x5764198c),LL(0x503a4861,0x9702cec2),LL(0xce53ec1b,0x886169de),LL(0xe97c752c,0x8d58887b),LL(0xeb1ce735,0xe1b8ee6d),L_(0x000001ca), + LL(0x2c437027,0x6213a0d1),LL(0x1d410b7b,0xba1aa221),LL(0xa8aa0a76,0xbd669af8),LL(0x06a3140b,0x196f88e0),LL(0xdc333cc8,0x1c9aa099),LL(0x60557f79,0xe1ff46ae),LL(0xa7b7da68,0xd7626d62),LL(0x2a5a72d1,0xffb47531),L_(0x000000d1), LL(0x1a06575e,0x72d98927),LL(0x8e521d9d,0x6acd3ed1),LL(0x855c8e97,0x47c48377),LL(0x42413164,0x89182c42),LL(0xae6a473b,0x59d11d10),LL(0x730eb702,0xd4abd611),LL(0x71fd7c4f,0x4d329537),LL(0xe8493373,0x5e745909),L_(0x00000143), + LL(0x3b653b69,0x4c6f3e4a),LL(0x688f6e0a,0x2a9b6d1a),LL(0x9e0c6a62,0xfebdd4b3),LL(0x7fb92759,0xe14d66ca),LL(0x3d698e7d,0x0a2edb8e),LL(0x08ea4d11,0x10b21ab8),LL(0x0f855706,0xf8405b08),LL(0x104c29e8,0xadf7ff63),L_(0x0000007d), LL(0x2f741dbb,0x4757c2f5),LL(0xe671ed88,0xb38018e2),LL(0xd5288875,0x629331d9),LL(0x279d96a8,0x7e602196),LL(0x559eca5a,0x81f2dc09),LL(0xf274c146,0x96818fe5),LL(0x483d0424,0x1c8246f6),LL(0x76f5286b,0x44ba1052),L_(0x00000179), +}, +/* digit=79 base_pwr=2^395 */ +{ + LL(0x0f7e89bf,0x42d44166),LL(0x0a8f4b54,0x924764d2),LL(0xf7bf31e5,0x26e4af60),LL(0x7be28350,0x9fc53bdf),LL(0x08056835,0xa12726ea),LL(0x726e147e,0xd5e175b1),LL(0xca2a0207,0x833e5911),LL(0x4d322cae,0x6cba51a1),L_(0x00000093), LL(0x4e7b937f,0xdaa1d653),LL(0xe0948dd2,0xca2b565c),LL(0x2f88e2ac,0x94ccb3a4),LL(0x85d91e6c,0xd8fe4775),LL(0xb59cebc7,0x0807f46d),LL(0xf4237821,0xfc9eb940),LL(0x402ecff0,0xd173f94e),LL(0xbf7bc598,0xaf975145),L_(0x00000116), + LL(0x75e9fe79,0x15942a4c),LL(0x3dad29e1,0x350dec67),LL(0xe6be55a2,0xca3c399f),LL(0x71245659,0x87e22652),LL(0x8f51c63c,0x1bd4c445),LL(0x758ae1a1,0x319b57db),LL(0x547db810,0x7d5c89dc),LL(0x62c8ba84,0x959a5bbe),L_(0x00000039), LL(0x3876f024,0x8c3d490f),LL(0x482b690e,0x50d48521),LL(0x83aada08,0x82c13331),LL(0x45c4a535,0xae5a3425),LL(0x31c1467b,0x51c50d6f),LL(0x2d093b81,0x84a7d97c),LL(0x82d6fbdb,0xdb41ffbe),LL(0x4953468e,0xae0e9fad),L_(0x000001e6), + LL(0xd4f44715,0xc26f9e1d),LL(0xb8562be5,0x71fd7b5e),LL(0x039f5b0d,0xe3196121),LL(0x073e4db0,0x902cc367),LL(0x22f8b999,0x604d9b78),LL(0xc1cfc4a4,0x52f26ece),LL(0x88d45487,0xa175f394),LL(0x9f16c268,0x9fe9a65c),L_(0x000000ea), LL(0x3d6adc59,0x54ed1ed1),LL(0x4d516f86,0xb3a46011),LL(0xa9ca3304,0x8c5e216d),LL(0x2030c9a9,0x86a5904f),LL(0xc0c4c573,0x9467bf24),LL(0xb4a6fdae,0xa652162c),LL(0xd3472536,0xb6166589),LL(0x31361b4b,0xfa3bddac),L_(0x000001b7), + LL(0x6c223a73,0xaa60773f),LL(0xe8255739,0x58459ee1),LL(0x672547b1,0x3f4e65fb),LL(0x0030639d,0x5059b89e),LL(0xc29a4f63,0x3ffb7b9a),LL(0x267e1823,0x6854cad3),LL(0x7f7d6bbb,0x79c3b99a),LL(0x20d91b6e,0xbec22c47),L_(0x0000005e), LL(0x6be80718,0xd72fbb0f),LL(0x307d9b73,0x7f8c46dc),LL(0xc6fa2f52,0xfbf3d06f),LL(0xd62e537c,0xd7888122),LL(0x4ebd3818,0x11ea61a8),LL(0x479a6f83,0xeafa5532),LL(0xa4bf3325,0xc7a31554),LL(0x5a717809,0x9520a809),L_(0x00000039), + LL(0x320654ac,0x4f775892),LL(0xb974b129,0x5ce12c3e),LL(0xe8064395,0x2e69ed62),LL(0x31e3ae39,0x9c220c3b),LL(0xd59ea4b1,0xb79732ea),LL(0x6f45918a,0x6600e529),LL(0x08681af2,0x4ed3f5dd),LL(0x507438a2,0xac5ccefd),L_(0x0000006e), LL(0xf3fc209f,0x5b14db7c),LL(0x2484d059,0x411826aa),LL(0x6005d933,0xca8c9d7f),LL(0x15dd0c44,0x1900212d),LL(0x96825f72,0xa3ea7e14),LL(0x921c8c87,0x20d2b6f4),LL(0xfdd63f04,0xbe8c25ac),LL(0x9f6a6126,0xa2592316),L_(0x00000128), + LL(0x8f2ae670,0x27b80789),LL(0x4f60fc89,0xc97bbd7e),LL(0xbd2ecac1,0x5b008578),LL(0xaaf75b36,0x3b54d53e),LL(0x6540b6f5,0xce5a0ab8),LL(0xac54c1d9,0x52d909c6),LL(0x67e6b65d,0x6607ea02),LL(0x7ea06112,0x0a94e0f9),L_(0x0000011c), LL(0x077ac647,0x7fdd52c7),LL(0x49bb24e9,0xfc9b8e68),LL(0xe60101c7,0xdcbe3ce9),LL(0xc2ae52f0,0x3c779930),LL(0x85f01602,0xa50df581),LL(0x4a7c2cfe,0x8c89a686),LL(0x52f720dc,0xe2d3b3fd),LL(0x566aecf1,0x1ac6cabd),L_(0x000001e5), + LL(0x9e01d468,0x1a616c59),LL(0x8c145c49,0x1dbffc3d),LL(0x2990c86d,0x842bad94),LL(0x7914907c,0xb0b65826),LL(0x8753b549,0x09f19586),LL(0x29d809d0,0x3f813f8e),LL(0x3e155797,0x293bf471),LL(0x87d08f2b,0xc41c49e5),L_(0x000000a9), LL(0x8769b86f,0xd9473ed1),LL(0x7fc8f692,0x7012e2ef),LL(0x5014f2ae,0xe7047394),LL(0x086b300a,0xb63d8e90),LL(0xe77f5c2e,0xf739fb33),LL(0x700eca1b,0x22cfcea2),LL(0x8bf8652d,0x4bb3d357),LL(0xe61f7839,0xba0c3888),L_(0x00000189), + LL(0x1f417baa,0xe42f4734),LL(0xbb740324,0x9909bcd8),LL(0xe45bfed8,0xe9550616),LL(0x77889a7f,0x861f9828),LL(0x446a53b7,0xc8946a71),LL(0xfbe6cdbf,0x72916502),LL(0x70373dcf,0x59c8ba5f),LL(0x1ab59a0e,0x07606ee3),L_(0x000000ed), LL(0x7cf7e716,0xfa3f237a),LL(0xb5ddd50e,0x0f211f61),LL(0x56f3a63a,0x8319e664),LL(0x3124dae6,0xe5acc6a3),LL(0xf2c7d090,0x5d6c5fe3),LL(0x5cc8df62,0xa0165f43),LL(0x20f4229f,0x5adf8ff4),LL(0x2e9c92c9,0xd951affc),L_(0x00000003), + LL(0x28c3778e,0x85b95034),LL(0xdf5cb44e,0xffda53ce),LL(0x976139f3,0x126f8dfa),LL(0xe003499a,0x25a0d493),LL(0xb521d994,0x43bb5822),LL(0x4978624a,0x1b4bdd9b),LL(0xf19f8465,0xb8e6e89b),LL(0x51953414,0x3b559f26),L_(0x00000070), LL(0xb01bfe66,0x67904448),LL(0x23ee72a6,0x7f3ba24c),LL(0xefa5df96,0x4a910e6e),LL(0x2d36ae87,0xbee250f2),LL(0xa3310ea5,0xbebd708e),LL(0x5db14894,0x94f849b9),LL(0x7b2cb1b7,0x830487c0),LL(0xd8532e27,0x72d6bdee),L_(0x000001cc), + LL(0x1a632371,0x098e856f),LL(0xcfb4864e,0x2a48dc38),LL(0xa9b6c8ef,0x070c9954),LL(0xc3e565ee,0x47d25c31),LL(0xeec5d7b2,0xdd383653),LL(0x5ac0ce45,0x1f5f3381),LL(0xd40035f3,0x486c7281),LL(0xb516fd77,0x5bbe546e),L_(0x00000014), LL(0x9535f589,0x0df8a00f),LL(0x3f432f2f,0x740ff679),LL(0xa22b5ed0,0xff907935),LL(0x3d446e71,0xd013a668),LL(0xeeee1f6e,0x16c4dc5f),LL(0x37116783,0x8ed6e49d),LL(0x07fabe84,0xd710493c),LL(0x29dbccd8,0x9263e99e),L_(0x00000193), + LL(0x1f79b2df,0x906df6cd),LL(0x835e52c6,0x8a2f7445),LL(0x66d4156a,0xb891e801),LL(0xaa5c0324,0x852e9f5b),LL(0xd3f81a7a,0xce23b51a),LL(0xcdc49377,0xe9c8bbe9),LL(0x7da33db2,0xd5ae561f),LL(0x90aafd3d,0x2f82d67e),L_(0x00000046), LL(0x0d755e30,0xd53059fe),LL(0x43cd9246,0xca05f22d),LL(0x0c50a7ad,0x23745904),LL(0x612405ce,0x1e2b644e),LL(0x1783439c,0xbb5598f4),LL(0x8299ab36,0xfdba24f0),LL(0x36c6a428,0x085bc781),LL(0x42e84aa9,0xe954acbf),L_(0x00000123), + LL(0xb7caf2a5,0xeb9123d0),LL(0x9813e058,0x170c410c),LL(0xb14ecd8d,0x43a445ef),LL(0x12caafb5,0xb13251ba),LL(0x692c7666,0x6a58ee44),LL(0x93734f73,0x1ecf9f41),LL(0x0ff39b6d,0x582e2c3a),LL(0xdd49d16c,0xbc874e11),L_(0x00000054), LL(0x2f1debe5,0x9986c142),LL(0xbe161bed,0xadb85be6),LL(0x94c11992,0x974392b1),LL(0x4727d02a,0x1355dd0b),LL(0x9ef84fdc,0x001779ff),LL(0xdbbcd1c3,0x09218dc1),LL(0x615360f7,0xce2bff57),LL(0xa1cd90ce,0x5b2b4772),L_(0x00000008), + LL(0x3b91b808,0x797035ff),LL(0x63db3368,0x2ea4d5db),LL(0xd58aac93,0x5c1f4e24),LL(0xf46d30bc,0x87b8b188),LL(0xfe246192,0x177c783a),LL(0x2b7ec253,0x7157d89d),LL(0x93dd18f3,0x609635a4),LL(0xebb06e2d,0x9bb84085),L_(0x00000182), LL(0x7977b4f8,0x5291bba2),LL(0x68aae159,0xd5d8d4a8),LL(0xc3c56f74,0x8f01b0f8),LL(0xd20ce087,0x0018bf4f),LL(0xcf62fd4a,0x58c66e12),LL(0xe583beb9,0xcba992f3),LL(0x7c22cd07,0x82ea861b),LL(0xffb459bf,0x757b22f8),L_(0x0000010c), + LL(0xc9b1cbb9,0xf9bac8ff),LL(0xbeb1c7a5,0x70f6e41f),LL(0x32bb744d,0x1f50a7aa),LL(0x2a66377a,0x611c61b4),LL(0x324a6008,0x1ce936f3),LL(0xc3dc201c,0xc5bca084),LL(0xb70db700,0x47988c82),LL(0xfef0caf3,0x2dfe3675),L_(0x0000005e), LL(0xe9f94248,0x76fee427),LL(0x165d8d1c,0x937f269a),LL(0xdc1fbe1e,0xdfee35cc),LL(0x745db180,0x5475d4c9),LL(0xefc6cdb5,0xb36d0a2f),LL(0x26ccd4e8,0xf7ab4767),LL(0x3783e580,0x31ff3c1f),LL(0xce5d079b,0xed18ab19),L_(0x0000009c), + LL(0x0e0b8581,0x231ac4a1),LL(0x0790eab4,0xec50bbe6),LL(0xde5da610,0xc5c0bffc),LL(0xc2382715,0xd201e1fa),LL(0xaac50e22,0x27c0dbe5),LL(0xd06ae9b5,0x8715ddc4),LL(0x77f1683f,0x592ddcce),LL(0x410624c0,0xffb7a12c),L_(0x00000167), LL(0x73c64862,0xf76139b5),LL(0xf8b9348f,0x29844cbe),LL(0x30deae53,0x941fdd94),LL(0x5f88797a,0xa3612765),LL(0xbbb4a54a,0x8e2ccc8e),LL(0x4a469e09,0x47a174f2),LL(0xe17602e5,0xc6f2b50b),LL(0x5edc6700,0x887e7ea0),L_(0x000000bb), + LL(0xffa11839,0x350806a0),LL(0x8c8ad937,0xbc207b36),LL(0x45322006,0xf3feb2a7),LL(0x29b356ce,0x9772041b),LL(0x4db4360d,0x71532653),LL(0xffd40033,0x62b4c3a4),LL(0x525bb0cb,0xef8cba94),LL(0xb23dfc1a,0x5f95e7e7),L_(0x0000003c), LL(0xaafe7a77,0x3a9f0d5f),LL(0x05623688,0xe7ca479f),LL(0xe0f2e02f,0x821e669e),LL(0xd9320a75,0x3208dc12),LL(0x2a90f1fc,0xac95bd4d),LL(0xb836d5d0,0x0f15127a),LL(0xa56f1f5b,0x35a8c806),LL(0x4b53533b,0xb5d6ee48),L_(0x0000001d), +}, +/* digit=80 base_pwr=2^400 */ +{ + LL(0x321cc486,0xb1b291af),LL(0x63480c29,0x6afae0d2),LL(0xec906027,0xd90afac7),LL(0x067489d0,0x62d3da37),LL(0xe31b78ec,0xd88f38b3),LL(0xff5fafe7,0x3dfa7f35),LL(0x88536101,0x7a6237d7),LL(0x80270f89,0x42a2eac9),L_(0x0000002a), LL(0x69198d03,0x95b6527c),LL(0x38b21960,0xab3e28c0),LL(0x56c8573a,0x49dbf002),LL(0xb06c2993,0x016fc238),LL(0xc550c59c,0x7c26c63d),LL(0x22cb4395,0xd6c87128),LL(0x4a5765b7,0x42f34ea4),LL(0x360be87f,0x9436ed50),L_(0x0000001a), + LL(0x92004107,0xcf6c0f3c),LL(0x23b6d0c7,0x01323c15),LL(0xb2a524b7,0x445c4f05),LL(0xfb280e1d,0xb721bd24),LL(0x43a450cf,0x34d74f50),LL(0x459a3690,0x9ed3c3ca),LL(0xf8a99776,0x9bd35cd1),LL(0xaf456934,0xa1b94559),L_(0x000001a7), LL(0xda50d868,0xc7479e9f),LL(0x93e5dd5a,0xfdaf1391),LL(0xfcfc382f,0x727251e2),LL(0x4776e937,0x9f976e0e),LL(0xfaf93681,0x7e0dea37),LL(0x18ec38c9,0x45662b32),LL(0x0308bb26,0xd581e3f2),LL(0x1a441534,0xc275dc07),L_(0x00000046), + LL(0xe20ed2c8,0x9bcb5968),LL(0x702c5bb8,0xdfb47a52),LL(0x33f897ad,0xe16c51a7),LL(0x078f030c,0x5e8bc092),LL(0xb9a4c194,0xa0a224a9),LL(0x0d2a2dbe,0x244c74fb),LL(0x00b01506,0xef3b3eda),LL(0xf4403180,0x44f09c72),L_(0x000000d5), LL(0xc261b7e8,0x4a7f0289),LL(0x3c0211ff,0x88323d80),LL(0x1ffe93b1,0x81a127a4),LL(0x7deb5031,0x0bd65111),LL(0x65ffd296,0xff238b15),LL(0x80cef133,0x643d7062),LL(0xddb33d18,0x93ccc6e4),LL(0xc957a463,0x1fcc4678),L_(0x0000011d), + LL(0x7eb6c9cf,0x1ccb1806),LL(0xe0e23232,0x5eae4904),LL(0xc2c362eb,0xda675b34),LL(0x2c14f20a,0x13d2fa91),LL(0x60f4ae95,0x6c8c7ff9),LL(0x78df2064,0x6790ea32),LL(0xb702cc14,0x7608da34),LL(0xe2b87bb5,0xadae0fb9),L_(0x0000011f), LL(0x0ca7a84d,0xa26a843b),LL(0xc89f77d5,0x5a368ffa),LL(0x265d14c3,0x957c89a9),LL(0xbd1486e7,0x514b7e05),LL(0xa9030ef9,0x537cf3d5),LL(0xb9ea3998,0x4fb32008),LL(0x0c45cfba,0x61ff9565),LL(0x078d5a15,0x1cc6a564),L_(0x000001c0), + LL(0xa9ac7ecf,0x665bb52e),LL(0xf7ce0ec3,0x69d2fe28),LL(0x4c059fd5,0x76b354dc),LL(0x1290e892,0x674b639e),LL(0x6d828313,0x951c9220),LL(0xd6285250,0xee815fd8),LL(0x3a0ba16e,0x8ee38518),LL(0x5323ff40,0x6678fced),L_(0x00000073), LL(0xba67d240,0x73a8b28d),LL(0xba84ecb6,0xbd048216),LL(0x34998afb,0xfb264967),LL(0xfcd4e06c,0xc024c958),LL(0xd668c764,0x9c3e07fd),LL(0xee500455,0x0ea9e902),LL(0x7be48424,0xdf78504c),LL(0xb185d1cb,0xc315ffe9),L_(0x0000001f), + LL(0xd554604f,0xdb3f67e6),LL(0xdbdf25d6,0x23503a7e),LL(0x4a86faf8,0x58cbd82a),LL(0x727561d4,0x50ad1fb2),LL(0x0994b8fd,0x1d57e2f0),LL(0x1b2b4725,0x15736a57),LL(0x5dcba3cd,0x4df58192),LL(0xec335163,0xf716e579),L_(0x00000190), LL(0xfd15e62f,0x3615c741),LL(0xe4509eb4,0xef754782),LL(0xbb1dede8,0x7a793f6d),LL(0xa02e32d8,0x27d972fe),LL(0x00b65ecb,0xd0af4ace),LL(0x80ede0c9,0x96816659),LL(0x6c809dcb,0x979a653f),LL(0x943f6f1f,0x0638c8e6),L_(0x000001a7), + LL(0x69a82f95,0xc3695735),LL(0x79907894,0x6a94274c),LL(0x923a4f54,0xa698895f),LL(0x6213c148,0x57afe3d1),LL(0x14eca3c3,0x40597be1),LL(0x57638ac1,0x23258bde),LL(0xb1f30c4d,0xb9d09cea),LL(0xe2c1e648,0x544e3974),L_(0x0000015b), LL(0x2d296b84,0x28a45f39),LL(0x0111ad40,0xbe28f874),LL(0xc3e262fd,0x830a9ee4),LL(0xe3cc3453,0x0fdaa4f4),LL(0x044defae,0xcdb8b9c8),LL(0x06665f64,0x4a06827b),LL(0xb0bfa5e9,0x926f3364),LL(0xaf288ab3,0xd9d3c3ec),L_(0x0000015c), + LL(0xf210670e,0x5eb181f2),LL(0x047db30c,0xbb73f9ac),LL(0xf0b9977a,0xc5355db3),LL(0xa26685fe,0xa356655d),LL(0x48c1a2cb,0x950bb7fd),LL(0xc2c2dc45,0xe766094a),LL(0x437ff8a8,0x80146ca2),LL(0x0b26425d,0x5b7aafe5),L_(0x00000121), LL(0x7260e44b,0x1c8a54b7),LL(0x683c3c46,0xee244200),LL(0x806c758f,0xb90af5d8),LL(0x71dee02f,0x67ac6f65),LL(0x53aeab51,0x0f1dfed2),LL(0x762d7338,0x59bf51fc),LL(0x059d5565,0x1432973f),LL(0xa9143049,0xbf133e93),L_(0x0000017e), + LL(0x355d9c01,0xa86b08e9),LL(0x4f81ab77,0x4d06707c),LL(0x8f1bf925,0xaaec66fa),LL(0x4de91bfe,0x49683907),LL(0xb9d75b89,0xac105bac),LL(0x91f21593,0x6506b77d),LL(0xab4390d7,0x7287e5ee),LL(0x7487ed45,0x6c88b2c6),L_(0x000001c1), LL(0xd0ceccec,0x11c071b8),LL(0x97c81ddb,0x1bbbfe37),LL(0xd3ad9501,0x7a1b3885),LL(0x68eaad09,0x7027528c),LL(0xf0a64aef,0xcf860284),LL(0x3c25fb62,0x37b3f120),LL(0x5ae492f1,0xd454de89),LL(0x61c7d5ed,0x02ac32db),L_(0x000001a3), + LL(0x3afa7fde,0x245c4af0),LL(0x97406073,0x02c143ac),LL(0xda94e70d,0x1c3ce7f2),LL(0x639942c8,0x9cde176b),LL(0x7bd79b38,0x0883b19a),LL(0xac9d2c91,0xb4709f2a),LL(0x4423fc39,0x794a6aaa),LL(0x89d178db,0x2e0aecc8),L_(0x000000c4), LL(0xd62ec51a,0xb872cf73),LL(0xd957093e,0x85b4776d),LL(0x2acbd72e,0x0656f9a4),LL(0xd468fb6e,0xe2017205),LL(0xc79f3d11,0xe6a367a5),LL(0x012b1362,0x73088cac),LL(0x370e3947,0xb86f1933),LL(0x3bdf96c5,0xe8162ca6),L_(0x000000dd), + LL(0x432ab140,0x9b09e656),LL(0xeb6d1849,0x81aa411a),LL(0xaae86725,0xd8ee02c2),LL(0x3008becf,0x574081d1),LL(0x53cb17df,0x37bd5637),LL(0xe3afcfbe,0x116f90cd),LL(0xe91cbaad,0x9a6da94d),LL(0x023933b7,0xbabdb470),L_(0x0000008f), LL(0x7351bde3,0xf3d722e1),LL(0xef4074df,0x2345ac8e),LL(0x1492cda0,0xe92886d5),LL(0x0fa0e7ed,0x31537a63),LL(0x0778badc,0xa8b49a51),LL(0x2034669c,0x85153992),LL(0xace88cb9,0xfd679e52),LL(0x4c6fbbf5,0x48dc5511),L_(0x00000167), + LL(0xe07de72e,0xe7ef2e60),LL(0xcd7037d9,0x52aa0cf6),LL(0x7dc7381f,0x24da2a63),LL(0xc3562613,0x43cb3857),LL(0xfa06da1c,0x5a1b2f21),LL(0xc7806190,0xec8d6d1e),LL(0xf2d57ff1,0x5a0e8065),LL(0x3e9088a1,0xc30f4ccc),L_(0x0000005d), LL(0x5dde7f70,0x5912141f),LL(0x1bc58b8c,0x19ee91c4),LL(0xbf2996d5,0x41d465ca),LL(0xb39d820d,0x281052af),LL(0x890d83a9,0xa8cb9903),LL(0x32dbf7a5,0xbfad3855),LL(0x309c32fb,0x81d1982b),LL(0x44dec521,0xe04b5e8d),L_(0x000000cb), + LL(0xb2b331bb,0xf141f663),LL(0x6c39f9cb,0x458b556c),LL(0xea205dcc,0xb6c66d04),LL(0x96d440e6,0xa0f1a8e6),LL(0xd4e4d37e,0x3d5c97ee),LL(0xd69c9913,0x92914c77),LL(0x12d04cdd,0xb8bb2a33),LL(0xa42c1185,0x2dee2451),L_(0x00000084), LL(0x8f1a5bb3,0xe8feb727),LL(0xf2ee4afa,0xb371de96),LL(0x0f971611,0x605c486d),LL(0x50e27936,0x313e3113),LL(0xdf801ab6,0x7cfe0fbb),LL(0xfbec3f73,0x50864708),LL(0x55057cd2,0x3b426634),LL(0xa56700f5,0xe23875b8),L_(0x000000de), + LL(0xf2c197f3,0x923016ac),LL(0x17eaa1a0,0x28e3f935),LL(0x128b1c02,0xa23b14eb),LL(0x9183c2c0,0x504289c9),LL(0x5ea313d0,0xb5fd0ea2),LL(0x135c854c,0x03f5e8df),LL(0x491bfca7,0xd9abdd7a),LL(0x8b6af733,0x29d5f4e7),L_(0x0000014e), LL(0x535e0af4,0x50113295),LL(0xb89fd770,0x861c50a9),LL(0x6fd08a16,0x1b7e4c60),LL(0xb1a9800a,0xadc9902a),LL(0xbed31f30,0xa5b9d65d),LL(0xc9a0a6d4,0x75c10264),LL(0xe1743115,0xc317e935),LL(0x8c13233d,0xc0c350f5),L_(0x000000d2), + LL(0xbd31c0b0,0xfdaa94e2),LL(0x4ea46af8,0x2200fec9),LL(0x3139256c,0x4f2fd88a),LL(0xb9b83d67,0xe894a212),LL(0x5e6f7bed,0x449c5bc8),LL(0xbeebbe0e,0x1e599f77),LL(0x74fc58a0,0x7f4a9123),LL(0x6073c6cf,0x1a08d5bf),L_(0x00000113), LL(0x2d5f04f1,0xacb45b52),LL(0x06826375,0xbd9c13b3),LL(0xf41e6cac,0xc5b15016),LL(0x79237672,0x2f2bfd3b),LL(0xa99e108a,0x1681dcfd),LL(0x22fd3033,0x224f0276),LL(0xc24a2525,0x05e14660),LL(0xf9a0ff9c,0x09f9360e),L_(0x000001fb), + LL(0x442343fb,0x1e2ff226),LL(0x33f17650,0x597f6cd8),LL(0x95759918,0x5f1a7fb7),LL(0xb518de41,0x316d41dc),LL(0x966644e4,0x829d89c6),LL(0x7775eaca,0xed7a513d),LL(0x9d45da4e,0xdd14ef8a),LL(0x232852aa,0xc016fff6),L_(0x0000002e), LL(0x7b230dd6,0x0791f95c),LL(0x2cc46d0d,0xd4641a73),LL(0x04775f23,0xb9dcdfdd),LL(0xc9bd9d15,0x727ace99),LL(0xaec6f67a,0xa77fe3e5),LL(0xeac1ee2f,0x75b923be),LL(0x8f21d632,0x55852cf0),LL(0xb6658853,0xbe6d4550),L_(0x000000e1), +}, +/* digit=81 base_pwr=2^405 */ +{ + LL(0x51c61dc7,0x2c6e3212),LL(0x0a1fff24,0x4b5a6256),LL(0xa45d5589,0xbc1ece0d),LL(0x852bc8f1,0x655945af),LL(0xf4152e99,0xf81d51a6),LL(0x4573b7d9,0x15c74818),LL(0x69e42e80,0x69dba53f),LL(0x55b2c206,0x96245123),L_(0x0000009f), LL(0x553b866e,0x11c96019),LL(0x8a5146d0,0xc8d9fabe),LL(0x6c8e83c7,0x3fede45c),LL(0x3d5d33f9,0xb87d2fad),LL(0xe1fe306f,0x67a48456),LL(0xf030c243,0x2aa7e6e9),LL(0xb8f59e2c,0x8097392a),LL(0x7ae2ecee,0xdbed7e8f),L_(0x000000e7), + LL(0x1e3892c1,0x8efeaf2d),LL(0x8c7ed96e,0xf3e2fab6),LL(0x3cb959b9,0xfe65989c),LL(0x4cec2e34,0xb397dfd2),LL(0x2a821089,0x95a4f7a7),LL(0x194fcfc2,0xcd183d4d),LL(0x009eac36,0xc2005f34),LL(0x384df54a,0xc4355ce3),L_(0x0000017a), LL(0x52880022,0x6218f15a),LL(0xab158f0c,0xc9db684a),LL(0x919d3c1c,0x22157c5c),LL(0x733c654f,0xa5d7e7c5),LL(0x1bb67f61,0x6dc89cd4),LL(0x0cac1f78,0xf6e74669),LL(0x2b55f183,0xb445fa4f),LL(0x9df41e4c,0x69c4dd42),L_(0x000001a9), + LL(0xcf794718,0xd8d9bdcb),LL(0x3dd4ca53,0x1306c74d),LL(0x1af7d8d5,0x3e680d58),LL(0x9c6b82f0,0x884ca0be),LL(0x0aacdba7,0x5c62e372),LL(0xd633f595,0x1c4cad9a),LL(0xc84d067a,0x54e3c550),LL(0x4fe24eee,0xbe3f67b5),L_(0x0000019d), LL(0xc026b9ed,0xb999b839),LL(0xd75cb7b6,0xa5275bc7),LL(0x5e6b4aa8,0xfaa9f40a),LL(0xe6b156cd,0x1992d1c2),LL(0x16e51f4a,0x0b180928),LL(0x00c94afd,0x6b3427a9),LL(0x0f9d0fb1,0x09eefa51),LL(0x098f98b4,0xd3cae463),L_(0x000000ae), + LL(0xd17fe65d,0x881adc31),LL(0xbb3a93b6,0x8ce1cba5),LL(0xb603dd9e,0x4f5b70c1),LL(0xaed8b0cf,0x5f958dd3),LL(0x5eae2517,0xb70f44e1),LL(0xa5b942f5,0xc526177a),LL(0x02efb949,0xc8dd1153),LL(0x132ba3a2,0xf9288a95),L_(0x00000189), LL(0xede20db4,0xc45972eb),LL(0xbc841aed,0x7853d5b1),LL(0x933a99b8,0x6e1536aa),LL(0x85259727,0x238abf3c),LL(0x05488fa0,0x8485ab11),LL(0x1debe07d,0x6d6f6d52),LL(0xf1ad18f1,0x54637f92),LL(0xdfd3c55a,0xa2b58773),L_(0x00000004), + LL(0xed1c0bc7,0x66d98564),LL(0x72366f09,0x9a3f0f97),LL(0x00008259,0xb1a9b87a),LL(0x33f3b0e5,0xe8074b36),LL(0xd83471a9,0x68f935c3),LL(0x59dc097c,0x5ab59d2f),LL(0x049d3329,0xae3c2a44),LL(0x523ad362,0xd39de2e5),L_(0x000000b0), LL(0x55bdbbc5,0x10a229a9),LL(0xad11b358,0xdcf6cbc9),LL(0x8a7d993d,0x2d5c5b91),LL(0x31b67dc1,0xd2d684f6),LL(0x5dd81c8d,0x29c17938),LL(0xec292f8a,0x2fb94c2a),LL(0xe9c267eb,0x67c899b3),LL(0x31e831bd,0xd72dd6a7),L_(0x00000058), + LL(0x310d60d9,0x37a99dda),LL(0xabc73772,0xb8a9bdaf),LL(0x18907ad9,0xac790211),LL(0xd35c8ab3,0x56550490),LL(0x483d71b5,0x9c473d52),LL(0xaed32863,0x796ddfe4),LL(0xc175ce1c,0x39329661),LL(0xf0af8692,0x3411279a),L_(0x000001f3), LL(0x085548fc,0x2f981ba2),LL(0xe7ed779e,0x25706bd9),LL(0x385062b4,0x8826d6b1),LL(0x50749b03,0x8f92597e),LL(0x4d1f3b1e,0xcc7ec8dc),LL(0x5ffacc7e,0xc4c11580),LL(0x903de537,0x82ed5c34),LL(0x92e3ccb2,0x829a6dc1),L_(0x00000197), + LL(0xc22b2da3,0x8a8271c3),LL(0x46e669a1,0x9bedd70e),LL(0x6dbeb99e,0xe5038aaf),LL(0xc8d58c8e,0xe202e790),LL(0x312f9e8d,0x5eb0a99d),LL(0x2b3b3990,0x6e033ac6),LL(0x8ddb53b5,0xdd9938af),LL(0xcad94c88,0x8fee9f14),L_(0x00000103), LL(0xe8c634af,0x2d18c7f7),LL(0xf6c9ab51,0xddec3950),LL(0xd2e14fab,0x8eb24aaa),LL(0x85f6d87b,0xbff04dfa),LL(0x7d46acbb,0x458d7c7a),LL(0x949d94f0,0x5596e98b),LL(0x60ee6372,0x37137ad5),LL(0x20231d27,0xe0d4f3a2),L_(0x0000006d), + LL(0x95b0fc9d,0x1f851427),LL(0x75a8973a,0xf65e532b),LL(0x125d27c2,0xb7e6ca7e),LL(0x680245ef,0xd37a1c1d),LL(0xc0ac3fa3,0xc8ed8871),LL(0x73ed1f61,0x25a3f922),LL(0x1c0619b1,0xbaaf99bf),LL(0x2fc151e1,0xb9c92ca1),L_(0x0000015f), LL(0x02c28006,0x4cd45f13),LL(0x46eca65c,0x2f16b28a),LL(0x181d940c,0xdae561c3),LL(0xdef08156,0xdb51b5db),LL(0x5aff9fff,0xd7d0f3f0),LL(0x56731470,0x9f642167),LL(0x3e4323b0,0xc5c736fd),LL(0xc2c256f1,0xaf757eba),L_(0x00000046), + LL(0xe3fa203a,0xfc825ff9),LL(0xa28756a7,0x6fc9ac38),LL(0xf07539fe,0x19a8908a),LL(0x89d74956,0xc69dedbb),LL(0x934712d9,0x2aaee4e8),LL(0xbbd47741,0x1053c866),LL(0xda8058c4,0x00b68d70),LL(0xc019bbaf,0xb1236281),L_(0x000000ad), LL(0x7020f123,0xad679598),LL(0xffe5c58c,0x5fe8d191),LL(0xfb4d9415,0x46edaa9c),LL(0x44747329,0xdd0f1cc6),LL(0x34e406e5,0x79f7cddc),LL(0x40ad1213,0xab39e94c),LL(0x470ac094,0x885cb3a4),LL(0x12891647,0x3224c564),L_(0x00000150), + LL(0x5baed419,0x17473053),LL(0x116c3934,0x97765c7e),LL(0x74d1a056,0xf7e7734d),LL(0x2fb92919,0x22455583),LL(0x880ac302,0xc6198a57),LL(0xc983ea10,0x32d7f501),LL(0x3adab6b9,0xa4c1c306),LL(0x3997a013,0x14822e84),L_(0x000001b6), LL(0x1a3dfcbb,0xf8efc5c2),LL(0x2e766181,0xda380c24),LL(0x158c4baa,0x4d96447c),LL(0x6acaba32,0xe953e90e),LL(0x4ed2e3bf,0x86ffae71),LL(0x452c6d1b,0x3af83523),LL(0x49a52fe9,0x41a86c00),LL(0xec5b4f72,0x9c65d29c),L_(0x0000011d), + LL(0x3e3efd7e,0x15231af9),LL(0x83eb4905,0x29b6e8f6),LL(0x35420b50,0x76561721),LL(0xd8dddbf9,0x83f7c4f4),LL(0x776812d9,0x460a8666),LL(0xd3c1656b,0x6901dcbc),LL(0x22e1e397,0xd9c17a67),LL(0xd83adc99,0x7a32d3cd),L_(0x000000d0), LL(0xe72700dc,0x66102687),LL(0x8578e51e,0x957c6151),LL(0x9a86b387,0xb2bd85fe),LL(0x553c599b,0x21884750),LL(0xb8b8c27a,0x920c65b6),LL(0xf81924dd,0x6f14a6d0),LL(0x28619568,0x18db08c4),LL(0xab2d8a3f,0x11b85385),L_(0x0000015a), + LL(0x880a5659,0x5aa706e4),LL(0x0e7a9fe5,0x2110fcf2),LL(0xa8b2d6f6,0x67bdfcf9),LL(0xeaac5dca,0x0435935d),LL(0x39631926,0x399fbc31),LL(0x60795bd1,0x34625175),LL(0x0f561153,0xced09fb3),LL(0xe23ff49c,0xf1b45cd7),L_(0x00000126), LL(0xb5ed6363,0x4ec88551),LL(0xd20c517f,0x3c8be0dc),LL(0x221ae5ee,0xd895d43a),LL(0x6705afa1,0x00fa5270),LL(0xb373ab4d,0x75678bdf),LL(0x64e2886b,0x47650c25),LL(0x7e0e12f4,0xab5c87d2),LL(0x0c9aec40,0x15ef4537),L_(0x000000e3), + LL(0xfd1186f9,0x69bd1ae9),LL(0xe2ef5aee,0x0585cc2f),LL(0x1e6188c6,0xa3c01465),LL(0x301c46a3,0x1cc1ea41),LL(0xf7f76048,0x16944109),LL(0xedd90482,0xdc473809),LL(0x3da1ef77,0xf7267c80),LL(0xfefdbcd3,0xfd92f40f),L_(0x00000037), LL(0xbd7e42e8,0x365d3aae),LL(0x5f29db47,0x93437f82),LL(0xd2267583,0xf4a57394),LL(0xe0388ea5,0xb28bca60),LL(0x5cea0f68,0xbf640edf),LL(0xc68dbd2f,0x8db4f9b6),LL(0x24be537f,0x9d943d05),LL(0x7704cfe6,0x4a1f249a),L_(0x0000003e), + LL(0xca968ae2,0x15eb727b),LL(0x79b64a1c,0x82dd22e4),LL(0x51a73cb4,0xa1e0df42),LL(0x5140d8ce,0xf6d38530),LL(0xe8523991,0xbf2d199e),LL(0x578593ff,0xaabd945b),LL(0xcfef51c1,0x789ae01d),LL(0x9c35735e,0x685cad78),L_(0x0000015a), LL(0xa7f90d48,0x9ac2d994),LL(0x5bf21520,0x5119208b),LL(0x3671fecc,0x477379bf),LL(0xcfbe3b2b,0xdc4fe9f8),LL(0x98a8aee0,0x6e915903),LL(0xa7edc740,0xc4cb91a1),LL(0x71211193,0x1e307364),LL(0x96026ba1,0x814b8b56),L_(0x000000b1), + LL(0x6b636a3e,0xdfef19d0),LL(0x30a583d8,0x8521445c),LL(0x192a3c38,0x8651795c),LL(0xe24d5be6,0x6cfd9cef),LL(0x273daff4,0x78499d8a),LL(0xbe3d67fc,0xf4d9c05a),LL(0xfff37ca4,0x9d03588e),LL(0xbe977f9e,0x2b3a2963),L_(0x00000043), LL(0x5504ebb6,0x6c46a157),LL(0x146210b3,0x092ba668),LL(0x00fc64e7,0xcfe9730b),LL(0xc41f8e66,0xc11dab42),LL(0x79417f9d,0xff89645e),LL(0xddf57c66,0xd0e35f15),LL(0x98273f3c,0x49f211d9),LL(0xea35684c,0x1b8dcf07),L_(0x000000a4), + LL(0xa5cf0865,0x9d8b99d3),LL(0x7dad9f18,0x96f2ea6f),LL(0x96139562,0xf5d410b4),LL(0x86c29eba,0xc6b1f46d),LL(0x2dbba6d4,0xb1709ad9),LL(0x9de07504,0xeea80cc0),LL(0xc7c9ec95,0x47d01eb6),LL(0x99076486,0x19b1d6cd),L_(0x00000139), LL(0xe61ba7e9,0x0a5f9f34),LL(0x53cc24a0,0x5e367eff),LL(0x672781ea,0x275cfce0),LL(0xedc5266f,0x92d98139),LL(0x0d9e2099,0xc2c0efd5),LL(0xf3d9cb26,0x687bde18),LL(0x647d23fe,0xd97b9ccb),LL(0x0c54a71a,0x258eaff2),L_(0x0000007a), +}, +/* digit=82 base_pwr=2^410 */ +{ + LL(0x99ea3941,0x30cbad8f),LL(0xfa23022a,0x0f3c6cf3),LL(0x9f3a186b,0x33420e3c),LL(0xdc6e922b,0xbffbbdb1),LL(0xeac227c1,0x7aa59cde),LL(0x43d5b878,0xe3673dd9),LL(0x513a5be5,0xae77a5b5),LL(0xbc4c0fef,0x8e4c10fc),L_(0x00000151), LL(0x28229a86,0x2a245057),LL(0x020fe0ed,0x5e8e914b),LL(0x39e8625b,0xdbd2dbf4),LL(0x3dfac893,0x5bf5b95c),LL(0xc2cfde2a,0x29c6d879),LL(0xca30a315,0x05a9a75f),LL(0x3ac05ce9,0x894b84cf),LL(0xb5445553,0xeb87696c),L_(0x000001c2), + LL(0x6bc3c710,0xf9b134fd),LL(0x05a92256,0x23245303),LL(0x7ccfce0b,0x6d8cb621),LL(0xd61ca36a,0x6d0ef54f),LL(0x210c1e23,0x1a182b1b),LL(0x8ae4f253,0xb1f2e2c4),LL(0xa16671b7,0xd29f38da),LL(0xae1cf556,0x3fa6c8ea),L_(0x000001c8), LL(0x7396e499,0xb9a18df6),LL(0xa098406d,0xd588a978),LL(0xd15a5ed3,0x781ea818),LL(0x5e68786d,0xad06fce1),LL(0xf98680c4,0x66d7a550),LL(0x981589bd,0xd3ff4140),LL(0x7ff83976,0x6ffe6dff),LL(0x9c88eabc,0x47479f18),L_(0x00000116), + LL(0x71975c6a,0xdb783254),LL(0xe1b91784,0xcc2bd843),LL(0x7a80b2cc,0xda0b8166),LL(0xaa3b4ff9,0xaff9f442),LL(0x88dcdac5,0x38067551),LL(0x47d782df,0xba990049),LL(0x82a02e17,0x02eb92a1),LL(0x61467fcd,0x29ea45e3),L_(0x0000017b), LL(0x7050f019,0x15b7f2b2),LL(0x3170353b,0x4a58e306),LL(0x4a6890e8,0x0dcaea11),LL(0x2ee85176,0x198b5c45),LL(0x79793d5c,0x9872dd7c),LL(0x5ad3ba16,0x2940cc17),LL(0xacaf46a5,0x9d812262),LL(0xb1cff849,0xee571706),L_(0x0000000f), + LL(0x78fb5075,0xeb0d7e33),LL(0xb0731c42,0x657bac9b),LL(0x5dc372a3,0xd967282a),LL(0xa9374ab3,0xf9ac8856),LL(0xbdf21057,0x3b740967),LL(0xfec8274b,0x56933024),LL(0x5596459a,0x94a16871),LL(0xc8d21c17,0xef7bcfc7),L_(0x00000098), LL(0xb0a89f08,0xf9af7b9a),LL(0xbfd8b660,0x7b728a24),LL(0x9cb13ed9,0xee5e0227),LL(0xa450fd15,0xbd3b7d28),LL(0x972ff1fe,0xf367bf5b),LL(0x08f71ea2,0xaa4191b6),LL(0x496276ed,0x52d016bf),LL(0xc4a6a4c1,0x52e7dadd),L_(0x000000bf), + LL(0x78e99bd7,0x1afab699),LL(0x049e7f00,0x88380090),LL(0xb4090ebe,0xfc780e00),LL(0x9088eeb6,0x294d8c96),LL(0x7d582ac2,0x4027746c),LL(0x5e897916,0xc4fa7517),LL(0x24defc92,0xb30f7062),LL(0x7efacebb,0xe29a1ed9),L_(0x000000b6), LL(0xf3cced8f,0x856dc4c3),LL(0xee3e4e0d,0x6b56aa33),LL(0x971e660f,0xb6b834b0),LL(0xabff8d16,0x4f4e4f36),LL(0x791ab99f,0x1407b72f),LL(0x49828107,0x81293d19),LL(0xdc829510,0x1f2a3c51),LL(0x359122da,0x3bfa8d54),L_(0x000001db), + LL(0x6ad994ce,0xdf15ca96),LL(0x121949d5,0x705e6cb5),LL(0x15f3e1fa,0x844205ea),LL(0xda2168d9,0xb4128cbd),LL(0x793edfff,0x751feb1f),LL(0x332f4e92,0x4c804349),LL(0x4429da30,0x26bc232c),LL(0x8cea38b7,0x753baf61),L_(0x0000013c), LL(0xe7083ee5,0x88e43827),LL(0xb46e7eb4,0x41b7f39b),LL(0xc6fd29ba,0x98261154),LL(0x4974c56e,0x502ce35f),LL(0x0efcc622,0xad5a1de8),LL(0xfd41558a,0xa51d36ff),LL(0x1a681fdd,0xf2ea91c6),LL(0xaa082cc2,0xf7b13d69),L_(0x000000ed), + LL(0x0c38ee14,0xa66a52a8),LL(0x208e5e12,0x9cf1d09b),LL(0xa19a455d,0x4d39c2fb),LL(0xbd4d9e44,0x0adaf826),LL(0xd068570e,0xfd187cbc),LL(0x93225311,0x2398aca7),LL(0x490180a0,0x2dcbb906),LL(0xcee10c1a,0x40723e9c),L_(0x000001fc), LL(0x26d6a477,0x0caf9248),LL(0x7f7fda0a,0x722d1676),LL(0xbe8a621f,0x96407066),LL(0xe524661e,0x3a360a23),LL(0xaa8ac484,0x22dc1578),LL(0x9532f3a1,0x70f20b2b),LL(0x936a98d0,0x4e640ba5),LL(0x10c24716,0xc78e13ef),L_(0x000001ab), + LL(0xc4c781ab,0x8ced176d),LL(0x49979e6d,0x1d8f2f21),LL(0xb3c390a7,0xd1cc9018),LL(0xff49ec42,0x805d407d),LL(0x56592d13,0xb92c79f6),LL(0x69b4fae8,0x7816250a),LL(0xea40b75d,0x9c23c189),LL(0x98e49fbb,0x012080c6),L_(0x00000030), LL(0xf3d27ef2,0xde297ec2),LL(0x394adc76,0x39d2e9b4),LL(0x084a2dca,0xa4a3c98c),LL(0xff54162f,0x4df52d9f),LL(0x847a48de,0xa6af6c27),LL(0x9a4d9dcc,0x128a4972),LL(0x96ed3609,0x8323c41b),LL(0x0a200116,0x81fed229),L_(0x0000011e), + LL(0xa657dd9b,0x20d87189),LL(0x08f2635f,0x6ccede28),LL(0x6b5f8018,0xc2a0e842),LL(0x1a1eaa9e,0x09c6bbfb),LL(0x590c83ae,0xba912027),LL(0xb05367b2,0x98c59c39),LL(0x363d5716,0xc607367d),LL(0x2738b72f,0x76fa4bf8),L_(0x00000080), LL(0x7042e613,0x04011fd9),LL(0x6b33299c,0x42860e67),LL(0xfb00a1e0,0x95c30a5c),LL(0xc7ee4546,0xc597748e),LL(0xbb5a242e,0x3ed03861),LL(0xf0e0361f,0x87708c93),LL(0x2149d193,0x7590a638),LL(0x483134f1,0xc66e8a9d),L_(0x00000176), + LL(0xd5f27f39,0x3813f012),LL(0x5ffac9a4,0xc0d4789d),LL(0x3e79348e,0x955e2b14),LL(0xdb2d28b7,0xe25f9f8b),LL(0xc1656e1c,0x6715563b),LL(0xec42f2f4,0x79f7e28f),LL(0x2aead585,0x106b8506),LL(0x201ba118,0x7f8c385c),L_(0x00000033), LL(0x4d129b93,0x0507ebcf),LL(0xa8ba08d5,0x035af551),LL(0x5db82217,0xe87f61ae),LL(0x97aaf3ee,0xeaf13d4a),LL(0x5f02aeda,0x0476d73b),LL(0x05c9a1af,0x8d9103dd),LL(0xbbd0da23,0x4b324ed6),LL(0x4516539b,0x0edd4904),L_(0x000001b2), + LL(0x7ae64cdd,0xfad3e533),LL(0x9fbd42a4,0x2540903f),LL(0xd7649f2b,0xd8fbba0e),LL(0xc6189edd,0x6a06d4e6),LL(0x1ecce6bf,0xcbba2a3b),LL(0x5c61ec36,0xdb49be34),LL(0x447d3062,0xe4b0ad6c),LL(0x49fc7fee,0x1394cdb0),L_(0x0000018e), LL(0x1b4e3c31,0x1172029d),LL(0x30a8e021,0xbd509669),LL(0x6f8f7c76,0x34a06ccf),LL(0x28f39de2,0x1f805a36),LL(0x5b84e8ae,0x95bcde27),LL(0xa9d053ba,0xd196b2b5),LL(0x4777e8cb,0xbc21fa82),LL(0x1bd75421,0x58e2b8c6),L_(0x00000120), + LL(0xf6fda5e4,0x7a23ceea),LL(0x51aba8d2,0x241c7fb7),LL(0x58878762,0x60069d96),LL(0x51c9d281,0x30ade2dd),LL(0x9b631718,0x91c3eb50),LL(0xf909879a,0xca8dcb86),LL(0xee7eb48a,0x9244bc3d),LL(0x3828bf79,0x2a064705),L_(0x00000120), LL(0x2843df98,0xa015f7fc),LL(0x3ba48f85,0x2b6a12f5),LL(0x95ae129c,0x444e10a6),LL(0x8aeddb2a,0xecfba54d),LL(0x0e8bdf6a,0xc7c39b4f),LL(0x0a72c4d1,0x73b3e601),LL(0x5cdfd0f3,0x99b50a0d),LL(0x5168c9e0,0xbed1929d),L_(0x000001c8), + LL(0xac116c7f,0xa9be603a),LL(0xd22b9ef5,0xc53e7872),LL(0x9558bd84,0x154e0475),LL(0x65401d38,0xbb71dcda),LL(0xf42190a1,0x79891c59),LL(0xcce4f8f7,0x3876a596),LL(0x5ec40734,0x91d0b32b),LL(0xef02bcea,0xe2d3dc41),L_(0x00000083), LL(0x290b052f,0x48a15725),LL(0x7ac11538,0xde31cc2a),LL(0x59724de2,0x949e5fc7),LL(0x1c37f0ed,0xc39190ad),LL(0xc058927a,0xb63be947),LL(0x6bda54d7,0x1d0ceb2d),LL(0xcff0cf31,0xac38d621),LL(0x23f872f5,0x04b46672),L_(0x00000119), + LL(0xf8c3ff8b,0x13238feb),LL(0x8c998602,0xca47bfa7),LL(0xebe29b0a,0xe22527b3),LL(0x7f9eebf4,0x127b07f2),LL(0xa6e2939d,0xca0c1812),LL(0x6469e55a,0x65afd9e7),LL(0x46a7771a,0xf84da7e1),LL(0xbd0864a1,0xb5dbfbc9),L_(0x000000c0), LL(0x89796136,0xb3a29f77),LL(0xbf5bf543,0xb2d15c04),LL(0xd9bcacc1,0xb632b37f),LL(0x5041f46d,0x9bed186f),LL(0xb690ffbb,0x4873f91b),LL(0xeebd39f1,0xff2ad723),LL(0x79a50cae,0xd0c46d7f),LL(0x6b3bea02,0x4a5b4d01),L_(0x000000c1), + LL(0x59fc0874,0x0bb28261),LL(0xe9314bb7,0xa04bdd40),LL(0xaab60946,0xe0b7c3f9),LL(0x20cc2e25,0x1e792761),LL(0x60fd58e5,0x7823f278),LL(0x561086d3,0xfa3bf02e),LL(0x7b6170af,0x0390ab9e),LL(0x4a18459c,0x0c346fa4),L_(0x00000089), LL(0x1b1fad83,0xb623e6eb),LL(0xe73d3d7e,0x43490dbf),LL(0xc1f1a1fd,0x49e9e831),LL(0x0a18b7b1,0xe619f992),LL(0x0c6e526d,0xdb9252a1),LL(0x954cd738,0x6a826c49),LL(0x41105a8b,0xddcbb9cf),LL(0xe9217743,0x750efcbf),L_(0x00000193), + LL(0x83ba5b5f,0x90c2c466),LL(0x57128a20,0x1345257c),LL(0xdbf610a8,0xd16c4a33),LL(0xbff009fd,0x8f1d5b65),LL(0xb49af8c7,0xf560ad02),LL(0x0eb8499d,0xfb45ea45),LL(0xa52dc630,0x7e352023),LL(0x8ada8ac5,0x2fd6cb3d),L_(0x000001b7), LL(0x9d102023,0x8f6d7783),LL(0x4a52a42a,0x3f44af1f),LL(0x4175b2fd,0x05fe7f14),LL(0x079ac149,0x1757d0c0),LL(0x475fac70,0x88ae6d1c),LL(0x7bfd9387,0x33f3d56b),LL(0xb7dbf13b,0xcacad131),LL(0xebd1df20,0x8aef62c8),L_(0x00000009), +}, +/* digit=83 base_pwr=2^415 */ +{ + LL(0xa0c45001,0x96fd03e2),LL(0xf3f782f7,0x0bf4ad5c),LL(0x3ffeae62,0x8514f603),LL(0xe39ca015,0x5633e085),LL(0xa88f4e54,0x94884fbe),LL(0x85fc77f2,0xf605882a),LL(0x2678c646,0xa505f9b1),LL(0x799ba323,0x23217b43),L_(0x00000092), LL(0xf4170bf2,0x032e8744),LL(0x29194f6c,0x6cab181a),LL(0xa932a791,0xf60ec063),LL(0xb94fb0f9,0x217a0ff0),LL(0x8f066aff,0xd03ea56b),LL(0x56ee8b26,0xc1ffed4a),LL(0xefbf8ce2,0xed130515),LL(0x922eb114,0x74474339),L_(0x0000003d), + LL(0xd1837634,0x88affb50),LL(0x5fc7f37a,0x170a0c1c),LL(0xa1f73a2f,0x83474ff9),LL(0xea811929,0x4738ed4f),LL(0xdd78686b,0x24d293dc),LL(0x16188a23,0x36670cd9),LL(0xc585fd52,0xa2e54dbb),LL(0xd3b67188,0x10b37344),L_(0x000000ed), LL(0x6336003d,0xb4548b08),LL(0xfbe0b348,0xad120991),LL(0xef3cdca5,0x034c9a59),LL(0xd16cfcd2,0x56699960),LL(0x10f2524f,0xf6df1f5f),LL(0x4733c5f1,0x98a50032),LL(0x757f84ed,0x2ce4fa9a),LL(0x032f7eec,0xd296f3ba),L_(0x000000e3), + LL(0x5b81d69e,0x4c96fb7e),LL(0x71db44c9,0x7ac3b1c1),LL(0xe9e2107d,0x6305a422),LL(0xa60259b2,0xc0b70492),LL(0x17e1f71b,0x3dedfaf7),LL(0x57f8d178,0x15a8c62d),LL(0x7a704e50,0x81a4724b),LL(0x3b26accf,0xdf992c3c),L_(0x000000a0), LL(0xe21a6ddc,0xe35cce3f),LL(0xc9cc0ded,0xc51c6e93),LL(0x00949cad,0xba4d9081),LL(0x8de8b3c3,0x274926f6),LL(0x839aef68,0xf13d3b8a),LL(0x40a2e3c7,0x90113a91),LL(0xef4e7433,0x1f472807),LL(0x8320657c,0xd4accc1a),L_(0x000001d6), + LL(0xb32007cb,0xc34e9382),LL(0xcd77eae7,0xad7f0d81),LL(0x13604a2d,0xd0e3fde5),LL(0x70443d13,0x6cb59871),LL(0xc5f3e64c,0x7a8441d1),LL(0xadfd909c,0x00532361),LL(0xefba7861,0x0eb9abb4),LL(0x05155907,0xee4fe6fb),L_(0x00000108), LL(0xa4f6cc8a,0x7579200c),LL(0x128ad5ec,0x49006579),LL(0xc2659737,0x8cf2fa39),LL(0xc9df0a7f,0xdb548c37),LL(0x648b652d,0x3da31069),LL(0x075eeef1,0x34916a7e),LL(0x04d0e409,0xea6b7825),LL(0x08fd613b,0xba92eb2c),L_(0x0000015b), + LL(0x3dfeb0af,0x47521ca4),LL(0xc24e86b5,0x555d9a6b),LL(0x572924ea,0x878c09b9),LL(0x379539ca,0xb6b82a15),LL(0x8666f974,0x72ba5827),LL(0x2d9ff656,0xb17ece57),LL(0xc68cfdb5,0xfe917da6),LL(0x22f34e2a,0x08992968),L_(0x00000021), LL(0x174d7bcd,0xf77f07dd),LL(0xba763f11,0xf3886c72),LL(0x11e3d2aa,0xa4c6b62d),LL(0x550a0527,0xe4eac1e4),LL(0x6a7880c8,0x9b86f1e0),LL(0x00ea68f4,0x87cc2f01),LL(0x55e1d3f3,0xa6daef73),LL(0xa5a26f0f,0x2061095b),L_(0x000000d8), + LL(0x4a4107d1,0x8b5ab4da),LL(0xbf85411a,0x7b991f0d),LL(0xa933992a,0xce47a748),LL(0xc820accd,0x662f2eb8),LL(0x12508cc5,0x964b5fdd),LL(0xadddfe6b,0x1358db73),LL(0xf97a44a3,0xfefeacaa),LL(0xf544c5dd,0x3a084f6f),L_(0x00000140), LL(0xec21428a,0xa4ad406f),LL(0x54cddbeb,0xec844e89),LL(0x92a7fe19,0xa4c49f5f),LL(0x30484bff,0x8eb76b96),LL(0xf75a70ec,0x414948f0),LL(0xc139503c,0x7606dff2),LL(0x2fdf031b,0xead62083),LL(0xc5fa11a5,0xf7a1eba7),L_(0x0000007e), + LL(0xe0553aec,0x7ec431d8),LL(0x2ebaaf47,0x3c1a318b),LL(0xd9561acd,0x57cb3287),LL(0xf1f803a4,0x97882cee),LL(0x97ac71c6,0x08071010),LL(0xf26f5efd,0x60ed1a2f),LL(0xe4333f2a,0x76efc905),LL(0xc5d26fd2,0x4101ca9f),L_(0x0000013c), LL(0x3f266f96,0x75779876),LL(0xf709921a,0x294a8042),LL(0x5ab89053,0x0dc515ed),LL(0x5f21558b,0xd5146f68),LL(0x1cdf8e37,0x8dcdaaee),LL(0xe4930f54,0xacb8f4a6),LL(0x58634cb4,0x507294c8),LL(0x20ac12b8,0x38b15ebe),L_(0x000000ba), + LL(0xbdb35346,0x6973a3da),LL(0x5d8f611c,0x3f4e86dd),LL(0x799bf33f,0x749fb625),LL(0x4a326f63,0x667bd358),LL(0xa8161392,0x060fa9fe),LL(0xa3a8de55,0xc4af999e),LL(0xdd75d71a,0x18b1e644),LL(0xc8783194,0xe995c857),L_(0x00000067), LL(0x52eb541b,0x32c7afa5),LL(0x22bb4a07,0x08c59a72),LL(0xc7e0e1a6,0xe1132506),LL(0x1fa09057,0x5c6a1998),LL(0x6b7bf39c,0x41920509),LL(0x8ab7490c,0x462ba7d3),LL(0x9e016ba7,0xdc3595d9),LL(0x1207f474,0x6a3d8c9f),L_(0x00000063), + LL(0xb27c0a7b,0x2b8fd9cd),LL(0x606dd096,0xaea5acbf),LL(0x3ef0fe25,0x4859d96e),LL(0x078a8287,0xf923a972),LL(0x0d66d4df,0x660fec36),LL(0xb27b9ec1,0x102d1cc4),LL(0x1d991f63,0x730fac93),LL(0x0c8cdfad,0x601a8644),L_(0x0000005b), LL(0xb0aea91e,0x55d58869),LL(0x0d313b90,0xde263f83),LL(0x11dbdf4c,0xcde2808c),LL(0x824b53bf,0x6aad6afb),LL(0x94547c69,0x864c21db),LL(0x6771d459,0x65c1681d),LL(0x7f15bc35,0x95681ca9),LL(0xeac68bf9,0x78f00d83),L_(0x000000db), + LL(0xf35f241d,0x2f058d88),LL(0xb2e8d253,0xaca02a59),LL(0x15502597,0xbd7d1caf),LL(0xa0dfb1e8,0x680e361d),LL(0x9355f155,0x2ed31cfd),LL(0xb0064d2b,0x308047c0),LL(0xf348830f,0xbffaf7d4),LL(0x7bb7440f,0x553b98e1),L_(0x0000005a), LL(0xb6375708,0x9f0f6eac),LL(0x25881bc3,0xa7105d49),LL(0xd655e7e9,0x033db883),LL(0xdc82f09a,0x7d5a4975),LL(0xa17847e9,0x4036e619),LL(0xe9b20930,0x048479d0),LL(0x34fdadf8,0xe6c7daa4),LL(0x19412216,0x152f330b),L_(0x000000eb), + LL(0x6fe66f8e,0xc41a7939),LL(0xcb26a326,0x78d31411),LL(0xa365755c,0xd3a391dc),LL(0xe18d1f8e,0x25f34512),LL(0x72a34b02,0x345c60dd),LL(0xd3613f12,0xe2be6e2f),LL(0xee2aebd8,0x116aa632),LL(0x92435b38,0xa7cff2c3),L_(0x000000c9), LL(0x3f4fa083,0x342a1414),LL(0x9da3251e,0x650f3cbb),LL(0x1c4cfe7a,0x93ddee66),LL(0x90734ecc,0x1551811f),LL(0x6d901fe7,0xcc1cef07),LL(0x0c7d5cd4,0x3cf5fe87),LL(0xf0de3068,0x628a5bd8),LL(0x56e883b0,0x077d4d10),L_(0x00000195), + LL(0x8e2db279,0x2d362cca),LL(0xfb4260cc,0x99614b58),LL(0xe2d527b8,0x67cb8aa3),LL(0x08c1f0b4,0xef71b82f),LL(0xc4649ad1,0x4dc68072),LL(0x11a9313c,0x1ac298aa),LL(0x65002fbe,0x9f237961),LL(0xe90bf539,0x537dfdd6),L_(0x0000004b), LL(0xade2b535,0xfcf6c830),LL(0xf2e76469,0x4ed4174a),LL(0x51f1bc5f,0x3a450f7e),LL(0x83c0406c,0xb53708a6),LL(0xaa7dce02,0x2428a6e3),LL(0x44b377b6,0xf1abc0df),LL(0x9c1a58f5,0x0f02c35e),LL(0x27c5458b,0xea8718da),L_(0x00000116), + LL(0xedc91665,0x1709bab6),LL(0xd1568e14,0x5208725a),LL(0x11bb7351,0x3937bb45),LL(0xe6e5c9ed,0x51ec95b9),LL(0xcf7661c1,0x2e23be41),LL(0x6ad385c7,0x743b0e8c),LL(0xe91385d0,0xdfe84bbc),LL(0x6fd97535,0xdce16477),L_(0x000000c9), LL(0xdbe30fec,0xef8587b7),LL(0x0e47600f,0x8f375855),LL(0x6859ef6b,0x529ee446),LL(0xc03de2c7,0x84c625e6),LL(0x6f7cbb53,0x3af54a02),LL(0x29eb9d15,0x37cbf19d),LL(0x750b2d9f,0x2dc61071),LL(0x28f78635,0x6b20c9d0),L_(0x000001cc), + LL(0xc05c59ca,0x78134d37),LL(0x3d575868,0x09f53723),LL(0x6cf5af74,0xfc4fd018),LL(0x39a8ae6d,0xb54df4cc),LL(0xb1d402e8,0x25f3046d),LL(0xeece717b,0x98af312a),LL(0xa13a0c5d,0x5f96c47d),LL(0x7f73d630,0xc80a3e3a),L_(0x00000187), LL(0x516f5d07,0x7f27d5a2),LL(0x38bbf8fa,0xc4360f93),LL(0x2109c7d0,0xe57b26a1),LL(0xeea6004b,0x32aad5ae),LL(0xf9dededa,0x341aa5da),LL(0x6abc8307,0xfdd6b0a1),LL(0x88080bda,0x4cecd6e0),LL(0x1d4fa881,0x24b2b7fe),L_(0x0000015f), + LL(0x923ad134,0x9a724982),LL(0x5ec2d29e,0xa69853d7),LL(0xd1a7ff3b,0x4e7a173c),LL(0xffeab3d8,0x9f30735d),LL(0xdf72352a,0x92186213),LL(0x469f7a55,0x2d184184),LL(0x15c208a0,0x29514acf),LL(0xa37b763a,0x91393991),L_(0x0000015b), LL(0xfb958196,0xa16bd801),LL(0xf0588b07,0xc8dafbba),LL(0xda133516,0x3aebd875),LL(0xf622ae4f,0x34fb368f),LL(0x4f336c4d,0x0b5d9d6e),LL(0xc6eb3519,0xa586248a),LL(0x6d28f06f,0x6daf558c),LL(0x2f3e52a1,0xfb60040b),L_(0x000000b3), + LL(0xd74494af,0x05cc19d6),LL(0x6d31ebaa,0x2f30e929),LL(0x3edd43b0,0xc72cbbb4),LL(0x827c79aa,0x7829df3e),LL(0x4e4cf8d5,0x98bb6262),LL(0xffe745fb,0xad894f05),LL(0x50b350aa,0xb566ef19),LL(0xf2ea5e2d,0x37f6dcf4),L_(0x0000001e), LL(0xe7f3c59f,0xfa34202e),LL(0xd748da48,0x68fd9ed5),LL(0xe1cf505b,0xc7778cb3),LL(0x3af31b86,0xdadb4507),LL(0xfe717fdb,0xb4b6e80b),LL(0x13036b30,0x2c3f1ee4),LL(0x482b138b,0x4fc01593),LL(0x71e1ed1e,0xd788bd27),L_(0x00000084), +}, +/* digit=84 base_pwr=2^420 */ +{ + LL(0x836ce2cd,0x7d9afb73),LL(0x1c085462,0x2eb9d35d),LL(0xacf6649f,0x38a8a9ec),LL(0x095acf4c,0x78be52c0),LL(0xf7d7ea21,0x48f6e06d),LL(0x115ce7bb,0xbedc8285),LL(0x7f232680,0xe51e8f4b),LL(0xd24103a6,0x09aa0bc0),L_(0x000000aa), LL(0x39f42b13,0xe11c4bc5),LL(0xf757159c,0xbc8d09f1),LL(0x0e9e10c0,0xe3621884),LL(0x60d7345b,0x822e5e0a),LL(0xddc802d1,0x6ae792ac),LL(0xf49763d7,0x868a6be0),LL(0xff0f1717,0x7cae1bcd),LL(0x69443786,0x8bfe19f2),L_(0x0000010e), + LL(0xb02326b4,0xe01b8994),LL(0xe496416a,0xa213f31c),LL(0xdc0825ce,0x0281aa93),LL(0xe4450bdf,0x236853f9),LL(0xc8c09e59,0x841e294d),LL(0x595c72a5,0x1e14b03a),LL(0x2bc6e538,0x46b3008c),LL(0xef20b035,0xa57d1874),L_(0x000000ec), LL(0xe5948afb,0x5951a61c),LL(0xd36a1693,0x5b84b925),LL(0x1712f765,0xc1e05016),LL(0xa422f3fd,0xa758020b),LL(0xb022819a,0xb6927405),LL(0xfced2aa8,0x13a79822),LL(0x9ae63d93,0x87cbebb1),LL(0xcdd9c078,0x13e45feb),L_(0x0000005e), + LL(0xe4998422,0xc434f1f7),LL(0x878049fe,0xf0c9ca48),LL(0x692a3fe1,0x277fbbb0),LL(0x7ad48261,0x263dc0fe),LL(0xc6fec032,0x7e09052a),LL(0x04d38aeb,0x37fd6838),LL(0x6c55fb12,0x53925e9d),LL(0x63bce4b4,0xc33e2d82),L_(0x0000016a), LL(0xc6d3730d,0x70a764c5),LL(0x705b2adb,0xb9f1bed9),LL(0x3bab4631,0xb4850149),LL(0x37a14535,0x2385e829),LL(0xcd9ea79c,0x07aa5ebd),LL(0x17e5b5eb,0xd3216565),LL(0x8fb6885f,0xc6b2bd87),LL(0xce87f791,0xd44bfcfc),L_(0x000001ca), + LL(0xda995024,0xd4a7c1bb),LL(0xad86ee95,0xf741fd13),LL(0x1edd6ede,0x85b8fada),LL(0x08912214,0xb65b3c0a),LL(0x30092262,0x3583aac0),LL(0x6716727e,0x817eec2e),LL(0xd6729d8a,0x33bfe296),LL(0xa92ad342,0x779921dc),L_(0x00000006), LL(0x0bc37c31,0xc1373f0b),LL(0x5632a2bf,0x2c2b1683),LL(0xa8cd4f47,0xa4cedeb1),LL(0x32a4b6c9,0x180690c1),LL(0xfa510c76,0x1eca05f9),LL(0x51eb02c4,0x2e0f4e05),LL(0x63213fc5,0xb6165cde),LL(0xd91b429b,0x1fdd188c),L_(0x000000bc), + LL(0x58d599bc,0x52629f76),LL(0x87d39d0d,0x07dbdb1e),LL(0xcaaa9976,0xeb7d6dd2),LL(0x02ffcf23,0x7e0cd30a),LL(0x78d51085,0x83ecd227),LL(0x961e1f15,0x54655c5e),LL(0x8aa70a14,0xf96ec7c6),LL(0x62c8fb1f,0x64d2f55e),L_(0x000001b4), LL(0x0ac79416,0xb6e904d2),LL(0x0bec2602,0xef59ae4f),LL(0x80f6effa,0x330793e6),LL(0x54960688,0x2442ae08),LL(0xff5a5fdb,0xc5e3d773),LL(0xc6ac0199,0xcacfcecb),LL(0x2fa7a795,0xc57e52d1),LL(0xdfd6f9bf,0x2e4eaeaf),L_(0x000000e3), + LL(0xba2a7d7d,0xefeeeccb),LL(0xd77ed0ff,0x9db74ad1),LL(0x5e752d76,0x0b6200a7),LL(0xfc315b24,0x7b48ab8c),LL(0x38ef4859,0x6f975045),LL(0x980da41b,0x09695a2e),LL(0x0010c201,0x0d2b23ac),LL(0x7b7fe53f,0xfe8b4888),L_(0x0000001e), LL(0x8dd021c4,0xc97b952e),LL(0x6b8cb163,0x16bdd26e),LL(0xd62feeed,0x6129cfd5),LL(0x9be7e3db,0xcfa4489e),LL(0xd804ec9d,0x8f551707),LL(0xadb1fbcf,0x055312a2),LL(0x05be2283,0x7d87937c),LL(0x33c9f74b,0xe5edd3be),L_(0x000000d7), + LL(0x46a8f7e5,0x854b2f45),LL(0x223a8078,0x954aed2a),LL(0x155358a6,0x30349ae5),LL(0xae186e4b,0x982c7a2a),LL(0x15564aa5,0x26ca4f64),LL(0xbb73fd2e,0x3fca013a),LL(0x2d10cb06,0x3b01aea9),LL(0x33610496,0xd0f1f68f),L_(0x000001bc), LL(0xf43f1a25,0x7f69aedb),LL(0x1ef2c9b4,0x7e6d3ebe),LL(0x3246ee72,0x1a5a6120),LL(0xfc23f5bd,0x7e829b05),LL(0x02380698,0xfb70444c),LL(0xe903fa23,0xb14cf0d6),LL(0xaba52743,0x7e9872db),LL(0x4a9994d9,0xcbf74f83),L_(0x00000052), + LL(0x0feb8ccb,0x077ea324),LL(0xae770944,0x13db8792),LL(0x1201418e,0xbc07e7f8),LL(0x7f43920c,0xd56c9383),LL(0xa87da3ae,0xf13ae3d2),LL(0xddc44f1d,0x220944b8),LL(0x9328fcaa,0xe3819646),LL(0xeeb928d8,0x3e55f26b),L_(0x00000131), LL(0x1bd80088,0x42ec4f45),LL(0xfe195b7e,0x8b6807bd),LL(0xe12b70e6,0x2b99f5fa),LL(0x7ea15922,0x4e065912),LL(0x9b50760c,0x2efe8690),LL(0xb9b66711,0x16a38c58),LL(0x7b7026cf,0x0ea01a94),LL(0x80737374,0x39639895),L_(0x00000015), + LL(0x5664a574,0x514bdbc0),LL(0xb7081a5f,0xfa8d102a),LL(0x8b95f28e,0xbeeb8bec),LL(0x89435618,0x71f4af84),LL(0xdf609827,0xb479c91f),LL(0x14f1207b,0xcb1fbb1b),LL(0xde646157,0xf9e5fc99),LL(0x7b096333,0x34873e5f),L_(0x0000012f), LL(0x7966170e,0xd78f796f),LL(0x3fa5ab5e,0xa3fca0b1),LL(0xacf76276,0x8003771d),LL(0x1726ca8d,0xe6d96044),LL(0x3e43296d,0x93fa9826),LL(0x42531228,0x527acf14),LL(0x60a72222,0x7a97cba9),LL(0x3d7c6f6c,0x1948eafd),L_(0x0000012b), + LL(0x5463af9c,0x4e12b5af),LL(0x08d068df,0x886cf2ad),LL(0x79260e0d,0x52a61dfd),LL(0x3e9745ee,0x3f586c5d),LL(0xe1f15978,0x566b3b18),LL(0x87842ee1,0x9ad3de40),LL(0xa8f52a76,0xfa06fa9e),LL(0x391add0b,0x9e84e7b8),L_(0x00000042), LL(0x95c6c5aa,0xbb6677c3),LL(0xa05bc623,0x52d3e69c),LL(0xb274df3e,0x974dd07a),LL(0x630e22a1,0x5e6204a2),LL(0xb9a3958d,0x1b227ffa),LL(0xd0e0f634,0x8fc075d7),LL(0x37062afb,0xbe7fbf5c),LL(0x34adc02d,0x77906471),L_(0x0000012f), + LL(0xa73219ef,0x6c9a6028),LL(0x259579a0,0x6f5bb2d1),LL(0x0285a43e,0x154ac0bc),LL(0x167a3f99,0x36795c46),LL(0x728f737d,0xb73fd19e),LL(0x4d08d004,0xa6cc0016),LL(0x85ddb728,0xe72a83f5),LL(0x2e0d295e,0xbfbd9eeb),L_(0x000000f5), LL(0x30d1ecb0,0x90f9c914),LL(0x0eb6be44,0xaf866e12),LL(0xfe01cd22,0x5fd8a835),LL(0x7bf70c61,0x7f0a7679),LL(0x38813f80,0xc8998bdd),LL(0x3833317e,0xb796096b),LL(0xfec3ea9b,0x95f4e76d),LL(0xdad9de4b,0x341f07a8),L_(0x000000d0), + LL(0x9808d4ec,0xd3c33742),LL(0x200fc40a,0xb0a42399),LL(0x130ccf0d,0xcba30b44),LL(0x5e03c03f,0x8e5dcd29),LL(0x6cdd189a,0x1e3d9c12),LL(0x5a30516d,0x81b3e6ac),LL(0x6b10cabc,0xcc86cceb),LL(0x69bda2cd,0xc43e19c0),L_(0x0000003a), LL(0x616887a8,0xdc567bde),LL(0xe0559109,0x6575e24d),LL(0xe2ac30ae,0x67aa07b7),LL(0x2b6c5e32,0xdaf439f7),LL(0x407ff042,0x2ed76986),LL(0x0a686f32,0x1581e9c9),LL(0x5cacfb27,0x18e37622),LL(0xc3e3f9bf,0x387fd7f7),L_(0x000001ce), + LL(0x11961504,0xcd152b7e),LL(0xe1bfaadb,0x9f66edb2),LL(0x2faec2d9,0x65371362),LL(0xe758a385,0x03888c8c),LL(0x70736030,0x84bd514a),LL(0xf31e004a,0x391a903e),LL(0x246ee862,0x36e788ca),LL(0x4e1e02aa,0xf4e00df9),L_(0x00000137), LL(0x6c1152dc,0x13ba50a4),LL(0x4eff7c1a,0x941609eb),LL(0x6d04b057,0x4cf29ce1),LL(0x414be663,0xc8df1ba5),LL(0x12291ebf,0xaa401069),LL(0xeb298db0,0x8ded5bfb),LL(0x893e7d57,0x549bd728),LL(0x01b7b4f4,0xac5af1f1),L_(0x00000045), + LL(0x757b26d3,0x5f6b726e),LL(0xb153c049,0x1a260428),LL(0x30c06c83,0x9ce7c010),LL(0x5b4fc0ff,0x894ae091),LL(0x5657bc21,0x0aae49b6),LL(0x571a85ef,0x79cfa3cb),LL(0x65689c51,0x2187e975),LL(0xe76412d7,0x992bb278),L_(0x00000189), LL(0x11fe345d,0x0cf8c519),LL(0xaa2b60ae,0x9b1824f2),LL(0xc0f1fe55,0x9db0e06e),LL(0xd91bb72b,0xa610996c),LL(0x58e02f73,0xf2c3a8c2),LL(0x28d213dc,0xeae1bfc7),LL(0x13f362dc,0x943517da),LL(0xf7bdbaa1,0x496789e0),L_(0x00000049), + LL(0xb7668d30,0x6c383c56),LL(0x23ca0c67,0x0e7599c4),LL(0x99283769,0xfeb47aad),LL(0xe99af616,0xaf666d85),LL(0xf45b2fa8,0x9231d83a),LL(0x7c7b482a,0x98e9e83d),LL(0x8568ac48,0x1e56ad0b),LL(0xb6b0ac15,0x724bffe2),L_(0x00000169), LL(0x5217d4f7,0xb11260bb),LL(0x877aec8d,0xc03ce819),LL(0x4e72375a,0xccf9f47b),LL(0x54033f3a,0xac8daf52),LL(0x19b599aa,0xdea12842),LL(0x175a56a0,0x7ec2c31e),LL(0x8b550edc,0x1acab966),LL(0xf427902c,0xb52abab5),L_(0x00000147), + LL(0x70bcb2f8,0x52546c7f),LL(0x20270840,0x3c3eb721),LL(0x13f0d6d8,0x142e7b62),LL(0x019eaca7,0xf973a763),LL(0x025efc29,0x8be00c2a),LL(0x6f6199ab,0x03943eca),LL(0xb5618bcc,0xfbfb9ab2),LL(0x5fb02749,0x0ae9ab79),L_(0x0000013c), LL(0x3c3d712e,0x50f295b9),LL(0xab6d0e9f,0x083d8bf1),LL(0xbf53d7cb,0x07076abe),LL(0xf29d744e,0x51a53561),LL(0x5c2fc15a,0x7d647b91),LL(0x253f8428,0x51a29ead),LL(0x91bd9d62,0x006b7bce),LL(0xe106dd74,0xc770e4ef),L_(0x00000092), +}, +/* digit=85 base_pwr=2^425 */ +{ + LL(0x4d5c812f,0x412703ae),LL(0xe95afaa3,0xeb61e427),LL(0x271961d1,0x509a4412),LL(0xcf60fb71,0xef04644e),LL(0xd64abf7b,0xf44d4556),LL(0xfd03e651,0x9b052a7b),LL(0x4add4bdd,0xbb2156b9),LL(0x2a438d74,0x6c6c1657),L_(0x000001a3), LL(0x6be29111,0x0eae16f9),LL(0xafa2d73a,0x44951d52),LL(0x5bc81a5a,0x324d90b4),LL(0x8490fc89,0xcab36337),LL(0x7db83818,0xdcb411d8),LL(0xd6cb710a,0x80af21f7),LL(0xad265214,0xf370ca06),LL(0x3194a666,0xd8bc966e),L_(0x00000175), + LL(0xed0cc632,0xbbdc15d0),LL(0x35b540e1,0x979e91ce),LL(0x973de7aa,0x745fa684),LL(0x84d5e965,0x999e957c),LL(0x2675f78a,0x671ead70),LL(0xec63eb2b,0x9f4e2bee),LL(0x2262a934,0xa151a2d6),LL(0x885b027f,0x5a633743),L_(0x000001b4), LL(0xcbfe121a,0x63ade9fd),LL(0x64a13f40,0x23957199),LL(0xea6e576b,0x21e1e294),LL(0x1a39fa20,0x3e8e8d0c),LL(0xeb1db8f9,0x2627a8aa),LL(0xa239f73f,0x4ba11c83),LL(0xdc2704f3,0xe1591a19),LL(0x6d53062b,0x75cd2132),L_(0x000000c9), + LL(0x2f5961b0,0x615287cf),LL(0x9beb6f84,0x5201fcef),LL(0xa7eaedc7,0xabd8969f),LL(0x485d39df,0x52116284),LL(0x9a298f42,0xc2945b41),LL(0xb98d985b,0x02540f03),LL(0xbe292f99,0xfb5c8297),LL(0x927d1f59,0x379bfc25),L_(0x0000002d), LL(0x2527211f,0xbd1179af),LL(0x24128932,0x49c7ff27),LL(0x920719e9,0x1ce441c4),LL(0x7bdf184c,0xd4d77a7c),LL(0xd00ec091,0x286d6826),LL(0x3901be8d,0x8de5125b),LL(0x6d7da5fd,0x7b71c09c),LL(0x0a84f058,0xd13d8a20),L_(0x000000cb), + LL(0xfe1ac328,0x4047691d),LL(0xb25141cf,0x9109a95b),LL(0xcb8b02c5,0x46b0d286),LL(0x43ceb570,0xb7de8163),LL(0x45b33ca8,0xd109a4f7),LL(0xe4de2617,0xfe1ebf34),LL(0x50869270,0x1ca61709),LL(0xd0a3a1b0,0x38634a9a),L_(0x0000012e), LL(0x9fbe1e8e,0x38f07910),LL(0xce15260c,0xc1bab71b),LL(0x59aca1ef,0x5da68584),LL(0x8166c7ad,0x0bec54d9),LL(0x8f461203,0x49a9c32e),LL(0xd88a5170,0xefea1154),LL(0xbe8180a7,0xec5ede1e),LL(0xad08abc6,0xe0d26459),L_(0x0000017d), + LL(0x40dfaf92,0x75d35af0),LL(0xc6d5609c,0x78abf2cc),LL(0xe0503dac,0x06b14f6b),LL(0x2edd7321,0x1d8b929f),LL(0xf2198c61,0xae13548d),LL(0x510cab55,0x7dda7de6),LL(0xd681dc83,0x9c5b80c7),LL(0xf75528c0,0x3789ecc5),L_(0x00000012), LL(0xf941d11e,0x96096d94),LL(0x43aa0894,0xc108e7c3),LL(0x6b10f225,0x662c2a13),LL(0x6a846090,0xfd5f03e1),LL(0x2ef6dd38,0x6481ee80),LL(0x5be580e2,0x5fdbc53d),LL(0xa958cf09,0x11a46f11),LL(0xc02fca90,0x664e67d4),L_(0x00000194), + LL(0x9ae3c944,0x2e504292),LL(0xd30defda,0x0fa7d618),LL(0xea344a1b,0x8b5c684d),LL(0x88f031e8,0xc536e574),LL(0x4b824800,0x3cbde674),LL(0xad6b63cc,0x56514289),LL(0x9617ab5d,0x85b3cf5e),LL(0x7dc3ec36,0x7fc09058),L_(0x000001b6), LL(0x555deeda,0x846008dd),LL(0x6898fae8,0x9e7f0e84),LL(0xd8281d5a,0x10825f5c),LL(0x2d75ee2c,0x12dbdb85),LL(0x7c0aafd5,0x9070014b),LL(0xbf482380,0x9ee025e3),LL(0x0fc3aa99,0x63b81901),LL(0x508f2832,0x84535f01),L_(0x000000f2), + LL(0xc010cf33,0x9b68b1b0),LL(0xc08c3333,0xc47488c1),LL(0xab449170,0xb2a5ff61),LL(0xcf64ef35,0xb9e5f70f),LL(0x7b502c46,0x417cfca7),LL(0x88cbf950,0x1cf7bedc),LL(0xd4ad741c,0xf0d8aab2),LL(0xbcf1b6ad,0xf5a48560),L_(0x000000fe), LL(0xf9b68aec,0x502beb84),LL(0x2cc6cdcb,0x584dda42),LL(0x094da6af,0x129f5a40),LL(0xd42b7286,0x1002d93c),LL(0x5c173674,0x725b278c),LL(0xefa77633,0xd75f98ff),LL(0x791931d8,0x21527308),LL(0x512e096c,0x53d3565e),L_(0x000000de), + LL(0x140932ce,0x00019394),LL(0x62bec3a2,0xa2c579a3),LL(0x5ab884c7,0xa06cf4c6),LL(0x083cab2b,0xe7eda004),LL(0xf5f6cd3e,0xa107bb12),LL(0xe8649e92,0x492e8a0a),LL(0x5a344683,0x91fc3fba),LL(0x8dcfb242,0x94cfd171),L_(0x00000038), LL(0x1d1f30f0,0x03751b1b),LL(0xce223739,0xc2f1043f),LL(0x23da45da,0xf9307b4a),LL(0x3110105d,0x2b6d1b2c),LL(0xb39e6a89,0x2f645131),LL(0xd8e317be,0x90466331),LL(0xb2f395de,0xf1e436a0),LL(0x1cd56345,0x547fe748),L_(0x000001b0), + LL(0x6d41f81e,0x9c18c216),LL(0xfd251565,0x18071963),LL(0xf23633a9,0x005c3056),LL(0x1e0819de,0x46ebbb94),LL(0xb549b1c1,0xb17a6d7d),LL(0x92577078,0x991b825e),LL(0x8f6ca39e,0x98671ca8),LL(0x90ccdc4b,0x393fb1b5),L_(0x00000102), LL(0xcb6f46fe,0x3e2fd629),LL(0x150480ec,0x0aca508b),LL(0xcbfc5f51,0x0864f022),LL(0x14650672,0x3b3979b5),LL(0xbff63c37,0xb97ea4a4),LL(0x3b1cb3cc,0x3ef296ec),LL(0xcb4a6db9,0xff3bd186),LL(0xbd5b637b,0xedab5309),L_(0x00000052), + LL(0x7249f522,0xba62ee82),LL(0xd77204b3,0x77e6a3df),LL(0x710f93b4,0x02f4ab22),LL(0xcbb03368,0x5e0ecd43),LL(0xb189c2ee,0xc2847328),LL(0x9bdf60eb,0x0ebd36bf),LL(0xb67d93ed,0x0c17c388),LL(0x70c2279c,0xe8fdeaec),L_(0x000000d6), LL(0x8f80aff8,0x48b19e4c),LL(0xc3191f20,0x04206c4e),LL(0x8bdfd660,0x9a16a00f),LL(0xac2112af,0x61997c61),LL(0x8ab3667b,0x7b84c760),LL(0x06bdff10,0xf4e1645e),LL(0x607df1e8,0x352d17a0),LL(0xa63979fb,0x83c7eb22),L_(0x0000016b), + LL(0x223ba032,0xe2ef6187),LL(0xe76e649d,0x50e4618d),LL(0x6b987f60,0xc357d1ec),LL(0xad4f8231,0xd4a60be3),LL(0xb315fdb0,0x2eced057),LL(0xa9fb56ad,0x19319996),LL(0xa2da27cf,0x2f11dee2),LL(0x916ddbdc,0x7e40b2e7),L_(0x00000059), LL(0xd93389b7,0xb7a77c12),LL(0xb4cf9881,0x7da427fd),LL(0xcc7a5c06,0x80b0089c),LL(0xdef4d8fc,0x93e7089d),LL(0xe325b955,0x16ead4d1),LL(0xb039b2a1,0x5dfcb305),LL(0x1820d74b,0xe64404b5),LL(0xd5916176,0x6e3b8d75),L_(0x000001ef), + LL(0xf55f7a6b,0xe5919bbd),LL(0xff1cd958,0xef53a31b),LL(0xf4f94077,0x95740f28),LL(0xd731ab60,0xae2285d4),LL(0xa1ecbb47,0x8c0691df),LL(0x258e27d0,0x3a34371a),LL(0x6fcf67a4,0xe81db787),LL(0x31ad117e,0xcd7a8c18),L_(0x00000101), LL(0xc8eb7eab,0x1c27e10f),LL(0x7aa7d331,0x36d353fb),LL(0x484bc381,0x5feb0814),LL(0x14c00e62,0x5a897087),LL(0x4174c50b,0x73330526),LL(0xa9a0af61,0x7d611726),LL(0x60873261,0xca260561),LL(0xe43e2bb4,0x9e169695),L_(0x000001ef), + LL(0xb03b9785,0xb1e778d1),LL(0xe387f978,0x9b9aee5d),LL(0xd76337e3,0x92cf14d1),LL(0xe5ccf490,0x72921fc6),LL(0x5b43aefb,0x65d3e5c1),LL(0x795ec0f9,0x3904e5d3),LL(0xa4928380,0xa5751b73),LL(0xdfd26a4c,0x351de543),L_(0x00000098), LL(0x34b9f259,0x30b62509),LL(0x2b0ddc19,0xf6f4cca7),LL(0xe118ecd8,0xed5b6edf),LL(0x0c9d677b,0x4f5d7b07),LL(0x34b851c9,0x67daaa19),LL(0xade6f3a0,0xf4cef90c),LL(0x4e977f04,0xddf591d7),LL(0xa5a8e05f,0x51707198),L_(0x00000040), + LL(0x3afab893,0x8fe855a9),LL(0x6f37b397,0xfc53c756),LL(0x11e9e126,0x0be45fa0),LL(0x72b050d7,0x0064ab05),LL(0xfc69a454,0x2c4f8ec5),LL(0xe8ca3f9e,0xfd23251a),LL(0x3713bf5d,0xfbba4b98),LL(0xf9f17348,0x022b3ded),L_(0x00000105), LL(0xdf4c7717,0x496ec6da),LL(0x02b42501,0x3666ead7),LL(0xc5995675,0x03f0c9c9),LL(0xfafe4fc1,0x92c23f3c),LL(0x6e5dcc14,0x49d6013c),LL(0xa80274fd,0x6faecdf5),LL(0x3b89a119,0x6951f782),LL(0x04ebafd5,0xe5c75fdf),L_(0x00000059), + LL(0x2477dccc,0x4c63dd71),LL(0xecaaf994,0xdb563551),LL(0xfa0452c4,0x034243a2),LL(0x35dd1a49,0xc1e98b35),LL(0x67ebb1d1,0xd64a14c8),LL(0x65aa32b2,0xa40fe3b8),LL(0xf36b9b35,0x74b94620),LL(0xde048775,0x9c4f81d6),L_(0x000001ff), LL(0x2154a1a5,0x317a309b),LL(0x27e42884,0xea18f52e),LL(0xab7b0d80,0x6a303cb3),LL(0xd883eedd,0x6960a553),LL(0x5fa720c0,0x4a30771c),LL(0xaf1c642d,0x8c3afce3),LL(0x0aea4951,0x5b832460),LL(0x8daa7e14,0x3626a795),L_(0x00000086), + LL(0x084325f9,0x5078f4b7),LL(0xf8bc0fa4,0xf5948727),LL(0x1eb36b7b,0xea0106cf),LL(0xc704268c,0xc80e06d4),LL(0x300186e8,0x5c216323),LL(0x6e94d507,0x72ccb301),LL(0x882e86c3,0x053b8cfd),LL(0x3148a28a,0xd1710019),L_(0x00000143), LL(0x64ae003b,0x24c02876),LL(0x6d8c63b2,0xc0738276),LL(0xb8220989,0xd418521b),LL(0xa59b967c,0x52d761e7),LL(0x601efc41,0x0fa019e4),LL(0x41cb4d7c,0x9d52f11b),LL(0xcac0131d,0xe27a162d),LL(0xaac991e4,0x975ffa76),L_(0x00000189), +}, +/* digit=86 base_pwr=2^430 */ +{ + LL(0x0347ad37,0x53760212),LL(0xe9ad4077,0x52251dd3),LL(0x2f89c76c,0x5957d5e2),LL(0xb5844aaf,0x57c28051),LL(0x630bd361,0x369eb597),LL(0xacac6af3,0x5611a387),LL(0xaad0ef87,0xde2edc32),LL(0x0bccba64,0x731d170b),L_(0x000001f1), LL(0x5f219574,0xc2557f66),LL(0x48e1ee52,0x5f44f645),LL(0x0cf95561,0x64b6036a),LL(0x1e52bf2c,0x89213e75),LL(0x34bd858d,0x15aa570d),LL(0xbf1cb2e7,0xcc205ae3),LL(0xf29e703a,0xfb1911e3),LL(0x98a34a47,0x78616fc4),L_(0x00000095), + LL(0x7ca50557,0x301fc12e),LL(0x91d6adbf,0xdc42e25e),LL(0x587a0d31,0xcecd10d8),LL(0xf88cc8a9,0x8d1b41cc),LL(0x90bd959d,0x33b3f964),LL(0x999c5a51,0x47bee028),LL(0xc3a08825,0xb39f5f74),LL(0xd4fc3438,0x5af39ed4),L_(0x000001a7), LL(0x1f9ad9d9,0xbf02b54c),LL(0x0a29ee5f,0x4f0d6dbd),LL(0x57045bbc,0xe6fde293),LL(0xc7d0fce2,0x040027df),LL(0xcda45a33,0x23be51ff),LL(0x9841c14d,0x6d045486),LL(0x56ff1e33,0xfdc85221),LL(0x5ea634d9,0x619b696b),L_(0x000001ae), + LL(0x3ea82d4d,0xa104ae49),LL(0x5ed4f918,0x7f6fea1f),LL(0xfb0d6721,0xb83838c1),LL(0x99caa790,0xd116daa3),LL(0xc2f1d19e,0xe68d0873),LL(0x105c2468,0x247798a8),LL(0x672f66e0,0x740bca2a),LL(0xe789f0d7,0x86c229b4),L_(0x0000014c), LL(0xc9d1c796,0xd8e83bf0),LL(0x77be9153,0xaa767cd0),LL(0x71074af9,0xb3e46e21),LL(0x17f8ca53,0x7796d98d),LL(0x215b3c92,0xd098831f),LL(0x362429bd,0x0b75e1e5),LL(0x75d41b1b,0xc1e2cba3),LL(0x0101e963,0xbf3622fb),L_(0x000000fe), + LL(0xc6beb7ff,0xb9019f2f),LL(0x646b70de,0x2e533666),LL(0x90a5717d,0xd38f5274),LL(0x7916810f,0x3bb33bad),LL(0xfd952623,0x59b6b88e),LL(0x5dad71c0,0xfe6ed574),LL(0xca300788,0x21a98ac4),LL(0x6ded4a6a,0xeeff5891),L_(0x00000010), LL(0x03a67cf9,0x48da52e8),LL(0x98c1a5fc,0x1901e7d3),LL(0x42b185c0,0xcb2700b0),LL(0xaca4eda6,0xeec867cd),LL(0x0a19ae4e,0x7dcb7e93),LL(0x88471dcc,0x17e010a2),LL(0x198ed175,0xbb36f683),LL(0xbb4e8756,0xf915ed36),L_(0x0000007d), + LL(0x899986a2,0x56948883),LL(0xfd191432,0x519679d8),LL(0xcdcf1fa3,0x35711035),LL(0x87a0e867,0x663b1a45),LL(0x6c52192e,0xd46a0d9f),LL(0xd90e9a02,0x01cece65),LL(0x0dbf88f7,0x5032d5f7),LL(0xd5855448,0xc3ecbbe3),L_(0x00000083), LL(0x7299fb8b,0xda71595b),LL(0x4bb4a8ce,0x6609b74e),LL(0x23abfae6,0x76172b1d),LL(0xc43ff5b4,0xc2d69d21),LL(0x275e500d,0xf9af38b2),LL(0x8ab54bd9,0xe6a2d266),LL(0x983f2f3c,0xb11e43bc),LL(0x0d27f996,0x3ef88279),L_(0x00000141), + LL(0x235721d1,0x9058183f),LL(0xdd40e348,0x8259229c),LL(0x5e67db56,0x8582282f),LL(0xa5fd9ed5,0xea641a1b),LL(0x1a634c9e,0x6d89c3eb),LL(0x44359789,0x3d617332),LL(0xb7458f8e,0x974b6efb),LL(0xb6863988,0xd2857c41),L_(0x0000018e), LL(0x70c629ba,0xd0bd1dc8),LL(0x08757c7c,0x4edda34c),LL(0x5a31fc77,0xfa2bc7b5),LL(0xf6b159cc,0x36e4a532),LL(0x904f6b01,0x603edf4b),LL(0x96a6ebff,0xb699fdb4),LL(0xad76014c,0x888c04c9),LL(0x52da1b11,0x9675ab25),L_(0x00000151), + LL(0xee308f65,0x76cc2e7a),LL(0x23209d8a,0x39ac3634),LL(0x493f6e5e,0x2a49b5d9),LL(0xaac1e790,0x83449f3c),LL(0x85d5aef7,0x919c353b),LL(0x45144182,0x9673a9fe),LL(0xc1d4f2ca,0x9f1f328f),LL(0x8f5eeda5,0xf2216169),L_(0x00000042), LL(0x8673cd02,0x3f509f9a),LL(0x96feece0,0x85e87ba8),LL(0xe9be0b9b,0x186dcdb0),LL(0x7207011b,0x590eaaa1),LL(0x60f3a8d4,0x3aeb9b3f),LL(0xac89f276,0xd21f7163),LL(0xfd4e5813,0x05998e42),LL(0x80b7e4e3,0x38cd8e87),L_(0x00000006), + LL(0x2bd11682,0xc09d95ae),LL(0xd6f73873,0x7cafdfb9),LL(0x27b3bc9e,0x1f14d40f),LL(0x6fc05fa8,0x4eebf4e6),LL(0x4e6b76ed,0x7bb6036a),LL(0x736cb738,0x58e19e3f),LL(0x2277b08a,0xa4986beb),LL(0xbb55c0d7,0x8f830ada),L_(0x000000d1), LL(0x82356bcd,0xb25e59f9),LL(0x9a3ae563,0x18997059),LL(0xd06e16bf,0x7505891e),LL(0x3d688dac,0x76b3eb97),LL(0x10cb084c,0xc6498b2d),LL(0x2d6e8e95,0xafa574af),LL(0x503f71b8,0xc16133ca),LL(0x9966999b,0x6f3443fa),L_(0x000001d7), + LL(0xdbea5491,0xa62c57f7),LL(0x797a0980,0x03e966f7),LL(0x3a33197c,0x5565e857),LL(0x31df23bc,0x1feaa7a8),LL(0x240894a7,0xec48825c),LL(0x8f886413,0x9a2b6b14),LL(0x98c1f268,0x041a9538),LL(0x89254981,0x46c27078),L_(0x000001a7), LL(0x6917dfc0,0x3f850405),LL(0x16453f02,0x5e4fd155),LL(0xb6a3db9b,0xcd6306e4),LL(0xc259c564,0x89b34c35),LL(0x1cbd8b5f,0x7420eabc),LL(0x22f6885a,0xfe106029),LL(0x2bf9ec49,0xaef5fd1c),LL(0x73da5ef8,0x01c0496b),L_(0x000000d0), + LL(0x6be83fa9,0xccf43b6c),LL(0x2b2fd87d,0xb29b9f9b),LL(0x42c15930,0x75dead34),LL(0x8a992ae9,0x93d238f6),LL(0x835b46d8,0x537768fc),LL(0x59b7cdac,0x97a9b4f9),LL(0x8690f7d0,0xe523d489),LL(0xddd3b40f,0x2dcb85bf),L_(0x00000107), LL(0x29e746ab,0x2a6612ff),LL(0xa015a7b6,0xc599e46f),LL(0x72d01eaf,0x9b23b6a1),LL(0xc21d7f10,0xdff644dc),LL(0xac717daa,0x215cc0c4),LL(0x31113eff,0x29fa625d),LL(0x2da218c6,0x3ddc80ac),LL(0x8399eca4,0x5d17f283),L_(0x00000143), + LL(0x0f0161a0,0xfd958a4c),LL(0x656636ff,0x022fd87f),LL(0x77efdadd,0x3a9ac51c),LL(0x96597b30,0x2fbf14ce),LL(0x01364596,0x7d940bec),LL(0x1f7b85dc,0xbde90f3c),LL(0x3116d774,0x40e30d60),LL(0x9289d920,0x52c8b93a),L_(0x000001a8), LL(0x3eba46a9,0x2ebd850f),LL(0xa0eaa3d4,0x590d1f54),LL(0xbe22470c,0x53e4e7c3),LL(0xf88fc1e8,0xd8883aaa),LL(0x7782a946,0xe78d366f),LL(0x567fbeed,0xf47e81a5),LL(0xadd3d4f0,0xb2aaddfe),LL(0x29a661cf,0xa342a4e6),L_(0x00000027), + LL(0xfdf300d6,0x8c9cc5a1),LL(0xe5aa101a,0x329070ed),LL(0x53342a7d,0x852398fa),LL(0x007bb4fe,0x673ceb9c),LL(0xa9431236,0x3c9247b3),LL(0x1cb68268,0x115b1ee5),LL(0x5c07f2ef,0x3f6486b7),LL(0x7183b014,0x1585ec37),L_(0x00000110), LL(0xd84cedcc,0x0680da54),LL(0x9a2e8c00,0xe41a7e56),LL(0xe94c0472,0xb7c713ec),LL(0xf2aa41c1,0x742d18cc),LL(0x3e3162e5,0xedf3c376),LL(0x77417a84,0x57916b99),LL(0x165b44c7,0x246f2e2f),LL(0x1c45d988,0x8287828d),L_(0x0000015c), + LL(0x1ca07749,0xab8f038d),LL(0x27f0c816,0xc4edbd37),LL(0xc8522ec7,0x2f8f0b54),LL(0xe399477b,0x074481dc),LL(0x0fb69890,0xa0499848),LL(0xa5280b6b,0x88cb4213),LL(0x3dff5cd9,0x41c11365),LL(0xe549bd1e,0xea8a3c58),L_(0x000000fa), LL(0x19cc70e2,0x2b200b5c),LL(0x2ec791da,0xb1772087),LL(0xb33233b1,0x9df4451f),LL(0xaa1032a0,0xbaa72f9e),LL(0x645358e3,0xc7982461),LL(0x88b9794b,0xecd3c965),LL(0xcecd6313,0xa2d9359f),LL(0xf7b6ca2d,0x7f382df2),L_(0x000001ce), + LL(0xcb2fcc0b,0x02037c91),LL(0x41a52ada,0x82ddca3f),LL(0x1b403720,0x4b6a1108),LL(0x6acc9873,0x6d306779),LL(0x495190be,0x80fb3f34),LL(0x13ec341a,0x0307762f),LL(0x2ea03fd7,0x422a6d1a),LL(0xfacb4f11,0x014521e4),L_(0x000000f6), LL(0x869fd036,0x0ac6d565),LL(0xb343a25b,0xdead79aa),LL(0xf7c846e5,0x54b1471e),LL(0xccc5d545,0xf9ccfd6a),LL(0x31f5ccd5,0x43e993a3),LL(0x8fc6e767,0x07c79ed2),LL(0xd716a9ab,0x4894cc48),LL(0xae33e7f3,0xd6c5646c),L_(0x0000004f), + LL(0x7a287c20,0x37b15d9b),LL(0xc16b5265,0x43945dbd),LL(0x7a53a14b,0x994df966),LL(0xd272ba24,0xfbfe62a0),LL(0x80e0b451,0xb488717c),LL(0x7153f565,0x22303a33),LL(0x6397afa8,0xc85bf638),LL(0x0216bd4f,0x096b646c),L_(0x00000189), LL(0xf17f6fdf,0xf00721ec),LL(0x500e6e18,0xab5ca93b),LL(0xea2d8a0d,0x06a3b0f0),LL(0x44feaba9,0xf766ff9a),LL(0xea82aa95,0x05be709f),LL(0x0829c7c9,0x36a07ae0),LL(0xa1cfe409,0xc83c032c),LL(0x780ac746,0xde7f9ef0),L_(0x000001db), + LL(0xfdb1c069,0x18f9c015),LL(0x25767d44,0x49d6199f),LL(0x77c536b7,0xb54e847b),LL(0x1af54bfd,0x7979776c),LL(0x0b838623,0x51eefa22),LL(0xdf9bb4a7,0x028f18ac),LL(0xce45beb5,0x7dd86218),LL(0xb930f98a,0x3f055e3a),L_(0x00000102), LL(0x2c7f6a23,0x1df321b7),LL(0xeeb57c14,0x53902659),LL(0x0b2255cf,0xf2a776fd),LL(0x4cc9dbec,0x453cf8ab),LL(0x63e94ee6,0xf2d56478),LL(0x93a4007a,0x027149fe),LL(0x9cf116d0,0xa6376053),LL(0x17dc8184,0xe7465f73),L_(0x0000001d), +}, +/* digit=87 base_pwr=2^435 */ +{ + LL(0x4e8c16e6,0x6ea55f73),LL(0xf042fa2f,0xab1d226e),LL(0xb24c1848,0x6862a1dd),LL(0x413acbe8,0x1f4168e7),LL(0x91408365,0x9d596e07),LL(0x23961d18,0x01b379ca),LL(0x6d536797,0x05ec7b7e),LL(0x13cf35fa,0xd7f6b707),L_(0x00000120), LL(0x4c707b18,0xbc18785b),LL(0x76095f2d,0xa0054386),LL(0xe28a0370,0x50c89610),LL(0xfeeaf09e,0x144bba0b),LL(0x455cf10f,0x34cf6dd7),LL(0xf509d978,0xf94fe722),LL(0x05c279e5,0x8092debb),LL(0xe71244fe,0xb314f061),L_(0x00000153), + LL(0xbfafe94c,0x7f803868),LL(0xeb771971,0x05b4c2dd),LL(0x911e1ad0,0x0df34f87),LL(0x57076f1e,0x9958d5da),LL(0x6f49ecb2,0x55d1ebf6),LL(0x2ca7b49e,0xfcb4f571),LL(0xb2ff1b32,0x42a971c5),LL(0x49f97d3d,0x838bb327),L_(0x00000004), LL(0x789eb651,0xa52e7908),LL(0x86529c3b,0x7ddd9af7),LL(0xe7159473,0x6dd64d51),LL(0x8305400f,0x922bf016),LL(0x21ca4239,0x1db4bbaa),LL(0x8c94ee85,0xafda935d),LL(0x80623440,0x0a576c9c),LL(0xd9110efc,0xd79f58dc),L_(0x00000196), + LL(0x4946a27b,0xe1a7c929),LL(0xeff94575,0x7f265399),LL(0x076c7864,0xdf2046cc),LL(0xfe889ff0,0xe23267f8),LL(0x4b83615e,0x171f661f),LL(0x32637340,0xc3fd3e6c),LL(0x946ccba9,0xafb9463a),LL(0x9deb55e5,0xbdcb574f),L_(0x00000142), LL(0x21b6bfa5,0xd84d6148),LL(0x6362d37c,0xe1724e4e),LL(0x80e307d8,0x2dee8134),LL(0xd661095b,0x596062c6),LL(0xbc1b3be1,0x5779751a),LL(0x36f36543,0xd0be963f),LL(0x7fee02b0,0x75c65486),LL(0xb82030d1,0x72d27424),L_(0x00000046), + LL(0x9df95f7d,0xaeb38ccc),LL(0x49ee6062,0xc1411d18),LL(0x164333de,0xa4a4727a),LL(0x566e81b4,0x74b241dc),LL(0x290aa59c,0x85069fb7),LL(0x2865cb6d,0x28389d32),LL(0xcbd64839,0xce3c8f7e),LL(0x8c909864,0xfef248d2),L_(0x000000c9), LL(0x126259dc,0x4261d435),LL(0xb96bec85,0x62a6f5a6),LL(0x1f509bbc,0xb43f9c90),LL(0xc53ddbc8,0x94118466),LL(0x9ad3885e,0x22a38677),LL(0xd109dd2e,0x60b6db58),LL(0xeef4f2af,0xadf1adca),LL(0xd5bc0cd7,0x0fb47811),L_(0x00000022), + LL(0x04e04d29,0x456d2584),LL(0xca189af8,0x85bb0ebb),LL(0xf9e4ffa5,0x3e647534),LL(0x1136b7ab,0x0e1d213d),LL(0x91ac330c,0x6aa7c7de),LL(0x4345ad0a,0xf0255a49),LL(0xa156a357,0xe3967bfd),LL(0x2bdb3ace,0x8dff208b),L_(0x000001d8), LL(0xa955f611,0xad93b86d),LL(0xff9685e0,0x990e1837),LL(0xd599bc9a,0xa491c185),LL(0x320375e6,0x59cacf47),LL(0x427b05e7,0x03c76cd9),LL(0x9565d0db,0x073ece2e),LL(0x72f50d14,0x2045534c),LL(0x31281552,0xa43a812b),L_(0x00000114), + LL(0xb2b9d580,0x49f1b4d9),LL(0xd4601e45,0xd37e9635),LL(0x5712ad28,0x2c3143dd),LL(0xaf7e19b4,0xc6366f04),LL(0xaa565afd,0xc4b34637),LL(0xc12b452b,0xdd135b2a),LL(0x77fe7f5b,0xfeea8b42),LL(0x9ec6ff31,0x5cdaec8e),L_(0x000000c4), LL(0x8366af15,0x3f781317),LL(0x664c221b,0x652c3e83),LL(0xafac4ecd,0x6da93d03),LL(0xcfa7c466,0x2ceac0d6),LL(0x039d2b65,0xcb1a4cf4),LL(0xa6eb1946,0xa2285c58),LL(0x422c9b53,0x1b4d8367),LL(0xcc7349ed,0x4c55a379),L_(0x000000fa), + LL(0xa9e31a4b,0x85b7a89c),LL(0x1ea7ad0b,0x9f47c4ab),LL(0xe5150449,0x85cda5f6),LL(0xb891dfa1,0xe69386b4),LL(0xd89edbbe,0xf4ac48bb),LL(0x1b2a00ae,0x3c163ff2),LL(0x37bb423a,0x73d0c87f),LL(0x393a234b,0xe80b0fdc),L_(0x000001ac), LL(0x833e5248,0xc52fc82f),LL(0xa9154aa9,0xe56ec070),LL(0xb5b87f0b,0xb996ced1),LL(0x2c118a27,0xbb7f2dff),LL(0x3e0161d2,0x9ee991cd),LL(0x45e9acd1,0xb300f7d4),LL(0xfb934e98,0x20f357aa),LL(0x370589a8,0xfe8af1f8),L_(0x000001b4), + LL(0xac4010e5,0xb59ee173),LL(0x42f467a4,0x41b66fb9),LL(0x5770a301,0x02513df6),LL(0xf1d41988,0x7e0148f2),LL(0xf54abf2c,0xd9a1b6c6),LL(0xbb47b51b,0xfba956f2),LL(0xf5846505,0xb02618f4),LL(0x502a3ddc,0x69ec8c64),L_(0x0000007b), LL(0x3aa9231a,0x194b6953),LL(0x04316d8f,0x8f7bcb08),LL(0x006107c5,0x9f43afd0),LL(0x32f310f2,0xa15ea5dc),LL(0xe2b91ae7,0x3849a363),LL(0x2b4966c6,0x6a457445),LL(0x1d63455b,0xb8835c17),LL(0xcd39b535,0x1ae86f54),L_(0x00000109), + LL(0xdcd0412f,0x413915c5),LL(0xd3aa0f40,0x553e50ff),LL(0xc139c1fd,0x408079ac),LL(0xcaeedf51,0x5702513c),LL(0xf43dc271,0x1c08e5b0),LL(0xf5e1208e,0x48d91655),LL(0x813375b3,0x91b427b6),LL(0x0fa6be8b,0x833896b7),L_(0x000000d0), LL(0xbf42c4ce,0x7c3676f3),LL(0xac9ff585,0x853930b5),LL(0x94c9266b,0xc6b73b6c),LL(0xbc211c89,0x277b6c8b),LL(0x2fc248c5,0x93fd3dec),LL(0xcefb839e,0x4a5c85d3),LL(0x10bf217b,0x2a276f95),LL(0x3a708326,0xc15a4206),L_(0x00000147), + LL(0x614daa59,0x7bde1ec8),LL(0x34c4db4a,0xa70a6f08),LL(0x48a29f6b,0x587d1015),LL(0x49dbe231,0x998c9a20),LL(0x8aceafbb,0xe5eedcf8),LL(0xe6b738f7,0xcab08878),LL(0x9b693ecf,0xb374ede1),LL(0x008dd1dd,0x7a6cd94f),L_(0x000001be), LL(0x16a2f123,0x83bd130c),LL(0xc20757ed,0x17bff343),LL(0x228e06b2,0x1bef19ad),LL(0xc51046a9,0x0e88da5c),LL(0x011b5840,0x70a9f961),LL(0x49ae6f04,0xc6f83f90),LL(0x9d079a03,0x912f072e),LL(0x9a401435,0x0b78fe3f),L_(0x00000149), + LL(0x81fae141,0x20b6a171),LL(0xc0eec582,0x531fdbd5),LL(0x7a0400cc,0xe2cd895a),LL(0x9c308aa1,0xf0bb2d5a),LL(0x08b021d8,0xde68db65),LL(0xad0f2bcd,0xac747060),LL(0x3fd807ea,0xfe64802d),LL(0x8b4648ee,0x38d8773a),L_(0x00000176), LL(0xc5e737e0,0xcb637daf),LL(0x16605703,0xdffc80b5),LL(0x4845eee4,0xd28e60ca),LL(0x57f98d2e,0xdfda4fa3),LL(0xac95f77a,0x0d0b9220),LL(0x05da7201,0x60ec2cb7),LL(0x9a49b6df,0x201396c1),LL(0x611b2a93,0xb535c0ce),L_(0x0000000f), + LL(0xf4245fc8,0x1481fdff),LL(0x68c14a66,0x84edc501),LL(0x764d4d2b,0xbe356501),LL(0x736aaa60,0xe6771588),LL(0x0c40e330,0x5714d50c),LL(0x7b8fe887,0xfbdf2915),LL(0x9215aac4,0x549e25da),LL(0x58b4091b,0x442d2b00),L_(0x000001ad), LL(0xc5841383,0x88ee6a91),LL(0x934568ba,0x307efd84),LL(0x10dd1585,0xcb644b24),LL(0x3219b046,0x2bd376e1),LL(0xef0c68dc,0xdcecc49b),LL(0x56c2d2d1,0xc907f765),LL(0x810f8810,0x052b3ddd),LL(0x1ba20da5,0xc3448a3c),L_(0x0000019e), + LL(0x099d2210,0xb0648b7e),LL(0x193cb76d,0xf7fec768),LL(0xb90f6558,0x86126ace),LL(0x237a7fc7,0x749a6fe8),LL(0x83ab837e,0xb5c3035a),LL(0x3e9ae2d9,0x8de4bf68),LL(0x9c620970,0x0b3fa791),LL(0x8ef69888,0xc5e8388c),L_(0x0000005f), LL(0x8f4a5084,0x892086c5),LL(0xc04ac82e,0xfb491292),LL(0xc0d38a50,0x52d706d1),LL(0x42c8a5e7,0x582ce44e),LL(0xc9853494,0x96312a80),LL(0x04da6643,0x74ef5508),LL(0xbbc8dc30,0x9a8e3322),LL(0xfa669919,0xa1f29644),L_(0x000000d5), + LL(0xf4dadcf3,0x0e084366),LL(0x5cc0a55e,0x7ef0187f),LL(0xa139c3fe,0x49d53f7b),LL(0x5423f2e7,0xd809a727),LL(0x5a94a4ff,0xe2e74f9e),LL(0x541f08d0,0x22541929),LL(0x14dd0793,0x49159841),LL(0xdbf53ad1,0x408f5bb7),L_(0x0000018a), LL(0xdde4d64f,0x1db74ade),LL(0x46e28711,0x3b22deeb),LL(0x1ad3605f,0x8863541b),LL(0x1fe070fc,0xfdd530f5),LL(0x4af47e93,0x7f3d69c0),LL(0xd93cb647,0x6d16f551),LL(0xbc684cde,0x50cd6852),LL(0xb5154a9f,0x16ac0cc2),L_(0x0000009f), + LL(0x78202e6d,0xec5c2c31),LL(0xc90def9a,0xe4b46e4f),LL(0x67d0d316,0x8901b941),LL(0x2bb65bec,0xc96e5167),LL(0xf836eba2,0x096ab2fb),LL(0x996167b5,0x2719f2e6),LL(0xdde8e72c,0x12437287),LL(0x62d48d9a,0xf4ef64e5),L_(0x00000188), LL(0x8698b358,0x91da5b6b),LL(0xa56f46e1,0xc13b6841),LL(0x97107435,0xfbe3e2a6),LL(0xa446c520,0x759315a1),LL(0x0c5bba8d,0x861aec20),LL(0x852f2659,0x775fa0ec),LL(0xfbe06684,0xa91ab0fa),LL(0x03bd8b0d,0x3006d391),L_(0x00000023), + LL(0xcf090898,0x1b6190bc),LL(0xa4a386bf,0x17c47de2),LL(0x95703cbb,0x3bf84891),LL(0x3f013d22,0x12474267),LL(0x6fdb827a,0x0290f2b8),LL(0x50e9b7e1,0x79a8f44e),LL(0xcc658260,0x89a9228e),LL(0xab4d12b5,0x83a119d1),L_(0x000001b6), LL(0x93cc6375,0x2d25950f),LL(0x6b02229a,0x38c46b7a),LL(0xfb0617d5,0x6bc581dc),LL(0x0ce1dd7b,0x6b522d59),LL(0xd0dcdf5b,0x9133e3f5),LL(0x5cce47e7,0xd71f5bdf),LL(0x21b8ecd0,0x17d9aefe),LL(0x7aac21b7,0x7b609025),L_(0x00000188), +}, +/* digit=88 base_pwr=2^440 */ +{ + LL(0x35d8a1ec,0xf89fcba2),LL(0x59a63f98,0x86d07ca3),LL(0xf60025c0,0x590915cb),LL(0x68c18d4e,0x15cc7c3b),LL(0x85575ec9,0x09334801),LL(0xe8d10d82,0x4789511a),LL(0x82704b90,0xdb2e76c0),LL(0xf6a4e997,0xf5824d99),L_(0x000000c3), LL(0x8f32dc9f,0x6953628d),LL(0xa7575550,0x8504400e),LL(0x8537e141,0x609d8295),LL(0xc7b7f7a0,0x5da70118),LL(0xc50379c5,0x79ad1223),LL(0xc936f6ea,0xbde48629),LL(0x4f7f839c,0x1ba01725),LL(0xdff8def6,0x1bef09eb),L_(0x0000015c), + LL(0x5fe3f41c,0xe82eeedc),LL(0x330d665a,0x0753a4f9),LL(0x3f5e64a3,0x9e477096),LL(0xef9e92f3,0x07f9d297),LL(0x388062aa,0xc48c3ddf),LL(0x60ab0df5,0x55e6e61e),LL(0x5a47567e,0x9872a6f9),LL(0x3a66d012,0x425f368c),L_(0x00000153), LL(0xf66ffa47,0x03b7cc7b),LL(0xb2825eba,0xba3cef16),LL(0x90e67535,0x4aec5704),LL(0xcc34aef1,0x511ac67b),LL(0xd95c0e01,0x51002739),LL(0x0f4f3657,0x45e92922),LL(0x465557ab,0x1baabf91),LL(0x0e9abecf,0x8337c976),L_(0x000001e9), + LL(0xd2b325ae,0xa1025751),LL(0x6a01039d,0x28499cde),LL(0x5ba84622,0x47232500),LL(0x4da34907,0x523417ab),LL(0x54b07c1a,0xd3451baf),LL(0x3fa7e4ff,0x7ce5516f),LL(0x2fbff214,0xfc522cc4),LL(0xa33f1b0a,0x95c7010c),L_(0x000000b3), LL(0x5af51c66,0x665ed5f5),LL(0x980e5684,0xd596415b),LL(0xa5a1b30b,0x8834a37b),LL(0xfeebb04e,0xcf282494),LL(0xb29d17be,0x340dc6ce),LL(0x8d5399a5,0xa50f4a86),LL(0x76012bce,0x83faa312),LL(0x4bc769aa,0x6550a065),L_(0x0000018e), + LL(0xf4dbc144,0xec66fa0e),LL(0x134a53f7,0xa7b2871d),LL(0x1ee39cda,0x83070c04),LL(0x9749b3f1,0x6da77991),LL(0x867841c7,0xe916f1eb),LL(0x21e5438a,0xe409b274),LL(0x1b0e12d8,0x3842a6e5),LL(0xde5e08b8,0x74b9e008),L_(0x00000143), LL(0x63a63405,0xfea4cba7),LL(0x08e07acd,0x06789133),LL(0xdb2143a9,0x815c887a),LL(0x85ffe6dd,0xa9d2043c),LL(0xa68d05e2,0xd3ceab79),LL(0x93674d33,0x7a8a9863),LL(0x12ee73ca,0xd54b7afd),LL(0x6403b9bb,0x2eead112),L_(0x00000170), + LL(0x57f80d54,0xfa0b987f),LL(0x67c06145,0xfd55dd43),LL(0x34438e79,0xe8ca9c52),LL(0xfad0f9f8,0x810f12c2),LL(0xa97a7136,0xb3ec5af1),LL(0x3d0eabab,0xb7b58561),LL(0xecb3da01,0x8aadf26a),LL(0xbb015079,0x9cce9cad),L_(0x00000002), LL(0x43839606,0x265a72e5),LL(0xd025e951,0x90e3ddec),LL(0x5b2c9143,0x4955e972),LL(0x05386478,0xdae63ed0),LL(0x60c28f8c,0x4aa5ded8),LL(0x0fb99e77,0xb74c1dd8),LL(0x0f07854b,0x2caae0f2),LL(0x6691581c,0x069f6ba7),L_(0x00000148), + LL(0x6126647d,0xf5f13583),LL(0x8b738df6,0xe91f4420),LL(0x786c7341,0x1ae2188a),LL(0xda384ed9,0x08e3293b),LL(0x19b1a00b,0x9e09af31),LL(0x65267666,0x322f3662),LL(0xd07b9f37,0x764ea40f),LL(0x11ae129d,0xc16a911b),L_(0x00000185), LL(0x95ad18bc,0x59021aec),LL(0xeb3197c7,0x4daed80e),LL(0xdfd4a433,0x606234ad),LL(0xab3ff78d,0xf98a1d73),LL(0x9f90a43f,0x2c9cac66),LL(0x99bed176,0x5e8063cf),LL(0x8f03fcd4,0x50672a22),LL(0xfdb17bf7,0x027e080b),L_(0x00000079), + LL(0xfcf8e230,0x236a647c),LL(0x5dafe047,0x30081d74),LL(0xc3212b4d,0x0f548f13),LL(0x51f94578,0xd885e14f),LL(0x941059a0,0x06ed3092),LL(0x189c478f,0x5042651e),LL(0x7a26e8c7,0xf36b6ee0),LL(0x09a14b52,0x32dfdec0),L_(0x000001a0), LL(0x79eb582d,0xbe8ca673),LL(0x527f0a50,0x4c6beb6f),LL(0xadaba76e,0xfc7fd1fd),LL(0x9909b987,0x47c90091),LL(0x992155a0,0x06d6f45b),LL(0x2da697e0,0x740de37e),LL(0x1a38bcd1,0xce3867f1),LL(0x509b93e8,0x503be8b2),L_(0x00000160), + LL(0x1ab9de21,0xe8c16ca7),LL(0x4d3bbd16,0x519f4d4b),LL(0x53785c45,0x6454947e),LL(0x1aafab77,0xd9b9416f),LL(0x6883b419,0xb337e34e),LL(0x0208ba8a,0x7e584157),LL(0xab67774a,0x5a84d18c),LL(0x108ac516,0x77b69d31),L_(0x000001a9), LL(0x652943fd,0x91e5bcfd),LL(0xcd5e892a,0x5aa27743),LL(0x502744c8,0xa0414bf5),LL(0xe26bb91b,0xbc4ef773),LL(0x8bcd45f8,0x8f9e301b),LL(0x3589038c,0x30d42898),LL(0x9a5f5e5a,0xa609f771),LL(0xaf5c6671,0xade09eb4),L_(0x0000003a), + LL(0x284eb84a,0x775485ec),LL(0xf826fcc0,0xa66e99e3),LL(0xfac7759b,0x6006cfb6),LL(0x13b284ca,0x1fbb2a30),LL(0x53d194ad,0x2c2b6910),LL(0xf54ebb36,0xa49dd337),LL(0x46a6edea,0x8fc79498),LL(0x0d6aff86,0x842dc894),L_(0x000000f6), LL(0x954ada11,0x34121245),LL(0xb4cfd050,0xafb75e83),LL(0xaf8c43b1,0x77a38e5e),LL(0x4ff38619,0xe7485f16),LL(0xfa745e75,0x7e4f2466),LL(0xcdc30bb6,0x009d4a36),LL(0x9994c740,0x25e09cb4),LL(0x66ca76f4,0xf59131dd),L_(0x0000019e), + LL(0x7773c26b,0x6edaa49d),LL(0x0aa308ea,0x7669b865),LL(0x72c7a072,0xf06e514b),LL(0xa7ddba09,0xca616052),LL(0x126487b0,0xd64c8323),LL(0x713cc701,0xb7fd1abe),LL(0x84ce35d3,0x7bfbc16b),LL(0xba894fdd,0xba61b8c2),L_(0x0000009f), LL(0x27a3cc9b,0x18d74478),LL(0xa66248c8,0x62b773bd),LL(0x3c1ca3e3,0xde584f76),LL(0x5c3541fa,0x475ec797),LL(0xe7fdfe89,0xd692c26f),LL(0xe8463461,0xe888fcbd),LL(0x682f9099,0x1a8aba10),LL(0x0ae8eea4,0x2cc79e0c),L_(0x000000cc), + LL(0x2edafeba,0x901772ff),LL(0x6d000499,0x481b7323),LL(0xb7a27eb4,0x69fc5685),LL(0x4a7abcd1,0x47ec07ac),LL(0xcde4a9ac,0x56f5f84a),LL(0x45545bfb,0x6ef7da38),LL(0x33e7eca8,0x2edec324),LL(0xd8a46ddf,0xd29093de),L_(0x00000155), LL(0x94de9831,0x163bac52),LL(0xbeecc923,0x637f0966),LL(0x0af4893a,0x1759af91),LL(0x1d38f097,0xe0aea79f),LL(0xf9d81651,0x8ae541b7),LL(0x510d4c3c,0x32bd0e43),LL(0xd73faaea,0x6891a73c),LL(0x3864a690,0x6feafb02),L_(0x00000019), + LL(0xeacf95da,0xa0c16a35),LL(0xb6681c57,0x415f0571),LL(0xb2c83a60,0xcfe1f331),LL(0x4b9088d9,0x1279d3aa),LL(0x2ab5f2f1,0x29c29c20),LL(0xd47ee149,0x16735420),LL(0xfbb44304,0xb8379216),LL(0x23034403,0x20a6f133),L_(0x000000aa), LL(0xc23990b9,0xe0a94e50),LL(0x38217da2,0x2ba297d2),LL(0x18816b2c,0xe566aa72),LL(0xca63550f,0x1c7b21ca),LL(0xdfe51644,0x10c887fa),LL(0xef849ed8,0x4faeda58),LL(0xb92e8367,0x03636294),LL(0x2414c0ef,0x8476a050),L_(0x000001ab), + LL(0xfd6f68b0,0x93412483),LL(0x9f53d923,0x4403bdcc),LL(0xe30fa97e,0x6c9d0aa5),LL(0x1601e86b,0x9c1a2ec2),LL(0x19610105,0x431d5f14),LL(0x6cc0662b,0xb7bbdb4e),LL(0x84ed40f8,0x266aca0a),LL(0x1b8a27f1,0x198bae2a),L_(0x00000036), LL(0x95509e62,0x33afa5e4),LL(0x023ed8a6,0x8523afba),LL(0x036adaa0,0x83cbabb2),LL(0xf5cebadd,0xac3f99aa),LL(0x20899c44,0xca5f46cf),LL(0x0e94933a,0x7a04e2c1),LL(0x9a3fee46,0xb0015196),LL(0x367a01a4,0x1715a693),L_(0x0000001d), + LL(0xb429b2e7,0xc2d951af),LL(0xd89fbf0d,0x4488a068),LL(0x172ac7ad,0x7772ecaa),LL(0x0409a3f1,0xf7780ac8),LL(0x6d541e69,0x6ffa0a05),LL(0x4e8fccac,0x7509c471),LL(0xeff8ec93,0x018bbf89),LL(0x101b9048,0x2b2d5626),L_(0x00000102), LL(0x4bb1ca6a,0xbd338134),LL(0xda4a1896,0xaeb1aa9a),LL(0xdfd1cb54,0xc0b310a4),LL(0xeea1a455,0xbed91e2c),LL(0xaae7927d,0xd3502cea),LL(0xdfdf4808,0xd31ee1ce),LL(0xecc68f6a,0x893f08dc),LL(0x9350fda5,0x2a6f281d),L_(0x00000114), + LL(0xb4f8335f,0xd5d022dc),LL(0x6e654db2,0xe4aaf49d),LL(0xaa763047,0x24820282),LL(0xfe8aa2dd,0xef229292),LL(0xb7ff78ba,0x0170b38f),LL(0xe0a88558,0x0aca63f0),LL(0x66a526d3,0x97a4873e),LL(0xc069b5d3,0x28c88b56),L_(0x000001f1), LL(0xb3873204,0xa3de237d),LL(0xb57187d4,0x20e27844),LL(0x59762170,0xf8485db5),LL(0xf8fe71f6,0x47186213),LL(0xccceb1f1,0x3ddfa68a),LL(0xe9e1e35a,0x3805a749),LL(0x048090bc,0xeea89d03),LL(0xd04309c8,0x451591c1),L_(0x000001d8), + LL(0xe0dbf609,0x2e8fa162),LL(0x86c08a44,0xb15f83f4),LL(0xd94f9cd2,0x50bb6a89),LL(0x7bbf2a23,0x606cc572),LL(0x74b1325b,0xb03f198a),LL(0x73b79d3a,0xcf731a6f),LL(0xc95046a9,0x298efd11),LL(0x095ed71e,0xd622bb24),L_(0x00000074), LL(0xb59eae12,0x08c383b3),LL(0xb2f19275,0xe14dee81),LL(0x0d888be6,0xce4b12e8),LL(0x213fb612,0x78248f53),LL(0x43092c13,0x4330dbca),LL(0xe40c52b2,0x952b9ef5),LL(0x9d869889,0x31f1126e),LL(0xfbc05f41,0xfd03ae1d),L_(0x00000103), +}, +/* digit=89 base_pwr=2^445 */ +{ + LL(0xbdfd1e31,0x6e868b8b),LL(0x244e266c,0x05a4369e),LL(0x7c7bf40b,0xe296776b),LL(0xaed3b7e1,0x2cfd9c18),LL(0xddbc31e2,0x1ea90d63),LL(0x98abb7bc,0xe50b9291),LL(0x791f36a5,0xc2f87e55),LL(0xfe737c71,0x75c6d8e7),L_(0x0000004f), LL(0xd596ad7c,0xd54c9eb7),LL(0xa0fb486a,0x91d1e1e1),LL(0xd820f02c,0x9160a67f),LL(0xe5d16017,0xf1163f25),LL(0x8b61c557,0xfcbc9a92),LL(0x84ed79f2,0x6a33df9b),LL(0x54ba6955,0xc8febe18),LL(0x43c5cb8a,0xec5a3443),L_(0x000001a7), + LL(0xef36f15e,0xa725a8ec),LL(0x5828c615,0xf603a049),LL(0xac113424,0x87a77e81),LL(0x34642c16,0x761c2762),LL(0x9d0db298,0x29b1a474),LL(0x8ac3391f,0x143a9782),LL(0x050c5b69,0x5ae00925),LL(0xc578b0a2,0x144730ed),L_(0x0000016c), LL(0x4e6437c4,0x3bbfa384),LL(0x606aeb93,0x6e3daf55),LL(0x97e41356,0x4263527e),LL(0x4ac1c1ae,0x3a037893),LL(0x8c336382,0xb2143f58),LL(0x2bb7d997,0x69412726),LL(0x2419935b,0x9cb555c8),LL(0x724eeef9,0x2ef7f7cb),L_(0x000001b8), + LL(0x463c9476,0xf81335ca),LL(0xd6526151,0x999ff056),LL(0x3c494f0f,0x0a7433ca),LL(0x41b82dc1,0x7fa3bcdc),LL(0x9af11e06,0xec803bb3),LL(0x0ac7bb35,0x457b31fc),LL(0xb5e185aa,0xed555915),LL(0x586ab2cb,0x33044819),L_(0x00000097), LL(0x2112108d,0xbfc07b3b),LL(0xae813666,0x4f0a957b),LL(0x8eee1f42,0x82cf0958),LL(0xc3321225,0x4daeb7bd),LL(0x458ec031,0xe4de4e23),LL(0xd0f97884,0xfc50a768),LL(0x1655c201,0xb424f36a),LL(0x14f1a537,0x0cdff481),L_(0x0000006c), + LL(0xf3b637b5,0x22f8cb24),LL(0xf131c203,0x0c2b076a),LL(0x815ccfff,0xf056364e),LL(0xdbdfbdbf,0xc8028853),LL(0x41ab5760,0x8af0ee08),LL(0xca93ac08,0x3094da56),LL(0x30135092,0x5054010d),LL(0x74228a25,0x8e7dde67),L_(0x00000005), LL(0x5c512c3d,0x8a5176aa),LL(0x3779fd86,0xbe16420d),LL(0x7658fb3d,0x41f45c5c),LL(0x110cf130,0x49dea64d),LL(0x19e0e350,0x73f6746e),LL(0x87ca4575,0xfe7da390),LL(0x108ab4e2,0x874c5458),LL(0xe39cce4e,0x1d64965c),L_(0x00000179), + LL(0xb7099c8e,0xfa76cb01),LL(0x80465e82,0x38560d7e),LL(0xf4fd03ea,0xa649c8ff),LL(0x150b3815,0x72398c4a),LL(0xc0e6baed,0x1ba3da88),LL(0xfa79ad8b,0x6f43120b),LL(0x8353fc42,0xbd32e2fe),LL(0x7dbd7876,0x148c548b),L_(0x000001d9), LL(0xea549c25,0x18cf351b),LL(0x9ca7db2a,0x80485f13),LL(0x16240b9e,0xdfdb85c4),LL(0xc2dc15ee,0xbb4121d8),LL(0xcce3d597,0x0e963371),LL(0xafb37db7,0x7c69e287),LL(0xed3b5fd8,0x6c8d52b6),LL(0x608eaea5,0x053f2384),L_(0x00000165), + LL(0x625fe0c4,0xbbf62c47),LL(0x1eac543b,0x11eca801),LL(0x4e45d301,0x682f5663),LL(0x054cb071,0xe9473698),LL(0xf1ad1950,0x860a714e),LL(0xa8a339c8,0x96d39034),LL(0xe04ab8cd,0xde22b09f),LL(0x2a845e02,0xdfec2116),L_(0x000001e6), LL(0xaad333d8,0xeaca7b49),LL(0x3e7c928e,0xb0fcae9a),LL(0xca8b3e2f,0xc41fdaef),LL(0x9529863b,0xe1843977),LL(0xd56a624a,0xee7e83a7),LL(0x3438606a,0x81db821d),LL(0x06cdb198,0x3aa0eeb4),LL(0x9b12775b,0xe0d60750),L_(0x000001da), + LL(0xb1f4b70e,0xbe8e1de1),LL(0x9b0a79d8,0xc345fcdd),LL(0x77d93da0,0xb2f5a213),LL(0x7028f9ce,0x800fcc19),LL(0x1306f2ff,0x469efe59),LL(0x4cd68bf6,0xc5ffe046),LL(0x62b03f93,0x53010575),LL(0x5af4940d,0x46961f0f),L_(0x00000171), LL(0x354ba888,0xad6952b3),LL(0x9045b751,0x43f410dc),LL(0x217ebd7d,0x0d11a22c),LL(0x6ddeefda,0x6d1775a7),LL(0x43965993,0x055c0203),LL(0xe7060f57,0x3548b71e),LL(0xa89da1f0,0x805eb428),LL(0x6f8231a6,0xf7b78a97),L_(0x000000f7), + LL(0xb3368550,0xcf1f7c72),LL(0xe247b483,0x97b6bf76),LL(0x202781dc,0xc0f81747),LL(0x8b65bb58,0x92efba88),LL(0x9611a60c,0xd9612af5),LL(0xaf54a57c,0x20d7ccbe),LL(0xf8689ba5,0x6d3cbf9e),LL(0x0591cc36,0xdc1abfe9),L_(0x00000056), LL(0xcd538940,0xd4a04bec),LL(0x1e600b02,0x2a15021b),LL(0x6c3ebe8f,0x9586be60),LL(0xb8507cfe,0xf4028af5),LL(0x54dda762,0x4d392e89),LL(0x519d3758,0xbde8dadc),LL(0x58c3813e,0x81db641b),LL(0x91557ce6,0x23fa3b99),L_(0x00000063), + LL(0x3b03dea1,0x51bb00fd),LL(0x32b04d9e,0x8aefebe4),LL(0x22b78b18,0x8698b63c),LL(0x7da3c01d,0xa71b8bc0),LL(0x8d71ee46,0x27b7a39c),LL(0xb0583313,0xbd156109),LL(0x49d2846e,0x931258ab),LL(0x86e6af4e,0x3ca87258),L_(0x000000b6), LL(0x429e4df7,0x79f7c689),LL(0x39041060,0x6229b813),LL(0x9028538d,0x3a4aa59f),LL(0x517bfaf7,0x2d1cb542),LL(0x71d33bc2,0x882030de),LL(0x9ba76285,0x91ba5fcd),LL(0x25f86ca8,0x9ae0fc6d),LL(0x47f08f0a,0x3948678f),L_(0x00000089), + LL(0x22c29c44,0xcd9eb593),LL(0x18cd9b8b,0xac677eb2),LL(0x0d8705ef,0x6b203fcc),LL(0x934fa783,0x39fcfd85),LL(0x571b28eb,0x58bd6d8e),LL(0xd8f1d221,0x215fad4b),LL(0x3e44e705,0x827adc24),LL(0x5ff00393,0x1ec35c0f),L_(0x000001ac), LL(0xc588165d,0x14fc0a02),LL(0x5fce2e10,0x71f9384c),LL(0xf0f2ac5b,0x90d699f4),LL(0x7b00891b,0x43b6bcdf),LL(0xe8c4a652,0x7bc04d87),LL(0x0ac9f698,0x2ab126b5),LL(0x3eb3d860,0x849b38d0),LL(0x426d6e94,0xb6985535),L_(0x000000cf), + LL(0xe89608fa,0x0725d65d),LL(0x1183558e,0xb6a14f9d),LL(0x44070253,0x20d9075f),LL(0x6c243902,0x486b1799),LL(0x6c1a9d8a,0xf5efa075),LL(0x8ae5a14e,0x4ea72292),LL(0x2d7b9c93,0x0ca5c12a),LL(0x992cae02,0x91e3345c),L_(0x0000005b), LL(0x306b8949,0xc89bcdf7),LL(0xd10410ec,0x89966bf7),LL(0xcf680bd5,0x6ee731b3),LL(0xa0c3db72,0xe37f14f5),LL(0x2aa5a376,0xf554bdb7),LL(0x23be47dd,0xef1712c3),LL(0x96ab9b1d,0x1c7594e0),LL(0x9ed66d28,0x032ce687),L_(0x0000003d), + LL(0xc8516fe7,0x9e3351dd),LL(0x68d68f89,0x9fb7334d),LL(0x40ebf359,0xc5209aaa),LL(0x120177c0,0xe5d00b75),LL(0x2f0e6bbb,0xbf188e69),LL(0x110d2427,0x8e2e5213),LL(0xd6344a1b,0xdcf577cf),LL(0xa7331f94,0x3c553feb),L_(0x0000013c), LL(0x64f458fb,0x795a2fa2),LL(0x524f4a9f,0x6609f22b),LL(0x6b23609b,0x2c95b3f8),LL(0x0500bc47,0x8df999a3),LL(0x042c79e9,0x9db59925),LL(0x12a07a8b,0x55be1532),LL(0x07f62419,0x33c89540),LL(0x8df78722,0xfe671ad7),L_(0x00000141), + LL(0xe38c8109,0x18059a16),LL(0x52e9bed4,0x717c36f7),LL(0x49d5e825,0xb56dd6dd),LL(0x7783b6cd,0x667fac4c),LL(0xcf53b558,0x116a1985),LL(0x7c15cf14,0x9913c6ee),LL(0xe08410c5,0x6728a2a4),LL(0x9d771edb,0x331fb13d),L_(0x000000be), LL(0x34b06991,0x394ecd2b),LL(0x8da76d72,0x341e75ff),LL(0xf52f78d6,0x29ea6d71),LL(0x46d211ab,0xaf402bfc),LL(0x386ae83b,0x7e9586dd),LL(0x909f5bf1,0x11c7f555),LL(0x1b8a537f,0x427868fe),LL(0xcf05f9d7,0x32daf130),L_(0x00000055), + LL(0xc5d1aadd,0x4ef2bfbd),LL(0x360e62a2,0xc1081697),LL(0xa6a207aa,0x28d01fdb),LL(0x18abc7ac,0x204fc30b),LL(0xbcff0be1,0xe5cdb570),LL(0x48ef40e1,0x1f0b1c2e),LL(0xeb79790f,0x63136e14),LL(0x3d4fe961,0xb9d45c94),L_(0x00000140), LL(0x79d14142,0xa61088da),LL(0x8938b0fa,0xb39c86bd),LL(0xc33f1d7c,0xa2380177),LL(0x530d6911,0xaab3667d),LL(0x7b52bed9,0xd815d83b),LL(0x5c596749,0x44b95fe0),LL(0x5148c157,0x202c91ff),LL(0x406b7485,0x8bf24d49),L_(0x00000006), + LL(0xf808d1d3,0xd37bc919),LL(0xe29da36d,0x00b56fef),LL(0x7458f713,0x8621718f),LL(0x286883b4,0x448b7c11),LL(0x363d4ba5,0x6114fd6e),LL(0x04011c7c,0x0d4b7500),LL(0xe765f7ee,0x491c6545),LL(0xc2b827eb,0xd01f3320),L_(0x000001f5), LL(0x464d4102,0x6902bd96),LL(0xda599389,0xe2b47365),LL(0xfe7e3528,0xe9079def),LL(0x3aa4556e,0xc96d3bc8),LL(0x610e35fb,0xb585febd),LL(0xa0b2ea82,0x70988a63),LL(0x60d1db4d,0xd27f19aa),LL(0x6eee4c02,0x248d0f40),L_(0x00000073), + LL(0xa17e11d6,0xba327209),LL(0x4da0ba85,0xfac1ed29),LL(0x2e3b7145,0x48cf218d),LL(0x5cbfef12,0xde112f17),LL(0x76f3e234,0x194a8f16),LL(0x65787086,0xde1af9c2),LL(0x18958d56,0x495c76a5),LL(0xdd3dbcba,0xa5e9c9c9),L_(0x000000bf), LL(0xf9f1e953,0xb1fcebaf),LL(0x30a1b712,0x73d82709),LL(0x8296f1f2,0xfa6e1f41),LL(0x5ef71edd,0x7dd19081),LL(0xc4a2f8af,0x2f6fda9b),LL(0x85b1234b,0x541a4825),LL(0x23556036,0x79e6b22e),LL(0x911ac1cc,0x88ea71f9),L_(0x00000199), +}, +/* digit=90 base_pwr=2^450 */ +{ + LL(0xd86efa4b,0xa22cc5d8),LL(0x8ee779a0,0x5dd5d86c),LL(0xfd215954,0xfd5e2c81),LL(0xab1c7262,0x75f13cf7),LL(0x4f36ad82,0xe759a0b7),LL(0x8c3ddc91,0xd2223c0c),LL(0x10948a51,0x9b2c7f7d),LL(0x977160cf,0x285822b5),L_(0x0000000f), LL(0x92851c33,0x1a1e9623),LL(0xd7c127ef,0x7e5e2b50),LL(0xd984c528,0x3999dfdf),LL(0xaad5ce7d,0xd1373907),LL(0x5c84726f,0x97f8f082),LL(0x5ebbc32d,0x68dcb5c5),LL(0xbd51b3a0,0xa1b4f592),LL(0x36935287,0xf3eb9dca),L_(0x000000c8), + LL(0x56f967bd,0x95c0c51d),LL(0xdbb76844,0x7bb768dd),LL(0x35dbc45f,0xeda49098),LL(0x39df9e6d,0x3639006a),LL(0x47f77ee7,0xd878e5a2),LL(0xd141b2c8,0x2c8ccd83),LL(0x04a47e33,0x2d4027f8),LL(0x2c9dc7a0,0x9934bb00),L_(0x0000011b), LL(0x8efe0042,0x8d777a83),LL(0xd9919c1d,0x0f685368),LL(0x9dd72165,0x892863f1),LL(0x2a92b1de,0x8f2b25a3),LL(0x90ff3dd7,0x12a43206),LL(0xaf7bb8bc,0xe03505a4),LL(0xd763efcf,0xcf4f256c),LL(0x53701c70,0xef267753),L_(0x00000043), + LL(0xf628351d,0xecb197aa),LL(0x593ffafe,0xdd6e37e8),LL(0x40fd5b94,0x41331e2a),LL(0x298fef4f,0x60fb849c),LL(0xfd38fb42,0x7e149e93),LL(0xe83fe7b3,0x22e02e59),LL(0x5d08e682,0x58b84ec3),LL(0xcaa4bfd0,0x7f8e6b6a),L_(0x00000004), LL(0xc3c8a59d,0x89faa591),LL(0x3bdb5b23,0x6e000f52),LL(0xa824fd00,0x630a4795),LL(0xcbf7e717,0x4e000837),LL(0xc37102e6,0x5656508a),LL(0x40d36c3a,0xe0b06b84),LL(0x0a694f94,0xd89beeae),LL(0x647088c8,0x682c3563),L_(0x000001d9), + LL(0xd4220577,0x49e0800c),LL(0xd1b7504d,0x59ebe077),LL(0x04fd80e1,0x714afb4f),LL(0x90ea18a8,0x28810d8b),LL(0xf02c3cde,0x719cff83),LL(0x19367a86,0x8786eb9e),LL(0x952bac43,0xecceb4e9),LL(0x460e0748,0x55aefa66),L_(0x000001aa), LL(0x1f2623e4,0x7aaa315b),LL(0x44f96e09,0x7a5db2b5),LL(0x5dae237e,0xa9362519),LL(0x163873d0,0x69799223),LL(0x4d0fbf55,0xc1a58ea8),LL(0xd3bb728a,0x661ed43f),LL(0x100cfe43,0xf1cd21af),LL(0xa24f55c6,0x25dcbe9f),L_(0x00000118), + LL(0x33be448b,0x5ef36acd),LL(0xd5225f3c,0x5770b7a0),LL(0x90f00a62,0xebdaa1b7),LL(0xc0ab750b,0x1bcb88b8),LL(0x4d9be029,0x06bbf584),LL(0x9dfcba75,0x606f29fe),LL(0x74e426d7,0xd113e261),LL(0x2931cda9,0x0453e382),L_(0x00000056), LL(0x2b727cee,0xba1e3830),LL(0x95a907f6,0x1922ca15),LL(0x760c0c2f,0x24719cf1),LL(0x383ccd2c,0x11f794fe),LL(0xa0495e03,0xaf40e690),LL(0x3eba817e,0x1fab7cdd),LL(0xa83e8359,0x9846062a),LL(0x737b3c03,0x52241afa),L_(0x0000005b), + LL(0x747efd45,0x2d0ad5b6),LL(0x49587f7b,0x321154b0),LL(0x12ada5b1,0xdc3aa007),LL(0x0fe792e5,0xf996b5f9),LL(0x0e4944ed,0x8197e83f),LL(0x06340a72,0x6a72b742),LL(0xb3002f8a,0xbc8a8319),LL(0x173328b5,0x81f8ab11),L_(0x00000139), LL(0x964808f4,0x2774c6fa),LL(0xbe5f49f5,0x674e04ad),LL(0x8fbf7faa,0xe6de1313),LL(0x5724658c,0x38fee508),LL(0x6a0cb3e6,0xeb3e2c17),LL(0x24438695,0xe7eaef00),LL(0x43ac8a73,0x4dc94b9f),LL(0xd190f6ea,0x422b705b),L_(0x000001cd), + LL(0x1bd57124,0xd43e3b34),LL(0x02b39b5b,0xd46524f2),LL(0xabfa1c64,0xa4dd7015),LL(0x8ec6eade,0xfac38f67),LL(0x78cba481,0x1123582f),LL(0x61a4550d,0x1caa4894),LL(0x42f7ada2,0x83747c68),LL(0x17f7f74c,0xffb17df5),L_(0x00000130), LL(0x02e47bb1,0xeb2b93af),LL(0x7f0ef78c,0xcf301d2b),LL(0x8e9f267c,0x8e246b2f),LL(0xb4a2acd2,0x2035c962),LL(0x50846229,0x97e899e6),LL(0xe23609a5,0xfbcb2b53),LL(0x1483eb63,0xfc3f203e),LL(0x4d6ddbe6,0x2861a320),L_(0x00000039), + LL(0x5858d75c,0x41fc794a),LL(0xffcd84d6,0xf5985a4a),LL(0x5082ece4,0xcf3bb3f4),LL(0x850b4853,0xb1d8af65),LL(0x670d980b,0x6953dc3e),LL(0xf579458a,0x7963424e),LL(0xac2f2e4a,0x540b6858),LL(0x920d771e,0x1f5fed22),L_(0x000001be), LL(0xc4864af0,0x731be223),LL(0x662c4dc5,0x1419cfbb),LL(0xa5701752,0xd65099ca),LL(0xfbcc0240,0x3af88f3b),LL(0x1643acb1,0xfbc4861e),LL(0x67405bcd,0x35f067ed),LL(0x9351f1c8,0xcb8018ed),LL(0xeed0e188,0xd276f971),L_(0x000001f8), + LL(0x7974a311,0x8cc00e47),LL(0xd167d662,0x311413fa),LL(0xb1947f2a,0x68fd100f),LL(0xc373b68f,0xd96895d1),LL(0x259f8c2b,0xb6277660),LL(0x495d6470,0x6dd59691),LL(0x9eee1f91,0x8e4a7fc7),LL(0x1b01dab7,0xf1319245),L_(0x0000009c), LL(0x2efe3e85,0x3751b5a9),LL(0xb3dd751f,0x352c6ed0),LL(0x36c470b2,0xbb64e49d),LL(0x58925906,0x5b0b62a0),LL(0x9089d01b,0xe64e7de4),LL(0xf631915c,0xcd161d83),LL(0xfa1f87e7,0x44c46466),LL(0xcced2cc6,0xec7eb165),L_(0x00000125), + LL(0xda580a2f,0x0360b595),LL(0xc1265889,0x3fa41625),LL(0x9e6d3563,0xa19100cb),LL(0x10accaeb,0xaec86fbb),LL(0x7b3f3d8a,0x80771b15),LL(0xdb2ccbe9,0x803f9c49),LL(0x07a460fa,0xf34e5b14),LL(0x4b602490,0x6a99a6e6),L_(0x0000005d), LL(0x47562df1,0x308acd32),LL(0x65b0ad8e,0x3dceea03),LL(0x13ece697,0xe1bef19b),LL(0x0643badc,0x4cbdd893),LL(0x470bd8ab,0xafc33073),LL(0x39f6bceb,0x17b3cfed),LL(0xf3aad086,0x198868e1),LL(0x0e329726,0x9f6251f8),L_(0x00000016), + LL(0x2910e070,0x2c636d48),LL(0xd1bce1b0,0x51f92c94),LL(0x1ad3a1e4,0x88368755),LL(0x9be2c281,0xd8124a18),LL(0xe3a680f7,0x9e5bc7e6),LL(0x5e952f4f,0x2fefbd16),LL(0x5cef9135,0x19b6c616),LL(0x5576fffd,0xbf997c16),L_(0x000001f9), LL(0x7be439b2,0x01681747),LL(0xdbf38f7d,0x00a3bfc8),LL(0x549f1e54,0x39cded9a),LL(0x23fdd541,0x8b94ded9),LL(0x89eeca5b,0xfd084a5a),LL(0x5123eaa5,0x834be49d),LL(0xae42403e,0x02444c83),LL(0x25abfb57,0xbcb65841),L_(0x00000091), + LL(0x9621f3da,0x606a9e44),LL(0x91ee418d,0xd2dd052e),LL(0x7e5ea0e7,0x92e787f3),LL(0x69d6f73c,0x4508ea48),LL(0xfe248ecc,0xa2d461ac),LL(0x529ffcc1,0x22dcbd24),LL(0xf90dc5dc,0xa3364562),LL(0x542f8abb,0xd254e4f9),L_(0x0000016c), LL(0xc0262fa6,0x990aa036),LL(0xc4f4234a,0xb59ee2ed),LL(0x1031cef9,0xd5d4b081),LL(0x984145d5,0xdf2c037b),LL(0xc7b07787,0x2a2d6af2),LL(0x31d56853,0x6d5e6ff4),LL(0x309a7c9b,0xbb40c66e),LL(0xda6a3ee9,0xaf9db41c),L_(0x00000081), + LL(0x959a8be0,0x2ab124d1),LL(0xf806ef78,0x2f7a4e31),LL(0x0cea295c,0x32a4553d),LL(0x8a3bdbfd,0x19971283),LL(0x3b9e4766,0x8810a423),LL(0xcdcf9f57,0x8e85dc85),LL(0x020a6262,0x129bd8b1),LL(0xbc50d2d2,0x9a64395d),L_(0x00000068), LL(0x5b27b0a7,0xde8938c9),LL(0x2bd0d178,0xb0b45608),LL(0xde7e99ed,0x36896362),LL(0x3851afd1,0xe2cc94c2),LL(0xdfbefd8a,0x14aa57ae),LL(0x3c1abef1,0xd8652bb1),LL(0x0cc39736,0xaf001e99),LL(0x72b91536,0x9fdf9a10),L_(0x00000157), + LL(0x29021dc7,0x4403c833),LL(0x48e45088,0x26afc7d8),LL(0x41225474,0xa4629460),LL(0xcc7a0cd5,0xff951c59),LL(0xaba7cf1f,0x0d1f3526),LL(0xaf07c8ca,0x18426df5),LL(0x2746f85b,0x42d2e91b),LL(0xfabc76e4,0x58debcd4),L_(0x0000017d), LL(0x888a4dcd,0xa9da44e7),LL(0xeb092c08,0x1d496006),LL(0x0b68b024,0x67bb5005),LL(0xbf7c5c64,0x0562f97c),LL(0xca481cfe,0xb0ce28f4),LL(0xc88ad0f0,0x5bcd7411),LL(0x79f82640,0xcfb08fb0),LL(0x9dac9879,0x22beef72),L_(0x00000060), + LL(0x4092ef3c,0x055ab7d2),LL(0xb9344cd4,0xcbb171f1),LL(0xf24efa25,0x63c2684d),LL(0x4f8a3dbe,0xf0b702dd),LL(0xda3c37c4,0xa46de3f5),LL(0x17afe9a8,0xdeacde9a),LL(0x55f90e6b,0xb3ba7f88),LL(0x4d24b1d2,0xa174d6f6),L_(0x000000cc), LL(0x8d8e436b,0xbb7dee1a),LL(0xa172733c,0x562c1d69),LL(0x4325a01b,0x851c2792),LL(0x2307ef6a,0x99968ef0),LL(0x69918f4f,0xb12cbce1),LL(0xcd378fb1,0x9c21abf8),LL(0x48639036,0xe801ee02),LL(0x8268fb51,0x7205404d),L_(0x000000a5), + LL(0x7a194c8a,0x3a0f6f06),LL(0x7ad7abcf,0x531ca66d),LL(0x41cf832b,0x4ac4965a),LL(0x0f9f470a,0xe00766a0),LL(0xa92657cb,0xb432af80),LL(0xac40c892,0xa94b8968),LL(0xbd44eded,0x4be8b74c),LL(0x2a8d4620,0x98f760bd),L_(0x000001dd), LL(0x2479db10,0x2a7f464f),LL(0xdb3e7dcb,0x00b58c0f),LL(0x1a96b289,0x3c1d7ee5),LL(0x7a30a299,0xbf0ce935),LL(0xb49f5f57,0xf1f5d39e),LL(0x8f0c1970,0xac9cfb8b),LL(0x0718d4fa,0xe1c25a36),LL(0x66ec4ed9,0xb0d7504b),L_(0x0000004b), +}, +/* digit=91 base_pwr=2^455 */ +{ + LL(0x9db6d8d2,0x72de604b),LL(0xda62b655,0xfb7d9262),LL(0x9db8d0d3,0x8e9c2aa3),LL(0x9b867b7f,0xf2912d3c),LL(0x1a5ad674,0x279e6d83),LL(0xc6935b1c,0x82236f3c),LL(0x9a75e08b,0xfcf8f6f0),LL(0x1baaa28c,0x5ff40727),L_(0x000000f7), LL(0x120a90c1,0xab24706c),LL(0x991d9aa4,0x2ed85b4f),LL(0x767e0695,0x0793e3c1),LL(0xf7d06ffc,0x6115d975),LL(0x1316443d,0x9c57472c),LL(0x0b8651df,0x8b972443),LL(0xda1a64f2,0xf1bbd8db),LL(0x0c6db846,0x62aea165),L_(0x000000f5), + LL(0x6eab9379,0xd5d810c8),LL(0xe6eb51bd,0xe42198e6),LL(0x9bc2e3a0,0xc4b3dc48),LL(0xeda7e391,0x6ed77fe5),LL(0xcd0e3d73,0xc5e60972),LL(0x05f70f41,0x26a51aaa),LL(0xb07669f4,0x9830a47d),LL(0x34591483,0x45b98cf4),L_(0x00000090), LL(0x69325242,0xdc9d9c57),LL(0xf4c3b8bd,0x95086409),LL(0xd467dc92,0x3e6cf0fa),LL(0xfbebdef2,0x9684c1e0),LL(0x1daa3a72,0x8c3a2301),LL(0xf40ca0da,0x850f8c4e),LL(0x4dda12c0,0x990ccbe3),LL(0x8f2c5e51,0xc2f0adaf),L_(0x00000094), + LL(0x45c80cae,0x248df475),LL(0xb7ac228f,0x869c271f),LL(0xc18f18ee,0x46d75c07),LL(0xb3a5cead,0xa1a299f1),LL(0xf8c96489,0x7ba98e94),LL(0x805d6dbc,0x1a1d6b09),LL(0x485c463f,0x5bca1865),LL(0xfae82626,0x54594fd7),L_(0x000000b1), LL(0x88e41bdb,0x13e4d735),LL(0xb2ef61b8,0x5d0af04c),LL(0x3f79b3f0,0xe86965cf),LL(0x8df70be4,0x6a326017),LL(0x93f5bd3a,0x59b253c0),LL(0xcbf1399d,0xa14a5e9e),LL(0x5112a46b,0x98f7dd60),LL(0x84b48e76,0x9c0c1a6e),L_(0x0000005f), + LL(0x390065e7,0xf887b36a),LL(0xa9bb0064,0x651f2c93),LL(0x4572329d,0xd988aed3),LL(0x6f510e01,0x48541e9e),LL(0xabc023be,0x90ac10a8),LL(0xa8621efc,0xf943b700),LL(0xeb208400,0x768bb3b3),LL(0x64d85b1c,0x634af0db),L_(0x00000142), LL(0x250b4be1,0xa7e0a10a),LL(0x4e42e593,0xacaa8063),LL(0xadef0026,0x6f2f96cb),LL(0x955002da,0xca66aa2e),LL(0x57271d8d,0x15a69e81),LL(0x66dc7629,0xcf29f326),LL(0x378977dd,0x07d6619c),LL(0xe10e5eaa,0x48e47a94),L_(0x000000a5), + LL(0x9612b84f,0x60476dab),LL(0xd5ce8836,0x1ca95649),LL(0xd950d005,0xf0b56f1d),LL(0xe5ff1bc5,0x06d36f22),LL(0x6e683386,0xe2bbd6a0),LL(0x249d13ac,0x69fc343c),LL(0x15998eb3,0x47a41e75),LL(0xf1860545,0xc5dfbed2),L_(0x00000169), LL(0x972e1e1b,0xc1e80885),LL(0x5129b884,0xf74074e8),LL(0x9d124cde,0xb7fa9540),LL(0x9c58531f,0x291d5e4a),LL(0xaf4422a4,0xb30d5ded),LL(0x4cd8b631,0x30a12b16),LL(0x4d0ff100,0xb2d9901c),LL(0x450557f4,0x1b1ee29b),L_(0x00000026), + LL(0x85c6eadf,0x6dcd5109),LL(0xc399a187,0x5c966d1c),LL(0xfc77243f,0x4abf82c9),LL(0x3aec3b2c,0xfaf22c71),LL(0x4988df1e,0xaa22e170),LL(0x8f28a287,0x7d192a2d),LL(0xf724ea96,0x6179ade0),LL(0x8ced48e7,0x18acdc5b),L_(0x000001f3), LL(0xc8ad685c,0x0c63196a),LL(0x8e70052b,0xa08be270),LL(0xd8ff45f4,0xe9bd37fa),LL(0x46164862,0xcc39748e),LL(0x3cc067ef,0x7dcfa284),LL(0x68836731,0xc4586ae8),LL(0x8aabfd38,0x85ecef5b),LL(0x815642ea,0xaf78e84b),L_(0x00000073), + LL(0xbecd1954,0x55d6e970),LL(0xf86e2eca,0xa3c6d699),LL(0x35fd609b,0x277a6dcb),LL(0x8e0371bf,0x73c53bc0),LL(0x69861b02,0x747c9b1f),LL(0x121dd3fe,0xf6c83ab7),LL(0x1a4b5c05,0xa41b4b4b),LL(0x103beb00,0x6b773426),L_(0x0000002a), LL(0x2273e127,0x590624d3),LL(0x968b4935,0xdc61d6e8),LL(0x57ff9d1f,0xc882f647),LL(0x3774a13e,0xd8068210),LL(0xc8094f39,0x52df5a78),LL(0x9ce26b44,0xd80b9309),LL(0x06b76a9e,0x51126d68),LL(0x26a298cf,0x20129462),L_(0x0000002a), + LL(0x4a23b337,0x00b763fd),LL(0xe905f611,0x31c9604c),LL(0xad5b3ffa,0x059b27d0),LL(0xe85817bf,0x957d997f),LL(0xc9cea64f,0xa8adabfc),LL(0xbf24cb58,0x74dd2fa1),LL(0xfe218a31,0x08cb0dea),LL(0xcfee69ca,0x310fed00),L_(0x000001df), LL(0xb0001602,0xd66e131e),LL(0x18c779a4,0x47a1fa4d),LL(0x13180dd7,0x23f27ad3),LL(0x3169340c,0xdf4a2f35),LL(0xa8c2be04,0xaec77b35),LL(0x3a1f8aa1,0x2eed7fb2),LL(0xe69edc27,0x6110abc9),LL(0xa5d58ddb,0xe7590226),L_(0x0000002a), + LL(0xc035887a,0x2331223f),LL(0xe2c10ac8,0xb1509b5c),LL(0xf8262756,0x8c8a002b),LL(0x4f5e4c0e,0xccc65314),LL(0xaa63ef65,0x5d26b24c),LL(0x91432899,0xaef2d2ee),LL(0xace0562c,0x284cbe21),LL(0xdd5ba0e2,0x4e06f44f),L_(0x00000176), LL(0x604bd262,0x9becc83c),LL(0xb164c12c,0xc02c461a),LL(0x811e7743,0x621b38ad),LL(0x8fde0227,0x61151b61),LL(0x1eee0c18,0x90989162),LL(0xa3e85682,0x6d28169b),LL(0x478f9519,0xd980aa5d),LL(0x35d9dbcd,0xf940dd13),L_(0x00000120), + LL(0xe818a77b,0xf6f3cc8a),LL(0xede150db,0x71295f54),LL(0xc2f06bc3,0x173c2266),LL(0xbd26ab2d,0xda0b8b46),LL(0x958aad7f,0x470909e3),LL(0xbeb03518,0x0c135242),LL(0x6b5aad80,0xe6b782b0),LL(0xf43a70dc,0x1ebe42a4),L_(0x00000176), LL(0x0792877e,0xe9ee87f4),LL(0x5c7acfe8,0xcb05380b),LL(0x7c5775ef,0x2d540e71),LL(0x4507bf0a,0x839d644d),LL(0xc6b81a8c,0xbd1ff451),LL(0xa45f8834,0xc6531b7e),LL(0x06bfc9c3,0xf1a607c3),LL(0xfacabe92,0x152a3731),L_(0x00000013), + LL(0x42b39a61,0x972ed0b4),LL(0x6043f03e,0x2b923d0d),LL(0xef133e76,0x91d9dd18),LL(0xbb6feffc,0x25386141),LL(0x9f4c0085,0x7c52d849),LL(0x279b119e,0x1a74529c),LL(0xfbddc6be,0x3bb14fa6),LL(0x0fc37390,0x5b0469e7),L_(0x00000130), LL(0x2eb5ef21,0xc18b1c46),LL(0x93ed9948,0xc3e5b7a6),LL(0xef0ebe97,0x98b816ed),LL(0x5877dfeb,0x369d99d9),LL(0x3e4311e4,0x2bf65ec4),LL(0x311d0134,0x8c99e5b8),LL(0xc89dbf6c,0x0a9bf18b),LL(0x0c03cf95,0x67a8c9b6),L_(0x00000134), + LL(0x05361f9f,0x4dad1ff8),LL(0x536327ce,0x2d1bf3b3),LL(0x5b0b6267,0x47367af0),LL(0xf13ddf38,0x8798d158),LL(0xa948f127,0x84fbc252),LL(0x3d0fe92b,0x978e6fc0),LL(0x0138676b,0xef9334ec),LL(0x96ea4ba8,0xb13cf224),L_(0x00000198), LL(0x7d693fd2,0xa8ac5c62),LL(0x19d6d21f,0xb670515f),LL(0xb4a2fb70,0x28de441a),LL(0x4762c399,0x0fd9e912),LL(0x3371fcdb,0x896888c1),LL(0x85fb68ee,0xd0e213f1),LL(0x2d86c189,0xcea1f849),LL(0x151cf6fa,0x49b94eff),L_(0x000001b9), + LL(0x1baf476f,0xab2c72b8),LL(0x009b461a,0x349ca815),LL(0x64f4891c,0x07d9a898),LL(0x0c2c3617,0xe9f189a3),LL(0x6792792c,0x226e9a21),LL(0xf620343c,0xc9fabe1a),LL(0x72bb7a93,0xda748299),LL(0x321acb19,0x1b2afe12),L_(0x00000023), LL(0x72c1891c,0x938118e1),LL(0xbfadb006,0xb72a44a8),LL(0x9204c26a,0x32e10cf5),LL(0xbf6c9c27,0xd43d80b7),LL(0xbe5c3cec,0x0afdab99),LL(0x915d7960,0x1a5a24d1),LL(0x933f89c3,0xfe011c0b),LL(0x93e4d990,0xfd09b45f),L_(0x00000123), + LL(0x8c930d5e,0x5c3e8550),LL(0x67c79888,0xf4f27501),LL(0x18ef2850,0x0d7f6b01),LL(0xf40547f0,0x9cb15ed2),LL(0x89d6f189,0x1bc417a9),LL(0x5c937894,0x22180816),LL(0xb0e0c28f,0x72ddfe6b),LL(0x95839d95,0xb0b70e2e),L_(0x0000016a), LL(0x5a755fcb,0x0092c31b),LL(0x3261be8d,0x81547562),LL(0x2ed776ec,0x2bc72da3),LL(0x7afed7b9,0x943fbdb4),LL(0x7b5a5d6d,0x896c4516),LL(0x44f20815,0x23f06fef),LL(0x5bd5c28e,0x8f4f6c6d),LL(0xaeda432e,0x355da25e),L_(0x000001b3), + LL(0x69860706,0xfa6c2c0b),LL(0x097c5794,0x8878edbb),LL(0xc8776355,0xf986d296),LL(0xb4dc28da,0xcb68b5fd),LL(0x7d364b0a,0xf5c63307),LL(0xa8df5161,0xe81fb7b4),LL(0x5837723e,0x9cfefa9e),LL(0x681c02e5,0xf9182f1b),L_(0x000001f5), LL(0x9e5af2e3,0x43741dfd),LL(0xeeeaf6e7,0x0f7f7db0),LL(0x912e6cbd,0x1b17b840),LL(0xfbc2bc3c,0xff0906cb),LL(0xaed2e1f9,0x79085d0d),LL(0x413a3a6e,0x3e5f9190),LL(0xf10a18b2,0x5e4a7967),LL(0xbe3ecf44,0xb4e7b709),L_(0x000000ae), + LL(0x8c349a09,0x9f76c8ab),LL(0x693d1574,0x254fd850),LL(0xd18a6991,0x4944e4ef),LL(0xf78f60f5,0xbc73879c),LL(0x125696fe,0x04c63e00),LL(0x8e1e2dbe,0x88e142f6),LL(0x688fd93a,0xbbe7321b),LL(0xd6f1e83a,0xa7b2fa13),L_(0x000001aa), LL(0xa9d97c57,0xcbcaa293),LL(0x19ff3b97,0xb39ad595),LL(0xcedb5893,0x7e59369f),LL(0x47364ed0,0x15852af3),LL(0x7c32e933,0x4d5a40d4),LL(0x82768fe3,0x06533865),LL(0x53f8e4c6,0xf43bf2b4),LL(0x8b4a96dc,0x2542e182),L_(0x00000033), +}, +/* digit=92 base_pwr=2^460 */ +{ + LL(0xe3fec0f9,0x0611a7aa),LL(0x83626ec8,0x91b56818),LL(0xaebe5044,0xecc113fd),LL(0x3fc31dd2,0x68171e0f),LL(0x5fdadf10,0x4d2fbb6b),LL(0xe94a492b,0x06c20ee0),LL(0x0723f06e,0xe2fed2d9),LL(0x8316b906,0x2f32d5d0),L_(0x0000013b), LL(0xf27e685a,0x63731c43),LL(0x2924bed0,0xf8996c91),LL(0x58df5bd8,0x04d16a64),LL(0xb4780e3a,0x3ff6f14c),LL(0x5aed0330,0xcf56c817),LL(0xb62e0f3e,0xf8163011),LL(0xee8bd1d8,0x5c28fa8e),LL(0x9fa055be,0x5edb8d9c),L_(0x00000163), + LL(0x9d6a36d0,0x0deb4c2e),LL(0x0e66e6ba,0xb2451f1c),LL(0x058a747c,0x20962d66),LL(0x0214b10a,0xda104e82),LL(0xe594cae1,0xb4693d32),LL(0xf837609c,0x059d3bb7),LL(0x53eda7c5,0x1dd16cae),LL(0x3eb60275,0xc67ede2a),L_(0x0000007a), LL(0xc49b1452,0xc838202c),LL(0x35c208ae,0xb2fb2035),LL(0x56079145,0x55be9713),LL(0x95c814d4,0xfd8f0bb3),LL(0xc09f6782,0xcc755426),LL(0x8edafadb,0x4ecf0b74),LL(0xeaf4bc3a,0x553943ed),LL(0xa2bff049,0xe542c407),L_(0x0000017e), + LL(0x76e77cb5,0x0d581933),LL(0xc0bba438,0x9ddc3e72),LL(0xbc54c3a8,0x1bbc6d0f),LL(0xf2cdd63a,0x1518f660),LL(0xaf2f62bf,0x27e5bf5f),LL(0xbfe7727a,0x7b164682),LL(0xa33defeb,0x40ec257d),LL(0x6132b5ef,0x902b2e89),L_(0x0000000e), LL(0x87524e3b,0xe3ab4683),LL(0x69a88271,0x8299824e),LL(0x4545479e,0x373761e0),LL(0x397121a7,0xe3b1f753),LL(0x0aaebe93,0xd463acb4),LL(0x1af707da,0x1d6dcb72),LL(0x14e15233,0x48e5280c),LL(0x91655530,0xaaad009d),L_(0x00000012), + LL(0x964e28ff,0x9792b39f),LL(0xcfd79768,0xc556dfa7),LL(0xf279c07e,0xd7670e8c),LL(0xdad1cf40,0x5e04abde),LL(0x90b1377b,0x46fcc199),LL(0xf5067ad9,0x6a088572),LL(0x28101c96,0x2ad58aca),LL(0x43b5e33c,0xe333e24c),L_(0x000000b0), LL(0x475ac89b,0x54f0361a),LL(0xf79463ef,0xc22ce2dc),LL(0xdd053538,0x4e6817ca),LL(0x2bc1013b,0x4b01a6e1),LL(0x844a6eb4,0xad109d85),LL(0xbfdaef54,0x4481b985),LL(0x6830be54,0x297f121f),LL(0x8bfd8dd0,0x6d0b67a6),L_(0x000001cb), + LL(0xe23b3385,0x45ba65ed),LL(0x45dcbcdc,0xdd83e268),LL(0x0a0b1cdd,0x63968b4f),LL(0x35e092ea,0x71e6e72d),LL(0x3f1ddc4b,0xa46c5ed0),LL(0x5a166bbf,0xb7c76efc),LL(0x0de0b5f6,0xc1387b79),LL(0xb6445136,0x8923450a),L_(0x000001a7), LL(0x4005e747,0x99a85019),LL(0x0cae44ea,0x1e4af9c0),LL(0xe5e17b63,0x51c0dae8),LL(0xee91c80d,0x0c3cbe08),LL(0xa4093612,0xa23c8041),LL(0x3446b0ed,0xf215ada7),LL(0x43d14026,0x7813fb66),LL(0xb26fbac3,0x0031b68b),L_(0x000001c7), + LL(0x1660b409,0xe275ab90),LL(0x45ee7e0e,0x399f21e6),LL(0x37b06a37,0xe9bd12b1),LL(0x96080496,0xa5d50d58),LL(0x05ba3f26,0x7c1e3f37),LL(0x1d4a0081,0x4d39274a),LL(0x2d00866a,0x0317c40d),LL(0x64f146bf,0x2ec71ea0),L_(0x0000004e), LL(0x5c7f5630,0x933c199f),LL(0x4f78f168,0x3fb1362e),LL(0xa194ad2a,0x5259655c),LL(0x5f408022,0x5898f9da),LL(0xd8a6c1bc,0x7553edff),LL(0x2793c479,0xe26de20b),LL(0x1ea73083,0x533d9374),LL(0xb2a2971a,0x4fd22035),L_(0x000000e5), + LL(0x9a275851,0xf8f9d4de),LL(0xdebb5987,0xe3156400),LL(0xd526ec09,0xe0023f66),LL(0x5578bd03,0xd7d715cc),LL(0x3099f2ae,0x858cc9c0),LL(0x4417ca0c,0x2ea5506b),LL(0x23b4df57,0x7420ffbd),LL(0xe0c5dc14,0x99652bdf),L_(0x00000128), LL(0xf8e9148e,0x0d31987c),LL(0x9461ad7d,0xe6bbab25),LL(0xc859a4a7,0x3d2a289c),LL(0x08730e2b,0xbe629139),LL(0xcf6e14e9,0xc3904cf1),LL(0xdab045a7,0xacb2cca9),LL(0x2a43de3e,0xfa439f68),LL(0xd4d82c3a,0xd187ae70),L_(0x000001ec), + LL(0x78085816,0x5587b449),LL(0xe293d334,0x6f8ad12b),LL(0x4ce4906c,0x8d521bfa),LL(0xb26a6693,0xa914bdb3),LL(0x9dc3e746,0x3ae5f6e9),LL(0xb0881c2e,0xac2559db),LL(0x9191a1b5,0x72e53430),LL(0xa6c6a97a,0x07226ad4),L_(0x000001dd), LL(0x234483b3,0x3a39e249),LL(0x419af206,0xc72bc669),LL(0x42122752,0x9a44c7a3),LL(0xc28aa7e1,0x188ac573),LL(0x14ec6b11,0x09a3360e),LL(0xc8624588,0x3af7bc0b),LL(0x41e19299,0x42bd4817),LL(0xa8d8d757,0x8768555b),L_(0x000000cd), + LL(0x5b9a4085,0xe3a45fe5),LL(0x95591624,0xaebe1b9d),LL(0xa30f27ef,0xcc5daccf),LL(0xf894bab2,0x753b9ddd),LL(0x456446cc,0xa2185cca),LL(0xfed1e127,0x12d28159),LL(0x0339e65d,0x698c68d1),LL(0x1a9ca283,0xe9c06f97),L_(0x00000087), LL(0xb7266ccb,0xfa3c3bf1),LL(0x0574a504,0x1f37ca7f),LL(0xdbe81703,0xebc6693f),LL(0x6a068b27,0x9068b291),LL(0x03b786c6,0x3dcacbac),LL(0x74d197dc,0x5766087c),LL(0x8dd304a2,0x60b034d5),LL(0x45cead24,0x0eb18561),L_(0x0000010a), + LL(0x6e27781a,0x247df0f3),LL(0x5379241e,0xe00bbfb5),LL(0xad8e1bf2,0x4971d453),LL(0xffdae98c,0x4dac08bb),LL(0x43c392ba,0x1094f61d),LL(0x34435f45,0x25c82ce3),LL(0xc0379951,0x86ddd573),LL(0xbe3f91c8,0xa4a47405),L_(0x0000017c), LL(0x16746966,0xd140309f),LL(0x7d5ffd2b,0xb5c6fd08),LL(0xf1092ec1,0x3fef2aa9),LL(0xb1d3ec33,0x5486d81c),LL(0x414b5e87,0x09d0d988),LL(0x7faa17ff,0xd5d8f9a4),LL(0xed9cefef,0xa9baa755),LL(0xcec21b69,0xb4d04c7c),L_(0x000000b0), + LL(0x446d0af9,0x705db89c),LL(0x84f6bfda,0xac8dec0a),LL(0x7e47ba03,0x11601ccd),LL(0xa270c8be,0x5c7ebbe6),LL(0x6d1474f3,0xf62ef9fc),LL(0xe04a6269,0xfb77f59b),LL(0x9e393c86,0x150112d3),LL(0x5df9c04c,0x64d1ef7d),L_(0x0000006b), LL(0x430bfacf,0x4c8472a1),LL(0xfafe59da,0x41ca0a2b),LL(0x2ad99761,0x1e89c29e),LL(0x4b041df1,0xfe4fe5b6),LL(0x8b8ac33a,0xd36ab4d9),LL(0xf8473963,0x53e1a21e),LL(0x75363c3b,0x21c36ab1),LL(0xe0592363,0xe35ac3a5),L_(0x00000121), + LL(0x432d1107,0xee1e4fba),LL(0x2e463b38,0xafc972de),LL(0x1010af35,0xbe876d0d),LL(0x49188274,0x060ac231),LL(0xac345fe8,0xb568289e),LL(0xe6dec43c,0x9e9a1cec),LL(0xc8cf61bd,0xa7d9e863),LL(0x4480624e,0x84470564),L_(0x0000010f), LL(0x0aeb84ba,0x7730874b),LL(0xb07f3f33,0x5d9261b6),LL(0x122fcc85,0xe27f8557),LL(0x06820d8c,0x073c1847),LL(0x82e3f6be,0x3976550d),LL(0x0e6c3609,0x9a68ea2f),LL(0xf48dbeee,0x99ffba71),LL(0xcc24a469,0x3d0bdb6c),L_(0x00000168), + LL(0x910a6eae,0x01e0bacb),LL(0xfaf69acc,0xde8618ba),LL(0xc2f50b6a,0x14a7dc8c),LL(0x255c91c8,0x2f3c4d3d),LL(0x3e41e3ed,0xb39008da),LL(0x3885fe89,0xe57622e4),LL(0x0199693a,0x4d2436d5),LL(0x8a6ba080,0xdafeb6e6),L_(0x00000023), LL(0x5f1b83a0,0x2580e973),LL(0x8519d427,0xa920070b),LL(0x0b67d6f2,0x90fc96fb),LL(0x14566ecc,0x25c7716a),LL(0x30c1cfb5,0xb8dd507a),LL(0xd8b7f726,0xa175dc05),LL(0x24a60ede,0x99f15332),LL(0x78236bba,0x7ca4a569),L_(0x0000015f), + LL(0x4901b9ef,0x1e3479cf),LL(0xdd8491ad,0xd79c5592),LL(0xcd2995f5,0x8ece5732),LL(0x7f9162e1,0x7f928933),LL(0xff64a3a5,0xdd90d4ea),LL(0x82a7e6d8,0xd296e0f0),LL(0x8dc81d30,0xf317ef62),LL(0xa5bbd68c,0xc3c72a97),L_(0x000000bd), LL(0x43a9d0a2,0x8de59597),LL(0xbad8e310,0xe46b5cc5),LL(0xae543536,0x3be10fe7),LL(0xb038a518,0x22e5dfa8),LL(0x98fc1a73,0xda531be8),LL(0x4395fad8,0xa64d7d12),LL(0x81e9b112,0x5b7b8eee),LL(0x6b371c5e,0xf97cc8db),L_(0x00000072), + LL(0xac5fef79,0xeddf9eed),LL(0xac48ddad,0x3fafda2a),LL(0xb0386572,0xe2cfe37f),LL(0x4a95ae1d,0x2a2d6f2e),LL(0xf0b70c4d,0x5539faa3),LL(0x1e2738ed,0x855ae2b7),LL(0x680d7df5,0x5fa4d703),LL(0x047f7d72,0x981799eb),L_(0x000001a0), LL(0xc9f2a2dc,0xefb32d4e),LL(0x14e1364c,0x0008de76),LL(0x2af04490,0x56298a56),LL(0xa488b32b,0xe10ef61b),LL(0x7c93d9c0,0x74302f60),LL(0xe50aeca3,0x40b43584),LL(0x7cf8baf8,0xb9ab8a52),LL(0x29e97768,0xf0c44bc5),L_(0x000000b0), + LL(0x266fcb30,0xc7ebaeb9),LL(0x9166afc8,0x8df096c4),LL(0xb1a4fb9f,0x9ef63e0b),LL(0x0a63a275,0xd0e62d1d),LL(0xa13c16de,0xf215cb79),LL(0x82d5b46e,0x45439424),LL(0x5cf39033,0xc9b239aa),LL(0x4a39ce21,0xfcf03ed3),L_(0x000000a2), LL(0xdf517f0c,0xd8466a8a),LL(0x523be0b6,0x493a7775),LL(0x74759167,0x4894bb12),LL(0x5e2284c6,0x864e9ca2),LL(0xd07d26e2,0x08b7f98f),LL(0x6d662061,0x8e1e3fdb),LL(0xf64b5a66,0xa0ba6cae),LL(0xedd31c44,0xdac14a11),L_(0x00000091), +}, +/* digit=93 base_pwr=2^465 */ +{ + LL(0x5b2f805e,0xd37e005c),LL(0x6d99c24d,0x9fa0210b),LL(0x813da140,0xd53cbcd5),LL(0x9488bf13,0xb6d8655f),LL(0x5b2d055b,0xb21f224e),LL(0x3ba305b4,0x059a77dd),LL(0x5337f568,0x783aa9f0),LL(0xb88b4b1e,0xe8c56442),L_(0x0000016c), LL(0x71f23b13,0x9b7e0acb),LL(0xe0e90fde,0xda1867f2),LL(0x336f8ff1,0x14e3d072),LL(0x8e647516,0x87e51c7e),LL(0x1ca72a31,0x27ef1710),LL(0x61c42d89,0x641d8a97),LL(0xbb69cc0c,0x6138250e),LL(0xc12903e9,0xd2873a54),L_(0x000000b8), + LL(0x06415e13,0x0523f47f),LL(0x1fe7219c,0x11a49ec5),LL(0xbd8a88f4,0x6713e8b2),LL(0xd3f30897,0xe0f84892),LL(0x410c616e,0x4957e9fa),LL(0x60b01558,0xfce0903d),LL(0x41fc07f1,0x82117eff),LL(0x3ffa3ce1,0xb039b569),L_(0x00000054), LL(0x9c0d6884,0x04f700d5),LL(0x693fd9aa,0xa0743bcc),LL(0xb8b0e7fe,0x81c35812),LL(0xfcb182c1,0x64896cc8),LL(0x9f019f88,0x8c77cf49),LL(0xa6594c50,0x2c4110bb),LL(0x88406e14,0x0fcaee7e),LL(0xb8b45fd5,0x4dc1ba3e),L_(0x00000189), + LL(0x7eef0c8d,0x07c446c4),LL(0x0878421c,0xf275544a),LL(0x8722c55c,0x424a48fb),LL(0x028ec763,0xf6b5b3b9),LL(0xca8f7bf4,0xf78d4fe3),LL(0x77d82e20,0x04e23f42),LL(0xbc6300a7,0xf5f71bbf),LL(0x3aa908b8,0x0bc8e8a5),L_(0x00000109), LL(0x49fc8da7,0xd0dcad65),LL(0x35d31de3,0xe5fbc4e6),LL(0x9ac9c9da,0x525deba7),LL(0x0b85d812,0x465a1ffb),LL(0x08542228,0xc039c002),LL(0x1962a343,0x60c9d143),LL(0x729577d4,0x0fe4b631),LL(0x05befcdb,0x25528067),L_(0x00000198), + LL(0xe896c288,0xbc7bb607),LL(0x894887a4,0x14230e0a),LL(0x6eb1e976,0xe2c653f8),LL(0xe9303e71,0xdded494b),LL(0x9fc0dd96,0x98ac95d0),LL(0x63fba061,0x738abea5),LL(0xf3c1624d,0x4a0ea988),LL(0x389df64d,0xc6ae1823),L_(0x000001d5), LL(0x0454516f,0x7feeeb90),LL(0x7d7a8b0a,0xde36c637),LL(0xa9c345a7,0x611067e9),LL(0x0a9100d5,0x6bcdcedd),LL(0xf6c68c80,0x92b5dec6),LL(0x8d7d4a34,0xad3651f3),LL(0x2d5061b9,0xf739c0f2),LL(0xd15c9ea7,0x34e6cedb),L_(0x000000e7), + LL(0xcc5b33ab,0x5ef9ab41),LL(0xda8a106f,0x3b8cfab2),LL(0xb72948ff,0x3a4d0cd5),LL(0xf95a1457,0x5f6b94ff),LL(0xa636c12b,0x4c711bcc),LL(0x1e6c9e9f,0xffdba7aa),LL(0xe7eacce4,0xfce23073),LL(0x8fbc9275,0x5935eb69),L_(0x00000059), LL(0x00d4e588,0x05d931ae),LL(0x918b9aff,0x1f8a1b79),LL(0x844b544c,0xaa1709c8),LL(0x7e08066d,0x5258c624),LL(0xb640f1c6,0x176bbba4),LL(0xa22bddd0,0xc24ede16),LL(0xd090e0e9,0x4685aca4),LL(0xd8b8736b,0x64e8e6dc),L_(0x000000f2), + LL(0xd9d5f173,0x35476aba),LL(0x78928ff0,0xd948696e),LL(0x989109f6,0x5f254f30),LL(0x44ed9a63,0xd3543664),LL(0xa497e106,0xec63e4f7),LL(0x54a3d56c,0x4cb1418d),LL(0xbfbcd507,0x2a5c778f),LL(0x548f00b1,0x3ba6c12a),L_(0x00000150), LL(0xeeb4939d,0x4db14381),LL(0xc2817a38,0x86547af0),LL(0xb6947c7e,0x6d9e6104),LL(0x70ddd5de,0x2c369c27),LL(0x2f6e17ee,0x04550b40),LL(0x2c52689a,0xb0ead30b),LL(0x3892ae0d,0x99d74e20),LL(0x145321b6,0xd38ac454),L_(0x00000068), + LL(0x3ffb08c5,0x9a014b31),LL(0x2b898264,0x9e8130d3),LL(0x12fa12a6,0x014372da),LL(0x94999852,0x86eb5c63),LL(0x2a214084,0xdfb3f74b),LL(0x889d0eaf,0x9c182b54),LL(0x4f4c24a5,0x023efe1f),LL(0x0c3bbe75,0x3089629d),L_(0x0000012f), LL(0x76a8709a,0xdbfd5856),LL(0x138e9e46,0x49f8b60f),LL(0x8855a365,0x00624aa1),LL(0x358ac67f,0xff0d2d03),LL(0xd4f8c970,0xb9b15a4c),LL(0x244d4dda,0x60864d2f),LL(0x7db18004,0x1d1483da),LL(0xd00cd704,0xfbce4196),L_(0x000000b0), + LL(0x3cb5aab3,0x0feb0501),LL(0xca55bd42,0x8695f9af),LL(0x9c3c71fd,0xcc6e5ed1),LL(0x3d500caf,0x8edc89ca),LL(0x4e21b872,0x77647185),LL(0x0ff872ac,0xee45201f),LL(0xe23036d8,0xc8bee8b3),LL(0x5f2c13f7,0xa1d51a1e),L_(0x00000107), LL(0xbf685327,0x8022b011),LL(0x529ed8e2,0x2b6ff0cf),LL(0xb8a477a5,0x3b6e8238),LL(0xcf5cd2f6,0x291c55c9),LL(0x42ab247c,0x3f4796ab),LL(0x9f93937b,0x5dbfc098),LL(0xca7b47aa,0x7620e79b),LL(0x296f0a6b,0xdd4ea007),L_(0x00000158), + LL(0x26121c08,0xf01a2f06),LL(0x96bedf48,0x7e8f7f41),LL(0xd452fc32,0xcfab9384),LL(0xf1693df4,0x57e90144),LL(0xde828634,0xf5773fb8),LL(0xf8ca5704,0x123913d5),LL(0x2119d8cb,0x7eb6dfc1),LL(0xfdf9f63e,0x3c675fe5),L_(0x0000000b), LL(0x63fe6950,0x1f4969fc),LL(0x18608a42,0x92fc7ae8),LL(0x270c4cd5,0xf9035119),LL(0xd6e64853,0xc4832b26),LL(0x6b21d3fa,0xa4c2c4c7),LL(0x726b2dcb,0x725c739d),LL(0xb649408c,0xcaaec71e),LL(0xa0b38b9c,0x5b557fb0),L_(0x0000017b), + LL(0x2d41c6fd,0xffa160b7),LL(0x6ed5bb92,0xd76d4830),LL(0x174db5f3,0x56439bbf),LL(0x9cf210a6,0xcf76e11e),LL(0x0a183944,0xb1458e01),LL(0xa3ae6e4a,0x034db573),LL(0xc26a236a,0xe322c7d5),LL(0x3184159a,0xef56cb0f),L_(0x000000f3), LL(0x0a135217,0x83bf41c6),LL(0xc9df776e,0x9cd9b688),LL(0x5709e999,0xec730800),LL(0x9d5ce348,0x9c9f3378),LL(0xa53d30c8,0xdcaf4c9c),LL(0x09b66b9c,0xb8aeaba1),LL(0xc4d0530f,0xea0f22f3),LL(0x73581f25,0xc1d28f6a),L_(0x0000015a), + LL(0xcf778121,0x47e984e8),LL(0x20c4ae50,0xd1b90de4),LL(0xc94af252,0xdf5adf83),LL(0x41b573a2,0xb6b25c5d),LL(0x003e17b9,0x47e2aa64),LL(0x600d2bda,0xf75489f9),LL(0x595799ce,0x1b49400e),LL(0x7a9784ae,0xba0298f7),L_(0x0000009f), LL(0x3c7f600f,0xc18830c9),LL(0xc0e76415,0xb7a4b3e7),LL(0x46646194,0x210e39ff),LL(0x46e16ac1,0x94dd48b0),LL(0xe09df941,0x5657d728),LL(0x75b23925,0x987d7dae),LL(0xf5484304,0xb8bad70f),LL(0xee4753cb,0xc44a0313),L_(0x0000005e), + LL(0x8d604911,0x301bb718),LL(0x3ba3b59e,0x7ed618c3),LL(0x2225703f,0xe6e7b1ec),LL(0x3d9b8d85,0x2ddd2443),LL(0x526b020e,0xbbb89c6b),LL(0xf99d3527,0x9694dbcf),LL(0x1bef732e,0x34415736),LL(0x42d5d4d5,0x5cdafabf),L_(0x00000140), LL(0xd173ef6b,0x3da6214f),LL(0x2ffdc730,0x3a1a49d4),LL(0xc640e584,0x9755bbcb),LL(0xd90466dd,0x6a1bb6be),LL(0xcc97293e,0xd094e422),LL(0x23c9d622,0xf2ec9cc4),LL(0xcc616321,0xafe8382a),LL(0xc1a93af5,0x26522de8),L_(0x0000004e), + LL(0xdcb7addb,0x73e6acc5),LL(0x9117f654,0xea525fd4),LL(0xd6399efb,0x5316271b),LL(0xbf78249c,0xe30685a3),LL(0x7737d7b2,0xb95bf177),LL(0x4cfca353,0x138bd305),LL(0xa3671bc7,0x088b1877),LL(0x110ae487,0x1ff3771d),L_(0x000000fd), LL(0x0a4086e7,0xaf501744),LL(0x732576c6,0x4a538a56),LL(0xdfe16416,0xf3e1aa2e),LL(0xfe886ca1,0x95495af6),LL(0xfad421b0,0x2e5633e8),LL(0x14deea0f,0x87a33bf0),LL(0x59e08514,0x7333d917),LL(0x92bad09b,0x773222e4),L_(0x000001d4), + LL(0xd9f416e2,0x3565ca51),LL(0x0fd12235,0x5d3c8d16),LL(0xdc033287,0x948d4bd6),LL(0xd175dce3,0x9d5a6616),LL(0xd08718eb,0x4afce525),LL(0xd0bbb22a,0x9107b243),LL(0x93527f91,0x45382cdd),LL(0x6c46b7fb,0x8d893d42),L_(0x0000018d), LL(0xf7906107,0xbddfc4e9),LL(0x1d8f3a25,0x1c16029a),LL(0x51ce74f6,0x0f857730),LL(0x72d22f72,0x9d6f7b83),LL(0xa805ac0c,0xf970cb65),LL(0x6193a324,0xef9afdbc),LL(0x579b13d8,0x29a49024),LL(0x2ea3de42,0x4d4f92d1),L_(0x00000167), + LL(0x68b78453,0x3687601b),LL(0x61173b64,0x19ad59c7),LL(0x5cdaa0ac,0x37e94d1c),LL(0xb01a110c,0x46a8a192),LL(0xb73fb28d,0x89ea578e),LL(0x32a829b7,0xc1c111ef),LL(0x73a214d8,0x1c8ded15),LL(0x1cfef495,0xbf036574),L_(0x000001dd), LL(0x79e6c9f4,0xde40c899),LL(0x451757da,0x9bdf62c8),LL(0xde505c58,0xf590c16c),LL(0x0833d1ea,0x4f878ef4),LL(0xc9e82fbb,0x8ef0ccb2),LL(0x87ab08dc,0x1d1f4efd),LL(0x115ad9da,0x4e21d1ff),LL(0xd549cc87,0x88e6e9e7),L_(0x000001dd), + LL(0xc9cea1d4,0x424f87aa),LL(0xffc3ba23,0x162b1fab),LL(0xa3b2b167,0xb86c7978),LL(0x83e73da5,0x9b5f991d),LL(0x8d484c76,0x2cb3d908),LL(0xe085b439,0x28064542),LL(0xeba2ea8f,0x2b91d2b4),LL(0xcdc46cb8,0xe83321f8),L_(0x000000c0), LL(0x9d31426b,0xfd97601a),LL(0x4a0c50df,0x305e99ef),LL(0xfc4b8056,0xa29f6e86),LL(0xf5b0c1c8,0xbe1babed),LL(0x558d2cd4,0x75e98d4d),LL(0xd17d7bc8,0x45e57fd7),LL(0x9a0a33b7,0xc3cf9b60),LL(0x2d8c2a2b,0x5277b76d),L_(0x00000023), +}, +/* digit=94 base_pwr=2^470 */ +{ + LL(0x62b479f7,0x3e614e7a),LL(0x1806f150,0x0773591b),LL(0xc937295d,0xb432690c),LL(0x6d3468f3,0x7af2bc37),LL(0xc765b502,0xf1568b1f),LL(0x4508081c,0x4f2d04c8),LL(0x3b08d2fa,0x0d438419),LL(0xfdaa2353,0xd4118eb0),L_(0x000000dd), LL(0xa74bf7e0,0x7395d916),LL(0x879c30fb,0x732a652f),LL(0x33e906c3,0xd707078b),LL(0xeb09ecd6,0x4fe7914f),LL(0x76e24476,0xf1644295),LL(0x1ef70830,0x90ff7060),LL(0x8d1a94c2,0x8e38b393),LL(0x347e067e,0x7b7a7e79),L_(0x0000004d), + LL(0xb1fc1341,0x7e61fdc0),LL(0xe59da03f,0x98bd359f),LL(0x51831e76,0xf982fb68),LL(0x4079f81d,0x64253ce4),LL(0xffbd0a1c,0x684a0c0f),LL(0x24ab0837,0x0fa3fd27),LL(0xaefd7b90,0x8cd54b9a),LL(0x4f017be0,0x39893203),L_(0x00000159), LL(0x71b358dd,0x5f8bfed5),LL(0xbd73f12c,0xbd9b46e5),LL(0x7574722c,0x672fc532),LL(0x95b789de,0x9d4c5de7),LL(0x313e84cf,0x48e00647),LL(0x002f1934,0xdda401e2),LL(0x649a15d6,0x96114ef7),LL(0x37a4f04c,0x4a9dc085),L_(0x000000c3), + LL(0xf21720e5,0x8f0a82c0),LL(0x87ae4c3b,0x74f004b6),LL(0x384b1146,0xce02e119),LL(0xd665e4e4,0x1859c7a2),LL(0xe5dbd5ec,0xadfa269c),LL(0xa30b0013,0x74ac1d2b),LL(0xb4b5ebac,0xbc73c88d),LL(0xacdb48e2,0x872a2a2e),L_(0x00000061), LL(0x8570d85d,0x1044c064),LL(0x1021647f,0x3c4c4561),LL(0x05bc197e,0x8aef2b50),LL(0xdd7066b6,0x53c751b1),LL(0x10e7a8e7,0x3b7ee577),LL(0x2667f737,0xee8825eb),LL(0xd2baf066,0x2e6cd49a),LL(0xfaed0dee,0x4bbbae5e),L_(0x000000c0), + LL(0xea72f88d,0x755e5ed4),LL(0x7eafbd5d,0x22817dc2),LL(0x8274c8f2,0x6ed11c56),LL(0xa0be4b95,0xc506cd96),LL(0xfca3c62a,0x6c56121c),LL(0x160f6437,0x94e1e3c5),LL(0xcd969d97,0x2a1b9ac1),LL(0x6a2818ba,0x5d12cb94),L_(0x000001bd), LL(0x838f81f8,0x67065269),LL(0x6eaf2423,0x2f4afa25),LL(0x74390891,0x2342e954),LL(0xec048d33,0x5565f855),LL(0x3a8816f7,0x8e4ec59a),LL(0x55b015d4,0x6a715052),LL(0xbf898ef0,0x385313b6),LL(0x8baf90ae,0x415dc868),L_(0x000001d5), + LL(0xb57c6620,0x117822b3),LL(0x37c2f26e,0x4762f6de),LL(0x9377e35f,0xfb62c99f),LL(0x3c9faf6d,0xa5fec0c5),LL(0xe6f2602d,0x6a84d794),LL(0x1e2e5844,0x2e9c376a),LL(0x2b97dfd3,0xcdd28547),LL(0x24977fdb,0x3efe1f48),L_(0x00000189), LL(0x28c17d00,0xd2462621),LL(0x19582902,0x6e455352),LL(0x638d59c6,0xc3bea880),LL(0xd74f8133,0x1df686ac),LL(0xae8224be,0xdefcc095),LL(0x467de606,0xc63376c6),LL(0x1bbcfb09,0x644acbfc),LL(0xe6c4cc04,0xce83b441),L_(0x00000126), + LL(0x02da505b,0x27486dd1),LL(0x6bd544fa,0xc8ec4c55),LL(0xc9e28c7b,0xa7531516),LL(0x22d696c9,0xfbb565f5),LL(0x5bc69de8,0xf4a6f6f5),LL(0x3c4607fb,0x2183944a),LL(0xfbf34142,0xc088d4ca),LL(0x6a6a25a0,0xebf86497),L_(0x000000be), LL(0x17ac892f,0x08805ee9),LL(0xe10f4906,0xb9740059),LL(0x2a74e9d7,0x72ae3ee2),LL(0x2294188b,0x0359108b),LL(0xffa569a8,0x5438b3ae),LL(0xf5a918ef,0x5b1543c1),LL(0xbe32cd1e,0x857bce3b),LL(0xf67721ef,0xcb0f4756),L_(0x00000137), + LL(0x5bd5d895,0x7f863db9),LL(0xb81bcc90,0xc94e7e86),LL(0x24d3e88a,0x59ad89c4),LL(0xf5ed10e7,0xce7d1f0e),LL(0x1bd7de70,0x6932d3ed),LL(0xf1561272,0x12569b60),LL(0x393d97a6,0xf0741124),LL(0xa46a9516,0xc9b5b179),L_(0x00000130), LL(0xe867277a,0xda1e1906),LL(0x6ff9aa73,0x64a918f8),LL(0x1673d460,0xf233bf04),LL(0xe548f086,0x4ac69dac),LL(0xb984ce8a,0x12e45aea),LL(0x294fb6e3,0x5a19d674),LL(0xd8993346,0x00bc5dd3),LL(0x0c254d86,0x90107b13),L_(0x00000014), + LL(0xdf6f2ba1,0xec115486),LL(0xb08c738e,0x93e59803),LL(0x23024435,0x4f00e934),LL(0xbdbe60eb,0x7c91438e),LL(0x23e859e5,0x580e89c5),LL(0xba0053e0,0xb329d75c),LL(0xd11317c9,0x3d389550),LL(0xd235e570,0x5aac6426),L_(0x00000184), LL(0xfa06b949,0x42349105),LL(0x21c69e6c,0x9a81de9b),LL(0xf70eb151,0x2990539b),LL(0xafc827ae,0xfe9bf0bb),LL(0xee9dd548,0xb8dbb3c1),LL(0x8c4d4274,0xf3be2a90),LL(0xb7224476,0x6e6842bc),LL(0x8428346b,0x39da0c73),L_(0x000001ad), + LL(0xaf5db393,0xdaa88388),LL(0xe569ed69,0x7f0f1377),LL(0x60762027,0x2fe1fac0),LL(0xa5dd03fe,0x91b1f27a),LL(0xb60b2ae5,0xc4161046),LL(0x0c72417b,0xf9d8850d),LL(0x16e6bfc4,0xa63fb7e8),LL(0x1c03a1ed,0x8baa08f0),L_(0x0000017d), LL(0xaf898d56,0xba237ffd),LL(0x7b0680d7,0x0d439e05),LL(0x64dd0307,0x5d17a507),LL(0xded7a46c,0xaa6f686f),LL(0x46bc0763,0x302e95a9),LL(0x16dd6fb8,0x32a15017),LL(0xd01bdd13,0x4b4868e4),LL(0xe250803d,0x1558017b),L_(0x00000168), + LL(0xc67a04f1,0x275e8267),LL(0xb1b59985,0x6c0a68a7),LL(0x44944a18,0x537ca1e2),LL(0xd9ed65ad,0x30e1107c),LL(0x1eddede3,0x802fb267),LL(0xeaf5fb68,0x27195ee0),LL(0xd0137c18,0x11b69677),LL(0x331a8cb2,0xab2ade03),L_(0x00000147), LL(0x872e7d11,0x0449d382),LL(0x0033feb4,0x9c7c44c6),LL(0xb2576dae,0x83e4fe6e),LL(0xde49f678,0x8e0a0c88),LL(0x049a1944,0x873f101a),LL(0x2f0dcfaf,0x0c6ecf9c),LL(0xf4b7cbf4,0xb8b0d918),LL(0x5cf46641,0xd2760eea),L_(0x00000082), + LL(0xdb4a36c2,0x8ae1b4af),LL(0x5fa3cb57,0xd120dbcb),LL(0x79a8d192,0xec8bbea1),LL(0x8ab00e0a,0x0fa43f9b),LL(0x8da0324e,0x320ae51d),LL(0x53f9b52f,0xd7d8355e),LL(0x28f25abd,0x800a5d6c),LL(0xce8c317a,0x8116a102),L_(0x00000024), LL(0x539d001d,0x5e187817),LL(0xdc1f3bba,0x3d0941a2),LL(0x42b1a905,0xa2a2cc7f),LL(0x7392c3e9,0xc8a2218b),LL(0xf81e4937,0x50e22321),LL(0xec9f9e7e,0x650f7010),LL(0x905ed136,0xce7ae424),LL(0x143d78dd,0xf4b39b10),L_(0x000000de), + LL(0xfa007b15,0xeaa9a342),LL(0x15bb7a90,0x26771654),LL(0xdcc6aa0c,0xc720264a),LL(0x986a1f0b,0xe93f7bc1),LL(0xa182f9b5,0x1bf6b3fc),LL(0x5c22f84c,0x14eb9a7b),LL(0x7bdec7ad,0x42c3b078),LL(0x108902c9,0xd7973e78),L_(0x00000069), LL(0x4676602b,0x20230ff4),LL(0x64b88212,0x8928dd30),LL(0xd730a522,0xc0f54e1a),LL(0x528ea087,0xbe8035de),LL(0x2188b80a,0x1d9e98d6),LL(0x12fe3f39,0xc4b4d85f),LL(0x5d1c13b3,0x0436d0a9),LL(0x02c9a494,0xcc5f2436),L_(0x000001d2), + LL(0xbce8d401,0xfa9544c4),LL(0x3a3c6860,0xc43438df),LL(0x7ba2b50e,0x8cfb61b2),LL(0x160b337f,0x177729b0),LL(0xccab10b1,0x69458502),LL(0xb9a0ff5a,0x78449ba9),LL(0x67ca5074,0xa879a311),LL(0x73b77e61,0x91f5cf0f),L_(0x00000119), LL(0xa663023b,0xe9dd38a6),LL(0x7f1ab441,0xdb97e39f),LL(0x8836c427,0x0e31501d),LL(0x26be55f8,0x88c80de0),LL(0xf094f5db,0x161288e2),LL(0x239cdfc2,0xb27ca6c4),LL(0x6d31f6e4,0xbff28243),LL(0x6db05886,0xadc659a4),L_(0x000000f4), + LL(0x46bc5619,0xbc76d262),LL(0x3a81dc1b,0x1f94ed62),LL(0xb5ab7c11,0xcb89fed9),LL(0x7bd8c4df,0xf48a8846),LL(0xc61ecacd,0xe68265b3),LL(0x88e6ef63,0xfbdfdb92),LL(0x8bd95324,0x94692afd),LL(0xb7f81080,0x8b73dbb0),L_(0x000001a3), LL(0x42f27046,0xed59ec62),LL(0xe2c2f288,0x6fafd323),LL(0x11cda8bd,0x6af36bba),LL(0x324f4f07,0x06098acb),LL(0x091a65fb,0x74b50485),LL(0xf28dae6c,0x93e109e7),LL(0x1a6b91b6,0x648a962a),LL(0xba4a27c8,0x4dfe3efb),L_(0x000000ac), + LL(0x3b488e90,0xf43286a4),LL(0x56196c6d,0x6ba6cc59),LL(0xec8e64cd,0x7bd6708a),LL(0x98aaa1d7,0xb45c7cd9),LL(0xb5bcc0ce,0xce3eb2eb),LL(0x2653d9af,0x5b7387d0),LL(0xf3afdf31,0xe27833cf),LL(0x17806f1f,0x3ec743c7),L_(0x00000144), LL(0x5e969a67,0x96863b2d),LL(0xf200831e,0x1d6c065f),LL(0x1613e78e,0xa1366e05),LL(0xee600a0a,0xc0223e24),LL(0x06065867,0xe94c7976),LL(0x81ff94bd,0xf203aa4b),LL(0xf9511ac2,0xb7c19e3c),LL(0xd9eef849,0x75211256),L_(0x00000153), + LL(0x708d61a9,0x5cc484e2),LL(0x446b62d8,0x1d84ce14),LL(0x9c0fff45,0x08f1ae70),LL(0xe53a49ca,0x71899e1a),LL(0x9917f93d,0x3709a90d),LL(0x12fbb050,0x045ef39b),LL(0x38af72a2,0x0b8cc9a0),LL(0xf0817cd9,0x52b4ed83),L_(0x000000f3), LL(0xae023ef9,0xa7a71d48),LL(0x44e50d53,0x67e7ad87),LL(0x192ec226,0x37867d3b),LL(0xce32e194,0x24825f0c),LL(0xfce90271,0x5aa41f07),LL(0x4b826212,0xc9cefc67),LL(0x1f602e03,0xc071ae6d),LL(0xf9f93cc0,0xe4c52cae),L_(0x000001fa), +}, +/* digit=95 base_pwr=2^475 */ +{ + LL(0x58773b9a,0xebcc18c3),LL(0xf6385f12,0x8caf536b),LL(0x8a0c24ba,0xd83891ca),LL(0x08b3093c,0xb8c37621),LL(0xb26a0ef5,0xb41e3399),LL(0xca8c426d,0x0263fadf),LL(0x173bf676,0xd40bf584),LL(0xd8a6677a,0xa4760acd),L_(0x000000ec), LL(0x4207fa10,0xfbc42b8b),LL(0x5a60d34e,0x678686dc),LL(0xa367e08d,0x3e942c85),LL(0x2dd8cead,0x9d289bdc),LL(0xa6d1bb40,0xa4b034c3),LL(0x04955940,0x4e438893),LL(0x0034f368,0xddeee0c8),LL(0x63808a7c,0x8f3d9aa2),L_(0x00000114), + LL(0xd3e1babb,0x3ddf4a3f),LL(0x7d84daca,0x17e3e628),LL(0xd7eaf570,0x4870b354),LL(0x9fed1a4e,0x26a3e3ca),LL(0x5710e04c,0x0ce1ea5e),LL(0x17a2ff92,0xee67709e),LL(0xf8a3bc06,0xc019a660),LL(0xdb788ab8,0x3d909c0f),L_(0x000000f4), LL(0xebda5c3a,0x51c0c61e),LL(0xc130704b,0x5d086395),LL(0x762ffbcb,0xf6639983),LL(0x337f660b,0x46d9fb03),LL(0x8fa37c16,0x865cf06a),LL(0x3f14b6d2,0xe7365f2e),LL(0x8227d360,0xc5c3e588),LL(0xb6a48fcd,0x8c2eaf07),L_(0x0000019e), + LL(0x3e7b660c,0x226084ff),LL(0xaf5d90f5,0xe8626b6c),LL(0xa900e635,0x22c0e157),LL(0x22e31c96,0x1a4ad1af),LL(0x9e88afb5,0x3aadc5f1),LL(0xff5f6050,0xb11e90fa),LL(0xc0677ea3,0xf77875e2),LL(0xaed6a977,0x841145e6),L_(0x000001e8), LL(0x587f301e,0x6bb84d1a),LL(0xbfc80743,0xf386ce67),LL(0xb28c1dbb,0x43c48ae6),LL(0x88b71460,0xf88870e8),LL(0x4e3895ad,0x71c30d54),LL(0xcdbb1a28,0xa8e29d09),LL(0x71499052,0x6fbd1362),LL(0x3608395e,0x9cdda95f),L_(0x000000b2), + LL(0xf2dc50d1,0xb01ce2b5),LL(0x9f8c4d01,0xf417d7e3),LL(0x78d34284,0xcbf04214),LL(0xf59d157a,0xc4238071),LL(0xf8a594c0,0x7b0a1e05),LL(0xbaf85cdc,0xc9cfd81b),LL(0x1d1329e8,0xc9be4f2d),LL(0x3168fc55,0x5c20884e),L_(0x0000009b), LL(0x9cb47277,0x6e9fd410),LL(0x96d54227,0x16c1621d),LL(0xd61e57db,0x8656adf3),LL(0x2da52da5,0xd546ecce),LL(0x2098e089,0xb41508ee),LL(0x7499c874,0x9cf31199),LL(0xf525839d,0x96548966),LL(0xa0de08e5,0x1cdd85c0),L_(0x00000051), + LL(0x7be106c4,0xb8ede8af),LL(0xa3de8360,0xfdee27bf),LL(0x4341bdb3,0x376db3df),LL(0x851382eb,0x309206d9),LL(0x6325d433,0xff416946),LL(0x8994d6c3,0x0e775cfd),LL(0xfe50149c,0xee627cff),LL(0xee7b578d,0xcd01235e),L_(0x0000005b), LL(0xc03a13e9,0x5a46c19a),LL(0xeec8d37d,0xab92e082),LL(0xa6ae3bae,0x2deb57b4),LL(0xe3c4d075,0xce5d2ec0),LL(0x962e7d64,0xbd42e96f),LL(0xc56b57d5,0x513d5228),LL(0x68f2747e,0x7ec6010d),LL(0x1f92f153,0x8ad259fb),L_(0x000000ca), + LL(0x0c4d9937,0x1fdb1361),LL(0x018344e5,0x016f0192),LL(0xcb8a7e81,0x1ca2c27e),LL(0xc36425ff,0xa8df5318),LL(0x56d5d247,0x84872bcd),LL(0xa2e0d261,0x4866d142),LL(0x83feb22e,0x0999b14a),LL(0xab13dac7,0x07863be6),L_(0x000000d0), LL(0xd62b467d,0xce023bbb),LL(0xf8f48d21,0x35940e6e),LL(0xea9c5f9c,0x2bd76e0a),LL(0xa1f9af53,0x8ff97911),LL(0x750c500f,0xdefcff41),LL(0x3985ad13,0x9c027cfa),LL(0x36812ef9,0x34694b31),LL(0x5d319ee5,0x9722dca8),L_(0x0000019d), + LL(0xa6ef59c2,0xa78cdc7b),LL(0x67114f96,0x1a506e84),LL(0x909080ed,0xe3ccc90c),LL(0xe770488f,0xe93a6e81),LL(0x0b332add,0x6e681e90),LL(0x494adeb9,0x13abbb36),LL(0x580a5070,0xbf271178),LL(0xa19a151b,0xebb4d25d),L_(0x000000ae), LL(0x8202ce50,0x2353100e),LL(0x4b162883,0xf7cdd45e),LL(0x57659fda,0x4f79c844),LL(0x95b94da4,0x3ca165b0),LL(0xa6d4f4d9,0x3565f5c9),LL(0xc13d6186,0x288f561b),LL(0x81efd295,0x51b5a1dd),LL(0x0dee47df,0x0f774131),L_(0x0000005e), + LL(0xac4c9233,0x8240d25b),LL(0x6132b9fa,0x74ec9502),LL(0xddc2ef3e,0xa9db4e16),LL(0x29d151b0,0x5ad95c14),LL(0x9bb57bff,0x08144cde),LL(0xf2a19e48,0xef980c02),LL(0x655b0b6a,0x1f2df6c5),LL(0x2138725b,0x346457ed),L_(0x00000072), LL(0xe12bd180,0x8e3077ff),LL(0x2804b9bb,0x8db75e68),LL(0xb8a3a732,0x0cb1bbec),LL(0xb587b6f5,0x823e8549),LL(0xe705757a,0xdd7be7a7),LL(0xb60b8617,0x23677103),LL(0x131d7bc3,0x128ac224),LL(0x03713f91,0xadb3b9bf),L_(0x000000de), + LL(0x1a0be84b,0x625c8b6a),LL(0x8fc09173,0x14eb1426),LL(0xbb0b06c4,0xd925dd0a),LL(0x28f4f79f,0x5a160baf),LL(0x6a240ffc,0x4f7c033a),LL(0xcb7f6751,0x98adaaee),LL(0xc349dd94,0x192aa587),LL(0xee546461,0x189c51b1),L_(0x00000007), LL(0xeb23ea03,0x46d637ca),LL(0xaa3d1efa,0x01cfe315),LL(0xf7d6f7f9,0x4164c61b),LL(0x64b9530e,0x1a339a05),LL(0xce33c2f5,0xc30d67f4),LL(0xbcb863c9,0x79f8f963),LL(0x2bb9ff68,0x0799af64),LL(0xcfca4893,0xe7b1b3d8),L_(0x000000c3), + LL(0x58e4f6cf,0xa5b46eea),LL(0x00cef9c4,0x0381ae85),LL(0xe36179b5,0x317e7dbb),LL(0xac6498cb,0x2d824ab9),LL(0x328707df,0x6aa97d96),LL(0x80e79f5f,0xc19368fd),LL(0xe03799c5,0x109d20be),LL(0xa4688d4b,0x5dfd91a5),L_(0x000001ae), LL(0x49aaa1b4,0xfce4aa86),LL(0x8a4a894f,0x5f3c5caf),LL(0xf0a6af85,0x0a082826),LL(0x869fa6ef,0x4cf46392),LL(0x5a750056,0x1d906025),LL(0xb437590a,0x5afd7688),LL(0xfa2a2142,0x5b91f195),LL(0x46dd69d6,0x53028951),L_(0x00000004), + LL(0x0870d771,0x110cffe0),LL(0x4f03a88a,0xb44dbaac),LL(0x68ebc98c,0x849e6d09),LL(0x9e197499,0x126aa5d3),LL(0x374e4b92,0x9e50c62e),LL(0x9406118f,0xf4a6d99b),LL(0x4e25c845,0xc9df6238),LL(0xb15d2756,0xa10c0e52),L_(0x000001fe), LL(0x347cab66,0x816212f7),LL(0x43351049,0x52076e7a),LL(0x90d0771e,0xac804061),LL(0x50393b27,0x509ba99e),LL(0xb81254b5,0x6fa16ea2),LL(0xbe5e2613,0x1a907d04),LL(0xf4aab035,0x2ee00b2d),LL(0x00a0f275,0xab599862),L_(0x0000009e), + LL(0xecfc0941,0xfb9a872a),LL(0xde3af050,0xbec0fc8c),LL(0xebe6b500,0xe7c4ef2e),LL(0x28e4d4b7,0xb38a6c42),LL(0x82362d94,0xc4f9fb0e),LL(0x4e229d20,0xa3690dbf),LL(0xa6e45bdf,0x730c74e3),LL(0xa7b1c90f,0xf2fc481f),L_(0x00000106), LL(0xb887a36f,0xe5e496c4),LL(0xe46148f8,0x16f8ae6a),LL(0x4268188f,0x60936452),LL(0xdcecf1b3,0x828f2ec9),LL(0xeec097ea,0x8a581be5),LL(0x3e062b3a,0x85430a09),LL(0x4da12b49,0x562092de),LL(0xcbb50541,0x33c27b17),L_(0x000000ae), + LL(0xfb667016,0x4148520a),LL(0xb05dd749,0x6530988c),LL(0x4882c146,0x38e93ea7),LL(0xf98af47a,0x6360b046),LL(0x75158008,0x670a2092),LL(0xa8d210f6,0xcea39485),LL(0x590b4493,0xd54fb04e),LL(0xe30eec4b,0xea6ce05c),L_(0x000000fe), LL(0xc43f1354,0x095bed5a),LL(0xc9bc887c,0x40c45485),LL(0x2639073e,0x060df364),LL(0x9ad162fc,0x0ed461e0),LL(0xd17260de,0x48f9f001),LL(0xcef6cf88,0x5e44883d),LL(0xc42e028e,0x78ade819),LL(0x7ee983d2,0x24ef3daf),L_(0x00000059), + LL(0xad0684c9,0xfbbed4ff),LL(0x64e57bff,0x825f2bb2),LL(0x9eb6b035,0xfd8b6643),LL(0x3c213466,0x9c353790),LL(0x7313deab,0x9b0366be),LL(0x2121723c,0xac2996ae),LL(0x953e87c3,0xbd382785),LL(0xf9b6974b,0x3a30236c),L_(0x000001e4), LL(0xffdea7ff,0x5f3b2707),LL(0x68809f79,0xca4a12da),LL(0x374c5228,0x32cc5a86),LL(0x15cef9a1,0xae5f8c0d),LL(0x72616f2b,0xe61ce206),LL(0x75c41da6,0xde33abed),LL(0xa5fc5af7,0x50659126),LL(0x5776a4d1,0x4c16e788),L_(0x000000d4), + LL(0x59994686,0x017dab60),LL(0xe869faef,0x1a3d2819),LL(0xa91c965c,0x95cacbcd),LL(0x1c63a302,0x28898d33),LL(0x91791e04,0xe5b4e674),LL(0x2669fe66,0x4ee8bdb3),LL(0x55d62682,0x333ebff9),LL(0x2111714d,0x88832299),L_(0x000001e4), LL(0x1080f065,0x4df0c3cb),LL(0x7975bc08,0xa4a0f0d9),LL(0x6243d2cd,0xf978a250),LL(0x447d6ec4,0xca8ffce0),LL(0x3c8e28e1,0xa6bda9ff),LL(0x45d5e419,0x3acf30c2),LL(0x7bf52151,0x2b66a867),LL(0x21d9061e,0xbba7056e),L_(0x00000013), + LL(0x8e534a08,0x8f349801),LL(0x029f064c,0x07be931b),LL(0xb893aedc,0x14f71f6a),LL(0x242b0eea,0xe179067b),LL(0x8af895ee,0x99f6bf52),LL(0x5e852a27,0x1d5c2098),LL(0x94bc1969,0x296ab7db),LL(0x7605deba,0x31b9475f),L_(0x000000b2), LL(0xf8d3bc51,0xf8c45d63),LL(0x0d9145a0,0xb3a1daab),LL(0xbc0cd8bb,0x614875d3),LL(0x4f51299d,0xad650d62),LL(0x7baf748b,0xdb91d840),LL(0x83b9d385,0xf5cc54a3),LL(0x840ae765,0xbe2653a6),LL(0xab5a54bd,0x5728a0ed),L_(0x00000086), +}, +/* digit=96 base_pwr=2^480 */ +{ + LL(0x4732d33f,0x2be41906),LL(0x9f1fdd6e,0x9ab150fb),LL(0xb458dd16,0xf3f55fa5),LL(0xb1bb79de,0xd9b88ebf),LL(0xc1d98e1f,0x7b8b17a8),LL(0x7f6beb8b,0x6c86e6b3),LL(0xbc72340b,0x7bb70edd),LL(0xdc7c19d3,0x67a99418),L_(0x00000168), LL(0x22c0fb38,0x56a4a09f),LL(0x9cb6bc12,0x8ded9bb1),LL(0x77d8b51c,0x9f35ca45),LL(0xeb257480,0xf1168ba7),LL(0x770b52be,0x12cdae11),LL(0xed4f42bd,0xde9dff68),LL(0xd326b225,0x5631a8c3),LL(0x1d37f144,0xb14a3c37),L_(0x0000012c), + LL(0x0603f033,0x66b0b95f),LL(0xf969adad,0x57813fa7),LL(0x1acf7746,0x220707f6),LL(0x712a2615,0x71d4cd53),LL(0x2fd4ef2f,0x1f82a44a),LL(0xd9e26293,0x0681773f),LL(0xf763ad20,0xe31fd702),LL(0xa99b206f,0xc3a8767f),L_(0x00000165), LL(0x84569e51,0x41a7f8ce),LL(0x21c3dd47,0xcc9159a8),LL(0xe90e3290,0x06b623fa),LL(0x9e8cf993,0x531760ae),LL(0x2874afd7,0xc9e7cf28),LL(0xe6527ae8,0x293d6e1a),LL(0xf99eef73,0x03d3d878),LL(0x9237109e,0xe1efdba8),L_(0x0000016e), + LL(0xd074ce95,0xcc51928e),LL(0x2af7a58d,0xfb374b29),LL(0x5ec5d4bd,0xd01fb1db),LL(0x6d8cdd85,0x62636565),LL(0x641e476e,0x674fc478),LL(0xe28d244d,0xb39d16a5),LL(0xdbaa94dd,0x5fd5183a),LL(0x6b7fdde9,0xea66d862),L_(0x00000147), LL(0x62ab02a5,0x574c9d49),LL(0x88f7fd2b,0x31232213),LL(0x6c23d660,0xb2ca0c2c),LL(0xce3a1a6a,0x664a406b),LL(0x2ca19917,0x8f549744),LL(0x6f2fc149,0xab32866b),LL(0x41cbc3b0,0x7a277aea),LL(0x25557ca3,0x16026538),L_(0x000000f0), + LL(0x2a8dfc22,0x4fdb7562),LL(0x9be9e5c7,0x29bd5547),LL(0x548d39ec,0x29c79da4),LL(0xf3f7942d,0xc4bc1f5d),LL(0x948e1f79,0x34a7cecb),LL(0xb63229ed,0x76898793),LL(0x39c1a7d6,0xbe3b3419),LL(0x9157ad78,0x2801351b),L_(0x0000012f), LL(0xfece8891,0x59cbeae6),LL(0x85ddee3b,0x8140db30),LL(0xeeab1d34,0x41a033c2),LL(0xb676bba9,0x85703aaf),LL(0x23a9d8b6,0x35046b64),LL(0xb832a7c8,0x9e2475da),LL(0x5b8c259b,0xb51f8631),LL(0xdb18a6bb,0x53eb5dc8),L_(0x000001a2), + LL(0x85cafbb2,0x23cca37a),LL(0x57f26e36,0xc4d2ab36),LL(0x787ec793,0x520b9137),LL(0x436337f7,0xbcfb7906),LL(0x2caa7a0d,0x418cfaf2),LL(0x5a502d75,0x0ba14462),LL(0x066c6a13,0x1d083e40),LL(0xd21212f5,0xb9541e99),L_(0x000000eb), LL(0xe2ab22a5,0xa39384f4),LL(0x07cf7953,0xbdfbaff7),LL(0xaa5f9b05,0x1b083e95),LL(0x782626e8,0xfb350599),LL(0x06f421de,0xe92399d2),LL(0x415729d3,0x04ad8bd9),LL(0xcf103879,0x9370ad78),LL(0x766e0bc1,0xf2c002a0),L_(0x00000148), + LL(0xd8adb3ed,0xd6b8bb85),LL(0x9a142f9b,0x979dc67b),LL(0xfc51be0f,0xf84e32d8),LL(0xf9ccb118,0xf5b6ca36),LL(0x5e79aba7,0x3a900f56),LL(0xfcfd2df6,0x15163143),LL(0x22db9b75,0x5f85f9f1),LL(0xd886015e,0xe7c48af6),L_(0x000001f0), LL(0xd0dec7fa,0x5dcbc466),LL(0xc13f4daf,0x043aefcf),LL(0x613ac2b0,0x60909041),LL(0x9567d2ec,0xf4b79cb6),LL(0x57b5e5ef,0x8e04188e),LL(0x9dd05dcf,0x759c45aa),LL(0xcd8106c6,0xc6c633a7),LL(0x694b84b0,0xe7963345),L_(0x0000001e), + LL(0x68b4a3f3,0xfe8ed21f),LL(0xf39b982a,0xdf9459e0),LL(0xef033664,0x1245ad2f),LL(0x4c26109c,0x6578f9c3),LL(0x7b73834d,0x28e9fc09),LL(0x21a085c7,0x84bd7b31),LL(0x65666df5,0xd5585963),LL(0x9d7af58e,0xfd1e18ec),L_(0x000001e9), LL(0xaf6bc16e,0xe717df29),LL(0xf468848d,0x7c888dbb),LL(0xd747cd3b,0x51097e9d),LL(0xe70801a0,0x8bb9b824),LL(0x172bbff6,0xc27a8a5f),LL(0xf45d5351,0x402074f9),LL(0x0ba6fcc2,0xd7e5a578),LL(0xc1d4e050,0xcb9d2f1c),L_(0x000000eb), + LL(0x3b9d7737,0xf2b990fe),LL(0x1096bf3d,0x3b2d5eb8),LL(0xeb580e65,0xa2ad7396),LL(0xca4cfd31,0xcddd150b),LL(0x4cdae865,0x5cde916b),LL(0x6ffe74e3,0x1b6f19b5),LL(0x1e7dc0b2,0x333016e2),LL(0xc799d8bf,0x46cec318),L_(0x000000b1), LL(0xab36d519,0x9830acdb),LL(0xdd1e911c,0x1a0df89d),LL(0x891db580,0x646bbddd),LL(0xe25f1a5d,0xc4d27510),LL(0x10d55b0a,0x144af2f9),LL(0x5bcea08e,0x50da24a7),LL(0x7ae5f37d,0x9ad211e3),LL(0x73d37273,0xd9d5c417),L_(0x000000a2), + LL(0x2bd93615,0x785d4516),LL(0xd201173c,0x5cbe43f9),LL(0x6f813c93,0xfc65024c),LL(0x5174f5db,0xcbde45cf),LL(0x98aed5fb,0x29d4641e),LL(0xe15ff504,0xb6befd4a),LL(0x92a16838,0x3fb27455),LL(0x7017d508,0xa78ba07c),L_(0x000001cc), LL(0x3aef4cab,0xad9a35f0),LL(0x16e47d7b,0xbfe092d3),LL(0x75d728c5,0xd99290d8),LL(0x8ba65183,0xae8ed203),LL(0x9af2b287,0xe9db0d4e),LL(0x433a1079,0xe6c8ae7a),LL(0x21dd82ce,0x5486b431),LL(0xfbc30bb9,0x7775c8a8),L_(0x00000150), + LL(0x623589c4,0x92135986),LL(0xaae951c6,0x9a74fa72),LL(0x3dfd82d1,0x4bc31a1f),LL(0x060156b1,0x9ab6f26a),LL(0x8b245f24,0xa98e8084),LL(0x317596e1,0xc80c4dc3),LL(0x11d5e680,0xe262106e),LL(0xbee2a8d6,0x60234555),L_(0x000001ea), LL(0x90ed18c1,0x8b82b6cd),LL(0xa80d5059,0xb2182943),LL(0xa584e869,0xa8841e7d),LL(0x7e59ceee,0x74e1e538),LL(0x201d2b08,0x6d2519ec),LL(0x76d5cc62,0x41a115fc),LL(0xdedbfb6e,0xa6f152e5),LL(0x5c18feb7,0xd80d529e),L_(0x000000b3), + LL(0xecd211cd,0x3682bcf5),LL(0xe91e53c5,0x6ca16c30),LL(0x3355812c,0x6b8e8ce2),LL(0x18e076f5,0x77cbae05),LL(0x45a2864e,0xae50657d),LL(0x29b224b8,0x5b740476),LL(0x1853045f,0x9cd59d4a),LL(0x4fef40e2,0x6e774f0e),L_(0x00000092), LL(0x338e4f9d,0x0f66fd40),LL(0x87e39c23,0xd4b5406a),LL(0xb9d5824a,0x309845c7),LL(0x4567fe70,0x40e6539b),LL(0xc9f3a53e,0x4965ee0b),LL(0x0d799507,0x06d618a8),LL(0x28bea4e0,0x01fa0a00),LL(0x8b356252,0xb43cd562),L_(0x00000162), + LL(0xd82f0bd0,0x198a02d5),LL(0xcf78de3a,0xa89bcfff),LL(0x6931d65b,0x98eb3ac0),LL(0x204cbef2,0x796db40b),LL(0xdbd652c6,0x82883eda),LL(0x8c7c0479,0x6355b755),LL(0x3ccc26fb,0xb1589be6),LL(0xdcd445ae,0x8ce9a7eb),L_(0x00000063), LL(0x3ed1177d,0x34daeef1),LL(0x92e7ebd6,0x33e4d5b6),LL(0x252f990a,0x7af9fecc),LL(0xa16a7b3d,0x19533f3a),LL(0x07d26ab6,0xf0584373),LL(0xa41a7a2a,0xc7584589),LL(0x32ddecef,0xf36c6f17),LL(0xf2956cde,0xe47377e9),L_(0x000001ef), + LL(0xc14a919e,0x68fb10a1),LL(0xe2e1dbf7,0x1bce1a5b),LL(0x23f22cb2,0x865d95ed),LL(0x3d7b8ca9,0x9350d70c),LL(0x0559d55e,0xf39cff5a),LL(0x634be668,0x1f6fcd80),LL(0xbb740491,0x31d2120d),LL(0x7202a974,0x2efc5e17),L_(0x000001d7), LL(0x7fe1fd4a,0x6ff361af),LL(0xbd828851,0x26eff873),LL(0x96db8923,0x8d394d9b),LL(0x6a1cb060,0x3ebd8f2b),LL(0x2c56b043,0x71b88fe8),LL(0x91925e0f,0x39b0cfe1),LL(0xea28e59d,0xcb53dd25),LL(0x933a3cad,0x8fbf4361),L_(0x00000112), + LL(0x08433d22,0x13e495fa),LL(0x7698266f,0x51931514),LL(0xd385a184,0x7057cc40),LL(0x7fd1998f,0x8ffed935),LL(0x5d2e260a,0x55f9858d),LL(0x34fdc952,0x353e16aa),LL(0x3d6d1e16,0xd91adeda),LL(0x9e8895ec,0xa78987af),L_(0x00000106), LL(0x77ff974c,0x62e40103),LL(0x869a5ca3,0xc9ddcb20),LL(0x777bb6c1,0x0f3e3498),LL(0x4f97ec1c,0x18133992),LL(0xa7ddecae,0x1c9b2738),LL(0x280ea610,0xae01d593),LL(0xc9770c84,0x30145dcf),LL(0x7c4ed00d,0xa2a8b818),L_(0x00000107), + LL(0x9aa418d2,0xd4f005b4),LL(0x56eacf75,0xf05cca8e),LL(0x8a05a713,0x2382e841),LL(0x3f19077a,0x3c0079f4),LL(0xef823326,0x07e9f310),LL(0x71d13043,0x6311fb89),LL(0x0c6d6593,0x63ca3188),LL(0x0c592a1b,0xfce1253d),L_(0x00000042), LL(0x36fe1597,0x9089e935),LL(0x994e32d5,0xdc455b1d),LL(0x643872ac,0x914013f8),LL(0xac2eba70,0x35f0c433),LL(0x5a85e638,0x59b2430e),LL(0xa786ce7a,0x5225b772),LL(0x920543ca,0x51228731),LL(0x1e47ebe9,0xc56f0daf),L_(0x0000018f), + LL(0x3660052e,0x31ce7476),LL(0xe409da17,0x5b328da3),LL(0x098b5f71,0x607382a2),LL(0x51c3538d,0xc3ee7b06),LL(0xabf1dd7b,0x96d5eed9),LL(0xe8c0d16d,0x1a4ceb18),LL(0x3fe464dc,0x6b9f8f1e),LL(0x0c30d6fa,0x359d987d),L_(0x000000fa), LL(0x10803ed6,0x2947d098),LL(0xb97b5789,0x05d737b5),LL(0xcc27fc50,0x2087e2c1),LL(0x62d40feb,0xdd0d9606),LL(0xf37345b7,0x225ee555),LL(0x7f3858a7,0x9ae8d7c1),LL(0x0cf2ae73,0xdcf4e1aa),LL(0xee00ee77,0x649e41ec),L_(0x00000012), +}, +/* digit=97 base_pwr=2^485 */ +{ + LL(0xd619b611,0x808de672),LL(0x8326922a,0x156260ea),LL(0x1a0841b0,0x63e3e317),LL(0xacb0f8a9,0x806aeb44),LL(0x33483737,0xad9d8a14),LL(0x761a3419,0xbffd26bd),LL(0x2e7a343f,0x6d361b6d),LL(0x4d86e32c,0xf433219c),L_(0x0000003e), LL(0x22d4f25e,0x1f25620f),LL(0xd5c03d38,0x3a87f67d),LL(0x80f73464,0xe876505e),LL(0xe4906c5e,0x491baac4),LL(0x178a012b,0x93e07deb),LL(0x0f735b86,0xd75fad06),LL(0x76ce5dd8,0xc97cb185),LL(0xf5dd4cd9,0x634bbb55),L_(0x000000bc), + LL(0x4a6d5dde,0x37703361),LL(0xc5f2fac1,0x4dee5fdd),LL(0xe014aa4b,0x2218fde8),LL(0xa684a9b7,0x0e229612),LL(0x1d9b66c1,0x7cb5b99f),LL(0x1796c130,0x71c7eff0),LL(0xc0871522,0x27930b1e),LL(0xd19f171d,0x3091f21a),L_(0x000001d5), LL(0x4172f540,0xa74c873e),LL(0xbd512368,0xbc31a6ec),LL(0xd3ea21d4,0x62eff689),LL(0xbd43a95f,0x73a33474),LL(0x1413507c,0xf88fa97a),LL(0xb01846ef,0x8f06b4d3),LL(0xfbac8f6a,0xdc2a3015),LL(0x159ddd58,0x7b911f1a),L_(0x0000001f), + LL(0x5f77e97a,0x2fe873b0),LL(0x5ebf3c8d,0x32fe371d),LL(0x5b9ca7cc,0xc245b054),LL(0x2658798c,0xeaf83f8b),LL(0xf09afde6,0x761d87bd),LL(0x29e1b970,0xa4fd48a5),LL(0x1501c97b,0x5ab0a100),LL(0x1dca9665,0x0ec7beee),L_(0x0000002f), LL(0xc7a92892,0x28296b82),LL(0x171dfdb2,0x171bb70c),LL(0x1dac3a3a,0xad9a13af),LL(0xe21b7ea6,0x1fe361dd),LL(0x2f8b8125,0xccea9acd),LL(0xe8df3c1e,0xa4b48480),LL(0x8a5f495c,0xb8ecc783),LL(0x07fd225c,0xbc6bffc7),L_(0x00000054), + LL(0xb807b638,0x84cef36a),LL(0xcdf4c999,0x5f8d7040),LL(0xc211953a,0xfaefc5ed),LL(0x563ab4c0,0xa17066a1),LL(0x0c339a5c,0xafb2c094),LL(0x517a5667,0xb135b1e8),LL(0x3d2a94a0,0x4526e2ec),LL(0xd9185e4d,0x3c05d493),L_(0x000001b5), LL(0x676f8435,0x2fc5ced3),LL(0xff470fab,0x21ddb195),LL(0xed29f4a2,0x2d94f5fe),LL(0x69f0868b,0xaf8fcc50),LL(0x8631be3c,0x3dcfc141),LL(0x43a07062,0x1c9d9989),LL(0xbafa5f73,0x1cc4a069),LL(0xe1c5c56c,0xf502e626),L_(0x00000082), + LL(0xeb28400f,0xaaec0dfb),LL(0xb5f2559e,0x37f92069),LL(0xf82c9e25,0xde3d65ad),LL(0xca0987ef,0x52dba2b0),LL(0x110760de,0x6f1e9d7b),LL(0xec3c5a7a,0xb68a52e5),LL(0xe6b61974,0x47ef0970),LL(0xa12dbde7,0x952831ff),L_(0x000000e3), LL(0x193e5166,0x2a4c3695),LL(0x2be66d3f,0x90213a6f),LL(0xb1043636,0xecffb364),LL(0x0ea64838,0xaf651989),LL(0x059f3995,0xd04bda10),LL(0x8aa19045,0xfff61b8d),LL(0x76712e84,0xb77b5575),LL(0x6970c5d5,0xfb11370a),L_(0x0000011e), + LL(0x472b3293,0xe37454a0),LL(0x644b36eb,0x2bc89a6d),LL(0xe5b95fef,0x705a9c84),LL(0xc8e9527d,0x6ad037b8),LL(0xc306c56f,0xa3d9152e),LL(0x99bdd442,0x3acd8434),LL(0xd2e50d9f,0xfb35013e),LL(0x10c1418e,0xa1aaf42a),L_(0x000001ea), LL(0xd3620e3c,0x301c9fea),LL(0xa12968d1,0xf9c8c259),LL(0x796a5743,0x1c0237b8),LL(0x92290293,0x56baf809),LL(0xb04d2746,0x81ca3b50),LL(0x93109cb5,0xd70a42b1),LL(0x1ffad7c2,0x829c0f93),LL(0x90fb8081,0x00473bdc),L_(0x00000185), + LL(0xec7bc5fd,0x78a1fc22),LL(0x1e0d9fae,0xfe3efcad),LL(0x7975003b,0x1a1d9870),LL(0x5a8555e6,0x08399c40),LL(0x13808c98,0x1f10285e),LL(0xf7ae407f,0x6b16e9b6),LL(0x95f47114,0x5ef970a5),LL(0x42ba4017,0x58f89d1a),L_(0x000000be), LL(0x09085b73,0xb52c0fec),LL(0x68533122,0xc427c0ed),LL(0xd8c2fad9,0xbd46322d),LL(0x870ca81f,0xccd1cd67),LL(0xab6ba984,0x5510a68c),LL(0x6f619ce9,0x2516fdb1),LL(0xd13d0213,0x89ce2a78),LL(0xd4ddba71,0x33ef2f0f),L_(0x000000d6), + LL(0x2d019ecd,0xd29edf28),LL(0x8e335e18,0xe046e99e),LL(0x4ace8ce0,0x72c0503a),LL(0x42f01d0f,0x9c6d09e2),LL(0xfcb4567e,0x3998b6c2),LL(0x0686ceb1,0x91430be4),LL(0xb8fca6af,0x2236ef5d),LL(0x01c77e85,0x718e1a29),L_(0x000001a3), LL(0xab427d9a,0xab5ae430),LL(0xa843a1b6,0x025f63d8),LL(0xc9500fb6,0xd803e788),LL(0xfb7b9cb8,0xea023d9b),LL(0xcdad70fc,0x803f3ec5),LL(0xa7e50d4c,0x9c07188d),LL(0x9eb540fd,0x822ee2af),LL(0x0d14ab57,0xaff12ba0),L_(0x00000174), + LL(0x0f113d06,0x8230400e),LL(0xed3531cf,0x20fd0e05),LL(0x442851e2,0xd6869a7e),LL(0x1568acb2,0xae871699),LL(0xd7c29d8f,0xad380219),LL(0x512e57e5,0x17e73a2a),LL(0x0239d8b5,0xff1100de),LL(0xa4cc3700,0x3960bc57),L_(0x000000b5), LL(0xd0f458fc,0xec6e136b),LL(0x7a2013e5,0xb3934a8d),LL(0xdeac099e,0x7585325a),LL(0xc5fcf6e8,0xa4aae387),LL(0x73e275f5,0xe0a1bb17),LL(0xc599d358,0x78aeadce),LL(0x5e5ee001,0xf20a237c),LL(0xbc670ce3,0xc755c2ed),L_(0x00000056), + LL(0x980d56e7,0x9e1ae8f1),LL(0xb8a1be4d,0xec417dcf),LL(0xa0d53ddd,0x13c7c494),LL(0x565a5779,0xe8460798),LL(0x4157d87f,0x865e6ed9),LL(0x5fcc1adb,0x43eb5613),LL(0xff942117,0xf8951241),LL(0x65dffe8f,0xbc9c1cd1),L_(0x00000069), LL(0xb27b24a4,0x9c6c39b5),LL(0xdc72853e,0x60f36e47),LL(0x8941b5fe,0x036e5482),LL(0xd9f274e7,0x2bbb4450),LL(0xd2f8bf2a,0x900ba078),LL(0x48bef6a0,0x9a34b9c0),LL(0x548c40a6,0xa419ecbd),LL(0x3d7bc93f,0x5929867e),L_(0x000000b9), + LL(0xe3977001,0x2d5481cb),LL(0x900dd0cf,0x94bad4ae),LL(0x06d3c0a0,0xbee25614),LL(0x2d0029ba,0x4f1ce8bf),LL(0x7ae14d24,0x12c5aff4),LL(0xcb8bd567,0xd5130b01),LL(0xa1cb296e,0x13ab0e47),LL(0x287ae4a9,0x1c30c115),L_(0x000001e6), LL(0xdf0986d0,0x91cfcc0a),LL(0x9e6287d6,0xf8cffb98),LL(0x0a5d81d7,0x6e40495f),LL(0xfe24065c,0x4ac91688),LL(0x6ef91697,0x0bce1292),LL(0xfa7c3394,0x082d9558),LL(0x334da954,0x0d5bbff2),LL(0x41fa885a,0x6904d684),L_(0x0000009c), + LL(0x88636e5f,0x30ed1da7),LL(0x34a6a52a,0x610afcab),LL(0x9193baed,0x00ab78dc),LL(0x40598146,0x40d27bbd),LL(0xfc2510b3,0xdf263e04),LL(0x2c222200,0x4f8a34f4),LL(0xe2fa7ec8,0x7ecddf41),LL(0xf5c8a69f,0xb69fa963),L_(0x00000065), LL(0xdd29b7e1,0xde38eab5),LL(0x1dc06ecf,0xbce53abf),LL(0x287aff4f,0x123a0ff7),LL(0x865d5801,0x9bc53dd9),LL(0x7f2760b0,0xf4d19de7),LL(0x2617ed79,0x59b16830),LL(0xfb36b9bb,0x86d6b37c),LL(0xc68164d5,0x3ce542b7),L_(0x000001d5), + LL(0x83343459,0x785c9888),LL(0xd5898c8f,0x0f97f6b8),LL(0xa5e5e010,0x25a6849d),LL(0xdb272a5b,0x5b826b6b),LL(0xf1d7d775,0x8319ab20),LL(0x81fab2fc,0x051b545f),LL(0x13836d82,0xf3f0508b),LL(0x79a2e73a,0xc87d4ab2),L_(0x00000089), LL(0xe7797e18,0x9d6fea08),LL(0x0b7f377d,0x285c3784),LL(0x13a96505,0x5e5f0355),LL(0x80e5351a,0x92ff2d7f),LL(0xa4907b9b,0xf478e9fa),LL(0xec7c1179,0xd90b6dba),LL(0xd2c36f50,0xbe1d562a),LL(0x797351a1,0xf65a7374),L_(0x000001a1), + LL(0x6dcfe3ad,0x8caa24b9),LL(0xdad4ac58,0xe55e016b),LL(0x42a35993,0x04d4925f),LL(0xf4d85232,0x8c2cb262),LL(0x654bec90,0x44564228),LL(0xd9274933,0x58349da0),LL(0x55dc684a,0xb18184ce),LL(0xe08bebb4,0xe4015bbc),L_(0x0000006e), LL(0xa2f8db31,0xcea81cd2),LL(0xb89f906a,0x5a1b62b0),LL(0xc0a88adc,0x46897bd0),LL(0x3422a9ae,0x6bfb70df),LL(0x0d20f649,0x113b8338),LL(0x197424dc,0xf43ab4e9),LL(0x11c7f33e,0xc17b56d7),LL(0x3e3697c4,0xf0f21e9a),L_(0x00000141), + LL(0x0bbb8295,0xb3c8d4e4),LL(0x3221af32,0x786f9cb5),LL(0xef78da7b,0x6228aab0),LL(0x460bf9aa,0x4fd179a4),LL(0xf900af46,0x2bd49daa),LL(0x42fb7206,0xcadd2655),LL(0xfa16e111,0x992a0506),LL(0x4726c9f4,0xbd04e990),L_(0x000001c5), LL(0x8281f418,0xc3ac19be),LL(0xeed5408d,0xfa218d10),LL(0x2bf29af8,0x2a7befd5),LL(0x3b0d28b4,0xeabb7643),LL(0x28d2a823,0xfa48a66e),LL(0x34709b21,0x92c650bc),LL(0xdf1a83ea,0x261706b5),LL(0xa9a5f258,0xbeb0a33b),L_(0x00000019), + LL(0xe0952a3f,0xd53c20db),LL(0xb6013f7f,0x09d4a480),LL(0x447348d1,0xcb6a7da1),LL(0xd8bfe6fe,0x64e8c529),LL(0xa6067265,0x9034045f),LL(0xa8df68fa,0xff3f3ee2),LL(0x1796dbc7,0xdedc2792),LL(0xe9a130fe,0x4c3f368a),L_(0x000001e2), LL(0x9eed66d4,0x792961eb),LL(0xed55f272,0x9b014919),LL(0x0068193a,0x44cb0bf8),LL(0x32ef3174,0xe22227ee),LL(0x4cb4a896,0x147c8b85),LL(0xc6a73b28,0x2ed1bf6d),LL(0x6804296e,0x77be001d),LL(0x223e6f8a,0x89b143ab),L_(0x0000004c), +}, +/* digit=98 base_pwr=2^490 */ +{ + LL(0x8ea1f1fa,0xec0e9921),LL(0x21044500,0xcba88ccf),LL(0x0c873630,0x6fd4e4b8),LL(0x45764f80,0x056645dd),LL(0x4551a9a7,0x72ed8739),LL(0x025ba6b1,0xa9a78987),LL(0xdd01b45f,0x1f9f1355),LL(0xd2ccea3a,0x807cbab8),L_(0x00000192), LL(0xe24e1198,0x4c6c96e8),LL(0x1a51e813,0x57065d92),LL(0x2ab97599,0xa89e1baa),LL(0xabc4035c,0x057c2aaf),LL(0x1a6716df,0x9c0890aa),LL(0xf802387d,0xc7786bd3),LL(0xa39383e5,0x1f627056),LL(0x00601e4e,0xf2265779),L_(0x00000096), + LL(0x1b109b21,0xd4e8955d),LL(0xe1d0f381,0x45a79e1d),LL(0x6407a6cf,0xe689a76d),LL(0x407f2393,0xd92aed2a),LL(0xac261bd3,0x95547cc6),LL(0x9e62fcac,0x49835e0b),LL(0x1e291077,0xdde8f908),LL(0x7b3d6780,0xc4cab77f),L_(0x000001c5), LL(0xf3dc82cd,0xbde052d3),LL(0xaa1aeecd,0x958c939c),LL(0x24153092,0xca5c0f7b),LL(0x5c0c11b7,0xc9284796),LL(0x3698d827,0xd732af64),LL(0x351c0ba1,0xa3ee0367),LL(0x1b1bf491,0xec2302cf),LL(0xafdf3514,0x4436d640),L_(0x0000014c), + LL(0x1e921eea,0xe09f3da3),LL(0xb7ed8d41,0x97fc0836),LL(0x33451dcc,0x1b62ac0b),LL(0x4bb0f328,0xc7985f30),LL(0x76f68d69,0x7e5bf130),LL(0xa8fcc12a,0x87f28f61),LL(0x0c13fc90,0x097e8f18),LL(0x9299a913,0xec6104d5),L_(0x00000150), LL(0xa95b34c6,0xf8ef488b),LL(0x2c0a7e4f,0x94b33bee),LL(0x5db54ad3,0x4a72bd81),LL(0x0abb0c63,0x57fa905b),LL(0xa7b05810,0x98b0da0b),LL(0xc18c2e82,0xa6507965),LL(0xd53207a1,0xba323d2e),LL(0x1a96c29e,0xe70d8f52),L_(0x000001e6), + LL(0x7179d881,0xc54265bd),LL(0x14b97128,0xe0b08320),LL(0xfd3dafdf,0xb3fd6699),LL(0xb8bb1956,0x416a87bb),LL(0x038f8691,0x01dd4344),LL(0x88826c84,0x7456566c),LL(0x07a8a4b7,0xb2fca59f),LL(0x037671e1,0x797dc52a),L_(0x00000000), LL(0xe8d7f705,0x5d7843bb),LL(0xee9b4c46,0xfa39c4f9),LL(0x303b1652,0xc4a55ae2),LL(0xc15ae7c4,0x3ccdcb67),LL(0xda8ac526,0x7a17fd06),LL(0x8d1d1e92,0x685ac10d),LL(0x5bfc6232,0x048bbb8e),LL(0x233162cd,0xc2cffebb),L_(0x00000111), + LL(0x789c58d4,0x20c13569),LL(0x9b91ab1e,0x14810705),LL(0x032808d4,0x6428e5a7),LL(0xdd56117a,0xb86a6737),LL(0x9ed920a9,0x32ca9ded),LL(0x46d45de0,0x0898b533),LL(0xaef720c0,0xc4b5cbd6),LL(0xe8b625d5,0xd262cadb),L_(0x00000025), LL(0x69824bf0,0xbe07a63a),LL(0xae8c0455,0xc0c992ba),LL(0x56ade4e3,0xd8c4dd74),LL(0xaddf367e,0x1fb487a3),LL(0x10d03d26,0x978961e7),LL(0xced02543,0xb1fb8d4a),LL(0xd9cb94bb,0x4067c3be),LL(0xfd0c7063,0x1e63aa4d),L_(0x0000003e), + LL(0xc18c25f0,0x43a818af),LL(0x8e882098,0xa14d3397),LL(0x2df8f9ff,0xfba08f0f),LL(0xc3139e9a,0xd6f4162c),LL(0xd35b42ed,0xddc9743c),LL(0xa29eeda8,0xeaef65fa),LL(0x1d1cf761,0xc4cffc87),LL(0xf5204083,0x9c04512b),L_(0x000000af), LL(0x76c92be0,0xcce1fc0e),LL(0x4ca92fa2,0x0756de13),LL(0x7ef7ab66,0x6b218d95),LL(0xa4befba4,0xec5df862),LL(0x028018d1,0xb0fb4797),LL(0x0fba684f,0xbb1872b1),LL(0x035fcdb9,0x727d62c3),LL(0xa85f1754,0x52c190b4),L_(0x000001a0), + LL(0x62904b65,0x8c02d54a),LL(0xa33c5b35,0x9d3a0d5a),LL(0xda74fd32,0xcf0e2fac),LL(0xdf27160e,0xa9cf0042),LL(0xce44dc1f,0xf9b4e2f6),LL(0x8815ba75,0x71f7406a),LL(0x5d29fed7,0xba78d604),LL(0x1b44fac2,0xa544aa8d),L_(0x000001da), LL(0x44983ea1,0x4f2a1a98),LL(0xb052a60b,0x2ee09590),LL(0xfd68dd3b,0xf82abd62),LL(0x4cd7b68f,0x3747e4a5),LL(0xa7a82c3e,0x6580fff5),LL(0xfc1c77a5,0x878185a1),LL(0x29848ebd,0x3507cf8f),LL(0xe376e805,0x3d153708),L_(0x000000fa), + LL(0xec3c1780,0xd12cb202),LL(0xde81ad92,0x7dfd0285),LL(0xb71b7749,0x0c150a03),LL(0xf597a8a1,0xe99f4ad3),LL(0xaef51dbb,0x1f4533b0),LL(0x838ed493,0xbcf13b27),LL(0xffc95a8e,0x5623ddf8),LL(0x2b2cefcf,0x37c16683),L_(0x00000107), LL(0xda2ff7e4,0x8ce740d8),LL(0xf3d3ab04,0xd03ed624),LL(0x376415ce,0x76391c7d),LL(0x22ffbe56,0x671ffe7b),LL(0xcc5f4981,0x90390438),LL(0xae289dd4,0xef0984a5),LL(0xf9a1f5bb,0x41c66528),LL(0x3eb05c49,0xef77ff07),L_(0x0000003a), + LL(0x75a1acd6,0xdc003b87),LL(0x913da2c9,0x7357222f),LL(0xb56a4216,0x44f79a9a),LL(0x435f9dd6,0x6316da1e),LL(0x76bee0c1,0x44a0a348),LL(0x528570a1,0x1fc2528a),LL(0x9e402d88,0xadaf6615),LL(0x23fd690f,0x1b05eafc),L_(0x00000050), LL(0x66a31ca1,0x22ceb0f5),LL(0x9817f70b,0xb1910295),LL(0x586cc9da,0xfd424885),LL(0x92631348,0x902c218d),LL(0x2d6b4b6b,0x3b06ab9d),LL(0xf6ce2b8d,0xc2ef9db4),LL(0x5440a618,0x86f795ee),LL(0xe7115329,0x241af150),L_(0x0000019d), + LL(0x9e7167f7,0xb84d399a),LL(0x5d1b603b,0x6e75095d),LL(0x3ebd519c,0x0c38a4d8),LL(0x5a2e8ab6,0x703ee9c9),LL(0xbe621b2f,0x46425097),LL(0xeda0425f,0x3bda2722),LL(0x7eaff2bd,0xc82eaa59),LL(0xddb8e21a,0x9f181562),L_(0x0000011b), LL(0xa357af4a,0x319323ed),LL(0xf55ade52,0x1195febd),LL(0x72cc0544,0xa5494291),LL(0x34dc9234,0xaa005164),LL(0x2b24e83a,0x2e6ec50c),LL(0x61a67644,0x300b585a),LL(0x2a2cce42,0x251bf8fe),LL(0xc53eb03f,0xa53e93a0),L_(0x0000007c), + LL(0x7d64ddd7,0xbd450c1b),LL(0xb6233906,0xf8cf8ce0),LL(0x2f163b01,0x37f9bc73),LL(0x121c5a4d,0x74a3b3e9),LL(0x84c581f0,0xd2aa3a2e),LL(0xa16b9ae6,0x7a258259),LL(0x5182e300,0x6d279587),LL(0x3b163221,0x9054a8d4),L_(0x00000100), LL(0xa5941c9e,0x650238b5),LL(0xbdfac6cd,0xbc313548),LL(0x60bb5887,0x98e5a28f),LL(0x3b8aace3,0xb0c9ed3d),LL(0x1f5ded63,0x943fdf61),LL(0x3ad1b6fe,0x292fee9d),LL(0x3bea9f61,0x765fb8a8),LL(0xb3aff102,0x1c05be97),L_(0x0000005e), + LL(0xc6900b62,0xdf604a9b),LL(0xe5f51ce9,0xad97f878),LL(0xb763f98c,0x89d3ab54),LL(0x6fef5a13,0x3d0efdef),LL(0xdf2543ab,0x39be5cf0),LL(0x51869676,0xd8322a0d),LL(0x34850193,0x90477611),LL(0xe0860abf,0xf04b3d8d),L_(0x000000b4), LL(0xa5518941,0x65e57c44),LL(0xa66238e0,0x78a12a3f),LL(0xd140af78,0x55dcc858),LL(0x65fde14d,0xc7b391cc),LL(0x4d882561,0x2d7dbf32),LL(0xaa74e78f,0x03f85d09),LL(0xed166a45,0x7ad860c8),LL(0xe6da0c2a,0x8d8fe387),L_(0x00000004), + LL(0xb38db64f,0x06980a45),LL(0xdae5a808,0xcd56237c),LL(0xdb2a4bfb,0x1628aed0),LL(0x029fee0a,0x5baf2e6b),LL(0xcfe94b4b,0xbaa05917),LL(0x186a18d2,0x6fb7aa6f),LL(0x3e611766,0x97f81882),LL(0x759b437f,0x25dc9d58),L_(0x000000c3), LL(0x4bf0133d,0x16ed73fe),LL(0x1dbc5e5f,0xae04984f),LL(0xb2380921,0x00f6d531),LL(0x941b93a0,0x9956e547),LL(0xafda37c5,0xbcd796ef),LL(0xeb23c73c,0xd18e7739),LL(0xec321455,0x9f3b6f4a),LL(0xc9846e98,0x7c09f2f9),L_(0x0000010b), + LL(0xe0076809,0x447d7aeb),LL(0x4ec21998,0x691aa304),LL(0x062e1539,0xd6b055f4),LL(0x350b4c67,0x310e0e47),LL(0xeee31b24,0xe16697e6),LL(0x24a9c93a,0xc742c710),LL(0x50e8a296,0x5f791c6d),LL(0x511ff4a6,0x4536286d),L_(0x00000179), LL(0xc52ae1e0,0x36a695b3),LL(0x301dfa2f,0x07145edb),LL(0x89ff6d58,0xb1a2232a),LL(0xd04785f8,0x27854372),LL(0x37e21f65,0x82b68a6c),LL(0x56a575ee,0xb3063755),LL(0x45ea52c0,0x0e5cba73),LL(0x1f5a3458,0xb3b90431),L_(0x00000131), + LL(0xc80f270b,0x05a677e3),LL(0x7473a30c,0x5a28a98f),LL(0x8039064e,0x69a1890c),LL(0x559f0687,0xdf1716eb),LL(0x5a68dd8c,0x216c64b2),LL(0x665fa083,0x89b49b44),LL(0x27c0f780,0x952e61c5),LL(0x5e57e0f3,0xc9f22b25),L_(0x0000015f), LL(0x96d8f8cb,0x9de5f532),LL(0xa33b1f2d,0x6ef509f5),LL(0xd526630e,0x80ca2834),LL(0x2e72073f,0x551bac6e),LL(0x8bdc5409,0xa93f6103),LL(0x74c46ebb,0x78de2a49),LL(0xe616b99b,0xbd4c9f5c),LL(0x5a4f27fb,0x9a70865d),L_(0x000001c4), + LL(0x7162a8e4,0x66505ddc),LL(0x5ed0deab,0x89dc51fb),LL(0x5e972cc6,0xb69dc78b),LL(0x0c6e495f,0x3f182669),LL(0x96d605ca,0x85f61868),LL(0xf6678928,0xedfbfd32),LL(0x863f96f8,0xdece22e8),LL(0x64e644e5,0xf857a4d5),L_(0x000001b7), LL(0xeebfc5de,0x55691ddb),LL(0x1566a700,0x7fa43590),LL(0xe1f4e606,0x796a672c),LL(0x30886e62,0xe5ee14cf),LL(0x40aacab4,0x9b327c1a),LL(0x3294bd68,0x2c002fed),LL(0x103e5699,0xf8494f59),LL(0x9eb1323d,0x51fec2bb),L_(0x0000017b), +}, +/* digit=99 base_pwr=2^495 */ +{ + LL(0xaea3c081,0xcb769c73),LL(0xa9593240,0xb057cdf9),LL(0x4e8217f3,0x220054ab),LL(0x34dceca2,0x5ba1c2a7),LL(0x22b719c9,0x16500d13),LL(0x1b436038,0x0ba1ec57),LL(0xf76e87cb,0x59db2c76),LL(0x77c59385,0xbe033b58),L_(0x00000197), LL(0xeb075ca6,0xddb6a77f),LL(0xc6a6cde3,0x8f578d9c),LL(0xe4987253,0xeaf37819),LL(0xa10e469f,0xdc9c48cd),LL(0x0f98832d,0x51a6e545),LL(0xbf05ea5f,0x195d2d31),LL(0x75cd216a,0x89987c43),LL(0xd18007e6,0x358f3e07),L_(0x000000bc), + LL(0xf2770c43,0xd66a73f6),LL(0x68ca281c,0xa08670f9),LL(0x827efca6,0x180e8f32),LL(0xeac3a96b,0x9979b757),LL(0xbff7df80,0x2d9223bf),LL(0x166015fc,0x30d747dd),LL(0x5475a887,0x9ea9d126),LL(0xfbce1622,0x23756de3),L_(0x0000016d), LL(0xee27c6e9,0xa8ed537b),LL(0xe46c7c15,0x6d7df943),LL(0x4b3f8765,0x335be530),LL(0xdb8a9213,0xcb0ee208),LL(0xb61ee376,0xa4f5fc16),LL(0x4ee85495,0x2d47c111),LL(0x53ced62c,0x453ad352),LL(0xaf641c92,0xe1a21d73),L_(0x000001e4), + LL(0x45292063,0x515c16f0),LL(0xa0b8ef31,0x11c31db7),LL(0x6aca4236,0xa62abc90),LL(0xc862474f,0x94eec3da),LL(0x600cabaf,0x3cb82e64),LL(0xec07e4a3,0x3c98aa82),LL(0xef67e3ea,0x548e4dc2),LL(0x3817b7a5,0x6160dfbe),L_(0x000001b0), LL(0xd5114978,0x212ed3d8),LL(0xa341dfe1,0x71cd3579),LL(0x96070d42,0x9ade2ce0),LL(0x0d16c811,0x6ffd05f0),LL(0x1f2ea906,0x4add5623),LL(0x9034240f,0x871a6489),LL(0x92dc3317,0xd71bf3b7),LL(0xd6ec77d9,0x3faf4d88),L_(0x0000010a), + LL(0xc9de102a,0xab483ff2),LL(0x0cb9492b,0x1910717f),LL(0x1999673c,0x5ba40ad7),LL(0x61f5c7a7,0xec8c1ec8),LL(0x7a954022,0x87870445),LL(0xab6d023c,0x607c1194),LL(0xdaf5008c,0x53612330),LL(0x4ad39492,0x5bf20a93),L_(0x000001ab), LL(0x1b16277b,0xab8ed330),LL(0x045574d7,0xf38e3b31),LL(0xed11cb44,0xab10bb4e),LL(0xa511af67,0x33cee10c),LL(0x9fe7c1d0,0x1549874c),LL(0x6999489f,0xa85b392d),LL(0xfcfe4a15,0x3684decf),LL(0xb3b006a1,0xbaefda3e),L_(0x000001b8), + LL(0x8b50487e,0x3d2bc261),LL(0x1ee7f1c9,0xd8d2c223),LL(0x25894a88,0x1159870b),LL(0x106d0fe5,0xac7070a4),LL(0x37cd1eb7,0x45edd566),LL(0xfcc105e8,0x0d87aaf2),LL(0xbbc54886,0x650d534b),LL(0x4c68c1c6,0xf8b73f4b),L_(0x00000118), LL(0xdb0789c6,0x1c67e752),LL(0x9fdf5e33,0xda66e59f),LL(0x1506a29f,0x04eb7efc),LL(0x566aa91a,0xaf0c681f),LL(0x41ad74ab,0xcd6fd019),LL(0xa0e6609f,0x90d32f02),LL(0xf9f03394,0xc3f5d1da),LL(0xbd5dbcf1,0xa63c99d2),L_(0x0000015c), + LL(0xe7306a80,0x94e82cbf),LL(0x52832eb4,0x61b4102e),LL(0xb381c8b4,0x7f0afbca),LL(0xf1ba6e87,0x8482ae88),LL(0xc8cc076b,0xd709eb28),LL(0x5fc8e5ce,0x0c640cd4),LL(0x1363d1cf,0x6be8f78b),LL(0x4993a63d,0x7a8e7f6a),L_(0x00000140), LL(0x088bf641,0x1bfd703c),LL(0xb3415df0,0x8b57f708),LL(0xf82eeb2c,0x407aa69d),LL(0xa723ed35,0x9767c6b4),LL(0x4dbc3f44,0x52e1a818),LL(0xdffc3e96,0xad89d25e),LL(0xe8855e29,0x89f2e493),LL(0xb2c695a8,0xae2a995a),L_(0x0000005d), + LL(0x32b63b4e,0x7948d6db),LL(0x1bf35a68,0x6953ece3),LL(0x2e4f6945,0x5f0ffde7),LL(0x8e48e233,0x45b68e1d),LL(0x805943a8,0x7c03271d),LL(0x83acacfb,0x136c96e1),LL(0x20b340c8,0xb32c48cc),LL(0xbf7ec9c0,0x9a5cdcd3),L_(0x0000013c), LL(0x14fd95e6,0xeda9d905),LL(0x02288f5d,0x6d5bb5e0),LL(0xe250809d,0x26841f8e),LL(0x3ede31ca,0x5b8a74f6),LL(0x941707b2,0xcd744281),LL(0xf31b5a51,0xcb81c384),LL(0xa2b751ad,0x1c7d45f0),LL(0x98cd14ff,0x0b61414f),L_(0x000000eb), + LL(0x3dacf223,0x49c300d6),LL(0x149cc932,0x984e1f84),LL(0x4f71e87a,0xa635f884),LL(0x2ebebead,0xc51f4894),LL(0x8d815dca,0xb76c6b87),LL(0x60dede95,0xc3f25874),LL(0x83c91cf8,0x53753878),LL(0x6d13e9be,0x4ce987a5),L_(0x0000016d), LL(0x1675f42e,0x22fb2015),LL(0x4bf1c2d6,0x8bc4abf5),LL(0x22da7f9e,0xe7b83f3a),LL(0xe42051b0,0xeda536a6),LL(0x9d89d573,0x3ce8431a),LL(0x64d23c5a,0x3eec2b7a),LL(0x07a6be7b,0xb5fb43a0),LL(0x5b672919,0x4a7d1800),L_(0x00000045), + LL(0x98c6c900,0x54bf1abe),LL(0x835f5200,0x88be0ac1),LL(0x80a69454,0x37a520fe),LL(0x1ae181ec,0xfbf51fae),LL(0x1dbbcb25,0xe0a144d5),LL(0x7481d4f7,0xbb7cdca1),LL(0x89f680f7,0x0d5abe7d),LL(0xb4b624a0,0x391425d4),L_(0x000000f3), LL(0xe424aa1c,0xb4188e07),LL(0xbfd1f50f,0x6575b3b7),LL(0xa43201e2,0xbba2a525),LL(0xe0fef193,0xc9408771),LL(0x2d5a89d3,0x5fad41dc),LL(0x95a6705b,0x7d9dc257),LL(0x9020f8bb,0x290e4eb8),LL(0xfbd17a63,0xbb41e71c),L_(0x000000d2), + LL(0x29e97a00,0x929f484a),LL(0x0aa411ba,0x72a995e9),LL(0x16fc135a,0xc8dd8a3a),LL(0x2226cfa3,0xefeed6df),LL(0xebb1a266,0xbe66eb40),LL(0xd15ad7b0,0x9e390f8a),LL(0x0c3a1992,0x4d13a05e),LL(0xd151e340,0xcf393bac),L_(0x00000175), LL(0x4d898149,0x956cdbeb),LL(0x4f6ce102,0x1a20db88),LL(0x3138d132,0x4bd065b9),LL(0x3956528c,0x082878ee),LL(0x1ab3833d,0xc2946565),LL(0x49e6b0dc,0xe955cb4e),LL(0x10248d30,0x1ae9cc37),LL(0x9e6e01a5,0x5567eab8),L_(0x000000fc), + LL(0x17fc3dfc,0x263e83bd),LL(0x531fddd1,0x56ab0f10),LL(0x11105bea,0xe11b8ec7),LL(0x98797be3,0x1992abc2),LL(0xec87b621,0x2f8db083),LL(0x8d258c0b,0xb3d171f4),LL(0xac2ced2a,0xd1cb21ab),LL(0x7bcca55e,0x47bc4dad),L_(0x0000000d), LL(0xe7fc240a,0xf1f3098d),LL(0xb97a56f7,0x30359457),LL(0x13b63e78,0x5c0291e4),LL(0x0560b59b,0xfd66b1ed),LL(0xffcd1e35,0xab62195d),LL(0xa68ddbd9,0x5e1a88f2),LL(0xc8f7a10f,0x5805f677),LL(0xc770a044,0x145b476e),L_(0x00000166), + LL(0x072f54a0,0xfec59218),LL(0xbad5f014,0x29cb3195),LL(0xdeabd554,0xfb9c1406),LL(0x8cab2ab5,0xf39524ff),LL(0x1480bd6c,0x6fbb57c0),LL(0xc932f537,0x34f118cb),LL(0x9e4e5da0,0x6eb8d83c),LL(0xa6fb16c3,0xc80fc4ea),L_(0x00000048), LL(0x1e1cfe69,0xbe668aa1),LL(0x614afa98,0x9a412d4f),LL(0x4cadab47,0xb94d7822),LL(0x933864b0,0x51053a74),LL(0x424e5f26,0x3dd43fe6),LL(0x600bdaac,0xf8a04f2c),LL(0xc0b432cc,0x4257110e),LL(0x2f4d8257,0x58edc3e1),L_(0x0000004e), + LL(0x2d89ca71,0x42ad308f),LL(0x07b9420d,0xf2ff228d),LL(0xf800adda,0x959a36a9),LL(0xa755b758,0xe791f71f),LL(0x06570844,0xc9aeec3a),LL(0x2525c1b6,0xf99b3c09),LL(0xfae920c4,0x93e1e24e),LL(0x54cdd224,0x4a6949cb),L_(0x00000108), LL(0xbd54c1b3,0xc1192578),LL(0xf0af3d86,0x042c14ae),LL(0x035da967,0x54675869),LL(0x65aba6d1,0x00ddf870),LL(0xfb6525b8,0xcb3618ed),LL(0xa58a9ef1,0xa2776a71),LL(0xd7373181,0x9adaa84e),LL(0x5acac693,0x7c38e845),L_(0x00000017), + LL(0x2f298451,0x17d5163c),LL(0xb8856ff7,0xc5c08271),LL(0xa79d0557,0x32810d21),LL(0x16514840,0xa6e95174),LL(0xce8f06a5,0x393b8782),LL(0xf14b15d0,0xdf3da6f7),LL(0xa398eeac,0xc040c1ac),LL(0xd2eb31c7,0xe9ed34f4),L_(0x000001ac), LL(0xcc4ff509,0x6386bac2),LL(0x294f7bd0,0xf986d8d5),LL(0xec0b7a55,0x55592285),LL(0x96568681,0x15833824),LL(0x307162e2,0x03366d9a),LL(0x196efa15,0x78331de9),LL(0x6afbb75f,0xce11aa87),LL(0x4246ce65,0xa207e319),L_(0x000000f7), + LL(0xe290f3f8,0xdf95cccc),LL(0x2c3a928b,0x954c7e07),LL(0xb87fc4e2,0xc556dd0f),LL(0x5dde9996,0xa10a09f2),LL(0x017edf41,0x126b8ebf),LL(0x11739253,0xe86b6fe5),LL(0x1f9be2dd,0x7b89b849),LL(0x4acd9273,0x572a4b24),L_(0x00000018), LL(0x695c6282,0xe035ed7d),LL(0x8545db72,0x1c4fb913),LL(0x52bc38af,0xe8b8d046),LL(0x500cd8de,0xd8bd36cf),LL(0xc86ac9e5,0x7fd4ed73),LL(0x82941dac,0x968c57f8),LL(0xcd1842b9,0x806ab108),LL(0x4885bf1c,0x7821dec4),L_(0x0000014e), + LL(0x7d0fa54f,0xf8d6129f),LL(0xbddf5a7c,0xb2f43150),LL(0xb4988625,0x3c2f3809),LL(0x1299bbfb,0xb080f7b3),LL(0x84ed45f5,0x20ab0abb),LL(0x824f7bed,0x533e510d),LL(0xd6447243,0xb64fbbb6),LL(0x67c576b7,0xcaa9ee82),L_(0x00000016), LL(0xea0b07fd,0x4253a269),LL(0xf68fe622,0x4572de06),LL(0xa777b687,0xcf599bf5),LL(0xa16d5f86,0x2a811045),LL(0x94a33dfe,0x08732642),LL(0xac970a0c,0xd6867a04),LL(0xeb2b7d05,0x0e51a57c),LL(0xad29a28a,0xbf79a38e),L_(0x000001ac), +}, +/* digit=100 base_pwr=2^500 */ +{ + LL(0x6dc2f627,0x103efe86),LL(0xcb9e407c,0xaaa13853),LL(0x0e6c71ed,0x1b70a2af),LL(0x5ae18f75,0xcf7e3e82),LL(0x1ec11bb0,0x45c36a5a),LL(0x56ab4b56,0xa4e5487f),LL(0x86df052e,0x3868ef9a),LL(0xc7d75031,0xee422740),L_(0x00000060), LL(0xd47edd5d,0xb7ee652b),LL(0xfaa97f40,0xa3dd7397),LL(0x294d2d1b,0x453daecf),LL(0xe65344d3,0x24bd3f56),LL(0x1c9985d3,0xc78c6f61),LL(0x2675985e,0x8ad7e24f),LL(0x8b4060d3,0x6c928213),LL(0xffdfb749,0x27a6ad57),L_(0x000001e6), + LL(0xe5a166a7,0x589be6a7),LL(0x313031bd,0x84289d06),LL(0x704962d9,0x522512df),LL(0x4932ec5b,0x6a669eef),LL(0x4db07538,0xcfe74767),LL(0xabd7aebb,0xe7944dba),LL(0xd27fb22c,0x458ef814),LL(0x6ae70494,0xc9680563),L_(0x000001ad), LL(0x11b3ff8e,0x6c5b60c5),LL(0x915adb1e,0x108c6584),LL(0xf8937e8e,0x64ea3a9f),LL(0xab88406d,0x61d268f9),LL(0xd6126f44,0xae3ff279),LL(0xed1f3032,0x22a5d3b3),LL(0x3a1b63af,0x2fd7a532),LL(0x90caf928,0xf7a42a75),L_(0x00000199), + LL(0x014a6bc9,0x9fdfb005),LL(0x179f05f7,0x91c70d36),LL(0x1f0a00c5,0x1bdb8aa0),LL(0x13b09f86,0x1877e4cc),LL(0xab098b85,0x804921bd),LL(0x47ca3471,0xb874265f),LL(0x78a5d59c,0xeb734d84),LL(0xbb8aac74,0xa87e5bc7),L_(0x000000f6), LL(0x082ec4f4,0x635dd559),LL(0x340b409a,0x4cff9fd3),LL(0xe395e617,0x83237476),LL(0x0e435fdf,0x995df5d9),LL(0xacf9026e,0x0535beb0),LL(0xf60fe3f2,0xc3baddc2),LL(0x21d68a60,0x5d079f08),LL(0x8b800543,0xf7a20c38),L_(0x0000004e), + LL(0x4902c85a,0x061ce962),LL(0xee54d7fc,0xe676ddac),LL(0x8e982883,0xfc5997cf),LL(0xe8ccea68,0x4f3d06f1),LL(0x42831bb9,0x57b80d82),LL(0x625c3604,0x7389ec2e),LL(0x5466a1d8,0x13ff25b1),LL(0xf9c50093,0xfb5d45d1),L_(0x0000005e), LL(0xf2463011,0xab4e32c6),LL(0x08ec78b5,0x24014646),LL(0x53a9c0b3,0xd5ffb795),LL(0x5594ddea,0xf933bdcc),LL(0xdc110f8b,0xd5249289),LL(0x5d52f652,0xcb4147d2),LL(0x95ab06c0,0x968adb2d),LL(0xa1bbff17,0x4039be6f),L_(0x00000066), + LL(0xee64527d,0x186c159d),LL(0x6c1da851,0x87f87d48),LL(0x2ffecac8,0x326e8ac4),LL(0x70a401c5,0x68a082d2),LL(0x0bd703fe,0xbb17b4ec),LL(0x9fe82427,0xd5029b1a),LL(0xf75c6b25,0x8859df58),LL(0xfaac56e2,0x0f97f9bf),L_(0x000001c7), LL(0xe4e3e216,0xaf73b932),LL(0xe721dc13,0x98f115f8),LL(0x825bde14,0x102f60c7),LL(0xf0780fd3,0x92e5a9e9),LL(0x9fcac3f2,0x96a4edbc),LL(0x51ae4427,0x123f8f26),LL(0x622ca1bb,0x3d2d6a72),LL(0x4fa4ca10,0x77d0f199),L_(0x000000a5), + LL(0x10e1302b,0x33e577af),LL(0xbf557eaa,0x98005156),LL(0xca7cba0f,0xe3d41486),LL(0x73d1e9d6,0x3a30b9f9),LL(0xaa3443ad,0x99856b55),LL(0x81921940,0x0ca4724f),LL(0x250cf13f,0xba1696b8),LL(0x9b6f0f69,0xba1deb1e),L_(0x0000003b), LL(0x72f9435a,0xa037cb66),LL(0x7437eb08,0xfaf55d7e),LL(0x579149c6,0x43f7c61c),LL(0x921e4c69,0xeba03e36),LL(0xc342ab26,0x0a3ade34),LL(0x86264eb6,0xcf120523),LL(0xffc57653,0x914e6e20),LL(0x5770eec7,0x2d678456),L_(0x00000091), + LL(0x072e1bd0,0x3106fc3e),LL(0x9739a62b,0xd166966e),LL(0x6ea97e6a,0xbc843284),LL(0x0fb2aa2b,0x1768bc4f),LL(0xbd1f5f87,0x0a51f2d1),LL(0x8890f99b,0x5f53245a),LL(0x6a0000aa,0x38e0dc0b),LL(0x49547c15,0x3dbece21),L_(0x000001c3), LL(0x7b97c512,0xeab56de1),LL(0xcb15a8e3,0x8e0ff0fc),LL(0xc00352a7,0x0dae3bf3),LL(0x716e7d48,0x273cee30),LL(0x83dde892,0x00962a42),LL(0xe18ae53b,0xcb2ce674),LL(0xb8c78835,0x12176412),LL(0xc6faee27,0x94e0f5e7),L_(0x00000062), + LL(0x2d854d6e,0x78569761),LL(0x266d871a,0xe7df9590),LL(0x015aa94b,0x4e8bbf72),LL(0x150482ac,0x12880439),LL(0xaa4b2f7c,0x53495f23),LL(0x5ef777bb,0x04a67481),LL(0x38a798c0,0xc7aadc88),LL(0x308a425c,0x2ccb5f57),L_(0x00000043), LL(0x640f2881,0xb506af85),LL(0x718ebc6c,0xdd21e3a6),LL(0x2c50bba9,0x56098fde),LL(0x7b6e3c4d,0xa8a72185),LL(0x02f4d7a4,0x9218f9c4),LL(0x55530e5d,0xa541a6f2),LL(0xbf7b3c63,0x97421bb0),LL(0x04f0181b,0xe7a08f28),L_(0x0000003d), + LL(0x611c41b4,0xb7a89c2a),LL(0x59bf73de,0x701684bf),LL(0x59572fc8,0x8134fb14),LL(0x116042d5,0xc2db37ef),LL(0x49c61bec,0xf05c9753),LL(0x0b65e976,0xbbefa454),LL(0x3b1cd2a7,0x2cb97f65),LL(0x790f0086,0xc011e095),L_(0x000000d8), LL(0x7cfb4920,0x170eba17),LL(0xc6e8d2e1,0x609e9ebc),LL(0xe69224f6,0xa74251dc),LL(0x88ae4a2d,0xf5331e8f),LL(0xf73bb04a,0xfb6fb779),LL(0xf42091f4,0x7aa76758),LL(0x23ea8c88,0xf921d2fe),LL(0xb43fc164,0xb83f6c60),L_(0x0000014b), + LL(0xae89583e,0xb2f332da),LL(0x1235350c,0x3fc1272d),LL(0x28803380,0xcc94724d),LL(0x52679e63,0xa1b6d063),LL(0x5f59afa4,0x8fd15f1d),LL(0x1998f9ec,0x67514283),LL(0x856a5843,0x0b1f071f),LL(0x44e35d97,0xa4396ed8),L_(0x00000024), LL(0x1fa181f4,0x209d5128),LL(0x36d77579,0x3d71f02d),LL(0x6903b9cc,0x242255da),LL(0x255e80b9,0x0d577ece),LL(0x8c99ca99,0xcce7a8ce),LL(0x6e67b351,0x8274de19),LL(0xb1789c3b,0xd9d46d98),LL(0xb68f17a4,0xbe658d62),L_(0x00000190), + LL(0xb5bb0a17,0xb2d11384),LL(0xc0e43052,0x18ea1e36),LL(0x6499f986,0x67f0543f),LL(0x6c81b7cb,0xfe7f0035),LL(0x0049686c,0xc866a608),LL(0xd1d9672f,0xad63f7a9),LL(0x402ddc59,0x0430d4e2),LL(0x90a63a4e,0x212afc89),L_(0x0000011f), LL(0xd4b7e0dd,0xa5a4f004),LL(0xb4669198,0xe3c06d85),LL(0x9dbced32,0xd46b4406),LL(0x6dc0df8d,0x13bbdb5b),LL(0x26fe23c2,0x9e72fe86),LL(0xf3f82db4,0x9908a610),LL(0x21de6fb9,0xacd7a7dc),LL(0x84215e98,0x8d6e691f),L_(0x0000016f), + LL(0x5bb0cb53,0x6a2a68fa),LL(0x1a45088f,0x097c3677),LL(0x93e569b4,0xf6248b6e),LL(0x76442347,0xf5aa90b0),LL(0x7db67859,0x47468b1b),LL(0x0335b7c8,0xc14d722e),LL(0xbdb192bb,0xa89357c4),LL(0x6091e296,0xabbee708),L_(0x000001a6), LL(0x25d75afe,0xb79d5c35),LL(0xfa81b0bd,0xb8c1d912),LL(0x9c98ad08,0x6f43a564),LL(0x487e8e86,0xb8305a15),LL(0x68d70b11,0x4f253374),LL(0xda0a3387,0xc3ee2674),LL(0x12970085,0x7d1dcff2),LL(0xabf94fcc,0x3abb8f41),L_(0x000000ca), + LL(0x791890ca,0x43ae6f56),LL(0x2ecfea4f,0xbfbd972d),LL(0x1717ff60,0x50d6adf6),LL(0x54f2d354,0x6c6e24d6),LL(0x2e42a5a9,0x8ef967c8),LL(0x2c029c93,0xef490e17),LL(0x74b0a604,0x3a515366),LL(0x835d8fbf,0x878ca8ab),L_(0x000000f9), LL(0x25b70439,0xb19f5e63),LL(0x0a7f849d,0xe7343af4),LL(0xa13e0960,0x93d2a93e),LL(0xd7e39973,0x58e4acec),LL(0x89b42cb1,0xb14fea72),LL(0x128188b6,0xaf37faab),LL(0xfe8c3b50,0x51d9f7ee),LL(0x2b121d0a,0x5997d399),L_(0x000001a5), + LL(0x40c610da,0x70a88d95),LL(0x0ae00340,0x2c6b6f8f),LL(0x941fb569,0xd4a8a83d),LL(0xfd77e569,0xfe3e5239),LL(0x2906296c,0x5d2269e6),LL(0x779ada15,0x65afaeca),LL(0x46fce296,0x41d7fedd),LL(0x0ceb36d9,0x39bc7e8f),L_(0x00000013), LL(0x385b4015,0x5ebed878),LL(0x04b873d1,0xb3e0c338),LL(0x061d2bb8,0xd36ffa7e),LL(0x596e92bc,0xbde7e857),LL(0x58c9a5ab,0x78a8b297),LL(0x34780b9b,0x6bd40718),LL(0xac60511a,0x2d3c02e9),LL(0x83e6ac72,0x6d092687),L_(0x00000112), + LL(0xa33a65c3,0xfd72e9bb),LL(0x99783135,0x67ff52e3),LL(0x8dc913fd,0xab427847),LL(0x8986e4a7,0x5377b12a),LL(0x8a8d9fc8,0x57fef8f8),LL(0xb61bed13,0xf189dc79),LL(0xd4bb14a1,0x2e6d28f7),LL(0xcd3d6f13,0xf00e6579),L_(0x000001a6), LL(0xf42d044e,0x14efe417),LL(0x9fb6caaf,0xabbb19b6),LL(0x951205f5,0x787b1023),LL(0x440fd20f,0x4968e195),LL(0x6d5f6164,0x33cbed34),LL(0xa2722dcb,0xf576c320),LL(0xbddf2d94,0xb80ba0c8),LL(0x414feeda,0xeb25ced6),L_(0x00000008), + LL(0x9ee2c247,0xaa61ebad),LL(0x6aced3dd,0xe5fe2dd4),LL(0x0bd3e3fd,0xfe14f9f4),LL(0x09520569,0xd818d1a2),LL(0xb4968b88,0x82f0bdc9),LL(0x0b8b7732,0x6520e3de),LL(0xfe9e8edc,0x272ff767),LL(0xbc017cf0,0x0f65dc99),L_(0x000000da), LL(0x3dc034f9,0x0a9b50b0),LL(0x4ea634ab,0xe6308ff0),LL(0x7b191db6,0xee04399a),LL(0xddea9de7,0xda7bdea8),LL(0x492d45e6,0xb54c55ae),LL(0x39e666b7,0xf573f4e9),LL(0x0c925a51,0x0292c159),LL(0x71f91622,0x80fc7f50),L_(0x000000d3), +}, +/* digit=101 base_pwr=2^505 */ +{ + LL(0xb83d191c,0x53639a92),LL(0x1ce7b1b4,0x1474fb3a),LL(0x0d260e43,0xaefab808),LL(0xc4e32954,0xfaf9e670),LL(0xb5bae76d,0xde42d0d7),LL(0x9eb4687f,0x5f45bfa8),LL(0x7d89b1ca,0xba638bbc),LL(0xc0ecce4f,0x1de873fc),L_(0x000000a2), LL(0x9165fd5a,0x36c9c2b4),LL(0x318f9f96,0x2cb815fb),LL(0x6f676d6c,0x7560919e),LL(0x41cc633a,0xe2d47525),LL(0x9a79e211,0xe199c599),LL(0xfbee081e,0x265f515d),LL(0x3107dec5,0xd0ea4e25),LL(0x6f2bdc9e,0x5a539ab3),L_(0x0000001c), + LL(0x2d570179,0x6ffa11c2),LL(0x1bfc6586,0x1f882706),LL(0xe25c9a78,0xeb5d1a25),LL(0x12852d54,0xe5fcb1fe),LL(0x4f331734,0xd8c5dfb7),LL(0xe48b7e54,0xcfadcca2),LL(0xe9b41639,0x193402d5),LL(0xb8e1f8c2,0xc49e8f71),L_(0x00000148), LL(0x943f50d3,0x4e33ca43),LL(0xe98a1f64,0xa7cb416c),LL(0xbd595ac8,0xe328cd45),LL(0xf8fd4eb1,0xec3fd8cb),LL(0xf768cfe8,0x9eb626b0),LL(0x476b1bbc,0x069d1524),LL(0x8d0ffe31,0x220edd8d),LL(0x2925aa89,0x60b37558),L_(0x00000036), + LL(0xf4f778c1,0xe83c1031),LL(0xc5cc621c,0x434bc9bb),LL(0xac957c67,0x27bcbc47),LL(0x430a3686,0xace0a905),LL(0xed2cb5ad,0x38aa0831),LL(0x0f4f5d32,0x1fa3c11d),LL(0xc48e91c9,0xedbfb351),LL(0x98229765,0xdf2591e4),L_(0x0000012b), LL(0x72bcfd0f,0xe38e4555),LL(0xa53bea9a,0x97db1ecb),LL(0x196518c9,0x58970b56),LL(0x6895c332,0xc6b46d1e),LL(0x0fe772a3,0x18e44ad3),LL(0x48216056,0xaf1a5dd6),LL(0x86d1933f,0x18ae6deb),LL(0xdf5a53d7,0x4345a6ba),L_(0x0000004e), + LL(0x7fbc903f,0x0deafce5),LL(0xa11885f4,0xf942172b),LL(0xd640aa98,0x17ce6b52),LL(0xdc1bfcf8,0x9d8e40bc),LL(0x14a7d638,0x3c804e7d),LL(0x80e95516,0x63048fa8),LL(0x4af7c92b,0x15381b03),LL(0x88fd6851,0x73ec6a96),L_(0x000001af), LL(0xf3848b20,0xf4f85d1a),LL(0x6ba36666,0xf47de871),LL(0xfbfc6c17,0xf9474540),LL(0x70e03b35,0x72b1ddc6),LL(0xad63874b,0x7f48bdd3),LL(0xd249ea68,0x6a15d7cf),LL(0xc1614192,0xe9d101f9),LL(0xacd8d963,0x1b9b1c2f),L_(0x00000077), + LL(0xb30a7c89,0xc3b944a7),LL(0x8b0aff56,0x97b94164),LL(0xacb2b2a3,0xfade0d57),LL(0xe22c59c3,0x72ad3ddf),LL(0xd8b6f7d4,0x4a332f59),LL(0xe436d0e5,0xb28ca267),LL(0xa69516f2,0xc620d57f),LL(0xdd5b988f,0xb3a24be6),L_(0x00000126), LL(0x0e17b40b,0x7dc7da50),LL(0xf918114f,0xa2d07106),LL(0xb35f8ba3,0xfe3e2734),LL(0x583e00bb,0x27f4b785),LL(0x5b427e96,0xb8110ec6),LL(0x6198344d,0x2666790f),LL(0xc2f9267c,0x9a6bed52),LL(0x15a1c587,0x05e6612c),L_(0x00000067), + LL(0xa3837d8a,0xeddc635b),LL(0xb97227a6,0xf72fcd23),LL(0xcc5023cf,0x16ff8449),LL(0x4d184e1b,0xa6e3cc25),LL(0xb6176789,0x14e442c7),LL(0x200d2b68,0x147e4e2b),LL(0x9903da3b,0xa26bbf3b),LL(0x6b6aeb08,0x849b5dc2),L_(0x0000005d), LL(0x8b8b5947,0xbb6ef597),LL(0x250dac2d,0xac7eb4e4),LL(0x0cadff32,0xfb97f1fe),LL(0x1e4bc394,0x2549b2e2),LL(0xa4d9b459,0xc81df54d),LL(0xedb4d3d9,0x885b7f7b),LL(0xa8c245eb,0xc2642e0e),LL(0xfd485de4,0xbbbd9d24),L_(0x000000f9), + LL(0x41537261,0x82620992),LL(0x9bdd489e,0xf9ff35e3),LL(0xe0746b33,0xfea9ab53),LL(0xc4e354c0,0x5438eb93),LL(0x660ea2e7,0xe43d99e6),LL(0x15697bab,0xcb7634f2),LL(0xacd2ac86,0xcf1f1144),LL(0xd531df3b,0x2d6020c4),L_(0x0000005c), LL(0xba8f9454,0x5eadffbf),LL(0x79147d9d,0x6799957b),LL(0xf22818e2,0xfc746f29),LL(0x7b9b1ddc,0x87c40fbc),LL(0x5518ebb6,0x12ffe947),LL(0x9a81391e,0x997b0c31),LL(0x3a724bee,0x02662680),LL(0xe08a5f24,0x1ebe1250),L_(0x00000065), + LL(0x5f304612,0x3b6bccc6),LL(0x4b51d15f,0x4d9896e4),LL(0x61b0ed08,0xdf4f3be0),LL(0xcedf84c9,0x6ed8e29b),LL(0xf45bde62,0xaa49395c),LL(0x128499be,0x812a1bfb),LL(0x30e7b9a9,0x442e44bd),LL(0xe50016d9,0x251b4710),L_(0x00000177), LL(0xfaa5f8e4,0xb4c58ea8),LL(0x8972e1af,0x6a3b7639),LL(0x2d603c3a,0x64b41953),LL(0xad6090f4,0x1c1cc6d5),LL(0x644bda3e,0x79fc4551),LL(0xe003b3c6,0xc4fe3bf0),LL(0x879a2d4d,0x9f3993d4),LL(0xd0249205,0x90933a0b),L_(0x00000094), + LL(0xb1b095cf,0xbaa0eb60),LL(0x64977e8a,0xed1a5135),LL(0xdb1eee40,0x7fe9e6fc),LL(0xef595c17,0xbc2a7a81),LL(0x4d74eea7,0xafbb2385),LL(0x34f92af2,0xffa66ed1),LL(0x9f246323,0x91252082),LL(0xd49955a9,0xa2901a50),L_(0x00000001), LL(0xa661dce3,0x19a6a510),LL(0xae624c6c,0xf34dd865),LL(0xbf77202a,0x6cbac9c8),LL(0x0d692aaf,0x2471eb03),LL(0xae2ff6e2,0x4e8b52f6),LL(0xfc37aa01,0x96b6740a),LL(0x5ca85277,0x4fc2258e),LL(0x1a66c773,0xd7c07ad3),L_(0x00000122), + LL(0x8dbc9582,0x3efb63d7),LL(0xb52733b4,0xadb5f371),LL(0xa39008b4,0xbebb4df3),LL(0xb7dee0b6,0x6cbc3a0b),LL(0x889767ea,0x0970ea6b),LL(0xfb4bece2,0x7f67b6be),LL(0x890b0f75,0xe72afc5e),LL(0x8395198d,0x6c13b8e8),L_(0x000001e1), LL(0xd3420042,0xcd597e49),LL(0x85730a39,0xd16d451d),LL(0xbb9ddfb7,0xc5ad35f5),LL(0x46f1005a,0x04cccc76),LL(0xfc9aa038,0x199ada1e),LL(0x03f6f34d,0x48f0a0bd),LL(0x200aa943,0x2532adab),LL(0x83389203,0xa871ac66),L_(0x00000098), + LL(0xac1a4aa6,0xd09b5635),LL(0xc63c436d,0x1bf51ded),LL(0x36468adf,0x71acd515),LL(0x98fc5e09,0xf48ba93b),LL(0xed9c3a1c,0x0c6b0d79),LL(0x01b9c574,0x24610fb1),LL(0xb968199b,0xe209e9f6),LL(0xbbce4f03,0xeee7632a),L_(0x00000095), LL(0x2b338eea,0x7c32158c),LL(0xce330212,0xeb40d5de),LL(0x1694cc96,0x981c6977),LL(0xddf4fc29,0x9ea41d2c),LL(0x89f8c78d,0x86af7d31),LL(0xd67b4c4c,0xf2c9d3cf),LL(0xad9e3351,0x16fcf6af),LL(0x702ac15c,0x16ccd30f),L_(0x000000b0), + LL(0xef338e43,0x31986e71),LL(0xafcb621c,0xe99b97d2),LL(0x3f65ee43,0x0c39ca80),LL(0x14bf4d0a,0x4ebb930d),LL(0x840ff2ab,0x894804fb),LL(0x76798c37,0x89a1b227),LL(0x3aa6a099,0x34d5a9a4),LL(0x6f4a66e9,0xdd3ebcad),L_(0x0000005a), LL(0xda731fcf,0x63a60589),LL(0x9cc5953b,0x6c38743c),LL(0x85a6854a,0x08e7cd4c),LL(0xf39a75be,0x936c5fc4),LL(0xfc799df1,0x93f15bcf),LL(0x739e6699,0x4c317bf9),LL(0x2e5c38de,0x6db73251),LL(0x427a1224,0xa307eb83),L_(0x00000136), + LL(0xaa579162,0xdc077b67),LL(0xa1669c8a,0x49cbb3e1),LL(0xd5e45b94,0x26a91035),LL(0xe7362c4d,0x3ddf32d0),LL(0x643b77a9,0x27b4f14e),LL(0xaa5ac709,0x1246b2b6),LL(0xfed505b3,0xce87322e),LL(0x6473f9c6,0xffc4c045),L_(0x00000121), LL(0xf98bfb3d,0x4300e539),LL(0x1c3ea4fc,0x79dff91b),LL(0xe9151768,0xe8106a01),LL(0xced48484,0xd2bbaae2),LL(0xb0b62aaa,0xec766cbf),LL(0x4fd3762d,0x3740af93),LL(0x1969f618,0xeff16696),LL(0x481e8d46,0x08f70c0b),L_(0x00000191), + LL(0xf8786a71,0xe6e4ffd4),LL(0xd4cc0359,0xdeb8f8ef),LL(0x6a7bc267,0xb0f6a0db),LL(0xceec7c5d,0xbe8f401f),LL(0x328be59a,0x0120834c),LL(0xea6a0206,0x5b979c4e),LL(0xc2f2cb42,0x6693b49a),LL(0xc70270e4,0xcb7ea005),L_(0x0000003e), LL(0x4229943e,0xd93cd84b),LL(0x59acbb8f,0x771ac6f4),LL(0x37d3f220,0x5f43f61d),LL(0xe983e186,0xe1ef31c8),LL(0x66433715,0xe567c88b),LL(0x4ca008fa,0x4f949b3b),LL(0x164fa949,0x7f0981e7),LL(0x55a6f6fe,0x85c2f160),L_(0x00000030), + LL(0x7091f78d,0x0fe60315),LL(0x778f4301,0x3d35c1e7),LL(0x42ff27b9,0x4e622807),LL(0x9ba721f4,0x9122d492),LL(0xc966361a,0x0b7b9eb9),LL(0x824265f0,0xd71fbe97),LL(0x90d81101,0xf6012c22),LL(0x3aa81035,0x77c80e09),L_(0x000000c0), LL(0x23dea1b1,0xc562080c),LL(0x0edd3c7c,0xd4bfec34),LL(0x6ac1e8f3,0x8628425d),LL(0x3dfaff6a,0x6eeb0125),LL(0x2bd725ab,0xf2cb02f5),LL(0xeb0228d9,0x63e15d94),LL(0x41589407,0xaa99cd36),LL(0x94061e78,0xf8ab88d8),L_(0x00000109), + LL(0x21abe88e,0xb83d39ba),LL(0xfe497366,0x1a08bb8d),LL(0x3fb2e32d,0x68fadea4),LL(0x90040dcc,0xc78d9e5b),LL(0x6201359b,0x8999242f),LL(0x4ca94e09,0x83f1e38d),LL(0x3e9a9377,0xf5e42def),LL(0xc5c271ed,0xd3d42e09),L_(0x000000da), LL(0xb7e48974,0x7f14caf1),LL(0x52b71020,0x42fb2920),LL(0x840b578b,0x4e0acb78),LL(0x9f5c859d,0xbd1059c6),LL(0xf5ce1fef,0xdfc0d187),LL(0xd99b98e1,0xb702018d),LL(0xd9d695b6,0x7056ff1f),LL(0x1187f703,0x73121d9b),L_(0x0000000b), +}, +/* digit=102 base_pwr=2^510 */ +{ + LL(0x7a6e63bf,0x89622bfd),LL(0x12bebe9c,0xcafd2ca2),LL(0x487abee2,0xd290457b),LL(0xc04143f4,0x05d13bf4),LL(0x716aab7d,0x067f0ae3),LL(0x7740d413,0x5925a309),LL(0xebe6d02d,0x14370b8d),LL(0xe8ef2c27,0xfae20be9),L_(0x000001cf), LL(0x7eac0b8f,0x8d09dc72),LL(0xb463618a,0x49d83802),LL(0xe00f8249,0x50666aae),LL(0xd5a21e88,0x54be3730),LL(0x258522a3,0xa6ce164b),LL(0xbf3fd223,0xfefa386c),LL(0xb7ba5ba4,0x479bc6a0),LL(0x378ecff5,0xece410bb),L_(0x000000c6), + LL(0x97fc3142,0xb8ebdae4),LL(0xfcc42342,0x0addd068),LL(0x2e8a76fa,0x1ba58a99),LL(0x972d98aa,0x585d2056),LL(0x5a290a6f,0x51a66712),LL(0xa47990be,0xab19e664),LL(0xe44696be,0x2f64d1c3),LL(0x490ab4a2,0x0b8ce484),L_(0x0000014a), LL(0x88951457,0x0acf9a53),LL(0x3b3fd199,0xdaaafe7e),LL(0x1c1f1592,0x3015bdb3),LL(0xadb01684,0xb2dbb2d0),LL(0x670d1295,0x3f77ef5d),LL(0xb3f98aca,0x51408bf1),LL(0xb5280fd3,0x0b5ee9d3),LL(0x3a7a5866,0xe5879122),L_(0x00000186), + LL(0x43cd6b28,0x2aa4eb8b),LL(0x719fd8c6,0x2ae67788),LL(0xd8b75613,0xd4b10cdf),LL(0x691c837c,0x7871303d),LL(0x169b2b0b,0x0d01af02),LL(0x6b821f74,0x33573229),LL(0x82eb3840,0x782b872c),LL(0xef815609,0x64bda6ba),L_(0x000000a9), LL(0x4db2512d,0x654fa37e),LL(0x9665a8db,0x761f0aa2),LL(0xe37a4531,0x4eac5b19),LL(0x586ef6d9,0x886dc010),LL(0x014c7183,0x075d0e7c),LL(0x55263f06,0x8a38c3bb),LL(0x5b8b13c3,0xf18380c5),LL(0xcefec3fb,0xb50c9c44),L_(0x000000ec), + LL(0x5b59b03b,0x5131c51f),LL(0x65ab5849,0xd5115c76),LL(0x8739b754,0x840523eb),LL(0xb96b253e,0x8a1f77e3),LL(0x765d9707,0x8742a046),LL(0x7e942e5b,0x1539823d),LL(0x0b3194bd,0x560b9978),LL(0xb52679bf,0xbda6ff32),L_(0x0000001a), LL(0xe66dad83,0x92820e93),LL(0x881e08a8,0x208f9f2c),LL(0x7e5fd839,0x4e86968c),LL(0x305d2580,0x76aeb554),LL(0xb44037fc,0x24c686c9),LL(0xb80d02e0,0x20e62e51),LL(0x5774d5a6,0x653df90e),LL(0xf0000eae,0xc9b31961),L_(0x0000012a), + LL(0xe873ab5d,0x0eaaf4f9),LL(0xfbc7cc75,0x23c2ade5),LL(0xd4d5bade,0x734024b7),LL(0xaca16532,0xb04a1289),LL(0x748144eb,0x599436fa),LL(0x2dc69353,0x292c7b3d),LL(0xe403f7a9,0xb831d302),LL(0xb390c4b2,0xefcb4c53),L_(0x0000003e), LL(0xbd287a5b,0xe0c0eb7f),LL(0xf6a7b102,0x411f77a6),LL(0xc71476dc,0xae920f7c),LL(0x2edbab42,0x28d62eb0),LL(0x3c6b13bc,0xc5296531),LL(0x35b583ce,0xfb32a342),LL(0x66e0af0c,0x23a5daad),LL(0xa449b545,0xa7558c28),L_(0x00000033), + LL(0x52d4b721,0x6383559c),LL(0x52741d3a,0x6d33fc8c),LL(0x98c5775c,0x595adc9e),LL(0xdeefc5ad,0xc4827819),LL(0x94326f99,0x384312e5),LL(0xc9b59642,0xfb777603),LL(0x6642d1f6,0xfebd7a7d),LL(0x04e71c64,0xd51e2c8e),L_(0x00000072), LL(0xf9aa2f14,0xfe188e45),LL(0xc6c9a2ae,0xcfd37d07),LL(0x870985c6,0xb64a1b0a),LL(0xf54e2700,0xd57c3e8a),LL(0x97a88233,0x4ccc22c2),LL(0xe7db1ecd,0xfff2895e),LL(0xb9be3600,0x2a410545),LL(0x8934c5cb,0x71986e7a),L_(0x0000016c), + LL(0x9dfa6b98,0x8664ea8c),LL(0xb006fef1,0x18fac77e),LL(0x2d10f805,0x144d6d0c),LL(0xcefe66ce,0x7ca13881),LL(0x6bc51a22,0x93e1d2aa),LL(0xfe6fc406,0xdde6594a),LL(0xcfe90933,0x92333bca),LL(0x2f2f839c,0xf1434e7e),L_(0x000000b9), LL(0x2169be2f,0x84887df0),LL(0xbd49b873,0xf75f870f),LL(0x4f7c90e8,0xe89d9ed4),LL(0xf464a50f,0x9a76a69e),LL(0x3d9a6cbe,0x0a745d9f),LL(0x0f9b1034,0x87bec297),LL(0x5fe8ae9e,0x7f1ab569),LL(0xad783c5d,0xea58b4b0),L_(0x00000074), + LL(0x0917b4f6,0x9ff8d786),LL(0x35735942,0x05d76bb3),LL(0x995f4b0a,0x58d0fe01),LL(0xe40e0f1c,0x21dccd2e),LL(0x3af9c629,0x40ab0ca3),LL(0x074069c3,0xa30b637c),LL(0x098a102f,0x44888bc2),LL(0xde377018,0xb2e96e33),L_(0x0000016c), LL(0xa96d1903,0x0f70d506),LL(0xe57ed3ba,0xe59c4f2c),LL(0x2492cf26,0x5879a0eb),LL(0x3d130599,0x75760ae9),LL(0x4103b206,0x89f9d0d0),LL(0xa2b74089,0x4b0ad618),LL(0x723e7b44,0xab5c813e),LL(0xebb80451,0x305a1f27),L_(0x00000016), + LL(0xd00287bc,0xd06cea6e),LL(0x063477df,0x7ad6ed83),LL(0xdfc2e4d9,0x7d58a8ed),LL(0x1a10d461,0x9fae700f),LL(0x9ad7943b,0x37aee0fa),LL(0x575deb90,0xef0a0865),LL(0x926de4b2,0x3a26c380),LL(0xc5d7be4b,0x910a980d),L_(0x000001a4), LL(0x66fcdff6,0xdc85a306),LL(0x8a336a7f,0x078dd7d7),LL(0x3626cc20,0x5cdeb063),LL(0xc2b171da,0x273ae54e),LL(0xab82b41f,0x10b49e9f),LL(0x9d867301,0xca9e1b59),LL(0xe2e4e776,0x7eb0c998),LL(0x2437d70b,0x3320fabf),L_(0x0000006b), + LL(0xa0ce92cd,0x7b0fa120),LL(0xb86bacca,0xfbcd14e3),LL(0xa097b60e,0xd21921a2),LL(0x57795942,0x96c19ceb),LL(0x8537f555,0x2145d8ff),LL(0xe5e61d05,0xc2c7c89b),LL(0xbe0f1c2c,0xb88cf04d),LL(0x77a2f17b,0x47c65308),L_(0x000001d7), LL(0xc686be37,0x9b1b0b2e),LL(0x3ba530cb,0xac5182ed),LL(0xea3a3af8,0xfa4f9dd8),LL(0x03a0d517,0x7b2d9856),LL(0x2bd4dfe1,0x6d8ccb18),LL(0xa68f896d,0xc8c4d1ec),LL(0xa1acec0c,0x0fdaed1a),LL(0xdc43340b,0x1a6552ac),L_(0x000000d7), + LL(0xa74443d6,0x1cfe1d00),LL(0x35fc4f26,0x06e2cd4f),LL(0x7dce43d2,0xfe4a6fd7),LL(0x8884f4ad,0x8bc475be),LL(0x9fb2b07e,0x9fe1c66d),LL(0xde1173c5,0xa0cf5d6a),LL(0x6059a297,0x4938219f),LL(0x49237fdb,0x01e57227),L_(0x0000003d), LL(0x5f7cc32e,0x35f11932),LL(0x63db0e8e,0x431b9b60),LL(0x0b8d8078,0xdb56a2b7),LL(0xa040057a,0x856dd526),LL(0x87409cca,0x3d5f500d),LL(0xb482e56d,0xd9cd1b6f),LL(0xbf890467,0x815814ee),LL(0xafa8c19c,0x2dd2fd09),L_(0x0000015f), + LL(0x0680f460,0x4ab480b6),LL(0x71a65ccb,0x061e197a),LL(0x2360920f,0xb306dab3),LL(0x9d9428ae,0xee526750),LL(0xcbaf9d5a,0xf58e47b1),LL(0xd9a6f7e9,0x696a3350),LL(0x5af36c30,0x1f66ddb3),LL(0xeaff438c,0xd4937e17),L_(0x00000119), LL(0x7821be61,0xac9df61e),LL(0x22655044,0x106e2b83),LL(0x1ae7bb1e,0x343bc8e6),LL(0x99139508,0xbc1e06e0),LL(0x166453a2,0x966bd6b8),LL(0x3756d0eb,0xb4bb44e2),LL(0x3795c5a1,0x625fe170),LL(0xb7605deb,0x426f42f1),L_(0x00000048), + LL(0xa4866558,0x06cfe4f0),LL(0x868d9076,0x47442b11),LL(0xec69d70f,0x7bd07599),LL(0x5e554262,0x4c93a1e5),LL(0x9ba31acc,0x5fa3f8f0),LL(0x5118c586,0xed99e567),LL(0x9ed35f7d,0x9e3fd347),LL(0xce15a315,0x0a315f79),L_(0x000001ce), LL(0x3173a0a9,0x86dda811),LL(0xb40d1386,0xb2f0ff2d),LL(0x0d2ec043,0xc03a536d),LL(0x01fe94d1,0x753c381c),LL(0x6d3c523b,0x468beaaf),LL(0xeb47f9a1,0x5d8bebfe),LL(0xeaf18315,0xb071abcd),LL(0x1a924dbc,0x3c1a4715),L_(0x00000080), + LL(0x68ec9abe,0xcce52519),LL(0xd9a705a8,0xef1ab816),LL(0xb6139e2f,0x4fdd8131),LL(0xdab1fe19,0xf7fb9e55),LL(0x94460d6e,0xeb0d1405),LL(0x6a211783,0x6ba4226e),LL(0x8b3a8c56,0xf2eeb428),LL(0xf7e95eb9,0x0b4ffc60),L_(0x000001aa), LL(0xfb4dfe97,0xe66a7792),LL(0x7209db94,0xdb0eb453),LL(0x0352a746,0x3c883ac1),LL(0xdd4b846f,0xb4107c7d),LL(0xe960d5ec,0xf20e2f77),LL(0x61292f8f,0x8b9e3ba1),LL(0x85963097,0x1218ba8c),LL(0xbba1103f,0x61201057),L_(0x000001d3), + LL(0x86acebee,0x08fb83e5),LL(0x0419bb93,0xc48ce791),LL(0x7a851af1,0xd3ade5e1),LL(0xedcfe59c,0xbf3e625d),LL(0xd7763ab7,0xd2c5aaf2),LL(0x625d14ad,0xb7b3d23e),LL(0xbf8e7638,0x7079ecb0),LL(0xc7d9e9b0,0x9c8fcf47),L_(0x00000001), LL(0x8c2d591a,0x074d6a10),LL(0xceed2d69,0x1a1995f3),LL(0x61d18bb0,0xf02767e3),LL(0x33398884,0x68db2be1),LL(0xbdeb7872,0x3e3fa104),LL(0x82f62909,0x2e4ab79a),LL(0x55582545,0x67badda6),LL(0xa7bb473e,0x26c76ce9),L_(0x00000000), + LL(0x8b6d8d82,0xc0ebb49b),LL(0x76edd0b2,0x7ef78c95),LL(0x089746d8,0x86ff89a1),LL(0x30dee546,0x51992a8e),LL(0x8362adcd,0xafcb70ff),LL(0x883f2631,0xa55108d8),LL(0xa13e25b5,0x93138472),LL(0x1fd32baa,0x64387fbe),L_(0x00000097), LL(0xd0f2fcd5,0xe8652373),LL(0xe1299928,0xce8fd7e9),LL(0x16c54d21,0x938b0123),LL(0xad0e62d7,0x4d602bac),LL(0xf9df41ce,0xc55138cb),LL(0x25dfe098,0xbc01e0e6),LL(0xbf9a6851,0x2bdbc63d),LL(0xa70b0da1,0x8b07ceba),L_(0x000000ec), +}, +/* digit=103 base_pwr=2^515 */ +{ + LL(0x9de2eac9,0xaca511d0),LL(0x698e16fa,0xcb4d0031),LL(0x2e96a74c,0x7b609854),LL(0x679b8501,0x87d91373),LL(0x6f39c358,0xa39fd4a5),LL(0x3aea2bb2,0xc7eef60f),LL(0x4e8edd3f,0xd4812888),LL(0x89e1d001,0x2f4d1fa9),L_(0x00000173), LL(0x855b7b6b,0x0a629c27),LL(0xd6fdccbf,0x6bc14652),LL(0x5f32800f,0xd29c1358),LL(0x69e7f62a,0xf3a9fdce),LL(0x9418d0db,0xdaa9f4b6),LL(0xf492796f,0x525ae5fe),LL(0x32f4a27a,0xd91d1353),LL(0xcc1a7293,0xdc6b1bb1),L_(0x000000fa), + LL(0x4b410aaa,0xe776fe9e),LL(0xe46e8257,0xc90ea2e2),LL(0x285dece1,0x02e70a0d),LL(0xd2e4f07e,0x9e22652f),LL(0x7fb0667d,0x5325ca4b),LL(0xe36daaa9,0x04df1305),LL(0xeebe9d3d,0xfcd0755f),LL(0x570e3be0,0x4f74e603),L_(0x000001f7), LL(0xf1587145,0xb8a7ff33),LL(0x37b93b28,0x02791127),LL(0xa408dc4f,0x219fcf52),LL(0x7589c78d,0xbf0f03e3),LL(0x0bb10f8f,0x8d6cdb0c),LL(0xd517b4d7,0x99047428),LL(0x37e06b1f,0xa69f3aed),LL(0x98f8786b,0x8624d396),L_(0x00000150), + LL(0xf569dc20,0xa4ea5b5a),LL(0x734ae209,0x90fc2a73),LL(0x5823b56b,0x7673ea6e),LL(0x7dbb26e5,0xd6657bcd),LL(0xd1742aca,0xc34cd032),LL(0xe888df76,0x8065b09e),LL(0xa8dad269,0xc00f61b1),LL(0x3a07c5aa,0xd150d657),L_(0x00000038), LL(0x13c019ca,0xd0a14535),LL(0x608fa78a,0xfb7e5603),LL(0x1e082856,0x643ad480),LL(0x7bf543a7,0xf5b0db8f),LL(0xfadd24cd,0x7206f2c4),LL(0x1806c9ff,0x3a60a387),LL(0x0a68bae7,0x1164c0d4),LL(0x51de4b72,0xb512a4da),L_(0x00000076), + LL(0xfead3406,0x77314e66),LL(0xc35e0de9,0x4220d9f5),LL(0x86281b42,0x69b8f421),LL(0xe5f95c9d,0x5fd90a74),LL(0x9a89c707,0x8c09fc49),LL(0xe12f3480,0xa6764b64),LL(0x8c161166,0x886f3c36),LL(0x55f40cf1,0x68ad8aaa),L_(0x00000049), LL(0x36790bc9,0xf91f4bec),LL(0x489002d4,0xe8177d77),LL(0x1759ca38,0xe14e5a1f),LL(0xab2e759a,0x9005868d),LL(0xa02b4128,0xaa1dff8c),LL(0x2b9cd06e,0x12d6a4d8),LL(0x578741ea,0x641aef64),LL(0x1a343e8d,0xc6a85c8b),L_(0x000000d4), + LL(0x6d2e1752,0x4fe44c0c),LL(0x7a97ea09,0xec2c9500),LL(0x455f3253,0xa4bedbb4),LL(0x902e1815,0x27a1df89),LL(0xfb3e392c,0x120f8330),LL(0x583ac267,0x9a9698e2),LL(0xe8c87240,0x675c3030),LL(0x32adecb4,0x4f7fb620),L_(0x000000c6), LL(0x402fa549,0x56a1c202),LL(0xb86d1cd8,0xdab68ef9),LL(0xed63845f,0x1723eef9),LL(0x480f0cda,0x7a1853f8),LL(0xb146da6a,0x7337ff75),LL(0x4dd1db53,0xd3685d26),LL(0x41863100,0x9ea6ba31),LL(0xef5caeec,0xa06b6815),L_(0x000000e2), + LL(0x9a0b3f1b,0xfa7c3363),LL(0xf791b828,0x5836e010),LL(0x69b98b78,0xba2d3b6c),LL(0x504f9367,0xcea4290d),LL(0x5860835e,0xf3dd0621),LL(0x22ac6245,0x208bde66),LL(0xd8153e71,0xd2a1a552),LL(0xcd91cd77,0xc7df7c52),L_(0x000001eb), LL(0x68ac84ed,0x0bfe4e12),LL(0xfdf620f3,0xe9aa6d05),LL(0x78c9c26d,0x7d0e875e),LL(0x581fef3a,0x45acf57d),LL(0x4f2f3f1f,0x89769d1b),LL(0xb516d4fb,0x7161f8a2),LL(0xa50e2afa,0xa831731c),LL(0xa569ea98,0x69e2a679),L_(0x00000040), + LL(0x3662a23a,0xafc78e61),LL(0x4dc242cf,0xb32ad972),LL(0xbb40309f,0x790edca8),LL(0x89505a9a,0x3c060f5d),LL(0x112ba1c3,0x52485cd5),LL(0xc5bdc888,0x9feed1cd),LL(0x68110e28,0x49202782),LL(0x49d8d8e4,0x7cea44e5),L_(0x00000130), LL(0xf027ab8a,0x33e94828),LL(0x54f9b84f,0xdd82a038),LL(0xf69e1a64,0x9e8e50f3),LL(0xf1691b4d,0x13fe2932),LL(0x5ffe5329,0x032ff352),LL(0x51f4070a,0x9dcce305),LL(0xc3145c5b,0x62f1400d),LL(0xd89cb5eb,0xbef8bde9),L_(0x00000117), + LL(0xb69b37b8,0x37573b5a),LL(0x464e098a,0x2eac199e),LL(0x4b6ae9bd,0x41109e44),LL(0x96c7e839,0x638c7109),LL(0x99e6beb3,0xa5b03740),LL(0x0943a1c3,0x8490e0cd),LL(0xfab6ecb1,0x4e71bae8),LL(0xb69f0eb1,0x29f06246),L_(0x00000112), LL(0x5b8fa8ca,0x536e2d86),LL(0xa2a3d8ca,0xf4f50e4b),LL(0x4c428120,0xcb6eaa4c),LL(0xb4203e5b,0x69871129),LL(0x6da13d6c,0x218bdacf),LL(0x4f621f85,0x52046a31),LL(0x1ea900e8,0x3a13fa03),LL(0xc7d28019,0x167b70a8),L_(0x00000037), + LL(0x0575434b,0x2bdb447a),LL(0xae8792f8,0x63aef018),LL(0xae6cf0fd,0x3291cfc7),LL(0xe0ee5c02,0x3ae122af),LL(0x5bd690e7,0xff276537),LL(0xbc3516ac,0xf83b9879),LL(0xa4255fcc,0x05236d5e),LL(0x4ca14e35,0xaf60c6b5),L_(0x0000010d), LL(0x7d6c65fb,0xc17c08e7),LL(0x737de42a,0xac0df2a7),LL(0x520e48ce,0xaceb43b8),LL(0x2f791d6a,0x57fe87f0),LL(0x662b9dfe,0xc51cfa7c),LL(0x884ed1f4,0x75a9efdd),LL(0xb5ee76b0,0x3f9fe081),LL(0x61e43ed8,0xb9598115),L_(0x000001b1), + LL(0xaa029d13,0x82e47d41),LL(0x1c1e4e8c,0xf692383c),LL(0x819d110b,0x0caaa47b),LL(0xcb280e34,0xf6e315e8),LL(0x49f5a7e4,0x68659604),LL(0x6db3e3bf,0xc18d2a73),LL(0xb38233a8,0xab54c2d8),LL(0x216ab95c,0x670af6e7),L_(0x000001b4), LL(0x5df2f21c,0xf79cd8d1),LL(0x95f873cc,0x8946ec9e),LL(0xd4ae259b,0x352c8cec),LL(0x6383026a,0x4b6773b4),LL(0x574f14c8,0x7327edc3),LL(0x43f9e116,0xfc2d9802),LL(0x58a2e8d3,0x26360b9c),LL(0x2ae789b4,0xeac487c7),L_(0x0000004b), + LL(0x38176573,0xcfa66e36),LL(0xe576c7d6,0x40446421),LL(0x5a9fe083,0xa7e0a9d1),LL(0x43da69bc,0xcd5cfda4),LL(0xaca35d4c,0xdb98b2d8),LL(0xcc88e119,0x238da31d),LL(0xe775938e,0xd74d2fe0),LL(0x0e845777,0xee458b07),L_(0x00000086), LL(0xe869d146,0xfc21befe),LL(0x6f8ded71,0xf57aacc7),LL(0xa1f5602a,0x8a4a8706),LL(0xdf77dad6,0xe88d5556),LL(0xf0fb8eab,0x38891e24),LL(0x3a9313c2,0xd33f4e50),LL(0x4e334d02,0x267f4849),LL(0x45dbbeac,0x7b8b078d),L_(0x0000001d), + LL(0xa128f198,0x77a504d7),LL(0x706161d0,0x0d717bec),LL(0x8d3f449e,0xcd6aa437),LL(0x4c327553,0xbfa09758),LL(0xecfed023,0x131032e9),LL(0x4abfe666,0x2301de73),LL(0x972524c6,0xd67cd7b5),LL(0x1b68a20a,0xd8e4bd98),L_(0x000001d7), LL(0x75c85608,0xa0dd411d),LL(0x136bd0f9,0x8521a20a),LL(0xe9c06f6a,0x34117a07),LL(0xb1417701,0x625cc2c0),LL(0x534fecc6,0xe6f01c93),LL(0x698e9742,0xcea5bcd3),LL(0x43a9724a,0x54b554d5),LL(0x77820ced,0x7954cbcc),L_(0x0000005a), + LL(0x07759296,0xef4a4c48),LL(0x67e62639,0xef609727),LL(0x91ba0b01,0xc40bb739),LL(0x7f62ccbc,0xb7eda85e),LL(0xb14485bc,0x2a55f22b),LL(0xeab4bc94,0x091f3fde),LL(0xb13e2d7b,0x72b44ddf),LL(0xa6958062,0xd4d990ab),L_(0x000000af), LL(0x579003db,0x14ea0dda),LL(0xc0e83ba2,0x10cf6ee9),LL(0xc9f677ef,0x8e3eae8b),LL(0x1862146e,0xcfe0b037),LL(0xa6cdb8a8,0x6d8e5bd3),LL(0xf50d8419,0xdd3f0ea7),LL(0xf42ab2a9,0x8a6b5e3c),LL(0x12f65451,0xe62ff71b),L_(0x00000097), + LL(0xd1a8afb2,0x0460a7ae),LL(0x6526a7b4,0x80ab037a),LL(0xf752335e,0xf14c43bd),LL(0xd11a8f65,0x9b989eab),LL(0x23fa924f,0x0976a80f),LL(0xf16dad87,0x4e440171),LL(0x3baaac45,0xb9635c0d),LL(0xf0f34704,0x0e1ca863),L_(0x0000011b), LL(0x47c957ab,0x40fcc076),LL(0x420a4e5f,0x24cf57e8),LL(0xcec847e3,0x1f189869),LL(0xb20db2fe,0x27f88dee),LL(0x48712498,0x423c17a4),LL(0x284e6344,0xfe029568),LL(0x10b9bfa3,0xd426c180),LL(0x07626ee2,0x96cbf2d8),L_(0x000000c4), + LL(0xfdd27a3f,0x8a898f02),LL(0x37d5f07a,0x9e4691c0),LL(0xdc3f3126,0xb1ce0f65),LL(0x81c10b29,0x73e7fd11),LL(0xc595f2a8,0x5a3fd848),LL(0x47d0e340,0x439d759b),LL(0xbc668622,0x87538694),LL(0x461b9eba,0x3484697e),L_(0x0000013b), LL(0xbd5a6c1c,0x3d20296f),LL(0xf99ebce4,0x59fd2232),LL(0x51ac1eec,0x2fb986d6),LL(0xe71f9d4a,0xa5d1433c),LL(0xa289f5b9,0x4ece5225),LL(0xd51b9288,0x6ec5e037),LL(0xcce86717,0xb823e469),LL(0xac199283,0x45b78f23),L_(0x000000aa), + LL(0x16cc257b,0xbfa48e10),LL(0x08472093,0xc8ac9e7a),LL(0xb7492e4e,0x471d73ed),LL(0x53d1bd1c,0x243428ce),LL(0x40adeca7,0xb7d49d17),LL(0xd60077c0,0x7a95c66d),LL(0x737593cb,0xf237e1e5),LL(0x6a7c6f6e,0xa7929ff0),L_(0x00000060), LL(0x6c3da59f,0x894bcc2e),LL(0x7b3c416f,0xa493d3e8),LL(0x21231992,0x4c0a5993),LL(0xf376d082,0x0d5c6c61),LL(0x9e1550e6,0x2a3430f2),LL(0xf1d1beb9,0x11f95e81),LL(0x7b6bc5ab,0xa78bae02),LL(0x17858c60,0xbdd9ea90),L_(0x00000195), +}, +/* digit=104 base_pwr=2^520 */ +{ + LL(0xc0a103c0,0x0b8b69f6),LL(0xb96747cd,0x3f30c14c),LL(0x9e707ed2,0x029690c8),LL(0xd1292336,0x884265ce),LL(0x55cfec2b,0xc87e6275),LL(0xa097ab7d,0xa5558f62),LL(0xf635118e,0x23dda1ad),LL(0x5770b69e,0x1b70ffff),L_(0x0000011d), LL(0xe9851576,0x4f2606bc),LL(0x8c9c6bbd,0xccb92cf3),LL(0x58d1b308,0x2ce6913c),LL(0xbbdfb6fe,0x1967bfef),LL(0xe5cb515b,0x453132fc),LL(0x2527584e,0xc389b2f2),LL(0x2a1591df,0x0460d618),LL(0x602d5761,0x7498b0d1),L_(0x000001c5), + LL(0x59381757,0xb38cffee),LL(0xa135a61f,0x8ac17028),LL(0x2c9fe1a1,0x30418249),LL(0x3ecb968e,0x4c958b81),LL(0x6f834a0b,0x523285e8),LL(0x54df836f,0x29e4b05e),LL(0xf77b5fb2,0xe864d898),LL(0x48371505,0xcce3c32e),L_(0x000001b3), LL(0xce8bcd8c,0xc4a0e5e3),LL(0xfb74fd48,0x18996dc2),LL(0x875aaa20,0x5e9f86f5),LL(0x5eb82718,0x6642422b),LL(0x87431019,0xd6d6f0dd),LL(0x9d7e7982,0xcb46f571),LL(0x346044e6,0xa348c675),LL(0xdc02d250,0x517508af),L_(0x000001ec), + LL(0xa7a865fb,0x18145140),LL(0xded9f8c1,0xba498095),LL(0x40df2b19,0x3834dace),LL(0xa0c7dec0,0x3687e664),LL(0xfeee9a9e,0x09d971f2),LL(0xa032ee72,0x07148796),LL(0xa0b2cc31,0xb65bcec3),LL(0x46067d48,0xd8c2af65),L_(0x000001f5), LL(0xb094864d,0xd715c856),LL(0x79c821c5,0xbebe04f7),LL(0x62cf6310,0x3844c220),LL(0x89986c83,0x2644f899),LL(0x43935732,0xf80ac439),LL(0xfd741ec2,0x299a9f2d),LL(0x3022b589,0xbcf4790c),LL(0xf5663af1,0x767587f6),L_(0x00000046), + LL(0x67f89299,0xf1cdfa06),LL(0xda7b7826,0x97fea55e),LL(0xdd938459,0xc39699cc),LL(0x17d73f54,0x5971fbd4),LL(0x9f08e1d8,0x3631f804),LL(0x65a870ee,0x51a73b8e),LL(0xa3cdc8e3,0x54f69b2a),LL(0x85999014,0x59cbe0fe),L_(0x000000d6), LL(0x9c778792,0x085bfaeb),LL(0x17f6a7aa,0xe07f9788),LL(0x1e5f6c61,0xb271762a),LL(0xcc6a5dff,0x5d3e3cd3),LL(0x895eb0df,0x6eadd52e),LL(0x2cd665b4,0x92523b34),LL(0x93078c5b,0x2ebcff33),LL(0x48620212,0x777dd50c),L_(0x000001f4), + LL(0x57bfa90e,0xfe3fcdbd),LL(0x118dee98,0xcfc4ad1b),LL(0x0a524cec,0x6d1ffa5c),LL(0xe0d7420c,0x919d859d),LL(0x44af5553,0xf81fb745),LL(0x0981d6de,0xa17a7c3f),LL(0x6680c297,0xe0cc4fe7),LL(0xf6d22135,0xde36e57a),L_(0x00000115), LL(0xd78b0a1b,0x0b8ae03f),LL(0xac0c4195,0xc2638c06),LL(0xc994e2c7,0x4523ebb6),LL(0x49b75a94,0xdb72a765),LL(0x73310e2b,0x57a73d05),LL(0x8ce6d6b8,0x1e23ae4c),LL(0x4309eb9e,0x842f06ae),LL(0xf98aedc0,0xedf71264),L_(0x000000ed), + LL(0x7f81bbbd,0xbe47c62f),LL(0xd4c38f8c,0x49173568),LL(0x085a52c0,0x7d057b19),LL(0x612151d8,0x2de5c929),LL(0x8a4ea946,0xc4036de1),LL(0x0a8300dd,0x5a6e290b),LL(0x195518bc,0x894ed83e),LL(0x35e21097,0xd8dd898c),L_(0x00000074), LL(0x9a47676d,0xd0eee2e2),LL(0xc0b89ddf,0x8a7bf325),LL(0x0c5c995a,0xa5131a24),LL(0x9573e514,0xe5998b6e),LL(0x93979878,0x7ba59336),LL(0x7906f3c8,0xff513215),LL(0x093cead5,0xb76588a1),LL(0x243bc2e1,0x5abc03c8),L_(0x00000005), + LL(0x1074b7de,0x07c9d208),LL(0x234d2231,0x524b8b4b),LL(0xc6fe7d17,0x469a9182),LL(0x688b3e26,0x35461f18),LL(0xb0f49d1d,0x78f9ef1e),LL(0x4bb2130a,0xe72b7216),LL(0x9240661e,0xec1e4e46),LL(0x294674d4,0x95346b15),L_(0x000001f1), LL(0x2beae096,0xa594d4cd),LL(0xf533e3fe,0xdb75a21c),LL(0x99051c26,0xea1a808b),LL(0x9c10237a,0x230cfcd1),LL(0xe6396566,0x90aedfcf),LL(0x18cdd934,0x8c399655),LL(0x446502d8,0x0de6be10),LL(0x4a6de065,0x7b3640aa),L_(0x00000048), + LL(0x4f3b2020,0xf6315f2c),LL(0xf2e07c55,0x62ee6844),LL(0x9040fc8f,0xaf168881),LL(0xaec6bb02,0xa1922677),LL(0x968e4dee,0x53b90132),LL(0x92b21f9e,0x2b9cb75f),LL(0x4ae4c1d6,0x65408d1b),LL(0xd20f3732,0x70be8f94),L_(0x0000013d), LL(0xb2d1ac5c,0x05c00995),LL(0x4e16ae2b,0xf20b5acf),LL(0xd04ee882,0x1f8c5834),LL(0x35db5e06,0xc9dbfe41),LL(0x3a42f540,0x3f7b55b6),LL(0x500e266e,0x52ce3b47),LL(0xda6911e6,0x0c53d0a4),LL(0xee43c4db,0x6052a6d7),L_(0x00000194), + LL(0xfdcd47d9,0xf7f0245d),LL(0xc1b6e80d,0xd2bf51a5),LL(0x9077fd5b,0x4a61b710),LL(0x97dc422c,0x3089abb3),LL(0x2c00b155,0x893fa316),LL(0x40add702,0xb9d804ac),LL(0xc77985f1,0xdfaaf13d),LL(0x4150d1b3,0xaf9d85b4),L_(0x0000016c), LL(0x24cad038,0x92589d47),LL(0xd0dfdf3a,0x06f77875),LL(0x6b4012d5,0xfe067b9f),LL(0x4426c933,0x2ed7298d),LL(0x5353c502,0x67948217),LL(0x156d92c1,0xf0f675fa),LL(0xee2b368d,0xba020dd5),LL(0x3367f41b,0xd5aae0ea),L_(0x00000182), + LL(0xf394cccf,0x75917fbb),LL(0x1b52b513,0x87569974),LL(0xbeb1242f,0x4a568f34),LL(0xb03e5543,0x86a66b14),LL(0xc094f1b5,0x93edd5ae),LL(0x7da45bde,0x2fb2c47a),LL(0x960491e1,0x678ce19f),LL(0x890efc78,0xf4853187),L_(0x00000031), LL(0x7699808a,0x4363742f),LL(0xd3a570ac,0x1812091b),LL(0x4961aaea,0xd32e88d4),LL(0x97dd4a2f,0x1b33e755),LL(0xb3987552,0x79ed95a8),LL(0x669664b0,0x0837ce5b),LL(0x3584f341,0x3f9a0cc0),LL(0x116266b7,0x64a902cc),L_(0x000000e0), + LL(0x68c78f5b,0x0660e9a1),LL(0xb0698be3,0x5c045493),LL(0xc1ca94da,0x82b986b9),LL(0x4e8d0287,0xfafe7c0e),LL(0x02cfcf45,0x9235965c),LL(0xcf187fb8,0x316bebe3),LL(0x3fb90363,0xc42b951d),LL(0xc6248aec,0x26710295),L_(0x00000078), LL(0x5d4d89fa,0xc24f0956),LL(0xa11f38e2,0x398a2047),LL(0x9a0accc0,0x9e479268),LL(0x8ccafb17,0x468f61d6),LL(0x0c01f836,0x52f96c75),LL(0xeb043b98,0xacc6ee07),LL(0xa59405be,0xc63baaf6),LL(0xeddd33c9,0xba136442),L_(0x00000009), + LL(0x6e231385,0xc33b7b64),LL(0x1eb1f532,0x3bd3bca3),LL(0x3a7aee90,0x0882ad60),LL(0xad95f2eb,0xddaf31c3),LL(0x4c9d8e29,0x59c130df),LL(0xbdd1470a,0x7ac309bd),LL(0xe3bf394f,0xaafb8369),LL(0xedba4812,0x7f598209),L_(0x000000e8), LL(0x8bea28c8,0xb5d505d9),LL(0x81e772f9,0xfa065dfd),LL(0x15db5d5b,0x7e590809),LL(0x141e8679,0xc4ffd236),LL(0xf14602ff,0xbf521149),LL(0xcfd2b215,0xc500a7a4),LL(0xc04e3706,0xe4c06db1),LL(0x3e28ec80,0x93d40cbc),L_(0x00000009), + LL(0xb6cf9729,0x530c705b),LL(0x1a470405,0xe8292e38),LL(0x85b81d18,0xc5fdb2b2),LL(0x484b843b,0xa859e4fe),LL(0xeeae06f4,0x4e3895d5),LL(0x0a67c915,0xe6119b5b),LL(0xed968aa0,0x9264e00f),LL(0x66ed4c20,0xbf1d85ad),L_(0x00000091), LL(0x8c186038,0xd8a9d836),LL(0xee5d4b51,0xcf0b68ca),LL(0x89ad96c1,0xbcd75bca),LL(0xae1fa341,0x667fbe53),LL(0xa297bd27,0xe40caf5f),LL(0xd6165d4f,0x329b45f8),LL(0x62d6bced,0xc11413bc),LL(0xd1c11022,0x7e0d7384),L_(0x000000b1), + LL(0x422577fe,0xa39b851e),LL(0xa0188e61,0xf5e55bb8),LL(0xe96325dc,0x8cd5c092),LL(0x9087223b,0xe2c1a18b),LL(0xca56bfcd,0x548d8395),LL(0x4ae67f8d,0xaa6db861),LL(0x8f44b4e9,0x218ba5b9),LL(0x317abb0e,0x04386052),L_(0x000000d2), LL(0x27c1441a,0x91b2e14e),LL(0x6ac7c848,0x5377e2b3),LL(0xd40844b6,0x9d93badb),LL(0xe505b8cb,0x4d7f3493),LL(0x615a64c9,0x20a05d5d),LL(0x90a5eb78,0xa4cb086a),LL(0xed4783ac,0x36415b6c),LL(0xf10f3d20,0x56659094),L_(0x00000179), + LL(0x8da0847c,0x0eeb045d),LL(0xac240867,0x98145c00),LL(0xe7ce6952,0xb5d0c780),LL(0x315dd662,0x189fc413),LL(0x41646f48,0x4392d048),LL(0xc963ad1a,0x1e77199f),LL(0xebc649ee,0x83e1f918),LL(0xcd6ca624,0x13b6a99b),L_(0x00000169), LL(0xa873cec3,0x2108af54),LL(0x0b55c26f,0x3989bd71),LL(0xe0f27726,0x1e5e0053),LL(0x5c7e0958,0xa8452157),LL(0xf8e7b504,0xb64d38bf),LL(0xf180ac5c,0x8c8c65f5),LL(0x32a84a9b,0x8f00c232),LL(0x898ca7ed,0x1a1639de),L_(0x00000096), + LL(0x5f1724ec,0x6746f213),LL(0xe1073527,0xa966b0a8),LL(0x3ad14203,0x5bc4272f),LL(0x39620db2,0xbcd33a93),LL(0xe5eae695,0x26bac2fb),LL(0x0e4497dc,0xb7d647bd),LL(0x3a195407,0x7f7ed906),LL(0x899ce3f6,0xadd76129),L_(0x000000f3), LL(0xd7a2bdd5,0x333eb7cb),LL(0x51de1b18,0x8e185580),LL(0xa269b8e8,0x486cd055),LL(0x3555823a,0x6689b4be),LL(0xaa52baba,0xcd8d6ffd),LL(0xd072a45c,0x9cba9f57),LL(0xba53f86f,0x74a8d5fb),LL(0x16481f57,0x10747d58),L_(0x00000010), +} +}; +#else +const __ALIGN64 P521_POINT_AFFINE ec_p521r1_precomputed[105][16] = { +/* digit=0 base_pwr=2^0 */ +{ + LL(0xb331a163,0x18e172de),LL(0x4dfcbf3f,0xe0c2b521),LL(0x6f19a459,0x93d17fd4),LL(0x947f0ee0,0x3bf7f3ac),LL(0xdd50a5af,0xb035a69e),LL(0x90fc1457,0x9c829fda),LL(0x214e3240,0xb311cada),LL(0xe6cf1f65,0x5b820274),L_(0x00000103), LL(0x28460e4a,0x3b4fe8b3),LL(0x20445f4a,0x43513961),LL(0xb09a9e38,0x809fd683),LL(0x2062a85c,0x4caf7a13),LL(0x164bf739,0x8b939f33),LL(0x340bd7de,0x24abcda2),LL(0xeccc7aa2,0xda163e8d),LL(0x022e452f,0x3c4d1de0),L_(0x000000b5), + LL(0x1e90cf08,0x99dd36bc),LL(0xb3fa1f1c,0xb26b07ec),LL(0xa0e797d1,0x1d1ae2d7),LL(0x83d50825,0x6d377aaa),LL(0x4bd9d902,0x82ebb4df),LL(0x1a96372a,0xcd8e6603),LL(0x3a3a0193,0x40a46141),LL(0x3417e594,0x1213bf3f),L_(0x000000c8), LL(0x813d2ee3,0x6b30fa0d),LL(0x7b8df1ab,0x4af6e07a),LL(0x7a757e5f,0xd4cd1924),LL(0xb5c9c9bf,0xef4f928f),LL(0xddd9f1bb,0x4c836216),LL(0xa05590d1,0x3e26d4bb),LL(0x5ae35a88,0x777769f8),LL(0x8053f9f6,0xfc36d933),L_(0x00000063), + LL(0xbee9cf4d,0x976f1bd6),LL(0x02d2c8ce,0x43161975),LL(0x0dd75a48,0x8b5acff1),LL(0x028ed35e,0x251d2419),LL(0xe8d69f8b,0x0896bd46),LL(0x5cf2d6bd,0x2d891ecd),LL(0x3cda9537,0x325acaca),LL(0xaeec8eb5,0x21ef148c),L_(0x00000092), LL(0x72cfa6c0,0x50f74360),LL(0x212fac46,0xde49d2c8),LL(0x867882e4,0x68ef61e3),LL(0xd816ad67,0x67c6e2ba),LL(0x761716ea,0x8fd1aae7),LL(0x8be97c55,0xd4154e81),LL(0x7978aabf,0x63655c0a),LL(0xeccbcfc3,0xbefd316e),L_(0x000001dc), + LL(0x798d6d77,0x09438c81),LL(0xf17de43a,0x2d8472d2),LL(0x3350ea81,0x4f83c578),LL(0xa8745c47,0x257f1e83),LL(0x56432cf1,0xaaa0e9e7),LL(0x7e0362ea,0x8e2ff9cd),LL(0x66e30e44,0xa43e4838),LL(0x61aa5a41,0x57156102),L_(0x000001b2), LL(0x86a0825b,0xfe1a3726),LL(0xecf10fa3,0x78234ce8),LL(0xf75dbfd7,0xa854adc2),LL(0xa029127b,0xf2a5d1c2),LL(0xf93cf941,0xf178cc83),LL(0x0dad731f,0x7b737197),LL(0xdb2a90d7,0xc7585a55),LL(0x5b39f00b,0x13093ce9),L_(0x000001c2), + LL(0x3c194afc,0x4b764798),LL(0x9c6ad5a8,0xf36c498b),LL(0xd194ebf0,0x5789bf3c),LL(0x11b8897f,0x36af180a),LL(0x721c1e06,0x5c78bbd6),LL(0x926781ed,0x7eda9f86),LL(0x5fbd2cb7,0xc8e02758),LL(0x639ede19,0x9493d219),L_(0x000001e2), LL(0x65d6f9bb,0xf46f5848),LL(0xfc0b9e61,0x92b9aa7b),LL(0xbce8f803,0x108e7aff),LL(0xba188aa0,0xe4839679),LL(0x43ddb44b,0xe4d01a38),LL(0x28f6ec0b,0x47439700),LL(0x488e6c7f,0x88a54089),LL(0x764515b9,0xeeb300eb),L_(0x0000018d), + LL(0x5cfb915a,0xf6fbc903),LL(0x5711b98d,0xab2bf9c0),LL(0x4617b374,0x11ca98df),LL(0xca70393d,0x0b0a9fb9),LL(0xa92fde65,0x56f25580),LL(0x79cc0a83,0x4bbfeb8e),LL(0xcab11e98,0x7ca24068),LL(0xa9977f9a,0x66dac90c),L_(0x000000eb), LL(0x8bf78095,0xbc2c6a27),LL(0x6f0f5666,0x5aae506d),LL(0x101ee3dc,0x26f13a79),LL(0x4efcb64c,0x872b3246),LL(0x4b655b96,0x93100d45),LL(0x47392054,0xb9ed2d40),LL(0x889555dd,0x82a371d8),LL(0x35716e93,0x866327ac),L_(0x000000d6), + LL(0x6a15b574,0xc4140b76),LL(0xcd00e756,0xa87ee130),LL(0xe237ca9f,0x986e71dd),LL(0x6c64d36f,0x855fe34c),LL(0x2ec61846,0x617b88a6),LL(0x14780c69,0x747aa419),LL(0x062f9170,0xed05839d),LL(0xa3775b2f,0xceadbfb1),L_(0x000000ec), LL(0x8d8f4b46,0xe4829292),LL(0x3dae35c5,0x952eef7e),LL(0x2fcf3b38,0xa2c8e70d),LL(0x15ca91d1,0x49e6f64f),LL(0x2ab5e879,0xc51365ef),LL(0x6eb8edec,0x68141278),LL(0x3c5ae2c1,0xbd1ceb42),LL(0x8868ec18,0xcdd55150),L_(0x000001be), + LL(0x340208b1,0x2a553c67),LL(0x02c37cf5,0xdad37a02),LL(0x0d5ab144,0x0de46bcf),LL(0xf845acc6,0xdc2bcfa4),LL(0xc7adff52,0x82fc1314),LL(0x0545c51d,0xc54d801f),LL(0x2dea714e,0xcb580871),LL(0x31541a41,0xdb180fb9),L_(0x00000007), LL(0x0e58cc64,0x788f8bc0),LL(0xa9c56b21,0xa004a389),LL(0x34cf9dd4,0xf832e2bc),LL(0x5ff85d06,0x552c88da),LL(0x78c4f4e0,0x30833bd4),LL(0xada841ef,0xf4f16038),LL(0xcd227c76,0xb73c429d),LL(0x10247ed5,0xaaa17e24),L_(0x0000008e), + LL(0x03d614e2,0x3cb3e5f1),LL(0x330fa2b1,0x5ec2e694),LL(0xc7a7a85d,0x6fb92d18),LL(0x1af9e2ab,0x9cb09a6d),LL(0x32ba14f0,0xa2dc635b),LL(0x4c962558,0x0dcc43a3),LL(0x44769a2a,0x8ab8ab6a),LL(0x13517adf,0xacf0f10c),L_(0x000000f1), LL(0x270a8b98,0x435cd695),LL(0x9a1d5075,0x74944407),LL(0x3eb9b615,0x67a55fec),LL(0x4207fab7,0xbab02bd6),LL(0x3706b4f2,0x131eeda2),LL(0xdb6412dd,0x2a770e75),LL(0xc7184453,0xbd13d749),LL(0xcf85aaaa,0x4d4576f9),L_(0x000000c6), + LL(0x90643ae3,0xd0279799),LL(0xcd43ff84,0x8f8d4740),LL(0xf9be1720,0xd4c21049),LL(0x94ced526,0xbc7e6131),LL(0x885163e0,0x588d6d44),LL(0xe1a54923,0xcd604d63),LL(0x66c80ec3,0xdb390f62),LL(0x0efe7f3d,0x5ed30f94),L_(0x00000191), LL(0xacfad765,0x4c09af63),LL(0x0b94aa5e,0x1cb9cfd5),LL(0xf7f9b44b,0xb1ee2bf5),LL(0x5da9b7b0,0xea26cc32),LL(0x695fa9a4,0xc53177b1),LL(0x05d4bfeb,0x0a2128d9),LL(0xb0617759,0xd86515d3),LL(0x4edd7559,0x248b0a5d),L_(0x0000015c), + LL(0xcd229d61,0x16ed066d),LL(0x81766070,0x27d264d4),LL(0x4280ae01,0x75f18c88),LL(0x0de8cd8d,0x999331ed),LL(0x2979ede2,0x2a794c8b),LL(0x4aa1f796,0xf6be0bc2),LL(0xe7f6aee3,0xab9da18a),LL(0xaa378d1c,0x2cfef6ff),L_(0x00000009), LL(0x0425becc,0xc9672464),LL(0xaeac43a7,0x71fa40cd),LL(0x9b6e5640,0x8c8a54a9),LL(0x559c4919,0x8745a152),LL(0x158de454,0xea705cdc),LL(0x49f6974a,0x31085e82),LL(0x149d6eab,0x3b82a7d9),LL(0xc24e8654,0xe5dd1094),L_(0x0000001f), + LL(0xc6703677,0xb8dfc216),LL(0x58bdf34a,0xae298494),LL(0x009bba63,0x4d30749c),LL(0x98a3bff7,0xbe1fdc0d),LL(0xd3227485,0xd2cb3b89),LL(0xdb083e7b,0xabcac22b),LL(0x0f40c3a0,0xba4e3fef),LL(0x2ef27d74,0x49fc110e),L_(0x00000076), LL(0xf850122d,0xd1940d8b),LL(0x363cf3d8,0x91127ad7),LL(0x184425d3,0x7ca8dcdb),LL(0xdb660853,0x21ec37e6),LL(0x4beb68c4,0x5fb50be0),LL(0xd22f2025,0xd5b8a4a6),LL(0x23b1ff32,0x81d34165),LL(0x7f1e70e8,0x352493e0),L_(0x000000ca), + LL(0xd39f8fa6,0x46997de6),LL(0xfc0c43c2,0x80559c77),LL(0x74a5f61d,0xf3cd5c47),LL(0xb51aa852,0x84701e4c),LL(0x3099622c,0x5f57adc3),LL(0x1c2776e9,0x66f0da61),LL(0x0d49fb9b,0x95a49243),LL(0xce6bc32e,0xc261cb56),L_(0x0000007c), LL(0xe4c23b96,0xfd811538),LL(0xfb948d00,0x57c88bf4),LL(0xe1b0ccf5,0x9a8e5fdd),LL(0x1f936fee,0xac1c5e3b),LL(0x9560deae,0xd72e0f10),LL(0xe34e3d33,0x1c36aa10),LL(0x04676a85,0x6d51f6ad),LL(0xd48d0c93,0xb6bc0ff6),L_(0x00000075), + LL(0xba61d6f1,0x16e9a196),LL(0xae964f34,0x5c0977df),LL(0x5533b3fb,0x25bde3a2),LL(0x16bcef9f,0x645c4b91),LL(0xccba7e03,0xd5f0598c),LL(0x17ea7d85,0x0a4a08b5),LL(0x68cac5a0,0x7d57c26f),LL(0xd4f0dc66,0x2a8d8db2),L_(0x000001ee), LL(0xa5172924,0xc293806a),LL(0x5f20b34f,0xa9d43e42),LL(0x06adb487,0xf8e899ee),LL(0x2608f44d,0xd8da79ac),LL(0xb1683bc0,0x4dc36bf6),LL(0x350423e7,0x15c728c1),LL(0xfdc23809,0x5dd5da5e),LL(0xe96b3148,0xbaed65ad),L_(0x0000011a), + LL(0xaa9adab6,0x9bde5c6d),LL(0x5ef4d7f1,0x4c0f1fc9),LL(0x8ecdc6cb,0x8e47e019),LL(0xff3c3ade,0x13ede807),LL(0x08dc8e67,0x996f8947),LL(0x296b4bda,0x185a0504),LL(0x07dc7de6,0xe2a36a18),LL(0xf820aac7,0x5e5c9232),L_(0x0000004d), LL(0x32ed1a36,0x4050a3aa),LL(0xecab1a1a,0x6622355e),LL(0xc9237ec8,0x11964b64),LL(0x4010a471,0x644ca385),LL(0x6abf4831,0x34cba42f),LL(0x5d25b108,0x54dd6906),LL(0xb1ef824b,0x9199f6df),LL(0xb53e7326,0x8ab89c8e),L_(0x00000113), + LL(0x17f45f44,0x47c0d420),LL(0x299d3628,0x4003ee6e),LL(0x6f86dfae,0xd3cc2b18),LL(0x5072cb2e,0xc430e500),LL(0x294a1ff8,0x69058c45),LL(0x9eeb197c,0x30c97e9f),LL(0x859543fb,0x13563a1d),LL(0x2eed4bed,0x544e4568),L_(0x0000006c), LL(0x5f1e8dcd,0x7c4de39d),LL(0x99a97139,0x69468c31),LL(0x294f1802,0xd64eccc4),LL(0xd505983a,0x9cc6daa3),LL(0x60e0c170,0x64a5b5c1),LL(0x8763e518,0x61dc006a),LL(0xb9099af2,0xc69e9f34),LL(0x0fe38a58,0x2c16bb80),L_(0x0000001e), +}, +/* digit=1 base_pwr=2^5 */ +{ + LL(0x4f59ae8d,0x5c3b72b5),LL(0xc123c8ad,0xae6b62e3),LL(0xca4e7ba3,0xeb5d2e8f),LL(0x7633eb4d,0xaee39acb),LL(0xeb750251,0x77dcf2f5),LL(0x1c6dd3f6,0x32a70340),LL(0x9a1f1f0b,0x7ca5d0ad),LL(0x4a21b83d,0x69051d23),L_(0x000000cb), LL(0x5511dec3,0x4c8a50ab),LL(0x1b4ae54d,0xb2d5cb0b),LL(0x6fe6cc64,0xe3f5079b),LL(0x5cdba6f0,0xdc0c66eb),LL(0xbef10266,0xe32e16eb),LL(0xb3e0ef80,0x5faff80c),LL(0x347fbdec,0xaff9f041),LL(0x088b1af3,0x68ca055c),L_(0x000001fa), + LL(0xa16ae6a8,0xc671907b),LL(0xa0da9ee1,0x014d027e),LL(0x0efad55c,0x0b01f380),LL(0xb9d3e016,0xe2f7ed1b),LL(0x938df5e9,0xe67c4396),LL(0xd65b0a5c,0xd40c305d),LL(0x533f5edd,0x01c97f61),LL(0x68a79ebe,0xe7088976),L_(0x000000be), LL(0xad9d235c,0x2c52fc6f),LL(0xea2d78b8,0xedd0329e),LL(0x21e0e0d2,0xf1d7cac0),LL(0x887a53dc,0xd0a846a5),LL(0xc5d60b2d,0x1b4f43d5),LL(0x0e8631dd,0x4597d8ee),LL(0x36cdd8e6,0xbe2bdcac),LL(0xb9d50810,0xc2474902),L_(0x00000075), + LL(0x571ccaa7,0xcc45066e),LL(0x3f9b1980,0x05f6f02b),LL(0x6757639e,0xe84f381f),LL(0x4e03775c,0x5ef348d1),LL(0x4770c353,0x3845ff7f),LL(0x5fb50c57,0xedbd5036),LL(0x5b16a317,0xe0ae9613),LL(0xbda8a1ca,0xab626b1c),L_(0x000000f2), LL(0x2efb5af8,0xce499e42),LL(0xdea5dcca,0xf85f9a34),LL(0x8fc76f8d,0x07899ffe),LL(0x0f62f621,0xbdb94b70),LL(0x648a20af,0xfe99ecf5),LL(0x436c353f,0xd5421253),LL(0x3fcac929,0xe3d53ffd),LL(0xa9413337,0xdbd78092),L_(0x00000158), + LL(0xf05f3504,0x72b9b7d4),LL(0xdabb5813,0x90711a09),LL(0xa79713f3,0x40b52fc0),LL(0xbea8efb8,0xb43b138e),LL(0xcba724a4,0x7bae703e),LL(0x96698925,0x5de16b6e),LL(0xd41e4b4c,0x80a9811f),LL(0xc1b18e03,0xf93f1c0f),L_(0x0000012c), LL(0x989bdb97,0x0964efd8),LL(0x725e184a,0x049a3954),LL(0x09871a1a,0x92325673),LL(0x462734e1,0x586e4cd6),LL(0x54d24ffa,0xe1d8d7ce),LL(0xb30da7d5,0x69f3efbc),LL(0x416e700f,0xef7de3d3),LL(0xe729987f,0x8354141e),L_(0x000000fa), + LL(0x08a54bd7,0x32ffe09a),LL(0x456c0723,0xd6fc852f),LL(0x40d62750,0x2ffa7f72),LL(0x4aeb61e4,0xe7b77ac3),LL(0x502b124f,0xfe47dfb6),LL(0x4dd6a90a,0x06cd1ac4),LL(0xa6600862,0x47e8bdf2),LL(0x9f7f4801,0x3d29e3f1),L_(0x0000009f), LL(0x0fde861b,0xc7487c32),LL(0x94ff60ca,0x23970cbd),LL(0x72b3e644,0x6984aedb),LL(0xcc2f8476,0x43e3b1aa),LL(0x4e288fca,0xd6b84507),LL(0x5c070a30,0xcde70c2f),LL(0x806889c8,0x9397e29b),LL(0x4c71559f,0x35a4b1e0),L_(0x0000000b), + LL(0x85707b44,0xfaa21460),LL(0xdf0df8fc,0x6496b635),LL(0xa66cdf1c,0x72a3871c),LL(0x9e220e51,0x55171f57),LL(0x76519fbf,0x2bc7ff1e),LL(0x1fe67c09,0x4f8bd386),LL(0x55ed0240,0xc5765e29),LL(0x1c77281d,0x27bc3df5),L_(0x00000160), LL(0x9e78f5c3,0x38d57586),LL(0x6a14b857,0xa24ce77d),LL(0x956a40cd,0x6eeb21f0),LL(0x384b0097,0x30d4fd92),LL(0x3f99bf29,0xce9aade0),LL(0x0b162be5,0xc168443c),LL(0x056730f0,0x8b3af3cd),LL(0x86e7a481,0x043e9d2a),L_(0x0000010a), + LL(0xd9b7e5ae,0xd2292194),LL(0xc11a98a6,0x0b648125),LL(0x59e37b44,0xaf635b08),LL(0x25aea6af,0x19039f0b),LL(0xd7528475,0xd304853b),LL(0x17b80f08,0x08f86bd4),LL(0x16cad388,0x4ba43b52),LL(0xd0f1e285,0x084ad45e),L_(0x000000b9), LL(0x7719f6ae,0x0e419d0c),LL(0x808c65ad,0x5ed42353),LL(0x2e40948b,0xc831c79b),LL(0x95dfdfbe,0x1f8615c2),LL(0x19810fc6,0xd5083188),LL(0xc73c4dd3,0x9df9fc10),LL(0xb9ee4c0f,0xb094cd65),LL(0x75870f78,0x4c55caa4),L_(0x0000009c), + LL(0x56757d02,0xc834df00),LL(0xc0d9c745,0xe5caf285),LL(0x91f23599,0x620faea3),LL(0x2d4e48a9,0xb7461523),LL(0x99bdc7a7,0xf47934e5),LL(0xd4dc2fd4,0x4f65ada3),LL(0x4c81e39c,0x3c079897),LL(0x64a2c57e,0x815ad610),L_(0x00000107), LL(0x3fa38d40,0x22a435ef),LL(0xc247d609,0xa826a53b),LL(0x3d8a1866,0xcede94d3),LL(0x75ac695c,0x2b9c71de),LL(0x8bcb2e7b,0xd52b9aa7),LL(0x1a8316e1,0x40f2da2b),LL(0xe2a07695,0x49881db4),LL(0x7e4c0ddd,0x39a09e21),L_(0x000001d5), + LL(0x7118ea7a,0xe663ce23),LL(0xd6a550c8,0x67612dfd),LL(0x45b8de7d,0xd0a752b6),LL(0x1b9bd789,0x60ad3301),LL(0x023b6c29,0xcc26ecce),LL(0xa078b41a,0x61239a1a),LL(0xee942cd7,0x6922505f),LL(0x5e08263e,0xe76fdb87),L_(0x00000029), LL(0x1108fef0,0xe9adb593),LL(0xe4610492,0x509096b8),LL(0x26279733,0x4c917c92),LL(0xc7a80802,0x7516cc5c),LL(0x8edbea9c,0x131d3769),LL(0x1db92a9d,0xe32f86b9),LL(0xc3bfb615,0x16237fcf),LL(0xdaad00e7,0xe0767305),L_(0x000001c0), + LL(0x726d1da3,0xd0f5eba7),LL(0x5afe82c9,0x00a2bafd),LL(0xa7f8f99a,0x8c282afe),LL(0x5344cf5a,0xa7ab3e18),LL(0xb4f699ab,0x2626fca8),LL(0x345363ae,0xc44f5f11),LL(0x9cac1c3a,0xa135f6b3),LL(0x2cc9c6d3,0x3b77c483),L_(0x000001e8), LL(0x90784ad0,0x4491c85e),LL(0xc87a8f35,0x23793bcb),LL(0x9606baed,0xcd6ee91d),LL(0xaa42a14a,0x54d429b3),LL(0x40a29e37,0x89ff244a),LL(0xd4a2c066,0x0bb505cb),LL(0xdc545060,0xfc93a903),LL(0xad7e26a4,0xc3b67c6e),L_(0x000001fe), + LL(0x8ceb07a2,0x96bcc3d3),LL(0x99d9281c,0x3db83972),LL(0x3ff2e9a3,0x16268498),LL(0x00d03fc3,0xf0d72767),LL(0x974db3bc,0x52e2c15d),LL(0xcfc51b17,0xe4156324),LL(0x10aa8cfe,0x989f0141),LL(0x8e68c302,0x9e3bb348),L_(0x00000058), LL(0xb1ff4858,0x798b01e9),LL(0xb7bcaeeb,0xb107b933),LL(0x0bdcd04d,0x5499a0b1),LL(0x26fd1d2e,0xacddcbd8),LL(0x56837ddc,0xa9081a22),LL(0x3bdf1491,0x05c3276e),LL(0xc07890c9,0x91891ac9),LL(0xa184d413,0x925157ab),L_(0x00000171), + LL(0x8bff7233,0x91b8350b),LL(0xf62b4383,0xb265f67b),LL(0xc46f7226,0x21d7036a),LL(0xef90907e,0x8034aa28),LL(0xdabc0434,0xd005b709),LL(0xb12cb388,0x06bb608b),LL(0xe65c7159,0xeb7b8a18),LL(0x11e0f987,0x7aab899b),L_(0x000001f6), LL(0x8d53586a,0x0ab2f2c0),LL(0xe3db9e6a,0x80aa8220),LL(0xb7b44599,0x2bea87eb),LL(0xa54e5ad3,0x6c5ac479),LL(0x93b927af,0x83fb3fac),LL(0x62a4775c,0x9c4bd501),LL(0x657b8d9a,0x88136dc2),LL(0x31811cf2,0x2b887e73),L_(0x00000185), + LL(0x4d4e2e15,0xa3ad1cf1),LL(0xf792f378,0x7aef7449),LL(0x4d833ce0,0x5394ba78),LL(0x06fcfedb,0xf33fd365),LL(0x76965949,0x9c4ccb42),LL(0x4e9fbd73,0x61aaa0a9),LL(0x9fa1995c,0x3ba114e8),LL(0x462ee846,0x7540468d),L_(0x000000cf), LL(0x0442839b,0x46e8546b),LL(0x4cfa345f,0x0d411f16),LL(0xc1e9119e,0xf8d99149),LL(0x0deb6f34,0xb98975e2),LL(0x6508c235,0x6e32684a),LL(0x741c5884,0x99583d46),LL(0xacaecb2f,0xd61998e0),LL(0xdc28ccee,0x1ef321fc),L_(0x000001db), + LL(0x22a3dc2c,0xbc8dbffe),LL(0xf713e616,0xbe6a57a2),LL(0xbe89cc5f,0x5dfb0ead),LL(0xb5bd5287,0x5dba909b),LL(0xff87fb08,0x124b1f29),LL(0xd39afe41,0x8ad8951f),LL(0x0e13a626,0x2f09f744),LL(0x4826695e,0x44419020),L_(0x00000036), LL(0x89f11d49,0xd8b689b1),LL(0x42cf8f40,0x8f4bb929),LL(0x1093f58a,0x41b6334a),LL(0x5f1b0229,0xcbfc9d3f),LL(0xfa09f9c8,0x4f838812),LL(0x4ae0b40b,0x114194e2),LL(0x6d9844d6,0x69722fe6),LL(0x15e4c6d7,0xb8f4264f),L_(0x000001cf), + LL(0x2f86d0f6,0xf27fea27),LL(0xb102e317,0xf76070d1),LL(0xb05afc5b,0x1c9d3a3b),LL(0x5dd0f5d9,0x00e4d9fc),LL(0xee4d6689,0x65f0f1c6),LL(0x2a86ba85,0xde562216),LL(0x3e6bfc0d,0xdbfc35a2),LL(0x9af0f242,0x5ad61eda),L_(0x00000142), LL(0x941bae5d,0x2b9df6f5),LL(0x5be1e379,0x818b63c5),LL(0x35a1da29,0x7c374ecf),LL(0x81936096,0x91cdc4c0),LL(0x32597a76,0x72e4e5df),LL(0x3e8a2fa3,0x5b7351e8),LL(0x916e7f8d,0x19372aca),LL(0xabd62e9d,0xe3d4016a),L_(0x0000015c), + LL(0xc98396a0,0x1d4dea3f),LL(0x2852471e,0xf1b1b604),LL(0x9e270a42,0xbff87527),LL(0xe46c1327,0xfe022231),LL(0xfc05c823,0xe4c1b07e),LL(0xa4581988,0x46e86dbf),LL(0xc3803e03,0xf3ea14d7),LL(0x8c2f4163,0x3536b269),L_(0x00000094), LL(0x474df73f,0x3f77cba0),LL(0x82f0ebae,0x9fac52f4),LL(0xeabe2a5c,0x4d046303),LL(0x5a86c777,0xd8716f60),LL(0x16157561,0x76cfe4cf),LL(0x564b6dae,0xf10528e0),LL(0x9113bb26,0x878d8ad6),LL(0x933ccc8b,0xc9676c2f),L_(0x00000038), +}, +/* digit=2 base_pwr=2^10 */ +{ + LL(0x7c6312ff,0xb394c36d),LL(0xd8c526b5,0xae9f8123),LL(0x6b7fb3e1,0x7287a461),LL(0x2d9f22f9,0xd21b31a9),LL(0x895d4a0f,0xd7cbfded),LL(0x81ff2d23,0x5c105748),LL(0xe830bd0b,0x4fe2bd04),LL(0x9dfeb777,0xdf56afd6),L_(0x000001fe), LL(0x83b243fc,0x7e7441cd),LL(0xa23e166b,0x5c91b009),LL(0x85f70865,0x122f85c7),LL(0x22e7768c,0x6db40321),LL(0x2fb75185,0xd6df94b8),LL(0x80b31836,0x98df3edc),LL(0xeea7ce80,0x05298e9a),LL(0x048ecb96,0x1e0476e6),L_(0x000001ed), + LL(0x8ec6fc14,0x292021e4),LL(0xa9680402,0x9500ecd0),LL(0xed719b16,0x41202339),LL(0xb81e8a19,0xb85440eb),LL(0xd40e8e4d,0x3f6a53c2),LL(0x84a12a31,0x2796c5c6),LL(0x497c0088,0x91636765),LL(0x751837b7,0x8b09820c),L_(0x0000014e), LL(0xd4740897,0xfeb6c7cf),LL(0x05fd0f39,0x66755043),LL(0x24da0165,0x915708d7),LL(0xcde5846c,0xc7bb1c3f),LL(0x0cbcc847,0x5d5c58a4),LL(0xd0093587,0x531dd999),LL(0x178ab52f,0x88ff3f98),LL(0x4485d318,0x35266a7c),L_(0x00000171), + LL(0x7f523b68,0x91acdac6),LL(0xe5501216,0x656f99d2),LL(0x9d6ec374,0xe158465f),LL(0xf67a8845,0x15ed0b99),LL(0x0ea75aec,0x01226fd6),LL(0xc000f5ba,0x0a951866),LL(0x2eb378e5,0x185feb1f),LL(0x746f4b9e,0xd7b2048c),L_(0x00000097), LL(0xae887bf0,0xc9deb828),LL(0x2d928546,0x4d8afcb8),LL(0x7759681c,0x47a77426),LL(0x1f2422bc,0x9941fb7f),LL(0xc9c44935,0x3b4f41a6),LL(0x50ea43ef,0x708dbefd),LL(0x5c9f2544,0xcef3425f),LL(0x8d085b3a,0x17fe443d),L_(0x00000114), + LL(0x4dbc092f,0xa2f27fd6),LL(0xfff03850,0x2ad51407),LL(0x2ffc14aa,0xc4b80840),LL(0xbe516b67,0x4499107f),LL(0x0f027098,0x715688b4),LL(0x5e2c9af3,0xbddce779),LL(0x26ec8f7d,0xcc8a5dc6),LL(0xcc9e1305,0x6a9d132a),L_(0x000001c4), LL(0xcd14a595,0xce664e13),LL(0x678ff921,0xb7485d5a),LL(0xed6fe685,0xdd61d65f),LL(0x2b7d0453,0xa066d915),LL(0x81e48dc7,0x0c3395f0),LL(0xc1cb1256,0x6053e587),LL(0x630f2cdd,0xc776afca),LL(0xf0d70553,0x61da1814),L_(0x000000dc), + LL(0x71ac09f5,0xd012b2e5),LL(0x9190907b,0xc03bb972),LL(0xab45bb80,0x8ed0d272),LL(0x3b41e8eb,0xaa3449d8),LL(0xd2d64ef1,0x4e6b21d4),LL(0x9f7e0342,0x9eb72363),LL(0xb6336622,0x69f35a65),LL(0x9114adb9,0x24fc697e),L_(0x000001f5), LL(0x18b88dd7,0x489c82e7),LL(0x1d050011,0x5e1bdb72),LL(0x80ac7d35,0x3a785f6c),LL(0x6bb1ceb8,0x4d0595c0),LL(0x47ba8e65,0xf29ab5dc),LL(0xfba4c7c5,0x768427d3),LL(0xf250f0c9,0x38fed5ff),LL(0x60390918,0x655b698f),L_(0x000000eb), + LL(0xd8129c74,0x33a20918),LL(0x56ec57a8,0x44da27b8),LL(0xfe03052c,0x5c69a6e2),LL(0xb8645b34,0x61e0489c),LL(0xedf7eb89,0x0d9cee51),LL(0xb459ccf4,0x4bbdc11a),LL(0x2e3c7f1a,0x22591a2d),LL(0xab74c4c7,0xa982d5f8),L_(0x00000083), LL(0xffdc8f5b,0x9755e96d),LL(0x00d903b0,0x73fc3336),LL(0xa3ed2567,0xa44f5c0a),LL(0x78da9c2e,0x130585a8),LL(0x5d2a5778,0xf488bddc),LL(0x203a9db6,0x0d642fb8),LL(0x49bb8671,0x86aadd4d),LL(0xc216425a,0x20f6b6a8),L_(0x0000012f), + LL(0x106b0907,0x6c59b6a4),LL(0x77bee1fa,0x082792c1),LL(0x39609b3e,0x4e300675),LL(0x9586b280,0x41820c34),LL(0xf4b318a9,0x568da4bf),LL(0x504b9f0d,0x18b54e1d),LL(0x7cd449b1,0xea63bc73),LL(0x35d4426b,0x9a0a15d7),L_(0x0000001e), LL(0x0b6fffaa,0x5b9ffa5e),LL(0xc584b1b6,0xb1ee386e),LL(0x00bfc921,0x2e48b6f0),LL(0xc1a25580,0x90b9e7af),LL(0x232ccaec,0x60d7386e),LL(0xbcde0a94,0x27832dfe),LL(0x20ca19ad,0xa34dad1d),LL(0x2a628682,0xad3601ca),L_(0x00000146), + LL(0x61a19c36,0x5540d3da),LL(0xde90b954,0xfed5fc9d),LL(0x08cbe546,0x6579be89),LL(0x931292ec,0x31c8bf2b),LL(0xde0b2215,0x64709233),LL(0xf0e33dcf,0xa91e2913),LL(0x99299206,0x933880d8),LL(0xab37b024,0x63ef0d07),L_(0x0000010d), LL(0x6eb1d587,0x5f29f3ee),LL(0x6f46862b,0x13755e24),LL(0xe2652ae3,0x952c2e51),LL(0xba6a65e2,0x013b9446),LL(0x3fd1b792,0x5e7bffb4),LL(0x96a14917,0x66af7dd8),LL(0x68a41011,0x553d0d5f),LL(0x4ff29cf9,0x381be3a3),L_(0x00000010), + LL(0x1d7e25bf,0x707d2643),LL(0xb62058b6,0x3eddf1f7),LL(0xcf147bf5,0x09f87dab),LL(0x11a1e31b,0x9b643ba2),LL(0x4287faad,0x31ecf4ec),LL(0xfdf5220a,0xa4f09336),LL(0x8916b869,0xd2c73095),LL(0xe07b7112,0xe676b1a5),L_(0x00000038), LL(0xea00c98e,0x31798ea9),LL(0xee9f1bb9,0xa0db3168),LL(0x33aa5ab3,0x5107a1fa),LL(0xbb110cf5,0xccdd22ec),LL(0xedd17aae,0x8bb0cd07),LL(0x610d689f,0xcf178778),LL(0xcca4e56f,0x95d696e3),LL(0xaef30431,0x5f284888),L_(0x0000008d), + LL(0x7352fa9e,0x4590e4bc),LL(0x951e01ab,0x42b51fb7),LL(0x3643ff6f,0x1a3be50c),LL(0xdad9a3a4,0x5c6479b6),LL(0xb0a91741,0x5f9d3ca9),LL(0x841c9d52,0xbed2f70d),LL(0xdc8331dc,0x3fce8436),LL(0x0a312707,0xf8c4ccb2),L_(0x000001e6), LL(0x4d9d7ef9,0xaec9953a),LL(0x62242fd1,0x04665dd7),LL(0x49b9eb5e,0x7d7f1a35),LL(0x6a03ee74,0xcabc639f),LL(0x22cc5c02,0xf26d2603),LL(0xbb312bf5,0x05ee7955),LL(0x10cf1634,0x00c226f0),LL(0x3baa95d6,0x49ecb4bc),L_(0x00000144), + LL(0x968950c6,0xa826a58f),LL(0xe858945d,0x3a7fc7e7),LL(0xd63d6714,0x63d3c677),LL(0xc319d1ba,0x349e7bde),LL(0xb4155a1d,0x03a4c66e),LL(0x3ddc0044,0x77aa278e),LL(0xccce8941,0xd867d113),LL(0x4e46021e,0x06424305),L_(0x000000ad), LL(0x6dd54385,0xd8308ab7),LL(0x2e1458d6,0x0d0a4aaf),LL(0x924e3bd4,0x309fb2fb),LL(0x2f7cd47f,0x5161e4da),LL(0xbc75672b,0x27fa09f8),LL(0x0e420bf8,0x6bf78336),LL(0x83d1b09e,0x3c3d3117),LL(0x89323d7e,0x585a4d97),L_(0x00000059), + LL(0xe8b9e5f2,0x9e4c557e),LL(0xa63316be,0xc510883d),LL(0xfba63955,0x58616eed),LL(0x5eba66cb,0x1f901bb5),LL(0x7d93dd07,0xe4c33f46),LL(0xd7520d11,0x9c2288bd),LL(0x3c9b7282,0xa3f22d4f),LL(0xf979cce9,0xbc4f2416),L_(0x000001cb), LL(0x69f91fa6,0x1780ab39),LL(0x9f2b3904,0x1e17f9e9),LL(0x0408a22e,0xf102825a),LL(0xe814b39a,0x4077db13),LL(0x717c70c1,0x116e8d04),LL(0x1642fd91,0x5157bba1),LL(0x072760c2,0x223d53fd),LL(0xf596860d,0x68119130),L_(0x000000d6), + LL(0xd03914a2,0x1f8fa1cb),LL(0xbc0f726e,0xc55472bc),LL(0x9dcf7393,0xcc596835),LL(0x86ab65ea,0x0c9b7622),LL(0x90362f16,0x8c0ca08c),LL(0xe8de2a3c,0xec48a671),LL(0xbde41568,0x0286ac32),LL(0xd27da64f,0xdc224c38),L_(0x00000016), LL(0xf6c82cd6,0x53a87083),LL(0x3e753ee6,0xab548bed),LL(0x07afab6b,0xc34ddb60),LL(0xc0dc2ddc,0x378f8e85),LL(0x399c4261,0x5087e698),LL(0x6f7e49f2,0x07f39938),LL(0x6345ae4d,0xc730c9c6),LL(0xb6c2765f,0xfb522df3),L_(0x0000014d), + LL(0xe4292c6d,0xf4cf1a41),LL(0xfb9a8b65,0xf774c321),LL(0x5046d341,0x7b28d6b2),LL(0xfe598075,0xb06becbe),LL(0xc3187f95,0xd220a206),LL(0xc278703d,0x54ba06d2),LL(0xb514e8c6,0xda1d824d),LL(0xc959300e,0xc32680df),L_(0x000000c4), LL(0x7fbd13f4,0xec877f9e),LL(0xd0e494f0,0x209c6b0a),LL(0x529b0f0c,0xc6b1073b),LL(0x50fb2f00,0xd17f2e67),LL(0x80cd82a4,0x62378ddb),LL(0x9f57c57d,0x0162b312),LL(0xc234e4cb,0x8483d5e6),LL(0x501d8ec9,0x5438453f),L_(0x00000176), + LL(0x0d037502,0xd53d7a18),LL(0x424ed14d,0xd13f986d),LL(0x29de6753,0x3e4dbff0),LL(0x6d33dc1f,0xf6b77dc2),LL(0x87ad5722,0xbf6050c2),LL(0xaea8f254,0x83742064),LL(0xb17406b4,0x7d90e061),LL(0x13b29245,0x6d00e002),L_(0x00000097), LL(0x7f20e8bb,0x5bf160f5),LL(0x8562b323,0x1d2d2e90),LL(0x4b31d400,0xea7b242c),LL(0x4a1acb5c,0x229d7510),LL(0xc93f9b92,0x3eba408b),LL(0xb068a0e7,0xb0525ab0),LL(0xb376d6b0,0xd96dff43),LL(0xf1b03f82,0x78a56db4),L_(0x00000168), + LL(0x77ddddf5,0x631d2365),LL(0xc4b6db39,0x5fc5e812),LL(0xd1cccab0,0xc38ec807),LL(0x8729f1a1,0x1629e92c),LL(0xc999e406,0x6b4c00d1),LL(0x781d88f5,0x3cac8f29),LL(0xcce3380c,0x16b02141),LL(0xc7e0e0cc,0xcb8c9920),L_(0x000001a3), LL(0x76234580,0xd02da7d0),LL(0xe2d27b0a,0xcc82cf5a),LL(0x3adad7f2,0x2c08a15c),LL(0x7009305d,0x55fa7b4d),LL(0xde9e632a,0x0b55b693),LL(0x2a821156,0xb565732e),LL(0x3788cf98,0x89f0adb6),LL(0x2d1f6054,0x0705738e),L_(0x000001b1), +}, +/* digit=3 base_pwr=2^15 */ +{ + LL(0xb0c6a7b6,0x23a29c73),LL(0x9c3eafc7,0x392643c3),LL(0xf81be3c4,0x88c0b213),LL(0xec734fa3,0x33b98ae3),LL(0x9b26d37a,0x23074268),LL(0x687a332e,0x28354ec1),LL(0x6935b64e,0xf60d4b7e),LL(0x9d55aecf,0x437bcba7),L_(0x000000b6), LL(0xd6073362,0x8bcd336b),LL(0x5b5f67fb,0xb6c7a784),LL(0x5633e845,0xdf601730),LL(0xa907be72,0x2814a576),LL(0xfe65734d,0xc7084b86),LL(0x0758f113,0xd7bad9f2),LL(0x5030c22c,0x3ef6af2a),LL(0x7ff1cabc,0x15f43164),L_(0x00000122), + LL(0x6184cce9,0xac65525f),LL(0x5051a406,0xc9acc4f2),LL(0x651c4a44,0xb637bdd2),LL(0x571fa6bd,0x2ae9ce59),LL(0x4cf1489d,0xf56bdf32),LL(0x61b0a821,0xe5fa827f),LL(0x9dcea620,0x4b46a244),LL(0x7027c9ed,0x6e4d3c94),L_(0x000001d6), LL(0x0495f1c5,0x27a410cd),LL(0xee6432c2,0xbc9ba135),LL(0x73536858,0x53142570),LL(0x7e39c350,0xd0616e0b),LL(0x316eeb65,0xa694a069),LL(0x55bbe949,0x9aba0dc4),LL(0x1f9d7b76,0x32d36d72),LL(0x1dcb7a1d,0x9a8a0a04),L_(0x0000001a), + LL(0x0fce6d79,0x49c3fb55),LL(0x3a2f9da5,0x3a8e9a7e),LL(0x44e158ff,0xd771a67e),LL(0x7de21bd3,0xa6180b0e),LL(0x5cf6b900,0x349f9cad),LL(0x53ff2b3f,0x783786f1),LL(0xe350b1ce,0xec23cb86),LL(0x58690faa,0xbca103ef),L_(0x0000009a), LL(0xe8902691,0x4d7ea0cd),LL(0xdfaca68b,0x13648702),LL(0x595a974f,0x5bd316f2),LL(0xbf226a22,0xbb11b239),LL(0xeaee978b,0x2ab1e433),LL(0xc7607b51,0x870c9a0f),LL(0x43795a95,0xe00a29c5),LL(0x53d7cad7,0xd68ee860),L_(0x00000013), + LL(0x9b30d330,0x8312448f),LL(0x499ca6a8,0x27c12fd1),LL(0xaf5a132e,0xc3fb765e),LL(0x01b2d2a5,0x07951a8d),LL(0xce3517c8,0x97c68ed6),LL(0xe67d936a,0x8cdd161c),LL(0xad5eb28f,0x795d9876),LL(0x6496ac4a,0x2bca3997),L_(0x000001f8), LL(0x7fd91252,0x6e4dff62),LL(0xe44601e6,0xa96a9194),LL(0x84a673b1,0xf81ccae8),LL(0x06054966,0x2eba8c5d),LL(0x53226945,0x77e70b53),LL(0x17deba76,0x98891e5c),LL(0x2fe55a92,0xccf9a70e),LL(0x8b39032d,0xcf81d4d4),L_(0x0000009b), + LL(0x2c87d9f4,0xce45ab56),LL(0x0de1dc21,0xa16d3789),LL(0x72ace7c2,0xe08192c8),LL(0xe7012d3c,0x4840d465),LL(0x2d9fcc09,0xd2d9e7c8),LL(0xb83abe6f,0x4dc89aa4),LL(0x57f505dd,0x58ef6f90),LL(0xc12ca416,0x3b0f2ce4),L_(0x00000096), LL(0x9e8dd733,0x4f047388),LL(0x1231cdd3,0x536cd1c8),LL(0x45523810,0xd1e5a85f),LL(0x4bcff7cb,0x3fceb99e),LL(0x86ad3d2f,0x00ae1467),LL(0xddf93ca7,0xab6574df),LL(0x4160edd9,0x611238b6),LL(0x0bbbbc9e,0xc6a872eb),L_(0x00000014), + LL(0x4b9dc9a7,0x4e86b3f7),LL(0xd56a4df5,0xb7fc672c),LL(0xc91daa4c,0x047ac313),LL(0xd8b04fac,0x71df8b53),LL(0xd047ffb7,0x48cf7c44),LL(0xe196a8ad,0xbf663542),LL(0xea4fed68,0x45aa68b0),LL(0xdbd49e0b,0x9fea1483),L_(0x0000005e), LL(0xd77d603e,0x33664de2),LL(0x5ef7dee2,0x994f9685),LL(0xc8ab10b1,0x5e3c5bf8),LL(0xf5ab3d23,0xff2ae5c2),LL(0xdbff37af,0x9d0fd0f4),LL(0x50db50de,0xa6d91d52),LL(0xe2c950fc,0xa742da0b),LL(0x0ec3836f,0x3cb961c7),L_(0x00000071), + LL(0xea3797f2,0x00f8c37d),LL(0x0b3e1166,0xce0936fa),LL(0xb532c55c,0x204a444f),LL(0xeef2ac73,0xa6b09c79),LL(0x31515d9e,0xac9e3e09),LL(0xdd05ab36,0xe9cef435),LL(0x319eb710,0xfa2d9fd3),LL(0x1d7ac545,0x6af2d4bc),L_(0x00000199), LL(0x595b4001,0x2f76c04a),LL(0x0f70018c,0x74e5849f),LL(0xa9c62272,0xb2abd908),LL(0xaecd915f,0x5ffbaabb),LL(0x9fa73bfe,0x111c8c5f),LL(0x35b0554e,0x77c9c2a7),LL(0xcc8177e6,0xe83b44a5),LL(0x3bc6ae04,0x18cd6dec),L_(0x000000b1), + LL(0x229b222a,0xb1bb114a),LL(0xff59f733,0x887f6c13),LL(0x2679cded,0xbbad5dfb),LL(0xd35dec8b,0xea94d41f),LL(0x90930770,0xd4f0a601),LL(0x2ad07da8,0x2142901c),LL(0x48f142ed,0x692aaa86),LL(0x252e4559,0x82af1f42),L_(0x000000a7), LL(0x9b4f335e,0x78c42f0d),LL(0xc2716105,0xfda89975),LL(0x2c49b195,0x35776137),LL(0x3ac76051,0x4de0d058),LL(0xfcd0c4d5,0x47ffa549),LL(0xe11bc35f,0x31f21817),LL(0x3f57a567,0x46ac2b10),LL(0xcde0cd71,0xa72a1284),L_(0x0000008e), + LL(0xaecaf4a5,0xcd468ef8),LL(0x60b977fb,0xbcb8a438),LL(0x3938f4bc,0xcfcf5c2b),LL(0x2c7337c9,0x7bb844f3),LL(0x23c47750,0xdea5e248),LL(0xf126971b,0x47ee8dea),LL(0x6f1d664c,0xd5392932),LL(0x3efa21b6,0x9886378b),L_(0x00000075), LL(0x5940abfb,0x28ef7f36),LL(0x3e9bee76,0x8f415722),LL(0x360759cd,0x11a30e1c),LL(0x3c8733e8,0x78196a73),LL(0xc43394c7,0xf3a60c7e),LL(0xac3864e9,0x776e1d00),LL(0x0c19158c,0x2e4681b7),LL(0x517321cc,0x2a4a5040),L_(0x0000007c), + LL(0x69bb2a3b,0x59d22ba1),LL(0x18bc1523,0xee4d727c),LL(0xbabfd9ca,0x4c8338aa),LL(0xe3550512,0xa9cc3cca),LL(0xe599b6e8,0x15386807),LL(0xc5ab3c64,0x3919da2f),LL(0xd2ee43d4,0x801a4c6f),LL(0x38ead934,0x40a087be),L_(0x00000197), LL(0x8b8c66b5,0x7834d44e),LL(0x74807217,0x690ef307),LL(0x926feb1c,0x54c7151d),LL(0xbe2f1f34,0x456bd03f),LL(0xc48ce8e6,0x04a6964d),LL(0xafec270c,0xe8febbc7),LL(0x483b3a5f,0xd30f159a),LL(0x96cb139a,0x52fa9aca),L_(0x000000c9), + LL(0x0e87f867,0xf99ad667),LL(0x75faf57c,0x011dcb9c),LL(0x6c05cb53,0x4f1f75a2),LL(0x3556cade,0x2dea9ad0),LL(0x3f87760d,0xb590f7b4),LL(0xe73b9512,0xc497a74f),LL(0x5a5a684b,0x8d18f07d),LL(0x8e2fa89c,0xbe126a50),L_(0x000000ef), LL(0xc3adce30,0x12408706),LL(0x4d73c59c,0xcce1c5bc),LL(0x2ddcd22c,0x381eb1ab),LL(0x0b77c42b,0x43827dd9),LL(0xaee2e20f,0x0ecadad8),LL(0x4d7ed6ba,0x141b0bef),LL(0x69fa3aa0,0x9ae275eb),LL(0x3d138706,0x2d98b314),L_(0x0000016a), + LL(0x7fc0e976,0x65d7b1d3),LL(0xc8c06baa,0x608a4b87),LL(0x2e527b8c,0xa2d8c259),LL(0xcc19bb3a,0xb09308aa),LL(0x4ce5b0ad,0x2458761d),LL(0x7a6ee0f4,0xd73d4f70),LL(0xd791c442,0x0d3867f8),LL(0x3ba7a1a6,0x14027c94),L_(0x0000007b), LL(0x0e7ffca3,0x467af3d9),LL(0x60c44d23,0x9427b9fa),LL(0xe4a16358,0xaff54ce0),LL(0x55e4129a,0x275c2816),LL(0xcbefd5ea,0x7c03c7fc),LL(0xb7160ce2,0xc97ca421),LL(0x84bb35f0,0xea69ee6f),LL(0x35e0436e,0x360ec7ec),L_(0x000001ca), + LL(0xf585af17,0xbac45c7f),LL(0xf7251745,0xd6aa93a1),LL(0x8a56414c,0x8fa35248),LL(0xf6e64410,0x1720b12e),LL(0x81f59ca8,0x6cb0f80a),LL(0x232a9916,0x205cfe62),LL(0x872efe0b,0xdcba9616),LL(0xa3d26e5c,0x2c5a0421),L_(0x000001fc), LL(0x06a36051,0x478ec567),LL(0x7d42157a,0xa110b6a7),LL(0x0c863ff6,0xb1e77441),LL(0xa6979407,0x7c13c78a),LL(0x6a0ad3b6,0x08c47fd0),LL(0x34e0edd0,0xcd2ed5cb),LL(0x8df0c73d,0x41a8e1a2),LL(0x73883967,0x85e0312e),L_(0x00000135), + LL(0x7d33b8c9,0xa6572311),LL(0x3fceee3a,0x1482e2ca),LL(0x52560262,0x6d96dfdb),LL(0xa105a9eb,0xbdc41e36),LL(0x8c0fd8b7,0xa2f2edd5),LL(0xb271c58b,0x050043d8),LL(0x4a51907c,0xa79966a3),LL(0x0fa52e13,0x60842aee),L_(0x00000128), LL(0xdac2d706,0x62accbe2),LL(0x0b78e0d4,0x8397028d),LL(0x2c9d107f,0x711b525e),LL(0xfedd5666,0x0c96203d),LL(0x88395725,0x2be09463),LL(0xf9856d0f,0x6dd96c8f),LL(0x9c7a6702,0x4398fe82),LL(0xfc430b6d,0xbf922dac),L_(0x000000da), + LL(0xaa02764f,0x7d06225f),LL(0x36596aaf,0x23dab345),LL(0x0047b230,0x1f940005),LL(0x1c2f1ccf,0xb4fb0f0c),LL(0x82a82a8c,0x589309ef),LL(0xc66190cb,0x19fbd0a3),LL(0x839f41c1,0x0fe2846b),LL(0xcc1c9536,0xeb188d9c),L_(0x00000082), LL(0x729f81c7,0x27782d0b),LL(0x55359881,0x76e1016b),LL(0xcaad48a7,0x26d82543),LL(0xc89767f1,0xcf1f4470),LL(0xd4acb529,0xe5b4bfed),LL(0x7b75fd29,0xae8ee068),LL(0xc3d34db9,0x3b3ffbcb),LL(0x9c535467,0xf84d77d7),L_(0x00000122), + LL(0x9faba8ba,0x5f421abd),LL(0xe82276fc,0x94ac402c),LL(0x91f2efc8,0x7d55bead),LL(0x8241f32e,0xcc1090d2),LL(0xe8bce170,0x19f59df3),LL(0xe27350cb,0x4ac35c2d),LL(0x3e6cfc43,0xd13cf90c),LL(0x84bc2847,0xe00912a7),L_(0x000001c7), LL(0xfd3f87f7,0x2713cbe9),LL(0x4fd8d338,0x34163c33),LL(0x46cada61,0x7214cbe3),LL(0x6aa94a54,0x30a042dd),LL(0xf7b92358,0xe120acf2),LL(0x09be500b,0x30c3e8d0),LL(0x51dc7f0d,0x6f225e27),LL(0xb7edd06e,0xe3546714),L_(0x000000a9), +}, +/* digit=4 base_pwr=2^20 */ +{ + LL(0x20c1256f,0x2fd11810),LL(0x5aa78701,0xc4a46931),LL(0xea26a86c,0x056b1163),LL(0xbe00b905,0xa0ac68e4),LL(0x52f1dad4,0xc19c5769),LL(0xc6fde2d8,0xbbc11dae),LL(0x6293f810,0x3a3baf9c),LL(0x5056fba0,0xfa278d37),L_(0x000001c8), LL(0x5973f08b,0x4cc716b5),LL(0x8efce6c1,0xb5b613b1),LL(0x64d3ad94,0x248f005d),LL(0xba83b800,0xa375eb34),LL(0xc9ee4cf2,0x413af2a4),LL(0x68a27d29,0x25ea8722),LL(0x8d12fde5,0xc9c082bd),LL(0x2d233189,0x935ee6fa),L_(0x0000017f), + LL(0x85f1bef2,0xa73fb5cd),LL(0x111a8c9c,0x1a80d76a),LL(0x8d3b7461,0x2e325f88),LL(0x7765b87f,0xc8ad9e3f),LL(0x92e36012,0x2c7cf6c4),LL(0xbf5a9dc4,0x7d5db366),LL(0x6228a81d,0x915359f9),LL(0x725123cd,0x62477772),L_(0x00000183), LL(0x8b6c7a0e,0xa38cc5da),LL(0xee14f97f,0xb43bb38e),LL(0x770c4afd,0xaa0f15c0),LL(0x138850f3,0x3953b993),LL(0x2658cf7e,0xb70f0779),LL(0x1d447c8b,0xd78fd38c),LL(0x681177a0,0x8e23ebe4),LL(0x704ca751,0xf974bc9d),L_(0x00000059), + LL(0xba8fa7e4,0x4e9fda74),LL(0x334944db,0x404855f4),LL(0x65201753,0x31df130f),LL(0x19a9846d,0x661cb9d7),LL(0xbc651ab9,0xc04c2995),LL(0x91c2b653,0x1b2c3fb5),LL(0x1b65fb33,0xc90b91d2),LL(0x9233b624,0xf53a7c1f),L_(0x000000a4), LL(0xfceac108,0x86d9cc5a),LL(0x4cdd0a2e,0x0e2ec8f9),LL(0x309b7d38,0xf2c40675),LL(0x0d2223f6,0xc1e34e32),LL(0xa3be480d,0xb364f62b),LL(0xec527b72,0x3595753d),LL(0xf6639f06,0x4e283d90),LL(0x67ed0c35,0x3f3d71d5),L_(0x0000000c), + LL(0x5667e2e3,0x2b33e937),LL(0x711cfb9d,0x5cc9c7d0),LL(0xedf0adb9,0xc5aaa7c2),LL(0x610b704f,0x770150b6),LL(0x1107368e,0xf9af2a47),LL(0x06e6cc4e,0xfe1e566d),LL(0x814dd0ca,0x7ca67146),LL(0x6c67f663,0xf0648fd3),L_(0x000001a8), LL(0xecb744b3,0xd960d19d),LL(0xc0bcfa2a,0x99ff41db),LL(0x933b28a6,0xb97977ca),LL(0x951faf63,0xca3752a7),LL(0x15168f23,0x01e0f16b),LL(0x4ea397d9,0x05f55f96),LL(0x3b374a51,0x813c0d40),LL(0xe408ed3a,0x4d48dd43),L_(0x00000079), + LL(0x937586c5,0x83c230d0),LL(0x61062265,0x9c8f1eaf),LL(0x10419f67,0x2c698769),LL(0x8d67dbad,0x4407836e),LL(0x4c3c184d,0x99fd2f81),LL(0x52a37538,0x7825fefa),LL(0x45a721e3,0xfff07585),LL(0xa4b823d5,0xabf0c498),L_(0x00000014), LL(0x96e376eb,0x6a23fbe4),LL(0x5f76504d,0xb69ec350),LL(0x545afc26,0xfb28fe6b),LL(0x87ed2073,0xaf95f20e),LL(0xa6145047,0x4d27cd1b),LL(0xc4cc53f8,0xa35d865d),LL(0x9ee96b7f,0xb07b711a),LL(0x430aefde,0xda4b08ec),L_(0x000001a9), + LL(0xc7354ba1,0x9fdb88ca),LL(0xdc64a8c7,0x9f85e2ef),LL(0x7f3a69d0,0x5631012d),LL(0xd2bed232,0xfd4d1f17),LL(0x04dfd89c,0xe64d46be),LL(0xd5598288,0x9f8bf20f),LL(0x1f269d18,0xc11d0864),LL(0x333e29ff,0x28d5f9fc),L_(0x000000d2), LL(0x9cc7dab1,0xbbef8e94),LL(0x5c714223,0x7f10fed7),LL(0xbb61d266,0x09c647b0),LL(0xe823dbf3,0xf58db2b8),LL(0x4601c5a1,0x3a71fa3e),LL(0x344f9c02,0x0b5cbdd6),LL(0x77b11f1a,0xf8df6a65),LL(0x6eb12db5,0x640b327e),L_(0x00000170), + LL(0x0db94e9f,0x9c4cc346),LL(0x646b9dff,0x339e03c0),LL(0x7ae26f18,0x64dca76c),LL(0x2ba1712f,0x2c804061),LL(0x16950e5f,0xb5bf0fae),LL(0x13d1569e,0x185858b6),LL(0x5b35ba86,0x6880b124),LL(0x3c937406,0xc9854d9a),L_(0x00000058), LL(0xee5f1c44,0x96f83044),LL(0x10924610,0xe69176fe),LL(0x5cfb2614,0x324a7887),LL(0x825516a8,0xfbad9007),LL(0xc065d69c,0x3d71727b),LL(0x06621f87,0x85c81c53),LL(0xe21856f1,0x3ac1471a),LL(0x68582e4e,0x92c8d47e),L_(0x000001c3), + LL(0x9ace1c67,0x5469124c),LL(0x24f3ddfa,0xccd6db20),LL(0xaadd52b4,0x74a2fc6f),LL(0x24af0765,0x17b27151),LL(0xb5105915,0x118a106b),LL(0x7e240081,0xcffda2d6),LL(0xc6925ec7,0x88b3b39f),LL(0x37b374e2,0x1e0f8120),L_(0x0000016d), LL(0xefd91b81,0x3a683e17),LL(0xa72b7c41,0x952a60ed),LL(0x0495c130,0x9c4b61d8),LL(0xbf06a574,0x872c4bf6),LL(0x0c7cbd39,0xe01cb7ce),LL(0x989f1a82,0x726d7547),LL(0x44906b41,0x52742de9),LL(0x2e02ff37,0x3d8a309f),L_(0x000000a8), + LL(0x8dd4b66d,0x73240fb0),LL(0xb6e39bb1,0x1303b771),LL(0x195468c6,0x7bd7b8e4),LL(0x9c3d4d09,0xa8684e6d),LL(0x724f9017,0x31c9bec6),LL(0x2fd691e3,0x727ff44a),LL(0x5f3d4db7,0x89060924),LL(0x984ffa88,0xbadb47b0),L_(0x000001dd), LL(0x0c1bfdeb,0x6d85f7e8),LL(0x4b0cbc59,0xccccc632),LL(0xe1faf1ce,0xc7a0620b),LL(0xf6e95c18,0x0aa71a3a),LL(0x8fa50a9c,0x49a07249),LL(0x8cf3e64b,0xaed36f6a),LL(0x94bd6377,0xa4cf33ed),LL(0xeb0073d7,0x331d113f),L_(0x0000019f), + LL(0x87c3614b,0x53d1d5ef),LL(0xaa78183f,0x8ea4ef11),LL(0x5c12de26,0x84a5c0b0),LL(0x3315c75e,0x4f1d31e4),LL(0x0d7a1bdb,0x97ef83ed),LL(0x8d2ce325,0x91dd29d3),LL(0x3340be93,0x184b8ada),LL(0x0f8f6087,0x8ea5afc1),L_(0x0000015b), LL(0x8ce9dccb,0x96d5b0b9),LL(0x641d29f6,0xb48f1d66),LL(0x50813cfd,0x8881842c),LL(0x7ff13a32,0x3fa7e856),LL(0x78eaa08d,0x7984e336),LL(0xfbc798ef,0x67c2e064),LL(0xb3fc5a0a,0x23e92b2d),LL(0xe38a9115,0x5b238a08),L_(0x0000013f), + LL(0x3691b2b4,0x9b215aee),LL(0x5cf3f209,0xa1d61cff),LL(0x4a0f639e,0x2783ba69),LL(0x62cddd5b,0x1490682d),LL(0x5648b7c2,0x074ffd3c),LL(0xeb8c7a90,0x0e9bddbd),LL(0xf3751b7e,0xbce879c7),LL(0xa3bfa1d5,0xb7fba2d9),L_(0x0000016c), LL(0x20a30aeb,0xe42922b6),LL(0x3498a06c,0xa129b145),LL(0x26cc79e5,0x9d307ed3),LL(0x1c156f64,0x2baf6215),LL(0x14f481b2,0x9b79cac0),LL(0x2eb38c02,0xc564e629),LL(0xde0cf55a,0x24a1582a),LL(0xad5a0f7d,0x0a21a7d5),L_(0x0000011a), + LL(0xa68ccc39,0x6997bdf9),LL(0x05de1b77,0x553e670c),LL(0xee7e6f6d,0xadae5917),LL(0x199db8cd,0xecf2088a),LL(0xc33572de,0xa41c0fa0),LL(0x7b432d00,0x1dc922e3),LL(0xc58b8529,0xb5615888),LL(0xb9f1df40,0x310ba77e),L_(0x000001d5), LL(0xe11804a4,0x9626517b),LL(0xb50d5ff0,0x34399d8f),LL(0xeb302429,0x18d0e9c5),LL(0x47c13487,0x18ec316e),LL(0x851d2afc,0x1b701f2c),LL(0x167d9f22,0x883a9116),LL(0xa8f932f6,0xc3a9d8fd),LL(0x0da0e796,0xa92d4102),L_(0x00000116), + LL(0x029cd0ab,0x5d899070),LL(0xb5278a7b,0xe7ecc032),LL(0xabde08a8,0x9fa79438),LL(0xd3d2c019,0x80363047),LL(0x66ae4725,0x141bc210),LL(0xeab542c4,0xdf1d8696),LL(0xd1c060a0,0x3908c1f0),LL(0xfa788627,0x34d35bd3),L_(0x000001a3), LL(0x677d387d,0x76fb5476),LL(0x05f7c0d8,0x3a298203),LL(0x2097dc46,0xe5119f75),LL(0x1f8bd7b9,0x5996aa0e),LL(0x61982b25,0x92c77df7),LL(0x119a9371,0xac8b008e),LL(0x7f9b0675,0x346bbe9d),LL(0x8f2407d2,0xffdfc237),L_(0x00000063), + LL(0xab5b8457,0x52f7bee1),LL(0xbc2db536,0x07aae0d0),LL(0x58623dce,0xa3e03da9),LL(0xf444f5c3,0xe7b9b75d),LL(0xdb04bbba,0x8fa837de),LL(0xb065c230,0xed346416),LL(0x51909adb,0x16d6222a),LL(0x3c084209,0x7ace49f1),L_(0x000000d1), LL(0xdf2f5e19,0xbbb390fd),LL(0x38ba1ac8,0xc03099bd),LL(0xa1e828bc,0x6f1d5124),LL(0xb83c59a2,0xfd863005),LL(0xc17a0d0d,0x97857c15),LL(0x2d92fc6c,0xd53a6942),LL(0xb70e5498,0x36d4d6e3),LL(0x368524fa,0x19450987),L_(0x000001c5), + LL(0xc5143f3d,0xcda50c31),LL(0x34bd47c0,0x5f0ed71c),LL(0x1dee87b7,0x24882cc9),LL(0x7e3bbffb,0xd1c7d1fb),LL(0x1e403329,0x92474fcc),LL(0x53d4ee41,0xbcade2fb),LL(0xbd25257b,0x48035309),LL(0x1ff4afd1,0xa15dc4f3),L_(0x00000163), LL(0xf76b6465,0x54f67634),LL(0x6c3176f4,0x76970368),LL(0x7bbaa2a0,0xe3ebf308),LL(0xc5082d82,0x65ccfb3f),LL(0xec84bb5f,0xf3b5795a),LL(0x4999273e,0xa51dd238),LL(0x7a764cec,0xe6317658),LL(0xd0cd1c12,0xc77140d5),L_(0x0000006b), + LL(0x440872f1,0x28a369cd),LL(0xf5dafa9c,0x07ce63b6),LL(0xc12c54cd,0x4f79c12d),LL(0x0a67eec8,0xee7a34a0),LL(0xdbc7a4ed,0xff4e7f63),LL(0x40f7ea82,0x50ab4929),LL(0xdc3353f9,0xcd953027),LL(0x1b37a3c6,0x4f307be2),L_(0x00000197), LL(0x4c5f758a,0x416ce18f),LL(0xa1adfbab,0xb9e45e70),LL(0xa53a6ae6,0xf8e38074),LL(0x19bbc5ea,0x639d3359),LL(0x66580a40,0x3c3446f8),LL(0xabe333f8,0xf20b5de7),LL(0x703c0639,0xd4a42c48),LL(0x59dbbfad,0xfadceed6),L_(0x0000016e), +}, +/* digit=5 base_pwr=2^25 */ +{ + LL(0xd025b49c,0xa66a4107),LL(0x4a0b9d5c,0x09a27870),LL(0x383562d7,0xb8b27e26),LL(0x8bfb3e1f,0xcd156bc2),LL(0x06132452,0x49ca82ac),LL(0x92d3d9a2,0xa22d3a0e),LL(0xd34655b5,0xa2b19aaf),LL(0x36ff20de,0xa0e8848d),L_(0x0000018f), LL(0x86f6b302,0x1143727c),LL(0x8192a853,0xc5c00385),LL(0x52b2fd4b,0xdaf1da93),LL(0x5e79d62b,0xde7c27d2),LL(0x23c74903,0x18fc4730),LL(0xd6b3ad47,0x3e48286e),LL(0xe1fb2466,0xde5e2dbe),LL(0x558feb16,0x6b4b7857),L_(0x000000b0), + LL(0xf33aa997,0x399f83a0),LL(0xac6eebd0,0xf152344e),LL(0x1e36ba58,0xb233cc5d),LL(0xbcb7b01b,0xb95ccb58),LL(0x0e6e01c3,0xbd70b347),LL(0x28b73f3c,0x471f1429),LL(0x192a8fa5,0x72f33fe8),LL(0x51b82a42,0xe6f5e677),L_(0x000001ef), LL(0x91ea1f80,0x29d67e20),LL(0x9219d0e0,0x355df0fe),LL(0x86128187,0xb88069ec),LL(0x196ceb57,0xc5d6f3e9),LL(0xcb831fc0,0x61a90e72),LL(0x76ccb185,0x875cf514),LL(0x775daca6,0x5e9d0fdc),LL(0xf11fb0d6,0x37c46556),L_(0x00000034), + LL(0x4f6f0528,0x8365a540),LL(0xd607f5f7,0xa6130da7),LL(0x85d457a1,0x282ce8f3),LL(0x7a71db8c,0xf22e8e12),LL(0x675b1780,0x1ed981d1),LL(0x54455f27,0x879b7161),LL(0xd4c8d2d0,0xe53f2395),LL(0xfbb885ea,0x533bd02f),L_(0x0000018d), LL(0xdd4e8a69,0xc1a29f03),LL(0xd15d6f60,0x620f1843),LL(0x51aa6497,0xb9253703),LL(0x3bdc3d85,0xb15aa83f),LL(0xd5713630,0x38bafd76),LL(0x46f5b250,0xa324c450),LL(0xc7ef0e3a,0x428ff854),LL(0x3074cd9a,0x61029f87),L_(0x00000130), + LL(0xdb38150e,0x98edde21),LL(0x02c3ed54,0x10e8c4a8),LL(0x92441512,0x5d3d5708),LL(0xf9b009f9,0x785d69a6),LL(0xac80e954,0xa197859e),LL(0x166d8dbf,0x9a954ad6),LL(0x5cac81cf,0xd5536054),LL(0x01aaa83b,0xd431342d),L_(0x000001b4), LL(0x578a0342,0x2cf62469),LL(0xb8dd8820,0x8c4987d0),LL(0xc2b48e0f,0x3b1becfe),LL(0xd5c621a6,0xa90343a0),LL(0xf668f2b6,0xfe098700),LL(0x9c088bd4,0xbd8fa882),LL(0x3f0936a8,0x67794851),LL(0xae92e720,0xc5748733),L_(0x00000186), + LL(0x45dfbf2f,0x3cd0f06a),LL(0x39d3c154,0xf941b6e2),LL(0x350febe9,0x2d076edd),LL(0x4d560b52,0x55928e0b),LL(0x2b184b05,0x9f1196dc),LL(0x85b25794,0x6b4f0570),LL(0x942eaf69,0xe7c1e0be),LL(0xd7c08589,0x63f1f217),L_(0x0000012d), LL(0x7bc564b2,0x3dbde8c2),LL(0x8d95012a,0xeff3d407),LL(0xa7b28d48,0x08aae60c),LL(0x1b0df773,0xe7d844e8),LL(0xf5cd9389,0xe907a4ac),LL(0x0fa28183,0x6e40930b),LL(0x381dd63b,0x20fcef80),LL(0xa89662a0,0x5312183a),L_(0x000001c7), + LL(0x33be7ac9,0xf700265a),LL(0xf01390c9,0x82de52e5),LL(0x6ffa798e,0xd0dd94ce),LL(0xd994953e,0x2a40701e),LL(0xdc206da1,0xf9937624),LL(0x9641d8d6,0x68b09fde),LL(0xfb08d3aa,0x515609bd),LL(0xc7a5c685,0x916b2107),L_(0x0000017a), LL(0x34c11eaf,0x007f99c6),LL(0x3e931cb9,0xfb91b432),LL(0x97268272,0xb16a5f9f),LL(0x14f51fca,0x22fdf379),LL(0x448325ce,0xa5ef4116),LL(0x36a8c292,0xb6fe6cbe),LL(0xee4ed4ed,0xd30012d0),LL(0x5ebc5cb5,0x50b1fe61),L_(0x00000103), + LL(0x6edf3f2a,0xc68eb226),LL(0x00200346,0x880aea4d),LL(0x8d31f029,0x006895c8),LL(0xf085775e,0x89fcfc5e),LL(0xd25cf0d0,0xb2a191ee),LL(0x4508a2eb,0xc9428552),LL(0x90ca4e90,0x22abade9),LL(0x985e1eac,0x8ec6b7a8),L_(0x00000092), LL(0x31e5ce6f,0x96705235),LL(0xdb11d250,0xee984de1),LL(0xfa51e382,0x06a31aa7),LL(0x47482dcf,0xb9587e74),LL(0x988048ce,0x51593c99),LL(0xca987f12,0x6fe30df1),LL(0xffd978f6,0xfa604f59),LL(0x8e6925df,0xbebed832),L_(0x00000121), + LL(0x24f8517a,0x039e54c1),LL(0x2549eb0e,0x1ec83d52),LL(0xeb02627a,0x18a8ead8),LL(0x95771e96,0x497c2cef),LL(0xd4f3a6b4,0x1c4917c7),LL(0x080b55a4,0xe8babd80),LL(0x90585b7b,0x6c548c79),LL(0xdabe54a0,0x2dcbdc64),L_(0x00000099), LL(0x2b269bac,0xdf9b98b0),LL(0xc4ed8c6b,0x0247e075),LL(0xd8e1a299,0xbc830716),LL(0x019c94ed,0x3458f33a),LL(0x9c1665fb,0xcd4d82fa),LL(0xff63e700,0x2326e507),LL(0x41c9bbfa,0x5b452b83),LL(0xc9214a3f,0x1a7050df),L_(0x0000012f), + LL(0x3a39f3de,0x6207af25),LL(0x48efebf2,0xe105123f),LL(0xc14c4d56,0xe2f0d038),LL(0xe91d26ba,0x4b6de581),LL(0x4ec88307,0x8f223417),LL(0x0cd6e3f2,0x6ecd94a5),LL(0x986d1a76,0x75765814),LL(0x6e247362,0x2e292aa2),L_(0x00000047), LL(0x8b83a718,0x6e918a1b),LL(0xc07b54ba,0x67b07959),LL(0x22adb8dc,0x96a7a38b),LL(0x2810db83,0xd80ec3cd),LL(0x512687fb,0x3b7485fc),LL(0x6a994329,0x746aec5a),LL(0xd193ecb8,0x6e8d782e),LL(0x72720264,0xf4c3b7e3),L_(0x00000160), + LL(0x18efe68e,0x7f9ec076),LL(0xd1c7c3de,0x2cb36021),LL(0x39cd1c50,0x4584b14d),LL(0x997b5332,0x4275cf71),LL(0x8ac5db4a,0x47f829e4),LL(0x0bc80acc,0x41dd84b4),LL(0x93021863,0x498e9e29),LL(0x6c2be4f1,0x514dc4f7),L_(0x000001aa), LL(0xa59f6acf,0x50030fe6),LL(0xbe6f68fb,0x21480f26),LL(0x217fd94a,0x7a75421a),LL(0xfc5b6dc5,0x7f8cd384),LL(0x19e9f3af,0x674a2419),LL(0xd009b54a,0x454fa1ef),LL(0x6bd82b7f,0xf5bea7db),LL(0x31688d56,0x02858efb),L_(0x00000076), + LL(0x24d7621f,0x92168834),LL(0x1fc413da,0x47855bbd),LL(0x13a9e461,0x17199daf),LL(0x4e3536bc,0x42c2a13c),LL(0x2b4a64a6,0xa8670cf2),LL(0x38d86ada,0x6c221d69),LL(0x68ee0d8d,0x91def21c),LL(0xa5126dec,0x99a859f3),L_(0x000000bd), LL(0x03f14fd3,0x378bafe9),LL(0xef0ab60c,0x11be6eda),LL(0xe18440ad,0x59ce941a),LL(0xead8d861,0x59cf0928),LL(0xd51985bf,0x608dbb83),LL(0x54f87d88,0xb568f14f),LL(0x15b84493,0xb574903f),LL(0xbc432c12,0x90bc4e6d),L_(0x0000010e), + LL(0x79d38b74,0x7eaa6aae),LL(0xf84e0117,0xdb942db5),LL(0x77a0b212,0xd22fb9bf),LL(0xc56ffa35,0x3c3e0310),LL(0xdf48ef3e,0x9f901558),LL(0x372e6e7b,0xa69d9dbe),LL(0x0da5fd74,0xeaa231af),LL(0x1a06775c,0xc9f04db6),L_(0x000001cf), LL(0xd09d3577,0x7802613d),LL(0x61b4d69e,0x19ba2ee8),LL(0xa78fefe1,0x46a7d09b),LL(0x7be0e7b7,0xedbd7a6a),LL(0xed289bee,0x894b4148),LL(0x613e6a64,0x4ced52a0),LL(0xf2a487de,0x58c856b6),LL(0x802d7250,0xe3475832),L_(0x0000000e), + LL(0xd1a6df86,0x365ca885),LL(0xca64af53,0xfb62f8cc),LL(0xbd654db8,0x7be4b7f7),LL(0x029300b0,0x8e5d1bd5),LL(0xc892e13f,0x2c67f87f),LL(0xcc4777a4,0x8b3a46c0),LL(0xba6516c2,0x7400b2a4),LL(0x6cbe8178,0xce032c3d),L_(0x000001ff), LL(0x24153eca,0x540f05b7),LL(0x19221aa5,0x97f95061),LL(0x09e289a2,0xbba5984c),LL(0xf2867a5f,0xa0f6739b),LL(0x0f075e42,0xa0718647),LL(0xab768457,0xa0b31eb8),LL(0xae89b1c6,0xf7e66ea8),LL(0x127d85c9,0xabebbadf),L_(0x00000087), + LL(0x6d5a6301,0x8cd94792),LL(0x3dd861e6,0x97b8baf4),LL(0xfe513cef,0x04f520d2),LL(0x71773dad,0x05832bb5),LL(0xe9f256e1,0x644c2754),LL(0x40df1966,0x700981f0),LL(0x9215b453,0x5dacb6f1),LL(0xc1eb3c97,0xc9b5d3dd),L_(0x00000093), LL(0xbd504fe7,0x365fddbf),LL(0x8fd2242a,0xc2c35f7a),LL(0x4bda285f,0xc1adbae0),LL(0xe964cfd7,0x9f4dbf4e),LL(0x29f5bb5b,0xebe1db2e),LL(0xaab3521e,0xeb26c679),LL(0x17b1e0d2,0x460499ac),LL(0x5a9383dc,0x366f4a0a),L_(0x000001a5), + LL(0x13ec332c,0xb2a67fff),LL(0xb50e03f5,0x50971210),LL(0xfca3bc03,0xa67e12aa),LL(0x4ad52b09,0xbb90dbda),LL(0xff71aa12,0x0edb116e),LL(0x6d43afd2,0x0827d59a),LL(0x97de7056,0x29d216bb),LL(0x7a131b7f,0xe3451b60),L_(0x0000017c), LL(0xb922ac22,0xfa7cdb03),LL(0xf832d6ed,0x58c5a459),LL(0xd373c013,0x7af850a7),LL(0xec4706eb,0x2c0a7082),LL(0x67642828,0x10a909d7),LL(0xae5e13c9,0xab317113),LL(0x5697d646,0x62e5e289),LL(0x83fffcd8,0xb1590922),L_(0x000000d9), + LL(0x331109fa,0x16fb0987),LL(0x7dc5b06e,0xf23db224),LL(0x62c76a0d,0x3f2f53ef),LL(0xf6843163,0x41397218),LL(0x63cbe6a9,0x6d05ffc2),LL(0xb14a253b,0x7ed103a8),LL(0x8ee0b454,0xf389d693),LL(0x3310d7c6,0x9221bf4f),L_(0x000001e6), LL(0xe3ae5ea5,0xeba63d9e),LL(0x281cc803,0xff5d7d44),LL(0x5a289454,0xfa26da52),LL(0x8b5bf3e7,0x8b9a0868),LL(0xf6c91ab8,0x565a2877),LL(0x10a39907,0x737c0212),LL(0x1223d6a1,0xe131db55),LL(0x19f277aa,0x3e1fc414),L_(0x000001cc), +}, +/* digit=6 base_pwr=2^30 */ +{ + LL(0x205e28d0,0x97a1a0b2),LL(0x662c120e,0x6679b5d8),LL(0xd256806d,0x56f0da8b),LL(0x5177882d,0xb3a0c292),LL(0x1fd447d9,0x15eb4fc8),LL(0xd44b077e,0xf15d98b7),LL(0x44ac5bb0,0x2721ac99),LL(0x38b1a539,0x90bc5964),L_(0x000000ee), LL(0x2615ec8d,0x7c193099),LL(0x6014ef39,0xde0771a4),LL(0x7929bb3e,0x2d975e02),LL(0xcd3fad00,0xd387bd9d),LL(0x38c634c3,0x4ebf2479),LL(0x08079c3f,0x2c2da2b5),LL(0xe2b2209a,0xd2e85151),LL(0xbb80ad2e,0x2c3845f4),L_(0x00000166), + LL(0x8f90c95c,0xb8f686b4),LL(0x17d91844,0xa7467794),LL(0xb498e1e2,0x354bd2f6),LL(0xdf2e61e9,0xaea1ed2a),LL(0xb0db08d1,0xf347f08d),LL(0x02cba497,0x78713784),LL(0x5de0850c,0x51753c73),LL(0xed5b7079,0x1738e1f9),L_(0x0000007a), LL(0xd4d045e4,0x91e83186),LL(0xd4180245,0x2f07b1ad),LL(0x059bab68,0x1d467074),LL(0x2b0f371c,0xb0e391ea),LL(0x80e5f219,0xf22e68c2),LL(0xb058f844,0x79b7f960),LL(0x6ca98996,0xd1250203),LL(0x1f63acdb,0x0336e28a),L_(0x000000bf), + LL(0x6f5a9e54,0xbda97f65),LL(0xaf7af291,0x3726d93c),LL(0x49f05287,0x2f95260a),LL(0x41ba40c2,0x77d547dd),LL(0x957f51b5,0x05f74755),LL(0x8ef3d171,0xabb35056),LL(0x131fa675,0x73e0ad7e),LL(0xf2cdf14a,0xc6b88e0e),L_(0x00000119), LL(0xe74e4c04,0x5dd32692),LL(0x56454c03,0xbec75c70),LL(0xcf373902,0xf969fbf4),LL(0xcc237598,0xc6be1f8c),LL(0x25a52045,0x1317acb0),LL(0xbd2ef98d,0x4c09bec9),LL(0x87e896e1,0xa4033c40),LL(0x16b064d0,0x35a420c5),L_(0x00000082), + LL(0x2223b656,0xf580463d),LL(0x7a55f8bf,0xbd3d896f),LL(0x19d6f3c8,0xdc1bc358),LL(0xf8dfdd32,0xa2786cfc),LL(0x57477a57,0xae5589b6),LL(0x9f47a3b0,0x06de5eae),LL(0x93f21788,0x2c782fcd),LL(0xd9479ee3,0x96eb796d),L_(0x00000135), LL(0x0929435c,0x52a35de2),LL(0x5b11dd83,0x4ce809af),LL(0x080ab8c3,0xe25a76cf),LL(0x50478b1c,0x216bfb22),LL(0x103b3ff7,0xd87f4762),LL(0x99b34133,0x41327480),LL(0x95f627aa,0xa729b689),LL(0x62b3cf30,0xd78ba6fb),L_(0x0000017d), + LL(0x721c2b47,0x251bc403),LL(0xb04b42b0,0x43b1fedb),LL(0x73ae072d,0xe91e26a3),LL(0x0d23d9bf,0xbe237693),LL(0x40c88f4f,0x3424beb1),LL(0x6a55a156,0xa0854744),LL(0x26b2b7ac,0xd61d14b6),LL(0x4ee7f1ff,0x2b8c920f),L_(0x0000005f), LL(0xa08faeb0,0x9b46600b),LL(0xfac400c9,0x541f8756),LL(0xfa469aa7,0x961867a9),LL(0x9f699a76,0xaf3ea90e),LL(0x6fa24828,0x87c1ec18),LL(0xb4eae9ea,0x13538c6c),LL(0xd16f2fd6,0x3f2706c2),LL(0x1ba2b83f,0x49f12b47),L_(0x0000001b), + LL(0xbdd4d4a0,0x305ae6fd),LL(0x1d7bb497,0xfd52a449),LL(0x143fec43,0xf6e16ac4),LL(0xebf21b11,0x8a70f179),LL(0xdc6c0238,0xb6be355d),LL(0xd156ead0,0x45178899),LL(0x91cfd7f1,0xdeb4b443),LL(0x63af6554,0x56edae1a),L_(0x000000f2), LL(0xc755678e,0x78e21693),LL(0x747e0d46,0x04110693),LL(0x8c5bb7f8,0x878fdb74),LL(0xb933a4bb,0x90576198),LL(0x4a53f37a,0xb6123d17),LL(0x9e11e3a2,0x6fc61c48),LL(0xf4f2778d,0x0bd5422a),LL(0x78936064,0x0834022a),L_(0x00000075), + LL(0x1cdfc255,0x0e89d4d5),LL(0x620f831a,0x353a0af0),LL(0xb276e069,0x687747f6),LL(0x848dd0c6,0xf27dfd62),LL(0x0bb6c7d1,0x05846ed2),LL(0x561ca5d7,0x17ec1af8),LL(0x39a2756b,0x4280d8ec),LL(0xf4bc9db5,0x3c387c2c),L_(0x00000111), LL(0x827b9727,0x96e34ce9),LL(0x3fe216b6,0x79c77aa0),LL(0x6720f347,0xfad412f9),LL(0x695ff127,0x74dee9fa),LL(0x5f1b0845,0xe65f23d5),LL(0xbfbfe9e6,0x3e545ad7),LL(0xe1c9f0ed,0x92d8678d),LL(0x9914fd4c,0x34192088),L_(0x000001ca), + LL(0x479267d4,0x0396f4d9),LL(0x88692b5b,0xb1e5f7f0),LL(0xfd9437e3,0x9509c3c1),LL(0xb9b9b7fa,0x15a75e6c),LL(0xba37b6c1,0x5b42f650),LL(0xdbbc62d0,0x5b3cf510),LL(0x6d5fe62d,0xc6a49bf0),LL(0x824e6593,0x21168049),L_(0x0000007a), LL(0xcc7f70f3,0xeb94fc53),LL(0x6a2b34a3,0x82c0e60f),LL(0xea4bc9d4,0x8f42888a),LL(0x7ab9ed31,0xc0fe3ed3),LL(0x1a505ae0,0xe94fad3b),LL(0x545382c9,0xfa0f3128),LL(0x9ae4fb8e,0x44bd4be6),LL(0xa516fe88,0x3f6a7003),L_(0x000000d6), + LL(0x5455aac0,0x2a8889b3),LL(0xc9b0d66d,0x1c5bf83e),LL(0x736f64af,0xeb5d7b9e),LL(0xb0411171,0xfd19cd82),LL(0x22d46926,0xdc65529f),LL(0x1a902efa,0x2603fa26),LL(0xa829e75c,0x4e40cf1f),LL(0x25ea27fc,0xe0d4e87d),L_(0x00000161), LL(0x4b68fce9,0xe6bccc3f),LL(0x83ed8b75,0x047d0099),LL(0x864c3567,0x85060d1b),LL(0x1ddc1dd9,0x3ee4ae76),LL(0x974e9669,0x389206d2),LL(0x6126092a,0x91ac46ae),LL(0xe4c7cd87,0x450710f2),LL(0x9e581f06,0xc5c30640),L_(0x000000ae), + LL(0x291b3bd9,0x42264245),LL(0x08f40ce6,0x9254ae8e),LL(0x244beb5d,0x6f14d800),LL(0x0b345a89,0xe0141c82),LL(0xb5b0de4a,0xf52d10c3),LL(0x2ca8fb1f,0xebf183aa),LL(0xdb884d75,0x8611998a),LL(0xd2bb9afb,0xce37079a),L_(0x00000152), LL(0xa6d267bb,0xe6bc257b),LL(0xf0568557,0x2160d875),LL(0x6b9e2dd7,0xb1fad31a),LL(0x9c709714,0xed3d24a8),LL(0xf7d05de5,0x61431dc5),LL(0xdc526178,0x0501b15f),LL(0xe40e89ee,0xb2aa5088),LL(0xf3025f1d,0x365bc9f8),L_(0x000001e3), + LL(0x859759f0,0xb249eb45),LL(0xff4d25bf,0xb8020268),LL(0xd9737355,0x9d4a831f),LL(0x84343cef,0xba7e139e),LL(0x5bb5557e,0xb24bfdbf),LL(0x3ab8fb19,0x31a1b161),LL(0x3fa57146,0x4ec60d24),LL(0x01fac245,0xf4776d25),L_(0x000000f7), LL(0x3e8c0bac,0x5b116d34),LL(0x0f307133,0x62db28ba),LL(0x9a0533b5,0x0939b6eb),LL(0xacf1cf00,0x89d1c401),LL(0x693e7c92,0x4b14c89d),LL(0x0945b630,0x4ed7b53a),LL(0x4b162dc1,0x79a7fa4b),LL(0x3e6b7dc4,0xbe6da5c0),L_(0x0000009b), + LL(0xb35e8088,0x24bb4a7a),LL(0x9e47ec6e,0x968fd370),LL(0xe7d95582,0x474d55e3),LL(0x783f2538,0x9d35c0f8),LL(0x83b454e1,0x4578fbe5),LL(0x114b724c,0x98f51326),LL(0x97ee546b,0xf190d99b),LL(0x801229b8,0x3b184691),L_(0x000000b2), LL(0x7ee4a0a2,0x8b19b4b6),LL(0xd4f5faa0,0xc220cb11),LL(0x6ad0862b,0xce1c7d42),LL(0xd7a63508,0x3236a72d),LL(0xeaae4777,0x36cda4ac),LL(0xbe3fcce9,0x808ba7a0),LL(0x972c3840,0x79c14a5b),LL(0xcda4655c,0x0cecb887),L_(0x00000155), + LL(0xf72e10a1,0x23f718f8),LL(0x07515abf,0x820e2eb6),LL(0xc81af50b,0xea92c2b4),LL(0x06af11fd,0x71048468),LL(0x755ab069,0x6ada8b6e),LL(0x5f2e0402,0xe7b549c5),LL(0xa45c27e8,0x13b56922),LL(0x1760f8ae,0x07a6f401),L_(0x00000085), LL(0xacb0965f,0xc6437d60),LL(0x30e79f87,0x93a0e68a),LL(0xcb0f4dcf,0x3613f9b0),LL(0x02e2383c,0x32355c62),LL(0x3ec75821,0x4a835dde),LL(0x4891bfda,0xa38a3a87),LL(0xd0206185,0x9de0b9b9),LL(0xa9cedb8c,0x54d8a9f4),L_(0x0000018f), + LL(0x8b5ddb45,0xdc46fb16),LL(0x8166a790,0x41a5e4a7),LL(0xc854c900,0x342b51ba),LL(0xeb4a908e,0x116e0741),LL(0xb76499d7,0xc1551311),LL(0x6cc7399a,0xccaf37b4),LL(0xf2094cc8,0x9c29f265),LL(0xc4a1fb65,0xcca404c7),L_(0x0000005f), LL(0xa4cf272a,0xcfc8b9dd),LL(0x9fd1f1c4,0xb8e8cb4f),LL(0x8ddfade3,0xfd72dadc),LL(0xf084903f,0x764ba410),LL(0x9094e8a7,0xf43466f6),LL(0xb9869fee,0x1564c4ac),LL(0x82a6cc83,0x7742c943),LL(0x89c1b557,0x2b44656c),L_(0x0000001a), + LL(0x7e25b95d,0x08c1d55c),LL(0x50011066,0x2de90a9b),LL(0x63db375d,0x46d6ba3b),LL(0x2d3fa9bd,0x9ae8aa53),LL(0x9fb12874,0xabf67a35),LL(0xd1f1f45f,0x05d8099d),LL(0xdc479fc0,0xdf2cdfde),LL(0x779d8b52,0xe51f87a9),L_(0x000000da), LL(0x8f62ac2a,0x4ab94029),LL(0x56a0908f,0xbb03c1b4),LL(0x9ce0943b,0x6319d2ed),LL(0xe2d743f7,0x910c2363),LL(0x871683b7,0x78ddf2fb),LL(0x43f08ff4,0xb7c5e230),LL(0x2ebe58e7,0x1c175b19),LL(0xc02e1520,0x9f7bb585),L_(0x0000013c), + LL(0xec7786aa,0x70017bcc),LL(0x26576a40,0x926b1ae0),LL(0x5e810d17,0x6c002685),LL(0xdef24d30,0xd826c439),LL(0x78ac55c7,0xe40f66cc),LL(0xfbf9b521,0xe26697c9),LL(0x01aaaf8a,0x46a75caf),LL(0xd094aec3,0x12da82a9),L_(0x00000072), LL(0x99a161f4,0xcc3c7e5f),LL(0x88977e72,0xfefa749b),LL(0xd6e24cf4,0x56863737),LL(0x4607a910,0x4dcfe4ed),LL(0x01768685,0xac7bc09c),LL(0x9ec211ac,0xc731394d),LL(0xac76909c,0xd59e4dd8),LL(0x246fb612,0x6cf6ebc7),L_(0x00000183), +}, +/* digit=7 base_pwr=2^35 */ +{ + LL(0x5592f380,0x0dd54d04),LL(0x44d05f6f,0x298fe241),LL(0xe2d191b2,0xad46274a),LL(0x9113fc1e,0xcc54c590),LL(0x0dea702c,0x763ea8c1),LL(0x74f208c5,0x35441e72),LL(0x578c5736,0x3706b2f2),LL(0x81343695,0x93b46c3b),L_(0x00000063), LL(0x88ada464,0x4c2605a5),LL(0xd5464606,0x004bbf31),LL(0xca04e18d,0x03210805),LL(0x998cccf9,0x89627867),LL(0x48398a6f,0x8d2faed1),LL(0xca85dacf,0xde01ea7e),LL(0x92784742,0x0feb7d82),LL(0x9fe5859b,0xb9f9aeb8),L_(0x00000150), + LL(0x05a73aaf,0xe5e08d95),LL(0x18e919ab,0x9fd8bcd3),LL(0x5b386f06,0xd6891589),LL(0x0b5abd07,0x07bc244a),LL(0x339901b0,0xc3693691),LL(0x684cff05,0x179dcef7),LL(0xe69b7993,0xb0125196),LL(0x23db7308,0x03553a28),L_(0x000000be), LL(0xf88e8361,0x0c66e267),LL(0x2f9f93bf,0x6d375b82),LL(0x09656c0d,0x6d5dc021),LL(0x0c1fc16b,0xba81db53),LL(0x86bf6541,0xb4caf94b),LL(0x1a6d443c,0x87f59d3c),LL(0x0e10fe1a,0xb72afcd6),LL(0x8d7e1995,0x1a0c6a08),L_(0x000000da), + LL(0x3c2640b0,0xef7a7df4),LL(0xf6850378,0x2a55c175),LL(0xed04fb24,0x8b6f667a),LL(0xa7e743ff,0x93061f72),LL(0xdf46d114,0xdaa59d80),LL(0x9d4ed766,0x49c1e1f8),LL(0x4b991a82,0x2fe74922),LL(0x2e41804d,0xe9a27b22),L_(0x00000196), LL(0xf37b7a2c,0x140444d6),LL(0x8f14ff32,0xf54bdbb6),LL(0x34a9f316,0xde42c24e),LL(0x9515de8e,0x96a88eec),LL(0xf1959aa3,0xdcd09006),LL(0x30671815,0x058c4643),LL(0xb6c6d2af,0x65dac0ec),LL(0xd1a5f578,0xc7fbdbf4),L_(0x00000150), + LL(0x0a5b8daa,0xd4667616),LL(0x8dcdbdbe,0x16e7a7e7),LL(0x1b0f5e9c,0x9178e1fc),LL(0x3cb9503e,0xf2294c93),LL(0xe6f78b36,0x420442aa),LL(0xfb2d6ec3,0x01e17c7f),LL(0x7d30fddc,0x9c7293b8),LL(0x36bed473,0xa2fcf928),L_(0x000000a0), LL(0x4975943e,0x84cf37b8),LL(0xab2857b7,0x645eaa02),LL(0xe12a0d65,0x3d30b0ac),LL(0x267c93aa,0x30be3eff),LL(0x6fa69542,0x903aa7ed),LL(0x2cdb01e1,0x311ae0c8),LL(0xc5056d4c,0x275f0bd3),LL(0x94d54a98,0x7fa09220),L_(0x0000005d), + LL(0x8c942dd7,0x62d7fcf1),LL(0xc893077d,0x643ea78e),LL(0xda471b28,0x4e9d9fdb),LL(0xfb673df7,0xe64cf76a),LL(0x51536a0a,0xfccba343),LL(0x5fc020fd,0x279bce99),LL(0xdee7d155,0xb7f5fd72),LL(0x68c90549,0xdc14172d),L_(0x00000119), LL(0x4ecec5bb,0xe6b0baf4),LL(0x6a4bc82c,0x18f9ed0f),LL(0x93eb1856,0xdc69912e),LL(0x6f327c7d,0x2be0e2b3),LL(0x52fbc6f3,0xe7e8bfdb),LL(0x24b3acf0,0xd81de1d4),LL(0x62a84326,0xe49b2810),LL(0xc3912b42,0x55d57336),L_(0x000001ab), + LL(0x551c0660,0x2fe061f4),LL(0x91d40c3d,0xa2b6dd70),LL(0x7cc4883d,0xbd2bf743),LL(0x58011dda,0xaee852f2),LL(0x71bdaba7,0x39f463ce),LL(0xbd7c195b,0xfb33b881),LL(0xede085fe,0xbf7d695a),LL(0xf8fa2471,0xe577b380),L_(0x00000149), LL(0xc96eeab9,0x8a6f5a6d),LL(0x07f71f13,0xb29f154f),LL(0x242d7a9e,0xb707d1c7),LL(0xf3c7f1af,0x44aee842),LL(0x0af2896b,0x01f56999),LL(0xc5c5b556,0x1e36ed07),LL(0xcca26786,0x245430fa),LL(0xa78eb214,0xaa96816e),L_(0x00000106), + LL(0x0f4d9fba,0xa4eddc14),LL(0x9ebcb886,0x420f1510),LL(0x3c114b65,0x4e6c23f6),LL(0xba3fc70d,0x5f6bd766),LL(0x718ac792,0x532bdc38),LL(0x5c2b8f46,0x03efcc7f),LL(0x72c3a472,0x5c2b9a45),LL(0x9dc110c9,0xba2fb9f4),L_(0x00000147), LL(0x1a6af2f7,0xb5c3ea1c),LL(0x8052b6c2,0xab27ec8a),LL(0xa3cff07f,0x4c92575d),LL(0x00fa9778,0x4b43265d),LL(0xff1e188a,0x6c53a129),LL(0xedc9e95e,0xfcb7ddb8),LL(0xfb7de911,0xc1dd8dd6),LL(0x6e53bbb5,0xd488fa72),L_(0x000000b1), + LL(0x7daeaaab,0x6eaee317),LL(0x6e35398b,0xf5c1267a),LL(0xd01bbbbb,0xa2b92ae2),LL(0x3af1a4f4,0x5cdfa4bf),LL(0x5532088e,0x638a58f5),LL(0x2a0ec8ee,0xbce5e7aa),LL(0x79c3f50b,0x46a16448),LL(0x378387c2,0x1c40f25e),L_(0x000001cd), LL(0x6769ee88,0x8ec2984b),LL(0x62fea90f,0x1f9e47bd),LL(0xe7522665,0x3d9f9867),LL(0x916cb049,0x5be2cccb),LL(0x1106987f,0x71156e1c),LL(0x7bad869c,0x011576bd),LL(0x945007a5,0x38b76a6e),LL(0xc18de41f,0xd11c30ae),L_(0x00000109), + LL(0x6ee5a7d6,0x2831843a),LL(0x58bef9ba,0xe4f07223),LL(0xd00147a3,0xe1c40e6b),LL(0xd10e53e0,0xe7025e87),LL(0x5d5f797a,0xf0dd838b),LL(0xe3b65b5f,0x16931c80),LL(0xd1ff0eb8,0x42704c55),LL(0xffc29bf2,0x999442d4),L_(0x00000150), LL(0x279cdc4f,0xed03963d),LL(0x51c15682,0x4a0c613e),LL(0xc3d504ac,0x6ef1db42),LL(0xba61ce4e,0xf8742eaf),LL(0x2e74a6d4,0x3efa3cf5),LL(0xd53943f5,0x201c12e1),LL(0x253da54a,0xa468664c),LL(0xa35c5fd4,0x6014ba12),L_(0x000001c6), + LL(0x687c5994,0x7a229d23),LL(0x8aa53c89,0x118e3e78),LL(0x080e973b,0x15c794e3),LL(0x5a91046f,0x30c0f281),LL(0x5ac04ed5,0xfe35eb92),LL(0x0c5478a4,0xfe18e707),LL(0x7d9c57a1,0x4bf409ce),LL(0xfcd17edd,0xdecf58aa),L_(0x00000138), LL(0x3d2850fe,0x23469475),LL(0x2c45c37e,0xc528ec7d),LL(0xf346792a,0xa36bdd41),LL(0xca964be2,0x11d67932),LL(0x3fef3e79,0xc72673a0),LL(0xf0bc0101,0x53d6358e),LL(0xb9caaba6,0x2bf874e2),LL(0xbbcc3188,0xe8aaf45e),L_(0x00000102), + LL(0xf22d90c1,0x24df4deb),LL(0x449aa491,0x37a9290e),LL(0xd1657f10,0xc6bbdf04),LL(0x4ba3ee4b,0x4075fc0e),LL(0xe3e07500,0xf134a6c8),LL(0x7661f086,0x2b3b9213),LL(0xfd4c08de,0x6f51f169),LL(0xcae8fa7f,0x275a9f04),L_(0x000000b3), LL(0x9303da25,0x1e8df3c5),LL(0x6a92d1ac,0xb326f2a6),LL(0xafc6ed8c,0x46ef258d),LL(0xe7d3b364,0x53fe6ba5),LL(0x7b2d91d1,0x669d23c2),LL(0x150f6968,0xc05b6015),LL(0x230a0105,0x376b2356),LL(0x3df6108a,0xea51c972),L_(0x00000165), + LL(0x5490baa5,0x148d7d84),LL(0xae379993,0xd948464c),LL(0xac4f8ace,0xddaf46fe),LL(0x68d1b7c8,0x6c39b672),LL(0x10d4770e,0x5967ac57),LL(0x30a2c195,0x2098b748),LL(0x19047735,0x858d17fe),LL(0xe5071ced,0x709c0191),L_(0x0000000a), LL(0x44b4f003,0x2a7e4193),LL(0x767f8087,0x1262c90a),LL(0x91770fa4,0x25315a97),LL(0x4bc5606e,0x696dd31a),LL(0x6a665041,0x7ec13746),LL(0xbd01da6d,0xac07c562),LL(0x59de1e2c,0x4bc4a92e),LL(0x8f7d2999,0xcaa3e706),L_(0x0000016a), + LL(0x37e7ccad,0x0f93acce),LL(0x909c5616,0x3802c7e4),LL(0x50e428ab,0x3b424aa7),LL(0xdee4af73,0xc7e0291d),LL(0x5bbec3a0,0x3889c563),LL(0x7854dc66,0xe5ea7c0c),LL(0x7701185f,0x9cdf7704),LL(0x0b88a64a,0x5a2a21ad),L_(0x000000be), LL(0xe5481dba,0xc168583c),LL(0xbb4cbf39,0xec84a971),LL(0x16e6876e,0xdde2891b),LL(0x4940610a,0x9e03560a),LL(0x5406bed1,0x992e1390),LL(0x475b7757,0xc2108310),LL(0x7cc85b3f,0xffbf072b),LL(0x82170451,0x18e357eb),L_(0x000000d8), + LL(0x9d172698,0x0dde3c4b),LL(0xb8a86057,0x4dad7e2a),LL(0x376d08bb,0x709cc4fe),LL(0x2d396e50,0xb38f91df),LL(0x78138716,0x6a45d957),LL(0x7a064e9a,0xeda47781),LL(0xf79bf83f,0x037062a4),LL(0x2aff8e01,0xb537eca1),L_(0x00000160), LL(0x49748577,0xa23c2d1e),LL(0xdb9b468c,0x79fc968b),LL(0x9d5d5807,0xb4f35908),LL(0xae7478af,0x7ffb5a37),LL(0x6f6ac1a1,0x0ad6d095),LL(0x35d076fb,0xcc73da49),LL(0x4e896c83,0xf0b38ddd),LL(0xcd942654,0x1d47691f),L_(0x0000017a), + LL(0xf362300f,0x940a7fee),LL(0x0c8996fe,0xff7970e2),LL(0x434f05d2,0x3ed8edff),LL(0xc3ed10ba,0x5ebc5312),LL(0xdee87d6d,0xa445169b),LL(0x12a8674e,0xc4cceb87),LL(0xd8da7725,0x51c6dc7d),LL(0xea8956f4,0x656977af),L_(0x0000019f), LL(0x7d3585a5,0xa5517351),LL(0x13fe8bb4,0x74f18cb5),LL(0xe68912ee,0x7950c8df),LL(0x1512d84f,0x2fbcbaca),LL(0x7df13019,0x0de4c1a1),LL(0xe29f312a,0xabb943b7),LL(0x656e2b95,0x7ae2e506),LL(0xfc56ddff,0xb10146ab),L_(0x00000105), + LL(0xccd06bf8,0xa717a0ac),LL(0x6ceb8a0c,0xb428946c),LL(0x2bd2594e,0x5c199265),LL(0x80688236,0x3a824141),LL(0x74f2f352,0xb634a60c),LL(0x83ce2e27,0xf4680f2b),LL(0x33426806,0x485e159f),LL(0x18c76ca1,0x28f63537),L_(0x0000011d), LL(0x1f63dec4,0xd74bb74e),LL(0x6f95ccb7,0x4a5f0944),LL(0xc75d0662,0xd36f4555),LL(0xd6e7583f,0x23cc2b28),LL(0x783fa303,0x2c0c49bb),LL(0x4f771001,0x907cd3d6),LL(0xac90f899,0xe2c79e69),LL(0x2cb352bd,0x93baf6c1),L_(0x000001c1), +}, +/* digit=8 base_pwr=2^40 */ +{ + LL(0x93bc2a7f,0x46f49f7a),LL(0xcc0b41a2,0x91072db7),LL(0xd30ac7ec,0x53711f97),LL(0x9de685e7,0xef0acc16),LL(0xb8a2ae9a,0x6bded4c6),LL(0x4497e3c4,0x04789cdc),LL(0xb2b2ea26,0xfea082fe),LL(0xe890cba4,0xb570178f),L_(0x000001c0), LL(0x66edbc5c,0xa472aca4),LL(0x5ad15298,0xa8066050),LL(0x1b16ca97,0x38b0c1cc),LL(0xa2937bbf,0x4d56c3dd),LL(0x72545fac,0x7e35494b),LL(0x4b5790f0,0x903e9ca7),LL(0xb0c8bc40,0xc4b43111),LL(0x5923f9e9,0xed048546),L_(0x000000a0), + LL(0xc18b52bf,0x6410d489),LL(0x28530a4d,0xae318bb5),LL(0x3f7d7028,0xb534b71f),LL(0x0b21b686,0x1f599597),LL(0xf01ea275,0x663379dd),LL(0x800a1322,0x77c11915),LL(0x6db4beae,0xe3261c47),LL(0xe89c22a1,0x7fe9cae8),L_(0x0000016e), LL(0xfce16965,0xd6426762),LL(0x2e53d9af,0x532ec13f),LL(0x1e801ed5,0xe9efe413),LL(0x6963e2f9,0x8aecc3c8),LL(0x5f46e509,0xb47801a3),LL(0xc3d8faa2,0x3b6f3406),LL(0x349f37b4,0xadb57971),LL(0x49ef39f4,0x63bf47d7),L_(0x0000014f), + LL(0xf9dcaceb,0x5a73dec3),LL(0xd887946c,0x0e0ffb30),LL(0xe7e62831,0x84126935),LL(0xe074c83b,0x158a7df5),LL(0x18b04291,0x08eaada2),LL(0xfa20f72a,0x40d05438),LL(0x9aa8c4aa,0x8405e6ac),LL(0xb7559284,0xd94b0b92),L_(0x00000024), LL(0x7981326c,0xf1a037a8),LL(0x13ab2cdb,0x9887e290),LL(0x98bd3d86,0xfecffd65),LL(0x3c803a95,0x2fd8393b),LL(0x3e4c5072,0x129b699c),LL(0x5e3c4e70,0xfa72cdd6),LL(0x65f24da0,0x6c0ccbba),LL(0x5325682f,0xd21b5897),L_(0x000000e9), + LL(0x728d8231,0xeede07ff),LL(0x52ba3f46,0xc57dc8bc),LL(0xbbd28782,0x49d96a93),LL(0x9e0a7a0e,0x49576560),LL(0xe9fbe4aa,0x79dbfb8a),LL(0xcccb4c5f,0xe1789960),LL(0xd25ebfd5,0x09b74da3),LL(0x56df642b,0x37317c2e),L_(0x000001a4), LL(0xd17057a2,0xa759ce4d),LL(0xab9d226b,0x57744ef0),LL(0xb7115a63,0xdddc9ee2),LL(0xd77f24c2,0xdfee8900),LL(0x2142ea1a,0xa9d5346a),LL(0x6d500f3f,0xa84ecd7e),LL(0x7a1527e7,0xae35caeb),LL(0x10e6262d,0x07461c81),L_(0x0000017d), + LL(0xcd457989,0xbd6196a6),LL(0xdd85ca16,0x6f76c2cb),LL(0xcfd847e9,0xaa25840d),LL(0x8ea001b3,0x444e27ec),LL(0xa898be24,0x8a0c53dd),LL(0xb3e4397d,0xfa5f98a5),LL(0x64ea9863,0xbe8e1973),LL(0x922c6bbf,0x4f87cbbd),L_(0x0000015d), LL(0x0664b7db,0x54735a19),LL(0x990568c9,0x7371d65a),LL(0x52b4c902,0x600bdaf0),LL(0xc2cc9668,0x4697299e),LL(0xbadac668,0xeb4949cf),LL(0x33272f7c,0xeda14ca0),LL(0x3989fbe1,0xd9927092),LL(0x0f1714c9,0x09c6f2a8),L_(0x000000a1), + LL(0x00da9fad,0x39d9c06e),LL(0x69b9bebe,0x7878b1c2),LL(0x50e16aa4,0xa0545c03),LL(0x04f7fb31,0x5d57a4d6),LL(0xd233dc43,0x629977c5),LL(0x87e54a59,0xaa747e53),LL(0x0cca577c,0x80698068),LL(0x3aa24734,0xa5c6ac7f),L_(0x000001a5), LL(0xf46ecf72,0x87d84398),LL(0x5b5e3ea0,0xf8e3dbad),LL(0xc29bdf29,0xc86793b7),LL(0x8b4ad4a2,0x337e0dd4),LL(0x34cf9d25,0xc858ea72),LL(0xb282be01,0xe90a676b),LL(0x7590c7bf,0x7d306f50),LL(0x155053c4,0x2ef83915),L_(0x000001fe), + LL(0xfa311b42,0xa08e6727),LL(0x609fc56f,0x6285b2f7),LL(0x07ce1a3e,0xe94807c3),LL(0xc9c1df6d,0x19a317d8),LL(0xd70b9796,0x052a3379),LL(0x870efdde,0xaa7d20b3),LL(0x8f7406db,0xbb6e443d),LL(0x511beafe,0x2e19a355),L_(0x00000177), LL(0xd62e82c9,0x9d51e499),LL(0x9995a224,0x615ca9d8),LL(0xd99162d9,0xab897ac7),LL(0x51034000,0x24f35e95),LL(0xb70ca9d9,0x853be7df),LL(0xff11b526,0x38dc8c8a),LL(0x463b8a66,0x3331fb01),LL(0xb55e7404,0x69bac4e7),L_(0x0000000c), + LL(0x508d4f13,0xdd3ccf18),LL(0xf25524c5,0xd8ab1776),LL(0xe0bf0c9e,0x017a54a5),LL(0x1226e24a,0xb9626be8),LL(0xb0e5b1ec,0x8b7b3bc5),LL(0xf24c6acb,0x14da0130),LL(0x46736054,0x7db2f1d9),LL(0x73af8b9c,0x690b53ae),L_(0x00000140), LL(0xb11a4baf,0xf7b99ef6),LL(0x1e9bb68c,0xb7054990),LL(0xa5ca0071,0xe57cc5d4),LL(0x55009f7b,0x501abab5),LL(0x08dbaab6,0x2d17b21b),LL(0xcde35c58,0x9921c7ba),LL(0x9991c48d,0x3f13fb4b),LL(0xc6f664c9,0xc04b4d72),L_(0x00000100), + LL(0xaf8f0fbd,0x3fffd972),LL(0x513637b7,0xa381ede6),LL(0x3a907a7b,0xef4d7386),LL(0xdec53a87,0xfaf3ac39),LL(0x6072c595,0x7077416d),LL(0x25742340,0x8d4d4598),LL(0x0272fbab,0xc3dce550),LL(0x44d3c246,0xe453c93a),L_(0x000000c7), LL(0x8a45d7f7,0xf5989e00),LL(0xa25dc323,0x31cb7128),LL(0xd19e79bc,0x87f0b5cb),LL(0xf782a69b,0x62e18e62),LL(0x4a3bc664,0xacc62f0f),LL(0x8a21efa3,0x855aaaab),LL(0x5dc442d8,0x895a7f3a),LL(0x7fccf9f6,0xd3bb417f),L_(0x0000012b), + LL(0xf6a194a7,0x21638407),LL(0x8d081ace,0xe465a985),LL(0x1e6d4f3f,0x596aa1dd),LL(0x800bb059,0xf63247cf),LL(0x88ecdd17,0xd80d0066),LL(0xd6196a9e,0x359a8606),LL(0x6d1c0b4e,0xf12ac0e0),LL(0x1f003c05,0x0a4696e8),L_(0x000000d9), LL(0xe591e392,0x88389649),LL(0x09f83a93,0x2b4134e0),LL(0x9d2fd6ac,0x3ada50c0),LL(0xd488e638,0xa2f5e7c7),LL(0x6ae6d5dd,0xece41bdd),LL(0x626ed9b1,0x83fc37eb),LL(0x0ec94ba6,0x390a5c6f),LL(0xd316539d,0x016da834),L_(0x000001fa), + LL(0xf8cf81f2,0xdd03c701),LL(0xce67a0a5,0x957637c9),LL(0x4af6b68e,0x49c7193a),LL(0x2b716eb7,0xa9f1106e),LL(0x04a50d86,0x5cdc8e58),LL(0x29fe3e8a,0x404173b6),LL(0x2217e337,0x8d0fe7b4),LL(0x41f85927,0x04f5e2ee),L_(0x00000085), LL(0xf0033298,0x7982223c),LL(0xeed36f1b,0xc078a101),LL(0xc8a52b8f,0x54b52769),LL(0xfd843c12,0x0c71b06d),LL(0xdaa31445,0xd139607d),LL(0x996c457f,0x3373eded),LL(0x0d6abc25,0x616b57db),LL(0x27a4f9a0,0x2f577a05),L_(0x000000d9), + LL(0x24d46da3,0xc28ba5d6),LL(0x84ca0be2,0x69aff480),LL(0xb7d623cf,0xeee1ba2a),LL(0xc4a065a8,0xe236787f),LL(0xd893b3f3,0xaa351426),LL(0x2106fcf4,0xc4d98be5),LL(0xf2dfc4d7,0x534e82e2),LL(0x4f43180c,0xc094b8bf),L_(0x00000037), LL(0xea92fe7d,0x3cd0971d),LL(0xb9b4d4bf,0x5fa502b8),LL(0x56e42bc0,0x2f95fd57),LL(0x9a55a6ac,0xefd75261),LL(0x9c01cac4,0xc54d4200),LL(0x8b9c411f,0x9a2d86c0),LL(0x84f22245,0x0123f4e9),LL(0x924fe183,0xc4d22790),L_(0x00000014), + LL(0x5adfc431,0xed2fd11e),LL(0x1a785308,0xb3ad4ad2),LL(0x534b1813,0x19e08445),LL(0x77328159,0x557af465),LL(0xcd28509e,0x114e6813),LL(0x908aacef,0xdd6f9e0a),LL(0xea30d82c,0x5aec37e4),LL(0x56efd94a,0xaae09d84),L_(0x000000b3), LL(0x9a808c1f,0x8215d192),LL(0x00e65251,0x2e216a64),LL(0x8be89e79,0xa21b58aa),LL(0x1bae586d,0xde6dc431),LL(0x6074af45,0xd9ffe269),LL(0x144f7409,0x7968f9ca),LL(0x4c70bef4,0x057ee0b0),LL(0x464dfc55,0x96645958),L_(0x00000139), + LL(0xda8f0d55,0xbc3cfc53),LL(0xf7a0b6a9,0xc3851a8d),LL(0xd221e3bc,0x3b6631e9),LL(0x73e218ec,0xd802d5a9),LL(0xbb393674,0x357ad609),LL(0x17e839e5,0x26a2911a),LL(0xfd4ff33d,0xa9163042),LL(0x40c85178,0xaab7ae88),L_(0x00000084), LL(0xebbb0dce,0x628d1685),LL(0xc4b138ed,0xad6058b9),LL(0x1ab4e65c,0xd77f3507),LL(0xa315e387,0x01e25773),LL(0xc1c7fc22,0x5f337f59),LL(0x9dd402d9,0xc4922f4d),LL(0x8947a84e,0x52e76d6f),LL(0x83ef2457,0xdca5e48b),L_(0x000001a4), + LL(0x67dd4533,0xed953e34),LL(0x0ffa9336,0x4fc44042),LL(0xb44d3a69,0x0c1288b1),LL(0x7f745c6a,0x0c5f14a6),LL(0x345f8ac2,0x765ee067),LL(0xcfed50e8,0x659b1874),LL(0x5ef0443b,0x26abda6a),LL(0x894afeee,0x4aa8ff63),L_(0x000001af), LL(0xabe2ed4e,0x46dcead8),LL(0x196272e0,0x64053114),LL(0x13a8b18e,0xbcb0e703),LL(0xf9b6c7a1,0xaecaa246),LL(0xb17e245a,0xd0c5c4d7),LL(0xce6786b6,0x01f4866b),LL(0x12c94128,0xea713e45),LL(0x75975359,0x68aeda04),L_(0x000000de), + LL(0xb900e09c,0xf45b409f),LL(0x7837bf97,0xff4a0108),LL(0x2bcafcbc,0x6b8d204b),LL(0x0da165ec,0x8423a60a),LL(0xb1171697,0xf8295351),LL(0x3eb1f2f7,0x1f58e2d1),LL(0x2b669228,0xbbed8459),LL(0x5f9819ae,0xe7cb925f),L_(0x0000002b), LL(0x7b7ea077,0x53ee2ff7),LL(0x5b359b96,0x98e8334b),LL(0x87baabe1,0x85a52104),LL(0x95a5886c,0xc237881a),LL(0x809649f4,0x7f95c6f6),LL(0xd3395612,0xed6c6419),LL(0x657d29fa,0xa5be49aa),LL(0x7ae0b376,0xd04f1da6),L_(0x000001ac), +}, +/* digit=9 base_pwr=2^45 */ +{ + LL(0x45fb32ba,0x8a721e35),LL(0x5c4674f0,0x584020b4),LL(0x84a774fc,0xadafd3a2),LL(0x477afffe,0x266e1004),LL(0xd6a2c4ec,0x326c6652),LL(0x428066dc,0x0b3a65b9),LL(0x4c7d5c77,0xe355b810),LL(0x4b6af237,0x7bce6d8c),L_(0x00000197), LL(0x1c0b97b7,0xde135822),LL(0xcc7ac435,0x876cab38),LL(0x8f30b09e,0xec654cdc),LL(0xcb3a4f5a,0x26a9da0c),LL(0xb2ac30ca,0x8e2a6fa3),LL(0x77ee1103,0x545c20a5),LL(0xf50fb144,0x97bff8e2),LL(0x58359a6d,0x39ffa931),L_(0x00000194), + LL(0xb2c8ba9c,0xddf8d1b9),LL(0x7f24e874,0x27291ffd),LL(0x563287c7,0xd028bd9d),LL(0xd01bdb48,0x3b0c1265),LL(0x71b99b97,0x618319b9),LL(0xf686050d,0x8420d531),LL(0xc411c3a3,0xaed7c201),LL(0x468eb84c,0x13a48d97),L_(0x0000019f), LL(0xf6eb2fc0,0x1dab9da1),LL(0xc275b73e,0x49847c3a),LL(0x54d322f9,0xf0578805),LL(0xdd0cd2b7,0x4958eafe),LL(0x185bb3e7,0xd9061a48),LL(0x5c6dfcd8,0xf9ac370d),LL(0xa0217866,0xf54cb188),LL(0xa132c3b5,0xec13457a),L_(0x000000b7), + LL(0xf197825d,0x1340276a),LL(0x4bbcc96d,0xd82fe632),LL(0xcad6233b,0xc290475e),LL(0x0cd8d04a,0x738cce9a),LL(0x8e8e067d,0xaa038ad0),LL(0xd83e4317,0xa7ce55aa),LL(0xd5e91f49,0x856a1887),LL(0x5efeae92,0x16577a13),L_(0x0000013c), LL(0x9bfa0b6a,0x3d153ead),LL(0xef7bc585,0xca7f6fb4),LL(0x0b798e2a,0xf8abfbb3),LL(0x53595cf1,0x79182066),LL(0x1774e7d1,0x862d3928),LL(0x8b4548df,0xdb1e4086),LL(0x6e38fc52,0x72153b33),LL(0xe2e4b80e,0xb610b52c),L_(0x00000006), + LL(0xf5595043,0xe1b752f3),LL(0x1b9318d9,0xf6e2b364),LL(0x5c02bb70,0x38d64e0f),LL(0x9d8f2870,0x07542416),LL(0xa62f3a1b,0x3b8c6755),LL(0xd59701bf,0x2b642127),LL(0x20fbe8ba,0xfac17f0c),LL(0x3410177d,0x46466526),L_(0x000001b8), LL(0x2b08cc56,0x70680750),LL(0xe532cef6,0x7a1e8980),LL(0x29a4a8b8,0x3b679637),LL(0xcb3a4f19,0x0043db7c),LL(0x92e07ae8,0x346fea83),LL(0x0da35be0,0xef33f7a0),LL(0xcb41f4e9,0x271ea778),LL(0xbb760e77,0xcbdf796e),L_(0x00000018), + LL(0x120e5ac8,0xcad14e90),LL(0xd45b7941,0x130b3936),LL(0x78bbd634,0x3839fe90),LL(0x8f94ae22,0xfb2c2b29),LL(0xbd4d9761,0xb2caaa91),LL(0xf6e513d3,0x37bd3dff),LL(0xa0f24baa,0x9dd2846a),LL(0x1d27a8db,0x025f7ece),L_(0x000001d1), LL(0xd4e2cdab,0x8296eec9),LL(0xee13214a,0xce1e6780),LL(0x6fed4902,0x8ec28ea6),LL(0x28576525,0xa9bf0652),LL(0x0afbfe7d,0x0c66edcd),LL(0x9e743eb7,0xc8ec4a8a),LL(0x64589360,0x09bf2d23),LL(0x7a6453a2,0x48ef097a),L_(0x000000c5), + LL(0x4d44bd26,0x5e7c9a8b),LL(0xfa441738,0x3f4fd525),LL(0x8cdf278d,0x5b1fa4df),LL(0x60600772,0xb7e79779),LL(0x15388443,0x6b7719f4),LL(0xd7a3aeca,0x17dd158d),LL(0x02441c0d,0x3d070ec1),LL(0xd5eb5d02,0x8264d6c9),L_(0x00000123), LL(0x0ab898cb,0x01117e64),LL(0xee325365,0x6f680374),LL(0xbc1ae420,0xdaebee10),LL(0x98a23bbf,0xfec8e51b),LL(0xb59efdf3,0xbbf08b12),LL(0xa18137ff,0x1532459f),LL(0x04b7fdbe,0x60238519),LL(0x37b3447b,0x4a54c60e),L_(0x00000108), + LL(0x6b53a82a,0xad2c8749),LL(0x5a5a47d3,0x75f76d03),LL(0x60558c44,0x6ecf38ff),LL(0x957fd8a0,0x7695311e),LL(0xcd47da64,0x215ee9fe),LL(0x35b22e22,0x4796f4b7),LL(0x949a56db,0xf62c912e),LL(0x74debc0c,0x62bb4591),L_(0x00000058), LL(0x9bd8df8c,0x203b317a),LL(0x637e055c,0x03c45bfe),LL(0x90fbadef,0x1132b50f),LL(0xaf36e7bf,0x20a98c58),LL(0x4f36088b,0xdebbd429),LL(0xcbb98ba8,0x391e4230),LL(0x3091f3e1,0xb3356938),LL(0xd86355bd,0xb1556872),L_(0x000000ab), + LL(0xf79ba658,0xb48e1df3),LL(0x3eb15b18,0x5fc03a10),LL(0x3bed592a,0x3591ad26),LL(0x127b78a3,0x07e9d80a),LL(0xc0337c7b,0x349dd74f),LL(0x364ed2a0,0xb1a807c5),LL(0x588d4203,0xecd92cca),LL(0x772a1716,0xf28c1d9e),L_(0x00000183), LL(0xf6fc1df3,0x42d25980),LL(0x8922f157,0x36f0fdb0),LL(0xa583206a,0x8cc1fe47),LL(0xc73f8816,0x1d279801),LL(0xe1b77767,0x7ac8979c),LL(0x3dba6831,0xa98b4836),LL(0x60d40152,0xc7f36b74),LL(0xc3d46c62,0xde52bda1),L_(0x000001ec), + LL(0xed4a0395,0x99b5cc1c),LL(0x4cddc23e,0x1d30267a),LL(0x16bee440,0xcd4130db),LL(0x553abd41,0x6652be0e),LL(0x6e659595,0x22061ff2),LL(0xf0c20235,0x72c720f6),LL(0x077f6daf,0x8079b1de),LL(0x1ad9ac77,0x52a9e551),L_(0x00000074), LL(0x6701fea0,0xdd8e0cb1),LL(0x5849b249,0xf395a61b),LL(0xb92466c3,0xc2b702c7),LL(0x77432a31,0xbd7899d3),LL(0x28b4ebc6,0x307f0a10),LL(0x0b06f919,0x5c8246fb),LL(0x7154af20,0x8f032be2),LL(0xc88de5c5,0xd4f96409),L_(0x00000076), + LL(0x344eafa5,0x0a16f77b),LL(0x724f29ca,0xdbafe962),LL(0x94bbb419,0x30985479),LL(0x2b2c87d2,0x3775b2ba),LL(0xe0e3814b,0xbd366c77),LL(0x1130e80c,0x7b644025),LL(0xf10ea539,0xe1da2161),LL(0xf66677b2,0xcab81415),L_(0x00000060), LL(0x11454e50,0xd8ab26fa),LL(0x45948446,0x2a4b8bd4),LL(0x35518731,0x34c59cba),LL(0xcc005baf,0xbd4d3f49),LL(0x06c483a0,0xa3e5d238),LL(0xd77da187,0xc4657e79),LL(0xa31fff1d,0x33918629),LL(0x0e898785,0xce7defc6),L_(0x00000010), + LL(0xd39844b5,0xc0ae95fc),LL(0x0cd04d32,0x608fd1bf),LL(0x2b33bcf9,0x8e195302),LL(0x3567e13c,0xb9784d4d),LL(0x6f12914d,0xf39a6a6c),LL(0xf4d361ba,0xcf170781),LL(0x366e62a5,0x70b10e90),LL(0xa3bce706,0x4f54bbdc),L_(0x00000091), LL(0x65a7fa0b,0xdf710618),LL(0x93abe742,0x7805a257),LL(0x738295fd,0x76e1d4b4),LL(0xcb5b0f15,0xc121708d),LL(0x716ee171,0x14725b57),LL(0xd2227241,0x2e484d37),LL(0x34400369,0xef0bb7f5),LL(0xebdf59e5,0xe564b505),L_(0x00000082), + LL(0xf35d7da4,0x0ee635c0),LL(0x6c91936e,0xdd72a103),LL(0xa9f8eae7,0x2a073b1f),LL(0xff539491,0x6c35942b),LL(0x0a881a03,0x35498b7c),LL(0x67e4af9a,0x59bde411),LL(0xf903d1e5,0x517835ca),LL(0xf0b93b5b,0xf4238664),L_(0x000001ed), LL(0x079d614c,0x550a47c5),LL(0x1c8515d3,0x1f9595ac),LL(0x1557c55d,0xf301c894),LL(0xb5548c2e,0xecc6608b),LL(0x6ed92475,0xf17244f1),LL(0x9b9d35aa,0x9b6083ca),LL(0x82abcca4,0x902eead4),LL(0x45a99fbc,0x489ec5a2),L_(0x00000074), + LL(0xc92a2f72,0xb4d59736),LL(0x46a97747,0xec9ee773),LL(0x92e9e427,0x54eed174),LL(0xe62769e0,0xb25c6252),LL(0x26eca3d7,0xb5598a2e),LL(0x72728c2b,0x73ee8036),LL(0x6cefdf35,0x4ee8ce4c),LL(0x700d3d8f,0x80153dd7),L_(0x000000e7), LL(0x2ffb5bc1,0xb8175b8e),LL(0xd9c451df,0xbdb5cc88),LL(0x9445c144,0x846b2eaf),LL(0x92957da0,0x5ff2e582),LL(0x2da50816,0xe7cc1a15),LL(0x4dc70abf,0xe4999b07),LL(0x24220cc0,0x1b3556b4),LL(0xb4413c1d,0x112c52e6),L_(0x0000006a), + LL(0xb5ee8957,0x45878f54),LL(0x112fbfa1,0x1879f721),LL(0xe9f0dfae,0xb007f421),LL(0xf113817d,0xeb000fbc),LL(0x35d8e979,0x206151d8),LL(0x0bf9caf2,0x258ab3b7),LL(0x6e8e8e3a,0x92a042db),LL(0x7a4dc496,0x53dcf8da),L_(0x00000112), LL(0x3d64ea94,0x7360c36c),LL(0xbf2b13c7,0xfb77c37c),LL(0x73884c74,0x65a78a55),LL(0x5d8600a0,0x888762bf),LL(0x77475414,0xc8ba0daf),LL(0x975e6be1,0x59f8b668),LL(0x14cf6707,0x185c7c67),LL(0xfef650be,0x043f0a23),L_(0x00000051), + LL(0x14dc97d9,0xa3e0a482),LL(0xe962fe1d,0x44364f6d),LL(0x19480b73,0x9ffa10f7),LL(0x28fc88ac,0x7993eaa2),LL(0x8a5db808,0xd4bb9db5),LL(0x4464dfad,0x9088a081),LL(0x903605db,0x86f98ca4),LL(0x87bd4fc7,0x5fb11fcc),L_(0x0000014a), LL(0xba5ec771,0xec2c3e51),LL(0x3078a6cd,0x1ad83e79),LL(0x66717c17,0xad871d3e),LL(0x8530527c,0x0e3f9442),LL(0x92315ca1,0x49c67cb7),LL(0x2fc5cd79,0x4eb1ba39),LL(0x256788a6,0x10b0e6f6),LL(0xb9cd18a5,0xef51548b),L_(0x000000c8), + LL(0xd61fb046,0x90213473),LL(0x4f9db0e1,0xcbb6e9b8),LL(0x36fcff78,0x6aa8fb8a),LL(0x7cd5e9d1,0x337a00c4),LL(0x2c2601e9,0xfe8445d7),LL(0xbbab713e,0x0681fd15),LL(0x0b2dd233,0x2151cff9),LL(0x00ab444b,0xf5654249),L_(0x000001cf), LL(0x06de9a88,0xcb8ede52),LL(0x209abe3b,0xe1369e32),LL(0xb711e224,0x53136516),LL(0x533569db,0x59d96525),LL(0x5419656e,0xf2d68025),LL(0x326eee21,0xd59bb004),LL(0x073cca71,0x1cbb722c),LL(0xaa784f93,0x50513866),L_(0x00000019), +}, +/* digit=10 base_pwr=2^50 */ +{ + LL(0x51931359,0x672b4b0a),LL(0x14acc3b3,0x78ea42e6),LL(0x22fe0a9a,0xe72784cd),LL(0xc20faf43,0x8f9c3ea6),LL(0x5e49f303,0x4c50987c),LL(0x12d1fb91,0x0c76e9b9),LL(0x96a89b90,0x74dc2b7b),LL(0x238b29a0,0x3a4808de),L_(0x00000076), LL(0xca68ea37,0x06adb168),LL(0x5ecbae96,0xe58dde88),LL(0x4d422e92,0xeba17742),LL(0xa609937c,0x1451998a),LL(0x8f30fc81,0x9eba807b),LL(0xa724c9f9,0x200db6e7),LL(0x651c126e,0xc9db2dc7),LL(0xb58e38f0,0x63ee509c),L_(0x000000e0), + LL(0x71e8870b,0x8ea654a6),LL(0xa23dd690,0xc3eb3660),LL(0x673dbdf6,0xa5ddaf70),LL(0x9bbf5d38,0x1e7af5c1),LL(0x0fe1371d,0xcc1eff61),LL(0x1572e30b,0x1308bdd3),LL(0x20ce33cf,0xc60db70b),LL(0x6ab6b3ed,0xbf718e03),L_(0x00000103), LL(0xae357b86,0x3ce6e16b),LL(0x94e06b89,0xa3849b8d),LL(0xb6058ad8,0xacee1675),LL(0x6add0f99,0x39df12ed),LL(0x43cd380c,0x5c645ff1),LL(0x0481e233,0x94a0f618),LL(0xc84b4bf9,0x805a52a4),LL(0x49a710f4,0xe454ce98),L_(0x000001a9), + LL(0x99d73698,0x68ebe9cf),LL(0x7fcd4216,0xc625e525),LL(0x4922e8d6,0xe579cc68),LL(0xe272485c,0x58eef2df),LL(0x1aedb9d5,0x6bba0e47),LL(0xf69dbcc2,0x6afac0cf),LL(0xd8f85c14,0x4dfdd56a),LL(0xc7e717e4,0xfa08e4f2),L_(0x000000ae), LL(0x7d4e9483,0x056cb0ab),LL(0x8a2580c9,0xee676f9d),LL(0x031109c0,0x0e2ecd89),LL(0x784c6d24,0xedf27261),LL(0xeac131cc,0xdb6b9edd),LL(0x9428ee22,0xf59f93aa),LL(0x90347b1c,0xd59691aa),LL(0xcb3849d1,0x0b74214c),L_(0x000001ca), + LL(0xc42ea299,0x33ae1a32),LL(0xd0ddacf9,0xb43b79b9),LL(0x30561bd9,0x0ad2636c),LL(0x12241370,0xd830def9),LL(0x85a779a0,0xda5f6561),LL(0x28b8580f,0x7e785d86),LL(0x8bafa8c6,0x48ce8b18),LL(0xc75df63d,0x746f16d2),L_(0x000000df), LL(0x5a90afd2,0xc72b304c),LL(0x1b4b2e57,0x40d7dec2),LL(0xe0f45d07,0x3eb94cfd),LL(0xaabbfa71,0xae1b3f10),LL(0x37fa8b4f,0xb080d24d),LL(0x6f6447d2,0x142abdb3),LL(0x20453501,0xfd470df7),LL(0x76e433f8,0xc036f47e),L_(0x000001af), + LL(0x1f809e00,0x9eb8b4e3),LL(0x91e1d4a1,0xa399e369),LL(0x9b1aa8fe,0xf15f9651),LL(0x80a83b4c,0xea343c7b),LL(0x1c2fb2b2,0xc40680cb),LL(0x4d003567,0xe7a338fc),LL(0x65bc46fe,0x4519127e),LL(0x3a269638,0x40c08630),L_(0x00000163), LL(0x8811cc38,0xcd6861c0),LL(0xd2e2abb7,0xccb0e7ea),LL(0xce5461a0,0x8c05450d),LL(0x28a458ea,0xeff9ba00),LL(0x51ce8e58,0x3e543072),LL(0x41ebfad1,0xa43fc5d6),LL(0x2acf8a4f,0xc0d63fae),LL(0xd16efc25,0x156cea94),L_(0x00000000), + LL(0x0c31be44,0x25c21bb4),LL(0x171af22e,0x91e02b25),LL(0xa0756859,0x87db4292),LL(0xd07cf03c,0xd52aff6a),LL(0xf2199b54,0x476b6c0b),LL(0x4c50edf5,0x1bd465b7),LL(0xdeb36507,0x9e6301c1),LL(0x957f58a4,0x0a904754),L_(0x00000067), LL(0xf73b742f,0x5e9d3550),LL(0x6b92e894,0x419be8da),LL(0xd785e55e,0x95d412eb),LL(0x6018e5bc,0xadbd35ab),LL(0x079447ae,0x5f3359bd),LL(0x21b9bd0a,0xc4db3315),LL(0x5774802e,0x4978d805),LL(0x18a2368a,0x662a2d67),L_(0x0000011c), + LL(0xd53479d6,0x5c26f234),LL(0x2d429971,0x107f7e92),LL(0x8c689924,0x334d8841),LL(0x2a5fa3eb,0xe5ebe430),LL(0xc519e325,0xe8291ba7),LL(0xf2242ce5,0x8a0c19be),LL(0x20419cdc,0x804a91ee),LL(0x70dcad32,0xacb0db7f),L_(0x000000c4), LL(0x78a46e2e,0x18b297a6),LL(0x21fc2dc8,0x3ba036e4),LL(0x30517e2f,0x7a021835),LL(0x49f89605,0x19710681),LL(0x84156ac1,0xd61e5109),LL(0x05c42243,0x31ade9f9),LL(0x7b661ab8,0x83c25735),LL(0x22eb398b,0x6193abdf),L_(0x0000012d), + LL(0xad081cbd,0x1433b543),LL(0x88d8cd2c,0x94641d24),LL(0x2da0394f,0xd8e36e70),LL(0x48288ca4,0x461fe782),LL(0xa112c8a6,0x6f063613),LL(0xb8624a48,0x77efb66b),LL(0x511d90ff,0x016e8d41),LL(0xce809694,0x5bb229a1),L_(0x000000d0), LL(0x36feced9,0x3ecdac71),LL(0x921f42e8,0xee8e2857),LL(0xe82b293d,0x2c3ef9bb),LL(0x182b25ab,0xac32f4bd),LL(0x297ad819,0x74b598de),LL(0xdd15916b,0xd5e666a5),LL(0x51456a24,0x447be0b1),LL(0x4dc25c5c,0x8726ab79),L_(0x0000003f), + LL(0xb1762839,0xba507049),LL(0xed038901,0xdcef710b),LL(0x4b349ec8,0x2489f866),LL(0x37b4ec4e,0x991460d8),LL(0x94e1cecc,0xbf2a63d2),LL(0x33d105a8,0xc7e7415f),LL(0xbf883b5c,0x268241cd),LL(0x2f565fda,0x65d5bd35),L_(0x000001f3), LL(0x8bf3904a,0x8e823f54),LL(0x0c2d77f6,0x06de5eb9),LL(0x2c00d580,0x89b51b4d),LL(0x41ce4b94,0x794caf3e),LL(0x177cd9b2,0x7c62716f),LL(0xfe0ae88a,0xb7e50074),LL(0x4d023907,0x49a489fb),LL(0x545f8faa,0xe82852b9),L_(0x00000070), + LL(0x05813dea,0x62545e9a),LL(0xc84039c9,0x70606ec0),LL(0xcdf6907b,0xec7e8e9b),LL(0x2e4e87c0,0x9d6e053f),LL(0xffa08764,0x22a2e351),LL(0xe5b305f1,0x95345fe3),LL(0x65c90711,0x4f24c950),LL(0x139d472c,0xbf685d44),L_(0x00000179), LL(0x9eeab46c,0xac8b67e8),LL(0x267bcc83,0x702c21e5),LL(0x0dac9b29,0xce390fe4),LL(0x60429071,0x6ef71376),LL(0x4a80e0d7,0x47100322),LL(0x0ef6a473,0xdc625a85),LL(0x759024e7,0xea01db5d),LL(0x1e4722a2,0x155020a2),L_(0x0000014b), + LL(0x5678bfca,0xb048b1b3),LL(0x5bacba68,0xf518ba8c),LL(0x1626088b,0x7054f024),LL(0xa686c886,0x933a9118),LL(0xbb623954,0x1c3c471e),LL(0xc4da98f0,0xf1b8c9b1),LL(0xa0619dd5,0xaeebf226),LL(0x24b28dc5,0xcbe9fb08),L_(0x00000092), LL(0x47814012,0x49c3a34f),LL(0x1cf06d59,0xfcdcc300),LL(0x6d4a798d,0xe86df54b),LL(0xa1a4dd57,0x1534b80c),LL(0xaf606d64,0xbdfde769),LL(0xde1cbaa0,0x649c3a2a),LL(0xbf6c9950,0x763574e1),LL(0xaaf6f737,0x7fdbd339),L_(0x0000015a), + LL(0xf687c377,0xca314119),LL(0x3eacfd33,0x2512d094),LL(0x9c0e1850,0xe55f9fd6),LL(0xc3c6ea7e,0xc20685b7),LL(0x66291556,0x4868b07c),LL(0xb5895337,0xf9f339d7),LL(0x9238a109,0x75d6855b),LL(0xac6af37f,0x78b54491),L_(0x00000184), LL(0x6eb5d5b2,0xe7603bff),LL(0xf7552855,0x8f73b087),LL(0xc19b7320,0xe8f5c0ad),LL(0x55df5442,0xb6aeabd3),LL(0x3a4b8876,0x8dc2b22b),LL(0xf8bca737,0x26f89265),LL(0x3dbb040c,0xfb6645f2),LL(0xb09ab1bb,0x30259f38),L_(0x000001e9), + LL(0x9e2dc755,0xea8b03a8),LL(0x5618c490,0xd5d01455),LL(0x01a7a348,0x9622ab8a),LL(0xa6b5c4df,0x9adea853),LL(0x303519de,0xa9b99058),LL(0xb3d0934b,0x0fbd9ea2),LL(0x2cdee030,0xe856d6fd),LL(0xa351d2a2,0x89fdbc8b),L_(0x000001d3), LL(0xea40ff5a,0x4859e663),LL(0x906f2d7a,0x71904b77),LL(0x411180a4,0xbd7ebd35),LL(0xe50b9460,0x0ec190c2),LL(0x2e7f4d73,0x4c9e4aac),LL(0x76a98ae4,0x4323017a),LL(0x7f0e29c1,0x22ea8f39),LL(0x31c71758,0x89d8efe3),L_(0x000000d5), + LL(0xcf98bee6,0x54f3df55),LL(0x672d0a69,0xe9759866),LL(0x1addc9d4,0x9c17b622),LL(0x6c819f7f,0xc42650ec),LL(0x6b1209c4,0x6a1aa1b9),LL(0x2b341fef,0xdbcf91bf),LL(0xc99d2b99,0xed76cdf2),LL(0x27467cd2,0x05f190ee),L_(0x00000008), LL(0x64754f19,0xc40615cf),LL(0xb8f1e46f,0x9a8d5587),LL(0x540f1fa5,0x804f7dd7),LL(0x21752096,0x2c95388c),LL(0x9444e15c,0x133319bb),LL(0xb1d5a817,0x29552f4e),LL(0x79fc1cba,0x93730e70),LL(0xcae8a131,0x4445951a),L_(0x000000dd), + LL(0x88863b50,0xbfdb676e),LL(0xb9545954,0xfcfc0194),LL(0xe74bedd7,0x888694d2),LL(0xe59a14c4,0x236680d1),LL(0x4cd674c0,0xacdf13c1),LL(0x52151e94,0xcdbecfcb),LL(0x6a28bc34,0x641d77e2),LL(0x6293af48,0x38e4eee3),L_(0x00000057), LL(0x2e8f361b,0x67004141),LL(0x39634681,0x5db1f02f),LL(0xf975c602,0xe645bd3a),LL(0x8b39a53a,0xfafccb60),LL(0xa58e37f9,0x33ab2637),LL(0xcf611fd4,0x8b8cc6bb),LL(0xe7f89e7f,0x28eb10f6),LL(0x5f527820,0xfdcde1d1),L_(0x00000151), + LL(0x9411ca0d,0x92267e14),LL(0xb385c8ea,0xbbfcc2ab),LL(0xbfd56d29,0x34b29656),LL(0x5f2180a7,0x06f72807),LL(0x6dc34000,0x02310437),LL(0x854af754,0x1bae73e6),LL(0xbc753242,0x06a8d2dd),LL(0x11770a34,0x9848b3d7),L_(0x0000011f), LL(0x14476594,0xeb8cb497),LL(0x6ba99aed,0xc86324ad),LL(0xc49863ca,0x8a316428),LL(0x2e5cfc3d,0xcb62d82a),LL(0x79adc3e0,0x9e5f3fda),LL(0xcaddeff7,0xb4f990b6),LL(0xae15a98e,0xb9b0e410),LL(0xedf394c7,0xac6cb00a),L_(0x00000171), +}, +/* digit=11 base_pwr=2^55 */ +{ + LL(0x20c391c2,0x96d1c5f4),LL(0xeaef76b3,0x6bb17f5e),LL(0x7feb16a1,0x3f16a57b),LL(0xcc801552,0x4aadf126),LL(0xcded6e6d,0xe23393c9),LL(0x6848f602,0x2c8dbcb3),LL(0x49f3a9ae,0xf811e23c),LL(0xc0c1ebfa,0x27162730),L_(0x000001e5), LL(0xaf1b88cd,0xb4c22029),LL(0xaa7f22fc,0x9624d6d5),LL(0xbb120735,0x416db935),LL(0xa8308449,0x85fd3219),LL(0xc467f9f1,0xb4d3e00f),LL(0xa69d57d8,0x187052a8),LL(0x0528c91e,0xb79e6638),LL(0x2a603bc9,0xb4ac37a4),L_(0x000001ab), + LL(0xca26efe1,0xa3ad38a0),LL(0xf5cd529d,0xec34abea),LL(0x94808b1e,0x27c847ac),LL(0x87ade961,0xfa6df215),LL(0x6a43fa8c,0xbcfdb5ad),LL(0x947fbb39,0xdd4d0c9f),LL(0xbca687c5,0xe8772a4e),LL(0x7d79e215,0x9fc12922),L_(0x0000008e), LL(0xbf926e1c,0xb04fbc5a),LL(0xb9c12ffd,0x34707ba5),LL(0x4ee8c89b,0x81aa347c),LL(0x367a152d,0x4cd56572),LL(0x74511a3a,0xa6642939),LL(0xd0e3b8f1,0x60ea13e9),LL(0xee14ab42,0x81a19a28),LL(0xea76ba4c,0x96065d5d),L_(0x00000103), + LL(0x6b0c75c4,0x23a4b0aa),LL(0xdb181c23,0xdc940ab7),LL(0x7b70983d,0x328a76b8),LL(0xd5b473a0,0xadcb9bcd),LL(0x3863dc05,0x646b4949),LL(0xe5090fd0,0x0b996e3d),LL(0xd0261360,0x7c332b20),LL(0xcc6b2f86,0x415e4fc7),L_(0x00000009), LL(0x28cd5819,0xf08cb2c4),LL(0xfbdf661a,0x9b1c2455),LL(0x2be7d7be,0x38fbe0c1),LL(0xbd91e037,0x84e69e29),LL(0x1cdba496,0xc6f94abf),LL(0xa8445728,0x8e9508e3),LL(0x4a144f07,0x8ee0e340),LL(0xc5b72f6a,0x80119066),L_(0x00000075), + LL(0xaff1e1b3,0x87421ab9),LL(0xbbfcc6da,0xa3305ebd),LL(0x4b75a8e1,0x8f4cb778),LL(0x4410056f,0xb5abdc6a),LL(0x4ff65612,0xd83f32f5),LL(0x21c44b1c,0xb989d251),LL(0x80a7bb1f,0x5214abed),LL(0x8f200e11,0x5e3475ba),L_(0x000001fc), LL(0xf4fb8525,0xf5f23c02),LL(0xaa8e02a4,0x405911d9),LL(0x45abb8b1,0x7a6dae03),LL(0x4834d14e,0x4621957b),LL(0xeb31fdc5,0x7cbf9b75),LL(0x26ee5dca,0xee84304e),LL(0x37349cc1,0xcc6a2c7d),LL(0x5a34c3af,0x740fffc8),L_(0x000001ac), + LL(0x9f122c9e,0x71cfc92d),LL(0x9c6ec42c,0xb86c84b7),LL(0xc8d12bf6,0x1c821b85),LL(0xe8432cc5,0x197e0f04),LL(0x4258bc34,0xc4f03c70),LL(0x60ae518c,0x811512ff),LL(0xf050c9b8,0xe8038335),LL(0x0b215595,0x84b70af0),L_(0x00000078), LL(0x423ac4f7,0xb15c3155),LL(0x2fd13662,0x7684c454),LL(0x5cf8078d,0x1a7bfb14),LL(0x2b928e98,0x1d05b843),LL(0x3bbf2a85,0xeeb1e658),LL(0x356da90c,0x179bc7a9),LL(0x11d26c87,0xf524843e),LL(0xf4159e0d,0x47c538ec),L_(0x00000128), + LL(0x08cefac8,0xb1885068),LL(0xe8422939,0x5985dd6e),LL(0xab14cf0e,0xcda94a64),LL(0xc27af983,0xd127851a),LL(0xf24f6eaf,0xbab20f8f),LL(0xda3b25d8,0xa549d9c6),LL(0xed810bd4,0x5bf18f37),LL(0xf630e4c9,0x14bb2655),L_(0x0000018b), LL(0xae18594e,0x52697460),LL(0xf8de9d89,0xaec56660),LL(0x294777cd,0xe3a93a39),LL(0xf7dc98fd,0x63fcc0bd),LL(0xc0c53dc3,0x5d2c2708),LL(0x55da9198,0x3692d050),LL(0xebcde249,0xcdc4d312),LL(0x8d0017f5,0x5aeda0c1),L_(0x000000a3), + LL(0x1d9471ec,0x737ba42a),LL(0xd3401c6a,0x33fd5fca),LL(0x3c2758a5,0xb016b83f),LL(0x79b1c693,0x3409d0d4),LL(0x5f80d853,0x4f2a7005),LL(0x4d1634ee,0x799e106c),LL(0x1e92ef70,0x632dcd5e),LL(0x86232632,0xb8cf2e87),L_(0x00000008), LL(0x1acab1e2,0x92c31a31),LL(0x91455009,0x740223dd),LL(0x15a695ed,0xa95f341b),LL(0xe601b98b,0x17db74b3),LL(0x19ccbb77,0xd916a466),LL(0x44573d27,0xc31a7a19),LL(0x093c0363,0x1bb20e06),LL(0x6715c5f0,0xc4532de7),L_(0x0000000c), + LL(0xa1a3f86e,0x2df7ec8e),LL(0xf1f8f7f3,0xd8551991),LL(0xb16ec397,0xbc80f4ee),LL(0xebe5be1a,0xa1e6cbf5),LL(0xaf8233b8,0x5c403702),LL(0x41483767,0xbf97ecb0),LL(0x2899a5cc,0x58655568),LL(0x0720d399,0x092de028),L_(0x000000bf), LL(0x88312054,0xd550df72),LL(0xf87e274d,0x193eb1e2),LL(0xa715c43f,0x97773656),LL(0xcb67dce2,0x8a585c6e),LL(0x0aacb5db,0x6332fcd1),LL(0x4f16d92a,0xdeebccba),LL(0x2b8001ac,0x8936c8da),LL(0x7b627657,0x4f5f2781),L_(0x000001b8), + LL(0x8d63a794,0x35fd304e),LL(0xdd225fc9,0xa6aae952),LL(0xc40c9b7f,0xd5054f16),LL(0x42316d8c,0xf663b3dc),LL(0xb3d7abe3,0x13c94097),LL(0x7aa82bbf,0x78263190),LL(0x2a622ce8,0x819c0b14),LL(0x2b1dba5c,0xcf88a41a),L_(0x00000049), LL(0xe69bb850,0xdf7bd8ef),LL(0x985f2ed6,0xc76c2599),LL(0x44f156c5,0x46e2c0c0),LL(0x7cfc49ae,0xf5fb07c3),LL(0x6f59a7a0,0x2f48e451),LL(0x1b89eefd,0x88119cbb),LL(0x1c41ec61,0xa18666db),LL(0x53014a3d,0x4b3591f0),L_(0x00000078), + LL(0x792d6d08,0x4d84ecf1),LL(0xe0110c24,0xc93fa7f1),LL(0xc72b1bb4,0x908f695c),LL(0x1730f1b2,0x8d0bc692),LL(0xdb0b36b3,0xe4bf469c),LL(0xa1db29c0,0x1d41428d),LL(0x7a577f2c,0x2cd1253d),LL(0x23b65522,0x07ab206a),L_(0x0000009c), LL(0xa4ba5fbd,0x9808ec8a),LL(0xa5383520,0x49718327),LL(0x2c210a5f,0xdc5bb249),LL(0xef53e1db,0x7e38e02e),LL(0xc9d3c171,0x7b41e983),LL(0x3a07d487,0x2d8aedea),LL(0x6c0e3ba1,0xa17e058b),LL(0x22c8be6d,0x7c63dfd0),L_(0x000000fd), + LL(0x6a5713b6,0xac6235d1),LL(0x19855a0f,0x32d1869e),LL(0x093a8212,0x8afdb213),LL(0x89861196,0x3402ba32),LL(0xb3676c48,0x5e54b89e),LL(0x53597329,0xbdde3064),LL(0x94cdc873,0xc3d273b6),LL(0xfd911ed5,0xa0110df1),L_(0x000000a2), LL(0x0d98f860,0xf6cf3683),LL(0xa681e586,0x6f5c1e3f),LL(0xc6905825,0x7d626d06),LL(0x571b75e4,0x00a44322),LL(0xf9fe1aa4,0x34ece73d),LL(0x3975b815,0x38add31a),LL(0xfa3db092,0x499ecb33),LL(0x2ce86fab,0x9eb9f7bf),L_(0x000000cb), + LL(0xf5870ab9,0xa353002c),LL(0xa12da044,0x6150f34f),LL(0x0086b83e,0x69e6eea2),LL(0xa2cdf131,0x5e80e0a0),LL(0x528616b2,0x2d13e0cc),LL(0x4a67c598,0x9702e01a),LL(0x83d6e661,0x15b60ef1),LL(0x6f9172f8,0xd6247a1f),L_(0x00000115), LL(0x7b2b5776,0xe6acc547),LL(0x055811a0,0xba422b24),LL(0xa9873020,0x8c990991),LL(0x310acf2c,0x96459d45),LL(0x78701ea7,0x917c30ec),LL(0xd1688c83,0xdb51be44),LL(0xb42ce9e9,0x0b514c3b),LL(0x0b03fd87,0x0c8ed48d),L_(0x000000c7), + LL(0xbcc82868,0x69816459),LL(0x580f7a11,0x9b94ac07),LL(0x11b4de1d,0x120451de),LL(0x8f21a7aa,0xc048b454),LL(0x0f6b490e,0xca8d647e),LL(0x5d0f4e1d,0xf1f7c090),LL(0x3e12d889,0x1ad27c80),LL(0x5b341256,0x381024e5),L_(0x00000161), LL(0x35f1970a,0x4366eed0),LL(0x1134e984,0x55c0352d),LL(0x7ea259fe,0xfad7d83b),LL(0xda4dcbce,0xdd5f6008),LL(0xb2924c78,0x01b25214),LL(0xac404086,0xf325f997),LL(0x2b613948,0xf37e21a0),LL(0x26e31be0,0xf87b2cb4),L_(0x000000bc), + LL(0x017edbd6,0xf483d72a),LL(0xb08491c6,0x58a225c5),LL(0x568a7e71,0x7fde8697),LL(0x821bf73d,0xef4bc022),LL(0xec765e3a,0x8d1daf2f),LL(0xb59a1d2e,0x72d486e7),LL(0x1edfc037,0x2a595f95),LL(0xf1683f88,0x9376cfff),L_(0x00000081), LL(0x55fc5381,0x96f30cc5),LL(0xc6ce2141,0x76a3af64),LL(0x339f5668,0x5449bfff),LL(0xe438adb5,0xf3c48dff),LL(0x1aa59ae8,0xce59b544),LL(0xc0fd6c57,0xb7bdc7b7),LL(0x8e51d10e,0x973b8e1d),LL(0x6427d578,0x99827d6a),L_(0x00000098), + LL(0xe76cf424,0x4eac44ab),LL(0x559e7a5a,0x0ddf44f2),LL(0xc58d75d8,0xfb0d499a),LL(0xab62039f,0x6cf6c677),LL(0xd4e76825,0x2e427953),LL(0xa955fdca,0xe1d73f88),LL(0x049f7f5d,0x89dc4a2e),LL(0xd5493485,0x3b5c3f5a),L_(0x000001ec), LL(0xa5dc86bb,0x2a75769b),LL(0x606d9e57,0x550fb22b),LL(0x260bcabb,0x7bccdd84),LL(0x2e3ee7a3,0xc4b6b979),LL(0x03bd7f7d,0xfc3349bd),LL(0x122b5333,0x95f84290),LL(0x4bdf7095,0x3057b4f5),LL(0x6af3cf31,0x5d341e22),L_(0x0000015b), + LL(0x1d055192,0xf272a08c),LL(0x343f766f,0x142d545c),LL(0xb8bd86e9,0x860ef117),LL(0x60c69c66,0xb6de931c),LL(0x1b54e53c,0x9924f2f5),LL(0x878c0c9b,0x0b949095),LL(0xfba7e2a3,0x6916f5f1),LL(0x7da79c3a,0x9166a581),L_(0x000000c9), LL(0xe06ad6ba,0xd551de11),LL(0x3b3cbbe6,0x6c45d4c3),LL(0xcc4aa553,0xe3c9e3df),LL(0x1bb5c238,0x05a1e382),LL(0x8dfc012d,0x84d8d463),LL(0x3b856506,0x05b7e241),LL(0xcdcfd8e8,0x27718949),LL(0xc1a85e66,0xab32f2ef),L_(0x0000017a), +}, +/* digit=12 base_pwr=2^60 */ +{ + LL(0x0af6a9bd,0xdca0a6cc),LL(0xfe5f904b,0xd9e6d336),LL(0xd87d0339,0x3b8c9d8b),LL(0x4d463bab,0xfb629c3f),LL(0xc203e46d,0x4ea62ed4),LL(0x998a0ef3,0x64035458),LL(0x62783285,0x7769592c),LL(0x3c56ebb3,0xb1cb181b),L_(0x00000013), LL(0x259a17aa,0xd51ce441),LL(0x8666df8e,0xc62b1c65),LL(0x437c7966,0x74db6999),LL(0x0fecb364,0x7c60998f),LL(0x1f725b1f,0x71fdafc2),LL(0x5b56396c,0xa547fb5a),LL(0x9d888686,0x0f566ae7),LL(0x130033ff,0xd990e0f4),L_(0x00000172), + LL(0x12a6c73f,0x66164319),LL(0xfe4c8bf4,0x9c6ffbd2),LL(0x42f313ec,0x2869e4fd),LL(0xf8b100ba,0xbae712b9),LL(0x0e18229b,0x61a1f1da),LL(0xffe55501,0x032c80f2),LL(0x3bfaa0e0,0x48f0b1d5),LL(0xb83c7607,0x49fa235f),L_(0x00000083), LL(0xa0ed3335,0x8b3c031f),LL(0xc141575b,0x53c30e33),LL(0xfa62217c,0xf9f945a8),LL(0x8b667de4,0x889399aa),LL(0x7c4952fb,0xb711abc7),LL(0xabedb6e3,0x59e7e12f),LL(0x5a1b2cb9,0x1857ebfe),LL(0x4206e243,0xef43f534),L_(0x00000079), + LL(0xa95f9c5a,0x0d937115),LL(0x2ee0eb80,0x4b1412c8),LL(0xdf5a5904,0xe6f39cf3),LL(0xcd50327e,0x9a796b16),LL(0x0841dfd1,0xc493ac5c),LL(0x19d15d79,0x7275eb23),LL(0x4b9d4479,0x1a3b6feb),LL(0xe1eb10df,0x3bcfa542),L_(0x00000060), LL(0x7551bac5,0x7a907c78),LL(0x232dc783,0x82e7d67e),LL(0x5acaf222,0x5ebc3c22),LL(0xe17100c9,0x62250256),LL(0x3198b234,0x4beb3ba2),LL(0x16986b8a,0x492d3035),LL(0x973e4135,0xfcc0dd28),LL(0x2e1155d6,0x2fe7131b),L_(0x000000da), + LL(0x1f14d7bc,0xe951da15),LL(0x3d397c45,0x3964143f),LL(0x24be6549,0x2e556c9c),LL(0xe1293e25,0x3aed330a),LL(0x4bfda40e,0xdf82159a),LL(0x3b13e72c,0x514f7b17),LL(0xa5b859ff,0xe20684bf),LL(0x90812f67,0xed303fce),L_(0x0000009b), LL(0xbca9abf7,0x3b0b3a0f),LL(0x72194a82,0x11d27090),LL(0x17f5564f,0x9bbb7a7f),LL(0x87f0af99,0x96c01479),LL(0x69d62017,0x45cce25a),LL(0x0c43d35c,0x26584337),LL(0xcbff6e89,0x19a55401),LL(0xb503e2ea,0x4184b0ea),L_(0x000001d3), + LL(0xaecaabc4,0xdeccde50),LL(0x9ffdf34c,0x395d2404),LL(0x25068e1b,0x40559189),LL(0x93fb9ea4,0xd141ad3f),LL(0x2a60ba95,0xc42f76f0),LL(0x414a5981,0x946bf800),LL(0x138c47b5,0x38435023),LL(0xf314147e,0x9aa25f3c),L_(0x0000016b), LL(0xe38bdcc6,0x3cebd917),LL(0x0966bac7,0xc3533788),LL(0x2718c3e8,0x33ee6ede),LL(0x10236ae8,0x4f5b88fd),LL(0x44797bb1,0x485e76bd),LL(0xb2b31296,0x68194c12),LL(0xe45112ba,0x0cb75dc1),LL(0x8574000b,0xc33c7dd8),L_(0x00000135), + LL(0x8f37d315,0xf77a65b3),LL(0x37731160,0xde279622),LL(0x6f06ae65,0x87ebd334),LL(0x25b38b15,0x2a1d2c7a),LL(0xa55c6b9f,0xb1687394),LL(0x0ccf2f34,0x4f27c66a),LL(0xecf3de75,0xa9866c84),LL(0xa4a0f4aa,0x44979b4d),L_(0x000001dd), LL(0x0366dd8b,0xbfeaeff0),LL(0xfe941121,0xa80b5c3b),LL(0xc3fed2fa,0x18a5b6a4),LL(0x23dfdf47,0x2ef007c6),LL(0xdb0791d2,0xcec61c6b),LL(0x6d79949c,0xe328d9cc),LL(0x0d03e696,0xaa14a153),LL(0xfdb36710,0xffd70d45),L_(0x0000011b), + LL(0x736dcfa6,0x77e26493),LL(0x6af49ff9,0x089ee4ac),LL(0x1720bd71,0x2f3b86d6),LL(0x48d2c5df,0xbcc66a78),LL(0xd78e07af,0x1f230a9e),LL(0x077a7ceb,0xd2f61bf5),LL(0xfbf99e70,0x92770c3f),LL(0x7ae5f084,0x1a79a148),L_(0x00000035), LL(0xa6ee44c4,0x990f4f03),LL(0x4aacbd8f,0x45377427),LL(0x0ef447b6,0x55b5c873),LL(0xe02e661f,0x11e65ae5),LL(0x99f13f10,0xfe17d3ed),LL(0x393cf4c8,0xdbeb35dd),LL(0x23277110,0x65a7d1cd),LL(0x444802cd,0x8532c3fb),L_(0x000000c9), + LL(0xea71a842,0x4c057a1d),LL(0xfc8fb859,0xe1689c80),LL(0xadc9a8e1,0x09c22f52),LL(0xc47b8163,0x0a960c99),LL(0x90c495f0,0x0a0f356d),LL(0x88242e20,0x87494b79),LL(0xb7f9ca6a,0x6fdcd587),LL(0xd76d2c39,0x43fae9c4),L_(0x000001ec), LL(0x1e35970d,0xaee47a26),LL(0x8df13449,0xadfd394c),LL(0x67553f2c,0x71cdfbec),LL(0xa43c6154,0xf09db2ac),LL(0x4606556e,0xf2e04011),LL(0x12eca225,0x9dfb28da),LL(0x87a4c839,0x28812bc5),LL(0x8cba8984,0xd9e2b1ab),L_(0x000001ad), + LL(0xcb554ab1,0x3446834e),LL(0x21810284,0x2ab359a6),LL(0xf95fa59b,0xf33f9ef0),LL(0x16db657d,0x0f8d940a),LL(0x38fe2897,0x39b668bf),LL(0xdeba7f4c,0xc6452278),LL(0x7471cf19,0xb96dd1e3),LL(0x732f77c2,0x0834711b),L_(0x00000092), LL(0x745c3f1b,0xbca782f9),LL(0x8bd5ef13,0xc4e21488),LL(0x8211733f,0x8f6c1b78),LL(0x50b780cb,0x4b628b50),LL(0x1a1a0206,0x78e4de6a),LL(0x44975c37,0xf9f51865),LL(0x6ef7e616,0xbadf032d),LL(0x3882a9ad,0x821f6884),L_(0x000001f4), + LL(0xe84ad756,0xf2f7ceab),LL(0x6545847a,0xfb04aded),LL(0xdd4cb1ba,0x87083ecc),LL(0xf4c8bbb0,0x452249b7),LL(0x6531b732,0x868536ed),LL(0x6968e15d,0x1d0209ca),LL(0xf0285aff,0xfaefc194),LL(0xbed23705,0xa154b4bd),L_(0x000001c9), LL(0x3ea47ce4,0x69374c35),LL(0x2a6d8757,0xdc6375ee),LL(0xc6f768ea,0xaeba5bab),LL(0x327c743e,0xda6790e0),LL(0x9a01ae4a,0x1a9de4f9),LL(0x3ae6cb85,0x9ac5b7b3),LL(0x6d32a174,0xf134b615),LL(0xdf38a0f3,0x136a4fec),L_(0x000000c9), + LL(0x688d2325,0x79db6c85),LL(0x5359ff24,0x764f954a),LL(0xc7801c4a,0xfa78e8b8),LL(0x098ede82,0xb52cd1ab),LL(0xd34f03a8,0x66adb2da),LL(0xcfcfe244,0xfc69d130),LL(0xb5e52304,0xf88483bc),LL(0xab73db68,0x38339bc7),L_(0x000000fb), LL(0x077d01af,0x7eb9fb07),LL(0x8abf2d7a,0xcb62a0d5),LL(0x37a4ecbb,0x1a28e347),LL(0x06b68356,0x35c05ae1),LL(0xadaac01c,0x2f3d8c6e),LL(0x712aa1f3,0x9ee5907c),LL(0x69606236,0xc9bdbb2c),LL(0x7b2e6894,0x1e267011),L_(0x00000045), + LL(0x09d420b2,0xccc993bc),LL(0xa3ad7d2a,0xd8b3ee97),LL(0x7986ac14,0x59fa9e76),LL(0x95dc5774,0x9477b42b),LL(0xfbe8e9d7,0x89d7ab26),LL(0x79b03712,0x017b7f94),LL(0x77f9bdea,0xbd8dcef1),LL(0x7a238609,0xb9973bfe),L_(0x00000018), LL(0xaee002b6,0xcdfc127c),LL(0x23640ec5,0xad2abcbc),LL(0xc6dc5bd5,0x05982646),LL(0x20400061,0x3c1c6b9f),LL(0x6ee16a76,0xc943d1fd),LL(0xd619a75d,0xd16a85f5),LL(0xa278715e,0xd8747be6),LL(0x34ec8668,0x3ff993c0),L_(0x00000037), + LL(0xcec9be5b,0x6f5e16ec),LL(0xfd62380c,0x192223b1),LL(0x27bda6fd,0xe1e75d7c),LL(0x0df8a788,0xd01bccf4),LL(0xbeed1a6b,0x6611a8ed),LL(0x01402436,0x17838dc7),LL(0x7f189fd3,0x615a507f),LL(0x760bd862,0xe4e17352),L_(0x000001b5), LL(0x791dac6d,0x2bb5aa65),LL(0x0457b859,0xece0f798),LL(0x1a8af3a8,0xde75b69f),LL(0x6625db63,0xcf064060),LL(0xeeda55fc,0x0d8f8c69),LL(0x05536430,0x27c6a431),LL(0xb1dc58b1,0x56c1ac3a),LL(0x0c1a1dab,0x1e026aae),L_(0x00000012), + LL(0xfb73e1e4,0x0f973c1b),LL(0x6977f355,0xca40d04d),LL(0x2797e4c7,0x01c089b0),LL(0x1b05804b,0x0064b701),LL(0x7b76fc1d,0x9677da0f),LL(0xb0b47105,0xd02ba9e2),LL(0x4fb9b758,0x6b2435ea),LL(0x2fd704c2,0xd08297a6),L_(0x00000031), LL(0xdc19942f,0x4734e848),LL(0x366f685b,0x6b9e935f),LL(0xb3827f30,0x81e91d77),LL(0x4cce7910,0x36ada690),LL(0xcb031a95,0x227eb763),LL(0x6ac3a0f0,0x527fc0d3),LL(0x7b60ac80,0x404eb0f3),LL(0x2c62b4f1,0xf1c63ec9),L_(0x0000004a), + LL(0x95cf6b06,0x69fd4f1d),LL(0xbdef736a,0x9044b8ce),LL(0xd7e44ee1,0x546a5d1c),LL(0xe3ac270c,0xbe0ace78),LL(0xe59e1538,0x27b93218),LL(0xe51fc4ac,0xf26796fb),LL(0x71f9328b,0x9137cac1),LL(0x07a55147,0xbda08657),L_(0x000001f4), LL(0x9ce1532c,0x37d59c98),LL(0x2c0e5b9e,0x62f632e6),LL(0x55146f87,0x29fd2249),LL(0x402150dd,0x12ea0f69),LL(0xf442153e,0xfa397b38),LL(0x9b5cefd9,0xc5ad174d),LL(0x8cae5294,0xb46b9f16),LL(0xcd8b0a60,0x773343a7),L_(0x000001db), + LL(0x1e614695,0x71344edf),LL(0x2b4a50c1,0xb3013081),LL(0x4896c770,0x2cf314a1),LL(0x96a68659,0x90053fe7),LL(0xd79226d8,0x5847ac79),LL(0x3ada869a,0xf60993a8),LL(0x7d156a5c,0x67e4b5fe),LL(0x7850cdf6,0xed437add),L_(0x00000084), LL(0xf35bcbb3,0x4e2d6021),LL(0x9877f0a3,0x90be9398),LL(0x4d6435bb,0x86130340),LL(0xe5919257,0x2710c007),LL(0xcc99d199,0x87d3586c),LL(0xc1451c79,0xe8681c58),LL(0xfa896da8,0x6659a487),LL(0xb1a9e543,0x67bd49a8),L_(0x000000bf), +}, +/* digit=13 base_pwr=2^65 */ +{ + LL(0xfa295332,0x0b259ba7),LL(0x94f8c958,0xa4092fea),LL(0xbe9d56f6,0x622efd38),LL(0x0f2ba425,0xa4d25a72),LL(0x57c0adb2,0x2498a9ea),LL(0x11f11875,0x893bbb4d),LL(0x195ec41d,0x2f56b02f),LL(0x2ad72c4b,0xe7bb8bca),L_(0x000000e7), LL(0x0fa4013f,0x521f983a),LL(0xebae7f17,0x5292b2f1),LL(0xdebce289,0xd6d75002),LL(0xb6cd203a,0x93bfe503),LL(0x3c3592c9,0xa40b351b),LL(0x180f5400,0x9b6bafed),LL(0x291283ae,0xd4d6a9f0),LL(0x036cf95d,0x6e0c1563),L_(0x00000035), + LL(0xa8cfa5a9,0x33878665),LL(0xa5401cc3,0x809b2a4b),LL(0x6cdc3f0e,0x90d9594f),LL(0x9bbfac67,0xd551d6e9),LL(0xfd836074,0xe874e847),LL(0x13f89d9a,0x264b3b0b),LL(0x7a6ec5fa,0x0a3ac51f),LL(0x6dd250c6,0x6ed0021d),L_(0x000000bb), LL(0xd1e14aba,0x7c3196ca),LL(0x1495ef12,0x78a62924),LL(0x0cbcf8af,0x1f4ded5d),LL(0x83d56ec3,0xfa54b15b),LL(0xcc6ef029,0x6f0a12c6),LL(0xae62cc51,0xce830e11),LL(0x964fd2d0,0x88747fe9),LL(0x56076a32,0xe8f7bc67),L_(0x0000014e), + LL(0xb73d3d92,0xc668cfa4),LL(0xda69c4d9,0x97ee2907),LL(0xbf4c3402,0xbf5fb743),LL(0x4034c59c,0xd60ae9cd),LL(0x99bc4b73,0xda82be72),LL(0xda1f7664,0xe3800a84),LL(0xfb007b67,0xb7700f12),LL(0xb546161e,0x1386e882),L_(0x0000011f), LL(0x4be150bb,0xfc5d0def),LL(0x660c9122,0x1ba0f43d),LL(0x3a5b4550,0x7224e926),LL(0x33c24e5b,0xba92b4ef),LL(0xd249e1b7,0x2b1856c8),LL(0xb2c9aa15,0x5fe68108),LL(0x6e540179,0x2fe766ae),LL(0xa379f58c,0xecd72903),L_(0x000001a1), + LL(0x4d5341f2,0xdff5ad0b),LL(0xbb141c66,0x6270a82e),LL(0x7912e413,0xfc62897d),LL(0x6b16ad87,0x348f2e6b),LL(0x0fe7c18f,0xae57af6d),LL(0x2f22a03b,0x6d2d6ab0),LL(0xefa7a28a,0xd717c3e7),LL(0x73423958,0xe64f19c6),L_(0x00000065), LL(0x0b4f0f2c,0x8c6c9219),LL(0xa884b55b,0xaff1be7f),LL(0xde74b331,0x8882c375),LL(0x7a676c7d,0x57c355f3),LL(0x71190b6b,0x180dbbfa),LL(0x599b9c95,0xd7dc77b1),LL(0x8f766481,0x227eba11),LL(0x840229ee,0x3dabc4aa),L_(0x000001c9), + LL(0x5d79f380,0xaa1f8835),LL(0xe2b5d59c,0xf1a96e2b),LL(0x281ece14,0x146a87f9),LL(0xb65f19cf,0x24f845c5),LL(0x2f123e45,0x9418bddf),LL(0x90bcee98,0xb6ea99d9),LL(0x64b9ae2d,0xd147e8b2),LL(0xda89ba61,0xf40e6b8f),L_(0x0000014d), LL(0x53c9380f,0x31e35563),LL(0x7d832bdb,0x7117919e),LL(0x5906fec2,0xbdd97ce8),LL(0x069087de,0x015486f6),LL(0x866bf3e5,0x909fd1d8),LL(0x1d61be88,0x770d7f9e),LL(0xa28b013b,0xe7653682),LL(0xddec12fa,0x6232ac60),L_(0x0000006d), + LL(0x38819383,0x0ad9906f),LL(0x9f83225e,0x089577d6),LL(0x8e97694b,0x264428ce),LL(0x0c33baa6,0x1bf2f80c),LL(0x0823ef0c,0xbf31819a),LL(0x2e4750ee,0x66c7b596),LL(0xd160d5df,0x886455a6),LL(0x575835ea,0x662cb270),L_(0x0000006f), LL(0x3c424c86,0x240ffedb),LL(0xd1be1d89,0x3c874c88),LL(0xed147fb3,0xccd38c51),LL(0xbbd7bef5,0x10af1392),LL(0x101c8dcb,0xd85a000f),LL(0x018793c4,0x0b435263),LL(0x24305a2a,0x989d12cc),LL(0x9c061ca0,0xa4553f3e),L_(0x0000005f), + LL(0x9a069590,0x8dfc3414),LL(0x1912aeb6,0x0b37df44),LL(0x1e30b3df,0xcbe1989a),LL(0xaf9f97dc,0x415de022),LL(0x4e406185,0x01cb0277),LL(0x2eb58373,0xbb545160),LL(0x0ab0a19c,0x1e53639f),LL(0xeecdb49f,0x1d63db69),L_(0x000001de), LL(0x2a5d2526,0x8e8397e0),LL(0xa679b16d,0x44c0a296),LL(0x7dcafbf3,0xee3f4022),LL(0x668a1c15,0x2ca74425),LL(0xea786663,0x4906d981),LL(0xda4fa2cc,0x0bbbb1fb),LL(0x21224a70,0x8859a117),LL(0x8ec6a6f4,0x1d1581ec),L_(0x0000019b), + LL(0xf1507b4b,0x4574b533),LL(0xb56c3906,0x76eaa707),LL(0xa9532376,0x16e5e98a),LL(0x611c9b67,0x12d9a934),LL(0x9eb6a261,0x8430b478),LL(0x3fab6e06,0x50fd9610),LL(0xd215cdde,0x1d509b62),LL(0xc4da786d,0xcdfc10b7),L_(0x0000007b), LL(0x752a0af8,0x3ebf635e),LL(0x680f5838,0x9175f3f7),LL(0x9f1a0d87,0x861b999d),LL(0x04cce1e2,0x75ef231f),LL(0xe86e6afd,0xaf7240e6),LL(0x04734476,0xe887f56c),LL(0x2837e095,0xc194ba35),LL(0x9e3dc524,0x97c14f46),L_(0x000000dc), + LL(0x9a9802d3,0x4d7d0f39),LL(0x0345d8d7,0x5e5fc037),LL(0x6923910c,0x0187070a),LL(0x5e15ce92,0x3e13ac42),LL(0x456e81c2,0xcc3c7cbf),LL(0xfc527716,0xcd65b4f8),LL(0x5fd7c9e9,0x305c5c1a),LL(0x4af9f3b6,0x023984a4),L_(0x00000196), LL(0x1b64c99c,0xc01b599b),LL(0x38f68420,0x53fbc68b),LL(0x1b40d8c5,0x91c4f402),LL(0xe87107e9,0x0e78fd77),LL(0x96fc64fc,0xf13016cf),LL(0x438fbd69,0x59324677),LL(0xa8e5ff95,0x50792eee),LL(0xb6a73a20,0xf0ee755c),L_(0x00000074), + LL(0xb1979059,0xaa4d1038),LL(0xf13725c9,0x5751fd0c),LL(0x22f3e709,0xe1b98f3c),LL(0x131dbcec,0x8e66976f),LL(0x64116cc1,0x2dc36526),LL(0xf955db38,0x6f94eedc),LL(0x6bee8490,0x2dc08c31),LL(0x65ce87a2,0x73c1c8c0),L_(0x0000016b), LL(0xf8a59cff,0x9035a740),LL(0x328c1388,0x61a4c1b8),LL(0x1625b5b4,0x8b1931b4),LL(0xf35b9bbd,0x075bce63),LL(0xfb1707dd,0x7a601744),LL(0x232aecf6,0x50241612),LL(0xa974c9fd,0xf1cae8d5),LL(0x7f4ba9ea,0x7ae0d836),L_(0x0000012d), + LL(0xe7c06e65,0xa5756aa5),LL(0xd57f322c,0xc48c9811),LL(0xd934c78f,0xf0128bf0),LL(0x15f53184,0xd880ad31),LL(0xadaedd19,0x967dfa08),LL(0x374ad4e4,0x94c0e608),LL(0x389863cb,0xcecf2255),LL(0xe582d71f,0x02511eed),L_(0x0000017c), LL(0x5b089de0,0xef23aed0),LL(0x89f1eb4b,0xb93d3851),LL(0x8967136f,0x4daacc4f),LL(0x2482bd87,0xd7b99516),LL(0x8a382f79,0x404615c9),LL(0x9daadb4f,0x114144be),LL(0xd38bfc82,0x173ad4ab),LL(0x7f91ad71,0xcab03e42),L_(0x0000008b), + LL(0xa28a3c93,0x299c2c3f),LL(0xf2024c1b,0x63be4df6),LL(0xcbefbff9,0x39510e86),LL(0xe4d80ac6,0xf1d2f2e9),LL(0x8653ba64,0x0165880a),LL(0xd4fac73e,0x73ce1252),LL(0x67da578f,0xeb225590),LL(0xc1817050,0x2d6d0dbb),L_(0x00000096), LL(0xdc671b44,0xa93c7bda),LL(0x82c05dfc,0xcb77f18a),LL(0xaea605e9,0x1df6368b),LL(0x295e78ce,0x67c3474a),LL(0x93547410,0x18ce6948),LL(0x7dc8108a,0x7c67eccb),LL(0x8d3a8c61,0x67a75d98),LL(0xbd7b3b73,0xbf3fd62e),L_(0x00000046), + LL(0xaca4c347,0x9fc92c8d),LL(0x994543ca,0xc6ad228f),LL(0xb218a1c0,0xc58614eb),LL(0x527974b7,0x6ca62054),LL(0x85ffca1f,0xc2129ab9),LL(0xc6c14b3f,0x019b6e5f),LL(0x2ae678d2,0x1e22f90d),LL(0x5b3ee46c,0x71e5f846),L_(0x00000084), LL(0xd3c4dd26,0x4e855850),LL(0x74af7741,0xf226ab4a),LL(0xe0d7e588,0x332581f0),LL(0xe826c3d6,0x4284a728),LL(0xcdf777fd,0x6bfcaba8),LL(0x5a83f0b7,0x6ddf35c7),LL(0x2fbd194d,0x212fa0d3),LL(0x072b793f,0xa6758649),L_(0x0000011c), + LL(0x9d02c3bc,0xb7ec87d2),LL(0xe0887e3d,0xb3caff01),LL(0x4d1f3674,0x567cdbdc),LL(0x39e61184,0x3d19e2c3),LL(0xbe0de4f9,0x0c3139a2),LL(0x500e0978,0x7ba6031f),LL(0x6f3470b9,0xf8e9a69a),LL(0x65297bad,0xe732fb59),L_(0x0000001d), LL(0xc7bd9625,0x69eb8288),LL(0xe4ec65aa,0x281d3a84),LL(0x2c8f25ab,0x27426301),LL(0x97e61a91,0x3672bc0b),LL(0x95476b11,0xbfd7d2dd),LL(0xbff37ddc,0x918c4eae),LL(0x82eea309,0xbd19084d),LL(0x26d27fb7,0x769378af),L_(0x0000016a), + LL(0xe6cbd721,0x4a4ba674),LL(0xc27ec19d,0xf1234a47),LL(0x16a6532d,0x3cf8bf88),LL(0xe2dbe535,0xbc4dc6be),LL(0x948616ba,0x46216f90),LL(0x76d1a242,0x02af6244),LL(0xd5de4770,0xd85e1029),LL(0x7fda32cb,0x5681f9f1),L_(0x0000012f), LL(0x34f944fd,0x944544cd),LL(0x0193b124,0x72b4685a),LL(0xe7601697,0x6ec14591),LL(0x9572360e,0x2184e096),LL(0xffc7295c,0x9127a0f3),LL(0x844306ef,0x818c91ad),LL(0xf603be79,0x81b486bc),LL(0xc1bd26da,0x0974514f),L_(0x0000014d), + LL(0x3c726e7d,0x7d1e874b),LL(0xa513da39,0x8c78c755),LL(0x92677915,0xe62fef13),LL(0xe9e24f3a,0x3d1cf9e1),LL(0xd96cf621,0xf503d4fc),LL(0x0e1204b1,0xf07e39bb),LL(0x71958180,0x7a406c60),LL(0x3b7b9a61,0x304ca46d),L_(0x00000095), LL(0xcdb43a8c,0xf37f3857),LL(0xed377a92,0x8ae49b6b),LL(0x0827d789,0xbd50e338),LL(0xdeff6865,0xfdb287e2),LL(0xe758e466,0xa0c560a0),LL(0x54321f3b,0xb418a264),LL(0xd44767fd,0xfaaa26f5),LL(0xcc7b7f8b,0x59390d36),L_(0x0000006b), +}, +/* digit=14 base_pwr=2^70 */ +{ + LL(0x580dde02,0x814d149b),LL(0xeaa3b1ca,0x3cc8c599),LL(0x43a45440,0xc98d833d),LL(0xdba29de3,0x6e31f2af),LL(0xab2ff205,0xf81e95cb),LL(0xb530ab3b,0x49419f19),LL(0x8a6e1bb6,0xd0585b64),LL(0x1dfbba1e,0xd49f7ff1),L_(0x000001e2), LL(0x2b473469,0xc0250d4a),LL(0xb9f4e199,0xd4759758),LL(0x326d4e2a,0xc32f68bc),LL(0xa78113ab,0x840b01ce),LL(0xcd248f92,0xbd87644a),LL(0xa8d8d61e,0xe9a32d38),LL(0x58a69c2a,0x6a0c706b),LL(0xf4b942e1,0x8c19f561),L_(0x000000d7), + LL(0x538c7fcc,0xff198eec),LL(0x4ec043bd,0x29ee8af8),LL(0x125b846f,0xd01572ea),LL(0x280cfc9e,0x4ba80325),LL(0x3f73f265,0x57e3b7be),LL(0x6bcaffbf,0xf83701a2),LL(0x1a2d3724,0x19d20a25),LL(0x410f80b3,0x50b7f3ec),L_(0x00000161), LL(0x3b961197,0xc136f93b),LL(0xc26463d8,0x000ba8d6),LL(0x8d99824e,0x0e084f84),LL(0xcfbb42b2,0x81fef33f),LL(0x138715f7,0x48ed1078),LL(0xca7dbdd0,0x42869724),LL(0x3c66b900,0xcfde2c20),LL(0x89fab2c4,0x1ca32904),L_(0x000001b8), + LL(0xaf4c0100,0xca7c1c9b),LL(0xd05a1cfa,0x096b7d5f),LL(0x5e939f07,0xc34c35b1),LL(0x11a408f8,0xef94d03f),LL(0x9c1a3053,0xa610576b),LL(0x89fbfdb7,0xce4bec40),LL(0xabf93595,0x66023f5e),LL(0xc5d43f87,0x958d40aa),L_(0x000001ef), LL(0xa58413e1,0xcbfb3671),LL(0x1874038a,0xd3ca207c),LL(0xb2e8a04a,0xccca2442),LL(0x3073c925,0x3c9baa99),LL(0x554b9664,0x6d9e1787),LL(0x70e99ee9,0x874df9a7),LL(0x312bf341,0x1b8e89dd),LL(0xfdf17994,0x5caa8ba7),L_(0x0000005e), + LL(0x5838bb0c,0xad2e37a1),LL(0x5a28cd2c,0x54d33ad6),LL(0xd4f1caa9,0x44b04b20),LL(0x80e4c9d2,0x8c65ceb1),LL(0x370a13f5,0xecff016c),LL(0xee758816,0x6ad260ae),LL(0x95c36fe1,0xbbdbb7b5),LL(0x4d06dfe8,0x48d94294),L_(0x000000f5), LL(0xf5c325d6,0x955c2a22),LL(0x79a376a5,0x16640925),LL(0xf8e9390b,0x36b3aac4),LL(0x2a8dbf22,0x034f2b72),LL(0x77b02d94,0xd5de86b9),LL(0x729cfcd8,0xa8bc9f80),LL(0xbe296bcf,0x1dbcbc03),LL(0xec1469f1,0x73243364),L_(0x000000f0), + LL(0x4141a618,0xa7081b57),LL(0x51abc1ef,0x738c30ac),LL(0xb0cb69e4,0x731cd9ef),LL(0x9b0b0c3b,0xf0dc5cc9),LL(0x40e54a92,0xfba2bbb5),LL(0x0dee7871,0x5ee7b5d3),LL(0x540e5eaf,0xad1a2eba),LL(0x7c44af5d,0xafd14dda),L_(0x000000d4), LL(0x451af808,0x1bf41212),LL(0xacac98b3,0xd6076452),LL(0xed40fc8f,0x868d93c4),LL(0xb7246c52,0xab7c14c6),LL(0x36bab138,0x789fa296),LL(0x330e1a06,0x88e8110a),LL(0x5dbf4ce1,0xff782421),LL(0x9cabebe8,0xa4afa38f),L_(0x00000046), + LL(0xbf3ef154,0x0cb92f46),LL(0x47026a02,0xe21fd797),LL(0x1dec53c2,0x0b5a2b41),LL(0xcfbf686e,0xdb7c6dc9),LL(0xd6d5c0b4,0xb4a8866b),LL(0xf8283374,0x14d9e7f9),LL(0x4dd48282,0xffba2822),LL(0xf9de17db,0x48083494),L_(0x00000075), LL(0x1814c604,0xbe3bf885),LL(0x12bae7dc,0xdf04c3eb),LL(0xe8ee1061,0xc658c3bb),LL(0x8d34f043,0x47642843),LL(0x2916bbb3,0xedee7c23),LL(0x70f93acd,0xc93b4f5c),LL(0x187bf7ea,0x75348b4d),LL(0xffda6e9d,0x9cb29eeb),L_(0x00000033), + LL(0xc855134e,0xdd4ae89b),LL(0x30b7a1ad,0xd9dc0b8c),LL(0xbee0416e,0x67e1dcbc),LL(0x66147ae2,0x7907e5e7),LL(0xb9dea373,0x537d932f),LL(0xf4450461,0x8d5aa671),LL(0x7b0644e6,0x33269776),LL(0x835848af,0x93b63489),L_(0x00000138), LL(0xfdca2fb9,0x400c0893),LL(0xe36e56e3,0x095f6119),LL(0xaaeab6ef,0x07fda371),LL(0xfdbbf61e,0x51034096),LL(0x65d823d3,0xc5284f49),LL(0xcec7f701,0x00cca32d),LL(0x443cce6e,0x13b673ca),LL(0x7c7c1332,0x75e93750),L_(0x000001f7), + LL(0x5f48e2fc,0xfafcc610),LL(0x2c5d9495,0xd34073fc),LL(0x321d1a08,0x30442510),LL(0x3427742e,0x8068ffd8),LL(0x2ec5f97b,0xa7faa8a5),LL(0xac14530d,0x5d010e52),LL(0xf277a140,0x3edf5701),LL(0xcdd53228,0x07eb00ea),L_(0x00000155), LL(0xf8f566cb,0x51771845),LL(0xa7b0e50c,0x66aafeee),LL(0x81cf4ee6,0x8fd52580),LL(0x1bc2c6ec,0x232a19c7),LL(0x6790d250,0xd4c06ab2),LL(0xdc4411be,0xacd06e0b),LL(0xe0fd2a20,0x19734273),LL(0xc9fb738a,0xe5b5e5c1),L_(0x00000162), + LL(0x2c36a49c,0x346bc599),LL(0xb17c0c24,0x4349ec6d),LL(0x1994a52e,0xc574a60f),LL(0x479b18ee,0x2cf1156f),LL(0x00b04364,0x587d6a8b),LL(0xe20999e4,0xb840bbc5),LL(0xdfbaad24,0x1056fcbe),LL(0xb84e8539,0x211c4015),L_(0x0000009b), LL(0x34b3807b,0xc83300dc),LL(0xcbae472d,0xff64a1e1),LL(0x60ef1e86,0x1368c7ee),LL(0xfa9cecf9,0x1a548595),LL(0x12c62bf5,0xbab6253e),LL(0xc9541d0a,0x2d9ef7be),LL(0x1e1f27f7,0x956cff19),LL(0xba2813eb,0x5c29eeaf),L_(0x00000030), + LL(0xdb558004,0xce8d95f8),LL(0x9dc5e59e,0xf1893cb5),LL(0xa7cb9fd4,0x77041349),LL(0x53461897,0xb8f3b00e),LL(0x4b8c1719,0x3acc8d5e),LL(0xe436769f,0x2b51a3ea),LL(0x15adc570,0x5fa4c1f5),LL(0x93a4046f,0x70b3a881),L_(0x000000f5), LL(0xc6ca2c5f,0x8e92a20f),LL(0xe70ae728,0x9e74b9b9),LL(0x6d7cbd1a,0x8837144f),LL(0x797825f3,0xa10a4e2b),LL(0xd3f2260a,0xbce98f37),LL(0x3d85debd,0x39cb6f79),LL(0xa763a30e,0xb1f85a09),LL(0x15a4a3f3,0xb462b7b8),L_(0x000000c0), + LL(0x4df25f56,0xf82561fb),LL(0x9e75f649,0x8c808470),LL(0xd7751a0f,0x0c1dcee3),LL(0x4322644e,0xf53e90ce),LL(0xd68db21e,0x8f8d9278),LL(0x72d07c10,0x94db3061),LL(0x821b176c,0x2c56677c),LL(0x51fa088e,0xabb0b5b6),L_(0x00000172), LL(0xbd496ef4,0x2f3203c7),LL(0x702f0af9,0xf4ee87af),LL(0xdeead7f4,0x1e240ba2),LL(0x6ba4a666,0x45c6b9bc),LL(0x5b558a22,0x69826d07),LL(0x82f16b78,0x1d474e7f),LL(0x1b099f67,0x6f9ab8f0),LL(0x79936b07,0xb7da7cd3),L_(0x00000140), + LL(0xdcb6ef22,0x8cd18dc5),LL(0x4111e26b,0x563a07bc),LL(0x3482455e,0x0e2f7391),LL(0xcb5ec4ad,0x21483bed),LL(0xc0caacaf,0xa5a48441),LL(0xc9e80f16,0xbffbf280),LL(0x90242b85,0x91f37a76),LL(0xd9544186,0x2dcd22ee),L_(0x00000157), LL(0xa766d6e3,0xfc4b8794),LL(0x38293f47,0xa81360ec),LL(0x31e9f867,0xbe34d77d),LL(0xdb92af31,0x882df842),LL(0xcd799976,0xd34a906c),LL(0xabb505dd,0x961ddfb3),LL(0xa3a37b0b,0x4fbbb326),LL(0xf7af85a7,0x508f35d4),L_(0x00000028), + LL(0xbb0940d9,0x3d70435a),LL(0xff491b63,0x1eb2a685),LL(0x22d3c808,0xa380de6e),LL(0x17e44c8f,0xd9df636d),LL(0x2a35379f,0xa903bf9f),LL(0x0f809249,0x387b8a0a),LL(0xabe12175,0x44dbe0cd),LL(0xfd759d00,0xcccc9723),L_(0x0000003c), LL(0x98196ddd,0x019c1122),LL(0x84c5a37e,0x254adbe9),LL(0x3981a4ac,0xbfc928a2),LL(0x62436eb9,0x7ad29c64),LL(0xc99f2914,0x2cc9ceca),LL(0x2f0c2529,0x2d8109c3),LL(0xbda5dcc9,0xe65ae3c9),LL(0x27c8461f,0x2300f14c),L_(0x00000101), + LL(0x7f2f4a50,0xdb205fa6),LL(0x977d6dff,0xb6a346a4),LL(0x39b1aa2b,0xc02f5c26),LL(0x6d520bed,0x7e9b3df6),LL(0x9fe2d1ad,0x8060eb41),LL(0xffe8b3d8,0x8de43158),LL(0x6b78c0bc,0xe900b6ef),LL(0xba5607f9,0xe9657827),L_(0x000001ae), LL(0x656a0930,0xe37eeb37),LL(0xf82e5547,0x817b9a2b),LL(0xebe57826,0x966c6b06),LL(0xd17239c5,0x0d566764),LL(0xb6e7e211,0xb736c18d),LL(0x67f60fe4,0x7f6de467),LL(0x91c330d3,0xdf003076),LL(0x6fe1ff1d,0xf3ffae38),L_(0x00000032), + LL(0xa38e4c83,0xee75b3b5),LL(0x42633219,0x96b76b44),LL(0xa702e22d,0xce624bec),LL(0x45df636e,0x087dc34c),LL(0x00b8ab39,0x7c3b41a0),LL(0xa4c92149,0xbe1f412c),LL(0xc186c0a7,0xa0e6b72f),LL(0x6f9b9c73,0x909b955c),L_(0x000001a3), LL(0x6d958194,0xfceeca76),LL(0xc25f90a1,0xd246f978),LL(0xf7a79529,0x5bdce3f4),LL(0x202fdba0,0x4516ada4),LL(0x9ccb5769,0x22fea769),LL(0x35cd1aca,0x9d5e791d),LL(0x72d93ad7,0x89481217),LL(0xd00224e0,0xdc53ffd6),L_(0x000000b6), + LL(0x0b6d183b,0xcdd1f50d),LL(0x88770143,0x721cf9d0),LL(0xd247118a,0x5a8338fa),LL(0xfa498ee5,0x33ff454e),LL(0xa8d98087,0x2107a954),LL(0x4eaefaa7,0x39298606),LL(0xc385af5a,0x3e0c503b),LL(0xfc7e0cec,0x37f8795c),L_(0x00000096), LL(0xa2f6c113,0xbf161ebf),LL(0x48b5f685,0xbb087e9c),LL(0x58eb481e,0x3b7987c9),LL(0xa465a54c,0x6e92e01e),LL(0xa8194344,0x1e66d88b),LL(0xb0c7a894,0x40dc6c71),LL(0x690cafad,0x057f59a3),LL(0xf02679ac,0x4a12e130),L_(0x0000007d), +}, +/* digit=15 base_pwr=2^75 */ +{ + LL(0x3a007995,0x19e3233b),LL(0xcd545767,0xc78c2194),LL(0xc744ff86,0x789e51d3),LL(0xafacd6dd,0x7a5cd253),LL(0x398cb1ba,0x18b56085),LL(0x273c4fb9,0xff1bce38),LL(0x0ba240c9,0xbca7efa9),LL(0x3bb2e372,0x05ab8ba7),L_(0x0000012d), LL(0x563a114b,0xf4adbd1d),LL(0x90284d2d,0xe9ad940e),LL(0xe3af63ef,0x61ffca7f),LL(0x96feaa4e,0xba0669de),LL(0xdbf94ff4,0xd7b8471a),LL(0x696c5279,0x1dda976e),LL(0x0a229117,0x1566b880),LL(0xba44b588,0x7a731f5f),L_(0x0000006a), + LL(0x29b896b5,0x025efc3b),LL(0x60e34ccb,0xbcd9f85c),LL(0xae29c1d9,0x1e85f821),LL(0x3dcc6356,0x27219e29),LL(0x3f95824c,0x9e01039f),LL(0xd3a9843c,0x8ef0f79f),LL(0xbba44b84,0x9cddb5a3),LL(0xf0a7f537,0xe6239011),L_(0x00000062), LL(0xd40be315,0x0d73669d),LL(0x0706daaa,0x90c605d6),LL(0x262a826a,0x2e67d62f),LL(0x90997b0e,0x5ac29c5e),LL(0xa4dc7322,0x9728fe4c),LL(0x46c28be3,0x8656b7a7),LL(0xcf46a3d3,0x318bd5a3),LL(0x0c58ac70,0xbdba780a),L_(0x0000012f), + LL(0xd012d32d,0xb1701460),LL(0x36257f7f,0x712c0989),LL(0xbc1511ff,0x948635e7),LL(0x929c254b,0x950b6b9e),LL(0x88fae008,0xc1ebc649),LL(0xb35e21c2,0x69de590b),LL(0x0c8fd948,0xc534704d),LL(0x03df94d1,0xc72f229f),L_(0x00000191), LL(0x417e1367,0xbe538962),LL(0xbaa7a81a,0x6d9ba3b4),LL(0x085df8a2,0x72446606),LL(0xc1a4f077,0x313aa0fe),LL(0x443486a7,0x310facd1),LL(0x4bb29bc8,0xe7424659),LL(0x70dbb24c,0x7a208944),LL(0x2ddd11bb,0x4190996b),L_(0x000000fa), + LL(0x5370e2b4,0xa39d68ac),LL(0x0b26e23e,0xf98a9904),LL(0xbe557ba1,0xcef362a9),LL(0x202765cc,0xa7731e3c),LL(0xa726d7b5,0xcd815e2b),LL(0x1c25faf8,0xa6579cd9),LL(0xdafb2e8b,0xc5ec8fb3),LL(0x648049fa,0xd9a36af4),L_(0x000001d2), LL(0xa04b6251,0x96110b89),LL(0xd1ddccb2,0x551f88c9),LL(0x0f26d015,0x9b8bbb0b),LL(0xb5bd39d3,0x9d52ffc2),LL(0x2dc18ef7,0xab6d006e),LL(0x142fb527,0x804f61d0),LL(0x391511fa,0x9dbe5992),LL(0xa3e717ea,0x408dd815),L_(0x0000008a), + LL(0x9289c640,0x6b7a800c),LL(0xc76b2795,0x581d42f0),LL(0x8ca1e81a,0x4472a2fa),LL(0xbf4b6019,0x6715dfdd),LL(0xb304a936,0xf9366e36),LL(0x391be48c,0x81f74b90),LL(0x6151e36b,0xd32b6b20),LL(0xc48b79a5,0x9a91aea9),L_(0x00000183), LL(0x831e89ed,0xfa6f04fa),LL(0x5ebcfa5b,0xec4ecc3d),LL(0x6b2117a0,0x7b376a81),LL(0x90bf1080,0xba3f0166),LL(0x0993a607,0xaf14e2f1),LL(0x8cf72c90,0xef21c633),LL(0xd173a6d3,0x187451c2),LL(0x13ec5f08,0xdd8ab011),L_(0x0000006c), + LL(0x0f29eadd,0x246b16c2),LL(0x9b833212,0x9da31a63),LL(0x297cf150,0x4995a63d),LL(0x90a2a3a8,0x26054531),LL(0x22ca8af6,0x0cdf918d),LL(0x62531849,0xa5ed4b64),LL(0x7ca9de65,0xfa4fec38),LL(0x72b35acf,0xa666610b),L_(0x0000018a), LL(0xf96d539b,0x221e3646),LL(0x256f3a4b,0xb6bf83af),LL(0xc408a90f,0x7abd62f0),LL(0xaefff14a,0x4069cdfd),LL(0xae41ce0c,0x29824953),LL(0xb47d1cba,0xa382ab7e),LL(0x9eb440b7,0x957f6459),LL(0x4c148b60,0xaf1d974a),L_(0x00000047), + LL(0x1086c5f8,0x26130fe0),LL(0x7b4b2e70,0x68d46ff5),LL(0x0c407c0a,0x3b8c39a0),LL(0x88577dcf,0x6dc35106),LL(0x5dafbff6,0x66c847b2),LL(0x2d675f1f,0x17ebe229),LL(0x834c9c2b,0x7dd924e2),LL(0xd5b6edf6,0x0a03148b),L_(0x000000d6), LL(0xf96321d7,0x6f67acfa),LL(0x9437a2b2,0xf66029f2),LL(0xe7b46c71,0x329167d4),LL(0xb24e796b,0xcf0f34c3),LL(0x9d6e95b5,0x4f9e7abb),LL(0xcb817cb5,0xb5258968),LL(0xe5382677,0xb89d3951),LL(0xb2e2fb7f,0xd0f6ea08),L_(0x000000c4), + LL(0xe65a160c,0x79edff6b),LL(0x1ae3af0e,0x7704970c),LL(0x8b957c42,0xb8aa395f),LL(0x5f0f181e,0xbdf3d6c8),LL(0xe7d8f529,0x4e626c58),LL(0xd83fd353,0x770dabfa),LL(0xe5ada98c,0xac3e65d7),LL(0x6430730b,0x954e2567),L_(0x0000010b), LL(0x09d0476d,0x4dcf97c3),LL(0xab9b5d71,0xa56a3252),LL(0x9648c08c,0xff297fc7),LL(0x897ba609,0xff84306b),LL(0x4c446a06,0xddb64374),LL(0xbc202d01,0x97ee1218),LL(0xf9b0f80f,0xf048db33),LL(0xb5f54bb8,0xf9d935e8),L_(0x000000e7), + LL(0xa010dba8,0xa2fb38a6),LL(0x31feab78,0x36899fbf),LL(0xacceb7c2,0x5bdd898c),LL(0x14de8e04,0xe42d5076),LL(0x459ea861,0xe51c21c6),LL(0xfbda1b0c,0xdc95445c),LL(0xa7d4e38e,0x7947f71f),LL(0xadcd66a5,0x2c40ef60),L_(0x00000199), LL(0xdcdf5f99,0xf1467b2f),LL(0x2de2bfcc,0x9c1ae772),LL(0xd43f04fe,0x41471183),LL(0x043333f6,0xc9a3cf00),LL(0xf1a6e8cb,0x7cc8a0c7),LL(0xa8c3f924,0x74a1cf04),LL(0x828052df,0x8be5596b),LL(0x50a45820,0xd5384c0f),L_(0x00000198), + LL(0xe9851bfc,0x51c40077),LL(0xbfe0982b,0x87cd565d),LL(0xd372a1cb,0xccd954bd),LL(0xbbff7b4d,0x294b36f6),LL(0x8237c51d,0x4ce0f879),LL(0x261403d6,0x569d6e3c),LL(0xb79e0e60,0xeba6224f),LL(0xc33dd3b5,0x701a34d7),L_(0x000001f7), LL(0xaaf054ad,0x5f93dace),LL(0x160bbbee,0x8aa260aa),LL(0xa9f4b722,0xb1b5025d),LL(0x817d1e67,0x81308a04),LL(0xfe002797,0x4afd2f00),LL(0x680cc208,0x154f68e6),LL(0xd4b7eccc,0x4cc6b0f1),LL(0xb8976118,0x39f42fe2),L_(0x000000b5), + LL(0x7283e8a9,0x8ab94a45),LL(0x0bb733f0,0x642a6645),LL(0x41a534e8,0x57ae42c1),LL(0xd4abc981,0x8be6ea3e),LL(0x7e62f50a,0xb3f01b7e),LL(0x98a38cc8,0x8865f98b),LL(0x7862605c,0xf1a738cf),LL(0xde02b3ce,0x7cbcb60e),L_(0x00000191), LL(0xbb74e488,0xc6aa6ad4),LL(0xf6d314f3,0x1691860c),LL(0x97214287,0x625fd28f),LL(0xab878ce4,0xd6ea61fb),LL(0x8ebed709,0x5a22486f),LL(0x6db8fc6f,0x0b71449e),LL(0x46c06119,0x0e8cd622),LL(0xe44026c8,0x2b868bc7),L_(0x0000017c), + LL(0xab7fbeff,0x954c6cef),LL(0x7291b35c,0x062277d6),LL(0x7553137c,0xaf482063),LL(0xb75730d8,0x710c68e8),LL(0xb68d2250,0xbae3e7c1),LL(0xff637a2f,0x6b643e1a),LL(0x1ef002bf,0xdcdc4699),LL(0x38ab4582,0x3d4d096e),L_(0x00000057), LL(0x92bea0d8,0x899ee626),LL(0x4d7631d6,0xcdd2a79a),LL(0x2ee3aea9,0x8162f50f),LL(0x1476eea0,0xc4d433fb),LL(0x27c81475,0xe8214237),LL(0xdc969d9f,0x56c76934),LL(0x6d918c87,0x7533eb08),LL(0x56d2a891,0x6eb1be30),L_(0x0000001e), + LL(0x1b47da4b,0x8a8331e9),LL(0x49ed1362,0x59fa2b27),LL(0x418a7189,0xe8454c56),LL(0xf46859d8,0xb777077d),LL(0xdf895326,0xa1ecfc76),LL(0x7c0bff3a,0xa0d40120),LL(0x4dc72f6f,0x863ee5b0),LL(0x6d7d35d2,0xf072a9db),L_(0x0000015b), LL(0x51536aa9,0x797cc189),LL(0x992a786b,0x424d6c36),LL(0x0bb1db2f,0x7b4a72f0),LL(0x99596f48,0xd38d470a),LL(0x7d9d8119,0xed3220cb),LL(0x7fe52443,0xd0f7efd8),LL(0xf334c76a,0x5b52a8c0),LL(0x289cf254,0x9c518382),L_(0x00000178), + LL(0x50ef0de9,0x06fa762c),LL(0xf226f768,0x770bcaf6),LL(0x47cff6b0,0xec1a0d3e),LL(0xb780ef8c,0x49872a4e),LL(0x434df874,0x82a505c8),LL(0xfc298963,0x6b56a94d),LL(0xc037f2d4,0x1721c7a5),LL(0xef98ff94,0xdf6aa659),L_(0x00000092), LL(0xce001c93,0x229e4010),LL(0xb7710f4a,0xb1be023a),LL(0x4f780b68,0x37373fb4),LL(0x44ddc611,0xb6129029),LL(0xeba09ab8,0x3d4bb157),LL(0x52d1be00,0x7557730f),LL(0x5d184bff,0x79506c27),LL(0xabea8b49,0x120e5c57),L_(0x000000a5), + LL(0x475e54a3,0xe76ea216),LL(0x641b111e,0xe7b2f367),LL(0x11e8b8cf,0xaa1845af),LL(0x3e44ec0e,0x634d35d6),LL(0xd9e05b86,0x9fef6a5e),LL(0xa66acc79,0x6151a902),LL(0xc7fadad6,0x95ee0cab),LL(0x4f71151e,0x5844d0c7),L_(0x000000b3), LL(0xc47ffd5b,0xd5a7577a),LL(0xa66a5bd1,0x1bbdd68c),LL(0x1a4070bb,0xafa4a6f8),LL(0xb0f9b28c,0x67075086),LL(0x966afc2e,0x63512dbd),LL(0xed5912ae,0x8f26597d),LL(0xd092a281,0xa415f451),LL(0xc2ef31e4,0x0ffffc3e),L_(0x00000147), + LL(0x4e004498,0xbea3c389),LL(0xaa39f9d0,0xa8f46458),LL(0x0b3654a0,0xf830008b),LL(0x4cd7392b,0xa46a22ca),LL(0xb12eb97a,0x80e1d7af),LL(0xd888b9cb,0xf74c8adc),LL(0xbb6e179c,0x73b51d04),LL(0x968eb224,0xb5490b50),L_(0x000001f5), LL(0x51f96d03,0x9de9e100),LL(0xfade3749,0xecddcd4f),LL(0xfc72771e,0x7aa9dbdf),LL(0x5e1bb964,0xdc24156b),LL(0xbaddb508,0x0de78eea),LL(0x6facddf4,0xb1c48894),LL(0x4a964d6a,0x49c82382),LL(0x6f258c87,0x62eac3b1),L_(0x000001b0), +}, +/* digit=16 base_pwr=2^80 */ +{ + LL(0x32cc508d,0xcd408945),LL(0x60b1057f,0xf3eb54e9),LL(0x77d231ff,0xbd6ea408),LL(0xe5110313,0xb9ee8343),LL(0x85209f6e,0x64924e77),LL(0xe7fa5897,0x32e258b2),LL(0x618a6eb3,0x11e2e038),LL(0x96067c35,0xf4c23aaf),L_(0x000000a7), LL(0x0f221560,0xef974e44),LL(0xa1b7a3a0,0x8ea1f931),LL(0x72932b48,0x720e4174),LL(0xbb75d745,0x1bf9c803),LL(0x996758e5,0x7f0b3909),LL(0x8d83f97c,0x39d56a48),LL(0x1fac932b,0x55fe1ded),LL(0xaaf43ccf,0x91fe8399),L_(0x0000002b), + LL(0x6ffb7834,0xad3e7387),LL(0xd3158a49,0x8771e37a),LL(0xb98ec469,0x6f103f2c),LL(0x1bd53110,0xf8325af3),LL(0xa434959f,0xe47f875b),LL(0xa10264ee,0xcf224bc0),LL(0x9ccf2f61,0x337d33a1),LL(0x86ce6031,0xe6ee9f21),L_(0x0000009e), LL(0x6706e91f,0x9179c5ea),LL(0x79ba3023,0x9aa4ed38),LL(0x7e239f26,0xaa83eb97),LL(0xef091443,0x82853a90),LL(0x336fc4d2,0x2b260d34),LL(0x56b3a0bb,0x119fbd07),LL(0xe0f16198,0x3453a3a6),LL(0xa2af0802,0xd2ee6016),L_(0x00000050), + LL(0x8000185b,0x02ebe1f2),LL(0x3ebcda00,0x30d3e5f2),LL(0x75cccaba,0x8ea9d40f),LL(0x108edd48,0x152a6563),LL(0x6028024e,0x732e422c),LL(0x17618296,0x142e6cc1),LL(0x9dea7266,0x5d4e4488),LL(0xb05325e9,0xf99254d1),L_(0x000001a3), LL(0xf3270a2e,0xc29b5dca),LL(0xbad984d9,0xd2f759d7),LL(0xad7bc046,0x347ff7c2),LL(0xa4e4f59d,0xc16d4c0f),LL(0x0a06be29,0x872d14ff),LL(0x2390bb31,0xb7a5b6ec),LL(0x66be2ce2,0x408ae4cc),LL(0x6b9b1fe0,0xc70e0700),L_(0x00000080), + LL(0x9cc4cee5,0xa99628d5),LL(0x4764e916,0x4417813a),LL(0x9a05da16,0xe423f0c4),LL(0x2babb644,0x24dca899),LL(0xd179a66e,0x894f6883),LL(0xc157cbef,0xed7756c7),LL(0x44c30131,0xcdff08e1),LL(0x78b0a3e9,0xf5fa08ac),L_(0x00000059), LL(0x43dabd75,0xbe7ba3ec),LL(0x93626426,0xd17b8f8b),LL(0xbfcd2a78,0x486d7ac0),LL(0xaeda53c9,0xfc3c49c2),LL(0xbc99eeae,0x49fb4a9c),LL(0x12ab3d09,0xdb075628),LL(0xc0f863b5,0x3d5da4c6),LL(0x8ec31fe4,0xc774d9e0),L_(0x00000072), + LL(0x098d00b0,0x4acb7a91),LL(0xf4f8175f,0xfe317cf8),LL(0xf8155d16,0x2bc9a77c),LL(0xac3ddeef,0x17520bd7),LL(0x0aeae3c4,0x44ee6fbc),LL(0x1aaae6ff,0xd8c23852),LL(0xf47bc828,0x553f42c9),LL(0xc09b26d0,0xb04bad2f),L_(0x000001e8), LL(0x9a32ac7c,0xc1e669bb),LL(0xc8aca498,0x697322f4),LL(0x43042d46,0x334625a5),LL(0xdf16aa69,0x67bda03c),LL(0xab4b67c2,0x205d341f),LL(0x83a55d6f,0x05daa2bd),LL(0xcdfd94e0,0x9ac9573f),LL(0x1cb76afe,0x2fdad4e8),L_(0x000000d1), + LL(0xf074fced,0x97863b91),LL(0xc43281c9,0x92d449a3),LL(0x7a68c2d7,0x063c9119),LL(0x3b2de0b3,0x55e1666a),LL(0xf3e7d825,0xf70b4227),LL(0x6aacf427,0x6c04e18d),LL(0xc2b9b616,0xaa4c82c2),LL(0x376fa210,0x3261f95a),L_(0x000001ca), LL(0x27de0f4a,0x63844f17),LL(0x70941232,0xfd0bec77),LL(0x3b5f4e85,0x68fe79f4),LL(0x5cbc9a57,0x826a7303),LL(0x65bb2328,0xda7d2209),LL(0x40788a77,0x6978fe42),LL(0x14c7cf99,0xc2ae1a05),LL(0x77943ce3,0x53eab85e),L_(0x00000074), + LL(0xdb0d1b45,0x25e9937f),LL(0x14c566a9,0xd1cf3988),LL(0xcd250848,0xcc300694),LL(0xbfd82b6c,0x135bc75f),LL(0xa47db4ae,0x8639e63f),LL(0x2295c1f1,0x30a5e5b3),LL(0x61f91b27,0x2841bb1c),LL(0x1335383b,0x5f90a945),L_(0x00000129), LL(0xe1df27e3,0x0fb695c7),LL(0x721a9ee4,0xc8c313d1),LL(0x267e9801,0xb288bc93),LL(0x9aafbe12,0x80d7a36a),LL(0x5e34c2b1,0x5e8b79ae),LL(0x22efe6b6,0x7da7f03b),LL(0x552e9134,0xee94e563),LL(0xab16538c,0xb9465f4a),L_(0x000001e5), + LL(0x336db42e,0x49f7546b),LL(0xf6900421,0x57093c06),LL(0x8e00d7d3,0x8d2dbce8),LL(0xd9ede742,0xd004dc4a),LL(0x01940521,0x0d3be2ce),LL(0xdc4ae497,0x1bb5cf60),LL(0x0a6ccb03,0x803df567),LL(0xdf04605d,0xa1f5bb64),L_(0x0000013c), LL(0xa05f0fb0,0x1f8ad28a),LL(0x39d0a695,0xe5b908dd),LL(0xd67e92ff,0x4165f76f),LL(0xf281077f,0xc8d52980),LL(0xc9ee2db2,0x9ff0b841),LL(0xe792e9a0,0x9a5850f2),LL(0xd74d1fff,0xdbe9887c),LL(0x468c4978,0xb14ef9aa),L_(0x0000018f), + LL(0x17cfe85b,0x645a5369),LL(0x96452c88,0xfe80ed90),LL(0xd777343c,0x53d07352),LL(0x57827fb3,0xa6c3e43b),LL(0x80b39b9b,0x2a8365b0),LL(0xaf5a3de0,0xac5e9239),LL(0x181d4337,0xea4d87d6),LL(0x51fe7247,0x91c5ece4),L_(0x00000037), LL(0x0dfb5a0c,0x7a7e75fb),LL(0x15428231,0x5c94ba5c),LL(0xdf444166,0xc521d0ff),LL(0x0f639c5d,0x74de2e37),LL(0xdc1e397f,0xc9356e9e),LL(0x93e3e959,0x0c9e3464),LL(0x39201886,0x6a78ad71),LL(0x2c0f3246,0x857cb413),L_(0x000001f9), + LL(0x9207f1ee,0xc358b92f),LL(0xcc5ec219,0x085c3194),LL(0x8df4ef1a,0x71e157f1),LL(0xb527e0ab,0xdd0a2a0d),LL(0xae481b48,0x0e0efe51),LL(0x27b717b7,0x5828b573),LL(0xc2a98225,0x3d2c0ff1),LL(0x099e23ff,0x7269417e),L_(0x00000083), LL(0xf9a97523,0x11db2059),LL(0x5fbc6f68,0x162229b6),LL(0x08c87928,0xc7000da2),LL(0xa03b55dc,0x3f37b2b0),LL(0x03e27759,0x2abf57b8),LL(0x6d3e3a66,0x3d925762),LL(0xa5335174,0xec43569d),LL(0xb9e4cda0,0xe4522ca8),L_(0x000000f8), + LL(0x991bdfe4,0x46da618a),LL(0x0642fbf7,0x2827ab9a),LL(0xcb3c80d7,0xa4c02a4a),LL(0x24ffe39b,0xdd1a385c),LL(0x81fb1f95,0x4fe48911),LL(0x7ed78cb0,0x42514e36),LL(0x12838990,0x2822adf0),LL(0xcd6245c9,0x17e195af),L_(0x00000018), LL(0x682326f4,0xeadeab18),LL(0xa6714b92,0x8194d4d8),LL(0xec34ca02,0xaf1a5a6c),LL(0x18aa0b61,0x21f3603a),LL(0xd56cb67c,0x9e65c6ef),LL(0x64df4053,0x9af9ff17),LL(0x6e1b769a,0x333abb94),LL(0xd74c3bd6,0xb764bf57),L_(0x00000185), + LL(0x6b395c0b,0xb5b87b52),LL(0x48432467,0x7efdcd9b),LL(0x9d4e858e,0x7c0c8922),LL(0x30d29dcc,0xd7b3a015),LL(0xe0639cac,0x02eb2112),LL(0xa2a88900,0x99cb32dd),LL(0x5ee2bf55,0xe98b88fa),LL(0xa3323b39,0x9b1c4ab7),L_(0x000001c6), LL(0xe5c08a49,0x2cf028a4),LL(0xf8ca8809,0x62209ac3),LL(0xfaac3cb2,0x91bff5ca),LL(0x2cd5c8b2,0x2f2a80df),LL(0x2244abea,0xd2c4cfb9),LL(0x1181848d,0x345b1ca4),LL(0xc35b7b60,0x3093dc03),LL(0x815a0c2b,0xccdd0f34),L_(0x000000a3), + LL(0x3f060a04,0xc9440b7b),LL(0xf892eb31,0x8c0aafa8),LL(0xc25c561f,0xebcb8c35),LL(0x12770520,0xe63e94d2),LL(0x6d5c16b3,0x1c828d52),LL(0x9bde42a4,0xcafff93a),LL(0x2b7abe51,0xc4851b92),LL(0xa9fcd2b9,0x41227e00),L_(0x0000009b), LL(0x24e44af4,0xebda81a0),LL(0xa695bf6c,0xdee85d40),LL(0x76c43176,0xf9ff0671),LL(0x8106e3cf,0x7e8060f0),LL(0x4ce1b2a1,0xa7e23360),LL(0x81d1ed8c,0xbfa62d39),LL(0x47f7b51a,0x4292ca9e),LL(0x9139a437,0x60e7fdba),L_(0x000000ae), + LL(0xa65d9111,0xb4e47850),LL(0x4221510a,0x7528def6),LL(0xeca10d8d,0x12fd7625),LL(0x4560fa05,0x7883ec3b),LL(0x5c64949f,0x2a961e63),LL(0x4df0271f,0x4eacd5e7),LL(0x9af4d45b,0x62f25ceb),LL(0x9ab3bb2a,0x7ef167e5),L_(0x000001f5), LL(0x92cdd0ea,0x6860d474),LL(0xae51b5ad,0xc69002e2),LL(0xccd9fe59,0xe138bd08),LL(0x00304cd8,0x9ec62f95),LL(0x329feb5b,0x0928d5e4),LL(0x8a27990e,0x6d9656bc),LL(0x0f0e6792,0xb7989c9a),LL(0x3f1c862c,0x1bbbc30e),L_(0x00000033), + LL(0xa5a66086,0xf622c1ac),LL(0x7ccf9bc1,0x3beaf465),LL(0xa68726c3,0x900cd78d),LL(0x2a903911,0x2be1c512),LL(0x8832203c,0xa8466703),LL(0xbf042dcb,0x1301ba91),LL(0xa8235209,0xfc06ed9e),LL(0xe1aef50a,0x04f3d4d5),L_(0x00000138), LL(0x47a0b87c,0x55a0038d),LL(0x9826631d,0x6ef49646),LL(0xe8d54ad8,0x53644562),LL(0xa200feff,0x773835a0),LL(0x5c4408b1,0xb6c0a0e2),LL(0x3199aac7,0x2d23ae2f),LL(0x4f2c5a36,0xdfbd8171),LL(0xccacb296,0xa7505cc8),L_(0x000001e2), + LL(0xcf45bc39,0xbca99c74),LL(0x16aa0dee,0xae132b38),LL(0xb713a9d8,0xf8d3fb57),LL(0x139b0131,0xc38156e4),LL(0xc42164bb,0x3293d5ff),LL(0x74da5a53,0x28a54d0e),LL(0x970fceff,0x56246758),LL(0x3ef8fd2b,0x1b3685cb),L_(0x000000ad), LL(0xa0e59886,0xe1a4b217),LL(0xaa3f69ad,0x23117719),LL(0xb7b4e45e,0x118c88c1),LL(0xf1294233,0xd7dfa5c3),LL(0xf1ed8c9d,0xfa2104f8),LL(0x22213ffa,0x89ed7138),LL(0x3ea0a97e,0x2cd0f857),LL(0xf09db9f8,0x28378a81),L_(0x00000167), +}, +/* digit=17 base_pwr=2^85 */ +{ + LL(0x7d77d542,0x6c76648a),LL(0xa16759eb,0xae9936fc),LL(0xd9556eeb,0x239c8db3),LL(0x1fc23af7,0x3df02c8f),LL(0xd956a664,0xfec894e1),LL(0x2a0d5264,0xaaa92f80),LL(0x14a90b90,0x93d756f0),LL(0x53d197ff,0xb190be07),L_(0x00000163), LL(0x06dc80b7,0x026f8974),LL(0xdd1ea5de,0xbf46ad6a),LL(0x1c416858,0x8f0f1c5d),LL(0x6b3d82ce,0xf587ce4d),LL(0xa459159d,0x3b92c19a),LL(0xb6baca47,0xda5bec10),LL(0x600af3c6,0x40e0b4be),LL(0xc81e2b9b,0x5a9bfcf6),L_(0x000001ec), + LL(0x7d27dadf,0x34d4f088),LL(0x4ff582d9,0x24ab4a3b),LL(0x151ee4b7,0x58c8094d),LL(0xdf116aee,0x1ceee559),LL(0x6c8ad814,0xf6cc0cd1),LL(0x3d13c277,0x9d41dd2c),LL(0x75e0cd1b,0xf318d63c),LL(0xbb0767f3,0x285b2c06),L_(0x000000cd), LL(0x1f3f3f36,0x8f1170f9),LL(0x44906780,0x6338ef63),LL(0x7495e6b8,0x66dcb459),LL(0x3f524ab7,0x7ec63428),LL(0xfd773112,0x86373e0a),LL(0x549575be,0xd2af0e3a),LL(0x9dfc2be8,0x6314c681),LL(0xffa7af5c,0xed6d3f7a),L_(0x0000013d), + LL(0xb01c752e,0x2db47294),LL(0x8755f110,0xefd03152),LL(0x233b9d7e,0x00f8e7a5),LL(0x69c62c75,0x1ee3b5dc),LL(0x551f4471,0xa5e280f0),LL(0xc7dd9d94,0x11042cd8),LL(0x0c2167eb,0xaf8b1437),LL(0x7f4636e4,0x9f076780),L_(0x00000112), LL(0x77598691,0x12f599a0),LL(0xcf61cb84,0x0459d6b4),LL(0x6f27cc0b,0x126405e2),LL(0xbc7fdaf5,0x4a3026dd),LL(0x0cdbba7c,0x658e4a3b),LL(0x25d0b262,0xf2e795bb),LL(0xeec95e90,0xc8766509),LL(0x52259c52,0x0502e5a4),L_(0x000001a9), + LL(0x35410967,0x88f9eb94),LL(0x20da6297,0xae5dfb3d),LL(0x527623e5,0x17557d88),LL(0xc844e99c,0xb4031115),LL(0xb6a57ec3,0x4aff5aa0),LL(0xbe7d0b59,0xea2e84ad),LL(0x1e84a37c,0x947fcbaf),LL(0x048c2935,0x17d978e3),L_(0x00000076), LL(0x93f58bed,0x503171b0),LL(0x7aaa2b21,0x7b8e1c73),LL(0x6261f263,0x620dfceb),LL(0x1e8e8701,0xdb241dd2),LL(0xce453d37,0x74e79c85),LL(0x7db88257,0x6f92bc71),LL(0x5a2566e6,0x6fb9d9ff),LL(0x4ae0bd7b,0xebbcbbfd),L_(0x000001c4), + LL(0xed12288f,0xe57e6348),LL(0x585fa9eb,0xa7abfcec),LL(0xb1b5c7dc,0x12939a1c),LL(0x76c09203,0x48eb6b41),LL(0xc4cc679c,0x2a08cff6),LL(0x4778574b,0x8519401e),LL(0xa530fac0,0x93672b0d),LL(0xe3ed0e37,0x42bda3c5),L_(0x0000011e), LL(0xb789e1a5,0x479a986c),LL(0x85a2af38,0x47001e80),LL(0x5a9c69c9,0x623b25fe),LL(0x05944f83,0xecb76557),LL(0x1f6667a7,0x0cb584d5),LL(0xc3bb24b7,0xdaf97923),LL(0x7f09e6d8,0x00ac1be5),LL(0x6d082075,0xfc0ef0db),L_(0x000001a8), + LL(0xcbbf6efa,0xd82adf79),LL(0x6f9a1c1d,0x1d21df42),LL(0x47e0a609,0x1902bf5b),LL(0x6c507d72,0xd92481e1),LL(0x732e3603,0x4e995e7f),LL(0x0a9f3c6f,0x57c9de5d),LL(0x171e968b,0xace10341),LL(0x27d7d96d,0x8b519973),L_(0x00000108), LL(0x093b8c99,0x68806375),LL(0x999003af,0x0227279e),LL(0x62158e85,0xd41629b8),LL(0x3479df9d,0xa2d25c33),LL(0x3570b3aa,0x61de636e),LL(0x4eb04a19,0x71b3bbeb),LL(0x0ce04d19,0x4c30f9b9),LL(0x15492470,0xb4fa9cb3),L_(0x00000082), + LL(0xcc212f6f,0xf1e3cf43),LL(0x2c575c09,0xdd247286),LL(0x0980e757,0x812b4ec0),LL(0x19cba5ed,0x883e3e28),LL(0x4641e487,0x0bb816e7),LL(0x6c96b70b,0x461f03f5),LL(0x26aaf764,0x918835cd),LL(0xe1cfcddb,0x0c7159a8),L_(0x00000104), LL(0xb11711e0,0x3f7282c1),LL(0xed552895,0x44826000),LL(0x25594479,0x065389a8),LL(0x70867768,0x96f127c7),LL(0x52ffb2b6,0xaa1f5abe),LL(0x0831bd74,0x9739a178),LL(0xf0a510a0,0x6154e726),LL(0x36902e78,0x51eacd77),L_(0x0000005b), + LL(0x42cd0f9e,0x7bbdb010),LL(0xa3fbb175,0x1996a380),LL(0xf39db731,0xbb69e651),LL(0xf3f08146,0xec6679cd),LL(0x8679b0b6,0x60847478),LL(0x90d1ae26,0x883e5a59),LL(0xa5e209ed,0xba61924f),LL(0x3c755c0b,0x53df41de),L_(0x000001ef), LL(0x847c1f82,0x8ace9c6a),LL(0xa1434ccf,0x857d9026),LL(0xad864d4c,0x0ee5b0b7),LL(0xf613e221,0x86a35718),LL(0x91165b2c,0x55984d67),LL(0x080d19fb,0x15401901),LL(0x3389eccf,0xd99a0e8b),LL(0x8b509b98,0x5d25fe60),L_(0x000001d7), + LL(0x7c660e1a,0xfd68d650),LL(0x85aa25df,0xe307472b),LL(0x3713d00e,0x3afed55d),LL(0xc091f93d,0xecc3137e),LL(0xa9f9d1a2,0xa3d44f8a),LL(0x32a1cdec,0x4344089c),LL(0xc8d64b46,0xe3575142),LL(0x11bd5244,0x0baf1702),L_(0x000001f4), LL(0xe4cb9635,0x8fa8648f),LL(0x3dc52f80,0xdf8e13dd),LL(0x058fc1b8,0xc1ab282d),LL(0x3abf2a6c,0xf290d505),LL(0xfb0841a1,0xea29f4f4),LL(0x3d94894a,0xb691fb1a),LL(0xf7a1cc29,0x0da25d00),LL(0xfc4326f1,0x3439b98c),L_(0x000001fa), + LL(0x1b3c8c62,0x1ffd1435),LL(0x58c3116e,0x54a96e6c),LL(0x90a8d92e,0x623a9ece),LL(0x891efe6b,0x66715556),LL(0xe72489c2,0xd3bfc0b3),LL(0x5b00b58c,0x8cf3d04d),LL(0x06c601e4,0x71f460fb),LL(0x26e3ef6c,0xab622fd1),L_(0x00000122), LL(0x626e2af2,0xa51ff90c),LL(0xec49ef66,0xd3f82493),LL(0x704277da,0x9f0e6e8a),LL(0xca17ce54,0x80cb9b26),LL(0x62bbba3d,0x39fb568b),LL(0x7ff82aae,0x978a0c13),LL(0x9cbfd867,0x6bd90fa9),LL(0x07be1717,0x35fbb751),L_(0x000001d2), + LL(0xd809dd1b,0x12107a4a),LL(0xc16d6592,0xdc70467d),LL(0xf8d5c83c,0x273b2243),LL(0x596e052c,0xeab68bd9),LL(0x8aaa0dde,0x11cdb329),LL(0x39baf3e4,0x65459e78),LL(0x004f292b,0xc654b1e0),LL(0x4da9e734,0x3bae514a),L_(0x00000186), LL(0x1ee009d1,0x45bc8ab1),LL(0x24563a0d,0xde252f6e),LL(0x04322137,0xd1d577ae),LL(0x29c5297d,0x554880c2),LL(0xf29e1a9e,0xfc0d4b7e),LL(0xa08f14c2,0xe648399e),LL(0xfd1007c8,0x23bed899),LL(0x2a7303ca,0xf8a75610),L_(0x000000b1), + LL(0x2e60230d,0x883ed27b),LL(0xf8aee1ce,0x57a9715e),LL(0x9d84b9da,0xa58cae2e),LL(0x59971acb,0x62ef042b),LL(0x5b1190ec,0xa8c70cf9),LL(0x1ecd90c6,0x1e5cf5d1),LL(0x0a20a7a4,0x3aec3e16),LL(0x7baf5a6c,0x8a25f66e),L_(0x00000149), LL(0xf18a3f41,0x0debc30a),LL(0xc54674cb,0x2dbc3b83),LL(0xc265a6e5,0xa8033fd7),LL(0xf450415c,0x2a50f527),LL(0x6adf277c,0x81475ec6),LL(0xff0d3a36,0x5f2c676b),LL(0xdffe6c53,0x0c1f159c),LL(0xf5ad7f78,0xa68f5906),L_(0x00000191), + LL(0x00058fc2,0x8bb729ca),LL(0x48246fa7,0xff563f60),LL(0xb23219d5,0x8c64a3a5),LL(0xf34e49ac,0xe82036c1),LL(0x9d2397f6,0xe392c964),LL(0x58216601,0xbeda885c),LL(0x3d7f9573,0xe289f5e8),LL(0xb1917dc6,0x15f32b69),L_(0x0000009e), LL(0x6d9791a6,0x37eb92d1),LL(0x066237a9,0x7995f34e),LL(0x764ae778,0x8d994f01),LL(0xde5ca0e3,0xaff07e24),LL(0x199bd467,0x544454f0),LL(0x4bf2e809,0x603eb80f),LL(0xe5054850,0xc95e16ef),LL(0xa4a6c672,0x0af00fa9),L_(0x00000060), + LL(0xc8db0700,0x32a1b788),LL(0x45d0d169,0xc842373b),LL(0x87531f66,0x674b4407),LL(0xdbe71b0e,0x8189664d),LL(0x42dd8323,0x7cdb6aec),LL(0x8df2c5a6,0x86c397bc),LL(0x86388fa4,0x110db0bc),LL(0xf6d18eb8,0xa1af0086),L_(0x00000118), LL(0x23e50391,0xa2dcf957),LL(0x6ae2419f,0x1c7362d5),LL(0xac9caab7,0xd238a731),LL(0xbef3c44d,0x28f6d7ae),LL(0x5c3ea7d4,0xdacef1fe),LL(0xd654307e,0xb31d909e),LL(0x01625227,0x6d2db310),LL(0xb2421d90,0x923e60f9),L_(0x000000b4), + LL(0x4083d41b,0xf3bbcb07),LL(0xe8c0bfc8,0x91274928),LL(0x9c0b763c,0xc81b7765),LL(0xbe1076e4,0xbf368625),LL(0x9568943e,0x8c2112ae),LL(0xb0cd4c71,0xf70e5fb5),LL(0x8fe2ec7e,0x76a5c64b),LL(0x867a5527,0xdac3d4b5),L_(0x0000016c), LL(0x3faf5b44,0x41c29ed5),LL(0x50ffd8e6,0xcb155068),LL(0xa9b2855c,0x28ae527b),LL(0x9d8e8d01,0x2a092960),LL(0x5cbf1edf,0x66ffe099),LL(0xb212b2a8,0xd340e610),LL(0xfc0600f9,0xa2f5aba0),LL(0xb08ba7b8,0x10dafa5b),L_(0x000001a7), + LL(0x0ca526cf,0xa9ed0f7a),LL(0x125b33af,0x1dd685ac),LL(0x516f5290,0x17e7ff8f),LL(0x927c416e,0x720475ad),LL(0x0fc77cc9,0x67e1e919),LL(0x3aec0717,0x6652fcb3),LL(0xcb2653df,0xd80f0d48),LL(0x8bf16720,0x98cc486c),L_(0x000000c0), LL(0x08404c44,0x7f43e4e4),LL(0x9c42e337,0xdefb2272),LL(0xc3b37e10,0xdaf241ae),LL(0xc795c866,0x0a07c892),LL(0xbf4d3079,0xb2425f5f),LL(0xa5db075b,0xb7cb5830),LL(0x0875f161,0x93c95089),LL(0x91cad664,0x21f9ae65),L_(0x0000000b), +}, +/* digit=18 base_pwr=2^90 */ +{ + LL(0xc3797559,0x266f6975),LL(0x22dfb9d9,0x599f544c),LL(0xdb081480,0x839c2be6),LL(0xeb8ec462,0xb5cdf12a),LL(0xd9d49cd3,0xa917fb29),LL(0x96146a8b,0x233b216f),LL(0xd936c0b8,0xf0abf1a9),LL(0x8c8a45a2,0xbb59a787),L_(0x000000fb), LL(0x0f5dd64a,0xd1625aef),LL(0x056528c2,0x5ff56fc7),LL(0x9b293d67,0xa5c323cb),LL(0x02b295cc,0x4d697cbd),LL(0xbc712910,0xe4eb4b02),LL(0xc1e4d83f,0xa4e9327c),LL(0x9c23cdd4,0x5af46cdc),LL(0x94640699,0xac05353f),L_(0x00000006), + LL(0x6147414f,0xaed9435d),LL(0xbd16db82,0x62ff16a9),LL(0x3b07e71a,0x52dcaf4a),LL(0xf9456ee7,0xea0d3e3f),LL(0x1d78dd65,0x64901fef),LL(0x31145bf8,0xbc9f4225),LL(0x366fd367,0x81cb13ee),LL(0x290083f4,0xbc209fe6),L_(0x00000193), LL(0x5232088e,0xc5e887c8),LL(0xcc7b3f38,0xf005e149),LL(0xe1bede78,0xe8c89874),LL(0x2dfeaf32,0xcb4e28c7),LL(0x3cb0a4d9,0xda48c711),LL(0xaf7daba5,0x1fe289a0),LL(0x0d3633ab,0xbd5d0dc0),LL(0xc0b05c86,0xefd7c801),L_(0x000000dc), + LL(0x85849a1c,0x74861653),LL(0xe1eb1a35,0xaf98abe0),LL(0x53b40a6e,0xf083ec36),LL(0x74acbc0b,0x0b5a921d),LL(0x28a65b06,0x5764e30a),LL(0x5588eb5e,0x62277d5e),LL(0xc8da671e,0x39cae462),LL(0x53bbf492,0xc9d38ca9),L_(0x000001ab), LL(0xa6baa014,0xd98a134c),LL(0x31e23fcb,0xdbe32368),LL(0xa8c7a352,0xef82abaa),LL(0xa98ba793,0xcbb55844),LL(0x1a07e161,0xaf3169a7),LL(0x4991ee4a,0x5cfbe290),LL(0x0c980dd6,0x56a21524),LL(0x1c07cd7d,0x6b70cccd),L_(0x0000000c), + LL(0x07567a7a,0x045155d8),LL(0x351697cf,0xce8bb246),LL(0xcba64633,0xdd5c2900),LL(0x24297174,0xe7f044c7),LL(0x0c3a3851,0x48c830bf),LL(0xd35ff595,0x817a26f0),LL(0x7d923f53,0xd93d8b02),LL(0x2b3dd7ca,0xbc01062c),L_(0x0000000d), LL(0xcc94cbf6,0x986d299b),LL(0xdf6f5c09,0xf89ccb5a),LL(0x5aee26f4,0x18699f82),LL(0xc1b545bb,0x6595e656),LL(0xb0d22aa5,0xa3953fae),LL(0x6abcba5e,0xa9580b4b),LL(0xd4e240d3,0xef465246),LL(0x13b6fdf7,0x97f3ed86),L_(0x00000124), + LL(0xbd979035,0xf355f70f),LL(0xc5a87f52,0x1b7365b1),LL(0x18c500a7,0x2dd3210c),LL(0xf547e418,0xc57a734c),LL(0x391d8bfc,0x928abb19),LL(0x5ec8dcf1,0x3e6991b5),LL(0x7de9b3d1,0x9e25eac7),LL(0x6ebd2b96,0x6a202a22),L_(0x000000d2), LL(0x7d70fd72,0xae0eb8d2),LL(0xf2114ba3,0xb57b0ed4),LL(0xe573b783,0xc65906cc),LL(0x0b2b3d21,0xfac74740),LL(0x9dbb25b7,0x6157bc5c),LL(0x6859f85a,0x21a21340),LL(0x0df614a7,0xbfacee18),LL(0xe4709b5a,0xa37c69b2),L_(0x000000ef), + LL(0x15b4091f,0x31ad054c),LL(0xdccd6f0a,0x57e2c372),LL(0x27698a20,0x9a3a4ffe),LL(0xcc03de8d,0xc754fb8f),LL(0xcd17cdf6,0xa956ebe7),LL(0x7cb9ef2e,0x9d18d38c),LL(0x190daaea,0xdc66f2ca),LL(0x7d783383,0x46f57c30),L_(0x0000012d), LL(0x727ea062,0xfefa898f),LL(0x773b6e5f,0xfbea71e8),LL(0x7460be92,0xc9254b56),LL(0x318caed7,0xc982d8e5),LL(0x7f1a16b2,0x9a4da06d),LL(0x38233ae7,0xe149d876),LL(0x3b8c2af7,0x70fc3c7a),LL(0xec4a5565,0x8d292554),L_(0x00000197), + LL(0x5f08f3c6,0xd02192ba),LL(0x05cbe08a,0xbb7b43e6),LL(0x4a339a92,0x2b4034e6),LL(0x1c6eea28,0x0145dd56),LL(0xb3481662,0x819f74ea),LL(0x5fdab086,0x2d08d669),LL(0x401f4d8b,0x87d855a8),LL(0x71a3977d,0x0a7c167c),L_(0x000000e6), LL(0x787e1654,0x7f231760),LL(0x1dcce655,0x2908ba05),LL(0x10acadce,0xe5a3ffa8),LL(0x60f31016,0xe1211553),LL(0x34c2a1c2,0xfe1b8dad),LL(0xa81d35d3,0xe7230bfc),LL(0x37a78d41,0xcfeaf774),LL(0x6e60ae9c,0x929625f5),L_(0x000001ed), + LL(0x898a64cd,0xda668233),LL(0xc8516d6d,0x1f40d7d1),LL(0x5fb1f564,0xd5a115e0),LL(0x7906c2d8,0x496ac4ad),LL(0x6f4efe00,0x3643f707),LL(0x6c892d97,0x414f5838),LL(0xe34b14c4,0x7c2d83c2),LL(0xc00d08bc,0xb5ea2437),L_(0x000000e4), LL(0xca445165,0x219e2e5b),LL(0xcbf3a3f2,0x118227ea),LL(0xf1a84019,0x6017e2ce),LL(0x58a5f9de,0x1922122d),LL(0xf1ecfa6e,0x98696eb0),LL(0xd3df6fb1,0x54826be2),LL(0x8fefc088,0x02dea006),LL(0x77c29791,0xd7ce374e),L_(0x000000d7), + LL(0xe65ce508,0xa5fec659),LL(0x0b177612,0xdd1bcbef),LL(0xfc84b3bf,0xe4569388),LL(0x984ce0ec,0x163caf98),LL(0x9cc1f201,0x3fa88dda),LL(0x39bac4dd,0x6c254803),LL(0x70562e22,0x75a93dd1),LL(0x2cbfcecc,0x16ebf295),L_(0x0000006d), LL(0x0ce600b2,0xe3dc17db),LL(0x5b060f2c,0x5399ddf1),LL(0xd74988f6,0x83e38dff),LL(0x2b96b9f4,0x87e221d8),LL(0x2c298bc9,0x0753b765),LL(0xd6bd45f4,0x27e5b1a1),LL(0x18ca1da9,0x41853811),LL(0x7849f1e1,0xf9e739bd),L_(0x000001fa), + LL(0x226b8a15,0xdb333954),LL(0x4e3975a7,0xaa52c0ba),LL(0x74176c01,0x1fd9d014),LL(0xa1a2b6b0,0xf61b81e1),LL(0x28415db7,0xd3614b3a),LL(0xa0f6000e,0x9e00e5a5),LL(0x5cf34986,0x9efe446d),LL(0x0b69d383,0x845d741b),L_(0x00000121), LL(0x2d0c1e46,0x1f77c4cb),LL(0x553d358a,0xe9f129ca),LL(0x071ebad1,0x088b2769),LL(0x8eb8c2d8,0xf3219a51),LL(0x877b3a25,0x17c5431a),LL(0x9c4b8adf,0xeac2ff93),LL(0xffc69d68,0x23d158f5),LL(0x2e0840d4,0xbd9e89fa),L_(0x00000048), + LL(0x6d2a4e6a,0xb45f9376),LL(0x6e78c290,0xbff42b2f),LL(0x5fb9af74,0x2af23d44),LL(0xaebdf547,0x7deaa238),LL(0x93a6597a,0x25df5576),LL(0x5dcbd040,0x6f6b456f),LL(0xa9249467,0xb63da0b0),LL(0x7f719d63,0xb3143ac8),L_(0x00000173), LL(0xda677478,0xcb218f05),LL(0xb119afe4,0x3e0b4052),LL(0x816ad70e,0x342ea279),LL(0x14498c92,0x7acbb776),LL(0xc4e09c12,0xbf8e66de),LL(0x0f290835,0x3edfdd90),LL(0x8ca19f41,0x794255ee),LL(0xa1bde671,0xd1cabfcb),L_(0x0000002b), + LL(0x4848eb20,0x80f21ad1),LL(0xde0fb7d6,0x8e50bcbd),LL(0x4c1119fd,0x83d6f607),LL(0xf9f2e435,0xd9961c79),LL(0x765361f1,0x625f26bb),LL(0x008a463e,0x2b47c8db),LL(0x87cd134f,0x328c3977),LL(0x36eea7ef,0xb645529b),L_(0x000000b4), LL(0x3d664722,0xdfe036fb),LL(0x206e887a,0x67daf1eb),LL(0x72f017c4,0xf7db19b3),LL(0xad33a99e,0x06ea7ba8),LL(0xf55c0da8,0x14bd637e),LL(0x9b12c024,0x59864973),LL(0xd282f3bb,0x55feed3d),LL(0xce69b372,0x299c53ec),L_(0x00000071), + LL(0xdf10c34e,0xb43675e3),LL(0xb8ad4110,0x19590538),LL(0x801f28aa,0x239ed388),LL(0xc602d7b2,0x3a5a6ad4),LL(0xbdc9fad5,0x62b5ae49),LL(0x135d222c,0x5042d74a),LL(0xc3f94c1d,0x28ba3dd9),LL(0xe0a1ec48,0x5abc7310),L_(0x000000f3), LL(0xceb5a088,0x77f171c2),LL(0x8f737348,0x31242fc1),LL(0xdbca643a,0xe44bcbd8),LL(0xcd573afd,0xbad62d6a),LL(0xc4c9f268,0x5d49ae75),LL(0x98fad2cf,0x9b8f817e),LL(0xd8431494,0x96bb2753),LL(0xc49e4432,0xa3c623a8),L_(0x000000d9), + LL(0xa4f9f22d,0x2ccfc8e8),LL(0x4cd46e2b,0xa48136e4),LL(0xa95491dd,0x8818327f),LL(0x6393487e,0x7a069a28),LL(0x98ab19e5,0x5c879344),LL(0x02491e0c,0x82e60002),LL(0x2abf1999,0xc0e10b00),LL(0x82724899,0x06d9bded),L_(0x0000010e), LL(0x4897df79,0x688c2193),LL(0x43479e7f,0xc705deaf),LL(0x5b79a04e,0x068f9205),LL(0xb2c04a79,0x3a42239d),LL(0x0245715d,0xa99eca86),LL(0x55284faa,0x9f983742),LL(0x07d9bbaa,0x5fe3bb74),LL(0x5a19431d,0xaebc434c),L_(0x0000013f), + LL(0x99e63ac2,0xac6175c0),LL(0xcb3bdf62,0x0d4bf222),LL(0xd9e5f622,0x83e1cd77),LL(0x4ca3e4a6,0xde0dce61),LL(0xf80cd49b,0xeac1e293),LL(0xf190c10e,0x901aa7ce),LL(0x30eda1d3,0x25f2f0a6),LL(0xa509dab1,0x7ead9599),L_(0x0000001e), LL(0x0ab974ef,0x9b6b41ad),LL(0xcc8fc9d8,0x20269236),LL(0x5472afda,0x81034020),LL(0xb7eb7c83,0x26dba78a),LL(0x7c59479d,0x81e829eb),LL(0xb0ba6216,0x1549f5f2),LL(0x843a5c45,0x18302134),LL(0xa2709e3b,0xf629ef4c),L_(0x000001d3), + LL(0xbde5f81a,0x8f81e9a3),LL(0x7f4da051,0x9720014a),LL(0xa02bf073,0x9ff1f457),LL(0xe074553e,0x3a46ea6b),LL(0x64351eaa,0x7e32f0dd),LL(0xd22b22b2,0xb488462f),LL(0xcafc2c23,0x566dddda),LL(0xfb75908f,0x6e12535b),L_(0x00000182), LL(0xe8987332,0xfe94dfd9),LL(0x4f63b44f,0xf9b91bc6),LL(0x0dbd772b,0xecb77f43),LL(0xd580392a,0x2fb67ddf),LL(0x1ddc69fb,0x2fdb69c9),LL(0x9ea2314d,0x754b9b9f),LL(0xe624f23e,0xf3f2e9c2),LL(0xc6e677e1,0x734230a3),L_(0x0000016b), +}, +/* digit=19 base_pwr=2^95 */ +{ + LL(0x14f438f4,0xd13ff0c3),LL(0xb8e45f22,0xd55796ec),LL(0x3dd2e2bb,0xf830ab87),LL(0x9eb71d33,0x14a36478),LL(0x83167e0b,0x46513aa4),LL(0x793502c2,0x03e86d3a),LL(0xb0fe98cb,0x4eb2db2b),LL(0x0404a0ec,0x002c10d7),L_(0x00000162), LL(0x8f4384c5,0x0cd19a5c),LL(0xc19b3257,0x1c33b468),LL(0x9210942f,0x29ccbac3),LL(0x36048a2a,0xd4ffa97d),LL(0x4f69ef5f,0xcd6b0a67),LL(0x82d0ece5,0x13229739),LL(0x4bce1b8b,0x491493bd),LL(0x4d6596bf,0x6c17983d),L_(0x0000018d), + LL(0x560db5e8,0x1117f6d4),LL(0x8e19b583,0x1106059c),LL(0xe8232c57,0xc78f908c),LL(0xd0f09782,0x4a24aa92),LL(0x8bd0fcb6,0xd766becf),LL(0xf59977e3,0x155f53d2),LL(0xfa9a727f,0x49389ae2),LL(0xff877e92,0x595c1920),L_(0x00000027), LL(0x9985d510,0x3b4e5204),LL(0xe4788dc7,0xaa68342b),LL(0x8a0ca8e6,0x7b14f89c),LL(0x66874892,0xf19eb3e8),LL(0xe17375dd,0x5e5f8b7c),LL(0x652a41d1,0x2912af54),LL(0xa86a7275,0x5ab9a777),LL(0xbaf706d8,0xa88b103b),L_(0x000001a5), + LL(0x8ca944bc,0x4852fbcb),LL(0xbefff557,0x0369a497),LL(0x3e3736fb,0xe2bb7551),LL(0xd21615dc,0xe6d3b727),LL(0xe319eb4c,0xe8de5f7d),LL(0x48fe4856,0x592a3396),LL(0x7516380a,0x80dc9aef),LL(0x51f52af0,0xddc54f51),L_(0x0000001a), LL(0x98c6421f,0x38523c52),LL(0xec51f4f2,0xf58cc95f),LL(0xbc6c8082,0x36ef370c),LL(0x7bc605a3,0x8ac270e3),LL(0x83d78da4,0x0412498a),LL(0x6de54abd,0x66b38131),LL(0xdb62d8c6,0xe06d3c3f),LL(0xf5e45a86,0x4e3ea929),L_(0x000000f5), + LL(0x4517d395,0x53d2cbb1),LL(0x5b733d69,0x44a3ef5b),LL(0x472126ff,0xee076565),LL(0xa4a1334d,0xb26c37b2),LL(0x12573d17,0xb5b29517),LL(0x129c2c7a,0xd328148c),LL(0xa2c72b08,0x08907f5a),LL(0x1d10e103,0x4f66b18b),L_(0x00000085), LL(0x5e159666,0x9359d888),LL(0x4827d5dd,0x0281b6f1),LL(0xa475f3a4,0x6b19bc4b),LL(0x2eef4469,0xdc6dfbc3),LL(0x782b50db,0x9ef4383e),LL(0x0583236a,0xd7320845),LL(0x7767db3f,0x0dd190b0),LL(0x3c0278a0,0x96afa315),L_(0x0000002a), + LL(0xb03b1675,0x97e68374),LL(0x1c499646,0xae2f3284),LL(0x74508785,0x0255084b),LL(0xf1921ad5,0x6ea40714),LL(0x2aabd8cb,0x516433de),LL(0xd1c8abdf,0x5d2d8ded),LL(0x8f5d7b6f,0x421e5a19),LL(0x2d6ae9c5,0x9eaf7e7b),L_(0x0000018a), LL(0xd9a89463,0x4310dcf0),LL(0x0122d1b0,0xd2489f91),LL(0x4e7b58d3,0x655fdd5d),LL(0x40c4379c,0xc7862d42),LL(0x6da55b4a,0x975cc64c),LL(0x1e1d3862,0x84484f68),LL(0x301cfa3f,0x2a16eb0a),LL(0x20ebbbdb,0x293922e4),L_(0x00000132), + LL(0xc0ba905f,0x5b3d8d39),LL(0x62a268ee,0xea2a0a44),LL(0x094457cb,0x80f032f3),LL(0xab36ceff,0x8790739b),LL(0x21b0fdf3,0x40209bce),LL(0x38249dc8,0xf0c1c8e0),LL(0x213ecb4d,0x70c51d81),LL(0x2b025e0d,0xac4a363c),L_(0x000000a8), LL(0x493bb32c,0x798bfbf2),LL(0xf622fccb,0x28838277),LL(0xc2594827,0x5cbe5b67),LL(0x2c07c4dd,0x2c4c703c),LL(0x1c19526a,0xdcd0df4c),LL(0xed390177,0x57a743a1),LL(0x3a4c5274,0xac32bea6),LL(0x1c302e78,0x133f075d),L_(0x00000097), + LL(0x18bc0976,0xa3094433),LL(0x7cfa690a,0xca509300),LL(0x660a165b,0x9645aafc),LL(0xdcef6d7c,0xd90c6f35),LL(0x07a41bbc,0xad0dd448),LL(0x702e476c,0x5c881f7f),LL(0x2185a821,0x9c91260c),LL(0x0622b914,0x6ca1be63),L_(0x0000001d), LL(0xfe64e60a,0x59c3b83d),LL(0x192bd9cf,0x5bdb24df),LL(0x69659379,0xa9cbc111),LL(0x437b31e8,0x2954601b),LL(0xb7b5589c,0x13c392f3),LL(0x97064b9e,0xf1845fb5),LL(0x8d803336,0x7ea9c980),LL(0xea2d2221,0x36a667ca),L_(0x00000196), + LL(0x23fc5003,0xa9841f43),LL(0x62407436,0xc6f35f8a),LL(0xfc260a1f,0xf5e9286e),LL(0xe74c4b2d,0x504bfafb),LL(0x7cb3568b,0x3548e504),LL(0x1af9dbcf,0xd92aaad7),LL(0x85e423c5,0x2d182410),LL(0x94d1d884,0x1fe174f8),L_(0x00000105), LL(0x9975b2a3,0x55f78348),LL(0x0939eec5,0x92b31a41),LL(0x23255263,0x4a1b7bc2),LL(0x65a25c26,0x283464db),LL(0x08fc1aed,0xecd1a9b7),LL(0xea335c70,0x90a7a2a0),LL(0x9f14ffdd,0x7fd21f2e),LL(0x9566dadd,0x36ebafb4),L_(0x000001e5), + LL(0x578b8f88,0x2bbf7009),LL(0x8c4ed5a7,0x30e7b9aa),LL(0x30aeacb4,0xe07a93f1),LL(0x58d02f17,0xbcf5fa42),LL(0x7c6f83ed,0x228f3e47),LL(0x91d5ba0c,0x815eef51),LL(0xe37c3b79,0x0528cf82),LL(0x507b5dce,0x7d4e9bf2),L_(0x000000aa), LL(0x5a682d56,0x14eac39b),LL(0x72ce45fa,0x90b36bc5),LL(0x5f62d8bb,0xadbc91ec),LL(0x9d5e2385,0xfae2e6dd),LL(0xef20a2d0,0x380bdc5e),LL(0x9c9037ea,0x2d48d188),LL(0xdd58b76d,0x6a4ce00a),LL(0xb6e3d5d7,0x2f506090),L_(0x00000135), + LL(0xfffa5757,0x64cbfd55),LL(0x1f1c3ae1,0x8a854545),LL(0xd0be9705,0x7272e007),LL(0x2a8f4c49,0x254138ed),LL(0xc97ed736,0x15e864c7),LL(0x83df5162,0xb624fc1b),LL(0xf4114fdb,0xcc0313aa),LL(0x8c7f0423,0xa6edea7a),L_(0x000000a8), LL(0xa4ed76ab,0x3e74f599),LL(0x8b381bfe,0x25d1f92d),LL(0x9e406956,0x06a5a359),LL(0x869bdf5e,0x5afaf671),LL(0xaec86f62,0xbbcc12cd),LL(0x0dd1d724,0xda751689),LL(0x21630603,0x409b6925),LL(0x1f15a18b,0xa4edb254),L_(0x00000191), + LL(0xc7b9233c,0xff6a30c8),LL(0x6ecefcff,0xf3d40688),LL(0xfad25722,0xc51d4aa6),LL(0xf9c6f8fa,0x3fae03f2),LL(0xc7864470,0x5ca7328e),LL(0x4201ca64,0x4b3ff739),LL(0x08cad454,0xb2721cee),LL(0x43bf4523,0x4594a3d1),L_(0x00000197), LL(0xaf0e573e,0x2a8f8fba),LL(0x3f5e67f8,0xae4ce46a),LL(0x616a72e1,0x906994c7),LL(0x3a7f4aec,0x055b94c2),LL(0x8bb69b90,0x67e39f74),LL(0x4eead34a,0x7a7cada3),LL(0xf92b70d7,0x9b22abe2),LL(0xca616691,0xa5cfde92),L_(0x00000052), + LL(0xd5e1c4e6,0x7ad659b7),LL(0xe2f2a298,0x3f7f338d),LL(0xaaeb06f1,0x6d9b55a5),LL(0xa60e84f2,0x30c6f8b9),LL(0x9d105631,0xd017d58e),LL(0xfa41e760,0x9e20b973),LL(0xb2f4acf3,0x840eaafd),LL(0xec9c6ab5,0xcbac2501),L_(0x00000065), LL(0x22f4549b,0xb140b897),LL(0xc2510a98,0x3e099225),LL(0x9117bbe6,0x18af31b1),LL(0xba7147bd,0x68bb5c46),LL(0xf0f540e3,0x29d33114),LL(0x1c9aeacf,0xe59588a0),LL(0xeb2d0e67,0x29ef0e25),LL(0x4bb1b8d0,0x4c60558e),L_(0x0000014b), + LL(0xfaf6783d,0xd0223b63),LL(0x4dcebc43,0x6472ecb7),LL(0x6ff8245b,0x1d3fdb09),LL(0x3439a621,0xa67bac9a),LL(0xd009a2d7,0x94c65387),LL(0xf330caae,0x19a02e17),LL(0xa6f7e5f9,0xb2396a55),LL(0x5fef60ca,0x08eb7e67),L_(0x0000007f), LL(0xefd6a160,0xefa0d1c9),LL(0xbd7c0837,0x2ede1f4e),LL(0xfcadb9c0,0xf4756366),LL(0x47882726,0xcd42f3e5),LL(0x5c040ce2,0xa61b16f9),LL(0xdb84713f,0xc4ef07e5),LL(0xe43320a0,0xecb2b8ce),LL(0xac70be15,0x4eb58c90),L_(0x00000124), + LL(0x0457915c,0x89a3e1eb),LL(0xe08cc88e,0xed12c670),LL(0x89133ab9,0x1faeab1d),LL(0x15d9bc0c,0x3c4250f6),LL(0x881504d6,0x084c8e8f),LL(0x3ead62cd,0xaf76dbe5),LL(0x49cfac6a,0x85bf1dcc),LL(0x007ea0b8,0x8816bfc7),L_(0x0000016a), LL(0x47472352,0x123835c7),LL(0x80692fa2,0x67bab29e),LL(0x8379c2a8,0xc2ecca00),LL(0x9065aafb,0x32da9779),LL(0xda605d2e,0x421bbbfb),LL(0x12432283,0xbdc2e115),LL(0x9c126b9d,0x437a9d89),LL(0x7ce3f8d6,0xa0a2a746),L_(0x000001f8), + LL(0x5b6dacf5,0x7ae17558),LL(0x1b0157e6,0x314ffaf0),LL(0x03819c6a,0x9d2d7013),LL(0x0e14b9b1,0xc8c2b439),LL(0x91c83635,0x9f636f0b),LL(0x98fed93b,0xe579e47a),LL(0x23b6808c,0xaeb0f547),LL(0x8d044288,0xcd184b66),L_(0x00000151), LL(0x891f32da,0xf965270d),LL(0x655e3634,0xe11f64ad),LL(0xacfd673c,0x1b496453),LL(0x68fb4507,0xdf15820b),LL(0x64f7f419,0xd816e8bb),LL(0xfdaf2edc,0x0858d605),LL(0xd9f619c9,0xbbe07451),LL(0xb9e75198,0x3e93a602),L_(0x00000116), + LL(0xe000c97f,0x5fa6cdb3),LL(0x595ed0a5,0xfd5fcd60),LL(0xa02a23c6,0x76e522bd),LL(0x61844a1d,0xbaf8c003),LL(0x0c6c179e,0xd0a47af4),LL(0x6aa1a6cc,0x71e2a115),LL(0x4eb00620,0x2ca1b0fc),LL(0xc1c5314a,0x3dce0bb4),L_(0x00000094), LL(0xcdc04837,0x5b4e8123),LL(0x242a32ef,0xa7d67834),LL(0xccb0fead,0xc2a3bc3a),LL(0xe65ed32f,0xe7119407),LL(0xab8b44e6,0xaeb1712a),LL(0x5a5977e9,0x39ce4f89),LL(0x708cfeb0,0xa19d43ed),LL(0x4957cd1c,0x0562d425),L_(0x000000ce), +}, +/* digit=20 base_pwr=2^100 */ +{ + LL(0xc29a3fb6,0xcfe3b9a1),LL(0xca033fc3,0x8c5a528b),LL(0xb4bc3e3f,0x23396cd7),LL(0x4bd134e2,0x39c3eb8c),LL(0x565224c7,0x25548c0a),LL(0xf01471ec,0xb0fb17f6),LL(0x5fc65797,0x18aee101),LL(0xc4d69b6d,0x07d23448),L_(0x0000005e), LL(0x0074be70,0xa14b550a),LL(0x52425042,0xf8e6b95a),LL(0x3f438c42,0x0174e981),LL(0x81004aa1,0x34ae4351),LL(0x510cd9a8,0xb1b67e29),LL(0x8cdf2105,0x41438bad),LL(0x8ec5ba91,0xc64d1130),LL(0x00e8c08d,0xc01f2343),L_(0x0000011e), + LL(0xb2625d11,0xdcf3505f),LL(0x9962934c,0xc306874b),LL(0x981e8fcd,0x02704410),LL(0x2bd0a650,0x249eb349),LL(0x6a534a84,0x6b6bb40b),LL(0x0c32e132,0xbe5d2914),LL(0xcb2ca52e,0x029956b2),LL(0x77c72251,0x01849b8c),L_(0x00000029), LL(0xfe2b4e07,0x6314442e),LL(0x4431d884,0x66618e79),LL(0xcd3eeea2,0x6a5a933f),LL(0x64415965,0xbda24f06),LL(0x522dc52f,0x82f45dda),LL(0xd412542f,0x0e5075c9),LL(0xff34a66e,0x9a2aba0f),LL(0x512c4a1d,0xb89bbd69),L_(0x000000f8), + LL(0xa3dc5b94,0x9872016b),LL(0x2889ccb7,0x1df5e18d),LL(0xe0129254,0xdbcaebf5),LL(0x4cd20b4c,0x63d6c33a),LL(0x1f301089,0xc46a1dad),LL(0x755fc0db,0x2c0e39b6),LL(0x6ef9e694,0xac07fa12),LL(0xd500d36f,0xd54aa805),L_(0x0000013e), LL(0xf90b5e7b,0x65e8dbbf),LL(0xeb8ecb37,0xe491cc2d),LL(0xd314c068,0xd31ab995),LL(0xe810513a,0xcca181b4),LL(0x9b50dc0f,0x0c1e0526),LL(0x89302958,0xb6453c85),LL(0xde5a7d26,0x3c9a98b2),LL(0x6f7a7718,0x3cbf6b38),L_(0x000001a3), + LL(0xd7d861fc,0x7b3c184c),LL(0x5c425a73,0x14a6e3b9),LL(0x85d4a651,0x532f514e),LL(0xb665bb45,0x8b87e598),LL(0xa66a39b0,0xdbcbbabb),LL(0xa46208db,0xa64b561f),LL(0x9520864b,0x05569250),LL(0xde8b31e2,0x46d15281),L_(0x000000a8), LL(0xdd1bb6a7,0xe76a8c5d),LL(0xc0f66932,0xc6ee633c),LL(0xc546bb80,0x911f68d0),LL(0x828f4e0c,0xc213a206),LL(0xeb2a4276,0x04a16b2c),LL(0xcbe98cb2,0x38c09aa1),LL(0xbcc1671d,0xa6f3ebee),LL(0x7a684ba9,0xe784232c),L_(0x00000088), + LL(0x2ca34639,0x14b160b2),LL(0xe3cc4351,0x57f2d520),LL(0x8707011b,0x1d30ff78),LL(0xb1a346a6,0xd69eda68),LL(0xa84618b8,0x4008115f),LL(0x06f520c0,0xfeeecaa8),LL(0x6a14e30d,0xcc7e0843),LL(0x68bc839c,0x9aec405f),L_(0x00000113), LL(0xb3ae58e3,0xd35d5af6),LL(0xf29bb951,0x38625415),LL(0x552cd755,0xb49fd087),LL(0x87ef7e8a,0xde9ebd20),LL(0x76b067b5,0x110309c1),LL(0x1ece1e74,0xb224505a),LL(0x91a5a2d7,0x03ba9629),LL(0x79263dad,0x31e7eb88),L_(0x00000193), + LL(0xa71b7e01,0x5ba885f2),LL(0x915b6877,0xb98305b9),LL(0x53769a90,0x10c19b27),LL(0x8d87ac0d,0x8f7acf63),LL(0x583c77c1,0x64d02af2),LL(0x6e04d239,0xe5be9202),LL(0x8f85294d,0x3a5a3099),LL(0xbb22f880,0xf9b4d2d2),L_(0x00000113), LL(0x150daae0,0xf1b9b104),LL(0x500967b9,0xe48eb13c),LL(0xec3e5c4d,0x895cf9cc),LL(0xb6158629,0x7d90f3f7),LL(0x8aa201ea,0xc0cda29f),LL(0x0c9a6e88,0x4c0d7015),LL(0x97d1c62f,0x56ee70bc),LL(0x4fd0f68d,0xedd26b8e),L_(0x00000130), + LL(0x413edbb8,0x792ccd3b),LL(0x1564f76a,0x72527a7c),LL(0x2778d6f2,0x45b55b68),LL(0x67ba3cee,0x6e138f21),LL(0x5d96d43a,0x38c932f1),LL(0x63638065,0x4892afee),LL(0x6ed7c45d,0xb82b82f0),LL(0x287b4614,0xad914fee),L_(0x00000089), LL(0xb0953f4f,0x5150d18c),LL(0x107a5e99,0x67e23c2e),LL(0x71a733f6,0x2e8bfba0),LL(0xca46066c,0x1d6c6108),LL(0x8cfb4987,0x39bb5a64),LL(0x48164ece,0x0cf34f51),LL(0x336996f4,0xe99b9250),LL(0xd6e08146,0x773a9c69),L_(0x00000183), + LL(0x4008e517,0x7a6c13d1),LL(0xc5d69ab8,0xc4597b07),LL(0xc66db12e,0xdbd74a68),LL(0xca40dec9,0x4579d719),LL(0x5a617fff,0x13172539),LL(0xe3835876,0x09e3b946),LL(0x2478942d,0x42d20c85),LL(0x82ecbef7,0x34ea5879),L_(0x00000124), LL(0x5f589886,0xc9ae2bbd),LL(0x8ba22c3f,0x2126ee97),LL(0xe212b5ab,0xff875595),LL(0x389b739e,0x9d6707a2),LL(0x263595af,0x87d12dd7),LL(0x92670f97,0xb014c330),LL(0x00678803,0x692f506a),LL(0x060764da,0x2c056f67),L_(0x000001b4), + LL(0xa7daee3e,0x99e57328),LL(0x3e8ff84f,0x8d930fda),LL(0x140c8ff0,0x29def859),LL(0x514bf396,0x6dc6556a),LL(0x917c0e6f,0x1b3aa174),LL(0x2fd52f94,0xbc6b4711),LL(0xec0486f8,0x10164218),LL(0xfc4d6fdb,0x53878087),L_(0x000000df), LL(0x143ce621,0xb9fa1174),LL(0xb63c7e44,0x5ae8a4f3),LL(0xa5424b58,0x8d2b2e47),LL(0x1058f848,0xc653cf99),LL(0x5d72c078,0x9ab7f092),LL(0x5a469db8,0x0eb39c94),LL(0xb34488b0,0xd4f2904e),LL(0xff8c9e4f,0xf8df541b),L_(0x00000041), + LL(0x5b0f1fb8,0x0aa30019),LL(0xcd7cdff0,0xb19b5360),LL(0x2a32083a,0x1a886b29),LL(0x6fd83895,0x3e954ca3),LL(0xfc9e4bd7,0xf6dfbbe3),LL(0xd49474b0,0xbe5e429b),LL(0x6cfc9bf8,0xb282ba89),LL(0xd4f6b8c7,0xffc5e2b0),L_(0x00000099), LL(0xf0239d41,0x54cf1a20),LL(0xf833aa3b,0xbe4f75ce),LL(0x667c55f3,0x210d085a),LL(0xf20673c4,0x90bc983c),LL(0xe6d37945,0x2b933ece),LL(0x7400e31b,0xa73815a0),LL(0x0016f173,0x2d12b0bb),LL(0x04943106,0x2f2ea6cc),L_(0x000000b4), + LL(0xf8f8e68e,0x93b485b5),LL(0xbbae7e17,0x21f99dd4),LL(0xd3ee3730,0x8f41688d),LL(0x869cee4c,0xa7149163),LL(0x7ffdc064,0x1dd2e6f5),LL(0x3e9c9ee6,0x3e5343f8),LL(0x1bea26ce,0xd599de2a),LL(0xc95e92f5,0x12eb8e21),L_(0x0000004b), LL(0xc38ccf06,0x17914783),LL(0x5b85620d,0x3066f430),LL(0x8a55dcb5,0x73026026),LL(0x8691669e,0xe37b2f3d),LL(0x6c8abb34,0xb0b3d5e7),LL(0xe607781f,0x1d40a38b),LL(0xbd4e966b,0xa4bb9c3f),LL(0xfa1cc8e1,0x52c54053),L_(0x0000019c), + LL(0x61cb5b4c,0xa0ed42bb),LL(0xb44afbfd,0x8be8a8dd),LL(0x0621480b,0xe9c3bbe4),LL(0x3bbd013b,0x3dad214a),LL(0xf63413c0,0xf44a0f7a),LL(0xfedcc70e,0xababaa32),LL(0x0bd6ea84,0xd282fada),LL(0x556a1b66,0x85e11955),L_(0x0000005f), LL(0x2eb9a417,0xd244439d),LL(0xe9ea5bcc,0x6a6b06e0),LL(0xc7910063,0x95bae48e),LL(0x065bc250,0x8f84ff6a),LL(0x70b64554,0x1fab9066),LL(0xe0a46dab,0xe07a85ee),LL(0x8bdd86f2,0x832625e0),LL(0x012550fd,0x518e586d),L_(0x0000008c), + LL(0xafed2f9f,0x24dc0f96),LL(0xe142d2cd,0xa6689657),LL(0xea2d5cc5,0xae481b79),LL(0xd0bdf4f2,0x8a560bee),LL(0x85db3226,0xa5d11f99),LL(0xfd86b657,0x77ab3c4c),LL(0x9ecb073c,0x1fa5f6fb),LL(0x59db760e,0x55d4b8e3),L_(0x0000012e), LL(0x09f0d045,0x3533500a),LL(0x2990659d,0x4eb8eebc),LL(0xe68fe462,0x60e87a33),LL(0x48e37752,0xcd62216e),LL(0x703956e1,0x0dcfead1),LL(0x3a09a4d6,0x491340a6),LL(0x49d2c6bb,0x0b013428),LL(0x50df11ed,0xd9a7913c),L_(0x00000048), + LL(0xd8099e2f,0x2aad5a31),LL(0x5920a298,0xb4992465),LL(0x07aa2d85,0x6aeecdc9),LL(0xe712f629,0x2a16e6d8),LL(0x8639dfb4,0x36815fc1),LL(0x2a477d95,0x0b2f5359),LL(0x7c3ca55a,0x896f9530),LL(0x05522116,0xa9274ead),L_(0x0000008d), LL(0x208b956c,0x65da1fcf),LL(0x1c1f9a0d,0x1534c8eb),LL(0x772539f2,0xb39694db),LL(0xbc2cb67e,0xf14a06ea),LL(0xf6e48c27,0x4aa51441),LL(0xe7141d18,0xbd52c5e3),LL(0x7983136e,0x77a0099a),LL(0xd7f96b63,0xa49e12a5),L_(0x00000083), + LL(0x0a99ca48,0xd8a3e48d),LL(0xb1ee6ff6,0xa6f4001b),LL(0x2ec4e0b2,0x04f03a29),LL(0xe977781e,0x0605bcbc),LL(0x0f8d3aa3,0xa1ff6ad7),LL(0x052c4409,0x13eedc9a),LL(0x211fe699,0x4cc42df8),LL(0xd70f26db,0x9c11a057),L_(0x000001fc), LL(0x00258026,0x08667174),LL(0x7f49a77b,0xa5dfdb47),LL(0x0446d9f9,0x6b646fc8),LL(0x24ebc0c9,0xb269fd06),LL(0x244a494a,0xd5fd906c),LL(0x9c16866d,0x27e6983d),LL(0x545f39fb,0x798c184b),LL(0x5fdefa1a,0x5cbdd99b),L_(0x0000013d), + LL(0x1e365108,0x765f67fb),LL(0x1aae9f80,0xb1b38d14),LL(0x7a9407e4,0xc8424d69),LL(0x9693e7cc,0x91d93e3f),LL(0xba50e7d2,0x385c13c5),LL(0xd94ecd34,0x6fcf73c9),LL(0x80eb0bb6,0x0d598f4a),LL(0x1c4d4c29,0x1f3a1b72),L_(0x00000172), LL(0x3b0fb9a3,0x326d546e),LL(0x4c6d27ba,0xa848cf09),LL(0x09d2dc18,0x46c0e416),LL(0x6f0069ca,0x9231b926),LL(0x1c4aef79,0xbdbead08),LL(0x9d4dabac,0x272ba195),LL(0x8e216ceb,0xcc49b720),LL(0x83cc03ec,0xd678f2ba),L_(0x00000177), +}, +/* digit=21 base_pwr=2^105 */ +{ + LL(0xd64c0998,0x7c43b599),LL(0x85a2ec86,0x4c9f6ac7),LL(0x59f29f0d,0xfaa4ec8b),LL(0xf16ae8c0,0x2017604d),LL(0xab8d8f78,0xbb59089d),LL(0x2772bc38,0x38403982),LL(0x6e88e817,0xf3d1571c),LL(0x7d4e8e0c,0x4a1b1a7b),L_(0x0000016a), LL(0x477bc572,0x39b5eb10),LL(0x8d22a645,0x77d71bc8),LL(0xd769223e,0x9734dc62),LL(0xfe2b562c,0x41cdb0ad),LL(0x3173fab2,0x70ddf3ff),LL(0x997d6033,0x70dbbbbd),LL(0x6d59561f,0x998a88a5),LL(0x64aafc32,0x1f2324be),L_(0x000001d4), + LL(0x449a64c8,0x674c0fe9),LL(0x81603532,0x01e88fe6),LL(0x595c6e13,0x913b8697),LL(0xf6f513d4,0x3b6d478c),LL(0x88c1e320,0xb2857351),LL(0x90cfb68d,0xdfb9fd43),LL(0x6c4bb93b,0xcc660149),LL(0x3f388af7,0x73e97063),L_(0x000001bd), LL(0x9a5258fb,0xacae0a8b),LL(0xb741a40d,0xba0560ab),LL(0x795d005e,0x3978bc6a),LL(0xcaa47999,0x1c0b2496),LL(0x1ff04fef,0x6ddcefe7),LL(0x47c3b092,0xf281ff39),LL(0xcc93f3d8,0x23027cc7),LL(0x773c9a3f,0x2ea46e78),L_(0x00000046), + LL(0x21f6a156,0xc4591dc9),LL(0xf7c15406,0xb36c8aaa),LL(0x3b0d0813,0x287834fd),LL(0x44ef9e76,0x94a18ce5),LL(0x52fb6092,0xbd019877),LL(0xb24d08d9,0xd4816092),LL(0x39d2d32c,0x188b097d),LL(0x5b9f00f2,0xa3f1ab3a),L_(0x00000003), LL(0xccda9d6f,0x41dad104),LL(0x637d2807,0x4ee619b7),LL(0x0f5a9cc8,0x4f3d7156),LL(0x97bb554b,0x367054b8),LL(0xe890a210,0x3f1f61c3),LL(0x0784aff6,0xb92963c2),LL(0xc9acc43c,0x309317af),LL(0xdadb0d3e,0xf3d3b5c1),L_(0x00000012), + LL(0x6d5e67b8,0x049665d8),LL(0x19993eb2,0xb56e1ced),LL(0x7a62ba87,0xdfb9c1fc),LL(0x6fc5cf75,0x4712b627),LL(0x554f5dad,0xe0548bd1),LL(0xecba89fb,0x1ee24125),LL(0xfa18f5ad,0x7e176a53),LL(0x8796b526,0x557064e1),L_(0x000001aa), LL(0xada0f1a1,0xe4e5f968),LL(0x89107584,0x8e12a3e0),LL(0xd6a2ba69,0x1ee9c73c),LL(0xe23b2a1f,0x43a76e02),LL(0x428e9adc,0xe3d7526f),LL(0xf09d62c6,0x0557ab8a),LL(0x37cd537d,0x2758b1d3),LL(0xe54434b8,0x3d68a000),L_(0x000000f5), + LL(0x2d008a6b,0xd28295a8),LL(0x6d2db388,0x2d6dd882),LL(0x7d1d9977,0x1f0a2196),LL(0xa51d2cc4,0x5e445be3),LL(0x52abdb6b,0x3146aac1),LL(0x1f1b1ee1,0xfac49408),LL(0x92df369d,0x84b1d832),LL(0xf67ffdf5,0xac7d64da),L_(0x0000005f), LL(0xba8a7d18,0xfa0e70ac),LL(0xf53c34b6,0x3063c19b),LL(0x4954b990,0xcac557d4),LL(0x2d1447f0,0xc89953df),LL(0xc7aef4e6,0x66df6476),LL(0xcb978dd6,0xc6d8f86c),LL(0xf9c4098f,0x024e891d),LL(0x25468ae2,0x0b1850aa),L_(0x0000006a), + LL(0x9893947c,0xe7519a11),LL(0xc44aa926,0x9787209d),LL(0xf096efcd,0x743501fd),LL(0xce9a1706,0x1f7551cc),LL(0x684716f6,0x8dae5405),LL(0x149e678c,0x4cc025ce),LL(0xa47c9f5e,0xb9f91415),LL(0x9acbec1a,0x39acf658),L_(0x0000019f), LL(0x1acf3849,0x7e3ea9f8),LL(0x0e3c4dae,0x4d104dc4),LL(0xadaff9d7,0x5ec06e69),LL(0x2c84d798,0x553ccd90),LL(0x93c28668,0x32f13211),LL(0x585b3068,0x2907a89a),LL(0xcdceca12,0xb9c0d594),LL(0x128fec44,0xf4db515d),L_(0x0000016d), + LL(0x1698ff12,0x50452a3f),LL(0x34e63149,0x70c02b2e),LL(0xa04d3b54,0x3afdb0b0),LL(0x61ed2692,0x91c5ad02),LL(0xd6e4e6a2,0x602aa43c),LL(0x134189db,0x4a9101c1),LL(0xf2cd081d,0x4ee8352b),LL(0xf7a72eda,0x24e8f999),L_(0x000000eb), LL(0x04ba1ce3,0xf4179875),LL(0xa4294fcb,0xe4a0b700),LL(0x05833496,0xb77b6e65),LL(0x8229a64f,0x204eabae),LL(0xe9137cfc,0x4ba1e0a9),LL(0x66c9fb36,0x5ece8d91),LL(0xabbbb589,0x725a0003),LL(0x62522294,0xdcc40fa3),L_(0x00000167), + LL(0xfe36c3a6,0x11953f80),LL(0xd15f704a,0x0b4cc57c),LL(0x3d3a8bb5,0x347df44d),LL(0xb5099398,0x3789e215),LL(0xd81f3a55,0x5f2bce30),LL(0x853ed011,0x0b7f91f0),LL(0x20d53ac9,0x8d7ec293),LL(0x63e7bfbe,0x8e08485b),L_(0x00000157), LL(0x56cdcd80,0xc636cf5e),LL(0x21241d8f,0x6e4c3d96),LL(0x6812f9d5,0x81fb84a8),LL(0x7741d3de,0xb50a7928),LL(0xbab3d77e,0x7cc80386),LL(0x8f374862,0x1901afee),LL(0xbf5ceb2a,0xdd95591f),LL(0xed0c8140,0x58db480a),L_(0x00000046), + LL(0xf412c36c,0x0ee07a01),LL(0x426518b8,0x54499ba5),LL(0x89e701b7,0x380c3953),LL(0xf3f8a9a9,0xd8749088),LL(0x809a3666,0xc559f6c7),LL(0xe768213d,0x64aff50b),LL(0xaad0b2e7,0x0535ff88),LL(0x68771b34,0xcb2d46cd),L_(0x0000017f), LL(0x7ec42d0f,0x4a3e7bea),LL(0xf9c7931c,0xe0127a41),LL(0xe2d8d114,0x88a09cfa),LL(0xc08a0d23,0xa27150fc),LL(0x052224fb,0x1880e3c3),LL(0xc80a285d,0xc9c2bfb6),LL(0xd52dcb46,0xfd0b235d),LL(0xfb31d609,0xc8ebb8df),L_(0x0000014b), + LL(0x058ee09e,0xf551f7ab),LL(0xd68828a3,0x85ee0c11),LL(0x6925306d,0x45a463ec),LL(0x9d6c7188,0xf851554d),LL(0x962b1441,0x66783d33),LL(0x32aca09e,0x856ca966),LL(0x9146adcd,0x5f7a7608),LL(0xfe631a24,0x269af51f),L_(0x00000087), LL(0x28d6e11e,0x0d442988),LL(0xc65a878d,0x0f8c7ce6),LL(0x332a94e8,0xf933213e),LL(0x0a2bf942,0x055f0bf3),LL(0x0e64c7cf,0x371deac5),LL(0x34d3b9fd,0x384367de),LL(0x6f42cc1a,0x15ed6027),LL(0x8f2a65d1,0x58437198),L_(0x000001ca), + LL(0x1243edd3,0x7356cd93),LL(0xc68eb0e6,0xfc213d5b),LL(0xa963c442,0xd8a42be8),LL(0x426acfcc,0x4e52c125),LL(0x3019a35c,0xcccbb098),LL(0xcbc21858,0xd433019c),LL(0x687acf0a,0x47525d0f),LL(0xce5e2216,0xe523322f),L_(0x000001cf), LL(0x88d6b26d,0x6dad247f),LL(0xa7f32d3f,0x70a43312),LL(0x340a2808,0xe2da73d8),LL(0xed020b20,0x477f5bf7),LL(0x752a7c57,0x84e84209),LL(0xaf283680,0xdce8d8d1),LL(0xf2a576b8,0x4e1b6410),LL(0x6cfe6e79,0x69ed0095),L_(0x0000008c), + LL(0xf28c9f4c,0xd2aab695),LL(0x7916e1ca,0x8e97f41e),LL(0xe9e93d40,0x40703872),LL(0xd7c8afdf,0x78640530),LL(0x8fe1f1c7,0xfb5cc433),LL(0x2060da94,0x7302096f),LL(0x953a5bfe,0x62d3763e),LL(0x4947fc61,0xeba19a33),L_(0x00000049), LL(0x92108fd5,0x8477b34f),LL(0xe9b6b7e8,0x31f306a1),LL(0x306db354,0xd16d3be6),LL(0xdf966d91,0x0deb9d15),LL(0x62b5d760,0x613eaff2),LL(0x22a569a2,0xae4f8efb),LL(0x0eeb67c4,0x9bca80a4),LL(0xac4e55dc,0x0d30055e),L_(0x00000183), + LL(0xe7b9391b,0xb559bea1),LL(0xd94e135b,0xbbc93e2a),LL(0xca27f88b,0xda4f9fae),LL(0x9fac28c9,0xe1b8ef16),LL(0x5f1ec2d6,0xfd385151),LL(0x49bb68ca,0xfb07a8a5),LL(0x55e48d66,0x146bc523),LL(0xe1490dad,0x0b9411ce),L_(0x00000010), LL(0xd3c160fa,0xec69a98d),LL(0xf736bcb4,0xb1767df2),LL(0x0bfa04f5,0x48e96045),LL(0x87d98ff0,0x977f6553),LL(0x6992858f,0x17332805),LL(0x22fe39cc,0xe8ffd592),LL(0xc326d64f,0x6551c98c),LL(0x12a83f56,0xa04ddf29),L_(0x00000192), + LL(0xdef70c89,0x5d8bd28a),LL(0xb3dcf1dc,0xe969cb6b),LL(0x157c6b46,0x846f656e),LL(0x30bef44a,0x3e3ab4ad),LL(0x14cc18c8,0x63d1dbea),LL(0x10262f96,0xfe27704f),LL(0xa567503f,0x1ff786bf),LL(0xff184f96,0x3fcb21d0),L_(0x000001f6), LL(0x92281a63,0x41b9f732),LL(0x4f7c669f,0x4f4c7a3a),LL(0xd9aab809,0x6fecf938),LL(0xece45010,0xbed97dd7),LL(0x20855b33,0x41c58421),LL(0xda8500b4,0x6bc3ec2e),LL(0xf0a33322,0xa1fd8aed),LL(0xe01eb188,0x1be357d3),L_(0x00000141), + LL(0x1799b47a,0x205f8a25),LL(0x5c7dce04,0x6bc7753c),LL(0x918feeda,0xb0bd3460),LL(0xba66aed0,0xfae70886),LL(0x5d2bdd64,0x37b93501),LL(0xa85e194e,0x12025c5a),LL(0x44f97270,0xb54faac3),LL(0x98c400c5,0x8d500c94),L_(0x00000190), LL(0xabe22aa5,0x3c01512e),LL(0x99aa80de,0xca5be145),LL(0x312f55d9,0x8dbfded4),LL(0x1ca51916,0x3f318a7b),LL(0xa0b3f9fb,0x42abfb1f),LL(0x2303713b,0x195a32bb),LL(0xb6968fd2,0x069ca809),LL(0x52819c4b,0x27ea438b),L_(0x00000079), + LL(0x662fcdd0,0xd8b1dbd1),LL(0x66c06c9a,0xaf6b6e15),LL(0x28998a9b,0xca45c4ad),LL(0x2d2005db,0xfcd947b1),LL(0x609acb17,0x6bf7b35f),LL(0x25ebaf2e,0xb8a8aba3),LL(0x599df520,0xe4302e3f),LL(0x2bf9b973,0xf871980d),L_(0x000000fd), LL(0x25aebd11,0x8868630c),LL(0xa5529c40,0xaf7c4f6b),LL(0xf5657b1a,0xc0fd49e0),LL(0x3fa70b84,0x4d86ecda),LL(0x39f53004,0xc59dce6d),LL(0x39513f7e,0xbdaf7343),LL(0x822c2924,0xce22dd61),LL(0xacb0786e,0x78182466),L_(0x00000054), +}, +/* digit=22 base_pwr=2^110 */ +{ + LL(0x55dedb27,0x5a0fc338),LL(0x03e8af53,0x788ccd88),LL(0xe10cabd0,0xa1f1f7d6),LL(0x5f889d7f,0x0487ee35),LL(0xa583e303,0x1885d800),LL(0x2fc9f3dd,0x09ae9a4a),LL(0x2887b5bf,0xa554fc30),LL(0xd91181d3,0x1d189678),L_(0x000001e4), LL(0x53b146d6,0x52f280d5),LL(0xdfaac466,0xe0b73d63),LL(0x0d77869d,0xa8a399cd),LL(0xba5ffe6a,0x5c61b757),LL(0xaffc1da6,0x71cf6c9e),LL(0x34d27387,0x20ae1248),LL(0xf184b956,0x7f6504de),LL(0x1c974cb4,0x94c62d76),L_(0x0000019b), + LL(0x659b9b53,0xd37156ff),LL(0xf8338bab,0xa115d2c7),LL(0x9d1175b5,0xa927371c),LL(0x53c22d6a,0xe5b07da3),LL(0xb79ee37b,0x3585421c),LL(0x8ac92029,0xbe2b0a93),LL(0xd489e47a,0x363622f3),LL(0xccd5811b,0xdf568ac9),L_(0x00000183), LL(0x41cb54f0,0xbf83fb74),LL(0x527b4fee,0x7d9fa2d5),LL(0x6d4a3597,0xf8ab4037),LL(0xe4619c87,0xd590e945),LL(0xab913b27,0xe8861075),LL(0x389b1da0,0xd8fb707f),LL(0xe0beb49d,0x35140b6f),LL(0x392dd172,0x0d587bf7),L_(0x000001ea), + LL(0xbf4176d0,0xfe15067e),LL(0x0120bf23,0xc1d45172),LL(0x0cb82143,0xfbe2cf59),LL(0xb0e80076,0xd69fd57c),LL(0xbd9b2caa,0x78503bca),LL(0x99823d72,0x2730e435),LL(0x31cc7be0,0x4b145b24),LL(0x10f8d6e3,0x7db8fea2),L_(0x000001cf), LL(0x4fc47dab,0x7a72c91b),LL(0x564e5846,0x5530b4bc),LL(0x1837936f,0x9913d2c3),LL(0xdf60105f,0x5f1a5851),LL(0x839ef0be,0x3d6d7b8d),LL(0x05890a65,0x48845fee),LL(0x57eb20fa,0xcc1288ca),LL(0xf7b7e05b,0xbf1a3cac),L_(0x0000015a), + LL(0x7082a01c,0x50e34426),LL(0x91616bf5,0x27cfd7b1),LL(0x426bd9ae,0xc299bf54),LL(0x5f468d0e,0x0487ca37),LL(0x695e6354,0xb93aa7dc),LL(0x9322f558,0x8f48edec),LL(0x818f0592,0x957ee742),LL(0xaca5b088,0x248afd23),L_(0x000001be), LL(0x11189720,0x45a01307),LL(0x348cb9e4,0x5bf246e5),LL(0xf5c183c6,0x3fd8ccf1),LL(0xe9a40aeb,0x0fbda6f2),LL(0x087abdef,0xdaf09cee),LL(0x90c450f5,0xe33344ee),LL(0x3abe1073,0xa3404424),LL(0x02a065d1,0x011c8b8f),L_(0x0000010a), + LL(0x5faa9290,0xbd275c4d),LL(0x69ab8c9c,0xebf0d548),LL(0x7a6bc4a9,0xe5ed16cd),LL(0x31faf28e,0x3e681735),LL(0xcf90331c,0xc624be8b),LL(0xfb66de1c,0x42603696),LL(0x2a65b006,0xc07466ec),LL(0x9d7f9688,0x84d634f3),L_(0x0000013b), LL(0x44e0d6d8,0x3db25f9e),LL(0xe7ca860d,0xa581c150),LL(0x1c6481b3,0x49e5b0b8),LL(0x62060736,0x82bc7eb1),LL(0x54daac9c,0x376c43d1),LL(0x68353454,0xfb293af5),LL(0x2dde1795,0x7e2ec37f),LL(0xed4ef8f0,0xbefc779b),L_(0x000001a4), + LL(0x4adfcff9,0x2b928ce2),LL(0xb2e63ba3,0xd02f461c),LL(0x73336d0f,0x04fd6cc5),LL(0x48e88a3a,0xc377597d),LL(0x56c730e8,0x5ac92cdd),LL(0xaf7486b7,0x0317d853),LL(0xe151b910,0x0978da40),LL(0x86c35051,0x6108c40e),L_(0x00000134), LL(0xb333edf0,0x1ff77a8a),LL(0xd24f0df5,0xed7f23b9),LL(0xdfae6385,0xaa4f024f),LL(0x2a1af93e,0x2ffef5f6),LL(0x6f45d8a5,0x9aa11e63),LL(0x853bb088,0x2271f40e),LL(0x8ed5445f,0x3ccb38c1),LL(0x24afc179,0x6883bc27),L_(0x000001d3), + LL(0x57906522,0x85f7ceb0),LL(0xce569864,0x059b3177),LL(0x45f8df95,0x60efada7),LL(0x2ee7a947,0x87aafc1c),LL(0xdce2c588,0x17fd804d),LL(0xdcf2f1ac,0x10e82f62),LL(0x4b1309c3,0xe852efd4),LL(0x0ef30c42,0x8810a12f),L_(0x000000d4), LL(0x95ed2fc9,0x55cd2f37),LL(0x5d0e9c83,0x107085fd),LL(0x479d2ac2,0x85ff5e6a),LL(0x7bd3eb35,0xd6ed74ec),LL(0xc8a77d96,0x366d8e34),LL(0x3be40939,0xf0ec3c90),LL(0x11212f04,0xc317d540),LL(0x5743bb20,0x4f1fdd52),L_(0x000000c4), + LL(0xffb66d41,0x6ea70c3f),LL(0xc5491789,0xcb17d54f),LL(0x1eaf6e4d,0x7c642a64),LL(0xb15be10c,0x99328296),LL(0x611efe5f,0x30829e9c),LL(0x3cdec049,0x5f18e861),LL(0x1a7c38da,0xab7985a5),LL(0x3536d908,0x8a46db8f),L_(0x00000102), LL(0x8ece50b4,0x5435f6e4),LL(0x13d04672,0x60505d74),LL(0x07efc4e5,0x8a551fa9),LL(0xcc601ad2,0x8fed3391),LL(0x9eeaf4b1,0x4338a854),LL(0x72c52e1a,0x61868d33),LL(0xce70bb6b,0x9c3a511b),LL(0x5c8d75eb,0xf313ce1f),L_(0x0000008f), + LL(0x69ee64f3,0x508ada39),LL(0xdd8c7134,0xda598cbd),LL(0x36c545ec,0x05133eca),LL(0x0c3f5caf,0x2df79eab),LL(0x71cd5c2a,0x920bc258),LL(0xfa67ae73,0x1fd1e4a4),LL(0x2ed2d89b,0x48726f90),LL(0xd5a8956c,0x2a4c3dab),L_(0x0000011a), LL(0x6f23ba5d,0xb20e4dfc),LL(0x4bab12af,0x5fcc6747),LL(0x0006c88a,0xaa6ea9a8),LL(0xe8e4646f,0x9db86f5a),LL(0x8f1f8a76,0xec7745fa),LL(0x455291dc,0xe15a4259),LL(0x155de9ad,0x6c3a6e93),LL(0xf9fdf6e0,0x54f6c4df),L_(0x00000005), + LL(0x3dc752ef,0x14921ae6),LL(0xdad5624e,0xf829dec8),LL(0x1d1460ab,0x5833de5b),LL(0x9c1ff203,0xd5ded33e),LL(0xfac09cf2,0xbb7c2ad4),LL(0x82d6f980,0xf32dbe91),LL(0xad650ca0,0x82ca75ed),LL(0x6c58e180,0x360a9033),L_(0x00000156), LL(0xea9d2225,0xd3ca85cc),LL(0x434cfef1,0x3bb6dff7),LL(0xce357f60,0x56328ba9),LL(0x0e4a179e,0x4a4804e3),LL(0xe56b9eac,0x254a5b01),LL(0x72a81cd7,0x8b35d349),LL(0xe054875e,0xc8c87ac4),LL(0xe8c11607,0xfef9a015),L_(0x00000001), + LL(0x1c6cc43f,0x6f96d167),LL(0x794436fa,0x586761cb),LL(0xd20a52b3,0x3ae24479),LL(0xc24cf7e8,0x5d299550),LL(0x7aabd2b7,0x9cea4b13),LL(0x09feb305,0xd75ffb4e),LL(0x5b6a28d7,0x5d3cd9c7),LL(0x4d85737b,0x3f800863),L_(0x00000186), LL(0x743b9f5d,0xe042f5f5),LL(0xe03225a4,0xc7dc3d4b),LL(0xdfce41ae,0x51cdf46c),LL(0xd3c05da2,0x69bf3a35),LL(0xc18cbac7,0xc0889d43),LL(0x51fc0084,0xf694d481),LL(0xbfa4cfd4,0x05438a3a),LL(0xcb44f4a9,0x037fbdcc),L_(0x0000003c), + LL(0x22d68d2a,0x6f65a633),LL(0xd77949bf,0x368db479),LL(0x738f46ed,0xe52e22f5),LL(0x7212d465,0x4758d194),LL(0x8bb783e2,0xd677a59c),LL(0x1b239d33,0x60904604),LL(0x9c2f2775,0x8df6497a),LL(0x9be5339a,0xfc0bbe7a),L_(0x0000007c), LL(0xb5804d78,0xbd5cd190),LL(0xbfbebfb6,0xd58769b6),LL(0x66d25685,0xf9ea5b23),LL(0x206ac283,0x09d14a84),LL(0x845e93a9,0xe03b612f),LL(0xc6807818,0x061fa312),LL(0xeb980705,0x6b501efd),LL(0xfa3670b6,0x8ea0643c),L_(0x00000010), + LL(0x509702e1,0xaf686102),LL(0xc4201f85,0x800e44cf),LL(0x7e6e7641,0x35e4ed58),LL(0xa3fd838b,0x5af78edd),LL(0x728e86da,0x0effaedc),LL(0x6fd05e38,0xfd668fa4),LL(0x4ebcbbb8,0xe45dbf60),LL(0x21bf82d8,0xafb31f6c),L_(0x000000c4), LL(0xf066dcdf,0x1ca3b2b7),LL(0x73890298,0x96fc550c),LL(0xcc4f19d8,0xfc004a6a),LL(0xc9fae54f,0xcdd730f4),LL(0xa5e22c5e,0x2754c10a),LL(0xb60dac89,0x49c473e4),LL(0xd5465b8b,0x30fa2831),LL(0x14688f19,0xa1a65375),L_(0x00000100), + LL(0x8f88f135,0x7034290c),LL(0x2f4cd77e,0x61556d3d),LL(0xe5aea948,0xee182466),LL(0xf7fd60b4,0xfaab2132),LL(0x2107919a,0xb164b7c6),LL(0x4909986e,0xb4de2fa4),LL(0xe1076a94,0x331fc36a),LL(0x8fd3234a,0xadcb78d4),L_(0x0000009f), LL(0x07524382,0x3f9109b1),LL(0xec9a0d40,0x11cb9eb3),LL(0x7e1084f2,0x1e06d740),LL(0x00717031,0x2928ea89),LL(0x4bde88cb,0xd69d113b),LL(0x20f91a03,0x2ff2dbf3),LL(0x7a3884fc,0xdf24441f),LL(0x198806cf,0x69d88d2a),L_(0x000001dc), + LL(0x076f7438,0x1544e142),LL(0x15274247,0x022e14c5),LL(0x96077c3c,0x30862489),LL(0x789e3935,0x50c53e4e),LL(0x1141fcf3,0x3047c405),LL(0x2cd7f2a5,0x57a60daf),LL(0x9bc52e3e,0xcd0ce692),LL(0x21830d42,0x6d7217a9),L_(0x000001f3), LL(0x77ac72bb,0x403e48dc),LL(0xad70038c,0xc084214e),LL(0x170491ff,0xf7b0b5fe),LL(0xd8cf2d1c,0x4af0ed5d),LL(0xd8301c30,0x00208a5b),LL(0x33f56a54,0x18e018d4),LL(0xfa224eea,0xef56c21e),LL(0xaa9262c2,0x692f082d),L_(0x0000006e), + LL(0x328296f3,0x4bb11c1f),LL(0x6050199a,0x15b40849),LL(0x53a7ca7c,0x81bc50ce),LL(0x0a2c1da2,0x682873f5),LL(0x52c0e34f,0x44102170),LL(0x9c5ef21f,0x9f354fbc),LL(0x0bba954e,0x6cd7990a),LL(0x02432a32,0x37dabca4),L_(0x00000019), LL(0x2be6dddd,0xe55cac7b),LL(0x8a3b8a57,0xda37392c),LL(0xec1dc93e,0x2e3fecd4),LL(0xcf4f78c9,0xfedf3f09),LL(0x1ff689fe,0x03374052),LL(0x092dabd5,0xdf4087ca),LL(0xb9e4e110,0x9d02763e),LL(0xf3f329b7,0xdaeda689),L_(0x0000012e), +}, +/* digit=23 base_pwr=2^115 */ +{ + LL(0xbd54382d,0x826f7b17),LL(0x32c36ec7,0xdce64f28),LL(0x22a16680,0xab2193ae),LL(0xaf6a85c2,0x52cc0a0a),LL(0x2f202702,0xcc1335b3),LL(0x2afbf317,0x3743776e),LL(0x0deb4740,0xf9a19900),LL(0x61591f25,0xd91b36bc),L_(0x00000002), LL(0x1d84eebf,0xb06a4eb6),LL(0x8bb72608,0xb23e7396),LL(0x2e886104,0x992a3ae8),LL(0x7c8605d2,0x418a91d2),LL(0xe33bec6e,0x5f2b49e6),LL(0xa9d829d4,0xd1f4a3f4),LL(0xeb2f044b,0x28bc4cea),LL(0xb1ef09fa,0x00e53c63),L_(0x00000050), + LL(0x04826845,0x6b1ee54a),LL(0x9240015b,0x608b1dc0),LL(0x25698b8a,0xa1390509),LL(0xb5e532ad,0xdf4acb31),LL(0x30c41c46,0x16e05bee),LL(0x07d190c1,0x642c1273),LL(0xb566eca9,0x1b365a48),LL(0x5c3cffa2,0x3f2fc46d),L_(0x000000c2), LL(0x21b8836b,0xa0057686),LL(0x520f579e,0x7bf51510),LL(0xa43d38cb,0x0a4bbc0c),LL(0xfe21891a,0x687446c9),LL(0x1242b093,0x8feab881),LL(0xcacb6d61,0x7a921f31),LL(0xcf611aa4,0xc5cb09d3),LL(0xfef9a8ef,0xa83137d8),L_(0x0000012e), + LL(0x4375da6c,0x13287748),LL(0x4fda63b3,0xfa3518ec),LL(0xbc16ce7a,0x996ffb04),LL(0xf3ffb5fd,0xf0a2d30e),LL(0x6da8bb6d,0xa23e83a8),LL(0x08e806aa,0x0642e4da),LL(0x0286c1dc,0x84837dcb),LL(0x8196eb23,0x999ea9ac),L_(0x00000060), LL(0x1c8d1ec4,0x9c108506),LL(0x77ca438c,0xdc8649fb),LL(0xdbfc198f,0xaf929bb8),LL(0xa5fbf701,0x60078f43),LL(0xe25fdf9d,0x3f03bcb4),LL(0xca36812b,0x53c1eaae),LL(0xb394d3a9,0x1f8445ff),LL(0x78a7b4ab,0x8305bbeb),L_(0x000000b0), + LL(0x6d42c81a,0x40e01936),LL(0x7487e815,0xd6072e7b),LL(0x32da913c,0xe7b4156c),LL(0xf1e87478,0xb217423d),LL(0x4880f5cc,0xca344dd5),LL(0x90182347,0x15da2c26),LL(0x87d2337d,0x8d993e28),LL(0x604cc23f,0x0e937379),L_(0x000001ea), LL(0xf9778d40,0xc9dd1808),LL(0x4345027e,0xdcd7b63f),LL(0x198a63ab,0x03bcf65f),LL(0xa7a4c388,0x1130c2d3),LL(0xb476f99f,0xc1ea5019),LL(0x991ad6b9,0x4f67377e),LL(0xa9f5ad13,0xd99047df),LL(0x80641e2f,0xd93815dc),L_(0x00000185), + LL(0x2a4af296,0xf45a67c1),LL(0x963ea378,0x3fc32889),LL(0xe19e2266,0x2477017e),LL(0x3e1c3af5,0x1bbfecd7),LL(0x9c6aea32,0x03afdf5f),LL(0xdd92f5ef,0xd2ffd177),LL(0xcff66e71,0x22d56579),LL(0xca369a53,0x098e3322),L_(0x000001b9), LL(0x3ecebaa8,0x87cbd3db),LL(0xfef4d6ca,0x92b7d8a0),LL(0xf81b8c47,0x4e50ecbb),LL(0xdd6768bf,0x916361ed),LL(0xf3c09bf3,0x6b31d1bf),LL(0x54e2879b,0x17c7f544),LL(0x44b470e7,0xb6fa811a),LL(0x32df7372,0x135177e9),L_(0x000000ad), + LL(0x83de0d9d,0x7ea57102),LL(0x4652ceba,0xdd543523),LL(0xb8a36856,0xb586f821),LL(0x19e00261,0x6ce309c3),LL(0x1ed079e5,0xe0f75ac3),LL(0x51ff1099,0x2442020d),LL(0x0c077aee,0x248b83fa),LL(0xc85e1f87,0xf5eebe6f),L_(0x000001f4), LL(0xaf872b79,0x311d3108),LL(0x2ca2a32b,0x5040c97d),LL(0x4fa4c2f0,0x7615703d),LL(0x80d5eb27,0x36c8c169),LL(0xb5f074a5,0x95daa1e3),LL(0xbda91813,0x672dad89),LL(0x395bd4fb,0xf61f3d94),LL(0xc4a2c81e,0x8ba214b4),L_(0x00000108), + LL(0x04e91ed9,0x861a2094),LL(0x2bca8d77,0xca4ca01a),LL(0x03590793,0xf16fc210),LL(0xc8ad1877,0x48a85346),LL(0x89666be4,0xf0fc0cfc),LL(0x6adba857,0xcd27d0e4),LL(0xdc6000be,0x66de6f3c),LL(0x410cd2f9,0x701dbfa3),L_(0x00000143), LL(0x27a30017,0x3566e721),LL(0x523a0305,0x793773f0),LL(0x1ee9afff,0x7fd66850),LL(0xdbc711c7,0x047ae5ad),LL(0x3acae945,0x203bb8df),LL(0xb932a42c,0xfe2439ff),LL(0xd51dba9f,0xe4630688),LL(0x268de595,0x2e9ee036),L_(0x0000012e), + LL(0xcb924e78,0xf61f664c),LL(0x2e404ee4,0x5ac67cb0),LL(0x6b002de5,0x87550da7),LL(0x537e3c3c,0xb6b43fc4),LL(0xcc36c052,0xb2d5ce01),LL(0x0c5bb204,0x8e7f6d0e),LL(0xf930fde8,0x09c188bb),LL(0x056f87d9,0xff08d168),L_(0x000000ad), LL(0xbd1106b6,0xe76203aa),LL(0x0182e8dc,0x02fff311),LL(0x1307d3fa,0x96bf7d1e),LL(0x71013392,0x6ed2ca34),LL(0x24a22e45,0x1b668eed),LL(0xe4102d01,0x79cf95e5),LL(0x681d10cb,0x6c6693b0),LL(0xf94e08ac,0x7a740355),L_(0x000000d1), + LL(0x4ea63be4,0x7d917da9),LL(0x5de2c336,0xadecf30f),LL(0x0955c66b,0x81dabb28),LL(0x6e473865,0x19491209),LL(0x1d9702d7,0xa1f53ee2),LL(0xa4bef482,0x26877597),LL(0x0315b12f,0x18b70eae),LL(0xbca15f03,0x5864f695),L_(0x000001d8), LL(0xe0496daa,0x1afe44da),LL(0x518d4571,0x824106af),LL(0xf72d3376,0xa6b1f64c),LL(0x1149d145,0xbe64f34b),LL(0x8c71ca30,0xd7b0b0f7),LL(0x3acfa7a5,0x58a3ea6e),LL(0xcc0bc394,0x5e42d97e),LL(0xe6ba0355,0xb8a8749f),L_(0x00000051), + LL(0xcc323242,0x25b57cdb),LL(0x0afacd89,0x4d31e696),LL(0x075e88b1,0x80ecc1c8),LL(0x4d853d58,0xe213393a),LL(0x5c2d17b4,0x1d02b340),LL(0x29f6c35a,0x7f4eb22a),LL(0x763f945f,0x991b6570),LL(0x38d9e916,0x8274a08a),L_(0x00000172), LL(0x919e262a,0xcdb707d0),LL(0x7b3ccd89,0xc28362e2),LL(0xe31adec3,0xd970a8d2),LL(0x2e5340b0,0x11f1ff4f),LL(0xb76d44ac,0x42bd388a),LL(0x528cddee,0x2165e718),LL(0x7cb055f5,0xa8c2384a),LL(0xd81cae87,0x5ee96e3b),L_(0x0000011f), + LL(0x95c8885f,0x504d777f),LL(0x6a9d125d,0x327807d9),LL(0xd624f272,0x0e53c679),LL(0x95ea222f,0xe1387ac7),LL(0xb1597752,0x656acd9a),LL(0x352cac8c,0x6190d15d),LL(0x61bf8856,0x01af7e20),LL(0xc719ce43,0x14d8c07b),L_(0x00000194), LL(0x83f36452,0xfdd73cd6),LL(0x314de5cf,0x46e7aa4c),LL(0x50ce899f,0x7424d707),LL(0x38e875fc,0x3221445c),LL(0x9f3a7a99,0x974ba6c9),LL(0xe06eb667,0xcc871495),LL(0xffe48ed2,0xda22c28b),LL(0x34965180,0x95e30bbd),L_(0x0000007e), + LL(0x8ee0ef29,0xfb175926),LL(0x1c82db3e,0x93d33d24),LL(0xb912da50,0xbb15ebc6),LL(0xcea7d557,0x53132904),LL(0x2a95c0c2,0x1728bce5),LL(0xd703338d,0x3e934774),LL(0x9ff62322,0xa4bdaa17),LL(0x5a25267e,0xfd8b69c0),L_(0x0000000c), LL(0x4db3f1bf,0x49cf21d2),LL(0xa18c0ec0,0x1567c730),LL(0xf359d391,0x8f78c3e0),LL(0xa1bf7eca,0x9f9aa64e),LL(0x4252d4d8,0x5b2ffd6d),LL(0x3cf77a2e,0x70d5197d),LL(0x420f1fde,0xc4ac046e),LL(0xfbaabfd6,0x197eef82),L_(0x00000092), + LL(0x6f164190,0xacca9818),LL(0xdc3afb13,0x75acc7ba),LL(0xf8e7944e,0xaec6267e),LL(0x266c89d3,0xc0d5983e),LL(0x7a35b6ea,0xc7025ed3),LL(0xcc6f8ba4,0xb3a78dfe),LL(0xe18e7845,0x43ed79a0),LL(0xd3e423c7,0x9847da1e),L_(0x00000022), LL(0x7ad878cd,0xf2ddd3e7),LL(0x7ad2ccc8,0xdd3af93f),LL(0xe988a2b1,0xc20e3266),LL(0xf31777a1,0xc508e478),LL(0x5f61decf,0x6ca64937),LL(0x38c983d0,0x63f7f656),LL(0xffbb003b,0xff837029),LL(0x32104839,0xc4d50105),L_(0x000001ee), + LL(0x0f23d833,0x787960d2),LL(0x1a82d064,0x1e23da2c),LL(0xf31fd1ab,0x632fca0d),LL(0x67beaa32,0x48f5480d),LL(0xb9e45d26,0xbb162f9b),LL(0xd434d563,0x10e02089),LL(0xa10eef01,0x647082d3),LL(0xb7735d1d,0x03650cb7),L_(0x0000017f), LL(0x40d95b89,0xe61d29d9),LL(0x68b4d3bb,0x1c7d5b4e),LL(0xd78df4bc,0xbd612a5a),LL(0xd83302ca,0x80982747),LL(0x511140b2,0x754f6279),LL(0x16e7211a,0x1d43610e),LL(0xf0dec954,0x3999e665),LL(0xbaca9f0f,0x3cdd9ee6),L_(0x000000e0), + LL(0x38877144,0x1ee1b012),LL(0xed46bb99,0xf5e943b8),LL(0x376d8b60,0x1bc6b4d7),LL(0x4b6cb863,0x7dc297da),LL(0x216481d0,0x123486d4),LL(0xc754e76d,0xb1c0a1e8),LL(0xdbcf937b,0xadf930fc),LL(0xdaf06ef4,0xa5ef1b3b),L_(0x000000c3), LL(0x03210a22,0x979dbfb5),LL(0xd444dbbf,0xd232212a),LL(0xef6c2520,0x35e7b199),LL(0xee0108b0,0x21bd8267),LL(0xe6dba29f,0xc9483241),LL(0x8b0b6ada,0x304a26d8),LL(0xebc36edd,0x88b58b4d),LL(0x0b4cd577,0x6441b89d),L_(0x00000108), + LL(0xee657257,0x208861aa),LL(0x5d4bf915,0x8adfc02b),LL(0x8b2a8792,0x51bf7839),LL(0xd1929e39,0x6ac2d82c),LL(0x51878fc6,0x2453f26a),LL(0x67c6a197,0x0ebd963c),LL(0x29e6f9cc,0xab6feb88),LL(0x6a8aecc7,0xb24a4c98),L_(0x00000165), LL(0x9f30636d,0xe6a5beb0),LL(0x5f6af11d,0x247b3767),LL(0xa04301fc,0x7893d122),LL(0x577167d7,0x4d974f3f),LL(0xea69addd,0x983fc60d),LL(0xd35bf8be,0x627055a8),LL(0x95c80a83,0x51c83aaf),LL(0xa21f06b1,0xee9e2a9a),L_(0x0000011b), +}, +/* digit=24 base_pwr=2^120 */ +{ + LL(0x1dc1e136,0x3cf04eac),LL(0x1c4f5e85,0x599f9890),LL(0xcbc44867,0x34ff0e3e),LL(0x5a12a7f8,0x851c12ee),LL(0xf066c152,0x7ca61be6),LL(0x73832df9,0x7153da2c),LL(0x14acdbe2,0xb73e882e),LL(0x87567338,0x9a4f930b),L_(0x000000cc), LL(0xd1fe8148,0x05fd56d3),LL(0x49ee53a4,0xe986a7db),LL(0x11101981,0x695cf7bc),LL(0x750760e2,0xb6aca2a9),LL(0x4815cb90,0x9f5ace2a),LL(0xc3dc9f29,0x6b06b61b),LL(0x3b28698d,0x6b2e5c22),LL(0x5687880a,0xb6015c0b),L_(0x000001bc), + LL(0x1af552c0,0xaaadcddf),LL(0x160c329a,0xf071e91a),LL(0x77f33e93,0xf9cbbaf7),LL(0x6e836178,0x74f3bc69),LL(0x430ecc6d,0x349ec647),LL(0x9e682571,0xbbec63ff),LL(0x3f624e0d,0x64eff8b4),LL(0x0d19e23a,0xab39a800),L_(0x000001cb), LL(0xce60d534,0x3eff3832),LL(0xe89d00d8,0x12f600da),LL(0xf8745dbd,0x3eb89d2d),LL(0x48217cd8,0xe79b868a),LL(0xc5ce0f8a,0xc2c4ae44),LL(0x0fe94021,0xa980ca2b),LL(0x5ab9482e,0xf0414674),LL(0xcffa33fc,0x96a9f1c2),L_(0x00000109), + LL(0xb176fd51,0xa2b01fa7),LL(0xbebf27e3,0x1a17875c),LL(0xca98073a,0x1a08df20),LL(0x73873253,0xcea9581c),LL(0xdc360b05,0xbad316bf),LL(0xb8a68986,0x9591db5f),LL(0x6941db20,0x838ce851),LL(0x0df495ad,0x337f3cd5),L_(0x000001fb), LL(0xb5d46b24,0xf0d09b27),LL(0xf2b04a4a,0xe34ef392),LL(0xc4e0cb50,0x9c028d0c),LL(0xbe127061,0x60b8995b),LL(0x202d9276,0xa9beaf92),LL(0x3a61c444,0x686effea),LL(0x7cc238c3,0xce321e42),LL(0x09075147,0x65266fe6),L_(0x0000014b), + LL(0x16e9dd16,0x8a3599a2),LL(0x05317187,0x0821091d),LL(0x24ef2697,0x6ed2cda3),LL(0x4950f2f1,0xeaefd2e9),LL(0xc815b553,0x9f00612d),LL(0x47c1f801,0x930eacc5),LL(0xfd1730a1,0x136fc4a1),LL(0x8252d52d,0xc56a7db8),L_(0x00000098), LL(0x6b77522a,0x2deb3842),LL(0xaea9f6b1,0xca869197),LL(0x3823d16a,0x187c4319),LL(0xf12c9d38,0xd5cc9828),LL(0xe31f43da,0x436529c3),LL(0x0781728a,0x63d40c6f),LL(0xbfbb0978,0x94da1798),LL(0x7a196933,0x97dcd1e1),L_(0x000000d8), + LL(0x95a20633,0x4ce60573),LL(0x98b05888,0xb9e9ac42),LL(0x9f28e7bc,0x770f80c7),LL(0xabb15751,0xd0147212),LL(0xcce75763,0x67296f82),LL(0x8034afbe,0xa2950d9f),LL(0x11791412,0x9731ca6f),LL(0x87c616f9,0x1f16d8bb),L_(0x00000104), LL(0xc7f27dc8,0x5fa5d017),LL(0x95bcc4fc,0x9fdb4deb),LL(0x39917e40,0x30051c1e),LL(0xbefa777d,0x3f36dfcf),LL(0x26ebd51f,0xd9696a85),LL(0x16cc089e,0x58a6c0bc),LL(0x6723f03c,0x3193efb5),LL(0xe4f7a675,0x97abbf77),L_(0x00000000), + LL(0x0082edbf,0x7ff0d41a),LL(0x1522ffb6,0x76aa53cf),LL(0x453dcda7,0x3ac99dda),LL(0x634bcd8a,0xca31a6bf),LL(0xdf09af12,0xda6aee65),LL(0xb96045d3,0xb2e1c131),LL(0x6f3c7e70,0x72188816),LL(0xcb58f8b9,0x9a1f5d21),L_(0x000000c7), LL(0x51f3e032,0xd4da7b11),LL(0xcafbe9a8,0xe3a95788),LL(0x39c010af,0x8c87071e),LL(0x05cb3faf,0xafcfc04b),LL(0x08a702fb,0x42c775b7),LL(0x5b3b6187,0x8aab53d6),LL(0xb84f9386,0x8bb27ffb),LL(0x08491b70,0x9bf23a75),L_(0x0000003b), + LL(0x2f20328d,0xb581eaa6),LL(0xd269e274,0x8fb2a285),LL(0x604b1779,0x5b26ea89),LL(0x3aa53ad7,0xd5119e93),LL(0x9fa62691,0x3e002a94),LL(0x8ba167e0,0x62921501),LL(0x195dffb0,0xe4ae2796),LL(0xdc1f93ea,0x2bba3282),L_(0x00000096), LL(0x980977c6,0x654950f7),LL(0x422ba8f7,0x9f0fcf77),LL(0xb7dc1d4c,0x6b970562),LL(0x0b2f7617,0x2de6068f),LL(0x894ad6c1,0xd3457950),LL(0xe569d53d,0xfc63f78f),LL(0x14981ae1,0xf2a90b52),LL(0x902dadf9,0xee87e5ab),L_(0x0000003e), + LL(0x5406fc86,0xc759885f),LL(0x5bd2a491,0x2d4cddc5),LL(0xc35aa122,0xb5461045),LL(0x2154985e,0x188b457a),LL(0xff0dcbe4,0x235148df),LL(0xd70c6a7e,0xa2535a30),LL(0x7d8e9016,0x6be2be33),LL(0x99a19ee9,0x8e953cf8),L_(0x000001ae), LL(0xa1dc1860,0x519771ba),LL(0xea4c9a1d,0xe6bf7f8d),LL(0xc44825c6,0xfd4b88d5),LL(0x48270d80,0x619d7b16),LL(0xcd7c088d,0x50ac4887),LL(0xcc2ce67f,0xd1ac72f9),LL(0xaafa6b89,0xafdce091),LL(0xb9365de8,0xe061fac9),L_(0x000000e8), + LL(0x7862482b,0x264a6de7),LL(0xcc327c76,0xf0e8036a),LL(0xdd64bd2f,0x58aff74c),LL(0xd63e620b,0x0d34c089),LL(0x792f3b2b,0x3cffd08d),LL(0xb8e40e30,0x2466d774),LL(0x5be025a8,0x83e235d8),LL(0xe9d3cf63,0x87a2f8e7),L_(0x0000016e), LL(0x93020cb2,0xfef8d504),LL(0x75b3c700,0xa8404df1),LL(0x1f7b3ca4,0x25cfc4f3),LL(0x3dca1055,0x21fc5f18),LL(0x73402205,0x096d5dd9),LL(0xf8afba2b,0xe13c530e),LL(0x23634751,0x7b6b3f2e),LL(0xca1be461,0x93159c76),L_(0x00000162), + LL(0x662154b2,0xf439d6d1),LL(0xc1e155ac,0x0ae5f642),LL(0xa2dd72ce,0x55e79db1),LL(0x71c8da23,0x7905f65a),LL(0x29ffbd0d,0x21383ef6),LL(0x39515d47,0x28c8f708),LL(0x739d692e,0x67130a0a),LL(0xe8283125,0x9d6fd698),L_(0x000000a8), LL(0xe6e2797b,0x7f499c43),LL(0x57f047a7,0x8571cbc2),LL(0xe0447784,0x8f068f3b),LL(0x85efc6b1,0x9497bb27),LL(0x4b52e9d2,0x5f954c9e),LL(0x896bc179,0x299e982a),LL(0xfe335eac,0x2fe557d1),LL(0x15ed5037,0x714710c8),L_(0x0000011d), + LL(0x45022f8e,0xe4a9bc90),LL(0xcb58c8df,0xd48d6951),LL(0xf9950f95,0x3a67fa88),LL(0xcfc52411,0x8aad81c0),LL(0xea907dba,0x193feef4),LL(0xbcf6e329,0x847c4744),LL(0x51539dcb,0xbfaf49f8),LL(0xe1705ff1,0x79078f89),L_(0x00000187), LL(0x7e920920,0x93070144),LL(0xf4a966bf,0xebc39792),LL(0x26b6e21a,0x5380b22b),LL(0xe6fd22df,0xf5ce884f),LL(0x866ea333,0xbbd94169),LL(0x3e0f11de,0x3a3c3087),LL(0x2fd9dd8d,0xefe676ed),LL(0x227a4771,0x250b4a16),L_(0x000001d2), + LL(0xe1817fcc,0x7ff2aece),LL(0xe4758b83,0x15a3e785),LL(0x768947ca,0x54660e77),LL(0x2c352eca,0x1486538c),LL(0xaac39b78,0x86e8fec5),LL(0xaa608004,0x414e550f),LL(0x32acb85c,0xa6493364),LL(0x2fd7f2f3,0x3d144499),L_(0x00000085), LL(0x92784c4d,0xcdd72f6c),LL(0x18258546,0x7a0d4685),LL(0xabc0f043,0xa00c87a4),LL(0xfab6104e,0xd492feca),LL(0x0edfb4e6,0xd10ae319),LL(0xa0ad3d18,0x74331002),LL(0xb3e27cea,0x1c928b0f),LL(0x43d33a89,0xb7eaffcb),L_(0x00000056), + LL(0x7f32173f,0x5c89c248),LL(0x82306719,0xa569342d),LL(0x5318cae6,0xaf11c888),LL(0xb3871b59,0x43ea3a04),LL(0xccb1894e,0x7d6e369f),LL(0x584ca2b8,0x4a0018e5),LL(0x01476d73,0xc7bd79b4),LL(0x6328258d,0xed62337f),L_(0x00000095), LL(0x607af994,0x27381ef0),LL(0x686aee98,0x28efc122),LL(0xe773f07b,0x1e1202b4),LL(0xb162dfca,0xd4141270),LL(0xfcd08076,0xdb1eed20),LL(0xd4dbdbd9,0xec8f2a2a),LL(0x31a47c86,0x153d8ef2),LL(0x74c6410f,0x33a76f22),L_(0x000001f8), + LL(0x0b272d33,0xb7894299),LL(0x2ade0047,0x7215a462),LL(0x8525f896,0xe97dd7d4),LL(0x8faa7fcd,0xf320c207),LL(0x2aaff4f3,0xce32f0aa),LL(0xf98216e5,0xe5a62be1),LL(0x4be7ec71,0x057e6071),LL(0xe8262bc9,0xd6f6ecc9),L_(0x000000e4), LL(0xe0348118,0x1e1c1702),LL(0x67b5b771,0x61dc410d),LL(0x540bfa59,0x52daedcf),LL(0x722428bd,0xa96118aa),LL(0xced4360f,0xaa07a68e),LL(0xd1ae09f8,0x4870992b),LL(0x98c1f34f,0xf97358de),LL(0xea267e80,0x8837a9e0),L_(0x00000086), + LL(0xb75d5e00,0xba7367a8),LL(0xcb6a4c5d,0x698ec043),LL(0xc4a8a172,0x58a0e780),LL(0x1c52f090,0x45a0c118),LL(0x7ba85810,0x41f652a4),LL(0x261486ed,0x14a0dead),LL(0xe61b0bd4,0x4a38be55),LL(0x881f7207,0x554bf84d),L_(0x000000fb), LL(0xb678cf5d,0x8cf0f2b1),LL(0x1c805e0f,0x50bc855f),LL(0x4c9f70d9,0xab5b49a8),LL(0x82a11ee6,0xbf5c0c4e),LL(0xecca8fd8,0x30c1e91a),LL(0x40104321,0x3bccd5ea),LL(0xf20e8305,0x7cc38a3b),LL(0xa89c9d80,0xf84d4b44),L_(0x0000008d), + LL(0x6dc98840,0x33535047),LL(0xa105e17e,0xf240ea34),LL(0x0ca7c1ed,0xe0a7225c),LL(0x60ee9bcd,0x8d5abc2e),LL(0xd1b7a04f,0xed201196),LL(0x421fd636,0xee08dcde),LL(0x3a41da5f,0x4d648f1c),LL(0x37a2b18a,0x637ab14b),L_(0x00000031), LL(0xe2574ca3,0x8f4aa46c),LL(0x42b5000c,0x5e6cb8b1),LL(0x2cc007b3,0xd18aeecc),LL(0x139d4602,0xfad62b8b),LL(0x4857b6e6,0x0b803515),LL(0xaaf5703a,0x7dfe5be4),LL(0x5b88d9b4,0xc7e255f1),LL(0xb42f23b0,0x642d5cee),L_(0x000001a2), +}, +/* digit=25 base_pwr=2^125 */ +{ + LL(0x9827bf41,0xd6c228d6),LL(0x53bd6003,0xac8482db),LL(0xedd6d84e,0x199f6c6c),LL(0x554b59c1,0xc80a2555),LL(0xbb3dd0d5,0x9a255d70),LL(0xb61698fd,0x8ce8ece5),LL(0x01602388,0x0910e4ff),LL(0x21f2b5b4,0xb877de8b),L_(0x00000006), LL(0xba9be6e9,0xe014bad7),LL(0xd6c8e28b,0x7941a6f1),LL(0x983d3be4,0x93e374aa),LL(0xb03efe8a,0x7787501a),LL(0x2ecc1517,0x3863f010),LL(0x8ce1a07f,0x2339ade0),LL(0xb1181652,0x142e138f),LL(0xed660839,0xade2437d),L_(0x00000072), + LL(0xb7c246de,0xbed2f33b),LL(0xc7b5006a,0xd46decde),LL(0x50c509c6,0x83eafeed),LL(0x09502cf6,0x6c8d2171),LL(0x6fa7b091,0xe284eb82),LL(0x6ef3971c,0x5478a9a0),LL(0x7e812b4b,0xbedbb05d),LL(0xbdf3afd0,0x5880bfa9),L_(0x000000a8), LL(0x17c0e4a6,0xde9fb976),LL(0x510f0d79,0x2d46f889),LL(0x5085caf9,0x57625cb7),LL(0x63379f4c,0x7679eef9),LL(0x202dc487,0x61e8da06),LL(0xd95a7481,0x933c7094),LL(0x6f198e77,0x7e527ab9),LL(0x3cef9bb6,0x3556a0a2),L_(0x000000a3), + LL(0x587ef556,0xadb4b17e),LL(0xe6db7725,0x223554b9),LL(0x298840a9,0x8ea40d6a),LL(0xb9987d3e,0x088f1989),LL(0x8c544359,0x98c4e679),LL(0x26877124,0xd4955574),LL(0xaeb47579,0x42531911),LL(0xedd6bd8c,0x876a0c25),L_(0x00000030), LL(0x17da2be3,0x578452ef),LL(0xf3506ed1,0x26ec7e64),LL(0x400c530b,0x0a9d93fd),LL(0x42c14bcb,0xeec28064),LL(0xdbc44330,0x21d894ab),LL(0x1784b7a3,0x83284ca2),LL(0xbd2fe673,0xbdaabf2c),LL(0x333a314b,0xdd217a0b),L_(0x00000109), + LL(0x6110cba8,0x3dead375),LL(0x261b1296,0x24e572ee),LL(0x4f710c53,0xa4d924c1),LL(0x3234879d,0x2bb72d3a),LL(0xf0242c6b,0x5319d73b),LL(0x56b72596,0x5d438ac3),LL(0x9c1467ec,0xe4eb1ea6),LL(0x40556d55,0x74c566ea),L_(0x00000000), LL(0x113bb0cf,0xf755482f),LL(0x1fdd8292,0xb750229b),LL(0x36eb56b3,0x8756dd9d),LL(0xd65055f0,0xad24bc9f),LL(0x305fbea1,0x29626eb1),LL(0xfcecb5ba,0xc9855409),LL(0xf6273264,0x81000d0a),LL(0x9d561b22,0x7b8d7b24),L_(0x0000017f), + LL(0xd531bd0b,0x1f33fdc4),LL(0x1e83cdaa,0x527f8e3e),LL(0x867d160e,0xf198e03e),LL(0x1f8e836d,0x319f12f2),LL(0xe5494da5,0x312ddaeb),LL(0x8aa887f2,0x0cacf5c7),LL(0xab111707,0x0ac8def9),LL(0xbe88c645,0x9ea7eeaa),L_(0x000000e2), LL(0xaae9a35d,0xecaceba5),LL(0x4a0a292e,0xbb26ecc4),LL(0x686acc28,0x1e45b0f2),LL(0x2a87d12e,0x3a62004b),LL(0x0c521e1f,0x1147391e),LL(0x2c697526,0x4d3ecffc),LL(0x940dd92a,0x45f78060),LL(0x3a2ded9e,0xeb3a17cc),L_(0x000000cb), + LL(0xf734ebf4,0x6012408c),LL(0x62256296,0xf1399678),LL(0x6234e097,0x152b073d),LL(0xd0a76b3e,0xbf3c9a35),LL(0x1dc1794f,0xca7a4461),LL(0x0ba3b03d,0xc31edda4),LL(0x3859cdff,0x8b3288b3),LL(0xf848ef1a,0x47d3b9b9),L_(0x00000067), LL(0x8f1d82f1,0x25e150c8),LL(0x41cecb20,0x4d109c13),LL(0xc9d21d04,0x7441f09e),LL(0x7778b13b,0xe84ff4c4),LL(0xa32c0c4b,0xc5ccc687),LL(0x309d686e,0x4ddb0a19),LL(0x9203c78d,0xba0868c4),LL(0x53181ea2,0x064154d0),L_(0x0000002f), + LL(0xe3ff2a58,0x31973b19),LL(0x2a26ad18,0x67b2d91a),LL(0x5fcc6c2f,0xe2db81e3),LL(0x0637d795,0x74742bd8),LL(0x1ed4fcac,0x26659e88),LL(0x30b9bfbf,0x232b6d3c),LL(0xae535c11,0x97bb1796),LL(0x32eef414,0xf6fe8c4f),L_(0x000000cb), LL(0x4a8e4230,0xc9a735e1),LL(0x8c58bcc2,0xa2ae4a3b),LL(0x1cf20755,0xcba626bb),LL(0x30e29d2d,0xc537d49e),LL(0xa170a87e,0x2ce7cb6e),LL(0x6a6c16d0,0x5f03a6b1),LL(0xa45e1673,0xe7f13685),LL(0x5d8c9454,0xdc67d748),L_(0x0000006a), + LL(0xaf97f8c2,0x16b51e78),LL(0xbb4d7657,0x4d4e4ac9),LL(0x12ece85a,0x2a2be63f),LL(0x2c2556ca,0x191c3b7f),LL(0x12341b0c,0x6c15ecee),LL(0xdf666379,0x2e302dd7),LL(0xce9cb829,0x76d162a4),LL(0xa7f8ba92,0xead863df),L_(0x000001b4), LL(0xd8403973,0x56dae839),LL(0xd9d38a99,0xd9da7dcb),LL(0xb69b8acf,0xd93d0fff),LL(0x4e0adb2a,0xf74f0454),LL(0xbb2ad644,0xb5de013b),LL(0xd489e7d5,0x944ef674),LL(0xa2d2bd3f,0x0ae01d0e),LL(0xdd32d1ec,0xf54aa8ae),L_(0x000000b0), + LL(0xe4705f69,0xcf1b879b),LL(0xacfbcaf7,0x1be6ac8d),LL(0x7318370c,0xdc61b734),LL(0x68c96561,0x0073d96f),LL(0xb94c34d8,0xc1901cf0),LL(0xf081cf45,0xe5c4c386),LL(0xf0fb0845,0xcbb72560),LL(0x26daccc8,0xc0c2c739),L_(0x0000003c), LL(0x649de0ae,0x12fe8c98),LL(0x69621218,0xdfb8607d),LL(0x8791c2b5,0xe9d74f1a),LL(0x3844e43f,0x58b63a6e),LL(0xa8d06c72,0x30e1aac8),LL(0xb6d9b103,0xc4264540),LL(0x3d6167d2,0x08191333),LL(0x68c04430,0x84141b67),L_(0x000000ed), + LL(0x1aa56828,0x0367e359),LL(0x3cdae245,0xb804f8f2),LL(0x72553e1f,0x67c4cfca),LL(0xb65b5da8,0xd9a5c285),LL(0x9a1f0411,0x613cad66),LL(0xba23bbe1,0xf8b4e4e2),LL(0x1cef34cc,0x8c65734b),LL(0x932e9f3f,0x02f73b5b),L_(0x00000186), LL(0x0351d0a0,0x59bb05ec),LL(0x31868efa,0x8bafab58),LL(0x0873b1c0,0xec7f2fe3),LL(0x3643b183,0xc1a9b522),LL(0x8e06d826,0x5a21bdcb),LL(0x8e78107b,0xd0770856),LL(0xf66af571,0xb9c9076a),LL(0xc46c020c,0x4aa90b62),L_(0x0000012f), + LL(0x5dc8f4b8,0xa33c20bc),LL(0x7ec83bd7,0xd911b15d),LL(0xd15a6121,0x4a74a6f1),LL(0x7ded1664,0x38816e97),LL(0x011743f8,0x3193fcc0),LL(0x2ce300dc,0xda43c181),LL(0x4a353b8c,0x15a04d1c),LL(0xc667d3b1,0x388d9585),L_(0x0000019c), LL(0x0bdf93dd,0xe0cceb86),LL(0x0ead0ff6,0x54678a08),LL(0x869bbb72,0xb18f6184),LL(0x7bd575cd,0x64b65577),LL(0xa032d6e4,0x7c7dc54a),LL(0x322afc12,0x30a518c1),LL(0xb73e6fec,0x94b0be46),LL(0xfb67de43,0x33b5236d),L_(0x000000fe), + LL(0x0f962f35,0x9c6f091e),LL(0x29586d09,0xc7324d43),LL(0xf0870634,0xfc9e4d8f),LL(0xa54095ca,0xb869d9b8),LL(0x750af3db,0x6d2001cc),LL(0x1a6baecc,0x24533332),LL(0x4d43331a,0x73cd1354),LL(0xe8c54106,0xc8a6bf97),L_(0x000001c2), LL(0x24bca00e,0xe99a79ea),LL(0xbfa3857b,0x6523d1b7),LL(0xf152a797,0x74d5c2b9),LL(0x7c8d0d7e,0xca070e93),LL(0x0c8c05da,0x90e17c3e),LL(0x6e856e17,0xc09e9cee),LL(0x45014958,0x157a95c9),LL(0x8be88b6b,0xb046e21d),L_(0x0000015b), + LL(0xcdd92148,0x641d359f),LL(0x2502c5ea,0x6f35d51a),LL(0x3893c7d7,0x20bf4812),LL(0x2ac899fa,0xea66bfcf),LL(0x3dd9d780,0x6686f753),LL(0x5853eeec,0x471826dd),LL(0x3f6607f4,0x63551e77),LL(0xab0845ea,0x9591457e),L_(0x0000004b), LL(0xc75e008e,0x1eb5093e),LL(0xf1fc3d61,0x48e575c1),LL(0x02888aae,0x04ab23f0),LL(0xe87f1ead,0xae16fee2),LL(0xb7f7d076,0x3eebdb5c),LL(0x94d4a8d3,0x1d42f789),LL(0x32f711dd,0xb65c5dfe),LL(0xffe8ae2e,0x5368ab2c),L_(0x0000014c), + LL(0xc71e34a7,0x960dca9f),LL(0xb04fd5f9,0xf94be13b),LL(0xcb350c8c,0xad91afd4),LL(0x507fe2c4,0x64d4307d),LL(0x965e3503,0x5c7ae781),LL(0x150451f8,0x6bf2a6c3),LL(0x730da87b,0x2d1774ae),LL(0x075f7ca9,0x5f606798),L_(0x000000a6), LL(0x7d9d82a2,0xdf7b7ba4),LL(0x9f994c7a,0x02b12659),LL(0xbc50a3bf,0x9dfdd3b8),LL(0x383c8539,0xfd8d4292),LL(0x17ae38e3,0xf28f2f03),LL(0x882096f8,0x5cc24a79),LL(0x4e0ef573,0xf15428a2),LL(0x57f145e4,0xb89880e7),L_(0x0000000c), + LL(0x457824fc,0x185ab84d),LL(0x1253397d,0x8d154628),LL(0x387df8c9,0x6bebdcd0),LL(0x9150bff8,0x556713ef),LL(0xe0119e69,0x47194289),LL(0xea336304,0xaea5316a),LL(0xfcab6f8a,0x32095630),LL(0x9256e44a,0xf5be137f),L_(0x000000e7), LL(0x91535ac7,0x1d1ebf3f),LL(0x100cda53,0x2af14479),LL(0xebfd994d,0x0287bad1),LL(0x075babf0,0x868eb0f1),LL(0x4f27433f,0x59c4864e),LL(0xb3ca6bbe,0x042e0b78),LL(0x36fc642d,0xc718e121),LL(0x457b51e4,0x6451668a),L_(0x000000f1), + LL(0x8e3c3743,0x2cea274d),LL(0x79b2083b,0xf6accb4a),LL(0xf7eff159,0x1a2ac9cf),LL(0xbd1a458b,0xc30597c5),LL(0xdaf5afd8,0x67ad0a34),LL(0xad0ce95f,0xfcb5f547),LL(0xf492633f,0xd42c927e),LL(0xd70d201b,0x677f0118),L_(0x0000014d), LL(0x7325271d,0x511be774),LL(0x532d9f83,0xe33f2540),LL(0x0e1e6624,0x6202d9c5),LL(0xf8f4394e,0x9c8fa1b9),LL(0xf8528991,0x2359d3b9),LL(0xd88ed641,0x4c00c9ea),LL(0x054c125a,0xbd626daa),LL(0xe0db1f33,0xbfaf8853),L_(0x00000029), +}, +/* digit=26 base_pwr=2^130 */ +{ + LL(0xd73be466,0x565e43ad),LL(0x8ce3b9fd,0x4a046e43),LL(0xef2d69e6,0xb337e9ed),LL(0x7f11d4e7,0xb4d2646c),LL(0x09fce23d,0x9cfe36cf),LL(0xf8577ee6,0xd497797f),LL(0x1e1b23f9,0xba0fa9f7),LL(0x813fdfce,0x1ec2f2d2),L_(0x000000cb), LL(0xbc5801d3,0xba1d6ad8),LL(0xfa8c88b9,0x38f8437e),LL(0x58d2c493,0xdf5755dc),LL(0xa5d4147a,0x9f31388e),LL(0x2454e0d1,0xd880f0ef),LL(0xed7c5174,0xf4ab4400),LL(0x2972f596,0x422f97c0),LL(0xfd1f05bd,0x1b6edbb7),L_(0x0000009e), + LL(0xe5308733,0xcdb37e83),LL(0x48081b75,0x60b5bfda),LL(0x38365296,0x9f69f061),LL(0x88a8974b,0x5fb9ec96),LL(0x75444cc0,0xf252002f),LL(0x899c5a67,0x664675a1),LL(0x11db7cc9,0xc6b6d7be),LL(0xe5e85617,0x8bf19549),L_(0x000001df), LL(0x650536e0,0x7897a846),LL(0x57bdeceb,0xb8acad39),LL(0x39f416b8,0xbb4ba894),LL(0xde12e814,0x45c679cf),LL(0xfa77e0ef,0xbfcd091b),LL(0xae92f35b,0xf3ea6cc5),LL(0xff4f9db9,0x15f66583),LL(0x67f0fed3,0x81b129a8),L_(0x0000009d), + LL(0x647774c8,0xd728f100),LL(0x8216c030,0x7565d29a),LL(0x38976a5d,0x0e8d40b2),LL(0xdebd4cac,0xe6c701d8),LL(0x3dc8c008,0x71a01dd8),LL(0x54f5f816,0x85aadb00),LL(0xe571a7d2,0x66dfeb71),LL(0x0d64dc32,0x2213f7ff),L_(0x000000b1), LL(0x8476568e,0x70c9c24c),LL(0x81e7d6f2,0xdfa45074),LL(0x8ce07818,0xc75e724b),LL(0x17be95c5,0xf85a8c49),LL(0x56216aaf,0x71eb7f6d),LL(0xf60fc3e9,0x4afdaffe),LL(0xb5697356,0x598d1d44),LL(0x2dfe785a,0x78a83874),L_(0x000001b6), + LL(0x70487d30,0xa5efc98a),LL(0x56482796,0x86f3d005),LL(0x81ed5742,0x41ac177d),LL(0x693c9188,0x41f63ff3),LL(0x544078e1,0xcb0cceba),LL(0xf396ad9c,0xcd9ca803),LL(0x1f2f8905,0xb9a3b9f8),LL(0x4318691b,0x399dbc5b),L_(0x000000c9), LL(0xf876e309,0xb6fd45a8),LL(0xf87881ff,0xa8a0715e),LL(0xb8d73d7d,0x074192a0),LL(0xca88981c,0xdc66d086),LL(0x00f41a80,0x8f279d46),LL(0x34882bbb,0xb5564038),LL(0x10c7a90c,0x5552b11c),LL(0xf89b04d8,0x834a5053),L_(0x000000bc), + LL(0x3f7dbd38,0xce9c88b4),LL(0xf194c13b,0xdc04befb),LL(0xdd6c7f32,0xd71b8746),LL(0xc7a2d3eb,0xb71fb09e),LL(0x497484b9,0x73e11c5b),LL(0x1fc70d7c,0x9831a6ee),LL(0x15940a74,0xc9a49067),LL(0xe36e9b20,0x365b0985),L_(0x00000144), LL(0x54606829,0x7dd8cbe9),LL(0x0d9bdc27,0xc774aae0),LL(0x36955f4f,0xcfe0f91f),LL(0x72271ae4,0x1d88d914),LL(0xc0f2a388,0xe1f3ebda),LL(0x63cec6da,0xf2b86354),LL(0xe4a5ad95,0xed0252cb),LL(0xedde22e8,0x7a31d7c3),L_(0x00000152), + LL(0x840ba74d,0xab9733bd),LL(0x35000794,0xc171b7dd),LL(0x7a0a699e,0x370bb4fe),LL(0xed68a491,0xfb486be6),LL(0xf15b9567,0x86467e73),LL(0x5a72e34b,0x007fbbba),LL(0x4fc2fd9c,0x07f9990e),LL(0xf83d0453,0x47ba1009),L_(0x0000014d), LL(0x3ed4cc4e,0xd0aa85e5),LL(0xfa4eda85,0xcc6de111),LL(0xab8aa3e2,0xd8d585dc),LL(0x43bc8ccf,0x69adf3a0),LL(0x9f03e827,0x2ce58643),LL(0x4e3d11d6,0xf05e13fa),LL(0x2820b6d0,0x7af921ff),LL(0x94e1a5fd,0x91b383f3),L_(0x00000193), + LL(0x8c47f3dc,0xfd8a756c),LL(0xca9eb3fd,0x31799e67),LL(0x5933facf,0x70399eb0),LL(0xe0504d9d,0xdc761184),LL(0x469e7106,0x8ef17d6d),LL(0xcd5f283f,0xb55ec3df),LL(0xdaa7f2c8,0x7711b93f),LL(0xa9a6a6b5,0x3add4e0a),L_(0x0000007c), LL(0xc75a128f,0x9a94b1bb),LL(0xa3a9b3b6,0x99889957),LL(0xd56e141d,0xc45c74e4),LL(0x969c754a,0x455c4484),LL(0xf069f686,0x7584cae6),LL(0xbd579d45,0x441fc298),LL(0x29bfd918,0xea727ee8),LL(0xb0624772,0xd66de027),L_(0x000000c7), + LL(0x7b2e1e8d,0x3ae58142),LL(0x2e71222e,0x90f869db),LL(0x9d393376,0xb1ce0668),LL(0x2d537bfd,0x1a9bff70),LL(0x47346bbf,0x4aeeb846),LL(0x8a0e90f5,0x73c9dd46),LL(0x54e3afa2,0xb6c871a6),LL(0x5945d8c3,0xf25b8746),L_(0x00000110), LL(0x100e770a,0xac70e87a),LL(0x1c87dbe4,0x797d6d91),LL(0x961a5c5e,0xc5b533fb),LL(0x548c0001,0xb560cfb8),LL(0xa9d47191,0xa65c8463),LL(0x37d39eec,0xcad37d21),LL(0x716bab4e,0x7b0514ad),LL(0x89ad5bc2,0x4b2c1f87),L_(0x000001e9), + LL(0x7c1ff897,0x4bd5aa83),LL(0x73534a22,0xb8d76f5e),LL(0x26abe76b,0x8f3282b7),LL(0x76978114,0x14a5cb17),LL(0x1bff40a3,0xb7375a3c),LL(0xb7209f08,0x91b36a89),LL(0xb4553b1e,0xcebaa86d),LL(0x73824616,0x8a3a95bd),L_(0x000000b5), LL(0xbcb95506,0x36ce8449),LL(0x45813245,0x0d1e1b38),LL(0xd6d0eea3,0x7fd0d6a4),LL(0x14a3ad4b,0x14bcb34d),LL(0x4fc99703,0xf4772d1d),LL(0xe5d8c8d7,0x1d59825b),LL(0x6cc8f63c,0x8d26276f),LL(0xba00b77b,0x4ea936fb),L_(0x00000180), + LL(0x06031f54,0x960c67a1),LL(0x32f38594,0x09357fe4),LL(0x3b745f59,0x2a14d637),LL(0x8fa90857,0x653eeaba),LL(0x65744c6c,0xfa37b71c),LL(0xf85872c8,0x3238cb4d),LL(0x9700049f,0xbb9a7dcb),LL(0x4c8ed8c4,0x47abe41a),L_(0x00000191), LL(0x49db6e5a,0x58e268fc),LL(0x4b45feb2,0x84cf99d5),LL(0x2045f9c6,0x80f4779c),LL(0xf44869f9,0xa220c8fc),LL(0x058ad702,0x7e09b470),LL(0x948098cc,0x5bc02559),LL(0x495b8c3f,0x33da20c4),LL(0x197459a3,0x4eda80f6),L_(0x000001da), + LL(0xd2a33c4e,0xb8c0a18c),LL(0x4edc42cb,0x42d862b6),LL(0xd775f940,0x1e91d30a),LL(0x6703500a,0xba0ea3ff),LL(0xa7531dda,0x2773ec8b),LL(0x39b7bed4,0x2d04e32b),LL(0xb4d1689d,0x9117e556),LL(0xd20ddefd,0x0946439e),L_(0x000000d4), LL(0xa10a2f30,0x55b7005a),LL(0x2bae1e82,0x53323c22),LL(0x2ff6304e,0x397190c6),LL(0xd9f087fd,0xa7a8b69d),LL(0xb68e3037,0x12602cd7),LL(0x25d350ef,0x22bf670f),LL(0x86cdc0d1,0x8a47dde9),LL(0x8ee7e2f2,0x974ab69b),L_(0x00000074), + LL(0xc6dbc583,0xd24b6766),LL(0xd31b0ef8,0x95890706),LL(0xe3a35296,0xc90c51cc),LL(0xb8ed7618,0x7cff3a80),LL(0x4973ebf1,0xd473b1c4),LL(0x3a129c68,0x098525e4),LL(0x5036c9f1,0xc374031f),LL(0x3955ea92,0x77e611d3),L_(0x000000d2), LL(0xf46f1c31,0x64ddf24c),LL(0x9e1fda40,0x70db5256),LL(0x5ea2c55e,0xf8940530),LL(0xf14297ac,0x034f59d5),LL(0xa46ea96e,0x42888331),LL(0x7dc4622a,0x102ad134),LL(0xe007741a,0xfe88a514),LL(0x1db8ec7c,0xc746e046),L_(0x00000099), + LL(0x514ddbd3,0x40394ddf),LL(0xc9c65dd0,0x539c2910),LL(0x679067ad,0x449575b2),LL(0x3e4b50f3,0x3cba3f07),LL(0x3ae8deca,0xdb855b46),LL(0xde55385b,0x16ac2f4b),LL(0xcf4ed383,0x1d879d47),LL(0x1e8113f4,0x90927036),L_(0x000000d6), LL(0xce4c202b,0x3d4593d6),LL(0x973ac87d,0xf0b4acfe),LL(0x01434726,0x6764442d),LL(0xfe9274d4,0x6b582005),LL(0xe308fe9e,0xf520a500),LL(0xaae35487,0x99c31e18),LL(0xcdca5ee1,0x1d99ed71),LL(0x0f6491b7,0x05d24b98),L_(0x000001b8), + LL(0x46f862a6,0x65cb077c),LL(0x631e9559,0x9b939d55),LL(0x25138071,0xf40d4552),LL(0x952fcfe6,0xb308097f),LL(0x0a6a5375,0x2e65e8e6),LL(0x3e9edb80,0x1310ce7e),LL(0x9008e712,0x36b60d2d),LL(0xef767e69,0x0706fc9a),L_(0x00000167), LL(0xfa1dc587,0x32ad8729),LL(0x4064ce9d,0xe1763571),LL(0x56c0be29,0x7963b458),LL(0xde3b2135,0x95d575d7),LL(0x66e40952,0xa842ef1e),LL(0x444bd560,0x5e446834),LL(0x9e4dbf26,0xf024c8aa),LL(0xf4d25cc1,0x3119e4bb),L_(0x00000080), + LL(0x31f1b543,0x5b8d2482),LL(0x0ec252b0,0x19b88e25),LL(0x0818329c,0xcd8bbb1b),LL(0xdb10a837,0x02e4893e),LL(0x81192510,0x84cd1c11),LL(0xbe980656,0x6c489430),LL(0x0f675008,0x346cc643),LL(0x57e72ed9,0xa6664b52),L_(0x00000049), LL(0x8c9e3525,0xc2a2b6e2),LL(0x50c3fc67,0xaf377b60),LL(0x018ff455,0xd7f347bb),LL(0xcd5a7fd3,0x820f28df),LL(0x7a766a20,0xbaa35047),LL(0x2e3e3c08,0xea0d932e),LL(0x620422f9,0x561b15cb),LL(0x78d9ad76,0xe4b810ff),L_(0x00000012), + LL(0x0f23847d,0xcb2c301d),LL(0x46a3121c,0x24b1883c),LL(0x64fb5faa,0x43263cce),LL(0xc10bc090,0x731fce3c),LL(0xe510506a,0x134986c0),LL(0xd2899a05,0xaa30a907),LL(0xd8592433,0x6671f165),LL(0xa5074a40,0xe0e30eea),L_(0x00000119), LL(0x39b1d8c9,0x4f03f7bd),LL(0xed9a2887,0x4a870054),LL(0xbd121753,0x510756ad),LL(0x9a0d0a37,0x85faa51a),LL(0x35296053,0xdf5c089f),LL(0x15a5c2ed,0x130a237e),LL(0xbd316fba,0x3774ff2c),LL(0x2c9d3ce1,0x66d3d7ee),L_(0x000001e7), +}, +/* digit=27 base_pwr=2^135 */ +{ + LL(0x45c384b6,0x2d4100e2),LL(0x7187b9e4,0xc5264e57),LL(0x2477a817,0xcb20ec20),LL(0xc146fbb4,0xa5dd079d),LL(0x6c49fc51,0xb66b540f),LL(0xa207dd34,0x18cb3114),LL(0xfc85f364,0x79042a4a),LL(0xa886f4d4,0xf32c0592),L_(0x00000021), LL(0x3c62b595,0x7df28ef3),LL(0x09a83c10,0xc98bc18d),LL(0x61720266,0xe8b908cb),LL(0xbfa40c64,0x3266ed34),LL(0xc5f7d00d,0x785d5c5a),LL(0xed6e6699,0x0fda50cd),LL(0x0528d637,0x9fa7129a),LL(0x226a0134,0xc857ddf6),L_(0x0000013a), + LL(0x854a83b5,0xa6b72500),LL(0x82b8a64e,0xf5cc5dee),LL(0xa44f4310,0x82f7e307),LL(0xa979f999,0x26038361),LL(0x36271c95,0x9d4a6e7e),LL(0x2c2e87bd,0x83121a68),LL(0x801461a1,0xdda0c42c),LL(0xc46dd1bb,0x16d6b9ef),L_(0x00000004), LL(0xf5ff9d53,0xac7e6d6f),LL(0xba6044cb,0x8a2a18c9),LL(0x4e0b1c61,0x47645723),LL(0x538c1881,0xff1d071b),LL(0x0d20849f,0x3d943038),LL(0x033ae333,0xd1326f05),LL(0xe89c6421,0x504a49c4),LL(0x0c637164,0x5b9d0e64),L_(0x00000191), + LL(0xc4db51e8,0x0ee6ce1d),LL(0x76a9fbe6,0x471be04f),LL(0xaee80fe4,0x63fea5d4),LL(0x13ed56ca,0xbb7b1989),LL(0xff53dd5a,0xbdd30335),LL(0x5aa48cba,0x8830cbd1),LL(0xced46a92,0x6ec07f47),LL(0x4d0d3e16,0x3e149703),L_(0x0000002c), LL(0x85d83aa8,0x030c528b),LL(0x3981ba7e,0xf6347818),LL(0x51c072a5,0x8851b9e3),LL(0x6bc6f46b,0x908af12a),LL(0xab612e82,0x11ae86d1),LL(0x194bfdad,0x855184ce),LL(0x3ed70ec9,0xbc5ba81b),LL(0x36a51b16,0xbe363f21),L_(0x0000017e), + LL(0x89a7c665,0x4ba50604),LL(0xf92c410e,0x03183bca),LL(0x325bb838,0xde751063),LL(0x4a227afa,0x61ce2f62),LL(0x8d611fad,0xe1c057fe),LL(0x63741f27,0x26a80815),LL(0xcc3f4944,0xdc51e188),LL(0x1fb19202,0x18a29e60),L_(0x000000fc), LL(0x23f5c4ba,0x8b90f284),LL(0xeac00c83,0xecc8f9f7),LL(0xc63ca5b1,0xabd4ae3b),LL(0x61f4eb49,0x5868250a),LL(0xde5e94c7,0x8aa62e59),LL(0x2e205082,0xa27ce17d),LL(0x4d94b7ec,0x3cf7dcd2),LL(0x84ff72ff,0xd9add4ed),L_(0x0000006b), + LL(0xd6250a4d,0xc4c48937),LL(0xb7e17582,0x30cd4a1c),LL(0x663cb672,0x4ecce3f6),LL(0x51a07652,0xe3e24952),LL(0x971076ab,0xb2837d4a),LL(0xcfa04494,0xae48378d),LL(0x2f234848,0x35aa4670),LL(0x5204cd94,0xdbb7f2c1),L_(0x0000010b), LL(0xce99c049,0x189c18e6),LL(0x1251a582,0xe65b23d2),LL(0x1ea8f76e,0x50f4154e),LL(0xde65bbaa,0x55d8a624),LL(0xd1acdeb6,0x9745647b),LL(0xdbc7b696,0xa1a36741),LL(0xc3af97df,0x0e06b475),LL(0xcec9f674,0x09826835),L_(0x0000018d), + LL(0x9edae224,0xc93ceb2c),LL(0xc40b8881,0x376b68f1),LL(0x493ec443,0x2fe4d107),LL(0x2613f055,0x2adbc0de),LL(0xc264177a,0x6850f4d4),LL(0x999b4445,0x024b1759),LL(0xb5528e8c,0xa532c490),LL(0xfe9cb25f,0xfd3a94e4),L_(0x00000027), LL(0xea2401de,0x29f2c840),LL(0xae4f0565,0x6004e218),LL(0x9745c833,0x45a26d7d),LL(0x1aa8e8c7,0x2e1e3abc),LL(0xf254366c,0xd176c592),LL(0x5dba9a65,0x75f2ce2f),LL(0xcb70eda5,0xef390121),LL(0xdf3bd7c9,0x57bbfad4),L_(0x000000fb), + LL(0xf2fb4c5a,0x612c5e22),LL(0x90ec0ad8,0x19eef2ca),LL(0xf648d0a0,0xb08c2818),LL(0x56957806,0xc6fa4d71),LL(0xe858889f,0xd381edae),LL(0x0d311c34,0x51c58427),LL(0xc7d13fa9,0x223f6153),LL(0xe7ffd714,0x41bebc46),L_(0x00000125), LL(0xa9aa9baf,0xd185738b),LL(0xa46d0a8f,0xb3308a45),LL(0x74e9630a,0x50e76c6b),LL(0xa8af0eb5,0xe6d664ef),LL(0xb4263c27,0xd6ff5afe),LL(0x0d5ab8af,0xa29e25ea),LL(0x35f45527,0x9641d328),LL(0x8c614ae7,0xf344ace6),L_(0x0000007d), + LL(0xe352f406,0x44f05c5b),LL(0x36069000,0x3a7061d3),LL(0xe7fd3e15,0xd82371f2),LL(0xb123a32e,0xc0c29bdd),LL(0x1a15e8ee,0x0938b2d1),LL(0x9bba46b8,0xa2ae38c1),LL(0x66a69b9b,0x470c4e74),LL(0xe7a0607a,0x04517ba8),L_(0x00000067), LL(0x04e250e3,0x0d3d6116),LL(0x99aa8990,0x2850e69a),LL(0xe87aacf0,0x0f5ea018),LL(0xa9b70f5d,0xe629958e),LL(0xc9dfec50,0x67ad0ad8),LL(0xa19fef72,0xfbbc4dd8),LL(0x4e913349,0x44ef73af),LL(0x36506a6e,0xa2784d06),L_(0x0000009a), + LL(0x4fc61403,0xbe686c91),LL(0x3b319ce9,0xcaf2c252),LL(0x48b002e1,0xd7142b37),LL(0xfd368034,0x4805818a),LL(0x24a14bab,0xfef62905),LL(0x0f3cee8f,0x8e05459e),LL(0xdd641c3a,0xf8a79ba8),LL(0xda069476,0xb3c5de43),L_(0x00000084), LL(0x39168c60,0x3f0a7260),LL(0x3e3e3f10,0xefe9a0b6),LL(0x98f31f24,0x029755ad),LL(0x68a3bd55,0x56d48cfc),LL(0x08db9e00,0x180b09d6),LL(0xf41fc496,0x43518b4d),LL(0x0a026b4c,0x53fa9a78),LL(0x49c51679,0xd4505c06),L_(0x000000f2), + LL(0x04533f2b,0xc6b20735),LL(0xf37cab9f,0x59889c71),LL(0x2957243a,0xee6d3e3b),LL(0xcd4ef031,0xc82e2f33),LL(0xbe1fa792,0x9431aaa2),LL(0xd5df936a,0x5897dee2),LL(0x69038db7,0x3c5c1a27),LL(0x49337ba9,0xd62944c1),L_(0x00000141), LL(0xdaff077c,0x92632965),LL(0xd489db4e,0x73090129),LL(0x940397cb,0x3ab24c2f),LL(0x08747c46,0xa7844d0f),LL(0x4063f57e,0xde4ab15b),LL(0xdfb6a687,0x7bdc8db9),LL(0xc4b7272d,0x670393c5),LL(0xcc129fac,0x2bf452b3),L_(0x0000012b), + LL(0xd3620658,0xd490ca01),LL(0x480b6735,0xf6b97c19),LL(0x5a38261b,0xeb8077db),LL(0x3a0d7cfc,0x6cb95b1f),LL(0xd822b66c,0x027f3439),LL(0x204c12b4,0x5b1121d8),LL(0xd1662f63,0x50df8b79),LL(0x5a06b5c4,0xa26c4803),L_(0x00000195), LL(0xdd45df26,0x7d35a477),LL(0xdf45caba,0x37ca8bf1),LL(0xb9d5153b,0xa163bc4d),LL(0x2a09a7c7,0x79721ae6),LL(0xf16f3745,0x4901566b),LL(0x8b7edc54,0xda6d915a),LL(0x2073fe4c,0x59c5233d),LL(0x0e719f5b,0x8881557c),L_(0x00000041), + LL(0x34d85352,0xb5e1a9d1),LL(0x04b6fb3b,0x96ed674c),LL(0xc5869197,0x2201eaeb),LL(0xc13b24f0,0x43fe141e),LL(0xd5acb880,0x77717702),LL(0xb4c36b2f,0xf913c28e),LL(0xbd9e8fe1,0xbb8bc0cb),LL(0x871dc376,0xdf0d6014),L_(0x00000130), LL(0xb3b18239,0x9062a004),LL(0x0d96f561,0x84b9c0aa),LL(0xd38134a4,0x384e6a14),LL(0x62e9b9dd,0x434945b9),LL(0x0d2a3f87,0x26111d5b),LL(0x0558e17c,0xca088afb),LL(0x7e83601c,0x5f4109b6),LL(0xf3372d86,0x33a44ebe),L_(0x00000052), + LL(0x2e8b93c7,0x08562f0b),LL(0xbd858543,0x4642e58a),LL(0xf3108a95,0xe72a8e55),LL(0xc3b6dcd3,0x48efed30),LL(0xbf3f1b76,0xeef47f99),LL(0xbe7c393d,0x7808a959),LL(0xb13004f3,0xab865ef6),LL(0x937fdeea,0xd800a95c),L_(0x0000001e), LL(0xde622870,0x5b036454),LL(0x1435996c,0x253cdb02),LL(0xc939a75b,0x00181ca2),LL(0x497b4076,0xc885fd30),LL(0x89ffccfd,0x5be5d64a),LL(0xd221db4b,0xf3ff67a9),LL(0x7c1814ff,0xe534c2d3),LL(0xdff1b3c3,0x4c8996de),L_(0x00000045), + LL(0xc9c5a035,0x6fb80668),LL(0xd3037f61,0xf7001431),LL(0x61783bd7,0x7eb67860),LL(0xa8db044a,0x687c5be2),LL(0xbd63e80e,0x72619e19),LL(0x79bd6dba,0x3f54433d),LL(0xd3da5abb,0x53179eab),LL(0xbeded885,0x2f58ffce),L_(0x000000d6), LL(0x8c1156c4,0x0e48c339),LL(0x6a8706fa,0xd70c895f),LL(0xdef1e5d8,0x74e0aa32),LL(0x628036e7,0xb31a93a3),LL(0xa6fa3b42,0xe7bb3f2a),LL(0x91ab3f15,0xd667e0a4),LL(0x1d5276ef,0x172f04b6),LL(0xac2e330e,0x46c091fd),L_(0x000001a4), + LL(0xcaeed330,0x3479c8d8),LL(0x774a0f8d,0x2da43aae),LL(0x89fab1e0,0x5a52588d),LL(0x22017d07,0xf2088700),LL(0x3344f84f,0xf666f8bc),LL(0xcded1b00,0x98c10e11),LL(0x385b1f15,0x4a35267e),LL(0x4cb957d9,0x0bc3b5db),L_(0x000000b2), LL(0xb7f4f85a,0x70aae220),LL(0x50f81138,0xd0547dcc),LL(0x320d34eb,0xaa86f5d7),LL(0x4627a90c,0x313d3af2),LL(0x0d86c9fe,0x9d1708c0),LL(0x93baaabc,0x4bb0c611),LL(0x5e3713af,0x8c78d7cf),LL(0x23abcabf,0x24042f80),L_(0x00000150), + LL(0x8d760d23,0x2a50426c),LL(0x2fd748bd,0xd4451d72),LL(0x84a5084a,0x7d518774),LL(0x395bd1ac,0x41ad7719),LL(0x5dc03d65,0x54b40eaa),LL(0xf42c68a2,0xc699a962),LL(0x481b2b4d,0x78f2ecdd),LL(0xd9badbf1,0x6d040345),L_(0x00000038), LL(0x6890c940,0x7a9849bb),LL(0xe8615e51,0x5822be91),LL(0xe3c3e516,0x9ed67ca7),LL(0x5ebee67a,0x5438f44c),LL(0xbf03236f,0xf9e45ec0),LL(0x29c5029e,0x412d0011),LL(0x4fd4f4e3,0x09bad0b6),LL(0x5f591e3c,0xd09fc0e1),L_(0x0000006a), +}, +/* digit=28 base_pwr=2^140 */ +{ + LL(0x6867ca62,0xb2e8cc83),LL(0x2abfd678,0xd7d6c96a),LL(0xbb6c702c,0xb7b75f62),LL(0x8eb9ab34,0x2a8eb698),LL(0x67b38227,0xee1d1728),LL(0xbff15e40,0x6f600751),LL(0x4ec3001b,0x30ff996b),LL(0x7fb8efdf,0x35fa815d),L_(0x000001bb), LL(0xfc62d76a,0x1c80dd81),LL(0x4a2f2f09,0xc1a9825d),LL(0x4ae9b61a,0xb05a4fb5),LL(0x71a812fc,0xa7baf2db),LL(0x8bb96eaa,0xcc434e4e),LL(0x53c2dfd9,0x8fce5672),LL(0xceeb8e7b,0xd6b948ee),LL(0xc787b7e9,0x44e8daba),L_(0x00000053), + LL(0x44566d20,0x816dab3c),LL(0xa555ef8d,0x68ad0a5e),LL(0x93fa3eae,0xb45ab760),LL(0xad51a41f,0x14a732bc),LL(0x3c784a11,0xcd96f357),LL(0x7e912d99,0x7808bc95),LL(0x547dff3f,0xd022a461),LL(0xd3f93d98,0x51efde9d),L_(0x0000010f), LL(0xdcf5792b,0x9e50e443),LL(0x1c5d0319,0xab35921f),LL(0xce7e3777,0x61acb763),LL(0xc69a2c80,0xd5a1f19e),LL(0xd4921d8b,0x86d49b86),LL(0x3effd3f1,0xd287849a),LL(0x969ee2c3,0x2319a1d3),LL(0x7987e8d9,0xda41b97e),L_(0x00000077), + LL(0x66e6b355,0x48d7c646),LL(0x494cec8b,0x4319bb26),LL(0x3c15f132,0xa4923bd5),LL(0xb25b7340,0xe36296a4),LL(0xd2c82187,0x62a70b23),LL(0x3a2676cb,0x3ce0a44b),LL(0x15ada951,0x93e13762),LL(0xcdd5bfa0,0x7e7884bd),L_(0x00000094), LL(0xe16f0577,0xeb1d23f2),LL(0x563bc2d6,0x74b1ae5a),LL(0x22ce417d,0xf0676c19),LL(0x8b56e586,0x64665c8d),LL(0xd3d21118,0x4a9d1f08),LL(0xb5b57a1f,0x9ad18a2e),LL(0x121b1440,0x31f16f69),LL(0xd3dba51f,0x94a4555a),L_(0x00000186), + LL(0x14a0990f,0x1571f4bd),LL(0x8a12a2ae,0xa7e98142),LL(0x64ea4bd5,0xf548a570),LL(0xc2f56d89,0x3a99f05e),LL(0x24fcfb51,0xb029c28b),LL(0x468881de,0x16eb364a),LL(0x54a22d8c,0x9df6df67),LL(0x8e7ba7c2,0x2986939a),L_(0x000000d8), LL(0x6d585b84,0x8b87eab6),LL(0xf8f2e668,0x61b8a4ae),LL(0x1b210ab1,0xcd5968d0),LL(0x38c32d9f,0x9469f27a),LL(0x2170203f,0x7e65bf26),LL(0xdf5327ba,0x268e8f3d),LL(0x0d743f23,0xbbd5d6a5),LL(0x6866dcf3,0xebf3b6f7),L_(0x00000010), + LL(0xfe75ceb3,0xc2dbf93c),LL(0x7d044fdd,0x05aa3d0f),LL(0x3459ab15,0x1e4c0404),LL(0xbea051fd,0xeeca2cbf),LL(0xa5c86723,0x428637a5),LL(0x81d9dd90,0xd3aca9d5),LL(0xf6461276,0x78277709),LL(0x5fdc5888,0xc80ded9e),L_(0x0000013d), LL(0x105fdead,0xb799ae3c),LL(0xc919db59,0x5e3595ac),LL(0x2aa1f7f7,0x4e9b6f6b),LL(0x519dab32,0x1054eecc),LL(0xd70aa0c8,0xdab1fa02),LL(0x45046840,0xe8162c46),LL(0x382d8fac,0x3f7fc117),LL(0xc63a2e34,0xf8b4f79c),L_(0x000001dc), + LL(0x40f45643,0x5e42072e),LL(0xb22b4730,0x6980bc47),LL(0x0c0959ae,0xd0091f48),LL(0x17382117,0xe76ce6df),LL(0x6fb6755f,0x083b1371),LL(0x8e338195,0x3ce92877),LL(0x57844465,0x22eadd23),LL(0x88650fd1,0xcb97b5f2),L_(0x00000194), LL(0xb832d4f9,0x40795011),LL(0x2f5eb6ec,0x56106a16),LL(0x439d72fa,0x7a360472),LL(0x9a695980,0x77c4b5ed),LL(0xbd3315f1,0xcd83808f),LL(0xc773b196,0x21f3f41d),LL(0xdcca40dc,0x42518607),LL(0xd975bf10,0x6fd38920),L_(0x000000cd), + LL(0x7643d0a4,0xca61488d),LL(0xc9a4ec9b,0x78d40864),LL(0x08ac32aa,0xd1f91912),LL(0xe2c33dbb,0x4ce17265),LL(0xa6b041d8,0xc73e5e84),LL(0x130222f6,0xcaf07f55),LL(0xbc20bdd0,0x2fe0bc76),LL(0x482195b2,0x6fe4ca43),L_(0x00000141), LL(0x37f04c87,0xbdd6ee14),LL(0x601822b2,0xb9431fd2),LL(0xf10879b1,0xebee54b7),LL(0xb8d5c027,0x530c61a6),LL(0x52358509,0x3b953e07),LL(0xc05d71ee,0xd055e247),LL(0xfc120f31,0x51f78c21),LL(0xb71a77f5,0x8d424c40),L_(0x0000008b), + LL(0xdcca1655,0xfcdcd83f),LL(0x6f13723c,0x6fe01dad),LL(0x48fc724e,0x10ea92f4),LL(0xe9506b35,0xbacd3171),LL(0x32c50316,0x5b9c3826),LL(0xb525812e,0xb06a2041),LL(0x6095bd35,0x29d1e51d),LL(0x8c9f2aff,0x03f8818f),L_(0x000001ba), LL(0xf5e8440d,0x5815a76f),LL(0x4652dd8f,0x0ba6e7eb),LL(0xa2d772d1,0x67d0cfa7),LL(0x2c2c10a3,0x9134fbe1),LL(0xe122408a,0x4d3fc999),LL(0x98396be7,0xf158ed72),LL(0xf1736447,0x2e445a86),LL(0x3589b3e7,0xd638a810),L_(0x0000013e), + LL(0xaa39db8a,0xcd1bfa5a),LL(0x3f690830,0xf20580fd),LL(0x47516625,0xc02a443b),LL(0x72df9c02,0x37c50805),LL(0x1f658c86,0x70ba4df8),LL(0xb9b7c15e,0x7863af7e),LL(0x4f048a5e,0xac437359),LL(0x985ed711,0x9b762ef5),L_(0x00000035), LL(0x31deb67a,0xf7ff8403),LL(0x277a75a7,0x9efc9dd1),LL(0x9e038122,0x72ab76fd),LL(0x380f72e2,0xa5bd7ec4),LL(0x55ee2df7,0xe6e012fa),LL(0x8dba5f73,0x3daacbbb),LL(0x7d57b1b9,0x706e065a),LL(0x2a1528ff,0x9e984f15),L_(0x000001c4), + LL(0x15c609d6,0xe5366bdb),LL(0xba4c8f5f,0xab004e8e),LL(0xa55c2b8f,0xbe220e5c),LL(0x9b0a693e,0x328cf3bb),LL(0xf0a01098,0x93659056),LL(0xba4d555d,0xa9299fb7),LL(0x705141f5,0xac2b6ea4),LL(0x44c2570f,0xd6c8f88d),L_(0x00000158), LL(0x4159e7f0,0xda0acb04),LL(0xd0b0f9e2,0x72227853),LL(0x9c81b6a3,0xfca5d947),LL(0xe37b62d0,0x89f8e6a5),LL(0xa2b087c1,0x397e6f2d),LL(0x79ab8dd4,0x0c2f8337),LL(0xe811e1ad,0xaed2062f),LL(0x41fc3c1b,0x6608ac6e),L_(0x00000196), + LL(0x62da0bcd,0x66e8ec0f),LL(0xa2436a22,0xb2614d9b),LL(0xc4f2fabe,0xd37ba7ca),LL(0x91730356,0xd6947b5c),LL(0x74afd26f,0xf62dae98),LL(0x24fc84c9,0xa5d82a0e),LL(0x01183e91,0x6d7bad82),LL(0x9ae00850,0x45968698),L_(0x00000176), LL(0xf94e5ea9,0x6a5a8c8b),LL(0x370f5efd,0x4a208fd5),LL(0x0abfbfb6,0xd3eba761),LL(0xb4577a64,0xaea020f7),LL(0x9d9fbff8,0xee185b5a),LL(0x7590eb6e,0xde37c8c6),LL(0x110f6564,0x087e5b3d),LL(0xf182e709,0x2a62e074),L_(0x00000022), + LL(0x35933656,0x11881664),LL(0x57d6289e,0xb5dfe85d),LL(0xb19a5774,0x03f55586),LL(0x84a3823a,0x83e66aba),LL(0x819d0f7f,0xe6540e46),LL(0x8229f91b,0xf8e60b64),LL(0x0ebba171,0x3cb7174c),LL(0x13a992ea,0xfc144f32),L_(0x000000dc), LL(0x66b10914,0xaf137af3),LL(0x22188a39,0xa99be2ef),LL(0xc9e8bf06,0x9f80153c),LL(0xb82d6f97,0xb70bb797),LL(0x713e0f8f,0x08001bac),LL(0x2900ebf9,0xc349df5c),LL(0x2dc5150c,0xcda05b0f),LL(0x705ef690,0xf953a13e),L_(0x0000006c), + LL(0x2275d0c6,0x43c2cbe5),LL(0xb77f7c23,0x426913e3),LL(0xa4d09bcb,0x193a8beb),LL(0x1c330bb2,0x9694aec2),LL(0xf90a1043,0x466c8910),LL(0x47794b4c,0x013120f4),LL(0x92db08ad,0x27504b4e),LL(0xfd2c4ee7,0x2771ac8c),L_(0x00000039), LL(0x3983eaf2,0x1520fd40),LL(0x43f9f523,0x10ab4804),LL(0xbbc7abbe,0x4c94f219),LL(0xb3da18c6,0x0653b434),LL(0x34410d29,0xa49aa62c),LL(0x475b1588,0x3fb54eff),LL(0x1efe3b74,0xe35ee322),LL(0xb5457582,0x0e28d1fe),L_(0x000001da), + LL(0xdfafafb9,0xd5ac6ec8),LL(0x3e035e11,0xf1bfe6e7),LL(0x0cdfda44,0x99f86b4b),LL(0xd82854c9,0x51eb2ba8),LL(0xe9378d3c,0xfc70edec),LL(0x0488564d,0x78099daa),LL(0x4df1eac2,0x106d93e9),LL(0xfcd2965d,0x9106f692),L_(0x0000018f), LL(0xeb73c32f,0x65c6a4f0),LL(0x12e6dfd1,0x613a95bd),LL(0xc4753f02,0x64c8ae6d),LL(0x6ee36689,0x82594975),LL(0x85faeab2,0xff936e47),LL(0xfd968563,0x16aa8dfb),LL(0xfae37039,0x6a6051eb),LL(0x090bfcd6,0xa79a4c77),L_(0x000000f5), + LL(0x1283d38d,0xab80a4e1),LL(0xab4695b9,0xb05894a6),LL(0x37378243,0x7f2984bd),LL(0x1227f75f,0xdf654236),LL(0xe2ef58d5,0x290dd3fb),LL(0xdf64907f,0x38ba14af),LL(0xf1d428ec,0x0c38bf2f),LL(0xc2c54bbc,0x86299078),L_(0x000000eb), LL(0x07837c73,0x31dfd968),LL(0xda6621ef,0x28b820a5),LL(0xe6fe2937,0xb934b778),LL(0x2622aba5,0xe627cb53),LL(0xdff94dc8,0xa81ea0cd),LL(0x560bd971,0x9c8b6e45),LL(0x2209f943,0xdbaad845),LL(0x6e9d457d,0xf9a711fc),L_(0x00000166), + LL(0x8402ef56,0xe6626b55),LL(0x5c34e569,0xbb9dc4c8),LL(0x009d6dab,0x6746cac4),LL(0xcf68656c,0x3336b947),LL(0xfe65ab97,0xe266a898),LL(0x0371ecf3,0x5830a2ee),LL(0x1d57e75b,0xc9710982),LL(0x3e097669,0x4a1d3264),L_(0x000000c9), LL(0x78e2ad77,0xddfb754e),LL(0x284311de,0x4aaa3d53),LL(0xac9d56ca,0xfe5f5938),LL(0x19e9ec29,0x24185a04),LL(0xe89e92d3,0x746f628d),LL(0xfd0968c4,0x6959a461),LL(0x2cc1b198,0x7f39e175),LL(0x5c4efa86,0x9030ef68),L_(0x000001fd), +}, +/* digit=29 base_pwr=2^145 */ +{ + LL(0x15578941,0x1a86314a),LL(0x6a7421e8,0xe2ec4906),LL(0xe975bc97,0xa7485f37),LL(0xd59fd20a,0xe5e712ab),LL(0x5b001318,0x951133a1),LL(0x1259bdca,0x057f57ee),LL(0xcbd3b2c6,0x33dad04a),LL(0xef3153ef,0xb54dd9e7),L_(0x000001a7), LL(0x2ed37d50,0xa4e81e7b),LL(0xf8f36d87,0x5a01a3ef),LL(0x0288c3e4,0x8b372673),LL(0x846f5208,0xa991189b),LL(0x6f560651,0x71db52e5),LL(0x431caeef,0x58e36c06),LL(0xa3f98d5e,0xd8d03f83),LL(0x020099b8,0xc4c7aadc),L_(0x00000118), + LL(0x52ab1b79,0x7e2040bd),LL(0x95a122c6,0x89ab0660),LL(0xf1cb78af,0x01a20058),LL(0xc77cb751,0x31375e35),LL(0x5e133615,0xea159ba6),LL(0x524c75ea,0x7ecbfca3),LL(0xab8ae0fa,0x5719d039),LL(0x623ac91c,0x7f17199c),L_(0x000001ae), LL(0x6b1430a2,0x8450eb5d),LL(0xc47b9efe,0xafb92b30),LL(0xa9991147,0xf6824bee),LL(0xe1752c3f,0x2b160b39),LL(0x7fd6a625,0x6256f4b4),LL(0x574646e7,0x076f7bff),LL(0xe5bbdfa9,0xcc3f350c),LL(0x4642b5db,0xa6dbfda8),L_(0x00000093), + LL(0xfa21d74b,0x37719209),LL(0x8cfe5b17,0x00c8bba2),LL(0x1c2878b2,0xa620523f),LL(0x170331c9,0xa5843ac0),LL(0x8cd83b50,0x0381135b),LL(0xb047131d,0xa643b75e),LL(0xd2ab54c3,0xc5ef1464),LL(0x62ed0e42,0x2a2e87f3),L_(0x000000fe), LL(0x91bb20fb,0x78f86132),LL(0x7805c40a,0x895f7e0d),LL(0xa2a8624a,0x3ce4b54c),LL(0x6579a871,0x1b0cde0e),LL(0xd626e2cc,0x6377df41),LL(0x045193c6,0xcd6454de),LL(0x1c3ca349,0x4909db1f),LL(0xb047b0a1,0x2ac29d91),L_(0x0000015a), + LL(0xf432b93e,0x4a6f35d7),LL(0x611248d4,0x62f74f5a),LL(0xff45509d,0xef98d968),LL(0xf78b11dc,0x540d2d90),LL(0x8e0fdb4e,0xf1948691),LL(0xf839178d,0x775c9c48),LL(0x1546952b,0x2da4516e),LL(0xb05a9a42,0x7e1f1548),L_(0x0000016d), LL(0x5a0e6542,0x5c40801a),LL(0xc9bfcea8,0x8cf4381f),LL(0xecff5ed1,0x04226551),LL(0xe3765708,0x3addaf06),LL(0xbf10bb39,0xe6d6327d),LL(0xa7a94c0b,0xde98dcbd),LL(0xc9cc265a,0x9445d1d2),LL(0x39198006,0x0a4801d2),L_(0x000001ce), + LL(0x6fd53bcb,0x11b88070),LL(0x89212039,0x0fd4310b),LL(0x7c570d88,0xfb34d160),LL(0xe29cc2db,0x8d8b6c1d),LL(0x98ac6419,0x633a2387),LL(0x48918f6b,0x3767a8fb),LL(0xc7f5fff0,0x1de5bf8a),LL(0x517008cf,0x0be25167),L_(0x0000010f), LL(0xc8a802dc,0x31a9aa05),LL(0x3f45d1a4,0x955dbbaa),LL(0x019bc5a3,0xf987ec6f),LL(0x7819e65f,0xa521ab1b),LL(0x6a8b4966,0x9db12d33),LL(0x1c418ebe,0x5c25c25e),LL(0xd371d986,0x05758d98),LL(0xcdb745fe,0x95301bd1),L_(0x00000034), + LL(0x82dda7a2,0x8cb7272f),LL(0x285a44b5,0xd0fa019c),LL(0x772202b8,0x256b2dc1),LL(0xf7a1827e,0x70cc578d),LL(0xf561fd03,0xf01369b8),LL(0x4b48b6ea,0xb34eeab3),LL(0xf869dc36,0xf55466de),LL(0x10fbfa49,0xd2c42ed5),L_(0x00000049), LL(0xa79b35c8,0x72243fd6),LL(0x48942459,0x6f4d4b6f),LL(0xbe3c7cfe,0x4b050256),LL(0x273326f3,0xccad925d),LL(0xcfe66f8b,0x63feb094),LL(0xd430d816,0xe74dd574),LL(0xf5ea27b5,0x45e6d69a),LL(0xe57442d8,0x62ad1e6a),L_(0x0000013c), + LL(0x91a18dca,0x52fb15ad),LL(0x64cc9794,0x889fc872),LL(0x76b7b4b5,0x516a4447),LL(0x7f78f44e,0xe0dc9367),LL(0x03435817,0x6c0ef141),LL(0x3e179290,0xdcc3815b),LL(0xa243fcb3,0x57d2c5d1),LL(0x33e3e4cd,0xcfefc1bf),L_(0x00000110), LL(0x373d3db8,0x567532fe),LL(0xa4edcdd2,0xe2cdd2ad),LL(0x313da102,0x7dc4c171),LL(0x9b6477b7,0x10610301),LL(0xd6614ed9,0xe5dbb13f),LL(0x093e9d03,0xc78d8181),LL(0x34692c91,0xd1998555),LL(0xfad9c4a4,0xbc55ef3f),L_(0x00000114), + LL(0x22136d3a,0x5150ed1e),LL(0xf12f4a61,0x48b602d1),LL(0x58c86ca8,0x8f3a438f),LL(0x2ad94dbc,0xfd28616c),LL(0xa1741520,0xfc8f344f),LL(0x97e96926,0xa2867b76),LL(0x3f74f49a,0xc963769f),LL(0x9eafe4ec,0xf81b0338),L_(0x000001be), LL(0x8d3271ab,0xeb904c8b),LL(0x361247ec,0xcf0e8b6b),LL(0x9dc846a9,0xf58b8dfe),LL(0x1bd5a3dc,0x46766ec7),LL(0xabb872ef,0x7028f76a),LL(0x5976ea25,0x7d56cad7),LL(0xa7a4c1e3,0x50e6e410),LL(0xd9ef6dff,0x18098aa4),L_(0x00000110), + LL(0x54ca4d62,0xa4adf367),LL(0x2f1d9120,0x9de3bdf9),LL(0xa199c49a,0x911112e9),LL(0x918e1ab3,0x51c4e324),LL(0x1ab9377d,0xdecbb2fc),LL(0x089f9423,0xfbdc7272),LL(0x61643ec7,0x297b6a31),LL(0x8eafbdcf,0x2cfa0899),L_(0x000001dd), LL(0xb9b29381,0x9618730d),LL(0x5c79e6e7,0x984e3379),LL(0x9a017cdf,0x6a46a60e),LL(0xb44ef6fe,0x6fd9e713),LL(0x8cf5836b,0x2e3b6ebb),LL(0x29b6614b,0x741582d0),LL(0xa7c94b36,0xb93abf5a),LL(0xc0822faa,0x19fa7da6),L_(0x00000070), + LL(0x6d40ef9f,0x75d98fac),LL(0x52ee8497,0x4f994b00),LL(0xb0754aa8,0xae60032c),LL(0x19b6eb82,0xb89fa32d),LL(0x3aea1e12,0xd3d62cba),LL(0xa47b84ef,0x7b3e3f24),LL(0x3738323f,0xa1811a10),LL(0xa83238ea,0x5ad6df01),L_(0x000000fb), LL(0xe600e837,0x2ba5692b),LL(0x25fadbb6,0x8c4ff4f6),LL(0xad437e54,0xfa9d42cd),LL(0x14c8f3b0,0x79e73eb8),LL(0xa0355c3e,0xee8fbd21),LL(0xefee74e8,0xb4ebba9f),LL(0x0e987b86,0x0e79123c),LL(0xa0018bb1,0x9f828609),L_(0x000000b4), + LL(0x68def816,0x3f12ff36),LL(0x4b57900c,0x7fffe102),LL(0xd2939213,0x70f61f2a),LL(0x4ecb6d5f,0x351a0540),LL(0xca3d4a8e,0x51a7737d),LL(0x887af4be,0xbc6bf04e),LL(0xfca084af,0xafb6ef2e),LL(0x80de41d0,0x1f45db0a),L_(0x000001a1), LL(0xa72b2fa9,0x9b7182ad),LL(0xa970074e,0x18bf55a2),LL(0x056574a9,0xb8d1ebac),LL(0xeba9a5ac,0x4bbdf7b3),LL(0xd324a4b9,0x20cc2ce0),LL(0x56572fe4,0x1a2b2538),LL(0xf24f0245,0xef07dd5e),LL(0x5ab8b3cb,0x4d463bcb),L_(0x0000013a), + LL(0x41cdedbf,0xcbb3dcf6),LL(0x8c3fb216,0x19500d09),LL(0xe213167c,0xe814428b),LL(0xac93cb34,0x1a28a2b6),LL(0x861cf475,0xbc74e6e7),LL(0xcc2d45ad,0x0f8c1d18),LL(0xbd9bdb71,0x6d7baa7c),LL(0x43067701,0x1f4e3b7d),L_(0x00000024), LL(0x8a8c2d8f,0xc8929c33),LL(0xc43c1d40,0x819f1cba),LL(0xdda7d3c4,0x598c12f3),LL(0xe612ee48,0xaa092a4b),LL(0x97324657,0xd55e9103),LL(0x1b8a4a06,0xd7a8f2d0),LL(0x010537d8,0xf7a0ab83),LL(0x9ae31bf0,0xc6e0b42d),L_(0x00000083), + LL(0xf56f7c26,0xac1feb24),LL(0x4df5d838,0x15563b3a),LL(0x1ca4e8a0,0xbf968a88),LL(0x62060557,0x3ca8c519),LL(0x46507367,0x743fec64),LL(0x374e7834,0xd6eda8e3),LL(0xe0db390d,0x64260f14),LL(0x96c53e95,0x0f1c8b0a),L_(0x00000171), LL(0xb4ebbd85,0x2786a497),LL(0x2343b68c,0xbc5660f7),LL(0x3871cff1,0xa03e99a5),LL(0x32a3116c,0x91a2e2b2),LL(0x39a66a33,0xf1e21170),LL(0xb4a691a2,0x0b59581c),LL(0x760bf647,0xbf35d6e4),LL(0xccdb4699,0x35e4ec1b),L_(0x00000125), + LL(0x8ea4e81f,0x6f0c46da),LL(0x6349f724,0xa1f6221f),LL(0x3d1cb710,0x801a6d7d),LL(0x9a8daaa8,0xdff7216c),LL(0xaabb78f1,0x0d054787),LL(0x6a1b8dee,0x9342cf54),LL(0x7426ffaf,0x8839548a),LL(0x7e189575,0xc54c3c21),L_(0x0000007a), LL(0x645473ec,0xf45138ac),LL(0xcb977455,0x51f3e82d),LL(0x23de028c,0xcb2096e6),LL(0x1236013a,0xd60fa53f),LL(0x790031f5,0x590da1dc),LL(0x41383462,0xd75ce15d),LL(0xaac7003b,0x5c3cf3c2),LL(0xe97d1507,0x738f1700),L_(0x00000179), + LL(0x552d3a3e,0xf4eca93b),LL(0xae1c95c5,0xdc45bd28),LL(0x0c12e32d,0x5dd7eb7c),LL(0x50ac538b,0x692eb87e),LL(0xc65147a8,0xa055973b),LL(0x8ff87281,0x23507ab9),LL(0x63636873,0x1c85fb4c),LL(0x794d2027,0x597db57e),L_(0x0000006f), LL(0xfe67d871,0x0c89c0f4),LL(0x7e7fa083,0x3b0ba9a3),LL(0x25bac099,0x1416b2e6),LL(0xcbc2cc9b,0xbbcf2943),LL(0xc5a1f7b3,0x354fa11e),LL(0xb195e363,0x61adb945),LL(0xcce31308,0xfde526a9),LL(0x5e8055b0,0x2d4a42f0),L_(0x00000100), + LL(0x7e3f21a0,0x83926d83),LL(0x58ff928b,0x46424f50),LL(0x50a21088,0x656540b1),LL(0xd69839e2,0x2172157b),LL(0xc836bb43,0x34535e3b),LL(0x1f818f5d,0x61ec6b27),LL(0xf4cd40aa,0x8714bd57),LL(0xfdb8302a,0x633209c8),L_(0x000000d1), LL(0x0a6b22f2,0x8b7033af),LL(0xdbf4f3b5,0x213a07cd),LL(0xd71e271e,0xfa9434d1),LL(0xc069f3af,0xd5d23e3a),LL(0xc4ccd448,0xe785990c),LL(0xdd215a3d,0x500536e9),LL(0x43168909,0xe45a1f48),LL(0x9f92d8e2,0x747468d3),L_(0x000000aa), +}, +/* digit=30 base_pwr=2^150 */ +{ + LL(0xf661dbbf,0x43121ba6),LL(0x2325e379,0x176edfca),LL(0xf0cef68c,0xa3861e28),LL(0x617ac6ed,0xa77e7f84),LL(0x57535e8c,0xd31f498d),LL(0xf36e23d1,0x546d78b2),LL(0x2c3f8810,0xcfc7d55e),LL(0x156a1cb9,0xd5aca891),L_(0x0000001a), LL(0x974ce76b,0x894a4c0a),LL(0xc178af73,0xe4d65f8f),LL(0x5d4f42d7,0xf71cb940),LL(0xf73dac29,0x1d35c689),LL(0x32814192,0xe3cb66f4),LL(0x753255de,0xaf9effca),LL(0xa9814253,0xd34e3d9e),LL(0x22e23b71,0x8168c420),L_(0x000001f1), + LL(0xaa0bda51,0x62c59939),LL(0x9d4f255c,0x1e39fae8),LL(0x74c49bbe,0x09372aef),LL(0x180fc9e6,0xde724860),LL(0x163da12a,0xfa823f50),LL(0xa72a28de,0x965a30e8),LL(0x3c600eca,0x905cf108),LL(0x9f8b9687,0xd2d094af),L_(0x000000c3), LL(0x26afd7d7,0x13810cfd),LL(0x986aa03b,0x37d1ddbf),LL(0xeede05c2,0x2715d035),LL(0xb7ae0b88,0x95ef9e71),LL(0x08124878,0xe5042346),LL(0x9f87f170,0x3054f163),LL(0xebc09360,0xce2e674e),LL(0x593b42f2,0x6d4f4654),L_(0x000001b2), + LL(0x673811ec,0x60b858de),LL(0xeb464fae,0x677b9369),LL(0xd5f16b47,0x26383f92),LL(0xc119870e,0x3f8c6fe9),LL(0x5da1cbb2,0xf7124d37),LL(0xf6c7c1d8,0xdb2b9c75),LL(0x96be948d,0x93746dbd),LL(0x9988eb57,0x760b9ec6),L_(0x00000024), LL(0x03e8f45c,0x3d8c21e4),LL(0x9b0b40cd,0xc1186513),LL(0xb44deee2,0xf970a928),LL(0x2d95e66b,0xa6ac8009),LL(0x8387cee5,0xbddad6f7),LL(0xfec87180,0x0d3ded17),LL(0x2404e11c,0x41ea3e64),LL(0x725101e4,0x4f7761e0),L_(0x000001b3), + LL(0xc6a2f3a1,0xa09feeb1),LL(0x180b5e19,0xc54628c6),LL(0xe8c61be2,0x773cefba),LL(0x054eeedc,0x90648d31),LL(0x0005e411,0x36489351),LL(0x44b74925,0x54e90646),LL(0x573a22ca,0xd626639e),LL(0xa6074dac,0x850593b5),L_(0x00000197), LL(0x9e2e1f28,0x11161ac9),LL(0x4f328fba,0xaaf012b0),LL(0xb74a91c1,0xadb60a6c),LL(0x0cf3c48d,0x1b818269),LL(0xf7c4e07d,0x9eb0dacb),LL(0x2e6fbed1,0xaba09048),LL(0x9ea1ef81,0xaab8c6fb),LL(0x4b567809,0x87318d6d),L_(0x00000196), + LL(0xdee1b8d1,0xeadd7be3),LL(0x42ed487b,0xcb4bb355),LL(0x508d338c,0x1d927c01),LL(0x671a9478,0xd1e3ea8e),LL(0x6482584e,0x83bdc72c),LL(0xb63d17da,0xe52363b8),LL(0x49266941,0x4b78813a),LL(0xcb9e3414,0x5ec6b5cf),L_(0x00000092), LL(0x1f691526,0xb7f97367),LL(0x8a234b55,0xb87b73bd),LL(0x107f953d,0x2944bffc),LL(0x7c0ce6fb,0x6166fb64),LL(0xe784fca9,0x0a71a69e),LL(0x864d9dbf,0xa770d1de),LL(0x1d767a82,0x641a01bc),LL(0xcb0ce972,0xe39d1b9b),L_(0x000001fd), + LL(0xc878d60b,0xeb346a1c),LL(0x6b8a06cb,0x38f8292a),LL(0x28e10a9b,0xa02441ad),LL(0x110ae3e5,0x374d8f2e),LL(0x9df680d7,0x622d31b8),LL(0x0be1994b,0x98b8d29a),LL(0x35da2573,0xcf273b8e),LL(0x5a38591a,0x797f3e99),L_(0x00000156), LL(0x49364b7a,0xd85cd33a),LL(0x18db5402,0x71a1b4e6),LL(0x7ccbb0bd,0xda26853d),LL(0xc76e0476,0x1360631f),LL(0x888e44f3,0xf6b0ad63),LL(0x2c3af0f8,0xbec71f59),LL(0xdbf01e8d,0x723b0fd6),LL(0x92661703,0x31867e0b),L_(0x00000137), + LL(0x28ddccde,0x38aeac57),LL(0x4e993e85,0x0db5dd87),LL(0x38abc090,0xb465add4),LL(0x0c1739c3,0x43d0e74a),LL(0x70bd3e21,0x0b277d58),LL(0xac3af0a3,0x7b2c5422),LL(0x770a41ce,0x08580ab5),LL(0x9864e070,0xa3af8219),L_(0x0000002c), LL(0xe88e2a8f,0x6783af82),LL(0x37cfc10e,0x6261464c),LL(0x8e8c7510,0xcc9c836e),LL(0x503598b9,0x560d6425),LL(0xcf7c6100,0x4d90b834),LL(0xa8db43b7,0x2444a629),LL(0xd2cb3f5e,0xcabe2a81),LL(0x64aa2ef3,0xfca35c68),L_(0x0000013b), + LL(0x8b6de757,0xa9eb572b),LL(0xe5a924f4,0x650813f9),LL(0x4cddfbbc,0x0f808102),LL(0x750529ae,0x8dbdc23e),LL(0xfc407a67,0x3db36c6d),LL(0x549e5c64,0xadee9ab1),LL(0x55d46bd9,0xacadd1f8),LL(0xf68182d8,0xdd3bbaed),L_(0x00000171), LL(0x66e2fb66,0x24a72828),LL(0xe75f104b,0x50b3c877),LL(0xe38bb301,0x2f8590fa),LL(0xb7b5535d,0xf87c6208),LL(0xe1b50eae,0x41ba355d),LL(0x3f0d0c45,0x8bfe9602),LL(0xdc159699,0x3cdcf2ea),LL(0xab8c033f,0xe7e25f41),L_(0x000001dd), + LL(0x29dba96e,0xc3f8209e),LL(0xb1d945a2,0x08e3fd6a),LL(0x6511a3da,0x9263ef8c),LL(0x2562d483,0xd579038d),LL(0xc4c88945,0x5094d203),LL(0x75e4003b,0x54ec1258),LL(0xbe6102b0,0xe7874a8c),LL(0x8d34a4cc,0x27868839),L_(0x000000ea), LL(0x018d87b3,0x19a05999),LL(0xaacd7c73,0xd3cb884f),LL(0xe794b313,0x5a8d6b1c),LL(0x598893b1,0x47ab4f51),LL(0x7e862cb6,0xcd145d37),LL(0x58ebff95,0xa0ddf0aa),LL(0xcb716afe,0xa0791796),LL(0xc7f724f2,0xf2c9729e),L_(0x000000ee), + LL(0x5cadd4b7,0x080a8ec1),LL(0xb97d34b1,0x7709d928),LL(0x24150cc5,0xfda92711),LL(0x9f76ea4a,0xa5dd93a6),LL(0x4e4f7b83,0x826a2138),LL(0xb1f097c8,0xdb8dc9be),LL(0x877e5a70,0xf1a434cb),LL(0xdd51585b,0x86a8a774),L_(0x000000d3), LL(0x7f07e424,0x166eed95),LL(0x061bf4f5,0x469e3126),LL(0x06b67307,0xacfcc07d),LL(0x87971f8e,0x96d964d3),LL(0xe5aebd3f,0xa4d18cb2),LL(0x5d286291,0x3fbd829f),LL(0x7560bbae,0x277863a6),LL(0x6e83e561,0x690089b3),L_(0x000001ba), + LL(0x6a2ec0c8,0x6902a948),LL(0x82ea938a,0x1db5acbd),LL(0x9ed742a7,0x26e981b2),LL(0x64d1ad56,0x075f4b10),LL(0x30adf93f,0xfd5008eb),LL(0xdc51091f,0x7f4f1467),LL(0x907912b5,0x0fb17ba1),LL(0x567270fd,0xa348b34f),L_(0x00000094), LL(0xbf940606,0x18337830),LL(0xabbf9925,0x7019fd78),LL(0xbb5e1175,0xc937b351),LL(0x1359c463,0xd19eff42),LL(0xfe68dfe8,0xa8892734),LL(0xed1005b7,0x7cc639ba),LL(0x780e1feb,0xb6ff755e),LL(0x1f0082fa,0x16899eda),L_(0x000001ec), + LL(0x191bfbe8,0x87367459),LL(0x58859da0,0xd000c521),LL(0x4373d9cc,0x560dbafd),LL(0xfeee235e,0x8d303a3e),LL(0x1fe980f9,0x2a6082ad),LL(0xb5244f01,0x567ed43e),LL(0x8306748e,0xa7eddca6),LL(0x4e531e38,0xec76ef95),L_(0x0000006a), LL(0x100101b4,0x4e959563),LL(0x7ab1df8f,0xbcb6c605),LL(0x0218cd6f,0x3a152b14),LL(0x217b7b09,0x9b32670a),LL(0x7924c99c,0x8550cfd6),LL(0xf9af0b38,0xda396f8c),LL(0x27557bfe,0x01351543),LL(0xf74a0d9f,0xdcef098c),L_(0x000000cb), + LL(0xafc3d641,0x7e899074),LL(0xeac59b4a,0xab6f7e5f),LL(0x9036a3e5,0xd5685de7),LL(0x32b71856,0x6c3ebc40),LL(0xbe82a80b,0x46fa8ac4),LL(0x8d567d33,0x8f1ba3f5),LL(0xe3d61024,0x9622947c),LL(0x175ff060,0xe735b06d),L_(0x0000015b), LL(0x79460a8f,0x28338621),LL(0xa409e48e,0x6cee22e8),LL(0x04f98fee,0x448a258d),LL(0xb3e86ccf,0xedd8a07d),LL(0x94212741,0xe7c10493),LL(0x0eae65b6,0x72b816f3),LL(0x3c05e156,0xaf3b8cf2),LL(0xbeed59dd,0xd6c91b11),L_(0x0000017d), + LL(0x09ba81aa,0x3d989afa),LL(0xf91b3d7c,0x24d1650d),LL(0xff5c31f0,0xdd5b9bae),LL(0x20976038,0xf21ca860),LL(0x119240c2,0xcea2f4f0),LL(0x317b48b0,0x894a28c8),LL(0x18cdb521,0x70a13f92),LL(0x613d3aff,0xe3c74422),L_(0x00000003), LL(0xe8dd6883,0x18e6d135),LL(0xeb2f0e32,0x2b49078e),LL(0xf0a3dbd6,0x80094f8b),LL(0xad97e7ba,0x7ec56e28),LL(0x9e31d818,0x1a28c019),LL(0x2e151983,0x924258ad),LL(0x40e237bd,0x4e48eb49),LL(0x7b03fcb6,0x865da13c),L_(0x00000181), + LL(0x023dd329,0x3dc56c97),LL(0x2622fb40,0x2ec06f52),LL(0xca023e51,0x70809db6),LL(0x4a297188,0x1e390795),LL(0x4a3bacea,0xc30dbed7),LL(0x0824bcd6,0x3a9a9a8c),LL(0x3e051eda,0x98211435),LL(0xbf2fd10b,0xec587a65),L_(0x0000000f), LL(0x6a685349,0x272a03cd),LL(0xecc543f9,0x54d18fe3),LL(0xdf8cb8ba,0x9b7c5d19),LL(0x5f3f336d,0x9e4ff288),LL(0x5e647a61,0x2dc0aea7),LL(0xbdca4466,0x24b8d191),LL(0x33cd397e,0x4bd62cf4),LL(0xf66b542f,0x572d2230),L_(0x000000a6), + LL(0x6acd9335,0xcbf85583),LL(0x52fb515f,0x7808f963),LL(0x374fc6d2,0x5865c5cb),LL(0xebbbb50f,0xa4d0c81e),LL(0x47f3a5b9,0x29ed702e),LL(0x97b4bc63,0x64f0fada),LL(0xf73be9b2,0xe65e3bbc),LL(0x42f9f14d,0x40320dd4),L_(0x000001bd), LL(0x4729cdbc,0x47ce590b),LL(0xf363cdb2,0x5be836dd),LL(0x6a8da968,0x5fd32e4b),LL(0x49bdb981,0x076e41c0),LL(0x8d8f7528,0x097db4cb),LL(0x7fc1d50b,0xf829470b),LL(0x75b1cc67,0xd2b6caef),LL(0xd55324b1,0x829ebed3),L_(0x000001ac), +}, +/* digit=31 base_pwr=2^155 */ +{ + LL(0xf69ff50f,0xfc1e2456),LL(0x02b84001,0x1c124be7),LL(0xa6c9b545,0x2857f671),LL(0x07337c72,0xb0a89f0e),LL(0x7d3661d0,0xaf022308),LL(0x61f17db2,0xd9b173b2),LL(0xd0457b51,0xf8c65404),LL(0xefc1cd30,0x8f61986e),L_(0x00000196), LL(0xfffafb34,0xc5bd6770),LL(0xfc7a7db4,0x0de59815),LL(0x342e8ca8,0x843b5602),LL(0x0e1c9e4f,0x3bfe9122),LL(0x8b0b7c5b,0xa1e2826c),LL(0xe442b313,0x88ce465c),LL(0xf2ef9e99,0x77217ce5),LL(0xa10ff590,0x672e5682),L_(0x000001f7), + LL(0x9441390c,0x054f8022),LL(0x7143ab58,0x9b74e159),LL(0x8a901ba0,0x116652a4),LL(0x9b4f3635,0x0afb314b),LL(0x45e2ee30,0xd4622886),LL(0x4d2f79f7,0xb66e6167),LL(0x298ff3c1,0x2505aad2),LL(0x27d64009,0xc36ddbd3),L_(0x00000066), LL(0x0eb20dd5,0x4b51c2ba),LL(0x74c9cdac,0xbf1d3648),LL(0x10d4063e,0x6b726013),LL(0xc8c6fbaa,0xf8b94ac6),LL(0x6ce6639f,0x91f488ec),LL(0xf454066c,0x24c600b8),LL(0xf37706e5,0x1cff656e),LL(0x434286c2,0xe127b8a0),L_(0x00000087), + LL(0x35d5b009,0x7857cb4d),LL(0x2bf04d1e,0xc85eb560),LL(0xaf5c9697,0x3e426a2c),LL(0x140d9785,0xe234a765),LL(0xfae3a667,0x6a198191),LL(0x6a2fce6c,0x3a779c8f),LL(0x217e7e57,0xb35dd0c7),LL(0xb20040f7,0xad54aae8),L_(0x000000c4), LL(0x36df1d99,0x46ee68e7),LL(0x884f54d7,0x2f9e3760),LL(0x70670755,0x1a8bd746),LL(0xd3fc19b5,0xc34c78ba),LL(0xaf9a102e,0x9b57be0c),LL(0xfe21514a,0x9e7c2e6d),LL(0xde90d865,0xd44207f4),LL(0x19f36d3f,0x344544a7),L_(0x00000123), + LL(0xd877e284,0x37f60445),LL(0xca4191dc,0x69f0b4df),LL(0x358c7759,0x12aaa285),LL(0x72cf55e1,0x7f71ae31),LL(0x0ffea4f4,0xfc352eb3),LL(0x5b8d412a,0xc7ffc3d9),LL(0xabdbf74b,0x239ccbac),LL(0xd4b6acd1,0xfa89f30d),L_(0x00000163), LL(0xa6870d63,0xd1598751),LL(0xc6b0d1f9,0xc925f0b6),LL(0xa890fd44,0x106c309b),LL(0xcedd20fe,0xc46673f1),LL(0x2408588d,0xebfbcf6f),LL(0xb54153cb,0xa52fed53),LL(0x7b4aaced,0x672bbf3f),LL(0x84a22a21,0xf033a8d4),L_(0x0000004d), + LL(0x2197649c,0xa6098bec),LL(0x132b7114,0xe879a5ea),LL(0xbf33520a,0x5ec11946),LL(0xf7eb2f05,0x76724ae5),LL(0x8b00135e,0xa281ab75),LL(0xe322da16,0x75ecccf9),LL(0xf00478b6,0x5f741662),LL(0x77d420d9,0x27da38cf),L_(0x000001c3), LL(0x8deebf19,0x38cf892d),LL(0xdd1ae54e,0x4a486822),LL(0x8e9572bf,0x15d5deb4),LL(0x83965350,0xa31f170a),LL(0x098efd39,0x225bfe44),LL(0x3effcffa,0xd17f63bf),LL(0xf8e3659d,0x72dce9ce),LL(0x561fceba,0x7a4df1b0),L_(0x0000009f), + LL(0xaed633fc,0x573eaf11),LL(0xc05113c8,0x5f254d2b),LL(0x283764aa,0xee71c7fd),LL(0x70135776,0x88759ff7),LL(0x33df5ba9,0x84205188),LL(0xd52265da,0x809c0705),LL(0x912507fb,0x641067f4),LL(0x28d91a94,0x4a47efb6),L_(0x000000d6), LL(0xc5e6e2ac,0x0000ebfa),LL(0xa6c0565c,0xced796bd),LL(0x6c90c0d4,0x100a3283),LL(0xee187fc8,0x82bcb3d2),LL(0x8d7848e9,0x290e6b62),LL(0x4a59be08,0x5ab586db),LL(0xb9a00808,0xf4b07e2f),LL(0x210d8de2,0xc7559d56),L_(0x00000067), + LL(0x224b3264,0x7213c7fe),LL(0x43204c94,0x13a1a9d8),LL(0xf7f1cdee,0x68201c17),LL(0xd60991d9,0xbe9464cd),LL(0x4334ef4d,0x715fe2ea),LL(0x590e3478,0x7284a69e),LL(0xe07f24de,0x7c088851),LL(0x5ce9bed5,0xf517b62c),L_(0x000001c5), LL(0x3aeb8798,0x222d178c),LL(0xa01cdb6d,0xe0c1815b),LL(0x29424615,0xec65cc42),LL(0xfd885c8b,0xfd5df228),LL(0xd9564da3,0x9775d121),LL(0xb060eb3c,0xdc43087b),LL(0xbf975586,0x0c723af3),LL(0x941c0856,0x081262ab),L_(0x0000006c), + LL(0xbf0ac7a2,0x744e57f4),LL(0x8834334c,0x2edb448a),LL(0x4c1f9523,0xc8e4d56a),LL(0x85d4cde6,0x0bd23e3b),LL(0x83063d71,0x45b52f37),LL(0x14ca833d,0x2012d08a),LL(0xff85aaed,0x02ccbe55),LL(0x9fa9b95c,0x0a4b1e91),L_(0x000001fa), LL(0x04999b76,0xf355b09a),LL(0x9309a1f5,0x00d64b66),LL(0xb2bd55ad,0x57889605),LL(0x6b121bac,0x20d91b65),LL(0xed693b72,0x1faab888),LL(0x344453ea,0x45d07a30),LL(0x75e36d67,0xf7e7a52e),LL(0x86433618,0xdc4aaa79),L_(0x000000c8), + LL(0x9079f10e,0x7efb5f88),LL(0x2c050909,0x1cc662d3),LL(0x7e0de0a5,0x5ee0da97),LL(0xb01a8aa9,0x4922eaaa),LL(0xbf868cba,0x64bbc9e2),LL(0xba2129b9,0x0afac1c6),LL(0x38f86242,0xc8be3270),LL(0x7520a9f0,0x711b9939),L_(0x00000014), LL(0x0cf7a18c,0x5d48ee16),LL(0xc85d4499,0x04528b19),LL(0x2452bab5,0x65b2b9c3),LL(0xbcfb2531,0xd43a545f),LL(0xc03362e0,0x07cc670e),LL(0x5d9aafa1,0x58f98004),LL(0xd816f41a,0x324a8340),LL(0x352c0783,0x948019c3),L_(0x00000060), + LL(0x4b2484fc,0xfc2dc91c),LL(0x3b5be485,0xd7e9f840),LL(0x42217cb5,0x8585ec85),LL(0x3deede9d,0xa1f0053f),LL(0x48c56ddd,0x845902ce),LL(0xb2e99028,0xdbb111fe),LL(0x8f6659bc,0xf1537c2b),LL(0x89960f5b,0x2b6c075d),L_(0x000001de), LL(0x88a9e85b,0x39c6986f),LL(0x69af11de,0x0fa555ee),LL(0x0c555b9d,0x411f3b27),LL(0x62266b30,0x0b0e864c),LL(0x784a1194,0x112da824),LL(0xb7ec5b26,0xc56950bf),LL(0x8a57ba0f,0x3866d81f),LL(0x72e0aa00,0x33d123b7),L_(0x0000012f), + LL(0xf2f274e7,0x9f7aad66),LL(0xd84871e2,0x040c2554),LL(0x2a4885e0,0x8ddb8ec0),LL(0xbfd317bb,0x2a407fab),LL(0xaa27b70d,0x17f03cf3),LL(0x1bed7718,0x8c3de6bc),LL(0x34f5d378,0x0e550353),LL(0xfef609bb,0xf9faf54a),L_(0x000000e9), LL(0xc5275edd,0xd325a149),LL(0x33824d71,0x14d92534),LL(0xba4131f2,0xf74f4dc1),LL(0x81fdb0c7,0xd354ab8c),LL(0xc33be6cd,0x96e68610),LL(0x7d362d2c,0x967ca304),LL(0xad3a9c9c,0x90a06f8b),LL(0xaf6da5b3,0x05921c7a),L_(0x00000012), + LL(0xc1710f55,0x67ff0e8d),LL(0x7ef6718f,0x601481b6),LL(0xb39b462d,0x57d09ffa),LL(0xad90ba10,0xf83bbbb5),LL(0x918d94f2,0xed4c7a16),LL(0x2bee8d2e,0x9ddb61a1),LL(0xdadd0291,0xde96ab74),LL(0x2e5753e9,0xf3c27390),L_(0x000001bb), LL(0xa3926dcd,0x5af3e375),LL(0x827a6eab,0xeb250dce),LL(0x08623cdc,0x52408bd1),LL(0x9a7d0e9c,0x236fdad4),LL(0xf66e3019,0x55ed033a),LL(0x55cf40ab,0x67077bc7),LL(0x33b49be6,0x3d6972e6),LL(0x34396ea4,0xc069b953),L_(0x000000fb), + LL(0xa4c22061,0x30374f9c),LL(0x83feabc1,0x5043f74b),LL(0xf24a71b6,0x0f58d08b),LL(0x5ac8cda0,0xe7084b9c),LL(0x20120c5a,0xc241d2a2),LL(0xa700c2dd,0x246b4a2f),LL(0xe50e9154,0xe1b127ec),LL(0x240be13a,0x7313d8aa),L_(0x00000117), LL(0x0178da07,0x4ac8c26e),LL(0xa3abe616,0xeaa2008e),LL(0x58d98d73,0x16a0bdbf),LL(0xf5f03b56,0x0dd5224a),LL(0xafd2d956,0x656cc265),LL(0xe30a653c,0x896d53dd),LL(0xc8ac8028,0x8038e832),LL(0xe07a2ee0,0x1df13c5a),L_(0x00000179), + LL(0xf7671c0c,0xdb94fffe),LL(0x0361d956,0xbca8fdc3),LL(0x860aa7a6,0xff4ebfa5),LL(0xca2b724b,0xd506fbfe),LL(0xe572f34f,0x2e88a7d1),LL(0x430c48ff,0x74822e19),LL(0xeb20b178,0x623c0129),LL(0x07cc6f01,0x013ca3db),L_(0x00000040), LL(0xe9244f5d,0x8954a885),LL(0x2bf3bfbd,0xf3969954),LL(0xa7e331fd,0x80dc93a6),LL(0x16b29c51,0xe85d8098),LL(0xfad960c8,0x7931b35d),LL(0x74ab3a3a,0x2e570f29),LL(0xf4422349,0x54904daa),LL(0x5e1f7007,0x16920cf2),L_(0x0000014c), + LL(0x11b4e5eb,0x5620232e),LL(0x8a138aa8,0x46f706eb),LL(0xa03d24cf,0x6e11ca59),LL(0x7337f5d0,0xad37149e),LL(0xfa3336f2,0xb68bf40b),LL(0xcb9ee77e,0xa7c9f76e),LL(0x8719bf3b,0xf9bd4330),LL(0x45e4e081,0xe7e07596),L_(0x00000148), LL(0x4d1bc133,0x8d32cc0d),LL(0x1bd3eff3,0xa11a2038),LL(0xde1eb1a9,0xa2e7f299),LL(0xb382b9cb,0xac50dfdc),LL(0x62fa8c40,0xe2272381),LL(0xa696bb54,0xf025e3e7),LL(0x68bf08ed,0x608f07d7),LL(0x91eb5365,0xd0e1062b),L_(0x00000157), + LL(0xea56e1b7,0x5f8128c4),LL(0x028409b6,0xe5e0d92d),LL(0x8ed0e1c4,0xf3b74f68),LL(0xc55f66d6,0x35d3f9f9),LL(0xdeb2ab80,0x3bde4296),LL(0xa7cb6b64,0x25e29f7e),LL(0x9b9d057f,0x087f5f23),LL(0x17e3fac2,0x1dc271af),L_(0x0000015d), LL(0xd463cbb3,0xd926fd3e),LL(0x014b12b6,0xab9ee679),LL(0x3a1bcb9a,0x1f47e609),LL(0x17170593,0xf44f73dc),LL(0x3b0a4387,0xd2a12e51),LL(0x3ce5c7cd,0x473ec3b7),LL(0x7f341e3a,0x6aef1796),LL(0x09a474c8,0x8fa42a38),L_(0x00000047), +}, +/* digit=32 base_pwr=2^160 */ +{ + LL(0xc2dc9808,0x7f1bc655),LL(0xb688a237,0x67de245f),LL(0xd7a61e34,0x30b260cd),LL(0x9aaf28a3,0x9aeae5d8),LL(0xd4e07803,0x53d349d8),LL(0xd7aea422,0x38cabcfe),LL(0x3728bd24,0x25a9960f),LL(0x58af5683,0x16a09af6),L_(0x0000003e), LL(0x816e52ab,0x1bc21ee4),LL(0x31a5819d,0x26613d4c),LL(0x2a5969b3,0x1a8c1407),LL(0xabfa75ee,0xd357015b),LL(0x7c563bc4,0xd2086ecb),LL(0xa4a80425,0x9b8fafb1),LL(0xc2661a2c,0x547ef737),LL(0xe7afb2d6,0xac816c20),L_(0x0000002f), + LL(0x65726f32,0xb7a9942b),LL(0xa33e2204,0x4a26b80f),LL(0xbbf82a56,0x73c6f40a),LL(0x970dfcc9,0xf9548526),LL(0xf1c38e96,0xd2bbae55),LL(0x2ecb19ab,0x1edd71d6),LL(0x6d97496c,0x2e20adf2),LL(0x17e1cf32,0x714bc0d9),L_(0x00000130), LL(0x76aaf44a,0xb67e29ba),LL(0x1031c67b,0x3d1213c2),LL(0xe37fdfde,0xb4f3b345),LL(0xb46f2bbe,0xef5d5bda),LL(0x53442227,0xbdace910),LL(0x75a65c11,0x0e12dac1),LL(0x99010c36,0x58cdb1cf),LL(0x06f25026,0x3222c871),L_(0x00000187), + LL(0xee441882,0xdeb1c61c),LL(0xf8ff5eb8,0x6080b71c),LL(0x7b2ccc29,0x214b75b5),LL(0xffb3c6aa,0xe80f53b9),LL(0x90a50e70,0xfeb156be),LL(0x0211fd2f,0xa94620e8),LL(0x15422e55,0x085db41e),LL(0x20305265,0x492c355d),L_(0x00000093), LL(0x139e1933,0x50841313),LL(0x976e986b,0xb6d55898),LL(0x36a0866b,0xa443f795),LL(0xe06bc0b2,0x63ba00b8),LL(0x734e5428,0xdd7a73a3),LL(0x213440e4,0xb2efa382),LL(0xb0905af8,0xe95312ec),LL(0xb084f884,0x3260e8ef),L_(0x000001cc), + LL(0x3f172c5e,0xd8ddaafa),LL(0xeb6e8784,0x785f2ae4),LL(0xf77d65ef,0x4e5db162),LL(0xdec5c58d,0x2375c785),LL(0x4a30bffa,0xc92e0f7f),LL(0x0c920bb7,0x294b17a0),LL(0x26f93d72,0x0a9107e0),LL(0xce9dc095,0xcde64911),L_(0x0000018d), LL(0x2b841c67,0x72452329),LL(0xb0490079,0x0e7ddb4c),LL(0x55646515,0xc3ad47f9),LL(0x4b2a0877,0xd8708db4),LL(0xa4c3de4b,0xb4a9131f),LL(0x938e9d24,0x85e650ae),LL(0x80176c45,0x60bb2e49),LL(0x0248559a,0xe3e93182),L_(0x000000cd), + LL(0x28cef71c,0x2e3e2609),LL(0x5311578b,0xb15a4e84),LL(0x66031c77,0xc30c76cc),LL(0xf2c06ffc,0xc352a0e2),LL(0xea471db8,0x9a687b94),LL(0x2e1e184b,0xb1979864),LL(0x08e1a1c9,0x7d1d84cf),LL(0xa36c823a,0x25036462),L_(0x00000135), LL(0x47b77555,0x1b0d298e),LL(0xe7833c92,0x071e1319),LL(0xe5e5ae43,0xbf6e6f4c),LL(0x48ff7cbd,0x44726013),LL(0xec042f31,0x861a992a),LL(0x820461f1,0x0e5f80d4),LL(0x5b728532,0x588846b7),LL(0x4edf14c0,0xdbf695ef),L_(0x0000003f), + LL(0x0f51608c,0x41c6cf4e),LL(0x15b1b366,0x263e7b75),LL(0x6eb6d459,0x041a5063),LL(0x53679a56,0x6ef1d0df),LL(0x9b4abcaa,0xb47a0301),LL(0xae975077,0xd2d427ef),LL(0x62f30c49,0x5a3dfa91),LL(0xc801e565,0xee86df8b),L_(0x00000124), LL(0x68202783,0x26d59f48),LL(0x4e17501a,0x3895e666),LL(0x202e3866,0xaa8031f4),LL(0xd2af7613,0x8ddf2869),LL(0xa21cc1e5,0xe13d84ff),LL(0x5da3159e,0x8f6eb59a),LL(0xb87bbc9c,0xdc5df9b7),LL(0xb8b6006c,0x68fc1771),L_(0x00000158), + LL(0x3e6aa5bb,0x9e7a21c0),LL(0xee3c40e2,0x9e430844),LL(0x91ca8307,0x420584b0),LL(0xfb05a033,0x5dc3546a),LL(0x515d7ef6,0xdfae44d0),LL(0x8e97acb0,0xad35608c),LL(0x1c181a0b,0x85a78e5d),LL(0xd8ba90d8,0xd453e7ca),L_(0x0000010d), LL(0x4b1cba50,0x3d89eff9),LL(0x1828d959,0xb8883419),LL(0x9cd1acbe,0xe7788137),LL(0xd9c16250,0xf51b1fc4),LL(0x2f4d66db,0xbf985d68),LL(0xe78a703c,0x98e4fae0),LL(0x8125e5c9,0x9fe12466),LL(0x7096d179,0xcfe7195c),L_(0x0000004d), + LL(0x96c267db,0x2de8af3a),LL(0xd7a0da68,0xfc2373c7),LL(0x5ae71058,0x00846c04),LL(0xb05a94e6,0x87910867),LL(0x49ec9a78,0x0df20f65),LL(0xaecf973f,0xd4a6c168),LL(0x30604ed3,0x0b50f6bc),LL(0x2722d421,0x1ef35f34),L_(0x000001d2), LL(0x3c89badc,0x32767a9d),LL(0x76ac95ea,0xdc1a4baa),LL(0x3eced60d,0x114219cd),LL(0x2d3cddf3,0x557cfa7d),LL(0x4c14e1ea,0xd40b6e23),LL(0x77a3c466,0x24ae1830),LL(0x9bfca752,0x8ee59e15),LL(0x0d62fa0c,0x91ffb4c3),L_(0x00000106), + LL(0x98c15e86,0xdbdbd0e0),LL(0x13fe0c31,0x3f1495d3),LL(0x611ba4e9,0xcaa1f174),LL(0xd93815b9,0xec434016),LL(0x3bdec28e,0xb9edffc7),LL(0x7d039312,0x995ffc03),LL(0x340b94bb,0xf4d0bdad),LL(0xd62628f0,0xd171e355),L_(0x000001b2), LL(0x874bb93e,0xd381244e),LL(0xb9a019ce,0x2710057d),LL(0xa746e7f7,0x5f04bc77),LL(0xc973f2cc,0x16b90cee),LL(0x45b7cdcc,0x3bf24131),LL(0xf860483e,0xa97598d3),LL(0xd873a041,0x5da07fc7),LL(0x13ee03df,0x1370307d),L_(0x000001cb), + LL(0x04785e61,0x6896551c),LL(0xc769142d,0x1f7de113),LL(0xdc5c38a3,0x6f6444e4),LL(0x619b0fe5,0x442a0f4e),LL(0xc1f930e3,0x0e3d13dc),LL(0xc2166fc3,0x16566439),LL(0x1264bd78,0x043b1c6c),LL(0x55bd407b,0x85bcb31d),L_(0x00000096), LL(0x51809e05,0x37cc449e),LL(0xcbcd15b1,0xc268f122),LL(0xde98d3eb,0xd1094f76),LL(0x2f691855,0x38e9385f),LL(0x940e99ca,0xebc0ca85),LL(0x7a41361e,0x633585a2),LL(0xe77d0dba,0xf4c9fedd),LL(0xffae9098,0x6c29f55e),L_(0x000001b5), + LL(0xa2f1a549,0xf13eb703),LL(0xc846b81d,0xcf6235f8),LL(0xba752969,0x28bf7176),LL(0xa83689cf,0x4f491b5c),LL(0xac203f35,0xa5c72127),LL(0x17a19c66,0xc5180b7d),LL(0xe3fefda7,0x2a895472),LL(0xbc0194d4,0x7cd212ca),L_(0x00000155), LL(0x854caa47,0x822cdfe0),LL(0x7cac3eb5,0x3c7db833),LL(0x13d80239,0x03909920),LL(0xd8e93f09,0xde83b6de),LL(0xb075d1a2,0x53e966c9),LL(0x372a1d5e,0x5b917dec),LL(0x60dd5294,0x8284dac3),LL(0xff014a15,0xac0f873e),L_(0x00000047), + LL(0x73b6bc0c,0xf1bfa63a),LL(0x10296f8e,0xabe3e152),LL(0x9dd0aaa4,0xd3d4285b),LL(0x0c15dad8,0x5f828ba3),LL(0x3be85ac5,0x343ddcee),LL(0x2204e02a,0x532735c7),LL(0x7ba86652,0x2a530b1b),LL(0x262994c6,0x13b8c7c1),L_(0x00000129), LL(0x391a6b29,0x05dfaa4f),LL(0x6b8878ef,0x3e5666ae),LL(0xb1b8a9fa,0xa4a12d5f),LL(0x9ace0b8d,0xc27561ea),LL(0x7b4c8164,0xef8504c5),LL(0xd8cc29d3,0x16570313),LL(0x483fc408,0x3b7b5ec3),LL(0x2f18c762,0x6d721583),L_(0x000001c3), + LL(0xb6ecbf65,0x8116e2fc),LL(0x5e8f5e16,0x59e26819),LL(0x5609fad9,0x8fafa607),LL(0x02dca647,0x7fb0c319),LL(0x1e28746a,0x62d45955),LL(0x6e8dafba,0x53e7625a),LL(0x83169dc2,0xe60b1042),LL(0xebf6fde7,0x8e9c9927),L_(0x00000153), LL(0x7b2d8bde,0x5592a1b4),LL(0x1e6bd0f2,0xcdd5271e),LL(0x566eb6fd,0x7f5033b8),LL(0x4ca0b581,0xd99ab0fb),LL(0xb6096f1e,0x3953fc59),LL(0xcf65a6f4,0x1d2ec4b4),LL(0x1920c542,0xd24e43ac),LL(0xbc37795d,0x3e1940bf),L_(0x00000100), + LL(0x154c2ad3,0x9ab5e4b3),LL(0xa83af5c6,0xaf86c5e5),LL(0x17feec16,0xa5cec56c),LL(0x98ec6557,0x84e83213),LL(0x0f7fdcf1,0x4c26d215),LL(0xffda8a76,0x453ea210),LL(0x9ecd3b2e,0x5f3f4d74),LL(0x45856be9,0x8864bfe9),L_(0x000000bc), LL(0xa68371b9,0x1b7d95b0),LL(0xe827845c,0xee539828),LL(0x8d12cb9b,0xe4618579),LL(0x7d751e1d,0x49a508f4),LL(0xf62e7726,0x71d8ff6b),LL(0x1aa5f1f4,0x1b002961),LL(0xb185989f,0xc7af8411),LL(0x436bb002,0x902fe574),L_(0x000000c6), + LL(0x83b10389,0x5487e28e),LL(0x6c59c4b1,0xdbe03ec9),LL(0x5812b87f,0x800f9a8d),LL(0xa69e4288,0x0042610f),LL(0xa98baf31,0xa41914ae),LL(0x04c78aca,0xad52d4dd),LL(0x200e6b24,0xe64f0db1),LL(0xef061a8b,0xb301c9f3),L_(0x00000118), LL(0x9064a4d4,0xf05bb7bd),LL(0xdb0e1035,0x95a8d7b1),LL(0xbed0afed,0x5aa18c8f),LL(0x1db27276,0xbed5ae9a),LL(0x21d6647d,0x87ff9181),LL(0xd1b9171c,0x25ddbbf6),LL(0x6afd3974,0x58651838),LL(0xd5394b3e,0x6640e3ca),L_(0x00000126), + LL(0x97acef3f,0xdda16fb0),LL(0x1fbdca04,0xe90de335),LL(0xff197a3e,0x7011f9ba),LL(0x10909fc0,0xbf835536),LL(0xa3d538e2,0xd3c214c4),LL(0xd1adfbd9,0x4b2db047),LL(0xa7800e16,0xe30b9e3f),LL(0x3ba0bb0c,0x566a5103),L_(0x000001b3), LL(0x3ccb2552,0xb20301de),LL(0xc8c0dcda,0xaf7c3af2),LL(0x06c79c8a,0x7eefe996),LL(0x38fb5284,0x7cb586d6),LL(0x59bf5673,0xec4f260f),LL(0x36f200b1,0x62ff887d),LL(0x39132913,0xe5ed3b69),LL(0xc40f0d7a,0x2b5086e3),L_(0x000001e0), +}, +/* digit=33 base_pwr=2^165 */ +{ + LL(0x9d8a516e,0x02a3fb8a),LL(0x5fedd472,0xecb3edff),LL(0x7fb9838f,0x762220d1),LL(0x8b9ac40f,0x23ad98dc),LL(0x59a8311e,0xfb615d6c),LL(0x86c784ed,0xe6c85dc4),LL(0xee5f8f84,0x6bbf81a7),LL(0x58d5bb86,0x91b5becc),L_(0x000000ac), LL(0xa1d41ef1,0x007acce1),LL(0x20e9778f,0xd8b8126b),LL(0x438944de,0x437a71b3),LL(0x4e76c73c,0x9a1b4b13),LL(0x14a56abd,0xb7385f9b),LL(0x29b4de8d,0x3115d582),LL(0x91b40784,0x15347258),LL(0xba8c32f8,0xf9f46c55),L_(0x000000ed), + LL(0x99943818,0x2684f683),LL(0x9d27b5d4,0x68a5f913),LL(0xa3ed9c84,0x9f9d03a1),LL(0xe699de7f,0xe3117424),LL(0x6ddd7e41,0x967769d8),LL(0xf6fd89cc,0x0e9e00b5),LL(0x4a6926ea,0x3d7b6393),LL(0x5b068a8b,0x5f0b111c),L_(0x000001a3), LL(0x297d21cf,0x9a2aecb4),LL(0xcf147f52,0x251f8677),LL(0xd2a35774,0xf0bbad3b),LL(0xbedc57bb,0xfe5b3790),LL(0xaa31f1db,0xb3cb7422),LL(0x01bb1e75,0x476bcd99),LL(0xb31cdbf6,0x8c278bd8),LL(0x6fb17125,0xd548282a),L_(0x00000150), + LL(0x19f12734,0x3d48fbed),LL(0xf69ad2bb,0x49bdd26f),LL(0x985b989b,0x61bfbf26),LL(0x451c21eb,0x35f12cad),LL(0xf237a30e,0x680a082d),LL(0x2751a3b3,0x88ebe4c9),LL(0xc7316941,0x0887a8fb),LL(0xa8bdfe94,0xd925bb03),L_(0x00000187), LL(0x356f89f4,0xde19c8b9),LL(0xeab9cd80,0x77afef27),LL(0xf941390d,0x16f538f8),LL(0x8c79f62a,0x9a2c1a2b),LL(0x84a907ee,0xb7aa5d96),LL(0x9877951e,0xfe7d75aa),LL(0x59fbafe1,0xc17b983b),LL(0xb437db42,0xab1072ba),L_(0x000001c1), + LL(0x3a57f7ec,0x4ce56b58),LL(0x1306d958,0xce15377e),LL(0x1e23a49b,0xbad5b26b),LL(0x2d98c317,0xae8b11f4),LL(0xdc523283,0xf50073f0),LL(0xe7af81dd,0xab516099),LL(0x519277c4,0x6a29299e),LL(0x8cb7cfdd,0xf9d5fc0d),L_(0x000000ae), LL(0xad29a85f,0x9213cb42),LL(0x37030b7e,0x364e5e4a),LL(0xf8a54d03,0x771a3941),LL(0xb7d507ec,0xd6f8ad50),LL(0xddb1def4,0xbd493bf4),LL(0xc65eeab3,0x716822a9),LL(0x7e2f6019,0x1d5d463b),LL(0x062fa75d,0x39044728),L_(0x00000094), + LL(0x6ebc6aeb,0x755f610d),LL(0x7e269fb2,0xbc7a68af),LL(0x9a7e6748,0x576c91d2),LL(0xaa653529,0x8b42e1e9),LL(0xe03c250b,0x9e921ac8),LL(0xf313cd04,0x500a0736),LL(0x48b57315,0xfbe580a3),LL(0xd15496dd,0xbe103270),L_(0x000000c6), LL(0xe43286f3,0x42537712),LL(0xfad38cc2,0x5ba8dd4a),LL(0xe8e53c49,0x940cf7d8),LL(0x88cb201e,0xe105c906),LL(0x0310db91,0x14eb5137),LL(0xbdf5c752,0x04b87caa),LL(0x73be9996,0x32ce177d),LL(0x545383f4,0xa2679329),L_(0x00000146), + LL(0x181fa26f,0xe91dfab4),LL(0x8f94d28e,0x500e4763),LL(0x031df707,0x2cdd284c),LL(0xfc76fd9e,0xb532df91),LL(0xdbbb6032,0x95140af2),LL(0x0796b18c,0x1a08045a),LL(0xf970af5c,0xb920694c),LL(0x325b81c7,0xb373cbe7),L_(0x00000087), LL(0x6128ccb5,0x68880de0),LL(0x2e3ad7bc,0xb9bdf74f),LL(0x769e9e60,0x43ac2084),LL(0x0eb7035f,0x71aa1b0a),LL(0x443fc7a8,0x9b9cb064),LL(0x8d6eb3bc,0xcda0b792),LL(0x35030dfe,0xcc362ccd),LL(0x29fff962,0xda9f8e29),L_(0x0000017c), + LL(0x01f40601,0x78cfd6f2),LL(0x993f944b,0x042c6de6),LL(0xe197b472,0x7bbfb051),LL(0xe877f763,0xa554df58),LL(0x82d5094c,0x5d801ed2),LL(0xd75061e2,0x89c183e2),LL(0x060481cf,0xe8a754c6),LL(0x43706037,0x94d020ef),L_(0x00000096), LL(0xc842ce7d,0x1ecea7b9),LL(0x0c3c295e,0xd56a995a),LL(0x2352f8cc,0x2d519fed),LL(0x7a9172aa,0x95d8bcc3),LL(0x546f4f90,0x6b8cea31),LL(0xaeee4bb3,0x3f188de2),LL(0xb3d9fe63,0x63e62bb8),LL(0x9f32b579,0x1d4076c1),L_(0x000000be), + LL(0xa42ee214,0xde6a66c4),LL(0x6c6c7d51,0xc1ce9444),LL(0xd2eee21b,0xeaac0d5d),LL(0x8f8f4a8d,0x5914a3e8),LL(0x755296fd,0x6c394520),LL(0xe647dd87,0x3798ebb4),LL(0x696a7a68,0xc9fd6484),LL(0x66ec9d8e,0x1c7ad1d8),L_(0x0000015a), LL(0xf4ca34e1,0x01e46446),LL(0x0fc27507,0x4431e6d9),LL(0x0b310d40,0x9766b761),LL(0xe5199614,0x04e26686),LL(0xae7e80f7,0x4f7efe74),LL(0x9829aa76,0xd9535c6d),LL(0x5702e183,0x755a23c2),LL(0x457bd92a,0x92bf30a0),L_(0x00000040), + LL(0xf84e8b92,0xa379a575),LL(0xfe2e00bb,0xf17caafe),LL(0xf713e2fe,0xfca28897),LL(0x56bf2a80,0x484598f6),LL(0x6ed19617,0xda495469),LL(0x3fe63788,0x9a48ed8c),LL(0x32c6923d,0xea4ef749),LL(0xdd905e15,0xc8a6856d),L_(0x00000125), LL(0xa7aa5cfa,0xcd183286),LL(0xb2b44bda,0xf345a8f6),LL(0xb4add52f,0xfc8c57af),LL(0x001d629f,0x9943972a),LL(0x1f5f64d0,0xee2dc970),LL(0xc7523ceb,0x078e50ed),LL(0xa1b8fb92,0x98c1c85f),LL(0x69d2866f,0x12275bf9),L_(0x00000129), + LL(0x225d5458,0xae1049c6),LL(0x1f4012ec,0xe45c8c16),LL(0x89c61650,0xe63f6f74),LL(0x97d5ef6b,0x13c990c2),LL(0x7718d93d,0x3b2af534),LL(0x388b9ecf,0xb4a19bd4),LL(0x53cfc179,0xc67cc8de),LL(0xea4e62f5,0xccf5963e),L_(0x00000185), LL(0xd67d72c3,0xa67e3213),LL(0xb4e77b7d,0x2c9d3a3a),LL(0x77d06f89,0x5e517015),LL(0x66b06c1c,0x79e0be47),LL(0x25f78836,0xacaba839),LL(0xf6cdc997,0x60da7988),LL(0x18ee069c,0x75dac3e5),LL(0x028ed009,0x3f7a195d),L_(0x0000014e), + LL(0x77706fdb,0xce2d42d6),LL(0x121b5db1,0xbfb3bb0d),LL(0xc38da042,0xcc50a951),LL(0x690091d1,0xe0527354),LL(0xad28eb90,0x6d30c1cf),LL(0x4621b3fa,0x72f783b8),LL(0xecc35c39,0x05168f18),LL(0x7e7054f1,0x744bd75d),L_(0x00000158), LL(0x437313ab,0xe9455bf8),LL(0xe4a7314e,0x131fad1a),LL(0xb8e1f53e,0xd17720a7),LL(0xbbc2ae3a,0x41e60518),LL(0xde92e4d6,0x53833db7),LL(0xd91c6976,0xa3c1ec67),LL(0x002fce6d,0x353e4fb8),LL(0xf35678d9,0x008548f1),L_(0x0000003d), + LL(0xe8b6d7a1,0xfb16d9c1),LL(0x2d6fbafd,0x4603abda),LL(0xc1c342fe,0xc3174b01),LL(0xf763e29a,0x6ef24fdc),LL(0x081c9b95,0x6ff0c881),LL(0xadc9659b,0xc6c4ce5a),LL(0xb4df1c4c,0x1e8123e9),LL(0x9a4d9154,0xaa2cc087),L_(0x00000047), LL(0x08513662,0x1126c683),LL(0x7efb0353,0xe6d3af24),LL(0x506609c7,0x31b758c2),LL(0x8f74142c,0x5c5f34e5),LL(0x0a3e4fdd,0x3f4b19a6),LL(0xe42c81b5,0xf8223898),LL(0x57c04ea3,0x21041b37),LL(0x44625a29,0x52ac9b84),L_(0x000000ea), + LL(0xafb45817,0x273a1b81),LL(0xe782c707,0x122cfd64),LL(0xf60341eb,0x55516c45),LL(0x19dc551b,0x646dbbd6),LL(0x33015a2f,0xc8d289cb),LL(0x1e2ea096,0x01dbc5f5),LL(0x04e60127,0xb40b7fdc),LL(0xaa434764,0xd6732b6e),L_(0x0000008d), LL(0x3cf9666a,0x4a435c55),LL(0xd7f50159,0xa122c995),LL(0xfe948450,0x52defdfc),LL(0xb5bd3afb,0x272ef1f4),LL(0x702b6fc2,0x86c9c7cf),LL(0x578c41a5,0xce279630),LL(0xafedf374,0x57fd35a9),LL(0xdd29b0d6,0x359e1bc5),L_(0x0000016b), + LL(0x8c313c96,0x81430d8f),LL(0xb3ef9728,0x10f97c7d),LL(0x57ddefb8,0x07066ad1),LL(0xca00506e,0x586f421c),LL(0xdaf65ab2,0x2bf4f170),LL(0x2f754fdd,0xf2415152),LL(0x88f0654f,0x18776438),LL(0x5393b3f2,0xad67f2ff),L_(0x00000150), LL(0x560341d5,0xd6250f20),LL(0xce99e680,0x4c66a708),LL(0xd56bd29f,0x440c3774),LL(0x8b248864,0x9f32acf5),LL(0xad54b8ec,0x99d6dad5),LL(0x45a78e51,0xedbfc6a1),LL(0xa505dcaa,0x28bf41f3),LL(0x8a48ad87,0xae2d610a),L_(0x000000c3), + LL(0x548d3329,0x8505c5bc),LL(0x76050884,0x4dfc2894),LL(0x6b686f2c,0x1e02d4a1),LL(0x9d97a4c0,0x2a447f5a),LL(0x5475b435,0xc828b6cc),LL(0xd2791aa0,0xe6e9d956),LL(0x3b328dfd,0x69ef6cbe),LL(0x988497f9,0xd58525b0),L_(0x000000b4), LL(0x628f0906,0x503dd445),LL(0xc52d4b69,0xdd2f1758),LL(0x438796f6,0x16280d32),LL(0xce2abc46,0x2fb22aa7),LL(0xbdfd0070,0xa5833469),LL(0xd5120c6e,0x80d303f0),LL(0x047308c9,0xd719acb8),LL(0x2a731dfc,0x1df4d436),L_(0x000001be), + LL(0xfb9cf085,0xff178cc9),LL(0x717cdc0a,0xcd1f6670),LL(0x8870fa8c,0xc4d58854),LL(0x99c44c6b,0xd627431a),LL(0xd7a4c31e,0x552f232d),LL(0x85daf88c,0x940140f0),LL(0x1d886818,0x9aa8211a),LL(0x16e4c1b0,0x985e3412),L_(0x00000004), LL(0x4b6cac59,0x3374279c),LL(0xc878a0ba,0x8991eda2),LL(0x84ea0b3f,0x32e3b4cf),LL(0x5e729a39,0xcc5f3102),LL(0xd47222c0,0xb4346c5b),LL(0xc5c9ba94,0x2995032e),LL(0x41a4babe,0x7ddb493f),LL(0x7b6e042b,0x862b224e),L_(0x000001cd), +}, +/* digit=34 base_pwr=2^170 */ +{ + LL(0xfe921c0f,0x079475b7),LL(0x410ea1a2,0xea0fd52e),LL(0x77d4bbcb,0x212e44af),LL(0x260a54b0,0xc66a7d1f),LL(0x4269af2e,0x4993bda8),LL(0xd04f3479,0x0b15e358),LL(0x0bdfadc5,0x1c67a4d3),LL(0x2250ea3d,0x4f317e91),L_(0x00000130), LL(0x2783de4f,0xfb63579e),LL(0xd5ac84fd,0x1abe0cba),LL(0x4b8a145c,0x84082001),LL(0x5d987c51,0xcfadaba8),LL(0xd9eba9aa,0xf5fccfd5),LL(0x82de291a,0x85e551a9),LL(0x372c4557,0x5e2bcee4),LL(0x9d89842d,0x7d7f549c),L_(0x000001cf), + LL(0xd17b0f39,0x5ae74e03),LL(0xb17b1a43,0xfbdcdaf3),LL(0xe5084910,0x63c90868),LL(0x9102285b,0xd8e63c01),LL(0xd5454d88,0x80d185fe),LL(0x50f99e23,0xf9e19dfe),LL(0xce8d3eba,0x728e09d7),LL(0x51277498,0x1de72015),L_(0x00000011), LL(0x777fda1b,0x7188feeb),LL(0xf597fdfd,0x6801e0f6),LL(0x29652f82,0x252e9d17),LL(0x58dec034,0xc6aa0c9d),LL(0x43cc68d0,0x6779b37e),LL(0x9c62a4e7,0x8d509f56),LL(0x0558ca70,0xd90c4133),LL(0x56b5657b,0x3de96569),L_(0x00000006), + LL(0x9ea07210,0xa60d5a9b),LL(0xe0116982,0xb1b1d6d6),LL(0x275ea7bc,0x7f932848),LL(0x607da14c,0x3af5ede1),LL(0x931400f1,0x03040c84),LL(0x3c889175,0x6c5973c9),LL(0xe7a0f614,0x293b333f),LL(0x5267024f,0xfd3993bc),L_(0x00000111), LL(0x81f51a89,0x99b337fa),LL(0xc1ab5f24,0x9bde774a),LL(0x5f059cb6,0x070a9fc0),LL(0xdd8da34f,0x8499182e),LL(0x7e9ea166,0x829389a7),LL(0xaf460691,0x5d843a97),LL(0x6edc8515,0xf45adbcb),LL(0x1fe7439d,0x998d92a0),L_(0x00000006), + LL(0x8e9008a5,0x241f1037),LL(0x92193f76,0x4e7ecf00),LL(0xf8905d70,0x28097f48),LL(0x6b4870ad,0x389acac8),LL(0x2f86eb6f,0xc3b9a313),LL(0xfe5a3ffb,0x9c6b9598),LL(0x14fb463a,0x2f429f10),LL(0x40890855,0x26f84e06),L_(0x000000ad), LL(0x530c94ae,0xd0399afa),LL(0xac70ca6d,0x7da5ef17),LL(0x854eb299,0xe2c80b49),LL(0x4afd62b1,0x77d7cf10),LL(0xf0b13757,0xb8dbecfe),LL(0xdfbd794d,0xf21b1b05),LL(0x47404dbf,0x96f1e68e),LL(0x28abdaf2,0x049bd809),L_(0x00000082), + LL(0xe27f7207,0xf97a3f2a),LL(0x787c8ad9,0xd2383086),LL(0x5dd8b1e6,0xfa851816),LL(0x13c110cb,0x0056cac9),LL(0xffc6bc3c,0xca2b8de4),LL(0x9e086187,0xd596553e),LL(0x4495145a,0xa323bbac),LL(0x799ae6f9,0x259cbe64),L_(0x000001f6), LL(0x53c95598,0x813fad43),LL(0x8941d128,0xee24f158),LL(0x81fe1387,0x7ca3f8b2),LL(0xec9a8f90,0xf4bc106e),LL(0x14a7e155,0x9b049dce),LL(0xacb41c88,0xf8e36863),LL(0x1985dcc1,0xc3075358),LL(0xa78ad338,0x5bf6eb17),L_(0x00000015), + LL(0x764f13da,0x1663cb94),LL(0xef025b32,0xdbea0296),LL(0xc7c10036,0x846d4ce1),LL(0xa4ebe01c,0xdcd331ba),LL(0x36bdd387,0x165ca514),LL(0xbf0ef724,0x1500e9b6),LL(0xf3d31456,0x2e001ed2),LL(0x0726f097,0x03718369),L_(0x00000050), LL(0x8451ea7f,0x6ab1aeac),LL(0x04486599,0x2c2e44e3),LL(0x2f540159,0x22a4d1df),LL(0x2edd9124,0x1b2aa406),LL(0x2d29f8fe,0x3d860387),LL(0xaed1f58f,0x2d78d5df),LL(0xbde871a5,0x65a5c46d),LL(0x6c0a2f54,0xde41cdf5),L_(0x00000189), + LL(0xbc829d92,0xb48b90b7),LL(0x41e85ef8,0x52345df2),LL(0xdc154eba,0x0472e3d9),LL(0xf8b4b2db,0x59486c6e),LL(0xd8c1f468,0x2a84dd3b),LL(0x1d593d50,0xdf33e197),LL(0x775fa504,0x0cadf964),LL(0xa92cc156,0x9927b85b),L_(0x00000083), LL(0x6a0487ca,0xfc3c5102),LL(0x03084458,0x9fa6a745),LL(0xcfa05014,0x51d7c6b7),LL(0x8f18942a,0x8c314152),LL(0x2258ef9c,0x789bb4fe),LL(0x5dcef195,0x1bcd3685),LL(0x2b822a02,0x4e898c2f),LL(0xbded6e36,0x9c74bcf9),L_(0x000001c6), + LL(0xaa163689,0x5ab01bff),LL(0x58bf7fb7,0xba4b1f3e),LL(0x50bc67e3,0xe8b59cab),LL(0xcea4689c,0x0d30cf8a),LL(0xaf1932a3,0xb3d1d8ed),LL(0x92c06e5c,0xfa7949e4),LL(0xb16d8b25,0x6f41db2d),LL(0x10d851b9,0xe9a23b06),L_(0x00000117), LL(0xb154a2b3,0x96d36fe2),LL(0x09d47ca1,0x766e2a61),LL(0xf1863dba,0xa8ef9263),LL(0xe92b5a5b,0xa5da163b),LL(0xcaad9918,0x520c8298),LL(0xf5e79189,0x0a27963a),LL(0x4ab05f91,0x20b8c3b8),LL(0xfd0103c4,0xced227ba),L_(0x000000cc), + LL(0xdd063123,0xa94faf37),LL(0x027545cc,0x751ad636),LL(0x5633398c,0xe2af69a4),LL(0xdb3a42f8,0x9980ca80),LL(0x3ef70c7f,0x2f9f4827),LL(0x03c8083d,0x4bd7694f),LL(0x2297619a,0xcba6106f),LL(0xb857e944,0x5d665130),L_(0x00000068), LL(0xed99704a,0x9b54a475),LL(0xd04dd3a0,0x45640d66),LL(0xe5bc76a8,0xb7f7aff6),LL(0xffcfd663,0x7c51db6e),LL(0x1b1b7d77,0x4ab9daa0),LL(0x312b124a,0xd1e4a043),LL(0xa2044f37,0x1147cf83),LL(0xc70b0257,0x3b2bed84),L_(0x000000bd), + LL(0x240110f2,0x7cc2f02a),LL(0xebf0be7a,0x2ac9b5c2),LL(0x24af14f6,0x9a211862),LL(0x782fa4b4,0x50410353),LL(0xf137e0ec,0x1b26ae96),LL(0x241ccb89,0x2cdf1d3c),LL(0x2b213449,0xac6249df),LL(0x063b93cd,0xae3ea116),L_(0x000001e7), LL(0x243cd7a1,0xf821c5c8),LL(0xd3358ef9,0x1dea9bcc),LL(0x06149e77,0x744a2dd7),LL(0x76a25a6f,0x510fe3ad),LL(0x126f991b,0x2bff6928),LL(0x9ff56b6f,0x26743e80),LL(0xb7342a0d,0x75a539a6),LL(0x1e395f1d,0xade39b8f),L_(0x00000039), + LL(0xc5b17046,0x6c3972f8),LL(0x30453d81,0xa45c6e8c),LL(0xf914e1ed,0x3fd2d983),LL(0x8df9d87a,0x465d7bda),LL(0x0b35e4f5,0x6fe2ce33),LL(0xd6b328dc,0x54ed3799),LL(0xfe08ef94,0x7a45c9eb),LL(0x18e443cd,0xf274d81a),L_(0x000000d6), LL(0x785e8d35,0x203318e2),LL(0x5c35d7f0,0x7b61a2b1),LL(0x652767b2,0x861e2602),LL(0x830e75ba,0x44dc9f10),LL(0x31d6ffa6,0x2ba2cf1f),LL(0x61cf1408,0x6ea5aa79),LL(0x8a5f9d9e,0xc5f5c401),LL(0x983aa3d8,0xa970331d),L_(0x000001d7), + LL(0xaae3e45f,0xebb6d11c),LL(0x08fdeeb8,0x0f274ee5),LL(0x562c576b,0xee620c83),LL(0x10e47bae,0x88c57185),LL(0x279b8105,0x919ff42e),LL(0x927c894a,0x7edf259b),LL(0x23100a00,0x169d16a2),LL(0x2acb9ccb,0xe6d07a5b),L_(0x0000011a), LL(0xfee8415a,0x94d74f07),LL(0xcacca4b5,0x4fb6e0e5),LL(0x08cc549b,0x08e788b7),LL(0xc62edae5,0xc2847dc0),LL(0x9b9ef886,0x64c8eee6),LL(0x9deee406,0xed24b57a),LL(0xa5474323,0xca2b9d44),LL(0x16f12261,0xf3a0da6f),L_(0x00000122), + LL(0x7ec4af3a,0x1798ff0a),LL(0x06583190,0xce45de42),LL(0xa85d801e,0x661ec8ee),LL(0xe6f87169,0x12391657),LL(0xa304ed8e,0x70c4e172),LL(0x8bed9dc1,0xd437a386),LL(0xa1738ce7,0xd75a62c8),LL(0xfe484c38,0x91a53279),L_(0x000000bc), LL(0x9f0ace8e,0xb886c429),LL(0x274adb1a,0x61800342),LL(0x733bc7ba,0x05688d04),LL(0xe49410a8,0x9d06b25e),LL(0x1965c8e7,0x2b710949),LL(0xad70d74f,0x3d588ffa),LL(0x3d3fbe49,0x7220b560),LL(0xe246db1c,0x9e8e9616),L_(0x0000006e), + LL(0x8d34890c,0x59239eaf),LL(0x15030568,0x6578691d),LL(0xd8dd39f6,0xcb5a2489),LL(0x121c7f85,0xf808f7b5),LL(0x952a8bc8,0x99daec7f),LL(0x84b94629,0xd8fb611d),LL(0xbad11517,0xcd32215c),LL(0x967dfc54,0x781070b7),L_(0x000001fe), LL(0x4a7b1ca7,0x3e19d6b8),LL(0x4aecc72b,0xc71b44ff),LL(0xa89c3596,0xc6ae9705),LL(0x99c2157d,0x4697a093),LL(0x95d8264e,0xd6e1d1c0),LL(0x6704d656,0x1e3c6190),LL(0xc7b65104,0x91499ef5),LL(0x466ffc24,0x879387d1),L_(0x00000177), + LL(0x9b81bb10,0x3ae92664),LL(0x35d0ee3c,0xc1f7e1a2),LL(0x20c5e62f,0x8f740935),LL(0xb7936d32,0x451ab7a1),LL(0xc573c20c,0x126bba09),LL(0x3152bb2a,0xa66454b3),LL(0xb17e342e,0x784051fe),LL(0x3681b560,0x1536e827),L_(0x000000a8), LL(0x8f5b18c5,0xcc09f617),LL(0x6e34c103,0x22747cab),LL(0xf8a96755,0xde36110e),LL(0x4dcfc108,0x67ee6834),LL(0x342676fc,0x8502fdf2),LL(0x5b48ae34,0x3b4a1019),LL(0x85dba2fb,0x75c0d58b),LL(0xda298efd,0xb34a8e88),L_(0x0000011f), + LL(0x335496bc,0xd3205850),LL(0x88067d33,0xcf402bd3),LL(0x0074be0b,0x2913e673),LL(0xe8db4e94,0x62a0cb43),LL(0xcf7beef4,0x8c2ead81),LL(0xf06de58a,0x97eccd5d),LL(0x501f23bb,0x1d5954e3),LL(0xa8b8e4e1,0xcdca24b4),L_(0x00000085), LL(0x1d598c1e,0xbf9648fc),LL(0x9987cb09,0xd224f8ad),LL(0xd88fba1f,0x60665a60),LL(0xd86a1d9f,0xf1e7f754),LL(0x26c4ad1d,0x4acf77f7),LL(0x7713e1da,0x938971a2),LL(0x0f78da10,0x3b7fc94e),LL(0x92811d7d,0xbb436839),L_(0x00000004), +}, +/* digit=35 base_pwr=2^175 */ +{ + LL(0x5c665e2b,0x4232c5cb),LL(0x9a24517e,0x26232f5f),LL(0x5981cd79,0x0a27a027),LL(0xe253d4d8,0x783b1d5a),LL(0xd6c00bb7,0x89b5ab0b),LL(0x40ea4c25,0x6c48caf7),LL(0xfc5351cf,0xa482e177),LL(0x2b0e714b,0xb60940fd),L_(0x0000017f), LL(0x8552b5ad,0x11881ba7),LL(0xfb2067a3,0x4ccac10c),LL(0x5d449097,0x9013dcbe),LL(0x8873accd,0xb2cf2a8b),LL(0x380d70e5,0x2c281733),LL(0x4631440b,0x7c3b711d),LL(0x3747bc66,0xd6b99662),LL(0x423c70b2,0x8e7bb383),L_(0x000000d5), + LL(0xa8b11d17,0x2eed227f),LL(0x7fd9a16c,0x916842cb),LL(0xe12a5d02,0x6758564c),LL(0xe59ed474,0x4b48f9be),LL(0x8e675f35,0xece126be),LL(0xf7c75d69,0x8ce3aca2),LL(0x00f88d21,0x9a768d60),LL(0x26ea6ff2,0xb90d290f),L_(0x00000113), LL(0x5d96ef4c,0xa0efb2f0),LL(0xac3a2f2d,0xa99dc276),LL(0x757c443d,0x23ce0342),LL(0x390d2a5e,0x9b674e3b),LL(0x7e7ea78e,0x32e72b98),LL(0xdca485e1,0xb6c21856),LL(0xda17d0d6,0xee5bed8c),LL(0x220788bd,0x3c4e1237),L_(0x000001cd), + LL(0x2333e9ce,0x1a6d9a86),LL(0xaadfe3f4,0x56277e4f),LL(0x389e2b48,0xc71ed641),LL(0x5717d1e3,0x5f0642b9),LL(0x56053a56,0x042b6345),LL(0xafc491b1,0x5ee182db),LL(0x9a47e510,0x6ccadf49),LL(0xf6da1632,0x3b09fd0e),L_(0x00000044), LL(0x05064c02,0x708a5ae7),LL(0xd681e121,0x3b41c0e8),LL(0xcbc4f74a,0xe5968e75),LL(0x47ee9e3e,0x6b8f739b),LL(0xe232789d,0xcd2e53fe),LL(0xd5500a30,0xde9e78f3),LL(0x2cad3174,0x4522aa08),LL(0x295cf494,0x7b02adf4),L_(0x0000007a), + LL(0x0e236499,0x8d70cd44),LL(0x61afc7ec,0xd8467c51),LL(0x4c9aa882,0xbc140872),LL(0x62a215cb,0xad6d3cb9),LL(0x6c4986c1,0x912aaf7a),LL(0x83691332,0x6db2702d),LL(0x7d4a1ecd,0x3fa17e01),LL(0xbf2405e9,0x44b4cf2d),L_(0x000000d9), LL(0x8a064116,0xe42ae3e6),LL(0x395b06fc,0x39938713),LL(0x5bb1098f,0xca3394a1),LL(0x97734c1b,0xe8c0bedb),LL(0x1edfc62a,0x9b0452cb),LL(0xc661bc2f,0x04c79c90),LL(0xe6253323,0x2a0dada4),LL(0x2e4ae434,0x218bc6fc),L_(0x00000101), + LL(0xd1cc1e71,0x144c819c),LL(0x6629502b,0xbd39770d),LL(0x8c2f7831,0xe5075a30),LL(0x4ec45cef,0x641b65dd),LL(0xa56294ef,0xe6aa4eb3),LL(0x75854d3b,0xbdb743c7),LL(0x3fdd169d,0x5176a409),LL(0x181ac2e1,0x0bd27e3a),L_(0x000000a2), LL(0x0ec035b9,0x3f057fe2),LL(0xa4b87bdc,0x3feadb94),LL(0xf7f87024,0x5db56992),LL(0x6a6cdc10,0x57cd02da),LL(0xdd69ecb6,0xc54a8f0e),LL(0xe1da3c2e,0x10cf592f),LL(0x8fe5cc2e,0x3b4989f2),LL(0x37d88e4b,0xe43fe00f),L_(0x00000135), + LL(0xb01bd7fc,0xb25b337f),LL(0xe1e660dd,0xac5d025a),LL(0x6a73c379,0xb69c2605),LL(0xdede6af2,0xb7e81b95),LL(0xa1ae9121,0x6cd030d2),LL(0x5f7f754f,0xb47d1e9a),LL(0xc9b7c0f8,0xe925d238),LL(0xd6fa902c,0xc4a56d32),L_(0x0000002a), LL(0xf95ace42,0x6ab6a655),LL(0x709a356f,0x8fa78d47),LL(0x9d32f258,0xc583f5d3),LL(0xab0a90b8,0x5d68b6be),LL(0x8bc51799,0xcae9d65d),LL(0x4a80ea4a,0xd569fb10),LL(0x2cb3b12f,0x67b09db0),LL(0x4b3e1e5f,0x28308c62),L_(0x0000007b), + LL(0xf5df84a6,0xcdd9e65f),LL(0xf9cc5322,0x1ae57f11),LL(0x997b8e3d,0x4ec38b17),LL(0x52bddbee,0x99728cf4),LL(0x3daa3db0,0xc592cfa0),LL(0xe99b31ff,0x0d3ad893),LL(0x71adf8f6,0x21a14a01),LL(0x748065a6,0x7604c00e),L_(0x0000008a), LL(0x313c6b68,0x45073a51),LL(0x4026740f,0xe3c1a79a),LL(0x03ea00aa,0x7f5aaba8),LL(0xae0bdab5,0x1d349675),LL(0x81ecbbfd,0x611cda3e),LL(0x62377d6e,0xc226bbb5),LL(0x4eee5f15,0xf8b5d257),LL(0x8afee162,0x44124202),L_(0x0000010a), + LL(0x1e93cbca,0x14655922),LL(0x200324a2,0x6c357ebc),LL(0xfbe29569,0x537e73da),LL(0xf1c77b70,0x59d41573),LL(0x12b0a8e9,0xc50a71dc),LL(0xd18455d9,0x7c9b3656),LL(0x7fcbc173,0xd9283b61),LL(0x6acf8093,0x9416c097),L_(0x000000db), LL(0xfa8b5737,0xf0027024),LL(0x3b38a173,0x386bfccf),LL(0xbbc99c54,0xbdb95480),LL(0x8bfbf241,0xcc88d566),LL(0x8353dffb,0x7968e885),LL(0xaa2a216b,0xe22f661f),LL(0xf0cc373d,0x0c189437),LL(0x5601679c,0x68e69c1f),L_(0x0000002b), + LL(0xb1e74cee,0x969661a7),LL(0x9719c192,0x390ae167),LL(0x76ffd55a,0xe6fc4921),LL(0x17827dab,0xb57cd8f2),LL(0x4435c383,0x16123417),LL(0x1dcf73d3,0x8d4cffd7),LL(0x0c91ecfd,0xe6e70928),LL(0x8412502a,0x82f5ef71),L_(0x0000012b), LL(0x019a0e1b,0xecf807dc),LL(0xfb488f96,0x6c466766),LL(0xedde1e48,0x756a682c),LL(0x37d6152f,0x2661ee25),LL(0x5b6f467e,0x96e2b2d4),LL(0xb97a8d49,0x1c1589d1),LL(0xbcf05602,0xdf83ce24),LL(0x04b7cee0,0xda686197),L_(0x0000000c), + LL(0xfca86967,0x34e40148),LL(0xfa3981f6,0x1c864ffb),LL(0x42879632,0x5079d6d1),LL(0x3e4b6047,0x9eef5744),LL(0x6e1e5a87,0xfd7f7f13),LL(0x38d5d2b8,0x19b63788),LL(0x1c2726dd,0xc17815ed),LL(0xf17abcb4,0x58a0242e),L_(0x00000030), LL(0x7bb9a599,0x40e55822),LL(0x3d146be1,0x7f28ae92),LL(0x8852f582,0x8cdd00a5),LL(0x60ada16c,0x7def110e),LL(0x8158a85a,0x1d1152d2),LL(0xa55ae5c6,0x4be61bf1),LL(0x0a31606d,0x8fcf413c),LL(0xd625cdfd,0x2b64bb3c),L_(0x00000131), + LL(0x6c3d008c,0xd11d6fc4),LL(0x0786f8b9,0x26066afc),LL(0x6f28cd76,0x6a57afc2),LL(0x9d41e208,0x2ed8fbfb),LL(0x32ce6027,0xda94edc4),LL(0xf08d764b,0x94110774),LL(0xd4093a46,0x8526b334),LL(0x084fdb6f,0x41d9b8a0),L_(0x0000002d), LL(0x652dbbe1,0x75164543),LL(0x605ecb71,0x6687cd0a),LL(0x3962a1b6,0xfe7869a3),LL(0x347a147b,0x1cab34bd),LL(0x634a95b8,0xbb85dfab),LL(0xbfeffee8,0x4995b282),LL(0xf245a753,0x69b18723),LL(0x5d6e7794,0x653a65a4),L_(0x000001cd), + LL(0x643f2f8c,0xcd45c4a5),LL(0x24a0afd5,0xa6b24112),LL(0x2258c4c8,0xfa87a5ca),LL(0x8f855fed,0x3975cb67),LL(0xc55199f4,0x9edc6298),LL(0x5a48e9a3,0x7312684e),LL(0xf55daba0,0xbfadaeb9),LL(0xc9f5f377,0xc404fb39),L_(0x00000199), LL(0x5a3e0968,0xc70ffd11),LL(0xfcccb869,0xba001f2a),LL(0xfe8068fd,0x124107bd),LL(0x06868f7c,0x28b9fe02),LL(0x3821a909,0x33728dac),LL(0xac94afc5,0x3e9edff0),LL(0x7f67565b,0x0bd10c69),LL(0x250773ba,0x54f9dbea),L_(0x00000070), + LL(0x70ff7fb7,0xad0ad081),LL(0x7b1f1709,0x88afd9fd),LL(0x2c52599e,0xa4b49d9d),LL(0x58896d4a,0x6df73899),LL(0x001961cd,0x1c7f535c),LL(0xd4c3ed4d,0x75c903a7),LL(0x4c699591,0xab8339d4),LL(0x939fc682,0xa4d28226),L_(0x0000016d), LL(0x921bc00d,0xe64a70d3),LL(0x0644b2c5,0xb7245016),LL(0x4fad690f,0x8e863833),LL(0x52268bfe,0xcab84fe2),LL(0xc76f784d,0x75b08768),LL(0xbd5df903,0x97114157),LL(0x49a7a2a6,0x5dbab306),LL(0xb4ae419a,0x15ea0320),L_(0x000001eb), + LL(0x24275a2c,0xc241f782),LL(0xba60fbb4,0xc4fd93b4),LL(0x616268c1,0x0872941b),LL(0x107f7964,0x25e04f20),LL(0x831b4388,0x7786625f),LL(0xc5f61924,0x8de20083),LL(0x791c6d52,0xb0abde39),LL(0x75c25ecf,0x22801b3b),L_(0x0000011f), LL(0xb58e09f4,0x4c6bc5ce),LL(0xb3112937,0x38e27941),LL(0xcee2666e,0xca0e3235),LL(0x8dbee896,0x9b498dea),LL(0x53066000,0x0f289764),LL(0x58ff5f8f,0xfb5ee444),LL(0xa7b5e140,0xf9fb559a),LL(0xac85e138,0x3b186272),L_(0x000000f7), + LL(0x0d7ebeca,0x20feaf54),LL(0x19cc13ae,0x3c5c75ec),LL(0x38ba79ba,0x78f3f1f0),LL(0xf2dc8803,0x911501c4),LL(0x8fb64807,0x6448cf01),LL(0xa8bff66a,0x206b2cb4),LL(0xce9b312c,0x195342ba),LL(0x219fa1d7,0x26a20ccd),L_(0x00000180), LL(0xe91053ed,0x41af0398),LL(0xadc91c1f,0x6166dc26),LL(0x5d9c3eef,0x055887cd),LL(0x90ea6dfd,0x3d270166),LL(0xa4280b95,0x206854af),LL(0x7b358dc6,0xa6ae166a),LL(0x03623eb4,0x34af3892),LL(0xe4258201,0x02a9b53b),L_(0x00000143), + LL(0xbb9f0c61,0x99256667),LL(0x3489213e,0x27fdbbfc),LL(0x1218ca33,0x5630d2c7),LL(0x5a83f00e,0xbdc8df91),LL(0x0d628331,0x28ee96b8),LL(0xbfe73e81,0x6a5f7e06),LL(0x2a7cd331,0xdd16364a),LL(0x8cd2a08b,0x20b4bc74),L_(0x00000127), LL(0x221d90fa,0x814ecb88),LL(0x8b5df20c,0x00fc7920),LL(0x76343a10,0xc99f2520),LL(0x14b68032,0x71413b8a),LL(0x654fe0dc,0x9a173cb4),LL(0xa9acd97c,0x85a386e9),LL(0x14a40bfc,0x87bf160a),LL(0x849e9970,0xa7001032),L_(0x000000a3), +}, +/* digit=36 base_pwr=2^180 */ +{ + LL(0x77384b0d,0x16c289b4),LL(0xf9601e0c,0x9eabe48c),LL(0x71ddca51,0xb3f199d6),LL(0x3fce7863,0xa3ecba6f),LL(0x2e01be3e,0x67c58c7d),LL(0xfbf4b701,0x4893679a),LL(0x2cb78d1f,0xe019a436),LL(0x15a3d7fe,0xff764615),L_(0x00000185), LL(0x25f2840f,0x160c51fc),LL(0x516e72ef,0x97156a16),LL(0xd9625db3,0xbf6e8398),LL(0x3f5b2c0e,0x651404d6),LL(0xfc5b6523,0xd10c4d87),LL(0x8eef476d,0xf40ffa31),LL(0xe5d39771,0xe3788025),LL(0x98fa2547,0xdce443c2),L_(0x000000e8), + LL(0xb2523e81,0x8e050759),LL(0x42659aef,0xb0377d50),LL(0x2b36823c,0x419b9ae7),LL(0xff957169,0xf46fc17e),LL(0x59705ceb,0xb61ce7ad),LL(0x2fffbd18,0xa7135b60),LL(0xfe9192a7,0x96f2e092),LL(0x30a3a8e5,0x14c4a74d),L_(0x000000cb), LL(0x667c895e,0x3da48897),LL(0xeb732652,0x467afe86),LL(0x5b7a7cf8,0x393a5ee2),LL(0xf2568e46,0xb15dd000),LL(0xb79a3304,0x203f1569),LL(0xd91a36bd,0xa5e938c0),LL(0x1a346459,0x521da127),LL(0x88c575bf,0x2cea24c6),L_(0x0000015b), + LL(0xc8c62a6b,0xeaef0121),LL(0x98cc53c0,0x58d73540),LL(0x925273a9,0xc04448cc),LL(0x73c56bf4,0xc52be46f),LL(0x542b800b,0x39147d47),LL(0x30298d6b,0x44cb5cfe),LL(0xb2312e04,0x9ed4247f),LL(0x4c4d89dd,0x77e09705),L_(0x000000c6), LL(0x460edd6a,0xed4f4d5b),LL(0x9206a582,0xb9ca6130),LL(0x3e18c6dd,0xa3efafa9),LL(0xa68f9bb8,0xb2d783bd),LL(0xb70a52c8,0xc0dda564),LL(0xdbe47728,0x0dc789e7),LL(0xe8a6481c,0xe4119aa3),LL(0x27f421a4,0x522c7bed),L_(0x0000012f), + LL(0xf301ee13,0x055dadf3),LL(0xd3d6ab94,0x8803374b),LL(0xa078817f,0xc730e431),LL(0xe1298465,0xaae8170a),LL(0xba08da98,0x8b779119),LL(0xf12876bc,0x1b8f7410),LL(0xbe46247c,0x67bc98dc),LL(0x18059808,0x0a103176),L_(0x00000068), LL(0xf59de67d,0xd0125b70),LL(0x1682d3d2,0x8c5ad3a0),LL(0x9c7c1b26,0x62fcf59a),LL(0xa095cf63,0x6482c8ad),LL(0x5b79b1ed,0xc253c84e),LL(0xd6952b3b,0x56917d1d),LL(0xdfad9c37,0x5c8f439f),LL(0xa63232aa,0xb95c4651),L_(0x00000156), + LL(0x1f3a0552,0x620ca4b5),LL(0x48133bce,0x765c9fcc),LL(0x710e23a7,0x7a6387e5),LL(0xd9c29479,0xa6621b41),LL(0x9fe4eedd,0x3bf9d9ac),LL(0x5df19f73,0x4cb8a304),LL(0x4f51d70e,0x45d5c96c),LL(0x25c50ad2,0xc3e2641a),L_(0x000000eb), LL(0xb3acd866,0x4b6a5c45),LL(0x9f7d19bc,0x3f85a2b0),LL(0x0758494b,0xd50f6942),LL(0x554c9337,0x40c2407e),LL(0x0ccb9c2f,0xc5dfc1a6),LL(0x11e7e482,0x6ad44e8b),LL(0x0fea5311,0xedd080e6),LL(0x9fd549f4,0x409ed188),L_(0x0000010f), + LL(0x7e7e29c5,0x2b2e558b),LL(0x3e6bc46b,0x4702314f),LL(0x56eeaa30,0x06726fae),LL(0x5ca44a1b,0x2ee6f214),LL(0x0ea8da79,0x829cf968),LL(0x141e7e4c,0x723cb279),LL(0x45b326cd,0xdac514c6),LL(0x5e8e8931,0xb9bb3b3d),L_(0x000001de), LL(0x61e5ed08,0x4a34e74e),LL(0x83644940,0xe1d4a984),LL(0x1f65c56b,0x3e5f4500),LL(0x062ee718,0xaa764b8e),LL(0x6a39ef75,0x9012ed64),LL(0xbddef450,0x42837f0e),LL(0xfaa786a7,0xf89ab588),LL(0x474accf0,0x7a91f8d7),L_(0x000000c5), + LL(0x1ef31aa7,0x70683b34),LL(0x5ed33b5f,0x2190eb5f),LL(0xf3278604,0x8f6e3b2b),LL(0xdb29e400,0xc911a62c),LL(0xd42f0700,0x688f5189),LL(0x7efff5e3,0xc2de5c25),LL(0xe2d46677,0x6189c193),LL(0x5de47c98,0xa4dc1cca),L_(0x0000015b), LL(0xa7dddac1,0xcaddccac),LL(0xcf555803,0x0778df17),LL(0x5faf93e7,0xb029278c),LL(0x7cfbb523,0xa7546c0e),LL(0x33ef004b,0xd52d052d),LL(0xc8957290,0x54a34c36),LL(0xcc555faf,0xa3e1b89d),LL(0x77136cbc,0x07995587),L_(0x00000007), + LL(0x4760b5ef,0x30fe1ead),LL(0xd1479bf4,0x3a480e70),LL(0xba684ec7,0x54c97c0a),LL(0x99909719,0xd306cb54),LL(0xea1c5645,0xcc5c264c),LL(0x3d9ecc85,0x39efac32),LL(0x465cbfa7,0xe63b20c4),LL(0xe9cad749,0x41d04b4e),L_(0x000000d8), LL(0x13242934,0x860bc182),LL(0x41b298a9,0x7a452bdb),LL(0x51ceda44,0x0786c3f6),LL(0x53ca2965,0x7845a5f1),LL(0x7e0cd8cc,0x5913baf8),LL(0x060bc9cd,0x0312de2e),LL(0x79bfb315,0xf7a14442),LL(0xa16f8265,0x104fcff8),L_(0x00000101), + LL(0x350ffc55,0xd2d93315),LL(0xecd8b90d,0x88f5e22f),LL(0x9eccd42a,0xdc1f662e),LL(0xc4f29c7e,0x42b4d8d9),LL(0x4f6798fc,0xe485f1a4),LL(0x46c699bd,0x6c52567f),LL(0xf81e6fde,0xccefcbe0),LL(0xcd5234c4,0xb5f3a75d),L_(0x00000147), LL(0xc73f9043,0x8ce6bb52),LL(0xbdeaccaa,0x746080b0),LL(0x3424a5b8,0x0eee571d),LL(0x785554a3,0x4bc343de),LL(0x6aadb674,0x44652a58),LL(0x2ff3c998,0x5fd0a875),LL(0x84f6f7fc,0xba89cfbf),LL(0xd08e7a6f,0x3566a002),L_(0x00000104), + LL(0x533b8a60,0x3e2d62d2),LL(0xda0545e5,0x476d9e76),LL(0xb53693d1,0x8749ddb7),LL(0x78864741,0x6623b715),LL(0x737a1960,0xb1899ac7),LL(0xf216ba69,0x057f8862),LL(0xb25babc9,0xcb288274),LL(0x927aa4d0,0xa7dcbe28),L_(0x00000159), LL(0xd9a6f518,0xfa70c9cf),LL(0xfefa0627,0xcdcaaa25),LL(0xceb9750a,0x15a2f18a),LL(0x9cc57e80,0xba45323a),LL(0x0cadb63b,0xe3f19ccd),LL(0xa55c80d9,0x1e511bb0),LL(0x3bb4df11,0x164359dc),LL(0x3e271d06,0x1f1ae900),L_(0x00000000), + LL(0xaa6cb262,0xb186d04e),LL(0xe56a357b,0x750898af),LL(0xb3fa3a15,0x4d60c192),LL(0xc07d177e,0x9679fa78),LL(0xf75650e4,0x3ad024b4),LL(0xfc2fc8b2,0x0bddcaf6),LL(0x559b0ced,0x604f3f34),LL(0x995261e0,0x1b5f8e9d),L_(0x000001a0), LL(0xfb2e6335,0xc5822803),LL(0x102a3be3,0xc4f23418),LL(0x2683d799,0x446dc4f9),LL(0x87d5a82b,0x82fb7bba),LL(0xba06b349,0x859d405f),LL(0xdacb2e84,0xf7fdeed1),LL(0xa51f1588,0x8b67135b),LL(0xc2217c58,0xbd5966c7),L_(0x000000b7), + LL(0x8ed9d71b,0xf4c69057),LL(0x0ee9b6ca,0xb90a3ad0),LL(0x690215b5,0xe1a72991),LL(0x9dc86f3e,0x4e4042ba),LL(0x076b900e,0x7d9520d4),LL(0xf559233d,0xa6fe5f79),LL(0xd16f05cc,0x6290cb9a),LL(0x2c55a35f,0x2d4f8345),L_(0x0000017e), LL(0x02fbcf5d,0xc7fdf1f3),LL(0x78d6c024,0x3c5ac58d),LL(0x180724dc,0xeafba33f),LL(0x0f2d4859,0xa9ec392b),LL(0x9adb7f75,0x10b122b3),LL(0xa1699e54,0x8be6fae5),LL(0xcfb1317b,0x3a96cd81),LL(0x9a5bff09,0x44a057c8),L_(0x00000143), + LL(0xf6d8c638,0xd287d869),LL(0x6aabc1f2,0x26b0f715),LL(0xbdea2e8a,0x8e33c1dc),LL(0x689a9c3d,0x8c56f036),LL(0xa841ff6c,0x527eaefd),LL(0x10032f78,0x0c199e97),LL(0x6215f00a,0xd8293042),LL(0x0262f60a,0xfb4b2139),L_(0x00000098), LL(0x0105c4ab,0xa09207d0),LL(0x2a3ccda5,0x7549d228),LL(0x67ad8625,0x0483ecfd),LL(0x12b83a0c,0x0eee9667),LL(0xe653fd39,0x14bf0bf5),LL(0xdd617912,0x5b9e1025),LL(0x58e59489,0xb42fb14a),LL(0x6b6fe3f4,0xd59f64f0),L_(0x00000022), + LL(0x82f2b927,0xfe39c03a),LL(0x595b30e1,0xa1ad263a),LL(0xcc294836,0xe59896e0),LL(0x55678ebc,0xdab6cc2a),LL(0x3b48be12,0xf27aff9b),LL(0x1525c60c,0x72f22657),LL(0xbfa65ac2,0xe179fdb5),LL(0x957d9762,0x5bc621c4),L_(0x00000066), LL(0xaabb8ddd,0x2e567a04),LL(0xfc24eb81,0xcd0abafe),LL(0x3ab9ba57,0x7ece5b5f),LL(0x94233802,0xfa49f2c9),LL(0x192ad8a9,0x7c6c9e7b),LL(0xd9733712,0x97c62d5b),LL(0x608ec02f,0x3b573c6e),LL(0x90c6dba5,0x100eafa7),L_(0x0000017c), + LL(0x41a926ac,0xea7b4b5d),LL(0x6a5e3301,0x595fce21),LL(0xee8aa9a4,0x4300c92a),LL(0xeb1b3325,0xfb3d0ddc),LL(0xe7231d36,0xdd2028ea),LL(0x0407b0dc,0xb99d20da),LL(0xf0f51dbb,0xe418d5a9),LL(0x31d74a02,0xb34fb225),L_(0x000001ea), LL(0xf7fd4389,0xbfaf4600),LL(0xe8d861a3,0xf167dda3),LL(0x92cf759f,0xc46df950),LL(0x32d3e4f2,0xb7815d1f),LL(0x91ed2fcf,0x6d421190),LL(0x12864b88,0xf9dfcf39),LL(0x04988ed3,0x74a6a7f4),LL(0x44aba25d,0x1a08a720),L_(0x000000d7), + LL(0x4033bcc8,0xd518ffb7),LL(0x2da2c2e9,0x1ae0cd73),LL(0x9a4290d6,0x357cfbc1),LL(0x784c1f06,0x3ca1aed8),LL(0x3fe20989,0x85a8dedb),LL(0x2f87969a,0xc8eb2e93),LL(0x550ff529,0xfbbcc740),LL(0x54bf85aa,0x9d6aa4f5),L_(0x00000124), LL(0xd5bc6372,0x557f4f2e),LL(0xc2efbdc0,0x9d0c30f3),LL(0x262ac2fc,0xd5dddabb),LL(0xa05b87d4,0x769d1cf7),LL(0xfc91e745,0x4d0a4907),LL(0xdcd38c99,0x89250072),LL(0xc453a288,0x0dffae1a),LL(0xe7245800,0xf08702a8),L_(0x000001af), +}, +/* digit=37 base_pwr=2^185 */ +{ + LL(0x06d9177f,0x5e3cbf74),LL(0x39ed5397,0x2f09def0),LL(0x0caf736f,0xbc534da8),LL(0xfcc790fe,0x46448c8e),LL(0xb0ad47e7,0x36b92fa7),LL(0xc7671ca3,0x90e92c64),LL(0x637080ab,0x52dfd8d6),LL(0x5711517b,0xed4b932f),L_(0x00000178), LL(0x56a0257f,0x24412386),LL(0x77234bc0,0x364971ed),LL(0x9b2d316f,0x0cc8b1d0),LL(0xdf4ae5e0,0x307856bf),LL(0x1468fa8d,0xe3791c04),LL(0xfc69805b,0xfa589236),LL(0x0c1fe733,0x89a33762),LL(0x37b57609,0x42b031e7),L_(0x000001e9), + LL(0x8a01e6a2,0x71b4886b),LL(0x83120c39,0xdae7cb78),LL(0xf3efe6ce,0xfd659d28),LL(0xe1699713,0x75625028),LL(0x0252af65,0xcd7c4a21),LL(0x81fe2a3a,0x7efc9c5a),LL(0x2fab4ecf,0x8ae2a5f8),LL(0x92444155,0xf68655c0),L_(0x00000034), LL(0x95495dd4,0xdf9b0e72),LL(0xbb0facca,0xab5a8f70),LL(0x01ed29d2,0x6b65b325),LL(0xf439adfe,0x2e9c2436),LL(0x40a6c720,0xbcd403e2),LL(0x97776531,0x4526a2b6),LL(0x90cd1256,0xa61dc2d5),LL(0x170acdcf,0xac792b64),L_(0x00000126), + LL(0x8be8d883,0x770de7cc),LL(0x95107be3,0x07c65e3e),LL(0x780e3eca,0xcf6ac96f),LL(0x3d615089,0x2549b641),LL(0xf585b5b2,0xacd5da79),LL(0x4c0d8b5f,0x3c8b5c5a),LL(0x970b49ff,0xadd6dfaa),LL(0xc025c0e7,0xdf3f065c),L_(0x00000179), LL(0x0b3c64dd,0xc797b7cd),LL(0xbc308343,0x1f367813),LL(0x138ae118,0xe7bfbf3f),LL(0x1f8c6302,0xe3cc546f),LL(0xef35ea2e,0x904ac34e),LL(0x852c3a0b,0x2596f106),LL(0xb1310ec5,0x1e6e533a),LL(0x763b1938,0x2a9b4abf),L_(0x00000068), + LL(0x29b5e462,0xab5e9c2f),LL(0xc87a1f3a,0x40c3ae00),LL(0x8fdfc7cb,0xf72aade9),LL(0xf671ec86,0xaa376ff2),LL(0x369dd7b2,0x0c4b0748),LL(0x1a9eb6f9,0xe5c39e83),LL(0xb8bdb31a,0xc9ef6929),LL(0x5a4c5224,0xaad6c712),L_(0x000001ba), LL(0x3d80ab90,0x8a32a994),LL(0x0edbbb7c,0x51b7d4fd),LL(0x9eb83ad0,0xbe08eaa8),LL(0x343de0eb,0xc33cc9fb),LL(0xad3c4d0c,0x24b0953f),LL(0x9c30b151,0x582773fc),LL(0x3a021a47,0x75ab2c19),LL(0xddfb8816,0x2171ab73),L_(0x0000011a), + LL(0x8da3d9ef,0x644b8138),LL(0x80531565,0xf0f2d302),LL(0x3bfd457e,0x64c28e98),LL(0x93b685b7,0x24eeda6c),LL(0x2b149454,0x97f74e2b),LL(0x1420398e,0xce3c2017),LL(0x93fa9e0c,0xa9df8bc6),LL(0x0fc6b820,0x951fb9ae),L_(0x00000066), LL(0x52bf8c38,0x06c9ae45),LL(0x500fd912,0x2fbca6d5),LL(0x5e0fd35b,0xe9e18d3d),LL(0x39985525,0x3bfa858d),LL(0x3a3dab8f,0x0bc682dc),LL(0x51f2882f,0x5632ba53),LL(0xd2912672,0xa5d16cfd),LL(0xefb27960,0xa8e18261),L_(0x00000078), + LL(0xf235a5ba,0x39db43f4),LL(0xdd5bbd91,0x4d8ac038),LL(0xc1e864b1,0xbb5ec32b),LL(0x9c3d8682,0x0da419a0),LL(0x7fa3e54d,0xbbcc85f7),LL(0x4911605b,0x16bf46df),LL(0x459ed701,0x42b3919e),LL(0x4a6f67ab,0xc54ccf4e),L_(0x0000016e), LL(0x761f44af,0x78dc3aaf),LL(0x7ec577f5,0x443c49b1),LL(0xcedfe95e,0x4ca71a23),LL(0x80d161de,0x88a46fa9),LL(0xa3a812c1,0x8060703c),LL(0x5d69c965,0x52f25061),LL(0xebe46263,0xf14ae427),LL(0x2518ad4d,0x8ea0658a),L_(0x00000112), + LL(0xb29db4b3,0x3eb4e951),LL(0x73934c0c,0xd205a31b),LL(0xcde75602,0xf7d9ceca),LL(0x652846c3,0xa5604560),LL(0xf53ed6dc,0xfcef8ee2),LL(0xa3dda8b0,0x73763d47),LL(0x5dcfc88c,0x3f72bc6b),LL(0x61afbead,0xb25f5862),L_(0x0000019c), LL(0x706c2fef,0xb896e8ba),LL(0x91189666,0xa5f8d9e7),LL(0x6dc25f9d,0x98f8493f),LL(0x29210ade,0x77e1557d),LL(0xc803167f,0x80aaf764),LL(0x746e916c,0x9a02bf22),LL(0x6f8c70cb,0x692f9669),LL(0xe6efe144,0x7721e1e7),L_(0x00000116), + LL(0xd6f014ad,0x10e7e9c3),LL(0x91edf75a,0x042dda6a),LL(0xfbe9047f,0x6df69276),LL(0x497f9141,0xfce4035a),LL(0xab982ab7,0x1e6adadf),LL(0xd973b8b6,0x218a9fd9),LL(0xe2c23f1d,0x9e1c8c04),LL(0x2274d47d,0x94c92cb1),L_(0x00000053), LL(0x090ec3de,0x272cecd7),LL(0x6d724d9a,0x50e492db),LL(0xe32d2f19,0x68f82a50),LL(0x6bf40e9c,0x0678afdb),LL(0x4b25727f,0xe6ae7819),LL(0x06b77a36,0xbb096d18),LL(0xeedfa35c,0xf41afd3f),LL(0xc17d9b9f,0xf7315e57),L_(0x00000072), + LL(0xaac42698,0x9e517311),LL(0x90d691a0,0xbbe7b23f),LL(0x3efcc598,0xc5b3ba4a),LL(0xf044505e,0x8a70a012),LL(0x818530c1,0xb73eeaec),LL(0xd4496b66,0x25f453bb),LL(0xabda0862,0x6dcb9832),LL(0x76d60bb1,0x6fccf460),L_(0x00000068), LL(0xa348f0b1,0xdb8646ed),LL(0x1af6f002,0x451f5839),LL(0x3087f4a7,0xa66aaeeb),LL(0x47adc893,0xbec9934e),LL(0x6e6950e9,0xb35294f1),LL(0x31d5e186,0x7bf79296),LL(0x590c3c8b,0xc1942a2c),LL(0x1b804ef3,0xf152ba51),L_(0x00000074), + LL(0xd23e1c8a,0x01715db9),LL(0x41d8f90c,0xbbc2c6b8),LL(0x4e56a842,0x83c0fc75),LL(0x986646d8,0x4fe3bcf5),LL(0x5fdbc4e4,0x2ada0ebd),LL(0x0e534106,0x8c28b66c),LL(0xb1f981f9,0x56f04488),LL(0x79a1b1cb,0x291ee458),L_(0x000000d5), LL(0x98d4263b,0xb7ee4dad),LL(0xb8425937,0x9ce9983c),LL(0xaff51ac5,0x798d12f4),LL(0x772fa5da,0xb536f2fe),LL(0x9c00b0c8,0xb35431b3),LL(0xf4789358,0x2ee8e687),LL(0x664cbdfe,0x29120ae4),LL(0xfa9435a4,0x94cd44dc),L_(0x000000a8), + LL(0xde30af0b,0x64ca10f1),LL(0x7af2cf68,0xf23265e4),LL(0xcdd4b45d,0x4fc85e40),LL(0x3c687440,0x3ef2a535),LL(0x6a698fc3,0x9efabe8e),LL(0x63e4d298,0x18de82a4),LL(0xb24c2816,0x775b0ff1),LL(0xf09e7eee,0xae091929),L_(0x00000039), LL(0x1cbca914,0x7dbc5bb9),LL(0xcc9dfa8d,0x432844d1),LL(0xb35c10a9,0xdb4db17a),LL(0xf5e1db87,0xf9910dba),LL(0x86ff1ebc,0xb2c9c01b),LL(0x189bbc27,0xa7d616b4),LL(0x5df3f754,0xe6cc2fbf),LL(0x274e1d3a,0xb0f2916e),L_(0x000000f8), + LL(0xcbf1d173,0xd9410d43),LL(0xb76d4376,0x656599eb),LL(0x900d071f,0x2fb9b595),LL(0x5fbadcc3,0xe781b5f4),LL(0xc0a2440b,0x50f63654),LL(0xbfcd2d0c,0x1e522100),LL(0x2f21286a,0x4f742889),LL(0x482b198e,0x3ab8b88a),L_(0x00000159), LL(0xd5622874,0x47efa194),LL(0x1f58794f,0xbd93f7de),LL(0xc2129e69,0x5496a993),LL(0x1b271db6,0x8f7ac06b),LL(0x5b18ae06,0x78e56286),LL(0x6111cab0,0xbf1dc2cb),LL(0x641b9597,0x9c602e3b),LL(0x6826b02e,0xeec26ee4),L_(0x000000a9), + LL(0xcc35919a,0x2497cb5e),LL(0x94f52d11,0x4ef3c830),LL(0xdf80522a,0x94fd85cb),LL(0xf72be2d7,0x29671043),LL(0x75499b11,0x540e521b),LL(0x0bd6a835,0x5b01c741),LL(0xc2f40e1e,0x4828498e),LL(0xb6d6e72b,0x3a3120eb),L_(0x00000072), LL(0x87e9147b,0x4ff41c4c),LL(0x7947091e,0x283731b8),LL(0x31294652,0xf259b874),LL(0x0f36636f,0x8ee00f38),LL(0xfc2118ab,0x8017118b),LL(0x5f13103b,0xc3d2d9af),LL(0xc3feb59d,0xca5c4199),LL(0x39888318,0x669fc868),L_(0x000000d1), + LL(0x5d7424e4,0x59ff2a43),LL(0x6be7810b,0x30694fb4),LL(0x78ec13fc,0x92716d06),LL(0x1d9c5aae,0x8fbb9bb8),LL(0x416a4a81,0x1881c6a4),LL(0x15a0a324,0x489236d7),LL(0x23235b6b,0x685caeab),LL(0xa5c2734b,0x61eb3c66),L_(0x000000d4), LL(0x383cc04c,0x2d3601e9),LL(0xd66a0119,0xa6e151c6),LL(0x9e61fd22,0x8339ddd5),LL(0x91be32a7,0x235b6f9c),LL(0x7155449c,0x322c55d0),LL(0xa7e5e410,0xbe0a861c),LL(0xce4ac258,0x0323587c),LL(0xd78b88ca,0x00416746),L_(0x00000149), + LL(0xb1cdbf59,0x59991ed2),LL(0x646f1d97,0xae034cb5),LL(0xcf9f8f62,0xa6cfbf1e),LL(0x9a35acdb,0xb02eab43),LL(0x0993f86d,0x172ffcec),LL(0xc65c756e,0x1b44bc51),LL(0x5ec6620c,0x19230f70),LL(0xc9e7a1a7,0xec7cab96),L_(0x0000015b), LL(0xf7bed1a1,0x28d9a36c),LL(0x391142c7,0x3d4288e4),LL(0x0485a093,0xf59f8fae),LL(0x0209a097,0x94df4e25),LL(0x5fdf8f3e,0xbc0be074),LL(0xb3140419,0xba7e0344),LL(0x5cbb3260,0x95c0673b),LL(0x536a91bd,0x49e9efa6),L_(0x0000007b), + LL(0x9f78e57a,0x123d2b21),LL(0xa3cf981a,0xecb0183c),LL(0x1eddfd07,0x6998ed9f),LL(0x8f90e3c6,0x0e05152f),LL(0xfad41bb2,0x7dab5c5e),LL(0x939419c7,0xca783006),LL(0xde605b32,0x98ae5cd1),LL(0x3d6039cc,0xb8a93393),L_(0x00000160), LL(0x590ae5b0,0x68c4bf82),LL(0xfdf4f711,0x01a66f3e),LL(0xa65b0015,0x241e1da4),LL(0x0665dbdf,0x4c3387ba),LL(0xf15f360d,0xc88fe301),LL(0x8acf4e85,0x061a8e04),LL(0x9ca9957c,0x51bcc011),LL(0x8585dfcf,0x656e99ea),L_(0x000000b7), +}, +/* digit=38 base_pwr=2^190 */ +{ + LL(0x37a0e4fa,0xd96e52e4),LL(0x3aebdd33,0x29f2632f),LL(0x4fd15682,0x2c70c85e),LL(0x4f3be789,0x0a1634de),LL(0xc7d9fb18,0x638b44c2),LL(0x3e6cb175,0xe33499b5),LL(0x0b60dc32,0x2ecdad29),LL(0xcf1fcbab,0x3e0d39d1),L_(0x0000014b), LL(0x5d1854ce,0xb5b7539f),LL(0xb9d47257,0x96df1240),LL(0x561ffc72,0x5e9f9e9a),LL(0x6d945271,0x9f0df30d),LL(0x25aea910,0x1e814b45),LL(0x4c475d52,0x7037d6e7),LL(0x2239acac,0x6b60afbb),LL(0x3a178a1e,0x3b53fdfe),L_(0x000001f6), + LL(0x3a760e88,0x1c8cff61),LL(0x662259f9,0x3af1d337),LL(0x798ee44c,0x04c2f55a),LL(0x7171b763,0x6b42022c),LL(0x451b89de,0xe995dc45),LL(0x0166754c,0x5d7e90f4),LL(0x45f5e9ea,0x1437fe2c),LL(0x5f81a1be,0xdd209d83),L_(0x000000f7), LL(0x34c04a7d,0x2e0b1aaf),LL(0x62ecd7d2,0xad928156),LL(0x3ef4d947,0xdc8b88ed),LL(0x90778ccc,0xd0a75501),LL(0x1ea32bf7,0x615d6df4),LL(0x370394ed,0x6cdce7de),LL(0xa5a2d856,0x5b5d94b6),LL(0xb1500a75,0x79b1d869),L_(0x0000015f), + LL(0x49071d49,0x08ad1120),LL(0x6075e725,0x64ba748d),LL(0xec7f443e,0xf8b1338b),LL(0xe9769df7,0xe04bfcf3),LL(0x276b48ae,0x8c536f3f),LL(0x51362d75,0x91347181),LL(0x7270c649,0x8771d27e),LL(0xd7277846,0x78d1b0b0),L_(0x0000007b), LL(0xf80d5fba,0xb1ff417a),LL(0x8984b71a,0x7e8990b5),LL(0x052a6765,0xa10e9e6f),LL(0xa3a8ec04,0x68613043),LL(0xf8edeb0d,0x94eee364),LL(0x38d79bc5,0xb9b0283c),LL(0x1f04d202,0x240928e2),LL(0xef3aafdb,0xc9d0a1bb),L_(0x0000017c), + LL(0x7b556d5f,0x0eb6f5de),LL(0xe1fefbb4,0x92b00751),LL(0xadf10d77,0x245d985b),LL(0xa78c0fd8,0x1ec6c5bd),LL(0x197cec62,0x6f653476),LL(0xf59e9de3,0x29578b20),LL(0x48b6a349,0xdd081291),LL(0x858df1e4,0x1947d260),L_(0x00000196), LL(0x4a2df7b8,0x72cf54b6),LL(0x702ccf08,0x5ba02c9e),LL(0x4fa2136f,0x4316a469),LL(0x62ca46c9,0x2a601fae),LL(0x6a69d6c7,0xf210ce68),LL(0x3ca9ff0a,0x108647e2),LL(0x7301dc8a,0xbc72d54b),LL(0xc0d011e4,0x8176db4f),L_(0x00000002), + LL(0xaff92b49,0x3a47424a),LL(0x1313f0d7,0xd74e27bd),LL(0xc984d57a,0xa310f0f4),LL(0x7f2762aa,0xf81b869f),LL(0xc1c0028a,0x73626037),LL(0x16619502,0xb6eafd5a),LL(0x02aa41a3,0x7a26f16a),LL(0xee8393c6,0x76a73209),L_(0x00000172), LL(0xc3e1533a,0x770bf892),LL(0xe55dcb14,0x37abc785),LL(0x49d52ff8,0x88ea32f9),LL(0xca2e3d46,0xf41fb729),LL(0x28df94aa,0xd1d7fe42),LL(0x6b931662,0x453917fd),LL(0xf0e1ad47,0x15504e62),LL(0xac5c9f2e,0x74231808),L_(0x00000140), + LL(0x1643c1a4,0x253c03de),LL(0x2114e9cd,0xd7e1536e),LL(0x1b41ae52,0xc0d640bc),LL(0xe9135dab,0xb1a92fcb),LL(0x5a9ef7aa,0xf491bd34),LL(0xd3e367c8,0x6cfcfac3),LL(0x6970f4aa,0x28093242),LL(0x12bc2a52,0x16e9d71c),L_(0x00000064), LL(0x68fd0341,0x14138a14),LL(0x1a1ea358,0xd7ebb375),LL(0x0313c60f,0xb31aaf76),LL(0x7f4e2cd8,0x63d1b78f),LL(0x376b2b87,0xea4746f2),LL(0x9adb2628,0x7159cd2f),LL(0x3fee262c,0x45cb3634),LL(0x258e2340,0x9f14787d),L_(0x0000002e), + LL(0x1f614296,0x5dcc4da5),LL(0xa12ada20,0x6dbe5329),LL(0xd7a52ac6,0x2fb42468),LL(0x898121b3,0xe9f5b08e),LL(0x86a37006,0xf5a42f83),LL(0xbe3e6de0,0x271bec98),LL(0xc405b595,0x486c9095),LL(0x9adc363e,0xec7821b2),L_(0x0000011c), LL(0x3c99889c,0x5b663d6b),LL(0x0c06893d,0xa00328d8),LL(0x8fc3f4f3,0x2578283f),LL(0x3d264389,0x57571710),LL(0x28e44b9a,0xd62bb6bf),LL(0x87dd3c9c,0xd7a2f5d0),LL(0xf55cade8,0xcb792986),LL(0xfa60b3a6,0x5a4730dc),L_(0x00000189), + LL(0x04c2c927,0x8890056b),LL(0xc5944824,0x319a82e4),LL(0x5d37d95b,0x5a8bd6a5),LL(0xd80dfb73,0xa7edec74),LL(0xdb368732,0x60fac47d),LL(0x4f46dbb9,0xb7d14924),LL(0xe4ae15b9,0x255c8153),LL(0xd6f56370,0x73d96dd7),L_(0x00000042), LL(0x8323077c,0xc814ef11),LL(0x2b965e01,0xd2dfe1b5),LL(0xcad600e8,0x2cdd66c5),LL(0xcd44f8d0,0xad1f4964),LL(0xbb170f04,0x6b03da74),LL(0x09f8b95d,0x721ac428),LL(0xc3ee7059,0xab3fd08d),LL(0x69cd062a,0xeb3f55bd),L_(0x0000006f), + LL(0x02685d92,0xb24b23c8),LL(0xbb2912fa,0x700eb07a),LL(0x547f3fb5,0x51442fab),LL(0xf8090af2,0x91ae8f36),LL(0xd9f38784,0x7db330a6),LL(0x213e5f98,0xc4904ecd),LL(0xd61a36f0,0x18124e05),LL(0xdf7f8676,0x7f34e23c),L_(0x00000047), LL(0x2a682aca,0xa540909b),LL(0x1f256aed,0x28d1d810),LL(0x643a464f,0xc1d65b95),LL(0xc56ce322,0xe242b555),LL(0xf79c9363,0x165401c2),LL(0x90b17574,0xb89e030b),LL(0xd9ba6bec,0xad9d3eed),LL(0x3cf323eb,0xb926c391),L_(0x00000199), + LL(0x68365daf,0x6bf21ffc),LL(0x9e0da99b,0x2a11bae7),LL(0x4b632c36,0xd82e9b91),LL(0x8aba8d4a,0xd3edcc69),LL(0x7bcaa8bf,0x0780abc1),LL(0x4b5bb38e,0xf449e1ff),LL(0xb3a33e0f,0x8bf9427c),LL(0x5f153607,0xd3095501),L_(0x000001f9), LL(0xc25e9667,0xa71cb23b),LL(0xf1192121,0xb5c951ff),LL(0xe5267dca,0xde9bed29),LL(0x45f5f5dc,0x62c0dc77),LL(0x5c58640e,0xb410973e),LL(0xab71fb6e,0x2ca60e2a),LL(0xbd3de2bc,0x91e919a4),LL(0xb16029e8,0xcc5ea62b),L_(0x00000057), + LL(0x920ffb1b,0x2f766525),LL(0x5063e19e,0x0c6dbf49),LL(0x2d7c8225,0xc1e5d3a2),LL(0x5919b3b0,0xc8eabb36),LL(0x9bd4d72c,0x7daca33a),LL(0xe43be366,0x6a2d3407),LL(0xd8bf85b2,0xcb065c13),LL(0x74ca1514,0x12ff272a),L_(0x000001ed), LL(0x240e3231,0xdfa8142f),LL(0xbfc4ea2d,0xe35ddb6b),LL(0xac61b3dd,0xf38f22ff),LL(0x0b6750d6,0x6e04a783),LL(0x0ae7817b,0x43182e3c),LL(0x5fc3f142,0x70dd88d4),LL(0x8958a84f,0xe8d996f8),LL(0x988beb73,0xfbc25676),L_(0x000000d3), + LL(0x2fb883d8,0xae353f2a),LL(0xb61aaafb,0x473d0ade),LL(0x890f51bd,0xd1b37ae3),LL(0x0f0c4103,0xeca49348),LL(0x0087e22d,0xc669a58f),LL(0x3462ae96,0xd7ec27b9),LL(0x71fe3af0,0x345f63a7),LL(0x5d6f6927,0x415c1403),L_(0x0000008b), LL(0x2047ce82,0x2d78ca1a),LL(0xbf2a2a03,0xffe80d92),LL(0x8144148d,0x41e35712),LL(0xf4375651,0x70453a65),LL(0x4044794e,0x74d6e72f),LL(0xc3b6ed9d,0xc9dec888),LL(0x03c9efa2,0x01d35b17),LL(0x4a8b5ee1,0x0c747556),L_(0x0000016c), + LL(0xfedb9ea3,0x89f7c11a),LL(0xcc1f90fd,0x930eb52d),LL(0x56a442b3,0xb33c2951),LL(0x6f35d3db,0x189b9ef0),LL(0x5065c93b,0x03375bf3),LL(0xb5e57110,0x9efd6440),LL(0xacf2c750,0xada5967b),LL(0x09a6e279,0x2239bfb7),L_(0x00000034), LL(0xdb8bd33e,0x5a1a6302),LL(0x0c057175,0xd07e6a63),LL(0x12ebc219,0xd7415a35),LL(0x5c53acb6,0xffce8b04),LL(0xc61aee58,0xb8a197ba),LL(0x3531c053,0x76809753),LL(0xd0b9df9b,0x5e6fa51d),LL(0xdfc91e09,0x0d89124e),L_(0x0000018d), + LL(0x8909747e,0x7284d674),LL(0x9a35c4f1,0xaebd8339),LL(0x80afa728,0xcd763811),LL(0xe7b55292,0xbfb44242),LL(0x2f4e7b9f,0xa0832cc1),LL(0x4b2452f9,0xeb50e9df),LL(0xfb6f4f77,0x557d53b4),LL(0x081a5219,0x421a55ef),L_(0x00000078), LL(0xda08f2ab,0x71ddd646),LL(0xdc6cd83b,0x827b9770),LL(0x2342c2c2,0x40af5e14),LL(0x1b228d0c,0xc507fb1d),LL(0x01646580,0x8f89f75f),LL(0x38a92b52,0xe00b0563),LL(0x154282c5,0x27686a53),LL(0x4f688875,0xa595819d),L_(0x000000ac), + LL(0x5b121bb1,0xfd31a6ee),LL(0x6472e541,0x1922fc41),LL(0xcfaa3052,0xffa68d9b),LL(0x6480d380,0xb9a18b55),LL(0xe83a1c00,0xb0e4740a),LL(0x0caf0d03,0x6f130693),LL(0x4ea7894d,0x36b54495),LL(0x3a9ed5ca,0x2a06c77e),L_(0x0000019d), LL(0x6328956b,0x8cad3c5b),LL(0x903ec0c4,0x38be6a97),LL(0x4788849b,0x6a8af4cc),LL(0xb82169df,0x753b96f7),LL(0xf48e2aee,0xbe19a762),LL(0x3765ed66,0xc53900a8),LL(0x5283437e,0x77012317),LL(0x86b0a458,0x28a77627),L_(0x00000076), + LL(0xb5408c0e,0x1466bddd),LL(0x7d090375,0x68631831),LL(0x1d77faab,0xfd6f5c35),LL(0xe1c56990,0xed7bc7e6),LL(0x544f54fd,0x65874640),LL(0x9b1f7a03,0xcb87ac9e),LL(0xd060b45a,0x33e1a951),LL(0xd46b22b1,0x419962d4),L_(0x000001e8), LL(0x7cb30863,0xb9f34068),LL(0xa3dfc88f,0x545e0d0d),LL(0x7b5897e6,0xbbf7c012),LL(0x8cdc1322,0x5bfb3570),LL(0x13ce8bdc,0xe13ad999),LL(0xe1589aef,0xb6cb7333),LL(0x92265f86,0x5f5d1b9d),LL(0x9dfffba1,0x3f3bb65e),L_(0x0000013c), +}, +/* digit=39 base_pwr=2^195 */ +{ + LL(0x60d3d920,0xe5091b5f),LL(0xae3143dd,0xec304735),LL(0xe360b755,0xf78afe69),LL(0x119298c9,0xe3ed2ff3),LL(0x5c6a7738,0x24d64036),LL(0x06b1298a,0x8b486bf0),LL(0x3448a967,0x81e9050b),LL(0x6d50f02b,0x435dfac4),L_(0x0000006a), LL(0xf1b1ce68,0x59ff13a9),LL(0xfc1bfb85,0x23011f5e),LL(0x1d2b17a5,0xbec4e57b),LL(0xfdcb9ac6,0x53d5a58e),LL(0x109c3a11,0x4b16461a),LL(0xe6c06b2a,0xa5edc709),LL(0xc93e99c5,0x4ed62c80),LL(0x18529aa9,0xb0d33b02),L_(0x00000143), + LL(0xbce7cc65,0x49c78f7a),LL(0xe90f1135,0x6489c7f6),LL(0x7145775f,0xbe8e5262),LL(0xc353f1ce,0x36a4b927),LL(0xda2f29fc,0x3ef5bace),LL(0x3d4c0acc,0xb8074e6a),LL(0x43a9c64f,0x638d3fd6),LL(0x70fffe4c,0xb73e74c0),L_(0x000000bb), LL(0x5efbd2cd,0x9bf19e6e),LL(0xe86655bd,0x7942ab0c),LL(0xe82b0e8c,0xb1c0c790),LL(0x2f2b552c,0x9dab8e1e),LL(0x390a098c,0xa67eba46),LL(0x9b4d9810,0x6a4756fc),LL(0xc97785ec,0x8fe8cb25),LL(0xf5f5b6c1,0xf04de215),L_(0x000001ea), + LL(0x3e59c5ce,0x3ea8b15a),LL(0xda44d978,0xd1bea64a),LL(0xe489d3d2,0x8d887f59),LL(0xcaf7d8fe,0xf6f90986),LL(0x76e4ba07,0xeeb4dfe6),LL(0x19aece18,0xaf8390c7),LL(0x4b163792,0x8ecf88f9),LL(0xfe44fa1c,0xa188e3dd),L_(0x0000006a), LL(0x631704b8,0xceaf9552),LL(0xd06dc6ba,0x319c43cf),LL(0x86d813d4,0xc141f1cb),LL(0x1dcaf56d,0x59594026),LL(0xe9fcecb7,0x7334a724),LL(0xda2f8a7f,0xd2a3a54d),LL(0x40d320ca,0x376b3d8c),LL(0xee9740bc,0xf59a7790),L_(0x000000f0), + LL(0xf72bf06c,0x3dcbcaa8),LL(0xcedc2a2b,0x559f9fa4),LL(0x707cbdc6,0x65301ff4),LL(0xb59f1a1d,0xfc409d5e),LL(0xbb9620b3,0x6c53a5fe),LL(0x48b591f7,0x766a3eea),LL(0xc3fc458e,0x1913597e),LL(0xb4cf309e,0x0cff2a5e),L_(0x000000d6), LL(0x8bb24162,0xdda1da6f),LL(0xcd895e57,0x92393366),LL(0x02de8414,0x59ad0cbc),LL(0x65ce8f07,0x893b6573),LL(0xbcfa2564,0x73186b40),LL(0x4fbea748,0xd0156cb0),LL(0x512a03d4,0x0e490f66),LL(0x328165e7,0xb9d1236b),L_(0x0000015c), + LL(0xff9d55c9,0xf62fa8e3),LL(0x4867369d,0xa6218d53),LL(0x3d4df374,0x462df770),LL(0x0e4446f3,0x5002dbe2),LL(0xc6146393,0xee0caf51),LL(0x1bc9af4e,0x615e075e),LL(0x2e28e88d,0x750b8016),LL(0xc58c8ffa,0xc7d3f7ff),L_(0x0000019e), LL(0x91860ab2,0x88c92592),LL(0x4340d28a,0xa23735bf),LL(0x72c0db10,0x9e762765),LL(0xb8f03780,0xf1edfe96),LL(0x7d6eee09,0x395f70ab),LL(0x2e30082e,0xa5b4a747),LL(0xa42e66ca,0x42b9fa7f),LL(0x3482cf5e,0x48386fbb),L_(0x00000130), + LL(0xd1d63dbd,0xe4cd39f5),LL(0x09f98c29,0xaa873723),LL(0x561fec44,0xdc9951ea),LL(0xfd07b42d,0x6c46651a),LL(0x34575586,0x7bf78c6a),LL(0xa447b2e0,0x9284f87c),LL(0x9cdea2ee,0x7235d419),LL(0x677e1753,0x3057349a),L_(0x000001cf), LL(0x14229ff9,0xc8f306c8),LL(0xa79e93cb,0x1a7861e7),LL(0x05616521,0x7842d63c),LL(0x95f90f64,0xcf737ed9),LL(0xc0e16cd8,0xff0413e3),LL(0xedcf1408,0xc3a4f30d),LL(0x43a170a9,0x11a0a6b4),LL(0xcc49b5c7,0x21ebf6e0),L_(0x000000a0), + LL(0xd2ecdcea,0xd4a0eaa9),LL(0x7d01a093,0xc04512bd),LL(0x177d0211,0xa88e2872),LL(0xe8ab8a12,0xf6040bce),LL(0xec0d3b20,0xbaded143),LL(0xef133b8e,0xc0271e57),LL(0x57beaa78,0xa57bec42),LL(0x6cf36a58,0xb50afa18),L_(0x000001f2), LL(0x2b962ada,0x54a163f7),LL(0xdc75f9c2,0x9214028b),LL(0xf894f882,0x29ad8172),LL(0x454f24fe,0x0197a015),LL(0xb080990f,0x1b117443),LL(0x1d17b86f,0x15991c9a),LL(0x66059551,0x66f5d53f),LL(0x4e2d70c8,0x1de3adc7),L_(0x000000d4), + LL(0x6bac308c,0xd825f740),LL(0x34b0da8d,0x0fbb496f),LL(0xde870fb5,0x365075e1),LL(0x7841bcf2,0xd3c98322),LL(0x3b8e05ff,0xc39c86d8),LL(0x74cbe33d,0xf0fd6d0e),LL(0x8904ae19,0x6ed62a5a),LL(0x2b1e2805,0xb06988c3),L_(0x000000e9), LL(0x97d2a267,0x4c1cde55),LL(0x05031449,0xc0326e91),LL(0x50606033,0x423b1ee1),LL(0xf18317b0,0x398c9c61),LL(0x5cc474ed,0x96a97237),LL(0x7eb4df47,0x96b52ef0),LL(0x1372ae46,0xb1ed9607),LL(0xabf9d1fe,0xcf8f25ff),L_(0x00000036), + LL(0x31528630,0xd54e33b3),LL(0x4ada5d83,0x74c3835a),LL(0x9759009e,0xdb44c86f),LL(0x44543433,0xb0ab6930),LL(0x244966ca,0x9d760ccb),LL(0xc7b02622,0x48a60f21),LL(0xe78f542a,0xb0eae43b),LL(0x57db1786,0x7809642e),L_(0x000001f7), LL(0xabc5cfbf,0x88276635),LL(0x5fba9138,0xcf1d1a57),LL(0x568aba80,0x955f0fe2),LL(0x9cc142b7,0x00c55c57),LL(0x897ffd8e,0x15579a99),LL(0xb910a1ec,0x58a539a3),LL(0x68c6e345,0xa76c02ea),LL(0x37272cb2,0x81b23d5b),L_(0x00000083), + LL(0x9045b8e2,0x054435bd),LL(0xe377c286,0x12ddcca1),LL(0xd747b1a8,0x3eb9f510),LL(0x775c0ea6,0x4fcce9d3),LL(0x5865c783,0xd37d19f7),LL(0x2eb67bba,0xbbc7cb40),LL(0x53271117,0xd5530a0f),LL(0x600a1a8b,0x320b3908),L_(0x0000011e), LL(0xddabbe5e,0xb10fe3c6),LL(0x3587db17,0x11b65599),LL(0x1c163208,0xad78aa4f),LL(0xe7539751,0x51c18792),LL(0xc229bfb7,0x5719f77d),LL(0xf84f03ce,0xdd5c3eed),LL(0xbb9c60b9,0xc4b8c257),LL(0xe60da1b9,0x2658c345),L_(0x00000004), + LL(0xe70fa9ae,0x80aea17a),LL(0x3cc6237a,0xa1a05142),LL(0xbfbb8572,0x82ef4062),LL(0x1a092a36,0x584063eb),LL(0x083d9b48,0x0ca19b36),LL(0xd64fde39,0x92047044),LL(0x84e4f4c7,0x3a0049a4),LL(0x025a777c,0x234042e3),L_(0x000001eb), LL(0xa8f92448,0x1e5063b2),LL(0xfb85d333,0x72d2c93b),LL(0x0f374579,0x6c2c1440),LL(0x599d4bd1,0xf99fc78f),LL(0xf8d879ab,0x7157a6f4),LL(0x24350117,0x6511ce35),LL(0xf5039be6,0x3e9cc395),LL(0xa82c44a2,0x9a34c956),L_(0x0000003c), + LL(0xe6e7d409,0x83ea2a70),LL(0x4c9ba2b5,0x137328e5),LL(0xd4654390,0xb93501ec),LL(0xd733683a,0x118e98e5),LL(0x89f374dd,0x00d407bd),LL(0x5295b907,0x13b0afb6),LL(0x57db6bfc,0x6b480958),LL(0x95fc47c6,0xef3d4708),L_(0x0000006a), LL(0xaa37df30,0x84543a49),LL(0x5c127536,0xaf148309),LL(0xbf08a1c2,0xcb7176db),LL(0xbab267dc,0xa7bbd2fb),LL(0xabd6efdf,0x8aeeb27e),LL(0xe86eabfc,0xc902ad03),LL(0x4e44e718,0xf09e682a),LL(0x064991f1,0xe6ec4fe0),L_(0x00000084), + LL(0xc501c914,0xa64386af),LL(0x86838cd0,0x4dd63878),LL(0xe353d214,0x2b6e52eb),LL(0x298c7007,0xb94c5abb),LL(0x4bbcef96,0x3cdd0d98),LL(0xfb73d97d,0xe31b50a6),LL(0xd4d6c5d5,0x63019a2f),LL(0xaac04770,0x2424268c),L_(0x00000045), LL(0x1496527b,0x959a4aac),LL(0xe7cd0ef4,0x5fb6b5e8),LL(0xcf4b2051,0x77e30f99),LL(0x1b0c7952,0x70b054fe),LL(0x38ba1d97,0x5de49575),LL(0x947a5a05,0x8fe1e2d6),LL(0x4246cd4c,0x3254f07e),LL(0x238ade18,0x21122733),L_(0x000001a1), + LL(0x99aed77f,0xf6fa1112),LL(0x9c04ed64,0xd4feb2e7),LL(0x7a120999,0x5d57b4f9),LL(0x70550af6,0xd07357d3),LL(0x41340660,0xe4afb7c6),LL(0x05ac084c,0x0826e572),LL(0xae197ca4,0x3ea7fc0b),LL(0x8f07d680,0x1c2a2823),L_(0x0000009b), LL(0xaf454a02,0xb5cdf6d6),LL(0xb32bd0de,0xf3bb89c8),LL(0x1bd8c3f6,0x5deb355a),LL(0x3db355ab,0x2f043ae6),LL(0xd5c6b398,0x0e90987d),LL(0xabe8910f,0x380521ad),LL(0x4bf6a241,0x3dfa044a),LL(0xfb752ed2,0x6a745fd8),L_(0x00000196), + LL(0xaad132cf,0x696e2831),LL(0x49f240e0,0x0d4e57f0),LL(0x3025b776,0xf18f53bc),LL(0x0b5878b5,0x56b2575b),LL(0x576025b0,0x452417b5),LL(0x51986dad,0xa57a7837),LL(0x5444a7c0,0x9f4452b9),LL(0x9f945ebb,0x8aa46532),L_(0x000001a7), LL(0xca0455ee,0xd553d885),LL(0x83b12fcc,0xd68fe49a),LL(0x3da8d9a1,0xb71fad5d),LL(0xa984d589,0x1f435980),LL(0x5db787bb,0x659a3f24),LL(0xa908e510,0xdd95c91e),LL(0xbe7d501b,0x4a9245db),LL(0xdaa920fc,0xccd618d3),L_(0x00000107), + LL(0x58fdd1be,0x26f6dba6),LL(0x1bc799a8,0xcb4e9512),LL(0xf00f6eae,0x56df676a),LL(0xe75a521d,0x29d333ce),LL(0x4eca7d77,0x27fb68ac),LL(0x206d2e50,0xa49aec5f),LL(0xaa272aa4,0x1b6a988f),LL(0x341efc69,0x6924c47f),L_(0x000000e1), LL(0xe6df0f07,0x3afd9193),LL(0x782686ca,0xe7785c7d),LL(0x3c2a9148,0x4c330f1e),LL(0xa49f1fa5,0x82ded4aa),LL(0xde962297,0xb845da08),LL(0x79a993b6,0x0729c991),LL(0xf8fef022,0xad904b0d),LL(0xe6016c6a,0x2a8c39ce),L_(0x00000108), +}, +/* digit=40 base_pwr=2^200 */ +{ + LL(0x938c22f8,0x943a3471),LL(0x837e8130,0xe2773aac),LL(0x4a3c4f46,0xf24010c6),LL(0x2b750229,0x138446be),LL(0x007131ff,0x731813b9),LL(0xc2c90ce7,0xe94672d6),LL(0xdd149a00,0x69dcb075),LL(0x7531381b,0x7f7c96d0),L_(0x00000019), LL(0xb6b38c7b,0x39ced7c6),LL(0xb63d5a97,0xa61fbc4f),LL(0x8f6b6bae,0x075fe4d1),LL(0x6ae1dbab,0xc1ebedbd),LL(0x12c3dbf8,0x6dce109a),LL(0xc087b051,0x4a2962c4),LL(0xa1e1733e,0xf40db685),LL(0x9f800e79,0xcdbc11ad),L_(0x000001d1), + LL(0x663feea7,0x4fde5cbe),LL(0xdd02746e,0x3f440437),LL(0x33232942,0x8cb2a089),LL(0x21f2f603,0x50b4f0e4),LL(0xa8d7b95a,0x18c0b0f4),LL(0x35a473ae,0xc9451cc0),LL(0x8955b22b,0x9c5154ba),LL(0x9d1fd085,0x73fe1d34),L_(0x00000117), LL(0xaff04652,0xe19db868),LL(0x961dcb73,0xeea574f2),LL(0xf8c3e1f9,0xf4327664),LL(0x9b512b73,0xb683e483),LL(0xb02a0ec5,0x0fb615a0),LL(0x7991b38a,0xb1e55bb8),LL(0x3f719551,0x19309417),LL(0x0ba8f164,0x5179fd3c),L_(0x00000183), + LL(0x20ed0fcb,0xe26950f9),LL(0xa5916c61,0x8ac76960),LL(0x2ae5b02a,0x17149cfc),LL(0xda5eb1a5,0x5c8f4a8c),LL(0x9118595b,0x0004518e),LL(0x9e0cc88e,0x9dcbce69),LL(0xb0b05838,0x5edca7cd),LL(0x8f7a0d45,0x48f7b967),L_(0x000000c6), LL(0x773e3080,0xd9ebc5ab),LL(0x32f8567c,0x92748ad1),LL(0x2f890896,0xd6e0eb81),LL(0x83d0c649,0xdc173290),LL(0xed13cd26,0x6815ffda),LL(0x775e539d,0x727168b4),LL(0x09166ff7,0xbfad2565),LL(0x7a36c1d3,0xa58e9174),L_(0x000001a8), + LL(0x46e6e936,0xf2627126),LL(0x96d6c787,0x7ee8f552),LL(0x6540e78b,0x53eb4432),LL(0x2fb88504,0x073a9cef),LL(0x0e4739b6,0x9bbfb39a),LL(0x5b6532c1,0xbec805ba),LL(0x4331c495,0xedb74df4),LL(0x002e8ec8,0xb24b19d3),L_(0x0000014a), LL(0xcbd7ba6c,0x4bae18ce),LL(0x3a66d73d,0xa9e818b4),LL(0x5a439da7,0x422e109d),LL(0xe2bd60c3,0x71574884),LL(0x482785ad,0xf6bd330b),LL(0x8c0c9a5b,0x6c8383da),LL(0x0007cc56,0x2a9a00bf),LL(0xa489783e,0xd0b67256),L_(0x00000090), + LL(0x9d4a8e40,0xd221fba2),LL(0xd46ad86a,0xfddda1e9),LL(0x14fcb5bf,0xf9686431),LL(0x60db0e24,0xb0468c5a),LL(0xa659be98,0x5c91bca8),LL(0xcabd0c78,0x1e072204),LL(0xd9453dfb,0x50ebbe04),LL(0x8aef77cf,0x599b2768),L_(0x000000f4), LL(0xb8b62e34,0x7872d194),LL(0x43334446,0xddbd4e1c),LL(0xaad0f260,0x42324117),LL(0x7d8cfb9c,0xbd6c92ea),LL(0x883e18f4,0x68768225),LL(0x72898dd5,0x59ac483a),LL(0x25923bd3,0xffec7082),LL(0x48de2e57,0x84d8a621),L_(0x00000136), + LL(0xf64d07f7,0x07a33d42),LL(0x4d434a03,0x8948a2a4),LL(0x03ccc6f0,0xbb8f90db),LL(0x6ff7592e,0x69af4969),LL(0x87ff2ae9,0xec7fca3c),LL(0x687414fc,0xd6cec6f5),LL(0xb3255410,0x9a9ae9c6),LL(0x961c9823,0x2da713ea),L_(0x0000010f), LL(0x4284bbf9,0x472d0237),LL(0x86ea89ac,0xd63ca5a6),LL(0x6cd552ac,0xe6161434),LL(0x2fb24ab8,0xdc07d107),LL(0x880d3677,0xd1833f7c),LL(0xd225c8c0,0x17c50635),LL(0x2bf84ae7,0xc1f8a219),LL(0x2e83c678,0xfc9d327f),L_(0x0000003e), + LL(0xbf7ea965,0xe0762dd1),LL(0x7e003dff,0xd60aa791),LL(0x62c54da8,0xfa92fc72),LL(0x21eaa7ed,0xff6dc244),LL(0x62c86ea7,0x29f82d5e),LL(0x68214737,0x535c6df4),LL(0xb69bc00d,0x494bda6b),LL(0xf34e2601,0xa37c7b33),L_(0x00000181), LL(0x1096ecec,0x12da1692),LL(0xcc5db4fd,0x2f945903),LL(0xc4f95586,0x15014cb4),LL(0xf70f6fbb,0x3d80e47a),LL(0x0a6967e2,0xadb489f2),LL(0x65a13ebe,0xa0094906),LL(0xa28958e6,0x9fafaa96),LL(0xda82221f,0xc7b75a31),L_(0x0000006a), + LL(0xbab5dfd8,0x59f2c453),LL(0xe3fa69de,0x0c9b2cc1),LL(0xb6318b0b,0x56b33c17),LL(0x2d5d399a,0x6dc3c1e9),LL(0xf08f8a6f,0x9e28633f),LL(0x51ffb7f3,0xf1fbcd43),LL(0x77388eef,0x9d013e8c),LL(0x953e5ebf,0x98e2766a),L_(0x000001e9), LL(0xbdbfd2f4,0x95e5a481),LL(0x017a31d1,0x1ef3e1ae),LL(0xb5b37267,0x8988b706),LL(0x748e8ba9,0x3916983f),LL(0x0e9de7d3,0x6e3e3c93),LL(0xe7e37cde,0xee4ca324),LL(0xae3cdd99,0x2fb6a772),LL(0x5fda48d8,0x470f3423),L_(0x00000033), + LL(0xa88f1006,0x32101e9d),LL(0xec995155,0xbc724136),LL(0x50786c7c,0x718458f0),LL(0x942d96ed,0xd4b44c20),LL(0xa25702b9,0xf245f9d1),LL(0x060c2adf,0xb57e7b0b),LL(0xb9d35bf5,0x7ec560d8),LL(0x06f41b22,0x66ddd256),L_(0x00000163), LL(0xe70f3385,0xbb3cf7d0),LL(0xaf72af3d,0xf5c80879),LL(0x24797994,0xcff4c536),LL(0x60ff916d,0x9be0e09a),LL(0xab9b1069,0x069c4c8f),LL(0x4df20300,0x1f6d9a1c),LL(0xe23cffcc,0xe8dd8a18),LL(0xf91c6b69,0x08a175c4),L_(0x00000066), + LL(0x95fbe896,0x77feee72),LL(0x5cc61767,0x8880fcfd),LL(0x37faec22,0x520b4347),LL(0xb528d647,0x08c7efa3),LL(0xd89b3eae,0x7f34be1f),LL(0x0837f588,0x08e5ae1a),LL(0x1bd21d07,0xdd6b478f),LL(0x2fe84dcc,0xa5eab71c),L_(0x000001a1), LL(0x834a2481,0xe505f7b4),LL(0x2410091c,0x6e308dff),LL(0xc904e585,0x881cb270),LL(0x8e50edd4,0xb510da38),LL(0x1a9f6193,0xdd2ee182),LL(0x09d449a4,0x27fbd7e1),LL(0x8ae922fd,0x099fa1c5),LL(0xd2acc7bd,0xfa56f751),L_(0x00000073), + LL(0x0a20a859,0x0116816e),LL(0x589d2f96,0x148078e7),LL(0xaaa24c2b,0x28f3d3c2),LL(0x171935f6,0x48826cdd),LL(0x84d3a584,0x4e4018a0),LL(0x3aa25c41,0x20c105fb),LL(0xfbf31507,0x27ff55a5),LL(0x109df084,0xb2e48b26),L_(0x000000a9), LL(0x7739d21b,0xd2b7193f),LL(0xacf34cf4,0x3d27ea07),LL(0xab2591de,0x55176728),LL(0x3ceb2fdd,0x2cd960ff),LL(0x716fc560,0xb6983fee),LL(0x90fd8f68,0xb59a98f6),LL(0xe1bb13d6,0x476cf07b),LL(0x119c8087,0xe73f3c19),L_(0x000001d7), + LL(0x09bcac55,0xcc9f7ef1),LL(0xf3382f38,0x8ea6fbed),LL(0x08f8a371,0x93651490),LL(0x97354cca,0xa5d779cd),LL(0xd6ff00ea,0x17e28bc6),LL(0xcb936676,0xf2c7be3b),LL(0x4d0cca52,0xe6fcf731),LL(0xfe1b3242,0x64c801b7),L_(0x0000017b), LL(0x998790dc,0x13a64152),LL(0x616ab4fc,0x4787beee),LL(0x0f38f16c,0x8c14e216),LL(0x13dc0561,0x5144326b),LL(0xab46b249,0x49417c95),LL(0x0e55d9c5,0x87bf06de),LL(0x4c1e541e,0xc0681d93),LL(0x2c9b452b,0xa73364e9),L_(0x00000032), + LL(0x8a76574f,0x406e1518),LL(0xe6346b0d,0xb53c0598),LL(0xbdfc6afa,0x9ca435a5),LL(0x50f31dc3,0x9cc07001),LL(0x1695c15e,0x1fd27db3),LL(0x80905ca0,0x1aadf518),LL(0xad7428a8,0x926949ce),LL(0xff033643,0x07ed4c7c),L_(0x000000a0), LL(0x44dec074,0x2345ae4a),LL(0xe0173b35,0x3909da4f),LL(0x56f4144f,0xe4ed2bfd),LL(0x95f4401f,0x198f03a8),LL(0x1227b3e8,0x49d6d509),LL(0x249ec281,0xe4d920ef),LL(0xc12324e5,0x0d321781),LL(0x00c158a5,0x04e56106),L_(0x00000049), + LL(0xe0633553,0x3677eb41),LL(0xa7b03785,0x9c729924),LL(0x166ddf87,0xac9990ff),LL(0x9eba58ed,0x2f6b0f44),LL(0x29db9988,0x2b93f534),LL(0x7f4a4f14,0x92b08207),LL(0x695177a9,0xc1d4f27c),LL(0x1d65e607,0xa7f4bdbf),L_(0x000000c5), LL(0x1902ce34,0xc2625795),LL(0x99c8b2c8,0xb763d2c5),LL(0x4d39eb77,0xa75ff101),LL(0xd8865e52,0x2e31245d),LL(0x51ed16bc,0xfc608d17),LL(0xd782bd3f,0x441d7032),LL(0x8f51fc25,0x47fab287),LL(0xe871d582,0x4836c86e),L_(0x000000fe), + LL(0xec4b364a,0xfbb65219),LL(0xd42017be,0x359e7f30),LL(0x3fc15863,0x5c218315),LL(0xb1a3700b,0xf2cbaf1a),LL(0x040dad16,0xb6cd3ff6),LL(0xbf23d44a,0xd045f02a),LL(0x83befb28,0x4160599e),LL(0x467f747b,0xdf89f01d),L_(0x000001ac), LL(0xc40b618b,0xf743241b),LL(0x64d1d40f,0x576ba83f),LL(0x5ece3029,0x4dc64148),LL(0x47769772,0x6d3057fb),LL(0xd175fe83,0x0884e64d),LL(0x33875e4e,0x859df923),LL(0x481b7714,0x655fbae2),LL(0xdef5f5e2,0xbc699a44),L_(0x000001cf), + LL(0x7dce56e1,0xcfdef637),LL(0x3df3dfe4,0x35b39936),LL(0xa715e9e9,0xffaca630),LL(0x1011f820,0x33f64da2),LL(0x222d3bc6,0xc5987552),LL(0x523adab0,0xc95736a8),LL(0x787c715d,0x058fef5b),LL(0x66393c63,0xeecbe32a),L_(0x00000038), LL(0x5434ecf8,0x891f4ace),LL(0x7708ad40,0x51c5f6bf),LL(0xfe89ee25,0xc377ca62),LL(0xe6011a07,0xd24cd2ea),LL(0xd028c949,0xc094a1a4),LL(0xcf84ab99,0x1fed19d9),LL(0x036f7f03,0x0551c154),LL(0x37b50c32,0xede913d4),L_(0x0000012f), +}, +/* digit=41 base_pwr=2^205 */ +{ + LL(0x8936aa0b,0xb069d26c),LL(0xd3718b4b,0x2e3dd1d0),LL(0xeaaab400,0x26ebf3da),LL(0x315a34c4,0x04c67676),LL(0x1eb5f386,0x97e2eb1b),LL(0xbc318051,0x7226db1a),LL(0xf5e17eb3,0x109ad73d),LL(0x97d098f5,0xfde98d37),L_(0x000001eb), LL(0x5982cf08,0x48806f5a),LL(0x5f09406b,0xabd27505),LL(0x4db94328,0xb3c49a50),LL(0x38e43c40,0x08d386ec),LL(0x582b99e6,0x3b07fe47),LL(0x9089cf44,0x7186cac2),LL(0xac474a1b,0x22d982cf),LL(0x7b0368d4,0x59d409aa),L_(0x00000105), + LL(0xba6cb60e,0xf6d82cb3),LL(0x989499c2,0xf86e04fd),LL(0xcb66fb0c,0x35487bb6),LL(0x7e6257cf,0xdbe642cb),LL(0x7caa5a38,0x0132dd97),LL(0xcf0c7ff7,0x9fe7cec8),LL(0x8b24a15a,0x54f2a9f9),LL(0x52eb7ce9,0x66352c75),L_(0x000000a3), LL(0x47aec9e8,0x766fc554),LL(0xced0e96d,0x99065768),LL(0x493166bd,0x2b9adad9),LL(0x28be045e,0x222a08c3),LL(0xae70d305,0x4f554727),LL(0x61d8ec1d,0xfed1873d),LL(0x41e23d82,0x9f1c46d5),LL(0xf348b3f1,0xf15cb42e),L_(0x00000085), + LL(0x82d58220,0x488e8bdb),LL(0xaabd7c77,0xba1084c7),LL(0x67c7272c,0xca6ae765),LL(0xab82a151,0x1ca84a79),LL(0xa826c75c,0x06316ad0),LL(0xf3c64348,0x0dd7f329),LL(0xde3d6a04,0x3f33bd2d),LL(0x689c3e45,0x5cdf3c43),L_(0x00000090), LL(0x8c61be6e,0xb8361cc4),LL(0x624c3291,0xb8a7b622),LL(0x4c3e4e24,0x3c10547d),LL(0x37a21d5f,0xfa09bfc9),LL(0x407a153f,0x54fa325a),LL(0x5c7fa63d,0x8d13bea7),LL(0xfb7c45a4,0x2fe1e55b),LL(0x1a8b8531,0x15e6e55a),L_(0x000001d3), + LL(0x0246d25c,0xaf11f361),LL(0x0ca563e7,0xd246f5c0),LL(0x3dbda22c,0xc1f5f271),LL(0x1517fe92,0x5b82f357),LL(0xae558aff,0xc7e4811a),LL(0xfe93852e,0x0a7d7a0c),LL(0x80c69efa,0x5a9a571e),LL(0xcf0c8dc8,0x625dad5e),L_(0x00000141), LL(0xfa629e03,0x70463933),LL(0x97e91195,0x4d8bdb29),LL(0x1670ff63,0x3758d16c),LL(0xac0e352c,0xc3148942),LL(0xebf5c218,0x789d4077),LL(0x68df9269,0xedd91114),LL(0xc949bb1e,0x1c9e59bd),LL(0xa97ced00,0xd4c36694),L_(0x000000c5), + LL(0xa992abd6,0xe820b690),LL(0x598bc6c2,0x5618c97b),LL(0xdba62f4c,0x4cfa34df),LL(0x13d15fa5,0xe73cd8e4),LL(0xf2867cb6,0x0a135ff7),LL(0xc48f860e,0x8e8f76ce),LL(0xfdd988f5,0x3ad07476),LL(0xcb3a07b0,0x6690129d),L_(0x00000156), LL(0x59630aad,0xf07162de),LL(0x6a1c1e29,0x47eefc5f),LL(0xd695eb4b,0x69323852),LL(0xf81fbfef,0xeb840ba2),LL(0xe9a13161,0x99e6fb6f),LL(0x82ed26e9,0x0282e82e),LL(0xe498cc70,0x4b2c80ab),LL(0x6264860e,0xadc02c02),L_(0x0000012e), + LL(0x9f6dc20b,0x5c25edfa),LL(0x51ff9b10,0x2267d84b),LL(0x77cd497c,0xecebc8bb),LL(0xd41da61c,0x2e90633d),LL(0x0c2e11a1,0x61552b8c),LL(0xe7151d1b,0xfb4a699d),LL(0xa5898fdf,0x74297e23),LL(0x1a45323e,0xea3846f7),L_(0x00000192), LL(0xfe2e5183,0xf83ccca0),LL(0x0905c1e0,0xc44aa9ba),LL(0x0b1cf7ee,0xfedaabc5),LL(0xc2ab4a81,0x7f856296),LL(0xb2c2c3b9,0x7c377576),LL(0x110f594f,0x248077d8),LL(0x0920b595,0x7dcc7073),LL(0xa5e1a393,0x04275ba3),L_(0x0000019e), + LL(0x003d8cbe,0xf2309e12),LL(0x1dd14f55,0x6aafea42),LL(0xd6b6d08d,0x9f501238),LL(0xd63623cd,0xe37ddae1),LL(0x079b78ad,0xc658ecac),LL(0x62a7d933,0xffca3243),LL(0xf37ff209,0xefd095aa),LL(0x8e2c83df,0x1ba236dd),L_(0x000000b7), LL(0x5e0a6e24,0x2d2f9a76),LL(0x1a69b899,0xe6914b5b),LL(0xf621c0e1,0x8472fe93),LL(0xdd02cec9,0x92c73abd),LL(0x8f7b6745,0x3d6f438d),LL(0xeceedfc7,0x290f69f2),LL(0x98f33394,0x0ed2ebb0),LL(0x2536905b,0x9076c523),L_(0x00000010), + LL(0x5e0e658f,0xa4e263b1),LL(0xb787e362,0xf1a72f5c),LL(0xade21c8a,0x3468dd85),LL(0x51d6c477,0xd69f8f93),LL(0x4ea4254f,0xae15e0af),LL(0xdb86c982,0xdd836935),LL(0x98d3a2d5,0xdc232783),LL(0x5ffb0769,0xca75482c),L_(0x000000c6), LL(0x4c2ae6ec,0x043db3d9),LL(0x09230456,0x73642e7c),LL(0x6f9dd795,0x2a8692df),LL(0x2cd98576,0xa83e7242),LL(0x16ac0a49,0xa2e9e20f),LL(0xd3fe59cf,0xd0093708),LL(0x10a46920,0x10c84d0b),LL(0xa5bda12a,0xeef3a35f),L_(0x000001a4), + LL(0x8efb0b81,0x586ad55f),LL(0x8d830110,0xef747d77),LL(0xb3bf603e,0x0f99e447),LL(0xb48e1874,0x9da3060c),LL(0x389d594d,0x10ea78c4),LL(0xc7644272,0xd0f0eafb),LL(0xaf1ebb75,0x7afab007),LL(0xcc569782,0xe37efd54),L_(0x000001db), LL(0xebfccb68,0xf0400a4a),LL(0x474a550e,0x9ca24b23),LL(0xbf9aef56,0x5613d5ac),LL(0xab9d9c2b,0xb8267455),LL(0xbf868bc0,0xdb52868c),LL(0x909956ec,0x50005f46),LL(0x49928d00,0xac611000),LL(0x904632ee,0xeb25bd62),L_(0x0000001d), + LL(0xe148f242,0x2dd2babc),LL(0xa0bb05f4,0xd775106b),LL(0xbff23d31,0xf2919560),LL(0xe97c018c,0x44a63043),LL(0x4871d249,0x87f10683),LL(0x189ec6a3,0x0d74d0f1),LL(0x257dbb86,0xd0bbd041),LL(0xcc6bf0e5,0x0f7bd5b9),L_(0x0000010c), LL(0xad1a6c62,0x9dacf60b),LL(0xde935e97,0x77ba8de8),LL(0x73dfb2df,0xeb7f0da2),LL(0x64121541,0x18d0ee67),LL(0x024e0b69,0x7b37359f),LL(0x84be521c,0xea621f3f),LL(0xcd285848,0xf996c437),LL(0x6391d449,0x64593630),L_(0x00000129), + LL(0x6d04608b,0x807ec468),LL(0x80a83900,0xb77f8649),LL(0x5eb64b76,0x07caea9c),LL(0xc62e7f72,0x0ec672b0),LL(0x465c546e,0x378afe4c),LL(0x4b627b99,0xcc8adc18),LL(0x929649b3,0x8be7d42e),LL(0x967ccf92,0xd72c8057),L_(0x000001e7), LL(0xa6efca22,0x4f172127),LL(0xf3a643dc,0x9bf5975f),LL(0x4fe72fa7,0x90cb1c95),LL(0x4c4df518,0xba142ada),LL(0x799862f3,0xca2d035c),LL(0x68d4ff7c,0x98b9e83b),LL(0x7d49b932,0x13c2887b),LL(0x21991bff,0x550f6a51),L_(0x000000f5), + LL(0x881d7401,0x5a044cc4),LL(0xe24e7a25,0x768d2e23),LL(0xfaed9fe1,0x8f7705f4),LL(0x383ba961,0xd4a1b2c2),LL(0x435ea5ef,0xa2b9797e),LL(0x3b37f12c,0xe7f7c215),LL(0xb091bf0d,0xf2391054),LL(0xb7ba542b,0xf1e35e4c),L_(0x000001d9), LL(0x701310b7,0x9eb25738),LL(0x04099f3f,0x6dcb2223),LL(0x7798cb36,0x4d56786a),LL(0x95c47f22,0x919da3c8),LL(0xdaeee06f,0xb6abb8e6),LL(0xa660ceaa,0xa641a64e),LL(0xa1891535,0xc989455e),LL(0x627a7ec8,0x2de2ad9f),L_(0x000000f1), + LL(0x834e7f6b,0xa4f8a8ed),LL(0x8d0ab54c,0x7e835385),LL(0x86915b58,0x726730be),LL(0xba996ac1,0xe84c9776),LL(0x0ac27d99,0x59603bec),LL(0xbcd0e3fb,0xc7989ede),LL(0x1f1942c9,0xf303b7a8),LL(0x9f4547f6,0x7f3e4ae4),L_(0x00000115), LL(0x53ff8c21,0x01ac096d),LL(0xdb4b7aa1,0x69d169b0),LL(0x9723cdd3,0x802b82f6),LL(0x2051bc9d,0x256afff2),LL(0x3d4a1ea2,0x321bcf00),LL(0x7da3a724,0x26d669c5),LL(0x4ccffc5e,0x72d30cad),LL(0x212847aa,0x2089c0bc),L_(0x000000d0), + LL(0x7dca0317,0xad5ed722),LL(0xe8f1e786,0x8f7b7ae7),LL(0x30b8f677,0x9bd08de3),LL(0x03bfd2ff,0x1955b540),LL(0x599706c8,0xfa5ccf69),LL(0x66c7232f,0xa98d3152),LL(0xa531f734,0x35d69728),LL(0xa518ffd1,0x2fb7fbdd),L_(0x00000104), LL(0xc9f5e869,0x5ce90640),LL(0x7f70dba5,0x4d6b1828),LL(0xa187b391,0x636739ff),LL(0xc3f757a7,0x6788a6b6),LL(0xca8804a7,0xc2a0400e),LL(0x6dfa8acb,0x499f91eb),LL(0x8bd0499a,0x28af9210),LL(0x45056091,0xbfab21ea),L_(0x00000051), + LL(0x35af5ce4,0x33f86de1),LL(0x24f47c0f,0x237c0920),LL(0x0133d426,0x76b14d92),LL(0xdebe095d,0x989c257d),LL(0x7569cd5b,0x8d5f30b4),LL(0xeebd5dcb,0x7daee4b5),LL(0x381f623b,0x5a27d2f9),LL(0xc7fab47d,0x55d628c0),L_(0x000001ba), LL(0x87c8f748,0x04d7a15d),LL(0x0ac1e9b0,0x96216dea),LL(0xb1634ac6,0x3c5fec65),LL(0xf61cf904,0xab711a92),LL(0x3d592940,0x9bf5392f),LL(0x2eefb59a,0x96b616e2),LL(0x6a36ed7e,0x5c5c3417),LL(0xb15b4b78,0x0085b08c),L_(0x00000166), + LL(0x109924ed,0x46d2968a),LL(0x9147a28a,0xfe84ed7a),LL(0x49744c91,0x88e478aa),LL(0xfd889651,0x65a34f30),LL(0x8dc8d99e,0x21fd955c),LL(0x740206f2,0x7ea7cd99),LL(0xdedce892,0xd4f83ab9),LL(0xa7c26d23,0x35d566e8),L_(0x000001f7), LL(0x91728e18,0x0fa5f320),LL(0x3ad9d78b,0x760a4e2a),LL(0x65aca369,0x6812b50f),LL(0x46ee027e,0xdb993f3d),LL(0xe5a7e2b8,0x2acac076),LL(0x60290375,0xa179054a),LL(0xddbfa0d3,0xf87bff0f),LL(0xee0dfeef,0x197178fe),L_(0x00000059), +}, +/* digit=42 base_pwr=2^210 */ +{ + LL(0xbda60a63,0x090aae19),LL(0xcb3f577f,0xe7638c32),LL(0xf59abf93,0x34b2a6dd),LL(0x35486136,0x3c50db1d),LL(0x91b5e651,0x49476ec8),LL(0xf4bbb5bd,0x83b636d2),LL(0x3dd95f7a,0xd5071e3a),LL(0x77c02f69,0x2c32cef9),L_(0x0000017a), LL(0xc6860379,0x9c9ad3cf),LL(0x35f1eec4,0x4e50cb96),LL(0x26b39588,0xd703ca9d),LL(0x3bd6a0e5,0x3fe9036f),LL(0x08ef03a9,0x605b0ecc),LL(0x070faad2,0x6abd3a9b),LL(0xf3494eab,0x7fa81977),LL(0x164f95f6,0x63b07831),L_(0x0000012a), + LL(0x57205d81,0xe8f546e2),LL(0x442871cf,0x87afe8b6),LL(0xd5e346c5,0x748676ba),LL(0xa964afa3,0xca39baf4),LL(0xe1422f71,0x0e9e0a58),LL(0xd62c328b,0xd31cca18),LL(0x07714d71,0x75787f65),LL(0x810168e3,0xd69279ad),L_(0x000000ce), LL(0xb730f78a,0x509d6354),LL(0xec14ff7e,0xbeae80e0),LL(0x9793053f,0x019f7cc0),LL(0xb6b1fd1b,0xe4fca025),LL(0x44558d48,0x7ed4a037),LL(0x86992aae,0x0e2db1c4),LL(0xf0333757,0x557f4b02),LL(0x30117649,0x88191af4),L_(0x00000142), + LL(0x23e0df8d,0x58c8cbc9),LL(0x1732c3e0,0x3466783b),LL(0xf47836e0,0x4713b9a8),LL(0x79e1d15c,0xa517b03d),LL(0xefa174a4,0x63c15938),LL(0x49e8d766,0xea4a3245),LL(0xd01d6313,0xbc5db16f),LL(0x83758c05,0x98199196),L_(0x00000032), LL(0xbf271b78,0x3af2e11a),LL(0x60042746,0xa3ab648d),LL(0x79d1c019,0x5000aba1),LL(0x253b9712,0xd9239c9e),LL(0x0e930854,0x85a1b532),LL(0xab5ac676,0xe00c287d),LL(0x57eaaede,0xe2d767a0),LL(0x43b264bf,0x2e801980),L_(0x000000ef), + LL(0xd715e37a,0x436c808c),LL(0x8615d6bb,0x78232591),LL(0x58c6e6b2,0xca6d68ce),LL(0xa40e8f75,0xc4c37875),LL(0xc01da381,0xbe962879),LL(0x58a155d9,0x5dd3d4cf),LL(0x847d5de7,0xee99fd85),LL(0x8f7f76b4,0x8718ee15),L_(0x0000005a), LL(0xd4c9f66a,0x5302a76b),LL(0x647086e4,0x1b679cdf),LL(0x93b84a7e,0xd412c242),LL(0x92243bc4,0x519ccba0),LL(0xd5c3c375,0x585371f2),LL(0x8ba3d06b,0x90f4c0f2),LL(0x1daa7685,0xf573b409),LL(0x6342e78b,0xd831b73c),L_(0x00000099), + LL(0xf1f9c0d3,0xf2f4b2d5),LL(0xbf3b998b,0x6f543cfb),LL(0x0d744317,0x25e2b138),LL(0xda23008b,0xc81ea703),LL(0xa6df5808,0x322e0c8c),LL(0xdabad20e,0x654aa6ef),LL(0x3a6c3719,0x64c8d439),LL(0xf75f2e54,0x0e935567),L_(0x00000017), LL(0x50778fd6,0xbd5f5197),LL(0xcbe15e1b,0x16df4a37),LL(0x36cef226,0xb2ae2273),LL(0xfaaea71f,0xb8561402),LL(0x40dd546c,0xed28c500),LL(0xa3b837f1,0xe3cde0fb),LL(0x7315fb7b,0x893aced1),LL(0x828da346,0x811cfb0b),L_(0x00000120), + LL(0x136ca413,0x7447323e),LL(0xc3f1d660,0xfd177667),LL(0x0d79142a,0xeab4ffc2),LL(0x1d798fb8,0x8d6c7790),LL(0x65d4f135,0x6db0f7cc),LL(0x3bd9feb2,0xeb9db217),LL(0xf714d78c,0x48ba0ac8),LL(0xa3ca23a0,0xe83bb0ce),L_(0x0000013f), LL(0xe69b95fe,0x3047872d),LL(0x2f7264f0,0x968482d0),LL(0xedcd1afb,0x4afd067b),LL(0x21685d4f,0xd2b0a788),LL(0x8c222e50,0xee57ad29),LL(0x86dde86a,0xb70520ec),LL(0xb933bc16,0x0ddb5005),LL(0x16594b1c,0xca06f19c),L_(0x00000169), + LL(0xbbca315f,0x5bb1b6c3),LL(0x0036d456,0x7469a0dc),LL(0xd48baaad,0x017a26b6),LL(0x27d8ab41,0x02fa6e32),LL(0x15045224,0x0c7090d4),LL(0xdb62af96,0x85ab46be),LL(0x89adad56,0x45b4363b),LL(0x449f71e2,0x1636a279),L_(0x00000056), LL(0xf8377adc,0x4bc2c5ac),LL(0xa5878a73,0x58a681e7),LL(0x18006b8a,0x8426f06f),LL(0xacb84fbb,0xf862f722),LL(0x32b8466b,0x041f4d43),LL(0xe59c6ec4,0xac4fc1fd),LL(0xda852fdc,0xa52e3ee8),LL(0xf6e11234,0x001b1376),L_(0x0000012c), + LL(0x9ddcb7fa,0x44679a2a),LL(0x8069c471,0x3be369b3),LL(0x0de84dbc,0x82ca8262),LL(0xd5e1f28d,0x89e87798),LL(0xa205de89,0x84051f10),LL(0x5c22abac,0xc26a5b9c),LL(0x99bba5fa,0xdd74ecbb),LL(0x359fa6c2,0x94787253),L_(0x0000013f), LL(0x2e2ac09a,0x2ac08c57),LL(0x71535b42,0xc5c720e0),LL(0xa7bd1c3d,0x18f95966),LL(0x0a4b0c9e,0x49a62e3a),LL(0x2faa6c64,0x4f85595c),LL(0x529cbdd4,0x7bb9f75f),LL(0x93955cf4,0x50f46a64),LL(0xdefa5af6,0x84215622),L_(0x00000105), + LL(0xedc6f59f,0xa294c7f6),LL(0xa2e1643a,0x5eabc120),LL(0x41a385d8,0x0ff38f95),LL(0x6b429a90,0xa608b840),LL(0x5f0ae2e2,0xf1d02f3b),LL(0x9b8946a7,0x2525fc8f),LL(0xc76a7386,0x49cc1359),LL(0xca2a7f4d,0x40152b0a),L_(0x00000034), LL(0xbe8aed8e,0x60d4bdb6),LL(0x86d41413,0x937209e4),LL(0x3c716d10,0xf538d0a6),LL(0xa096c4bf,0xd3c035a2),LL(0x04283ffd,0xb280d9ae),LL(0x4964e73c,0x3893b1c0),LL(0x75d67682,0xc2768753),LL(0x005e5f85,0x0d82ec22),L_(0x00000103), + LL(0x8d283446,0x498b2996),LL(0x86abf4d3,0xfde09c12),LL(0x2dfa3c50,0x0e695616),LL(0x29f9b0d2,0xb697398e),LL(0x48792e33,0x568f5e3a),LL(0x7493cab0,0x6ee081c2),LL(0x657411b7,0xb996a914),LL(0x6bba20f2,0x9947f5c4),L_(0x0000006c), LL(0xcc21ceb0,0xf3a30195),LL(0x04798bbc,0x8327746a),LL(0xa982d5b1,0x2b585e77),LL(0xd3a733ef,0x8fc21ff1),LL(0x683b1710,0xf43ba5ab),LL(0xca115f83,0xeb98616c),LL(0xa31d56f2,0x89236402),LL(0x2fd3f97b,0xd91441e5),L_(0x00000043), + LL(0x71daa8fc,0xcd75aec1),LL(0x1ba03e2e,0x3c07bc51),LL(0x57185304,0xca3e327c),LL(0xb1122c0c,0xfb82f00a),LL(0x4ab98ef9,0x143ac664),LL(0xed517312,0xb67dba09),LL(0x3be9088b,0xf6425b41),LL(0x9abf3748,0xb19d9dad),L_(0x000001fb), LL(0x58516a7c,0x937668fa),LL(0x1a5b0042,0xee40f983),LL(0x70389a31,0x7bddb2f1),LL(0xc5b229d2,0x5e0553f6),LL(0x5a0d088b,0xeae9e1a5),LL(0xf99b1b9a,0xcda92dcb),LL(0x5ef8199d,0x7e285536),LL(0x50942301,0xef8656fb),L_(0x00000141), + LL(0x276df335,0x996f91a7),LL(0xbba5b9b6,0x8be8a5f6),LL(0xa13311be,0x1f26ef24),LL(0xfe2a95d5,0xee11bab7),LL(0x684a38b4,0x3233a4f7),LL(0x2ff2caf9,0x3771e476),LL(0x50176c17,0x5e24adb0),LL(0x7e6a96ea,0x59a02a9e),L_(0x000001d1), LL(0xd1540fbe,0x9f444fde),LL(0xc5cef3f3,0x6ce01534),LL(0x25f5b460,0x1d8a8861),LL(0x37e2199b,0xa77157b4),LL(0xea994fb5,0x599e6b65),LL(0xa91dfcae,0x599cf24d),LL(0x48964c92,0x5d9f0d72),LL(0x171d2191,0xc2a827ac),L_(0x0000019d), + LL(0xc7cc5161,0x5e82226c),LL(0x71009ee1,0x4bc92633),LL(0xa4a6f458,0x4b11e8ce),LL(0x27cf64fc,0xa35c3f83),LL(0x247f343b,0x08c548d2),LL(0x5b9231c1,0x8238f13a),LL(0x98e33dc3,0xb0046a52),LL(0x4e6220c7,0xf590670d),L_(0x00000160), LL(0x60df1456,0xc43a8ffe),LL(0xb03d69e6,0xf8867110),LL(0x760c6b46,0x1cbd5f52),LL(0xbfe67f56,0xee1342df),LL(0xf514ecfe,0x7fa3d377),LL(0xeacf358c,0xb7abe871),LL(0xf91db13e,0xf63ff9a1),LL(0xfd2c6720,0xc9c8d118),L_(0x00000048), + LL(0xfcd4da3a,0x01aa7fce),LL(0x5590f28d,0x891c86bc),LL(0x3fde019f,0xc74e219a),LL(0x47bfd522,0x17dfc33e),LL(0xcd902b91,0x1e1f0327),LL(0x19b7adb8,0x46fb1987),LL(0x4aa71b67,0x7c133f73),LL(0xc87d00cc,0x8fc4f89a),L_(0x00000179), LL(0x15897652,0x484cb85c),LL(0x6d5a49dd,0x08cbe8b3),LL(0x0398f1d3,0xc2570a5f),LL(0xcf0588fd,0x75aff023),LL(0x9a086ca6,0x7d708565),LL(0xf0ab637b,0x478e9220),LL(0x4da37998,0x79f64eda),LL(0x9f09ba72,0x8789e929),L_(0x00000049), + LL(0x3cf6f674,0x4d0bf9f0),LL(0xe45ebcfb,0xf49cc75a),LL(0xf02c0b22,0x198a7df8),LL(0xce88b254,0x798e3ccc),LL(0x452bc12b,0xcb1f3272),LL(0xf7e1eeab,0x204a39c3),LL(0x03001038,0xbf6a035b),LL(0x8bb9d05a,0x464b8506),L_(0x000001e2), LL(0x519b1e12,0xff47d24d),LL(0xecef22c6,0x1e6f218f),LL(0xb9743d3d,0x7fe5e0b0),LL(0x0de37bec,0xb84cd289),LL(0x2f608290,0x640acc09),LL(0x0325466e,0xf1f489de),LL(0x523a4381,0x5a2fd923),LL(0x78483089,0xc78d0d2b),L_(0x0000017d), + LL(0x51ca4b89,0x0c9b5cc2),LL(0xcb500f85,0xb99572d6),LL(0xb82356c8,0x9eef8952),LL(0x575ebe20,0x607df288),LL(0x389253a4,0xa421a6b3),LL(0xd513ff9b,0x99f0eb4d),LL(0xc84b56b7,0x253ed789),LL(0xe6a5c4d4,0xbbc17293),L_(0x000000ab), LL(0x98ff217c,0x6930ae01),LL(0x8abc5898,0x12fd1e77),LL(0x94fad4d8,0xf2ff5095),LL(0x6733dcce,0xf10fab65),LL(0x36afad83,0x17d75c59),LL(0xd99578f7,0x8a1a313c),LL(0x16027411,0x0bcd7387),LL(0x46078ec4,0x30a61f54),L_(0x0000001e), +}, +/* digit=43 base_pwr=2^215 */ +{ + LL(0x011cfb71,0x25479ac0),LL(0x785ff67b,0x2f48bcdb),LL(0xc454fd25,0xf62da267),LL(0x4e460f65,0xac759a15),LL(0xe6fa21e0,0xdb56d239),LL(0xfa07770b,0xb91d7cce),LL(0xfe275a8d,0x68f07f46),LL(0xa1529273,0x477f1527),L_(0x00000196), LL(0xd7969a0d,0x36f066b2),LL(0xd81a759c,0x3a664061),LL(0x929c3e1c,0x5ba8c41d),LL(0x033ad63b,0xfdb5a8a4),LL(0xf387b665,0x2b433c84),LL(0x1e742e2c,0x4f2d4d93),LL(0xb030d2de,0x8d631141),LL(0xe3845c24,0xac94286d),L_(0x000000e9), + LL(0x650c039d,0xab7b6907),LL(0xb4f64d44,0x26b42700),LL(0x0576d051,0x2c72c1e2),LL(0x70d2ad71,0x4322ae9e),LL(0x315f4631,0x89904b57),LL(0xef02dfb1,0x24905a45),LL(0x8a7b4701,0xde26cce6),LL(0x5f0db2ca,0x183ba4ac),L_(0x00000004), LL(0x3a42cd20,0x6fcfeb99),LL(0xee9920d8,0x1cc7f2d2),LL(0x46bdb299,0xb71c2095),LL(0x1516a6ea,0x99e62c53),LL(0x95f1492f,0x197ae770),LL(0xe95c2cf5,0x013e12e8),LL(0x2aad7be2,0x60b78cb7),LL(0x0e70b967,0x48ef0032),L_(0x000000aa), + LL(0x7b152b81,0x2adc31d1),LL(0x1cf4d989,0xf817919d),LL(0xea745ea9,0x86caaaf3),LL(0x035cfaca,0xbd62e874),LL(0x533bc33d,0xb1acfb8f),LL(0x2cc2ce8d,0xb212b5b7),LL(0x5b9ba7f0,0x350192b7),LL(0xce04c178,0x39e7b1a3),L_(0x000000bb), LL(0x88563e49,0xaa946b95),LL(0xe4c90142,0x0e515aee),LL(0x5062c2f0,0x99e87538),LL(0xcd39192d,0x52eb943e),LL(0x0c893238,0x0201b73c),LL(0xfbac7e1c,0x1ab36a78),LL(0xc6d833b4,0x58d01a7c),LL(0xe359c01f,0x84ab5208),L_(0x000001b6), + LL(0x7511fae2,0x9afd1135),LL(0x428434f6,0xab322fa2),LL(0xcbe5e3de,0xd89f361e),LL(0xc1b08880,0xc1fbd2b7),LL(0x6a50aa80,0x9e40537e),LL(0x7fdf104d,0xd4f51df5),LL(0xe707164e,0xb78a6cfb),LL(0xd887e3d0,0x0a3d0f36),L_(0x0000012f), LL(0x96365e7f,0x129d87d9),LL(0x64aad3c6,0x2952186a),LL(0xf8224d3c,0x45209284),LL(0xc689c1d4,0x2c194d7b),LL(0x03f44aec,0x0e7c6b2d),LL(0xf18a57e0,0xe28c9eb3),LL(0xfac4981d,0x4b8b7de6),LL(0x55215906,0xb3077009),L_(0x00000134), + LL(0x3a3a13a5,0xbeace249),LL(0xc22b9755,0xd063181b),LL(0x27e6ebf5,0x80b61753),LL(0x0eb1c7f3,0x8b95d0ba),LL(0x791e9667,0x5d4d0f0f),LL(0x5f5189e4,0xdd28c3e4),LL(0x9162d716,0x2eca4da1),LL(0x40913d2a,0xa84739de),L_(0x000000ee), LL(0x85fea8d0,0x3c004f10),LL(0xa6620dad,0xb3680a4e),LL(0x31990981,0x8a6964de),LL(0x3b4c2f57,0xd1986197),LL(0x3b4fdf64,0x5bc5cf00),LL(0xea0f7010,0xbf1dee68),LL(0x0d263581,0x3edd2000),LL(0x8ba6db98,0xac05f412),L_(0x00000043), + LL(0x79147285,0xbe273f4c),LL(0x71c0654b,0xec66bd4d),LL(0xdd0505e9,0xed4771b4),LL(0x4c891619,0x316ff12d),LL(0x821a0542,0x0c65ede4),LL(0xa23cffae,0xb678c0c5),LL(0xbf05e02d,0x32bbf48a),LL(0x91ff4f9e,0xe3d08326),L_(0x0000018b), LL(0x944ec586,0x5c351b2d),LL(0x00e74766,0x3939c0c8),LL(0xfe310474,0xb780a9b2),LL(0x5e63ba55,0x11cfcec0),LL(0xc11119c9,0x322972c9),LL(0x0c8e0043,0x35f5ba3b),LL(0xa0946bc2,0x222b78da),LL(0x622257dc,0x2be73a82),L_(0x000001e9), + LL(0x2b9dae76,0xe9b24e64),LL(0x1b491ac4,0x9716bae3),LL(0x162c28a6,0x192c7196),LL(0x00752a8a,0x10883cfa),LL(0xf0af2cfc,0x77a5388d),LL(0x5f1ffb0c,0xebf78534),LL(0x49abf2a7,0xe9343bc9),LL(0x54bcef89,0x3a94d690),L_(0x00000175), LL(0xa864f2a7,0x1bdde3a5),LL(0x0362ffa7,0x45782cba),LL(0x113c94f8,0xb94efa60),LL(0xb5969326,0xd2b826b0),LL(0x92ab6bc9,0x228e2d7d),LL(0x1f6dc09e,0xe5f07f40),LL(0x3371efd7,0x87e6028d),LL(0x90fcb8fa,0x35577bbe),L_(0x000001a7), + LL(0xcf53147b,0x97a83e73),LL(0x0fba4048,0xd11d0f51),LL(0xfcf9128e,0x6dfde4ab),LL(0x095c92d5,0xbcfa2a47),LL(0xff3cd334,0xa5b6f4d4),LL(0xa2490038,0x5df73169),LL(0x663bcbc9,0x41aba80e),LL(0xa47769e8,0x45877c64),L_(0x00000076), LL(0x2104365a,0xa582938f),LL(0xc439a531,0x43321e6d),LL(0xf7387146,0xee861253),LL(0x52424fe3,0x13590283),LL(0x0a676302,0xe71ab2ae),LL(0xa837702f,0xf460396c),LL(0xf7bce515,0x01882e04),LL(0xd9362399,0x899edf93),L_(0x000000c5), + LL(0x9bd2b2b9,0xc33456a5),LL(0x6dcdccc4,0x97a89ed3),LL(0x865bca41,0xbdb83821),LL(0xf50a59bc,0x46a8f43a),LL(0x47c10299,0xbaa69204),LL(0x03139274,0x1da8242a),LL(0xa899da5a,0x49b0bcc8),LL(0x2caf68d5,0x7ba862a8),L_(0x0000003c), LL(0x4dc88c23,0xc1c2ff36),LL(0x896e10e9,0x10c6c1db),LL(0x7a488fa2,0x58161012),LL(0x6b4ceefa,0x5188acf7),LL(0xea1e2f11,0x72073c50),LL(0xefaa1151,0xe696e16a),LL(0xd65f38a5,0xe4dea3d4),LL(0x6710e2d4,0x849ce811),L_(0x00000165), + LL(0x6cdc17de,0x2c374c6c),LL(0xe3454080,0xbd6a5b20),LL(0x337b0f58,0x40246c40),LL(0x31d7aff9,0x49292d34),LL(0x01dba6a8,0x37486a26),LL(0x41a0f90b,0x12d61cbe),LL(0x782067dc,0x3033f828),LL(0xebc39fcd,0x519d9486),L_(0x00000109), LL(0x026818da,0xf42673be),LL(0xf2e2739e,0x40e906d3),LL(0xd8ecf6f0,0x37b0e1f1),LL(0x5c024d38,0x6708c065),LL(0x3417cf59,0xc5788208),LL(0x17020946,0x8942a103),LL(0x7086a405,0x279752bc),LL(0xbd8d65b0,0x96d15474),L_(0x00000127), + LL(0x9abc3d40,0xf756f14e),LL(0x748874bf,0xf9192ae1),LL(0x50f7dc14,0xfb847314),LL(0x424a6e96,0xd7141cc9),LL(0xe0539e22,0xefedb9cd),LL(0x7f5ec1a8,0x7abbcc13),LL(0xe6446e18,0x0050521f),LL(0x3706445a,0xedca13d0),L_(0x000001b1), LL(0x2f62e709,0xa5e744f0),LL(0x8279d454,0x8aad355f),LL(0x03801f2d,0x88a334e7),LL(0x30342602,0x34cb8228),LL(0xed180e75,0x6d4243df),LL(0x5bbbe349,0xd0752a72),LL(0x0977e643,0xd5658e43),LL(0xf968b621,0xa8dfe2a7),L_(0x0000009d), + LL(0x3757bccc,0xce9d3ef4),LL(0xb10a83cd,0x021aa7d7),LL(0x804bf03e,0x79a14071),LL(0x09822fa4,0x1adecf50),LL(0xec4d2fe9,0x8fc3b061),LL(0x479fbad1,0xffe82ea4),LL(0xb70c5762,0x25b0bed7),LL(0x8a667da4,0x6f5ffd5c),L_(0x00000042), LL(0xd836c22b,0x9beeade4),LL(0xd28e01a7,0xd7c41634),LL(0x124715f6,0x83d30d7e),LL(0x8e33c4f9,0x2ee2bdb2),LL(0x5442a068,0xa18cfc1f),LL(0x68390a48,0xa6cb1637),LL(0x44abd789,0xa8ecc588),LL(0xbd6682cb,0x8ea17383),L_(0x000000ae), + LL(0xd8199c9b,0x86b3c132),LL(0x77f229f5,0x1a49a2e0),LL(0x6f74d75e,0x15ed7662),LL(0x8dd2368b,0x7d799783),LL(0x313152ab,0xa636aa73),LL(0x925319e7,0x2947bd12),LL(0xacf4c559,0xb2b023d8),LL(0x0322c16a,0xc8a967d9),L_(0x000001c4), LL(0x8a09b0a3,0xb1025eae),LL(0x8ae5f654,0xf817133f),LL(0x081d6d59,0xea1aa5d9),LL(0xe9090ae2,0xe61c2d54),LL(0x0784905d,0xf542221b),LL(0x5789e25c,0xa9b09c19),LL(0x1b8dadc2,0x7d4f5221),LL(0xabe01efb,0x446d7ebe),L_(0x000000da), + LL(0x8fc36944,0x1a86a356),LL(0x88a55222,0xa9332303),LL(0xfc1f186c,0xf8c2ca0c),LL(0x1f24d4d5,0xa82a6905),LL(0x753ac024,0xb6761f2a),LL(0x8a0fe4e0,0xb4a03fa9),LL(0xd1d20586,0xcb862d5f),LL(0x2bcb9949,0x48673692),L_(0x00000144), LL(0xa285574f,0xb1192295),LL(0x23eb5f8c,0x2436f1d4),LL(0x3f4febd2,0x842f6ae2),LL(0x0f1f266f,0x8091b264),LL(0xee94d349,0xf4561f25),LL(0x6f4a16f1,0x6e303b52),LL(0x80ed7ffc,0x08bbc14e),LL(0x6f957c19,0xb48f99ac),L_(0x000000a1), + LL(0x73d10fa8,0x1b164c81),LL(0xc855b452,0xdd4617a8),LL(0x9be0c270,0x94a57ba1),LL(0xb28b6ed5,0xc1383246),LL(0xc74bc4a3,0x17910967),LL(0x2a2a0e34,0x8aab9202),LL(0x93b8d150,0x122babf8),LL(0x1dffa251,0x32612c73),L_(0x00000069), LL(0x5fd7acd3,0x906979fc),LL(0xf242bd9e,0xc7fe62d6),LL(0x59507727,0xa94a1beb),LL(0x966af710,0xe7344f5a),LL(0xd397803f,0x5b835cef),LL(0xcd196fb1,0x1ca9e8ac),LL(0xe701eb7e,0x09bdf0a9),LL(0x521354ea,0x16f0f7c2),L_(0x0000018d), + LL(0xc389a398,0x92326c61),LL(0xc80b1e6a,0x0b658149),LL(0x23067f4a,0xd5cf4f64),LL(0xc96735ae,0x1cf4dd22),LL(0x43cb53c3,0x4b478ba1),LL(0xb353c721,0x3b1bbb1e),LL(0x46f2e84d,0x7b5aa79b),LL(0x4a4fa3d6,0x0db507d4),L_(0x000001a4), LL(0x5ad31f07,0x5402e45d),LL(0x19f0b31e,0x55650578),LL(0x214cdf81,0x3fd50ebe),LL(0x3efd8e06,0xd808dee6),LL(0x0c82d63a,0x39ccec30),LL(0x39dfd0ee,0x5249be7d),LL(0xb6c788e8,0xebe9271a),LL(0x7ffa3cad,0xe23c5a72),L_(0x0000019e), +}, +/* digit=44 base_pwr=2^220 */ +{ + LL(0xfd61361b,0xf3125658),LL(0xd69b66e1,0x96b636a1),LL(0x0c7ac9e9,0xe69b9e47),LL(0xf9bb3617,0x1b1e895b),LL(0x12050a8c,0xfa5a11a5),LL(0xa2492213,0xc2919aff),LL(0x08d55c3b,0xc4be1b10),LL(0x6dcf2c08,0xdd2f7392),L_(0x000001c7), LL(0xb4fb57fd,0x6ce4aac2),LL(0x6292f827,0x9277cab1),LL(0xc90518a9,0x144e677d),LL(0x0432d015,0xea4408ab),LL(0x35d9214e,0x49b20eb2),LL(0x2560b8a6,0x48a45d8a),LL(0x37dd269b,0xcecf7d1d),LL(0x71a47616,0x7df8d49d),L_(0x00000152), + LL(0xa734d86d,0x475ac257),LL(0x324330b9,0x3a9d12f1),LL(0xcecaa5dc,0x048adf08),LL(0x33641cc3,0xbfcc4fdc),LL(0xe40352a6,0x76f01bad),LL(0x6e93dac8,0x5bd7dfb0),LL(0xc0e1ca53,0xb21d4494),LL(0xb51965b8,0x20e57219),L_(0x00000031), LL(0x46f90e9b,0x76cb857a),LL(0x1b517407,0x31907caf),LL(0x38843e17,0x37770400),LL(0xb3c14ab5,0xb47cfe14),LL(0x58c99d12,0xd18daa18),LL(0xd4551590,0x84db2817),LL(0xc4d8f7ef,0x5a8544bb),LL(0x752b595c,0xed331ee1),L_(0x000001cf), + LL(0xa8718c9c,0xd016412e),LL(0x3d459798,0xc7059eeb),LL(0x07afd251,0x7f6e9107),LL(0x6ae603f5,0x7c29b336),LL(0x1f1d424b,0x1f08b6d1),LL(0x06f8f459,0xfa0b1884),LL(0x6dfa46f4,0x22a09ce4),LL(0x1ee6193a,0x050edf3e),L_(0x00000067), LL(0x59ef37aa,0xbc282a7b),LL(0x0490844a,0x16eca5fb),LL(0xfa414af6,0xdd42a4a0),LL(0x748c915e,0x6f1ab810),LL(0xa3d4af5a,0x31ef86bb),LL(0xe768aff3,0xd8ffc35c),LL(0xb6c4e536,0x6e278d36),LL(0x8f61ddd5,0xb2c50623),L_(0x0000019f), + LL(0x108204ec,0x1febacf8),LL(0x0a0c3a40,0xa1fbe66e),LL(0x2570b727,0x343654b7),LL(0x5299d8ae,0x83e0647b),LL(0x158b323a,0x3d86ee0d),LL(0x758534d1,0xe946224b),LL(0xcb77d0eb,0xe9a38321),LL(0xb845ec39,0xd329eea9),L_(0x0000003f), LL(0x4421327c,0x13edfc50),LL(0x0c4b9b9c,0xefee69b8),LL(0xce07a452,0x110b9736),LL(0x2779de28,0x810433de),LL(0x19cb506b,0x1468d237),LL(0xd2930983,0xb54615ee),LL(0x895d360f,0xa6a72f30),LL(0x2a939f9f,0x0f67c082),L_(0x0000019a), + LL(0x613613a0,0x41479bab),LL(0x64b3ba28,0xae853dfe),LL(0xd7d5f8f3,0xeddc5d69),LL(0x3c023c98,0xc2af1c91),LL(0x5e51c064,0x3e811beb),LL(0x3e28caf2,0x297f73a1),LL(0xb2c63f7a,0xe92c2db1),LL(0x272783c6,0xa58f0869),L_(0x00000175), LL(0x4d9e33e7,0x2930859a),LL(0xf96d2a6e,0x21c82319),LL(0x234b3a37,0x99962855),LL(0xe1e952c7,0x834c3fcf),LL(0xa9fff526,0x2d66290a),LL(0xc1aa8293,0xd0618b6f),LL(0x65d795be,0xd8f51b17),LL(0x7ad3a784,0x61d170fa),L_(0x00000185), + LL(0x6fb99ec5,0x5b03ab3b),LL(0xdf673d70,0xd514df02),LL(0x97926d51,0x68bd9794),LL(0x7f3cad99,0xa7bbd732),LL(0x807b8edf,0x1dee6527),LL(0x9072fc5c,0x6db8f170),LL(0x8a088d03,0xfad430f3),LL(0xf3373c9b,0xac0bdf9d),L_(0x00000068), LL(0x562932b2,0x98ce826d),LL(0x15d43a46,0xd64992b9),LL(0x0e1471fe,0x12cf137e),LL(0x11a1c256,0x19907c68),LL(0x97e5e746,0x5dcff6a5),LL(0xb4d10f45,0xfe503afc),LL(0x2daf8e96,0xbfaf4738),LL(0x51a1e9e2,0x47ef585e),L_(0x000000ca), + LL(0x57c899ed,0x309ddc9f),LL(0x000b8805,0xf9ec0561),LL(0x61be65ad,0xdb755990),LL(0xc6ac2e8b,0xf8f392c7),LL(0xc546a9f7,0x709f90fb),LL(0xaa4eb38f,0xb81ee256),LL(0xe3bb73d5,0x920cd9ff),LL(0xe54f7913,0x8fd7ead1),L_(0x000000a5), LL(0x6fe432ea,0x7d99d437),LL(0x42314efd,0xecbf2570),LL(0x0d11bf19,0x1a26524d),LL(0xc070e881,0x80db7170),LL(0x69bb46ac,0xed697625),LL(0x6e7f5dec,0xf5d4f199),LL(0x35c855b1,0x63c6d1bc),LL(0xcfaf131b,0xfd69fb7f),L_(0x000001d9), + LL(0x4c7faf89,0xe289504c),LL(0x7c67c701,0xc21c143d),LL(0x51104808,0x429b8b10),LL(0x8547ea3f,0x643f8b1a),LL(0x442d1597,0x8e30463a),LL(0x1322c20d,0x9700b9ca),LL(0x12313e31,0x53c7c741),LL(0x429e582d,0xebcdb21d),L_(0x000001bf), LL(0xa6df174d,0xf30a9c65),LL(0x734a8421,0xcca7bd70),LL(0x12f441c1,0x42e970e9),LL(0xda35c856,0x990f29b0),LL(0x7fbc6108,0x201a5ca8),LL(0x006704f9,0xb4ba5b8a),LL(0xd79c4200,0x5332f15d),LL(0xaa2c5720,0xbc6411dc),L_(0x00000184), + LL(0xa1d7fa9f,0x366d01e5),LL(0x0b24e344,0x05318d76),LL(0x9ed7c092,0xd4cb6907),LL(0xdf8af7fc,0xe97d11bc),LL(0xcad57852,0x4e593cff),LL(0x966648ef,0xeb5229ac),LL(0x0513e9bd,0xc0b6887d),LL(0xc4457c4d,0x8e5e6129),L_(0x0000001e), LL(0xe56e3c4a,0x33909c43),LL(0x6b6ecd98,0xab630f3e),LL(0x58c96e3b,0x604359ea),LL(0xa6fb1a48,0x4c23bc41),LL(0x01509a8e,0x735b6fd3),LL(0x55538a0d,0xc1e6f88e),LL(0xca10627f,0x43b7d098),LL(0x7a0bef75,0x237cd9ab),L_(0x00000034), + LL(0x382d4ce3,0xd57fa9f7),LL(0x39f2057e,0x5a8fe9d0),LL(0x819adb6c,0xf4dd28c4),LL(0x26c97625,0xa0db7a3e),LL(0x6c864ae6,0x8bee872f),LL(0xfd35c90d,0x42a5de0c),LL(0xf6b8c643,0xa0d68766),LL(0x613a35a1,0x32e15e02),L_(0x00000001), LL(0x7201d2d4,0xeb515a90),LL(0xb3006a97,0x293640a4),LL(0xdf1363d1,0x4e898435),LL(0xc4282ef1,0xf5eec35c),LL(0xf5395189,0xf9d8e87e),LL(0xecf465a2,0x621a8997),LL(0x2d4680db,0xfe197810),LL(0xf2c35d16,0xdc319b13),L_(0x0000019b), + LL(0x35c78619,0x976fd128),LL(0x4fea3432,0x9d455cba),LL(0x94c9bc3b,0xb7bb73c4),LL(0x0a3d4425,0x90c2ec3a),LL(0xc1f93e1c,0xa95d87c6),LL(0xe60757b3,0x60d5d399),LL(0xd3d27995,0xce2c84d1),LL(0x81c8808b,0x750e4526),L_(0x0000017f), LL(0x32b7091e,0x76361e05),LL(0xe202d497,0x074baca9),LL(0xd7df203d,0x18d9c96d),LL(0x60965442,0x6777a2e9),LL(0xcf542875,0xcea9162e),LL(0xd6cf9057,0x0e08540f),LL(0x118843d2,0x12ce0e32),LL(0x64680618,0x84e4eabb),L_(0x000001d9), + LL(0x3de4c30b,0x1640f2bb),LL(0xe1cd2dab,0x67a49e8e),LL(0x68415129,0xce1a4189),LL(0x5f6655b9,0xf8e55d14),LL(0x58251e9d,0x2a896a11),LL(0x2990bcd1,0x5ee2f029),LL(0xc1b51e11,0xe59128b6),LL(0x7f1b211c,0x753ab1e4),L_(0x0000009b), LL(0x0e4c3270,0x75ae7bca),LL(0x0ff0470f,0xe26fb687),LL(0xdde851ff,0x6353acb2),LL(0x661a72f5,0xa014b448),LL(0x6da88895,0x20ab0fca),LL(0x41dc7a9c,0x90420ccc),LL(0x97a7f272,0xf726e07e),LL(0x23c2dfa2,0xb43c28ae),L_(0x000001db), + LL(0x19987e83,0xc4bea825),LL(0xb4d6f37a,0x86bba8ac),LL(0x1e145dc1,0xa3d3860d),LL(0x9d377493,0x538d1baa),LL(0x9ce67c55,0x507bf776),LL(0x0148462a,0x563563c0),LL(0xc2891330,0xba0e7843),LL(0xc76b64b0,0x2ff4ccb3),L_(0x0000012d), LL(0xca889e53,0x632d2ce5),LL(0xe7e3d3fe,0xeabe8bf3),LL(0xb7126e82,0xcc6bda06),LL(0xf58c6361,0x69480574),LL(0x6a99c0d0,0x660e6906),LL(0xd1fdf14a,0x465778f0),LL(0xddbb43b6,0xb843d29c),LL(0x575fd92f,0xa63f0dcd),L_(0x00000158), + LL(0x2cd4fd82,0x0e9a73c7),LL(0x52c71a96,0x4d3a4557),LL(0x067ccd35,0x3e412b8b),LL(0x7bd77ec2,0x951e31d9),LL(0x4100bac7,0x2ac6482b),LL(0xccccb053,0x6361906b),LL(0x79655211,0x269a7fe0),LL(0x756f8ab0,0x3b6f68fb),L_(0x00000062), LL(0x8ce6630b,0xcbecac0c),LL(0x4a29a7b7,0xf7ba2d3b),LL(0x47e9ba07,0x2e6c073a),LL(0x5aa6a94b,0x73cf6f20),LL(0xa8a42977,0x50805433),LL(0xf9945386,0x74ac62ed),LL(0xc8f04c45,0x14e4baea),LL(0xe37de45f,0x3e21f3c5),L_(0x0000015b), + LL(0x73f77d7e,0x6ff96082),LL(0x58430006,0x328b95a9),LL(0x82931188,0x968102c5),LL(0xbb40cb55,0x6514614a),LL(0x370bf205,0xe5de48d5),LL(0xb3d94790,0x73c104b4),LL(0xa5683b4f,0xfb96fb1a),LL(0x36a9677e,0x17789e50),L_(0x0000010c), LL(0x32bfd837,0x4594f7a0),LL(0x3fb584bb,0x9be3e3c1),LL(0xc86bb1e7,0xef4aecee),LL(0xfa93fc70,0x9de5dfe2),LL(0xbfd3b8de,0xe7670296),LL(0xe1fa5638,0x228cc6d9),LL(0xd230d44e,0xf7797644),LL(0x82d4c20c,0xcfd9942c),L_(0x00000018), + LL(0xe19f88e8,0x828b99d9),LL(0xc2beefa3,0x51fa512e),LL(0xd33e3c3c,0xbe14d684),LL(0x34a6c37a,0x5b5936ae),LL(0x89d4bea5,0x2802583c),LL(0xd938e649,0x98da605b),LL(0x1f045420,0x55288cfc),LL(0x659c47e4,0x85b5fed0),L_(0x000001de), LL(0xb9ddaea5,0x22472a49),LL(0x0aefe3e4,0xc2da9569),LL(0x6e21cee3,0x0c7dcef3),LL(0xb14f0abf,0x98c3c9ea),LL(0xb64941e1,0x4819eee0),LL(0x433cfcf6,0xbfe77fa8),LL(0x2f7686dd,0x03c3b28a),LL(0xdbfd2334,0xd4d1d9c3),L_(0x00000008), +}, +/* digit=45 base_pwr=2^225 */ +{ + LL(0xc3b7107c,0xe8887082),LL(0xd218cc42,0xcd304c29),LL(0x7a96d44b,0xf1c4d847),LL(0xee7f483f,0x530d4bec),LL(0xc951d19c,0x4d6bf1fd),LL(0x71d2d68d,0xe03d009b),LL(0x4d3bd1df,0xc4553769),LL(0xdb4cb1a2,0x5f77a3c3),L_(0x000000a4), LL(0x5e86f0e3,0xcaf5fb4c),LL(0x715c3388,0xb2f14ba7),LL(0x81191db4,0x38426103),LL(0xf68a08e3,0xb75d25e0),LL(0xd342059c,0x92f767fc),LL(0x4dec2bd2,0x7a696b41),LL(0x16057d6a,0x8adfb670),LL(0x99c277b1,0x2be66359),L_(0x000001f4), + LL(0xd6f16b10,0x088e20bf),LL(0xb30d007e,0x859fdd39),LL(0xd4c40b6e,0xc9196072),LL(0x0a59d2a4,0x3c4f607d),LL(0xaf5b531a,0x5c546c30),LL(0xfdc40588,0xdc1d5df2),LL(0x2971b1ad,0xfb26f4df),LL(0x7cb15104,0x70c15bb6),L_(0x000000e6), LL(0x9ff74646,0xe3bee8f3),LL(0x96143e3d,0x560fc63e),LL(0x2e0395d9,0xfd9d7aab),LL(0x099cc808,0x422f153f),LL(0x4e3c3dca,0xefabb0d7),LL(0xed2c2c61,0xd736943a),LL(0x78f87c18,0x96d74e41),LL(0xb76afadf,0xfdd923d1),L_(0x000001a2), + LL(0x204ae17c,0xab5b4b2b),LL(0x780e0409,0xc4c863d0),LL(0xf73d00d9,0xd243a003),LL(0xd17b97ed,0xca6e6ef7),LL(0xd335ec2e,0x3f246d97),LL(0xd6c07def,0x0e2518ce),LL(0xb8b0595f,0xfdf728fd),LL(0xc1ccb10b,0x9df076dc),L_(0x000000d8), LL(0x869f012d,0x0a767d8e),LL(0xbe6fa9d5,0x1bbb0510),LL(0xdeeebbfe,0x8f4cdab7),LL(0x8332cdf9,0x75d651f0),LL(0x54687821,0xdd0fc83f),LL(0xa428610a,0x965277af),LL(0x38635dc4,0x9dff5c34),LL(0x0961df5b,0x5ea091fc),L_(0x000001a0), + LL(0xe4290556,0xea12a072),LL(0x762385fe,0xd082bac9),LL(0x1be16424,0x8697c433),LL(0xf06c7a59,0xa812dd21),LL(0x7de72b68,0x02f90069),LL(0xa2e56525,0x9acaec02),LL(0xa89c7efd,0x4f9120c4),LL(0xdf713a32,0xd965ba1b),L_(0x000001d6), LL(0x3367220e,0xc9641453),LL(0x9b048cab,0x29a9fac9),LL(0x0e9e757a,0xcb21c285),LL(0xde9cc170,0xbef96957),LL(0x7d017c03,0x0f3534a3),LL(0xa9c18cbf,0x5a627fd6),LL(0xed78ac58,0xb9c29da8),LL(0xef092aca,0x4f469ae4),L_(0x00000020), + LL(0x1cb1ec3d,0x669f05f8),LL(0x03d201d9,0x6bb4d70b),LL(0x898f998e,0x96c9a0f0),LL(0xe22c0440,0xfa56577c),LL(0x7b9dee1b,0x00502b66),LL(0x47777bc8,0x483cf935),LL(0xffb71b74,0x0b52d0b2),LL(0x39112d39,0x752546a7),L_(0x0000004f), LL(0x75c45aee,0x6b3c757b),LL(0xf1bef81e,0x7e3d1f7a),LL(0x21a74b7e,0x2bfe9dfd),LL(0xb0da2bc4,0x90c8361c),LL(0x0c5783ca,0xf50913fe),LL(0x5dd47036,0x34045f12),LL(0xd0f87837,0x89dc259f),LL(0x560578ab,0x22e987bd),L_(0x000000dd), + LL(0x1a521d30,0x8270aad1),LL(0xd7dd0f9e,0x5bd17ae9),LL(0x2341942e,0xc840cb6d),LL(0xdc090118,0x1c49ccd4),LL(0x60e02fa4,0x28d04bf0),LL(0x3570b020,0x1dc79e5a),LL(0xbd1e244b,0x5f5e1042),LL(0xeebf73ef,0xf6c77a24),L_(0x000001ba), LL(0x9d82fe22,0xf2a4d116),LL(0xfbb0437b,0x538ffa89),LL(0xac67b2fd,0x70b0e36d),LL(0xcd72f925,0x3b843326),LL(0x26042e6e,0xd58ef907),LL(0xa54711c2,0x40c00366),LL(0xa62c4885,0x33e0a1f1),LL(0x758c2c5e,0x508c7bec),L_(0x0000014e), + LL(0xf11f835f,0xd4f6e5fa),LL(0x4a99daab,0xbfa7d98f),LL(0x3f46a163,0xe712c72d),LL(0x869f8036,0x194ee078),LL(0x1283d5ab,0xe9c40094),LL(0xf4b53cf8,0x8eccbfb3),LL(0x865de4fb,0x27d82426),LL(0xc4399d77,0x5783591a),L_(0x00000093), LL(0xa59eaa03,0xb1a23e95),LL(0x99d72c23,0x48055f02),LL(0xf5f1e1c0,0x04a35336),LL(0x448cee4d,0xcb7b9a8c),LL(0x5862f10a,0xb9974831),LL(0x93b3f7d9,0x93c6f79e),LL(0xb09c629c,0x2a3760fa),LL(0x5d47957f,0x310ac801),L_(0x00000186), + LL(0xf060edf1,0x83a0baf3),LL(0x26de7078,0xeb8cb2a3),LL(0xf5a8d631,0xcfa95554),LL(0x6ba14fb5,0x9d950c23),LL(0xc53769e2,0xc3b6e6d4),LL(0x857d43ea,0x00e396b6),LL(0x28cc2c64,0x79b3b40a),LL(0x42ca52de,0x66e9ef01),L_(0x00000037), LL(0x4e536d43,0xbb696fec),LL(0x60fe980d,0x86879c73),LL(0x6a564e15,0x6796d473),LL(0x62e7ce9d,0xee75c812),LL(0xd4cced89,0xdd4f732c),LL(0xa3a75fe2,0x4a4d97d2),LL(0x62def6c0,0xfce046cc),LL(0x0bfe781a,0x9b2b5c59),L_(0x000000e8), + LL(0xecebff30,0x5ddecacd),LL(0x66805c77,0xc732fa10),LL(0x62e2a037,0x9f8318aa),LL(0xf535306d,0xcb188ede),LL(0xf3ae9b72,0xd4215242),LL(0x76515ea4,0x19ee9251),LL(0x04b30ff0,0xfdb4add0),LL(0xc2006bc5,0xc094d7f3),L_(0x0000008c), LL(0x08f431a5,0x9a41c273),LL(0x4ef9856a,0x29336f92),LL(0x89a37aaf,0xb4f719c9),LL(0xb6ae5b4c,0xcb8b83d7),LL(0x0b93351c,0x579d32e9),LL(0x95ac6c1d,0xed14de1a),LL(0xc9a3144d,0x59d39bac),LL(0x4809d3c0,0x021a5392),L_(0x00000076), + LL(0x46b4f926,0x31967f76),LL(0xedbec7a4,0x9540a500),LL(0x7d4cba0a,0x3664934a),LL(0x44eee05e,0xacaf15ab),LL(0x7e273dff,0x0d40978b),LL(0xb55d54b7,0x7a7b2c2d),LL(0xb2400231,0xcc267a21),LL(0x31dec63e,0x45427d34),L_(0x00000169), LL(0x9ea998cb,0x5e543e59),LL(0x22a0befc,0x0dee7b14),LL(0x1aa7e66e,0x052bb589),LL(0x51c3c220,0xe7133b54),LL(0x64b84789,0x86fee1bb),LL(0xacd3a9a6,0x6d321f88),LL(0xf3dee610,0x17435c7d),LL(0x64e0790f,0xea978193),L_(0x0000000a), + LL(0x824c77d2,0x1d5087de),LL(0x3c424cb9,0xce8e9c97),LL(0xb3b1c7ef,0x65ef8961),LL(0x36bdf7df,0x936db078),LL(0xe1a9152e,0x9ff36717),LL(0xbde711fb,0x1b59074d),LL(0xc2a17a75,0x9070200e),LL(0xa0657ad1,0xd3cc9c6b),L_(0x000001fb), LL(0xf471b9b4,0x63dca49d),LL(0x23fe450a,0x7aa88a47),LL(0x569744e7,0xbac5c025),LL(0x557519ba,0xc31b16cd),LL(0x90672c5b,0x7609de28),LL(0xeb879e6d,0x06a4f8d7),LL(0xa50f3b9e,0x45a7792d),LL(0x5ef4893d,0x5794fb38),L_(0x00000186), + LL(0xa8c609cd,0x3fc0a251),LL(0xe916d838,0x09bae516),LL(0xdc8442b3,0x70eda9b4),LL(0xdbbd2829,0xe66c09a8),LL(0x44077102,0x46f0d061),LL(0x0c8b9a97,0xe7c4e4e6),LL(0x96dd9b50,0xd93b4696),LL(0xf2abcbd4,0x2e694650),L_(0x00000152), LL(0xdcc67ed7,0x34c98a81),LL(0x1a864f2a,0xd79eca11),LL(0x46960243,0xaf4ad720),LL(0x02db8ea4,0xe8d429d1),LL(0xd9dae172,0x9cac2c01),LL(0xf704ba3a,0x052267c9),LL(0x0ee381b0,0x768c8223),LL(0xf95050b5,0x3558199f),L_(0x00000037), + LL(0xdbee340c,0x7596d517),LL(0x7a4d90d8,0xe88332d5),LL(0x27312835,0xadc174b8),LL(0xbf7deda3,0x9fa8589b),LL(0x30a04404,0x50ccb83e),LL(0x9ea9cbf7,0xb73afec5),LL(0x8a699ab5,0x0382ff44),LL(0x682b0e1c,0x12fbb9a1),L_(0x00000024), LL(0xa48bc771,0x219fed54),LL(0x20bed549,0xd7a8f53a),LL(0x9f897a30,0xccad31ab),LL(0xf4ab61be,0xe4253812),LL(0xd02eb63b,0x02c0bd98),LL(0xf369795f,0x940fa395),LL(0x6661d80d,0x4fc30a92),LL(0xdb54dfb5,0xf14f8652),L_(0x000001a5), + LL(0x7f4fad25,0x917eadce),LL(0x33c5c41f,0xfac932b3),LL(0x8175e0ba,0x0c6ce437),LL(0xd6df4caa,0x628f6bd2),LL(0xd8cef71f,0xbbd0808b),LL(0x1bf4e3d8,0x82a3b927),LL(0xf2573f56,0x18b7f7d3),LL(0x20a5d5d4,0x68ebb93b),L_(0x00000151), LL(0x6296df1a,0x511300a6),LL(0x48f698fe,0x0a451118),LL(0xe69d9b6b,0xbda90f27),LL(0x6b99560f,0x00ad6b22),LL(0x576639a4,0x70498a28),LL(0xe3fb0685,0x766cc9eb),LL(0x910ed9d6,0xa13d4e5f),LL(0x45079f4a,0xdd2c7f6f),L_(0x00000047), + LL(0xb3a0df18,0xcdee46b0),LL(0x0f7b0b8f,0x0dee6d65),LL(0xdc8df7a6,0xca19127a),LL(0x6d034f50,0xd6d74c77),LL(0xcd8b7301,0xd01a93fc),LL(0x7b8e12a8,0x77799926),LL(0xc0b3bdfe,0x157d532e),LL(0x5444b9cf,0x90f700d1),L_(0x000001f9), LL(0x2776ac60,0xbab425f8),LL(0x947e525b,0x66085567),LL(0x6d095956,0xbcb7adb8),LL(0x4d3075ac,0x99a0d6ce),LL(0xc684b9ba,0x0e134c5c),LL(0x4c65fec0,0x30477674),LL(0x5db48af9,0xf3744581),LL(0xfcc9963e,0xc6d87a4c),L_(0x000001c2), + LL(0xb51e39da,0x258dcf68),LL(0x1c50cc68,0xb1289a02),LL(0x54112229,0x48928ef6),LL(0xc73b83c7,0xf0df33f4),LL(0x6a0ebdbd,0x5f166393),LL(0x09883324,0xbdfac3bf),LL(0x21bceec6,0x871bcc9c),LL(0x64a15de9,0xa6e53bcb),L_(0x0000007a), LL(0x7efb2f27,0xce8d66f9),LL(0x6a48e2f0,0xc4f49e10),LL(0x0d05177f,0x04e720d6),LL(0x9e354273,0x746f841b),LL(0xe08c355b,0x7b7b7cd5),LL(0x81226157,0x586eb9a8),LL(0x38ce3838,0x8c03b21c),LL(0x02872c5a,0x40058aec),L_(0x00000197), +}, +/* digit=46 base_pwr=2^230 */ +{ + LL(0x8a4b6723,0xa08c6852),LL(0x79a5b3fd,0xc7195714),LL(0xd8054fed,0x7c611c75),LL(0x503c3580,0xe88e5dbf),LL(0xb37b9ea9,0x2b4c4521),LL(0x75f1c942,0x950eb17e),LL(0x79508472,0x238eca42),LL(0x845c91f9,0x0657bc0f),L_(0x000000fe), LL(0xa0c1e80e,0x978ae396),LL(0x14b242e1,0x2c0d00f9),LL(0xb47bf7bf,0xe944e43f),LL(0xb416e50a,0x906c3634),LL(0xe7c8d114,0x347f03a3),LL(0x65b00ad0,0xa6eba251),LL(0x53a14e26,0xf521e9dc),LL(0x11eb83f4,0x82020769),L_(0x00000151), + LL(0xcf0b0ab2,0x1ea0d8e5),LL(0x30d499c7,0x4b6204df),LL(0x8f748b9e,0xc4dd3f54),LL(0xba95c754,0xc443876e),LL(0xd7ad6cc9,0x08e10896),LL(0x544e9e2d,0xa9428b4a),LL(0x1a2d3e1e,0xb6fe7189),LL(0x05c7660e,0x77bc86b9),L_(0x0000013a), LL(0xb366e641,0x77106f8f),LL(0x42b36e28,0x2d9d82f5),LL(0x63743c2e,0xaf2b4eca),LL(0xc887146a,0xb28d08cf),LL(0x4ed669e0,0xc7391371),LL(0xda28c885,0x14ee1f56),LL(0x84424da9,0x42d4479d),LL(0x33646227,0x4a2f215e),L_(0x00000082), + LL(0x12e89265,0x1eeb47d8),LL(0xcbb7e582,0x5b702942),LL(0x0f041cf4,0x64d44f59),LL(0x50d39295,0x5d20f47c),LL(0x0a8b51ba,0x15a076aa),LL(0x79ae768a,0xa2141ba3),LL(0xcf638bb5,0xe3c47d36),LL(0x582394d5,0x3311139a),L_(0x00000047), LL(0xde6f722f,0x7bcb9faa),LL(0xc544456b,0x7edbbe33),LL(0xc58f127c,0x178e1289),LL(0xcd6856a6,0x29c72942),LL(0x7d0ce889,0xdca59772),LL(0x951589f5,0x6908ef3f),LL(0x4f00ce63,0x110a84b5),LL(0xc1a89443,0x72833218),L_(0x0000003c), + LL(0x1fbd25b5,0xc3abc5bd),LL(0xa3138347,0x78a11d29),LL(0xf2283223,0xadac6d62),LL(0x4af4ece3,0x72b0dc7b),LL(0xf1c75e43,0xa7308e28),LL(0x99139560,0x0ea7127d),LL(0x9cb3c31d,0xee0172da),LL(0xc69386a7,0xd4d20c36),L_(0x000001db), LL(0x293be2a2,0x82933139),LL(0xaf8d3077,0x41cdee07),LL(0x4118b415,0xd6d0895f),LL(0xb3a9502a,0x242767b9),LL(0x404e1d44,0x3924f383),LL(0xe7a91a84,0x3c5c40dc),LL(0x0f30db5a,0x2d443e9e),LL(0x38df60b6,0xc16acc53),L_(0x00000195), + LL(0x57eacd01,0x8b237781),LL(0x47bc0a58,0x21bf6b08),LL(0x72f947bb,0x5c6b0c6d),LL(0x0c58bea7,0xc78326d2),LL(0x8a6feb8c,0xf3157ca1),LL(0xe147ad97,0x4be255e4),LL(0xa6917b35,0x7006cd50),LL(0x1ceacc56,0x7d62c8eb),L_(0x0000000d), LL(0xc4957a8d,0x5e969282),LL(0x3e3eb59d,0x1216fdc0),LL(0x2f2b0b06,0x51a13162),LL(0x5b88e211,0xff4a6d02),LL(0x6b68e6ae,0xc7c80a3f),LL(0xa7fca940,0x2128145a),LL(0x3a205f85,0x459b75e6),LL(0x48874b7e,0xef227f89),L_(0x0000019c), + LL(0xf1c2d135,0x363e7de4),LL(0xede249f0,0x2374169d),LL(0x92b91c52,0x34ca05e7),LL(0x42f9c460,0xe96d13d2),LL(0xb8dc141b,0x9c04a0eb),LL(0x11349888,0x86f45f6f),LL(0xa3c9d21f,0x34d9dd7d),LL(0x6359abc3,0xa35e3eae),L_(0x00000172), LL(0x906d8bdb,0x82617ad1),LL(0x5f4fb81e,0xd70d26c6),LL(0x68367c91,0xe835f648),LL(0x4d712331,0x5dda13b7),LL(0x06ca4385,0x97f662ae),LL(0xcbeb485a,0x211b18b3),LL(0xfe5f6ad4,0x178f31f1),LL(0xbfc76ef2,0x88d953e4),L_(0x000001a9), + LL(0xab40e045,0xcdbdf2d4),LL(0xbc7f59a3,0x25655817),LL(0x7d6adfae,0x5b05af23),LL(0x3b9e8819,0x1f7f265d),LL(0x4d41011d,0xa02a10b1),LL(0xc4403b75,0x4598d47a),LL(0x21678b80,0x72bcfd2d),LL(0x0a91ddc2,0x53e0dc1a),L_(0x0000003b), LL(0x5f9a8c36,0x38d6c310),LL(0x22226823,0x4b065228),LL(0xdf4b4ac7,0x99e6867b),LL(0x1562f4fb,0xc9c2e12f),LL(0xe5bf9e6b,0xef6dd2fe),LL(0xa02f573e,0x99a366b2),LL(0xf15b202c,0x46de40b9),LL(0x8fa6fe0d,0x0fbe1e76),L_(0x0000011f), + LL(0x95d88264,0x5a265bdb),LL(0x771ed60f,0x717063ef),LL(0x0dd813de,0xd974ca00),LL(0x4d348196,0x9d26915a),LL(0x3def1612,0xfcdfb352),LL(0x1ecabcbb,0x7290d698),LL(0x44e08c75,0x5c0c5d24),LL(0x65a063c6,0x1673491c),L_(0x0000000e), LL(0x751941d0,0xa5845e0e),LL(0x90c04cc0,0xc51c09cc),LL(0xf11f3a2b,0x536ce8af),LL(0x3944ca09,0xd614a230),LL(0xe5cb9d10,0x4cf73a00),LL(0x11f36267,0x39b52629),LL(0xe681d436,0x27379f49),LL(0x7634bb09,0x6f9a6ea0),L_(0x00000193), + LL(0x3b3162c4,0xf5ba50b2),LL(0x27a53ca3,0x5b2f88a0),LL(0x4d9f7313,0xff015375),LL(0x4100d075,0xefc66c49),LL(0xaa939a6b,0x25334ce7),LL(0x4d836d2e,0x0ff09c49),LL(0x9aa3f59b,0xce13f150),LL(0x6bd297a9,0x80466de2),L_(0x00000106), LL(0x3c8984bd,0x8be1ce28),LL(0x7a30734a,0x83fbedb8),LL(0x56e66999,0xc37e9e22),LL(0xeb69a4db,0x8b3de542),LL(0x15192947,0x4a7280d3),LL(0x67515315,0x5c05b359),LL(0x0b9a8604,0x5ec92a80),LL(0x973deb80,0x04619c68),L_(0x00000027), + LL(0x1256d99a,0xa13ac401),LL(0x9e017800,0xed3810cb),LL(0x84b7702c,0xb6d9eff1),LL(0xcdc98f94,0xf4a42e06),LL(0xa3cf6c58,0x658f00c3),LL(0x17b79fe0,0x4db4bd0a),LL(0xc0cd6ebd,0x79f4a662),LL(0xb6716eec,0x94db6c69),L_(0x0000014d), LL(0x6f9c5845,0x5d9256ec),LL(0x540f768a,0xe910b5c3),LL(0x4e7e7b8d,0x3c907d51),LL(0xf387c3d5,0xeecfe723),LL(0x213d5d27,0xd78500a0),LL(0xaa244815,0xb117a5b8),LL(0x7049f488,0xfb72f9ed),LL(0x3651c83c,0x24f438bb),L_(0x00000079), + LL(0xed918816,0x14555dbb),LL(0x178d8d22,0x4a72d14b),LL(0x1ebffe3d,0xa3a172eb),LL(0x476c82fd,0x4c45f724),LL(0x700c837a,0xd80825d0),LL(0x16888e37,0x104ac32f),LL(0xfaf93105,0xed3ecf8b),LL(0x503fb78e,0xcfbd4bb3),L_(0x00000067), LL(0x3c014136,0x4a1d6c14),LL(0x3ae0a56f,0xa38d42d7),LL(0x765e846a,0xdc241d4a),LL(0x087fcd52,0x0a7a4e2e),LL(0xf088c4c3,0x81043b53),LL(0x0b74dc81,0x030c4d6d),LL(0xf465c63a,0xb63f1e30),LL(0xa7aa0a71,0x1a64238f),L_(0x00000006), + LL(0xe2aea3b4,0x22e76e20),LL(0x6eca1ede,0x7e03afde),LL(0x08d1c44e,0x4edf70d6),LL(0x88d58544,0xd8ec390b),LL(0x474d298c,0xc4c3f675),LL(0x9ce21146,0x3274110e),LL(0xb12ae7e7,0xce9cf1f4),LL(0x67e9f4b4,0x8102a4fa),L_(0x000000ee), LL(0x102ef931,0x3efdaff6),LL(0x7a8b94cb,0x59bd3104),LL(0x4b8b1235,0x25afad7c),LL(0x24884827,0xdd939da0),LL(0x43b462ba,0x45cb99ab),LL(0x6a4b9f89,0x0f6b65c6),LL(0xbddb4b1c,0x39e97dd0),LL(0x5a1f7976,0xef2919d7),L_(0x00000117), + LL(0xcf01fbe1,0xc5bd929e),LL(0x47c26018,0x2bbeb492),LL(0x050e6a5c,0x04741b6f),LL(0x0f665d42,0x4c0e1ab0),LL(0x15137e0c,0x3ee524ae),LL(0x88980ffb,0x86e225d0),LL(0xa5de3190,0xde9d18c0),LL(0xe6bcf986,0xc6d1d2db),L_(0x0000009d), LL(0xc1e09545,0xa8ab4958),LL(0x7b416dfc,0x05e9411d),LL(0xb5a5de50,0x916238cc),LL(0x7da6c853,0x0e933ce2),LL(0xcff26d33,0x7fab03f6),LL(0xc2410d1b,0x70a12092),LL(0x130ded07,0xa7ff89e1),LL(0xf34872a0,0xde280eea),L_(0x000001cb), + LL(0x6107960e,0x08dca3fc),LL(0x9f978b8d,0x2367f8bf),LL(0x41ab4ac8,0xbfb304bc),LL(0xc14127da,0xb2161643),LL(0x89ee83de,0x91433f74),LL(0x76ad8bee,0x3abcb595),LL(0xa74d6d8b,0xa8b1935f),LL(0x314e698a,0x23ecc61e),L_(0x000000e7), LL(0xbd5f1508,0x1fda31cb),LL(0x4401991d,0x67b33d1f),LL(0x5dcaee66,0x80ef50d3),LL(0x58f1d026,0xc6160acf),LL(0x00a13e57,0x3be539f3),LL(0x88dd96e9,0xa5a0a5ae),LL(0x97419f0c,0xcf6a8f10),LL(0x44b2ca4c,0x1b9f2c18),L_(0x000001dc), + LL(0x5c77a891,0x745da1cc),LL(0xa4b913df,0xa1006271),LL(0xec1779e4,0xb3fe3fca),LL(0xcceebf8a,0x7b0d3f86),LL(0x82fd16d4,0xeb20fdbf),LL(0x4e29270b,0x450edccc),LL(0xa783c064,0x2c637dd3),LL(0x7337d5ce,0x9b0b50ee),L_(0x00000005), LL(0x819b276f,0x71e05cf6),LL(0x45bd1439,0x748c488f),LL(0x280a8add,0xbb099ca8),LL(0xe8e6e69d,0x19429d88),LL(0x4f4c80b6,0x2d2698e8),LL(0x0e4ab44b,0x1eb3fc49),LL(0xcd46fe76,0x02d1a2ca),LL(0x15543eb8,0xacd0b316),L_(0x00000129), + LL(0x700ec0ac,0x13ee9845),LL(0xe55b42fb,0xe5ad047b),LL(0x45bc6993,0xe8f73a08),LL(0xee41f2ae,0x1ac680ff),LL(0xd3c83204,0x740b12fc),LL(0x5e36bb4a,0x21ebc164),LL(0x35a45c95,0xb8dfcc77),LL(0x92b0fbda,0xa1f288ef),L_(0x00000153), LL(0xb5805ed4,0x829d754d),LL(0x6c810584,0x36cc488a),LL(0xc9632468,0x39f60f1e),LL(0xf2ebc30d,0x390502c0),LL(0xde960758,0x6d1feec9),LL(0x63adf462,0xf944bca3),LL(0xdeea2824,0x9c375dbf),LL(0xa887d095,0xbc1d650b),L_(0x000001c8), +}, +/* digit=47 base_pwr=2^235 */ +{ + LL(0x2fcf8b0a,0x07415442),LL(0xe5dadc06,0x07a17d21),LL(0xc237ee85,0x83c01e78),LL(0x08f5fd21,0xada49ad1),LL(0x998eec2d,0x3e35f765),LL(0x5a121c30,0x1f207544),LL(0xf5fdddb9,0x537426d8),LL(0x1dd7a92c,0x6cdbd6af),L_(0x000001c6), LL(0x9ac31da2,0x22850494),LL(0xd3189e72,0x7646877e),LL(0xe04fe426,0xee0cbac9),LL(0xf8802494,0xd9fdf793),LL(0xa975ea85,0x0c9c6045),LL(0x18cf38fb,0x02b88cb1),LL(0xa12c8778,0x9757e39e),LL(0x8e571e06,0xb126d254),L_(0x000001ec), + LL(0xd91f89ee,0xd77e7f7e),LL(0x2cf0f860,0xe9a24986),LL(0x705ade19,0xa4b26963),LL(0x0f929eaf,0xf6f18b5d),LL(0x1a12f7ee,0x0ff1861d),LL(0x4ef59d59,0x6b67d295),LL(0x18bcbd0d,0x70540dd4),LL(0x83b51e51,0x2670896e),L_(0x000001b3), LL(0x0b0a67de,0x090cb0d5),LL(0x89c8fd0f,0xb8b3eeba),LL(0x289c0d96,0xa8a26dc5),LL(0xe6dd7431,0xafde18e0),LL(0xf0a660c8,0x2cc76374),LL(0x397bcfa0,0x7654494d),LL(0xd958a15b,0x24476b8f),LL(0x53a314e3,0x67dea97d),L_(0x000000de), + LL(0x74ca8553,0x099db6d0),LL(0x79fa68c5,0x4c5dd75b),LL(0x75880817,0xccf37a0a),LL(0x92dc167c,0xf900a103),LL(0xc92d1684,0x8386aa09),LL(0xfba8f79b,0x53d25b65),LL(0xb1822202,0x183fbc1b),LL(0xfccc5a8e,0x21f550ce),L_(0x0000013b), LL(0xf2f85858,0x4925f513),LL(0x1b5dd268,0x3efe01ed),LL(0x65fdd3e5,0x47d317de),LL(0x4621cca2,0xdf49fdd8),LL(0x60e7bc31,0x66f9ab90),LL(0xe320caa6,0x2ca76c6f),LL(0x41361bf3,0x602d539d),LL(0x267e9ebe,0xf83bf8f0),L_(0x00000140), + LL(0x925d2d91,0xd471ad5a),LL(0xe6ad36d9,0x18a88bdc),LL(0x599734aa,0xb20b90be),LL(0xaa6c0fb7,0x33c88799),LL(0xceda14b2,0x7dad41f1),LL(0x3bb924ee,0xb7c643d1),LL(0xa63db6fc,0xcfe7d84e),LL(0x46de4666,0x71afb3a8),L_(0x00000072), LL(0xebe6128e,0x4a9235f4),LL(0x780d1146,0x2db34ea2),LL(0xb3ecdbc0,0x31fad3e8),LL(0xcdbe1207,0x7ab5ebf8),LL(0xdf431809,0xbf1d4990),LL(0x0f8e79eb,0xe93c1583),LL(0xfba03ee7,0xefc40a1c),LL(0xf76de664,0xd188b68d),L_(0x00000103), + LL(0xa3f4bcf5,0xf02dfbba),LL(0xf379ed8d,0x7857a0d6),LL(0x6c580cb5,0x7b4a59a6),LL(0x6d6738fc,0x1654de01),LL(0x7c102c44,0x8d6ededc),LL(0xb8a37f11,0xe92baa08),LL(0x6ecce15f,0x1b1b1c96),LL(0xf542a7b0,0x118752af),L_(0x00000169), LL(0xcf9be3be,0x02fd3092),LL(0x22a60f7a,0xd4c8209a),LL(0xdeeb8950,0x8636e6d7),LL(0x29511c76,0x09037c4f),LL(0x419f1652,0x4196d645),LL(0x935dc02e,0xeaef1b05),LL(0xcec8a6ec,0x2f5c9bf8),LL(0xb432cd40,0x375285bc),L_(0x00000116), + LL(0x7f267eeb,0x2a1279f1),LL(0xf11c0e49,0xc76f729a),LL(0x92b8f2a3,0x5437339e),LL(0xe00c8ca3,0x41f96b14),LL(0x1dbcc016,0x449dde57),LL(0x5fa3755e,0x0a0df11e),LL(0x42dc646a,0x5d317707),LL(0x3610d1c6,0x149d76f5),L_(0x0000008b), LL(0x76e5b808,0xc99fe831),LL(0x61f75499,0x19eef1a0),LL(0xd4c21a60,0xcc67deb8),LL(0x192fdd7e,0xec37ce33),LL(0x13250ef2,0xf6ed9344),LL(0xac4baff8,0x8d1e9777),LL(0x0d67d5c5,0x7183407f),LL(0xd1b52871,0x7887079f),L_(0x000000aa), + LL(0x64430a84,0x5637bc1f),LL(0xc9bb1131,0x102f522c),LL(0x38d318b3,0xb1dfff39),LL(0x0967edf8,0xc54708cf),LL(0xff7bf052,0xb7dea363),LL(0x5d9deef1,0x44846b8a),LL(0x5e3fdc0e,0x2ccf785a),LL(0xdf1138dc,0x16844b0a),L_(0x0000000f), LL(0x3b1beaf4,0x5076af3c),LL(0xac3bbbf4,0xdda27ca5),LL(0xdc76f92a,0x7c5f4d64),LL(0x67fb8aff,0x98258450),LL(0xfa7eea13,0x183f4c0d),LL(0xc38dcddf,0xc7ca9f82),LL(0x0789054d,0xa0ad28e2),LL(0xf484ac51,0x2b54cf18),L_(0x000001ce), + LL(0x0d9d6152,0x28f43810),LL(0x4ecd5aff,0x0a2146e6),LL(0x7e0c0df1,0x0d19ad5b),LL(0x6775181b,0x151d7c3f),LL(0x43c7a1b8,0x38a3ef9f),LL(0x2dd235c3,0x1fdd8171),LL(0x1f2597ee,0x4734726a),LL(0x083971de,0x92167cb3),L_(0x00000173), LL(0xfac04120,0x85ff590d),LL(0x952b767c,0x8b72aaea),LL(0x843ae9c1,0x004265e7),LL(0x307d7542,0x395a8932),LL(0xc2dea503,0xc7f73a8f),LL(0x3399e1ee,0xb86c7eb5),LL(0x2926b2c4,0x0d21cf86),LL(0x47a4f082,0x37deb193),L_(0x0000001b), + LL(0x685e8e4b,0xd88c27c3),LL(0x0240c0b1,0xeaa149a2),LL(0x79b4d2d7,0x18bd3d71),LL(0x368319a1,0xdb5d6ca6),LL(0x44a8c42f,0x587ba79d),LL(0x77c41337,0xdb57f6f2),LL(0xea1f7c4f,0x8ae39899),LL(0x77750728,0x8ec08a85),L_(0x00000083), LL(0x22355d19,0xa2946cf3),LL(0x598a6616,0x4d6a2c7a),LL(0x4bb8cec6,0xfe918a85),LL(0x94e93f01,0x833d8970),LL(0x64d5ba4a,0xa972b798),LL(0x05a95b5c,0x636ab756),LL(0xca11412a,0x2b50cab7),LL(0xeca1dc55,0x886d7097),L_(0x00000181), + LL(0xcb4f7b38,0x1a4787e2),LL(0xea069acc,0xa5590739),LL(0xa99d3f7a,0x601be73f),LL(0xcf6cca23,0xaa83e6e7),LL(0x3960cc53,0x5920b117),LL(0x20e0b86b,0xa7ac9dbe),LL(0x6b3ef99f,0xf468bb6f),LL(0x60ac6c56,0xae52f640),L_(0x00000099), LL(0xdc626c18,0xb7dd82a1),LL(0x008f892e,0x142f8425),LL(0x705cbf37,0x01a4b241),LL(0x4c8483d1,0x275951c0),LL(0x82075cfd,0xa7dae45a),LL(0xa9d9f282,0x63c98e81),LL(0x30bf2902,0xefc331b1),LL(0xc5f55add,0x2fdd6c95),L_(0x0000013f), + LL(0x4a6e860f,0xaa1272c0),LL(0x85afdbd7,0x53123c17),LL(0x796e466f,0x327374bf),LL(0x19701fb1,0x8c2b076b),LL(0x755659d8,0x8fee24c8),LL(0xc6001497,0x45d95463),LL(0x144b9a21,0x98a62be8),LL(0x9e1d51ac,0xd46ea2e1),L_(0x00000017), LL(0xff126e8b,0x87dd796d),LL(0x5e933815,0xad6f752f),LL(0x0f756584,0xa0d6329c),LL(0xc8afd335,0xef95a2f8),LL(0xa429ff1c,0xe9d51af6),LL(0x06c04336,0xbb9ac481),LL(0x0e389129,0x80d2d2dd),LL(0xb75b239e,0x128834d2),L_(0x000001f5), + LL(0x04ac9a1c,0x9fb461cd),LL(0x2abd08e0,0x7bca251b),LL(0xfa6e8384,0xf5d98ad8),LL(0x7e953f04,0xca3dcc08),LL(0xfd9f6c57,0x679c5992),LL(0xfac8f179,0xc9cf93cd),LL(0x0e29622c,0x050d1a32),LL(0x703c650a,0xc7848580),L_(0x0000006d), LL(0xc1188e58,0x60a160b5),LL(0x6d729218,0xbe9a3f42),LL(0x311911d3,0x864f8747),LL(0x8a79eb1a,0xe2ff5eb7),LL(0x66b881de,0xef83107b),LL(0x8784fc87,0xd139997a),LL(0xd894e5ee,0x9a80e84f),LL(0x1a2f4197,0x2ffb83aa),L_(0x000001a7), + LL(0x6619c168,0xbefe84df),LL(0x7fe74db9,0x7efd2da4),LL(0xc983adbf,0x0bbc28a9),LL(0x6535bd9b,0x169b3680),LL(0xcb39ad32,0xd2112121),LL(0x3f60a9f2,0xa91386ce),LL(0x3a1b138f,0xb1ef230d),LL(0x36ea6d68,0x8198043e),L_(0x00000105), LL(0x1ef3660b,0x58bee126),LL(0x3a1e13eb,0x84fe2098),LL(0x1044e049,0xd3d31ea9),LL(0x6e975e84,0xbdc58274),LL(0xcbeca3ca,0x4c27aa9f),LL(0xe9036bd4,0xcc717651),LL(0x982f94af,0x20d99920),LL(0x0268a0da,0x96c3d4be),L_(0x0000013e), + LL(0x8e317075,0x01712fc5),LL(0x00607c95,0x10e30ced),LL(0x77b023a2,0x53b465e6),LL(0x620a31c6,0xb10af467),LL(0xa8ea62e5,0x48d08ca5),LL(0x31ddd990,0x65af8778),LL(0x5be8899d,0x2b29f5ab),LL(0x21d38a08,0x7aa79283),L_(0x000000c1), LL(0xd56bb423,0xba4bc4d5),LL(0x28b7afb8,0xa4ef2e4b),LL(0xc314822c,0xae9b41f7),LL(0x882d9a51,0xf2b327a2),LL(0x1e6c2280,0x7ce965ab),LL(0xa49d1969,0x5708dae4),LL(0xea346d97,0xcff1175e),LL(0x572d2fbc,0x8dc700e9),L_(0x000000f1), + LL(0x971c75d0,0x3dd39efc),LL(0x11aab543,0xe47dc537),LL(0xf6290b96,0x38ce4cde),LL(0xe7a164cd,0xd9dcb339),LL(0xebee41ba,0xbc0e8d41),LL(0x18ca0dd5,0xd2a8a073),LL(0x92054e4f,0x366d13cf),LL(0x188fdcbb,0xa61f6fe4),L_(0x000000d5), LL(0x587afc9e,0xf03b4067),LL(0x4ac1fdd7,0x43baf1b7),LL(0x900c0863,0x35bd9902),LL(0xe71367f8,0x3f6d6815),LL(0xfd613341,0xc12cdb6c),LL(0xa8569f79,0x6e58f3c3),LL(0xd9ca8493,0xd420b011),LL(0x6d1ebe67,0x18a83579),L_(0x0000014d), + LL(0xbd7d4bad,0x4af0bab5),LL(0xade61274,0xcd1e04b0),LL(0xe21c3a53,0x4ec80a7f),LL(0x4b987124,0x28f48386),LL(0x23f6fd14,0x63b180bf),LL(0xe2b4889d,0xacf748b0),LL(0x2996dea6,0xf3a06107),LL(0x7cdec9d5,0x7248e660),L_(0x00000130), LL(0xc2a9858e,0x2d38ce4f),LL(0xc709b1fa,0x87521e69),LL(0xf5996fd9,0x4d05bae8),LL(0x131ac99f,0xc3bad75e),LL(0x62578fe3,0xc9d13920),LL(0x3ada2279,0xb6fbac90),LL(0x696da364,0x6c292885),LL(0xd02ce135,0x5f814ddb),L_(0x00000048), +}, +/* digit=48 base_pwr=2^240 */ +{ + LL(0x3d1074e0,0xff10eeb7),LL(0xe087c1a0,0xc77549f2),LL(0x5e2e0837,0x74d6808a),LL(0x48c7156c,0x11f82ce9),LL(0xbc13bf7c,0x72ee287e),LL(0x06f6a514,0x28c4e6f9),LL(0x165038cb,0x320fef0b),LL(0xb6f1c9d9,0x369852d8),L_(0x000001dc), LL(0x57d310ce,0xadd6b6df),LL(0xcd825c08,0xdcd4cb28),LL(0x48bebf85,0x644e1cbe),LL(0xf1d5aa6a,0xbebbd351),LL(0x908ba85c,0x4d8a2aa1),LL(0x518b1bc8,0x343d2d77),LL(0x29b988ed,0x7ea90982),LL(0x940fc8d0,0xeb433a9f),L_(0x00000059), + LL(0x8f96ebc1,0x54a67c95),LL(0x984cd0ed,0xef67809a),LL(0x8dd8453d,0xde0abb72),LL(0x4fe5f363,0x4b73609f),LL(0x8e4fc461,0xaab1b83c),LL(0xb989fee4,0x06d2158c),LL(0xfe56f7d6,0x52096597),LL(0x70734a0c,0x4bb17572),L_(0x000000e7), LL(0x094ef503,0x32b2f44d),LL(0xebd6d9b7,0x2c29898d),LL(0xe0ef3ff7,0xe5d5ffd4),LL(0x30b99ae2,0xe1c94a38),LL(0x2dd5fca5,0x15b084de),LL(0x6d08e970,0xe94504be),LL(0x90fe0fe0,0xdb79eaed),LL(0xafa2897d,0x1b99b51e),L_(0x00000143), + LL(0x45dd470c,0x907208e4),LL(0x551b38ea,0x1157ba3e),LL(0x1b72b693,0xcf9f94c5),LL(0x83c616c9,0xaf1c59b1),LL(0x2fe84fca,0x7ed67f1d),LL(0xf1bd77c9,0x1d1e1a09),LL(0x51550daa,0xbac2f477),LL(0x58d345e7,0x84ab938d),L_(0x000001dc), LL(0xa5d95b5f,0xa404c99b),LL(0x29f414da,0x5a1bac7d),LL(0x81c9d673,0x56fda469),LL(0x2c1bc499,0x66bdcd65),LL(0xfe505f2a,0x9783eab5),LL(0x92378106,0x6f9996ab),LL(0xa7330e63,0xa6238170),LL(0xfa70e33d,0x9a7f0670),L_(0x0000010a), + LL(0xfee86a58,0xf15be04b),LL(0x32b87572,0xe35f663e),LL(0x94b48632,0x165f5c52),LL(0x62fb4267,0x7d3b9413),LL(0x407dadeb,0x189d86c3),LL(0x5689012e,0x63e5f780),LL(0x61c7907d,0xa2b7e335),LL(0xc0dc085f,0xd9e3c3c0),L_(0x00000070), LL(0xa9cbe4f2,0x0cfb081d),LL(0x5ac2d2f3,0x11fe0a52),LL(0x32662679,0x8fcbbd46),LL(0x3e344c2d,0xb1ff4122),LL(0xa0a08757,0xa48229e9),LL(0x6f7101c0,0x56b8c92c),LL(0x4af0e804,0xbb9a086c),LL(0xc21360d8,0xed22dd74),L_(0x00000169), + LL(0x5a00ce71,0xb34ef3a4),LL(0xac5c1fed,0x800620f8),LL(0xba6237c0,0x97dc7c79),LL(0xff56f449,0x5563d588),LL(0xe8ab9474,0xb2b00a9a),LL(0xb8d1df21,0x6a286295),LL(0x4cf9b378,0x00426dbb),LL(0x30b70043,0xa6195ec8),L_(0x000000c9), LL(0x1fa97f98,0x5a4e4b25),LL(0xd3a62f9d,0x61b4d385),LL(0xad987701,0xb90b4cda),LL(0x41727a73,0x84f90823),LL(0xec4abf7e,0xd814c4a3),LL(0xed3df0de,0xc453671b),LL(0x0c8945f2,0xdda88869),LL(0x94087855,0x25b6219a),L_(0x000001ef), + LL(0x0ce5f28e,0x705e5e78),LL(0x3af745ca,0x93ca952d),LL(0x7351cb28,0xfedeccdc),LL(0xc12e9837,0x7e7bfbbf),LL(0xd14b9356,0x47999f34),LL(0xc6295462,0x3f729887),LL(0x5692b0da,0xc96edd28),LL(0x812e383c,0x60ea4fc9),L_(0x000000ff), LL(0xef1bc941,0x70199860),LL(0x493b99ba,0x2d090c33),LL(0x279e0c37,0xbe1503ff),LL(0x3fbe286b,0x80f6465c),LL(0x06d81c3e,0x0a9257bf),LL(0x2a448d3a,0x402ee72a),LL(0x6a5669fe,0xe592b91c),LL(0x5315497c,0x5e220ad9),L_(0x000001ae), + LL(0x93b397c9,0x3318572b),LL(0xa3327857,0x0e667f17),LL(0x7af5bff7,0x2e60b913),LL(0xfb65a96b,0x9e25ef17),LL(0x9188dfee,0xdd117ab2),LL(0x830a2c9e,0x5472d03d),LL(0x063aa4e1,0xd8512ca1),LL(0x9593f42b,0x67ebcadf),L_(0x00000181), LL(0xa143be47,0xbbccf7b7),LL(0x40a8123d,0x80d4a4b1),LL(0x8562ea6a,0xe8424774),LL(0x53e144c7,0xa39d882f),LL(0x49ac8b6f,0xb451fe15),LL(0xffe4b1ea,0xc4538f34),LL(0x0527aede,0x1076cc29),LL(0xbcd8efdc,0x4f2ddd83),L_(0x000000b7), + LL(0x670c5c5b,0x046d9801),LL(0xf7b5e720,0x0601ff33),LL(0xa07084cc,0x900e6b0a),LL(0x791af83b,0xd2391e07),LL(0x2dd856fd,0x0430654d),LL(0x63408d88,0x82606889),LL(0xc176e8bf,0x2fa4b443),LL(0x18c663e6,0x5deafc4e),L_(0x0000019b), LL(0x48d8ae04,0x950e8cec),LL(0x11b8065c,0x1316d53d),LL(0x3a5ccc8c,0xfeecd686),LL(0xed41a668,0xbb41648c),LL(0xf76ba771,0x9a4bb4b6),LL(0xa9b2b49a,0x370974d5),LL(0xf9f130ae,0x200485de),LL(0x4c25f49d,0x4b31a1b8),L_(0x00000083), + LL(0x24ffe291,0x23a38bed),LL(0x41132058,0x4bf85483),LL(0xba359818,0xbdd23ab5),LL(0x3571fdd9,0xd8f1fbf5),LL(0x740422c0,0x46b3d29b),LL(0xdeca158b,0xf7c74726),LL(0x69a6765b,0xe058b8b2),LL(0x3a247a5b,0x9d49dfe5),L_(0x00000125), LL(0x49bfcbe2,0x37b93476),LL(0x3090fa48,0x219565ef),LL(0x30eb0e12,0x996b7d11),LL(0xe2ef23ae,0x935017a2),LL(0xfedd570a,0x59810960),LL(0x510b5963,0xf4feef53),LL(0x4eeb57ef,0xa6aed7bc),LL(0x500fc7dc,0x4dabb423),L_(0x0000009c), + LL(0x502210f9,0x2f6be4ac),LL(0x36ffcfaf,0x97a1a521),LL(0x874cac2d,0xd4b6d8f6),LL(0x4e03da3a,0xebd4b9c9),LL(0x058f53fa,0x05bdde0c),LL(0x54da0035,0xea492dfd),LL(0xf1d437ba,0x083ae453),LL(0x837cc36f,0x84792d73),L_(0x000001a7), LL(0x7ab16b91,0x08283456),LL(0xe627feae,0x5b031ea9),LL(0x5ab7febf,0xbc9c1a1e),LL(0xdf744ce8,0x0a851efb),LL(0x944a28d0,0x92d1258a),LL(0xa2c52c0a,0x8db3d01a),LL(0x3272efed,0xc3e55528),LL(0xdd38ae95,0x67075092),L_(0x000000ca), + LL(0x6c8b9288,0x201a8b72),LL(0x55d5044c,0xcd7dd04a),LL(0x5fefdd8a,0xf135da4f),LL(0x326e5c4a,0x93ab679e),LL(0x0ebd3dab,0x9e1c63de),LL(0x3da9430d,0xb6139a96),LL(0x1c3e1a0b,0x0bbe99d7),LL(0x31ad9e61,0xb4c624a5),L_(0x00000041), LL(0xd69aa80c,0x312fa8a7),LL(0x10699493,0xb830105a),LL(0x432ef74c,0x427ed742),LL(0x21457fc6,0x32a8f306),LL(0xed7ee077,0xde91b340),LL(0x7b2b2143,0x97655415),LL(0x2c36d1f0,0x3e70f7df),LL(0x8cb68be3,0x10666362),L_(0x000001a0), + LL(0x1c2c5d0a,0xbdac0199),LL(0xf48304d4,0x58ea0344),LL(0x7badcb3c,0x61cc3620),LL(0x7fa76693,0xee100174),LL(0xcbbd3041,0xb845bb4b),LL(0x2ab9fe8e,0xbc927037),LL(0x7e87d3ef,0x7c6a4dcf),LL(0x87fd392a,0x4efc311a),L_(0x0000016b), LL(0xf99d4987,0x3518a503),LL(0x093c711b,0x5ed52438),LL(0x8de87657,0x1283fea5),LL(0xa57d89d4,0x1d760642),LL(0x37377224,0xbf397e80),LL(0x0c073eb0,0x2d948da0),LL(0x19066ff6,0x02fb3665),LL(0x6f15cfd1,0xfb0a4e90),L_(0x000001b3), + LL(0x0703434d,0x36c93ab6),LL(0x08a36b13,0x54182f0b),LL(0x6018b27b,0x1eb09d80),LL(0x52b2eff5,0x5cab8c14),LL(0x54baf54d,0x09c5f439),LL(0x9128c26e,0xa0c91a3e),LL(0x3c462ce0,0xc8a5d523),LL(0x0dcb5998,0x2d1570bd),L_(0x00000066), LL(0x168afb3c,0x28b7e59a),LL(0x51e02e1e,0x7db3beb7),LL(0x95d9b53f,0xd954033d),LL(0x4280b408,0xfca4117d),LL(0xd87fffcb,0xa8a2c41d),LL(0xedfe3235,0xa4e146b0),LL(0xc9ab6206,0x23f56e8a),LL(0x6cc8df44,0x7a8946c5),L_(0x00000054), + LL(0xe30728ab,0xeda148e9),LL(0x09a1b090,0xb3c8e802),LL(0x4fed9972,0x6feab8c2),LL(0xbd0bf024,0x684e8ba5),LL(0xb6e56fc0,0xc679d0b7),LL(0x3a2827e6,0x05aad3cf),LL(0x9605e502,0x154eec73),LL(0xc7d72f2b,0x9336c1e2),L_(0x0000009a), LL(0x04181bc3,0x17c5f246),LL(0x0c346607,0xf9d4abe6),LL(0x9d5267c4,0x53390747),LL(0x76bf72c0,0xe2e74911),LL(0xd4ea29a3,0x25cb2342),LL(0x5c9021dd,0x7e8d6c56),LL(0xac452ce1,0x1fd1d0fd),LL(0x7f5955ad,0x61f3d3df),L_(0x00000094), + LL(0x7a984abf,0x46f086dc),LL(0xd299e678,0x3ae95a65),LL(0xb7f240f5,0xadf74e6d),LL(0xdd46db9e,0xc95ae291),LL(0x3dae88ec,0xbc688dc9),LL(0x4474ebcc,0xe0bd1aae),LL(0x3a30df47,0xd32d0317),LL(0x222e18e3,0x088bd254),L_(0x000001ad), LL(0xda1caa7c,0x3b9e4f98),LL(0x584fa9c3,0xe3bb36fe),LL(0xa33fbb40,0xece5a0db),LL(0x990ebace,0x2d1efc59),LL(0x1c0c2167,0x0ea367aa),LL(0x18eb3285,0xaa6c72bd),LL(0x4a9d7d68,0x69e3bcf6),LL(0xcd396faf,0xb1399f8f),L_(0x00000027), + LL(0x20f43bc2,0x14169ebb),LL(0x776ec8b3,0x4d9ce016),LL(0x40cc814f,0x64db6f3c),LL(0xde6701fd,0xbc27b375),LL(0x1be16687,0x41b2641f),LL(0xaaf1c8ca,0x5c7ebdd8),LL(0x667e429e,0x9221918a),LL(0x84d2d4cd,0x68099d93),L_(0x000001bf), LL(0x65966739,0x502a6f1f),LL(0xe772626c,0x612fc65a),LL(0xd1717c45,0x06b47588),LL(0xcd0bd273,0xfd9dbcb0),LL(0xf7b68702,0x5beaed3a),LL(0x60d2a43a,0xd9e3bfb5),LL(0x618c6158,0xeec14b9b),LL(0x1dad1537,0x9166a242),L_(0x00000117), +}, +/* digit=49 base_pwr=2^245 */ +{ + LL(0x7b9e7889,0xe957ebe2),LL(0x3ae1f292,0x2bd715ca),LL(0x6ae08fea,0x3b2ea475),LL(0xcab67aaf,0xf247c9f5),LL(0xd47caa37,0x53af0925),LL(0x409f1b89,0x57e7bd1f),LL(0x4ee5b8e5,0x0b979eb1),LL(0x0e289030,0x16dd58bb),L_(0x0000017f), LL(0x1d78d6b7,0x3dfcb1d0),LL(0x595db0c6,0x8e6aaf05),LL(0x9217ec90,0x45106ff4),LL(0xab12df36,0x8489adb8),LL(0xc4207aff,0x257b835b),LL(0xabbbb85d,0x706e08a1),LL(0x71ad10a3,0x6d2a5b7d),LL(0xe224792f,0x528cf56a),L_(0x0000003d), + LL(0x8295b8b6,0x65b39aaa),LL(0x0690f149,0x1270c951),LL(0x763e6fce,0xbafb3a9f),LL(0x3f839143,0x0dc990be),LL(0x3866c189,0xa0d6a0e7),LL(0x087b74c8,0xb520d476),LL(0xd8910a14,0xbd81006e),LL(0x16e6fb91,0xfae3e52b),L_(0x000000a1), LL(0x9757756a,0xbe4b13d0),LL(0xc54fba1e,0x3f1884ce),LL(0x4519ff97,0x68950392),LL(0xfb9e4f42,0x2d309b59),LL(0xf2ce5e20,0x004b85f0),LL(0x35d898e0,0x05c20b17),LL(0xd4f54e0b,0x34add1fb),LL(0x178e2a7f,0xee6a8dcb),L_(0x000000c7), + LL(0x9fa52c89,0x8fc48ea8),LL(0x6a680dfd,0x4b09bae8),LL(0x78d67917,0x7cea1e12),LL(0x0f37ae3b,0x8383d337),LL(0x9f51107a,0x157913dc),LL(0xbbd05c8e,0xc347e479),LL(0xe7f7f024,0xc27dfb63),LL(0xa32c2410,0xa1e55ac4),L_(0x0000005a), LL(0xa0c5983d,0xbc8a42a9),LL(0x31f03f3e,0xdb0533ac),LL(0x07b4440f,0x1b5cb9b7),LL(0x522041e9,0x18816d64),LL(0xa0763672,0x78c44489),LL(0xa7d823be,0x0289668f),LL(0xa033e066,0x14b7bda9),LL(0x1bf9880e,0xb68f4c04),L_(0x0000004e), + LL(0x992b024d,0xcc47fd44),LL(0x301e6aa5,0x4ca3c2ae),LL(0xa239d460,0xb59f6635),LL(0x72a93968,0x93da741e),LL(0x6f3e7cb4,0xe451c847),LL(0x958457a0,0x0539a4ae),LL(0x0ccc6f49,0x70df123a),LL(0x4b36ee4a,0xa7af1f3e),L_(0x00000171), LL(0xa5bf5964,0xd54e934e),LL(0xa4f9d8d0,0x838881f3),LL(0x5759057f,0xd231904c),LL(0xf74d21e3,0x65fa2854),LL(0x09110e09,0x3e3fbb9d),LL(0x73f82547,0x66595687),LL(0xc3213d46,0x4ee05953),LL(0xc6c9fbf7,0x8044dde0),L_(0x0000015d), + LL(0xf19f1768,0xfbd4f466),LL(0xccc78e3b,0x4ab17eab),LL(0xe0f582bd,0x42edf70d),LL(0x32c21454,0xe1c56694),LL(0x7f57c601,0x01c830d3),LL(0xe9eae160,0xe56900b5),LL(0xca26d56c,0x36688674),LL(0xb2fb4c7c,0x145b107e),L_(0x0000012f), LL(0x53c6182e,0x47ff90cc),LL(0xaaf18b16,0x8fc84257),LL(0x96b0582e,0x9e3a6661),LL(0x4532767f,0x0d14fb71),LL(0x29d6ef11,0xdc2f950b),LL(0x54eb6cc6,0x85acbd0a),LL(0x525b30dd,0x5c05fb17),LL(0x67dd5268,0x2af3ff75),L_(0x0000006d), + LL(0xaabdb0b2,0x1475f80e),LL(0x6b7bb07c,0xd90a1e1c),LL(0x92ecf09d,0xc193105d),LL(0x5feeb4d3,0x322cd2b8),LL(0xd3b68b08,0xb3acd3e0),LL(0xb0ed276b,0x50511672),LL(0x512d83e9,0x5830b5d3),LL(0xda968b0a,0x849009db),L_(0x000000cd), LL(0xa9aed6cf,0xadb30375),LL(0xb42997c1,0xc6687e52),LL(0x1d88f275,0x5c1d5e8e),LL(0x5d9e895e,0xdbf775d5),LL(0x1f149b28,0xc29aed12),LL(0xe2724e7a,0x220a70ba),LL(0x7e781bbf,0xcf9cd146),LL(0xfb0950fc,0x40e01766),L_(0x00000026), + LL(0x26492a48,0x8f120fe7),LL(0xb833386c,0xbca86762),LL(0x7d77fbcb,0x2f67d175),LL(0x5165ed7e,0xf29932da),LL(0x40520604,0x607db461),LL(0x88627d90,0x74dd9734),LL(0x9d6e8589,0xff8795e0),LL(0x0898a1bd,0xeba49f57),L_(0x00000104), LL(0xf0c19be8,0x3e449e91),LL(0xc086fd4a,0xce081f35),LL(0x60f6bfc7,0x6b980172),LL(0xf116eb17,0x438fccb9),LL(0xb036eed0,0x3b9d80a6),LL(0x355bcf69,0x17f28db4),LL(0x1d897ded,0x5b488d87),LL(0xb2564e1a,0xa12f88be),L_(0x00000147), + LL(0x6223dd4a,0x84db5faf),LL(0x0b0f5e29,0xd39d495d),LL(0x0a14e52d,0x723841ce),LL(0x365ed8de,0xb4ef0fcb),LL(0xdbcb1fe9,0x3aa7d5d2),LL(0xb19047ee,0x2d33c7e0),LL(0xe2978f53,0x2ad3b9dd),LL(0x9dbc97d1,0x7d591532),L_(0x00000104), LL(0x02d16555,0xb3172c7a),LL(0x8804b57f,0xbeeb32c4),LL(0xad774958,0x59e99dad),LL(0x34b2bc96,0x90ab3c79),LL(0x33fd281f,0xe477effe),LL(0xfaa713ab,0x78b329a3),LL(0xf3df2353,0xac36cbb9),LL(0x62e824d0,0x01585145),L_(0x000000d8), + LL(0xd8f323fb,0x81a85944),LL(0x3dd3e0be,0x51a21fab),LL(0xd871d4d4,0xb561f31e),LL(0xb4ce4cde,0x4449b15f),LL(0xce67b526,0x64493f22),LL(0x82ddd4ad,0x546ec9b8),LL(0x0adc07a9,0x4dba63e9),LL(0x82628c7e,0xcc85d728),L_(0x000000a9), LL(0x5e900de4,0x358c4d59),LL(0x8391c4fa,0xd7be91ac),LL(0x82f89ceb,0xac49480b),LL(0x0dcd6532,0x48237726),LL(0xd2dee6e8,0xaeda17a0),LL(0x13850532,0x2d6729ba),LL(0x201426a9,0xf52c6ebc),LL(0x188c6ec6,0xe02e54ae),L_(0x000001c3), + LL(0x9b37123b,0x274b93dc),LL(0x0d11cfe3,0x01caef2f),LL(0xbe8ef001,0xef0d218a),LL(0x2810cd03,0xace6e761),LL(0x5c17a13d,0x65a61b64),LL(0xcfaaba81,0x669bf078),LL(0xd429ba49,0xd9a6abec),LL(0x0f71e96f,0xf999276c),L_(0x00000185), LL(0x592f7894,0xe42d0a9c),LL(0xfcc98aed,0x3730d520),LL(0xbb02c8f7,0xe10a8edd),LL(0x4886062f,0x57fa6238),LL(0xdab28f83,0x308d0fc8),LL(0x13c4d161,0x8db6b346),LL(0xf5b7f11e,0x70d7617e),LL(0xd7fc6e4f,0x6c6606b4),L_(0x000001dc), + LL(0xe3c773fc,0x30cb34f5),LL(0x854f6660,0xd3093680),LL(0x5abdfb30,0x7939c865),LL(0x3d739567,0xa46ffe9f),LL(0x3a7253f9,0x8fd8f096),LL(0x151f1baa,0xa1443d09),LL(0x9a29f4f0,0xb2ed7af8),LL(0x0ad8104e,0x3a019dfe),L_(0x0000001d), LL(0x76fc8041,0x0868fcae),LL(0xf81d0998,0xdba2642e),LL(0x7f6f6367,0x57d4f243),LL(0x6a189847,0x92fa4eaf),LL(0xc022a76b,0x4e736ea2),LL(0xdd251f4b,0x06ebdd88),LL(0x39b7f55b,0xb9f83aec),LL(0xf34e682b,0xe8264e21),L_(0x000000b1), + LL(0xba33a960,0x502d83da),LL(0x377dd454,0x04a7d732),LL(0x60c9b2c8,0xfa223630),LL(0xe06fc03b,0x7e497e85),LL(0x145bb405,0x456567db),LL(0x39898314,0x2d3a76ad),LL(0xeb33a535,0xd36b4686),LL(0xebbad130,0x134c3479),L_(0x000001f7), LL(0x8a5778a7,0x10340c30),LL(0xf7b160b7,0x1f4c9e0a),LL(0x729201ef,0x09fe7ed4),LL(0xe28d29ca,0x57eb2d3f),LL(0x3e4bfbfc,0xfe99fffb),LL(0xe7397e68,0x62215a66),LL(0x18c1dc25,0x16278a5c),LL(0x1045ab03,0x3d04aaca),L_(0x000000a3), + LL(0xb45b8788,0xc3b9c902),LL(0xe8e7e9b5,0xe1deb5a2),LL(0x6f0ccb02,0xcc118df4),LL(0x931e8bf3,0x44d4d935),LL(0x8d49c80c,0xae880f5f),LL(0x294917c6,0xdaf4ba46),LL(0xf7ee5bdb,0xe09285b3),LL(0x11c3f0c5,0x981a5efe),L_(0x000001a2), LL(0x6f1e6d94,0x150c77e6),LL(0xc23026fa,0x8e7cd3f5),LL(0x567d2494,0xfba4ae31),LL(0x380b42d3,0x7135a34e),LL(0xdf2c5139,0xf390e611),LL(0xfd3ff544,0x1f3b21ae),LL(0x4254bcbc,0x67fca024),LL(0x68f4c9e3,0x2b49a43b),L_(0x000001ff), + LL(0x68ed5662,0x0fa1e08c),LL(0x31f2c30a,0xb65f32e1),LL(0x2d06bb12,0xfec7dae7),LL(0x95c78209,0xb2e0fbb5),LL(0x7d228e9d,0x84a73eeb),LL(0x0a7cd176,0x4968f085),LL(0xb3a9c57a,0xf1034b1b),LL(0x12dc73c6,0x942d801d),L_(0x0000003e), LL(0xd74b5636,0x90895a11),LL(0x8864d82d,0x8f322a4c),LL(0x1db6f8b1,0xef6cb619),LL(0xdc8f7532,0x95fe8c47),LL(0x6bb6ba07,0xb376849e),LL(0x6839da29,0x55ee699f),LL(0x9e4c3d09,0x013177ca),LL(0x06ac03b0,0x0b1b8598),L_(0x00000187), + LL(0x68b8e4b1,0x556e1c75),LL(0xb2cfde0a,0xbf8d1b12),LL(0x77dcfbef,0xbfb5199d),LL(0x8803a28c,0xb3b33660),LL(0x9ab9ee66,0xffcfec92),LL(0x75479f96,0x66780a53),LL(0x9e3f38c0,0xe21a1107),LL(0x8d2c8147,0x61fe7ba1),L_(0x0000002e), LL(0xf14b8da9,0xf175c60b),LL(0xc9c58bd9,0x94928d32),LL(0xf4158167,0xa1977e7b),LL(0x3720a8c9,0x50e84c76),LL(0xb29acf00,0xa73b8ebd),LL(0xdf12fc49,0x8d8a4296),LL(0xf75a8ea8,0x1abde921),LL(0xaa9b61ff,0xf9a127a6),L_(0x000001d8), + LL(0xd278de5a,0x458b071b),LL(0x5b584d1f,0x0313ed9b),LL(0x5f476cd6,0xc54ce5fe),LL(0x007678c1,0x6d145a77),LL(0xe258964a,0x0c62f8fe),LL(0xfa68420c,0x595f7056),LL(0x63621f28,0x2d891192),LL(0x5d8cf9f8,0x32cc419c),L_(0x000001d6), LL(0x130243d2,0x8d5a87bb),LL(0xbcf908d7,0xc4e517b3),LL(0xd0ef67bc,0xba70f65d),LL(0x30e5dd35,0x41af2fd9),LL(0xaeb9fb07,0x448314ee),LL(0xf71f75eb,0x3702fdd3),LL(0xbac48350,0x1d2c0a91),LL(0x1ebd11ec,0x96f4f8e9),L_(0x000001cd), +}, +/* digit=50 base_pwr=2^250 */ +{ + LL(0xf4bf04df,0x31cb2cfc),LL(0x0c3adb10,0x13e25b6e),LL(0x3dd2ab40,0xff8fa4cf),LL(0x758e2edf,0xfeffd307),LL(0xf31ad907,0x4baf111b),LL(0xeba1b456,0x3c4f6a12),LL(0x13a81607,0x94fad755),LL(0xe7fc43bd,0xaee6974d),L_(0x00000100), LL(0xb8d44eee,0x369ffff5),LL(0x5334df1e,0x64f19da6),LL(0x5c2b9ceb,0x01321d0f),LL(0xc5ca3390,0x02b87e91),LL(0x45689acf,0x3a49c8b5),LL(0x049dbf7c,0x93f7ed7c),LL(0x8d277840,0x73a0a1d5),LL(0x726f20ba,0x702173d7),L_(0x000000b2), + LL(0xe3d95d4b,0xe45eed99),LL(0x8fe19237,0xddc0cb97),LL(0x7eb46e14,0x4df73f68),LL(0x57bdaf6e,0x8670ac6f),LL(0x847741a1,0xa46fbe2b),LL(0x02454925,0x82f9632d),LL(0xc15a10d2,0xaf2e144f),LL(0xc55aed10,0xfcdce815),L_(0x000000af), LL(0xc44dce06,0xa506cebb),LL(0x03aaab25,0x48b6559b),LL(0x933863a2,0xc348048a),LL(0xd37a9de4,0x26cd5e20),LL(0xc20f4402,0xee95db69),LL(0xff1c74e9,0x2f425e1e),LL(0xd820bb88,0x1933a6f8),LL(0x6f95cad2,0x93758ef0),L_(0x000000b3), + LL(0x6d40379a,0x87db0c0b),LL(0x3e1edc80,0x4c5fed50),LL(0x8c80df24,0x0d788315),LL(0x5caf06ac,0x12556a93),LL(0x95b47183,0x76d86da1),LL(0xc5714cff,0xcec43480),LL(0xc2f30fbd,0x4e4b2ab3),LL(0xc0cb91c6,0x3a9665b8),L_(0x00000121), LL(0xcbc11bf0,0x75b252f6),LL(0x87c8dce3,0x501537b7),LL(0x84253d3b,0x6a40795c),LL(0xa38fd372,0xb45a08cc),LL(0x22234e1d,0x09918d4e),LL(0x76319208,0xea70d97b),LL(0x3ef6521d,0x0bdcb67c),LL(0xae2d874b,0x5bf47d8d),L_(0x00000024), + LL(0x8fe4e5c5,0x4aedde8b),LL(0xc0d36b58,0xaba27830),LL(0xc0f869dc,0x32bc7d59),LL(0x14a35cbc,0x22d71ab7),LL(0x04bed4bc,0x00680d9e),LL(0xdf25061f,0x3bf0836a),LL(0xb3d768c3,0x3b0d7fed),LL(0x616b984e,0xaa12f95c),L_(0x000000c3), LL(0x3c8c5d5c,0xbd923c28),LL(0x9a32e9b2,0xfbaf0321),LL(0xf0d8e95a,0xed15bdb0),LL(0xd3039b5f,0xd8942727),LL(0xcc59ce26,0xa1dec9a4),LL(0xf0b3676f,0x21992696),LL(0xb8c7cfb1,0x27260c98),LL(0xc1c97929,0x66cb2541),L_(0x0000002d), + LL(0x97538cbd,0xee612332),LL(0xd25447d7,0xa1040800),LL(0x9c9bdc12,0x06fb815b),LL(0x191fed4b,0xae49fbdb),LL(0xd9407747,0x3d19e592),LL(0x9715df76,0x4613ac78),LL(0x1e9e20a7,0x6c932530),LL(0xe2bfff7a,0x92e124fd),L_(0x0000000c), LL(0x6bfd0423,0x13390bb3),LL(0xb87dda88,0x40017ffb),LL(0x979fe6b8,0x635bb57d),LL(0xb6fc9a61,0x8ce87e55),LL(0x535b6b63,0xea1ec56c),LL(0xf567cddc,0x06b927fc),LL(0x72c516ee,0x6fb0868c),LL(0x2ebdac5a,0xfb20cdff),L_(0x000001c3), + LL(0x76e19265,0x53bc630e),LL(0x506faf0e,0x88127211),LL(0x64c166ff,0xad3fc9e2),LL(0x6308dc18,0xb271bc9b),LL(0xec631a3e,0x23be699b),LL(0x2e23525b,0xbfded0b1),LL(0x2391574a,0x69f0d2b6),LL(0xa8ede972,0xcfd5d67b),L_(0x0000001b), LL(0x194cc299,0xf244f4b2),LL(0x911e4585,0x7871cfc3),LL(0x52af7b51,0x331dbf96),LL(0xd41147d5,0x7a399291),LL(0x48e46193,0xb0e20d54),LL(0xd985a24f,0x98e92da0),LL(0x7266525c,0xe9b74352),LL(0xe84bc9e9,0x0956c443),L_(0x000000b9), + LL(0xe9d37b18,0x456032b1),LL(0x30dc5e77,0xd6168724),LL(0x47f55c35,0x18a17037),LL(0x154ea414,0x86d54c7d),LL(0xe14c43c8,0x8d092542),LL(0x78f9b9e8,0x986d7498),LL(0x98519065,0xcc71fd0e),LL(0x4d22c2b8,0x3de62260),L_(0x000000f5), LL(0x91d1b267,0x4de89064),LL(0x601dee13,0x91ac8ed6),LL(0xd375837e,0x8587c0dc),LL(0xa03d56de,0xd2c2524e),LL(0xa4331dd9,0xf36ec517),LL(0x0f8bb8e6,0xb100599c),LL(0x0dc65f7a,0x8ada0049),LL(0xa298259e,0xdd8602f3),L_(0x00000083), + LL(0x1b2f821d,0x01e839c8),LL(0xcb7e709f,0xf9520b0a),LL(0xc4857ab9,0x3a85ef6e),LL(0x09f9eda6,0x51f47572),LL(0x96daca66,0x6e717337),LL(0xccecd697,0x40b37bfa),LL(0x29f4ce02,0xcbb44372),LL(0x0fe8a0ff,0x50a9f191),L_(0x0000018a), LL(0xa3781970,0xcf2ed75e),LL(0xf948f559,0x524a7d80),LL(0x3e1fceef,0x058a1573),LL(0x25fd5510,0x5865318e),LL(0x14c29ed3,0xcb7d9fa1),LL(0xcaa64e51,0xf171b487),LL(0x25580546,0x006163b8),LL(0xde740000,0xb9c9d686),L_(0x00000055), + LL(0x87d0f1ad,0x4cf7363c),LL(0xd4fe9fd3,0x9bd13abd),LL(0x4b7d7e77,0x66face9e),LL(0x42746d44,0x0edf9d57),LL(0xd5f51826,0x888b45bf),LL(0x37b7e3fa,0xde49e8c9),LL(0x0262004d,0x8fd87627),LL(0xdc4da423,0x9122108a),L_(0x00000012), LL(0xce31cec6,0x2990a670),LL(0x1be9607f,0xf2081d18),LL(0x8855d0c8,0x11fb1c34),LL(0xc4c2574d,0xf1b8ff1c),LL(0x3e444ec2,0x4404e3fc),LL(0x2db84189,0xb7726488),LL(0x0dd78e74,0x7de996b1),LL(0x7da11b57,0x12b7b41c),L_(0x000000b4), + LL(0x6a5bd2b7,0x343d29b0),LL(0xd33329b0,0x82fbc88b),LL(0xd3fc973d,0xe1e7bcdd),LL(0x111c0001,0x1c56ee4a),LL(0x0cb45e7f,0x65818c84),LL(0xaccf98e4,0x69029f68),LL(0x6bbf8831,0x53ac7e98),LL(0xe2fa2c45,0x8eccfe60),L_(0x000001db), LL(0xb7950225,0x0e5094fb),LL(0xf11a656a,0x3eed5459),LL(0xd3afccdd,0xe6e4111f),LL(0xc0d31cdb,0x822775ae),LL(0xfb39d140,0x04034f9a),LL(0x5954dd7b,0x8adace51),LL(0xc58c7b83,0xeef24d4c),LL(0xe9d767e5,0x40051c4d),L_(0x0000000b), + LL(0x7f21ed73,0xfb8a9c16),LL(0x300cbdfb,0x12e137b8),LL(0x22e8279d,0xefc00fd0),LL(0x173a4228,0xe30fee24),LL(0xaf4fb8a2,0xaa67fa02),LL(0x6171abf8,0xda82a49d),LL(0x418d47f2,0x3ea61949),LL(0x572fdfa4,0x581b7570),L_(0x0000001a), LL(0xbfe14768,0xceb46b56),LL(0x36fbf6b3,0x2855bf4d),LL(0xd4e1ce80,0xeeceaddf),LL(0x5130ec7d,0xeb1ca189),LL(0x57123316,0xabed8057),LL(0xb7e8b4c8,0xcb8de9f3),LL(0xa878fb40,0x81b143a5),LL(0x2fa96496,0x9080408b),L_(0x0000012c), + LL(0x22b150de,0x9a87c082),LL(0xad76f636,0x222ff210),LL(0xf2177234,0x874b4d66),LL(0xfb6d673f,0x7a63aa6f),LL(0x559b847a,0x1fb601b8),LL(0x7f528818,0x1d5a56e1),LL(0x2dceae56,0x159cad3b),LL(0x64799ea5,0xe50c2b70),L_(0x000001fe), LL(0xc6717776,0xfb840953),LL(0xe6943ee3,0x6c82ee1f),LL(0x45986586,0xa8804763),LL(0xb2c01a1d,0xd2e62027),LL(0x81dd9ed1,0x7ac9ecf9),LL(0xa86a93b5,0xea3ed52c),LL(0xab42d43a,0x9783c732),LL(0x4badd572,0xf7f2c527),L_(0x000000f1), + LL(0x4e7db852,0x1ee36133),LL(0x5d2ce89b,0x7925fde7),LL(0x344442cb,0x7be13983),LL(0x933fd989,0xf818aa84),LL(0x868cf674,0xcf763eff),LL(0x970119fa,0x1161eea1),LL(0x91b1cf2b,0xf803b198),LL(0x47f45bbc,0x358bf9d3),L_(0x0000004a), LL(0x2871ba24,0x03c6e820),LL(0x946bbdda,0x1e7f5b10),LL(0x6d786c0b,0x901e63e0),LL(0xf905444d,0x61291f2a),LL(0xa07d991d,0x26f8514d),LL(0x5dd4a768,0x8caa8bed),LL(0xba23453d,0x625c627a),LL(0x55ae73dc,0x566ebb14),L_(0x00000172), + LL(0x7616ee6c,0x76716c8f),LL(0x9e16dd77,0xd9dc6964),LL(0x424e43ef,0x4f1ab6f7),LL(0x3307372b,0x853acdd0),LL(0xb131b10b,0xd0481561),LL(0x6c779030,0x8833d896),LL(0xb43c81fb,0xf49c69e7),LL(0x013b71e6,0x2d5044d9),L_(0x00000171), LL(0xf6938c6c,0xcb8f8364),LL(0x7d7772d3,0xb55fa4df),LL(0x5c5bb6fc,0xe309036f),LL(0x87518233,0x00458dd9),LL(0x5ae0cb46,0xab6a628e),LL(0x80a93940,0xd00141d6),LL(0xe42dc460,0xe62c337f),LL(0xf594561c,0x32989bd5),L_(0x00000113), + LL(0xe8ff16d6,0xd0f00eb5),LL(0x060c9ece,0x76dc2763),LL(0x8d24bcf1,0xe04a4e63),LL(0x8049d5a2,0x1f378724),LL(0xad86dce1,0xee568d6b),LL(0xbd4ecf75,0x064ed8ea),LL(0x23b4afb4,0x3066bb9b),LL(0xe8ba5019,0x59524e62),L_(0x00000113), LL(0x16a5e07c,0x0317cfe4),LL(0xb05c4eb0,0xe263fd3a),LL(0x87619f5b,0xe43b9d32),LL(0x04548fa9,0x5fe60636),LL(0x1e3bb4ee,0x177080d6),LL(0x80dd88dd,0x6b920ffa),LL(0x50a4adb9,0x6cf839eb),LL(0x579a402c,0xbaad4226),L_(0x0000016a), + LL(0xd3860f86,0x51b92975),LL(0xd751ffa4,0xe9c89ace),LL(0x0fff3b8f,0x22e82df0),LL(0x44ceed1d,0x9ef4bd4d),LL(0x322e7d38,0xec43e5b6),LL(0x5dafe91d,0x3ac6cd72),LL(0x385f22e0,0xc23c7139),LL(0xecc87ca1,0x8c786dee),L_(0x00000186), LL(0xd6515802,0xfbf97dbf),LL(0x72372941,0x689ac9e8),LL(0x4611974e,0xce7740f8),LL(0xe04ba0c6,0x7a8f9746),LL(0x7419caa0,0x05b0cdaf),LL(0x30755659,0xcd257003),LL(0x5af7e403,0x8e3b2c01),LL(0x7d54b47d,0xc1f90328),L_(0x00000043), +}, +/* digit=51 base_pwr=2^255 */ +{ + LL(0x06aeb844,0x171b7252),LL(0x2f67f278,0x3ef95a8f),LL(0x411d7c3d,0x1341fdfb),LL(0xbc9db5d5,0x9c831f2c),LL(0x64cd3d49,0x5fa0db40),LL(0xb8bb90a7,0x2c8d72cf),LL(0x050fdef7,0x9770a986),LL(0x584d26e8,0xcb48ff2a),L_(0x000001c1), LL(0x8a357b6c,0x75fe114c),LL(0xe1fc26b3,0xaa2296d0),LL(0xe2fe623c,0xe037cba1),LL(0xca73315c,0x36843eb8),LL(0xb7e86db2,0xb5b70ddf),LL(0x4b155e04,0x20198f9d),LL(0x06921394,0x51535cfa),LL(0xaa06d437,0x98bee5a2),L_(0x000001b1), + LL(0x89cc4566,0x68f57feb),LL(0xa2543b28,0x4bd3cbd6),LL(0x0bf63c0e,0x66da5e56),LL(0x648f4a56,0xb7d9cc0e),LL(0x7591427c,0xab848b1a),LL(0xe85c5977,0xf4656829),LL(0x4025667a,0xcdae8f7a),LL(0xab876527,0x93eb73b4),L_(0x00000077), LL(0x204ed818,0x30db96c4),LL(0x1b3e5e48,0x26c352dd),LL(0x497308c9,0x54703369),LL(0x3370174e,0xa9534502),LL(0x7c6d8497,0xae86058c),LL(0xae7aecbf,0xa32e4cdc),LL(0x67daf0b8,0x3a4e9eb5),LL(0xaf8dd7df,0xff79b85e),L_(0x00000081), + LL(0x5b9e36c3,0xe50c2a6e),LL(0x316c41f6,0x6af25999),LL(0xe48ac795,0x813a1e7e),LL(0x65d44dd3,0x7fc2f7f1),LL(0x4d3b130b,0x08cc4038),LL(0x7c00e333,0x4484ccd4),LL(0x8e7636fc,0xf9a80322),LL(0x1688e5f3,0xb4b52a8f),L_(0x000000e2), LL(0x0987f80d,0x2cd48e4d),LL(0x9fe4562b,0xaa48e7e6),LL(0xf168a311,0x7fdc1a14),LL(0xdf4018fc,0xc463e403),LL(0x6c8979b5,0xd6d0bb4b),LL(0x62cddf39,0xdf09f24f),LL(0x9b318fce,0xca7e6578),LL(0xcab54343,0x48ea638d),L_(0x0000000a), + LL(0xd7deae24,0xcb23734e),LL(0x23939762,0x66bcd84d),LL(0x989a46bd,0x85ec037c),LL(0x65439883,0xcc808ec0),LL(0xa3f08c8a,0x680dc66c),LL(0xa76800e7,0x4c3c5332),LL(0xcc98ee9e,0x8663204a),LL(0xa0ef46de,0xa2348db0),L_(0x000001ce), LL(0x05b4a4e2,0xb14e22b3),LL(0x930a37ee,0xe9d3141e),LL(0x35f5cd09,0x3364f154),LL(0xf55ccf3f,0x55f31352),LL(0xf6c93770,0x0c74549e),LL(0x4bf80f61,0x8d0207da),LL(0xb1c2b15c,0xafb6ee97),LL(0x0992fd2c,0xfe513092),L_(0x000000ff), + LL(0xea69c80c,0xc95caa93),LL(0xdeca4025,0x95bc2026),LL(0xc8c86ea9,0xe3161b91),LL(0x73fc72d8,0x46b441b7),LL(0x033e25a4,0x6b7c1805),LL(0x2d4e9335,0x5a4b1a06),LL(0xd30b7dc3,0x992637db),LL(0xdaac9a90,0x7c72add5),L_(0x00000146), LL(0x4f4c9063,0xc14619bd),LL(0xfbc662e7,0x3c65896e),LL(0x8176a953,0x1c5790f3),LL(0x4f51c6bd,0x0ef460cd),LL(0xc6fa754c,0xee3cd226),LL(0x5e872735,0x05291b65),LL(0x79e3b5c0,0x734e1b22),LL(0xfa256432,0xf18b1394),L_(0x000000cf), + LL(0x0643f252,0xedb2cca1),LL(0x00e32811,0xc996f279),LL(0x6f6af92c,0xbf992edb),LL(0xbdef8275,0x3384462d),LL(0xa4dd3d26,0x818a7ff9),LL(0x8e214401,0x60e7694d),LL(0xa7aec62d,0x9d54e87f),LL(0x8bdd2244,0x5d769f98),L_(0x000000ba), LL(0xfb63c9fb,0x7e7ff11a),LL(0x7eec026d,0xc6b3e18f),LL(0xe08b80f1,0x84b5c983),LL(0x5d6b5a4c,0x4b0fd4b7),LL(0x85f99e3a,0xfc4904ce),LL(0x7afd5a7c,0xc336a99a),LL(0xba1e62f6,0x24e4a736),LL(0xbe20ba29,0xcceea595),L_(0x000001c9), + LL(0xbb592469,0x29ecccb0),LL(0x89ef0925,0xce6e29e1),LL(0xf98f60f1,0x36216c3c),LL(0x0848c8bd,0x63a73874),LL(0x085409b9,0xd4abc07d),LL(0x2319eb0d,0x0e39c02a),LL(0x6bda97a9,0x393de5dc),LL(0x0140ddd0,0x5302f915),L_(0x00000104), LL(0xa2f22a24,0x54381101),LL(0x423788d2,0xb694bdb7),LL(0xc151a89e,0xd0ef2b67),LL(0xb01ee242,0x01c2b082),LL(0xc07af292,0x10fd1158),LL(0x3639401a,0x8e3f86e2),LL(0x1ed8f101,0xcf21ea60),LL(0x83b3b62f,0x186fd3fb),L_(0x000000d5), + LL(0x718c92f0,0x3423f6e8),LL(0xdae8d246,0x5f129e35),LL(0xaeff7db0,0xdbad59f4),LL(0x963932f5,0x3cf82c0c),LL(0xf5e468db,0x6b7d10e1),LL(0x10e6e23c,0x6e085959),LL(0xc76fb1b0,0x538880e8),LL(0xe8c12594,0xf8f02134),L_(0x00000021), LL(0x89506649,0x272ea4f0),LL(0x9dd1a14c,0xaa274066),LL(0xa6cc0d62,0x191622f8),LL(0x92244f6f,0xd28338e3),LL(0xc3dcbd9d,0xa8dd7166),LL(0xa39c0c61,0x4930a90c),LL(0xb979b8ce,0x6cd41296),LL(0xaa5c88b7,0xf6ce2037),L_(0x000001b0), + LL(0x8ef889ab,0x6a8258a9),LL(0x87f34cfa,0x6e977272),LL(0x538468d6,0x198bf996),LL(0xe9cb2903,0xea7ac40d),LL(0x389f9bff,0x50fd922d),LL(0x88f4717c,0xcb0c2bca),LL(0xb57f0298,0x5d670088),LL(0x812c3767,0x7a4062fa),L_(0x000001d3), LL(0xd95da33d,0x89860058),LL(0x74f4e9a6,0x9df658db),LL(0x46a06ddb,0x8faf5c15),LL(0x36b96ffe,0xe9bbc867),LL(0x1a07dce2,0x19a59e1c),LL(0x9536a09f,0x1683c160),LL(0x7fabb0f1,0x183d2bdd),LL(0x06b7a416,0x75975bd0),L_(0x0000015b), + LL(0x5451d16f,0x720fff5e),LL(0xeacd93fd,0x6c62e42c),LL(0x6f74fc83,0xff9b7285),LL(0x8a51db93,0x7b6bb42b),LL(0x66ca983e,0x8fd893a3),LL(0x08eee06f,0x491c6c89),LL(0xe1230942,0x638e9f64),LL(0x4984e580,0x9b2ab576),L_(0x00000063), LL(0x0bad9aba,0x7b9d835c),LL(0x84846555,0x89b78779),LL(0xc6bb325d,0x88fce8c3),LL(0x0fb571c3,0x237c5f2e),LL(0x27185f17,0x37bcf483),LL(0x53b0ac57,0xf037df6f),LL(0x34a972e5,0x73b6f7ae),LL(0xf685c7b2,0x3e68f821),L_(0x0000012f), + LL(0xfab07625,0x52bebe14),LL(0xd1fbd06a,0x7d25c686),LL(0xe5149dc2,0x20b2f012),LL(0x6707c1b6,0xe4fdb06a),LL(0x5cf7e0a4,0x124b0592),LL(0xdadcb97c,0x9ef54e16),LL(0x97f26141,0x6b91bf50),LL(0x689c475b,0xf26ed365),L_(0x0000001c), LL(0x31ac5e9e,0x95772aee),LL(0xf333125d,0xfb3bbb8c),LL(0x906a459b,0xda033a3e),LL(0xd9a3800f,0x7aebdf94),LL(0xea08c76a,0x4600cde7),LL(0xecd96496,0x1b4f8404),LL(0xb9fdf8c7,0x58389c23),LL(0xd186fc48,0x9a44e137),L_(0x00000045), + LL(0xace5575b,0xabf66053),LL(0x6dc5328e,0x54861cbd),LL(0xea9fdaff,0x8555b123),LL(0x0ecf823c,0x09e411c0),LL(0xa5d8934d,0x0ae97a01),LL(0x170ceb09,0x91dabc9c),LL(0x73c40a75,0xd8f751f2),LL(0x52861011,0xba95af0f),L_(0x00000181), LL(0x352bc9a3,0x79de4fde),LL(0xf0130bb7,0x2eb1b199),LL(0x3c4457b6,0xa95e2900),LL(0xf04878d3,0xc1a9dc9f),LL(0xe04ebfee,0x097a6545),LL(0xf5aa7d0d,0x673c7b41),LL(0x3c5c4ce8,0xa67894e6),LL(0x385d1700,0xeb9b105b),L_(0x00000060), + LL(0x16680fb2,0x370df675),LL(0xbeb0847a,0xbefb427d),LL(0xbbe54c19,0x9a4770e1),LL(0x44a1916f,0x7f5945d3),LL(0xc14ef507,0x731b2da3),LL(0x17aa92af,0x07208217),LL(0xf69f649d,0xa27c5c7b),LL(0xabd89463,0xc25bbff1),L_(0x000001e6), LL(0xe551752e,0x9733c080),LL(0x98e86236,0xc5bfef7f),LL(0x936a2ebe,0xcc36a721),LL(0x25c227b9,0x11dd6248),LL(0xf8d96ae8,0x83440604),LL(0x3b2dca5f,0xc74d7e75),LL(0x3d8a998d,0x3c210303),LL(0xdcf4cf75,0x2fdbfd9c),L_(0x000000d1), + LL(0xe646d7ef,0x2f349cfb),LL(0xfd187fbb,0x22f14a9c),LL(0x7fb5a2ff,0x781ef46f),LL(0x084df701,0xada115d8),LL(0xab2e7da6,0x37b36285),LL(0x21432735,0x779e5cbe),LL(0x42159b5d,0x987b1bb3),LL(0x182d17ef,0xbd5c0321),L_(0x00000197), LL(0x0c974b93,0x3e904667),LL(0x4f31ca20,0xc9fa51be),LL(0xed87df23,0x5530167f),LL(0x7ab1aee3,0x34d6716d),LL(0x16c8a7b3,0xb3f82160),LL(0xf3eb37b8,0x77ee013d),LL(0x13ff1326,0xa57a3a10),LL(0xd7d1a2e9,0xab837a7e),L_(0x0000012b), + LL(0xfafe3733,0xb917d893),LL(0x17024a0a,0xfd27b406),LL(0x89eda4ec,0xcd3182c3),LL(0x4e2244fc,0xcec915fc),LL(0x083e32ec,0xbc2fe85f),LL(0x26668631,0x3458ec27),LL(0x23dec845,0x2e647e96),LL(0x35986103,0x80f7dfe3),L_(0x000000af), LL(0x9c1dd0c7,0xc0f6c814),LL(0xdb6594dc,0x59eee455),LL(0x2db7ed2b,0xc7b946f5),LL(0xc94ac2ca,0x45521872),LL(0x1f918bfa,0xe23366a4),LL(0x3439b349,0x50d8220c),LL(0x347cd4a8,0xc2e30ec6),LL(0x9274e0c9,0x10e0889c),L_(0x0000004d), + LL(0x18b3fd00,0x605cfac8),LL(0x11efdfd4,0x85d2f0b8),LL(0x5bb41efb,0xee216714),LL(0x3c03cac7,0xade36a6e),LL(0x485c4b2d,0xcd3725a1),LL(0x50bc220d,0x2cf525a5),LL(0xb11c84f3,0xe314db66),LL(0x664e47ac,0x0c8cd013),L_(0x00000050), LL(0xa7a48858,0x6f7bbfd1),LL(0x7d04c227,0xe24ada56),LL(0xadced466,0x03a6a941),LL(0x70addbb1,0xf14e02c2),LL(0xc761ca82,0x94b62798),LL(0x03264d07,0xa0bec3f9),LL(0x966e8d47,0xe6caf618),LL(0x1f211c02,0xa8c9821a),L_(0x000001cf), +}, +/* digit=52 base_pwr=2^260 */ +{ + LL(0x26c24408,0x1e48c2aa),LL(0x2fb6cd86,0x8746f93b),LL(0x515690c4,0x71e5f018),LL(0x76a3c1b7,0x99fbb28d),LL(0x993035c8,0xc338e004),LL(0xa3d8d18a,0xb4e7f02f),LL(0x804c0351,0x09fabf9b),LL(0x3e6175e3,0x697832f8),L_(0x0000016b), LL(0xf6830680,0xc1a8a622),LL(0x50d94962,0xf94a3f34),LL(0x0a44d62d,0x8057a83e),LL(0x05319e21,0xd2bed201),LL(0x3a4a1ebd,0x3d6076c1),LL(0x7368f486,0x4672ca13),LL(0xef4b1a43,0xf96135e4),LL(0x6692537f,0xb40a920b),L_(0x00000090), + LL(0xb81b7a5e,0xb048815a),LL(0x11e1229e,0x2e0a161f),LL(0xecb84207,0xf8e1801b),LL(0x5b394a58,0x890edfad),LL(0x37512807,0xb3e4e477),LL(0x5d81f675,0xc9984105),LL(0x1050ce18,0xf43ed35c),LL(0x17bd56ac,0xfc61c6c2),L_(0x00000011), LL(0xcf0d6c8c,0xaf53db5d),LL(0x6ee72ddb,0xc6ab4e4f),LL(0x32c8481a,0x1b4e6860),LL(0x5c545af6,0x0e7c0e41),LL(0xc3595ad6,0x261ffe75),LL(0xbf47f59b,0xf66fa7cf),LL(0xfa1aaf6c,0x212e7097),LL(0x86b7977f,0x3c00aa83),L_(0x000000b4), + LL(0x4b0029a7,0xf5c33307),LL(0x1eec5f29,0x9e45f3bb),LL(0xf8396133,0x17635aad),LL(0x25d3e2a3,0xbdaba508),LL(0xc34ef799,0x574f4d09),LL(0x78d47f38,0x085e8086),LL(0x3db03879,0x1c4a5748),LL(0x65ae9f6d,0x3c8df544),L_(0x000000c3), LL(0xb52fb74f,0xb901e46e),LL(0x1868eef4,0x68ec4a86),LL(0x7bab1199,0x9f2f51a0),LL(0x8f19df10,0x6a75a074),LL(0x2d75da4f,0x61385965),LL(0x59f7f255,0x60c80677),LL(0x6b7b569e,0x40b66382),LL(0x0533f4d4,0xd7897f45),L_(0x00000195), + LL(0x3b8e6670,0xd20de07d),LL(0x7aafab5a,0xdab27e9b),LL(0x3fb66eb5,0x5ac52dbe),LL(0x7ce85634,0xdf84c8cf),LL(0x9025496c,0x95b8e1e2),LL(0x776182f0,0x8e6db4cd),LL(0x21aaa54e,0x3bb0faa3),LL(0xe73fef00,0x9223c88a),L_(0x00000098), LL(0x16d643fd,0x70e138d6),LL(0x5612fc48,0x69ca59b8),LL(0x0889a9e6,0xfb3a26a0),LL(0x93e3dad6,0x43df1bc0),LL(0xe6ce66bb,0xae036271),LL(0xfcd4244d,0x05182a82),LL(0x958ca2ac,0x102559d8),LL(0x26838c85,0x588a9a75),L_(0x0000009f), + LL(0x0184b954,0x6cd88b38),LL(0xdcf3ba77,0x70c99422),LL(0x86f66f43,0xbc4bd450),LL(0x7b81c0e8,0x93575c5f),LL(0x704cad24,0x4091825d),LL(0x4b9f70ce,0xf1ff4bbf),LL(0xac2a0a24,0xb5d28bd9),LL(0xe5ebf7a3,0x5e706c23),L_(0x00000198), LL(0xeb270e7e,0xeb783548),LL(0xfd0b1050,0x81562bc5),LL(0x96b8a59a,0xedb5f688),LL(0xbc130375,0x8ab1fc73),LL(0xa1c5bd93,0x89b28fea),LL(0x7f18c19c,0x8d6e4b1c),LL(0xe98a494f,0x409f7384),LL(0x55131bf6,0xf382aa3f),L_(0x0000017b), + LL(0x25a27923,0xffb6351b),LL(0xc9bc2f04,0x7f29f1e1),LL(0x91e80528,0x37069b7d),LL(0x8a56cb26,0xff75d6d4),LL(0x7d9a9a20,0x3f52dd39),LL(0x52270b39,0x703dee3c),LL(0x67288a63,0x13f9c1c2),LL(0x49651d47,0x663b9838),L_(0x000001b2), LL(0x1cd56c85,0xbc1a8d52),LL(0x46598a93,0x87351736),LL(0x4685de4b,0x418967ee),LL(0xf35701ef,0x6dbbce4c),LL(0x380b116b,0xc5acf7cf),LL(0x35416b03,0xe839b424),LL(0x8d1a9cff,0x15841fbd),LL(0xe1730d1f,0x5ba5c3a6),L_(0x00000078), + LL(0xc0fdbea9,0xf76bf65a),LL(0xf573be01,0x0c6778ad),LL(0xffd85a6f,0x927d0f51),LL(0xfe98c72d,0x1738874a),LL(0x187e8ec8,0x032ae57d),LL(0x00c6d76d,0xdf95e888),LL(0xdee55d14,0x0dec4042),LL(0xcd5760c3,0xb172c8e8),L_(0x00000132), LL(0x14eac108,0x7bdb463a),LL(0xcbf771c3,0x76281603),LL(0xd48543b5,0xfc634037),LL(0xb965ac3e,0x6e5426f3),LL(0x49a7be5b,0x87fba366),LL(0x2e203d0f,0xddb5ca9f),LL(0xabbc3174,0x55052649),LL(0x2eb60836,0x21e004b7),L_(0x00000098), + LL(0x5c1bc4f6,0x64a43b5d),LL(0x1c634029,0x5d7c2617),LL(0x93c0fb82,0x1b7967cc),LL(0x96145dca,0xd068364b),LL(0x3b5c4ddf,0xec5bd3c7),LL(0xd5007f0e,0x2d7bf8f0),LL(0x771d6fd8,0x215b93c7),LL(0xf222990f,0x70529475),L_(0x00000028), LL(0x1d26e01b,0x340c9a0a),LL(0x04b5b926,0x7edb2bec),LL(0x0417ca25,0xefd5d17d),LL(0xb41c7280,0x70df3372),LL(0x93c942f6,0xfbfcef99),LL(0x7e3d7910,0x37ef3a57),LL(0x39005c54,0x8c4d4c90),LL(0xdb0ceb3a,0x446ceedd),L_(0x0000008e), + LL(0x65b80a9e,0xb3de9aec),LL(0x47955751,0xebd70107),LL(0xb9795325,0x326b6e2a),LL(0xb7ebfc01,0xb10f9b62),LL(0x6ec48711,0x145049a1),LL(0xb7dced78,0xd8c85f83),LL(0x6736770f,0xf3878209),LL(0x41fd70b3,0x4d4517ea),L_(0x000000ec), LL(0x1245bc60,0x228424d3),LL(0x6d611151,0xee80416f),LL(0x92f6a019,0x18ef86e2),LL(0x2e88fbc7,0x9f5d9f5c),LL(0xb51a1205,0x3ee14394),LL(0xd989aea0,0xe73a0ff8),LL(0x81623fdf,0x10ed321a),LL(0x3d71a6e8,0xa6b2f1dd),L_(0x0000018b), + LL(0x98f6bd7a,0xef684d8d),LL(0xaac1e35c,0x1c71e036),LL(0x19611929,0x9428ed8d),LL(0x24f7251a,0xf90f6e8a),LL(0xcd34ddd1,0x9742ae40),LL(0xf7d22290,0x9b5b15a7),LL(0x5d805418,0x91f1f6ec),LL(0xe50e28f5,0x604d11c6),L_(0x000001d0), LL(0x2c60d848,0x5ad92240),LL(0xf18911d8,0xcfb4c90b),LL(0x3c5b71d6,0x1a2c26ff),LL(0x5d53d732,0x9fada03e),LL(0x8bfbe9ec,0xd69c81a8),LL(0x443458b3,0xce7f11ce),LL(0xca59b490,0x2489ddd2),LL(0xdaf9ecde,0xceb4c71d),L_(0x000001b3), + LL(0xd052454d,0x6844d9d2),LL(0x667d1be2,0xec3032ef),LL(0xefd476cd,0x67f7c660),LL(0x47628345,0x80c64c50),LL(0x57751538,0xbb8da5d7),LL(0xb8ef3bee,0xb395bca7),LL(0x3bc2ad45,0xc2e7e012),LL(0x610d67aa,0x9e6187ca),L_(0x000000eb), LL(0x3dc2b1ef,0xf5d2b364),LL(0x4ee3fb91,0x38966853),LL(0xe0446916,0xc5fb0623),LL(0x2a6bfc98,0x6bce11c9),LL(0x65c6b297,0xeb6233b0),LL(0xce8c355a,0x6ac473c3),LL(0xd97c1dd8,0x56091541),LL(0x0514bb6d,0x01792023),L_(0x00000043), + LL(0x92033f14,0x17a6ef25),LL(0x89a53d65,0xaa5a50af),LL(0xd872b73d,0xa475ab82),LL(0x53f65ce3,0xe34acbbd),LL(0x1d33affd,0x33a80ee5),LL(0xe7690066,0x969e21f0),LL(0xcc415a0a,0xca2e0920),LL(0xb0325f4a,0x901c4d50),L_(0x00000172), LL(0x0f43e61e,0x40fc3a90),LL(0xa48abf41,0x601d8bcd),LL(0xe374c532,0x786a5e70),LL(0x7446fbbf,0x484d4fec),LL(0x0ec5b3b2,0xdc5db3b9),LL(0xeeb6c43c,0xe60cf40d),LL(0xca2ac048,0x1ff63739),LL(0x4475d66f,0xea8de3b9),L_(0x00000143), + LL(0x9adfc451,0xe4847519),LL(0x294337f9,0xbe446976),LL(0x4e7dac1b,0xf00a5441),LL(0x94ab8aec,0xc582caec),LL(0x5ac9bb3b,0xeacdc76b),LL(0x3fe0e66a,0x6e00689b),LL(0x246c86c1,0x6c266a0e),LL(0xbf0ade72,0xa0a2515c),L_(0x00000043), LL(0x70658a73,0xff30554a),LL(0xc9871f1a,0x4b2448f2),LL(0x8dd50a2b,0x87a01756),LL(0x3ea8f62c,0xb6b9a2ea),LL(0x7311e04c,0x0d165122),LL(0xfc6c9f2d,0xb7efc2eb),LL(0x07406007,0x92c33ebb),LL(0xbd8e5282,0xd904c21b),L_(0x000001cf), + LL(0xe495d18f,0x9b0f5222),LL(0xac5ca929,0x126086f0),LL(0xf4d4507f,0x41908fc8),LL(0x3355d26d,0x65791c82),LL(0x31183a9c,0xf00a9e80),LL(0x018f0189,0x644ffd95),LL(0xe447af71,0xbffa0975),LL(0x7424f93b,0x849e9345),L_(0x000001b3), LL(0x0d6949a4,0x78519510),LL(0xb79c2b5f,0xa59828a5),LL(0x9ae92003,0x6c54e38e),LL(0xfb93be38,0x67bd521a),LL(0x4c71aeb4,0x04b0340d),LL(0xa3451e4d,0xcd2e92f4),LL(0xa9a77ad0,0xfe218b65),LL(0x656db073,0xd2b4300f),L_(0x000001f4), + LL(0xa3b27cf0,0x160f039b),LL(0x82ec5f83,0xcff71736),LL(0xba9364b6,0xebc485a6),LL(0xddbcec8c,0xdc80329f),LL(0x3bda1715,0xcc71e664),LL(0x46fc4c3b,0x592819ef),LL(0xd1da3eb7,0x2cb62fe8),LL(0xb29cfca6,0xc417d055),L_(0x00000061), LL(0x8c802541,0xfc158c58),LL(0xe84b30eb,0xf6625ae1),LL(0xd137d022,0x441de79b),LL(0x42e42c6f,0x7d99126c),LL(0xf2ec0ec8,0xfbbd41bb),LL(0xb8f928d4,0x2851ec63),LL(0xf3ff5c1d,0x5e5a9ca7),LL(0xd3429e7d,0xdfb3e4f5),L_(0x000000ac), + LL(0x56a3a063,0x8d143249),LL(0x627718f3,0x2232fc35),LL(0x5a0479ef,0x6a6d389e),LL(0x82744b80,0xf4d435b8),LL(0xb0bd687a,0x2792b960),LL(0x4cecd317,0xf792e60e),LL(0x063be911,0xb09dcb17),LL(0x02f6ffb4,0x79c0ec59),L_(0x00000094), LL(0xb04fbf6f,0xe7c66f8f),LL(0x07b90918,0x95b38bbb),LL(0x01c5b207,0xc67022d2),LL(0x4fdf3937,0xee01b834),LL(0xe5a11142,0xc7b97506),LL(0x11b8cb5f,0x2ae40433),LL(0x2450b7bc,0xe3e1937f),LL(0xa26a70cf,0xb747b45f),L_(0x00000042), +}, +/* digit=53 base_pwr=2^265 */ +{ + LL(0xf2fdc439,0x7edbcb88),LL(0xdb91292b,0x19d35421),LL(0xd5dee79a,0xa420a538),LL(0x035e9ea2,0x9cf14f3b),LL(0xe21709fe,0x49703f94),LL(0x690ca5b7,0x495b47e8),LL(0x4deb7af2,0xcc2ef057),LL(0xb09d6324,0xf4a72dd4),L_(0x000000d7), LL(0x7ff7df3b,0x25c764e6),LL(0x1593ef9e,0xb9153d85),LL(0xef6d9489,0x117822d7),LL(0x238e5449,0x1e34e4c9),LL(0xbbd3333b,0x58cc8198),LL(0x416c6cfb,0x7b487650),LL(0xa8085b4b,0xb3068c07),LL(0x5e20cc8e,0xcdc16b7b),L_(0x0000013f), + LL(0xf98b837d,0xe15922b2),LL(0x62c29919,0xc8afde9c),LL(0x95a1a3c5,0xffe9534c),LL(0x604b1043,0xfa2f638f),LL(0x27a01a13,0x04cd8a8d),LL(0x2660393b,0xe26fd0c2),LL(0x72545d96,0xcf0808a0),LL(0x1dd10699,0xded0e387),L_(0x00000097), LL(0x037dfe3a,0xc38223ef),LL(0xd36e2094,0xe8b66c87),LL(0xe28405ae,0xe3e2766e),LL(0xa065b535,0xed4b87f0),LL(0x084a317d,0x3f53ac0b),LL(0x0ca5866d,0xa0ee5586),LL(0x82b21bd7,0x1fa70803),LL(0xff1d58cf,0x4ad6e2f3),L_(0x0000009d), + LL(0x905b2c93,0xef13c9b6),LL(0xd34ff12d,0xa5eaf60e),LL(0x46115d13,0xad4eed45),LL(0xc0704820,0x0761b0ac),LL(0xf0c499c6,0x5dd51e45),LL(0x4abd13af,0xa978e552),LL(0xb1ec09b7,0xa79f811d),LL(0x0dab7d3a,0x34d1f590),L_(0x00000147), LL(0xa75f21e6,0xea3d3b19),LL(0x0364a9c0,0x68df5edf),LL(0x5e1d6b4a,0xb44c93c9),LL(0x33e2dcc0,0x07832283),LL(0xea8fc7be,0x37cb9512),LL(0xe9e13504,0xc965c20f),LL(0x887068c5,0x62d3176a),LL(0xe870a541,0x0903d004),L_(0x00000092), + LL(0xf1dd3a67,0x7824703a),LL(0xb39c772c,0xbf0c69c5),LL(0x46f942bc,0x31d0901a),LL(0x9e0174be,0xd38bdff8),LL(0xab6326f7,0x91bfcc1e),LL(0x8787eadb,0x541868bb),LL(0xa0385662,0x6ba48a8b),LL(0x6d878761,0x3094b460),L_(0x00000185), LL(0x69e290f5,0x1d33d545),LL(0x1ba52eb4,0x662b634f),LL(0x9dfd1d1b,0x876fc504),LL(0xbe51c909,0x93e42059),LL(0xcb7406ba,0x49355b9d),LL(0x2651475a,0x8963ea18),LL(0xbcf76704,0x08985cee),LL(0xaa85c805,0xf1d1062e),L_(0x000001ab), + LL(0x6c2616bb,0xeb31ab0e),LL(0x01e38aad,0xbcd43f2e),LL(0xda068909,0x7f990c18),LL(0x9fb2c072,0xa82ff220),LL(0x757bff88,0x81327a89),LL(0x28c2afd8,0x1d3a1126),LL(0x0c2079b4,0x95685773),LL(0xa957db38,0xe18a18bb),L_(0x00000058), LL(0x45f5c72a,0x53fe6703),LL(0x42ce353e,0xbe22096d),LL(0x4a3251c0,0x601d33ed),LL(0xaaaf17c5,0xfe2c8cbc),LL(0x3d4b4185,0x242a9581),LL(0xb32328dc,0xbc79d78a),LL(0x03bf4442,0xa103c8f5),LL(0x64e28853,0xf9b6130c),L_(0x00000094), + LL(0xb447a9f7,0x2f9482b0),LL(0x31ede472,0xc7c55120),LL(0x00d8bc4d,0x627457bc),LL(0x5c471ca7,0x4a9f36d6),LL(0x14a28cac,0x436c70c1),LL(0x38a173b4,0x011c4897),LL(0x96f4df0f,0xdde3c9d7),LL(0x587a661d,0x8ddb7793),L_(0x00000026), LL(0x143023ce,0xbe0156de),LL(0x68012aa1,0xeabfa04c),LL(0x3ed6803f,0x204765fc),LL(0x762dc13e,0x2e5fcd9a),LL(0x5b5cd65c,0x04a542e1),LL(0xd6b6a2d7,0x3dcadeb8),LL(0x57f74a74,0x0da1060f),LL(0xe953f87b,0x2e478948),L_(0x000000e2), + LL(0x326c0546,0x63e8f854),LL(0x35eb9eac,0x39f46433),LL(0xf4944efa,0xc8688704),LL(0x91ff1606,0xfeaa7186),LL(0x99316708,0xa92605cc),LL(0x3fbb0f25,0x2252affe),LL(0xa90598c4,0xcbb64aaa),LL(0xb34934f1,0x3ba7e244),L_(0x000001f6), LL(0x763915ed,0xc814b6b0),LL(0xe697e570,0x69866f7d),LL(0x63fc73af,0xb1f0f7a2),LL(0xb634f283,0x17533e2f),LL(0x423d910b,0xe17bdbad),LL(0xfbcd888f,0x778dac12),LL(0x4c46f8f4,0xfb0bef09),LL(0x72d4d626,0x4fcdf97f),L_(0x000000b1), + LL(0x13b5648e,0x34c2f6e4),LL(0x202b7ba7,0x8535398b),LL(0xebd7f177,0x7eea8b23),LL(0xd3b0fc5e,0xc7a0f19f),LL(0xa3df55dd,0x577641e2),LL(0xcb9f261d,0xf496646e),LL(0x112454e8,0xdaf2be9c),LL(0xc23da6a9,0x64522e02),L_(0x0000003d), LL(0xe59d35c5,0x1efd66d8),LL(0x1b401767,0x904f29b9),LL(0x084f27ae,0x129c352b),LL(0x23e88566,0xa3263601),LL(0xd229faae,0xc04620cf),LL(0xc91c87f4,0x535f695e),LL(0x27c58545,0x3cfc2a21),LL(0xc94873fa,0xdacf3fa1),L_(0x0000005f), + LL(0xb13cb473,0x88b9f2da),LL(0x2dda613b,0x3a43eba4),LL(0xedbf11f5,0x9cddb3f6),LL(0xdf7a9b8a,0x1e00c8ae),LL(0x7e0c7ceb,0x84c5eb6c),LL(0x03d1dbf7,0xc6747572),LL(0x80122fe1,0x5d6814d2),LL(0x639c5254,0x1d4a0602),L_(0x000001bc), LL(0xec1fdac5,0xa4651a82),LL(0x690ed4f8,0x4ef3c551),LL(0xe2f0cf8b,0x20e94507),LL(0x61b6144a,0xeb258124),LL(0x11bde361,0x51d9d605),LL(0x541da730,0x397e8ce6),LL(0x06c00c29,0xb4a5d672),LL(0x43c098bc,0x76423a86),L_(0x000001bc), + LL(0xf6656606,0x82cb8136),LL(0xd96edef6,0x84c50ccd),LL(0x9f0978ff,0x37e0a146),LL(0x17bb0d3a,0xbf780900),LL(0xb2dca4ae,0x1d528632),LL(0xb6bd3e16,0x1bee4b87),LL(0x8c609327,0x16432d3a),LL(0x4aa7829a,0x9ea4ecd7),L_(0x00000068), LL(0x3f9c377c,0x5046622f),LL(0xa18dda52,0xd98abc09),LL(0x722fdd39,0x23fb42b0),LL(0xa78f3825,0xf2a75675),LL(0x13487db2,0x33200560),LL(0x244aa1c4,0xc0bf37b5),LL(0x86de25a6,0x57b73e86),LL(0x079d95dd,0xb0aa0cbc),L_(0x00000013), + LL(0x17341b4f,0x1e7f52de),LL(0x52ebce6c,0x882af4b0),LL(0x673d8b9f,0xbbd95fb1),LL(0xd64ea8ef,0x28e628db),LL(0x7889079e,0x54b7908d),LL(0x26e0abe9,0x3df2e6e8),LL(0xc20813db,0xfde7f3a9),LL(0x978bffe2,0x4f06b898),L_(0x00000019), LL(0xead10e9e,0xad34ad6f),LL(0x1076d303,0x8df8f495),LL(0x8819ee4a,0xc0d57db8),LL(0x70fb03b5,0xb14472ea),LL(0xbc0a100a,0x18cc104f),LL(0x7fbf87ac,0x45839e8e),LL(0x64d66536,0x58fe7198),LL(0x5bfbac43,0x04ede156),L_(0x00000050), + LL(0x1836614b,0xcb97f199),LL(0xc897ce78,0x6d967571),LL(0x940b810e,0xd145156b),LL(0x850c5939,0x4b73e9e6),LL(0x04a9944a,0x6e833bad),LL(0x2f7df8e9,0x2cd53823),LL(0x3b222e7b,0xf7a26c91),LL(0x1034f78b,0xb14d3854),L_(0x00000027), LL(0xfe320dfb,0x5c8c95a5),LL(0xcb9240ce,0xfffec63d),LL(0xc515192c,0x000718b0),LL(0x4259ce4f,0xdbfc0155),LL(0x6f7a6ff2,0xb1ff6013),LL(0x312bdeae,0x4cc245a6),LL(0x79a65a6f,0x29aa5006),LL(0xd3b4632f,0x920b0db1),L_(0x00000080), + LL(0x356fabd3,0x9eb2fa3e),LL(0x32e2213b,0x296648f1),LL(0x5464f17a,0xa0bf8f36),LL(0xf19ca8ef,0x4c8a5c7a),LL(0x876e29a8,0xa58d7e8a),LL(0xd8f86aee,0x0ec00506),LL(0x04f4e1a3,0x05b4072d),LL(0x1462f8a1,0x3da650c1),L_(0x00000175), LL(0x564895eb,0x5b19fee9),LL(0x8c62d05b,0x7809d8ab),LL(0x0cb573dc,0x4ad42bc1),LL(0x1884e984,0x9b3e8a9c),LL(0x6b2d6773,0x2d81f0f3),LL(0x072385dc,0x2f91b4d1),LL(0xca372aa3,0xe4038277),LL(0xd16c3a45,0x64dcfcfa),L_(0x000000bc), + LL(0x9757a335,0x1143084a),LL(0x38952f06,0xa4710659),LL(0x025fc38c,0x1698caaa),LL(0xcf127f48,0x7f55805b),LL(0x39cb3c58,0x621feb96),LL(0x58068b85,0x3a91b62b),LL(0xa4e48dd8,0xa7ba8220),LL(0xbd22ff75,0xc1b17c5b),L_(0x000000a0), LL(0x05196c43,0x142e4222),LL(0xaa56e765,0xeee6393d),LL(0x8f13ec6c,0xa88f8eb7),LL(0x536554a7,0x6720e144),LL(0x66972f38,0xadb6408b),LL(0x9d95e37f,0x67ab92ba),LL(0xe96c2792,0xa2d1345c),LL(0x3fb8e9b5,0x138c93a7),L_(0x000001f8), + LL(0x61c11852,0xf484baad),LL(0x1e71dab0,0xad2e9168),LL(0xe0ea71a3,0x6e1a90b0),LL(0x2b244009,0xfb37ada7),LL(0x0bd3281f,0x38140203),LL(0x1599d34d,0xf278746c),LL(0x3790a7db,0x17f577dc),LL(0x483a5cb9,0xbbf0dc49),L_(0x00000128), LL(0x8f5c56dd,0x6ec17c1a),LL(0x767ebf21,0xf0a141c4),LL(0xf1091ff1,0x051b5811),LL(0x3ac1c024,0x396942d9),LL(0x692ece19,0x5725cecf),LL(0xf1e6de73,0x75b56339),LL(0x2f629ac2,0x45030754),LL(0x6207c855,0x5090ae5d),L_(0x00000045), + LL(0xd14b028b,0x8c73e470),LL(0x684d5fa1,0x46af781c),LL(0xeb44feae,0x3e3aadf0),LL(0x4610320e,0xfd9c5960),LL(0xd8fffa44,0x70d9d9d3),LL(0xebbc9082,0x8a6283f3),LL(0xfcec5348,0xdd60d649),LL(0x44a603fa,0x8998ae1d),L_(0x0000011e), LL(0xce740ae3,0x5f6a91eb),LL(0x2000b013,0x3780772b),LL(0xe7ea71ec,0x0e54747a),LL(0x6f03b13d,0xfc299d7f),LL(0xd6603e33,0xb6e9df68),LL(0x86040d28,0x2043747f),LL(0x3aeee37a,0xe4608968),LL(0x9926fb8d,0x47be624e),L_(0x00000060), +}, +/* digit=54 base_pwr=2^270 */ +{ + LL(0xabb9ad39,0x6af8e430),LL(0x523480bf,0xf59d55e6),LL(0x4bc56b8b,0xc072bd61),LL(0x3df0a6ec,0x25c98f18),LL(0xbee1786d,0xbcc84059),LL(0xf26f3fea,0xb20a09a6),LL(0x79a2dfb7,0x93d600ce),LL(0xcf2e6f03,0x73c31f39),L_(0x000000e1), LL(0x9b72a39f,0x2b1470af),LL(0x4804a704,0x7da313b0),LL(0xe67c9622,0xc290e590),LL(0xbec90ccc,0x796f29ca),LL(0xf5e76e6a,0xcadb620b),LL(0x8ec01637,0x15b03af3),LL(0x4087520d,0xd8dcf763),LL(0x6c0ca6b7,0x0f970190),L_(0x0000006a), + LL(0x4f37e57a,0xc030f2d5),LL(0x649effc2,0xb84aa880),LL(0x3ad19d77,0x1cab55a1),LL(0x91bd296d,0x8c081620),LL(0xc8f7f0b2,0x0c469726),LL(0xb847d758,0x1840b8cf),LL(0xc59a8b12,0xc8fd3c1c),LL(0x0e2778fd,0x8c61a8b9),L_(0x00000186), LL(0x446cc1f4,0x3bf2f826),LL(0x4fc8626c,0x8d5f8bb7),LL(0x7df08423,0x3a877c74),LL(0x41c77ea1,0x471d935d),LL(0x5556d8fe,0xe98cde5b),LL(0x279a8287,0x068f4d40),LL(0xe400538c,0xdb305a88),LL(0xc091d74b,0x31201e77),L_(0x0000005f), + LL(0x766c809c,0xdd78e4e1),LL(0x81bcdd43,0xa933555b),LL(0xac8729aa,0xb8964c85),LL(0x4f18e8ec,0x87096359),LL(0x580f05ae,0xffab1de5),LL(0xe800a6a9,0x797b2184),LL(0xb6212cc3,0xea98c5ce),LL(0xc923afe4,0x287facf1),L_(0x000001ce), LL(0x66353c22,0xfe0b78e5),LL(0x0345fac6,0x27358467),LL(0x180e49dd,0x1ccff0c2),LL(0x92bcdaf0,0xea3ef331),LL(0xf7e4fb3d,0x99b89e87),LL(0x092ef793,0x9bdcca2a),LL(0x1418dec5,0xf9f9dccb),LL(0x314595b4,0x2d188c67),L_(0x0000013f), + LL(0x6e190a6f,0xeee766f8),LL(0x360709e0,0x5b775cd5),LL(0xd4566a98,0xe4057c69),LL(0x45df1e07,0xc0672257),LL(0x947733f9,0x1b1c2a5a),LL(0x4bcd6e2b,0xc80987a2),LL(0xe7293fbf,0x89f4061a),LL(0x11f7042b,0x9f0c87d7),L_(0x00000025), LL(0xc53c1b03,0xdaedd9c1),LL(0xc48bf537,0x495a12d7),LL(0x2c8c9765,0xe7c2d4a3),LL(0x662fe9df,0xfaed525a),LL(0x27c6bad9,0x5c4df70a),LL(0x24dd660c,0xba7fb076),LL(0x21abac8b,0x4a91b1d6),LL(0xb618ce5d,0xcf239771),L_(0x00000083), + LL(0x7875f26a,0x3212d16b),LL(0x3953e4c1,0x80a99a23),LL(0x42f909d9,0x457c9b9d),LL(0x68f18c26,0x62cfee59),LL(0x0b1b0fbd,0xd6e74f93),LL(0x99d73ca6,0x898ff611),LL(0xecc60074,0x0a0cf8cf),LL(0x0e4b48e0,0x264c8682),L_(0x0000007c), LL(0xd086fac9,0x30dd79af),LL(0xe0e34e51,0x0cb837fd),LL(0x052d2441,0xaefa0933),LL(0xbdea4988,0x44aec8de),LL(0xdfac83ea,0x46cb2469),LL(0xfad769d2,0x7cb77050),LL(0x18dd28c7,0x8001a60c),LL(0x8fe3d888,0xf467b922),L_(0x000001a1), + LL(0x5b24df92,0x2affa8b5),LL(0x207215e0,0x3c816de9),LL(0x199ff528,0xe11ab159),LL(0xfcc61eda,0xc8d67190),LL(0x661fbf7d,0xdda50129),LL(0x76defd37,0xd466e3a4),LL(0xf14fb3e4,0xc11ac280),LL(0x7620efce,0x396bb6d8),L_(0x00000179), LL(0xb257b1f0,0x4204720c),LL(0xa9d04719,0x620bab0b),LL(0xcf599cb7,0x4b89783a),LL(0x4c38e784,0x96aa7914),LL(0xa4374ec1,0xa42b74a4),LL(0x1d57fa44,0x5d9da37f),LL(0x9e98081b,0x907073d1),LL(0xd1274d9a,0x616e9807),L_(0x000001ae), + LL(0x3e1d7c3a,0xde4b8ce4),LL(0x0940b9c2,0x3515847d),LL(0x594f371d,0xc37d20b4),LL(0x0d44e03d,0x4b2281a5),LL(0xa133895a,0x03246afc),LL(0x69dc40a2,0x1243d0ed),LL(0x6acc7d98,0xc664eb78),LL(0xa9ddc8a1,0xf6274641),L_(0x00000197), LL(0x1a66ad76,0x3359c96c),LL(0xebb34cc8,0xe69f9794),LL(0xd1662749,0xf9f1455f),LL(0xb162a274,0x40fc34f6),LL(0x9d860e20,0x4fb62774),LL(0xa70d36c2,0x6f971a18),LL(0x990d79fe,0x19225101),LL(0xdd30d9b3,0x30c5bb02),L_(0x000001fc), + LL(0x51c91503,0x673999f7),LL(0xfe14668b,0x08c22b2a),LL(0xfc300d5b,0xecabf6a5),LL(0xe178c0bc,0x020b90b3),LL(0x79d38258,0x81c171fe),LL(0xa2f11763,0x86f32623),LL(0xf3a66cde,0x6ff64b9b),LL(0x5668f5ac,0xf47332c5),L_(0x00000014), LL(0xac181251,0x3ea58c5d),LL(0xde279aea,0x356a5f10),LL(0x7e9f7153,0xd08295b0),LL(0x86ce9eb4,0xf7c783f5),LL(0x4daab1a3,0x030b2d7c),LL(0x603300a0,0x2198f316),LL(0xd0c0475c,0xba184aa9),LL(0xa6fe88ca,0xe6bb2b11),L_(0x000000dc), + LL(0xade6d9e8,0xaf408fd2),LL(0xfdbd3baf,0xf2f9bf1e),LL(0x437c785f,0x502fb232),LL(0xeb964445,0xeaf5771f),LL(0xb08bd2ce,0x4c23d40b),LL(0x71ed9783,0xa9fbdb0a),LL(0xf481a53f,0x3a79f04f),LL(0x8241d897,0x33076e6c),L_(0x000001be), LL(0xdbc73cb0,0x6c1463d2),LL(0x900c1c58,0x38e9d58f),LL(0x9115aabb,0xc0a95554),LL(0xd544f068,0xb7066734),LL(0x00d18a77,0x10cc9da3),LL(0x482d67a5,0xf9d64b4f),LL(0xfe8b6e80,0xff8c0ceb),LL(0xa32070d7,0x7154c1c3),L_(0x00000058), + LL(0x1fdd6bf9,0xbe1106f7),LL(0xc44a87ae,0x43749975),LL(0x3d0c3b77,0xb45ea397),LL(0xf0fbd03e,0xd44cf903),LL(0xb4c5c47c,0xaa4a5ca8),LL(0x38153aea,0x2980014f),LL(0x58f964fd,0x7baccd46),LL(0x2458a3d8,0xeff4ee59),L_(0x0000010e), LL(0x9c8ef0d9,0x8ec22b8f),LL(0x99582fc5,0x2b8aa8c6),LL(0xd629663b,0xcebbdb32),LL(0x2ff405c2,0xbd666f05),LL(0x8989d659,0x6c986174),LL(0x2eb13e6b,0x7fa4c2f5),LL(0x5660e9a9,0x30206c27),LL(0xcd2e9b5f,0xc16a517c),L_(0x00000126), + LL(0x1b2c97bc,0x3e8c6c5f),LL(0x97ea6bef,0xe6975580),LL(0x1fa05de0,0x45dcf226),LL(0x28ce5e22,0x6b3296e1),LL(0xbc5ea09c,0x355e867e),LL(0x171e0d4c,0x93a02b45),LL(0xef953fca,0x72562a6a),LL(0x283e85d5,0x0a4baa50),L_(0x0000012e), LL(0x00384cef,0xc61925fd),LL(0xb15e2b9b,0x98c42ea4),LL(0xe51f203e,0xa9da7eaf),LL(0x80b7c7d8,0x00d05b8b),LL(0xd0e8cb9d,0x5b984aa6),LL(0xe2223126,0x97eb783a),LL(0xcbd154b4,0x60eeeb46),LL(0x77d65106,0x51c5f225),L_(0x00000100), + LL(0xecba5c8d,0x4c52e9d2),LL(0x1a2c0764,0x084d971b),LL(0x04071452,0x5b00ba29),LL(0x420810c9,0xe37ace16),LL(0x8726c12e,0xfb3b3465),LL(0x76e95cdb,0xddb8f121),LL(0x5a782ea1,0xe1266546),LL(0x9e91fb9e,0x2763e598),L_(0x00000003), LL(0xde0c16d5,0xb01d1b1a),LL(0xcdfa5a5a,0x186cc016),LL(0x907f643d,0x951f20c1),LL(0x19ce2692,0x499758b8),LL(0xa1e463db,0xae173a15),LL(0x1a60551b,0x9960164a),LL(0xa7db4dec,0x5c5b509d),LL(0xd9cec887,0x8bf36254),L_(0x000000a4), + LL(0x16cce787,0xd4a2fc01),LL(0xa425fa18,0xf155769e),LL(0xaf00539b,0x60eb1b90),LL(0x688fdaf6,0xfa6d7481),LL(0xf34ab7ee,0x4f289d5b),LL(0x07dbb72e,0x1e391abd),LL(0xbc2da7ee,0x95c48dba),LL(0x566bd167,0xd547663d),L_(0x00000043), LL(0x9ce2304f,0xd19a287e),LL(0x13ab6992,0x4405fdb4),LL(0x96fa0864,0x4139a060),LL(0x41a760d2,0xbccd999e),LL(0x5ba64bc7,0xb10009b9),LL(0xd8deab9a,0x984258d2),LL(0x79776c54,0x221ad688),LL(0x10e8fea6,0x936b124d),L_(0x00000087), + LL(0x5b9e9b6c,0x0d488b76),LL(0x718de1b3,0x479d6241),LL(0xc5e08581,0x835e01af),LL(0x2980a85e,0x5861c30b),LL(0xb6dbc1d7,0xa410ea56),LL(0x2db982c5,0xe2d1be8c),LL(0x9bd416d2,0x8b2c0849),LL(0xff9c097a,0x3959a07c),L_(0x0000017a), LL(0xbf3507bb,0x84f97fa2),LL(0x638f765c,0xc3635cb6),LL(0xddc089f6,0xadab3335),LL(0x03a11712,0x45c1fc02),LL(0x6d411e20,0x92e35990),LL(0x10bb8db9,0x9c8fd9b8),LL(0xa138660a,0x9289131a),LL(0x37f6b49c,0xa3d2ae4c),L_(0x000000b7), + LL(0xa1146df7,0xfdfcfffc),LL(0x81511cf3,0x23bc0f93),LL(0x2b2a7c87,0xdc82a234),LL(0x03b40d7d,0xa08a7dd1),LL(0x710d66e6,0x21695339),LL(0xb2de413b,0x70d88f9e),LL(0xa639823f,0x35b8b90e),LL(0x023e9ff4,0x841fb110),L_(0x000000b7), LL(0x67b34d07,0x68b9df98),LL(0xae385a70,0x9e1b7b1b),LL(0x69f1b285,0xf3ca9831),LL(0xacc09537,0xd4b56b4f),LL(0x9d566211,0x5fe60450),LL(0x8cabfe34,0xfe548cc5),LL(0xb714db58,0x6edccbce),LL(0x1b3de9cb,0xae7588f9),L_(0x00000018), + LL(0x3fb44e11,0xd6d98611),LL(0xf902dfd5,0x894e506c),LL(0xfd0b1d00,0x85a52247),LL(0x782247b1,0xee7a96a2),LL(0x73bd1827,0x817a81ab),LL(0xb5a675cc,0x58e21da1),LL(0x96f3b0ad,0xba6b1f8c),LL(0x0b4feab1,0x47a0c5eb),L_(0x00000186), LL(0x94e1e70f,0xa2caaa6a),LL(0xe4a5a160,0xc595ff3d),LL(0x4e2aab67,0xc4ca75c8),LL(0x55f145b7,0x7731bee5),LL(0xfc6003a4,0xe7fe03b5),LL(0x0bfb8f07,0xc95ac06b),LL(0x062bb217,0x970ec8f9),LL(0xa73aafef,0x436ead58),L_(0x0000004e), +}, +/* digit=55 base_pwr=2^275 */ +{ + LL(0x10edee70,0x5fd3e47e),LL(0x3597fca0,0x77535436),LL(0xd14d9e5f,0x3e8b8ab9),LL(0x09ae6cb6,0x74096598),LL(0xc8a4dd84,0x9f1a5c96),LL(0xcb6edd24,0xd2f79af0),LL(0x61d2b7a4,0x0e166e53),LL(0xfe3d22a6,0x57342692),L_(0x0000001c), LL(0xf72fa6f8,0x96d7a363),LL(0x488ad6e3,0xd92f57cc),LL(0x8510a286,0xf0a9d195),LL(0xb888aa8b,0x317136c0),LL(0xf42decec,0xdf9fc71b),LL(0xb6cc8b9b,0xc0298d41),LL(0x49e5d99a,0x109ecbb2),LL(0x314b57f8,0xb46534db),L_(0x00000132), + LL(0xb2d708c6,0xce839038),LL(0xce28ed96,0x20e1ea62),LL(0x8763eab5,0xed2713c5),LL(0xc4523fd6,0x71027fe2),LL(0x7eae1cb2,0x9c4b8cf6),LL(0x24a95e4f,0x601ad020),LL(0x07164949,0xdd7d73ac),LL(0x37442ffc,0x6e6ae643),L_(0x000000d1), LL(0xb77851b2,0xdb8574d7),LL(0xb645bbee,0x286ebfe9),LL(0x0c8710d8,0x766e45ce),LL(0xa79aecb4,0xc2d31256),LL(0xbf379f83,0x340ea164),LL(0x164bbbc5,0xf851521a),LL(0x1ac3081d,0x7e9d5d5e),LL(0xb205779b,0x9097f081),L_(0x0000007d), + LL(0xde9114db,0xe63af665),LL(0x818c463b,0x6295501a),LL(0x35a127bf,0xdce47ef1),LL(0x007d2c0c,0xcdab36d7),LL(0xccb851cf,0xfdd117a8),LL(0xf238753e,0x0f305c31),LL(0x8e2817b4,0x7fa2c0d7),LL(0xf487e902,0xc3ae8d53),L_(0x000001a5), LL(0x170a9d8a,0xfdab89fd),LL(0xd0296988,0xf7158579),LL(0x9d9469d7,0xe8a6f604),LL(0x10415652,0x2b54a37b),LL(0xcc4eb51c,0x0c719573),LL(0xae48f5a3,0xa83c1dff),LL(0x30b12c01,0x72dbb726),LL(0x57308088,0x979e1572),L_(0x0000003f), + LL(0x313e4b56,0x08231734),LL(0x6717f045,0x479ae527),LL(0xe3d436d8,0xd02cb05c),LL(0xf2257834,0x00a63fdd),LL(0xa7cf8043,0x55acde6a),LL(0x457f48d5,0x3233d0de),LL(0xde5db66b,0x81aa55b4),LL(0x0379d9ac,0x13ffc8a6),L_(0x00000189), LL(0x13e90717,0xc47bead6),LL(0xf2111132,0x90e8a449),LL(0xb92dfa6c,0x2861c278),LL(0x0e5052e4,0xbb21a8a2),LL(0xdd62ef7f,0xda29cea2),LL(0x06ce5d03,0xb1054057),LL(0x321921a2,0x2bdda27b),LL(0xa8070a21,0xe0b1ed13),L_(0x0000000c), + LL(0xb57fe1e1,0x13d1fd3d),LL(0xaf39e976,0x9d300d5d),LL(0x3c4bd73d,0x394a792d),LL(0x22ea164b,0xdb8ad2a0),LL(0x94ca8b71,0x46b5c44e),LL(0x29573de6,0x4faada81),LL(0xa68e6f0b,0x9bb3e293),LL(0x2829705e,0x8e9779a5),L_(0x00000001), LL(0x562f24c2,0x10d7ee7c),LL(0xebc4c9ff,0x1bfc2a5b),LL(0x9e849995,0x9ab67877),LL(0x29bf2cd5,0xebbab48f),LL(0x1c14b040,0xc34becb0),LL(0x0f56d5be,0xa06f84dd),LL(0x74ea8bd3,0x16998590),LL(0x240441e5,0xafa1dab5),L_(0x000001fe), + LL(0x99d23cf5,0x8778f5f2),LL(0xc8b025b9,0x1705a5c8),LL(0x3919b71b,0x0d5b88f4),LL(0xcb92372e,0x60fa371f),LL(0x943296e1,0x0a89cc71),LL(0x5fe1a497,0x34a3ae69),LL(0x5dec2f93,0x1251e4b9),LL(0x275a5942,0x8b24d4d6),L_(0x000000b8), LL(0xff47e08d,0xaa12f1b2),LL(0xad152f2f,0xffb55ea0),LL(0x2d49016e,0x9565927a),LL(0xe898a743,0x6cdbde63),LL(0x47e768ee,0x9201bbe7),LL(0x0a069ce7,0x64e8832a),LL(0x4d3af5cb,0x22cff077),LL(0x58cc25a5,0xb382381d),L_(0x000001b0), + LL(0xb0b126b6,0xaeabb8d7),LL(0xc38359f0,0x58505b26),LL(0xb9091af1,0xe930c10c),LL(0x1baa2a57,0x4ceb63b6),LL(0xf34a1cb3,0x2ce30eb5),LL(0xe695563a,0xb46502e9),LL(0xbc4a4498,0x3de11285),LL(0x7d46bb82,0xc6d8d037),L_(0x000001b5), LL(0xd0a4132d,0x4b0d3d99),LL(0x19c65f2c,0xbdec1fb8),LL(0xe96bad16,0xf98a7a22),LL(0xfc740b98,0x4ac9e432),LL(0x327482c4,0x19b02fa9),LL(0xee365754,0x2b71db69),LL(0x8c4b6fcb,0x3b059127),LL(0x1ff3d7d8,0x9083d509),L_(0x00000063), + LL(0x5b27f0e1,0xc1b9a1af),LL(0xc491a3f1,0x3c2c754f),LL(0x5cbaab1f,0x2a77d316),LL(0x0310665c,0x760e6436),LL(0xda6d64bd,0xabfa1968),LL(0xfb5f4ce1,0x2b0e1701),LL(0xb466c4ed,0x7ed3c4d7),LL(0x2ebf2125,0xdfa00f91),L_(0x00000168), LL(0x50f6e44d,0xddd0d096),LL(0x8ca37cde,0xacd3234c),LL(0xb7244def,0xd39cba5f),LL(0xacca56c2,0x42e4fef3),LL(0x04d3ff0e,0xd03959e1),LL(0xe6513498,0x101ed923),LL(0x40deadab,0x3a0842e2),LL(0x40cf65c5,0xe8f52471),L_(0x0000005d), + LL(0x24abeced,0x8027d5fa),LL(0x54bd40b9,0x6c207959),LL(0xaeb9dac1,0xd464b86a),LL(0x419d1ea2,0x6820d398),LL(0x5e25c94d,0x7cb4e131),LL(0x65e1ca01,0x407cd9e9),LL(0xe5ede0b5,0x9cc6a7e8),LL(0xccfcd5ef,0x5004892e),L_(0x000000dd), LL(0x38ae86f8,0x198931f5),LL(0xff746c8b,0xd54d7f12),LL(0x309a79bf,0x1246b150),LL(0x42f00081,0x77449920),LL(0xc47ea560,0x3c1ca128),LL(0x7f0d691e,0x4a7cd82a),LL(0x389f0267,0x3325d3c5),LL(0xd3d69318,0x8d437ee6),L_(0x00000114), + LL(0xdc420de7,0x40b5d961),LL(0x0c56a78c,0x0669c065),LL(0x20ea2fcd,0xd7d6512b),LL(0x0cfdc1ce,0x793f28c8),LL(0x12dc4c42,0x2a2c66b6),LL(0x65ef14af,0x8712d0f4),LL(0x498de283,0xaba3e10f),LL(0xd43378b1,0x99525eef),L_(0x00000147), LL(0x29182339,0x2743e625),LL(0x75c8a0f5,0x7cb967f9),LL(0x3b942e5d,0xa6fa495f),LL(0x2c93c4c7,0xaf911e44),LL(0xe5ea4e81,0x61393032),LL(0x453b1c33,0x6ad975cf),LL(0xbd844374,0x598fb85f),LL(0x19bef583,0x5bc44e9f),L_(0x0000015e), + LL(0xca0bf18d,0xa3d7dbe9),LL(0x4163c3c8,0xfea7d95a),LL(0xc8c760db,0xbedeb961),LL(0xbe4aaf54,0x6366da72),LL(0x184e2e0f,0x4b391d6f),LL(0xc176d3d7,0xda402a6d),LL(0x58e13d8b,0x35c88b87),LL(0x9e868f1c,0x28cd51ed),L_(0x00000137), LL(0x060d87ba,0x2fb088fa),LL(0x7bb887ef,0xd9fecef3),LL(0x5f6918e8,0xd8d0ab29),LL(0x584e5e50,0xa68549e0),LL(0x1a0e8dad,0xccee2619),LL(0x6b94fb63,0x2fe6d355),LL(0x41620a75,0x2f9f5687),LL(0x3bbe2240,0xb1f43624),L_(0x0000013f), + LL(0x7b1c8a03,0x5d3c5d98),LL(0x9b0532ff,0x8d28755f),LL(0xa6811aec,0x526696c9),LL(0x1a05c762,0x85c6fe67),LL(0x9b509178,0xb1cd9732),LL(0x9ce1e435,0xd6424b12),LL(0xd25c0017,0x7dfb69b5),LL(0x8fd3449e,0xb06e65bf),L_(0x0000014e), LL(0xdb3b3b47,0x4f13caa8),LL(0x8c65da30,0xc42581d3),LL(0xf1637449,0x31ba26e3),LL(0x086fb178,0x1f20a375),LL(0xe70bccf5,0x794f66bb),LL(0x8fb9ec67,0x8759114e),LL(0xba15c3e0,0xbae55fef),LL(0x6274e840,0x541f86b0),L_(0x0000008b), + LL(0x9940c688,0xd17c3809),LL(0x1ad4bb7a,0x02bbb8e4),LL(0x62901ee1,0xdef212fe),LL(0xf758e2a0,0x902e165b),LL(0x41df4c90,0x813c93c1),LL(0x2d980715,0xdfe446b0),LL(0x16925c07,0x26180355),LL(0x6a7e91aa,0x2feac9e3),L_(0x00000096), LL(0xa45e236d,0xf8e5447a),LL(0x116cff34,0xe8d1be37),LL(0x05f97032,0xb0f21f66),LL(0x2f2c027f,0xa11df664),LL(0xa89a55e5,0x02fec70b),LL(0x0cbe911b,0x281dbb5c),LL(0xf7515075,0xff0a3fb5),LL(0x5eb63dce,0xde9df17e),L_(0x0000012f), + LL(0xa91392a8,0xbdba8a73),LL(0x3162337e,0x57c70feb),LL(0xfe6064d0,0x1cbf9841),LL(0xb8ea5857,0x0f9265ab),LL(0x8f4d78e1,0x4ea34ee6),LL(0x86c61019,0xe932bccc),LL(0xa4d88afe,0x9518b05b),LL(0x1b666c9e,0xdccb7c7f),L_(0x000000bd), LL(0xf8bfc49e,0x05208c75),LL(0x9df54ae4,0xd373c5a9),LL(0x79933e71,0x5b8a9772),LL(0x64aa0b3f,0xa96b6ada),LL(0x636d2c0d,0x5227ca04),LL(0xc4142eed,0x6e0ffb64),LL(0x11d0f26b,0x03baa051),LL(0x6cce8a32,0xd1fa3649),L_(0x000000b3), + LL(0x1b22c75f,0xf889ab10),LL(0x198462a9,0xdc726c9f),LL(0x4488dc01,0x03497dbf),LL(0x6173d9f9,0x44668664),LL(0xa2ec1a91,0x97d57504),LL(0xc515fce6,0xe38e8b7b),LL(0xc94145f8,0x9a57e3c8),LL(0x7f491462,0xf8e43df6),L_(0x000001e3), LL(0xfab74bc2,0x21aa76a0),LL(0x5cde4de2,0xcabcd328),LL(0x58febacf,0xfc5df376),LL(0x164a8b43,0x53cf2abb),LL(0xd59cb6ca,0x4bdbc3e7),LL(0x5fa72448,0x61888f39),LL(0xec291663,0xa63d2680),LL(0x8efa4a04,0x6b003337),L_(0x000001c7), + LL(0xbfc44034,0xda15115c),LL(0x7be6519e,0x4c0ca17c),LL(0x1e404697,0xd39b899e),LL(0xd56cb968,0xafaffc6b),LL(0x977f25af,0x0c48baa8),LL(0x33c5e846,0xb8ddcae7),LL(0x8db3dd6c,0xea4e7b4d),LL(0x30c42ef0,0xdf2a5d47),L_(0x00000075), LL(0x3bded433,0x1c36485b),LL(0x60a85776,0x3a9ef9b4),LL(0xc163b2e2,0x9caea119),LL(0x699c32d3,0x6fbf2af8),LL(0xc9afc21d,0xb2f30acb),LL(0x5cd46105,0x2782e179),LL(0xc5de1ebd,0x94bd0296),LL(0x40db331e,0x1d32d3f5),L_(0x000000dc), +}, +/* digit=56 base_pwr=2^280 */ +{ + LL(0xb3566c4d,0x70856265),LL(0xffa63a05,0xcbace31a),LL(0x64645336,0xd792b4ed),LL(0xe49945b2,0xcdc41c6d),LL(0x4ffedb2c,0xfc3fec1e),LL(0xfb381239,0x6c094341),LL(0xb5868f95,0xa828185b),LL(0xf680572d,0xb8368ada),L_(0x000001ae), LL(0xbfe0585a,0x480f8f0f),LL(0x30bd3b95,0x5be334d5),LL(0xc2d3c86e,0xd762f278),LL(0x676d6c82,0x1488b56b),LL(0x539dec8e,0x756194ec),LL(0xc0fc3e4b,0x4e5ad8a2),LL(0xe01cce49,0x1e1d4129),LL(0x9cb7e94c,0xb0ed4c45),L_(0x00000145), + LL(0x2ee172f7,0x76d70712),LL(0xcc86b737,0x07937b43),LL(0xc42e1bf4,0x0a15775a),LL(0xe0abab13,0x8a6f4155),LL(0x8a4930e5,0x7af5f75c),LL(0xe4d25a23,0xf7ffdb8e),LL(0x47745ba4,0x1c8fe7af),LL(0xda1f09c1,0x298ae467),L_(0x00000026), LL(0x58ab833a,0xbed40193),LL(0x74f45669,0x07b7fe4d),LL(0x00a5516c,0x8fed834f),LL(0x35a36d31,0x46f3d5f0),LL(0xcf8a31bb,0x46e1df3b),LL(0x86c99729,0xef24a7e8),LL(0x383ab5e1,0x3285864c),LL(0x85a50c0d,0x14d71a5b),L_(0x00000172), + LL(0x82d65b66,0x2c2925e3),LL(0xc79d0387,0x80fde81a),LL(0xa2432027,0xbc3a2b38),LL(0xa34422b6,0x24a6595f),LL(0xb948d55e,0x7ed8f149),LL(0xfac14efa,0x1011867c),LL(0xe578bbe7,0x01bd1e94),LL(0xefa02765,0x4d118803),L_(0x000001b0), LL(0xc9f7389f,0x4f58dfa2),LL(0xd3bff99b,0x27af2024),LL(0xe0ca7fb1,0xa09a0ca5),LL(0x782dbf7b,0x01098d83),LL(0x19fa5d61,0xabdbaa4b),LL(0xddb85231,0xc03b0d70),LL(0xbb859499,0x26cba60c),LL(0x8d596ce3,0xda0073ba),L_(0x00000111), + LL(0x8aec0422,0xb67d56f7),LL(0x85f0af78,0xd0dd5f00),LL(0xd1879493,0xede77ef6),LL(0xd412929c,0x6254c309),LL(0x2f8f47fc,0x6cfbb6a1),LL(0x4af63813,0x0a561c30),LL(0x70283e09,0xb1640127),LL(0xc77dd363,0x061fa6e8),L_(0x00000023), LL(0x65973023,0x29101040),LL(0x7036678a,0x6adb22ec),LL(0x5919dc9a,0xb9607eaa),LL(0xff75ea6b,0x2f103f0e),LL(0xa1b15402,0xc4300dcf),LL(0xfdcf2e9d,0x3ebbf1ab),LL(0x1be47bf2,0x3bc7610a),LL(0xe48e43b2,0x13834ee8),L_(0x0000002d), + LL(0xb1366222,0xc255282f),LL(0xcf45bdd6,0xe5f65eaf),LL(0x15c02959,0x0d59b305),LL(0xd5074c2f,0x69d34def),LL(0xf460bf31,0x98c4daf0),LL(0xcd4dd881,0x2d35aae9),LL(0xa3e2b924,0xcd6f8fb3),LL(0x7ee19179,0x3fe40e5b),L_(0x0000000f), LL(0x3015c0c8,0x24998853),LL(0x9b18e9fc,0xc27ee4ed),LL(0x01ee8c44,0x961e30d9),LL(0x38c4d057,0x4e4722a4),LL(0x27a847d7,0xbd34c3ce),LL(0xfc175e9e,0x9ec1f371),LL(0x28ad6264,0x9962cfa6),LL(0x8cb8ba21,0xa4d69025),L_(0x0000017b), + LL(0x48414eef,0x4b16d635),LL(0x466a22f4,0x91c53690),LL(0xb9b4826e,0xdfa700b4),LL(0xba41d6fd,0xa9d1c269),LL(0x3f48b184,0x9a1ae562),LL(0x1d66af1a,0x88445e2f),LL(0x16e76216,0x5528a0ef),LL(0xa509e874,0xe8df0ea0),L_(0x000000c2), LL(0x4d16c4eb,0x405c88d6),LL(0xcb159002,0x21995ea2),LL(0x9206340d,0xc1b476f5),LL(0xbdb47138,0x73c4a87f),LL(0xf1fc51a6,0xd81d7d81),LL(0x68d2c132,0x3035e2c5),LL(0xc2e86c33,0xdd1981e0),LL(0x25fcaa15,0xa3e61c9f),L_(0x000001f3), + LL(0x4ab2a49c,0x70aadb7c),LL(0x983438d3,0x9f4ebf13),LL(0xd25c9ac8,0xd8d4d610),LL(0x9c7f0f75,0xca14e0b4),LL(0xdcfed5c3,0x4d6f7590),LL(0x36f5cd7f,0xd93cbfaa),LL(0x65cb3d17,0x1ddce79a),LL(0xbd97f101,0xbf7024b4),L_(0x0000014b), LL(0x8b012070,0x77a85a51),LL(0x279a4494,0x7671964d),LL(0xff88af2c,0xf271d11a),LL(0xdbb6c2c5,0x82395ca1),LL(0x85ba326b,0x98f43101),LL(0x0cb73c28,0x63dc513c),LL(0x6b203054,0x4469f278),LL(0xc5c18db9,0xbad9b61f),L_(0x0000010f), + LL(0x3d80adc6,0x4e403a26),LL(0xab320624,0xad04e1de),LL(0xdcbb6130,0x2a259720),LL(0x6e850532,0x231b1ad6),LL(0xa60fb3f5,0xcd79b6e2),LL(0x663d49e1,0x179c366e),LL(0x01277eb6,0xe6c6ea0d),LL(0x883c4ffd,0x0c383473),L_(0x0000006c), LL(0x7e40167f,0xaec71f68),LL(0x9a231d95,0x1c63bde5),LL(0x4af79a44,0xdec74dfa),LL(0xfd79e68d,0xa1952760),LL(0x0b613ae2,0x08e61ca7),LL(0x9e73036b,0xe30d2b54),LL(0xd922e0f3,0x6d8fb383),LL(0x28c14621,0xa421baa0),L_(0x00000194), + LL(0x1137f8e6,0x66cf0839),LL(0x9b11d642,0xa909855d),LL(0xc2008fb3,0xcfac98d0),LL(0x7141e8cb,0xf021a4df),LL(0xf143c405,0xab358375),LL(0x67bc2904,0x17ae0177),LL(0xcc509637,0x3e96013d),LL(0xa1e7d9dc,0x4facea49),L_(0x000000db), LL(0x957910b6,0x497bb1a3),LL(0x0139fb02,0xbd519647),LL(0xb1e83186,0x3a05bf5b),LL(0x19b27ed1,0xcae014aa),LL(0x82e975ee,0x2c7c3285),LL(0x88e9df86,0x55efa48e),LL(0xeb606052,0x325f6177),LL(0xa47eec1a,0x99b1ff61),L_(0x000001df), + LL(0x3a878798,0xb073983a),LL(0xb57577f3,0x911ef229),LL(0x88b3000a,0xb6a2c941),LL(0xfe60609c,0x2014f532),LL(0x505cb96c,0x56c1e07d),LL(0x4b65f80c,0xacfa88b0),LL(0x6d867481,0xe4164f16),LL(0x5599d374,0x862f8d51),L_(0x000000d4), LL(0x0c1e1e98,0xfcb0b84f),LL(0x19e6efb8,0xb9207b90),LL(0x51572182,0xa8d8dbea),LL(0x3bef29fc,0x48664299),LL(0xf1204f85,0x601bc4d3),LL(0x0e9a0fd0,0x04ed7b5b),LL(0x79ce5a54,0x2efbde9a),LL(0xa61da12c,0x620aebbd),L_(0x00000125), + LL(0x50fecf46,0x92f8cb5b),LL(0x920174e1,0xf0d81ba1),LL(0x1067e00e,0xdf0090d3),LL(0x0f944d92,0x7cf84daa),LL(0x29760c45,0xe2fe5c35),LL(0xdcd64aa5,0x0fc3d3ab),LL(0x926c41d9,0xd093a0e6),LL(0xf508ca71,0x5b0f1f7a),L_(0x0000008a), LL(0x2ec2b366,0x41b5fe84),LL(0x68d89f52,0x6a8252ee),LL(0x7f37105c,0x12bfbd7a),LL(0x28e7afdd,0xa8c8a6ce),LL(0xc9ac312b,0x64f59820),LL(0xfc0802de,0x91abcca1),LL(0xdd3daa77,0x3663e8e4),LL(0xcba87568,0xb39b3267),L_(0x0000003b), + LL(0x167f86e9,0x77ecbc5f),LL(0xb273c517,0x1e1c4580),LL(0x7b0cac28,0x7d9e9af5),LL(0x8231d2eb,0x6aedfd99),LL(0xaaffb728,0xf1c98f12),LL(0xe925ce44,0x74f1520c),LL(0xc049eb63,0x494b788d),LL(0xd9d95935,0xe226209f),L_(0x00000130), LL(0xccc2d2a6,0x0b127e1a),LL(0x3b751841,0xcd70c11d),LL(0x64418006,0x1b4ae955),LL(0xbadb0a24,0x604cb635),LL(0x9f330b20,0xcfd3da79),LL(0x73a90953,0xef2d17df),LL(0x1b20635f,0x4743671d),LL(0xd4c67374,0x2f665bab),L_(0x00000186), + LL(0x9ec2635a,0xbc72074c),LL(0x1fe4598a,0x2c820e92),LL(0xe36ea0ca,0x2497dd7b),LL(0x4a8623f2,0xe23eb8d9),LL(0x60085b98,0x40f9b504),LL(0x1c20ee53,0x201ec927),LL(0x00761ac1,0xd442c9fb),LL(0x4f448cc8,0x9ba54c26),L_(0x00000025), LL(0x6e43f9a4,0xccd45383),LL(0xdd223c39,0xe5a9e554),LL(0xb7183f2b,0xbabb5193),LL(0xcaf50569,0xd8970128),LL(0x1fd8dbce,0xa8d54145),LL(0x334b381f,0x13a9b729),LL(0x728ba78e,0x437dd328),LL(0xd0616bed,0x028df6a9),L_(0x00000104), + LL(0xaa17f0bc,0xd2eb9fb6),LL(0xcdb03b75,0x6da41301),LL(0xff96e100,0xd003b0fd),LL(0x0f341e2c,0xeffd7580),LL(0x00a92b40,0xb4e94e73),LL(0x2f420bab,0x7e020897),LL(0x3a5a980e,0x0e9d7689),LL(0x44d12101,0x6759e798),L_(0x000000aa), LL(0x2631776c,0xf32bcc90),LL(0x4363aa54,0x81eb128d),LL(0x290c3760,0xc99f8366),LL(0x24d51de3,0xa9c8e087),LL(0x2212897d,0x63ef80d8),LL(0xe1731f84,0x4ba53d2d),LL(0x71a09c6c,0xf3c92c58),LL(0x1de91d0d,0x1015dbba),L_(0x0000009b), + LL(0x007f0db0,0xfc29c9c2),LL(0xc5a04df7,0x7ce23069),LL(0x889ac9f2,0xabf8339c),LL(0x48b685ae,0x57639632),LL(0x3111646e,0xf8b9f075),LL(0x94f37131,0xb2897670),LL(0xf80a60f0,0x834d23a7),LL(0xeca2d9a7,0x27dcb428),L_(0x0000003e), LL(0xc94b1130,0x12c12cdf),LL(0x7f1285c7,0x53433711),LL(0xa3fdb413,0x6c9aab37),LL(0xd50e87db,0xc28229bf),LL(0x6e31c080,0x9878eb5e),LL(0x647c7885,0x6255df90),LL(0x3c367034,0x64a27e7b),LL(0x359e7554,0x26d5090f),L_(0x00000077), + LL(0xc3090e22,0x856faf70),LL(0xcf9c3c63,0xa537fff7),LL(0xd0317a7a,0x61c02007),LL(0xdc853b32,0x61687510),LL(0x36ccdf2c,0xd6f188d5),LL(0x26fd385c,0x2955fa1d),LL(0xf2d7d6ea,0x3087bdae),LL(0x3173148e,0x0abc6b0f),L_(0x00000121), LL(0xd784c9a2,0xc0770de6),LL(0x4bf8c47c,0x35e4c8fa),LL(0xaece660d,0x8910f637),LL(0xedb7b99e,0x5ced4fad),LL(0xa82ce72b,0x0fa07446),LL(0x15701d4d,0x94600c85),LL(0x4152f301,0xf34ffcea),LL(0xf31c15ed,0x42ddbc96),L_(0x000000a8), +}, +/* digit=57 base_pwr=2^285 */ +{ + LL(0x3f5b1c78,0x7bafc7d2),LL(0xbe5d1d02,0x50c19efd),LL(0x3cdce225,0x5357a753),LL(0x3af279d2,0x14412057),LL(0x76015a5e,0x57141209),LL(0xc91c803b,0xb203384a),LL(0x12ba72de,0x2bedd680),LL(0x825c3d8d,0x7b99091d),L_(0x00000063), LL(0x6423553a,0xeaf27fe1),LL(0xef9335b8,0xc4539fb7),LL(0xfa5830e5,0x66badc9f),LL(0x0a5e5034,0x1dcbb895),LL(0xd3a2a96a,0xa62dca0d),LL(0x8a881a89,0xd5f98db4),LL(0x06e0a311,0xe2554b95),LL(0x69efeec8,0xf070bdcd),L_(0x00000073), + LL(0xb00f4f23,0xd445a023),LL(0xdd54cf07,0x551441ad),LL(0xe62fb5bf,0x33408f85),LL(0xd275f3aa,0x1f4e87ff),LL(0x701eb4ed,0xb08c2f8d),LL(0xcbec0af1,0x3b5987fa),LL(0xf0bc3119,0x37542336),LL(0x219a8c12,0xa528b587),L_(0x00000106), LL(0x5d52f04d,0x1a68b0a5),LL(0xd5fa5e22,0x5bdff6f9),LL(0xda24e831,0x9487e3f2),LL(0x8b43c649,0x8cc54962),LL(0xca393f54,0x934d621c),LL(0x52e8acee,0x0ce12d1b),LL(0xf0e6025e,0xcb0b93c4),LL(0x16663ffd,0xe4979771),L_(0x0000015c), + LL(0x5de71627,0xf54385d8),LL(0x8b05712b,0x7bb4ab47),LL(0xcc6b5489,0x5a9a3b2c),LL(0x7fa6f6a0,0x3bfef8f6),LL(0x773cd523,0xf505a80c),LL(0x5c9cc4c4,0xa3fe8c18),LL(0xf37ae336,0x6259e110),LL(0xb440dc7a,0x10c5254c),L_(0x0000016e), LL(0x1e50c98b,0x4b379333),LL(0x4a8aa5a0,0x3b8d3103),LL(0xc900df01,0xc99c95da),LL(0x52027ee1,0xb891a1fd),LL(0x4b5be3ae,0xb6857422),LL(0x5ba842f3,0xe00bb37d),LL(0x3f36375a,0x13a7f31f),LL(0x04743a28,0x61c3d8c3),L_(0x0000018f), + LL(0x69c14cb5,0xb3bf6b75),LL(0xa0bac5f2,0xcc275187),LL(0x6c2d559b,0x0eb925ef),LL(0xa5453338,0xd0382a25),LL(0x119f0d6c,0xbbf74a02),LL(0xaf681433,0xbd994d4c),LL(0x53999a9b,0x8d27772c),LL(0x249226a3,0xd03393af),L_(0x0000006f), LL(0x796a46b5,0x887c8f77),LL(0xabbf7bee,0x45dd063e),LL(0xa0a8e117,0xea429199),LL(0x8d22e28a,0x7e69a991),LL(0xb0ecebe7,0xb0ed2dea),LL(0x0edf7cec,0x37aa98d2),LL(0x0e528886,0x0ca19479),LL(0x73078fbd,0x2624f60c),L_(0x00000169), + LL(0x1809c965,0x4cbc62e0),LL(0x5c062be1,0xeaea9560),LL(0x1162dcba,0x07340ec9),LL(0xcf90fad2,0x6f6f4573),LL(0x8b6c2347,0x41df3d0b),LL(0x4bc4f7c5,0x6aba94c9),LL(0x9ce77c5e,0x25055fe2),LL(0xb9dd3d1f,0x3fae0593),L_(0x00000195), LL(0x6e430120,0x31a5bf9b),LL(0x31ef29c8,0x4ea14b33),LL(0x1451c355,0x3d7a4759),LL(0x515170ed,0xc3011219),LL(0x21413d55,0xb375db75),LL(0xacf6ce68,0x846f6627),LL(0x2facb2b3,0xdee7b99f),LL(0x3ead4f94,0x9859484f),L_(0x000001db), + LL(0x4f0117ae,0xd22f1814),LL(0xe3424149,0xad5ce6fa),LL(0x58b2d2e6,0x70e2e877),LL(0x5de9d268,0x6eb2a9a7),LL(0x6da54502,0xddee04cc),LL(0x3079a618,0xaa37f095),LL(0xef9c41ec,0x0ab072b7),LL(0x4f475d8e,0x2bbf4182),L_(0x00000199), LL(0x139a7b97,0x482e5296),LL(0x56474127,0xc87e2a1d),LL(0x0fe9253d,0x4757efac),LL(0xbdc55bb0,0xaa65a406),LL(0xfb5b47e5,0xfa5b9027),LL(0xd99ad5b0,0x1f6adf92),LL(0x3a4363de,0x6339348e),LL(0x6d9cf0af,0x2a0a19fa),L_(0x00000161), + LL(0x926c08a5,0x58be047c),LL(0x00018285,0x4ede436f),LL(0xb7c0f8da,0x267d40d4),LL(0x754e0583,0xf821bcca),LL(0x29b55f1c,0x356838ff),LL(0xef10fd05,0xd3bb98d7),LL(0xbf4cd160,0xcf20597a),LL(0xfe3d8718,0x20dd7039),L_(0x00000074), LL(0xbe1a541a,0x2ab231ad),LL(0x833de73e,0x3217ac39),LL(0xde117a87,0xe86047c6),LL(0xa460f6e7,0xbf61ad63),LL(0x3e4086cc,0x10646884),LL(0xeedbe45c,0x97bd568e),LL(0x9c11e90d,0x23655180),LL(0xb96c7748,0x1e4144ca),L_(0x000001aa), + LL(0x469af391,0x5bb73205),LL(0x3f5f97fe,0x80d05ad0),LL(0x73b1a3ca,0x00af9b79),LL(0xb52add98,0xcc82c533),LL(0x93cc487c,0x0da2ae06),LL(0xe46cf71a,0x060c7047),LL(0xaeb64abc,0x3aa21503),LL(0x0075b1d3,0xd2fdb4ed),L_(0x00000150), LL(0x29a2bed9,0x9a79f6b1),LL(0xda4630c4,0x22374a19),LL(0x62f001a1,0x90a13059),LL(0xa3cc4dcc,0x026cefa3),LL(0xb188b4cc,0x0fbb1d3f),LL(0xad092ff3,0x36e3761c),LL(0x6354b93c,0xf3dbdbdc),LL(0xb73317cf,0x30aa2b2a),L_(0x00000025), + LL(0x478cc5ff,0x39f6b453),LL(0x4e6379ca,0x063068cd),LL(0xe4f74a40,0xdb751bed),LL(0x94052b38,0xb0c9cab3),LL(0x8ee4c1e6,0xf5c6aa29),LL(0x3ca0dbb3,0x4a2497c4),LL(0x79c6ee9a,0x9ea63ecb),LL(0x00e2354a,0x9aadab88),L_(0x000001b4), LL(0x3a6196f9,0x62b157a8),LL(0xfa37ba58,0x59648d99),LL(0xe7f1758b,0x1b51e49d),LL(0x2e82eb6d,0xbcb59206),LL(0xc0686a28,0x337d156a),LL(0x6197f5d2,0x69bbd81d),LL(0x9b64ab0e,0x45283587),LL(0x8e4d1a8a,0x49019de8),L_(0x000001c9), + LL(0xed1f56a5,0xe00396a8),LL(0xc29bf98d,0x671ac7e0),LL(0x61a8021c,0xa76a8082),LL(0x40244556,0x85eaf05e),LL(0xd4493c4b,0xacc79ffa),LL(0xf3dd2e24,0xa065de83),LL(0xc2899229,0x616b4043),LL(0x26ec1649,0xa4be7f5b),L_(0x0000011f), LL(0x177b6d5f,0x5bdfa715),LL(0xbb1b8e86,0x1c8a38c6),LL(0x673dccd1,0xb8eb3119),LL(0x26507bdf,0x5e9ab066),LL(0x52444c41,0x63e06aca),LL(0xc41d72f8,0x6233cfed),LL(0xf0a4cf0f,0xa4204c93),LL(0x2e510168,0xdcd3c22c),L_(0x0000011b), + LL(0xb9ae2c20,0xa8e73672),LL(0x60eb4910,0x76d9acc7),LL(0xef341ec9,0xf8c3ff90),LL(0x71631b58,0xc5d46ddd),LL(0xf784f3a3,0xaca2efa5),LL(0x9fd6ccc9,0xb432ab0f),LL(0xe0c19b00,0xe9336780),LL(0x8e4851bc,0xeadaf077),L_(0x00000005), LL(0xeb8a4214,0x1ec45488),LL(0xab894d10,0xb3592330),LL(0xfe84be74,0xea5cac9d),LL(0xd6ce52df,0xd6fc5829),LL(0x52b83766,0xada3c1ed),LL(0x6e5007f8,0x07f9020f),LL(0x9dc77e51,0xf5261fb1),LL(0x3d52bde6,0x7203a776),L_(0x000000ec), + LL(0x9c19cb9b,0xa33466a1),LL(0x59900a19,0x2eeef601),LL(0xd1a35b61,0x70079dee),LL(0x043f6bdc,0xcd5ea5fd),LL(0xc0119db9,0x3a3b272f),LL(0x5b2eb1b8,0xe45f974f),LL(0xba8b6e51,0x5f770445),LL(0xd0419f04,0xc16877b4),L_(0x000000c3), LL(0x4b8a6a2f,0x21c82a7e),LL(0x356eb5d6,0x1f805802),LL(0xc510787f,0xb2d54598),LL(0x678fa9fa,0x14375d0b),LL(0x27fba413,0x6ed82aca),LL(0xce44cfc3,0xbe259313),LL(0x826f662b,0xdd1eec94),LL(0xc3f7e810,0x387cd50e),L_(0x0000011e), + LL(0xafd7f180,0x784a36b9),LL(0xa71f98dd,0x55359374),LL(0xb80637d2,0xeb356304),LL(0x319eb954,0xe36b826f),LL(0x6caacbc8,0x5436ed41),LL(0xc8f9a6b4,0x1ad27bfc),LL(0x05822de6,0x66b9e6c5),LL(0x6448fc9c,0x3e7fb461),L_(0x0000008a), LL(0xd306ac81,0x92fe9fac),LL(0xdda81241,0x90144259),LL(0x08d31ec6,0x70a69700),LL(0x59532bbb,0x548a4797),LL(0x25db5e1c,0xfe84a6c9),LL(0xdb376141,0xd82e648e),LL(0xaf5e43aa,0xd86080c6),LL(0xa3c129c0,0xcae35c47),L_(0x00000037), + LL(0x2343be89,0xd0ffaccb),LL(0x1cb1ffd2,0x3a03a0c7),LL(0x899f4ff5,0x6266f542),LL(0x5f5c983d,0xbcc25c41),LL(0xcccfd128,0xdfd7dc3b),LL(0x971841bb,0xb315e6d3),LL(0xab458be6,0xf423c907),LL(0x18de71ba,0xc4028ce5),L_(0x0000002d), LL(0x8643db82,0xdf24bfc8),LL(0xae140a96,0x7d249f93),LL(0xcc1b0809,0x4a944e10),LL(0xbb9f2bb4,0x2cf2ab30),LL(0x2a9df9a6,0x3e7a3348),LL(0x8877de2f,0xc1bae4c1),LL(0x6e777963,0x95df6e0c),LL(0x12289ec3,0xc2fe3876),L_(0x000001c5), + LL(0x84f63e6e,0xb92016c3),LL(0x9065995b,0xcfc7edd3),LL(0x7e853e34,0xe921ec35),LL(0xe5b9d192,0x48df779c),LL(0x22c1257c,0x2377e36c),LL(0xc67f15b2,0x2dd7559c),LL(0x56741ee4,0x8133583f),LL(0x266292d1,0x3e412477),L_(0x00000034), LL(0xcd5b0dfd,0xd6ec8a55),LL(0x0e18cbd7,0xa02b3661),LL(0xd0b4c82b,0x54051775),LL(0x2328d0e9,0xa2bff3fb),LL(0xbab00086,0x8724078f),LL(0xf6183452,0xe2d3f99f),LL(0x1d9f7aa2,0x3419a97e),LL(0x6878b1f4,0x67fe5413),L_(0x00000051), + LL(0x6c03f366,0x7fd85da2),LL(0xd5f694e1,0x52fd006e),LL(0x2f043a61,0x51032d25),LL(0x8bc9cc74,0x9348d55c),LL(0x6f5370ca,0x56333c4c),LL(0x3610540b,0xc9a5ca53),LL(0x716d25cb,0x39d8071b),LL(0x7337f70a,0x6bc9d236),L_(0x000000ac), LL(0x97db6fc5,0x3251b143),LL(0xdb755cfd,0xd84aa2bf),LL(0x0cc3e62a,0x9e9e3810),LL(0x6071f1f8,0xe47fb104),LL(0x3e7012d9,0x97ec5c7c),LL(0xf6c7e6ad,0x98bc4de4),LL(0xa4e7cef6,0x240c6a07),LL(0xa03a3a12,0xf8236198),L_(0x00000070), +}, +/* digit=58 base_pwr=2^290 */ +{ + LL(0x7663b161,0xc93f0cc5),LL(0x071af3d7,0x22d6ac11),LL(0xb9149bdc,0xe6312d84),LL(0xe44e4632,0xc50d4c88),LL(0xc448cc8e,0x6c85277a),LL(0xbfe4f89a,0x128700ea),LL(0xa38e5f2e,0xb742928e),LL(0x3e261880,0xbb296e29),L_(0x000001bb), LL(0xfa51028c,0x9b6a14e2),LL(0x09549191,0x82dfd5da),LL(0xe13022a3,0x233ca662),LL(0x96fafc24,0x505fe429),LL(0xa18dea4f,0x96182166),LL(0x15ea5a2d,0x199ba558),LL(0x22a4ac80,0x33772326),LL(0xb13c3b81,0x276d1311),L_(0x00000182), + LL(0x215bf15f,0x61cb09c8),LL(0xe5912a10,0xae2de789),LL(0xd84851c6,0xd74ceccc),LL(0xb95ccd21,0x6a285101),LL(0xd32dddf2,0x0122d3f6),LL(0xdb554921,0x02c5d952),LL(0x96a4aa1f,0xb24be997),LL(0x8cde88aa,0x7caeb407),L_(0x0000008d), LL(0x2cd753b8,0x88aff9b9),LL(0xcc49d782,0xf7cdce61),LL(0xdac4e445,0xfad48cc3),LL(0x0ac2a937,0x956fdfcb),LL(0x98c5bdda,0x81841ce2),LL(0x9f12bb3e,0x170e6c81),LL(0xcab58ad5,0x30efd73e),LL(0x76a3a481,0xabf3f0f4),L_(0x00000010), + LL(0x361c2c61,0xb9e4b6ab),LL(0x810de58e,0x3d0db7a6),LL(0xe085a8d1,0x45a31ec7),LL(0xa7bb9df2,0x35378c99),LL(0xd8933f30,0x186da525),LL(0x1033d24d,0xddb7a3b3),LL(0x8af19819,0xb5c9012c),LL(0x57d17203,0x722478c1),L_(0x0000002f), LL(0xce3ccda4,0x41697bbe),LL(0x5c6d7a27,0xb8be57fb),LL(0x1fba9fbf,0xc562ecca),LL(0x9a3fed12,0x1bcd8090),LL(0xf597a3fc,0x74a6954d),LL(0xce4e5ded,0xb4910fbc),LL(0xa5ed9cf8,0x79902452),LL(0x9c77346d,0x4ebfad77),L_(0x000001cf), + LL(0x3a4f332d,0x3d82a438),LL(0x56af6321,0x0fc06213),LL(0xca05acfd,0x8864ca32),LL(0xe80103ea,0xc2e9c7c1),LL(0x99dd0ff4,0x4c64b758),LL(0xc9889d76,0x80c128a8),LL(0x881a1256,0x77fdb7cd),LL(0xf09f58ad,0xf478aec5),L_(0x0000018c), LL(0xc6ecd4f9,0x41a633a8),LL(0xecb1fc4c,0x847c2f58),LL(0x92dae51a,0xa7d7b295),LL(0x268f50e3,0x3b5eef6e),LL(0x3a27de2a,0x0d7a599e),LL(0x14916d54,0xd01f9b57),LL(0x204fbca0,0x675e52e5),LL(0xeb48615c,0x50dac3d9),L_(0x0000019b), + LL(0x0a639042,0x0fd5737f),LL(0x31d2eb63,0x801bd86d),LL(0x85ffa7ea,0x1011c35d),LL(0x8d043e51,0x1ef5b87b),LL(0xcb405068,0xec30dbd9),LL(0xc20daf68,0xe48310c1),LL(0xcee24a41,0xa65b8aca),LL(0x119d1da9,0x4a04288e),L_(0x000000ec), LL(0x716e9def,0x8b2825b1),LL(0x5d82926b,0xb4cd0082),LL(0xe1a8a7c8,0x6474e309),LL(0x12620e3a,0xb0da6f90),LL(0x2d673d4a,0x574adf3d),LL(0x628b88b6,0xb210b971),LL(0x9d1b96a3,0x6b2d573e),LL(0xedcd56fc,0x8269ac81),L_(0x000000b1), + LL(0xe07f6e9e,0x0f0b6a27),LL(0xa4a6f307,0xddeaaa37),LL(0x9c430a1d,0xeacad6b5),LL(0x9620fd47,0x9a8128c0),LL(0xf279790d,0x3bf5952b),LL(0xfb97ad6f,0xe0561485),LL(0x0fe7692a,0x482c591f),LL(0xe268f3cb,0x8a7a037f),L_(0x0000013a), LL(0xdb21bfbb,0xde3dc674),LL(0xdbf154f3,0xf4401caa),LL(0x32e63083,0x462197d4),LL(0xb9452cf6,0x46240ddd),LL(0x10344368,0x1c6dbfe6),LL(0xa986f17f,0x94ccbb69),LL(0x4632a20c,0x3f6277d7),LL(0x33029382,0x0f710d97),L_(0x00000010), + LL(0x2b718a28,0xd8752e23),LL(0x65e87e31,0xa158249b),LL(0x88c3f123,0xff7b1118),LL(0xd9121432,0xbe4461fe),LL(0xa0850e4f,0x9ef5bc2d),LL(0x6350e71c,0xf28780a7),LL(0xc6dabd80,0xfc8f574b),LL(0xc3c266b1,0x7bb0ad43),L_(0x00000016), LL(0x13bf6ff9,0x7db218bc),LL(0x9d000699,0x0dec75e5),LL(0x7cf7628c,0xb95f2df4),LL(0x2fa7aacc,0xe8ff9a7d),LL(0x96555722,0x1076129b),LL(0xa11a1984,0x57afac1e),LL(0xfd1e9ec4,0x16d64a31),LL(0x008b70a7,0xf6e81632),L_(0x00000000), + LL(0xb0d9eaa8,0xf5caa2dd),LL(0xddc322f7,0xc31038c0),LL(0x08fb4b57,0xa651596a),LL(0xd9ca6980,0xab32e2a6),LL(0xc95c78a8,0xe5808eea),LL(0x5a32ba78,0xf5f9923d),LL(0x3bbece34,0x26ad1c8d),LL(0x8f8b8459,0x84189709),L_(0x000000ce), LL(0x16843645,0x8875d753),LL(0xa90e9fa2,0xe13608c5),LL(0xaf90c364,0x57e5556e),LL(0xcc40e058,0x9e332dda),LL(0x9c5012b5,0x2b76768f),LL(0x8a76230b,0x2932d53c),LL(0x573bdff3,0x14999fbd),LL(0xeee93001,0xdef0f840),L_(0x0000017d), + LL(0x1a3f60e5,0x7e88fc70),LL(0x29625c0d,0x396b1851),LL(0x610e5833,0x947ef062),LL(0xd8dd1f5d,0x47f1a571),LL(0xa0f65294,0x7850d950),LL(0x49f087e9,0x22c8e733),LL(0x18807434,0xce5508b2),LL(0xd0fc8fca,0x25125ea0),L_(0x00000121), LL(0xc3d1360f,0x69072d8a),LL(0x8b9d1e81,0x8cba6305),LL(0x01ed34a6,0x7a1a3844),LL(0xbc37f296,0x20c61572),LL(0xd409e84e,0xd54640e7),LL(0xeb6c948c,0xc9243fc3),LL(0xc754fba2,0xb39c9166),LL(0xc28f4f28,0x15bec182),L_(0x00000188), + LL(0x32c7f33f,0xa0e6751c),LL(0x47474b7e,0x7f265069),LL(0xe1e4a2ce,0x0460d889),LL(0xeab5839d,0xb51b9a7f),LL(0xe7cad388,0xa5032a25),LL(0x46ee855a,0x6621ee7b),LL(0x16fbfccf,0x5a6f1501),LL(0x73af1329,0xbadeb575),L_(0x000001f6), LL(0x694e09ae,0x0646542a),LL(0xaceb179f,0x27a867a4),LL(0xeb30df16,0x14cf3975),LL(0xf9d85fe3,0xa18e96b2),LL(0x37e6f97d,0x7781a0f2),LL(0xb09bd1ce,0xa8de0b13),LL(0x278a1089,0xc3a91cf5),LL(0x02296583,0x1b9593f9),L_(0x0000000e), + LL(0xeca2c791,0x86fe1661),LL(0xd1c18fea,0x4aedcbc7),LL(0x3f879f46,0x0545d544),LL(0x4cb96993,0x95120a10),LL(0x595026f4,0x1d335198),LL(0xc959c824,0xa814ec2a),LL(0x8fbede3e,0x0e062b6d),LL(0x3832b5b2,0x6bbcce86),L_(0x0000017b), LL(0xfad2bbc3,0xcf89249c),LL(0xf01f3803,0x63266c42),LL(0xb86898e0,0x685db0a5),LL(0x3becca8e,0x7e4eaa63),LL(0x3ec31fe1,0x90fcf86c),LL(0x394a64f7,0x967f0628),LL(0xc6f81bcd,0x635c81b7),LL(0xec462896,0xb8022758),L_(0x000000f1), + LL(0x67236236,0x67484461),LL(0xdb36fbe6,0x5f5c47c0),LL(0x9fb54024,0x71f2fbb3),LL(0x44525a78,0xbedf63e3),LL(0xe71375bd,0x9f085fb8),LL(0x47d50bd8,0x1c59b6e6),LL(0x2f2ea430,0x578031fa),LL(0x58012b66,0xd2034d9b),L_(0x0000007e), LL(0x473015fd,0xfde2151d),LL(0x9616e82c,0xe4d908c9),LL(0xd4ec3b2f,0x04a04977),LL(0x13df9ad2,0x3ee923f5),LL(0x2b66641e,0x175bb5d9),LL(0x0fcd9df2,0xcdb5c3c9),LL(0xfac57254,0x7fde8809),LL(0xc6981a62,0xd951b985),L_(0x00000091), + LL(0xb19296e9,0x9d659d99),LL(0x6d5aa5b9,0x489a28d9),LL(0x5422e69c,0x5cf35829),LL(0x06993c4a,0xed0dce41),LL(0x4bccea69,0x21d11ff6),LL(0xa2e82c7c,0x44f73388),LL(0xda3168f8,0xf6117d7c),LL(0xdff018c7,0x0454467f),L_(0x00000112), LL(0xeb022661,0xe965ba34),LL(0xfeff852b,0x1a68518b),LL(0xbe9a9ee4,0x53aa84d0),LL(0x31f46a2c,0x112327ea),LL(0x6855b874,0x06311411),LL(0x43d26e75,0x7348f329),LL(0x65628948,0x0582ac08),LL(0xe3244339,0x44f3001a),L_(0x00000184), + LL(0x5a842868,0x12b97327),LL(0xd4a2fefb,0x8f7a410f),LL(0x84c0e584,0x98e19862),LL(0x3fbe93da,0xe566e4f8),LL(0xe44a9540,0xec1d03a0),LL(0xa377131e,0x1b99313e),LL(0x27336d2c,0xc15f4f38),LL(0x8c27d958,0xeeef149e),L_(0x000001b6), LL(0xe75811d3,0xc54aa0e8),LL(0xe1cbcf6b,0xc195aaf4),LL(0x5596682f,0xe98a5845),LL(0xe0cf229c,0x70256db2),LL(0x5a921b26,0x10d830e7),LL(0x37fc26fd,0xe3def649),LL(0x16a810c8,0xdb834a77),LL(0x613433c1,0x71908e34),L_(0x000000f8), + LL(0x7624b24b,0x7db58054),LL(0xabebbd07,0x20618f15),LL(0x3a5e2752,0x72097df2),LL(0x4f72e3fd,0xf79b4cba),LL(0x6f03686a,0x3cfd9643),LL(0x2d89778e,0x683c4a14),LL(0xeade01a4,0x7cca2771),LL(0x533fd14a,0x7a145968),L_(0x000001b1), LL(0x5f84ba35,0xb97b68e1),LL(0x4e0b9bd1,0x6e47fe3d),LL(0xcdba9cfa,0x3026f026),LL(0x6e415889,0x057de03a),LL(0x7c12c31f,0xa54231cf),LL(0xcdfae481,0x68a6cb37),LL(0x3908080c,0x259ee9d4),LL(0xa3c797b1,0x5a90709c),L_(0x00000030), + LL(0x021a0d91,0xd49fbe57),LL(0xde5b21b0,0xbb57b277),LL(0x7291e7e7,0x1e6a4b2d),LL(0x16da29ce,0x4426f88c),LL(0x68f8b71f,0x6a6ebaff),LL(0x9995fbf7,0xab510adb),LL(0x6ec18d2c,0x8d4b996a),LL(0x3ce11f1f,0x0c8233bc),L_(0x0000004b), LL(0x8e04c405,0x34703d79),LL(0x6b0a33af,0xdd55e68b),LL(0xb14161e8,0x4737f09c),LL(0x57558d9c,0x90c00b53),LL(0x9d9a485a,0x508e73fc),LL(0xbb09dac2,0xd252e5f5),LL(0x4ba2132e,0x33b1efcb),LL(0xc58bf239,0x43e79593),L_(0x00000026), +}, +/* digit=59 base_pwr=2^295 */ +{ + LL(0x5b781a84,0x584291d5),LL(0x6ea73ad7,0x992c0a26),LL(0x20e9954c,0x169e02af),LL(0x4d73d175,0x2718e0ca),LL(0xe1612ee1,0xed50926d),LL(0xc638cf1d,0xc1060d91),LL(0xb5998df8,0x4b7dc332),LL(0x4eb7dc88,0x46da4262),L_(0x00000157), LL(0xd78eae21,0xa9f4bf2c),LL(0x372725c2,0x86c74c6b),LL(0xb5b5158f,0x736a4de4),LL(0xba4800d6,0x451f4693),LL(0x5138590e,0xd2239cb9),LL(0x6f5d263e,0x45bdc4c5),LL(0xc0f8acf5,0xd06676d4),LL(0x8bbd0743,0xbe979403),L_(0x00000155), + LL(0x714fe80d,0x2da244cb),LL(0x06f04145,0xcda8b722),LL(0x84ee9fa6,0xc3d58870),LL(0x0e111da7,0x1c267392),LL(0x53bb35ef,0x906e57c4),LL(0x6a858e61,0x3eebaa20),LL(0xf4582387,0x20dd3b5b),LL(0xcf71c4a7,0x82d3b15d),L_(0x00000009), LL(0x91605cdd,0xe98756fb),LL(0xcda7aac0,0xf0286c4c),LL(0xb4372718,0xa4017819),LL(0xdae0a5d8,0x21935131),LL(0x0720f8cb,0x261dafa4),LL(0x40e03217,0x6fb18c8c),LL(0x34851940,0xcd3c7d48),LL(0xe02770ca,0xc52abd46),L_(0x00000103), + LL(0x826415dc,0xaee30325),LL(0xe70ecca1,0x8395cad3),LL(0x053fd2fe,0x148ed662),LL(0x3520779d,0xad7a6345),LL(0xc9cad78e,0x02a99616),LL(0xb2b3d15b,0xa5bc3102),LL(0x9cfe5a4d,0x4cc19d74),LL(0x59fc0a6f,0xd407b106),L_(0x00000075), LL(0x3a1ba1c9,0x487fa3ac),LL(0xb4dfc9f7,0xe0d152d8),LL(0xf699a6f5,0x8f345525),LL(0x20633fdf,0x76ba6839),LL(0xca7d8a08,0x2bd4f59f),LL(0x88585003,0x05078df0),LL(0xcface355,0x01da05b9),LL(0xfd2eb0a9,0x05a56ca7),L_(0x0000007e), + LL(0x52b16ebe,0xf47e5b46),LL(0xe6ae36e7,0x71c81701),LL(0xe3a4cc33,0x1ae7f8d6),LL(0xb29431c3,0xaeb29c67),LL(0xb869125f,0xe16be2af),LL(0x052cfd62,0x934ce6d0),LL(0x10638824,0xe1cd7490),LL(0x2b021800,0xede192e6),L_(0x000000ac), LL(0x53a39e35,0x21f34b32),LL(0xaa3b5761,0xd9c36b4a),LL(0x2e0442ed,0x419b2399),LL(0xd2725144,0x4d374723),LL(0x0ced8d6e,0x6d58a708),LL(0x1b1f1118,0x0cd63ed1),LL(0xd7a4d0b5,0xf4c6faa6),LL(0x9c897561,0x53bcb255),L_(0x00000047), + LL(0x0cb3fdd3,0x8e8db5d1),LL(0xaee5321b,0x87aea4fe),LL(0x3857736e,0x16711165),LL(0x0c1ddf85,0x0fefea55),LL(0x5866facc,0xf4b819a9),LL(0x33eac999,0xde7464eb),LL(0x2a31e0a7,0x4f70e413),LL(0x152ce312,0xfbd1795a),L_(0x00000194), LL(0xbf9b85c5,0xba27db9c),LL(0x153c54c7,0x65beb177),LL(0x0c1db955,0x38a15bc3),LL(0x12ad15c1,0x1bbf7edd),LL(0x99f39d44,0x0cddc300),LL(0x017014dd,0x7ea43a2c),LL(0x2d23d878,0xbec5a12d),LL(0x5eda0b7d,0x53c1c755),L_(0x00000156), + LL(0x06657d80,0xd29c3627),LL(0x8365a95f,0xe70b9e02),LL(0x084d6b18,0xc68fa40d),LL(0x80bbbcfe,0xae56df9d),LL(0x8a06728a,0xb6253373),LL(0xbd4a361e,0xc77e1c92),LL(0xd3c11b1d,0xa94fbeef),LL(0xa52dffaf,0xca64acfb),L_(0x000000c8), LL(0xf19acb1f,0xe668d3f2),LL(0x23cbf024,0x095d2d5d),LL(0x7f105b84,0x89f76b69),LL(0x5b550d74,0xe73345da),LL(0xbc9d3a15,0xd2b26a8f),LL(0xd6a293e0,0xe4494adf),LL(0xe0387451,0x818e6417),LL(0xb0518331,0x48fe9fb0),L_(0x00000001), + LL(0x4d2a42df,0x808fa6d8),LL(0x68651170,0x445f0d4b),LL(0xec0d410c,0xc6980698),LL(0x1ad2d890,0x005f7ee7),LL(0xbcda7089,0xa7d283e2),LL(0xf5b48062,0xcee64fee),LL(0x0051a180,0x73e72ad4),LL(0x848f7b7c,0x9759cdbe),L_(0x00000137), LL(0x690a81c3,0x3ce8b5c2),LL(0xac33d774,0x15e254f0),LL(0xa423baea,0x948fbc87),LL(0x2fe89ca2,0x80cdde65),LL(0x9a165eb6,0xb1b05690),LL(0x1c84102d,0xc135f5d4),LL(0x73f34a94,0xc61329b0),LL(0x8ced1268,0x2e0d7d89),L_(0x000001f4), + LL(0x9713685f,0x93a123f3),LL(0x6ef5a591,0xf28bae61),LL(0x67b33050,0xda7d2c65),LL(0xae5b6596,0x59fc9e4d),LL(0x0481adbd,0x18dde4a4),LL(0x2fe92c16,0xf2a19468),LL(0x5adc0431,0xeac05bc1),LL(0xdb30fcce,0x7e2cf13e),L_(0x000001dc), LL(0xee47d7a2,0x4cf35b8d),LL(0xc425ee95,0x74908606),LL(0xcdac359e,0x22c94a88),LL(0xef2c8586,0x10d20bfb),LL(0x8d0f7458,0x1ca77f65),LL(0x426741d7,0x82f59f1e),LL(0x640314c7,0x294af36c),LL(0xec0709c8,0x18dd5e35),L_(0x00000120), + LL(0x1e52ac8f,0x0894401e),LL(0x36abe923,0x9f482e3b),LL(0x81ce378a,0x32d1efaf),LL(0xae954687,0x62ff86e1),LL(0xd9afd8e0,0x085da5ec),LL(0x5871a105,0xd026254f),LL(0xe7a32717,0x83b4a648),LL(0x40288d2e,0x8d29644b),L_(0x00000197), LL(0xb0b6dc10,0xa8898440),LL(0x699ececa,0xe53262d1),LL(0xdf9ba3e4,0xdb180197),LL(0xa82e89ff,0x786042aa),LL(0x97af8b17,0x663faa6b),LL(0x00b45e0a,0xf346c12e),LL(0x03c76f2d,0xe8ec3f00),LL(0xbda1be57,0xf05e51a3),L_(0x00000078), + LL(0x55c5c739,0xfcd89a8d),LL(0xc0dcdb75,0x71a4d047),LL(0xd498ccfb,0x8baef9a3),LL(0x669e7edb,0x20d8ae8f),LL(0x98d6b13c,0xd16a48c7),LL(0x3f4ca564,0x50d24170),LL(0x509f9dc2,0x8c680ef9),LL(0xb2c17a38,0x29830970),L_(0x000001e1), LL(0x23778808,0xe8d8d33c),LL(0x9dd269ab,0x9bd2fb18),LL(0x1e8769b1,0x62b11258),LL(0x657bacd6,0x8521c19d),LL(0x584fbcaa,0x46adc05d),LL(0xe93891c5,0x68fad3f4),LL(0x8617aebc,0x0857bce7),LL(0xe39a4226,0x50c3c701),L_(0x00000128), + LL(0x17cc51bf,0x6d70f113),LL(0x546beb5c,0xc94a2150),LL(0x62ee8e73,0x3d11d590),LL(0x255999c7,0x8c21f26b),LL(0x42e22a6e,0x3732f418),LL(0xb9e01e7a,0x910608f5),LL(0xea10cdc0,0xef7f9669),LL(0xf54f17b3,0x271d06b5),L_(0x000001f9), LL(0xe994f9cf,0x5a87a958),LL(0xe54c87e2,0x6e7d0b5e),LL(0xa8cc0493,0xb0873928),LL(0x837b814a,0x92c0806c),LL(0x1e804d39,0x8cb9df2c),LL(0x1d8a5f76,0x2377456b),LL(0x33c3155b,0xe6d63e09),LL(0x55cb5f9c,0x572ecf1e),L_(0x00000107), + LL(0xcfd79f62,0xc5d103b9),LL(0x295548e9,0x71b65a58),LL(0xb8462f3d,0x40934991),LL(0x1453c2b2,0x41c087b5),LL(0x1cf62fb3,0x37eea50d),LL(0x02ab6cdb,0x7ada571b),LL(0x1b49b692,0xda9e677d),LL(0xb52dc4d6,0x1d635004),L_(0x0000009f), LL(0x2e42f171,0xb6cceda3),LL(0xdbcc6765,0x09960889),LL(0x2e336d9f,0xe5440e8e),LL(0x34cefd7d,0xb410a81f),LL(0xd8bc6b5a,0x91782d60),LL(0x46d80cbb,0x95e22f30),LL(0xc99291ea,0x9e775cf7),LL(0xca473be0,0x73ab5282),L_(0x00000170), + LL(0x06ac0186,0xbd98b833),LL(0xdc7e7944,0xae0a1564),LL(0x0b94fb31,0x8b85aba8),LL(0xfd0db6b7,0x3fa21f5d),LL(0xbb92a7c4,0x7133a3ce),LL(0xfc2b2cf0,0xd39f3731),LL(0xb63fc2cb,0x376e5f1f),LL(0x4d89e9a6,0x3c36dece),L_(0x00000199), LL(0xe98c5f25,0x2178e890),LL(0x51c9207d,0x1be32aa0),LL(0x8d2cb6c8,0x629881ee),LL(0xdb210410,0xe3099a6e),LL(0xfd24a488,0x62aff70d),LL(0x6e705a1e,0xb843d997),LL(0x41319b69,0x8bfa95c3),LL(0xd9b9376a,0x9990510b),L_(0x00000032), + LL(0xc2b6a03b,0xfefa851e),LL(0x0a7bf6b7,0x3a92f668),LL(0xbf4905c0,0xe8a75dea),LL(0xff483a6a,0x78467396),LL(0xdc163b2b,0x8999b6fb),LL(0x8968be09,0xc4a53538),LL(0x419a12c9,0x40b8e919),LL(0xd87c8896,0xc3d8cb0b),L_(0x00000147), LL(0x68f69e36,0x5971b3c8),LL(0x53ea08fc,0xbd601db4),LL(0x0ff01a96,0xa72aee96),LL(0x347158b6,0xef1dc3a0),LL(0xc8994151,0xb70d9ea4),LL(0x39937de4,0xa2906842),LL(0x73a17885,0x1ae4276c),LL(0x34f8bee7,0x73c8f8a8),L_(0x00000103), + LL(0x7fdb683d,0x5075fbb5),LL(0x35997d35,0x4f7513bc),LL(0xb65dda5b,0xcca089f1),LL(0x0e8d30ed,0xf394427e),LL(0x66ecf608,0x7394ebe5),LL(0x80d0cb61,0x2babf408),LL(0x9903b671,0xb8208316),LL(0xe416cdcf,0xb4a46cb6),L_(0x000001fc), LL(0x26090ace,0x026550ab),LL(0xf1bd714b,0xceedda36),LL(0xd83c3071,0x6a6fe427),LL(0x5704c9b9,0x9e328311),LL(0xa3fbc241,0xfceb37ff),LL(0xf54b88ee,0x2f82304a),LL(0xb6315a8d,0x77a230c7),LL(0xeda8682e,0xb1bd2d77),L_(0x000001e3), + LL(0x5759366b,0xabadb724),LL(0x75b84d60,0x87caf5b9),LL(0x983f793d,0x1cecc3f1),LL(0x94d8de54,0xd8885c2e),LL(0xf90b687c,0xd952f2ac),LL(0xd5046b6c,0x16cc05b6),LL(0x266e5bb0,0x52f97cdb),LL(0x5959c784,0x417167c9),L_(0x000000e1), LL(0x931e70e0,0xc8989bfc),LL(0x36663528,0xc244410e),LL(0xf14f2667,0xbca645d5),LL(0xf8059cc8,0x1c94e08c),LL(0x1d2134f7,0x1d1a84e9),LL(0x3489ba7b,0x21c35b98),LL(0xdddbab1e,0xa303bfc5),LL(0x86156a8f,0xa3ec6998),L_(0x000001df), +}, +/* digit=60 base_pwr=2^300 */ +{ + LL(0xec83fd79,0x9640b910),LL(0x5166dc54,0x1f3c076c),LL(0x553ff932,0x6d2b7e4e),LL(0x7e2f7e67,0x097c11e9),LL(0xaa2cdad5,0xc5cedff5),LL(0x397f4bc0,0x57d09eef),LL(0x8d95f667,0xe7743495),LL(0xe32eada9,0x34e574ee),L_(0x000000d6), LL(0xe641f85b,0xa0e3496c),LL(0xa6b773ab,0x7931548e),LL(0x3aecbf5b,0x2ef14927),LL(0x58c42c5d,0x1528b818),LL(0xfddc70d9,0x1c157c1e),LL(0x0a328d34,0x690e10df),LL(0x0760fd8a,0xa364f5d3),LL(0x4ec3b44e,0x0d06d0d1),L_(0x0000007e), + LL(0xb187f011,0x26f1e48d),LL(0xb62010d1,0x0d05c4b7),LL(0xe0fbe5e8,0x3f0a2ba1),LL(0x24d802f1,0x189e9602),LL(0xde0b8698,0xc0b8af43),LL(0x28591b4b,0x706b742b),LL(0x36bf7aec,0x63d91963),LL(0xec0d2fd3,0x576d907f),L_(0x000000ae), LL(0x440b0b6c,0xe982e3e2),LL(0xd99d67cd,0xd814a6fa),LL(0x7edfbb4d,0x4ac7d349),LL(0x46c6afc8,0x63bbd77a),LL(0x84cd907a,0x18bcc3d6),LL(0xf098909a,0x756b5193),LL(0xd6e0581d,0x02f37ab5),LL(0x06adf4d1,0x2f363ffb),L_(0x000001ce), + LL(0xe90415e4,0x6bd8544c),LL(0xc664d2f1,0x3e7600d6),LL(0x01912f05,0x1e8d9aa4),LL(0x3e0c268e,0x43dcdadd),LL(0x2f140134,0xbf936e21),LL(0x252cf59c,0xb8aaec39),LL(0x00b8cef4,0x7ed652c2),LL(0xc64c11e4,0x639c50aa),L_(0x000001e9), LL(0xa7c08e2e,0x59176893),LL(0xafe8a484,0xc7c5a8f0),LL(0x6f043d0d,0xe33e999b),LL(0xb3c3cfd6,0x2a496c00),LL(0xcb4fbc5e,0xe2690de5),LL(0x1dfaf5ee,0x3d2db451),LL(0x743e9277,0x8bfecdd2),LL(0xade6a194,0x5bfc14f4),L_(0x000000f4), + LL(0x8d039609,0x488fdf25),LL(0x1e037339,0x0e39945a),LL(0x4d16fa60,0xf539a844),LL(0x54408c23,0xf7f8ccc4),LL(0xbef729fb,0x69b8abc6),LL(0x76661fe3,0x3dbd87a4),LL(0xa8903415,0x790f6266),LL(0xc7b4bcd2,0xbe422de3),L_(0x00000195), LL(0xa406e205,0xce1af477),LL(0x72e7763c,0xf7fcf645),LL(0x43d00999,0x1f7c9317),LL(0x02adccdc,0x8b87a139),LL(0x541be26f,0x9ffcb96f),LL(0xbb590677,0xc0636264),LL(0xc0db1d61,0x19484331),LL(0x5585c8ae,0x3548f76e),L_(0x000001d9), + LL(0xe85899b3,0x5fff92e2),LL(0xb4a44f31,0xa8cf599e),LL(0xca1a3e87,0x17a2e4cf),LL(0x205f34e2,0xe4f28f3f),LL(0xe9eb1362,0x4aa7c205),LL(0x6ace61c8,0xfa76515b),LL(0x7e550586,0xec83aca3),LL(0x35e870f8,0x982b8304),L_(0x000000ef), LL(0x2e02203e,0x318327d4),LL(0x004d9655,0x5d903904),LL(0x7cd1d2f1,0x9d294adb),LL(0xca1242c8,0x3b5bb4eb),LL(0x4af2a142,0x93818e82),LL(0x15e366f3,0x9a304441),LL(0xd6a53de7,0x49183b2c),LL(0xfd324d82,0x27bcf47b),L_(0x000001a2), + LL(0x4d1ee196,0xa1d04903),LL(0xe53d6718,0xacaf386a),LL(0x6605e4e6,0xcc306d74),LL(0x458e136f,0x77dc40c4),LL(0xab7a1ac6,0xbb331955),LL(0xd47f6ee4,0xb95c386e),LL(0x43841037,0x526640fb),LL(0x00a37bb7,0x59e53810),L_(0x000001b6), LL(0x8b8cc55b,0x71d84cd7),LL(0xe7e03251,0x1b8eb12e),LL(0x64c45f59,0x2d7bd8c5),LL(0xc2283df1,0xb03bfc76),LL(0x28a36461,0xafd6fc81),LL(0xa0580c8c,0xe72b6275),LL(0x511c376a,0xce438282),LL(0x0ca3213f,0x379b6878),L_(0x0000011e), + LL(0x1e2c0f6f,0x17179cb8),LL(0x0810156c,0xaf140db0),LL(0x63cd1fa8,0x2171bb5e),LL(0x729533d8,0x544c77b0),LL(0xf6676827,0x0143af56),LL(0x599efe1b,0x5a759df2),LL(0x3accffd4,0x55962b59),LL(0xc61321e5,0x795b77a3),L_(0x0000006a), LL(0x66e58a6a,0xbb4d42d5),LL(0xe0cfd739,0xe6ef3760),LL(0x0620ef46,0x6804fb37),LL(0x253e0f9e,0xf4e9cdc2),LL(0xcb5c8c64,0xec6f6658),LL(0xa80d08da,0x04153037),LL(0xcc959be5,0x3d215b47),LL(0x7aaaa865,0x6b1b8e4e),L_(0x0000014a), + LL(0xf52bc233,0x9700029a),LL(0x2a837019,0x3201dddf),LL(0xd02e2c74,0xd8885cda),LL(0xaea6cd36,0x7e35126d),LL(0xbc3f784e,0xfa40cea6),LL(0x5130e882,0xeec16a3a),LL(0xace2f121,0x13c43706),LL(0xf3c3a16a,0xc703c9cc),L_(0x0000002c), LL(0xc964823f,0xe8e4729a),LL(0xeed8d7ba,0x3fe54edf),LL(0xcfba42bd,0xec6a4e7b),LL(0xe917bf88,0x06be2039),LL(0x604163c6,0x17d5a63a),LL(0x6bdf09b0,0x276869bd),LL(0xf021410c,0x4b94144f),LL(0xf038cdd9,0xf4d84a13),L_(0x0000013c), + LL(0xeb8a6d24,0x724db1ba),LL(0x545ff43f,0x850f42b2),LL(0xfcaf8079,0x5c1fdccc),LL(0xde18c209,0x57404da7),LL(0x83097de7,0x267842f8),LL(0x8706015d,0xab9a893d),LL(0xe62c08dd,0xe2b0c7a0),LL(0x736bf358,0x34fb3742),L_(0x000001e1), LL(0x535f7766,0x4779dd4d),LL(0x10a98c32,0xc095243e),LL(0x0e7bc245,0x3cd82df8),LL(0x5fac69de,0x59efca16),LL(0xa2f0af19,0x692e1ddd),LL(0x40c91226,0x9b21f9d2),LL(0xa682e04c,0xa51cafc2),LL(0x831c0e79,0x9830cc49),L_(0x0000004a), + LL(0x94e4798e,0x00af5508),LL(0x9afdccb8,0x987426d2),LL(0x0f5f64d2,0x4aaeb57e),LL(0x76899b88,0x9a8859b4),LL(0xcf38ab59,0x31a64817),LL(0x8dc36916,0x9b757c1f),LL(0x28b27539,0xfb6a189c),LL(0xc63802b8,0x03d47dba),L_(0x000000c2), LL(0x225f33e4,0x9d6bb39a),LL(0x880025e5,0xffb62ebd),LL(0xb7d05691,0x6aacd544),LL(0x3e434b4f,0xba8454f9),LL(0x4bc06244,0xbe5d3fe0),LL(0x941bd419,0x6732d1a1),LL(0x6794ada7,0x6efab77e),LL(0x1058a767,0xf531c68b),L_(0x00000132), + LL(0xe58d6aac,0x5f7430b9),LL(0xb9ab109f,0xedf696b8),LL(0x252205ff,0x65163bdb),LL(0xb3a47496,0xfd8eca02),LL(0x915f0458,0x6d6b0ec4),LL(0xb0096e8b,0xddddf89f),LL(0x874c9e5e,0xab9c669c),LL(0x156cfc6b,0x83d55b9a),L_(0x000000fa), LL(0x8c6752e9,0xc103803a),LL(0x0196650f,0x653fb161),LL(0xe597cf0d,0xa211c0cf),LL(0x66902c1d,0x126bc8cd),LL(0xff4436dc,0x502c8df4),LL(0xcf0ef89e,0x30d9c137),LL(0xadabc266,0x509eb349),LL(0xcbdaa030,0xa426a8ae),L_(0x00000101), + LL(0x9d067e75,0x0846b23b),LL(0xb348d35d,0x6044d60f),LL(0x4e0c74fc,0x4437fb47),LL(0x2124f846,0xc898c89a),LL(0x30abdb7d,0xa1ecdf1d),LL(0x496e747c,0x09be44bd),LL(0x9381d368,0x6a34a28e),LL(0x2cd1e7c2,0xdcbc2c2c),L_(0x00000101), LL(0xece95ae5,0x705df30c),LL(0x08cdb28f,0x8f6f90ca),LL(0x15b2caca,0xdb9f29f0),LL(0xf8e2597f,0x537c5e8b),LL(0x91e195ad,0xa54bf828),LL(0x1ef3bc47,0x5f5b1233),LL(0x6c98e99d,0x096c5cfe),LL(0x0acf39b3,0x88fd1f55),L_(0x00000043), + LL(0x5e49ea9e,0x8278d27c),LL(0x1462a5cf,0xb6b188b1),LL(0xd1c3e723,0x7adfe4be),LL(0x9f4ab12b,0x4f74656e),LL(0x86f9e399,0x15f9fd6f),LL(0x647aabe4,0x356f176c),LL(0xdc05df56,0x037f39e8),LL(0x6f8c9681,0xc0c1a4ab),L_(0x000001cf), LL(0xfd586565,0x4f9b62dd),LL(0x9a07decc,0x07d1665c),LL(0xb14d451c,0xb6af032b),LL(0x2d75e1e7,0x7c7044cc),LL(0xac2df8ec,0xcac8a69f),LL(0x0906e20d,0x20c70582),LL(0xa4167466,0x1ca47052),LL(0x6fe2b8c1,0x576c602e),L_(0x000000ff), + LL(0xfb8bf3d9,0xcaac45d1),LL(0x39bf904a,0x4d2bf532),LL(0xa7f1203e,0xa3246448),LL(0x9ecdf560,0x6d3940b2),LL(0xa6d3b3a3,0x89bdcf57),LL(0x40dcc76f,0xdcc95c73),LL(0x9fa5791b,0xa442ea24),LL(0xc94b40b3,0x4596aafd),L_(0x00000184), LL(0x8473e7d8,0xbc97c4e6),LL(0xe17e4cb3,0x8daa65ee),LL(0x1d315fc6,0x20c6ff94),LL(0xeaec9cae,0x74f3ffd6),LL(0x1b61c251,0x015f6f88),LL(0x6a1cf873,0xee694708),LL(0xd187e0b9,0x37da53ba),LL(0x332ba61a,0x7b15c4b9),L_(0x0000017f), + LL(0x409292b4,0x27a8575f),LL(0x23606aaf,0xeb1afe84),LL(0x6df6bd82,0x546ebac4),LL(0xd77cd802,0xd19a6a0e),LL(0x9ac38a98,0x701bcf92),LL(0xae1c0504,0xd6247bc1),LL(0x90bada7a,0x7a406e08),LL(0xcc3c49ab,0x75a3d1ac),L_(0x00000089), LL(0x5dcf7e44,0x5f7352bf),LL(0x2b674d19,0x0ee31d4e),LL(0x35a3e68d,0x92916c56),LL(0x5e153a5d,0x1bbba324),LL(0xa3d55e06,0x27d3691c),LL(0x74f6e553,0xa9153a98),LL(0x2c8a7473,0x6e737b68),LL(0x545acbac,0x0b3ed514),L_(0x00000035), + LL(0x892f8bd1,0x31776371),LL(0x6c0b1d4f,0x60a2c8b1),LL(0x9a688a19,0xa0d12ea4),LL(0x56eef3ec,0x4746c345),LL(0x90c83381,0x842db71e),LL(0x3fdb8d8b,0x1bf4ae9e),LL(0xbc576b9b,0x9e0ee706),LL(0x85a8de64,0x3528254e),L_(0x000000e8), LL(0x3c32799b,0x25980592),LL(0xc4760975,0x76f95241),LL(0xd6c8d637,0xdd3d3b18),LL(0x677dee0d,0xe5bceafa),LL(0x12ad9334,0x7ca46478),LL(0x3990ceb2,0x94d56dac),LL(0x39d6e555,0x8e338deb),LL(0x7a9d83d7,0x33dbf96c),L_(0x0000013e), +}, +/* digit=61 base_pwr=2^305 */ +{ + LL(0x52d7b490,0x4215e0ae),LL(0x3a781f13,0x84df8cb3),LL(0xaf858985,0xc3ab10bf),LL(0x449e13a9,0xea1a4d65),LL(0xb859368b,0x2c8134df),LL(0xb51fd78c,0xa7d94439),LL(0x4b1e6f5b,0x003f1c75),LL(0x4cf5d47b,0x6428ed46),L_(0x00000029), LL(0x6dc28640,0xd6597a8e),LL(0xe3d2ac15,0xe5e853fb),LL(0x8bd192da,0xf804c989),LL(0x76032d13,0xdb66ee3c),LL(0x738d5e8b,0x349bcb16),LL(0xe16c7ea1,0xe4001679),LL(0xd22ec201,0x82f1f584),LL(0xd9f317a9,0xb67757b7),L_(0x00000079), + LL(0xec0a67f0,0x2efbca10),LL(0x3767ee14,0xcc433a34),LL(0x68131944,0x370f6b9e),LL(0x36fcc884,0xb4d1b5e8),LL(0x85328231,0x85b956e6),LL(0x3e4b895d,0x17afd7ce),LL(0x23cd96a1,0xfc28a48e),LL(0x6cbf1cc4,0xfc04d775),L_(0x00000125), LL(0x5cc45bb9,0x4201b856),LL(0xbec2c277,0xb4f52020),LL(0xcd76ab62,0x3aabda65),LL(0x2fb221ac,0xd348e9ae),LL(0xb7ba962f,0x81d5e875),LL(0xacdde7c3,0x8ae119ed),LL(0x7186eb96,0xd5b495d5),LL(0xdf795bd0,0x1bbdd56f),L_(0x0000011a), + LL(0xabe42367,0x83477d80),LL(0xae655393,0x9bf84781),LL(0xf1389d3f,0x50d527ea),LL(0x02ffac63,0x30005241),LL(0x5a3b0583,0x332af83e),LL(0x30a51cf3,0xd633aac0),LL(0x7e87b5f6,0x6133508c),LL(0x54cf5544,0xae4be94e),L_(0x00000153), LL(0x79f352eb,0x3d5e304a),LL(0xc8a8525e,0x673478be),LL(0xcd082890,0x65272acf),LL(0x6528a7ef,0x07746b44),LL(0x6120a7a6,0xaa126f2d),LL(0xed8dc8aa,0x0714c411),LL(0x242ecc1e,0x09219322),LL(0x4dd29b99,0xf4c3776f),L_(0x000001bb), + LL(0x9d1e3e79,0x6e1c349e),LL(0x7cb862d6,0x411c782a),LL(0x774f6f73,0xf914b067),LL(0xd88b7029,0xbe145ff3),LL(0x68ac9342,0x0730a2fc),LL(0x77dcfba1,0x7ace014c),LL(0xe34f0621,0x876ebecf),LL(0xb7a85b90,0xa9ebdf04),L_(0x0000005e), LL(0x2be45d39,0x2af68cfe),LL(0xeda14612,0xbeab553a),LL(0xc8cf47bb,0x185338ec),LL(0x9f26575d,0xbcf5707a),LL(0x2dafc93e,0x9b4f2615),LL(0x85006b56,0x1c517096),LL(0x58e4408a,0x2759575a),LL(0xa451b6b3,0x3b494d43),L_(0x000001b3), + LL(0x64081168,0x9fca1732),LL(0x9e2e68b9,0xc2ec6027),LL(0xbd01c53b,0xcf8e3aa9),LL(0x3c299cf0,0xc31ff566),LL(0x69a934da,0x3a869b7d),LL(0xee4c3bf3,0x17fb711a),LL(0xe353eab9,0xb5b7fc05),LL(0xd300c851,0x0ef4f1da),L_(0x00000006), LL(0x7326c782,0xf868e4b0),LL(0x7616e981,0x9cbec832),LL(0x0d0b19fb,0x0355a1b8),LL(0x7504ef78,0x9b3d9f50),LL(0x75e429da,0x0924def0),LL(0x130ecd97,0x07187605),LL(0x844d6f96,0x7c14ae9f),LL(0x8921d3a7,0xccff1a1b),L_(0x0000016c), + LL(0x9415e0cc,0xdd321662),LL(0x7be013aa,0xad261af9),LL(0x46c47707,0xedc263d9),LL(0xabc20130,0x2f265a39),LL(0xe142f5b7,0xcfeec142),LL(0x90dba064,0xa08536a7),LL(0x488a0175,0x6d419631),LL(0xcf748207,0xe8415c3f),L_(0x000000dd), LL(0x6def509f,0x7ce6774c),LL(0x4af30ccf,0x09bc7469),LL(0x3bf7cbb3,0x12464f68),LL(0x648dc8e6,0x93c3b96f),LL(0xb39085a1,0x1000d207),LL(0xe4a3e7cf,0xcb93a762),LL(0x39e62f3d,0x08cde0be),LL(0xd6284f5b,0x8545da92),L_(0x000000b3), + LL(0x4fd9fb1f,0x0a484ffb),LL(0x4a91b446,0x5c5120d6),LL(0xcfe786b3,0x521a227a),LL(0x1f347861,0xb4d52b1b),LL(0x005afae4,0x28a0b22b),LL(0xaed64316,0x96565ad0),LL(0xcf8e1f9b,0x3bd3818c),LL(0xd717a7fd,0xea3705d7),L_(0x0000019d), LL(0x17132d83,0x4d75e2a9),LL(0xf9bc8f55,0xbaf9947b),LL(0x91f937e7,0xcb9bb75f),LL(0x232e92ab,0x4c1c8f93),LL(0x92cbf962,0x90cd09cd),LL(0xf2dcee1e,0x07dbfe55),LL(0xb89e680d,0xc41cd340),LL(0x51007568,0xa6275fb3),L_(0x000001e0), + LL(0xb831eea8,0x9b33eda2),LL(0xdd1ac75a,0xccb42fd2),LL(0xe55769fc,0xa865ab8f),LL(0x2ea2c9e4,0xc60208ee),LL(0x28effb93,0x321dff3f),LL(0x2d6b1522,0x5124fb78),LL(0x6e29dc3e,0x4cf961e9),LL(0x2f39193c,0x554efa6d),L_(0x000000ca), LL(0x035e63b7,0x67da851f),LL(0x7e9f39d6,0x7183aa79),LL(0xe886c75f,0x4aa59f9f),LL(0x6d9e6857,0x706045d9),LL(0x2ee25277,0xb18ceb8f),LL(0xe4bcaa94,0x5bc3971a),LL(0x57fc8f0d,0x7b7b6081),LL(0xdd642848,0x2e8f2150),L_(0x000001c6), + LL(0x44c8a327,0x1f4b8713),LL(0xa7ee221a,0x5258be1d),LL(0xe41af48e,0xa737a8cd),LL(0x83aeee1c,0x3f320ac3),LL(0xbb3a2bdf,0x8abc18a8),LL(0x2bfe7f09,0x9da43962),LL(0x5ab55046,0x1318b08d),LL(0xc7f6a7c8,0x4ae4a0c2),L_(0x000001fe), LL(0x072464c5,0xe109557b),LL(0x52cb5223,0x2e18e586),LL(0xbb44cf23,0x58cf033e),LL(0x8273746f,0x9cb3f3f2),LL(0x15027b4d,0x0badc23f),LL(0x03dd0534,0xebde0563),LL(0x94ef00b2,0xf31a2a6f),LL(0x88f86782,0x089879e2),L_(0x000001cb), + LL(0x5644a483,0x012e151c),LL(0xec46d8f5,0xcb60c92d),LL(0x61f8e693,0xf704143e),LL(0x53579b44,0x06b44ecd),LL(0x98807645,0xdda8c89f),LL(0x17c64951,0xfaab400d),LL(0x2e39e25a,0x16b86130),LL(0xfef9c912,0x627a2e4f),L_(0x000000ad), LL(0xd2367ca5,0xf18ebddc),LL(0xbcf685ba,0x59f6d4cf),LL(0xe159807e,0x8c3c8195),LL(0x2bb8d624,0x02e20259),LL(0x3ad24a15,0x5eef3266),LL(0xe9a952f5,0xc8d0e08e),LL(0x37d37845,0x4cf4addb),LL(0x778df76b,0x420c462e),L_(0x0000004a), + LL(0x55047df4,0xa605ea07),LL(0x579060a2,0x728d81cf),LL(0x3b8900fa,0x1ccc1e61),LL(0xf49f8c7a,0xd633ce10),LL(0x4957105f,0xe467c698),LL(0xc3e2024e,0xa14d0dd3),LL(0xe7cc2be3,0xc90176ba),LL(0x00090c73,0x28835ef8),L_(0x000000f4), LL(0x39d8bbe5,0x229fde87),LL(0x1982808c,0x9d5d0ade),LL(0xb83a93a2,0x56715396),LL(0x4130f493,0xf5d0d1b5),LL(0xc39ee248,0xcbf57700),LL(0x0662cf56,0xf41ee620),LL(0x121da851,0x2397e72c),LL(0x1ab413bf,0x6c00ad7e),L_(0x000001f2), + LL(0xbdd73dc1,0x1f7a793b),LL(0x7252dfce,0xb3c777ad),LL(0x1ad25f35,0xd306f90f),LL(0x314c1227,0x4538596b),LL(0x28e31145,0x15f73822),LL(0x1808f8a9,0x5e4847ef),LL(0x6eb8175d,0x9d57409f),LL(0x2ae642ae,0x19b0b71d),L_(0x000001bb), LL(0x3f28e667,0xa45a1d7d),LL(0x84954816,0xe5f147ca),LL(0xccfb8bd1,0xcd78f915),LL(0x9a693642,0xb02b310f),LL(0xb6cb5362,0x1f01047b),LL(0x7529c74d,0x81d1fb13),LL(0xdf4ae21d,0x80b9dd94),LL(0x6a1afeec,0xd7840bbc),L_(0x000000db), + LL(0xc0abf15f,0x81d957db),LL(0x65ae7f67,0xe0fe8725),LL(0x40e4566b,0x7c42febe),LL(0x6340b7ff,0xcf060fa6),LL(0xea1d2782,0x9a689bd9),LL(0xb66eed98,0xc45b992d),LL(0x8f5646a0,0x969dc412),LL(0xd272048a,0xf48e06d2),L_(0x000000f7), LL(0x50900d7f,0x701c6e38),LL(0xaad1803d,0x92065d64),LL(0xf9668fa4,0x361ef75c),LL(0xfcc216b4,0x962eb248),LL(0x575be56a,0x89b9828a),LL(0xfde9ba30,0x202a575f),LL(0xc435f2ce,0xba3890aa),LL(0xf83734f6,0x4860b39a),L_(0x00000180), + LL(0x3ede16d4,0xa59cdc00),LL(0xd0de9f29,0x9f3b7991),LL(0xda7b6269,0x832ec0d2),LL(0xf2e16e2b,0xa8e7c828),LL(0xd0c41727,0x7f0878f9),LL(0xc4546447,0x7356692c),LL(0xc4af90b2,0x5fbe130e),LL(0xaa2e9ec4,0x17691590),L_(0x000001bb), LL(0x0074a183,0x16d0ccbe),LL(0x351544e0,0xf7b675ff),LL(0xf37d43c1,0x87055e7e),LL(0xc371f0a0,0x3e668989),LL(0xefcfca1d,0x8323227b),LL(0x36507d20,0x38f76084),LL(0x25498782,0x75a23d95),LL(0xacb8cb75,0x40481354),L_(0x00000151), + LL(0x20886bbb,0x9a6dec74),LL(0x296f5cd9,0xfd24a18e),LL(0x6092fe28,0xbb4a7907),LL(0xdeab539d,0x869a8ccb),LL(0x67f524d1,0x61521c17),LL(0xbbe3aaa1,0x5f79a2c2),LL(0x8be17e72,0x7d8ce0cb),LL(0x647da5af,0xf36cb2d8),L_(0x0000003a), LL(0x0f2ab363,0x6d93da70),LL(0xdfde9d3e,0xa76d10f8),LL(0xe72ce040,0x17308d11),LL(0x075467a8,0xed2aabb3),LL(0x9aa69a1d,0xc10f78d3),LL(0x4caac399,0x13d4d378),LL(0x7e54c473,0xda4d8f8d),LL(0x911cb804,0x55d905b0),L_(0x000001aa), + LL(0x253f45b8,0x2af99e2d),LL(0x112b491f,0x76414d1e),LL(0x4a8e2b12,0x1c380001),LL(0xa17691a7,0xb0f6f9ff),LL(0x4bd4233a,0xcf4e764a),LL(0xccb8bd49,0x012735f0),LL(0xc7fc0714,0xf6037d3d),LL(0x3da811dd,0x4eeecae0),L_(0x00000135), LL(0xf24a8fb9,0xb39e1a42),LL(0xa3c4048e,0x07ea05b5),LL(0xd581bb93,0xfa0b7bd9),LL(0xeeb6ce36,0x8b7b8d8f),LL(0xa0329bb2,0xfcc8cab6),LL(0x5c44e608,0x04c03b08),LL(0x3f2c4c95,0x51593cce),LL(0xb2ef5a3d,0xa645193b),L_(0x000000c1), +}, +/* digit=62 base_pwr=2^310 */ +{ + LL(0x052a5d7a,0xbd77fdcc),LL(0x91e67c5a,0x7ae80845),LL(0x30c17511,0x4fb64810),LL(0x92e55b53,0x6c21b31f),LL(0x96bd1eba,0x4d056aef),LL(0x3bd89651,0xf597cfd9),LL(0x39ce9f5d,0x10c4de29),LL(0x6737cbaf,0xca8494ef),L_(0x0000005f), LL(0x726ccf17,0xa070d7b2),LL(0xef8a249b,0x33084cb0),LL(0xfece71d7,0xdbf18fa7),LL(0xf11b328d,0x8e7f8fe7),LL(0xdb8c5b89,0xc5842d33),LL(0x38ef699b,0x44b71419),LL(0x619477d9,0xcd39a13d),LL(0x28db36f5,0x6954dd52),L_(0x000001eb), + LL(0xae5a71e3,0x160db974),LL(0xb267126a,0x4df55ba5),LL(0xf2bfd214,0xcf291fe0),LL(0xe4215d39,0x3dc0a627),LL(0x8849498b,0xfec311ed),LL(0x5b220c7d,0x9fbb5099),LL(0xa3d83cc2,0xc55f9ca4),LL(0x32f62dd6,0xa473d853),L_(0x0000012b), LL(0xe73278db,0xcaebf5d4),LL(0x38b01c56,0xe4ab979b),LL(0x7e210f66,0xfe305e1e),LL(0x00e35bf7,0xbbd247e8),LL(0xf41625bd,0x64eabbca),LL(0xb3c01407,0xe49d3fb6),LL(0xc31a840a,0x6ebed09d),LL(0x6c67185e,0xd891e44a),L_(0x000000d3), + LL(0xeea3bb5d,0xa7efe273),LL(0x382ccbad,0x1d1fd154),LL(0x321aecf3,0xba4c80f8),LL(0x3a3eb329,0x44874ee5),LL(0xfc744e55,0xc89ec973),LL(0xd83775b1,0x9ac52665),LL(0x7c8cecd7,0xe149472e),LL(0xffa02e1a,0xabeedb19),L_(0x00000188), LL(0x4863bc6f,0x5e342dcd),LL(0x30b568ee,0xba377da7),LL(0x61a3cd5a,0xdb7394c9),LL(0x7e13d011,0x655ca62a),LL(0x531b03ef,0x687df8b5),LL(0xa07d97a8,0xc1cc63e0),LL(0xc3579f84,0x4f51c0a2),LL(0x1f68d107,0x8b2a0959),L_(0x00000021), + LL(0x73976185,0x5ac29ab9),LL(0x1049200d,0x5376ef50),LL(0xb7cabe96,0xb29fcfde),LL(0x2ebeaa6e,0x849702e8),LL(0x9856863e,0xd5820c1d),LL(0xadb32b7d,0x0b85b8b6),LL(0xcb2a1da8,0x4ecc2beb),LL(0x911240a2,0xb6f1d1cb),L_(0x0000019d), LL(0x8e9339d6,0x65738f28),LL(0xc9389868,0xec9ab31a),LL(0xb78b477c,0xb756dfc0),LL(0x2531d4c9,0x2bea7bd2),LL(0xa957b1f7,0x19668750),LL(0xb7acf908,0x23544082),LL(0xfa97aa90,0xd310dd35),LL(0x7c9376d4,0xe3e85065),L_(0x000000c8), + LL(0xe690991e,0xb53d0f3e),LL(0x7a8ed401,0xe7688a48),LL(0xc975d343,0x0a163c7f),LL(0xbbe320a1,0x3c38f3b4),LL(0x637ba641,0x0cc94ae6),LL(0x2fa7a438,0xf036a2cd),LL(0x85cf8f8c,0xdd8d3d2d),LL(0xf3dec45f,0x83f293f7),L_(0x00000007), LL(0x0fc0fe23,0xf9f35894),LL(0x22cf1f3c,0x5367a382),LL(0x0a85bdb7,0xa486155c),LL(0x43d0dc60,0xa045fb49),LL(0x28e031ef,0x239e6d10),LL(0xbc646dab,0xf3c58cd8),LL(0x37a252ad,0xa190c29a),LL(0x729ace13,0xafb3edde),L_(0x00000004), + LL(0x2c2d00f5,0x309124ad),LL(0x92f17a73,0x90896f6d),LL(0xd107aa3f,0x6655b0bf),LL(0x28ed385a,0x4393b8b0),LL(0x64efc785,0xa72dcb01),LL(0xdc6d4959,0xedcf6e0d),LL(0xad09fb16,0xa138cb63),LL(0x5a264a29,0x9242e638),L_(0x00000154), LL(0x23fa857e,0xa4afca8f),LL(0x362d992a,0x4718e360),LL(0xe0e7ef99,0x51da204d),LL(0x0a263a3e,0x76d92100),LL(0xc54159bd,0xb90bd792),LL(0x6992a7d6,0x2d4d5792),LL(0x34429060,0xea9796c5),LL(0x2d91640d,0x11109304),L_(0x00000023), + LL(0x69a4e57c,0xd8256cea),LL(0x1bf79944,0x4d134e77),LL(0x4e8b215e,0x63a4641d),LL(0x83621b34,0x8da5f102),LL(0x530939c0,0x9d6baa6f),LL(0x78356025,0x0a919eb7),LL(0x9cebfe30,0x523c04c9),LL(0xba70fc3b,0x06c24660),L_(0x000001b0), LL(0xb404acda,0xbeff381e),LL(0xf36c4399,0xc6bfdda5),LL(0x193ff430,0xddaf4961),LL(0x43e642a9,0x86bb6b08),LL(0x4ebe4623,0xd3326377),LL(0x8dc4af24,0x33ce6709),LL(0xb168c749,0x3757e6ab),LL(0x451bf0a9,0xdd6738f7),L_(0x00000114), + LL(0x95d393a7,0x15af1e76),LL(0x09f6c873,0x5dad48c9),LL(0x168b010a,0x03c65a7e),LL(0xd86fdc56,0x73f51c26),LL(0x88f52d53,0x697c8b7d),LL(0xbc64a497,0x670982de),LL(0xaf7a0676,0x809f942a),LL(0xb15cc57a,0x593420be),L_(0x000001c2), LL(0x71728397,0xf5563ef9),LL(0x305f3c8f,0x4e73a2dd),LL(0xa80ae4a1,0x828cc516),LL(0x2258160b,0xe74db735),LL(0x108533e6,0x14ad6801),LL(0x3b320283,0x541598a0),LL(0x763ab107,0x56c3d815),LL(0xf632644f,0x8ad49089),L_(0x000001fa), + LL(0x3fb5de8b,0x8bc4deb0),LL(0x8d93c4e8,0x1e8e7ab9),LL(0x3dd24d9d,0x201baf56),LL(0xcada68d7,0x0a384ece),LL(0x503d4f19,0x5dcc59f7),LL(0x6763d7ad,0x7849c18f),LL(0xc66f3753,0x6951161c),LL(0xfc052118,0x74242704),L_(0x000000c6), LL(0x90fd23ab,0xfbb20d46),LL(0xd8a0eeac,0xd979406d),LL(0x508b0789,0xeb2d48ad),LL(0x8cad1e65,0x2f16458c),LL(0x7615ee48,0x8941144f),LL(0x2d4a611a,0x57baf847),LL(0x706729a1,0x04864e43),LL(0x13b7d8ff,0xa2cca9c9),L_(0x0000001f), + LL(0x6b13d691,0x15c5966e),LL(0x5adf4806,0xe79886a9),LL(0xb44e7b28,0xc0149ae3),LL(0x14ea5297,0x3f2176d8),LL(0xd637170e,0x3d5f7f20),LL(0xfd66e46a,0x5f76d12c),LL(0x998ccf72,0x8fbfc2d6),LL(0xc2738301,0xc299587b),L_(0x00000093), LL(0x48202d24,0x5d1ab080),LL(0x22169c9a,0xf44f3ef3),LL(0x4f9cc0b4,0x5a6ea1fc),LL(0xd8a38b0c,0x1e3f8f7c),LL(0xfc2a4b0f,0x2f6b7ea1),LL(0x85236ace,0x7c4797d9),LL(0x507ad976,0x30db4704),LL(0x70b62118,0x15efb0d0),L_(0x000000f2), + LL(0xa8e006d6,0xa8350ceb),LL(0x0b5d40f6,0xd0476f3e),LL(0x7d6beb64,0x8a7277db),LL(0x9a1052c1,0xb78ba330),LL(0x6fb67a25,0xa921f295),LL(0x937d5f7f,0x58e2fb78),LL(0xb3c5ee8b,0x224a8a6c),LL(0x3ba51856,0xdd623558),L_(0x0000004f), LL(0x472c8eee,0x46bd4fc5),LL(0xd6bbe5d7,0xbccb9c2d),LL(0x8704f8a7,0xd4145962),LL(0xf0c09b77,0xe9ce9fc8),LL(0xe24e89e3,0x091189c9),LL(0x34dfd23c,0xa0008822),LL(0xddeaf170,0x43b08954),LL(0xe569f253,0x867e76b2),L_(0x00000020), + LL(0x21969535,0xd200f675),LL(0x1aa95306,0x8a20dfb4),LL(0x450070d5,0xe56ecbdc),LL(0xa73c2aa2,0x93697944),LL(0x8cf15e09,0x2bf1cc5f),LL(0xb81e3982,0xa98dee98),LL(0x39d2614b,0x4249763f),LL(0x88bf80d0,0xdc125dce),L_(0x00000041), LL(0xea90be49,0xeb5f7526),LL(0x9d76e09e,0x42892f62),LL(0x665e7661,0xdd2de6b9),LL(0xdb45bef0,0xe66edde4),LL(0x0f0c29ed,0xd947a3fe),LL(0x39bccdcb,0xdc0bb667),LL(0x97600929,0xeeaa185f),LL(0xf355b62f,0x29e7ad98),L_(0x00000083), + LL(0xc2d3d21b,0xbe07feb4),LL(0x9e1d3138,0x90b99ecf),LL(0x358fe997,0x2ec0a3d5),LL(0x95008edf,0x7f72a6c8),LL(0xc6e6cc84,0x2b8ec523),LL(0x351d40b6,0x10aa3646),LL(0x30961dc3,0xdff38b1a),LL(0x6d776cf7,0xc458ba47),L_(0x0000000a), LL(0xfce248cf,0xad217997),LL(0xe663a7a1,0x423b10cc),LL(0x8d65dc51,0x0215b195),LL(0x0f10bc35,0x0f4e07a2),LL(0x23278029,0x19d23499),LL(0x304b98d9,0x6127a2da),LL(0x9fb0c81f,0xed0c0943),LL(0xcd486835,0x4d63eec4),L_(0x0000015f), + LL(0xd0ac0ecd,0xbaefab89),LL(0x0a9db571,0xe4775843),LL(0x6c283a9c,0xfebcf91f),LL(0xd37751b1,0xe02d1251),LL(0x4c69aef4,0x93ca62f3),LL(0x756b2ab1,0x017751a3),LL(0x9921e7da,0xf0df5f26),LL(0x31fbf868,0xc6385112),L_(0x0000015e), LL(0xc24912f9,0x6ce422f4),LL(0xe5bc3bbe,0x0c518f2e),LL(0x00e5a237,0x51ed5f7e),LL(0xb6da428f,0x1a77cbdb),LL(0x248c6951,0x086ad3a0),LL(0xfd285428,0x00d65807),LL(0x460a5bc1,0xc6265db2),LL(0x728547e4,0x8fcc39fe),L_(0x0000002a), + LL(0xcdbb56b7,0xea484f2e),LL(0xebecf09a,0xbe1705e2),LL(0xdc0d7050,0xe83a83b6),LL(0x5c8fdff7,0x1cd41a57),LL(0x58f038ee,0x975aeb28),LL(0x858f75ce,0xddbb66ee),LL(0x7455106f,0x7e1bcafd),LL(0x54e1961f,0x1e151b28),L_(0x0000006f), LL(0xc329e633,0x32cfb8ca),LL(0x596dc91e,0xe8fb4aa3),LL(0x19c60dcd,0x0c27fe63),LL(0x9c2411d9,0x49228e82),LL(0xf4420f99,0x5075f5a2),LL(0x38a95326,0xadb26b0e),LL(0x7345059b,0x67709e35),LL(0x428212fb,0xa6804139),L_(0x000001a5), + LL(0x3a95d178,0xb909e614),LL(0x1860388a,0x89b7600c),LL(0x942112c1,0xa080f4aa),LL(0x5a1967f7,0x5057c08b),LL(0x13543a0e,0xf9ac78fd),LL(0x1598cafb,0x9408a20a),LL(0xfa7974b4,0x8fb58bcd),LL(0x17ad4e19,0xf91b6362),L_(0x000001b1), LL(0x57138c5b,0xf8960264),LL(0x185f172a,0xea9d78b2),LL(0x8652917b,0x62148231),LL(0x9b757159,0xb7470a8b),LL(0x4f2c7ae3,0x532d7747),LL(0xc96fd10f,0x6b40b8bf),LL(0x77081dbd,0xa54da232),LL(0x2cd44f13,0x0796d519),L_(0x000001ac), +}, +/* digit=63 base_pwr=2^315 */ +{ + LL(0xc1b5f874,0x37761e89),LL(0xc768e709,0x052402d8),LL(0xa7063fcc,0x437e0d8f),LL(0x5032ca28,0xd7049706),LL(0xe0560b81,0xfcc5af72),LL(0xdac1a63b,0xabb68cfd),LL(0xe89f3917,0x257b3b85),LL(0x80d7454a,0x28153e38),L_(0x00000088), LL(0x001d4cbe,0xe9651818),LL(0x66b1014f,0x64d65a97),LL(0x511b3639,0xd56646ec),LL(0x26e7c4e0,0xdfae8dcc),LL(0xa94ae11a,0x86e8d406),LL(0x6e9a1a68,0x47bbf4ad),LL(0x3004a685,0x13e8901b),LL(0x5981c480,0x3994bc8d),L_(0x00000179), + LL(0xc63223ff,0xeed01f7e),LL(0xb10c656e,0xa95759d3),LL(0xc8ceacdd,0x5f68fe9f),LL(0xb6ab8ec3,0x50e97936),LL(0x28c0b215,0xe4ccc3b9),LL(0x92be8e1d,0xfc6be17e),LL(0x20828c77,0xb81bdb63),LL(0x13352a78,0xe00699d2),L_(0x0000016f), LL(0x3d9b3c1e,0x0b5daa42),LL(0x94567b3a,0x6bf95aa5),LL(0xa3d149d4,0x8d0fbfb5),LL(0x4e997958,0x5e636b3d),LL(0xc1e08ca4,0x8fa3b11b),LL(0x73645c35,0x552e11b1),LL(0x931ab993,0x0db67bc8),LL(0xce614c5f,0x53c262d3),L_(0x0000019f), + LL(0xf005a100,0x9be2e7f2),LL(0x06f26644,0x47f29a13),LL(0x7b580ed9,0x3f3b60f9),LL(0x22198889,0x7e6ec70b),LL(0x9f87a7be,0xfc2d715c),LL(0xb2ebc47c,0xfc003ea3),LL(0xfa7b2218,0x79438acf),LL(0xbfd9d6c5,0x930053c2),L_(0x000000f1), LL(0xdac555ca,0x80f3c546),LL(0xe5a4dad1,0xa25a6ba7),LL(0x2d4bd9fe,0xa68cfbbf),LL(0x35faf13a,0x4e3df8bd),LL(0xcfc847de,0x8434c00b),LL(0x4dfdf245,0x40669463),LL(0x6e619d42,0x5f688c19),LL(0x13d3a517,0x9cabea4b),L_(0x000001af), + LL(0x3aed7148,0xbb901084),LL(0x6321bcc4,0x996ac002),LL(0x53c74ad5,0xcb634535),LL(0x741982c7,0xdc48a041),LL(0x1196d8fe,0x44c9f092),LL(0xfabc20d2,0x0a8fce97),LL(0x32828d27,0x62a6d447),LL(0xcab0c775,0xe6a8960b),L_(0x00000089), LL(0x3a61da39,0x4b749d7b),LL(0xfb5b6d67,0x9df6dae1),LL(0xec275083,0x4fd05c30),LL(0x7da1a928,0xb7bd6dae),LL(0xec82a28e,0x66cd19ba),LL(0xa08ca71d,0xd599b2c6),LL(0x6c312c52,0x5bfaa154),LL(0x6795e306,0x2d3a57f0),L_(0x00000087), + LL(0xe9c779ff,0x99bf4200),LL(0x2e4558d4,0xb9ba6e9a),LL(0x636dc521,0x836fe297),LL(0xa1c7e0bd,0x461d465e),LL(0xa229d229,0x287fba32),LL(0xe43c8f80,0x5fa34491),LL(0x76cbe0ad,0x0e6b8f16),LL(0x7a25d2a9,0x6221b3d0),L_(0x000001ae), LL(0xd420ce9f,0x59654147),LL(0x2c11cbe4,0x582dde44),LL(0x73168c78,0xcfbe66e7),LL(0x5f455763,0x55778942),LL(0xd782c483,0x9b69f069),LL(0xff95fe3e,0xaa1addcb),LL(0x00a4bd0b,0x47541c1d),LL(0x8ad93857,0x5c6d8f2a),L_(0x000001b8), + LL(0x0afff918,0x80ad064a),LL(0x89af5deb,0x7114ab96),LL(0x4dbde778,0x099fad0b),LL(0xfd29cd3c,0x525d6055),LL(0xbd379d42,0x4df50e85),LL(0xdfc116d0,0x3602e006),LL(0x374d96b5,0x2ee6c63f),LL(0x6509a7f3,0x4fb180f2),L_(0x00000115), LL(0xee822c17,0xfc039cb1),LL(0x6c2fdf58,0x5872cad5),LL(0xea665324,0x3b9e8ae0),LL(0xaf2e64bf,0xb8314c4d),LL(0xa8f96bb4,0x63c57f41),LL(0x3df990e0,0x5149d306),LL(0x1d5f9b0d,0x08ba6128),LL(0x3d6cc9c6,0x52f204ba),L_(0x000000d5), + LL(0x84b34c8d,0x0908a3fa),LL(0x06e343a5,0x8e41dac7),LL(0xca844102,0x83411f49),LL(0x0712aa99,0xd4bcaa5f),LL(0xf85d2ba8,0x0278367b),LL(0xbdc302b7,0x5016082b),LL(0x54ed82be,0xfea00712),LL(0x1e47617b,0x97de79a6),L_(0x000000b6), LL(0x5025ca72,0xa8a5db48),LL(0xd3c98c1a,0xeb113cc6),LL(0x259b9a28,0x1b35c6d2),LL(0x49923a55,0x266d75d3),LL(0x644a3ecc,0x9590fb6b),LL(0x221e1f1d,0xa7f663c5),LL(0x9c9bd811,0x30cacfb5),LL(0x8f25a4f3,0x459bddad),L_(0x0000017d), + LL(0x5c5d4a76,0x224e9112),LL(0x74621c3b,0x62ab184a),LL(0x17406495,0xedfb682a),LL(0xc3f7c8cd,0x16ae2053),LL(0xd8e38d44,0xdf044060),LL(0x39ed9c28,0x86143e57),LL(0xf327b97f,0x8b95f9f7),LL(0x53853147,0x9200a872),L_(0x000001b4), LL(0x43b98e46,0x05661b39),LL(0xc1bcc1fc,0x9ee23198),LL(0x64ff1647,0x115744fc),LL(0x0f20d871,0xcdf5ac56),LL(0x92c9feea,0x63cba9c3),LL(0xa72f70b2,0xadbac8fd),LL(0x365c71db,0x171aad35),LL(0x9d51687d,0xaa0202f9),L_(0x00000102), + LL(0x7242ca8b,0x45faaf9a),LL(0xc08d85f8,0x550ef4dc),LL(0xb82ff28e,0x814b8cba),LL(0xbbe121da,0x1eb4cd63),LL(0x081656e6,0x82eece40),LL(0xf4405b11,0xe9889d6a),LL(0xf6c9d001,0xbc4f3c1e),LL(0xe85dc906,0x7466fe1b),L_(0x000000f9), LL(0xbcfa56fc,0xb8894301),LL(0xa60a71ba,0xc8290de3),LL(0x5b4cf893,0xfa8203e9),LL(0xa8602943,0xb0d9fec8),LL(0x6b75b5c6,0xacaeb1bd),LL(0x40f20d5b,0x228fdb83),LL(0xe7477d37,0x967812d8),LL(0x3271b8d9,0x20f42fd0),L_(0x000001a9), + LL(0x69acd4ec,0x76812a69),LL(0xd47ef468,0x62f921ab),LL(0xc8ee3434,0xb7930834),LL(0xc08c033b,0x369c3e87),LL(0xee51d0a2,0xd98cac8f),LL(0xc675c1fb,0xa309b704),LL(0x3fcbb3c6,0x69a173a4),LL(0x32c49495,0x0559e094),L_(0x00000126), LL(0x2b5e781f,0x79bd3a70),LL(0x68837e34,0xd74e86eb),LL(0x22881aa5,0xecb38496),LL(0x91b89a84,0xdd2964ba),LL(0x7caeee87,0xb0230b75),LL(0x83a10f40,0x7853cadc),LL(0x465657ae,0xe45f5ad1),LL(0x100e5033,0xe6dbce75),L_(0x0000005d), + LL(0x27034a2f,0xae458db9),LL(0xe02cc805,0x361c4604),LL(0xc6c6e812,0xf53dab3f),LL(0xe1de7819,0xa93944c6),LL(0x77575b10,0x7d127be6),LL(0x4580ec67,0x18920ad0),LL(0x6451a6a6,0x595f7341),LL(0xe3b018ad,0x87c2f62f),L_(0x000000a8), LL(0x73fafabe,0xacea82d0),LL(0xb8e018e3,0xb66d3c1a),LL(0xf0a068d2,0xa0a76281),LL(0x2960ab23,0xd3310f1e),LL(0x1ade815c,0x5df5a459),LL(0x4830c68e,0x9bc40618),LL(0x506f8ded,0xa5b32181),LL(0xb64aea9e,0xdef4d655),L_(0x000000f4), + LL(0xd2e44f39,0x7f22b492),LL(0xf166288f,0x72f850db),LL(0x45a14853,0x6743ab2a),LL(0x480b82ee,0x235a84e1),LL(0xbca609c8,0x422668b9),LL(0x2f4e85d8,0x5d6f0bf0),LL(0x792321da,0x61afb880),LL(0x2c095f02,0xca3c8cb8),L_(0x00000022), LL(0xe6bc2f57,0x43f99381),LL(0x4a832e1f,0xe6089c84),LL(0x51ad7011,0x65600aa9),LL(0x2a695207,0x11447728),LL(0xa07e689c,0x7bb9a4c6),LL(0xcd7b0e53,0xdf06eaf0),LL(0x78952329,0x777c474d),LL(0x10d2b00b,0x9d539f07),L_(0x0000004b), + LL(0x20322841,0x736265c0),LL(0x91a20b69,0x7c956777),LL(0x530024a3,0xbcd4358f),LL(0x2cfdf5c4,0xe32fe9e3),LL(0x69c3c240,0x7dd472b9),LL(0x16947a8a,0xeeaaeb78),LL(0x03c5cf25,0x9aa3433d),LL(0x521f3b6b,0xf3a8ba18),L_(0x00000085), LL(0x88ddc3b1,0x00a93570),LL(0xc5b29c7d,0x5d8a1581),LL(0xb3793bd2,0x1877f26e),LL(0x4435f13a,0x4fc7c6c4),LL(0x5b6c76af,0xb032ee18),LL(0x5465338c,0xd7e32969),LL(0x5e8d0f72,0x82259fe6),LL(0x38f0d401,0xe8265e82),L_(0x0000012d), + LL(0x8093c8e0,0x248ef981),LL(0x84b971a4,0x5353d713),LL(0x8ab8dd10,0xf3f56422),LL(0x9e95615c,0x163427ee),LL(0x1dbed91f,0x7cd8d83e),LL(0xa05bff8c,0x2a117b26),LL(0x9094b7c0,0x28d65130),LL(0x8d73f3a7,0x75d255fa),L_(0x0000015d), LL(0xfb45cf4f,0x5dd01b3b),LL(0xa983b69a,0xe3b24278),LL(0xdda15e64,0x0beba6e8),LL(0xeabdafeb,0xce3cbe7c),LL(0xe28dd1f4,0x03c3a01e),LL(0x315483c0,0x286b68f0),LL(0x44cc13c4,0x653661bd),LL(0xa5a2b18a,0x8d8536d7),L_(0x0000007f), + LL(0xf83ea93b,0xc7a428c9),LL(0x308d0ccb,0xdeef06fc),LL(0xe456dad3,0xd27b9dce),LL(0x04dd575c,0x8c1bbcfc),LL(0x1c63319a,0x479f6f73),LL(0xed4daaeb,0x7cb52d7f),LL(0x9fe5930e,0xd171cdbc),LL(0xcd65b54d,0xf8449b00),L_(0x00000106), LL(0xbb8079a0,0x2ea98b08),LL(0xda724133,0x24b7505e),LL(0xedf1d97e,0x5aed5e6f),LL(0x283c1e51,0xc39ad307),LL(0xf64812a4,0x76820a6c),LL(0x13b5c88a,0xb32f91ce),LL(0xb8954a33,0x211cbd9c),LL(0x31a311e0,0x72994cae),L_(0x000001f3), + LL(0x3ef4f2d2,0xcf466ff0),LL(0x47044584,0x78dc82b8),LL(0xb52d320e,0x8b5110dc),LL(0xdfe140d7,0xe07b117a),LL(0x0b45fd46,0x39af6581),LL(0xbda19439,0x26b6d5c5),LL(0x8309f53f,0x8091095a),LL(0xaad23c7d,0xb35060d4),L_(0x000001f0), LL(0x90dd82bc,0x1ee78b60),LL(0x839c5155,0x453fc776),LL(0x2966b875,0x6bf1d026),LL(0x9e2a2996,0x825d3c72),LL(0x48c49cfc,0x1345ab1d),LL(0xad600996,0x7e6049f3),LL(0xf1b39850,0x0b007da4),LL(0x2c8f36cb,0xcf942c38),L_(0x00000153), +}, +/* digit=64 base_pwr=2^320 */ +{ + LL(0xeba284c6,0xdcd5cc91),LL(0x292fa3ee,0xea471fd6),LL(0x5841cc32,0x9fb23a12),LL(0x35810a74,0x4e18eb2e),LL(0xd6133648,0x5228a2bf),LL(0x52cab6f6,0x07542e74),LL(0x40c74692,0x62526cd9),LL(0x36b9b329,0xc8dbbd29),L_(0x00000120), LL(0x17573e4f,0xca996c2a),LL(0xc3d42418,0x33e1f9c1),LL(0x970fbd47,0x246b3cbf),LL(0x8c0b1561,0xf0853508),LL(0x16e93234,0x8ff90188),LL(0x74d99d7f,0xaa556f85),LL(0xd3b1d290,0xdda5d989),LL(0xab78218e,0x7de54ed0),L_(0x0000000c), + LL(0x87233d65,0x32cea9ff),LL(0xb1454f2b,0x6963e65d),LL(0xd5f2627e,0xd15b05b7),LL(0x68cad15b,0xf2c9215a),LL(0xf9679cfe,0x982da4ec),LL(0xdd21cc1f,0x73910763),LL(0x3925aff6,0xce110fdc),LL(0xad0858b1,0x0efaca3c),L_(0x000000ae), LL(0x160c51bf,0x88f58f75),LL(0x9e39ee8b,0x713a7cf3),LL(0x5af813ed,0x8ac4ac36),LL(0xa788e43e,0xe789c040),LL(0xc10b5e01,0x3d0cb49e),LL(0x26fdddf3,0x3f2d9bd7),LL(0x504e525a,0x776e3c30),LL(0xa456acdf,0xf6acce51),L_(0x000001b6), + LL(0x23707d7d,0x4b6d85bc),LL(0x2372ec00,0x4b483dd7),LL(0x838f63c9,0x869b15c9),LL(0x40b6584e,0x291644dd),LL(0x05bb5ad6,0x693ec1c9),LL(0xc10969be,0xb5c6018d),LL(0xb81150c7,0x04c9c113),LL(0xbd460de8,0x933a940b),L_(0x0000014c), LL(0x5ad558a1,0xa356589b),LL(0xf88e046d,0xf093ea9c),LL(0xede9de0f,0x39acd54e),LL(0x19ec3f88,0xfcbf451f),LL(0x44ec243f,0xf8b02c0e),LL(0x981fd0d1,0x42d2cc07),LL(0xf4701bca,0x3f363b43),LL(0x30f2e9e4,0x2eafb7f9),L_(0x00000170), + LL(0x006b0772,0xa8c7c3d7),LL(0x1ba3ff28,0xc1d96b23),LL(0xc17a4f5f,0xda50f432),LL(0xca88f653,0xfce5ef14),LL(0x31ac5da9,0xd10257bb),LL(0x18d3105d,0x06b910de),LL(0x4f950082,0x8ed121d6),LL(0x748b9a29,0xeb08be83),L_(0x000001b3), LL(0xfedc9456,0xcda193af),LL(0x30addb34,0xf39dff50),LL(0xa3a58a0d,0x586d72c1),LL(0xd7c02e84,0x7190f71a),LL(0xb6dddddd,0xd7f7815a),LL(0x93fd431d,0xb059af28),LL(0xdb90a301,0x626d66eb),LL(0xb55b2545,0xa7c95ac5),L_(0x00000015), + LL(0x02a70327,0x60173b4c),LL(0x700e187b,0x853a0c8b),LL(0xebfa5d41,0xa74d3fcb),LL(0x6636a248,0x7f152910),LL(0xcd439df1,0x433bf866),LL(0x3d361a48,0xd52b92a9),LL(0x96508fce,0xcdde5dbf),LL(0x08fb48db,0x4598b64c),L_(0x000000ab), LL(0x41e6d707,0xa287bad7),LL(0xc1d0199a,0xaca83d8b),LL(0x3248272c,0xdeea6deb),LL(0x81490886,0xd5830e62),LL(0x803b3e7e,0x0b551501),LL(0x329bb8f5,0xe61ae410),LL(0x1b1ec67e,0x2add209b),LL(0x9d8f057d,0xac0fb9f7),L_(0x000000af), + LL(0x159240ac,0x701cba0d),LL(0xbe49e9de,0x1e2030d5),LL(0xa8d80ea6,0x891f5b9f),LL(0x389aa0a7,0x281d5c9e),LL(0xbf08f46a,0x42c2a6a9),LL(0x30133d89,0xcae5c626),LL(0x26d80fbd,0x976ed6f2),LL(0xd7445273,0xdd69ee7c),L_(0x00000095), LL(0x4b64112e,0x1ba10f05),LL(0x3c715556,0x076de398),LL(0x051c721c,0x1b6338a0),LL(0xed93ec2b,0x0b18e617),LL(0xe40d08e3,0x2546a805),LL(0x39d986d0,0x289546bf),LL(0x87fe36cb,0xcb29a40d),LL(0x28ca6d96,0xcb42c038),L_(0x000001d7), + LL(0xb8888aaa,0xeb89b8a3),LL(0x504b24ba,0x13c31ce0),LL(0x1577d88f,0x1d308489),LL(0x01541da0,0xc31edb15),LL(0xfbe18906,0xcb88a0c0),LL(0xb123cf8a,0xe0a54814),LL(0xce17eb8d,0x12d30b10),LL(0x5435ad11,0x16c373c6),L_(0x000001e1), LL(0x7a3c3081,0x198cbd6e),LL(0x18481bd1,0x9feff602),LL(0x8a4e33b7,0x4dc9559a),LL(0x242155d3,0x49b265ae),LL(0x0458dbdb,0x66003375),LL(0x19c33688,0x53753ede),LL(0xac09e0c8,0x0eb6969a),LL(0x25b27567,0xc165da52),L_(0x000001ad), + LL(0xfd4c030e,0x4bb182d8),LL(0x45ba7b8e,0x24d5733c),LL(0x9bbae322,0x857e0992),LL(0xe18395c5,0x7a4b7ec5),LL(0xbbeb3431,0x9d2ffacf),LL(0x70996597,0x0dac7ff4),LL(0x634b33c0,0xd22ac181),LL(0x5a113dab,0xb7220b32),L_(0x00000027), LL(0x1af6d0b4,0x4b60b5e3),LL(0x60067ebd,0x7c6c236a),LL(0xccf47b3d,0x199b1be8),LL(0x1dbd1cc7,0xe888eba2),LL(0xb4932466,0x034c21f8),LL(0x19ff1dee,0xf9da1696),LL(0xe040c95f,0xee7e95c7),LL(0x9dbe56ef,0x308a2b5a),L_(0x00000094), + LL(0x33a41f31,0xb89596d6),LL(0xde8f6d08,0x36e37e25),LL(0x09152867,0x911d84ea),LL(0x4f3476b2,0xba9def4c),LL(0x44e0d519,0x12065979),LL(0xdfef8c30,0x91c87d28),LL(0xa45cf33c,0x6b8dd103),LL(0x5a48975d,0x7c11f3b7),L_(0x000001e4), LL(0xe73885a7,0x36da50e9),LL(0xdc731cf5,0xb97f8cf8),LL(0x67bb07a8,0x922b0be5),LL(0xfb414a3b,0xe9cbd504),LL(0xd391785e,0x2631b899),LL(0x9eb65672,0x50f31f7b),LL(0x4ee45215,0x0d4d0798),LL(0x23e25b24,0x83771e39),L_(0x00000062), + LL(0x6178bac5,0x9eabc3c5),LL(0xbfa39955,0x503d57c5),LL(0xda006222,0x8e465ace),LL(0xe390a3d7,0x363ca671),LL(0x7ca51f49,0xe0376d27),LL(0xab1c9afc,0x325dbeb1),LL(0xf303951f,0x2ac46079),LL(0x4fcc04e4,0xe25bdab1),L_(0x00000049), LL(0x1e1f126b,0x7a1f14f0),LL(0x0b9e64fc,0x49fb5176),LL(0x394e56b2,0xc1fff51a),LL(0x0b50d33b,0xf135bc4a),LL(0x41e4f563,0xf07911e8),LL(0x0008c3a8,0xfd1855b2),LL(0xd0455066,0xb1f8cdf1),LL(0x3e4e10b9,0xb720d66c),L_(0x00000140), + LL(0x47688773,0x75ef0188),LL(0x81134439,0x6a21abe0),LL(0x3ff532d6,0x9e0177ad),LL(0x3c27f56d,0xe284df24),LL(0xd99892e8,0xfcaf2cd5),LL(0x668c2ac2,0x72c31d05),LL(0x450ea985,0x617df772),LL(0xd1386608,0xffb181fb),L_(0x00000007), LL(0xa81bbd89,0xb7cb40e7),LL(0x6545e4d7,0x4e799290),LL(0x48c0ef0c,0x129414b6),LL(0xf9bc6b77,0x75cac719),LL(0xcf3cf61a,0x7090a084),LL(0x5de671da,0x573167fe),LL(0x53c2428e,0x9be66bcd),LL(0x581cfd76,0xf5419505),L_(0x0000014f), + LL(0x868b1c0c,0x6fc274a1),LL(0x83263833,0x7470faca),LL(0x28bbd5e9,0x10a9a5ed),LL(0x92bc266f,0x1a2df530),LL(0xc1420bb6,0x26088825),LL(0x6de27806,0xb843fbcd),LL(0x96eddc77,0xfaacb0c0),LL(0xe58f23ff,0x249016a2),L_(0x00000132), LL(0x304fead2,0x46f8b4da),LL(0xcdc2767a,0x030d0ccb),LL(0x4ecb91a4,0xa6cdee79),LL(0x546f1657,0x2f10c656),LL(0xa2c85665,0xdaca38a8),LL(0xb2b32405,0xa84dd381),LL(0x29386bb0,0x4d4926a6),LL(0x3ed722f3,0xbe6190e5),L_(0x0000004c), + LL(0xb158c617,0x70c2bdde),LL(0x1e929730,0x03b67c3d),LL(0x83aa10cb,0x207b6554),LL(0xf8bd1f2f,0x65897412),LL(0x52c6a1bf,0xd2b9e2bf),LL(0x4072e449,0x4573028a),LL(0x51728cdb,0x08f548f4),LL(0xa5fb2b4e,0x6569c4cb),L_(0x000000ce), LL(0xc30af1b8,0xf64d0544),LL(0x91fac057,0xe6a44ca8),LL(0x97b402f5,0xf3758797),LL(0x7a48f50b,0x9ec9c59e),LL(0xa20d052f,0x7e0c3edb),LL(0x50d02201,0xba6cf070),LL(0xc4603d10,0x0fd79a40),LL(0x04379719,0x030a1a47),L_(0x000000fd), + LL(0x33d05c0c,0xf4e66422),LL(0x28d54dca,0x3f1a0e0a),LL(0x1b90591f,0x8319bf69),LL(0x50c92b63,0x45f8cdc5),LL(0x2b172382,0xc8908923),LL(0xce47651d,0x282da333),LL(0xe6f22c70,0x6dc02842),LL(0x73a13e20,0xf357a052),L_(0x00000154), LL(0x9bb811ca,0x2aa27cb4),LL(0xff020d4c,0x8f138cd3),LL(0x181a1cec,0x35013750),LL(0x47863f93,0x37481122),LL(0xe6028031,0x18f58c65),LL(0xf01c48b5,0x565f6657),LL(0x9f20924b,0xa7b0ed3f),LL(0x643987ef,0x13cf4aeb),L_(0x0000004e), + LL(0x151f4865,0x1b42c497),LL(0xf30bdc50,0x055325f0),LL(0x144e0aa2,0xf8e98fe1),LL(0xa165a395,0x2e0f9b5a),LL(0x25afa523,0x3ceadf0d),LL(0x70ed634b,0x55dbb9b8),LL(0x1b25f855,0xe2ddb61f),LL(0x8a54708c,0x4ce03600),L_(0x0000004b), LL(0x74847fca,0xf91dd3a7),LL(0x92445716,0xe74dda4f),LL(0xfe51e6c5,0xd2ebe9fa),LL(0xe3bfd67c,0xec65184e),LL(0x51f3767f,0xc26dcf5e),LL(0x6092d164,0x7562e715),LL(0x48053ca5,0xc341746a),LL(0xfe264b56,0x6aa0cec8),L_(0x000001af), + LL(0x688f0816,0x55ab4efe),LL(0x012b1be7,0x48d9609c),LL(0xdb068e78,0x0fb98843),LL(0x958488ad,0xff5eda2d),LL(0x83f6d23c,0x3ec7372a),LL(0xb176c41d,0x5d185ec0),LL(0x925e4903,0x476314a5),LL(0xe4ff9579,0x0d644a3d),L_(0x00000029), LL(0xd529cc94,0xd2ad417d),LL(0x2dfe7d43,0x360ab4fa),LL(0x52cc454d,0xfb2e9eb5),LL(0xa4732c24,0xcf82a235),LL(0x68d1e843,0x67bb7a40),LL(0xbca2ab8e,0x91877eaf),LL(0x99566cd1,0x62574ab0),LL(0x22f9872f,0x687549f9),L_(0x00000183), +}, +/* digit=65 base_pwr=2^325 */ +{ + LL(0x2d85f820,0x287db7bb),LL(0x702fbecc,0xe568667e),LL(0xa157f36d,0xf4ecbeb7),LL(0x484f3352,0x941bbfbf),LL(0x558da014,0x3d5fe38d),LL(0x7b22586c,0x8a8ef1b3),LL(0x7a9e7fea,0xba594962),LL(0x0c422ebe,0x7c9fc474),L_(0x00000117), LL(0x34c2ec0f,0xeb882690),LL(0x8c7ffbe0,0x16f607ed),LL(0x0a729f09,0x9cab235b),LL(0xfb783d21,0xe85a3bb2),LL(0x7a1f91a0,0xaf1659ef),LL(0x067ef36e,0x3c3d4be9),LL(0x2b43e992,0x3b5e5bd9),LL(0xe81391aa,0x6e4981e3),L_(0x000001cc), + LL(0x58cf4fb6,0x2d970cf0),LL(0x6108b652,0x8db98564),LL(0x1c524ec7,0xc375bd09),LL(0x8ded01ba,0xeaf41a1c),LL(0xca7571d5,0x1513bf75),LL(0x83433ae4,0x831a58ce),LL(0xb1cbad60,0xd4b5c1d0),LL(0x7e3558b4,0x054b3a12),L_(0x000000d2), LL(0x5fc5bcaa,0xcd90416c),LL(0x1fd11e95,0xf9cd3e85),LL(0x77429d71,0x4b143cec),LL(0x818263e7,0xb694e333),LL(0x7b0bed2a,0x078fef20),LL(0x900d9d3b,0x22c62d90),LL(0xb2dcc393,0xf713057f),LL(0xee2cd8c7,0x46aefb0a),L_(0x00000048), + LL(0xc6a3697b,0x36666b6c),LL(0x771114f0,0x4615de0a),LL(0x4656bc00,0x27ed1a54),LL(0xfe1b0ffd,0x7a367a4c),LL(0xa7b011fe,0x9395287b),LL(0x1539028f,0xe474d2cd),LL(0x67ab6630,0x50df81c9),LL(0xd416f7d8,0xc4e94809),L_(0x000000c5), LL(0xcd4ff017,0x14e9eb99),LL(0x68ce9cb6,0xefe50131),LL(0x878690dc,0xf0500068),LL(0xa58b25b4,0xfe708a3e),LL(0x1697bfbe,0x4cbc1887),LL(0x109e6148,0xd61b572d),LL(0xea6e538f,0xcd507a67),LL(0x8cf0642b,0x44822c10),L_(0x0000002a), + LL(0x254f2817,0x74f04275),LL(0xe534cfb9,0x22ee390d),LL(0xf368e25e,0xbc4caed9),LL(0x0d3f5a1e,0xd7010447),LL(0x26bb7427,0x91f02404),LL(0x7f0d8308,0x993cd5f3),LL(0x4ebe5786,0x1ba9d89f),LL(0x2549a02c,0x89e62c7b),L_(0x00000089), LL(0x575527d2,0x33afd2d8),LL(0x6240eaa7,0x200435a3),LL(0x0df72a8e,0x104dbed5),LL(0xe9f3dcc7,0x1f5c3464),LL(0x98404140,0x791b398c),LL(0x1581b281,0xd77cd49e),LL(0xa203aa2a,0x2329530c),LL(0x70040738,0x782c024a),L_(0x000001de), + LL(0x8715b292,0x5fca8e16),LL(0x0659f3f5,0x904a8960),LL(0x7cdfccac,0xd46df8f8),LL(0xe8078ecb,0xae2184c0),LL(0xc1352930,0x904b839e),LL(0xf3fd8786,0x36602186),LL(0xc3ec21de,0x2a20030f),LL(0xe08de817,0x592eec46),L_(0x00000098), LL(0xd7e4cb2f,0x812cd124),LL(0x5ac9609a,0x83d9653b),LL(0x690acbe2,0xf981cac9),LL(0xc894c3a8,0xd274538e),LL(0x286285d9,0xc202f8f4),LL(0x24269d02,0x51bb2579),LL(0xe768b7b8,0x121f910a),LL(0x8baea845,0x76c44d58),L_(0x0000006d), + LL(0xf6a81b1e,0xcf66e145),LL(0x2a3bda3a,0xa88833db),LL(0x44164a42,0x4e0df1ec),LL(0x754bd187,0x89a6c53a),LL(0xff8dc770,0x961d8b4b),LL(0xa3761531,0x87d46b93),LL(0x31b05601,0xf7105b06),LL(0x4ef74177,0x500c6356),L_(0x00000082), LL(0xbc131ab6,0x4688d4bb),LL(0x6977d5eb,0x82e94cb0),LL(0x8706473c,0xb785ac18),LL(0x7362c724,0xae704972),LL(0x3b45de5c,0x2e6bdb68),LL(0x2c67f7d5,0x99b0063e),LL(0xa06ed86c,0x4969a5c7),LL(0xc4bf63b2,0xe55691fa),L_(0x000000b8), + LL(0x32f70f29,0x4140b7bb),LL(0xbcda177c,0x0fdc8422),LL(0xad59d7db,0xd2a3a29c),LL(0x9c1893f4,0x8f80936a),LL(0xe60fd330,0xa903804b),LL(0x6b3cc7d2,0x0e38278b),LL(0xfcb7a0c8,0x31deb3a4),LL(0xb47a9458,0x3c25cfff),L_(0x00000030), LL(0xc70a541d,0xcbbeaa75),LL(0x6f004b75,0xd7f127d6),LL(0x1d4ef334,0x15636fa0),LL(0x758ac159,0xa2548921),LL(0xb047a7de,0x60705693),LL(0x128b4b7e,0xa5696c87),LL(0xf499a64e,0xa206ac49),LL(0xf272bcca,0x1f6883bd),L_(0x0000005b), + LL(0x7a4b896d,0xb2e00072),LL(0x47a68dd1,0xd43d9655),LL(0x1ce79a50,0x6abc1f1e),LL(0xec87252c,0xe7160fe5),LL(0xb60d6a5a,0xdcf45caa),LL(0xd235985f,0x1b3180d9),LL(0xf8982569,0xdc646ab8),LL(0x446d6798,0x7ef649f0),L_(0x000001d0), LL(0x6eec719b,0x5f9db663),LL(0xcc9431de,0x38f4a187),LL(0xef383b03,0x3dbac366),LL(0x2fa36674,0xcc0b0d02),LL(0x5760fac1,0x67948b1b),LL(0xe8d42650,0x93934495),LL(0x4f889216,0xd3c64b8f),LL(0x1cd8ec2e,0x0eb0d528),L_(0x00000107), + LL(0x107d6bfc,0x5816dc43),LL(0xb8b5b662,0x1fe1463b),LL(0x973f88f0,0x4370119d),LL(0xc84691e3,0x153d37fa),LL(0xd0059b51,0x9a4e583e),LL(0xe99b060f,0x24c8671a),LL(0x8d5b7c8c,0xfdf410b1),LL(0x0db2233f,0x9b0e342f),L_(0x0000006a), LL(0x08c95cdd,0x07dbde25),LL(0xf5273466,0xf0969049),LL(0x76e98baa,0x8cba001d),LL(0xae173b76,0xcde8bcc2),LL(0xf10bd659,0x09ae5065),LL(0xc7bec674,0x5cecdf22),LL(0x4bdaed4b,0x41bc9eb9),LL(0xc459b9e5,0x7e368546),L_(0x0000007c), + LL(0xd08a3672,0xbcb161c1),LL(0x52ff0a6b,0x0593dede),LL(0x4b01e845,0xb5c016b2),LL(0x421052e7,0x65c310a3),LL(0x6c1dd249,0xf97c5a2d),LL(0x5a4e53fc,0xe551417d),LL(0x0e004126,0xbf9b31fa),LL(0xc101bc8b,0x43a5bd25),L_(0x000000c2), LL(0x2811e723,0xeaad08c2),LL(0xcc586f7d,0xc771ce2f),LL(0x8c0556ea,0xef8166e5),LL(0x32556e9b,0x2d11bd0c),LL(0x11e66977,0xfe7d92c3),LL(0x646a8dfa,0x58b69181),LL(0x8a624576,0x5de6bd35),LL(0xdeb3cceb,0xffa1c827),L_(0x000001e2), + LL(0x88f96f05,0xc1d3ecbd),LL(0x3593555a,0x3073315c),LL(0xb56c336e,0xaec91693),LL(0x5eea4cf3,0x2eade86a),LL(0x90e05846,0xe134f505),LL(0x9189d76c,0x8d429f7e),LL(0x0feade8a,0xf0013b30),LL(0xe083daf3,0xa7b56e97),L_(0x00000103), LL(0x817e33d7,0x9c2537c7),LL(0xa44bf13f,0x9a6a317a),LL(0xda31eee4,0x35ae34c0),LL(0xa0379d97,0xd66b27c5),LL(0x76a48571,0x1ae6d028),LL(0x5d83028d,0xbb8dadac),LL(0x1e4ebd89,0x8aaff54e),LL(0x67e3f97d,0x528b5683),L_(0x000001d8), + LL(0x9a7f8ba8,0x2992a59b),LL(0xa9efff68,0x1a328627),LL(0x3c1d097f,0x555f21eb),LL(0xf75afbb3,0x48ff742c),LL(0x4bf1016a,0xa10b236e),LL(0x5c770a94,0xff3e57b4),LL(0x5915a516,0xc042020f),LL(0x9df7440a,0x8d21ed86),L_(0x00000017), LL(0xecaea7dc,0xbf14c4c4),LL(0xd0035f1c,0x8343dde4),LL(0x19dfc08d,0x3a483722),LL(0x5b130cf1,0xc107a176),LL(0x30fc7b7f,0x6717617b),LL(0xf8cab932,0x626ffb5f),LL(0x59269de8,0x74af7c16),LL(0xbf37b9d1,0x4c20580d),L_(0x00000031), + LL(0xd31224fc,0xe9393683),LL(0x52cacad0,0xd4fad3f5),LL(0xd36819ae,0x14ee7de5),LL(0xb8c9f302,0xdb882d76),LL(0x103a3f52,0xb3362378),LL(0x1728ce43,0x0f7553db),LL(0x29c76302,0x56ba2d84),LL(0xc2f44114,0x28cc209f),L_(0x00000102), LL(0x3c639715,0xd8b8276f),LL(0xacdd7488,0xac8ea8c0),LL(0x912aac3a,0x8de7940b),LL(0x3dabab2d,0xd5bb2c71),LL(0xbc3f4a45,0xbcc2e33c),LL(0x2bf9f840,0x9d0edeba),LL(0x4b00c80e,0xcbe852a6),LL(0xbe4c368d,0x48bbcf91),L_(0x0000011d), + LL(0xc2974837,0x06284fef),LL(0xe24d882e,0x7041703c),LL(0x6f37e52f,0x5e1e37fd),LL(0x7853375d,0x145a0690),LL(0xcc75898e,0x270c5225),LL(0x61f33577,0x82c5658c),LL(0x777ab969,0x47024eca),LL(0x315f136f,0x85165412),L_(0x00000180), LL(0x8c1cda05,0xb7db1fe6),LL(0xedf31bed,0x7f07ba36),LL(0x8079b5f2,0xcdbc2d84),LL(0x22d262c8,0x4bb1ece6),LL(0xb457935a,0x79793249),LL(0x8614cc62,0xe89f8430),LL(0x258e4fa5,0x836da5fd),LL(0x759ca7b4,0xc0d83009),L_(0x000000d5), + LL(0xfa6cf668,0xce181709),LL(0x4b169dbb,0x5bee0010),LL(0x618d280d,0x28a88cdb),LL(0xa9c8042e,0xd8239424),LL(0x8ce15b95,0xcfce331d),LL(0xda740deb,0x6f83c378),LL(0x3c616a52,0xfc05f1b1),LL(0xda236e18,0x3be27f3b),L_(0x0000011a), LL(0x8647ebee,0x64ec0125),LL(0x0db33259,0x91422332),LL(0xf49a7d94,0xa2819732),LL(0xcccef356,0x9b90c693),LL(0x87f18954,0x805deff2),LL(0x22aa64c0,0x294b5e96),LL(0x352be09c,0xa610a1bc),LL(0x8368526f,0xd20e437e),L_(0x00000183), + LL(0x8a8479d8,0x893e0d1c),LL(0x045ea96a,0x972cecee),LL(0x37445b26,0xb250ed0b),LL(0x1c0a16a9,0x08e477bf),LL(0x7509c768,0xc826b683),LL(0xb74870c4,0x008a3f9f),LL(0x4d580408,0x4b88d0f7),LL(0x18d474fc,0xb871463d),L_(0x0000003f), LL(0x2fa143fb,0x726dcabd),LL(0x36d39ee2,0xe017d3cb),LL(0xd8d9e011,0x43bd77e2),LL(0x332c8650,0xc8965069),LL(0x3231a13f,0xf7a775c4),LL(0x1e3de078,0xe93c91cf),LL(0xe0f7c892,0x50d48604),LL(0x27097492,0xf1d9b31d),L_(0x00000042), +}, +/* digit=66 base_pwr=2^330 */ +{ + LL(0x9e603471,0xfb50e447),LL(0x2536308c,0xf617634a),LL(0x9549272f,0x302ba17f),LL(0x3e264556,0x0f6ed916),LL(0xddb056ef,0xc67d2e92),LL(0xf04b9449,0x7bf608be),LL(0xe0fd2d62,0x5ba41494),LL(0xfaa0f9ad,0x7c857eca),L_(0x00000074), LL(0xac0394a9,0x9ed3e6c1),LL(0x9017e273,0x2bf950e4),LL(0x441c9b0d,0x9856ca69),LL(0x717b7978,0x3315b53c),LL(0x66f1bf12,0xc5bce131),LL(0xdc85d5ff,0xf7b0dacb),LL(0x32365700,0xf85306a7),LL(0x22ce0f19,0x406e80fc),L_(0x00000011), + LL(0x4ae474b2,0x3a2640ed),LL(0x3684de2b,0x21f91da9),LL(0x424bab62,0x9955df67),LL(0x50d60209,0x5193b4b5),LL(0x196c99d5,0x8e9f2748),LL(0x7e7f3c74,0x31e6b3fe),LL(0x257248f6,0x73f47f4e),LL(0x56db9ba3,0xb50ccf8f),L_(0x00000137), LL(0x48bc357e,0x3556892c),LL(0xfa4353a3,0xb72124cf),LL(0x6ea40bee,0x1dbe3505),LL(0xb37aa3c7,0x2e951ca1),LL(0xaddc96c7,0x71de6fca),LL(0x58ae291c,0xf88244eb),LL(0x96fd42af,0x7e89a7bf),LL(0x1c5d8ae9,0x9ba25eb4),L_(0x00000178), + LL(0x5894fe38,0xa61f78f7),LL(0x1b7d597b,0x4cc9003f),LL(0x94b8d7ad,0x0c50139b),LL(0x899c26b5,0xd4d5af57),LL(0x5801efbb,0x7d0705ee),LL(0xc45f009b,0x345f5d52),LL(0xc52fa5f2,0xda142009),LL(0x8d175fe7,0x7812ca18),L_(0x0000004c), LL(0x3b59142d,0xc3639f82),LL(0x8ec202a8,0x87394d85),LL(0xb2496e6e,0xa4035f4b),LL(0x7b3e291d,0xcab52bdd),LL(0xda4a9abd,0x41430674),LL(0x23a5aab4,0x7c18c413),LL(0x57ee045c,0xe39c61ef),LL(0x6008e4d9,0x54734790),L_(0x0000017f), + LL(0xc78ac751,0x51e9a589),LL(0x6a5f85af,0xc09d63e5),LL(0x8ce24a89,0x9b655b51),LL(0x5441652a,0x9b445da7),LL(0xf4ffab48,0x9523b0e9),LL(0x23e77128,0x90aaf7cb),LL(0x99234af2,0xb5c9bc78),LL(0xa65dc198,0x8246a1dd),L_(0x0000003f), LL(0xfaeeacca,0x00e72c44),LL(0xa2bbbd59,0x47a63782),LL(0xf531aecb,0x67c393bb),LL(0xc7dda450,0x31630b09),LL(0xe719fa6f,0xc95e46b2),LL(0xf849f3a0,0xdeaf5d70),LL(0x68299654,0x827dd5d4),LL(0x286bc1f0,0xb4def96c),L_(0x0000017a), + LL(0x9eae7cc9,0x62e9226e),LL(0x3410c389,0x2f9c24f5),LL(0x71f68cfe,0x090966e9),LL(0xa4f25f7a,0xbb4733af),LL(0x45a0c4d6,0x6303208f),LL(0x0dc0c0a0,0xc45e9f18),LL(0x9a589e6d,0x5c94f082),LL(0xb9c34b5b,0x9266dd1d),L_(0x000000e1), LL(0x9fc32695,0xaccb375d),LL(0x15ea0e6d,0x5d3b353b),LL(0xc2dc172e,0xdcb147d5),LL(0x96265816,0x7b5ea6b9),LL(0x7188496d,0x6c199f62),LL(0x067983d5,0x8be2b6ee),LL(0x804ab5f0,0xb8433a5a),LL(0xfb9701f7,0x08ec1b7f),L_(0x00000005), + LL(0xf346db2e,0x75d8b8a7),LL(0x6d441b3f,0x33c9dd4c),LL(0x49bf3101,0x43c1a96f),LL(0x586195d3,0xc64bca08),LL(0x797aa157,0x35872dbe),LL(0xf8494dd7,0x155f257f),LL(0x370900d2,0x42b380b7),LL(0x2ec6eba8,0x0de4106f),L_(0x00000122), LL(0xd54c0210,0x9ba3b710),LL(0xdf76d347,0x79738efe),LL(0x168b7e7a,0x59395338),LL(0x5be21cb3,0x786b578f),LL(0x6c93997c,0xb42b9419),LL(0xda95deb9,0x1b5aa55d),LL(0x1941c038,0x6282d548),LL(0xdc8f9a51,0x155e294b),L_(0x00000038), + LL(0xc54fa3d4,0xc9331956),LL(0x37673a44,0x95ef7146),LL(0x8699a77e,0x35d322ef),LL(0xe5aa4366,0xb0a5fb37),LL(0x1c2f8160,0xdaf11474),LL(0x6c679654,0x4ae2c0a4),LL(0x25f5bd9f,0x0cd1a20c),LL(0x2812f915,0xddb9cbb7),L_(0x00000116), LL(0x6ba35704,0xddf43f70),LL(0x66ebac5d,0xbd74a353),LL(0x5836be03,0x8be404aa),LL(0x606cb997,0xa72b6949),LL(0xe5570eb3,0xe6f5f53a),LL(0x1550fdd3,0xd841cc12),LL(0x83299eb2,0xcf113a07),LL(0x76117e23,0xb031e54d),L_(0x000001d5), + LL(0x0412d583,0xe9241a8a),LL(0xac6aa964,0x24077ab2),LL(0xe1536c1d,0xad189da3),LL(0xdf56af4a,0xfbfb6e01),LL(0x46ef9e57,0x8ca8e624),LL(0x7c9eb17f,0xcbcb351a),LL(0xf3eda4f9,0x0985fb54),LL(0x69708739,0x4d5e2c29),L_(0x00000119), LL(0x4656d8d3,0x4d7980be),LL(0x8d7ed681,0x65dbc6a6),LL(0xe9b6528d,0x433cebeb),LL(0x9dfcc27d,0x25c88d26),LL(0x273aec82,0x43f7caaf),LL(0x8662d906,0x1c78327e),LL(0xe4a09eb4,0x04763fda),LL(0xad9bd2f6,0xbfba2b0e),L_(0x0000005c), + LL(0x56bd282f,0xc5002426),LL(0x8905b0bd,0xc776fda9),LL(0x5fecb366,0x691702ab),LL(0x8c3c6d77,0x3eafba03),LL(0x7b9bb860,0xa5a38b66),LL(0x288652df,0x9d4dccfc),LL(0x8b876824,0xf937bd4d),LL(0xedcfeccb,0x8bbe45b7),L_(0x0000009b), LL(0x4223e6a4,0xffdb1d9f),LL(0x44384141,0x4395593e),LL(0x68a0f97c,0x4b9f43be),LL(0xa0157cb8,0x99290f7c),LL(0xcccef6ba,0xef0777fc),LL(0x4349cd23,0xc43d71c0),LL(0x17318622,0xa3bb9fb0),LL(0x5a9a1343,0x039a4c9d),L_(0x000000c9), + LL(0xd92ddb9e,0x22ba5c1f),LL(0x65cc6bf6,0xa333c9c6),LL(0xb76d024f,0x521f0218),LL(0x51355f8c,0xb277d241),LL(0x061005e2,0x36014e7f),LL(0x322c2c56,0x52d2dee0),LL(0x8916c0e7,0xf40b1b75),LL(0x0e73d5bd,0xcfe8f526),L_(0x000001ea), LL(0x2bc1b23d,0x34905a27),LL(0x31577092,0xac41b5aa),LL(0xf6758a71,0x0e95917e),LL(0xeccda7b1,0x35f458d2),LL(0x52d35c5f,0x61af1ed0),LL(0x42e21d1f,0x99ea1f96),LL(0x1b6c4f11,0x1c9ff42a),LL(0x5a729a64,0xdaa6b431),L_(0x00000185), + LL(0xa860137a,0x2bb38059),LL(0x7d4fcbe1,0xaebc7ba3),LL(0x3de80141,0xabc6d3af),LL(0x0ce79f66,0xecaf11f9),LL(0x9743b455,0x23367c7f),LL(0xcf9fcf33,0x458f9b06),LL(0xaf18324a,0x1dd894da),LL(0xc4c95fe7,0x09129d0b),L_(0x000000be), LL(0xe5b4edaf,0xff94f2f7),LL(0xdb54a58e,0xc8f912e0),LL(0x655ce3bf,0xdffa0710),LL(0xb0757830,0x0f2d4402),LL(0xb963f905,0x95d5e868),LL(0x098d24ba,0xcdbb826b),LL(0xba591cbd,0xb02b1feb),LL(0x6a8df836,0xd82281ce),L_(0x000001b1), + LL(0x108f42a8,0x9cf21db6),LL(0xddcfd187,0x555cfe2b),LL(0xaf269c11,0xe7b4d452),LL(0xc4011856,0x3cb7c3fc),LL(0xfe6e4559,0x415957c0),LL(0x8996b215,0x3b983b2c),LL(0xbe5cf31d,0x0312f953),LL(0x78abc3a0,0x37adff70),L_(0x00000185), LL(0xd52c0d6b,0x847e724e),LL(0x42bcb7b2,0xcfdfc839),LL(0x2ddac314,0x2aac1c0a),LL(0xa690e67e,0x60736a1d),LL(0xe310507c,0x15f2f407),LL(0xa30b8b85,0x03447dd4),LL(0xf3ddc7c2,0x87208fe5),LL(0x482e1135,0x70157470),L_(0x00000073), + LL(0x4b696b0d,0x98400d78),LL(0x671aa2b1,0x05e6f78c),LL(0x19c7b31c,0x2f26896d),LL(0x537e98fd,0xc925cf6a),LL(0x31498b2c,0xe43ae0bb),LL(0x5b628896,0x2cc9c889),LL(0xf81936b3,0xd5a79df1),LL(0xaa558d67,0xdd9726a6),L_(0x00000127), LL(0x947e26f1,0xd8102ba2),LL(0xcba2e206,0xf57819ca),LL(0xe1b53333,0xb14e41e4),LL(0x89b722a1,0xa87036c9),LL(0x43d9c2e8,0x8c5e594a),LL(0x8a9f09ad,0xfc198885),LL(0x96afae31,0xd164a447),LL(0xf6ad8705,0xa304b415),L_(0x0000018c), + LL(0x51de1f32,0xfe892ec2),LL(0xde0ad941,0xa5e0c485),LL(0x252b5759,0x04504e2c),LL(0xe069dceb,0xceea659d),LL(0x80061659,0xb3fe1e63),LL(0xee236d87,0x846da87b),LL(0xf97ae613,0xa90f8433),LL(0xe7abcaf3,0x2f8213d9),L_(0x0000008a), LL(0x583a7dc4,0x6598f335),LL(0x8097f299,0x89f7734d),LL(0xff15633b,0x8aebbf6c),LL(0xb01c7b6a,0xb5108c62),LL(0xc7f93ae2,0xf807ee31),LL(0xf990d4e3,0x34992a71),LL(0x9962859e,0x282fca85),LL(0x8047dde1,0xc000a572),L_(0x00000113), + LL(0x5945bc53,0xcf237d07),LL(0xe31f2468,0x1ef4bb2f),LL(0x641e2901,0x2c562b14),LL(0x06773e1d,0x2ff0373c),LL(0xca66e36e,0xc519e2a9),LL(0x716d0497,0xbfb75cdb),LL(0x9a1fa9e2,0x0dddfb32),LL(0x1e517999,0x6b953285),L_(0x000000a8), LL(0x9daa8f05,0xe87b5c36),LL(0xba374fa1,0xcd0b7e7d),LL(0xf9ec22c4,0xd20b7cb2),LL(0xdd4d581d,0x516610e0),LL(0x74ddb0da,0x126aa3e2),LL(0x4e09fb27,0x35d95ce7),LL(0x0c242711,0x40d02e9e),LL(0x926ede13,0x8d5dfa4d),L_(0x0000010b), + LL(0xb7d7682e,0xd8734ac8),LL(0xd322358b,0x75fcdec6),LL(0x56d9d86e,0xe72ffc35),LL(0x6c363b61,0xd1066f3a),LL(0x3d03c2ea,0x7095dbd7),LL(0xcd1674fd,0xb42f9972),LL(0xacc3e682,0x9370acc2),LL(0x80c71149,0xa71a76fd),L_(0x0000016a), LL(0x5a49c095,0xe1070948),LL(0xd0d2a294,0xe55bcaed),LL(0xa073e38c,0xd2884da3),LL(0x25346561,0x95d34747),LL(0xbed8195e,0x990b2c19),LL(0xefe701ec,0x17664aec),LL(0x4c59ce88,0xcb5f1246),LL(0xa23cc1c6,0x846bef85),L_(0x00000113), +}, +/* digit=67 base_pwr=2^335 */ +{ + LL(0x72a8d985,0x39a70c2f),LL(0xea7d5b7d,0xf6f8acab),LL(0x73f642dc,0xdeb95d02),LL(0x8a54bc56,0x3f3cc37d),LL(0x5a63f188,0x1ef1bba0),LL(0x8c074b83,0xe112f14c),LL(0x17f937a8,0x29897379),LL(0x4d335ca2,0x57340487),L_(0x000001ef), LL(0x998374f7,0x8ee96d7e),LL(0x7d432cad,0xbff9e11a),LL(0xe6665366,0xf6d56384),LL(0x9b692423,0xfc7e344a),LL(0xf75f044e,0xee1b3ddd),LL(0xa827ee60,0x9cd00df7),LL(0xb2612c93,0x3d529eb5),LL(0xc4ffa6a1,0x5e23ecd7),L_(0x000000e1), + LL(0x39c16671,0x3753ce58),LL(0x7356faf0,0x54c5f8c0),LL(0x1165a356,0xf5c1afc7),LL(0xd6adf86c,0x903b89f9),LL(0xba6b4966,0xbff86c3f),LL(0xb1519f4e,0xa87b0151),LL(0xbe4f95b4,0x85efd27b),LL(0x0513d263,0xee9e2b04),L_(0x0000014c), LL(0xd4622a63,0x2aca99c7),LL(0xa6efd8d9,0x1d6acf3a),LL(0x7e55d6dd,0x95a1b738),LL(0xdb119c22,0xd9703d10),LL(0xd11a67da,0x427c0f52),LL(0xe412eedb,0xe055192f),LL(0x174c7a31,0x404a5758),LL(0xfd4b1dde,0xbb21a55c),L_(0x000001f4), + LL(0xc51693d1,0x57ada379),LL(0x7b463243,0x9dea1156),LL(0x65ad7ad1,0x3431ffb8),LL(0x22995c21,0xbfb6e47d),LL(0xa47b2e96,0xecc87ed5),LL(0xb5d8ce10,0xab93da78),LL(0xe414756b,0x0b5319cc),LL(0x327dee6e,0x82e08d68),L_(0x0000010d), LL(0x68a2cd80,0xd1a1a9cc),LL(0x62b72c52,0x2c285a2a),LL(0x23638d39,0x1736a146),LL(0x90e668b9,0x99d14c12),LL(0xe73e0b34,0x0861672c),LL(0x88955dcb,0x7dc3cdab),LL(0x06284ae9,0x4fad41f5),LL(0xfe7ca883,0x16e468d7),L_(0x0000001a), + LL(0x370d119f,0x9990254b),LL(0x764d3fe4,0x8a86cb40),LL(0x39be0e2c,0x4b9820af),LL(0x458321b0,0x29475227),LL(0x2c2ba583,0xa07a5c7f),LL(0x52e9ae89,0xfa6d5206),LL(0xbdc0eee8,0x435604c9),LL(0x6f2e4842,0xd30a60c0),L_(0x000000ff), LL(0x1714a30a,0x66400030),LL(0xa3f37cbd,0xab87a938),LL(0xf3132874,0xecbe1c91),LL(0x7e9d7ac6,0x1734fae5),LL(0x7e33fb88,0x600765b0),LL(0xedfc073b,0x428cbfb5),LL(0x85e9a209,0xbd290285),LL(0x77ef7692,0xf50381e0),L_(0x00000057), + LL(0xec2103fa,0xa28d165e),LL(0x31bc9c0a,0x9e3c2272),LL(0x1db480d5,0xeefa29bf),LL(0x5fe970c7,0x0625c44f),LL(0xa6473b7b,0xd9e52858),LL(0x66b89d6b,0xea4cd7c9),LL(0xc3e3b579,0x3b6547a2),LL(0xa8bdce16,0x517b2e4e),L_(0x00000152), LL(0x66f8ad71,0x8a1901b7),LL(0x08802270,0x0ceae5cf),LL(0x853e6e60,0x63ae6ec1),LL(0xd89e54bb,0x1f365d84),LL(0x9e03d94f,0x49df44dd),LL(0xddc9e1b3,0xf70096c2),LL(0xba5865a7,0x6cc7a69e),LL(0x01800fb8,0xcbb03fad),L_(0x00000080), + LL(0x91cf34a0,0xc22b2802),LL(0xfd975370,0xe87a26b1),LL(0x662b3666,0x46d8088a),LL(0xa7466010,0x0988f2ee),LL(0xd8edbdfb,0x1f7fc1eb),LL(0xf4b2f213,0x266b6d41),LL(0x00896949,0xc83c1c02),LL(0x7c849de8,0xedffcbe2),L_(0x000001ad), LL(0x90bcda37,0x480ec74a),LL(0xdde9d726,0x26216e2d),LL(0xbef16495,0xa5c64b02),LL(0xfba3c749,0xc3f630aa),LL(0xea872930,0x8695df3b),LL(0xb21d654a,0xb5372491),LL(0xa2f3f6fc,0xe917f3b3),LL(0x13fe01cf,0x4b29c946),L_(0x000001e7), + LL(0x12faa2d3,0x93a86247),LL(0x67da197c,0xd4612a0b),LL(0xfd72cd4e,0x145bbdec),LL(0x8483b822,0xb4dba8a8),LL(0xf56d58c6,0x81ec1ed7),LL(0x708f26ec,0x631a5032),LL(0xf4782ede,0xb4b04fc8),LL(0x34a251ec,0x0d9db88d),L_(0x000001ac), LL(0x1b8eedbc,0xadbb93b8),LL(0x1b68a64f,0x2308b6f8),LL(0xc37e6bd6,0x392cec36),LL(0x7b419db4,0x0aee0e63),LL(0x1777f3d5,0xff167cd9),LL(0x706d5278,0xede81ea9),LL(0x2ecab8c0,0x63b7e96c),LL(0xb7b3cb43,0xb821ee5a),L_(0x000001b0), + LL(0xec8937b4,0xbb7112fd),LL(0x3ed5a415,0x66e9e19d),LL(0x01eab0fa,0xbbe65978),LL(0x740c409b,0xba92675e),LL(0xa050b19b,0x3e8b56da),LL(0xe6eedf2c,0x2fbcf099),LL(0xa55e0691,0x80195262),LL(0xf2c7d1e9,0x72121ee2),L_(0x000000f6), LL(0xeec1384c,0x51ffea5b),LL(0xeaca6749,0xd0ad477d),LL(0x45756473,0x1fd4ee32),LL(0x80864216,0xa3069430),LL(0x850c8b97,0x405f653b),LL(0x9de4340f,0x5a543cae),LL(0x347d550d,0xa331ca24),LL(0x75f4312e,0x7ca7992a),L_(0x00000002), + LL(0x6f04ee1f,0x5987ecf0),LL(0xe39114e7,0x8c92a999),LL(0x734b40d0,0x3eb0ff94),LL(0x35b97b9a,0xf3a7b34c),LL(0xd5b35118,0xd35276ae),LL(0xfa36fe0d,0x9e933110),LL(0x3c37067d,0xbe64ccaa),LL(0x9629f86f,0xf4e80957),L_(0x000001ef), LL(0x2f066fb6,0x69fb1dea),LL(0xd4e489f4,0x65ba5821),LL(0x3abd59aa,0xf0068abc),LL(0xf152d51c,0xfa26b25b),LL(0x4c7900c0,0x929fd963),LL(0x22beebe6,0xd19508e6),LL(0x6c8e147f,0x84fd88ff),LL(0x8a8eae8c,0xf8e5d746),L_(0x00000039), + LL(0x58c8162c,0x15dbf9ea),LL(0x7dd87f67,0x5523d821),LL(0x2bc5b0b4,0x8780b2c5),LL(0xb8903ecd,0xf92e785d),LL(0x54296f75,0x6397e404),LL(0xcce33c6e,0x84bad1cc),LL(0xd3c5f54e,0x95b82162),LL(0x33f935ae,0xddedf093),L_(0x000000e8), LL(0xf54ff1c3,0xb4ec10e1),LL(0xc2886785,0xa1634274),LL(0xe5822d49,0xbfab5d5e),LL(0xbe9122e0,0x955a062f),LL(0xf03c2cc2,0x579ad9b7),LL(0xf2e5e08e,0xdd6ee255),LL(0x1b65e701,0x934f08f7),LL(0xcd7d23cb,0x4d45cc9d),L_(0x0000007c), + LL(0xaeacb5da,0x9e604f26),LL(0xf00003d7,0xb32a37b8),LL(0x5e05fada,0x771ab3ac),LL(0x03aa3a60,0xe17eba9f),LL(0x31442064,0x7239319d),LL(0x021c13e4,0xa35b4712),LL(0x3f7e400f,0xc6e5283a),LL(0x5888abe0,0xbae7d910),L_(0x0000015b), LL(0x0081a506,0x59fee06e),LL(0x6e2f4f1a,0x372aea8b),LL(0x11014792,0x2416f852),LL(0x35841e3f,0x71de69f4),LL(0x710be3f1,0xfa1b9018),LL(0xd8151855,0x92a94717),LL(0x52addc4d,0xff148def),LL(0xea65eaa7,0x9642343b),L_(0x0000008b), + LL(0x01927819,0xd986cf50),LL(0xab17488d,0xeb6ebddd),LL(0x8628281f,0x92fa1e38),LL(0x4511bc03,0xe1691b01),LL(0x0b79c2a5,0xb842f844),LL(0x8805d866,0x343c71a4),LL(0xd5d795d0,0x042ac5c5),LL(0xedd85588,0xb87124aa),L_(0x000001b0), LL(0xfa896067,0x0d43dab8),LL(0xd6f1580b,0x9a3104ec),LL(0x840b3b59,0x76717c31),LL(0x41201091,0x2243eb78),LL(0x88de871b,0xe2323a38),LL(0x53ac3f7b,0x764799c3),LL(0xdaf476ff,0x999d244e),LL(0xc595b87c,0xbe221513),L_(0x0000010e), + LL(0x8985e912,0x3a4ec3d9),LL(0x639aa554,0x9515b9e8),LL(0x2b90b395,0xe330201b),LL(0x5d9f9a07,0x95f372a0),LL(0x83696835,0x0d679c0c),LL(0x8c8132ef,0xbc4509be),LL(0x5efb5013,0x1a4c14f9),LL(0x7a8442ff,0x59dd0f6b),L_(0x000000f6), LL(0x1d220323,0x607be429),LL(0x7b931a60,0x3b156b2a),LL(0x38e10514,0x36aabd76),LL(0x8ff10073,0x384c71ad),LL(0x3603a7b3,0x1c1f643f),LL(0xe305de49,0x08714206),LL(0x63b53241,0x57add901),LL(0xfba33a9a,0xf2cced51),L_(0x0000004e), + LL(0x03800832,0xceb77120),LL(0xc3398df2,0x625cbf7e),LL(0x4c74b442,0xdf67b9eb),LL(0x424f2515,0x6b977890),LL(0x2abc1051,0x2462b490),LL(0x7c8f5df8,0x6d60d980),LL(0x1bb838ca,0xff606aee),LL(0x1e7a2ff1,0xfb6f830e),L_(0x000001cd), LL(0x238fce9b,0x49a4c209),LL(0x73f7bb70,0xae2b39f7),LL(0xcfbf4b1e,0x53e2f55b),LL(0x7309ae96,0x026775b0),LL(0xfcf869d6,0x1b0e83da),LL(0x72ed99fe,0x02f8a21d),LL(0xe81bf7a2,0xd737619d),LL(0x2bf238b7,0xd0a7eb7b),L_(0x00000119), + LL(0x38b832fc,0xb158ca8a),LL(0xb03ae8bb,0x7420634b),LL(0xac8efa9e,0xaa8dadd3),LL(0x259a7c31,0x525d1488),LL(0xf727fdfe,0x8b687898),LL(0x48ffe9fc,0xfbbb696d),LL(0xb00d2cb4,0xf4a8353d),LL(0x9be76892,0x472fe4b6),L_(0x000001a2), LL(0xf73fea91,0x0e2a8788),LL(0xfce9c9d4,0x18c2b050),LL(0x3412a938,0x1844f4f0),LL(0xc14eff85,0x4ad5ef54),LL(0x0ad47ee3,0x01446b7d),LL(0x4c160f3e,0xc22336c4),LL(0xc43f42cf,0x64f51a4f),LL(0x34833c47,0xc43f0955),L_(0x00000133), + LL(0x7f8ca9dd,0x308270c4),LL(0x433780e1,0x2be3ff97),LL(0xb186cdf8,0xeb6471b0),LL(0x747ce696,0x05f60ab1),LL(0x815fe310,0x5a457da8),LL(0x6bb9ac1b,0x4901af0f),LL(0x925bd14f,0x652f972c),LL(0xacd8b58a,0x53143186),L_(0x000000a8), LL(0x3e0f3553,0xf7ba0c69),LL(0x6dbda5e1,0x5b11db3d),LL(0xe46fae1c,0xf1f51b89),LL(0x7c0b46b0,0xa48d5ec9),LL(0x1cc037ca,0xbdcc7599),LL(0xe5f40355,0x28784dc0),LL(0xd1a3fa4f,0x0b837ead),LL(0xb5a1926d,0x3098c9ee),L_(0x000000e4), +}, +/* digit=68 base_pwr=2^340 */ +{ + LL(0xde847fc3,0x2cc0ff17),LL(0x8ee8bde0,0x63e6fe38),LL(0x774123d0,0x97b1e73a),LL(0xf8cb9f55,0xee20c8da),LL(0x8b938535,0xa8b1bfac),LL(0x3df10d7d,0xe2a81936),LL(0xe7d42738,0x9979861a),LL(0xaf0a6779,0xc4dba03e),L_(0x0000012f), LL(0xbb611141,0xaefc5b58),LL(0x9b3d91d2,0x33633ce2),LL(0x0d5e3306,0x2544bfef),LL(0x8956c10a,0xbeaa2dd7),LL(0xcfa84101,0x8ba22402),LL(0x96a5f958,0x0df8f462),LL(0x34ace12e,0xc6c70187),LL(0xe0e00d25,0x01914ca8),L_(0x000001bc), + LL(0x8cc26d9a,0x4788e280),LL(0xddb52ac3,0xa2e3c895),LL(0x305bc55d,0x6e792cc3),LL(0xee807637,0x5f9d9986),LL(0x0da2d9cb,0xaea185a3),LL(0x56f8f0d8,0x8ef462b9),LL(0x096bebd6,0x300f61c2),LL(0x7be11b59,0x9b4cc44e),L_(0x00000051), LL(0x9dea58bb,0x2585fb77),LL(0x8271cd5b,0xd75c3d97),LL(0x5ac1a4b5,0x4eee827a),LL(0x6fe4d480,0x0147df6c),LL(0x7f66c09f,0x3f6a4c21),LL(0x92c3b320,0x1eeb9502),LL(0x17768418,0xebda3f44),LL(0x12f688be,0xfba26717),L_(0x0000002b), + LL(0xea4586de,0xdc25b073),LL(0xf24773b7,0x49b36fcc),LL(0x8466e794,0x3cc03245),LL(0x36f85404,0x02d71d6f),LL(0x251e24c9,0x1a81ea4c),LL(0x9e39ec68,0x0e710d11),LL(0x0e27e696,0x44dfa8e5),LL(0x6885ae0f,0x4f91bbc9),L_(0x00000100), LL(0x7872cc7e,0xb8224b0f),LL(0xb5696e54,0xe5d4bd3d),LL(0x64ead37d,0xcbad27cc),LL(0x5b2f36d2,0xa45e529b),LL(0x521210e2,0x788fbf74),LL(0xe1bd2c8d,0x40c5440b),LL(0x2b990184,0xbc157b39),LL(0xeeecc510,0xca9c145d),L_(0x000001c7), + LL(0x241f3444,0xaf242ed2),LL(0x73f71786,0x82ffb7bd),LL(0xee8f684b,0x0b940040),LL(0x0e0a7766,0x3cb9180b),LL(0x05509077,0x41ff934d),LL(0xd7e63b43,0xfe154397),LL(0x597b1f2e,0x73566086),LL(0x115dad8d,0x8383d38e),L_(0x00000032), LL(0xa0320279,0x44ad3c59),LL(0x95d0b92b,0x9546c02d),LL(0x17644016,0x0cf317d6),LL(0xb1278de8,0x0cc035b3),LL(0xa150eec2,0xa454911c),LL(0x40f34047,0xb15350f1),LL(0x64a854ce,0x24297dd6),LL(0x45fd389a,0xfddaec15),L_(0x000001d2), + LL(0x1b54c6d4,0xd46442b3),LL(0xc94971a5,0x8eb97dd3),LL(0x9e3e1ff6,0xe2df3525),LL(0x058db0b2,0xe0b69449),LL(0x46968077,0x4ea6ec9f),LL(0xba76456e,0x8457796a),LL(0x77183369,0x255412cc),LL(0xb4e4306f,0x9bbd6c5e),L_(0x0000005e), LL(0xcef4a91b,0x57dadc33),LL(0xb934de9a,0x51f56ade),LL(0x2e8fe341,0x49ab29d2),LL(0x85723b5e,0xfabf520f),LL(0x1ee66b41,0xbffe6761),LL(0x0a605253,0x50202f55),LL(0x9e49468c,0xef6d250b),LL(0x956ea13f,0x0c1b81a2),L_(0x000001e4), + LL(0xe30dc097,0xb5323240),LL(0x3dd49c5e,0x082d33c5),LL(0x17c692b6,0x12bc5d69),LL(0x7e9d695a,0x02c7fd33),LL(0xb78372d6,0x85e92117),LL(0xefa0ef29,0xceacebcb),LL(0x9b4e8e18,0x3d68cc3e),LL(0xe3e50ee1,0x56a48db3),L_(0x000001d4), LL(0x5f980bec,0x9b062f58),LL(0x03edaeaa,0x83ee3b51),LL(0x4f1f8028,0xe4a60553),LL(0xd292d29e,0xf95f8dad),LL(0x7f9ba28d,0xfb5785f5),LL(0x54b8d134,0x0fe162fe),LL(0x2bd8287f,0x2247f090),LL(0x769ce1a1,0x5ebc6de3),L_(0x0000001a), + LL(0x416e3d53,0x74341e71),LL(0xa2b4eeb7,0x9e6b225b),LL(0xff742236,0xd3c824dc),LL(0x5e3b9c67,0xd547224d),LL(0xc6e4276d,0xcc105d46),LL(0x8a001d2d,0x92da4b3a),LL(0x3b252483,0x67e2f395),LL(0xcbb5174b,0x659222fe),L_(0x00000118), LL(0x2308d720,0x4e2e8071),LL(0x9562db9b,0x1bf6b116),LL(0xb14e6841,0x45f6d2ac),LL(0xbfd32572,0x8cdf458c),LL(0xe189ac1c,0x3f894fd3),LL(0xb5d6ee49,0xa59cbf0a),LL(0x672a0c8f,0xd8476d8e),LL(0xfbb78753,0x104d6ed9),L_(0x000000a1), + LL(0x4e39334c,0x0560e3e7),LL(0xc62b58ab,0xfa2ff6e2),LL(0x25b0cfee,0xe3dacd7d),LL(0x56f469ae,0x6476d0c4),LL(0xaa862fad,0x0d8d2340),LL(0x468a688f,0x648a9494),LL(0x9d4d2feb,0xe9f4ed20),LL(0x3e7c5890,0xb2f118a9),L_(0x00000039), LL(0x12cdd07e,0x60784376),LL(0x8d6d1246,0xea183586),LL(0x85400753,0x5c35ba6a),LL(0xf252808d,0x57ba4df7),LL(0x07b45d68,0x048ddde7),LL(0x8c60d683,0x759393e3),LL(0x919b183c,0x760c630e),LL(0x90171725,0xc0e42720),L_(0x0000007e), + LL(0xc6d9113b,0xdfe5709e),LL(0x1b269289,0x74793368),LL(0x0c1477a3,0x3c2fbb12),LL(0x544d389c,0xbce04baa),LL(0x4fcb6058,0x52137546),LL(0x62503bea,0x306eaabe),LL(0xb0495976,0x49afd268),LL(0xa62777c0,0x9d57af71),L_(0x000001c4), LL(0x5a2c2301,0x122ed53b),LL(0x23b67f05,0x68235f24),LL(0x0b4f5601,0x850a3bcc),LL(0xfc63cfca,0xfb6987d1),LL(0x7f3a86aa,0xeb70f694),LL(0x726395f3,0xe2648fe0),LL(0xfc883eec,0x04f6906b),LL(0x551243f7,0xb7f9a9a2),L_(0x000001b1), + LL(0x1329d8d9,0x09b5385e),LL(0xe206cc29,0x8ad6ff5e),LL(0xaa658f94,0x8f946f98),LL(0x428d98ec,0x2b10de6d),LL(0x3ec6baf6,0x7efb3f61),LL(0x21982753,0x47b7f15b),LL(0x578e65b3,0x63e00597),LL(0x5b3f1a2b,0x4f67290b),L_(0x0000003d), LL(0xec6244fb,0xb103454a),LL(0xbf7ff50a,0xc09eeea3),LL(0x5b8194a6,0x117b25e8),LL(0xb8d95b3b,0x4719134d),LL(0x98e3eda5,0xfb408475),LL(0x184c4131,0xaae4a703),LL(0x433cd4bd,0xee1c89f5),LL(0xec3f5308,0xffc4a25f),L_(0x00000190), + LL(0x0a7c8b60,0x2ac3e5be),LL(0x35df434c,0x73674292),LL(0x218ab0b7,0x377ea177),LL(0x971e7bf8,0x979969d0),LL(0xa6fc69b3,0xf051da86),LL(0x0f199014,0x50a5af5c),LL(0x0bd76f2a,0x68112265),LL(0x6ef71ee6,0x92fc81c9),L_(0x0000002a), LL(0x2c92d1d2,0xa2718b70),LL(0x8bc153fa,0xefc700c8),LL(0x7d63139e,0x11bfab90),LL(0x63a1a94a,0x1c81d0a8),LL(0x0af04b86,0x5010811a),LL(0x65f6b2d4,0xd230f7b9),LL(0x346b9acb,0x92db7ff2),LL(0x791ad571,0x423539f4),L_(0x0000016d), + LL(0x0bfa96fe,0x4b219ff8),LL(0xe186bb8d,0x34f0b2a5),LL(0x4bbcffce,0x74e4771d),LL(0x4fa8ab52,0xa8af538f),LL(0x0bd9aa17,0x55b8ec20),LL(0x24b82cd8,0x55a1e2c4),LL(0xf54b7fae,0x34a02ce9),LL(0x011a367d,0x3f61c656),L_(0x000000e6), LL(0xb4418223,0xf5b325cc),LL(0x48062883,0xae0ea746),LL(0x3f6511a6,0xa6b89032),LL(0x057c30cb,0xe93a6d10),LL(0x00156e39,0x0c08d7cc),LL(0x7ef1eeb2,0x713dea1e),LL(0xeb905073,0x1c6abbcd),LL(0x4e60ee01,0xb23a579c),L_(0x00000093), + LL(0x1db48f4d,0x239e10c2),LL(0x3cbc9af9,0xde4c09f2),LL(0xabdb2bfd,0xed72b749),LL(0xf64b181a,0x928d6d6b),LL(0xc85d2a91,0x9af06cb2),LL(0x32a985b4,0x67685c7b),LL(0x4cf68328,0x607e3b33),LL(0x96bdc7c4,0xa7e4db64),L_(0x00000004), LL(0xc31b32d2,0xd33b72d3),LL(0xf9f3e673,0xd3e9eee6),LL(0x42fb631c,0x43164033),LL(0x6eedae95,0xaab1e76b),LL(0x986360ea,0x9e6c1afa),LL(0x30e60546,0xffd16939),LL(0x06fbdf1b,0x989aed4f),LL(0x7870903a,0x905f7535),L_(0x0000008f), + LL(0x1065b996,0x62ecea90),LL(0x92968f64,0x48a2356f),LL(0xe74f9d69,0xc38162fc),LL(0x799bc2e4,0x109958a5),LL(0xc2cf37fd,0xc511c5bf),LL(0xb810f3aa,0xba774e2d),LL(0x04423cab,0x7df651dd),LL(0xfa41efc5,0xd82de4be),L_(0x00000005), LL(0x843bf3e9,0xca2ddefb),LL(0x07001aa8,0x2325dc6b),LL(0xfb77a925,0x19ebdc37),LL(0x984cbf76,0x991269aa),LL(0x0ea96690,0x5a99df72),LL(0xf5a76c72,0x261ea01a),LL(0x11ed492c,0xec2261aa),LL(0x511c25de,0x10d398dd),L_(0x0000018c), + LL(0x79d76ece,0xdf3265ef),LL(0xdd97cbec,0x1201ac02),LL(0x82ed80a6,0x3e075507),LL(0x2e6d2b22,0x16c3d5bb),LL(0x9b31411b,0x89a53e5c),LL(0x4570ab97,0x5d2d34d3),LL(0x76b65f73,0xd36a94bd),LL(0x3b4c1384,0x10e67e38),L_(0x00000044), LL(0x91c17d4a,0x850a6308),LL(0x5b75be0c,0xa8eb3e41),LL(0xc4b1a960,0x5176e7e2),LL(0xef74a564,0xf493e8d5),LL(0xd77a5683,0x2824a377),LL(0xdc2b6e93,0x3d90456b),LL(0xf50c072e,0xfe49afc1),LL(0x3768ae59,0x7326363a),L_(0x000000f2), + LL(0xc160ead5,0xf73cb6f0),LL(0xe2cd2bce,0x53fb7eb3),LL(0x9bb7bc05,0xb6b62b31),LL(0x8a471706,0xdaf298a3),LL(0x846c6b42,0x9d404cf9),LL(0x085e05e5,0x8fac2e73),LL(0xc53893b4,0xa8ff0af6),LL(0xe3d8eea7,0x0310bde9),L_(0x00000017), LL(0xe934470a,0xc1af94ed),LL(0x89b3f0c4,0x4fba6b38),LL(0x9af80496,0x3e423ee4),LL(0xd89fd53a,0x793fadf0),LL(0x353cc302,0xcd463b61),LL(0x51fbb36d,0x782e102e),LL(0xd6d1c6c3,0xbbc63732),LL(0xc97f604b,0xb60ab59d),L_(0x0000019a), +}, +/* digit=69 base_pwr=2^345 */ +{ + LL(0x5fdb3148,0x0a1bc3a0),LL(0xbda07174,0xc265e95e),LL(0x43b1d3c4,0xedc546d6),LL(0x611b9709,0x807b04c2),LL(0xe15784fc,0x473ceec4),LL(0x58afe5bd,0xc97fb33e),LL(0x7b94dc53,0x39d6d532),LL(0x914fc6dc,0xfc2ee62a),L_(0x00000094), LL(0x67cbcf73,0xdf80627a),LL(0x67d3e029,0x91ccf95a),LL(0xfd91b52d,0xa53ec7ae),LL(0xa855273c,0x1131573a),LL(0x07213a95,0x49db550c),LL(0xfa5fb98a,0x470643af),LL(0x0fa67f5b,0x6bf0628a),LL(0xf906186e,0x01bb6d2c),L_(0x000001b1), + LL(0x7f1be067,0x7f152d2c),LL(0x8659f773,0x242f8dec),LL(0x8df47d63,0xcbac7d95),LL(0x91c63b0a,0xf620881b),LL(0x8c2c6ad3,0xda54c002),LL(0x701999ad,0x3012f693),LL(0x99516f6a,0x3c14f4c4),LL(0x68d44cb7,0xc5e6fda0),L_(0x000001c7), LL(0x1d209ec5,0x76dcc740),LL(0x51dc58e8,0x93231067),LL(0x4e14fe98,0x5a4f75f2),LL(0x88f5086e,0x4ccd6aac),LL(0xf294dbdb,0xedf86543),LL(0x30e3be99,0x767f48ab),LL(0x22dcd0b3,0xd87667c6),LL(0xd6615681,0x36829d7f),L_(0x0000011a), + LL(0x0d5b110b,0x1e6c13e6),LL(0xa0842e9d,0x58585159),LL(0x443356f4,0x265b46fe),LL(0x3b25c086,0x87729185),LL(0x0ebbff20,0xc71c4d1d),LL(0x79c2136e,0x3ebeca5e),LL(0x3a96ed20,0x32a0af3b),LL(0x012e330a,0xb9939f93),L_(0x0000016d), LL(0x7d17ed67,0xde583a56),LL(0xe5703cea,0x316840b4),LL(0x03d396ce,0x0cfc4143),LL(0x6970ea48,0xb1974ced),LL(0xa63761b9,0x3de4383d),LL(0xe400788f,0xd07d6726),LL(0xa545993a,0xa33056bd),LL(0xd3fe822e,0x9a96c385),L_(0x000001b5), + LL(0x2f073480,0xf78fb060),LL(0xb2604448,0xc2ac8aeb),LL(0x277ca03f,0x27864b78),LL(0x2c6f473d,0xeb1664eb),LL(0x7cb793a9,0x58eee9a3),LL(0x9c2ce2b3,0x94f18cbc),LL(0xc87a3dc1,0x2df6078b),LL(0x0e93cd11,0xc8181622),L_(0x00000009), LL(0x3fd1ed5d,0xbe10752f),LL(0x0c38fb72,0xd1e47666),LL(0x9b093c35,0x73ab1d6d),LL(0xd898dff7,0x00e4cf26),LL(0xc4b445df,0x422c1136),LL(0x9ee0d1ce,0xb6e821b5),LL(0xe82511ed,0x1ade6252),LL(0x81c804e4,0xada5e2f4),L_(0x0000012d), + LL(0x47ebbcfa,0x75ed251a),LL(0xc45b660c,0xef50c145),LL(0x733af648,0x00ba4ec8),LL(0x5119e848,0xb32a226b),LL(0x3e048abf,0x1f648014),LL(0x7a1eccf1,0xfab712cd),LL(0x2badbf53,0x1ebfb8b6),LL(0xca11e7be,0x07ff38dd),L_(0x000001b2), LL(0x789ae7e1,0x42374e6e),LL(0xa641227e,0xb2f0016d),LL(0xa82cadf5,0x8ca0d43e),LL(0x552e57fb,0x5a71e67c),LL(0xa8bcdb55,0xc3a4e9f5),LL(0x6e0e2bb2,0x124b38d4),LL(0xc1f40c89,0x60b28fe4),LL(0xabb2620f,0x69bcc1f8),L_(0x00000140), + LL(0xfaecca00,0x028ae659),LL(0x3a3c5350,0x656b887c),LL(0x5b3bb83d,0x0c6fbb7d),LL(0xc2d4fb00,0x8f5877ba),LL(0xea3d0289,0x01e01a3d),LL(0x943983b2,0xdcb878b4),LL(0x382b3bfc,0x7e566dbe),LL(0xe238f00d,0x8ef9130d),L_(0x000000a9), LL(0xcf1a3019,0x6043d5c3),LL(0x3f970442,0xd72b4978),LL(0x47e77b41,0x51765a88),LL(0xd9f49bc6,0xc2232c0f),LL(0xf1592cf2,0xba6cd5b1),LL(0x7bdf89de,0x33024471),LL(0xa56dd8d4,0x4e3c8e30),LL(0x6718468d,0x5ba5d7f6),L_(0x00000062), + LL(0x1c158b81,0x3cf9ac17),LL(0xe81b84be,0xb9565ca7),LL(0xcf197d97,0x30adec74),LL(0x0f74f924,0xbbec4f93),LL(0xc6810de9,0xdd1aa6b4),LL(0x68e13e67,0x31b1e868),LL(0x3b5ec800,0xb287ff46),LL(0x970d17f9,0x56052d48),L_(0x00000171), LL(0xfc348975,0x3990d639),LL(0x2c996afc,0xbf8e72bb),LL(0xc43038ef,0xf3f9e817),LL(0x673d9367,0xb8d625be),LL(0x33a4c44f,0xb72553a3),LL(0x6135473a,0x7365b95f),LL(0x91b434c7,0x70995374),LL(0x2e7887e0,0x42eef182),L_(0x0000009c), + LL(0xf8d7989a,0x1b1a9086),LL(0xb30f16f9,0x2b4d5672),LL(0x919c3dae,0x64f72009),LL(0x3b6ec0a9,0x64ce5600),LL(0xc12b7f4f,0xe0f0d4fb),LL(0x144476bf,0x3eb40f82),LL(0x0332a8a4,0x80cbb448),LL(0xf3755660,0x689dbf3f),L_(0x000001e5), LL(0xe518caf3,0xa53d6591),LL(0xe170d9ab,0x4f1b1b50),LL(0x3e56ed3e,0x7c4f9eca),LL(0x8dfe4cc6,0xfa0dd028),LL(0xa59c7726,0x0a01f234),LL(0x6c8fa066,0x704007db),LL(0x8366767a,0x4570b32e),LL(0x94810fb8,0xbae24969),L_(0x000000bc), + LL(0xce3ac7a6,0x5ab23199),LL(0x54772e62,0xd7eb0723),LL(0x8caedc24,0x76dd866b),LL(0xa41bb763,0xba5b9e92),LL(0xe6d92de4,0x28e72bc3),LL(0xce269dcb,0x2b7dc535),LL(0x7e64fab4,0xe7cccbfd),LL(0x02a03896,0xcec461c3),L_(0x0000014b), LL(0x3dc101fd,0x079c3e36),LL(0x2332d512,0xf51726ad),LL(0xdbc6dd2c,0x81e5e3b6),LL(0xbd26d73b,0xb2e70917),LL(0x24b0b54b,0x23cdf2f9),LL(0x0353dd40,0x5e2acffb),LL(0x2e871e61,0x07f5e7c3),LL(0x5b299ee3,0x191257c5),L_(0x0000004f), + LL(0x3ec2ad9c,0x115859c9),LL(0x1f8e247e,0x17b4d84b),LL(0x2585ef39,0x657d1198),LL(0xa3f20465,0x3619c497),LL(0x6901431a,0xe9a26e53),LL(0xa5276fd0,0x496e3f3c),LL(0x1f276a6e,0x348de17d),LL(0xf8c3af85,0xc8630fca),L_(0x000000cc), LL(0xb1e10393,0x7b5ce8d7),LL(0x6deb4f48,0xc7dc292e),LL(0xc43c502b,0x6b03f2ee),LL(0xb1a1182c,0x39273259),LL(0x7622b369,0x4498e583),LL(0xfe7d43c5,0xa50df54f),LL(0x61447fad,0xc4339095),LL(0x3a9a1f2a,0xda206eb7),L_(0x0000005a), + LL(0x4af9a66e,0x310c7a00),LL(0x8db46828,0x9223ef88),LL(0x2cf042cc,0x6fa5a04f),LL(0x7c559200,0x6daaab2b),LL(0x91afc691,0x98a36fb2),LL(0x00b03676,0x188b128b),LL(0x51264914,0xb9803ddb),LL(0x75459c1a,0xf41eea62),L_(0x0000019d), LL(0xe3aaa000,0x7a690ecd),LL(0xceb2c1c6,0x8880b402),LL(0x45703d2c,0xc0613e4c),LL(0x8b8c9eb5,0xc5db208f),LL(0x4c846b8c,0xa9987715),LL(0x6bd91493,0x42b48717),LL(0x9a26dd34,0x3e788715),LL(0xd9b2c58c,0xb5cd631e),L_(0x0000017d), + LL(0x361ab1b5,0xccf7e9f1),LL(0x24eb3b91,0x5a00b663),LL(0x3a047d69,0x70c649d7),LL(0xff025b44,0xbfc42749),LL(0xf2772669,0xfa1fa0b0),LL(0x1d856fe2,0x59b50aeb),LL(0xdd9c5baa,0x07db1f1d),LL(0x63853e29,0x1535fdcc),L_(0x00000054), LL(0x924ac747,0xaa197f48),LL(0x269fd103,0x40db5d37),LL(0x4cdd9698,0x1838b760),LL(0x491094a5,0xfe6931b4),LL(0x6e498775,0xf5d608e3),LL(0x0107968f,0x1ebcbfad),LL(0x4aa5111c,0x9da743a4),LL(0xcba06022,0x90a35fc8),L_(0x00000151), + LL(0x47445d4b,0x06b03d8a),LL(0x4a0fee6b,0x3277857c),LL(0x1fa95d1d,0x87c700bf),LL(0x48c9ebf0,0x2a50e2b3),LL(0x43aa2c31,0x9a26e1ce),LL(0xfdbdd0e9,0x52ca3b19),LL(0xfcedcc05,0x2999c847),LL(0xbff002c4,0x267e4e00),L_(0x00000125), LL(0x64d4906f,0x556e52bc),LL(0x42f805ae,0x90f2f1b6),LL(0xda19f3db,0xd869ad98),LL(0x21709755,0xf5bb8103),LL(0x573f3fb7,0x593f7e34),LL(0x2bac9d40,0x02d84c5f),LL(0xa4e8ad8e,0xea53406e),LL(0x80797c51,0xde43a1d4),L_(0x0000007e), + LL(0x02db6ff9,0x9a077100),LL(0x442b39ab,0xa4a99831),LL(0x658c33f0,0xd154641a),LL(0x6b185008,0xbb725f43),LL(0x1c42ee9d,0xc802ceb4),LL(0xd4f04052,0xde1c27e0),LL(0xaa13dd25,0x5e0fdafb),LL(0x59845ec9,0x89bca83c),L_(0x00000039), LL(0xd7b6958f,0xc632359e),LL(0x00693302,0xd897df30),LL(0x3a9976fe,0x07f17ce0),LL(0xb3320f5d,0x0634d694),LL(0x9281633f,0xf481d736),LL(0xa1f17a34,0xc568493c),LL(0x7973f20c,0xc81657ab),LL(0xe331f1c3,0xf1485d0e),L_(0x0000001f), + LL(0x02479c06,0xf0c74cf8),LL(0x82afef4d,0x3fe27e63),LL(0xc6ed0fe5,0x2473be19),LL(0x88c3bb5a,0xf0f01c24),LL(0x8b895708,0xff99c560),LL(0xda940b97,0x19fe223c),LL(0x3a3e9972,0xfce2e466),LL(0x48aaf92b,0x2976902b),L_(0x00000085), LL(0x1f0040f3,0xc569b506),LL(0x6e592ad0,0xa59e0b36),LL(0xe1a1ad5f,0x506ba908),LL(0xdd28fbcb,0x35a90e9a),LL(0x5dfd21f8,0xae82baef),LL(0x5c23fd74,0x03b13133),LL(0xe27cfbd6,0xa41bf476),LL(0x7af7e895,0xc5b57a28),L_(0x0000005f), + LL(0xdf92a189,0x7fe35612),LL(0x9f97d902,0xa5105c7f),LL(0xba021326,0x9c521853),LL(0xd2cadb21,0x9b3ed45d),LL(0xb3ab9d25,0xddadc1eb),LL(0x7b4a857f,0xaddab060),LL(0xbbc92a34,0x7dff916f),LL(0xee7e6c52,0x9816b721),L_(0x000000f6), LL(0xa3ba1dab,0x7b0e564a),LL(0x27f6c008,0xb9c963dd),LL(0x3c030970,0x76828eca),LL(0x3cd7a457,0x0834d3c2),LL(0x417b833d,0xf44c50d5),LL(0xf0d8be25,0x153d4a6b),LL(0xa49590ea,0x47e9c71d),LL(0xf3a30dc2,0xeb2b1d43),L_(0x0000018d), +}, +/* digit=70 base_pwr=2^350 */ +{ + LL(0x995d0979,0xe0c6b8f7),LL(0xbaad255c,0xcdfa7a67),LL(0x2a84102f,0x2ed9a546),LL(0x5fff322b,0x238b4cca),LL(0x316895f0,0x27a1fc8f),LL(0x4369e1fc,0x399300db),LL(0x708ed717,0x5f16f718),LL(0x931503fe,0x22cd3aa4),L_(0x000000b1), LL(0xa040da9e,0xf9967269),LL(0x8694422a,0x5871908b),LL(0x8350fa73,0x61c801c8),LL(0x80cd1b5d,0x26cc63f6),LL(0xaf0b4158,0x3474f5f7),LL(0x1e938b36,0x7800e540),LL(0x2f20f7f2,0x56230526),LL(0xd27bb44c,0xe18d65e6),L_(0x0000003f), + LL(0x474f3cfa,0x7075c330),LL(0xb3ee576a,0xb446bdd4),LL(0x1b7efe90,0xa143371a),LL(0x826a3c98,0x45fcf113),LL(0xa412cdba,0x4b5601ca),LL(0x614348a4,0x06ebe3f7),LL(0x9f111b62,0xd750e443),LL(0x1f6bb4a9,0xa9ce6845),L_(0x000000fc), LL(0x63c0d59b,0xcb29ae28),LL(0xe216b29b,0x3a9a3c78),LL(0x2465c5b5,0x9567856c),LL(0x36d155fd,0xdfbfe047),LL(0xa1ce6b07,0xc3a4fa43),LL(0xf19b8361,0x0e9f03c0),LL(0xb21f548d,0x9de803f9),LL(0x85460ccb,0x5a9172f8),L_(0x000000be), + LL(0x28dc3d9e,0xb1947d50),LL(0x707f8201,0xa08b9636),LL(0x1277e4a9,0x253ea9d3),LL(0x38720146,0x7e71d50f),LL(0x62240fd9,0x79e96c95),LL(0x99a338fd,0x14656a4b),LL(0x3386ac0d,0xaed2f11c),LL(0x56e7d58c,0xb1645e8f),L_(0x00000026), LL(0xd77da8a6,0xb38b4a7f),LL(0xdb95d39e,0xb6a3eb09),LL(0xc15d78b6,0x97f12510),LL(0xc3aba47d,0x56c5ceaa),LL(0x1799be9b,0x71c67e7c),LL(0x8203dc96,0xc27e5165),LL(0xaf4cd822,0x0bd2ed9a),LL(0xb4f47f2a,0xcb8f3ca9),L_(0x00000141), + LL(0xd26b287c,0x978130d6),LL(0xd433a71c,0x7aadd881),LL(0xb4ad7bb1,0x52e4f45f),LL(0xb1940d6b,0x722e2bd7),LL(0x72d44569,0xf91dc84e),LL(0x6d3fde70,0xd4254643),LL(0xbd1bc41e,0xb847e41a),LL(0x544a7be2,0xace3e10f),L_(0x0000017d), LL(0xe2e82545,0x3d8e5d2b),LL(0x1c9863cf,0x5f30317d),LL(0x98fb8ec2,0x81f15ce0),LL(0x8947db85,0xf3e6c315),LL(0x055d8793,0x43a7feb5),LL(0x8417f508,0xac153d3d),LL(0x248bbc38,0x664329e7),LL(0xfcfbcb03,0xb0314d2f),L_(0x00000064), + LL(0x0ffb3aeb,0xf0f73b2b),LL(0xb5b42e5a,0xfb4aa83f),LL(0x6fe3f768,0x166957fb),LL(0x1777755a,0x9be2e29b),LL(0xb21920be,0xd39b15af),LL(0x9d78ee52,0x5cfe70fd),LL(0x4ed3196b,0x55c6cafd),LL(0x34efaf79,0xaab07209),L_(0x0000011b), LL(0x9f0e1288,0xc5b10817),LL(0xef0c0487,0x2d0a8114),LL(0xd48fc96d,0x7cfa2d81),LL(0x641e3931,0x6424b3ef),LL(0xe6f897a1,0x3828d18c),LL(0xf36345b8,0x162c7445),LL(0xa9afd810,0x86dd67b2),LL(0x34f52c33,0xc05d931e),L_(0x000000c0), + LL(0xbf4033d7,0x282d7d2f),LL(0x4816fc56,0xbba4acf6),LL(0x0d36fbc1,0xed8c95f6),LL(0xc3d935ee,0xf5c04db6),LL(0x0f8e468f,0xf1e487eb),LL(0xe95fc738,0x07ce9d5f),LL(0xd25bbce8,0x3ea06aeb),LL(0xd6e4d92f,0xe1653616),L_(0x000000e3), LL(0xd1ebfa2e,0x8c6285b5),LL(0xff9b06cd,0x7f8da651),LL(0x19efacac,0xfb702fe7),LL(0xf38839db,0xfd2c9b61),LL(0xb8c0cb9c,0x674adc03),LL(0xd42ebdf8,0x326b91f3),LL(0x8993821a,0x0fbb4e81),LL(0xbc7b32f8,0xc1690f2a),L_(0x0000011d), + LL(0x2fd676b6,0x892e4204),LL(0x41e60d2e,0xdf88f03d),LL(0x05344e65,0x322deae7),LL(0xabc0c748,0xf3bee9da),LL(0x7dbffa8f,0xe2cbe049),LL(0xf2922597,0xb0e73ee1),LL(0xa1ab5a4f,0xd18b70e5),LL(0xf7b0232d,0x6b85e543),L_(0x000000f3), LL(0x7a93eb9f,0x998f9412),LL(0x6520852d,0x9fffa5de),LL(0xa83ca014,0x9492e738),LL(0x29961bc9,0x38356eab),LL(0xbaa587e6,0x9aad6aa8),LL(0x5bde3fd3,0x15f7c437),LL(0xc663a8aa,0x2b85ba68),LL(0x922641a9,0xb030aa17),L_(0x00000175), + LL(0x90b91e88,0x9329a6f3),LL(0x5e0fd985,0x332091e8),LL(0xc7233994,0x069d0a1f),LL(0x7172741e,0x953488c0),LL(0x1870fafc,0x3b040fb9),LL(0xbb2fd807,0x9e841e1b),LL(0x16872728,0x48f58216),LL(0xa72dc0f5,0x590057bf),L_(0x0000010e), LL(0x4ec2f404,0xc898f3ae),LL(0x893dfe9f,0xd6d16346),LL(0xcfc2a2d6,0xdcfcc356),LL(0x9780e14a,0x52a0f58f),LL(0xe40c34a9,0x1f74017f),LL(0x9637bf1f,0xe85cc7e4),LL(0x7db8273a,0xf7240054),LL(0xd4e119d7,0x2baa4eaf),L_(0x0000008d), + LL(0x8bec2bfe,0xe147d2d0),LL(0x0f381def,0x63d396f3),LL(0xd719925a,0xa70c4ee8),LL(0x48335ffe,0x70cbdf17),LL(0xba86e989,0x70acf1a0),LL(0xa8e07900,0x3fdcbfe3),LL(0xf724a710,0xd0dd93f1),LL(0xcce8d0db,0x6cd982dc),L_(0x000001e7), LL(0x312db409,0xe842388e),LL(0x51c17662,0x5cfc4c86),LL(0x2819cf9f,0x3d2f2e78),LL(0xcb5f9278,0xc84b5c07),LL(0x6807319b,0x91966c94),LL(0xd459389d,0xf68df64d),LL(0x22ed9565,0x30d2c264),LL(0x2d12cd4a,0x1496895c),L_(0x00000055), + LL(0xa8198a4a,0x7fd3cc79),LL(0xcf8b16bd,0x973e67ac),LL(0x27c827e8,0xc9207496),LL(0x6d1709a1,0x9157f587),LL(0xc23a1cd6,0xbedd9d1c),LL(0x8b1088ef,0xb005d24b),LL(0xc08bed38,0x6477a806),LL(0x0681e2ab,0x213943da),L_(0x000000ba), LL(0x2de2bdb8,0xe996e18b),LL(0xf9e30d5c,0x2c137e56),LL(0x5c521393,0x6041c771),LL(0x8cdc666b,0x4ee56b76),LL(0x2695a1d6,0x518a5638),LL(0x41e2f039,0xdb59944d),LL(0xaf0a4010,0x5b0e9f3f),LL(0x829ed1fe,0x2cb7de1f),L_(0x00000068), + LL(0x3b382180,0x93922bbc),LL(0x1503eb8a,0xd83a3aec),LL(0x4f056b98,0x0f661abc),LL(0x1dc0c0de,0x20cd5ab5),LL(0x25a3a29e,0x9409a1df),LL(0x2ec3d724,0x2311b333),LL(0x9cff8f0d,0xd299c4b1),LL(0x30ec4ca0,0x7f770084),L_(0x000001d0), LL(0x674d075d,0x79f5e3c1),LL(0x3e4d547c,0xaceff143),LL(0xa985c66e,0xceb1d3f0),LL(0xc8735e90,0x46311b8a),LL(0xecdbba45,0xd47b3dd0),LL(0x8237affa,0xc21e6384),LL(0x6b4f82bd,0x851d53b8),LL(0xff59ca4e,0xf5e1b60e),L_(0x0000000b), + LL(0x29a5355a,0xa2f29baf),LL(0x3e11c687,0x653058a2),LL(0xab5abb63,0x533110b2),LL(0xead1d1b9,0x254324e4),LL(0x65d1b7b6,0x9ad5a861),LL(0xd1007405,0x0712ab62),LL(0x6f88f2a9,0x78e9d501),LL(0xefd62c6b,0x1145cafe),L_(0x000001fc), LL(0x62d2d42c,0x8d15289e),LL(0x5d68b919,0x4baf7b53),LL(0x4a9af773,0xbfd8566e),LL(0x2c278158,0x0f0f5b40),LL(0xa603f631,0x366d639e),LL(0xed79331a,0x457655be),LL(0x5b5f4bc7,0x744b4617),LL(0x6ced0122,0xa2e7748f),L_(0x000001c1), + LL(0x0a6157b9,0x62225ddd),LL(0x63f4338a,0x3ff075f1),LL(0x19a505d3,0xa170cf14),LL(0xdfa6bdb1,0x35bb45bd),LL(0x3014f03b,0x01eadcfb),LL(0xdf884f38,0xed9ce5a3),LL(0xeb67d796,0xbe4fe92b),LL(0x01a5f419,0x57e98411),L_(0x00000129), LL(0x7ddc4629,0x8bd2744c),LL(0x40075673,0xa1d1f6a9),LL(0x6ac9b5bb,0xbff10759),LL(0x7709ddbb,0xd617233f),LL(0x04b71ff3,0x91a5fcd2),LL(0x45458de2,0x699b54de),LL(0x5147375c,0x4e0307cc),LL(0x2fe5917c,0xf27f33bc),L_(0x0000003f), + LL(0x4252a836,0xfbc6e97a),LL(0xbffdbb20,0xee422c57),LL(0x5769cb47,0x501f912b),LL(0x3924d571,0xae0c25a9),LL(0x239c3442,0x7ed84b21),LL(0xd601103f,0x74478136),LL(0xebe1703d,0xc5087e65),LL(0x67b021e4,0xf61c2d56),L_(0x000001f8), LL(0x8ce094d8,0xaa167fe1),LL(0x70c73af6,0x79b93fdc),LL(0x0e41e095,0x3eab3290),LL(0x4d79fd87,0xb7162ca3),LL(0x66141590,0x583d1391),LL(0xe0bd38e8,0xa393b806),LL(0x169f55ab,0x4bdc04a4),LL(0xd1df6260,0x445d5212),L_(0x000001e5), + LL(0x6ef19cc6,0x8fb90054),LL(0x349f0b5e,0x7a588763),LL(0xd381418c,0xe35ce1af),LL(0xec0fd49b,0xecb4e203),LL(0x7786513b,0xa276ad19),LL(0xfe701187,0x99f4ee10),LL(0x47d026fa,0x58423dc0),LL(0xa22bbaf1,0xbd8a6065),L_(0x000000fc), LL(0x9aa1efcf,0x5af392c9),LL(0x8ba7989e,0x7c5174d3),LL(0xd1616f2e,0xd20d0632),LL(0xa5636d7c,0xd761cf0d),LL(0xce718261,0x701e7d69),LL(0x08d4d0bc,0x66e92aa8),LL(0x819aee8a,0x0d6fcd90),LL(0xc96d5138,0x2da0618d),L_(0x000000a1), + LL(0x81b20efd,0x117c40b0),LL(0xec008c40,0x0a6d9c2a),LL(0x693270e3,0x6114d3e0),LL(0x4266a5ea,0x44a6af67),LL(0x81ebf621,0xee3917e3),LL(0x7fca3d45,0xc35ff5d6),LL(0xa3526048,0x0f6e79db),LL(0x7e7bfed4,0x0da1a81a),L_(0x0000016e), LL(0x0c06eb8d,0x3343c5a2),LL(0x3418cfe3,0x3e67d0a5),LL(0x15eb001a,0x7e48959e),LL(0x0ead5e7c,0x0962e6ac),LL(0x32e4162f,0x3e28513c),LL(0xfb8117bb,0x317568fa),LL(0x3a2e3034,0x0c912ceb),LL(0x55938174,0x24701102),L_(0x0000018d), +}, +/* digit=71 base_pwr=2^355 */ +{ + LL(0xfec70f6d,0x92ef06ac),LL(0xc38aac0c,0x790f3344),LL(0x53c30edf,0x1b40fed7),LL(0xfc800650,0xf6da803d),LL(0xfdf722f2,0x284a42e2),LL(0x5cecc340,0xa0f15400),LL(0xb36ac652,0xef82f0df),LL(0xd1506b21,0x5fdee7be),L_(0x00000096), LL(0x84d76b78,0xa1c3d7b8),LL(0xc6050edc,0x307e9a89),LL(0x1519baa6,0x8c7e0ccc),LL(0x3495eff8,0x22e91666),LL(0xea17475b,0x69639f1c),LL(0x8c53c39e,0x1f0e827f),LL(0xede8121b,0xeb066355),LL(0x91249281,0x0c0c2d5b),L_(0x000000b2), + LL(0x3accccf3,0x16751720),LL(0xa2ac7465,0x1bf938ea),LL(0x83a983dd,0xb73d73d6),LL(0x598a6f1d,0x630b4f7f),LL(0x6235f9ed,0xdb784cb5),LL(0x52bdf332,0xb330540f),LL(0x221e5e83,0xc25843b0),LL(0x09499b4e,0x6e45244a),L_(0x000000b4), LL(0x010fb3ca,0x8b6e52dc),LL(0x60a24896,0x22c046dc),LL(0xae1e187a,0x482695be),LL(0x006acf49,0x35934bd3),LL(0xb960f105,0x1e1d0143),LL(0x4cfddf01,0x5de371d8),LL(0x56c439e8,0x64828414),LL(0x582ff3b5,0x43605585),L_(0x00000161), + LL(0x3578ec4a,0x83ac7330),LL(0x74417a7c,0xb971e045),LL(0xbee09246,0xc391f3f4),LL(0xb2cfe806,0xf8fb9526),LL(0xe574f401,0x2d57d573),LL(0x9ef64156,0xf2047705),LL(0x2e13ebeb,0xe2e05adf),LL(0x97b783be,0x2b746c25),L_(0x00000086), LL(0xac801f25,0xbb184270),LL(0x1d0026d9,0x772cb73c),LL(0xc5ae7636,0xf37dccd7),LL(0x5106f975,0x276aad31),LL(0x2b41e6c0,0x0c355836),LL(0x7d420351,0x169e0a27),LL(0x25cdd7da,0xe67ef6ec),LL(0xdd1ddbf0,0x988f96fe),L_(0x00000103), + LL(0xbce94ac6,0x89803b61),LL(0xe8afe442,0x2e798f31),LL(0x9a42f37b,0x9475e43d),LL(0x77aef7a7,0xa8f685f3),LL(0x203a6947,0x969c3b8c),LL(0xdf0dc1b4,0x9c542cdb),LL(0xc76bc8fb,0xff501682),LL(0x2a768660,0xe2ff6697),L_(0x000001f7), LL(0xc8f9daec,0xc4a46652),LL(0x89b1c325,0xb09499ac),LL(0x721c0cae,0x5e63ccd5),LL(0xda46e344,0xaca0b998),LL(0x32db691c,0x793a1fc7),LL(0x049e845a,0xd927f614),LL(0x7aea310a,0x80024bf0),LL(0x5359be8b,0x60137724),L_(0x000000be), + LL(0x57afd605,0xb3d701e7),LL(0x608eeeaf,0xa1e2923b),LL(0x3a52d1fe,0xb5ba2517),LL(0xf6c570be,0x6f830092),LL(0xa2b946db,0x63e9fd66),LL(0x2b96de58,0xd1292d2e),LL(0x89c80a56,0xf418bcf0),LL(0x02901646,0x626f5025),L_(0x000000f4), LL(0x9f28725f,0xb36ad666),LL(0x73de9bb2,0x25e09cb1),LL(0xf102152a,0x3474fc24),LL(0xb0389a9f,0x9e0b3083),LL(0x245ecf47,0xefc86ff8),LL(0xedc1e824,0xd024fd72),LL(0x022e7528,0x0e37477f),LL(0xa7fdb14f,0x2d504191),L_(0x00000176), + LL(0x130db159,0x31d0f49f),LL(0xbd9c79a2,0xf1d499d6),LL(0x7ea9b7cf,0x367757ac),LL(0x05a4d545,0x0a42a453),LL(0x1f2c8548,0xd461a5b5),LL(0xb8ed29ef,0xe691c9e6),LL(0xa549541f,0x6490ea1c),LL(0x09c0153e,0x93d0058f),L_(0x000000d9), LL(0x7f525f59,0xc140a1f6),LL(0xa98aaedb,0xf5bef166),LL(0x750be5c2,0xd457a559),LL(0x8cba58b2,0xc5d96cfb),LL(0x514d93d0,0x70bfa2f9),LL(0xb86d7234,0xf6b79058),LL(0xa8a78584,0x18d58f11),LL(0x2b2d0ad4,0x341dbfe3),L_(0x0000007f), + LL(0x147202dd,0x3c8a33ae),LL(0xd39c4b9d,0x1d455649),LL(0x3f56141d,0xc0ac51ab),LL(0xa5a57669,0xfc529732),LL(0x1ee307ea,0x1861bed9),LL(0x5f1e4bec,0xc7a796b8),LL(0x5b0d925f,0x06f456f4),LL(0x2257c76d,0x7d497d17),L_(0x000000ab), LL(0x4360118e,0x218cb0cb),LL(0xaa1c1dca,0xf6db0e7a),LL(0x2be1c968,0x5a3744ed),LL(0x46bb2acc,0xaf262fce),LL(0x7e16d340,0xaec37ad7),LL(0xe6df41bd,0x1060715f),LL(0xee38cd22,0x87b94898),LL(0x34109b20,0x0eb71ca2),L_(0x000001fc), + LL(0xb416f6d1,0xc2c062af),LL(0x3dacb0a7,0x1630676c),LL(0xd74ee6b3,0x95fc297a),LL(0x8f736e49,0x48a7a2d1),LL(0xd64edc25,0x6b5d5f53),LL(0x83303159,0x0e945b2c),LL(0xdfaa52c4,0xb0587c06),LL(0x462a8f05,0xa859437c),L_(0x00000141), LL(0x2fcd636b,0x0835ddc0),LL(0x86482b2b,0xd2333470),LL(0xc7f1c7b2,0xb32bf92b),LL(0xd5f30c92,0x49153950),LL(0x9ce136c0,0x29288cec),LL(0x4ac8254d,0x34eb3849),LL(0x0b3117ac,0x14ba2a1b),LL(0x3a85376a,0x1b0e1c47),L_(0x0000009e), + LL(0xb3ec3510,0xc014692a),LL(0x988a8cb2,0x70309a41),LL(0xb83a1155,0x92367194),LL(0x22c65f09,0x558d49bd),LL(0x17ac8e14,0xd539b194),LL(0x281a7ecc,0xa19213fa),LL(0xc69fe80b,0x1ef427d6),LL(0x744a4f4d,0x5f94b5db),L_(0x00000136), LL(0xf07169a2,0x4ff0070e),LL(0xe17c5e0d,0x42362609),LL(0xd97efa2b,0xab4374f3),LL(0x59d17f1f,0xc3027acb),LL(0x8cb9e348,0xc305c872),LL(0x320eb648,0x861bfbe3),LL(0xf68b129b,0x2a98fbd9),LL(0xc35aa741,0x8fa1ae4d),L_(0x000000c9), + LL(0xba2a6162,0xfef26d3a),LL(0xaf22bedc,0x4bac42f2),LL(0x32bd0514,0xaf54da8b),LL(0x474d59e6,0x846ca3be),LL(0xac190f17,0x7e7c79bf),LL(0xaea0f3e1,0x13543ecb),LL(0x0ff996ba,0xabe74acd),LL(0xe27a5f5a,0xcea6ecbd),L_(0x00000136), LL(0xa8ccc73f,0x1e746179),LL(0x62af882b,0xb19b717d),LL(0x4e0895be,0xb255045a),LL(0x8f194a8b,0x0b37366b),LL(0x089f1cd5,0x7b3da3e1),LL(0x68b1f2a5,0x91e4f674),LL(0x9c4602f6,0x698976ca),LL(0x3ed98ad9,0x99fe05e5),L_(0x000001fd), + LL(0xfe476990,0x532b6850),LL(0x345630b7,0x7ff00f29),LL(0xecbc85e6,0x9c3d2e4c),LL(0x5f322ba8,0x6869142c),LL(0x9e246c23,0xf5d1ef76),LL(0x0affd2a0,0x2e6d871e),LL(0xb40893c1,0x66c72704),LL(0x1665fed8,0x01d321d9),L_(0x00000042), LL(0x06507414,0x9d9489fd),LL(0xa1379411,0x7d5c53f0),LL(0x7e6d3de2,0x097595be),LL(0xc486fb28,0x85e5a09c),LL(0xa9fd8f9e,0x2996fa66),LL(0xd71e16d0,0x040664e3),LL(0x7c75e965,0x14b60428),LL(0x9d686380,0xf4fa3032),L_(0x00000123), + LL(0x86f79bf5,0xbf36d744),LL(0xbf45f97e,0x0fe48147),LL(0x6235b3ae,0xd40868d4),LL(0xa9b13d93,0x45fa9173),LL(0x34e9264c,0xb5705f4c),LL(0xd0d58c79,0xfd4b166f),LL(0xaf4ff870,0x7aaca2ed),LL(0xb68a488f,0xd9bb2955),L_(0x00000047), LL(0xf5f3e0b1,0x747dd972),LL(0xf6fc0c26,0x92d84bc8),LL(0x088102b5,0x201255f7),LL(0x84970893,0x7f6288c9),LL(0x6a679170,0x9309b54e),LL(0xd5327276,0x389f4da5),LL(0xc48b5de6,0x8fc1eb23),LL(0xac794b85,0xa3f64d1b),L_(0x00000132), + LL(0xb408df01,0x66056a5d),LL(0x5f945157,0xc30a058e),LL(0xf0071848,0xb615e360),LL(0xe6a8d838,0x132d49ae),LL(0x1bb7f3ca,0x092ee873),LL(0x398ab7d7,0xbf83bd98),LL(0xba362639,0x73208c1b),LL(0x49dd5ba3,0xf4382ccd),L_(0x000001a7), LL(0x0d07ce4b,0x724cd057),LL(0xf7baff54,0x31245cf8),LL(0xff518822,0xff5f1211),LL(0xea1813a7,0x89f90332),LL(0xfa74413f,0x9e68455b),LL(0x9e49a7a6,0x182fad31),LL(0x30e8a2ef,0x233ce0d5),LL(0xbd55ab52,0x38f1c599),L_(0x00000010), + LL(0x28e2d8df,0x708c9183),LL(0x1dbe8e5a,0x9b9fb00f),LL(0xa3695cb8,0xaeafe9c7),LL(0x9205b4ca,0xd6ec0b74),LL(0x756f204b,0xa9e0254c),LL(0xd51d1a73,0x152441cf),LL(0x0ca91564,0x370d2b8b),LL(0xe3cdd9e9,0xab50f4de),L_(0x000001a5), LL(0x9dab1375,0x5322d78e),LL(0x78b8ab5d,0x0adbfc55),LL(0xa2b97f9c,0xd5d0ce27),LL(0x9cd573f2,0xe94e39b4),LL(0x06ee23d2,0x213bd15a),LL(0x708d61ea,0x561b9d34),LL(0x6271f59e,0x41fb576c),LL(0x9ae94507,0xd828d166),L_(0x0000009b), + LL(0x6669c984,0x46ae7251),LL(0x8c23b4d3,0x3738a807),LL(0xad75f8b4,0x3ef8e3cc),LL(0xc8e8bbf5,0x029e586a),LL(0xa7111c4e,0xa4d1beb9),LL(0x238f36b4,0xc4da1680),LL(0xbe34bad6,0x9409b124),LL(0x6ab824c2,0x48e9ec02),L_(0x00000062), LL(0x0666df3f,0xa0015f27),LL(0xd7f90fcc,0x3e7eb3a2),LL(0xcd91c4ca,0xa8bd0fd8),LL(0x56907857,0xc0b60059),LL(0x3559db95,0xc6ed4fe7),LL(0x0c8beff5,0x2ba51007),LL(0x5bc2116c,0x409f6b4e),LL(0xa6198a28,0x360e46d8),L_(0x00000076), + LL(0x231a6637,0xb888102e),LL(0xaecdfa06,0x430b0efa),LL(0xd888793c,0xb123b1e3),LL(0x8beed2db,0x8887df4f),LL(0x3ea5e72a,0x2425e985),LL(0xccc7d4aa,0xd98e93f3),LL(0xe9181719,0x07cba97f),LL(0x8ea6eef3,0xa9e6f6b0),L_(0x000000d8), LL(0x5719b171,0x5d9f40be),LL(0xe16ecd5e,0xfea96313),LL(0xfe1e359f,0xd9f1461e),LL(0xde9904f3,0x38f6d943),LL(0x881bb7a0,0x2c5787d5),LL(0xa74eed55,0x67fc2cd9),LL(0x7ccb483a,0x2c643f37),LL(0x7070b576,0x386889db),L_(0x000001ea), +}, +/* digit=72 base_pwr=2^360 */ +{ + LL(0xca188556,0xbd7c7672),LL(0x20ced736,0xa4667058),LL(0xb83d6897,0xe3f39ba4),LL(0x174ecbf7,0x34188faf),LL(0x203dc58b,0xdb5dba0b),LL(0x5206b453,0xf54df32d),LL(0x52fcf51e,0x08d08e3c),LL(0x2f551f34,0x72856373),L_(0x0000016a), LL(0x6b937ade,0x7f7f5dfd),LL(0x632c9b6a,0x3151876b),LL(0x3ee4a789,0x7a59040d),LL(0x41b009fd,0xdfc2d274),LL(0x68b427fe,0x1c0ceded),LL(0x07d57f92,0x220fc8f2),LL(0x83c79a42,0x7b0f6753),LL(0x410a2e83,0xc0765d6f),L_(0x00000124), + LL(0x01fb8b79,0x82addc43),LL(0xfdb0062d,0xb9cdf1d9),LL(0xb1cf25ad,0xb5a42255),LL(0xaaea42eb,0x1990669d),LL(0x1dffd105,0x88f20764),LL(0x613001d6,0xda7769bd),LL(0xa275aa11,0xe04ea507),LL(0xea612e43,0xf381e073),L_(0x0000005f), LL(0x21f18b4b,0x268a5e08),LL(0xa7554f72,0xad126436),LL(0x714fe1c3,0x5b5ba02f),LL(0xb7c7cde4,0xf2da3519),LL(0xfc576f09,0x4fb328e0),LL(0x185faef3,0xa0386e0f),LL(0x2adc73d1,0xc97a6bb3),LL(0xda21be9a,0x70df7733),L_(0x00000048), + LL(0x6a23d540,0x500040b2),LL(0xf87554cc,0xe6a09fa7),LL(0x548aea96,0x0cf27fbb),LL(0xfa1d8c06,0x1a618765),LL(0x1943cfee,0xe6a8c7ea),LL(0x20bf61bc,0x99730b0b),LL(0x744528dd,0x42eac170),LL(0xd049742c,0x85700423),L_(0x000001aa), LL(0x1e2bba63,0xe601cd80),LL(0xcbeefa62,0xc9e240a6),LL(0x8106469f,0xda8103af),LL(0xc7109e54,0xdcc44907),LL(0xdb9a3ec3,0xe44b6df8),LL(0xe34b7788,0x0e67c93e),LL(0x4a58495d,0x63e8347b),LL(0x23b5096e,0x468b5372),L_(0x000000db), + LL(0xbf417e03,0xb60364c1),LL(0xdda37ca1,0xea847f52),LL(0x28527f5c,0x98d517fb),LL(0xa1e399f7,0xff102f07),LL(0xc452c79f,0x87dfab3c),LL(0xc5aa688a,0x490b0295),LL(0x0dbc6056,0x3dd17acd),LL(0xb4f6972c,0x2e16b5ac),L_(0x000000b7), LL(0x23fa3555,0x3a903ffa),LL(0x262814e3,0xc46f6e35),LL(0x1cec4214,0x7267bc6b),LL(0xca2e1dc8,0x20b7474b),LL(0xeb500457,0xf394811d),LL(0x4304c697,0xa5001f3d),LL(0x0f7a5e2b,0x4c9ea7fd),LL(0xead3d012,0x84c6a90d),L_(0x00000120), + LL(0xb7e47c23,0xd475bc3c),LL(0xff4599af,0x1acc6490),LL(0x39b1950f,0xee09f5dc),LL(0xd14540f1,0xe51c9564),LL(0xf1b75050,0xbebd088f),LL(0x17895647,0x240dba4c),LL(0x097400cf,0x559b95e8),LL(0x5d4b8420,0x3bc0185b),L_(0x000001a3), LL(0x85986a76,0xc2ffb653),LL(0xa478ed7f,0x18e264c5),LL(0x4841d184,0x17a5a278),LL(0x21d9e8a0,0x297fe2fe),LL(0x1bf52154,0x072d6d91),LL(0x88327dad,0xae77c8ea),LL(0x02d1fcbe,0x13786b6a),LL(0x25554500,0xdf7f6e68),L_(0x000001a0), + LL(0x17de731f,0x7753cff6),LL(0x972b7a35,0x73655403),LL(0x28d73a10,0x0c70484d),LL(0x46d46c14,0x516a9dc8),LL(0x455b7ef1,0x0b552594),LL(0xb8161489,0xf418ade1),LL(0x64a91645,0x76a465f2),LL(0x7693e9a1,0x91415ed3),L_(0x0000013e), LL(0xe5e5c3bf,0xdc135bc4),LL(0xca946121,0x3d39b5c7),LL(0x55877498,0x318be468),LL(0x9fb5d801,0xe5cb6287),LL(0x3afd92b1,0xcd7f8034),LL(0x34ed24ae,0xa3835c84),LL(0x6aa7d954,0xd325764c),LL(0x41780668,0x7945ea02),L_(0x000001b9), + LL(0x34c0928c,0x37933b17),LL(0x74a56f2c,0x6bae2a29),LL(0xb1d26ac8,0x6bd1e8bd),LL(0x84c336cb,0xcdaa1b9c),LL(0xaca41014,0x7838c44f),LL(0xe2ce24b8,0x525239ca),LL(0x515f204f,0x8bcb0507),LL(0xdbd0e0a5,0x50183993),L_(0x000001e4), LL(0x631411bd,0x3e15aa53),LL(0x8672c87f,0xade47bf6),LL(0x93da50d5,0x148028e4),LL(0x0048f8cd,0x03c75612),LL(0xb5ecfaeb,0x7b7867aa),LL(0xafcde134,0xa0208953),LL(0xe2411e3b,0x24be9b23),LL(0x848d40b4,0x24d9da2e),L_(0x00000187), + LL(0x16583ec0,0x7b7ba7d9),LL(0x2bbb4768,0x2b3f0b4e),LL(0xe0e4b3fb,0x9172caac),LL(0xe6fb63a6,0x22aab4b0),LL(0xa00520c8,0x7930e37a),LL(0x4dcbf41f,0x6bfa91da),LL(0xf521a694,0xb88bd604),LL(0xa707c1f0,0x95a5140f),L_(0x0000011a), LL(0xd8520b88,0x22333018),LL(0xa6bc2bb8,0xaa6a00bc),LL(0x011553af,0x5def3469),LL(0x0ed5fc0a,0xbcfec7c2),LL(0xdee0e8c5,0x2f464224),LL(0x8adb476e,0xd844542e),LL(0xd3c1bdb2,0xa709924f),LL(0xac98d161,0x472a5f2f),L_(0x0000001a), + LL(0x5054b047,0xa4a3faff),LL(0xd966b478,0x3d33573c),LL(0x1081c0ed,0x928c644f),LL(0x0d2ce409,0xb6d01835),LL(0x9e6a2193,0x52176b02),LL(0xa876fcb1,0xa48ba61b),LL(0x717040ec,0xa24784a2),LL(0x063597d5,0xbbb8ca8d),L_(0x000000e2), LL(0x24d496ff,0xb9c0170d),LL(0xf08c120b,0xee06f00f),LL(0x24a5f375,0x9e3d247d),LL(0x21f556fe,0x23ca02e0),LL(0x9baf2fa5,0xc33aa42b),LL(0x87dfcff1,0x165eef36),LL(0xf9dd5e4c,0x54afa097),LL(0xfe4014f0,0xe713f89f),L_(0x00000058), + LL(0x690b2052,0x836e06e2),LL(0x3509fc01,0x7b24d732),LL(0xc3a11a6f,0xf970953d),LL(0xb1e661c4,0x19774ec5),LL(0x1a995696,0xa05e5145),LL(0x777347df,0x1c3c2550),LL(0x77882ae3,0x5a7928a8),LL(0x77fbd07d,0x69195185),L_(0x000000a4), LL(0x41e2d40c,0x16368545),LL(0x29dc25b0,0xfa97fb70),LL(0xb7b7f0ff,0xdec5a377),LL(0x2c841e96,0x205a5df4),LL(0x01390c3a,0xe68d053a),LL(0xa1116c1b,0x83274721),LL(0x62015852,0xe92e4364),LL(0x6a3178e8,0xfafb3a16),L_(0x00000013), + LL(0xf2f16fe9,0x75049d40),LL(0x2503f2a9,0x67afd1db),LL(0xe86bcd13,0x1004d640),LL(0x5322f07a,0xf5a688a9),LL(0xb4accb02,0x5af14887),LL(0xe07764b0,0xd65fb0f3),LL(0x03224e24,0x5db913de),LL(0xb8433f0c,0x4e0f4a11),L_(0x00000109), LL(0xf3884513,0xe8ba41a9),LL(0xaded3528,0x8ea44980),LL(0x1a82f302,0x31375544),LL(0x0614f686,0x4f3dc64f),LL(0x8ad34274,0x7f906c7a),LL(0xc479a89a,0xe50987f1),LL(0x17709633,0xaa307609),LL(0x33922a61,0x8a16309d),L_(0x00000035), + LL(0x7b97be7b,0xd004b0c4),LL(0xd609efba,0x341b0b8f),LL(0x20475d65,0x373000c9),LL(0x00101859,0x5bcb5ae6),LL(0x518d7514,0xfb3c86f2),LL(0xf5d314f2,0x2c8aceec),LL(0x2307be1e,0x778a0fa6),LL(0x5d168daa,0xebfa491d),L_(0x0000017e), LL(0xa440b9a3,0xa70ba700),LL(0xa47a36dd,0xef6fc566),LL(0x17be0829,0x8e295843),LL(0x63809dc9,0xa317bc2f),LL(0x1a787c34,0xbce26a9e),LL(0x943ad796,0xc651a487),LL(0xe010f911,0x636a6efc),LL(0xcd31e255,0x66b98f88),L_(0x000001af), + LL(0xed5ce1f5,0x8d6c9e3b),LL(0x8f008e9a,0x2a13d48e),LL(0xa44f1754,0x32cde8f7),LL(0xaa90e24c,0x8660b8f9),LL(0x3052b86b,0xd73a03c1),LL(0x79244ef9,0xe4f8a628),LL(0xc2475432,0x35d52164),LL(0x9c11d0d4,0x8b588259),L_(0x000001c5), LL(0xb6ab40a4,0x3d6aa080),LL(0x9ca82551,0x92a4dc90),LL(0x8044f304,0x5989211e),LL(0x4878d275,0xc33afe23),LL(0x50bb5ea4,0x2b031b41),LL(0x889e9545,0x4a4f6a74),LL(0xdcce463e,0x6a9c23ca),LL(0x260ab0c8,0x066be87c),L_(0x0000007b), + LL(0x2372213b,0x84ab40ec),LL(0xdaad8de9,0xcb476943),LL(0x2dbf8cf6,0x9149cbb6),LL(0x72626b77,0x4935f2cb),LL(0xae5d765c,0x79eb7a36),LL(0x65f4be84,0x44c54fc0),LL(0x2049ba34,0xcbfa4bf0),LL(0x9c904608,0x5711da83),L_(0x000001c0), LL(0xa5d29aec,0x3010853b),LL(0xdf573f6e,0xcdc13fa1),LL(0x9737e298,0x24add1ef),LL(0xa7f64bf7,0x2431c698),LL(0xb2b280e2,0xbdff9a1e),LL(0x93c22a36,0xd70876a8),LL(0x0c7227cd,0xaf483376),LL(0x04873b2d,0x78b4c89e),L_(0x0000007b), + LL(0xecb1f89d,0x21b6a936),LL(0xf18c9695,0x805badb2),LL(0x45242b9a,0x2c7430dc),LL(0x7856a265,0x5aacc16c),LL(0x281a24a8,0xbe56330d),LL(0xd8f8608d,0x50d0b225),LL(0x62852160,0x4403a1c3),LL(0x7038362e,0x86d7f075),L_(0x0000010c), LL(0x7dc02e97,0x816727a7),LL(0xca536177,0xc5451dd1),LL(0x304d66d8,0xb89ef533),LL(0x360da6a3,0xe6b58c96),LL(0x3f234bec,0xc18619f4),LL(0xb4bfa580,0x891d516d),LL(0x40c1bed3,0x04f1453b),LL(0x7060a227,0x3adae1e8),L_(0x0000005b), + LL(0x30c9a655,0xcf48b04d),LL(0xb68ea01a,0x04344830),LL(0x174d6fa7,0x1ac58a53),LL(0x9044eeb3,0x0524d6f5),LL(0xf87d51a6,0xfb882d4d),LL(0xaa0ba344,0x1ed41d08),LL(0xa85fb93d,0x6086b6ae),LL(0x7fa57f48,0xe80f97f2),L_(0x0000006c), LL(0x596f6fa7,0x2aae9562),LL(0x04813ef9,0xabc183c4),LL(0xded30d2a,0x6a011be4),LL(0x20b7ae96,0x88e77be2),LL(0xbc3e6cfc,0x77d5e0ca),LL(0xac06a92b,0xd7f99c6d),LL(0xc76c3023,0xa80a6be4),LL(0x1d55150d,0xfd7af32d),L_(0x0000016f), +}, +/* digit=73 base_pwr=2^365 */ +{ + LL(0xea0d37f0,0xd1a1ebfa),LL(0xb07ea564,0xbce4cf04),LL(0x4677d784,0x74184f2b),LL(0x14a4f867,0x3b8741db),LL(0x00b95ce9,0x35b5960b),LL(0xfae4a317,0xd2c80a76),LL(0x23107ec3,0x0522c4d1),LL(0x8678a9d7,0x19e856cd),L_(0x00000060), LL(0x304b58dc,0x88c3d5f2),LL(0x45f5267d,0xf847248f),LL(0x28ef4e85,0xc0f3da26),LL(0x7945a7b9,0xea2c17e3),LL(0x8a2da387,0xe84de988),LL(0x211a8e98,0x8290c88f),LL(0x75574343,0x6b4ce366),LL(0xca4612f5,0x827b2040),L_(0x0000015d), + LL(0x6c7a73bd,0x1e3ef4b5),LL(0x104fc4ba,0x835871e0),LL(0x4759b57a,0x9b4ed462),LL(0xd3c95d4d,0x648a71ee),LL(0x829d8353,0xece81ad2),LL(0xb2a56bed,0x452c12f2),LL(0xb67ec3e1,0x35ab19b8),LL(0x3f8f88bf,0xeb6e6ccd),L_(0x000000f2), LL(0x76062e0d,0x4abff696),LL(0x6641cdd3,0x9b89962a),LL(0x0add12e1,0x9f42be1c),LL(0xa078191a,0x72d9da14),LL(0x7c488cf9,0x607f65fc),LL(0xa7e790e9,0x5cdadd7d),LL(0x83b3584d,0x381ca37f),LL(0x3c6df02d,0xfe16cdd4),L_(0x000000b8), + LL(0x3a1e6367,0xfd8b4117),LL(0xcdcf0fd6,0x0c717ef3),LL(0xb3300d01,0x07e608fd),LL(0x527d7c8e,0xe69c0a0d),LL(0xd39cd9ec,0x11bdaf48),LL(0xa5576772,0x5d520c7f),LL(0x92f3c61b,0x1fbf8426),LL(0x814bffe3,0xa920b055),L_(0x000001bd), LL(0x3dc94502,0x8193bb95),LL(0xb1a23052,0xd7bdda5a),LL(0x19650b25,0x4c67c4a2),LL(0x78abb7ba,0xb9dbe10f),LL(0x3eb157bd,0xb3d0ff94),LL(0x1a32ace6,0x8180c4dc),LL(0x9e9b36a4,0xeb0124b6),LL(0xfee72796,0xc1e93da7),L_(0x00000141), + LL(0xedaed08c,0x46195fa3),LL(0xbcb9e4e5,0xc31c13ef),LL(0xb2cc8a6a,0x483f2eae),LL(0x912ca200,0x5ee60fa1),LL(0xff0ff27a,0x9e9c56cf),LL(0xdac70a7b,0x4977503c),LL(0xbda5a3e4,0xa564deab),LL(0xfe3a9fcb,0xa761938e),L_(0x000000f7), LL(0x7e821113,0x560a5e57),LL(0x4ec38a12,0xae4b9aaf),LL(0x8358d926,0x1cb5c9a3),LL(0x7b69c24b,0x0e546449),LL(0xc0748541,0x660a2d50),LL(0xa4c426fb,0x87263ee5),LL(0xc286e0b9,0xf7b3ba20),LL(0x4bed6c50,0x9dd6eae5),L_(0x000000f0), + LL(0x9263865c,0x4434e163),LL(0xf3a80e48,0x9ca69373),LL(0x9608a668,0xc4b09404),LL(0xb3964738,0xbe4c6ca8),LL(0xe169bce3,0x7c62a7e9),LL(0x4efa6e4a,0xb46b0f85),LL(0xe2e5aeb2,0xe0111694),LL(0x6babf49a,0xb472f5f5),L_(0x0000003e), LL(0x2589af29,0x286fb826),LL(0xc48651eb,0x97cf3fde),LL(0x6f9d0884,0x78bdc9d0),LL(0x16ca5665,0x7e3d1e7d),LL(0x5a2f1773,0xc2f14e0d),LL(0x5e7f3258,0x6606eb12),LL(0x77a28311,0x67442ff0),LL(0xc80cdb6d,0x08ac890b),L_(0x00000177), + LL(0x6f4c68a5,0xfaab85ab),LL(0x18a863e0,0x8722f321),LL(0x96627616,0xe3cfe440),LL(0x1b986ba1,0xc819b534),LL(0x03e0ab51,0x7155ab76),LL(0x9cf682fc,0xc9e37547),LL(0x4f4f98e5,0x137f31be),LL(0x3cda736a,0x3a508340),L_(0x00000001), LL(0x91d6868c,0x2e86f052),LL(0x70f48703,0x14f3d533),LL(0x6c353990,0x099af7f9),LL(0x0b2a6c71,0x3d4612fb),LL(0x9b98b62b,0x406a6c59),LL(0xe6249353,0x3b1b8dd5),LL(0xc92d7a1b,0x1f751af1),LL(0xf81ef140,0x4692b4e0),L_(0x000000ee), + LL(0x2c9d1106,0xdc34aa30),LL(0xa7bfe1cd,0x28ee1801),LL(0xd18bd43a,0x6d6e3e49),LL(0x52b35eba,0x25fc7059),LL(0x7bfa8888,0xe767a889),LL(0x487fe05c,0xbcac8ca5),LL(0x09996d23,0x52fe2328),LL(0xe81bfb43,0xe19e490d),L_(0x0000006e), LL(0x121f3bab,0xa0be654b),LL(0xd4307448,0x230e8622),LL(0xa9492744,0xcf01fc0b),LL(0xe7b1abae,0xcc98dab2),LL(0x4d9cffdc,0xf6504381),LL(0xe9697cb5,0x506e8f37),LL(0xc3dfae33,0xc253ed02),LL(0x141d1dda,0xb927e31d),L_(0x0000001f), + LL(0xd4e2418e,0x60bac026),LL(0xe3153f45,0x08c6a85e),LL(0x8e05a0fb,0xabec7d7d),LL(0x5867d053,0xca6918d3),LL(0x1ebaaa06,0x7d8627f0),LL(0x6ea92220,0xdfe74b9c),LL(0x27dc332f,0x54478deb),LL(0x33ddba7b,0x1e5dc6b6),L_(0x0000017d), LL(0x2feb3b84,0xd254321c),LL(0xbd92cdce,0xae0be12c),LL(0x91edd7e2,0x65ab5fae),LL(0xacef4485,0x607c22ad),LL(0xe3f288c1,0x01e22b70),LL(0xa3baea8d,0xe3598c73),LL(0xc24e3c94,0xf89cd9f6),LL(0x595791d3,0x15c887a5),L_(0x000001b5), + LL(0xa048c822,0xbacfe4cd),LL(0xc1680ce0,0xd2bf80ae),LL(0x4efcb3dd,0x3024028c),LL(0x029e5c0d,0x63d006d1),LL(0x5acb256c,0x1d3229cd),LL(0xee644462,0x6e4f2a9d),LL(0x4aa18f75,0xd4fdba43),LL(0x437e2a93,0x45e800b4),L_(0x0000013d), LL(0x7de729a4,0xcdd3e499),LL(0xf4a84a14,0x9fba5f0d),LL(0x87d56bc9,0xa92225d1),LL(0x637de402,0xa9f81afe),LL(0x142558b2,0x2061f42e),LL(0xd09b2789,0x1e15d846),LL(0x8753411a,0xf0c0c378),LL(0x83c3fa31,0x8e19c92c),L_(0x000001ff), + LL(0xda11421c,0x8eaf7d82),LL(0x6c3eef0e,0x9224cdcb),LL(0xd8359bd5,0x34b2e501),LL(0x734d08d8,0xd9f7f27c),LL(0x60136559,0xa91fd047),LL(0x7da1c7f8,0xcb1bc103),LL(0xf3f7e7c7,0x65c241e1),LL(0x7555ce39,0xf11e10f9),L_(0x00000055), LL(0x6b91fc57,0x5484eacb),LL(0x8a825cda,0x888b470b),LL(0xf0a2ebdb,0x81202cfc),LL(0x3ac37a5b,0x05c01038),LL(0xfe8f11c9,0x5bf196e7),LL(0x76123e92,0x19cd94ed),LL(0x353febf7,0x9c972db2),LL(0x65d70280,0x988f859d),L_(0x00000064), + LL(0x217d03c5,0x56db67a5),LL(0xf3d76a7f,0x992b8bdf),LL(0x4fb50dfa,0xf9702a82),LL(0x5593b5d5,0x9088a3ea),LL(0x0ef00d4d,0xbf26b47f),LL(0xeb497149,0x0793417a),LL(0x58262023,0x465f75db),LL(0x4abc908a,0x2c6c549e),L_(0x000001e0), LL(0x1667b32f,0x4ef57c25),LL(0x0a4bdcb0,0x6e8c095e),LL(0x696391d7,0xabbc605b),LL(0x1dd44220,0xc3e47f6e),LL(0x38adb47e,0x8f35f645),LL(0xd3c084d2,0xf253b25e),LL(0xfaa3b241,0x0b53ca2e),LL(0x2e3d4cbe,0x6c56c789),L_(0x000001f4), + LL(0xf26ba24d,0x0d68d639),LL(0xbc6bcbf3,0x76bdcf5c),LL(0xd445a425,0x4ce8f583),LL(0x036223e7,0x12c0333e),LL(0x6b6c4cf4,0x411cd547),LL(0xfa6d4a89,0x6d3fb3c8),LL(0x7e41166b,0x906f6895),LL(0xd5a83001,0x28190a6a),L_(0x00000076), LL(0x33fa874d,0xdb9b9bf4),LL(0x6efa8bd8,0xc1467c44),LL(0x78067572,0x41957d49),LL(0x68a286e7,0x4563827a),LL(0x3562fe94,0xd87962e4),LL(0x5b2ba1f2,0xbce9e3b5),LL(0xeb40dfc9,0xedcbc4f7),LL(0x6ddd5a2c,0x2d14704c),L_(0x00000009), + LL(0x70bab965,0x738ba18f),LL(0x90ebea6a,0x3526ec84),LL(0x544312d2,0x606e765e),LL(0x4f7ce18b,0xe015cac8),LL(0x0b6de72b,0xea01d5b4),LL(0x2ff4bd72,0x0c7eba91),LL(0x90d594c1,0x8a32c97a),LL(0x881a1a5b,0x326bf5cb),L_(0x00000188), LL(0x8272c1c5,0x72a5ff2c),LL(0x0ff0a2ad,0x4bb6c7a4),LL(0x640ae3bc,0x87804672),LL(0x00da0040,0x94aaf22e),LL(0x2ebbcebb,0xd8f3e9bf),LL(0x8646dab7,0x40c90d99),LL(0xd08b3434,0x8f9d970b),LL(0xd1f0de73,0x2362e1e3),L_(0x0000004a), + LL(0x893123ae,0xc8b47342),LL(0xedaef283,0xde3df2dd),LL(0x303757c2,0xc96f2592),LL(0x089845fe,0xa0d3f290),LL(0x18b0b508,0x4bf7b214),LL(0x93c88975,0x3cd67758),LL(0xed218a4a,0x81a61b63),LL(0xfa78dcc6,0xdc5a49e5),L_(0x000000ff), LL(0x90ce8d10,0x6c24429c),LL(0xe40a36fc,0xc3ad8e81),LL(0x6178bcf5,0x16d9b177),LL(0x488d2cf8,0xc063b1e0),LL(0x57f41dcf,0x1cbca7c3),LL(0x200bb41d,0xec7a80d6),LL(0xd6366c22,0x84ceffb4),LL(0x66439f2f,0x6b57b092),L_(0x0000016f), + LL(0x869f06cd,0x97f9882a),LL(0xaa57537b,0xe3b1c07d),LL(0x917cf4ff,0xf6d1aa12),LL(0x2683a59f,0x73bca1f9),LL(0x613d785e,0x496708d8),LL(0x66ede999,0xf45fff24),LL(0x4e9727c1,0xf71c4572),LL(0x39995099,0xf55978fb),L_(0x00000090), LL(0x60222373,0xf249fc50),LL(0xdb62572f,0x95b4f7eb),LL(0x4efd7ca7,0xb4994b20),LL(0x0762c1c5,0x99292d14),LL(0x4d4c1985,0x140a608e),LL(0x7f0ba7f9,0x489023fc),LL(0x77e472d8,0xac039583),LL(0xca8aeb86,0x303e3f76),L_(0x000001e6), + LL(0xa1cd049d,0x859af0b3),LL(0x32b70dbe,0xd9aa6b96),LL(0x83656cba,0xa5229dc4),LL(0x02bc7ba1,0x574b487d),LL(0xffc68a06,0x9518ff35),LL(0xad36470a,0xaf20c720),LL(0xcf8b908a,0xee3bb49e),LL(0xf8b9d88a,0x204ddf66),L_(0x00000075), LL(0xc04ae92a,0xff799aa5),LL(0x352ca9e2,0x48de6d0a),LL(0xeb0f3051,0xe2b8f5f2),LL(0xa98f1062,0xdfe726fe),LL(0xc285eca4,0x22419400),LL(0x527244d3,0x441ba1f9),LL(0x3ec0c841,0x9ac0f611),LL(0xf7b09376,0x483ffb67),L_(0x00000159), +}, +/* digit=74 base_pwr=2^370 */ +{ + LL(0xce2d58e6,0x4fd2b399),LL(0xe56a0a18,0xd46ffafd),LL(0x43a772e3,0x61832664),LL(0x5e99ec73,0x5a652a9e),LL(0x068acc97,0xda22ced1),LL(0x829eb99d,0x17534159),LL(0xc94c616f,0xf6ab0176),LL(0xa334609d,0xf8e1dfbd),L_(0x000001bc), LL(0xf0e586eb,0x03144a03),LL(0xdf49ef2f,0x70d82d13),LL(0xf054795a,0x3d4fad35),LL(0xca4e83c9,0x7178dcbf),LL(0xdccd2e81,0x06f96d5d),LL(0x059906d9,0x99860a4c),LL(0xb0cc8989,0x0b7c4473),LL(0xc7a2422f,0xc031d2a9),L_(0x00000158), + LL(0x990d40ea,0xfd292d5f),LL(0xfe2aa0bf,0x0b3c033e),LL(0x350ffa07,0x7093caf5),LL(0xcba18d05,0x8e77aa62),LL(0x5de1ef34,0x8dcafce9),LL(0x8d305062,0x54c13b97),LL(0xa2184206,0x024b7581),LL(0xc1eed7a3,0x7a430aeb),L_(0x000000fb), LL(0xd2467c3c,0x5919f6e7),LL(0x6f3a2cc6,0xe4ef4ee6),LL(0xd95dc335,0x8b15339d),LL(0x53862418,0xc47f7183),LL(0xdc9f6ee9,0x0164075a),LL(0x8fc3c2d0,0xfb8c9b9b),LL(0x82f15ec0,0x2cab4250),LL(0x6da80b24,0xef0f9370),L_(0x000001fe), + LL(0xce799013,0x9e5e77aa),LL(0xdc0e8efd,0x74901979),LL(0x87335bd4,0x8d25ae87),LL(0x7663b155,0x30b14eb3),LL(0x42427def,0x9f7acb63),LL(0x504e9e47,0x8f787f03),LL(0xb68c9ee0,0x9fb2d8ed),LL(0x3fafed1d,0x168ae687),L_(0x0000008e), LL(0x84c837fd,0x1b05f526),LL(0x361ad6ff,0xa178f8a8),LL(0xba6a96b6,0x0afa0765),LL(0xf1a53f48,0xee02b40e),LL(0x455203e1,0x280a052e),LL(0xa80a8929,0xcc11be29),LL(0x6815682d,0x4811eb83),LL(0xd7ede303,0xae7746c4),L_(0x000001bd), + LL(0x33981c54,0xcebb5e69),LL(0x32546345,0x544b1b16),LL(0x84cafbc6,0x7981b01f),LL(0x5cddc181,0xaa139311),LL(0x1378ad86,0x68cbb494),LL(0x7e2675fe,0x588ce3ac),LL(0x7f2694e9,0xab708d62),LL(0xeda381dd,0xea9b5a9d),L_(0x00000155), LL(0x31d5f56f,0x1a13df6f),LL(0xcdb4564c,0x02f2a54c),LL(0x586ae362,0x19118f47),LL(0xdb9ebb1f,0x7fa3e3f6),LL(0x0b71b651,0x82c695b0),LL(0x82ecf8c2,0x58306aa8),LL(0xc8a72bb7,0x24bb71fa),LL(0x1671f4f9,0x8041f535),L_(0x000001c7), + LL(0xccfdb09e,0x9138999d),LL(0xfc40b806,0x190da0c1),LL(0x248d01f8,0x660fe973),LL(0x04db0124,0xd2a3e26a),LL(0xae1441c8,0x3b5b69d9),LL(0x542d784a,0xe47e9fb1),LL(0xc8706904,0x07fd3e18),LL(0x7b0252dc,0xac397897),L_(0x0000003e), LL(0x9bf565f8,0x7a07a372),LL(0xb2a08e69,0x88b9b9fe),LL(0x23737883,0x4af5f0ac),LL(0x3ee2589f,0x4d74831e),LL(0x0c99ecfe,0xe461011a),LL(0xccb75730,0x5f6ac945),LL(0x44dfe861,0xde67a0c4),LL(0x0a4190dc,0xc1f45c07),L_(0x0000010a), + LL(0x8eb6ed93,0x8217bc59),LL(0x480c8b3d,0xf9f73e7b),LL(0xed85e1cf,0xa742114e),LL(0xff0dd45c,0x5d90782b),LL(0x6499236f,0xa8a56eb6),LL(0x81a46542,0xc0a1d718),LL(0xb645bf88,0x5ad3645d),LL(0xda5b3451,0xe35c84b5),L_(0x00000062), LL(0x8177efc2,0x91a87849),LL(0xd64e2a8c,0xb25866c7),LL(0x7fcea597,0x7ba86329),LL(0xf0a84157,0x4ae784c7),LL(0x6d45e6a0,0xe8a4e8e8),LL(0xbad02a45,0x2b8a78de),LL(0xcb445d9a,0x1a096a8f),LL(0x1c606af7,0xcd9a146c),L_(0x000001e4), + LL(0x9ad71dbf,0x013860f9),LL(0xefc2f3af,0x0bf5cd2e),LL(0xcc51df06,0x366cecf5),LL(0x8b6d4774,0xff5f3234),LL(0xb320cd40,0xa48903e0),LL(0x331e170d,0xf463c308),LL(0x7b602dac,0x3d097dcb),LL(0xca3b41f6,0x1437b881),L_(0x000001d6), LL(0x1f06f98d,0x9812b78b),LL(0x0835cbf9,0xa1b46e69),LL(0x96985a6e,0xc756e3cb),LL(0x8c83dd55,0xe0033ed2),LL(0xc71730ab,0x7a46dc00),LL(0x333b6de9,0xe8045912),LL(0x8f656577,0x4a453d9c),LL(0x385fe0c5,0x9c113ca9),L_(0x0000014a), + LL(0xcb831835,0x9d0103c9),LL(0x47a18b26,0xbca486ca),LL(0x2d151fe9,0x8a31af06),LL(0xc2bcd6c3,0x4c38dae7),LL(0x6a2dd494,0xd0cab5f5),LL(0x0e8dd5e2,0xaaffb6b8),LL(0xf1aa0451,0xd1b4ab87),LL(0xd7ec926b,0x23418baa),L_(0x0000000d), LL(0x61149962,0xd2761793),LL(0x4dbe67f0,0x5f42f5e1),LL(0xbdaa4d19,0x9aea3745),LL(0x2770c1ff,0x336a10b0),LL(0x730746fe,0x965986ef),LL(0x500d5a19,0x98b61ae7),LL(0x2c56a5b5,0xfbc2b7c9),LL(0xf20ece26,0x484287c0),L_(0x00000016), + LL(0x47a76498,0xc197a42e),LL(0xa29d1821,0x8a2704b7),LL(0x54a27b18,0x6d7a65e1),LL(0x382392a3,0x6e25e555),LL(0x583ca3d8,0xba1298fb),LL(0x92305de9,0x8762634a),LL(0xc416a724,0xe3560751),LL(0x52b73064,0x9264b444),L_(0x0000019b), LL(0xd4b7caca,0x4fc9c7a4),LL(0x0d61d52c,0x88ca3d7d),LL(0x39c96801,0x9d85c914),LL(0x072c2d52,0xeae1af66),LL(0xdc7e4834,0x4567d964),LL(0xcf69cc8b,0x225a1435),LL(0xec05edf7,0xcf0fd41b),LL(0x16674dc5,0xa580cfec),L_(0x00000120), + LL(0xc929fb7d,0x666eee50),LL(0xa82ad2d3,0x943d9f3f),LL(0xe959c5db,0x01361c6c),LL(0x413dcd10,0x810dc990),LL(0xba8e95a8,0x111bfce1),LL(0x144ccf37,0x37942ccc),LL(0xbba23cc8,0x6250c86e),LL(0x64797a98,0x95ec8524),L_(0x000001d4), LL(0xa61fd6b9,0x324caf26),LL(0x208fb38d,0x802296a3),LL(0x923005fe,0x1cab4d64),LL(0x545d2ffc,0x7edf08f3),LL(0xfd85bdf3,0xbbb0b3e1),LL(0x9feb12a7,0xab0ed8c1),LL(0xeb4e517d,0x45179c50),LL(0x5c75791e,0x1bac0194),L_(0x000001cb), + LL(0xebddb001,0x809051bf),LL(0x932eff69,0xb46ff016),LL(0x5ce81f11,0xb49261d6),LL(0x39183971,0x75ef3047),LL(0x65753518,0xefad1e5a),LL(0x7887db59,0x147b2e1c),LL(0x6c93b47e,0x239259f0),LL(0xb5f34e30,0xe56b4fc3),L_(0x000001c8), LL(0x905217df,0x861e1dc6),LL(0x9fada5a7,0x3986b470),LL(0xf9e88cd0,0xcaab1d92),LL(0x839c290b,0x02e99a54),LL(0x39b3ffa9,0x910523a5),LL(0x655e6f7c,0x42d47f30),LL(0xb367bd8a,0x1e0a7f1b),LL(0xe25d7561,0xb79355b2),L_(0x000001ad), + LL(0x4987e69d,0xd0493c78),LL(0xba0a5cbe,0x59557a83),LL(0x75bbdb17,0xf2acabc4),LL(0x65e4a623,0x8bf3c53d),LL(0xd71fb7df,0xc8eb2466),LL(0x7545f576,0xc6d8140f),LL(0x620a0123,0x9f02bd4e),LL(0x67837a46,0xbdab7cb9),L_(0x000000e5), LL(0x8b871f92,0xc2538248),LL(0x4b3a3a4d,0xef93af7d),LL(0xdbf730c3,0xbfd3ed77),LL(0xf764526e,0x07f935a6),LL(0xcb6152b4,0x1c016476),LL(0x2bf32571,0xdee3ad5a),LL(0x88aaec73,0xfb09ed56),LL(0x5a614fc9,0x9f8877cb),L_(0x0000002d), + LL(0x7c127a9f,0x26712a11),LL(0x642533c6,0xf98c4d57),LL(0xc60c2ae3,0x65b1d46e),LL(0x444527b6,0x71dbb3f8),LL(0xe33842c0,0x4381ace7),LL(0x9e839852,0x6a3b078e),LL(0x2c3ba212,0x4de7e214),LL(0xed5d5463,0x3d30ea71),L_(0x00000163), LL(0x2312ab46,0x8e3acd07),LL(0x631b2001,0xb498759a),LL(0xcc66ba39,0x22f04bfe),LL(0xd634fae4,0x0f57a006),LL(0x6fb05b0c,0xd36ff867),LL(0x74e2535a,0x85b24cf7),LL(0xe1ec7865,0x1a8674c2),LL(0x8caeb36c,0xac17ecae),L_(0x000001bc), + LL(0x41ce000d,0xbbc4d762),LL(0x3f8e4f4a,0x850dfa23),LL(0x315d2f3b,0xc39228da),LL(0x146ba937,0xe938195d),LL(0x05ba80b7,0xa6946ca0),LL(0x5996d5e9,0x8582cc92),LL(0x20830fe7,0x9fbffd5e),LL(0x49bd8864,0x35f18f3a),L_(0x0000016f), LL(0x52a9ca5f,0x342678f8),LL(0x0fde0f1b,0x03e8057f),LL(0xb03f731a,0x68041b2c),LL(0xc1b65ef3,0x5e348390),LL(0x812a39aa,0x6058d643),LL(0x85301a5f,0xf291dfe0),LL(0x3f43fec2,0x4f9f3872),LL(0x0ea19231,0xfc5949c4),L_(0x00000109), + LL(0x2071dbfe,0x4b89bb6a),LL(0x243135cc,0x0ff935bd),LL(0x7a1a0770,0xe0c7a9ac),LL(0x227a5593,0x6dd5bad8),LL(0x3493e1c5,0x8ba3715d),LL(0x35c53b09,0xc6b271ea),LL(0x744d2bd3,0x2e9feb1a),LL(0x214dd692,0x2db119cb),L_(0x000000b1), LL(0xeec6e175,0x1f9d847f),LL(0x63ead5ad,0x5419e32c),LL(0xd6156a57,0xe233586a),LL(0xe66a5622,0x39029648),LL(0x4092e8b6,0xbf26b933),LL(0x1e5f719a,0xcd5ad746),LL(0x4f5fc4b9,0xf552f21c),LL(0xd86c96b0,0x997ee591),L_(0x000000b3), + LL(0x9ad2983e,0x090fc8d4),LL(0x3313e6f8,0x55cd8ed4),LL(0x8651d168,0x6a906312),LL(0x55315e35,0x591a919d),LL(0x83731bdb,0x5172884d),LL(0x209c90f3,0xe2ac6b9b),LL(0xfbd125cc,0x92e9decc),LL(0x7d19839d,0xff6024c6),L_(0x0000016a), LL(0xed7835fb,0x29318405),LL(0x7858c38c,0xc88969fe),LL(0x39c13839,0x9e3d9e6b),LL(0x193b8588,0xa0e5421a),LL(0xe0167b73,0xc6ebad78),LL(0x3cbbe3a6,0xea506121),LL(0xa8cbcf0f,0x20f99af7),LL(0xdbdf82d3,0x2e935934),L_(0x00000010), +}, +/* digit=75 base_pwr=2^375 */ +{ + LL(0xa97c0378,0x851811ca),LL(0x042ad325,0x0425fd5a),LL(0xae223e37,0x93b83181),LL(0xf721e5a1,0x90e949dc),LL(0x0e3457f0,0x3df6a852),LL(0xb654bab8,0x5f22447c),LL(0x720ad354,0xec0dfbcc),LL(0x64eddb51,0x81daf300),L_(0x00000178), LL(0xa97e2d48,0x5a22dda9),LL(0xa660386d,0x4ad7ed63),LL(0xf4eac86c,0x0e9bdfd6),LL(0xa0b3cfe9,0xf3d0576e),LL(0xb9746e43,0x38598ede),LL(0x5a4f247c,0x6c63f53c),LL(0x9110d7d6,0x937de7ce),LL(0xedc5628f,0xd7ef1580),L_(0x00000149), + LL(0x07ceb75a,0x22d17ed3),LL(0x1d003cd4,0x816d46db),LL(0xacb6fef8,0xb000a452),LL(0x9bbe93ac,0xa33425cb),LL(0x8e7044e1,0xd94105ed),LL(0xbf04fc32,0x7b448d72),LL(0x8a8006d7,0x77527b27),LL(0xce1c27af,0x481ed3f3),L_(0x00000072), LL(0xede21a04,0xbea8a562),LL(0xfe905ba2,0x7dcaa390),LL(0x8bd01814,0x8c2e3be5),LL(0xad37906a,0x4bd6ba24),LL(0xe4147c93,0xab35993e),LL(0x54b18700,0xeb32c196),LL(0xc4b62833,0xfde9fb6d),LL(0xefce4982,0xfd482fff),L_(0x00000009), + LL(0x326389a5,0x8c87ee07),LL(0xc8d06b64,0xf1eb33ca),LL(0x9f6c9bde,0x905394b5),LL(0xe7cff087,0x9daef572),LL(0x15defa0d,0xdcb4d146),LL(0x48372dac,0x9d9bc2a2),LL(0x558be40e,0x918fbab0),LL(0x093092b7,0xd1e9cb05),L_(0x00000158), LL(0x80e77a4e,0x1e399561),LL(0x7faf6193,0xe636d3ab),LL(0xe0b54eab,0x0991ea6a),LL(0x12db09ef,0xe300cb6a),LL(0x2c4871c5,0x5b2c3ec2),LL(0x74b476ca,0x01ab0f81),LL(0x571997b2,0xe7647206),LL(0x7f0593e9,0x19083320),L_(0x00000084), + LL(0xcb8ddce8,0x53953842),LL(0xc94eac86,0x510be22e),LL(0x18af52b2,0x5204fc68),LL(0x6cd384c6,0xb08bd4d3),LL(0x40918e38,0xbbca8f66),LL(0xc2ac8cd3,0x9b3d5866),LL(0x2e4fdaef,0xbd15b5a2),LL(0xcebfa696,0x2334128f),L_(0x0000013b), LL(0x8986becf,0xa9f4f0a0),LL(0xb5bde2b0,0x2781857a),LL(0xf623a384,0xf48f7d34),LL(0x2fc32d5d,0x5aed2eab),LL(0x3357b29a,0x85d8000c),LL(0xd7a02a4f,0x47c091a8),LL(0x83e2289a,0x748c8758),LL(0x78f3991d,0xb7dfe04c),L_(0x000001eb), + LL(0xe09895cd,0x8dc487b8),LL(0x1ee06c73,0x4cd52925),LL(0x3615a586,0xb40cf5e8),LL(0x67c6302f,0xe2444c40),LL(0xad0f9fd1,0xf79a9138),LL(0x21560ea8,0x53b1f139),LL(0xdfcbf2e3,0x42f5c15e),LL(0x937e18f2,0x4b1cf2d1),L_(0x00000141), LL(0x2b5822b9,0x4289e6fd),LL(0xcce11b47,0x6b3fbfe6),LL(0xfc6c35ad,0x42db4435),LL(0xf40269b9,0xd9e6571d),LL(0x052ecbf9,0x43b34e97),LL(0x85c17cbf,0xc1ac2947),LL(0x2cf45704,0xb8e4df72),LL(0xccda58a9,0x77f6ed90),L_(0x00000164), + LL(0x736929bd,0x2018c224),LL(0x183d085f,0x4be72094),LL(0xb8504d5a,0xa1b86c43),LL(0xb6f18e21,0x4ff46986),LL(0x6ad297dd,0x8a9142ac),LL(0xaf090f36,0x09fe86be),LL(0x3b6921ae,0x3c8c552b),LL(0x953006e9,0x8c728bce),L_(0x000000e1), LL(0xaf72ac6e,0x80956c6b),LL(0xfa741157,0xc4c2c5f7),LL(0x5e9870c9,0x7ec1eba5),LL(0xf57cfca1,0xc10b4b60),LL(0xa6e490cc,0xb0618cc6),LL(0x39e98a9d,0x31fe5f00),LL(0xf0f53611,0xb1970dbc),LL(0xa442baa6,0x0c613700),L_(0x00000000), + LL(0x90a308db,0x61cca596),LL(0xcbaa218f,0x6743511c),LL(0x6574da13,0x71c34c1b),LL(0x50cf99a2,0x570b3140),LL(0x8ae9a0ed,0x8fffc78b),LL(0x816fb1cc,0xe8d131b4),LL(0x052abb5d,0x64b0d1f3),LL(0xe564075c,0x02a02c6f),L_(0x0000014e), LL(0x5fb1b653,0x57fab65e),LL(0x72bc2ab8,0xcd37e51f),LL(0x6c8bb1d5,0x57d81547),LL(0x75b37fd5,0x572bd385),LL(0x1441e8d8,0x0eb239d2),LL(0xd5a6c392,0x7cc7ae14),LL(0x16b857f6,0x7141c32a),LL(0x931901c6,0xd7fbdf6f),L_(0x00000041), + LL(0xe53ec842,0x16678799),LL(0xeb3f78fb,0xdb8c26f2),LL(0x1091298b,0xe09307e4),LL(0x265d3b22,0x79682bc2),LL(0xf829161a,0x8a62536f),LL(0xdea99410,0x002fb6de),LL(0x3d369ec2,0xca79fdbc),LL(0xf58b2f20,0x3d6462ba),L_(0x00000130), LL(0xf756d9fe,0x61419646),LL(0x9934641a,0x308b017c),LL(0x5de627fc,0xbea2b76a),LL(0x23ee7d29,0xba8603cb),LL(0x6ab47900,0x85c79476),LL(0x004b96bb,0xf41684cc),LL(0x94d547ed,0x8b7656ed),LL(0x2003142b,0x30eaee30),L_(0x0000014d), + LL(0x8759d864,0xfb67c7fb),LL(0x0407a03f,0xfb9982ce),LL(0x020231ea,0x55103874),LL(0x9d3dc0fe,0x9a32c3ea),LL(0xc54c5166,0x8e76b967),LL(0x7422e59f,0x538d7969),LL(0x1567215f,0xc1772e51),LL(0x1bee3ac0,0xeb37df59),L_(0x00000159), LL(0x44e31ef8,0xeef4a1e2),LL(0xf4d1de52,0x78709a3f),LL(0x144880f0,0x90e1bf50),LL(0xc5a2e2e7,0x6576ad05),LL(0x963afdb1,0x858a5053),LL(0xfb62a6ae,0x720e44be),LL(0xf7d3d903,0x85ea2a35),LL(0x1ce3e300,0xbc7fd8b2),L_(0x00000178), + LL(0xc85cdf02,0xe346fabb),LL(0x2b0945e1,0xe07629de),LL(0x76a1e2b5,0x45e5724f),LL(0x43bc885c,0x6f8c506b),LL(0xbd1f5350,0xc4a247ae),LL(0x9759458e,0xe8c49a8e),LL(0xad9f81fe,0xef961f24),LL(0x0789ce81,0x4d378c78),L_(0x00000045), LL(0xbdac3a5a,0x0d120ebb),LL(0x2f38cfe3,0x29a29c91),LL(0x470f8673,0xa93d27e5),LL(0x85f54b6a,0x347ce7f7),LL(0x869bc2c6,0x681c6e83),LL(0x240291c1,0x5f895132),LL(0xd778a681,0x9354c132),LL(0x35657182,0xd9159913),L_(0x000000a6), + LL(0xdc0c7615,0xd08ada52),LL(0xf64fa06b,0x6fe343d4),LL(0x0f9f191b,0x269d74ba),LL(0x2b582fb8,0x3e1302f6),LL(0x4f3fa209,0x3dd58666),LL(0xbaec4e8c,0x2346df80),LL(0x2addc663,0x961745b5),LL(0x6358e5f3,0x18d6fbc0),L_(0x000000c0), LL(0xbca3dd73,0x0f473bff),LL(0x97d4d8fb,0xcbcc7f23),LL(0x592d62f3,0x8c21a728),LL(0x2d18d7d2,0x08669251),LL(0x5acddad3,0x10138815),LL(0x5eb1d5da,0xaf710391),LL(0xf88b7078,0x200a8738),LL(0x9614df01,0x7fc7b078),L_(0x000001a8), + LL(0x39f8e71a,0xf8ccaa40),LL(0x89f9fde7,0x4a565eb3),LL(0x3a88c7c8,0x2241445b),LL(0xb88e20b6,0x0479b1b1),LL(0xe22d8db0,0x96695cdc),LL(0x02fe3690,0x48a70132),LL(0xaba6a66d,0xafe3713b),LL(0x5be868e0,0x4959a9b6),L_(0x0000007a), LL(0xc518718a,0x3e38b8c6),LL(0x00a613fe,0xee16d3bd),LL(0x8bab2dac,0x5f51a73a),LL(0xecd0dde7,0x5d598b1d),LL(0xf9a19d5d,0xcc2ed8e1),LL(0x74ed2f5d,0xb66c7686),LL(0x1a036457,0x45717b78),LL(0xeb14d9fd,0xebae3cd5),L_(0x000000c6), + LL(0x42bb5d7b,0x69ebb0a9),LL(0xf6f6e0b1,0x24c217db),LL(0x751a1bfd,0xd4eab425),LL(0x1570cf87,0x46afeada),LL(0x55c17749,0x84f69779),LL(0x72264346,0x0e0b6e91),LL(0x43f9c928,0x2a080641),LL(0x5face2cd,0xd6f99ce6),L_(0x0000011d), LL(0xb97b52aa,0xb84b4c35),LL(0x079d267b,0xea1ade5e),LL(0xfd3d3470,0x64da3bef),LL(0x603b5d99,0x4e2ca35b),LL(0xc90c1bef,0x267ee929),LL(0xa4dcd6fa,0x2e371559),LL(0xeadf09d2,0xdc90620c),LL(0x7ae7bd5e,0xa0074931),L_(0x00000049), + LL(0xe19a4c40,0x3bef2c63),LL(0x57c68d2d,0x922215ae),LL(0x03e85348,0xb54763ee),LL(0x7a4a0d2c,0x4381fb33),LL(0x747d2320,0x3d971222),LL(0xc828be44,0xd96627f6),LL(0x6d1199b7,0xabfb6b5f),LL(0x3d2170b3,0x7e019a43),L_(0x00000146), LL(0xbb1d3366,0x8c946a2e),LL(0xa111d7a3,0xb29fb103),LL(0x75997def,0xa9647d36),LL(0x82824e10,0xa45fdefd),LL(0xf29d6d05,0xa9b94f37),LL(0xe35c500f,0x317e08c1),LL(0x4c601022,0x2a6ed921),LL(0xa2afcd4a,0x376d3ee0),L_(0x0000012e), + LL(0x14a651b3,0x6443bb77),LL(0xc4e092bf,0x3d523243),LL(0xb725b204,0x563f7657),LL(0xf0d19ab6,0x6dd80a2f),LL(0x0c09a035,0xa3a7805c),LL(0x72bfb218,0x767659a8),LL(0x5001304f,0x06ad0ad0),LL(0xae6cf2cf,0xa76a4e48),L_(0x000000c0), LL(0x73b1275f,0x58294610),LL(0xf2619fd4,0x12455781),LL(0xc2991198,0x822a98ac),LL(0xc52b1be7,0x6f92e55b),LL(0x85c5dde4,0x6f912a88),LL(0x71070200,0xc6ff80dc),LL(0xed86ff4f,0x5fb4c0fb),LL(0xf6cd415c,0xb7ec4feb),L_(0x00000025), + LL(0xe13291f0,0xa731176a),LL(0xc2f095b8,0x53e5b4c8),LL(0x22d8f01b,0x9b8d5a23),LL(0xf09c9053,0x6cfad192),LL(0xac4c2264,0x016016f3),LL(0xc2d48df5,0x500c56f4),LL(0xbed57312,0x206618c9),LL(0x249d3807,0xcc5fef52),L_(0x00000059), LL(0xa752bb21,0xa854f0db),LL(0xad82109b,0x2bd9fbff),LL(0x39d0e928,0xe612cee3),LL(0x5cfc63fe,0x3aca9e51),LL(0x18541bf3,0x0fd5f823),LL(0x1df11f0f,0xdccc44f5),LL(0xe5d7f0f8,0xc8e26d92),LL(0xdc204c43,0x7278c334),L_(0x000000c9), +}, +/* digit=76 base_pwr=2^380 */ +{ + LL(0x90df775a,0xa877dfb5),LL(0x0e8c657f,0xf628f95d),LL(0xc58775b2,0x1f94b622),LL(0x55966c52,0xd94ba3e7),LL(0x2b826bca,0x536e1836),LL(0x1429585e,0x6a8bf64e),LL(0xab9cff45,0xfdc0d065),LL(0x7ad254f1,0xd4d31c6b),L_(0x00000099), LL(0x10be2241,0xa869cd60),LL(0xb5cc49f5,0x399cde94),LL(0x029dfb84,0x53bc96f6),LL(0xc7d08220,0xf3d33d68),LL(0xb6cb5f4b,0xd70bd72f),LL(0x81e790ba,0x5f85b782),LL(0xd6b87ddc,0xd6fbd3aa),LL(0x9bddab2f,0xd049e67e),L_(0x0000007c), + LL(0xcff0963f,0x1215b91a),LL(0x7ce3e2ea,0x276c4b8c),LL(0xae4d76fd,0x27c2c599),LL(0xcf3a3b9f,0x985a8106),LL(0x706667b3,0x3dd5545f),LL(0xe5bf95ab,0x9ae8ea63),LL(0x5a494d9e,0xac8eb301),LL(0x36df8e2a,0x0aa3c14b),L_(0x000000cb), LL(0x241e0605,0xec384ae0),LL(0x19fc3d54,0xbfb3e2ee),LL(0xce0a2d7c,0xa5ac041b),LL(0x7e0aa0d1,0x22b978b5),LL(0xcf6adf10,0x50508726),LL(0x13804525,0x77d6d81e),LL(0x02fbac9b,0x34536c98),LL(0x6666d2c2,0xdd6867d1),L_(0x000001f2), + LL(0x0fd75964,0x2c167fba),LL(0x79fb34e2,0x68658968),LL(0x3d2ac14b,0x6bb85f11),LL(0xf9265032,0xe567c4fc),LL(0x09815c6a,0x30478f2d),LL(0xb2da7033,0x2d4e045b),LL(0x7450186c,0xeb491702),LL(0x3d6ff5bb,0x394b01cb),L_(0x00000186), LL(0xa96aee5e,0x7fa7f974),LL(0x54b866f5,0xd445b199),LL(0x7edd540d,0xda3cc41b),LL(0x6672b9ea,0x3c302e2f),LL(0xf5adb45c,0x5ea3de1b),LL(0x201f8535,0x70efd3fa),LL(0x9bc11d2a,0x3d2e2804),LL(0x4d97e055,0x461aca9e),L_(0x000001ac), + LL(0xdb5aaa83,0x020dd38c),LL(0xf16c4ef4,0x0a2db89c),LL(0xa5cd426b,0x43e8b727),LL(0xf5617c8e,0x23ddc0ba),LL(0xb43d6e58,0x259e17f2),LL(0xc826180e,0x06737413),LL(0x55f63ef5,0x434e7412),LL(0x23e6163c,0x6e327096),L_(0x000001c1), LL(0x9695e5ae,0x47505a19),LL(0x6e1e36e5,0x74ec16e2),LL(0x43d8b0e2,0x4831814a),LL(0x037ed439,0x5b1a104c),LL(0x375672e8,0x5bc4b456),LL(0x9fdc64a0,0xf4e8604b),LL(0xdb5b0994,0xa1d8d54c),LL(0x035e5850,0xc99108d7),L_(0x00000018), + LL(0x6b6358ca,0xae97ec9d),LL(0xf89cd326,0x96931bf3),LL(0x0db33ff8,0x1728a8a2),LL(0x5df6988b,0x8b413bf1),LL(0xc9cd5efc,0x7876052f),LL(0x980dbb18,0x662d8014),LL(0x7d44d414,0x56d9235f),LL(0xf0c89214,0xc3bb2edf),L_(0x00000183), LL(0x7d2553d4,0xd43349dd),LL(0xc275fa25,0x98c5095d),LL(0x7e9d6a23,0x5dae8169),LL(0x48607095,0x004d6221),LL(0x0de66e5e,0x88753853),LL(0x407e61fc,0x0cbeddeb),LL(0xb1e576ab,0x85968acb),LL(0x6df65046,0xd7b6cee3),L_(0x000000d4), + LL(0x1d1376b4,0x2c927fc0),LL(0xf4b1b912,0xcb3e3200),LL(0xd6a633a1,0x54477db8),LL(0x4c991410,0xbf0c1c32),LL(0xa9a4d4dd,0xda008df0),LL(0x30c04f89,0xf68e5507),LL(0x1a10f51d,0x5ce5c51e),LL(0x41031547,0x5069b8da),L_(0x000001be), LL(0xdec76b03,0x4ca12b9b),LL(0x53a8bf3f,0x4e3a3297),LL(0x6ef86a89,0xc5a499c8),LL(0x38a372fc,0x97d666c8),LL(0x6ec44e4a,0x41b99123),LL(0x95600ea2,0x650c8dbf),LL(0x4eb71cc1,0x4c7627fd),LL(0x54f79c84,0x0226e786),L_(0x0000002b), + LL(0x69ce7225,0xf1a6e969),LL(0x77785cf6,0x9cc6268e),LL(0xc7d8303c,0x2b8308ef),LL(0x6b0e5276,0xcba9dc8a),LL(0xa4bf9968,0x416fd26f),LL(0x8c4cdb7c,0xe7d932fa),LL(0xde7df0b0,0x472063b5),LL(0xd8e36d94,0x1f940a1b),L_(0x00000187), LL(0x852f11b9,0x528d0c6d),LL(0xe34ebb6a,0x0491c222),LL(0x572cf3b4,0x3235246f),LL(0x6a507a97,0x5419c482),LL(0x151b4954,0xa45d1468),LL(0x1e9ff246,0x555cac59),LL(0x74cf9098,0x3f67c66c),LL(0x4d10852a,0x1f28bc81),L_(0x000001b1), + LL(0x4d4d6495,0x86a81e7f),LL(0x54df2c4b,0x4316eb10),LL(0x19b90005,0x41b6877c),LL(0x63ac12d9,0xdbe38379),LL(0x7bdc46a9,0xe68280f8),LL(0x04afbefb,0x1d97d1dc),LL(0x64f8fe97,0xfdaabbcc),LL(0x3ef9d7ec,0x355cb854),L_(0x00000021), LL(0x4256ebb2,0x5f744a3f),LL(0x462bbab0,0xc6587a4d),LL(0x8de305cc,0xf7cc14a1),LL(0xd2c0e8c5,0x1c2fa456),LL(0xd7f552fd,0x93096db0),LL(0xdf5d165b,0xeef9e935),LL(0x0a7d4ef3,0x8313440e),LL(0x2ecbc3b6,0x31c6420c),L_(0x00000033), + LL(0x4ec080d5,0xc4429668),LL(0x00e4c47a,0x0c5790c3),LL(0x2bb3e90a,0x76060854),LL(0xb06d2fa8,0x51871594),LL(0xb8a8220c,0x35fa01c4),LL(0x96cfa275,0x351722c6),LL(0xd2b5aea0,0xcac00f2f),LL(0x9a7e1203,0x2e7bac23),L_(0x0000010c), LL(0xa79d695c,0xe8bbd2be),LL(0xc13d4a47,0xdd17ddc3),LL(0x3a2ef1d7,0x835e7fe3),LL(0x3a0d7223,0xdaab1fae),LL(0xeadcb841,0x2baa3375),LL(0x25d48c28,0x3675311d),LL(0x1cfce1d1,0x8cca0828),LL(0x6d648baa,0xe0c0ce1e),L_(0x00000060), + LL(0x39772678,0x4730fdee),LL(0x4814bf38,0x7da6e5cf),LL(0x717ace32,0x2440a79c),LL(0x1fa530fc,0xee76a431),LL(0xf0840ed5,0x64a9d867),LL(0x9b3e52c9,0x01e024e8),LL(0x388a3167,0xbc5a3de8),LL(0xb45ab215,0xddc3e927),L_(0x000001d6), LL(0xdb989f10,0xb0c72279),LL(0x88321c3c,0x461d5212),LL(0xe2e0c887,0x22583d6f),LL(0x0422ef3c,0x5319c021),LL(0xf26dbc88,0x3aba5f48),LL(0x62bbe876,0x7e742165),LL(0x411f00ae,0xd32aa7f6),LL(0x6608e197,0x1c414ee2),L_(0x000001fe), + LL(0xb2aed406,0x5c51d66d),LL(0xe49e1c0a,0x341d6090),LL(0xeca16754,0x50aaa76d),LL(0x3d4ae66e,0x23dd6ea7),LL(0xc264093c,0xb964aff3),LL(0x124124d1,0x6a903309),LL(0x320f2ccc,0x040a80c9),LL(0xa2fc450d,0x54b0cef1),L_(0x000001d0), LL(0x660ce624,0x876d5eb7),LL(0xb113de73,0xd6a08a24),LL(0x8fdedfa3,0x7e981617),LL(0xccca4ec9,0xbf6d63e9),LL(0x1cbf7303,0xd5b865e9),LL(0x06258e51,0x8b4e0432),LL(0xae5f01e7,0xed7485d8),LL(0x1fb3ec8e,0x5f9ac27f),L_(0x00000088), + LL(0xcd3a614c,0xaac40dbd),LL(0x933beab8,0x23a3080d),LL(0x4ca0b1cd,0x3c101e4c),LL(0x2d0376e4,0xcb246c76),LL(0x8ef2560e,0x3a96d882),LL(0x54be7604,0x792be430),LL(0x4ed7cfd6,0xe67d1eea),LL(0x924a5689,0x8c271a2c),L_(0x00000139), LL(0x63647d6b,0x9642188f),LL(0x514cab56,0xbcdfd904),LL(0x529ea4a2,0x876c6668),LL(0xcbd5305d,0xfa8c20c0),LL(0x58b69ea6,0x1ac42596),LL(0xf62b1c30,0x80232775),LL(0xee3d0824,0xc5b975d6),LL(0xc483c2ea,0x84841ab2),L_(0x000000c9), + LL(0x8bc6d688,0x6c7bd269),LL(0x7652c729,0xc96e37bd),LL(0x405791df,0x410e2904),LL(0x5ab8090e,0xf9bde0c1),LL(0x1a7b424c,0x37d8159f),LL(0x26876e27,0x6e7212ab),LL(0x5e21bc5d,0x2ff3af58),LL(0xa29dba58,0xd72ccea6),L_(0x000001a5), LL(0x7dd665a0,0xe048ab97),LL(0x19984a71,0x17b7a849),LL(0x8f61b833,0x9bdf5b57),LL(0xeb63a0f7,0x32adf9dd),LL(0x8eaf0eb1,0xe30814ac),LL(0x799c8225,0x35be0b92),LL(0xa082ff80,0x7e52495d),LL(0x196f3154,0xf5e9fa54),L_(0x00000164), + LL(0x3b3fccf2,0xf8b435c7),LL(0x7f599023,0x4b07f56c),LL(0xc6614964,0x1f9e8dd5),LL(0x7bfaa97f,0x57a41fcb),LL(0x056d5124,0x69dc4c19),LL(0x33956d10,0x2e70770f),LL(0xac607019,0x17397aa9),LL(0xbda3b4ee,0xb5df8c82),L_(0x000001c8), LL(0x11800cef,0x9f451540),LL(0x5de0b39d,0x9cd9ec18),LL(0xf09ca421,0x7bff70d3),LL(0x3fc8f958,0x6af7560e),LL(0x348716d4,0x2601495a),LL(0x4ed53bb1,0x39eaa8c2),LL(0x97052bec,0x87acae02),LL(0xb4363c7a,0xe01ba269),L_(0x00000117), + LL(0x010e4074,0x23ceb817),LL(0x68de1fa6,0xb6c3610b),LL(0x79b5037c,0x794616e3),LL(0x38ca34e7,0x6964c6c9),LL(0x64ba9eb2,0x9d828a84),LL(0x713e3f60,0xc6cafea4),LL(0x69bdca04,0x0035da7c),LL(0xc53921ac,0xe8c47770),L_(0x000001f9), LL(0xbe97815a,0x678d3502),LL(0x09bb25d5,0x417b9931),LL(0xadbec401,0xa021e930),LL(0xef1be11c,0xb53777ab),LL(0xfb3f04c1,0x2e6bc85d),LL(0xab9fbf13,0x22a4d27b),LL(0xb988012f,0xddee5ad9),LL(0xfda8aea8,0xf2fa9a7a),L_(0x000000cd), + LL(0xc43ac524,0x9805e79d),LL(0x95a2175f,0x2bee2dfe),LL(0x6125c31c,0x2b6284b0),LL(0x10319508,0x2264eba0),LL(0x8cedfa4a,0x25bc143e),LL(0x3199afa3,0xe3ae2485),LL(0x63067c6a,0xebebe969),LL(0x54a7cecd,0xb0d6a96c),L_(0x0000007d), LL(0x6de3a522,0x3a1a5044),LL(0xf721555f,0x644f2db4),LL(0x6dc38924,0xb72ad43f),LL(0x39beb126,0xe7dd7722),LL(0xd840de05,0xd6caacd0),LL(0xc67a2862,0xce6fa639),LL(0xba53021b,0x71087602),LL(0xec9b5982,0x69c6c679),L_(0x00000128), +}, +/* digit=77 base_pwr=2^385 */ +{ + LL(0xb75ed4da,0x7aaab2f5),LL(0x1ee37679,0x30159305),LL(0x0b02f44a,0x021962a3),LL(0xd622cf13,0x55a3eea1),LL(0x9a7dfa05,0x4fcd685c),LL(0x7a2a6aca,0xd2e75077),LL(0xbd4c914a,0x1e6aa905),LL(0xeec52d7b,0x4fd3f8ff),L_(0x00000173), LL(0xfa95204a,0x539f85e4),LL(0x36eeec34,0xe8ddc16e),LL(0x74599d1c,0x50244a9e),LL(0xb343c6c5,0x714c017a),LL(0xb07951ae,0x4503f92d),LL(0x44d15c8c,0x830499e5),LL(0x94680ff6,0xe7188a7b),LL(0xd6c4809f,0xdf43c147),L_(0x0000007e), + LL(0x1bf14718,0xf89fcfd4),LL(0xc079070b,0xa403ed89),LL(0xa107b324,0x18c3f861),LL(0x4dade6e3,0x665e9f9b),LL(0x332b6327,0xb408e3b3),LL(0xf62f16ec,0x11ee2181),LL(0x67bbd1bc,0xd0ba5904),LL(0x4b5440b9,0x4a8d7a0f),L_(0x00000134), LL(0x9cb1aa7a,0x8a32a33d),LL(0xedb96d1a,0x9ae722d5),LL(0x0654bd1b,0x1664c777),LL(0x03d0e5a5,0x6a4a631d),LL(0xfb01ee81,0x3d1d9344),LL(0xc7691584,0x1e1821b3),LL(0xbe2d285c,0xafc22520),LL(0xe0834d6f,0x0ccc1fe7),L_(0x000001bd), + LL(0x08d12f4d,0xf2407064),LL(0xd7255fb6,0x1c6799f2),LL(0xad7d86d9,0x8c32b1c7),LL(0x259fb289,0x172083c6),LL(0xcdc9f2eb,0xc85a5a26),LL(0xdca9f61e,0x303eee79),LL(0x82cff2b2,0x283cc245),LL(0xaea38a1c,0xd4cf3018),L_(0x0000004a), LL(0xf7fee514,0xb0b385f6),LL(0x0a48204d,0x785b0f88),LL(0xaff858dd,0xae66256a),LL(0x24f69e65,0x92f5e352),LL(0xe99c8d90,0xb2e1ccee),LL(0x40cfa4d5,0x5201f6ee),LL(0xfefbb836,0xbcabc908),LL(0x61f21689,0x688f6f44),L_(0x00000158), + LL(0xd756d8ef,0xaef07505),LL(0xdd4c26d4,0xfd1d5c09),LL(0xb43bb4e7,0x53c26645),LL(0x6a817eb2,0xf92f5487),LL(0x6209c32a,0x3e205a08),LL(0xb9b883a6,0x802502f8),LL(0xa6f9cb8d,0x2169b5a7),LL(0x87089ec1,0x07a66e8d),L_(0x000000a3), LL(0x4cc2981d,0xc0b6dcff),LL(0x39135fd7,0xe2051e1d),LL(0x0f800ca1,0x76456f99),LL(0xae12c766,0x987a86bb),LL(0xdfc5fcbf,0xcbf344cf),LL(0xa853db02,0x65a4f55e),LL(0x24b115f9,0xf4d8cf4b),LL(0x28cffa2b,0x7ef16ddb),L_(0x000000b8), + LL(0x9698f500,0x46e1cf5e),LL(0xa37c3c3b,0xb10d9d35),LL(0xb6d8d81a,0x6814e15a),LL(0x65739419,0x9282b5ea),LL(0xf890c6b6,0x80d764c7),LL(0x7d309653,0xda043f0c),LL(0xd070b1e7,0xc9e15f63),LL(0xb143ef10,0x31da4456),L_(0x0000010a), LL(0x47fd9b69,0x525c5453),LL(0xc9876e94,0xe52b9bcb),LL(0x3078ffdd,0xab8f2cfa),LL(0x7e1a8a45,0x338e4367),LL(0x3382f009,0x5b0092de),LL(0x06454df0,0xdb5d1cbb),LL(0xbbf3f2d3,0x17b40f75),LL(0xb1b7961b,0xc510c4b1),L_(0x0000018d), + LL(0x7aa9d1e5,0x71e12980),LL(0x69777935,0xdfdae6a9),LL(0x2f45dffd,0xf2900457),LL(0xabe3441f,0x6471580e),LL(0x5aece4d0,0xd983618d),LL(0x85cd6571,0xf5a1d861),LL(0xc2bd978c,0x955894c4),LL(0x81cae98a,0x762f897f),L_(0x00000078), LL(0x3176598c,0x4b7790dd),LL(0xc91def7d,0xb21152ba),LL(0x279d992a,0xd92aafdd),LL(0x7dab9fb5,0x59dd70d0),LL(0x7039956b,0x14d13b0f),LL(0x7cfe20de,0x7bd5d4c8),LL(0x45b10bc2,0x59724543),LL(0x906a5d0d,0xda689ccd),L_(0x00000096), + LL(0x80b9235f,0x380affbc),LL(0x0b4024a8,0xcd02b098),LL(0xc98a3c0b,0x36545f0a),LL(0xe62cf7ea,0x3b089c99),LL(0xdade8fc7,0xf9b4c955),LL(0xf404f355,0x7650e822),LL(0xae64ac11,0xcc0fb628),LL(0x593a1b6a,0x278ddfdf),L_(0x00000041), LL(0xa51f1936,0x091d5493),LL(0xeabe8135,0x5285ec41),LL(0x6bc185fe,0x9a7df8de),LL(0x25f54dac,0xc1b5836c),LL(0x4e5fc638,0xde102ff5),LL(0x81a60442,0x56a67f9d),LL(0xaeaff6ca,0x7cc5f40e),LL(0xdaecfbdc,0x63449fd0),L_(0x00000084), + LL(0xcd4c9f1b,0x64ba5d43),LL(0x262898c5,0xb6d345ac),LL(0x0f11b91d,0xf8015622),LL(0xfa53c395,0xeb32e3dc),LL(0x4e8f0647,0x125c49a2),LL(0x537471de,0x5c783701),LL(0x7a9741e3,0x00bc6a87),LL(0x1c5d3aa6,0x77962a3e),L_(0x00000003), LL(0x18eae31d,0x1336f732),LL(0x7555dff2,0x63097ec4),LL(0x8f16a8d5,0x928e41ac),LL(0xc063790a,0x72cf7210),LL(0x2842e2a8,0xbed4668e),LL(0xe7f0d214,0x7b5aab91),LL(0xbd94783e,0x472089cb),LL(0x55df6f3c,0x7e5f6d77),L_(0x000000b9), + LL(0x6ba36bea,0xd6462023),LL(0x51b0110e,0x0299f400),LL(0x5ad94e5d,0x1c56f2f6),LL(0xd8d6e619,0x0b4ea27d),LL(0xe73e18fa,0xdcfdac26),LL(0x61a026ce,0x27dab320),LL(0x4ebfdadb,0xb6af0729),LL(0xe9561c2a,0x2c10a6e9),L_(0x000000bf), LL(0x1418a240,0x7a5020b9),LL(0x2b5125f4,0x7dac2ce9),LL(0x6985bdc7,0x7a36a07a),LL(0x6d385362,0xc0a58550),LL(0x940163b2,0xd28cbf38),LL(0xfb9a9d22,0x8eddbcd2),LL(0x1cfcdeb4,0xad40ff84),LL(0x41da1441,0x0dfc69a8),L_(0x0000018f), + LL(0xf1ab6a88,0x32d8d898),LL(0xb4c956c4,0xbd9881d2),LL(0xbc516c73,0x5116a36c),LL(0x91e840e1,0xff4abf28),LL(0x14bf8bab,0x2bc617a5),LL(0xb012c75f,0xba5d811c),LL(0xf333effb,0x37bcddc4),LL(0x771a4567,0x0f294bff),L_(0x000001ea), LL(0xa48d6dfa,0xf2542b71),LL(0x495f434b,0xe3f39bde),LL(0x9f969883,0x179f2c63),LL(0xa68cdccf,0x44e28315),LL(0x7408c1bf,0xeb7b9849),LL(0xf6615345,0x823ede15),LL(0xdf405d5f,0x17f01e94),LL(0x0efd64e2,0xd1d65356),L_(0x00000094), + LL(0x6794bc7e,0xfa3914f9),LL(0x044623b2,0x3e94e3f2),LL(0xc7e42b96,0x85cc2a9d),LL(0xe0fbde7f,0x2e0e1f42),LL(0xde5ec740,0x0a1b4e4e),LL(0x99d96c6e,0xa3d7e876),LL(0x9b31d8b9,0xcce96a38),LL(0x4fd9fc85,0xaa4475dc),L_(0x0000016f), LL(0x5b7a322f,0x4317247b),LL(0x01f0266b,0xfd3d2a1d),LL(0xa9fbb760,0x75fc993e),LL(0x60905a87,0x51ce6740),LL(0x3c6c984d,0x16580d6e),LL(0xe79ccbef,0x585522ad),LL(0x57fa547e,0x42f80ccf),LL(0x8ad71acb,0x04c2d188),L_(0x0000000b), + LL(0xb16ee0ab,0x28366fe7),LL(0x0365fed1,0xc78aeb84),LL(0xf329889f,0xee74063c),LL(0x16267a5a,0x9c5a9197),LL(0x040a619f,0xc54b40cc),LL(0x16e7345f,0x9d5f609d),LL(0x957fc0b1,0xaf8fbbc8),LL(0xb1cb4d02,0xae0bc117),L_(0x0000016d), LL(0xd023f32f,0x1deda80b),LL(0xacb49a95,0x4c11c95e),LL(0x00e1f672,0xde8db891),LL(0x19ee52cc,0x27a5b5d8),LL(0x6176a0d3,0xd6752e2d),LL(0xdfccd883,0xe819161e),LL(0xe1282202,0xe71376c0),LL(0xb92fddf6,0x36e67047),L_(0x00000013), + LL(0xfecc3203,0x7168d714),LL(0x1726e41f,0xea5d7e70),LL(0x69dd518d,0xe7aec797),LL(0x2cfd3a6e,0x9ab3823a),LL(0xb90a5bea,0x43831c4a),LL(0xdbb35cc6,0xd5853b0b),LL(0x95c10ae1,0x55834ab5),LL(0xd668a3cb,0x91daa6d5),L_(0x000001e9), LL(0xc0f7662f,0x98346aa6),LL(0x173a0008,0xabe91f50),LL(0x0508c118,0x97cdd826),LL(0x7255d2e1,0x01d1e340),LL(0xeb07ccd4,0xf2f7ac53),LL(0xc6829327,0x966981ba),LL(0x0fb3f4fa,0x1ee6ae0f),LL(0xea212110,0x67b02691),L_(0x00000154), + LL(0x3c713d51,0x946494a7),LL(0x1e6130d8,0xbd5ab69c),LL(0x6cbf341e,0xcf57622a),LL(0x94d1578b,0xc7327897),LL(0xb1db7d17,0x2874559e),LL(0xf5607998,0xd15eab48),LL(0x18199bc9,0xfe1d9b44),LL(0xd0aedc11,0x95354256),L_(0x00000169), LL(0x50c646ed,0xb75609c0),LL(0x9721b326,0x4a7a787a),LL(0xed16025f,0xbf8835a3),LL(0x78dc9b8f,0xee0c8b9a),LL(0xf388850b,0x5a1458a1),LL(0xc5cee39e,0xb7bfeb06),LL(0xecfd0e49,0x8795b039),LL(0x70bdfc80,0x3f5f6711),L_(0x00000126), + LL(0xa3c85c78,0xbd49ba23),LL(0xe512fde9,0xdb860d3f),LL(0xbf99d4e9,0x3cb574a7),LL(0xbc4c4fcb,0x9af33dfa),LL(0x88710836,0xfd7ca4de),LL(0x4b83fe29,0xa2306095),LL(0x03d40abe,0x5f9ae75f),LL(0x5e83ab1a,0xe6d9a07e),L_(0x00000154), LL(0x97d0048f,0xf861736c),LL(0xde0ffdb3,0x5e0c4806),LL(0x335dce14,0x76e125cd),LL(0xcc1ff27f,0x16822cb3),LL(0x8414749b,0x44ae46e0),LL(0x68085d31,0xe89d0a2c),LL(0x9c734630,0x569b26c1),LL(0x2d89e8f0,0xc04e824b),L_(0x000000c9), + LL(0x1dc45be8,0xf3a736f8),LL(0xb7b948b5,0xf684853d),LL(0xa1d8a120,0x7de7af37),LL(0xe3760299,0xadb401c1),LL(0xa1ba866c,0x24e3e6ca),LL(0xcd94496a,0xc3eae9d6),LL(0x06f82a82,0x4a74e14e),LL(0x069d9983,0xb9dff343),L_(0x00000018), LL(0x2a60baf5,0x3855bae4),LL(0xfbb6aadc,0x20017824),LL(0x950ab7cd,0x9dfd2439),LL(0x96b97104,0x65cd1e13),LL(0x4925d16b,0xf2ec4dd1),LL(0xd82c0815,0x9059e2e1),LL(0x3ba03a79,0x2f723163),LL(0xf35a3c60,0xc7cb8c17),L_(0x0000004b), +}, +/* digit=78 base_pwr=2^390 */ +{ + LL(0x05eac10d,0x1d0d93fa),LL(0x3fb80a45,0x30dbe366),LL(0x69a67246,0x1373df55),LL(0x3a7b7740,0x3b688d58),LL(0x9a1bf576,0x6f4ce0d1),LL(0xc2030797,0x72348a80),LL(0xd874e749,0x24296b9d),LL(0x26fc8724,0xdf80dad7),L_(0x00000154), LL(0xa4626a33,0x0b206a34),LL(0x661fc19c,0x597e1cfe),LL(0x1f98c11b,0x010915e6),LL(0x00c41adb,0xbdcb2d64),LL(0xfed21859,0x07de82c2),LL(0xeffa2475,0xba1295da),LL(0x73a4f297,0x570842d6),LL(0x1fc083b4,0x94c92172),L_(0x00000159), + LL(0x05474c79,0xf27eccc5),LL(0x9bbd81b9,0x4061bf3c),LL(0x1385eed6,0xe5477666),LL(0x2d5cbcfe,0xa0f415ca),LL(0xab0a8618,0x18675f88),LL(0x47e7c454,0x0d5df7ae),LL(0x75324e3e,0xe2f88082),LL(0xe818ccf0,0x9cc4c5bf),L_(0x0000007e), LL(0xc2d8f452,0xef224de6),LL(0x9ce12031,0x70dcb7f7),LL(0xd958660b,0xc4597b18),LL(0x4dfd1c89,0xda0fa522),LL(0x2d69ce10,0x989afa82),LL(0x46e81b65,0x4e1b3fa1),LL(0xdcac5039,0x832ae217),LL(0x7d3eef41,0x7f7b4632),L_(0x00000159), + LL(0x5daa1b89,0x542982e7),LL(0x98f2622c,0x5b7e32c5),LL(0xa09068bf,0x6931e543),LL(0xeb5500e2,0x9089e794),LL(0x2a396afe,0xe5f72142),LL(0x68403342,0xf625b926),LL(0x14e9fc8c,0x6171d647),LL(0xe3e5f19c,0x41a98a6f),L_(0x0000012b), LL(0x9dfea1fb,0xaf413d88),LL(0x0a1d61f5,0xb4928484),LL(0xc08e23bd,0xe1c147cc),LL(0x3ba7bae4,0x0b6e0358),LL(0x7c12896b,0xa84a4aed),LL(0x0195f08d,0x7bcd8649),LL(0xeba99c7c,0x799d1ac2),LL(0x0193b278,0xa35f128c),L_(0x00000072), + LL(0x91f424e1,0xb2d18470),LL(0x482b31fa,0xc18abe11),LL(0x4548d244,0xc9e02a20),LL(0x901117bf,0xb1dc1cd2),LL(0x286c34a4,0x8ee98a6d),LL(0x04c325a4,0xb76b3ae6),LL(0xdb518aec,0x981c8298),LL(0xb7194b24,0x8c90f08f),L_(0x000001b3), LL(0xfd055d97,0xc7061a3b),LL(0x8a107a2c,0x5a667bf8),LL(0xaf0a63ac,0x21028dc3),LL(0x3e8641fb,0xa80d9487),LL(0xd40a558d,0x63f8d8c7),LL(0x3accfc09,0xccc46547),LL(0xfbd53079,0xe85ce362),LL(0x15006f15,0xb994381d),L_(0x000000d4), + LL(0x913326ba,0xe7e20db3),LL(0xe0e58c40,0x4dd3a381),LL(0x4f08867b,0x8c0db229),LL(0x1f57cf8e,0x862604e5),LL(0x9951f866,0x588c2473),LL(0xdd04385b,0x2a4f4278),LL(0x13235969,0x334e1277),LL(0xe14a7afb,0x0a02da7c),L_(0x00000164), LL(0xce866486,0x4c0e101a),LL(0xfa8ba6d4,0x8dee1016),LL(0x1cd14dc2,0x5e8be2aa),LL(0xcc56a3c9,0x5bdaa677),LL(0x26fe9020,0x787cbbf4),LL(0x6059a9eb,0xf27c2ccd),LL(0x52b29d1e,0x431ad6f9),LL(0xe9273bbd,0x47f193d1),L_(0x0000006c), + LL(0x531296c8,0xa4ea86fa),LL(0xe8dd3548,0x107a45cd),LL(0xd8b7e446,0x1bfd985a),LL(0x280a4e8e,0x4fc2dc66),LL(0x4b39f03c,0x86dfa832),LL(0xc44021ef,0x55d3ccb1),LL(0x210fe263,0x65b0f1db),LL(0x68364a82,0x4fc0acb2),L_(0x00000132), LL(0x2aec7e7e,0x7906edd3),LL(0x941505c5,0x5f4fddad),LL(0x657d2929,0x320073ac),LL(0xefaffd81,0x7ab9edd6),LL(0xeb40eb69,0x23d38d94),LL(0xc8df60f9,0x1840540c),LL(0xbfea59c1,0x3958b571),LL(0x2f7620df,0xcd1712e7),L_(0x000000d6), + LL(0xa8d54597,0x3ace77d1),LL(0x23823b9e,0xb05ab9ad),LL(0x7d6803d7,0x36f8db67),LL(0x8ed91e87,0x8fbc69ec),LL(0xe0042a07,0x6a246110),LL(0x5131b89e,0xf857a2db),LL(0x4c1861d6,0x311746a3),LL(0x84c00e5b,0xa2b64ab3),L_(0x0000008e), LL(0x95ff285c,0xdd895ec5),LL(0x9a709b4c,0xadcb32cb),LL(0x148a78f6,0x7c40e4b0),LL(0xf62708ee,0xd37d48a3),LL(0x68232a30,0x382492cc),LL(0x79b622f3,0x92d657a0),LL(0x09992500,0x6a420a8f),LL(0x501d8de9,0x759f5a62),L_(0x00000051), + LL(0xc122141c,0x66149a76),LL(0xc602d382,0xfdcc4f6e),LL(0x16b23844,0xea0673e3),LL(0x7ff0591c,0x53082d2f),LL(0x82264497,0xafeaf6b5),LL(0xca528c9c,0x01a84f6b),LL(0xe2403638,0x43738672),LL(0x54c4174c,0x50ff80fb),L_(0x00000186), LL(0x4205f340,0x0b6de62a),LL(0xa84853c5,0xc67ea2a5),LL(0x92e0d583,0x296584f2),LL(0x6bb7a441,0xa9f0cd97),LL(0xce6eb31f,0xd0d33dfa),LL(0x850b5886,0xca9b5dbb),LL(0xe75a3fb5,0x143bbd95),LL(0xfc35ee42,0x1eab0aa9),L_(0x00000157), + LL(0xfe747fdd,0x0c47e2c5),LL(0xf9ae16bd,0xa07b48d2),LL(0x01c347f0,0x2cf4c727),LL(0x747fa6ef,0x850ead03),LL(0xff312b98,0x8726545d),LL(0x5218270e,0xa6f367cc),LL(0x7680a110,0x37efad3c),LL(0xe34b12a3,0x693b75a8),L_(0x00000069), LL(0xe1fe8e14,0x5767922c),LL(0xe0242975,0x20a012f1),LL(0xba21fc56,0x49ff848f),LL(0xbd4ab80a,0xde1b4b0f),LL(0x24104f37,0x314917b2),LL(0x221548c9,0xde9c2909),LL(0x6f74cf12,0x75ca1868),LL(0x44d62839,0x6d64b7c2),L_(0x000000bb), + LL(0x3109d76a,0x044307f0),LL(0x7372da8e,0x9ba083bb),LL(0xd35f32f6,0xd25b7e53),LL(0x0a6c99ed,0x30794ede),LL(0x1523c414,0xed97a2a7),LL(0x7dbb6798,0x6d897d7a),LL(0xb996ba83,0x3c0356d5),LL(0xad5ef539,0x3f933449),L_(0x000000e3), LL(0x37a40ab6,0xfbb2d425),LL(0x468766c5,0xd03d4766),LL(0xe6a77a37,0x09eb18e1),LL(0x9d7f1644,0xbdb7ff41),LL(0xeeb2f1b2,0xc98d5a19),LL(0x8377c08e,0xda52ef67),LL(0x1078c0d1,0xbd2d3d66),LL(0x9181c3b8,0x0015c0fd),L_(0x000001f0), + LL(0x4f7f8d77,0x663aa882),LL(0x9e806c61,0xcd6d9ab0),LL(0x434a485b,0x575aafa5),LL(0x0477ae83,0x1fb377fe),LL(0x7b98c782,0x4ac83912),LL(0xe2f16d17,0xd098d31f),LL(0xeaff3b56,0xfba7077f),LL(0x13382e53,0x8ba7a6cb),L_(0x00000048), LL(0x4aad91ac,0xce1a9e84),LL(0xc8fc10e4,0xb01bef42),LL(0x6afd2473,0xb1a03e1c),LL(0x507c30ff,0x3760275a),LL(0x162b2e12,0xabb38d7f),LL(0x558877e8,0x88916de2),LL(0x9f519917,0x2fdccf2d),LL(0xf70b4695,0x1d641420),L_(0x00000025), + LL(0x4848296d,0xe0f71789),LL(0xf3835d1a,0x9fea01e8),LL(0xaf761a04,0xc7d32c83),LL(0x1060dc76,0x6c5ff4c8),LL(0xc3751915,0xec6a8a45),LL(0x6f1dbd4a,0xdf6166cd),LL(0xa64bcecc,0xa942edd4),LL(0x53c4d352,0x8df04873),L_(0x000001f2), LL(0x957c5d10,0xe757c12a),LL(0x60819db9,0x5aae5302),LL(0xfbc64c70,0x72a99d6e),LL(0xb00bbd49,0x3bf68d69),LL(0xf0b755c1,0x125fdc71),LL(0x4502c9bf,0xe7e83089),LL(0x7e20c9f2,0xaefe0993),LL(0x5fa7bc0d,0x58ce802b),L_(0x0000001f), + LL(0x596ae14c,0xc255b673),LL(0x1bb0ac11,0x19c21f40),LL(0x04f2091c,0x1838e69c),LL(0x3e9cb56d,0x51439522),LL(0xc1cecfc9,0x52a9b2f6),LL(0xc564467e,0x3576c345),LL(0xeb0fdcd3,0xd04ad0d7),LL(0x3c3aa2c6,0x1cfc7def),L_(0x00000038), LL(0x54b2e727,0x044d2e1b),LL(0x8701d001,0x794b02f7),LL(0xd8153787,0x4f463b8e),LL(0xa0dfa5ea,0x253e324d),LL(0x8fc87618,0x19c78239),LL(0x6165c650,0x97b77e54),LL(0x134c2cb2,0x5e23fc18),LL(0x5ade1fce,0x10c19f2a),L_(0x00000174), + LL(0xaad903e8,0xb407a7f0),LL(0xa64c2d6e,0xc577e3d9),LL(0xd68527c3,0x398177a5),LL(0xdf38bff9,0xde6b9d9e),LL(0x746c3309,0x1d7385a5),LL(0x98512661,0x3b8d9701),LL(0xe82c0d22,0xe2b7edd9),LL(0x16ce165c,0x92301ae2),L_(0x0000002f), LL(0x491b562d,0xd2049870),LL(0x1de2def6,0xcf01bb60),LL(0xaff2152d,0x20717620),LL(0x5764198c,0x503a4861),LL(0x9702cec2,0xce53ec1b),LL(0x886169de,0xe97c752c),LL(0x8d58887b,0xeb1ce735),LL(0xe1b8ee6d,0x01ad85ca),L_(0x00000093), + LL(0x6213a0d1,0x1d410b7b),LL(0xba1aa221,0xa8aa0a76),LL(0xbd669af8,0x06a3140b),LL(0x196f88e0,0xdc333cc8),LL(0x1c9aa099,0x60557f79),LL(0xe1ff46ae,0xa7b7da68),LL(0xd7626d62,0x2a5a72d1),LL(0xffb47531,0x86e04ed1),L_(0x00000058), LL(0x72d98927,0x8e521d9d),LL(0x6acd3ed1,0x855c8e97),LL(0x47c48377,0x42413164),LL(0x89182c42,0xae6a473b),LL(0x59d11d10,0x730eb702),LL(0xd4abd611,0x71fd7c4f),LL(0x4d329537,0xe8493373),LL(0x5e745909,0x0caebd43),L_(0x00000034), + LL(0x4c6f3e4a,0x688f6e0a),LL(0x2a9b6d1a,0x9e0c6a62),LL(0xfebdd4b3,0x7fb92759),LL(0xe14d66ca,0x3d698e7d),LL(0x0a2edb8e,0x08ea4d11),LL(0x10b21ab8,0x0f855706),LL(0xf8405b08,0x104c29e8),LL(0xadf7ff63,0xca76d27d),L_(0x00000076), LL(0x4757c2f5,0xe671ed88),LL(0xb38018e2,0xd5288875),LL(0x629331d9,0x279d96a8),LL(0x7e602196,0x559eca5a),LL(0x81f2dc09,0xf274c146),LL(0x96818fe5,0x483d0424),LL(0x1c8246f6,0x76f5286b),LL(0x44ba1052,0xe83b7779),L_(0x0000005e), +}, +/* digit=79 base_pwr=2^395 */ +{ + LL(0x42d44166,0x0a8f4b54),LL(0x924764d2,0xf7bf31e5),LL(0x26e4af60,0x7be28350),LL(0x9fc53bdf,0x08056835),LL(0xa12726ea,0x726e147e),LL(0xd5e175b1,0xca2a0207),LL(0x833e5911,0x4d322cae),LL(0x6cba51a1,0xfd137e93),L_(0x0000001e), LL(0xdaa1d653,0xe0948dd2),LL(0xca2b565c,0x2f88e2ac),LL(0x94ccb3a4,0x85d91e6c),LL(0xd8fe4775,0xb59cebc7),LL(0x0807f46d,0xf4237821),LL(0xfc9eb940,0x402ecff0),LL(0xd173f94e,0xbf7bc598),LL(0xaf975145,0xf726ff16),L_(0x0000009c), + LL(0x15942a4c,0x3dad29e1),LL(0x350dec67,0xe6be55a2),LL(0xca3c399f,0x71245659),LL(0x87e22652,0x8f51c63c),LL(0x1bd4c445,0x758ae1a1),LL(0x319b57db,0x547db810),LL(0x7d5c89dc,0x62c8ba84),LL(0x959a5bbe,0xd3fcf239),L_(0x000000eb), LL(0x8c3d490f,0x482b690e),LL(0x50d48521,0x83aada08),LL(0x82c13331,0x45c4a535),LL(0xae5a3425,0x31c1467b),LL(0x51c50d6f,0x2d093b81),LL(0x84a7d97c,0x82d6fbdb),LL(0xdb41ffbe,0x4953468e),LL(0xae0e9fad,0xede049e6),L_(0x00000070), + LL(0xc26f9e1d,0xb8562be5),LL(0x71fd7b5e,0x039f5b0d),LL(0xe3196121,0x073e4db0),LL(0x902cc367,0x22f8b999),LL(0x604d9b78,0xc1cfc4a4),LL(0x52f26ece,0x88d45487),LL(0xa175f394,0x9f16c268),LL(0x9fe9a65c,0xe88e2aea),L_(0x000001a9), LL(0x54ed1ed1,0x4d516f86),LL(0xb3a46011,0xa9ca3304),LL(0x8c5e216d,0x2030c9a9),LL(0x86a5904f,0xc0c4c573),LL(0x9467bf24,0xb4a6fdae),LL(0xa652162c,0xd3472536),LL(0xb6166589,0x31361b4b),LL(0xfa3bddac,0xd5b8b3b7),L_(0x0000007a), + LL(0xaa60773f,0xe8255739),LL(0x58459ee1,0x672547b1),LL(0x3f4e65fb,0x0030639d),LL(0x5059b89e,0xc29a4f63),LL(0x3ffb7b9a,0x267e1823),LL(0x6854cad3,0x7f7d6bbb),LL(0x79c3b99a,0x20d91b6e),LL(0xbec22c47,0x4474e65e),L_(0x000000d8), LL(0xd72fbb0f,0x307d9b73),LL(0x7f8c46dc,0xc6fa2f52),LL(0xfbf3d06f,0xd62e537c),LL(0xd7888122,0x4ebd3818),LL(0x11ea61a8,0x479a6f83),LL(0xeafa5532,0xa4bf3325),LL(0xc7a31554,0x5a717809),LL(0x9520a809,0xd00e3039),L_(0x000000d7), + LL(0x4f775892,0xb974b129),LL(0x5ce12c3e,0xe8064395),LL(0x2e69ed62,0x31e3ae39),LL(0x9c220c3b,0xd59ea4b1),LL(0xb79732ea,0x6f45918a),LL(0x6600e529,0x08681af2),LL(0x4ed3f5dd,0x507438a2),LL(0xac5ccefd,0x0ca9586e),L_(0x00000064), LL(0x5b14db7c,0x2484d059),LL(0x411826aa,0x6005d933),LL(0xca8c9d7f,0x15dd0c44),LL(0x1900212d,0x96825f72),LL(0xa3ea7e14,0x921c8c87),LL(0x20d2b6f4,0xfdd63f04),LL(0xbe8c25ac,0x9f6a6126),LL(0xa2592316,0xf8413f28),L_(0x000001e7), + LL(0x27b80789,0x4f60fc89),LL(0xc97bbd7e,0xbd2ecac1),LL(0x5b008578,0xaaf75b36),LL(0x3b54d53e,0x6540b6f5),LL(0xce5a0ab8,0xac54c1d9),LL(0x52d909c6,0x67e6b65d),LL(0x6607ea02,0x7ea06112),LL(0x0a94e0f9,0x55cce11c),L_(0x0000011e), LL(0x7fdd52c7,0x49bb24e9),LL(0xfc9b8e68,0xe60101c7),LL(0xdcbe3ce9,0xc2ae52f0),LL(0x3c779930,0x85f01602),LL(0xa50df581,0x4a7c2cfe),LL(0x8c89a686,0x52f720dc),LL(0xe2d3b3fd,0x566aecf1),LL(0x1ac6cabd,0xf58c8fe5),L_(0x0000000e), + LL(0x1a616c59,0x8c145c49),LL(0x1dbffc3d,0x2990c86d),LL(0x842bad94,0x7914907c),LL(0xb0b65826,0x8753b549),LL(0x09f19586,0x29d809d0),LL(0x3f813f8e,0x3e155797),LL(0x293bf471,0x87d08f2b),LL(0xc41c49e5,0x03a8d0a9),L_(0x0000013c), LL(0xd9473ed1,0x7fc8f692),LL(0x7012e2ef,0x5014f2ae),LL(0xe7047394,0x086b300a),LL(0xb63d8e90,0xe77f5c2e),LL(0xf739fb33,0x700eca1b),LL(0x22cfcea2,0x8bf8652d),LL(0x4bb3d357,0xe61f7839),LL(0xba0c3888,0xd370df89),L_(0x0000010e), + LL(0xe42f4734,0xbb740324),LL(0x9909bcd8,0xe45bfed8),LL(0xe9550616,0x77889a7f),LL(0x861f9828,0x446a53b7),LL(0xc8946a71,0xfbe6cdbf),LL(0x72916502,0x70373dcf),LL(0x59c8ba5f,0x1ab59a0e),LL(0x07606ee3,0x82f754ed),L_(0x0000003e), LL(0xfa3f237a,0xb5ddd50e),LL(0x0f211f61,0x56f3a63a),LL(0x8319e664,0x3124dae6),LL(0xe5acc6a3,0xf2c7d090),LL(0x5d6c5fe3,0x5cc8df62),LL(0xa0165f43,0x20f4229f),LL(0x5adf8ff4,0x2e9c92c9),LL(0xd951affc,0xefce2c03),L_(0x000000f9), + LL(0x85b95034,0xdf5cb44e),LL(0xffda53ce,0x976139f3),LL(0x126f8dfa,0xe003499a),LL(0x25a0d493,0xb521d994),LL(0x43bb5822,0x4978624a),LL(0x1b4bdd9b,0xf19f8465),LL(0xb8e6e89b,0x51953414),LL(0x3b559f26,0x86ef1c70),L_(0x00000051), LL(0x67904448,0x23ee72a6),LL(0x7f3ba24c,0xefa5df96),LL(0x4a910e6e,0x2d36ae87),LL(0xbee250f2,0xa3310ea5),LL(0xbebd708e,0x5db14894),LL(0x94f849b9,0x7b2cb1b7),LL(0x830487c0,0xd8532e27),LL(0x72d6bdee,0x37fccdcc),L_(0x00000160), + LL(0x098e856f,0xcfb4864e),LL(0x2a48dc38,0xa9b6c8ef),LL(0x070c9954,0xc3e565ee),LL(0x47d25c31,0xeec5d7b2),LL(0xdd383653,0x5ac0ce45),LL(0x1f5f3381,0xd40035f3),LL(0x486c7281,0xb516fd77),LL(0x5bbe546e,0xc646e214),L_(0x00000034), LL(0x0df8a00f,0x3f432f2f),LL(0x740ff679,0xa22b5ed0),LL(0xff907935,0x3d446e71),LL(0xd013a668,0xeeee1f6e),LL(0x16c4dc5f,0x37116783),LL(0x8ed6e49d,0x07fabe84),LL(0xd710493c,0x29dbccd8),LL(0x9263e99e,0x6beb1393),L_(0x0000012a), + LL(0x906df6cd,0x835e52c6),LL(0x8a2f7445,0x66d4156a),LL(0xb891e801,0xaa5c0324),LL(0x852e9f5b,0xd3f81a7a),LL(0xce23b51a,0xcdc49377),LL(0xe9c8bbe9,0x7da33db2),LL(0xd5ae561f,0x90aafd3d),LL(0x2f82d67e,0xf365be46),L_(0x0000003e), LL(0xd53059fe,0x43cd9246),LL(0xca05f22d,0x0c50a7ad),LL(0x23745904,0x612405ce),LL(0x1e2b644e,0x1783439c),LL(0xbb5598f4,0x8299ab36),LL(0xfdba24f0,0x36c6a428),LL(0x085bc781,0x42e84aa9),LL(0xe954acbf,0xeabc6123),L_(0x0000001a), + LL(0xeb9123d0,0x9813e058),LL(0x170c410c,0xb14ecd8d),LL(0x43a445ef,0x12caafb5),LL(0xb13251ba,0x692c7666),LL(0x6a58ee44,0x93734f73),LL(0x1ecf9f41,0x0ff39b6d),LL(0x582e2c3a,0xdd49d16c),LL(0xbc874e11,0x95e54a54),L_(0x0000016f), LL(0x9986c142,0xbe161bed),LL(0xadb85be6,0x94c11992),LL(0x974392b1,0x4727d02a),LL(0x1355dd0b,0x9ef84fdc),LL(0x001779ff,0xdbbcd1c3),LL(0x09218dc1,0x615360f7),LL(0xce2bff57,0xa1cd90ce),LL(0x5b2b4772,0x3bd7ca08),L_(0x0000005e), + LL(0x797035ff,0x63db3368),LL(0x2ea4d5db,0xd58aac93),LL(0x5c1f4e24,0xf46d30bc),LL(0x87b8b188,0xfe246192),LL(0x177c783a,0x2b7ec253),LL(0x7157d89d,0x93dd18f3),LL(0x609635a4,0xebb06e2d),LL(0x9bb84085,0x23701182),L_(0x00000077), LL(0x5291bba2,0x68aae159),LL(0xd5d8d4a8,0xc3c56f74),LL(0x8f01b0f8,0xd20ce087),LL(0x0018bf4f,0xcf62fd4a),LL(0x58c66e12,0xe583beb9),LL(0xcba992f3,0x7c22cd07),LL(0x82ea861b,0xffb459bf),LL(0x757b22f8,0xef69f10c),L_(0x000000f2), + LL(0xf9bac8ff,0xbeb1c7a5),LL(0x70f6e41f,0x32bb744d),LL(0x1f50a7aa,0x2a66377a),LL(0x611c61b4,0x324a6008),LL(0x1ce936f3,0xc3dc201c),LL(0xc5bca084,0xb70db700),LL(0x47988c82,0xfef0caf3),LL(0x2dfe3675,0x6397725e),L_(0x00000193), LL(0x76fee427,0x165d8d1c),LL(0x937f269a,0xdc1fbe1e),LL(0xdfee35cc,0x745db180),LL(0x5475d4c9,0xefc6cdb5),LL(0xb36d0a2f,0x26ccd4e8),LL(0xf7ab4767,0x3783e580),LL(0x31ff3c1f,0xce5d079b),LL(0xed18ab19,0xf284909c),L_(0x000001d3), + LL(0x231ac4a1,0x0790eab4),LL(0xec50bbe6,0xde5da610),LL(0xc5c0bffc,0xc2382715),LL(0xd201e1fa,0xaac50e22),LL(0x27c0dbe5,0xd06ae9b5),LL(0x8715ddc4,0x77f1683f),LL(0x592ddcce,0x410624c0),LL(0xffb7a12c,0x170b0367),L_(0x0000001c), LL(0xf76139b5,0xf8b9348f),LL(0x29844cbe,0x30deae53),LL(0x941fdd94,0x5f88797a),LL(0xa3612765,0xbbb4a54a),LL(0x8e2ccc8e,0x4a469e09),LL(0x47a174f2,0xe17602e5),LL(0xc6f2b50b,0x5edc6700),LL(0x887e7ea0,0x8c90c4bb),L_(0x000000e7), + LL(0x350806a0,0x8c8ad937),LL(0xbc207b36,0x45322006),LL(0xf3feb2a7,0x29b356ce),LL(0x9772041b,0x4db4360d),LL(0x71532653,0xffd40033),LL(0x62b4c3a4,0x525bb0cb),LL(0xef8cba94,0xb23dfc1a),LL(0x5f95e7e7,0x4230723c),L_(0x000001ff), LL(0x3a9f0d5f,0x05623688),LL(0xe7ca479f,0xe0f2e02f),LL(0x821e669e,0xd9320a75),LL(0x3208dc12,0x2a90f1fc),LL(0xac95bd4d,0xb836d5d0),LL(0x0f15127a,0xa56f1f5b),LL(0x35a8c806,0x4b53533b),LL(0xb5d6ee48,0xfcf4ee1d),L_(0x00000155), +}, +/* digit=80 base_pwr=2^400 */ +{ + LL(0xb1b291af,0x63480c29),LL(0x6afae0d2,0xec906027),LL(0xd90afac7,0x067489d0),LL(0x62d3da37,0xe31b78ec),LL(0xd88f38b3,0xff5fafe7),LL(0x3dfa7f35,0x88536101),LL(0x7a6237d7,0x80270f89),LL(0x42a2eac9,0x39890c2a),L_(0x00000064), LL(0x95b6527c,0x38b21960),LL(0xab3e28c0,0x56c8573a),LL(0x49dbf002,0xb06c2993),LL(0x016fc238,0xc550c59c),LL(0x7c26c63d,0x22cb4395),LL(0xd6c87128,0x4a5765b7),LL(0x42f34ea4,0x360be87f),LL(0x9436ed50,0x331a061a),L_(0x000000d2), + LL(0xcf6c0f3c,0x23b6d0c7),LL(0x01323c15,0xb2a524b7),LL(0x445c4f05,0xfb280e1d),LL(0xb721bd24,0x43a450cf),LL(0x34d74f50,0x459a3690),LL(0x9ed3c3ca,0xf8a99776),LL(0x9bd35cd1,0xaf456934),LL(0xa1b94559,0x00820fa7),L_(0x00000124), LL(0xc7479e9f,0x93e5dd5a),LL(0xfdaf1391,0xfcfc382f),LL(0x727251e2,0x4776e937),LL(0x9f976e0e,0xfaf93681),LL(0x7e0dea37,0x18ec38c9),LL(0x45662b32,0x0308bb26),LL(0xd581e3f2,0x1a441534),LL(0xc275dc07,0xa1b0d046),L_(0x000001b4), + LL(0x9bcb5968,0x702c5bb8),LL(0xdfb47a52,0x33f897ad),LL(0xe16c51a7,0x078f030c),LL(0x5e8bc092,0xb9a4c194),LL(0xa0a224a9,0x0d2a2dbe),LL(0x244c74fb,0x00b01506),LL(0xef3b3eda,0xf4403180),LL(0x44f09c72,0x1da590d5),L_(0x000001c4), LL(0x4a7f0289,0x3c0211ff),LL(0x88323d80,0x1ffe93b1),LL(0x81a127a4,0x7deb5031),LL(0x0bd65111,0x65ffd296),LL(0xff238b15,0x80cef133),LL(0x643d7062,0xddb33d18),LL(0x93ccc6e4,0xc957a463),LL(0x1fcc4678,0xc36fd11d),L_(0x00000184), + LL(0x1ccb1806,0xe0e23232),LL(0x5eae4904,0xc2c362eb),LL(0xda675b34,0x2c14f20a),LL(0x13d2fa91,0x60f4ae95),LL(0x6c8c7ff9,0x78df2064),LL(0x6790ea32,0xb702cc14),LL(0x7608da34,0xe2b87bb5),LL(0xadae0fb9,0x6d939f1f),L_(0x000000fd), LL(0xa26a843b,0xc89f77d5),LL(0x5a368ffa,0x265d14c3),LL(0x957c89a9,0xbd1486e7),LL(0x514b7e05,0xa9030ef9),LL(0x537cf3d5,0xb9ea3998),LL(0x4fb32008,0x0c45cfba),LL(0x61ff9565,0x078d5a15),LL(0x1cc6a564,0x4f509bc0),L_(0x00000019), + LL(0x665bb52e,0xf7ce0ec3),LL(0x69d2fe28,0x4c059fd5),LL(0x76b354dc,0x1290e892),LL(0x674b639e,0x6d828313),LL(0x951c9220,0xd6285250),LL(0xee815fd8,0x3a0ba16e),LL(0x8ee38518,0x5323ff40),LL(0x6678fced,0x58fd9e73),L_(0x00000153), LL(0x73a8b28d,0xba84ecb6),LL(0xbd048216,0x34998afb),LL(0xfb264967,0xfcd4e06c),LL(0xc024c958,0xd668c764),LL(0x9c3e07fd,0xee500455),LL(0x0ea9e902,0x7be48424),LL(0xdf78504c,0xb185d1cb),LL(0xc315ffe9,0xcfa4801f),L_(0x00000174), + LL(0xdb3f67e6,0xdbdf25d6),LL(0x23503a7e,0x4a86faf8),LL(0x58cbd82a,0x727561d4),LL(0x50ad1fb2,0x0994b8fd),LL(0x1d57e2f0,0x1b2b4725),LL(0x15736a57,0x5dcba3cd),LL(0x4df58192,0xec335163),LL(0xf716e579,0xa8c09f90),L_(0x000001aa), LL(0x3615c741,0xe4509eb4),LL(0xef754782,0xbb1dede8),LL(0x7a793f6d,0xa02e32d8),LL(0x27d972fe,0x00b65ecb),LL(0xd0af4ace,0x80ede0c9),LL(0x96816659,0x6c809dcb),LL(0x979a653f,0x943f6f1f),LL(0x0638c8e6,0x2bcc5fa7),L_(0x000001fa), + LL(0xc3695735,0x79907894),LL(0x6a94274c,0x923a4f54),LL(0xa698895f,0x6213c148),LL(0x57afe3d1,0x14eca3c3),LL(0x40597be1,0x57638ac1),LL(0x23258bde,0xb1f30c4d),LL(0xb9d09cea,0xe2c1e648),LL(0x544e3974,0x505f2b5b),L_(0x000000d3), LL(0x28a45f39,0x0111ad40),LL(0xbe28f874,0xc3e262fd),LL(0x830a9ee4,0xe3cc3453),LL(0x0fdaa4f4,0x044defae),LL(0xcdb8b9c8,0x06665f64),LL(0x4a06827b,0xb0bfa5e9),LL(0x926f3364,0xaf288ab3),LL(0xd9d3c3ec,0x52d7095c),L_(0x0000005a), + LL(0x5eb181f2,0x047db30c),LL(0xbb73f9ac,0xf0b9977a),LL(0xc5355db3,0xa26685fe),LL(0xa356655d,0x48c1a2cb),LL(0x950bb7fd,0xc2c2dc45),LL(0xe766094a,0x437ff8a8),LL(0x80146ca2,0x0b26425d),LL(0x5b7aafe5,0x20ce1d21),L_(0x000001e4), LL(0x1c8a54b7,0x683c3c46),LL(0xee244200,0x806c758f),LL(0xb90af5d8,0x71dee02f),LL(0x67ac6f65,0x53aeab51),LL(0x0f1dfed2,0x762d7338),LL(0x59bf51fc,0x059d5565),LL(0x1432973f,0xa9143049),LL(0xbf133e93,0xc1c8977e),L_(0x000000e4), + LL(0xa86b08e9,0x4f81ab77),LL(0x4d06707c,0x8f1bf925),LL(0xaaec66fa,0x4de91bfe),LL(0x49683907,0xb9d75b89),LL(0xac105bac,0x91f21593),LL(0x6506b77d,0xab4390d7),LL(0x7287e5ee,0x7487ed45),LL(0x6c88b2c6,0xbb3803c1),L_(0x0000006a), LL(0x11c071b8,0x97c81ddb),LL(0x1bbbfe37,0xd3ad9501),LL(0x7a1b3885,0x68eaad09),LL(0x7027528c,0xf0a64aef),LL(0xcf860284,0x3c25fb62),LL(0x37b3f120,0x5ae492f1),LL(0xd454de89,0x61c7d5ed),LL(0x02ac32db,0x9d99d9a3),L_(0x000001a1), + LL(0x245c4af0,0x97406073),LL(0x02c143ac,0xda94e70d),LL(0x1c3ce7f2,0x639942c8),LL(0x9cde176b,0x7bd79b38),LL(0x0883b19a,0xac9d2c91),LL(0xb4709f2a,0x4423fc39),LL(0x794a6aaa,0x89d178db),LL(0x2e0aecc8,0xf4ffbcc4),L_(0x00000075), LL(0xb872cf73,0xd957093e),LL(0x85b4776d,0x2acbd72e),LL(0x0656f9a4,0xd468fb6e),LL(0xe2017205,0xc79f3d11),LL(0xe6a367a5,0x012b1362),LL(0x73088cac,0x370e3947),LL(0xb86f1933,0x3bdf96c5),LL(0xe8162ca6,0x5d8a34dd),L_(0x000001ac), + LL(0x9b09e656,0xeb6d1849),LL(0x81aa411a,0xaae86725),LL(0xd8ee02c2,0x3008becf),LL(0x574081d1,0x53cb17df),LL(0x37bd5637,0xe3afcfbe),LL(0x116f90cd,0xe91cbaad),LL(0x9a6da94d,0x023933b7),LL(0xbabdb470,0x5562808f),L_(0x00000086), LL(0xf3d722e1,0xef4074df),LL(0x2345ac8e,0x1492cda0),LL(0xe92886d5,0x0fa0e7ed),LL(0x31537a63,0x0778badc),LL(0xa8b49a51,0x2034669c),LL(0x85153992,0xace88cb9),LL(0xfd679e52,0x4c6fbbf5),LL(0x48dc5511,0xa37bc767),L_(0x000000e6), + LL(0xe7ef2e60,0xcd7037d9),LL(0x52aa0cf6,0x7dc7381f),LL(0x24da2a63,0xc3562613),LL(0x43cb3857,0xfa06da1c),LL(0x5a1b2f21,0xc7806190),LL(0xec8d6d1e,0xf2d57ff1),LL(0x5a0e8065,0x3e9088a1),LL(0xc30f4ccc,0xfbce5c5d),L_(0x000001c0), LL(0x5912141f,0x1bc58b8c),LL(0x19ee91c4,0xbf2996d5),LL(0x41d465ca,0xb39d820d),LL(0x281052af,0x890d83a9),LL(0xa8cb9903,0x32dbf7a5),LL(0xbfad3855,0x309c32fb),LL(0x81d1982b,0x44dec521),LL(0xe04b5e8d,0xbcfee0cb),L_(0x000000bb), + LL(0xf141f663,0x6c39f9cb),LL(0x458b556c,0xea205dcc),LL(0xb6c66d04,0x96d440e6),LL(0xa0f1a8e6,0xd4e4d37e),LL(0x3d5c97ee,0xd69c9913),LL(0x92914c77,0x12d04cdd),LL(0xb8bb2a33,0xa42c1185),LL(0x2dee2451,0x66637684),L_(0x00000165), LL(0xe8feb727,0xf2ee4afa),LL(0xb371de96,0x0f971611),LL(0x605c486d,0x50e27936),LL(0x313e3113,0xdf801ab6),LL(0x7cfe0fbb,0xfbec3f73),LL(0x50864708,0x55057cd2),LL(0x3b426634,0xa56700f5),LL(0xe23875b8,0x34b766de),L_(0x0000011e), + LL(0x923016ac,0x17eaa1a0),LL(0x28e3f935,0x128b1c02),LL(0xa23b14eb,0x9183c2c0),LL(0x504289c9,0x5ea313d0),LL(0xb5fd0ea2,0x135c854c),LL(0x03f5e8df,0x491bfca7),LL(0xd9abdd7a,0x8b6af733),LL(0x29d5f4e7,0x832fe74e),L_(0x000001e5), LL(0x50113295,0xb89fd770),LL(0x861c50a9,0x6fd08a16),LL(0x1b7e4c60,0xb1a9800a),LL(0xadc9902a,0xbed31f30),LL(0xa5b9d65d,0xc9a0a6d4),LL(0x75c10264,0xe1743115),LL(0xc317e935,0x8c13233d),LL(0xc0c350f5,0xbc15e8d2),L_(0x000000a6), + LL(0xfdaa94e2,0x4ea46af8),LL(0x2200fec9,0x3139256c),LL(0x4f2fd88a,0xb9b83d67),LL(0xe894a212,0x5e6f7bed),LL(0x449c5bc8,0xbeebbe0e),LL(0x1e599f77,0x74fc58a0),LL(0x7f4a9123,0x6073c6cf),LL(0x1a08d5bf,0x63816113),L_(0x0000017a), LL(0xacb45b52,0x06826375),LL(0xbd9c13b3,0xf41e6cac),LL(0xc5b15016,0x79237672),LL(0x2f2bfd3b,0xa99e108a),LL(0x1681dcfd,0x22fd3033),LL(0x224f0276,0xc24a2525),LL(0x05e14660,0xf9a0ff9c),LL(0x09f9360e,0xbe09e3fb),L_(0x0000005a), + LL(0x1e2ff226,0x33f17650),LL(0x597f6cd8,0x95759918),LL(0x5f1a7fb7,0xb518de41),LL(0x316d41dc,0x966644e4),LL(0x829d89c6,0x7775eaca),LL(0xed7a513d,0x9d45da4e),LL(0xdd14ef8a,0x232852aa),LL(0xc016fff6,0x4687f62e),L_(0x00000088), LL(0x0791f95c,0x2cc46d0d),LL(0xd4641a73,0x04775f23),LL(0xb9dcdfdd,0xc9bd9d15),LL(0x727ace99,0xaec6f67a),LL(0xa77fe3e5,0xeac1ee2f),LL(0x75b923be,0x8f21d632),LL(0x55852cf0,0xb6658853),LL(0xbe6d4550,0x461bace1),L_(0x000000f6), +}, +/* digit=81 base_pwr=2^405 */ +{ + LL(0x2c6e3212,0x0a1fff24),LL(0x4b5a6256,0xa45d5589),LL(0xbc1ece0d,0x852bc8f1),LL(0x655945af,0xf4152e99),LL(0xf81d51a6,0x4573b7d9),LL(0x15c74818,0x69e42e80),LL(0x69dba53f,0x55b2c206),LL(0x96245123,0x8c3b8e9f),L_(0x000000a3), LL(0x11c96019,0x8a5146d0),LL(0xc8d9fabe,0x6c8e83c7),LL(0x3fede45c,0x3d5d33f9),LL(0xb87d2fad,0xe1fe306f),LL(0x67a48456,0xf030c243),LL(0x2aa7e6e9,0xb8f59e2c),LL(0x8097392a,0x7ae2ecee),LL(0xdbed7e8f,0x770cdce7),L_(0x000000aa), + LL(0x8efeaf2d,0x8c7ed96e),LL(0xf3e2fab6,0x3cb959b9),LL(0xfe65989c,0x4cec2e34),LL(0xb397dfd2,0x2a821089),LL(0x95a4f7a7,0x194fcfc2),LL(0xcd183d4d,0x009eac36),LL(0xc2005f34,0x384df54a),LL(0xc4355ce3,0x7125837a),L_(0x0000003c), LL(0x6218f15a,0xab158f0c),LL(0xc9db684a,0x919d3c1c),LL(0x22157c5c,0x733c654f),LL(0xa5d7e7c5,0x1bb67f61),LL(0x6dc89cd4,0x0cac1f78),LL(0xf6e74669,0x2b55f183),LL(0xb445fa4f,0x9df41e4c),LL(0x69c4dd42,0x100045a9),L_(0x000000a5), + LL(0xd8d9bdcb,0x3dd4ca53),LL(0x1306c74d,0x1af7d8d5),LL(0x3e680d58,0x9c6b82f0),LL(0x884ca0be,0x0aacdba7),LL(0x5c62e372,0xd633f595),LL(0x1c4cad9a,0xc84d067a),LL(0x54e3c550,0x4fe24eee),LL(0xbe3f67b5,0xf28e319d),L_(0x0000019e), LL(0xb999b839,0xd75cb7b6),LL(0xa5275bc7,0x5e6b4aa8),LL(0xfaa9f40a,0xe6b156cd),LL(0x1992d1c2,0x16e51f4a),LL(0x0b180928,0x00c94afd),LL(0x6b3427a9,0x0f9d0fb1),LL(0x09eefa51,0x098f98b4),LL(0xd3cae463,0x4d73daae),L_(0x00000180), + LL(0x881adc31,0xbb3a93b6),LL(0x8ce1cba5,0xb603dd9e),LL(0x4f5b70c1,0xaed8b0cf),LL(0x5f958dd3,0x5eae2517),LL(0xb70f44e1,0xa5b942f5),LL(0xc526177a,0x02efb949),LL(0xc8dd1153,0x132ba3a2),LL(0xf9288a95,0xffccbb89),L_(0x000001a2), LL(0xc45972eb,0xbc841aed),LL(0x7853d5b1,0x933a99b8),LL(0x6e1536aa,0x85259727),LL(0x238abf3c,0x05488fa0),LL(0x8485ab11,0x1debe07d),LL(0x6d6f6d52,0xf1ad18f1),LL(0x54637f92,0xdfd3c55a),LL(0xa2b58773,0xc41b6804),L_(0x000001db), + LL(0x66d98564,0x72366f09),LL(0x9a3f0f97,0x00008259),LL(0xb1a9b87a,0x33f3b0e5),LL(0xe8074b36,0xd83471a9),LL(0x68f935c3,0x59dc097c),LL(0x5ab59d2f,0x049d3329),LL(0xae3c2a44,0x523ad362),LL(0xd39de2e5,0x38178eb0),L_(0x000001da), LL(0x10a229a9,0xad11b358),LL(0xdcf6cbc9,0x8a7d993d),LL(0x2d5c5b91,0x31b67dc1),LL(0xd2d684f6,0x5dd81c8d),LL(0x29c17938,0xec292f8a),LL(0x2fb94c2a,0xe9c267eb),LL(0x67c899b3,0x31e831bd),LL(0xd72dd6a7,0x7b778a58),L_(0x000000ab), + LL(0x37a99dda,0xabc73772),LL(0xb8a9bdaf,0x18907ad9),LL(0xac790211,0xd35c8ab3),LL(0x56550490,0x483d71b5),LL(0x9c473d52,0xaed32863),LL(0x796ddfe4,0xc175ce1c),LL(0x39329661,0xf0af8692),LL(0x3411279a,0x1ac1b3f3),L_(0x00000062), LL(0x2f981ba2,0xe7ed779e),LL(0x25706bd9,0x385062b4),LL(0x8826d6b1,0x50749b03),LL(0x8f92597e,0x4d1f3b1e),LL(0xcc7ec8dc,0x5ffacc7e),LL(0xc4c11580,0x903de537),LL(0x82ed5c34,0x92e3ccb2),LL(0x829a6dc1,0xaa91f997),L_(0x00000010), + LL(0x8a8271c3,0x46e669a1),LL(0x9bedd70e,0x6dbeb99e),LL(0xe5038aaf,0xc8d58c8e),LL(0xe202e790,0x312f9e8d),LL(0x5eb0a99d,0x2b3b3990),LL(0x6e033ac6,0x8ddb53b5),LL(0xdd9938af,0xcad94c88),LL(0x8fee9f14,0x565b4703),L_(0x00000184), LL(0x2d18c7f7,0xf6c9ab51),LL(0xddec3950,0xd2e14fab),LL(0x8eb24aaa,0x85f6d87b),LL(0xbff04dfa,0x7d46acbb),LL(0x458d7c7a,0x949d94f0),LL(0x5596e98b,0x60ee6372),LL(0x37137ad5,0x20231d27),LL(0xe0d4f3a2,0x8c695e6d),L_(0x000001d1), + LL(0x1f851427,0x75a8973a),LL(0xf65e532b,0x125d27c2),LL(0xb7e6ca7e,0x680245ef),LL(0xd37a1c1d,0xc0ac3fa3),LL(0xc8ed8871,0x73ed1f61),LL(0x25a3f922,0x1c0619b1),LL(0xbaaf99bf,0x2fc151e1),LL(0xb9c92ca1,0x61f93b5f),L_(0x0000012b), LL(0x4cd45f13,0x46eca65c),LL(0x2f16b28a,0x181d940c),LL(0xdae561c3,0xdef08156),LL(0xdb51b5db,0x5aff9fff),LL(0xd7d0f3f0,0x56731470),LL(0x9f642167,0x3e4323b0),LL(0xc5c736fd,0xc2c256f1),LL(0xaf757eba,0x85000c46),L_(0x00000005), + LL(0xfc825ff9,0xa28756a7),LL(0x6fc9ac38,0xf07539fe),LL(0x19a8908a,0x89d74956),LL(0xc69dedbb,0x934712d9),LL(0x2aaee4e8,0xbbd47741),LL(0x1053c866,0xda8058c4),LL(0x00b68d70,0xc019bbaf),LL(0xb1236281,0xf44074ad),L_(0x000001c7), LL(0xad679598,0xffe5c58c),LL(0x5fe8d191,0xfb4d9415),LL(0x46edaa9c,0x44747329),LL(0xdd0f1cc6,0x34e406e5),LL(0x79f7cddc,0x40ad1213),LL(0xab39e94c,0x470ac094),LL(0x885cb3a4,0x12891647),LL(0x3224c564,0x41e24750),L_(0x000000e0), + LL(0x17473053,0x116c3934),LL(0x97765c7e,0x74d1a056),LL(0xf7e7734d,0x2fb92919),LL(0x22455583,0x880ac302),LL(0xc6198a57,0xc983ea10),LL(0x32d7f501,0x3adab6b9),LL(0xa4c1c306,0x3997a013),LL(0x14822e84,0x5da833b6),L_(0x000000b7), LL(0xf8efc5c2,0x2e766181),LL(0xda380c24,0x158c4baa),LL(0x4d96447c,0x6acaba32),LL(0xe953e90e,0x4ed2e3bf),LL(0x86ffae71,0x452c6d1b),LL(0x3af83523,0x49a52fe9),LL(0x41a86c00,0xec5b4f72),LL(0x9c65d29c,0x7bf9771d),L_(0x00000034), + LL(0x15231af9,0x83eb4905),LL(0x29b6e8f6,0x35420b50),LL(0x76561721,0xd8dddbf9),LL(0x83f7c4f4,0x776812d9),LL(0x460a8666,0xd3c1656b),LL(0x6901dcbc,0x22e1e397),LL(0xd9c17a67,0xd83adc99),LL(0x7a32d3cd,0x7dfafcd0),L_(0x0000007c), LL(0x66102687,0x8578e51e),LL(0x957c6151,0x9a86b387),LL(0xb2bd85fe,0x553c599b),LL(0x21884750,0xb8b8c27a),LL(0x920c65b6,0xf81924dd),LL(0x6f14a6d0,0x28619568),LL(0x18db08c4,0xab2d8a3f),LL(0x11b85385,0x4e01b95a),L_(0x000001ce), + LL(0x5aa706e4,0x0e7a9fe5),LL(0x2110fcf2,0xa8b2d6f6),LL(0x67bdfcf9,0xeaac5dca),LL(0x0435935d,0x39631926),LL(0x399fbc31,0x60795bd1),LL(0x34625175,0x0f561153),LL(0xced09fb3,0xe23ff49c),LL(0xf1b45cd7,0x14acb326),L_(0x00000110), LL(0x4ec88551,0xd20c517f),LL(0x3c8be0dc,0x221ae5ee),LL(0xd895d43a,0x6705afa1),LL(0x00fa5270,0xb373ab4d),LL(0x75678bdf,0x64e2886b),LL(0x47650c25,0x7e0e12f4),LL(0xab5c87d2,0x0c9aec40),LL(0x15ef4537,0xdac6c6e3),L_(0x0000016b), + LL(0x69bd1ae9,0xe2ef5aee),LL(0x0585cc2f,0x1e6188c6),LL(0xa3c01465,0x301c46a3),LL(0x1cc1ea41,0xf7f76048),LL(0x16944109,0xedd90482),LL(0xdc473809,0x3da1ef77),LL(0xf7267c80,0xfefdbcd3),LL(0xfd92f40f,0x230df237),L_(0x000001fa), LL(0x365d3aae,0x5f29db47),LL(0x93437f82,0xd2267583),LL(0xf4a57394,0xe0388ea5),LL(0xb28bca60,0x5cea0f68),LL(0xbf640edf,0xc68dbd2f),LL(0x8db4f9b6,0x24be537f),LL(0x9d943d05,0x7704cfe6),LL(0x4a1f249a,0xfc85d03e),L_(0x0000017a), + LL(0x15eb727b,0x79b64a1c),LL(0x82dd22e4,0x51a73cb4),LL(0xa1e0df42,0x5140d8ce),LL(0xf6d38530,0xe8523991),LL(0xbf2d199e,0x578593ff),LL(0xaabd945b,0xcfef51c1),LL(0x789ae01d,0x9c35735e),LL(0x685cad78,0x2d15c55a),L_(0x00000195), LL(0x9ac2d994,0x5bf21520),LL(0x5119208b,0x3671fecc),LL(0x477379bf,0xcfbe3b2b),LL(0xdc4fe9f8,0x98a8aee0),LL(0x6e915903,0xa7edc740),LL(0xc4cb91a1,0x71211193),LL(0x1e307364,0x96026ba1),LL(0x814b8b56,0xf21a90b1),L_(0x0000014f), + LL(0xdfef19d0,0x30a583d8),LL(0x8521445c,0x192a3c38),LL(0x8651795c,0xe24d5be6),LL(0x6cfd9cef,0x273daff4),LL(0x78499d8a,0xbe3d67fc),LL(0xf4d9c05a,0xfff37ca4),LL(0x9d03588e,0xbe977f9e),LL(0x2b3a2963,0xc6d47c43),L_(0x000000d6), LL(0x6c46a157,0x146210b3),LL(0x092ba668,0x00fc64e7),LL(0xcfe9730b,0xc41f8e66),LL(0xc11dab42,0x79417f9d),LL(0xff89645e,0xddf57c66),LL(0xd0e35f15,0x98273f3c),LL(0x49f211d9,0xea35684c),LL(0x1b8dcf07,0x09d76ca4),L_(0x000000aa), + LL(0x9d8b99d3,0x7dad9f18),LL(0x96f2ea6f,0x96139562),LL(0xf5d410b4,0x86c29eba),LL(0xc6b1f46d,0x2dbba6d4),LL(0xb1709ad9,0x9de07504),LL(0xeea80cc0,0xc7c9ec95),LL(0x47d01eb6,0x99076486),LL(0x19b1d6cd,0x9e10cb39),L_(0x0000014b), LL(0x0a5f9f34,0x53cc24a0),LL(0x5e367eff,0x672781ea),LL(0x275cfce0,0xedc5266f),LL(0x92d98139,0x0d9e2099),LL(0xc2c0efd5,0xf3d9cb26),LL(0x687bde18,0x647d23fe),LL(0xd97b9ccb,0x0c54a71a),LL(0x258eaff2,0x374fd27a),L_(0x000001cc), +}, +/* digit=82 base_pwr=2^410 */ +{ + LL(0x30cbad8f,0xfa23022a),LL(0x0f3c6cf3,0x9f3a186b),LL(0x33420e3c,0xdc6e922b),LL(0xbffbbdb1,0xeac227c1),LL(0x7aa59cde,0x43d5b878),LL(0xe3673dd9,0x513a5be5),LL(0xae77a5b5,0xbc4c0fef),LL(0x8e4c10fc,0xd4728351),L_(0x00000133), LL(0x2a245057,0x020fe0ed),LL(0x5e8e914b,0x39e8625b),LL(0xdbd2dbf4,0x3dfac893),LL(0x5bf5b95c,0xc2cfde2a),LL(0x29c6d879,0xca30a315),LL(0x05a9a75f,0x3ac05ce9),LL(0x894b84cf,0xb5445553),LL(0xeb87696c,0x45350dc2),L_(0x00000050), + LL(0xf9b134fd,0x05a92256),LL(0x23245303,0x7ccfce0b),LL(0x6d8cb621,0xd61ca36a),LL(0x6d0ef54f,0x210c1e23),LL(0x1a182b1b,0x8ae4f253),LL(0xb1f2e2c4,0xa16671b7),LL(0xd29f38da,0xae1cf556),LL(0x3fa6c8ea,0x878e21c8),L_(0x000000d7), LL(0xb9a18df6,0xa098406d),LL(0xd588a978,0xd15a5ed3),LL(0x781ea818,0x5e68786d),LL(0xad06fce1,0xf98680c4),LL(0x66d7a550,0x981589bd),LL(0xd3ff4140,0x7ff83976),LL(0x6ffe6dff,0x9c88eabc),LL(0x47479f18,0x2dc93316),L_(0x000000e7), + LL(0xdb783254,0xe1b91784),LL(0xcc2bd843,0x7a80b2cc),LL(0xda0b8166,0xaa3b4ff9),LL(0xaff9f442,0x88dcdac5),LL(0x38067551,0x47d782df),LL(0xba990049,0x82a02e17),LL(0x02eb92a1,0x61467fcd),LL(0x29ea45e3,0x2eb8d57b),L_(0x000000e3), LL(0x15b7f2b2,0x3170353b),LL(0x4a58e306,0x4a6890e8),LL(0x0dcaea11,0x2ee85176),LL(0x198b5c45,0x79793d5c),LL(0x9872dd7c,0x5ad3ba16),LL(0x2940cc17,0xacaf46a5),LL(0x9d812262,0xb1cff849),LL(0xee571706,0xa1e0320f),L_(0x000000e0), + LL(0xeb0d7e33,0xb0731c42),LL(0x657bac9b,0x5dc372a3),LL(0xd967282a,0xa9374ab3),LL(0xf9ac8856,0xbdf21057),LL(0x3b740967,0xfec8274b),LL(0x56933024,0x5596459a),LL(0x94a16871,0xc8d21c17),LL(0xef7bcfc7,0xf6a0ea98),L_(0x000000f1), LL(0xf9af7b9a,0xbfd8b660),LL(0x7b728a24,0x9cb13ed9),LL(0xee5e0227,0xa450fd15),LL(0xbd3b7d28,0x972ff1fe),LL(0xf367bf5b,0x08f71ea2),LL(0xaa4191b6,0x496276ed),LL(0x52d016bf,0xc4a6a4c1),LL(0x52e7dadd,0x513e10bf),L_(0x00000161), + LL(0x1afab699,0x049e7f00),LL(0x88380090,0xb4090ebe),LL(0xfc780e00,0x9088eeb6),LL(0x294d8c96,0x7d582ac2),LL(0x4027746c,0x5e897916),LL(0xc4fa7517,0x24defc92),LL(0xb30f7062,0x7efacebb),LL(0xe29a1ed9,0xd337aeb6),L_(0x000000f1), LL(0x856dc4c3,0xee3e4e0d),LL(0x6b56aa33,0x971e660f),LL(0xb6b834b0,0xabff8d16),LL(0x4f4e4f36,0x791ab99f),LL(0x1407b72f,0x49828107),LL(0x81293d19,0xdc829510),LL(0x1f2a3c51,0x359122da),LL(0x3bfa8d54,0x99db1fdb),L_(0x000001e7), + LL(0xdf15ca96,0x121949d5),LL(0x705e6cb5,0x15f3e1fa),LL(0x844205ea,0xda2168d9),LL(0xb4128cbd,0x793edfff),LL(0x751feb1f,0x332f4e92),LL(0x4c804349,0x4429da30),LL(0x26bc232c,0x8cea38b7),LL(0x753baf61,0xb3299d3c),L_(0x000000d5), LL(0x88e43827,0xb46e7eb4),LL(0x41b7f39b,0xc6fd29ba),LL(0x98261154,0x4974c56e),LL(0x502ce35f,0x0efcc622),LL(0xad5a1de8,0xfd41558a),LL(0xa51d36ff,0x1a681fdd),LL(0xf2ea91c6,0xaa082cc2),LL(0xf7b13d69,0x107dcaed),L_(0x000001ce), + LL(0xa66a52a8,0x208e5e12),LL(0x9cf1d09b,0xa19a455d),LL(0x4d39c2fb,0xbd4d9e44),LL(0x0adaf826,0xd068570e),LL(0xfd187cbc,0x93225311),LL(0x2398aca7,0x490180a0),LL(0x2dcbb906,0xcee10c1a),LL(0x40723e9c,0x71dc29fc),L_(0x00000018), LL(0x0caf9248,0x7f7fda0a),LL(0x722d1676,0xbe8a621f),LL(0x96407066,0xe524661e),LL(0x3a360a23,0xaa8ac484),LL(0x22dc1578,0x9532f3a1),LL(0x70f20b2b,0x936a98d0),LL(0x4e640ba5,0x10c24716),LL(0xc78e13ef,0xad48efab),L_(0x0000004d), + LL(0x8ced176d,0x49979e6d),LL(0x1d8f2f21,0xb3c390a7),LL(0xd1cc9018,0xff49ec42),LL(0x805d407d,0x56592d13),LL(0xb92c79f6,0x69b4fae8),LL(0x7816250a,0xea40b75d),LL(0x9c23c189,0x98e49fbb),LL(0x012080c6,0x8f035630),L_(0x00000189), LL(0xde297ec2,0x394adc76),LL(0x39d2e9b4,0x084a2dca),LL(0xa4a3c98c,0xff54162f),LL(0x4df52d9f,0x847a48de),LL(0xa6af6c27,0x9a4d9dcc),LL(0x128a4972,0x96ed3609),LL(0x8323c41b,0x0a200116),LL(0x81fed229,0xa4fde51e),L_(0x000001e7), + LL(0x20d87189,0x08f2635f),LL(0x6ccede28,0x6b5f8018),LL(0xc2a0e842,0x1a1eaa9e),LL(0x09c6bbfb,0x590c83ae),LL(0xba912027,0xb05367b2),LL(0x98c59c39,0x363d5716),LL(0xc607367d,0x2738b72f),LL(0x76fa4bf8,0xafbb3680),L_(0x0000014c), LL(0x04011fd9,0x6b33299c),LL(0x42860e67,0xfb00a1e0),LL(0x95c30a5c,0xc7ee4546),LL(0xc597748e,0xbb5a242e),LL(0x3ed03861,0xf0e0361f),LL(0x87708c93,0x2149d193),LL(0x7590a638,0x483134f1),LL(0xc66e8a9d,0x85cc2776),L_(0x000000e0), + LL(0x3813f012,0x5ffac9a4),LL(0xc0d4789d,0x3e79348e),LL(0x955e2b14,0xdb2d28b7),LL(0xe25f9f8b,0xc1656e1c),LL(0x6715563b,0xec42f2f4),LL(0x79f7e28f,0x2aead585),LL(0x106b8506,0x201ba118),LL(0x7f8c385c,0xe4fe7233),L_(0x000001ab), LL(0x0507ebcf,0xa8ba08d5),LL(0x035af551,0x5db82217),LL(0xe87f61ae,0x97aaf3ee),LL(0xeaf13d4a,0x5f02aeda),LL(0x0476d73b,0x05c9a1af),LL(0x8d9103dd,0xbbd0da23),LL(0x4b324ed6,0x4516539b),LL(0x0edd4904,0x253727b2),L_(0x0000009a), + LL(0xfad3e533,0x9fbd42a4),LL(0x2540903f,0xd7649f2b),LL(0xd8fbba0e,0xc6189edd),LL(0x6a06d4e6,0x1ecce6bf),LL(0xcbba2a3b,0x5c61ec36),LL(0xdb49be34,0x447d3062),LL(0xe4b0ad6c,0x49fc7fee),LL(0x1394cdb0,0xcc99bb8e),L_(0x000000f5), LL(0x1172029d,0x30a8e021),LL(0xbd509669,0x6f8f7c76),LL(0x34a06ccf,0x28f39de2),LL(0x1f805a36,0x5b84e8ae),LL(0x95bcde27,0xa9d053ba),LL(0xd196b2b5,0x4777e8cb),LL(0xbc21fa82,0x1bd75421),LL(0x58e2b8c6,0x9c786320),L_(0x00000036), + LL(0x7a23ceea,0x51aba8d2),LL(0x241c7fb7,0x58878762),LL(0x60069d96,0x51c9d281),LL(0x30ade2dd,0x9b631718),LL(0x91c3eb50,0xf909879a),LL(0xca8dcb86,0xee7eb48a),LL(0x9244bc3d,0x3828bf79),LL(0x2a064705,0xfb4bc920),L_(0x000001ed), LL(0xa015f7fc,0x3ba48f85),LL(0x2b6a12f5,0x95ae129c),LL(0x444e10a6,0x8aeddb2a),LL(0xecfba54d,0x0e8bdf6a),LL(0xc7c39b4f,0x0a72c4d1),LL(0x73b3e601,0x5cdfd0f3),LL(0x99b50a0d,0x5168c9e0),LL(0xbed1929d,0x87bf31c8),L_(0x00000050), + LL(0xa9be603a,0xd22b9ef5),LL(0xc53e7872,0x9558bd84),LL(0x154e0475,0x65401d38),LL(0xbb71dcda,0xf42190a1),LL(0x79891c59,0xcce4f8f7),LL(0x3876a596,0x5ec40734),LL(0x91d0b32b,0xef02bcea),LL(0xe2d3dc41,0x22d8fe83),L_(0x00000158), LL(0x48a15725,0x7ac11538),LL(0xde31cc2a,0x59724de2),LL(0x949e5fc7,0x1c37f0ed),LL(0xc39190ad,0xc058927a),LL(0xb63be947,0x6bda54d7),LL(0x1d0ceb2d,0xcff0cf31),LL(0xac38d621,0x23f872f5),LL(0x04b46672,0x160a5f19),L_(0x00000052), + LL(0x13238feb,0x8c998602),LL(0xca47bfa7,0xebe29b0a),LL(0xe22527b3,0x7f9eebf4),LL(0x127b07f2,0xa6e2939d),LL(0xca0c1812,0x6469e55a),LL(0x65afd9e7,0x46a7771a),LL(0xf84da7e1,0xbd0864a1),LL(0xb5dbfbc9,0x87ff16c0),L_(0x000001f1), LL(0xb3a29f77,0xbf5bf543),LL(0xb2d15c04,0xd9bcacc1),LL(0xb632b37f,0x5041f46d),LL(0x9bed186f,0xb690ffbb),LL(0x4873f91b,0xeebd39f1),LL(0xff2ad723,0x79a50cae),LL(0xd0c46d7f,0x6b3bea02),LL(0x4a5b4d01,0xf2c26cc1),L_(0x00000112), + LL(0x0bb28261,0xe9314bb7),LL(0xa04bdd40,0xaab60946),LL(0xe0b7c3f9,0x20cc2e25),LL(0x1e792761,0x60fd58e5),LL(0x7823f278,0x561086d3),LL(0xfa3bf02e,0x7b6170af),LL(0x0390ab9e,0x4a18459c),LL(0x0c346fa4,0xf810e889),L_(0x000000b3), LL(0xb623e6eb,0xe73d3d7e),LL(0x43490dbf,0xc1f1a1fd),LL(0x49e9e831,0x0a18b7b1),LL(0xe619f992,0x0c6e526d),LL(0xdb9252a1,0x954cd738),LL(0x6a826c49,0x41105a8b),LL(0xddcbb9cf,0xe9217743),LL(0x750efcbf,0x3f5b0793),L_(0x00000036), + LL(0x90c2c466,0x57128a20),LL(0x1345257c,0xdbf610a8),LL(0xd16c4a33,0xbff009fd),LL(0x8f1d5b65,0xb49af8c7),LL(0xf560ad02,0x0eb8499d),LL(0xfb45ea45,0xa52dc630),LL(0x7e352023,0x8ada8ac5),LL(0x2fd6cb3d,0x74b6bfb7),L_(0x00000107), LL(0x8f6d7783,0x4a52a42a),LL(0x3f44af1f,0x4175b2fd),LL(0x05fe7f14,0x079ac149),LL(0x1757d0c0,0x475fac70),LL(0x88ae6d1c,0x7bfd9387),LL(0x33f3d56b,0xb7dbf13b),LL(0xcacad131,0xebd1df20),LL(0x8aef62c8,0x20404609),L_(0x0000013a), +}, +/* digit=83 base_pwr=2^415 */ +{ + LL(0x96fd03e2,0xf3f782f7),LL(0x0bf4ad5c,0x3ffeae62),LL(0x8514f603,0xe39ca015),LL(0x5633e085,0xa88f4e54),LL(0x94884fbe,0x85fc77f2),LL(0xf605882a,0x2678c646),LL(0xa505f9b1,0x799ba323),LL(0x23217b43,0x88a00292),L_(0x00000141), LL(0x032e8744,0x29194f6c),LL(0x6cab181a,0xa932a791),LL(0xf60ec063,0xb94fb0f9),LL(0x217a0ff0,0x8f066aff),LL(0xd03ea56b,0x56ee8b26),LL(0xc1ffed4a,0xefbf8ce2),LL(0xed130515,0x922eb114),LL(0x74474339,0x2e17e43d),L_(0x000001e8), + LL(0x88affb50,0x5fc7f37a),LL(0x170a0c1c,0xa1f73a2f),LL(0x83474ff9,0xea811929),LL(0x4738ed4f,0xdd78686b),LL(0x24d293dc,0x16188a23),LL(0x36670cd9,0xc585fd52),LL(0xa2e54dbb,0xd3b67188),LL(0x10b37344,0x06ec68ed),L_(0x000001a3), LL(0xb4548b08,0xfbe0b348),LL(0xad120991,0xef3cdca5),LL(0x034c9a59,0xd16cfcd2),LL(0x56699960,0x10f2524f),LL(0xf6df1f5f,0x4733c5f1),LL(0x98a50032,0x757f84ed),LL(0x2ce4fa9a,0x032f7eec),LL(0xd296f3ba,0x6c007ae3),L_(0x000000c6), + LL(0x4c96fb7e,0x71db44c9),LL(0x7ac3b1c1,0xe9e2107d),LL(0x6305a422,0xa60259b2),LL(0xc0b70492,0x17e1f71b),LL(0x3dedfaf7,0x57f8d178),LL(0x15a8c62d,0x7a704e50),LL(0x81a4724b,0x3b26accf),LL(0xdf992c3c,0x03ad3ca0),L_(0x000000b7), LL(0xe35cce3f,0xc9cc0ded),LL(0xc51c6e93,0x00949cad),LL(0xba4d9081,0x8de8b3c3),LL(0x274926f6,0x839aef68),LL(0xf13d3b8a,0x40a2e3c7),LL(0x90113a91,0xef4e7433),LL(0x1f472807,0x8320657c),LL(0xd4accc1a,0x34dbb9d6),L_(0x000001c4), + LL(0xc34e9382,0xcd77eae7),LL(0xad7f0d81,0x13604a2d),LL(0xd0e3fde5,0x70443d13),LL(0x6cb59871,0xc5f3e64c),LL(0x7a8441d1,0xadfd909c),LL(0x00532361,0xefba7861),LL(0x0eb9abb4,0x05155907),LL(0xee4fe6fb,0x400f9708),L_(0x00000166), LL(0x7579200c,0x128ad5ec),LL(0x49006579,0xc2659737),LL(0x8cf2fa39,0xc9df0a7f),LL(0xdb548c37,0x648b652d),LL(0x3da31069,0x075eeef1),LL(0x34916a7e,0x04d0e409),LL(0xea6b7825,0x08fd613b),LL(0xba92eb2c,0xed99155b),L_(0x00000149), + LL(0x47521ca4,0xc24e86b5),LL(0x555d9a6b,0x572924ea),LL(0x878c09b9,0x379539ca),LL(0xb6b82a15,0x8666f974),LL(0x72ba5827,0x2d9ff656),LL(0xb17ece57,0xc68cfdb5),LL(0xfe917da6,0x22f34e2a),LL(0x08992968,0xfd615e21),L_(0x0000007b), LL(0xf77f07dd,0xba763f11),LL(0xf3886c72,0x11e3d2aa),LL(0xa4c6b62d,0x550a0527),LL(0xe4eac1e4,0x6a7880c8),LL(0x9b86f1e0,0x00ea68f4),LL(0x87cc2f01,0x55e1d3f3),LL(0xa6daef73,0xa5a26f0f),LL(0x2061095b,0x9af79ad8),L_(0x0000002e), + LL(0x8b5ab4da,0xbf85411a),LL(0x7b991f0d,0xa933992a),LL(0xce47a748,0xc820accd),LL(0x662f2eb8,0x12508cc5),LL(0x964b5fdd,0xadddfe6b),LL(0x1358db73,0xf97a44a3),LL(0xfefeacaa,0xf544c5dd),LL(0x3a084f6f,0x820fa340),L_(0x00000094), LL(0xa4ad406f,0x54cddbeb),LL(0xec844e89,0x92a7fe19),LL(0xa4c49f5f,0x30484bff),LL(0x8eb76b96,0xf75a70ec),LL(0x414948f0,0xc139503c),LL(0x7606dff2,0x2fdf031b),LL(0xead62083,0xc5fa11a5),LL(0xf7a1eba7,0x4285147e),L_(0x000001d8), + LL(0x7ec431d8,0x2ebaaf47),LL(0x3c1a318b,0xd9561acd),LL(0x57cb3287,0xf1f803a4),LL(0x97882cee,0x97ac71c6),LL(0x08071010,0xf26f5efd),LL(0x60ed1a2f,0xe4333f2a),LL(0x76efc905,0xc5d26fd2),LL(0x4101ca9f,0xaa75d93c),L_(0x000001c0), LL(0x75779876,0xf709921a),LL(0x294a8042,0x5ab89053),LL(0x0dc515ed,0x5f21558b),LL(0xd5146f68,0x1cdf8e37),LL(0x8dcdaaee,0xe4930f54),LL(0xacb8f4a6,0x58634cb4),LL(0x507294c8,0x20ac12b8),LL(0x38b15ebe,0x4cdf2cba),L_(0x0000007e), + LL(0x6973a3da,0x5d8f611c),LL(0x3f4e86dd,0x799bf33f),LL(0x749fb625,0x4a326f63),LL(0x667bd358,0xa8161392),LL(0x060fa9fe,0xa3a8de55),LL(0xc4af999e,0xdd75d71a),LL(0x18b1e644,0xc8783194),LL(0xe995c857,0x66a68c67),L_(0x0000017b), LL(0x32c7afa5,0x22bb4a07),LL(0x08c59a72,0xc7e0e1a6),LL(0xe1132506,0x1fa09057),LL(0x5c6a1998,0x6b7bf39c),LL(0x41920509,0x8ab7490c),LL(0x462ba7d3,0x9e016ba7),LL(0xdc3595d9,0x1207f474),LL(0x6a3d8c9f,0xd6a83663),L_(0x000000a5), + LL(0x2b8fd9cd,0x606dd096),LL(0xaea5acbf,0x3ef0fe25),LL(0x4859d96e,0x078a8287),LL(0xf923a972,0x0d66d4df),LL(0x660fec36,0xb27b9ec1),LL(0x102d1cc4,0x1d991f63),LL(0x730fac93,0x0c8cdfad),LL(0x601a8644,0xf814f65b),L_(0x00000164), LL(0x55d58869,0x0d313b90),LL(0xde263f83,0x11dbdf4c),LL(0xcde2808c,0x824b53bf),LL(0x6aad6afb,0x94547c69),LL(0x864c21db,0x6771d459),LL(0x65c1681d,0x7f15bc35),LL(0x95681ca9,0xeac68bf9),LL(0x78f00d83,0x5d523cdb),L_(0x00000161), + LL(0x2f058d88,0xb2e8d253),LL(0xaca02a59,0x15502597),LL(0xbd7d1caf,0xa0dfb1e8),LL(0x680e361d,0x9355f155),LL(0x2ed31cfd,0xb0064d2b),LL(0x308047c0,0xf348830f),LL(0xbffaf7d4,0x7bb7440f),LL(0x553b98e1,0xbe483a5a),L_(0x000001e6), LL(0x9f0f6eac,0x25881bc3),LL(0xa7105d49,0xd655e7e9),LL(0x033db883,0xdc82f09a),LL(0x7d5a4975,0xa17847e9),LL(0x4036e619,0xe9b20930),LL(0x048479d0,0x34fdadf8),LL(0xe6c7daa4,0x19412216),LL(0x152f330b,0x6eae10eb),L_(0x0000016c), + LL(0xc41a7939,0xcb26a326),LL(0x78d31411,0xa365755c),LL(0xd3a391dc,0xe18d1f8e),LL(0x25f34512,0x72a34b02),LL(0x345c60dd,0xd3613f12),LL(0xe2be6e2f,0xee2aebd8),LL(0x116aa632,0x92435b38),LL(0xa7cff2c3,0xccdf1cc9),L_(0x000000df), LL(0x342a1414,0x9da3251e),LL(0x650f3cbb,0x1c4cfe7a),LL(0x93ddee66,0x90734ecc),LL(0x1551811f,0x6d901fe7),LL(0xcc1cef07,0x0c7d5cd4),LL(0x3cf5fe87,0xf0de3068),LL(0x628a5bd8,0x56e883b0),LL(0x077d4d10,0x9f410795),L_(0x0000007e), + LL(0x2d362cca,0xfb4260cc),LL(0x99614b58,0xe2d527b8),LL(0x67cb8aa3,0x08c1f0b4),LL(0xef71b82f,0xc4649ad1),LL(0x4dc68072,0x11a9313c),LL(0x1ac298aa,0x65002fbe),LL(0x9f237961,0xe90bf539),LL(0x537dfdd6,0x5b64f24b),L_(0x0000011c), LL(0xfcf6c830,0xf2e76469),LL(0x4ed4174a,0x51f1bc5f),LL(0x3a450f7e,0x83c0406c),LL(0xb53708a6,0xaa7dce02),LL(0x2428a6e3,0x44b377b6),LL(0xf1abc0df,0x9c1a58f5),LL(0x0f02c35e,0x27c5458b),LL(0xea8718da,0xc56a6b16),L_(0x0000015b), + LL(0x1709bab6,0xd1568e14),LL(0x5208725a,0x11bb7351),LL(0x3937bb45,0xe6e5c9ed),LL(0x51ec95b9,0xcf7661c1),LL(0x2e23be41,0x6ad385c7),LL(0x743b0e8c,0xe91385d0),LL(0xdfe84bbc,0x6fd97535),LL(0xdce16477,0x922ccac9),L_(0x000001db), LL(0xef8587b7,0x0e47600f),LL(0x8f375855,0x6859ef6b),LL(0x529ee446,0xc03de2c7),LL(0x84c625e6,0x6f7cbb53),LL(0x3af54a02,0x29eb9d15),LL(0x37cbf19d,0x750b2d9f),LL(0x2dc61071,0x28f78635),LL(0x6b20c9d0,0xc61fd9cc),L_(0x000001b7), + LL(0x78134d37,0x3d575868),LL(0x09f53723,0x6cf5af74),LL(0xfc4fd018,0x39a8ae6d),LL(0xb54df4cc,0xb1d402e8),LL(0x25f3046d,0xeece717b),LL(0x98af312a,0xa13a0c5d),LL(0x5f96c47d,0x7f73d630),LL(0xc80a3e3a,0xb8b39587),L_(0x00000180), LL(0x7f27d5a2,0x38bbf8fa),LL(0xc4360f93,0x2109c7d0),LL(0xe57b26a1,0xeea6004b),LL(0x32aad5ae,0xf9dededa),LL(0x341aa5da,0x6abc8307),LL(0xfdd6b0a1,0x88080bda),LL(0x4cecd6e0,0x1d4fa881),LL(0x24b2b7fe,0xdeba0f5f),L_(0x000000a2), + LL(0x9a724982,0x5ec2d29e),LL(0xa69853d7,0xd1a7ff3b),LL(0x4e7a173c,0xffeab3d8),LL(0x9f30735d,0xdf72352a),LL(0x92186213,0x469f7a55),LL(0x2d184184,0x15c208a0),LL(0x29514acf,0xa37b763a),LL(0x91393991,0x75a2695b),L_(0x00000124), LL(0xa16bd801,0xf0588b07),LL(0xc8dafbba,0xda133516),LL(0x3aebd875,0xf622ae4f),LL(0x34fb368f,0x4f336c4d),LL(0x0b5d9d6e,0xc6eb3519),LL(0xa586248a,0x6d28f06f),LL(0x6daf558c,0x2f3e52a1),LL(0xfb60040b,0x2b032cb3),L_(0x000001f7), + LL(0x05cc19d6,0x6d31ebaa),LL(0x2f30e929,0x3edd43b0),LL(0xc72cbbb4,0x827c79aa),LL(0x7829df3e,0x4e4cf8d5),LL(0x98bb6262,0xffe745fb),LL(0xad894f05,0x50b350aa),LL(0xb566ef19,0xf2ea5e2d),LL(0x37f6dcf4,0x89295e1e),L_(0x000001ae), LL(0xfa34202e,0xd748da48),LL(0x68fd9ed5,0xe1cf505b),LL(0xc7778cb3,0x3af31b86),LL(0xdadb4507,0xfe717fdb),LL(0xb4b6e80b,0x13036b30),LL(0x2c3f1ee4,0x482b138b),LL(0x4fc01593,0x71e1ed1e),LL(0xd788bd27,0xe78b3e84),L_(0x000001cf), +}, +/* digit=84 base_pwr=2^420 */ +{ + LL(0x7d9afb73,0x1c085462),LL(0x2eb9d35d,0xacf6649f),LL(0x38a8a9ec,0x095acf4c),LL(0x78be52c0,0xf7d7ea21),LL(0x48f6e06d,0x115ce7bb),LL(0xbedc8285,0x7f232680),LL(0xe51e8f4b,0xd24103a6),LL(0x09aa0bc0,0xd9c59aaa),L_(0x00000106), LL(0xe11c4bc5,0xf757159c),LL(0xbc8d09f1,0x0e9e10c0),LL(0xe3621884,0x60d7345b),LL(0x822e5e0a,0xddc802d1),LL(0x6ae792ac,0xf49763d7),LL(0x868a6be0,0xff0f1717),LL(0x7cae1bcd,0x69443786),LL(0x8bfe19f2,0xe856270e),L_(0x00000073), + LL(0xe01b8994,0xe496416a),LL(0xa213f31c,0xdc0825ce),LL(0x0281aa93,0xe4450bdf),LL(0x236853f9,0xc8c09e59),LL(0x841e294d,0x595c72a5),LL(0x1e14b03a,0x2bc6e538),LL(0x46b3008c,0xef20b035),LL(0xa57d1874,0x464d68ec),L_(0x00000160), LL(0x5951a61c,0xd36a1693),LL(0x5b84b925,0x1712f765),LL(0xc1e05016,0xa422f3fd),LL(0xa758020b,0xb022819a),LL(0xb6927405,0xfced2aa8),LL(0x13a79822,0x9ae63d93),LL(0x87cbebb1,0xcdd9c078),LL(0x13e45feb,0x2915f65e),L_(0x000001cb), + LL(0xc434f1f7,0x878049fe),LL(0xf0c9ca48,0x692a3fe1),LL(0x277fbbb0,0x7ad48261),LL(0x263dc0fe,0xc6fec032),LL(0x7e09052a,0x04d38aeb),LL(0x37fd6838,0x6c55fb12),LL(0x53925e9d,0x63bce4b4),LL(0xc33e2d82,0x3308456a),L_(0x000001c9), LL(0x70a764c5,0x705b2adb),LL(0xb9f1bed9,0x3bab4631),LL(0xb4850149,0x37a14535),LL(0x2385e829,0xcd9ea79c),LL(0x07aa5ebd,0x17e5b5eb),LL(0xd3216565,0x8fb6885f),LL(0xc6b2bd87,0xce87f791),LL(0xd44bfcfc,0xa6e61bca),L_(0x0000018d), + LL(0xd4a7c1bb,0xad86ee95),LL(0xf741fd13,0x1edd6ede),LL(0x85b8fada,0x08912214),LL(0xb65b3c0a,0x30092262),LL(0x3583aac0,0x6716727e),LL(0x817eec2e,0xd6729d8a),LL(0x33bfe296,0xa92ad342),LL(0x779921dc,0x32a04806),L_(0x000001b5), LL(0xc1373f0b,0x5632a2bf),LL(0x2c2b1683,0xa8cd4f47),LL(0xa4cedeb1,0x32a4b6c9),LL(0x180690c1,0xfa510c76),LL(0x1eca05f9,0x51eb02c4),LL(0x2e0f4e05,0x63213fc5),LL(0xb6165cde,0xd91b429b),LL(0x1fdd188c,0x86f862bc),L_(0x00000017), + LL(0x52629f76,0x87d39d0d),LL(0x07dbdb1e,0xcaaa9976),LL(0xeb7d6dd2,0x02ffcf23),LL(0x7e0cd30a,0x78d51085),LL(0x83ecd227,0x961e1f15),LL(0x54655c5e,0x8aa70a14),LL(0xf96ec7c6,0x62c8fb1f),LL(0x64d2f55e,0xab3379b4),L_(0x000000b1), LL(0xb6e904d2,0x0bec2602),LL(0xef59ae4f,0x80f6effa),LL(0x330793e6,0x54960688),LL(0x2442ae08,0xff5a5fdb),LL(0xc5e3d773,0xc6ac0199),LL(0xcacfcecb,0x2fa7a795),LL(0xc57e52d1,0xdfd6f9bf),LL(0x2e4eaeaf,0x8f282ce3),L_(0x00000015), + LL(0xefeeeccb,0xd77ed0ff),LL(0x9db74ad1,0x5e752d76),LL(0x0b6200a7,0xfc315b24),LL(0x7b48ab8c,0x38ef4859),LL(0x6f975045,0x980da41b),LL(0x09695a2e,0x0010c201),LL(0x0d2b23ac,0x7b7fe53f),LL(0xfe8b4888,0x54fafa1e),L_(0x00000174), LL(0xc97b952e,0x6b8cb163),LL(0x16bdd26e,0xd62feeed),LL(0x6129cfd5,0x9be7e3db),LL(0xcfa4489e,0xd804ec9d),LL(0x8f551707,0xadb1fbcf),LL(0x055312a2,0x05be2283),LL(0x7d87937c,0x33c9f74b),LL(0xe5edd3be,0xa04388d7),L_(0x0000011b), + LL(0x854b2f45,0x223a8078),LL(0x954aed2a,0x155358a6),LL(0x30349ae5,0xae186e4b),LL(0x982c7a2a,0x15564aa5),LL(0x26ca4f64,0xbb73fd2e),LL(0x3fca013a,0x2d10cb06),LL(0x3b01aea9,0x33610496),LL(0xd0f1f68f,0x51efcbbc),L_(0x0000008d), LL(0x7f69aedb,0x1ef2c9b4),LL(0x7e6d3ebe,0x3246ee72),LL(0x1a5a6120,0xfc23f5bd),LL(0x7e829b05,0x02380698),LL(0xfb70444c,0xe903fa23),LL(0xb14cf0d6,0xaba52743),LL(0x7e9872db,0x4a9994d9),LL(0xcbf74f83,0x7e344a52),L_(0x000001e8), + LL(0x077ea324,0xae770944),LL(0x13db8792,0x1201418e),LL(0xbc07e7f8,0x7f43920c),LL(0xd56c9383,0xa87da3ae),LL(0xf13ae3d2,0xddc44f1d),LL(0x220944b8,0x9328fcaa),LL(0xe3819646,0xeeb928d8),LL(0x3e55f26b,0xd7199731),L_(0x0000001f), LL(0x42ec4f45,0xfe195b7e),LL(0x8b6807bd,0xe12b70e6),LL(0x2b99f5fa,0x7ea15922),LL(0x4e065912,0x9b50760c),LL(0x2efe8690,0xb9b66711),LL(0x16a38c58,0x7b7026cf),LL(0x0ea01a94,0x80737374),LL(0x39639895,0xb0011015),L_(0x00000037), + LL(0x514bdbc0,0xb7081a5f),LL(0xfa8d102a,0x8b95f28e),LL(0xbeeb8bec,0x89435618),LL(0x71f4af84,0xdf609827),LL(0xb479c91f,0x14f1207b),LL(0xcb1fbb1b,0xde646157),LL(0xf9e5fc99,0x7b096333),LL(0x34873e5f,0xc94ae92f),L_(0x000000ac), LL(0xd78f796f,0x3fa5ab5e),LL(0xa3fca0b1,0xacf76276),LL(0x8003771d,0x1726ca8d),LL(0xe6d96044,0x3e43296d),LL(0x93fa9826,0x42531228),LL(0x527acf14,0x60a72222),LL(0x7a97cba9,0x3d7c6f6c),LL(0x1948eafd,0xcc2e1d2b),L_(0x000000f2), + LL(0x4e12b5af,0x08d068df),LL(0x886cf2ad,0x79260e0d),LL(0x52a61dfd,0x3e9745ee),LL(0x3f586c5d,0xe1f15978),LL(0x566b3b18,0x87842ee1),LL(0x9ad3de40,0xa8f52a76),LL(0xfa06fa9e,0x391add0b),LL(0x9e84e7b8,0xc75f3842),L_(0x000000a8), LL(0xbb6677c3,0xa05bc623),LL(0x52d3e69c,0xb274df3e),LL(0x974dd07a,0x630e22a1),LL(0x5e6204a2,0xb9a3958d),LL(0x1b227ffa,0xd0e0f634),LL(0x8fc075d7,0x37062afb),LL(0xbe7fbf5c,0x34adc02d),LL(0x77906471,0x8d8b552f),L_(0x0000012b), + LL(0x6c9a6028,0x259579a0),LL(0x6f5bb2d1,0x0285a43e),LL(0x154ac0bc,0x167a3f99),LL(0x36795c46,0x728f737d),LL(0xb73fd19e,0x4d08d004),LL(0xa6cc0016,0x85ddb728),LL(0xe72a83f5,0x2e0d295e),LL(0xbfbd9eeb,0x6433def5),L_(0x0000014e), LL(0x90f9c914,0x0eb6be44),LL(0xaf866e12,0xfe01cd22),LL(0x5fd8a835,0x7bf70c61),LL(0x7f0a7679,0x38813f80),LL(0xc8998bdd,0x3833317e),LL(0xb796096b,0xfec3ea9b),LL(0x95f4e76d,0xdad9de4b),LL(0x341f07a8,0xa3d960d0),L_(0x00000061), + LL(0xd3c33742,0x200fc40a),LL(0xb0a42399,0x130ccf0d),LL(0xcba30b44,0x5e03c03f),LL(0x8e5dcd29,0x6cdd189a),LL(0x1e3d9c12,0x5a30516d),LL(0x81b3e6ac,0x6b10cabc),LL(0xcc86cceb,0x69bda2cd),LL(0xc43e19c0,0x11a9d83a),L_(0x00000130), LL(0xdc567bde,0xe0559109),LL(0x6575e24d,0xe2ac30ae),LL(0x67aa07b7,0x2b6c5e32),LL(0xdaf439f7,0x407ff042),LL(0x2ed76986,0x0a686f32),LL(0x1581e9c9,0x5cacfb27),LL(0x18e37622,0xc3e3f9bf),LL(0x387fd7f7,0xd10f51ce),L_(0x000000c2), + LL(0xcd152b7e,0xe1bfaadb),LL(0x9f66edb2,0x2faec2d9),LL(0x65371362,0xe758a385),LL(0x03888c8c,0x70736030),LL(0x84bd514a,0xf31e004a),LL(0x391a903e,0x246ee862),LL(0x36e788ca,0x4e1e02aa),LL(0xf4e00df9,0x2c2a0937),L_(0x00000023), LL(0x13ba50a4,0x4eff7c1a),LL(0x941609eb,0x6d04b057),LL(0x4cf29ce1,0x414be663),LL(0xc8df1ba5,0x12291ebf),LL(0xaa401069,0xeb298db0),LL(0x8ded5bfb,0x893e7d57),LL(0x549bd728,0x01b7b4f4),LL(0xac5af1f1,0x22a5b845),L_(0x000000d8), + LL(0x5f6b726e,0xb153c049),LL(0x1a260428,0x30c06c83),LL(0x9ce7c010,0x5b4fc0ff),LL(0x894ae091,0x5657bc21),LL(0x0aae49b6,0x571a85ef),LL(0x79cfa3cb,0x65689c51),LL(0x2187e975,0xe76412d7),LL(0x992bb278,0xf64da789),L_(0x000000ea), LL(0x0cf8c519,0xaa2b60ae),LL(0x9b1824f2,0xc0f1fe55),LL(0x9db0e06e,0xd91bb72b),LL(0xa610996c,0x58e02f73),LL(0xf2c3a8c2,0x28d213dc),LL(0xeae1bfc7,0x13f362dc),LL(0x943517da,0xf7bdbaa1),LL(0x496789e0,0xfc68ba49),L_(0x00000023), + LL(0x6c383c56,0x23ca0c67),LL(0x0e7599c4,0x99283769),LL(0xfeb47aad,0xe99af616),LL(0xaf666d85,0xf45b2fa8),LL(0x9231d83a,0x7c7b482a),LL(0x98e9e83d,0x8568ac48),LL(0x1e56ad0b,0xb6b0ac15),LL(0x724bffe2,0xcd1a6169),L_(0x0000016e), LL(0xb11260bb,0x877aec8d),LL(0xc03ce819,0x4e72375a),LL(0xccf9f47b,0x54033f3a),LL(0xac8daf52,0x19b599aa),LL(0xdea12842,0x175a56a0),LL(0x7ec2c31e,0x8b550edc),LL(0x1acab966,0xf427902c),LL(0xb52abab5,0x2fa9ef47),L_(0x000000a4), + LL(0x52546c7f,0x20270840),LL(0x3c3eb721,0x13f0d6d8),LL(0x142e7b62,0x019eaca7),LL(0xf973a763,0x025efc29),LL(0x8be00c2a,0x6f6199ab),LL(0x03943eca,0xb5618bcc),LL(0xfbfb9ab2,0x5fb02749),LL(0x0ae9ab79,0x7965f13c),L_(0x000000e1), LL(0x50f295b9,0xab6d0e9f),LL(0x083d8bf1,0xbf53d7cb),LL(0x07076abe,0xf29d744e),LL(0x51a53561,0x5c2fc15a),LL(0x7d647b91,0x253f8428),LL(0x51a29ead,0x91bd9d62),LL(0x006b7bce,0xe106dd74),LL(0xc770e4ef,0x7ae25c92),L_(0x00000078), +}, +/* digit=85 base_pwr=2^425 */ +{ + LL(0x412703ae,0xe95afaa3),LL(0xeb61e427,0x271961d1),LL(0x509a4412,0xcf60fb71),LL(0xef04644e,0xd64abf7b),LL(0xf44d4556,0xfd03e651),LL(0x9b052a7b,0x4add4bdd),LL(0xbb2156b9,0x2a438d74),LL(0x6c6c1657,0xb9025fa3),L_(0x0000009a), LL(0x0eae16f9,0xafa2d73a),LL(0x44951d52,0x5bc81a5a),LL(0x324d90b4,0x8490fc89),LL(0xcab36337,0x7db83818),LL(0xdcb411d8,0xd6cb710a),LL(0x80af21f7,0xad265214),LL(0xf370ca06,0x3194a666),LL(0xd8bc966e,0xc5222375),L_(0x000000d7), + LL(0xbbdc15d0,0x35b540e1),LL(0x979e91ce,0x973de7aa),LL(0x745fa684,0x84d5e965),LL(0x999e957c,0x2675f78a),LL(0x671ead70,0xec63eb2b),LL(0x9f4e2bee,0x2262a934),LL(0xa151a2d6,0x885b027f),LL(0x5a633743,0x198c65b4),L_(0x000001da), LL(0x63ade9fd,0x64a13f40),LL(0x23957199,0xea6e576b),LL(0x21e1e294,0x1a39fa20),LL(0x3e8e8d0c,0xeb1db8f9),LL(0x2627a8aa,0xa239f73f),LL(0x4ba11c83,0xdc2704f3),LL(0xe1591a19,0x6d53062b),LL(0x75cd2132,0xfc2434c9),L_(0x00000197), + LL(0x615287cf,0x9beb6f84),LL(0x5201fcef,0xa7eaedc7),LL(0xabd8969f,0x485d39df),LL(0x52116284,0x9a298f42),LL(0xc2945b41,0xb98d985b),LL(0x02540f03,0xbe292f99),LL(0xfb5c8297,0x927d1f59),LL(0x379bfc25,0xb2c3602d),L_(0x0000005e), LL(0xbd1179af,0x24128932),LL(0x49c7ff27,0x920719e9),LL(0x1ce441c4,0x7bdf184c),LL(0xd4d77a7c,0xd00ec091),LL(0x286d6826,0x3901be8d),LL(0x8de5125b,0x6d7da5fd),LL(0x7b71c09c,0x0a84f058),LL(0xd13d8a20,0x4e423ecb),L_(0x0000004a), + LL(0x4047691d,0xb25141cf),LL(0x9109a95b,0xcb8b02c5),LL(0x46b0d286,0x43ceb570),LL(0xb7de8163,0x45b33ca8),LL(0xd109a4f7,0xe4de2617),LL(0xfe1ebf34,0x50869270),LL(0x1ca61709,0xd0a3a1b0),LL(0x38634a9a,0x3586512e),L_(0x000001fc), LL(0x38f07910,0xce15260c),LL(0xc1bab71b,0x59aca1ef),LL(0x5da68584,0x8166c7ad),LL(0x0bec54d9,0x8f461203),LL(0x49a9c32e,0xd88a5170),LL(0xefea1154,0xbe8180a7),LL(0xec5ede1e,0xad08abc6),LL(0xe0d26459,0x7c3d1d7d),L_(0x0000013f), + LL(0x75d35af0,0xc6d5609c),LL(0x78abf2cc,0xe0503dac),LL(0x06b14f6b,0x2edd7321),LL(0x1d8b929f,0xf2198c61),LL(0xae13548d,0x510cab55),LL(0x7dda7de6,0xd681dc83),LL(0x9c5b80c7,0xf75528c0),LL(0x3789ecc5,0xbf5f2412),L_(0x00000081), LL(0x96096d94,0x43aa0894),LL(0xc108e7c3,0x6b10f225),LL(0x662c2a13,0x6a846090),LL(0xfd5f03e1,0x2ef6dd38),LL(0x6481ee80,0x5be580e2),LL(0x5fdbc53d,0xa958cf09),LL(0x11a46f11,0xc02fca90),LL(0x664e67d4,0x83a23d94),L_(0x000001f2), + LL(0x2e504292,0xd30defda),LL(0x0fa7d618,0xea344a1b),LL(0x8b5c684d,0x88f031e8),LL(0xc536e574,0x4b824800),LL(0x3cbde674,0xad6b63cc),LL(0x56514289,0x9617ab5d),LL(0x85b3cf5e,0x7dc3ec36),LL(0x7fc09058,0xc79289b6),L_(0x00000135), LL(0x846008dd,0x6898fae8),LL(0x9e7f0e84,0xd8281d5a),LL(0x10825f5c,0x2d75ee2c),LL(0x12dbdb85,0x7c0aafd5),LL(0x9070014b,0xbf482380),LL(0x9ee025e3,0x0fc3aa99),LL(0x63b81901,0x508f2832),LL(0x84535f01,0xbbddb4f2),L_(0x000000aa), + LL(0x9b68b1b0,0xc08c3333),LL(0xc47488c1,0xab449170),LL(0xb2a5ff61,0xcf64ef35),LL(0xb9e5f70f,0x7b502c46),LL(0x417cfca7,0x88cbf950),LL(0x1cf7bedc,0xd4ad741c),LL(0xf0d8aab2,0xbcf1b6ad),LL(0xf5a48560,0x219e66fe),L_(0x00000180), LL(0x502beb84,0x2cc6cdcb),LL(0x584dda42,0x094da6af),LL(0x129f5a40,0xd42b7286),LL(0x1002d93c,0x5c173674),LL(0x725b278c,0xefa77633),LL(0xd75f98ff,0x791931d8),LL(0x21527308,0x512e096c),LL(0x53d3565e,0x6d15d8de),L_(0x000001f3), + LL(0x00019394,0x62bec3a2),LL(0xa2c579a3,0x5ab884c7),LL(0xa06cf4c6,0x083cab2b),LL(0xe7eda004,0xf5f6cd3e),LL(0xa107bb12,0xe8649e92),LL(0x492e8a0a,0x5a344683),LL(0x91fc3fba,0x8dcfb242),LL(0x94cfd171,0x12659c38),L_(0x00000028), LL(0x03751b1b,0xce223739),LL(0xc2f1043f,0x23da45da),LL(0xf9307b4a,0x3110105d),LL(0x2b6d1b2c,0xb39e6a89),LL(0x2f645131,0xd8e317be),LL(0x90466331,0xb2f395de),LL(0xf1e436a0,0x1cd56345),LL(0x547fe748,0x3e61e1b0),L_(0x0000003a), + LL(0x9c18c216,0xfd251565),LL(0x18071963,0xf23633a9),LL(0x005c3056,0x1e0819de),LL(0x46ebbb94,0xb549b1c1),LL(0xb17a6d7d,0x92577078),LL(0x991b825e,0x8f6ca39e),LL(0x98671ca8,0x90ccdc4b),LL(0x393fb1b5,0x83f03d02),L_(0x000000da), LL(0x3e2fd629,0x150480ec),LL(0x0aca508b,0xcbfc5f51),LL(0x0864f022,0x14650672),LL(0x3b3979b5,0xbff63c37),LL(0xb97ea4a4,0x3b1cb3cc),LL(0x3ef296ec,0xcb4a6db9),LL(0xff3bd186,0xbd5b637b),LL(0xedab5309,0xde8dfc52),L_(0x00000196), + LL(0xba62ee82,0xd77204b3),LL(0x77e6a3df,0x710f93b4),LL(0x02f4ab22,0xcbb03368),LL(0x5e0ecd43,0xb189c2ee),LL(0xc2847328,0x9bdf60eb),LL(0x0ebd36bf,0xb67d93ed),LL(0x0c17c388,0x70c2279c),LL(0xe8fdeaec,0x93ea44d6),L_(0x000000e4), LL(0x48b19e4c,0xc3191f20),LL(0x04206c4e,0x8bdfd660),LL(0x9a16a00f,0xac2112af),LL(0x61997c61,0x8ab3667b),LL(0x7b84c760,0x06bdff10),LL(0xf4e1645e,0x607df1e8),LL(0x352d17a0,0xa63979fb),LL(0x83c7eb22,0x015ff16b),L_(0x0000011f), + LL(0xe2ef6187,0xe76e649d),LL(0x50e4618d,0x6b987f60),LL(0xc357d1ec,0xad4f8231),LL(0xd4a60be3,0xb315fdb0),LL(0x2eced057,0xa9fb56ad),LL(0x19319996,0xa2da27cf),LL(0x2f11dee2,0x916ddbdc),LL(0x7e40b2e7,0x77406459),L_(0x00000044), LL(0xb7a77c12,0xb4cf9881),LL(0x7da427fd,0xcc7a5c06),LL(0x80b0089c,0xdef4d8fc),LL(0x93e7089d,0xe325b955),LL(0x16ead4d1,0xb039b2a1),LL(0x5dfcb305,0x1820d74b),LL(0xe64404b5,0xd5916176),LL(0x6e3b8d75,0x67136fef),L_(0x000001b2), + LL(0xe5919bbd,0xff1cd958),LL(0xef53a31b,0xf4f94077),LL(0x95740f28,0xd731ab60),LL(0xae2285d4,0xa1ecbb47),LL(0x8c0691df,0x258e27d0),LL(0x3a34371a,0x6fcf67a4),LL(0xe81db787,0x31ad117e),LL(0xcd7a8c18,0xbef4d701),L_(0x000001ea), LL(0x1c27e10f,0x7aa7d331),LL(0x36d353fb,0x484bc381),LL(0x5feb0814,0x14c00e62),LL(0x5a897087,0x4174c50b),LL(0x73330526,0xa9a0af61),LL(0x7d611726,0x60873261),LL(0xca260561,0xe43e2bb4),LL(0x9e169695,0xd6fd57ef),L_(0x00000191), + LL(0xb1e778d1,0xe387f978),LL(0x9b9aee5d,0xd76337e3),LL(0x92cf14d1,0xe5ccf490),LL(0x72921fc6,0x5b43aefb),LL(0x65d3e5c1,0x795ec0f9),LL(0x3904e5d3,0xa4928380),LL(0xa5751b73,0xdfd26a4c),LL(0x351de543,0x772f0a98),L_(0x00000160), LL(0x30b62509,0x2b0ddc19),LL(0xf6f4cca7,0xe118ecd8),LL(0xed5b6edf,0x0c9d677b),LL(0x4f5d7b07,0x34b851c9),LL(0x67daaa19,0xade6f3a0),LL(0xf4cef90c,0x4e977f04),LL(0xddf591d7,0xa5a8e05f),LL(0x51707198,0x73e4b240),L_(0x00000069), + LL(0x8fe855a9,0x6f37b397),LL(0xfc53c756,0x11e9e126),LL(0x0be45fa0,0x72b050d7),LL(0x0064ab05,0xfc69a454),LL(0x2c4f8ec5,0xe8ca3f9e),LL(0xfd23251a,0x3713bf5d),LL(0xfbba4b98,0xf9f17348),LL(0x022b3ded,0xf5712705),L_(0x00000075), LL(0x496ec6da,0x02b42501),LL(0x3666ead7,0xc5995675),LL(0x03f0c9c9,0xfafe4fc1),LL(0x92c23f3c,0x6e5dcc14),LL(0x49d6013c,0xa80274fd),LL(0x6faecdf5,0x3b89a119),LL(0x6951f782,0x04ebafd5),LL(0xe5c75fdf,0x98ee2e59),L_(0x000001be), + LL(0x4c63dd71,0xecaaf994),LL(0xdb563551,0xfa0452c4),LL(0x034243a2,0x35dd1a49),LL(0xc1e98b35,0x67ebb1d1),LL(0xd64a14c8,0x65aa32b2),LL(0xa40fe3b8,0xf36b9b35),LL(0x74b94620,0xde048775),LL(0x9c4f81d6,0xefb999ff),L_(0x00000048), LL(0x317a309b,0x27e42884),LL(0xea18f52e,0xab7b0d80),LL(0x6a303cb3,0xd883eedd),LL(0x6960a553,0x5fa720c0),LL(0x4a30771c,0xaf1c642d),LL(0x8c3afce3,0x0aea4951),LL(0x5b832460,0x8daa7e14),LL(0x3626a795,0xa9434a86),L_(0x00000042), + LL(0x5078f4b7,0xf8bc0fa4),LL(0xf5948727,0x1eb36b7b),LL(0xea0106cf,0xc704268c),LL(0xc80e06d4,0x300186e8),LL(0x5c216323,0x6e94d507),LL(0x72ccb301,0x882e86c3),LL(0x053b8cfd,0x3148a28a),LL(0xd1710019,0x864bf343),L_(0x00000010), LL(0x24c02876,0x6d8c63b2),LL(0xc0738276,0xb8220989),LL(0xd418521b,0xa59b967c),LL(0x52d761e7,0x601efc41),LL(0x0fa019e4,0x41cb4d7c),LL(0x9d52f11b,0xcac0131d),LL(0xe27a162d,0xaac991e4),LL(0x975ffa76,0x5c007789),L_(0x000000c9), +}, +/* digit=86 base_pwr=2^430 */ +{ + LL(0x53760212,0xe9ad4077),LL(0x52251dd3,0x2f89c76c),LL(0x5957d5e2,0xb5844aaf),LL(0x57c28051,0x630bd361),LL(0x369eb597,0xacac6af3),LL(0x5611a387,0xaad0ef87),LL(0xde2edc32,0x0bccba64),LL(0x731d170b,0x8f5a6ff1),L_(0x00000006), LL(0xc2557f66,0x48e1ee52),LL(0x5f44f645,0x0cf95561),LL(0x64b6036a,0x1e52bf2c),LL(0x89213e75,0x34bd858d),LL(0x15aa570d,0xbf1cb2e7),LL(0xcc205ae3,0xf29e703a),LL(0xfb1911e3,0x98a34a47),LL(0x78616fc4,0x432ae895),L_(0x000000be), + LL(0x301fc12e,0x91d6adbf),LL(0xdc42e25e,0x587a0d31),LL(0xcecd10d8,0xf88cc8a9),LL(0x8d1b41cc,0x90bd959d),LL(0x33b3f964,0x999c5a51),LL(0x47bee028,0xc3a08825),LL(0xb39f5f74,0xd4fc3438),LL(0x5af39ed4,0x4a0aafa7),L_(0x000000f9), LL(0xbf02b54c,0x0a29ee5f),LL(0x4f0d6dbd,0x57045bbc),LL(0xe6fde293,0xc7d0fce2),LL(0x040027df,0xcda45a33),LL(0x23be51ff,0x9841c14d),LL(0x6d045486,0x56ff1e33),LL(0xfdc85221,0x5ea634d9),LL(0x619b696b,0x35b3b3ae),L_(0x0000003f), + LL(0xa104ae49,0x5ed4f918),LL(0x7f6fea1f,0xfb0d6721),LL(0xb83838c1,0x99caa790),LL(0xd116daa3,0xc2f1d19e),LL(0xe68d0873,0x105c2468),LL(0x247798a8,0x672f66e0),LL(0x740bca2a,0xe789f0d7),LL(0x86c229b4,0x505a9b4c),L_(0x0000007d), LL(0xd8e83bf0,0x77be9153),LL(0xaa767cd0,0x71074af9),LL(0xb3e46e21,0x17f8ca53),LL(0x7796d98d,0x215b3c92),LL(0xd098831f,0x362429bd),LL(0x0b75e1e5,0x75d41b1b),LL(0xc1e2cba3,0x0101e963),LL(0xbf3622fb,0xa38f2cfe),L_(0x00000193), + LL(0xb9019f2f,0x646b70de),LL(0x2e533666,0x90a5717d),LL(0xd38f5274,0x7916810f),LL(0x3bb33bad,0xfd952623),LL(0x59b6b88e,0x5dad71c0),LL(0xfe6ed574,0xca300788),LL(0x21a98ac4,0x6ded4a6a),LL(0xeeff5891,0x7d6ffe10),L_(0x0000018d), LL(0x48da52e8,0x98c1a5fc),LL(0x1901e7d3,0x42b185c0),LL(0xcb2700b0,0xaca4eda6),LL(0xeec867cd,0x0a19ae4e),LL(0x7dcb7e93,0x88471dcc),LL(0x17e010a2,0x198ed175),LL(0xbb36f683,0xbb4e8756),LL(0xf915ed36,0x4cf9f27d),L_(0x00000007), + LL(0x56948883,0xfd191432),LL(0x519679d8,0xcdcf1fa3),LL(0x35711035,0x87a0e867),LL(0x663b1a45,0x6c52192e),LL(0xd46a0d9f,0xd90e9a02),LL(0x01cece65,0x0dbf88f7),LL(0x5032d5f7,0xd5855448),LL(0xc3ecbbe3,0x330d4483),L_(0x00000113), LL(0xda71595b,0x4bb4a8ce),LL(0x6609b74e,0x23abfae6),LL(0x76172b1d,0xc43ff5b4),LL(0xc2d69d21,0x275e500d),LL(0xf9af38b2,0x8ab54bd9),LL(0xe6a2d266,0x983f2f3c),LL(0xb11e43bc,0x0d27f996),LL(0x3ef88279,0x33f71741),L_(0x000000e5), + LL(0x9058183f,0xdd40e348),LL(0x8259229c,0x5e67db56),LL(0x8582282f,0xa5fd9ed5),LL(0xea641a1b,0x1a634c9e),LL(0x6d89c3eb,0x44359789),LL(0x3d617332,0xb7458f8e),LL(0x974b6efb,0xb6863988),LL(0xd2857c41,0xae43a38e),L_(0x00000046), LL(0xd0bd1dc8,0x08757c7c),LL(0x4edda34c,0x5a31fc77),LL(0xfa2bc7b5,0xf6b159cc),LL(0x36e4a532,0x904f6b01),LL(0x603edf4b,0x96a6ebff),LL(0xb699fdb4,0xad76014c),LL(0x888c04c9,0x52da1b11),LL(0x9675ab25,0x8c537551),L_(0x000000e1), + LL(0x76cc2e7a,0x23209d8a),LL(0x39ac3634,0x493f6e5e),LL(0x2a49b5d9,0xaac1e790),LL(0x83449f3c,0x85d5aef7),LL(0x919c353b,0x45144182),LL(0x9673a9fe,0xc1d4f2ca),LL(0x9f1f328f,0x8f5eeda5),LL(0xf2216169,0x611eca42),L_(0x000001dc), LL(0x3f509f9a,0x96feece0),LL(0x85e87ba8,0xe9be0b9b),LL(0x186dcdb0,0x7207011b),LL(0x590eaaa1,0x60f3a8d4),LL(0x3aeb9b3f,0xac89f276),LL(0xd21f7163,0xfd4e5813),LL(0x05998e42,0x80b7e4e3),LL(0x38cd8e87,0xe79a0406),L_(0x0000010c), + LL(0xc09d95ae,0xd6f73873),LL(0x7cafdfb9,0x27b3bc9e),LL(0x1f14d40f,0x6fc05fa8),LL(0x4eebf4e6,0x4e6b76ed),LL(0x7bb6036a,0x736cb738),LL(0x58e19e3f,0x2277b08a),LL(0xa4986beb,0xbb55c0d7),LL(0x8f830ada,0xa22d04d1),L_(0x00000057), LL(0xb25e59f9,0x9a3ae563),LL(0x18997059,0xd06e16bf),LL(0x7505891e,0x3d688dac),LL(0x76b3eb97,0x10cb084c),LL(0xc6498b2d,0x2d6e8e95),LL(0xafa574af,0x503f71b8),LL(0xc16133ca,0x9966999b),LL(0x6f3443fa,0x6ad79bd7),L_(0x00000104), + LL(0xa62c57f7,0x797a0980),LL(0x03e966f7,0x3a33197c),LL(0x5565e857,0x31df23bc),LL(0x1feaa7a8,0x240894a7),LL(0xec48825c,0x8f886413),LL(0x9a2b6b14,0x98c1f268),LL(0x041a9538,0x89254981),LL(0x46c27078,0xd4a923a7),L_(0x000001b7), LL(0x3f850405,0x16453f02),LL(0x5e4fd155,0xb6a3db9b),LL(0xcd6306e4,0xc259c564),LL(0x89b34c35,0x1cbd8b5f),LL(0x7420eabc,0x22f6885a),LL(0xfe106029,0x2bf9ec49),LL(0xaef5fd1c,0x73da5ef8),LL(0x01c0496b,0x2fbf80d0),L_(0x000000d2), + LL(0xccf43b6c,0x2b2fd87d),LL(0xb29b9f9b,0x42c15930),LL(0x75dead34,0x8a992ae9),LL(0x93d238f6,0x835b46d8),LL(0x537768fc,0x59b7cdac),LL(0x97a9b4f9,0x8690f7d0),LL(0xe523d489,0xddd3b40f),LL(0x2dcb85bf,0xd07f5307),L_(0x000000d7), LL(0x2a6612ff,0xa015a7b6),LL(0xc599e46f,0x72d01eaf),LL(0x9b23b6a1,0xc21d7f10),LL(0xdff644dc,0xac717daa),LL(0x215cc0c4,0x31113eff),LL(0x29fa625d,0x2da218c6),LL(0x3ddc80ac,0x8399eca4),LL(0x5d17f283,0xce8d5743),L_(0x00000053), + LL(0xfd958a4c,0x656636ff),LL(0x022fd87f,0x77efdadd),LL(0x3a9ac51c,0x96597b30),LL(0x2fbf14ce,0x01364596),LL(0x7d940bec,0x1f7b85dc),LL(0xbde90f3c,0x3116d774),LL(0x40e30d60,0x9289d920),LL(0x52c8b93a,0x02c341a8),L_(0x0000001e), LL(0x2ebd850f,0xa0eaa3d4),LL(0x590d1f54,0xbe22470c),LL(0x53e4e7c3,0xf88fc1e8),LL(0xd8883aaa,0x7782a946),LL(0xe78d366f,0x567fbeed),LL(0xf47e81a5,0xadd3d4f0),LL(0xb2aaddfe,0x29a661cf),LL(0xa342a4e6,0x748d5227),L_(0x0000007d), + LL(0x8c9cc5a1,0xe5aa101a),LL(0x329070ed,0x53342a7d),LL(0x852398fa,0x007bb4fe),LL(0x673ceb9c,0xa9431236),LL(0x3c9247b3,0x1cb68268),LL(0x115b1ee5,0x5c07f2ef),LL(0x3f6486b7,0x7183b014),LL(0x1585ec37,0xe601ad10),L_(0x000001fb), LL(0x0680da54,0x9a2e8c00),LL(0xe41a7e56,0xe94c0472),LL(0xb7c713ec,0xf2aa41c1),LL(0x742d18cc,0x3e3162e5),LL(0xedf3c376,0x77417a84),LL(0x57916b99,0x165b44c7),LL(0x246f2e2f,0x1c45d988),LL(0x8287828d,0x99db995c),L_(0x000001b0), + LL(0xab8f038d,0x27f0c816),LL(0xc4edbd37,0xc8522ec7),LL(0x2f8f0b54,0xe399477b),LL(0x074481dc,0x0fb69890),LL(0xa0499848,0xa5280b6b),LL(0x88cb4213,0x3dff5cd9),LL(0x41c11365,0xe549bd1e),LL(0xea8a3c58,0x40ee92fa),L_(0x00000039), LL(0x2b200b5c,0x2ec791da),LL(0xb1772087,0xb33233b1),LL(0x9df4451f,0xaa1032a0),LL(0xbaa72f9e,0x645358e3),LL(0xc7982461,0x88b9794b),LL(0xecd3c965,0xcecd6313),LL(0xa2d9359f,0xf7b6ca2d),LL(0x7f382df2,0x98e1c5ce),L_(0x00000033), + LL(0x02037c91,0x41a52ada),LL(0x82ddca3f,0x1b403720),LL(0x4b6a1108,0x6acc9873),LL(0x6d306779,0x495190be),LL(0x80fb3f34,0x13ec341a),LL(0x0307762f,0x2ea03fd7),LL(0x422a6d1a,0xfacb4f11),LL(0x014521e4,0x5f9816f6),L_(0x00000196), LL(0x0ac6d565,0xb343a25b),LL(0xdead79aa,0xf7c846e5),LL(0x54b1471e,0xccc5d545),LL(0xf9ccfd6a,0x31f5ccd5),LL(0x43e993a3,0x8fc6e767),LL(0x07c79ed2,0xd716a9ab),LL(0x4894cc48,0xae33e7f3),LL(0xd6c5646c,0x3fa06c4f),L_(0x0000010d), + LL(0x37b15d9b,0xc16b5265),LL(0x43945dbd,0x7a53a14b),LL(0x994df966,0xd272ba24),LL(0xfbfe62a0,0x80e0b451),LL(0xb488717c,0x7153f565),LL(0x22303a33,0x6397afa8),LL(0xc85bf638,0x0216bd4f),LL(0x096b646c,0x50f84189),L_(0x000000f4), LL(0xf00721ec,0x500e6e18),LL(0xab5ca93b,0xea2d8a0d),LL(0x06a3b0f0,0x44feaba9),LL(0xf766ff9a,0xea82aa95),LL(0x05be709f,0x0829c7c9),LL(0x36a07ae0,0xa1cfe409),LL(0xc83c032c,0x780ac746),LL(0xde7f9ef0,0xfedfbfdb),L_(0x000001e2), + LL(0x18f9c015,0x25767d44),LL(0x49d6199f,0x77c536b7),LL(0xb54e847b,0x1af54bfd),LL(0x7979776c,0x0b838623),LL(0x51eefa22,0xdf9bb4a7),LL(0x028f18ac,0xce45beb5),LL(0x7dd86218,0xb930f98a),LL(0x3f055e3a,0x6380d302),L_(0x000001fb), LL(0x1df321b7,0xeeb57c14),LL(0x53902659,0x0b2255cf),LL(0xf2a776fd,0x4cc9dbec),LL(0x453cf8ab,0x63e94ee6),LL(0xf2d56478,0x93a4007a),LL(0x027149fe,0x9cf116d0),LL(0xa6376053,0x17dc8184),LL(0xe7465f73,0xfed4461d),L_(0x00000058), +}, +/* digit=87 base_pwr=2^435 */ +{ + LL(0x6ea55f73,0xf042fa2f),LL(0xab1d226e,0xb24c1848),LL(0x6862a1dd,0x413acbe8),LL(0x1f4168e7,0x91408365),LL(0x9d596e07,0x23961d18),LL(0x01b379ca,0x6d536797),LL(0x05ec7b7e,0x13cf35fa),LL(0xd7f6b707,0x182dcd20),L_(0x0000009d), LL(0xbc18785b,0x76095f2d),LL(0xa0054386,0xe28a0370),LL(0x50c89610,0xfeeaf09e),LL(0x144bba0b,0x455cf10f),LL(0x34cf6dd7,0xf509d978),LL(0xf94fe722,0x05c279e5),LL(0x8092debb,0xe71244fe),LL(0xb314f061,0xe0f63153),L_(0x00000098), + LL(0x7f803868,0xeb771971),LL(0x05b4c2dd,0x911e1ad0),LL(0x0df34f87,0x57076f1e),LL(0x9958d5da,0x6f49ecb2),LL(0x55d1ebf6,0x2ca7b49e),LL(0xfcb4f571,0xb2ff1b32),LL(0x42a971c5,0x49f97d3d),LL(0x838bb327,0x5fd29804),L_(0x0000017f), LL(0xa52e7908,0x86529c3b),LL(0x7ddd9af7,0xe7159473),LL(0x6dd64d51,0x8305400f),LL(0x922bf016,0x21ca4239),LL(0x1db4bbaa,0x8c94ee85),LL(0xafda935d,0x80623440),LL(0x0a576c9c,0xd9110efc),LL(0xd79f58dc,0x3d6ca396),L_(0x000000f1), + LL(0xe1a7c929,0xeff94575),LL(0x7f265399,0x076c7864),LL(0xdf2046cc,0xfe889ff0),LL(0xe23267f8,0x4b83615e),LL(0x171f661f,0x32637340),LL(0xc3fd3e6c,0x946ccba9),LL(0xafb9463a,0x9deb55e5),LL(0xbdcb574f,0x8d44f742),L_(0x00000092), LL(0xd84d6148,0x6362d37c),LL(0xe1724e4e,0x80e307d8),LL(0x2dee8134,0xd661095b),LL(0x596062c6,0xbc1b3be1),LL(0x5779751a,0x36f36543),LL(0xd0be963f,0x7fee02b0),LL(0x75c65486,0xb82030d1),LL(0x72d27424,0x6d7f4a46),L_(0x00000043), + LL(0xaeb38ccc,0x49ee6062),LL(0xc1411d18,0x164333de),LL(0xa4a4727a,0x566e81b4),LL(0x74b241dc,0x290aa59c),LL(0x85069fb7,0x2865cb6d),LL(0x28389d32,0xcbd64839),LL(0xce3c8f7e,0x8c909864),LL(0xfef248d2,0xf2befac9),L_(0x0000013b), LL(0x4261d435,0xb96bec85),LL(0x62a6f5a6,0x1f509bbc),LL(0xb43f9c90,0xc53ddbc8),LL(0x94118466,0x9ad3885e),LL(0x22a38677,0xd109dd2e),LL(0x60b6db58,0xeef4f2af),LL(0xadf1adca,0xd5bc0cd7),LL(0x0fb47811,0xc4b3b822),L_(0x00000024), + LL(0x456d2584,0xca189af8),LL(0x85bb0ebb,0xf9e4ffa5),LL(0x3e647534,0x1136b7ab),LL(0x0e1d213d,0x91ac330c),LL(0x6aa7c7de,0x4345ad0a),LL(0xf0255a49,0xa156a357),LL(0xe3967bfd,0x2bdb3ace),LL(0x8dff208b,0xc09a53d8),L_(0x00000009), LL(0xad93b86d,0xff9685e0),LL(0x990e1837,0xd599bc9a),LL(0xa491c185,0x320375e6),LL(0x59cacf47,0x427b05e7),LL(0x03c76cd9,0x9565d0db),LL(0x073ece2e,0x72f50d14),LL(0x2045534c,0x31281552),LL(0xa43a812b,0xabec2314),L_(0x00000152), + LL(0x49f1b4d9,0xd4601e45),LL(0xd37e9635,0x5712ad28),LL(0x2c3143dd,0xaf7e19b4),LL(0xc6366f04,0xaa565afd),LL(0xc4b34637,0xc12b452b),LL(0xdd135b2a,0x77fe7f5b),LL(0xfeea8b42,0x9ec6ff31),LL(0x5cdaec8e,0x73ab00c4),L_(0x00000165), LL(0x3f781317,0x664c221b),LL(0x652c3e83,0xafac4ecd),LL(0x6da93d03,0xcfa7c466),LL(0x2ceac0d6,0x039d2b65),LL(0xcb1a4cf4,0xa6eb1946),LL(0xa2285c58,0x422c9b53),LL(0x1b4d8367,0xcc7349ed),LL(0x4c55a379,0xcd5e2afa),L_(0x00000106), + LL(0x85b7a89c,0x1ea7ad0b),LL(0x9f47c4ab,0xe5150449),LL(0x85cda5f6,0xb891dfa1),LL(0xe69386b4,0xd89edbbe),LL(0xf4ac48bb,0x1b2a00ae),LL(0x3c163ff2,0x37bb423a),LL(0x73d0c87f,0x393a234b),LL(0xe80b0fdc,0xc63497ac),L_(0x00000153), LL(0xc52fc82f,0xa9154aa9),LL(0xe56ec070,0xb5b87f0b),LL(0xb996ced1,0x2c118a27),LL(0xbb7f2dff,0x3e0161d2),LL(0x9ee991cd,0x45e9acd1),LL(0xb300f7d4,0xfb934e98),LL(0x20f357aa,0x370589a8),LL(0xfe8af1f8,0x7ca491b4),L_(0x00000106), + LL(0xb59ee173,0x42f467a4),LL(0x41b66fb9,0x5770a301),LL(0x02513df6,0xf1d41988),LL(0x7e0148f2,0xf54abf2c),LL(0xd9a1b6c6,0xbb47b51b),LL(0xfba956f2,0xf5846505),LL(0xb02618f4,0x502a3ddc),LL(0x69ec8c64,0x8021ca7b),L_(0x00000158), LL(0x194b6953,0x04316d8f),LL(0x8f7bcb08,0x006107c5),LL(0x9f43afd0,0x32f310f2),LL(0xa15ea5dc,0xe2b91ae7),LL(0x3849a363,0x2b4966c6),LL(0x6a457445,0x1d63455b),LL(0xb8835c17,0xcd39b535),LL(0x1ae86f54,0x52463509),L_(0x00000075), + LL(0x413915c5,0xd3aa0f40),LL(0x553e50ff,0xc139c1fd),LL(0x408079ac,0xcaeedf51),LL(0x5702513c,0xf43dc271),LL(0x1c08e5b0,0xf5e1208e),LL(0x48d91655,0x813375b3),LL(0x91b427b6,0x0fa6be8b),LL(0x833896b7,0xa0825ed0),L_(0x000001b9), LL(0x7c3676f3,0xac9ff585),LL(0x853930b5,0x94c9266b),LL(0xc6b73b6c,0xbc211c89),LL(0x277b6c8b,0x2fc248c5),LL(0x93fd3dec,0xcefb839e),LL(0x4a5c85d3,0x10bf217b),LL(0x2a276f95,0x3a708326),LL(0xc15a4206,0x85899d47),L_(0x0000017e), + LL(0x7bde1ec8,0x34c4db4a),LL(0xa70a6f08,0x48a29f6b),LL(0x587d1015,0x49dbe231),LL(0x998c9a20,0x8aceafbb),LL(0xe5eedcf8,0xe6b738f7),LL(0xcab08878,0x9b693ecf),LL(0xb374ede1,0x008dd1dd),LL(0x7a6cd94f,0x9b54b3be),L_(0x000000c2), LL(0x83bd130c,0xc20757ed),LL(0x17bff343,0x228e06b2),LL(0x1bef19ad,0xc51046a9),LL(0x0e88da5c,0x011b5840),LL(0x70a9f961,0x49ae6f04),LL(0xc6f83f90,0x9d079a03),LL(0x912f072e,0x9a401435),LL(0x0b78fe3f,0x45e24749),L_(0x0000002d), + LL(0x20b6a171,0xc0eec582),LL(0x531fdbd5,0x7a0400cc),LL(0xe2cd895a,0x9c308aa1),LL(0xf0bb2d5a,0x08b021d8),LL(0xde68db65,0xad0f2bcd),LL(0xac747060,0x3fd807ea),LL(0xfe64802d,0x8b4648ee),LL(0x38d8773a,0xf5c28376),L_(0x00000103), LL(0xcb637daf,0x16605703),LL(0xdffc80b5,0x4845eee4),LL(0xd28e60ca,0x57f98d2e),LL(0xdfda4fa3,0xac95f77a),LL(0x0d0b9220,0x05da7201),LL(0x60ec2cb7,0x9a49b6df),LL(0x201396c1,0x611b2a93),LL(0xb535c0ce,0xce6fc00f),L_(0x0000018b), + LL(0x1481fdff,0x68c14a66),LL(0x84edc501,0x764d4d2b),LL(0xbe356501,0x736aaa60),LL(0xe6771588,0x0c40e330),LL(0x5714d50c,0x7b8fe887),LL(0xfbdf2915,0x9215aac4),LL(0x549e25da,0x58b4091b),LL(0x442d2b00,0x48bf91ad),L_(0x000001e8), LL(0x88ee6a91,0x934568ba),LL(0x307efd84,0x10dd1585),LL(0xcb644b24,0x3219b046),LL(0x2bd376e1,0xef0c68dc),LL(0xdcecc49b,0x56c2d2d1),LL(0xc907f765,0x810f8810),LL(0x052b3ddd,0x1ba20da5),LL(0xc3448a3c,0x0827079e),L_(0x0000018b), + LL(0xb0648b7e,0x193cb76d),LL(0xf7fec768,0xb90f6558),LL(0x86126ace,0x237a7fc7),LL(0x749a6fe8,0x83ab837e),LL(0xb5c3035a,0x3e9ae2d9),LL(0x8de4bf68,0x9c620970),LL(0x0b3fa791,0x8ef69888),LL(0xc5e8388c,0x3a44205f),L_(0x00000013), LL(0x892086c5,0xc04ac82e),LL(0xfb491292,0xc0d38a50),LL(0x52d706d1,0x42c8a5e7),LL(0x582ce44e,0xc9853494),LL(0x96312a80,0x04da6643),LL(0x74ef5508,0xbbc8dc30),LL(0x9a8e3322,0xfa669919),LL(0xa1f29644,0x94a108d5),L_(0x0000011e), + LL(0x0e084366,0x5cc0a55e),LL(0x7ef0187f,0xa139c3fe),LL(0x49d53f7b,0x5423f2e7),LL(0xd809a727,0x5a94a4ff),LL(0xe2e74f9e,0x541f08d0),LL(0x22541929,0x14dd0793),LL(0x49159841,0xdbf53ad1),LL(0x408f5bb7,0xb5b9e78a),L_(0x000001e9), LL(0x1db74ade,0x46e28711),LL(0x3b22deeb,0x1ad3605f),LL(0x8863541b,0x1fe070fc),LL(0xfdd530f5,0x4af47e93),LL(0x7f3d69c0,0xd93cb647),LL(0x6d16f551,0xbc684cde),LL(0x50cd6852,0xb5154a9f),LL(0x16ac0cc2,0xc9ac9e9f),L_(0x000001bb), + LL(0xec5c2c31,0xc90def9a),LL(0xe4b46e4f,0x67d0d316),LL(0x8901b941,0x2bb65bec),LL(0xc96e5167,0xf836eba2),LL(0x096ab2fb,0x996167b5),LL(0x2719f2e6,0xdde8e72c),LL(0x12437287,0x62d48d9a),LL(0xf4ef64e5,0x405cdb88),L_(0x000000f0), LL(0x91da5b6b,0xa56f46e1),LL(0xc13b6841,0x97107435),LL(0xfbe3e2a6,0xa446c520),LL(0x759315a1,0x0c5bba8d),LL(0x861aec20,0x852f2659),LL(0x775fa0ec,0xfbe06684),LL(0xa91ab0fa,0x03bd8b0d),LL(0x3006d391,0x3166b023),L_(0x0000010d), + LL(0x1b6190bc,0xa4a386bf),LL(0x17c47de2,0x95703cbb),LL(0x3bf84891,0x3f013d22),LL(0x12474267,0x6fdb827a),LL(0x0290f2b8,0x50e9b7e1),LL(0x79a8f44e,0xcc658260),LL(0x89a9228e,0xab4d12b5),LL(0x83a119d1,0x121131b6),L_(0x0000019e), LL(0x2d25950f,0x6b02229a),LL(0x38c46b7a,0xfb0617d5),LL(0x6bc581dc,0x0ce1dd7b),LL(0x6b522d59,0xd0dcdf5b),LL(0x9133e3f5,0x5cce47e7),LL(0xd71f5bdf,0x21b8ecd0),LL(0x17d9aefe,0x7aac21b7),LL(0x7b609025,0x98c6eb88),L_(0x00000127), +}, +/* digit=88 base_pwr=2^440 */ +{ + LL(0xf89fcba2,0x59a63f98),LL(0x86d07ca3,0xf60025c0),LL(0x590915cb,0x68c18d4e),LL(0x15cc7c3b,0x85575ec9),LL(0x09334801,0xe8d10d82),LL(0x4789511a,0x82704b90),LL(0xdb2e76c0,0xf6a4e997),LL(0xf5824d99,0xb143d8c3),L_(0x0000006b), LL(0x6953628d,0xa7575550),LL(0x8504400e,0x8537e141),LL(0x609d8295,0xc7b7f7a0),LL(0x5da70118,0xc50379c5),LL(0x79ad1223,0xc936f6ea),LL(0xbde48629,0x4f7f839c),LL(0x1ba01725,0xdff8def6),LL(0x1bef09eb,0x65b93f5c),L_(0x0000011e), + LL(0xe82eeedc,0x330d665a),LL(0x0753a4f9,0x3f5e64a3),LL(0x9e477096,0xef9e92f3),LL(0x07f9d297,0x388062aa),LL(0xc48c3ddf,0x60ab0df5),LL(0x55e6e61e,0x5a47567e),LL(0x9872a6f9,0x3a66d012),LL(0x425f368c,0xc7e83953),L_(0x000000bf), LL(0x03b7cc7b,0xb2825eba),LL(0xba3cef16,0x90e67535),LL(0x4aec5704,0xcc34aef1),LL(0x511ac67b,0xd95c0e01),LL(0x51002739,0x0f4f3657),LL(0x45e92922,0x465557ab),LL(0x1baabf91,0x0e9abecf),LL(0x8337c976,0xdff48fe9),L_(0x000001ec), + LL(0xa1025751,0x6a01039d),LL(0x28499cde,0x5ba84622),LL(0x47232500,0x4da34907),LL(0x523417ab,0x54b07c1a),LL(0xd3451baf,0x3fa7e4ff),LL(0x7ce5516f,0x2fbff214),LL(0xfc522cc4,0xa33f1b0a),LL(0x95c7010c,0x664b5cb3),L_(0x000001a5), LL(0x665ed5f5,0x980e5684),LL(0xd596415b,0xa5a1b30b),LL(0x8834a37b,0xfeebb04e),LL(0xcf282494,0xb29d17be),LL(0x340dc6ce,0x8d5399a5),LL(0xa50f4a86,0x76012bce),LL(0x83faa312,0x4bc769aa),LL(0x6550a065,0xea38cd8e),L_(0x000000b5), + LL(0xec66fa0e,0x134a53f7),LL(0xa7b2871d,0x1ee39cda),LL(0x83070c04,0x9749b3f1),LL(0x6da77991,0x867841c7),LL(0xe916f1eb,0x21e5438a),LL(0xe409b274,0x1b0e12d8),LL(0x3842a6e5,0xde5e08b8),LL(0x74b9e008,0xb7828943),L_(0x000001e9), LL(0xfea4cba7,0x08e07acd),LL(0x06789133,0xdb2143a9),LL(0x815c887a,0x85ffe6dd),LL(0xa9d2043c,0xa68d05e2),LL(0xd3ceab79,0x93674d33),LL(0x7a8a9863,0x12ee73ca),LL(0xd54b7afd,0x6403b9bb),LL(0x2eead112,0x4c680b70),L_(0x000000c7), + LL(0xfa0b987f,0x67c06145),LL(0xfd55dd43,0x34438e79),LL(0xe8ca9c52,0xfad0f9f8),LL(0x810f12c2,0xa97a7136),LL(0xb3ec5af1,0x3d0eabab),LL(0xb7b58561,0xecb3da01),LL(0x8aadf26a,0xbb015079),LL(0x9cce9cad,0xf01aa802),L_(0x000000af), LL(0x265a72e5,0xd025e951),LL(0x90e3ddec,0x5b2c9143),LL(0x4955e972,0x05386478),LL(0xdae63ed0,0x60c28f8c),LL(0x4aa5ded8,0x0fb99e77),LL(0xb74c1dd8,0x0f07854b),LL(0x2caae0f2,0x6691581c),LL(0x069f6ba7,0x072c0d48),L_(0x00000087), + LL(0xf5f13583,0x8b738df6),LL(0xe91f4420,0x786c7341),LL(0x1ae2188a,0xda384ed9),LL(0x08e3293b,0x19b1a00b),LL(0x9e09af31,0x65267666),LL(0x322f3662,0xd07b9f37),LL(0x764ea40f,0x11ae129d),LL(0xc16a911b,0x4cc8fb85),L_(0x000000c2), LL(0x59021aec,0xeb3197c7),LL(0x4daed80e,0xdfd4a433),LL(0x606234ad,0xab3ff78d),LL(0xf98a1d73,0x9f90a43f),LL(0x2c9cac66,0x99bed176),LL(0x5e8063cf,0x8f03fcd4),LL(0x50672a22,0xfdb17bf7),LL(0x027e080b,0x5a317879),L_(0x0000012b), + LL(0x236a647c,0x5dafe047),LL(0x30081d74,0xc3212b4d),LL(0x0f548f13,0x51f94578),LL(0xd885e14f,0x941059a0),LL(0x06ed3092,0x189c478f),LL(0x5042651e,0x7a26e8c7),LL(0xf36b6ee0,0x09a14b52),LL(0x32dfdec0,0xf1c461a0),L_(0x000001f9), LL(0xbe8ca673,0x527f0a50),LL(0x4c6beb6f,0xadaba76e),LL(0xfc7fd1fd,0x9909b987),LL(0x47c90091,0x992155a0),LL(0x06d6f45b,0x2da697e0),LL(0x740de37e,0x1a38bcd1),LL(0xce3867f1,0x509b93e8),LL(0x503be8b2,0xd6b05b60),L_(0x000000f3), + LL(0xe8c16ca7,0x4d3bbd16),LL(0x519f4d4b,0x53785c45),LL(0x6454947e,0x1aafab77),LL(0xd9b9416f,0x6883b419),LL(0xb337e34e,0x0208ba8a),LL(0x7e584157,0xab67774a),LL(0x5a84d18c,0x108ac516),LL(0x77b69d31,0x73bc43a9),L_(0x00000035), LL(0x91e5bcfd,0xcd5e892a),LL(0x5aa27743,0x502744c8),LL(0xa0414bf5,0xe26bb91b),LL(0xbc4ef773,0x8bcd45f8),LL(0x8f9e301b,0x3589038c),LL(0x30d42898,0x9a5f5e5a),LL(0xa609f771,0xaf5c6671),LL(0xade09eb4,0x5287fa3a),L_(0x000000ca), + LL(0x775485ec,0xf826fcc0),LL(0xa66e99e3,0xfac7759b),LL(0x6006cfb6,0x13b284ca),LL(0x1fbb2a30,0x53d194ad),LL(0x2c2b6910,0xf54ebb36),LL(0xa49dd337,0x46a6edea),LL(0x8fc79498,0x0d6aff86),LL(0x842dc894,0x9d7094f6),L_(0x00000050), LL(0x34121245,0xb4cfd050),LL(0xafb75e83,0xaf8c43b1),LL(0x77a38e5e,0x4ff38619),LL(0xe7485f16,0xfa745e75),LL(0x7e4f2466,0xcdc30bb6),LL(0x009d4a36,0x9994c740),LL(0x25e09cb4,0x66ca76f4),LL(0xf59131dd,0x95b4239e),L_(0x0000012a), + LL(0x6edaa49d,0x0aa308ea),LL(0x7669b865,0x72c7a072),LL(0xf06e514b,0xa7ddba09),LL(0xca616052,0x126487b0),LL(0xd64c8323,0x713cc701),LL(0xb7fd1abe,0x84ce35d3),LL(0x7bfbc16b,0xba894fdd),LL(0xba61b8c2,0xe784d69f),L_(0x000000ee), LL(0x18d74478,0xa66248c8),LL(0x62b773bd,0x3c1ca3e3),LL(0xde584f76,0x5c3541fa),LL(0x475ec797,0xe7fdfe89),LL(0xd692c26f,0xe8463461),LL(0xe888fcbd,0x682f9099),LL(0x1a8aba10,0x0ae8eea4),LL(0x2cc79e0c,0x479936cc),L_(0x0000004f), + LL(0x901772ff,0x6d000499),LL(0x481b7323,0xb7a27eb4),LL(0x69fc5685,0x4a7abcd1),LL(0x47ec07ac,0xcde4a9ac),LL(0x56f5f84a,0x45545bfb),LL(0x6ef7da38,0x33e7eca8),LL(0x2edec324,0xd8a46ddf),LL(0xd29093de,0xb5fd7555),L_(0x0000005d), LL(0x163bac52,0xbeecc923),LL(0x637f0966,0x0af4893a),LL(0x1759af91,0x1d38f097),LL(0xe0aea79f,0xf9d81651),LL(0x8ae541b7,0x510d4c3c),LL(0x32bd0e43,0xd73faaea),LL(0x6891a73c,0x3864a690),LL(0x6feafb02,0xbd306219),L_(0x00000129), + LL(0xa0c16a35,0xb6681c57),LL(0x415f0571,0xb2c83a60),LL(0xcfe1f331,0x4b9088d9),LL(0x1279d3aa,0x2ab5f2f1),LL(0x29c29c20,0xd47ee149),LL(0x16735420,0xfbb44304),LL(0xb8379216,0x23034403),LL(0x20a6f133,0x9f2bb4aa),L_(0x000001d5), LL(0xe0a94e50,0x38217da2),LL(0x2ba297d2,0x18816b2c),LL(0xe566aa72,0xca63550f),LL(0x1c7b21ca,0xdfe51644),LL(0x10c887fa,0xef849ed8),LL(0x4faeda58,0xb92e8367),LL(0x03636294,0x2414c0ef),LL(0x8476a050,0x732173ab),L_(0x00000184), + LL(0x93412483,0x9f53d923),LL(0x4403bdcc,0xe30fa97e),LL(0x6c9d0aa5,0x1601e86b),LL(0x9c1a2ec2,0x19610105),LL(0x431d5f14,0x6cc0662b),LL(0xb7bbdb4e,0x84ed40f8),LL(0x266aca0a,0x1b8a27f1),LL(0x198bae2a,0xded16036),L_(0x000001fa), LL(0x33afa5e4,0x023ed8a6),LL(0x8523afba,0x036adaa0),LL(0x83cbabb2,0xf5cebadd),LL(0xac3f99aa,0x20899c44),LL(0xca5f46cf,0x0e94933a),LL(0x7a04e2c1,0x9a3fee46),LL(0xb0015196,0x367a01a4),LL(0x1715a693,0xa13cc41d),L_(0x0000012a), + LL(0xc2d951af,0xd89fbf0d),LL(0x4488a068,0x172ac7ad),LL(0x7772ecaa,0x0409a3f1),LL(0xf7780ac8,0x6d541e69),LL(0x6ffa0a05,0x4e8fccac),LL(0x7509c471,0xeff8ec93),LL(0x018bbf89,0x101b9048),LL(0x2b2d5626,0x5365cf02),L_(0x00000168), LL(0xbd338134,0xda4a1896),LL(0xaeb1aa9a,0xdfd1cb54),LL(0xc0b310a4,0xeea1a455),LL(0xbed91e2c,0xaae7927d),LL(0xd3502cea,0xdfdf4808),LL(0xd31ee1ce,0xecc68f6a),LL(0x893f08dc,0x9350fda5),LL(0x2a6f281d,0x6394d514),L_(0x00000097), + LL(0xd5d022dc,0x6e654db2),LL(0xe4aaf49d,0xaa763047),LL(0x24820282,0xfe8aa2dd),LL(0xef229292,0xb7ff78ba),LL(0x0170b38f,0xe0a88558),LL(0x0aca63f0,0x66a526d3),LL(0x97a4873e,0xc069b5d3),LL(0x28c88b56,0xf066bff1),L_(0x00000169), LL(0xa3de237d,0xb57187d4),LL(0x20e27844,0x59762170),LL(0xf8485db5,0xf8fe71f6),LL(0x47186213,0xccceb1f1),LL(0x3ddfa68a,0xe9e1e35a),LL(0x3805a749,0x048090bc),LL(0xeea89d03,0xd04309c8),LL(0x451591c1,0x0e6409d8),L_(0x00000167), + LL(0x2e8fa162,0x86c08a44),LL(0xb15f83f4,0xd94f9cd2),LL(0x50bb6a89,0x7bbf2a23),LL(0x606cc572,0x74b1325b),LL(0xb03f198a,0x73b79d3a),LL(0xcf731a6f,0xc95046a9),LL(0x298efd11,0x095ed71e),LL(0xd622bb24,0xb7ec1274),L_(0x000001c1), LL(0x08c383b3,0xb2f19275),LL(0xe14dee81,0x0d888be6),LL(0xce4b12e8,0x213fb612),LL(0x78248f53,0x43092c13),LL(0x4330dbca,0xe40c52b2),LL(0x952b9ef5,0x9d869889),LL(0x31f1126e,0xfbc05f41),LL(0xfd03ae1d,0x3d5c2503),L_(0x0000016b), +}, +/* digit=89 base_pwr=2^445 */ +{ + LL(0x6e868b8b,0x244e266c),LL(0x05a4369e,0x7c7bf40b),LL(0xe296776b,0xaed3b7e1),LL(0x2cfd9c18,0xddbc31e2),LL(0x1ea90d63,0x98abb7bc),LL(0xe50b9291,0x791f36a5),LL(0xc2f87e55,0xfe737c71),LL(0x75c6d8e7,0xfa3c624f),L_(0x0000017b), LL(0xd54c9eb7,0xa0fb486a),LL(0x91d1e1e1,0xd820f02c),LL(0x9160a67f,0xe5d16017),LL(0xf1163f25,0x8b61c557),LL(0xfcbc9a92,0x84ed79f2),LL(0x6a33df9b,0x54ba6955),LL(0xc8febe18,0x43c5cb8a),LL(0xec5a3443,0x2d5af9a7),L_(0x000001ab), + LL(0xa725a8ec,0x5828c615),LL(0xf603a049,0xac113424),LL(0x87a77e81,0x34642c16),LL(0x761c2762,0x9d0db298),LL(0x29b1a474,0x8ac3391f),LL(0x143a9782,0x050c5b69),LL(0x5ae00925,0xc578b0a2),LL(0x144730ed,0x6de2bd6c),L_(0x000001de), LL(0x3bbfa384,0x606aeb93),LL(0x6e3daf55,0x97e41356),LL(0x4263527e,0x4ac1c1ae),LL(0x3a037893,0x8c336382),LL(0xb2143f58,0x2bb7d997),LL(0x69412726,0x2419935b),LL(0x9cb555c8,0x724eeef9),LL(0x2ef7f7cb,0xc86f89b8),L_(0x0000009c), + LL(0xf81335ca,0xd6526151),LL(0x999ff056,0x3c494f0f),LL(0x0a7433ca,0x41b82dc1),LL(0x7fa3bcdc,0x9af11e06),LL(0xec803bb3,0x0ac7bb35),LL(0x457b31fc,0xb5e185aa),LL(0xed555915,0x586ab2cb),LL(0x33044819,0x7928ec97),L_(0x0000008c), LL(0xbfc07b3b,0xae813666),LL(0x4f0a957b,0x8eee1f42),LL(0x82cf0958,0xc3321225),LL(0x4daeb7bd,0x458ec031),LL(0xe4de4e23,0xd0f97884),LL(0xfc50a768,0x1655c201),LL(0xb424f36a,0x14f1a537),LL(0x0cdff481,0x24211a6c),L_(0x00000042), + LL(0x22f8cb24,0xf131c203),LL(0x0c2b076a,0x815ccfff),LL(0xf056364e,0xdbdfbdbf),LL(0xc8028853,0x41ab5760),LL(0x8af0ee08,0xca93ac08),LL(0x3094da56,0x30135092),LL(0x5054010d,0x74228a25),LL(0x8e7dde67,0x6c6f6a05),L_(0x000001e7), LL(0x8a5176aa,0x3779fd86),LL(0xbe16420d,0x7658fb3d),LL(0x41f45c5c,0x110cf130),LL(0x49dea64d,0x19e0e350),LL(0x73f6746e,0x87ca4575),LL(0xfe7da390,0x108ab4e2),LL(0x874c5458,0xe39cce4e),LL(0x1d64965c,0xa2587b79),L_(0x000000b8), + LL(0xfa76cb01,0x80465e82),LL(0x38560d7e,0xf4fd03ea),LL(0xa649c8ff,0x150b3815),LL(0x72398c4a,0xc0e6baed),LL(0x1ba3da88,0xfa79ad8b),LL(0x6f43120b,0x8353fc42),LL(0xbd32e2fe,0x7dbd7876),LL(0x148c548b,0x13391dd9),L_(0x0000016e), LL(0x18cf351b,0x9ca7db2a),LL(0x80485f13,0x16240b9e),LL(0xdfdb85c4,0xc2dc15ee),LL(0xbb4121d8,0xcce3d597),LL(0x0e963371,0xafb37db7),LL(0x7c69e287,0xed3b5fd8),LL(0x6c8d52b6,0x608eaea5),LL(0x053f2384,0xa9384b65),L_(0x000001d4), + LL(0xbbf62c47,0x1eac543b),LL(0x11eca801,0x4e45d301),LL(0x682f5663,0x054cb071),LL(0xe9473698,0xf1ad1950),LL(0x860a714e,0xa8a339c8),LL(0x96d39034,0xe04ab8cd),LL(0xde22b09f,0x2a845e02),LL(0xdfec2116,0xbfc189e6),L_(0x000000c4), LL(0xeaca7b49,0x3e7c928e),LL(0xb0fcae9a,0xca8b3e2f),LL(0xc41fdaef,0x9529863b),LL(0xe1843977,0xd56a624a),LL(0xee7e83a7,0x3438606a),LL(0x81db821d,0x06cdb198),LL(0x3aa0eeb4,0x9b12775b),LL(0xe0d60750,0xa667b1da),L_(0x00000155), + LL(0xbe8e1de1,0x9b0a79d8),LL(0xc345fcdd,0x77d93da0),LL(0xb2f5a213,0x7028f9ce),LL(0x800fcc19,0x1306f2ff),LL(0x469efe59,0x4cd68bf6),LL(0xc5ffe046,0x62b03f93),LL(0x53010575,0x5af4940d),LL(0x46961f0f,0xe96e1d71),L_(0x00000163), LL(0xad6952b3,0x9045b751),LL(0x43f410dc,0x217ebd7d),LL(0x0d11a22c,0x6ddeefda),LL(0x6d1775a7,0x43965993),LL(0x055c0203,0xe7060f57),LL(0x3548b71e,0xa89da1f0),LL(0x805eb428,0x6f8231a6),LL(0xf7b78a97,0x975110f7),L_(0x0000006a), + LL(0xcf1f7c72,0xe247b483),LL(0x97b6bf76,0x202781dc),LL(0xc0f81747,0x8b65bb58),LL(0x92efba88,0x9611a60c),LL(0xd9612af5,0xaf54a57c),LL(0x20d7ccbe,0xf8689ba5),LL(0x6d3cbf9e,0x0591cc36),LL(0xdc1abfe9,0x6d0aa056),L_(0x00000166), LL(0xd4a04bec,0x1e600b02),LL(0x2a15021b,0x6c3ebe8f),LL(0x9586be60,0xb8507cfe),LL(0xf4028af5,0x54dda762),LL(0x4d392e89,0x519d3758),LL(0xbde8dadc,0x58c3813e),LL(0x81db641b,0x91557ce6),LL(0x23fa3b99,0xa7128063),L_(0x0000019a), + LL(0x51bb00fd,0x32b04d9e),LL(0x8aefebe4,0x22b78b18),LL(0x8698b63c,0x7da3c01d),LL(0xa71b8bc0,0x8d71ee46),LL(0x27b7a39c,0xb0583313),LL(0xbd156109,0x49d2846e),LL(0x931258ab,0x86e6af4e),LL(0x3ca87258,0x07bd42b6),L_(0x00000076), LL(0x79f7c689,0x39041060),LL(0x6229b813,0x9028538d),LL(0x3a4aa59f,0x517bfaf7),LL(0x2d1cb542,0x71d33bc2),LL(0x882030de,0x9ba76285),LL(0x91ba5fcd,0x25f86ca8),LL(0x9ae0fc6d,0x47f08f0a),LL(0x3948678f,0x3c9bee89),L_(0x00000085), + LL(0xcd9eb593,0x18cd9b8b),LL(0xac677eb2,0x0d8705ef),LL(0x6b203fcc,0x934fa783),LL(0x39fcfd85,0x571b28eb),LL(0x58bd6d8e,0xd8f1d221),LL(0x215fad4b,0x3e44e705),LL(0x827adc24,0x5ff00393),LL(0x1ec35c0f,0x853889ac),L_(0x00000045), LL(0x14fc0a02,0x5fce2e10),LL(0x71f9384c,0xf0f2ac5b),LL(0x90d699f4,0x7b00891b),LL(0x43b6bcdf,0xe8c4a652),LL(0x7bc04d87,0x0ac9f698),LL(0x2ab126b5,0x3eb3d860),LL(0x849b38d0,0x426d6e94),LL(0xb6985535,0x102cbacf),L_(0x0000018b), + LL(0x0725d65d,0x1183558e),LL(0xb6a14f9d,0x44070253),LL(0x20d9075f,0x6c243902),LL(0x486b1799,0x6c1a9d8a),LL(0xf5efa075,0x8ae5a14e),LL(0x4ea72292,0x2d7b9c93),LL(0x0ca5c12a,0x992cae02),LL(0x91e3345c,0x2c11f45b),L_(0x000001d1), LL(0xc89bcdf7,0xd10410ec),LL(0x89966bf7,0xcf680bd5),LL(0x6ee731b3,0xa0c3db72),LL(0xe37f14f5,0x2aa5a376),LL(0xf554bdb7,0x23be47dd),LL(0xef1712c3,0x96ab9b1d),LL(0x1c7594e0,0x9ed66d28),LL(0x032ce687,0xd712923d),L_(0x00000060), + LL(0x9e3351dd,0x68d68f89),LL(0x9fb7334d,0x40ebf359),LL(0xc5209aaa,0x120177c0),LL(0xe5d00b75,0x2f0e6bbb),LL(0xbf188e69,0x110d2427),LL(0x8e2e5213,0xd6344a1b),LL(0xdcf577cf,0xa7331f94),LL(0x3c553feb,0xa2dfcf3c),L_(0x00000190), LL(0x795a2fa2,0x524f4a9f),LL(0x6609f22b,0x6b23609b),LL(0x2c95b3f8,0x0500bc47),LL(0x8df999a3,0x042c79e9),LL(0x9db59925,0x12a07a8b),LL(0x55be1532,0x07f62419),LL(0x33c89540,0x8df78722),LL(0xfe671ad7,0xe8b1f741),L_(0x000000c9), + LL(0x18059a16,0x52e9bed4),LL(0x717c36f7,0x49d5e825),LL(0xb56dd6dd,0x7783b6cd),LL(0x667fac4c,0xcf53b558),LL(0x116a1985,0x7c15cf14),LL(0x9913c6ee,0xe08410c5),LL(0x6728a2a4,0x9d771edb),LL(0x331fb13d,0x190212be),L_(0x000001c7), LL(0x394ecd2b,0x8da76d72),LL(0x341e75ff,0xf52f78d6),LL(0x29ea6d71,0x46d211ab),LL(0xaf402bfc,0x386ae83b),LL(0x7e9586dd,0x909f5bf1),LL(0x11c7f555,0x1b8a537f),LL(0x427868fe,0xcf05f9d7),LL(0x32daf130,0x60d32255),L_(0x00000069), + LL(0x4ef2bfbd,0x360e62a2),LL(0xc1081697,0xa6a207aa),LL(0x28d01fdb,0x18abc7ac),LL(0x204fc30b,0xbcff0be1),LL(0xe5cdb570,0x48ef40e1),LL(0x1f0b1c2e,0xeb79790f),LL(0x63136e14,0x3d4fe961),LL(0xb9d45c94,0xa355bb40),L_(0x0000018b), LL(0xa61088da,0x8938b0fa),LL(0xb39c86bd,0xc33f1d7c),LL(0xa2380177,0x530d6911),LL(0xaab3667d,0x7b52bed9),LL(0xd815d83b,0x5c596749),LL(0x44b95fe0,0x5148c157),LL(0x202c91ff,0x406b7485),LL(0x8bf24d49,0xa2828406),L_(0x000000f3), + LL(0xd37bc919,0xe29da36d),LL(0x00b56fef,0x7458f713),LL(0x8621718f,0x286883b4),LL(0x448b7c11,0x363d4ba5),LL(0x6114fd6e,0x04011c7c),LL(0x0d4b7500,0xe765f7ee),LL(0x491c6545,0xc2b827eb),LL(0xd01f3320,0x11a3a7f5),L_(0x000001f0), LL(0x6902bd96,0xda599389),LL(0xe2b47365,0xfe7e3528),LL(0xe9079def,0x3aa4556e),LL(0xc96d3bc8,0x610e35fb),LL(0xb585febd,0xa0b2ea82),LL(0x70988a63,0x60d1db4d),LL(0xd27f19aa,0x6eee4c02),LL(0x248d0f40,0x9a820473),L_(0x0000008c), + LL(0xba327209,0x4da0ba85),LL(0xfac1ed29,0x2e3b7145),LL(0x48cf218d,0x5cbfef12),LL(0xde112f17,0x76f3e234),LL(0x194a8f16,0x65787086),LL(0xde1af9c2,0x18958d56),LL(0x495c76a5,0xdd3dbcba),LL(0xa5e9c9c9,0xfc23acbf),L_(0x00000142), LL(0xb1fcebaf,0x30a1b712),LL(0x73d82709,0x8296f1f2),LL(0xfa6e1f41,0x5ef71edd),LL(0x7dd19081,0xc4a2f8af),LL(0x2f6fda9b,0x85b1234b),LL(0x541a4825,0x23556036),LL(0x79e6b22e,0x911ac1cc),LL(0x88ea71f9,0xe3d2a799),L_(0x000001f3), +}, +/* digit=90 base_pwr=2^450 */ +{ + LL(0xa22cc5d8,0x8ee779a0),LL(0x5dd5d86c,0xfd215954),LL(0xfd5e2c81,0xab1c7262),LL(0x75f13cf7,0x4f36ad82),LL(0xe759a0b7,0x8c3ddc91),LL(0xd2223c0c,0x10948a51),LL(0x9b2c7f7d,0x977160cf),LL(0x285822b5,0xddf4960f),L_(0x000001b0), LL(0x1a1e9623,0xd7c127ef),LL(0x7e5e2b50,0xd984c528),LL(0x3999dfdf,0xaad5ce7d),LL(0xd1373907,0x5c84726f),LL(0x97f8f082,0x5ebbc32d),LL(0x68dcb5c5,0xbd51b3a0),LL(0xa1b4f592,0x36935287),LL(0xf3eb9dca,0x0a3866c8),L_(0x00000125), + LL(0x95c0c51d,0xdbb76844),LL(0x7bb768dd,0x35dbc45f),LL(0xeda49098,0x39df9e6d),LL(0x3639006a,0x47f77ee7),LL(0xd878e5a2,0xd141b2c8),LL(0x2c8ccd83,0x04a47e33),LL(0x2d4027f8,0x2c9dc7a0),LL(0x9934bb00,0xf2cf7b1b),L_(0x000000ad), LL(0x8d777a83,0xd9919c1d),LL(0x0f685368,0x9dd72165),LL(0x892863f1,0x2a92b1de),LL(0x8f2b25a3,0x90ff3dd7),LL(0x12a43206,0xaf7bb8bc),LL(0xe03505a4,0xd763efcf),LL(0xcf4f256c,0x53701c70),LL(0xef267753,0xfc008443),L_(0x0000011d), + LL(0xecb197aa,0x593ffafe),LL(0xdd6e37e8,0x40fd5b94),LL(0x41331e2a,0x298fef4f),LL(0x60fb849c,0xfd38fb42),LL(0x7e149e93,0xe83fe7b3),LL(0x22e02e59,0x5d08e682),LL(0x58b84ec3,0xcaa4bfd0),LL(0x7f8e6b6a,0x506a3a04),L_(0x000001ec), LL(0x89faa591,0x3bdb5b23),LL(0x6e000f52,0xa824fd00),LL(0x630a4795,0xcbf7e717),LL(0x4e000837,0xc37102e6),LL(0x5656508a,0x40d36c3a),LL(0xe0b06b84,0x0a694f94),LL(0xd89beeae,0x647088c8),LL(0x682c3563,0x914b3bd9),L_(0x00000187), + LL(0x49e0800c,0xd1b7504d),LL(0x59ebe077,0x04fd80e1),LL(0x714afb4f,0x90ea18a8),LL(0x28810d8b,0xf02c3cde),LL(0x719cff83,0x19367a86),LL(0x8786eb9e,0x952bac43),LL(0xecceb4e9,0x460e0748),LL(0x55aefa66,0x440aefaa),L_(0x000001a8), LL(0x7aaa315b,0x44f96e09),LL(0x7a5db2b5,0x5dae237e),LL(0xa9362519,0x163873d0),LL(0x69799223,0x4d0fbf55),LL(0xc1a58ea8,0xd3bb728a),LL(0x661ed43f,0x100cfe43),LL(0xf1cd21af,0xa24f55c6),LL(0x25dcbe9f,0x4c47c918),L_(0x0000003e), + LL(0x5ef36acd,0xd5225f3c),LL(0x5770b7a0,0x90f00a62),LL(0xebdaa1b7,0xc0ab750b),LL(0x1bcb88b8,0x4d9be029),LL(0x06bbf584,0x9dfcba75),LL(0x606f29fe,0x74e426d7),LL(0xd113e261,0x2931cda9),LL(0x0453e382,0x7c891656),L_(0x00000067), LL(0xba1e3830,0x95a907f6),LL(0x1922ca15,0x760c0c2f),LL(0x24719cf1,0x383ccd2c),LL(0x11f794fe,0xa0495e03),LL(0xaf40e690,0x3eba817e),LL(0x1fab7cdd,0xa83e8359),LL(0x9846062a,0x737b3c03),LL(0x52241afa,0xe4f9dc5b),L_(0x00000056), + LL(0x2d0ad5b6,0x49587f7b),LL(0x321154b0,0x12ada5b1),LL(0xdc3aa007,0x0fe792e5),LL(0xf996b5f9,0x0e4944ed),LL(0x8197e83f,0x06340a72),LL(0x6a72b742,0xb3002f8a),LL(0xbc8a8319,0x173328b5),LL(0x81f8ab11,0xfdfa8b39),L_(0x000000e8), LL(0x2774c6fa,0xbe5f49f5),LL(0x674e04ad,0x8fbf7faa),LL(0xe6de1313,0x5724658c),LL(0x38fee508,0x6a0cb3e6),LL(0xeb3e2c17,0x24438695),LL(0xe7eaef00,0x43ac8a73),LL(0x4dc94b9f,0xd190f6ea),LL(0x422b705b,0x9011e9cd),L_(0x0000012c), + LL(0xd43e3b34,0x02b39b5b),LL(0xd46524f2,0xabfa1c64),LL(0xa4dd7015,0x8ec6eade),LL(0xfac38f67,0x78cba481),LL(0x1123582f,0x61a4550d),LL(0x1caa4894,0x42f7ada2),LL(0x83747c68,0x17f7f74c),LL(0xffb17df5,0xaae24930),L_(0x00000037), LL(0xeb2b93af,0x7f0ef78c),LL(0xcf301d2b,0x8e9f267c),LL(0x8e246b2f,0xb4a2acd2),LL(0x2035c962,0x50846229),LL(0x97e899e6,0xe23609a5),LL(0xfbcb2b53,0x1483eb63),LL(0xfc3f203e,0x4d6ddbe6),LL(0x2861a320,0xc8f76239),L_(0x00000005), + LL(0x41fc794a,0xffcd84d6),LL(0xf5985a4a,0x5082ece4),LL(0xcf3bb3f4,0x850b4853),LL(0xb1d8af65,0x670d980b),LL(0x6953dc3e,0xf579458a),LL(0x7963424e,0xac2f2e4a),LL(0x540b6858,0x920d771e),LL(0x1f5fed22,0xb1aeb9be),L_(0x000000b0), LL(0x731be223,0x662c4dc5),LL(0x1419cfbb,0xa5701752),LL(0xd65099ca,0xfbcc0240),LL(0x3af88f3b,0x1643acb1),LL(0xfbc4861e,0x67405bcd),LL(0x35f067ed,0x9351f1c8),LL(0xcb8018ed,0xeed0e188),LL(0xd276f971,0x0c95e1f8),L_(0x00000189), + LL(0x8cc00e47,0xd167d662),LL(0x311413fa,0xb1947f2a),LL(0x68fd100f,0xc373b68f),LL(0xd96895d1,0x259f8c2b),LL(0xb6277660,0x495d6470),LL(0x6dd59691,0x9eee1f91),LL(0x8e4a7fc7,0x1b01dab7),LL(0xf1319245,0xe946229c),L_(0x000000f2), LL(0x3751b5a9,0xb3dd751f),LL(0x352c6ed0,0x36c470b2),LL(0xbb64e49d,0x58925906),LL(0x5b0b62a0,0x9089d01b),LL(0xe64e7de4,0xf631915c),LL(0xcd161d83,0xfa1f87e7),LL(0x44c46466,0xcced2cc6),LL(0xec7eb165,0xfc7d0b25),L_(0x0000005d), + LL(0x0360b595,0xc1265889),LL(0x3fa41625,0x9e6d3563),LL(0xa19100cb,0x10accaeb),LL(0xaec86fbb,0x7b3f3d8a),LL(0x80771b15,0xdb2ccbe9),LL(0x803f9c49,0x07a460fa),LL(0xf34e5b14,0x4b602490),LL(0x6a99a6e6,0xb0145e5d),L_(0x000001b4), LL(0x308acd32,0x65b0ad8e),LL(0x3dceea03,0x13ece697),LL(0xe1bef19b,0x0643badc),LL(0x4cbdd893,0x470bd8ab),LL(0xafc33073,0x39f6bceb),LL(0x17b3cfed,0xf3aad086),LL(0x198868e1,0x0e329726),LL(0x9f6251f8,0xac5be216),L_(0x0000008e), + LL(0x2c636d48,0xd1bce1b0),LL(0x51f92c94,0x1ad3a1e4),LL(0x88368755,0x9be2c281),LL(0xd8124a18,0xe3a680f7),LL(0x9e5bc7e6,0x5e952f4f),LL(0x2fefbd16,0x5cef9135),LL(0x19b6c616,0x5576fffd),LL(0xbf997c16,0x21c0e1f9),L_(0x00000052), LL(0x01681747,0xdbf38f7d),LL(0x00a3bfc8,0x549f1e54),LL(0x39cded9a,0x23fdd541),LL(0x8b94ded9,0x89eeca5b),LL(0xfd084a5a,0x5123eaa5),LL(0x834be49d,0xae42403e),LL(0x02444c83,0x25abfb57),LL(0xbcb65841,0xc8736491),L_(0x000000f7), + LL(0x606a9e44,0x91ee418d),LL(0xd2dd052e,0x7e5ea0e7),LL(0x92e787f3,0x69d6f73c),LL(0x4508ea48,0xfe248ecc),LL(0xa2d461ac,0x529ffcc1),LL(0x22dcbd24,0xf90dc5dc),LL(0xa3364562,0x542f8abb),LL(0xd254e4f9,0x43e7b56c),L_(0x0000012c), LL(0x990aa036,0xc4f4234a),LL(0xb59ee2ed,0x1031cef9),LL(0xd5d4b081,0x984145d5),LL(0xdf2c037b,0xc7b07787),LL(0x2a2d6af2,0x31d56853),LL(0x6d5e6ff4,0x309a7c9b),LL(0xbb40c66e,0xda6a3ee9),LL(0xaf9db41c,0x4c5f4c81),L_(0x00000180), + LL(0x2ab124d1,0xf806ef78),LL(0x2f7a4e31,0x0cea295c),LL(0x32a4553d,0x8a3bdbfd),LL(0x19971283,0x3b9e4766),LL(0x8810a423,0xcdcf9f57),LL(0x8e85dc85,0x020a6262),LL(0x129bd8b1,0xbc50d2d2),LL(0x9a64395d,0x3517c068),L_(0x0000012b), LL(0xde8938c9,0x2bd0d178),LL(0xb0b45608,0xde7e99ed),LL(0x36896362,0x3851afd1),LL(0xe2cc94c2,0xdfbefd8a),LL(0x14aa57ae,0x3c1abef1),LL(0xd8652bb1,0x0cc39736),LL(0xaf001e99,0x72b91536),LL(0x9fdf9a10,0x4f614f57),L_(0x000000b6), + LL(0x4403c833,0x48e45088),LL(0x26afc7d8,0x41225474),LL(0xa4629460,0xcc7a0cd5),LL(0xff951c59,0xaba7cf1f),LL(0x0d1f3526,0xaf07c8ca),LL(0x18426df5,0x2746f85b),LL(0x42d2e91b,0xfabc76e4),LL(0x58debcd4,0x043b8f7d),L_(0x00000052), LL(0xa9da44e7,0xeb092c08),LL(0x1d496006,0x0b68b024),LL(0x67bb5005,0xbf7c5c64),LL(0x0562f97c,0xca481cfe),LL(0xb0ce28f4,0xc88ad0f0),LL(0x5bcd7411,0x79f82640),LL(0xcfb08fb0,0x9dac9879),LL(0x22beef72,0x149b9a60),L_(0x00000111), + LL(0x055ab7d2,0xb9344cd4),LL(0xcbb171f1,0xf24efa25),LL(0x63c2684d,0x4f8a3dbe),LL(0xf0b702dd,0xda3c37c4),LL(0xa46de3f5,0x17afe9a8),LL(0xdeacde9a,0x55f90e6b),LL(0xb3ba7f88,0x4d24b1d2),LL(0xa174d6f6,0x25de78cc),L_(0x00000081), LL(0xbb7dee1a,0xa172733c),LL(0x562c1d69,0x4325a01b),LL(0x851c2792,0x2307ef6a),LL(0x99968ef0,0x69918f4f),LL(0xb12cbce1,0xcd378fb1),LL(0x9c21abf8,0x48639036),LL(0xe801ee02,0x8268fb51),LL(0x7205404d,0x1c86d6a5),L_(0x0000011b), + LL(0x3a0f6f06,0x7ad7abcf),LL(0x531ca66d,0x41cf832b),LL(0x4ac4965a,0x0f9f470a),LL(0xe00766a0,0xa92657cb),LL(0xb432af80,0xac40c892),LL(0xa94b8968,0xbd44eded),LL(0x4be8b74c,0x2a8d4620),LL(0x98f760bd,0x329915dd),L_(0x000000f4), LL(0x2a7f464f,0xdb3e7dcb),LL(0x00b58c0f,0x1a96b289),LL(0x3c1d7ee5,0x7a30a299),LL(0xbf0ce935,0xb49f5f57),LL(0xf1f5d39e,0x8f0c1970),LL(0xac9cfb8b,0x0718d4fa),LL(0xe1c25a36,0x66ec4ed9),LL(0xb0d7504b,0xf3b6204b),L_(0x00000048), +}, +/* digit=91 base_pwr=2^455 */ +{ + LL(0x72de604b,0xda62b655),LL(0xfb7d9262,0x9db8d0d3),LL(0x8e9c2aa3,0x9b867b7f),LL(0xf2912d3c,0x1a5ad674),LL(0x279e6d83,0xc6935b1c),LL(0x82236f3c,0x9a75e08b),LL(0xfcf8f6f0,0x1baaa28c),LL(0x5ff40727,0x6db1a4f7),L_(0x0000013b), LL(0xab24706c,0x991d9aa4),LL(0x2ed85b4f,0x767e0695),LL(0x0793e3c1,0xf7d06ffc),LL(0x6115d975,0x1316443d),LL(0x9c57472c,0x0b8651df),LL(0x8b972443,0xda1a64f2),LL(0xf1bbd8db,0x0c6db846),LL(0x62aea165,0x152182f5),L_(0x00000024), + LL(0xd5d810c8,0xe6eb51bd),LL(0xe42198e6,0x9bc2e3a0),LL(0xc4b3dc48,0xeda7e391),LL(0x6ed77fe5,0xcd0e3d73),LL(0xc5e60972,0x05f70f41),LL(0x26a51aaa,0xb07669f4),LL(0x9830a47d,0x34591483),LL(0x45b98cf4,0x5726f290),L_(0x000000dd), LL(0xdc9d9c57,0xf4c3b8bd),LL(0x95086409,0xd467dc92),LL(0x3e6cf0fa,0xfbebdef2),LL(0x9684c1e0,0x1daa3a72),LL(0x8c3a2301,0xf40ca0da),LL(0x850f8c4e,0x4dda12c0),LL(0x990ccbe3,0x8f2c5e51),LL(0xc2f0adaf,0x64a48494),L_(0x000000d2), + LL(0x248df475,0xb7ac228f),LL(0x869c271f,0xc18f18ee),LL(0x46d75c07,0xb3a5cead),LL(0xa1a299f1,0xf8c96489),LL(0x7ba98e94,0x805d6dbc),LL(0x1a1d6b09,0x485c463f),LL(0x5bca1865,0xfae82626),LL(0x54594fd7,0x90195cb1),L_(0x0000008b), LL(0x13e4d735,0xb2ef61b8),LL(0x5d0af04c,0x3f79b3f0),LL(0xe86965cf,0x8df70be4),LL(0x6a326017,0x93f5bd3a),LL(0x59b253c0,0xcbf1399d),LL(0xa14a5e9e,0x5112a46b),LL(0x98f7dd60,0x84b48e76),LL(0x9c0c1a6e,0xc837b65f),L_(0x00000111), + LL(0xf887b36a,0xa9bb0064),LL(0x651f2c93,0x4572329d),LL(0xd988aed3,0x6f510e01),LL(0x48541e9e,0xabc023be),LL(0x90ac10a8,0xa8621efc),LL(0xf943b700,0xeb208400),LL(0x768bb3b3,0x64d85b1c),LL(0x634af0db,0x00cbcf42),L_(0x00000072), LL(0xa7e0a10a,0x4e42e593),LL(0xacaa8063,0xadef0026),LL(0x6f2f96cb,0x955002da),LL(0xca66aa2e,0x57271d8d),LL(0x15a69e81,0x66dc7629),LL(0xcf29f326,0x378977dd),LL(0x07d6619c,0xe10e5eaa),LL(0x48e47a94,0x1697c2a5),L_(0x0000004a), + LL(0x60476dab,0xd5ce8836),LL(0x1ca95649,0xd950d005),LL(0xf0b56f1d,0xe5ff1bc5),LL(0x06d36f22,0x6e683386),LL(0xe2bbd6a0,0x249d13ac),LL(0x69fc343c,0x15998eb3),LL(0x47a41e75,0xf1860545),LL(0xc5dfbed2,0x25709f69),L_(0x0000012c), LL(0xc1e80885,0x5129b884),LL(0xf74074e8,0x9d124cde),LL(0xb7fa9540,0x9c58531f),LL(0x291d5e4a,0xaf4422a4),LL(0xb30d5ded,0x4cd8b631),LL(0x30a12b16,0x4d0ff100),LL(0xb2d9901c,0x450557f4),LL(0x1b1ee29b,0x5c3c3626),L_(0x0000012e), + LL(0x6dcd5109,0xc399a187),LL(0x5c966d1c,0xfc77243f),LL(0x4abf82c9,0x3aec3b2c),LL(0xfaf22c71,0x4988df1e),LL(0xaa22e170,0x8f28a287),LL(0x7d192a2d,0xf724ea96),LL(0x6179ade0,0x8ced48e7),LL(0x18acdc5b,0x8dd5bff3),L_(0x0000010b), LL(0x0c63196a,0x8e70052b),LL(0xa08be270,0xd8ff45f4),LL(0xe9bd37fa,0x46164862),LL(0xcc39748e,0x3cc067ef),LL(0x7dcfa284,0x68836731),LL(0xc4586ae8,0x8aabfd38),LL(0x85ecef5b,0x815642ea),LL(0xaf78e84b,0x5ad0b873),L_(0x00000191), + LL(0x55d6e970,0xf86e2eca),LL(0xa3c6d699,0x35fd609b),LL(0x277a6dcb,0x8e0371bf),LL(0x73c53bc0,0x69861b02),LL(0x747c9b1f,0x121dd3fe),LL(0xf6c83ab7,0x1a4b5c05),LL(0xa41b4b4b,0x103beb00),LL(0x6b773426,0x9a32a82a),L_(0x0000017d), LL(0x590624d3,0x968b4935),LL(0xdc61d6e8,0x57ff9d1f),LL(0xc882f647,0x3774a13e),LL(0xd8068210,0xc8094f39),LL(0x52df5a78,0x9ce26b44),LL(0xd80b9309,0x06b76a9e),LL(0x51126d68,0x26a298cf),LL(0x20129462,0xe7c24e2a),L_(0x00000044), + LL(0x00b763fd,0xe905f611),LL(0x31c9604c,0xad5b3ffa),LL(0x059b27d0,0xe85817bf),LL(0x957d997f,0xc9cea64f),LL(0xa8adabfc,0xbf24cb58),LL(0x74dd2fa1,0xfe218a31),LL(0x08cb0dea,0xcfee69ca),LL(0x310fed00,0x47666fdf),L_(0x00000094), LL(0xd66e131e,0x18c779a4),LL(0x47a1fa4d,0x13180dd7),LL(0x23f27ad3,0x3169340c),LL(0xdf4a2f35,0xa8c2be04),LL(0xaec77b35,0x3a1f8aa1),LL(0x2eed7fb2,0xe69edc27),LL(0x6110abc9,0xa5d58ddb),LL(0xe7590226,0x002c042a),L_(0x00000160), + LL(0x2331223f,0xe2c10ac8),LL(0xb1509b5c,0xf8262756),LL(0x8c8a002b,0x4f5e4c0e),LL(0xccc65314,0xaa63ef65),LL(0x5d26b24c,0x91432899),LL(0xaef2d2ee,0xace0562c),LL(0x284cbe21,0xdd5ba0e2),LL(0x4e06f44f,0x6b10f576),L_(0x00000180), LL(0x9becc83c,0xb164c12c),LL(0xc02c461a,0x811e7743),LL(0x621b38ad,0x8fde0227),LL(0x61151b61,0x1eee0c18),LL(0x90989162,0xa3e85682),LL(0x6d28169b,0x478f9519),LL(0xd980aa5d,0x35d9dbcd),LL(0xf940dd13,0x97a4c520),L_(0x000000c0), + LL(0xf6f3cc8a,0xede150db),LL(0x71295f54,0xc2f06bc3),LL(0x173c2266,0xbd26ab2d),LL(0xda0b8b46,0x958aad7f),LL(0x470909e3,0xbeb03518),LL(0x0c135242,0x6b5aad80),LL(0xe6b782b0,0xf43a70dc),LL(0x1ebe42a4,0x314ef776),L_(0x000001d0), LL(0xe9ee87f4,0x5c7acfe8),LL(0xcb05380b,0x7c5775ef),LL(0x2d540e71,0x4507bf0a),LL(0x839d644d,0xc6b81a8c),LL(0xbd1ff451,0xa45f8834),LL(0xc6531b7e,0x06bfc9c3),LL(0xf1a607c3,0xfacabe92),LL(0x152a3731,0x250efc13),L_(0x0000000f), + LL(0x972ed0b4,0x6043f03e),LL(0x2b923d0d,0xef133e76),LL(0x91d9dd18,0xbb6feffc),LL(0x25386141,0x9f4c0085),LL(0x7c52d849,0x279b119e),LL(0x1a74529c,0xfbddc6be),LL(0x3bb14fa6,0x0fc37390),LL(0x5b0469e7,0x6734c330),L_(0x00000085), LL(0xc18b1c46,0x93ed9948),LL(0xc3e5b7a6,0xef0ebe97),LL(0x98b816ed,0x5877dfeb),LL(0x369d99d9,0x3e4311e4),LL(0x2bf65ec4,0x311d0134),LL(0x8c99e5b8,0xc89dbf6c),LL(0x0a9bf18b,0x0c03cf95),LL(0x67a8c9b6,0x6bde4334),L_(0x0000005d), + LL(0x4dad1ff8,0x536327ce),LL(0x2d1bf3b3,0x5b0b6267),LL(0x47367af0,0xf13ddf38),LL(0x8798d158,0xa948f127),LL(0x84fbc252,0x3d0fe92b),LL(0x978e6fc0,0x0138676b),LL(0xef9334ec,0x96ea4ba8),LL(0xb13cf224,0x6c3f3f98),L_(0x0000000a), LL(0xa8ac5c62,0x19d6d21f),LL(0xb670515f,0xb4a2fb70),LL(0x28de441a,0x4762c399),LL(0x0fd9e912,0x3371fcdb),LL(0x896888c1,0x85fb68ee),LL(0xd0e213f1,0x2d86c189),LL(0xcea1f849,0x151cf6fa),LL(0x49b94eff,0xd27fa5b9),L_(0x000000fa), + LL(0xab2c72b8,0x009b461a),LL(0x349ca815,0x64f4891c),LL(0x07d9a898,0x0c2c3617),LL(0xe9f189a3,0x6792792c),LL(0x226e9a21,0xf620343c),LL(0xc9fabe1a,0x72bb7a93),LL(0xda748299,0x321acb19),LL(0x1b2afe12,0x5e8ede23),L_(0x00000037), LL(0x938118e1,0xbfadb006),LL(0xb72a44a8,0x9204c26a),LL(0x32e10cf5,0xbf6c9c27),LL(0xd43d80b7,0xbe5c3cec),LL(0x0afdab99,0x915d7960),LL(0x1a5a24d1,0x933f89c3),LL(0xfe011c0b,0x93e4d990),LL(0xfd09b45f,0x83123923),L_(0x000000e5), + LL(0x5c3e8550,0x67c79888),LL(0xf4f27501,0x18ef2850),LL(0x0d7f6b01,0xf40547f0),LL(0x9cb15ed2,0x89d6f189),LL(0x1bc417a9,0x5c937894),LL(0x22180816,0xb0e0c28f),LL(0x72ddfe6b,0x95839d95),LL(0xb0b70e2e,0x261abd6a),L_(0x00000119), LL(0x0092c31b,0x3261be8d),LL(0x81547562,0x2ed776ec),LL(0x2bc72da3,0x7afed7b9),LL(0x943fbdb4,0x7b5a5d6d),LL(0x896c4516,0x44f20815),LL(0x23f06fef,0x5bd5c28e),LL(0x8f4f6c6d,0xaeda432e),LL(0x355da25e,0xeabf97b3),L_(0x000000b4), + LL(0xfa6c2c0b,0x097c5794),LL(0x8878edbb,0xc8776355),LL(0xf986d296,0xb4dc28da),LL(0xcb68b5fd,0x7d364b0a),LL(0xf5c63307,0xa8df5161),LL(0xe81fb7b4,0x5837723e),LL(0x9cfefa9e,0x681c02e5),LL(0xf9182f1b,0x0c0e0df5),L_(0x000000d3), LL(0x43741dfd,0xeeeaf6e7),LL(0x0f7f7db0,0x912e6cbd),LL(0x1b17b840,0xfbc2bc3c),LL(0xff0906cb,0xaed2e1f9),LL(0x79085d0d,0x413a3a6e),LL(0x3e5f9190,0xf10a18b2),LL(0x5e4a7967,0xbe3ecf44),LL(0xb4e7b709,0xb5e5c6ae),L_(0x0000013c), + LL(0x9f76c8ab,0x693d1574),LL(0x254fd850,0xd18a6991),LL(0x4944e4ef,0xf78f60f5),LL(0xbc73879c,0x125696fe),LL(0x04c63e00,0x8e1e2dbe),LL(0x88e142f6,0x688fd93a),LL(0xbbe7321b,0xd6f1e83a),LL(0xa7b2fa13,0x693413aa),L_(0x00000118), LL(0xcbcaa293,0x19ff3b97),LL(0xb39ad595,0xcedb5893),LL(0x7e59369f,0x47364ed0),LL(0x15852af3,0x7c32e933),LL(0x4d5a40d4,0x82768fe3),LL(0x06533865,0x53f8e4c6),LL(0xf43bf2b4,0x8b4a96dc),LL(0x2542e182,0xb2f8ae33),L_(0x00000153), +}, +/* digit=92 base_pwr=2^460 */ +{ + LL(0x0611a7aa,0x83626ec8),LL(0x91b56818,0xaebe5044),LL(0xecc113fd,0x3fc31dd2),LL(0x68171e0f,0x5fdadf10),LL(0x4d2fbb6b,0xe94a492b),LL(0x06c20ee0,0x0723f06e),LL(0xe2fed2d9,0x8316b906),LL(0x2f32d5d0,0xfd81f33b),L_(0x000001c7), LL(0x63731c43,0x2924bed0),LL(0xf8996c91,0x58df5bd8),LL(0x04d16a64,0xb4780e3a),LL(0x3ff6f14c,0x5aed0330),LL(0xcf56c817,0xb62e0f3e),LL(0xf8163011,0xee8bd1d8),LL(0x5c28fa8e,0x9fa055be),LL(0x5edb8d9c,0xfcd0b563),L_(0x000001e4), + LL(0x0deb4c2e,0x0e66e6ba),LL(0xb2451f1c,0x058a747c),LL(0x20962d66,0x0214b10a),LL(0xda104e82,0xe594cae1),LL(0xb4693d32,0xf837609c),LL(0x059d3bb7,0x53eda7c5),LL(0x1dd16cae,0x3eb60275),LL(0xc67ede2a,0xd46da07a),L_(0x0000013a), LL(0xc838202c,0x35c208ae),LL(0xb2fb2035,0x56079145),LL(0x55be9713,0x95c814d4),LL(0xfd8f0bb3,0xc09f6782),LL(0xcc755426,0x8edafadb),LL(0x4ecf0b74,0xeaf4bc3a),LL(0x553943ed,0xa2bff049),LL(0xe542c407,0x3628a57e),L_(0x00000189), + LL(0x0d581933,0xc0bba438),LL(0x9ddc3e72,0xbc54c3a8),LL(0x1bbc6d0f,0xf2cdd63a),LL(0x1518f660,0xaf2f62bf),LL(0x27e5bf5f,0xbfe7727a),LL(0x7b164682,0xa33defeb),LL(0x40ec257d,0x6132b5ef),LL(0x902b2e89,0xcef96a0e),L_(0x000000ed), LL(0xe3ab4683,0x69a88271),LL(0x8299824e,0x4545479e),LL(0x373761e0,0x397121a7),LL(0xe3b1f753,0x0aaebe93),LL(0xd463acb4,0x1af707da),LL(0x1d6dcb72,0x14e15233),LL(0x48e5280c,0x91655530),LL(0xaaad009d,0xa49c7612),L_(0x0000010e), + LL(0x9792b39f,0xcfd79768),LL(0xc556dfa7,0xf279c07e),LL(0xd7670e8c,0xdad1cf40),LL(0x5e04abde,0x90b1377b),LL(0x46fcc199,0xf5067ad9),LL(0x6a088572,0x28101c96),LL(0x2ad58aca,0x43b5e33c),LL(0xe333e24c,0x9c51feb0),L_(0x0000012c), LL(0x54f0361a,0xf79463ef),LL(0xc22ce2dc,0xdd053538),LL(0x4e6817ca,0x2bc1013b),LL(0x4b01a6e1,0x844a6eb4),LL(0xad109d85,0xbfdaef54),LL(0x4481b985,0x6830be54),LL(0x297f121f,0x8bfd8dd0),LL(0x6d0b67a6,0xb59137cb),L_(0x0000008e), + LL(0x45ba65ed,0x45dcbcdc),LL(0xdd83e268,0x0a0b1cdd),LL(0x63968b4f,0x35e092ea),LL(0x71e6e72d,0x3f1ddc4b),LL(0xa46c5ed0,0x5a166bbf),LL(0xb7c76efc,0x0de0b5f6),LL(0xc1387b79,0xb6445136),LL(0x8923450a,0x76670ba7),L_(0x000001c4), LL(0x99a85019,0x0cae44ea),LL(0x1e4af9c0,0xe5e17b63),LL(0x51c0dae8,0xee91c80d),LL(0x0c3cbe08,0xa4093612),LL(0xa23c8041,0x3446b0ed),LL(0xf215ada7,0x43d14026),LL(0x7813fb66,0xb26fbac3),LL(0x0031b68b,0x0bce8fc7),L_(0x00000080), + LL(0xe275ab90,0x45ee7e0e),LL(0x399f21e6,0x37b06a37),LL(0xe9bd12b1,0x96080496),LL(0xa5d50d58,0x05ba3f26),LL(0x7c1e3f37,0x1d4a0081),LL(0x4d39274a,0x2d00866a),LL(0x0317c40d,0x64f146bf),LL(0x2ec71ea0,0xc168124e),L_(0x0000002c), LL(0x933c199f,0x4f78f168),LL(0x3fb1362e,0xa194ad2a),LL(0x5259655c,0x5f408022),LL(0x5898f9da,0xd8a6c1bc),LL(0x7553edff,0x2793c479),LL(0xe26de20b,0x1ea73083),LL(0x533d9374,0xb2a2971a),LL(0x4fd22035,0xfeac60e5),L_(0x000000b8), + LL(0xf8f9d4de,0xdebb5987),LL(0xe3156400,0xd526ec09),LL(0xe0023f66,0x5578bd03),LL(0xd7d715cc,0x3099f2ae),LL(0x858cc9c0,0x4417ca0c),LL(0x2ea5506b,0x23b4df57),LL(0x7420ffbd,0xe0c5dc14),LL(0x99652bdf,0x4eb0a328),L_(0x00000134), LL(0x0d31987c,0x9461ad7d),LL(0xe6bbab25,0xc859a4a7),LL(0x3d2a289c,0x08730e2b),LL(0xbe629139,0xcf6e14e9),LL(0xc3904cf1,0xdab045a7),LL(0xacb2cca9,0x2a43de3e),LL(0xfa439f68,0xd4d82c3a),LL(0xd187ae70,0xd2291dec),L_(0x000001f1), + LL(0x5587b449,0xe293d334),LL(0x6f8ad12b,0x4ce4906c),LL(0x8d521bfa,0xb26a6693),LL(0xa914bdb3,0x9dc3e746),LL(0x3ae5f6e9,0xb0881c2e),LL(0xac2559db,0x9191a1b5),LL(0x72e53430,0xa6c6a97a),LL(0x07226ad4,0x10b02ddd),L_(0x000000f0), LL(0x3a39e249,0x419af206),LL(0xc72bc669,0x42122752),LL(0x9a44c7a3,0xc28aa7e1),LL(0x188ac573,0x14ec6b11),LL(0x09a3360e,0xc8624588),LL(0x3af7bc0b,0x41e19299),LL(0x42bd4817,0xa8d8d757),LL(0x8768555b,0x890766cd),L_(0x00000046), + LL(0xe3a45fe5,0x95591624),LL(0xaebe1b9d,0xa30f27ef),LL(0xcc5daccf,0xf894bab2),LL(0x753b9ddd,0x456446cc),LL(0xa2185cca,0xfed1e127),LL(0x12d28159,0x0339e65d),LL(0x698c68d1,0x1a9ca283),LL(0xe9c06f97,0x34810a87),L_(0x000000b7), LL(0xfa3c3bf1,0x0574a504),LL(0x1f37ca7f,0xdbe81703),LL(0xebc6693f,0x6a068b27),LL(0x9068b291,0x03b786c6),LL(0x3dcacbac,0x74d197dc),LL(0x5766087c,0x8dd304a2),LL(0x60b034d5,0x45cead24),LL(0x0eb18561,0x4cd9970a),L_(0x0000016e), + LL(0x247df0f3,0x5379241e),LL(0xe00bbfb5,0xad8e1bf2),LL(0x4971d453,0xffdae98c),LL(0x4dac08bb,0x43c392ba),LL(0x1094f61d,0x34435f45),LL(0x25c82ce3,0xc0379951),LL(0x86ddd573,0xbe3f91c8),LL(0xa4a47405,0x4ef0357c),L_(0x000000dc), LL(0xd140309f,0x7d5ffd2b),LL(0xb5c6fd08,0xf1092ec1),LL(0x3fef2aa9,0xb1d3ec33),LL(0x5486d81c,0x414b5e87),LL(0x09d0d988,0x7faa17ff),LL(0xd5d8f9a4,0xed9cefef),LL(0xa9baa755,0xcec21b69),LL(0xb4d04c7c,0xe8d2ccb0),L_(0x0000002c), + LL(0x705db89c,0x84f6bfda),LL(0xac8dec0a,0x7e47ba03),LL(0x11601ccd,0xa270c8be),LL(0x5c7ebbe6,0x6d1474f3),LL(0xf62ef9fc,0xe04a6269),LL(0xfb77f59b,0x9e393c86),LL(0x150112d3,0x5df9c04c),LL(0x64d1ef7d,0xda15f26b),L_(0x00000088), LL(0x4c8472a1,0xfafe59da),LL(0x41ca0a2b,0x2ad99761),LL(0x1e89c29e,0x4b041df1),LL(0xfe4fe5b6,0x8b8ac33a),LL(0xd36ab4d9,0xf8473963),LL(0x53e1a21e,0x75363c3b),LL(0x21c36ab1,0xe0592363),LL(0xe35ac3a5,0x17f59f21),L_(0x00000086), + LL(0xee1e4fba,0x2e463b38),LL(0xafc972de,0x1010af35),LL(0xbe876d0d,0x49188274),LL(0x060ac231,0xac345fe8),LL(0xb568289e,0xe6dec43c),LL(0x9e9a1cec,0xc8cf61bd),LL(0xa7d9e863,0x4480624e),LL(0x84470564,0x5a220f0f),L_(0x00000086), LL(0x7730874b,0xb07f3f33),LL(0x5d9261b6,0x122fcc85),LL(0xe27f8557,0x06820d8c),LL(0x073c1847,0x82e3f6be),LL(0x3976550d,0x0e6c3609),LL(0x9a68ea2f,0xf48dbeee),LL(0x99ffba71,0xcc24a469),LL(0x3d0bdb6c,0xd7097568),L_(0x00000015), + LL(0x01e0bacb,0xfaf69acc),LL(0xde8618ba,0xc2f50b6a),LL(0x14a7dc8c,0x255c91c8),LL(0x2f3c4d3d,0x3e41e3ed),LL(0xb39008da,0x3885fe89),LL(0xe57622e4,0x0199693a),LL(0x4d2436d5,0x8a6ba080),LL(0xdafeb6e6,0x14dd5c23),L_(0x00000122), LL(0x2580e973,0x8519d427),LL(0xa920070b,0x0b67d6f2),LL(0x90fc96fb,0x14566ecc),LL(0x25c7716a,0x30c1cfb5),LL(0xb8dd507a,0xd8b7f726),LL(0xa175dc05,0x24a60ede),LL(0x99f15332,0x78236bba),LL(0x7ca4a569,0x3707415f),L_(0x000000be), + LL(0x1e3479cf,0xdd8491ad),LL(0xd79c5592,0xcd2995f5),LL(0x8ece5732,0x7f9162e1),LL(0x7f928933,0xff64a3a5),LL(0xdd90d4ea,0x82a7e6d8),LL(0xd296e0f0,0x8dc81d30),LL(0xf317ef62,0xa5bbd68c),LL(0xc3c72a97,0x0373debd),L_(0x00000092), LL(0x8de59597,0xbad8e310),LL(0xe46b5cc5,0xae543536),LL(0x3be10fe7,0xb038a518),LL(0x22e5dfa8,0x98fc1a73),LL(0xda531be8,0x4395fad8),LL(0xa64d7d12,0x81e9b112),LL(0x5b7b8eee,0x6b371c5e),LL(0xf97cc8db,0x53a14472),L_(0x00000087), + LL(0xeddf9eed,0xac48ddad),LL(0x3fafda2a,0xb0386572),LL(0xe2cfe37f,0x4a95ae1d),LL(0x2a2d6f2e,0xf0b70c4d),LL(0x5539faa3,0x1e2738ed),LL(0x855ae2b7,0x680d7df5),LL(0x5fa4d703,0x047f7d72),LL(0x981799eb,0xbfdef3a0),L_(0x00000158), LL(0xefb32d4e,0x14e1364c),LL(0x0008de76,0x2af04490),LL(0x56298a56,0xa488b32b),LL(0xe10ef61b,0x7c93d9c0),LL(0x74302f60,0xe50aeca3),LL(0x40b43584,0x7cf8baf8),LL(0xb9ab8a52,0x29e97768),LL(0xf0c44bc5,0xe545b8b0),L_(0x00000193), + LL(0xc7ebaeb9,0x9166afc8),LL(0x8df096c4,0xb1a4fb9f),LL(0x9ef63e0b,0x0a63a275),LL(0xd0e62d1d,0xa13c16de),LL(0xf215cb79,0x82d5b46e),LL(0x45439424,0x5cf39033),LL(0xc9b239aa,0x4a39ce21),LL(0xfcf03ed3,0xdf9660a2),L_(0x0000004c), LL(0xd8466a8a,0x523be0b6),LL(0x493a7775,0x74759167),LL(0x4894bb12,0x5e2284c6),LL(0x864e9ca2,0xd07d26e2),LL(0x08b7f98f,0x6d662061),LL(0x8e1e3fdb,0xf64b5a66),LL(0xa0ba6cae,0xedd31c44),LL(0xdac14a11,0xa2fe1891),L_(0x000001be), +}, +/* digit=93 base_pwr=2^465 */ +{ + LL(0xd37e005c,0x6d99c24d),LL(0x9fa0210b,0x813da140),LL(0xd53cbcd5,0x9488bf13),LL(0xb6d8655f,0x5b2d055b),LL(0xb21f224e,0x3ba305b4),LL(0x059a77dd,0x5337f568),LL(0x783aa9f0,0xb88b4b1e),LL(0xe8c56442,0x5f00bd6c),L_(0x000000b6), LL(0x9b7e0acb,0xe0e90fde),LL(0xda1867f2,0x336f8ff1),LL(0x14e3d072,0x8e647516),LL(0x87e51c7e,0x1ca72a31),LL(0x27ef1710,0x61c42d89),LL(0x641d8a97,0xbb69cc0c),LL(0x6138250e,0xc12903e9),LL(0xd2873a54,0xe47626b8),L_(0x000000e3), + LL(0x0523f47f,0x1fe7219c),LL(0x11a49ec5,0xbd8a88f4),LL(0x6713e8b2,0xd3f30897),LL(0xe0f84892,0x410c616e),LL(0x4957e9fa,0x60b01558),LL(0xfce0903d,0x41fc07f1),LL(0x82117eff,0x3ffa3ce1),LL(0xb039b569,0x82bc2654),L_(0x0000000c), LL(0x04f700d5,0x693fd9aa),LL(0xa0743bcc,0xb8b0e7fe),LL(0x81c35812,0xfcb182c1),LL(0x64896cc8,0x9f019f88),LL(0x8c77cf49,0xa6594c50),LL(0x2c4110bb,0x88406e14),LL(0x0fcaee7e,0xb8b45fd5),LL(0x4dc1ba3e,0x1ad10989),L_(0x00000138), + LL(0x07c446c4,0x0878421c),LL(0xf275544a,0x8722c55c),LL(0x424a48fb,0x028ec763),LL(0xf6b5b3b9,0xca8f7bf4),LL(0xf78d4fe3,0x77d82e20),LL(0x04e23f42,0xbc6300a7),LL(0xf5f71bbf,0x3aa908b8),LL(0x0bc8e8a5,0xde191b09),L_(0x000000fd), LL(0xd0dcad65,0x35d31de3),LL(0xe5fbc4e6,0x9ac9c9da),LL(0x525deba7,0x0b85d812),LL(0x465a1ffb,0x08542228),LL(0xc039c002,0x1962a343),LL(0x60c9d143,0x729577d4),LL(0x0fe4b631,0x05befcdb),LL(0x25528067,0xf91b4f98),L_(0x00000093), + LL(0xbc7bb607,0x894887a4),LL(0x14230e0a,0x6eb1e976),LL(0xe2c653f8,0xe9303e71),LL(0xdded494b,0x9fc0dd96),LL(0x98ac95d0,0x63fba061),LL(0x738abea5,0xf3c1624d),LL(0x4a0ea988,0x389df64d),LL(0xc6ae1823,0x2d8511d5),L_(0x000001d1), LL(0x7feeeb90,0x7d7a8b0a),LL(0xde36c637,0xa9c345a7),LL(0x611067e9,0x0a9100d5),LL(0x6bcdcedd,0xf6c68c80),LL(0x92b5dec6,0x8d7d4a34),LL(0xad3651f3,0x2d5061b9),LL(0xf739c0f2,0xd15c9ea7),LL(0x34e6cedb,0xa8a2dee7),L_(0x00000008), + LL(0x5ef9ab41,0xda8a106f),LL(0x3b8cfab2,0xb72948ff),LL(0x3a4d0cd5,0xf95a1457),LL(0x5f6b94ff,0xa636c12b),LL(0x4c711bcc,0x1e6c9e9f),LL(0xffdba7aa,0xe7eacce4),LL(0xfce23073,0x8fbc9275),LL(0x5935eb69,0xb6675659),L_(0x00000198), LL(0x05d931ae,0x918b9aff),LL(0x1f8a1b79,0x844b544c),LL(0xaa1709c8,0x7e08066d),LL(0x5258c624,0xb640f1c6),LL(0x176bbba4,0xa22bddd0),LL(0xc24ede16,0xd090e0e9),LL(0x4685aca4,0xd8b8736b),LL(0x64e8e6dc,0xa9cb10f2),L_(0x00000001), + LL(0x35476aba,0x78928ff0),LL(0xd948696e,0x989109f6),LL(0x5f254f30,0x44ed9a63),LL(0xd3543664,0xa497e106),LL(0xec63e4f7,0x54a3d56c),LL(0x4cb1418d,0xbfbcd507),LL(0x2a5c778f,0x548f00b1),LL(0x3ba6c12a,0xabe2e750),L_(0x000001b3), LL(0x4db14381,0xc2817a38),LL(0x86547af0,0xb6947c7e),LL(0x6d9e6104,0x70ddd5de),LL(0x2c369c27,0x2f6e17ee),LL(0x04550b40,0x2c52689a),LL(0xb0ead30b,0x3892ae0d),LL(0x99d74e20,0x145321b6),LL(0xd38ac454,0x69273a68),L_(0x000001dd), + LL(0x9a014b31,0x2b898264),LL(0x9e8130d3,0x12fa12a6),LL(0x014372da,0x94999852),LL(0x86eb5c63,0x2a214084),LL(0xdfb3f74b,0x889d0eaf),LL(0x9c182b54,0x4f4c24a5),LL(0x023efe1f,0x0c3bbe75),LL(0x3089629d,0xf6118b2f),L_(0x0000007f), LL(0xdbfd5856,0x138e9e46),LL(0x49f8b60f,0x8855a365),LL(0x00624aa1,0x358ac67f),LL(0xff0d2d03,0xd4f8c970),LL(0xb9b15a4c,0x244d4dda),LL(0x60864d2f,0x7db18004),LL(0x1d1483da,0xd00cd704),LL(0xfbce4196,0x50e134b0),L_(0x000000ed), + LL(0x0feb0501,0xca55bd42),LL(0x8695f9af,0x9c3c71fd),LL(0xcc6e5ed1,0x3d500caf),LL(0x8edc89ca,0x4e21b872),LL(0x77647185,0x0ff872ac),LL(0xee45201f,0xe23036d8),LL(0xc8bee8b3,0x5f2c13f7),LL(0xa1d51a1e,0x6b556707),L_(0x00000079), LL(0x8022b011,0x529ed8e2),LL(0x2b6ff0cf,0xb8a477a5),LL(0x3b6e8238,0xcf5cd2f6),LL(0x291c55c9,0x42ab247c),LL(0x3f4796ab,0x9f93937b),LL(0x5dbfc098,0xca7b47aa),LL(0x7620e79b,0x296f0a6b),LL(0xdd4ea007,0xd0a64f58),L_(0x0000017e), + LL(0xf01a2f06,0x96bedf48),LL(0x7e8f7f41,0xd452fc32),LL(0xcfab9384,0xf1693df4),LL(0x57e90144,0xde828634),LL(0xf5773fb8,0xf8ca5704),LL(0x123913d5,0x2119d8cb),LL(0x7eb6dfc1,0xfdf9f63e),LL(0x3c675fe5,0x2438100b),L_(0x0000004c), LL(0x1f4969fc,0x18608a42),LL(0x92fc7ae8,0x270c4cd5),LL(0xf9035119,0xd6e64853),LL(0xc4832b26,0x6b21d3fa),LL(0xa4c2c4c7,0x726b2dcb),LL(0x725c739d,0xb649408c),LL(0xcaaec71e,0xa0b38b9c),LL(0x5b557fb0,0xfcd2a17b),L_(0x000000c7), + LL(0xffa160b7,0x6ed5bb92),LL(0xd76d4830,0x174db5f3),LL(0x56439bbf,0x9cf210a6),LL(0xcf76e11e,0x0a183944),LL(0xb1458e01,0xa3ae6e4a),LL(0x034db573,0xc26a236a),LL(0xe322c7d5,0x3184159a),LL(0xef56cb0f,0x838dfaf3),L_(0x0000005a), LL(0x83bf41c6,0xc9df776e),LL(0x9cd9b688,0x5709e999),LL(0xec730800,0x9d5ce348),LL(0x9c9f3378,0xa53d30c8),LL(0xdcaf4c9c,0x09b66b9c),LL(0xb8aeaba1,0xc4d0530f),LL(0xea0f22f3,0x73581f25),LL(0xc1d28f6a,0x26a42f5a),L_(0x00000014), + LL(0x47e984e8,0x20c4ae50),LL(0xd1b90de4,0xc94af252),LL(0xdf5adf83,0x41b573a2),LL(0xb6b25c5d,0x003e17b9),LL(0x47e2aa64,0x600d2bda),LL(0xf75489f9,0x595799ce),LL(0x1b49400e,0x7a9784ae),LL(0xba0298f7,0xef02429f),L_(0x0000019e), LL(0xc18830c9,0xc0e76415),LL(0xb7a4b3e7,0x46646194),LL(0x210e39ff,0x46e16ac1),LL(0x94dd48b0,0xe09df941),LL(0x5657d728,0x75b23925),LL(0x987d7dae,0xf5484304),LL(0xb8bad70f,0xee4753cb),LL(0xc44a0313,0xfec01e5e),L_(0x00000078), + LL(0x301bb718,0x3ba3b59e),LL(0x7ed618c3,0x2225703f),LL(0xe6e7b1ec,0x3d9b8d85),LL(0x2ddd2443,0x526b020e),LL(0xbbb89c6b,0xf99d3527),LL(0x9694dbcf,0x1bef732e),LL(0x34415736,0x42d5d4d5),LL(0x5cdafabf,0xc0922340),L_(0x0000011a), LL(0x3da6214f,0x2ffdc730),LL(0x3a1a49d4,0xc640e584),LL(0x9755bbcb,0xd90466dd),LL(0x6a1bb6be,0xcc97293e),LL(0xd094e422,0x23c9d622),LL(0xf2ec9cc4,0xcc616321),LL(0xafe8382a,0xc1a93af5),LL(0x26522de8,0xe7ded64e),L_(0x000001a2), + LL(0x73e6acc5,0x9117f654),LL(0xea525fd4,0xd6399efb),LL(0x5316271b,0xbf78249c),LL(0xe30685a3,0x7737d7b2),LL(0xb95bf177,0x4cfca353),LL(0x138bd305,0xa3671bc7),LL(0x088b1877,0x110ae487),LL(0x1ff3771d,0x6f5bb6fd),L_(0x000001b9), LL(0xaf501744,0x732576c6),LL(0x4a538a56,0xdfe16416),LL(0xf3e1aa2e,0xfe886ca1),LL(0x95495af6,0xfad421b0),LL(0x2e5633e8,0x14deea0f),LL(0x87a33bf0,0x59e08514),LL(0x7333d917,0x92bad09b),LL(0x773222e4,0x810dcfd4),L_(0x00000014), + LL(0x3565ca51,0x0fd12235),LL(0x5d3c8d16,0xdc033287),LL(0x948d4bd6,0xd175dce3),LL(0x9d5a6616,0xd08718eb),LL(0x4afce525,0xd0bbb22a),LL(0x9107b243,0x93527f91),LL(0x45382cdd,0x6c46b7fb),LL(0x8d893d42,0xe82dc58d),L_(0x000001b3), LL(0xbddfc4e9,0x1d8f3a25),LL(0x1c16029a,0x51ce74f6),LL(0x0f857730,0x72d22f72),LL(0x9d6f7b83,0xa805ac0c),LL(0xf970cb65,0x6193a324),LL(0xef9afdbc,0x579b13d8),LL(0x29a49024,0x2ea3de42),LL(0x4d4f92d1,0x20c20f67),L_(0x000001ef), + LL(0x3687601b,0x61173b64),LL(0x19ad59c7,0x5cdaa0ac),LL(0x37e94d1c,0xb01a110c),LL(0x46a8a192,0xb73fb28d),LL(0x89ea578e,0x32a829b7),LL(0xc1c111ef,0x73a214d8),LL(0x1c8ded15,0x1cfef495),LL(0xbf036574,0x6f08a7dd),L_(0x000000d1), LL(0xde40c899,0x451757da),LL(0x9bdf62c8,0xde505c58),LL(0xf590c16c,0x0833d1ea),LL(0x4f878ef4,0xc9e82fbb),LL(0x8ef0ccb2,0x87ab08dc),LL(0x1d1f4efd,0x115ad9da),LL(0x4e21d1ff,0xd549cc87),LL(0x88e6e9e7,0xcd93e9dd),L_(0x000000f3), + LL(0x424f87aa,0xffc3ba23),LL(0x162b1fab,0xa3b2b167),LL(0xb86c7978,0x83e73da5),LL(0x9b5f991d,0x8d484c76),LL(0x2cb3d908,0xe085b439),LL(0x28064542,0xeba2ea8f),LL(0x2b91d2b4,0xcdc46cb8),LL(0xe83321f8,0x9d43a8c0),L_(0x00000193), LL(0xfd97601a,0x4a0c50df),LL(0x305e99ef,0xfc4b8056),LL(0xa29f6e86,0xf5b0c1c8),LL(0xbe1babed,0x558d2cd4),LL(0x75e98d4d,0xd17d7bc8),LL(0x45e57fd7,0x9a0a33b7),LL(0xc3cf9b60,0x2d8c2a2b),LL(0x5277b76d,0x6284d623),L_(0x0000013a), +}, +/* digit=94 base_pwr=2^470 */ +{ + LL(0x3e614e7a,0x1806f150),LL(0x0773591b,0xc937295d),LL(0xb432690c,0x6d3468f3),LL(0x7af2bc37,0xc765b502),LL(0xf1568b1f,0x4508081c),LL(0x4f2d04c8,0x3b08d2fa),LL(0x0d438419,0xfdaa2353),LL(0xd4118eb0,0x68f3eedd),L_(0x000000c5), LL(0x7395d916,0x879c30fb),LL(0x732a652f,0x33e906c3),LL(0xd707078b,0xeb09ecd6),LL(0x4fe7914f,0x76e24476),LL(0xf1644295,0x1ef70830),LL(0x90ff7060,0x8d1a94c2),LL(0x8e38b393,0x347e067e),LL(0x7b7a7e79,0x97efc04d),L_(0x0000014e), + LL(0x7e61fdc0,0xe59da03f),LL(0x98bd359f,0x51831e76),LL(0xf982fb68,0x4079f81d),LL(0x64253ce4,0xffbd0a1c),LL(0x684a0c0f,0x24ab0837),LL(0x0fa3fd27,0xaefd7b90),LL(0x8cd54b9a,0x4f017be0),LL(0x39893203,0xf8268359),L_(0x00000163), LL(0x5f8bfed5,0xbd73f12c),LL(0xbd9b46e5,0x7574722c),LL(0x672fc532,0x95b789de),LL(0x9d4c5de7,0x313e84cf),LL(0x48e00647,0x002f1934),LL(0xdda401e2,0x649a15d6),LL(0x96114ef7,0x37a4f04c),LL(0x4a9dc085,0x66b1bac3),L_(0x000000e3), + LL(0x8f0a82c0,0x87ae4c3b),LL(0x74f004b6,0x384b1146),LL(0xce02e119,0xd665e4e4),LL(0x1859c7a2,0xe5dbd5ec),LL(0xadfa269c,0xa30b0013),LL(0x74ac1d2b,0xb4b5ebac),LL(0xbc73c88d,0xacdb48e2),LL(0x872a2a2e,0x2e41ca61),L_(0x000001e4), LL(0x1044c064,0x1021647f),LL(0x3c4c4561,0x05bc197e),LL(0x8aef2b50,0xdd7066b6),LL(0x53c751b1,0x10e7a8e7),LL(0x3b7ee577,0x2667f737),LL(0xee8825eb,0xd2baf066),LL(0x2e6cd49a,0xfaed0dee),LL(0x4bbbae5e,0xe1b0bac0),L_(0x0000010a), + LL(0x755e5ed4,0x7eafbd5d),LL(0x22817dc2,0x8274c8f2),LL(0x6ed11c56,0xa0be4b95),LL(0xc506cd96,0xfca3c62a),LL(0x6c56121c,0x160f6437),LL(0x94e1e3c5,0xcd969d97),LL(0x2a1b9ac1,0x6a2818ba),LL(0x5d12cb94,0xe5f11bbd),L_(0x000001d4), LL(0x67065269,0x6eaf2423),LL(0x2f4afa25,0x74390891),LL(0x2342e954,0xec048d33),LL(0x5565f855,0x3a8816f7),LL(0x8e4ec59a,0x55b015d4),LL(0x6a715052,0xbf898ef0),LL(0x385313b6,0x8baf90ae),LL(0x415dc868,0x1f03f1d5),L_(0x00000107), + LL(0x117822b3,0x37c2f26e),LL(0x4762f6de,0x9377e35f),LL(0xfb62c99f,0x3c9faf6d),LL(0xa5fec0c5,0xe6f2602d),LL(0x6a84d794,0x1e2e5844),LL(0x2e9c376a,0x2b97dfd3),LL(0xcdd28547,0x24977fdb),LL(0x3efe1f48,0xf8cc4189),L_(0x0000016a), LL(0xd2462621,0x19582902),LL(0x6e455352,0x638d59c6),LL(0xc3bea880,0xd74f8133),LL(0x1df686ac,0xae8224be),LL(0xdefcc095,0x467de606),LL(0xc63376c6,0x1bbcfb09),LL(0x644acbfc,0xe6c4cc04),LL(0xce83b441,0x82fa0126),L_(0x00000051), + LL(0x27486dd1,0x6bd544fa),LL(0xc8ec4c55,0xc9e28c7b),LL(0xa7531516,0x22d696c9),LL(0xfbb565f5,0x5bc69de8),LL(0xf4a6f6f5,0x3c4607fb),LL(0x2183944a,0xfbf34142),LL(0xc088d4ca,0x6a6a25a0),LL(0xebf86497,0xb4a0b6be),L_(0x00000005), LL(0x08805ee9,0xe10f4906),LL(0xb9740059,0x2a74e9d7),LL(0x72ae3ee2,0x2294188b),LL(0x0359108b,0xffa569a8),LL(0x5438b3ae,0xf5a918ef),LL(0x5b1543c1,0xbe32cd1e),LL(0x857bce3b,0xf67721ef),LL(0xcb0f4756,0x59125f37),L_(0x0000002f), + LL(0x7f863db9,0xb81bcc90),LL(0xc94e7e86,0x24d3e88a),LL(0x59ad89c4,0xf5ed10e7),LL(0xce7d1f0e,0x1bd7de70),LL(0x6932d3ed,0xf1561272),LL(0x12569b60,0x393d97a6),LL(0xf0741124,0xa46a9516),LL(0xc9b5b179,0xabb12b30),L_(0x000000b7), LL(0xda1e1906,0x6ff9aa73),LL(0x64a918f8,0x1673d460),LL(0xf233bf04,0xe548f086),LL(0x4ac69dac,0xb984ce8a),LL(0x12e45aea,0x294fb6e3),LL(0x5a19d674,0xd8993346),LL(0x00bc5dd3,0x0c254d86),LL(0x90107b13,0xce4ef414),L_(0x000001d0), + LL(0xec115486,0xb08c738e),LL(0x93e59803,0x23024435),LL(0x4f00e934,0xbdbe60eb),LL(0x7c91438e,0x23e859e5),LL(0x580e89c5,0xba0053e0),LL(0xb329d75c,0xd11317c9),LL(0x3d389550,0xd235e570),LL(0x5aac6426,0xde574384),L_(0x000001be), LL(0x42349105,0x21c69e6c),LL(0x9a81de9b,0xf70eb151),LL(0x2990539b,0xafc827ae),LL(0xfe9bf0bb,0xee9dd548),LL(0xb8dbb3c1,0x8c4d4274),LL(0xf3be2a90,0xb7224476),LL(0x6e6842bc,0x8428346b),LL(0x39da0c73,0x0d7293ad),L_(0x000001f4), + LL(0xdaa88388,0xe569ed69),LL(0x7f0f1377,0x60762027),LL(0x2fe1fac0,0xa5dd03fe),LL(0x91b1f27a,0xb60b2ae5),LL(0xc4161046,0x0c72417b),LL(0xf9d8850d,0x16e6bfc4),LL(0xa63fb7e8,0x1c03a1ed),LL(0x8baa08f0,0xbb67277d),L_(0x0000015e), LL(0xba237ffd,0x7b0680d7),LL(0x0d439e05,0x64dd0307),LL(0x5d17a507,0xded7a46c),LL(0xaa6f686f,0x46bc0763),LL(0x302e95a9,0x16dd6fb8),LL(0x32a15017,0xd01bdd13),LL(0x4b4868e4,0xe250803d),LL(0x1558017b,0x131aad68),L_(0x0000015f), + LL(0x275e8267,0xb1b59985),LL(0x6c0a68a7,0x44944a18),LL(0x537ca1e2,0xd9ed65ad),LL(0x30e1107c,0x1eddede3),LL(0x802fb267,0xeaf5fb68),LL(0x27195ee0,0xd0137c18),LL(0x11b69677,0x331a8cb2),LL(0xab2ade03,0xf409e347),L_(0x0000018c), LL(0x0449d382,0x0033feb4),LL(0x9c7c44c6,0xb2576dae),LL(0x83e4fe6e,0xde49f678),LL(0x8e0a0c88,0x049a1944),LL(0x873f101a,0x2f0dcfaf),LL(0x0c6ecf9c,0xf4b7cbf4),LL(0xb8b0d918,0x5cf46641),LL(0xd2760eea,0x5cfa2282),L_(0x0000010e), + LL(0x8ae1b4af,0x5fa3cb57),LL(0xd120dbcb,0x79a8d192),LL(0xec8bbea1,0x8ab00e0a),LL(0x0fa43f9b,0x8da0324e),LL(0x320ae51d,0x53f9b52f),LL(0xd7d8355e,0x28f25abd),LL(0x800a5d6c,0xce8c317a),LL(0x8116a102,0x946d8424),L_(0x000001b6), LL(0x5e187817,0xdc1f3bba),LL(0x3d0941a2,0x42b1a905),LL(0xa2a2cc7f,0x7392c3e9),LL(0xc8a2218b,0xf81e4937),LL(0x50e22321,0xec9f9e7e),LL(0x650f7010,0x905ed136),LL(0xce7ae424,0x143d78dd),LL(0xf4b39b10,0x3a003ade),L_(0x000000a7), + LL(0xeaa9a342,0x15bb7a90),LL(0x26771654,0xdcc6aa0c),LL(0xc720264a,0x986a1f0b),LL(0xe93f7bc1,0xa182f9b5),LL(0x1bf6b3fc,0x5c22f84c),LL(0x14eb9a7b,0x7bdec7ad),LL(0x42c3b078,0x108902c9),LL(0xd7973e78,0x00f62a69),L_(0x000001f4), LL(0x20230ff4,0x64b88212),LL(0x8928dd30,0xd730a522),LL(0xc0f54e1a,0x528ea087),LL(0xbe8035de,0x2188b80a),LL(0x1d9e98d6,0x12fe3f39),LL(0xc4b4d85f,0x5d1c13b3),LL(0x0436d0a9,0x02c9a494),LL(0xcc5f2436,0xecc057d2),L_(0x0000008c), + LL(0xfa9544c4,0x3a3c6860),LL(0xc43438df,0x7ba2b50e),LL(0x8cfb61b2,0x160b337f),LL(0x177729b0,0xccab10b1),LL(0x69458502,0xb9a0ff5a),LL(0x78449ba9,0x67ca5074),LL(0xa879a311,0x73b77e61),LL(0x91f5cf0f,0xd1a80319),L_(0x00000179), LL(0xe9dd38a6,0x7f1ab441),LL(0xdb97e39f,0x8836c427),LL(0x0e31501d,0x26be55f8),LL(0x88c80de0,0xf094f5db),LL(0x161288e2,0x239cdfc2),LL(0xb27ca6c4,0x6d31f6e4),LL(0xbff28243,0x6db05886),LL(0xadc659a4,0xc60476f4),L_(0x0000014c), + LL(0xbc76d262,0x3a81dc1b),LL(0x1f94ed62,0xb5ab7c11),LL(0xcb89fed9,0x7bd8c4df),LL(0xf48a8846,0xc61ecacd),LL(0xe68265b3,0x88e6ef63),LL(0xfbdfdb92,0x8bd95324),LL(0x94692afd,0xb7f81080),LL(0x8b73dbb0,0x78ac33a3),L_(0x0000008d), LL(0xed59ec62,0xe2c2f288),LL(0x6fafd323,0x11cda8bd),LL(0x6af36bba,0x324f4f07),LL(0x06098acb,0x091a65fb),LL(0x74b50485,0xf28dae6c),LL(0x93e109e7,0x1a6b91b6),LL(0x648a962a,0xba4a27c8),LL(0x4dfe3efb,0xe4e08cac),L_(0x00000085), + LL(0xf43286a4,0x56196c6d),LL(0x6ba6cc59,0xec8e64cd),LL(0x7bd6708a,0x98aaa1d7),LL(0xb45c7cd9,0xb5bcc0ce),LL(0xce3eb2eb,0x2653d9af),LL(0x5b7387d0,0xf3afdf31),LL(0xe27833cf,0x17806f1f),LL(0x3ec743c7,0x911d2144),L_(0x00000076), LL(0x96863b2d,0xf200831e),LL(0x1d6c065f,0x1613e78e),LL(0xa1366e05,0xee600a0a),LL(0xc0223e24,0x06065867),LL(0xe94c7976,0x81ff94bd),LL(0xf203aa4b,0xf9511ac2),LL(0xb7c19e3c,0xd9eef849),LL(0x75211256,0x2d34cf53),L_(0x000000bd), + LL(0x5cc484e2,0x446b62d8),LL(0x1d84ce14,0x9c0fff45),LL(0x08f1ae70,0xe53a49ca),LL(0x71899e1a,0x9917f93d),LL(0x3709a90d,0x12fbb050),LL(0x045ef39b,0x38af72a2),LL(0x0b8cc9a0,0xf0817cd9),LL(0x52b4ed83,0x1ac352f3),L_(0x000000e1), LL(0xa7a71d48,0x44e50d53),LL(0x67e7ad87,0x192ec226),LL(0x37867d3b,0xce32e194),LL(0x24825f0c,0xfce90271),LL(0x5aa41f07,0x4b826212),LL(0xc9cefc67,0x1f602e03),LL(0xc071ae6d,0xf9f93cc0),LL(0xe4c52cae,0x047df3fa),L_(0x0000015c), +}, +/* digit=95 base_pwr=2^475 */ +{ + LL(0xebcc18c3,0xf6385f12),LL(0x8caf536b,0x8a0c24ba),LL(0xd83891ca,0x08b3093c),LL(0xb8c37621,0xb26a0ef5),LL(0xb41e3399,0xca8c426d),LL(0x0263fadf,0x173bf676),LL(0xd40bf584,0xd8a6677a),LL(0xa4760acd,0xee7734ec),L_(0x000000b0), LL(0xfbc42b8b,0x5a60d34e),LL(0x678686dc,0xa367e08d),LL(0x3e942c85,0x2dd8cead),LL(0x9d289bdc,0xa6d1bb40),LL(0xa4b034c3,0x04955940),LL(0x4e438893,0x0034f368),LL(0xddeee0c8,0x63808a7c),LL(0x8f3d9aa2,0x0ff42114),L_(0x00000084), + LL(0x3ddf4a3f,0x7d84daca),LL(0x17e3e628,0xd7eaf570),LL(0x4870b354,0x9fed1a4e),LL(0x26a3e3ca,0x5710e04c),LL(0x0ce1ea5e,0x17a2ff92),LL(0xee67709e,0xf8a3bc06),LL(0xc019a660,0xdb788ab8),LL(0x3d909c0f,0xc37576f4),L_(0x000001a7), LL(0x51c0c61e,0xc130704b),LL(0x5d086395,0x762ffbcb),LL(0xf6639983,0x337f660b),LL(0x46d9fb03,0x8fa37c16),LL(0x865cf06a,0x3f14b6d2),LL(0xe7365f2e,0x8227d360),LL(0xc5c3e588,0xb6a48fcd),LL(0x8c2eaf07,0xb4b8759e),L_(0x000001d7), + LL(0x226084ff,0xaf5d90f5),LL(0xe8626b6c,0xa900e635),LL(0x22c0e157,0x22e31c96),LL(0x1a4ad1af,0x9e88afb5),LL(0x3aadc5f1,0xff5f6050),LL(0xb11e90fa,0xc0677ea3),LL(0xf77875e2,0xaed6a977),LL(0x841145e6,0xf6cc19e8),L_(0x0000007c), LL(0x6bb84d1a,0xbfc80743),LL(0xf386ce67,0xb28c1dbb),LL(0x43c48ae6,0x88b71460),LL(0xf88870e8,0x4e3895ad),LL(0x71c30d54,0xcdbb1a28),LL(0xa8e29d09,0x71499052),LL(0x6fbd1362,0x3608395e),LL(0x9cdda95f,0xfe603cb2),L_(0x000000b0), + LL(0xb01ce2b5,0x9f8c4d01),LL(0xf417d7e3,0x78d34284),LL(0xcbf04214,0xf59d157a),LL(0xc4238071,0xf8a594c0),LL(0x7b0a1e05,0xbaf85cdc),LL(0xc9cfd81b,0x1d1329e8),LL(0xc9be4f2d,0x3168fc55),LL(0x5c20884e,0xb8a1a29b),L_(0x000001e5), LL(0x6e9fd410,0x96d54227),LL(0x16c1621d,0xd61e57db),LL(0x8656adf3,0x2da52da5),LL(0xd546ecce,0x2098e089),LL(0xb41508ee,0x7499c874),LL(0x9cf31199,0xf525839d),LL(0x96548966,0xa0de08e5),LL(0x1cdd85c0,0x68e4ee51),L_(0x00000139), + LL(0xb8ede8af,0xa3de8360),LL(0xfdee27bf,0x4341bdb3),LL(0x376db3df,0x851382eb),LL(0x309206d9,0x6325d433),LL(0xff416946,0x8994d6c3),LL(0x0e775cfd,0xfe50149c),LL(0xee627cff,0xee7b578d),LL(0xcd01235e,0xc20d885b),L_(0x000000f7), LL(0x5a46c19a,0xeec8d37d),LL(0xab92e082,0xa6ae3bae),LL(0x2deb57b4,0xe3c4d075),LL(0xce5d2ec0,0x962e7d64),LL(0xbd42e96f,0xc56b57d5),LL(0x513d5228,0x68f2747e),LL(0x7ec6010d,0x1f92f153),LL(0x8ad259fb,0x7427d2ca),L_(0x00000180), + LL(0x1fdb1361,0x018344e5),LL(0x016f0192,0xcb8a7e81),LL(0x1ca2c27e,0xc36425ff),LL(0xa8df5318,0x56d5d247),LL(0x84872bcd,0xa2e0d261),LL(0x4866d142,0x83feb22e),LL(0x0999b14a,0xab13dac7),LL(0x07863be6,0x9b326ed0),L_(0x00000018), LL(0xce023bbb,0xf8f48d21),LL(0x35940e6e,0xea9c5f9c),LL(0x2bd76e0a,0xa1f9af53),LL(0x8ff97911,0x750c500f),LL(0xdefcff41,0x3985ad13),LL(0x9c027cfa,0x36812ef9),LL(0x34694b31,0x5d319ee5),LL(0x9722dca8,0x568cfb9d),L_(0x000001ac), + LL(0xa78cdc7b,0x67114f96),LL(0x1a506e84,0x909080ed),LL(0xe3ccc90c,0xe770488f),LL(0xe93a6e81,0x0b332add),LL(0x6e681e90,0x494adeb9),LL(0x13abbb36,0x580a5070),LL(0xbf271178,0xa19a151b),LL(0xebb4d25d,0xdeb384ae),L_(0x0000014d), LL(0x2353100e,0x4b162883),LL(0xf7cdd45e,0x57659fda),LL(0x4f79c844,0x95b94da4),LL(0x3ca165b0,0xa6d4f4d9),LL(0x3565f5c9,0xc13d6186),LL(0x288f561b,0x81efd295),LL(0x51b5a1dd,0x0dee47df),LL(0x0f774131,0x059ca05e),L_(0x00000104), + LL(0x8240d25b,0x6132b9fa),LL(0x74ec9502,0xddc2ef3e),LL(0xa9db4e16,0x29d151b0),LL(0x5ad95c14,0x9bb57bff),LL(0x08144cde,0xf2a19e48),LL(0xef980c02,0x655b0b6a),LL(0x1f2df6c5,0x2138725b),LL(0x346457ed,0x99246672),L_(0x00000158), LL(0x8e3077ff,0x2804b9bb),LL(0x8db75e68,0xb8a3a732),LL(0x0cb1bbec,0xb587b6f5),LL(0x823e8549,0xe705757a),LL(0xdd7be7a7,0xb60b8617),LL(0x23677103,0x131d7bc3),LL(0x128ac224,0x03713f91),LL(0xadb3b9bf,0x57a300de),L_(0x000001c2), + LL(0x625c8b6a,0x8fc09173),LL(0x14eb1426,0xbb0b06c4),LL(0xd925dd0a,0x28f4f79f),LL(0x5a160baf,0x6a240ffc),LL(0x4f7c033a,0xcb7f6751),LL(0x98adaaee,0xc349dd94),LL(0x192aa587,0xee546461),LL(0x189c51b1,0x17d09607),L_(0x00000034), LL(0x46d637ca,0xaa3d1efa),LL(0x01cfe315,0xf7d6f7f9),LL(0x4164c61b,0x64b9530e),LL(0x1a339a05,0xce33c2f5),LL(0xc30d67f4,0xbcb863c9),LL(0x79f8f963,0x2bb9ff68),LL(0x0799af64,0xcfca4893),LL(0xe7b1b3d8,0x47d406c3),L_(0x000001d6), + LL(0xa5b46eea,0x00cef9c4),LL(0x0381ae85,0xe36179b5),LL(0x317e7dbb,0xac6498cb),LL(0x2d824ab9,0x328707df),LL(0x6aa97d96,0x80e79f5f),LL(0xc19368fd,0xe03799c5),LL(0x109d20be,0xa4688d4b),LL(0x5dfd91a5,0xc9ed9fae),L_(0x000000b1), LL(0xfce4aa86,0x8a4a894f),LL(0x5f3c5caf,0xf0a6af85),LL(0x0a082826,0x869fa6ef),LL(0x4cf46392,0x5a750056),LL(0x1d906025,0xb437590a),LL(0x5afd7688,0xfa2a2142),LL(0x5b91f195,0x46dd69d6),LL(0x53028951,0x55436804),L_(0x00000093), + LL(0x110cffe0,0x4f03a88a),LL(0xb44dbaac,0x68ebc98c),LL(0x849e6d09,0x9e197499),LL(0x126aa5d3,0x374e4b92),LL(0x9e50c62e,0x9406118f),LL(0xf4a6d99b,0x4e25c845),LL(0xc9df6238,0xb15d2756),LL(0xa10c0e52,0xe1aee3fe),L_(0x00000010), LL(0x816212f7,0x43351049),LL(0x52076e7a,0x90d0771e),LL(0xac804061,0x50393b27),LL(0x509ba99e,0xb81254b5),LL(0x6fa16ea2,0xbe5e2613),LL(0x1a907d04,0xf4aab035),LL(0x2ee00b2d,0x00a0f275),LL(0xab599862,0xf956cc9e),L_(0x00000068), + LL(0xfb9a872a,0xde3af050),LL(0xbec0fc8c,0xebe6b500),LL(0xe7c4ef2e,0x28e4d4b7),LL(0xb38a6c42,0x82362d94),LL(0xc4f9fb0e,0x4e229d20),LL(0xa3690dbf,0xa6e45bdf),LL(0x730c74e3,0xa7b1c90f),LL(0xf2fc481f,0xf8128306),L_(0x000001d9), LL(0xe5e496c4,0xe46148f8),LL(0x16f8ae6a,0x4268188f),LL(0x60936452,0xdcecf1b3),LL(0x828f2ec9,0xeec097ea),LL(0x8a581be5,0x3e062b3a),LL(0x85430a09,0x4da12b49),LL(0x562092de,0xcbb50541),LL(0x33c27b17,0x0f46deae),L_(0x00000171), + LL(0x4148520a,0xb05dd749),LL(0x6530988c,0x4882c146),LL(0x38e93ea7,0xf98af47a),LL(0x6360b046,0x75158008),LL(0x670a2092,0xa8d210f6),LL(0xcea39485,0x590b4493),LL(0xd54fb04e,0xe30eec4b),LL(0xea6ce05c,0xcce02cfe),L_(0x000001f6), LL(0x095bed5a,0xc9bc887c),LL(0x40c45485,0x2639073e),LL(0x060df364,0x9ad162fc),LL(0x0ed461e0,0xd17260de),LL(0x48f9f001,0xcef6cf88),LL(0x5e44883d,0xc42e028e),LL(0x78ade819,0x7ee983d2),LL(0x24ef3daf,0x7e26a859),L_(0x00000188), + LL(0xfbbed4ff,0x64e57bff),LL(0x825f2bb2,0x9eb6b035),LL(0xfd8b6643,0x3c213466),LL(0x9c353790,0x7313deab),LL(0x9b0366be,0x2121723c),LL(0xac2996ae,0x953e87c3),LL(0xbd382785,0xf9b6974b),LL(0x3a30236c,0x0d0993e4),L_(0x0000015a), LL(0x5f3b2707,0x68809f79),LL(0xca4a12da,0x374c5228),LL(0x32cc5a86,0x15cef9a1),LL(0xae5f8c0d,0x72616f2b),LL(0xe61ce206,0x75c41da6),LL(0xde33abed,0xa5fc5af7),LL(0x50659126,0x5776a4d1),LL(0x4c16e788,0xbd4ffed4),L_(0x000001ff), + LL(0x017dab60,0xe869faef),LL(0x1a3d2819,0xa91c965c),LL(0x95cacbcd,0x1c63a302),LL(0x28898d33,0x91791e04),LL(0xe5b4e674,0x2669fe66),LL(0x4ee8bdb3,0x55d62682),LL(0x333ebff9,0x2111714d),LL(0x88832299,0x328d0de4),L_(0x000000b3), LL(0x4df0c3cb,0x7975bc08),LL(0xa4a0f0d9,0x6243d2cd),LL(0xf978a250,0x447d6ec4),LL(0xca8ffce0,0x3c8e28e1),LL(0xa6bda9ff,0x45d5e419),LL(0x3acf30c2,0x7bf52151),LL(0x2b66a867,0x21d9061e),LL(0xbba7056e,0x01e0ca13),L_(0x00000021), + LL(0x8f349801,0x029f064c),LL(0x07be931b,0xb893aedc),LL(0x14f71f6a,0x242b0eea),LL(0xe179067b,0x8af895ee),LL(0x99f6bf52,0x5e852a27),LL(0x1d5c2098,0x94bc1969),LL(0x296ab7db,0x7605deba),LL(0x31b9475f,0xa69410b2),L_(0x0000011c), LL(0xf8c45d63,0x0d9145a0),LL(0xb3a1daab,0xbc0cd8bb),LL(0x614875d3,0x4f51299d),LL(0xad650d62,0x7baf748b),LL(0xdb91d840,0x83b9d385),LL(0xf5cc54a3,0x840ae765),LL(0xbe2653a6,0xab5a54bd),LL(0x5728a0ed,0xa778a286),L_(0x000001f1), +}, +/* digit=96 base_pwr=2^480 */ +{ + LL(0x2be41906,0x9f1fdd6e),LL(0x9ab150fb,0xb458dd16),LL(0xf3f55fa5,0xb1bb79de),LL(0xd9b88ebf,0xc1d98e1f),LL(0x7b8b17a8,0x7f6beb8b),LL(0x6c86e6b3,0xbc72340b),LL(0x7bb70edd,0xdc7c19d3),LL(0x67a99418,0x65a67f68),L_(0x0000008e), LL(0x56a4a09f,0x9cb6bc12),LL(0x8ded9bb1,0x77d8b51c),LL(0x9f35ca45,0xeb257480),LL(0xf1168ba7,0x770b52be),LL(0x12cdae11,0xed4f42bd),LL(0xde9dff68,0xd326b225),LL(0x5631a8c3,0x1d37f144),LL(0xb14a3c37,0x81f6712c),L_(0x00000045), + LL(0x66b0b95f,0xf969adad),LL(0x57813fa7,0x1acf7746),LL(0x220707f6,0x712a2615),LL(0x71d4cd53,0x2fd4ef2f),LL(0x1f82a44a,0xd9e26293),LL(0x0681773f,0xf763ad20),LL(0xe31fd702,0xa99b206f),LL(0xc3a8767f,0x07e06765),L_(0x0000000c), LL(0x41a7f8ce,0x21c3dd47),LL(0xcc9159a8,0xe90e3290),LL(0x06b623fa,0x9e8cf993),LL(0x531760ae,0x2874afd7),LL(0xc9e7cf28,0xe6527ae8),LL(0x293d6e1a,0xf99eef73),LL(0x03d3d878,0x9237109e),LL(0xe1efdba8,0xad3ca36e),L_(0x00000108), + LL(0xcc51928e,0x2af7a58d),LL(0xfb374b29,0x5ec5d4bd),LL(0xd01fb1db,0x6d8cdd85),LL(0x62636565,0x641e476e),LL(0x674fc478,0xe28d244d),LL(0xb39d16a5,0xdbaa94dd),LL(0x5fd5183a,0x6b7fdde9),LL(0xea66d862,0xe99d2b47),L_(0x000001a0), LL(0x574c9d49,0x88f7fd2b),LL(0x31232213,0x6c23d660),LL(0xb2ca0c2c,0xce3a1a6a),LL(0x664a406b,0x2ca19917),LL(0x8f549744,0x6f2fc149),LL(0xab32866b,0x41cbc3b0),LL(0x7a277aea,0x25557ca3),LL(0x16026538,0x56054af0),L_(0x000000c5), + LL(0x4fdb7562,0x9be9e5c7),LL(0x29bd5547,0x548d39ec),LL(0x29c79da4,0xf3f7942d),LL(0xc4bc1f5d,0x948e1f79),LL(0x34a7cecb,0xb63229ed),LL(0x76898793,0x39c1a7d6),LL(0xbe3b3419,0x9157ad78),LL(0x2801351b,0x1bf8452f),L_(0x00000055), LL(0x59cbeae6,0x85ddee3b),LL(0x8140db30,0xeeab1d34),LL(0x41a033c2,0xb676bba9),LL(0x85703aaf,0x23a9d8b6),LL(0x35046b64,0xb832a7c8),LL(0x9e2475da,0x5b8c259b),LL(0xb51f8631,0xdb18a6bb),LL(0x53eb5dc8,0x9d1123a2),L_(0x000001fd), + LL(0x23cca37a,0x57f26e36),LL(0xc4d2ab36,0x787ec793),LL(0x520b9137,0x436337f7),LL(0xbcfb7906,0x2caa7a0d),LL(0x418cfaf2,0x5a502d75),LL(0x0ba14462,0x066c6a13),LL(0x1d083e40,0xd21212f5),LL(0xb9541e99,0x95f764eb),L_(0x0000010b), LL(0xa39384f4,0x07cf7953),LL(0xbdfbaff7,0xaa5f9b05),LL(0x1b083e95,0x782626e8),LL(0xfb350599,0x06f421de),LL(0xe92399d2,0x415729d3),LL(0x04ad8bd9,0xcf103879),LL(0x9370ad78,0x766e0bc1),LL(0xf2c002a0,0x56454b48),L_(0x000001c5), + LL(0xd6b8bb85,0x9a142f9b),LL(0x979dc67b,0xfc51be0f),LL(0xf84e32d8,0xf9ccb118),LL(0xf5b6ca36,0x5e79aba7),LL(0x3a900f56,0xfcfd2df6),LL(0x15163143,0x22db9b75),LL(0x5f85f9f1,0xd886015e),LL(0xe7c48af6,0x5b67dbf0),L_(0x000001b1), LL(0x5dcbc466,0xc13f4daf),LL(0x043aefcf,0x613ac2b0),LL(0x60909041,0x9567d2ec),LL(0xf4b79cb6,0x57b5e5ef),LL(0x8e04188e,0x9dd05dcf),LL(0x759c45aa,0xcd8106c6),LL(0xc6c633a7,0x694b84b0),LL(0xe7963345,0xbd8ff41e),L_(0x000001a1), + LL(0xfe8ed21f,0xf39b982a),LL(0xdf9459e0,0xef033664),LL(0x1245ad2f,0x4c26109c),LL(0x6578f9c3,0x7b73834d),LL(0x28e9fc09,0x21a085c7),LL(0x84bd7b31,0x65666df5),LL(0xd5585963,0x9d7af58e),LL(0xfd1e18ec,0x6947e7e9),L_(0x000000d1), LL(0xe717df29,0xf468848d),LL(0x7c888dbb,0xd747cd3b),LL(0x51097e9d,0xe70801a0),LL(0x8bb9b824,0x172bbff6),LL(0xc27a8a5f,0xf45d5351),LL(0x402074f9,0x0ba6fcc2),LL(0xd7e5a578,0xc1d4e050),LL(0xcb9d2f1c,0xd782dceb),L_(0x0000015e), + LL(0xf2b990fe,0x1096bf3d),LL(0x3b2d5eb8,0xeb580e65),LL(0xa2ad7396,0xca4cfd31),LL(0xcddd150b,0x4cdae865),LL(0x5cde916b,0x6ffe74e3),LL(0x1b6f19b5,0x1e7dc0b2),LL(0x333016e2,0xc799d8bf),LL(0x46cec318,0x3aee6eb1),L_(0x00000077), LL(0x9830acdb,0xdd1e911c),LL(0x1a0df89d,0x891db580),LL(0x646bbddd,0xe25f1a5d),LL(0xc4d27510,0x10d55b0a),LL(0x144af2f9,0x5bcea08e),LL(0x50da24a7,0x7ae5f37d),LL(0x9ad211e3,0x73d37273),LL(0xd9d5c417,0x6daa32a2),L_(0x00000156), + LL(0x785d4516,0xd201173c),LL(0x5cbe43f9,0x6f813c93),LL(0xfc65024c,0x5174f5db),LL(0xcbde45cf,0x98aed5fb),LL(0x29d4641e,0xe15ff504),LL(0xb6befd4a,0x92a16838),LL(0x3fb27455,0x7017d508),LL(0xa78ba07c,0xb26c2bcc),L_(0x00000057), LL(0xad9a35f0,0x16e47d7b),LL(0xbfe092d3,0x75d728c5),LL(0xd99290d8,0x8ba65183),LL(0xae8ed203,0x9af2b287),LL(0xe9db0d4e,0x433a1079),LL(0xe6c8ae7a,0x21dd82ce),LL(0x5486b431,0xfbc30bb9),LL(0x7775c8a8,0xde995750),L_(0x00000075), + LL(0x92135986,0xaae951c6),LL(0x9a74fa72,0x3dfd82d1),LL(0x4bc31a1f,0x060156b1),LL(0x9ab6f26a,0x8b245f24),LL(0xa98e8084,0x317596e1),LL(0xc80c4dc3,0x11d5e680),LL(0xe262106e,0xbee2a8d6),LL(0x60234555,0x6b1389ea),L_(0x000000c4), LL(0x8b82b6cd,0xa80d5059),LL(0xb2182943,0xa584e869),LL(0xa8841e7d,0x7e59ceee),LL(0x74e1e538,0x201d2b08),LL(0x6d2519ec,0x76d5cc62),LL(0x41a115fc,0xdedbfb6e),LL(0xa6f152e5,0x5c18feb7),LL(0xd80d529e,0xda3182b3),L_(0x00000121), + LL(0x3682bcf5,0xe91e53c5),LL(0x6ca16c30,0x3355812c),LL(0x6b8e8ce2,0x18e076f5),LL(0x77cbae05,0x45a2864e),LL(0xae50657d,0x29b224b8),LL(0x5b740476,0x1853045f),LL(0x9cd59d4a,0x4fef40e2),LL(0x6e774f0e,0xa4239a92),L_(0x000001d9), LL(0x0f66fd40,0x87e39c23),LL(0xd4b5406a,0xb9d5824a),LL(0x309845c7,0x4567fe70),LL(0x40e6539b,0xc9f3a53e),LL(0x4965ee0b,0x0d799507),LL(0x06d618a8,0x28bea4e0),LL(0x01fa0a00,0x8b356252),LL(0xb43cd562,0x1c9f3b62),L_(0x00000067), + LL(0x198a02d5,0xcf78de3a),LL(0xa89bcfff,0x6931d65b),LL(0x98eb3ac0,0x204cbef2),LL(0x796db40b,0xdbd652c6),LL(0x82883eda,0x8c7c0479),LL(0x6355b755,0x3ccc26fb),LL(0xb1589be6,0xdcd445ae),LL(0x8ce9a7eb,0x5e17a063),L_(0x000001b0), LL(0x34daeef1,0x92e7ebd6),LL(0x33e4d5b6,0x252f990a),LL(0x7af9fecc,0xa16a7b3d),LL(0x19533f3a,0x07d26ab6),LL(0xf0584373,0xa41a7a2a),LL(0xc7584589,0x32ddecef),LL(0xf36c6f17,0xf2956cde),LL(0xe47377e9,0xa22efbef),L_(0x0000007d), + LL(0x68fb10a1,0xe2e1dbf7),LL(0x1bce1a5b,0x23f22cb2),LL(0x865d95ed,0x3d7b8ca9),LL(0x9350d70c,0x0559d55e),LL(0xf39cff5a,0x634be668),LL(0x1f6fcd80,0xbb740491),LL(0x31d2120d,0x7202a974),LL(0x2efc5e17,0x95233dd7),L_(0x00000182), LL(0x6ff361af,0xbd828851),LL(0x26eff873,0x96db8923),LL(0x8d394d9b,0x6a1cb060),LL(0x3ebd8f2b,0x2c56b043),LL(0x71b88fe8,0x91925e0f),LL(0x39b0cfe1,0xea28e59d),LL(0xcb53dd25,0x933a3cad),LL(0x8fbf4361,0xc3fa9512),L_(0x000000ff), + LL(0x13e495fa,0x7698266f),LL(0x51931514,0xd385a184),LL(0x7057cc40,0x7fd1998f),LL(0x8ffed935,0x5d2e260a),LL(0x55f9858d,0x34fdc952),LL(0x353e16aa,0x3d6d1e16),LL(0xd91adeda,0x9e8895ec),LL(0xa78987af,0x867a4506),L_(0x00000010), LL(0x62e40103,0x869a5ca3),LL(0xc9ddcb20,0x777bb6c1),LL(0x0f3e3498,0x4f97ec1c),LL(0x18133992,0xa7ddecae),LL(0x1c9b2738,0x280ea610),LL(0xae01d593,0xc9770c84),LL(0x30145dcf,0x7c4ed00d),LL(0xa2a8b818,0xff2e9907),L_(0x000000ef), + LL(0xd4f005b4,0x56eacf75),LL(0xf05cca8e,0x8a05a713),LL(0x2382e841,0x3f19077a),LL(0x3c0079f4,0xef823326),LL(0x07e9f310,0x71d13043),LL(0x6311fb89,0x0c6d6593),LL(0x63ca3188,0x0c592a1b),LL(0xfce1253d,0x4831a442),L_(0x00000135), LL(0x9089e935,0x994e32d5),LL(0xdc455b1d,0x643872ac),LL(0x914013f8,0xac2eba70),LL(0x35f0c433,0x5a85e638),LL(0x59b2430e,0xa786ce7a),LL(0x5225b772,0x920543ca),LL(0x51228731,0x1e47ebe9),LL(0xc56f0daf,0xfc2b2f8f),L_(0x0000006d), + LL(0x31ce7476,0xe409da17),LL(0x5b328da3,0x098b5f71),LL(0x607382a2,0x51c3538d),LL(0xc3ee7b06,0xabf1dd7b),LL(0x96d5eed9,0xe8c0d16d),LL(0x1a4ceb18,0x3fe464dc),LL(0x6b9f8f1e,0x0c30d6fa),LL(0x359d987d,0xc00a5cfa),L_(0x0000006c), LL(0x2947d098,0xb97b5789),LL(0x05d737b5,0xcc27fc50),LL(0x2087e2c1,0x62d40feb),LL(0xdd0d9606,0xf37345b7),LL(0x225ee555,0x7f3858a7),LL(0x9ae8d7c1,0x0cf2ae73),LL(0xdcf4e1aa,0xee00ee77),LL(0x649e41ec,0x007dac12),L_(0x00000021), +}, +/* digit=97 base_pwr=2^485 */ +{ + LL(0x808de672,0x8326922a),LL(0x156260ea,0x1a0841b0),LL(0x63e3e317,0xacb0f8a9),LL(0x806aeb44,0x33483737),LL(0xad9d8a14,0x761a3419),LL(0xbffd26bd,0x2e7a343f),LL(0x6d361b6d,0x4d86e32c),LL(0xf433219c,0x336c223e),L_(0x000001ac), LL(0x1f25620f,0xd5c03d38),LL(0x3a87f67d,0x80f73464),LL(0xe876505e,0xe4906c5e),LL(0x491baac4,0x178a012b),LL(0x93e07deb,0x0f735b86),LL(0xd75fad06,0x76ce5dd8),LL(0xc97cb185,0xf5dd4cd9),LL(0x634bbb55,0xa9e4bcbc),L_(0x00000045), + LL(0x37703361,0xc5f2fac1),LL(0x4dee5fdd,0xe014aa4b),LL(0x2218fde8,0xa684a9b7),LL(0x0e229612,0x1d9b66c1),LL(0x7cb5b99f,0x1796c130),LL(0x71c7eff0,0xc0871522),LL(0x27930b1e,0xd19f171d),LL(0x3091f21a,0xdabbbdd5),L_(0x00000094), LL(0xa74c873e,0xbd512368),LL(0xbc31a6ec,0xd3ea21d4),LL(0x62eff689,0xbd43a95f),LL(0x73a33474,0x1413507c),LL(0xf88fa97a,0xb01846ef),LL(0x8f06b4d3,0xfbac8f6a),LL(0xdc2a3015,0x159ddd58),LL(0x7b911f1a,0xe5ea801f),L_(0x00000082), + LL(0x2fe873b0,0x5ebf3c8d),LL(0x32fe371d,0x5b9ca7cc),LL(0xc245b054,0x2658798c),LL(0xeaf83f8b,0xf09afde6),LL(0x761d87bd,0x29e1b970),LL(0xa4fd48a5,0x1501c97b),LL(0x5ab0a100,0x1dca9665),LL(0x0ec7beee,0xefd2f42f),L_(0x000000be), LL(0x28296b82,0x171dfdb2),LL(0x171bb70c,0x1dac3a3a),LL(0xad9a13af,0xe21b7ea6),LL(0x1fe361dd,0x2f8b8125),LL(0xccea9acd,0xe8df3c1e),LL(0xa4b48480,0x8a5f495c),LL(0xb8ecc783,0x07fd225c),LL(0xbc6bffc7,0x52512454),L_(0x0000018f), + LL(0x84cef36a,0xcdf4c999),LL(0x5f8d7040,0xc211953a),LL(0xfaefc5ed,0x563ab4c0),LL(0xa17066a1,0x0c339a5c),LL(0xafb2c094,0x517a5667),LL(0xb135b1e8,0x3d2a94a0),LL(0x4526e2ec,0xd9185e4d),LL(0x3c05d493,0x0f6c71b5),L_(0x00000170), LL(0x2fc5ced3,0xff470fab),LL(0x21ddb195,0xed29f4a2),LL(0x2d94f5fe,0x69f0868b),LL(0xaf8fcc50,0x8631be3c),LL(0x3dcfc141,0x43a07062),LL(0x1c9d9989,0xbafa5f73),LL(0x1cc4a069,0xe1c5c56c),LL(0xf502e626,0xdf086a82),L_(0x000000ce), + LL(0xaaec0dfb,0xb5f2559e),LL(0x37f92069,0xf82c9e25),LL(0xde3d65ad,0xca0987ef),LL(0x52dba2b0,0x110760de),LL(0x6f1e9d7b,0xec3c5a7a),LL(0xb68a52e5,0xe6b61974),LL(0x47ef0970,0xa12dbde7),LL(0x952831ff,0x50801ee3),L_(0x000001d6), LL(0x2a4c3695,0x2be66d3f),LL(0x90213a6f,0xb1043636),LL(0xecffb364,0x0ea64838),LL(0xaf651989,0x059f3995),LL(0xd04bda10,0x8aa19045),LL(0xfff61b8d,0x76712e84),LL(0xb77b5575,0x6970c5d5),LL(0xfb11370a,0x7ca2cd1e),L_(0x00000032), + LL(0xe37454a0,0x644b36eb),LL(0x2bc89a6d,0xe5b95fef),LL(0x705a9c84,0xc8e9527d),LL(0x6ad037b8,0xc306c56f),LL(0xa3d9152e,0x99bdd442),LL(0x3acd8434,0xd2e50d9f),LL(0xfb35013e,0x10c1418e),LL(0xa1aaf42a,0x566527ea),L_(0x0000008e), LL(0x301c9fea,0xa12968d1),LL(0xf9c8c259,0x796a5743),LL(0x1c0237b8,0x92290293),LL(0x56baf809,0xb04d2746),LL(0x81ca3b50,0x93109cb5),LL(0xd70a42b1,0x1ffad7c2),LL(0x829c0f93,0x90fb8081),LL(0x00473bdc,0xc41c7985),L_(0x000001a6), + LL(0x78a1fc22,0x1e0d9fae),LL(0xfe3efcad,0x7975003b),LL(0x1a1d9870,0x5a8555e6),LL(0x08399c40,0x13808c98),LL(0x1f10285e,0xf7ae407f),LL(0x6b16e9b6,0x95f47114),LL(0x5ef970a5,0x42ba4017),LL(0x58f89d1a,0xf78bfabe),L_(0x000001d8), LL(0xb52c0fec,0x68533122),LL(0xc427c0ed,0xd8c2fad9),LL(0xbd46322d,0x870ca81f),LL(0xccd1cd67,0xab6ba984),LL(0x5510a68c,0x6f619ce9),LL(0x2516fdb1,0xd13d0213),LL(0x89ce2a78,0xd4ddba71),LL(0x33ef2f0f,0x10b6e6d6),L_(0x00000012), + LL(0xd29edf28,0x8e335e18),LL(0xe046e99e,0x4ace8ce0),LL(0x72c0503a,0x42f01d0f),LL(0x9c6d09e2,0xfcb4567e),LL(0x3998b6c2,0x0686ceb1),LL(0x91430be4,0xb8fca6af),LL(0x2236ef5d,0x01c77e85),LL(0x718e1a29,0x033d9ba3),L_(0x0000005a), LL(0xab5ae430,0xa843a1b6),LL(0x025f63d8,0xc9500fb6),LL(0xd803e788,0xfb7b9cb8),LL(0xea023d9b,0xcdad70fc),LL(0x803f3ec5,0xa7e50d4c),LL(0x9c07188d,0x9eb540fd),LL(0x822ee2af,0x0d14ab57),LL(0xaff12ba0,0x84fb3574),L_(0x00000156), + LL(0x8230400e,0xed3531cf),LL(0x20fd0e05,0x442851e2),LL(0xd6869a7e,0x1568acb2),LL(0xae871699,0xd7c29d8f),LL(0xad380219,0x512e57e5),LL(0x17e73a2a,0x0239d8b5),LL(0xff1100de,0xa4cc3700),LL(0x3960bc57,0x227a0cb5),L_(0x0000001e), LL(0xec6e136b,0x7a2013e5),LL(0xb3934a8d,0xdeac099e),LL(0x7585325a,0xc5fcf6e8),LL(0xa4aae387,0x73e275f5),LL(0xe0a1bb17,0xc599d358),LL(0x78aeadce,0x5e5ee001),LL(0xf20a237c,0xbc670ce3),LL(0xc755c2ed,0xe8b1f856),L_(0x000001a1), + LL(0x9e1ae8f1,0xb8a1be4d),LL(0xec417dcf,0xa0d53ddd),LL(0x13c7c494,0x565a5779),LL(0xe8460798,0x4157d87f),LL(0x865e6ed9,0x5fcc1adb),LL(0x43eb5613,0xff942117),LL(0xf8951241,0x65dffe8f),LL(0xbc9c1cd1,0x1aadce69),L_(0x00000130), LL(0x9c6c39b5,0xdc72853e),LL(0x60f36e47,0x8941b5fe),LL(0x036e5482,0xd9f274e7),LL(0x2bbb4450,0xd2f8bf2a),LL(0x900ba078,0x48bef6a0),LL(0x9a34b9c0,0x548c40a6),LL(0xa419ecbd,0x3d7bc93f),LL(0x5929867e,0xf64948b9),L_(0x00000164), + LL(0x2d5481cb,0x900dd0cf),LL(0x94bad4ae,0x06d3c0a0),LL(0xbee25614,0x2d0029ba),LL(0x4f1ce8bf,0x7ae14d24),LL(0x12c5aff4,0xcb8bd567),LL(0xd5130b01,0xa1cb296e),LL(0x13ab0e47,0x287ae4a9),LL(0x1c30c115,0x2ee003e6),L_(0x000001c7), LL(0x91cfcc0a,0x9e6287d6),LL(0xf8cffb98,0x0a5d81d7),LL(0x6e40495f,0xfe24065c),LL(0x4ac91688,0x6ef91697),LL(0x0bce1292,0xfa7c3394),LL(0x082d9558,0x334da954),LL(0x0d5bbff2,0x41fa885a),LL(0x6904d684,0x130da09c),L_(0x000001be), + LL(0x30ed1da7,0x34a6a52a),LL(0x610afcab,0x9193baed),LL(0x00ab78dc,0x40598146),LL(0x40d27bbd,0xfc2510b3),LL(0xdf263e04,0x2c222200),LL(0x4f8a34f4,0xe2fa7ec8),LL(0x7ecddf41,0xf5c8a69f),LL(0xb69fa963,0xc6dcbe65),L_(0x00000110), LL(0xde38eab5,0x1dc06ecf),LL(0xbce53abf,0x287aff4f),LL(0x123a0ff7,0x865d5801),LL(0x9bc53dd9,0x7f2760b0),LL(0xf4d19de7,0x2617ed79),LL(0x59b16830,0xfb36b9bb),LL(0x86d6b37c,0xc68164d5),LL(0x3ce542b7,0x536fc3d5),L_(0x000001ba), + LL(0x785c9888,0xd5898c8f),LL(0x0f97f6b8,0xa5e5e010),LL(0x25a6849d,0xdb272a5b),LL(0x5b826b6b,0xf1d7d775),LL(0x8319ab20,0x81fab2fc),LL(0x051b545f,0x13836d82),LL(0xf3f0508b,0x79a2e73a),LL(0xc87d4ab2,0x6868b289),L_(0x00000106), LL(0x9d6fea08,0x0b7f377d),LL(0x285c3784,0x13a96505),LL(0x5e5f0355,0x80e5351a),LL(0x92ff2d7f,0xa4907b9b),LL(0xf478e9fa,0xec7c1179),LL(0xd90b6dba,0xd2c36f50),LL(0xbe1d562a,0x797351a1),LL(0xf65a7374,0xf2fc31a1),L_(0x000001ce), + LL(0x8caa24b9,0xdad4ac58),LL(0xe55e016b,0x42a35993),LL(0x04d4925f,0xf4d85232),LL(0x8c2cb262,0x654bec90),LL(0x44564228,0xd9274933),LL(0x58349da0,0x55dc684a),LL(0xb18184ce,0xe08bebb4),LL(0xe4015bbc,0x9fc75a6e),L_(0x000000db), LL(0xcea81cd2,0xb89f906a),LL(0x5a1b62b0,0xc0a88adc),LL(0x46897bd0,0x3422a9ae),LL(0x6bfb70df,0x0d20f649),LL(0x113b8338,0x197424dc),LL(0xf43ab4e9,0x11c7f33e),LL(0xc17b56d7,0x3e3697c4),LL(0xf0f21e9a,0xf1b66341),L_(0x00000145), + LL(0xb3c8d4e4,0x3221af32),LL(0x786f9cb5,0xef78da7b),LL(0x6228aab0,0x460bf9aa),LL(0x4fd179a4,0xf900af46),LL(0x2bd49daa,0x42fb7206),LL(0xcadd2655,0xfa16e111),LL(0x992a0506,0x4726c9f4),LL(0xbd04e990,0x77052bc5),L_(0x00000017), LL(0xc3ac19be,0xeed5408d),LL(0xfa218d10,0x2bf29af8),LL(0x2a7befd5,0x3b0d28b4),LL(0xeabb7643,0x28d2a823),LL(0xfa48a66e,0x34709b21),LL(0x92c650bc,0xdf1a83ea),LL(0x261706b5,0xa9a5f258),LL(0xbeb0a33b,0x03e83019),L_(0x00000105), + LL(0xd53c20db,0xb6013f7f),LL(0x09d4a480,0x447348d1),LL(0xcb6a7da1,0xd8bfe6fe),LL(0x64e8c529,0xa6067265),LL(0x9034045f,0xa8df68fa),LL(0xff3f3ee2,0x1796dbc7),LL(0xdedc2792,0xe9a130fe),LL(0x4c3f368a,0x2a547fe2),L_(0x000001c1), LL(0x792961eb,0xed55f272),LL(0x9b014919,0x0068193a),LL(0x44cb0bf8,0x32ef3174),LL(0xe22227ee,0x4cb4a896),LL(0x147c8b85,0xc6a73b28),LL(0x2ed1bf6d,0x6804296e),LL(0x77be001d,0x223e6f8a),LL(0x89b143ab,0xdacda84c),L_(0x0000013d), +}, +/* digit=98 base_pwr=2^490 */ +{ + LL(0xec0e9921,0x21044500),LL(0xcba88ccf,0x0c873630),LL(0x6fd4e4b8,0x45764f80),LL(0x056645dd,0x4551a9a7),LL(0x72ed8739,0x025ba6b1),LL(0xa9a78987,0xdd01b45f),LL(0x1f9f1355,0xd2ccea3a),LL(0x807cbab8,0x43e3f592),L_(0x0000011d), LL(0x4c6c96e8,0x1a51e813),LL(0x57065d92,0x2ab97599),LL(0xa89e1baa,0xabc4035c),LL(0x057c2aaf,0x1a6716df),LL(0x9c0890aa,0xf802387d),LL(0xc7786bd3,0xa39383e5),LL(0x1f627056,0x00601e4e),LL(0xf2265779,0x9c233096),L_(0x000001c4), + LL(0xd4e8955d,0xe1d0f381),LL(0x45a79e1d,0x6407a6cf),LL(0xe689a76d,0x407f2393),LL(0xd92aed2a,0xac261bd3),LL(0x95547cc6,0x9e62fcac),LL(0x49835e0b,0x1e291077),LL(0xdde8f908,0x7b3d6780),LL(0xc4cab77f,0x213643c5),L_(0x00000036), LL(0xbde052d3,0xaa1aeecd),LL(0x958c939c,0x24153092),LL(0xca5c0f7b,0x5c0c11b7),LL(0xc9284796,0x3698d827),LL(0xd732af64,0x351c0ba1),LL(0xa3ee0367,0x1b1bf491),LL(0xec2302cf,0xafdf3514),LL(0x4436d640,0xb9059b4c),L_(0x000001e7), + LL(0xe09f3da3,0xb7ed8d41),LL(0x97fc0836,0x33451dcc),LL(0x1b62ac0b,0x4bb0f328),LL(0xc7985f30,0x76f68d69),LL(0x7e5bf130,0xa8fcc12a),LL(0x87f28f61,0x0c13fc90),LL(0x097e8f18,0x9299a913),LL(0xec6104d5,0x243dd550),L_(0x0000003d), LL(0xf8ef488b,0x2c0a7e4f),LL(0x94b33bee,0x5db54ad3),LL(0x4a72bd81,0x0abb0c63),LL(0x57fa905b,0xa7b05810),LL(0x98b0da0b,0xc18c2e82),LL(0xa6507965,0xd53207a1),LL(0xba323d2e,0x1a96c29e),LL(0xe70d8f52,0xb6698de6),L_(0x00000152), + LL(0xc54265bd,0x14b97128),LL(0xe0b08320,0xfd3dafdf),LL(0xb3fd6699,0xb8bb1956),LL(0x416a87bb,0x038f8691),LL(0x01dd4344,0x88826c84),LL(0x7456566c,0x07a8a4b7),LL(0xb2fca59f,0x037671e1),LL(0x797dc52a,0xf3b10200),L_(0x000000e2), LL(0x5d7843bb,0xee9b4c46),LL(0xfa39c4f9,0x303b1652),LL(0xc4a55ae2,0xc15ae7c4),LL(0x3ccdcb67,0xda8ac526),LL(0x7a17fd06,0x8d1d1e92),LL(0x685ac10d,0x5bfc6232),LL(0x048bbb8e,0x233162cd),LL(0xc2cffebb,0xafee0b11),L_(0x000001d1), + LL(0x20c13569,0x9b91ab1e),LL(0x14810705,0x032808d4),LL(0x6428e5a7,0xdd56117a),LL(0xb86a6737,0x9ed920a9),LL(0x32ca9ded,0x46d45de0),LL(0x0898b533,0xaef720c0),LL(0xc4b5cbd6,0xe8b625d5),LL(0xd262cadb,0x38b1a825),L_(0x000000f1), LL(0xbe07a63a,0xae8c0455),LL(0xc0c992ba,0x56ade4e3),LL(0xd8c4dd74,0xaddf367e),LL(0x1fb487a3,0x10d03d26),LL(0x978961e7,0xced02543),LL(0xb1fb8d4a,0xd9cb94bb),LL(0x4067c3be,0xfd0c7063),LL(0x1e63aa4d,0x0497e03e),L_(0x000000d3), + LL(0x43a818af,0x8e882098),LL(0xa14d3397,0x2df8f9ff),LL(0xfba08f0f,0xc3139e9a),LL(0xd6f4162c,0xd35b42ed),LL(0xddc9743c,0xa29eeda8),LL(0xeaef65fa,0x1d1cf761),LL(0xc4cffc87,0xf5204083),LL(0x9c04512b,0x184be0af),L_(0x00000183), LL(0xcce1fc0e,0x4ca92fa2),LL(0x0756de13,0x7ef7ab66),LL(0x6b218d95,0xa4befba4),LL(0xec5df862,0x028018d1),LL(0xb0fb4797,0x0fba684f),LL(0xbb1872b1,0x035fcdb9),LL(0x727d62c3,0xa85f1754),LL(0x52c190b4,0x9257c1a0),L_(0x000000ed), + LL(0x8c02d54a,0xa33c5b35),LL(0x9d3a0d5a,0xda74fd32),LL(0xcf0e2fac,0xdf27160e),LL(0xa9cf0042,0xce44dc1f),LL(0xf9b4e2f6,0x8815ba75),LL(0x71f7406a,0x5d29fed7),LL(0xba78d604,0x1b44fac2),LL(0xa544aa8d,0x2096cbda),L_(0x000000c5), LL(0x4f2a1a98,0xb052a60b),LL(0x2ee09590,0xfd68dd3b),LL(0xf82abd62,0x4cd7b68f),LL(0x3747e4a5,0xa7a82c3e),LL(0x6580fff5,0xfc1c77a5),LL(0x878185a1,0x29848ebd),LL(0x3507cf8f,0xe376e805),LL(0x3d153708,0x307d42fa),L_(0x00000089), + LL(0xd12cb202,0xde81ad92),LL(0x7dfd0285,0xb71b7749),LL(0x0c150a03,0xf597a8a1),LL(0xe99f4ad3,0xaef51dbb),LL(0x1f4533b0,0x838ed493),LL(0xbcf13b27,0xffc95a8e),LL(0x5623ddf8,0x2b2cefcf),LL(0x37c16683,0x782f0107),L_(0x000001d8), LL(0x8ce740d8,0xf3d3ab04),LL(0xd03ed624,0x376415ce),LL(0x76391c7d,0x22ffbe56),LL(0x671ffe7b,0xcc5f4981),LL(0x90390438,0xae289dd4),LL(0xef0984a5,0xf9a1f5bb),LL(0x41c66528,0x3eb05c49),LL(0xef77ff07,0x5fefc83a),L_(0x000001b4), + LL(0xdc003b87,0x913da2c9),LL(0x7357222f,0xb56a4216),LL(0x44f79a9a,0x435f9dd6),LL(0x6316da1e,0x76bee0c1),LL(0x44a0a348,0x528570a1),LL(0x1fc2528a,0x9e402d88),LL(0xadaf6615,0x23fd690f),LL(0x1b05eafc,0x4359ac50),L_(0x000000eb), LL(0x22ceb0f5,0x9817f70b),LL(0xb1910295,0x586cc9da),LL(0xfd424885,0x92631348),LL(0x902c218d,0x2d6b4b6b),LL(0x3b06ab9d,0xf6ce2b8d),LL(0xc2ef9db4,0x5440a618),LL(0x86f795ee,0xe7115329),LL(0x241af150,0x4639439d),L_(0x000000cd), + LL(0xb84d399a,0x5d1b603b),LL(0x6e75095d,0x3ebd519c),LL(0x0c38a4d8,0x5a2e8ab6),LL(0x703ee9c9,0xbe621b2f),LL(0x46425097,0xeda0425f),LL(0x3bda2722,0x7eaff2bd),LL(0xc82eaa59,0xddb8e21a),LL(0x9f181562,0xe2cfef1b),L_(0x0000013c), LL(0x319323ed,0xf55ade52),LL(0x1195febd,0x72cc0544),LL(0xa5494291,0x34dc9234),LL(0xaa005164,0x2b24e83a),LL(0x2e6ec50c,0x61a67644),LL(0x300b585a,0x2a2cce42),LL(0x251bf8fe,0xc53eb03f),LL(0xa53e93a0,0xaf5e947c),L_(0x00000146), + LL(0xbd450c1b,0xb6233906),LL(0xf8cf8ce0,0x2f163b01),LL(0x37f9bc73,0x121c5a4d),LL(0x74a3b3e9,0x84c581f0),LL(0xd2aa3a2e,0xa16b9ae6),LL(0x7a258259,0x5182e300),LL(0x6d279587,0x3b163221),LL(0x9054a8d4,0xc9bbaf00),L_(0x000000fa), LL(0x650238b5,0xbdfac6cd),LL(0xbc313548,0x60bb5887),LL(0x98e5a28f,0x3b8aace3),LL(0xb0c9ed3d,0x1f5ded63),LL(0x943fdf61,0x3ad1b6fe),LL(0x292fee9d,0x3bea9f61),LL(0x765fb8a8,0xb3aff102),LL(0x1c05be97,0x28393c5e),L_(0x0000014b), + LL(0xdf604a9b,0xe5f51ce9),LL(0xad97f878,0xb763f98c),LL(0x89d3ab54,0x6fef5a13),LL(0x3d0efdef,0xdf2543ab),LL(0x39be5cf0,0x51869676),LL(0xd8322a0d,0x34850193),LL(0x90477611,0xe0860abf),LL(0xf04b3d8d,0x2016c4b4),L_(0x0000018d), LL(0x65e57c44,0xa66238e0),LL(0x78a12a3f,0xd140af78),LL(0x55dcc858,0x65fde14d),LL(0xc7b391cc,0x4d882561),LL(0x2d7dbf32,0xaa74e78f),LL(0x03f85d09,0xed166a45),LL(0x7ad860c8,0xe6da0c2a),LL(0x8d8fe387,0xa3128204),L_(0x0000014a), + LL(0x06980a45,0xdae5a808),LL(0xcd56237c,0xdb2a4bfb),LL(0x1628aed0,0x029fee0a),LL(0x5baf2e6b,0xcfe94b4b),LL(0xbaa05917,0x186a18d2),LL(0x6fb7aa6f,0x3e611766),LL(0x97f81882,0x759b437f),LL(0x25dc9d58,0x1b6c9ec3),L_(0x00000167), LL(0x16ed73fe,0x1dbc5e5f),LL(0xae04984f,0xb2380921),LL(0x00f6d531,0x941b93a0),LL(0x9956e547,0xafda37c5),LL(0xbcd796ef,0xeb23c73c),LL(0xd18e7739,0xec321455),LL(0x9f3b6f4a,0xc9846e98),LL(0x7c09f2f9,0xe0267b0b),L_(0x00000097), + LL(0x447d7aeb,0x4ec21998),LL(0x691aa304,0x062e1539),LL(0xd6b055f4,0x350b4c67),LL(0x310e0e47,0xeee31b24),LL(0xe16697e6,0x24a9c93a),LL(0xc742c710,0x50e8a296),LL(0x5f791c6d,0x511ff4a6),LL(0x4536286d,0x0ed01379),L_(0x000001c0), LL(0x36a695b3,0x301dfa2f),LL(0x07145edb,0x89ff6d58),LL(0xb1a2232a,0xd04785f8),LL(0x27854372,0x37e21f65),LL(0x82b68a6c,0x56a575ee),LL(0xb3063755,0x45ea52c0),LL(0x0e5cba73,0x1f5a3458),LL(0xb3b90431,0x55c3c131),L_(0x0000018a), + LL(0x05a677e3,0x7473a30c),LL(0x5a28a98f,0x8039064e),LL(0x69a1890c,0x559f0687),LL(0xdf1716eb,0x5a68dd8c),LL(0x216c64b2,0x665fa083),LL(0x89b49b44,0x27c0f780),LL(0x952e61c5,0x5e57e0f3),LL(0xc9f22b25,0x1e4e175f),L_(0x00000190), LL(0x9de5f532,0xa33b1f2d),LL(0x6ef509f5,0xd526630e),LL(0x80ca2834,0x2e72073f),LL(0x551bac6e,0x8bdc5409),LL(0xa93f6103,0x74c46ebb),LL(0x78de2a49,0xe616b99b),LL(0xbd4c9f5c,0x5a4f27fb),LL(0x9a70865d,0xb1f197c4),L_(0x0000012d), + LL(0x66505ddc,0x5ed0deab),LL(0x89dc51fb,0x5e972cc6),LL(0xb69dc78b,0x0c6e495f),LL(0x3f182669,0x96d605ca),LL(0x85f61868,0xf6678928),LL(0xedfbfd32,0x863f96f8),LL(0xdece22e8,0x64e644e5),LL(0xf857a4d5,0xc551c9b7),L_(0x000000e2), LL(0x55691ddb,0x1566a700),LL(0x7fa43590,0xe1f4e606),LL(0x796a672c,0x30886e62),LL(0xe5ee14cf,0x40aacab4),LL(0x9b327c1a,0x3294bd68),LL(0x2c002fed,0x103e5699),LL(0xf8494f59,0x9eb1323d),LL(0x51fec2bb,0x7f8bbd7b),L_(0x000001dd), +}, +/* digit=99 base_pwr=2^495 */ +{ + LL(0xcb769c73,0xa9593240),LL(0xb057cdf9,0x4e8217f3),LL(0x220054ab,0x34dceca2),LL(0x5ba1c2a7,0x22b719c9),LL(0x16500d13,0x1b436038),LL(0x0ba1ec57,0xf76e87cb),LL(0x59db2c76,0x77c59385),LL(0xbe033b58,0x47810397),L_(0x0000015d), LL(0xddb6a77f,0xc6a6cde3),LL(0x8f578d9c,0xe4987253),LL(0xeaf37819,0xa10e469f),LL(0xdc9c48cd,0x0f98832d),LL(0x51a6e545,0xbf05ea5f),LL(0x195d2d31,0x75cd216a),LL(0x89987c43,0xd18007e6),LL(0x358f3e07,0x0eb94cbc),L_(0x000001d6), + LL(0xd66a73f6,0x68ca281c),LL(0xa08670f9,0x827efca6),LL(0x180e8f32,0xeac3a96b),LL(0x9979b757,0xbff7df80),LL(0x2d9223bf,0x166015fc),LL(0x30d747dd,0x5475a887),LL(0x9ea9d126,0xfbce1622),LL(0x23756de3,0xee18876d),L_(0x000001e4), LL(0xa8ed537b,0xe46c7c15),LL(0x6d7df943,0x4b3f8765),LL(0x335be530,0xdb8a9213),LL(0xcb0ee208,0xb61ee376),LL(0xa4f5fc16,0x4ee85495),LL(0x2d47c111,0x53ced62c),LL(0x453ad352,0xaf641c92),LL(0xe1a21d73,0x4f8dd3e4),L_(0x000001dc), + LL(0x515c16f0,0xa0b8ef31),LL(0x11c31db7,0x6aca4236),LL(0xa62abc90,0xc862474f),LL(0x94eec3da,0x600cabaf),LL(0x3cb82e64,0xec07e4a3),LL(0x3c98aa82,0xef67e3ea),LL(0x548e4dc2,0x3817b7a5),LL(0x6160dfbe,0x5240c7b0),L_(0x0000008a), LL(0x212ed3d8,0xa341dfe1),LL(0x71cd3579,0x96070d42),LL(0x9ade2ce0,0x0d16c811),LL(0x6ffd05f0,0x1f2ea906),LL(0x4add5623,0x9034240f),LL(0x871a6489,0x92dc3317),LL(0xd71bf3b7,0xd6ec77d9),LL(0x3faf4d88,0x2292f10a),L_(0x000001aa), + LL(0xab483ff2,0x0cb9492b),LL(0x1910717f,0x1999673c),LL(0x5ba40ad7,0x61f5c7a7),LL(0xec8c1ec8,0x7a954022),LL(0x87870445,0xab6d023c),LL(0x607c1194,0xdaf5008c),LL(0x53612330,0x4ad39492),LL(0x5bf20a93,0xbc2055ab),L_(0x00000193), LL(0xab8ed330,0x045574d7),LL(0xf38e3b31,0xed11cb44),LL(0xab10bb4e,0xa511af67),LL(0x33cee10c,0x9fe7c1d0),LL(0x1549874c,0x6999489f),LL(0xa85b392d,0xfcfe4a15),LL(0x3684decf,0xb3b006a1),LL(0xbaefda3e,0x2c4ef7b8),L_(0x00000036), + LL(0x3d2bc261,0x1ee7f1c9),LL(0xd8d2c223,0x25894a88),LL(0x1159870b,0x106d0fe5),LL(0xac7070a4,0x37cd1eb7),LL(0x45edd566,0xfcc105e8),LL(0x0d87aaf2,0xbbc54886),LL(0x650d534b,0x4c68c1c6),LL(0xf8b73f4b,0xa090fd18),L_(0x00000116), LL(0x1c67e752,0x9fdf5e33),LL(0xda66e59f,0x1506a29f),LL(0x04eb7efc,0x566aa91a),LL(0xaf0c681f,0x41ad74ab),LL(0xcd6fd019,0xa0e6609f),LL(0x90d32f02,0xf9f03394),LL(0xc3f5d1da,0xbd5dbcf1),LL(0xa63c99d2,0x0f138d5c),L_(0x000001b6), + LL(0x94e82cbf,0x52832eb4),LL(0x61b4102e,0xb381c8b4),LL(0x7f0afbca,0xf1ba6e87),LL(0x8482ae88,0xc8cc076b),LL(0xd709eb28,0x5fc8e5ce),LL(0x0c640cd4,0x1363d1cf),LL(0x6be8f78b,0x4993a63d),LL(0x7a8e7f6a,0x60d50140),L_(0x000001ce), LL(0x1bfd703c,0xb3415df0),LL(0x8b57f708,0xf82eeb2c),LL(0x407aa69d,0xa723ed35),LL(0x9767c6b4,0x4dbc3f44),LL(0x52e1a818,0xdffc3e96),LL(0xad89d25e,0xe8855e29),LL(0x89f2e493,0xb2c695a8),LL(0xae2a995a,0x17ec825d),L_(0x00000011), + LL(0x7948d6db,0x1bf35a68),LL(0x6953ece3,0x2e4f6945),LL(0x5f0ffde7,0x8e48e233),LL(0x45b68e1d,0x805943a8),LL(0x7c03271d,0x83acacfb),LL(0x136c96e1,0x20b340c8),LL(0xb32c48cc,0xbf7ec9c0),LL(0x9a5cdcd3,0x6c769d3c),L_(0x00000065), LL(0xeda9d905,0x02288f5d),LL(0x6d5bb5e0,0xe250809d),LL(0x26841f8e,0x3ede31ca),LL(0x5b8a74f6,0x941707b2),LL(0xcd744281,0xf31b5a51),LL(0xcb81c384,0xa2b751ad),LL(0x1c7d45f0,0x98cd14ff),LL(0x0b61414f,0xfb2bcceb),L_(0x00000029), + LL(0x49c300d6,0x149cc932),LL(0x984e1f84,0x4f71e87a),LL(0xa635f884,0x2ebebead),LL(0xc51f4894,0x8d815dca),LL(0xb76c6b87,0x60dede95),LL(0xc3f25874,0x83c91cf8),LL(0x53753878,0x6d13e9be),LL(0x4ce987a5,0x59e4476d),L_(0x0000007b), LL(0x22fb2015,0x4bf1c2d6),LL(0x8bc4abf5,0x22da7f9e),LL(0xe7b83f3a,0xe42051b0),LL(0xeda536a6,0x9d89d573),LL(0x3ce8431a,0x64d23c5a),LL(0x3eec2b7a,0x07a6be7b),LL(0xb5fb43a0,0x5b672919),LL(0x4a7d1800,0xebe85c45),L_(0x0000002c), + LL(0x54bf1abe,0x835f5200),LL(0x88be0ac1,0x80a69454),LL(0x37a520fe,0x1ae181ec),LL(0xfbf51fae,0x1dbbcb25),LL(0xe0a144d5,0x7481d4f7),LL(0xbb7cdca1,0x89f680f7),LL(0x0d5abe7d,0xb4b624a0),LL(0x391425d4,0x8d9200f3),L_(0x00000131), LL(0xb4188e07,0xbfd1f50f),LL(0x6575b3b7,0xa43201e2),LL(0xbba2a525,0xe0fef193),LL(0xc9408771,0x2d5a89d3),LL(0x5fad41dc,0x95a6705b),LL(0x7d9dc257,0x9020f8bb),LL(0x290e4eb8,0xfbd17a63),LL(0xbb41e71c,0x495438d2),L_(0x000001c8), + LL(0x929f484a,0x0aa411ba),LL(0x72a995e9,0x16fc135a),LL(0xc8dd8a3a,0x2226cfa3),LL(0xefeed6df,0xebb1a266),LL(0xbe66eb40,0xd15ad7b0),LL(0x9e390f8a,0x0c3a1992),LL(0x4d13a05e,0xd151e340),LL(0xcf393bac,0xd2f40175),L_(0x00000053), LL(0x956cdbeb,0x4f6ce102),LL(0x1a20db88,0x3138d132),LL(0x4bd065b9,0x3956528c),LL(0x082878ee,0x1ab3833d),LL(0xc2946565,0x49e6b0dc),LL(0xe955cb4e,0x10248d30),LL(0x1ae9cc37,0x9e6e01a5),LL(0x5567eab8,0x130292fc),L_(0x0000009b), + LL(0x263e83bd,0x531fddd1),LL(0x56ab0f10,0x11105bea),LL(0xe11b8ec7,0x98797be3),LL(0x1992abc2,0xec87b621),LL(0x2f8db083,0x8d258c0b),LL(0xb3d171f4,0xac2ced2a),LL(0xd1cb21ab,0x7bcca55e),LL(0x47bc4dad,0xf87bf80d),L_(0x0000002f), LL(0xf1f3098d,0xb97a56f7),LL(0x30359457,0x13b63e78),LL(0x5c0291e4,0x0560b59b),LL(0xfd66b1ed,0xffcd1e35),LL(0xab62195d,0xa68ddbd9),LL(0x5e1a88f2,0xc8f7a10f),LL(0x5805f677,0xc770a044),LL(0x145b476e,0xf8481566),L_(0x000001cf), + LL(0xfec59218,0xbad5f014),LL(0x29cb3195,0xdeabd554),LL(0xfb9c1406,0x8cab2ab5),LL(0xf39524ff,0x1480bd6c),LL(0x6fbb57c0,0xc932f537),LL(0x34f118cb,0x9e4e5da0),LL(0x6eb8d83c,0xa6fb16c3),LL(0xc80fc4ea,0x5ea94048),L_(0x0000000e), LL(0xbe668aa1,0x614afa98),LL(0x9a412d4f,0x4cadab47),LL(0xb94d7822,0x933864b0),LL(0x51053a74,0x424e5f26),LL(0x3dd43fe6,0x600bdaac),LL(0xf8a04f2c,0xc0b432cc),LL(0x4257110e,0x2f4d8257),LL(0x58edc3e1,0x39fcd24e),L_(0x0000003c), + LL(0x42ad308f,0x07b9420d),LL(0xf2ff228d,0xf800adda),LL(0x959a36a9,0xa755b758),LL(0xe791f71f,0x06570844),LL(0xc9aeec3a,0x2525c1b6),LL(0xf99b3c09,0xfae920c4),LL(0x93e1e24e,0x54cdd224),LL(0x4a6949cb,0x1394e308),L_(0x0000005b), LL(0xc1192578,0xf0af3d86),LL(0x042c14ae,0x035da967),LL(0x54675869,0x65aba6d1),LL(0x00ddf870,0xfb6525b8),LL(0xcb3618ed,0xa58a9ef1),LL(0xa2776a71,0xd7373181),LL(0x9adaa84e,0x5acac693),LL(0x7c38e845,0xa9836617),L_(0x0000017a), + LL(0x17d5163c,0xb8856ff7),LL(0xc5c08271,0xa79d0557),LL(0x32810d21,0x16514840),LL(0xa6e95174,0xce8f06a5),LL(0x393b8782,0xf14b15d0),LL(0xdf3da6f7,0xa398eeac),LL(0xc040c1ac,0xd2eb31c7),LL(0xe9ed34f4,0x5308a3ac),L_(0x0000005e), LL(0x6386bac2,0x294f7bd0),LL(0xf986d8d5,0xec0b7a55),LL(0x55592285,0x96568681),LL(0x15833824,0x307162e2),LL(0x03366d9a,0x196efa15),LL(0x78331de9,0x6afbb75f),LL(0xce11aa87,0x4246ce65),LL(0xa207e319,0x9fea12f7),L_(0x00000198), + LL(0xdf95cccc,0x2c3a928b),LL(0x954c7e07,0xb87fc4e2),LL(0xc556dd0f,0x5dde9996),LL(0xa10a09f2,0x017edf41),LL(0x126b8ebf,0x11739253),LL(0xe86b6fe5,0x1f9be2dd),LL(0x7b89b849,0x4acd9273),LL(0x572a4b24,0x21e7f018),L_(0x000001c5), LL(0xe035ed7d,0x8545db72),LL(0x1c4fb913,0x52bc38af),LL(0xe8b8d046,0x500cd8de),LL(0xd8bd36cf,0xc86ac9e5),LL(0x7fd4ed73,0x82941dac),LL(0x968c57f8,0xcd1842b9),LL(0x806ab108,0x4885bf1c),LL(0x7821dec4,0xb8c5054e),L_(0x000000d2), + LL(0xf8d6129f,0xbddf5a7c),LL(0xb2f43150,0xb4988625),LL(0x3c2f3809,0x1299bbfb),LL(0xb080f7b3,0x84ed45f5),LL(0x20ab0abb,0x824f7bed),LL(0x533e510d,0xd6447243),LL(0xb64fbbb6,0x67c576b7),LL(0xcaa9ee82,0x1f4a9e16),L_(0x000000fa), LL(0x4253a269,0xf68fe622),LL(0x4572de06,0xa777b687),LL(0xcf599bf5,0xa16d5f86),LL(0x2a811045,0x94a33dfe),LL(0x08732642,0xac970a0c),LL(0xd6867a04,0xeb2b7d05),LL(0x0e51a57c,0xad29a28a),LL(0xbf79a38e,0x160ffbac),L_(0x000001d4), +}, +/* digit=100 base_pwr=2^500 */ +{ + LL(0x103efe86,0xcb9e407c),LL(0xaaa13853,0x0e6c71ed),LL(0x1b70a2af,0x5ae18f75),LL(0xcf7e3e82,0x1ec11bb0),LL(0x45c36a5a,0x56ab4b56),LL(0xa4e5487f,0x86df052e),LL(0x3868ef9a,0xc7d75031),LL(0xee422740,0x85ec4e60),L_(0x000000db), LL(0xb7ee652b,0xfaa97f40),LL(0xa3dd7397,0x294d2d1b),LL(0x453daecf,0xe65344d3),LL(0x24bd3f56,0x1c9985d3),LL(0xc78c6f61,0x2675985e),LL(0x8ad7e24f,0x8b4060d3),LL(0x6c928213,0xffdfb749),LL(0x27a6ad57,0xfdbabbe6),L_(0x000001a8), + LL(0x589be6a7,0x313031bd),LL(0x84289d06,0x704962d9),LL(0x522512df,0x4932ec5b),LL(0x6a669eef,0x4db07538),LL(0xcfe74767,0xabd7aebb),LL(0xe7944dba,0xd27fb22c),LL(0x458ef814,0x6ae70494),LL(0xc9680563,0x42cd4fad),L_(0x000001cb), LL(0x6c5b60c5,0x915adb1e),LL(0x108c6584,0xf8937e8e),LL(0x64ea3a9f,0xab88406d),LL(0x61d268f9,0xd6126f44),LL(0xae3ff279,0xed1f3032),LL(0x22a5d3b3,0x3a1b63af),LL(0x2fd7a532,0x90caf928),LL(0xf7a42a75,0x67ff1d99),L_(0x00000023), + LL(0x9fdfb005,0x179f05f7),LL(0x91c70d36,0x1f0a00c5),LL(0x1bdb8aa0,0x13b09f86),LL(0x1877e4cc,0xab098b85),LL(0x804921bd,0x47ca3471),LL(0xb874265f,0x78a5d59c),LL(0xeb734d84,0xbb8aac74),LL(0xa87e5bc7,0x94d792f6),L_(0x00000002), LL(0x635dd559,0x340b409a),LL(0x4cff9fd3,0xe395e617),LL(0x83237476,0x0e435fdf),LL(0x995df5d9,0xacf9026e),LL(0x0535beb0,0xf60fe3f2),LL(0xc3baddc2,0x21d68a60),LL(0x5d079f08,0x8b800543),LL(0xf7a20c38,0x5d89e84e),L_(0x00000010), + LL(0x061ce962,0xee54d7fc),LL(0xe676ddac,0x8e982883),LL(0xfc5997cf,0xe8ccea68),LL(0x4f3d06f1,0x42831bb9),LL(0x57b80d82,0x625c3604),LL(0x7389ec2e,0x5466a1d8),LL(0x13ff25b1,0xf9c50093),LL(0xfb5d45d1,0x0590b45e),L_(0x00000092), LL(0xab4e32c6,0x08ec78b5),LL(0x24014646,0x53a9c0b3),LL(0xd5ffb795,0x5594ddea),LL(0xf933bdcc,0xdc110f8b),LL(0xd5249289,0x5d52f652),LL(0xcb4147d2,0x95ab06c0),LL(0x968adb2d,0xa1bbff17),LL(0x4039be6f,0x8c602266),L_(0x000001e4), + LL(0x186c159d,0x6c1da851),LL(0x87f87d48,0x2ffecac8),LL(0x326e8ac4,0x70a401c5),LL(0x68a082d2,0x0bd703fe),LL(0xbb17b4ec,0x9fe82427),LL(0xd5029b1a,0xf75c6b25),LL(0x8859df58,0xfaac56e2),LL(0x0f97f9bf,0xc8a4fbc7),L_(0x000001dc), LL(0xaf73b932,0xe721dc13),LL(0x98f115f8,0x825bde14),LL(0x102f60c7,0xf0780fd3),LL(0x92e5a9e9,0x9fcac3f2),LL(0x96a4edbc,0x51ae4427),LL(0x123f8f26,0x622ca1bb),LL(0x3d2d6a72,0x4fa4ca10),LL(0x77d0f199,0xc7c42ca5),L_(0x000001c9), + LL(0x33e577af,0xbf557eaa),LL(0x98005156,0xca7cba0f),LL(0xe3d41486,0x73d1e9d6),LL(0x3a30b9f9,0xaa3443ad),LL(0x99856b55,0x81921940),LL(0x0ca4724f,0x250cf13f),LL(0xba1696b8,0x9b6f0f69),LL(0xba1deb1e,0xc260563b),L_(0x00000021), LL(0xa037cb66,0x7437eb08),LL(0xfaf55d7e,0x579149c6),LL(0x43f7c61c,0x921e4c69),LL(0xeba03e36,0xc342ab26),LL(0x0a3ade34,0x86264eb6),LL(0xcf120523,0xffc57653),LL(0x914e6e20,0x5770eec7),LL(0x2d678456,0xf286b491),L_(0x000000e5), + LL(0x3106fc3e,0x9739a62b),LL(0xd166966e,0x6ea97e6a),LL(0xbc843284,0x0fb2aa2b),LL(0x1768bc4f,0xbd1f5f87),LL(0x0a51f2d1,0x8890f99b),LL(0x5f53245a,0x6a0000aa),LL(0x38e0dc0b,0x49547c15),LL(0x3dbece21,0x5c37a1c3),L_(0x0000000e), LL(0xeab56de1,0xcb15a8e3),LL(0x8e0ff0fc,0xc00352a7),LL(0x0dae3bf3,0x716e7d48),LL(0x273cee30,0x83dde892),LL(0x00962a42,0xe18ae53b),LL(0xcb2ce674,0xb8c78835),LL(0x12176412,0xc6faee27),LL(0x94e0f5e7,0x2f8a2462),L_(0x000000f7), + LL(0x78569761,0x266d871a),LL(0xe7df9590,0x015aa94b),LL(0x4e8bbf72,0x150482ac),LL(0x12880439,0xaa4b2f7c),LL(0x53495f23,0x5ef777bb),LL(0x04a67481,0x38a798c0),LL(0xc7aadc88,0x308a425c),LL(0x2ccb5f57,0x0a9adc43),L_(0x0000005b), LL(0xb506af85,0x718ebc6c),LL(0xdd21e3a6,0x2c50bba9),LL(0x56098fde,0x7b6e3c4d),LL(0xa8a72185,0x02f4d7a4),LL(0x9218f9c4,0x55530e5d),LL(0xa541a6f2,0xbf7b3c63),LL(0x97421bb0,0x04f0181b),LL(0xe7a08f28,0x1e51023d),L_(0x000000c8), + LL(0xb7a89c2a,0x59bf73de),LL(0x701684bf,0x59572fc8),LL(0x8134fb14,0x116042d5),LL(0xc2db37ef,0x49c61bec),LL(0xf05c9753,0x0b65e976),LL(0xbbefa454,0x3b1cd2a7),LL(0x2cb97f65,0x790f0086),LL(0xc011e095,0x388368d8),L_(0x000000c2), LL(0x170eba17,0xc6e8d2e1),LL(0x609e9ebc,0xe69224f6),LL(0xa74251dc,0x88ae4a2d),LL(0xf5331e8f,0xf73bb04a),LL(0xfb6fb779,0xf42091f4),LL(0x7aa76758,0x23ea8c88),LL(0xf921d2fe,0xb43fc164),LL(0xb83f6c60,0xf692414b),L_(0x000000f9), + LL(0xb2f332da,0x1235350c),LL(0x3fc1272d,0x28803380),LL(0xcc94724d,0x52679e63),LL(0xa1b6d063,0x5f59afa4),LL(0x8fd15f1d,0x1998f9ec),LL(0x67514283,0x856a5843),LL(0x0b1f071f,0x44e35d97),LL(0xa4396ed8,0x12b07c24),L_(0x0000015d), LL(0x209d5128,0x36d77579),LL(0x3d71f02d,0x6903b9cc),LL(0x242255da,0x255e80b9),LL(0x0d577ece,0x8c99ca99),LL(0xcce7a8ce,0x6e67b351),LL(0x8274de19,0xb1789c3b),LL(0xd9d46d98,0xb68f17a4),LL(0xbe658d62,0x4303e990),L_(0x0000003f), + LL(0xb2d11384,0xc0e43052),LL(0x18ea1e36,0x6499f986),LL(0x67f0543f,0x6c81b7cb),LL(0xfe7f0035,0x0049686c),LL(0xc866a608,0xd1d9672f),LL(0xad63f7a9,0x402ddc59),LL(0x0430d4e2,0x90a63a4e),LL(0x212afc89,0x76142f1f),L_(0x0000016b), LL(0xa5a4f004,0xb4669198),LL(0xe3c06d85,0x9dbced32),LL(0xd46b4406,0x6dc0df8d),LL(0x13bbdb5b,0x26fe23c2),LL(0x9e72fe86,0xf3f82db4),LL(0x9908a610,0x21de6fb9),LL(0xacd7a7dc,0x84215e98),LL(0x8d6e691f,0x6fc1bb6f),L_(0x000001a9), + LL(0x6a2a68fa,0x1a45088f),LL(0x097c3677,0x93e569b4),LL(0xf6248b6e,0x76442347),LL(0xf5aa90b0,0x7db67859),LL(0x47468b1b,0x0335b7c8),LL(0xc14d722e,0xbdb192bb),LL(0xa89357c4,0x6091e296),LL(0xabbee708,0x6196a7a6),L_(0x000000b7), LL(0xb79d5c35,0xfa81b0bd),LL(0xb8c1d912,0x9c98ad08),LL(0x6f43a564,0x487e8e86),LL(0xb8305a15,0x68d70b11),LL(0x4f253374,0xda0a3387),LL(0xc3ee2674,0x12970085),LL(0x7d1dcff2,0xabf94fcc),LL(0x3abb8f41,0xaeb5fcca),L_(0x0000004b), + LL(0x43ae6f56,0x2ecfea4f),LL(0xbfbd972d,0x1717ff60),LL(0x50d6adf6,0x54f2d354),LL(0x6c6e24d6,0x2e42a5a9),LL(0x8ef967c8,0x2c029c93),LL(0xef490e17,0x74b0a604),LL(0x3a515366,0x835d8fbf),LL(0x878ca8ab,0x312194f9),L_(0x000000f2), LL(0xb19f5e63,0x0a7f849d),LL(0xe7343af4,0xa13e0960),LL(0x93d2a93e,0xd7e39973),LL(0x58e4acec,0x89b42cb1),LL(0xb14fea72,0x128188b6),LL(0xaf37faab,0xfe8c3b50),LL(0x51d9f7ee,0x2b121d0a),LL(0x5997d399,0x6e0873a5),L_(0x0000004b), + LL(0x70a88d95,0x0ae00340),LL(0x2c6b6f8f,0x941fb569),LL(0xd4a8a83d,0xfd77e569),LL(0xfe3e5239,0x2906296c),LL(0x5d2269e6,0x779ada15),LL(0x65afaeca,0x46fce296),LL(0x41d7fedd,0x0ceb36d9),LL(0x39bc7e8f,0x8c21b413),L_(0x00000081), LL(0x5ebed878,0x04b873d1),LL(0xb3e0c338,0x061d2bb8),LL(0xd36ffa7e,0x596e92bc),LL(0xbde7e857,0x58c9a5ab),LL(0x78a8b297,0x34780b9b),LL(0x6bd40718,0xac60511a),LL(0x2d3c02e9,0x83e6ac72),LL(0x6d092687,0xb6802b12),L_(0x00000070), + LL(0xfd72e9bb,0x99783135),LL(0x67ff52e3,0x8dc913fd),LL(0xab427847,0x8986e4a7),LL(0x5377b12a,0x8a8d9fc8),LL(0x57fef8f8,0xb61bed13),LL(0xf189dc79,0xd4bb14a1),LL(0x2e6d28f7,0xcd3d6f13),LL(0xf00e6579,0x74cb87a6),L_(0x00000146), LL(0x14efe417,0x9fb6caaf),LL(0xabbb19b6,0x951205f5),LL(0x787b1023,0x440fd20f),LL(0x4968e195,0x6d5f6164),LL(0x33cbed34,0xa2722dcb),LL(0xf576c320,0xbddf2d94),LL(0xb80ba0c8,0x414feeda),LL(0xeb25ced6,0x5a089c08),L_(0x000001e8), + LL(0xaa61ebad,0x6aced3dd),LL(0xe5fe2dd4,0x0bd3e3fd),LL(0xfe14f9f4,0x09520569),LL(0xd818d1a2,0xb4968b88),LL(0x82f0bdc9,0x0b8b7732),LL(0x6520e3de,0xfe9e8edc),LL(0x272ff767,0xbc017cf0),LL(0x0f65dc99,0xc5848eda),L_(0x0000013d), LL(0x0a9b50b0,0x4ea634ab),LL(0xe6308ff0,0x7b191db6),LL(0xee04399a,0xddea9de7),LL(0xda7bdea8,0x492d45e6),LL(0xb54c55ae,0x39e666b7),LL(0xf573f4e9,0x0c925a51),LL(0x0292c159,0x71f91622),LL(0x80fc7f50,0x8069f2d3),L_(0x0000007b), +}, +/* digit=101 base_pwr=2^505 */ +{ + LL(0x53639a92,0x1ce7b1b4),LL(0x1474fb3a,0x0d260e43),LL(0xaefab808,0xc4e32954),LL(0xfaf9e670,0xb5bae76d),LL(0xde42d0d7,0x9eb4687f),LL(0x5f45bfa8,0x7d89b1ca),LL(0xba638bbc,0xc0ecce4f),LL(0x1de873fc,0x7a3238a2),L_(0x00000170), LL(0x36c9c2b4,0x318f9f96),LL(0x2cb815fb,0x6f676d6c),LL(0x7560919e,0x41cc633a),LL(0xe2d47525,0x9a79e211),LL(0xe199c599,0xfbee081e),LL(0x265f515d,0x3107dec5),LL(0xd0ea4e25,0x6f2bdc9e),LL(0x5a539ab3,0xcbfab41c),L_(0x00000122), + LL(0x6ffa11c2,0x1bfc6586),LL(0x1f882706,0xe25c9a78),LL(0xeb5d1a25,0x12852d54),LL(0xe5fcb1fe,0x4f331734),LL(0xd8c5dfb7,0xe48b7e54),LL(0xcfadcca2,0xe9b41639),LL(0x193402d5,0xb8e1f8c2),LL(0xc49e8f71,0xae02f348),L_(0x0000005a), LL(0x4e33ca43,0xe98a1f64),LL(0xa7cb416c,0xbd595ac8),LL(0xe328cd45,0xf8fd4eb1),LL(0xec3fd8cb,0xf768cfe8),LL(0x9eb626b0,0x476b1bbc),LL(0x069d1524,0x8d0ffe31),LL(0x220edd8d,0x2925aa89),LL(0x60b37558,0x7ea1a636),L_(0x00000128), + LL(0xe83c1031,0xc5cc621c),LL(0x434bc9bb,0xac957c67),LL(0x27bcbc47,0x430a3686),LL(0xace0a905,0xed2cb5ad),LL(0x38aa0831,0x0f4f5d32),LL(0x1fa3c11d,0xc48e91c9),LL(0xedbfb351,0x98229765),LL(0xdf2591e4,0xeef1832b),L_(0x000001e9), LL(0xe38e4555,0xa53bea9a),LL(0x97db1ecb,0x196518c9),LL(0x58970b56,0x6895c332),LL(0xc6b46d1e,0x0fe772a3),LL(0x18e44ad3,0x48216056),LL(0xaf1a5dd6,0x86d1933f),LL(0x18ae6deb,0xdf5a53d7),LL(0x4345a6ba,0x79fa1e4e),L_(0x000000e5), + LL(0x0deafce5,0xa11885f4),LL(0xf942172b,0xd640aa98),LL(0x17ce6b52,0xdc1bfcf8),LL(0x9d8e40bc,0x14a7d638),LL(0x3c804e7d,0x80e95516),LL(0x63048fa8,0x4af7c92b),LL(0x15381b03,0x88fd6851),LL(0x73ec6a96,0x79207faf),L_(0x000000ff), LL(0xf4f85d1a,0x6ba36666),LL(0xf47de871,0xfbfc6c17),LL(0xf9474540,0x70e03b35),LL(0x72b1ddc6,0xad63874b),LL(0x7f48bdd3,0xd249ea68),LL(0x6a15d7cf,0xc1614192),LL(0xe9d101f9,0xacd8d963),LL(0x1b9b1c2f,0x09164077),L_(0x000001e7), + LL(0xc3b944a7,0x8b0aff56),LL(0x97b94164,0xacb2b2a3),LL(0xfade0d57,0xe22c59c3),LL(0x72ad3ddf,0xd8b6f7d4),LL(0x4a332f59,0xe436d0e5),LL(0xb28ca267,0xa69516f2),LL(0xc620d57f,0xdd5b988f),LL(0xb3a24be6,0x14f91326),L_(0x00000166), LL(0x7dc7da50,0xf918114f),LL(0xa2d07106,0xb35f8ba3),LL(0xfe3e2734,0x583e00bb),LL(0x27f4b785,0x5b427e96),LL(0xb8110ec6,0x6198344d),LL(0x2666790f,0xc2f9267c),LL(0x9a6bed52,0x15a1c587),LL(0x05e6612c,0x2f681667),L_(0x0000001c), + LL(0xeddc635b,0xb97227a6),LL(0xf72fcd23,0xcc5023cf),LL(0x16ff8449,0x4d184e1b),LL(0xa6e3cc25,0xb6176789),LL(0x14e442c7,0x200d2b68),LL(0x147e4e2b,0x9903da3b),LL(0xa26bbf3b,0x6b6aeb08),LL(0x849b5dc2,0x06fb145d),L_(0x00000147), LL(0xbb6ef597,0x250dac2d),LL(0xac7eb4e4,0x0cadff32),LL(0xfb97f1fe,0x1e4bc394),LL(0x2549b2e2,0xa4d9b459),LL(0xc81df54d,0xedb4d3d9),LL(0x885b7f7b,0xa8c245eb),LL(0xc2642e0e,0xfd485de4),LL(0xbbbd9d24,0x16b28ef9),L_(0x00000117), + LL(0x82620992,0x9bdd489e),LL(0xf9ff35e3,0xe0746b33),LL(0xfea9ab53,0xc4e354c0),LL(0x5438eb93,0x660ea2e7),LL(0xe43d99e6,0x15697bab),LL(0xcb7634f2,0xacd2ac86),LL(0xcf1f1144,0xd531df3b),LL(0x2d6020c4,0xa6e4c25c),L_(0x00000082), LL(0x5eadffbf,0x79147d9d),LL(0x6799957b,0xf22818e2),LL(0xfc746f29,0x7b9b1ddc),LL(0x87c40fbc,0x5518ebb6),LL(0x12ffe947,0x9a81391e),LL(0x997b0c31,0x3a724bee),LL(0x02662680,0xe08a5f24),LL(0x1ebe1250,0x1f28a865),L_(0x00000175), + LL(0x3b6bccc6,0x4b51d15f),LL(0x4d9896e4,0x61b0ed08),LL(0xdf4f3be0,0xcedf84c9),LL(0x6ed8e29b,0xf45bde62),LL(0xaa49395c,0x128499be),LL(0x812a1bfb,0x30e7b9a9),LL(0x442e44bd,0xe50016d9),LL(0x251b4710,0x608c2577),L_(0x000000be), LL(0xb4c58ea8,0x8972e1af),LL(0x6a3b7639,0x2d603c3a),LL(0x64b41953,0xad6090f4),LL(0x1c1cc6d5,0x644bda3e),LL(0x79fc4551,0xe003b3c6),LL(0xc4fe3bf0,0x879a2d4d),LL(0x9f3993d4,0xd0249205),LL(0x90933a0b,0x4bf1c894),L_(0x000001f5), + LL(0xbaa0eb60,0x64977e8a),LL(0xed1a5135,0xdb1eee40),LL(0x7fe9e6fc,0xef595c17),LL(0xbc2a7a81,0x4d74eea7),LL(0xafbb2385,0x34f92af2),LL(0xffa66ed1,0x9f246323),LL(0x91252082,0xd49955a9),LL(0xa2901a50,0x612b9e01),L_(0x00000163), LL(0x19a6a510,0xae624c6c),LL(0xf34dd865,0xbf77202a),LL(0x6cbac9c8,0x0d692aaf),LL(0x2471eb03,0xae2ff6e2),LL(0x4e8b52f6,0xfc37aa01),LL(0x96b6740a,0x5ca85277),LL(0x4fc2258e,0x1a66c773),LL(0xd7c07ad3,0xc3b9c722),L_(0x0000014c), + LL(0x3efb63d7,0xb52733b4),LL(0xadb5f371,0xa39008b4),LL(0xbebb4df3,0xb7dee0b6),LL(0x6cbc3a0b,0x889767ea),LL(0x0970ea6b,0xfb4bece2),LL(0x7f67b6be,0x890b0f75),LL(0xe72afc5e,0x8395198d),LL(0x6c13b8e8,0x792b05e1),L_(0x0000011b), LL(0xcd597e49,0x85730a39),LL(0xd16d451d,0xbb9ddfb7),LL(0xc5ad35f5,0x46f1005a),LL(0x04cccc76,0xfc9aa038),LL(0x199ada1e,0x03f6f34d),LL(0x48f0a0bd,0x200aa943),LL(0x2532adab,0x83389203),LL(0xa871ac66,0x84008498),L_(0x000001a6), + LL(0xd09b5635,0xc63c436d),LL(0x1bf51ded,0x36468adf),LL(0x71acd515,0x98fc5e09),LL(0xf48ba93b,0xed9c3a1c),LL(0x0c6b0d79,0x01b9c574),LL(0x24610fb1,0xb968199b),LL(0xe209e9f6,0xbbce4f03),LL(0xeee7632a,0x34954c95),L_(0x00000158), LL(0x7c32158c,0xce330212),LL(0xeb40d5de,0x1694cc96),LL(0x981c6977,0xddf4fc29),LL(0x9ea41d2c,0x89f8c78d),LL(0x86af7d31,0xd67b4c4c),LL(0xf2c9d3cf,0xad9e3351),LL(0x16fcf6af,0x702ac15c),LL(0x16ccd30f,0x671dd4b0),L_(0x00000056), + LL(0x31986e71,0xafcb621c),LL(0xe99b97d2,0x3f65ee43),LL(0x0c39ca80,0x14bf4d0a),LL(0x4ebb930d,0x840ff2ab),LL(0x894804fb,0x76798c37),LL(0x89a1b227,0x3aa6a099),LL(0x34d5a9a4,0x6f4a66e9),LL(0xdd3ebcad,0x671c865a),L_(0x000001de), LL(0x63a60589,0x9cc5953b),LL(0x6c38743c,0x85a6854a),LL(0x08e7cd4c,0xf39a75be),LL(0x936c5fc4,0xfc799df1),LL(0x93f15bcf,0x739e6699),LL(0x4c317bf9,0x2e5c38de),LL(0x6db73251,0x427a1224),LL(0xa307eb83,0xe63f9f36),L_(0x000001b4), + LL(0xdc077b67,0xa1669c8a),LL(0x49cbb3e1,0xd5e45b94),LL(0x26a91035,0xe7362c4d),LL(0x3ddf32d0,0x643b77a9),LL(0x27b4f14e,0xaa5ac709),LL(0x1246b2b6,0xfed505b3),LL(0xce87322e,0x6473f9c6),LL(0xffc4c045,0xaf22c521),L_(0x00000154), LL(0x4300e539,0x1c3ea4fc),LL(0x79dff91b,0xe9151768),LL(0xe8106a01,0xced48484),LL(0xd2bbaae2,0xb0b62aaa),LL(0xec766cbf,0x4fd3762d),LL(0x3740af93,0x1969f618),LL(0xeff16696,0x481e8d46),LL(0x08f70c0b,0x17f67b91),L_(0x000001f3), + LL(0xe6e4ffd4,0xd4cc0359),LL(0xdeb8f8ef,0x6a7bc267),LL(0xb0f6a0db,0xceec7c5d),LL(0xbe8f401f,0x328be59a),LL(0x0120834c,0xea6a0206),LL(0x5b979c4e,0xc2f2cb42),LL(0x6693b49a,0xc70270e4),LL(0xcb7ea005,0xf0d4e23e),L_(0x000001f0), LL(0xd93cd84b,0x59acbb8f),LL(0x771ac6f4,0x37d3f220),LL(0x5f43f61d,0xe983e186),LL(0xe1ef31c8,0x66433715),LL(0xe567c88b,0x4ca008fa),LL(0x4f949b3b,0x164fa949),LL(0x7f0981e7,0x55a6f6fe),LL(0x85c2f160,0x53287c30),L_(0x00000084), + LL(0x0fe60315,0x778f4301),LL(0x3d35c1e7,0x42ff27b9),LL(0x4e622807,0x9ba721f4),LL(0x9122d492,0xc966361a),LL(0x0b7b9eb9,0x824265f0),LL(0xd71fbe97,0x90d81101),LL(0xf6012c22,0x3aa81035),LL(0x77c80e09,0x23ef1ac0),L_(0x000000e1), LL(0xc562080c,0x0edd3c7c),LL(0xd4bfec34,0x6ac1e8f3),LL(0x8628425d,0x3dfaff6a),LL(0x6eeb0125,0x2bd725ab),LL(0xf2cb02f5,0xeb0228d9),LL(0x63e15d94,0x41589407),LL(0xaa99cd36,0x94061e78),LL(0xf8ab88d8,0xbd436309),L_(0x00000047), + LL(0xb83d39ba,0xfe497366),LL(0x1a08bb8d,0x3fb2e32d),LL(0x68fadea4,0x90040dcc),LL(0xc78d9e5b,0x6201359b),LL(0x8999242f,0x4ca94e09),LL(0x83f1e38d,0x3e9a9377),LL(0xf5e42def,0xc5c271ed),LL(0xd3d42e09,0x57d11cda),L_(0x00000043), LL(0x7f14caf1,0x52b71020),LL(0x42fb2920,0x840b578b),LL(0x4e0acb78,0x9f5c859d),LL(0xbd1059c6,0xf5ce1fef),LL(0xdfc0d187,0xd99b98e1),LL(0xb702018d,0xd9d695b6),LL(0x7056ff1f,0x1187f703),LL(0x73121d9b,0xc912e80b),L_(0x0000016f), +}, +/* digit=102 base_pwr=2^510 */ +{ + LL(0x89622bfd,0x12bebe9c),LL(0xcafd2ca2,0x487abee2),LL(0xd290457b,0xc04143f4),LL(0x05d13bf4,0x716aab7d),LL(0x067f0ae3,0x7740d413),LL(0x5925a309,0xebe6d02d),LL(0x14370b8d,0xe8ef2c27),LL(0xfae20be9,0xdcc77fcf),L_(0x000000f4), LL(0x8d09dc72,0xb463618a),LL(0x49d83802,0xe00f8249),LL(0x50666aae,0xd5a21e88),LL(0x54be3730,0x258522a3),LL(0xa6ce164b,0xbf3fd223),LL(0xfefa386c,0xb7ba5ba4),LL(0x479bc6a0,0x378ecff5),LL(0xece410bb,0x58171ec6),L_(0x000000fd), + LL(0xb8ebdae4,0xfcc42342),LL(0x0addd068,0x2e8a76fa),LL(0x1ba58a99,0x972d98aa),LL(0x585d2056,0x5a290a6f),LL(0x51a66712,0xa47990be),LL(0xab19e664,0xe44696be),LL(0x2f64d1c3,0x490ab4a2),LL(0x0b8ce484,0xf862854a),L_(0x0000012f), LL(0x0acf9a53,0x3b3fd199),LL(0xdaaafe7e,0x1c1f1592),LL(0x3015bdb3,0xadb01684),LL(0xb2dbb2d0,0x670d1295),LL(0x3f77ef5d,0xb3f98aca),LL(0x51408bf1,0xb5280fd3),LL(0x0b5ee9d3,0x3a7a5866),LL(0xe5879122,0x2a28af86),L_(0x00000111), + LL(0x2aa4eb8b,0x719fd8c6),LL(0x2ae67788,0xd8b75613),LL(0xd4b10cdf,0x691c837c),LL(0x7871303d,0x169b2b0b),LL(0x0d01af02,0x6b821f74),LL(0x33573229,0x82eb3840),LL(0x782b872c,0xef815609),LL(0x64bda6ba,0x9ad650a9),L_(0x00000087), LL(0x654fa37e,0x9665a8db),LL(0x761f0aa2,0xe37a4531),LL(0x4eac5b19,0x586ef6d9),LL(0x886dc010,0x014c7183),LL(0x075d0e7c,0x55263f06),LL(0x8a38c3bb,0x5b8b13c3),LL(0xf18380c5,0xcefec3fb),LL(0xb50c9c44,0x64a25aec),L_(0x0000009b), + LL(0x5131c51f,0x65ab5849),LL(0xd5115c76,0x8739b754),LL(0x840523eb,0xb96b253e),LL(0x8a1f77e3,0x765d9707),LL(0x8742a046,0x7e942e5b),LL(0x1539823d,0x0b3194bd),LL(0x560b9978,0xb52679bf),LL(0xbda6ff32,0xb360761a),L_(0x000000b6), LL(0x92820e93,0x881e08a8),LL(0x208f9f2c,0x7e5fd839),LL(0x4e86968c,0x305d2580),LL(0x76aeb554,0xb44037fc),LL(0x24c686c9,0xb80d02e0),LL(0x20e62e51,0x5774d5a6),LL(0x653df90e,0xf0000eae),LL(0xc9b31961,0xdb5b072a),L_(0x000001cc), + LL(0x0eaaf4f9,0xfbc7cc75),LL(0x23c2ade5,0xd4d5bade),LL(0x734024b7,0xaca16532),LL(0xb04a1289,0x748144eb),LL(0x599436fa,0x2dc69353),LL(0x292c7b3d,0xe403f7a9),LL(0xb831d302,0xb390c4b2),LL(0xefcb4c53,0xe756ba3e),L_(0x000001d0), LL(0xe0c0eb7f,0xf6a7b102),LL(0x411f77a6,0xc71476dc),LL(0xae920f7c,0x2edbab42),LL(0x28d62eb0,0x3c6b13bc),LL(0xc5296531,0x35b583ce),LL(0xfb32a342,0x66e0af0c),LL(0x23a5daad,0xa449b545),LL(0xa7558c28,0x50f4b633),L_(0x0000017a), + LL(0x6383559c,0x52741d3a),LL(0x6d33fc8c,0x98c5775c),LL(0x595adc9e,0xdeefc5ad),LL(0xc4827819,0x94326f99),LL(0x384312e5,0xc9b59642),LL(0xfb777603,0x6642d1f6),LL(0xfebd7a7d,0x04e71c64),LL(0xd51e2c8e,0xa96e4272),L_(0x000000a5), LL(0xfe188e45,0xc6c9a2ae),LL(0xcfd37d07,0x870985c6),LL(0xb64a1b0a,0xf54e2700),LL(0xd57c3e8a,0x97a88233),LL(0x4ccc22c2,0xe7db1ecd),LL(0xfff2895e,0xb9be3600),LL(0x2a410545,0x8934c5cb),LL(0x71986e7a,0x545e296c),L_(0x000001f3), + LL(0x8664ea8c,0xb006fef1),LL(0x18fac77e,0x2d10f805),LL(0x144d6d0c,0xcefe66ce),LL(0x7ca13881,0x6bc51a22),LL(0x93e1d2aa,0xfe6fc406),LL(0xdde6594a,0xcfe90933),LL(0x92333bca,0x2f2f839c),LL(0xf1434e7e,0xf4d730b9),L_(0x0000013b), LL(0x84887df0,0xbd49b873),LL(0xf75f870f,0x4f7c90e8),LL(0xe89d9ed4,0xf464a50f),LL(0x9a76a69e,0x3d9a6cbe),LL(0x0a745d9f,0x0f9b1034),LL(0x87bec297,0x5fe8ae9e),LL(0x7f1ab569,0xad783c5d),LL(0xea58b4b0,0xd37c5e74),L_(0x00000042), + LL(0x9ff8d786,0x35735942),LL(0x05d76bb3,0x995f4b0a),LL(0x58d0fe01,0xe40e0f1c),LL(0x21dccd2e,0x3af9c629),LL(0x40ab0ca3,0x074069c3),LL(0xa30b637c,0x098a102f),LL(0x44888bc2,0xde377018),LL(0xb2e96e33,0x2f69ed6c),L_(0x00000012), LL(0x0f70d506,0xe57ed3ba),LL(0xe59c4f2c,0x2492cf26),LL(0x5879a0eb,0x3d130599),LL(0x75760ae9,0x4103b206),LL(0x89f9d0d0,0xa2b74089),LL(0x4b0ad618,0x723e7b44),LL(0xab5c813e,0xebb80451),LL(0x305a1f27,0xda320616),L_(0x00000152), + LL(0xd06cea6e,0x063477df),LL(0x7ad6ed83,0xdfc2e4d9),LL(0x7d58a8ed,0x1a10d461),LL(0x9fae700f,0x9ad7943b),LL(0x37aee0fa,0x575deb90),LL(0xef0a0865,0x926de4b2),LL(0x3a26c380,0xc5d7be4b),LL(0x910a980d,0x050f79a4),L_(0x000001a0), LL(0xdc85a306,0x8a336a7f),LL(0x078dd7d7,0x3626cc20),LL(0x5cdeb063,0xc2b171da),LL(0x273ae54e,0xab82b41f),LL(0x10b49e9f,0x9d867301),LL(0xca9e1b59,0xe2e4e776),LL(0x7eb0c998,0x2437d70b),LL(0x3320fabf,0xf9bfec6b),L_(0x000000cd), + LL(0x7b0fa120,0xb86bacca),LL(0xfbcd14e3,0xa097b60e),LL(0xd21921a2,0x57795942),LL(0x96c19ceb,0x8537f555),LL(0x2145d8ff,0xe5e61d05),LL(0xc2c7c89b,0xbe0f1c2c),LL(0xb88cf04d,0x77a2f17b),LL(0x47c65308,0x9d259bd7),L_(0x00000141), LL(0x9b1b0b2e,0x3ba530cb),LL(0xac5182ed,0xea3a3af8),LL(0xfa4f9dd8,0x03a0d517),LL(0x7b2d9856,0x2bd4dfe1),LL(0x6d8ccb18,0xa68f896d),LL(0xc8c4d1ec,0xa1acec0c),LL(0x0fdaed1a,0xdc43340b),LL(0x1a6552ac,0x0d7c6ed7),L_(0x0000018d), + LL(0x1cfe1d00,0x35fc4f26),LL(0x06e2cd4f,0x7dce43d2),LL(0xfe4a6fd7,0x8884f4ad),LL(0x8bc475be,0x9fb2b07e),LL(0x9fe1c66d,0xde1173c5),LL(0xa0cf5d6a,0x6059a297),LL(0x4938219f,0x49237fdb),LL(0x01e57227,0x8887ac3d),L_(0x0000014e), LL(0x35f11932,0x63db0e8e),LL(0x431b9b60,0x0b8d8078),LL(0xdb56a2b7,0xa040057a),LL(0x856dd526,0x87409cca),LL(0x3d5f500d,0xb482e56d),LL(0xd9cd1b6f,0xbf890467),LL(0x815814ee,0xafa8c19c),LL(0x2dd2fd09,0xf9865d5f),L_(0x000000be), + LL(0x4ab480b6,0x71a65ccb),LL(0x061e197a,0x2360920f),LL(0xb306dab3,0x9d9428ae),LL(0xee526750,0xcbaf9d5a),LL(0xf58e47b1,0xd9a6f7e9),LL(0x696a3350,0x5af36c30),LL(0x1f66ddb3,0xeaff438c),LL(0xd4937e17,0x01e8c119),L_(0x0000000d), LL(0xac9df61e,0x22655044),LL(0x106e2b83,0x1ae7bb1e),LL(0x343bc8e6,0x99139508),LL(0xbc1e06e0,0x166453a2),LL(0x966bd6b8,0x3756d0eb),LL(0xb4bb44e2,0x3795c5a1),LL(0x625fe170,0xb7605deb),LL(0x426f42f1,0x437cc248),L_(0x000000f0), + LL(0x06cfe4f0,0x868d9076),LL(0x47442b11,0xec69d70f),LL(0x7bd07599,0x5e554262),LL(0x4c93a1e5,0x9ba31acc),LL(0x5fa3f8f0,0x5118c586),LL(0xed99e567,0x9ed35f7d),LL(0x9e3fd347,0xce15a315),LL(0x0a315f79,0x0ccab1ce),L_(0x00000149), LL(0x86dda811,0xb40d1386),LL(0xb2f0ff2d,0x0d2ec043),LL(0xc03a536d,0x01fe94d1),LL(0x753c381c,0x6d3c523b),LL(0x468beaaf,0xeb47f9a1),LL(0x5d8bebfe,0xeaf18315),LL(0xb071abcd,0x1a924dbc),LL(0x3c1a4715,0xe7415280),L_(0x00000062), + LL(0xcce52519,0xd9a705a8),LL(0xef1ab816,0xb6139e2f),LL(0x4fdd8131,0xdab1fe19),LL(0xf7fb9e55,0x94460d6e),LL(0xeb0d1405,0x6a211783),LL(0x6ba4226e,0x8b3a8c56),LL(0xf2eeb428,0xf7e95eb9),LL(0x0b4ffc60,0xd9357daa),L_(0x000000d1), LL(0xe66a7792,0x7209db94),LL(0xdb0eb453,0x0352a746),LL(0x3c883ac1,0xdd4b846f),LL(0xb4107c7d,0xe960d5ec),LL(0xf20e2f77,0x61292f8f),LL(0x8b9e3ba1,0x85963097),LL(0x1218ba8c,0xbba1103f),LL(0x61201057,0x9bfd2fd3),L_(0x000001f6), + LL(0x08fb83e5,0x0419bb93),LL(0xc48ce791,0x7a851af1),LL(0xd3ade5e1,0xedcfe59c),LL(0xbf3e625d,0xd7763ab7),LL(0xd2c5aaf2,0x625d14ad),LL(0xb7b3d23e,0xbf8e7638),LL(0x7079ecb0,0xc7d9e9b0),LL(0x9c8fcf47,0x59d7dc01),L_(0x0000010d), LL(0x074d6a10,0xceed2d69),LL(0x1a1995f3,0x61d18bb0),LL(0xf02767e3,0x33398884),LL(0x68db2be1,0xbdeb7872),LL(0x3e3fa104,0x82f62909),LL(0x2e4ab79a,0x55582545),LL(0x67badda6,0xa7bb473e),LL(0x26c76ce9,0x5ab23400),L_(0x00000118), + LL(0xc0ebb49b,0x76edd0b2),LL(0x7ef78c95,0x089746d8),LL(0x86ff89a1,0x30dee546),LL(0x51992a8e,0x8362adcd),LL(0xafcb70ff,0x883f2631),LL(0xa55108d8,0xa13e25b5),LL(0x93138472,0x1fd32baa),LL(0x64387fbe,0xdb1b0497),L_(0x00000116), LL(0xe8652373,0xe1299928),LL(0xce8fd7e9,0x16c54d21),LL(0x938b0123,0xad0e62d7),LL(0x4d602bac,0xf9df41ce),LL(0xc55138cb,0x25dfe098),LL(0xbc01e0e6,0xbf9a6851),LL(0x2bdbc63d,0xa70b0da1),LL(0x8b07ceba,0xe5f9aaec),L_(0x000001a1), +}, +/* digit=103 base_pwr=2^515 */ +{ + LL(0xaca511d0,0x698e16fa),LL(0xcb4d0031,0x2e96a74c),LL(0x7b609854,0x679b8501),LL(0x87d91373,0x6f39c358),LL(0xa39fd4a5,0x3aea2bb2),LL(0xc7eef60f,0x4e8edd3f),LL(0xd4812888,0x89e1d001),LL(0x2f4d1fa9,0xc5d59373),L_(0x0000013b), LL(0x0a629c27,0xd6fdccbf),LL(0x6bc14652,0x5f32800f),LL(0xd29c1358,0x69e7f62a),LL(0xf3a9fdce,0x9418d0db),LL(0xdaa9f4b6,0xf492796f),LL(0x525ae5fe,0x32f4a27a),LL(0xd91d1353,0xcc1a7293),LL(0xdc6b1bb1,0xb6f6d6fa),L_(0x0000010a), + LL(0xe776fe9e,0xe46e8257),LL(0xc90ea2e2,0x285dece1),LL(0x02e70a0d,0xd2e4f07e),LL(0x9e22652f,0x7fb0667d),LL(0x5325ca4b,0xe36daaa9),LL(0x04df1305,0xeebe9d3d),LL(0xfcd0755f,0x570e3be0),LL(0x4f74e603,0x821555f7),L_(0x00000096), LL(0xb8a7ff33,0x37b93b28),LL(0x02791127,0xa408dc4f),LL(0x219fcf52,0x7589c78d),LL(0xbf0f03e3,0x0bb10f8f),LL(0x8d6cdb0c,0xd517b4d7),LL(0x99047428,0x37e06b1f),LL(0xa69f3aed,0x98f8786b),LL(0x8624d396,0xb0e28b50),L_(0x000001e2), + LL(0xa4ea5b5a,0x734ae209),LL(0x90fc2a73,0x5823b56b),LL(0x7673ea6e,0x7dbb26e5),LL(0xd6657bcd,0xd1742aca),LL(0xc34cd032,0xe888df76),LL(0x8065b09e,0xa8dad269),LL(0xc00f61b1,0x3a07c5aa),LL(0xd150d657,0xd3b84038),L_(0x000001ea), LL(0xd0a14535,0x608fa78a),LL(0xfb7e5603,0x1e082856),LL(0x643ad480,0x7bf543a7),LL(0xf5b0db8f,0xfadd24cd),LL(0x7206f2c4,0x1806c9ff),LL(0x3a60a387,0x0a68bae7),LL(0x1164c0d4,0x51de4b72),LL(0xb512a4da,0x80339476),L_(0x00000027), + LL(0x77314e66,0xc35e0de9),LL(0x4220d9f5,0x86281b42),LL(0x69b8f421,0xe5f95c9d),LL(0x5fd90a74,0x9a89c707),LL(0x8c09fc49,0xe12f3480),LL(0xa6764b64,0x8c161166),LL(0x886f3c36,0x55f40cf1),LL(0x68ad8aaa,0x5a680c49),L_(0x000001fd), LL(0xf91f4bec,0x489002d4),LL(0xe8177d77,0x1759ca38),LL(0xe14e5a1f,0xab2e759a),LL(0x9005868d,0xa02b4128),LL(0xaa1dff8c,0x2b9cd06e),LL(0x12d6a4d8,0x578741ea),LL(0x641aef64,0x1a343e8d),LL(0xc6a85c8b,0xf21792d4),L_(0x0000006c), + LL(0x4fe44c0c,0x7a97ea09),LL(0xec2c9500,0x455f3253),LL(0xa4bedbb4,0x902e1815),LL(0x27a1df89,0xfb3e392c),LL(0x120f8330,0x583ac267),LL(0x9a9698e2,0xe8c87240),LL(0x675c3030,0x32adecb4),LL(0x4f7fb620,0x5c2ea4c6),L_(0x000000da), LL(0x56a1c202,0xb86d1cd8),LL(0xdab68ef9,0xed63845f),LL(0x1723eef9,0x480f0cda),LL(0x7a1853f8,0xb146da6a),LL(0x7337ff75,0x4dd1db53),LL(0xd3685d26,0x41863100),LL(0x9ea6ba31,0xef5caeec),LL(0xa06b6815,0x5f4a92e2),L_(0x00000080), + LL(0xfa7c3363,0xf791b828),LL(0x5836e010,0x69b98b78),LL(0xba2d3b6c,0x504f9367),LL(0xcea4290d,0x5860835e),LL(0xf3dd0621,0x22ac6245),LL(0x208bde66,0xd8153e71),LL(0xd2a1a552,0xcd91cd77),LL(0xc7df7c52,0x167e37eb),L_(0x00000134), LL(0x0bfe4e12,0xfdf620f3),LL(0xe9aa6d05,0x78c9c26d),LL(0x7d0e875e,0x581fef3a),LL(0x45acf57d,0x4f2f3f1f),LL(0x89769d1b,0xb516d4fb),LL(0x7161f8a2,0xa50e2afa),LL(0xa831731c,0xa569ea98),LL(0x69e2a679,0x5909da40),L_(0x000000d1), + LL(0xafc78e61,0x4dc242cf),LL(0xb32ad972,0xbb40309f),LL(0x790edca8,0x89505a9a),LL(0x3c060f5d,0x112ba1c3),LL(0x52485cd5,0xc5bdc888),LL(0x9feed1cd,0x68110e28),LL(0x49202782,0x49d8d8e4),LL(0x7cea44e5,0xc5447530),L_(0x0000006c), LL(0x33e94828,0x54f9b84f),LL(0xdd82a038,0xf69e1a64),LL(0x9e8e50f3,0xf1691b4d),LL(0x13fe2932,0x5ffe5329),LL(0x032ff352,0x51f4070a),LL(0x9dcce305,0xc3145c5b),LL(0x62f1400d,0xd89cb5eb),LL(0xbef8bde9,0x4f571517),L_(0x000001e0), + LL(0x37573b5a,0x464e098a),LL(0x2eac199e,0x4b6ae9bd),LL(0x41109e44,0x96c7e839),LL(0x638c7109,0x99e6beb3),LL(0xa5b03740,0x0943a1c3),LL(0x8490e0cd,0xfab6ecb1),LL(0x4e71bae8,0xb69f0eb1),LL(0x29f06246,0x366f7112),L_(0x0000016d), LL(0x536e2d86,0xa2a3d8ca),LL(0xf4f50e4b,0x4c428120),LL(0xcb6eaa4c,0xb4203e5b),LL(0x69871129,0x6da13d6c),LL(0x218bdacf,0x4f621f85),LL(0x52046a31,0x1ea900e8),LL(0x3a13fa03,0xc7d28019),LL(0x167b70a8,0x1f519437),L_(0x000000b7), + LL(0x2bdb447a,0xae8792f8),LL(0x63aef018,0xae6cf0fd),LL(0x3291cfc7,0xe0ee5c02),LL(0x3ae122af,0x5bd690e7),LL(0xff276537,0xbc3516ac),LL(0xf83b9879,0xa4255fcc),LL(0x05236d5e,0x4ca14e35),LL(0xaf60c6b5,0xea86970d),L_(0x0000000a), LL(0xc17c08e7,0x737de42a),LL(0xac0df2a7,0x520e48ce),LL(0xaceb43b8,0x2f791d6a),LL(0x57fe87f0,0x662b9dfe),LL(0xc51cfa7c,0x884ed1f4),LL(0x75a9efdd,0xb5ee76b0),LL(0x3f9fe081,0x61e43ed8),LL(0xb9598115,0xd8cbf7b1),L_(0x000000fa), + LL(0x82e47d41,0x1c1e4e8c),LL(0xf692383c,0x819d110b),LL(0x0caaa47b,0xcb280e34),LL(0xf6e315e8,0x49f5a7e4),LL(0x68659604,0x6db3e3bf),LL(0xc18d2a73,0xb38233a8),LL(0xab54c2d8,0x216ab95c),LL(0x670af6e7,0x053a27b4),L_(0x00000154), LL(0xf79cd8d1,0x95f873cc),LL(0x8946ec9e,0xd4ae259b),LL(0x352c8cec,0x6383026a),LL(0x4b6773b4,0x574f14c8),LL(0x7327edc3,0x43f9e116),LL(0xfc2d9802,0x58a2e8d3),LL(0x26360b9c,0x2ae789b4),LL(0xeac487c7,0xe5e4384b),L_(0x000000bb), + LL(0xcfa66e36,0xe576c7d6),LL(0x40446421,0x5a9fe083),LL(0xa7e0a9d1,0x43da69bc),LL(0xcd5cfda4,0xaca35d4c),LL(0xdb98b2d8,0xcc88e119),LL(0x238da31d,0xe775938e),LL(0xd74d2fe0,0x0e845777),LL(0xee458b07,0x2ecae686),L_(0x00000070), LL(0xfc21befe,0x6f8ded71),LL(0xf57aacc7,0xa1f5602a),LL(0x8a4a8706,0xdf77dad6),LL(0xe88d5556,0xf0fb8eab),LL(0x38891e24,0x3a9313c2),LL(0xd33f4e50,0x4e334d02),LL(0x267f4849,0x45dbbeac),LL(0x7b8b078d,0xd3a28c1d),L_(0x000001d0), + LL(0x77a504d7,0x706161d0),LL(0x0d717bec,0x8d3f449e),LL(0xcd6aa437,0x4c327553),LL(0xbfa09758,0xecfed023),LL(0x131032e9,0x4abfe666),LL(0x2301de73,0x972524c6),LL(0xd67cd7b5,0x1b68a20a),LL(0xd8e4bd98,0x51e331d7),L_(0x00000142), LL(0xa0dd411d,0x136bd0f9),LL(0x8521a20a,0xe9c06f6a),LL(0x34117a07,0xb1417701),LL(0x625cc2c0,0x534fecc6),LL(0xe6f01c93,0x698e9742),LL(0xcea5bcd3,0x43a9724a),LL(0x54b554d5,0x77820ced),LL(0x7954cbcc,0x90ac105a),L_(0x000000eb), + LL(0xef4a4c48,0x67e62639),LL(0xef609727,0x91ba0b01),LL(0xc40bb739,0x7f62ccbc),LL(0xb7eda85e,0xb14485bc),LL(0x2a55f22b,0xeab4bc94),LL(0x091f3fde,0xb13e2d7b),LL(0x72b44ddf,0xa6958062),LL(0xd4d990ab,0xeb252caf),L_(0x0000000e), LL(0x14ea0dda,0xc0e83ba2),LL(0x10cf6ee9,0xc9f677ef),LL(0x8e3eae8b,0x1862146e),LL(0xcfe0b037,0xa6cdb8a8),LL(0x6d8e5bd3,0xf50d8419),LL(0xdd3f0ea7,0xf42ab2a9),LL(0x8a6b5e3c,0x12f65451),LL(0xe62ff71b,0x2007b697),L_(0x000000af), + LL(0x0460a7ae,0x6526a7b4),LL(0x80ab037a,0xf752335e),LL(0xf14c43bd,0xd11a8f65),LL(0x9b989eab,0x23fa924f),LL(0x0976a80f,0xf16dad87),LL(0x4e440171,0x3baaac45),LL(0xb9635c0d,0xf0f34704),LL(0x0e1ca863,0x515f651b),L_(0x000001a3), LL(0x40fcc076,0x420a4e5f),LL(0x24cf57e8,0xcec847e3),LL(0x1f189869,0xb20db2fe),LL(0x27f88dee,0x48712498),LL(0x423c17a4,0x284e6344),LL(0xfe029568,0x10b9bfa3),LL(0xd426c180,0x07626ee2),LL(0x96cbf2d8,0x92af56c4),L_(0x0000008f), + LL(0x8a898f02,0x37d5f07a),LL(0x9e4691c0,0xdc3f3126),LL(0xb1ce0f65,0x81c10b29),LL(0x73e7fd11,0xc595f2a8),LL(0x5a3fd848,0x47d0e340),LL(0x439d759b,0xbc668622),LL(0x87538694,0x461b9eba),LL(0x3484697e,0xa4f47f3b),L_(0x000001fb), LL(0x3d20296f,0xf99ebce4),LL(0x59fd2232,0x51ac1eec),LL(0x2fb986d6,0xe71f9d4a),LL(0xa5d1433c,0xa289f5b9),LL(0x4ece5225,0xd51b9288),LL(0x6ec5e037,0xcce86717),LL(0xb823e469,0xac199283),LL(0x45b78f23,0xb4d838aa),L_(0x0000017a), + LL(0xbfa48e10,0x08472093),LL(0xc8ac9e7a,0xb7492e4e),LL(0x471d73ed,0x53d1bd1c),LL(0x243428ce,0x40adeca7),LL(0xb7d49d17,0xd60077c0),LL(0x7a95c66d,0x737593cb),LL(0xf237e1e5,0x6a7c6f6e),LL(0xa7929ff0,0x984af660),L_(0x0000002d), LL(0x894bcc2e,0x7b3c416f),LL(0xa493d3e8,0x21231992),LL(0x4c0a5993,0xf376d082),LL(0x0d5c6c61,0x9e1550e6),LL(0x2a3430f2,0xf1d1beb9),LL(0x11f95e81,0x7b6bc5ab),LL(0xa78bae02,0x17858c60),LL(0xbdd9ea90,0x7b4b3f95),L_(0x000000d8), +}, +/* digit=104 base_pwr=2^520 */ +{ + LL(0x0b8b69f6,0xb96747cd),LL(0x3f30c14c,0x9e707ed2),LL(0x029690c8,0xd1292336),LL(0x884265ce,0x55cfec2b),LL(0xc87e6275,0xa097ab7d),LL(0xa5558f62,0xf635118e),LL(0x23dda1ad,0x5770b69e),LL(0x1b70ffff,0x4207811d),L_(0x00000181), LL(0x4f2606bc,0x8c9c6bbd),LL(0xccb92cf3,0x58d1b308),LL(0x2ce6913c,0xbbdfb6fe),LL(0x1967bfef,0xe5cb515b),LL(0x453132fc,0x2527584e),LL(0xc389b2f2,0x2a1591df),LL(0x0460d618,0x602d5761),LL(0x7498b0d1,0x0a2aedc5),L_(0x000001d3), + LL(0xb38cffee,0xa135a61f),LL(0x8ac17028,0x2c9fe1a1),LL(0x30418249,0x3ecb968e),LL(0x4c958b81,0x6f834a0b),LL(0x523285e8,0x54df836f),LL(0x29e4b05e,0xf77b5fb2),LL(0xe864d898,0x48371505),LL(0xcce3c32e,0x702eafb3),L_(0x000000b2), LL(0xc4a0e5e3,0xfb74fd48),LL(0x18996dc2,0x875aaa20),LL(0x5e9f86f5,0x5eb82718),LL(0x6642422b,0x87431019),LL(0xd6d6f0dd,0x9d7e7982),LL(0xcb46f571,0x346044e6),LL(0xa348c675,0xdc02d250),LL(0x517508af,0x179b19ec),L_(0x0000019d), + LL(0x18145140,0xded9f8c1),LL(0xba498095,0x40df2b19),LL(0x3834dace,0xa0c7dec0),LL(0x3687e664,0xfeee9a9e),LL(0x09d971f2,0xa032ee72),LL(0x07148796,0xa0b2cc31),LL(0xb65bcec3,0x46067d48),LL(0xd8c2af65,0x50cbf7f5),L_(0x0000014f), LL(0xd715c856,0x79c821c5),LL(0xbebe04f7,0x62cf6310),LL(0x3844c220,0x89986c83),LL(0x2644f899,0x43935732),LL(0xf80ac439,0xfd741ec2),LL(0x299a9f2d,0x3022b589),LL(0xbcf4790c,0xf5663af1),LL(0x767587f6,0x290c9a46),L_(0x00000161), + LL(0xf1cdfa06,0xda7b7826),LL(0x97fea55e,0xdd938459),LL(0xc39699cc,0x17d73f54),LL(0x5971fbd4,0x9f08e1d8),LL(0x3631f804,0x65a870ee),LL(0x51a73b8e,0xa3cdc8e3),LL(0x54f69b2a,0x85999014),LL(0x59cbe0fe,0xf12532d6),L_(0x000000cf), LL(0x085bfaeb,0x17f6a7aa),LL(0xe07f9788,0x1e5f6c61),LL(0xb271762a,0xcc6a5dff),LL(0x5d3e3cd3,0x895eb0df),LL(0x6eadd52e,0x2cd665b4),LL(0x92523b34,0x93078c5b),LL(0x2ebcff33,0x48620212),LL(0x777dd50c,0xef0f25f4),L_(0x00000138), + LL(0xfe3fcdbd,0x118dee98),LL(0xcfc4ad1b,0x0a524cec),LL(0x6d1ffa5c,0xe0d7420c),LL(0x919d859d,0x44af5553),LL(0xf81fb745,0x0981d6de),LL(0xa17a7c3f,0x6680c297),LL(0xe0cc4fe7,0xf6d22135),LL(0xde36e57a,0x7f521d15),L_(0x000000af), LL(0x0b8ae03f,0xac0c4195),LL(0xc2638c06,0xc994e2c7),LL(0x4523ebb6,0x49b75a94),LL(0xdb72a765,0x73310e2b),LL(0x57a73d05,0x8ce6d6b8),LL(0x1e23ae4c,0x4309eb9e),LL(0x842f06ae,0xf98aedc0),LL(0xedf71264,0x161436ed),L_(0x000001af), + LL(0xbe47c62f,0xd4c38f8c),LL(0x49173568,0x085a52c0),LL(0x7d057b19,0x612151d8),LL(0x2de5c929,0x8a4ea946),LL(0xc4036de1,0x0a8300dd),LL(0x5a6e290b,0x195518bc),LL(0x894ed83e,0x35e21097),LL(0xd8dd898c,0x03777a74),L_(0x000000ff), LL(0xd0eee2e2,0xc0b89ddf),LL(0x8a7bf325,0x0c5c995a),LL(0xa5131a24,0x9573e514),LL(0xe5998b6e,0x93979878),LL(0x7ba59336,0x7906f3c8),LL(0xff513215,0x093cead5),LL(0xb76588a1,0x243bc2e1),LL(0x5abc03c8,0x8eceda05),L_(0x00000134), + LL(0x07c9d208,0x234d2231),LL(0x524b8b4b,0xc6fe7d17),LL(0x469a9182,0x688b3e26),LL(0x35461f18,0xb0f49d1d),LL(0x78f9ef1e,0x4bb2130a),LL(0xe72b7216,0x9240661e),LL(0xec1e4e46,0x294674d4),LL(0x95346b15,0xe96fbdf1),L_(0x00000020), LL(0xa594d4cd,0xf533e3fe),LL(0xdb75a21c,0x99051c26),LL(0xea1a808b,0x9c10237a),LL(0x230cfcd1,0xe6396566),LL(0x90aedfcf,0x18cdd934),LL(0x8c399655,0x446502d8),LL(0x0de6be10,0x4a6de065),LL(0x7b3640aa,0xd5c12c48),L_(0x00000057), + LL(0xf6315f2c,0xf2e07c55),LL(0x62ee6844,0x9040fc8f),LL(0xaf168881,0xaec6bb02),LL(0xa1922677,0x968e4dee),LL(0x53b90132,0x92b21f9e),LL(0x2b9cb75f,0x4ae4c1d6),LL(0x65408d1b,0xd20f3732),LL(0x70be8f94,0x7640413d),L_(0x0000009e), LL(0x05c00995,0x4e16ae2b),LL(0xf20b5acf,0xd04ee882),LL(0x1f8c5834,0x35db5e06),LL(0xc9dbfe41,0x3a42f540),LL(0x3f7b55b6,0x500e266e),LL(0x52ce3b47,0xda6911e6),LL(0x0c53d0a4,0xee43c4db),LL(0x6052a6d7,0xa358b994),L_(0x00000165), + LL(0xf7f0245d,0xc1b6e80d),LL(0xd2bf51a5,0x9077fd5b),LL(0x4a61b710,0x97dc422c),LL(0x3089abb3,0x2c00b155),LL(0x893fa316,0x40add702),LL(0xb9d804ac,0xc77985f1),LL(0xdfaaf13d,0x4150d1b3),LL(0xaf9d85b4,0x9a8fb36c),L_(0x000001fb), LL(0x92589d47,0xd0dfdf3a),LL(0x06f77875,0x6b4012d5),LL(0xfe067b9f,0x4426c933),LL(0x2ed7298d,0x5353c502),LL(0x67948217,0x156d92c1),LL(0xf0f675fa,0xee2b368d),LL(0xba020dd5,0x3367f41b),LL(0xd5aae0ea,0x95a07182),L_(0x00000049), + LL(0x75917fbb,0x1b52b513),LL(0x87569974,0xbeb1242f),LL(0x4a568f34,0xb03e5543),LL(0x86a66b14,0xc094f1b5),LL(0x93edd5ae,0x7da45bde),LL(0x2fb2c47a,0x960491e1),LL(0x678ce19f,0x890efc78),LL(0xf4853187,0x29999e31),L_(0x000001e7), LL(0x4363742f,0xd3a570ac),LL(0x1812091b,0x4961aaea),LL(0xd32e88d4,0x97dd4a2f),LL(0x1b33e755,0xb3987552),LL(0x79ed95a8,0x669664b0),LL(0x0837ce5b,0x3584f341),LL(0x3f9a0cc0,0x116266b7),LL(0x64a902cc,0x330114e0),L_(0x000000ed), + LL(0x0660e9a1,0xb0698be3),LL(0x5c045493,0xc1ca94da),LL(0x82b986b9,0x4e8d0287),LL(0xfafe7c0e,0x02cfcf45),LL(0x9235965c,0xcf187fb8),LL(0x316bebe3,0x3fb90363),LL(0xc42b951d,0xc6248aec),LL(0x26710295,0x8f1eb678),L_(0x000000d1), LL(0xc24f0956,0xa11f38e2),LL(0x398a2047,0x9a0accc0),LL(0x9e479268,0x8ccafb17),LL(0x468f61d6,0x0c01f836),LL(0x52f96c75,0xeb043b98),LL(0xacc6ee07,0xa59405be),LL(0xc63baaf6,0xeddd33c9),LL(0xba136442,0x9b13f409),L_(0x000000ba), + LL(0xc33b7b64,0x1eb1f532),LL(0x3bd3bca3,0x3a7aee90),LL(0x0882ad60,0xad95f2eb),LL(0xddaf31c3,0x4c9d8e29),LL(0x59c130df,0xbdd1470a),LL(0x7ac309bd,0xe3bf394f),LL(0xaafb8369,0xedba4812),LL(0x7f598209,0x46270ae8),L_(0x000000dc), LL(0xb5d505d9,0x81e772f9),LL(0xfa065dfd,0x15db5d5b),LL(0x7e590809,0x141e8679),LL(0xc4ffd236,0xf14602ff),LL(0xbf521149,0xcfd2b215),LL(0xc500a7a4,0xc04e3706),LL(0xe4c06db1,0x3e28ec80),LL(0x93d40cbc,0xd4519009),L_(0x00000117), + LL(0x530c705b,0x1a470405),LL(0xe8292e38,0x85b81d18),LL(0xc5fdb2b2,0x484b843b),LL(0xa859e4fe,0xeeae06f4),LL(0x4e3895d5,0x0a67c915),LL(0xe6119b5b,0xed968aa0),LL(0x9264e00f,0x66ed4c20),LL(0xbf1d85ad,0x9f2e5291),L_(0x0000016d), LL(0xd8a9d836,0xee5d4b51),LL(0xcf0b68ca,0x89ad96c1),LL(0xbcd75bca,0xae1fa341),LL(0x667fbe53,0xa297bd27),LL(0xe40caf5f,0xd6165d4f),LL(0x329b45f8,0x62d6bced),LL(0xc11413bc,0xd1c11022),LL(0x7e0d7384,0x30c070b1),L_(0x00000118), + LL(0xa39b851e,0xa0188e61),LL(0xf5e55bb8,0xe96325dc),LL(0x8cd5c092,0x9087223b),LL(0xe2c1a18b,0xca56bfcd),LL(0x548d8395,0x4ae67f8d),LL(0xaa6db861,0x8f44b4e9),LL(0x218ba5b9,0x317abb0e),LL(0x04386052,0x4aeffcd2),L_(0x00000084), LL(0x91b2e14e,0x6ac7c848),LL(0x5377e2b3,0xd40844b6),LL(0x9d93badb,0xe505b8cb),LL(0x4d7f3493,0x615a64c9),LL(0x20a05d5d,0x90a5eb78),LL(0xa4cb086a,0xed4783ac),LL(0x36415b6c,0xf10f3d20),LL(0x56659094,0x82883579),L_(0x0000004f), + LL(0x0eeb045d,0xac240867),LL(0x98145c00,0xe7ce6952),LL(0xb5d0c780,0x315dd662),LL(0x189fc413,0x41646f48),LL(0x4392d048,0xc963ad1a),LL(0x1e77199f,0xebc649ee),LL(0x83e1f918,0xcd6ca624),LL(0x13b6a99b,0x4108f969),L_(0x0000011b), LL(0x2108af54,0x0b55c26f),LL(0x3989bd71,0xe0f27726),LL(0x1e5e0053,0x5c7e0958),LL(0xa8452157,0xf8e7b504),LL(0xb64d38bf,0xf180ac5c),LL(0x8c8c65f5,0x32a84a9b),LL(0x8f00c232,0x898ca7ed),LL(0x1a1639de,0xe79d8696),L_(0x00000150), + LL(0x6746f213,0xe1073527),LL(0xa966b0a8,0x3ad14203),LL(0x5bc4272f,0x39620db2),LL(0xbcd33a93,0xe5eae695),LL(0x26bac2fb,0x0e4497dc),LL(0xb7d647bd,0x3a195407),LL(0x7f7ed906,0x899ce3f6),LL(0xadd76129,0x2e49d8f3),L_(0x000000be), LL(0x333eb7cb,0x51de1b18),LL(0x8e185580,0xa269b8e8),LL(0x486cd055,0x3555823a),LL(0x6689b4be,0xaa52baba),LL(0xcd8d6ffd,0xd072a45c),LL(0x9cba9f57,0xba53f86f),LL(0x74a8d5fb,0x16481f57),LL(0x10747d58,0x457baa10),L_(0x000001af), +} +}; +#endif + +#endif /* _DISABLE_ECP_521R1_HARDCODED_BP_TBL_ */ +#endif /* _IPP_DATA */ + + +IPP_OWN_DEFN (const cpPrecompAP*, gfpec_precom_nistP521r1_fun, (void)) +{ + static cpPrecompAP t = { + /* w */ 5, + /* select function */ p521r1_select_ap_w5, + /* precomputed data */ (BNU_CHUNK_T*)ec_p521r1_precomputed + }; + return &t; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprime_isco.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprime_isco.h new file mode 100644 index 0000000..1f2c53c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprime_isco.h @@ -0,0 +1,63 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA Functions +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpprimeg.h" +#include "pcpprng.h" +#include "pcpngrsa.h" + +/* test if E and A are coprime */ +static int cpIsCoPrime(BNU_CHUNK_T* pA, int nsA, + BNU_CHUNK_T* pB, int nsB, + BNU_CHUNK_T* pBuffer) +{ + if (nsA>nsB) { + SWAP_PTR(BNU_CHUNK_T, pA, pB); + SWAP(nsA, nsB); + } + { + __ALIGN8 IppsBigNumState bnA, bnB, bnGcd; + BNU_CHUNK_T* pDataA = pBuffer; + BNU_CHUNK_T* pBuffA = pDataA + nsA + 1; + BNU_CHUNK_T* pDataB = pBuffA + nsA + 1; + BNU_CHUNK_T* pBuffB = pDataB + nsB + 1; + BNU_CHUNK_T* pDataGcd = pBuffB + nsB + 1; + BNU_CHUNK_T* pBuffGcd = pDataGcd + nsB + 1; + + BN_Make(pDataA, pBuffA, nsA, &bnA); + BN_Make(pDataB, pBuffB, nsB, &bnB); + BN_Make(pDataGcd, pBuffGcd, nsB, &bnGcd); + + COPY_BNU(pDataA, pA, nsA) + BN_Set(pDataA, nsA, &bnA); + COPY_BNU(pDataB, pB, nsB) + BN_Set(pDataB, nsB, &bnB); + ippsGcd_BN(&bnA, &bnB, &bnGcd); + return 0 == cpBN_cmp(&bnGcd, cpBN_OneRef()); + } +} \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprime_isprob.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprime_isprob.h new file mode 100644 index 0000000..eab2c31 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprime_isprob.h @@ -0,0 +1,137 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA Functions +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpprimeg.h" +#include "pcpprng.h" +#include "pcpngrsa.h" + + +static int cpMillerRabinTest(BNU_CHUNK_T* pW, cpSize nsW, + const BNU_CHUNK_T* pE, cpSize bitsizeE, + int k, + const BNU_CHUNK_T* pPrime1, + gsModEngine* pMont, + BNU_CHUNK_T* pBuffer) +{ + cpSize nsP = MOD_LEN(pMont); + + /* to Montgomery Domain */ + ZEXPAND_BNU(pW, nsW, nsP); + MOD_METHOD(pMont)->encode(pW, pW, pMont); + + /* w = exp(w,e) */ + #if !defined(_USE_WINDOW_EXP_) + gsMontExpBin_BNU_sscm(pW, pW, nsP, pE, bitsizeE, pMont, pBuffer); + #else + gsMontExpWin_BNU_sscm(pW, pW, nsP, pE, bitsizeE, pMont, pBuffer); + #endif + + /* if (w==1) ||(w==prime-1) => probably prime */ + if ((0 == cpCmp_BNU(pW, nsP, MOD_MNT_R(pMont), nsP)) + || (0 == cpCmp_BNU(pW, nsP, pPrime1, nsP))) + return 1; /* witness of the primality */ + + while (--k) { + MOD_METHOD(pMont)->sqr(pW, pW, pMont); + + if (0 == cpCmp_BNU(pW, nsP, MOD_MNT_R(pMont), nsP)) + return 0; /* witness of the compositeness */ + if (0 == cpCmp_BNU(pW, nsP, pPrime1, nsP)) + return 1; /* witness of the primality */ + } + return 0; +} + +/* test if P is prime + +returns: +IPP_IS_PRIME (==1) - prime value has been detected +IPP_IS_COMPOSITE (==0) - composite value has been detected +-1 - if internal error (ippStsNoErr != rndFunc()) +*/ +static int cpIsProbablyPrime(BNU_CHUNK_T* pPrime, int bitSize, + int nTrials, + IppBitSupplier rndFunc, void* pRndParam, + gsModEngine* pME, + BNU_CHUNK_T* pBuffer) +{ + /* if test for trivial divisors passed*/ + int ret = cpMimimalPrimeTest((Ipp32u*)pPrime, BITS2WORD32_SIZE(bitSize)); + + /* appy Miller-Rabin test */ + if (ret) { + int ns = BITS_BNU_CHUNK(bitSize); + BNU_CHUNK_T* pPrime1 = pBuffer; + BNU_CHUNK_T* pOdd = pPrime1 + ns; + BNU_CHUNK_T* pWitness = pOdd + ns; + BNU_CHUNK_T* pMontPrime1 = pWitness + ns; + BNU_CHUNK_T* pScratchBuffer = pMontPrime1 + ns; + int k, a, lenOdd; + + /* prime1 = prime-1 = odd*2^a */ + cpDec_BNU(pPrime1, pPrime, ns, 1); + for (k = 0, a = 0; k0; n--) { + Ipp64u tmp = IPP_MAKEDWORD(pX[n-1],r); + r = (Ipp32u)(tmp%divider); + } + return r; +} + +/* minimal prime test */ +static Ipp32u PrimeList[] = { + 2, 3, 5, 7, 11, 13, 17, 19, + 23, 29, 31, 37, 41, 43, 47, 53, + 59, 61, 67, 71, 73, 79, 83, 89, + 97, 101, 103, 107, 109, 113, 127, 131, + 137, 139, 149, 151, 157, 163, 167, 173, + 179, 181, 191, 193, 197, 199, 211, 223, + 227, 229, 233, 239, 241, 251, 257, 263, + 269, 271, 277, 281, 283, 293, 307, 311, + 313, 317, 331, 337, 347, 349, 353, 359, + 367, 373, 379, 383, 389, 397, 401, 409, + 419, 421, 431, 433, 439, 443, 449, 457, + 461, 463, 467, 479, 487, 491, 499, 503, + 509, 521, 523, 541, 547, 557, 563, 569, + 571, 577, 587, 593, 599, 601, 607, 613, + 617, 619, 631, 641, 643, 647, 653, 659, + 661, 673, 677, 683, 691, 701, 709, 719, + 727, 733, 739, 743, 751, 757, 761, 769, + 773, 787, 797, 809, 811, 821, 823, 827, + 829, 839, 853, 857, 859, 863, 877, 881, + 883, 887, 907, 911, 919, 929, 937, 941, + 947, 953, 967, 971, 977, 983, 991, 997, + 1009,1013,1019,1021,1031,1033,1039,1049, + 1051,1061,1063,1069,1087,1091,1093,1097, + 1103,1109,1117,1123,1129,1151,1153,1163, + 1171,1181,1187,1193,1201,1213,1217,1223, + 1229,1231,1237,1249,1259,1277,1279,1283, + 1289,1291,1297,1301,1303,1307,1319,1321, + 1327,1361,1367,1373,1381,1399,1409,1423, + 1427,1429,1433,1439,1447,1451,1453,1459, + 1471,1481,1483,1487,1489,1493,1499,1511, + 1523,1531,1543,1549,1553,1559,1567,1571, + 1579,1583,1597,1601,1607,1609,1613,1619, + 1621,1627,1637,1657,1663,1667,1669,1693, + 1697,1699,1709,1721,1723,1733,1741,1747, + 1753,1759,1777,1783,1787,1789,1801,1811, + 1823,1831,1847,1861,1867,1871,1873,1877, + 1879,1889,1901,1907,1913,1931,1933,1949, + 1951,1973,1979,1987,1993,1997,1999,2003, + 2011,2017,2027,2029,2039,2053,2063,2069, + 2081,2083,2087,2089,2099,2111,2113,2129, + 2131,2137,2141,2143,2153,2161,2179,2203, + 2207,2213,2221,2237,2239,2243,2251,2267, + 2269,2273,2281,2287,2293,2297,2309,2311, + 2333,2339,2341,2347,2351,2357,2371,2377, + 2381,2383,2389,2393,2399,2411,2417,2423, + 2437,2441,2447,2459,2467,2473,2477,2503, + 2521,2531,2539,2543,2549,2551,2557,2579, + 2591,2593,2609,2617,2621,2633,2647,2657, + 2659,2663,2671,2677,2683,2687,2689,2693, + 2699,2707,2711,2713,2719,2729,2731,2741, + 2749,2753,2767,2777,2789,2791,2797,2801, + 2803,2819,2833,2837,2843,2851,2857,2861, + 2879,2887,2897,2903,2909,2917,2927,2939, + 2953,2957,2963,2969,2971,2999,3001,3011, + 3019,3023,3037,3041,3049,3061,3067,3079, + 3083,3089,3109,3119,3121,3137,3163,3167, + 3169,3181,3187,3191,3203,3209,3217,3221, + 3229,3251,3253,3257,3259,3271,3299,3301, + 3307,3313,3319,3323,3329,3331,3343,3347, + 3359,3361,3371,3373,3389,3391,3407,3413, + 3433,3449,3457,3461,3463,3467,3469,3491, + 3499,3511,3517,3527,3529,3533,3539,3541, + 3547,3557,3559,3571,3581,3583,3593,3607, + 3613,3617,3623,3631,3637,3643,3659,3671, + 3673,3677,3691,3697,3701,3709,3719,3727, + 3733,3739,3761,3767,3769,3779,3793,3797, + 3803,3821,3823,3833,3847,3851,3853,3863, + 3877,3881,3889,3907,3911,3917,3919,3923, + 3929,3931,3943,3947,3967,3989,4001,4003, + 4007,4013,4019,4021,4027,4049,4051,4057, + 4073,4079,4091,4093,4099,4111,4127,4129, + 4133,4139,4153,4157,4159,4177,4201,4211, + 4217,4219,4229,4231,4241,4243,4253,4259, + 4261,4271,4273,4283,4289,4297,4327,4337, + 4339,4349,4357,4363,4373,4391,4397,4409, + 4421,4423,4441,4447,4451,4457,4463,4481, + 4483,4493,4507,4513,4517,4519,4523,4547, + 4549,4561,4567,4583,4591,4597,4603,4621, + 4637,4639,4643,4649,4651,4657,4663,4673, + 4679,4691,4703,4721,4723,4729,4733,4751, + 4759,4783,4787,4789,4793,4799,4801,4813, + 4817,4831,4861,4871,4877,4889,4903,4909, + 4919,4931,4933,4937,4943,4951,4957,4967, + 4969,4973,4987,4993,4999,5003,5009,5011, + 5021,5023,5039,5051,5059,5077,5081,5087, + 5099,5101,5107,5113,5119,5147,5153,5167, + 5171,5179,5189,5197,5209,5227,5231,5233, + 5237,5261,5273,5279,5281,5297,5303,5309, + 5323,5333,5347,5351,5381,5387,5393,5399, + 5407,5413,5417,5419,5431,5437,5441,5443, + 5449,5471,5477,5479,5483,5501,5503,5507, + 5519,5521,5527,5531,5557,5563,5569,5573, + 5581,5591,5623,5639,5641,5647,5651,5653, + 5657,5659,5669,5683,5689,5693,5701,5711, + 5717,5737,5741,5743,5749,5779,5783,5791, + 5801,5807,5813,5821,5827,5839,5843,5849, + 5851,5857,5861,5867,5869,5879,5881,5897, + 5903,5923,5927,5939,5953,5981,5987,6007, + 6011,6029,6037,6043,6047,6053,6067,6073, + 6079,6089,6091,6101,6113,6121,6131,6133, + 6143,6151,6163,6173,6197,6199,6203,6211, + 6217,6221,6229,6247,6257,6263,6269,6271, + 6277,6287,6299,6301,6311,6317,6323,6329, + 6337,6343,6353,6359,6361,6367,6373,6379, + 6389,6397,6421,6427,6449,6451,6469,6473, + 6481,6491,6521,6529,6547,6551,6553,6563, + 6569,6571,6577,6581,6599,6607,6619,6637, + 6653,6659,6661,6673,6679,6689,6691,6701, + 6703,6709,6719,6733,6737,6761,6763,6779, + 6781,6791,6793,6803,6823,6827,6829,6833, + 6841,6857,6863,6869,6871,6883,6899,6907, + 6911,6917,6947,6949,6959,6961,6967,6971, + 6977,6983,6991,6997,7001,7013,7019,7027, + 7039,7043,7057,7069,7079,7103,7109,7121, + 7127,7129,7151,7159,7177,7187,7193,7207, + 7211,7213,7219,7229,7237,7243,7247,7253, + 7283,7297,7307,7309,7321,7331,7333,7349, + 7351,7369,7393,7411,7417,7433,7451,7457, + 7459,7477,7481,7487,7489,7499,7507,7517, + 7523,7529,7537,7541,7547,7549,7559,7561, + 7573,7577,7583,7589,7591,7603,7607,7621, + 7639,7643,7649,7669,7673,7681,7687,7691, + 7699,7703,7717,7723,7727,7741,7753,7757, + 7759,7789,7793,7817,7823,7829,7841,7853, + 7867,7873,7877,7879,7883,7901,7907,7919, + 7927,7933,7937,7949,7951,7963,7993,8009, + 8011,8017,8039,8053,8059,8069,8081,8087, + 8089,8093,8101,8111,8117,8123,8147,8161, + 8167,8171,8179,8191,8209,8219,8221,8231, + 8233,8237,8243,8263,8269,8273,8287,8291, + 8293,8297,8311,8317,8329,8353,8363,8369, + 8377,8387,8389,8419,8423,8429,8431,8443, + 8447,8461,8467,8501,8513,8521,8527,8537, + 8539,8543,8563,8573,8581,8597,8599,8609, + 8623,8627,8629,8641,8647,8663,8669,8677, + 8681,8689,8693,8699,8707,8713,8719,8731, + 8737,8741,8747,8753,8761,8779,8783,8803, + 8807,8819,8821,8831,8837,8839,8849,8861, + 8863,8867,8887,8893,8923,8929,8933,8941, + 8951,8963,8969,8971,8999,9001,9007,9011, + 9013,9029,9041,9043,9049,9059,9067,9091, + 9103,9109,9127,9133,9137,9151,9157,9161, + 9173,9181,9187,9199,9203,9209,9221,9227, + 9239,9241,9257,9277,9281,9283,9293,9311, + 9319,9323,9337,9341,9343,9349,9371,9377, + 9391,9397,9403,9413,9419,9421,9431,9433, + 9437,9439,9461,9463,9467,9473,9479,9491, + 9497,9511,9521,9533,9539,9547,9551,9587, + 9601,9613,9619,9623,9629,9631,9643,9649, + 9661,9677,9679,9689,9697,9719,9721,9733, + 9739,9743,9749,9767,9769,9781,9787,9791, + 9803,9811,9817,9829,9833,9839,9851,9857, + 9859,9871,9883,9887,9901,9907,9923,9929, + 9931,9941,9949,9967,9973,10007,10009,10037, + 10039,10061,10067,10069,10079,10091,10093,10099, + 10103,10111,10133,10139,10141,10151,10159,10163, + 10169,10177,10181,10193,10211,10223,10243,10247, + 10253,10259,10267,10271,10273,10289,10301,10303, + 10313,10321,10331,10333,10337,10343,10357,10369, + 10391,10399,10427,10429,10433,10453,10457,10459, + 10463,10477,10487,10499,10501,10513,10529,10531, + 10559,10567,10589,10597,10601,10607,10613,10627, + 10631,10639,10651,10657,10663,10667,10687,10691, + 10709,10711,10723,10729,10733,10739,10753,10771, + 10781,10789,10799,10831,10837,10847,10853,10859, + 10861,10867,10883,10889,10891,10903,10909,10937, + 10939,10949,10957,10973,10979,10987,10993,11003, + 11027,11047,11057,11059,11069,11071,11083,11087, + 11093,11113,11117,11119,11131,11149,11159,11161, + 11171,11173,11177,11197,11213,11239,11243,11251, + 11257,11261,11273,11279,11287,11299,11311,11317, + 11321,11329,11351,11353,11369,11383,11393,11399, + 11411,11423,11437,11443,11447,11467,11471,11483, + 11489,11491,11497,11503,11519,11527,11549,11551, + 11579,11587,11593,11597,11617,11621,11633,11657, + 11677,11681,11689,11699,11701,11717,11719,11731, + 11743,11777,11779,11783,11789,11801,11807,11813, + 11821,11827,11831,11833,11839,11863,11867,11887, + 11897,11903,11909,11923,11927,11933,11939,11941, + 11953,11959,11969,11971,11981,11987,12007,12011, + 12037,12041,12043,12049,12071,12073,12097,12101, + 12107,12109,12113,12119,12143,12149,12157,12161, + 12163,12197,12203,12211,12227,12239,12241,12251, + 12253,12263,12269,12277,12281,12289,12301,12323, + 12329,12343,12347,12373,12377,12379,12391,12401, + 12409,12413,12421,12433,12437,12451,12457,12473, + 12479,12487,12491,12497,12503,12511,12517,12527, + 12539,12541,12547,12553,12569,12577,12583,12589, + 12601,12611,12613,12619,12637,12641,12647,12653, + 12659,12671,12689,12697,12703,12713,12721,12739, + 12743,12757,12763,12781,12791,12799,12809,12821, + 12823,12829,12841,12853,12889,12893,12899,12907, + 12911,12917,12919,12923,12941,12953,12959,12967, + 12973,12979,12983,13001,13003,13007,13009,13033, + 13037,13043,13049,13063,13093,13099,13103,13109, + 13121,13127,13147,13151,13159,13163,13171,13177, + 13183,13187,13217,13219,13229,13241,13249,13259, + 13267,13291,13297,13309,13313,13327,13331,13337, + 13339,13367,13381,13397,13399,13411,13417,13421, + 13441,13451,13457,13463,13469,13477,13487,13499, + 13513,13523,13537,13553,13567,13577,13591,13597, + 13613,13619,13627,13633,13649,13669,13679,13681, + 13687,13691,13693,13697,13709,13711,13721,13723, + 13729,13751,13757,13759,13763,13781,13789,13799, + 13807,13829,13831,13841,13859,13873,13877,13879, + 13883,13901,13903,13907,13913,13921,13931,13933, + 13963,13967,13997,13999,14009,14011,14029,14033, + 14051,14057,14071,14081,14083,14087,14107,14143, + 14149,14153,14159,14173,14177,14197,14207,14221, + 14243,14249,14251,14281,14293,14303,14321,14323, + 14327,14341,14347,14369,14387,14389,14401,14407, + 14411,14419,14423,14431,14437,14447,14449,14461, + 14479,14489,14503,14519,14533,14537,14543,14549, + 14551,14557,14561,14563,14591,14593,14621,14627, + 14629,14633,14639,14653,14657,14669,14683,14699, + 14713,14717,14723,14731,14737,14741,14747,14753, + 14759,14767,14771,14779,14783,14797,14813,14821, + 14827,14831,14843,14851,14867,14869,14879,14887, + 14891,14897,14923,14929,14939,14947,14951,14957, + 14969,14983,15013,15017,15031,15053,15061,15073, + 15077,15083,15091,15101,15107,15121,15131,15137, + 15139,15149,15161,15173,15187,15193,15199,15217, + 15227,15233,15241,15259,15263,15269,15271,15277, + 15287,15289,15299,15307,15313,15319,15329,15331, + 15349,15359,15361,15373,15377,15383,15391,15401, + 15413,15427,15439,15443,15451,15461,15467,15473, + 15493,15497,15511,15527,15541,15551,15559,15569, + 15581,15583,15601,15607,15619,15629,15641,15643, + 15647,15649,15661,15667,15671,15679,15683,15727, + 15731,15733,15737,15739,15749,15761,15767,15773, + 15787,15791,15797,15803,15809,15817,15823,15859, + 15877,15881,15887,15889,15901,15907,15913,15919, + 15923,15937,15959,15971,15973,15991,16001,16007, + 16033,16057,16061,16063,16067,16069,16073,16087, + 16091,16097,16103,16111,16127,16139,16141,16183, + 16187,16189,16193,16217,16223,16229,16231,16249, + 16253,16267,16273,16301,16319,16333,16339,16349, + 16361,16363,16369,16381,16411,16417,16421,16427, + 16433,16447,16451,16453,16477,16481,16487,16493, + 16519,16529,16547,16553,16561,16567,16573,16603, + 16607,16619,16631,16633,16649,16651,16657,16661, + 16673,16691,16693,16699,16703,16729,16741,16747, + 16759,16763,16787,16811,16823,16829,16831,16843, + 16871,16879,16883,16889,16901,16903,16921,16927, + 16931,16937,16943,16963,16979,16981,16987,16993, + 17011,17021,17027,17029,17033,17041,17047,17053, + 17077,17093,17099,17107,17117,17123,17137,17159, + 17167,17183,17189,17191,17203,17207,17209,17231, + 17239,17257,17291,17293,17299,17317,17321,17327, + 17333,17341,17351,17359,17377,17383,17387,17389, + 17393,17401,17417,17419,17431,17443,17449,17467, + 17471,17477,17483,17489,17491,17497,17509,17519, + 17539,17551,17569,17573,17579,17581,17597,17599, + 17609,17623,17627,17657,17659,17669,17681,17683, + 17707,17713,17729,17737,17747,17749,17761,17783, + 17789,17791,17807,17827,17837,17839,17851,17863 +}; + +/*F* +// Name: cpMimimalPrimeTest +// +// Purpose: Test a number for being a mimnimal probable prime from list. +// +// Returns: Reason: +// 0 not prime number +// 1 prime number +// +// Parameters: +// pPrime prime number +// len32 length of prime number +*F*/ + +IPP_OWN_DEFN (int, cpMimimalPrimeTest, (const Ipp32u* pPrime, cpSize len32)) +{ + cpSize i; + + FIX_BNU32(pPrime, len32); + + /* take a look in the list */ + if(1==len32) { + for(i=0; i<(cpSize)(sizeof(PrimeList)/sizeof(Ipp32u)); i++) { + if(pPrime[0]==PrimeList[i]) + return 1; + } + } + + /* test if value under the test is divisible by first prime numbers 2,3,5, ... */ + for(i=0; i<(cpSize)(sizeof(PrimeList)/sizeof(Ipp32u)); i++) { + if(0 == cpMod32(pPrime, len32, PrimeList[i])) + return 0; + } + + return 1; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprime_packctx.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprime_packctx.c new file mode 100644 index 0000000..e4951a3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprime_packctx.c @@ -0,0 +1,56 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptographic Primitives (ippcp) +// Prime Number Primitives. +// +// Contents: +// cpPackPrimeCtx() +// +*/ + +#include "owncp.h" +#include "pcpprimeg.h" +#include "pcptool.h" + +/*F* +// Name: cpPackPrimeCtx +// +// Purpose: Serialize prime context +// +// Parameters: +// pCtx context +// pBuffer buffer +*F*/ + +IPP_OWN_DEFN (void, cpPackPrimeCtx, (const IppsPrimeState* pCtx, Ipp8u* pBuffer)) +{ + IppsPrimeState* pB = (IppsPrimeState*)(pBuffer); + + /* max length of prime */ + cpSize nsPrime = BITS_BNU_CHUNK(PRIME_MAXBITSIZE(pCtx)); + + CopyBlock(pCtx, pB, sizeof(IppsPrimeState)); + + cpSize dataAlignment = (cpSize)(IPP_INT_PTR(PRIME_NUMBER(pCtx)) - IPP_INT_PTR(pCtx) - (IPP_INT64)sizeof(IppsPrimeState)); + cpSize gsMontOffset = (cpSize)(IPP_INT_PTR(PRIME_MONT(pCtx)) - IPP_INT_PTR(pCtx) - dataAlignment); + + CopyBlock(PRIME_NUMBER(pCtx), (Ipp8u*)pB + sizeof(IppsPrimeState), nsPrime*(Ipp32s)sizeof(BNU_CHUNK_T)); + gsPackModEngineCtx(PRIME_MONT(pCtx), (Ipp8u*)pB + gsMontOffset); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprime_test.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprime_test.c new file mode 100644 index 0000000..311df8e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprime_test.c @@ -0,0 +1,187 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptographic Primitives (ippcp) +// Prime Number Primitives. +// +// Contents: +// cpPrimeTest() +// +*/ + +#include "owncp.h" +#include "pcpprimeg.h" +#include "pcpprng.h" +#include "pcptool.h" + +/* Rabin-Miller test */ +/* -1 is returned when pBuffer cannot be allocated */ +static int RabinMiller(int a, BNU_CHUNK_T* pZ, BNU_CHUNK_T* pR, cpSize nsR, BNU_CHUNK_T* pM, cpSize nsM, gsModEngine* pModEngine) +{ + /* modulus and it length and other parameters */ + const BNU_CHUNK_T* pModulus = MOD_MODULUS(pModEngine); + cpSize modLen = MOD_LEN(pModEngine); + + const int usedPoolLen = 1; + BNU_CHUNK_T* pBuffer = 0; + + /* compute z = r^m mod prime */ + nsR = cpMontEnc_BNU_EX(pR, pR, nsR, pModEngine); + cpMontExpBin_BNU(pZ, pR, nsR, pM, nsM, pModEngine); + + /* if z==1 => probably prime */ + if(0==cpCmp_BNU(pZ, modLen, MOD_MNT_R(pModEngine), modLen)) + return 1; + + pBuffer = gsModPoolAlloc(pModEngine, usedPoolLen); + if(NULL == pBuffer) + return -1; + + /* if z==prime-1 => probably prime */ + cpSub_BNU(pBuffer, pModulus, MOD_MNT_R(pModEngine), modLen); + if(0==cpCmp_BNU(pZ, modLen, pBuffer, modLen)) + { + gsModPoolFree(pModEngine, usedPoolLen); + return 1; + } + + while(--a) { + + /* z = z^2 mod w */ + cpMontSqr_BNU(pZ, pZ, pModEngine); + + /* if z==1 => definitely composite */ + if(0==cpCmp_BNU(pZ, modLen, MOD_MNT_R(pModEngine), modLen)) + { + gsModPoolFree(pModEngine, usedPoolLen); + return 0; + } + + /* if z==w-1 => probably prime */ + cpSub_BNU(pBuffer, pModulus, MOD_MNT_R(pModEngine), modLen); + if(0==cpCmp_BNU(pZ, modLen, pBuffer, modLen)) + { + gsModPoolFree(pModEngine, usedPoolLen); + return 1; + } + } + + gsModPoolFree(pModEngine, usedPoolLen); + + /* if we are here, then we deal with composize */ + return 0; +} + +/* + returns: + IPP_IS_PRIME (==1) - prime value has been detected + IPP_IS_COMPOSITE (==0) - composite value has been detected + -1 - if internal error (ippStsNoErr != rndFunc()) +*/ + +/*F* +// Name: cpPrimeTest +// +// Purpose: Test a number for being a probable prime. +// +// Returns: Reason: +// 0 not prime number +// 1 prime number +// +// Parameters: +// pPrime prime number +// primeLen length of prime number +// nTrials parameter for the Miller-Rabin probable primality test +// pCtx pointer to the context +// rndFunc external PRNG +// pRndParam pointer to the external PRNG parameters +*F*/ + +IPP_OWN_DEFN (int, cpPrimeTest, (const BNU_CHUNK_T* pPrime, cpSize primeLen, cpSize nTrials, IppsPrimeState* pCtx, IppBitSupplier rndFunc, void* pRndParam)) +{ + FIX_BNU(pPrime, primeLen); + + if( primeLen==1 && pPrime[0]==0) + return 0; + + /* 2 is prime number */ + else if( primeLen==1 && pPrime[0]==2) + return 1; + + /* + // test number + */ + else { + cpSize primeBitsize = BITSIZE_BNU(pPrime, primeLen); + cpSize primeLen32 = BITS2WORD32_SIZE(primeBitsize); + + /* apply easy prime test */ + if( 0==cpMimimalPrimeTest((Ipp32u*)pPrime, primeLen32) ) + return 0; + + /* continue test */ + else { + cpSize n, a; + + gsModEngine* pModEngine = PRIME_MONT(pCtx); + BNU_CHUNK_T* pMdata = PRIME_TEMP1(pCtx); + BNU_CHUNK_T* pRdata = PRIME_TEMP2(pCtx); + BNU_CHUNK_T* pZdata = PRIME_TEMP3(pCtx); + cpSize lenM, lenR; + + /* set up Montgomery engine (and save value being under the test) */ + gsModEngineInit(pModEngine, (Ipp32u*)pPrime, BITSIZE_BNU(pPrime, primeLen), MONT_DEFAULT_POOL_LENGTH, gsModArithMont()); + + /* express w = m*2^a + 1 */ + cpDec_BNU(pMdata, pPrime, primeLen, 1); + for(n=0,a=0; n=1 */ + if(!cpTst_BNU(pRdata, lenR)) + pRdata[0] |= 1; + FIX_BNU(pRdata, lenR); + + /* Rabin-Miller test */ + int result = RabinMiller(a, pZdata, pRdata,primeLen, pMdata,lenM, pModEngine); + if(-1 == result) //internal error + return -1; + if(0 == result) + return 0; + } + + return 1; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprime_unpackctx.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprime_unpackctx.c new file mode 100644 index 0000000..b2d83d6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprime_unpackctx.c @@ -0,0 +1,68 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptographic Primitives (ippcp) +// Prime Number Primitives. +// +// Contents: +// cpUnpackPrimeCtx() +// +*/ + +#include "owncp.h" +#include "pcpprimeg.h" +#include "pcptool.h" + +/*F* +// Name: cpUnpackPrimeCtx +// +// Purpose: Deserialize prime context +// +// Parameters: +// pCtx context +// pBuffer buffer +*F*/ + +IPP_OWN_DEFN (void, cpUnpackPrimeCtx, (const Ipp8u* pBuffer, IppsPrimeState* pCtx)) +{ + IppsPrimeState* pB = (IppsPrimeState*)(pBuffer); + + /* max length of prime */ + cpSize nsPrime = BITS_BNU_CHUNK(PRIME_MAXBITSIZE(pB)); + + CopyBlock(pB, pCtx, sizeof(IppsPrimeState)); + + Ipp8u* ptr = (Ipp8u*)pCtx; + ptr += sizeof(IppsPrimeState); + ptr = IPP_ALIGNED_PTR(ptr, PRIME_ALIGNMENT); + PRIME_NUMBER(pCtx)= (BNU_CHUNK_T*)(ptr); + ptr += nsPrime*(Ipp32s)sizeof(BNU_CHUNK_T); + PRIME_TEMP1(pCtx) = (BNU_CHUNK_T*)(ptr); + ptr += nsPrime*(Ipp32s)sizeof(BNU_CHUNK_T); + PRIME_TEMP2(pCtx) = (BNU_CHUNK_T*)(ptr); + ptr += nsPrime*(Ipp32s)sizeof(BNU_CHUNK_T); + PRIME_TEMP3(pCtx) = (BNU_CHUNK_T*)(ptr); + ptr += nsPrime*(Ipp32s)sizeof(BNU_CHUNK_T); + PRIME_MONT(pCtx) = (gsModEngine*)(ptr); + + cpSize gsMontOffset = (cpSize)(IPP_INT_PTR(PRIME_MONT(pCtx)) - IPP_INT_PTR(pCtx)); + + CopyBlock((Ipp8u*)pB + sizeof(IppsPrimeState), PRIME_NUMBER(pCtx), nsPrime*(Ipp32s)sizeof(BNU_CHUNK_T)); + gsUnpackModEngineCtx((Ipp8u*)pB + gsMontOffset, PRIME_MONT(pCtx)); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimeg.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimeg.h new file mode 100644 index 0000000..ac0cb8e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimeg.h @@ -0,0 +1,91 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// Prime Number Primitives. +// +// +*/ + + +#if !defined(_CP_PRIME_H) +#define _CP_PRIME_H + +#include "pcpbn.h" +#include "pcpmontgomery.h" + + +/* +// Prime context +*/ +struct _cpPrime { + Ipp32u idCtx; /* Prime context identifier */ + cpSize maxBitSize; /* max bit length */ + BNU_CHUNK_T* pPrime; /* prime value */ + BNU_CHUNK_T* pT1; /* temporary BNU */ + BNU_CHUNK_T* pT2; /* temporary BNU */ + BNU_CHUNK_T* pT3; /* temporary BNU */ + gsModEngine* pMont; /* montgomery engine */ +}; + +/* alignment */ +#define PRIME_ALIGNMENT ((int)sizeof(void*)) + +/* Prime accessory macros */ +#define PRIME_SET_ID(ctx) ((ctx)->idCtx = (Ipp32u)idCtxPrimeNumber ^ (Ipp32u)IPP_UINT_PTR(ctx)) +#define PRIME_MAXBITSIZE(ctx) ((ctx)->maxBitSize) +#define PRIME_NUMBER(ctx) ((ctx)->pPrime) +#define PRIME_TEMP1(ctx) ((ctx)->pT1) +#define PRIME_TEMP2(ctx) ((ctx)->pT2) +#define PRIME_TEMP3(ctx) ((ctx)->pT3) +#define PRIME_MONT(ctx) ((ctx)->pMont) + +#define PRIME_VALID_ID(ctx) ((((ctx)->idCtx) ^ (Ipp32u)IPP_UINT_PTR((ctx))) == (Ipp32u)idCtxPrimeNumber) + +/* +// Number of Miller-Rabin rounds for an error rate of less than 1/2^80 for random 'b'-bit input, b >= 100. +// (see Table 4.4, Handbook of Applied Cryptography [Menezes, van Oorschot, Vanstone; CRC Press 1996] +*/ +#define MR_rounds_p80(b) ((b) >= 1300 ? 2 : \ + (b) >= 850 ? 3 : \ + (b) >= 650 ? 4 : \ + (b) >= 550 ? 5 : \ + (b) >= 450 ? 6 : \ + (b) >= 400 ? 7 : \ + (b) >= 350 ? 8 : \ + (b) >= 300 ? 9 : \ + (b) >= 250 ? 12 : \ + (b) >= 200 ? 15 : \ + (b) >= 150 ? 18 : \ + /*(b) >= 100*/ 27) + +/* easy prime test */ +#define cpMimimalPrimeTest OWNAPI(cpMimimalPrimeTest) + IPP_OWN_DECL (int, cpMimimalPrimeTest, (const Ipp32u* pPrime, cpSize ns)) + +/* prime test */ +#define cpPrimeTest OWNAPI(cpPrimeTest) + IPP_OWN_DECL (int, cpPrimeTest, (const BNU_CHUNK_T* pPrime, cpSize primeLen, cpSize nTrials, IppsPrimeState* pCtx, IppBitSupplier rndFunc, void* pRndParam)) + +#define cpPackPrimeCtx OWNAPI(cpPackPrimeCtx) + IPP_OWN_DECL (void, cpPackPrimeCtx, (const IppsPrimeState* pCtx, Ipp8u* pBuffer)) +#define cpUnpackPrimeCtx OWNAPI(cpUnpackPrimeCtx) + IPP_OWN_DECL (void, cpUnpackPrimeCtx, (const Ipp8u* pBuffer, IppsPrimeState* pCtx)) + +#endif /* _CP_PRIME_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimegen_bn.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimegen_bn.c new file mode 100644 index 0000000..97d8315 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimegen_bn.c @@ -0,0 +1,124 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptographic Primitives (ippcp) +// Prime Number Primitives. +// +// Contents: +// ippsPrimeGen_BN() +// +*/ + +#include "owncp.h" +#include "pcpprimeg.h" +#include "pcpprng.h" +#include "pcptool.h" + +/*F* +// Name: ippsPrimeGen_BN +// +// Purpose: Generates a random probable prime Big number of the specified bitlength. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pCtx +// NULL == rndFunc +// ippStsContextMatchErr !PRIME_VALID_ID() +// ippStsLengthErr 1 > nBits +// ippStsOutOfRangeErr nBits > PRIME_MAXBITSIZE() +// ippStsBadArgErr 1 > nTrials +// ippStsInsufficientEntropy when prime generation fails due +// to a poor choice of seed/context bitstream. +// ippStsNoErr no error +// +// Parameters: +// pPrime BigNum context +// nBits bitlength for the desired probable prime number +// nTrials parameter for the Miller-Rabin probable primality test +// pCtx pointer to the context +// rndFunc external PRNG +// pRndParam pointer to the external PRNG parameters +// +// +// Notes: +// ippsPrimeGen_BN() returns ippStsInsufficientEntropy, if it +// detects that it needs more entropy seed during its probable prime +// generation. In this case, the user should update PRNG parameters +// and call the primitive again. +*F*/ + +IPPFUN(IppStatus, ippsPrimeGen_BN, (IppsBigNumState* pPrime, int nBits, + int nTrials, + IppsPrimeState* pCtx, + IppBitSupplier rndFunc, void* pRndParam)) +{ + /* test generator context */ + IPP_BAD_PTR1_RET(pCtx); + IPP_BADARG_RET(!PRIME_VALID_ID(pCtx), ippStsContextMatchErr); + + /* test BN context */ + IPP_BAD_PTR1_RET(pPrime); + IPP_BADARG_RET(!BN_VALID_ID(pPrime), ippStsContextMatchErr); + + IPP_BADARG_RET(nBits<1, ippStsLengthErr); + IPP_BADARG_RET(nBits>PRIME_MAXBITSIZE(pCtx), ippStsOutOfRangeErr); + IPP_BADARG_RET(BN_ROOM(pPrime) < BITS_BNU_CHUNK(nBits), ippStsOutOfRangeErr); + + IPP_BADARG_RET(nTrials < 0, ippStsBadArgErr); + IPP_BAD_PTR1_RET(rndFunc); + + { + cpSize count; + Ipp32u result = IPP_IS_COMPOSITE; + + BNU_CHUNK_T botPattern = 0x1; + BNU_CHUNK_T topPattern = (BNU_CHUNK_T)1 << ((nBits-1)&(BNU_CHUNK_BITS-1)); + BNU_CHUNK_T topMask = MASK_BNU_CHUNK(nBits); + + BNU_CHUNK_T* pRand = BN_NUMBER(pPrime); + cpSize randLen = BITS_BNU_CHUNK(nBits); + + ZEXPAND_BNU(pRand, 0, BN_ROOM(pPrime)); + BN_SIZE(pPrime) = randLen; + BN_SIGN(pPrime) = ippBigNumPOS; + + if (nTrials < 1) + nTrials = MR_rounds_p80(nBits); + + #define MAX_COUNT (1000) + for(count=0; count +/*F* +// Name: ippsPrimeGen +// +// Purpose: Generates a random probable prime number of the specified bitlength. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pCtx +// NULL == rndFunc +// ippStsContextMatchErr !PRIME_VALID_ID() +// ippStsLengthErr 1 > nBits +// ippStsOutOfRangeErr nBits > PRIME_MAXBITSIZE() +// ippStsBadArgErr 1 > nTrials +// ippStsInsufficientEntropy when prime generation fails due +// to a poor choice of seed/context bitstream. +// ippStsNoErr no error +// +// Parameters: +// nBits bitlength for the desired probable prime number +// nTrials parameter for the Miller-Rabin probable primality test +// pCtx pointer to the context +// rndFunc external PRNG +// pRndParam pointer to the external PRNG parameters +// +// +// Notes: +// ippsPrimeGen()returns ippStsInsufficientEntropy, if it +// detects that it needs more entropy seed during its probable prime +// generation. In this case, the user should update PRNG parameters +// and call the primitive again. +*F*/ +IPPFUN(IppStatus, ippsPrimeGen, (int nBits, int nTrials, IppsPrimeState* pCtx, + IppBitSupplier rndFunc, void* pRndParam)) +{ + IPP_BAD_PTR2_RET(pCtx, rndFunc); + + IPP_BADARG_RET(!PRIME_VALID_ID(pCtx), ippStsContextMatchErr); + + IPP_BADARG_RET(nBits<1, ippStsLengthErr); + IPP_BADARG_RET(nBits>PRIME_MAXBITSIZE(pCtx), ippStsOutOfRangeErr); + + IPP_BADARG_RET(nTrials < 0, ippStsBadArgErr); + + { + cpSize count; + + BNU_CHUNK_T botPattern = 0x1; + BNU_CHUNK_T topPattern = (BNU_CHUNK_T)1 << ((nBits-1)&(BNU_CHUNK_BITS-1)); + BNU_CHUNK_T topMask = MASK_BNU_CHUNK(nBits); + + BNU_CHUNK_T* pRand = PRIME_NUMBER(pCtx); + cpSize randLen = BITS_BNU_CHUNK(nBits); + + ZEXPAND_BNU(pRand, 0, BITS_BNU_CHUNK(PRIME_MAXBITSIZE(pCtx))); + + if(nTrials < 1) + nTrials = MR_rounds_p80(nBits); + + #define MAX_COUNT (1000) + for(count=0; countidCtx +// ippStsNoErr no error +// +// Parameters: +// pPrime pointer to the BNU value +// pSize pointer to the BNU wordsize +// pCtx pointer to the context +// +*F*/ +IPPFUN(IppStatus, ippsPrimeGet, (Ipp32u* pPrime, int* pSize, const IppsPrimeState* pCtx)) +{ + IPP_BAD_PTR3_RET(pCtx, pPrime, pSize); + + IPP_BADARG_RET(!PRIME_VALID_ID(pCtx), ippStsContextMatchErr); + + { + Ipp32u* pValue = (Ipp32u*)PRIME_NUMBER(pCtx); + cpSize len32 = BITS2WORD32_SIZE(PRIME_MAXBITSIZE(pCtx)); + FIX_BNU32(pValue, len32); + + COPY_BNU(pPrime, pValue, len32); + *pSize = len32; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimeget_bn.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimeget_bn.c new file mode 100644 index 0000000..11973bd --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimeget_bn.c @@ -0,0 +1,69 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptographic Primitives (ippcp) +// Prime Number Primitives. +// +// Contents: +// ippsPrimeGet_BN() +// +*/ + +#include "owncp.h" +#include "pcpprimeg.h" + +/*F* +// Name: ippsPrimeGet_BN +// +// Purpose: Extracts probable prime and store into BN. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pCtx +// NULL == pPrime +// ippStsContextMatchErr illegal pCtx->idCtx +// illegal pPrime->idCtx +// ippStsOutOfRangeErr BN_ROOM(pPrime) < BITS_BNU_CHUNK(PRIME_MAXBITSIZE(pCtx)) +// ippStsNoErr no error +// +// Parameters: +// pPrime pointer to the BN +// pCtx pointer to the context +// +*F*/ +IPPFUN(IppStatus, ippsPrimeGet_BN, (IppsBigNumState* pPrime, const IppsPrimeState* pCtx)) +{ + IPP_BAD_PTR2_RET(pCtx, pPrime); + + IPP_BADARG_RET(!BN_VALID_ID(pPrime), ippStsContextMatchErr); + IPP_BADARG_RET(!PRIME_VALID_ID(pCtx), ippStsContextMatchErr); + + { + BNU_CHUNK_T* pValue = PRIME_NUMBER(pCtx); + cpSize ns = BITS_BNU_CHUNK(PRIME_MAXBITSIZE(pCtx)); + FIX_BNU(pValue, ns); + + IPP_BADARG_RET(BN_ROOM(pPrime) < ns, ippStsOutOfRangeErr); + + COPY_BNU(BN_NUMBER(pPrime), pValue, ns); + BN_SIZE(pPrime) = ns; + BN_SIGN(pPrime) = ippBigNumPOS; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimegetsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimegetsize.c new file mode 100644 index 0000000..5593983 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimegetsize.c @@ -0,0 +1,67 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptographic Primitives (ippcp) +// Prime Number Primitives. +// +// Contents: +// ippsPrimeGetSize() +// +*/ + +#include "owncp.h" +#include "pcpprimeg.h" +#include "pcptool.h" + +/*F* +// Name: ippsPrimeGetSize +// +// Purpose: Returns size of Prime Number Generator context (bytes). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pSize +// ippStsLengthErr 1 > nMaxBits +// ippStsNoErr no error +// +// Parameters: +// nMaxBits max length of a prime number +// pSize pointer to the size of internal context +*F*/ +IPPFUN(IppStatus, ippsPrimeGetSize, (int nMaxBits, int* pSize)) +{ + IPP_BAD_PTR1_RET(pSize); + IPP_BADARG_RET(nMaxBits<1, ippStsLengthErr); + + { + cpSize len = BITS_BNU_CHUNK(nMaxBits); + cpSize montSize; + + gsModEngineGetSize(nMaxBits, MONT_DEFAULT_POOL_LENGTH, &montSize); + + *pSize = (Ipp32s)sizeof(IppsPrimeState) + +len*(Ipp32s)sizeof(BNU_CHUNK_T) + +len*(Ipp32s)sizeof(BNU_CHUNK_T) + +len*(Ipp32s)sizeof(BNU_CHUNK_T) + +len*(Ipp32s)sizeof(BNU_CHUNK_T) + +montSize + +PRIME_ALIGNMENT-1; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimeginitca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimeginitca.c new file mode 100644 index 0000000..dbc86a4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimeginitca.c @@ -0,0 +1,78 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptographic Primitives (ippcp) +// Prime Number Primitives. +// +// Contents: +// ippsPrimeInit() +// +*/ + +#include "owncp.h" +#include "pcpprimeg.h" +#include "pcptool.h" + +/*F* +// Name: ippsPrimeInit +// +// Purpose: Initializes Prime Number Generator context +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pCtx +// ippStsLengthErr 1 > nMaxBits +// ippStsNoErr no error +// +// Parameters: +// nMaxBits max length of a prime number +// pCtx pointer to the context to be initialized +*F*/ +IPPFUN(IppStatus, ippsPrimeInit, (int nMaxBits, IppsPrimeState* pCtx)) +{ + IPP_BAD_PTR1_RET(pCtx); + IPP_BADARG_RET(nMaxBits<1, ippStsLengthErr); + + { + Ipp8u* ptr = (Ipp8u*)pCtx; + + cpSize len = BITS_BNU_CHUNK(nMaxBits); + + PRIME_SET_ID(pCtx); + PRIME_MAXBITSIZE(pCtx) = nMaxBits; + + ptr += sizeof(IppsPrimeState); + ptr = (Ipp8u*)(IPP_ALIGNED_PTR(ptr, PRIME_ALIGNMENT)); + PRIME_NUMBER(pCtx) = (BNU_CHUNK_T*)ptr; + + ptr += len*(Ipp32s)sizeof(BNU_CHUNK_T); + PRIME_TEMP1(pCtx) = (BNU_CHUNK_T*)ptr; + + ptr += len*(Ipp32s)(Ipp32s)sizeof(BNU_CHUNK_T); + PRIME_TEMP2(pCtx) = (BNU_CHUNK_T*)ptr; + + ptr += len*(Ipp32s)sizeof(BNU_CHUNK_T); + PRIME_TEMP3(pCtx) = (BNU_CHUNK_T*)ptr; + + ptr += len*(Ipp32s)sizeof(BNU_CHUNK_T); + PRIME_MONT(pCtx) = (gsModEngine*)(ptr); + gsModEngineInit(PRIME_MONT(pCtx), NULL, nMaxBits, MONT_DEFAULT_POOL_LENGTH, gsModArithMont()); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimegsetca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimegsetca.c new file mode 100644 index 0000000..2a4c029 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimegsetca.c @@ -0,0 +1,74 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptographic Primitives (ippcp) +// Prime Number Primitives. +// +// Contents: +// ippsPrimeSet() +// +*/ + +#include "owncp.h" +#include "pcpprimeg.h" + +/*F* +// Name: ippsPrimeSet +// +// Purpose: Sets a trial BNU for further testing +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pCtx +// NULL == pPrime +// ippStsContextMatchErr illegal pCtx->idCtx +// ippStsLengthErr 1 > nBits +// ippStsOutOfRangeErr nBits > PRIME_MAXBITSIZE(pCtx) +// ippStsNoErr no error +// +// Parameters: +// pPrime pointer to the number to be set +// nBits bitlength of input number bitlength +// pCtx pointer to the context +// +*F*/ +IPPFUN(IppStatus, ippsPrimeSet, (const Ipp32u* pPrime, int nBits, IppsPrimeState* pCtx)) +{ + IPP_BAD_PTR2_RET(pCtx, pPrime); + IPP_BADARG_RET(nBits<1, ippStsLengthErr); + + IPP_BADARG_RET(!PRIME_VALID_ID(pCtx), ippStsContextMatchErr); + + IPP_BADARG_RET(nBits > PRIME_MAXBITSIZE(pCtx), ippStsOutOfRangeErr); + + /* clear prime container */ + ZEXPAND_BNU(PRIME_NUMBER(pCtx), 0, BITS_BNU_CHUNK(PRIME_MAXBITSIZE(pCtx))); + + { + Ipp32u* pValue = (Ipp32u*)PRIME_NUMBER(pCtx); + + cpSize len32 = BITS2WORD32_SIZE(nBits); + Ipp32u mask = MAKEMASK32(nBits); + FIX_BNU32(pPrime, len32); + + ZEXPAND_COPY_BNU(pValue, BITS2WORD32_SIZE(PRIME_MAXBITSIZE(pCtx)), pPrime, len32); + pValue[len32-1] &= mask; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimeset_bn.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimeset_bn.c new file mode 100644 index 0000000..6a567ab --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimeset_bn.c @@ -0,0 +1,72 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptographic Primitives (ippcp) +// Prime Number Primitives. +// +// Contents: +// ippsPrimeSet_BN() +// +*/ + +#include "owncp.h" +#include "pcpprimeg.h" + +/*F* +// Name: ippsPrimeSet_BN +// +// Purpose: Sets a trial BN for further testing +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pCtx +// NULL == pPrime +// ippStsContextMatchErr illegal pCtx->idCtx +// illegal pPrime->idCtx +// ippStsOutOfRangeErr BITSIZE_BNU(BN_NUMBER(pPrime), BN_SIZE(pPrime)) +// > PRIME_MAXBITSIZE(pCtx) +// ippStsNoErr no error +// +// Parameters: +// pPrime pointer to the BN to be set +// pCtx pointer to the context +// +*F*/ +IPPFUN(IppStatus, ippsPrimeSet_BN, (const IppsBigNumState* pPrime, IppsPrimeState* pCtx)) +{ + IPP_BAD_PTR2_RET(pCtx, pPrime); + + IPP_BADARG_RET(!BN_VALID_ID(pPrime), ippStsContextMatchErr); + IPP_BADARG_RET(!PRIME_VALID_ID(pCtx), ippStsContextMatchErr); + + IPP_BADARG_RET(BITSIZE_BNU(BN_NUMBER(pPrime), BN_SIZE(pPrime)) > PRIME_MAXBITSIZE(pCtx), ippStsOutOfRangeErr); + + { + BNU_CHUNK_T* pPrimeU = BN_NUMBER(pPrime); + cpSize ns = BN_SIZE(pPrime); + cpSize nBits = BITSIZE_BNU(pPrimeU, ns); + + BNU_CHUNK_T* pPrimeCtx = PRIME_NUMBER(pCtx); + BNU_CHUNK_T topMask = MASK_BNU_CHUNK(nBits); + + ZEXPAND_COPY_BNU(pPrimeCtx, BITS_BNU_CHUNK(PRIME_MAXBITSIZE(pCtx)), pPrimeU, ns); + pPrimeCtx[ns-1] &= topMask; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimetest.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimetest.c new file mode 100644 index 0000000..5fdcc11 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimetest.c @@ -0,0 +1,78 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptographic Primitives (ippcp) +// Prime Number Primitives. +// +// Contents: +// ippsPrimeTest() +// +*/ + +#include "owncp.h" +#include "pcpprimeg.h" +#include "pcpprng.h" +#include "pcptool.h" + +/*F* +// Name: ippsPrimeTest +// +// Purpose: Test a number for being a probable prime. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pCtx +// NULL == rndFunc +// NULL == pResult +// ippStsContextMatchErr !PRIME_VALID_ID() +// ippStsBadArgErr 1 > nTrials +// ippStsNoErr no error +// +// Parameters: +// pResult result of test +// nTrials parameter for the Miller-Rabin probable primality test +// pCtx pointer to the context +// rndFunc external PRNG +// pRndParam pointer to the external PRNG parameters +*F*/ + +IPPFUN(IppStatus, ippsPrimeTest, (int nTrials, + Ipp32u* pResult, IppsPrimeState* pCtx, + IppBitSupplier rndFunc, void* pRndParam)) +{ + IPP_BAD_PTR3_RET(pResult, pCtx, rndFunc); + IPP_BADARG_RET(nTrials<1, ippStsBadArgErr); + + IPP_BADARG_RET(!PRIME_VALID_ID(pCtx), ippStsContextMatchErr); + + { + BNU_CHUNK_T* pPrime = PRIME_NUMBER(pCtx); + cpSize ns = BITS_BNU_CHUNK(PRIME_MAXBITSIZE(pCtx)); + FIX_BNU(pPrime, ns); + + { + int ret = cpPrimeTest(pPrime, ns, nTrials, pCtx, rndFunc, pRndParam); + if(-1 == ret) + return ippStsErr; + else { + *pResult = ret? IPP_IS_PRIME : IPP_IS_COMPOSITE; + return ippStsNoErr; + } + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimetest_bn.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimetest_bn.c new file mode 100644 index 0000000..2144c8f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprimetest_bn.c @@ -0,0 +1,81 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Intel(R) Integrated Performance Primitives. Cryptographic Primitives (ippcp) +// Prime Number Primitives. +// +// Contents: +// ippsPrimeTest_BN() +// +*/ + +#include "owncp.h" +#include "pcpprimeg.h" +#include "pcpprng.h" +#include "pcptool.h" + +/*F* +// Name: ippsPrimeTest_BN +// +// Purpose: Test a Big number for being a probable prime. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pCtx +// NULL == rndFunc +// NULL == pResult +// ippStsContextMatchErr !PRIME_VALID_ID() +// ippStsBadArgErr 1 > nTrials +// ippStsNoErr no error +// +// Parameters: +// pPrime BigNum context +// pResult result of test +// nTrials parameter for the Miller-Rabin probable primality test +// pCtx pointer to the context +// rndFunc external PRNG +// pRndParam pointer to the external PRNG parameters +*F*/ + +IPPFUN(IppStatus, ippsPrimeTest_BN, (const IppsBigNumState* pPrime, + int nTrials, + Ipp32u* pResult, IppsPrimeState* pCtx, + IppBitSupplier rndFunc, void* pRndParam)) +{ + IPP_BAD_PTR4_RET(pPrime, pResult, pCtx, rndFunc); + IPP_BADARG_RET(nTrials<1, ippStsBadArgErr); + + IPP_BADARG_RET(!PRIME_VALID_ID(pCtx), ippStsContextMatchErr); + + IPP_BADARG_RET(!BN_VALID_ID(pPrime), ippStsContextMatchErr); + + { + BNU_CHUNK_T* pPrimeBN = BN_NUMBER(pPrime); + cpSize ns = BN_SIZE(pPrime); + + { + int ret = cpPrimeTest(pPrimeBN, ns, nTrials, pCtx, rndFunc, pRndParam); + if(-1 == ret) + return ippStsErr; + else { + *pResult = ret? IPP_IS_PRIME : IPP_IS_COMPOSITE; + return ippStsNoErr; + } + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprng.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprng.h new file mode 100644 index 0000000..a62e1b5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprng.h @@ -0,0 +1,65 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Internal Definitions and +// Internal Pseudo Random Generator Function Prototypes +// +*/ + +#if !defined(_CP_PRNG_H) +#define _CP_PRNG_H + +/* +// Pseudo-random generation context +*/ + +#define MAX_XKEY_SIZE 512 +#define DEFAULT_XKEY_SIZE 512 /* must be >=160 || <=512 */ + +struct _cpPRNG { + Ipp32u idCtx; /* PRNG identifier */ + cpSize seedBits; /* secret seed-key bitsize */ + BNU_CHUNK_T Q[BITS_BNU_CHUNK(160)]; /* modulus */ + BNU_CHUNK_T T[BITS_BNU_CHUNK(160)]; /* parameter of SHA_G() funct */ + BNU_CHUNK_T xAug[BITS_BNU_CHUNK(MAX_XKEY_SIZE)]; /* optional entropy augment */ + BNU_CHUNK_T xKey[BITS_BNU_CHUNK(MAX_XKEY_SIZE)]; /* secret seed-key */ +}; + +/* alignment */ +#define PRNG_ALIGNMENT ((int)(sizeof(void*))) + +#define RAND_SET_ID(ctx) ((ctx)->idCtx = (Ipp32u)idCtxPRNG ^ (Ipp32u)IPP_UINT_PTR(ctx)) +#define RAND_SEEDBITS(ctx) ((ctx)->seedBits) +#define RAND_Q(ctx) ((ctx)->Q) +#define RAND_T(ctx) ((ctx)->T) +#define RAND_XAUGMENT(ctx) ((ctx)->xAug) +#define RAND_XKEY(ctx) ((ctx)->xKey) + +#define RAND_VALID_ID(ctx) ((((ctx)->idCtx) ^ (Ipp32u)IPP_UINT_PTR((ctx))) == (Ipp32u)idCtxPRNG) + +#define cpPRNGen OWNAPI(cpPRNGen) + IPP_OWN_DECL (int, cpPRNGen, (Ipp32u* pBuffer, cpSize bitLen, IppsPRNGState* pCtx)) +#define cpPRNGenPattern OWNAPI(cpPRNGenPattern) + IPP_OWN_DECL (int, cpPRNGenPattern, (BNU_CHUNK_T* pRand, int bitSize, BNU_CHUNK_T botPattern, BNU_CHUNK_T topPattern, IppBitSupplier rndFunc, void* pRndParam)) +#define cpPRNGenRange OWNAPI(cpPRNGenRange) + IPP_OWN_DECL (int, cpPRNGenRange, (BNU_CHUNK_T* pRand, const BNU_CHUNK_T* pLo, cpSize loLen, const BNU_CHUNK_T* pHi, cpSize hiLen, IppBitSupplier rndFunc, void* pRndParam)) + +#endif /* _CP_PRNG_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprng_gen.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprng_gen.c new file mode 100644 index 0000000..22e4622 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprng_gen.c @@ -0,0 +1,149 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// PRNG Functions +// +// Contents: +// cpPRNGen() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcphash.h" +#include "pcpprng.h" +#include "pcptool.h" + + +/* +// G() function based on SHA1 +// +// Parameters: +// T 160 bit parameter +// pHexStr input hex string +// hexStrLen size of hex string (Ipp8u segnments) +// xBNU 160 bit BNU result +// +// Note 1: +// must to be hexStrLen <= 64 (512 bits) +*/ +static void SHA1_G(Ipp32u* xBNU, const Ipp32u* T, Ipp8u* pHexStr, int hexStrLen) +{ + /* select processing function */ + cpHashProc updateFunc; + #if (_SHA_NI_ENABLING_==_FEATURE_ON_) + updateFunc = UpdateSHA1ni; + #elif (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_) + updateFunc = IsFeatureEnabled(ippCPUID_SHA)? UpdateSHA1ni : UpdateSHA1; + #else + updateFunc = UpdateSHA1; + #endif + + /* pad HexString zeros */ + PadBlock(0, pHexStr+hexStrLen, BITS2WORD8_SIZE(MAX_XKEY_SIZE)-hexStrLen); + + /* reset initial HASH value */ + xBNU[0] = T[0]; + xBNU[1] = T[1]; + xBNU[2] = T[2]; + xBNU[3] = T[3]; + xBNU[4] = T[4]; + + /* SHA1 */ + //UpdateSHA1(xBNU, pHexStr, BITS2WORD8_SIZE(MAX_XKEY_SIZE), SHA1_cnt); + updateFunc(xBNU, pHexStr, BITS2WORD8_SIZE(MAX_XKEY_SIZE), SHA1_cnt); + + /* swap back */ + SWAP(xBNU[0],xBNU[4]); + SWAP(xBNU[1],xBNU[3]); +} + +/*F* +// Name: cpPRNGen +// +// Purpose: Returns bitsize of the bitstring has beed added +// +// Returns: +// bitsize of the bitstring has beed added +// +// Parameters: +// pRand pointer to the buffer +// nBits number of bits be requested +// pRnd pointer to the context +*F*/ + +IPP_OWN_DEFN (int, cpPRNGen, (Ipp32u* pRand, cpSize nBits, IppsPRNGState* pRnd)) +{ + BNU_CHUNK_T Xj [BITS_BNU_CHUNK(MAX_XKEY_SIZE)]; + BNU_CHUNK_T XVAL[BITS_BNU_CHUNK(MAX_XKEY_SIZE)]; + + Ipp8u TXVAL[BITS2WORD8_SIZE(MAX_XKEY_SIZE)]; + + /* XKEY length in BNU_CHUNK_T */ + cpSize xKeyLen = BITS_BNU_CHUNK(RAND_SEEDBITS(pRnd)); + /* XKEY length in bytes */ + cpSize xKeySize= BITS2WORD8_SIZE(RAND_SEEDBITS(pRnd)); + /* XKEY word's mask */ + BNU_CHUNK_T xKeyMsk = MASK_BNU_CHUNK(RAND_SEEDBITS(pRnd)); + + /* number of Ipp32u chunks to be generated */ + cpSize genlen = BITS2WORD32_SIZE(nBits); + + ZEXPAND_BNU(Xj, 0, BITS_BNU_CHUNK(MAX_XKEY_SIZE)); + ZEXPAND_BNU(XVAL, 0, BITS_BNU_CHUNK(MAX_XKEY_SIZE)); + + while(genlen) { + cpSize len; + + /* Step 1: XVAL=(Xkey+Xseed) mod 2^b */ + BNU_CHUNK_T carry = cpAdd_BNU(XVAL, RAND_XKEY(pRnd), RAND_XAUGMENT(pRnd), xKeyLen); + XVAL[xKeyLen-1] &= xKeyMsk; + + /* Step 2: xj=G(t, XVAL) mod Q */ + cpToOctStr_BNU(TXVAL, xKeySize, XVAL, xKeyLen); + SHA1_G((Ipp32u*)Xj, (Ipp32u*)RAND_T(pRnd), TXVAL, xKeySize); + + { + cpSize sizeXj = BITS_BNU_CHUNK(160); + if(0 <= cpCmp_BNU(Xj, BITS_BNU_CHUNK(IPP_SHA1_DIGEST_BITSIZE), RAND_Q(pRnd),BITS_BNU_CHUNK(IPP_SHA1_DIGEST_BITSIZE)) ) + sizeXj = cpMod_BNU(Xj, BITS_BNU_CHUNK(IPP_SHA1_DIGEST_BITSIZE), RAND_Q(pRnd), BITS_BNU_CHUNK(IPP_SHA1_DIGEST_BITSIZE)); + FIX_BNU(Xj, sizeXj); + ZEXPAND_BNU(Xj, sizeXj, BITS_BNU_CHUNK(MAX_XKEY_SIZE)); + } + + /* Step 3: Xkey=(1+Xkey+Xj) mod 2^b */ + cpInc_BNU(RAND_XKEY(pRnd), RAND_XKEY(pRnd), xKeyLen, 1); + carry = cpAdd_BNU(RAND_XKEY(pRnd), RAND_XKEY(pRnd), Xj, xKeyLen); + RAND_XKEY(pRnd)[xKeyLen-1] &= xKeyMsk; + + /* fill out result */ + len = genlen=_IPP_G9) || (_IPP32E>=_IPP32E_E9)) +__INLINE int cpRand_hw_sample(BNU_CHUNK_T* pSample) +{ +#define LOCAL_COUNTER (8) + int n; + int success = 0; + for(n=0; n=_IPP32E_E9) +__INLINE int cpRand_hw_sample32(Ipp32u* pSample) +{ +#define LOCAL_COUNTER (8) + int n; + int success = 0; + for(n=0; n=_IPP32E_E9) + if( bufLen%((Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(Ipp32u))) ) { + if( !cpRand_hw_sample32(pBuffer)) { + return 0; + } + } + #endif + return 1; +} +#endif + +#endif /* #if !defined (_PCP_PRN_GEN_HW_H) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngen_bn.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngen_bn.c new file mode 100644 index 0000000..67c47f3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngen_bn.c @@ -0,0 +1,90 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// PRNG Functions +// +// Contents: +// ippsPRNGen_BN() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcphash.h" +#include "pcpprng.h" +#include "pcptool.h" + +/*F* +// Name: ippsPRNGen_BN +// +// Purpose: Generates a pseudorandom big number of the specified nBits length. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pCtx +// NULL == pRand +// +// ippStsContextMatchErr illegal pCtx->idCtx +// illegal pRand->idCtx +// +// ippStsLengthErr 1 > nBits +// nBits > BN_ROOM(pRand) +// +// ippStsNoErr no error +// +// Parameters: +// pRand pointer to the BN random +// nBits number of bits be requested +// pCtx pointer to the context +*F*/ +IPPFUN(IppStatus, ippsPRNGen_BN,(IppsBigNumState* pRand, int nBits, void* pCtx)) +{ + IppsPRNGState* pRndCtx; + + /* test PRNG context */ + IPP_BAD_PTR1_RET(pCtx); + pRndCtx = (IppsPRNGState*)(pCtx); + IPP_BADARG_RET(!RAND_VALID_ID(pRndCtx), ippStsContextMatchErr); + + /* test random BN */ + IPP_BAD_PTR1_RET(pRand); + IPP_BADARG_RET(!BN_VALID_ID(pRand), ippStsContextMatchErr); + + /* test sizes */ + IPP_BADARG_RET(nBits< 1, ippStsLengthErr); + IPP_BADARG_RET(nBits> BN_ROOM(pRand)*BNU_CHUNK_BITS, ippStsLengthErr); + + + { + BNU_CHUNK_T* pRandBN = BN_NUMBER(pRand); + cpSize rndSize = BITS_BNU_CHUNK(nBits); + BNU_CHUNK_T rndMask = MASK_BNU_CHUNK(nBits); + + cpPRNGen((Ipp32u*)pRandBN, nBits, pRndCtx); + pRandBN[rndSize-1] &= rndMask; + + FIX_BNU(pRandBN, rndSize); + BN_SIZE(pRand) = rndSize; + BN_SIGN(pRand) = ippBigNumPOS; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngen_pattern.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngen_pattern.c new file mode 100644 index 0000000..07f424e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngen_pattern.c @@ -0,0 +1,53 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// PRNG Functions +// +// Contents: +// cpPRNGenPattern() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcphash.h" +#include "pcpprng.h" +#include "pcptool.h" + +/* generates random string of specified bitSize length + returns: + 1 random bit string generated + -1 detected internal error (ippStsNoErr != rndFunc()) +*/ +IPP_OWN_DEFN (int, cpPRNGenPattern, (BNU_CHUNK_T* pRand, int bitSize, BNU_CHUNK_T botPattern, BNU_CHUNK_T topPattern, IppBitSupplier rndFunc, void* pRndParam)) +{ + BNU_CHUNK_T topMask = MASK_BNU_CHUNK(bitSize); + cpSize randLen = BITS_BNU_CHUNK(bitSize); + + IppStatus sts = rndFunc((Ipp32u*)pRand, bitSize, pRndParam); + if(ippStsNoErr!=sts) return -1; + + pRand[randLen-1] &= topMask; + pRand[0] |= botPattern; + pRand[randLen-1] |= topPattern; + return 1; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngen_range.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngen_range.c new file mode 100644 index 0000000..93a9328 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngen_range.c @@ -0,0 +1,63 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// PRNG Functions +// +// Contents: +// cpPRNGenRange() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcphash.h" +#include "pcpprng.h" +#include "pcptool.h" + +/* generates random string of specified bitSize length + within specified ragnge lo < r < Hi + returns: + 0 random bit string not generated + 1 random bit string generated + -1 detected internal error (ippStsNoErr != rndFunc()) +*/ +IPP_OWN_DEFN (int, cpPRNGenRange, (BNU_CHUNK_T* pRand, const BNU_CHUNK_T* pLo, cpSize loLen, const BNU_CHUNK_T* pHi, cpSize hiLen, IppBitSupplier rndFunc, void* pRndParam)) +{ + int bitSize = BITSIZE_BNU(pHi,hiLen); + BNU_CHUNK_T topMask = MASK_BNU_CHUNK(bitSize); + + #define MAX_COUNT (1000) + int n; + for(n=0; nidCtx +// +// ippStsLengthErr 1 > nBits +// +// ippStsNoErr no error +// +// Parameters: +// pRand pointer to the buffer +// nBits number of bits be requested +// pCtx pointer to the context +*F*/ + +IPPFUN(IppStatus, ippsPRNGen,(Ipp32u* pRand, int nBits, void* pCtx)) +{ + IppsPRNGState* pCtxCtx = (IppsPRNGState*)pCtx; + + /* test PRNG context */ + IPP_BAD_PTR2_RET(pRand, pCtx); + IPP_BADARG_RET(!RAND_VALID_ID(pCtxCtx), ippStsContextMatchErr); + + /* test sizes */ + IPP_BADARG_RET(nBits< 1, ippStsLengthErr); + + { + cpSize rndSize = BITS2WORD32_SIZE(nBits); + Ipp32u rndMask = MAKEMASK32(nBits); + + cpPRNGen(pRand, nBits, pCtxCtx); + pRand[rndSize-1] &= rndMask; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngenhw_bn.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngenhw_bn.c new file mode 100644 index 0000000..c0798ba --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngenhw_bn.c @@ -0,0 +1,93 @@ +/******************************************************************************* +* Copyright (C) 2015 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// PRNG Functions +// +// Contents: +// ippsPRNGenRDRAND_BN() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcptool.h" +#include "pcpprng_genhw.h" + +/*F* +// Name: ippsPRNGenRDRAND_BN +// +// Purpose: Generates a pseudorandom big number +// based on RDRAND instruction of the specified nBits length. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pBuffer +// +// ippStsLengthErr 1 > nBits +// nBits > BN_ROOM(pRand) +// +// ippStsNotSupportedModeErr unsupported rdrand instruction +// +// ippStsErr random big number can't be generated +// +// ippStsNoErr no error +// +// Parameters: +// pRand pointer to the big number +// nBits number of bits be requested +// pCtx pointer to the context +*F*/ +IPPFUN(IppStatus, ippsPRNGenRDRAND_BN,(IppsBigNumState* pRand, int nBits, void* pCtx)) +{ + /* test random BN */ + IPP_BAD_PTR1_RET(pRand); + IPP_BADARG_RET(!BN_VALID_ID(pRand), ippStsContextMatchErr); + + /* test sizes */ + IPP_BADARG_RET(nBits< 1, ippStsLengthErr); + IPP_BADARG_RET(nBits> BN_ROOM(pRand)*BNU_CHUNK_BITS, ippStsLengthErr); + + IPP_UNREFERENCED_PARAMETER(pCtx); + + #if ((_IPP>=_IPP_G9) || (_IPP32E>=_IPP32E_E9)) + if( IsFeatureEnabled(ippCPUID_RDRAND) ) { + BNU_CHUNK_T* pRandBN = BN_NUMBER(pRand); + cpSize rndSize = BITS_BNU_CHUNK(nBits); + BNU_CHUNK_T rndMask = MASK_BNU_CHUNK(nBits); + + if(cpRandHW_buffer((Ipp32u*)pRandBN, rndSize*(Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(Ipp32u)))) { + pRandBN[rndSize-1] &= rndMask; + + FIX_BNU(pRandBN, rndSize); + BN_SIZE(pRand) = rndSize; + BN_SIGN(pRand) = ippBigNumPOS; + + return ippStsNoErr; + } + else + return ippStsErr; + } + + /* unsupported rdrand instruction */ + else + #endif + IPP_ERROR_RET(ippStsNotSupportedModeErr); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngenhwca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngenhwca.c new file mode 100644 index 0000000..eeeb3a0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngenhwca.c @@ -0,0 +1,83 @@ +/******************************************************************************* +* Copyright (C) 2015 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// PRNG Functions +// +// Contents: +// ippsPRNGenRDRAND() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcptool.h" +#include "pcpprng_genhw.h" + +/*F* +// Name: ippsPRNGenRDRAND +// +// Purpose: Generates a pseudorandom bit sequence +// based on RDRAND instruction of the specified nBits length. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pRand +// +// ippStsLengthErr 1 > nBits +// +// ippStsNotSupportedModeErr unsupported rdrand instruction +// +// ippStsErr random bit sequence can't be generated +// +// ippStsNoErr no error +// +// Parameters: +// pRand pointer to the buffer +// nBits number of bits be requested +// pCtx pointer to the context +*F*/ +IPPFUN(IppStatus, ippsPRNGenRDRAND,(Ipp32u* pRand, int nBits, void* pCtx)) +{ + /* test PRNG buffer */ + IPP_BAD_PTR1_RET(pRand); + + /* test sizes */ + IPP_BADARG_RET(nBits< 1, ippStsLengthErr); + + IPP_UNREFERENCED_PARAMETER(pCtx); + + #if ((_IPP>=_IPP_G9) || (_IPP32E>=_IPP32E_E9)) + if( IsFeatureEnabled(ippCPUID_RDRAND) ) { + cpSize rndSize = BITS2WORD32_SIZE(nBits); + Ipp32u rndMask = MAKEMASK32(nBits); + + if(cpRandHW_buffer(pRand, rndSize)) { + pRand[rndSize-1] &= rndMask; + return ippStsNoErr; + } + else + return ippStsErr; + } + /* unsupported rdrand instruction */ + else + #endif + IPP_ERROR_RET(ippStsNotSupportedModeErr); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprnggetseed.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprnggetseed.c new file mode 100644 index 0000000..8da2254 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprnggetseed.c @@ -0,0 +1,67 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// PRNG Functions +// +// Contents: +// ippsPRNGGetSeed() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpprng.h" + +/*F* +// Name: ippsPRNGGetSeed +// +// Purpose: Get current SEED value from the state +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pCtx +// NULL == pSeed +// +// ippStsContextMatchErr illegal pCtx->idCtx +// illegal pSeed->idCtx +// ippStsOutOfRangeErr length of the actual SEED > length SEED destination +// +// ippStsNoErr no error +// +// Parameters: +// pCtx pointer to the context +// pSeed pointer to the SEED +*F*/ +IPPFUN(IppStatus, ippsPRNGGetSeed, (const IppsPRNGState* pCtx, IppsBigNumState* pSeed)) +{ + /* test PRNG context */ + IPP_BAD_PTR1_RET(pCtx); + IPP_BADARG_RET(!RAND_VALID_ID(pCtx), ippStsContextMatchErr); + + /* test seed */ + IPP_BAD_PTR1_RET(pSeed); + IPP_BADARG_RET(!BN_VALID_ID(pSeed), ippStsContextMatchErr); + + return ippsSet_BN(ippBigNumPOS, + BITS2WORD32_SIZE(RAND_SEEDBITS(pCtx)), + (Ipp32u*)RAND_XKEY(pCtx), + pSeed); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprnggetsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprnggetsize.c new file mode 100644 index 0000000..3ab23a4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprnggetsize.c @@ -0,0 +1,56 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// PRNG Functions +// +// Contents: +// ippsPRNGGetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpprng.h" +#include "pcphash.h" +#include "pcptool.h" + +/*F* +// Name: ippsPRNGGetSize +// +// Purpose: Returns size of PRNG context (bytes). +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pSize +// +// ippStsNoErr no error +// +// Parameters: +// pSize pointer to the size of internal context +*F*/ +IPPFUN(IppStatus, ippsPRNGGetSize, (int* pSize)) +{ + IPP_BAD_PTR1_RET(pSize); + + *pSize = sizeof(IppsPRNGState); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprnginitca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprnginitca.c new file mode 100644 index 0000000..8ef0230 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprnginitca.c @@ -0,0 +1,87 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// PRNG Functions +// +// Contents: +// ippsPRNGInit() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpprng.h" +#include "pcphash.h" +#include "pcptool.h" + +/*F* +// Name: ippsPRNGInit +// +// Purpose: Initializes PRNG context +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pCtx +// +// ippStsLengthErr seedBits < 1 +// seedBits < MAX_XKEY_SIZE +// seedBits % 8 !=0 +// +// ippStsNoErr no error +// +// Parameters: +// seedBits seed bitsize +// pCtx pointer to the context to be initialized +*F*/ +IPPFUN(IppStatus, ippsPRNGInit, (int seedBits, IppsPRNGState* pCtx)) +{ + /* test PRNG context */ + IPP_BAD_PTR1_RET(pCtx); + + /* test sizes */ + IPP_BADARG_RET((1>seedBits) || (seedBits>MAX_XKEY_SIZE) ||(seedBits&7), ippStsLengthErr); + + { + int hashIvSize = cpHashIvSize(ippHashAlg_SHA1); + const Ipp8u* iv = cpHashIV[ippHashAlg_SHA1]; + + /* cleanup context */ + ZEXPAND_BNU((Ipp8u*)pCtx, 0, (cpSize)(sizeof(IppsPRNGState))); + + RAND_SET_ID(pCtx); + RAND_SEEDBITS(pCtx) = seedBits; + + /* default Q parameter */ + ((Ipp32u*)RAND_Q(pCtx))[0] = 0xFFFFFFFF; + ((Ipp32u*)RAND_Q(pCtx))[1] = 0xFFFFFFFF; + ((Ipp32u*)RAND_Q(pCtx))[2] = 0xFFFFFFFF; + ((Ipp32u*)RAND_Q(pCtx))[3] = 0xFFFFFFFF; + ((Ipp32u*)RAND_Q(pCtx))[4] = 0xFFFFFFFF; + + /* workaround to avoid false positive stringop-overflow error on gcc10.1 and gcc11.1 */ + hashIvSize = ( IPP_MIN(hashIvSize, BITS2WORD8_SIZE(160)) ); + + /* default T parameter */ + CopyBlock(iv, RAND_T(pCtx), hashIvSize); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngsetaugment.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngsetaugment.c new file mode 100644 index 0000000..1c175a6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngsetaugment.c @@ -0,0 +1,73 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// PRNG Functions +// +// Contents: +// ippsPRNGSetAugment() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpprng.h" + +/*F* +// Name: ippsPRNGSetAugment +// +// Purpose: Sets the Entropy Augmentation +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pCtx +// NULL == pAug +// +// ippStsContextMatchErr illegal pCtx->idCtx +// illegal pAug->idCtx +// +// ippStsNoErr no error +// +// Parameters: +// pAug pointer to the entropy eugmentation +// pCtx pointer to the context +*F*/ + +IPPFUN(IppStatus, ippsPRNGSetAugment, (const IppsBigNumState* pAug, IppsPRNGState* pCtx)) +{ + /* test PRNG context */ + IPP_BAD_PTR1_RET(pCtx); + IPP_BADARG_RET(!RAND_VALID_ID(pCtx), ippStsContextMatchErr); + + /* test augmentation */ + IPP_BAD_PTR1_RET(pAug); + IPP_BADARG_RET(!BN_VALID_ID(pAug), ippStsContextMatchErr); + + { + cpSize argSize = BITS_BNU_CHUNK( RAND_SEEDBITS(pCtx) ); + BNU_CHUNK_T mask = MASK_BNU_CHUNK(RAND_SEEDBITS(pCtx)); + cpSize size = IPP_MIN(BN_SIZE(pAug), argSize); + + ZEXPAND_COPY_BNU(RAND_XAUGMENT(pCtx), (cpSize)(sizeof(RAND_XAUGMENT(pCtx))/sizeof(BNU_CHUNK_T)), BN_NUMBER(pAug), size); + RAND_XAUGMENT(pCtx)[argSize-1] &= mask; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngsetca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngsetca.c new file mode 100644 index 0000000..d167b6e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngsetca.c @@ -0,0 +1,69 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// PRNG Functions +// +// Contents: +// ippsPRNGSetH0() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpprng.h" + +/*F* +// Name: ippsPRNGSetH0 +// +// Purpose: Sets 160-bit parameter of G() function. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pCtx +// NULL == pH0 +// +// ippStsContextMatchErr illegal pCtx->idCtx +// illegal pH0->idCtx +// +// ippStsNoErr no error +// +// Parameters: +// pH0 pointer to the parameter used into G() function +// pCtx pointer to the context +*F*/ +IPPFUN(IppStatus, ippsPRNGSetH0,(const IppsBigNumState* pH0, IppsPRNGState* pCtx)) +{ + /* test PRNG context */ + IPP_BAD_PTR1_RET(pCtx); + IPP_BADARG_RET(!RAND_VALID_ID(pCtx), ippStsContextMatchErr); + + /* test H0 */ + IPP_BAD_PTR1_RET(pH0); + IPP_BADARG_RET(!BN_VALID_ID(pH0), ippStsContextMatchErr); + + { + cpSize len = IPP_MIN(5, BN_SIZE(pH0)*((Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(Ipp32u)))); + ZEXPAND_BNU(RAND_T(pCtx), 0, (int)(sizeof(RAND_T(pCtx))/sizeof(BNU_CHUNK_T))); + ZEXPAND_COPY_BNU((Ipp32u*)RAND_T(pCtx), (int)(sizeof(RAND_T(pCtx))/(sizeof(Ipp32u))), + (Ipp32u*)BN_NUMBER(pH0), len); + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngsetmodulus.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngsetmodulus.c new file mode 100644 index 0000000..8d4879b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngsetmodulus.c @@ -0,0 +1,67 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// PRNG Functions +// +// Contents: +// ippsPRNGSetModulus() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpprng.h" + +/*F* +// Name: ippsPRNGSetModulus +// +// Purpose: Sets 160-bit modulus Q. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pCtx +// NULL == pMod +// +// ippStsContextMatchErr illegal pCtx->idCtx +// illegal pMod->idCtx +// +// ippStsBadArgErr 160 != bitsize(pMod) +// +// ippStsNoErr no error +// +// Parameters: +// pMod pointer to the 160-bit modulus +// pCtx pointer to the context +*F*/ +IPPFUN(IppStatus, ippsPRNGSetModulus, (const IppsBigNumState* pMod, IppsPRNGState* pCtx)) +{ + /* test PRNG context */ + IPP_BAD_PTR1_RET(pCtx); + IPP_BADARG_RET(!RAND_VALID_ID(pCtx), ippStsContextMatchErr); + + /* test modulus */ + IPP_BAD_PTR1_RET(pMod); + IPP_BADARG_RET(!BN_VALID_ID(pMod), ippStsContextMatchErr); + IPP_BADARG_RET(160 != BITSIZE_BNU(BN_NUMBER(pMod),BN_SIZE(pMod)), ippStsBadArgErr); + + ZEXPAND_COPY_BNU(RAND_Q(pCtx), (int)(sizeof(RAND_Q(pCtx))/sizeof(BNU_CHUNK_T)), BN_NUMBER(pMod), BN_SIZE(pMod)); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngsetseed.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngsetseed.c new file mode 100644 index 0000000..9801353 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpprngsetseed.c @@ -0,0 +1,72 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// PRNG Functions +// +// Contents: +// ippsPRNGSetSeed() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpprng.h" + +/*F* +// Name: ippsPRNGSetSeed +// +// Purpose: Sets the initial state with the SEED value +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pCtx +// NULL == pSeed +// +// ippStsContextMatchErr illegal pCtx->idCtx +// illegal pSeed->idCtx +// +// ippStsNoErr no error +// +// Parameters: +// pSeed pointer to the SEED +// pCtx pointer to the context +*F*/ +IPPFUN(IppStatus, ippsPRNGSetSeed, (const IppsBigNumState* pSeed, IppsPRNGState* pCtx)) +{ + /* test PRNG context */ + IPP_BAD_PTR1_RET(pCtx); + IPP_BADARG_RET(!RAND_VALID_ID(pCtx), ippStsContextMatchErr); + + /* test seed */ + IPP_BAD_PTR1_RET(pSeed); + IPP_BADARG_RET(!BN_VALID_ID(pSeed), ippStsContextMatchErr); + + { + cpSize argSize = BITS_BNU_CHUNK( RAND_SEEDBITS(pCtx) ); + BNU_CHUNK_T mask = MASK_BNU_CHUNK(RAND_SEEDBITS(pCtx)); + cpSize size = IPP_MIN(BN_SIZE(pSeed), argSize); + + ZEXPAND_COPY_BNU(RAND_XKEY(pCtx), (cpSize)(sizeof(RAND_XKEY(pCtx))/sizeof(BNU_CHUNK_T)), BN_NUMBER(pSeed), size); + RAND_XKEY(pCtx)[argSize-1] &= mask; + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprij.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprij.h new file mode 100644 index 0000000..d7034f1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprij.h @@ -0,0 +1,286 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Internal Definitions and +// Internal Rijndael based Encrypt/Decrypt Function Prototypes +// +// +*/ + +#if !defined(_PCP_RIJ_H) +#define _PCP_RIJ_H + +#include "pcpaesnoise.h" + +/* +// The GF(256) modular polynomial and elements +*/ +#define WPOLY 0x011B +#define BPOLY 0x1B + +/* +// Make WORD using 4 arbitrary bytes +*/ +#define BYTES_TO_WORD(b0,b1,b2,b3) ( ( ((Ipp32u)((Ipp8u)(b3))) <<24 ) \ + |( ((Ipp32u)((Ipp8u)(b2))) <<16 ) \ + |( ((Ipp32u)((Ipp8u)(b1))) << 8 ) \ + |( ((Ipp32u)((Ipp8u)(b0))) ) ) +/* +// Make WORD setting byte in specified position +*/ +#define BYTE0_TO_WORD(b) BYTES_TO_WORD((b), 0, 0, 0) +#define BYTE1_TO_WORD(b) BYTES_TO_WORD( 0, (b), 0, 0) +#define BYTE2_TO_WORD(b) BYTES_TO_WORD( 0, 0, (b), 0) +#define BYTE3_TO_WORD(b) BYTES_TO_WORD( 0, 0, 0, (b)) + +/* +// Extract byte from specified position n. +// Sure, n=0,1,2 or 3 only +*/ +#define EBYTE(w,n) ((Ipp8u)((w) >> (8 * (n)))) + +/* alignment */ +#define RIJ_ALIGNMENT (16) +/* alignment in words */ +#define RIJ_ALIGNMENT_WORD (16/4) +/* Assuming word is 4 bytes in rij calculations */ +#define RIJ_BYTES_IN_WORD (4) + +#define MBS_RIJ128 (128/8) /* message block size (bytes) */ +#define MBS_RIJ192 (192/8) +#define MBS_RIJ256 (256/8) + +#define SR (4) /* number of rows in STATE data */ + +#define NB(msgBlks) ((msgBlks)/32) /* message block size (words) */ + /* 4-word for 128-bits data block */ + /* 6-word for 192-bits data block */ + /* 8-word for 256-bits data block */ + +#define NK(keybits) ((keybits)/32) /* key length (words): */ +#define NK128 NK(ippRijndaelKey128) /* 4-word for 128-bits security key */ +#define NK192 NK(ippRijndaelKey192) /* 6-word for 192-bits security key */ +#define NK256 NK(ippRijndaelKey256) /* 8-word for 256-bits security key */ + +#define NR128_128 (10) /* number of rounds data: 128 bits key: 128 bits are used */ +#define NR128_192 (12) /* number of rounds data: 128 bits key: 192 bits are used */ +#define NR128_256 (14) /* number of rounds data: 128 bits key: 256 bits are used */ +#define NR192_128 (12) /* number of rounds data: 192 bits key: 128 bits are used */ +#define NR192_192 (12) /* number of rounds data: 192 bits key: 192 bits are used */ +#define NR192_256 (14) /* number of rounds data: 192 bits key: 256 bits are used */ +#define NR256_128 (14) /* number of rounds data: 256 bits key: 128 bits are used */ +#define NR256_192 (14) /* number of rounds data: 256 bits key: 192 bits are used */ +#define NR256_256 (14) /* number of rounds data: 256 bits key: 256 bits are used */ + +#define NSK128_256 ((NR128_256+1)*NK128) /* max number of scheduled keys for 128-bit data (256-bit key) (in words) */ +#define NSK192_256 ((NR192_256+1)*NK192) /* max number of scheduled keys for 192-bit data (256-bit key) (in words) */ +#define NSK256_256 ((NR256_256+1)*NK256) /* max number of scheduled keys for 256-bit data (256-bit key) (in words) */ + +/* +// Rijndael's spec +// +// Rijndael128, Rijndael192 and Rijndael256 +// reserve space for maximum number of expanded keys +*/ +IPP_OWN_FUNPTR (void, RijnCipher, (const Ipp8u* pInpBlk, Ipp8u* pOutBlk, int nr, const Ipp8u* pKeys, const void* pTbl)) + +struct _cpRijndael128 { + Ipp32u idCtx; /* Rijndael spec identifier */ + int nk; /* security key length (words) */ + int nb; /* data block size (words) */ + int nr; /* number of rounds */ + RijnCipher encoder; /* encoder/decoder */ + RijnCipher decoder; /* entry point */ + Ipp32u* pEncTbl; /* expanded S-boxes for */ + Ipp32u* pDecTbl; /* encryption and decryption */ + Ipp8u* pEncKeys; /* pointer to array of keys for encryption */ + Ipp8u* pDecKeys; /* pointer to array of keys for decryption */ + Ipp32u aesNI; /* AES instruction available */ + Ipp32u safeInit; /* SafeInit performed */ + Ipp32u keys[2*NSK128_256 + RIJ_ALIGNMENT_WORD]; /* array of keys for encryption/decryption */ +#if (_AES_PROB_NOISE == _FEATURE_ON_) + cpAESNoiseParams noiseParams; +#endif +}; + +struct _cpRijndael192 { + Ipp32u idCtx; /* Rijndael spec identifier */ + int nk; /* security key length (words) */ + int nb; /* data block size (words) */ + int nr; /* number of rounds */ + RijnCipher encoder; /* encoder/decoder */ + RijnCipher decoder; /* entry point */ + Ipp32u* pEncTbl; /* expanded S-boxes for */ + Ipp32u* pDecTbl; /* encryption and decryption */ + Ipp8u* pEncKeys; /* pointer to array of keys for encryption */ + Ipp8u* pDecKeys; /* pointer to array of keys for decryption */ + Ipp32u aesNI; /* AES instruction available */ + Ipp32u safeInit; /* SafeInit performed */ + Ipp32u keys[2*NSK192_256 + RIJ_ALIGNMENT_WORD]; /* array of keys for encryption/decryption */ +#if (_AES_PROB_NOISE == _FEATURE_ON_) + cpAESNoiseParams noiseParams; +#endif +}; + + struct _cpRijndael256 { + Ipp32u idCtx; /* Rijndael spec identifier */ + int nk; /* security key length (words) */ + int nb; /* data block size (words) */ + int nr; /* number of rounds */ + RijnCipher encoder; /* encoder/decoder */ + RijnCipher decoder; /* entry point */ + Ipp32u* pEncTbl; /* expanded S-boxes for */ + Ipp32u* pDecTbl; /* encryption and decryption */ + Ipp8u* pEncKeys; /* pointer array of keys for encryption */ + Ipp8u* pDecKeys; /* pointer array of keys for decryprion */ + Ipp32u aesNI; /* AES instruction available */ + Ipp32u safeInit; /* SafeInit performed */ + Ipp32u keys[2*NSK256_256 + RIJ_ALIGNMENT_WORD]; /* array of keys for encryption/decryption */ +#if (_AES_PROB_NOISE == _FEATURE_ON_) + cpAESNoiseParams noiseParams; +#endif +}; + +#define AES_MB_MAX_KERNEL_SIZE (16) /* max number of buffers in multi buffer */ +#define CFB16_BLOCK_SIZE (16) /* CFB mode block size for multi buffer */ + +/* +// Useful macros +*/ +#define RIJ_SET_ID(ctx) ((ctx)->idCtx = (Ipp32u)idCtxRijndael ^ (Ipp32u)IPP_UINT_PTR(ctx)) +#define RIJ_RESET_ID(ctx) ((ctx)->idCtx = (Ipp32u)idCtxRijndael) +#define RIJ_NB(ctx) ((ctx)->nb) +#define RIJ_NK(ctx) ((ctx)->nk) +#define RIJ_NR(ctx) ((ctx)->nr) +#define RIJ_ENCODER(ctx) ((ctx)->encoder) +#define RIJ_DECODER(ctx) ((ctx)->decoder) +#define RIJ_ENC_SBOX(ctx) ((ctx)->pEncTbl) +#define RIJ_DEC_SBOX(ctx) ((ctx)->pDecTbl) +#define RIJ_EKEYS(ctx) ((ctx)->pEncKeys) +#define RIJ_DKEYS(ctx) ((ctx)->pDecKeys) +#define RIJ_AESNI(ctx) ((ctx)->aesNI) +#define RIJ_SAFE_INIT(ctx) ((ctx)->safeInit) +#define RIJ_KEYS_BUFFER(ctx) ((ctx)->keys) + +#if (_AES_PROB_NOISE == _FEATURE_ON_) +#define RIJ_NOISE_PARAMS(ctx) ((ctx)->noiseParams) +#endif + +#define VALID_RIJ_ID(ctx) ((((ctx)->idCtx) ^ (Ipp32u)IPP_UINT_PTR((ctx))) == (Ipp32u)idCtxRijndael) + +/* +// Internal functions +*/ +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPOSITE_GF_) +#define SafeEncrypt_RIJ128 OWNAPI(SafeEncrypt_RIJ128) + IPP_OWN_DECL (void, SafeEncrypt_RIJ128, (const Ipp8u* pInpBlk, Ipp8u* pOutBlk, int nr, const Ipp8u* pKeys, const void* pTbl)) +#define SafeDecrypt_RIJ128 OWNAPI(SafeDecrypt_RIJ128) + IPP_OWN_DECL (void, SafeDecrypt_RIJ128, (const Ipp8u* pInpBlk, Ipp8u* pOutBlk, int nr, const Ipp8u* pKeys, const void* pTbl)) +#endif + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) +#define Safe2Encrypt_RIJ128 OWNAPI(Safe2Encrypt_RIJ128) + IPP_OWN_DECL (void, Safe2Encrypt_RIJ128, (const Ipp8u* pInpBlk, Ipp8u* pOutBlk, int nr, const Ipp8u* pKeys, const void* pTbl)) +#define Safe2Decrypt_RIJ128 OWNAPI(Safe2Decrypt_RIJ128) + IPP_OWN_DECL (void, Safe2Decrypt_RIJ128, (const Ipp8u* pInpBlk, Ipp8u* pOutBlk, int nr, const Ipp8u* pKeys, const void* pTbl)) +#endif + +#if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) +#define Encrypt_RIJ128_AES_NI OWNAPI(Encrypt_RIJ128_AES_NI) + IPP_OWN_DECL (void, Encrypt_RIJ128_AES_NI, (const Ipp8u* pInpBlk, Ipp8u* pOutBlk, int nr, const Ipp8u* pKeys, const void* pTbl)) +#define EncryptECB_RIJ128pipe_AES_NI OWNAPI(EncryptECB_RIJ128pipe_AES_NI) + IPP_OWN_DECL (void, EncryptECB_RIJ128pipe_AES_NI, (const Ipp8u* pSrc, Ipp8u* pDst, int nr, const Ipp8u* pKeys, int len)) +#define EncryptCBC_RIJ128_AES_NI OWNAPI(EncryptCBC_RIJ128_AES_NI) + IPP_OWN_DECL (void, EncryptCBC_RIJ128_AES_NI, (const Ipp8u* pSrc, Ipp8u* pDst, int nr, const Ipp8u* pKeys, int len, const Ipp8u* pIV)) +#define EncryptCTR_RIJ128pipe_AES_NI OWNAPI(EncryptCTR_RIJ128pipe_AES_NI) + IPP_OWN_DECL (void, EncryptCTR_RIJ128pipe_AES_NI, (const Ipp8u* pSrc, Ipp8u* pDst, int nr, const Ipp8u* pKeys, int len, Ipp8u* pCtrValue, const Ipp8u* pCtrBitMask)) +#define EncryptStreamCTR32_AES_NI OWNAPI(EncryptStreamCTR32_AES_NI) + IPP_OWN_DECL (void, EncryptStreamCTR32_AES_NI, (const Ipp8u* pSrc, Ipp8u* pDst, int nr, const Ipp8u* pKeys, int len, Ipp8u* pCtrValue)) + +#define EncryptCFB_RIJ128_AES_NI OWNAPI(EncryptCFB_RIJ128_AES_NI) + IPP_OWN_DECL (void, EncryptCFB_RIJ128_AES_NI, (const Ipp8u* pSrc, Ipp8u* pDst, int nr, const Ipp8u* pKeys, int len, int cfbBlkSize, const Ipp8u* pIV)) +#define EncryptCFB32_RIJ128_AES_NI OWNAPI(EncryptCFB32_RIJ128_AES_NI) + IPP_OWN_DECL (void, EncryptCFB32_RIJ128_AES_NI, (const Ipp8u* pSrc, Ipp8u* pDst, int nr, const Ipp8u* pKeys, int len, int cfbBlkSize, const Ipp8u* pIV)) +#define EncryptCFB128_RIJ128_AES_NI OWNAPI(EncryptCFB128_RIJ128_AES_NI) + IPP_OWN_DECL (void, EncryptCFB128_RIJ128_AES_NI, (const Ipp8u* pSrc, Ipp8u* pDst, int nr, const Ipp8u* pKeys, int len, const Ipp8u* pIV)) +#define EncryptOFB_RIJ128_AES_NI OWNAPI(EncryptOFB_RIJ128_AES_NI) + IPP_OWN_DECL (void, EncryptOFB_RIJ128_AES_NI, (const Ipp8u* pSrc, Ipp8u* pDst, int nr, const Ipp8u* pKeys, int len, int ofbBlkSize, Ipp8u* pIV)) +#define EncryptOFB128_RIJ128_AES_NI OWNAPI(EncryptOFB128_RIJ128_AES_NI) + IPP_OWN_DECL (void, EncryptOFB128_RIJ128_AES_NI, (const Ipp8u* pSrc, Ipp8u* pDst, int nr, const Ipp8u* pKeys, int len, Ipp8u* pIV)) + +#define Decrypt_RIJ128_AES_NI OWNAPI(Decrypt_RIJ128_AES_NI) + IPP_OWN_DECL (void, Decrypt_RIJ128_AES_NI, (const Ipp8u* pInpBlk, Ipp8u* pOutBlk, int nr, const Ipp8u* pKeys, const void* pTbl)) +#define DecryptECB_RIJ128pipe_AES_NI OWNAPI(DecryptECB_RIJ128pipe_AES_NI) + IPP_OWN_DECL (void, DecryptECB_RIJ128pipe_AES_NI, (const Ipp8u* pSrc, Ipp8u* pDst, int nr, const Ipp8u* pKeys, int len)) +#define DecryptCBC_RIJ128pipe_AES_NI OWNAPI(DecryptCBC_RIJ128pipe_AES_NI) + IPP_OWN_DECL (void, DecryptCBC_RIJ128pipe_AES_NI, (const Ipp8u* pSrc, Ipp8u* pDst, int nr, const Ipp8u* pKeys, int len, const Ipp8u* pIV)) +#define DecryptCFB_RIJ128pipe_AES_NI OWNAPI(DecryptCFB_RIJ128pipe_AES_NI) + IPP_OWN_DECL (void, DecryptCFB_RIJ128pipe_AES_NI, (const Ipp8u* pSrc, Ipp8u* pDst, int nr, const Ipp8u* pKeys, int cfbBlocks, int cfbBlkSize, const Ipp8u* pIV)) +#define DecryptCFB32_RIJ128pipe_AES_NI OWNAPI(DecryptCFB32_RIJ128pipe_AES_NI) + IPP_OWN_DECL (void, DecryptCFB32_RIJ128pipe_AES_NI, (const Ipp8u* pSrc, Ipp8u* pDst, int nr, const Ipp8u* pKeys, int cfbBlocks, int cfbBlkSize, const Ipp8u* pIV)) +#define DecryptCFB128_RIJ128pipe_AES_NI OWNAPI(DecryptCFB128_RIJ128pipe_AES_NI) + IPP_OWN_DECL (void, DecryptCFB128_RIJ128pipe_AES_NI, (const Ipp8u* pSrc, Ipp8u* pDst, int nr, const Ipp8u* pKeys, int len, const Ipp8u* pIV)) + +#define cpExpandAesKey_NI OWNAPI(cpExpandAesKey_NI) + IPP_OWN_DECL (void, cpExpandAesKey_NI, (const Ipp8u* pSecret, IppsAESSpec* pCtx)) + +#define cpAESEncryptXTS_AES_NI OWNAPI(cpAESEncryptXTS_AES_NI) + IPP_OWN_DECL (void, cpAESEncryptXTS_AES_NI, (Ipp8u* outBlk, const Ipp8u* inpBlk, int nBlks, const Ipp8u* pRKey, int nr, Ipp8u* pTweak)) +#define cpAESDecryptXTS_AES_NI OWNAPI(cpAESDecryptXTS_AES_NI) + IPP_OWN_DECL (void, cpAESDecryptXTS_AES_NI, (Ipp8u* outBlk, const Ipp8u* inpBlk, int nBlks, const Ipp8u* pRKey, int nr, Ipp8u* pTweak)) + +#if (_IPP32E>=_IPP32E_K1) +#define cpAESEncryptXTS_VAES OWNAPI(cpAESEncryptXTS_VAES) + IPP_OWN_DECL (void, cpAESEncryptXTS_VAES, (Ipp8u* outBlk, const Ipp8u* inpBlk, int nBlks, const Ipp8u* pRKey, int nr, Ipp8u* pTweak)) +#define cpAESDecryptXTS_VAES OWNAPI(cpAESDecryptXTS_VAES) + IPP_OWN_DECL (void, cpAESDecryptXTS_VAES, (Ipp8u* outBlk, const Ipp8u* inpBlk, int nBlks, const Ipp8u* pRKey, int nr, Ipp8u* pTweak)) + +#define EncryptECB_RIJ128pipe_VAES_NI OWNAPI(EncryptECB_RIJ128pipe_VAES_NI) + IPP_OWN_DECL (void, EncryptECB_RIJ128pipe_VAES_NI, (const Ipp8u* pSrc, Ipp8u* pDst, int len, const IppsAESSpec* pCtx)) +#define EncryptCTR_RIJ128pipe_VAES_NI OWNAPI(EncryptCTR_RIJ128pipe_VAES_NI) + IPP_OWN_DECL (void, EncryptCTR_RIJ128pipe_VAES_NI, (const Ipp8u* pSrc, Ipp8u* pDst, int nr, const Ipp8u* pKeys, int len, Ipp8u* pCtrValue, const Ipp8u* pCtrBitMask)) +#define EncryptStreamCTR32_VAES_NI OWNAPI(EncryptStreamCTR32_VAES_NI) + IPP_OWN_DECL (void, EncryptStreamCTR32_VAES_NI, (const Ipp8u* pSrc, Ipp8u* pDst, int nr, const Ipp8u* pKeys, int len, Ipp8u* pCtrValue)) + +#define DecryptECB_RIJ128pipe_VAES_NI OWNAPI(DecryptECB_RIJ128pipe_VAES_NI) + IPP_OWN_DECL (void, DecryptECB_RIJ128pipe_VAES_NI, (const Ipp8u* pSrc, Ipp8u* pDst, int len, const IppsAESSpec* pCtx)) +#define DecryptCBC_RIJ128pipe_VAES_NI OWNAPI(DecryptCBC_RIJ128pipe_VAES_NI) + IPP_OWN_DECL (void, DecryptCBC_RIJ128pipe_VAES_NI, (const Ipp8u* pSrc, Ipp8u* pDst, int len, const IppsAESSpec* pCtx, const Ipp8u* pIV)) + +#define DecryptCFB_RIJ128pipe_VAES_NI OWNAPI(DecryptCFB_RIJ128pipe_VAES_NI) + IPP_OWN_DECL (void, DecryptCFB_RIJ128pipe_VAES_NI, (const Ipp8u* pSrc, Ipp8u* pDst, int len, int cfbBlkSize, const IppsAESSpec* pCtx, const Ipp8u* pIV)) +#define DecryptCFB64_RIJ128pipe_VAES_NI OWNAPI(DecryptCFB64_RIJ128pipe_VAES_NI) + IPP_OWN_DECL (void, DecryptCFB64_RIJ128pipe_VAES_NI, (const Ipp8u* pSrc, Ipp8u* pDst, int len, const IppsAESSpec* pCtx, const Ipp8u* pIV)) +#define DecryptCFB128_RIJ128pipe_VAES_NI OWNAPI(DecryptCFB128_RIJ128pipe_VAES_NI) + IPP_OWN_DECL (void, DecryptCFB128_RIJ128pipe_VAES_NI, (const Ipp8u* pSrc, Ipp8u* pDst, int len, const IppsAESSpec* pCtx, const Ipp8u* pIV)) +#endif /* _IPP32E>=_IPP32E_K1 */ + +#endif /* _IPP>=_IPP_P8 || _IPP32E>=_IPP32E_Y8 */ + +#define ExpandRijndaelKey OWNAPI(ExpandRijndaelKey) + IPP_OWN_DECL (void, ExpandRijndaelKey, (const Ipp8u* pKey, int NK, int NB, int NR, int nKeys, Ipp8u* pEncKeys, Ipp8u* pDecKeys)) + +#if(_IPP>_IPP_PX || _IPP32E>_IPP32E_PX) +#define Touch_SubsDword_8uT OWNAPI(Touch_SubsDword_8uT) + IPP_OWN_DECL (Ipp32u, Touch_SubsDword_8uT, (Ipp32u inp, const Ipp8u* pTbl, int tblLen)) +#endif + +#endif /* _PCP_RIJ_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprij128safe.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprij128safe.c new file mode 100644 index 0000000..88015d0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprij128safe.c @@ -0,0 +1,243 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Internal Safe Rijndael Encrypt, Decrypt +// (It's the special free from Sbox/tables implementation) +// +*/ + +#include "owndefs.h" +#include "owncp.h" + +#if ((_IPP <_IPP_V8) && (_IPP32E <_IPP32E_U8)) /* no pshufb instruction */ + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPOSITE_GF_) +#include "pcprij128safe.h" + +#if defined _PCP_RIJ_SAFE_OLD +/* +// old version +*/ + +IPP_OWN_DEFN (Ipp8u, TransformByte, (Ipp8u x, const Ipp8u Transformation[])) +{ + Ipp32u y = 0; + + Ipp32u testBit = 0x01; + int bit; + for(bit=0; bit<8; bit++) { + Ipp32u mask = (x & testBit)? 0xFF : 0; + y ^= Transformation[bit] & mask; + testBit <<= 1; + } + + return (Ipp8u)y; +} + +static Ipp8u Native2CompositeTransformation[] = {0x01,0x2E,0x49,0x43,0x35,0xD0,0x3D,0xE9}; + +IPP_OWN_DEFN (void, TransformNative2Composite, (Ipp8u out[], const Ipp8u inp[])) +{ + int n; + for(n=0; n<16; n++) + out[n] = TransformByte(inp[n], Native2CompositeTransformation); +} + +static Ipp8u Composite2NativeTransformation[] = {0x01,0x5C,0xE0,0x50,0x1F,0xEE,0x55,0x6A}; + +IPP_OWN_DEFN (void, TransformComposite2Native, (Ipp8u out[], const Ipp8u inp[])) +{ + int n; + for(n=0; n<16; n++) + out[n] = TransformByte(inp[n], Composite2NativeTransformation); +} + +IPP_OWN_DEFN (void, AddRoundKey, (Ipp8u out[], const Ipp8u inp[], const Ipp8u pKey[])) +{ + int n; + for(n=0; n<16; n++) + out[n] = (Ipp8u)( inp[n] ^ pKey[n] ); +} + +#define MASK_BIT(x,n) ((Ipp32s)((x)<<(31-n)) >>31) +#define GF16mulX(x) ( ((x)<<1) ^ ( MASK_BIT(x,3) & 0x13) ) + +static Ipp8u GF16mul(Ipp8u a, Ipp8u b) +{ + Ipp32u a0 = a; + Ipp32u a1 = GF16mulX(a0); + Ipp32u a2 = GF16mulX(a1); + Ipp32u a4 = GF16mulX(a2); + + Ipp32u r = (a0 & MASK_BIT(b,0)) + ^(a1 & MASK_BIT(b,1)) + ^(a2 & MASK_BIT(b,2)) + ^(a4 & MASK_BIT(b,3)); + + return (Ipp8u)r; +} + +static Ipp8u GF16_sqr[] = {0x00,0x01,0x04,0x05,0x03,0x02,0x07,0x06, + 0x0C,0x0D,0x08,0x09,0x0F,0x0E,0x0B,0x0A}; +static Ipp8u GF16_sqr1[]= {0x00,0x09,0x02,0x0B,0x08,0x01,0x0A,0x03, + 0x06,0x0F,0x04,0x0D,0x0E,0x07,0x0C,0x05}; +static Ipp8u GF16_inv[] = {0x00,0x01,0x09,0x0E,0x0D,0x0B,0x07,0x06, + 0x0F,0x02,0x0C,0x05,0x0A,0x04,0x03,0x08}; +IPP_OWN_DEFN (Ipp8u, InverseComposite, (Ipp8u x)) +{ + /* split x = {bc} => b*t + c */ + int b = (x>>4) & 0x0F; + int c = x & 0x0F; + + int D = GF16mul((Ipp8u)b, (Ipp8u)c) + ^GF16_sqr[c] + ^GF16_sqr1[b]; + + D = GF16_inv[D]; + + c = GF16mul((Ipp8u)(b^c), (Ipp8u)D); + b = GF16mul((Ipp8u)b, (Ipp8u)D); + + /* merge p*t + q => {pq} = x */ + x = (Ipp8u)((b<<4) + c); + return x; +} +#endif /* _PCP_RIJ_SAFE_OLD */ + +#if !defined _PCP_RIJ_SAFE_OLD +/* +// new version +*/ + +/* GF(2^128) -> GF((2^4)^2) isomorphous transformation matrix Ipp8u Native2CompositeTransformation[] = {0x01,0x2E,0x49,0x43,0x35,0xD0,0x3D,0xE9}; + is defined in reference code, see doc for details */ +static Ipp8u Native2CompositeTransformationLO[] = { /* defived from Native2CompositeTransformation[i], i=0,1,2,3 */ + /* 0 */ 0x00, + /* 1 */ 0x01, + /* 2 */ 0x2E, + /* 3 */ 0x2E^0x01, + /* 4 */ 0x49, + /* 5 */ 0x49^0x01, + /* 6 */ 0x49^0x2E, + /* 7 */ 0x49^0x2E^0x01, + /* 8 */ 0x43, + /* 9 */ 0x43^0x01, + /* a */ 0x43^0x2E, + /* b */ 0x43^0x2E^0x01, + /* c */ 0x43^0x49, + /* d */ 0x43^0x49^0x01, + /* e */ 0x43^0x49^0x2E, + /* f */ 0x43^0x49^0x2E^0x01 +}; +static Ipp8u Native2CompositeTransformationHI[] = { /* defived from Native2CompositeTransformation[i], i=4,5,6,7 */ + /* 0 */ 0x00, + /* 1 */ 0x35, + /* 2 */ 0xD0, + /* 3 */ 0xD0^0x35, + /* 4 */ 0x3D, + /* 5 */ 0x3D^0x35, + /* 6 */ 0x3D^0xD0, + /* 7 */ 0x3D^0xD0^0x35, + /* 8 */ 0xE9, + /* 9 */ 0xE9^0x35, + /* a */ 0xE9^0xD0, + /* b */ 0xE9^0xD0^0x35, + /* c */ 0xE9^0x3D, + /* d */ 0xE9^0x3D^0x35, + /* e */ 0xE9^0x3D^0xD0, + /* f */ 0xE9^0x3D^0xD0^0x35 +}; +IPP_OWN_DEFN (void, TransformNative2Composite, (Ipp8u out[16], const Ipp8u inp[16])) +{ + Ipp8u blk_lo[16], blk_hi[16]; + ((Ipp64u*)blk_lo)[0] = ((Ipp64u*)inp)[0] & 0x0F0F0F0F0F0F0F0F; + ((Ipp64u*)blk_lo)[1] = ((Ipp64u*)inp)[1] & 0x0F0F0F0F0F0F0F0F; + ((Ipp64u*)blk_hi)[0] = (((Ipp64u*)inp)[0]>>4) & 0x0F0F0F0F0F0F0F0F; + ((Ipp64u*)blk_hi)[1] = (((Ipp64u*)inp)[1]>>4) & 0x0F0F0F0F0F0F0F0F; + { + int n; + for(n=0; n<16; n++) { + Ipp8u lo = Native2CompositeTransformationLO[blk_lo[n]]; + Ipp8u hi = Native2CompositeTransformationHI[blk_hi[n]]; + out[n] = lo^hi; + } + } +} + +/* GF((2^4)^2) -> GF(2^128) isomorphous transformation matrix Ipp8u Composite2NativeTransformation[] = {0x01,0x5C,0xE0,0x50,0x1F,0xEE,0x55,0x6A}; + is defined in reference code, see doc for details */ +static Ipp8u Composite2NativeTransformationLO[] = { /* defived from Composite2NativeTransformation[i], i=0,1,2,3 */ + /* 0 */ 0x00, + /* 1 */ 0x01, + /* 2 */ 0x5C, + /* 3 */ 0x5C^0x01, + /* 4 */ 0xE0, + /* 5 */ 0xE0^0x01, + /* 6 */ 0xE0^0x5C, + /* 7 */ 0xE0^0x5C^0x01, + /* 8 */ 0x50, + /* 9 */ 0x50^0x01, + /* a */ 0x50^0x5C, + /* b */ 0x50^0x5C^0x01, + /* c */ 0x50^0xE0, + /* d */ 0x50^0xE0^0x01, + /* e */ 0x50^0xE0^0x5C, + /* f */ 0x50^0xE0^0x5C^0x01 +}; +static Ipp8u Composite2NativeTransformationHI[] = {/* defived from Composite2NativeTransformation[i], i=4,5,6,7 */ + /* 0 */ 0x00, + /* 1 */ 0x1F, + /* 2 */ 0xEE, + /* 3 */ 0xEE^0x1F, + /* 4 */ 0x55, + /* 5 */ 0x55^0x1F, + /* 6 */ 0x55^0xEE, + /* 7 */ 0x55^0xEE^0x1F, + /* 8 */ 0x6A, + /* 9 */ 0x6A^0x1F, + /* a */ 0x6A^0xEE, + /* b */ 0x6A^0xEE^0x1F, + /* c */ 0x6A^0x55, + /* d */ 0x6A^0x55^0x1F, + /* e */ 0x6A^0x55^0xEE, + /* f */ 0x6A^0x55^0xEE^0x1F +}; +IPP_OWN_DEFN (void, TransformComposite2Native, (Ipp8u out[16], const Ipp8u inp[16])) +{ + Ipp8u blk_lo[16], blk_hi[16]; + ((Ipp64u*)blk_lo)[0] = ((Ipp64u*)inp)[0] & 0x0F0F0F0F0F0F0F0F; + ((Ipp64u*)blk_lo)[1] = ((Ipp64u*)inp)[1] & 0x0F0F0F0F0F0F0F0F; + ((Ipp64u*)blk_hi)[0] = (((Ipp64u*)inp)[0]>>4) & 0x0F0F0F0F0F0F0F0F; + ((Ipp64u*)blk_hi)[1] = (((Ipp64u*)inp)[1]>>4) & 0x0F0F0F0F0F0F0F0F; + { + int n; + for(n=0; n<16; n++) { + Ipp8u lo = Composite2NativeTransformationLO[blk_lo[n]]; + Ipp8u hi = Composite2NativeTransformationHI[blk_hi[n]]; + out[n] = lo^hi; + } + } +} +#endif /* !_PCP_RIJ_SAFE_OLD */ + +#endif /* _ALG_AES_SAFE_COMPOSITE_GF_ */ + +#endif /* (_IPP <_IPP_V8) && (_IPP32E <_IPP32E_U8) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprij128safe.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprij128safe.h new file mode 100644 index 0000000..d5e2d6f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprij128safe.h @@ -0,0 +1,94 @@ +/******************************************************************************* +* Copyright (C) 2007 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Internal Safe Rijndael Encrypt, Decrypt +// +// +*/ + +#if !defined(_PCP_RIJ_SAFE_H) +#define _PCP_RIJ_SAFE_H + +#include "owncp.h" +#include "pcprijtables.h" +#include "pcpbnuimpl.h" +#include "pcpmask_ct.h" + +#if defined _PCP_RIJ_SAFE_OLD +/* old version */ +#define TransformByte OWNAPI(TransformByte) + IPP_OWN_DECL (Ipp8u, TransformByte, (Ipp8u x, const Ipp8u Transformation[])) +#define TransformNative2Composite OWNAPI(TransformNative2Composite) + IPP_OWN_DECL (void, TransformNative2Composite, (Ipp8u out[16], const Ipp8u inp[16])) +#define TransformComposite2Native OWNAPI(TransformComposite2Native) + IPP_OWN_DECL (void, TransformComposite2Native, (Ipp8u out[16], const Ipp8u inp[16])) +#define InverseComposite OWNAPI(InverseComposite) + IPP_OWN_DECL (Ipp8u, InverseComposite, (Ipp8u x)) +#define AddRoundKey OWNAPI(AddRoundKey) + IPP_OWN_DECL (void, AddRoundKey, (Ipp8u out[], const Ipp8u inp[], const Ipp8u pKey[])) +#endif + + +#if !defined _PCP_RIJ_SAFE_OLD +/* new version */ +#define TransformNative2Composite OWNAPI(TransformNative2Composite) + IPP_OWN_DECL (void, TransformNative2Composite, (Ipp8u out[16], const Ipp8u inp[16])) +#define TransformComposite2Native OWNAPI(TransformComposite2Native) + IPP_OWN_DECL (void, TransformComposite2Native, (Ipp8u out[16], const Ipp8u inp[16])) + +/* add round key operation */ +__INLINE void AddRoundKey(Ipp8u out[16], const Ipp8u inp[16], const Ipp8u rkey[16]) +{ + ((Ipp64u*)out)[0] = ((Ipp64u*)inp)[0] ^ ((Ipp64u*)rkey)[0]; + ((Ipp64u*)out)[1] = ((Ipp64u*)inp)[1] ^ ((Ipp64u*)rkey)[1]; +} + +/* add logs of GF(2^4) elements +// the exp table has been build matched for that implementation +*/ +__INLINE Ipp8u AddLogGF16(Ipp8u loga, Ipp8u logb) +{ + //Ipp8u s = loga+logb; + //return (s>2*14)? 15 : (s>14)? s-15 : s; + Ipp8u s = loga+logb; + Ipp8u delta = ((0xF-1)-s)>>7; + s -= delta; + s |= 0-(s>>7); + return s & (0xF); +} +#endif + +#define SELECTION_BITS ((sizeof(BNU_CHUNK_T)/sizeof(Ipp8u)) -1) + +__INLINE Ipp8u getSboxValue(Ipp8u x) +{ + BNU_CHUNK_T selection = 0; + const Ipp8u* SboxEntry = RijEncSbox; + + Ipp32u i; + for (i = 0; i>7)); +} + +__INLINE Ipp32u xtime4(Ipp32u x) +{ + Ipp32u t = (x+x) &0xFEFEFEFE; + t ^= mask4(x) & 0x1B1B1B1B; + return t; +} + +#endif /* _PCP_RIJ_SAFE2_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprij128safedec2pxca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprij128safedec2pxca.c new file mode 100644 index 0000000..0d56a28 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprij128safedec2pxca.c @@ -0,0 +1,157 @@ +/******************************************************************************* +* Copyright (C) 2015 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Decrypt 128-bit data block according to Rijndael +// (compact S-box based implementation) +// +// Contents: +// Safe2Decrypt_RIJ128() +// +// +*/ + +#include "owncp.h" + +#if ((_IPP <_IPP_V8) && (_IPP32E <_IPP32E_U8)) /* no pshufb instruction */ + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + +#include "pcprij128safe2.h" +#include "pcprijtables.h" + + +#include "pcpbnuimpl.h" +#define SELECTION_BITS ((sizeof(BNU_CHUNK_T)/sizeof(Ipp8u)) -1) + +#if defined(__INTEL_COMPILER) || defined(__INTEL_LLVM_COMPILER) +__INLINE Ipp8u getInvSboxValue(Ipp8u x) +{ + BNU_CHUNK_T selection = 0; + const BNU_CHUNK_T* SboxEntry = (BNU_CHUNK_T*)RijDecSbox; + + BNU_CHUNK_T i_sel = x/sizeof(BNU_CHUNK_T); /* selection index */ + BNU_CHUNK_T i; + for(i=0; i>= (x & SELECTION_BITS)*8; + return (Ipp8u)(selection & 0xFF); +} + +#else +#include "pcpmask_ct.h" +__INLINE Ipp8u getInvSboxValue(Ipp8u x) +{ + BNU_CHUNK_T selection = 0; + const BNU_CHUNK_T* SboxEntry = (BNU_CHUNK_T*)RijDecSbox; + + Ipp32u _x = x/sizeof(BNU_CHUNK_T); + Ipp32u i; + for(i=0; i>= (x & SELECTION_BITS)*8; + return (Ipp8u)(selection & 0xFF); +} +#endif + +__INLINE void invSubBytes(Ipp8u state[]) +{ + int i; + for(i=0;i<16;i++) + state[i] = getInvSboxValue(state[i]); +} + +__INLINE void invShiftRows(Ipp32u* state) +{ + state[1] = ROR32(state[1], 24); + state[2] = ROR32(state[2], 16); + state[3] = ROR32(state[3], 8); +} + +__INLINE void invMixColumns(Ipp32u* state) +{ + Ipp32u y0 = state[1] ^ state[2] ^ state[3]; + Ipp32u y1 = state[0] ^ state[2] ^ state[3]; + Ipp32u y2 = state[0] ^ state[1] ^ state[3]; + Ipp32u y3 = state[0] ^ state[1] ^ state[2]; + Ipp32u t02, t13, t0123; + + state[0] = xtime4(state[0]); + state[1] = xtime4(state[1]); + state[2] = xtime4(state[2]); + state[3] = xtime4(state[3]); + + y0 ^= state[0] ^ state[1]; + y1 ^= state[1] ^ state[2]; + y2 ^= state[2] ^ state[3]; + y3 ^= state[3] ^ state[0]; + + t02 = state[0] ^ state[2]; + t13 = state[1] ^ state[3]; + t02 = xtime4(t02); + t13 = xtime4(t13); + + t0123 = t02^t13; + t0123 = xtime4(t0123); + + state[0] = y0 ^t02 ^t0123; + state[1] = y1 ^t13 ^t0123; + state[2] = y2 ^t02 ^t0123; + state[3] = y3 ^t13 ^t0123; +} + + +IPP_OWN_DEFN (void, Safe2Decrypt_RIJ128, (const Ipp8u* in, Ipp8u* out, int Nr, const Ipp8u* RoundKey, const void* sbox)) +{ + Ipp32u state[4]; + + int round=0; + + IPP_UNREFERENCED_PARAMETER(sbox); + + // copy input to the state array + TRANSPOSE((Ipp8u*)state, in); + + // add the round key to the state before starting the rounds. + XorRoundKey((Ipp32u*)state, (Ipp32u*)(RoundKey+Nr*16)); + + // there will be Nr rounds + for(round=Nr-1;round>0;round--) { + invShiftRows(state); + invSubBytes((Ipp8u*)state); + XorRoundKey(state,(Ipp32u*)(RoundKey+round*16)); + invMixColumns(state); + } + + // last round + invShiftRows(state); + invSubBytes((Ipp8u*)state); + XorRoundKey(state,(Ipp32u*)(RoundKey+0*16)); + + // copy from the state to output + TRANSPOSE(out, (Ipp8u*)state); +} +#endif /* _ALG_AES_SAFE_COMPACT_SBOX_ */ + +#endif /* (_IPP <_IPP_V8) && (_IPP32E <_IPP32E_U8) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprij128safedecpxca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprij128safedecpxca.c new file mode 100644 index 0000000..e4e034d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprij128safedecpxca.c @@ -0,0 +1,411 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Decrypt 128-bit data block according to Rijndael +// (It's the special free from Sbox/tables implementation) +// +// Contents: +// SafeDecrypt_RIJ128() +// +// +*/ + +#include "owncp.h" + +#if ((_IPP <_IPP_V8) && (_IPP32E <_IPP32E_U8)) /* no pshufb instruction */ + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPOSITE_GF_) + +#include "pcprij.h" +#include "pcprij128safe.h" + +#if defined _PCP_RIJ_SAFE_OLD +/* +// old version +*/ + +static Ipp8u AffineMatrix[] = {0x50,0x36,0x15,0x82,0x01,0x34,0x40,0x3E}; +static void InvSubByte(Ipp8u inp_out[]) +{ + Ipp8u AffineCnt = 0x48; + int n; + for(n=0; n<16; n++) { + Ipp8u x = inp_out[n]; + x = TransformByte(x, AffineMatrix); + x^= AffineCnt; + x = InverseComposite(x); + inp_out[n] = x; + } +} + +static int ShiftRowsInx[] = {0,13,10,7,4,1,14,11,8,5,2,15,12,9,6,3}; +static void InvShiftRows(Ipp8u inp_out[]) +{ + Ipp8u tmp[16]; + + int n; + for(n=0; n<16; n++) + tmp[n] = inp_out[n]; + + for(n=0; n<16; n++) { + int idx = ShiftRowsInx[n]; + inp_out[n] = tmp[idx]; + } +} + +static Ipp8u GF16mul_4_2x[] = {0x00,0x24,0x48,0x6C,0x83,0xA7,0xCB,0xEF, + 0x36,0x12,0x7E,0x5A,0xB5,0x91,0xFD,0xD9}; +static Ipp8u GF16mul_1_6x[] = {0x00,0x61,0xC2,0xA3,0xB4,0xD5,0x76,0x17, + 0x58,0x39,0x9A,0xFB,0xEC,0x8D,0x2E,0x4F}; + +static Ipp8u GF16mul_C_6x[] = {0x00,0x6C,0xCB,0xA7,0xB5,0xD9,0x7E,0x12, + 0x5A,0x36,0x91,0xFD,0xEF,0x83,0x24,0x48}; +static Ipp8u GF16mul_3_Ax[] = {0x00,0xA3,0x76,0xD5,0xEC,0x4F,0x9A,0x39, + 0xFB,0x58,0x8D,0x2E,0x17,0xB4,0x61,0xC2}; + +static Ipp8u GF16mul_B_0x[] = {0x00,0x0B,0x05,0x0E,0x0A,0x01,0x0F,0x04, + 0x07,0x0C,0x02,0x09,0x0D,0x06,0x08,0x03}; +static Ipp8u GF16mul_0_Bx[] = {0x00,0xB0,0x50,0xE0,0xA0,0x10,0xF0,0x40, + 0x70,0xC0,0x20,0x90,0xD0,0x60,0x80,0x30}; + +static Ipp8u GF16mul_2_4x[] = {0x00,0x42,0x84,0xC6,0x38,0x7A,0xBC,0xFE, + 0x63,0x21,0xE7,0xA5,0x5B,0x19,0xDF,0x9D}; +static Ipp8u GF16mul_2_6x[] = {0x00,0x62,0xC4,0xA6,0xB8,0xDA,0x7C,0x1E, + 0x53,0x31,0x97,0xF5,0xEB,0x89,0x2F,0x4D}; +static void InvMixColumn(Ipp8u inp_out[]) +{ + + Ipp8u out[16]; + Ipp32u* pInp32 = (Ipp32u*)inp_out; + + int n; + + for(n=0; n<16; n++) { + int xL = inp_out[n] & 0xF; + int xH = (inp_out[n]>>4) & 0xF; + out[n] = (Ipp8u)( GF16mul_4_2x[xL] ^ GF16mul_1_6x[xH] ); + } + + pInp32[0] = ROR32(pInp32[0], 8); + pInp32[1] = ROR32(pInp32[1], 8); + pInp32[2] = ROR32(pInp32[2], 8); + pInp32[3] = ROR32(pInp32[3], 8); + for(n=0; n<16; n++) { + int xL = inp_out[n] & 0xF; + int xH = (inp_out[n]>>4) & 0xF; + out[n]^= (Ipp8u)( GF16mul_C_6x[xL] ^ GF16mul_3_Ax[xH] ); + } + + pInp32[0] = ROR32(pInp32[0], 8); + pInp32[1] = ROR32(pInp32[1], 8); + pInp32[2] = ROR32(pInp32[2], 8); + pInp32[3] = ROR32(pInp32[3], 8); + for(n=0; n<16; n++) { + int xL = inp_out[n] & 0xF; + int xH = (inp_out[n]>>4) & 0xF; + out[n]^= (Ipp8u)( GF16mul_B_0x[xL] ^ GF16mul_0_Bx[xH] ); + } + + pInp32[0] = ROR32(pInp32[0], 8); + pInp32[1] = ROR32(pInp32[1], 8); + pInp32[2] = ROR32(pInp32[2], 8); + pInp32[3] = ROR32(pInp32[3], 8); + for(n=0; n<16; n++) { + int xL = inp_out[n] & 0xF; + int xH = (inp_out[n]>>4) & 0xF; + out[n]^= (Ipp8u)( GF16mul_2_4x[xL] ^ GF16mul_2_6x[xH] ); + } + + for(n=0; n<16; n++) + inp_out[n] = out[n]; +} + +/* define number of column in the state */ +#define SC NB(128) +#define STATE_SIZE (sizeof(Ipp32u)*SC) + +IPP_OWN_DEFN (void, SafeDecrypt_RIJ128, (const Ipp8u* pInpBlk, Ipp8u* pOutBlk, int nr, const Ipp8u* pKeys, const void* pTables)) +{ + int r; + + Ipp8u state[STATE_SIZE]; /* state */ + + IPP_UNREFERENCED_PARAMETER(pTables); + + /* native => composite */ + TransformNative2Composite(state, pInpBlk); + + pKeys += nr*STATE_SIZE; + + /* input whitening */ + AddRoundKey(state, state, pKeys); + pKeys -= STATE_SIZE; + + /* regular (nr-1) rounds */ + for(r=1; r native */ + TransformComposite2Native(pOutBlk, state); +} +#endif /* _PCP_RIJ_SAFE_OLD */ + +#if !defined _PCP_RIJ_SAFE_OLD +/* +// new version +*/ + +/* +// SubByte operation in composite GF((2^4)^2) +// sdetails are in the doc. +// +// multiplication in basic GF(2^4) performs by log-and-exp sequence +*/ +static Ipp8u GF16_sqr1[] = {0x00,0x09,0x02,0x0B,0x08,0x01,0x0A,0x03, /* (x^2)*{9} */ + 0x06,0x0F,0x04,0x0D,0x0E,0x07,0x0C,0x05}; + +static Ipp8u GF16_log[] = {0xC0,0x00,0x01,0x04,0x02,0x08,0x05,0x0A, /* log element x */ + 0x03,0x0E,0x09,0x07,0x06,0x0D,0x0B,0x0C}; +static Ipp8u GF16_invlog[] = {0xC0,0x00,0x0E,0x0B,0x0D,0x07,0x0A,0x05, /* log of multiple inversion element x^-1 */ + 0x0C,0x01,0x06,0x08,0x09,0x02,0x04,0x03}; +static Ipp8u GF16_exp[] = {0x01,0x02,0x04,0x08,0x03,0x06,0x0C,0x0B, + 0x05,0x0A,0x07,0x0E,0x0F,0x0D,0x09,0x00}; /* exp[15]= 0!!! */ + +/* affine transformation matrix Ipp8u AffineMatrix[] = {0x50,0x36,0x15,0x82,0x01,0x34,0x40,0x3E}; + is defined in reference code, see doc for details */ +static Ipp8u InvAffineMatrixLO[] = { /* defived from AffineMatrix[i], i=0,1,2,3 */ + /* 0 */ 0x00, + /* 1 */ 0x50, + /* 2 */ 0x36, + /* 3 */ 0x36^0x50, + /* 4 */ 0x15, + /* 5 */ 0x15^0x50, + /* 6 */ 0x15^0x36, + /* 7 */ 0x15^0x36^0x50, + /* 8 */ 0x82, + /* 9 */ 0x82^0x50, + /* a */ 0x82^0x36, + /* b */ 0x82^0x36^0x50, + /* c */ 0x82^0x15, + /* d */ 0x82^0x15^0x50, + /* e */ 0x82^0x15^0x36, + /* f */ 0x82^0x15^0x36^0x50 +}; +static Ipp8u InvAffineMatrixHI[] = { /* defived from AffineMatrix[i], i=4,5,6,7 */ + /* 0 */ 0x00, + /* 1 */ 0x01, + /* 2 */ 0x34, + /* 3 */ 0x34^0x01, + /* 4 */ 0x40, + /* 5 */ 0x40^0x01, + /* 6 */ 0x40^0x34, + /* 7 */ 0x40^0x34^0x01, + /* 8 */ 0x3E, + /* 9 */ 0x3E^0x01, + /* a */ 0x3E^0x34, + /* b */ 0x3E^0x34^0x01, + /* c */ 0x3E^0x40, + /* d */ 0x3E^0x40^0x01, + /* e */ 0x3E^0x40^0x34, + /* f */ 0x3E^0x40^0x34^0x01 +}; + +static void InvSubByte(Ipp8u blk[16]) +{ + Ipp8u blk_c[16], blk_b[16]; + ((Ipp64u*)blk_c)[0] = ((Ipp64u*)blk)[0] & 0x0F0F0F0F0F0F0F0F; + ((Ipp64u*)blk_c)[1] = ((Ipp64u*)blk)[1] & 0x0F0F0F0F0F0F0F0F; + ((Ipp64u*)blk_b)[0] = (((Ipp64u*)blk)[0]>>4) & 0x0F0F0F0F0F0F0F0F; + ((Ipp64u*)blk_b)[1] = (((Ipp64u*)blk)[1]>>4) & 0x0F0F0F0F0F0F0F0F; + + { + const Ipp8u affineCnt = 0x48; /* /* value of H(0x05) */ + int n; + for(n=0; n<16; n++) { + Ipp8u c = blk_c[n]; + Ipp8u b = blk_b[n]; + + /* affine transformation y = (AT)*(x) + affineCnt */ + Ipp8u t = InvAffineMatrixLO[c] ^ InvAffineMatrixHI[b] ^ affineCnt; + c = t & 0xF; + b = t >> 4; + + /* {c+b*t} element inversion => {c+b*t} */ + { + Ipp8u log_c = GF16_log[c]; + Ipp8u log_b = GF16_log[b]; + Ipp8u log_cb = GF16_log[c^b]; + + Ipp8u d = GF16_sqr1[b]; + Ipp8u t = AddLogGF16(log_c, log_cb); + d ^= GF16_exp[t]; + + d = GF16_invlog[d]; + c = AddLogGF16(log_cb, d); + b = AddLogGF16(log_b, d); + c = GF16_exp[c]; + b = GF16_exp[b]; + } + + blk[n] = b<<4 | c; + } + } +} + +/* inplace ShiftRows operation */ +/* int ShiftRowsInx[] = {0,13,10,7, 4,1,14,11, 8,5,2,15, 12,9,6,3}; */ +static void InvShiftRows(Ipp8u blk[16]) +{ + Ipp8u x = blk[13]; + blk[13]= blk[9]; + blk[9] = blk[5]; + blk[5] = blk[1]; + blk[1] = x; + + x = blk[10]; + blk[10]= blk[2]; + blk[2] = x; + x = blk[14]; + blk[14]= blk[6]; + blk[6] = x; + + x = blk[3]; + blk[3] = blk[7]; + blk[7] = blk[11]; + blk[11]= blk[15]; + blk[15]= x; +} + +static Ipp8u GF16mul_4_2x[] = {0x00,0x24,0x48,0x6C,0x83,0xA7,0xCB,0xEF, + 0x36,0x12,0x7E,0x5A,0xB5,0x91,0xFD,0xD9}; +static Ipp8u GF16mul_1_6x[] = {0x00,0x61,0xC2,0xA3,0xB4,0xD5,0x76,0x17, + 0x58,0x39,0x9A,0xFB,0xEC,0x8D,0x2E,0x4F}; + +static Ipp8u GF16mul_C_6x[] = {0x00,0x6C,0xCB,0xA7,0xB5,0xD9,0x7E,0x12, + 0x5A,0x36,0x91,0xFD,0xEF,0x83,0x24,0x48}; +static Ipp8u GF16mul_3_Ax[] = {0x00,0xA3,0x76,0xD5,0xEC,0x4F,0x9A,0x39, + 0xFB,0x58,0x8D,0x2E,0x17,0xB4,0x61,0xC2}; + +static Ipp8u GF16mul_B_0x[] = {0x00,0x0B,0x05,0x0E,0x0A,0x01,0x0F,0x04, + 0x07,0x0C,0x02,0x09,0x0D,0x06,0x08,0x03}; +static Ipp8u GF16mul_0_Bx[] = {0x00,0xB0,0x50,0xE0,0xA0,0x10,0xF0,0x40, + 0x70,0xC0,0x20,0x90,0xD0,0x60,0x80,0x30}; + +static Ipp8u GF16mul_2_4x[] = {0x00,0x42,0x84,0xC6,0x38,0x7A,0xBC,0xFE, + 0x63,0x21,0xE7,0xA5,0x5B,0x19,0xDF,0x9D}; +static Ipp8u GF16mul_2_6x[] = {0x00,0x62,0xC4,0xA6,0xB8,0xDA,0x7C,0x1E, + 0x53,0x31,0x97,0xF5,0xEB,0x89,0x2F,0x4D}; +static void InvMixColumn(Ipp8u blk[16]) +{ + Ipp8u out[16]; + Ipp32u* pInp32 = (Ipp32u*)blk; + + int n; + for(n=0; n<16; n++) { + int xL = blk[n] & 0xF; + int xH = (blk[n]>>4) & 0xF; + out[n] = GF16mul_4_2x[xL] ^ GF16mul_1_6x[xH]; + } + pInp32[0] = ROR32(pInp32[0], 8); + pInp32[1] = ROR32(pInp32[1], 8); + pInp32[2] = ROR32(pInp32[2], 8); + pInp32[3] = ROR32(pInp32[3], 8); + + for(n=0; n<16; n++) { + int xL = blk[n] & 0xF; + int xH = (blk[n]>>4) & 0xF; + out[n]^= GF16mul_C_6x[xL] ^ GF16mul_3_Ax[xH]; + } + pInp32[0] = ROR32(pInp32[0], 8); + pInp32[1] = ROR32(pInp32[1], 8); + pInp32[2] = ROR32(pInp32[2], 8); + pInp32[3] = ROR32(pInp32[3], 8); + + for(n=0; n<16; n++) { + int xL = blk[n] & 0xF; + int xH = (blk[n]>>4) & 0xF; + out[n]^= GF16mul_B_0x[xL] ^ GF16mul_0_Bx[xH]; + } + pInp32[0] = ROR32(pInp32[0], 8); + pInp32[1] = ROR32(pInp32[1], 8); + pInp32[2] = ROR32(pInp32[2], 8); + pInp32[3] = ROR32(pInp32[3], 8); + + for(n=0; n<16; n++) { + int xL = blk[n] & 0xF; + int xH = (blk[n]>>4) & 0xF; + blk[n] = out[n] ^ GF16mul_2_4x[xL] ^ GF16mul_2_6x[xH]; + } +} + +/* define number of column in the state */ +#define SC NB(128) +#define STATE_SIZE (sizeof(Ipp32u)*SC) + +IPP_OWN_DEFN (void, SafeDecrypt_RIJ128, (const Ipp8u* pInpBlk, Ipp8u* pOutBlk, int nr, const Ipp8u* pKeys, const void* pTables)) +{ + int r; + + Ipp8u state[STATE_SIZE]; /* state */ + + IPP_UNREFERENCED_PARAMETER(pTables); + + /* native => composite */ + TransformNative2Composite(state, pInpBlk); + + pKeys += nr*STATE_SIZE; + + /* input whitening */ + AddRoundKey(state, state, pKeys); + pKeys -= STATE_SIZE; + + /* regular (nr-1) rounds */ + for(r=1; r native */ + TransformComposite2Native(pOutBlk, state); +} +#endif /* !_PCP_RIJ_SAFE_OLD */ + +#endif /* _ALG_AES_SAFE_COMPOSITE_GF_ */ + +#endif /* (_IPP <_IPP_V8) && (_IPP32E <_IPP32E_U8) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprij128safeenc2pxca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprij128safeenc2pxca.c new file mode 100644 index 0000000..8e8c9e4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprij128safeenc2pxca.c @@ -0,0 +1,113 @@ +/******************************************************************************* +* Copyright (C) 2015 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Encrypt 128-bit data block according to Rijndael +// (compact S-box based implementation) +// +// Contents: +// Safe2Encrypt_RIJ128() +// +// +*/ + +#include "owncp.h" + +#if ((_IPP <_IPP_V8) && (_IPP32E <_IPP32E_U8)) /* no pshufb instruction */ + +#if (_ALG_AES_SAFE_==_ALG_AES_SAFE_COMPACT_SBOX_) + +#include "pcprij128safe.h" +#include "pcprij128safe2.h" +#include "pcprijtables.h" + +__INLINE void SubBytes(Ipp8u state[]) +{ + int i; + for(i=0;i<16;i++) { + state[i] = getSboxValue(state[i]); + } +} + + +__INLINE void ShiftRows(Ipp32u* state) +{ + state[1] = ROR32(state[1], 8); + state[2] = ROR32(state[2], 16); + state[3] = ROR32(state[3], 24); +} + +// MixColumns4 function mixes the columns of the state matrix +__INLINE void MixColumns(Ipp32u* state) +{ + Ipp32u y0 = state[1] ^ state[2] ^ state[3]; + Ipp32u y1 = state[0] ^ state[2] ^ state[3]; + Ipp32u y2 = state[0] ^ state[1] ^ state[3]; + Ipp32u y3 = state[0] ^ state[1] ^ state[2]; + + state[0] = xtime4(state[0]); + state[1] = xtime4(state[1]); + state[2] = xtime4(state[2]); + state[3] = xtime4(state[3]); + + y0 ^= state[0] ^ state[1]; + y1 ^= state[1] ^ state[2]; + y2 ^= state[2] ^ state[3]; + y3 ^= state[3] ^ state[0]; + + state[0] = y0; + state[1] = y1; + state[2] = y2; + state[3] = y3; +} + +IPP_OWN_DEFN (void, Safe2Encrypt_RIJ128, (const Ipp8u* in, Ipp8u* out, int Nr, const Ipp8u* RoundKey, const void* sbox)) +{ + Ipp32u state[4]; + + int round=0; + + IPP_UNREFERENCED_PARAMETER(sbox); + + // copy input to the state array + TRANSPOSE((Ipp8u*)state, in); + + // add round key to the state before starting the rounds. + XorRoundKey(state, (Ipp32u*)(RoundKey+0*16)); + + // there will be Nr rounds + for(round=1;round>4) & 0xF ); + tmp[n] = (Ipp8u)( GF16mul_E_2x[xL] ^ GF16mul_1_Cx[xH] ); + } + + pTmp32[0] ^= ROR32(pTmp32[0], 8); + pTmp32[1] ^= ROR32(pTmp32[1], 8); + pTmp32[2] ^= ROR32(pTmp32[2], 8); + pTmp32[3] ^= ROR32(pTmp32[3], 8); + + pInp32[0] = ROR32(pInp32[0], 8); + pInp32[1] = ROR32(pInp32[1], 8); + pInp32[2] = ROR32(pInp32[2], 8); + pInp32[3] = ROR32(pInp32[3], 8); + pTmp32[0] ^= pInp32[0]; + pTmp32[1] ^= pInp32[1]; + pTmp32[2] ^= pInp32[2]; + pTmp32[3] ^= pInp32[3]; + + pInp32[0] = ROR32(pInp32[0], 8); + pInp32[1] = ROR32(pInp32[1], 8); + pInp32[2] = ROR32(pInp32[2], 8); + pInp32[3] = ROR32(pInp32[3], 8); + pTmp32[0] ^= pInp32[0]; + pTmp32[1] ^= pInp32[1]; + pTmp32[2] ^= pInp32[2]; + pTmp32[3] ^= pInp32[3]; + + pInp32[0] = ROR32(pInp32[0], 8); + pInp32[1] = ROR32(pInp32[1], 8); + pInp32[2] = ROR32(pInp32[2], 8); + pInp32[3] = ROR32(pInp32[3], 8); + pInp32[0]^= pTmp32[0]; + pInp32[1]^= pTmp32[1]; + pInp32[2]^= pTmp32[2]; + pInp32[3]^= pTmp32[3]; +} + +/* define number of column in the state */ +#define SC NB(128) +#define STATE_SIZE (sizeof(Ipp32u)*SC) + +IPP_OWN_DEFN (void, SafeEncrypt_RIJ128, (const Ipp8u* pInpBlk, Ipp8u* pOutBlk, int nr, const Ipp8u* pKeys, const void* pTables)) +{ + int r; + + Ipp8u state[STATE_SIZE]; /* state */ + + IPP_UNREFERENCED_PARAMETER(pTables); + + /* native => composite */ + TransformNative2Composite(state, pInpBlk); + + /* input whitening */ + AddRoundKey(state, state, pKeys); + pKeys += STATE_SIZE; + + /* regular (nr-1) rounds */ + for(r=1; r native */ + TransformComposite2Native(pOutBlk, state); +} +#endif /* _PCP_RIJ_SAFE_OLD */ + +#if !defined _PCP_RIJ_SAFE_OLD +/* +// new version +*/ + +/* +// SubByte operation in composite GF((2^4)^2) +// sdetails are in the doc. +// +// multiplication in basic GF(2^4) performs by log-and-exp sequence +*/ +static Ipp8u GF16_sqr1[] = {0x00,0x09,0x02,0x0B,0x08,0x01,0x0A,0x03, /* (x^2)*{9} */ + 0x06,0x0F,0x04,0x0D,0x0E,0x07,0x0C,0x05}; + +static Ipp8u GF16_log[] = {0xC0,0x00,0x01,0x04,0x02,0x08,0x05,0x0A, /* log element x */ + 0x03,0x0E,0x09,0x07,0x06,0x0D,0x0B,0x0C}; +static Ipp8u GF16_invlog[] = {0xC0,0x00,0x0E,0x0B,0x0D,0x07,0x0A,0x05, /* log of multiple inversion element x^-1 */ + 0x0C,0x01,0x06,0x08,0x09,0x02,0x04,0x03}; +static Ipp8u GF16_exp[] = {0x01,0x02,0x04,0x08,0x03,0x06,0x0C,0x0B, + 0x05,0x0A,0x07,0x0E,0x0F,0x0D,0x09,0x00}; /* exp[15]= 0!!! */ + +/* affine transformation matrix Ipp8u AffineMatrix[] = {0x10,0x22,0x55,0x82,0x41,0x34,0x40,0x2A}; + is defined in reference code, see doc for details */ +static Ipp8u FwdAffineMatrixLO[] = { /* defived from AffineMatrix[i], i=0,1,2,3 */ + /* 0 */ 0x00, + /* 1 */ 0x10, + /* 2 */ 0x22, + /* 3 */ 0x22^0x10, + /* 4 */ 0x55, + /* 5 */ 0x55^0x10, + /* 6 */ 0x55^0x22, + /* 7 */ 0x55^0x22^0x10, + /* 8 */ 0x82, + /* 9 */ 0x82^0x10, + /* a */ 0x82^0x22, + /* b */ 0x82^0x22^0x10, + /* c */ 0x82^0x55, + /* d */ 0x82^0x55^0x10, + /* e */ 0x82^0x55^0x22, + /* f */ 0x82^0x55^0x22^0x10 +}; +static Ipp8u FwdAffineMatrixHI[] = { /* defived from AffineMatrix[i], i=4,5,6,7 */ + /* 0 */ 0x00, + /* 1 */ 0x41, + /* 2 */ 0x34, + /* 3 */ 0x34^0x41, + /* 4 */ 0x40, + /* 5 */ 0x40^0x41, + /* 6 */ 0x40^0x34, + /* 7 */ 0x40^0x34^0x41, + /* 8 */ 0x2A, + /* 9 */ 0x2A^0x41, + /* a */ 0x2A^0x34, + /* b */ 0x2A^0x34^0x41, + /* c */ 0x2A^0x40, + /* d */ 0x2A^0x40^0x41, + /* e */ 0x2A^0x40^0x34, + /* f */ 0x2A^0x40^0x34^0x41 +}; + +/* inplace SubByte */ +static void FwdSubByte(Ipp8u blk[16]) +{ + Ipp8u blk_c[16], blk_b[16]; + ((Ipp64u*)blk_c)[0] = ((Ipp64u*)blk)[0] & 0x0F0F0F0F0F0F0F0F; + ((Ipp64u*)blk_c)[1] = ((Ipp64u*)blk)[1] & 0x0F0F0F0F0F0F0F0F; + ((Ipp64u*)blk_b)[0] = (((Ipp64u*)blk)[0]>>4) & 0x0F0F0F0F0F0F0F0F; + ((Ipp64u*)blk_b)[1] = (((Ipp64u*)blk)[1]>>4) & 0x0F0F0F0F0F0F0F0F; + + { + const Ipp8u affineCnt = 0xc2; /* value of H(0x63) */ + int n; + for(n=0; n<16; n++) { + Ipp8u c = blk_c[n]; + Ipp8u b = blk_b[n]; + + /* {c+b*t} element inversion => {c+b*t} */ + Ipp8u log_c = GF16_log[c]; + Ipp8u log_b = GF16_log[b]; + Ipp8u log_cb = GF16_log[c^b]; + + Ipp8u d = GF16_sqr1[b]; + Ipp8u t = AddLogGF16(log_c, log_cb); + d ^= GF16_exp[t]; + + d = GF16_invlog[d]; + c = AddLogGF16(log_cb, d); + b = AddLogGF16(log_b, d); + c = GF16_exp[c]; + b = GF16_exp[b]; + + /* affine transformation y = (AT)*(x^-1) + affineCnt */ + c = FwdAffineMatrixLO[c]; + b = FwdAffineMatrixHI[b]; + blk[n] = (c^b) ^affineCnt; + } + } +} + +/* inplace ShifttRows operation */ +/* int ShiftRowsInx[] = {0,5,10,15, 4,9,14,3, 8,13,2,7, 12,1,6,11}; */ +__INLINE void FwdShiftRows(Ipp8u blk[16]) +{ + Ipp8u x = blk[1]; + blk[1] = blk[5]; + blk[5] = blk[9]; + blk[9] = blk[13]; + blk[13]= x; + + x = blk[2]; + blk[2] = blk[10]; + blk[10]= x; + x = blk[6]; + blk[6] = blk[14]; + blk[14]= x; + + x = blk[15]; + blk[15] = blk[11]; + blk[11] = blk[7]; + blk[7] = blk[3]; + blk[3] = x; +} + +/* inplace Mixcolumns operation */ +static Ipp8u GF16mul_E_2x[] = {0x00,0x2E,0x4F,0x61,0x8D,0xA3,0xC2,0xEC, + 0x39,0x17,0x76,0x58,0xB4,0x9A,0xFB,0xD5}; +static Ipp8u GF16mul_1_Cx[] = {0x00,0xC1,0xB2,0x73,0x54,0x95,0xE6,0x27, + 0xA8,0x69,0x1A,0xDB,0xFC,0x3D,0x4E,0x8F}; +static void FwdMixColumn(Ipp8u blk[16]) +{ + Ipp8u tmp[16]; + Ipp32u* pTmp32 = (Ipp32u*)tmp; + Ipp32u* pBlk32 = (Ipp32u*)blk; + + int n; + for(n=0; n<16; n++) { + Ipp8u xL = blk[n] & 0xF; + Ipp8u xH =(blk[n]>>4) & 0xF; + tmp[n] = GF16mul_E_2x[xL] ^ GF16mul_1_Cx[xH]; + } + + pTmp32[0] ^= ROR32(pTmp32[0], 8); + pTmp32[1] ^= ROR32(pTmp32[1], 8); + pTmp32[2] ^= ROR32(pTmp32[2], 8); + pTmp32[3] ^= ROR32(pTmp32[3], 8); + + pBlk32[0] = ROR32(pBlk32[0], 8); + pBlk32[1] = ROR32(pBlk32[1], 8); + pBlk32[2] = ROR32(pBlk32[2], 8); + pBlk32[3] = ROR32(pBlk32[3], 8); + pTmp32[0] ^= pBlk32[0]; + pTmp32[1] ^= pBlk32[1]; + pTmp32[2] ^= pBlk32[2]; + pTmp32[3] ^= pBlk32[3]; + + pBlk32[0] = ROR32(pBlk32[0], 8); + pBlk32[1] = ROR32(pBlk32[1], 8); + pBlk32[2] = ROR32(pBlk32[2], 8); + pBlk32[3] = ROR32(pBlk32[3], 8); + pTmp32[0] ^= pBlk32[0]; + pTmp32[1] ^= pBlk32[1]; + pTmp32[2] ^= pBlk32[2]; + pTmp32[3] ^= pBlk32[3]; + + pBlk32[0] = ROR32(pBlk32[0], 8); + pBlk32[1] = ROR32(pBlk32[1], 8); + pBlk32[2] = ROR32(pBlk32[2], 8); + pBlk32[3] = ROR32(pBlk32[3], 8); + pBlk32[0]^= pTmp32[0]; + pBlk32[1]^= pTmp32[1]; + pBlk32[2]^= pTmp32[2]; + pBlk32[3]^= pTmp32[3]; +} + + +/* define number of column in the state */ +#define SC NB(128) +#define STATE_SIZE (sizeof(Ipp32u)*SC) + + +IPP_OWN_DEFN (void, SafeEncrypt_RIJ128, (const Ipp8u* pInpBlk, Ipp8u* pOutBlk, int nr, const Ipp8u* pKeys, const void* pTables)) +{ + int r; + + Ipp8u state[STATE_SIZE]; /* local state */ + + IPP_UNREFERENCED_PARAMETER(pTables); + + /* native => composite */ + TransformNative2Composite(state, pInpBlk); + + /* input whitening */ + AddRoundKey(state, state, pKeys); + pKeys += STATE_SIZE; + + /* regular (nr-1) rounds */ + for(r=1; r native */ + TransformComposite2Native(pOutBlk, state); +} +#endif /* !_PCP_RIJ_SAFE_OLD */ + +#endif /* _ALG_AES_SAFE_COMPOSITE_GF_ */ + +#endif /* (_IPP <_IPP_V8) && (_IPP32E <_IPP32E_U8) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprijdecsboxca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprijdecsboxca.c new file mode 100644 index 0000000..4293331 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprijdecsboxca.c @@ -0,0 +1,47 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Encrypt tables for Rijndael +// +// Contents: +// RijDecSbox[256] +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcprijtables.h" + + +/* +// Reference to pcrijencryptpxca.c +// for details +*/ + +/* +// Pure Decription S-boxes +*/ +#if defined( _IPP_DATA ) + +const __ALIGN64 Ipp8u RijDecSbox[256] = { DEC_SBOX(none_t) }; + +#endif /* _IPP_DATA */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprijencsboxca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprijencsboxca.c new file mode 100644 index 0000000..5984408 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprijencsboxca.c @@ -0,0 +1,47 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Encrypt tables for Rijndael +// +// Contents: +// RijEncSbox[256] +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcprijtables.h" + + +/* +// Reference to pcrijencryptpxca.c +// for details +*/ + +/* +// Pure Encryprion S-boxes +*/ +#if defined( _IPP_DATA ) + +const __ALIGN64 Ipp8u RijEncSbox[256] = { ENC_SBOX(none_t) }; + +#endif /* _IPP_DATA */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprijkeysca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprijkeysca.c new file mode 100644 index 0000000..0dbdd39 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprijkeysca.c @@ -0,0 +1,301 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Initialization of Rijndael +// +// Contents: +// EncRijndaelKeys() +// DecRijndaelKeys() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcprij.h" +#include "pcprijtables.h" +#include "pcptool.h" +#include "pcprij128safe.h" +#include "pcprij128safe2.h" +/* +// Pseudo Code for Key Expansion +// was shown in Sec 5.2 of FIPS-197 +// +// KeyExpansion(byte key[4*Nk], word w[Nb*(Nr+1)], Nk) +// begin +// word temp +// +// i = 0 +// +// while (i < Nk) +// w[i] = word(key[4*i], key[4*i+1], key[4*i+2], key[4*i+3]) +// i = i+1 +// end while +// +// i = Nk +// +// while (i < Nb * (Nr+1)] +// temp = w[i-1] +// if (i mod Nk = 0) +// temp = SubWord(RotWord(temp)) xor Rcon[i/Nk] +// else if (Nk > 6 and i mod Nk = 4) +// temp = SubWord(temp) +// end if +// w[i] = w[i-Nk] xor temp +// i = i + 1 +// end while +// end +// +// Note: +// I see nothing any reason for optimizing reference code above +// because it run once for each encryption/decryption procedure. +// +// +// We are going to use so called Equivalent Inverse Cipher. +// Look the reason are in pcaesdecryptpxca.c. +// +// For the Equivalent Inverse Cipher, the following pseudo code is added at +// the end of the Key Expansion routine (Sec. 5.2): +// +// for i = 0 step 1 to (Nr+1)*Nb-1 +// dw[i] = w[i] +// end for +// +// for round = 1 step 1 to Nr-1 +// InvMixColumns(dw[round*Nb, (round+1)*Nb-1]) // note change of type +// end for +// +// Note that, since InvMixColumns operates on a two-dimensional array of bytes +// while the Round Keys are held in an array of words, the call to +// InvMixColumns in this code sequence involves a change of type (i.e. the +// input to InvMixColumns() is normally the State array, which is considered +// to be a two-dimensional array of bytes, whereas the input here is a Round +// Key computed as a one-dimensional array of words). +// +// +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// Brief Consideration of InvMixColumn() +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// +// Let words U and V are input and output of InvMixColumn() operation. +// Let U(0), U(1), U(2) and U(3) are bytes of word U. +// And V(0), V(1), V(2) and V(3) are bytes of word V. +// +// According to sec 5.3.3. of FIPS-197 +// V(0) = {E}U(0) xor {B}U(1) xor {D}U(2) xor {9}U(3) +// V(1) = {9}U(0) xor {E}U(1) xor {B}U(2) xor {D}U(3) +// V(2) = {D}U(0) xor {9}U(1) xor {E}U(2) xor {B}U(3) +// V(3) = {B}U(0) xor {D}U(1) xor {9}U(2) xor {E}U(3) +// where {hex}U(n) means GF(256) multiplication +// +// Or +// V = word( {E}U(0), {9}U(0), {D}U(0), {B}U(0) ) xor +// word( {B}U(1), {E}U(1), {9}U(1), {D}U(1) ) xor +// word( {D}U(2), {B}U(2), {E}U(2), {9}U(2) ) xor +// word( {9}U(3), {D}U(3), {B}U(3), {E}U(3) ) +// +// Word values +// word( {E}x, {9}x, {D}x, {B}x ) +// word( {B}y, {E}y, {9}y, {D}y ) +// word( {D}z, {B}z, {E}z, {9}z ) +// word( {9}t, {D}t, {B}t, {E}t ) +// are precomputed tables (for x,y,z,t = 0x00, ... 0xff) +// +// Tables InvMixCol_Tbl[4] are contents exactly as we want +// and macro InvMixColumn() provide necassary operation. +*/ + +/* +// RconTbl[] contains [x**(i),{00},{00},{00}], i=0,..,10 GF(256) +// +// Note: +// Reference sec 4.2 of FIPS-197 for calculation +*/ +static const Ipp32u RconTbl[] = { + BYTE0_TO_WORD(0x01), BYTE0_TO_WORD(0x02), BYTE0_TO_WORD(0x04), BYTE0_TO_WORD(0x08), + BYTE0_TO_WORD(0x10), BYTE0_TO_WORD(0x20), BYTE0_TO_WORD(0x40), BYTE0_TO_WORD(0x80), + BYTE0_TO_WORD(0x1B), BYTE0_TO_WORD(0x36), BYTE0_TO_WORD(0x6C), BYTE0_TO_WORD(0xD8), + BYTE0_TO_WORD(0xAB), BYTE0_TO_WORD(0x4D), BYTE0_TO_WORD(0x9A), BYTE0_TO_WORD(0x2F), + BYTE0_TO_WORD(0x5E), BYTE0_TO_WORD(0xBC), BYTE0_TO_WORD(0x63), BYTE0_TO_WORD(0xC6), + BYTE0_TO_WORD(0x97), BYTE0_TO_WORD(0x35), BYTE0_TO_WORD(0x6A), BYTE0_TO_WORD(0xD4), + BYTE0_TO_WORD(0xB3), BYTE0_TO_WORD(0x7D), BYTE0_TO_WORD(0xFA), BYTE0_TO_WORD(0xEF), + BYTE0_TO_WORD(0xC5) +}; + +/// commented due to mitigation +// +///* precomputed table for InvMixColumn() operation */ +//static const Ipp32u InvMixCol_Tbl[4][256] = { +// { LINE(inv_t0) }, +// { LINE(inv_t1) }, +// { LINE(inv_t2) }, +// { LINE(inv_t3) } +//}; +// +//#define InvMixColumn(x, tbl) \ +// ( (tbl)[0][ EBYTE((x),0) ] \ +// ^(tbl)[1][ EBYTE((x),1) ] \ +// ^(tbl)[2][ EBYTE((x),2) ] \ +// ^(tbl)[3][ EBYTE((x),3) ] ) + +__INLINE Ipp32u InvMixColumn(Ipp32u x) +{ + Ipp32u x_mul_2 = xtime4(x); + Ipp32u x_mul_4 = xtime4(x_mul_2); + Ipp32u x_mul_8 = xtime4(x_mul_4); + + Ipp32u x_mul_9 = x_mul_8 ^ x; + Ipp32u x_mul_B = x_mul_8 ^ x_mul_2 ^ x; + Ipp32u x_mul_D = x_mul_8 ^ x_mul_4 ^ x; + Ipp32u x_mul_E = x_mul_8 ^ x_mul_4 ^ x_mul_2; + + x = x_mul_E ^ ROR32(x_mul_B, 8) ^ ROR32(x_mul_D, 16) ^ ROR32(x_mul_9, 24); + return x; +} + +/* +// Expansion of key for Rijndael's Encryption +*/ +IPP_OWN_DEFN (void, ExpandRijndaelKey, (const Ipp8u* pKey, int NK, int NB, int NR, int nKeys, Ipp8u* pEncKeys, Ipp8u* pDecKeys)) +{ + Ipp32u* enc_keys = (Ipp32u*)pEncKeys; + Ipp32u* dec_keys = (Ipp32u*)pDecKeys; + /* convert security key to WORD and save into the enc_key array */ + int n; + for(n=0; n=_IPP_W7) || (_IPP32E==_IPP32E_M7)) + _mm_lfence(); /* lfence added because of potential exploit of speculative execution (KW); lfence accessible on SSE2 and above */ + #endif + dec_keys[n] = InvMixColumn(dec_keys[n]); + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprijtables.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprijtables.h new file mode 100644 index 0000000..ee0414c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprijtables.h @@ -0,0 +1,197 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Rijndael Tables Declarations +// +// +*/ + +#if !defined(_PCP_RIJTBLES_H) +#define _PCP_RIJTBLES_H + +#include "owndefs.h" +#include "owncp.h" +#include "pcprij.h" + +/* +// GF(256) multiplication operations +*/ +#define gf_m2(x) ((x<<1) ^ (((x>>7) & 1) * WPOLY)) +#define gf_m4(x) ((x<<2) ^ (((x>>6) & 1) * WPOLY) ^ (((x>>6) & 2) * WPOLY)) +#define gf_m8(x) ((x<<3) ^ (((x>>5) & 1) * WPOLY) ^ (((x>>5) & 2) * WPOLY) \ + ^ (((x>>5) & 4) * WPOLY)) +#define gf_m1(x) ((x)) +#define gf_m3(x) (gf_m2(x) ^ x) +#define gf_m9(x) (gf_m8(x) ^ x) +#define gf_mB(x) (gf_m8(x) ^ gf_m2(x) ^ x) +#define gf_mD(x) (gf_m8(x) ^ gf_m4(x) ^ x) +#define gf_mE(x) (gf_m8(x) ^ gf_m4(x) ^ gf_m2(x)) + +/* +// The following particular transformations +// are used for create Encryption Tables +*/ +#define fwd_t0(x) BYTES_TO_WORD(gf_m2(x), gf_m1(x), gf_m1(x), gf_m3(x)) +#define fwd_t1(x) BYTES_TO_WORD(gf_m3(x), gf_m2(x), gf_m1(x), gf_m1(x)) +#define fwd_t2(x) BYTES_TO_WORD(gf_m1(x), gf_m3(x), gf_m2(x), gf_m1(x)) +#define fwd_t3(x) BYTES_TO_WORD(gf_m1(x), gf_m1(x), gf_m3(x), gf_m2(x)) + +/* +// The following particular transformations +// are used for create Decryption Tables +*/ +#define inv_t0(x) BYTES_TO_WORD(gf_mE(x), gf_m9(x), gf_mD(x), gf_mB(x)) +#define inv_t1(x) BYTES_TO_WORD(gf_mB(x), gf_mE(x), gf_m9(x), gf_mD(x)) +#define inv_t2(x) BYTES_TO_WORD(gf_mD(x), gf_mB(x), gf_mE(x), gf_m9(x)) +#define inv_t3(x) BYTES_TO_WORD(gf_m9(x), gf_mD(x), gf_mB(x), gf_mE(x)) + +#define exp_b3(x) BYTES_TO_WORD(0, 0, 0, (x)) +#define exp_b2(x) BYTES_TO_WORD(0, 0, (x),0) +#define exp_b1(x) BYTES_TO_WORD(0, (x),0, 0) +#define exp_b0(x) BYTES_TO_WORD((x),0, 0, 0) + +/* +// The following particular transformations +// are used for create pure Encryption/Decryption Sboxes +*/ +#define none_t(x) (x) + + +/* +// Just sequence of byte, beginning 0x00 upto 0xff +// (be parametrized by any transformation 't') +*/ +#define LINE(t) \ + t(0x00), t(0x01), t(0x02), t(0x03), t(0x04), t(0x05), t(0x06), t(0x07),\ + t(0x08), t(0x09), t(0x0a), t(0x0b), t(0x0c), t(0x0d), t(0x0e), t(0x0f),\ + t(0x10), t(0x11), t(0x12), t(0x13), t(0x14), t(0x15), t(0x16), t(0x17),\ + t(0x18), t(0x19), t(0x1a), t(0x1b), t(0x1c), t(0x1d), t(0x1e), t(0x1f),\ + t(0x20), t(0x21), t(0x22), t(0x23), t(0x24), t(0x25), t(0x26), t(0x27),\ + t(0x28), t(0x29), t(0x2a), t(0x2b), t(0x2c), t(0x2d), t(0x2e), t(0x2f),\ + t(0x30), t(0x31), t(0x32), t(0x33), t(0x34), t(0x35), t(0x36), t(0x37),\ + t(0x38), t(0x39), t(0x3a), t(0x3b), t(0x3c), t(0x3d), t(0x3e), t(0x3f),\ + t(0x40), t(0x41), t(0x42), t(0x43), t(0x44), t(0x45), t(0x46), t(0x47),\ + t(0x48), t(0x49), t(0x4a), t(0x4b), t(0x4c), t(0x4d), t(0x4e), t(0x4f),\ + t(0x50), t(0x51), t(0x52), t(0x53), t(0x54), t(0x55), t(0x56), t(0x57),\ + t(0x58), t(0x59), t(0x5a), t(0x5b), t(0x5c), t(0x5d), t(0x5e), t(0x5f),\ + t(0x60), t(0x61), t(0x62), t(0x63), t(0x64), t(0x65), t(0x66), t(0x67),\ + t(0x68), t(0x69), t(0x6a), t(0x6b), t(0x6c), t(0x6d), t(0x6e), t(0x6f),\ + t(0x70), t(0x71), t(0x72), t(0x73), t(0x74), t(0x75), t(0x76), t(0x77),\ + t(0x78), t(0x79), t(0x7a), t(0x7b), t(0x7c), t(0x7d), t(0x7e), t(0x7f),\ + t(0x80), t(0x81), t(0x82), t(0x83), t(0x84), t(0x85), t(0x86), t(0x87),\ + t(0x88), t(0x89), t(0x8a), t(0x8b), t(0x8c), t(0x8d), t(0x8e), t(0x8f),\ + t(0x90), t(0x91), t(0x92), t(0x93), t(0x94), t(0x95), t(0x96), t(0x97),\ + t(0x98), t(0x99), t(0x9a), t(0x9b), t(0x9c), t(0x9d), t(0x9e), t(0x9f),\ + t(0xa0), t(0xa1), t(0xa2), t(0xa3), t(0xa4), t(0xa5), t(0xa6), t(0xa7),\ + t(0xa8), t(0xa9), t(0xaa), t(0xab), t(0xac), t(0xad), t(0xae), t(0xaf),\ + t(0xb0), t(0xb1), t(0xb2), t(0xb3), t(0xb4), t(0xb5), t(0xb6), t(0xb7),\ + t(0xb8), t(0xb9), t(0xba), t(0xbb), t(0xbc), t(0xbd), t(0xbe), t(0xbf),\ + t(0xc0), t(0xc1), t(0xc2), t(0xc3), t(0xc4), t(0xc5), t(0xc6), t(0xc7),\ + t(0xc8), t(0xc9), t(0xca), t(0xcb), t(0xcc), t(0xcd), t(0xce), t(0xcf),\ + t(0xd0), t(0xd1), t(0xd2), t(0xd3), t(0xd4), t(0xd5), t(0xd6), t(0xd7),\ + t(0xd8), t(0xd9), t(0xda), t(0xdb), t(0xdc), t(0xdd), t(0xde), t(0xdf),\ + t(0xe0), t(0xe1), t(0xe2), t(0xe3), t(0xe4), t(0xe5), t(0xe6), t(0xe7),\ + t(0xe8), t(0xe9), t(0xea), t(0xeb), t(0xec), t(0xed), t(0xee), t(0xef),\ + t(0xf0), t(0xf1), t(0xf2), t(0xf3), t(0xf4), t(0xf5), t(0xf6), t(0xf7),\ + t(0xf8), t(0xf9), t(0xfa), t(0xfb), t(0xfc), t(0xfd), t(0xfe), t(0xff) + +/* +// Encrypt/Decrypt S-box data +// (be parametrized by any transformation 't') +*/ +#define ENC_SBOX(t) \ + t(0x63), t(0x7c), t(0x77), t(0x7b), t(0xf2), t(0x6b), t(0x6f), t(0xc5),\ + t(0x30), t(0x01), t(0x67), t(0x2b), t(0xfe), t(0xd7), t(0xab), t(0x76),\ + t(0xca), t(0x82), t(0xc9), t(0x7d), t(0xfa), t(0x59), t(0x47), t(0xf0),\ + t(0xad), t(0xd4), t(0xa2), t(0xaf), t(0x9c), t(0xa4), t(0x72), t(0xc0),\ + t(0xb7), t(0xfd), t(0x93), t(0x26), t(0x36), t(0x3f), t(0xf7), t(0xcc),\ + t(0x34), t(0xa5), t(0xe5), t(0xf1), t(0x71), t(0xd8), t(0x31), t(0x15),\ + t(0x04), t(0xc7), t(0x23), t(0xc3), t(0x18), t(0x96), t(0x05), t(0x9a),\ + t(0x07), t(0x12), t(0x80), t(0xe2), t(0xeb), t(0x27), t(0xb2), t(0x75),\ + t(0x09), t(0x83), t(0x2c), t(0x1a), t(0x1b), t(0x6e), t(0x5a), t(0xa0),\ + t(0x52), t(0x3b), t(0xd6), t(0xb3), t(0x29), t(0xe3), t(0x2f), t(0x84),\ + t(0x53), t(0xd1), t(0x00), t(0xed), t(0x20), t(0xfc), t(0xb1), t(0x5b),\ + t(0x6a), t(0xcb), t(0xbe), t(0x39), t(0x4a), t(0x4c), t(0x58), t(0xcf),\ + t(0xd0), t(0xef), t(0xaa), t(0xfb), t(0x43), t(0x4d), t(0x33), t(0x85),\ + t(0x45), t(0xf9), t(0x02), t(0x7f), t(0x50), t(0x3c), t(0x9f), t(0xa8),\ + t(0x51), t(0xa3), t(0x40), t(0x8f), t(0x92), t(0x9d), t(0x38), t(0xf5),\ + t(0xbc), t(0xb6), t(0xda), t(0x21), t(0x10), t(0xff), t(0xf3), t(0xd2),\ + t(0xcd), t(0x0c), t(0x13), t(0xec), t(0x5f), t(0x97), t(0x44), t(0x17),\ + t(0xc4), t(0xa7), t(0x7e), t(0x3d), t(0x64), t(0x5d), t(0x19), t(0x73),\ + t(0x60), t(0x81), t(0x4f), t(0xdc), t(0x22), t(0x2a), t(0x90), t(0x88),\ + t(0x46), t(0xee), t(0xb8), t(0x14), t(0xde), t(0x5e), t(0x0b), t(0xdb),\ + t(0xe0), t(0x32), t(0x3a), t(0x0a), t(0x49), t(0x06), t(0x24), t(0x5c),\ + t(0xc2), t(0xd3), t(0xac), t(0x62), t(0x91), t(0x95), t(0xe4), t(0x79),\ + t(0xe7), t(0xc8), t(0x37), t(0x6d), t(0x8d), t(0xd5), t(0x4e), t(0xa9),\ + t(0x6c), t(0x56), t(0xf4), t(0xea), t(0x65), t(0x7a), t(0xae), t(0x08),\ + t(0xba), t(0x78), t(0x25), t(0x2e), t(0x1c), t(0xa6), t(0xb4), t(0xc6),\ + t(0xe8), t(0xdd), t(0x74), t(0x1f), t(0x4b), t(0xbd), t(0x8b), t(0x8a),\ + t(0x70), t(0x3e), t(0xb5), t(0x66), t(0x48), t(0x03), t(0xf6), t(0x0e),\ + t(0x61), t(0x35), t(0x57), t(0xb9), t(0x86), t(0xc1), t(0x1d), t(0x9e),\ + t(0xe1), t(0xf8), t(0x98), t(0x11), t(0x69), t(0xd9), t(0x8e), t(0x94),\ + t(0x9b), t(0x1e), t(0x87), t(0xe9), t(0xce), t(0x55), t(0x28), t(0xdf),\ + t(0x8c), t(0xa1), t(0x89), t(0x0d), t(0xbf), t(0xe6), t(0x42), t(0x68),\ + t(0x41), t(0x99), t(0x2d), t(0x0f), t(0xb0), t(0x54), t(0xbb), t(0x16) + +#define DEC_SBOX(t) \ + t(0x52), t(0x09), t(0x6a), t(0xd5), t(0x30), t(0x36), t(0xa5), t(0x38),\ + t(0xbf), t(0x40), t(0xa3), t(0x9e), t(0x81), t(0xf3), t(0xd7), t(0xfb),\ + t(0x7c), t(0xe3), t(0x39), t(0x82), t(0x9b), t(0x2f), t(0xff), t(0x87),\ + t(0x34), t(0x8e), t(0x43), t(0x44), t(0xc4), t(0xde), t(0xe9), t(0xcb),\ + t(0x54), t(0x7b), t(0x94), t(0x32), t(0xa6), t(0xc2), t(0x23), t(0x3d),\ + t(0xee), t(0x4c), t(0x95), t(0x0b), t(0x42), t(0xfa), t(0xc3), t(0x4e),\ + t(0x08), t(0x2e), t(0xa1), t(0x66), t(0x28), t(0xd9), t(0x24), t(0xb2),\ + t(0x76), t(0x5b), t(0xa2), t(0x49), t(0x6d), t(0x8b), t(0xd1), t(0x25),\ + t(0x72), t(0xf8), t(0xf6), t(0x64), t(0x86), t(0x68), t(0x98), t(0x16),\ + t(0xd4), t(0xa4), t(0x5c), t(0xcc), t(0x5d), t(0x65), t(0xb6), t(0x92),\ + t(0x6c), t(0x70), t(0x48), t(0x50), t(0xfd), t(0xed), t(0xb9), t(0xda),\ + t(0x5e), t(0x15), t(0x46), t(0x57), t(0xa7), t(0x8d), t(0x9d), t(0x84),\ + t(0x90), t(0xd8), t(0xab), t(0x00), t(0x8c), t(0xbc), t(0xd3), t(0x0a),\ + t(0xf7), t(0xe4), t(0x58), t(0x05), t(0xb8), t(0xb3), t(0x45), t(0x06),\ + t(0xd0), t(0x2c), t(0x1e), t(0x8f), t(0xca), t(0x3f), t(0x0f), t(0x02),\ + t(0xc1), t(0xaf), t(0xbd), t(0x03), t(0x01), t(0x13), t(0x8a), t(0x6b),\ + t(0x3a), t(0x91), t(0x11), t(0x41), t(0x4f), t(0x67), t(0xdc), t(0xea),\ + t(0x97), t(0xf2), t(0xcf), t(0xce), t(0xf0), t(0xb4), t(0xe6), t(0x73),\ + t(0x96), t(0xac), t(0x74), t(0x22), t(0xe7), t(0xad), t(0x35), t(0x85),\ + t(0xe2), t(0xf9), t(0x37), t(0xe8), t(0x1c), t(0x75), t(0xdf), t(0x6e),\ + t(0x47), t(0xf1), t(0x1a), t(0x71), t(0x1d), t(0x29), t(0xc5), t(0x89),\ + t(0x6f), t(0xb7), t(0x62), t(0x0e), t(0xaa), t(0x18), t(0xbe), t(0x1b),\ + t(0xfc), t(0x56), t(0x3e), t(0x4b), t(0xc6), t(0xd2), t(0x79), t(0x20),\ + t(0x9a), t(0xdb), t(0xc0), t(0xfe), t(0x78), t(0xcd), t(0x5a), t(0xf4),\ + t(0x1f), t(0xdd), t(0xa8), t(0x33), t(0x88), t(0x07), t(0xc7), t(0x31),\ + t(0xb1), t(0x12), t(0x10), t(0x59), t(0x27), t(0x80), t(0xec), t(0x5f),\ + t(0x60), t(0x51), t(0x7f), t(0xa9), t(0x19), t(0xb5), t(0x4a), t(0x0d),\ + t(0x2d), t(0xe5), t(0x7a), t(0x9f), t(0x93), t(0xc9), t(0x9c), t(0xef),\ + t(0xa0), t(0xe0), t(0x3b), t(0x4d), t(0xae), t(0x2a), t(0xf5), t(0xb0),\ + t(0xc8), t(0xeb), t(0xbb), t(0x3c), t(0x83), t(0x53), t(0x99), t(0x61),\ + t(0x17), t(0x2b), t(0x04), t(0x7e), t(0xba), t(0x77), t(0xd6), t(0x26),\ + t(0xe1), t(0x69), t(0x14), t(0x63), t(0x55), t(0x21), t(0x0c), t(0x7d), + +/* +// Internal cipher tables +*/ +extern const __ALIGN64 Ipp8u RijEncSbox[256]; /* pure encryption S-box */ +extern const __ALIGN64 Ipp8u RijDecSbox[256]; /* pure decryption S-box */ + +extern const __ALIGN16 Ipp32u RijEncTbl[5][256]; /* precomputed encryption tables */ +extern const __ALIGN16 Ipp32u RijDecTbl[5][256]; /* precomputed decryption tables */ + +#endif /* _PCP_RIJTBLES_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_decrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_decrypt.c new file mode 100644 index 0000000..8eb949c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_decrypt.c @@ -0,0 +1,93 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA Functions +// +// Contents: +// ippsRSA_Decrypt() +// +*/ + +#include "owncp.h" +#include "pcpbn.h" +#include "pcpngrsa.h" +#include "pcpngrsamethod.h" + +/*F* +// Name: ippsRSA_Decrypt +// +// Purpose: Performs RSA Decryprion +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pKey +// NULL == pCtxt +// NULL == pPtxt +// NULL == pBuffer +// +// ippStsContextMatchErr !RSA_PUB_KEY_VALID_ID() +// !BN_VALID_ID(pCtxt) +// !BN_VALID_ID(pPtxt) +// +// ippStsIncompleteContextErr private key is not set up +// +// ippStsOutOfRangeErr pCtxt >= modulus +// pCtxt <0 +// +// ippStsSizeErr BN_ROOM(pPtxt) is not enough +// +// ippStsNoErr no error +// +// Parameters: +// pCtxt pointer to the ciphertext +// pPtxt pointer to the plaintext +// pKey pointer to the key context +// pScratchBuffer pointer to the temporary buffer +*F*/ +IPPFUN(IppStatus, ippsRSA_Decrypt,(const IppsBigNumState* pCtxt, + IppsBigNumState* pPtxt, + const IppsRSAPrivateKeyState* pKey, + Ipp8u* pBuffer)) +{ + IPP_BAD_PTR2_RET(pKey, pBuffer); + IPP_BADARG_RET(!RSA_PRV_KEY_VALID_ID(pKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PRV_KEY_IS_SET(pKey), ippStsIncompleteContextErr); + + IPP_BAD_PTR1_RET(pCtxt); + IPP_BADARG_RET(!BN_VALID_ID(pCtxt), ippStsContextMatchErr); + IPP_BADARG_RET(BN_NEGATIVE(pCtxt), ippStsOutOfRangeErr); + IPP_BADARG_RET(0 <= cpCmp_BNU(BN_NUMBER(pCtxt), BN_SIZE(pCtxt), + MOD_MODULUS(RSA_PRV_KEY_NMONT(pKey)), MOD_LEN(RSA_PRV_KEY_NMONT(pKey))), ippStsOutOfRangeErr); + + IPP_BAD_PTR1_RET(pPtxt); + IPP_BADARG_RET(!BN_VALID_ID(pPtxt), ippStsContextMatchErr); + IPP_BADARG_RET(BN_ROOM(pPtxt) < BITS_BNU_CHUNK(RSA_PRV_KEY_BITSIZE_N(pKey)), ippStsSizeErr); + + { + BNU_CHUNK_T* pScratchBuffer = (BNU_CHUNK_T*)( IPP_ALIGNED_PTR(pBuffer, (int)sizeof(BNU_CHUNK_T)) ); + + if(RSA_PRV_KEY1_VALID_ID(pKey)) + gsRSAprv_cipher(pPtxt, pCtxt, pKey, pScratchBuffer); + else + gsRSAprv_cipher_crt(pPtxt, pCtxt, pKey, pScratchBuffer); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_emsa_pkcs1v15.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_emsa_pkcs1v15.h new file mode 100644 index 0000000..ba92d2c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_emsa_pkcs1v15.h @@ -0,0 +1,59 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSASSA-PKCS-v1_5 +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpngrsa.h" +#include "pcphash.h" +#include "pcptool.h" + +static int EMSA_PKCSv15(const Ipp8u* msgDg, int lenMsgDg, + const Ipp8u* fixPS, int lenFixPS, + Ipp8u* pEM, int lenEM) +{ + /* + // encoded message format: + // EM = 00 || 01 || PS=(FF..FF) || 00 || T + // T = fixPS || msgDg + // len(PS) >= 8 + */ + int tLen = lenFixPS + lenMsgDg; + + if (lenEM >= tLen + 11) { + int psLen = lenEM - 3 - tLen; + + PadBlock(0xFF, pEM, lenEM); + pEM[0] = 0x00; + pEM[1] = 0x01; + pEM[2 + psLen] = 0x00; + CopyBlock(fixPS, pEM + 3 + psLen, lenFixPS); + if (msgDg) { + CopyBlock(msgDg, pEM + 3 + psLen + lenFixPS, lenMsgDg); + } + return 1; + } + else + return 0; /* encoded message length too long */ +} \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_encrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_encrypt.c new file mode 100644 index 0000000..95cfdf9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_encrypt.c @@ -0,0 +1,88 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA Functions +// +// Contents: +// ippsRSA_Encrypt() +// +*/ + +#include "owncp.h" +#include "pcpbn.h" +#include "pcpngrsa.h" +#include "pcpngrsamethod.h" + +/*F* +// Name: ippsRSA_Encrypt +// +// Purpose: Performs RSA Encryprion +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pKey +// NULL == pPtxt +// NULL == pCtxt +// NULL == pBuffer +// +// ippStsContextMatchErr !RSA_PUB_KEY_VALID_ID() +// !BN_VALID_ID(pPtxt) +// !BN_VALID_ID(pCtxt) +// +// ippStsIncompleteContextErr public key is not setup +// +// ippStsOutOfRangeErr pPtxt >= modulus +// pPtxt <0 +// +// ippStsSizeErr BN_ROOM(pCtxt) is not enough +// +// ippStsNoErr no error +// +// Parameters: +// pPtxt pointer to the plaintext +// pCtxt pointer to the ciphertext +// pKey pointer to the key context +// pScratchBuffer pointer to the temporary buffer +*F*/ +IPPFUN(IppStatus, ippsRSA_Encrypt,(const IppsBigNumState* pPtxt, + IppsBigNumState* pCtxt, + const IppsRSAPublicKeyState* pKey, + Ipp8u* pBuffer)) +{ + IPP_BAD_PTR2_RET(pKey, pBuffer); + IPP_BADARG_RET(!RSA_PUB_KEY_VALID_ID(pKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PUB_KEY_IS_SET(pKey), ippStsIncompleteContextErr); + + IPP_BAD_PTR1_RET(pPtxt); + IPP_BADARG_RET(!BN_VALID_ID(pPtxt), ippStsContextMatchErr); + IPP_BADARG_RET(BN_NEGATIVE(pPtxt), ippStsOutOfRangeErr); + IPP_BADARG_RET(0 <= cpCmp_BNU(BN_NUMBER(pPtxt), BN_SIZE(pPtxt), + MOD_MODULUS(RSA_PUB_KEY_NMONT(pKey)), MOD_LEN(RSA_PUB_KEY_NMONT(pKey))), ippStsOutOfRangeErr); + + IPP_BAD_PTR1_RET(pCtxt); + IPP_BADARG_RET(!BN_VALID_ID(pCtxt), ippStsContextMatchErr); + IPP_BADARG_RET(BN_ROOM(pCtxt) < BITS_BNU_CHUNK(RSA_PUB_KEY_BITSIZE_N(pKey)), ippStsSizeErr); + + { + BNU_CHUNK_T* pScratchBuffer = (BNU_CHUNK_T*)(IPP_ALIGNED_PTR(pBuffer, (int)sizeof(BNU_CHUNK_T)) ); + gsRSApub_cipher(pCtxt, pPtxt, pKey, pScratchBuffer); + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_generatekeys.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_generatekeys.c new file mode 100644 index 0000000..a45c786 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_generatekeys.c @@ -0,0 +1,300 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA Functions +// +// Contents: +// ippsRSA_GenerateKeys() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpprimeg.h" +#include "pcpprng.h" +#include "pcpngrsa.h" + +#include "pcpprime_isco.h" +#include "pcpprime_isprob.h" + +/*F* +// Name: ippsRSA_GenerateKeys +// +// Purpose: Generate RSA keys +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pSrcPublicExp +// NULL == pPublicExp +// NULL == pModulus +// NULL == pPrivateKeyType2 +// NULL == pPrimeGen +// NULL == pBuffer +// NULL == rndFunc +// +// ippStsContextMatchErr !RSA_PRV_KEY2_VALID_ID(pPrivateKeyType2) +// !RSA_PRV_KEY1_VALID_ID(pPrivateKeyType1) +// !BN_VALID_ID(pSrcPublicExp) +// !BN_VALID_ID(pPublicExp) +// !BN_VALID_ID(pModulus) +// !PRIME_VALID_ID(pPrimeGen) +// +// ippStsSizeErr BN_ROOM(pPublicExp) < BN_SIZE(pSrcPublicExp) +// BN_ROOM(pModulus) < SIZE(factorPbitSize+factorQbitSize) +// PRIME_MAXBITSIZE(pPrimeGen) < factorPbitSize +// +// ippStsOutOfRangeErr 0 >= pSrcPublicExp +// +// ippStsBadArgErr nTrials < 1 +// +// ippStsNoErr no error +// +// Parameters: +// pSrcPublicExp pointer to the beginning public exponent +// pPublicExp pointer to the resulting public exponent (E) +// pModulus pointer to the resulting modulus (N) +// pPrivateKeyType2 pointer to the private key type2 context +// pPrivateKeyType1 (optional) pointer to the private key type1 context +// pBuffer pointer to the temporary buffer +// nTrials parameter of Miller-Rabin Test +// pPrimeGen pointer to the Prime generator context +// rndFunc external PRNG +// pRndParam pointer to the external PRNG parameters +*F*/ +IPPFUN(IppStatus, ippsRSA_GenerateKeys,(const IppsBigNumState* pSrcPublicExp, + IppsBigNumState* pModulus, + IppsBigNumState* pPublicExp, + IppsBigNumState* pPrivateExp, /* optional */ + IppsRSAPrivateKeyState* pPrivateKeyType2, + Ipp8u* pBuffer, + int nTrials, + IppsPrimeState* pPrimeGen, /* NULL */ + IppBitSupplier rndFunc, void* pRndParam)) +{ + IPP_BAD_PTR1_RET(pSrcPublicExp); + IPP_BADARG_RET(!BN_VALID_ID(pSrcPublicExp), ippStsContextMatchErr); + IPP_BADARG_RET(!(0 < cpBN_tst(pSrcPublicExp)), ippStsOutOfRangeErr); + /* test if e is odd and e>=3 */ + IPP_BADARG_RET(!(BN_NUMBER(pSrcPublicExp)[0] &1), ippStsBadArgErr); + IPP_BADARG_RET((0 > cpBN_cmp(pSrcPublicExp, cpBN_ThreeRef())), ippStsBadArgErr); + + IPP_BAD_PTR1_RET(pModulus); + IPP_BADARG_RET(!BN_VALID_ID(pModulus), ippStsContextMatchErr); + + IPP_BAD_PTR1_RET(pPublicExp); + IPP_BADARG_RET(!BN_VALID_ID(pPublicExp), ippStsContextMatchErr); + IPP_BADARG_RET(BN_ROOM(pPublicExp) ret) break; /* internal error */ + if(0 ==ret) continue; /* composite factor */ + found = 1; + } + if(!found) + goto err; /* internal error or ippStsInsufficientEntropy */ + + /* + // generate prime Q + */ + topPattern = (BNU_CHUNK_T)1 << ((factorQbitSize-1)&(BNU_CHUNK_BITS-1)); + nRounds = 5*factorQbitSize; + + for(r=0,found=0; r=512) { + int bitsize = BITSIZE_BNU(pFreeBuffer, nsP); + if(bitsize < (factorPbitSize-100)) continue; /* abs(P-Q) <=2^(factorPbitSize-100)*/ + } + } + } + + /* test if bitsize(N) = bitsize(P)+bitsize(Q) */ + cpMul_BNU_school(pProdN, pFactorP, nsP, pFactorQ, nsQ); + if(rsaModulusBitSize != BITSIZE_BNU(pProdN, nsN)) continue; + + /* chek if E and (Q-1) co-prime */ + cpDec_BNU(pFactorQ, pFactorQ, nsQ, 1); + if(0 == cpIsCoPrime(BN_NUMBER(pPublicExp), BN_SIZE(pPublicExp), pFactorQ, nsQ, pFreeBuffer)) continue; + + /* test Q for primality */ + cpInc_BNU(pFactorQ, pFactorQ, nsQ, 1); + gsModEngineInit(pMontQ, (Ipp32u*)pFactorQ, factorQbitSize, MOD_ENGINE_RSA_POOL_SIZE, gsModArithRSA()); + //if(0==cpIsProbablyPrime(pFactorQ, factorQbitSize, mrTrials, + // rndFunc, pRndParam, + // pMontQ, pFreeBuffer)) continue; + //found = 1; + ret = cpIsProbablyPrime(pFactorQ, factorQbitSize, mrTrials, + rndFunc, pRndParam, + pMontQ, pFreeBuffer); + if(0 > ret) break; /* internal error */ + if(0 ==ret) continue; /* composite factor */ + found = 1; + } + if(!found) + goto err; /* internal error or ippStsInsufficientEntropy */ + + { + BNU_CHUNK_T* pExpD = pFreeBuffer; + BNU_CHUNK_T* pExpDBuf = pExpD +nsN+1; + BNU_CHUNK_T* pPhi = pExpDBuf+nsN+1; + BNU_CHUNK_T* pPhiBuf = pPhi +nsN+1; + int nsD, ns; + + /* phi = (P-1) * (Q-1) */ + cpDec_BNU(pFactorP, pFactorP, nsP, 1); + cpDec_BNU(pFactorQ, pFactorQ, nsQ, 1); + cpMul_BNU_school(pPhi, pFactorP, nsP, pFactorQ, nsQ); + + /* D = 1/E mod (phi) */ + nsD = cpModInv_BNU(pExpD, BN_NUMBER(pPublicExp),BN_SIZE(pPublicExp), pPhi,nsN, pExpDBuf, BN_BUFFER(pPublicExp), pPhiBuf); + /* if D exp requested */ + if(pPrivateExp) + BN_Set(pExpD, nsD, pPrivateExp); + + /* compute dP = D mod(P-1) */ + COPY_BNU(pExpDBuf, pExpD, nsD); + ns = cpMod_BNU(pExpDBuf, nsD, pFactorP, nsP); + ZEXPAND_COPY_BNU(pExpDp, nsP, pExpDBuf, ns); + /* compute dQ = D mod(Q-1) */ + COPY_BNU(pPhi, pExpD, nsD); + ns = cpMod_BNU(pPhi, nsD, pFactorQ, nsQ); + ZEXPAND_COPY_BNU(pExpDq, nsQ, pPhi, ns); + + /* restore P and Q */ + pFactorP[0]++; + pFactorQ[0]++; + /* re-init Montgomery Engine */ + gsModEngineInit(pMontP, (Ipp32u*)pFactorP, factorPbitSize, MOD_ENGINE_RSA_POOL_SIZE, gsModArithRSA()); + gsModEngineInit(pMontQ, (Ipp32u*)pFactorQ, factorQbitSize, MOD_ENGINE_RSA_POOL_SIZE, gsModArithRSA()); + + /* compute Qinv = 1/Q mod P */ + COPY_BNU(pPhiBuf, pFactorP,nsP); + ns = cpModInv_BNU(pInvQ, pFactorQ,nsQ, pPhiBuf,nsP, pExpD, pExpDBuf, pPhi); + /* expand invQ */ + ZEXPAND_BNU(pInvQ, ns, nsP); + + cpMul_BNU_school(pProdN, pFactorP, nsP, pFactorQ, nsQ); + gsModEngineInit(pMontN, (Ipp32u*)pProdN, factorPbitSize+factorQbitSize, MOD_ENGINE_RSA_POOL_SIZE, gsModArithRSA()); + /* setup modulus */ + BN_Set(pProdN, nsN, pModulus); + + /* actual size of modulus in bits */ + RSA_PRV_KEY_BITSIZE_N(pPrivateKeyType2) = BITSIZE_BNU(pProdN, nsN); + + ret = 1; + return ippStsNoErr; + } + + err: + ZEXPAND_BNU(pFactorP, 0, nsP); + ZEXPAND_BNU(pFactorQ, 0, nsQ); + ZEXPAND_BNU(pExpDp, 0, nsP); + ZEXPAND_BNU(pExpDq, 0, nsQ); + ZEXPAND_BNU(pInvQ, 0, nsP); + return ret<0? ippStsErr : ippStsInsufficientEntropy; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_generatesign_pkcs1v15.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_generatesign_pkcs1v15.h new file mode 100644 index 0000000..c0fc4d7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_generatesign_pkcs1v15.h @@ -0,0 +1,85 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSASSA-PKCS-v1_5 +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpngrsa.h" +#include "pcphash.h" +#include "pcptool.h" + +#include "pcprsa_emsa_pkcs1v15.h" + +static int GenerateSign(const Ipp8u* pMsg, int msgLen, /* message representation */ + const Ipp8u* pSalt, int saltLen, /* fied string */ + Ipp8u* pSign, + const IppsRSAPrivateKeyState* pPrvKey, + const IppsRSAPublicKeyState* pPubKey, + BNU_CHUNK_T* pBuffer) +{ + /* size of RSA modulus in bytes and chunks */ + cpSize rsaBits = RSA_PRV_KEY_BITSIZE_N(pPrvKey); + cpSize k = BITS2WORD8_SIZE(rsaBits); + cpSize nsN = BITS_BNU_CHUNK(rsaBits); + + /* EMSA-PKCS-v1_5 encoding */ + int result = EMSA_PKCSv15(pMsg, msgLen, pSalt, saltLen, pSign, k); + + if (result) { + /* temporary BNs */ + __ALIGN8 IppsBigNumState bnC; + __ALIGN8 IppsBigNumState bnP; + + /* make BNs */ + BN_Make(pBuffer, pBuffer + nsN + 1, nsN, &bnC); + pBuffer += (nsN + 1) * 2; + BN_Make(pBuffer, pBuffer + nsN + 1, nsN, &bnP); + pBuffer += (nsN + 1) * 2; + + /* + // private-key operation + */ + ippsSetOctString_BN(pSign, k, &bnC); + + if (RSA_PRV_KEY1_VALID_ID(pPrvKey)) + gsRSAprv_cipher(&bnP, &bnC, pPrvKey, pBuffer); + else + gsRSAprv_cipher_crt(&bnP, &bnC, pPrvKey, pBuffer); + + ippsGetOctString_BN(pSign, k, &bnP); + + /* check the result before send it out (fault attack mitigatioin) */ + if (pPubKey) { + gsRSApub_cipher(&bnP, &bnP, pPubKey, pBuffer); + + /* check signature before send it out (fault attack mitigatioin) */ + if (0 != cpBN_cmp(&bnP, &bnC)) { + PadBlock(0, pSign, k); + result = 0; + } + } + } + + return result; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_getbuffersizeprivatekey.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_getbuffersizeprivatekey.c new file mode 100644 index 0000000..26001e9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_getbuffersizeprivatekey.c @@ -0,0 +1,91 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA Functions +// +// Contents: +// ippsRSA_GetBufferSizePrivateKey() +// +*/ + +#include "owncp.h" +#include "pcpbn.h" +#include "pcpngrsa.h" +#include "pcpngrsamethod.h" + +#include "pcprsa_getdefmeth_priv.h" + +/*F* +// Name: ippsRSA_GetBufferSizePrivateKey +// +// Purpose: Returns size of temporary buffer (in bytes) for private key operation +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pKey +// NULL == pBufferSize +// +// ippStsContextMatchErr !RSA_PRV_KEY_VALID_ID() +// +// ippStsIncompleteContextErr (type1) private key is not set up +// +// ippStsNoErr no error +// +// Parameters: +// pBufferSize pointer to size of temporary buffer +// pKey pointer to the key context +*F*/ +IPPFUN(IppStatus, ippsRSA_GetBufferSizePrivateKey,(int* pBufferSize, const IppsRSAPrivateKeyState* pKey)) +{ + IPP_BAD_PTR1_RET(pKey); + IPP_BADARG_RET(!RSA_PRV_KEY_VALID_ID(pKey), ippStsContextMatchErr); + IPP_BADARG_RET(RSA_PRV_KEY1_VALID_ID(pKey) && !RSA_PRV_KEY_IS_SET(pKey), ippStsIncompleteContextErr); + + IPP_BAD_PTR1_RET(pBufferSize); + + { + cpSize modulusBits = (RSA_PRV_KEY1_VALID_ID(pKey))? RSA_PRV_KEY_BITSIZE_N(pKey) : + IPP_MAX(RSA_PRV_KEY_BITSIZE_P(pKey), RSA_PRV_KEY_BITSIZE_Q(pKey)); + gsMethod_RSA* m = getDualExpMethod_RSA_private(RSA_PRV_KEY_BITSIZE_P(pKey), RSA_PRV_KEY_BITSIZE_Q(pKey)); + if (NULL == m) + m = getDefaultMethod_RSA_private(modulusBits); + + cpSize bitSizeN = (RSA_PRV_KEY1_VALID_ID(pKey))? modulusBits : modulusBits*2; + cpSize nsN = BITS_BNU_CHUNK(bitSizeN); + + cpSize bn_scheme = (nsN+1)*2; /* BN for RSA schemes */ + cpSize bn3_gen = (RSA_PRV_KEY2_VALID_ID(pKey))? (nsN+1)*2*3 : 0; /* 3 BN for generation/validation */ + + cpSize bufferNum = bn_scheme*2 /* (1)2 BN for RSA (enc)/sign schemes */ + + 1; /* BNU_CHUNK_T alignment */ + bufferNum += m->bufferNumFunc(modulusBits); /* RSA private key operation */ + + bufferNum = IPP_MAX(bufferNum, bn3_gen); /* generation/validation resource overlaps RSA resource */ + + *pBufferSize = bufferNum*(Ipp32s)sizeof(BNU_CHUNK_T); + + #if defined(_USE_WINDOW_EXP_) + /* pre-computed table should be CACHE_LINE aligned*/ + *pBufferSize += CACHE_LINE_SIZE; + #endif + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_getbuffersizepublickey.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_getbuffersizepublickey.c new file mode 100644 index 0000000..f52a237 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_getbuffersizepublickey.c @@ -0,0 +1,77 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA Functions +// +// Contents: +// ippsRSA_GetBufferSizePublicKey() +// +*/ + +#include "owncp.h" +#include "pcpbn.h" +#include "pcpngrsa.h" +#include "pcpngrsamethod.h" + +#include "pcprsa_getdefmeth_pub.h" + +/*F* +// Name: ippsRSA_GetBufferSizePublicKey +// +// Purpose: Returns size of temporary buffer (in bytes) for public key operation +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pKey +// NULL == pBufferSize +// +// ippStsContextMatchErr !RSA_PUB_KEY_VALID_ID() +// +// ippStsIncompleteContextErr no ippsRSA_SetPublicKey() call +// +// ippStsNoErr no error +// +// Parameters: +// pBufferSize pointer to size of temporary buffer +// pKey pointer to the key context +*F*/ +IPPFUN(IppStatus, ippsRSA_GetBufferSizePublicKey,(int* pBufferSize, const IppsRSAPublicKeyState* pKey)) +{ + IPP_BAD_PTR1_RET(pKey); + IPP_BADARG_RET(!RSA_PUB_KEY_VALID_ID(pKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PUB_KEY_IS_SET(pKey), ippStsIncompleteContextErr); + + IPP_BAD_PTR1_RET(pBufferSize); + + { + cpSize bitSizeN = RSA_PUB_KEY_BITSIZE_N(pKey); + cpSize nsN = BITS_BNU_CHUNK(bitSizeN); + + gsMethod_RSA* m = getDefaultMethod_RSA_public(bitSizeN); + + cpSize bufferNum = ((nsN+1)*2)*2 /* (1)2 BN for RSA (enc)/sign schemes */ + + 1; /* BNU_CHUNK_T alignment */ + bufferNum += m->bufferNumFunc(bitSizeN); /* RSA public key operation */ + + *pBufferSize = bufferNum*(Ipp32s)sizeof(BNU_CHUNK_T); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_getdefmeth_priv.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_getdefmeth_priv.h new file mode 100644 index 0000000..3f927f5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_getdefmeth_priv.h @@ -0,0 +1,67 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA Functions +// +*/ + +#include "owncp.h" +#include "pcpbn.h" +#include "pcpngrsa.h" +#include "pcpngrsamethod.h" + +static gsMethod_RSA* getDefaultMethod_RSA_private(int modulusBitSize) +{ + gsMethod_RSA* m; + +#if(_IPP32E>=_IPP32E_K1) + m = IsFeatureEnabled(ippCPUID_AVX512IFMA) ? gsMethod_RSA_avx512_private() : gsMethod_RSA_avx2_private(); + +#elif(_IPP32E>=_IPP32E_L9) + m = IsFeatureEnabled(ippCPUID_ADCOX) ? gsMethod_RSA_gpr_private() : gsMethod_RSA_avx2_private(); + +#elif(_IPP>=_IPP_W7) + m = gsMethod_RSA_sse2_private(); + +#else + m = gsMethod_RSA_gpr_private(); +#endif + + if (!(m->loModulusBisize <= modulusBitSize && modulusBitSize <= m->hiModulusBisize)) + m = gsMethod_RSA_gpr_private(); + return m; +} + +static gsMethod_RSA* getDualExpMethod_RSA_private(int bitSizeDP, int bitSizeDQ) +{ + /* Dual exp kernels assume same bitsizes of private exponents */ + if ((bitSizeDP != bitSizeDQ) || (bitSizeDP == 0)) + return NULL; + +#if(_IPP32E>=_IPP32E_K1) + gsMethod_RSA* m = NULL; + m = gsMethod_RSA_avx512_crt_private(bitSizeDP); + if (m && m->dualExpFun) + return m; +#endif + + return NULL; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_getdefmeth_pub.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_getdefmeth_pub.h new file mode 100644 index 0000000..85cf09e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_getdefmeth_pub.h @@ -0,0 +1,50 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA Functions +// +*/ + +#include "owncp.h" +#include "pcpbn.h" +#include "pcpngrsa.h" +#include "pcpngrsamethod.h" + + +/* get default method based on CPU's features */ +static gsMethod_RSA* getDefaultMethod_RSA_public(int modulusBitSize) +{ + gsMethod_RSA* m; + +#if(_IPP32E>=_IPP32E_K1) + m = IsFeatureEnabled(ippCPUID_AVX512IFMA) ? gsMethod_RSA_avx512_public() : gsMethod_RSA_avx2_public(); +#elif(_IPP32E>=_IPP32E_L9) + m = gsMethod_RSA_avx2_public(); +#elif(_IPP>=_IPP_W7) + m = gsMethod_RSA_sse2_public(); +#else + m = gsMethod_RSA_gpr_public(); +#endif + + if (!(m->loModulusBisize <= modulusBitSize && modulusBitSize <= m->hiModulusBisize)) + m = gsMethod_RSA_gpr_public(); + return m; +} \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_getprivatekeytype1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_getprivatekeytype1.c new file mode 100644 index 0000000..47d7022 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_getprivatekeytype1.c @@ -0,0 +1,85 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA Functions +// +// Contents: +// ippsRSA_GetPrivateKeyType1() +// +*/ + +#include "owncp.h" +#include "pcpbn.h" +#include "pcpngrsa.h" + +/*F* +// Name: ippsRSA_GetPrivateKeyType1 +// +// Purpose: Extract key component from the key context +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pKey +// +// ippStsContextMatchErr !RSA_PRV_KEY_VALID_ID() +// !BN_VALID_ID(pModulus) +// !BN_VALID_ID(pExp) +// +// ippStsIncompleteContextErr private key is not set up +// +// ippStsSizeErr BN_ROOM(pModulus), BN_ROOM(pExp) is not enough +// +// ippStsNoErr no error +// +// Parameters: +// pModulus (optional) pointer to the modulus (N) +// pExp (optional) pointer to the public exponent (E) +// pKey pointer to the key context +*F*/ +IPPFUN(IppStatus, ippsRSA_GetPrivateKeyType1,(IppsBigNumState* pModulus, + IppsBigNumState* pExp, + const IppsRSAPrivateKeyState* pKey)) +{ + IPP_BAD_PTR1_RET(pKey); + IPP_BADARG_RET(!RSA_PRV_KEY1_VALID_ID(pKey), ippStsContextMatchErr); + + if(pModulus) { + IPP_BADARG_RET(!BN_VALID_ID(pModulus), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PRV_KEY_IS_SET(pKey), ippStsIncompleteContextErr); + IPP_BADARG_RET(BN_ROOM(pModulus) rsaModulusBitSize +// MAX_RSA_SIZE < rsaModulusBitSize +// +// ippStsBadArgErr 0 >= privateExpBitSize +// privateExpBitSize > rsaModulusBitSize +// +// ippStsNoErr no error +// +// Parameters: +// rsaModulusBitSize bitsize of RSA modulus (bitsize of N) +// privateExpBitSize bitsize of private exponent (bitsize of D) +// pSize pointer to the size of RSA key context (bytes) +*F*/ + + +IPPFUN(IppStatus, ippsRSA_GetSizePrivateKeyType1,(int rsaModulusBitSize, int privateExpBitSize, int* pKeySize)) +{ + IPP_BAD_PTR1_RET(pKeySize); + IPP_BADARG_RET((MIN_RSA_SIZE>rsaModulusBitSize) || (rsaModulusBitSize>MAX_RSA_SIZE), ippStsNotSupportedModeErr); + IPP_BADARG_RET(!((0 (factorPbitSize+factorQbitSize) +// MAX_RSA_SIZE < (factorPbitSize+factorQbitSize) +// +// ippStsBadArgErr 0 >= factorPbitSize +// 0 >= factorQbitSize +// factorQbitSize > factorPbitSize +// +// ippStsNoErr no error +// +// Parameters: +// factorPbitSize bitsize of RSA modulus (bitsize of P) +// factorPbitSize bitsize of private exponent (bitsize of Q) +// pSize pointer to the size of RSA key context (bytes) +*F*/ + + +IPPFUN(IppStatus, ippsRSA_GetSizePrivateKeyType2,(int factorPbitSize, int factorQbitSize, int* pKeySize)) +{ + IPP_BAD_PTR1_RET(pKeySize); + IPP_BADARG_RET((factorPbitSize<=0) || (factorQbitSize<=0), ippStsBadArgErr); + //25.09.2019 gres: IPP_BADARG_RET((factorPbitSize < factorQbitSize), ippStsBadArgErr); + IPP_BADARG_RET((MIN_RSA_SIZE>(factorPbitSize+factorQbitSize) || (factorPbitSize+factorQbitSize)>MAX_RSA_SIZE), ippStsNotSupportedModeErr); + + *pKeySize = cpSizeof_RSA_privateKey2(factorPbitSize, factorQbitSize); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_getsizepublickey.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_getsizepublickey.c new file mode 100644 index 0000000..ad0c158 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_getsizepublickey.c @@ -0,0 +1,65 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA Functions +// +// Contents: +// ippsRSA_GetSizePublicKey() +// +*/ + +#include "owncp.h" +#include "pcpbn.h" +#include "pcpngrsa.h" + +#include "pcprsa_sizeof_pubkey.h" + + +/*F* +// Name: ippsRSA_GetSizePublicKey +// +// Purpose: Returns context size (bytes) of RSA public key context +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pSize +// +// ippStsNotSupportedModeErr MIN_RSA_SIZE > rsaModulusBitSize +// MAX_RSA_SIZE < rsaModulusBitSize +// +// ippStsBadArgErr 0 >= publicExpBitSize +// publicExpBitSize > rsaModulusBitSize +// +// ippStsNoErr no error +// +// Parameters: +// rsaModulusBitSize bitsize of RSA modulus (bitsize of N) +// publicExpBitSize bitsize of public exponent (bitsize of E) +// pSize pointer to the size of RSA key context (bytes) +*F*/ +IPPFUN(IppStatus, ippsRSA_GetSizePublicKey,(int rsaModulusBitSize, int publicExpBitSize, int* pKeySize)) +{ + IPP_BAD_PTR1_RET(pKeySize); + IPP_BADARG_RET((MIN_RSA_SIZE>rsaModulusBitSize) || (rsaModulusBitSize>MAX_RSA_SIZE), ippStsNotSupportedModeErr); + IPP_BADARG_RET(!((0=_IPP32E_L9) +#include "pcpngmontexpstuff_avx2.h" + +IPP_OWN_DEFN (gsMethod_RSA*, gsMethod_RSA_avx2_private, (void)) +{ + static gsMethod_RSA m = { + RSA_AVX2_MIN_BITSIZE, RSA_AVX2_MAX_BITSIZE, /* RSA range */ + + /* private key exponentiation: private, window, avx2 */ + #if !defined(_USE_WINDOW_EXP_) + gsMontExpBinBuffer_avx2, + gsMontExpBin_BNU_sscm_avx2 + #else + gsMontExpWinBuffer_avx2, + gsMontExpWin_BNU_sscm_avx2 + #endif + , NULL + }; + return &m; +} +#endif /* _IPP32E_L9 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsmethod__avx2_public.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsmethod__avx2_public.c new file mode 100644 index 0000000..9a30735 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsmethod__avx2_public.c @@ -0,0 +1,51 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Montgomery engine preparation (GetSize/init/Set) +*/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// gsMethod_RSA_gpr_public() +// +*/ + +#include "owncp.h" +#include "pcpngmontexpstuff.h" +#include "gsscramble.h" +#include "pcpngrsamethod.h" +#include "pcpngrsa.h" + +#if (_IPP32E>=_IPP32E_L9) +#include "pcpngmontexpstuff_avx2.h" + +IPP_OWN_DEFN (gsMethod_RSA*, gsMethod_RSA_avx2_public, (void)) +{ + static gsMethod_RSA m = { + RSA_AVX2_MIN_BITSIZE, RSA_AVX2_MAX_BITSIZE, /* RSA range */ + + /* public key exponentiation: public, binary gpr */ + gsMontExpBinBuffer, + gsModExpBin_BNU, + NULL + }; + return &m; +} +#endif /* _IPP32E_L9 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsmethod__avx512_private.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsmethod__avx512_private.c new file mode 100644 index 0000000..b257c09 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsmethod__avx512_private.c @@ -0,0 +1,85 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Montgomery engine preparation (GetSize/init/Set) +*/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// gsMethod_RSA_gpr_private() +// +*/ + +#include "owncp.h" +#include "pcpngmontexpstuff.h" +#include "gsscramble.h" +#include "pcpngrsamethod.h" +#include "pcpngrsa.h" + +#if (_IPP32E>=_IPP32E_K1) +#include "pcpngmontexpstuff_avx512.h" + +IPP_OWN_DEFN (gsMethod_RSA*, gsMethod_RSA_avx512_private, (void)) +{ + static gsMethod_RSA m = { + RSA_AVX512_MIN_BITSIZE, RSA_AVX512_MAX_BITSIZE, /* RSA range */ + + /* private key exponentiation: private, window, avx512 */ + #if !defined(_USE_WINDOW_EXP_) + gsMontExpBinBuffer_avx512, + gsMontExpBin_BNU_sscm_avx512 + #else + gsMontExpWinBuffer_avx512, + gsMontExpWin_BNU_sscm_avx512 + #endif + , NULL + }; + return &m; +} + +#define RSA_DUAL_EXP_AVX512_MIN_BITSIZE 2048 +#define RSA_DUAL_EXP_AVX512_MAX_BITSIZE 4096 + +IPP_OWN_DEFN (gsMethod_RSA*, gsMethod_RSA_avx512_crt_private, (int privExpBitSize)) { + static gsMethod_RSA m = { + RSA_DUAL_EXP_AVX512_MIN_BITSIZE, RSA_DUAL_EXP_AVX512_MAX_BITSIZE, /* RSA range */ + gsMontDualExpWinBuffer_avx512, + NULL, + NULL + }; + + if (IsFeatureEnabled(ippCPUID_AVX512IFMA)) { + ngMontDualExp dexpFunc = NULL; + switch (privExpBitSize) { + /* RSA 2k,3k,4k only supported */ + case 1024: + case 1536: + case 2048: + dexpFunc = gsMontDualExpWin_BNU_sscm_avx512; + break; + default: + dexpFunc = NULL; + } + m.dualExpFun = dexpFunc; + } + + return &m; +} +#endif /* _IPP32E_K1 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsmethod__avx512_public.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsmethod__avx512_public.c new file mode 100644 index 0000000..6106d3f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsmethod__avx512_public.c @@ -0,0 +1,51 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Montgomery engine preparation (GetSize/init/Set) +*/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// gsMethod_RSA_gpr_public() +// +*/ + +#include "owncp.h" +#include "pcpngmontexpstuff.h" +#include "gsscramble.h" +#include "pcpngrsamethod.h" +#include "pcpngrsa.h" + +#if (_IPP32E>=_IPP32E_K1) +#include "pcpngmontexpstuff_avx512.h" + +IPP_OWN_DEFN (gsMethod_RSA*, gsMethod_RSA_avx512_public, (void)) +{ + static gsMethod_RSA m = { + RSA_AVX512_MIN_BITSIZE, RSA_AVX512_MAX_BITSIZE, /* RSA range */ + + /* public key exponentiation: public, binary, avx512 */ + gsMontExpBinBuffer_avx512, + gsMontExpBin_BNU_avx512, + NULL + }; + return &m; +} +#endif /* _IPP32E_L9 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsmethod__gpr_private.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsmethod__gpr_private.c new file mode 100644 index 0000000..dd93e7e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsmethod__gpr_private.c @@ -0,0 +1,55 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Montgomery engine preparation (GetSize/init/Set) +*/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// gsMethod_RSA_gpr_private() +// +*/ + +#include "owncp.h" +#include "pcpngmontexpstuff.h" +#include "gsscramble.h" +#include "pcpngrsamethod.h" +#include "pcpngrsa.h" + +/* +// definition of RSA exponentiation (PX/GPR based) +*/ + +IPP_OWN_DEFN (gsMethod_RSA*, gsMethod_RSA_gpr_private, (void)) +{ + static gsMethod_RSA m = { + MIN_RSA_SIZE, MAX_RSA_SIZE, /* RSA range */ + + /* private key exponentiation: private, window, gpr */ + gsMontExpWinBuffer, + #if !defined(_USE_WINDOW_EXP_) + gsModExpBin_BNU_sscm + #else + gsModExpWin_BNU_sscm + #endif + , NULL + }; + return &m; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsmethod__gpr_public.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsmethod__gpr_public.c new file mode 100644 index 0000000..bed08e6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsmethod__gpr_public.c @@ -0,0 +1,47 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Montgomery engine preparation (GetSize/init/Set) +*/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// gsMethod_RSA_gpr_public() +// +*/ + +#include "owncp.h" +#include "pcpngmontexpstuff.h" +#include "gsscramble.h" +#include "pcpngrsamethod.h" +#include "pcpngrsa.h" + +IPP_OWN_DEFN (gsMethod_RSA*, gsMethod_RSA_gpr_public, (void)) +{ + static gsMethod_RSA m = { + MIN_RSA_SIZE, MAX_RSA_SIZE, /* RSA range */ + + /* public key exponentiation: public, binary, gpr */ + gsMontExpBinBuffer, + gsModExpBin_BNU, + NULL + }; + return &m; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsmethod__sse2_private.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsmethod__sse2_private.c new file mode 100644 index 0000000..f07b9cb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsmethod__sse2_private.c @@ -0,0 +1,57 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Montgomery engine preparation (GetSize/init/Set) +*/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// gsMethod_RSA_sse2_private() +// +*/ + +#include "owncp.h" +#include "pcpngmontexpstuff.h" +#include "gsscramble.h" +#include "pcpngrsamethod.h" +#include "pcpngrsa.h" + + +#if (_IPP>=_IPP_W7) +#include "pcpngmontexpstuff_sse2.h" + +IPP_OWN_DEFN (gsMethod_RSA*, gsMethod_RSA_sse2_private, (void)) +{ + static gsMethod_RSA m = { + RSA_SSE2_MIN_BITSIZE, RSA_SSE2_MAX_BITSIZE, /* RSA range */ + + /* private key exponentiation: private, window, sse2 */ + #if !defined(_USE_WINDOW_EXP_) + gsMontExpBinBuffer_sse2, + gsMontExpBin_BNU_sscm_sse2 + #else + gsMontExpWinBuffer_sse2, + gsMontExpWin_BNU_sscm_sse2 + #endif + , NULL + }; + return &m; +} +#endif /* _IPP_W7 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsmethod__sse2_public.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsmethod__sse2_public.c new file mode 100644 index 0000000..e8deda8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsmethod__sse2_public.c @@ -0,0 +1,52 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Montgomery engine preparation (GetSize/init/Set) +*/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// gsMethod_RSA_sse2_public() +// +*/ + +#include "owncp.h" +#include "pcpngmontexpstuff.h" +#include "gsscramble.h" +#include "pcpngrsamethod.h" +#include "pcpngrsa.h" + +#if (_IPP>=_IPP_W7) + +#include "pcpngmontexpstuff_sse2.h" + +IPP_OWN_DEFN (gsMethod_RSA*, gsMethod_RSA_sse2_public, (void)) +{ + static gsMethod_RSA m = { + RSA_SSE2_MIN_BITSIZE, RSA_SSE2_MAX_BITSIZE, /* RSA range */ + + /* public key exponentiation: public, binary, sse2 */ + gsMontExpBinBuffer_sse2, + gsMontExpBin_BNU_sse2, + NULL + }; + return &m; +} +#endif /* _IPP_W7 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsprv_cipher.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsprv_cipher.c new file mode 100644 index 0000000..5aef08e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsprv_cipher.c @@ -0,0 +1,49 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA Functions +// +// Contents: +// gsRSAprv_cipher() +// +*/ + +#include "owncp.h" +#include "pcpbn.h" +#include "pcpngrsa.h" +#include "pcpngrsamethod.h" + +#include "pcprsa_getdefmeth_priv.h" + +IPP_OWN_DEFN (void, gsRSAprv_cipher, (IppsBigNumState* pY, const IppsBigNumState* pX, const IppsRSAPrivateKeyState* pKey, BNU_CHUNK_T* pBuffer)) +{ + gsMethod_RSA* m = getDefaultMethod_RSA_private(RSA_PRV_KEY_BITSIZE_N(pKey)); + + BNU_CHUNK_T* dataY = BN_NUMBER(pY); + cpSize nsY = m->expFun(dataY, + BN_NUMBER(pX), BN_SIZE(pX), + RSA_PRV_KEY_D(pKey), RSA_PRV_KEY_BITSIZE_N(pKey), + RSA_PRV_KEY_NMONT(pKey), + pBuffer); + FIX_BNU(dataY, nsY); + BN_SIZE(pY) = nsY; + BN_SIGN(pY) = ippBigNumPOS; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsprv_cipher_crt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsprv_cipher_crt.c new file mode 100644 index 0000000..1f75cc7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gsprv_cipher_crt.c @@ -0,0 +1,159 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA Functions +// +// Contents: +// gsRSAprv_cipher_crt() +// +*/ + +#include "owncp.h" +#include "pcpbn.h" +#include "pcpngrsa.h" +#include "pcpngrsamethod.h" + +#include "pcprsa_getdefmeth_priv.h" + +/* CTE version of CRT based RSA decrypt */ +IPP_OWN_DEFN (void, gsRSAprv_cipher_crt, (IppsBigNumState* pY, const IppsBigNumState* pX, const IppsRSAPrivateKeyState* pKey, BNU_CHUNK_T* pBuffer)) +{ + const BNU_CHUNK_T* dataX = BN_NUMBER(pX); + cpSize nsX = BN_SIZE(pX); + + BNU_CHUNK_T* dataY = BN_NUMBER(pY); + + BNU_CHUNK_T* dataXp = BN_NUMBER(pY); + BNU_CHUNK_T* dataXq = BN_BUFFER(pY); + + /* P- and Q- montgometry engines */ + gsModEngine* pMontP = RSA_PRV_KEY_PMONT(pKey); + gsModEngine* pMontQ = RSA_PRV_KEY_QMONT(pKey); + cpSize nsP = MOD_LEN(pMontP); + cpSize nsQ = MOD_LEN(pMontQ); + cpSize bitSizeP = RSA_PRV_KEY_BITSIZE_P(pKey); + cpSize bitSizeQ = RSA_PRV_KEY_BITSIZE_Q(pKey); + cpSize bitSizeDP = bitSizeP; //BITSIZE_BNU(RSA_PRV_KEY_DP(pKey), nsP); /* bitsize of dP exp */ + cpSize bitSizeDQ = bitSizeQ; //BITSIZE_BNU(RSA_PRV_KEY_DQ(pKey), nsQ); /* bitsize of dQ exp */ + + /* Prefer dual exponentiation method if available */ + gsMethod_RSA* m = getDualExpMethod_RSA_private(bitSizeDP, bitSizeDQ); + if (m) { + ZEXPAND_COPY_BNU(pBuffer, nsQ+nsQ, dataX, nsX); + MOD_METHOD(pMontQ)->red(dataXq, pBuffer, pMontQ); + MOD_METHOD(pMontQ)->mul(dataXq, dataXq, MOD_MNT_R2(pMontQ), pMontQ); + + ZEXPAND_COPY_BNU(pBuffer, nsP+nsP, dataX, nsX); + MOD_METHOD(pMontP)->red(dataXp, pBuffer, pMontP); + MOD_METHOD(pMontP)->mul(dataXp, dataXp, MOD_MNT_R2(pMontP), pMontP); + + BNU_CHUNK_T* pDataX[2] = {0}; + pDataX[0] = dataXq; + pDataX[1] = dataXp; + + cpSize pSize[2] = {0}; + pSize[0] = nsQ; + pSize[1] = nsP; + + BNU_CHUNK_T* pPrvExp[2] = {0}; + pPrvExp[0] = RSA_PRV_KEY_DQ(pKey); + pPrvExp[1] = RSA_PRV_KEY_DP(pKey); + + gsModEngine* pMont[2] = {0}; + pMont[0] = pMontQ; + pMont[1] = pMontP; + + m->dualExpFun(pDataX, (const BNU_CHUNK_T**)pDataX, pSize, (const BNU_CHUNK_T**)pPrvExp, pMont, pBuffer); + } else { + /* compute xq = x^dQ mod Q */ + if (bitSizeP== bitSizeQ) { /* believe it's enough conditions for correct Mont application */ + ZEXPAND_COPY_BNU(pBuffer, nsQ+nsQ, dataX, nsX); + MOD_METHOD(pMontQ)->red(dataXq, pBuffer, pMontQ); + MOD_METHOD(pMontQ)->mul(dataXq, dataXq, MOD_MNT_R2(pMontQ), pMontQ); + } + else { + COPY_BNU(dataXq, dataX, nsX); + cpMod_BNU(dataXq, nsX, MOD_MODULUS(pMontQ), nsQ); + } + + m = getDefaultMethod_RSA_private(bitSizeDQ); + m->expFun(dataXq, dataXq, nsQ, RSA_PRV_KEY_DQ(pKey), bitSizeDQ, pMontQ, pBuffer); + + /* compute xp = x^dP mod P */ + if (bitSizeP== bitSizeQ) { /* believe it's enough conditions for correct Mont application */ + ZEXPAND_COPY_BNU(pBuffer, nsP+nsP, dataX, nsX); + MOD_METHOD(pMontP)->red(dataXp, pBuffer, pMontP); + MOD_METHOD(pMontP)->mul(dataXp, dataXp, MOD_MNT_R2(pMontP), pMontP); + } + else { + COPY_BNU(dataXp, dataX, nsX); + cpMod_BNU(dataXp, nsX, MOD_MODULUS(pMontP), nsP); + } + + m = getDefaultMethod_RSA_private(bitSizeDP); + m->expFun(dataXp, dataXp, nsP, RSA_PRV_KEY_DP(pKey), bitSizeDP, pMontP, pBuffer); + } + + /* + // recombination + */ + /* xq = xq mod P + must be sure that xq in the same residue domain as xp + because of following (xp-xq) mod P operation + */ + if (bitSizeP == bitSizeQ) { /* believe it's enough conditions for correct Mont application */ + ZEXPAND_COPY_BNU(pBuffer, nsP+nsP, dataXq, nsQ); + //MOD_METHOD(pMontP)->red(pBuffer, pBuffer, pMontP); + //MOD_METHOD(pMontP)->mul(pBuffer, pBuffer, MOD_MNT_R2(pMontP), pMontP); + MOD_METHOD(pMontP)->sub(pBuffer, pBuffer, MOD_MODULUS(pMontP), pMontP); + /* xp = (xp - xq) mod P */ + MOD_METHOD(pMontP)->sub(dataXp, dataXp, pBuffer, pMontP); + } + else { + COPY_BNU(pBuffer, dataXq, nsQ); + { + cpSize nsQP = cpMod_BNU(pBuffer, nsQ, MOD_MODULUS(pMontP), nsP); + BNU_CHUNK_T cf = cpSub_BNU(dataXp, dataXp, pBuffer, nsQP); + if(nsP-nsQP) + cf = cpDec_BNU(dataXp+nsQP, dataXp + nsQP, (nsP-nsQP), cf); + if (cf) + cpAdd_BNU(dataXp, dataXp, MOD_MODULUS(pMontP), nsP); + } + } + + /* xp = xp*qInv mod P */ + /* convert invQ into Montgomery domain */ + MOD_METHOD(pMontP)->encode(pBuffer, RSA_PRV_KEY_INVQ(pKey), (gsModEngine*)pMontP); + /* and multiply xp *= mont(invQ) mod P */ + MOD_METHOD(pMontP)->mul(dataXp, dataXp, pBuffer, pMontP); + + /* Y = xq + xp*Q */ + cpMul_BNU_school(pBuffer, dataXp, nsP, MOD_MODULUS(pMontQ), nsQ); + { + BNU_CHUNK_T cf = cpAdd_BNU(dataY, pBuffer, dataXq, nsQ); + cpInc_BNU(dataY + nsQ, pBuffer + nsQ, nsP, cf); + } + + nsX = nsP + nsQ; + FIX_BNU(dataY, nsX); + BN_SIZE(pY) = nsX; + BN_SIGN(pY) = ippBigNumPOS; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gspub_cipher.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gspub_cipher.c new file mode 100644 index 0000000..59948ff --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_gspub_cipher.c @@ -0,0 +1,49 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA Functions +// +// Contents: +// gsRSApub_cipher() +// +*/ + +#include "owncp.h" +#include "pcpbn.h" +#include "pcpngrsa.h" +#include "pcpngrsamethod.h" + +#include "pcprsa_getdefmeth_pub.h" + +IPP_OWN_DEFN (void, gsRSApub_cipher, (IppsBigNumState* pY, const IppsBigNumState* pX, const IppsRSAPublicKeyState* pKey, BNU_CHUNK_T* pBuffer)) +{ + gsMethod_RSA* m = getDefaultMethod_RSA_public(RSA_PRV_KEY_BITSIZE_N(pKey)); + + BNU_CHUNK_T* dataY = BN_NUMBER(pY); + cpSize nsY = m->expFun(dataY, + BN_NUMBER(pX), BN_SIZE(pX), + RSA_PUB_KEY_E(pKey), RSA_PUB_KEY_BITSIZE_E(pKey), + RSA_PUB_KEY_NMONT(pKey), + pBuffer); + FIX_BNU(dataY, nsY); + BN_SIZE(pY) = nsY; + BN_SIGN(pY) = ippBigNumPOS; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_initprivatekeytype1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_initprivatekeytype1.c new file mode 100644 index 0000000..c21eb07 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_initprivatekeytype1.c @@ -0,0 +1,110 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA Functions +// +// Contents: +// ippsRSA_InitPrivateKeyType1() +// +*/ + +#include "owncp.h" +#include "pcpbn.h" +#include "pcpngrsa.h" + +#include "pcprsa_sizeof_privkey1.h" + +/*F* +// Name: ippsRSA_InitPrivateKeyType1 +// +// Purpose: Init RSA private key context +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pKey +// +// ippStsNotSupportedModeErr MIN_RSA_SIZE > rsaModulusBitSize +// MAX_RSA_SIZE < rsaModulusBitSize +// +// ippStsBadArgErr 0 >= privateExpBitSize +// privateExpBitSize > rsaModulusBitSize +// +// ippStsMemAllocErr keyCtxSize is not enough for operation +// +// ippStsNoErr no error +// +// Parameters: +// rsaModulusBitSize bitsize of RSA modulus (bitsize of N) +// privateExpBitSize bitsize of private exponent (bitsize of D) +// pKey pointer to the key context +// keyCtxSize size of memmory accosizted with key comtext +*F*/ + +IPPFUN(IppStatus, ippsRSA_InitPrivateKeyType1,(int rsaModulusBitSize, int privateExpBitSize, + IppsRSAPrivateKeyState* pKey, int keyCtxSize)) +{ + IPP_BAD_PTR1_RET(pKey); + + IPP_BADARG_RET((MIN_RSA_SIZE>rsaModulusBitSize) || (rsaModulusBitSize>MAX_RSA_SIZE), ippStsNotSupportedModeErr); + IPP_BADARG_RET(!((0 (factorPbitSize+factorQbitSize) +// MAX_RSA_SIZE < (factorPbitSize+factorQbitSize) +// +// ippStsBadArgErr 0 >= factorPbitSize +// 0 >= factorQbitSize +// factorQbitSize > factorPbitSize +// +// ippStsMemAllocErr keyCtxSize is not enough for operation +// +// ippStsNoErr no error +// +// Parameters: +// factorPbitSize bitsize of RSA modulus (bitsize of P) +// factorQbitSize bitsize of private exponent (bitsize of Q) +// pKey pointer to the key context +// keyCtxSize size of memmory accosizted with key comtext +*F*/ + +IPPFUN(IppStatus, ippsRSA_InitPrivateKeyType2,(int factorPbitSize, int factorQbitSize, + IppsRSAPrivateKeyState* pKey, int keyCtxSize)) +{ + IPP_BAD_PTR1_RET(pKey); + IPP_BADARG_RET((factorPbitSize<=0) || (factorQbitSize<=0), ippStsBadArgErr); + //25.09.2019 gres: IPP_BADARG_RET((factorPbitSize < factorQbitSize), ippStsBadArgErr); + IPP_BADARG_RET((MIN_RSA_SIZE>(factorPbitSize+factorQbitSize) || (factorPbitSize+factorQbitSize)>MAX_RSA_SIZE), ippStsNotSupportedModeErr); + + /* test available size of context buffer */ + IPP_BADARG_RET(keyCtxSize rsaModulusBitSize +// MAX_RSA_SIZE < rsaModulusBitSize +// +// ippStsBadArgErr 0 >= publicExpBitSize +// publicExpBitSize > rsaModulusBitSize +// +// ippStsMemAllocErr keyCtxSize is not enough for operation +// +// ippStsNoErr no error +// +// Parameters: +// rsaModulusBitSize bitsize of RSA modulus (bitsize of N) +// publicExpBitSize bitsize of public exponent (bitsize of E) +// pKey pointer to the key context +// keyCtxSize size of memmory accosizted with key comtext +*F*/ +IPPFUN(IppStatus, ippsRSA_InitPublicKey,(int rsaModulusBitSize, int publicExpBitSize, + IppsRSAPublicKeyState* pKey, int keyCtxSize)) +{ + IPP_BAD_PTR1_RET(pKey); + + IPP_BADARG_RET((MIN_RSA_SIZE>rsaModulusBitSize) || (rsaModulusBitSize>MAX_RSA_SIZE), ippStsNotSupportedModeErr); + IPP_BADARG_RET(!((0hashAlgId, ippStsNotSupportedModeErr); + IPP_BADARG_RET(ippHashAlg_SM3 == pMethod->hashAlgId, ippStsNotSupportedModeErr); + + /* use aligned public key context if defined */ + if (*pPubKey) { + IPP_BADARG_RET(!RSA_PUB_KEY_VALID_ID(*pPubKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PUB_KEY_IS_SET(*pPubKey), ippStsIncompleteContextErr); + } + + /* test data pointer */ + IPP_BAD_PTR2_RET(pMsg, pSign); + /* test length */ + IPP_BADARG_RET(msgLen < 0, ippStsLengthErr); + + return ippStsNoErr; +} + +// Check all the ippsRSAVerify_PKCS1v15_rmf parameters, set valid=0, align pKey pointer +__INLINE IppStatus SingleVerifyPkcs1v15RmfPreproc(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pSign, int* pIsValid, + const IppsRSAPublicKeyState** pKey, + const IppsHashMethod* pMethod, + Ipp8u* pScratchBuffer) +{ + /* test public key context */ + IPP_BAD_PTR3_RET(*pKey, pScratchBuffer, pMethod); + IPP_BADARG_RET(!RSA_PUB_KEY_VALID_ID(*pKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PUB_KEY_IS_SET(*pKey), ippStsIncompleteContextErr); + + /* test hash algorithm ID */ + IPP_BADARG_RET(ippHashAlg_Unknown == pMethod->hashAlgId, ippStsNotSupportedModeErr); + IPP_BADARG_RET(ippHashAlg_SM3 == pMethod->hashAlgId, ippStsNotSupportedModeErr); + + /* test data pointer */ + IPP_BAD_PTR3_RET(pMsg, pSign, pIsValid); + /* test length */ + IPP_BADARG_RET(msgLen < 0, ippStsLengthErr); + + *pIsValid = 0; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_pss_preproc.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_pss_preproc.h new file mode 100644 index 0000000..cb5d4c0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_pss_preproc.h @@ -0,0 +1,80 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +#include "owncp.h" +#include "pcphash_rmf.h" + +// Check all the ippsRSASign_PSS_rmf parameters and align pPrvKey, pPubKey pointers +__INLINE IppStatus SingleSignPssRmfPreproc(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pSalt, int saltLen, + Ipp8u* pSign, + const IppsRSAPrivateKeyState** pPrvKey, + const IppsRSAPublicKeyState** pPubKey, + const IppsHashMethod* pMethod, + Ipp8u* pScratchBuffer) +{ + /* test message length */ + IPP_BADARG_RET((msgLen < 0), ippStsLengthErr); + /* test message pointer */ + IPP_BADARG_RET((msgLen && !pMsg), ippStsNullPtrErr); + + /* test data pointer */ + IPP_BAD_PTR2_RET(pSign, pMethod); + + /* test salt length and salt pointer */ + IPP_BADARG_RET(saltLen < 0, ippStsLengthErr); + IPP_BADARG_RET((saltLen && !pSalt), ippStsNullPtrErr); + + /* test private key context */ + IPP_BAD_PTR2_RET(*pPrvKey, pScratchBuffer); + IPP_BADARG_RET(!RSA_PRV_KEY_VALID_ID(*pPrvKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PRV_KEY_IS_SET(*pPrvKey), ippStsIncompleteContextErr); + + /* use aligned public key context if defined */ + if (*pPubKey) { + IPP_BADARG_RET(!RSA_PUB_KEY_VALID_ID(*pPubKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PUB_KEY_IS_SET(*pPubKey), ippStsIncompleteContextErr); + } + + return ippStsNoErr; +} + +// Check all the ippsRSAVerify_PSS_rmf parameters, set valid=0, align pKey pointer +__INLINE IppStatus SingleVerifyPssRmfPreproc(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pSign, + int* pIsValid, + const IppsRSAPublicKeyState** pKey, + const IppsHashMethod* pMethod, + Ipp8u* pScratchBuffer) +{ + /* test message length */ + IPP_BADARG_RET((msgLen < 0), ippStsLengthErr); + /* test message pointer */ + IPP_BADARG_RET((msgLen && !pMsg), ippStsNullPtrErr); + + /* test data pointer */ + IPP_BAD_PTR3_RET(pSign, pIsValid, pMethod); + + /* test public key context */ + IPP_BAD_PTR2_RET(*pKey, pScratchBuffer); + IPP_BADARG_RET(!RSA_PUB_KEY_VALID_ID(*pKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PUB_KEY_IS_SET(*pKey), ippStsIncompleteContextErr); + + *pIsValid = 0; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_setprivatekeytype1.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_setprivatekeytype1.c new file mode 100644 index 0000000..fa374e4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_setprivatekeytype1.c @@ -0,0 +1,89 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA Functions +// +// Contents: +// ippsRSA_SetPrivateKeyType1() +// +*/ + +#include "owncp.h" +#include "pcpbn.h" +#include "pcpngrsa.h" + +/*F* +// Name: ippsRSA_SetPrivateKeyType1 +// +// Purpose: Set up the RSA private key +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pModulus +// NULL == pPrivateExp +// NULL == pKey +// +// ippStsContextMatchErr !BN_VALID_ID(pModulus) +// !BN_VALID_ID(pPrivateExp) +// !RSA_PRV_KEY_VALID_ID() +// +// ippStsOutOfRangeErr 0 >= pModulus +// 0 >= pPrivateExp +// +// ippStsSizeErr bitsize(pModulus) exceeds requested value +// bitsize(pPrivateExp) exceeds requested value +// +// ippStsNoErr no error +// +// Parameters: +// pModulus pointer to modulus (N) +// pPrivateExp pointer to public exponent (D) +// pKey pointer to the key context +*F*/ +IPPFUN(IppStatus, ippsRSA_SetPrivateKeyType1,(const IppsBigNumState* pModulus, + const IppsBigNumState* pPrivateExp, + IppsRSAPrivateKeyState* pKey)) +{ + IPP_BAD_PTR1_RET(pKey); + IPP_BADARG_RET(!RSA_PRV_KEY1_VALID_ID(pKey), ippStsContextMatchErr); + + IPP_BAD_PTR1_RET(pModulus); + IPP_BADARG_RET(!BN_VALID_ID(pModulus), ippStsContextMatchErr); + IPP_BADARG_RET(!(0 < cpBN_tst(pModulus)), ippStsOutOfRangeErr); + IPP_BADARG_RET(BITSIZE_BNU(BN_NUMBER(pModulus), BN_SIZE(pModulus)) > RSA_PRV_KEY_MAXSIZE_N(pKey), ippStsSizeErr); + + IPP_BAD_PTR1_RET(pPrivateExp); + IPP_BADARG_RET(!BN_VALID_ID(pPrivateExp), ippStsContextMatchErr); + IPP_BADARG_RET(!(0 < cpBN_tst(pPrivateExp)), ippStsOutOfRangeErr); + IPP_BADARG_RET(BITSIZE_BNU(BN_NUMBER(pPrivateExp), BN_SIZE(pPrivateExp)) > RSA_PRV_KEY_MAXSIZE_D(pKey), ippStsSizeErr); + + { + /* store D */ + ZEXPAND_COPY_BNU(RSA_PRV_KEY_D(pKey), BITS_BNU_CHUNK(RSA_PRV_KEY_MAXSIZE_D(pKey)), BN_NUMBER(pPrivateExp), BN_SIZE(pPrivateExp)); + + /* setup montgomery engine */ + gsModEngineInit(RSA_PRV_KEY_NMONT(pKey), (Ipp32u*)BN_NUMBER(pModulus), cpBN_bitsize(pModulus), MOD_ENGINE_RSA_POOL_SIZE, gsModArithRSA()); + + RSA_PRV_KEY_BITSIZE_N(pKey) = cpBN_bitsize(pModulus); + RSA_PRV_KEY_BITSIZE_D(pKey) = cpBN_bitsize(pPrivateExp); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_setprivatekeytype2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_setprivatekeytype2.c new file mode 100644 index 0000000..04bcc83 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_setprivatekeytype2.c @@ -0,0 +1,142 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA Functions +// +// Contents: +// ippsRSA_SetPrivateKeyType2() +// +*/ + +#include "owncp.h" +#include "pcpbn.h" +#include "pcpngrsa.h" + +/*F* +// Name: ippsRSA_SetPrivateKeyType2 +// +// Purpose: Set up the RSA private key +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pFactorP, NULL == pFactorQ +// NULL == pCrtExpP, NULL == pCrtExpQ +// NULL == pInverseQ +// NULL == pKey +// +// ippStsContextMatchErr !BN_VALID_ID(pFactorP), !BN_VALID_ID(pFactorQ) +// !BN_VALID_ID(pCrtExpP), !BN_VALID_ID(pCrtExpQ) +// !BN_VALID_ID(pInverseQ) +// !RSA_PRV_KEY_VALID_ID() +// +// ippStsOutOfRangeErr 0 >= pFactorP, 0 >= pFactorQ +// 0 >= pCrtExpP, 0 >= pCrtExpQ +// 0 >= pInverseQ +// +// ippStsSizeErr bitsize(pFactorP) exceeds requested value +// bitsize(pFactorQ) exceeds requested value +// bitsize(pCrtExpP) > bitsize(pFactorP) +// bitsize(pCrtExpQ) > bitsize(pFactorQ) +// bitsize(pInverseQ) > bitsize(pFactorP) +// +// ippStsNoErr no error +// +// Parameters: +// pFactorP, pFactorQ pointer to the RSA modulus (N) prime factors +// pCrtExpP, pCrtExpQ pointer to CTR's exponent +// pInverseQ 1/Q mod P +// pKey pointer to the key context +*F*/ +IPPFUN(IppStatus, ippsRSA_SetPrivateKeyType2,(const IppsBigNumState* pFactorP, + const IppsBigNumState* pFactorQ, + const IppsBigNumState* pCrtExpP, + const IppsBigNumState* pCrtExpQ, + const IppsBigNumState* pInverseQ, + IppsRSAPrivateKeyState* pKey)) +{ + IPP_BAD_PTR1_RET(pKey); + IPP_BADARG_RET(!RSA_PRV_KEY2_VALID_ID(pKey), ippStsContextMatchErr); + + IPP_BAD_PTR1_RET(pFactorP); + IPP_BADARG_RET(!BN_VALID_ID(pFactorP), ippStsContextMatchErr); + IPP_BADARG_RET(!(0 < cpBN_tst(pFactorP)), ippStsOutOfRangeErr); + IPP_BADARG_RET(BITSIZE_BNU(BN_NUMBER(pFactorP), BN_SIZE(pFactorP)) > RSA_PRV_KEY_BITSIZE_P(pKey), ippStsSizeErr); + + IPP_BAD_PTR1_RET(pFactorQ); + IPP_BADARG_RET(!BN_VALID_ID(pFactorQ), ippStsContextMatchErr); + IPP_BADARG_RET(!(0 < cpBN_tst(pFactorQ)), ippStsOutOfRangeErr); + IPP_BADARG_RET(BITSIZE_BNU(BN_NUMBER(pFactorQ), BN_SIZE(pFactorQ)) > RSA_PRV_KEY_BITSIZE_Q(pKey), ippStsSizeErr); + + /* let P>Q */ + //IPP_BADARG_RET(0>=cpBN_cmp(pFactorP,pFactorQ), ippStsBadArgErr); + + IPP_BAD_PTR1_RET(pCrtExpP); + IPP_BADARG_RET(!BN_VALID_ID(pCrtExpP), ippStsContextMatchErr); + IPP_BADARG_RET(!(0 < cpBN_tst(pCrtExpP)), ippStsOutOfRangeErr); + IPP_BADARG_RET(BITSIZE_BNU(BN_NUMBER(pCrtExpP), BN_SIZE(pCrtExpP)) > RSA_PRV_KEY_BITSIZE_P(pKey), ippStsSizeErr); + + IPP_BAD_PTR1_RET(pCrtExpQ); + IPP_BADARG_RET(!BN_VALID_ID(pCrtExpQ), ippStsContextMatchErr); + IPP_BADARG_RET(!(0 < cpBN_tst(pCrtExpQ)), ippStsOutOfRangeErr); + IPP_BADARG_RET(BITSIZE_BNU(BN_NUMBER(pCrtExpQ), BN_SIZE(pCrtExpQ)) > RSA_PRV_KEY_BITSIZE_Q(pKey), ippStsSizeErr); + + IPP_BAD_PTR1_RET(pInverseQ); + IPP_BADARG_RET(!BN_VALID_ID(pInverseQ), ippStsContextMatchErr); + IPP_BADARG_RET(!(0 < cpBN_tst(pInverseQ)), ippStsOutOfRangeErr); + IPP_BADARG_RET(BITSIZE_BNU(BN_NUMBER(pInverseQ), BN_SIZE(pInverseQ)) > RSA_PRV_KEY_BITSIZE_P(pKey), ippStsSizeErr); + + /* set bitsize(N) = 0, so the key contex is not ready */ + RSA_PRV_KEY_BITSIZE_N(pKey) = 0; + RSA_PRV_KEY_BITSIZE_D(pKey) = 0; + + /* setup montgomery engine P */ + gsModEngineInit(RSA_PRV_KEY_PMONT(pKey), (Ipp32u*)BN_NUMBER(pFactorP), cpBN_bitsize(pFactorP), MOD_ENGINE_RSA_POOL_SIZE, gsModArithRSA()); + /* setup montgomery engine Q */ + gsModEngineInit(RSA_PRV_KEY_QMONT(pKey), (Ipp32u*)BN_NUMBER(pFactorQ), cpBN_bitsize(pFactorQ), MOD_ENGINE_RSA_POOL_SIZE, gsModArithRSA()); + + /* actual size of key components */ + RSA_PRV_KEY_BITSIZE_P(pKey) = cpBN_bitsize(pFactorP); + RSA_PRV_KEY_BITSIZE_Q(pKey) = cpBN_bitsize(pFactorQ); + + /* store CTR's exp dp */ + ZEXPAND_COPY_BNU(RSA_PRV_KEY_DP(pKey), BITS_BNU_CHUNK(RSA_PRV_KEY_BITSIZE_P(pKey)), BN_NUMBER(pCrtExpP), BN_SIZE(pCrtExpP)); + /* store CTR's exp dq */ + ZEXPAND_COPY_BNU(RSA_PRV_KEY_DQ(pKey), BITS_BNU_CHUNK(RSA_PRV_KEY_BITSIZE_Q(pKey)), BN_NUMBER(pCrtExpQ), BN_SIZE(pCrtExpQ)); + /* store CTR's invQ */ + ZEXPAND_COPY_BNU(RSA_PRV_KEY_INVQ(pKey), BITS_BNU_CHUNK(RSA_PRV_KEY_BITSIZE_P(pKey)), BN_NUMBER(pInverseQ), BN_SIZE(pInverseQ)); + + /* setup montgomery engine N = P*Q */ + { + BNU_CHUNK_T* pN = MOD_MODULUS(RSA_PRV_KEY_NMONT(pKey)); + cpSize nsN = BITS_BNU_CHUNK(RSA_PRV_KEY_BITSIZE_P(pKey) + RSA_PRV_KEY_BITSIZE_Q(pKey)); + + cpMul_BNU_school(pN, + BN_NUMBER(pFactorP), BN_SIZE(pFactorP), + BN_NUMBER(pFactorQ), BN_SIZE(pFactorQ)); + + gsModEngineInit(RSA_PRV_KEY_NMONT(pKey), (Ipp32u*)MOD_MODULUS(RSA_PRV_KEY_NMONT(pKey)), + RSA_PRV_KEY_BITSIZE_P(pKey) + RSA_PRV_KEY_BITSIZE_Q(pKey), MOD_ENGINE_RSA_POOL_SIZE, gsModArithRSA()); + + FIX_BNU(pN, nsN); + RSA_PRV_KEY_BITSIZE_N(pKey) = BITSIZE_BNU(pN, nsN); + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_setpublickey.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_setpublickey.c new file mode 100644 index 0000000..f3030de --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_setpublickey.c @@ -0,0 +1,92 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA Functions +// +// Contents: +// ippsRSA_SetPublicKey() +// +*/ + +#include "owncp.h" +#include "pcpbn.h" +#include "pcpngrsa.h" + +/*F* +// Name: ippsRSA_SetPublicKey +// +// Purpose: Set up the RSA public key +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pKey +// NULL == pPublicExp +// NULL == pKey +// +// ippStsContextMatchErr !BN_VALID_ID(pModulus) +// !BN_VALID_ID(pPublicExp) +// !RSA_PUB_KEY_VALID_ID() +// +// ippStsOutOfRangeErr 0 >= pModulus +// 0 >= pPublicExp +// +// ippStsSizeErr bitsize(pModulus) exceeds requested value +// bitsize(pPublicExp) exceeds requested value +// +// ippStsNoErr no error +// +// Parameters: +// pModulus pointer to modulus (N) +// pPublicExp pointer to public exponent (E) +// pKey pointer to the key context +*F*/ +IPPFUN(IppStatus, ippsRSA_SetPublicKey,(const IppsBigNumState* pModulus, + const IppsBigNumState* pPublicExp, + IppsRSAPublicKeyState* pKey)) +{ + IPP_BAD_PTR1_RET(pKey); + IPP_BADARG_RET(!RSA_PUB_KEY_VALID_ID(pKey), ippStsContextMatchErr); + + IPP_BAD_PTR1_RET(pModulus); + IPP_BADARG_RET(!BN_VALID_ID(pModulus), ippStsContextMatchErr); + IPP_BADARG_RET(!(0 < cpBN_tst(pModulus)), ippStsOutOfRangeErr); + IPP_BADARG_RET(BITSIZE_BNU(BN_NUMBER(pModulus), BN_SIZE(pModulus)) > RSA_PUB_KEY_MAXSIZE_N(pKey), ippStsSizeErr); + + IPP_BAD_PTR1_RET(pPublicExp); + IPP_BADARG_RET(!BN_VALID_ID(pPublicExp), ippStsContextMatchErr); + IPP_BADARG_RET(!(0 < cpBN_tst(pPublicExp)), ippStsOutOfRangeErr); + IPP_BADARG_RET(BITSIZE_BNU(BN_NUMBER(pPublicExp), BN_SIZE(pPublicExp)) > RSA_PUB_KEY_MAXSIZE_E(pKey), ippStsSizeErr); + + { + RSA_PUB_KEY_BITSIZE_N(pKey) = 0; + RSA_PUB_KEY_BITSIZE_E(pKey) = 0; + + /* store E */ + ZEXPAND_COPY_BNU(RSA_PUB_KEY_E(pKey), BITS_BNU_CHUNK(RSA_PUB_KEY_MAXSIZE_E(pKey)), BN_NUMBER(pPublicExp), BN_SIZE(pPublicExp)); + + /* setup montgomery engine */ + gsModEngineInit(RSA_PUB_KEY_NMONT(pKey), (Ipp32u*)BN_NUMBER(pModulus), cpBN_bitsize(pModulus), MOD_ENGINE_RSA_POOL_SIZE, gsModArithRSA()); + + RSA_PUB_KEY_BITSIZE_N(pKey) = cpBN_bitsize(pModulus); + RSA_PUB_KEY_BITSIZE_E(pKey) = cpBN_bitsize(pPublicExp); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_sizeof_privkey1.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_sizeof_privkey1.h new file mode 100644 index 0000000..1adea98 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_sizeof_privkey1.h @@ -0,0 +1,37 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA Functions +// +*/ + +static int cpSizeof_RSA_privateKey1(int rsaModulusBitSize, int privateExpBitSize) +{ + int prvExpLen = BITS_BNU_CHUNK(privateExpBitSize); + int modulusLen32 = BITS2WORD32_SIZE(rsaModulusBitSize); + int montNsize; + rsaMontExpGetSize(modulusLen32, &montNsize); + + return (Ipp32s)sizeof(IppsRSAPrivateKeyState) + + prvExpLen * (Ipp32s)sizeof(BNU_CHUNK_T) + + (Ipp32s)sizeof(BNU_CHUNK_T) - 1 + + montNsize; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_sizeof_privkey2.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_sizeof_privkey2.h new file mode 100644 index 0000000..7e8de54 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_sizeof_privkey2.h @@ -0,0 +1,48 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA Functions +// +*/ + +static int cpSizeof_RSA_privateKey2(int factorPbitSize, int factorQbitSize) +{ + int factorPlen = BITS_BNU_CHUNK(factorPbitSize); + int factorQlen = BITS_BNU_CHUNK(factorQbitSize); + int factorPlen32 = BITS2WORD32_SIZE(factorPbitSize); + int factorQlen32 = BITS2WORD32_SIZE(factorQbitSize); + int rsaModulusLen32 = BITS2WORD32_SIZE(factorPbitSize + factorQbitSize); + int montPsize; + int montQsize; + int montNsize; + rsaMontExpGetSize(factorPlen32, &montPsize); + rsaMontExpGetSize(factorQlen32, &montQsize); + rsaMontExpGetSize(rsaModulusLen32, &montNsize); + + return (Ipp32s)sizeof(IppsRSAPrivateKeyState) + + factorPlen * (Ipp32s)sizeof(BNU_CHUNK_T) /* dp slot */ + + factorQlen * (Ipp32s)sizeof(BNU_CHUNK_T) /* dq slot */ + + factorPlen * (Ipp32s)sizeof(BNU_CHUNK_T) /* qinv slot */ + + (Ipp32s)sizeof(BNU_CHUNK_T) - 1 /* alignment */ + + montPsize + + montQsize + + montNsize; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_sizeof_pubkey.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_sizeof_pubkey.h new file mode 100644 index 0000000..7fcbe1c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_sizeof_pubkey.h @@ -0,0 +1,38 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA Functions +// +*/ + +/* compute size of RSA public key context */ +static int cpSizeof_RSA_publicKey(int rsaModulusBitSize, int publicExpBitSize) +{ + int pubExpLen = BITS_BNU_CHUNK(publicExpBitSize); + int modulusLen32 = BITS2WORD32_SIZE(rsaModulusBitSize); + int montNsize; + rsaMontExpGetSize(modulusLen32, &montNsize); + + return (Ipp32s)sizeof(IppsRSAPublicKeyState) + + pubExpLen*(Ipp32s)sizeof(BNU_CHUNK_T) + + (Ipp32s)sizeof(BNU_CHUNK_T)-1 /* alignment */ + + montNsize; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_validatekeys.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_validatekeys.c new file mode 100644 index 0000000..34f062c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_validatekeys.c @@ -0,0 +1,355 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA Functions +// +// Contents: +// ippsRSA_ValidateKeys() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpbn.h" +#include "pcpprimeg.h" +#include "pcpprng.h" +#include "pcpngrsa.h" + +#include "pcpprime_isco.h" +#include "pcpprime_isprob.h" + +/*F* +// Name: ippsRSA_ValidateKeys +// +// Purpose: Validate RSA keys +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pPublicKey +// NULL == pPrivateKeyType2 +// NULL == pPrivateKeyType1 +// NULL == pBuffer +// NULL == pPrimeGen +// NULL == rndFunc +// NULL == pResult +// +// ippStsContextMatchErr !RSA_PUB_KEY_VALID_ID(pPublicKey) +// !RSA_PRV_KEY2_VALID_ID(pPrivateKeyType2) +// !RSA_PRV_KEY1_VALID_ID(pPrivateKeyType1) +// !PRIME_VALID_ID(pPrimeGen) +// +// ippStsIncompleteContextErr public and.or private key is not set up +// +// ippStsSizeErr PRIME_MAXBITSIZE(pPrimeGen) < factorPbitSize +// +// ippStsBadArgErr nTrials < 1 +// +// ippStsNoErr no error +// +// Parameters: +// pResult pointer to the validation result +// pPublicKey pointer to the public key context +// pPrivateKeyType2 pointer to the private key type2 context +// pPrivateKeyType1 (optional) pointer to the private key type1 context +// pBuffer pointer to the temporary buffer +// nTrials parameter of Miller-Rabin Test +// pPrimeGen pointer to the Prime generator context +// rndFunc external PRNG +// pRndParam pointer to the external PRNG parameters +*F*/ +/* +// make sure D*E = 1 mod(phi(P,Q)) +// where phi(P,Q) = (P-1)*(Q-1) +*/ +static +int isValidPriv1_classic(const BNU_CHUNK_T* pN, int nsN, + const BNU_CHUNK_T* pE, int nsE, + const BNU_CHUNK_T* pD, int nsD, + const BNU_CHUNK_T* pFactorP, int nsP, + const BNU_CHUNK_T* pFactorQ, int nsQ, + BNU_CHUNK_T* pBuffer) +{ + BNU_CHUNK_T* pPhi = pBuffer; + BNU_CHUNK_T* pProduct = pPhi + nsN; + BNU_CHUNK_T c = cpSub_BNU(pPhi, pN, pFactorP, nsP); + int prodLen; + if (nsN>1) cpDec_BNU(pPhi + nsP, pN + nsP, nsQ, c); + c = cpSub_BNU(pPhi, pPhi, pFactorQ, nsQ); + if (nsN>1) cpDec_BNU(pPhi + nsQ, pPhi + nsQ, nsP, c); + cpInc_BNU(pPhi, pPhi, nsP + nsQ, 1); + + cpMul_BNU_school(pProduct, pE, nsE, pD, nsD); + prodLen = cpMod_BNU(pProduct, nsE + nsD, pPhi, nsN); + + return 1 == cpEqu_BNU_CHUNK(pProduct, prodLen, 1) ? IPP_IS_VALID : IPP_IS_INVALID; +} + +/* +// make sure D*E = 1 mod(lcm(P-1,Q-1)) +// where lcm(P-1,Q-1) = (P-1)*(Q-1)/gcd(P-1,Q-1) +*/ +static +int isValidPriv1_rsa(const BNU_CHUNK_T* pN, int nsN, + const BNU_CHUNK_T* pE, int nsE, + const BNU_CHUNK_T* pD, int nsD, + BNU_CHUNK_T* pFactorP, int nsP, + BNU_CHUNK_T* pFactorQ, int nsQ, + BNU_CHUNK_T* pBuffer) +{ + __ALIGN8 IppsBigNumState tmpBN1; + __ALIGN8 IppsBigNumState tmpBN2; + __ALIGN8 IppsBigNumState tmpBN3; + + BNU_CHUNK_T* pProduct = pBuffer; + BNU_CHUNK_T* pGcd = pProduct + (nsN + 1); + BNU_CHUNK_T* pLcm; + int nsLcm; + int prodLen; + pBuffer = pGcd + (nsP + 1) * 2; + + /* P = P-1 and Q = Q-1 */ + pFactorP[0]--; + pFactorQ[0]--; + + /* compute product (P-1)*(Q-1) = P*Q -P -Q +1 = N -(P-1) -(Q-1) -1 */ + { + BNU_CHUNK_T c = cpSub_BNU(pProduct, pN, pFactorP, nsP); + if (nsN>1) cpDec_BNU(pProduct + nsP, pN + nsP, nsQ, c); + c = cpSub_BNU(pProduct, pProduct, pFactorQ, nsQ); + if (nsN>1) cpDec_BNU(pProduct + nsQ, pProduct + nsQ, nsP, c); + cpDec_BNU(pProduct, pProduct, nsN, 1); + } + + /* compute gcd(p-1, q-1) */ + BN_Make(pGcd, pGcd + nsP + 1, nsP, &tmpBN1); /* BN(gcd) */ + BN_SIZE(&tmpBN1) = nsP; + BN_Make(pFactorP, pBuffer, nsP, &tmpBN2); /* BN(P-1) */ + BN_SIZE(&tmpBN2) = nsP; + BN_Make(pFactorQ, pBuffer + nsP + 1, nsQ, &tmpBN3); /* BN(Q-1) */ + BN_SIZE(&tmpBN3) = nsQ; + ippsGcd_BN(&tmpBN2, &tmpBN3, &tmpBN1); + + /* compute lcm(p-1, q-1) = (p-1)(q-1)/gcd(p-1, q-1) */ + pLcm = pBuffer; + cpDiv_BNU(pLcm, &nsLcm, pProduct, nsN, pGcd, BN_SIZE(&tmpBN1)); + + /* test E*D = 1 mod lcm */ + cpMul_BNU_school(pProduct, pE, nsE, pD, nsD); + prodLen = cpMod_BNU(pProduct, nsE + nsD, pLcm, nsLcm); + + /* restore P and Q */ + pFactorP[0]++; + pFactorQ[0]++; + + return 1 == cpEqu_BNU_CHUNK(pProduct, prodLen, 1) ? IPP_IS_VALID : IPP_IS_INVALID; +} + +IPPFUN(IppStatus, ippsRSA_ValidateKeys,(int* pResult, + const IppsRSAPublicKeyState* pPublicKey, + const IppsRSAPrivateKeyState* pPrivateKeyType2, + const IppsRSAPrivateKeyState* pPrivateKeyType1, /*optional */ + Ipp8u* pBuffer, + int nTrials, + IppsPrimeState* pPrimeGen, /*NULL */ + IppBitSupplier rndFunc, void* pRndParam)) +{ + IPP_BAD_PTR1_RET(pPublicKey); + IPP_BADARG_RET(!RSA_PUB_KEY_VALID_ID(pPublicKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PUB_KEY_IS_SET(pPublicKey), ippStsIncompleteContextErr); + + IPP_BAD_PTR1_RET(pPrivateKeyType2); + IPP_BADARG_RET(!RSA_PRV_KEY2_VALID_ID(pPrivateKeyType2), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PRV_KEY_IS_SET(pPrivateKeyType2), ippStsIncompleteContextErr); + + if(pPrivateKeyType1) { /* pPrivateKeyType1 is optional */ + IPP_BADARG_RET(!RSA_PRV_KEY1_VALID_ID(pPrivateKeyType1), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PRV_KEY_IS_SET(pPrivateKeyType1), ippStsIncompleteContextErr); + } + + IPP_BAD_PTR3_RET(pResult, pBuffer, rndFunc); + + IPP_UNREFERENCED_PARAMETER(pPrimeGen); + + { + /* E key component */ + BNU_CHUNK_T* pExpE = RSA_PUB_KEY_E(pPublicKey); + cpSize nsE = BITS_BNU_CHUNK(RSA_PUB_KEY_BITSIZE_E(pPublicKey)); + + /* N key component */ + BNU_CHUNK_T* pN = MOD_MODULUS(RSA_PUB_KEY_NMONT(pPublicKey)); + cpSize nsN = MOD_LEN(RSA_PUB_KEY_NMONT(pPublicKey)); + + /* P, Q, dP, dQ, invQ key components */ + gsModEngine* pMontP = RSA_PRV_KEY_PMONT(pPrivateKeyType2); + gsModEngine* pMontQ = RSA_PRV_KEY_QMONT(pPrivateKeyType2); + BNU_CHUNK_T* pFactorP= MOD_MODULUS(pMontP); + BNU_CHUNK_T* pFactorQ= MOD_MODULUS(pMontQ); + BNU_CHUNK_T* pExpDp = RSA_PRV_KEY_DP(pPrivateKeyType2); + BNU_CHUNK_T* pExpDq = RSA_PRV_KEY_DQ(pPrivateKeyType2); + BNU_CHUNK_T* pInvQ = RSA_PRV_KEY_INVQ(pPrivateKeyType2); + + int factorPbitSize = RSA_PRV_KEY_BITSIZE_P(pPrivateKeyType2); + int factorQbitSize = RSA_PRV_KEY_BITSIZE_Q(pPrivateKeyType2); + cpSize nsP = MOD_LEN(RSA_PRV_KEY_PMONT(pPrivateKeyType2)); + cpSize nsQ = MOD_LEN(RSA_PRV_KEY_QMONT(pPrivateKeyType2)); + + int ret = IPP_IS_VALID; + + BNU_CHUNK_T* pTmp = (BNU_CHUNK_T*)(IPP_ALIGNED_PTR(pBuffer, (int)sizeof(BNU_CHUNK_T))); + BNU_CHUNK_T* pFreeBuffer = pTmp+nsN; + + /* choose security parameter */ + int mrTrials = (nTrials<1)? MR_rounds_p80(factorPbitSize) : nTrials; + + IppStatus sts = ippStsNoErr; + do { + /* test if E is odd and 3 <= E < N */ + if(0==(pExpE[0]&1)) { + ret = IPP_IS_INVALID; + break; + } + if(1==nsE && pExpE[0]<3) { + ret = IPP_IS_INVALID; + break; + } + if(0 <= cpCmp_BNU(pExpE, nsE, pN, nsN)) { + ret = IPP_IS_INVALID; + break; + } + + /* test if N==P*Q */ + cpMul_BNU_school(pFreeBuffer, pFactorP, nsP, pFactorQ, nsQ); + { + cpSize ns = cpFix_BNU(pFreeBuffer, nsP+nsQ); + if(cpCmp_BNU(pFreeBuffer, ns, pN, nsN)) { + ret = IPP_IS_INVALID; + break; + } + } + /* test if PubKey(N)==PrivKeytype2(N) */ + if(cpCmp_BNU(pN, nsN, MOD_MODULUS(RSA_PRV_KEY_NMONT(pPrivateKeyType2)), MOD_LEN(RSA_PRV_KEY_NMONT(pPrivateKeyType2)))) { + ret = IPP_IS_INVALID; + break; + } + /* test if PubKey(N)==PrivKeytype1(N) */ + if(pPrivateKeyType1) { + if(cpCmp_BNU(pN, nsN, MOD_MODULUS(RSA_PRV_KEY_NMONT(pPrivateKeyType1)), MOD_LEN(RSA_PRV_KEY_NMONT(pPrivateKeyType1)))) { + ret = IPP_IS_INVALID; + break; + } + } + + /* test if P is prime */ + //if(0==cpIsProbablyPrime(pFactorP, factorPbitSize, mrTrials, + // rndFunc, pRndParam, + // pMontP, pFreeBuffer)) { + // ret = IPP_IS_COMPOSITE; + // break; + //} + { + int r = cpIsProbablyPrime(pFactorP, factorPbitSize, mrTrials, + rndFunc, pRndParam, + pMontP, pFreeBuffer); + if(0>r) { + sts = ippStsErr; + break; + } + if(0==r) { + ret = IPP_IS_COMPOSITE; + break; + } + } + /* test if E and (P-1) co-prime */ + cpDec_BNU(pTmp, pFactorP, nsP, 1); + if(0 == cpIsCoPrime(pExpE, nsE, pTmp, nsP, pFreeBuffer)) { + ret = IPP_IS_INVALID; + break; + } + /* test if E*dP = 1 mod (P-1) */ + cpMul_BNU_school(pFreeBuffer, pExpDp, nsP, pExpE, nsE); + cpMod_BNU(pFreeBuffer, nsP+nsE, pTmp, nsP); + if(!cpEqu_BNU_CHUNK(pFreeBuffer, nsP, 1)) { + ret = IPP_IS_INVALID; + break; + } + + /* test if Q is prime */ + //if(0==cpIsProbablyPrime(pFactorQ, factorQbitSize, mrTrials, + // rndFunc, pRndParam, + // pMontQ, pFreeBuffer)) { + // ret = IPP_IS_COMPOSITE; + // break; + //} + { + int r = cpIsProbablyPrime(pFactorQ, factorQbitSize, mrTrials, + rndFunc, pRndParam, + pMontQ, pFreeBuffer); + if(0>r) { + sts = ippStsErr; + break; + } + if(0==r) { + ret = IPP_IS_COMPOSITE; + break; + } + } + /* test if E and (Q-1) co-prime */ + cpDec_BNU(pTmp, pFactorQ, nsQ, 1); + if(0 == cpIsCoPrime(pExpE, nsE, pTmp, nsQ, pFreeBuffer)) { + ret = IPP_IS_INVALID; + break; + } + /* test if E*dQ = 1 mod (Q-1) */ + cpMul_BNU_school(pFreeBuffer, pExpDq, nsQ, pExpE, nsE); + cpMod_BNU(pFreeBuffer, nsQ+nsE, pTmp, nsQ); + if(!cpEqu_BNU_CHUNK(pFreeBuffer, nsQ, 1)) { + ret = IPP_IS_INVALID; + break; + } + + /* test if 1==(Q*Qinv) mod P */ + cpMul_BNU_school(pFreeBuffer, pInvQ, nsP, pFactorQ,nsQ); + cpMod_BNU(pFreeBuffer, nsP+nsQ, pFactorP, nsP); + if(!cpEqu_BNU_CHUNK(pFreeBuffer, nsP, 1)) { + ret = IPP_IS_INVALID; + break; + } + + /* test private exponent (optional) */ + if(pPrivateKeyType1) { + const BNU_CHUNK_T* pExpD = RSA_PRV_KEY_D(pPrivateKeyType1); + cpSize nsD = nsN; + int resilt1 = isValidPriv1_classic(pN,nsN, pExpE,nsE, pExpD,nsD, + pFactorP,nsP, pFactorQ,nsQ, + pTmp); + int resilt2 = isValidPriv1_rsa(pN,nsN, pExpE,nsE, pExpD,nsD, + pFactorP,nsP, pFactorQ,nsQ, + pTmp); + if(IPP_IS_VALID!=resilt1 && IPP_IS_VALID!=resilt2) { + ret = IPP_IS_INVALID; + break; + } + } + } while(0); + + *pResult = ret; + return sts; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_verifysign_pkcs1v15.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_verifysign_pkcs1v15.h new file mode 100644 index 0000000..42c22f0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsa_verifysign_pkcs1v15.h @@ -0,0 +1,73 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSASSA-PKCS-v1_5 +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpngrsa.h" +#include "pcphash.h" +#include "pcptool.h" + +#include "pcprsa_emsa_pkcs1v15.h" + +static int VerifySign(const Ipp8u* pMsg, int msgLen, /* message representation */ + const Ipp8u* pSalt, int saltLen, /* fied string */ + const Ipp8u* pSign, + int* pIsValid, + const IppsRSAPublicKeyState* pKey, + BNU_CHUNK_T* pBuffer) +{ + /* size of RSA modulus in bytes and chunks */ + cpSize rsaBits = RSA_PUB_KEY_BITSIZE_N(pKey); + cpSize k = BITS2WORD8_SIZE(rsaBits); + cpSize nsN = BITS_BNU_CHUNK(rsaBits); + + /* temporary BNs */ + __ALIGN8 IppsBigNumState bnC; + __ALIGN8 IppsBigNumState bnP; + + /* make BNs */ + BN_Make(pBuffer, pBuffer + nsN + 1, nsN, &bnC); + pBuffer += (nsN + 1) * 2; + BN_Make(pBuffer, pBuffer + nsN + 1, nsN, &bnP); + pBuffer += (nsN + 1) * 2; + + /* + // public-key operation + */ + ippsSetOctString_BN(pSign, k, &bnP); + gsRSApub_cipher(&bnC, &bnP, pKey, pBuffer); + + /* convert EM into the string */ + ippsGetOctString_BN((Ipp8u*)(BN_BUFFER(&bnC)), k, &bnC); + + /* EMSA-PKCS-v1_5 encoding */ + if (EMSA_PKCSv15(pMsg, msgLen, pSalt, saltLen, (Ipp8u*)(BN_NUMBER(&bnC)), k)) { + *pIsValid = 1 == EquBlock((Ipp8u*)(BN_BUFFER(&bnC)), (Ipp8u*)(BN_NUMBER(&bnC)), k); + return 1; + } + else + return 0; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsadecrypt_oaep.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsadecrypt_oaep.c new file mode 100644 index 0000000..5d19ab8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsadecrypt_oaep.c @@ -0,0 +1,215 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSAES-OAEP Encryption/Decription Functions +// +// Contents: +// ippsRSADecrypt_OAEP() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcptool.h" +#include "pcpngrsa.h" +#include "pcphash.h" + +/*F* +// Name: ippsRSADecrypt_OAEP +// +// Purpose: Performs RSAES-OAEP decryprion scheme +// +// Returns: Reason: +// ippStsNotSupportedModeErr unknown hashAlg +// +// ippStsNullPtrErr NULL == pKey +// NULL == pSrc +// NULL == pLab +// NULL == pDst +// NULL == pDstLen +// NULL == pBuffer +// +// ippStsLengthErr labLen <0 +// RSAsize < 2*hashLen +2 +// +// ippStsIncompleteContextErr private key is not set up +// +// ippStsContextMatchErr !RSA_PRV_KEY_VALID_ID() +// +// ippStsOutOfRangeErr ciphertext > RSA(N) +// +// ippStsUnderRunErr decoding error +// +// ippStsNoErr no error +// +// Parameters: +// pSrc pointer to the ciphertext +// assumed that length of the ciphertext is equal to k - sizeof RSA modulus in bytes +// pLab (optional) pointer to the label associated with plaintext +// labLen label length (bytes) +// pDst pointer to the plaintext +// pDstLen pointer to the plaintext length +// assumed that length of the recovered message is at least k-hashLen*2-2 bytes; +// maximum message length is (k-hashLen*2-2) bytes and is for choosen RSA and Hash-function +// pKey pointer to the RSA private key context +// hashAlg hash alg ID +// pBuffer pointer to scratch buffer +*F*/ +IPPFUN(IppStatus, ippsRSADecrypt_OAEP, (const Ipp8u* pSrc, + const Ipp8u* pLab, int labLen, + Ipp8u* pDst, int* pDstLen, + const IppsRSAPrivateKeyState* pKey, + IppHashAlgId hashAlg, + Ipp8u* pScratchBuffer)) +{ + int hashLen; + + /* test hash algorith ID */ + hashAlg = cpValidHashAlg(hashAlg); + IPP_BADARG_RET(ippHashAlg_Unknown == hashAlg, ippStsNotSupportedModeErr); + + /* test data pointer */ + IPP_BAD_PTR3_RET(pSrc, pDst, pDstLen); + + IPP_BADARG_RET(!pLab && labLen, ippStsNullPtrErr); + + IPP_BAD_PTR2_RET(pKey, pScratchBuffer); + IPP_BADARG_RET(!RSA_PRV_KEY_VALID_ID(pKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PRV_KEY_IS_SET(pKey), ippStsIncompleteContextErr); + + /* test hash length */ + IPP_BADARG_RET(labLen < 0, ippStsLengthErr); + + hashLen = cpHashSize(hashAlg); + /* test compatibility of RSA and hash length */ + IPP_BADARG_RET(BITS2WORD8_SIZE(RSA_PRV_KEY_BITSIZE_N(pKey)) < (2 * hashLen + 2), ippStsLengthErr); + + { + /* size of RSA modulus in bytes and chunks */ + int k = BITS2WORD8_SIZE(RSA_PRV_KEY_BITSIZE_N(pKey)); + cpSize nsN = BITS_BNU_CHUNK(RSA_PRV_KEY_BITSIZE_N(pKey)); + + /* align resource */ + BNU_CHUNK_T* pResource = (BNU_CHUNK_T*)(IPP_ALIGNED_PTR(pScratchBuffer, (int)sizeof(BNU_CHUNK_T))); + + /* temporary BN */ + __ALIGN8 IppsBigNumState tmpBN; + BN_Make(pResource, pResource+ nsN + 1, nsN, &tmpBN); + pResource += (nsN + 1) * 2; /* update buffer pointer */ + ippsSetOctString_BN(pSrc, k, &tmpBN); /* convert ciphertext to BigNum */ + /* make sure ciphertext < RSA modulus N */ + IPP_BADARG_RET(0 <= cpCmp_BNU(BN_NUMBER(&tmpBN), BN_SIZE(&tmpBN), + MOD_MODULUS(RSA_PRV_KEY_NMONT(pKey)), MOD_LEN(RSA_PRV_KEY_NMONT(pKey))), ippStsOutOfRangeErr); + + /* RSA decryption */ + + if (RSA_PRV_KEY1_VALID_ID(pKey)) + gsRSAprv_cipher(&tmpBN, &tmpBN, pKey, pResource); + else + gsRSAprv_cipher_crt(&tmpBN, &tmpBN, pKey, pResource); + + /* + // EME-OAEP decoding + */ + { + Ipp8u* pEM = (Ipp8u*)(BN_BUFFER(&tmpBN)); + Ipp8u* pBuffer = (Ipp8u*)BN_NUMBER(&tmpBN); + + int i; + /* convert RSA encoded into EM */ + for (i = 0; i < k; i++) { + pEM[i] = pBuffer[k - 1 - i]; + } + + /* + // OAEP EM decoding, EM = Y || maskedSeed || maskedDB + */ + { + /* check that Y == 0 */ + BNU_CHUNK_T res = cpIsZero_ct(pEM[0]); + + Ipp8u* pSeed = pEM + 1; + Ipp8u* pDB = pEM + 1 + hashLen; /* DB = lHash || PS || Msg */ + int dbLen = k - 1 - hashLen; + int maxMsgLen = dbLen - hashLen - 1; + + /* seed = maskedSeed ^ seedMask, seedMask = MGF(maskedDB, hashLen) */ + ippsMGF(pDB, dbLen, pBuffer, hashLen, hashAlg); + XorBlock(pSeed, pBuffer, pSeed, hashLen); + + /* DB = maskedDB ^ dbMask, dbMask = MGF (seed, dbLen) */ + ippsMGF(pSeed, hashLen, pBuffer, dbLen, hashAlg); + XorBlock(pDB, pBuffer, pDB, dbLen); + + /* re-compute Hash(Label) and compare with lHash */ + ippsHashMessage(pLab, labLen, pBuffer, hashAlg); + res &= cpIsEquBlock_ct(pDB, pBuffer, hashLen); + + /* detect the padding consists of a number of 0x00-bytes, followed by a 0x01 */ + BNU_CHUNK_T byte_01_found = 0; + BNU_CHUNK_T byte_01_idx = 0; + for (i = hashLen; i < dbLen; i++) { + BNU_CHUNK_T byte_00 = cpIsZero_ct(pDB[i]); /* mask if byte is 0x00 */ + BNU_CHUNK_T byte_01 = cpIsZero_ct(pDB[i] ^ 0x01); /* mask if byte is 0x01 */ + /* set index of byte 01_idx if byte 01 found */ + byte_01_idx = cpSelect_ct(~byte_01_found & byte_01, (BNU_CHUNK_T)i, byte_01_idx); + /* set flag byte_01_found */ + byte_01_found |= byte_01; + /* update EM encoding result */ + res &= (byte_01_found | byte_00); + } + /* update EM encoding result if byte_01_found */ + res &= byte_01_found; + + /* copy decoded message to pDst */ + { + /* move decoded message by (maxMsgLen-msgLen) left + */ + int msgIdx = (int)byte_01_idx + 1; /* decoded message index inside DB */ + int msgLen = dbLen - msgIdx; /* length of decoded message */ + + BNU_CHUNK_T mask; + for(msgIdx = 1; msgIdx < maxMsgLen; msgIdx <<= 1) { + mask = cpIsEqu_ct((BNU_CHUNK_T)(msgIdx & (maxMsgLen - msgLen)), (BNU_CHUNK_T)msgIdx); + for(i = hashLen+1; i < (dbLen-msgIdx); i++) + pDB[i] = cpSelect_ct_8u(mask, pDB[i + msgIdx], pDB[i]); + } + + /* copy decoded message */ + for(i = 0; i < maxMsgLen; i++) { + mask = res & cpIsLt_ct((BNU_CHUNK_T)i, (BNU_CHUNK_T)msgLen); + pDst[i] = cpSelect_ct_8u(mask, pDB[i + hashLen + 1], pDst[i]); + } + + /* decoded message length */ + *pDstLen = cpSelect_ct_int(res, msgLen, -1); + } + + /* clean */ + PurgeBlock(pEM, k); + PurgeBlock(pBuffer, k); + + /* return error status if is */ + return (IppStatus)cpSelect_ct_int(res, ippStsNoErr, ippStsUnderRunErr); + } + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsadecrypt_oaep_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsadecrypt_oaep_rmf.c new file mode 100644 index 0000000..565d75b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsadecrypt_oaep_rmf.c @@ -0,0 +1,210 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Cryptography Primitive. +// RSAES-OAEP Encryption/Decription Functions +// +// Contents: +// ippsRSADecrypt_OAEP_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcptool.h" +#include "pcpngrsa.h" +#include "pcphash_rmf.h" + +/*F* +// Name: ippsRSADecrypt_OAEP_rmf +// +// Purpose: Performs RSAES-OAEP decryprion scheme +// +// Returns: Reason: +// ippStsNotSupportedModeErr unknown hashAlg +// +// ippStsNullPtrErr NULL == pKey +// NULL == pSrc +// NULL == pLab +// NULL == pDst +// NULL == pDstLen +// NULL == pMethod +// NULL == pBuffer +// +// ippStsLengthErr labLen <0 +// RSAsize < 2*hashLen +2 +// +// ippStsIncompleteContextErr private key is not set up +// +// ippStsContextMatchErr !RSA_PRV_KEY_VALID_ID() +// +// ippStsOutOfRangeErr ciphertext > RSA(N) +// +// ippStsUnderRunErr decoding error +// +// ippStsNoErr no error +// +// Parameters: +// pSrc pointer to the ciphertext +// assumed that length of the ciphertext is equal to k - sizeof RSA modulus in bytes +// pLab (optional) pointer to the label associated with plaintext +// labLen label length (bytes) +// pDst pointer to the encoded plaintext +// assumed that length of the recovered message is at least k-hashLen*2-2 bytes; +// maximum message length is (k-hashLen*2-2) bytes and is for choosen RSA and Hash-function +// pDstLen pointer to the plaintext length +// pKey pointer to the RSA private key context +// pMethod hash methods +// pBuffer pointer to scratch buffer +*F*/ +IPPFUN(IppStatus, ippsRSADecrypt_OAEP_rmf, (const Ipp8u* pSrc, + const Ipp8u* pLab, int labLen, + Ipp8u* pDst, int* pDstLen, + const IppsRSAPrivateKeyState* pKey, + const IppsHashMethod* pMethod, + Ipp8u* pScratchBuffer)) +{ + int hashLen; + + /* test data pointer */ + IPP_BAD_PTR4_RET(pSrc, pDst, pDstLen, pMethod); + + IPP_BADARG_RET(!pLab && labLen, ippStsNullPtrErr); + + IPP_BAD_PTR2_RET(pKey, pScratchBuffer); + IPP_BADARG_RET(!RSA_PRV_KEY_VALID_ID(pKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PRV_KEY_IS_SET(pKey), ippStsIncompleteContextErr); + + /* test hash length */ + IPP_BADARG_RET(labLen < 0, ippStsLengthErr); + + hashLen = pMethod->hashLen; + /* test compatibility of RSA and hash length */ + IPP_BADARG_RET(BITS2WORD8_SIZE(RSA_PRV_KEY_BITSIZE_N(pKey)) < (2 * hashLen + 2), ippStsLengthErr); + + { + /* size of RSA modulus in bytes and chunks */ + int k = BITS2WORD8_SIZE(RSA_PRV_KEY_BITSIZE_N(pKey)); + cpSize nsN = BITS_BNU_CHUNK(RSA_PRV_KEY_BITSIZE_N(pKey)); + + /* align resource */ + BNU_CHUNK_T* pResource = (BNU_CHUNK_T*)(IPP_ALIGNED_PTR(pScratchBuffer, (int)sizeof(BNU_CHUNK_T))); + + /* temporary BN */ + __ALIGN8 IppsBigNumState tmpBN; + BN_Make(pResource, pResource+nsN+1, nsN, &tmpBN); + pResource += (nsN + 1) * 2; /* update buffer pointer */ + ippsSetOctString_BN(pSrc, k, &tmpBN); /* convert ciphertext to BigNum */ + /* make sure ciphertext < RSA modulus N */ + IPP_BADARG_RET(0 <= cpCmp_BNU(BN_NUMBER(&tmpBN), BN_SIZE(&tmpBN), + MOD_MODULUS(RSA_PRV_KEY_NMONT(pKey)), MOD_LEN(RSA_PRV_KEY_NMONT(pKey))), ippStsOutOfRangeErr); + + /* RSA decryption */ + if (RSA_PRV_KEY1_VALID_ID(pKey)) + gsRSAprv_cipher(&tmpBN, &tmpBN, pKey, pResource); + else + gsRSAprv_cipher_crt(&tmpBN, &tmpBN, pKey, pResource); + + /* + // EME-OAEP decoding + */ + { + Ipp8u* pEM = (Ipp8u*)(BN_BUFFER(&tmpBN)); + Ipp8u* pBuffer = (Ipp8u*)BN_NUMBER(&tmpBN); + + int i; + /* convert RSA encoded into EM */ + for (i = 0; i < k; i++) { + pEM[i] = pBuffer[k-1-i]; + } + + /* + // OAEP EM decoding, EM = Y || maskedSeed || maskedDB + */ + { + /* check that Y == 0 */ + BNU_CHUNK_T res = cpIsZero_ct(pEM[0]); + + Ipp8u* pSeed = pEM + 1; + Ipp8u* pDB = pEM + 1 + hashLen; /* DB = lHash || PS || Msg */ + int dbLen = k - 1 - hashLen; + int maxMsgLen = dbLen - hashLen - 1; + + /* seed = maskedSeed ^ seedMask, seedMask = MGF(maskedDB, hashLen) */ + ippsMGF1_rmf(pDB, dbLen, pBuffer, hashLen, pMethod); + XorBlock(pSeed, pBuffer, pSeed, hashLen); + + /* DB = maskedDB ^ dbMask, dbMask = MGF (seed, dbLen) */ + ippsMGF1_rmf(pSeed, hashLen, pBuffer, dbLen, pMethod); + XorBlock(pDB, pBuffer, pDB, dbLen); + + /* re-compute Hash(Label) and compare with lHash */ + ippsHashMessage_rmf(pLab, labLen, pBuffer, pMethod); + res &= cpIsEquBlock_ct(pDB, pBuffer, hashLen); + + /* detect the padding consists of a number of 0x00-bytes, followed by a 0x01 */ + BNU_CHUNK_T byte_01_found = 0; + BNU_CHUNK_T byte_01_idx = 0; + for (i = hashLen; i < dbLen; i++) { + BNU_CHUNK_T byte_00 = cpIsZero_ct(pDB[i]); /* mask if byte is 0x00 */ + BNU_CHUNK_T byte_01 = cpIsZero_ct(pDB[i] ^ 0x01); /* mask if byte is 0x01 */ + /* set index of byte 01_idx if byte 01 found */ + byte_01_idx = cpSelect_ct(~byte_01_found & byte_01, (BNU_CHUNK_T)i, byte_01_idx); + /* set flag byte_01_found */ + byte_01_found |= byte_01; + /* update EM encoding result */ + res &= (byte_01_found | byte_00); + } + /* update EM encoding result if byte_01_found */ + res &= byte_01_found; + + /* copy decoded message to pDst */ + { + /* move decoded message by (maxMsgLen-msgLen) left + */ + int msgIdx = (int)byte_01_idx + 1; /* decoded message index inside DB */ + int msgLen = dbLen - msgIdx; /* length of decoded message */ + + BNU_CHUNK_T mask; + for(msgIdx = 1; msgIdx < maxMsgLen; msgIdx <<= 1) { + mask = cpIsEqu_ct((BNU_CHUNK_T)(msgIdx & (maxMsgLen - msgLen)), (BNU_CHUNK_T)msgIdx); + for(i = hashLen+1; i < (dbLen-msgIdx); i++) + pDB[i] = cpSelect_ct_8u(mask, pDB[i + msgIdx], pDB[i]); + } + + /* copy decoded message */ + for(i = 0; i < maxMsgLen; i++) { + mask = res & cpIsLt_ct((BNU_CHUNK_T)i, (BNU_CHUNK_T)msgLen); + pDst[i] = cpSelect_ct_8u(mask, pDB[i + hashLen + 1], pDst[i]); + } + + /* decoded message length */ + *pDstLen = cpSelect_ct_int(res, msgLen, -1); + } + + /* clean */ + PurgeBlock(pEM, k); + PurgeBlock(pBuffer, k); + + /* return error status if is */ + return (IppStatus)cpSelect_ct_int(res, ippStsNoErr, ippStsUnderRunErr); + } + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsadecrypt_pkcsv15.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsadecrypt_pkcsv15.c new file mode 100644 index 0000000..0806b0b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsadecrypt_pkcsv15.c @@ -0,0 +1,153 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA-PKCS1-v1_5 Encryption Scheme +// +// Contents: +// ippsRSADecrypt_PKCSv15() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpngrsa.h" +#include "pcptool.h" + + +static Ipp32u NonZeroBlockLen(const Ipp8u* pData, int dataLen) +{ + int i; + for(i=0; i emLen); + + /* + // decoded message format: + // EM = 00 || 02 || PS || 00 || Msg + // len(PS) >= 8 + */ + errDecodeFlag |= (0x00 != pEM[0]); + errDecodeFlag |= (0x02 != pEM[1]); + errDecodeFlag |= (0x00 != pEM[3+psLen-1]); + CopyBlock(pEM+3+psLen, pMsg, msgLen); + *pMsgLen = msgLen; + return !errDecodeFlag; +} + + +/* +// returns 0 decription error +/ 1 OK +*/ +static int Decryption(const Ipp8u* pCipherTxt, + Ipp8u* pMsg, int* pMsgLen, + const IppsRSAPrivateKeyState* pKey, + BNU_CHUNK_T* pBuffer) +{ + /* size of RSA modulus in bytes and chunks */ + int k = BITS2WORD8_SIZE(RSA_PRV_KEY_BITSIZE_N(pKey)); + cpSize nsN = BITS_BNU_CHUNK(RSA_PRV_KEY_BITSIZE_N(pKey)); + + /* temporary BN */ + __ALIGN8 IppsBigNumState tmpBN; + BN_Make(pBuffer, pBuffer+nsN+1, nsN, &tmpBN); + + /* update buffer pointer */ + pBuffer += (nsN+1)*2; + + /* + // private-key operation + */ + ippsSetOctString_BN(pCipherTxt, k, &tmpBN); + if( 0 <= cpCmp_BNU(BN_NUMBER(&tmpBN), BN_SIZE(&tmpBN), + MOD_MODULUS(RSA_PRV_KEY_NMONT(pKey)), nsN) ) + return 0; + + if(RSA_PRV_KEY1_VALID_ID(pKey)) + gsRSAprv_cipher(&tmpBN, &tmpBN, pKey, pBuffer); + else + gsRSAprv_cipher_crt(&tmpBN, &tmpBN, pKey, pBuffer); + + ippsGetOctString_BN((Ipp8u*)(BN_BUFFER(&tmpBN)), k, &tmpBN); + + /* EME-PKCS-v1_5 decoding */ + return DecodeEME_PKCSv15((Ipp8u*)(BN_BUFFER(&tmpBN)), k, pMsg, pMsgLen); +} + +/*F* +// Name: ippsRSADecrypt_PKCSv15 +// +// Purpose: Performs Decrption according to RSA-ES-PKCS1_v1.5 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pSrc +// NULL == pDst +// NULL == pDstLen +// NULL == pKey +// NULL == pBuffer +// +// ippStsContextMatchErr !RSA_PRV_KEY_VALID_ID() +// +// ippStsIncompleteContextErr private key is not set up +// +// ippStsSizeErr RSA modulus size too short (k < 11, see PKCS#1) !! runtime error +// ippStsPaddingErr pSrc > size of RSA modulus !! runtime error +// DecodeEME_PKCS_v1_5 error +// ippStsNoErr no error +// +// Parameters: +// pSrc pointer to the ciphertext +// pDst pointer to the plaintext +// pDstLen pointer to the length (bytes) of decrypted plaintext +// pKey pointer to the RSA private key context +// pBuffer pointer to scratch buffer +*F*/ +IPPFUN(IppStatus, ippsRSADecrypt_PKCSv15,(const Ipp8u* pSrc, + Ipp8u* pDst, int* pDstLen, + const IppsRSAPrivateKeyState* pKey, + Ipp8u* pScratchBuffer)) +{ + /* use aligned key context */ + IPP_BAD_PTR2_RET(pKey, pScratchBuffer); + IPP_BADARG_RET(!RSA_PRV_KEY_VALID_ID(pKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PRV_KEY_IS_SET(pKey), ippStsIncompleteContextErr); + + /* test data pointer */ + IPP_BAD_PTR3_RET(pSrc, pDst, pDstLen); + + if(RSA_PRV_KEY_BITSIZE_N(pKey) < 11*BYTESIZE) + return ippStsSizeErr; + + { + BNU_CHUNK_T* pBuffer = (BNU_CHUNK_T*)(IPP_ALIGNED_PTR((pScratchBuffer), (int)sizeof(BNU_CHUNK_T))); + return Decryption(pSrc, pDst, pDstLen, + pKey, + pBuffer)? ippStsNoErr : ippStsPaddingErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsaencrypt_oaep.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsaencrypt_oaep.c new file mode 100644 index 0000000..01229a5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsaencrypt_oaep.c @@ -0,0 +1,159 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSAES-OAEP Encryption/Decription Functions +// +// Contents: +// ippsRSAEncrypt_OAEP() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcptool.h" +#include "pcpngrsa.h" +#include "pcphash.h" + +/*F* +// Name: ippsRSAEncrypt_OAEP +// +// Purpose: Performs RSAES-OAEP encryprion scheme +// +// Returns: Reason: +// ippStsNotSupportedModeErr unknown hashAlg +// +// ippStsNullPtrErr NULL == pKey +// NULL == pSrc +// NULL == pDst +// NULL == pLabel +// NULL == pSeed +// NULL == pBuffer +// +// ippStsLengthErr srcLen <0 +// labLen <0 +// srcLen > RSAsize -2*hashLen -2 +// RSAsize < 2*hashLen +2 +// +// ippStsContextMatchErr !RSA_PUB_KEY_VALID_ID() +// +// ippStsIncompleteContextErr public key is not set up +// +// ippStsNoErr no error +// +// Parameters: +// pSrc pointer to the plaintext +// srcLen plaintext length (bytes) +// pLabel (optional) pointer to the label associated with plaintext +// labLen label length (bytes) +// pSeed seed string of hashLen size +// pDst pointer to the ciphertext (length of pdst is not less then size of RSA modulus) +// pKey pointer to the RSA public key context +// hashAlg hash alg ID +// pBuffer pointer to scratch buffer +*F*/ +IPPFUN(IppStatus, ippsRSAEncrypt_OAEP,(const Ipp8u* pSrc, int srcLen, + const Ipp8u* pLabel, int labLen, + const Ipp8u* pSeed, + Ipp8u* pDst, + const IppsRSAPublicKeyState* pKey, + IppHashAlgId hashAlg, + Ipp8u* pScratchBuffer)) +{ + int hashLen; + + /* test hash algorith ID */ + hashAlg = cpValidHashAlg(hashAlg); + IPP_BADARG_RET(ippHashAlg_Unknown==hashAlg, ippStsNotSupportedModeErr); + + /* test data pointer */ + IPP_BAD_PTR3_RET(pSrc,pDst, pSeed); + + IPP_BADARG_RET(!pLabel && labLen, ippStsNullPtrErr); + + /* test public key context */ + IPP_BAD_PTR2_RET(pKey, pScratchBuffer); + IPP_BADARG_RET(!RSA_PUB_KEY_VALID_ID(pKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PUB_KEY_IS_SET(pKey), ippStsIncompleteContextErr); + + /* test length */ + IPP_BADARG_RET(srcLen<0||labLen<0, ippStsLengthErr); + + hashLen = cpHashSize(hashAlg); + /* test compatibility of RSA and hash length */ + IPP_BADARG_RET(BITS2WORD8_SIZE(RSA_PRV_KEY_BITSIZE_N(pKey)) < (2*hashLen +2), ippStsLengthErr); + /* test compatibility of msg length and other (RSA and hash) lengths */ + IPP_BADARG_RET(BITS2WORD8_SIZE(RSA_PRV_KEY_BITSIZE_N(pKey))-(2*hashLen +2) < srcLen, ippStsLengthErr); + + { + /* size of RSA modulus in bytes and chunks */ + int k = BITS2WORD8_SIZE(RSA_PUB_KEY_BITSIZE_N(pKey)); + cpSize nsN = BITS_BNU_CHUNK(RSA_PUB_KEY_BITSIZE_N(pKey)); + + /* + // EME-OAEP encoding + */ + { + Ipp8u seedMask[BITS2WORD8_SIZE(IPP_SHA512_DIGEST_BITSIZE)]; + + Ipp8u* pMaskedSeed = pDst+1; + Ipp8u* pMaskedDB = pDst +hashLen +1; + + pDst[0] = 0; + + /* maskedDB = MGF(seed, k-1-hashLen)*/ + ippsMGF(pSeed, hashLen, pMaskedDB, k-1-hashLen, hashAlg); + + /* seedMask = HASH(pLab) */ + ippsHashMessage(pLabel, labLen, seedMask, hashAlg); + + /* maskedDB ^= concat(HASH(pLab),PS,0x01,pSc) */ + XorBlock(pMaskedDB, seedMask, pMaskedDB, hashLen); + pMaskedDB[k-srcLen-hashLen-2] ^= 0x01; + XorBlock(pMaskedDB+k-srcLen-hashLen-2+1, pSrc, pMaskedDB+k-srcLen-hashLen-2+1, srcLen); + + /* seedMask = MGF(maskedDB, hashLen) */ + ippsMGF(pMaskedDB, k-1-hashLen, seedMask, hashLen, hashAlg); + /* maskedSeed = seed ^ seedMask */ + XorBlock(pSeed, seedMask, pMaskedSeed, hashLen); + } + + /* RSA encryption */ + { + /* align buffer */ + BNU_CHUNK_T* pBuffer = (BNU_CHUNK_T*)(IPP_ALIGNED_PTR(pScratchBuffer, (int)sizeof(BNU_CHUNK_T)) ); + + /* temporary BN */ + __ALIGN8 IppsBigNumState tmpBN; + BN_Make(pBuffer, pBuffer+nsN+1, nsN, &tmpBN); + + /* updtae buffer pointer */ + pBuffer += (nsN+1)*2; + + ippsSetOctString_BN(pDst, k, &tmpBN); + + gsRSApub_cipher(&tmpBN, &tmpBN, pKey, pBuffer); + + ippsGetOctString_BN(pDst, k, &tmpBN); + } + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsaencrypt_oaep_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsaencrypt_oaep_rmf.c new file mode 100644 index 0000000..08dd1b7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsaencrypt_oaep_rmf.c @@ -0,0 +1,153 @@ +/******************************************************************************* +* Copyright (C) 2016 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Cryptography Primitive. +// RSAES-OAEP Encryption/Decription Functions +// +// Contents: +// ippsRSAEncrypt_OAEP_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcptool.h" +#include "pcpngrsa.h" +#include "pcphash_rmf.h" + +/*F* +// Name: ippsRSAEncrypt_OAEP_rmf +// +// Purpose: Performs RSAES-OAEP encryprion scheme +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pKey +// NULL == pSrc +// NULL == pDst +// NULL == pLabel +// NULL == pSeed +// NULL == pMethod +// NULL == pBuffer +// +// ippStsLengthErr srcLen <0 +// labLen <0 +// srcLen > RSAsize -2*hashLen -2 +// RSAsize < 2*hashLen +2 +// +// ippStsContextMatchErr !RSA_PUB_KEY_VALID_ID() +// +// ippStsIncompleteContextErr public key is not set up +// +// ippStsNoErr no error +// +// Parameters: +// pSrc pointer to the plaintext +// srcLen plaintext length (bytes) +// pLabel (optional) pointer to the label associated with plaintext +// labLen label length (bytes) +// pSeed seed string of hashLen size +// pDst pointer to the ciphertext (length of pdst is not less then size of RSA modulus) +// pKey pointer to the RSA public key context +// pMethod hash methods +// pBuffer pointer to scratch buffer +*F*/ +IPPFUN(IppStatus, ippsRSAEncrypt_OAEP_rmf,(const Ipp8u* pSrc, int srcLen, + const Ipp8u* pLabel, int labLen, + const Ipp8u* pSeed, + Ipp8u* pDst, + const IppsRSAPublicKeyState* pKey, + const IppsHashMethod* pMethod, + Ipp8u* pScratchBuffer)) +{ + int hashLen; + + /* test data pointer */ + IPP_BAD_PTR4_RET(pSrc, pDst, pSeed, pMethod); + + IPP_BADARG_RET(!pLabel && labLen, ippStsNullPtrErr); + + /* test public key context */ + IPP_BAD_PTR2_RET(pKey, pScratchBuffer); + IPP_BADARG_RET(!RSA_PUB_KEY_VALID_ID(pKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PUB_KEY_IS_SET(pKey), ippStsIncompleteContextErr); + + /* test length */ + IPP_BADARG_RET(srcLen<0||labLen<0, ippStsLengthErr); + + hashLen = pMethod->hashLen; + /* test compatibility of RSA and hash length */ + IPP_BADARG_RET(BITS2WORD8_SIZE(RSA_PRV_KEY_BITSIZE_N(pKey)) < (2*hashLen +2), ippStsLengthErr); + /* test compatibility of msg length and other (RSA and hash) lengths */ + IPP_BADARG_RET(BITS2WORD8_SIZE(RSA_PRV_KEY_BITSIZE_N(pKey))-(2*hashLen +2) < srcLen, ippStsLengthErr); + + { + /* size of RSA modulus in bytes and chunks */ + int k = BITS2WORD8_SIZE(RSA_PUB_KEY_BITSIZE_N(pKey)); + cpSize nsN = BITS_BNU_CHUNK(RSA_PUB_KEY_BITSIZE_N(pKey)); + + /* + // EME-OAEP encoding + */ + { + Ipp8u seedMask[BITS2WORD8_SIZE(IPP_SHA512_DIGEST_BITSIZE)]; + + Ipp8u* pMaskedSeed = pDst+1; + Ipp8u* pMaskedDB = pDst +hashLen +1; + + pDst[0] = 0; + + /* maskedDB = MGF(seed, k-1-hashLen)*/ + ippsMGF1_rmf(pSeed, hashLen, pMaskedDB, k-1-hashLen, pMethod); + + /* seedMask = HASH(pLab) */ + ippsHashMessage_rmf(pLabel, labLen, seedMask, pMethod); + + /* maskedDB ^= concat(HASH(pLab),PS,0x01,pSc) */ + XorBlock(pMaskedDB, seedMask, pMaskedDB, hashLen); + pMaskedDB[k-srcLen-hashLen-2] ^= 0x01; + XorBlock(pMaskedDB+k-srcLen-hashLen-2+1, pSrc, pMaskedDB+k-srcLen-hashLen-2+1, srcLen); + + /* seedMask = MGF(maskedDB, hashLen) */ + ippsMGF1_rmf(pMaskedDB, k-1-hashLen, seedMask, hashLen, pMethod); + /* maskedSeed = seed ^ seedMask */ + XorBlock(pSeed, seedMask, pMaskedSeed, hashLen); + } + + /* RSA encryption */ + { + /* align buffer */ + BNU_CHUNK_T* pBuffer = (BNU_CHUNK_T*)(IPP_ALIGNED_PTR(pScratchBuffer, (int)sizeof(BNU_CHUNK_T)) ); + + /* temporary BN */ + __ALIGN8 IppsBigNumState tmpBN; + BN_Make(pBuffer, pBuffer+nsN+1, nsN, &tmpBN); + + /* updtae buffer pointer */ + pBuffer += (nsN+1)*2; + + ippsSetOctString_BN(pDst, k, &tmpBN); + + gsRSApub_cipher(&tmpBN, &tmpBN, pKey, pBuffer); + + ippsGetOctString_BN(pDst, k, &tmpBN); + } + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsaencrypt_pkcsv15.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsaencrypt_pkcsv15.c new file mode 100644 index 0000000..c456a52 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsaencrypt_pkcsv15.c @@ -0,0 +1,136 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSA-PKCS1-v1_5 Encryption Scheme +// +// Contents: +// ippsRSAEncrypt_PKCSv15() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpngrsa.h" +#include "pcptool.h" + + +static int EncodeEME_PKCSv15(const Ipp8u* msg, Ipp32u msgLen, + const Ipp8u* rndPS, + Ipp8u* pEM, Ipp32u lenEM) +{ + /* + // encoded message format: + // EM = 00 || 02 || PS || 00 || Msg + // len(PS) >= 8 + */ + Ipp32u psLen = lenEM - msgLen - 3; + + pEM[0] = 0x00; + pEM[1] = 0x02; + if(rndPS) + CopyBlock(rndPS, pEM+2, (cpSize)psLen); + else + PadBlock(0xFF, pEM+2, (cpSize)psLen); + pEM[2+psLen] = 0x00; + CopyBlock(msg, pEM+3+psLen, (cpSize)msgLen); + return 1; +} + + +static int Encryption(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pRndPS, + Ipp8u* pCipherTxt, + const IppsRSAPublicKeyState* pKey, + BNU_CHUNK_T* pBuffer) +{ + /* size of RSA modulus in bytes and chunks */ + int k= BITS2WORD8_SIZE(RSA_PUB_KEY_BITSIZE_N(pKey)); + cpSize nsN = BITS_BNU_CHUNK(RSA_PUB_KEY_BITSIZE_N(pKey)); + + if( (msgLen+11)<=k ) { + /* temporary BN */ + __ALIGN8 IppsBigNumState tmpBN; + BN_Make(pBuffer, pBuffer+nsN, nsN, &tmpBN); + + /* EME-PKCS-v1_5 encoding */ + EncodeEME_PKCSv15(pMsg, (Ipp32u)msgLen, pRndPS, (Ipp8u*)(BN_BUFFER(&tmpBN)), (Ipp32u)k); + /* + // public-key operation + */ + ippsSetOctString_BN((Ipp8u*)(BN_BUFFER(&tmpBN)), k, &tmpBN); + gsRSApub_cipher(&tmpBN, &tmpBN, pKey, pBuffer+nsN*2); + + /* convert into the cipher text */ + ippsGetOctString_BN(pCipherTxt, k, &tmpBN); + return 1; + } + else + return 0; +} + +/*F* +// Name: ippsRSAEncrypt_PKCSv15 +// +// Purpose: Performs Encrption according to RSA-ES-PKCS1_v1.5 +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pSrc +// NULL == pDst +// NULL == pKey +// NULL == pBuffer +// +// ippStsContextMatchErr !RSA_PUB_KEY_VALID_ID() +// +// ippStsIncompleteContextErr public key is not set up +// +// //ippStsLengthErr 0 == srcLen +// ippStsSizeErr RSA modulus too short (srcLen > k-11, see PKCS#1) !!runtime error +// ippStsNoErr no error +// +// Parameters: +// pSrc pointer to the plaintext to be encrypted +// srcLen plaintext length (bytes) +// pRandPS pointer to the random nonzero string of suitable length (psLen >= k -3 -srcLen) +// pDst pointer to the ciphertext (k bytes length, == length of RSA modulus +// pKey pointer to the public key context context +// pBuffer pointer to scratch buffer +*F*/ +IPPFUN(IppStatus, ippsRSAEncrypt_PKCSv15,(const Ipp8u* pSrc, int srcLen, + const Ipp8u* pRndPS, + Ipp8u* pDst, + const IppsRSAPublicKeyState* pKey, + Ipp8u* pScratchBuffer)) +{ + /* test public key context */ + IPP_BAD_PTR2_RET(pKey, pScratchBuffer); + IPP_BADARG_RET(!RSA_PUB_KEY_VALID_ID(pKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PUB_KEY_IS_SET(pKey), ippStsIncompleteContextErr); + + /* test data pointer */ + IPP_BAD_PTR2_RET(pSrc, pDst); + + { + BNU_CHUNK_T* pBuffer = (BNU_CHUNK_T*)(IPP_ALIGNED_PTR((pScratchBuffer), (int)sizeof(BNU_CHUNK_T))); + return Encryption(pSrc, srcLen, pRndPS, pDst, + pKey, + pBuffer)? ippStsNoErr : ippStsSizeErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsasign_pkcs1v15.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsasign_pkcs1v15.c new file mode 100644 index 0000000..ce74772 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsasign_pkcs1v15.c @@ -0,0 +1,86 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSASSA-PKCS-v1_5 +// +// Signatire Scheme with Appendix Signatute Generation +// +// Contents: +// ippsRSASign_PKCS1v15() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpngrsa.h" +#include "pcphash.h" +#include "pcptool.h" + +#include "pcprsa_pkcs1c15_data.h" +#include "pcprsa_generatesign_pkcs1v15.h" + +IPPFUN(IppStatus, ippsRSASign_PKCS1v15,(const Ipp8u* pMsg, int msgLen, + Ipp8u* pSign, + const IppsRSAPrivateKeyState* pPrvKey, + const IppsRSAPublicKeyState* pPubKey, + IppHashAlgId hashAlg, + Ipp8u* pScratchBuffer)) +{ + /* test private key context */ + IPP_BAD_PTR2_RET(pPrvKey, pScratchBuffer); + IPP_BADARG_RET(!RSA_PRV_KEY_VALID_ID(pPrvKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PRV_KEY_IS_SET(pPrvKey), ippStsIncompleteContextErr); + + /* test hash algorith ID */ + hashAlg = cpValidHashAlg(hashAlg); + IPP_BADARG_RET(ippHashAlg_Unknown==hashAlg, ippStsNotSupportedModeErr); + IPP_BADARG_RET(ippHashAlg_SM3==hashAlg, ippStsNotSupportedModeErr); + + /* use aligned public key context if defined */ + if(pPubKey) { + IPP_BADARG_RET(!RSA_PUB_KEY_VALID_ID(pPubKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PUB_KEY_IS_SET(pPubKey), ippStsIncompleteContextErr); + } + + /* test data pointer */ + IPP_BAD_PTR2_RET(pMsg, pSign); + /* test length */ + IPP_BADARG_RET(msgLen<0, ippStsLengthErr); + + { + Ipp8u md[IPP_SHA512_DIGEST_BITSIZE/BYTESIZE]; + int mdLen = cpHashSize(hashAlg); + ippsHashMessage(pMsg, msgLen, md, hashAlg); + + { + const Ipp8u* pSalt = pksc15_salt[hashAlg].pSalt; + int saltLen = pksc15_salt[hashAlg].saltLen; + + int sts = GenerateSign(md, mdLen, + pSalt, saltLen, + pSign, + pPrvKey, pPubKey, + (BNU_CHUNK_T*)(IPP_ALIGNED_PTR((pScratchBuffer), (int)sizeof(BNU_CHUNK_T)))); + + return (1==sts)? ippStsNoErr : ippStsSizeErr; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsasign_pkcs1v15_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsasign_pkcs1v15_rmf.c new file mode 100644 index 0000000..01e5b76 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsasign_pkcs1v15_rmf.c @@ -0,0 +1,73 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSASSA-PKCS-v1_5 +// +// Signatire Scheme with Appendix Signatute Generation +// +// Contents: +// ippsRSASign_PKCS1v15_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpngrsa.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +#include "pcprsa_pkcs1c15_data.h" +#include "pcprsa_generatesign_pkcs1v15.h" +#include "pcprsa_pkcs1v15_preproc.h" + +IPPFUN(IppStatus, ippsRSASign_PKCS1v15_rmf,(const Ipp8u* pMsg, int msgLen, + Ipp8u* pSign, + const IppsRSAPrivateKeyState* pPrvKey, + const IppsRSAPublicKeyState* pPubKey, + const IppsHashMethod* pMethod, + Ipp8u* pScratchBuffer)) +{ + const IppStatus preprocResult = SingleSignPkcs1v15RmfPreproc(pMsg, msgLen, pSign, + &pPrvKey, &pPubKey, pMethod, pScratchBuffer); // badargs and pointer alignments + + if (ippStsNoErr != preprocResult) { + return preprocResult; + } + + { + Ipp8u md[IPP_SHA512_DIGEST_BITSIZE/BYTESIZE]; + int mdLen = pMethod->hashLen; + ippsHashMessage_rmf(pMsg, msgLen, md, pMethod); + + { + const Ipp8u* pSalt = pksc15_salt[pMethod->hashAlgId].pSalt; + int saltLen = pksc15_salt[pMethod->hashAlgId].saltLen; + + int sts = GenerateSign(md, mdLen, + pSalt, saltLen, + pSign, + pPrvKey, pPubKey, + (BNU_CHUNK_T*)(IPP_ALIGNED_PTR((pScratchBuffer), (int)sizeof(BNU_CHUNK_T)))); + + return (1==sts)? ippStsNoErr : ippStsSizeErr; + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsasign_pss.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsasign_pss.c new file mode 100644 index 0000000..e66c908 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsasign_pss.c @@ -0,0 +1,218 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSASSA-PSS +// +// Signatire Scheme with Appendix Signatute Generation +// (Ppobabilistic Signature Scheme) +// +// Contents: +// ippsRSASign_PSS() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpngrsa.h" +#include "pcphash.h" +#include "pcptool.h" + +/*F* +// Name: ippsRSASign_PSS +// +// Purpose: Performs Signature Generation according to RSASSA-PSS +// +// Returns: Reason: +// ippStsNotSupportedModeErr invalid hashAlg value +// +// ippStsNullPtrErr NULL == pMsg +// NULL == pSalt +// NULL == pSign +// NULL == pPrvKey +// NULL == pPubKey +// NULL == pBuffer +// +// ippStsLengthErr msgLen<0 +// saltLen<0 +// emLen < (hashLen +saltLen +2), +// where emLen = (BITSIZE(RSA)-1)/8 +// +// ippStsContextMatchErr !RSA_PRV_KEY_VALID_ID() +// !RSA_PUB_KEY_VALID_ID() +// +// ippStsIncompleteContextErr private or/and public key is not set up +// +// ippStsNoErr no error +// +// Parameters: +// pMsg pointer to the message to be signed +// msgLen lenfth of the message +// pSalt "salt" pointer to random string +// saltLen length of the "salt" string (bytes) +// pSign pointer to the signature string of the RSA length +// pPrvKey pointer to the RSA private key context +// pPubKey (optional) pointer to the RSA public key context +// hashAlg hash ID +// pBuffer pointer to scratch buffer +*F*/ +IPPFUN(IppStatus, ippsRSASign_PSS,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pSalt, int saltLen, + Ipp8u* pSign, + const IppsRSAPrivateKeyState* pPrvKey, + const IppsRSAPublicKeyState* pPubKey, + IppHashAlgId hashAlg, + Ipp8u* pScratchBuffer)) +{ + /* test hash algorith ID */ + hashAlg = cpValidHashAlg(hashAlg); + IPP_BADARG_RET(ippHashAlg_Unknown==hashAlg, ippStsNotSupportedModeErr); + + /* test message length */ + IPP_BADARG_RET((msgLen<0), ippStsLengthErr); + /* test message pointer */ + IPP_BADARG_RET((msgLen && !pMsg), ippStsNullPtrErr); + + /* test data pointer */ + IPP_BAD_PTR1_RET(pSign); + + /* test salt length and salt pointer */ + IPP_BADARG_RET(saltLen<0, ippStsLengthErr); + IPP_BADARG_RET((saltLen && !pSalt), ippStsNullPtrErr); + + /* test private key context */ + IPP_BAD_PTR2_RET(pPrvKey, pScratchBuffer); + IPP_BADARG_RET(!RSA_PRV_KEY_VALID_ID(pPrvKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PRV_KEY_IS_SET(pPrvKey), ippStsIncompleteContextErr); + + /* use public key context if defined */ + if(pPubKey) { + IPP_BADARG_RET(!RSA_PUB_KEY_VALID_ID(pPubKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PUB_KEY_IS_SET(pPubKey), ippStsIncompleteContextErr); + } + + { + Ipp8u hashMsg[MAX_HASH_SIZE]; + + /* hash length */ + int hashLen = cpHashSize(hashAlg); + + /* size of RSA modulus in bytes and chunks */ + cpSize rsaBits = RSA_PRV_KEY_BITSIZE_N(pPrvKey); + cpSize k = BITS2WORD8_SIZE(rsaBits); + cpSize nsN = BITS_BNU_CHUNK(rsaBits); + + /* align buffer */ + BNU_CHUNK_T* pBuffer = (BNU_CHUNK_T*)(IPP_ALIGNED_PTR(pScratchBuffer, (int)sizeof(BNU_CHUNK_T)) ); + + /* temporary BNs */ + __ALIGN8 IppsBigNumState bnC; + __ALIGN8 IppsBigNumState bnP; + + /* message presentative size */ + int emBits = rsaBits-1; + int emLen = BITS2WORD8_SIZE(emBits); + + /* size of padding string (PS) */ + int psLen = emLen -hashLen -saltLen -2; + + /* test size consistence */ + if(0 > psLen) + IPP_ERROR_RET(ippStsLengthErr); + + /* compute hash of the message */ + ippsHashMessage(pMsg, msgLen, hashMsg, hashAlg); + + /* make BNs */ + BN_Make(pBuffer, pBuffer+nsN+1, nsN, &bnC); + pBuffer += (nsN+1)*2; + BN_Make(pBuffer, pBuffer+nsN+1, nsN, &bnP); + pBuffer += (nsN+1)*2; + + /* + // EMSA-PSS encoding + */ + { + Ipp8u* pM = (Ipp8u*)BN_NUMBER(&bnP); + Ipp8u* pEM = pSign; + Ipp8u* pDB = pSign; + int dbLen = emLen-hashLen-1; + Ipp8u* pH = pSign+dbLen; + + /* construct message M' + // M' = (00 00 00 00 00 00 00 00) || mHash || salt + // where: + // mHash = HASH(pMsg) + */ + PadBlock(0, pM, 8); + CopyBlock(hashMsg, pM+8, hashLen); + CopyBlock(pSalt, pM+8+hashLen, saltLen); + + /* construct EM + // EM = maskedDB || H || 0xBC + // where: + // H = HASH(M') + // maskedDB = DB ^ MGF(H) + // where: + // DB = PS || 0x01 || salt + // + // by other words + // EM = (dbMask ^ (PS || 0x01 || salt)) || HASH(M) || 0xBC + */ + pEM[emLen-1] = 0xBC; /* tail octet */ + ippsHashMessage(pM, 8+hashLen+saltLen, pH, hashAlg); /* H = HASH(M) */ + ippsMGF(pH, hashLen, pDB, dbLen, hashAlg); /* dbMask = MGF(H) */ + + XorBlock(pDB+psLen+1, pSalt, pDB+psLen+1, saltLen); + pDB[psLen] ^= 0x01; + + /* make sure that top 8*emLen-emBits bits are clear */ + pDB[0] &= MAKEMASK32(8-8*emLen+emBits); + } + + /* + // private-key operation + */ + ippsSetOctString_BN(pSign, emLen, &bnC); + + if(RSA_PRV_KEY1_VALID_ID(pPrvKey)) + gsRSAprv_cipher(&bnP, &bnC, pPrvKey, pBuffer); + else + gsRSAprv_cipher_crt(&bnP, &bnC, pPrvKey, pBuffer); + + ippsGetOctString_BN(pSign, k, &bnP); + + /* no check requested */ + if(!pPubKey) + return ippStsNoErr; + + /* check the result before send it out (fault attack mitigatioin) */ + else { + gsRSApub_cipher(&bnP, &bnP, pPubKey, pBuffer); + if(0==cpBN_cmp(&bnP, &bnC)) + return ippStsNoErr; + /* discard signature if check failed */ + else { + PadBlock(0, pSign, k); + return ippStsErr; + } + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsasign_pss_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsasign_pss_rmf.c new file mode 100644 index 0000000..61d5951 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsasign_pss_rmf.c @@ -0,0 +1,201 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSASSA-PSS +// +// Signatire Scheme with Appendix Signatute Generation +// (Ppobabilistic Signature Scheme) +// +// Contents: +// ippsRSASign_PSS_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpngrsa.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +#include "pcprsa_pss_preproc.h" + +/*F* +// Name: ippsRSASign_PSS_rmf +// +// Purpose: Performs Signature Generation according to RSASSA-PSS +// +// Returns: Reason: +// ippStsNotSupportedModeErr invalid hashAlg value +// +// ippStsNullPtrErr NULL == pMsg +// NULL == pSalt +// NULL == pSign +// NULL == pPrvKey +// NULL == pPubKey +// NULL == pMethod +// NULL == pBuffer +// +// ippStsLengthErr msgLen<0 +// saltLen<0 +// emLen < (hashLen +saltLen +2), +// where emLen = (BITSIZE(RSA)-1)/8 +// +// ippStsContextMatchErr !RSA_PRV_KEY_VALID_ID() +// !RSA_PUB_KEY_VALID_ID() +// +// ippStsIncompleteContextErr private or/and public key is not set up +// +// ippStsNoErr no error +// +// Parameters: +// pMsg pointer to the message to be signed +// msgLen lenfth of the message +// pSalt "salt" pointer to random string +// saltLen length of the "salt" string (bytes) +// pSign pointer to the signature string of the RSA length +// pPrvKey pointer to the RSA private key context +// pPubKey (optional) pointer to the RSA public key context +// pMethod hash method +// pBuffer pointer to scratch buffer +*F*/ +IPPFUN(IppStatus, ippsRSASign_PSS_rmf,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pSalt, int saltLen, + Ipp8u* pSign, + const IppsRSAPrivateKeyState* pPrvKey, + const IppsRSAPublicKeyState* pPubKey, + const IppsHashMethod* pMethod, + Ipp8u* pScratchBuffer)) +{ + const IppStatus preprocResult = SingleSignPssRmfPreproc(pMsg, msgLen, pSalt, saltLen, + pSign, &pPrvKey, &pPubKey, pMethod, pScratchBuffer); // badargs and pointer alignments + + if (ippStsNoErr != preprocResult) { + return preprocResult; + } + + { + Ipp8u hashMsg[MAX_HASH_SIZE]; + + /* hash length */ + int hashLen = pMethod->hashLen; + + /* size of RSA modulus in bytes and chunks */ + cpSize rsaBits = RSA_PRV_KEY_BITSIZE_N(pPrvKey); + cpSize k = BITS2WORD8_SIZE(rsaBits); + cpSize nsN = BITS_BNU_CHUNK(rsaBits); + + /* align buffer */ + BNU_CHUNK_T* pBuffer = (BNU_CHUNK_T*)(IPP_ALIGNED_PTR(pScratchBuffer, (int)sizeof(BNU_CHUNK_T)) ); + + /* temporary BNs */ + __ALIGN8 IppsBigNumState bnC; + __ALIGN8 IppsBigNumState bnP; + + /* message presentative size */ + int emBits = rsaBits-1; + int emLen = BITS2WORD8_SIZE(emBits); + + /* size of padding string (PS) */ + int psLen = emLen -hashLen -saltLen -2; + + /* test size consistence */ + if(0 > psLen) + IPP_ERROR_RET(ippStsLengthErr); + + /* compute hash of the message */ + ippsHashMessage_rmf(pMsg, msgLen, hashMsg, pMethod); + + /* make BNs */ + BN_Make(pBuffer, pBuffer+nsN+1, nsN, &bnC); + pBuffer += (nsN+1)*2; + BN_Make(pBuffer, pBuffer+nsN+1, nsN, &bnP); + pBuffer += (nsN+1)*2; + + /* + // EMSA-PSS encoding + */ + { + Ipp8u* pM = (Ipp8u*)BN_NUMBER(&bnP); + Ipp8u* pEM = pSign; + Ipp8u* pDB = pSign; + int dbLen = emLen-hashLen-1; + Ipp8u* pH = pSign+dbLen; + + /* construct message M' + // M' = (00 00 00 00 00 00 00 00) || mHash || salt + // where: + // mHash = HASH(pMsg) + */ + PadBlock(0, pM, 8); + CopyBlock(hashMsg, pM+8, hashLen); + CopyBlock(pSalt, pM+8+hashLen, saltLen); + + /* construct EM + // EM = maskedDB || H || 0xBC + // where: + // H = HASH(M') + // maskedDB = DB ^ MGF(H) + // where: + // DB = PS || 0x01 || salt + // + // by other words + // EM = (dbMask ^ (PS || 0x01 || salt)) || HASH(M) || 0xBC + */ + pEM[emLen-1] = 0xBC; /* tail octet */ + ippsHashMessage_rmf(pM, 8+hashLen+saltLen, pH, pMethod); /* H = HASH(M) */ + ippsMGF1_rmf(pH, hashLen, pDB, dbLen, pMethod); /* dbMask = MGF(H) */ + + XorBlock(pDB+psLen+1, pSalt, pDB+psLen+1, saltLen); + pDB[psLen] ^= 0x01; + + /* make sure that top 8*emLen-emBits bits are clear */ + pDB[0] &= MAKEMASK32(8-8*emLen+emBits); + } + + /* + // private-key operation + */ + ippsSetOctString_BN(pSign, emLen, &bnC); + + if(RSA_PRV_KEY1_VALID_ID(pPrvKey)) + gsRSAprv_cipher(&bnP, &bnC, pPrvKey, pBuffer); + else + gsRSAprv_cipher_crt(&bnP, &bnC, pPrvKey, pBuffer); + + ippsGetOctString_BN(pSign, k, &bnP); + + /* no check requested */ + if(!pPubKey) + return ippStsNoErr; + + /* check the result before send it out (fault attack mitigatioin) */ + else { + gsRSApub_cipher(&bnP, &bnP, pPubKey, pBuffer); + if(0==cpBN_cmp(&bnP, &bnC)) + return ippStsNoErr; + /* discard signature if check failed */ + else { + PadBlock(0, pSign, k); + return ippStsErr; + } + } + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsasignhash_pkcs1v15.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsasignhash_pkcs1v15.c new file mode 100644 index 0000000..c6651b4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsasignhash_pkcs1v15.c @@ -0,0 +1,85 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSASSA-PKCS-v1_5 +// +// Signatire Scheme with Appendix Signatute Generation +// +// Contents: +// ippsRSASignHash_PKCS1v15() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpngrsa.h" +#include "pcphash.h" +#include "pcptool.h" + +#include "pcprsa_pkcs1c15_data.h" +#include "pcprsa_generatesign_pkcs1v15.h" + +#if defined( _ABL_ ) + +IPPFUN(IppStatus, ippsRSASignHash_PKCS1v15,(const Ipp8u* md, int mdLen, + Ipp8u* pSign, + const IppsRSAPrivateKeyState* pPrvKey, + const IppsRSAPublicKeyState* pPubKey, + IppHashAlgId hashAlg, + Ipp8u* pScratchBuffer)) +{ + /* test private key context */ + IPP_BAD_PTR2_RET(pPrvKey, pScratchBuffer); + IPP_BADARG_RET(!RSA_PRV_KEY_VALID_ID(pPrvKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PRV_KEY_IS_SET(pPrvKey), ippStsIncompleteContextErr); + + /* test hash algorith ID */ + hashAlg = cpValidHashAlg(hashAlg); + IPP_BADARG_RET(ippHashAlg_Unknown==hashAlg, ippStsNotSupportedModeErr); + IPP_BADARG_RET(ippHashAlg_SM3==hashAlg, ippStsNotSupportedModeErr); + + /* use public key context if defined */ + if(pPubKey) { + IPP_BADARG_RET(!RSA_PUB_KEY_VALID_ID(pPubKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PUB_KEY_IS_SET(pPubKey), ippStsIncompleteContextErr); + } + + /* test data pointer */ + IPP_BAD_PTR2_RET(md, pSign); + /* test length */ + IPP_BADARG_RET(mdLen!=cpHashSize(hashAlg), ippStsLengthErr); + + { + const Ipp8u* pSalt = pksc15_salt[hashAlg].pSalt; + int saltLen = pksc15_salt[hashAlg].saltLen; + + int sts = GenerateSign(md, mdLen, + pSalt, saltLen, + pSign, + pPrvKey, pPubKey, + (BNU_CHUNK_T*)(IPP_ALIGNED_PTR((pScratchBuffer), (int)sizeof(BNU_CHUNK_T)))); + + return (1==sts)? ippStsNoErr : ippStsSizeErr; + } +} + +#endif /* #if defined( _ABL_ ) */ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsasignhash_pkcs1v15_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsasignhash_pkcs1v15_rmf.c new file mode 100644 index 0000000..a468d46 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsasignhash_pkcs1v15_rmf.c @@ -0,0 +1,84 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSASSA-PKCS-v1_5 +// +// Signatire Scheme with Appendix Signatute Generation +// +// Contents: +// ippsRSASignHash_PKCS1v15_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpngrsa.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +#include "pcprsa_pkcs1c15_data.h" +#include "pcprsa_generatesign_pkcs1v15.h" + +#if defined( _ABL_ ) + +IPPFUN(IppStatus, ippsRSASignHash_PKCS1v15_rmf,(const Ipp8u* md, + Ipp8u* pSign, + const IppsRSAPrivateKeyState* pPrvKey, + const IppsRSAPublicKeyState* pPubKey, + const IppsHashMethod* pMethod, + Ipp8u* pScratchBuffer)) +{ + IppHashAlgId hashAlg; + + /* test private key context */ + IPP_BAD_PTR3_RET(pPrvKey, pScratchBuffer, pMethod); + IPP_BADARG_RET(!RSA_PRV_KEY_VALID_ID(pPrvKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PRV_KEY_IS_SET(pPrvKey), ippStsIncompleteContextErr); + + /* test hash algorith ID */ + hashAlg = pMethod->hashAlgId; + IPP_BADARG_RET(ippHashAlg_SM3==hashAlg, ippStsNotSupportedModeErr); + + /* use aligned public key context if defined */ + if(pPubKey) { + IPP_BADARG_RET(!RSA_PUB_KEY_VALID_ID(pPubKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PUB_KEY_IS_SET(pPubKey), ippStsIncompleteContextErr); + } + + /* test data pointer */ + IPP_BAD_PTR2_RET(md, pSign); + + { + const Ipp8u* pSalt = pksc15_salt[hashAlg].pSalt; + int saltLen = pksc15_salt[hashAlg].saltLen; + + int sts = GenerateSign(md, pMethod->hashLen, + pSalt, saltLen, + pSign, + pPrvKey, pPubKey, + (BNU_CHUNK_T*)(IPP_ALIGNED_PTR((pScratchBuffer), (int)sizeof(BNU_CHUNK_T)))); + + return (1==sts)? ippStsNoErr : ippStsSizeErr; + } +} + +#endif /* #if defined( _ABL_ ) */ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsaverify_pkcs1v15.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsaverify_pkcs1v15.c new file mode 100644 index 0000000..246c075 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsaverify_pkcs1v15.c @@ -0,0 +1,73 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSASSA-PKCS-v1_5 +// +// Signatire Scheme with Appendix Signatute Generation +// +// Contents: +// ippsRSAVerify_PKCS1v15() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpngrsa.h" +#include "pcphash.h" +#include "pcptool.h" + +#include "pcprsa_pkcs1c15_data.h" +#include "pcprsa_verifysign_pkcs1v15.h" + +IPPFUN(IppStatus, ippsRSAVerify_PKCS1v15,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pSign, int* pIsValid, + const IppsRSAPublicKeyState* pKey, + IppHashAlgId hashAlg, + Ipp8u* pScratchBuffer)) +{ + /* test public key context */ + IPP_BAD_PTR2_RET(pKey, pScratchBuffer); + IPP_BADARG_RET(!RSA_PUB_KEY_VALID_ID(pKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PUB_KEY_IS_SET(pKey), ippStsIncompleteContextErr); + + /* test hash algorith ID */ + hashAlg = cpValidHashAlg(hashAlg); + IPP_BADARG_RET(ippHashAlg_Unknown==hashAlg, ippStsNotSupportedModeErr); + IPP_BADARG_RET(ippHashAlg_SM3==hashAlg, ippStsNotSupportedModeErr); + + /* test data pointer */ + IPP_BAD_PTR3_RET(pMsg, pSign, pIsValid); + /* test length */ + IPP_BADARG_RET(msgLen<0, ippStsLengthErr); + + *pIsValid = 0; + { + Ipp8u md[IPP_SHA512_DIGEST_BITSIZE/BYTESIZE]; + int mdLen = cpHashSize(hashAlg); + ippsHashMessage(pMsg, msgLen, md, hashAlg); + + return VerifySign(md, mdLen, + pksc15_salt[hashAlg].pSalt, pksc15_salt[hashAlg].saltLen, + pSign, pIsValid, + pKey, + (BNU_CHUNK_T*)(IPP_ALIGNED_PTR((pScratchBuffer), (int)sizeof(BNU_CHUNK_T))))? ippStsNoErr : ippStsSizeErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsaverify_pkcs1v15_rmf.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsaverify_pkcs1v15_rmf.c new file mode 100644 index 0000000..3d17a77 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsaverify_pkcs1v15_rmf.c @@ -0,0 +1,64 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSASSA-PKCS-v1_5 +// +// Signatire Scheme with Appendix Signatute Generation +// +// Contents: +// ippsRSAVerify_PKCS1v15_rmf() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpngrsa.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +#include "pcprsa_pkcs1c15_data.h" +#include "pcprsa_verifysign_pkcs1v15.h" +#include "pcprsa_pkcs1v15_preproc.h" + +IPPFUN(IppStatus, ippsRSAVerify_PKCS1v15_rmf,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pSign, int* pIsValid, + const IppsRSAPublicKeyState* pKey, + const IppsHashMethod* pMethod, + Ipp8u* pScratchBuffer)) +{ + const IppStatus preprocResult = SingleVerifyPkcs1v15RmfPreproc(pMsg, msgLen, pSign, + pIsValid, &pKey, pMethod, pScratchBuffer); // badargs, pointer alignments, set valid = 0 + + if (ippStsNoErr != preprocResult) { + return preprocResult; + } + + { + Ipp8u md[IPP_SHA512_DIGEST_BITSIZE/BYTESIZE]; + ippsHashMessage_rmf(pMsg, msgLen, md, pMethod); + + return VerifySign(md, pMethod->hashLen, + pksc15_salt[pMethod->hashAlgId].pSalt, pksc15_salt[pMethod->hashAlgId].saltLen, + pSign, pIsValid, + pKey, + (BNU_CHUNK_T*)(IPP_ALIGNED_PTR((pScratchBuffer), (int)sizeof(BNU_CHUNK_T))))? ippStsNoErr : ippStsSizeErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsaverify_pss.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsaverify_pss.c new file mode 100644 index 0000000..81bb0c4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcprsaverify_pss.c @@ -0,0 +1,194 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// RSASSA-PSS +// +// Signatire Scheme with Appendix Signatute Generation +// (Ppobabilistic Signature Scheme) +// +// Contents: +// ippsRSAVerify_PSS() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpngrsa.h" +#include "pcphash.h" +#include "pcptool.h" + +//////////////////////////////////////////////////////////////////////////////////////////////////// +/*F* +// Name: ippsRSAVerify_PSS +// +// Purpose: Performs Signature Verification according to RSASSA-PSS +// +// Returns: Reason: +// ippStsNotSupportedModeErr invalid hashAlg value +// +// ippStsNullPtrErr NULL == pMsg +// NULL == pSign +// NULL == pIsValid +// NULL == pKey +// NULL == pBuffer +// +// ippStsLengthErr msgLen<0 +// RSAsize <=hashLen +2 +// +// ippStsContextMatchErr !RSA_PUB_KEY_VALID_ID() +// +// ippStsIncompleteContextErr public key is not set up +// +// ippStsNoErr no error +// +// Parameters: +// pMsg pointer to the message to be verified +// msgLen length of the message +// pSign pointer to the signature string of the RSA length +// pIsValid pointer to the verification result +// pKey pointer to the RSA public key context +// hashAlg hash ID +// pBuffer pointer to scratch buffer +*F*/ +IPPFUN(IppStatus, ippsRSAVerify_PSS,(const Ipp8u* pMsg, int msgLen, + const Ipp8u* pSign, + int* pIsValid, + const IppsRSAPublicKeyState* pKey, + IppHashAlgId hashAlg, + Ipp8u* pScratchBuffer)) +{ + /* test hash algorith ID */ + hashAlg = cpValidHashAlg(hashAlg); + IPP_BADARG_RET(ippHashAlg_Unknown==hashAlg, ippStsNotSupportedModeErr); + + /* test message length */ + IPP_BADARG_RET((msgLen<0), ippStsLengthErr); + /* test message pointer */ + IPP_BADARG_RET((msgLen && !pMsg), ippStsNullPtrErr); + + /* test data pointer */ + IPP_BAD_PTR2_RET(pSign, pIsValid); + + /* test public key context */ + IPP_BAD_PTR2_RET(pKey, pScratchBuffer); + IPP_BADARG_RET(!RSA_PUB_KEY_VALID_ID(pKey), ippStsContextMatchErr); + IPP_BADARG_RET(!RSA_PUB_KEY_IS_SET(pKey), ippStsIncompleteContextErr); + + { + Ipp8u hashMsg[MAX_HASH_SIZE]; + + /* hash length */ + int hashLen = cpHashSize(hashAlg); + + /* size of RSA modulus in bytes and chunks */ + cpSize rsaBits = RSA_PUB_KEY_BITSIZE_N(pKey); + cpSize k = BITS2WORD8_SIZE(rsaBits); + cpSize nsN = BITS_BNU_CHUNK(rsaBits); + + /* align buffer */ + BNU_CHUNK_T* pBuffer = (BNU_CHUNK_T*)(IPP_ALIGNED_PTR(pScratchBuffer, (int)sizeof(BNU_CHUNK_T)) ); + + /* temporary BNs */ + __ALIGN8 IppsBigNumState bnC; + __ALIGN8 IppsBigNumState bnP; + + /* message presentative size */ + int emBits = rsaBits-1; + int emLen = BITS2WORD8_SIZE(emBits); + + /* test size consistence */ + if(k <= (hashLen+2)) + IPP_ERROR_RET(ippStsLengthErr); + + /* compute hash of the message */ + ippsHashMessage(pMsg, msgLen, hashMsg, hashAlg); + + /* make BNs */ + BN_Make(pBuffer, pBuffer+nsN+1, nsN, &bnC); + pBuffer += (nsN+1)*2; + BN_Make(pBuffer, pBuffer+nsN+1, nsN, &bnP); + pBuffer += (nsN+1)*2; + + /* + // public-key operation + */ + ippsSetOctString_BN(pSign, k, &bnP); + gsRSApub_cipher(&bnC, &bnP, pKey, pBuffer); + + *pIsValid = 0; + /* + // EMSA-PSS verification + */ + { + /* convert BN into octet string EM + // EM = maskedDB || H || 0xBC + */ + Ipp8u* pEM = (Ipp8u*)BN_BUFFER(&bnC); + ippsGetOctString_BN(pEM, emLen, &bnC); + + /* test last byte and top of (8*emLen-emBits) bits */ + if(0xBC==pEM[emLen-1] && 0x00==(pEM[0] >>(8-(8*emLen-emBits)))) { + int psLen; + Ipp8u* pM = (Ipp8u*)BN_NUMBER(&bnP); + + /* pointers to the EM fields */ + int dbLen = emLen-hashLen-1; + Ipp8u* pDB = pEM; + Ipp8u* pH = pEM+dbLen; + + /* recover DB = maskedDB ^ MGF(H) */ + ippsMGF(pH, hashLen, pM, dbLen, hashAlg); + XorBlock(pDB, pM, pDB, dbLen); + + /* make sure that top 8*emLen-emBits bits are clear */ + pDB[0] &= MAKEMASK32(8-8*emLen+emBits); + + /* skip over padding sring (PS) */ + for(psLen=0; psLenhashLen; + + /* size of RSA modulus in bytes and chunks */ + cpSize rsaBits = RSA_PUB_KEY_BITSIZE_N(pKey); + cpSize k = BITS2WORD8_SIZE(rsaBits); + cpSize nsN = BITS_BNU_CHUNK(rsaBits); + + /* align buffer */ + BNU_CHUNK_T* pBuffer = (BNU_CHUNK_T*)(IPP_ALIGNED_PTR(pScratchBuffer, (int)sizeof(BNU_CHUNK_T)) ); + + /* temporary BNs */ + __ALIGN8 IppsBigNumState bnC; + __ALIGN8 IppsBigNumState bnP; + + /* message presentative size */ + int emBits = rsaBits-1; + int emLen = BITS2WORD8_SIZE(emBits); + + /* test size consistence */ + if(k <= (hashLen+2)) + IPP_ERROR_RET(ippStsLengthErr); + + /* compute hash of the message */ + ippsHashMessage_rmf(pMsg, msgLen, hashMsg, pMethod); + + /* make BNs */ + BN_Make(pBuffer, pBuffer+nsN+1, nsN, &bnC); + pBuffer += (nsN+1)*2; + BN_Make(pBuffer, pBuffer+nsN+1, nsN, &bnP); + pBuffer += (nsN+1)*2; + + /* + // public-key operation + */ + ippsSetOctString_BN(pSign, k, &bnP); + gsRSApub_cipher(&bnC, &bnP, pKey, pBuffer); + + /* + // EMSA-PSS verification + */ + { + /* convert BN into octet string EM + // EM = maskedDB || H || 0xBC + */ + Ipp8u* pEM = (Ipp8u*)BN_BUFFER(&bnC); + ippsGetOctString_BN(pEM, emLen, &bnC); + + /* test last byte and top of (8*emLen-emBits) bits */ + if(0xBC==pEM[emLen-1] && 0x00==(pEM[0] >>(8-(8*emLen-emBits)))) { + int psLen; + Ipp8u* pM = (Ipp8u*)BN_NUMBER(&bnP); + + /* pointers to the EM fields */ + int dbLen = emLen-hashLen-1; + Ipp8u* pDB = pEM; + Ipp8u* pH = pEM+dbLen; + + /* recover DB = maskedDB ^ MGF(H) */ + ippsMGF1_rmf(pH, hashLen, pM, dbLen, pMethod); + XorBlock(pDB, pM, pDB, dbLen); + + /* make sure that top 8*emLen-emBits bits are clear */ + pDB[0] &= MAKEMASK32(8-8*emLen+emBits); + + /* skip over padding sring (PS) */ + for(psLen=0; psLenhashAlgId; + IPP_BADARG_RET(ippHashAlg_SM3==hashAlg, ippStsNotSupportedModeErr); + + /* test data pointer */ + IPP_BAD_PTR3_RET(md, pSign, pIsValid); + + *pIsValid = 0; + return VerifySign(md, pMethod->hashLen, + pksc15_salt[hashAlg].pSalt, pksc15_salt[hashAlg].saltLen, + pSign, pIsValid, + pKey, + (BNU_CHUNK_T*)(IPP_ALIGNED_PTR((pBuffer), (int)sizeof(BNU_CHUNK_T))))? ippStsNoErr : ippStsSizeErr; +} + +#endif /* #if defined( _ABL_ ) */ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpscramble.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpscramble.h new file mode 100644 index 0000000..42c20a4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpscramble.h @@ -0,0 +1,169 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Fixed window exponentiation scramble/unscramble +// +// Contents: +// cpScramblePut() +// cpScrambleGet() +// +// +*/ + +#if !defined(_PC_SCRAMBLE_H) +#define _PC_SCRAMBLE_H + +/* +// cpsScramblePut/cpsScrambleGet +// stores to/retrieves from pScrambleEntry position +// pre-computed data if fixed window method is used +*/ +__INLINE void cpScramblePut(Ipp8u* pArray, cpSize colummSize, + const Ipp32u* pData, cpSize dataSize) +{ + int i; + switch(colummSize) { + case 1: // column - byte + dataSize *= sizeof(Ipp32u); + for(i=0; i 2 dword) + for(; dataSize>=2; dataSize-=2, pArray+=CACHE_LINE_SIZE, pData+=2) { + ((Ipp32u*)pArray)[0] = pData[0]; + ((Ipp32u*)pArray)[1] = pData[1]; + } + if(dataSize) + ((Ipp32u*)pArray)[0] = pData[0]; + break; + case 16: // column - oword (16 bytes => 4 dword) + for(; dataSize>=4; dataSize-=4, pArray+=CACHE_LINE_SIZE, pData+=4) { + ((Ipp32u*)pArray)[0] = pData[0]; + ((Ipp32u*)pArray)[1] = pData[1]; + ((Ipp32u*)pArray)[2] = pData[2]; + ((Ipp32u*)pArray)[3] = pData[3]; + } + for(; dataSize>0; dataSize--, pArray+=sizeof(Ipp32u), pData++) + ((Ipp32u*)pArray)[0] = pData[0]; + break; + case 32: // column - 2 oword (32 bytes => 8 dword) + for(; dataSize>=8; dataSize-=8, pArray+=CACHE_LINE_SIZE, pData+=8) { + ((Ipp32u*)pArray)[0] = pData[0]; + ((Ipp32u*)pArray)[1] = pData[1]; + ((Ipp32u*)pArray)[2] = pData[2]; + ((Ipp32u*)pArray)[3] = pData[3]; + ((Ipp32u*)pArray)[4] = pData[4]; + ((Ipp32u*)pArray)[5] = pData[5]; + ((Ipp32u*)pArray)[6] = pData[6]; + ((Ipp32u*)pArray)[7] = pData[7]; + } + for(; dataSize>0; dataSize--, pArray+=sizeof(Ipp32u), pData++) + ((Ipp32u*)pArray)[0] = pData[0]; + break; + default: + break; + } +} + + +/* +// Retrieve data from pArray +*/ +#define u8_to_u32(b0,b1,b2,b3, x) \ + ((x) = (b0), \ + (x)|=((b1)<<8), \ + (x)|=((b2)<<16), \ + (x)|=((b3)<<24)) +#define u16_to_u32(w0,w1, x) \ + ((x) = (w0), \ + (x)|=((w1)<<16)) +#define u32_to_u64(dw0,dw1, x) \ + ((x) = (Ipp64u)(dw0), \ + (x)|= (((Ipp64u)(dw1))<<32)) + +__INLINE void cpScrambleGet(Ipp32u* pData, cpSize dataSize, + const Ipp8u* pArray, cpSize colummSize) +{ + int i; + switch(colummSize) { + case 1: // column - byte + for(i=0; i 2 dword) + for(; dataSize>=2; dataSize-=2, pArray+=CACHE_LINE_SIZE, pData+=2) { + pData[0] = ((Ipp32u*)pArray)[0]; + pData[1] = ((Ipp32u*)pArray)[1]; + } + if(dataSize) + pData[0] = ((Ipp32u*)pArray)[0]; + break; + case 16: // column - oword (16 bytes => 4 dword) + for(; dataSize>=4; dataSize-=4, pArray+=CACHE_LINE_SIZE, pData+=4) { + pData[0] = ((Ipp32u*)pArray)[0]; + pData[1] = ((Ipp32u*)pArray)[1]; + pData[2] = ((Ipp32u*)pArray)[2]; + pData[3] = ((Ipp32u*)pArray)[3]; + + } + for(; dataSize>0; dataSize--, pArray+=sizeof(Ipp32u), pData++) + pData[0] = ((Ipp32u*)pArray)[0]; + break; + case 32: // column - 2 oword (32 bytes => 8 dword) + for(; dataSize>=8; dataSize-=8, pArray+=CACHE_LINE_SIZE, pData+=8) { + pData[0] = ((Ipp32u*)pArray)[0]; + pData[1] = ((Ipp32u*)pArray)[1]; + pData[2] = ((Ipp32u*)pArray)[2]; + pData[3] = ((Ipp32u*)pArray)[3]; + pData[4] = ((Ipp32u*)pArray)[4]; + pData[5] = ((Ipp32u*)pArray)[5]; + pData[6] = ((Ipp32u*)pArray)[6]; + pData[7] = ((Ipp32u*)pArray)[7]; + } + for(; dataSize>0; dataSize--, pArray+=sizeof(Ipp32u), pData++) + pData[0] = ((Ipp32u*)pArray)[0]; + break; + default: + break; + } +} + +#endif /* _PC_SCRAMBLE_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1ca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1ca.c new file mode 100644 index 0000000..1bf44e1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1ca.c @@ -0,0 +1,66 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA1 +// +// Contents: +// cpFinalizeSHA1() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha1stuff.h" + + +IPP_OWN_DEFN (void, cpFinalizeSHA1, (DigestSHA1 pHash, const Ipp8u* inpBuffer, int inpLen, Ipp64u processedMsgLen)) +{ + /* select processing function */ + #if (_SHA_NI_ENABLING_==_FEATURE_ON_) + cpHashProc updateFunc = UpdateSHA1ni; + #elif (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_) + cpHashProc updateFunc = IsFeatureEnabled(ippCPUID_SHA)? UpdateSHA1ni : UpdateSHA1; + #else + cpHashProc updateFunc = UpdateSHA1; + #endif + + /* local buffer and it length */ + Ipp8u buffer[MBS_SHA1*2]; + int bufferLen = inpLen < (MBS_SHA1-(int)MLR_SHA1)? MBS_SHA1 : MBS_SHA1*2; + + /* copy rest of message into internal buffer */ + CopyBlock(inpBuffer, buffer, inpLen); + + /* padd message */ + buffer[inpLen++] = 0x80; + PadBlock(0, buffer+inpLen, (cpSize)(bufferLen-inpLen-(int)MLR_SHA1)); + + /* put processed message length in bits */ + processedMsgLen = ENDIANNESS64(processedMsgLen<<3); + ((Ipp64u*)(buffer+bufferLen))[-1] = processedMsgLen; + + /* copmplete hash computation */ + updateFunc(pHash, buffer, bufferLen, sha1_cnt); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1duplicate.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1duplicate.c new file mode 100644 index 0000000..f57b044 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1duplicate.c @@ -0,0 +1,66 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA1 +// +// Contents: +// ippsSHA1Duplicate() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsSHA1Duplicate +// +// Purpose: Clone SHA1 state. +// +// Returns: Reason: +// ippStsNullPtrErr pSrcState == NULL +// pDstState == NULL +// ippStsContextMatchErr pSrcState->idCtx != idCtxSHA1 +// pDstState->idCtx != idCtxSHA1 +// ippStsNoErr no errors +// +// Parameters: +// pSrcState pointer to the source SHA1 state +// pDstState pointer to the target SHA1 state +// +// Note: +// pDstState may to be uninitialized by ippsSHA1Init() +// +*F*/ +IPPFUN(IppStatus, ippsSHA1Duplicate,(const IppsSHA1State* pSrcState, IppsSHA1State* pDstState)) +{ + /* test state pointers */ + IPP_BAD_PTR2_RET(pSrcState, pDstState); + IPP_BADARG_RET(!HASH_VALID_ID(pSrcState, idCtxSHA1), ippStsContextMatchErr); + + /* copy state */ + CopyBlock(pSrcState, pDstState, sizeof(IppsSHA1State)); + HASH_SET_ID(pDstState, idCtxSHA1); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1final.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1final.c new file mode 100644 index 0000000..a434058 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1final.c @@ -0,0 +1,75 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA1 +// +// Contents: +// ippsSHA1Final() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha1stuff.h" + +/*F* +// Name: ippsSHA1Final +// +// Purpose: Stop message digesting and return digest. +// +// Returns: Reason: +// ippStsNullPtrErr pMD == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxSHA1 +// ippStsNoErr no errors +// +// Parameters: +// pMD address of the output digest +// pState pointer to the SHS state +// +*F*/ +IPPFUN(IppStatus, ippsSHA1Final,(Ipp8u* pMD, IppsSHA1State* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxSHA1), ippStsContextMatchErr); + + /* test digest pointer */ + IPP_BAD_PTR1_RET(pMD); + + cpFinalizeSHA1(HASH_VALUE(pState), HASH_BUFF(pState), HASH_BUFFIDX(pState), HASH_LENLO(pState)); + /* convert hash into big endian */ + ((Ipp32u*)pMD)[0] = ENDIANNESS32(HASH_VALUE(pState)[0]); + ((Ipp32u*)pMD)[1] = ENDIANNESS32(HASH_VALUE(pState)[1]); + ((Ipp32u*)pMD)[2] = ENDIANNESS32(HASH_VALUE(pState)[2]); + ((Ipp32u*)pMD)[3] = ENDIANNESS32(HASH_VALUE(pState)[3]); + ((Ipp32u*)pMD)[4] = ENDIANNESS32(HASH_VALUE(pState)[4]); + + /* re-init hash value */ + HASH_BUFFIDX(pState) = 0; + HASH_LENLO(pState) = 0; + sha1_hashInit(HASH_VALUE(pState)); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1getsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1getsize.c new file mode 100644 index 0000000..7984cbd --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1getsize.c @@ -0,0 +1,56 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA1 +// +// Contents: +// ippsSHA1GetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsSHA1GetSize +// +// Purpose: Returns size (bytes) of IppsSHA1State state. +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to state size +// +*F*/ +IPPFUN(IppStatus, ippsSHA1GetSize,(int* pSize)) +{ + /* test pointer */ + IPP_BAD_PTR1_RET(pSize); + + *pSize = sizeof(IppsSHA1State); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1gettag.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1gettag.c new file mode 100644 index 0000000..20b7273 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1gettag.c @@ -0,0 +1,78 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA1 +// +// Contents: +// ippsSHA1GetTag() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha1stuff.h" + +/*F* +// Name: ippsSHA1GetTag +// +// Purpose: Compute digest based on current state. +// Note, that futher digest update is possible +// +// Returns: Reason: +// ippStsNullPtrErr pTag == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxSHA1 +// ippStsLengthErr max_SHA_digestLen < tagLen <1 +// ippStsNoErr no errors +// +// Parameters: +// pTag address of the output digest +// tagLen length of digest +// pState pointer to the SHS state +// +*F*/ +IPPFUN(IppStatus, ippsSHA1GetTag,(Ipp8u* pTag, Ipp32u tagLen, const IppsSHA1State* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxSHA1), ippStsContextMatchErr); + + /* test digest pointer */ + IPP_BAD_PTR1_RET(pTag); + IPP_BADARG_RET((tagLen<1)||(sizeof(DigestSHA1)idCtx != idCtxSHA1 +// ippStsNoErr no errors +// +// Parameters: +// pState pointer to the hash state +// pBuffer pointer to the destination buffer +// +*F*/ +IPPFUN(IppStatus, ippsSHA1Pack,(const IppsSHA1State* pState, Ipp8u* pBuffer)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pState, pBuffer); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxSHA1), ippStsContextMatchErr); + + CopyBlock(pState, pBuffer, sizeof(IppsSHA1State)); + IppsSHA1State* pCopy = (IppsSHA1State*)pBuffer; + HASH_RESET_ID(pCopy, idCtxSHA1); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1stuff.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1stuff.h new file mode 100644 index 0000000..a1a93a6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1stuff.h @@ -0,0 +1,87 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA1 +// +// Contents: +// SHA1 stuff +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" + +#if !defined(_CP_HASH_SHA1) +#define _CP_HASH_SHA1 + +/* SHA-1 constants */ +static const Ipp32u sha1_iv[] = { + 0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476, 0xC3D2E1F0}; + +static __ALIGN16 const Ipp32u sha1_cnt[] = { + 0x5A827999, 0x6ED9EBA1, 0x8F1BBCDC, 0xCA62C1D6 +}; + +IPP_OWN_DEFN (static void, sha1_hashInit, (void* pHash)) +{ + /* setup initial digest */ + ((Ipp32u*)pHash)[0] = sha1_iv[0]; + ((Ipp32u*)pHash)[1] = sha1_iv[1]; + ((Ipp32u*)pHash)[2] = sha1_iv[2]; + ((Ipp32u*)pHash)[3] = sha1_iv[3]; + ((Ipp32u*)pHash)[4] = sha1_iv[4]; +} + +IPP_OWN_DEFN (static void, sha1_hashUpdate, (void* pHash, const Ipp8u* pMsg, int msgLen)) +{ + UpdateSHA1(pHash, pMsg, msgLen, sha1_cnt); +} + +#if (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_ || _SHA_NI_ENABLING_==_FEATURE_ON_) +IPP_OWN_DEFN (static void, sha1_ni_hashUpdate, (void* pHash, const Ipp8u* pMsg, int msgLen)) +{ + UpdateSHA1ni(pHash, pMsg, msgLen, sha1_cnt); +} +#endif + +IPP_OWN_DEFN (static void, sha1_hashOctString, (Ipp8u* pMD, void* pHashVal)) +{ + /* convert hash into big endian */ + ((Ipp32u*)pMD)[0] = ENDIANNESS32(((Ipp32u*)pHashVal)[0]); + ((Ipp32u*)pMD)[1] = ENDIANNESS32(((Ipp32u*)pHashVal)[1]); + ((Ipp32u*)pMD)[2] = ENDIANNESS32(((Ipp32u*)pHashVal)[2]); + ((Ipp32u*)pMD)[3] = ENDIANNESS32(((Ipp32u*)pHashVal)[3]); + ((Ipp32u*)pMD)[4] = ENDIANNESS32(((Ipp32u*)pHashVal)[4]); +} + +IPP_OWN_DEFN (static void, sha1_msgRep, (Ipp8u* pDst, Ipp64u lenLo, Ipp64u lenHi)) +{ + IPP_UNREFERENCED_PARAMETER(lenHi); + lenLo = ENDIANNESS64(lenLo<<3); + ((Ipp64u*)(pDst))[0] = lenLo; +} + +#define cpFinalizeSHA1 OWNAPI(cpFinalizeSHA1) + IPP_OWN_DECL (void, cpFinalizeSHA1, (DigestSHA1 pHash, const Ipp8u* inpBuffer, int inpLen, Ipp64u processedMsgLen)) + +#endif /* #if !defined(_CP_HASH_SHA1) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1unpack.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1unpack.c new file mode 100644 index 0000000..16e0287 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1unpack.c @@ -0,0 +1,59 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA1 +// +// Contents: +// ippsSHA1Unpack() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsSHA1Unpack +// +// Purpose: Unpack buffer content into the initialized context. +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL +// pBuffer == NULL +// ippStsNoErr no errors +// +// Parameters: +// pBuffer pointer to the input buffer +// pCtx pointer hash state +// +*F*/ +IPPFUN(IppStatus, ippsSHA1Unpack,(const Ipp8u* pBuffer, IppsSHA1State* pCtx)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pCtx, pBuffer); + + CopyBlock(pBuffer, pCtx, sizeof(IppsSHA1State)); + HASH_SET_ID(pCtx, idCtxSHA1); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1update.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1update.c new file mode 100644 index 0000000..2ce1299 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha1update.c @@ -0,0 +1,122 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA1 +// +// Contents: +// ippsSHA1Update() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha1stuff.h" + +/*F* +// Name: ippsSHA1Update +// +// Purpose: Updates intermediate digest based on input stream. +// +// Returns: Reason: +// ippStsNullPtrErr pSrc == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxSHA1 +// ippStsLengthErr len <0 +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the input stream +// len input stream length +// pState pointer to the SHA1 state +// +*F*/ +IPPFUN(IppStatus, ippsSHA1Update,(const Ipp8u* pSrc, int len, IppsSHA1State* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxSHA1), ippStsContextMatchErr); + + /* test input length */ + IPP_BADARG_RET((len<0), ippStsLengthErr); + /* test source pointer */ + IPP_BADARG_RET((len && !pSrc), ippStsNullPtrErr); + + /* + // handle non empty message + */ + if(len) { + /* select processing function */ + #if (_SHA_NI_ENABLING_==_FEATURE_ON_) + cpHashProc updateFunc = UpdateSHA1ni; + #elif (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_) + cpHashProc updateFunc = IsFeatureEnabled(ippCPUID_SHA)? UpdateSHA1ni : UpdateSHA1; + #else + cpHashProc updateFunc = UpdateSHA1; + #endif + + int procLen; + + int idx = HASH_BUFFIDX(pState); + Ipp8u* pBuffer = HASH_BUFF(pState); + Ipp64u lenLo = HASH_LENLO(pState) + (Ipp64u)len; + + /* if non empty internal buffer filling */ + if(idx) { + /* copy from input stream to the internal buffer as match as possible */ + procLen = IPP_MIN(len, (MBS_SHA1-idx)); + CopyBlock(pSrc, pBuffer+idx, procLen); + + /* update message pointer and length */ + idx += procLen; + pSrc += procLen; + len -= procLen; + + /* update digest if buffer full */ + if( MBS_SHA1 == idx) { + updateFunc(HASH_VALUE(pState), pBuffer, MBS_SHA1, sha1_cnt); + idx = 0; + } + } + + /* main message part processing */ + procLen = len & ~(MBS_SHA1-1); + if(procLen) { + updateFunc(HASH_VALUE(pState), pSrc, procLen, sha1_cnt); + pSrc += procLen; + len -= procLen; + } + + /* store rest of message into the internal buffer */ + if(len) { + CopyBlock(pSrc, pBuffer, len); + idx += len; + } + + /* update length of processed message */ + HASH_LENLO(pState) = lenLo; + HASH_BUFFIDX(pState) = idx; + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha224duplicate.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha224duplicate.c new file mode 100644 index 0000000..e12c77d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha224duplicate.c @@ -0,0 +1,59 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsSHA224Duplicate() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsSHA224Duplicate +// +// Purpose: Clone SHA224 state. +// +// Returns: Reason: +// ippStsNullPtrErr pSrcState == NULL +// pDstState == NULL +// ippStsContextMatchErr pSrcState->idCtx != idCtxSHA256 +// pDstState->idCtx != idCtxSHA256 +// ippStsNoErr no errors +// +// Parameters: +// pSrcState pointer to the source SHA224 state +// pDstState pointer to the target SHA224 state +// +// Note: +// pDstState may to be uninitialized by ippsSHA224Init() +// +*F*/ + +IPPFUN(IppStatus, ippsSHA224Duplicate,(const IppsSHA224State* pSrcState, IppsSHA224State* pDstState)) +{ + return ippsSHA256Duplicate(pSrcState, pDstState); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha224final.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha224final.c new file mode 100644 index 0000000..6869bdb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha224final.c @@ -0,0 +1,78 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsSHA224Final() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +/*F* +// Name: ippsSHA224Final +// +// Purpose: Stop message digesting and return digest. +// +// Returns: Reason: +// ippStsNullPtrErr pMD == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxSHA256 +// ippStsNoErr no errors +// +// Parameters: +// pMD address of the output digest +// pState pointer to the SHA224 state +// +*F*/ + +IPPFUN(IppStatus, ippsSHA224Final,(Ipp8u* pMD, IppsSHA224State* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxSHA256), ippStsContextMatchErr); + + /* test digest pointer */ + IPP_BAD_PTR1_RET(pMD); + + cpFinalizeSHA256(HASH_VALUE(pState), HASH_BUFF(pState), HASH_BUFFIDX(pState), HASH_LENLO(pState)); + /* convert hash into big endian */ + ((Ipp32u*)pMD)[0] = ENDIANNESS32(HASH_VALUE(pState)[0]); + ((Ipp32u*)pMD)[1] = ENDIANNESS32(HASH_VALUE(pState)[1]); + ((Ipp32u*)pMD)[2] = ENDIANNESS32(HASH_VALUE(pState)[2]); + ((Ipp32u*)pMD)[3] = ENDIANNESS32(HASH_VALUE(pState)[3]); + ((Ipp32u*)pMD)[4] = ENDIANNESS32(HASH_VALUE(pState)[4]); + ((Ipp32u*)pMD)[5] = ENDIANNESS32(HASH_VALUE(pState)[5]); + ((Ipp32u*)pMD)[6] = ENDIANNESS32(HASH_VALUE(pState)[6]); + + /* re-init hash value */ + HASH_BUFFIDX(pState) = 0; + HASH_LENLO(pState) = 0; + sha224_hashInit(HASH_VALUE(pState)); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha224getsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha224getsize.c new file mode 100644 index 0000000..cb0b472 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha224getsize.c @@ -0,0 +1,53 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsSHA224GetSize() +// +*/ + +/*F* +// Name: ippsSHA224GetSize +// +// Purpose: Returns size (bytes) of IppsSHA224State state. +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to state size +// +*F*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +IPPFUN(IppStatus, ippsSHA224GetSize,(int* pSize)) +{ + return GetSizeSHA256(pSize); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha224gettag.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha224gettag.c new file mode 100644 index 0000000..4d3f570 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha224gettag.c @@ -0,0 +1,82 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsSHA224GetTag() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +/*F* +// Name: ippsSHA224GetTag +// +// Purpose: Compute digest based on current state. +// Note, that futher digest update is possible +// +// Returns: Reason: +// ippStsNullPtrErr pTag == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxSHA256 +// ippStsLengthErr max_SHA_digestLen < tagLen <1 +// ippStsNoErr no errors +// +// Parameters: +// pTag address of the output digest +// tagLen length of digest +// pState pointer to the SHA224 state +// +*F*/ + +IPPFUN(IppStatus, ippsSHA224GetTag,(Ipp8u* pTag, Ipp32u tagLen, const IppsSHA224State* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxSHA256), ippStsContextMatchErr); + + /* test digest pointer */ + IPP_BAD_PTR1_RET(pTag); + IPP_BADARG_RET((tagLen<1)||(sizeof(DigestSHA224)idCtx != idCtxSHA256 +// ippStsLengthErr len <0 +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the input stream +// len input stream length +// pState pointer to the SHA224 state +// +*F*/ + +IPPFUN(IppStatus, ippsSHA224Update,(const Ipp8u* pSrc, int len, IppsSHA224State* pState)) +{ + return ippsSHA256Update(pSrc, len, pState); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha256_messagedigest.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha256_messagedigest.c new file mode 100644 index 0000000..e33565b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha256_messagedigest.c @@ -0,0 +1,87 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// cpSHA256MessageDigest() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +IPP_OWN_DEFN (IppStatus, cpSHA256MessageDigest, (DigestSHA256 hash, const Ipp8u* pMsg, int msgLen, const DigestSHA256 IV)) +{ + /* test digest pointer */ + IPP_BAD_PTR1_RET(hash); + /* test message length */ + IPP_BADARG_RET((msgLen<0), ippStsLengthErr); + /* test message pointer */ + IPP_BADARG_RET((msgLen && !pMsg), ippStsNullPtrErr); + + { + /* select processing function */ + #if (_SHA_NI_ENABLING_==_FEATURE_ON_) + cpHashProc updateFunc = UpdateSHA256ni; + #elif (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_) + cpHashProc updateFunc = IsFeatureEnabled(ippCPUID_SHA)? UpdateSHA256ni : UpdateSHA256; + #else + cpHashProc updateFunc = UpdateSHA256; + #endif + + /* message length in the multiple MBS and the rest */ + int msgLenBlks = msgLen & (-MBS_SHA256); + int msgLenRest = msgLen - msgLenBlks; + + /* init hash */ + hash[0] = IV[0]; + hash[1] = IV[1]; + hash[2] = IV[2]; + hash[3] = IV[3]; + hash[4] = IV[4]; + hash[5] = IV[5]; + hash[6] = IV[6]; + hash[7] = IV[7]; + + /* process main part of the message */ + if(msgLenBlks) { + updateFunc(hash, pMsg, msgLenBlks, sha256_cnt); + pMsg += msgLenBlks; + } + + cpFinalizeSHA256(hash, pMsg, msgLenRest, (Ipp64u)msgLen); + hash[0] = ENDIANNESS32(hash[0]); + hash[1] = ENDIANNESS32(hash[1]); + hash[2] = ENDIANNESS32(hash[2]); + hash[3] = ENDIANNESS32(hash[3]); + hash[4] = ENDIANNESS32(hash[4]); + hash[5] = ENDIANNESS32(hash[5]); + hash[6] = ENDIANNESS32(hash[6]); + hash[7] = ENDIANNESS32(hash[7]); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha256ca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha256ca.c new file mode 100644 index 0000000..f2f78dd --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha256ca.c @@ -0,0 +1,65 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// cpFinalizeSHA256() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +IPP_OWN_DEFN (void, cpFinalizeSHA256, (DigestSHA256 pHash, const Ipp8u* inpBuffer, int inpLen, Ipp64u processedMsgLen)) +{ + /* select processing function */ + #if (_SHA_NI_ENABLING_==_FEATURE_ON_) + cpHashProc updateFunc = UpdateSHA256ni; + #elif (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_) + cpHashProc updateFunc = IsFeatureEnabled(ippCPUID_SHA)? UpdateSHA256ni : UpdateSHA256; + #else + cpHashProc updateFunc = UpdateSHA256; + #endif + + /* local buffer and it length */ + Ipp8u buffer[MBS_SHA256*2]; + int bufferLen = inpLen < (MBS_SHA256-(int)MLR_SHA256)? MBS_SHA256 : MBS_SHA256*2; + + /* copy rest of message into internal buffer */ + CopyBlock(inpBuffer, buffer, inpLen); + + /* padd message */ + buffer[inpLen++] = 0x80; + PadBlock(0, buffer+inpLen, (cpSize)(bufferLen-inpLen-(int)MLR_SHA256)); + + /* put processed message length in bits */ + processedMsgLen = ENDIANNESS64(processedMsgLen<<3); + ((Ipp64u*)(buffer+bufferLen))[-1] = processedMsgLen; + + /* copmplete hash computation */ + updateFunc(pHash, buffer, bufferLen, sha256_cnt); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha256duplicate.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha256duplicate.c new file mode 100644 index 0000000..ad7bb24 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha256duplicate.c @@ -0,0 +1,67 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsSHA256Duplicate() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsSHA256Duplicate +// +// Purpose: Clone SHA256 state. +// +// Returns: Reason: +// ippStsNullPtrErr pSrcState == NULL +// pDstState == NULL +// ippStsContextMatchErr pSrcState->idCtx != idCtxSHA256 +// pDstState->idCtx != idCtxSHA256 +// ippStsNoErr no errors +// +// Parameters: +// pSrcState pointer to the source SHA256 state +// pDstState pointer to the target SHA256 state +// +// Note: +// pDstState may to be uninitialized by ippsSHA256Init() +// +*F*/ +IPPFUN(IppStatus, ippsSHA256Duplicate,(const IppsSHA256State* pSrcState, IppsSHA256State* pDstState)) +{ + /* test state pointers */ + IPP_BAD_PTR2_RET(pSrcState, pDstState); + /* test states ID */ + IPP_BADARG_RET(!HASH_VALID_ID(pSrcState, idCtxSHA256), ippStsContextMatchErr); + + /* copy state */ + CopyBlock(pSrcState, pDstState, sizeof(IppsSHA256State)); + HASH_SET_ID(pDstState, idCtxSHA256); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha256final.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha256final.c new file mode 100644 index 0000000..5f488fd --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha256final.c @@ -0,0 +1,79 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsSHA256Final() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + + +/*F* +// Name: ippsSHA256Final +// +// Purpose: Stop message digesting and return digest. +// +// Returns: Reason: +// ippStsNullPtrErr pMD == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxSHA256 +// ippStsNoErr no errors +// +// Parameters: +// pMD address of the output digest +// pState pointer to the SHA256 state +// +*F*/ +IPPFUN(IppStatus, ippsSHA256Final,(Ipp8u* pMD, IppsSHA256State* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxSHA256), ippStsContextMatchErr); + + /* test digest pointer */ + IPP_BAD_PTR1_RET(pMD); + + cpFinalizeSHA256(HASH_VALUE(pState), HASH_BUFF(pState), HASH_BUFFIDX(pState), HASH_LENLO(pState)); + /* convert hash into big endian */ + ((Ipp32u*)pMD)[0] = ENDIANNESS32(HASH_VALUE(pState)[0]); + ((Ipp32u*)pMD)[1] = ENDIANNESS32(HASH_VALUE(pState)[1]); + ((Ipp32u*)pMD)[2] = ENDIANNESS32(HASH_VALUE(pState)[2]); + ((Ipp32u*)pMD)[3] = ENDIANNESS32(HASH_VALUE(pState)[3]); + ((Ipp32u*)pMD)[4] = ENDIANNESS32(HASH_VALUE(pState)[4]); + ((Ipp32u*)pMD)[5] = ENDIANNESS32(HASH_VALUE(pState)[5]); + ((Ipp32u*)pMD)[6] = ENDIANNESS32(HASH_VALUE(pState)[6]); + ((Ipp32u*)pMD)[7] = ENDIANNESS32(HASH_VALUE(pState)[7]); + + /* re-init hash value */ + HASH_BUFFIDX(pState) = 0; + HASH_LENLO(pState) = 0; + sha256_hashInit(HASH_VALUE(pState)); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha256getsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha256getsize.c new file mode 100644 index 0000000..6ca1db6 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha256getsize.c @@ -0,0 +1,52 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsSHA256GetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +/*F* +// Name: ippsSHA256GetSize +// +// Purpose: Returns size (bytes) of IppsSHA256State state. +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to state size +// +*F*/ +IPPFUN(IppStatus, ippsSHA256GetSize,(int* pSize)) +{ + return GetSizeSHA256(pSize); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha256gettag.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha256gettag.c new file mode 100644 index 0000000..d398460 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha256gettag.c @@ -0,0 +1,81 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SHA256 +// +// Contents: +// ippsSHA256GetTag() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha256stuff.h" + +/*F* +// Name: ippsSHA256GetTag +// +// Purpose: Compute digest based on current state. +// Note, that futher digest update is possible +// +// Returns: Reason: +// ippStsNullPtrErr pTag == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxSHA256 +// ippStsLengthErr max_SHA_digestLen < tagLen <1 +// ippStsNoErr no errors +// +// Parameters: +// pTag address of the output digest +// tagLen length of digest +// pState pointer to the SHA256 state +// +*F*/ +IPPFUN(IppStatus, ippsSHA256GetTag,(Ipp8u* pTag, Ipp32u tagLen, const IppsSHA256State* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxSHA256), ippStsContextMatchErr); + + /* test digest pointer */ + IPP_BAD_PTR1_RET(pTag); + IPP_BADARG_RET((tagLen<1)||(sizeof(DigestSHA256)idCtx != idCtxSHA256 +// ippStsLengthErr len <0 +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the input stream +// len input stream length +// pState pointer to the SHA256 state +// +*F*/ +IPPFUN(IppStatus, ippsSHA256Update,(const Ipp8u* pSrc, int len, IppsSHA256State* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxSHA256), ippStsContextMatchErr); + + /* test input length */ + IPP_BADARG_RET((len<0), ippStsLengthErr); + /* test source pointer */ + IPP_BADARG_RET((len && !pSrc), ippStsNullPtrErr); + + /* + // handle non empty message + */ + if(len) { + /* select processing function */ + #if (_SHA_NI_ENABLING_==_FEATURE_ON_) + cpHashProc updateFunc = UpdateSHA256ni; + #elif (_SHA_NI_ENABLING_==_FEATURE_TICKTOCK_) + cpHashProc updateFunc = IsFeatureEnabled(ippCPUID_SHA)? UpdateSHA256ni : UpdateSHA256; + #else + cpHashProc updateFunc = UpdateSHA256; + #endif + + int procLen; + + int idx = HASH_BUFFIDX(pState); + Ipp8u* pBuffer = HASH_BUFF(pState); + Ipp64u lenLo = HASH_LENLO(pState) + (Ipp64u)len; + + /* if non empty internal buffer filling */ + if(idx) { + /* copy from input stream to the internal buffer as match as possible */ + procLen = IPP_MIN(len, (MBS_SHA256-idx)); + CopyBlock(pSrc, pBuffer+idx, procLen); + + /* update message pointer and length */ + pSrc += procLen; + len -= procLen; + idx += procLen; + + /* update digest if buffer full */ + if( MBS_SHA256 == idx) { + updateFunc(HASH_VALUE(pState), pBuffer, MBS_SHA256, sha256_cnt); + idx = 0; + } + } + + /* main message part processing */ + procLen = len & ~(MBS_SHA256-1); + if(procLen) { + updateFunc(HASH_VALUE(pState), pSrc, procLen, sha256_cnt); + pSrc += procLen; + len -= procLen; + } + + /* store rest of message into the internal buffer */ + if(len) { + CopyBlock(pSrc, pBuffer, len); + idx += len; + } + + /* update length of processed message */ + HASH_LENLO(pState) = lenLo; + HASH_BUFFIDX(pState) = idx; + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha384duplicate.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha384duplicate.c new file mode 100644 index 0000000..9faabfd --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha384duplicate.c @@ -0,0 +1,58 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsSHA384Duplicate() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsSHA384Duplicate +// +// Purpose: Clone SHA512 state. +// +// Returns: Reason: +// ippStsNullPtrErr pSrcState == NULL +// pDstState == NULL +// ippStsContextMatchErr pSrcState->idCtx != idCtxSHA512 +// pDstState->idCtx != idCtxSHA512 +// ippStsNoErr no errors +// +// Parameters: +// pSrcState pointer to the source SHA384 state +// pDstState pointer to the target SHA384 state +// Note: +// pDstState may to be uninitialized by ippsSHA384Init() +// +*F*/ + +IPPFUN(IppStatus, ippsSHA384Duplicate,(const IppsSHA384State* pSrcState, IppsSHA384State* pDstState)) +{ + return ippsSHA512Duplicate(pSrcState, pDstState); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha384final.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha384final.c new file mode 100644 index 0000000..7433342 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha384final.c @@ -0,0 +1,80 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsSHA384Final() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha512stuff.h" + +/*F* +// Name: ippsSHA384Final +// +// Purpose: Stop message digesting and return digest. +// +// Returns: Reason: +// ippStsNullPtrErr pMD == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxSHA512 +// ippStsNoErr no errors +// +// Parameters: +// pMD address of the output digest +// pState pointer to the SHA384 state +// +*F*/ + +IPPFUN(IppStatus, ippsSHA384Final,(Ipp8u* pMD, IppsSHA384State* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxSHA512), ippStsContextMatchErr); + + /* test digest pointer */ + IPP_BAD_PTR1_RET(pMD); + + cpFinalizeSHA512(HASH_VALUE(pState), + HASH_BUFF(pState), HASH_BUFFIDX(pState), + HASH_LENLO(pState), HASH_LENHI(pState)); + /* convert hash into big endian */ + ((Ipp64u*)pMD)[0] = ENDIANNESS64(HASH_VALUE(pState)[0]); + ((Ipp64u*)pMD)[1] = ENDIANNESS64(HASH_VALUE(pState)[1]); + ((Ipp64u*)pMD)[2] = ENDIANNESS64(HASH_VALUE(pState)[2]); + ((Ipp64u*)pMD)[3] = ENDIANNESS64(HASH_VALUE(pState)[3]); + ((Ipp64u*)pMD)[4] = ENDIANNESS64(HASH_VALUE(pState)[4]); + ((Ipp64u*)pMD)[5] = ENDIANNESS64(HASH_VALUE(pState)[5]); + + /* re-init hash value */ + HASH_BUFFIDX(pState) = 0; + HASH_LENLO(pState) = 0; + HASH_LENHI(pState) = 0; + sha512_384_hashInit(HASH_VALUE(pState)); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha384getsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha384getsize.c new file mode 100644 index 0000000..84bdbc2 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha384getsize.c @@ -0,0 +1,53 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsSHA384GetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha512stuff.h" + +/*F* +// Name: ippsSHA384GetSize +// +// Purpose: Returns size (bytes) of IppsSHA384State state. +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to state size +// +*F*/ + +IPPFUN(IppStatus, ippsSHA384GetSize,(int* pSize)) +{ + return GetSizeSHA512(pSize); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha384gettag.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha384gettag.c new file mode 100644 index 0000000..40af46b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha384gettag.c @@ -0,0 +1,82 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsSHA384GetTag() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha512stuff.h" + +/*F* +// Name: ippsSHA384GetTag +// +// Purpose: Compute digest based on current state. +// Note, that futher digest update is possible +// +// Returns: Reason: +// ippStsNullPtrErr pTag == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxSHA512 +// ippStsLengthErr max_SHA_digestLen < tagLen <1 +// ippStsNoErr no errors +// +// Parameters: +// pTag address of the output digest +// tagLen length of digest +// pState pointer to the SHA384 state +// +*F*/ + +IPPFUN(IppStatus, ippsSHA384GetTag,(Ipp8u* pTag, Ipp32u tagLen, const IppsSHA384State* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxSHA512), ippStsContextMatchErr); + + /* test digest pointer */ + IPP_BAD_PTR1_RET(pTag); + IPP_BADARG_RET((tagLen<1)||(sizeof(DigestSHA384)idCtx != idCtxSHA512 +// ippStsNoErr no errors +// +// Parameters: +// pState pointer hash state +// pBuffer pointer to the destination buffer +// +*F*/ +IPPFUN(IppStatus, ippsSHA384Pack,(const IppsSHA384State* pState, Ipp8u* pBuffer)) +{ + return ippsSHA512Pack(pState, pBuffer); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha384unpack.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha384unpack.c new file mode 100644 index 0000000..00d27ec --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha384unpack.c @@ -0,0 +1,54 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsSHA384Unpack() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsSHA384Unpack +// +// Purpose: Unpack buffer content into the initialized context. +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL +// pBuffer== NULL +// ippStsNoErr no errors +// +// Parameters: +// pBuffer pointer to the input buffer +// pState pointer hash state +// +*F*/ + +IPPFUN(IppStatus, ippsSHA384Unpack,(const Ipp8u* pBuffer, IppsSHA384State* pState)) +{ + return ippsSHA512Unpack(pBuffer, pState); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha384update.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha384update.c new file mode 100644 index 0000000..dec3ec8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha384update.c @@ -0,0 +1,57 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsSHA384Update() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsSHA384Update +// +// Purpose: Updates intermadiate digest based on input stream. +// +// Returns: Reason: +// ippStsNullPtrErr pSrc == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxSHA512 +// ippStsLengthErr len <0 +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the input stream +// len input stream length +// pState pointer to the SHA384 state +// +*F*/ + +IPPFUN(IppStatus, ippsSHA384Update,(const Ipp8u* pSrc, int len, IppsSHA384State* pState)) +{ + return ippsSHA512Update(pSrc, len, pState); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512_init.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512_init.c new file mode 100644 index 0000000..adc9dc5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512_init.c @@ -0,0 +1,52 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// InitSHA512() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha512stuff.h" + +IPP_OWN_DEFN (IppStatus, InitSHA512, (IppsSHA512State* pState, const DigestSHA512 IV)) +{ + /* test state pointer */ + IPP_BAD_PTR1_RET(pState); + + /* set state ID */ + HASH_SET_ID(pState, idCtxSHA512); + /* zeros message length */ + HASH_LENLO(pState) = 0; + HASH_LENHI(pState) = 0; + /* message buffer is free */ + HASH_BUFFIDX(pState) = 0; + /* setup initial digest */ + hashInit(HASH_VALUE(pState), IV); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512_messagedigest.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512_messagedigest.c new file mode 100644 index 0000000..d551fdb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512_messagedigest.c @@ -0,0 +1,71 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// cpSHA512MessageDigest() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha512stuff.h" + +IPP_OWN_DEFN (IppStatus, cpSHA512MessageDigest, (DigestSHA512 hash, const Ipp8u* pMsg, int msgLen, const DigestSHA512 IV)) +{ + /* test digest pointer */ + IPP_BAD_PTR1_RET(hash); + /* test message length */ + IPP_BADARG_RET((msgLen<0), ippStsLengthErr); + /* test message pointer */ + IPP_BADARG_RET((msgLen && !pMsg), ippStsNullPtrErr); + + { + /* message length in the multiple MBS and the rest */ + int msgLenBlks = msgLen & (-MBS_SHA512); + int msgLenRest = msgLen - msgLenBlks; + + /* init hash */ + hashInit(hash, IV); + + /* process main part of the message */ + if(msgLenBlks) { + UpdateSHA512(hash, pMsg, msgLenBlks, sha512_cnt); + pMsg += msgLenBlks; + } + + cpFinalizeSHA512(hash, pMsg, msgLenRest, (Ipp64u)msgLen, 0); + hash[0] = ENDIANNESS64(hash[0]); + hash[1] = ENDIANNESS64(hash[1]); + hash[2] = ENDIANNESS64(hash[2]); + hash[3] = ENDIANNESS64(hash[3]); + hash[4] = ENDIANNESS64(hash[4]); + hash[5] = ENDIANNESS64(hash[5]); + hash[6] = ENDIANNESS64(hash[6]); + hash[7] = ENDIANNESS64(hash[7]); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512duplicate.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512duplicate.c new file mode 100644 index 0000000..0a93ecd --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512duplicate.c @@ -0,0 +1,66 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsSHA512Duplicate() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsSHA512Duplicate +// +// Purpose: Clone SHA512 state. +// +// Returns: Reason: +// ippStsNullPtrErr pSrcState == NULL +// pDstState == NULL +// ippStsContextMatchErr pSrcState->idCtx != idCtxSHA512 +// pDstState->idCtx != idCtxSHA512 +// ippStsNoErr no errors +// +// Parameters: +// pSrcState pointer to the source SHA512 state +// pDstState pointer to the target SHA512 state +// Note: +// pDstState may to be uninitialized by ippsSHA512Init() +// +*F*/ +IPPFUN(IppStatus, ippsSHA512Duplicate,(const IppsSHA512State* pSrcState, IppsSHA512State* pDstState)) +{ + /* test state pointers */ + IPP_BAD_PTR2_RET(pSrcState, pDstState); + /* test states ID */ + IPP_BADARG_RET(!HASH_VALID_ID(pSrcState, idCtxSHA512), ippStsContextMatchErr); + + /* copy state */ + CopyBlock(pSrcState, pDstState, sizeof(IppsSHA512State)); + HASH_SET_ID(pDstState, idCtxSHA512); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512final.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512final.c new file mode 100644 index 0000000..f1721b8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512final.c @@ -0,0 +1,81 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsSHA512Final() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha512stuff.h" + +/*F* +// Name: ippsSHA512Final +// +// Purpose: Stop message digesting and return digest. +// +// Returns: Reason: +// ippStsNullPtrErr pMD == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxSHA512 +// ippStsNoErr no errors +// +// Parameters: +// pMD address of the output digest +// pState pointer to the SHA512 state +// +*F*/ +IPPFUN(IppStatus, ippsSHA512Final,(Ipp8u* pMD, IppsSHA512State* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxSHA512), ippStsContextMatchErr); + + /* test digest pointer */ + IPP_BAD_PTR1_RET(pMD); + + cpFinalizeSHA512(HASH_VALUE(pState), + HASH_BUFF(pState), HASH_BUFFIDX(pState), + HASH_LENLO(pState), HASH_LENHI(pState)); + /* convert hash into big endian */ + ((Ipp64u*)pMD)[0] = ENDIANNESS64(HASH_VALUE(pState)[0]); + ((Ipp64u*)pMD)[1] = ENDIANNESS64(HASH_VALUE(pState)[1]); + ((Ipp64u*)pMD)[2] = ENDIANNESS64(HASH_VALUE(pState)[2]); + ((Ipp64u*)pMD)[3] = ENDIANNESS64(HASH_VALUE(pState)[3]); + ((Ipp64u*)pMD)[4] = ENDIANNESS64(HASH_VALUE(pState)[4]); + ((Ipp64u*)pMD)[5] = ENDIANNESS64(HASH_VALUE(pState)[5]); + ((Ipp64u*)pMD)[6] = ENDIANNESS64(HASH_VALUE(pState)[6]); + ((Ipp64u*)pMD)[7] = ENDIANNESS64(HASH_VALUE(pState)[7]); + + /* re-init hash value */ + HASH_BUFFIDX(pState) = 0; + HASH_LENLO(pState) = 0; + HASH_LENHI(pState) = 0; + sha512_hashInit(HASH_VALUE(pState)); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512getsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512getsize.c new file mode 100644 index 0000000..c9277fb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512getsize.c @@ -0,0 +1,52 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsSHA512GetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha512stuff.h" + +/*F* +// Name: ippsSHA512GetSize +// +// Purpose: Returns size (bytes) of IppsSHA512State state. +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to state size +// +*F*/ +IPPFUN(IppStatus, ippsSHA512GetSize,(int* pSize)) +{ + return GetSizeSHA512(pSize); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512gettag.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512gettag.c new file mode 100644 index 0000000..c657cc7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512gettag.c @@ -0,0 +1,83 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsSHA512GetTag() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha512stuff.h" + +/*F* +// Name: ippsSHA512GetTag +// +// Purpose: Compute digest based on current state. +// Note, that futher digest update is possible +// +// Returns: Reason: +// ippStsNullPtrErr pTag == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxSHA512 +// ippStsLengthErr max_SHA_digestLen < tagLen <1 +// ippStsNoErr no errors +// +// Parameters: +// pTag address of the output digest +// tagLen length of digest +// pState pointer to the SHA512 state +// +*F*/ +IPPFUN(IppStatus, ippsSHA512GetTag,(Ipp8u* pTag, Ipp32u tagLen, const IppsSHA512State* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxSHA512), ippStsContextMatchErr); + + /* test digest pointer */ + IPP_BAD_PTR1_RET(pTag); + IPP_BADARG_RET((tagLen<1)||(sizeof(DigestSHA512)idCtx != idCtxSHA512 +// ippStsNoErr no errors +// +// Parameters: +// pState pointer hash state +// pBuffer pointer to the destination buffer +// +*F*/ +IPPFUN(IppStatus, ippsSHA512Pack,(const IppsSHA512State* pState, Ipp8u* pBuffer)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pState, pBuffer); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxSHA512), ippStsContextMatchErr); + + CopyBlock(pState, pBuffer, sizeof(IppsSHA512State)); + IppsSHA512State* pCopy = (IppsSHA512State*)pBuffer; + HASH_RESET_ID(pCopy, idCtxSHA512); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512stuff.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512stuff.h new file mode 100644 index 0000000..2c0fc98 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512stuff.h @@ -0,0 +1,200 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// SHA512 stuff +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +#if !defined(_PCP_SHA512_STUFF_H) +#define _PCP_SHA512_STUFF_H + +/* SHA-512, SHA-384, SHA512-224, SHA512 constants */ +static const Ipp64u sha512_iv[] = { + CONST_64(0x6A09E667F3BCC908), CONST_64(0xBB67AE8584CAA73B), + CONST_64(0x3C6EF372FE94F82B), CONST_64(0xA54FF53A5F1D36F1), + CONST_64(0x510E527FADE682D1), CONST_64(0x9B05688C2B3E6C1F), + CONST_64(0x1F83D9ABFB41BD6B), CONST_64(0x5BE0CD19137E2179)}; +static const Ipp64u sha512_384_iv[] = { + CONST_64(0xCBBB9D5DC1059ED8), CONST_64(0x629A292A367CD507), + CONST_64(0x9159015A3070DD17), CONST_64(0x152FECD8F70E5939), + CONST_64(0x67332667FFC00B31), CONST_64(0x8EB44A8768581511), + CONST_64(0xDB0C2E0D64F98FA7), CONST_64(0x47B5481DBEFA4FA4)}; +static const Ipp64u sha512_256_iv[] = { + CONST_64(0x22312194FC2BF72C), CONST_64(0x9F555FA3C84C64C2), + CONST_64(0x2393B86B6F53B151), CONST_64(0x963877195940EABD), + CONST_64(0x96283EE2A88EFFE3), CONST_64(0xBE5E1E2553863992), + CONST_64(0x2B0199FC2C85B8AA), CONST_64(0x0EB72DDC81C52CA2)}; +static const Ipp64u sha512_224_iv[] = { + CONST_64(0x8C3D37C819544DA2), CONST_64(0x73E1996689DCD4D6), + CONST_64(0x1DFAB7AE32FF9C82), CONST_64(0x679DD514582F9FCF), + CONST_64(0x0F6D2B697BD44DA8), CONST_64(0x77E36F7304C48942), + CONST_64(0x3F9D85A86A1D36C8), CONST_64(0x1112E6AD91D692A1)}; + +static __ALIGN16 const Ipp64u sha512_cnt[] = { + CONST_64(0x428A2F98D728AE22), CONST_64(0x7137449123EF65CD), CONST_64(0xB5C0FBCFEC4D3B2F), CONST_64(0xE9B5DBA58189DBBC), + CONST_64(0x3956C25BF348B538), CONST_64(0x59F111F1B605D019), CONST_64(0x923F82A4AF194F9B), CONST_64(0xAB1C5ED5DA6D8118), + CONST_64(0xD807AA98A3030242), CONST_64(0x12835B0145706FBE), CONST_64(0x243185BE4EE4B28C), CONST_64(0x550C7DC3D5FFB4E2), + CONST_64(0x72BE5D74F27B896F), CONST_64(0x80DEB1FE3B1696B1), CONST_64(0x9BDC06A725C71235), CONST_64(0xC19BF174CF692694), + CONST_64(0xE49B69C19EF14AD2), CONST_64(0xEFBE4786384F25E3), CONST_64(0x0FC19DC68B8CD5B5), CONST_64(0x240CA1CC77AC9C65), + CONST_64(0x2DE92C6F592B0275), CONST_64(0x4A7484AA6EA6E483), CONST_64(0x5CB0A9DCBD41FBD4), CONST_64(0x76F988DA831153B5), + CONST_64(0x983E5152EE66DFAB), CONST_64(0xA831C66D2DB43210), CONST_64(0xB00327C898FB213F), CONST_64(0xBF597FC7BEEF0EE4), + CONST_64(0xC6E00BF33DA88FC2), CONST_64(0xD5A79147930AA725), CONST_64(0x06CA6351E003826F), CONST_64(0x142929670A0E6E70), + CONST_64(0x27B70A8546D22FFC), CONST_64(0x2E1B21385C26C926), CONST_64(0x4D2C6DFC5AC42AED), CONST_64(0x53380D139D95B3DF), + CONST_64(0x650A73548BAF63DE), CONST_64(0x766A0ABB3C77B2A8), CONST_64(0x81C2C92E47EDAEE6), CONST_64(0x92722C851482353B), + CONST_64(0xA2BFE8A14CF10364), CONST_64(0xA81A664BBC423001), CONST_64(0xC24B8B70D0F89791), CONST_64(0xC76C51A30654BE30), + CONST_64(0xD192E819D6EF5218), CONST_64(0xD69906245565A910), CONST_64(0xF40E35855771202A), CONST_64(0x106AA07032BBD1B8), + CONST_64(0x19A4C116B8D2D0C8), CONST_64(0x1E376C085141AB53), CONST_64(0x2748774CDF8EEB99), CONST_64(0x34B0BCB5E19B48A8), + CONST_64(0x391C0CB3C5C95A63), CONST_64(0x4ED8AA4AE3418ACB), CONST_64(0x5B9CCA4F7763E373), CONST_64(0x682E6FF3D6B2B8A3), + CONST_64(0x748F82EE5DEFB2FC), CONST_64(0x78A5636F43172F60), CONST_64(0x84C87814A1F0AB72), CONST_64(0x8CC702081A6439EC), + CONST_64(0x90BEFFFA23631E28), CONST_64(0xA4506CEBDE82BDE9), CONST_64(0xBEF9A3F7B2C67915), CONST_64(0xC67178F2E372532B), + CONST_64(0xCA273ECEEA26619C), CONST_64(0xD186B8C721C0C207), CONST_64(0xEADA7DD6CDE0EB1E), CONST_64(0xF57D4F7FEE6ED178), + CONST_64(0x06F067AA72176FBA), CONST_64(0x0A637DC5A2C898A6), CONST_64(0x113F9804BEF90DAE), CONST_64(0x1B710B35131C471B), + CONST_64(0x28DB77F523047D84), CONST_64(0x32CAAB7B40C72493), CONST_64(0x3C9EBE0A15C9BEBC), CONST_64(0x431D67C49C100D4C), + CONST_64(0x4CC5D4BECB3E42B6), CONST_64(0x597F299CFC657E2A), CONST_64(0x5FCB6FAB3AD6FAEC), CONST_64(0x6C44198C4A475817) +}; + +/* setup init hash value */ +__INLINE void hashInit(Ipp64u* pHash, const Ipp64u* iv) +{ + pHash[0] = iv[0]; + pHash[1] = iv[1]; + pHash[2] = iv[2]; + pHash[3] = iv[3]; + pHash[4] = iv[4]; + pHash[5] = iv[5]; + pHash[6] = iv[6]; + pHash[7] = iv[7]; +} +IPP_OWN_DEFN (static void, sha512_hashInit, (void* pHash)) +{ + hashInit((Ipp64u*)pHash, sha512_iv); +} +IPP_OWN_DEFN (static void, sha512_384_hashInit, (void* pHash)) +{ + hashInit((Ipp64u*)pHash, sha512_384_iv); +} +IPP_OWN_DEFN (static void, sha512_256_hashInit, (void* pHash)) +{ + hashInit((Ipp64u*)pHash, sha512_256_iv); +} +IPP_OWN_DEFN (static void, sha512_224_hashInit, (void* pHash)) +{ + hashInit((Ipp64u*)pHash, sha512_224_iv); +} + +IPP_OWN_DEFN (static void, sha512_hashUpdate, (void* pHash, const Ipp8u* pMsg, int msgLen)) +{ + UpdateSHA512(pHash, pMsg, msgLen, sha512_cnt); +} + +/* convert hash into big endian */ +IPP_OWN_DEFN (static void, sha512_hashOctString, (Ipp8u* pMD, void* pHashVal)) +{ + ((Ipp64u*)pMD)[0] = ENDIANNESS64(((Ipp64u*)pHashVal)[0]); + ((Ipp64u*)pMD)[1] = ENDIANNESS64(((Ipp64u*)pHashVal)[1]); + ((Ipp64u*)pMD)[2] = ENDIANNESS64(((Ipp64u*)pHashVal)[2]); + ((Ipp64u*)pMD)[3] = ENDIANNESS64(((Ipp64u*)pHashVal)[3]); + ((Ipp64u*)pMD)[4] = ENDIANNESS64(((Ipp64u*)pHashVal)[4]); + ((Ipp64u*)pMD)[5] = ENDIANNESS64(((Ipp64u*)pHashVal)[5]); + ((Ipp64u*)pMD)[6] = ENDIANNESS64(((Ipp64u*)pHashVal)[6]); + ((Ipp64u*)pMD)[7] = ENDIANNESS64(((Ipp64u*)pHashVal)[7]); +} +IPP_OWN_DEFN (static void, sha512_384_hashOctString, (Ipp8u* pMD, void* pHashVal)) +{ + ((Ipp64u*)pMD)[0] = ENDIANNESS64(((Ipp64u*)pHashVal)[0]); + ((Ipp64u*)pMD)[1] = ENDIANNESS64(((Ipp64u*)pHashVal)[1]); + ((Ipp64u*)pMD)[2] = ENDIANNESS64(((Ipp64u*)pHashVal)[2]); + ((Ipp64u*)pMD)[3] = ENDIANNESS64(((Ipp64u*)pHashVal)[3]); + ((Ipp64u*)pMD)[4] = ENDIANNESS64(((Ipp64u*)pHashVal)[4]); + ((Ipp64u*)pMD)[5] = ENDIANNESS64(((Ipp64u*)pHashVal)[5]); +} +IPP_OWN_DEFN (static void, sha512_256_hashOctString, (Ipp8u* pMD, void* pHashVal)) +{ + ((Ipp64u*)pMD)[0] = ENDIANNESS64(((Ipp64u*)pHashVal)[0]); + ((Ipp64u*)pMD)[1] = ENDIANNESS64(((Ipp64u*)pHashVal)[1]); + ((Ipp64u*)pMD)[2] = ENDIANNESS64(((Ipp64u*)pHashVal)[2]); + ((Ipp64u*)pMD)[3] = ENDIANNESS64(((Ipp64u*)pHashVal)[3]); +} +IPP_OWN_DEFN (static void, sha512_224_hashOctString, (Ipp8u* pMD, void* pHashVal)) +{ + ((Ipp64u*)pMD)[0] = ENDIANNESS64(((Ipp64u*)pHashVal)[0]); + ((Ipp64u*)pMD)[1] = ENDIANNESS64(((Ipp64u*)pHashVal)[1]); + ((Ipp64u*)pMD)[2] = ENDIANNESS64(((Ipp64u*)pHashVal)[2]); + ((Ipp32u*)pMD)[6] = ENDIANNESS32(((Ipp32u*)pHashVal)[7]); +} + +IPP_OWN_DEFN (static void, sha512_msgRep, (Ipp8u* pDst, Ipp64u lenLo, Ipp64u lenHi)) +{ + lenHi = LSL64(lenHi,3) | LSR64(lenLo,63-3); + lenLo = LSL64(lenLo,3); + ((Ipp64u*)(pDst))[0] = ENDIANNESS64(lenHi); + ((Ipp64u*)(pDst))[1] = ENDIANNESS64(lenLo); +} + +IPP_OWN_DEFN (static IppStatus, GetSizeSHA512, (int* pSize)) +{ + /* test pointer */ + IPP_BAD_PTR1_RET(pSize); + *pSize = sizeof(IppsSHA512State); + return ippStsNoErr; +} + +/* #define cpFinalizeSHA512 OWNAPI(cpFinalizeSHA512) */ + /* IPP_OWN_DECL (void, cpFinalizeSHA512, (DigestSHA512 pHash, const Ipp8u* inpBuffer, int inpLen, Ipp64u lenLo, Ipp64u lenHi)) */ +#define cpSHA512MessageDigest OWNAPI(cpSHA512MessageDigest) + IPP_OWN_DECL (IppStatus, cpSHA512MessageDigest, (DigestSHA512 hash, const Ipp8u* pMsg, int msgLen, const DigestSHA512 IV)) +#define InitSHA512 OWNAPI(InitSHA512) + IPP_OWN_DECL (IppStatus, InitSHA512, (IppsSHA512State* pState, const DigestSHA512 IV)) + +IPP_OWN_DEFN (static void, cpFinalizeSHA512, (DigestSHA512 pHash, const Ipp8u* inpBuffer, int inpLen, Ipp64u lenLo, Ipp64u lenHi)) +{ + /* local buffer and it length */ + Ipp8u buffer[MBS_SHA512*2]; + int bufferLen = inpLen < (MBS_SHA512-(int)MLR_SHA512)? MBS_SHA512 : MBS_SHA512*2; + + /* copy rest of message into internal buffer */ + CopyBlock(inpBuffer, buffer, inpLen); + + /* padd message */ + buffer[inpLen++] = 0x80; + PadBlock(0, buffer+inpLen, (cpSize)(bufferLen-inpLen-(int)MLR_SHA512)); + + /* message length representation */ + lenHi = LSL64(lenHi,3) | LSR64(lenLo,63-3); + lenLo = LSL64(lenLo,3); + ((Ipp64u*)(buffer+bufferLen))[-2] = ENDIANNESS64(lenHi); + ((Ipp64u*)(buffer+bufferLen))[-1] = ENDIANNESS64(lenLo); + + /* copmplete hash computation */ + UpdateSHA512(pHash, buffer, bufferLen, sha512_cnt); +} + +#endif /* #if !defined(_PCP_SHA512_STUFF_H) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512unpack.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512unpack.c new file mode 100644 index 0000000..0278675 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512unpack.c @@ -0,0 +1,58 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsSHA512Unpack() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsSHA512Unpack +// +// Purpose: Unpack buffer content into the initialized context. +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL +// pBuffer== NULL +// ippStsNoErr no errors +// +// Parameters: +// pBuffer pointer to the input buffer +// pState pointer hash state +// +*F*/ +IPPFUN(IppStatus, ippsSHA512Unpack,(const Ipp8u* pBuffer, IppsSHA512State* pState)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pState, pBuffer); + + CopyBlock(pBuffer, pState, sizeof(IppsSHA512State)); + HASH_SET_ID(pState, idCtxSHA512); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512update.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512update.c new file mode 100644 index 0000000..80c512f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsha512update.c @@ -0,0 +1,116 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SHA512 message digest +// +// Contents: +// ippsSHA512Update() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsha512stuff.h" + +/*F* +// Name: ippsSHA512Update +// +// Purpose: Updates intermadiate digest based on input stream. +// +// Returns: Reason: +// ippStsNullPtrErr pSrc == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxSHA512 +// ippStsLengthErr len <0 +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the input stream +// len input stream length +// pState pointer to the SHA512 state +// +*F*/ +IPPFUN(IppStatus, ippsSHA512Update,(const Ipp8u* pSrc, int len, IppsSHA512State* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxSHA512), ippStsContextMatchErr); + + /* test input length */ + IPP_BADARG_RET((len<0), ippStsLengthErr); + /* test source pointer */ + IPP_BADARG_RET((len && !pSrc), ippStsNullPtrErr); + + /* + // handle non empty message + */ + if(len) { + int procLen; + + int idx = HASH_BUFFIDX(pState); + Ipp8u* pBuffer = HASH_BUFF(pState); + Ipp64u lenLo = HASH_LENLO(pState) + (Ipp64u)len; + Ipp64u lenHi = HASH_LENHI(pState); + if(lenLo < HASH_LENLO(pState)) lenHi++; + + /* if non empty internal buffer filling */ + if(idx) { + /* copy from input stream to the internal buffer as match as possible */ + procLen = IPP_MIN(len, (MBS_SHA512-idx)); + CopyBlock(pSrc, pBuffer+idx, procLen); + + /* update message pointer and length */ + pSrc += procLen; + len -= procLen; + idx += procLen; + + /* update digest if buffer full */ + if(MBS_SHA512 == idx) { + UpdateSHA512(HASH_VALUE(pState), pBuffer, MBS_SHA512, sha512_cnt); + idx = 0; + } + } + + /* main message part processing */ + procLen = len & ~(MBS_SHA512-1); + if(procLen) { + UpdateSHA512(HASH_VALUE(pState), pSrc, procLen, sha512_cnt); + pSrc += procLen; + len -= procLen; + } + + /* store rest of message into the internal buffer */ + if(len) { + CopyBlock(pSrc, pBuffer, len); + idx += len; + } + + /* update length of processed message */ + HASH_LENLO(pState) = lenLo; + HASH_LENHI(pState) = lenHi; + HASH_BUFFIDX(pState) = idx; + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpshsmgfca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpshsmgfca.c new file mode 100644 index 0000000..121814e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpshsmgfca.c @@ -0,0 +1,113 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// HASH based Mask Generation Functions +// +// Contents: +// ippsMGF_SHA1() +// ippsMGF_SHA224() +// ippsMGF_SHA256() +// ippsMGF_SHA384() +// ippsMGF_SHA512() +// ippsMGF_MD5() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcptool.h" + + +/*F* +// Name: ippsMGF_SHA1 +// ippsMGF_SHA224 +// ippsMGF_SHA256 +// ippsMGF_SHA384 +// ippsMGF_SHA512 +// ippsMGF_MD5 +// +// Purpose: Mask Generation Functions. +// +// Returns: Reason: +// ippStsNullPtrErr pMask == NULL +// ippStsLengthErr seedLen <0 +// maskLen <0 +// ippStsNotSupportedModeErr if algID is not match to supported hash alg +// ippStsNoErr no errors +// +// Parameters: +// pSeed pointer to the input stream +// seedLen input stream length (bytes) +// pMask pointer to the ouput mask +// maskLen desired length of mask (bytes) +// hashAlg identifier of the hash algorithm +// +*F*/ +IPPFUN(IppStatus, ippsMGF, (const Ipp8u* pSeed, int seedLen, Ipp8u* pMask, int maskLen, IppHashAlgId hashAlg)) +{ + /* get algorithm id */ + hashAlg = cpValidHashAlg(hashAlg); + /* test hash alg */ + IPP_BADARG_RET(ippHashAlg_Unknown==hashAlg, ippStsNotSupportedModeErr); + + IPP_BAD_PTR1_RET(pMask); + IPP_BADARG_RET((seedLen<0)||(maskLen<0), ippStsLengthErr); + + { + /* hash specific */ + int hashSize = cpHashSize(hashAlg); + + int i, outLen; + + IppsHashState hashCtx; + ippsHashInit(&hashCtx, hashAlg); + + if(!pSeed) + seedLen = 0; + + for(i=0,outLen=0; outLen>24) & 0xFF); + cnt[1] = (Ipp8u)((i>>16) & 0xFF); + cnt[2] = (Ipp8u)((i>>8) & 0xFF); + cnt[3] = (Ipp8u)(i & 0xFF); + + cpReInitHash(&hashCtx, hashAlg); + ippsHashUpdate(pSeed, seedLen, &hashCtx); + ippsHashUpdate(cnt, 4, &hashCtx); + + if((outLen + hashSize) <= maskLen) { + ippsHashFinal(pMask+outLen, &hashCtx); + outLen += hashSize; + } + else { + Ipp8u md[MAX_HASH_SIZE]; + ippsHashFinal(md, &hashCtx); + CopyBlock(md, pMask+outLen, maskLen-outLen); + outLen = maskLen; + } + } + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm2pprecomca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm2pprecomca.c new file mode 100644 index 0000000..15ed1bc --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm2pprecomca.c @@ -0,0 +1,2540 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// EC over Prime Finite Field (SM2 precomputed) +// +// +*/ +#include "owncp.h" +#include "pcpgfpecstuff.h" + + +#define OPERAND_BITSIZE (256) +#define LEN_P256 (BITS_BNU_CHUNK(OPERAND_BITSIZE)) + +/* SM2 affine point */ +typedef struct{ + BNU_CHUNK_T X[LEN_P256]; + BNU_CHUNK_T Y[LEN_P256]; +} SM2_POINT_AFFINE; + +extern const __ALIGN64 SM2_POINT_AFFINE precomputed_ec_sm2[37][64]; + + +#if defined( _IPP_DATA ) + +#if !defined(_DISABLE_ECP_SM2_HARDCODED_BP_TBL_) +/* see ippcp_baseptbl.cpp test for generation details */ + +const __ALIGN64 SM2_POINT_AFFINE precomputed_ec_sm2[37][64] = { +/* digit=0 base_pwr=2^0 */ +{ + LL(0xf418029e,0x61328990),LL(0xdca6c050,0x3e7981ed),LL(0xac24c3c3,0xd6a1ed99),LL(0xe1c13b05,0x91167a5e), LL(0x3c2d0ddd,0xc1354e59),LL(0x8d3295fa,0xc1f5e578),LL(0x6e2a48f8,0x8d4cfb06),LL(0x81d735bd,0x63cd65d4), + LL(0xbc3be46a,0x0af037bf),LL(0x2d8fa938,0x83bdc9ba),LL(0x5788cd24,0x5349d94b),LL(0xcaa5736a,0x0d7e9c18), LL(0x69db9ac1,0x6a7e1a1d),LL(0xc4a8e82b,0xccbd8d37),LL(0x9b7157ac,0xc7b14516),LL(0x6c21bdf5,0x947e7465), + LL(0xab589e4a,0x1cda54fd),LL(0xdb4f0a0d,0x26765289),LL(0x8ceb4a0a,0x0a265a30),LL(0xfe887c64,0x3019fd6b), LL(0x4b2fc190,0x0a10fbe9),LL(0x87cbce60,0xf40aa52b),LL(0x6dc13c97,0xcc496bfa),LL(0x5bb3fbb4,0x28ad3478), + LL(0x98615060,0x393f7c5a),LL(0xe9016209,0x487ea27f),LL(0xa09f9020,0x8a86bcb4),LL(0xc899dbe1,0x50dc8e3a), LL(0xfd619998,0xfc099043),LL(0x7c7383bd,0x1de135ea),LL(0x32cf70ed,0x4d0bd556),LL(0x25bce9e3,0x6ffc31c5), + LL(0x6a9c8162,0x9a575633),LL(0x21dfcc53,0x15aa58f2),LL(0x1ef5f4c5,0x7ad354bf),LL(0x63f875b9,0x0f443ef3), LL(0xd3450133,0x2e81d68f),LL(0xe3607d18,0xb30f4bbd),LL(0x362258ef,0xb1826a4c),LL(0x142a6768,0x7b415276), + LL(0x0acd72ba,0x136a9c4c),LL(0x5e7ec73c,0xb1274a25),LL(0x5de34db6,0xf15a876e),LL(0x8cba8047,0x85e74ca0), LL(0xb469eb37,0x08454cdd),LL(0xc99754f8,0x8fbf6d1f),LL(0xec30e984,0x1060e7f8),LL(0x4b8c598a,0xb568bc97), + LL(0x81f06784,0xaa3531c7),LL(0x07132520,0x0b894193),LL(0xacfe18c5,0x84ee5b69),LL(0xd9fbec28,0xbbf492e0), LL(0xe5f6186d,0x313a35c1),LL(0x757a01b8,0x0e449a2e),LL(0x2bd99baf,0x96c9b992),LL(0x3b84d777,0x2ba05a8f), + LL(0x09122670,0xde523a1c),LL(0x22cc810c,0x90be6f2a),LL(0x4387df9e,0x086e6341),LL(0xd9c44134,0x115c2fc0), LL(0x8799302a,0x9334430d),LL(0xe27b7ea4,0x693b3500),LL(0x9a8f3382,0xcbe1136f),LL(0xb5778247,0xe77fd5f2), + LL(0x30fbde86,0x98e795c3),LL(0xab21af8f,0x8e5e0495),LL(0xb48669b4,0x3925bf83),LL(0x469522c8,0x77d88740), LL(0x987b04ce,0x8fbf8b5b),LL(0x3aff4428,0x63c563a8),LL(0x53a6e969,0x5dc11165),LL(0x32697f4c,0x822a6c24), + LL(0x642cb143,0x6774298a),LL(0x2d110e71,0xecdb60d8),LL(0x1388728e,0xe810b11b),LL(0xd8603a8a,0x2e8237d8), LL(0x50aeeae1,0x673968fc),LL(0x6746a3f4,0x08c65d19),LL(0xd7dd7165,0x7a61a6b5),LL(0xa9b6df3a,0xe31bbfd9), + LL(0x3421e115,0x2b252ad0),LL(0xc6affc01,0x7557c8c7),LL(0x8a509267,0xd90c19fd),LL(0xe0d871c8,0x483da168), LL(0xc10729bf,0x72d6f9b3),LL(0x15b7061e,0x5dd84021),LL(0x9f2c587d,0x9bfea2db),LL(0x98641ec2,0x528398a7), + LL(0xf3afdd62,0x18a65d8d),LL(0xd6d7e4e4,0x89f38500),LL(0x9d8d4f07,0x65708c6a),LL(0xd0bdc7f4,0xb90ea13c), LL(0x8c3e2b32,0x58985855),LL(0xbcfad3a1,0xfa48d5c5),LL(0x62385ffa,0x5c3544e7),LL(0x7e72aeb7,0xb6bd39ed), + LL(0x8000fe4e,0x34e51c6a),LL(0x89c46941,0x7da2bdfd),LL(0xe1bc2b2e,0x667ba91d),LL(0x10a73e5c,0x3c80c9d0), LL(0xc7f5c64d,0x4fadebbe),LL(0x3ea35052,0xaef09eb4),LL(0x26ec55f9,0x167ee11b),LL(0x85189260,0x45fa508a), + LL(0x22542fc3,0xa0e9a439),LL(0xddac78dc,0x3f194a6c),LL(0x6f74d053,0xa75ae72d),LL(0x097c6617,0x0f8babeb), LL(0x4303c247,0x1d12bc5c),LL(0xbd1e246c,0xfe0c027a),LL(0xb69b55ad,0xe9ca1a99),LL(0x117cd63a,0xff6cd2b0), + LL(0xdde97d4d,0xf3489343),LL(0xbbb2ce1f,0x9c14e38a),LL(0xcfddf221,0x25866911),LL(0x460efef1,0x0df89411), LL(0x73ae8326,0xf713f30e),LL(0xcdd274a1,0xd9be66a8),LL(0x36885947,0xdf915ae2),LL(0x7878b781,0x2c5c1e9e), + LL(0x39e8a120,0xf71560c9),LL(0x7273b59a,0x7121d6b8),LL(0x8ef4639d,0x649535ce),LL(0x14cc6d58,0xcd01076e), LL(0x96e74f8a,0x2705729a),LL(0x5533037e,0xb07e3230),LL(0x663c5c62,0x0846dcc1),LL(0x10fc3ac1,0x6a4759c1), + LL(0xcfbdfeff,0x3c126193),LL(0x4996d845,0x4a31dd20),LL(0x19f2b658,0x48a76ba0),LL(0x8890a8bc,0xbe330142), LL(0x308aa041,0x287b34e1),LL(0x813adf29,0xcbf5da24),LL(0xcdcdc439,0xcdfc5a58),LL(0x198a6075,0xbda3bda2), + LL(0x1497fe38,0x639f92bc),LL(0xd58bd278,0x8ed8eeac),LL(0xb417bfe4,0xcf5d7ce6),LL(0x44400c59,0xf617c54e), LL(0x7d8dc939,0xde635635),LL(0x241baaff,0x2e6a3a75),LL(0xe07e8e97,0x02f324e5),LL(0x70f9fc9d,0xeb715487), + LL(0x86712116,0xbefd3380),LL(0x884efe46,0x9b9e9707),LL(0x8c9e513f,0x611a1eec),LL(0x3b6dbcec,0xe2d8e3f5), LL(0x4f8964e4,0x7cedab1c),LL(0xf4e139f8,0xee12d062),LL(0x9a9af4f3,0x8e63c9c0),LL(0x8b907b23,0xe3246dbb), + LL(0x71099001,0x70d5bda2),LL(0x15fae7dd,0x3d876d4a),LL(0x7b69c20e,0xaba0500f),LL(0x9834adf2,0xa8e3e094), LL(0x980b21b9,0x69db851b),LL(0x788c2a30,0x274c1de2),LL(0xd47d153d,0x5caa5336),LL(0x57cef318,0xada69877), + LL(0xa0551c80,0x83879486),LL(0x658e61be,0x1611dea0),LL(0x1b935068,0x1fe95c82),LL(0x5b229223,0x8f01e019), LL(0x7e93c389,0x23017e05),LL(0x9840dd64,0xce4ac99d),LL(0x1de86399,0xddc9b900),LL(0x88015785,0x6abe5cc3), + LL(0xb3c50898,0xc09545a9),LL(0x6c05d902,0xbd443361),LL(0x2c6bcc8c,0xed71f70c),LL(0xbdf8e908,0x8dbc0b88), LL(0x4fcbcd9a,0x56eb5b98),LL(0x08114397,0xafb6fedc),LL(0xb35f7927,0x0500ce5b),LL(0x95efe710,0x7005bcf9), + LL(0x2eba7f39,0x125cbed2),LL(0x6c488d44,0xc7c42e76),LL(0x676915c4,0xdb8991f9),LL(0x9183839f,0xdf6ae594), LL(0xc79f8bd1,0x4f69c304),LL(0xaa1662fa,0x638cb070),LL(0xba6f2599,0xc7f68c72),LL(0x1f6edfa9,0x11bb84d9), + LL(0xa215fda2,0x9ed156ec),LL(0x20c5ddb6,0x19de7a91),LL(0x0668c65d,0xc1ed949d),LL(0xd0826f6a,0x96683044), LL(0x1adaa8ff,0x1e6325e0),LL(0x07ac392a,0xbc53bc24),LL(0xd9f06e44,0x2c342db5),LL(0x30db8c1a,0x3f529385), + LL(0xe7492326,0xc5957d29),LL(0x0663f829,0x3addc3df),LL(0x728cfdc1,0x8faa3169),LL(0x6b975134,0xde53aa7c), LL(0xefddc764,0xf481759b),LL(0x09edaff3,0xd605474b),LL(0x653d48c9,0xc7df1eb9),LL(0xc5040212,0xa71e6854), + LL(0xafe945b5,0x136d8342),LL(0xe9d239c7,0x91707e7d),LL(0xfb2e80de,0xeda23dc5),LL(0xff614966,0x892bed73), LL(0x838dc12d,0x2ded2367),LL(0xb002bd9c,0x73fd298c),LL(0x2c4629df,0xc548b426),LL(0x8f7e03b7,0x93605d17), + LL(0xd37c24cc,0x32861816),LL(0xe427975a,0x5bb54ee2),LL(0x32f943a9,0x6da013d2),LL(0x9bc202e5,0x0746a77a), LL(0xcd1def5b,0x6db07a84),LL(0x861d9f9b,0x9421fe7f),LL(0x692181fb,0x71767292),LL(0xc9d2441d,0x0560e7e5), + LL(0x4d7e922a,0xf1496afd),LL(0xe11fa533,0x67f42a3f),LL(0x977956cd,0x9f903e5b),LL(0x1eb49608,0x37671e24), LL(0x21fb2047,0x967950a0),LL(0x35da3c6b,0x141f96fb),LL(0xd27bba59,0xe07c3c40),LL(0x0e1af754,0xbde5ed1d), + LL(0x54f1f257,0xdc64c4b0),LL(0xb01196dc,0xecb033c8),LL(0x8202d5bd,0x54e65f4d),LL(0x2b2fd451,0x63afcc93), LL(0x30640fb7,0x1e929a39),LL(0x5b361718,0xdc91387e),LL(0xf8f0bbe8,0x10aadecb),LL(0x0977e2bb,0x81d8f466), + LL(0xbd64cd96,0xdcaa3790),LL(0xcee698d3,0xbc8ac152),LL(0xa1143c45,0xde7192f7),LL(0xf5fb9ea0,0xf7c9d826), LL(0xc9468f50,0x54aea92e),LL(0xcc427ed4,0x340f4459),LL(0x02ad5467,0x3fec5be9),LL(0x2cc6c8b5,0xec780d9c), + LL(0xb889c78a,0x7b179a8b),LL(0x0aca32c5,0x069a7ab9),LL(0x591b9a36,0xe4e5215e),LL(0x3bd54630,0x7802fb3e), LL(0x233c6eeb,0x9a479313),LL(0x4e1cbabc,0x18c612ad),LL(0xc0e36f3b,0x28a29273),LL(0x7d3deb26,0xf4e2dfb1), + LL(0xadbb3c8b,0xa6c11369),LL(0x4c8ec378,0xd78af40b),LL(0x03f0a982,0xffb3a80d),LL(0xa83be50a,0x550e3e71), LL(0x418ee45b,0x845c0fb2),LL(0x0791b964,0x5297cf43),LL(0xcc47e33b,0x676b638c),LL(0xfecf85b2,0xb1c52fac), + LL(0x3dba2c0e,0xf011b5e5),LL(0x026d4f11,0xa6c68448),LL(0xc3f206fb,0x11596db3),LL(0x29414a3c,0xc91c76dc), LL(0xb94ddc7c,0x1839b9d1),LL(0x56ae8610,0xdfb20ce7),LL(0xd8734400,0x3e2b1cd9),LL(0xf01ea540,0x59f9329a), + LL(0x2351a2a9,0x7d4c140c),LL(0xbf4c9823,0x575c1e1b),LL(0x31068df9,0x8f11c2ca),LL(0x05e6def0,0xe3c17aa0), LL(0x501c8630,0xe6281c70),LL(0xc88a412e,0xad240917),LL(0x390492d7,0x6f21bfb7),LL(0xc3a3ccb7,0x61ea1385), + LL(0x33733cbc,0x60494a83),LL(0x27ed8157,0x8da622a0),LL(0x0471ad90,0x0022b154),LL(0xd3568003,0x3bd0a4c5), LL(0xd932df23,0xdc8e2d03),LL(0x7a1f5159,0x859ed940),LL(0x2a375b0f,0xad670e63),LL(0x9520db97,0x15922fae), + LL(0x59eb1a9b,0xfb73d16f),LL(0x8511e541,0x3ee8cc1f),LL(0x1590c321,0x20d72d59),LL(0x3bd075d4,0x62eab566), LL(0xfae123ab,0xac07a7c7),LL(0x1f10af6e,0x83b89abf),LL(0x1da8ac5d,0x469962ec),LL(0x8c58c3b3,0x09761c35), + LL(0x7da90fc9,0x2c086d5e),LL(0x5cc27782,0x458e5ffd),LL(0xb9268939,0xc3f48611),LL(0xde4b9110,0x39fed873), LL(0xfda698cc,0x16ef8f78),LL(0xa973bb50,0xb028dc21),LL(0xe29b725b,0x45eb849e),LL(0x14c6eae9,0xd41b5b6d), + LL(0xc55d5720,0x5e931b21),LL(0xa0e40b19,0xb628ccb2),LL(0x000651a5,0x42044ffe),LL(0x076544e7,0x2130b4de), LL(0x3677c70f,0x38428594),LL(0xf8945d86,0xfdcdb038),LL(0x4169ae44,0xfb2e3d4c),LL(0x0d13bce2,0xd4695e9b), + LL(0x039d646d,0x45191390),LL(0xb12ba339,0x983b7a2e),LL(0x5923e7d6,0xdfd30d3e),LL(0xba9d206a,0xae3590f0), LL(0xb6d5e62a,0x7d58d334),LL(0x7e402b12,0xb15b0544),LL(0x62ae8e01,0xac57e113),LL(0xf473edee,0x4d83804c), + LL(0xc81bc828,0x2faa7c4d),LL(0xfb62561f,0xb16ed9d7),LL(0x49c2fa93,0x4c9da270),LL(0xb311d90d,0x3b014c73), LL(0xf5443332,0xd29c5d65),LL(0xeebdb7c2,0xb6457d54),LL(0x4cce9480,0xc6a0bf3a),LL(0x85355854,0xd434a3b0), + LL(0x8b2c703c,0x178ca01b),LL(0x0ab71a51,0x605bba53),LL(0x3db948d5,0x2140948e),LL(0x5fb6b8c1,0xc45b2689), LL(0xf17b47bd,0x421f66de),LL(0x2e9b3ee5,0x57627a5a),LL(0x66614339,0xedf3920a),LL(0x4b638a46,0x7ea61903), + LL(0x3c030643,0x7203d542),LL(0x5e631461,0x7112bb3d),LL(0x2bc3da9c,0x2604eac7),LL(0x32d2541b,0x2e4964e7), LL(0xe8b6482a,0x940faf46),LL(0x24d27c9e,0x8f772fcb),LL(0xca7c5f88,0x125c34d7),LL(0xd1f47795,0x9903eadb), + LL(0xe2147129,0x11aaa417),LL(0xf88a0a30,0x3ccef5c2),LL(0x90283f97,0x78d5207a),LL(0xd25226b6,0xba1261e9), LL(0xd1e7a01c,0xbfc79248),LL(0x941ab2bd,0x373f1cd5),LL(0x19a0668b,0xf0881e21),LL(0x1f77bf0a,0x7b793789), + LL(0x63d4523d,0x49c2769b),LL(0xf0399eaf,0xf8df2cba),LL(0x22a2a74d,0x5ae94c69),LL(0xefd1e193,0xd08f8d45), LL(0xc681f376,0x64341fc4),LL(0xec918711,0x3a8e25c8),LL(0x0608f50f,0xdf35304d),LL(0x9a973742,0x9b4c6967), + LL(0xbfba043b,0xb5c1f5d3),LL(0xe975f03b,0xaff4f896),LL(0xae2cbb01,0xea1f39bd),LL(0xa62915ff,0x4cc1c4cb), LL(0x89e943b8,0x5eb4afa3),LL(0x154e565a,0x8c4d27e5),LL(0x7f2bced6,0x4e2e5a7e),LL(0x4487f6a3,0x7af408e2), + LL(0x97a60de7,0xe5dacbae),LL(0x4401b0ad,0x9774834c),LL(0x8a9113f9,0x7683bb00),LL(0x42b2ba67,0xc6fe7e8b), LL(0x54e760c8,0xc0c0564d),LL(0x118606c2,0xf7b05401),LL(0xec3cd7b9,0x554a9b0f),LL(0x27916a21,0xce75ecfb), + LL(0x12118abd,0xf6638997),LL(0x097da3a7,0x2ba6e754),LL(0x0fdf9985,0x1df82085),LL(0x546c864a,0xbf73502a), LL(0xc02d9ce0,0xdfde9323),LL(0xe4dd0e7d,0x580491e2),LL(0xae43b9b4,0xe71522d2),LL(0x6a231a41,0x876e3627), + LL(0xb36362ec,0xfa8ff511),LL(0x34085959,0x11c5a9f6),LL(0x9770c62b,0x272b86f2),LL(0x7c7e8827,0xf0626225), LL(0xea1e13eb,0x929168bf),LL(0xce59b0f5,0xdb892971),LL(0x4f826f34,0x6769e31d),LL(0x0a955cec,0xfa1dd934), + LL(0xa294d7ea,0x123d9ca2),LL(0x4492569b,0x8699063b),LL(0xa8dd86c3,0x6a50eae9),LL(0x12c06c38,0x3d757d10), LL(0x3e41e556,0x5a92c2c0),LL(0x6330c21a,0xa64595eb),LL(0xe184d925,0x70d8141a),LL(0xa2f10304,0x8543f2ce), + LL(0x9eaca504,0x4559b0a2),LL(0x2617bc9b,0xb9843a4b),LL(0x1b641003,0x5b28d4ee),LL(0x4ced538a,0x3e9af8e1), LL(0x7bdf7dc2,0x3790fe89),LL(0xc32549ee,0xc7c74941),LL(0xabcd2f42,0xdcc8295b),LL(0xead078b6,0x48b29a4f), + LL(0x2040178e,0x8e8b28e3),LL(0x971725fc,0xceff8f3e),LL(0xfcee2cc1,0x4a97b6fa),LL(0xbac85b56,0x775df6a9), LL(0xd28a21cc,0x32e5cbe6),LL(0xae2b82db,0xe8b86ada),LL(0x86e38e96,0x44dfbb50),LL(0x1afc2d4b,0x45d3fe7d), + LL(0xd23f620d,0x838b356e),LL(0x4592fe4b,0x2e8fa8ac),LL(0x3af5b1d8,0x1396e1b3),LL(0xcbf50fb0,0x9c0c2ef3), LL(0x836e93e9,0xd9efb6c9),LL(0x0899163f,0xe6eb5870),LL(0xdca00d1b,0x3a2f6d77),LL(0xb40ba0d6,0x36f55f89), + LL(0x32866e57,0xf3b1701f),LL(0x59de0f2e,0xf0768473),LL(0xab57962d,0xe55d7aed),LL(0x2b60cabb,0x45004985), LL(0xd5498888,0x8d539d6e),LL(0xa5e0ff6a,0x176ce1a0),LL(0xdc088c50,0xcb7c15ef),LL(0xc9a9ae2f,0x90393d7a), + LL(0xd396bdce,0xd9c1a140),LL(0x6fb2800f,0x4215b78b),LL(0x2f76b0df,0x8939109f),LL(0x2adb40a8,0x0f250897), LL(0x3a86e009,0x4db0007c),LL(0xf968a635,0x6ef0ad95),LL(0x8eaefa78,0x58a82d4b),LL(0x493604a4,0xe8a181cb), + LL(0x520d216d,0x36c84e34),LL(0xc666171c,0x2b2ef6b5),LL(0x2ce29d37,0x9469b91f),LL(0xc15f20aa,0x3ecd84e7), LL(0x292edd2c,0xf1090635),LL(0x7c3447f6,0x6d439362),LL(0x3eea3fdf,0x51b9a0a9),LL(0x9e57e450,0x68e0d1f8), + LL(0x7380931e,0x25d249d5),LL(0x2011a45b,0x87f03fad),LL(0xefde1ca3,0x89df0324),LL(0x9a9b4330,0x52ae43cd), LL(0xa1867c1b,0xfe48bc64),LL(0x9866920e,0xdd874f66),LL(0xfcf50251,0x6942a7e4),LL(0x9c5f6298,0xf5c10048), + LL(0x00973d66,0x305183eb),LL(0x95baf07c,0x1ce66760),LL(0x74822e13,0x74c9d971),LL(0x76b5e6ef,0x2ccd7fbb), LL(0xa3e1ca18,0x51688b49),LL(0xa603f2f1,0x1beb5bbb),LL(0x962534b6,0x09a231d1),LL(0xafa92f75,0x70417ce1), + LL(0xe154bc00,0xb86b6d82),LL(0x895483e5,0x5a0b19e8),LL(0xa0ff1e44,0xb15f6c05),LL(0xfdd8615d,0x2938b88a), LL(0x971615c3,0x81800a05),LL(0xc03d2039,0x6be6d56b),LL(0xc476ce64,0xff3e57d2),LL(0x6f583ee8,0x5b509b7b), + LL(0x7c1f5d3b,0x1d92c36c),LL(0xe11df757,0x1e60b19b),LL(0xe37e36f6,0x20261501),LL(0x29bc86e3,0xb68a9aaa), LL(0xf61d23ca,0xfba81eaa),LL(0xd5adaa18,0x63440834),LL(0xa5f93bb8,0xa80d76ed),LL(0x5a728480,0x3264283d), + LL(0xe4b8c48e,0xd6171111),LL(0xde557cca,0x3ee227a1),LL(0x3cb59841,0x2bebc09a),LL(0x99bf6205,0x2f8047fe), LL(0x4c43845f,0xb78b243e),LL(0x46d3b5e0,0x484ac183),LL(0x0314524d,0xa07be476),LL(0x1ab4c447,0xc0a3aa35), + LL(0x9c341f84,0x2f302d58),LL(0x84f130ba,0x264911a7),LL(0x3ee64343,0x30bed408),LL(0x5dc5868a,0xd7d6e92d), LL(0x80adb3fb,0x92074568),LL(0xa133123e,0x005ab33c),LL(0x42e1da50,0x105119fd),LL(0xb7f6b1e8,0x6987117d), + LL(0xc2bccb7a,0xa2315af3),LL(0x8672c98a,0x95ddd3ee),LL(0x5f48f607,0xa9032645),LL(0xc5273603,0x76861e62), LL(0x88817217,0x71aaa35f),LL(0x2892afac,0x57e95b6c),LL(0x9e84c791,0xf65e909b),LL(0xaa52f3b1,0x257bcc2d), + LL(0x865c665a,0xd5f6110a),LL(0x30c08b4c,0xddc3afe1),LL(0xefec26fc,0x4df3d04a),LL(0xb035af5d,0xf229bddf), LL(0xd191b439,0x364913cf),LL(0x5a7fa8a4,0xf41b8f6d),LL(0x6f6c1219,0x677cc51b),LL(0x148b7f64,0x593afe4a), + LL(0x0d038ad4,0x80ffa5ae),LL(0x36256c8f,0xf44d3df3),LL(0xbc978dce,0x0a3077c8),LL(0x745b8317,0xf3d9b4b0), LL(0xb6b1852c,0x8bbf4484),LL(0x0e78ff07,0x0cd02ed4),LL(0x49c24238,0x91cb827e),LL(0xdaa3cb55,0x58adaee5), +}, +/* digit=1 base_pwr=2^7 */ +{ + LL(0x033fc12a,0x07e6ce4d),LL(0x4886f316,0xba4f98a1),LL(0xe66f3f11,0xb24b38f3),LL(0x5ea4bde3,0xe3f6205a), LL(0xa77b998f,0x00705387),LL(0x9549f3b1,0x2c9b4457),LL(0x533a61d6,0xdef6625b),LL(0x7e4f781a,0x4eda7e2a), + LL(0xfd120134,0xe9730aaa),LL(0xc057309c,0xb22b9089),LL(0x84726ce7,0x98e79565),LL(0xd635a584,0x0e1431a0), LL(0xe834ffa6,0xbd387023),LL(0x036ab1ae,0x64198ddf),LL(0x9124b684,0x46e5ebb1),LL(0x233b3c6d,0xa316fa44), + LL(0x84782513,0xec2a9325),LL(0x2903d20b,0xd67c8ab7),LL(0x157f9aee,0x6b65b262),LL(0x69f964a2,0x547be60c), LL(0xee0419db,0x001bf327),LL(0xf20c7005,0x92fa0800),LL(0xcdc1ccda,0x1e11e745),LL(0xe471f822,0xa785ec10), + LL(0xa1371aa4,0xbc970210),LL(0x54b5424e,0xaff481a0),LL(0x0e64269b,0xbcdf91fd),LL(0xb02fc7cf,0x18bb37bb), LL(0x6f69d439,0xd99edd79),LL(0x169514b2,0x4e27a58f),LL(0x66e19ae4,0x80eca1ca),LL(0x0788696d,0x0470e965), + LL(0x8c9d34f6,0xa6b1992f),LL(0xa5ed969e,0xaf062ffe),LL(0x3a6d7ae2,0xbca2580d),LL(0xc8999158,0xf30cd9e6), LL(0x49d1ab0d,0x93e57897),LL(0x30214280,0xcfa3aa4d),LL(0x0a814831,0x0ca8b4fd),LL(0x0b10097c,0xdad179db), + LL(0x3dfdc228,0x63778bfc),LL(0xb9648a36,0xc0bae0ad),LL(0x015a99b5,0xda8cb8ab),LL(0x8366b58a,0xb045cccb), LL(0x4164cebd,0x74ef8ef4),LL(0xc5e00e5f,0x41e71fc8),LL(0x4479468e,0x753cf906),LL(0x332ea72d,0x78b5223f), + LL(0xddebafa2,0x8fc3e370),LL(0x351f9f04,0x15ffcce0),LL(0x45b0efdc,0x3fbd5f5c),LL(0x0fe3b460,0xb8216623), LL(0x533c7db6,0xe8322fbd),LL(0x00a243ff,0xf3866d15),LL(0xa0e5aaea,0xf1194ae2),LL(0xb9287b3d,0x3e93eb01), + LL(0x5876d6e8,0x528a9e2f),LL(0xd2b622d7,0x93c48f85),LL(0x3e5411d7,0x88d9eac8),LL(0x00a70e91,0xb7e4a6ba), LL(0xf1c43b2e,0xaf18e5ba),LL(0xa2f347de,0x46578c7e),LL(0xf009638f,0x19ca736d),LL(0xbd1acd29,0xa6563f1e), + LL(0x2f4126e7,0xdf9bcd3a),LL(0xd62efebd,0xecc22d13),LL(0x10943242,0xd9b29b4b),LL(0x670136f9,0x499ffa74), LL(0x2b889952,0xa2a9ad2c),LL(0xfd132dad,0x945f306e),LL(0x15cebfd7,0xfd05b884),LL(0xc7a5627f,0x653e70af), + LL(0x577dae35,0xfefc54b5),LL(0xaac3a655,0x9d2f0546),LL(0xfac31d00,0xb96bd298),LL(0xee69563d,0x3328a51c), LL(0x43195f4e,0x5e19098e),LL(0xa998010b,0x657f4ba5),LL(0x4047ccb6,0x45f29a84),LL(0x6e423da6,0x833a5443), + LL(0xca33f42b,0x97e480c6),LL(0x06e52a05,0x20a51033),LL(0x0a9be572,0x85e87255),LL(0xb988b582,0xe8bc857a), LL(0xc183c500,0x782495e8),LL(0xfee0ae2f,0xf33a87fd),LL(0xc64d1dec,0xf069fe20),LL(0xf4b08816,0x0b6dd98a), + LL(0x99229a90,0x6e6cf298),LL(0x1d71d532,0xa6840bc8),LL(0x71e3a8b7,0x803e5407),LL(0x6afd9a0e,0xd5611ee4), LL(0xbbbefa73,0xd739ca0e),LL(0xc5ec48b7,0x6082dbab),LL(0xbbdea0ec,0xa0ab10df),LL(0xf1633e03,0xb1b7ebe4), + LL(0x7be26441,0xfa752496),LL(0x0ef683e6,0xf52cb1b6),LL(0x39dd611d,0x1c96401f),LL(0x7bb19083,0x09c5a35b), LL(0x00a5d5a1,0xa2f002b8),LL(0xacf4e8ed,0x4e300ddd),LL(0xb4cc58c6,0x0d26b600),LL(0x50062651,0x5a53863a), + LL(0xad1cac22,0x62e64475),LL(0xc7e11395,0x2008653e),LL(0xd9479c4a,0xa875ad01),LL(0x804b30d1,0x3e6cf633), LL(0xb6b06e46,0x58b3ef6e),LL(0xf7b8410b,0x74c45dbe),LL(0xc278458d,0x02675759),LL(0xacd30bd1,0xb2ef4956), + LL(0x339aae8d,0x1a4a5773),LL(0x0c0fe175,0xa775b952),LL(0x5d5d5ac1,0x7b39ac1b),LL(0x11a511b6,0x3f183d49), LL(0x045ac045,0x9524e286),LL(0x4934c52f,0x0498d296),LL(0x9b636528,0x1fec5474),LL(0xc3e9b84b,0xec6f7a37), + LL(0x12ee579d,0x870b12dd),LL(0x06dd62d6,0x2a9a12ab),LL(0x071d7582,0xbcd52599),LL(0xa869c457,0x7a36193a), LL(0xe976ae5b,0xd29e6592),LL(0xadfecd58,0xe82c8712),LL(0xf714686d,0xbc83a440),LL(0x0c21e3ba,0xfe19344a), + LL(0xd7a191ae,0x2a32c989),LL(0x4e58caca,0x00a25163),LL(0xe4a11597,0x2c6501b8),LL(0x7f1891e6,0xb3e45d09), LL(0x659fd516,0xb7f532b1),LL(0xa7002930,0x99cf64de),LL(0xf2cd2d4d,0x56357ed4),LL(0x3447951f,0xa94cf5c5), + LL(0x76a164be,0x26c7f244),LL(0xa72e974c,0xbd83e20b),LL(0xda31de06,0x64e9c241),LL(0x1cdb203d,0x022bc0f0), LL(0x55c0601f,0x5eec4fcb),LL(0xb168a484,0xa1504f91),LL(0x1243026d,0xb9cf98b1),LL(0xfb3e5a1c,0x6a009deb), + LL(0x60657650,0xf1df3752),LL(0xcb1b8d9e,0xa5bbd8f5),LL(0x81b6af13,0x9e0d9447),LL(0x624cb828,0x8572cecf), LL(0xd003617a,0x28319d57),LL(0x996fde09,0x175c4766),LL(0x04878e13,0x168514b2),LL(0xec83a771,0x58a541d7), + LL(0x29fb000f,0xafdaad3b),LL(0xc20f56f5,0x1977a8de),LL(0xc5b7ba77,0x450faf6f),LL(0xe5954518,0x93253964), LL(0x644c3385,0x11ee0f31),LL(0xa8a57bad,0x6c24de9d),LL(0x5533a7ba,0xe8ff408c),LL(0xeace56fa,0x660a74d9), + LL(0x8cc2a866,0xb4b2543b),LL(0xeffc0cbf,0x69f23f18),LL(0x5308b9b1,0x0db4682a),LL(0x17037e08,0xce7fac53), LL(0x0a885b01,0xf02098c4),LL(0xb2e4eb6e,0xd375c03d),LL(0x70d4b81b,0xb6d4f6c1),LL(0x7ce5f297,0xa2b5e9cd), + LL(0xb8a233c3,0x787229cc),LL(0x3419867f,0x44ef5dd8),LL(0x79d3d8dc,0x00316d22),LL(0x90bb1410,0xdcf32003), LL(0x835d2264,0x62ad0125),LL(0x0ed6605c,0x768c8658),LL(0xfc44e760,0xa31abf17),LL(0xbb22e570,0xc91848ac), + LL(0xb16c805f,0xad1882f5),LL(0x7ccf9e9a,0xb74cc0ed),LL(0x7b122dd7,0x9635af23),LL(0x5c3cd11b,0x48a20903), LL(0x34c1eb54,0xa24820b6),LL(0x5284dcb3,0x31a3c330),LL(0x069c2ee4,0xd966cf59),LL(0xb3ff9335,0xa74eec6f), + LL(0x4620e739,0xf44eeb99),LL(0xf4159a9a,0x7663a596),LL(0xb4b745b1,0x79c54f42),LL(0x59db9482,0xa8d34937), LL(0x579501df,0x35fad92a),LL(0x289d7c2b,0x1d81bbe3),LL(0xddf3d371,0x1d60a274),LL(0x46df1233,0xf08e23e5), + LL(0xf3a95f04,0x4bc4c079),LL(0xa8626015,0x0b43e660),LL(0x246ae3ac,0xedb31526),LL(0x41247209,0xa8536eb6), LL(0xfdfacc62,0x6893a7df),LL(0xc557777b,0xf3de226f),LL(0x0d7f4265,0xa68c8d8c),LL(0x15c685e3,0x55a628eb), + LL(0x5ecec6ee,0x8cad8f87),LL(0x2a06c242,0x4aefda2d),LL(0x57f00a7d,0x46a21033),LL(0x7ed125cf,0x91910c3a), LL(0x541165d2,0x0b7f0e4a),LL(0x553eeec1,0x15ed1b93),LL(0xd24e020b,0xadf5b4db),LL(0xa7493b8b,0xf05307a3), + LL(0x62070042,0x725548dc),LL(0xc274916a,0x74d71526),LL(0x6f098d01,0x3269851e),LL(0xf9ec928c,0xb2e01cb7), LL(0x2b4368cd,0x96c2d922),LL(0xa0ec45d1,0x8eb84b03),LL(0x26e5b3ac,0x733ad068),LL(0x93c5a962,0xced3679e), + LL(0xdd6eb876,0x23c6a22d),LL(0xa343dc3b,0xbd98ad9a),LL(0x56054515,0x61933d03),LL(0xe45cd744,0x4a64b769), LL(0x12586de6,0x617a63f3),LL(0x7976e7d1,0x04984a9f),LL(0xcd2a0a6b,0xb00ba446),LL(0x7d059d46,0x5b64e7f5), + LL(0x6a4b08e6,0x8801ce04),LL(0xb13bbe9c,0x66f31460),LL(0x4d87114e,0xb174e887),LL(0xf348e94f,0xb2fee192), LL(0x7c822d05,0xfede2283),LL(0x8f82b14a,0x8d50c49c),LL(0x0f5f1b5d,0x21ea4f6e),LL(0xc1818095,0x68627cf0), + LL(0x8a7b2458,0xc1c0650c),LL(0x8bbc6aff,0x82ab62bb),LL(0x6ce6989d,0x7b3665d7),LL(0x7579e973,0x2ad7991f), LL(0x7e9e8510,0x701287aa),LL(0x0a18da53,0xb296a038),LL(0x2bf00fdc,0xf8c3af86),LL(0xb220dc06,0x55776951), + LL(0x7d7dd541,0x4e6e4b4f),LL(0xfe5c7431,0x812feac7),LL(0x340297b1,0x6bdfa63e),LL(0x98009910,0xecc11e55), LL(0xb25b98c0,0xee4c6165),LL(0x02c5939c,0x8a07b0d2),LL(0x23147c40,0x9b36c176),LL(0xde2eab3a,0x396054a2), + LL(0x2c439171,0x1f41010b),LL(0xe8139388,0x3ff85ee6),LL(0x8f077633,0x4ada4c7d),LL(0x824e6023,0x9976011a), LL(0xeaf49f63,0xa2501197),LL(0xd60b0c4c,0xdff2122f),LL(0xbab3df90,0x1a6a3abb),LL(0xb66ffd5f,0x854bbcc6), + LL(0x728572c1,0x525964a9),LL(0xfadbd14b,0x8a4923a2),LL(0xcd90b61b,0x03830df9),LL(0x79c2afe9,0xcdb00f4a), LL(0xa6c3f13d,0xff2f84bb),LL(0x5c0de4dd,0xdee45c30),LL(0xfba2e933,0x3e1dd748),LL(0x7c51124c,0xe9dcc690), + LL(0x28e11f62,0x725177af),LL(0x8a64fdf5,0xc8e120a1),LL(0xf24fb357,0x82ab73df),LL(0x44724879,0x2d5d1618), LL(0x96c66b86,0x09627e26),LL(0xc81d38c1,0x1d547cae),LL(0xd0f76658,0xbe8991a4),LL(0xcf11a976,0xf1508662), + LL(0x3be3e582,0xa5dafebd),LL(0x07399295,0xd9f545ba),LL(0x676f9598,0xd9f564a4),LL(0x9294431e,0xec00bddf), LL(0xc1fdc758,0xc1971113),LL(0x69a001de,0xe32f572f),LL(0xb907f044,0x048d7776),LL(0x5ca10e67,0x4a474e6e), + LL(0x3039a4b7,0x6476dd40),LL(0x018ee2b8,0x85de9baa),LL(0xfd7365f2,0x0c945aeb),LL(0x96c7267e,0x2b47dc0d), LL(0x0410de25,0xb12b48a7),LL(0x177242c1,0x3ba7a11a),LL(0x6504ff87,0x44e6cee7),LL(0x9d19f26c,0xb2605ff6), + LL(0x50fb1b6b,0xa56bb589),LL(0x71d2fb53,0x98dc1180),LL(0xa1b78e04,0xa4fdc6f8),LL(0x39d9349d,0xbea745b0), LL(0x62d7eb73,0xac474229),LL(0x8b808ac3,0x7b765138),LL(0xd0ca219f,0x882370af),LL(0x9d1c23e8,0x28dcff7b), + LL(0x3872f0a9,0xc6dc70eb),LL(0xdfb642b1,0xb2f21248),LL(0x65bbdfc9,0x86838f0f),LL(0x40b28364,0x1d04a8b5), LL(0x1e4d8d58,0xd4fa229d),LL(0xfad0a9cd,0x74ee5e20),LL(0x5a40ec4a,0x25a59aae),LL(0x3727d6cd,0x73be91f3), + LL(0xd63f64eb,0x9c31405e),LL(0x91d2f1c1,0x9943c34c),LL(0x4fcdbf34,0x70ad75d7),LL(0xb239e00d,0xa6ce7145), LL(0xcd04b9e9,0x136bceed),LL(0x44ed7f96,0xb9ebeb8d),LL(0x5d136280,0x068b43a5),LL(0x4c559b6b,0x2e1b6624), + LL(0x5472d67b,0xe3808e72),LL(0xce74546e,0x73450378),LL(0xea1d58f7,0xc1b1b66e),LL(0xe34c2a7d,0x2b576e4f), LL(0x2f732803,0xc2b1bdf7),LL(0x9f8e48c3,0x37aea390),LL(0x944f1cf3,0x8bbbb61e),LL(0x86c59643,0x5cc7ccaa), + LL(0x8d5b000a,0xaf4c18e3),LL(0x2b6d561c,0x23b0edd0),LL(0x0d6cbe27,0x11b67ef0),LL(0xb1b50e9d,0x679d789b), LL(0x372c4015,0xda198336),LL(0x65781ea7,0x5da50baf),LL(0x550201ba,0x00b3a6d4),LL(0xecfffc72,0x988b89f7), + LL(0x25948852,0xf2f08a09),LL(0x406d1a34,0x4036bbb7),LL(0x23d2dd87,0x1cd57f08),LL(0x4704dac3,0x11a4387e), LL(0xc5413b59,0xb8091a7a),LL(0x09b5fa71,0xe58940c6),LL(0x6a75397c,0x70fd5154),LL(0x5c59ff75,0xea544375), + LL(0xac25bd3a,0x15e5bed3),LL(0x6b17971e,0x1bed3c33),LL(0xbaa96968,0x046fc1cd),LL(0x7090256f,0xda1b010d), LL(0xe6677d20,0xeec55752),LL(0x24c9bb42,0x8eac5d06),LL(0x8f4120e0,0xc2f6270e),LL(0x07748faa,0xd9ae9fff), + LL(0xe8f7b7af,0x5a1b2634),LL(0x81b1612d,0x1fcd743d),LL(0x3d420398,0x6b065aa2),LL(0x41e06643,0xe758b9c7), LL(0x7f111b3b,0xe1e52b53),LL(0x83498731,0xb9ee0a5d),LL(0xea8154f4,0x49c19631),LL(0xe1c08746,0x8f5a3479), + LL(0x65dd5561,0xe032d7c1),LL(0x442bef09,0x6c3420fe),LL(0xa64eff47,0x1d390561),LL(0x902763bf,0x0d8fbf07), LL(0xa4bc6856,0x0262f26d),LL(0x9f4f2101,0x7c1b59a7),LL(0x51240642,0x663d9b38),LL(0x77ce53cc,0x39a0b4c2), + LL(0x61f5e655,0x1c896beb),LL(0x9f4bfd2d,0x75c4c049),LL(0x10111b02,0xb8799a15),LL(0xa4c2fa0e,0xc76f8641), LL(0x185fc036,0xd77ff7fd),LL(0xf5acbd16,0x53212bd6),LL(0x0408cff8,0x4ef7431f),LL(0xfb082c4b,0x45aa9d99), + LL(0xf0438565,0x22c1fa8e),LL(0x4cb43ab5,0x8e3a2ee3),LL(0x232081d1,0x457df338),LL(0x482ff47b,0xd1293d9b), LL(0x68106365,0x802a300e),LL(0xe51978c9,0xa8f27aa1),LL(0xa6a6a4d3,0x6ca0edda),LL(0x24c9526a,0x4cab1223), + LL(0x56730245,0x26234b2e),LL(0xe1b54be4,0x9a04c15d),LL(0xee89282b,0x153fb6cf),LL(0xd79d81ad,0x5901ca12), LL(0x7c3c5ffd,0xbe6853d8),LL(0x35e1942a,0x16d3efb5),LL(0x3b56bece,0x3491f207),LL(0x5b818cfd,0x0d75e0c1), + LL(0x40969df4,0x79a0e319),LL(0x9ae34b31,0x75e4632c),LL(0x68e8df30,0x4a47585c),LL(0x2a495467,0x4a4a40e4), LL(0x2762eae9,0x92b8a6f5),LL(0xc9a3d133,0xa204cd80),LL(0xd1ff23cf,0xa441ecfd),LL(0x4550ee57,0xd06feb58), + LL(0xdc032002,0xe14ca6a1),LL(0x05505a36,0x9a780e57),LL(0x08cb2b29,0xad93852e),LL(0x008b00c4,0xa54deaab), LL(0xe1042350,0x8cd2c71a),LL(0xa8915596,0x2014b85d),LL(0x97ddd1dc,0x1228b3e4),LL(0x4a3b3ab7,0xa97282ce), + LL(0x0f1559ad,0xd978cd73),LL(0x86b14d3c,0x2e877fa2),LL(0x3660f189,0x01d3dc94),LL(0x0d2b4ddd,0x90ad950d), LL(0x92245e3e,0xa8d26760),LL(0x4964245d,0xfc1bf8d5),LL(0xac3d97eb,0x31206c72),LL(0xa58c64cb,0x39dfd972), + LL(0x7efbbd16,0xd631907f),LL(0x174f1fd5,0x4fdc84e2),LL(0x5c277996,0xe81e89b3),LL(0x5f79f1de,0xcb277b4e), LL(0x2ed1962c,0x2eff44b3),LL(0x72883505,0xbe36a640),LL(0x545e69a0,0x14a1fac0),LL(0xd6f658b9,0x76dbbcbd), + LL(0x78e2e86d,0x0720c584),LL(0xcaeead35,0x52fccffb),LL(0x587fd1b2,0x06f28c72),LL(0x9e48bf69,0xec36a912), LL(0xdaa3cdbd,0x74874436),LL(0xcdc2f2a3,0xb3f7409f),LL(0x1951c078,0x0e50d7fa),LL(0xee8949f0,0xd97ff34e), + LL(0x742d7b1d,0x00db635e),LL(0x29f0d0f9,0x5c0b280e),LL(0xeabf9b35,0xafa7e616),LL(0x2c8a76e8,0x7341e2c7), LL(0x2e96f198,0x9679e34d),LL(0x90ee26ca,0x8c2661c0),LL(0x67a6516e,0x9c6dab35),LL(0x46b4b34f,0x7c8edc4b), + LL(0x2afba4fe,0xc502cf2f),LL(0x6776dbf1,0x76847ae0),LL(0xa2c3c83e,0xace02706),LL(0x4601c550,0x0012645f), LL(0xef6189bd,0x1940e14a),LL(0x2cdf5e89,0xba7f615f),LL(0x438a3781,0x698101aa),LL(0xa9e22357,0xf568a45d), + LL(0x1f913210,0x83af640e),LL(0x8d505edc,0x529a29fd),LL(0xd6b0c85a,0xdf3d3090),LL(0x6897ea43,0x46e23886), LL(0x416577ae,0x97cca980),LL(0x9aa08fc3,0x1f5a96a8),LL(0x56c05c30,0xcb014b33),LL(0x05ec9be4,0x1944765a), + LL(0xddc4371d,0x2d6789cc),LL(0xf3618fc2,0xd768f5a6),LL(0x3da93c1c,0x77065e11),LL(0x0e27b3eb,0x4ea3fbc3), LL(0x11ba30e9,0x7c1bfba0),LL(0x1036ebe6,0xfc6fba67),LL(0xd3231aed,0x0053a30c),LL(0xee3ac524,0x7f0613d9), + LL(0x63093df6,0x95ec2fd9),LL(0x7c0eea52,0xfbc63768),LL(0x8b64ea48,0xf767b286),LL(0xf75bc633,0x6959b0ec), LL(0xc9f63154,0x47e34c3b),LL(0xa524bc76,0xd616b19f),LL(0x632ac100,0xefc9bb54),LL(0xb4d96a7d,0xd9abba10), + LL(0xfe2ad7e8,0x3b7dd91a),LL(0xb4ebf343,0x29134cd7),LL(0x152864fd,0x49d1c305),LL(0x80efc220,0x3afd83d0), LL(0x3f2f0d27,0x3552517e),LL(0xfda48969,0x0a2b5006),LL(0x3c3e8ec9,0x568863ed),LL(0x891edec9,0xd99d5c62), + LL(0xd1c9d6ee,0xb0ddc129),LL(0x57db23b4,0x373dad74),LL(0xb416c7df,0x7c178b0b),LL(0x4f8a7153,0x77431dac), LL(0x41c1367e,0xf5288888),LL(0xb838c91c,0xf1518939),LL(0x541f3281,0x81e17838),LL(0x65b2bde5,0x70030244), + LL(0x7350251c,0xdc309424),LL(0x7c811130,0xfac0c6ad),LL(0x6a141269,0x3817aa1a),LL(0xe10b4a6d,0x1aa5a92f), LL(0x34648a96,0x996cca7f),LL(0x4e2a4f52,0x517a25b9),LL(0x38b1547c,0xff95ac42),LL(0xd9b9cd4f,0x01d981b6), + LL(0x88d60eba,0xcc34d15e),LL(0xa0ea1a51,0x45851bf4),LL(0x82854ee0,0x5d5f9b30),LL(0x176ea156,0x914be21f), LL(0x2a05c368,0xecac86d1),LL(0x73a666a8,0x255cb9c0),LL(0x78c0eec5,0x5e4799d9),LL(0x8fc05a71,0x40ed8989), + LL(0x8ae03edd,0x54888ac2),LL(0xa83b554b,0xef3e9865),LL(0xb7612fe4,0x47b41822),LL(0x8f76cd2e,0xf6e16fd5), LL(0xa977b5dd,0x091c7b12),LL(0x8f99d4aa,0x7051bf6b),LL(0xfed218fe,0x9f737902),LL(0xb752c612,0xd8112477), + LL(0x18d13bd3,0xbb45c287),LL(0x23c6dd1a,0xbbf3a894),LL(0x13b9cf87,0xc8171c5e),LL(0x34f5348d,0x2dfc7792), LL(0x985cabd4,0x9b9a662d),LL(0x4d971de0,0x588a6ebc),LL(0x574cba64,0xda9fd894),LL(0x651e6e67,0x7e0f0cca), +}, +/* digit=2 base_pwr=2^14 */ +{ + LL(0x4b8bceb8,0x88ca276c),LL(0x752d1106,0x6d4ec101),LL(0xf834dcbf,0x2ad98063),LL(0xdfff19d0,0x4da81d19), LL(0x3a9828ff,0x4ccc7cd2),LL(0x2e64b332,0xf1e389b0),LL(0x7308b81c,0xe2fb6a6c),LL(0x5bcc0ac6,0xc6df66b2), + LL(0xe1c58c80,0x5ccb8c75),LL(0x83fcc95a,0x2ba9de04),LL(0xdfccbcf9,0xccdeb0ee),LL(0x70f3d3ad,0x1d667d4f), LL(0x36269820,0xc6aa14a5),LL(0x0fe87940,0x329a308b),LL(0xede5cfb2,0x39869970),LL(0xf601bb2c,0xc33c3068), + LL(0xa1a8781b,0x3087444a),LL(0x5cff3cbf,0x6cb5b706),LL(0x83082714,0x7673a8e4),LL(0x0842a792,0xc4bce015), LL(0x53e2a531,0xae71a033),LL(0x8b5315f9,0x147b28f8),LL(0x6c5ab37a,0xcc460133),LL(0x540dde16,0xb1dd088b), + LL(0x11c09289,0xec250455),LL(0x164079c9,0x83042ba7),LL(0x6e3879a2,0x4881640c),LL(0x802452ee,0x77c5babc), LL(0x7088f360,0x7a7759a6),LL(0xb74be7e9,0x02da352c),LL(0xe0338289,0x15800cdb),LL(0x501688c6,0xad69f7c9), + LL(0x57ae1213,0xb7d35063),LL(0x97024ecd,0xd536753a),LL(0x24938196,0x9d680716),LL(0x44ed6d4e,0xac1bee4c), LL(0x33e95503,0x6dd9c9bf),LL(0x88fc1c3d,0x5ee9f1fd),LL(0x21654473,0x4a701ff4),LL(0xbd2ffe36,0x9a316450), + LL(0x103b5fa2,0xe9130a63),LL(0x8eee983b,0xe97f7120),LL(0xe8749cba,0x54b7f85b),LL(0xbb1bca55,0x69976910), LL(0xf4e621d3,0x9ec4034f),LL(0x695e17da,0xaad567ed),LL(0xcedb2ea8,0x7647f054),LL(0x09fc7433,0xf85f944c), + LL(0xb95eeddd,0x30af23b3),LL(0x89985f3d,0xfd1d565a),LL(0x4c254738,0xfbb53173),LL(0x171170a4,0xb07ba56a), LL(0x294d55d1,0x5069882c),LL(0x792694c1,0xae0385c4),LL(0x11225dc6,0x0a0c7927),LL(0xe22867c9,0xadcc5f08), + LL(0xaee03999,0x164ac67f),LL(0x79ff7f91,0x4de174d3),LL(0x548da6ea,0x063e4943),LL(0xdb7ccdf7,0x5264880b), LL(0x49b992cc,0x4a18f34b),LL(0x14065870,0xe16b6f4d),LL(0x4cdb0e21,0xd32479ac),LL(0x162bc9f8,0xce8151f6), + LL(0xe8f78545,0x0f8d9a2f),LL(0x3145b086,0x091643db),LL(0x23a1bcc9,0x5915a582),LL(0x8a280fc7,0x97348efd), LL(0x65eccf5d,0x3f9d6236),LL(0x01ac8146,0xd1a34937),LL(0x8ad0d5c1,0x1b8e5128),LL(0xd581dd11,0x5cbcc9ef), + LL(0xed059f1d,0x947ceaff),LL(0x7460a186,0xf5754d03),LL(0x0164ff7b,0x37698fa6),LL(0x35805339,0x630900d2), LL(0xeddd6bbc,0xe467a6be),LL(0x5e36b12e,0xc53bffec),LL(0xf831fc7d,0x06dfd3f9),LL(0xdaef86ac,0xd995fcc4), + LL(0x32d5b2e3,0x7d148468),LL(0x6335f566,0x7796b94c),LL(0x6769b8bd,0x693983d6),LL(0xed5244fa,0xff0306aa), LL(0x89b8e801,0x2e90d41a),LL(0x39e732f3,0x1af09d86),LL(0x320ccb1d,0x96d14e1f),LL(0xc05dcece,0xbaf21c6f), + LL(0xc216cf37,0x8ae82a1c),LL(0x773828bf,0xac437f45),LL(0x9d51a85b,0x8c12ff18),LL(0x34c16578,0xfeb563be), LL(0xc6706966,0x9d9353b6),LL(0x0cda8733,0xcdc6eb5a),LL(0x3e4953db,0x033c186e),LL(0xb2e37f7c,0x2ba46a66), + LL(0xb9f3ee06,0xb32115e2),LL(0xdd6346a9,0x1bc12cec),LL(0x321242fe,0x6b9c2142),LL(0x5c68ea06,0xcf9b9bb3), LL(0x920d49bc,0x7fe554ac),LL(0x37aedebb,0x90b3a9b4),LL(0x7695af86,0xacb181e0),LL(0xfd567fea,0xd1c99c55), + LL(0xfccf76eb,0xb7c18083),LL(0xf93113a3,0xc693bdbb),LL(0x66e03205,0x215ff05d),LL(0xf76d2a12,0x4424aaea), LL(0xe7f30891,0xb23f2782),LL(0x062db479,0xad814d5e),LL(0x4aea78c3,0x347ec1d0),LL(0x6a2332f2,0x3d0f0a7e), + LL(0x8ad9e323,0x02ecefa6),LL(0x0d45e0c9,0x16c81248),LL(0x2757306c,0xd4b6253d),LL(0x81e42d04,0xe90203a3), LL(0xc13782f0,0xbcef10fb),LL(0x156267d4,0x823efe5d),LL(0xfddb0092,0x18add11a),LL(0xb104561a,0x27068a29), + LL(0xda0abf3e,0x7eb7f516),LL(0x61b3381f,0x3c92ac94),LL(0xd3418870,0xbad7320e),LL(0x07dbe066,0xbab7a126), LL(0x2def303f,0xe7ce59be),LL(0x2d1e0c9f,0x0bf1f237),LL(0x38f418dc,0x12c18d1e),LL(0xb85bb676,0x7fcc5e3e), + LL(0x1b038ac6,0x0bcf25ad),LL(0xddf1becb,0x35388760),LL(0x8a1ad137,0x5734bf37),LL(0xb7645056,0x92f3a250), LL(0x718a5ace,0x6ed926a4),LL(0xb967f1cf,0x8e63f0a2),LL(0xd835fe33,0x6d9cccc9),LL(0x31f82e18,0xb1b5efee), + LL(0x24f2c6b1,0x997aa2a4),LL(0x9e536a91,0xde87114f),LL(0x0f819ec8,0x01938bd2),LL(0xef772a43,0x012e9031), LL(0x77aa9256,0x1578eb4c),LL(0x61a0c8ed,0x052b4088),LL(0x6ab5a380,0x1153a330),LL(0x132f5675,0xa3e7f085), + LL(0x909b8a41,0x5e946e00),LL(0x16a3c156,0x55f7d231),LL(0x8ac8f8e3,0xcd952464),LL(0xcd8d67f7,0x7c5184d4), LL(0x39ef93a7,0xb3468964),LL(0x4e9058c8,0xf4aa0b7e),LL(0x4b7c713f,0xa409403e),LL(0x41a83e50,0x9d55e33c), + LL(0x7e1754b8,0x9efee704),LL(0xb1c0027b,0x54085471),LL(0x45af4e6d,0xc5e7a6fa),LL(0x30048569,0xb4d3cd58), LL(0xf3ae8e79,0xd2c20014),LL(0x849f3f23,0xd0b6af13),LL(0x17982a8c,0x3a1db915),LL(0xf9ffcf90,0xea3c8099), + LL(0x7bada472,0x25d9eb82),LL(0x09afd498,0xff84d98c),LL(0x56ff21f4,0x5e2c1ffe),LL(0x2f2f3a94,0xafd07201), LL(0xcdb673bc,0xb0227fe6),LL(0xfe8d7326,0x58fc0e7e),LL(0x191bfd4d,0xb988d3eb),LL(0x2474d8b6,0x82499093), + LL(0x68caff21,0xd1ef53cb),LL(0x5074160a,0x3cff018c),LL(0x98f982fc,0x609a4688),LL(0x562a099e,0xee5caaac), LL(0xf8c6cfd7,0xf650365b),LL(0x9cbc10ee,0x2652aa23),LL(0x6ab86f4e,0x904fd66e),LL(0x2d82f3d8,0x6a25bbc3), + LL(0x19c7a275,0xd3e6ecad),LL(0x3604b2dd,0x05ed0451),LL(0x00c71863,0xdd1d87e2),LL(0x8cd23356,0xd9fc8793), LL(0x0036b81f,0x3337f8ba),LL(0xb5300622,0x63b5a762),LL(0xce8800e3,0x4cf696f1),LL(0x07e3cbc3,0x12cb3261), + LL(0x0fa12b5b,0x18eac953),LL(0x77d159b5,0x45ccf073),LL(0x6e844a0d,0xa7480444),LL(0xd77d1c18,0x4404e6c6), LL(0xce1af18f,0x003e43a6),LL(0x17fdffcc,0x8a828081),LL(0xcabf3d17,0x91b63c11),LL(0xad26f286,0xa4dedc21), + LL(0x1a2b1579,0x6bf62b69),LL(0xceeb29ff,0x3b67b87b),LL(0x40d4b996,0x451ffadb),LL(0x080978f8,0x10c6ae50), LL(0x2c242dc5,0x959d47e2),LL(0x5423e158,0xced9e922),LL(0x8d8a68f1,0x9a212d4c),LL(0x3708393f,0xeff3d644), + LL(0xfbaffded,0x43f51810),LL(0x0f6fd7c3,0x3886ccb4),LL(0x13c31946,0xb939247b),LL(0xaa1fd72a,0xbc1ee613), LL(0x631fd790,0x6d40140a),LL(0xd26b3fd9,0x9382e3ba),LL(0xb3af96c3,0xff414370),LL(0xe0ea9ad6,0x38c813cf), + LL(0x157594da,0xf8844c3c),LL(0xcac628bd,0x2a7b514f),LL(0xc08c5107,0xc023e4e2),LL(0x3f2722fe,0x6c164496), LL(0xc03a22ad,0x842e1d06),LL(0x37ddae0d,0x5dbc2865),LL(0x0342bc72,0x46dfc88d),LL(0xa4a3c65c,0x873c805c), + LL(0x60aa5c14,0xd202853b),LL(0x3850cc05,0x1dc35d34),LL(0x0cabccfd,0x8014357e),LL(0xc5a5225a,0x1aa44ce9), LL(0x3a8444b4,0xa3cef920),LL(0xc95384b1,0xcf3f91b3),LL(0xc9e5da54,0x1d625ba1),LL(0xb1d0f46a,0xbf1fba37), + LL(0xfb3f4885,0xdcef4fad),LL(0x3267f912,0xa49debb2),LL(0x1e121cb8,0x6417d37a),LL(0x533e94c9,0xa6d871fc), LL(0x2e4834fb,0x89f80208),LL(0xb353452f,0x27e83f0f),LL(0xe1f8f322,0xaf009f3c),LL(0x89319fd8,0xa5b77a77), + LL(0xedf71900,0x0a89e741),LL(0xd514d93f,0xd679b841),LL(0xb0a03702,0x8878577f),LL(0x85a209ae,0xc9607b78), LL(0x59432a28,0xb7bd0616),LL(0xed567145,0x0da060a2),LL(0x1a449f52,0x44e35a7a),LL(0xbbaccc0f,0x9c9a2c82), + LL(0xd83701f4,0x83abd436),LL(0x4bb9cbe8,0x56e8bfe8),LL(0xc631cd1e,0x5b545cc8),LL(0x955aca7d,0x6d03426f), LL(0x2f8db817,0x049fc9fa),LL(0xdc59675f,0xfcec1799),LL(0x0455f095,0xa00ed392),LL(0xe5096b18,0x6d7cfa5f), + LL(0xe30ae90b,0x2cda5cae),LL(0xcaabea0d,0x2cc34290),LL(0x41e67856,0x564afcd9),LL(0xcf6ef8b7,0x210c7a09), LL(0xf82a591d,0xc316d352),LL(0xab43d2a1,0x5fe8cc4d),LL(0x8b4e9470,0xd8ebce97),LL(0xba321a07,0x26c78f44), + LL(0xd75e509c,0xa63f4b34),LL(0xb9a6c63b,0x9122bbc5),LL(0x17942443,0x8bf792a3),LL(0x7f4f70f9,0x95b05d68), LL(0x13b70dc8,0x57d7dee5),LL(0xfc376fdd,0xe84259ed),LL(0xe3e313b4,0xf8c4c4ff),LL(0x13fa8ff1,0xf8e2d3da), + LL(0xf8e4eede,0x9692c390),LL(0x0e95a902,0x3b514551),LL(0x7360623e,0x45c1670c),LL(0x6abd2a82,0xf7a74f55), LL(0x24e8e721,0x99b16e7e),LL(0x512f1401,0xae52fa2a),LL(0x3f3a09d5,0x46c60e80),LL(0x0750e968,0xf803d1b3), + LL(0x1791644e,0x17840d2f),LL(0x3b7981e6,0x3e32b3db),LL(0xd3dfae10,0x2d0830a5),LL(0x6cc6dd0d,0x1b28d118), LL(0x78368274,0x944a9889),LL(0x55b1bf81,0x310da94a),LL(0x0d739056,0x503061ec),LL(0xb4d73288,0x1947e940), + LL(0x0228346f,0x760ee846),LL(0xc5cff077,0x108765b3),LL(0xbeb12160,0x22092b39),LL(0xb63001af,0xa631d553), LL(0x0af3d43a,0x9340cac4),LL(0x60d338a3,0xe6cbfb54),LL(0x7ca3f604,0x2280ff0c),LL(0x3ba738cb,0xaf48f86b), + LL(0x47d372ff,0x7435dd78),LL(0xf005c006,0xbf9c7149),LL(0x7a8d0e81,0x624084b9),LL(0x4840496c,0x50b578f3), LL(0xb52a4266,0x414ca2c1),LL(0x5535ef0b,0xa3c30275),LL(0xb50f7f47,0xd4b808c1),LL(0x9a199920,0xe6781ae2), + LL(0x27a91ef8,0x50141234),LL(0x5b77d060,0x2f4f5937),LL(0xc2dcb03d,0x1be8269e),LL(0x9f65043f,0xa293017c), LL(0x8caac401,0x1678dfe0),LL(0x968b1716,0x4942d8ce),LL(0xae36e201,0xa9b55fae),LL(0xd5279632,0xcfe4bde3), + LL(0x79d637e6,0x6126d744),LL(0xd63b4aad,0x8491f1a8),LL(0x9816b82c,0xdf97b736),LL(0x796408c1,0xafca2c36), LL(0x7a8e8058,0xc17f3f01),LL(0xe74705e2,0xb3335a24),LL(0x46e3e3b0,0xee200023),LL(0x40630e08,0x07bce061), + LL(0xee8f9dfc,0x46b42c00),LL(0x9e9b7f58,0x3b8e8509),LL(0xd36e8e89,0x83df4b18),LL(0x15d50555,0x09631af5), LL(0xef1ee3f1,0xb7906b77),LL(0x4bd1e17b,0x8272dc83),LL(0x4903faac,0xf160bfd9),LL(0x0dc71e59,0x7fe9e999), + LL(0xe714187d,0x6ee9b790),LL(0x9d5a656f,0x7391ec2a),LL(0xe10b20f0,0xcbb55ec6),LL(0xec3645d6,0xbba3b57b), LL(0xe18322e8,0x9c3265bc),LL(0x93328c91,0xdb49b0f3),LL(0x49c2bbec,0xa911db72),LL(0x6e5bd229,0xf71b4df3), + LL(0x7ba27baa,0xdccede33),LL(0x4b712a97,0x1af4476a),LL(0x8a8683ad,0xf0aaabec),LL(0x6fa8e84c,0x138cdac5), LL(0xdc78b1ad,0xd2d50b00),LL(0x696442b9,0x26fc0b72),LL(0x125bf11b,0x12cd5d8b),LL(0xc4f82ca6,0x2a2ce980), + LL(0x52e00dd3,0x9921c0a6),LL(0xf1d7e1af,0x98e8707a),LL(0xdf03b040,0xaa7aa8b8),LL(0xdff6bd74,0xb3ba8b23), LL(0x31db8c0b,0x2fd0faab),LL(0x2819b732,0x4697e9bf),LL(0x0425b866,0x2dc3a5d0),LL(0xd97816f1,0x4b9e7899), + LL(0x4c756c70,0x1355c412),LL(0x0fa089af,0x2d4c4eee),LL(0x3b8a01b5,0x4d8425a8),LL(0xa3531d3a,0xcc26b8a9), LL(0x7ebd9eea,0x6eebe11b),LL(0x92c0f858,0xd511a797),LL(0xec49a0c8,0xaa863f01),LL(0xa8242995,0x7fb65625), + LL(0x3dbc00c3,0x9de9d3f4),LL(0x3f7d61ab,0xb846152f),LL(0xd0d74549,0xc060fdbd),LL(0x7b273702,0xe722aab2), LL(0xd81b6f6e,0x9e54f098),LL(0x9e2fde1f,0x32dbaa5f),LL(0x9ebbc796,0x14cc9995),LL(0x0eb83921,0x4ca6686c), + LL(0x10a0c0bd,0x6e65d7c6),LL(0xb3c0f6cd,0x1f6930d7),LL(0x4d783d6f,0xe4e0a933),LL(0x70b20ad4,0xc945ee7f), LL(0x034b0265,0x521bd135),LL(0x0fa9be95,0xeb5d96e0),LL(0x357ef592,0x834c28c2),LL(0xb81df99f,0x08ab5b4c), + LL(0xf464825d,0x6be99d80),LL(0x9a0c1293,0x1cc83719),LL(0xe7e43c6a,0x76616803),LL(0x91cc47ac,0x6fa33715), LL(0xdbfc08b9,0xc3fdb99b),LL(0x68e2b249,0x66e1ef2d),LL(0x64a4a438,0xd3d8ef7f),LL(0xa6f25b00,0x775a70fc), + LL(0xa0cb5443,0x2444c682),LL(0x4b743ee7,0x264c2662),LL(0xa303eb20,0xd7a1adc4),LL(0xf60a5b98,0x3f14821b), LL(0x1a1d7661,0xa439102d),LL(0x8d8a5a1a,0x47c25a37),LL(0xa34c66a9,0xdf4a48db),LL(0x4c828c73,0xab467364), + LL(0x3459cc8b,0xd3caad73),LL(0x181b16c2,0x08eeb442),LL(0x70600d33,0x3444abbb),LL(0xcd0f8e70,0xaa2a39c4), LL(0x24836d70,0x5fc6ae8f),LL(0x47d32fd4,0xc119be84),LL(0x0d6000ce,0x2b3f3771),LL(0xe602337a,0x439893a8), + LL(0xc1e8e564,0x4b75ff6e),LL(0xe451cf42,0x6185413c),LL(0x162c3150,0x0276d3b6),LL(0x3aea9c55,0x844539e0), LL(0x42e9d70b,0xfc629ee6),LL(0x0be610c9,0x4eb9b7e6),LL(0x39ca3d92,0x8c53fda1),LL(0x14c2e9e2,0xd2e4cfa6), + LL(0xf14b31b2,0x3c1f6895),LL(0xeb951fad,0xad42d951),LL(0xb8f10fc1,0x5b20a169),LL(0x586c61cd,0x284810bd), LL(0xe863d781,0x0c4a89aa),LL(0x9c235d5c,0x2eda4847),LL(0xe6005150,0x8e141950),LL(0x52785efa,0x75716e1b), + LL(0x8305624e,0x290ced58),LL(0x06650920,0x398956a8),LL(0xdb5bd5b6,0xd057a47b),LL(0xbe9e119c,0xf2d85299), LL(0x7c5fc039,0x4783095c),LL(0x05363915,0x72f7e7cd),LL(0xdf3e2968,0xe46b90d1),LL(0xaaea2e53,0xaadb3dae), + LL(0x0cc4f426,0xf5d37496),LL(0x59d78369,0xa59bffa8),LL(0xf0a46b04,0x7ad4cc11),LL(0xb8e21b9e,0xcbd63351), LL(0x5653ebbf,0x60d255e6),LL(0x4d6b5843,0x3eaa59af),LL(0x9e1df2e2,0x90049d25),LL(0xe56aa105,0x9a185a6d), + LL(0x80e3d909,0xbd31c5cf),LL(0xa1f034d1,0x30caad3b),LL(0xd9c7c342,0xaca74fa1),LL(0x9565cf8a,0xac722cfc), LL(0x5b42e582,0x8b172ce6),LL(0x9b0607b2,0x9e99e4e5),LL(0x9446ca45,0x284eb579),LL(0xc57c9feb,0x6c5464ba), + LL(0xe511bc3b,0x1437fc95),LL(0x834d0889,0x22d7bc16),LL(0xc5071c43,0x62e545b2),LL(0x8cb4acd6,0x4c644d48), LL(0x68246492,0xd9efbe50),LL(0xcbd8ad0e,0xc9d169e7),LL(0x798ae01f,0xcb7365dc),LL(0x6d0dea3a,0x5783f98f), + LL(0xec454423,0x9b4a7e38),LL(0x96ff4c8c,0x27405d08),LL(0x0c462f7c,0x9769f097),LL(0x7dc946aa,0xcbda5412), LL(0xe7dd5146,0xdacb510f),LL(0x30507b37,0x9c9a0d39),LL(0x05ded0ac,0xa605730b),LL(0x6c6c7b5b,0x7e683472), + LL(0x7c952984,0xb378d92c),LL(0x72ae34d6,0xec76370d),LL(0xacda665b,0x1fde0bde),LL(0xb931afc1,0xc8f648f4), LL(0xb960f6ce,0x2b55adb2),LL(0x7336a643,0x71b3bdd4),LL(0x73cc39e7,0xf66e77bf),LL(0x2fa3999a,0xf582c5e8), + LL(0xaf986d1d,0x30ecd0c7),LL(0x4557dd65,0xa2ae53ed),LL(0x7d618a1d,0x97ebccfb),LL(0x11eed889,0xcbf54149), LL(0xd8f2bdd4,0xdd0ff0e7),LL(0xfa769e74,0x6ac4a9fb),LL(0x93e5abab,0xdfdfc7e9),LL(0xdffc6fcc,0x0c7151c5), + LL(0x5cbae56c,0x6d75e962),LL(0x96dccb89,0x77fae152),LL(0x6cc0e535,0x275c4946),LL(0x81781318,0xc4a400a9), LL(0x77ba50e6,0x8b9f872c),LL(0xa138eeb4,0x971b6cb3),LL(0x53f552a7,0xa7e7d1f9),LL(0x8447c730,0x360512ce), + LL(0xc5454439,0xf0c73bbb),LL(0xa3a24b5c,0x7f1b9b18),LL(0x51fa7d6b,0xc5bb48dc),LL(0x8b05a553,0xd264d6ec), LL(0xe9371f83,0x123caaf2),LL(0xb149f564,0xdf5da393),LL(0x853b9bab,0x38e02eb6),LL(0x95bf6647,0xc6aab96e), + LL(0x3141219b,0x4890be89),LL(0x7883fe8e,0x7afe4c2f),LL(0x59b86241,0xc27bd13c),LL(0xaacebdc9,0x1b9720f5), LL(0xf6b2174c,0xa054e203),LL(0x60f6de8e,0xd4e7b952),LL(0xf4558633,0xcf7b1aea),LL(0xbefa40a6,0x43fc1881), + LL(0xe23cef63,0x592164dd),LL(0xf7b4aaf2,0xfe57d6e8),LL(0xe8aef9bc,0x38a5e2c9),LL(0x1ac2b10b,0x576bd78c), LL(0x14309d10,0x2357944c),LL(0xed0ed94a,0x9933d7ed),LL(0x0339f299,0xb8792ea3),LL(0x87fd9bd1,0xcfb44322), + LL(0x92966739,0x864f2fd5),LL(0xd3cfd83e,0x7435ecc5),LL(0xec4249f2,0x8516d277),LL(0xfc158b34,0xaa7e1a8a), LL(0xfbe640a1,0xfc0fc22b),LL(0x91121fec,0xf287767f),LL(0x3f590dcb,0x0ce48273),LL(0xf087c249,0x5e994e2f), + LL(0x65604726,0x681a38c7),LL(0x247a421e,0x4f8c6ae3),LL(0x1294956e,0x1a51eaa0),LL(0x47c9b324,0x0984b1ef), LL(0x597b7696,0x3749bd0d),LL(0x08e57ee7,0x9d432b78),LL(0x2ba112d2,0x3092afe1),LL(0x16c5a7f5,0x89ccee49), +}, +/* digit=3 base_pwr=2^21 */ +{ + LL(0x54089685,0x355e9d7b),LL(0x40818349,0x9f0ec68f),LL(0x3861b80f,0x4cf4d8cd),LL(0xc1f5fa14,0xcce669fd), LL(0x1788f9da,0xea212509),LL(0xf3ccf239,0x32953613),LL(0x50027f3b,0x1048d092),LL(0x4270fbcb,0xe807b39d), + LL(0x95e388c3,0x5099dc55),LL(0xea44e3ea,0xd0670ff5),LL(0x61b41f7b,0xd212c993),LL(0xfaf13305,0x4f594af9), LL(0x05c01232,0xbc508bf2),LL(0x39ff08a5,0x76833536),LL(0xb837741a,0xa1cf70bd),LL(0xaaf7bd2a,0xba8e6616), + LL(0xdef27938,0xde04c343),LL(0x48cee32a,0x3f15ca91),LL(0x9dd142da,0xcb61573b),LL(0x126dd9bc,0xc094eefd), LL(0x136bb4da,0x5d42f1a5),LL(0xdb2f3449,0x75693952),LL(0x5c16795e,0x98017cd6),LL(0x2afb67db,0x9e401530), + LL(0x9b7c6c75,0x6376749f),LL(0xacbca35d,0x680eacdc),LL(0x5e145b32,0xe87fd5b5),LL(0x36b886af,0xeb20d1ba), LL(0x779b12bb,0xca499055),LL(0x0be39fb7,0x6f290ff2),LL(0xf4a128ce,0x33ad6fe0),LL(0x9b31da81,0xf09e2a40), + LL(0x12039372,0xb2ed3d70),LL(0x2ff46c13,0xb87e02c4),LL(0xfb27dce2,0x164246c6),LL(0xe6d95811,0xe34ee8f6), LL(0x3ec1fde9,0x66cc601c),LL(0x80ffdd56,0x056b3194),LL(0x9626aa21,0xff009868),LL(0x2d931092,0xc3e4982c), + LL(0xc3d42729,0xbc0da9c1),LL(0x720df0a0,0x4905da24),LL(0x45f6eadf,0x0e5e1fa0),LL(0x2aab7523,0xc02033f3), LL(0xedde75e1,0x45ba916f),LL(0x75c68e52,0xf43919bd),LL(0x84892e6a,0x00e7c076),LL(0x70dfeb08,0x259f8488), + LL(0xd8a869a0,0x3bfd5f2c),LL(0x574e7d67,0x1df48669),LL(0xe14cfd3b,0x16d6ed5a),LL(0xfcf78465,0x583aac2c), LL(0x67da2ae9,0x67210e6b),LL(0xcfee511d,0x0b024e70),LL(0x13839a4f,0xf27e122c),LL(0xb79dfa97,0xfa5356c9), + LL(0xf357999b,0xf0c24783),LL(0x26bfacb3,0x2c21474c),LL(0xd3ddb945,0xe3abed6a),LL(0x6031a5ea,0xbb21b764), LL(0x8afc2a09,0x6db3b68b),LL(0x81306b71,0x1aac2f08),LL(0x852eb6f5,0x882c3371),LL(0xd98e9b6f,0xadfe0c1a), + LL(0x7edcb9e5,0x0247ee7b),LL(0x1f29918b,0xe29ec013),LL(0x6099b6ce,0x5d1629e6),LL(0xcb534584,0x68587803), LL(0x8ce551d3,0x6ccfeddb),LL(0xf85123a8,0x7ef98b72),LL(0xf9711dcd,0x19af4771),LL(0xfd80e4dd,0x8f67858b), + LL(0x7d607ee3,0xa4c8c016),LL(0x4015a479,0x15db36d7),LL(0x9d28ea30,0x0cb58eee),LL(0xbecb7b4e,0xb3d469b0), LL(0x6f476e2c,0x811081b9),LL(0x59c78fab,0x264da3aa),LL(0x3cd73147,0xd6e5813d),LL(0xe905362c,0xce9e34a4), + LL(0xcb3afa55,0xe551ec2e),LL(0x4b05589c,0x2c9bef25),LL(0xbcd083bc,0xd36ddeb7),LL(0xddb54a24,0x1c180b52), LL(0xc0961f32,0xb84220f3),LL(0xfe3ae670,0xa71103fb),LL(0x46902477,0x6a14d319),LL(0x778b8eee,0x516701d2), + LL(0x4c3166d5,0x1cdb1025),LL(0x3d6fcb6e,0x3a0ba2c2),LL(0xb3820def,0xa218b4af),LL(0xbfe8a8f8,0xda6de958), LL(0x4ceabdfa,0xc2b3c755),LL(0x8d73edcb,0xd3534691),LL(0x0ce17182,0x453b8e63),LL(0x01654263,0x6507a5b0), + LL(0xd5da0e59,0xb2b8e424),LL(0x61ac4c2e,0x7e599c75),LL(0x41aff49a,0xc64cb4c3),LL(0xea3e378e,0x0e231e63), LL(0xe08edace,0x707cc0e3),LL(0x5410779f,0x18918dd2),LL(0x2eef6bb3,0xcdd57690),LL(0xff758569,0x4c54d7d8), + LL(0x2c89683c,0x49459204),LL(0x7827e518,0x93596a16),LL(0x2b20c939,0x6198954b),LL(0x8044d3ba,0x6672c94d), LL(0x199b16dd,0x55e95fd3),LL(0x4185999a,0xa8484135),LL(0xfe36e449,0x5e8709c8),LL(0x91401957,0x47470e2e), + LL(0x0874afce,0x0058bb09),LL(0x606c3e52,0x19fb1d56),LL(0x710903a0,0xe1208b2a),LL(0xd47dfd1c,0xecabc372), LL(0x5e94818f,0xd9daa7f4),LL(0x5dc99882,0x1302ac8f),LL(0xc44d37be,0x7b4c6b15),LL(0x72d19e0d,0x0bcf6d4c), + LL(0x3fd5a1de,0x1e0bf063),LL(0xa75b5b8c,0x5d05e901),LL(0xcb3c617a,0xbbbdb1ab),LL(0x1aef4706,0x44954a8c), LL(0xff6a6e47,0xbc3ceea3),LL(0x0ded1275,0x6140f421),LL(0x4dabe95f,0xbb4b4c04),LL(0x7135e813,0xc55e87da), + LL(0xd963dd6b,0x15ad105c),LL(0x666941a3,0x33d18f73),LL(0x5d9253d6,0x860ccabe),LL(0xd16e8b69,0x2af702fd), LL(0x74e525c0,0x7e46aadd),LL(0xaf59f48f,0xd9958a44),LL(0x8e7de482,0xd8ca872f),LL(0xcf7d007d,0xc2270c14), + LL(0xa200e574,0x87c6204e),LL(0x7b69e79e,0x0ee014cb),LL(0x82b23226,0x176ff378),LL(0x8dbbb2f3,0x802d829d), LL(0xe0a4dc31,0xb902924f),LL(0x5fe522f2,0x1f1a9ec7),LL(0x4da7c04a,0xbcd95d85),LL(0xb1543c0c,0x3a3a2e63), + LL(0xf3271bf8,0x9e70a3ff),LL(0xd2cd68ad,0xd2522d88),LL(0xa6b727b9,0xb777851b),LL(0x63ff5264,0x58953d6f), LL(0xb65c70d2,0x5e111c22),LL(0xd3a5143f,0xaae73c5b),LL(0x85ef5dc0,0x2daa2bfc),LL(0xea13ded3,0x5e7258d2), + LL(0x2e3ce423,0x4161127c),LL(0x6b1af415,0x7e35a0a2),LL(0xeed24b7b,0x004483a8),LL(0x9f9d44f1,0x2816180a), LL(0x062829a1,0x214add93),LL(0x225e847c,0x262a0bef),LL(0x5d6c53c4,0x4bb1b1ce),LL(0x91d06e53,0xd02f829a), + LL(0x784da27c,0xcdc8ba5c),LL(0x161b5836,0x78a6c0d2),LL(0x8373c6a4,0x6bea92c4),LL(0xa881f59a,0x815f1a30), LL(0x227cb8e2,0x699c8642),LL(0x25a2b9d0,0x515d1e2b),LL(0x1787b3e5,0xcb5f1c6c),LL(0x104dddc6,0xc9a10260), + LL(0x0f3811e5,0x18be4f2a),LL(0x71e727d3,0x8c05d3fc),LL(0xfa707140,0xecae3e5f),LL(0xd275b463,0x4bb05b16), LL(0xb02a5ac8,0x74bad373),LL(0x520344ee,0x7232875a),LL(0x65059d8f,0x32cef98c),LL(0x54e1b11d,0x68e0fdb6), + LL(0x3f3db43f,0x683424f3),LL(0xabf4a83f,0xf5f0878f),LL(0x4ac2c5c9,0x681350d9),LL(0x47dd3652,0x825e9ecb), LL(0x20713db6,0x420743f0),LL(0xd1b082e5,0x95db7427),LL(0x1affa57d,0xa0e1117f),LL(0xf940f325,0x62c87b5e), + LL(0x4e1d5d9a,0x6a65fda8),LL(0x345ccdef,0x0c0fe385),LL(0xd6d72c0a,0x19ff360f),LL(0xfb016131,0x1be1e8d7), LL(0x025b45e1,0xe2f27e91),LL(0x05259bf1,0x25bec266),LL(0xe51cc67e,0xd7b8b4e7),LL(0xab80a20e,0x3a839aa5), + LL(0x9f85320d,0x04a9b695),LL(0x98d669f3,0xb939cd83),LL(0xe6948957,0x24464ced),LL(0xa43928e8,0x463de507), LL(0xf8755845,0x4e1844e7),LL(0x5447e61c,0xc9c71091),LL(0x599d4bd7,0x1798f394),LL(0x1e072c64,0x758f7630), + LL(0x739b1925,0x83c93728),LL(0xfa8eb048,0x692017d7),LL(0x478d1ee3,0x4a3a2a59),LL(0x022640cd,0xb8e62912), LL(0x8572b8d7,0x4689a4dd),LL(0x8f79da63,0x6281ddfe),LL(0x212a153c,0x788bf9aa),LL(0xb3438da6,0xb67e18f5), + LL(0x31cebdb8,0x3fbafc51),LL(0xb042bd47,0x7f8ad590),LL(0xe3055004,0xf5d26c88),LL(0x3d7d6f5c,0x7f23a149), LL(0x8758ccc0,0x2fee5428),LL(0xe1b80dfa,0xb08c91b7),LL(0xea0c0a53,0xf2bcc903),LL(0x04e684ff,0xcdf2eae0), + LL(0xe1d9a693,0x354b2c07),LL(0x97a833a8,0x93b1fa2d),LL(0xe9e5f2b1,0x2dcd22c7),LL(0x18aa3163,0xf040a69c), LL(0x76939874,0x4f9a4b29),LL(0x15e24d44,0x58e5947f),LL(0xb0c2ef6f,0x9b47a945),LL(0xf630e92c,0xc4a15b7d), + LL(0x7b1d4141,0x8d7a33e7),LL(0x966486bc,0x44dabde9),LL(0xef31dc9d,0x387a6016),LL(0x1462ff61,0x76744b23), LL(0x20cdd726,0x2ad63954),LL(0x0e7803da,0x9cff7e86),LL(0xfd124ed3,0xaf5b8b4a),LL(0x050c1149,0x466dbbbd), + LL(0x06b296a3,0x68352636),LL(0x7f3fe1ef,0x0ab40080),LL(0x05bf08f8,0x1fc38951),LL(0x633c457f,0x69b54ae4), LL(0x1a206c53,0x2ad428c6),LL(0x8b09b3f9,0xd6725687),LL(0x0bc619c9,0x552d4d0e),LL(0x3113c689,0x0e88b313), + LL(0xe87a91b4,0xb2483b80),LL(0x0c75377b,0xb9f842d7),LL(0x5a78145e,0x50463f38),LL(0x830817a9,0xf2d3810d), LL(0x39cc886a,0x1819261e),LL(0x8415699b,0x697de51d),LL(0x5cab106e,0x688a874e),LL(0xcb8692ec,0xde48f3bb), + LL(0x38f4194d,0xffa1de97),LL(0x3b996b63,0x33d2726a),LL(0x0d2053a7,0x787c0ec3),LL(0xeecd5c0c,0x9447e9cb), LL(0x284773c0,0x077f121c),LL(0x815829a1,0x496427e4),LL(0x94def08b,0x4b119786),LL(0x9c15a778,0x9e7b29e6), + LL(0xfd4a8a47,0xa4d6d2be),LL(0x4333baef,0x4f000a12),LL(0x642c570b,0xc9049d86),LL(0x25e6aa6a,0x9424e8f9), LL(0xe011cfec,0x84de7fe9),LL(0x1e8c83b0,0xf273f956),LL(0xa47a40a6,0x98960835),LL(0x0a13c27b,0xd91a20f1), + LL(0xed703e13,0xaf08b4ef),LL(0xc9994946,0xefcbcf34),LL(0x2d53b069,0x019e6f38),LL(0x9b160894,0x3d62c3c0), LL(0xadfc8f3b,0xac7ad700),LL(0x0042fce6,0x41cc0cc3),LL(0x21cf742c,0x0228ae75),LL(0xf4c9a1a9,0x56a1152a), + LL(0xfebd27dc,0x5d8a3321),LL(0x7c525f7f,0x89bce700),LL(0x1c1039ee,0xe8f815a9),LL(0x62e86536,0x9f6db698), LL(0x66fe804d,0x1ea6e7a6),LL(0x261aea16,0x652acc41),LL(0xf9df596b,0xde28e5d8),LL(0x1553a545,0x18f453c1), + LL(0x84eeb5c8,0xa224f763),LL(0x835ba87e,0x8ac452f5),LL(0xc5f4c054,0x9b2b5939),LL(0x3ac1cdcc,0xb2577943), LL(0x772c60dc,0x1ba2cd0d),LL(0xd7a9bd1c,0x1fa52c43),LL(0x60444f34,0x2efd4f4a),LL(0x2bdcfc9d,0x7d188c05), + LL(0xe1913711,0x49ef6825),LL(0x600d6c46,0xbca95ded),LL(0xaf8d66d3,0x63916baa),LL(0x2dc837a8,0x04981202), LL(0x0d3ae79d,0xb501e517),LL(0xb4edb859,0x99ff7864),LL(0xaf4ec081,0x5099edee),LL(0x964f4052,0x89574889), + LL(0x52066d70,0x1690fdb8),LL(0x671f4e7f,0xb403207d),LL(0xd7413111,0x8ebc1d1b),LL(0xb4cfdf14,0x1432d7fe), LL(0x65ad5d0e,0x9277666a),LL(0xa928e194,0xbd5ae578),LL(0xb64962fb,0x2f6c10d5),LL(0x2e794187,0xe3d756c0), + LL(0xd3e6349c,0xf04fd82a),LL(0xcc7d39b6,0xde602dba),LL(0x044e7deb,0x0886e20a),LL(0xe9ba917e,0x6e30c75f), LL(0x4a322ede,0x763961fc),LL(0x2324bb92,0x6df4a3cb),LL(0x8f2ac967,0x9fe82323),LL(0x2345372a,0x3c372afe), + LL(0x50b66fec,0xbf7e9c55),LL(0x0c065cfa,0x5db7dd71),LL(0x50d459ea,0x3525e310),LL(0x8122941a,0xad7abe5a), LL(0x122d92fa,0xc7aeba80),LL(0xefcc1c24,0x066c3765),LL(0x8ffd71b1,0xa6d767ca),LL(0x9cc16dbc,0x4a75fab5), + LL(0xbb58b63d,0x9acf8b89),LL(0x6fc8c090,0x226cdcd3),LL(0xae7fbd0b,0x852965b7),LL(0xb8bfe65f,0x4cadd176), LL(0xcfa2ac11,0x4ccc11d1),LL(0x800319ab,0x8abf7420),LL(0x88bb3ef1,0x24ab82cb),LL(0x524c0ce1,0x4d3db003), + LL(0x3a431e8c,0x38483641),LL(0x792848ad,0xfc0c04a0),LL(0xa07701b0,0x2fc52bb8),LL(0xf29c72cb,0xdfdced3d), LL(0x5280c2e0,0x677e3d84),LL(0xe98cbec5,0x2dda1451),LL(0xaec26be2,0xba28b181),LL(0x5ddea39b,0x16694717), + LL(0x4b9aa9b0,0x911ec5f0),LL(0x3594ae7d,0x24b9aaa0),LL(0xc3c136a0,0x0ccfa661),LL(0xb7474319,0x5518964d), LL(0x2175c3dc,0xf0b0427b),LL(0x966b7bad,0x08db4cfc),LL(0x5e888ad1,0x6f61428a),LL(0x57b52d37,0xfaa96176), + LL(0x10aac879,0xe834013b),LL(0x95a62172,0x73397bb0),LL(0x33a244b2,0x97806839),LL(0xc3bec0d0,0x0ab3806c), LL(0x2a72512d,0x4fc7a859),LL(0x0a4228b9,0x96474939),LL(0x4de4b4a5,0x8e5d79a8),LL(0x05d62667,0x5a60d1b0), + LL(0x08d90c20,0xd31be21d),LL(0xcc14dbb1,0x3f7ed5f2),LL(0xd7d431c4,0xdc8f58f9),LL(0x82b5c63f,0x714f6dee), LL(0x76d2de93,0x6b285466),LL(0xc39dd98c,0x3c2f5d8f),LL(0xea3760a2,0x9bba0075),LL(0x2411742e,0x75e0389a), + LL(0x7ffdb955,0x87d6715a),LL(0x9efb199d,0x702108fc),LL(0x6c010f8a,0xf11db1f9),LL(0x7eb6871b,0xf52b1e0f), LL(0x97c3ed9e,0xc49c0dc7),LL(0x77220a50,0x18846f95),LL(0x97afddcb,0xdb2273bc),LL(0xcc469f75,0x5b9a16d6), + LL(0x3beedaf4,0xee364394),LL(0x528a9239,0x825e01d6),LL(0xffd0f17c,0xb60ba965),LL(0xb888384b,0xc00106b0), LL(0x31751f74,0x6e24322f),LL(0x1821d05a,0xfe4d074c),LL(0xbf072932,0xf2493c73),LL(0x21089f21,0xa797e208), + LL(0x2988abcd,0xf1b318af),LL(0x8e7da518,0xf887558f),LL(0x97836b57,0xb8b9939c),LL(0xc0a74cf3,0xf793e3b5), LL(0x37684170,0xe191008a),LL(0x05cb453c,0x7708823b),LL(0x361beb2c,0xec221d40),LL(0xeb1b68f4,0x0e5a6cce), + LL(0x9644e937,0x3dc535f0),LL(0xfda6c1b7,0xf506d720),LL(0xf99437bd,0xc78c0e0b),LL(0xcc9e2b09,0xa920b4d3), LL(0xf089b0e0,0x550965fe),LL(0x0109d910,0xf9813492),LL(0x8c9d5d83,0xd2496f20),LL(0x3e3e661f,0x751b6900), + LL(0x9e6ac190,0x921edbde),LL(0xf02d0e7a,0x75891359),LL(0x1c4da092,0xdeb0f83b),LL(0x4feb2375,0x7b427915), LL(0x7c3a85c3,0x24637c72),LL(0x3f214ac3,0xbbfabf86),LL(0xae22fbfa,0xe8765740),LL(0x5f14045a,0x3a09fab0), + LL(0x8190dd41,0x546d574f),LL(0x48b5a39f,0xdfcf0b73),LL(0x74097b2d,0xf26c69de),LL(0x3a7e3e90,0x37aa27ff), LL(0x83bbe3df,0x0942447b),LL(0x9ab378aa,0xe779fe20),LL(0x91e2264f,0xad18ad23),LL(0xaaabd6d1,0xe1dad926), + LL(0x5db5e5c8,0x9de0aa4f),LL(0xdb67e44e,0x45c3d73e),LL(0x5cd83936,0x440862a1),LL(0xffce9a79,0x9f2b9a88), LL(0x76cc6265,0x63299069),LL(0x7a8830f5,0xf596a67f),LL(0x8d1d8284,0x7051c842),LL(0x3e5561fc,0xa00d05a8), + LL(0x7a34d5bc,0x15ce42d5),LL(0xb0e37254,0x4d9b3f5f),LL(0x38841ab4,0x26e84094),LL(0x9a8ede27,0xa7afd35d), LL(0x14835fa9,0x4e8bcdb8),LL(0x79493e39,0x85d04ddc),LL(0xdf8f65ae,0xbfa8fa79),LL(0xda6c7c62,0xe31d759a), + LL(0x7600aea7,0x76f27e70),LL(0xd4d9acf5,0xbec79f15),LL(0x5eae2ff6,0x0f10bd0f),LL(0x7116a0c4,0x96c9eef1), LL(0x0cb6f595,0x30add2cc),LL(0x943efe90,0x0c70b548),LL(0x0a05f4a8,0x2ce8026f),LL(0xb7c53c00,0xaa3da153), + LL(0x8e385a26,0xcc46bf67),LL(0x99bae0f6,0x64bcf06e),LL(0x035dcb4c,0x49480a36),LL(0xe3cbae58,0x2cc1a299), LL(0xb5480cb2,0x849f8633),LL(0x5607d83e,0x1d8fa56d),LL(0xcea9f22b,0xcc3f0eee),LL(0x1a23f3da,0x7d5ece29), + LL(0x8ae66f00,0xc6f0a006),LL(0x78d648f0,0x2620157e),LL(0x40d2880f,0xfc717762),LL(0xbe105017,0x2e0e293c), LL(0x854116f4,0xb320f214),LL(0x2d5cd4ec,0x5e4fa700),LL(0xdffc1c55,0x83fa0a23),LL(0xc9a9ca15,0x18fcb8d2), + LL(0xd0ac70fe,0x9e9baccd),LL(0x27fe06ce,0x8ba02fb7),LL(0x3868fdd4,0x2708804c),LL(0x9ba83df4,0x355eaf0c), LL(0xbe43993a,0x014089ba),LL(0x469cccd6,0xc8b59eda),LL(0xb893a5a7,0x77c94507),LL(0x8e517fd5,0x0dffd39b), + LL(0x13dbeadf,0x71b6edb7),LL(0xfea2d0cb,0x1617b77f),LL(0x48ff989f,0xf7454736),LL(0xb618bfa3,0x27357890), LL(0xa7181331,0xf08c70ac),LL(0xb8bc036d,0x33b6cfe5),LL(0x7163f883,0x75ed10f9),LL(0x47d1cbbd,0x979875fc), + LL(0x7ad23938,0x6644b234),LL(0xd82e2bc8,0x0f09e7f1),LL(0x5588a39d,0x1e6c512b),LL(0xce8eae85,0xb44e6694), LL(0xf392a4c7,0x107336e2),LL(0xdbcd7b43,0x2619b284),LL(0xb7f476a5,0x7b7ec516),LL(0x43081af2,0x0de74ef3), + LL(0xd8d54faf,0x93d08bc6),LL(0xf2ae6c9f,0x88d343a7),LL(0xbc147c27,0x7cdb9003),LL(0x69248562,0xd740b19d), LL(0x464b3b60,0x7f3c48bb),LL(0xc91d92c1,0xfc4cd7e9),LL(0xd7420ac9,0x8172af80),LL(0xb9a50be9,0x66907b77), + LL(0x9ec8e974,0xed99fea1),LL(0x54f39b1c,0x624a8c94),LL(0xce9798d1,0x9c4d608a),LL(0xa4812277,0x81e1652e), LL(0xf58b7db8,0xa2cf7509),LL(0x745e450e,0xef2cd193),LL(0x9d9da493,0x48ee8431),LL(0xb8ce96fd,0x7b471698), + LL(0xe7553998,0x14dbaff8),LL(0x822de823,0xb0b14e4a),LL(0x429d7c51,0x11032354),LL(0xd572d20e,0xc1bb3327), LL(0x6a9c189e,0xff473811),LL(0x9c7b3b83,0x7cf2354e),LL(0x7662df92,0x29681ff6),LL(0x51c297d1,0x09296227), + LL(0x5e3da635,0x1b800b34),LL(0x745116e4,0xb5fd32d2),LL(0x2565abb0,0xdae17a1f),LL(0x1fec80c2,0x4f39d3d7), LL(0x290c2f4b,0xb4a19cc2),LL(0x0b6e5ae0,0x1a1b049e),LL(0x6a823b6b,0x41be6e92),LL(0x969649ce,0x35648873), + LL(0x2a8ed3d7,0xe85f995e),LL(0x2f319e47,0x9dc712e8),LL(0x536d98a2,0xc4402eff),LL(0x37521e35,0xca61e310), LL(0xc3196672,0xfed39621),LL(0xff17e8a7,0x29e7743f),LL(0x412a7c49,0x47eca488),LL(0x33a2a6da,0xf0114513), +}, +/* digit=4 base_pwr=2^28 */ +{ + LL(0xe6880b5f,0x5675a12a),LL(0xe2606d25,0x9ba1e92c),LL(0xeb3b2125,0xb012facb),LL(0xc37b0099,0x3c50fdfb), LL(0x9ce223e9,0xc9ce461c),LL(0xeefbd8ac,0xcb90bdd6),LL(0xc631ea8e,0xf657e5a4),LL(0x38a83ff6,0x6584520b), + LL(0x635abcf0,0xd959f317),LL(0x99e17618,0xa516a43f),LL(0xce3bd99b,0xed90ccf2),LL(0xa9fb3290,0x2fc6d460), LL(0x0cde4302,0xb61ebe09),LL(0xf908003b,0x5a3b061f),LL(0xf60f5787,0xf51bb736),LL(0x057efc2f,0x1717f6e9), + LL(0x1ca260ef,0x565acf93),LL(0xf1811d23,0x7d6e797d),LL(0x783e42c8,0xe63c6920),LL(0x8dcb5158,0xdc9dbce8), LL(0xc8e39022,0x1426dc7a),LL(0x30ebfe47,0xf3037f34),LL(0xf87d6395,0x75aa6845),LL(0x61f53539,0xbf792fd5), + LL(0x6ddc3d83,0xa8bf2172),LL(0xd88207bb,0xf68deb6e),LL(0xcd03bd7e,0xa8eae2eb),LL(0x951f59a4,0x64c7f57e), LL(0xa1786d57,0x8badb223),LL(0x71182790,0x2e7fda60),LL(0x9a5a9457,0x9dc90e36),LL(0xf4b07e07,0x6eca838b), + LL(0x03264871,0xad2e235b),LL(0xb8b933de,0xb4c56243),LL(0x91354c8e,0xd9c2bdda),LL(0x6a73fc76,0x97d743ff), LL(0xce88013e,0xbed4109d),LL(0xf3b3bf4f,0xa2428275),LL(0x011e761c,0x900d3560),LL(0xe24fd6c2,0x34925d7d), + LL(0xa8198235,0x08b966ca),LL(0xed2d764a,0x355d098c),LL(0xa3d63f3a,0xfac27f7c),LL(0xd3edc140,0x3e553f6c), LL(0x11ff4334,0x64d72c7f),LL(0xbc62cb57,0x48735aab),LL(0xeba21082,0xcf064294),LL(0xbb8d96fd,0xc1f9e456), + LL(0x293cd945,0x1d24bdbc),LL(0xea254e36,0x76985bcb),LL(0x876fb485,0x3df2cb6a),LL(0xcd1f673d,0x0176969f), LL(0x642133a7,0x8b41cacb),LL(0x373880e2,0x31ea88f8),LL(0xb3b1463f,0xccf1ff85),LL(0xaca74a27,0x88fffa15), + LL(0x167cdd1f,0x9a4b9b92),LL(0xf879b894,0xa9118fc0),LL(0xc55479f5,0xf6e73387),LL(0xc626d292,0xfadf82ed), LL(0x56e80e6a,0xa03bb761),LL(0xf27555d1,0x59a783f9),LL(0x3d087e43,0x027d63b6),LL(0x02fdeded,0x29f9ff32), + LL(0x371d0ec5,0x88a9173d),LL(0x08c0227a,0x04ac4d0d),LL(0x9c7ec715,0x00213011),LL(0xd9d6b472,0x0d2b7c76), LL(0x5050bdff,0xe678d53a),LL(0x65a5fcd5,0x8f929d57),LL(0x1dc3e712,0x0793920b),LL(0x4b073699,0x9a6a690f), + LL(0x758bdc9d,0x329d9a81),LL(0x7d867b66,0xebbaadd9),LL(0xe6025f68,0x0d7e6b19),LL(0xc53dce26,0x50184374), LL(0x3ed13916,0x298cb00f),LL(0xf5d45b26,0x835fe31e),LL(0xf5a7fb7a,0x373a9c49),LL(0x34d3d8a8,0x59ed7e23), + LL(0x3baf6fa3,0x1a8dfe33),LL(0xda53714f,0x926ccce7),LL(0x18ef6fe2,0xda4feaed),LL(0xc3ca5cdd,0xeddaf090), LL(0xbfe06d45,0xc39c2046),LL(0x0d7f549f,0x1d9e889e),LL(0x8d537d0a,0x209ace73),LL(0x0e31e1ce,0x6f182c88), + LL(0x1b8c82e2,0x865e0761),LL(0xa659f2ab,0xcf11bcb9),LL(0x7c868143,0x1804bbeb),LL(0x453e36eb,0x2fa89a0e), LL(0x2e17bad1,0x42d69d8f),LL(0xdc2ec741,0xe7fcea6f),LL(0x379ceb37,0xe7f19b45),LL(0x49bb35a0,0x84f0bd89), + LL(0x5264b33d,0xa8a50678),LL(0xab1c9e26,0x8cfae763),LL(0xff9b931a,0x1e837dc3),LL(0x796ac029,0x76164be8), LL(0x1266db27,0x26a8bb2b),LL(0x54822255,0xfba4ab83),LL(0x38524458,0x7a5adcfd),LL(0x44ee212c,0xa056c882), + LL(0x55018577,0xe8db6fee),LL(0x91955960,0xf71256b6),LL(0x10abe8d8,0xeb1c118e),LL(0xd45a8426,0x984efc9f), LL(0x00f2c6ed,0x4e1b323a),LL(0x331baae2,0x1759a7af),LL(0x2e00ba6c,0xf1587189),LL(0xbb385d39,0xbd8a877e), + LL(0x57d6c1ae,0x440d1eae),LL(0xa957dc67,0x092abdef),LL(0x74554b3f,0x1065cbc6),LL(0x710566c7,0x67062382), LL(0x6d04ae2b,0xd327679d),LL(0xb0340551,0x11507b00),LL(0xa2f52d80,0x2e571583),LL(0xe8578507,0x673628f4), + LL(0x0cf4efe5,0xecb8f92d),LL(0x960e2d22,0x88c47214),LL(0x6059f079,0xca9549ef),LL(0x7016da7c,0xd0a3774a), LL(0x1d001cab,0xd51c95f6),LL(0xa3feeec1,0x2d744def),LL(0x0afedf2b,0xb7c20cc2),LL(0x71d144a5,0xbf16c5f1), + LL(0x3dc0d12e,0x00384727),LL(0xb01cc80f,0xaa95f450),LL(0xa6f8e927,0x19be3106),LL(0x0417ba8b,0x6d6e10aa), LL(0x870e3491,0x149f120c),LL(0x026dde94,0x27380b41),LL(0xf29b04e6,0x97d00840),LL(0x4bf9eb19,0x21d5d7e3), + LL(0xd5327f05,0xea1daad9),LL(0x9c88c17c,0xf1f45d94),LL(0x3f8ee0ab,0xc5f3dee2),LL(0x75238a56,0x706b777c), LL(0xf834c60b,0xf7aee379),LL(0x13cfe17b,0x5c24dae6),LL(0x8091804b,0x354c82e5),LL(0x102a577b,0x0dec2fdf), + LL(0x5253f8fc,0xbf3b7030),LL(0xd913c01c,0xe516fa69),LL(0xa105ba64,0x053afef4),LL(0xc89c1e76,0x91a1f36c), LL(0x7e724e18,0x3375865c),LL(0x29327b2b,0x43132144),LL(0x6f7bb24e,0x9cb2fc3b),LL(0x6319e789,0x20a6a16d), + LL(0x642c467a,0x20bfbd77),LL(0x259d50c8,0x3452bb12),LL(0xec7ffab2,0x0d3ba9c7),LL(0x3560e541,0xbbdb5454), LL(0xd63ba04b,0xab1d6e22),LL(0x7d24f015,0xdf6f11d3),LL(0xf3df15fa,0x7c4d61d2),LL(0x40b3288c,0xd5269f79), + LL(0x0e7c7b6c,0xf8516b9e),LL(0xc203dac8,0x48750d82),LL(0xa13d3083,0x89845d36),LL(0x280a131a,0xb3db3cfa), LL(0xfbf752e6,0x40045401),LL(0x1432e856,0x0289f97b),LL(0x5fc1aa11,0x41a9f371),LL(0x8d464042,0xe5c1e5a5), + LL(0x589b71a7,0xfbee2ea2),LL(0x5de7056c,0xdd6ee5bd),LL(0x8fd6b6de,0xcf8a4541),LL(0xb15e33b1,0xb47831dc), LL(0x2064321e,0x126a2169),LL(0x6e517edd,0xa21d2d22),LL(0x5ba5a30b,0x1f8072be),LL(0xc6a24b7d,0x24cca576), + LL(0x5c282027,0x57eab82f),LL(0x557344b3,0x1620f5e6),LL(0x460b3385,0x59e852e0),LL(0xf050816c,0xc906e3db), LL(0x3eb398e8,0xc031f8cf),LL(0x507ac07f,0x9c25b69b),LL(0x9cf7bdbe,0x652baf2b),LL(0x5ad91107,0x06fedc53), + LL(0x4c4b12c5,0xa8ca0be2),LL(0x28762d5d,0x633292b6),LL(0x827c0d5e,0xc04983f2),LL(0xc707ef03,0xcb6b867d), LL(0xb9ac1124,0xa7fc0d5b),LL(0xaab7dcaf,0xa5ce085b),LL(0x1cfda998,0xb85e8f1c),LL(0x27822503,0x8208df42), + LL(0xa8dd6d76,0xeaa82320),LL(0xad36eb73,0x7b2fb4ae),LL(0x97a7b040,0x24d73191),LL(0x4001e02f,0xc3ff64ae), LL(0x88799d94,0xd5d87157),LL(0x93ceb95a,0x559142d0),LL(0x59c3009a,0x798a453c),LL(0x7d6c83a2,0x546b6fab), + LL(0x5c76029a,0xe263b23a),LL(0x4ac62973,0x85630532),LL(0xecb007ac,0x14ee0643),LL(0x7ca60905,0xf9e06297), LL(0x92f1f170,0x21b2fb23),LL(0x46528ab2,0x31c40918),LL(0x3395cfd2,0x43b53242),LL(0x6d14fb40,0x4042138f), + LL(0x4464f342,0x80899c8c),LL(0x084be305,0x0f54c993),LL(0xfbf84810,0xfacecac3),LL(0x8ae5244f,0xa6585936), LL(0x9a9f8d4a,0xb467c3c0),LL(0xfd394895,0x3e5f219c),LL(0x9bf85fa8,0x39f0767a),LL(0xd8ee6022,0xd97cc55d), + LL(0xc83f86c4,0xc480938f),LL(0xe43bfcc6,0x6479b8ef),LL(0x38cabad7,0x8e6f2e22),LL(0x31f8c6aa,0x48e57fdd), LL(0xcfbbdcac,0x66dd6a77),LL(0x50ece329,0xc7d9950b),LL(0x0747a937,0x2e31f205),LL(0xa07acb8a,0xc0f8a7e2), + LL(0x15eaa686,0x578477bd),LL(0xf2f58b50,0xd72fb935),LL(0xd3a64d22,0xe9fdbc6f),LL(0x492dc89f,0xa3e42674), LL(0xa8fb7d24,0x42410ffd),LL(0x52676ed7,0x08a37dfd),LL(0xcb5d6125,0x4607c41b),LL(0x4001fa42,0x7db48af8), + LL(0x50cd30f0,0xe2264eb1),LL(0xe215f8d7,0xbb6fe952),LL(0x97e3fe73,0xf3ce2411),LL(0x37f19247,0xe52e2179), LL(0x20c233c1,0x9c7fc8c0),LL(0xb383b101,0x91c7e721),LL(0xa7ac883f,0x1163c472),LL(0x9d3b0f1e,0xbe1c3b3a), + LL(0xa3536baf,0x07be716f),LL(0x62e9c19a,0x764d9f4e),LL(0x8eaf19f4,0x15af3499),LL(0x38ea0ace,0x987a7c47), LL(0x4a1f0117,0xb03740b8),LL(0xfe098a9f,0x5cd1164f),LL(0xc9d6fee5,0xaf952cef),LL(0x3c0ad28b,0x4e86dcbb), + LL(0x677b7a8f,0x81125450),LL(0xe69273d2,0xba889fce),LL(0x582c5990,0x4a40a859),LL(0xf48934c3,0x836638b3), LL(0xf3596ba6,0xe964e189),LL(0xde8b0754,0x2f417c0e),LL(0xd5f93f1b,0xd883169f),LL(0xd45bb389,0x0318fe4e), + LL(0xdb03273e,0xe2c998a1),LL(0x33ec151b,0xc34f544d),LL(0xeb92d963,0xae0456b1),LL(0x9738857f,0xaab61ec4), LL(0xf71d9c39,0x4fb6a34e),LL(0xd816ec44,0xaa9dbd8c),LL(0x5efdf950,0xf6532e37),LL(0x67109c55,0x7151dc44), + LL(0x3f4e322c,0xb18b586a),LL(0x3553a18b,0x27b30066),LL(0x1ae4cd85,0xbd31ea24),LL(0xa64de69a,0xe8f88f4a), LL(0x609c13bb,0x8c946a97),LL(0x0eebd9f0,0xbf8cc55a),LL(0x7a8892b9,0x446aa2e4),LL(0x65b98c31,0x660c0a55), + LL(0xd3463522,0x568c56fc),LL(0xeb130aa5,0xfa6bf3a6),LL(0x008dc0da,0x16c1568b),LL(0xfed70a87,0x9c4132cc), LL(0xd497fdff,0x3e983d09),LL(0xf0ebe6b0,0xd7a0e542),LL(0x68b542ca,0x193a07e0),LL(0x07c6ab4f,0x4909776b), + LL(0x418acd7b,0x55b77ef4),LL(0x47a77d32,0x64ba62d3),LL(0x2d1f562e,0xaec1aa93),LL(0x10dc5999,0x3468725b), LL(0x6ff0d478,0x422851b1),LL(0x8e7dddcc,0x15da8429),LL(0xb8ac5238,0x38567920),LL(0x2e3344d8,0xfd29eb4a), + LL(0x4fc636b5,0x7b2af70c),LL(0x879e7640,0x242acfc8),LL(0xb5e25c7b,0x88e89786),LL(0x16ec1bfd,0x85576b1b), LL(0x1891e595,0xb31c8253),LL(0xca5608a5,0x14315dfe),LL(0xb0c14fd9,0xb9d61b76),LL(0x734b6cab,0x5d5ad8a3), + LL(0x44aee005,0xc2ea321d),LL(0x147ed658,0xd68abd2c),LL(0x893db877,0x31152d60),LL(0x281487b6,0x4807ac46), LL(0x65da04b5,0x58ebd15e),LL(0xb2f9d1fd,0xf0f74fd4),LL(0x393c7d91,0x3d04aa65),LL(0x8e7e6a2c,0xb46fb59a), + LL(0xae1eed5d,0x9236fdf1),LL(0x7810e2be,0x71936f56),LL(0x6d9ff147,0xa1ead7d5),LL(0x149a9b6d,0x32670ed8), LL(0xcb58ea59,0x12772fdd),LL(0x9df52ddb,0xfce260b3),LL(0xccab1e97,0x3221f2fb),LL(0x57762484,0xf8ff7e37), + LL(0x855512cf,0xb0a31a1c),LL(0xd71d4c4e,0x293a819e),LL(0xcd6a900f,0xc1ebc896),LL(0x9b9e0a4b,0xc727a646), LL(0x0018f29f,0x06124fc0),LL(0x41b7730c,0x67bd8fed),LL(0xc77be72e,0xeeebf0f0),LL(0x474d747a,0x427fe6fe), + LL(0x932ccbf0,0xa7fb9a4f),LL(0x5f3d489f,0xabb9c85e),LL(0xbdf26442,0xe7e4f956),LL(0x38d17422,0xd014848e), LL(0xd3e9bff6,0xae37d855),LL(0xca5aeb09,0x88fbae1d),LL(0xf025feaa,0x1a8a740b),LL(0xb9475ebb,0xc1a67821), + LL(0xde2bf8a2,0xb6cb6acc),LL(0x66a0f14e,0x9b2ab1ca),LL(0x83b2ba59,0xcbfbc068),LL(0x68447934,0x336ab62c), LL(0xf19719b8,0xd3a016a9),LL(0x0b5b9d6e,0x819a31bb),LL(0x3e1c6c0b,0x7b24be2b),LL(0x013f6821,0x10834b4a), + LL(0x86f21d2c,0xe5e5de27),LL(0xe9e35ad5,0x56b46a2d),LL(0xe2111e59,0xfc4e861d),LL(0x6e37ca63,0x7472ce5e), LL(0x27d2210e,0xafab9a71),LL(0x9ff6245a,0x1644a0a6),LL(0x8dbef51f,0xee498acb),LL(0x2e9604d3,0xd4c70da1), + LL(0x6fecb64c,0xde487364),LL(0xd15fb62f,0xa8fda1fd),LL(0x088de028,0x97e2febe),LL(0xecdce095,0x4a769019), LL(0x50a58ddb,0x4cb6a338),LL(0x17028d36,0x08df59d8),LL(0xb51722b7,0xfe3a80ff),LL(0x963c2383,0xa3cc2fe2), + LL(0x53cc5341,0x40b2df49),LL(0xa3c4bf2f,0xf3e90d4c),LL(0x20f02731,0x3f25c5ec),LL(0x69065d9a,0xd84f5b5a), LL(0x129921be,0x156d350e),LL(0x1b116922,0xe98787cc),LL(0x39e77b13,0xba5f9b82),LL(0x044449a5,0xee4d79f5), + LL(0xdd6d852d,0xb54b7388),LL(0xf9ca5fdf,0xf7554c5c),LL(0x51228a81,0x864d1fbf),LL(0x9a80f90b,0x721e1add), LL(0xad0efa62,0x89d4e297),LL(0x6dba9404,0x4e471a87),LL(0x1c1008b0,0x9a38158b),LL(0x95c47ec2,0x3dfe81a7), + LL(0x28603026,0xcb02ce9b),LL(0x3bd357fc,0xfd3207aa),LL(0xf296f5f2,0xb3807bdd),LL(0x23c2ea7e,0x7895918d), LL(0x88feb3ba,0xdc0eb62f),LL(0xbdd75674,0x024dfd84),LL(0x0a1e0496,0xe5bd3828),LL(0x24c8f30c,0xb8b1cd86), + LL(0x674d10cf,0xb559e34d),LL(0x9f962ec5,0x6955bb69),LL(0x542af42d,0x8bf1ab6c),LL(0xdfa61256,0x3f2f33fa), LL(0x73d1049e,0x32140195),LL(0xdfd7f39b,0xf5089278),LL(0xb4237be0,0xb42eb51c),LL(0x874d0e57,0xdf747f44), + LL(0x77b5d475,0xbe64bb22),LL(0x28308634,0x2c3d5ecb),LL(0xcb999c46,0x936a2987),LL(0xe26489ea,0x5a30ddfa), LL(0xc8eabf9c,0x8bfc782e),LL(0x74c8c6e3,0xb9995bb0),LL(0x391f5c5a,0x4f99c7ac),LL(0x5270c4ad,0x67f4092b), + LL(0xe6e8135e,0x6771a29d),LL(0x6c698cec,0x988dfb2b),LL(0x77812aa1,0x7818600f),LL(0xfd98e1c1,0x04393c83), LL(0x864ef146,0xe448232e),LL(0xa465ab71,0x9b70ecf4),LL(0xb13cc704,0x31df0531),LL(0x16e48426,0x401ae0b3), + LL(0x7fc514ed,0xa8106630),LL(0xde4b1614,0xda798170),LL(0xc2c684fe,0xde892efc),LL(0x05d64eff,0xd5205bc1), LL(0xe1d59ba5,0x84df4ead),LL(0x89bb2ea7,0x65245ca1),LL(0x64edbf51,0x3de6ca34),LL(0x56bcebf9,0x115296e4), + LL(0x7fd52a3d,0x0851631f),LL(0x949ad4be,0x9881db71),LL(0x88caf772,0x4b2337dd),LL(0x33ec7979,0x02da59de), LL(0xafe840df,0x2473c620),LL(0xa92ef1d0,0x2965ebff),LL(0x6fcd9651,0x2452854f),LL(0xbac2ed99,0x97092935), + LL(0x08242246,0xf0743ce7),LL(0x6d1a8439,0x76fdd82c),LL(0x61079258,0x3627c890),LL(0x82b21983,0x312f98f1), LL(0xe9173891,0xd87dceec),LL(0xd7a30e32,0xad16cfe0),LL(0xc9c7efaf,0xc404a1a6),LL(0xd6df357e,0x46e34671), + LL(0x7a02aa32,0x92fec7c4),LL(0x5a6a7bb9,0x567fef7e),LL(0xa3f97b5d,0x35fd570c),LL(0x4a4b0dfa,0x456bad8c), LL(0xa677f090,0x85a3f42c),LL(0x22a68d53,0x35060bb8),LL(0x53567530,0x1cea9d11),LL(0x8169fbce,0xf2cbc8dd), + LL(0x86cde794,0xa3e1d52d),LL(0xb3bdf344,0x72a258cb),LL(0x31b8614d,0x2997cd59),LL(0x8164b632,0x31ce2ea4), LL(0x0eba7545,0xe495e9b7),LL(0xbc4403b5,0xaad69130),LL(0x45760d9b,0x37f6389b),LL(0xb871b17d,0x00f4d58d), + LL(0x4aa359d7,0x91973d4d),LL(0xc8dd0582,0x249f510c),LL(0x7608be27,0xef11ac87),LL(0xd940b1c7,0xce116714), LL(0xfef20037,0xf34881f3),LL(0x98203f4e,0x26222472),LL(0xe9363154,0x4c9e98ed),LL(0x03a8158d,0xa806b3a6), + LL(0x09d16ce3,0xdd974d66),LL(0x59ae977a,0xe1bcc513),LL(0x218464d6,0x0e6201c7),LL(0x9e35c7af,0x05789811), LL(0x8b33a863,0xb1596f7a),LL(0x42bd8284,0x8fa93aeb),LL(0x46e11559,0xf197c202),LL(0xadd27d86,0x356b9c81), + LL(0x1695cb70,0x3c4080fd),LL(0x20f20318,0xc10c28cc),LL(0xce1ffab9,0xe9d7ed93),LL(0x4f9de9bd,0xb23976b3), LL(0x6d61a6f2,0x9b1b81dd),LL(0xf6318874,0x7537d729),LL(0x20cee7ab,0xb75022f4),LL(0xaa430952,0x425fddba), + LL(0x1ccfb3fd,0x54c4306d),LL(0x46a30a37,0xf10a54f1),LL(0x74fd4925,0x2d332a29),LL(0x1438feb2,0x8d2fa921), LL(0xfbb41bd2,0x46a9c6b5),LL(0xd30c65fd,0x87e98550),LL(0x66cd9a20,0xfbcb2ca6),LL(0x91719670,0xc176530e), + LL(0xcec38056,0xdd4a1a18),LL(0x75544998,0xe6ef1793),LL(0x30583fb6,0xf58f69cf),LL(0xaa76bf2b,0x12197860), LL(0x3bb686fc,0x717813e5),LL(0x0f06c403,0x9beeb1ae),LL(0x2782dc86,0xd83416ee),LL(0xb5500ccc,0x5fc89c01), + LL(0x8e110ed3,0x063aee25),LL(0x45963073,0x1a87377c),LL(0x5110634f,0x86944f59),LL(0x0ba76459,0x50659ae1), LL(0xde9eb40d,0xa00e48ff),LL(0xfe5b118c,0x49235afa),LL(0xc425ee38,0x81705008),LL(0x2d5f2f92,0x3c01abc8), + LL(0x6fdf148b,0x4a21bc95),LL(0x0b7e6871,0xea5cc30e),LL(0x11713844,0x90b4abb6),LL(0xf2001af9,0x3b7d734f), LL(0x782b2020,0xfc616b89),LL(0xd8b0e02d,0x68b3935c),LL(0x1cbb2de4,0x54cf5b8c),LL(0xa7c0f7ac,0x42b0432a), + LL(0xbdffae5e,0xa04e06ef),LL(0xd4b636ee,0x36cac28e),LL(0xc3a98127,0x08a06b2f),LL(0x290c5385,0x1ef0b57b), LL(0x27154c46,0x14e184b8),LL(0x60910b3e,0xa5dd3444),LL(0x3c67a74b,0xd0008ac4),LL(0xefed9fd1,0x2649cba4), + LL(0xf0c1bb4f,0x26bc537a),LL(0xd06b90f5,0x37f376ff),LL(0xbe7c89cf,0x4d48d994),LL(0x8d572003,0x511c2158), LL(0x088dda1e,0xc26fbac1),LL(0x7ad4934c,0xc3d55189),LL(0x85dcaf7c,0x5233c176),LL(0xa88b473d,0xec3a8a29), +}, +/* digit=5 base_pwr=2^35 */ +{ + LL(0xb1f0c175,0xfd96667a),LL(0x2ab99e7d,0xa256a611),LL(0x05e43f9d,0xff07c1ea),LL(0x7e1c9cd6,0x305700bc), LL(0x2b2887a3,0x3f1e2546),LL(0xc772fd14,0xdd782f49),LL(0x38584057,0x9125f996),LL(0x16a02cf9,0x19fd0396), + LL(0x4c58174d,0xa8d62bd3),LL(0xa900551f,0x872251d3),LL(0xf12802c3,0x06f5862d),LL(0xdd925555,0x5d93c48a), LL(0xbd6006f8,0xc39b67d5),LL(0xf96ccc67,0xea6f756b),LL(0x543014db,0x140e853e),LL(0xe9de42c0,0x2bdc5674), + LL(0xe01c073d,0x01dfda7b),LL(0xff9e1234,0x07a6bb65),LL(0x622cee4e,0x2a4f7f18),LL(0x50f0a3a7,0xdf4cead8), LL(0x1b8c2903,0x152b3c8e),LL(0x5f2a89b3,0x9e82e999),LL(0x68ce7a3c,0x0e6cfa7e),LL(0x0ca0464c,0xebb34d90), + LL(0xeda49f74,0xfa1a58fa),LL(0x0e4545a3,0xddb89957),LL(0xc74c07e4,0xd3576489),LL(0xb59b1008,0x64e4b39e), LL(0xf66b546c,0x3b090340),LL(0xcdeb912f,0x0e0f4013),LL(0x01e55cca,0xbb00b46c),LL(0x99ad0768,0x55b61b34), + LL(0xe8bbda5b,0xb06b71fc),LL(0xa24b0a63,0x8de64d84),LL(0xb5d4603f,0xb73dc262),LL(0x1965a916,0x5d5fa964), LL(0x3bc98966,0xb48a4053),LL(0x6f564743,0xaa871863),LL(0xe76a6a3e,0x88b00822),LL(0xb38d9e0d,0x58c9e92e), + LL(0xe989963e,0xc0d22337),LL(0xd3778d5a,0x2c4831ce),LL(0xee8c4178,0xd775c6a5),LL(0x9d0c2894,0xe2391654), LL(0x5d0eb314,0xf7d4fe86),LL(0x8b2290d3,0x42801b8f),LL(0xcdcefa78,0x73e9b332),LL(0x3e877fea,0xc0d169d9), + LL(0xffee23fa,0x29c8138b),LL(0xfb92e3b8,0xbff98230),LL(0x8fa75007,0x14077ad5),LL(0x88e61b81,0x4d3a6e10), LL(0x3bcf733d,0x218a867d),LL(0x665e37fc,0x20ff6566),LL(0xda5cbf67,0xe39c0581),LL(0x8add8c4c,0x4a6e1d7c), + LL(0x734a1327,0xcab02370),LL(0x1951afa8,0xa1df7afc),LL(0x42638b8a,0x581cfbaf),LL(0x2130eaa6,0x39db6d2b), LL(0xda2f91a5,0x4bbc805b),LL(0xe569add8,0x3dcb0a7e),LL(0xd721fa7d,0x724ab65a),LL(0xf88f8008,0xa5152b95), + LL(0x281615ba,0x7fe7f1b9),LL(0x41d5aa0c,0x419d1a53),LL(0x9fb0917e,0xafc556dc),LL(0x616ce893,0xab2a69f3), LL(0xc0861e05,0xfb9a6eb1),LL(0x5eb02b8f,0x0b74ae11),LL(0x3b1e44fe,0xccff0ad5),LL(0x88824f53,0x86dfe0e6), + LL(0x41177a46,0xedf38dc4),LL(0x7f039a7b,0xd9a955bb),LL(0x4d8ae7c2,0x4f152581),LL(0x4f848819,0x063c9f83), LL(0x841e8783,0x54ea4526),LL(0xaa5f2b32,0xe86a4119),LL(0x19846dcf,0xb7529a3b),LL(0x35689d70,0x91356a07), + LL(0x8f049ef8,0xbe66f5db),LL(0xc38dd5ed,0x0f5fd99e),LL(0x1b4ae7a7,0x1896d52b),LL(0x480b1ebb,0xf27c45c6), LL(0x3fede5c1,0xd88cff4c),LL(0xda27560b,0x57d902c9),LL(0x52d57deb,0x84aa7f07),LL(0x08bb6028,0x8da4c7c8), + LL(0x8910763e,0x658f4dea),LL(0x076a0f80,0x6e5fcb48),LL(0xab65f9b9,0x6a5447a4),LL(0xa75bb0c5,0xd7d863d4), LL(0xe87e7916,0x806c34a7),LL(0xcd961e88,0x05391559),LL(0x74fe6aeb,0x5def2d88),LL(0xf9226ca1,0x8ac350b2), + LL(0x12401813,0xffa8a649),LL(0x5337c55d,0xd6182762),LL(0x3be902e3,0xfce9d7ff),LL(0xea0dd7a5,0xb3b275d1), LL(0x2cb48ac9,0x342620f4),LL(0xa8b38a74,0xc0369384),LL(0xc0695d3a,0x04b0ee6a),LL(0x94c5394d,0x4d025585), + LL(0x81443d16,0xff9635d0),LL(0xa6cc364b,0x2342cbfa),LL(0x25bf8438,0x63b0a032),LL(0xa078d298,0x6ccd3ce5), LL(0x91292fd3,0xf93bd108),LL(0x14073286,0xc887a31b),LL(0x9f62cd16,0xeb1275bf),LL(0x61578b46,0x0335bae3), + LL(0x53348e4e,0x810d5efd),LL(0x63c74225,0xf9cd822a),LL(0xa426bf44,0x93d2e810),LL(0x019d36b3,0x95a47a97), LL(0xd5d1f840,0x1da421b9),LL(0xd6c46e3c,0xe5b8a55f),LL(0xc9244881,0x2dd3a5e7),LL(0x70c1fd2f,0xd50f9cde), + LL(0x614d9ff3,0xbee2aca7),LL(0x358f245a,0xd1f13b2c),LL(0xc46f62ab,0x9e92d83f),LL(0x827d7374,0xc1dd32dd), LL(0xc3e566e7,0x1636d593),LL(0x04ccb02b,0x81c2f4e7),LL(0xcd35b652,0xb57782c6),LL(0x88210d42,0xad88787e), + LL(0xfbd7d35a,0x3ad52d72),LL(0x37a2a095,0x4117f502),LL(0xd356b3b6,0xed03d415),LL(0x15ca6087,0x135d5a8c), LL(0xef5dca2a,0xfbaba41f),LL(0xafb4787d,0x660e5cd0),LL(0xa55e9ef0,0xe0e66378),LL(0x69939f56,0xf24513cf), + LL(0xab4f6bd9,0x0f38f09c),LL(0x922dcac1,0xec3037b4),LL(0x08a1a51e,0x706b201a),LL(0x8ffff040,0x15911351), LL(0xccf63d87,0x239d7b6a),LL(0x5187f595,0xeca37dc8),LL(0xad5a0ab3,0x04ea79e4),LL(0xe9520e8f,0xcdd81522), + LL(0xc35e1020,0x7fe6b6aa),LL(0x140ac884,0x57b63c9e),LL(0x33f19077,0xc45c23fc),LL(0xb71273c5,0x468d2c36), LL(0xfc305ac2,0xeb6839d6),LL(0x0183793a,0xf6e310ff),LL(0x32da639d,0xbca206e4),LL(0x8518e27e,0x8eb5cac1), + LL(0x66ed96f9,0xfeed0feb),LL(0xecc3a8dc,0x1632632e),LL(0x1455c8ae,0x90449363),LL(0x0aeada65,0x8d7619d4), LL(0x9f630ee9,0x2f2fa898),LL(0x370db87c,0xd78caf0c),LL(0xc45898cf,0x46fa0fc9),LL(0x2d84244f,0xa509cc3e), + LL(0xa5b099aa,0xbdbea4b4),LL(0x7592587b,0x8e8fe284),LL(0x42000897,0x0226d387),LL(0x36db5cd9,0xb6780551), LL(0xca64f047,0xd8fe5eb1),LL(0xb77cf8cb,0x6f21474b),LL(0xee45ae34,0xab8fcae7),LL(0x1f19cd67,0x73eaf9eb), + LL(0xee4df6ce,0x5bb96415),LL(0xa3ae4cf3,0xd1e27bcf),LL(0xc7f1868e,0x9bf7ace3),LL(0x82091dca,0xe821aa8b), LL(0xd381b6c4,0xf732e6bc),LL(0xdd01864f,0x5feda346),LL(0xb6387846,0x0933b92c),LL(0xa0028029,0xbf1f1e83), + LL(0xa3e38124,0x0848bf8c),LL(0x208fda8f,0xfe295fdf),LL(0x8913a1c4,0x73379239),LL(0x7e78564e,0x59354b24), LL(0x32dcafbc,0x042b7529),LL(0xfa93c5c7,0x752173d3),LL(0x9737135a,0x6ffd4590),LL(0x0f983005,0x249712b0), + LL(0x1f25da8b,0xdbba2874),LL(0x097ba4a9,0x14027f11),LL(0x34b8e4a2,0xe429b3c7),LL(0x056b4afc,0xd66a43e3), LL(0x2ac351e6,0x158644ad),LL(0x164bc6cc,0xff4aecd9),LL(0xf6c615ee,0xbb5b0c87),LL(0xd7679b1b,0xc497d8ee), + LL(0xf1c6e97a,0xf666c625),LL(0xc73a277f,0xe89f84b2),LL(0x746af4c0,0x2403d513),LL(0xb7101feb,0xe6858fdf), LL(0x84f1dcb7,0x1a42c51b),LL(0x8202bc04,0xc57f12e0),LL(0x754df5ae,0xf8326a93),LL(0x81a46aef,0x3d3daf04), + LL(0x01232d03,0x8bb8c276),LL(0xfb371cf1,0xd446c82e),LL(0xefa495f4,0xe5e8b639),LL(0x477e6493,0x51a7b34a), LL(0x824f2b6e,0xffba5466),LL(0xf0eaa6a9,0xcc67ddad),LL(0xfee19b24,0xcf0f8ce1),LL(0x83b3df41,0x34309127), + LL(0x9719a6cd,0xc8b13e8c),LL(0x619d5c33,0xb408e505),LL(0xa3158864,0x8c1b831b),LL(0x0b3d02bb,0x506b3c16), LL(0xbf11ff8d,0xf23846bc),LL(0x16e0328a,0xf0f043e8),LL(0x65986a7a,0x30b7b9cd),LL(0x21b660cd,0x0951b102), + LL(0x52bf29a5,0x72a26c5f),LL(0xb6534592,0xb513d669),LL(0x578195ea,0xb8ac15ad),LL(0xc0785f88,0xd6ed33ea), LL(0xb9e33946,0x39e23dbf),LL(0xf43e88eb,0xeadb2453),LL(0x2746c34b,0x6d82fefa),LL(0xcc542b54,0xe9172aa0), + LL(0xecb50699,0x8af6b819),LL(0x1c1d0af9,0x4af76939),LL(0x99dddb1a,0x5a7dbbbe),LL(0x891ea41d,0x97b0a3aa), LL(0x6e35ea4f,0x32b457e6),LL(0x9d77b900,0xe2a21c2a),LL(0x2ac991cf,0xb18718d6),LL(0x740743cd,0xc4416237), + LL(0x6a05ab55,0xcc3f76b6),LL(0x98091425,0x2ab4b29e),LL(0xb6478fc8,0xbf373ad1),LL(0x178b5844,0x8a1a9489), LL(0x09daf4be,0xb5295edf),LL(0x4ed54766,0x07fbb119),LL(0x7d0b9d8f,0x6e44367b),LL(0xedb96a10,0x6dc4d8f6), + LL(0x37fc19a3,0x2ba69106),LL(0x0b138296,0x522eba39),LL(0xfda58cf3,0x751544c7),LL(0x0ba33938,0xaba6fe16), LL(0x94dac7d6,0x48e085be),LL(0x19f99faa,0x06c87014),LL(0x1a587f89,0x33b9a8d6),LL(0x3fd8d8fe,0xdae382ca), + LL(0x150b0fcd,0xb5b383c6),LL(0xed9b0f4c,0xf948da80),LL(0xccd05413,0xcf075225),LL(0x4f62be64,0x3f31b12c), LL(0x368c17f6,0x23b21fc8),LL(0x400bc690,0x423d5369),LL(0xdeac140e,0x5335dd1e),LL(0x9493ad61,0xe631c249), + LL(0x32fe490a,0xc274c695),LL(0x6d8ebd70,0x42bcb4e1),LL(0x65d7a1d0,0x69059e1e),LL(0x29fdd109,0xf36dfe2f), LL(0x0c4e6370,0xacfea1ec),LL(0x7a935ff4,0x97e7f722),LL(0xf8006bbd,0x83e7c7c3),LL(0x78e6792c,0x87a8b84d), + LL(0x94d3d60f,0x5cbe4883),LL(0x91cbc054,0x6eba464d),LL(0x021c38fa,0xf9c880d0),LL(0x21af4942,0x6200faf1), LL(0x5f03e261,0xd5b2b12d),LL(0xf3ea3e07,0x1659a0ac),LL(0x836757a8,0x8008f18d),LL(0x75a8f8e0,0xfb2f467b), + LL(0x9c9b00cc,0x9a6183c7),LL(0x3bf842b0,0x82ca07e3),LL(0xee1f83d9,0xe7089191),LL(0x2d0cd2da,0xc41ecde4), LL(0x4d1feacd,0x0ce421b0),LL(0x431c53f1,0xe80a1395),LL(0xe6bfccf5,0xae9b2018),LL(0x8b359c3b,0xdf9f86ad), + LL(0xb6170a5f,0x9887e28f),LL(0xf3c0c30c,0xf5b85d21),LL(0x632af7a4,0x30861cf8),LL(0xbb4ec123,0x2fb670ad), LL(0x3c425976,0x0668b84c),LL(0x02883af7,0x55c21b4e),LL(0xf8698d29,0x0fad58b5),LL(0x68b671c5,0xef210770), + LL(0x23f232b8,0x534d510a),LL(0x49c99708,0xdb66fec1),LL(0x6d54721b,0xf1a6f3e7),LL(0x4480f858,0x8d37ab64), LL(0xb0f7f354,0x7fcfca6c),LL(0x95bfd318,0x58c7ff5f),LL(0x903f9d91,0x3048e9af),LL(0x75357af0,0xe480bc0e), + LL(0xa5a1162e,0x4f915e1c),LL(0x37efa40c,0xdd539c21),LL(0x789201c2,0x61a45c53),LL(0xe7890746,0x1bc2333d), LL(0xbbed8f77,0xeed38f50),LL(0x178501a0,0xc1e93732),LL(0xa8fb8623,0xfed5b1d1),LL(0xdc3e1148,0xa3be3e2c), + LL(0xa71a390a,0x62fc1633),LL(0x2891c4c3,0x4be2868e),LL(0x2a0c3c23,0x6573fe49),LL(0xde1589d0,0x182d0bd4), LL(0x5a7aa63d,0x17c6a780),LL(0x12543191,0x9d84cfa8),LL(0x950c85c9,0xcdb22db7),LL(0x119010c4,0xd03589e0), + LL(0x8220dee8,0xbcd02e8a),LL(0x705632fd,0xbd4d1f2a),LL(0x22f8e30b,0x00119bfd),LL(0x6eb97c32,0x06c6e73e), LL(0x35abff53,0xa26f0a6a),LL(0x8564c37d,0x7d79a89f),LL(0x1b207495,0x0347bb17),LL(0xb5c8253a,0x1baf90e9), + LL(0x37affc96,0x01059b5f),LL(0xffee0a60,0xbe76c578),LL(0x75d6b83c,0x45d7291b),LL(0xe0b58129,0x212ff131), LL(0xaa5d46ed,0x4acc5748),LL(0x9193931b,0x9fc557d9),LL(0xda4eba9b,0x17568fcf),LL(0xa0edc975,0x2cf3690c), + LL(0x953df6fd,0x0e8b0e7e),LL(0x62036a87,0x38ea7cea),LL(0x655c3685,0x57e01428),LL(0xc39d8a43,0xaedfee73), LL(0x5fb27e0a,0xed7f6598),LL(0x946888e0,0x524c3653),LL(0xe949b72f,0xd84a299b),LL(0xb0c61ea4,0x76c1397a), + LL(0x1afe465a,0xfd9f7ed0),LL(0xdbbaf852,0x832c69ad),LL(0x03713338,0xcd888c22),LL(0xe3306617,0x4e1fe026), LL(0x23521b97,0xa87adf86),LL(0xf9fbb2a0,0x673d3625),LL(0x5d8f5b80,0xf29a1413),LL(0xd3526501,0x6e9be0c4), + LL(0xe8bfd84d,0x6129f861),LL(0x77e35a47,0x1df491d6),LL(0xa84a82cb,0xefe0e9a9),LL(0x6d949612,0x972bc3bc), LL(0x3a766eca,0x8d7795f5),LL(0x12fcc6d4,0x6119383f),LL(0xc95f0e21,0xa66d9836),LL(0x684e434b,0x77a0aa0a), + LL(0x7dd7b05a,0x3d55d256),LL(0x0fed8362,0xda616243),LL(0x383e94fe,0x24bd0fe8),LL(0x6bfd0cd2,0xbc2b7334), LL(0x321f7a70,0xf9497232),LL(0x6a3df54f,0x37a4c2f6),LL(0x4ddc49d6,0x7ba783bf),LL(0x04408c76,0x4d142317), + LL(0x38b99f23,0x7502146b),LL(0x21992e8f,0x479ab73c),LL(0xd52c41d3,0xf605370a),LL(0x3a60435f,0x358b746d), LL(0x5bc537b8,0xb2cbab94),LL(0xb99057d3,0x1fd24431),LL(0xb8510f3c,0xff2242a0),LL(0x0733bc53,0x74b4965d), + LL(0x86edc9b2,0x30a3a634),LL(0x49c07c7f,0x99c9cf19),LL(0x5b0cd506,0x9d8a50c2),LL(0xbbcb3d65,0x0ed9da5a), LL(0x013f88ec,0x6de1fb5e),LL(0x09086f8c,0xc9356bff),LL(0x2b8825d7,0xa272e1ac),LL(0xf2c5ba33,0x3ad83acb), + LL(0x275bce43,0x721ca22c),LL(0xd24f78e8,0xf058b8a7),LL(0xeed46b97,0xd178eb57),LL(0x259fdb5b,0x4ad7d425), LL(0x1b515fe5,0x669ed853),LL(0x76fa1b5e,0x9f14b8e5),LL(0x3da46b02,0xfaba8d0c),LL(0x338f7652,0x759c2c95), + LL(0xb5c0ceb3,0x9a369cb0),LL(0x28a2a633,0xc1d2d1ab),LL(0xfcb48cd3,0x676190e3),LL(0xee09c3b1,0x9af97ab3), LL(0xf7e918f5,0x39323719),LL(0xfd3cd554,0xc657cb57),LL(0xa2a02d5c,0x78a33d05),LL(0x64ada03f,0xda3b79df), + LL(0x61b3a72a,0x7115ab5c),LL(0x337194fc,0xdd19f34b),LL(0x8f0a14c3,0x0f23bfec),LL(0xa60485d3,0x1fe19eec), LL(0xa463dc9b,0x1ca308c3),LL(0x5e1ae8be,0x83e18dd0),LL(0xd928c0e7,0x849eabdf),LL(0x6bd3e7b3,0x2d131ff5), + LL(0x45be4c14,0xc84cd284),LL(0xf8f4c719,0xdee94092),LL(0x3cb73831,0xe8f223ef),LL(0x18c2361e,0x24382f88), LL(0xbe91c8dd,0x205366d0),LL(0x56024b95,0x1e17b50c),LL(0x742cabd3,0x3c3487da),LL(0x8bad494c,0xbe451387), + LL(0x18ffaef0,0xfae6c0bf),LL(0x85ed1ede,0x2e7b0ee3),LL(0x125d1488,0x3cebaa05),LL(0x7c8b7fb8,0xcd0de0fe), LL(0x464bc74a,0x59434d54),LL(0xa03fd77b,0x17472da2),LL(0x2c1a9edc,0xab23d042),LL(0xd9cf4b37,0x5390625e), + LL(0x0531264e,0x43b85844),LL(0xee7aedca,0x8d71805e),LL(0xfbe643ad,0x4ace3068),LL(0x5f7d46c1,0xc98d1cd2), LL(0xf59b3acd,0xd4888744),LL(0x27288b99,0xcf662d61),LL(0x5bce2649,0xf2704561),LL(0x206ae654,0x33a8f3f9), + LL(0x9bce2b39,0xe834843f),LL(0xa90cfc7d,0x8de8e41d),LL(0xd81115b4,0x398800ed),LL(0xff2532da,0x4d33f7c5), LL(0xdcc59e2c,0x5ae37fb2),LL(0x24015306,0xca27b622),LL(0x11e8d6e6,0x51beca89),LL(0xa9693774,0x08c0b7e2), + LL(0x72fa713e,0x795e1a21),LL(0x4be58266,0x5ec1c123),LL(0x1be14fc3,0x5d8e87da),LL(0x80283ad5,0x82cefc1e), LL(0xdab7865e,0x820a385b),LL(0xf3daf96c,0x11e32d62),LL(0x5835a989,0xf022ade7),LL(0x00242233,0x2cbc2554), + LL(0xe7ce649c,0x653335a0),LL(0x6857eff7,0x8b30baef),LL(0xf3288377,0x7ea7c856),LL(0xe8572f5d,0x1387b347), LL(0xbe10c0cf,0x8a6b0352),LL(0x037c97b9,0x2a74e834),LL(0x197b122e,0xfe10bf59),LL(0x1918aced,0xd1ee174c), + LL(0x3958c20d,0x568e5fb9),LL(0x0484a92f,0x1188cbe6),LL(0x4b0d29e3,0x00ec14f4),LL(0x16a2796d,0x2b2e078e), LL(0x20440444,0x48b8cffa),LL(0x661ab68d,0xd4b877a0),LL(0xc4b459fa,0x1f352ab1),LL(0xc53aa54c,0x33accbe6), + LL(0x02bb383b,0xce4ff566),LL(0xfd62813d,0xcad561c6),LL(0x01dfc9a8,0x0927c348),LL(0x00fb9a61,0x0dde73fb), LL(0xfce59f34,0xd859809f),LL(0x81872a46,0x225bd9b6),LL(0x0314bb90,0x2642add2),LL(0x0ae61eb8,0x82dc7958), + LL(0x22d5b667,0x84c97478),LL(0x6214f46d,0xb2fe94d1),LL(0x12cb20de,0x834740f2),LL(0x8aa69c94,0x336dc7a7), LL(0x939a33e6,0x8ca085a4),LL(0x75a94543,0xd59c9ae9),LL(0x3c47dd07,0x83c97f98),LL(0xe3177231,0x0985f73e), + LL(0xebbc623d,0xe556c3fc),LL(0xb1b968fa,0x30a3242f),LL(0xbcd05a51,0x842ce9b0),LL(0x0ad576ce,0x241a35ed), LL(0xbb4a793e,0x49ccaf3c),LL(0x4492a828,0x6e6c7a7b),LL(0xba53eb42,0x72f4f5fc),LL(0x3ea74dab,0x0ca4ba53), + LL(0xbbaf9d5f,0xe7b5fb06),LL(0xb02d3b20,0xd49c2e17),LL(0x2d933cc8,0x4d31052a),LL(0x07299aec,0x5346e0b4), LL(0x79aa99ec,0x952a6205),LL(0xecb34e97,0xaab9bc32),LL(0x58ffe9ae,0xd539d7e4),LL(0x9d994472,0x91599393), + LL(0xe8822711,0x6b1d4868),LL(0x73d452b8,0x8857e282),LL(0xf08ed046,0xad59adfd),LL(0xc1c47abe,0xdb755d65), LL(0x63275d49,0x2df8520b),LL(0x7f8a3249,0xc3c712ec),LL(0x5215ef57,0x55f2a108),LL(0x3ee2f149,0x955e07a3), + LL(0x33f344f4,0x2194ff53),LL(0xbad16820,0xb455b9fe),LL(0x610b4e4c,0xfe69ea78),LL(0x8ab11fe8,0x2957be96), LL(0x2ce14366,0x3efdee3c),LL(0x01eddf9f,0x42043f9f),LL(0x93524f6c,0xfb7998b1),LL(0xdfecf763,0x95ea64c0), + LL(0x21afa86f,0xb23d2620),LL(0x86b11457,0xea757f03),LL(0xb0148d30,0x0bc4d2d1),LL(0x88ce4170,0x119b5535), LL(0x0aa9c8f6,0xaab5bb67),LL(0x88e05de2,0xdfc9693c),LL(0xe3f1e9c3,0x6cae7e57),LL(0x6f6c3b9c,0x2b1ceb54), +}, +/* digit=6 base_pwr=2^42 */ +{ + LL(0x87636183,0x12e335ca),LL(0x719d1ca3,0x1461a65a),LL(0xb14161d8,0x8150080a),LL(0xc612e112,0x08da4ebf), LL(0xa8498a9a,0xc95dfb6b),LL(0xba0f8dba,0x099cf91d),LL(0x4fb4f497,0x12d2ae14),LL(0x33cb7306,0xfa3a28b0), + LL(0x0f01c7ce,0xc89fc5d0),LL(0x7283bdf0,0x6fc45ffd),LL(0x81151923,0x71dece81),LL(0xc433fcc9,0xed1cb14c), LL(0xd3959bcf,0x4279612b),LL(0x35b5732f,0xe163880b),LL(0x71d0a1ca,0x35414ca7),LL(0x2c1e47f3,0xe8b9e651), + LL(0xc8df0a74,0x4ff11b0c),LL(0xe095ea9a,0x346ba520),LL(0xcc2bc6c0,0x81dd2268),LL(0xc2701468,0x2fb2e99f), LL(0x98053f0e,0x0d213361),LL(0xf7ae879a,0xe0b8280d),LL(0x952560f7,0xd92b7a75),LL(0x9723b62e,0x8d17dfad), + LL(0x08b21362,0x5ce8a78a),LL(0xd9fe0b36,0xf37f5e7f),LL(0x2c87837c,0xdca66c7f),LL(0x0bf2e993,0x92524b94), LL(0x71745788,0xfc0f020c),LL(0x3cbfbf4c,0x6018de46),LL(0xac3de1c8,0xa8446691),LL(0x5de5ae41,0xb194d419), + LL(0x2ff27af2,0x1586cdff),LL(0xde26b5ef,0xee628535),LL(0xc682923e,0x58480040),LL(0x5e37da30,0x4dd4596b), LL(0x2f64225f,0x247b9fd7),LL(0x51ca2121,0xdcc6de5f),LL(0x86e7ab9a,0x99fb41ac),LL(0x952b413a,0x54c782a0), + LL(0x7298c7d9,0x7641190e),LL(0x716eda14,0x499c35ed),LL(0xbb764e90,0x316134bf),LL(0x884fc34e,0x4d23467e), LL(0xf1d13484,0xfd1208a9),LL(0xcd298a74,0x089d9605),LL(0x73c4346a,0xb398c85a),LL(0xf37f13de,0x50064076), + LL(0xa6ebb83d,0xfe10d25a),LL(0xa834b30d,0xc5e3edf8),LL(0x683e09ff,0x546b5d5c),LL(0xc6dc44c6,0x02f96218), LL(0xc0edfc04,0x64528c55),LL(0xb0fc3058,0xb5a44a2c),LL(0xceeff21c,0x9f09b116),LL(0x6b0fbbcd,0x077bcd67), + LL(0x9ce76a94,0x29aaa4a8),LL(0xc0725c97,0x847cd081),LL(0x97e16665,0x0c099e90),LL(0x8f7b1fc4,0xe409ffc9), LL(0x690941ed,0xc0575b80),LL(0x92c0ee9d,0x8e25100a),LL(0x9b75837d,0x71662d27),LL(0xe56bb22b,0x6eeb9e97), + LL(0x85c6a60b,0xf1d6333f),LL(0x1d7ccfaa,0x982fee9d),LL(0xd4634535,0x1c5e28e7),LL(0x94fec469,0xa76e1d27), LL(0xafe377ec,0x1fe944d6),LL(0x2f68ae6b,0xbd579a25),LL(0xab6b515e,0x10eabb93),LL(0x31b4e4b8,0xa17b5f6c), + LL(0xaf72c239,0x05e785fb),LL(0x8af42e92,0x597e2016),LL(0xb32ae6c9,0x663f5a72),LL(0x45541cc6,0x3040ff13), LL(0xdeca6b32,0x6140081f),LL(0xc735001b,0xcdaccaf7),LL(0xdaef0e68,0x62de5066),LL(0xd837df99,0x056e9021), + LL(0x16cd1be7,0xba399283),LL(0xcfacf7ad,0x2a486323),LL(0x277777ce,0x00c15730),LL(0xd49a254c,0x5d2f200f), LL(0xdb68078d,0xf38a1f3b),LL(0x33604a36,0x595dea3f),LL(0x904b60b2,0x14749d8c),LL(0x246348ff,0xe70c96d8), + LL(0x390e35da,0x04340d52),LL(0x27a9947c,0xc098e3d3),LL(0x9ecc7a3f,0xe6d78198),LL(0x23aa6df6,0x2c39102e), LL(0x300f3cb1,0xb83fed0d),LL(0xdcfbe054,0xc0b1e356),LL(0x20cf45a8,0x3da2224c),LL(0x2f30deda,0x5be55df7), + LL(0x2faa9530,0x4d31c29d),LL(0x49d42f79,0x1d5783ae),LL(0xf618b3f3,0xe588c224),LL(0xf8f5b65d,0x7d8a6f90), LL(0x62d09174,0xa802a3d2),LL(0xbddd1cb7,0x4f1a93d9),LL(0x35a5c1dc,0xe08e1d3c),LL(0xf9d2958e,0x856b2323), + LL(0x96f00090,0xefd1e3ba),LL(0x3e0d25de,0xd489943f),LL(0x30c8626f,0x082c40ae),LL(0xa4f428e0,0xf6e5b5ef), LL(0x38a7f623,0x660414a3),LL(0x23eefed8,0xcd4e68de),LL(0xfc14e750,0x6dcadc62),LL(0xbeae89b6,0xcb78b3bc), + LL(0x1d5e580e,0x445acc56),LL(0xc43abe19,0xbf6547ef),LL(0xc922d50f,0xd160a81b),LL(0xf68eed4e,0x3877c7f8), LL(0xf8a9f64a,0x395745ea),LL(0x603350f6,0x9085b253),LL(0x8b1df366,0x2a4c71f1),LL(0xabe332dc,0x49b9e818), + LL(0x528960b1,0xb3e76e66),LL(0xd84aecb3,0x445dc393),LL(0x1612ad64,0x13618436),LL(0x8c831e37,0x3ccbeccc), LL(0x6121383c,0x0fb0bd41),LL(0x80d895a3,0x316164a3),LL(0x233f2f1e,0xc3d34153),LL(0xe0d92225,0x2905906f), + LL(0x95456622,0xe12d66e2),LL(0xff554b13,0x10469942),LL(0xf7126c09,0xa894af86),LL(0xf581d3f5,0x448f3267), LL(0xa2b5e758,0xb5512926),LL(0x43fddd90,0x08f02988),LL(0x8ba319e6,0x5f437035),LL(0x865b37e7,0xd254188e), + LL(0x8a5cb63a,0x5b281b23),LL(0x6dd136c2,0xa15a2712),LL(0x169beae4,0x00fab229),LL(0xde31b4a1,0x400d3f37), LL(0xf8545cb0,0x275877a4),LL(0x36df0277,0xb396a513),LL(0x838ced07,0xf9896978),LL(0x715cea8d,0x86e68167), + LL(0x06a5a96d,0x0eb0f0de),LL(0x1fcf91ae,0x2c7a3672),LL(0x630eca3a,0x287bf614),LL(0xf60c9d2d,0x65347473), LL(0x906efa7f,0xed15a229),LL(0xd549b5b3,0xe7be6381),LL(0x2ce9141b,0x23f32972),LL(0xfcf823f8,0x9618d9a1), + LL(0xa3d89e15,0x3d0ef0d3),LL(0x0d07f5eb,0x4d5a30c9),LL(0x73e4887a,0xc359e310),LL(0xdbdec349,0x2f4c6b7e), LL(0xba142643,0xc5a1d3e9),LL(0x11c794b6,0x8f4fd58e),LL(0x1810c63d,0xcad091d1),LL(0xf0bfa76c,0x5b616239), + LL(0xa838792a,0xe3433562),LL(0x54148e98,0x4aead02b),LL(0xdb66f216,0x809f2baf),LL(0xeabfe5da,0x09cc90ff), LL(0x63e8edad,0x69eb235a),LL(0xa7f95997,0x64f7acb5),LL(0xfae20f25,0xe999ea18),LL(0x3c4966b3,0xcd7ff208), + LL(0x345c8929,0x595e0cc0),LL(0xde5e2498,0xfe43c73c),LL(0x503f216d,0x0cdefc98),LL(0xf98826fb,0x8e4e170d), LL(0xb6c79b1c,0x1492247d),LL(0xef0532aa,0xf8e24b38),LL(0x044bc458,0x9f349d51),LL(0x1002d315,0x2ef04ead), + LL(0xda60d581,0xaf322f23),LL(0x0681173f,0x07deaa88),LL(0xa78feca0,0x86b97444),LL(0xc633a54d,0x64d336ea), LL(0x2a426cbf,0x10dd4b1f),LL(0x7af59869,0x08d97c15),LL(0x2d7fe97e,0xb8cc814b),LL(0x3bfb60fe,0x7eacd2e1), + LL(0xb790881c,0x967dafb7),LL(0x3663e76c,0x2002b8e4),LL(0xf8e82490,0x3bd28ede),LL(0x4bb2a47a,0x44dd2e81), LL(0xdbc3f2f8,0xde750dfe),LL(0x6e2eec70,0xd9b6e912),LL(0x1e4c4d2f,0xe8400e2f),LL(0x23217be2,0xd3325697), + LL(0xd4231a1d,0x030b7e39),LL(0x613d17d8,0x1f72e8b1),LL(0x01857d37,0xcd423512),LL(0x0b4b7926,0x9ecd682c), LL(0x8ec44636,0xfe4ac1c3),LL(0x9aacc091,0x4a030cbf),LL(0x0b133f54,0x12bb252e),LL(0xf970d266,0xbf90ea5d), + LL(0xf5484410,0xe00d25f7),LL(0x2a922894,0xb4984eeb),LL(0x8e1e09ce,0x498102fd),LL(0xe1d731bf,0x8f8c9fcb), LL(0x0b4983b7,0xdb897669),LL(0x7b2468f5,0x7a7767f9),LL(0x72f4d5f4,0x1a2f9fe8),LL(0xa6169daa,0x10e95aa9), + LL(0x520166e9,0x9487e50f),LL(0xc4ee6a95,0x6f6b115b),LL(0xcf7560f8,0xaf29926f),LL(0x1f989e46,0x20a32458), LL(0xd3bd2473,0x165a2232),LL(0xe9fecdf8,0x93d841ff),LL(0xbf9978c0,0x71d63fa7),LL(0xe7de184b,0x381bcf34), + LL(0x347dfaef,0x317c8e40),LL(0x64464bf3,0x795b0f7d),LL(0x1364ec20,0x15dc99d6),LL(0x91600d3f,0xc07fce28), LL(0xc8bebbda,0x9825338b),LL(0xa8547c03,0x5e5e89f6),LL(0x1a040b84,0x3c50032f),LL(0x2b3a533d,0xcea7544f), + LL(0x43247e19,0xea26d87d),LL(0xfba8368e,0x7e753390),LL(0x3c7bcfc6,0xb35e75cb),LL(0x7e44aab3,0xf78cb5ce), LL(0xa98d7104,0x4a3534e9),LL(0x6f5852ea,0x2b83ea6c),LL(0x68dced7c,0x11337fff),LL(0xd1a2a294,0xcca0f2c6), + LL(0x426bf202,0xb547c662),LL(0x66194a34,0xec50423e),LL(0x78161e84,0x11d34865),LL(0x64f04dec,0x83508c06), LL(0xf7732345,0xd1c72976),LL(0x18e77e0a,0xd624bacd),LL(0xba79bdd9,0x71344b75),LL(0x8d6c1474,0xe4bfe085), + LL(0xcc5eb43a,0x505e8fd9),LL(0xdaaf0621,0x612ab1d0),LL(0xe6672368,0xde170783),LL(0x83788673,0xfee7df44), LL(0xd119271d,0x364d6885),LL(0xe1b0cea2,0xdd70bae8),LL(0x5832adba,0xb4b873ad),LL(0x8c683749,0xad3ecc18), + LL(0x4d217a2e,0x963d8793),LL(0x1fa4702d,0x099e8c56),LL(0xe6431f1b,0x6d91bc47),LL(0xa5f61399,0x3fd21287), LL(0x682fa08b,0x2fc90bae),LL(0xc1ca371c,0x51699c85),LL(0x831c428f,0x16f29d74),LL(0x9fa2b504,0x0ecefb66), + LL(0xa75c5a91,0xd04ac53f),LL(0x3bf0524b,0xcbe62421),LL(0xb8792826,0x91dcb3ce),LL(0x7885092a,0x28a6bf88), LL(0x64c1e218,0x24798e59),LL(0x3fec97dc,0x18e848dc),LL(0x9da457b7,0x935e0f50),LL(0xb8f497a6,0x46b67ab7), + LL(0x7651e4ff,0x15a38140),LL(0x4890cd7f,0x6ba6c617),LL(0x5fe253ca,0xa527b8d2),LL(0xff3d603b,0x945277b8), LL(0x75392f01,0x10796155),LL(0x2bd9619d,0xcac8f713),LL(0xdebb8e28,0x71a87eca),LL(0x52ab1792,0xe8e6179e), + LL(0xe33705e7,0x4ce3998b),LL(0x48ba56e4,0xf9a0661a),LL(0xd9e4e184,0x47f06b30),LL(0xf9f8f6df,0xda465f75), LL(0xc0ad3e20,0xb05acbbe),LL(0x92bc2c13,0xec8776a4),LL(0x240a908b,0xbb3971b7),LL(0xbbd0cecc,0x80a14367), + LL(0x086949bc,0x40911e50),LL(0x4064a19d,0x39b3ab69),LL(0x6b07eaa7,0x538c6d96),LL(0xd3723bde,0x38c05b47), LL(0x080d2a64,0x1e669308),LL(0x2a77601f,0x6b44dbe5),LL(0xe7c6ce9a,0x35579681),LL(0xa16afa75,0xd2950b0e), + LL(0xeaf7fafc,0xd228a3ba),LL(0xee878f00,0x9324e71d),LL(0xe413c1ce,0xa853bfc1),LL(0xdcf17d1c,0xfe916368), LL(0x2546154b,0x8611383a),LL(0xe715b375,0xdbdf225d),LL(0x8dbb0651,0x874d70a6),LL(0x9ed56391,0x84e58895), + LL(0x3776503c,0xca83d8ad),LL(0x46e82d65,0x2cf38b4e),LL(0xadf3a8d1,0x65af46e6),LL(0x1d31056d,0x4f09a4ab), LL(0xcacc81d5,0xdba27b42),LL(0x5d6e1bdd,0xb6caa0ca),LL(0xf7198b26,0x1086e441),LL(0xac572f9b,0x15dfe6cb), + LL(0xd2051dd5,0xd9444337),LL(0x834cd72d,0x6c34b236),LL(0x58df3f28,0x84783216),LL(0x2e921372,0x59b8808e), LL(0x55835302,0x3b268249),LL(0x9f4863f1,0x3299cbe0),LL(0x4c4e213a,0x616e3cdd),LL(0x8c824317,0xa3c84868), + LL(0x460ed764,0x884be61c),LL(0xb6041177,0x388df47b),LL(0x60b29b0b,0x27089763),LL(0xe502ba08,0xd66d7d53), LL(0x5acbfaf4,0xadec85ca),LL(0x470c9519,0xfbacf9b7),LL(0xdbcda5b2,0x5d18b7f6),LL(0x0f228ed5,0x7615c036), + LL(0xd6000825,0xdfcd8273),LL(0x9d440eb3,0xdacfcf11),LL(0x7738fa46,0xa8257834),LL(0x76281df3,0x7db548af), LL(0x3e0b052c,0x71dd19f6),LL(0xd876613f,0x811feef2),LL(0xf9c3155c,0x7536e854),LL(0x4c8c1220,0x3e594973), + LL(0x069b4228,0xf8c5c72d),LL(0x1f2f6b02,0xc077d394),LL(0x76393a52,0x0032dfb9),LL(0x706c3572,0x5e52c880), LL(0xe003750f,0x4a9d6e9d),LL(0x6147cee7,0x3d54814d),LL(0x23b655fa,0x09ed7f77),LL(0xf1988361,0x14fff651), + LL(0xfb263d48,0x742f3abd),LL(0x53699a0c,0xedb557dc),LL(0x7ecd0f61,0xc692a474),LL(0x058f0d17,0xdc64f583), LL(0x3227d3ed,0x68a9ce75),LL(0x4601d083,0xfd0b0320),LL(0x9c2cee38,0x7167b330),LL(0x710e350d,0xef658993), + LL(0x16910648,0x75a83be1),LL(0x2e7d446c,0x5b32e77d),LL(0xa86ba2de,0x8e0534e5),LL(0xb692aeee,0xc8a92eac), LL(0xf663150f,0x3cf686eb),LL(0x01962baf,0x840eaade),LL(0xa264d877,0x3e06154f),LL(0x24630300,0xbbd04137), + LL(0x58631468,0x0b0151bd),LL(0x9f99bbe5,0x570ef82c),LL(0xb30f7b96,0x03565f47),LL(0x98c04b24,0x000628e0), LL(0xd6ccdb2b,0xd34a90ae),LL(0xa99a761c,0x1a584858),LL(0x65e29f1b,0xa640ddca),LL(0x728d3208,0xffb672f9), + LL(0x5433abd8,0x550f6392),LL(0x13ff0107,0x4f35e116),LL(0xe731a37b,0xbb2b0fab),LL(0xc83d6e74,0x1e8a5a08), LL(0xc6565e23,0xf617e177),LL(0x76da0795,0x8e370e5a),LL(0x03936a55,0xa5631e02),LL(0xd41293ad,0xe576bee9), + LL(0x5381bc98,0xfcfd9bc7),LL(0x000a98ab,0x8a42ddfd),LL(0xb49463c4,0xd6091ca6),LL(0x9754ce07,0xf37f6b9f), LL(0xfa399fda,0xe1543897),LL(0x8810063c,0x7b029ead),LL(0xec5a5b52,0xa98a46bd),LL(0xd50cc504,0xdd162811), + LL(0x67a95e56,0x4d725c1d),LL(0x8e17af44,0xc36d6e8b),LL(0x313454c1,0x38ffb699),LL(0x991e4eaa,0x22c3da8c), LL(0x0bb72dc0,0xfa36ee15),LL(0x4fd01d32,0x356bbf74),LL(0x3c7939a3,0x9ff71a30),LL(0x691786e9,0xa0ad2fb5), + LL(0x283c34c1,0x7d7f4770),LL(0x2454a31c,0x0148a4f0),LL(0xeab3b646,0xdcbb138a),LL(0xf101223e,0x7834bdb9), LL(0x965baa81,0x49de6cc3),LL(0x15471215,0x5462f15e),LL(0x81d17760,0xd77e7a56),LL(0x53f00de3,0xa08c5ad9), + LL(0x397ed010,0x2e6e6686),LL(0x2bef28cb,0xe444a5a9),LL(0x4073cbe4,0x1ab9d616),LL(0x18f0b7dd,0x24c6b9e0), LL(0xc2a93055,0x456482b6),LL(0xefbe8715,0x0f89129f),LL(0x62e5f6f2,0xb50818c3),LL(0x3d63c663,0x1d74a1ca), + LL(0xd84bfa55,0x9a9124ee),LL(0x254b3f04,0x6cf81f41),LL(0xa1051109,0x7c9b7c3e),LL(0x640e8df2,0x71c3d6d6), LL(0x62f6af2f,0x56571157),LL(0xe1bc9ae8,0x2ec61a3f),LL(0x2fcc848b,0x20caa2ff),LL(0xdc5c297e,0x71e30dac), + LL(0xfbc0740a,0x11901efc),LL(0xb6e35fca,0x4994fc5f),LL(0x177de7dd,0x4dc09eba),LL(0x0494bebf,0xedfdd25c), LL(0x0cbaeb8b,0xc4821ed9),LL(0x66788fbd,0xa9ef7a48),LL(0xd65efbc2,0x5b7a7ca5),LL(0xa9cb1fc6,0xe18feb42), + LL(0x56b00ab5,0x2cc74b9c),LL(0xdb4bf3f4,0xf559a140),LL(0xb8a4b54e,0x283136d4),LL(0x37032aa2,0xe969e4f8), LL(0xd85beed3,0x5635fb66),LL(0xa72a54bb,0x32bc4fdd),LL(0x4c386a49,0xc1e5ee2b),LL(0x795a0b08,0x979fd877), + LL(0x431f0b8e,0x5acef24d),LL(0xd13cafef,0x9f1c4a80),LL(0x4659f447,0xf19ac70b),LL(0x10f561ae,0x82bab6b6), LL(0xbbc879a7,0x1268e7f3),LL(0x79c37165,0x7e7d7141),LL(0x597e313f,0x491f049d),LL(0xecc98736,0x6ca7e73e), + LL(0x80a31eb9,0xd7712aa4),LL(0x2d8b99d7,0xbf7376ca),LL(0x2b8e5f7b,0xc1166cdc),LL(0xf1a48c9b,0x562bf290), LL(0x31c38c75,0xa6e72238),LL(0xb5f42def,0x51a9a100),LL(0xa100b75f,0xa0931d81),LL(0x967830be,0x7022479d), + LL(0xc192bc29,0x53eaaa1f),LL(0x4123a9f2,0x09504e7a),LL(0x90671997,0xe897397f),LL(0x4294fda2,0xc56185d2), LL(0x9819b185,0xb531f278),LL(0xe9dda4cc,0x390155ff),LL(0x14d26bf9,0x1258a5d9),LL(0x7e5f13a1,0x47d8f5ae), + LL(0xe9591945,0xef9e05e3),LL(0x846441be,0x92d20e07),LL(0x1d897ad1,0x28cc70ef),LL(0x0bac861f,0xee962e74), LL(0xbed368e4,0x9b7a4236),LL(0xe49b9720,0xe65ac22a),LL(0x22c1bd82,0x851f0032),LL(0x1e75ab15,0x771573ec), + LL(0x702eb71a,0x2e0a4635),LL(0xee924cd9,0x65167c74),LL(0x10ccabb5,0xe16b3515),LL(0x10ea5907,0x63cf15c4), LL(0x616f5290,0x59dacdc6),LL(0x8e372a43,0x19eb409b),LL(0xe3c36beb,0x5c879337),LL(0x0555fa1e,0x5841e7e2), + LL(0xf346ec2d,0xce197347),LL(0x221db43d,0xe14818a8),LL(0xc935c025,0x1bf37115),LL(0xfee14ce0,0xb22bdb03), LL(0xf0e3dfd6,0x893c5efa),LL(0xb1f95e1e,0x8fe9d56c),LL(0x407e4db0,0x6ee580e5),LL(0x9fb33911,0x0292bfc4), + LL(0xb2869cac,0x598ce787),LL(0x4798690a,0xd2f77720),LL(0x9cb6015a,0x2689e0f3),LL(0x7127b0e8,0x8831ddeb), LL(0x5c4fd58c,0x44db087b),LL(0xcda33285,0x04a0c68e),LL(0xe1a4c364,0xe55f00d7),LL(0xb6393b21,0xb3a2ea9a), + LL(0x5e9d8db9,0x99ef4da3),LL(0x17b484bc,0xa01f11d9),LL(0x1767f6ca,0xc2283fbf),LL(0x9e77f389,0xbb524479), LL(0x10149726,0xc4ea3c66),LL(0x82ec5f64,0x4b71d644),LL(0x642944c7,0x0fe49d52),LL(0x5a17a7bd,0x69fef689), + LL(0x8c3dce23,0x2f3588fc),LL(0x68e0c237,0x9d429231),LL(0x86fa61d2,0x63756076),LL(0x729bc192,0x1d89c6b8), LL(0x00d3ffd1,0x85e098d2),LL(0xde6f9109,0x5bf292c2),LL(0x3e7b8f23,0xb20dc994),LL(0x87c6eb54,0xcbe51bad), + LL(0x0517b914,0x263fd862),LL(0x225499a7,0x447624ad),LL(0x71f807d9,0xfbb831bb),LL(0x2fe2e021,0x9514fe38), LL(0x52418e9a,0x881e8763),LL(0xf1d9b43b,0x268e4655),LL(0x1f780af9,0xf917044a),LL(0x3d758ba5,0x3727b2d9), + LL(0x68755cf3,0x8487eb90),LL(0x7fe12541,0x1887394e),LL(0x46af8ca8,0x2e4c65d4),LL(0xb9e119dc,0x72aae645), LL(0x1ec6ad73,0x958e0094),LL(0x8ce4573e,0x84a7eec4),LL(0xf9254b96,0x3d6d00d4),LL(0x8e421732,0x4ef44f58), +}, +/* digit=7 base_pwr=2^49 */ +{ + LL(0x7d3ad2ac,0xf59de0f8),LL(0xc0f92c5c,0xd2670cb1),LL(0xc900b6a5,0x8f05944a),LL(0x8d9668eb,0x11aeed23), LL(0x7c488ea6,0x21b038e4),LL(0x78083d75,0x406ea3f7),LL(0x3bd31fe2,0xd22197b4),LL(0x28a6ef9a,0xdc8f8ccb), + LL(0x02887281,0x679a6483),LL(0x7f9de66f,0x13f3d39b),LL(0xf1a7dee2,0x289c3c50),LL(0x40b698b8,0xa510a53c), LL(0x06f799ad,0xc566c3fb),LL(0xb5374650,0xcc95a879),LL(0x61c947b2,0xbd7343c0),LL(0x9543e51b,0xbbbff69d), + LL(0xba75aba9,0xb80d38dc),LL(0xd150f881,0xe9b61ac6),LL(0xca7b47c5,0x9f56af52),LL(0x77fb3128,0x040300d9), LL(0xc01fd0c1,0x36877184),LL(0x8b6e134b,0x40112a04),LL(0xccd71653,0x56daed90),LL(0xb74bd62e,0xec553aa6), + LL(0x11476dde,0x0398381d),LL(0x1ea0923f,0x4959204d),LL(0x017745bd,0xd67427ad),LL(0x6935e467,0xef022a74), LL(0x24e0380a,0x57e799f5),LL(0xb5f1a730,0x6ee2b64f),LL(0x521771d8,0x9aeaac48),LL(0x0992a13c,0x02c8521c), + LL(0x48f6934a,0x25dd9f41),LL(0x3e38e31f,0x8d191988),LL(0x32899837,0x35548844),LL(0xf56b07d6,0xf87c696b), LL(0x73e927b9,0xbc66e7d7),LL(0xbb0bedc5,0x04cdac77),LL(0x0bcd022a,0x1e6f2903),LL(0x22c574b5,0xafa637be), + LL(0x55c1759b,0xcdca4b1e),LL(0xa6819d39,0x3d46ee3b),LL(0xb4b0fce9,0xf7497ade),LL(0xdcb613c3,0x54aef506), LL(0x522ff464,0xbc11d721),LL(0x37bd3c51,0xf53f16f2),LL(0x485695b9,0x88f29955),LL(0xdac00fe3,0x428ce742), + LL(0xa3520c27,0xd971fbd2),LL(0xe05e1b9d,0x2204fe54),LL(0xf0c15c89,0xb08be507),LL(0x901a15c3,0xfeeda919), LL(0x84b60eb5,0x6576ad3b),LL(0x59e951da,0x40d4b9a1),LL(0x76244608,0xbe96e1b8),LL(0x58ef9f37,0x3af35ec9), + LL(0x153adbf7,0xbfe5c43b),LL(0x80351fec,0x07a66edf),LL(0x3b109e60,0x3d804235),LL(0xa832c162,0x4dc97176), LL(0xb1db1e5c,0x03fec75f),LL(0xa15b9900,0x6aa02da6),LL(0x4faa1cff,0x5f9e808f),LL(0xa6412a26,0x90aa28bd), + LL(0xe041d453,0x2fb2c15b),LL(0x86c18ef0,0x2b847efa),LL(0x115b7318,0x84f5ee9d),LL(0x71699936,0xd568b080), LL(0x6ea5c852,0x34658ae7),LL(0x126d10ce,0x99f918b3),LL(0x09b9407f,0x75e3d9cc),LL(0x7883f978,0x11c6a0bb), + LL(0x876db0fd,0x522a6925),LL(0x7a9a4211,0xc5401ca1),LL(0x6789e755,0x89163b57),LL(0x0fd6f1b8,0xd2b2c99a), LL(0xa7b452dc,0x427eea22),LL(0xef9db65c,0xce8e6682),LL(0xda9c80f7,0xfd835810),LL(0xea916411,0xdb91bfbb), + LL(0x798b5051,0x7a5aefad),LL(0x42a0d4cd,0xbd7ebc88),LL(0xbac28520,0x958e327a),LL(0x7d010c77,0xfa8bf6d4), LL(0x579752f4,0x8a7040aa),LL(0xe573d895,0x47974d84),LL(0xfe16f77b,0xfd2a0cdc),LL(0xdbf71fdc,0x0f08f86a), + LL(0x2983bd4d,0xb19de6f1),LL(0x1e3a28b3,0xb990931e),LL(0x00cbc4f9,0x43b71b8b),LL(0xf4d75a0e,0x35d1ddd0), LL(0xc653f111,0xc211951d),LL(0x88750928,0xbbc46824),LL(0x174803e3,0x0cf6e752),LL(0x8960d80c,0x81f5e0ac), + LL(0x0c52fcf9,0xe03ca085),LL(0x0865ced4,0xa795382e),LL(0xe7117376,0x03bd561c),LL(0x3fd7184a,0x8608dde1), LL(0xa2a98acc,0xfd48fd50),LL(0x11df74b0,0x902fa587),LL(0xfa73b8f1,0x683f101d),LL(0xe7c0efa6,0xc805d31b), + LL(0x5b11d6c0,0xe5effb4e),LL(0x01c8374c,0xba30f747),LL(0x0c275aec,0x8733511b),LL(0x97354e36,0xf140b740), LL(0x0341268e,0xb01ded69),LL(0x27eac17b,0x17bc3176),LL(0x8984992c,0x88097703),LL(0x3e05061c,0x37bfafab), + LL(0x111d0eb8,0x7eca9f09),LL(0x8f243481,0xda7eb023),LL(0x59b5e491,0xac3cb2d6),LL(0x4f794842,0x56e725b1), LL(0x45b2dff6,0x43245254),LL(0xef10ec78,0xeafe73b9),LL(0x78819dbf,0x0d3cb2bc),LL(0xe784eb22,0xff1cd617), + LL(0x9ce0fcd6,0x0dbaf1c9),LL(0x3232a847,0x732ea65e),LL(0x6a75d822,0xdb2ce218),LL(0x3d2273ca,0x88ffd479), LL(0xf2f26b61,0x89092ad2),LL(0x686706ab,0xfb7041bd),LL(0x5e23597c,0xe3d5fa75),LL(0x5995fc84,0xa2035bf8), + LL(0x4514b8bd,0x1feecd2c),LL(0x434b9233,0x57cb78b4),LL(0x24215322,0x59bd2ad7),LL(0x1ce8daa0,0x41437de2), LL(0x7147ce80,0x401bbece),LL(0x5abb61e8,0x5e462137),LL(0x5a3790eb,0xbbf21033),LL(0x5134dee3,0x9a791c09), + LL(0xcedd2cc1,0xc8ded766),LL(0x6447b925,0xa3e48e9d),LL(0x69efa728,0xc73282a3),LL(0x8d408bec,0x8cb72c30), LL(0x41cf92eb,0xfb4f2797),LL(0x26f2412e,0xef3f42a0),LL(0xa941ab5a,0xdbc0f972),LL(0x98337613,0xc7bd62dd), + LL(0x4e45dcbd,0x31892744),LL(0xb51b7f91,0x3b2979cb),LL(0x29b27fec,0x41e002f5),LL(0x4dd51b0f,0x9007ee68), LL(0x6e23d565,0x82f417a3),LL(0x77127820,0x3321f343),LL(0x199b32be,0x8d09d965),LL(0x5bc2017b,0x948429eb), + LL(0x124eb248,0x22b639f9),LL(0x125f8c22,0xed097f74),LL(0x5f8bed34,0xdbc02517),LL(0x51aa29c3,0xb93f5b42), LL(0xc7368c44,0x6fedd599),LL(0x2c772a9a,0x99a5a795),LL(0x7a5f156e,0x30b35ba7),LL(0x191c45af,0x9dc50978), + LL(0xb5b4c4fe,0xe8d241f5),LL(0xb75f54f8,0xda89eac1),LL(0x9ef86ae5,0xb399dba0),LL(0x51c1b8c6,0x2337bb46), LL(0x4d02f348,0xfe60b0c5),LL(0x0afc6cd0,0x709f1235),LL(0xb40fce18,0x8a0b458b),LL(0xe3929cfa,0xefe143aa), + LL(0x0ca6cec0,0xab3a4b0d),LL(0x67246ec3,0xcb235374),LL(0x1ec2538f,0xdf9b0e89),LL(0x80c7b53d,0x3ec2ea13), LL(0xd0ae3146,0x920c55f2),LL(0x43946090,0xd3ac4e1e),LL(0x97ebe7a4,0xeba72583),LL(0x393d401f,0x5031644a), + LL(0x9714de1a,0x802c3409),LL(0xde5bacba,0xc62d66d0),LL(0x903b8148,0xb6c2abeb),LL(0x5bffe1c4,0x203531ef), LL(0xe862ead7,0x186266de),LL(0x1a23bebb,0x21e643d5),LL(0x6edda603,0x15c13d11),LL(0xb1bebc77,0x39b4a3a3), + LL(0xb9ac4754,0xdb456c1a),LL(0x4d3f305a,0xf497e6e9),LL(0x3fa62dc0,0x84d27e3a),LL(0x2524b94f,0xc18c3569), LL(0xe380f5cb,0x92198954),LL(0x272ea458,0x81d8221c),LL(0x5f328491,0x6fa082f6),LL(0x8e304ccf,0x810ca5af), + LL(0x0d76e6d5,0xda9f1c15),LL(0xb7abad72,0x4bd38afc),LL(0x08aa20f5,0x14b5cc26),LL(0x81061318,0x010a1af8), LL(0x03c287c0,0xaf9d7a73),LL(0xbc4d40ab,0x9ba5105a),LL(0xb07937a6,0x99e4b824),LL(0xc869f63c,0x026d294c), + LL(0xc910b521,0xaaebde75),LL(0xa7d5dd9c,0xc803ded4),LL(0x62764be2,0xc8b713b0),LL(0x92540cf3,0x5ea9ea2b), LL(0x6930bd0e,0xbaa999c6),LL(0x1f4b254c,0x57052e53),LL(0x2b0b27ee,0xfb9fd699),LL(0x4cc83793,0x86b70932), + LL(0xfba59bbe,0x09ab4dd7),LL(0x04f4609e,0x83204fee),LL(0x93934977,0x251cb390),LL(0x647c13e8,0x8add9e8b), LL(0xe7ea7006,0x444815d3),LL(0xbd032c32,0x22333c0a),LL(0x4058b7cb,0xe7728dc8),LL(0xd1bc061f,0xde8eb503), + LL(0x493d76c2,0x5d3ece2e),LL(0xd804954a,0xa425f3ae),LL(0xeac95eb8,0x49100271),LL(0x38b4be56,0x94e4dfa0), LL(0x650f9930,0xa855893f),LL(0x50264765,0x1fa0a07d),LL(0x0d1d40be,0x37a3c169),LL(0x2eed2a0d,0xfedb51e4), + LL(0x1b3348b4,0xa6e0c2b2),LL(0xc414464e,0x9e361f42),LL(0x176e109a,0x3e14e2ee),LL(0xf4af92fd,0x5f1a6bbe), LL(0x84beb8e5,0xf15d4647),LL(0x97d36132,0xac3f01c1),LL(0x84ca42ae,0x36e669bf),LL(0xd9433ca1,0xf789bdbd), + LL(0xf71e84d4,0x384f37f4),LL(0x59d6481a,0x57de9473),LL(0xf5e6fa70,0xa9a81f99),LL(0x6cb57bf3,0x26f0a64f), LL(0x061d38fe,0xc07e1c13),LL(0x4a475732,0x6fae70e9),LL(0x840e595c,0x6cfb6b1d),LL(0x62848351,0xb23cf1f2), + LL(0x4fcf8743,0xef6094c7),LL(0x05fab119,0x7dc42218),LL(0x5c220d15,0x3207463f),LL(0x22c4bfb2,0xdf51b3f0), LL(0x1572735b,0x13db445b),LL(0x2f6db574,0xd7662537),LL(0x7796f888,0x692f1e05),LL(0x33f45593,0x9f3d7a5b), + LL(0x313de667,0xb5deb892),LL(0x66a478a8,0x75c872d7),LL(0xc4992428,0xb67b5513),LL(0xf70fde09,0xf97e010e), LL(0x60ee268c,0x49b0f053),LL(0xf67cd321,0x981b5141),LL(0x4fbc187c,0xb5a1ac8d),LL(0xc12e6da8,0x162417e2), + LL(0x62914938,0x07bb6fff),LL(0x19f44438,0xd385285b),LL(0xa28904dc,0x05a610a1),LL(0x5a29b9f8,0xd80a7099), LL(0xc177af4a,0x72ccb553),LL(0x5e3752f4,0xac0bd91b),LL(0x7ae838a0,0x8e8ae668),LL(0x1fdfe7c3,0xcaa5a46c), + LL(0x93d34156,0x2cc2c1a5),LL(0x61fe4572,0x22beffb1),LL(0xfcdc7418,0x66f9f3ce),LL(0x6af66892,0xbaccda41), LL(0x1af43f56,0x775c783d),LL(0x0ae04034,0x1b23b187),LL(0xe99b486b,0x5a9325f4),LL(0x8367ab05,0x36d5bfe9), + LL(0xa519d028,0x17d8d2fb),LL(0xbe00e7e0,0x27b6beb2),LL(0x15a3f03a,0x8d51c36c),LL(0x5faac8dd,0xbf90e78b), LL(0xb27ab69f,0x4e8c28e7),LL(0x4a553da4,0x37ecf0c7),LL(0x210fe720,0x3a39682f),LL(0x0b0cdd0c,0x60c62e80), + LL(0xa16647cd,0x893aa225),LL(0x64ce0455,0xcffb728e),LL(0xc4f0fe79,0x81891d39),LL(0xf9c39f00,0x1abe3073), LL(0xf961d05c,0x88336c27),LL(0xa5fc96df,0xc9033a88),LL(0x864b39f8,0x0d084405),LL(0x851e95c9,0x866aa904), + LL(0x98bae4a8,0x0c36da08),LL(0xb5feb202,0x9f88d799),LL(0x8054e4da,0xcd9aeb4a),LL(0x1e9134cb,0x005206bf), LL(0x17ee6649,0xd5f32bf8),LL(0x60847ad2,0x9431dcd8),LL(0x8a3e4050,0xbe6d62c7),LL(0xedf10d40,0x3ae68f7a), + LL(0x4604d71f,0xa95c9ea0),LL(0x415f8028,0x01aa3fea),LL(0x5a41970a,0x3dd55ca5),LL(0x0b5776b4,0x05978ad4), LL(0x787fe20c,0x7c9f5bdd),LL(0x75fdba0b,0x23b9fb7d),LL(0x5fcf3a0f,0xfb1a724d),LL(0x87817071,0xd63b3515), + LL(0x44e40138,0xecae282d),LL(0x87605748,0x8732df23),LL(0xd11188cb,0x0ef49da0),LL(0x51146cc0,0xc0478138), LL(0x46621921,0x4ba42323),LL(0x47dfa4eb,0x8836dd44),LL(0x8ec16442,0xdb6a0100),LL(0x9cdd2e87,0xabdd9b81), + LL(0x502e26d1,0x205ee262),LL(0x3294e240,0xb961ef9c),LL(0x6da7733d,0x7178f1fb),LL(0x232ecf73,0x989b69fb), LL(0x9a9bccae,0xb7278a35),LL(0x400a01f3,0xb1c81a0b),LL(0xa6b213ba,0x0781855a),LL(0x3429817e,0x8acc1b78), + LL(0xfb4e1aae,0x527e3a9f),LL(0x4c0b0f4c,0xc18c1cfd),LL(0x1fa7d9f0,0x0676c365),LL(0x4454cc7c,0x3314509f), LL(0xc7c48245,0xb0f45371),LL(0x695ef470,0x913fe759),LL(0xc8d3e0ad,0xbb676070),LL(0x902e1638,0x0db98fcc), + LL(0xfc4dfaa8,0x42874e9c),LL(0x7084b2cb,0xcbf89462),LL(0x8a846ab8,0xd6d46f77),LL(0x14c183b1,0x9e4506ca), LL(0xc53b7631,0xc2d0f9b7),LL(0x294d6c34,0xe47c3d8f),LL(0xc05d3f1c,0x04e3c868),LL(0xa5957fef,0xbacec4f3), + LL(0x3b77893e,0x4f4530ba),LL(0x69a18bd9,0x4c234f54),LL(0x5071f5e3,0xb45aadd8),LL(0xd1bd0b86,0x73e4160a), LL(0x1c474f64,0x43fcb30d),LL(0x617d1612,0xedef0769),LL(0x0eec330e,0x92076734),LL(0x5b0a21b5,0xd7767770), + LL(0x183e26f4,0x4b7dea31),LL(0xc9fd2e70,0x59d6ff20),LL(0xd5d914f5,0x7bdea00f),LL(0x56376640,0xc736dd0d), LL(0x38ae8300,0x593ae6ef),LL(0xdf0355bf,0xdafe49f1),LL(0x0db4491f,0x094ccd86),LL(0xfe4271ab,0x32295701), + LL(0x5db7014e,0x2b7690e4),LL(0xd7766bfb,0x1bbc9c36),LL(0x7d986d0c,0xc52249f0),LL(0x324f20ae,0xc7eec37b), LL(0x0e912329,0xd376afa3),LL(0x04268fa3,0xbc35e949),LL(0x9e91a4ac,0x617bf765),LL(0x1d483ecc,0xb1e932ed), + LL(0xac493266,0xd4e31672),LL(0xecdafb85,0x1c779fe2),LL(0x06280499,0xed09eb4a),LL(0xcd4e1f33,0x3dd8d965), LL(0xf34576dc,0x0fb4308d),LL(0x85781a43,0xa8ccbf5e),LL(0xce623a24,0x8dbf488a),LL(0x6118cfd4,0xb0e71d30), + LL(0x8cc9d957,0xfc68da68),LL(0x83815670,0x7e5e6b65),LL(0x3f185dfe,0x2c16f5ef),LL(0x98952b33,0x23a4098b), LL(0xd515f551,0x15a80298),LL(0xa7f8f341,0x71a2e7fc),LL(0x8cf4f7b6,0xed42b1b6),LL(0x1504d390,0x02743db2), + LL(0x3016e513,0x2bded3a8),LL(0xfb0f7bfb,0xa3c508af),LL(0xaa2be716,0xa6a490de),LL(0xf4485b9f,0x5a04d9e5), LL(0x6ad25b5d,0xd07b99d1),LL(0x65a72cb4,0xa1840109),LL(0x14c45a95,0xc8e2b32d),LL(0xe4f2ecff,0x0fae6e86), + LL(0xd94b6fe7,0xd09f454b),LL(0x23006b62,0xa776a633),LL(0xd332b4b9,0x6c700a1c),LL(0xce016225,0x50c3fb34), LL(0x8af71463,0x4b805bc3),LL(0x5f1fb3b7,0x049143e2),LL(0x5a6d1dd3,0xbcaf4b61),LL(0x4733abac,0x02093dd7), + LL(0xdf59f061,0x1a23c3f6),LL(0x80c4efb7,0x87a6c141),LL(0xd88e4363,0x47635ae4),LL(0xbf8d2072,0x75e2089f), LL(0xac83803b,0xa2bc1b27),LL(0xe2aafecf,0x8ae61522),LL(0xd0010193,0x4b459205),LL(0x9205f876,0x900f6a31), + LL(0xf808f044,0x49cddbc9),LL(0x95094ead,0x94637692),LL(0xb87c9bbf,0x3c9c7c0c),LL(0x4e1844d1,0x1699670a), LL(0xcbcf85c3,0xd8a978f2),LL(0x6a36e1c9,0x83e7b806),LL(0xfaff9c52,0x6f28a73f),LL(0xb71eaa80,0x51341222), + LL(0x9328a676,0x195461da),LL(0x21766180,0xefcc93e5),LL(0x771a5485,0xed82c930),LL(0x205a8bff,0x34f15ce0), LL(0xb8b3bfd8,0x88ab72cb),LL(0x8110fe55,0xbb59a5be),LL(0xc7d61a31,0x9ce8a082),LL(0x5b1c63d2,0xfe81d072), + LL(0xe9ff8421,0x9fae0be1),LL(0x967e13a6,0x4254f89d),LL(0x35da926f,0x1c094620),LL(0x4a76583d,0x84eda272), LL(0xe0e0ffb8,0xa4033064),LL(0xabc72d0c,0x47951945),LL(0xb72c32e7,0x0af6bb4c),LL(0xda797f9e,0x6c73357b), + LL(0x2ac2e99d,0xd7a726c9),LL(0xcd62e7cc,0xf44b4731),LL(0xe6225822,0xf89f8e29),LL(0x8d713d92,0xa44bb9b0), LL(0x9404f6c6,0x3291e8d3),LL(0x37bdb22d,0x50b7a4ff),LL(0x216a0f13,0xe008662e),LL(0xcf382547,0x150fa2d6), + LL(0x3138acbc,0xe5e47c55),LL(0x40d7f3db,0x595cf1e2),LL(0x2ee1949d,0x2872392d),LL(0x8a4fb721,0xdbd15bf8), LL(0x183351dc,0x30e78cdc),LL(0x6b294729,0xa39b8efb),LL(0xc7b553e8,0x0df4d23e),LL(0x659d3ffc,0x434f38fa), + LL(0x55a0c931,0x1764115e),LL(0xa5c920a4,0x34ea18b9),LL(0xaf903710,0x6a099ddc),LL(0xe49f2c7a,0x4b937dc1), LL(0x430f0a7e,0xacfc4a1a),LL(0x421dbe96,0x8f106a58),LL(0x1811d3fe,0x48ac7026),LL(0xb80f13c5,0x5484226a), + LL(0x8da7ca79,0xf692e17b),LL(0x718691b9,0x4827aaa2),LL(0x5c5ea68c,0x881f1c38),LL(0x88bdf643,0x1620f5d6), LL(0x0b9a5add,0xe5703cb2),LL(0xbe925061,0x392e6ea5),LL(0xb0bab0d5,0x2a66ce20),LL(0xf98e8dad,0x83a8e0e5), + LL(0xdeec2329,0x53532223),LL(0x346eea96,0x6a740238),LL(0x1dde2a6a,0xa54afbdf),LL(0xf2b5b190,0x0e6ca8c1), LL(0xf3cd4e46,0xcccaa3c6),LL(0x0eb7bb3c,0x168d66bd),LL(0x08d4f4e9,0xf1275144),LL(0x139811fc,0x2ae8c946), + LL(0xc870713a,0x4973c726),LL(0xba54b13f,0x298465ee),LL(0x940f224f,0x9f901403),LL(0xb9950a40,0x5cd6a07b), LL(0x069a8484,0x9d4095e6),LL(0xd4f8831f,0xe6bf3181),LL(0x39243da8,0x37ceb29a),LL(0x2693653c,0xb3887f31), + LL(0x42c98a56,0x685d2172),LL(0x3969dd9a,0x350fbab8),LL(0xe8ac84ec,0x728edca9),LL(0x59bbb0c4,0xf42beab3), LL(0x27d3c3fd,0x9793e746),LL(0xc732b37e,0xbf6016de),LL(0xdf0f248f,0x3688173a),LL(0x7ed59dfa,0x84fbd040), + LL(0xa6731b1b,0x2bad638f),LL(0xb7355979,0x1c7b4b13),LL(0xb8e77093,0xf21550e0),LL(0x53efc63c,0x14d0bc9d), LL(0xd56e1f29,0x119ae9fb),LL(0x4d60bc5a,0x3511309c),LL(0xe3574e43,0xec654f06),LL(0xbef6aea2,0x2d6695df), + LL(0x5d6abff7,0x27ece611),LL(0x640c9ab8,0xa706d42d),LL(0x5a6f8fa6,0x7a7252d9),LL(0x349aaf8c,0x32be7195), LL(0xff734e23,0xffb48a3d),LL(0x7d27b99c,0xa9b36c82),LL(0x0ccaedbc,0x85b7a57e),LL(0xc04f2394,0xb93b14fd), + LL(0x160700e0,0x3a3a78c5),LL(0x961e4af8,0xbd7ae60a),LL(0xd9472cd7,0xe1deb736),LL(0x3880bbbe,0x276b51b7), LL(0x1aa99bfb,0xcf0c4b9a),LL(0x689d7f58,0xaf949d5f),LL(0x65f33328,0x00878488),LL(0xe7d7b484,0x0f1a178c), + LL(0x849e6d32,0xd44550f8),LL(0xfe16485e,0xe7bc29d4),LL(0x2f343924,0x29bbfec6),LL(0x40f2b5ce,0xeeb802f2), LL(0xbbb64f33,0x2b337542),LL(0x9f9bdb3c,0x4c1d3a36),LL(0xc7a1cb88,0x1067cf3b),LL(0x4601fb6e,0x3f12a31d), +}, +/* digit=8 base_pwr=2^56 */ +{ + LL(0x1f8a4a91,0xb720a78f),LL(0x753dbe73,0x59e22211),LL(0xadd0991a,0x9f5ad99c),LL(0x7380726f,0x3a0db802), LL(0x7dfb4f1c,0x37f0761c),LL(0x5ac819cd,0x68e7098a),LL(0x37ffe348,0x9683d610),LL(0x2b7b5140,0x5bf205e5), + LL(0x61a97262,0x9846b5f6),LL(0x974a82f7,0xedf2cacb),LL(0xaf439654,0x3dfab85f),LL(0xc724ee09,0x43fb0ef9), LL(0x53b0119a,0xd0d5016f),LL(0x5bc8fc81,0x68445363),LL(0x1f65d298,0x6d10b649),LL(0x21a4e64f,0x0f3c88c6), + LL(0x7f34c517,0x320372a1),LL(0x2378bc27,0x5602bd16),LL(0x91aae024,0x666a592d),LL(0x317bbdaa,0x716886ab), LL(0xe3045103,0xce55fe68),LL(0x7de1d701,0xf2c4b0b2),LL(0x7d724cb6,0x8da35885),LL(0x9ec47314,0x9aac623c), + LL(0xb8529a01,0x824cff46),LL(0x4856b95c,0x6e4d82a2),LL(0xc65af7f7,0x58c6b833),LL(0xae110e53,0x8a6c4125), LL(0x4f083340,0x38207c30),LL(0x176cdb31,0x71aa384b),LL(0x42882de1,0x1ada2941),LL(0xc16a2e4a,0x38b1ad2e), + LL(0x142bcb30,0xbdda2720),LL(0xfaf604d1,0x56175263),LL(0xe6796314,0x086189c1),LL(0x5b04dd19,0xdab01c68), LL(0xba8ed3c1,0xce54e4b0),LL(0xe281acfb,0xf616513b),LL(0x5e0a6319,0xaf179629),LL(0x328b587b,0x85e79ac9), + LL(0xc9fd7da0,0x11d84588),LL(0x1238d0c4,0xa78682d0),LL(0x829d6475,0x333ddde0),LL(0x69de9e18,0x80c88440), LL(0xc6d8176f,0x5d15f21a),LL(0xa509d470,0xdaff9434),LL(0x8bbbfcd5,0x0191bb0a),LL(0x08fc2688,0xff7732b8), + LL(0x5ab3d89e,0x02fe772d),LL(0x9a786c91,0xf1580ec9),LL(0x5a323866,0x8fd83417),LL(0xbadec96f,0x93711d49), LL(0x6b9b4a30,0x2020c34a),LL(0xb8b0de24,0xbf10e000),LL(0x28de3ce5,0x2a5f298d),LL(0xfe1a1c63,0x807a398e), + LL(0x73f7c45c,0x9fb640cd),LL(0x0afe059c,0xeb1f87ad),LL(0x52b168d4,0xa3c3979a),LL(0x7b1e403f,0x6eef460c), LL(0x2724bb3f,0x6d943e50),LL(0xf9d922d1,0x53f3f1bb),LL(0xcd538b4a,0x547e7a03),LL(0xd2c4145f,0x37631e20), + LL(0xb1f810bf,0xe7e49922),LL(0xf2645825,0xacafdb0f),LL(0x15f35bda,0x0f22216a),LL(0xd85bd0b7,0x6f2b4d95), LL(0xbedc9ecd,0x2f203db8),LL(0xb91e090d,0x26639ff6),LL(0x3486eb84,0x94cd6596),LL(0x42c05747,0x32747db3), + LL(0xcebfa9f1,0xcd3e7a52),LL(0xfb2b3007,0x5e792d76),LL(0xb9ecce81,0x9669523d),LL(0x04f191e1,0x9263cc85), LL(0x69655fe1,0x192019c0),LL(0x4d984e59,0x1c5cc5eb),LL(0xdf33f336,0x9ad10ed6),LL(0x41d94897,0x0ca48387), + LL(0xf222476c,0xbd1ddf67),LL(0x12d6dc4d,0xb4ad7126),LL(0x93ed702a,0x5c327b18),LL(0xfa70cd9f,0x7e3a27b1), LL(0xc0c4f415,0xdca750bd),LL(0x213a5d61,0x98197c90),LL(0x6f10fcc7,0x9bbd014a),LL(0x2ceed4fb,0xb06061e1), + LL(0xa8ad25f9,0xaf6dbbe2),LL(0x7ade697d,0xe70e9f40),LL(0x6eb872d7,0xb829e016),LL(0x1b04173f,0xc330e15c), LL(0x0d4763d8,0xd4868e29),LL(0x4c18c9fb,0x37867f72),LL(0x28019486,0x5fd2f47f),LL(0xb16e9bdd,0xe6bfdf81), + LL(0x783e43c5,0xace2a977),LL(0x76eed46a,0xe1791288),LL(0xd1767739,0x3884a5b2),LL(0x427c50a3,0x14eddddb), LL(0x1c9b1fcc,0xbeeed5ac),LL(0x4ecdb47a,0x50b1cb44),LL(0x0dcb78d5,0xcbf69555),LL(0xf2b17a99,0xe60bf9c7), + LL(0x9e9ade95,0x0edae6b0),LL(0xcb78b1e1,0xb5c6e13d),LL(0x1c257848,0x32860fba),LL(0xef7f5080,0xfc9aa9f4), LL(0x32aac870,0xccef8508),LL(0xfb5310a0,0x4b237704),LL(0xfeebb972,0x4c3cf970),LL(0x763d5f67,0x5dd3c7a0), + LL(0xccbf29c6,0xa656797e),LL(0x5a76a56b,0x6d77f211),LL(0x0e3daff3,0xc627156b),LL(0x7646fb1c,0xa4bd37f5), LL(0xa8cd3e5a,0x5fd7e286),LL(0x2f5fed51,0x3889951a),LL(0xe48c49be,0xf8186fc5),LL(0xc662ee38,0x0d3d308a), + LL(0x970e164d,0xb7c9bf06),LL(0xbd3d3087,0xc27a88d8),LL(0xf4e7c899,0x8a37c9cd),LL(0xab411371,0x18494d5a), LL(0xd9d8b29c,0x06532375),LL(0x915a2f74,0xb92dd45c),LL(0x515acb02,0x8a23f6bf),LL(0x435bfa89,0x0e69248c), + LL(0x6866c5e4,0x8bf41ec3),LL(0x0999159d,0xf059e652),LL(0xd29d7cd8,0xf906838f),LL(0x3a269735,0xc30100f6), LL(0x6280e70b,0xb7742bc8),LL(0x867b54e1,0x0067d971),LL(0xf544622a,0xafe9032b),LL(0x118a2042,0x6b441e39), + LL(0xcdd66b70,0x905c8655),LL(0xc1e2110d,0xe88cce1b),LL(0xee674093,0x8cc23c0c),LL(0xb2ea3fc3,0x55ded4d9), LL(0xb58dfbed,0xdd14502b),LL(0x49f698f8,0x523a4dd9),LL(0x01c83e5a,0xf843a501),LL(0xfe71ee1e,0xf11fd4c1), + LL(0x162d7c0b,0xeedd7229),LL(0x4ccad713,0xd42d6a9e),LL(0x2b0c7b93,0xa082fffd),LL(0x2a5016b9,0xee3abd48), LL(0xc117e22b,0x5079c95f),LL(0x814b8666,0x5d4b9169),LL(0x9bf90a6d,0x9e0f5e87),LL(0x744bf7ab,0x4346bd29), + LL(0xbfb551b6,0x4d85af0e),LL(0x31f7a958,0xb48e3da8),LL(0x6f5bc50d,0x3b474ce6),LL(0xe7c8dced,0x9fdb47bc), LL(0x53003272,0x2064450e),LL(0x839e69da,0x6bb230f3),LL(0x4d822be5,0xb6941512),LL(0xf11a9dc1,0xb51bc6aa), + LL(0xb23047dc,0x866447f8),LL(0xe5f52c2d,0xe02dbd63),LL(0x02770a76,0xe6ea43cb),LL(0x56fa6c25,0x853f5fe3), LL(0x960de6d5,0xfe9615f0),LL(0xf4b1b945,0x37c8b0c8),LL(0x4618629d,0xa6e83805),LL(0x23a2ac61,0x38fb5264), + LL(0x01751c20,0x5dfd7005),LL(0xce72773a,0x7e100245),LL(0x0776794a,0xdf09f92a),LL(0x1b730fdc,0xc4a8de81), LL(0xf0c7b031,0x72c302ab),LL(0x1283913b,0xdddff68e),LL(0xe32517b5,0x24889098),LL(0x856a2934,0x2483a0f5), + LL(0xa1c3d56d,0xdf6d7dcc),LL(0x09afb797,0x07f9c00b),LL(0x083d9557,0xe90da23d),LL(0xcbc03826,0x80ae6e53), LL(0x7c0e1b23,0x1fd6ff6d),LL(0xb1100226,0x1e90f3c8),LL(0x05a24e23,0xf179e00e),LL(0x946f16bd,0xe5361efe), + LL(0x4c662091,0x50f12e4a),LL(0x28608585,0xdad2c7a3),LL(0xf7429473,0x55c66749),LL(0x045ea1b4,0x440b77de), LL(0x91229927,0x9f707b49),LL(0xc6725715,0x3501e29e),LL(0x1225a8e6,0x5626fabb),LL(0x9507e709,0x270a9c2b), + LL(0xbdcb9039,0xe0d629da),LL(0x20255b7c,0xb4d7cd22),LL(0x5ed874a6,0x10c8614b),LL(0x4e67d406,0x36891e70), LL(0x1dce66fe,0x020da834),LL(0xabd64dea,0xae69e1e7),LL(0xcc71b37b,0x9cf153a1),LL(0x44771c7e,0xa6e9d024), + LL(0x8840fc17,0xb15e31c7),LL(0x349124a4,0x57853112),LL(0xbac542ee,0x78a9d807),LL(0x38fe1188,0xe7b4d812), LL(0xb3a3b801,0x874adc70),LL(0x4694cec2,0x80c0e02a),LL(0xe97805e1,0xd05c8c0e),LL(0x89d8cd40,0x8eaebceb), + LL(0x378d055f,0x888c777b),LL(0xb104a132,0x6956795e),LL(0xbe8472d7,0xe4bce719),LL(0x5f51729e,0x23c9f0bf), LL(0x36a3bf3e,0xfe7f7e19),LL(0x20a32d37,0xf8f5d2ca),LL(0x93b8a344,0xf383b467),LL(0x27a6e2c5,0x7eab76f5), + LL(0x93b54bc1,0x86c31b0e),LL(0xfc4ecab2,0xb9405ea9),LL(0xa0f6d341,0x09485578),LL(0x4b77e8e7,0x88053bb8), LL(0x29a07ddd,0xcde9b777),LL(0x97649102,0xec8ea63f),LL(0xc516777a,0xf74d082a),LL(0xbacf0dd3,0xf4e26d89), + LL(0xd0b3b578,0x6a919da8),LL(0xa0b5f7d8,0x0bcc3b29),LL(0x9e55924b,0xbf4565e5),LL(0x7889dbb6,0x13b36187), LL(0x533981bd,0xad0e59c6),LL(0x0bd0cb7a,0xea941b62),LL(0xa9e7aa7c,0xe5e35e9a),LL(0x088bfd7d,0x27f61727), + LL(0x8b3c7fbc,0xda2a5a20),LL(0xba55cb48,0x33cdd403),LL(0x90e7ff36,0xb72b51cf),LL(0x6f215840,0x8cc4b553), LL(0xd2671224,0xf7b80ad9),LL(0x6a45436b,0x560b4387),LL(0xff9e8fae,0xdca90694),LL(0xf97aa84e,0x2e7e9546), + LL(0xf37cd717,0x71e9ff45),LL(0x0d73e98f,0x6edf335e),LL(0x9f715170,0xf355690c),LL(0x3f5a82bd,0xf74df40b), LL(0x95e5b105,0x28b6d931),LL(0x2841a54c,0x8827f47c),LL(0x62b4312d,0x159cb943),LL(0x8db37edb,0x277943d7), + LL(0x6113a9f8,0x561454fd),LL(0xe70e67e6,0x78ebe733),LL(0x903f2feb,0x8764360b),LL(0x97902f36,0x2ba3b3d8), LL(0x87490b8a,0x28808cef),LL(0xf05f31b3,0xb1175954),LL(0x6c9b4f4d,0xbd5d6005),LL(0xdd254e60,0x12b13fca), + LL(0x14959566,0x38d4e812),LL(0x36fe9a6c,0xe253b750),LL(0x809450c1,0x24b2c81a),LL(0x8fec36b1,0x0aa89966), LL(0x053e97e7,0x9a99deb5),LL(0xe31d3a6e,0x5e57321c),LL(0x8dbe78a2,0xcd7a4f33),LL(0x3299e070,0x9f809d4f), + LL(0xa26a9eca,0xd6de8cfa),LL(0xa158a735,0x33d5705b),LL(0xc2293743,0x08dd3fcc),LL(0x68bbbaea,0x1f8d0a46), LL(0x61bc4105,0x53ff76f9),LL(0x7c4a8fc9,0x6445e88d),LL(0xc285d0e6,0xfd9a8d04),LL(0xfe62b449,0xf08d0d6b), + LL(0xc062810c,0x08c27292),LL(0x6663fa28,0x955629f6),LL(0x9d86fee8,0xbaf96c0e),LL(0x46bb9894,0x1dbc5406), LL(0x93dd45c7,0x8d6b6207),LL(0x3ee989fc,0xaf3baef6),LL(0x59b7b2f7,0xf66cfdb1),LL(0xda16c637,0x287fc2bf), + LL(0x2d71f863,0xa44ca8fa),LL(0x84d5dee5,0xa1161962),LL(0x3957b610,0x5a5c8ce3),LL(0x17f50b57,0xdbb32253), LL(0x76056358,0xc6a4eb7d),LL(0xc359d90f,0xff9eb424),LL(0xa88cb38c,0xdf4afe23),LL(0xa696b75d,0x2ae727cb), + LL(0xd20a58c8,0x47cc63ef),LL(0xc492ab36,0xd319dc3a),LL(0x36c7f76e,0x887a7d83),LL(0xfcd4cf49,0x65ed5e3e), LL(0xda301d39,0x0e6f2f34),LL(0x38ad4533,0xf2f7c102),LL(0xae834622,0x8a3a003b),LL(0xa060a0d4,0x94084169), + LL(0x13c8a1eb,0xb673168b),LL(0x459f8da1,0x80109609),LL(0x5c82007b,0x68003fa1),LL(0x248e0430,0x9f634159), LL(0xfb9b6510,0x188156ab),LL(0xe62844de,0xc35be1cc),LL(0xb0c84d39,0x21e8f908),LL(0xdad3ae53,0xa886d3eb), + LL(0x82b0f5fd,0x9e20cd56),LL(0xc465c721,0xc0c12f0b),LL(0x6f913a6e,0xfeeb1051),LL(0xaa32d6fe,0x9e7c76b9), LL(0xb8637b5f,0x820b49a0),LL(0xf4abccf0,0xe9ae172a),LL(0xfb270e67,0xccc050b1),LL(0x2269d1de,0x0b51d7e3), + LL(0x678c8d8b,0xca772ec1),LL(0x77ae7c7b,0x74eea3f8),LL(0x1e1bcbd3,0x51550df1),LL(0x3458b249,0xa931c17c), LL(0xf204aed5,0x192c3a45),LL(0xc993c881,0x93abf63d),LL(0x83421891,0xc60aa2cb),LL(0xf6b70284,0x11ce6735), + LL(0x69e152e4,0x53e8a3ee),LL(0x0033da23,0x6889ece0),LL(0x7d585418,0xada56904),LL(0xf5e5abb9,0xaf81a877), LL(0xdf515727,0x36e0267d),LL(0x3daad2a9,0xe04b532d),LL(0x1a11ced6,0x290e3ee7),LL(0x65e7a651,0x5be7c429), + LL(0x8ef9b498,0xc0662cd3),LL(0x6c4dcbf9,0x0ec5fbf0),LL(0xce4d7e3a,0x26694c70),LL(0xfa52de99,0xc1699a93), LL(0x6dae3e97,0x2e0d394b),LL(0x4c66e572,0xe3af28cf),LL(0xba1e27e4,0x9caf7bf8),LL(0xd5a4bdaa,0xd5c39337), + LL(0x9ec8ad6d,0xbb5d9551),LL(0x609fc2e1,0xfb3bc1f1),LL(0x95fe12b5,0x0d95ad2a),LL(0x5341dc74,0xf6fd6e89), LL(0x7537b803,0x1532991e),LL(0xeaf96f9c,0x77772fd3),LL(0xf832749a,0x4ed09840),LL(0x95f19d25,0x69a194ce), + LL(0x041cc340,0x5464471a),LL(0x1c442289,0x26f7e550),LL(0xb5ce9706,0x38f2c20e),LL(0x8a44efd3,0xcf73f8f2), LL(0x586e8f77,0x5176eda5),LL(0x63ece447,0x47e33844),LL(0x86b00be2,0x83826e8f),LL(0x539807b7,0x49cffcdb), + LL(0x414d3fb1,0x543d1fad),LL(0x38b1ef44,0xd56aac6a),LL(0x96c89050,0x9980bb64),LL(0xb169b8a9,0xc300cb46), LL(0x83413df4,0x5ab01a6b),LL(0xf3c91eda,0x179b8922),LL(0x43cccc06,0x4060b943),LL(0x9458ec1e,0x4f6adeb5), + LL(0xe339e40e,0x0a4c6437),LL(0x02aefe83,0x9cb6c532),LL(0x23dce7ea,0xb072d02b),LL(0x59a9032f,0x2cd7b117), LL(0x81dbfaef,0x01220cea),LL(0x0905332d,0xffe0026c),LL(0x0197adff,0x95ec2cb2),LL(0x4c3d0e49,0x853bf6f5), + LL(0x25d78f7c,0x04ed54fb),LL(0xbb68cb9f,0x45aae3e1),LL(0xe32d7421,0xf4f1a2c6),LL(0x45a05771,0x646ade65), LL(0x91eab45e,0xab241cfa),LL(0x7b214af0,0xb1cf204c),LL(0x851d311c,0x92dfb3e3),LL(0x144ae0da,0x56479ffb), + LL(0x9a7a4ede,0xbf847444),LL(0xf5cfd20f,0xb26b1f15),LL(0x83b33b64,0xf380ed7d),LL(0x3d1998c9,0xa21f9564), LL(0xa720e347,0xd985c7d3),LL(0x8bdf09d5,0x98078974),LL(0xce947692,0xa1f34ce2),LL(0xf419c385,0xf69e6144), + LL(0x8c3adcc6,0xe1926526),LL(0x42746263,0x848974fb),LL(0x97791569,0xa731261f),LL(0x065b721b,0xfed39da2), LL(0x836a7e20,0x8369b04c),LL(0x53c19f62,0x5758a761),LL(0x0ebea868,0x45746383),LL(0x3b7d71a8,0x20179927), + LL(0x57632243,0xb466ed4f),LL(0x120577c9,0xc8d918cb),LL(0xeda40e9c,0xbab307e5),LL(0xd5f65d1b,0xe6dbc7d4), LL(0x60619e10,0xcae0c649),LL(0x6b0df67c,0xffddf6d1),LL(0xb32ee5d1,0x60488755),LL(0x47164a55,0xcb278aaf), + LL(0x0bfb732d,0x354c3392),LL(0x649bc125,0xcd4fc821),LL(0x770ffdb8,0xa8e1253f),LL(0x0ff0c37e,0xf7eec595), LL(0x7149b102,0xe5a65279),LL(0xd0528224,0x1cbbb56b),LL(0xb51c5df4,0x40b1a8d9),LL(0x39e1ca25,0xccb43d26), + LL(0xfdcfe8c5,0x48f74dc2),LL(0xfa5b8daf,0x3ccb31b6),LL(0x7de6300f,0x6f8dc5bc),LL(0xf247bc0b,0x2a373fd3), LL(0x17825306,0xefe13539),LL(0xc50c47b4,0xeb253484),LL(0x3c739f02,0x4a7f2af3),LL(0x9a3c6746,0x3a3eb385), + LL(0x588978e2,0xa90afa2a),LL(0x8d80894f,0x501fcebf),LL(0x6bf1a4cb,0x1de1d06d),LL(0x6cc42a07,0xb0f4a61d), LL(0x78d406f0,0x975cb8de),LL(0xe3d293e3,0x560b0d7b),LL(0x32e686ca,0x5746227c),LL(0x3fcb0205,0xd12854f5), + LL(0x499512e3,0x8c0eaba8),LL(0xade99108,0x8d97c229),LL(0xff2b5782,0xd80da38e),LL(0xaef08107,0xf8c30ba1), LL(0x076b97c3,0x9068d7d0),LL(0xb1b7eba5,0x851d1cb9),LL(0x318e4675,0x02bb728c),LL(0x76ddc683,0x0efe9707), + LL(0x6a248b04,0x6985d358),LL(0xf8969ed1,0x75eb6019),LL(0x606a0c64,0xecb66a20),LL(0xfe39b5e5,0xd1252f64), LL(0x2aa222a9,0x93d5d61c),LL(0x1ffff8ec,0x16c0d6f9),LL(0x5dfab0fe,0x0f1f962d),LL(0xcedcccb0,0x88776fe1), + LL(0xa32cbff1,0x410333c6),LL(0x093bcbda,0xca13ce28),LL(0x0e479259,0xd97b0684),LL(0xbf505c93,0x8b2b3ad8), LL(0x71761412,0x42092d64),LL(0x918acf33,0x9d0c842d),LL(0xac9ade57,0x904d3add),LL(0xe0d5ef6a,0x025e4177), + LL(0x0b33d4ed,0xce406ec0),LL(0x57b5c958,0xf73ac4da),LL(0x6ef70849,0x5f96cb8c),LL(0x77b32d5d,0x702ccc6f), LL(0xcea6885c,0x75bda8d8),LL(0xc0c0432e,0xbfc3e62e),LL(0x54631c9a,0x46db9cc6),LL(0xba1d1550,0x1669075b), + LL(0x2d227656,0x5ccc4e34),LL(0x02cb0644,0x0724e41b),LL(0x435601fb,0xc5e2077d),LL(0x68d6aee2,0x356155c5), LL(0xfde58906,0x0ea00013),LL(0x37a9eda4,0x79fa13c3),LL(0x8f51a6a6,0x7d09479d),LL(0x1f979fed,0x86e955b7), + LL(0xe39ab804,0x9cb22960),LL(0x03535a39,0x6aeae783),LL(0xb9909be6,0xeb4741de),LL(0x1a5f4139,0xb957c5da), LL(0xedc1819f,0xafdb3e8b),LL(0xe7caa690,0x33545722),LL(0x8bb66ed0,0x0ef33e28),LL(0x70e667b5,0x59073742), + LL(0x5c7773b8,0x0390fb3c),LL(0x286a809d,0xb80b4a2f),LL(0xfac46467,0xd17d6103),LL(0x91a48972,0x9a09a0d6), LL(0x39e44585,0xa2124b62),LL(0x174d241a,0x14c8a671),LL(0xada8ba26,0x99abfa37),LL(0xfbb457ae,0x847f3040), + LL(0x7529a18c,0x0587aaa4),LL(0x9bb45ee6,0x23b3f724),LL(0x2aa81155,0x4d7f5712),LL(0xa4f16d09,0xa9185804), LL(0x3fc992d1,0xab638141),LL(0x0cad0bb0,0xb6c326fa),LL(0x60f2cb10,0xe21c3625),LL(0x2fac20a9,0x6c7af09e), + LL(0xdc6f72ab,0x31e892fa),LL(0x21b81f7b,0x71d5c6a3),LL(0x298a0dd2,0xc3e2d70d),LL(0x13ecdc80,0xbc0c37e2), LL(0xe6496ba4,0xd3191146),LL(0x35115466,0x15f81541),LL(0x07d1937f,0x162be77d),LL(0x7b176367,0x38b4d194), + LL(0xb8cafbc9,0x4485966d),LL(0xf44c2a81,0x7cfc0d67),LL(0xe624cefe,0xe9e7ec4d),LL(0x581d4e48,0x4db8bec3), LL(0x7fc8615a,0xe76edf00),LL(0x9a02cdb8,0x1b62c4a5),LL(0x83938a6d,0x8b565749),LL(0x50c86c19,0xd813864e), + LL(0x16f55d40,0x7fc071ef),LL(0x9bb45ea5,0x70195438),LL(0xa35543ca,0x83cf09f2),LL(0x20554c19,0x07e91a84), LL(0x62a9d06e,0x51ecd701),LL(0x2044a663,0x00e14c62),LL(0x00423dd9,0xb1317c13),LL(0xa46eab4c,0xf49431bc), +}, +/* digit=9 base_pwr=2^63 */ +{ + LL(0xd0614aa1,0x35118434),LL(0xd1418434,0x8bae9779),LL(0xb8c15b89,0xf5641d82),LL(0x416432eb,0x2383af56), LL(0x2c73f990,0xa552d3f0),LL(0xa6bbdc7d,0x8df82e9e),LL(0xd75ec634,0x0f336aa8),LL(0x1603e53f,0xc42e3b2d), + LL(0xbad830d2,0x4b33e020),LL(0x590dffb3,0x5c101f9e),LL(0xbc80ecb0,0xcd0e0498),LL(0x52aa293e,0x302787f8), LL(0x220f8fc8,0xbfd64ced),LL(0xbe0ee377,0xcf5cebe0),LL(0x8913b128,0xdc03a038),LL(0xfde23279,0x4b096971), + LL(0xd2d638ad,0xb0f8c0de),LL(0x4f299d5f,0x47fc8c77),LL(0x9b68d48e,0xd1720a92),LL(0xa1c6f103,0xf944e708), LL(0xa146889b,0x36e34e04),LL(0xe74a2a28,0xb0aad2d6),LL(0xca52f53c,0xedbb034b),LL(0x87fb2713,0xe987a8e1), + LL(0xf727ef3a,0x6c5389af),LL(0x33db88fb,0x95ffeb95),LL(0x9dae0777,0x27cb7042),LL(0x616dbf02,0xd20afe81), LL(0x914bf706,0x0fab8e18),LL(0x0517cd09,0x3b1e66f3),LL(0x12e40644,0x24b46dce),LL(0x08f2d8fa,0x0ff10168), + LL(0xea2d8d84,0xe08a10df),LL(0xe97dda79,0xe31f05e7),LL(0x4e9ab132,0xfe95f84a),LL(0x927e216f,0xacd6f7fc), LL(0x83c5a3ea,0x025e27bd),LL(0x50f120fc,0xed010c0d),LL(0xb828101f,0x443b3b8a),LL(0x8cfc0dea,0xd8384819), + LL(0x83dc5447,0xe55f34c8),LL(0x04e4e9a0,0xbe76243b),LL(0x819166a2,0x78fb4cbc),LL(0xae37f80a,0x0bdfb703), LL(0xc217cda8,0xf869288e),LL(0x62af4156,0x2662bb71),LL(0x50ae9d30,0xce64f291),LL(0xdc0353c9,0xee0d4440), + LL(0xbd25609c,0x3e61a9ea),LL(0xb3839c8b,0x4ccaea93),LL(0xe43736e2,0x721cefa3),LL(0xd0035908,0x229cb244), LL(0x7f10aebb,0x936bc1dc),LL(0xb67332e7,0xc93a1002),LL(0xf98d1132,0xf4b53dd4),LL(0xd5a75030,0x7b99a196), + LL(0xca9a9526,0xb13caadd),LL(0x69a303e9,0x701c63fa),LL(0xb0a50f3c,0xb97d667a),LL(0x68e6557f,0x27c03d7c), LL(0xeb105607,0xab24e712),LL(0x8dd86ccb,0x4936aedd),LL(0x0a986d68,0x32196f8a),LL(0x248f5a65,0x0307b826), + LL(0xfcadb2ad,0x20e14b4c),LL(0x8c3b8c23,0x4cb4a092),LL(0x1caa9db1,0x50fe3c1a),LL(0x81c0a4e9,0x23cc56e8), LL(0x0867753f,0x5ab09199),LL(0xf9d47c55,0x5a253d19),LL(0x1a9bcc88,0x422b4e03),LL(0x671e4f36,0x4e1ce22b), + LL(0xebbe949f,0x588f58b5),LL(0x6982215b,0xb7762296),LL(0xcff863c0,0x3cc83dd6),LL(0x01098f51,0x81ec094d), LL(0xbe0432d0,0x214d69aa),LL(0x6455957d,0xe4e52a9c),LL(0xfadc1eab,0x94743ba8),LL(0x8176316f,0x2c395d97), + LL(0xe6bb4d34,0xeab6400c),LL(0xc0d49bf2,0x7364dc55),LL(0xe6959c7e,0xd6fa6e40),LL(0x7eaae61c,0x7960a997), LL(0x94ea77c2,0x918b3c63),LL(0x76866dd1,0x2cf4997f),LL(0xbcbba8ca,0xc4214abf),LL(0x7aa4aab2,0x349a6133), + LL(0x99458b24,0xd64bab77),LL(0x2eba3064,0x6fe19e25),LL(0x74068f03,0x9aabd83d),LL(0x6fdf8655,0xaef81218), LL(0x65593fef,0xf506d27b),LL(0xfaa457b2,0x0a1ad85d),LL(0xa303dff4,0x266d0f06),LL(0xabb416e3,0xe8114f4e), + LL(0x6aa5a1b8,0xe743f617),LL(0x1b5b2bd6,0xaf84652d),LL(0x092e2c46,0x8b1beab1),LL(0xe2518383,0x7e857549), LL(0xa9383124,0x6be2ece1),LL(0x7fc20081,0x8309442a),LL(0xc3046cab,0x1f00eb8b),LL(0x26f39f8c,0x959f3155), + LL(0x8fc2ed93,0xaacfe2d3),LL(0x78f0f858,0x83446645),LL(0xdda35ec4,0x58de6f09),LL(0xf78c69b1,0x891e5ecd), LL(0x91c13d67,0xff4a4ba9),LL(0x487d5575,0x6e78063d),LL(0x8d303a7e,0x226b621e),LL(0xc95987ed,0x5c9bc103), + LL(0x5e3be13e,0x28980108),LL(0x414af955,0x5e8c0ac5),LL(0xeaaa71a5,0x0f08e93b),LL(0xce4524f0,0x1bc50407), LL(0x921be66b,0x6a6a2e6a),LL(0xc27da9f2,0x37113baa),LL(0x52e90e29,0xc7b3c636),LL(0xc8558307,0xc075d178), + LL(0x88a45b65,0x605f581a),LL(0x68e58c1c,0xcb789200),LL(0xbc5bfe1c,0x14cbed65),LL(0xf02b11d7,0xd1af7dc7), LL(0xcd3a7cc8,0xb8341bc0),LL(0xa01a77b7,0x8e9aefe8),LL(0x5ae2a402,0x8eeafe87),LL(0xc11f3859,0x27a0698f), + LL(0xf7af9756,0xc5e49f07),LL(0x9e5b871f,0xffd65bcc),LL(0x423eed7b,0x62a95357),LL(0xb2ec687c,0x93cf64d5), LL(0xbe5da479,0x04b87dd7),LL(0x1a134c0b,0xdcceabd7),LL(0xc5c6925c,0xa4875091),LL(0x8e9c098d,0x3bf947df), + LL(0x11d1323b,0xb2617271),LL(0xe4c6046d,0x7769247c),LL(0xcfa6aac3,0xf9c1aaab),LL(0x354492e2,0xf7f13317), LL(0xe91befb6,0x4bd65afd),LL(0xf25b8f8d,0x3e78cd8c),LL(0xe60ff4d9,0x2adf53ed),LL(0x3d288d4c,0x81ec3853), + LL(0xee3bf44a,0xda852a71),LL(0xd7b5c6da,0x39b4ef6c),LL(0x44f4e720,0x472e6996),LL(0x9191614b,0xbbd19d38), LL(0x30c0e99d,0xa2bcc2ec),LL(0x57ba0582,0x29318d7b),LL(0xd315963a,0x322faf40),LL(0x0c0619d1,0x49ba5570), + LL(0xe5dcd066,0xc28c1f81),LL(0xff9e3493,0x64d1268d),LL(0xbdf8992c,0xab0db38e),LL(0x320cce30,0xe3790c26), LL(0x26e3e4b0,0x59b408a0),LL(0x6ab8504e,0xe9e5fe29),LL(0x83c9eaf3,0x45c827bd),LL(0x89518edb,0xc298e236), + LL(0x8d3ab381,0xb79a8b15),LL(0xdb0bb7c0,0x6bb951e8),LL(0xbe4b3353,0x5ebd3854),LL(0x2eb3b0fe,0x107ba27d), LL(0x46786cb4,0x9d01654d),LL(0xcf3a1aa2,0xf46d8352),LL(0xa1662f72,0xa8f669a0),LL(0x68a1d3e1,0xc979209f), + LL(0x65471473,0xc64975fa),LL(0xff1f2aad,0x1f8eec02),LL(0x8d0dd401,0x1b520fcc),LL(0x15e14346,0xcd732092), LL(0x8f878a76,0x616478d8),LL(0x7423e0f5,0x3579d49c),LL(0x1b2af15f,0x119f6d6e),LL(0xb08c2c8c,0xbbe33d81), + LL(0x8534a355,0x051d99c9),LL(0x458b764b,0xe3f3ddd3),LL(0xbc8c03bd,0xbd7e51aa),LL(0xe8d42e38,0xcd7abf4a), LL(0x3160e63f,0xf0d97428),LL(0x34d13871,0x258bba07),LL(0x3dcb885e,0x4fedb647),LL(0x50f0a645,0x009fca27), + LL(0x99775c4e,0x3f06c146),LL(0xf66e7d05,0xb10a4ed3),LL(0x3a3ab903,0x9300e3ca),LL(0xde3c3e1f,0x0a5610e0), LL(0x1af56fb7,0xe2827312),LL(0xd75d9a9c,0x7e2a2365),LL(0xf11f8963,0x9c3bb05a),LL(0x30c80488,0xdf94cac7), + LL(0x2d1143f5,0xaff1682f),LL(0xb4d6ed7f,0x5837e83a),LL(0xb4bce662,0xf3e179be),LL(0x8caa5fbb,0xfa8d7862), LL(0x59ea54c1,0xbdde016f),LL(0x3c1ac962,0xc488c829),LL(0x14b46863,0xabe8b367),LL(0x82897d1a,0xbcfde363), + LL(0x8c152354,0x87ddf0ec),LL(0x7a953398,0xdec85db7),LL(0x0b57108f,0x927a8b10),LL(0x525f78f2,0xb38b732f), LL(0xeb306d56,0x7e696084),LL(0x50269227,0x9befefef),LL(0xcaddfa11,0xfa86e376),LL(0x404be227,0xd50a08da), + LL(0x03bb523c,0xb7408e33),LL(0xc093aaf1,0x6d21aa4a),LL(0xa85d6fcf,0x52aae4c9),LL(0xb726afa9,0xf5d057c9), LL(0xf92ca5b2,0x7979bb5c),LL(0xc4e3e4f3,0x4b1f7936),LL(0x071ec517,0x2c534200),LL(0x67d3f86a,0x47b52ffe), + LL(0x84d1c5b2,0x4a0b581d),LL(0x0dfa90cb,0xfc825a4a),LL(0x11c72996,0x2df2ec98),LL(0x7dde922e,0x82077a6e), LL(0x9f28b584,0x89acda10),LL(0xe49fe66f,0x54578eb8),LL(0x4a1c29d7,0x90a5f700),LL(0xb3b44748,0x2de4719c), + LL(0x18d85e6b,0x6944fe14),LL(0xde7b904f,0x90bd8208),LL(0xa4597137,0x5811f3b6),LL(0xd4ab5433,0x7ea43767), LL(0xa204a36f,0x7ec39109),LL(0xa30fb76e,0xa43a4a57),LL(0xe090f2be,0x4fd514f8),LL(0xda1c97f8,0x3918138e), + LL(0x15145a20,0x2b466ae2),LL(0xfbac86b7,0x28ccb2ce),LL(0x04106b98,0xb891b707),LL(0x29696a08,0xe40a2310), LL(0x636d9e11,0x1210fed0),LL(0x2043caa1,0xdaea218d),LL(0x0aef7dcd,0x10c2ed0f),LL(0xffa5db7b,0x926be98a), + LL(0x36abac30,0xe762191c),LL(0x8b75b5cb,0xe21acfaa),LL(0xd180cc32,0x4f5e6b9f),LL(0x55deffdd,0x01358309), LL(0x992a66f3,0x1b1ab943),LL(0xceef1a9c,0x1ebe0246),LL(0x7a01dcb9,0xa24c9e25),LL(0x326505f5,0x3d45c4e3), + LL(0xc8544885,0x9b805759),LL(0x7bfcad78,0xbe9b99ca),LL(0x2b8fe78e,0xd1db36e1),LL(0xd5387bcf,0x37255a2d), LL(0xa150ad32,0x044b3a3e),LL(0x6671ae59,0xc65bc2a3),LL(0x1d52384b,0x41ce078e),LL(0x9e72c300,0x3115f1b1), + LL(0xd0a358a0,0x487ff9da),LL(0x9c242aec,0x4b20c369),LL(0x1c7b145f,0x7813a44c),LL(0xd6f2d3ee,0x87c6bede), LL(0x47d393b1,0x34d2a89b),LL(0x73f78679,0x1e9f97c6),LL(0x2edce91c,0xcb614fe0),LL(0x7e9a5fa9,0x62b96009), + LL(0x58c34b85,0x7eb2aeb5),LL(0xcf5074fc,0xa256a478),LL(0x98c1de9b,0x73f23a56),LL(0x61ce6535,0xeffd490e), LL(0x4a6c15c8,0x2569df2a),LL(0xfffc97a5,0x91e202a0),LL(0x28dc7a57,0xd83c428e),LL(0x9fc8dca8,0x03bc53c7), + LL(0x9b60487b,0xed394cfa),LL(0xb483a686,0xa4259f91),LL(0x179a5cca,0x11f51779),LL(0x86c1d1c7,0x00b00ef0), LL(0xf1231aed,0x6e596d2a),LL(0xd80eaa89,0x6c1a702b),LL(0xd23f1d64,0xd28f8c15),LL(0x6d01728f,0x93e85bea), + LL(0x1c704922,0xd4288fb5),LL(0x8c1363c5,0xaadd1968),LL(0x52f2cc4e,0x9b5c42d7),LL(0xc298e794,0xf9e4bc96), LL(0xaf0804ac,0xd604f076),LL(0xb3bb2628,0xa441140a),LL(0xd37bf6bd,0x761eabca),LL(0xbe1cf79c,0x7d192922), + LL(0x3da7073d,0x2365739e),LL(0x8e2c078f,0xfb7423ea),LL(0x3adfb6f3,0x08f5132e),LL(0x710a26fe,0x470a4205), LL(0x2b6c9b33,0xbe0afeb4),LL(0x3cd813bf,0x94d9edc8),LL(0x440a1699,0xa2c7a7a0),LL(0x4eaf0c10,0xbdc4ea3b), + LL(0x52fdc8d3,0x5a5455db),LL(0xb2945868,0x60485f39),LL(0x00af0abe,0x54ce9567),LL(0xe8d15f54,0x17bff77b), LL(0x0e14a7ec,0x0021c531),LL(0xdc9c800a,0x3efdb22d),LL(0x4d947ada,0x9a9e2747),LL(0xb37fc947,0x19417bc4), + LL(0x8f02c551,0x71ca8da8),LL(0x074cebc0,0x782d5da4),LL(0xc1a43a2d,0x99094980),LL(0x24890d9b,0xe1b02ff0), LL(0x45d82f7c,0x4eedaddb),LL(0x5061c943,0x7ae170a5),LL(0x4d47c578,0xaf8c7ea0),LL(0xad3a6eae,0xcad17044), + LL(0x4f4c9c8b,0x51383e61),LL(0x9182fc81,0x78d17182),LL(0x90d72cb4,0xbed6f0d4),LL(0x7bea62f0,0x98761291), LL(0xef3cd3fc,0x27594570),LL(0x91a8c364,0xf4759534),LL(0x2744eb2d,0xf5c607c5),LL(0xd8d8f337,0x0d6264eb), + LL(0xa8701750,0xb54867a6),LL(0x87691191,0x1387e940),LL(0xbd2f75dc,0xc451f178),LL(0xd1da6080,0x31a099d3), LL(0x49f87f03,0x0d0fcf97),LL(0x0af6273d,0x0b7585f8),LL(0x1142265d,0x3619cf2c),LL(0x05c559a4,0xf84d3f96), + LL(0xb83f2cb9,0xc3d3c80e),LL(0x8f602141,0xf4ef0b54),LL(0xb9612928,0x3afb343d),LL(0x8db5c907,0x7abe5620), LL(0xcf019b08,0xcd692215),LL(0x9ae12255,0x98d70b38),LL(0x8dfda5f2,0xb459e256),LL(0x8f3f403e,0x066a445e), + LL(0x423fbbb6,0x5663e123),LL(0x5424d48f,0xcc55ce36),LL(0x3b6d5467,0x8bca99b9),LL(0x316fc956,0x299ff0ea), LL(0xa0ceb256,0xd973a8d8),LL(0x6d9956b9,0x443ecdb9),LL(0x2f64f912,0x8c16a75d),LL(0xbbf7ab50,0x89e490c2), + LL(0xb8dbf031,0x4bd00db0),LL(0x7d2cb92d,0x866e0bbe),LL(0x1dd3db2c,0xad36406e),LL(0xe4e3f194,0x969dc881), LL(0x2a115bc8,0xcb3ac9e4),LL(0xe0a5ab75,0xb45efd5d),LL(0x55377d5c,0x1709c293),LL(0xde6bc25d,0x06d11ba4), + LL(0xccf2d10b,0x84a09347),LL(0x08ee5aef,0x571cd4d9),LL(0xa450dd82,0x1379ac02),LL(0xae404542,0x5b7f02f5), LL(0x2a7df4ce,0x17366e7f),LL(0x9830ebec,0x5bb3560c),LL(0x7c254726,0x5c582580),LL(0x70ab7b3d,0xea13f8fd), + LL(0x314e2a25,0x868c0f8d),LL(0x0be90b12,0x4b3dad3a),LL(0x32aaffcf,0x09970da4),LL(0x8a6d894d,0xe711e9cf), LL(0x0a80d07a,0x511521af),LL(0x8a2a2851,0xe3814716),LL(0x1de9183e,0xde76d41b),LL(0xaac779e5,0x8a9fc79a), + LL(0x26879f8b,0xd7d1f235),LL(0xe37d5f9f,0xcc849c85),LL(0x6b9cd82f,0x26b5488a),LL(0x91099141,0x1b068e8d), LL(0x35ee636f,0x040dc00f),LL(0xd84a9cbb,0xab40f94b),LL(0xdb303776,0x2e4cf65c),LL(0x78e8affb,0x42eaa12e), + LL(0x876f8f38,0x7835e4e9),LL(0x090ca6b6,0xcd421d77),LL(0xad0604f7,0x71a1d12d),LL(0x1a22e872,0x51c2d158), LL(0x429e45e9,0xfe7dfcc8),LL(0x48224b6f,0x20028f5c),LL(0x50abf907,0xf7afed37),LL(0xc4ce1a69,0x92183692), + LL(0x2d128add,0x0b93365c),LL(0x13200221,0x883f43c3),LL(0x4d309b2d,0x9d3b5a53),LL(0xcf91a023,0x60f0db16), LL(0x5b0e47be,0x20f0ebbd),LL(0x317d8b4b,0xcc20dde8),LL(0x5303dd7c,0xab033b48),LL(0x7a9c1974,0x6703eac7), + LL(0x351c8f26,0x92f0b738),LL(0x08569159,0xadd39ac8),LL(0x61098dd5,0x80866e5e),LL(0xcae578f6,0x7d0c1c6f), LL(0x975f59e4,0x13d89cee),LL(0x0092de2c,0x86006ed4),LL(0x819adda4,0xda825b0a),LL(0xde710934,0x74fefb46), + LL(0xd3dc8683,0x7c2ec289),LL(0x690e5d08,0x25898cd8),LL(0xbcc67531,0x9bed0f32),LL(0xac762453,0x356ba80c), LL(0x7680da5e,0xd3224c57),LL(0x3399d679,0xaae2597b),LL(0x68df6e74,0xb2a2a1e5),LL(0x2301b373,0x49d23e8c), + LL(0x170dd677,0xcb89b484),LL(0x6b3ee110,0x36b1d3d1),LL(0x0d7b51b4,0xe50ada4f),LL(0xfd9afdbc,0xa2f4fb57), LL(0xaa6dd8e8,0xb1b9b81d),LL(0x0be328aa,0x616056a0),LL(0xe12b07c8,0x8f6dd943),LL(0x325abaf6,0x4bb551c6), + LL(0x68fbed5f,0xa5460380),LL(0x87ed0d37,0xa65d059f),LL(0x208aa8cc,0xff60beda),LL(0x33d2d65e,0xc91ff11b), LL(0xf65f65d2,0x078c4e5e),LL(0xf347dccf,0xa92ed905),LL(0xf59e3be9,0x261ad25d),LL(0x3b69facc,0x95903d91), + LL(0xe789d854,0xcf0a2f94),LL(0x10fbf531,0x9d39cd51),LL(0x6de44e3c,0x980ed5d4),LL(0x78425caa,0xaedbae37), LL(0x7bd278b8,0x35804bc1),LL(0x6a2d7bee,0xf4bee96a),LL(0xa605671c,0xc6c553a6),LL(0x86f010d2,0x182c9238), + LL(0x9cd6f37a,0x94343b7a),LL(0x237190a9,0xa71e3853),LL(0xa8a28451,0xfcbebde7),LL(0xd711d2be,0xfa928367), LL(0xc3668951,0xba8fd2ea),LL(0x2d241329,0x00fad1ed),LL(0x5dbdffd1,0x61b82e19),LL(0x5a181dfe,0x0e5e5782), + LL(0xc60f1799,0x1c1bf593),LL(0x64ef800f,0x388d6950),LL(0xce927a87,0xf78ef00f),LL(0x6abfff9f,0x2a010419), LL(0xb0b7ffe2,0x13a7b08e),LL(0x6da4cc8f,0x4619da3e),LL(0x7937e0bd,0x8ac19190),LL(0x1af4f84c,0xf97d3fcb), + LL(0x8ac425a1,0xaea2abd0),LL(0x4a02e136,0xc619c17d),LL(0x1b2c4acb,0xf09a57d3),LL(0x87b4eb40,0xc6fce6fc), LL(0xb21b02f7,0xa161bb70),LL(0x95bcb925,0x075301fb),LL(0xe1b440ce,0x1d408003),LL(0x606b3142,0xb42a47af), + LL(0x1c832c35,0xd4ad09c7),LL(0x0e17fb8f,0x5bebe913),LL(0x8b45b250,0xbf8efbcd),LL(0xe5ca21e4,0xbef3cafe), LL(0x688076f1,0x08a18be7),LL(0x0c3a2648,0xabbb3fc5),LL(0xfb54720e,0xa77086e8),LL(0x19c29e8e,0x84277757), + LL(0x5b95b36d,0x551768ca),LL(0xc7df6d3f,0x8850a9b0),LL(0x5008c00a,0xe5a2737f),LL(0xad076e3c,0x9a577c0d), LL(0x2afa6a8a,0xbe7c611c),LL(0x04259dac,0x5dd9142a),LL(0x422bf3d1,0xd14253bb),LL(0x6805c78b,0x8c9dc4c6), + LL(0xd430488c,0xb9837258),LL(0x7abc184b,0xf9fc178b),LL(0x0c5e6a11,0x035d3079),LL(0xfbc2182b,0x20cbe540), LL(0x9d76812f,0x849994e2),LL(0xf7a85553,0x166a9279),LL(0x19d70aff,0x15ff0643),LL(0x4bc6a829,0x3c58e0b0), + LL(0x84df75ff,0x3809904b),LL(0x67a7c214,0x454c63fd),LL(0x2d873c09,0x79e0ffde),LL(0xcef301bf,0x620a3293), LL(0x237c2bdf,0x8f38c8e8),LL(0x13203c2c,0x61cf96de),LL(0xd0bef848,0xdff401d6),LL(0xee4bcbb6,0x3c8ed7ce), + LL(0x07ff8f9a,0x3e128e2d),LL(0xad7e8d5e,0x0653c0b2),LL(0xb1930408,0x7bb30bb5),LL(0x4c158386,0x91d18705), LL(0x80c21fb4,0xc4cf843c),LL(0x8a04133a,0x97a72d75),LL(0x4218755a,0x6b7c49f3),LL(0x68a40f68,0xc1a5a447), + LL(0x15ca3037,0x0ab9650e),LL(0xac06feb0,0x16b1fa71),LL(0x0faa3dca,0x50179660),LL(0x1c1aaeae,0x368b2d89), LL(0xb46f0310,0xf6fa652a),LL(0x79fcbc59,0x86a4d627),LL(0x6106a132,0x78169b8e),LL(0x9e387d16,0x40a741eb), + LL(0x80eed542,0x14a45174),LL(0x3362ef7f,0xadd64561),LL(0xc5dd0396,0x39228bfc),LL(0xea0c538b,0xe9fdf903), LL(0x74d235de,0x6bfd91ec),LL(0x24aa0a47,0x96ec2378),LL(0xaf8d6168,0xf5699241),LL(0xc548a60b,0x0a7b9be3), +}, +/* digit=10 base_pwr=2^70 */ +{ + LL(0x2ade9556,0xe5255c30),LL(0x75ba2e9b,0xe328af1b),LL(0x41ce9e47,0x9d3391ef),LL(0xfb0ffcc9,0xb74cd668), LL(0xe3226acf,0xc67103e4),LL(0xd2959e42,0xa65ad22c),LL(0x99d490fc,0x3aaa8406),LL(0x9ecc6356,0x3e26a1c2), + LL(0x4e92defc,0x71c975de),LL(0xd0089883,0x81aeb173),LL(0x2ec09190,0x8a30ce4a),LL(0x69a36d64,0x426e7838), LL(0x309bd2d7,0x5899a0b6),LL(0x3cc1a4af,0x3b1c24af),LL(0x345163b3,0xb2aa8142),LL(0x2c78c86d,0xd2ad9a69), + LL(0x8e7a4174,0xde59fe5d),LL(0xab3b0f3d,0xaedff2d2),LL(0x1f053470,0x4688e9e0),LL(0x97c95c7c,0x29ff3fb1), LL(0x85e6a8df,0xffb930cc),LL(0x623b6068,0xaa4d3175),LL(0xf9716623,0x682101dd),LL(0x402db696,0xa3bc9f5f), + LL(0xba4e4651,0xf67233c6),LL(0x0714d4ac,0x8cf95660),LL(0x71f1f8da,0xd70decc3),LL(0x7078370e,0xb674732a), LL(0x4ccc773b,0x4270416d),LL(0xde87951e,0xc848ff35),LL(0x493dfb56,0xa61506a8),LL(0x767eb110,0xd8371ea9), + LL(0x9e6b1d70,0xb468c382),LL(0x4cd025fb,0x1a901978),LL(0x5e6879e8,0x4bf50c7e),LL(0x71cf7119,0x6b862c0f), LL(0x06240e95,0x6a53ce89),LL(0x04107ff4,0x3ddfaa0a),LL(0x65532b51,0x317093cc),LL(0x0e27b5fc,0xf1e0f859), + LL(0xfe4674b4,0x96a97a12),LL(0xf7c97b12,0x2a132ae6),LL(0xa5f5ae91,0x5afcd087),LL(0xd4805ddb,0xfd1d3a32), LL(0xd7b5c8bd,0x0a989dc0),LL(0x4429af19,0x35d186e4),LL(0x42935fba,0x65623ad2),LL(0xe79b867d,0x4e274314), + LL(0x08aaba2a,0x47d92016),LL(0xf3f4c812,0x12b62343),LL(0x464f4b4c,0xb35bf043),LL(0xd8e8ba16,0xdc9391c0), LL(0x5d460c0d,0xcc0f8c4a),LL(0xe20fc6ad,0x04ce64bf),LL(0xaa4b7db5,0xd0289df5),LL(0xe5299815,0xe0ea15c5), + LL(0xda3adfe0,0xc066ee2f),LL(0x0c964e7d,0xce6a9bdc),LL(0x0c859476,0x04a0115b),LL(0x9c95699c,0xb5e02dc9), LL(0x11377eb9,0xf8301f62),LL(0x172bca2e,0x57b245a2),LL(0x7b47cf1f,0xa7d9b470),LL(0x1774b1c1,0x1b469bab), + LL(0xda2dce70,0xbb9ec3e9),LL(0xd29bcdda,0x02d5353e),LL(0xb215233c,0xc193244a),LL(0xd27a4e2a,0xb8d5727f), LL(0xb6c5b114,0x79e56194),LL(0x5ce727f0,0xe2c20e71),LL(0x236cbfea,0xc92f34a5),LL(0x56a02b8f,0xcc47dfd1), + LL(0xe983ba13,0x5cdbda39),LL(0x6e96c8b2,0x20f3de57),LL(0x66b76faa,0x2ff05aa7),LL(0xd7f84b47,0xa876bc62), LL(0x0d677d1f,0x962ef8a9),LL(0x801d3001,0xabc7bb1e),LL(0x7d13a23f,0xdb5f0b1a),LL(0x20b819e4,0x2664f3ab), + LL(0xdc45375a,0x96be66c5),LL(0x4a6c24e8,0x780ee062),LL(0x013a13ee,0xc6fbfd1a),LL(0x21fc4f9c,0x6ce1496c), LL(0x81f272c5,0x03130c09),LL(0xa26609cd,0x06e59457),LL(0xee5363b4,0xf4c5e564),LL(0x7df0775d,0x1cd19a11), + LL(0xdfd6586e,0xcdfcfa67),LL(0x1ba23faa,0x358953e5),LL(0xaeec5d6f,0x0f467275),LL(0x5b0e6b2a,0xb815967a), LL(0x012b89b4,0xb01bf133),LL(0x6839cc04,0xdd924bbc),LL(0x120dfd73,0xa5cd2180),LL(0x19bf8098,0x1abb11ef), + LL(0x6a281d1d,0xd56c11ce),LL(0x70daeb19,0xfb01f455),LL(0x8f29fcc1,0xbb442a0d),LL(0xe9b2f829,0x9aa60157), LL(0x90ae8113,0x1f3f6e61),LL(0x6c946c0d,0xc701a185),LL(0x52ba7caa,0xb4b89268),LL(0xb0a5c77f,0xd657c679), + LL(0x0dd26330,0x0f14eb11),LL(0x9b036325,0xff622296),LL(0x186e735a,0xaf833fb8),LL(0xc7e710f5,0x7801b02f), LL(0x5c948f43,0xa0bf821f),LL(0x86225c71,0x3be31aea),LL(0xe60b1c88,0xe98f4f7b),LL(0x73c5281b,0x6306588d), + LL(0x83c9f829,0xd6178277),LL(0xc06675f1,0x67b20e6c),LL(0xcb2d5b9d,0x8a63fb89),LL(0x038c00fe,0xcb28128c), LL(0x8c7c6c6e,0x070a5141),LL(0x8789c3b7,0xc05e18c3),LL(0xd5c67731,0x09fd03c2),LL(0xf54010ec,0xc59e2abb), + LL(0xf1ef2232,0x03977889),LL(0x9c7409a5,0xbe2c82f1),LL(0x32004e03,0x35ac44f9),LL(0x856823a3,0x048bb359), LL(0xec1cf253,0x2e108d6c),LL(0x703eb1d2,0xe98e74d7),LL(0x570ac34d,0xcaf64f60),LL(0x4d7797fa,0xff814e7d), + LL(0x70818472,0x93b6abc3),LL(0x888357af,0x0e984be6),LL(0x3fe0c135,0x2a7ca1b0),LL(0x94a82d67,0x0c6c4a11), LL(0xbb83ae74,0x0c90c359),LL(0x328b8af1,0x49b25e5e),LL(0x798ff0a6,0x26a36032),LL(0x1fc28ca3,0xbbf89c99), + LL(0xe679eb71,0x4ce174e2),LL(0xd1c183bd,0x17c76176),LL(0x03a69f58,0x4bf67be8),LL(0xc0ee530c,0x937a391c), LL(0x1f7daaff,0x2daa9d90),LL(0xc54f14d0,0xa47e99b2),LL(0xc57feeca,0x6be357e7),LL(0xcfdfd5dd,0x3753fad2), + LL(0x48f90174,0x74e1457a),LL(0x9b4734da,0xb8092642),LL(0x5800ea72,0x291e75ba),LL(0xc72c28f7,0x25a21b38), LL(0x505aa4d2,0x2193e0c9),LL(0xada9d3f8,0x2f6829e3),LL(0x92900e29,0x66cd5a1d),LL(0x7414dc1d,0x1360d287), + LL(0x0d1b96aa,0x5deeb2eb),LL(0x3e52ccf1,0x25783ce3),LL(0x29d534ef,0xe4e251f4),LL(0x55797dad,0x9fe9693d), LL(0xc6ab9935,0x6a173d69),LL(0x7721ca8e,0x23991318),LL(0xc393eb26,0x38cbcd30),LL(0x71d95a9e,0xe3627ab9), + LL(0x7f6fe2d4,0xdf1218be),LL(0xfabd8326,0x850c8598),LL(0xb0f7cf49,0x1214d3d7),LL(0x1805345b,0xeaf60d31), LL(0xbfee2c5f,0xc5caf65b),LL(0x45e23043,0x70127979),LL(0x500fbad2,0xda36e794),LL(0x4156e3a6,0x38fa60b0), + LL(0x3cbab88a,0x45934bdd),LL(0x1b19dce4,0x72821e74),LL(0x8661e32d,0x532f706d),LL(0x73a9930e,0x3dbfc225), LL(0x3cbeb0f1,0x72d1cb2a),LL(0xe20f5613,0x795b0696),LL(0x9fc88717,0x6e3469e8),LL(0x483864d2,0xf4bf0531), + LL(0xfa19ddd9,0xc92e6a8c),LL(0x3528e628,0x7db7e2ee),LL(0xf321fc88,0x997a00eb),LL(0xacdf205f,0x7605a2c9), LL(0xea9c3ed0,0x9fca58cf),LL(0x56ff0e98,0x833078cb),LL(0x662a1695,0x75159b8f),LL(0x1919f51f,0x186560b7), + LL(0x5ef764b4,0xe9b60e57),LL(0xbe73c698,0x61ad29fc),LL(0xdd2be1ee,0x18489017),LL(0x7febda71,0xac95e3b6), LL(0x5ac96092,0xa6985346),LL(0xbfc83566,0x906265c3),LL(0x5972fa4a,0x1692c812),LL(0x00773540,0x4e76f879), + LL(0x542b137e,0xba9a6268),LL(0x4c7926e2,0x43a52b90),LL(0xfeae554e,0x28785bf5),LL(0x0ab61576,0xc023b688), LL(0x10933a55,0xb3ec8181),LL(0x6331678e,0x75634459),LL(0x17c50b5d,0xe0dfa141),LL(0xe2151f25,0x4cbe7fda), + LL(0xce81fbaf,0x3f3072ac),LL(0x0ff56a32,0xa387bb20),LL(0x99865968,0x2b08a811),LL(0x084cb1f2,0x7279f913), LL(0xdad70f5a,0x78cca6c9),LL(0xff47647d,0x72469f6a),LL(0xe358b597,0x2505c7ff),LL(0x998ff0dc,0x7c5268a8), + LL(0x99d5b1c1,0x32d70129),LL(0x24a90c34,0x72727c15),LL(0x715662b1,0x57dad21c),LL(0x132f3294,0x76b4b6ec), LL(0x267d246e,0xd03b46b1),LL(0x29b47516,0xc7c848ec),LL(0x1660af51,0x5eab3dbc),LL(0x04c66383,0x818894c4), + LL(0xa7b82f5c,0x26a45c3e),LL(0xea98adfb,0x494694de),LL(0x134b182c,0x44a06ec3),LL(0x75b82b41,0x5570befa), LL(0x129f952e,0x819f6808),LL(0x914f87c4,0xa6fad25f),LL(0x7c7be905,0x831e5668),LL(0xe623a5c2,0x93107f38), + LL(0x5e40c5f4,0xa9d88469),LL(0xaa5993c5,0x4314d233),LL(0x9c19bbc5,0x5eab88d0),LL(0xb385d3cd,0x795e1c21), LL(0xce163fbe,0x532a1871),LL(0xb867aea4,0x727cb126),LL(0xf7f30476,0xfc7047eb),LL(0xfcc4fe35,0x18031271), + LL(0x884a4c6d,0x4b84fa4a),LL(0x55c830ab,0x82cb9aee),LL(0x0cc927cc,0xd4cfdf04),LL(0xa16bef30,0x787efdde), LL(0x32e3c763,0xd1fb2dd6),LL(0x16737272,0x8739566f),LL(0x03a1055a,0xf9ae4f46),LL(0xf9a7472b,0x199970cd), + LL(0x1d33ac50,0xf9893cfb),LL(0x1e5ff4e5,0x74cf7dd4),LL(0xf7165331,0x72ec32e5),LL(0xbb4679cf,0xa082c59a), LL(0x5c75461a,0x3cd0a467),LL(0x40f06206,0xd2872d68),LL(0xb5122795,0x08271eef),LL(0x1475e22b,0x7006d350), + LL(0x89e35108,0xf7cd1bcc),LL(0x93f1cbaf,0x924efa43),LL(0xf35b13ac,0xe3716559),LL(0x60370a1d,0xa0a88e87), LL(0x8c286ea3,0x1203be0a),LL(0x6ebd50c7,0x97fc5ab6),LL(0x74284d08,0x2b5b3602),LL(0x694a20e0,0x3055716f), + LL(0x193520c0,0x793c8a89),LL(0x655f5ff2,0x356402d0),LL(0x1af5ddd6,0x0cf889ee),LL(0xb3f149b2,0x3eb7eb35), LL(0x68e07e0e,0x5254b57c),LL(0x9c5bbfa7,0xb1971de2),LL(0x0ad81e7e,0xcc85a53a),LL(0xed3cbb10,0xbaaa4d2b), + LL(0x8f3a7eec,0xbdf9941c),LL(0xa1f26666,0x6e1b7dab),LL(0x2d79a58f,0xe7a0dfa4),LL(0x1f2b9667,0x25e0ddb5), LL(0x5fd96351,0x4b3b5105),LL(0x8874abd1,0x12325832),LL(0x795d91a5,0x56e90310),LL(0x2c32eec8,0x376a79d2), + LL(0xa8a16445,0xd9dd8817),LL(0x0e00fa92,0xd61f6aec),LL(0x17d95f07,0x594a620d),LL(0xf4b15001,0xa1534fda), LL(0x0974f4a3,0xe9402601),LL(0x8f671f13,0x4c3fc130),LL(0xc5f35bfb,0x8eaab35a),LL(0x4626baca,0x13b98472), + LL(0xcdee6f8d,0xf48703ad),LL(0x12d39694,0xf1ba0309),LL(0x9fcda52c,0xeb2d4d92),LL(0x5507b401,0x984f5111), LL(0xf6bab9ec,0xe3aa26ae),LL(0x6b2e077d,0x557b5e3f),LL(0x2f9c5f35,0x7185ab4f),LL(0x1680bcbc,0x96f21a33), + LL(0x2ddb1173,0x2e7f6e07),LL(0x816ffc8d,0xa704416e),LL(0x52e26265,0x55acfaa3),LL(0x8b758c94,0x9c244253), LL(0x0479936d,0x4012e0a6),LL(0x6d8541d8,0x12749e93),LL(0xce56a2a1,0x374f420d),LL(0xc79da17f,0x6a8e3614), + LL(0x157cc9e1,0x3602ad09),LL(0x13603196,0xf3c4a54c),LL(0x8473ae27,0x354fc6ed),LL(0x651b8003,0xb4cf4251), LL(0x3415b5f0,0x456b1b9b),LL(0xc4c71507,0xe078a858),LL(0x9a0a11fb,0xf4210409),LL(0xf930ec45,0x76de42a0), + LL(0xcfa869a1,0x82ecd0af),LL(0xdccf2e47,0xa637938a),LL(0xc041648a,0x1d2858f2),LL(0xc0dfacd2,0xcf9abfe8), LL(0xbdddebe1,0x3af77e19),LL(0x180b535f,0x15f6b0bb),LL(0x549d34c1,0x497de678),LL(0x3dba7d6f,0x31495c9e), + LL(0x393ab61c,0x47b9368b),LL(0x27441f92,0xdb8ee3a8),LL(0xfb0b0117,0x214a6a5f),LL(0x43759430,0x429e61ad), LL(0x77316c0e,0x78116e88),LL(0xa6a29e98,0x59c82616),LL(0xaaef4619,0xbfed454f),LL(0x35926335,0x673327c4), + LL(0x18c616a7,0xaa66d8c5),LL(0x6d28fb98,0xa93946a6),LL(0x133336dd,0x4fc30de5),LL(0x7f0c805e,0x7e000a34), LL(0xcf7eab23,0xa82dcf54),LL(0x8dc24371,0x679e67a8),LL(0x1af582d8,0x26b2dffc),LL(0xd3fe2155,0x4c1d692a), + LL(0x2d024923,0x2475b110),LL(0xc303c1e7,0x0cc9245d),LL(0x03667a7a,0x290b7a77),LL(0xd87dbd9c,0x2ab8eb6d), LL(0xc098719e,0x7089e481),LL(0x12c022c8,0x17dd6d74),LL(0x8b7aca14,0x90efa01f),LL(0xf55fbe83,0x8b601fba), + LL(0x415aa7a5,0xf800bd76),LL(0x3aa74349,0x015573d3),LL(0xaf5ec789,0xd5143779),LL(0xd76dd2dd,0x32330b4b), LL(0x82331ef1,0xec6caa4d),LL(0x25ad1965,0x92cc8865),LL(0x134370b0,0xf8209a40),LL(0xb883cf95,0x320a37b9), + LL(0x5d39731d,0x94237ba2),LL(0x8532a968,0x6359d195),LL(0xf86b8b75,0x8bca94c9),LL(0xde316226,0xdb5c6a9c), LL(0xa2fa0c26,0xdf300c59),LL(0x48af4357,0x6dbf6082),LL(0x06535fc9,0x25066c6c),LL(0xa29b2477,0xba2e774e), + LL(0x1931299b,0x5157e93d),LL(0x3a8035a4,0xd6249c10),LL(0x81245740,0xcb18fcf1),LL(0xad5ebe1f,0xb4d84c9d), LL(0x8df0d59d,0x95050bec),LL(0xac2a2e0c,0x190a4860),LL(0xfd1bbb11,0x29029e0f),LL(0xb075b432,0x341f73de), + LL(0x74836028,0xa825c3c5),LL(0x8f55343a,0xec4fd74b),LL(0x60a683b3,0x009bcab5),LL(0xcd3adea6,0x29877303), LL(0x684a33ac,0x9f264bf2),LL(0x84b3c379,0xc8bf19e6),LL(0xa1215470,0x8ac35fb8),LL(0x405386d6,0x2919d9da), + LL(0x19780b2a,0xb4e4aa30),LL(0x356ddd4e,0x639b8fcb),LL(0x9322c245,0x6ed7b10c),LL(0x57f39c76,0x84ec0bc6), LL(0x879176fb,0x6a1be66c),LL(0xe10e0f77,0x4cab3151),LL(0xe2ae0777,0x01c6321f),LL(0x65e57ff1,0x04d6a04c), + LL(0x142596dc,0x8c1725ed),LL(0xb2d413a6,0xd321d49a),LL(0x2b5e8ae7,0x19c25fc3),LL(0xbd3c7dc6,0xfc32cbeb), LL(0x57b98ff5,0xf3ec98b8),LL(0xf75b1a00,0x52e5f1ad),LL(0x8f8ad463,0x16812bb4),LL(0xa274f0c3,0x9d67cb11), + LL(0x80064047,0xdec72055),LL(0x4319f87b,0x3f828c01),LL(0xff4d6c4a,0xffcad5c3),LL(0x67a6e030,0xee6659b2), LL(0x0478715f,0x9cb5c729),LL(0x5a1c926e,0xc63fc281),LL(0xdeb11155,0x1b8788ca),LL(0x4f0c00b2,0xbe2eebf1), + LL(0xa6af09d1,0x9b72ffd0),LL(0xa9a459f3,0xcbac42bd),LL(0xf560dc93,0x150806c0),LL(0xc57787ee,0x71c636e4), LL(0x2a951b0d,0xe4f3acb8),LL(0x3b18d294,0x510dc771),LL(0xb060e782,0xfbb3fb53),LL(0x0358210e,0x0659cadd), + LL(0xecde1629,0x23638245),LL(0xcc09daa5,0xee6e9a65),LL(0xee18f9cf,0xf440bb81),LL(0x955200e0,0x99e4d6e8), LL(0x93e69354,0x34555b58),LL(0xfb29b19b,0xa6080e13),LL(0x5100ab06,0x3bfa4796),LL(0x22eec8fc,0xf5db4b13), + LL(0xe5aaa980,0x2c1a229e),LL(0xd29eb83e,0x446cd46b),LL(0x7f5aa015,0xe0c044da),LL(0xa18f23f7,0xa55d5f23), LL(0xb6b70f51,0xd71e58c1),LL(0xb5862c51,0x77c72d10),LL(0xfce94ceb,0x01db731a),LL(0x13a47553,0x877703a8), + LL(0x3b75e6d9,0x4878b0b1),LL(0xfe60f98a,0xbe8421f0),LL(0xc69d6820,0x6312821b),LL(0x00d2db20,0x4c019374), LL(0x2a1d8b74,0xb1bd5175),LL(0xef7fdad6,0xa0a24ad2),LL(0x929fc488,0xf29fd27d),LL(0x162a02de,0x8e28b4ed), + LL(0xc166afcf,0x434cbdb3),LL(0xbf663e65,0x7b79e808),LL(0xa3c3b159,0xd445f1b0),LL(0xb35b2be9,0xdf9f92b7), LL(0x788a9bbc,0x815b57f3),LL(0xabbba2e0,0x9e03e357),LL(0x91a658d8,0x3fc574d5),LL(0xadf4e250,0x83b35d8a), + LL(0xacd1e4f0,0xa0e15175),LL(0x868b4e04,0xeca899a4),LL(0x782b7ee7,0x713b4e9e),LL(0xb7d58c1a,0xed177e1e), LL(0xac8364b2,0x4778df76),LL(0x2e8f7ef7,0x6898fb31),LL(0xa03975b0,0xfccf4c53),LL(0x8597436f,0x0f908d14), + LL(0x96671c53,0xbeaf1a16),LL(0x6bc4cbbf,0x9be64329),LL(0x80017bf3,0xc8f66f63),LL(0x8836ff35,0x92d700f2), LL(0x13a4daf1,0x9ddd7a81),LL(0x9b72664b,0xb3c42723),LL(0x81babf43,0x3d96f79a),LL(0x7ce5830d,0xa46c9c0c), + LL(0xf5999a71,0x54dfec97),LL(0xe586a578,0xdb5be461),LL(0xcfb4e7ad,0xf9bc3f04),LL(0xb11f07aa,0x6e5448a9), LL(0x70132d5a,0x29ef7791),LL(0x28ba851a,0x4fe486c3),LL(0xb62f7f8a,0x6743fecd),LL(0x44d24d01,0xeb264387), + LL(0x72ebb976,0xf93c05bc),LL(0xaaae099e,0xe65b30c0),LL(0xc8427104,0x4194721a),LL(0x3025853e,0x3af3487f), LL(0xdbf48435,0xb33a2893),LL(0x2d79334e,0x2c5ac639),LL(0xb16b05a6,0x8fc199f8),LL(0x7661a77b,0xc145358e), + LL(0x1841f719,0x15b580b6),LL(0xd7045528,0x24f5fadb),LL(0x98df2c22,0xe443c257),LL(0xd7eed58b,0x48acf5a7), LL(0xedeb9e4b,0xe24e6571),LL(0x562533fd,0xcd047b81),LL(0xd1566e36,0x618ddd86),LL(0xdba1eced,0x09a77b70), + LL(0x6968ddb9,0x0e9de410),LL(0x12985b32,0x10f0f429),LL(0x8ca7d2fa,0xbe21b10f),LL(0xc610ae2b,0x0844d8e8), LL(0xd57107d5,0x58a08694),LL(0xc34441f3,0x45f44bd5),LL(0x79a54186,0xe8b3b3df),LL(0xb8b5f26b,0x6496d668), + LL(0x192503be,0xd69cefb8),LL(0x40f928fc,0xb692128d),LL(0xa7ed8c47,0x13b11dfd),LL(0x5a196756,0x90bd279f), LL(0x17ff33eb,0x78f2e8c6),LL(0xaaf6c182,0xa7b233d4),LL(0x76a31f46,0x63d62503),LL(0x1449dc97,0x53143dc3), + LL(0x94922922,0x5bb86802),LL(0x89181334,0x2f45a7dc),LL(0x1ec5cce4,0xf7c466d5),LL(0xbb3bd5f3,0x52d15eed), LL(0xe6eacf86,0x150bd5f6),LL(0x0ba66988,0x7fecaf3a),LL(0x11f6f528,0xcdd7fadb),LL(0x55042faf,0x60f64c68), + LL(0x961ddf76,0x1615919d),LL(0x53f2f514,0xdba095cb),LL(0x1e6c076c,0xf04960ba),LL(0x4c9f17d5,0xe5276722), LL(0x61c186a9,0x93ff80f9),LL(0x3c58ab92,0xd17378b0),LL(0x769492e8,0xc67f9ae1),LL(0x0c3d023b,0xaccfc868), + LL(0x5b99708d,0x7d67a63d),LL(0x4b80189a,0xfb29bef7),LL(0x241c402e,0x3cb7eeaa),LL(0x2c5c2767,0x328cb6de), LL(0x9cec231d,0x0d24a7b4),LL(0x0e2e6a7f,0x725955fc),LL(0xb7f17b13,0xa2040cfa),LL(0xa25c71cf,0x215eff8d), +}, +/* digit=11 base_pwr=2^77 */ +{ + LL(0xc0d41a94,0xe4d9cab1),LL(0x9e60f7d4,0xc38b202a),LL(0x9336116c,0x2bbf6b17),LL(0x2e068f13,0x2f9aa877), LL(0xa4bac9fd,0xf8820627),LL(0x8a593cb4,0x2209cb9e),LL(0xc7417931,0xaa78ec63),LL(0x7cfccfbf,0x42d21251), + LL(0x3e611940,0x40cee5ae),LL(0x0aa090ec,0x4e2d9ea9),LL(0x1b926e42,0x73d167ef),LL(0x5d5112a3,0x7fff36df), LL(0xcaffa3fb,0x25587745),LL(0xc5a83504,0x224f7f4e),LL(0x3b47bf2a,0x5ceff018),LL(0xecfab5c6,0xed9bfa73), + LL(0xd9b429c9,0xf3d570b8),LL(0xc5ad81cb,0x69460116),LL(0x2f6825bd,0x30833a08),LL(0x7c99f36a,0xa297122a), LL(0x05c3abdf,0x6fc9b848),LL(0x5f2e24b2,0xefe95298),LL(0x8915d922,0xf045275a),LL(0x298a9726,0x79146aab), + LL(0x4f831d0b,0x0c7d5905),LL(0x2d47d4fe,0xfaaaa26c),LL(0x85042e12,0x5ac28599),LL(0x7796977d,0x7eda370b), LL(0x95c0be63,0x9f0bd71d),LL(0x8e821005,0x7c4601bc),LL(0x4c2ffae9,0xf1ecbc60),LL(0x9b688173,0x7e3efc57), + LL(0xf4ea7641,0x868c672b),LL(0xd94243dc,0x4fa90a82),LL(0xf5eab489,0xbd82913e),LL(0xe966b39d,0xceebc159), LL(0x35c51e2b,0x31fe4f5f),LL(0xc79c1725,0x2fea6ab1),LL(0x83afad40,0x5856cd85),LL(0x4ca89b88,0x7f960988), + LL(0x1ed8fed0,0x9d237c2d),LL(0x4e0c6f10,0x69b4ec80),LL(0xe4648395,0x11f83283),LL(0x306e152a,0x6f4636a7), LL(0x804539b3,0xf70fd23a),LL(0xb3cdb981,0x4db0903a),LL(0x6691eb18,0xe506ae6f),LL(0x1d8e9d9d,0xaa69c7aa), + LL(0x3e4e2088,0x540af950),LL(0x95f04e57,0x8fab01d5),LL(0xa8c51a67,0x51bf909a),LL(0xfd19beb7,0x01299f5e), LL(0xb8f15aeb,0xdf703400),LL(0x2d890394,0x19c70987),LL(0x203d2d99,0xf5fcc675),LL(0xc3d4ddea,0xabbf3f21), + LL(0x587feffa,0x8348ca15),LL(0x7d69e4ad,0x585d0740),LL(0x885a0745,0x6fbe5619),LL(0xb10b24dd,0x04ee9eba), LL(0x0f4c12d7,0x5c27075c),LL(0x3c51c605,0xacf4acdc),LL(0xfce336d0,0x782fa52b),LL(0x483621d2,0x6e1d078f), + LL(0xd4dc3277,0xa2253bfb),LL(0x4691bc12,0x3a014307),LL(0xebdef616,0x415aa5b2),LL(0x16fe065b,0x1008a44a), LL(0x16dfa94d,0x4004a90a),LL(0x8464785b,0x0e24f541),LL(0x88d30ea8,0xd2715c89),LL(0x1f05a312,0xaf81a9ff), + LL(0x3e8d5eef,0x958da470),LL(0xd3c80414,0x09561898),LL(0x8bb74691,0xba6b731c),LL(0x577f2ef9,0x6b7355cd), LL(0xb8a98efa,0xd1f416ed),LL(0x11590271,0xd4a1293a),LL(0x4658e9eb,0x2c4d5934),LL(0xd1f15d39,0x51803651), + LL(0x3c95fffe,0x9b9f0c05),LL(0x31acd6ca,0x8f021451),LL(0xf9dba549,0x5fee2961),LL(0x17ea0456,0x05217975), LL(0xd13a6a4a,0xc0591906),LL(0x90daf370,0xa7f5ed02),LL(0xfc4c304d,0x1f8b7158),LL(0x1b7f0246,0x77016c29), + LL(0x2ea265d2,0xc27d1847),LL(0x2862781e,0xec0789c6),LL(0x5d86a60e,0x0a79ac1f),LL(0x130670a5,0xe325b563), LL(0x6d33bfee,0xf4794460),LL(0xec25bb10,0x126e703e),LL(0xa7bf902f,0xeae22fd3),LL(0x28eef62e,0x8b2fb282), + LL(0x059138b4,0xb68de35b),LL(0xd46e68e3,0xfc44bf56),LL(0xff11f76a,0x71567daa),LL(0x6b17cd2a,0x9110e849), LL(0x69573b93,0x7c4027e3),LL(0x1eb9bf01,0x84ee945a),LL(0x28c26cdb,0xa3fadc6d),LL(0x575dfc1b,0x7037a74d), + LL(0x9b2223dd,0x58c96a91),LL(0x51461b94,0x912fc795),LL(0x2df3329a,0xc18ced63),LL(0x88a002d0,0x79d6f75f), LL(0xf44d3d84,0x73d7a089),LL(0x8c058073,0x98c78aa7),LL(0x333ae8ff,0x0ab8b3c7),LL(0xebd4e536,0xf5a8f5ec), + LL(0x83a5f52b,0x2c7df9fd),LL(0xcc062097,0x314fc7c3),LL(0xc5a3113c,0x6c3195f8),LL(0x2c25a629,0xf130cef9), LL(0x70c8dd70,0x10c8cc5b),LL(0x01cd40d3,0xecb7a35f),LL(0x6fe21c93,0xfbee538f),LL(0x2ba12ee8,0x57ec1959), + LL(0xb2b806a8,0x74387a1b),LL(0xbad5d0f4,0x14efa300),LL(0x23a0e9e4,0xee7e4421),LL(0x3b6abdec,0x504ae428), LL(0x927b1aac,0xb8c06fcb),LL(0x323b71d3,0x55e1c85c),LL(0x48d6dae1,0xf47e180f),LL(0xa84cb0b8,0x6d80dd63), + LL(0xd75d7785,0xf8e07d53),LL(0x37614325,0x3edf19b7),LL(0x357946ed,0xf03709b0),LL(0xd12105e7,0x567d8c0d), LL(0xa9383b49,0xecf6abc0),LL(0xcab329a7,0xfe9c760b),LL(0x43928852,0x425e62fa),LL(0x6194b51f,0x27945ae0), + LL(0x3ee4f7cd,0x756f0f54),LL(0x26e4c865,0x4337ac7b),LL(0x35b9c3ae,0xf94596c3),LL(0x4d6aa5d2,0x066fd3da), LL(0x43c8d94c,0xce1a5b7e),LL(0x614c0fc2,0xaed496a8),LL(0x2a6d5481,0x355e16f5),LL(0xa09d3afe,0x8f934639), + LL(0x2bf2a099,0xd170daef),LL(0xae6ee507,0x3920d87a),LL(0x248158e3,0xbdac1c8e),LL(0x05c54e69,0x99033a9a), LL(0x41872197,0x4513bdf0),LL(0xd3f0f889,0x15634020),LL(0x05d42aa8,0x76c1855a),LL(0xe8ba47cc,0x23079179), + LL(0x728429e6,0xf80b787b),LL(0x3dd8c4f8,0x896b6ea5),LL(0xc7d9fe06,0x711eef39),LL(0xebced829,0xfff28d03), LL(0x4ad40c88,0x5d7823b8),LL(0x3b112bd4,0x40a5a166),LL(0x63bce926,0x84230bfa),LL(0xbe17e7cd,0x39d2e6bd), + LL(0xef03ee6c,0xa772e242),LL(0xfa009e67,0x888bc969),LL(0x4893e1f0,0x0f06ee83),LL(0x6b89e124,0xf28f0d3c), LL(0x71f5cbc5,0xb3e70ef8),LL(0x6cad191e,0xff0f7e62),LL(0xf028d276,0x990697be),LL(0x4ad8f830,0x144c101c), + LL(0x3556d74f,0xbcaafb45),LL(0xeb4c7ea0,0xbc216224),LL(0x234a62c8,0x73ad1043),LL(0x2d95ff46,0xa644eb6a), LL(0x0a3373f8,0xd545b60a),LL(0xd4edaa10,0xf7a0572c),LL(0xa97a85b4,0xa7177049),LL(0x7d3ec769,0x529dbadd), + LL(0x7822dd37,0xc45610f6),LL(0x98258666,0xfad07fab),LL(0x87259c1b,0xac27001f),LL(0x1b849745,0xa9bdc6a9), LL(0x04c862b1,0xc7ee7216),LL(0x0012f591,0x32c1541e),LL(0x5a366d79,0x8198aadd),LL(0x68866e1b,0x03cd312e), + LL(0x9ec64698,0xa0dbc381),LL(0x1ef36dd2,0x770e4d11),LL(0x8d56bdfd,0x01d17915),LL(0x75eb9947,0xb48153cd), LL(0xfde98390,0xc1d17a54),LL(0x0fe2d6fc,0x0817eaf7),LL(0xa6a704f1,0x44a63591),LL(0x7f554182,0x9423465f), + LL(0x13e0574e,0xc7c23cbd),LL(0x439941b6,0x6e06e2cb),LL(0xafa39c79,0xa8aebd2c),LL(0xedede735,0x1b859e2b), LL(0x6b4f5465,0x2f485781),LL(0x624c81e8,0xec3093f0),LL(0xc282644c,0xc1f027c1),LL(0x2b74ab51,0x2f6e990b), + LL(0xed2ea3df,0x9a988d1c),LL(0xff39d3df,0xa3f50efd),LL(0xec1d7614,0x418a3627),LL(0x3d4fa3e8,0xafc1520c), LL(0x891a9c69,0x741305af),LL(0x5d6f8296,0xe87064d4),LL(0x12307b05,0x47c9394c),LL(0xc35f0f40,0x6b038acb), + LL(0x1ccca008,0xa6e77647),LL(0x9dd71746,0x58e4cfb6),LL(0xc1fe84ae,0xdf649c98),LL(0x90db4388,0x2e308ddc), LL(0xe9362400,0xc2641332),LL(0x42d265e5,0x92dd9842),LL(0x31eb91bd,0xe0e4ed9b),LL(0x145535c5,0x62ec7dd1), + LL(0xf810812f,0x1ff29a09),LL(0x15e9b102,0x56b64acb),LL(0x5b353184,0xb5f6d4d4),LL(0x1c593774,0xc3c9292c), LL(0x2c700292,0x16781036),LL(0x3ae2f0c6,0xf0948fc9),LL(0x4da778ea,0x40e353cc),LL(0xa34df03a,0x07febf09), + LL(0x9ec397ff,0x349812ae),LL(0x330f02d0,0x7c78812f),LL(0x7d241ea2,0xf956700b),LL(0xebed08be,0x864b1809), LL(0xb9eb1635,0xe4cec3df),LL(0xab49fb60,0x7dd65ad6),LL(0x86788a28,0x06551163),LL(0x11fb4214,0xda8792d5), + LL(0xcec09c3e,0x82140df3),LL(0x539db03f,0xcd34ca30),LL(0xe7dd0e09,0xf07cf030),LL(0x56ae3487,0x7b08a242), LL(0xbf5a6549,0x9c0fd607),LL(0xd189d68e,0x0b1fc745),LL(0x9cf52022,0x0d91be74),LL(0x43ff7fc3,0x6013f31f), + LL(0xb5654233,0x3bf90bd5),LL(0x202bf534,0xd0a17969),LL(0xc97e02ba,0xff373b8b),LL(0xd31dba07,0x4606de54), LL(0x8114562a,0xb045c50a),LL(0x7b8d8915,0xc123acac),LL(0xb60aa324,0xa8f7cd87),LL(0xabc48200,0x077cab67), + LL(0x0d7fff59,0x88a68643),LL(0x67bfe743,0x82b92193),LL(0xc2ce06f9,0x1a8b86cf),LL(0xf9ad6134,0xa38414a0), LL(0x28e2c39f,0x7f102610),LL(0x480856a0,0x34805c20),LL(0x18c3034d,0x1b3f9302),LL(0x574c0c9d,0x1713f457), + LL(0x690ce2a5,0xd84fa26f),LL(0xe01a4637,0xd4cfa19f),LL(0xcc6fad47,0x4807fb82),LL(0xf933226a,0xc9d84b48), LL(0x7cd2c9dd,0x9b7c530e),LL(0xf44818e3,0x6436a001),LL(0xdfb00461,0xbae0ceed),LL(0x51c8c8a3,0xed6a7c5f), + LL(0x0463ac73,0xa6e7fa54),LL(0xc77b19e5,0xa0deed89),LL(0xff218725,0x4e0a3396),LL(0x2edf2704,0x7cfbbd57), LL(0x4e8608c5,0x8114d0ca),LL(0x38c935b7,0xceae65b9),LL(0x330af8fd,0x052b1407),LL(0x723c422b,0x02e189a1), + LL(0x657560c8,0xf1cd6216),LL(0xe5068361,0x099eec2f),LL(0x3de78037,0x68ef58fb),LL(0xf3e399e9,0x83e0d34e), LL(0xf9a17095,0x3a2a14c8),LL(0xaaf9f08a,0xc7a360be),LL(0x30e99527,0x6420155f),LL(0x9f180405,0x8f610960), + LL(0x02bc97fe,0x871a832f),LL(0x8dc7f1f2,0xa14b3326),LL(0x87f78ad1,0xc9bd8b41),LL(0x0b59b9c5,0xd378d02a), LL(0x35c0dc14,0x418a32a5),LL(0xf53d85af,0x4c96979d),LL(0x08eb4b26,0xb6f40e97),LL(0xcaa6252f,0xa21349ca), + LL(0x5de38e2d,0xb13d8062),LL(0x9b43c5d6,0x54ea3684),LL(0xb1d6521d,0xc0ad58d7),LL(0x22085236,0x182f8823), LL(0x2a08449e,0x9d50cecc),LL(0x17ab0b68,0xeb85e785),LL(0x8d74e26b,0xb8a22ab7),LL(0x77d03987,0x77515524), + LL(0x77ad71de,0x117a63f2),LL(0xc94c8c82,0x1cca30d0),LL(0x2f05382d,0xe5fefba9),LL(0x9b4b42f1,0xcc9e8916), LL(0x9fe716c1,0xbe939e13),LL(0x95e38cc2,0xbf2b9c80),LL(0x37adde62,0xf60c4491),LL(0xf4df75a3,0x3eb3338a), + LL(0xfe4d84df,0x16398af3),LL(0xfaf3e5f2,0xed752cf8),LL(0xb4cf0e1c,0x746a4339),LL(0x39fb6018,0xb8bd479a), LL(0x57dffed3,0x3a9a045b),LL(0xa5ae3c78,0x2b46ea98),LL(0xde6b0399,0x74b5163f),LL(0x80e511c5,0x069628a0), + LL(0x1b96672b,0x19cfc882),LL(0x379d1f57,0x2759c76b),LL(0x2269a514,0xa6cc7a98),LL(0x683b1413,0x1bc2015b), LL(0x1bf4be62,0xc43b1178),LL(0x7bf2b0be,0xd2941975),LL(0xc4591cfd,0x1eac3587),LL(0x0e66d495,0x283169e6), + LL(0x052352e1,0xd39bedb7),LL(0xd719cd43,0xb04af7f2),LL(0xe92956d7,0x702143d4),LL(0xa0e5b847,0x53498722), LL(0x574885fb,0xf0e8edc5),LL(0x8b5187c6,0x4d9418ac),LL(0xd2a00019,0x70e99cb3),LL(0xe7f8a61b,0xf0da5be4), + LL(0x7dd34fde,0x52704cbe),LL(0x2926bb6a,0x0fb7224a),LL(0xf2b0af92,0x0d58bddd),LL(0x0e9cad36,0x2f986a07), LL(0x80e3a6f9,0xc85549d4),LL(0x322cb14c,0xa013e913),LL(0xf25ac201,0x8a19cf30),LL(0xffb8f2e4,0x130e4ce0), + LL(0x0ce56c13,0x21ad2c8c),LL(0xb15f6a2f,0x13ed2106),LL(0x9453ce96,0xa217b5f6),LL(0x64e0bf9c,0x93b1cdc7), LL(0xc4fe8e72,0x753d894d),LL(0xf3a3916a,0x46c6cea3),LL(0x383dd581,0xc1fb78e1),LL(0x17376a3e,0x1b7ba1a9), + LL(0x5df66852,0xa1411287),LL(0xa30445d3,0x4e9d333c),LL(0x917568a9,0xb5a26c14),LL(0xe857a6ac,0x885f1857), LL(0x84b1f8cf,0x05fbd3ee),LL(0x1e81e4e1,0x5c1f4097),LL(0x011f30e6,0x43999be4),LL(0xa890719d,0xa8aab3bd), + LL(0xc7088eb2,0x49d598ce),LL(0xe341047c,0x7a892468),LL(0x07cb6075,0x8e69b5c4),LL(0x8c37dc04,0x83d066fd), LL(0x6ffff7ac,0x4fcc6d02),LL(0x7edfb968,0x1afaaf74),LL(0x70d529de,0x2972b753),LL(0x08218b2e,0xf65bff0d), + LL(0x4182b9fc,0x119b3c4b),LL(0x27b5e101,0xcab66591),LL(0x2ab87a02,0xfff2e939),LL(0xeec5949b,0x1c10c40d), LL(0x30aa1242,0x98366224),LL(0xf225a4e7,0x833e9dee),LL(0x992e8475,0x07f1cfec),LL(0x1ef83a8a,0x377a9d79), + LL(0xc6715544,0xaf1d0863),LL(0x1fd71505,0x34dd65c1),LL(0x04fed293,0x74d55c22),LL(0x86d2f3be,0x31b1e50e), LL(0xc09594ac,0x876148b9),LL(0x8900b56e,0x73aace3b),LL(0xa2cf4c37,0x4617258a),LL(0xc6f38a92,0x554e8f16), + LL(0xda0723bc,0xd8594800),LL(0xf3c8381d,0x524452df),LL(0x138ca980,0x846dfa02),LL(0xe2d32e56,0xaa77a80c), LL(0x419c86b5,0x27573fbc),LL(0xb70216c3,0xe7486807),LL(0xc72036e6,0x8b7a685a),LL(0x15fae3d8,0xa1764627), + LL(0x815f379c,0x0a1f2361),LL(0x01ab64d2,0x9811607e),LL(0xff2c75cd,0x31841038),LL(0x474982aa,0x8751674e), LL(0x52a2523f,0x2f32b55b),LL(0xe85f2025,0x6ff8d2a7),LL(0x707b2dcb,0xd2ec31ee),LL(0x6e277971,0xdac81e59), + LL(0x0e78191b,0x5445e3a2),LL(0x8c80db2f,0x134dba0b),LL(0x94002b55,0xe9925a87),LL(0x4293c71d,0xe56fa2be), LL(0xa9d009c2,0x72aca4d2),LL(0x02fb0741,0x0c1219dd),LL(0x208fd227,0x689fbc66),LL(0xe4bb09d8,0x8266f2f7), + LL(0x2a61b8bb,0x1a791f9b),LL(0x3eff4f21,0xb29b31b7),LL(0xab7812db,0x2f97803a),LL(0x880ceb4c,0xdbf27bae), LL(0x45e9db5b,0xecb84887),LL(0x5cb7d0ec,0x3dfd84e1),LL(0x77c0b1e0,0xc89f61c2),LL(0xb7656544,0x7ada1d37), + LL(0x910a966c,0x0bca9585),LL(0x6f12c20c,0x80385b47),LL(0xa4b30374,0xf63a1605),LL(0x104b4783,0x2f91b24c), LL(0xb3ab423f,0x9210f5b9),LL(0x2fd424a6,0xb9aa656d),LL(0xf7e8d640,0x63c615d5),LL(0xbb59cfec,0xd567ff98), + LL(0xf7692947,0x78121697),LL(0xbd9f5ed5,0xb9166739),LL(0xb64b20e2,0x58d9a4f4),LL(0xc9fcc93c,0x291898d9), LL(0xd6c6065a,0xbce6509e),LL(0xb84834a4,0x39af658f),LL(0x94b49185,0x0f919d44),LL(0x5dbe7308,0x3b80fc51), + LL(0xe321c228,0xb9fd8ae4),LL(0x360692ba,0x4a46bd2d),LL(0xd05b84b0,0x91d5396e),LL(0xd6b058d0,0x266e89fd), LL(0xb2c42e38,0x6fb142d7),LL(0x994ebc2f,0x93c9fe18),LL(0x104b04a3,0x90e17885),LL(0x654eb6ac,0x6a5fa420), + LL(0x3f349b26,0x26c8a9b4),LL(0xb4e528ae,0x39387f7e),LL(0x5eb46726,0xa74bea43),LL(0x9150b043,0x0b3e82dc), LL(0xe2fc799f,0xc69ffac9),LL(0x48921338,0xd0479697),LL(0x0a4e061b,0x91a68264),LL(0x3f410bcc,0x93a6c41e), + LL(0x6b1fb104,0xaea8d055),LL(0x31fe146f,0x2ff339a4),LL(0xcf63c413,0x3d7ef85b),LL(0x289a097b,0x1f0f57c5), LL(0x5bda1160,0x82f2f83b),LL(0x6fea66e8,0x433eea4d),LL(0xcae11651,0x1f3fff4f),LL(0x6b1c243b,0xfa71c3fd), + LL(0x674832a4,0x59f36add),LL(0x2891e4e6,0x7b6d3802),LL(0x084fa3c6,0x47b313bc),LL(0x6584c9c0,0x90003ac6), LL(0xbc802849,0x9718c2dd),LL(0x2870ca08,0x9a5a2698),LL(0xcf68f352,0xb5cfe625),LL(0x6e6b0caa,0x90d0e2ed), + LL(0xba64d50b,0xb30780c3),LL(0x7acb4fca,0x16328345),LL(0x84b258de,0xf64e01fd),LL(0x35dcd2f1,0x2a25873e), LL(0xce4b39da,0x36606813),LL(0xa69a93e3,0x5285c91e),LL(0xdcb501d6,0x4da13aaa),LL(0x52e3dc24,0xb90d0a52), + LL(0x60a57d0f,0x6882d15e),LL(0x167612fe,0x52142caf),LL(0x463d39cc,0x532ccfb1),LL(0xe5a969f3,0xcdecde85), LL(0xd1bc4480,0xa89c1d1d),LL(0x83f32199,0x9373f362),LL(0x6d653c44,0x42f3493d),LL(0x6c80e27e,0xa867e4db), + LL(0x5cb7623d,0x954fbd83),LL(0x0b83d55c,0xba8b3007),LL(0xe2b23256,0x71946b92),LL(0xfaf95492,0xe0a2a7bf), LL(0x4e0c81ef,0x32ed3d91),LL(0x46f058d6,0xb8c8b14c),LL(0x67221924,0xc76c917f),LL(0x2ddf3cd4,0xd26c1d51), + LL(0x4fc9b14a,0x184e1395),LL(0xc1969b8b,0x651a0c29),LL(0xc9d5bf9c,0x05687179),LL(0xebcd85b6,0xb2f18ed1), LL(0xe446f1ef,0x8b662764),LL(0x71699f5a,0x6c0e051e),LL(0x27d93da8,0xf94a1151),LL(0xa05fe7a4,0x751235c6), + LL(0x624e9ae2,0x40aaf88f),LL(0xf5f6e5c5,0x6499b3f5),LL(0x98157f61,0x01fb0b8e),LL(0x33827220,0x070438f3), LL(0x50ab0b43,0x7409012f),LL(0x63c50e65,0xdbbba563),LL(0xc0d084ad,0x6b572ca3),LL(0x7b76cd6c,0xf10f6684), + LL(0x0c34363b,0x32bcca97),LL(0xb40e8157,0x7a9cef10),LL(0x6eaec234,0x3d5ffc51),LL(0x5f23d481,0x7d7b41a5), LL(0xeecdfe73,0xe5276c22),LL(0x8ac8c30d,0xa9b2725b),LL(0xed0c743b,0xee449588),LL(0x48df73b7,0x6d3b82a3), + LL(0x023cb0df,0xcb52edc2),LL(0xd5a24591,0x08773a4d),LL(0xe12a9072,0x0d9a6aaa),LL(0x5bf5586e,0x4261f56f), LL(0x60a08106,0x184b0402),LL(0xb09cfa61,0x1b398053),LL(0xd5dae483,0xdf7f55b1),LL(0x86ef2cde,0x9554210e), +}, +/* digit=12 base_pwr=2^84 */ +{ + LL(0x9204db30,0x564d6e85),LL(0x2aa84cdf,0x139bb928),LL(0x88476456,0x9413d7ea),LL(0x5a1ffa66,0x5c554483), LL(0x2ed18080,0x7b863089),LL(0xd14e5daf,0x589aaf20),LL(0x7b5f81ca,0xeee4f96f),LL(0x1bb0b415,0x88d47007), + LL(0x55c9bd11,0x1bb400d3),LL(0x06fc2851,0x8402465c),LL(0x65063b3e,0xa81ba22d),LL(0x6e1aa0c6,0xbab2dcbc), LL(0xbe645e25,0xe5f43f1a),LL(0x4df84be1,0x62320533),LL(0x21a2eaf4,0x14ac7080),LL(0x58beb26f,0x3f946464), + LL(0x7a82d20f,0x5f2a3e9a),LL(0x191011f2,0x399e015c),LL(0x886ac8e6,0xfbec312a),LL(0xeda47f96,0x0dd5140a), LL(0x26b47318,0x0d4df313),LL(0xe6685ec8,0xe2c9ec78),LL(0xcd8442cd,0x4df119ae),LL(0x7b32a1cf,0xdb1ca955), + LL(0x126506cc,0x7e2c5852),LL(0x08b3567d,0xba94aac7),LL(0xc05a3f24,0x6905cdf4),LL(0x3547f8b3,0xbf5f559b), LL(0xaade7a1d,0x9e4b4e62),LL(0x1fda3088,0x56b8b9d6),LL(0x4c43d89f,0xea3eb4c6),LL(0x9c69e047,0xfb7e537c), + LL(0xdfe5f6ab,0xc23d9491),LL(0xc1a9c0af,0x42fc362d),LL(0x127d2b35,0x04170b01),LL(0x04116aeb,0x4f0f17bc), LL(0xc9184cf6,0x716c01df),LL(0x895ceae7,0x914dc877),LL(0x390bff2e,0x696b2ae8),LL(0xf88af5db,0xf6ccd628), + LL(0x0f88095a,0xdada9bb9),LL(0x919ce305,0x7155c28f),LL(0x6d78b266,0x32a01e47),LL(0xb652c4f8,0x6da94459), LL(0x827ea8ef,0xa31783a6),LL(0xbdb1af2b,0x4d69b7c6),LL(0xaf31dab9,0x2874eb38),LL(0xafd9bace,0xa0ed9910), + LL(0x4037f17e,0x7d892e3a),LL(0x5f91a4fa,0x81fa9841),LL(0x961cf02f,0x17c7292d),LL(0x388bcc75,0x35af0c0e), LL(0x127a29b0,0x340bec90),LL(0x3d087445,0x955714a4),LL(0xa587c273,0xfd430880),LL(0xd24dfda2,0x715ecd50), + LL(0xaafd6cef,0x4ade066d),LL(0xf8c1decc,0xce59c8de),LL(0x77b96ece,0x3e12a24a),LL(0x44cc710c,0xee7c32fc), LL(0x240e9bb7,0x70700e4f),LL(0x6a63b06e,0x837ada54),LL(0xd19644ee,0xa58ce980),LL(0x27e7451c,0xcaa5d14d), + LL(0x387272fc,0x8e78d2ed),LL(0xfd8a0f13,0x9163a377),LL(0x635c55f0,0x858436bd),LL(0x5ba5b0eb,0x0a414f9b), LL(0x7d7383b1,0x2b58373a),LL(0x6030a376,0x5e7b9d36),LL(0x543514ef,0x9c69af86),LL(0x26080ff3,0x044698cc), + LL(0xa2e23074,0x76f54954),LL(0x17526081,0x90393264),LL(0xf3b78a50,0x0d095055),LL(0x69d8b26d,0x1f3a3776), LL(0xf5e7c8fb,0x0575e3bb),LL(0xee40b0c5,0xee7dd406),LL(0x55dab556,0xe6522e5d),LL(0xb61cd918,0x2d1b5709), + LL(0x01400b8d,0x0ea9278e),LL(0x6464f584,0x9552e745),LL(0x12fc094f,0x67f5645b),LL(0xde303128,0x77c40f3c), LL(0x0e3f3381,0x16d7e9a5),LL(0x59947693,0x017795ab),LL(0x9222eaf5,0xb69b5708),LL(0x1b77f122,0x61b213e0), + LL(0xdc8db00e,0xa7cc8bbf),LL(0x3aa7fc1f,0x1c51f5e4),LL(0xb4ac2d0c,0xb85b782e),LL(0x0468e5ea,0x32fde94b), LL(0x7f7ff0a9,0x8ad5b9a2),LL(0x8fdbb3f9,0xcd26f418),LL(0x6ebf89db,0x853bc95d),LL(0xa066b849,0x1da0a323), + LL(0x4bce0fa7,0xc4cc7aab),LL(0x6bc940f1,0xd4a05b69),LL(0x392dbd11,0xc77300e6),LL(0x21f70aae,0x0dc2eac6), LL(0x4b2ad7e0,0x9d4b513b),LL(0xa6daee1d,0x19822167),LL(0x69b98eee,0x7d71d202),LL(0x35f3f150,0xdfd435dc), + LL(0xddfd45ed,0x66d46ad3),LL(0xe50a2f01,0xf0325189),LL(0x3ec5683d,0xe19b9500),LL(0x91dd97e9,0xc46ab0a2), LL(0xed682c4a,0x74c971d7),LL(0xa14da289,0xafedac2d),LL(0xe39ba740,0xd17838fe),LL(0x053536bc,0xeb497bca), + LL(0xde6d4c38,0x551ba4ca),LL(0x4f52298b,0xa67be247),LL(0x9a5b40a8,0x98413188),LL(0xbb0acfb5,0x083a26aa), LL(0x11d16ebb,0x4929ff5e),LL(0xa942ae7e,0x91f08b63),LL(0x876663ec,0xaa428ef3),LL(0x1e97cbb2,0xfaabd309), + LL(0xf1edd62f,0xca0ed50c),LL(0xd29f48d9,0xc3c7ae6f),LL(0x8a72ae88,0xff47bf28),LL(0x348c6666,0x584ddfe5), LL(0x36731fdf,0x271137e9),LL(0x88d98bc8,0x714bc7db),LL(0x0da6be30,0xcea912c1),LL(0xbe62d6a5,0x91cb844d), + LL(0xec027bfa,0xe16ca42a),LL(0x17603e76,0x0c88f701),LL(0x63d5a31a,0x799418e3),LL(0xebb063f6,0x033bb53b), LL(0x625d3909,0xbcd05461),LL(0x85f23129,0x2d7b7868),LL(0x95090997,0x23b07887),LL(0x18d2c218,0x216c08ae), + LL(0xeebdbcf9,0xe1ccb6c1),LL(0xe873842e,0x89ca4552),LL(0x3c2fcdd5,0x4837f137),LL(0x108a8c0a,0x805874e8), LL(0x3d442fa7,0xe7e524f4),LL(0xf8131f8a,0x580d82be),LL(0x93d3d50f,0x6dcb7d27),LL(0xb5b39168,0x51207d3e), + LL(0x09110fe9,0x9a3ce117),LL(0x48721d93,0x8f3c6e4f),LL(0x87bdfa61,0x60a62b48),LL(0x7c01d84a,0x086dac65), LL(0x53841493,0x4af7878c),LL(0xb3bd5aa1,0x3b1a8935),LL(0x902e5686,0x65c8445b),LL(0x2e3b1822,0xde16cfa5), + LL(0x0a3e3684,0x19879e78),LL(0xee249180,0xec553912),LL(0xf8f4c1ee,0x8eb73fae),LL(0xb81fd20d,0xdee59877), LL(0x20b5ece3,0x2452e63f),LL(0xb632dddb,0x17be9422),LL(0x94311e6d,0x01f89220),LL(0xa332f84f,0x8f0fe052), + LL(0x1b9784d5,0x59657aab),LL(0xd8a7f347,0x6f2ce032),LL(0x6b95e6e9,0x84247793),LL(0x4395b044,0x34301cf4), LL(0xf7fb5401,0x98ebfd98),LL(0xfcdb31a4,0x14fd494b),LL(0xf90e0481,0x042f89d8),LL(0x4134ab52,0x6b90a008), + LL(0x7fe2ffec,0x8fa22555),LL(0xa778448f,0xc6dc3d32),LL(0x85f45aad,0x4886fedb),LL(0x51704d0c,0x5bdef90e), LL(0xe2d1fdaf,0x46ad596d),LL(0x04126f0d,0x914e0090),LL(0xaef960a6,0x71aaeb18),LL(0xac77472c,0x8f4601e5), + LL(0xd8d9768c,0x42e5a186),LL(0x00f6004f,0x8cbf3a6c),LL(0xc1ddebdc,0x9d4bf5ac),LL(0xa9c066fb,0x13354792), LL(0x923fe808,0x72e0b81c),LL(0xc526d6e4,0x1e73b868),LL(0xa81f1e24,0x3f7bedc6),LL(0xe920ba24,0xed1ff363), + LL(0x659604c5,0x58234c89),LL(0xce4b0872,0xa6a421ad),LL(0xcc19578f,0x5dc8848a),LL(0x4f28bdfc,0xfcb418d0), LL(0x8d6442f5,0xf2e74820),LL(0x4dcf6378,0x0c481d85),LL(0x4556438b,0x4987d1a6),LL(0x3157c6be,0x76359363), + LL(0x1c1dceef,0x29bbf3b7),LL(0x576f1dbd,0x0995c340),LL(0x8fa61304,0x0405db3d),LL(0xcc7d345e,0x63438f3d), LL(0x942120e5,0x688174dd),LL(0xcd70c93c,0xc7dd05bd),LL(0x5e871ae0,0xdc8a32dc),LL(0x6178647a,0x1a7896b9), + LL(0x59c437e3,0x1fc3f7a2),LL(0x24235e5e,0x737de2e3),LL(0x7a5eaabd,0x589a56e3),LL(0xcca140f3,0x5a79da8e), LL(0xa12463fa,0x3d8b0d82),LL(0x0875daf5,0x63fc83d8),LL(0xbd9211f7,0x42a30803),LL(0x32d3935f,0x62f6167f), + LL(0x6f269922,0x70cd6467),LL(0x96163b47,0xf694ca21),LL(0x5f5ba669,0xf3bafb2d),LL(0xb8ed8333,0xcf7cf341), LL(0x9997edc2,0x34b2022d),LL(0x309c6508,0x57e6f4b5),LL(0x64841008,0xf6fbf864),LL(0xed075d44,0xbc9821f5), + LL(0xf37cc6b7,0x78c80f73),LL(0x6ab88fc2,0x41d28626),LL(0x58ca26fc,0x2126981c),LL(0xbe3dbf87,0x7a956c64), LL(0xce0ce9f3,0x2f41e27d),LL(0xf4c98e5b,0x0cb49ae0),LL(0xcace473e,0xba6224a6),LL(0x393e092f,0x25dddbc0), + LL(0xa4fb974d,0x747daf46),LL(0xc76dbe2e,0xfb775fe7),LL(0x9670c22e,0xb7b3ad6d),LL(0x10a380bc,0xc6580b23), LL(0x92087c3d,0x4ea226f5),LL(0xb53aa3c7,0xe67c379f),LL(0x991c3c9b,0x4133f831),LL(0x4fa0dd18,0x80f9e5bd), + LL(0xc6f80fb4,0x0094e7c6),LL(0x351bebd3,0x16e99ebc),LL(0xaae16a6f,0xc555ed44),LL(0x2f6367eb,0xe9d2846f), LL(0x83d46d0f,0xb34c93d0),LL(0x894fadc6,0xc0cb137a),LL(0xab31f937,0x21e289f8),LL(0x1bc72a35,0xac5e0516), + LL(0xf3d4db0d,0x6221871b),LL(0xa039826c,0x72d1fdce),LL(0x668c8022,0x69d9cc8b),LL(0xfee064ff,0x0bf359ce), LL(0xe8d16f19,0xb8e636b7),LL(0x443160ac,0xde88f403),LL(0x032836ee,0x4228177a),LL(0xe9801d86,0xee8fac37), + LL(0x4626e343,0x496c9363),LL(0xf4e4c8fa,0xf6999578),LL(0xb8648a06,0xce7306f6),LL(0xae7996e5,0xe2775c8c), LL(0xbf09d221,0x7b47e678),LL(0x515c2ace,0xf5251e1e),LL(0x77b48b41,0x087f9121),LL(0xeb38d74b,0xc40e7725), + LL(0xce95134a,0x1d559f4a),LL(0x320c8bc6,0x1048a1bc),LL(0xe3085f1b,0xad2ddaf8),LL(0x0ad35636,0xf1cfc4cb), LL(0x57db1e96,0x2bd8d4fb),LL(0xe1976ab7,0xd1813026),LL(0x15867022,0xa80e501c),LL(0x01f68017,0xecaf1497), + LL(0x48ab68b7,0xd82c5e79),LL(0x204d2962,0xa0f117e4),LL(0x7dedbf05,0x99b3bda1),LL(0x52786ecd,0xb872dbff), LL(0x57592d3c,0x56253c32),LL(0x4d570c07,0x495fbb05),LL(0xfaecad3e,0x073c49cb),LL(0xb46bad46,0xec8c1f57), + LL(0xce3b07c7,0x13800a76),LL(0x0ffaec55,0x9bbf87d7),LL(0xaf2426c3,0xf69a9ee3),LL(0x2fd70c22,0x2d0c201f), LL(0xc42bb661,0x957e5be1),LL(0x1dc771df,0x3e6ae19d),LL(0xe3cfafa7,0x60af970d),LL(0x5ebd1883,0x721ce869), + LL(0xb87d0ede,0xab0a80a5),LL(0x2954a3e3,0x33576f02),LL(0xc413fc00,0xcc2fe8c0),LL(0xeb86a18b,0x5ae762bd), LL(0x3fe6c6dc,0xbc309dde),LL(0xbf0d1eb5,0xb4f9d001),LL(0xd4fa748c,0xf3f3c5b9),LL(0x2ca78fdd,0x78e8867f), + LL(0xcdf1624b,0x8f85f872),LL(0xa7db0248,0xfdce003b),LL(0x1ad9866b,0x0ad33ef7),LL(0x296248a4,0x27d12937), LL(0xc99c656a,0x23bf35eb),LL(0x17753ace,0xcfb64da2),LL(0x6fbf7969,0x8bc0e741),LL(0xe719cff9,0x131018ef), + LL(0xd1c02b67,0x98f4ef66),LL(0x1f81f578,0xe8aa6cdb),LL(0x159469de,0xa6f97fb3),LL(0xe3906d9e,0xf8e834cd), LL(0x71bbd3d1,0x33ccda6d),LL(0xf952c038,0xeac76a4a),LL(0xe5b53383,0x2891eaa0),LL(0xedcf6de7,0xd629dbdd), + LL(0xa3fb0fa1,0x4af093cd),LL(0x0d1ea294,0x130fd057),LL(0xb57747bf,0xb553cb13),LL(0x024e856b,0x107c0f0e), LL(0xbd631fef,0xfd63a2ff),LL(0x12c01222,0x8df62ec2),LL(0xc0af11a9,0xacbce197),LL(0x5c4922b5,0x35fa3e80), + LL(0xc3de57ba,0xbc257ccf),LL(0x293ad2df,0xb481ca1c),LL(0x2058e222,0xb123f3bb),LL(0xefe46989,0x219cde82), LL(0xe9a316da,0x58ac87b8),LL(0xd4d25c91,0xa8294237),LL(0x62d14158,0xb54dad88),LL(0xb3da2a84,0x9250885f), + LL(0xd54776bd,0xb4e3bedf),LL(0x78043ee5,0x81a4c582),LL(0x4eb87133,0x279a0963),LL(0xf2bfdb52,0x827d333c), LL(0xed71e119,0x3601c6d1),LL(0x0d64df1d,0x3d9b1772),LL(0x3fa3c40e,0x2f5bcc09),LL(0x8e26aef5,0x74b7b30d), + LL(0x3d3ac848,0x98fd949b),LL(0x92e259f1,0xd99e99d0),LL(0x8d353c77,0x34404265),LL(0x4d8dfb1f,0xffc05a7d), LL(0x4e9d92c9,0xbaf2f471),LL(0x5ea9cef3,0xf354f8b2),LL(0xb8b2c8a0,0xf2be0fea),LL(0xfbce308f,0xa392d3e3), + LL(0x02619258,0x58cd793d),LL(0xfea6eacc,0x16a8c9e7),LL(0xb90f9cb5,0x3fcae1ed),LL(0xd59bc4ce,0x1df76d07), LL(0x8574a3ce,0x39248217),LL(0x03b6e82e,0x9d0df2b7),LL(0x33206733,0x64227c0f),LL(0xb342da7d,0xb909614f), + LL(0xb8e15a20,0xe46e977f),LL(0x744eaa18,0xdf2aa89d),LL(0x7ff12f89,0xa40b36b7),LL(0x86b0e7d4,0xbf7ed788), LL(0x9e044a5b,0x35930c5c),LL(0x4ac6b8a0,0x599cfa2b),LL(0xa8f06839,0x68f5f40d),LL(0xe8a1b9d5,0xe838649b), + LL(0xdd45feda,0x2e3c91a9),LL(0x58de0625,0x5f73aa38),LL(0x7535cddc,0xcc2b2397),LL(0xca7825fa,0x60e69d0b), LL(0x62424bd7,0x8f1a95c4),LL(0xf6f21e23,0x5e175a13),LL(0x4fa48b20,0x594e5b82),LL(0x9b14fed3,0x2bfed204), + LL(0x74484bc3,0x87c925fc),LL(0x5639abc5,0x052b634f),LL(0x290426dc,0x169549b6),LL(0xdaaefd38,0xfe515a22), LL(0xb4d87ccb,0x8a63a39c),LL(0x4034acdc,0x3dec5f62),LL(0x61090db0,0x59969d81),LL(0xf157248d,0xb089b8f7), + LL(0x9d59a29f,0x42b0ca54),LL(0x9be7ee82,0x522b3e3e),LL(0xac166a7e,0x894aade2),LL(0x9184ec33,0x57aaf19a), LL(0x5e50711a,0x84406a11),LL(0x1614f8d3,0x0cafd148),LL(0x3f7d19f8,0xc6174fdc),LL(0xff4958be,0xca5bed9a), + LL(0xe4fdd396,0x8dc18aaa),LL(0xd371c3f4,0xf6e8a9ee),LL(0xa5dfefde,0xc6b58042),LL(0xfc4f3577,0xccc3bbb6), LL(0xdedfdd55,0x9f583e4a),LL(0xb48c5fb2,0x9ea45133),LL(0x232d61e0,0xca2b3229),LL(0xb0b5cb38,0x642101a8), + LL(0xa9ebda1a,0x0cfac5fc),LL(0xd2dc9c7c,0x02398bd6),LL(0x80591234,0xd95511d9),LL(0xe8230901,0x0e5cc99c), LL(0x140eaba1,0x943350f6),LL(0xe0623c93,0x9fe19108),LL(0xd74e189b,0x052bf5d9),LL(0x40cd7173,0x3e341bff), + LL(0xcb7d384d,0x89b5b355),LL(0x50b76f18,0xedee32da),LL(0x5804d9df,0x6a9cfb19),LL(0x376fc2d8,0xccf638f8), LL(0xe14de014,0xebdce7a5),LL(0x7f606fa5,0x0135085f),LL(0x69b58c3b,0xf8a3de5f),LL(0x59ca19d1,0xbaa80445), + LL(0x0ce7238d,0x3252147d),LL(0xd57bc36f,0xd446960b),LL(0xb275f5ca,0x9b1743ce),LL(0x27629de8,0xda048c48), LL(0xd3bbac67,0x005354db),LL(0x1ba1facc,0x62c392fb),LL(0xa18da892,0xb066bfae),LL(0x367a8320,0xdb090711), + LL(0x6f90249a,0xbb7092e2),LL(0xe22da86b,0x6de14155),LL(0xb38d4ad8,0xe16136d3),LL(0xd0fbb940,0x9deaa5c9), LL(0xaacf50e3,0x54a54ba3),LL(0xb9ba4570,0x66e5645a),LL(0x48cb742a,0x77e28d94),LL(0xed98a2c9,0xc795b138), + LL(0x1daa17ee,0x899331f6),LL(0x4a77734f,0xac950653),LL(0x71f3e3b6,0xd7f6304f),LL(0x65fc119c,0xe7256955), LL(0xbe527794,0x3e60a04c),LL(0x7c578fb0,0xdaf53be4),LL(0xebc0754b,0xf785a4f8),LL(0xde1b78b4,0x8b21b116), + LL(0x62fb1c56,0xfe47e04f),LL(0x229f1017,0x8a92f9e6),LL(0x68b7842c,0x2d73dd23),LL(0xa56dbc4f,0x3b43f7dc), LL(0xd0f3f4ca,0x9435defe),LL(0x500594e3,0xdabfb1ba),LL(0x428f5ead,0x70e929e8),LL(0xbdc7716e,0x44adf585), + LL(0x02204867,0x7b7ff077),LL(0x0c78476c,0xf2f306be),LL(0x7e783793,0x48849fd5),LL(0xaf77e3c7,0xc2dc3c7d), LL(0xa980cdf6,0x5eb2b691),LL(0x204e25df,0x7ca7b7a4),LL(0xc5070eab,0x1e7c2f82),LL(0x4eb7cd3b,0x32ca4b36), + LL(0xf94ad1ab,0x38ffde8f),LL(0x59921b25,0xb4757ae1),LL(0xb4d2f624,0x856cd3f3),LL(0x1eb40708,0x90593929), LL(0x1193b3e4,0xffc4b89a),LL(0xbd2f804f,0x6afba7a8),LL(0x69dc21ed,0x72aabbaa),LL(0xe7fb6de1,0x5d1da32e), + LL(0x98d1e26b,0x56c0f440),LL(0xf7cc7d6c,0x9456a6c3),LL(0x14f2f24d,0x9eb0aebb),LL(0x7dd788a5,0x51d7c699), LL(0x46a22e97,0x053b8098),LL(0x8c025be8,0x27d8ea2a),LL(0x10d5afaa,0xe0bd464a),LL(0xe7cf120c,0x137c452d), + LL(0xd091397b,0xd06bd227),LL(0x21bc796f,0x4b307bf3),LL(0x7f5a37b0,0x701eaf3a),LL(0xac7d4718,0x8d5a0f61), LL(0xed8b1a37,0x0cf9eea3),LL(0x2aa9061c,0x10854f10),LL(0xa30eb4e6,0x0aaf430c),LL(0x2a050dfb,0xb74342f5), + LL(0x20e1899f,0x2feee9d7),LL(0xf2a1dbfc,0x49464a8e),LL(0x5762d68e,0x4d7cf25e),LL(0x7bf43462,0xe7b6e759), LL(0x79daf6e0,0x71fce284),LL(0x03858705,0x2d3ff71f),LL(0xbc4af4e6,0x07d8d288),LL(0x18f1c7d4,0x6777d197), + LL(0x0e85f036,0xb5770041),LL(0x4c8d9768,0xe1bb263e),LL(0xe3917798,0x4fcc1d44),LL(0x07abcde4,0x274d1d90), LL(0xb7a10472,0xc9b8ae9f),LL(0x8d10e8ec,0x6632e3be),LL(0x50f3a172,0xb6876fb0),LL(0xb4cf4867,0x753692d4), + LL(0x58e598f7,0xfe3624e6),LL(0x6d81fb40,0x15f90418),LL(0x9bea3649,0xae762f7b),LL(0x161e85cb,0xc48d2262), LL(0xcf5a21f0,0x8e8726a1),LL(0xa1f6653b,0x536c441f),LL(0x67ec5b86,0x0716bad0),LL(0xb2147d1f,0xa423a957), + LL(0xdca2e393,0x8eec96c8),LL(0x2843ef12,0x3619e36d),LL(0x2ef695e1,0xdc16fe2d),LL(0xffea8124,0x04ed2cad), LL(0x180ce636,0x5018a0ce),LL(0xdce7b2f8,0xc34b0bbf),LL(0x0c54fc30,0x645a02a9),LL(0xf3f819d9,0x6ee6772b), + LL(0x7cecded6,0xe2bbbdcd),LL(0x3f038851,0x9ae4fd55),LL(0xa2f316c7,0xc30664ab),LL(0x63ffb50a,0x3cccf4a1), LL(0xd00fb8f2,0xc37ee6ca),LL(0xad906eb1,0x593db6d5),LL(0x4aa84505,0x8f75b594),LL(0x9e5939f0,0xeff39d82), + LL(0xc064f530,0x4b7fab3c),LL(0xde175892,0x731153ae),LL(0x3d4c4e60,0x335e6503),LL(0x776ce13a,0xb0876a8a), LL(0x22241ecd,0xa8a566ee),LL(0x011e861c,0xb7456b3e),LL(0x177dd490,0xa9aff4eb),LL(0xc8f77c40,0x189b1ed9), +}, +/* digit=13 base_pwr=2^91 */ +{ + LL(0x2857a1fc,0x624de687),LL(0x2ff8f505,0xbd0a0d9c),LL(0xc381bc9a,0xeecb4fad),LL(0xfa94e41b,0x72386292), LL(0xe75fc753,0x354d3f83),LL(0xa7a5a6bf,0x06afc753),LL(0xb2f568dc,0x1ce792ee),LL(0xbd2f9647,0xc5faaee3), + LL(0xf912b74f,0x175fbeb0),LL(0x6e0ceedd,0x45fbe8e1),LL(0xd9233ee7,0xf0e1aa68),LL(0x406a626e,0xe55fc1ce), LL(0xe08712e7,0x20efa1b9),LL(0xbcfd6360,0x5fd108b5),LL(0xeec1edac,0xea431df6),LL(0x940803f1,0xae1c0521), + LL(0x15407ffe,0x584a16d0),LL(0x08a82a69,0xa977f702),LL(0x67f8a198,0x52eefecf),LL(0x19f7a7e0,0xec213738), LL(0x35987b9a,0x6795cfef),LL(0x97028480,0xb243403b),LL(0x9c1b9124,0xac24b12b),LL(0xa90f8aeb,0x1f379501), + LL(0x64bc0f09,0xa8e97fb6),LL(0xc953cd08,0x0b913991),LL(0x7fc3bf00,0x8385a1b3),LL(0xb09ccd8f,0xb6e74dec), LL(0xec473ea7,0x6e1df026),LL(0x530766bd,0xf2f7fbbe),LL(0x3292052b,0xf18cb47a),LL(0x9114866a,0x7f8d4592), + LL(0x8bfa2c22,0xf0a1c565),LL(0x2b326c0e,0xc28518c3),LL(0xec107d66,0xabafc6f0),LL(0x8478907a,0xbc7a6abf), LL(0xa2920288,0x8c1c8f6a),LL(0x930c043e,0x6c87579d),LL(0xb309696d,0x25ee808d),LL(0xb7a71041,0x433bbbda), + LL(0xb3086691,0x48d6d957),LL(0x26640916,0x9946a29b),LL(0x43db59a9,0x932ca93c),LL(0xe4fe91ba,0xaa61a0c5), LL(0x815bf003,0x9e22e112),LL(0xc86ba8d3,0xa9ed1b18),LL(0x1069f434,0x1b5d3c14),LL(0x1cc01754,0x3cd2ebd0), + LL(0x3350f670,0x5c06b244),LL(0xf6f9c751,0x7557dc9d),LL(0xde66fd97,0xa7ebd3b8),LL(0x2befe6fe,0xc126dbaa), LL(0x396f434a,0x312f4897),LL(0x61a4124d,0xe05cfcd6),LL(0x1525c05e,0xc83b8688),LL(0x11899f64,0x4646dbf2), + LL(0x8e419e08,0x2b7507cb),LL(0xaf855eec,0x785328d7),LL(0x7b8683a5,0x875db0c7),LL(0x90a597e9,0x3d1bc968), LL(0x47eeeab4,0x7d4afa10),LL(0xd680ca71,0x2668dd43),LL(0x17365023,0xc3210d1f),LL(0x17fb31cc,0xd5bb2ee4), + LL(0x08e9ba09,0xbefb6a4f),LL(0xb0c1b6e1,0xc6beedb8),LL(0x3510ef35,0x59daf057),LL(0xdbbabc65,0x604047cf), LL(0xa06b7340,0xfabc80a8),LL(0xdf765977,0x7433dee7),LL(0xfd807cfb,0x149a2c4a),LL(0x3480a086,0x14e8ad3b), + LL(0xb22c5f89,0xb0c3156f),LL(0xbf78675a,0xd10ece4a),LL(0x80b8ad9f,0xe270e317),LL(0xb0c2b420,0xfe7a6210), LL(0x125ef635,0xf091d738),LL(0xc1a6f202,0xf1f277d6),LL(0x3587d9bb,0xe2727e7b),LL(0xb3e2b84b,0x83b209a9), + LL(0x7a13effa,0xc9eb445d),LL(0x0d697480,0x89b856f1),LL(0x25c03cb7,0x834bbae2),LL(0xe0b4a7b2,0x0d8adb85), LL(0xc7fbc240,0x7b6884af),LL(0xaa4f9097,0x6b485409),LL(0x290c106f,0x4d0a367f),LL(0x3f0efdfd,0xab87d218), + LL(0x50f2b65b,0x15b9bab7),LL(0x5e5d53e4,0xa7403d4b),LL(0x28529212,0x2e23e376),LL(0x6e050767,0x6fe903a2), LL(0x6cf570fb,0x4c5291a1),LL(0x7a30b326,0x4bfb8607),LL(0x27c572a9,0xec4905f8),LL(0x0f381c31,0x72eeb8c9), + LL(0x460adca0,0x33346cec),LL(0x7b34756a,0xd4d5bba8),LL(0xeac84add,0x02b2e2d4),LL(0xdc1053b5,0xa129845b), LL(0xdca6f9ce,0x53f067e0),LL(0x3526aba6,0x6e9998ed),LL(0x1c0982da,0xa4aef9e2),LL(0x93f5d96f,0xfe5b606e), + LL(0x9c14b699,0x26b8502e),LL(0x0948a291,0xf1bcdca6),LL(0x2aefd415,0x73e43a32),LL(0xd1e2cfb5,0x7f523357), LL(0x97d3fa94,0xa60151c0),LL(0x72129630,0x820c0d58),LL(0x5854acf5,0xb8f2e1ed),LL(0x3c656ac3,0x86d6646c), + LL(0xbef1d0da,0x2284a612),LL(0xa8c8faba,0x2e7c5f4e),LL(0x70303ea3,0xfd441ae7),LL(0x5161cf82,0x9613f329), LL(0x2e19531f,0x65a3cc65),LL(0x34281f69,0x177a2775),LL(0x7c82e094,0x0cc692a4),LL(0xb6f377f0,0x9d62a55b), + LL(0xf96ec2b8,0xa24cf6ac),LL(0xa961cc16,0xd06747c3),LL(0xbd17f0a2,0x57c7001c),LL(0x34afe2d6,0x5f298db0), LL(0xdf12f671,0x51b01ef2),LL(0x5ce712fe,0xc01c5066),LL(0x92a74776,0xac0f4034),LL(0x08d696bd,0xa3e9934f), + LL(0xe7daaff8,0xafb6981a),LL(0x73bdcafc,0x5f8998d9),LL(0xbaf9906c,0x23ec39e1),LL(0xc999c9c0,0x5e248410), LL(0x17dad895,0xd14c7a89),LL(0xcbb3f6b9,0xfde9d01a),LL(0x5f698f1b,0x1d6b26ef),LL(0xf0baff97,0xc6495cd1), + LL(0x587674ec,0x5a72dc07),LL(0xdb09cd65,0x100f9ff0),LL(0xb30cf6e6,0xec0fb71f),LL(0x81066143,0xf54cb597), LL(0x633857c4,0x0090e997),LL(0xda92c5d2,0x7326ed15),LL(0x47c56e86,0x794cd8af),LL(0xf89214c9,0xb272112f), + LL(0x3445879d,0x37960861),LL(0xf2fcfc55,0xc5e496b0),LL(0x6559e153,0xfe74e95f),LL(0x54a772af,0x1e18b2b5), LL(0x157c157c,0xd146980c),LL(0xa11d77b5,0x31ee3f25),LL(0x5707db6d,0x7762a07d),LL(0xbd2022b8,0x00804bcb), + LL(0xd571c59e,0xdf3f4658),LL(0xcf45c1ee,0xc49e7a34),LL(0x43972cff,0xf401ba3d),LL(0xe509c2b6,0x146d989c), LL(0xeb72392f,0x7c68d6c8),LL(0x0658b8e6,0xdd048de5),LL(0x9a0aeb72,0xc9dc61b7),LL(0xb741d644,0x98b080e0), + LL(0xb1c5982a,0xa6ec0eed),LL(0x5ebbc26f,0x58d28317),LL(0x33e5b7dc,0xac8f1e1e),LL(0x9d8f9fed,0x31e4f65e), LL(0x904ad76d,0x6c9af383),LL(0x9bdb0517,0xfc38c53c),LL(0x0e69f85e,0x9ae278ee),LL(0xefd9d887,0x18b362b7), + LL(0x5bbbd3ac,0x65a5f74b),LL(0x077bfb4f,0x41eb4593),LL(0x83b38100,0xb642934b),LL(0xac1a99bb,0x643ceed7), LL(0xee7cd5f7,0x9c27e66d),LL(0x6ddbaa6b,0x2ccf87d5),LL(0x447b1192,0xd51ca739),LL(0x95f5f142,0x78471053), + LL(0x3a650829,0x915f50cd),LL(0x898a6a1c,0xe032bdc5),LL(0x2d15959f,0xde8fb4f1),LL(0xbad56140,0x1fc5fc73), LL(0x8e60c3c3,0xdafbf206),LL(0xe428adb5,0x4963dc95),LL(0xd49584fb,0x1538e081),LL(0xbc0e5fa9,0xb34add66), + LL(0xa7f28b2f,0x404ecf12),LL(0x7fa9c435,0x6ddc3ce1),LL(0x61ee755e,0xda887e3f),LL(0x8f71280a,0x4b5da661), LL(0xdc79a4cd,0xee5a86df),LL(0x99be4d36,0xd8514b8a),LL(0xcc82c751,0x674793ea),LL(0x437aedcd,0xf3a2123a), + LL(0xfcd6f027,0xf825ff37),LL(0xa681a001,0x60a056d8),LL(0xaa92c135,0x92a39248),LL(0xdcd190a7,0x61884e23), LL(0x24cc911c,0xec0d1420),LL(0x5aa16ad7,0xbdb0baae),LL(0x8a1694d7,0xf12726b5),LL(0xc93673f9,0x8c7cf113), + LL(0x7f2edc38,0x02fb6c69),LL(0x2fbe8690,0xcc4d4304),LL(0xe89c80d5,0x405b2491),LL(0x3d938bc1,0xdef46c76), LL(0x2520e3b0,0xd92ec0fa),LL(0x1fe2dfda,0x2501cfa3),LL(0x1d5c8474,0xe7c5753d),LL(0xe6226dcf,0xc059abc0), + LL(0x55a9011d,0x2dceefe6),LL(0xbbbbef00,0x8799064a),LL(0x0b49b5ef,0x7fe944c2),LL(0x225b21dc,0x722bbef0), LL(0xd2bb14af,0x84687cbb),LL(0x9b6f6caf,0xfc4ab4f0),LL(0x2c146a52,0xb7b7bb59),LL(0x1dfea10b,0xb90d67f2), + LL(0x713e1d30,0xca4ca8c8),LL(0xf8a13db8,0x50cbb994),LL(0xa5b0e3e5,0x2bee12b2),LL(0xe71e19fb,0xa7c5d6d1), LL(0x9e0314cd,0x28442423),LL(0x66cda5c0,0xc95c2746),LL(0x1c5ffd19,0xfe52a79a),LL(0x38a0339f,0xb93875cc), + LL(0xb49fb049,0x6a94196c),LL(0xcc12a38d,0xbeb1eb4b),LL(0xf5e8d167,0xbc136771),LL(0xd5f8ae87,0xa91e1232), LL(0x95172460,0xb2e812c7),LL(0xb8f81870,0xc699d376),LL(0xa8900827,0x9e4a3b70),LL(0x506c0b29,0xe0d4b2f4), + LL(0x7246fd96,0x13b4d1c7),LL(0x33965581,0x84ea2158),LL(0x2e53c024,0x9b9f071b),LL(0x864a1b78,0xcb005908), LL(0x3f742c2f,0x03daddf5),LL(0xdf595911,0xd29230e5),LL(0xca0406a1,0x3f7d4e6b),LL(0xb1db7e47,0xeb646f66), + LL(0x590e3107,0xb896003e),LL(0xf754ac01,0x7a0dc361),LL(0xe63ab0ac,0xe877a6f3),LL(0xdf60d307,0xd43b54f3), LL(0x59cf0add,0x65ef91ba),LL(0x18990eb4,0x35e99393),LL(0x8e46fbf6,0xc186ab16),LL(0x8c1eaa91,0x4c0eb22f), + LL(0x1abd31f0,0x4599b894),LL(0x9a1da7d3,0xdb34198d),LL(0xa0f0217d,0xa8b89523),LL(0xe56b884e,0x2014cc43), LL(0x49efd4ee,0x6fb94f88),LL(0x287f4ae0,0xf1b81710),LL(0x99fd2deb,0x89d38a9a),LL(0x72b67a53,0x8179277a), + LL(0x1a03755b,0x0ef6ce56),LL(0xfcdb3469,0x8dc768f2),LL(0xa21d959d,0x0be58a91),LL(0x9b103cd0,0xea44861a), LL(0x808b8a46,0x332e86e7),LL(0x8772c3f8,0x9882015c),LL(0x9f4b5d29,0xe6b272fe),LL(0xa29b023b,0x0e183a28), + LL(0x2286ebf3,0xf2fab88f),LL(0xfce83e6f,0xb7532ced),LL(0xe0cde4fc,0x17999d7c),LL(0xc1b7668a,0x7230fd85), LL(0xef588309,0x97a57d39),LL(0xf906f6e7,0x7e175f28),LL(0x72b70bfe,0x51f67413),LL(0x2f82218c,0x2132f595), + LL(0x9d8727cb,0x9cc0746e),LL(0xbba1ec8e,0xa2af77fb),LL(0x31a67cc9,0xc75aee60),LL(0x57408325,0xaeab9e0f), LL(0xec34bb89,0xf24de697),LL(0x5d958bdf,0x06b90039),LL(0x0603d6cc,0x6f55222e),LL(0x2eb0b239,0x496537b5), + LL(0x8be08323,0x083e5889),LL(0xf8dc0a78,0xc573596e),LL(0xe8901eca,0xc3e988fa),LL(0x6e350257,0x7f7b48f6), LL(0xa216e329,0xed820567),LL(0x8ce989c1,0x55f46737),LL(0xeeab9441,0x7f48c5f1),LL(0x86fe0831,0x1d3cac11), + LL(0x408a0306,0xe0364bae),LL(0x7a4eb2cb,0xe8d8aba0),LL(0x1fd7d5da,0xe548725e),LL(0xed5ed958,0x8de04491), LL(0x61d73977,0x3e75eba2),LL(0x55420386,0x4f580400),LL(0xd859a690,0x54642fa4),LL(0x296e336e,0x2c905f7e), + LL(0x22e260bc,0x4e287e66),LL(0x4a28d5bd,0x71a2ec99),LL(0xa7c5c3e3,0x5528da21),LL(0xa49471e0,0xae9f6856), LL(0x587cd94f,0xdcd8e66b),LL(0x6c7b7db8,0x91afbd79),LL(0x067e3cdd,0xdf2e6625),LL(0xa6375f59,0x15b5a329), + LL(0xb6586c5f,0x3b8b3b1d),LL(0xd34f10fb,0xe4d50a77),LL(0x7c3c01f7,0x26cb86f5),LL(0x8c57e6f7,0x36e9d3cc), LL(0x62c6dbae,0xaa8e7ce1),LL(0x60d7fae5,0x7f6b7689),LL(0xc797ee16,0x519a7659),LL(0xb36a6b1b,0xa1c7b30e), + LL(0x74dff201,0x8da05ba6),LL(0x40d0a835,0xd2eac07f),LL(0x610a7d6f,0x2701eb31),LL(0xbf893c4f,0x5c17a91e), LL(0x6bc8b161,0x68b92e88),LL(0xf52e6ec0,0xa312fd5b),LL(0x6b7952cf,0xf7daf460),LL(0x18aeb57a,0x847f0cf3), + LL(0xb0146708,0x27b178ed),LL(0x54ca2aa5,0x85a23554),LL(0x395a7b16,0x80dd123c),LL(0x0058bfce,0x64a9337b), LL(0xf4addc4a,0xf6ae9380),LL(0x464536f1,0x0f84385a),LL(0x16534f6c,0x41fc2270),LL(0xb8795ec3,0x13d8976f), + LL(0x8e12c560,0x2e90b3e4),LL(0x239b2c58,0x242a86ec),LL(0x0768275c,0x6fb42ecc),LL(0xbd96de9e,0xee341cd0), LL(0x84355d11,0xfd1833ac),LL(0x5f55ec6c,0xf062392c),LL(0xfee33fba,0x6ee7b59b),LL(0xabf86e0f,0x8113f0ca), + LL(0xcc68033c,0x2285aaaf),LL(0x78430646,0x850b732b),LL(0x2b3fa03d,0x50fa4b61),LL(0x3caf050d,0x4d893ecc), LL(0x988df419,0x454764e6),LL(0xfb61f1a4,0x055d8a4b),LL(0x8475e07a,0x3b7c5f4b),LL(0xa6a228e4,0xf93a198b), + LL(0xec8d566a,0xe0a8ce61),LL(0xc55f4bd6,0xe41397d6),LL(0x654bdf55,0x4cc18d48),LL(0x9325ac25,0xe1b49f9e), LL(0x72c68871,0x79840752),LL(0x6d806fe8,0x8930d8b5),LL(0x0bd5f65e,0x11c8b5a8),LL(0xbf37d7a8,0xe931c025), + LL(0xae8af083,0x25b17fd9),LL(0xde4215ed,0xd589fd8b),LL(0x4b3f61fd,0x56378f04),LL(0x6bfb4f9a,0xf0f39213), LL(0xe906cc6a,0x6b0f9596),LL(0x096f8296,0x441f13da),LL(0x1e4940e0,0x08f933d4),LL(0x5a53e7ee,0x6c35391c), + LL(0x19c3d18e,0x5f41c4d3),LL(0x1d389d95,0xc0804e09),LL(0x18a5a3f2,0x7323a9ab),LL(0x410a6381,0x7b7c2475), LL(0xb02cfe18,0xd362eb9a),LL(0x553b2970,0x79ef3d0a),LL(0x3d2acdab,0x371f7760),LL(0x7f241dfd,0x6cd37890), + LL(0xdf4a28e4,0x592a242e),LL(0x1bb45217,0x1e623cdc),LL(0x494074d3,0x5a9633a6),LL(0xd52fbfd8,0x81b74307), LL(0xdec4c5ff,0x98292651),LL(0x3e0f6edf,0xe1b7bc86),LL(0x6bb8fb31,0x3d5fd86a),LL(0x1cf29f19,0xa830e9a2), + LL(0xcf69c747,0xfffc5482),LL(0xa83549fd,0x7748a0f4),LL(0xe7ccf4a6,0xba1c8a0d),LL(0xa2ede6b7,0x6cd1399a), LL(0x87bb90d9,0x8fb634e6),LL(0xc59a5304,0xfa8e659b),LL(0xa9122d95,0xcd6bfc75),LL(0xdfa6d75a,0xdb107def), + LL(0xcc27760a,0xb0ec4cfc),LL(0xbed3a1a1,0xf24c1e22),LL(0x819bffc7,0x4f8522a1),LL(0xa93d97e1,0x263c7b5b), LL(0xa4b4de49,0xab1d31e0),LL(0xebbfe8f5,0x374e968b),LL(0x51ca0d08,0xe82e9756),LL(0x7df3f2df,0xc05715a2), + LL(0x038004ad,0x941f02c5),LL(0xa0fd46d4,0xc136a2a5),LL(0x3d31d26c,0x85db7d24),LL(0xbfefeecc,0x05bba6af), LL(0x5a60aebf,0xf803b539),LL(0x813d0e6d,0x9bb8a479),LL(0x066abdfb,0xb689c813),LL(0x0072e530,0xd93b3f4b), + LL(0x987446ad,0x242140a4),LL(0x06a02f0c,0x40b3f709),LL(0xa0fd6018,0x33f9bf20),LL(0xf21abfdc,0x58517c18), LL(0xc1f80f3f,0xa33dc5db),LL(0x7ec91c80,0xbb7dfe27),LL(0x8ca97dd8,0xd2cf9338),LL(0x32e43d44,0x5761f871), + LL(0xe513ea90,0x3c8ffb0e),LL(0x79bcdecc,0x91ecda36),LL(0x9b1a5514,0xdad3fdd5),LL(0x640d3af0,0x8fb630f9), LL(0xf9d2e0be,0x82949b09),LL(0xeba23663,0x079ca0ff),LL(0x135238d4,0x51e62c53),LL(0xc446bd67,0xf5fa0c61), + LL(0xe8da43d6,0x19dcdd2f),LL(0x95f29b5b,0x46fbf7ea),LL(0x635e8388,0x7f3eaa05),LL(0x5369750b,0x5ef817c3), LL(0xc860c4aa,0x06025893),LL(0x5551c9ef,0xa2f6684d),LL(0xfbc0c667,0xd6af67dc),LL(0xcd2fe44b,0xfd8d7394), + LL(0x302a17cc,0x011968ae),LL(0xc3e5a5cb,0x2206ff24),LL(0xa20dbfb7,0x4c7f0df3),LL(0xa395dd6f,0x59566376), LL(0x373ea76b,0x68ff3d9f),LL(0xf6cf8ada,0x2394f93a),LL(0xe7514a94,0x3acc5dba),LL(0x5ddfa11b,0x0340da7a), + LL(0x1a05155d,0xc3f03022),LL(0x4f7656c0,0x6cbbdc6b),LL(0x0b0875f5,0x6e30dbdd),LL(0x3471b0d5,0x5e7c2883), LL(0x408b4bc8,0x49cfd71c),LL(0xf01c002b,0xd29a184e),LL(0xff415b0f,0x308be85c),LL(0x01a8fe7d,0x1b4176f0), + LL(0x0c33bed3,0xb850acc7),LL(0x23af7af0,0x76aac640),LL(0x21d5853f,0x049187ee),LL(0x6620195c,0x44fbf6e5), LL(0x36158178,0xf0abf14b),LL(0x90e419c2,0x9866ffca),LL(0x9e8523a8,0x7522e277),LL(0x08e90f1d,0x2f2590f3), + LL(0x66d3f75b,0xde1c0c52),LL(0x6c299b57,0x47dc9ceb),LL(0x51f7f2b5,0x4ad12847),LL(0x452b07a5,0xeedf9d8d), LL(0x3dad76c6,0x207b0627),LL(0x5d4c0280,0xccbb5201),LL(0xb019ae8d,0x0bdca05b),LL(0xf2da7eb4,0xb5f8d088), + LL(0x4626c00c,0x4e79a0be),LL(0x5af82c0f,0xf6fdd64f),LL(0x5f7cba0f,0x7a828224),LL(0xb0765493,0xc2214834), LL(0xabd53ccf,0x5b0d0d1a),LL(0xf4a1b517,0x3b03a22d),LL(0x6ece453e,0xb235c862),LL(0xf66471c7,0xf43ac344), + LL(0xa1552fa5,0xeaff93dd),LL(0xcf3ae702,0xef1b40dc),LL(0x9ca613a4,0x35ced3fd),LL(0xa2f33a0d,0x90e350ab), LL(0x002b5738,0x47bb89aa),LL(0x032b8b08,0xafc01bba),LL(0x61588b4b,0x688ae119),LL(0xcf66ef14,0xdb7d820a), + LL(0x084910bf,0x83b654db),LL(0x60ea75a1,0xbb581f8b),LL(0x1cdae7c4,0x6917f282),LL(0xa40a07c3,0xb719f931), LL(0x31297b3b,0xf8efb8b9),LL(0x4d6fc5c6,0x74246ded),LL(0xd2c61503,0x5a111754),LL(0x71654764,0xf64d2b88), + LL(0x6ad8f168,0x4b452692),LL(0x0cc6fc91,0x907beb21),LL(0xbf13c18b,0xe876d523),LL(0x4cf37ca1,0x4d28e457), LL(0x6d3d1172,0x4c0dc22d),LL(0x5a753137,0x7935a8d2),LL(0xda44d652,0x03484e3d),LL(0xc50025a9,0x05a3d80f), + LL(0xff477c6d,0x6d43c503),LL(0x1ccd416a,0x35f4c4cf),LL(0xd5088349,0x7070f471),LL(0x281d30c8,0x678460ca), LL(0xc6defb33,0x8923cd9a),LL(0xe2557cab,0x44930f56),LL(0xad156c4a,0x33b020bb),LL(0xbcaf4805,0xfdab31e3), + LL(0x864b5564,0xffe79bec),LL(0x10c60d52,0x0510e352),LL(0x328a652e,0x66203aaf),LL(0x54fea042,0x9d5403bf), LL(0x6e5e5c7c,0xb3fe6743),LL(0xecc66e02,0x6deef667),LL(0x44eacacb,0x199ee152),LL(0xd8803fd9,0x9f49fcd4), + LL(0x2d3a6e28,0xdd5fee9e),LL(0x686d8ca3,0x8eed56d4),LL(0x78083491,0x36889a27),LL(0xbef20457,0xea1a6555), LL(0xa501e2a3,0xe7e6b609),LL(0xfb23de2f,0x1ea0ae29),LL(0x632c9a6b,0x5f537d07),LL(0x9a3db961,0x61770d1f), +}, +/* digit=14 base_pwr=2^98 */ +{ + LL(0x7497e8a5,0x325c60db),LL(0x8c6949a9,0x05d8eab8),LL(0xc7bd5898,0x3169e466),LL(0x192d8e3f,0xadc06264), LL(0xd55959fe,0x1ff468f4),LL(0x202dba19,0x97b33ee0),LL(0x21cf84bb,0xaa0c3fe2),LL(0x04a8d176,0x48cdc0af), + LL(0x9c0d4009,0x53d8c448),LL(0x2e24dbaa,0xd3714617),LL(0x9b62e5f1,0xdd92e730),LL(0x9922cc8a,0x97b344d7), LL(0x0bfe3e8f,0x416b009b),LL(0xf3c82269,0x56873834),LL(0xe6623555,0xf82a980f),LL(0x5ce68e54,0xb027ecaa), + LL(0xfe87680d,0x005a4b24),LL(0x4cf6ee32,0xd92532dc),LL(0x4bd3e01f,0xfcd27c8c),LL(0xe1b59ffa,0xda7d9949), LL(0x2735d373,0xe3d5f31f),LL(0x288e71fb,0x7e139ca5),LL(0x93979cb4,0xe474bc80),LL(0xf6fcc665,0x7f4f6017), + LL(0xfbd613c3,0xe6982c86),LL(0xc366f17a,0xf4af69d5),LL(0xb85c2343,0x8683eed6),LL(0x24bc2116,0xf5bb244a), LL(0xc9fc77d4,0x997a74bc),LL(0x6f44b54b,0xe202eb91),LL(0xa6997e76,0x77886412),LL(0x02c8837e,0x6996c8fb), + LL(0x2c61382c,0x0986df8a),LL(0x667c8ee9,0x90607b92),LL(0x084eacde,0x051fcbf7),LL(0x6e685877,0x84e3dba4), LL(0xe458da50,0x35861a82),LL(0xcf392b51,0xd036823f),LL(0x3dd86e74,0x43181481),LL(0x3741a385,0x8dcfe17d), + LL(0x40f56786,0x8e1a77cf),LL(0x6d4b7774,0xc5bca7f6),LL(0xc81ec077,0x86b588a0),LL(0x9206354f,0x88952a01), LL(0x43a8519a,0x5444a989),LL(0x2857b210,0xe29dd68c),LL(0x9a144624,0x36658903),LL(0xe423e899,0x8c4dedb0), + LL(0xea886e5c,0x482040c5),LL(0x1cdd50f7,0x42fe5a56),LL(0x453b6e7f,0xf034f132),LL(0xb3c69762,0xba3fa97d), LL(0xcadb598e,0x34262560),LL(0x07afe0a8,0x7ed74b51),LL(0xebe0e8bb,0x2261d849),LL(0x608cea1c,0x23747e55), + LL(0x902c343d,0xb2e9371c),LL(0xda4fdba1,0xf57b2de8),LL(0xb67703a1,0x43f9afa4),LL(0xf79fe203,0xeafafb41), LL(0xf649a494,0xfec99dc9),LL(0xfe378232,0x14799ef9),LL(0x7184e31e,0xba3f8114),LL(0xdc0e987f,0x0abbb815), + LL(0x6fb5d02c,0x1dc7221f),LL(0x40f1607c,0xa0dfb113),LL(0xdbe8db86,0xd217f238),LL(0xe91f2859,0xc02547eb), LL(0xa98d0875,0x41df6bcb),LL(0x78f6be54,0xf51a8077),LL(0xba66bef5,0x3ebdf28d),LL(0x9175ec20,0xc65c7b38), + LL(0x475589d5,0x2302981d),LL(0x199d8ad4,0xbd479708),LL(0x8db46795,0xcb508be9),LL(0x6b224eb4,0xba1e066e), LL(0x1326ab8a,0xae909633),LL(0xf5bbe5e6,0xac989728),LL(0xb1729b41,0x1de663f5),LL(0x00c3d293,0x2018f58c), + LL(0x71efe25b,0xe647c0e6),LL(0x6104ebd6,0xaa30f871),LL(0x3c166de6,0x1003eebe),LL(0x1cbc42d3,0x0c308538), LL(0x56cba120,0x98b6c312),LL(0x860d2916,0x9065ae66),LL(0xa7dabb6b,0x2162062b),LL(0x8e011036,0x1422386b), + LL(0xe069de1c,0xd2589ad6),LL(0x7ee09300,0x3fe8e67a),LL(0x2f8ae49b,0x5b881860),LL(0x263a6a90,0x186a1482), LL(0x9b35b33a,0xc6079b39),LL(0x53d3411d,0x6c38d789),LL(0x720b2f99,0x743c4462),LL(0x29e14d08,0x4d903dd7), + LL(0x7b963913,0x625dfce0),LL(0xcc65e41f,0x60a83daf),LL(0x93e185a2,0x9e88c26c),LL(0x13707ac3,0x6d950e92), LL(0xa0a23dd0,0xbd7df2dc),LL(0xc2116cc8,0x5fad27f2),LL(0xda4430be,0x0703b868),LL(0x83cc41df,0x5ebf0e2f), + LL(0xf9e83fc8,0xebb91900),LL(0x6d60bb8a,0xf9539591),LL(0xb604935a,0x3bdd7a8b),LL(0x64e5eec0,0x2cae8c67), LL(0x60aaf21d,0x30cf58bb),LL(0x359f63cc,0x5e0f6f5d),LL(0x0547e03a,0xda245055),LL(0xb9e143ce,0xa83fd8bb), + LL(0x1b0215aa,0xa564435a),LL(0x354ba769,0xecffccec),LL(0xbdbbd594,0xd846149a),LL(0x65dd1597,0x9137df36), LL(0xa9f3ac33,0xc4f39f37),LL(0x961d7e8d,0xf594bb74),LL(0x835befbc,0x41fa4b58),LL(0xed79139e,0xa983eae9), + LL(0xb4e31aa4,0xd4085efc),LL(0x18b26adf,0xc760aec0),LL(0x76a7400d,0x14c1f78e),LL(0x317fe128,0x87b8aced), LL(0xcbd85bb4,0x4433582e),LL(0x86adc041,0x58f01426),LL(0x8f0d5781,0x3596dd50),LL(0x1a31a82f,0x2e7f3b80), + LL(0x2d1ede3e,0xcac7ccc8),LL(0xe89573db,0xc9b9a8f3),LL(0x54b40df9,0xbf744f69),LL(0xa85ecb47,0x88eb2281), LL(0x426ec49d,0x6b115026),LL(0xc8c41110,0xebda4660),LL(0xdf392aae,0x0a4a32ac),LL(0x37cb7080,0x2a28f9b3), + LL(0x6bd9414d,0xfed99d1a),LL(0x5715620e,0x15f59c41),LL(0xac431919,0x93edd9fc),LL(0xe1ccc47f,0xed1d43ae), LL(0x556d1ab5,0xafed2acd),LL(0x02e039c0,0x817b967d),LL(0xe02a68bb,0x335d15da),LL(0x67df767c,0x4fa75ea0), + LL(0x44833943,0x384704b3),LL(0x4084ef35,0x7809ed3c),LL(0x79c7ff41,0x2abab3c4),LL(0xa833fb73,0x2b7ef5b2), LL(0x4901a4ff,0x12b0335b),LL(0x44d58617,0x3eea607b),LL(0xd7f57746,0x7161b669),LL(0xb1e93442,0xee17e43f), + LL(0xd6d7878c,0x95c9bd80),LL(0x34ff7c75,0xe1ef20ee),LL(0xd2ccd599,0x3fab197a),LL(0x952ef4f9,0x9e480593), LL(0x06ea3410,0x69777fd2),LL(0x74fa7dd5,0xb0280454),LL(0xc43bb5fc,0x641b6860),LL(0xecd7b8a8,0x9f359d5b), + LL(0x70be68cd,0x4431d4ed),LL(0x08b55f8f,0x712117f4),LL(0x23d0b6ca,0x449e1319),LL(0xfdee5357,0x658323cc), LL(0x62879a95,0xa1ef8114),LL(0x69963eeb,0xc21257e5),LL(0xc5bbee13,0x1016ab74),LL(0x43d81a86,0x99bb310a), + LL(0x1d33a15a,0xdef03c44),LL(0x49127148,0x3e78cf18),LL(0x30b0cc00,0xe8d93368),LL(0xbd7ccd85,0xb05458fb), LL(0xdbaa76b0,0x8c2896dd),LL(0x79e4cacb,0x0d826600),LL(0xff730ed0,0x50a45b23),LL(0xeba9030e,0x4a0e079c), + LL(0xe3129aa0,0x3ead3fdc),LL(0x48aac890,0xa93b39f3),LL(0xf362465e,0x0fd73860),LL(0xf8df2764,0x69177f2c), LL(0x824ebddf,0x4cd58c50),LL(0x2fcef01d,0x1478981f),LL(0x980524b3,0x511bd380),LL(0x4d23e8e9,0xc95252b1), + LL(0x9ce08452,0x7ff12c37),LL(0xa3a87024,0x3dd8dd09),LL(0xb849dcb6,0x61ff0d39),LL(0xfefad6de,0x3f5eab86), LL(0x251523f9,0xb6146886),LL(0x5be2135b,0x45ac1d52),LL(0x41d2c5d4,0x63507995),LL(0xb3064e72,0x7f19f799), + LL(0xa2efb9be,0x7280ad9b),LL(0xba9659de,0xbc8fbb60),LL(0x875a8062,0xa4861c12),LL(0x2f2387ca,0x78c920c8), LL(0x03b7da99,0xfe0a6ea7),LL(0x48e3afa3,0x936a44b8),LL(0x89fd127b,0x618a0ecf),LL(0x06f24bc5,0xa35614cd), + LL(0x8d49c05f,0x21a1002e),LL(0x2fd7989d,0xceeacfd1),LL(0x8f5f4ea5,0x8c058f4b),LL(0x563e214b,0xf31bd38e), LL(0x245c7585,0xbe47bd98),LL(0x6bc2a404,0x70d1a05c),LL(0xc68257f7,0x830d7f30),LL(0x8136c932,0x1abbbfbb), + LL(0x6e03597b,0x6fbc43b7),LL(0x5fca14a2,0xc18fddb6),LL(0xe8f506e2,0xd3c4ca78),LL(0xa47e27de,0x6b8711dd), LL(0x16382b5b,0xc308c189),LL(0xcf96bd7a,0xe9d3145b),LL(0xafb26629,0x5c290ec1),LL(0x209438ff,0xb4eb8130), + LL(0xe1118539,0x7335044a),LL(0x5192a007,0xed6d43fc),LL(0xa43a2bd4,0x1a8bf622),LL(0xefa9f3a5,0xefec3fb6), LL(0x6d834bde,0x6d224bbc),LL(0xb0fbc744,0xaaebfcb8),LL(0xc4ea1652,0x383b2bfc),LL(0x751ae816,0x9cd26d90), + LL(0xf47f0f9a,0xae614386),LL(0x73c6ecd0,0x99873787),LL(0xe5414fc7,0x0b56c1ad),LL(0xfd40286c,0x9b85e6b5), LL(0xd64687dc,0x7117aacd),LL(0xad8a8c4d,0x85d148e7),LL(0x7962655c,0xf62f8eb5),LL(0x7f0c6a2c,0x8386b37e), + LL(0x90c47d7f,0x6b4715a3),LL(0x458a54e4,0x1fc4ced1),LL(0x0ed97b0a,0x01853943),LL(0x5b370e0e,0x58a280be), LL(0x344f3960,0x8d488cb6),LL(0x0741604b,0x9050c599),LL(0x07954771,0x0878fb1b),LL(0xdbb3c82b,0xd927ea8c), + LL(0x384f01fa,0x2fe71d59),LL(0x238bb66b,0x66d2b790),LL(0xb8fd803b,0xceaec11f),LL(0x6dd09c0c,0xbb919914), LL(0x2ccb2f67,0xab5992e6),LL(0xcca50326,0x2421878f),LL(0x9ee6dc73,0x363d933d),LL(0x084b1fa3,0xa374ab0b), + LL(0x161f6475,0x2d832a29),LL(0xfc8797eb,0x435b8d78),LL(0xd71b609c,0x66bc156d),LL(0xfe0c2004,0xb3dca798), LL(0x02fd92d7,0x445d47bf),LL(0xc8b03083,0x1d1c9798),LL(0x079a7c51,0xca46d98d),LL(0x1afeb89a,0xb93f286c), + LL(0xeeb6665d,0x1c174510),LL(0x7479a932,0x65874b6a),LL(0x2335e3b8,0x28d16a85),LL(0xc747eae6,0x5e22bd3b), LL(0x04be16b4,0xa770e0a7),LL(0x40b3ff57,0x9f5f9ca9),LL(0x845ec39f,0x3f39e529),LL(0xebe697ce,0x5d5f4d60), + LL(0xbd90d4f4,0xea2a262f),LL(0xfe1b6494,0xa1ce74ac),LL(0xfa0fc472,0x4e419004),LL(0x4e691060,0xdef0e440), LL(0xa9f4baf3,0x57195a3a),LL(0x5e758c53,0xf14e108b),LL(0x920895e0,0x10a34f9d),LL(0xfeb57a63,0xc3f18af9), + LL(0xda1bef0d,0x4b1c988c),LL(0x3df6915b,0x8b328cd9),LL(0xf45586d5,0x5ddc5ecc),LL(0x040322fc,0x426468b9), LL(0xbe981558,0xf7f44765),LL(0x1855504a,0x25093991),LL(0xf7d6df43,0x72c51f2e),LL(0x849c99e5,0x858637fb), + LL(0x0ee9f78b,0x68b84dfd),LL(0xf2ee390e,0xff42fc9b),LL(0x531e1dcf,0xaca71e10),LL(0x7feaedfa,0x391620e2), LL(0xacf3e5da,0x7b2d6a02),LL(0xd20a16d3,0x261823d2),LL(0xbb00cd30,0xf9afa5d6),LL(0x1084580d,0xba151f4a), + LL(0x26a3fcc3,0xb5f4b3a9),LL(0x6729f4da,0x5d84a823),LL(0xfc35f732,0x51540c41),LL(0xa6ae5bf7,0x81b0cb58), LL(0xbd81bd27,0x91c7ae07),LL(0x1d56ff5d,0x0868980e),LL(0x65224df6,0xaef85a31),LL(0x17a69e35,0x112eba3b), + LL(0xc3a9d932,0x07c34677),LL(0x8ac45e37,0x3b6b7cce),LL(0x31b6248a,0x5e0e2b6e),LL(0x453d9685,0x14ee5b66), LL(0xbd4d807a,0x4c5e2be7),LL(0xc121fea8,0xc03f37f8),LL(0x8df7b5e7,0xcf34911e),LL(0x5f754191,0x00e7f18e), + LL(0x2dcea4aa,0x89a8c9e1),LL(0x50f6db55,0xcc1cc31a),LL(0x9046955f,0x4a6f542c),LL(0xda2485d4,0x85fed580), LL(0x9ac53748,0xa70f62d1),LL(0x655870a7,0xc2fbb850),LL(0x8c859aef,0xaeb2d438),LL(0xcc9ff51e,0xe3cc5ae5), + LL(0x0a3ebbfc,0xf8d8c55d),LL(0xed48f61a,0xdcd838d5),LL(0xd4cba7ab,0x032f91ea),LL(0x2f70235e,0xeb0ed88d), LL(0x000ef325,0xd4498170),LL(0x4b923c4a,0xfd34e07f),LL(0xb19b84cb,0xf71c07a9),LL(0xed9690a3,0x000a669c), + LL(0xb5705e16,0xf45eb0ef),LL(0x7d9ce007,0x8cfd6a62),LL(0xd9e52885,0x76ba9a5f),LL(0x8aa9ffd6,0x13f72881), LL(0x85528e4a,0x0a11e8dd),LL(0xcee8d663,0x58f1993d),LL(0xa1c81fd3,0xb49d750b),LL(0xe7de2e6b,0xaae29861), + LL(0x5dd7de70,0x9a40644e),LL(0x937a5546,0x67fbae1c),LL(0x956c2fa8,0xb3e02907),LL(0x21b4aede,0xaf785374), LL(0x9c0a8bfe,0xf42a1e96),LL(0x78957181,0x3f6690e6),LL(0xa6c5e0a7,0x1b1c9575),LL(0xf9cfb9bd,0x6def8124), + LL(0x72faa1b0,0xde552cf9),LL(0x9b5ebbbc,0xfac2f4ad),LL(0x8ef89ba1,0x4b60a5a5),LL(0x8012f3b1,0xb6d9be57), LL(0x9b2b083d,0x3992a6f7),LL(0xac640f77,0xe79ec527),LL(0x3f1e632e,0xf6cca775),LL(0x8fb80790,0x5dae8413), + LL(0xb572407c,0xf0d4146c),LL(0x3f84cc39,0x829cfb38),LL(0xe31f007e,0xd7c9fed4),LL(0x09e68ce9,0x93b2a5bc), LL(0xd01482b9,0x073fb24a),LL(0xb8d44e62,0xfe494244),LL(0x3dc49858,0xe59a1649),LL(0xf005b31f,0x071776f7), + LL(0x285439af,0xaa368f59),LL(0xe27e783d,0xb0821574),LL(0x098746ba,0xe16903f6),LL(0x69dc00d0,0x436a95ae), LL(0x877bfcec,0x9ae5a91a),LL(0x1416ad3a,0x5d8758d9),LL(0x25fd9a9a,0xa420ce40),LL(0x39b0fbfa,0x99610fdf), + LL(0x41cc1e9b,0x782af222),LL(0x86ff4cf6,0x346a4a92),LL(0xabc900c1,0x9dc4c6cd),LL(0xe7fade51,0x7ed2476c), LL(0x4f1dd24b,0x68a72185),LL(0xb08260cb,0xcefcf0fd),LL(0x8a6f61b6,0x0199a147),LL(0xcb5769c5,0x176b6079), + LL(0x347d5a63,0x6dbcceb0),LL(0x9c4dc505,0x26433ebc),LL(0x05d5e74d,0x52570318),LL(0x057ca464,0x692f1d81), LL(0x477f424e,0xa09554a0),LL(0x176a695d,0xbd3f9bbd),LL(0x5972db27,0x8c7f52f3),LL(0xf28b2aa4,0xacee8234), + LL(0x3a4a0217,0xb853465a),LL(0x66f3a4d5,0x74a2534f),LL(0xeff0423b,0xae1a7ff3),LL(0xbb126028,0xd2a01a09), LL(0x4963e855,0xff84c6f0),LL(0x6bc18d50,0x63db264c),LL(0xcc6a5e25,0x39792dca),LL(0xedb37a25,0xf20cdf3e), + LL(0x8730f2c4,0x6a460f3d),LL(0x6a0ab6bb,0xe9b786c4),LL(0x084015c2,0xa9720a6b),LL(0x0dbe6f0f,0x28add2e2), LL(0x26be7de7,0x90fb0ba7),LL(0xe40f15fd,0xfae8b5d4),LL(0xceb9c856,0x007363a1),LL(0x586b770e,0x6d8bfe14), + LL(0xb7bcc0e0,0x63e7d78e),LL(0x3ed4ab62,0x56c569f8),LL(0x9e103abb,0x76c6a5bb),LL(0x638fc44d,0xeb24afeb), LL(0xf0be16f1,0x15e00239),LL(0x8778f084,0x7db92f67),LL(0x63de2bef,0x5198680e),LL(0xe0510421,0x69031d0e), + LL(0xb94753c1,0x8058f8aa),LL(0x454bf609,0x412d4c97),LL(0x95f9fd19,0xb8dbfe8a),LL(0x68b43233,0x6cd3221a), LL(0xa5adaaff,0x384a9f15),LL(0x42b2ef95,0x60c70f90),LL(0x2c7da919,0x085b2f3b),LL(0xbc8407e1,0x1e5d23d1), + LL(0xadb45b3e,0x9ea95bc9),LL(0xed06ec67,0xb5a28fec),LL(0x62a3c143,0xd678df46),LL(0x6793284a,0x80f0bc9d), LL(0x07d4afc7,0xeb7865a9),LL(0xc1301d87,0x0fc5eafe),LL(0x4823349b,0x50a8e7f5),LL(0x2d019e96,0x97800fa2), + LL(0x1bdd1d9c,0xfeff2579),LL(0x23886156,0x4d938c5d),LL(0x6979b9f6,0x25e3a806),LL(0x37bb6199,0xeeef8fa0), LL(0xd7d308b8,0x4d917977),LL(0x4ae672cc,0x60ca7ff9),LL(0x2a68db6e,0xb24ec154),LL(0x9e9942f5,0x7b0604ec), + LL(0xca4fad9e,0xfdf4794f),LL(0x3df5de22,0x086020f8),LL(0xd601533c,0x653da6ca),LL(0x735709f3,0xf97c1865), LL(0x7cbd6ab6,0x2457ffd0),LL(0xd003a502,0xce05a482),LL(0x33ee2740,0x5c0c0cba),LL(0xf37174aa,0x8146ca00), + LL(0x587262bb,0xec118827),LL(0x7e2a402c,0x8468c029),LL(0xbd6eb2aa,0xe1c3a9e3),LL(0xb496bde8,0x77d09b4d), LL(0x854693bf,0x454e767a),LL(0xa4de85bc,0x6bbcc22e),LL(0xb180f206,0x66452f10),LL(0x0f5b1744,0x110c5a05), + LL(0xa6b3f6e4,0xb7be75d8),LL(0x2c0d2e1d,0xf64bb3fd),LL(0x935ae640,0xad17a039),LL(0x8f243748,0x7304ad63), LL(0xd278caa9,0x04316bb4),LL(0x1e84f91d,0x19b89c62),LL(0x6e7a2511,0xdf4a47e9),LL(0x998b6bc0,0xdef32df9), + LL(0xcee95a1f,0xf1253ce0),LL(0x0ae96e31,0xbacf5206),LL(0x0b343e63,0x4ba2e24a),LL(0x79929dc6,0xca64d07f), LL(0x9424ce75,0xf2823ac8),LL(0x0207ee9f,0x2d4add37),LL(0x387cde5f,0x44d9ecd0),LL(0xe180a21f,0xa5095ccb), + LL(0x7c0cedf8,0x901cec8a),LL(0x3b8c759c,0xf291dc78),LL(0x49401234,0x98b8efdc),LL(0x058e9d9e,0x8f2b16e3), LL(0x27dba00f,0x16ce8007),LL(0x6d66d2f1,0x5bb8fca9),LL(0x7b85a96b,0x092eda98),LL(0x973eae20,0xec53f4bc), + LL(0x1b93a60f,0xe6e0df59),LL(0x2f6b0abf,0x65e06ecf),LL(0x569a9e1d,0xb8c2ec3e),LL(0xaa8c1cc3,0x27f9fe72), LL(0xccd4d5e2,0x9cf3908f),LL(0x725c8528,0x5a40e0a9),LL(0xd470b0b0,0x27b15a1e),LL(0x33d81bff,0x50a09ec1), + LL(0xda99fcf5,0xba976a58),LL(0xc3536b7c,0x3881ef1e),LL(0xfbc931b1,0xec65a069),LL(0xfc929a0e,0xab7f57b4), LL(0xbc61f452,0xc7c63491),LL(0xc1750dbc,0x5c1aa935),LL(0x9ff0465c,0x35b8789b),LL(0x727647b7,0x8ff8589b), + LL(0xa95022b6,0x2b56fe50),LL(0x2adbdbbd,0x24220569),LL(0xd2b80ea8,0x2370d94f),LL(0x71d9e6f7,0xe5d47b73), LL(0x8d858032,0x2fe84b72),LL(0x4ffd7cfe,0x0411034b),LL(0x7c2c84e0,0x0819a9f3),LL(0x30228f9e,0xf8248dfd), + LL(0x4fdf6c79,0x75c77f0a),LL(0x15579cc7,0x195b4b59),LL(0xf8d3705c,0x274c7c81),LL(0x70ee9be1,0x45a2209f), LL(0x0b1e818b,0x4a4fce69),LL(0xbb9926b1,0x74d05e5f),LL(0x3f2a796b,0xb64639ce),LL(0xae957d59,0x2d168d5b), + LL(0x16171846,0x067c2270),LL(0x1a03f8d1,0x7bb71d15),LL(0x495a09a1,0x2badd196),LL(0x51b990c4,0x024db163), LL(0xe79dcaac,0xc19edc61),LL(0x60df81e9,0xf17f54bd),LL(0x9ae347e4,0x4560262e),LL(0x59eb711d,0x1d2c2542), + LL(0x4919affe,0x40372d2d),LL(0xa009bd16,0x2d4a2ea3),LL(0xc1a62eb1,0x48f1e7f8),LL(0x587a1124,0xce083725), LL(0xe7ebadd3,0xe874261b),LL(0x057b93e9,0x7ca5c156),LL(0xfe39e6ff,0xe8b381e5),LL(0xd30815c3,0x2d16b32f), +}, +/* digit=15 base_pwr=2^105 */ +{ + LL(0xad2e996a,0xdfd30b28),LL(0x9df0fc37,0x64d4eeec),LL(0xddc39763,0x8e049e3d),LL(0x37ea2e8e,0x9f55e0ce), LL(0x4bf01984,0xf3fcba4e),LL(0x9d55bc4b,0x764d5c3d),LL(0xcfda895f,0x98cb92a1),LL(0x5c7bca69,0x27dfe795), + LL(0x23a86e2c,0x86dfdecf),LL(0x3387f712,0x02ac466b),LL(0xd63509d1,0xc30a1ac2),LL(0x63aebbd1,0xd3913422), LL(0x068ae34f,0xdc7b789f),LL(0x487dcd10,0x499f2d01),LL(0xa3e8a4b4,0x68e2a3bc),LL(0x14d2a26c,0xdf87ba71), + LL(0x34479e7b,0x9e3ab999),LL(0xb026e780,0x9d5f2dc3),LL(0x4f1bd429,0x131374fd),LL(0x9be1379a,0x92e2e738), LL(0xd13bc111,0x6cc32f80),LL(0x86f81c92,0x6fbfc350),LL(0x9263913a,0x12ca1b30),LL(0xe06ab75c,0x6f7da1ff), + LL(0x19301b16,0x4780f12a),LL(0xbc368a20,0x233bc231),LL(0xcbadb344,0xd9650892),LL(0xad9425a1,0x38a0d964), LL(0x4a8d4d7e,0x277abcf2),LL(0xb806ce9e,0x4ccd16b1),LL(0x7570d473,0x82ff40f0),LL(0xdf130780,0x57491414), + LL(0xa96ef42b,0x9f1f39f2),LL(0x107f1321,0x1fd967ed),LL(0x29d4767e,0x93832497),LL(0x3fa9e796,0x7761a38b), LL(0x66318df2,0x3e408c59),LL(0x41745f03,0x9283ca44),LL(0x2cce1a86,0xfedf8fa3),LL(0xb44600b4,0x8363860d), + LL(0x45f45a89,0x3dbfde55),LL(0x26ce399c,0x8800c860),LL(0xc25e9479,0xfb25e8da),LL(0xf7d367a2,0x6ff0d6cb), LL(0xa93f9890,0x70b0ba36),LL(0x49bd5a80,0xc07ca403),LL(0xed54d1aa,0x5f4feda6),LL(0x671ad0b4,0xfa1e2efd), + LL(0x8c56e7aa,0xda465467),LL(0x25e45bc0,0x39d04cdc),LL(0xaf21c637,0x26661bd6),LL(0xb55ddfa5,0xf757ff5c), LL(0x4394eb20,0x07318fd1),LL(0x2bcf3ad1,0xe010b19d),LL(0x8e5c7e7f,0x71e2031c),LL(0xedbfda69,0xee35f346), + LL(0x5d8f6fab,0x8d8d4120),LL(0xabed09c8,0x5e420839),LL(0x4aacbb1e,0x5120e079),LL(0xec1bc996,0xf7e840f8), LL(0x9707c3bb,0xd921d63b),LL(0xae891a71,0x3ab4b8b8),LL(0x73bb2278,0xbe465756),LL(0x553ee91c,0x776ce040), + LL(0x88222e0a,0x86f07c2e),LL(0xdf97816e,0x3f3688f1),LL(0xa6df9d3a,0x20d2d944),LL(0xb2cb925a,0xff399057), LL(0x27f1de19,0x010c747e),LL(0x7fc1fbc3,0xbe940669),LL(0x877ef709,0x3ab94600),LL(0x8f4c8a8e,0x9af40a6b), + LL(0x713f023a,0xf7c0f202),LL(0xd0a8dc63,0xbe0bf3db),LL(0x64a41337,0x09789926),LL(0x2c2823cf,0xbb4a8964), LL(0xb90e45c6,0x9279cb27),LL(0x382564ac,0x80283fd3),LL(0xfa5f0bcd,0x3a746b01),LL(0x0afaf10a,0x28667a8e), + LL(0x015d5295,0xeccfd0ee),LL(0x73ce8e21,0xbd6678a5),LL(0x336ecb65,0x132a87f2),LL(0xbe7dc659,0x09de4efc), LL(0x6f69b3bd,0xbedadd10),LL(0xe8303650,0x357c61e2),LL(0xa45777e6,0x6b3c613a),LL(0x08d41179,0x51dd30ad), + LL(0xf98feac3,0xa7b53f3f),LL(0xf9497319,0xe8e00328),LL(0x0ca20b40,0x1f3b080d),LL(0xe19b088e,0x06dc5456), LL(0x0dce02cb,0x1f3f18d7),LL(0x6fd1ade9,0x2d215544),LL(0x055550d0,0x4c6de38c),LL(0xf33f4973,0x5d535e6f), + LL(0x744c033d,0xdcfd4163),LL(0x76fc420a,0x0706a490),LL(0x394f5b6d,0xdc9db55c),LL(0xe040a093,0x1a050a62), LL(0xd0ab26f3,0x60367ebb),LL(0xc3d91820,0xb9c07239),LL(0xa3e32db2,0x14a8bed5),LL(0xd49f37dd,0x23b19cd3), + LL(0xea6c85ce,0xd048c64f),LL(0xd5716cfe,0x8aefff19),LL(0xab85bdcf,0x199fddb1),LL(0xbaea49d0,0x3f592e7e), LL(0xf1d58ff6,0x8c8decb6),LL(0x7fe8f425,0x02598a99),LL(0x64a93fd6,0xe4c73ae2),LL(0x878db06b,0x208a0f61), + LL(0x27c4a266,0xff0272fe),LL(0x69e7c6a2,0xccfc96ae),LL(0x8e78d74a,0xbd7e715f),LL(0x32ed35bc,0xd04ae9a4), LL(0x05614c5f,0x302af413),LL(0x33943b5a,0x98176637),LL(0x8a58cfd8,0xa4d1c9b2),LL(0xac2f3f79,0xf0ebd5f4), + LL(0xfdc7945f,0xebadb010),LL(0x03750a4b,0x62c9cff0),LL(0xa0c85b9d,0x75192383),LL(0x16eb0bf9,0x2aba7b53), LL(0xa4c06f9a,0x93d4adaa),LL(0x5f8e482c,0x573a86a2),LL(0xe791a79f,0x109c6fdc),LL(0xb37eeec2,0xd9ed02ce), + LL(0x7dd63c8b,0x7b1fb4b4),LL(0x22493b49,0xae6e2767),LL(0x6a411bc9,0x5ef6beb7),LL(0xe1bf4242,0x4d88918d), LL(0x02a97fbc,0x7ba26f8c),LL(0x7f41c518,0xf45b2a50),LL(0x83249e23,0x6c35fb69),LL(0x7a5778cc,0xc4a95134), + LL(0x08287cf7,0x6173f860),LL(0xfac3a444,0xdcfc71d9),LL(0x079ce3c9,0x894f3b33),LL(0x916b661c,0x842bf7da), LL(0xa758a243,0x94299d6f),LL(0xb242389a,0x0e23f984),LL(0xc03a7aa2,0x653050f0),LL(0x9559ad9c,0x2ec3041b), + LL(0x97cf6e9b,0xa61dd499),LL(0x448fa6c6,0xfd090f38),LL(0x39b126bf,0x4f1b75ac),LL(0x1ef1a280,0xb48d0372), LL(0xf2b2941f,0xe40c310d),LL(0x8d9d5ace,0x5b9a7369),LL(0x7ad9ad95,0xbe0415c1),LL(0xa8510622,0xffd864b6), + LL(0x898f28e8,0x2aceddcd),LL(0xba89a51f,0xa0cfc30a),LL(0xe105e448,0xd87db983),LL(0x5e5ea6fb,0x16ba795e), LL(0x352ad094,0x5f48e45a),LL(0x55fd75e7,0x1971a10f),LL(0xfd1c8d68,0xfb0f1767),LL(0x86df0665,0x904229d9), + LL(0xe87ab22b,0xc2c88671),LL(0x33611a50,0xcbe384a1),LL(0xad2eb708,0x38eec58e),LL(0x86d7bdee,0xaa19b174), LL(0x51832b61,0xa9f416d7),LL(0x355e7b6d,0x10b751ff),LL(0x4ff07a5a,0x6dd52063),LL(0x4e3505c1,0x6a6090c1), + LL(0xd4c80f29,0x53addd1c),LL(0x0d3d793e,0xe76d668b),LL(0x191eedd9,0xbdcdc4c9),LL(0x0f8e4877,0x80675320), LL(0x32f5661b,0xc41426dd),LL(0x06199185,0x9fcbe1ac),LL(0x404a1207,0x62fa2198),LL(0x33c8190e,0xc742fdc2), + LL(0x778ee765,0x16ec1b96),LL(0xe29d992d,0xda656f58),LL(0xb4369e7f,0x5393775b),LL(0x0674fc45,0x512f5c7b), LL(0x60efa8e0,0x55b8bd38),LL(0x155b57ab,0x1ab6d2df),LL(0xe26ad439,0xed0aa777),LL(0xd9b04376,0x5b16333e), + LL(0x8fc7ea72,0x321119d3),LL(0x7211ef45,0x390b4ef5),LL(0x1feda1a2,0x289f3945),LL(0x605c8596,0xcee9cb33), LL(0x971b6897,0x91109e96),LL(0x19701ea7,0xdf70c173),LL(0xa92c6b2b,0xa07d0ecd),LL(0xa9eab766,0xf8eb97fe), + LL(0x0e3cf9e8,0xbb2cf63b),LL(0xdda02b26,0xffa6c503),LL(0x9cb18efd,0x42c1ec1a),LL(0xc45d5b22,0x13cdda9c), LL(0xc820cdf5,0x6b3baf33),LL(0x610c8bfc,0xa337bc5e),LL(0x06a9ca6b,0x88186818),LL(0xa0c455af,0x382a34ee), + LL(0xe8fc59df,0x725006c9),LL(0xf929c670,0x0d46b697),LL(0x893a9f6e,0x7bd6eceb),LL(0x1cd04e5a,0xf25aa637), LL(0xf6612d4a,0xf1563d79),LL(0x5efc49d8,0x8c9372cf),LL(0x96c5bdc5,0x09cc0513),LL(0xd3cc34d8,0x70f19d46), + LL(0xfdfbe16c,0xe62f6891),LL(0xdc60110b,0x8b7db2fd),LL(0xf7868674,0x3557bff8),LL(0x95a638d6,0x2b414c8e), LL(0x6d19ac65,0x3b6610ac),LL(0xd641b0ea,0x20864204),LL(0x892161fb,0xee372a46),LL(0x4125868a,0xc7c5bd87), + LL(0xa61ee957,0x5edc6afc),LL(0xd37faed4,0xa4374ba1),LL(0x1e52b0ab,0xf0684f46),LL(0x2a007b66,0x0812cbca), LL(0xe68c4893,0xf3442afd),LL(0x2d9dd9a2,0xb02ac6df),LL(0x068c08bb,0xa4fe98dc),LL(0x7795e2e9,0xfcd5dc05), + LL(0xa0f55cd8,0x28983aee),LL(0xe96cf557,0xb390daf4),LL(0x3f119098,0xbfd2f5ab),LL(0x6386c0ad,0xed114587), LL(0x69539392,0x578995b9),LL(0x36b880ab,0xc8a77948),LL(0xe4c8b560,0x7e03cfb6),LL(0x06cdcbe0,0x018cd4db), + LL(0xb006f8d5,0xbd7f2e3a),LL(0xd6d9f30e,0xf25d1718),LL(0x1b22ae3d,0x9ef6e2ee),LL(0x416154ab,0x89a2695d), LL(0xda261e39,0x1975e0c8),LL(0xda2c2031,0x8fe66aed),LL(0x97e1e00c,0x6f93f839),LL(0xa742b328,0xcba9a773), + LL(0x3417df8b,0x9529fec1),LL(0x54e314b1,0x37a27cd9),LL(0x65f94a63,0x93feb0f8),LL(0xb5e029c1,0x65978b84), LL(0x81b705f1,0x576fd830),LL(0x688e8c52,0x37d07a37),LL(0x332838dc,0x3b91d360),LL(0xb0b14a94,0xcca9cbf8), + LL(0x8b63b669,0x86f18c44),LL(0x6972d2d1,0x53c6eba6),LL(0x8cff59bc,0x2a522d8c),LL(0x6ed25ce5,0xbc181d15), LL(0x5feb0eca,0x913f173d),LL(0xa207bd71,0x34273f8d),LL(0xfa1715ae,0x41b7572e),LL(0x7f16f4ae,0x8a8ffea2), + LL(0xf95bdf78,0x2b852908),LL(0xa26328b9,0xa75adbb3),LL(0xdae21d25,0x992ac179),LL(0x78e14467,0x8c99927a), LL(0x0c16e0c2,0x23fb2fee),LL(0xdbcb5f4e,0x58e797bb),LL(0xa07bd7de,0x33d6956e),LL(0x7172d36a,0xc921fdb3), + LL(0x158267b5,0x035f1744),LL(0xa291374d,0xc7495f33),LL(0x4a41a6a0,0xe07db2f5),LL(0xd1238792,0xfdb2987e), LL(0x49741ce6,0x616df624),LL(0x8a693880,0x90ecd21b),LL(0x341fe21b,0x447c729d),LL(0xc012a8ab,0x06ad3c90), + LL(0xddfd6b5a,0x13dc4fa9),LL(0x64cfc0f3,0x238a1add),LL(0xc34a2b1e,0x874a3c2f),LL(0x0deb1dd4,0x427b6e3c), LL(0x876f2621,0x78a1ad1d),LL(0x252f6837,0x34f9207c),LL(0x047d667b,0x1c812fbb),LL(0x3ee03ba7,0xc3512ea3), + LL(0x527a1b55,0x762de5f0),LL(0xae3221af,0x7873d692),LL(0xb112d35f,0xa8ac73c7),LL(0x815414f6,0x1d118c30), LL(0x865ab6f6,0xbc686118),LL(0xecf8c02d,0x01e75348),LL(0xe6220bf9,0x9b131840),LL(0xa67512b5,0x3d72dac4), + LL(0xaf95e16e,0xd9e49ecc),LL(0x1e2aa105,0x1297c560),LL(0xa1a5e8c2,0x925301ac),LL(0x990ab503,0x3b0ea863), LL(0x15f258c4,0x7860b660),LL(0x397253e4,0xa4497040),LL(0x14a4774d,0x88562ed0),LL(0x7adbd093,0x325d8b0d), + LL(0x41e24010,0xd4df8df2),LL(0x580031be,0xe7cb8663),LL(0xd7fc7e5f,0xd653593a),LL(0xe9f1fe9d,0x429a8863), LL(0x63e71ced,0x18a0a709),LL(0xa498140e,0x39d9316d),LL(0x40f40117,0x44466cff),LL(0x64602832,0x58d27cd6), + LL(0x86018a70,0xf4a4c22b),LL(0x6d703f04,0x7a4d4102),LL(0xb424b0fb,0x4f5a0037),LL(0x322b1876,0xfb591cfd), LL(0x632e0f56,0xb45798b0),LL(0xfdcbcf20,0x83ef9feb),LL(0x115749ac,0x0a23b09c),LL(0x946248f8,0x3b950273), + LL(0x1e13eaee,0x5ed75e68),LL(0x09dafdb9,0xbebd4744),LL(0x69941fc4,0x8b46621f),LL(0x91129bc0,0x1fd3c13f), LL(0xb7b9da22,0x371448d0),LL(0xd87a4742,0xd452ccea),LL(0xf91c38b9,0xe2f04778),LL(0xbd516bfe,0xfb443a5d), + LL(0x044d666b,0xd7bd4056),LL(0x2a9b17c4,0xb4546ffd),LL(0x818fe55a,0xf66765ae),LL(0x1b5dc7a0,0xc375126c), LL(0xc1a81c63,0xe9a7ed20),LL(0xf8cf06b9,0xaef2e23d),LL(0x3e67b95b,0xf4536865),LL(0x24309537,0x25cbb5a6), + LL(0x3256c020,0x8a230e68),LL(0x215dca7b,0x4a33e462),LL(0x2935b6d1,0xefef4936),LL(0x852c39f4,0xb383df4e), LL(0x57c21e90,0x197ddd77),LL(0x2164127f,0x236f98dd),LL(0xbbd61847,0x464b86ec),LL(0xfb89d515,0x107a387c), + LL(0x400d66d5,0xe01e50b7),LL(0x5f864426,0x4377af2b),LL(0xf8fe847a,0xde21c49a),LL(0x887c0674,0xc133e58f), LL(0xd2fda17c,0xda5b4c3b),LL(0xfed8fe68,0x24157f13),LL(0x8b6bb6bf,0x1c4483f3),LL(0xcf1bed73,0x940fab9e), + LL(0x3c15c7e5,0xce3fca79),LL(0x066de109,0xb83fce10),LL(0x0cd5a04a,0xbd42ed01),LL(0x407fcb03,0xba5446b8), LL(0xe5d35bda,0x4a8cb929),LL(0xbff8631f,0x6338fd7e),LL(0x4656a8cd,0xc85d4ee4),LL(0x92c99820,0x83b1f39a), + LL(0x7e90c823,0x153fa4d4),LL(0x15326196,0xc15809ba),LL(0x6eb4b379,0x320b8fe9),LL(0x58704d9e,0x27cc07c0), LL(0xf13d6ee7,0x301840b2),LL(0xc6d8c930,0xf1f6530f),LL(0x96a22034,0x3e9254ea),LL(0xaf5b8a2e,0xf8c0ee68), + LL(0xb8207fde,0x88e9e44e),LL(0x29bc1310,0xdea11cbd),LL(0x9c9d7df7,0xa20c2f17),LL(0xbceac495,0x2a954927), LL(0x2a58d6ba,0x3f405f5c),LL(0x2ac9aeaa,0x64df5364),LL(0xe8aa74a5,0xb618f6db),LL(0x74ef61e3,0x22dadc7f), + LL(0x9cfdc4cd,0x306ee832),LL(0x40698a5b,0xaff53321),LL(0x89567341,0x98893443),LL(0xac7c68ec,0xdfefbdd4), LL(0x3261a582,0xd3da7094),LL(0x62ce96e7,0xd23e3fa5),LL(0xd773337e,0x62c060c0),LL(0xa041f910,0x5cb2beca), + LL(0x7e8215d8,0xe21ab479),LL(0x923b4b27,0x84384686),LL(0x6a3d21ef,0xa93c08fe),LL(0x2fa6de1c,0x7bd96248), LL(0xca6de3e0,0xb858ecd7),LL(0x47c9fced,0x466a48c8),LL(0xc708239e,0x23ca9b75),LL(0xb5bbe833,0x860d553d), + LL(0x9e76e71d,0x45804f1a),LL(0x51e59be2,0x9fdb8b8d),LL(0x3bbc2a19,0xa76db4b7),LL(0x96f82cd0,0xaebb47ee), LL(0x97b1dd04,0x7a9b95b5),LL(0x5f0b1d7c,0xcc149a8d),LL(0x47a50eec,0xbba40e4d),LL(0x56b72539,0x4e635d70), + LL(0xb18659c4,0x31c40e90),LL(0x6f694b35,0x08026224),LL(0xe0cd6e15,0x8ed3d7b8),LL(0x9293cb36,0x157e2a9d), LL(0x96e54583,0x7b457bb1),LL(0x2609c44d,0x75647498),LL(0x970a8cf2,0x54667671),LL(0x3af72572,0x3b7da9c8), + LL(0x4d63473a,0x8fbba977),LL(0x23001221,0x7af5af43),LL(0xea29d97e,0x99093197),LL(0xfa564089,0x4409f6a9), LL(0x2f70e06f,0x64fd1cda),LL(0x2e9d55ea,0x8b7c83a6),LL(0x6385fcef,0x0dffbe4b),LL(0x7a6fe72d,0x4513f584), + LL(0xba1de4ae,0x6a64f166),LL(0x89069fbd,0x1f8550a9),LL(0xda7ef7fc,0x72b411fd),LL(0x829ea712,0xa185d2c3), LL(0xccc1868d,0x82f5ffb8),LL(0xff9fafa9,0xb484473a),LL(0x089132ed,0xe1857f3c),LL(0x08ef378f,0xdad885a9), + LL(0x7af9e2aa,0xbdbdfc0e),LL(0x95486202,0x366c07bb),LL(0xfc9d979f,0x37040d45),LL(0xa0f80630,0xf279ed10), LL(0x8f31b39c,0x27855261),LL(0xf26f91cb,0xea0c3b1c),LL(0xb38c050f,0x90b4e8c2),LL(0x1623ab47,0x7acb26b1), + LL(0xa4a064d2,0xb6cc3cd9),LL(0x97245482,0xa410880c),LL(0x3a6e6a1f,0xfb470e11),LL(0x93f347e4,0xf19080b1), LL(0xb75a53d9,0x23253dcf),LL(0x9c95d567,0x55047049),LL(0x7b20202a,0x8c75631e),LL(0x834b8380,0x58fccf32), + LL(0x243dddde,0xaf6bdbd8),LL(0xcf38f705,0xa3ca3e2c),LL(0xca38c9a2,0xa2357b4b),LL(0x6eba095b,0x8bf0d270), LL(0x9d998070,0xe4a43b7c),LL(0x8469214c,0xdf412faa),LL(0x000f1802,0xd2578cc4),LL(0xf8515863,0x2feb563f), + LL(0x5022112c,0xe66ce02a),LL(0x1461b1c6,0x8415df81),LL(0xad247c38,0xc2546e6a),LL(0x9a9c74d6,0x4b9788e6), LL(0xa22be3e8,0x9d0cb2e0),LL(0x295f76ad,0x15db086c),LL(0xa2337670,0x9758f99b),LL(0x9ab57f54,0x61ae09bb), + LL(0x93926a37,0x7af4d4aa),LL(0xf25cadb4,0xa895f386),LL(0xc6f4f514,0x90e13821),LL(0x4eef97ab,0x46738d95), LL(0xf0b479a2,0x66f559f3),LL(0x3262fb2b,0x9ea62dcd),LL(0x6a857025,0x91a39a5e),LL(0xbb3e6e11,0x11bdd19a), + LL(0x9353cc19,0xfa411fd6),LL(0x94cd2276,0x275d094c),LL(0x25243d1b,0xe2069b22),LL(0x630c8cbc,0x9f02f1e2), LL(0x379b6b63,0x85c942fd),LL(0xbdcc9307,0x293dcf87),LL(0xdc212ca8,0x56856d8e),LL(0x23a0c56d,0x1927e931), + LL(0x9c2f8b66,0xacfed288),LL(0x386ad1e3,0x20f6b94e),LL(0xdcbeff77,0x0e622304),LL(0x5978f2f6,0x67e895fa), LL(0x20029bfe,0x12a63bde),LL(0x8d968b8c,0x0ed75b6c),LL(0x57cec33e,0x611739ee),LL(0x42b9fc44,0xeffaae72), + LL(0x971a90a9,0xa7de79ce),LL(0x4fead0d5,0x529bd8a2),LL(0x9a1a43ef,0x78434c30),LL(0x4f3c848c,0x904d1db2), LL(0x6d4234ad,0xedb3c11e),LL(0x5975e9a0,0x4e0c9ec4),LL(0xff16ec74,0xc51236db),LL(0x36d09231,0xde652cff), + LL(0xe60a0bc5,0x0d7c18b7),LL(0x9d7df012,0xaf041383),LL(0x47d4cfd0,0x9c0ff3f4),LL(0x3d97bac7,0x64c7e6c2), LL(0x8cb44d50,0x239d2500),LL(0xbba96026,0x47189764),LL(0x31ddca37,0x22449328),LL(0xfb7c29ef,0xa53a1886), + LL(0x97a3789b,0x2515b665),LL(0x540ea258,0x825c5031),LL(0x09a5b24b,0x76680656),LL(0x60fb8bcc,0x833c240f), LL(0x01e55cc7,0x758e0b10),LL(0x8d260572,0x8b799c48),LL(0x6c5dd0cd,0x0981a802),LL(0x1b9c6cc9,0x6f6da55d), + LL(0x8c4d503a,0x3457b685),LL(0x009a7a94,0xc537730f),LL(0xd01dfdff,0x334d46b4),LL(0x15e20bc7,0x3e5dc6a8), LL(0x6ce8b8ab,0x1833b0bf),LL(0xc57a4292,0xe21b99ae),LL(0x1713ba15,0x743cb595),LL(0xe0bb44a7,0x550e41bc), + LL(0x2f8ebcf5,0xf356917a),LL(0x953f056c,0x6f2c400b),LL(0xdd84bb48,0x09d9ac41),LL(0xa61e98e3,0x50dc7a8e), LL(0x3d3a6776,0x1179a9d3),LL(0x16de8b3e,0xdda312e7),LL(0xce6e2bea,0x62a8b7c3),LL(0x645e4ca0,0x2b00036c), +}, +/* digit=16 base_pwr=2^112 */ +{ + LL(0xab6cf0b4,0x9ad2cbd7),LL(0xf13d1ddf,0x7a1e67f4),LL(0x746003ba,0xa58f0c73),LL(0xa64a8fcc,0x8263e888), LL(0xbe2452f7,0x535cbe37),LL(0x6ae81a76,0x93125766),LL(0x3a553701,0x7d2ed0ab),LL(0xb0717d78,0x93d7e7df), + LL(0xf9cf03ba,0x61bc013c),LL(0xfeee3a78,0x36223b88),LL(0x3d7e4c95,0x86efc923),LL(0x965625e4,0xaf3801be), LL(0x3f32fd9d,0xa7bba141),LL(0x4e564acb,0x70724dec),LL(0xb7edcac1,0x52587f32),LL(0xb3969985,0x0b1cd94c), + LL(0x661fbdab,0x9f08407a),LL(0x58e52151,0xc4d7c536),LL(0x63dfe954,0xa3765bce),LL(0xac2dddd1,0xc6829bfb), LL(0xdc6e4487,0x97393f65),LL(0xbf04c930,0x9ba29422),LL(0x18838c0a,0x493c6914),LL(0x4b2f35cd,0x41b137ff), + LL(0x4e1c5e60,0xc9e4aa24),LL(0x13253d51,0x54bb5289),LL(0x9bbabe53,0xf4a86ab3),LL(0x6ac0aa64,0xd561feae), LL(0x1911bad7,0x27a896ab),LL(0x64337303,0x9cb22b98),LL(0x161f6928,0xf14262fb),LL(0x59ba6c9f,0x1661885c), + LL(0xb82574db,0x4666ebd3),LL(0x8d8af3f6,0xc5e86672),LL(0x209319bf,0xcc645205),LL(0x3834d1a8,0xc183c12e), LL(0x49eb0f40,0x533d73da),LL(0x6aca649e,0x3bcab0bc),LL(0xe39e8361,0xa02f4c41),LL(0xa89bdc85,0x2391e7ae), + LL(0x608cbe2f,0x88067c5e),LL(0xf16c22f2,0xcdec82f6),LL(0xf1faf9dc,0x80aa719a),LL(0x2340185c,0x261fe951), LL(0x9713e72e,0xcb411366),LL(0x6d8a2a46,0xdb1e405e),LL(0x11106ce0,0xed174757),LL(0xf71c0e69,0x6d70cf6e), + LL(0xcf707c76,0xb5310322),LL(0x40b4b7d7,0x3a1eb2ff),LL(0xb1a2d26d,0xb83259fb),LL(0x799720c0,0xa83ffb0f), LL(0x0bedb326,0xeecb1928),LL(0xe9271715,0x4473e820),LL(0x2f2d71a9,0x506e6d20),LL(0x4319756d,0xe7b253b1), + LL(0xf576cb3c,0x27e19335),LL(0xdfb3b78f,0xe16e4573),LL(0x63da0001,0xaf96d785),LL(0xf7444c5e,0xb38deafd), LL(0xc0eb0e28,0xaca6b38c),LL(0x7fe41b98,0xa6ca3541),LL(0x18965b31,0xfeb37b47),LL(0x597d5bc6,0x739cc322), + LL(0x4cb1fbc3,0x82776041),LL(0x8e2a3ad1,0xcdaa873d),LL(0xc01318bf,0xb5f87b34),LL(0x229cb81e,0x0c692268), LL(0xa53089f5,0xb0724016),LL(0x05415313,0xb2976a53),LL(0xcee8fdf2,0x436eab75),LL(0xd82b13e5,0x8794e1a6), + LL(0xd5516e3d,0x0d51a072),LL(0x4b2fdb3c,0x7bae61ce),LL(0x550728ee,0x0d987e89),LL(0xee6778db,0xfd5a656e), LL(0xbb4d9d7a,0x942ee623),LL(0x2dc1baf8,0xfc06d64b),LL(0x47c3dc8e,0x5244fcd8),LL(0x8568653e,0x5e37e156), + LL(0x048c8355,0xe5c2c6ff),LL(0xa0474046,0x480872ec),LL(0x7ff92484,0x67e3089d),LL(0x29971b3e,0xdc071893), LL(0x5a63e8ad,0x3857db2b),LL(0x5f2e0a09,0xf617d94e),LL(0x5844de79,0x267e9818),LL(0x861f5f92,0xfdb103b2), + LL(0x1570d6e9,0xb969bd3c),LL(0xe0fb517e,0x7350b9db),LL(0x55aa543b,0x083b142e),LL(0xaa2bc581,0x424757be), LL(0x69ea3302,0x4bd50f64),LL(0xed4c8288,0x053dcf83),LL(0xc118ac52,0xac2b3074),LL(0xe76ca88b,0x57f066a8), + LL(0x0a3207cb,0xb9ac28fd),LL(0x205829b0,0x0ec2ad13),LL(0x4f403b64,0x76216e56),LL(0x6621dd5c,0x7eaef662), LL(0x4b06ac54,0x9e0cc836),LL(0x9648523c,0x16ac3c6c),LL(0x08eec1d8,0xe1208a1a),LL(0xbe054a54,0x1e90b3a2), + LL(0x23a836cb,0xdfab7f92),LL(0x6f7674c8,0x624d6bd2),LL(0xea06210a,0xc34a914c),LL(0xf26c4f23,0xba5314ec), LL(0xa33a11cd,0xd440b196),LL(0x75eb951f,0xf81ab63e),LL(0x39350f0c,0x05ebb91a),LL(0x92e9528f,0x3f3c08ec), + LL(0x4fe6f4e6,0x54ff8844),LL(0x79b7ba49,0x217c0e02),LL(0xbf3a4179,0x13c4372b),LL(0xa1434264,0x6e5ad10b), LL(0x62bd8ff2,0xc8426540),LL(0x85fe6ef1,0x7c3dd284),LL(0x05da0e84,0x2c59b300),LL(0x17468e18,0xf303ed94), + LL(0xa211ffd3,0xe19fac99),LL(0x3f1f6bca,0x408f9424),LL(0xa5648244,0x1f5b76d1),LL(0x95b2bd67,0xef3942e8), LL(0xb538f1d7,0x1b9dee7f),LL(0x444b8f85,0x1cb78620),LL(0xcb8ea6a3,0x9f8ecd63),LL(0xb9d3b71f,0xca111b2e), + LL(0xbdc4e8e2,0xff83d71e),LL(0x3f76a9d5,0x43745ddb),LL(0xa25856ee,0x72db22a9),LL(0x5e9a9ff7,0xf34d5aa2), LL(0xbc529902,0x01f6b5f3),LL(0x086f4867,0xadf5d31e),LL(0xca556b56,0xbd88674f),LL(0xfdc81625,0xfd00120d), + LL(0xfdde77f6,0x90fbaba0),LL(0x559ec6e7,0x266d3bfe),LL(0xc8094357,0x372acf54),LL(0x6c61bb78,0x772bd8e4), LL(0x1af9aefc,0xcb2ac592),LL(0x5b178386,0xacc3dc9b),LL(0x23438463,0x09963084),LL(0x8359f1e6,0xae84f973), + LL(0xa4cee642,0xc3b19aa0),LL(0xb19a918f,0xcd5ca5c8),LL(0xe67cb207,0x46ac0d2e),LL(0x73ffebf2,0x2ae45e10), LL(0x10ef065c,0xf84aad8e),LL(0x32a7e903,0xa0af57fa),LL(0x43d346dc,0x42295904),LL(0x7f170965,0x8d6f711d), + LL(0xb110cffe,0x11aa7070),LL(0x9755605d,0x091a100d),LL(0xd27d86a6,0xc5a0c654),LL(0xdb30feaa,0x1031a244), LL(0xc02228d9,0x36804045),LL(0x8b746039,0x1877fc67),LL(0xe09bb238,0xba554015),LL(0x1de9b40d,0xc50d8f92), + LL(0x032f7a2c,0x29e40d88),LL(0x1d315ec2,0x514b56dd),LL(0x61778f04,0x9052652e),LL(0xe1a1c195,0x0d2bc606), LL(0xb05043ae,0x375fd7ec),LL(0x6eba4d1b,0x03b82577),LL(0xc349b39a,0x8e61b567),LL(0xb35fc43b,0xa670973a), + LL(0x5571b4a7,0x80c05ca7),LL(0x8a4af0ba,0x95d14f49),LL(0x67bf4290,0x96767cdb),LL(0x46870ef1,0xb293372f), LL(0x1afe1027,0xc6494405),LL(0x9019c4c2,0x9f027a1c),LL(0x188a593a,0xa392ac59),LL(0xfcb6e1ca,0x68acca2f), + LL(0x68ed921b,0xd8f86cbe),LL(0x712d2c07,0x24679ac2),LL(0x4e672cd9,0x18fbdb21),LL(0x51d1f8e1,0x401bb718), LL(0xaa8da4a1,0x688792e1),LL(0x3ca93d06,0xedf9266f),LL(0xaed484df,0x5ddba14a),LL(0xb0ea10a5,0xa5dab102), + LL(0x833396ea,0xd397edcd),LL(0xed5e6747,0x78a75693),LL(0x1a5f8260,0xf2c844ba),LL(0x5fb9fec5,0xbcafe59d), LL(0xd3147e7e,0xa2413d5f),LL(0xafdf26cd,0x130dd9e3),LL(0x9ad1abde,0x44be87ec),LL(0x6e77fbe8,0xe925c495), + LL(0xf26ced16,0x07ce8d96),LL(0x86ef7306,0x36c85643),LL(0xc7e3d409,0x62545902),LL(0x2ed4d38e,0x1747bf4a), LL(0x55adc839,0x6fe6fc3d),LL(0x8eaf64a8,0x20a3cc09),LL(0x622887b1,0xc1e9b766),LL(0xc41ac659,0x7b9d2f96), + LL(0xf2a65e45,0xfdb897ce),LL(0x97889eb8,0x0c09c905),LL(0xe4becf5b,0xa15df10f),LL(0xccef7e40,0x14a3d4fe), LL(0xa8fc67bd,0xedaa11f6),LL(0x5d185b42,0x7bf6fe9b),LL(0x6f9cb5c9,0x7bb9f1f5),LL(0xf97ea9cd,0x1b4ab74e), + LL(0x07638d62,0xe9ebf11d),LL(0xa78cf538,0x413a4a87),LL(0x570dd371,0x93785f86),LL(0xfb48063c,0xba431a91), LL(0x4ed4e5fa,0xf1f2ea5b),LL(0x5308325a,0x91a72c47),LL(0xc9ea6acb,0x4e6518e7),LL(0x208f67e3,0xfeaf4c3c), + LL(0x2c16bb1a,0x98c5d768),LL(0xee31dc08,0xbf91b62d),LL(0x33a54959,0xe9ad408d),LL(0x38b7170b,0x9d754a64), LL(0xd9d6da2b,0x106db7bc),LL(0xadd533af,0xf556cbb4),LL(0xf16d3b58,0x62db0de0),LL(0x1fa579ba,0x78a1b0be), + LL(0x7b552058,0xda96740b),LL(0x626c4d93,0x0c689cc6),LL(0xaf68e53b,0xee3dd5c9),LL(0x134d763b,0x78653a9f), LL(0x3ca5aa67,0xec9c1b72),LL(0x7d56992e,0x67471dac),LL(0xad1d8580,0x0a97dffe),LL(0x0063c040,0x11c7d93d), + LL(0x6e90b157,0xb79e355c),LL(0xd9c894c4,0x2c06edcb),LL(0x71a75ed7,0x9b931897),LL(0x8e95ad91,0xd7f00247), LL(0xb85bf054,0xfce1b489),LL(0x503b38bf,0xa3ffb8fd),LL(0xe0fe2ec9,0xe7ea3ad4),LL(0x0049865d,0x0f32f520), + LL(0xcff21b51,0x33afa324),LL(0x62a1cd24,0x3d6847d7),LL(0x0b06ce2f,0xf534e159),LL(0xae7cdae0,0x24d27b3d), LL(0x4ad68ab5,0xb727db29),LL(0xb63c0bc9,0x7e8e47e3),LL(0x02389039,0xe81f3122),LL(0x88e6a17c,0x0c281f52), + LL(0x091e1c4c,0x3cc00e21),LL(0x867ccc75,0xd500db44),LL(0xf5ebbbe4,0xa8e2e84b),LL(0xc4279ac0,0xc3266881), LL(0x7a170658,0x2e8fb4de),LL(0x51da4a2e,0x219c5ec1),LL(0xeeacee19,0xda69a3fd),LL(0x30462243,0x9d4c6fbd), + LL(0xa978c29e,0x43673fe8),LL(0x5861bbc1,0x6e825c95),LL(0xdba8e7ba,0xb41d1435),LL(0xb885d004,0x0f286f78), LL(0xee57436f,0xea42b7fd),LL(0xef7e29c7,0xcdae14bc),LL(0x24251056,0x50cff3f0),LL(0x6f6e8cb1,0xf60809fe), + LL(0x12932e53,0xee9f1d15),LL(0x167d5097,0xa6e55cd6),LL(0x9d926359,0x5df8816d),LL(0x797b7eca,0x108e2117), LL(0x91921587,0x7ba20319),LL(0xad23652e,0x304138e4),LL(0x51ebc32f,0x73d0ed57),LL(0xf01d0fc3,0xe0c10a38), + LL(0x78c49b19,0x14c6b72f),LL(0x3b7c7418,0x4f952b7e),LL(0xa2d019bf,0x3fe75d21),LL(0x7ca33e87,0x4837bfd2), LL(0x3946e7ea,0x4597729b),LL(0x4c37ea10,0xbe22c14a),LL(0xd7909474,0x91106c7c),LL(0xbf5551a1,0xfbf1e7db), + LL(0x55ffea13,0x8e06336c),LL(0x0a1f99f5,0x0deaeda0),LL(0xfda98fc9,0x9b738c4b),LL(0xa59c98ba,0x061cc613), LL(0xb477692c,0x5ceb5b83),LL(0x1fcc473b,0x5db77559),LL(0x83df476c,0x77214b62),LL(0x427ea01d,0x2ffac971), + LL(0x49fd0ba7,0xf29f600b),LL(0x7233ef79,0x1106f8b2),LL(0xe8a0ca35,0x706bc171),LL(0xacbff08b,0x4da7a9e6), LL(0x7725f97c,0x17c2fa4e),LL(0xe84a5095,0xab459314),LL(0x6b24d47e,0x01556f14),LL(0xb016dc1a,0x01399059), + LL(0x28eca6c6,0x154b84c7),LL(0xd9084e68,0x88ed8612),LL(0x00bf9b5b,0x4dfd5080),LL(0xba9a0cca,0x853cd8ab), LL(0x8af0e94b,0x8cbf9bd8),LL(0x83035d16,0x50782b73),LL(0x4f44533c,0x694d3e65),LL(0xa6e534eb,0x155d4bf4), + LL(0xee908b6b,0x9028e2ab),LL(0xa6743060,0x36e5aac0),LL(0x3c37d6f1,0xd26f5a51),LL(0x33729b9e,0x8483703e), LL(0x2e5f82a5,0xf27a6603),LL(0xca55d187,0x33bf2bdc),LL(0x7724a85f,0x894c415c),LL(0xa2ea178d,0x9255d416), + LL(0x0a6dc5b9,0x35ecb4e2),LL(0x51c54ed2,0x8b3fc2c8),LL(0x9eede13d,0x059f86eb),LL(0x791dd5eb,0xa796c750), LL(0xea88555b,0xb2f33680),LL(0x1245348c,0x92730950),LL(0xd1e63bfb,0x1a5545f8),LL(0xbebb7858,0xfebc9a14), + LL(0x6bdf0c84,0x13cce767),LL(0xa1d341fa,0x1aa9dc07),LL(0x1ee6fa43,0xd46e8ff6),LL(0x4b1dda64,0x4198e5d6), LL(0x2782abd1,0xe82a8134),LL(0xb6d64830,0xe6f7b1b4),LL(0x7f0fb374,0xabe00274),LL(0x7494d7d3,0xf1a8e2b7), + LL(0xf274f296,0xd16b0d9e),LL(0x65246ee9,0x3c94a7ac),LL(0x91ec2262,0xd32c32da),LL(0x83116ec1,0x04c7bb90), LL(0x78524a06,0x70fa0406),LL(0x07df8064,0x8d2d5176),LL(0xe2c8d48a,0x13e589f2),LL(0x122aed4e,0x3b110ac4), + LL(0x34e972cf,0xe8e0eb52),LL(0xfb3a77fe,0xc082944a),LL(0x6a32c23b,0xcdaff7a3),LL(0xc37b4a2c,0x88cc568d), LL(0xe27b2552,0xc9979a9c),LL(0xd6ef51f9,0x8612ae7d),LL(0xef4e8f85,0x7bf0f937),LL(0x3f12d45c,0x2f360a58), + LL(0x9b336663,0x3ec9d0e3),LL(0xb1438d2b,0x5ac2df38),LL(0xff93fde4,0x7f2de910),LL(0xd92534ba,0xbbc460da), LL(0x59a94ab9,0x74de3159),LL(0xc45b80dc,0xd51cfd32),LL(0x6e5b2397,0x9f1f349c),LL(0x995f7271,0xbdbd16ed), + LL(0xf59709a6,0x4a7efc1c),LL(0x4b3db99a,0x74e2760d),LL(0x7726a2e1,0xa7238e9e),LL(0x1a376013,0x47764208), LL(0x7e33ebc0,0xbc96f396),LL(0xc9e4ec0d,0x31e159e6),LL(0x6a2ab9f6,0x26a5aef2),LL(0x320eeea7,0x23add07c), + LL(0x833b45b6,0xa79a97c9),LL(0xc51885e6,0xb08da907),LL(0xae8d5659,0x23f5e651),LL(0x1faff2f2,0x1eb0be48), LL(0xa1e090da,0xe414ee3d),LL(0x7fcb664f,0x16e4f8fa),LL(0x98c36865,0x7a15a7e4),LL(0xaf89dacf,0xea60e8fd), + LL(0x86c1a4b4,0x4e009f45),LL(0x769644ad,0x78c1bebf),LL(0x0b4b3501,0xa41b480f),LL(0x57f0a0e9,0x98be5037), LL(0x3af24657,0x06934887),LL(0x2b6260f9,0xe2503ddb),LL(0xd1d0caaa,0x37c936c2),LL(0x16431f50,0xd371e991), + LL(0x6087c5e5,0xd9621d16),LL(0xc53a8bc5,0xae49c2ce),LL(0xcad314cd,0xd7868882),LL(0xaa57af18,0xfde10dc7), LL(0x3800f397,0x3fa8a60d),LL(0x388b333c,0xcec8ae7b),LL(0x85fa8129,0xefd8d692),LL(0x90776c32,0x33d5685b), + LL(0x65971a6e,0x47ecec0a),LL(0xad7c5755,0xe8a20bbe),LL(0x87372016,0xbeed0a4d),LL(0x1d9b8dc0,0xd0d499bb), LL(0x2800907d,0xf4ce27cd),LL(0x8622bbb7,0x07a30b77),LL(0x77e264db,0x7532f435),LL(0xd64f0710,0xfdd1a9c3), + LL(0xa6543720,0x92ca210f),LL(0x5bb6985a,0x2f19ed66),LL(0xf9399b43,0x08a4ac61),LL(0x757f241d,0x0b7bac5d), LL(0x3763c20d,0x93ef27cc),LL(0xa86b1f05,0xda3b206e),LL(0xd62467c0,0x8f19c74a),LL(0x6a3ad263,0x3ec5ef6d), + LL(0x2bc8b9f0,0x249aa636),LL(0x7f70990c,0x0fca7318),LL(0x6f038497,0x6d4aabc5),LL(0x5afaaaef,0x4647f72b), LL(0x7bf90444,0xc7cbe03a),LL(0xcac2efb0,0x6beb69ac),LL(0xbb3c6dc0,0x58544eba),LL(0x96aefc2f,0x569cdcd6), + LL(0x5e4c950d,0x2e8a4c57),LL(0x4dd32002,0x6f5341d7),LL(0x6efa5908,0xd0345db6),LL(0xf1d2bbe6,0x4b043197), LL(0xe8730bae,0xde8a4cb6),LL(0xa485cb62,0x9a89231f),LL(0xfcd9bcd4,0xb24ace89),LL(0x3ed5801d,0x01892cc0), + LL(0x413408f9,0x80ce2f30),LL(0xf8773b6a,0xaf67343a),LL(0xdd6ade17,0x91acc6d2),LL(0xe5eb3def,0x9d2ffeca), LL(0x50b029fd,0x72f8af06),LL(0x79c0aabd,0x339c802c),LL(0xafc0a6ad,0x46161fff),LL(0xbac9a2d4,0x1046d9f8), + LL(0xab920e51,0x2f12eb06),LL(0x2892e552,0xfc004900),LL(0x13e61154,0x9aadf93e),LL(0xabcfd127,0x4468da94), LL(0x152f220e,0x6a5d3ffe),LL(0x4737fe79,0xe6260c23),LL(0x5e6b4251,0x8b5dd1c9),LL(0x9af02b98,0x044f0656), + LL(0xa97ff366,0x434d475c),LL(0x2c4bcc46,0xbae8db8e),LL(0xf9777251,0x2ba43a8b),LL(0xdd019f1a,0x7ff430a5), LL(0x630064ba,0x65e9f290),LL(0x7e871c0e,0xfc57a54e),LL(0x5791ae48,0x54107bbb),LL(0x5c334de0,0xdfce137f), + LL(0xaed5be73,0xab65c8f6),LL(0x174bf00b,0x838c3958),LL(0xf1c7e594,0x27c57199),LL(0x0d02fae6,0x62643d81), LL(0x5f4431fa,0xc1e70c12),LL(0xb2b41f7e,0xfac86124),LL(0xf0108e3c,0x062ac35a),LL(0xa43d3a28,0xd7d34dfc), + LL(0x3143de4d,0xc40fb44a),LL(0xd2e0f9ae,0x06eac4ed),LL(0x95d9a69a,0x998f3211),LL(0xe950bd2e,0xb268f6a0), LL(0x1e9d4e40,0xadfab210),LL(0x73ce9324,0xc453a68c),LL(0x80881171,0x5e2f2544),LL(0xee7e543e,0xe4f596db), + LL(0x76b6698e,0x77f48e4e),LL(0x227365c6,0x47b5165f),LL(0x14ef39e6,0xf56ec8d4),LL(0x74ce46fa,0x1638d647), LL(0x08aa8b9a,0x244d0fac),LL(0x298326c9,0x98ccc4d0),LL(0xce0d2983,0x492d7661),LL(0x73158cda,0x728b3e3f), + LL(0xc4948011,0x7959ca67),LL(0x08425a38,0x32044ae9),LL(0xb1070c2b,0x05a7a6a7),LL(0xcc129ba5,0x34ed541f), LL(0xb2f1c3e2,0x4b6bf65c),LL(0xd0d8aec8,0x6f090ce6),LL(0xd4fe39c1,0x11ade23a),LL(0xa5a35528,0x50510c08), + LL(0xad6fd7c6,0xb7e2a5de),LL(0x2745dca8,0x9d691939),LL(0xad053184,0xff669c38),LL(0xecd0703e,0x394ca6b7), LL(0x60b15e75,0x59e32e80),LL(0x13c14864,0x82dde889),LL(0x64d4f086,0x0fd1624c),LL(0xc9893d7f,0x7fb502a7), + LL(0x711479a1,0x59b86bcf),LL(0xc40b44bc,0xfd4bc4d8),LL(0x988100c3,0x2fae18f5),LL(0x615867d7,0xe4af2f4f), LL(0xbe479e28,0x7d45e1e8),LL(0xa04798a5,0x547accbd),LL(0x1c458b5e,0xe88a85b1),LL(0x6032f0cc,0xe144f7f2), + LL(0x3f807365,0xad5276d3),LL(0xb318a6ea,0x5b6519e7),LL(0x2d0fcf50,0x5889cbb5),LL(0x2bdab4e0,0xdce91cab), LL(0x41b78954,0x17b6249f),LL(0x6f10449b,0xc9320b65),LL(0xf264ae8f,0xe38a7cc0),LL(0x52b85829,0xaab8803e), + LL(0xdd97973c,0x63668621),LL(0xd04138c7,0x5aaedce7),LL(0x1762874c,0x8e8e6614),LL(0x163fc862,0xd0cefcf4), LL(0xffed1ace,0x0ebe0048),LL(0x7a8c2673,0x070c3348),LL(0x9b0d3fd7,0xb801d159),LL(0x922d4842,0xf1d55911), + LL(0x680dcbf9,0xf0acf768),LL(0x4f0a51df,0x5072b825),LL(0xd88df9c5,0x3a74751c),LL(0x1cc1a332,0x9d20f989), LL(0x6926c34a,0x4e90042b),LL(0x00766880,0x5c728b1e),LL(0xf76e9dcb,0xf2e3bfe8),LL(0x15a125ae,0xd9822f0a), +}, +/* digit=17 base_pwr=2^119 */ +{ + LL(0xf51b14b0,0xbf84db58),LL(0xa39a79f0,0xdf73ccf5),LL(0x2b5a1f11,0x0ce1e584),LL(0x185fc400,0x841fa6a3), LL(0x2455c32a,0x94b09c68),LL(0xbfa71cc3,0x383c9bde),LL(0x1e797929,0xb6381486),LL(0x623d0a5d,0x33036faf), + LL(0x90f17cba,0x41b6cf7c),LL(0x30c7c5f4,0x5d655ff4),LL(0x4ccc7f38,0xc64f29d5),LL(0x6124a79e,0xf28e8531), LL(0x67bf1e98,0x1efa8d51),LL(0x5d7a33b0,0x8610027f),LL(0xcb9a40a4,0x35fe2bb2),LL(0x43d50a0b,0xc5cc1bf1), + LL(0x46e33870,0x84dbc605),LL(0x843c4e1e,0x23d8d2e5),LL(0x4cf8b569,0x69964b5e),LL(0xe0c546a5,0x2a5228e8), LL(0x96d6e111,0x4c0467ed),LL(0xa12bd298,0x25764cdf),LL(0xfbaaad46,0x92a3e7fa),LL(0xd12fa469,0x08ac1d36), + LL(0xa32106c2,0x60ae2bbf),LL(0x3e917750,0xef155b2a),LL(0x13853a30,0x5567c3c7),LL(0xeddb305b,0xa6be8290), LL(0xade26eec,0x2db58c21),LL(0x003c17ed,0xfa3c895c),LL(0x6293f8a2,0x96ab0de1),LL(0xac3710c6,0xbd2365ec), + LL(0x6aa24f73,0x93ea8553),LL(0xe0410c40,0xf75140d0),LL(0xaff0f7f2,0x760cfa2f),LL(0x3e580d68,0xc6dfb3c7), LL(0xc16d39e2,0x25fc2091),LL(0x19e1d5e2,0xa05b0c81),LL(0x62bbec7a,0xd4d223d8),LL(0xf293f34a,0x11a93775), + LL(0xe194c642,0x9ab03e73),LL(0x789e3c85,0x607b7106),LL(0x4bdacd55,0x952aab02),LL(0x21cc6084,0x31ca3ee2), LL(0x1c6b93f9,0xd3149b2b),LL(0xead930f8,0xcbc5ef3b),LL(0x22872630,0xed04984f),LL(0x6c4b6fe2,0xef5d052d), + LL(0x6010ffa2,0x808ae6c0),LL(0x1143166a,0x88b6fcd8),LL(0x5ab945ec,0xa2780263),LL(0x36db5012,0x4777b4aa), LL(0x059aa3c7,0x2156bf83),LL(0x2a660260,0xcbef6fb7),LL(0x8b366ce5,0x189fa221),LL(0x08290914,0xd6f5bdaa), + LL(0x57028677,0xd2028d05),LL(0xce9aabdf,0x90eebeeb),LL(0x06d4e5d0,0xab977aee),LL(0xf9361873,0x7a98c527), LL(0xb7c2474d,0xe49b1251),LL(0x5f3e7b02,0xcdaf2a36),LL(0x6fe54af1,0x638bcaf4),LL(0x1dac06b7,0xfec42624), + LL(0x3741a88b,0x422be225),LL(0x5304f139,0x1f3b82c3),LL(0x181c2fc2,0x101eab8e),LL(0x5898163c,0x8522738e), LL(0x2d2bac1b,0x0372d05f),LL(0x1d1f3c42,0xb65c786c),LL(0x64e2f5b3,0xbd59694b),LL(0x24c5671b,0xf1c424bf), + LL(0x1eafe37b,0xda738cf5),LL(0x30dd7c2b,0x503eac24),LL(0x11dd77af,0xf9b7b7a5),LL(0xe9dcfe7c,0x0ade03af), LL(0xf479e3b5,0x489bd34a),LL(0x030a33f3,0x993ab403),LL(0x9fb64068,0xaef322bf),LL(0x0e27f110,0xa47cc71b), + LL(0xefab99c8,0x1c445554),LL(0xa7f10e58,0x7c3c51e7),LL(0x78a87474,0xaa8b43ee),LL(0x2418475a,0x037d6397), LL(0x10324911,0xc9c751fe),LL(0x3e0797d4,0x3d65d9e0),LL(0x7dea2a63,0x98b68d2b),LL(0xf4afca19,0xa211ed3b), + LL(0xc63b9e75,0xe19ff8f8),LL(0x0d7d130a,0x43f6b4fc),LL(0x7c8490b7,0xdba3385d),LL(0x0252c14a,0x97889df7), LL(0xb2f18b9f,0xfccfca86),LL(0xc3a87422,0xf092ff9e),LL(0x67474838,0xf96dd675),LL(0x5bad2e9f,0x039e8287), + LL(0x52e041f6,0x7ed85e70),LL(0xcfdeb19f,0x3d6ef1e7),LL(0x0d9ac66e,0x9f9fe399),LL(0x16cb8554,0x5825e7bf), LL(0xd954a4d5,0xecffdf90),LL(0x20678fc5,0x8617ffdd),LL(0x666df77b,0x3e974054),LL(0xb5d92788,0x748379d1), + LL(0x2da32c0a,0x46a60911),LL(0xb2676ca3,0xb2e1ac32),LL(0x17614dc6,0xfb74093f),LL(0x3f27f965,0xf44939e4), LL(0xc922422b,0x4656a402),LL(0x3ff5c56f,0xd60a55ba),LL(0xab9aa62e,0x0d061b41),LL(0xaca3acd2,0xc9ceacfe), + LL(0xd946003b,0x056d5718),LL(0x2c7815f3,0xf8530d6d),LL(0x706536b8,0xbae14342),LL(0x2b901188,0x45c96dda), LL(0xc64ed946,0x386d88b6),LL(0x6c00f1c2,0xb7017022),LL(0xec8988df,0x28519811),LL(0x5a05cffc,0x3b011fe2), + LL(0x515f954c,0x4f581d47),LL(0x7f470a40,0x145f925b),LL(0x736feaaf,0xfee6b6b0),LL(0x2ea5743b,0xf90744af), LL(0xa2f36f56,0x4d8e8cea),LL(0xe3ed4007,0x4239a6ce),LL(0xd515e6db,0x0906b5bd),LL(0x8ac973d1,0x53622990), + LL(0xeb2fe229,0x472ceb94),LL(0x6a121363,0x0775ed41),LL(0x761ddb38,0xc0492e07),LL(0xaef9be2f,0x80c24d51), LL(0xdcba73a1,0xa2a3982b),LL(0x4e26d062,0xe0d83978),LL(0xcd41c930,0x794959a8),LL(0x70131161,0x7d2a88d7), + LL(0xf4f966da,0x48f93fc3),LL(0xed5b6487,0xf92691a0),LL(0xada2c1fc,0xc5a93e5d),LL(0x4b7d9243,0x4a7aca52), LL(0xd7c5598b,0x810aba93),LL(0x25109638,0x98f9ead2),LL(0xa388481a,0xe8c6e893),LL(0xe91ce74b,0x56e96b9b), + LL(0xd935f591,0xfa1e5dc3),LL(0x555eb816,0x985bb06c),LL(0xc4d14e69,0x6478c518),LL(0xc7f47349,0x48afbdbc), LL(0x26fed96c,0xbde90933),LL(0xcd468186,0xf9b96f41),LL(0x730e8518,0x22de6a29),LL(0x915db842,0x7a3dc912), + LL(0xfc1f9449,0x8d13b665),LL(0xdd4bba75,0x6e9932a9),LL(0x564083da,0xa90ce8e5),LL(0xbbf7989d,0x8a7cf362), LL(0x1b07ee2f,0x652eccb7),LL(0x6857a141,0x0c0dcf1a),LL(0xb7bfb43e,0xa87ec410),LL(0x82b8d179,0xaebdb7e7), + LL(0x625a24dd,0xeb3bc468),LL(0x463b1f89,0x7e45e47b),LL(0x00c75a48,0xc3013535),LL(0x13778807,0xafea920d), LL(0x22dcef16,0x0d1e9277),LL(0x86cecfd6,0xa2a10f67),LL(0xd7160bf2,0xad40e29c),LL(0xeac1265e,0xe78e6589), + LL(0x0c62c041,0xd3a24310),LL(0x6c03c747,0x4d27344a),LL(0x7d3ee9d1,0x0b19e4a6),LL(0xcd90de33,0x9cf2eccd), LL(0xfda636a9,0x673a9d1f),LL(0xa86ee501,0xb7349981),LL(0xe43766ed,0x11ca1e49),LL(0xe3ff3b08,0x0806af6f), + LL(0x8a01f119,0x21304338),LL(0xf3cb268f,0x58a6d3be),LL(0xe37d7851,0x40ceacca),LL(0xef5b81e8,0x18694595), LL(0x84bad32a,0x35678ed7),LL(0xd1624256,0x4f280f92),LL(0xfb28709c,0xdecb1f1e),LL(0x164911d7,0x2a7f3048), + LL(0x579d8a41,0x32551d31),LL(0x60a5ee33,0x754c7c24),LL(0x6a88f85f,0x2c53fbff),LL(0x2c7a36a0,0x6ad0bda7), LL(0x15724d6c,0x8b3674f8),LL(0xb9b7b34a,0x614727ce),LL(0x82ca9cd7,0x384fba98),LL(0x0c566025,0x8ef4343c), + LL(0x64886c98,0x5645fefb),LL(0x0f5c29e8,0x702befb3),LL(0x46de1523,0x6d74a7e0),LL(0xb1302163,0xcb2bcdb9), LL(0xab4ca69b,0xe65cff39),LL(0xf2d4f6ec,0xeacb7103),LL(0x1770d3ef,0x15826c2d),LL(0x3f710600,0x38b008f1), + LL(0x4bc7dccb,0xc68450cb),LL(0x9e5f2400,0xb5f11b26),LL(0x9c3a833b,0x2af58e9e),LL(0xa058abaa,0xb60e2672), LL(0x75b080c0,0xe891bf8c),LL(0x2434bf38,0x5b09b276),LL(0x700b8739,0x0d90a040),LL(0xe69f4a0b,0x995cb042), + LL(0x44a56b84,0xe30df0a1),LL(0x1ead5a62,0xbaf92d16),LL(0x6e0193a4,0xe214a062),LL(0xe9758b9e,0xd41de5bc), LL(0x732d82d5,0xcf214213),LL(0xf949f07b,0xaa1421f6),LL(0xf7fb101c,0x5f38c91e),LL(0x2a3e41e4,0x47ce2ec2), + LL(0x240c7897,0x6bb34768),LL(0x7b45473e,0x80ff54ea),LL(0x82fe5aac,0x16acd40f),LL(0x4350449f,0xa3e76f52), LL(0xacacbeb9,0xf7a3579e),LL(0x7bc40732,0x9791e0e0),LL(0xbc58cb9d,0xb26da7b5),LL(0x987e18f4,0x11d9fc80), + LL(0x1d8e0d34,0xc3c189a8),LL(0x2d42e0b5,0x3011097c),LL(0x94ab9371,0x4e385932),LL(0x0c237147,0x79e0c2ce), LL(0x7172e6ce,0xc9f17122),LL(0x9b89a008,0xf8d73b1d),LL(0xa945995d,0x91690c6b),LL(0xc15306c6,0x089eb941), + LL(0x12ac8df5,0xee5f526d),LL(0x3bf94c65,0xf1dd23f7),LL(0x81a3cb0e,0x594ceaac),LL(0x9757cc8b,0x07d41d3b), LL(0xfc5603d5,0x9eb0081d),LL(0x57bd230c,0xfb5d3298),LL(0xcde3f845,0xf2c0402e),LL(0x41e8aba6,0xa2defd67), + LL(0x2dd9573d,0xb300802a),LL(0x60c1ded3,0x64e401a5),LL(0x8ab1d3d8,0x19d4a677),LL(0xcca04f74,0x3c2092f2), LL(0xac40056a,0xf4827ba5),LL(0x9c09ddc2,0x49d4cf22),LL(0xdbf20277,0xb2b00f6b),LL(0x5b281e9b,0xc9ac48d4), + LL(0x32efbbce,0x648d6674),LL(0xe9639719,0x64a6c2b3),LL(0x30662e7d,0x38c04657),LL(0x352c9184,0x15d1d7ca), LL(0xcc3020cc,0x70e8630c),LL(0xb09f038f,0xe4b56c9c),LL(0xfe76a744,0xdb9cb5ed),LL(0x6947b988,0x4c85f020), + LL(0x29d8add4,0x7e500126),LL(0xbfaf6d7e,0xdbcfd295),LL(0x38df80be,0xc1a1c228),LL(0xf606ce3d,0xcfa6272a), LL(0x8e0af540,0xbf2a5720),LL(0x5b599ab0,0xb9c544fd),LL(0xd0a22c9a,0xd6dc994d),LL(0xd23e4c0e,0xa8a12acf), + LL(0xba588a5e,0x41f7ac85),LL(0xccdb9687,0x5425fa00),LL(0xec9398f0,0x12fbce76),LL(0x4f550b9b,0x2ad69251), LL(0xbb318636,0x120ff0f2),LL(0x01ecd90b,0x9378346c),LL(0xd0ba149b,0x1b98fe99),LL(0xc9c481c8,0xd69d5462), + LL(0x959e428e,0x11c79184),LL(0xcff227cc,0x9de61a8d),LL(0x1e09b860,0x144dfdcd),LL(0xf8ebe350,0x110c3a47), LL(0xfadf86b0,0x59e574dc),LL(0xcf3b8d30,0xe6ff6e12),LL(0x19c77143,0xe2d512fc),LL(0x60279af1,0x63461543), + LL(0x32b4d371,0xff65189c),LL(0x0faf5ba7,0x022fecca),LL(0x414707b4,0xd08fe9bf),LL(0x722d5fd2,0x0ef8af2b), LL(0x4e6fa14a,0xbef06063),LL(0xcca29177,0x1c41f752),LL(0x65091fe1,0x17dc7e18),LL(0x23f13c18,0x693d72d2), + LL(0xce8e2d30,0xce88eb02),LL(0xe972faca,0x7071f98a),LL(0x549c38ee,0xb7388d61),LL(0x0b788b8c,0x7cfccee2), LL(0xcb93b5e8,0xdc470705),LL(0xab96d485,0xea053c18),LL(0xd634c9b3,0x70e96456),LL(0xd5999cf2,0x2c58c20b), + LL(0xa77c1767,0xcd392b3c),LL(0x7c4e6bd9,0x14471fab),LL(0x75c905ff,0x312e1547),LL(0xace47038,0x45764654), LL(0x8fc3d697,0xa34a0b0e),LL(0xd309ed3a,0x5d9ad31a),LL(0x0918f957,0xbba340c0),LL(0x31fd72a1,0x768e06e8), + LL(0x3e1a4a54,0x77e5dd92),LL(0x3fdbc1e1,0x0970719f),LL(0xb0371fe2,0xd4f1da6f),LL(0xfd7f895a,0x3635f433), LL(0x411c8e6f,0x0e8e40e6),LL(0xec18103c,0x31d126bd),LL(0xc299d7cc,0x415a0cc1),LL(0x3a8e97f1,0xdf528e7b), + LL(0xeed95e91,0x4551a8c7),LL(0x32bcfb03,0x8de89888),LL(0x2eac5c3a,0x25da4f5f),LL(0x5f88d63f,0x6d0b2e25), LL(0x575d6145,0x8d158d14),LL(0x345f62b0,0xe5601a6b),LL(0x113c6895,0x6f951699),LL(0xb87e50ef,0x79e29fd5), + LL(0xd5fa51ff,0xf1ab215c),LL(0xaf2c3094,0x4fc5c4ea),LL(0x2c006042,0x1baeda40),LL(0x3e30e75f,0xcdfcc37c), LL(0x467f57eb,0xdd64e5dd),LL(0x22902d21,0xa5b13731),LL(0x1c52cb7b,0x856866dd),LL(0x16a08caa,0x05cf0f7a), + LL(0x533b4d09,0xa46e8a55),LL(0x4e073af1,0xfc803998),LL(0xe0d589c3,0x8e3825c8),LL(0x4c1daef3,0x505e8e5d), LL(0xc5f3db12,0x9f8363b1),LL(0x74f569e2,0xe7d46700),LL(0x4d68461a,0x551fd2ed),LL(0xa8bbe83d,0x26248da5), + LL(0x65681dbd,0x8d90c47f),LL(0x2200ba6b,0xe726d25e),LL(0x65a3bc9b,0xa2fe408f),LL(0x9c443b57,0x94a80457), LL(0x07364677,0x95f7f024),LL(0xdaf0fb34,0xe9d9bc87),LL(0x5588e979,0xe9082548),LL(0xa0e61ff2,0xede1f94d), + LL(0x45e1c230,0xcb89a1e8),LL(0x50a15304,0xee014c23),LL(0x2bab57e1,0xf25d8ffa),LL(0x26223c6e,0x8a920680), LL(0xaadf7e6a,0xc5abb7af),LL(0x9e7d8da5,0xcb57c893),LL(0x7d589a91,0x839bcda0),LL(0x77e82027,0x1fa774c0), + LL(0xba6504d7,0xeca669cf),LL(0x6845e47d,0x7bf09544),LL(0x607b3641,0x5eb6c33e),LL(0x64bab450,0xf445556e), LL(0x86096fde,0xed0b1c02),LL(0x8ea41693,0x2c5ba666),LL(0x37ec248d,0xe578b315),LL(0xf64ed28f,0x97ef44fe), + LL(0xce419462,0xfa5a6c46),LL(0x9cce80e9,0x29336dc9),LL(0xeee7617f,0x9e9054b9),LL(0xf3d51cba,0xcea9a100), LL(0x13267ec6,0xc3cce5e8),LL(0xa4e807e7,0x650c8383),LL(0x9b2726dc,0x1338e22e),LL(0xbf79b47a,0x220c50b2), + LL(0xa0e0962a,0xe160d496),LL(0xe1ed5cdc,0xe1a26460),LL(0x31427c62,0x9a1ed8c3),LL(0xe99a096a,0x65ef5300), LL(0x4e3ad558,0x38abea5f),LL(0x0880ba0c,0x03bb15e9),LL(0x0141b036,0x1e6dda7e),LL(0x5bf37669,0xd31b08bf), + LL(0x68da20d2,0x948e0366),LL(0x4108fe36,0x36998a24),LL(0xf9d6563b,0x7606e6ed),LL(0xe42172ba,0xcf7cbdd3), LL(0xa1265b99,0x2335a9a4),LL(0x30ac3f69,0x64776cdc),LL(0xa59b595e,0x04040362),LL(0x2cbc03cd,0x82df96b9), + LL(0x6cea2796,0xe9d18c7f),LL(0xe1ea7e35,0x3112c4f6),LL(0x5f8a786d,0xf9cbc205),LL(0x2097da0d,0x36cc6d42), LL(0x2153e665,0x54093350),LL(0xce937bb9,0xebe9db0f),LL(0xd95942f8,0x9d1a5972),LL(0xd4bd5c74,0x81c1f94a), + LL(0xaa04152e,0x61dc7318),LL(0x95e5ec9f,0xdf636db1),LL(0x48090850,0x64a80d46),LL(0xce890a30,0x2b37679e), LL(0xff6025e3,0x9f46d5b9),LL(0xf24534dd,0x6eed5a44),LL(0xf740a84b,0xc56b5cb1),LL(0x228cc826,0xb4641c28), + LL(0xaf62b943,0x676289be),LL(0x1eae5409,0xe3f3810c),LL(0x04b5be78,0x73613f32),LL(0x398b556c,0xe6359179), LL(0xc0263f77,0x6a342b12),LL(0xc10a6db5,0x6b093bbd),LL(0x29832fb9,0x8f3fc90d),LL(0xff03b2ff,0xb3f2d8fc), + LL(0x64457331,0x1de7bd1c),LL(0x43bb1380,0x0a03a06b),LL(0x8bf17419,0x6720cc3d),LL(0x33938d5a,0x2627c7da), LL(0x8d98f34c,0x204da058),LL(0x51cbd284,0x80e29f46),LL(0xa46f93d5,0x11b22dd4),LL(0xe971a61a,0xd7341655), + LL(0xee69f782,0x36a748b7),LL(0x94f08ac0,0xa3740020),LL(0xc36099f3,0x383fb245),LL(0x00137fdc,0xa7cb0ef9), LL(0x6e1dd2e5,0x5371052f),LL(0x7976a1d3,0xed3ab7b5),LL(0x9df822e6,0xb0119c0d),LL(0x358685d1,0xafd2a477), + LL(0x4ae1103c,0x82879cb0),LL(0x94385be6,0x61cd6ca8),LL(0xd85d4a62,0x7c551809),LL(0xb318d965,0x9632ac5f), LL(0xe1340605,0x67baad2c),LL(0xac6ed4f7,0x39c2c8c7),LL(0x71211c2f,0x42c4a7b1),LL(0x9bf758f6,0x43c778bb), + LL(0xf519acb2,0x2dc8fc39),LL(0x08eff177,0xd3c30a6d),LL(0x5144484b,0xf889c021),LL(0xca376af3,0x01b82327), LL(0xd3e71253,0x168a0b2f),LL(0x3f9ff89d,0x5e92c6ba),LL(0x5b4c6844,0x8c74c132),LL(0x33de6858,0x707c4a40), + LL(0x9c36dd9e,0xb13f6abd),LL(0x9b3aa9f5,0x4baaef52),LL(0xcd156392,0x0a4fa929),LL(0x6506f42f,0xde0f1956), LL(0x150d4ee7,0xe43dd7f0),LL(0x7247f552,0xf7afe7db),LL(0x9072b450,0x9abc8d1c),LL(0xc9a8e159,0x5d0645d5), + LL(0x01c6f17a,0x863d3e8f),LL(0xdf080690,0x3a0575ac),LL(0x2b0fb150,0xcad62d87),LL(0x625c35c6,0xa1f54744), LL(0x41fe59ec,0x7d3bcec3),LL(0x169f1e04,0x0fd3e40e),LL(0x2ed9aa4b,0xbde8c827),LL(0x13046c6e,0x71562ee6), + LL(0xe9acac7a,0xaf049c5c),LL(0x261dd422,0x7efec06c),LL(0x095564c4,0xa530fbfd),LL(0x2a57af79,0x000c0c82), LL(0x2ce1315c,0x9f79868f),LL(0x1b5d575e,0x0dd99453),LL(0x1e534cfd,0xf1a49419),LL(0xed7e8b39,0xc7de8756), + LL(0x3ed2ccb2,0xef61f5c8),LL(0x34af2a15,0x032ee766),LL(0x9f69ae9d,0xe0692ed5),LL(0xf64900df,0xd34fc2d5), LL(0xaca6d51b,0x1c51c950),LL(0xa7717dfb,0x10ae0fb2),LL(0xa7ec7ca8,0x9fa305f7),LL(0xb5728214,0xb215a8ab), + LL(0x8819505b,0x62628fdf),LL(0x004ba54e,0x3cefd86c),LL(0xc571da3d,0xa17bed74),LL(0x93a5faa5,0x362dfef6), LL(0xf8aeea05,0x1bee6899),LL(0x16f18b7a,0xd7bf7e31),LL(0x1cb7685c,0x3f3cf39d),LL(0xe2e57c8e,0x1df41f23), + LL(0xe2fd94f1,0x8f62ecb8),LL(0x4c30a178,0x652099c9),LL(0x4262e9e6,0xaa2454e1),LL(0x2015d4a9,0x7f0d440f), LL(0xbb5b39fa,0xa2c76313),LL(0x1ab47bb3,0x46e57ab2),LL(0x8697e682,0xd181f444),LL(0x33273dfe,0x55db129e), + LL(0xe71d029f,0xda188361),LL(0xb5def631,0x3e3e19da),LL(0x087ad30b,0x7431f513),LL(0x9f27c84e,0x2537887e), LL(0xac9df89d,0x0c228c62),LL(0x10031289,0xdcd2c5e9),LL(0x0321d1b6,0x5cc76782),LL(0x6cb3d600,0x4e460bdf), + LL(0x9a870166,0x6f356aab),LL(0x497d4ac0,0x21aecb3b),LL(0xf0495ef1,0xd981a4b0),LL(0x0fb7704b,0x615e8bff), LL(0x8478bf12,0xc148e8ea),LL(0x364eee52,0x7011ec5b),LL(0xf692bc12,0xd9075965),LL(0xe622ad51,0x3019c824), + LL(0xec83c953,0x349e4873),LL(0x3a21ef0a,0xb4f59fb3),LL(0x40f7d93e,0x3872d314),LL(0xc2568c82,0x479e1d02), LL(0x65d43d22,0xd7e4dc9a),LL(0xe775efa8,0xcc068e81),LL(0x326defa6,0xb78ccae9),LL(0x2da64956,0x8f92b296), +}, +/* digit=18 base_pwr=2^126 */ +{ + LL(0xdea227ee,0xb721f8d5),LL(0x3dda8ba0,0xf48c766c),LL(0xe43e3520,0x0583d94b),LL(0xe1d898b6,0xebda36c9), LL(0x6627adaa,0x1808286a),LL(0x9938368e,0x19c4c620),LL(0xf750949f,0xe0dbd707),LL(0x0cf356d9,0xcadf4bea), + LL(0x2dc890a7,0xf5de2126),LL(0x95aa75a3,0x76b7b675),LL(0x2a070b32,0x475fc143),LL(0x8e31d68f,0x7429a646), LL(0x09be3dca,0xec3a9aaa),LL(0xaf780ed7,0x07e119a9),LL(0x64fd96c4,0x62125625),LL(0xe8e80577,0xb571494f), + LL(0x5228d735,0x955ee349),LL(0x8fc5d4b6,0xa04ef2bb),LL(0x3600814f,0x0c532891),LL(0x59f85bd4,0x41f1f637), LL(0xe3dcdfb4,0x72f1d731),LL(0x3aa5edb3,0x28a4ddb9),LL(0xf702dcdb,0x116a68e1),LL(0x3bde657e,0x1975bc42), + LL(0x8a914b50,0x7b9f561a),LL(0x9154d377,0x2bf7130e),LL(0x519b4c35,0x6800f696),LL(0x568b4c56,0xc9e65040), LL(0x6d98a331,0x30706e00),LL(0xe211ce1e,0x781a12f6),LL(0x40562e5f,0x1fff9e3d),LL(0x8c166747,0x6356cf46), + LL(0x429945a7,0x80e87329),LL(0xb7ab06ad,0xc619fe17),LL(0x6fd86b17,0x9116bc2e),LL(0xb9116aac,0x64a41877), LL(0x32ba4f3b,0xe3ed867e),LL(0x68b4ebe6,0x013e263b),LL(0xe779e4ec,0x305ebfe7),LL(0x50178251,0x5536d45d), + LL(0x8873a93d,0x5abb939f),LL(0x8c4c9cb1,0x0263ba48),LL(0x6b78a4b5,0x36764b8d),LL(0x28bebc1e,0x205bb45d), LL(0xae89dcd5,0x16df4bb0),LL(0x316fadb7,0x85994670),LL(0x3af3c724,0x71f75664),LL(0xe8520c9c,0x43e30313), + LL(0x29e91921,0x3ab9ec54),LL(0xe3299f47,0xd931436e),LL(0xb89cd49f,0xb56da7bf),LL(0xcff7f637,0x90623412), LL(0x714022de,0x751e7944),LL(0x2c338262,0x86bcc342),LL(0x314c23bb,0x85f6a9bc),LL(0x1f0a3991,0xedbe8e74), + LL(0x003b40dd,0x7a748d63),LL(0x3951b7ae,0x8a682402),LL(0x704a91b0,0x41e92dd9),LL(0x858cd3ea,0x2dfb3eb9), LL(0xf5094667,0xc3c2af35),LL(0x7435aa2d,0xffa287dc),LL(0x7462714f,0xd03f3979),LL(0x203e5b0e,0xdb550f67), + LL(0xe241ed0c,0x6df7801b),LL(0xb642fd3a,0xb52c0a3f),LL(0x1977a29d,0xdd35e1cf),LL(0xa661404c,0x8e793d60), LL(0x6b9442ae,0x393e2b87),LL(0x2aa6b707,0x123b893a),LL(0xdb8d306a,0xeec88682),LL(0xce847879,0x92c2d93d), + LL(0x80ec63b4,0x725f1e7d),LL(0x74113de0,0xcb8f53d9),LL(0xb819f730,0x2132a072),LL(0xb4c61f06,0xfabf3c47), LL(0x2cb243d8,0x79c1bc86),LL(0x757e3600,0x442833c5),LL(0x4e918b8a,0xfa4f69ad),LL(0x73bc193e,0x5816f3f3), + LL(0x30f40e93,0xc671c7a4),LL(0x5c51cfa4,0x6041aa03),LL(0x2fac25d7,0x3a713549),LL(0x24a7df01,0xf5053237), LL(0xd29f4ec5,0x99efb34a),LL(0x71d2cb1b,0x74810523),LL(0xf3a029ab,0xacefaf8f),LL(0x069d9545,0xc82e4f5a), + LL(0xd3341d80,0xd759549d),LL(0x31a2a0a4,0x079e9fa7),LL(0x2a164f75,0x75da56c7),LL(0xbeefc182,0x9313ef5a), LL(0xbde130ad,0x0aa365b6),LL(0x98411180,0x44265977),LL(0xaa26466a,0xa65373f7),LL(0x2e2cf404,0x1a43bee6), + LL(0xb37a9390,0xe029ed6d),LL(0x34970065,0x5c2351ca),LL(0x1c46d72c,0x7c4f3c30),LL(0x7262ce20,0x09ce770a), LL(0xdd58a9f8,0x0cfeefad),LL(0x408addaa,0x06797d79),LL(0x05aed325,0x76a87c06),LL(0x8a46d0c6,0xe002b672), + LL(0x05b6e1a4,0xcf77ea31),LL(0xa5d92b00,0x3bf900bc),LL(0xdccfe144,0x05996d8c),LL(0x951a602c,0x73d4dfd7), LL(0x0ed8885d,0x033f3959),LL(0x36400817,0x8332dc73),LL(0x2d8ebda7,0x96372295),LL(0xb5da0c67,0x3fb32cf6), + LL(0x3e36defc,0xcb521d65),LL(0xa67f00f0,0xc293d170),LL(0xfb35bd06,0x6a3a2fd4),LL(0x0bd490a5,0x537937dd), LL(0xc274ee5a,0x898d94bc),LL(0x7515b5e7,0xdc70f9bd),LL(0x3749900f,0xa94673db),LL(0x49ad3b04,0x3e6e2af0), + LL(0x207eecd9,0xb9dae1b8),LL(0xec07b47c,0xd3f50d63),LL(0x364292da,0x02b4d909),LL(0xfc35975b,0x919a6df3), LL(0xb616452e,0xb41ed4aa),LL(0x5cfc6abb,0xe58689cd),LL(0xf389b025,0xeac325d9),LL(0x8f255de5,0x45ceb1e6), + LL(0x5e46cdff,0xda4a0715),LL(0x0f6c761c,0x8a860a55),LL(0x5fe1eef1,0xe1395274),LL(0xf7bc535f,0x256e296a), LL(0x2755dd27,0xf3d4b06c),LL(0xbb530c26,0x3ced6ee5),LL(0x96ba599d,0x73249ad7),LL(0xe8a66027,0x5de8dab3), + LL(0xc2f97e01,0xa4892840),LL(0x427945be,0xbe0dbe49),LL(0xa57d4e4f,0x6fd86a7b),LL(0x04a2e778,0x7f56c3e0), LL(0xffc13d49,0x734708cc),LL(0x788d31fe,0x3c1d9413),LL(0x8d3e4c36,0xfe85545b),LL(0x8815129c,0xcca441fc), + LL(0x15e3d172,0x2e2095e2),LL(0x64b43e81,0xc0c8d3c4),LL(0xc68e802e,0x084557ab),LL(0x30d239b9,0xa6b73590), LL(0xb67b0548,0x61ec00a9),LL(0xb8ab138d,0x630059de),LL(0x36ca9888,0x800abf01),LL(0x9517149e,0xe26d644a), + LL(0x58bf21d9,0x775d5a98),LL(0xdbeab706,0x00eb6846),LL(0x8232d477,0x9d714c9f),LL(0xb70f91c2,0x7cde2c3e), LL(0xe9871f0c,0xe6d0a8ce),LL(0x19e8444a,0x902bc60b),LL(0xff0cd43a,0x8651ed57),LL(0xd480d222,0x4418cc07), + LL(0xf3cbe01d,0xb5e0c7e3),LL(0xe43adcdf,0xbf4a899f),LL(0x78f8f79d,0xb89b022c),LL(0xf42c797c,0x79cbbf97), LL(0x59d53cc1,0x46d73cc5),LL(0x4ffca67c,0x99f683e6),LL(0x98865e5b,0x527c16ec),LL(0xf68f8ee0,0xc844b70f), + LL(0xc9854994,0xcffcccc0),LL(0x74926d5d,0x4aafcc15),LL(0x835aea59,0xeb084832),LL(0x20df21cf,0xcb317b5f), LL(0xe43d1853,0x3c45b084),LL(0xb93b9167,0xd12c9411),LL(0x19316bdf,0xb0901982),LL(0xd11ab5e2,0x76bfa2ac), + LL(0x4e84d3e9,0x22bf23cb),LL(0xd1572d4a,0x96ec9f8e),LL(0x080ba39a,0x31594ae4),LL(0xadc6bae4,0x105b5677), LL(0xa644e230,0x501e45dd),LL(0x64573126,0xeb571f27),LL(0xa36ac1ef,0x1fc3d478),LL(0x327c7da7,0xbd5fcee8), + LL(0x34a70bfe,0x1b2b1885),LL(0xa36345c5,0xcfa421f7),LL(0x6f322ae9,0x2f33f4cc),LL(0x4dabb7a0,0xdac0bb75), LL(0x923cea0a,0xfba35536),LL(0x6d9cb50c,0xc16f73e5),LL(0x25812c96,0x23216dc6),LL(0x3d7ab424,0x82945e67), + LL(0x0796605d,0x829577b2),LL(0x5026907f,0x47fa0978),LL(0x2d0f68b2,0x99701169),LL(0xbc1e46db,0xa0d24be4), LL(0x2eb2ac98,0xcf409c2e),LL(0x97f3ff5c,0x7b59c3c5),LL(0x81ed7f02,0x2f4576bd),LL(0x10399c22,0xe41339e5), + LL(0x2ecce0e6,0x562d7744),LL(0x9a1656c2,0x1afc3869),LL(0x86200621,0x5714820e),LL(0x566da805,0xee36f7b6), LL(0x6e5a2a06,0xe6694104),LL(0x8caabaab,0xd4390b74),LL(0x93b0d142,0x9db20998),LL(0x7926baf3,0xe1811b81), + LL(0x08bc1965,0xd578f2ed),LL(0x35f00d5d,0x9a7e31e2),LL(0xc9007327,0x3725b65c),LL(0x29c36f38,0x157cfe95), LL(0x23a521d7,0xb1c3d0f1),LL(0xb8a9ae08,0x3e65fb7c),LL(0x690b8f78,0xed48bcf9),LL(0x90d5dfde,0xe5f46b2c), + LL(0x0b6da2b6,0x14aebb35),LL(0x7b65ee55,0x91fef336),LL(0x1a0a004d,0xdb77b57b),LL(0x23aef1f7,0x1c59b628), LL(0x3ec88d18,0xa79c8c89),LL(0x4fde31f1,0x52cca38a),LL(0xcf4e30b6,0xe2f64a94),LL(0x37ff1cbb,0x2b4cdbd7), + LL(0x0b566632,0xcb542f68),LL(0x676fae9f,0xedab69a6),LL(0xc45cb6f0,0xc4531e0b),LL(0xb88fe4a5,0xf967ec6e), LL(0x2919785d,0x4ab4e645),LL(0x7a17b20f,0x2dcaefca),LL(0xda7afaa0,0x65c89c05),LL(0x4dafc6a2,0x59ea00e9), + LL(0x8eb43733,0xa6362bf8),LL(0x12011803,0xae2dddc1),LL(0x0bb2aaf8,0xbbf516b1),LL(0xd8de21a3,0x9f2627e9), LL(0x43a20b74,0xaf30439a),LL(0x4ce86408,0xac7e07b0),LL(0x7c10566b,0xc54cdff2),LL(0x6279af0a,0xe3ee0622), + LL(0xf7770f95,0x57d09708),LL(0x123e020b,0x6f0ba223),LL(0x6cd41559,0x6c123fb9),LL(0x6fb30f58,0xc54f5c65), LL(0xbbf7101c,0x5e168af3),LL(0xce974455,0xf6d6dbdb),LL(0x88313516,0xa001f3b9),LL(0xdfb4ac20,0xe6e4a26d), + LL(0x506f7dcd,0x74e7b7fc),LL(0x5d037d69,0x985e8546),LL(0x1ec8d374,0xff00a4da),LL(0x505b4180,0x8c339ae3), LL(0x3a5f71c4,0x78bcd4f2),LL(0x67ac3e9f,0x2fb4d99f),LL(0xee65dad1,0x7dd25aa6),LL(0xb62c34dc,0x2fd63fc2), + LL(0xf7700192,0xdee42663),LL(0x2c3248e9,0x9925a206),LL(0x2ea9f052,0x4a55a55d),LL(0x16ac67fe,0xe1d6efcd), LL(0x9bb02415,0x7f82246d),LL(0x72cd7a6c,0x2fadbb9b),LL(0x712004dc,0xe977a037),LL(0xb3c9f4b9,0xe8c449b2), + LL(0x861ea138,0xa2cb838a),LL(0x356ae003,0xfcbe219a),LL(0x1838504f,0x15c02496),LL(0x0769d5dc,0x58cef52c), LL(0xb3fef999,0x7e94ff7d),LL(0x04e4fc87,0xf55501e0),LL(0xc05430dc,0xcdb5fd36),LL(0x778c5cd4,0x49872453), + LL(0x1b5e7ace,0x4c4855ff),LL(0xb159fe74,0x89fc6309),LL(0x3c9ebbe2,0xaca00404),LL(0x866bf867,0x4c030591), LL(0x9b18a535,0xa7e8f599),LL(0x5c0a0a44,0x9203ebfc),LL(0x463207c9,0xbf1b30cc),LL(0x9d135aeb,0x90b59001), + LL(0x794cb3ab,0xedc44d04),LL(0x0ad7be70,0xb3baa475),LL(0x6c09fc91,0xb7d8c7c5),LL(0xf45a5bd6,0x2a362d71), LL(0x8cf3e5a6,0x36e308c3),LL(0x0a649c31,0x4caf2cd1),LL(0xb3c501c7,0xbae328f5),LL(0x83a0eeb3,0x2efeca03), + LL(0x7086093a,0xc3a27585),LL(0x6d686d83,0x78e86515),LL(0xedf0def6,0x18cf3ac1),LL(0x5a1d6cf4,0x2f6a56da), LL(0x30084873,0x350c822e),LL(0x65843610,0x82d48087),LL(0xf393ecd1,0xa4e752c1),LL(0xeeb74f25,0xe3034d6d), + LL(0xb8b0c5c7,0x1793727c),LL(0x7ec9ce37,0xde561ca6),LL(0x6190f612,0xd9eddc50),LL(0xca89a191,0xb52dc77c), LL(0x4bf1e87a,0x990010b2),LL(0x15b91691,0x073136b2),LL(0x15546011,0x50111261),LL(0x0196cb8d,0x17d48864), + LL(0xfd61d824,0x7ec44104),LL(0xf088d3db,0x213550ef),LL(0xacbbb608,0x5e8d321f),LL(0x39312b64,0xc317c1f8), LL(0x27de4329,0x7a4a1cd0),LL(0xf9b135e4,0xbfb33f07),LL(0x59b94480,0xcf82b639),LL(0x70b118e6,0xca62d957), + LL(0x2b1d45fb,0x95b2ff03),LL(0x2570686c,0x472dd56c),LL(0xd3d50e32,0x4fbae8a0),LL(0x65796a08,0xa31c65dd), LL(0x037ce5bb,0xe261f6f8),LL(0xd042073b,0x3b7816bf),LL(0xbfba45f8,0x6d0ebbee),LL(0xc9522e36,0xf2d99979), + LL(0x77cb5b0c,0x707f2a18),LL(0xdfc02b82,0x954b5a91),LL(0xc20ae04b,0x246b9a55),LL(0x9dd79f93,0xa1486775), LL(0xc11f6d8e,0xd4092830),LL(0x267a4dab,0x74ca482f),LL(0x9c58706f,0xe3c80bb6),LL(0x099154c1,0x245f04b7), + LL(0xf149259f,0x3a4b25b5),LL(0x65ccbe91,0xeac735f8),LL(0x572788a4,0x260e749f),LL(0xe34d40cb,0x30b9c736), LL(0xf524a17f,0x65981d50),LL(0xcddbbefc,0x6c462f5d),LL(0xa1e57312,0x245bfa18),LL(0x46dc8ae0,0x3b4b003c), + LL(0x5d262a35,0xb1958797),LL(0xffafd8c5,0x83f6e604),LL(0xbc2e0801,0x60843f9c),LL(0xc783ad3d,0x11d85ac1), LL(0x2e016e43,0x1ce477dd),LL(0xfb4a0201,0x2b628f06),LL(0xbf4f77d7,0x897b7f62),LL(0x10277d8a,0x52e04f22), + LL(0x5f3f0d6a,0x17132351),LL(0x59a96c4d,0x13c9e064),LL(0x86f05ae8,0xc73892b0),LL(0x4212ad65,0x94545c8a), LL(0x3dc4984c,0x0591b091),LL(0xf2ec1ca9,0x06845698),LL(0xb3ac894b,0xb0e1e1d0),LL(0xa7c915cd,0x962ca1da), + LL(0x95331bd5,0xb0640de8),LL(0x478c1b6d,0x2544348a),LL(0x5647a67e,0x3c3bd415),LL(0x5b20e5fb,0xd7970ef8), LL(0xe06b4fa6,0xd6e6f6be),LL(0x871390ff,0x5ae29e5e),LL(0x7256daa1,0xc7924188),LL(0x59f61750,0xfae5e501), + LL(0xd1ef1d2b,0xfac83ece),LL(0x554736da,0xa567060c),LL(0x1dba8bc7,0x697571f4),LL(0x553fbcfc,0xd3fc5aeb), LL(0x9755fab0,0xe665970a),LL(0xb5537da8,0x30fbe8d9),LL(0x97c2b5f0,0x7a7d0013),LL(0x1b700a02,0x9fea5c9c), + LL(0xe9a377da,0xcfc0166e),LL(0xac502375,0xcc78f3d8),LL(0xba64c3b7,0x803fbbda),LL(0x4d70cc42,0xe53c7d6b), LL(0x5189b7da,0x6b927bba),LL(0x8b05322a,0x2c86253b),LL(0xf3869873,0x333e7491),LL(0x4b492879,0x9308348a), + LL(0xb9ab0a36,0x39bfa2a8),LL(0x18f71ac7,0x560f80a6),LL(0x45e24412,0xca9b265a),LL(0x8e2ddac3,0x6796bece), LL(0x17bfcabb,0x87f1eee5),LL(0x195c9bb2,0x624db4d9),LL(0x2b4db6d2,0xf7110fcf),LL(0xb432d19d,0x41d3fb0d), + LL(0x73554a3c,0x3344ea7d),LL(0x830a3881,0x4c968dad),LL(0x687f71ec,0x5df71ad2),LL(0x259cbc07,0x4c4df41f), LL(0xeb541d72,0x8d12d2e0),LL(0xa20fb162,0x94c0dab6),LL(0x1eda0516,0x9bbc2524),LL(0xdd7871ff,0x696c924e), + LL(0x1db84dc1,0x97efb495),LL(0x03cbfbf8,0x7d293ce5),LL(0xbc48d007,0x79e25d3e),LL(0x8591a1ea,0xc900a580), LL(0xd37508c3,0xf0277a09),LL(0xe84557bf,0xbf583aa4),LL(0xd8228591,0x2e258d60),LL(0x117da3a9,0xb037e17c), + LL(0x243d588d,0x4b35355e),LL(0xcce2539e,0xbe6dfa36),LL(0x4843c9da,0xa57d5823),LL(0xf59348fa,0xe3d91511), LL(0x2791c08f,0xb5d1395c),LL(0xf6fdcc93,0x04129e5d),LL(0x0f53087b,0x635a63ba),LL(0xf237612e,0x66da6bec), + LL(0x22755420,0xc3d052e5),LL(0xd7a1bd35,0xc37a9b47),LL(0x9b347e02,0xf19613f3),LL(0xbbda7ae0,0xee84dbac), LL(0x3a85f2e5,0x603be21d),LL(0xff679451,0x5f0927c2),LL(0x8674f8d7,0x799013ad),LL(0x00f465e5,0x17b248d3), + LL(0x96ca19de,0x2a29135f),LL(0x957d1844,0xc8e56e32),LL(0xa11a4349,0x935e7eaf),LL(0x741b73d3,0x717308e1), LL(0x7233a9df,0x40477acb),LL(0xd2c83b72,0x7a78dac2),LL(0x2c5d79d2,0xfb882461),LL(0x76f44fa0,0x984505fb), + LL(0xdfdc4a9d,0x5cdded16),LL(0x3f0ff597,0x4cbea135),LL(0x8a79078e,0x38daf27a),LL(0xce1bbf0e,0xb4b0085d), LL(0x86f19fd0,0xb6b0d8d7),LL(0x1778ca6a,0xe0fdcdae),LL(0x0b26b9b5,0x257c7df9),LL(0x141dcafc,0x4b82422c), + LL(0x4d3cf148,0xcf8a2dad),LL(0x5f17e914,0xf1a4e292),LL(0x60de8f64,0xc40755bb),LL(0x8718f49d,0x412449f8), LL(0x8737b6cb,0xdabb9968),LL(0x6236ea05,0xdd94ae81),LL(0x05c5aca2,0xb5223cd0),LL(0x762210ed,0x6b81bd33), + LL(0x5d4164df,0x1f0921db),LL(0x8d4a35df,0xf6fdb08f),LL(0xc602d4d8,0x1efcf3c7),LL(0x057f3aa0,0xa2ecd9e6), LL(0xeb4fcba2,0x13a6c576),LL(0x13130559,0x16425bd4),LL(0x416b4968,0xa9eac848),LL(0x2119600e,0x617c32a9), + LL(0x0bb49e40,0x1a84eca5),LL(0xbc2310b3,0x2ed98d25),LL(0x5edbc719,0xad191f88),LL(0x0376ae08,0xd8d667d5), LL(0xf0b4fe29,0xb855a8ee),LL(0xe75354f7,0xc3fe79fb),LL(0x403b651e,0x1ee9b9e6),LL(0x2baa2c6e,0x99ddbb3c), + LL(0xeccce37d,0xc6a84c47),LL(0x038c9821,0x71a05a24),LL(0x9a6353d8,0x8d32194c),LL(0xcf0a1462,0x14cd3ea6), LL(0x7bdbe521,0x40d70aa2),LL(0x95c80cd8,0x200f0b21),LL(0x3efdf656,0x4c79dab9),LL(0xa981d8b5,0xafa44e4c), + LL(0xa7111315,0x811b9387),LL(0x7590c35d,0x0255a234),LL(0xf1af875c,0xb18e87c0),LL(0xced5cc1f,0x0a930b41), LL(0x96094a55,0x6ff4fca4),LL(0x6a9dc519,0x74095b88),LL(0xafa4894a,0x44492273),LL(0xa2e6f56e,0x54f16f88), + LL(0x34485e31,0xd613fbb4),LL(0xd2464242,0xc716c370),LL(0x1644f2e1,0x21535837),LL(0xbe417c3a,0x7719474b), LL(0x2045d2be,0x31bfb158),LL(0xf50e6828,0x10855524),LL(0x98a67af1,0xdb9490ad),LL(0x1c281ff3,0x41a34aa6), + LL(0xa8bf2580,0x87109ba8),LL(0x2d7eb52d,0x70c2e936),LL(0xfb3fc109,0xefe9fe2c),LL(0x780526bf,0xfd3f4d7b), LL(0x9ed0c3bc,0x6f9a48d8),LL(0x5d8205b2,0x0aec850f),LL(0x1c6a13ef,0xa378f8c6),LL(0x9d10e11b,0xac02f367), + LL(0x3b9bbf54,0x79c6b396),LL(0x42779c58,0xfb586d71),LL(0x889eecb3,0x5d975728),LL(0x434537d8,0xda2ec867), LL(0x62f31813,0x15a3c9c3),LL(0x3c30433e,0xc4b357c8),LL(0xc464e972,0xf26d281f),LL(0x4512ffcf,0x99fa49e7), + LL(0x725b9753,0x456db1b2),LL(0xb42941c5,0xec501760),LL(0x7d6d406f,0xd822a9d5),LL(0x7bbcd4d6,0x4bb7a820), LL(0xcc96a5b7,0x079b1fe0),LL(0x24aa4901,0xf83e5755),LL(0x20da7fcb,0x317cdd1d),LL(0x93b04a81,0x487fd706), + LL(0xe43332ef,0x43e0671f),LL(0x441c2218,0x71c5dd5b),LL(0xe922ba18,0x4c1d2c1f),LL(0xd619cb67,0x558e9c2f), LL(0x1ec51255,0xd04acde0),LL(0xaf824507,0x824b3740),LL(0x744c6afe,0x62d1b9de),LL(0xab0d52e3,0xb99616db), +}, +/* digit=19 base_pwr=2^133 */ +{ + LL(0x7f6a1cda,0x5ec9c084),LL(0x823d6350,0x68839c14),LL(0x03bad007,0xcbbb678b),LL(0x4788854e,0x6a727255), LL(0xef5c7294,0xc747fea2),LL(0x4875e775,0x74852778),LL(0xaa61a893,0xad7b8e8b),LL(0x40da98b1,0x18ff3335), + LL(0x5529ec80,0xa51e9f4f),LL(0x6fd146d1,0x0420274a),LL(0x8e300c2c,0xbbf1ab66),LL(0x41653fea,0x2d0b3a9d), LL(0x23a495b9,0x2be2180f),LL(0x5415d73b,0x6ef3c374),LL(0xc67ae4fc,0x1d3e1ec8),LL(0x98d31f5f,0xa5839e9c), + LL(0x37d77c01,0xf54114d6),LL(0x41023c87,0xc2e18a4b),LL(0x9e6e1221,0x6fa6c3d3),LL(0x410e48f9,0x9a6cf4e2), LL(0xb181828f,0xe0881140),LL(0x78cb7833,0x17c6df29),LL(0xa7cd2367,0xc1eb8df1),LL(0xca89f922,0xb78f1c8d), + LL(0xd0d42887,0xf25d4777),LL(0x2b7a2707,0x4b489218),LL(0x2d3966fe,0x1b4dbf9b),LL(0x41ae2bec,0x4bac5f48), LL(0x1733964e,0x68db2733),LL(0x6a814a69,0xa10c5dff),LL(0xa9898348,0x84ebdaf0),LL(0xa74da3f4,0x60e46823), + LL(0x93420649,0x452b6b1d),LL(0x6ed5d7f6,0x9dd6452b),LL(0xe687b577,0x4a9b8fa1),LL(0x854c49d7,0x1e203166), LL(0xa45feba8,0xf523667e),LL(0x5f9f4a56,0x9ecb4d44),LL(0x7fb1c641,0xb8655a5f),LL(0x87c26201,0x5516401a), + LL(0x0d2face6,0x24677754),LL(0xa8ade59c,0xd9f7da7f),LL(0x7fa7df06,0x27e3ad77),LL(0xf60395ad,0x35a4caf0), LL(0xe4e701ac,0xfaef231c),LL(0x23755489,0x9e135976),LL(0x43554ad3,0x7caa73ab),LL(0x94f0d878,0x9d8554d9), + LL(0xa85b81d5,0xe42040ce),LL(0x40fa9631,0x4d28aca7),LL(0x7e04b755,0x076fed3d),LL(0x1129ce4c,0xdde3d347), LL(0x1179af95,0x77f785d7),LL(0xf74e0672,0x4782f842),LL(0x0b4597cb,0xbd068cc1),LL(0x8f4c65b7,0x3d6d4b2a), + LL(0xf9066d73,0xe0642d18),LL(0xa098b3bf,0xbe1d2ec3),LL(0x21b4954c,0xefee860c),LL(0x27b629bb,0x4d7c4e6d), LL(0x8e8b81b0,0xcd8f1e03),LL(0x7fe77eb0,0x4a80168e),LL(0xce247c73,0x4d977591),LL(0x857e0356,0x9b30c9f2), + LL(0x2940e9de,0xc02495ba),LL(0xb6d2b72c,0x357299f5),LL(0x06a9c2e4,0x132b4c63),LL(0x084d8c67,0xe90a90c5), LL(0xace1b471,0x0f0c9e94),LL(0xf1e3d8f6,0x769457e1),LL(0xd71118c6,0xc4c30ce3),LL(0x6b652a3d,0xdb5fd8d6), + LL(0x4def5978,0x090df107),LL(0x2d8a5f3a,0x1abcfa32),LL(0xa34b70db,0x2976b012),LL(0xfa5e75b9,0x90f541d4), LL(0x37a6e9a0,0x50c991a9),LL(0x903bffda,0xf51e8693),LL(0x8d344776,0xa2697ab4),LL(0xe34a7850,0x77134fe8), + LL(0xa72597ac,0x723e5d3d),LL(0x4269aff7,0x4a49847a),LL(0x443b8db6,0x75ad9088),LL(0xa51d80a1,0x9b7d00d5), LL(0xe5e04ac2,0xce1c7049),LL(0x2a792bde,0xb8c2793c),LL(0xe410e175,0xde9220a0),LL(0x9401bc2a,0x4b3a9b85), + LL(0xf037d15f,0xc7eaf2c5),LL(0xc7afbf8b,0x410b627e),LL(0xd7bedf50,0x243cdb79),LL(0xbe6512d0,0x04813b51), LL(0x26beca2f,0x2fb77cab),LL(0x7baa3099,0xbb601975),LL(0x40bda4d0,0x8c327e59),LL(0x13c23444,0x85b9c764), + LL(0x08ed59d8,0x26960d9c),LL(0x4a72854d,0x9b76dced),LL(0xfdc3b7f5,0xca2f579a),LL(0x6cae8b4f,0xac27028a), LL(0x42326aa5,0x48fd1a49),LL(0x5759c63f,0xb95fdb4f),LL(0xe0a96abf,0x27655358),LL(0x36ed53b0,0x26d38b64), + LL(0xfc6d1f3e,0x03cfdd49),LL(0x15adaba0,0x20af5886),LL(0x754dd268,0x74c6c943),LL(0x7977717e,0xe7d52cdf), LL(0x3b414dd2,0x9a81d440),LL(0xd790a4c7,0x697c7b4a),LL(0xedbce1f2,0xb1b7735f),LL(0xbefa7210,0xbd90e63f), + LL(0x7ab207d1,0x2e2b0dad),LL(0x9b373211,0x89abbd83),LL(0x8e35e2bb,0x45d34ebc),LL(0x064856f6,0x67ba3ac5), LL(0xa52c7676,0xb5527dbe),LL(0x71294012,0x906fb217),LL(0xab305260,0x65fca552),LL(0x14ee193b,0x89ac52a3), + LL(0x88c06b1c,0x673aead4),LL(0x49d9d4e8,0xea8af420),LL(0xcb9e86bf,0xa7b4409a),LL(0x5414aa56,0x49f76f71), LL(0x8c13857a,0x6603c801),LL(0xce742384,0x7c26f1c2),LL(0x2a665719,0x042fb224),LL(0xe175b0c6,0x2619f254), + LL(0x7c092397,0x5b3b71ea),LL(0xf18c29ae,0xd9087023),LL(0x2008841d,0x48dbecbd),LL(0x22622bba,0x658c998e), LL(0x578e463f,0x38a2cc6d),LL(0xcbb88796,0x7002380f),LL(0x71695101,0xc545baff),LL(0xce65b49c,0x064a0500), + LL(0xb1ae0398,0x3651d926),LL(0x4ace0e86,0x33c9ea8f),LL(0x1a6debd7,0x481fab1b),LL(0x4d365031,0x65b58a79), LL(0x811e8f01,0xb73ec84b),LL(0x51342ef2,0xb6aa3955),LL(0x9efcdbcc,0xdbce3d9f),LL(0xcfbf2a4f,0x5791b35f), + LL(0x6eaad1f0,0x67024158),LL(0x0063ae25,0xe8dbaa88),LL(0x9fedc144,0x6d2051cc),LL(0x18b5e86d,0x136c2ab1), LL(0xc89241d4,0x3b2d3d63),LL(0x4a82dec6,0x843cfa3d),LL(0xf0a5f163,0x64fa5860),LL(0x1ae3be83,0x2d9b6095), + LL(0xb01a91e5,0x75f97753),LL(0xcd0d8cac,0xd374dfa2),LL(0x8eb72ba0,0xe5dbffef),LL(0xd7b8a624,0x61049807), LL(0xa39277d3,0x9c8b5e93),LL(0x3b1cc635,0x6e5ba593),LL(0x21cde059,0x8bd0a69e),LL(0x071ec0c8,0xd0a19b53), + LL(0xd1bb088d,0x8c87785a),LL(0x7e249c39,0xd801d5a6),LL(0x8688914f,0x002ee598),LL(0x6b68413d,0x52b014fc), LL(0x507946df,0xaf1d7e88),LL(0x84ccebf1,0xa38e436f),LL(0xaa86a4b6,0x37d9b946),LL(0xc506a394,0x55da0db6), + LL(0x02b900bd,0x856928c3),LL(0x7bc6a67b,0x9eb926a3),LL(0xd0f39446,0x2f4d392d),LL(0x01c49daa,0xb12f2761), LL(0x13874ac7,0x07b8d23f),LL(0x1efaa157,0xa473ef4c),LL(0xdf8cf2ab,0x550765f6),LL(0xd23d3750,0xeba88504), + LL(0x2434fa2e,0xf05791d4),LL(0x4e2a05ea,0x8c0899c3),LL(0x898bc9b0,0x40a53bdd),LL(0x40c8bf7c,0x6c255f6f), LL(0xe164b910,0x203db8c5),LL(0xc1c4de69,0x070baaee),LL(0x5df5c0a7,0x89660629),LL(0xdb364b99,0x0b9c2f4b), + LL(0x44bb5a79,0x012c6994),LL(0x9bd1fdc0,0xf5928e0c),LL(0x3ce49191,0xd30b8a97),LL(0xe3a05dd3,0x52792b85), LL(0x1d3d69c3,0x0da08916),LL(0xed59a28d,0x931759e8),LL(0x6ca05485,0x412148d9),LL(0x3d6e9964,0xb1517aa0), + LL(0xde75812d,0x15204ba9),LL(0x5698b03f,0x49e377e0),LL(0x05c9072e,0xe7790d41),LL(0xdba28e80,0xf79adbed), LL(0x4644840d,0x6aad9f96),LL(0x2e0a695b,0xc3f3d032),LL(0xaa4aa737,0x3eb739d2),LL(0x37d8d520,0x45c6b665), + LL(0x9917cb85,0xc3ba2408),LL(0xd7bf6304,0x1c729ffb),LL(0xcc160245,0x56b9935e),LL(0xe03cb227,0x42379567), LL(0xb66bfc5d,0x2dc20028),LL(0x95de8ed3,0xfaf7d224),LL(0x3214024e,0xa7541158),LL(0x50aabdb6,0x2f7755d8), + LL(0x7ea9b93a,0xb74ac27b),LL(0xa2e0516c,0xc1c5a8fe),LL(0x6b64f56f,0xe9f4f222),LL(0x8fbc4a64,0xf3c0c7fb), LL(0xa16edc23,0x43ac0ac2),LL(0x6d086e9e,0x0e26e4ad),LL(0x5bc0961f,0x5b8ef949),LL(0xd2b77c29,0xa0d16d39), + LL(0x78845d09,0x50b43efa),LL(0xcb3acdd9,0x3899e1be),LL(0x18d4ec31,0xa93a28e3),LL(0x0a66fe47,0x18a4eeed), LL(0x87333831,0xd7a7bf46),LL(0xdbe14699,0xbbf5c1a8),LL(0x80b9c9d0,0xf2a3da73),LL(0x82bceb4e,0x133c138a), + LL(0x335a923a,0xcfd4b885),LL(0x8fc82f3b,0xf9b69b3f),LL(0x8784c35c,0x08908b60),LL(0xd843b66e,0x76bf1082), LL(0xbb57a641,0x1ba730bf),LL(0x34e9f095,0x3bb4a8d7),LL(0xc28d5414,0x0342d32b),LL(0xcfd99e1a,0x8fb13cbf), + LL(0x1d02f47c,0x3845e507),LL(0x14ef0b26,0x4d77af89),LL(0x5ef578d9,0x93454480),LL(0xbdc408ec,0x23138c57), LL(0x47cf528a,0xdac833ed),LL(0x29d7cf20,0xd18e9865),LL(0xcdc8e55a,0x93208743),LL(0x724025a0,0xbfe570c8), + LL(0x3aee838e,0xb75c3de0),LL(0xe0f21f23,0x29304886),LL(0x82791daf,0xe46792ab),LL(0x3f124394,0x3d798d92), LL(0x29a6fb5e,0x2446dc81),LL(0xbd68c23a,0x2446e5b3),LL(0x689b1176,0xe1b5c76d),LL(0x9a852082,0x3fb66661), + LL(0xd9b45206,0x8d6fbcc7),LL(0xeabc4640,0x00ab735d),LL(0x810e37d1,0x428c7017),LL(0x27af5718,0xa4365872), LL(0x0a910146,0x8f195823),LL(0x0ff76704,0xc13ccdd7),LL(0x44d6f1c8,0x59d34ad6),LL(0x795b61b4,0xd3dfa6b2), + LL(0x12eea439,0x1ec08010),LL(0x7b2cd52a,0xafbbea32),LL(0x68cfe98b,0x99428f9a),LL(0x95628fe7,0x4ff9a5bc), LL(0x7ac41e9a,0x212baeb7),LL(0x29206e86,0x595cf03f),LL(0x733f37c4,0x4b62a429),LL(0x4d3cb6a6,0xa1fac4ae), + LL(0x1aed3c45,0x2d6cb0e6),LL(0x4e6da48d,0xf6703493),LL(0x2d13f9c1,0xa0036fb4),LL(0x7fe3ea2e,0x7db5078a), LL(0xd5992163,0x152a1fc0),LL(0x744b44ff,0xd63270e9),LL(0xf177c075,0x56730292),LL(0x17c3e08c,0x470f5e72), + LL(0xecb927f5,0xbf53d223),LL(0x629e8aa1,0xc80fbc1b),LL(0x24d72477,0xed59f186),LL(0x38811583,0xc266f5a6), LL(0x7c404560,0xc6f37bc1),LL(0x0c5b68e9,0xd58c10e5),LL(0x916e8f3c,0x696de793),LL(0x56a7781f,0x7298af8e), + LL(0xb16679d5,0xaf063553),LL(0x4316ed7e,0xa509f449),LL(0xb53cc0e2,0xe3d6ec43),LL(0x16ba34cd,0x9e957ce0), LL(0x7857d80d,0x2b0c7fbc),LL(0x3daffbf3,0xc2c671fe),LL(0x0d251d41,0xebcbf012),LL(0xffef45f5,0xedcfe7f7), + LL(0x334a1734,0xf5b66555),LL(0xe505f4bb,0x4354ccfa),LL(0x52a59260,0x6ee0b5b9),LL(0x5a699a93,0xb7bb64c1), LL(0x6de84422,0x85e34c0e),LL(0x8bbe0560,0xca9bacfe),LL(0x952a52d2,0xa08c780f),LL(0x3919176b,0x0e794b05), + LL(0x154d282d,0x8a496598),LL(0xdc34508c,0xb2999dc4),LL(0x9db4410a,0xfc304fe3),LL(0xe1bc07c8,0xbc09aee4), LL(0xef6d497d,0x1d2f0147),LL(0x96488fc1,0x3b9e06e0),LL(0x34cb97a7,0x37635d04),LL(0x8757f955,0x9a294b89), + LL(0x59508819,0x38c568ac),LL(0x46e15b82,0x854370fc),LL(0xee57f0b4,0x9f676404),LL(0x8f45319c,0x268854cc), LL(0x63746274,0x4256d25c),LL(0x0496cf9c,0x0a553821),LL(0x15e2fc17,0xb6bf27de),LL(0x99bd538a,0x6848f83a), + LL(0x1685e460,0x00e15d0a),LL(0x155d00b6,0x6fae8b37),LL(0xdc561456,0x277126d8),LL(0x6bf70c63,0x331c02e5), LL(0x515f39b7,0xc9b7da4e),LL(0x966c2060,0xb7e0d135),LL(0xc401f926,0x9a801457),LL(0xffb0137e,0xcc560825), + LL(0x5c7e38fc,0xbcfac5f8),LL(0x174e97ba,0xd542c1a4),LL(0x0bb507b8,0xbea67b1e),LL(0x3b782fd8,0xf008cc2c), LL(0x0aa329bc,0x865834da),LL(0x2b6db70a,0x0fd746f2),LL(0x65fbe439,0x8e72e5f7),LL(0x005295ee,0xac23881d), + LL(0xad9d013c,0xc2c45fef),LL(0x71c311f9,0x0df74277),LL(0x6bb32b66,0x69caf967),LL(0xb8e4a3e5,0x9fbd32ff), LL(0x78c0c439,0x39d94e31),LL(0xffa4b625,0x7489a8f0),LL(0x8aac717c,0x59af0ec3),LL(0xa12d996f,0xdd3b470e), + LL(0x8da3fef0,0x6d60cb97),LL(0x044d64fc,0x5164d722),LL(0xfc21305b,0xefe06ead),LL(0xceed89c1,0x72b4c45e), LL(0x8cabf0df,0x072cf1dc),LL(0xa5371d3e,0x0a0d7c0c),LL(0x2ae831d5,0xb13ba707),LL(0x269f189e,0x7702c3c5), + LL(0xc8239fe7,0xfb8e903e),LL(0x524f213c,0x5805c2ef),LL(0x70645f7f,0xdf056e45),LL(0x454c4577,0xfe10ecfb), LL(0x990dc567,0x422126da),LL(0xbf55cd81,0x95a5d753),LL(0x8c2688ed,0x2705a00c),LL(0x2f8f91af,0xd079ecb4), + LL(0x2b69a2c8,0x8cd13fa0),LL(0x36b657b8,0x7b0f310a),LL(0x251c595b,0xa7247cfd),LL(0x5a36e4b1,0xda352dc8), LL(0xf43312de,0x588d2e88),LL(0xdb9f6dec,0xef80a48f),LL(0x3fb2d6e3,0x39583634),LL(0x5a46bc46,0x0fbfa769), + LL(0xfe701598,0x3570a3f2),LL(0xac815fbb,0xd1d0d091),LL(0xd7f2b1b2,0x4d7bfadd),LL(0x66496326,0x509298d4), LL(0xcad9fb50,0xb7accafc),LL(0x9c592dee,0xcdbcb762),LL(0x6888482a,0xfe47a3b1),LL(0xe8b8c133,0x312be210), + LL(0x00167f93,0xc474b07f),LL(0xa637f35e,0x19457000),LL(0x5005d8a1,0x3eafa14e),LL(0xadf25f29,0x2a84723a), LL(0xa741cf9e,0x2c9d7ebb),LL(0xc3913acf,0x94024dc2),LL(0x97b98f1f,0xac2db91d),LL(0x46a7bf92,0xfb9a0502), + LL(0x6487a5d4,0x8874ffb5),LL(0x2f53e25f,0xc02a12b5),LL(0x416ba8fc,0x38654a57),LL(0x0c0b25d6,0x226356f2), LL(0x6030f2ac,0x34f2eaa6),LL(0x9cea9176,0xb788baa1),LL(0x4e912104,0x66fbe9f7),LL(0x39a69e3d,0x982ef71d), + LL(0xbbe5733a,0x9f361d17),LL(0x1988f31e,0xc79569a0),LL(0x9e0f52fe,0xf2b96ecb),LL(0x80235136,0xc78e44dc), LL(0x8462ef4f,0x96053ab5),LL(0x81506701,0xf83c1f6d),LL(0xa65c09e9,0xc7313eb1),LL(0x4efcf558,0xf5dfaa4a), + LL(0xe65ede91,0x8b4819e4),LL(0x6dc0a533,0x5a5824ba),LL(0xb4c930f8,0x89d18b20),LL(0xfcefa378,0xaad7a5d8), LL(0x298dba63,0x2ef790c2),LL(0xe90c322f,0x3e4b31b6),LL(0x52ce2ee4,0xa257bb81),LL(0xd39c36bb,0xb8c2966e), + LL(0x487719c7,0x13954df8),LL(0x791b00e7,0xcb0f7ae5),LL(0xc8d21faf,0x367a1cad),LL(0x3fbd8a7c,0x44dd204d), LL(0x5f67ec30,0x778fdb56),LL(0x5de5caeb,0xfb288790),LL(0xca53300c,0x310b4d56),LL(0x325c54b1,0x37dbb7c4), + LL(0xfe771ef7,0xc80c83a4),LL(0x1c1c1b92,0xe212050f),LL(0xf09c666f,0x0f12bb88),LL(0x10a2eca2,0x8ec5f396), LL(0x90a22eb7,0xdaf96996),LL(0x450de941,0xeb77eee5),LL(0x58fb0165,0x13823c58),LL(0x31272111,0x2157ba6e), + LL(0x2b4f9e7e,0x110ee33e),LL(0xf682d48f,0x7e1b550b),LL(0x3e17cb9b,0x8fd8c6c1),LL(0xe1843894,0x91cfbcf7), LL(0x2917b1c7,0x5fc64346),LL(0xba86d14a,0x06f56d0f),LL(0xaf219f21,0xb8874d88),LL(0x11ab8b0b,0xf8803b37), + LL(0xbe12841e,0x7e63cf63),LL(0xbc90765a,0x9c9cc421),LL(0x1084fa84,0x0264a597),LL(0x252a9bbe,0xce260a60), LL(0x2fefa4f2,0xfaff225c),LL(0x05bd09b0,0x02b900ad),LL(0x11b1b81c,0x631e5cfb),LL(0x0a193140,0x4d93de46), + LL(0xe3173750,0xd92a710a),LL(0x671a3833,0xd712d3a1),LL(0x4116e26b,0xbc9caad1),LL(0xa72fbd71,0xeb24f658), LL(0x9055f802,0x3986a207),LL(0xe2707793,0x212446f8),LL(0x1721b395,0x602541d6),LL(0xb07160c2,0x4099a2e6), + LL(0x2369ce91,0x765390f6),LL(0x5754d219,0x2dc37639),LL(0x7c018afb,0xbc552369),LL(0x35bf6b66,0xca835077), LL(0x61d4b0a6,0x61b83e43),LL(0x27cf66c5,0x8f87f597),LL(0x9357cbf2,0xace57840),LL(0xabe47fb7,0x24834481), + LL(0xdb3c6e47,0xa434c950),LL(0xaa1da775,0x1f479519),LL(0xf14f9d5e,0x338c9cd2),LL(0x1e75f72e,0x4666ce7e), LL(0xe56564e5,0x4fce4d95),LL(0x89e0ff6f,0x0db55ed5),LL(0x00190b73,0x88796e85),LL(0x454e31d0,0xfdf6492a), + LL(0xb9ed6e3b,0x30cb3fbe),LL(0xde8f0544,0x5c796282),LL(0xb6af89bc,0xe11b36bd),LL(0xec439d95,0x0a91cf73), LL(0x0a93fe1c,0xbbe74a5e),LL(0xa5d75083,0xcf1f376f),LL(0xf7725460,0x6718bce5),LL(0xa316d17f,0x6654d7b1), + LL(0x0393aa3b,0xdaa925e5),LL(0x9446cdbd,0x81217e18),LL(0xa7afc408,0x07708483),LL(0x44709dfe,0xa4c76c4f), LL(0x3a1c412b,0x72557d71),LL(0xb49b0e1c,0xeb4c2648),LL(0xe4d6c002,0xcdd24b77),LL(0x77113e0d,0x3384ea5b), + LL(0x6a10a9ea,0x906fb748),LL(0x0a3b0e89,0x46cda42e),LL(0x7ae4ad43,0x10b9096d),LL(0x3bf2afea,0xe1f23996), LL(0x0dd82d19,0xcb50b941),LL(0x832d93a0,0x008e593c),LL(0xd86a71e0,0x0b1fb0e6),LL(0xb1730860,0x75f2aa6b), + LL(0x7efc480a,0xed5d4d7c),LL(0xc76c64de,0x500b9d8c),LL(0xec4fc026,0x28904003),LL(0xdec8b315,0xe41b3f23), LL(0x70c06860,0xa9b5caff),LL(0x28343b2d,0x5cb9a4d1),LL(0x9986a0c3,0xec157abd),LL(0xb5fc67e9,0xbcad3bc6), + LL(0x13aa9c17,0x6e64dd26),LL(0x271aef54,0xa347c4a2),LL(0x883d90bf,0x47b26cb9),LL(0xe1c412c9,0xe84d9c6a), LL(0x1c67439f,0xd2eacc10),LL(0xc61b2b5d,0xd7797bb3),LL(0x8ebdb4be,0x0deda652),LL(0xac3fc2f4,0x9e04455d), + LL(0x27c86688,0xbbfc6e69),LL(0xa1715a33,0xf7cf2947),LL(0x47bc6409,0xe047a3e3),LL(0x6f2a5b28,0xefeb573a), LL(0xd105ba3b,0xbf3ea3af),LL(0x426c6482,0x5f01b4c2),LL(0x968390b1,0x778a5240),LL(0x72bcf6a6,0xc9c71625), + LL(0x6fd8b309,0x698ec2c9),LL(0xa055809f,0x512ea17a),LL(0x8822943c,0x28cb44e7),LL(0xdeb7d3e6,0x434dc709), LL(0x1be76434,0xb8b324d1),LL(0x382ff0b1,0x7cf24ed3),LL(0x2905e726,0xda8265fe),LL(0xee6c3abc,0xd57b3915), +}, +/* digit=20 base_pwr=2^140 */ +{ + LL(0xd2a819b5,0xf9010095),LL(0x48f2f653,0x5291aaf9),LL(0xf0afe366,0xfa533907),LL(0x8e279e27,0x88a58ecf), LL(0xfae130bc,0x0f077127),LL(0xf8a54c75,0xee9ccf1a),LL(0xbed82b6a,0x38a6783e),LL(0xed414524,0x9a1acb3d), + LL(0xd9c12e2a,0xe4e53cee),LL(0x7fc1308f,0x11983fc1),LL(0x892c2d0f,0x3eb4d84d),LL(0x74499723,0xa0bfc1ca), LL(0x0145176b,0x708344d9),LL(0x6f12e75b,0xbb2988e0),LL(0xada67545,0xdf73cead),LL(0x2bb8f989,0xf37069d1), + LL(0x9cc17f65,0xa24a35e6),LL(0x89d9abe0,0xc49b3e9a),LL(0x2fc09ae3,0x82f40303),LL(0x002cc587,0xbffe7d4d), LL(0x424ef713,0x5511f4e6),LL(0xa658f660,0xb86bf654),LL(0x1c8baea2,0x623388d9),LL(0x33656759,0x60664a71), + LL(0xd8447e16,0x18996198),LL(0x662171dd,0x17195d76),LL(0xf448b8e6,0x28cfe6a1),LL(0x0658c923,0x8a3c2823), LL(0x9c35e852,0x0c548d89),LL(0x2b378157,0xadf1cd2f),LL(0xf30113b6,0x999e41af),LL(0x9cf4696f,0xf87515a5), + LL(0x9778aa8e,0x6c332c55),LL(0xd1b8d8b2,0x290ae3ea),LL(0xbf533883,0x3e2bfa0b),LL(0x1a523ee2,0xe48e3747), LL(0x50fde3ed,0x4d40f1d5),LL(0x48710433,0xb57e695b),LL(0x1b241f3f,0xa4101258),LL(0x3042cabc,0xa0cabf7b), + LL(0xedfea522,0x68eb19c7),LL(0xa400db7b,0x68e028b8),LL(0xa8f03a08,0x6cd97bf7),LL(0xf442fe36,0x09f4d266), LL(0x5d713a1f,0x1ac77f91),LL(0x2f58ccb5,0x356e3a35),LL(0xf8ddc47d,0x31d657f1),LL(0x90092115,0xfea7aede), + LL(0x4ad49f66,0x2aeba1d2),LL(0x9d40861b,0x16ff2bad),LL(0x7da225f5,0x25464f37),LL(0x1ffc3f0b,0xa2fe66cc), LL(0xac757f41,0x74074d7f),LL(0xcd0a2c20,0x5c85d3d1),LL(0xc974b266,0xccda2a05),LL(0xcc10a04f,0x5c2e2511), + LL(0x510d515f,0x01ea2535),LL(0x489e7856,0xc861c54c),LL(0x680d17bc,0x9bc8485b),LL(0x819ccc86,0x71472c11), LL(0x0e9b5d8b,0xa7ef9485),LL(0xd029720d,0x698c9fe8),LL(0x61f50161,0x6ce987d1),LL(0x9d240bf6,0x035f6f32), + LL(0x44ec2bed,0xe7c03c9d),LL(0x76cf95c5,0x0bc4f4a2),LL(0x88f014ee,0x0722d57c),LL(0x76fa941b,0xae406348), LL(0x23ee068d,0x046424df),LL(0xe8c130c5,0xd30b6530),LL(0x554f149d,0x17b69098),LL(0x92f95b71,0x887e04f7), + LL(0x941c1244,0x414e7371),LL(0x94f1da50,0x1d48fe53),LL(0x6519802a,0xc18bcd89),LL(0x48925019,0xfae7c2d8), LL(0xf2ece2af,0x0f311ddf),LL(0x0a779f79,0x7e8e0e08),LL(0xb6207944,0x47daa5f9),LL(0xefd08d6e,0xf29dc331), + LL(0x23e48f60,0x9c096e19),LL(0x8dd36f0c,0xbcc6fe53),LL(0xbb86a9ca,0x452e60f9),LL(0xed16cf06,0xad35f732), LL(0x2bf445f7,0xcbdd01a2),LL(0xf60ce893,0xb7848e94),LL(0x2939a977,0x5e65e8ca),LL(0x63cfa5e4,0x304ebedc), + LL(0x252cc655,0x79bae721),LL(0xc4743792,0xa3b9a4e5),LL(0x36fdba1c,0xf32dcfeb),LL(0x7ac19885,0xadbd0c1f), LL(0xdc42a2cd,0xefb4fb68),LL(0x2289a71f,0x78b1ca37),LL(0x87fc6df4,0x7e1f70fe),LL(0x90a9faec,0x8d024301), + LL(0x46cd4141,0x37c08672),LL(0x1a60d8e5,0x3c0fed17),LL(0x0f56fea1,0xab18bf06),LL(0x372e757b,0x879ee748), LL(0x1d280206,0x84b19b80),LL(0xd96ac240,0xa40d7ce3),LL(0xfea42ebc,0x5d493fb1),LL(0x40d79bbd,0x9a5fdafd), + LL(0x383b371c,0x790c0b30),LL(0x676f8168,0x6dae5df9),LL(0x4c584948,0x101bb4fe),LL(0x55faafeb,0xe3d7e99f), LL(0x134c2e3b,0xd2c9aefa),LL(0x79e27788,0x0aa2a71f),LL(0x7ed0a606,0x4082f7a6),LL(0x6a1be308,0x843c12bb), + LL(0x56e9e473,0xae72ee74),LL(0x743e16ee,0xcecde6c1),LL(0x7c48ca04,0x9a06f105),LL(0x5f822a31,0x79179cd2), LL(0xe3530605,0x570d3eeb),LL(0x4c7b03b0,0xbacb30c3),LL(0x0eea0cb4,0x0a8fe254),LL(0x2cdf203a,0xa052a555), + LL(0x9c34971b,0xee031587),LL(0xe76545cf,0x5829eb07),LL(0x33a81bb9,0xb7a3a6ae),LL(0x49c9f710,0xff42daff), LL(0xbffb951b,0x894eae85),LL(0xce70f324,0x815fe3e2),LL(0x428b1f12,0x636564cb),LL(0xa029b0bd,0x722e0050), + LL(0xd373a65b,0xf45cb816),LL(0x6078d95e,0xf2210e00),LL(0x20d2924a,0xf767d7a6),LL(0x25b66987,0x06d6b552), LL(0x790563a1,0x5c4a3999),LL(0x3c85510c,0xcea00a91),LL(0xd2db6297,0x7e37da9c),LL(0xf67303e8,0xfca4735f), + LL(0xaf76f475,0x324ca06e),LL(0x76391adb,0x3367845e),LL(0xa26fe169,0x222aa1ce),LL(0x7ede94c7,0xb15a8665), LL(0x6b6a1f33,0x5b736342),LL(0x8562f392,0x25db61e1),LL(0xf2066206,0xfd4d720d),LL(0x82c555c3,0x26ef773f), + LL(0xfde6caa3,0xb6e35b3a),LL(0x87fabf4a,0x34eb5e13),LL(0x86236a62,0x4ad68635),LL(0x28510f8f,0x2651d3e6), LL(0xe0873ba6,0x88073e34),LL(0x22f63746,0x3becce70),LL(0x7c08dac6,0xff8f9b61),LL(0x8c28aa65,0xc8b45a9c), + LL(0xd87b59dc,0xe09c063b),LL(0xcbbdd4ec,0xf3e4b9ef),LL(0x4855a43e,0x1b6b1793),LL(0x4ada74ac,0x594d5565), LL(0xb410c5ef,0x10ee400f),LL(0x35695fe9,0xfc118113),LL(0x8f75d723,0x766cfe48),LL(0xff63aa76,0xc72023eb), + LL(0x9df9a165,0xc503a858),LL(0x851acc4b,0x9b1099ef),LL(0x66202ca0,0x9246c61a),LL(0x81390ccd,0xaba97788), LL(0xba9e2785,0x3309fa65),LL(0x2220f385,0xbc0388be),LL(0x00ddc8ba,0x94c01a9e),LL(0xbccfdec8,0xaa54aed9), + LL(0x059fc0d6,0x1a959c58),LL(0xf518e1c1,0xd0f34c38),LL(0xb53be8fe,0x38aa2b1d),LL(0xacdc872f,0xd95a2a19), LL(0xb4140bd6,0x97bde382),LL(0x4cfd5718,0x4084ba9d),LL(0xfd22450c,0xed016bfa),LL(0xa5d1f5bc,0xf00cdccf), + LL(0x111696ea,0x905114cc),LL(0x3a46e782,0x1f58a4d3),LL(0xa5e57fa0,0x899d1856),LL(0x68c45c2f,0x25186954), LL(0xfa6e3eec,0x806deb4a),LL(0x65a063a6,0x3c358d48),LL(0x3feacdcd,0xce28ed1f),LL(0xaaa8e601,0xef9ee31c), + LL(0x87c1c66b,0xddd4fe3d),LL(0xb3dbfac5,0xfc2b063e),LL(0x20c81dc5,0x52d37cd0),LL(0x083b5f53,0xb628f163), LL(0x7e42860b,0xd9295094),LL(0x307316ab,0xb877a744),LL(0x6b8d99b6,0xadec0d2d),LL(0x190bc675,0xa75183bd), + LL(0x4ad6bd44,0x2b1e0215),LL(0x9e020c66,0xda01ad52),LL(0x0c2913d9,0x65afd73d),LL(0xf0035373,0x67024b45), LL(0x4d308206,0xf501bb4c),LL(0x77e2e936,0xfa020c88),LL(0x936476a3,0x662b72bd),LL(0xbae57d17,0x07f76845), + LL(0xf34ca404,0x77a43055),LL(0x4eebc853,0x8e403294),LL(0x402fde89,0xe19ee69a),LL(0xfe00df56,0x9092acd0), LL(0xfb225f92,0x640c035c),LL(0xdce3aa84,0x92d94246),LL(0x971e9886,0x7fe8d3f9),LL(0x014b2a74,0xc569905e), + LL(0x7b7c299b,0xbafb8c4d),LL(0xd534cd21,0x3d289c60),LL(0xd311dee4,0x95e7032b),LL(0x6e8892a4,0xac0c46dd), LL(0xe5bd6486,0x9fedef00),LL(0x99f703aa,0x3f4d8daa),LL(0xf0c0ecd2,0x78e47925),LL(0xfdac73de,0x8f143c2b), + LL(0xc14bd094,0x1f88f5a9),LL(0x99d9532b,0x6cc19e43),LL(0x639ba66c,0x6e474499),LL(0xf5d06b03,0x5d9a283b), LL(0xaa25dbb5,0xc7e8164f),LL(0xf03faec8,0x7ab42a48),LL(0x647a0d72,0x4135765b),LL(0xe196b571,0x9562a676), + LL(0xa720cc20,0x62cc4c05),LL(0x13fa1ad2,0x9ed3f637),LL(0x7f59bac9,0xe5816f51),LL(0xb6884359,0x738e1544), LL(0x34d0fb02,0x83bb2666),LL(0x8014c57b,0x0e582c6b),LL(0x145e2bff,0xbb4069ae),LL(0x4f5f1d7d,0xd1965cdf), + LL(0x3cead86d,0xaf77f98b),LL(0x0e51cbd8,0x0ba278bd),LL(0xac2ebb7c,0xf11f20f6),LL(0xc9992b55,0xafd2333f), LL(0x322472b6,0x425dd0e4),LL(0x0958215e,0x0027a74a),LL(0x4cf7e0e4,0xddb301e7),LL(0xcbb70c2e,0xd0656ed5), + LL(0x0e7662e8,0xa40f629f),LL(0xe399a5ca,0xdaa85755),LL(0x7297010c,0x4c119aba),LL(0xe5df7140,0x4a4a6a43), LL(0x474f7873,0x6d90d303),LL(0xd1f8f867,0xc5b0e19c),LL(0x4f6dc217,0x188bcae6),LL(0x6777357f,0x51ce999a), + LL(0x41aeb39f,0xdfc9578b),LL(0x7dd55c1f,0xeeda86fe),LL(0xfb814075,0xd4b8fc54),LL(0x33a1317c,0x12e32a78), LL(0x2fd217d1,0xeb79cd2b),LL(0xdbd07638,0x5f5f20c0),LL(0x53dc7d8b,0xfc57643a),LL(0xf08350e2,0x65126014), + LL(0x871b0d3d,0x737ef5b4),LL(0xae3143a5,0x6b7e04ce),LL(0xb7ae12b9,0x0e5ab52e),LL(0xdb66ee0e,0x1a956daa), LL(0xeaa7042e,0x59657e47),LL(0xbf84a2cf,0xbbc35318),LL(0x78679b8b,0xef55429c),LL(0x60cb7678,0xef92df9d), + LL(0x1dd267d6,0x17655580),LL(0xeb0bc1fa,0x00a3ec71),LL(0x50514840,0xafa0a256),LL(0xf161c685,0x68c28d0c), LL(0xb1c766dc,0x069f7862),LL(0xd5ad4568,0x6868a463),LL(0x70e46d7d,0xf9c3d670),LL(0x6c875260,0xd2432cc9), + LL(0x088cecd9,0x534c3425),LL(0xb4e34c6c,0x3f1818e6),LL(0x028f153b,0x3aedf0a8),LL(0x50d9433a,0xe0a1cb9d), LL(0xe523b764,0x9b4e225f),LL(0xe5f8542c,0xcba6cba9),LL(0xa8f6b81e,0x59c307e4),LL(0x01bb44fc,0x36abf4b7), + LL(0xdd62528c,0xf7692c14),LL(0xdf57773e,0x0d4a8341),LL(0xc9b4f968,0xece6957d),LL(0x52779586,0x82eda200), LL(0x2f06ec6b,0xb902c488),LL(0x91a876f0,0x127dd7ba),LL(0x33ad0c13,0x06eb96d8),LL(0xfc5985ce,0xd7394080), + LL(0x661aaa4d,0x624c8f61),LL(0x6717a3e1,0x6fe10a11),LL(0x53168ad0,0x6c288c53),LL(0x8b52d037,0x91b8779b), LL(0x1b5b0ab9,0x89e664d4),LL(0xf30d47d3,0x9f69b44f),LL(0x03176019,0xfe67cad5),LL(0xb346a205,0xb83efd48), + LL(0xaeea0c91,0x63fc4863),LL(0xdb56004b,0xbabf9042),LL(0xa9917def,0xdb19f2ee),LL(0x54c3fae1,0x1d12f2dc), LL(0x55e36d40,0x7bb496af),LL(0x6be63b27,0x1f6c11f8),LL(0xcaf9a5b9,0x96d79804),LL(0x0648051c,0x03a085c4), + LL(0xb56baf4c,0x3b54c223),LL(0x559c1fc1,0x04af8c4c),LL(0xabd3cebb,0x05d55266),LL(0xf865e262,0xd2d3ae9b), LL(0xedfedc54,0x3bd3ca3a),LL(0x922776c4,0x30a6ff1c),LL(0x1616a6f2,0xfecd8845),LL(0x94948d8c,0x4e7bc2e8), + LL(0xedca784b,0x16e0d824),LL(0x67ea1eea,0x84584f98),LL(0x8625626b,0xeceb1418),LL(0xc34fc1f3,0xa487cf9f), LL(0xa57cec36,0x4ecfedd2),LL(0xd24a0709,0x08624865),LL(0x6a48f3ee,0x47bb1909),LL(0xc69bc041,0x54c5dd0c), + LL(0x7527166e,0x15a291e6),LL(0x4a9a8315,0x8a92370d),LL(0xda584bd6,0xe9fe705d),LL(0x3625a669,0xed441dc3), LL(0x3063f2de,0xa57929ce),LL(0x6348cc31,0x2809fe4b),LL(0x04cc19c0,0x92041d54),LL(0xb62c1f94,0xd7c227fc), + LL(0xcd0d497c,0xecfeee53),LL(0x128818a8,0x8d1ea9fd),LL(0x2ae4725d,0xb5cf2c28),LL(0xc6abad3a,0x7de9f967), LL(0xb14a183c,0xc64a11fc),LL(0xd5777d77,0x7f14d0fe),LL(0x12957444,0xbe79846c),LL(0x0e3257ca,0x4cf23abf), + LL(0x6e01b48e,0x8da0fd8d),LL(0x5ee87ca4,0x63a7ff16),LL(0x5cc96b94,0x90dff4d3),LL(0x406fc905,0xff1b41c3), LL(0x3ac71c41,0xdd932925),LL(0xcf65e59d,0xec57f1b8),LL(0x3ce0512b,0xa3116d6f),LL(0xa2e28316,0x3b46fd3c), + LL(0x60156a5e,0x5a6c0314),LL(0x55d46fd0,0xfab3afe3),LL(0x9846f0db,0x4617926c),LL(0x121ef237,0xc2d5a447), LL(0xf1cda3b1,0x789498d1),LL(0xccd64aac,0xa195cf03),LL(0x9440be2b,0xe8d1a162),LL(0x3ad5373e,0x7399890a), + LL(0x4fbf1899,0x65dcea2f),LL(0x44ee1a5a,0x44d9c8ab),LL(0x406880fb,0x2e94f8c8),LL(0x51faab7d,0x70732bad), LL(0xf1e92b52,0xd69309dd),LL(0x3c7685d0,0x25f9a675),LL(0x1dbfbaa0,0x3604f30b),LL(0x2ff28c22,0x5ac0001b), + LL(0x52762d8d,0x648ec4fa),LL(0x0cef95f8,0x4fc248c6),LL(0xe5fbb57d,0xfc0f7030),LL(0x5ff2db51,0x2e644729), LL(0x3775471c,0xd85877ec),LL(0x6076a271,0xe2580058),LL(0x09cb3873,0x58a4a24f),LL(0xb142da8c,0xb412928b), + LL(0x5997987a,0x1da7964b),LL(0x1825d97b,0x69765ff0),LL(0x4c97095c,0xde8ae407),LL(0xbb59316c,0xb257a968), LL(0xcf2dfbd2,0x80e5240d),LL(0x1fccd0e7,0x2b3b04b0),LL(0x8ff1093e,0x949f1234),LL(0x65f62273,0xa4df3290), + LL(0xd0058ffa,0x305b7479),LL(0x180b0de6,0xe459ecff),LL(0x8ca0585f,0xfbe00c08),LL(0xc3dd4fa0,0xc169e23a), LL(0x44026f6e,0x65d9009a),LL(0x1d96fe58,0xbbc258c3),LL(0x328ed1e0,0xd7ed379c),LL(0x23970884,0xe8b45744), + LL(0xed1095b5,0x2f44c8ce),LL(0xc59404aa,0x25725b0d),LL(0xa273e09f,0xea926278),LL(0x529143d5,0x102b120b), LL(0x81174d10,0xbd2c012d),LL(0x4e8333ad,0x0bf5cf89),LL(0xb2f60645,0x6b93e3b0),LL(0xf38df3ce,0x040298b8), + LL(0xb89e18fd,0x6433b6fc),LL(0x6bd6af88,0x48d6584a),LL(0x0e1747a5,0x46c8a061),LL(0x1ed79faa,0xe225d3cd), LL(0x5c28a48b,0x6c579abb),LL(0xda3889d6,0xc7a5ff4d),LL(0x008febdf,0x037ebc4b),LL(0xa7c5cbc9,0x03b60593), + LL(0xdcaa3319,0x0840804d),LL(0xb7481f45,0x38b0f084),LL(0x112e794e,0xfdb059c9),LL(0xe1afb614,0xb62bce05), LL(0x53be7c02,0xc15035b6),LL(0x153ee8e5,0x66fc7106),LL(0x8258727f,0x27fd2ea4),LL(0xb470105e,0x8e012416), + LL(0xd7711aee,0x3d24685b),LL(0x021bab69,0x66a83c3b),LL(0x5501d69d,0x6e1112a7),LL(0x068b0504,0x2219fe0b), LL(0xa926ab1b,0xaaa553c1),LL(0x56d6ca58,0x1c81af95),LL(0x221ef447,0x0a997380),LL(0xf5f55487,0x881a62fa), + LL(0x4f1b618a,0xf88fa0bd),LL(0xcbac98e8,0xb20e161c),LL(0x3bc6a7ad,0x443352b5),LL(0x0fd5748a,0xbc8e03ff), LL(0x0ca427fc,0x64006aff),LL(0x7cbbda99,0x1a477593),LL(0x1a347c47,0x21ef1afd),LL(0xdee162b6,0xfe056287), + LL(0x797a0b14,0x9d4eb7da),LL(0x951c4bd0,0xe4e01a46),LL(0x7fe354a6,0xaf8fa17f),LL(0xc430b12b,0xd71f160c), LL(0x83d46be0,0x5bb68437),LL(0x619bba86,0x99d10d82),LL(0xf4327042,0x95c2219d),LL(0x9c19ab57,0xdace2322), + LL(0xc8750fe7,0x88abbc67),LL(0xd3abe7d2,0x81ab300a),LL(0x45aa8948,0x62a6d8d5),LL(0x5d4ce8c3,0x76175bbd), LL(0x8ea70976,0x095cb181),LL(0xf7e62a19,0x785de3fc),LL(0xed11a7fe,0xc478bce8),LL(0x1528aee2,0xb7e5993b), + LL(0x76c32e4b,0xb9ec58d7),LL(0x2dbc9a61,0xef815613),LL(0x5e07410d,0x372c3856),LL(0x033276d0,0xa1b16510), LL(0x82640d26,0xd8589581),LL(0x9053fff0,0x1cb98180),LL(0xc1ff11f1,0x41519ce7),LL(0x666431e4,0x2f21a48f), + LL(0xe83ee840,0x2c223ed1),LL(0x1e7cf4dc,0x833ae708),LL(0xacd13385,0xec8853d8),LL(0x6a7a8cb1,0x559115ab), LL(0xeb184e71,0xe2f4ab2a),LL(0xc10194df,0x679abbce),LL(0x3aca0828,0x10199c84),LL(0x978cc1d6,0x7474e113), + LL(0x44e8eb9a,0xa4460ef1),LL(0x828aa4bf,0x4cde5260),LL(0x249bff50,0xd3d23790),LL(0x6bc7fbbc,0x2e6660da), LL(0x61494df0,0x3e3cc146),LL(0x0bcda8ba,0x6e9a1571),LL(0x096e00b7,0x68ce233e),LL(0x5106c85b,0x247a5c49), + LL(0x55fc5757,0xe6f0cb5c),LL(0xb7585799,0x452682b9),LL(0x869e747e,0x84509dfe),LL(0x8d23be04,0x904366e5), LL(0xb0f72c6d,0x7324a14d),LL(0x1913a9ff,0x9fbe3116),LL(0x428a6b5d,0x2f9fa62a),LL(0xf8a34d9e,0x8039270f), + LL(0x407aec78,0x0e3ce7ae),LL(0x826458cd,0x4d935d3d),LL(0xfc6f46d4,0xf75db7db),LL(0x88586874,0x3ab7ba68), LL(0x9a198b50,0xec92749f),LL(0xefc51cdb,0x0ffc7681),LL(0xe17bc0e3,0x951406d5),LL(0xc898a018,0x39cd2d07), + LL(0xf2f79af6,0x9dc3803c),LL(0x0a56cd68,0x292f3164),LL(0xf6fbdbdf,0xdcac21f9),LL(0x23e9e959,0x6f9ce2a4), LL(0x970f6c34,0x2011d221),LL(0x9e2decfb,0xd2e63711),LL(0x118ff327,0x19c7a489),LL(0xbb6e534e,0xe19d7e83), + LL(0xbd1a426b,0xc685389a),LL(0x8c679952,0x432ff7b0),LL(0xc5e2687f,0x516cbdfa),LL(0xba1eac8f,0x8242405d), LL(0x0b09854e,0x63af3152),LL(0x231ec979,0xcecd0faa),LL(0x7273f0b2,0x4746733f),LL(0x8f001365,0x69b28d87), + LL(0xc6f2623c,0x0d87d506),LL(0x86c40ed2,0xd209a9c6),LL(0x0fa20f3b,0xa5b7fde2),LL(0x4f5b2129,0x54550dc5), LL(0xfeddaa1b,0x36884047),LL(0xd899a29f,0x51398fa0),LL(0xcdf11867,0x14a416be),LL(0x3e466b62,0x86351ac6), + LL(0x39bb481a,0xd63e91e1),LL(0x99984155,0xdcdc072a),LL(0xd2d8e622,0x9fce6e38),LL(0x8e8c8034,0xbf6978b6), LL(0x8c37990a,0xaa1ae874),LL(0x0e749b86,0xd1f0e068),LL(0xcbdc7c12,0x5aa303b1),LL(0xc9130211,0x9a78baaf), +}, +/* digit=21 base_pwr=2^147 */ +{ + LL(0xb3e2087b,0x5eeba910),LL(0x44a61a33,0xbd016dd3),LL(0x48cd5c0f,0xffd1f081),LL(0x2e6a8e94,0x041c6aa0), LL(0xc4ac3d91,0xe09c35c5),LL(0x634767a4,0x58298105),LL(0x1040c2b5,0x6120e7cb),LL(0xa713589f,0x32a661ef), + LL(0xbd74c70e,0x5a056a90),LL(0x8af672f3,0x44f7d00d),LL(0xef4e9a48,0xdc25ab68),LL(0xfdfb1069,0xadd15cc3), LL(0x9f3033bf,0xb1f4fd28),LL(0xebb8b8a7,0x088a49bb),LL(0xa8d2861c,0xea485869),LL(0x6b977fb5,0x46dbfdaf), + LL(0xd88ae888,0x04e6461e),LL(0x9be2d7ad,0x112d204a),LL(0xdb558427,0x952dc813),LL(0x27e35200,0x39b01652), LL(0xff02cdbb,0x8b89bfce),LL(0x3e854e4c,0x1381a99b),LL(0x999efd2b,0x68185218),LL(0x86dc62e1,0xeb41e1bb), + LL(0x02d0aaff,0xa264ef40),LL(0x6e679fe2,0xa678c07d),LL(0xfd88bdce,0xcff13be7),LL(0x617badb8,0x9a8efe8d), LL(0xad5a22f4,0x1388a815),LL(0xfec398b3,0x8f821400),LL(0xff4fc2da,0x85a6a565),LL(0x858dd4f3,0x681f0181), + LL(0x91ee75bc,0xdc870745),LL(0x64f20e50,0xbadbf940),LL(0x49d3d417,0xf3ea437f),LL(0xc02109d0,0x7bebd868), LL(0xd16bb240,0xe6c8d93c),LL(0xfdab9bbd,0x2087141a),LL(0x20a3b470,0x8dba80ff),LL(0x29d3a8d7,0x960a0c7b), + LL(0xf8ec1151,0xae61b637),LL(0xaadc8545,0x415dd36b),LL(0x69d0985a,0xed21d176),LL(0x97893f50,0xc4d062af), LL(0x337b81f9,0x4d93ba1a),LL(0xb7c163a2,0xb995fe9e),LL(0x5416e4ed,0x447eff3b),LL(0x0bf4a8e7,0xd7660300), + LL(0x56d9e00f,0x9e14c6a2),LL(0xfa1f60e0,0xa228491c),LL(0x8782a9be,0xd540713e),LL(0xdcd55e21,0x5d3fcce8), LL(0x35c87b90,0xa176c34e),LL(0xf9508f35,0xc1d80aa1),LL(0x92302d47,0x14f7e7fc),LL(0x2b076e72,0x459372ba), + LL(0x4e933b19,0x44168fbc),LL(0xe54ea969,0xaf2db74c),LL(0xaeacbb56,0x36fade13),LL(0x84e6cd1d,0x29708665), LL(0xb692df97,0x6f7ff1e1),LL(0x6ae66307,0x5a68c1a2),LL(0xe7685f20,0x85bc544c),LL(0x0f65eeec,0xb3f42e6d), + LL(0x5b91b644,0xef209f44),LL(0x50cb02b6,0x808b930a),LL(0x099f684f,0xc5da5e86),LL(0x4330c2d8,0xd8f1dbcc), LL(0xd8608776,0x52e8cab5),LL(0x16e56f5d,0x13c89771),LL(0xb135282b,0x7d7d802a),LL(0xe9be8a20,0x69762c39), + LL(0x2a08a1dc,0x13f6bbad),LL(0x7f2dba7a,0xa7131e4a),LL(0x7864f5a3,0x751dce48),LL(0x08135109,0xc5af1b45), LL(0x4f08636e,0x3c4d75f7),LL(0x2e251e48,0x9949b2f5),LL(0x9bd98853,0xd0497977),LL(0x909b0e84,0x09d8e627), + LL(0x4ceff1c9,0x505753ee),LL(0x460710ca,0x03ca4571),LL(0x5480abc0,0x0cf72dee),LL(0xc19c8ef3,0x55d5a30e), LL(0x86541f6f,0x9e47641b),LL(0x10c9d6fb,0x89b2e48f),LL(0x0860915c,0x9d16382b),LL(0xf54b337f,0x770ac417), + LL(0x77ef7f67,0x366d0781),LL(0x2b6340c9,0xfefec947),LL(0x097acf63,0x7ce3a056),LL(0xaf306dec,0x26538c7c), LL(0x287dc8d1,0x8a8bb55e),LL(0x448823ae,0x9431095b),LL(0x7970fc51,0x8358087a),LL(0x413509ac,0x95299959), + LL(0x8b0911d4,0xb165f92a),LL(0xbfb37459,0xdde855ee),LL(0x3d9ce36e,0xe672eed4),LL(0x74bae261,0xf7da91e5), LL(0xb741c8e6,0x1f67323c),LL(0xc92c91ce,0x8efd4661),LL(0x78e73c42,0x556f0b1a),LL(0x16101f96,0x7d326b7f), + LL(0xfb48bd3a,0x8814ef0b),LL(0xc508309e,0x1bbbe13e),LL(0x67709c10,0x7ddaf061),LL(0x6436f655,0x82b67847), LL(0x03712e64,0x2a5601c6),LL(0x3e3f9b2e,0xac1f0362),LL(0x09184b5d,0xcc7e6a09),LL(0xb4625149,0x1258b265), + LL(0x384a6b54,0xd9f21461),LL(0x6cfe9311,0xde483161),LL(0x889f43cc,0x593dae45),LL(0xedee221b,0x8454335b), LL(0x3a2cbced,0x90f3fb43),LL(0xcc8dcb75,0x895ed692),LL(0x14233aa7,0x9857d719),LL(0x48166d5f,0x91b1a2ab), + LL(0x675b47a0,0xfbf7033a),LL(0x6542378f,0xcb3669c4),LL(0x125ec248,0x96abb0f7),LL(0x795fc346,0x6d5d2047), LL(0x8f5cffb6,0xa6c6c9e8),LL(0xbea5ee09,0xb968f2c7),LL(0x844ffd6f,0x2f2ce735),LL(0x27e40ac8,0x7931b877), + LL(0x2b63d538,0xe1f62dcf),LL(0xf44d7bf9,0x395681dd),LL(0x54aec359,0xf02eedf7),LL(0xa0ad5eb7,0xc64b6233), LL(0x346b086a,0xc65093c7),LL(0xe957b243,0xfcf8ecc9),LL(0x1ca48020,0xe1accffa),LL(0x4047bbeb,0xe1f29792), + LL(0x9fc6be25,0xb1097d40),LL(0x923eb7b4,0x02d33d19),LL(0x1f58f545,0x9e25200c),LL(0xda51efcb,0x2ffae306), LL(0xc0b011f2,0x7e6d76c1),LL(0xf680676b,0xedbd8398),LL(0x8d7fc8c2,0x38517fc2),LL(0x5c3ab05c,0x55a7fcf9), + LL(0x8e6dd02d,0x047e2422),LL(0x1f290d6a,0x7b3bf0e6),LL(0x6a99a6d0,0xbcf326fc),LL(0x2eef8232,0x1e3f38fa), LL(0x15bac529,0x9890780e),LL(0x9f143ba0,0x94202e0e),LL(0x885e4ed5,0xbd574712),LL(0x396f938d,0x2404c223), + LL(0xacde8286,0xd54d401c),LL(0xe7af01fd,0x80397794),LL(0xf615a8eb,0x94457d07),LL(0xd22d9ef7,0x34680480), LL(0x2c489ccf,0x04d4b302),LL(0xc3673dae,0x11dea4bd),LL(0x58cdfe41,0x8fbb4df5),LL(0x0f10a70e,0x49425184), + LL(0x077a59ce,0x831b977a),LL(0x894627f3,0xee08fb0c),LL(0x2f8553f0,0x21360507),LL(0x0487171b,0xca77ccd1), LL(0x07e11059,0xc17d20c2),LL(0xbe613256,0xcf74be6b),LL(0xa5fe18c1,0x06f185e6),LL(0x2b57ce3e,0x8d2cf4f5), + LL(0xc9c983e7,0x7179273c),LL(0x153f58d8,0xc7d27357),LL(0x4f400bd4,0xc0273069),LL(0x26262553,0x23309c7f), LL(0x712d0314,0xf26b6e11),LL(0xf96ee39a,0xb925cebf),LL(0x73944251,0x6df57108),LL(0x589d90aa,0x95419b24), + LL(0x796a8ee2,0x57a1bcc5),LL(0x2acee09d,0x22a22530),LL(0x66fa2911,0xa4c2cc03),LL(0xd85f13dc,0x9cc2b7fa), LL(0xce152790,0xf2498b8a),LL(0x1caf39d1,0xd8406007),LL(0x84c0822f,0x7ff50064),LL(0x155f1280,0xaf14ca4b), + LL(0x89b781c2,0x113f094b),LL(0x013833a5,0x996bf893),LL(0xc0b9cf6d,0x26bc6210),LL(0x6a88f1cf,0x18e2d3ac), LL(0xa21a2d35,0xc0ff2b3c),LL(0xa79e918e,0x409c2598),LL(0xb6917e22,0xffcf65a0),LL(0xbb4f8f3c,0x8036897f), + LL(0x9ec27fd7,0xac660365),LL(0x0c56cbb1,0x3c5ca1a9),LL(0xbe9e9ec7,0x01c5dce1),LL(0x386edb4a,0xdc21b01a), LL(0x4b1dde01,0x47e02a92),LL(0x44af3e0b,0x0613b7ca),LL(0x1c445b6f,0x644ac708),LL(0x87243e2a,0xb5566f0f), + LL(0xba9f354a,0x5b244172),LL(0xeb653a5e,0xaca4e9d3),LL(0x514809f2,0x6ff6904a),LL(0x96595230,0xf87a329b), LL(0x8d4bd051,0x39ebe6eb),LL(0x07d17d59,0x66f05f5c),LL(0xe0f81731,0xfa1ee673),LL(0xd12804a9,0xf41c1042), + LL(0xacd14cf5,0x1c4a655a),LL(0xef47548f,0xdc72f5bc),LL(0x0b3ee6c7,0xab07ceff),LL(0xbb501a28,0xcfa88319), LL(0xd8f03f7c,0xcec9c2e2),LL(0xe0c98d62,0x3098d752),LL(0x0a8681b3,0xa41a0794),LL(0x23587932,0x0e588076), + LL(0x5ef86f7c,0x4617dc66),LL(0xcedb5377,0x51de8430),LL(0x737381b6,0x0dda704a),LL(0x008f4671,0x83a44653), LL(0x38401c11,0x71bbb73e),LL(0x4894d378,0x879fe49c),LL(0xeab186a2,0x8875eef9),LL(0x12a382a9,0xedefe278), + LL(0x13b897fd,0x95ef41b8),LL(0x2a98ddd9,0xfefd495f),LL(0x680b12e8,0x09cbccfc),LL(0x167da5dc,0xc1888a21), LL(0x3bb290b4,0x2a204912),LL(0xd9613190,0xdcac95fc),LL(0x709c76f7,0x4df94f62),LL(0xa5cea926,0xc8c3a8ab), + LL(0x37a2b813,0x15c876b2),LL(0x9c3c821c,0x9b52803e),LL(0xcb3d6ff1,0x40f2268c),LL(0xa1573601,0x689f1696), LL(0x8e921f56,0x8d7566dd),LL(0xd992335a,0x5d8a990c),LL(0x20dc4f4b,0x6339153a),LL(0xdc5d06ab,0x0b07419c), + LL(0x97c201f9,0xe9cc014d),LL(0xa635f472,0xec04a52e),LL(0xa538a84f,0x6aac504a),LL(0x5762fe7c,0x4d0288e3), LL(0x34cbd09a,0xaa8539f0),LL(0x2619bcf7,0x6f7e0e94),LL(0x0dd338d0,0x178303dd),LL(0x8326f40e,0x6b58c2b1), + LL(0xfe73e699,0x98bb15ec),LL(0x47526489,0x7533abdc),LL(0x491dcc6d,0x4b269607),LL(0x77187363,0x325ec2a0), LL(0x7e9ab865,0x766faa19),LL(0xc25a9448,0x1c105b4a),LL(0x0531b5ba,0x0b6b8963),LL(0x2db1a579,0x32691f11), + LL(0x643e479c,0x24d90a57),LL(0xb98578df,0x048b27cb),LL(0xe53bed53,0x0600f93f),LL(0x0aac585f,0x1fd57dfc), LL(0x71d0e4e0,0xc3d72121),LL(0xf612fc4e,0x5ff10dfb),LL(0xb5a7ec79,0x9edf4b23),LL(0xd87706ab,0x975165c7), + LL(0x7095c3c4,0x8b99db04),LL(0x897faf50,0x65196441),LL(0xdd5b64cb,0x5d23d7d9),LL(0xe95fe601,0xec734b06), LL(0x0b5fcde9,0x03a5f53f),LL(0xebe35310,0x0186ad22),LL(0x84846603,0xe9a65eef),LL(0xa7c6e5be,0xe99e5188), + LL(0xf0887da6,0xa917327d),LL(0xe3f9fa70,0x49965f78),LL(0x4f10b31d,0x02ed227d),LL(0xb6120be5,0x535b4386), LL(0xcc1bf98a,0xdff21a8a),LL(0xeb1634bc,0x5b52a09a),LL(0xa3f61fa2,0x60f8690d),LL(0xb863c790,0x58a02566), + LL(0x5c6b2929,0xf9b90a9e),LL(0x22fca36e,0xd552e84c),LL(0x9eabcb58,0x6b23da4f),LL(0x5d4136dc,0x01111d07), LL(0xb3642a09,0xfaa80059),LL(0x5f49d533,0x1de667f4),LL(0x17525176,0xb3268776),LL(0xda729fde,0x75b0b102), + LL(0x6e9fe6ed,0x4ec7f678),LL(0x2717f8b0,0x28d29578),LL(0xd4cc149f,0x6a713c37),LL(0x7dfdf8c6,0x4804e04f), LL(0x5c931aa6,0xe7c6daab),LL(0xa0394f29,0x793e411d),LL(0x79ed9819,0xc0741c0d),LL(0x24d5d992,0x3f2ba70b), + LL(0xca9c987a,0xa61dc03f),LL(0xe64b50c1,0x78201cb8),LL(0x1861f4e4,0x45a23c25),LL(0xc4ee5d82,0x10f19f4f), LL(0xf3f055f4,0xf1520547),LL(0x006ccf49,0x69ae26b3),LL(0x33d8d4ad,0xe96eec0b),LL(0x48a4fc2c,0x00765f0c), + LL(0xa3976c07,0xad47e14e),LL(0xd905b6b4,0x82b1f882),LL(0x91382bac,0x7a1b9d73),LL(0x18178290,0xcc84a820), LL(0xb4e845ab,0x1123c6f6),LL(0xb92e3b08,0x63216635),LL(0x183879fb,0x748be745),LL(0xa73e9ada,0x7f20e1f0), + LL(0x9224c152,0x05de3e11),LL(0xea8fda4e,0x2fa9a474),LL(0xf48055ec,0xf5c8df05),LL(0x9e23a599,0x48bbf43a), LL(0x148086db,0xf593f034),LL(0xef0a2b62,0x0173a87a),LL(0x3fbabb6f,0x90ef0132),LL(0x21ade107,0x56ced09a), + LL(0x73f1d3e0,0xcf1ce892),LL(0x22424580,0x765236c6),LL(0xd939d063,0x0d223937),LL(0x7cb2fe2f,0x9a21beda), LL(0x1ce3a7fc,0xa559a571),LL(0x1b060dd5,0x7fd6b255),LL(0xc5afdf1a,0x4dfbd210),LL(0x1239368a,0xa74751ce), + LL(0x6d9a3eec,0x93acdd06),LL(0x9832dcfd,0x7d97f794),LL(0x0cc645ca,0xdafa9a44),LL(0xcfee0817,0x1da27ddf), LL(0x01b8dd49,0x0c1e6319),LL(0xd91aeace,0x8267e508),LL(0x87f43f20,0x86a2cedc),LL(0x07db2f24,0x7dd0e670), + LL(0x9db25177,0x8ea1e973),LL(0xab8802df,0x659cccb8),LL(0x004940ab,0x2bd39c65),LL(0xd9419db5,0x57676876), LL(0x6d6f237c,0xd52058a3),LL(0xfb4a9a7b,0xd9812dcd),LL(0x53bec56e,0x879890d1),LL(0x7ac5d9d9,0x17b7f151), + LL(0x1ec6db6c,0x2db3f5df),LL(0xafdf29b0,0xe9a3b18f),LL(0xe63a9ae9,0xda633d62),LL(0x38d13c8c,0x0922b16d), LL(0x7462c8e6,0xaeb7e270),LL(0xa01b6984,0xa98c96d8),LL(0x297c242f,0x5586e0d3),LL(0xff587596,0xbeddd1ab), + LL(0xc02ea084,0x79ac33ce),LL(0x8e02ae2f,0xe7d06753),LL(0x94d526b8,0x05fffd7d),LL(0x5ebc46d0,0x4590d655), LL(0x855f85e6,0xfb79c066),LL(0x7400ed08,0xbb3f0a6d),LL(0x67fb3683,0x46f4c3cd),LL(0xd19804cf,0x62fc1931), + LL(0xd1b6f356,0x0480e772),LL(0x56320242,0xa5810f25),LL(0x64073c03,0x6cf6c9c3),LL(0x46a6bfbc,0x7dfe137b), LL(0xba59baf8,0xa5633fa0),LL(0x5fd4929a,0xb703e5db),LL(0xd7515518,0x09eef835),LL(0xa0e3b067,0x2e596aa8), + LL(0x8649bb99,0x793831fc),LL(0x5ba4c1b6,0x91cb0057),LL(0x270ec9e8,0x44e93dbd),LL(0xd139d219,0xbf2ed01a), LL(0xc9d68198,0x39697e05),LL(0xde2b6894,0xf04439cf),LL(0x1e6b8e6d,0x65b7a04a),LL(0xce35ae6d,0xce3e9425), + LL(0x9f102fb3,0x041e0aff),LL(0x106ae748,0x91b3a87c),LL(0xc426fa5d,0xfd969804),LL(0x28f95b76,0xe624f1cd), LL(0x34f2ea56,0x6fe28cce),LL(0xd230f37c,0xdea55947),LL(0xf92f2742,0xd5e336f2),LL(0x1899c751,0x86852e3c), + LL(0xa5d1bd04,0x5ef2a63b),LL(0xb6ca2b79,0x5f4721a2),LL(0x9f484f78,0xbdb27b7c),LL(0xb085b4ed,0x2b07bf5b), LL(0x501b62ab,0x96b8ae73),LL(0x3ba64e23,0x0b1e003a),LL(0x93024347,0x43f9ec00),LL(0xae180a03,0x3c8c0c7e), + LL(0x8c0b21d4,0x58c72237),LL(0x2b15a1fa,0x9d51a996),LL(0xec755eda,0xf5201743),LL(0x933800fb,0x0c845fa3), LL(0x0e82418e,0xb6b66cdb),LL(0x3ae3eeb7,0x875258e5),LL(0x1a8f2b3e,0xf2c30b1e),LL(0x250f3328,0xa10b3843), + LL(0xc47c53f8,0x9f449967),LL(0x8775e16e,0x5dfe8c76),LL(0x336f2194,0xb02813a1),LL(0x5636498b,0x90ad3ad5), LL(0xc7c647e0,0x095acf96),LL(0x1f57c069,0xc90ef12b),LL(0x1fb85cc1,0x52f51878),LL(0x25a125ad,0x582cfd67), + LL(0x0d43ffad,0x53b4bfc7),LL(0x04dcf047,0x143b0b48),LL(0xd4500bf4,0x65d16216),LL(0x9ab1e4cd,0x960c7910), LL(0xe1d08c70,0x38b7ef7e),LL(0x9806e01e,0x64ae69e1),LL(0x6796b923,0x07468184),LL(0x70af1e64,0x6480887a), + LL(0x02384b34,0x4eb2d6fb),LL(0x05be47f3,0xb29337a8),LL(0x6b744f9d,0xfec96fc0),LL(0xc8c9afc3,0xc3de2fb0), LL(0xcc6dd0a5,0xe8ccc3eb),LL(0x71d7de7a,0x0329a9b9),LL(0xe357c4f9,0x459fbc8c),LL(0x025fdc97,0x80287f50), + LL(0xa089583d,0xedf1b0aa),LL(0xfb08add3,0xb1ad1a57),LL(0xe1ae76c1,0xd6826d03),LL(0x541462c8,0x3070cd2e), LL(0x83e6f4da,0x7b03c859),LL(0x24bdb487,0x5b39a809),LL(0x453bebb7,0x70017570),LL(0xb8ebbfc6,0xfe4e6206), + LL(0x106defe3,0xbb8a1899),LL(0x8683287a,0x6f23dc7a),LL(0x65d96aed,0x2cf01995),LL(0xdda4ea18,0x4e4cf7e9), LL(0xd2d0316b,0x72ad201f),LL(0x61de6cd4,0xd6451150),LL(0xc84856be,0x12432dbf),LL(0xd2a8378a,0xdd4dca98), + LL(0xbf881f9e,0xe70af958),LL(0xc4e58ec4,0xd4cd35ad),LL(0x5a531924,0x3889d3d9),LL(0xb4ce15ec,0xac657424), LL(0xf41e1344,0xdbe384ca),LL(0x5ab8bb08,0x9a1aed23),LL(0x8561df1d,0x375a041f),LL(0xb7685c1c,0x19f7a238), + LL(0xa4ba6317,0x8ba59933),LL(0x271f4aa0,0x0c44b6df),LL(0xbd64e922,0x51f4e88f),LL(0x9095769f,0x7279df94), LL(0xeaf8c8d3,0x098c17b6),LL(0x1aa841d1,0xe602ff2c),LL(0x8b63ce81,0xbe4e4926),LL(0xfcc79573,0x85de277a), + LL(0x5b8304db,0x38253d40),LL(0xe422af76,0x58c50c3b),LL(0xbf95c27a,0x7f7ec0d1),LL(0x6041df33,0xcb7c3a8c), LL(0x35364c89,0xc55595c0),LL(0x2a6eb1e6,0xd1a72aa7),LL(0xdeb98a3d,0x1fa941de),LL(0xeff46690,0x1e9607ab), + LL(0xad46a05a,0x6633e398),LL(0xb585e241,0xb99e5784),LL(0xea558424,0xd63106a4),LL(0x5df0e501,0xf0a5f939), LL(0x59dacce3,0xba17aaef),LL(0xe907c457,0x03dc5a07),LL(0xa9800bc3,0xa59f6d63),LL(0x364e1ef7,0x294a3827), + LL(0xdd191356,0x741bbab9),LL(0xc43954a4,0xe8fe9161),LL(0x65341d90,0x6a711fa9),LL(0xadef2d82,0x09bd0faa), LL(0x21ffc303,0x2112f27e),LL(0x395b69e2,0xcd2214dd),LL(0x8670b06f,0xe4b503c9),LL(0xc4e13ef4,0x219a678a), + LL(0x4a993816,0xc4020eff),LL(0x1bac14d5,0x00a9f5de),LL(0xeba7c3a4,0xd00fce1f),LL(0x14b537ab,0x2c6d4993), LL(0x6b898739,0xe9b2b540),LL(0x29dbf826,0xae53e6e3),LL(0xc8438b2c,0x634606c7),LL(0xfabfd429,0x268a9ee4), + LL(0x173b5583,0xb0486aae),LL(0xbf222673,0xf88a2f80),LL(0x0b3178c8,0x49c56f76),LL(0xd77d1406,0xeab47059), LL(0x993b1a7a,0x95035846),LL(0xa9b83efa,0xd6446e94),LL(0xc4424fa8,0x1d1a71ce),LL(0x3d08b8d2,0x8d814c4d), + LL(0x7b9374ac,0xbc3ed8d2),LL(0x77a3c020,0x8dd2d56d),LL(0x97efca8d,0x93ada735),LL(0x37974cd3,0x072bb2d0), LL(0x7bd74e40,0xa7c86e7e),LL(0x5b52e0ed,0x7bff5613),LL(0x053af1f1,0xc8d0bb30),LL(0x840bcb7d,0xc5bdb8f9), + LL(0x41690d1c,0xabdf5f73),LL(0xf0edac8c,0x0e857a78),LL(0x8238cfb0,0x59f40fcf),LL(0x511d41d2,0xdcb54f67), LL(0x0e645117,0x3f036ac8),LL(0x7af5fdcc,0xdc4e833e),LL(0x3d7bab2c,0x67d859b2),LL(0x5f8b32bd,0x92489b23), +}, +/* digit=22 base_pwr=2^154 */ +{ + LL(0xb9e2f9f8,0xe412aaf7),LL(0x0ff419ac,0x0484c1aa),LL(0x417bcb90,0x9d944989),LL(0xfe7318ca,0x2b73dbe3), LL(0x52dd7b0a,0xb91b71e5),LL(0x3954afeb,0xd61f8eea),LL(0xa07e3958,0xaaeab13c),LL(0x02a1ff49,0xde442032), + LL(0x0b054a0f,0x8292d96d),LL(0x74b9077a,0xa978af88),LL(0xfff1d49f,0x70bd185b),LL(0x0279eab1,0xbe6d0844), LL(0xb8ed07e9,0xa8fffe45),LL(0xcb920e55,0x714824a1),LL(0xaf1bb143,0xcd5c628a),LL(0x7637dbb7,0xd151afcd), + LL(0x62d7ee7c,0x83fb0f37),LL(0x9a3bcb7e,0x58c2282f),LL(0xeac2ca5a,0x79f77476),LL(0x579a262b,0x7e80c351), LL(0xedb4f0fc,0x19e67272),LL(0x1fbbe9fe,0xe142bb31),LL(0x95ea6cc1,0x5c7d7cce),LL(0xabfdcf7f,0x6465a380), + LL(0x5a26a1d4,0xa433bd2e),LL(0xd1c2d78c,0x1148bb1d),LL(0x64102515,0x4aae419e),LL(0x66489384,0xd03b9939), LL(0xd61a9919,0xe21d58b1),LL(0x4a0ef3d4,0x17618c36),LL(0x6fe8c0dd,0x2519020d),LL(0x00b87a75,0x48d837d6), + LL(0x426c1aa6,0xe6e067ab),LL(0xb11d1280,0x431579d2),LL(0x2ead6552,0xb926943f),LL(0x057fed1f,0x8fd692bf), LL(0xe9a98fae,0xed11c0ed),LL(0x9bcb2abd,0xe2bc967a),LL(0x68729b8e,0x1b388d66),LL(0x6f74563d,0x2144d67c), + LL(0x03fcd3bf,0xbe519757),LL(0xc8c7b62f,0x3f9dbd8d),LL(0xfc476e0e,0xce91fce6),LL(0x2715393a,0x2f140c73), LL(0xf11da35b,0x8a149a94),LL(0x5367030d,0xf6a2be5e),LL(0x0269def3,0xb68c0d82),LL(0x8eecb775,0x32d58819), + LL(0x32845ab0,0xccea6f53),LL(0x2541c834,0x792bc041),LL(0xb1336aa7,0xd726425f),LL(0x3ddd6256,0x85b1d21e), LL(0xd9b1ba0b,0xd575bfa8),LL(0xb778b77a,0xd23084e2),LL(0x44bb1010,0xd44e7399),LL(0xa91623fc,0x3d665388), + LL(0x3f11fc00,0x5cfd3a69),LL(0x8bc8eade,0x1f2b5d01),LL(0xa6b9f7ae,0x5160359b),LL(0xfa696463,0x1e2601dc), LL(0x915f6084,0x7f5ac6d2),LL(0x679176d5,0x6e387789),LL(0xae26abee,0x7fb99f4b),LL(0xaa409d22,0x4798a2fc), + LL(0x5965615c,0x582164f7),LL(0x0472cbeb,0x2c9dfb60),LL(0x2266724f,0x36eacc3f),LL(0x5fcb8868,0x253eb08c), LL(0x760c15b3,0x749a0577),LL(0x5686b036,0x71e4ce1e),LL(0xb710196a,0x47893a8f),LL(0xdf51c4e8,0xe27dfbac), + LL(0xffb3df08,0xc9536d6b),LL(0x6dde9e09,0xc95169ce),LL(0xcc085766,0xcb050de7),LL(0x0df088dc,0x92fce77e), LL(0x88781592,0x10c124ca),LL(0xc81030f3,0x6429d30b),LL(0x09e20c50,0x2e37721d),LL(0xf3e3d604,0x43e7f9ca), + LL(0x5b095e01,0xa277a87e),LL(0x83a51a95,0x968bc951),LL(0x53aff355,0x3b375d45),LL(0xe1ebac06,0xb79d7cce), LL(0x022995b4,0xd929e1a6),LL(0x28164ff7,0x228cf7f4),LL(0x5d3e4608,0x7bd12900),LL(0x2f97ebd8,0xc57ac873), + LL(0x86b383b5,0xc192342d),LL(0x706b01d5,0xe85f303f),LL(0x88cdcb89,0x19e19213),LL(0x2ce0de2a,0xe88f1943), LL(0xe453aecc,0xf6fcf8cf),LL(0x9a67b49f,0x0dcd10b8),LL(0xafece961,0xb93d5b4d),LL(0xc39d0b53,0xe232f34a), + LL(0x30735384,0x1b8f6cc3),LL(0xe4f26c08,0xc35c5a82),LL(0xba98f626,0x9e0c933b),LL(0x4c70aed7,0x49868100), LL(0xb7f26c66,0x711a3aad),LL(0x7dac506b,0x786ea28d),LL(0x43935657,0xd3a7ab1e),LL(0xd1b69e9e,0xda7f5c1f), + LL(0x0e6c8579,0xc08c85e5),LL(0x8d991759,0x29d04ad4),LL(0x3a8ccd69,0xbae8f163),LL(0x1790a49c,0xade66539), LL(0x45915cc1,0xf9f5bc8c),LL(0x4f2b18c3,0x63461cf0),LL(0xd236e848,0xceb75a9c),LL(0x847ce6c2,0xac653e3b), + LL(0xdb088764,0xb93b3032),LL(0xa78e5943,0x567fe1c3),LL(0xe359cb34,0xba7a7acf),LL(0xe2c3827e,0x38f4fbfd), LL(0xc90abad2,0x761c36d4),LL(0x75027c02,0xac1af4e7),LL(0xd4715572,0x95e6d01c),LL(0xd621145d,0x5b06cf39), + LL(0x64ca2efc,0x799acd7c),LL(0x4e0bcb6c,0x3397a15b),LL(0x0358a26c,0xb9b10ced),LL(0x4b8ddfaa,0x0a30dbbe), LL(0xe20f6fac,0xa70e9712),LL(0xd11451b0,0x87c7f732),LL(0xd5eece8f,0xf0c967b1),LL(0xab370e2d,0xbc62882a), + LL(0x59ddb7cb,0x134fb08e),LL(0x3ae8f816,0xe937c663),LL(0x802ed184,0x083f73a7),LL(0x8cd69f8d,0xd4badd85), LL(0x987f389e,0x2d8bfaf5),LL(0x4454b1f2,0x5338c056),LL(0x2f104468,0xdce38439),LL(0x83c5278b,0xffd94d27), + LL(0x5628ad08,0x8740af50),LL(0x8b1284e8,0x30a233db),LL(0x57acc8cd,0xb3982d73),LL(0x37c5ff03,0x211d53d3), LL(0xf6578d40,0xb6371f1b),LL(0xa80dec53,0x7f749bea),LL(0xa9816ec0,0xe6b3f730),LL(0xf5423ec6,0xd26832fd), + LL(0x63e27b64,0x80127368),LL(0xd2d21879,0x17b7a4b2),LL(0x43cf40d4,0x7dcced37),LL(0x97cf7c4c,0x999bbb80), LL(0x6bafa0b0,0x191c84e5),LL(0x917f6b17,0x1d08c049),LL(0xf4715c99,0x02e5fe53),LL(0x0658f1de,0xa92c6085), + LL(0x16a010bc,0xe9c0ba85),LL(0xea4f3e8e,0x2fd90fba),LL(0x4570a1e5,0x8af18371),LL(0x7cca9004,0xe869e8f7), LL(0x2dd83019,0xe2c8afb7),LL(0xfd99b386,0xb877995d),LL(0xf5adab87,0x1e3efc16),LL(0xaa3b191a,0x93105fe4), + LL(0xae504c31,0x21690dca),LL(0x698f629d,0x2d51ead4),LL(0x724c9cbf,0x2af3eef1),LL(0x81a0d4aa,0xa6181e60), LL(0xa94f6b05,0x580982c7),LL(0x48653ad7,0xe8bea903),LL(0xa608598d,0x0270614c),LL(0x3d0d5360,0xa7cae0f0), + LL(0x96067f64,0x81407687),LL(0x77a62d7d,0xab2c2706),LL(0xae19786b,0xbe9c1edf),LL(0x887814eb,0xa313f2b2), LL(0x08fd3c04,0xe2bc4c1f),LL(0xe5a9d032,0x25387129),LL(0x8fbc5030,0x7b3ced22),LL(0xadbf1bdc,0xc22bea3b), + LL(0x7b1308da,0x4f6b6b6d),LL(0xd0e33069,0x0f2faaaf),LL(0x0d3677c4,0xb461990f),LL(0x0e6a876b,0x55c9df43), LL(0x316d252a,0x5ce9aaa4),LL(0x0e8e5097,0x7d98a813),LL(0x9aa3343e,0x047ecd13),LL(0x939277e1,0x15cc7072), + LL(0x0a020be7,0x305165d1),LL(0xf66eaf8b,0x48560411),LL(0xffd2380e,0x5ff898dd),LL(0x784b4b11,0x7da35f08), LL(0x38fd05c7,0x50f53e2c),LL(0x47ada3a5,0x64b3ee82),LL(0x678995de,0x672ae316),LL(0xdfe96605,0x74707460), + LL(0x441e7150,0xb346dc71),LL(0x55fd483c,0xd9505e7a),LL(0x94302331,0xca96e59f),LL(0x801930cf,0xcfde701c), LL(0x73c31e5d,0x02fc9936),LL(0x8cda0b51,0x4ef53a55),LL(0xa269a1f3,0xa934e268),LL(0x7cca8542,0x7ba4e5e0), + LL(0xa2ae339c,0x4c6408f9),LL(0x5a70ba33,0xf9ea4cb2),LL(0x5cac2af4,0x3eaa9364),LL(0x95eaea09,0x62686d46), LL(0x3e771722,0x5196e88f),LL(0x7108b198,0x749518e8),LL(0x29b25254,0x394107c4),LL(0x3a315aad,0xf9945ac1), + LL(0xaab9dbe5,0xce15c84d),LL(0x3940eb15,0xebb54d52),LL(0xa2fdd11d,0x69b649c7),LL(0x3f6ade80,0x4e2d1782), LL(0x2327f7d8,0x0f53ac9c),LL(0xc79eb564,0xf6158d6e),LL(0x4536f5c1,0x2903bfc0),LL(0xfb9e9e07,0x0a25518b), + LL(0x62a0b0ed,0x70cbce8b),LL(0x0abbc9be,0x92f5dc33),LL(0xf369c2d6,0xbb92b7d3),LL(0x79ef83e1,0x70dd90c8), LL(0x7937ab45,0xe0b33153),LL(0xc054af6d,0x3a8d1f74),LL(0xb05ebfc4,0x35cf7380),LL(0x58c2cd0c,0xefb8dac2), + LL(0x7d665d26,0xe7316f99),LL(0x800fba6f,0x59a7ead9),LL(0x08a2cb88,0xfa4d2a2a),LL(0xb441995d,0x2e7d3bab), LL(0x93046f2b,0x390988c9),LL(0x08869cf5,0xfd95b86e),LL(0x9a76537b,0x0185b6be),LL(0xb6cd3d59,0xa89563bd), + LL(0xecb1ad25,0xe79a4f63),LL(0x6948504d,0x1857cec7),LL(0xa497922f,0x03b7b3ad),LL(0x38930f36,0x9df2f2e4), LL(0x4bb5927c,0x355e4a7a),LL(0x636ec349,0x5ad3fd47),LL(0xc41b19ca,0x5400730d),LL(0x555afa93,0xbfeabac1), + LL(0x6cca58b7,0xb6232083),LL(0x76d0c53e,0x55faae6b),LL(0x40a8eb5a,0x64ef60e2),LL(0xe8f22c94,0xc68bc678), LL(0x10a0416e,0x5156dc1c),LL(0x5c2037e4,0xac779644),LL(0xc7162aaa,0xd2e30925),LL(0x2cf21e2f,0x7bb5275f), + LL(0x0c11e65a,0x7722cb40),LL(0x68ff2be5,0xc94a7f52),LL(0x8d9f9352,0x420085cc),LL(0xca4b2544,0x4addb986), LL(0x06264a47,0x3c6ceac0),LL(0xe2b48ccc,0xebc01a03),LL(0xea94fef2,0xc430e7ab),LL(0xbd94aa8a,0x973bb6f0), + LL(0x3225b585,0xd60e5feb),LL(0x01b56ad1,0x6cbab39c),LL(0x37d6d1b4,0xcb55a9cc),LL(0xfbce1d89,0xd7288c1e), LL(0x162d4b46,0xcb516843),LL(0x15edb910,0xf0aca3a6),LL(0x08a6685a,0xdb998b55),LL(0x07811873,0x16b442e6), + LL(0xa1a7e0c2,0xa9badd09),LL(0x9f813289,0x0a9a339b),LL(0xd4cda45b,0xabf1793f),LL(0xc7378a84,0xa9830a12), LL(0xd28165b1,0x1ae11c32),LL(0xf71bca14,0xbfd49ace),LL(0xfc035476,0x9a3990df),LL(0x6c32b72a,0x0fd2b153), + LL(0x3541b5ae,0xceece835),LL(0x8256c750,0x2f7429f5),LL(0x88104f8c,0x456c3478),LL(0x8b23da06,0x8a435588), LL(0xd817ce6a,0x6b6c14f2),LL(0xf35ab86a,0x83bf0acb),LL(0x364b83fd,0xdadb89ba),LL(0x5cfecaf3,0x2c8fcf90), + LL(0x20d12c92,0xa90f77ca),LL(0x69d1739c,0x2e278e0e),LL(0x5c1f9e82,0x29d24b44),LL(0x647c59b1,0xbf4fb4cb), LL(0x90ffd733,0x9c8ea39d),LL(0xf14db3fc,0xe37a1352),LL(0x8f3e1dca,0x3c9164a2),LL(0xaec86440,0x515c16f2), + LL(0x5c483906,0x736fee4c),LL(0xa3f651c7,0x2325cabb),LL(0x35b94e45,0x582324df),LL(0x45598c64,0xeacedb3a), LL(0xde9ea8cd,0x674e1740),LL(0x89d2b975,0x30f2f423),LL(0x9c8abe45,0x330bd76d),LL(0x5371c0c4,0xb97e89f6), + LL(0xb7569543,0xb1769248),LL(0xd85f4d72,0xd29cc9d2),LL(0x10a5b6dd,0x89e1fd0c),LL(0xa693a796,0x501be0ae), LL(0xe490e600,0xc70965b6),LL(0x1bb6c5cd,0xf518f8af),LL(0x76f6daa2,0xf51d40bb),LL(0x5ec7849c,0x83a83b67), + LL(0x6d8aa314,0x0fe0d975),LL(0xea664a8c,0x9bf9aed5),LL(0x96fad9aa,0xef8bb989),LL(0x04a0e441,0xd07dce35), LL(0xb3c5eb81,0x53bd2a16),LL(0xaf178b66,0x49e29fe2),LL(0x24dced32,0x62cf7a62),LL(0x0f541e36,0xcc111fba), + LL(0xda9dd111,0xc93cd7c1),LL(0x28c9c1b4,0x56b625ab),LL(0x3769f3a2,0xeff436ae),LL(0xcbd31a17,0xa0d8d46b), LL(0xc80dc873,0x241693fa),LL(0x3cd579ab,0x56083f64),LL(0x33fbd431,0x12ee753b),LL(0xd66c283a,0x1bde60ad), + LL(0x0243cd83,0x0db508dd),LL(0x1349307c,0x3b12c134),LL(0x61d86bda,0x8296aa6d),LL(0x630adc96,0x1d5c8a4f), LL(0xa30a8ae6,0x9d01dc28),LL(0x1dab8168,0xc555a743),LL(0x7abe577a,0x61fe0d14),LL(0xc8c93bb7,0xe26aa4d8), + LL(0xda2bab5b,0xfb4b03bf),LL(0x79b4e6c0,0xfbd49089),LL(0x86806aa4,0xda1a0108),LL(0xdc078112,0x281f76ae), LL(0xe0fbd693,0x9f662594),LL(0x49ec4ee0,0x1da897b0),LL(0xfc7d1578,0x20d52a97),LL(0x6b1f4ab4,0xdbf8d157), + LL(0x3b97d1e3,0xfc0a5936),LL(0x1aa091b6,0x00f0f283),LL(0x13aadeb0,0x505e183e),LL(0xa55b3f8a,0xe28041ad), LL(0x086c2d23,0x2e0f76da),LL(0xf2c5eceb,0x815b147d),LL(0x673ba5f2,0x02066c02),LL(0xce043d4d,0xb85d6a8a), + LL(0x113890f6,0xd5f023a3),LL(0xa9d2491b,0xaa4f9058),LL(0x16d175a3,0x6d82393e),LL(0x671e2aed,0x1d1e00b2), LL(0x40018bae,0xd47c4f28),LL(0x7b30838f,0xd08eac83),LL(0x5dfe910d,0xa0fde631),LL(0x5c66d5c6,0xfc16adf7), + LL(0x18d8c6b1,0x0ed2a8a2),LL(0x632b5b07,0x67ee6037),LL(0x21a89b8d,0x7eed42e5),LL(0x33e6da02,0xd99942cf), LL(0x39971405,0x759ec79e),LL(0x174dca4c,0x669a92c7),LL(0x9d1e7c55,0x85935ed7),LL(0xa82055c0,0x5f3f9e68), + LL(0x56aa5af3,0xab0507c8),LL(0x1bd2726f,0x354cac5d),LL(0xb864816f,0x46e85e16),LL(0xd1840add,0xef2548f6), LL(0xc3842f43,0xe494ea07),LL(0xedf6c13a,0xa6169c4a),LL(0xa460e30b,0x65d9cca3),LL(0x31e7dfc3,0xa6848e4f), + LL(0x5c8109dd,0x4309f315),LL(0xc5799833,0x7a4ec14e),LL(0xa8132b78,0xcb768a63),LL(0xb416c77c,0x229106d1), LL(0xded02f41,0x1ca71df6),LL(0xc1a1fc66,0xb6365d3e),LL(0x1431d1fa,0xf7c432a1),LL(0xa5654387,0x30364500), + LL(0xd5b13b2e,0xc9ed0cf8),LL(0xd18d5a28,0xdbd541bb),LL(0x754de9d2,0x6b78c887),LL(0x54651568,0x7d32fedb), LL(0x0d37c339,0x7f319680),LL(0x37d70b76,0x22304d1f),LL(0x6fb5e555,0x01b2709e),LL(0xfd5d1708,0x978b0d3e), + LL(0x96bc118d,0x83206b9d),LL(0xec7bfc1c,0xb1a4d7bf),LL(0xb6b41502,0x753f98a6),LL(0x4c5187ce,0x41139110), LL(0x587a8213,0x56e9e218),LL(0xad9aefd0,0x3b39955b),LL(0xb9947ceb,0x7428b03f),LL(0xbe8bda29,0xbbe82668), + LL(0x5c4b4c63,0x5142e8ba),LL(0xe92ee291,0x90c3e2e3),LL(0x8f6a076d,0x6947a55a),LL(0x61964435,0x9acdeec1), LL(0x181dac35,0x56bc8e4c),LL(0x7a824372,0x4f4f2c0a),LL(0xc1033f6b,0xd1958b99),LL(0xc83ecf98,0xeeaa6604), + LL(0xaca52cb3,0xe43c0b44),LL(0x75443f14,0x12446426),LL(0xddcc00b4,0x0d14e885),LL(0x6cfe5734,0xb0f5f11d), LL(0x1013afcb,0x0e160164),LL(0xed9f4535,0x4f570ca9),LL(0x73a307ad,0xe5162a12),LL(0x3321ae54,0x6a431695), + LL(0x5ae301b4,0xa6c7b0c5),LL(0xbd2d3f1d,0x6f5d42b1),LL(0x15c0c94b,0x4eb12c09),LL(0x28618c41,0xf1c40386), LL(0xc0f55c25,0x30302333),LL(0xbd1c19f0,0xa5e41426),LL(0xcfcc66f8,0xd5d4d4d7),LL(0x449253c5,0xcfdf3039), + LL(0xb30ec0ff,0x17b0eb72),LL(0x5e6424f9,0xbce593e2),LL(0x2a73184e,0xa5d82937),LL(0xebe58773,0x23d2857a), LL(0x067e1eac,0xe3f0f676),LL(0x50509d7f,0x073ded2d),LL(0xca405a7e,0xc22af8f0),LL(0x6df6a46c,0x7a4ef592), + LL(0x97067006,0xf9cb0178),LL(0x489d2a39,0x9ae132af),LL(0x6a2da1c1,0xc7c46b35),LL(0xd95850c9,0x0993353b), LL(0xa25d52ef,0x6c313a57),LL(0x93c852c3,0xa6bdb2b2),LL(0x7e9e296d,0x27ed916b),LL(0xc7aeb09b,0x10b58337), + LL(0xecebe36e,0x78800c35),LL(0x2234ce8a,0xd93e2423),LL(0xfa95019f,0xe4cf5cee),LL(0x71e13748,0x21396d37), LL(0x0c32fdad,0xeb028350),LL(0x61f1652b,0x31645697),LL(0xf6677491,0x9e1c6e0b),LL(0x74176c12,0x4d18f2e5), + LL(0x3832d713,0x78d559bf),LL(0xb6e00e15,0x04f0b57b),LL(0xe80add3a,0xd6c9cb16),LL(0x5c7b1d70,0xeabfabc5), LL(0x98a62cc3,0x40570866),LL(0x4abb2b1a,0x39ef8ff1),LL(0x0c19959c,0xadb40548),LL(0x388b1f7c,0xd61632d7), + LL(0xd73b7d50,0xd1f9b736),LL(0x560bf0aa,0x652ed78e),LL(0x50e3fc4f,0x58e71e33),LL(0x55df1ad1,0xbfaf5f44), LL(0x9106744f,0xefe8893b),LL(0x356d1fe9,0xabfbd51e),LL(0x9eb1cbaf,0xab03570b),LL(0x3919012c,0x92cfe2e4), + LL(0xb6f7c64d,0x7671e5fb),LL(0x6e0a44b7,0xf040c039),LL(0x62b36088,0xf430f593),LL(0x94c7c0ac,0xa85b4bc9), LL(0x16b54fff,0x07d5c40c),LL(0xc53a3788,0x47aa73ee),LL(0x7000d61e,0xa63c5b36),LL(0x91b9f09f,0x04e8f53d), + LL(0x87dc6a3d,0x7e48021d),LL(0x28ae0f74,0xa2b5516b),LL(0x705132e2,0x84412003),LL(0xe243d1fa,0xc55f69cf), LL(0x6a8f80bd,0x758c0f71),LL(0xd09b719d,0x69ecf887),LL(0xa9b45194,0x51b100f0),LL(0x90ae0769,0x1fb9ef66), + LL(0x30fcdfd2,0xfee82fab),LL(0x36a6990b,0xf36185be),LL(0x3d33027b,0x88f343f6),LL(0x38ae16c6,0xb775dcbb), LL(0x85a82e45,0xa107b9f0),LL(0xde6b9806,0xaff8b0ae),LL(0x0392fad0,0x3cd3980f),LL(0xf3cf7650,0xdd829fc6), + LL(0x0dc8d031,0x177190cc),LL(0x7fc491eb,0x3e21cd25),LL(0x0d929039,0xea0cc90e),LL(0x1dfc37b3,0x5f7e6292), LL(0xe23bdd04,0x66dd6dde),LL(0x64fa490a,0x70e7a317),LL(0x10a03dd8,0x59c90f81),LL(0x96d58314,0x425ee6ce), + LL(0x5f896ed1,0x868001eb),LL(0x91dad4fd,0xc4c003f5),LL(0xd9ef80b4,0xfb4782b2),LL(0x323e4fc5,0xb9edb975), LL(0x53ef4ccc,0xa2ec9b6c),LL(0xa77922b6,0x4af8b2ca),LL(0x6697874b,0x73850e89),LL(0x3568523f,0x76e0fd72), + LL(0xe9c400a6,0x64799f46),LL(0xa9c245de,0x6c5176e7),LL(0x93503700,0xbd97c80c),LL(0xffbe539f,0xa92d9ee5), LL(0x8376bb3b,0x76003d14),LL(0xac564679,0x2e75cc77),LL(0x3a333970,0x126af6c7),LL(0x6b6604bd,0xdbfd0133), + LL(0x24424a48,0x11cf4c2e),LL(0x37d4471c,0x843c73ee),LL(0x617a488b,0xb3047fc5),LL(0xe3cf861c,0xf2a91709), LL(0x1c3a60f7,0x84444421),LL(0x26679148,0x74787a36),LL(0x53d9404b,0x115fbd06),LL(0x6244cef0,0x70fd3365), +}, +/* digit=23 base_pwr=2^161 */ +{ + LL(0x2b574b7f,0x76695c9b),LL(0xc369b6be,0xcca80405),LL(0xe3108ded,0x1f4bae99),LL(0xea133fce,0x9e715ce2), LL(0x54c2ee1c,0x60d52055),LL(0x1680742e,0x56bab301),LL(0x3fe438b9,0xa409b5f6),LL(0x8036f7ce,0xe3a8e4d0), + LL(0x247fdfdf,0xe1d7ec0f),LL(0x4a23d1dc,0xfb9d90e7),LL(0x190fdc41,0x7012eb2c),LL(0xddced48c,0x5c2bbff6), LL(0x68cd7feb,0x8a93426a),LL(0x6b4854e1,0xb5963962),LL(0xe772bbd8,0x8ac72b8e),LL(0xa6b3040a,0xc10d24d2), + LL(0x94d5f347,0x8fdfef16),LL(0x2b04af0a,0xf3189490),LL(0x6d2ca633,0x30e3da7a),LL(0x4803814a,0x8d002aea), LL(0x95a0bfe9,0xc15e311f),LL(0x4b4cc50c,0x2891ec7e),LL(0x8834df25,0x0936fed8),LL(0x78e00289,0x7e5d7dbf), + LL(0xfbfcf1b5,0xb9a92d78),LL(0xe8427d74,0x17ce4fab),LL(0xac66e74e,0xbae98ffd),LL(0x145bb5e5,0x6d548304), LL(0x0992abe1,0xbf3dc603),LL(0xbefdc5c5,0x318cfbda),LL(0x59f8efb8,0xbb5fa37d),LL(0x4ef5bef8,0x347874a0), + LL(0xbf68688b,0xdf552b01),LL(0x8f96a57a,0x2fc542cb),LL(0x4edb340e,0x5a731b61),LL(0x181cf578,0x5143d103), LL(0x2cc936b6,0x749ab511),LL(0x0dd355c2,0xbc94c053),LL(0xa3900fa2,0xa825eff5),LL(0xc1dc2b31,0x60a909a3), + LL(0xaf5bcab5,0x59b33c78),LL(0x496fbcdf,0x0053d789),LL(0xd7883bc1,0x5a5afe02),LL(0xfa66951d,0xec9afe78), LL(0x728e56a6,0x38f28b83),LL(0x78cafb9d,0x21d0b6ac),LL(0x7042e327,0xd43996bc),LL(0x7c31c145,0x60686637), + LL(0x3d919304,0xe1f8d2e6),LL(0x456be82a,0x09cf437c),LL(0xf0c21973,0x6a01dae8),LL(0x246d9ef8,0x8bffcda8), LL(0x5d853975,0x7e03a0d4),LL(0x32533ba3,0xc3800ca8),LL(0xf02ce43c,0xd77152cc),LL(0x6392089a,0xb8bc17a6), + LL(0x4b4558fb,0x6f5fcb61),LL(0x1f2545aa,0x9602597b),LL(0xabe5e469,0xfd89ab3f),LL(0xfb2e16bc,0xf1daeea2), LL(0x3a12940f,0xe699acd7),LL(0x4d7c7311,0x24980f6c),LL(0x336c8ec6,0x4a5cf975),LL(0x8c27d3dc,0x8e180e32), + LL(0xd36cb503,0xafb66269),LL(0x754fdd67,0xe98b07d2),LL(0x5a1fe9bf,0x1e0b425b),LL(0x24fc4a85,0xb4ac13e9), LL(0xc05a9c3f,0xef693781),LL(0x5c0124dc,0x266c1216),LL(0x64ee22e2,0x7f3184c4),LL(0xcdb5f1a9,0x3f985fb3), + LL(0xfc01efaa,0xb258cd5f),LL(0x0775588e,0x861688b1),LL(0xfa46eae0,0x72184b18),LL(0x5003404a,0xd17c9dea), LL(0x92e7bf9e,0xa8791966),LL(0x7891ac50,0x049c63cb),LL(0x5d46b33d,0x2ed32928),LL(0x0623595a,0x49d1bfbf), + LL(0x36c8e3e9,0x9f871470),LL(0xb20d610d,0xdec7eb98),LL(0x7b151f4e,0x15b9326f),LL(0x04005d02,0xa624c23e), LL(0xd9cacded,0x89fc2a8e),LL(0x9a2c3a00,0x9eb8defa),LL(0xe8d7eab7,0x7c5dc2d6),LL(0xeb0a77cf,0x48fa5403), + LL(0xbf033733,0xcc4c31d0),LL(0xef211c17,0xf37d0072),LL(0xae35b246,0x8967fe49),LL(0x5cb1aa9b,0x8c4cbd66), LL(0x04840da3,0xab0097db),LL(0x5828733e,0x3946faec),LL(0x87d64045,0x96c6531e),LL(0x83bc0d0e,0x893d3780), + LL(0x53bec0dc,0xf833e355),LL(0x2803a655,0xc9ff7280),LL(0x42b99b53,0x300ff7aa),LL(0x6a7c3f2c,0x3b48a8db), LL(0xf617f8aa,0xf78c21d9),LL(0xcbe4d565,0x23684cb7),LL(0x7514e9a0,0xf64ae9c8),LL(0x8429d8ba,0x4ff5483c), + LL(0x5cb18391,0xdedab351),LL(0x769ae948,0xd3126ffc),LL(0xd3546ad9,0x6c2f9ba8),LL(0x69aabfb7,0x4567e48a), LL(0xaa284747,0x6fbe29b0),LL(0x98af4f2f,0x3185f0db),LL(0x5b4c14e3,0xf2a958a2),LL(0x27d04855,0x106150c5), + LL(0x68a19ca9,0x60a3b4fb),LL(0xfac47c70,0x65c5719a),LL(0x973e4cfd,0xe228e088),LL(0xcb63c89f,0x122a2429), LL(0xbaea08f7,0x39fda97e),LL(0x621c12cb,0xe7da5324),LL(0xff9b3c84,0x569c8a51),LL(0x4c3b8d54,0x5ab8bb6d), + LL(0x00e25a95,0x4f02ece4),LL(0x7ac1732e,0xef947402),LL(0x51149260,0xecdb65ac),LL(0xa9180d51,0x6043aa29), LL(0x852deca0,0x07fc92bd),LL(0x15237c8d,0xf3338297),LL(0xe84b3f38,0xecfb0e76),LL(0x6b89af17,0x21f2f5c5), + LL(0x9659963f,0xf7aec268),LL(0xa0cb213c,0x67fb5260),LL(0x66d931b7,0x5daa0fef),LL(0x34d309ff,0x95457a7e), LL(0xc21285b6,0xe7cf1a56),LL(0x244e11b4,0xcbff9b08),LL(0xc0ecce3d,0xd79ee62d),LL(0x8267c254,0xe3f20739), + LL(0x037ef2d3,0xee06dd39),LL(0xd522f762,0x790d1b0f),LL(0xf30c47d0,0xf0659106),LL(0xb5fdc6b5,0xcd83214b), LL(0x6593b717,0xc8621660),LL(0xfe3fa381,0xb10a6d99),LL(0xab254244,0xa5c3224c),LL(0x5854b18e,0xd15287e6), + LL(0x225806ae,0x6bf9594c),LL(0x57e554f2,0x75a97e21),LL(0x82b00b16,0x0ea199f3),LL(0x5389c90f,0xde81a726), LL(0x86922afe,0x8503609e),LL(0x254b75c3,0x6778ad88),LL(0xf3e660ba,0x6bc2ac1b),LL(0x209c04a4,0x7efc1550), + LL(0x2528ec51,0x6e90b6a5),LL(0x0548389e,0x9196a7c9),LL(0x7b5b5dde,0xf7e285c1),LL(0x223d4837,0x6335a624), LL(0x412d19c4,0x8acef5af),LL(0x9783256b,0xb22808a5),LL(0xf53e4b62,0x6ea3daaa),LL(0xfa7bada4,0x7ca4c51b), + LL(0xe4d3115e,0x3e40461e),LL(0x3646fc40,0x24889b50),LL(0xfa26ccf7,0x39e0eb1e),LL(0xa82af350,0xfcad5d47), LL(0x4862b1fd,0x90037503),LL(0x1a79283c,0x88e937e8),LL(0x9a0127fb,0x16dd07c0),LL(0x39fca31a,0xac62a168), + LL(0xa294dac9,0x26542e2a),LL(0x2a5dcfe8,0xefab45af),LL(0xe642bbe8,0x6166857d),LL(0xff6290a8,0x3f3ad480), LL(0x5f50633f,0x435d4c2b),LL(0x84451c8b,0x36da60a7),LL(0x261612e4,0x00f5e2e4),LL(0x2d04786a,0xe4318273), + LL(0x2c175edb,0x192bcda5),LL(0x59a6f637,0x74681e0a),LL(0x2d244985,0x696df08b),LL(0xfcf577c6,0xde61a87c), LL(0xf2c9de81,0xcbd2ceab),LL(0xd36162e8,0x878f06ce),LL(0xb3d22955,0xc4f312e0),LL(0xe903efee,0x736ed43f), + LL(0xca9bf60f,0x2c687134),LL(0xbc7da3a5,0x2473ea8f),LL(0xb45fb57e,0xf54ef685),LL(0x3383cadb,0x594e8445), LL(0x4a7df4bb,0xe1edd3fb),LL(0xc17c2c92,0xa783d987),LL(0xcf8fcba8,0x0d498637),LL(0x3acd6e4c,0xdebd801d), + LL(0x34d3761e,0x2ade8a7c),LL(0xd825cd19,0xc591c889),LL(0x39b42dec,0x3ffd60ba),LL(0xfd9674dc,0x136d4902), LL(0xda4842c4,0x373a70f8),LL(0x3f078bfd,0x3208c485),LL(0xef608639,0x3587f871),LL(0xf04e46ed,0xf990ab0f), + LL(0xa83a8450,0x39d542ab),LL(0xdacb7c65,0x634b9198),LL(0x82486a05,0x680cef78),LL(0x16eaf88b,0xab1d4d77), LL(0x699c7aa5,0x5e605279),LL(0x3c40a07f,0x7e37906f),LL(0xfb6926e3,0x4ae84ad8),LL(0xe2ebc73b,0x236b5f07), + LL(0x9e0939a5,0xa94e50ab),LL(0x2d9e10e2,0xabeed810),LL(0x4e6423d3,0xea8190fb),LL(0x17acb62c,0xc739d209), LL(0x6fdbe8dc,0xae38106e),LL(0x63204138,0x1c6532d7),LL(0xbb7d0510,0x03af879d),LL(0x8cd2b1a4,0x1d76faf0), + LL(0xd77386cc,0x2fcdaf9b),LL(0xe32d2633,0x30f9f5a4),LL(0x382e7298,0xa4fc8189),LL(0x588af205,0x946923a1), LL(0x114f2beb,0x2c527a79),LL(0x077762eb,0xa2ca55d3),LL(0xcc85e41e,0xe4b2eb7c),LL(0x89346ada,0x4b5938d2), + LL(0x4c2084cf,0x8e94e741),LL(0xa839ecb4,0x4ef32d29),LL(0x802f0897,0xc5371755),LL(0xc49ae8a1,0xb0274ff1), LL(0x417bff62,0xf7716d1c),LL(0x918f9555,0x6efb0748),LL(0x7aeb1e8d,0x7d3bb9c8),LL(0x20d51e18,0xee9bd5e1), + LL(0xd52033b1,0xfaf0a1a5),LL(0xb8626432,0x7967d3f4),LL(0x5574dc0e,0xe837ca4b),LL(0x2c11d8ff,0xf7eae237), LL(0x87dc4007,0xc0f2f1fa),LL(0x8dfb51f7,0xf5f1f153),LL(0x5bd9ac7f,0xa64b10ae),LL(0xa2198841,0xb3c2ba37), + LL(0x66c1ee7b,0x5a7ebac5),LL(0xdba62ea8,0x59e06f4c),LL(0x30944ef3,0xa2ea165e),LL(0x3e21385b,0xfd5c7dfa), LL(0xe3bb110d,0x4a012c73),LL(0x4fb2fdf3,0x16d85219),LL(0x7cad0594,0x1aac7f11),LL(0x4b098d9f,0xea7f7dbf), + LL(0x7fd181e7,0x88abaa5c),LL(0xca3ad1eb,0x136a0c9f),LL(0xf394aab5,0xe6e5e6c2),LL(0x9349e4a5,0x84d697d4), LL(0xf76f4b3b,0x8215578b),LL(0x12feeb5f,0x81a1cec6),LL(0x3e876bc3,0x5d336eb7),LL(0x071892ca,0xe8afdcb5), + LL(0x3da8d94c,0x22f16f6b),LL(0x2d150069,0x28b276c5),LL(0x643d3e58,0x49d20441),LL(0x3da3a7fb,0x3450c84a), LL(0x442ca3e3,0x8f5bf388),LL(0x9e615382,0xca31411c),LL(0x7798675f,0xbe63e34d),LL(0xd1ea01e1,0x551eb64d), + LL(0x34a00e27,0x1738a83b),LL(0xbf58ce70,0xe7591d15),LL(0x57d806d8,0xde2ace5a),LL(0xd0338020,0xe89e8110), LL(0x4e25756c,0x935ed5de),LL(0x46d0f00b,0x07ef8c2f),LL(0xa659592a,0xa28e5fb4),LL(0x7fa4986a,0xcb45c622), + LL(0x74de493c,0x6b7df066),LL(0x79aa5258,0x4d6bdaef),LL(0xe2b255ed,0xe9709c34),LL(0x7d0e7443,0xdba2653a), LL(0xa00eb3e4,0xeb8da5c8),LL(0x7ab0e45c,0xe978228e),LL(0x9d551a51,0x3a31bafd),LL(0x403352f5,0x1de0e9cf), + LL(0x23ddd51c,0xb94d5478),LL(0x130e78e3,0x7c215c91),LL(0xed547bce,0x556b92e0),LL(0x909f5c6f,0x0072da6b), LL(0xf0dc846b,0x4ec71b11),LL(0xbf7baaa1,0xd0f3b3b4),LL(0x47770705,0x896391c5),LL(0x66732587,0x41fe5736), + LL(0x4acd3c51,0x02a7e3e3),LL(0xd30407b3,0x217df736),LL(0xe47c33cb,0x503a31ae),LL(0x4912bbb0,0xe3186392), LL(0x75a5df9a,0x2491a08a),LL(0xc09294ad,0x2882f937),LL(0x979ad9f9,0xe2576b69),LL(0x26dc1ffc,0xf44ddc15), + LL(0x968268ae,0x7dad21d4),LL(0xbe9c6fc0,0x07378e90),LL(0x2b329579,0x9406a872),LL(0x761f10ae,0xb27b5c51), LL(0xd04cf60b,0xf5dad2f9),LL(0xdf950997,0x3154dff5),LL(0xd8534a9a,0xaaec9d30),LL(0xac43f212,0x4ac722f5), + LL(0x46464c70,0x722882f4),LL(0x6c3c702e,0x9b9b5226),LL(0x8325964e,0x4e3974bb),LL(0xaa0c5227,0xd3ceff9d), LL(0x9534dba5,0xd530c8f9),LL(0xbc751878,0xd26e547b),LL(0xea79b19a,0x184a3527),LL(0x74f1cdc4,0x8dab9214), + LL(0xc051e9f6,0x708abc8c),LL(0x4be2d9ca,0x75194e9f),LL(0xd6ab5348,0x031d69c1),LL(0x78b0e490,0x1785990e), LL(0xf6c41f8e,0xd825f125),LL(0x0fbf2fe6,0x429924ea),LL(0xfb87161e,0x53c044be),LL(0x0651d153,0xa3bbdf1b), + LL(0xec6ecb9c,0xda660697),LL(0xddb8c619,0x51b4a5fd),LL(0x230fbffb,0x80b87520),LL(0x8848da9d,0xa0587430), LL(0x864c2502,0x98715939),LL(0xaf973396,0x2b10cbfb),LL(0x09572b5f,0x28675184),LL(0x39adf777,0x0a40cdef), + LL(0x3ead6eef,0x2efa3bb4),LL(0xd1b9fe65,0xbd76b425),LL(0x5e527201,0x95f006cd),LL(0x38a7dc3f,0x00890f3b), LL(0x3a7ce6be,0x84ffa014),LL(0x89541c2e,0x3406aaa0),LL(0x9559d989,0x430542b6),LL(0xb53bddd8,0x9b427b08), + LL(0x49639170,0x2182bd91),LL(0x3299ae83,0xb9fb2b42),LL(0x423b7ea2,0xbc993d59),LL(0xc110039e,0x03e416ac), LL(0x3ffe24aa,0x90c2269a),LL(0x1c322c49,0x421ea02d),LL(0x0ef8fa01,0x40677b1c),LL(0xc59407d4,0xa1acd239), + LL(0x8f14decc,0xb8cd4f40),LL(0x69e16a6b,0x95e90d87),LL(0xc3c38fd3,0x85dcf163),LL(0x0c01f90a,0xf4fb87ba), LL(0xdcd0f994,0x8274f825),LL(0x2e6bf7d8,0x4c685fa5),LL(0x3d928011,0xc87d8847),LL(0xf9efa96a,0x9add0105), + LL(0x50db3113,0xed39152b),LL(0xb794e6b4,0x6b523496),LL(0x84630b17,0x6bb241b6),LL(0x1de3ae08,0x6e9f8ae0), LL(0xd94ce4fe,0x97bd7c09),LL(0x9e61057a,0xe887b02c),LL(0xc62c27fa,0x853e8feb),LL(0x01600ed6,0x3f9d951a), + LL(0xb57b9742,0x3e957b36),LL(0x82b72110,0x92bfd61e),LL(0xfdce7ec4,0x108b450b),LL(0xcc29c494,0xd8af107a), LL(0x47688c92,0x8d67ff70),LL(0x28b9b681,0x57f42933),LL(0xaaf8a48d,0xbbc98ef3),LL(0xe2d549b6,0x14113b1a), + LL(0x0b412b3c,0x1172b259),LL(0x1d42a63e,0xaf86ca6f),LL(0x83660d24,0x5f893135),LL(0x5a21a441,0xe7bfe9a8), LL(0x4ee5122e,0xecd0aa5b),LL(0x5e4df46e,0xbb68654c),LL(0x5e243845,0x0c3e820b),LL(0x5c46bfa5,0x042b1895), + LL(0x894f7f16,0x791b2085),LL(0xb5c353fb,0x42eb80f2),LL(0xdf8db0d4,0x377777f7),LL(0x34c42ef2,0x023c0963), LL(0xa34cb6d0,0xba05eb5e),LL(0x55cd1242,0xffb8b01e),LL(0x87cd9f24,0xeab6ff7d),LL(0xab3c09fc,0x175e94c9), + LL(0x7075fd9d,0x6dc68140),LL(0x4b203c44,0x63851566),LL(0x871d1be7,0x3071e924),LL(0x85ee9cd9,0xe6285b56), LL(0x4bcf8edc,0x738dd629),LL(0x4ace75f5,0xf3a36813),LL(0x3cf6feb4,0x37a09e34),LL(0x2cd0c8af,0x4c2eaef7), + LL(0x16205f2a,0xd945a28b),LL(0xabadde7a,0xfe9112a7),LL(0x2bbf97c2,0x7db6c5ee),LL(0xb5b54833,0x3eb84a8f), LL(0x273007d9,0x9732a49f),LL(0xc6a2e3ef,0xe61431c0),LL(0x10a101da,0x88aa1a06),LL(0xb972cc61,0x64b94de3), + LL(0xf8402027,0xe79eb6aa),LL(0xea6e7157,0xbb1fa5e3),LL(0x4ebdbe4b,0x457f33a2),LL(0x7a61b393,0xf4e955e0), LL(0x698d37cf,0x578e2e64),LL(0x82ecbb69,0xbb139e23),LL(0xcfe8d05f,0x268d0291),LL(0x625fa854,0x7dcfef41), + LL(0x9c4da5e3,0xe21d5b8f),LL(0x10bf3df1,0xb5e22209),LL(0x437bf2c6,0xb04dd106),LL(0x1d055404,0x807c5d04), LL(0x2c06fd15,0x6e983206),LL(0xed63ea25,0x773450af),LL(0x95c8dca3,0xc2dae106),LL(0xd82229e8,0x5323f6ba), + LL(0x57c062bb,0x647fabee),LL(0xcd5210ac,0xcd6adee7),LL(0x181f674f,0x11b4df3b),LL(0xf2a92b48,0x4e23bf4e), LL(0x84a83d6f,0xeea34e2e),LL(0x9cb197e5,0xeaa09d51),LL(0x845e5008,0x7f36a278),LL(0x1581c0ab,0x41fa9b52), + LL(0x23d1206a,0x58917f67),LL(0x11062b8d,0xc04601ce),LL(0xf31f7326,0xdcc60fb6),LL(0x4b071708,0xc5aeef46), LL(0xdc6939eb,0x5364069e),LL(0x034a1052,0x44bd15a2),LL(0x62a307fe,0x8177eeb1),LL(0x1907ad16,0x451ae4e7), + LL(0x27eb3193,0x80e49544),LL(0xaf88f4c9,0xd788e57a),LL(0xd944e00a,0xf062c60f),LL(0xeb4a609f,0x504463e6), LL(0x74f13c8b,0x3593ad20),LL(0xc50bce88,0xdc7c5a35),LL(0xb657d1f9,0xa6336115),LL(0x591425ef,0x18d14e5d), + LL(0x1454f76e,0x73896725),LL(0x425c87a9,0x52772de4),LL(0xc6efb7d6,0xe59e4516),LL(0xd76bbc11,0xdddb8bf3), LL(0xc6fd2066,0x1acbebd9),LL(0x1d7082ea,0x88c3b525),LL(0x6d69cea3,0x6a3b3d62),LL(0x8d065405,0xdbf73dfa), + LL(0x4a7bd06e,0xd659c8d6),LL(0x7bd10bb0,0x67867520),LL(0x97838647,0x7c4e3be5),LL(0xc5891864,0x545c7144), LL(0xfa78d62c,0xf64e1031),LL(0xfa71692b,0x1f046593),LL(0x71310c47,0xd35a9cb7),LL(0x0ea84922,0x10911b96), + LL(0x93a9f5ac,0x5647310d),LL(0x6c05eedb,0xa6785861),LL(0x43950b68,0x2f5aa7c8),LL(0xa9d03b3a,0x57580907), LL(0x42e15fe3,0xd581049b),LL(0x916c4e88,0x55dcf9d2),LL(0x27d1a183,0x87ebfd13),LL(0xf5aaa51e,0x13aee909), + LL(0x3b9fc03e,0xa651959d),LL(0x98997a74,0x05c28772),LL(0xae2e4a65,0x73e047f4),LL(0x783aa072,0x359e6c45), LL(0x7a04b710,0x1124e9f0),LL(0x6d2053f2,0xd35094de),LL(0x2575dab0,0x0d57d976),LL(0x69171229,0x822256fc), + LL(0x3d19de1c,0xbd46937a),LL(0x6f0be84d,0x71feede4),LL(0x7c4dc4b3,0xca205366),LL(0xe3e851cb,0xfbb97d0d), LL(0x2066e9a4,0x0270b5ea),LL(0x42ae150b,0xeade87ff),LL(0x8eb1bafa,0x9a7f9e81),LL(0x0eb5f68e,0xcb374aaf), + LL(0xd5525ab2,0xa5841c9a),LL(0x03e02cd0,0x3eed9ba8),LL(0x279fca98,0x29449bca),LL(0x3f450c92,0x4990ec0f), LL(0xbecbba58,0xa241a8e3),LL(0x2eb47817,0xd0e2487c),LL(0x8300837d,0x6db7d420),LL(0x2d7f59ef,0x78872895), + LL(0x1314fc73,0x1b3d5033),LL(0xe710aded,0x2cf4cd42),LL(0x6f4026b7,0x9159bc5d),LL(0x2e62cc45,0x403f947b), LL(0x47d97843,0x18d6ac70),LL(0x0694f7eb,0x69d5faaa),LL(0x6932e0f0,0x7711535c),LL(0x6ebd1488,0xc85c9616), + LL(0xd3542212,0x558e3750),LL(0x02921066,0x21fe02d7),LL(0x46b90554,0x1636a1a2),LL(0x0108cc04,0x8acf01ed), LL(0xb4d60d37,0x57a2b16a),LL(0x91f4fdb4,0x3301a33b),LL(0x8e09b548,0x70dc3d3a),LL(0x079c0c2f,0x35ae7d07), + LL(0x978f92cc,0x95792f06),LL(0x23196752,0xb11574d3),LL(0xb8cfcac1,0xc3249711),LL(0xcf93af67,0x2061c767), LL(0x2f63dbe7,0xeff09a1b),LL(0x48091edd,0x527776b6),LL(0x19bba5a9,0xf0fa985e),LL(0x66ae3221,0xc54f89f3), +}, +/* digit=24 base_pwr=2^168 */ +{ + LL(0x6a436476,0xbc5a6284),LL(0x35dbb9cb,0x6fcc2313),LL(0x5012ffbf,0xa77d2d9f),LL(0x4ae4bd14,0xcc25e9f4), LL(0x1a5e40c6,0xd17fcfc4),LL(0xff085322,0x7d716a5f),LL(0xee3077c4,0x9dcbc50b),LL(0xdb4a6e61,0xebfe953c), + LL(0xd3d777d7,0xe7e66f2f),LL(0xcf1a6b09,0x3519dc64),LL(0xdbf88dcf,0x0df07beb),LL(0xacd4e105,0x17b09654), LL(0x4e70c783,0xcbd7acd0),LL(0x96b9d577,0xda66e747),LL(0xe3e52f8a,0x6d0488a1),LL(0x6ff71c1b,0x3ec0fd11), + LL(0xbe4f2782,0x75474cb6),LL(0x41c2c0cd,0x10ef5e6b),LL(0x6a65e29c,0x592c6b06),LL(0xd12d0608,0x4d424662), LL(0xb1a714fe,0xf5280949),LL(0x1199f802,0x52697bcc),LL(0xe6a4ff3a,0xc68ba4f8),LL(0x351849ce,0x25a5380f), + LL(0x573ec6f5,0x33207f69),LL(0x67bd2e8b,0x7ecc4bbe),LL(0x8ffe2420,0xa07acd34),LL(0xa13f9cdd,0x0a957eb8), LL(0x9ec9c0c5,0x0bc7f95b),LL(0x6a8578cd,0xd82147cc),LL(0x9e61923c,0x07a2e7c5),LL(0x32e83f25,0x591eb066), + LL(0x957c94fa,0xaaa61588),LL(0x364911fb,0x6a2bc707),LL(0xc4907b19,0x09771450),LL(0x9694ccc4,0x4cc48773), LL(0x50c878ac,0x9db6216e),LL(0x6f3031f1,0x6e89210c),LL(0xced0d41e,0xb711dcbf),LL(0x0fbf9751,0xe39bfe3e), + LL(0x764636b5,0x18fd7a45),LL(0xb75d48f3,0xe437ee86),LL(0x60a80177,0xe323bb18),LL(0xbc94c0ea,0xedc3c8f3), LL(0xec8cb0cf,0xd8351164),LL(0x2472936d,0xccdd8829),LL(0x58059756,0xa8db1b85),LL(0xd55c184a,0x4eda8cf8), + LL(0x2923b8cb,0xdfb5727d),LL(0xe6773d5e,0x6e793e5c),LL(0xa0641165,0x8ecc901b),LL(0xd6da5095,0x6077ab26), LL(0x6b127d9d,0x00669b0c),LL(0xd63e2e1f,0x8140e4e0),LL(0x9641b6a2,0x1ad5b03c),LL(0x9baed7b0,0x44299f88), + LL(0x1ea4a056,0x1736296d),LL(0xd77811ba,0x6f74702c),LL(0x432dd74b,0x5c927548),LL(0xe7a194ab,0x9cc73271), LL(0xd6328dca,0x0f035ede),LL(0x28db755e,0x5292aa39),LL(0xa0192a4a,0xb5488385),LL(0xdfc6895c,0x6e7d2fa8), + LL(0x5d8bbec9,0xfa912a0a),LL(0x0087edb3,0x7051140a),LL(0x64865e5b,0x5293672b),LL(0xc82c48d5,0x6e8448c9), LL(0xa2c437b3,0xeece41cb),LL(0x21ce1ef4,0x148967d2),LL(0x6b05c2a5,0xf14391fa),LL(0x8fed2f1f,0x15ff5fc9), + LL(0x4557b49f,0x18ae5e74),LL(0x3db266b2,0xe33760c6),LL(0xb1b249b5,0xd5d830c7),LL(0xc5fff531,0x24c665b9), LL(0xc57df7c0,0x6b304406),LL(0xc3958e89,0x59706667),LL(0x790a5483,0xbf590ff2),LL(0x5ce77aaa,0xbcaea5a5), + LL(0x80ceb559,0x8578a002),LL(0xd8d61946,0x3639aadf),LL(0xadd3bb00,0x3fd52d94),LL(0xe09a8ce3,0x16c27846), LL(0x294c7967,0x75cfd6c6),LL(0x59195034,0xfb9b7f37),LL(0xaa972a86,0xae687a99),LL(0xebd2394e,0x04bdefdb), + LL(0x2f96144d,0x8e245a19),LL(0x3b61e5ab,0xc740d348),LL(0x293ddb25,0x8703710e),LL(0x2bbf8f63,0xf4bb6ac0), LL(0xde3b5805,0x86396457),LL(0x65d29e63,0x607022db),LL(0xcc930fe3,0xad0a0cdc),LL(0x1626abf6,0xd9997ebb), + LL(0x2a510565,0x2d872d17),LL(0x0357ba07,0x3e682079),LL(0xebfaf203,0x49edd962),LL(0xf81eda20,0x3a13edfb), LL(0x7a75f2d5,0x87b5b5e1),LL(0xddfd9511,0xf04de2b8),LL(0xcfc5c5ff,0xf29a1569),LL(0x07160ed3,0xa3995532), + LL(0xcb2b061b,0xb6247469),LL(0x2f10fe1e,0xe75c5351),LL(0xd20e1bf7,0xbaf44963),LL(0x2d93babf,0x216cb6ab), LL(0xf5109e45,0x7e0b655c),LL(0x6657450d,0xdcc712fc),LL(0xd51fc733,0xe06c408e),LL(0xed9c0912,0x85b11f96), + LL(0x37365c9b,0x954cb91c),LL(0xb2f74fe7,0xe0eaa047),LL(0x15716541,0x9af74b86),LL(0xf73dc7bd,0x4da06207), LL(0xe07890a1,0xdb0d089e),LL(0x73902f91,0x5bf09681),LL(0xa897f0fe,0x14e1710c),LL(0x3605b1c2,0x191ec9a1), + LL(0x0133903b,0x271b2e2a),LL(0xe495ee32,0x5b3686f2),LL(0x0c991f28,0x89bcc974),LL(0x34f93b8a,0xadd20cce), LL(0x680b65b6,0x5f5a1768),LL(0xaad41c40,0x0c453ab8),LL(0xa7fb4269,0xd479630f),LL(0x52c4e929,0x60039d01), + LL(0xff860883,0x0d8d112c),LL(0x723c6e29,0xe1dce5c9),LL(0x191ad70e,0xc19eadae),LL(0x62ce0e64,0x4af8194d), LL(0xcc81415c,0xf207bfb0),LL(0x008495c8,0x3ab92f3b),LL(0xfdb9534b,0xe7250e17),LL(0x6c0c1d1c,0xba67e9b8), + LL(0x072c793f,0x117ae3ff),LL(0x9fb3091e,0x5243e6ea),LL(0x31a59e39,0xf93ad514),LL(0xc93c1891,0x8ce9cfb0), LL(0x1ed08b0e,0xbfcbf901),LL(0xb53d687d,0x4d13cf2a),LL(0x5d81e4ad,0x25aa82db),LL(0x63c3cb41,0xd12f01f5), + LL(0xf8d1333a,0x1e799084),LL(0x653bcd0a,0x30c96c55),LL(0x44b5195c,0x9cf130fd),LL(0x13c77763,0x4cffc531), LL(0x9430619f,0x082287f8),LL(0xb08ce0d9,0x78bb037d),LL(0x3affe8e8,0x2e69d512),LL(0xba9ec693,0xe9dbb263), + LL(0x62f132b5,0x67b66ad8),LL(0xbeb47184,0x70318d2b),LL(0xf50a0e98,0x46c429ea),LL(0xe2b3542c,0xd7e32eba), LL(0xe096b4b7,0x625c1ce9),LL(0x389fd4dd,0x09221351),LL(0xfb0ee85a,0x08dc02d2),LL(0x853cd901,0x98c0ba7d), + LL(0x0deb1d99,0x88a0cd6d),LL(0x79a6b90c,0x989e4962),LL(0x24dd89d5,0xf5d19b95),LL(0xb37cf19e,0x189e5230), LL(0xb0c5fefa,0x84a607b8),LL(0xd8c7fbd1,0xe48450c9),LL(0x46479ad7,0x178f9b56),LL(0xcbcd2ae5,0x7d6a36c6), + LL(0x71ae6516,0x95a4d51f),LL(0x566e2171,0x0363349f),LL(0xed1f2fc7,0x4d4bb4b0),LL(0xf10fa10c,0xde435aaf), LL(0xb76e3b6e,0x711258a9),LL(0x2792e0b3,0x9a640eeb),LL(0x5fab8617,0x7953ead8),LL(0xdd64702a,0xd4b6d248), + LL(0x2d672209,0x95bbe528),LL(0xb6926b8a,0xfcc53cfc),LL(0x57659f87,0x05814190),LL(0x08d25069,0x4836e93b), LL(0x6a5ad81e,0xd1eb2006),LL(0xaf0d37f8,0x4bee145a),LL(0xd31ce6cb,0xd44362ad),LL(0x936c1060,0xdc03e581), + LL(0x16fcb889,0x13cffce9),LL(0xac7e709a,0xed7e6683),LL(0x5896e541,0xb655d098),LL(0xb92a6204,0x07124356), LL(0xa8f50043,0xa2ae43c8),LL(0x68731891,0xeb39255c),LL(0x3d9c408b,0xe07be0ad),LL(0x0b4f5c3a,0x0db7904f), + LL(0x4d70bb81,0x7ddc0235),LL(0x5347797a,0xe3b323c3),LL(0x3536deee,0x3536cd9d),LL(0x001bfd25,0x579b6894), LL(0xebe2922e,0x58ad5301),LL(0x92a88d43,0xe0aa2cae),LL(0x4409e205,0x24567a3b),LL(0x2258f0cb,0x3cece61a), + LL(0x3babf4f6,0x8da5cf46),LL(0x81fff8e6,0xb37428d9),LL(0x48495d23,0xcda1ff77),LL(0x34f392ad,0x98f9208f), LL(0x5bc88514,0x931f5b37),LL(0xcb375921,0xd49971be),LL(0xb5c01fab,0x9dcd4986),LL(0xc1ab1c94,0xcc26ec02), + LL(0xb4b874d6,0x34e8087d),LL(0x9d0a3761,0x224fc277),LL(0x3f7e5159,0xacc1f258),LL(0x8966d593,0xc82d71ec), LL(0x7dcd691a,0x5b1f9f40),LL(0xba28f416,0xd8fafdae),LL(0x43b6d90f,0xe8622ae6),LL(0x9ec71d5b,0xec13fce7), + LL(0xfd2e8214,0x07b6aeb8),LL(0x4cbc297a,0x813e718e),LL(0x81fd6931,0xfac0dfab),LL(0x3c48ffd7,0xa1fe8821), LL(0x85e03c08,0xd2715c18),LL(0x977c57f0,0xb6e4418a),LL(0x73418cde,0xfaa79ea4),LL(0x171e2a89,0x6ab8c25b), + LL(0x4ec7cf05,0x2800445c),LL(0xb66c6200,0x8e74a7b0),LL(0x481db950,0x081b1177),LL(0xb89f7c02,0x526d051c), LL(0x5c29c905,0x3c830942),LL(0x44c15ce5,0xbfbd9e3e),LL(0xa29472e6,0x6055c949),LL(0xa37c4912,0xab0010c7), + LL(0x5b7d3647,0xeb8492be),LL(0x1ee31caf,0x0b4cfd7b),LL(0x4b46304b,0x81cfcde2),LL(0xc554a5bc,0x968df75d), LL(0x8d0e043c,0x7ce78806),LL(0x345ea27c,0x1e896819),LL(0x6e287603,0xe040c19c),LL(0x138e8ece,0xa581856f), + LL(0xc354a9d6,0xe49f6558),LL(0xc0cfb2d3,0xc4ad763a),LL(0x1b76b8f3,0x4be2143b),LL(0xd0ad0247,0xa8caae14), LL(0x928b0ae5,0xcfe96bd5),LL(0x7724f8e4,0xcf5051f7),LL(0xec4af64a,0x9128916f),LL(0xcb437bfb,0xc211ff4b), + LL(0xbce59c0f,0xee6e8134),LL(0xd59f7f86,0x3d068b4c),LL(0x96283457,0xafa2753c),LL(0x1aedcbf0,0x453fe33c), LL(0x483c0b1a,0x781294c8),LL(0x5c2ad1ee,0x9e6f5133),LL(0x69383e0b,0x2a77b6ce),LL(0xfa9f0142,0xcb5a83ab), + LL(0x3b0e027f,0x2318aa98),LL(0xc2c68dd5,0xdea716a3),LL(0x9f548eb3,0x3f75c46d),LL(0x96120de9,0x71642513), LL(0xdbee488e,0xf733614c),LL(0xaad077f4,0xdf940026),LL(0x94a840cb,0xeda9c098),LL(0x393be3b9,0x5108bf0b), + LL(0x39980cee,0x137c08b0),LL(0x0839112b,0x2e31bba0),LL(0xba614ea3,0x9ec73de2),LL(0xd17822c0,0xd0bca8d4), LL(0x50b7805d,0x5d9f7482),LL(0x298becf6,0x16035a80),LL(0xd7c318e7,0x46571500),LL(0xd0ee6956,0x6bd30919), + LL(0xb2e13320,0x5c0ad747),LL(0xda47666d,0xe7f7f71e),LL(0x318a8e8e,0xce322037),LL(0xe9f84dd6,0xf15232ae), LL(0x915a03b7,0xc59709c5),LL(0x9a3040b4,0x2e2000f7),LL(0x8398a5a9,0x41955f77),LL(0x7086b69e,0xa8e6620e), + LL(0x8344224b,0x63acd70e),LL(0xc3145159,0x966efefc),LL(0xf5e0f955,0x406619ec),LL(0xec6de618,0xedd0efc9), LL(0xb2580ed4,0x6fe3e34e),LL(0x4139b95e,0x9d8875b5),LL(0x8e5be187,0x85baf0c1),LL(0x09553886,0x549cefca), + LL(0xae9ef2cc,0xc965b2a2),LL(0x15afee63,0xd43079fb),LL(0x076cdb05,0x02b8794a),LL(0xa0d1a953,0xd0ae7321), LL(0x2ac5fff0,0x5a8b5281),LL(0xcdda362d,0x73437d67),LL(0x1a95ff87,0x1866b2b9),LL(0x0420b3e1,0x5ff11398), + LL(0x92284adf,0x0d43b92c),LL(0x4da4c4a7,0x81425367),LL(0xdf17641a,0xc8093c56),LL(0xb5ccd14d,0xc418f19d), LL(0x506762ed,0xaad98608),LL(0xddb2c829,0xb6f45297),LL(0xd395692a,0xd0e49176),LL(0x3b1073d3,0xc05b4e27), + LL(0xe5808e53,0xe8ca133b),LL(0x06a64b56,0x6105cd0e),LL(0x53cf6d7e,0x89a64669),LL(0x1bebfea5,0xe281ca2d), LL(0x324b25d8,0x98ee67ac),LL(0xdca154ec,0x2227631f),LL(0x4406e8ba,0xa242c5a1),LL(0x49250026,0xced39f05), + LL(0xdd77d731,0xd256dd83),LL(0x7414d0c0,0x2faa6a0e),LL(0x3b90f004,0xa2e0f928),LL(0x8719bfd4,0x019bb3ef), LL(0xe2d515c2,0x3f4f6109),LL(0xbf88d7a6,0xb50a9907),LL(0x015ac4de,0x8e5fbc2d),LL(0xe78a2117,0x96992421), + LL(0x26e53df3,0x321e6086),LL(0xf42b2508,0x07eb1d15),LL(0x0ef22bc2,0x7b552108),LL(0x00f3e571,0x9eedb828), LL(0x6f0e883c,0x556abbaf),LL(0x40473ead,0x8025770b),LL(0x6ece1cc8,0x2fdab965),LL(0x00ec1adc,0xba07cf89), + LL(0x4be5ad18,0xefec4deb),LL(0xd59fa732,0x16625be8),LL(0x6808cdf7,0xffee542e),LL(0xd7a9f29b,0x85c19ef3), LL(0x82dc1ae3,0xca4ac1f9),LL(0xca5e8f58,0xa6c726d1),LL(0x66960edd,0x0bcc3d58),LL(0x56f94ea8,0x8e8445d0), + LL(0x938e64c9,0xd4d0177b),LL(0xf9a0288f,0x8d0199f1),LL(0x14a226c0,0x9176d559),LL(0xa00aea02,0x13b373ee), LL(0x6b629fea,0xc63b2d79),LL(0xa7e0cc42,0x36df7c09),LL(0x40bdbc8e,0x4628ba4f),LL(0x0de296f2,0x7f5b0228), + LL(0x3c63d73f,0xb0598130),LL(0x0431550e,0x55e59f61),LL(0x6693eb8c,0x6f2e109d),LL(0x470b10fe,0x3602ba82), LL(0x5ec7f357,0x3acd0af4),LL(0xb071c758,0xfa7479f4),LL(0xe13652c9,0xbf47caa0),LL(0xf5f5eca9,0x6fa139bb), + LL(0x8c0e197e,0xfa149b84),LL(0x60ae7755,0xca31714c),LL(0x8ccc4241,0x934ed1af),LL(0x781a024e,0x39772769), LL(0xbe24eb34,0x9f07dfb1),LL(0x0a3dac06,0xfa8a9c60),LL(0x8e410ce7,0x08fbbe21),LL(0x396a9702,0xea55fb96), + LL(0xf18882bb,0x4422bc58),LL(0x0ddd0dd7,0x1ccb7b47),LL(0xf40ea941,0x828580a8),LL(0x0db78350,0xf9ec9728), LL(0x1612f28a,0x2823b4fd),LL(0x82b26487,0x96dc3e29),LL(0x2497420a,0x1740fdae),LL(0x322f1c6f,0x3bb39dfa), + LL(0x4cb19878,0xf32a21e6),LL(0x9277c80b,0xeac04097),LL(0x13380801,0x67178d8f),LL(0x34bf8872,0xfe5e2694), LL(0x327129d3,0x8278bad4),LL(0x941c4e5c,0xb42a3f9b),LL(0x39de36f0,0x04eefb7d),LL(0x8d967703,0xed2aab7f), + LL(0x72aa1c89,0xa3283a2c),LL(0x2a4d513e,0x1969613e),LL(0xddd5ea18,0x0d4c0347),LL(0x43cee5fe,0xbbad9ce4), LL(0x57313b49,0xe8c050a8),LL(0xff09bf31,0x3b91c3cc),LL(0x610395cb,0xe6e5ab6d),LL(0xdeb31bef,0xfc36cde0), + LL(0x5d43c8a7,0x76f806f2),LL(0x63b7c746,0x08a64cb2),LL(0x45256a2a,0xb6cdcdab),LL(0x9bebae90,0x640f67ea), LL(0xcf016971,0x682eeeb6),LL(0x50f59261,0x4d16d566),LL(0xf41db99d,0xdaca66bb),LL(0xf8f04d96,0xccdb3da0), + LL(0xcf41b164,0x7c228cae),LL(0xedbefa7c,0x40bef27f),LL(0xecb43685,0x4efdd6c2),LL(0xa834a50b,0x4d0fa367), LL(0xb87f7ec7,0x2ec9c445),LL(0x23170d0f,0xc3663ced),LL(0xc5b47b29,0x189872e4),LL(0x746d6a13,0xf8047387), + LL(0xb75ac898,0x753837d3),LL(0x91959a78,0xaee88a60),LL(0xe6f59621,0xf46b0f6e),LL(0x10d981c8,0x0e92e271), LL(0x8d578b6d,0x610d0f80),LL(0xb4d9b9de,0x962bd7bb),LL(0x84a0c394,0xbe26960d),LL(0x3b5bd996,0x142a0c75), + LL(0x0be95497,0x442bb39a),LL(0x0f33c9de,0xce5d2c60),LL(0x283dc751,0x1ce0d08c),LL(0x79b3c1a8,0x106ed588), LL(0x7f8ee4d7,0x4b2e29c6),LL(0x08bbd660,0x7d61e3bb),LL(0x1e964a3e,0x11504dc5),LL(0xc77a957a,0x31544a52), + LL(0xcd7d0dac,0x1fc4161e),LL(0x370c15c9,0x83733f27),LL(0x853758cc,0x224976de),LL(0x47c1ab78,0x1bbb7730), LL(0x19c89029,0x94a3b697),LL(0x37dfc44f,0x031432f0),LL(0xd88090cb,0xf84593ac),LL(0x65bcfee8,0x381b51bc), + LL(0x10b412b7,0x38dac75b),LL(0xc7e06d08,0x6df5c9a1),LL(0x0e08c41c,0x9c6d8068),LL(0xc3600f4f,0x1544e3c5), LL(0x9c83e0a1,0xf827a48d),LL(0x06bcb3c4,0xd8539228),LL(0x6268cf12,0x149862b3),LL(0x6ec4e354,0x4829ee56), + LL(0xb712a1f9,0x44b2c3bf),LL(0xc90852af,0xe556b78a),LL(0x906a13b6,0x50f6de2e),LL(0x568a1293,0x1744efd5), LL(0x2b5745a1,0x942ad99e),LL(0xca362913,0x0f100bd9),LL(0x91e96cde,0xd9b6ad51),LL(0x5a2f88e9,0x4aa440bc), + LL(0x57a10070,0x53c4c956),LL(0xae6e4872,0x7d1be72e),LL(0xd427eda4,0xb704009c),LL(0x5f39b7d8,0x3e0aa93f), LL(0x3153a757,0xdea1ab48),LL(0x9ee60ead,0x10a070e7),LL(0xe6c916bf,0xd6a6e92d),LL(0xbd7bb216,0x02b1e0e6), + LL(0xb49138a3,0x6efb5f1b),LL(0xe88d2db0,0x11f7a9be),LL(0x3233df5b,0x0b9a2b11),LL(0x1824fcc5,0x0688afda), LL(0x5ff97f9a,0xcf1ea2a5),LL(0x4998e602,0xe8ad7b15),LL(0xa455aad1,0xdb4ae67e),LL(0x74a27ff3,0x823ac090), + LL(0x2573443f,0x5c431060),LL(0x94258714,0x92f9f9ab),LL(0xb1283d2e,0x1548fe21),LL(0x5c5be5f9,0xf86fe50b), LL(0x520c5fc6,0xd20dfc8a),LL(0x53b5e7c5,0x6e721dd9),LL(0x8f2a8969,0x8ef7eee5),LL(0x62d07bdf,0xe894859f), + LL(0x1cfc6627,0xaf279176),LL(0x483755e9,0x94b8cff4),LL(0x0fda4bcb,0xa5916f70),LL(0x47ba65f3,0x9c5318d0), LL(0x636cd7e3,0x9e9c8e54),LL(0x54c49da3,0x5c64a261),LL(0x690e932c,0x04d7ff61),LL(0xc84b0b78,0x92a357b3), + LL(0xc6f3bd8d,0x47f6144c),LL(0x71c19265,0xdf7b1ee4),LL(0x3fd5c30f,0xa7ea37f1),LL(0x79fa08cf,0xdc2d890b), LL(0x2fd40236,0x9813bced),LL(0x432dde17,0xa8a1055f),LL(0x7772c727,0x70011f47),LL(0x2e2e802f,0x965c130a), + LL(0xf5bd4ac5,0x31a6aca7),LL(0xd825db6f,0x83995bde),LL(0xfe521833,0xcbf20325),LL(0x0278f4a0,0x8dcd25a1), LL(0x5f2293ea,0xf1e83d97),LL(0x52317ad3,0x1717876b),LL(0x14181928,0x0df62167),LL(0x2fe203ce,0x24adfd6e), + LL(0x797f25ff,0x1d264af0),LL(0xd22e3da1,0x2cb7cc17),LL(0xe0016a19,0x10c4b51a),LL(0xd82b2a86,0x5956ce8f), LL(0xa3d4780e,0xdef0fefc),LL(0x6e92b93a,0x97e693ab),LL(0x20bcc98f,0x8fa3f4fa),LL(0xf9182867,0x4fc004f1), + LL(0x93e73961,0x1a206da3),LL(0x1e7db32c,0x37d75a90),LL(0x0455b815,0xa39f0db1),LL(0xb69ee770,0x841646e0), LL(0x0939f065,0xadb0aaaa),LL(0x0b147d7a,0x5608613b),LL(0x470f6875,0x84ce1a4c),LL(0x7983050e,0x501de5fe), + LL(0xc3d8ed98,0x19915b26),LL(0x9a66a6e5,0xf451e57a),LL(0x30dab6a3,0x29843607),LL(0x3d1a1ebb,0x1710267c), LL(0xe11d88c0,0xce4ecfd4),LL(0x11ce026a,0x12fc2787),LL(0x691227de,0x9801cecd),LL(0x76ce6dae,0x517a92f3), +}, +/* digit=25 base_pwr=2^175 */ +{ + LL(0x648c48e5,0x821b0fdf),LL(0x9f45a433,0x689e6d56),LL(0x2e572855,0xa5a9dca8),LL(0x8adfb547,0xb0f07eb7), LL(0x552c8d55,0x48ecb166),LL(0xce854171,0xfe3fc268),LL(0xeeee9bc0,0x323af5eb),LL(0x41ae1c80,0x0666a2a3), + LL(0x9ff262fb,0xa06d20bc),LL(0xd075868b,0xcba032fd),LL(0x943fd973,0x70376026),LL(0xe35c5e02,0x81c57cba), LL(0xba871f1b,0x1964e700),LL(0x6b265f57,0xf03a8c04),LL(0x0b950259,0xc8ebc912),LL(0xad32ca8b,0xd2b0ee30), + LL(0x89c8e719,0xe01bf0c2),LL(0xb2f4cdb0,0xbce1e20f),LL(0xa1053ca5,0x8c38eeaf),LL(0x7cd633a5,0x8c2d85ef), LL(0x9b53cdb1,0x75695364),LL(0x447eb1a5,0x5e999741),LL(0xdbd88626,0x6d6b2d88),LL(0x21876357,0x87eaf045), + LL(0xdeec441e,0x2c88f1ff),LL(0xd01b2157,0xab52096b),LL(0x6c45cf5c,0x37eee275),LL(0x0520ecaa,0xa070d24e), LL(0x546b9fd3,0x61d15bd1),LL(0x2c96db1c,0x3276fb74),LL(0xb95b29b7,0xc5c1b041),LL(0xbd7d3254,0xe18008db), + LL(0x98dfb69a,0xd56ae44f),LL(0x609d941c,0xd5f66b0b),LL(0xb91b5245,0xca6b6d35),LL(0x7b3f98a6,0x98e3a4e3), LL(0xf358c56a,0x0715dfa6),LL(0x36a66c64,0x3b02ff21),LL(0xcb22cbd3,0x737b1401),LL(0x6b8e9624,0x9dd15f5b), + LL(0xd360d017,0x25f5a71d),LL(0x29b0ed73,0x4c0779b5),LL(0x9825a018,0xc662fedc),LL(0x61d4add0,0xeee89125), LL(0x92163d14,0x1543814d),LL(0x27370d3c,0x79f2376f),LL(0xcbe1af7a,0xf80c6963),LL(0xeb9e41f7,0xf2d521bc), + LL(0xc1805864,0xe241619f),LL(0xb2de204a,0x6f1d6166),LL(0x50e68d0b,0x13c3f912),LL(0xc4a24f5a,0x32eb021d), LL(0x0e78c588,0x3f1452f5),LL(0xc9971e98,0xa267bf19),LL(0xe801c021,0x77a231a7),LL(0xc2666e80,0xf363c9b3), + LL(0xae309a0a,0xb8eb0bf0),LL(0x375b8fbc,0xa9f52f58),LL(0x1a4993b7,0xb8e4f948),LL(0x8f73c292,0x50ce578e), LL(0x02e503d6,0x2437a4a6),LL(0xe4c68ea3,0x20cdfc50),LL(0x3616f348,0xfec5993b),LL(0xc0c420df,0x5d96b4c5), + LL(0xcca0c48a,0x6b71cdf0),LL(0xc1aea033,0x00e71b54),LL(0x69f6d107,0x247f5baa),LL(0x050c3704,0x4e3ec504), LL(0x7a438e93,0xf2b2be8a),LL(0xa331e01e,0x240b632d),LL(0x91150675,0x61e66557),LL(0x95a99a28,0x32364134), + LL(0xd3399e1e,0x5e5de136),LL(0xfe2f8b75,0xe38bab00),LL(0x3a77db29,0x736126de),LL(0xf2aa121e,0x7b0d1865), LL(0xdecf9cde,0x5545e45e),LL(0x2318be70,0x9608ebce),LL(0xfa55b0e5,0xe6596006),LL(0xbc4b6ca0,0x0c8c2f41), + LL(0x92025945,0xda1c5c7a),LL(0x5d3b0775,0xb114ba22),LL(0xcedb69a0,0x11cc6888),LL(0x0f83c772,0x4365bea8), LL(0xbda8dbe3,0x006fe80b),LL(0xc2d3d266,0x334adcb6),LL(0x1521de1c,0x8c92c084),LL(0x78d8f72c,0x57873ef9), + LL(0x3b64dcd7,0xcfb0a7d0),LL(0x558c9d55,0xf4c2f1fc),LL(0xa0fbc656,0x110c2db2),LL(0xef5b6bea,0x3cad85ca), LL(0x4e0b1230,0x7099dd0e),LL(0x098a2fcd,0xc769b937),LL(0x1e1e7407,0x9209f550),LL(0x1ba7cb47,0x1b47255d), + LL(0x2c01b596,0xd8aed0cd),LL(0x30efcda3,0x1a1a2e11),LL(0x36b1a5b5,0xf771f93b),LL(0x14fcd251,0x2ea34e3d), LL(0xfd893979,0x6895cb31),LL(0x14f556b4,0x10b1d2c9),LL(0x6430bfa8,0x835fdf7e),LL(0x24bf4ba5,0x1f4bbef5), + LL(0xd562b5f1,0xbc805aa5),LL(0x35dac658,0x7101b9da),LL(0xddc28e5a,0x5b7f211d),LL(0xd3d1cd0a,0xea89f24c), LL(0x7567c80d,0xbaaa9ef5),LL(0x9a60c5ee,0xe0d1f26d),LL(0xab36cd64,0xc88a044c),LL(0x1b125de6,0xb8e03d02), + LL(0x3a707a66,0xda0c1047),LL(0x0c692d44,0x76ddb98f),LL(0xb15b7254,0xeccae586),LL(0xe7e82423,0xeadc1b51), LL(0x7c3cb698,0xd6779ff2),LL(0xdf6e7da6,0x0e271cb4),LL(0x45900023,0xeacf34c3),LL(0x03da2ba5,0xafd017ad), + LL(0x27c7e6eb,0x49266998),LL(0x6625bc7f,0x84ffa372),LL(0x05c9cb15,0xedec9247),LL(0x8075b84e,0xcfad0b90), LL(0xbc0898d3,0x94bed316),LL(0x11f92525,0x02481eec),LL(0x0d7e59d7,0x19896e1b),LL(0xf2bb3129,0xa06adb6c), + LL(0x62a0a690,0x1539228e),LL(0x8ae02bda,0x98ac50b9),LL(0xe5cf21b9,0xaf233c85),LL(0xd6a9f599,0x943348d3), LL(0xdb363eaa,0xf5a2f2d1),LL(0x7a8ea96b,0xe917e2c5),LL(0xbf5c8293,0xc80b56c8),LL(0xcdbb5c4f,0xcfc1c24f), + LL(0xfbddf07b,0x7812dce2),LL(0x0186013a,0xd4af2f9b),LL(0x6fe8d485,0x1fadcd16),LL(0xc968f0b7,0xc3c2cd95), LL(0x778bff58,0xdbdd2ef0),LL(0x8706da34,0x67369204),LL(0xb8e70e35,0x31cf3a66),LL(0xd333801f,0x0b9e5cc5), + LL(0xf7177c4a,0x1212a811),LL(0x2d379e12,0x9e266ec3),LL(0x2e8bbbf7,0xc7382848),LL(0xa973be5f,0x3f3f1dc1), LL(0x786e727e,0x534d04de),LL(0x225497dc,0xfd7a5fbb),LL(0xb63b6320,0x3c03a7fd),LL(0x5dc76e05,0xe77f8985), + LL(0x265f8b8f,0xe8d14f32),LL(0xb90549c9,0xfeaab021),LL(0x081ccea6,0x7cd36751),LL(0x1f1e8f7a,0x7a001441), LL(0x1fdfd622,0x2e87b8a2),LL(0x8bb4f812,0xe76138ce),LL(0x71e03be4,0x9a5e8722),LL(0x153e0a5f,0x795e60f3), + LL(0xd0eb7d4c,0x11d28438),LL(0x4254a845,0x147884e1),LL(0x2a8693fb,0x6795f20f),LL(0xee01bd1a,0x5a10d535), LL(0x218c0513,0xe39982c9),LL(0x1d4e6ab5,0x6c23e5be),LL(0x0f424e7c,0x20a8c27f),LL(0x3bbb486f,0x74ae9998), + LL(0xb90ce3a1,0x3fae61be),LL(0x571c968b,0xf0f5a1e4),LL(0x7780d91b,0x6b9dded8),LL(0x7597e866,0x10f60ce2), LL(0xf1eb7d1c,0xf268ed02),LL(0x6030bf9b,0xa49b5a46),LL(0x251f8676,0xc939c4e7),LL(0xe2b9928f,0xbdfe5036), + LL(0xbccf7f43,0x5abfbcc2),LL(0x28c560af,0xb22067b6),LL(0x04c6a2da,0xecf07771),LL(0x8c4ae7dc,0xa2bf88db), LL(0x616675e8,0x172396f2),LL(0x8bfcfbc2,0x9abbb19c),LL(0xe85edd21,0x52e26c06),LL(0xa65de52f,0xfca4c4e0), + LL(0x281d58be,0x255e2d10),LL(0x3614ed6c,0x93ec2934),LL(0x6dc71abe,0x36d6cc15),LL(0xf592ae78,0xaa2ad1ef), LL(0xcc9291fb,0x39a82740),LL(0x102600d8,0x6812b80f),LL(0x50c511d5,0x64f4843c),LL(0x03445793,0x28f5795e), + LL(0x29f20b0c,0x2c566372),LL(0x9e24474c,0xb168ca7a),LL(0xabe37949,0xfadd8f80),LL(0x4cd69856,0xafa1bea2), LL(0x46713b88,0x5ce6ed80),LL(0x4b3bb84d,0xaf8b5fb3),LL(0x29d53349,0x134e5120),LL(0xcdcedefa,0x1214f4f0), + LL(0x4bb405b9,0xc346821b),LL(0xddd624d6,0x753afa86),LL(0xc7c014e3,0x15fe543c),LL(0x43d08964,0x6b3c0c5d), LL(0x745221aa,0xc77c8fb0),LL(0x152995c9,0x3e144fce),LL(0xa61b96bc,0x57089907),LL(0x5e05c1ee,0xd19a912c), + LL(0xa6ddd20b,0x7bcdc697),LL(0x2d5090f3,0xcb07e229),LL(0xf089607e,0x76afc803),LL(0x637dae27,0x9f7a88b9), LL(0x3bd20d78,0x1352d8bd),LL(0x5ea79d4c,0xede1a780),LL(0xf389e31d,0x59a8222b),LL(0x5c09f3d4,0xed066aa6), + LL(0x684529d3,0xade16197),LL(0x96a2a159,0x97bed904),LL(0x1b695d68,0xdd3da765),LL(0x02fecb9e,0xb8fa37e8), LL(0xbc0f7b99,0x1af4311c),LL(0x2a492a7e,0x600bdd46),LL(0x45dc9d16,0x6aa9cb30),LL(0xc0b93019,0xaa494815), + LL(0xba052dd8,0x1211834b),LL(0x86822bf1,0xcdc0208e),LL(0x8c8362a0,0x515eebd4),LL(0x9b90cf96,0x9ea7b9f5), LL(0x3a0a5a48,0x8418fe34),LL(0x331a2db1,0x654d3c32),LL(0xafde743c,0x22362ddf),LL(0x6f6ee3ba,0x617a89e8), + LL(0xb7deb988,0xed5f3d04),LL(0xbbc8a6b2,0x31c2c9e6),LL(0x81a3f184,0x8faa80e1),LL(0x51ecc548,0xa7183488), LL(0xa3780d0a,0xe67512d0),LL(0x822db54d,0x9f868036),LL(0xe555beab,0x6c74490a),LL(0xd989d6be,0xe747e666), + LL(0xdf8cd308,0xf8346dd6),LL(0x4745cd8e,0xe7ca105f),LL(0x31055db8,0xee059c58),LL(0x18b38aa0,0x90f4053a), LL(0x41081a21,0xbb2e7fc3),LL(0x45b33a71,0x3602525e),LL(0x2b411945,0xff21f2aa),LL(0x064ccb11,0xbeaadbd3), + LL(0xfe94629d,0xc35f6950),LL(0x9f860b15,0x1cbaa935),LL(0xf24f8f15,0x29b4bcd3),LL(0xd29c8458,0x0ae5b06a), LL(0x1b6c2df1,0xa645c31d),LL(0xd804facc,0x640b0990),LL(0x122b33e6,0x7a4a7f59),LL(0x7479b828,0x94bb0b2b), + LL(0xc4cd4887,0x0567272a),LL(0xfc8e4b0b,0x676d6962),LL(0x8661c0c2,0xa712b020),LL(0x279454a7,0x660e6aff), LL(0x1cd25bfd,0xe1295106),LL(0x077496a8,0x7096885c),LL(0x3006ab7b,0xdbc47c92),LL(0x509205f3,0x498761fa), + LL(0xe85ecfee,0x5d1eaeca),LL(0x534f38f5,0x9fcddeed),LL(0x8af32f70,0x4d34ec80),LL(0x24b3b4e3,0x476dffc9), LL(0x8bbcda9f,0xb45cd869),LL(0xdf684c2d,0x3b0079e7),LL(0x765cd483,0xcaf3eeb5),LL(0x63945b62,0x0b9e25e6), + LL(0x06492e0a,0xfd73833e),LL(0x9507ea57,0x4d2937e1),LL(0xcf562953,0x3e532c2e),LL(0x81ca80c3,0xe4baa2d4), LL(0x28d22139,0x4699e5c4),LL(0x6b1c665a,0x69aab900),LL(0x641016ac,0xf6a37330),LL(0x5f3b7c71,0x335f14cb), + LL(0xfacd904f,0x94a6c868),LL(0x2ec2bf99,0xb1127cc4),LL(0xa4b72d69,0x0ccfceb3),LL(0x55172f5b,0x16b786a3), LL(0xe093a729,0x51ebe029),LL(0xc40c4487,0xf57f4a1e),LL(0xa8ed5a3d,0x8aaf0dd6),LL(0x811f35d6,0x617c51f7), + LL(0x11e98d29,0x18c7ac62),LL(0x2c904ea8,0x71c578c4),LL(0x3c4ef8bd,0x4626b0ad),LL(0xa390be8b,0x121eb676), LL(0x154e297d,0xcb7249f5),LL(0xc2288ba0,0x678ad966),LL(0x57cc9cbc,0x3c2ab066),LL(0x80c8fbda,0xe32c1d45), + LL(0xf0b35526,0xd2f152cb),LL(0x13877dfb,0xc7f75fd4),LL(0xe83ca4a2,0x8603feff),LL(0xcd680589,0x6be89bb3), LL(0x45e1f141,0x5650549e),LL(0xa55ffadc,0x7dab03b8),LL(0x2dc5d31f,0x342edda4),LL(0x9af8105a,0xa97451ac), + LL(0x705b8fd7,0x796e1fe3),LL(0x02d131b8,0x6252a7ba),LL(0x086c3099,0x3db2ab14),LL(0x9db0ce72,0xeb763df5), LL(0x52b62fa5,0xe7b57bab),LL(0x88b820bd,0x6076d449),LL(0x1b660123,0xc43e1f61),LL(0x189eace5,0xc802d40b), + LL(0x341309a1,0x1f2a2a91),LL(0x414db96b,0x8680be67),LL(0xc846e288,0x65dd0396),LL(0xb0bbea85,0x8a1d871e), LL(0x8ff931c6,0x623e2408),LL(0xe14c5941,0x4933ffda),LL(0xb2cbff67,0x72688986),LL(0x8cf79450,0xe51504d8), + LL(0xfeba1168,0x50cd0a3f),LL(0xcd833df8,0x08d2e0fe),LL(0x0a4370ed,0xdbd60827),LL(0x66f4f58d,0x010cf800), LL(0x144e9656,0xffa29252),LL(0x9d1e9d61,0x90b896a2),LL(0x81f7b4d3,0x1802257c),LL(0x595612a5,0xd7758e8b), + LL(0x241b4dd2,0x751882d8),LL(0xfe177abe,0x7dae3003),LL(0xee6fe1cd,0x8f4d5dc4),LL(0xb08f625d,0x93a9cd5b), LL(0xf91cc442,0xa4d6ee1a),LL(0x594d172f,0xe05976cd),LL(0x6e762b2e,0xfb4064c6),LL(0x51a0156d,0xb2068ad9), + LL(0x24f06e82,0x0d2d5b26),LL(0x8c85a9a9,0xad70f276),LL(0x0ed413cb,0x00ede5d5),LL(0x927064d2,0x245be28b), LL(0x2af70d77,0x06eb2825),LL(0x52b0592b,0x472af630),LL(0xd881d50e,0x493afd98),LL(0x1189c989,0x56fa76a8), + LL(0xfaa974f7,0x775665d2),LL(0xc3f54eff,0xe395ccdb),LL(0x3fc83a7f,0xf0a40e4e),LL(0x4c00087a,0xc3b11d22), LL(0xddb50678,0xef8d2f06),LL(0x6cd5f681,0x6e41f315),LL(0xa1b97891,0x7c9d7a3d),LL(0xa0a41260,0x8b297d75), + LL(0x3806a30a,0xca44b65c),LL(0x61a6206e,0x125c5702),LL(0x87003e1e,0x311842a2),LL(0x4513d726,0xe049a7d0), LL(0x7b123469,0x8022c2d0),LL(0x86294393,0x76533934),LL(0x892e7bc4,0x6a6e84e7),LL(0xdb2007fe,0x7daf8b11), + LL(0x923e185c,0x092d1914),LL(0x3def87c2,0x5ec11237),LL(0x18742a51,0x38019e96),LL(0x4808ca10,0xe05ea79e), LL(0x134cbf9b,0x1fc8ae26),LL(0x01b4c1c4,0x14054672),LL(0x64051972,0x32abf912),LL(0x1af62fca,0x0edaa925), + LL(0x3c47d01a,0x58fa82e2),LL(0x780762c6,0xdb12a452),LL(0xfdbf4683,0x16d5a733),LL(0x2f798deb,0x1d7e8507), LL(0x9eab12f7,0x259aa9b9),LL(0x91261397,0xb13e6e41),LL(0x32602f2e,0x564706fa),LL(0x9c2283ef,0x50daef9c), + LL(0xf07a196e,0x9275f219),LL(0xbb8fcd35,0xfc58ebea),LL(0xbad69c11,0x5d1025f1),LL(0x1605c11b,0xcf364154), LL(0x2992276c,0x427bd117),LL(0x6a73cbb3,0x5545bec5),LL(0x133f3266,0x86855c2f),LL(0x67d9e5b2,0xb3d753d1), + LL(0xc9fb343b,0x2134b384),LL(0xb0e12b7a,0xb572f5d6),LL(0x392d24fe,0x7ee5852f),LL(0xc4f285f9,0x73559fae), LL(0x7711c525,0x269cb9e7),LL(0xf00d5606,0x4badfc52),LL(0xb465df15,0xef66d726),LL(0xaa4a301b,0x83eb59a3), + LL(0xed329b12,0xdb406469),LL(0xd933eb45,0x6eb95cc9),LL(0x6b638bdd,0xe2dabfa4),LL(0x031df114,0x7a5d0098), LL(0x38dbfaaf,0xe22d8f3e),LL(0xd79d1ce1,0x2306fd54),LL(0x7acb7cce,0xda324535),LL(0x88f61a1e,0xde6fcc16), + LL(0xb730fe5c,0xaf3e4894),LL(0x28adf897,0x7a3e4a7d),LL(0xb160ae0e,0x352c0069),LL(0xee52c58a,0x225cfb67), LL(0x96b2505f,0x12391b71),LL(0x3758141d,0x8d811bee),LL(0x8cd82e11,0xc941524d),LL(0xbb83a251,0x0feb26a5), + LL(0x76da8411,0x60ad0665),LL(0x88d4a968,0xe3c033d9),LL(0xde120078,0x767b3c05),LL(0x9f31e1e8,0xab7da95a), LL(0xb93e7cb7,0x7ad9b263),LL(0xfd626ea3,0x280f6bc3),LL(0x62713cad,0x746c3945),LL(0x3a4edce8,0xfa2b4599), + LL(0x10ab7f93,0x8792d0cb),LL(0xc25a2a86,0xfa38d031),LL(0x08b028e8,0x6914db0b),LL(0x383cab40,0x75a98aa0), LL(0x6da884bf,0x462e6b6d),LL(0xd3aa74b1,0x2b0f682b),LL(0x5cee0a83,0xb3b7995e),LL(0x3cce609a,0xe99fca2b), + LL(0x45451744,0x342c41c9),LL(0xc81be29f,0xb00d3c24),LL(0x022e8d27,0xd1e64d86),LL(0xbcf67326,0x404550be), LL(0xc8aab829,0xc7c510f0),LL(0xa90c605d,0xb61ae647),LL(0x02db8602,0x582ad9c9),LL(0x71cb4397,0x732b19ed), + LL(0x265e5369,0xea097c35),LL(0x9d5ea687,0xea7c368f),LL(0x8fcae7f1,0x7fc3b213),LL(0x49c54942,0x641daa3f), LL(0x404c39a6,0x0696372b),LL(0x87b4b181,0x56815716),LL(0xfca24eb8,0xa6e156b3),LL(0xd078a39c,0xf278eeae), + LL(0xac762dbe,0x046566d1),LL(0x625ed2e8,0x662ef0f6),LL(0x650e4764,0x15499e72),LL(0x84edf50e,0x361ccef0), LL(0x1f4a2200,0x2441f6f4),LL(0xdb730d58,0xf36fff06),LL(0x3c01edc7,0xcc18624d),LL(0x8a77e5bb,0x4889078f), + LL(0x75f8dd8a,0x02294e3b),LL(0xfc4113c5,0x5f6f6057),LL(0x6f699f18,0xb5300e0d),LL(0x639dc977,0x52cce358), LL(0x328fd50b,0x5dbe59b8),LL(0x39d73c2f,0x81500be6),LL(0x96ae313d,0x409ac4d7),LL(0x5b16c316,0x15205b7b), + LL(0xfc688c09,0xe272300c),LL(0xbdf71f2d,0xb412cf39),LL(0xf85b23d4,0xe3ab9c44),LL(0x7b662694,0x5c14085c), LL(0x9956d07b,0x24b0b385),LL(0x30b2c82d,0xfa8ea968),LL(0x6d403b58,0xd443b2aa),LL(0xe7fc8d57,0x6da53ecb), + LL(0xdfdf488f,0x29655314),LL(0xadc57e2b,0xb418943d),LL(0x6395a287,0xbaf090f1),LL(0xd62f5b38,0x8fdb4fc8), LL(0x371c9db1,0x115653c0),LL(0x96463359,0x6f5e1f39),LL(0x825e6273,0x106aaf1e),LL(0x398cbe1b,0xba22b7db), + LL(0x62b6bf7e,0x3b545300),LL(0x5bb6f993,0x495d7d27),LL(0x3f00290b,0xf558fc5d),LL(0x2cfc2381,0xdddbeb3e), LL(0x65c79326,0xca402179),LL(0x33b1973c,0x376ce4ac),LL(0x9b30749a,0xd6e65ae4),LL(0x5f54bf5a,0xdf68ee04), + LL(0x4cfdb41a,0xa982948d),LL(0xf2a7b4b3,0xddd1d3ba),LL(0x581aaba1,0xf52a6b33),LL(0x894ebf68,0xc3258264), LL(0x84665ac1,0x26c32c27),LL(0x20119b0e,0xda0190eb),LL(0xb86745c1,0x4a599db7),LL(0x58964b41,0xf9570f50), + LL(0xe0648365,0xb34d039b),LL(0x5c5f61e1,0x2cd7fde0),LL(0xbc6b08cc,0x76f514a3),LL(0x18a3cabf,0xc957b50d), LL(0x2334cd1f,0x775fc56a),LL(0x67ec91c6,0x7bfe3864),LL(0x35ad3a9a,0x99037daa),LL(0xb7ca5657,0x17ffe391), + LL(0xfef04aef,0x19f6d369),LL(0xd1876f8c,0x8030b467),LL(0x3cd7878f,0xa014be02),LL(0x3358c943,0x03c22a58), LL(0x2a257094,0x3c77f083),LL(0xd962a04f,0x47386957),LL(0x82da3329,0x768da40c),LL(0x458219cf,0x1507350d), + LL(0x4397ee7c,0xf460aed3),LL(0x36fbc7fe,0xf249e4cc),LL(0xaab03dfe,0xc42d9da8),LL(0x435ab9f3,0xa3d73ce3), LL(0x3813a3f3,0x86dddbc3),LL(0xb79c32a3,0x86d5779a),LL(0x028a2c3f,0x7c3d9aff),LL(0xb1add2bf,0xc687e71b), +}, +/* digit=26 base_pwr=2^182 */ +{ + LL(0x09b3fed3,0x4612a562),LL(0x3579966a,0xf296c17a),LL(0x07960971,0xa124a36f),LL(0x380c4a05,0x6d03b214), LL(0x70f1f268,0xcb0746e2),LL(0x9341aea4,0xcc9b47ff),LL(0x6d2f59cc,0x1b3662d5),LL(0xd4b1a15d,0xa6c65b2d), + LL(0xcccb0a4b,0xf96c113a),LL(0x3615f016,0x24c26bba),LL(0xeead2f5b,0x52fe115a),LL(0x0d7aaabb,0x85623d26), LL(0x31a2564f,0x50791fd0),LL(0xcd0d59a4,0x3659974d),LL(0x7a8b7466,0x2cffdb74),LL(0x514787b0,0xcf6b36e5), + LL(0x4ab1ccd2,0x8afccd36),LL(0x1c03ab29,0x67314635),LL(0xd7ff3a32,0x458f36bf),LL(0xfcf71648,0x70e9e789), LL(0xa6e904cf,0xf3764534),LL(0xf4bdd378,0x2d6130b1),LL(0x1ca5ce34,0xc61c98fb),LL(0xa4a684f5,0xda11f502), + LL(0xb6223f04,0x8d9daa41),LL(0x841c3fab,0x803c9c0e),LL(0xc50b42cf,0x60eee3f9),LL(0x77298193,0xaf4a7a5a), LL(0xbf809ad6,0xd379c2e1),LL(0xf67c0ff2,0x903ab4b1),LL(0x90f8e637,0xc779d7ed),LL(0x2cf3d363,0x968b0cc0), + LL(0xacf51940,0xaadfa857),LL(0x50156581,0x0c789d1e),LL(0x62cff8f4,0x5e79cef7),LL(0x65eb0d49,0x54cdaba9), LL(0x3321c57e,0xdf7a5828),LL(0xa21a51a6,0x8571e6e2),LL(0xc3726e69,0x0b9b482b),LL(0x1d92b657,0x3bc201e3), + LL(0x8a3b4cf8,0x271c58bb),LL(0x717eb539,0x269fc422),LL(0x5b381fe1,0xe82644e9),LL(0xcb62982f,0x27fb0822), LL(0x5b5ec726,0xb0acd51c),LL(0xea4eff73,0xfd01053d),LL(0x00b11c80,0x465311dd),LL(0x2ed8460c,0xe50a8615), + LL(0x7b2243a0,0x3eade5eb),LL(0x77424d11,0xa59ec933),LL(0xf5c7c3b7,0x65a8e1aa),LL(0x0c1db536,0x008399fa), LL(0xfb194a74,0x80b20e97),LL(0x43be90dc,0x2316fb9b),LL(0x0da4d913,0xb2773b23),LL(0xce973d27,0x945d0671), + LL(0xb79f82af,0x64ca871c),LL(0x2dab52f6,0x31304b02),LL(0x928239a7,0x1825ab54),LL(0x8e4ad736,0x740413b2), LL(0x44071d19,0xc5c5d3fa),LL(0x3f0b2da8,0x83e438f1),LL(0xc70a1981,0xfd759448),LL(0x565ebae3,0x13e0c7ee), + LL(0x26bd7c0a,0x31b74b0a),LL(0xd280cb56,0x66e0e8e8),LL(0x3d1c83d2,0x086795e6),LL(0x396ecf25,0x59e678da), LL(0xf015a36e,0xab3c8d74),LL(0xadc03171,0x0d19aed3),LL(0x5a263686,0xc83b787f),LL(0x9057ed63,0x46b94ad0), + LL(0x90979da2,0xfbf783a7),LL(0xa335c784,0xf04dd6a0),LL(0x87d93c4c,0x6e3c2554),LL(0x47994eb3,0xe3e6b289), LL(0x1b74ba16,0x473c0684),LL(0xabe84e1c,0x4e959eb4),LL(0x7c4a67b7,0xdc3bfd51),LL(0x5095bd6e,0xb4e3cb85), + LL(0x3229fb05,0x96fc11f0),LL(0x4b36c83a,0x598227e4),LL(0xd46fca66,0xdc69ad06),LL(0x703ad6be,0x14cc98e5), LL(0x6b22cd50,0xf0fdd142),LL(0xf89c1a5d,0x9b821fe2),LL(0x829f9a74,0xa3762dca),LL(0xf0c320cc,0xf65a584a), + LL(0x5568f242,0x58f4eaba),LL(0x029afc1f,0x83b0c37b),LL(0x994d7dfc,0x93de2d27),LL(0xb1679532,0x0d9a6edb), LL(0x95f085b0,0x3b834279),LL(0xa299355f,0x46ebac98),LL(0x044427f8,0x0212e489),LL(0xa2f37d0e,0xf9e4ce34), + LL(0xfdc9e233,0x0fa328d6),LL(0x51ae732f,0xd5c8afab),LL(0x83c00cee,0x85e59553),LL(0x87505db4,0x9fce31f9), LL(0x7069d066,0x33ea5eb6),LL(0xf01c0ce9,0x10e32a39),LL(0x0c4f1a2e,0xf170233c),LL(0xbd4cb366,0x8a907760), + LL(0x79bf05e3,0xda593421),LL(0xce49a5bd,0x7730907a),LL(0x09be5c7b,0x0dfb8a97),LL(0x23eb936e,0x6f50c692), LL(0xcb18ff1a,0xc6160551),LL(0x661cc384,0xfa1d23fd),LL(0x7ddec262,0xfef12307),LL(0xb15af580,0xd1aca960), + LL(0x2ee50fbd,0x7eab8a59),LL(0xbe1e7a42,0xe7f71845),LL(0x6121e573,0x5f46a511),LL(0x38ff7eba,0xa25dacbf), LL(0xf58f047c,0xe8aefcc7),LL(0xb538aace,0xc343aaa7),LL(0x3c340b1f,0x3e58cdda),LL(0x1fb98ccb,0xb0e9867c), + LL(0xef7750f0,0x034dd314),LL(0x22da84d7,0x2ceaa705),LL(0xfc0d647b,0x4561a254),LL(0xbbe01157,0x81cf0915), LL(0x34b798eb,0x547a3d4e),LL(0x8b1c7544,0xbb5dd625),LL(0xc8194168,0x94fee01b),LL(0xfaeb010b,0xdb4c25ff), + LL(0xe4d4a176,0x1ff217fa),LL(0xaf87f4f5,0x8b46e6c2),LL(0x89734273,0xcf658775),LL(0x52746de9,0x73c4390d), LL(0xb01c7a06,0xb5c84899),LL(0xedd9ef63,0xfa5ffe4d),LL(0xa1a8b2d4,0x28a313c5),LL(0xdaf5a33f,0xadd45f47), + LL(0x1fdb8694,0xc2dc9d13),LL(0x9a90b4d1,0xaa5e026c),LL(0x208cbfa5,0x5edffd39),LL(0x72a4d6cc,0xf095b3fb), LL(0x6645fcc1,0xbfca4e10),LL(0x92408c6a,0x14b872ac),LL(0xd0b82d2d,0x3d9261e1),LL(0x0294e75b,0x13e4ecb6), + LL(0x3ac2ab9d,0xabd4541c),LL(0x4d5d1531,0x025355b2),LL(0xfb726ab8,0x3d85f67c),LL(0x6d6fc6bc,0x56e26c82), LL(0x495e05a0,0xb24608bb),LL(0xe5afdc5d,0x840e0978),LL(0x248727e2,0x2cc543b5),LL(0x3bc8c021,0xe48146da), + LL(0x530c98b7,0xa1b36baf),LL(0x5acf023b,0x04503d7b),LL(0x21de1503,0x96bc4449),LL(0xd2a9c89f,0xbb8a122f), LL(0xd5d4b133,0x66df99df),LL(0xc97d3e52,0x1bb4a13b),LL(0x79b318d6,0xdab370f3),LL(0x9f18552e,0xfa6c823e), + LL(0x6388a194,0xe5b27e78),LL(0xc88ba441,0x13270523),LL(0x4fecfef5,0x9f309fbf),LL(0x10afee60,0x72cd374b), LL(0x93dfe3af,0x16bd0e2e),LL(0x24bc7e8e,0x7e92096a),LL(0xfec7f0bb,0x144fdf82),LL(0xe1f765f7,0x5d1d4598), + LL(0x72c67697,0xb6b91efc),LL(0xb2487905,0xc7a2ceaf),LL(0x7fb24d99,0x4a4c9e63),LL(0x4d742089,0x7ed373ac), LL(0x9149ac54,0x3f9e6ae1),LL(0x0611efc8,0x64fd7fef),LL(0x3d779af6,0x1c38da32),LL(0x0a1681f5,0x6893691b), + LL(0x50a0fa72,0xbac29978),LL(0xba55c665,0x98d5c82e),LL(0x2d4b76bc,0xf3e5b58e),LL(0x90615b32,0xfae27d9a), LL(0xd49b2500,0xb93bc327),LL(0xbbc213cd,0x7d9d4bff),LL(0xd1ee81c4,0xf985fe72),LL(0x381f9e48,0x6e2a94d1), + LL(0x14fb9624,0x1f09b225),LL(0xca4229d7,0x2eba4ff8),LL(0x21dc8c19,0x5b159dd1),LL(0xb1aa553e,0x1e1f968f), LL(0xc7674d52,0x6ea25976),LL(0x7b283501,0x98e73adc),LL(0xd39468c2,0x7cfce0e1),LL(0x08438a62,0x7aad0af9), + LL(0xb2a3dde2,0x2291cdd0),LL(0xf77a0aa4,0x3a625d50),LL(0x5fbc5a0a,0x3be0fba2),LL(0xe794bf46,0x67b7598a), LL(0x531ad772,0x3673d805),LL(0x03e8479f,0xf9a9b392),LL(0x2e16a126,0x142d264c),LL(0x5a2f6f2c,0xc20409ac), + LL(0xcd43f273,0xd9d84438),LL(0xbda7be72,0xfecc561d),LL(0x241b7ec2,0xc4b113c6),LL(0x40dba9e3,0xfc5bc32b), LL(0xd56bca47,0x70853d39),LL(0xa5306217,0x2b9a902d),LL(0x2017bfd0,0x2bb1836d),LL(0xcd1c2768,0x829ce116), + LL(0x697097f5,0x42d5fcf8),LL(0x1e936db5,0xc1fe7be6),LL(0xcb6a81d4,0xcbc5cdcc),LL(0xafef5ffa,0xab1e4ecb), LL(0xb933c216,0x3cbbdf76),LL(0x503607e2,0xdb5808da),LL(0x6bc96246,0x5bdaab7c),LL(0x68274629,0x91e5d17c), + LL(0x2eb1db21,0xa3cd09f6),LL(0x92c3e3e1,0xbe370485),LL(0x6aa43da5,0xeb51fa29),LL(0xd726625e,0x2c7fa809), LL(0xf0ec0e99,0x90c6786f),LL(0x08135cbf,0xd315af33),LL(0x1504751b,0xc1b60172),LL(0x0e28781a,0x88674e2d), + LL(0xed74e633,0x6aa74055),LL(0x7d06ce02,0xc44e740f),LL(0xa33b8d5e,0x8b40bc5e),LL(0x20f00f14,0x42d3539f), LL(0x3307ef15,0xd9f1f5cd),LL(0xc8599bcc,0xa9fe4dfb),LL(0xefa80b8d,0x31cb6703),LL(0x53bb73fe,0x4172b46d), + LL(0x20e4c321,0x85a70280),LL(0x5ac075f3,0x999a0d07),LL(0x7bdb478c,0x59a62b62),LL(0x573c403b,0x9aeb710a), LL(0x950bb8fc,0x1c099614),LL(0x5dc09741,0xc1efafab),LL(0x7296a74b,0x0de58ca5),LL(0xf5be2ec4,0x657116a4), + LL(0xcb199b77,0x0ce52f0f),LL(0xbcd11438,0xdcdc5cb9),LL(0x4777327b,0x587a68ff),LL(0x1cc6fbb3,0x55d9abb7), LL(0x9eeb28a9,0xf1970b82),LL(0x4ceef00f,0xe1ab4e14),LL(0xf7462893,0x184d3fb6),LL(0xc8ea54fd,0x9942a008), + LL(0x1e33b2a3,0x1fee0f77),LL(0x9f789870,0xd4bed81f),LL(0x6ef05b7e,0x6396feea),LL(0x2640b62a,0x9c5d6a01), LL(0x6834bea4,0x170cfec9),LL(0xe131feca,0x68d16728),LL(0x00affb4d,0x4be9c5d6),LL(0x99a6f256,0xe34a423c), + LL(0x09b9ed61,0x1a254e4a),LL(0x30b10207,0x902bc066),LL(0x62121f53,0xd2d5ed01),LL(0x30f1b518,0x0ba86811), LL(0xabe139c9,0x7916c132),LL(0x62c4f337,0xb3a30fe0),LL(0xaa5693be,0x85d0a769),LL(0xe3c7687b,0x2d414379), + LL(0x94958719,0x92b0cb3c),LL(0x4ec6575d,0xb78aa37b),LL(0x4f1bf26a,0xd035aae1),LL(0xd31d5108,0x1383992d), LL(0x92bdd6f5,0x53ecc535),LL(0x08c622ca,0xa9925ff6),LL(0x916d890c,0xcaa3146e),LL(0xb9c10220,0x8cd0f12e), + LL(0x7e12a730,0xcb6ad82b),LL(0xac9485db,0x3f89047c),LL(0xfea2d970,0x6f926117),LL(0x46a19ecb,0x87b0cd9d), LL(0x01e45bf6,0x98bb5b02),LL(0x2ed7716d,0xfc814620),LL(0x4f5caa95,0x8d6808cf),LL(0x082f799e,0x3b57df03), + LL(0x2df84ca2,0x469e1854),LL(0x64aac069,0x00dd62eb),LL(0x88d9efff,0x7d3ee9ce),LL(0xbb830ffc,0x9faed6a2), LL(0xd2d74f58,0xd073aac1),LL(0x2d44199e,0xf69e96b4),LL(0x83ed62ca,0x6cb3a3b1),LL(0xd799acf8,0x472489fd), + LL(0xb63a36cc,0x5f84382d),LL(0x92d5b175,0x6ba1de87),LL(0x516101b7,0x25aab130),LL(0x5f039793,0x6f69c3fc), LL(0x89e3da4f,0xd28439ee),LL(0x5e6b2b61,0x8cb40a0e),LL(0xe3d6650d,0xdfa55805),LL(0x0be59fd2,0x2651f6c7), + LL(0x140d01c8,0x290e0044),LL(0x62ea488f,0x78afa0a4),LL(0x91eaa932,0xc4e39971),LL(0xfe2e79dc,0x8a9ef3a2), LL(0x50705b7e,0xdcfae315),LL(0xd4be3d75,0x73324dca),LL(0x03a91415,0x900bdd43),LL(0xedfdc74d,0xc3ed02ed), + LL(0xf22b4a66,0x509bd1d6),LL(0xb78d264b,0xfd8ed371),LL(0xa419134f,0x562b2d3a),LL(0x7a57a51e,0x80a2c274), LL(0x8c662472,0xebba5317),LL(0xa0be71fb,0xebafedf2),LL(0xb77899c8,0x0c5b9c1c),LL(0xc4888cb5,0x82448008), + LL(0x78401c3b,0xb795ea00),LL(0xa85ab971,0x86776990),LL(0x185739da,0xdd81393b),LL(0x58136c97,0x76d0031f), LL(0x641d39d1,0x6aceaa56),LL(0x39be7ca8,0x918844c7),LL(0xe18efc54,0xa63993f7),LL(0x4af0f30a,0xb5369150), + LL(0x3d04af4f,0x9bc2068c),LL(0xa7796ed2,0xf309dff9),LL(0x4e15b6a2,0x46e9a59d),LL(0xc22ef488,0x617aaeba), LL(0xa15cf0cb,0xd91a8f90),LL(0xc30fb779,0xc6ce12a4),LL(0xb9d0a7ff,0xf3b80254),LL(0x6e9b6fa1,0x32a63bf9), + LL(0x546fe4a8,0x3e1ac837),LL(0x1279c7ef,0x91ed89a5),LL(0xc73e9dea,0x8eb7b88e),LL(0x18238af0,0x96d07205), LL(0xe96abf80,0x56ebf306),LL(0x52c4b10f,0x5088ce24),LL(0xc979137f,0x65293176),LL(0x228d300a,0x824642fb), + LL(0x7836aea5,0x968963a8),LL(0xfabbfac1,0x2d4c556c),LL(0xd3f9977a,0xa4c389bb),LL(0x99b4ccb6,0x2e8b2818), LL(0x6cb67df6,0xc1fd8565),LL(0xa72d2be8,0x0ac57d2a),LL(0xb8224ead,0xa51ce6b8),LL(0xf417d744,0x33f7b468), + LL(0xf9f0bdf4,0xcf8c80af),LL(0xd3222dd6,0x0728f880),LL(0x653afc12,0x436188a3),LL(0x3c7940bb,0x0f8bf160), LL(0x424dcd2a,0xdc18c13f),LL(0x20d3cd1f,0x038c1842),LL(0x7b12fd42,0xed7f86a5),LL(0x7aaf1881,0xa75ab77b), + LL(0xdf0574e2,0x5c3d7612),LL(0x719414ce,0x2eeeeb6f),LL(0x90349fc4,0x797c5771),LL(0x2232eb33,0x0d850f73), LL(0x2638c051,0x0a0744f3),LL(0xb6e7dbfa,0x739e6278),LL(0x659fc5f5,0xa77f286d),LL(0x9654b0eb,0xb18b7cf1), + LL(0x6062e78e,0x5a2089ac),LL(0xdfa6fb44,0x152f1804),LL(0xb61e6faa,0xe8a404b4),LL(0x08d06ea8,0x4774d30f), LL(0x3c359648,0xd7119b91),LL(0x09473ff7,0x850b02bd),LL(0x936b7868,0x4db6f9a0),LL(0xae38c3c5,0x84064dd5), + LL(0xfe448461,0x294d6831),LL(0x42cd2105,0xc3c6f446),LL(0x3a2fdcae,0xa4412eb0),LL(0x3d5a9181,0x394c3774), LL(0x5ca87c4b,0x58f19024),LL(0x89ad5685,0xba1879db),LL(0x803c2589,0x43c55c6a),LL(0xa8249c65,0xae1fad20), + LL(0xe0aff809,0x4929e89f),LL(0x1769a00a,0x19755ec2),LL(0xc242f335,0x3b6a207b),LL(0x090edab0,0xeca054ef), LL(0xcd9e1c26,0x217e9c8b),LL(0x35d4ac57,0x917c2ecd),LL(0xad33911d,0xdc869d5d),LL(0x2e828bd7,0x22d9d860), + LL(0xf38dfaa1,0x89262252),LL(0xeb9cd8d7,0x155c96ce),LL(0xed5ebcc4,0xb0082b5d),LL(0x17182086,0x7b6f9203), LL(0xee92aa6d,0xaefe28aa),LL(0x9aaaa0eb,0xbe67090c),LL(0x2f8ef18d,0x88c5fbf1),LL(0xdd1fd65f,0xbdc8bef1), + LL(0xa9c7b483,0xfb7052f5),LL(0xbd6c8a99,0x49634258),LL(0xc9f424f8,0x1410a747),LL(0xe9805723,0xfda0a304), LL(0x0879bd30,0x1a438bd3),LL(0x7f6903cb,0xed09a9d3),LL(0x57e53497,0x920878f8),LL(0xa7fca0ed,0x87a12968), + LL(0x38590ca1,0x7c8207cb),LL(0xfae885c2,0x4cf52db1),LL(0xe8dc711f,0x6cf384c4),LL(0x221dc698,0x6fea20ff), LL(0xa07bb79f,0x6af56700),LL(0x33ca79c6,0xc7da3b52),LL(0xd05eb519,0x3a214691),LL(0x93d4f089,0xea94c4f1), + LL(0xba51f002,0x734039d0),LL(0xce206406,0xc45e2042),LL(0x4b3c3d53,0xc0e75dbb),LL(0x55b1b97c,0x3a701272), LL(0xd6addb6c,0xec856e95),LL(0xf283aae1,0xb63fe8c6),LL(0x405788d1,0x148fb239),LL(0xe0181912,0x42e7148b), + LL(0x7de07978,0x00bddcdd),LL(0x3c2e0a27,0xac79b657),LL(0xdf1dd3dd,0x94024ba6),LL(0x0bac41ad,0xcddeb357), LL(0x500c4f4b,0x51ec3dd7),LL(0xd31c8fbe,0xf00d594f),LL(0x373a3e93,0x6b8c6f43),LL(0xfc2b6be9,0x891ba3a5), + LL(0xddd72e36,0x3928225a),LL(0xcee362c1,0x1e6a63bf),LL(0xc5eb404c,0x317b78f4),LL(0x67c5e6b3,0xb882736b), LL(0x1f2f07aa,0xb1da56ce),LL(0xff83b633,0xab3c4fbe),LL(0x0ceeab99,0x9cc32f1c),LL(0x1062070e,0xf1dead0d), + LL(0x8a3e79c4,0x49ea0d9b),LL(0xec9f16d1,0x4e7abe3f),LL(0x5549ade0,0x19bda1c6),LL(0xe5885734,0xaae756a5), LL(0xcc2a1aaf,0xb3cff8ce),LL(0xf896ca47,0x812eebff),LL(0x9b2e1123,0x0951b2bb),LL(0xdef6d6a9,0x7f245699), + LL(0x1be7ef41,0xa1331e95),LL(0x9fa1be62,0xd1f0c3c3),LL(0x4383e451,0xb1d8295e),LL(0x9f08bc14,0x658d8a84), LL(0x3ba4b85b,0xb0587aef),LL(0x481cbb27,0xb519c587),LL(0x040d8f06,0x2b975db6),LL(0x1691d809,0x399f6417), + LL(0x7c6204fb,0x207a0e46),LL(0x62c3e9d7,0xe30f1420),LL(0x792f8208,0x6127b782),LL(0xb0d3fca9,0x38f806ab), LL(0x2ff46c64,0x38248542),LL(0x926ec166,0xc18ffe85),LL(0xc0c133fa,0xfd268866),LL(0xb93770e6,0xb7f63f5a), + LL(0xb13afb71,0xd8f1db26),LL(0x32a790de,0x5c5627eb),LL(0xdf50b6f8,0x7f41bc1d),LL(0x92d4c803,0x49d4ef17), LL(0xe8530065,0x577f909f),LL(0xe630ff2d,0x482cdede),LL(0x14f54de8,0x682c8c6a),LL(0xb4079343,0xe6b5a504), + LL(0xe58bde6b,0x00d927fc),LL(0xf34841f4,0x65d85f03),LL(0x2ac001d8,0x265aec02),LL(0x2dfe518d,0x1b785666), LL(0xc01e6e47,0x76142488),LL(0xdd5648dc,0x8e8b2044),LL(0xb3a340b3,0x2c422006),LL(0x3dd67b22,0xa5392113), + LL(0xa1567aaa,0xbd08d05b),LL(0x02acbec6,0x84a55e43),LL(0x5d898af0,0x744ffd21),LL(0x6682e78a,0x38067622), LL(0xffd08522,0xf3696ff2),LL(0x2bf02466,0x49dd0060),LL(0x59c3e65d,0xc9e0d1a5),LL(0x0a37fc25,0x29537f56), + LL(0xa5f6b17a,0x6f6cb9eb),LL(0x9c55857e,0xc1854320),LL(0x45dacc6e,0x959585c6),LL(0xe5f4e930,0xf4e97c94), LL(0x57d2a492,0x966deb5f),LL(0x55d2df12,0x98256831),LL(0xaa457eca,0xfdd65534),LL(0x03551474,0x76dbb021), + LL(0x09d9b4aa,0x0aeefee9),LL(0x784ca675,0x30fea11a),LL(0xff1d381a,0x56b4b509),LL(0x9fce40da,0xd1b26fea), LL(0x48d22911,0x4835b911),LL(0x8bbe57e8,0x6aaac57a),LL(0x19d02037,0xc8882792),LL(0x3ee49afa,0x301e0aa6), + LL(0x00e6b020,0x1641ce6b),LL(0xeac7cad8,0x846b97de),LL(0x61aa6886,0x9b74bfd8),LL(0xb0fa37ac,0xdd95e765), LL(0xf848a83b,0xda0cde52),LL(0x355b3528,0xd2cc831d),LL(0x5e22238f,0xc7fd2e03),LL(0xab9a6c34,0x6d5373fa), + LL(0xd8247f13,0x5dfc2874),LL(0xe3c11f56,0xc211a7a1),LL(0xa2503b97,0x7512563f),LL(0x5c007c82,0x124cd984), LL(0x491cd249,0x4f6eb682),LL(0xa683359d,0xaf4f70a3),LL(0xcc302b62,0x2f1dfe71),LL(0xe57fbf56,0x83c474bb), +}, +/* digit=27 base_pwr=2^189 */ +{ + LL(0x916a8016,0x43af7ab7),LL(0x532bfb9c,0xf93d487f),LL(0xe2174971,0xa5f9af3c),LL(0x2d59b4d4,0xd1b9cf1f), LL(0x44f4eb91,0x4a779418),LL(0xc226edc5,0x6a131fac),LL(0x80d4bb33,0x472ab897),LL(0x2f6ca1fe,0xb69687a5), + LL(0xfabd066a,0xffa73ca2),LL(0xf9c78bfd,0x494e03a8),LL(0xff55cfef,0xe585a878),LL(0xd7053784,0x00770b1f), LL(0x056fe70b,0xdec4da4a),LL(0x57bd444f,0xe37395d8),LL(0x685df668,0x666250d4),LL(0xbe6cc583,0x0549569e), + LL(0xab11639e,0x87629830),LL(0xa4488d53,0x869dd3ba),LL(0xbaf06eb6,0x10fe1c0b),LL(0x1687ac37,0x99034839), LL(0x7f1ffe7b,0x38418377),LL(0x25bd7c57,0x3334a74c),LL(0x7008ba67,0xc57cb7ed),LL(0xc1e4e12d,0x384c12d0), + LL(0xdb4bdb35,0xf48cdca6),LL(0x74d913a7,0x6bc23aec),LL(0x12ed94d5,0x8f0ccd9d),LL(0x86db09e7,0xe4aabd12), LL(0x1e948326,0x0cbff31a),LL(0x17a479a2,0xcf68c47c),LL(0xca7686f1,0x3cced8e2),LL(0x4eb62669,0x15ed1e99), + LL(0xbdb0c561,0xc373ab4b),LL(0x6a9066a7,0x15082022),LL(0x62d31801,0x330a60c3),LL(0xe35bea57,0x53d97f09), LL(0x9c5dbb92,0xf204e006),LL(0xf831262a,0xfb9a8219),LL(0x42136174,0x3468ae41),LL(0x0e03218e,0x0f8fb5bc), + LL(0x4ad8bba6,0x90337499),LL(0xe3ecb618,0xdb71e1fb),LL(0x3cf2a8ad,0x6955e874),LL(0xed691fee,0x594501f5), LL(0xd29bd364,0x7e2baef3),LL(0x6f766759,0x5cbd91ac),LL(0xb2201a96,0xaba54aaa),LL(0xcfa392ab,0x2cfea457), + LL(0x86f8f7da,0xa4da4162),LL(0xcbc0b934,0x88d70b86),LL(0xacff4f7b,0x9df02795),LL(0xc65ef81b,0x0fc80219), LL(0xa299ca0f,0x32d457de),LL(0x0896427b,0x97081b35),LL(0x41bab6b4,0x92d6c309),LL(0x73b8d601,0x5d5e56f3), + LL(0x202bde39,0xfb3992a4),LL(0x3d6bab98,0x2549f564),LL(0x87712512,0x0b564642),LL(0x7fde7e50,0xd52442b4), LL(0xa3d3e16e,0xa6cefd08),LL(0xc83b29bd,0x5b194f0a),LL(0x906dec8c,0x6db0edd8),LL(0x02570c1e,0x7a090959), + LL(0x4c41eb53,0xf6f74fcc),LL(0x5b944a6c,0xd07678a9),LL(0xb80677ea,0xf53bf11d),LL(0xbc5900f8,0x569a5761), LL(0xd3d4de91,0x34e5bba8),LL(0x8361f73e,0xc5774804),LL(0x59abdbd5,0xd637d3dd),LL(0x8772b615,0x64a81bf9), + LL(0x7f3d83ab,0x78bb12ea),LL(0x573f9b99,0xca22c31c),LL(0x2aed4c39,0x4283c173),LL(0x39f32bdb,0xda054c1d), LL(0x1da2cbd7,0x2ead717e),LL(0x62390b41,0x747d67cd),LL(0x6b9666a6,0x43739d9c),LL(0x8c827b12,0xb84e2f22), + LL(0xc0312773,0x0e4ac2b1),LL(0xe53f068e,0x571cfc75),LL(0x42bfe41e,0x6c44df85),LL(0x627e30bb,0xe7d2edb9), LL(0x0dd5cedc,0x9c2e4fd6),LL(0x0f7d22d7,0xe2d885ef),LL(0x1329bcfd,0x44b0b5db),LL(0xba1c96f6,0x006e872f), + LL(0x7e952317,0xdbadab5d),LL(0xc2a5bcaa,0xab849ed4),LL(0x1e72dbb1,0xe3acbb74),LL(0x5d4b7cb7,0xbf42c3d3), LL(0x3d748639,0xebe967b5),LL(0xc03af7a1,0x1fe93db5),LL(0xa944ea06,0x2ab14596),LL(0x76655c09,0xfb05a759), + LL(0x6f8a532b,0x5117890c),LL(0x59430c5b,0x2f57781f),LL(0x79e07b84,0xe70968b3),LL(0xe86d7223,0x05df2305), LL(0x31e32933,0x57af0dc5),LL(0x84afc419,0x5473e34a),LL(0x03d5feb4,0xa7337a42),LL(0x1b1c6bd8,0xd85c8602), + LL(0x753008e6,0x25ca1891),LL(0x5f0ff93a,0x4338ec98),LL(0xddd30a7c,0xd2ba8557),LL(0x09c51794,0xb4b65361), LL(0xd1cbc66e,0xfbb51399),LL(0xe53bca50,0x28853781),LL(0xfd5a9aaa,0x5b797232),LL(0x5b88c4f3,0x6249afd7), + LL(0xba6918a0,0xcc5ab6cb),LL(0x8fb65c7d,0x9f824ec1),LL(0x56b18754,0x4796d80b),LL(0x67721520,0x4c83d371), LL(0x63b03348,0xd77c373c),LL(0x54f27457,0x91930e5e),LL(0xaf40c03f,0x83f97370),LL(0x34eea661,0x65b55872), + LL(0xeb10175e,0x310695d0),LL(0xcd236aa1,0x79aaa6ea),LL(0x3edfff40,0xf78539ff),LL(0x02cd6063,0x2369c517), LL(0x5c8631ff,0x81e43ae5),LL(0x216a60bd,0x065e8212),LL(0xe761a5f9,0x225cb473),LL(0xab6de6fa,0x695ef860), + LL(0x7d7d98d4,0x03536a46),LL(0x18413673,0xa17d3a69),LL(0x295ae303,0xa6ddcd46),LL(0x61beae2b,0x86de0bbd), LL(0x7699458e,0xdd73dfcc),LL(0xb53f88dd,0x827deba5),LL(0x42a9a11c,0x213c376b),LL(0x12c73975,0xc854fd72), + LL(0x15ac27ff,0x1fa96547),LL(0xf49b6c9a,0xcb0dc17b),LL(0x709dd202,0xa3e44853),LL(0xcfe2bbea,0xd3905c5f), LL(0x6c35ce9c,0xb01e5799),LL(0x900ef600,0x0063e7ac),LL(0xfffa5cc0,0x8c70b87e),LL(0x74230b0c,0xebd76d34), + LL(0xed5f8529,0x914eec9e),LL(0xe8edf477,0x7a65ffd3),LL(0x70c74bee,0xf0cb206d),LL(0xd1b02e01,0x03445ff1), LL(0xe5dbf791,0x664ca356),LL(0x254e69c4,0xd678d4ae),LL(0x8617386b,0x370c9f0f),LL(0xfdcd985d,0x42af7a0c), + LL(0x83c3da54,0x8c4b5009),LL(0x4c8a87c8,0x086a7ec5),LL(0xaa166c4c,0x9ba0b368),LL(0xa658ac1c,0xa279670f), LL(0x5d0544da,0xc49f49bd),LL(0x15cb0b41,0x28c22323),LL(0xa4834d71,0x86293dfa),LL(0xd1e1d63b,0x283e191d), + LL(0xca188555,0x0cad6519),LL(0x0cbd0c5c,0x323ce5da),LL(0x38560254,0x6b7d2be1),LL(0x1696b9b9,0xb05ed385), LL(0x9ae59f92,0x8ce4b5a7),LL(0x4f7e61a3,0xabe5ff33),LL(0xdbfeb302,0xae15a3cc),LL(0x837fde82,0x691b1129), + LL(0x2e6d116b,0xb60b31f3),LL(0xecab5aa9,0xd49e9d11),LL(0x6787f23d,0x3e95f844),LL(0xa12f4846,0x2ab8834f), LL(0x5b6359cc,0xe70e2ab1),LL(0x9322a047,0x7a6349e9),LL(0x6c1e483a,0xc01e424c),LL(0x92bd5d1b,0x424b2027), + LL(0x254e49a3,0x8a6e6766),LL(0x97e70d58,0xb8d85d42),LL(0xb51b3abc,0xa859082f),LL(0xe7bb828a,0x2850573b), LL(0x7bfe8021,0x47cc95b2),LL(0x5853f12c,0x7c28fe9e),LL(0x10c0f389,0xe5fb0558),LL(0xdaf0a7e7,0xb99a639f), + LL(0xf60ee3e5,0xa6b9e6c9),LL(0xa236377f,0xb397af7f),LL(0x7da4af71,0xb7a318ac),LL(0x0a9d39fb,0xae64b613), LL(0x902b3411,0x66ce6c74),LL(0x5a199e53,0xea256a70),LL(0x550fb76f,0x8dcddd89),LL(0x03e70f9c,0x9443b477), + LL(0x142113a6,0x1787b8a5),LL(0x180aec95,0xa58b6c61),LL(0x947ff26d,0xcc381420),LL(0x3d8b8c36,0x22520e8f), LL(0xef2cc6ef,0x192ee945),LL(0xe9ca0c7a,0xea52afee),LL(0xe6e30d93,0x5970d794),LL(0x57c786ac,0x0a325e42), + LL(0x33ca1226,0x5e2dddf8),LL(0x588cb1e3,0x18e624b9),LL(0x21809265,0xf3ba597a),LL(0x5d575728,0x90247702), LL(0xc1f918db,0x48a5bf7b),LL(0xd6840541,0x17d1efaf),LL(0x3e2e754d,0x13dfe6fe),LL(0x707a531f,0xc471e16a), + LL(0x97d34b48,0x79085bbd),LL(0xc2e9bea9,0xfa5ba99d),LL(0x6c5a6dc2,0x70b9c9fc),LL(0x4e94c5db,0x4e042213), LL(0x25ebb95f,0x4a37b41f),LL(0x055d79fb,0x24691010),LL(0x3f572a8f,0xdaff9352),LL(0xf327ec2a,0xe63d55b0), + LL(0xdebd5116,0xc5a86d3c),LL(0xa2ddef2a,0xd547fe08),LL(0x6a149f12,0xbabb617f),LL(0x8a766128,0x14f69a1b), LL(0x48236f77,0xb83a1477),LL(0x35711279,0xd0d81be1),LL(0x5eab1c3a,0x706f9067),LL(0x16a1ffaf,0x8c4823f1), + LL(0xaff5ea89,0xd845c68b),LL(0x6b75eadb,0xa276eaeb),LL(0xcc230ec1,0x2d0fc819),LL(0xedaaf1f2,0xdfad96e8), LL(0x40868254,0x0f25dcbf),LL(0x5babd7f9,0x53bbe31e),LL(0xcf804a8d,0x7f8afc48),LL(0x5f9b9a0d,0x7f4922ef), + LL(0xd7422804,0x703cbf6d),LL(0x83349bdd,0xe5df61f3),LL(0x77d285ad,0x0fa3d8cd),LL(0x2e88e15e,0xe990f9e5), LL(0x8561d8a6,0x40ec61f7),LL(0x16650305,0x7fc498a6),LL(0x8e5beabf,0xa3bf5cb4),LL(0x76ae0350,0xfaa52008), + LL(0xe4fc3e72,0x99e24318),LL(0x2079c616,0x9241c8ab),LL(0x9584a258,0xefa5bf38),LL(0x1eebb098,0xd7b770b5), LL(0xe1fc18a7,0x28b714a3),LL(0x5b83dd9a,0xf0426bd2),LL(0x291b28ee,0x956d8972),LL(0x6eb553ff,0x8bb8cbde), + LL(0x95298003,0x396cfe2d),LL(0xad8412fc,0xcaa66550),LL(0x83997dd3,0xf41d0215),LL(0x45534587,0x7066e356), LL(0x5b6de0d7,0x0d5b5c3e),LL(0xcecd5f26,0x8ead45d0),LL(0xd252ae50,0xe2f24e2c),LL(0x815150bf,0xf71e5d4f), + LL(0x54527ce5,0x3872685d),LL(0x91fd99ee,0x59b343ae),LL(0x3462cc0c,0xd621d027),LL(0x8dbfbcf4,0xfa42481f), LL(0xaf7ae918,0xda481a9e),LL(0x7c909a18,0xfd5fd37c),LL(0x805fb7b7,0xa5ebb7bf),LL(0x165200b1,0xeac65687), + LL(0x7cef9b97,0x56302866),LL(0xae3ddb64,0x8f662dd6),LL(0x60c1aa98,0x90cb4e87),LL(0x986fb3bc,0x33f9fc60), LL(0x974593cd,0x76f41ecc),LL(0x6e0f01e8,0xb19501f9),LL(0x25760dd5,0x587d9035),LL(0x9391032e,0xa31c971c), + LL(0x95c9a84f,0x7650e3b1),LL(0x78c66087,0xbb83ea93),LL(0xdfcf1365,0xda08a04c),LL(0xca0b84a4,0xd23daeba), LL(0x2ca3bd2b,0xf89d395d),LL(0x6e53fc93,0x779e2aaf),LL(0x34216082,0xc0fc7dc8),LL(0x42a66691,0x6cd8bdf6), + LL(0x0fe014cf,0x836a2cf3),LL(0x0c08373d,0xdde5fc22),LL(0xcb3b2b54,0xc4fa2387),LL(0xe2aa434a,0x96253732), LL(0x1d502ce8,0x4c4f5979),LL(0xb6df40c4,0xf046f5a9),LL(0xac6b75b5,0xc7d05765),LL(0xb69f3c34,0xae1cd887), + LL(0x49b86680,0xafed4be6),LL(0x14f70463,0x17b75fa5),LL(0x90b7c07f,0xb12e8051),LL(0x39a8e99d,0xe2c8cf2b), LL(0xd5fdb65b,0x984744c4),LL(0xa28942e4,0xd8b1c012),LL(0x46693fb2,0x295d0199),LL(0xa0a3b4fa,0x5ab3a305), + LL(0x26755b3f,0x3c191023),LL(0xb6c084de,0x75f35df1),LL(0x63e6286b,0x30640e66),LL(0xd2c6c059,0x3b3720ec), LL(0x6ea837be,0x2c821a0f),LL(0x84f23bd0,0x238c4d93),LL(0x390ea4f5,0xbdc40703),LL(0xae68a2db,0xcb207d0a), + LL(0xe25f098f,0x487d18bd),LL(0x9ab84e10,0x39018023),LL(0x8b7ab4a2,0xaa19aa62),LL(0x89f08fbd,0xcb9cdebe), LL(0x2ca57917,0x26a4c9eb),LL(0xda92ce1b,0xaadfd472),LL(0xdaa907db,0x32b592d8),LL(0x7d6f995e,0x9bbebacc), + LL(0xe1d88c25,0xa27a4735),LL(0x9bd66b67,0x339905e1),LL(0x62af9942,0xa9bfa0ed),LL(0x2e2cb83c,0xd94dd9e0), LL(0xab28e268,0x279d8fda),LL(0x51a97518,0xf28ab69b),LL(0x9691f33e,0xce9bd2ea),LL(0x74be3d64,0xb9e8b2fe), + LL(0xabefa07d,0x35072fab),LL(0x7b51ba8e,0x1c2ba05c),LL(0xd32d6bf5,0x3bb1ec56),LL(0x5d7bd7dc,0x326bdfdc), LL(0xd95bdcb1,0x33f4f4f6),LL(0x453ef338,0x781bfd34),LL(0x1ef61a66,0x4d210cad),LL(0x2799bcc7,0x6ae7bb14), + LL(0x194f4d6a,0xb105e5ec),LL(0x52b77f08,0x204a5480),LL(0xa328ab98,0x13f4e022),LL(0xb7f9a240,0xa56863c4), LL(0xce4cf7bd,0x2780c9a7),LL(0xc497fdf3,0xf98e3f58),LL(0xf52738fc,0x635b8bc6),LL(0x58937500,0xc5fd89b8), + LL(0x75e98a64,0x57070428),LL(0x946f094b,0x66aabaae),LL(0x06d37944,0x7d2376e8),LL(0x09319f13,0x9b316827), LL(0xa77eb79b,0xbbde59a8),LL(0xf451fde0,0xb7f35bbb),LL(0x64aa79fd,0xb2f7914e),LL(0x9f68a486,0x4d2944b3), + LL(0xc1a7e537,0xbd8a92de),LL(0x4fc930a3,0x76695e9a),LL(0xbcb4153b,0x1b75f9eb),LL(0xf6485404,0xf5a34d2d), LL(0x26853a8e,0xe09ee965),LL(0x9dbb319a,0x63de8595),LL(0xda079d6d,0xbbbc1b07),LL(0xdfa71b9d,0x5956bb3d), + LL(0x209cbcc3,0x69709306),LL(0xe3360429,0xbe2a08d0),LL(0x92a58752,0xd377a9fe),LL(0x997bc322,0x37e175ea), LL(0x042ff2e4,0xfe355d4d),LL(0x4c4babd3,0x4332ef31),LL(0x2314b1af,0x634429c2),LL(0x91a7d5e5,0xae6e8275), + LL(0x250a1476,0x134a39c7),LL(0x08994f0c,0xec9bb642),LL(0xd38704cd,0x2a9e0ac0),LL(0x536a4ad0,0x16490507), LL(0x7c8dbfeb,0xc7f747d2),LL(0xc0bb24ac,0x91e67dd2),LL(0x959eca45,0x2dfc6c8a),LL(0xc54fefe8,0x78bafaf0), + LL(0x5da056f3,0xf3eb2d1b),LL(0x3b89c967,0xda14b631),LL(0xcb51f621,0x80923b1c),LL(0x6609791c,0xc3d5fd1f), LL(0x817b1063,0x68ad7bef),LL(0xa1f0b00c,0x3775b686),LL(0x6c7f0dc1,0xb207c9a5),LL(0xa9b91fff,0xb7c30a7d), + LL(0x8b9f8e8c,0x9274c090),LL(0x24e94ce1,0xa3757610),LL(0x4f0f3ec1,0x8f2b1f2c),LL(0x3938d26f,0x02e53bb2), LL(0x701e5ae8,0x90a3da2c),LL(0xa271dcca,0x60c2eaca),LL(0x31fb2372,0xc9c08e39),LL(0xb838908a,0xcaa3245e), + LL(0xa6a6a275,0x2e168b0b),LL(0x0030ef6b,0x986a30a3),LL(0x170ab683,0x79f768f9),LL(0xff70981e,0x7296fd6f), LL(0x13a323cd,0xbab6fedf),LL(0x186e9813,0xa86ec0dd),LL(0xcd56e7d5,0xd9f8db04),LL(0xaa8b1c96,0x47b20676), + LL(0xf1fb3b03,0xdff4574e),LL(0x1051f9fc,0x41a1f765),LL(0x7f289a4e,0x35779aee),LL(0x11c96180,0x93bd54c9), LL(0x37b95c67,0x1485074a),LL(0x0865b2f0,0x0b01af95),LL(0x90ce0b41,0x43033ffe),LL(0x71504d6f,0xffd6e34c), + LL(0x1aa946c8,0xb380cd60),LL(0x806b2e19,0x7f7cc73b),LL(0x2db88e6d,0xc17df7d8),LL(0x072e5416,0x7b767ca2), LL(0x0ad6134b,0xbb42d3ed),LL(0x640df8af,0x5444339f),LL(0x5bc73112,0x7e7c7e7b),LL(0xf899dba4,0xe8f139b4), + LL(0x43a06bf3,0xd13b436d),LL(0x773e4863,0xe43f8567),LL(0x56b814d7,0x35555cd5),LL(0xd429ccc8,0x54af8e53), LL(0x82ae0477,0xc346718f),LL(0xbe02c7a8,0x301fb382),LL(0xd2a70595,0xcd65b3b2),LL(0x5aad01d6,0xcfcff499), + LL(0x589feca8,0xd0fcc076),LL(0x7c603ed8,0x7b2b93c7),LL(0x6ddfc3b8,0x2dda7a8c),LL(0x74723d99,0x678d66e9), LL(0x6db60b07,0x0f7e4215),LL(0xc0bfa2f9,0x40666848),LL(0x8e116caf,0x70b46b5c),LL(0xfba46d90,0xbd753511), + LL(0x019d8279,0xe48374cd),LL(0x309b0fc6,0x7d40e6e2),LL(0x9dec7a42,0x226435ee),LL(0x4243e7d0,0x818e79cb), LL(0x54658634,0x3d7376d7),LL(0x9f8727ac,0xa40cafeb),LL(0x81f556bc,0xdc1d09f0),LL(0x63223573,0x32ca7367), + LL(0x5810a27d,0x92e10f91),LL(0x1fdf969f,0x6fb34bad),LL(0x657a067e,0xe5c2b2ff),LL(0x382ba37a,0x173c0900), LL(0x86d87c1e,0xdd5113c8),LL(0xcaf77294,0x56a2ca9d),LL(0x666a9821,0x9f956881),LL(0xa3b18c0f,0xc4bcafc7), + LL(0x2b02578b,0xb100f338),LL(0x64b2c607,0x4716339e),LL(0x5b161179,0x92c923ae),LL(0x0df442a0,0xada2e4da), LL(0x47f85474,0x4d4b90c5),LL(0x824e3195,0xa378bf79),LL(0x2478a0d4,0x4469339d),LL(0x0c1e69e2,0x0972e788), + LL(0x72edc384,0x1aedd761),LL(0x9898d672,0xcabcd976),LL(0xba814ca2,0xd370aa7a),LL(0xe88eca9c,0x20fa58db), LL(0x45a7ab8d,0x1540ada9),LL(0xbdca94fc,0x8dcf9860),LL(0xaa9df4f4,0xf0187e2c),LL(0x54a49576,0x9a197dc3), + LL(0xb4a1f341,0xb54f5cb2),LL(0xfe32827b,0x1439caf0),LL(0xd36783f5,0x3c1a356d),LL(0xc56a6e47,0x284e2f15), LL(0x4dcfaddf,0xc6abad59),LL(0x082bb2b4,0xe82993f7),LL(0x23655955,0x3cb46972),LL(0x992551e3,0x8ab06385), + LL(0xdaa13ab3,0xcbd6cb99),LL(0x2dc1333d,0x01375bbd),LL(0x972c4440,0x638a7f20),LL(0x24dcb1cc,0x150665c6), LL(0x1ea989c6,0x4044e12f),LL(0x61012ea3,0x204c4eba),LL(0xac2719c1,0x78b8edaa),LL(0x2ab50d99,0x6772643d), + LL(0x606d63de,0x94604146),LL(0x693aadc8,0xa876d9b0),LL(0x667044ff,0xf7401ffb),LL(0xb599ecb4,0xab98d73e), LL(0xda5cbee3,0xe2b2048f),LL(0xa2b3da50,0x526e3aa1),LL(0xb4ad2073,0x4d0885e3),LL(0x644a1a19,0x916ce3d2), + LL(0x96930e8d,0x952b5747),LL(0xb0cf7f5f,0x2a489fd6),LL(0xa8b3b538,0xbff4b59b),LL(0x6aff1cbe,0xba0e03ff), LL(0xd56b2285,0xfa614adc),LL(0x50d58e62,0x2305edd4),LL(0xe36877e9,0xb349fdce),LL(0x43a6103b,0x5f808fc2), + LL(0x86493abe,0x66d8404b),LL(0x9b08ff7d,0x18c92d3d),LL(0x89754762,0x6a60ab6b),LL(0x8233dee7,0xec69fd4c), LL(0x06beadfa,0x32449242),LL(0xe0df7084,0x421caf1e),LL(0xd7969339,0x6f89693b),LL(0xfa30a7a9,0xb9a53713), + LL(0x11556d9a,0xf89d9bf5),LL(0xee8cf993,0xe4e9c5f0),LL(0x17ed9a7e,0xe5b2a323),LL(0x93e80c9e,0xd4db3920), LL(0x1fda3726,0xae857864),LL(0xa3e88485,0xe5cb36a3),LL(0xf495b9a8,0xa6b85205),LL(0x38f3b180,0xc1be0108), + LL(0x36a1f3a8,0x79d0585b),LL(0x913ba5f2,0xa3d8f17f),LL(0x225acf11,0x1eaee5d6),LL(0x0d32de79,0xd4dfd0a2), LL(0x6b3ceff3,0x0cec324b),LL(0xab447870,0x3acc6dec),LL(0xb9c759ac,0xabbf7e6d),LL(0xa5196938,0x0d5c1f47), +}, +/* digit=28 base_pwr=2^196 */ +{ + LL(0x45e7ea91,0x781a2157),LL(0xacadfc40,0x4da3f86d),LL(0x162cd393,0xc81d6c7d),LL(0xad6e60fc,0x2c38a2a1), LL(0xf753479d,0x575b25d6),LL(0xbdec6025,0xc914e08e),LL(0x492d5547,0xf81cea34),LL(0xfb1b6969,0x6bbb8bb1), + LL(0x1279504c,0x1ee8082c),LL(0x2c92ffb7,0xa466abb2),LL(0x3e81c7e2,0x4118b26a),LL(0xfc60e33a,0x1a76cc50), LL(0x5736d7ae,0x34998bc2),LL(0xbd1ef993,0x20b39558),LL(0x5fbf2525,0xd669e2ae),LL(0x01cc7626,0xbf956ec6), + LL(0xb0ccbaa5,0xce817029),LL(0x279b78a6,0x57ef5bd2),LL(0x4df45d89,0xc9283747),LL(0x2ec4bfd3,0xe86b91a8), LL(0xfe565915,0xe5ab4c6d),LL(0x7c58a042,0xe6574716),LL(0x6301c4bc,0xe141deda),LL(0x8084513a,0x2f95d561), + LL(0xeecede3d,0xdc424508),LL(0x386440d0,0x11889b35),LL(0x98de0d77,0x7b229f93),LL(0x300a7447,0x73fced8a), LL(0xe31c8f88,0xf75e1c79),LL(0xbb277e4f,0x8db20bdd),LL(0x2b87c02c,0x8ded0a70),LL(0x4d164c1a,0x166281b5), + LL(0xeedd8e0c,0x887356cf),LL(0xe44c012b,0x8afab37f),LL(0xe4aa3eb6,0x0795935f),LL(0xda6dfa57,0x9b9efc0c), LL(0xa8ab0840,0x0ff0f8aa),LL(0xc8561605,0x0f3a4b63),LL(0xd5db9315,0x2ca911ef),LL(0xc8ded9f8,0xef70e5ba), + LL(0xa6aae58b,0x443d9209),LL(0x274edda3,0x3d0798e8),LL(0xc2be3c9a,0x5c2d462c),LL(0x439882dc,0xb5488239), LL(0x977d4de4,0x6391bb41),LL(0x1e8245c4,0x7fd91040),LL(0x3b093dc2,0x1a6d3c71),LL(0x7b22fe12,0x423a4e3a), + LL(0x3a9a04a3,0xe3156f40),LL(0x297d9aff,0x9b32c4e5),LL(0x62a89850,0x7e0b401e),LL(0xa84ef082,0xffbf542e), LL(0xf990caf5,0x377cc0e0),LL(0xec88ea9b,0x02704343),LL(0x63f96a51,0x846fd46c),LL(0xe9855c47,0x37f5cebf), + LL(0xe6ad29dc,0xbd140bd8),LL(0x6a04da28,0x7dca4b10),LL(0xade05b33,0xa84feafc),LL(0x7630aacf,0x44d031f8), LL(0xcdee269c,0x18af2fa6),LL(0x8697a40b,0x1e40571b),LL(0xf0e5f826,0xf71d44ad),LL(0xa434cfe6,0x2a47ddf9), + LL(0xad17cc54,0x22b97078),LL(0xf2c105b2,0x223732dc),LL(0xe284fae8,0x25168336),LL(0xb356407b,0x41b1bb94), LL(0x89933a11,0x299e7d7a),LL(0xff7dd9f6,0x19e13d3c),LL(0xf23d7ca7,0x9517bd16),LL(0x1eb978a4,0x9e5e9e34), + LL(0x5fa3f92f,0x4c222dae),LL(0xed489ca7,0xd5e38e84),LL(0x70ea613d,0x3d81aca4),LL(0xbe4e88f6,0xc7bed301), LL(0x0757d8db,0x6fd5a7bf),LL(0x7a9181b0,0x1472441d),LL(0x5a90b66f,0x78b78753),LL(0xabdae620,0xe3fd5e91), + LL(0x84535653,0xea6a77d8),LL(0x81d7e667,0x8d241deb),LL(0xfaf4ef1b,0x1af73798),LL(0x3e0dae27,0x5e1ae728), LL(0x2f7450b5,0x6a67088c),LL(0xda9cb3c6,0x7bccbe06),LL(0x5b808e05,0x520fabab),LL(0x702b247b,0x84222f68), + LL(0xe0bd7ef2,0x2471546a),LL(0x656a62a3,0x27d310dc),LL(0xad35da30,0xb8014eca),LL(0x7f35cd7a,0xbdfdcd82), LL(0x040ae645,0xf1e4d51f),LL(0xf42a4d9b,0x672ffadf),LL(0x2d0be1c0,0x9d874370),LL(0xc6e55471,0xcc3671c6), + LL(0xbb9c9667,0x39aa705c),LL(0xc51f661d,0x8c3e584a),LL(0xe570769c,0xe5645b1d),LL(0xbc97abf4,0x81923fda), LL(0x0caac97c,0x51d64f64),LL(0xff847f4a,0x45c17651),LL(0x8cbfa2c7,0xc7a6eaf9),LL(0xba8ab893,0x6c2ab9f7), + LL(0xf435624e,0xbdaa2c7b),LL(0x1d961058,0xc113e971),LL(0xa2021a1c,0xb230f1b0),LL(0x521a4816,0x6b34e1ff), LL(0x9b874f4d,0x159dc24d),LL(0xbeaab169,0xeaa0f951),LL(0xb56f4916,0x4f38733f),LL(0xdc9d3ac7,0x4ee689db), + LL(0x7bf8d03b,0x720254bb),LL(0xd31d7679,0x78b0e6d6),LL(0xf130d7b0,0x848fb878),LL(0xd3ba625a,0xe8e478ec), LL(0x100dfefb,0xb0ce9168),LL(0xe5098aa8,0xfe1463ab),LL(0xa11ec558,0xf780ac38),LL(0x8e474b9f,0x92f15c52), + LL(0x46410cb1,0x3b3892d3),LL(0x03a5a136,0x72097f22),LL(0x98de068d,0xdb3a1b80),LL(0x4b1a3890,0xfb7438e4), LL(0x3839d3d9,0x8a10d5ea),LL(0xf4bd8126,0xd9ad034d),LL(0xd4800261,0x07d108ef),LL(0x9c5d6c52,0x978d98ba), + LL(0xecb9ce1c,0x63ae69e1),LL(0x51b28f39,0x70d2b437),LL(0x77f848a2,0xc15696b6),LL(0xd8ab4d76,0x6b6e60f4), LL(0x030bf112,0x33a581a4),LL(0xc5e74a76,0x9cdb1a6e),LL(0x7a950053,0x6c6f6ec4),LL(0xb04ebcff,0xd47dc472), + LL(0x5e0970df,0xe85ca76a),LL(0x78988af2,0x74448d98),LL(0x81620019,0x5227649b),LL(0xaabc2027,0x47e2ac62), LL(0xfbffedf2,0xfea15155),LL(0x3b4cb501,0xa565c484),LL(0xd830cece,0x4c523be5),LL(0xc321a440,0x2e2de6bc), + LL(0x8d69887d,0xa7d62771),LL(0x4e138de4,0xf9d8ac67),LL(0x9fcb0a09,0xad3fbc08),LL(0xbfc3bc9a,0xcaabb0b0), LL(0xb1c20604,0x84646bc3),LL(0xd1574260,0xf1059ac4),LL(0xeefff298,0x5c15c6a2),LL(0xf3b0a393,0x7975ede6), + LL(0x27c2343a,0x0ea9d355),LL(0x4b32e339,0xe21c75e4),LL(0xa7fc353a,0x1438785e),LL(0xe9a1dd56,0x0b8d64ba), LL(0x75347c02,0xcacf9b64),LL(0xcaad57aa,0xf788c83e),LL(0x36ecf2e0,0x90df1ab8),LL(0xf45070ac,0x4db604a3), + LL(0xb4c4ed93,0xbc76e168),LL(0x85b65a6c,0x07177b5e),LL(0x00d21309,0x41e3c275),LL(0x76a48f42,0xcc9678e4), LL(0xb1c6256f,0x3a04d197),LL(0xb2cc7330,0x940920a9),LL(0x2523d52f,0x990e4da8),LL(0x5a59d733,0x34709b24), + LL(0x8e745028,0x2f0da81c),LL(0xcd5668ab,0x32b5f384),LL(0xee538e7e,0x82744a5a),LL(0xf3eb2516,0x1b019bab), LL(0xd79d735f,0xccbd28fb),LL(0x85f90aa2,0x0bb54a6e),LL(0x9a69ecaf,0xacf5552f),LL(0xd1f1e30b,0xbc51ee85), + LL(0xfa25193d,0x12bf8b0b),LL(0x5ba4b3c8,0x3f0f51b0),LL(0x66181f23,0xc1b65deb),LL(0xc0156b03,0xfeb037f9), LL(0xa9dc59ed,0xdd7a0a8c),LL(0x7b57e018,0x20b5c8ea),LL(0xefaadad5,0x0c3ebc94),LL(0x18758eba,0x146d05b6), + LL(0x09c7b43c,0xcb952e41),LL(0x1c1b8fb6,0x7f7a0ae3),LL(0x331dfb05,0xbca8a9cf),LL(0xe0db9d7d,0x4a1db4a1), LL(0xe5b9c892,0x988d36a3),LL(0x010ad00e,0x64640e55),LL(0xc011bffd,0x4c33c7e8),LL(0xa0ad217b,0x5d7cf370), + LL(0x71f3df52,0xbaf8b74a),LL(0xa5995b20,0x300963bc),LL(0xd6c27636,0x695cf7ee),LL(0x03ac244e,0x74d4d3a1), LL(0xc898e5bb,0xddba3bd6),LL(0xfe3499f7,0x27573a89),LL(0x4b0a6c98,0x666b4415),LL(0xaa4ccfaa,0xf4f3e8c5), + LL(0x1a5b919b,0x5f136875),LL(0x670d4b04,0xed8eb5db),LL(0x0d0d73bf,0x4cd83d19),LL(0xbdf22579,0xd2a5c62a), LL(0xc2d04c2b,0x8c41be16),LL(0xbf9ad465,0x5aa33bc4),LL(0x5e00d922,0x36e20646),LL(0x00b70e17,0x9df21e7c), + LL(0xd440af4d,0x0601e630),LL(0x963e87dc,0x4aab0d33),LL(0x36d39000,0x2712abdb),LL(0xf9147e2a,0x856d7e3b), LL(0xc8e5d2f4,0xadc4a96a),LL(0x2e70c206,0xac3e5336),LL(0x6f6f3d0e,0x1ee7d838),LL(0xead72426,0x4674ef20), + LL(0x6af5f580,0x3a804dd8),LL(0xd74ea5ce,0x724a756b),LL(0x0432d854,0x0c2968d0),LL(0xa4f262fe,0xe3be03f3), LL(0xc84c22bb,0xe446692a),LL(0x9647650d,0x156b3168),LL(0xb5d3d62a,0x4e06bc39),LL(0x80eea174,0xf99d4fec), + LL(0xc08f1144,0x3a2b7ae8),LL(0x12dae8d6,0x35e65bf9),LL(0xae3892b5,0xfa0af1cf),LL(0xac408112,0xa57062dc), LL(0x24bf1af9,0xef6a9ec3),LL(0x5decd8bc,0xdda3b476),LL(0x7bed3775,0x9314a36c),LL(0x60aa296e,0x9e254b0e), + LL(0x65b9cf2c,0x8be2de80),LL(0xcb3b96cf,0x1b110df6),LL(0x18957e21,0x0f647a12),LL(0x4f907766,0xa1e11238), LL(0xc5528977,0x751a0d82),LL(0x9a4b1260,0x958d8738),LL(0x773658ee,0x99087543),LL(0xf19f74cf,0x18148bbe), + LL(0x0a19a374,0x5f50ef19),LL(0x6bdd3392,0xc5bc4160),LL(0xb80ad74c,0x1bdf5e4b),LL(0xed7e68c8,0xc40ec2f7), LL(0xdecef5b8,0xedd7dd6a),LL(0x896c95a3,0x3d29a1cb),LL(0x70ad41d4,0xfa84c325),LL(0xc398c177,0x6a577072), + LL(0x7375f2de,0x4f942d01),LL(0x8aa1523a,0x968a7608),LL(0x377e5c4c,0x55dc7da6),LL(0x282b540e,0xb75fff53), LL(0xfd4b6951,0xfee35c15),LL(0xf04ddfae,0x6d1d64f6),LL(0xaf7c8714,0x320f1769),LL(0x482ba6fd,0x2b5f86a4), + LL(0x1ab9986b,0xcf691cb7),LL(0x7377ba6b,0x42913d71),LL(0x1e47bf08,0x120b4601),LL(0x764b2661,0xfb514e52), LL(0xa140ae04,0x371c0155),LL(0x2e186763,0x94e65b70),LL(0xd5764306,0x5e440f7b),LL(0x7b8a5eeb,0x3411dadf), + LL(0xf0e3e158,0x6c25e519),LL(0xe8898c80,0x46ee66d6),LL(0xec4f9b03,0xa0e9d4b1),LL(0x126c1f31,0xba48d97c), LL(0xbdbf0928,0xb9f96818),LL(0x7f51cb48,0x293ce87d),LL(0x76f09d38,0x077a7420),LL(0xedea4c81,0xc71cb875), + LL(0x9ddd1485,0xfeda750d),LL(0x51c10504,0x987876dc),LL(0x75ec7922,0x4d0253f8),LL(0xc676b336,0xbcc15e39), LL(0xb9370740,0x33d533d8),LL(0xcb8c88ab,0xc5c482db),LL(0xc7c08a75,0x1ff3f223),LL(0x401b00fd,0xccfaf564), + LL(0x6ac9757c,0x6ba93d3a),LL(0xec2c92a0,0xff09b546),LL(0xc5960be8,0x95d3436c),LL(0x69029082,0x90b7e8cb), LL(0xdb6b32e5,0xbdd1e2b9),LL(0xfd47ad85,0xf4d2e43b),LL(0xcb005dbe,0x8923251e),LL(0x662912e7,0xc21368a0), + LL(0x062d205c,0xc7ce2963),LL(0x9542b831,0x1e8f812f),LL(0x818c322d,0x4f8a7915),LL(0xfb678809,0x50073cba), LL(0x0cb91b3e,0xed7b5237),LL(0x60d3fe6b,0x22d1fa41),LL(0x2d690f75,0x3de39063),LL(0xf164ec1f,0x12b2e39e), + LL(0x332f408f,0xa28a0d83),LL(0x6a054db1,0xe6d9406c),LL(0x5ddd64e0,0x67369b76),LL(0x02b21c2d,0x6d671707), LL(0xac42170f,0xb9ad3368),LL(0xe5802ffa,0x5e8f5277),LL(0xd9b4a0a9,0x1b4468fb),LL(0x96c24331,0x0daf8269), + LL(0xc09ad26d,0x976c2f23),LL(0x19c68d38,0xd47afe88),LL(0xd3d8d84f,0x0e96c03b),LL(0x0932b2fe,0xe05b5fd8), LL(0x347fbbbd,0x13931043),LL(0xb0ccc752,0xe0fa842f),LL(0xc75bf745,0x7295ee0f),LL(0xb0aa9d61,0xebaae0dc), + LL(0x6355564c,0xb392d49b),LL(0x887c5a18,0x57e2f166),LL(0x230a278a,0x88b3a014),LL(0x4c194449,0x088e4908), LL(0x43d6af69,0xc6cd309f),LL(0x589a7f7e,0x394445e3),LL(0x031e7c08,0x0610077a),LL(0xa3519f78,0xd05547cc), + LL(0x926e26ed,0x0123b543),LL(0x62d06da6,0xcd430b80),LL(0x0dcd6db5,0xddb182d0),LL(0x8eb6e010,0x724c9bce), LL(0x50a4a597,0x985a2f0f),LL(0x900f2a49,0x35f2427f),LL(0x13cbf187,0xce6136fe),LL(0x1086c2aa,0xc893bdee), + LL(0x07eca624,0xe2410ccb),LL(0xddf9afb0,0xeda92913),LL(0x5bb66033,0x8fc0cfd0),LL(0x0509ffc8,0x0ab7d29b), LL(0xb3d4f10a,0xc063b004),LL(0xeb8cf642,0xed94a955),LL(0xa272ac4d,0xacfb2f14),LL(0xc4ebbf0b,0x10f2c91a), + LL(0x06ea04eb,0x73f6e02e),LL(0x8b97ea93,0xb969e8f8),LL(0x0cd48657,0xa9b27472),LL(0x99264937,0xe1a874ec), LL(0xf794332d,0x34e677a4),LL(0x5ee2daea,0x5e67865e),LL(0xe6886879,0x3fe02b91),LL(0x0f9776ad,0xe113432f), + LL(0x6a2c47d1,0x37567306),LL(0x62feb54a,0xf66df9b8),LL(0x3e07ce56,0xf734ee37),LL(0x659809fd,0x50c4982d), LL(0x9daf8faa,0xe2fa768f),LL(0x8b9fd7c3,0x66088ddc),LL(0x333683c6,0xb8265661),LL(0xdff2a0a7,0xe7dacf81), + LL(0x5e3da06c,0x1e99d6bd),LL(0xbae05a25,0xbd50e15c),LL(0x802d7b40,0x47a0d997),LL(0x193ef621,0x0a25b51b), LL(0x148ee5a3,0x24d3d4f4),LL(0x022a9df0,0x7012618f),LL(0xf68e648f,0xb3777339),LL(0xd7544352,0xcdfb557f), + LL(0x1f912c5f,0x4b0b2d46),LL(0x957515d9,0xddaf929a),LL(0x0ae46856,0x29e4bf1f),LL(0x44e32ab0,0x158b4c85), LL(0x7c48d2d2,0x17935398),LL(0x6f2430bc,0xe4ab6300),LL(0x0d8b24d4,0x71dd7284),LL(0xfc21d7e4,0xd9303af1), + LL(0x450f7f6d,0x816c616f),LL(0x3306df19,0x17875d8e),LL(0x087e86e0,0x7ce8d4a5),LL(0x36251f01,0xa53970ac), LL(0xfc98edaf,0x2037f12c),LL(0xabf72b6f,0xc359a382),LL(0x85130fa6,0x06acf1a6),LL(0xadfe4344,0x08f45064), + LL(0xdd857b31,0xc01e1f3b),LL(0xc9513734,0x92c2263a),LL(0x589327b8,0x562652d5),LL(0x96a1c164,0xa8edd065), LL(0x79f8df8d,0x2cbf8f98),LL(0x40847dde,0x3d5cf771),LL(0x597c0622,0x69b08ee4),LL(0x8a868f2a,0xfff18c4d), + LL(0xf5ad0686,0x28bca3d2),LL(0x2d4eef7b,0xf7992f89),LL(0x3977e15d,0xab10b9cc),LL(0x2db8ef03,0x47912ca1), LL(0xdf27884b,0x1f3e70e6),LL(0x0e9d8efa,0xdd9bb81e),LL(0x279e27f6,0x97a83b6f),LL(0x24daf922,0x47e259fb), + LL(0x124e71be,0x49eb72bc),LL(0x3a6778ba,0x01dba001),LL(0x3be03050,0x8d02baec),LL(0x6dd6c8ef,0xe3571b3c), LL(0x2cc11ffe,0x6e1ffbac),LL(0xf4e2e6f0,0x6d725c75),LL(0xf2b53a58,0x96c31b45),LL(0x97f1634b,0xa0e38dd7), + LL(0x143ce001,0xe8507959),LL(0x8b49cc63,0xad9a9f52),LL(0x8438b0fa,0x950fd33d),LL(0xbe0cbdf6,0x2b294c00), LL(0x937b00ce,0xb2076b47),LL(0x78041498,0x026153a1),LL(0xe9e53d27,0xe958f12d),LL(0xe8f8fad9,0xf49e1124), + LL(0x2bca0ae1,0xb78a5b74),LL(0xcccdc3d3,0x35180dec),LL(0xa97e519f,0x15e4fba5),LL(0xf5b8340e,0xe49dac9d), LL(0xe5978024,0xdbd8ed3a),LL(0xeb105505,0xd181f26a),LL(0x29f57098,0x38364818),LL(0x3900171a,0xd674fe1f), + LL(0xf1bd5803,0x5a2ff729),LL(0xeda23387,0x53de7261),LL(0xf0dc417c,0x7f1d84c8),LL(0x5360fa80,0xa65694a7), LL(0x96ed36e6,0x356e4518),LL(0x406bfd36,0x127a52de),LL(0xde925d04,0xb575a98e),LL(0xc0627c4f,0x35fb44be), + LL(0x471e745a,0xc85f2c69),LL(0x6213d79e,0x1c01e1ea),LL(0x2f5081f0,0x95ea99a1),LL(0xc3367864,0xdb38bd3e), LL(0xd8880436,0x0e8cafec),LL(0xf9c63d09,0x1d59fd74),LL(0x7f875dbb,0xe57b0b4f),LL(0x77084bd7,0xe266c939), + LL(0x2fc1f909,0x0e289c5a),LL(0x86c4fc40,0xece9d225),LL(0x5d79b4b3,0xe0a56fbe),LL(0xd4489041,0x2b96fae7), LL(0xe23c85e7,0x0f66316b),LL(0xadfef0c2,0x2d1a3c78),LL(0x9fbce9cd,0x1aece4ad),LL(0x6c32d32d,0xccd0f334), + LL(0xfb9ba6dd,0x958d7a5c),LL(0xe673275d,0xa0052032),LL(0x7f978d07,0x514ffd9d),LL(0x450b76e1,0x544bbce9), LL(0x6b5201b6,0xeaa25d74),LL(0x74d082a5,0x7528a4ea),LL(0x66609e27,0xa08c8d31),LL(0xda7c6fd9,0x5150d1be), + LL(0x39930618,0x864f5b4c),LL(0xcebb516e,0xe71e7f1a),LL(0xebf1f8ac,0xaeee7fa5),LL(0x0ea827c6,0x6efcad4a), LL(0x74e21dd8,0x6e0f4ecb),LL(0xf33a7939,0xc5311600),LL(0xa4d93fc4,0xdf62f3c3),LL(0x9a18476d,0xd3b62727), + LL(0xc0e1256b,0x0b54f5e6),LL(0x97ba9afa,0xe8987efb),LL(0x41d11c15,0x4b6ea064),LL(0x79b79f0f,0xfed7017e), LL(0x5bd04e40,0x5a6bcf9e),LL(0x8fd3b4bd,0xf3090153),LL(0x82240648,0xa23b5acb),LL(0xb16cf033,0x61d9a8b1), + LL(0xc9fbee1e,0x2feb1706),LL(0xd7e07918,0xfaa4cd69),LL(0x447cba7a,0x28562c58),LL(0xa61a1064,0x727926c4), LL(0x97ac7eff,0x1b32db7f),LL(0x452253ce,0xfd968b22),LL(0x5cdd3217,0x69d1842f),LL(0x26f4007d,0xba7689da), + LL(0x141c8b35,0x16445a64),LL(0x73c61779,0xc11c3101),LL(0x485300ee,0xa5aa0d18),LL(0x1cc02bf0,0x531b6de1), LL(0xc4efeb2c,0xf8b94155),LL(0xd015a9c8,0x83632663),LL(0xcba18b7f,0xc369b3ce),LL(0xc29e0f9b,0xe11b3ef6), + LL(0x903ca95b,0x1a5e9bf2),LL(0xa50cb332,0x2d9aefc6),LL(0xb29ce693,0xb5670264),LL(0xab1d7b7e,0x806d08ac), LL(0xc9107eac,0xcbdfdf28),LL(0x6cdf12ac,0xa8086243),LL(0x903d5999,0xe7d9c315),LL(0xc079d951,0x4379820b), + LL(0xbf1edf4c,0xe789ecad),LL(0x47bc7752,0xec086811),LL(0xc2fc8798,0xeea2eeb8),LL(0xe031a96b,0x763183e0), LL(0xf9a6bfaf,0xc7daf0b2),LL(0x4b957cf7,0x1a2a7ffb),LL(0xbf2d2e7d,0xa483c7c8),LL(0x58ff7f9c,0xf96921fc), + LL(0x574ee010,0x41386185),LL(0x2780c649,0x62e6a1d8),LL(0x60f2516e,0xdec553af),LL(0x7a04eb11,0x5b091537), LL(0x67eb90c5,0x1b53e9dd),LL(0xddfda333,0xc390a23a),LL(0x480568aa,0xdd4e7c6d),LL(0x59ccbe61,0xd6c1e8a8), + LL(0x9107901b,0x1c6fd7a9),LL(0xa211d116,0x5dc4a41e),LL(0xaf1b78a8,0x597e94e7),LL(0x53afcb6a,0xe72da34d), LL(0x74512c24,0xbc364db7),LL(0xb2811e91,0xc26a8fb9),LL(0x290469b1,0xfdd39d7f),LL(0x20612535,0x84515392), +}, +/* digit=29 base_pwr=2^203 */ +{ + LL(0x1539cf31,0x7fe996a0),LL(0x0ded7c6e,0x4a3f729a),LL(0x3016f614,0x86f1f299),LL(0x86cb9163,0xc3d44e18), LL(0x558fa36c,0x96984531),LL(0x369c89d6,0x58e8bf05),LL(0xf9ee923f,0x287da114),LL(0xec271fbc,0x2032e984), + LL(0xd39207ad,0x91b8579d),LL(0x0b1fe916,0x6f62c725),LL(0xd89e01bf,0x0f1599ac),LL(0x4d1e5843,0x8d9bb86d), LL(0x726e38d1,0x348b90d4),LL(0x52a8c6b9,0xb824a1ca),LL(0xb1d2f6f4,0x984d9309),LL(0x431ec12e,0xefa485b7), + LL(0x9d616a5c,0x24cafa66),LL(0x4c9d0ea8,0xc1c7445f),LL(0x90bee7b6,0xf733e085),LL(0xd251d2ba,0xa2f3ece3), LL(0x66aeba6c,0x6e422a45),LL(0x37c1337f,0x35e99b16),LL(0x6d4f8d03,0x52d0fdf7),LL(0x79c92672,0xa043420c), + LL(0x76ac1925,0x99725607),LL(0x3442fc58,0x086449db),LL(0x2e311e74,0x8dbab920),LL(0x7ea25561,0x29dee69b), LL(0x19a7cd6c,0x5a62b6ee),LL(0x0d0dd5a0,0xba38cc4c),LL(0x166d0ff1,0x779279e5),LL(0xf48b3dae,0x0eef53cc), + LL(0x0f82c6a6,0x0463dcaa),LL(0x11d7d6d8,0x75dfc96d),LL(0x6c100d92,0x61f05e7b),LL(0xe13eabb4,0xa118e548), LL(0xdcdf06b5,0xcc77e3c8),LL(0x6ac25960,0x902d37d6),LL(0x347d7116,0x967d9993),LL(0x9ae33561,0xd2828650), + LL(0x8c7c6d89,0x955b7840),LL(0x42c2555f,0xbfa78bc8),LL(0xa69c3165,0x8c56ae3d),LL(0xe33bb1bc,0x72b20e72), LL(0xd1aa6416,0x686870b4),LL(0x7db03cdd,0xf000040b),LL(0xb0a0fd40,0xd25b16a9),LL(0x36815f1f,0xeb89e932), + LL(0x349b549c,0xb3e5e912),LL(0x086c4e74,0x801f788e),LL(0x0082ae92,0xafb9ea4f),LL(0x0d740026,0x4e8f27a4), LL(0x05f4a6ac,0xc4f8285a),LL(0x0adcd58c,0xefea5297),LL(0xc52f8c21,0x9d5b6ae5),LL(0xe97af4dd,0x92622a7a), + LL(0x79f34a8a,0x39b43935),LL(0x30046435,0x61acf55f),LL(0x3f05fdb1,0xf0a23fe6),LL(0x0c4fa7ff,0x7d6baee1), LL(0xe2daf735,0x253f62b6),LL(0x2948637b,0xe370ead8),LL(0xd84e6206,0xda57c16a),LL(0x0dd22ad3,0xf19ffe09), + LL(0x95bf2003,0x701acab2),LL(0x9dff6efc,0x50e4e10a),LL(0x43b95430,0xe637bcf0),LL(0x85050cbc,0xac45cd3e), LL(0x80639e4d,0xc2ebff64),LL(0x3056f603,0xe5af1fb5),LL(0x08b17132,0x302791d5),LL(0xed517904,0x87775ac4), + LL(0x4263a566,0xfe64ed1e),LL(0x1d5e8f55,0x735bbee4),LL(0x8294342c,0x9ac61915),LL(0xd4904935,0x0f522e5a), LL(0x7c1e11f4,0x2ee883b5),LL(0xf0c237f4,0x0a2ce30f),LL(0x8d955086,0xf4a7157b),LL(0x022dc2db,0x7ec0462e), + LL(0xca391b0b,0x562fb75b),LL(0x3bb1efd6,0x13030aac),LL(0x347f44fe,0x305d9831),LL(0x94b2615b,0x9f70c1ad), LL(0x4162ff22,0xaaf935f4),LL(0xa68d160e,0x2b20f047),LL(0x39222d1b,0x30d52a97),LL(0x6551642f,0x051223b1), + LL(0x39620daa,0xae65a5c7),LL(0x6f7c078f,0x8ef6f93f),LL(0xb0177db8,0xb06d52bc),LL(0x68fdf535,0x915cdd08), LL(0xc5183222,0x0070d150),LL(0x7817a2ae,0x2b6495cd),LL(0x0b194f0b,0x3ce47614),LL(0x513bfdfb,0x2eec6acf), + LL(0xf3dbd34d,0x725dbede),LL(0x3621fc75,0x01c4412a),LL(0x3c07f048,0x17bd68de),LL(0x62e735eb,0x117df57e), LL(0xb249c407,0xb1596c6d),LL(0xd46c55c4,0xa878f56a),LL(0xb8aa0cb4,0x33385670),LL(0x800ec887,0xc7faa80e), + LL(0xd7daf836,0x2cd2814f),LL(0x0d616922,0x877b72b7),LL(0xdb066012,0xea73ca1b),LL(0xb0d4159d,0xbe336c7b), LL(0x0f8fcd76,0xb993b07f),LL(0x8a593562,0x5fdceaba),LL(0xf691ec19,0x716595fb),LL(0x8e68e3c0,0x51a77f61), + LL(0x7a7c18db,0xe9e4cdfe),LL(0x7b4f69b7,0x967d3575),LL(0xa9a434c1,0x6dd350a1),LL(0x00c79ba7,0xb92cdef9), LL(0xa6bb0f93,0x7a762493),LL(0x8158ad36,0x6c18cdc2),LL(0xc529ecfd,0xa0bd83e3),LL(0x3962f96d,0x98363c59), + LL(0x1d63aa7f,0xd80f45a5),LL(0xb3b32da2,0x8d5eba75),LL(0xa4708858,0x0ef233df),LL(0x52161c61,0x74c3f4f7), LL(0xe6420de4,0xfa9bfe6b),LL(0x97dd86d5,0x96c0c501),LL(0xcfce233b,0x28e6827b),LL(0x58e74d63,0x035cc9a9), + LL(0x7948782d,0x9ba64bf4),LL(0x80d9ce1a,0x5e5b7c72),LL(0xf51df862,0x7b9435db),LL(0xb4dd2421,0xe74ab6e8), LL(0x60954c75,0xb0d704db),LL(0x0b59ae5b,0xd31c5145),LL(0xd99ba307,0xe0ff4660),LL(0x986bd82b,0x1a3800fd), + LL(0x509a0a50,0xe7e06ab7),LL(0xe4539885,0xbdf63778),LL(0x16ddb433,0xf888320f),LL(0x18e18998,0x0f108304), LL(0xfa834b14,0x27e7ffd6),LL(0xc68b9434,0x16de9a71),LL(0x4d360436,0x53a12e2c),LL(0x5e110b02,0x5ad2c986), + LL(0x3cf53e0c,0x3439443c),LL(0x1d65a7a3,0xfeae29b0),LL(0x78ad7d78,0x1e7774f6),LL(0x6fee368c,0x0c79fb01), LL(0xe4faf3ee,0xbec71de1),LL(0x263868e8,0x1a88f3e5),LL(0x90e91a8c,0x975d8381),LL(0x0f999c60,0x69c5a65d), + LL(0x8b884857,0xbd3360d8),LL(0x31b7c579,0x04190413),LL(0x142cc145,0x40dd9229),LL(0xdad0b2df,0xb5faab94), LL(0x52df4831,0x3e7d7921),LL(0x228bf67d,0xcf5bd1ed),LL(0xc4980085,0xd8669635),LL(0x1c71903d,0x094b8973), + LL(0xc4abb028,0x493a7a3d),LL(0xb4ab8e35,0x0e1a8fac),LL(0x017aa5f2,0x26094ca2),LL(0x021476cb,0x94fcb8b1), LL(0x4abf3bcd,0x57f101f9),LL(0x2d7f12a8,0x1ac2c125),LL(0x2e42789c,0x575259d9),LL(0x22471eb3,0xa64a4a4b), + LL(0xe1c00144,0xcc02304d),LL(0x754734b2,0x6269dfb9),LL(0xf14fbc81,0x72e7a183),LL(0x2a05caa2,0xd92a5b1c), LL(0x15efc2fb,0xd593492e),LL(0xd8dd458b,0x1ace7dca),LL(0xaef2ae81,0x576b4bc8),LL(0x351b397e,0x6de6a2db), + LL(0x656cf9ab,0x73f13b48),LL(0xaee7e01d,0xc18df1c9),LL(0x560355e7,0x30fb5155),LL(0x9ad059d5,0xd2c9a0ee), LL(0x5e5e0c7c,0xd9f89936),LL(0xf0a6c9d7,0x5d0a2cbd),LL(0x58fa9be9,0x3c2c497d),LL(0x6ac61a2f,0xe6c6fcf2), + LL(0x35607bc4,0xf7ec89e3),LL(0x9946bf52,0x17ca00ca),LL(0x180c8bd8,0xee46be5b),LL(0xb2873893,0xd29d5eb0), LL(0x97487b3c,0x348ac939),LL(0xfeef78ce,0xc18f0ceb),LL(0xf797cce5,0xfc648dca),LL(0x442148d7,0xe2467e0c), + LL(0xda6dbaf6,0x8e201ee7),LL(0xabd0490c,0xc1a93ee4),LL(0x4de7c210,0xfd0007bf),LL(0x083ffce0,0x02eccb8b), LL(0x97844c8d,0xbba82bbb),LL(0x2747a430,0xb7ff954c),LL(0x18fb5f29,0xb05f0584),LL(0xee7c263f,0x35a29cf5), + LL(0x25282da8,0x5fd84efd),LL(0x7682db7c,0xc1fc84c1),LL(0x4bb291e6,0x8422b56a),LL(0x66a56078,0xce379feb), LL(0xe7f045a0,0x1c3c581e),LL(0x6b3f89ac,0x8f42985d),LL(0xa6b2ba59,0x112839e9),LL(0xc2a7b29a,0x3f0c7269), + LL(0x79bd3046,0xecdadb6f),LL(0x9617ff6e,0x669f559b),LL(0x68928ca9,0x242bb14d),LL(0xc19cafcc,0x28e2b0cb), LL(0x81330593,0xfb7d8954),LL(0x9fbf8665,0x5c3741fd),LL(0xc3b93e86,0xaf49e83a),LL(0x32822548,0xd60ecf7d), + LL(0x7cfb37cd,0x4bf36759),LL(0xdb7af2ed,0x94b0c7f0),LL(0xebf8b461,0x2f1bebf6),LL(0x073e5c18,0x90ebc9c7), LL(0x3774e905,0xe431c793),LL(0xe03265bb,0xb8a4bc2f),LL(0xfabb629e,0x0bee0825),LL(0x84557170,0xbd1481dc), + LL(0xe1a010a0,0xa2257b58),LL(0x4688bb89,0x43f5451c),LL(0x91b96371,0xb87a5ff0),LL(0xde3c7b25,0x445dd02f), LL(0x51a0964c,0x44c0c080),LL(0x9afdcefa,0xb0c3190e),LL(0x0044d258,0x14cc65ad),LL(0x374fdd44,0x8c500b3e), + LL(0x51b07812,0xed8d840f),LL(0x76530691,0xd414a4a2),LL(0x1468ef8d,0x9db9d138),LL(0x292b3870,0xfc6b7434), LL(0xc9d7ad96,0x80b66797),LL(0x2a9c1e99,0x81e74eb6),LL(0x9e92f64b,0x48657d9a),LL(0x4c851ddd,0xf5c60075), + LL(0xd99d5cfe,0x08fa89be),LL(0x4db4addf,0x78b1f26e),LL(0x3523ead9,0x03237177),LL(0x6a281494,0x0147af5c), LL(0x916836b0,0x8db3952a),LL(0xfd365650,0x0632b102),LL(0xccb3f2f1,0x3854a8e9),LL(0x586ad427,0x5048486c), + LL(0x17a86e18,0x22de9979),LL(0xbe029111,0xe2ac2321),LL(0x35cc5a17,0xbfd34397),LL(0x525e13cf,0x7a93461f), LL(0x5122d6f1,0xd433542c),LL(0x833982c7,0x41d2d9de),LL(0x8ec24d27,0xe9f1f29a),LL(0xf3b99d58,0x4ae251f3), + LL(0x10adb458,0x7234dd24),LL(0x88379ef5,0x0e4b6567),LL(0x748dba5d,0x3007df15),LL(0x35103772,0x1485ef01), LL(0x29c2382f,0xe21a9dc9),LL(0x6b6c1c8d,0xcf7e0c24),LL(0x30550c0a,0xf8a71820),LL(0xb797de2e,0xb30e5c0f), + LL(0x03705145,0xbe136119),LL(0xf94aadc7,0xe6d1f720),LL(0x255f5297,0x38ce1872),LL(0xc3143f58,0xbbba4793), LL(0x0984e265,0xda5345fe),LL(0xd895e0d7,0xe93989d6),LL(0xcaab40a3,0xb7392b18),LL(0x65e754fc,0x4a586963), + LL(0xa3afd381,0xb3e88445),LL(0x693ad961,0xa6cbab0a),LL(0x257d56dc,0x64d51359),LL(0xbbde137a,0xf9e70fcc), LL(0xadd016b1,0xa33872fa),LL(0x7344f234,0xd1d263d2),LL(0x24ba41b2,0xc2d51210),LL(0xe4ab65cb,0x8c9c830c), + LL(0x175b4039,0x3b47563c),LL(0x116b2025,0x53521dfd),LL(0x1a9f1cca,0xe4f3aa89),LL(0xe7cb1d2b,0xcc73485c), LL(0xbf58fe30,0xa6ca61ef),LL(0x531a2b6a,0x5d50e15d),LL(0x22611c31,0x71cfdb41),LL(0x61e3d46a,0x0dc15533), + LL(0x479074ba,0xb363c60b),LL(0x24cb405d,0x2a4d8f4c),LL(0x646b7f0a,0x3d3bee13),LL(0x5571af63,0xdfa9194c), LL(0xee76521c,0x951a61a7),LL(0x65eda1f1,0x67466ba5),LL(0x69ebc7ea,0xe41d33b8),LL(0xd4f4848f,0x8b6c992d), + LL(0xa5003eaa,0x3572faac),LL(0xabf54df1,0x01e36500),LL(0xac6f3af7,0x6622f12f),LL(0x0a8bb122,0xb5d7c17f), LL(0x240286b1,0xd1fc1b99),LL(0xad968ede,0x519d52ce),LL(0xece28bb4,0xcd1e7d0c),LL(0x93f0b097,0x64ffc699), + LL(0x25af7d81,0xb18d51f8),LL(0x19820fb2,0x8d0bb08f),LL(0xaa344723,0xe9f45919),LL(0x5f9e0501,0x558f18ea), LL(0x07cc79dc,0x56eff076),LL(0xd5fa9783,0xf171e880),LL(0x8be7f1fe,0xd5fb41f3),LL(0xd6fe9afc,0x19a34620), + LL(0x7d8a042a,0x74c2696b),LL(0x54230ded,0xcf4908c3),LL(0xdb32943b,0x98a870d8),LL(0x52f33e07,0x181cbe5c), LL(0x93709d8b,0x4d9d1172),LL(0x2b2b7939,0xb800c292),LL(0xa8920f60,0xd5a1fb7a),LL(0xbf7df622,0x8d0a7725), + LL(0xe9415cf4,0x83a370cf),LL(0x62a4ff4d,0x9f24d1e1),LL(0x0a6ff7cb,0xca33914b),LL(0xda1d1aaf,0x2576f2d8), LL(0xb4793101,0xbb2668bd),LL(0xca990b4f,0xb576672c),LL(0xff1d3988,0xfa165b5f),LL(0xba8c9e2c,0x273d5b18), + LL(0x720a5b3f,0x13594ae5),LL(0x2e65454c,0x38c4e04a),LL(0x55d0d903,0xc5b55d98),LL(0xf72db301,0xfabeb890), LL(0x2f3deaa2,0xe471f1d5),LL(0xc5ade08d,0x3f8d39f5),LL(0x6baf9d1a,0xe08486a4),LL(0xe5140d3b,0x77c6d30e), + LL(0xd23d4e09,0x828db96a),LL(0x84dcac15,0x2de4856d),LL(0xa6ac856d,0x313c7f8b),LL(0xfe7dea98,0x7c9f671c), LL(0xfebe9e72,0x4d6c14a5),LL(0x6014be55,0x185ac4e6),LL(0x5ed64e23,0x42809988),LL(0xbd6440fe,0xdc9395a1), + LL(0x24108055,0x10169080),LL(0x38961403,0xfe0d9cb0),LL(0xceb87d29,0xf88e6a48),LL(0x69618944,0x0365ca2f), LL(0x9fb59ec9,0x1504647e),LL(0xa4aadbb7,0xb6486b3b),LL(0x22ef3214,0xfe8701af),LL(0x5f56d985,0x4c895bc1), + LL(0x2e1e68d2,0x6fdc6cb6),LL(0x13f07350,0x0689b22b),LL(0x6d62eb1f,0xba94416b),LL(0x98996d72,0x5a2fcbba), LL(0x04b2afed,0x2ca2a2e9),LL(0x0bf61008,0x5b62c764),LL(0x37f4d486,0x30074e57),LL(0x31865287,0x4e02be2a), + LL(0x6842ab16,0x401cfb89),LL(0x5b2eb304,0x440fb52d),LL(0xd22eaa61,0x3245fd38),LL(0x373f201e,0x252120e8), LL(0xb2e724c9,0x4d253f5c),LL(0x27e5b5e4,0x9428d6be),LL(0x6785ee9c,0x00d4c598),LL(0x56208d4b,0x0b7fc5f9), + LL(0x92310137,0x4426665d),LL(0xfee8da95,0x75b96cd3),LL(0xb561c6d8,0xaaaac6c3),LL(0x3f09e1d9,0x0784a3c5), LL(0xdcac620b,0xac78c064),LL(0x119b8d90,0x49dd5f02),LL(0x57e5caf4,0xf1f5ebf2),LL(0x0eb825e2,0xd8a9fa2d), + LL(0xbfb6a2fd,0x3553633a),LL(0xa0c9ce9a,0x06533515),LL(0x04c71728,0x6decd9e8),LL(0x980b29bd,0xcbc0df55), LL(0x31826d15,0x17b1b559),LL(0x1c5cae17,0xc96ed7d7),LL(0x88cda83e,0x24f58740),LL(0x0c69f40f,0x9e2ee1bc), + LL(0x9e5604ed,0x138ebf0f),LL(0xf229f097,0x0577f4c2),LL(0x9762825f,0x0a44f975),LL(0xdd085e55,0x113b8639), LL(0x73acc59e,0x4be02fee),LL(0xada7a17d,0x7829f288),LL(0x84fb30d4,0x086bd736),LL(0xe5338eca,0xb2f120ee), + LL(0xfb778d2f,0x21701393),LL(0x6441fd75,0xd46bc61e),LL(0x135b55bc,0x466671de),LL(0x51c0f145,0xee1d9cbb), LL(0x6d9ce27c,0x7a7bce67),LL(0x26d82b1d,0xa8c9b1e0),LL(0x4c87bd6e,0x250bee03),LL(0xd3829702,0xd6b02f71), + LL(0x0e555d98,0xf14b3748),LL(0x6f775e78,0xf795e62a),LL(0x8f46de18,0xe9a4e4ac),LL(0xc5ab76ef,0x773bd32a), LL(0x5264cae9,0x4f2dcc68),LL(0xc63a6419,0x453b627e),LL(0xe441c6d9,0xc3aeddd1),LL(0x3027b8f3,0x669500b7), + LL(0x88d601e5,0x06b93413),LL(0x76c4483c,0x0e996e87),LL(0x00eb0c21,0xe2ff3a6d),LL(0xf4690674,0x86ec3a73), LL(0xe9f82ca7,0x673d123e),LL(0xbf611d0c,0x952c2765),LL(0x3676497d,0x26ed9a68),LL(0x9d29cefc,0x2c9c0049), + LL(0x2ae73af6,0x0b8c12fe),LL(0x6455c8e1,0x555b0ab3),LL(0x4804b006,0xd2f49f03),LL(0x02e829a0,0x408a22bc), LL(0xf2832c53,0xde2a59cf),LL(0x96a54082,0x01332439),LL(0xbbd38f9f,0x11dc4ab6),LL(0xa24764b5,0x0248fd93), + LL(0x1374972a,0xc7774c12),LL(0x92764b41,0xc7355966),LL(0x23143092,0x31c10ea5),LL(0x9070137f,0xe89d9f88), LL(0xa504d91e,0x7d074406),LL(0x70b7aa8c,0xc5210379),LL(0xea799dd8,0xa67904f8),LL(0xe7b02c04,0x81e6516b), + LL(0xb08cc6fe,0x18daf05f),LL(0xf59d49f6,0xfbbd3061),LL(0x93a78581,0x5429b117),LL(0x1906df65,0x795a4465), LL(0xd51a7866,0x643c37e3),LL(0x663a17b1,0x69b8118d),LL(0x3e8a2c53,0x5e385989),LL(0x50f007d2,0xbc18c2ea), + LL(0xb616aa15,0x4adec20a),LL(0xea085548,0x99f77e49),LL(0xc01b9a33,0x9108c205),LL(0x6ef3bcef,0x298fbeb1), LL(0xefd8ba0e,0xdf1a8d2e),LL(0xe9756e7b,0xf0ec9492),LL(0x7ff5fbc3,0x4fd33389),LL(0x03ac8371,0x122a6bfb), + LL(0x90d66732,0x7d053c8c),LL(0xf9b2909f,0x83f26571),LL(0x66cba4b6,0x350dd6d0),LL(0x40d0d47d,0x8c71c17a), LL(0x4d0be34a,0x3bf85053),LL(0xe11bd49f,0x91ae4f59),LL(0xa22c648f,0xf8a38b41),LL(0x58abaaea,0xcb96620e), + LL(0xa7fabcf5,0xa55cee46),LL(0x79c8fbce,0xd16a8b92),LL(0xcbf048bf,0x26ad700b),LL(0x47bb5f1d,0x83b3ce11), LL(0x6b310318,0x31a48f46),LL(0x00612ef3,0x13a7f781),LL(0xa18db234,0xcd840f2a),LL(0x30611c74,0x3be2a7a8), + LL(0x2b1c0447,0xbdf37cb2),LL(0xfe71722d,0x7f572382),LL(0x25535e86,0x085b3566),LL(0x3f5b9cc2,0xb5b43063), LL(0xdee66228,0x7c7cff51),LL(0xb676fd6e,0xe29236ae),LL(0xab0cdb1a,0xf0c0105e),LL(0x06b52812,0x0adc9d6e), + LL(0x483baf0f,0xc9e6ca97),LL(0xf9bf5055,0x09b063bf),LL(0xfc5a407f,0x8c4c6b2a),LL(0x99a6f540,0xe29cb487), LL(0xcb9a2758,0x18b72239),LL(0x8a5ed308,0xa0ae9f10),LL(0x6e402657,0x2a2cb603),LL(0xaf6f4d14,0x9c7f52cf), + LL(0x70ca9046,0x0ed032e7),LL(0x59cac9e9,0xe4b0b1d3),LL(0x18462dfd,0xd12c87b0),LL(0xbb8860dd,0xa25a23ee), LL(0x507fa3d9,0x6c9a8676),LL(0x218f325f,0xc6bb96c4),LL(0x2386b7b2,0xe141bbb8),LL(0xd4183c77,0xf86a72d0), + LL(0xaece96b6,0x35537f86),LL(0x63f7e1fa,0x83aa1df9),LL(0x7ac4aaf2,0xa39ab4aa),LL(0x8a202375,0xb8d0ffa6), LL(0x86514cd8,0xd916df09),LL(0x11902747,0x71f905b3),LL(0x8c744f32,0x6388c2ee),LL(0xa5431096,0x6282e1f5), + LL(0x7c388413,0x14bfa765),LL(0x7b4437aa,0x95dd04d9),LL(0x3c39d7c3,0xdf6ca849),LL(0x0c2ddf38,0x85cb1123), LL(0xc401529c,0xf7766d86),LL(0x99a4d031,0xe33416a8),LL(0xb874ace4,0x5c507c3f),LL(0xdad6fcb0,0x0e3a42b6), + LL(0x47920742,0x402da460),LL(0xb45f94cc,0xb142d6ef),LL(0x76608dd4,0xc2d613e8),LL(0x5d75d4b5,0xa2c06cdd), LL(0x3c564ff4,0xa1951bc5),LL(0xad1d5ecd,0xe60f126b),LL(0x702135ad,0xa634e765),LL(0x8df44695,0xa5a56a6e), +}, +/* digit=30 base_pwr=2^210 */ +{ + LL(0x9e2207b4,0x234b8c7a),LL(0xf7ee9f62,0x1f724f30),LL(0xc58e21b6,0xfa908ca2),LL(0xa74296ae,0x55587744), LL(0x02911ae1,0x7dbe9130),LL(0x9d3af02e,0xc2075433),LL(0x0f3955a1,0x505b724b),LL(0xcaeced66,0x480e1a92), + LL(0x446d9f66,0xb20f6128),LL(0xc354b5a1,0xd6e06b14),LL(0x63558aac,0xa72d287d),LL(0xae68a8fd,0x4819be29), LL(0x205fbdf2,0xb024c324),LL(0x210927f9,0x2fca94e7),LL(0xbe658f80,0x74798be7),LL(0xef07c046,0x618e07f1), + LL(0xb35a8c3d,0xfba715fc),LL(0xed1beba8,0xc2548193),LL(0x2ceb663c,0xb956c6dd),LL(0xaacafe85,0x13d4ddbe), LL(0x30a29cc3,0x2f8275b5),LL(0xf51b39ef,0x10432e15),LL(0x2509b2d0,0xd6c9277c),LL(0x849b946c,0x4ee0d4c3), + LL(0x54b01bbc,0x547ba946),LL(0x055d4821,0x7c56c36d),LL(0x05575f20,0x8e933620),LL(0x3a621cf4,0xaec65be9), LL(0x46287937,0x820b96df),LL(0x733c67e7,0x35cea883),LL(0x58cf3e05,0x30366a3a),LL(0x2da39773,0x2580d565), + LL(0xba4417ed,0x7717c42f),LL(0x654c1086,0xb2d66fc7),LL(0x57503cd8,0x07fe918e),LL(0x3cacf74f,0xf9385159), LL(0x63063029,0x157d9081),LL(0x659034cf,0x79c84c08),LL(0xa8048cb9,0x02976610),LL(0x03e81417,0xef822006), + LL(0x22e489c6,0x5fb5dd4d),LL(0x81e167e9,0x9a06d9c2),LL(0x6b974c90,0x83fc248f),LL(0x7110dca6,0xb78cab72), LL(0x370ff66a,0x73f8f311),LL(0x3b61d20f,0x8c5049eb),LL(0xc8516e05,0xaac47edb),LL(0x53f0201b,0x2ceba50d), + LL(0x0b93fbc7,0x6679dc5e),LL(0xa560bd27,0xf4457919),LL(0xb1acadc9,0x2561bfca),LL(0x46708164,0x338fbb6d), LL(0x8b9cfd27,0x9f407621),LL(0xd3123732,0xe806c1e6),LL(0x7f24a161,0xaa1eafc4),LL(0x68e6650b,0xbee3f4a1), + LL(0x5832cd6c,0x453b6181),LL(0x985e90ba,0xc002e337),LL(0x6414f686,0x4b33afde),LL(0x8511fd45,0xf9ab29e9), LL(0x6fb9a688,0x067f0972),LL(0x7202a1b3,0x7db6e14c),LL(0x73a881ab,0x0c15b6e9),LL(0xfad10660,0xc8c324e0), + LL(0x87d9f927,0xa997a6d2),LL(0xacd2f107,0x62307f24),LL(0x9c80a742,0xed7b48a5),LL(0xa7c81e7e,0xecd33ae5), LL(0xefa00a94,0xcf05c763),LL(0xd9ee5aa7,0x38844b0d),LL(0x214b293f,0x02e0b05d),LL(0x8a8a510e,0x732e776b), + LL(0x6c929e53,0x784cd909),LL(0x56a33da2,0xe436e294),LL(0xce9e06d2,0x68eeb727),LL(0xfce7e2f0,0x637890b3), LL(0xc3fde38c,0xc0681a1c),LL(0x76dda134,0x9cb729d9),LL(0x5c333ece,0xaa69eb97),LL(0x48eed8a6,0xe138a680), + LL(0x505dc701,0xd53cbd01),LL(0x6a64c3d1,0x413de346),LL(0x3170a5bf,0x91f6cde9),LL(0x8489b5fa,0x58ffdfd9), LL(0x5c584a48,0xcc0b89d7),LL(0x167f28de,0x74f8ceed),LL(0x8c878c91,0x250fa9f7),LL(0x630adfdb,0xeb960a79), + LL(0xc9489dcb,0xe43ed412),LL(0x112d084b,0xcec053a5),LL(0x664c7cd2,0x0fd4fe42),LL(0x82a917f7,0x48ee06f4), LL(0x83cd65f5,0xc5de1970),LL(0x5569c42f,0x3a176578),LL(0x4f876110,0xf24508f3),LL(0x0a415bc4,0xf350374b), + LL(0xc63aa8ad,0x9c2b11c2),LL(0x7a51c0cc,0x6ac1ae12),LL(0xe1db428f,0x75acd0d7),LL(0x19800684,0x9e391227), LL(0xb1050426,0x4f89e9c6),LL(0xdaf99eee,0x099d97cc),LL(0x1ffce97f,0x27a19ad0),LL(0x3c038d77,0x05fad057), + LL(0x8597d09c,0x96d6c678),LL(0x1e1d8b57,0x38f6336c),LL(0x6330ace7,0x1f945bef),LL(0x613f9faf,0x9d627bbb), LL(0x19176cb7,0xc0e7f21b),LL(0xbb9db710,0xf0e09be3),LL(0xc650133c,0x16b06011),LL(0xf3673d7c,0x8629b975), + LL(0x67115575,0x8c8230b2),LL(0x9c9f30c0,0x9bfc55b1),LL(0xce93fd71,0x132d0e07),LL(0x511c5947,0x08e4736e), LL(0xfe881630,0xd54a098e),LL(0x98efa501,0x8ec67a85),LL(0x2267db00,0x72975dc7),LL(0xa338290d,0x3d6fc706), + LL(0x55ade88b,0x0c891082),LL(0x4b212e85,0x0525b501),LL(0xb61362fa,0x9ede010b),LL(0x881eecac,0x52f3d088), LL(0xbc6f0ae4,0x49957b6e),LL(0x1659701d,0x25fe7263),LL(0x07b76f45,0x41e9b7f5),LL(0xbda77d42,0x5f2ad664), + LL(0xa9c418c4,0x5bdcb490),LL(0xe500a527,0xd0e2c38d),LL(0xca83fada,0x0af29f6b),LL(0x62273db6,0x1f75b1f2), LL(0x9e857e57,0x8454f751),LL(0xb9e79612,0x3fb816d1),LL(0x6412b5f3,0xbe3040ae),LL(0x843ca984,0x99535136), + LL(0x3257f19d,0xb26ec8a8),LL(0xe54dd785,0xd32dc622),LL(0xf8698ab5,0x0c52e874),LL(0x79183316,0xf9a60af5), LL(0xf7f4496a,0x38575d53),LL(0x3d5cd0de,0x33adfd1e),LL(0x7f502017,0x2133f4a1),LL(0x3e8676f8,0x46c09393), + LL(0x3fb4c7fe,0xca8a5a58),LL(0x328ff257,0x2ad58826),LL(0x13b8d08d,0xd9264875),LL(0xdc5a845a,0x661ae2b2), LL(0x49a408d3,0xd2dcaa06),LL(0x85c21e84,0x9ef164f8),LL(0xb7819b61,0x55efaf85),LL(0xf504c32a,0x9488bb1c), + LL(0x9bb777fc,0xb571518c),LL(0x82415187,0xf369c391),LL(0x2720284b,0x2d7c5dd9),LL(0x4eec41cc,0x6feab634), LL(0x24ecd0be,0x2522d5db),LL(0x00338736,0x1fca0d97),LL(0x20145279,0x74416105),LL(0x0496e925,0xf4492e1e), + LL(0xbc33b547,0xa62e09fc),LL(0xae063835,0x9434475a),LL(0xa139b769,0x51edd69f),LL(0xe5166a9d,0x17bbe224), LL(0x1b4b6c84,0x6ecb0a02),LL(0x97968c70,0x16439490),LL(0xbc8aa671,0x75af0456),LL(0x3b4411ff,0xaef056ab), + LL(0x05cef121,0x686b7714),LL(0x078f4500,0x5ad6bdf3),LL(0x072e70eb,0x56df858c),LL(0x254c0917,0xa0fc5e6f), LL(0xc15bf9cf,0x1a99de09),LL(0xd008aacb,0x8aeb587a),LL(0xb900d652,0xba2d8c53),LL(0xad0f69b6,0x60eb5d0c), + LL(0x10b27762,0x27098ff8),LL(0x13264ed3,0x33329ca9),LL(0x887e3f40,0xffceaf40),LL(0x930df9ef,0x854b8284), LL(0x7d5627be,0xdda913a8),LL(0x35e9823b,0x8eb94d64),LL(0x2eb9e9bd,0x94c527fd),LL(0xf57b9f74,0x18335b1b), + LL(0xd193a526,0x3c44dac0),LL(0xe2b2d54f,0xd0717099),LL(0xc4c67d11,0x65624fb4),LL(0xbccedad8,0x04aa7033), LL(0x31470c52,0x0c522fac),LL(0x33b05d54,0x08eb33ca),LL(0xb4565e57,0x940e0693),LL(0x7be56370,0x7e2fd553), + LL(0x12e206ff,0xf722793c),LL(0x155d3d02,0xb57e1f23),LL(0x94fc6ce3,0xd68be193),LL(0x22d4815b,0xb0f3606c), LL(0xb62d5741,0xdaf995a9),LL(0x3e8f5df5,0xa7d19980),LL(0xe4631583,0x7bcdb661),LL(0x5a97dc7e,0x013193e3), + LL(0xfc73494b,0x8210be46),LL(0x57e10efc,0x508817ee),LL(0x126f2466,0x7b6b8da2),LL(0x177bee35,0x2f3b0ec6), LL(0x48db1eef,0x5ceb71e0),LL(0xdc62b6bc,0xd989d9c3),LL(0xf78fac92,0x2cc38cb9),LL(0x955ba5f4,0xcd2a009f), + LL(0x453b668e,0x65a74191),LL(0xc081b447,0x40e9dc38),LL(0x8c3fdf2c,0x48eb63bf),LL(0x5763071a,0x7845cf66), LL(0x787754ca,0x30d9b771),LL(0x8783a05a,0x10b3729f),LL(0x6ab570d9,0xf8090b3b),LL(0x502b3558,0xc1dfbde1), + LL(0xbe4d36ec,0xa568f5d0),LL(0x28e952fe,0x1e9e5393),LL(0xeaad6b24,0x768113f9),LL(0xa8fbede9,0x2bc798fc), LL(0xaaa9010d,0x2c787f9b),LL(0x761834ca,0x32cbc77a),LL(0x032c25d8,0x449e55f7),LL(0xca6216dd,0xe824a5bf), + LL(0x17ac1130,0x3beaec52),LL(0x5e9d1aa8,0xcc28c64b),LL(0x3af8c635,0x355d68bf),LL(0xd3d1adaa,0xcd12e443), LL(0x6c2030d8,0xa456daca),LL(0x62427eff,0x0dfe5bbb),LL(0xf69e2484,0xae45325d),LL(0x7245c979,0xfc2a90ab), + LL(0xa008b446,0xc34f38e9),LL(0xaac355e0,0x5e86163d),LL(0xd586a2fa,0x61432646),LL(0x3d92e0c3,0xc68c7c8e), LL(0x020c1dd6,0xbfa8c268),LL(0x8bbcc223,0x25788741),LL(0xef62642c,0xbaf8c9a3),LL(0xa8c496d2,0x6d2f1ae5), + LL(0x44e57ce4,0x92d1c805),LL(0xaacd2100,0x34cdf4a3),LL(0x31e9c649,0xd5b89e4d),LL(0x232cfc85,0x558a6e26), LL(0xcea31a31,0xb40f3e4a),LL(0x35c5c924,0x346c1e07),LL(0x3fcede81,0x8ffedd8e),LL(0x3b55d143,0x35269ba3), + LL(0x848bdc53,0x0366065a),LL(0x078554dd,0xba2af074),LL(0x19ff3b4d,0x3c755fba),LL(0x35a22cbb,0x5ea93372), LL(0x1eb3e23b,0x0e55fe02),LL(0x765dede4,0x2626ecca),LL(0x81f445da,0x187bf094),LL(0x9df30578,0xba011017), + LL(0xd72507f2,0x81148037),LL(0x5db072d0,0x3a5841fc),LL(0xbd910aa1,0xfd631862),LL(0x23881c60,0x17b22d68), LL(0xfcc13359,0x6fa799cb),LL(0x2d39fc5a,0x55c40219),LL(0x9f1f6515,0xd50bfff6),LL(0x2e30fa1a,0x575090b5), + LL(0x9ba20c27,0x70343a0b),LL(0x749306a5,0xef34db86),LL(0x4ba85f8d,0xd7ad61d2),LL(0x7e091a33,0xe4d24ad8), LL(0xfc348493,0xbd6b49b9),LL(0x299c325d,0x4f11b543),LL(0x55036e93,0x4574a254),LL(0x676b75d9,0x534a623e), + LL(0x7d05afbd,0x54aa3d8a),LL(0x7f3f2c90,0x13349009),LL(0xa94b18aa,0xcd03de0f),LL(0x855c050d,0x13123c18), LL(0xf598b5f8,0x747c3cda),LL(0x8644df20,0x0f7ed9b6),LL(0x08d73585,0x45e010fd),LL(0xf8cec4af,0x3b0ff430), + LL(0xb028a832,0x3745a41f),LL(0xc4106172,0xcd6d2468),LL(0x1eceac6f,0x56c5a9b0),LL(0xe1e6e980,0x769c1285), LL(0xfa113196,0xbd163a36),LL(0x5840c242,0x206ffc36),LL(0x9c57ef67,0x12de1147),LL(0x03bea885,0x50260273), + LL(0x9684d63f,0x3099c21b),LL(0xc7c66691,0x06adb196),LL(0x3d63b3be,0x8464492c),LL(0x0bd38c15,0x86024ef4), LL(0x10565cbf,0x226022a2),LL(0xc9899033,0x2ae6b298),LL(0x262ffa14,0x5564856b),LL(0x7e038b55,0x9472d0e1), + LL(0xb1dd268f,0x1b17ea07),LL(0x4899352e,0x8340b9d5),LL(0x75242992,0x4f159400),LL(0xe6727e9d,0xe4392a31), LL(0xaef59026,0x4df1ef86),LL(0x950cfee6,0xe40671ff),LL(0xde4dd990,0x7b36d1cd),LL(0x3366ff4b,0x25df10a6), + LL(0x584ef631,0x83fb7e59),LL(0x91af7b6a,0xf12dd400),LL(0xe26f11c7,0x4a5ae41e),LL(0x96d90445,0xeb86d5dd), LL(0xa355d0a3,0x028ae37e),LL(0x99260127,0x3c118ef4),LL(0x76f51bd7,0xb8c7538c),LL(0x5fbadc4d,0x66b90aae), + LL(0x17dfd0cf,0x078de9df),LL(0xfe44b17c,0x938df6da),LL(0xc40bc950,0x4a09616b),LL(0xbc969aac,0x0b507845), LL(0x35f7fb82,0x23bae091),LL(0x9ad29b83,0xebc04d37),LL(0x26a3c5fa,0x9fa48a5b),LL(0xf08f3d8c,0xf67c661c), + LL(0xb2e9c3a1,0x21825747),LL(0x46c3eb43,0x3520a82e),LL(0x9646183d,0xe309ae9f),LL(0x26dac288,0xa19c31be), LL(0xc76623de,0x3c37018d),LL(0x64b51087,0x59b9a0fa),LL(0xf1954f4e,0xa32f8357),LL(0x1f6da143,0x24251df4), + LL(0x01fb07d0,0x825c61cf),LL(0xf2f0243c,0x69ae93fd),LL(0x0943f7bd,0xd507c99e),LL(0x463ee64a,0x5e6dfb09), LL(0x29afd00b,0x10a3c32a),LL(0xbe8cbada,0x92179b5f),LL(0x7d89f9a6,0x7bebad0a),LL(0xd13b3073,0x7ba9bbf6), + LL(0x6481cc75,0xfc23c215),LL(0xfeca7532,0x6a654801),LL(0x0fed841f,0x20785ec0),LL(0x25269b21,0xcb612be9), LL(0x414a0fab,0xe9a9c09b),LL(0x6b4fa0dd,0x5404c7a7),LL(0xcb418588,0xde62dae8),LL(0xc594334e,0x2d80afd4), + LL(0x95366a44,0xfe454df1),LL(0x755cf8b2,0xda7626c3),LL(0x41397051,0x4446f0ab),LL(0x70eb8b23,0xd1788064), LL(0x977b040a,0xbc7737f1),LL(0x57590edb,0xbfb39418),LL(0x343a7333,0xb094b4a8),LL(0xeb91372f,0xb15912ce), + LL(0x0b56002c,0x584e1d5e),LL(0xaa0cb90f,0x1460ce24),LL(0x8f7ffcb6,0x58f0c144),LL(0xbe0d802e,0x56e39f33), LL(0x529458d0,0xb02a6edb),LL(0xa730f9d5,0xa0fbae74),LL(0x1bf69928,0xd98c0ac5),LL(0x796f12e9,0x5c9f888f), + LL(0x22e065c8,0x749207b0),LL(0x88c61143,0x6e0232a4),LL(0x837adda6,0x27f30808),LL(0xd0c215d5,0x0b748853), LL(0xbf076ba7,0x97bc4008),LL(0xf157f4d2,0xadae0275),LL(0x8bcba88b,0x394e5d7a),LL(0x23ef50ad,0xf995ec14), + LL(0xaa9b60a9,0x6b207f9c),LL(0x52f9979f,0xcd7509c1),LL(0x0834e0ad,0xe3e8f6dc),LL(0xcd5b1314,0x6e2a4165), LL(0xfd60d975,0x073a2db3),LL(0x2c053b7a,0x5ad92ca4),LL(0xfba97ae8,0xceb10220),LL(0xd265e913,0xab82f6a0), + LL(0xaa68a13d,0xc7a755ad),LL(0x10dd277a,0x56c13360),LL(0xdef56183,0xbbcf6411),LL(0xb863a4e1,0xebffe360), LL(0x814e8aec,0x67ff26e5),LL(0xa0804732,0x90553561),LL(0xad5fe672,0x407396ac),LL(0x11ad53af,0x053a068b), + LL(0xc652cb95,0xb518dd04),LL(0x57e2b99f,0xc818563a),LL(0xb96432a7,0x217cf87d),LL(0x8cc2fdcd,0x7fdc35be), LL(0x4effaebf,0x8c2ef271),LL(0xf427c7c2,0x21879369),LL(0x5197ba08,0xd80ebbea),LL(0xfc6f4c66,0x1b00db45), + LL(0x3828a5c0,0x99b9d2be),LL(0xd24a69e8,0xa8855350),LL(0x3ceaa821,0x2f919695),LL(0x04296439,0x89390c91), LL(0xc5cf8a0c,0xc12b3852),LL(0x73afc431,0xdcf234ea),LL(0xf4adf01b,0xdddf5568),LL(0x1b426097,0x2d04fc76), + LL(0xcb27c5e0,0x02a21dd5),LL(0xa6b2f9b3,0x01b842c3),LL(0xd8bd7a07,0xefbd483e),LL(0xd13a781e,0x0220825e), LL(0xde802c17,0x8aa029a0),LL(0x3a3f0fbf,0xb62fcd6e),LL(0xe9635f48,0x80558aff),LL(0x65dbeb2d,0xbdc6851d), + LL(0xb5412271,0x99f48286),LL(0xe242a8a3,0xa53ef798),LL(0xc0371086,0x41d18606),LL(0xfcb6d1aa,0x5df3379c), LL(0x00a7a13e,0x53f2f5a1),LL(0x2bf51e2a,0x3565a6eb),LL(0x930c5a71,0xa2832b52),LL(0xee2abfcb,0x66071ec7), + LL(0x3496a532,0x75ce0a56),LL(0xbe0d69b7,0xa316dfbb),LL(0x353e94fc,0x35438d6a),LL(0x1e0ce775,0xf53433c1), LL(0x22ff3a1d,0x47ea3e8f),LL(0xcd7ccdb6,0x60ebfba8),LL(0x33c475d0,0x47c6b6e2),LL(0xb7959fd5,0xd18637e7), + LL(0xa1ae3404,0x8d7a35ce),LL(0x75b832bf,0xf15c71d6),LL(0x98b9d24b,0x65047419),LL(0x0dcf73f4,0x28625a55), LL(0xc7c99478,0x5aa9dce8),LL(0x0bde8d53,0x752d1625),LL(0x93e99ee4,0x7255ecfa),LL(0x22706f48,0x1c53bf71), + LL(0x4d478014,0x28335451),LL(0x3f392c0a,0xd64b05ff),LL(0x4fba1661,0x1d9ac89d),LL(0x34828cd8,0x281a0ffb), LL(0x577ed419,0x07abacdd),LL(0x3cfb2c56,0xa66949f5),LL(0x847ebe65,0x38e69105),LL(0x44d6236d,0x8fbbba5a), + LL(0x725ef820,0x0c85bd64),LL(0x25a152a7,0x4ef91524),LL(0xb019cebf,0x5237ef0e),LL(0xc9a7724f,0x48203f41), LL(0xc55fc0d4,0x1f61885c),LL(0xbcb3daeb,0x2c4dd07a),LL(0x4b7dafc5,0x9855d5e7),LL(0x5f3769af,0xd76e6fdf), + LL(0xbb547be6,0xb834df1a),LL(0x3e7a9586,0x43056b12),LL(0x4375fc7e,0x7459e0bb),LL(0x9c85fc6b,0x5f5f295a), LL(0xbb23b709,0x3f2d2bb7),LL(0x955983ce,0x0b53bd8c),LL(0xfaf68dcc,0x5aee93df),LL(0x509f09dc,0x5575770c), + LL(0x40b1b744,0x2bf23c0c),LL(0x9a5bb913,0x524154bb),LL(0x296bdb2e,0xb0e93d76),LL(0xda0b2925,0xb3c71f5b), LL(0x0c617939,0x8e77ae7a),LL(0x1aca9b0a,0x2bfea97e),LL(0x7897c5a8,0x8e3317c9),LL(0x4cee2716,0x850ddefb), + LL(0x107d9186,0x684ceee9),LL(0x0082c312,0x48708423),LL(0x6c556897,0x5300137c),LL(0x7e99bc76,0x6d464401), LL(0x8808ca2b,0x6b11e1e3),LL(0xcca6433e,0xedd30eee),LL(0xaa072ff0,0xa9099f60),LL(0x5830f69d,0x774662ec), + LL(0xf94547d9,0x0770355c),LL(0x42967865,0xb5041edb),LL(0xe585a93b,0x169a6274),LL(0xd04d6a81,0x06cebf5e), LL(0x2ebc1791,0x0a59450f),LL(0x765ac18a,0x69fd006e),LL(0xa54f7e7a,0x4711ec9c),LL(0x819c6af9,0xd72c8d58), + LL(0xb5418e15,0x89c97c6c),LL(0xa558a854,0xd520b03c),LL(0x0d76773a,0xe3c24aca),LL(0x67e5110e,0xc4deb5ce), LL(0xcbb04ba4,0x5bb40152),LL(0xde1b628a,0x672563b6),LL(0x2e8d9e54,0xaec916aa),LL(0x3c60ac70,0xa4e8cb47), + LL(0x452d5064,0x54a03e39),LL(0x1e7bb355,0x1e405c2d),LL(0x3bbd3ab0,0x2ab2d5df),LL(0xdbc9fbd8,0x0808410e), LL(0x27f23f6f,0x4180ceea),LL(0xba1d6beb,0x2b965b35),LL(0xf66d6546,0x14f1f010),LL(0xf85cfb4a,0xefdca6a8), + LL(0x6614c1c0,0x69e6188e),LL(0xe07cb7f8,0x00bd1531),LL(0x4bb7ee68,0x1b90f515),LL(0x51abb1f3,0x8afdf466), LL(0xb5f34316,0xf59a7327),LL(0x64c7bf22,0x43c3c19d),LL(0xcdb00a2b,0xb275733f),LL(0x602915ba,0x0160df79), + LL(0x1baea574,0x1ae4ee9a),LL(0x03ae5168,0x0d922f28),LL(0xca691124,0x07df28fd),LL(0x8dd457c4,0x5aad2f31), LL(0x137384ca,0xe0d4f443),LL(0x6620ea8c,0xd93d424a),LL(0x5067797a,0x21d544d3),LL(0x9d8a15bc,0xc8a8cc99), +}, +/* digit=31 base_pwr=2^217 */ +{ + LL(0x941d80a3,0x610f0e26),LL(0xad36514e,0x30927879),LL(0x98f22601,0xaa2dfd48),LL(0x88c8b0f6,0xbc5b31b7), LL(0x6c841cc8,0xb1281f37),LL(0x5a412b84,0xdae16719),LL(0x828f210d,0x9ec1f6c8),LL(0xe8d92901,0x1935d576), + LL(0xaf3840f2,0x47247921),LL(0xdf3fcdfc,0x348325d2),LL(0xc43961bd,0xef578508),LL(0x1bd98c29,0x7d5e8ccd), LL(0xf8a30164,0x59cdba10),LL(0x7cb8c653,0x0757408f),LL(0xc3056ef4,0xcd7ed73f),LL(0xfb99cd1b,0xd28e7cc1), + LL(0x6bb62286,0xa8a67494),LL(0xe7d87467,0x8d6ef816),LL(0xf673b6d5,0x3529f938),LL(0x8bbf101b,0xf5c0ee76), LL(0x49fdc949,0x3768ed61),LL(0xcf405ee0,0x53b6999e),LL(0x1718e51a,0xbf0108a0),LL(0x5181ebd7,0x38e609ec), + LL(0xd8595159,0x82051640),LL(0x579a3876,0x30e1c706),LL(0x0298a67c,0x091154c6),LL(0x51132d27,0x76d9a9c9), LL(0x5c661a4d,0xe41de8b7),LL(0x2a6dcbaf,0xf24b5e96),LL(0xd714e045,0x1ed4cb0c),LL(0x0c187c70,0x605926a4), + LL(0xf34b1c93,0x97672559),LL(0xe0b43506,0xa256be98),LL(0xe1a98eb3,0x7fcdd412),LL(0x295a061b,0x7fcfcd84), LL(0xcc2386cf,0xddbac22b),LL(0x928c2556,0x7741adb7),LL(0x4e6e1288,0x3a2bb869),LL(0x025bb4a1,0x53ed11da), + LL(0x4108c8e2,0xb114bd67),LL(0x3deb8e23,0x85948c6b),LL(0xd0e9434c,0x6a9e05d9),LL(0xb9c4fd70,0x395060b7), LL(0x18893751,0xa0ccfd16),LL(0xbbf65477,0xa941ff60),LL(0x59423e35,0x34ada238),LL(0x7e570058,0x4ba7a7d1), + LL(0x869ae596,0x3d05d455),LL(0x93368325,0x92a15400),LL(0x62eb7ffa,0xbd64ae0a),LL(0xd34b2c59,0x3f263a7a), LL(0xa22244e1,0xe95eece3),LL(0x706fc500,0x39ccef58),LL(0x22f1d2e6,0x39a88f93),LL(0x6d47c670,0x1ec796b3), + LL(0xaa1ff4af,0x1558e0f2),LL(0x390503a4,0x61f43487),LL(0x3c4f76f1,0x66164732),LL(0x9e13432e,0x50d0706e), LL(0xf5eba0fb,0x5f1a87ca),LL(0x80bda2c5,0x28a95c0f),LL(0x12ae6462,0x10d693e0),LL(0xf45e6ba6,0x79871340), + LL(0x887051c0,0x8abf0cad),LL(0xb3c6b540,0xd09f571d),LL(0xe30ab25d,0x3fb2e16b),LL(0x539e8bc4,0x12e057a7), LL(0x733c2597,0x6efe71ce),LL(0xe71058ac,0x72fa0df5),LL(0xccc037bf,0x49f14d06),LL(0xceb3eb6c,0x9a3ceb03), + LL(0x4195370a,0x9895e908),LL(0x927345e7,0xa75007e5),LL(0xd3850711,0x00b4c212),LL(0xdfee8b34,0x76e4060d), LL(0x184c1d07,0x15801862),LL(0x302f5711,0x234e39c0),LL(0x6aa74204,0x4c0bd387),LL(0x6f496836,0x0515eddc), + LL(0xc849afbe,0xa9fd0cb5),LL(0xcad5c0aa,0x041df5ba),LL(0xddff259e,0x9a54af37),LL(0x9b550a8e,0xa3f156bf), LL(0x7e3298d4,0x4df2d33d),LL(0x65ff0e1a,0x0957a0a0),LL(0x1e2b3a45,0xff7fb43d),LL(0x1a73285a,0xb86d386a), + LL(0x28b18e93,0x6e283c47),LL(0x4b4132ed,0x5458b92f),LL(0xba674332,0x7026345e),LL(0x5c9fc32d,0xc8e38151), LL(0x58e7b4fe,0xd6aaf8e1),LL(0x4267253a,0x3e77a5c9),LL(0xa0949607,0x6441cba2),LL(0xdee20b2e,0xfa205185), + LL(0xf5b44600,0x64f3d576),LL(0xef5c8446,0xf12125db),LL(0x487a0116,0x1467539b),LL(0xf380229a,0x3aa0fa49), LL(0xd79f36dc,0xcc6586f1),LL(0x1b9a0b42,0xebcf653d),LL(0x9c1df729,0x68af006d),LL(0xa7903ee6,0x507e055a), + LL(0xafd8ac9b,0xd1b48ef8),LL(0x3b5f6b51,0xd8920af7),LL(0x983339c8,0x9629e789),LL(0xfa9248d3,0xbfd2d787), LL(0x9a078882,0xb50ca302),LL(0xf5cb0529,0x1c34f848),LL(0x1646a9f8,0xb9b01519),LL(0x80d53f9d,0xc9e3679e), + LL(0x1af3e5f2,0x5a63e822),LL(0xff58e3d1,0x05480ad8),LL(0xd6b3626b,0x2d241643),LL(0xc1eda15f,0x33b15660), LL(0x8528e5d6,0x3e74f855),LL(0xf63188f4,0xafb6dc9a),LL(0xaeeb1d32,0x0cac44cb),LL(0xa627eff8,0x50661046), + LL(0x64b49667,0xadc4b012),LL(0x1e05f760,0xa4bdafa7),LL(0xf185d27a,0x171b28b3),LL(0x33425747,0x987e5163), LL(0xc3864a65,0x7c42ac4e),LL(0xbf449c12,0x2dae1bb8),LL(0x06965832,0x680d9743),LL(0x7e31d9f4,0x6ac1ef01), + LL(0x579d6ae4,0xdef57433),LL(0xd5240bf9,0xe055b087),LL(0x90a5e069,0xe4dbbe60),LL(0xddb3dc15,0x2161d5fe), LL(0xda297b71,0x7d303012),LL(0xd468046c,0x04481034),LL(0x0ac93c6c,0xaa94d5bb),LL(0xd8d8f53a,0x95bacd45), + LL(0x3e03e895,0x790a5d6f),LL(0x44fa5a81,0x27efd502),LL(0xe5998b32,0xd9d35230),LL(0xf22ade19,0xb36a0c07), LL(0xf979a2fe,0x46ec8691),LL(0xced8cb94,0xa67ba933),LL(0x2f856ab3,0x00d07245),LL(0x3c925dae,0xadc9ff42), + LL(0x563038a5,0x0e4eaa25),LL(0x8a8f6483,0xfef7e89c),LL(0xace61af8,0x50433633),LL(0x2e1a3515,0x8a1589e0), LL(0x5fdcb1ac,0x99f21e29),LL(0xc9a466da,0x8fd2d411),LL(0xf56b7f13,0x55323c6f),LL(0x5cff438c,0xa016d64a), + LL(0xdc05b5cc,0x3e3dfcbc),LL(0xfc3c70ec,0xc1956ca8),LL(0xe63f02df,0x7dbbd169),LL(0x240b87c8,0x95206689), LL(0x1aa6d48a,0x7bacda5e),LL(0x39280f78,0x51dcf19f),LL(0x660abac2,0x1511ae04),LL(0xd905af53,0x3a95adc9), + LL(0xea464737,0x0c8c4330),LL(0x34fc4b51,0x989c09c4),LL(0xe2cf3376,0x1852120d),LL(0x25c04de3,0x5a1cb8a8), LL(0x75fe7596,0x50486f98),LL(0x223030b1,0x8cd78d2e),LL(0xcfa1ab11,0x524cb8f8),LL(0x5a15b0b9,0xa716ea3f), + LL(0xb902d114,0x7618e95e),LL(0x084ebf5d,0x0a1a4146),LL(0xe3f87683,0xdfb909e9),LL(0x4107410c,0xa0b7eee1), LL(0xf02b0e12,0xa45a551c),LL(0x9efccb9f,0xceabbfd2),LL(0x740f4e3a,0xb0d1b6bc),LL(0x4cbfd0de,0xfc737250), + LL(0x32452b0e,0x3fad2d9e),LL(0xf523527d,0xb4e659fe),LL(0x6c0ff353,0xf0dcd701),LL(0xd67b6f98,0x28f06e2a), LL(0x82a789b4,0x2d0c36ce),LL(0x49c0d54e,0x20e577da),LL(0xae38dd0e,0x8d1d5c7f),LL(0x894d9143,0x72e01399), + LL(0x3958e055,0xf7806856),LL(0x5df44aee,0xac35ee40),LL(0x97c18b8d,0x2b478913),LL(0xfa2586cd,0x5396824e), LL(0x1b23f8c4,0x22b37b25),LL(0xcdecdefa,0xf9ced36e),LL(0xc2fc39c0,0x28c3bee5),LL(0x6d9db32b,0xa1731fae), + LL(0xbc3e2c91,0xa0445fa7),LL(0x75a4aa72,0xa1ab6955),LL(0xbbe0a1c7,0xf0cd61c6),LL(0x0123bc52,0x923c3b69), LL(0xafd7c4bc,0x818ad28c),LL(0x28b15b05,0x7c229224),LL(0x1f78a4f4,0xecde7efb),LL(0x03ef3ab3,0x550d68e7), + LL(0xfc5f8c35,0x0371021d),LL(0x0ed2b06e,0x4440aa1e),LL(0x9ba7247d,0x70c8ede9),LL(0x84f23fde,0x0d2b6ed3), LL(0x5ff4478c,0xd0119d95),LL(0xf79c69d5,0x66152d27),LL(0x02afd13b,0x56d8bea4),LL(0x15bb938a,0x035efe5f), + LL(0x2ccaa425,0xc5ca7d08),LL(0xeeee9376,0xc8c69ea6),LL(0x493a2051,0xb22cfe59),LL(0xdc7b90fb,0xcb50e618), LL(0xe05a8705,0x0f6fdf2b),LL(0x4814df31,0x081f3fe7),LL(0xeb1e3e76,0x6fefe18a),LL(0x03e06a50,0x81910050), + LL(0xdb45bfea,0x8a801df1),LL(0x7a828cf6,0x8c7fe1fd),LL(0x8d173cfd,0x1c1868b5),LL(0x0dbde1c8,0xe18f0a36), LL(0x9ac345b6,0x3b29ed64),LL(0x9dcd07a5,0xd56d5956),LL(0xc6813a88,0xf4191570),LL(0xeda3af42,0x39033ebc), + LL(0xad5d215d,0xdee5591b),LL(0xafbe5a28,0x9cfa11c6),LL(0x1823c28f,0x73d0f1e2),LL(0xafab1f67,0x75d49925), LL(0x7c521448,0x61c81e2c),LL(0x4a96edb5,0xc547be6f),LL(0x4ca368b3,0xccb9fc59),LL(0x04fc3303,0x175ebe48), + LL(0xfce42989,0x507620cf),LL(0x9abfadb2,0xf236e043),LL(0xab36ab58,0x381c50c3),LL(0xae22c6a3,0xed4cb73e), LL(0x2158dc4c,0xa68a2827),LL(0xe9fa53ff,0x1715ac43),LL(0xfa266797,0xb02fdf73),LL(0x7eefb203,0x3079f3c7), + LL(0x7f7545bd,0x0a41fb94),LL(0xcb923ace,0x6b9dd022),LL(0x3bea2541,0x582c7ff5),LL(0x5ecdbe2d,0x992f2379), LL(0xfe17bdca,0x821f1670),LL(0x2626bdde,0x521c06f2),LL(0x1864ca0b,0x6292748c),LL(0x1bc74d8b,0x554d4ece), + LL(0xea3d4446,0x745d4f74),LL(0x40ad1c7f,0xa439f178),LL(0x51374e92,0xc95d9510),LL(0x90229008,0x75870e9f), LL(0xc54e7e81,0x3fec98c2),LL(0x94b3860b,0xef537ee9),LL(0x40bfc8f6,0x139dd834),LL(0x0f114403,0x20b51364), + LL(0x30b4b4db,0x4752a49f),LL(0x8c3c90e0,0xdfbb8b17),LL(0x70f0b16a,0x60c8915b),LL(0x40528319,0x5e395000), LL(0xa641f2e3,0x8a1624c7),LL(0xbb4ca0dc,0x3c9925c6),LL(0x2c3152b5,0x2aae6edb),LL(0x08b896ff,0x8dbac580), + LL(0xe5a36bc8,0xe0516205),LL(0x3295707b,0xd7714332),LL(0x51c3513f,0x61db6804),LL(0xab552df8,0xf2ee6e20), LL(0x353c17f0,0x5ddcfa99),LL(0x046d5fd4,0x65687a2f),LL(0xfd1ccad4,0xef567e9f),LL(0xa0238b70,0x7cd5f7dd), + LL(0x92c01197,0x96fba79e),LL(0x83b50e70,0x46a9f2de),LL(0xfe287169,0x7efcbbb2),LL(0x4528d67d,0xe30d60cb), LL(0x6cb04d3a,0x88fed0cc),LL(0x0221ceb8,0x63eb9d0d),LL(0x748b5813,0xc6954e9f),LL(0x5c96e897,0xceef2bd8), + LL(0x85648f04,0x99503ae2),LL(0x923e87d7,0xeee51f99),LL(0xb6560ceb,0x90908fca),LL(0x80e0f6b3,0xafad5926), LL(0xaea32cf9,0xa50f31f3),LL(0xa74ae92d,0x7ea17064),LL(0xcda71d1a,0x0675ccc1),LL(0x1e0a464a,0xd1e3b630), + LL(0x2442872d,0xa361f2b7),LL(0x46e52c97,0xb21bcd39),LL(0x85574630,0x1405f89c),LL(0x8e0a96ab,0x0da7bfbd), LL(0x4220f57b,0x48af06c2),LL(0x6a333e4f,0x772a9b12),LL(0x6f712eb8,0x3afc661e),LL(0x2eba8817,0x29deff6c), + LL(0xd8c69e5a,0xbab680de),LL(0xe93daf10,0xf8615abb),LL(0xcef6fae6,0x7455ea1d),LL(0x868455fd,0xac0a30ea), LL(0xe47d628a,0xae967b17),LL(0x65f1f482,0xa6d703e2),LL(0x0bfcc371,0x2723a965),LL(0x6db4a042,0x9b06cc14), + LL(0xa77c8b21,0xa973d738),LL(0xc008f2ed,0x9a981f80),LL(0xaf27cdb3,0xecc7bbcb),LL(0xb5cb693a,0x514db964), LL(0xe75c93d1,0x24125414),LL(0x1f00d53c,0xd9308c0e),LL(0x831eba6d,0xdb56d155),LL(0x672b86f1,0x29eefc2c), + LL(0xdd13b3c9,0x332f6ab6),LL(0xe371f873,0x70e052f6),LL(0x125712ab,0x05740742),LL(0xb3512100,0x4239152d), LL(0x80b22915,0x98355eaa),LL(0xb896f6fa,0xd0e263ec),LL(0x442b4c8f,0x9378a8a6),LL(0xf65795bb,0x40c2b546), + LL(0xd572ead8,0x0cfa46ed),LL(0x78361300,0xb9b4abdb),LL(0x8c102020,0x5fe63ef1),LL(0x785a4b54,0x1805c84e), LL(0x805cb642,0x147cf487),LL(0x487e581f,0x87cf50aa),LL(0x9eaebcd0,0xe942fa5b),LL(0xd1af71f2,0x06d4fa96), + LL(0xc4fc3823,0x20c1a770),LL(0x85140885,0xcdffd09e),LL(0x6b3592e9,0x27ce78ab),LL(0x8ba82008,0xb8e8c15e), LL(0xfef74187,0x5fe8f3f0),LL(0x77ce808d,0x8e85a3a5),LL(0xc7395f64,0x8447dc69),LL(0x1181b854,0xae90769f), + LL(0x456114c8,0x54adc101),LL(0x9ca6a9c1,0xe7962b76),LL(0x909410a4,0x3f0e77fb),LL(0x9e2e44f9,0xe18151cd), LL(0x2cf6e29e,0x5e510a0a),LL(0xb1836b07,0x136896ab),LL(0x0fe11010,0x3ad4fdec),LL(0xdbddf038,0x35b36790), + LL(0x75903df9,0x7c4f5a68),LL(0x2f5b7193,0x3e9cb056),LL(0x591a4524,0x745e9452),LL(0x1a056e15,0xc406ad44), LL(0xa69e11ef,0x2e93edf2),LL(0x73a1cb88,0xa28b82fd),LL(0x1225c3d5,0xdc1c9cda),LL(0xa5569794,0x86e9a994), + LL(0x5b092dde,0xd698506e),LL(0xd1ca8b06,0x076a4c82),LL(0x2ef2bc6f,0x4516033b),LL(0xd78fa65f,0x0574c792), LL(0x735bb362,0xa3b1c3d8),LL(0x0da54317,0x22fca7a4),LL(0x60aaebb6,0x3e7ae709),LL(0x937638c1,0x42417d54), + LL(0x1dfe8b0e,0x32f00a5d),LL(0x8dcdbdbc,0x8ea5e8e1),LL(0x6b30ea52,0x38df57cb),LL(0xe94c30ca,0xd325aa1c), LL(0xdce4d256,0x7aa04a9d),LL(0x74c7db6b,0x78e98cd3),LL(0x443d5c9f,0x631475a8),LL(0x7adfbceb,0x34e5c73a), + LL(0x9f1e8828,0x7fb69bab),LL(0xc84149e3,0xcadc78be),LL(0x1fe86af8,0xe9424ecc),LL(0xbc504ea8,0x13160cc8), LL(0x4c96a680,0xcb380078),LL(0x845faae5,0x006fb9d8),LL(0x1e0e66d1,0xc6a64277),LL(0x428f526d,0x13f77d6e), + LL(0x28474530,0x9f80fe8c),LL(0xdb7fec00,0x5649a173),LL(0xd9cb05ca,0xdeed5bf4),LL(0xd7077c41,0x14b1a3a9), LL(0x096883ec,0x4c2ed239),LL(0x44ae671d,0xd550edfe),LL(0xf7b7362a,0xb233e5dc),LL(0x4fd464f2,0x32c15820), + LL(0x68880bf9,0x0ecb18f7),LL(0xaf230a34,0x53468bed),LL(0x370cd6ef,0xe3ba97b9),LL(0x3516d77e,0xf5cdabf4), LL(0x11462032,0x08d78a56),LL(0xd583ccc5,0x1393fa93),LL(0x0c1b3514,0x52af7f5d),LL(0x188ca043,0xf48cac66), + LL(0x5461a1d1,0x2524c8dd),LL(0x91b6e707,0x6eee8101),LL(0xca2fe87e,0x209fece6),LL(0x9ac56706,0x50b35727), LL(0xec373bb2,0x651a6701),LL(0x1a4c2e84,0x881de85b),LL(0xcfdb47d5,0x4892861d),LL(0x5cdc4424,0x5ae2e653), + LL(0xa1f90dd9,0xc58f4f59),LL(0xfcf158a4,0xa5584f85),LL(0xab072a7a,0xbde86fb0),LL(0x268bae62,0x7c69e25a), LL(0x44fc7b3e,0xee3478f3),LL(0x6b7d3647,0xec148394),LL(0xe1c8c0ca,0x2a542ebf),LL(0x161dc0c1,0x63d1d635), + LL(0x57ab9282,0x769acdbe),LL(0x2a119cb9,0x9c338971),LL(0x125e5b4c,0x049e366f),LL(0xf0c8fde4,0x3aec68e0), LL(0x324cefda,0x9d95b6e5),LL(0x704014b5,0x844cce33),LL(0x6a6bb216,0x03920a61),LL(0xf379db8e,0xd69d17e3), + LL(0xc5e386e5,0x1924ac16),LL(0xd64953c3,0x62373a48),LL(0x47f4e4a4,0x5b1f7d64),LL(0xffa115fd,0xc043b5b5), LL(0x87fb16b0,0xb2a2656e),LL(0xd8cd79a6,0xcac56b9b),LL(0xcc19d3af,0x544971f6),LL(0x0fd63db9,0xf539244b), + LL(0xfbf4d232,0x0f052d3c),LL(0x7a2a7280,0x6b3c8366),LL(0x48079b9f,0xaa6579db),LL(0xa4d9edcf,0xc5beb93d), LL(0x0f1599a3,0x8ad58825),LL(0x5f3f640b,0x3f3a2634),LL(0x9032fd7c,0xda15393a),LL(0xac0e7136,0x97c10230), + LL(0x599785ee,0xfa32ef9f),LL(0x6b4c7a65,0xe1ed3b28),LL(0x2da1dcdd,0xcee1af27),LL(0x6861e2c2,0x4e480c11), LL(0x9c8ad8c3,0x35b5ec42),LL(0x3fc55f23,0xfd07f6a4),LL(0xea76d444,0xab18ead2),LL(0x22ba099a,0xcb8bde14), + LL(0xc61ae967,0x252e6a81),LL(0x72a2e1e6,0xaf11042c),LL(0x1a211ef8,0xb353902a),LL(0xc99a25fc,0x644d16e0), LL(0x5b67e48a,0x637fd606),LL(0x51a0b665,0xfa570963),LL(0x7ee072b8,0xaa661c73),LL(0xf2e0a727,0xde1eb4fe), + LL(0x22ed7ee6,0x56096a0c),LL(0x5825908b,0x31aaf403),LL(0xbfa02db6,0xfd5f6ba7),LL(0xff798900,0x85f4f9a9), LL(0x4a0cd878,0xa0997d56),LL(0xb1b6982e,0xdd76909c),LL(0xeccf338e,0x874fab15),LL(0x4ce82bb1,0x5e072b3c), + LL(0x6dd0d997,0x5dbe883f),LL(0x41765fb6,0xa32117f2),LL(0x7d87fc5e,0x59ca4da3),LL(0xb95ec918,0xc91002cd), LL(0x6548248f,0xd53bc123),LL(0x6c6d1e0e,0xef10a373),LL(0x99d9893f,0xafb2d760),LL(0xce0ba0ca,0xb77c1f1b), + LL(0xcfb9f6b2,0xabce362c),LL(0x35f9be91,0xe6e108d2),LL(0x7187fa9d,0xb2331290),LL(0xfc7ddce6,0xdcd1f4fd), LL(0x9086eb29,0x3a129991),LL(0x53a56d57,0xb0730520),LL(0xabd421bd,0x9fcdf4cf),LL(0x08f3e8e0,0x96271270), + LL(0x401e0217,0x951ea7e2),LL(0x733f637b,0xa4d1d708),LL(0x4f4cd676,0xc75170f4),LL(0x832f0b4d,0x568279ba), LL(0x25c17ab7,0xda4c01f7),LL(0xfa30e1b9,0xfcc13028),LL(0xacba57ec,0x4d1d8f71),LL(0xef6b3913,0x0c7971cf), + LL(0xc014f166,0xdf16e73d),LL(0xf96f2c30,0xd5796183),LL(0x3f70dd7c,0xd13ee9f7),LL(0xdac738c5,0x3f9aa0dd), LL(0xad021e28,0xa200c7e4),LL(0x08414fd0,0x982abae3),LL(0xc3779882,0x76d16a8c),LL(0xe70a6ff5,0x41563d33), + LL(0x4b553a17,0xdbb9656e),LL(0xd9c87aa1,0x96af21a0),LL(0x7bd9a625,0x2de13a03),LL(0xfeb1fec2,0x29f8c49b), LL(0x1a4ce44a,0x84e2df47),LL(0x548b39ee,0x83bb2965),LL(0x94d996eb,0x38b91cce),LL(0x9441ae0b,0x41e0a3cd), + LL(0xdaa92f34,0x720d30d8),LL(0x06f30fbb,0xba587579),LL(0x4c96ad59,0x24f74676),LL(0x0d33bd5f,0xf40493f7), LL(0x126a7267,0x9068c3e9),LL(0x18927490,0xa51099df),LL(0xa9cfe02f,0x27452423),LL(0xb8749653,0xcfd035be), + LL(0xfda6a4a9,0x0dd9bc2a),LL(0x0106ae0e,0xdba0178a),LL(0x4969a4bb,0x3820c9f5),LL(0x99fbc715,0x5031e9fd), LL(0xc193d942,0x642a030a),LL(0x454cbb39,0xdc3d6ab7),LL(0x1c8fa77c,0x507c17b9),LL(0xe3642a95,0x8465bcc8), +}, +/* digit=32 base_pwr=2^224 */ +{ + LL(0xc25dfad3,0xe74e265b),LL(0x493f44b6,0xd03630b9),LL(0xbfd6d473,0xb3270892),LL(0x1c5ee992,0x5b2d9543), LL(0xa36f7c5f,0xeeb94537),LL(0x8ab0b81d,0x9befc01d),LL(0x188b45e5,0x483cdb08),LL(0x01e4648b,0x44c753b7), + LL(0xb2411618,0xee43bc87),LL(0xf07924c4,0x08754bd2),LL(0x4ac92557,0xef205033),LL(0xee0387f4,0x6e7e4fe6), LL(0x76961d0e,0x51f3e2e2),LL(0x37eac10f,0x2b69d417),LL(0x73757a88,0x36d0f45f),LL(0x2b0c7d35,0x38b967e5), + LL(0xb31fa779,0x94ba8fc4),LL(0x0f13036e,0x8024dc85),LL(0x82d754b7,0xfda2af63),LL(0xae9ea9ae,0x4a784242), LL(0xf9887947,0x67dd14ab),LL(0xcd555a0a,0x7f2ecfc4),LL(0xf63a46aa,0xb37c4244),LL(0xff71b4b5,0xd032cfc1), + LL(0x6b8a6a97,0x0aef84c1),LL(0x0b2bca36,0xd2e7f3de),LL(0x5b174d43,0x721c6c09),LL(0xd52ccc5b,0x5719cf31), LL(0x3adf9517,0x6c7361f0),LL(0xabe20ff5,0x1e264169),LL(0x69eacc0e,0x01f9d997),LL(0xc2e635d2,0x721eba63), + LL(0x25df8bb5,0x4225e9c8),LL(0xb5752d7e,0x931f721e),LL(0x0a3b281d,0x3c4ed475),LL(0x4a4668be,0xcf927682), LL(0x75b7e90c,0x1b7f358e),LL(0xb7a29b9a,0x06e5c24d),LL(0xa167f2c8,0x0058967a),LL(0xa4ee62d3,0x9f1a6fb9), + LL(0x278291f1,0xca899c4f),LL(0xf4e64c1d,0x69a90324),LL(0x8d62916e,0x46cc5d42),LL(0xec1007cc,0x3c802e65), LL(0x6219cfbb,0xdadcf2aa),LL(0xd10258b2,0x942870dc),LL(0xa5e142af,0x77264e68),LL(0x089cc7a3,0xf25675e2), + LL(0x7336aa16,0x177e8a3b),LL(0xbc5c622c,0x5a92cc2d),LL(0x1789e029,0x33a35a2c),LL(0x4e4d5573,0x6f91306e), LL(0xda0a46f5,0xe5a2a581),LL(0x42640bba,0xfb532bed),LL(0x4a7b3ae4,0x88ff0f11),LL(0xb8ff7a71,0x2223e7b6), + LL(0x5d21d261,0x75933133),LL(0xcabb1fbd,0xa336289a),LL(0x631b3b61,0x797db2f3),LL(0xd7e6a511,0xc2cedb25), LL(0x10355332,0xb8806f34),LL(0x5d0ae37f,0xe5f1fb4a),LL(0x5d17c5c7,0x57cf26a5),LL(0x68c43ec3,0x82e8df47), + LL(0xf86bd784,0x70fa23eb),LL(0x51b0ce75,0x711a9dbb),LL(0x82170008,0x83bb4a90),LL(0x630602dc,0x8f096ee9), LL(0x7f15e77a,0x7d275fc9),LL(0xfe727ec7,0x63516a6a),LL(0x1dce9d38,0x6b06827a),LL(0x023b31c2,0xa01a5382), + LL(0x886209b8,0x12537433),LL(0xc5a11b32,0xb7875fa8),LL(0xbd61176d,0xfa63cb99),LL(0x33378ebb,0xebb204ea), LL(0x70c135f6,0xf29a29a0),LL(0xfa29d69f,0xf53941e9),LL(0x9917da42,0xab97b39a),LL(0x45947ae4,0x4677cfea), + LL(0x0f6dd908,0xd4668cff),LL(0x987e0769,0x48bb09ed),LL(0x8d64b6fd,0x794ed298),LL(0xfac845da,0xaf89d530), LL(0x2d06e70d,0x57445649),LL(0x079e70a7,0xe2a1a8c2),LL(0xf524fc03,0xd2ef1779),LL(0xb137bb1b,0xeaccaacc), + LL(0x5d279f42,0x34d8ed87),LL(0x1cd68904,0x4dd5344c),LL(0x24abd550,0xb558b71d),LL(0x40df135e,0x3728e850), LL(0xcfe86519,0x9329e2b2),LL(0xac74cde2,0x48ad17fb),LL(0x30b388b5,0x2ad61b22),LL(0xfaea71e1,0xebcbc1ad), + LL(0x35990d9d,0x50d7b19e),LL(0x6eb70243,0xb17138e5),LL(0xaa8ae7e6,0xb61618f6),LL(0xabce12c6,0xedee15b0), LL(0xcc7205fc,0xa99ce250),LL(0x69e0d75c,0xe438efc9),LL(0x5084b390,0x1feb6a10),LL(0x9c80d42d,0x7b348954), + LL(0xe4b68140,0x67ac89d5),LL(0xc9b092af,0x34afd23b),LL(0xfe1ad661,0xad65cae9),LL(0xe02d884c,0x4f402947), LL(0x6b1c40c1,0xd0a48fcc),LL(0x8961487b,0xf950c9f7),LL(0x206d1752,0xdb1cd811),LL(0xe99fd891,0x863b0ded), + LL(0xbb2a4515,0xd3aad8c2),LL(0x797e223c,0xc93c8cb8),LL(0x12a77d46,0x0f471e49),LL(0x600872b6,0xa2ac9434), LL(0x915f730b,0x6fb1c7ef),LL(0xd254d363,0x9fb72401),LL(0x6b44796a,0xf521e33a),LL(0x97c9fafb,0xb7ed2e8d), + LL(0xffb5e7ce,0x60d41128),LL(0xaecb96c2,0xdbd8b542),LL(0x0b5ca788,0x029ab3dd),LL(0x190eb38c,0x8b1148a2), LL(0x69fb1924,0x59048db8),LL(0xb18391a8,0xcd2149f0),LL(0xfed311b9,0x6bece5b6),LL(0x9ffd29b9,0x5edbe9b9), + LL(0x1156ded2,0x53810556),LL(0x721f3e68,0xf812ce5d),LL(0x7ccdc8cb,0x50504d40),LL(0xc60fa4fc,0xb559ba08), LL(0x1d6bd879,0x862a83d9),LL(0x836e26ba,0x2f8f653b),LL(0xeb26ca11,0x8587e6df),LL(0x8c8aaf7b,0x127bd905), + LL(0xd67d9902,0xe26e690d),LL(0xb58e7e78,0x1a6061f4),LL(0x480dd4d1,0x960ef741),LL(0x75589610,0x7fd09736), LL(0x855a8b2b,0x5a20a1a2),LL(0x355b4e0f,0x3ed68662),LL(0xe76595b4,0xd3786f45),LL(0x0bdedcfb,0x72a6999d), + LL(0x6a175646,0x4e48e843),LL(0xe05dc02d,0xde53c427),LL(0x97d31bc6,0x9728a4c5),LL(0x5bb3bd37,0x01a07129), LL(0xa74a0fcc,0x83c08a98),LL(0xbc345df8,0x233e400f),LL(0xcc3e0edb,0x9578c5f2),LL(0xf144a31f,0x0fe89df2), + LL(0x14c5a2cc,0x308098a0),LL(0xba40c0bc,0xeda5a59d),LL(0xb718a5ae,0x0b10f7e0),LL(0x5b8ad9ba,0xdaf7da8c), LL(0x87394cde,0xddc71285),LL(0xe43458d3,0x9bdb27cd),LL(0x4bd7c11c,0xc698d972),LL(0x3540be14,0x2ee97fbc), + LL(0x72f98422,0x2c704995),LL(0xef8661c5,0xfc71fee2),LL(0xce08043e,0x6574e022),LL(0x5143733c,0x3d17162e), LL(0x730e5b80,0x3bf0b448),LL(0x7cf94b5e,0x56de346a),LL(0x6c797797,0xfa87a53e),LL(0x6487d014,0xe8b9edfa), + LL(0x7be60b03,0x09e74387),LL(0xec8750db,0x2277ebc3),LL(0x7aeaa545,0xf1e9d594),LL(0x44c03394,0x41564562), LL(0x4de9f7ec,0x57943adc),LL(0x2a220cd5,0x09dd58f9),LL(0x06973808,0xdf848ec8),LL(0xd3950024,0xf1d5def1), + LL(0x5a8707e5,0xd089eba5),LL(0xb0b90ebe,0x914046cb),LL(0x63fe6bc2,0xb01180b2),LL(0x7ede9d83,0x1ffbc968), LL(0x3c52c09f,0xe16d336f),LL(0xdf40338d,0x32270ecb),LL(0x7eec7039,0xb55ff5c6),LL(0x38a63fab,0xb5ffb314), + LL(0xee18ffec,0x3e9f284f),LL(0x1d1b4e80,0x702d97f5),LL(0x214c4da1,0x2005ee57),LL(0x2f5ea2f4,0x1c210413), LL(0xa4149949,0xd24a486c),LL(0x23c8e201,0x3869a339),LL(0x0149992e,0x00f6e410),LL(0xf0a367dd,0x54e97b46), + LL(0xe169de5c,0xd967726c),LL(0x6a0fcd03,0xa3e81f93),LL(0xdb1b1733,0x171faa9d),LL(0x3828e41b,0x0bbb5e91), LL(0xf0828387,0x789a7b2e),LL(0xfca60b9b,0x9465cc16),LL(0xab630d23,0xcb58e90a),LL(0xa339d4b4,0xe7d30293), + LL(0x18e75428,0x0bcac958),LL(0x9a95900a,0xd2f1554a),LL(0x03846833,0xc63c2afb),LL(0x6d1e8753,0x703d0220), LL(0x04a357a2,0x47f5fe57),LL(0xcdc17255,0xaafba53e),LL(0x821af8d5,0x8f94c8eb),LL(0x35e37920,0x4d9918bc), + LL(0xe32dd067,0xc029bd84),LL(0xf77f8962,0x25982357),LL(0x510b7cfb,0x7af256ca),LL(0x446925d7,0xca397f37), LL(0xe0614e1e,0xb3dc7be5),LL(0xbbc4cc93,0x3b64cd27),LL(0xfb99bbc9,0xbd762df5),LL(0x04d7177f,0xc1ef0d4d), + LL(0x65e75ed6,0x77b6d3d6),LL(0x53053b45,0xbe59c8da),LL(0x54fe92cc,0x054d089f),LL(0x6fa4758f,0x2f2327e0), LL(0x5d2b5a01,0x948cf03f),LL(0x9c23b83e,0x47544c4c),LL(0x6c128d69,0x338590fa),LL(0x76842160,0x5724008d), + LL(0xc4f063e6,0x4cbeb18e),LL(0x9c2f826c,0x507ba094),LL(0x6f4e49f3,0x0e877a6e),LL(0x34f56868,0x050c2040), LL(0x0f119e25,0x8fd667c4),LL(0x13b47d3f,0x881dd34d),LL(0xca8e9a6a,0x2a6b636d),LL(0x107ea18c,0x67b081fb), + LL(0x84af55d9,0xd3a46367),LL(0xd7626b67,0x0e709a00),LL(0x4c6dfc8e,0x135725fa),LL(0x133a6e20,0xbf6328d9), LL(0xa87031f2,0xa4b743b4),LL(0x13825d07,0x62e90e67),LL(0xb85f3c45,0x7004028e),LL(0x11751be0,0x0465c502), + LL(0x8a5ab076,0x4d5d467f),LL(0xf4fb8a45,0x9aa3f414),LL(0x5dc1fa84,0x9fa0422e),LL(0xd02cfd2b,0x3205c05f), LL(0x078836b6,0x3eac28fa),LL(0xfc3ff573,0x53bc0189),LL(0x00b02100,0x2c45ef09),LL(0x34360ef7,0x61bc02ae), + LL(0x532e8d6a,0xeb5593e7),LL(0xf02a1ee4,0x94092904),LL(0x200496c0,0x379b32e8),LL(0x360a27c8,0x46fb6e9e), LL(0x62005158,0x8a3377ba),LL(0x0de3f191,0x1a3266c1),LL(0xc94d2127,0xe60fad96),LL(0x646302f3,0x41553dd1), + LL(0x377e0766,0x88bf0bfa),LL(0xe75bf57b,0x870a24db),LL(0x79e77976,0xc133cb49),LL(0xf43b6f18,0x2f14924d), LL(0xaa94cd73,0xe561dc90),LL(0xd6eb695d,0x8c420eb2),LL(0x2f04ef79,0x99e41ba8),LL(0x71e6d054,0x7f427bdf), + LL(0x10bde8ce,0x7304bb25),LL(0xe48b16f8,0x5dbc4325),LL(0x8796db7b,0x47d17ab2),LL(0x94c77832,0x83426817), LL(0x9878ace2,0x6781850e),LL(0x019e97aa,0x7f747b90),LL(0x949f9b08,0xa0545c85),LL(0x244bc083,0xe0a0bbf8), + LL(0xa1f38ea6,0x8cb53666),LL(0x4989a568,0x9be29ff6),LL(0x083a7fcd,0xbc5a7f87),LL(0x44ca10f6,0x90d0129c), LL(0xd724b7e2,0x1ad274bb),LL(0xcad5f069,0xa5290cbd),LL(0x86a4e0a9,0x886b1a7c),LL(0x8d8fb13f,0xd2481b5a), + LL(0x4f606ac5,0x80075fb2),LL(0xbfc10e7f,0xf984b5a2),LL(0xf056142f,0xd3d91aea),LL(0x4afdc017,0x770bee0b), LL(0x6c49c827,0x3c42ca88),LL(0x0aaa3139,0xb620c4e8),LL(0xad87890c,0xac6b512d),LL(0x0eb61f92,0xaee62df7), + LL(0x21dad9ec,0xcf0f37fc),LL(0xc52e24c1,0xd780d315),LL(0x23a2b699,0x0263bcab),LL(0x9714b577,0xdc8dcd2f), LL(0x55622b11,0xeb16eca8),LL(0x01118edf,0x94b3d649),LL(0xec66879d,0x6bafea64),LL(0xc4ab9f48,0xc35739c0), + LL(0xf3232370,0x082ccf53),LL(0x71407825,0x01b55dd3),LL(0x4f7f4038,0x86e0fe94),LL(0xb04159e9,0x1a623675), LL(0xbc4df337,0xf908ca59),LL(0x816162ce,0x1b4f1ffe),LL(0x2d60e05b,0xb5128952),LL(0xd38cbdf7,0xb47ca0eb), + LL(0x8ee38219,0xdccba22f),LL(0x9fbb36ff,0xc9436453),LL(0x8ac12c9d,0x83cecbf5),LL(0xf4cb1ebf,0x591191b5), LL(0xf03c1632,0x693cf383),LL(0xcb6abaca,0xaebd3f9b),LL(0x0fa26e7a,0x1427c154),LL(0x01bf37af,0x4f0de894), + LL(0xd88da2a6,0x4e497acf),LL(0xe5c86482,0x8014a215),LL(0xcf94ee40,0xa627d78f),LL(0x9ca36aa3,0x7647708d), LL(0x87e108c8,0x3d4e8bb1),LL(0x516f8b28,0xacdc3223),LL(0x1d955038,0x74e4d436),LL(0xedd68905,0x7e4a198c), + LL(0xc4bfbad1,0x41dc4bdb),LL(0x85964942,0xfd1121b1),LL(0x0c0d85ff,0xe462eb9c),LL(0x2b012d88,0xade1ccb3), LL(0x318f2aa3,0x2eec3503),LL(0xdec8a13e,0x656b44da),LL(0xcda13a8c,0x90b7aac8),LL(0x214a530f,0xe7f3a5ff), + LL(0x159060b9,0xa0c8062a),LL(0xd9305873,0xc19f2608),LL(0xc9609521,0x0d6213c4),LL(0x9aec4539,0xde2b9134), LL(0xaeddf0a6,0x4a2a6476),LL(0x5cf2e85d,0x89222dff),LL(0x084a0245,0xad92a1d3),LL(0x2138063f,0x29644a60), + LL(0xb8164832,0x5b57a05b),LL(0x885ce4d1,0xecf6620f),LL(0x045d3b88,0xde28ed2f),LL(0xb4502308,0x3eb11759), LL(0x4d9f94b8,0xe97f1db2),LL(0xeb83f016,0xfa248583),LL(0xcda53ba0,0x63a273b4),LL(0xa228b7b9,0x9692973a), + LL(0xb6707cbd,0x5968cb12),LL(0x5c1a2b4d,0x1895ccb4),LL(0x37f0b240,0xff309157),LL(0xb90875c2,0x374d983e), LL(0xc4e815e7,0x22fc40c6),LL(0x98d18506,0xf2db27be),LL(0xaa9ae62e,0x2854a948),LL(0x79e39df1,0xd614a722), + LL(0xb3501c19,0xebeec551),LL(0xd89cefca,0xe2b878eb),LL(0x6b4cd6bc,0xa0a34757),LL(0x70bfdf88,0x0159129c), LL(0x489502ca,0x26fa3e53),LL(0x285be050,0x7932084f),LL(0x71912b0c,0xfe4998f4),LL(0xc60b88b7,0x3dce0a87), + LL(0xc718ee5c,0x5b93edb0),LL(0xbb075836,0xb93a225f),LL(0x7aa0595c,0x87a08c94),LL(0xc31e3249,0x401d2620), LL(0xdae2cdb9,0xbe6928b4),LL(0x5a226630,0x4b68e106),LL(0xc9d32e4f,0xdc38c2fc),LL(0x26542f89,0xc51a6245), + LL(0x1757f3c4,0x5fb37c1b),LL(0x89128aa4,0xa27d6c02),LL(0x5e629309,0x3b74f56f),LL(0x2f7aeef2,0x24b5ad84), LL(0xbdc89596,0x54a962cc),LL(0xcc2f3d5d,0x6e8bccf8),LL(0x312e9241,0x4c1df22c),LL(0xfc30f0dc,0x8ffe6b0d), + LL(0xa6ec0fe4,0x670431a2),LL(0x35964572,0x49da0b42),LL(0x9dda5c39,0xbb12d1b0),LL(0x72d3de51,0x64170fe1), LL(0xa4a2f5d9,0xea8b2b16),LL(0xe590be92,0xde1bad64),LL(0xf0b9b0b5,0xb7f93581),LL(0xe115d67e,0xb007f4dd), + LL(0x415732d4,0xab9d7523),LL(0x905ec0fe,0x2951149d),LL(0x74350478,0x94bb2c63),LL(0xe9b1ada4,0xe6b63bfd), LL(0x13e8528f,0xd09b4d4b),LL(0x685bf239,0x6bed3d25),LL(0x1a14b7ce,0x83023ad9),LL(0xd0505d6b,0x4bffff63), + LL(0x8bb1cfa0,0x2ccc180a),LL(0x3a09c912,0x70c18508),LL(0x5878201c,0x318c41c2),LL(0x64c01149,0xb9f207b1), LL(0xc58287d9,0x89fdd9ee),LL(0x05c43da0,0xdb6fa8db),LL(0x311a34a1,0xc31354f0),LL(0xf1521976,0xccfbaddb), + LL(0x0c5e8388,0xf4175f75),LL(0xa81226cf,0x7e090ce8),LL(0xeda6a1ab,0x5209107e),LL(0x7e5ccefd,0xf0a263cb), LL(0x10c704a1,0x9fe05a36),LL(0xcf133aee,0x3784d7ca),LL(0x37af86c8,0x066c3116),LL(0xd7ebeb8a,0xbf32ca04), + LL(0xa9b5bab4,0x0447a950),LL(0x41bb1f1c,0x3b2f98bd),LL(0x59c62f08,0xd89bbdd7),LL(0x3ded069b,0x26bab370), LL(0x69ea63ae,0xb0db4ca5),LL(0x9133df68,0x57b32f32),LL(0x5a17efbe,0xc79a22d0),LL(0xf8ae3c2d,0x576976a3), + LL(0x9d02d212,0x5754198b),LL(0x230d0213,0x9cc9e61e),LL(0x92889e33,0x76772179),LL(0xf5df6cba,0xb1246608), LL(0x8d491280,0x821766bc),LL(0x96bd3df5,0xe148f470),LL(0xed753b73,0xc1e9fc70),LL(0xd6cecfc5,0x840e40ed), + LL(0x93e2f3a0,0x03874679),LL(0x5b646b64,0x462e5abf),LL(0xd7ae0e67,0x6fb19eda),LL(0xc3d2dddf,0x01e8a27f), LL(0x4bacfe2d,0xc9e69639),LL(0x712e8afb,0xbc3a134e),LL(0x8af6d30f,0x5d943a86),LL(0x443c942c,0x65eb5f99), + LL(0x2339e348,0xf5000308),LL(0xeb0d80e6,0xd69b7693),LL(0x5b9d220b,0x7b00b43b),LL(0x497bbcf9,0xde0dfc80), LL(0x0c2e851f,0xcfe2e3f3),LL(0x7e91d378,0xef7793d1),LL(0x9d266a5b,0x9e43eeac),LL(0x1766c5c1,0x9c81d68b), + LL(0xf6a4d560,0x121db320),LL(0x073582a7,0xcd0a4f03),LL(0x6e841041,0xbf469f9a),LL(0x5eb2d753,0x4314f0f6), LL(0x8c498414,0x09021001),LL(0x859275b7,0xf63d00ee),LL(0xf1c0385a,0x228fa809),LL(0x694c3318,0x44259d51), + LL(0xa2ad4eac,0xb0a5d5fe),LL(0xabdedf11,0xbb950010),LL(0x6264c601,0x081024ce),LL(0xaefb4115,0x6cc7faf2), LL(0x8692a7a4,0x90511289),LL(0x60596012,0x2bef4135),LL(0x0f0becb6,0xfec5209a),LL(0xd1ceb42e,0xad0907a6), + LL(0xf517372c,0x6cde3f21),LL(0x26131459,0x9acd4f09),LL(0x491f59d7,0xf3629a43),LL(0x41a037dd,0xe944561a), LL(0x826d1b5c,0x07beeabe),LL(0x3a1476cd,0x0728a907),LL(0x7d4a1abf,0xa91f41a0),LL(0xa7a119c4,0xdf58ed06), + LL(0xba701704,0x19990669),LL(0x47b67175,0x8aa3f76b),LL(0xdd0a6e9a,0x8bccff3e),LL(0x24f49029,0x4173fcda), LL(0x61c18233,0x2a688911),LL(0x78b9fa8f,0xdf54b239),LL(0x37596f40,0x714cf627),LL(0x24e6a879,0x2c73ddba), + LL(0xf2547f19,0x1538fd36),LL(0x0e7e84ee,0xd85c4730),LL(0x306f5fc0,0x00799e9f),LL(0x49ce114c,0xfccc6a37), LL(0x3fe08120,0xf9cff5e8),LL(0xc2be9f27,0xdf876a1f),LL(0x6939fdb9,0xe817c72e),LL(0xd34d0e43,0x244a1800), + LL(0x78fa7f11,0x41e83eef),LL(0xba6367e5,0xecaa250c),LL(0x8def6ae6,0x9c420347),LL(0x250b9e58,0x99efb3b1), LL(0x79b2298c,0xdaf311ee),LL(0x69b6dff3,0xb49200cf),LL(0x559e51f5,0x5c7f17bb),LL(0x424be7e9,0x117d0cbe), + LL(0x36e3af54,0x290a35c4),LL(0xe3a643b1,0xd2326cd8),LL(0x3580f9ee,0xc208b2b3),LL(0x4464a9e0,0x2419c661), LL(0xbccb2759,0x87123d3a),LL(0x1a77d469,0x5d36fcf3),LL(0x49b07e5a,0x5aafd58a),LL(0x6b71e237,0xf534595b), + LL(0x6705039f,0x0f0d3161),LL(0xca701676,0x7282b08c),LL(0x13796941,0xb05e8c3e),LL(0xfca06e08,0x5250012e), LL(0x980c5ea3,0x7eb2341a),LL(0xa41defb2,0x92f5aeb1),LL(0x0e62453f,0x203244e0),LL(0x96181756,0x74341218), + LL(0xe12a94ab,0x3b0cd36f),LL(0xb5ad7c48,0xf364b3b9),LL(0x8e768469,0x96a7a2a7),LL(0x1bbc7cc5,0xccc31c7e), LL(0x080dbb92,0xe70ad5d0),LL(0x56fb0f1f,0xfb201e92),LL(0x29d99f57,0xdfce7a1e),LL(0x06457da5,0xc12a02b0), +}, +/* digit=33 base_pwr=2^231 */ +{ + LL(0x2a80f39c,0xdea72ba6),LL(0x68cbe088,0xcb2174b1),LL(0xd6bd1cc1,0x9c73ec69),LL(0xf20dcce6,0x6a2dbe20), LL(0xeeaae9da,0x20a5150b),LL(0x9df630da,0xc935e85d),LL(0xa1634cd8,0x2147144f),LL(0x44f3af02,0x5eccb56c), + LL(0xc0e2b70a,0xf77a79cf),LL(0xee8cbae7,0x2569c8bc),LL(0xfadb18fc,0x392a5dbe),LL(0x3ce6a0ff,0x59bc96b4), LL(0x8b551005,0x287f04f4),LL(0xa44b2bd8,0x7efe3aa5),LL(0x6ac447d7,0x0e9cb8ed),LL(0x7783bdd0,0x9b4eb10a), + LL(0xb981d96f,0x793c4880),LL(0xd56fb2a6,0xf719d828),LL(0x8149057e,0x9fcc236f),LL(0xb4d65c2b,0x318c63ec), LL(0x5f95c5de,0x5269c1d7),LL(0xd0efd6bc,0x33b3745f),LL(0x4ae87c64,0xace344d5),LL(0xdd30ba2c,0x238809d6), + LL(0x71192348,0x3cc32acc),LL(0x3f17ef60,0x545401bf),LL(0xcde25b0e,0xe209a493),LL(0x663abab9,0x5c11886b), LL(0x28ec7c90,0xe61a81b1),LL(0x75b57f5c,0x18b125a6),LL(0xfad91696,0x86d1b45a),LL(0xa4c3f7ff,0xb31a786d), + LL(0xf45831d8,0x2fd4cd72),LL(0x8be40d9f,0x85978fa6),LL(0xa9301111,0x38106329),LL(0x9e5979ee,0x1527e462), LL(0x76c5fc8a,0x97f71c7e),LL(0xf1f33056,0x205fa473),LL(0xa6546a05,0x7bb9d24e),LL(0xf84c4d35,0x0e282a5c), + LL(0x59471f1f,0x59d21896),LL(0xf6303573,0x2e613dec),LL(0x78bf5a4b,0xa7995794),LL(0xbf19fbe5,0x20adf6b5), LL(0x1574d34d,0x3a48c95f),LL(0x09323ceb,0x95488f09),LL(0x552df9cf,0x450aee7f),LL(0x53557500,0xdf016f7a), + LL(0x2da8a2a6,0xf2acedc6),LL(0x2f4a0632,0x03fc8cf8),LL(0x5b82f03a,0xe7ff136b),LL(0x9e88c421,0xd5841c4d), LL(0x7eef63f0,0x75a4d66f),LL(0x2865c14b,0x92091ade),LL(0x64fe7ba3,0x7060474c),LL(0xfe30cb3e,0x4056724c), + LL(0x8d9fceb6,0x38cf4c6f),LL(0xab528f38,0x11e85f78),LL(0x52303b2b,0xe2896d25),LL(0xed68c605,0xf929675a), LL(0x10c708a9,0xfbd22374),LL(0x40d7e5a7,0x4682ca17),LL(0x9041047f,0x4242b5c5),LL(0x0f9c6840,0xaf571053), + LL(0xa56af433,0x713b2bbb),LL(0x5e82f947,0x45aaf2ce),LL(0x106283c7,0x9882571a),LL(0x9b9c3c3c,0x37de12ca), LL(0xbef10529,0xcb463af2),LL(0xd771236c,0xe18d763d),LL(0x62935de3,0xb47a69ca),LL(0x9a41e09a,0x4798e91f), + LL(0x8e93edd5,0x89696697),LL(0xb7ea4f45,0x35cdb8e1),LL(0xfed33d87,0x36f8305d),LL(0x625642d6,0x57623440), LL(0x945dd7d6,0xdfd9b580),LL(0x731739bc,0x965ffcb5),LL(0x637bf127,0x34588e1f),LL(0x539d21c7,0x936c0ba0), + LL(0x71640eed,0x70832099),LL(0x3ff407e3,0x916b1952),LL(0x88440bc0,0x4cd58881),LL(0xc280e977,0xd9fcb83d), LL(0xdf6cda83,0x0d3df9db),LL(0x3d55047e,0xc629414e),LL(0xc16b84c9,0xe05738a8),LL(0xe8783570,0xf4bdc724), + LL(0x9a93a5c9,0x7d876a59),LL(0xa8d12f61,0x026be75c),LL(0xe9b2aa43,0xe49900ed),LL(0xb3a68dad,0x44d6dc80), LL(0x7d23e11b,0xf96b116b),LL(0xb6814209,0x12791212),LL(0x6cc65956,0x3e2807cf),LL(0xf405ffae,0xcc606ca7), + LL(0x5df47226,0x5484b2c5),LL(0x8802da81,0xfbaf9042),LL(0x087adadc,0x84146315),LL(0x58d593b3,0x6adbcbc1), LL(0x68b97533,0xc1fb3896),LL(0x954cc1b7,0xa6919aac),LL(0x27a4ddd0,0xf301b2e4),LL(0xdf1a07b1,0xa15c16eb), + LL(0xc145a6c7,0xb36c017d),LL(0x968798da,0xcca64382),LL(0x8d0eff5d,0xd13b6376),LL(0x2206e681,0x06e39e2d), LL(0x3add517a,0x1d9dffa4),LL(0x10d95fef,0xe670e6c8),LL(0xf1c7c7a8,0x0ecb51ab),LL(0x74945305,0xf17dff51), + LL(0xb00d9c0f,0xf71b09b1),LL(0x9c72c80a,0xc719cf62),LL(0xe00a49a1,0x310c5aeb),LL(0x1b33c5e6,0xd01285e5), LL(0x05aa6eb7,0x7b23e7c7),LL(0x6bc88677,0xf84188b1),LL(0x64be321a,0x7e034cb5),LL(0xe884323f,0x270df734), + LL(0xe5137d20,0x218e68f9),LL(0x0f7e70ad,0x79588cba),LL(0x58b86b0a,0xb6d37f52),LL(0x7cc591fe,0xcb281c98), LL(0x8fe40e06,0x30e03fed),LL(0xed9ca793,0x394ded95),LL(0xbcd3a3da,0xf1d22cdd),LL(0x0c591130,0xcb88cb27), + LL(0x8ff0cbf4,0x67786ba3),LL(0x0565d337,0x85738a44),LL(0xaf9a667b,0x9d3b35ec),LL(0x8f5f540a,0x45a17512), LL(0xade5a5ba,0xf1ae5171),LL(0x39869be4,0x720e2823),LL(0x5352d84b,0x6893f14a),LL(0xc784dc20,0x919a4f15), + LL(0xf7ae5a67,0x36136612),LL(0xeaff0924,0x11f43d1c),LL(0x39449b96,0xcfee088c),LL(0x70c42ff6,0x3dc48359), LL(0xbf208b67,0x4072683a),LL(0xbe15d18f,0x35a7c2be),LL(0xe2c3391b,0xe61d2748),LL(0x39921d95,0x0a4109b1), + LL(0x05545cfc,0xe2cd131b),LL(0x3ae20d7f,0xa898da2c),LL(0x50dc4c61,0x501cd849),LL(0x3374e7f0,0x10e287d4), LL(0x38aea879,0x90d6326f),LL(0xef43fa08,0xc48d9af7),LL(0x6c20f168,0xf8f4886a),LL(0x23ccac4b,0xc5d34a86), + LL(0xb3d7004e,0x72357752),LL(0x817bd377,0x167db0ed),LL(0xdfb64d05,0x5d45b3da),LL(0xf0b414ac,0xed4b7fc4), LL(0x0bf1dd64,0xc178941b),LL(0x8fe835a5,0x43feac17),LL(0x6a014609,0xe1c23a17),LL(0xd5e23bd5,0x63255991), + LL(0xd7dfec55,0xefc76468),LL(0xc0831696,0xb1bc3fee),LL(0x5f52433b,0x0996811b),LL(0x799649fc,0x6b8b6daa), LL(0xab518b64,0x6e9f7cb6),LL(0x38a3a2ab,0x6a67a009),LL(0x928209e2,0xe55de954),LL(0x3da81142,0x98b6d0a7), + LL(0xe3f832e8,0xdec30331),LL(0xe50fa9e3,0xa9b77f3b),LL(0x5167c6a6,0x20febc21),LL(0x76fb0f13,0x0ce07d1a), LL(0xe796f8a3,0x9745dead),LL(0xd95deba6,0x2cb4eb1f),LL(0x4caf2afe,0x062e7cac),LL(0x16ace879,0xf50ce065), + LL(0x1d99d3e7,0xdec8954b),LL(0xa48262c7,0x5287e954),LL(0xcc3530de,0x1c6fbd17),LL(0x53af4211,0x6bcbea50), LL(0x4dce058d,0xe3533bca),LL(0xfc9cdf00,0x6fe62e64),LL(0xe8ec4cf9,0xee29fdec),LL(0xc8d52f80,0x7361797d), + LL(0xf4e36023,0xb1d858da),LL(0x73e6dee1,0x4a1282ce),LL(0xce1d71cc,0x6ba8f8ba),LL(0xcbbd8eb9,0xf5b7d6b4), LL(0x5aed382b,0x60f8bd50),LL(0x3f3a46b1,0x47b40519),LL(0x8a997d93,0xaed13bb9),LL(0x4dc6e35d,0x6cc2260e), + LL(0xccf915d9,0x173bfcdd),LL(0xc2d46f6e,0xad4525e1),LL(0xcdd2382b,0xb7ecec0b),LL(0xd2b76c84,0x01ae8291), LL(0xbec6b980,0x2d1e2a91),LL(0x7008a00c,0x1b0040be),LL(0x7d555120,0x6ac708d7),LL(0x0d745eef,0xa6017568), + LL(0x1ed38ef4,0x735e3511),LL(0xcebe5a8c,0x7c97f284),LL(0x24fecbac,0xd4059313),LL(0xde18c42c,0xf874ca4b), LL(0xdbb829b6,0x9ab736a8),LL(0x82ff128d,0xe914bdde),LL(0xfd0f362b,0x6e949bab),LL(0xffea2e79,0x275824cf), + LL(0x8cc52417,0x81f57245),LL(0x2f274090,0xed0a9079),LL(0xfdd0ba2f,0x98c3372e),LL(0x3ae99478,0x49820f41), LL(0xce373d3e,0x1c47e09f),LL(0x6dd12846,0x875d7920),LL(0x15d5bbb1,0x7a9e7973),LL(0x6d227962,0x48512656), + LL(0x199241e3,0x2c167c88),LL(0x7209ca37,0x98c1df6a),LL(0xde89e412,0x09a1583f),LL(0xc792de48,0xc19ed5b9), LL(0x74dc0834,0xb8dd1b1d),LL(0xa04456e9,0x9d458529),LL(0xad0ad39d,0x66ef5463),LL(0xe641edc5,0x8d7df4a1), + LL(0x6bd322e4,0x97815de2),LL(0xc1f77fb3,0x0bf6fc83),LL(0x8b4f7152,0x49378167),LL(0xbf0a1712,0xfdd476ef), LL(0xf2f9883a,0xe87e1977),LL(0x9ad2d034,0xdbb2fcbf),LL(0x64e1a4c6,0x5afdd161),LL(0x3e435191,0x0e43f911), + LL(0x42253804,0xde2d1ee6),LL(0xdaf13e57,0x6def6ceb),LL(0x05423bab,0xae09fd4f),LL(0xad46e409,0x6f6c17b8), LL(0x6c345cd3,0x966fa09d),LL(0x47abc35d,0x6c8aa1e9),LL(0xe015a566,0x02991686),LL(0xcd2f925d,0x39b3aeee), + LL(0x9119c117,0xf9cda92a),LL(0xf4f833e1,0x7b21ce82),LL(0xe4f99193,0x87517bf5),LL(0x4eb412f9,0x1b7ddec9), LL(0x6b077498,0x7a30dd57),LL(0x0ec44230,0xe060625f),LL(0xb0e5446b,0x0f355dc4),LL(0xbbd2df28,0xdf324d65), + LL(0x649966a5,0x28c7eb34),LL(0x26639e19,0x97587f4f),LL(0x0bce0f38,0x0724cc00),LL(0x4ae6280a,0x63578add), LL(0xc7fd6a1a,0xf1beaa57),LL(0x7b017e35,0x83b1a533),LL(0xefdf2ed1,0x01c027e3),LL(0xd2d31852,0xf373d4ea), + LL(0x65b8f5d7,0xe568acb6),LL(0x1240a545,0xea8ce1b8),LL(0x555fac44,0xb95b0db2),LL(0x768333dc,0x01d18170), LL(0xab1798ad,0xf938b55e),LL(0x999a7e5b,0x73a0d9dd),LL(0x57fd9b51,0xd2359bb5),LL(0x77fb4e5e,0x20f1d4fd), + LL(0x932dc388,0xf5efd71f),LL(0x21a37385,0x40f86819),LL(0xff935ef3,0x05395fb2),LL(0x1b615e8e,0xc2ee43ac), LL(0xe82d509a,0xa3bb6518),LL(0x30b93347,0x3a87d5a2),LL(0x5b130bcc,0xac0a5ad0),LL(0x9154d73a,0x91fe8fdd), + LL(0xdeb203a2,0x677d7d48),LL(0x8b0168e1,0x4d4108fe),LL(0xddc3d24c,0x16be4ad1),LL(0x9865df69,0x9b0ea387), LL(0x4c50ec70,0x16daf932),LL(0x478c96a3,0xa4799bda),LL(0x7114d3bb,0x4ef24d3f),LL(0x46e6bbdc,0x30a31509), + LL(0x97f3cb4b,0x60137187),LL(0x0a29d865,0xf2b66d8f),LL(0x60064a5c,0x93a4a37a),LL(0xe8c3cf47,0x7dee9bed), LL(0x0b7ee8b8,0x748833ce),LL(0x56f89483,0xc07f2f6d),LL(0xd24b406f,0xd71a40d8),LL(0xebbb7655,0xbe3b2e8f), + LL(0x4a536998,0xa23c2b05),LL(0xa9be2e14,0xdcaf45b3),LL(0xfe346ad1,0x515ad15f),LL(0xb7847640,0xb9c06a18), LL(0xf35bff4a,0x8552eb06),LL(0x2b7a29f5,0x4fb792e7),LL(0xa41a38b4,0x1cce2af5),LL(0x02b42414,0xde68bd0d), + LL(0x7cd66728,0x8124d6e2),LL(0x55efbadd,0x5906d1b4),LL(0x827f2741,0x7e17855a),LL(0x12c6966c,0xab525dfb), LL(0x758e0cd3,0x065ae99a),LL(0x517318a9,0x0dcb8f5d),LL(0x42441f5e,0x48756645),LL(0xd79d535e,0x03859154), + LL(0x8217e4bf,0x99bb28cd),LL(0x8291e54d,0xd6aed2e5),LL(0x1c92a65e,0x8f9067e3),LL(0x1540b9b5,0x120890ea), LL(0xec60a215,0x227d7c86),LL(0x556d8c65,0xb6609e85),LL(0x47f8c8a3,0xa6a26c37),LL(0xf1204bdc,0x4c850fe3), + LL(0x42db4eb8,0x25f7e61a),LL(0x3d62869d,0xfdf05575),LL(0x52b31c23,0x8b36a744),LL(0x1a5e8d4c,0x83b83c89), LL(0x5d9208bf,0x72d38dd3),LL(0x8cf7b6f4,0xbeb8873b),LL(0xcf90bcb6,0xa3ec5c36),LL(0x9a6d5fe7,0x35adda6f), + LL(0xf61c68d9,0x7312423d),LL(0x20bcaf77,0xb1c4e10f),LL(0xf168ee57,0x4df2850d),LL(0x180985e1,0xed07a4de), LL(0x2fba1f23,0xcb353d6b),LL(0x778cc15e,0x00ea9094),LL(0x20857170,0x4967faaa),LL(0xd7153bc4,0x9ff70dbe), + LL(0x59f62fc6,0x49eb7994),LL(0x3c91862d,0x5f459faf),LL(0x46d8f2e0,0x1c10f621),LL(0x252954e7,0x7e669c9a), LL(0xa83f6c57,0x4ccf659a),LL(0xec0a0a03,0xdc2b77eb),LL(0x2cc6b8a2,0xcf09ad07),LL(0xa97aa2d0,0x231960fc), + LL(0xde227de8,0xc0609130),LL(0xf1d7ddf1,0x40d2691c),LL(0xf9a73890,0x60349cf4),LL(0xf9968132,0x3f50005d), LL(0xf16f44b9,0xb4be853e),LL(0x799caac5,0x48bf4804),LL(0x3c562849,0xe6a64876),LL(0x854f283f,0x2f4d487f), + LL(0x159484c4,0x64b77e39),LL(0x523e1529,0xd419d4bd),LL(0x143dcf7d,0x1bf9510c),LL(0xed5cb4e1,0xa85bea71), LL(0xec715818,0x73a4cfd2),LL(0x67f475f5,0x88b11d0e),LL(0x4d12361c,0xbfe170d8),LL(0x00a0f979,0x9fc48e64), + LL(0x65682105,0x6a8bb2dd),LL(0x00bd952a,0xc1362a9c),LL(0xa6013753,0xef5b3d89),LL(0x8fdfa22a,0xc87bbacb), LL(0x31bb19e4,0x74fbdfc0),LL(0x32bfe260,0x7d058029),LL(0xe53da990,0x54a4cce4),LL(0x822da672,0x01acdff6), + LL(0x95597766,0xd2a2d484),LL(0xd43dc7fd,0x5960ac1f),LL(0x8d6db685,0xcf095b6f),LL(0xa85618f3,0x87232088), LL(0x34753c7c,0x91497a48),LL(0xd6353024,0xf682e372),LL(0x0c9b271c,0x7889ceda),LL(0x18340951,0x7126504e), + LL(0x967c8a60,0xf786b821),LL(0xc17f3d99,0xfce01b37),LL(0x1f2a8575,0xe23c00a1),LL(0xab6ff8a0,0x7f56aa1b), LL(0xd193dfcb,0xdb73869d),LL(0xd644733e,0xbec02c94),LL(0xf7b43261,0x283117bc),LL(0xb4108e39,0x920acf5d), + LL(0xe49aebb8,0x33f1ef5e),LL(0x0fcea2c1,0x9ead51e4),LL(0xf8503f28,0x1f800a68),LL(0x34a75f67,0x78818531), LL(0xb70ffb27,0x1aeb3760),LL(0xcb6309e9,0x1cca590a),LL(0x7170d241,0x8d09f360),LL(0xa0e0d0f8,0xbc970b5b), + LL(0x31d038a3,0x2ec93eea),LL(0x5153f8a2,0x3482a0d7),LL(0x6641b5d8,0xedcbe914),LL(0xe516e387,0xc086e61b), LL(0x9b875513,0x03814266),LL(0x37340a4f,0x6d37fee3),LL(0xe5d17ab7,0xcf78515e),LL(0x4119a759,0x0c7cd830), + LL(0x54924618,0xbd49062b),LL(0x41e7e7a3,0x34c44f45),LL(0x706bd0af,0x0039f3d2),LL(0x0be9a628,0x146cadc6), LL(0x57d48746,0x6d5d5020),LL(0x82caf4b0,0x0ea43f7b),LL(0x8a064d34,0x11a08927),LL(0x95638fa2,0x30c0ef40), + LL(0x602a871b,0x4b950c04),LL(0x6a731537,0xf50cb1ef),LL(0xcbe5e4ef,0xb87a1cd3),LL(0x3dd1c601,0xb1fc4894), LL(0xa516617a,0xdf402711),LL(0xaaf63c65,0x5bdd1d67),LL(0x6e559bd9,0x020d1062),LL(0x81ec09fc,0x4dec26d0), + LL(0xeeeeb2bc,0x7b91fafd),LL(0x33aaf2c4,0x56290f98),LL(0x79c7bf9e,0x57abbd27),LL(0x2b1e1ecf,0x568bdee6), LL(0x470f1d01,0x58f8c80c),LL(0x1b9cb76b,0xeecfe398),LL(0x311a0634,0xc0ffa4de),LL(0x0ae99877,0x425fcd13), + LL(0xf7bd0748,0x1964c681),LL(0x9d920471,0xebcca16f),LL(0xab4aa03e,0xa72b40cb),LL(0xa05624fc,0x4397d6af), LL(0xa94fca0a,0x372d522c),LL(0x3035b9fc,0xe1010d60),LL(0x4f788d44,0x9f1f00cc),LL(0x6a88b672,0xfd00ec75), + LL(0x2983aef7,0x53706702),LL(0x9b458edb,0xa5f67b0b),LL(0x7db93ca8,0x10789b90),LL(0xfd27cd55,0x885346f0), LL(0x2ebb5f15,0x3af5b0c8),LL(0x2a36b2a7,0x282e4c4a),LL(0xa6d88bd4,0x2f9d5d8b),LL(0x9856b7aa,0x6f016bda), + LL(0xa8198c1d,0x990ae53e),LL(0xa07e7ac5,0x295beceb),LL(0x48c2d246,0x576f790f),LL(0xe3ea9864,0xe99ab2ae), LL(0x43e2d400,0xcf4959f2),LL(0x7a39dfea,0xdd1d8fad),LL(0xfcd7fda0,0xdd6ff9c2),LL(0xb6ace55e,0x61c25b3e), + LL(0xb4dcddad,0xf94742af),LL(0x44601959,0xc49cfa21),LL(0x30c18470,0x07b3f1d1),LL(0x6e6afc82,0x2736cb99), LL(0xe24a8785,0x401fb234),LL(0x074f51ea,0x9af8ba40),LL(0xa9faed0c,0xe1acc646),LL(0xc9667008,0xd5a5f789), + LL(0x68c3ab8f,0xc6436514),LL(0xfe8d6b46,0x6fa0d734),LL(0xaf7f49c7,0xe5fccbfc),LL(0xbebcc58c,0x42c88c53), LL(0xe2a0f7f2,0x7d2e2fed),LL(0x36a18b26,0x694eb76c),LL(0x6b0f657b,0xf0e6ae43),LL(0x48f1ece7,0x8a0f6255), + LL(0x8674bfee,0xd594c168),LL(0xac7d5ebd,0xe59ad38d),LL(0x21645a1e,0x080a6b97),LL(0xf221b37b,0xb900f0e1), LL(0x04cab97d,0x562dabce),LL(0x6f472462,0x5c308741),LL(0xc7c4cba8,0xa5d87e23),LL(0x9b061062,0x5237fe16), + LL(0x222021c1,0xeddfbeb4),LL(0x4e7a2a8e,0xa4fe57d0),LL(0x2de56c47,0x0fbf6bdb),LL(0x6fcebc6c,0x819588e7), LL(0xdf041e3a,0x14196961),LL(0x40cd4f23,0x76a31437),LL(0x8e1a877d,0x44acd14d),LL(0x37d7b7de,0x227a35c6), + LL(0x842a9534,0xe1934f1d),LL(0x53ed73e2,0x7a2ed2c1),LL(0x3903924b,0xcffedd58),LL(0xb306431d,0x7c9dbf55), LL(0x56e06ab5,0x61a72f10),LL(0x616bc5cb,0xb46cf5cc),LL(0xf7c22216,0xecf07e10),LL(0xd9364b3a,0xa4bddad9), + LL(0xda8b1c93,0x548b95b2),LL(0xa1e1c0cb,0xc09a9598),LL(0x21d80851,0xedd80ef1),LL(0xc283f047,0x4684c439), LL(0x87333da3,0x07ca41f3),LL(0xca79a8f4,0x173ec4de),LL(0xb4aec6eb,0x89ce06f2),LL(0x15aaf7f0,0xfe6b0e92), + LL(0x7c1b9ed3,0xdab8876d),LL(0xa2606f83,0x88aba90f),LL(0xbebaf9f6,0xcd21a408),LL(0x0042a162,0x09da6696), LL(0x2d66ccf6,0x4a9b8b21),LL(0x44d5a648,0x34c74904),LL(0x3b0e9564,0xf3fe98e9),LL(0x221aa4a5,0xe4a8a352), + LL(0x26c2b53e,0x6278b4b5),LL(0x1b1708ea,0x4ddf26ce),LL(0x6eb0d845,0x704207af),LL(0x0f5862ef,0x60533de3), LL(0xe54393c0,0x2b5945dd),LL(0x145ea298,0x55941df2),LL(0xc240f654,0xe2b500b6),LL(0xcf9f6934,0x5a49d8f1), + LL(0x27502203,0xfe8d5468),LL(0x58ade316,0x985039d4),LL(0x0a687415,0xefd373f1),LL(0x43526774,0xefccb791), LL(0x0f4497d9,0xeef8d46e),LL(0x1601ab9a,0x4152df71),LL(0xe47b2ad1,0x4250cd2f),LL(0xfb048180,0xa2b63fa5), +}, +/* digit=34 base_pwr=2^238 */ +{ + LL(0x787d1f1c,0xd8a6cb6f),LL(0x3d219a66,0x427bac94),LL(0x383146b0,0x51d7d49f),LL(0x7863d781,0x8164b77f), LL(0x2f9631b8,0x1646b084),LL(0x849388df,0xef5b3aa8),LL(0xe58cd383,0x60536422),LL(0xf43ea3a0,0xb079d911), + LL(0xcb73887e,0x504ac041),LL(0xc3ce3a33,0xf878b618),LL(0x56393e75,0x57ef73d5),LL(0xd276c08c,0xe4372d2e), LL(0x0924cf58,0xfd9bc894),LL(0xaaa317e2,0xfa2a4deb),LL(0x79608da5,0xe51edccc),LL(0x8cd4b960,0xadcc68fa), + LL(0xf8e156c7,0xaa66c201),LL(0x1ab2e3fe,0x7c7cf22e),LL(0x0a677d85,0xe479c393),LL(0xb87c412b,0xc0cd340f), LL(0xf95ff321,0x2b2bcef4),LL(0xb8409952,0x65da11c9),LL(0xeb67eb9c,0x143a2218),LL(0xe53508e4,0x8919ff25), + LL(0xa9e0eeae,0x6f154f09),LL(0xab05a657,0x2246e6fe),LL(0x1045b85d,0x4d7c1c81),LL(0xd3bb7432,0xde99ea37), LL(0x63184ff4,0x058f8187),LL(0xd134bfc3,0x2a223421),LL(0x23120320,0x1560dbed),LL(0x76a3de9c,0x37243c95), + LL(0xd36a81b1,0xb8f3851a),LL(0xbdad7ad9,0xfbc62bfc),LL(0x561e0f8c,0xf68215c7),LL(0x1bcf765b,0x894131d1), LL(0x45c5d736,0x8da01f9e),LL(0x7484e0c1,0x025de05c),LL(0x6858b504,0x62f4c66c),LL(0xd6dc5f93,0x754b85d6), + LL(0x822a3de0,0x5b37cecc),LL(0xa98a37c2,0x422e49b1),LL(0xbe41e927,0x3ef53d89),LL(0xf4d5bffa,0x0994dd11), LL(0xf7eacca3,0xa62ea556),LL(0x7c746025,0x37b4e230),LL(0xa8e14253,0xb4291e37),LL(0x2a2b666c,0x2bfc9eaa), + LL(0xc26e5588,0xf604604a),LL(0xa7ec3971,0xf75816ff),LL(0x26a30a6d,0x859e9ec7),LL(0xa1a5b815,0x2ce57b66), LL(0xd65e8ec2,0xc7aa4df4),LL(0xa5d82ede,0xbab6b3bb),LL(0x7b088314,0x7a11b25d),LL(0xc2c636ac,0x501a3891), + LL(0xe256b02b,0x9f116c8f),LL(0xfa5946e0,0x71495693),LL(0xc335452a,0xeb9696ff),LL(0x4971162e,0x01ca5929), LL(0xc0f28e72,0xee0a1f50),LL(0x70d8df1a,0x2baac62c),LL(0xf49110f8,0xcf65d297),LL(0x9a45e16a,0x041dbb01), + LL(0x5e1410c0,0x8db69426),LL(0xa70d0268,0xb21f3c6a),LL(0xbac0ddac,0x64a3c30e),LL(0x66a2d33a,0xdcebdedc), LL(0xa519de21,0xc5dcd769),LL(0x19322c69,0xa692b6a0),LL(0x154fca13,0x454add5b),LL(0x4935eba2,0xd2281cf0), + LL(0xf2602323,0xb5f44fe7),LL(0x5d68a3db,0x772fb6a6),LL(0x76eec37a,0xf519c5d4),LL(0xada6c3f4,0xbc8e9a15), LL(0xf18effee,0x9cd2a2f2),LL(0x1808ab42,0x9601c142),LL(0x0480ad18,0x05d110f7),LL(0xef6a7f33,0x5f7e0721), + LL(0xe6409e21,0x1afbeaec),LL(0x317f7967,0xf6714651),LL(0x80124751,0x34cd21ff),LL(0x931d9639,0xf85c70ec), LL(0x1ca19094,0x4e26bef6),LL(0x0b841b9a,0xc513f66b),LL(0xb9b41001,0xe25507bd),LL(0x94f49f7c,0xd77fee94), + LL(0xd39e1ee4,0x20b19223),LL(0x4e3e6c2c,0xc8832a2c),LL(0xa3a45c34,0x64a8f43d),LL(0x21fb291b,0x52a05eef), LL(0xe4b68e38,0x10d3e24a),LL(0xee2d8a40,0x5289120a),LL(0x425b7da8,0x33836b98),LL(0xb00c64e1,0x5bd418f3), + LL(0xd511c3f8,0x10e92e5a),LL(0x18b62b7d,0x17f1301d),LL(0x97f0fcae,0xf710b02d),LL(0xbd394477,0x8b1030f6), LL(0xe5aab897,0x49040009),LL(0xce75b4d3,0xfdb23ac1),LL(0xf2b70e1b,0x7a43d904),LL(0xf94fa56f,0xdc09e995), + LL(0xd075dd65,0x9f314e85),LL(0xb9e26b8d,0xc0d39ce0),LL(0xfdc3b678,0xd3f6778e),LL(0xfc8497df,0xce6573e9), LL(0x67abaf7a,0x1f4535f8),LL(0xa47dd948,0x80706dab),LL(0xc059242b,0x670ae5bc),LL(0xcf5f9308,0x3a29bc73), + LL(0x8af2bf74,0xd2f8e297),LL(0x6c48bbec,0x98dbb443),LL(0xeb448447,0x211a3a96),LL(0x5af4a2c0,0x88ffb240), LL(0x9cdf9e20,0x1959dd34),LL(0xf34627e0,0xa4d0d839),LL(0xf00057cc,0xa5cecad3),LL(0xc5d97b18,0x22f32cce), + LL(0xcedc2c97,0x31a02241),LL(0x2b632641,0xf0f5489b),LL(0xcbfb588f,0xb09091dd),LL(0x5d9478e7,0x5ffd0f38), LL(0xdae35eda,0x13f141a1),LL(0x62f0b26c,0xd25563a6),LL(0x1b9dde18,0x80014b17),LL(0x7da49b36,0x9fcf8f81), + LL(0x93519f31,0x68aac84a),LL(0xc0b3660a,0xe14c35c1),LL(0x08195bc5,0x29f447dd),LL(0x10bba62f,0xc61fbbe6), LL(0x4ed8621a,0xc471624c),LL(0x0950a4c6,0x8005f67f),LL(0x93a2a33e,0xdfc3b3e5),LL(0x3c1d0e42,0x9c3c815e), + LL(0x93904766,0x1ed1aedb),LL(0x376fd0bc,0xcd5e0cf6),LL(0x90d03033,0xdd4c3378),LL(0xd85dca30,0xde39db01), LL(0xe6fab58b,0x49d01dc2),LL(0x6d449f51,0xd16d7940),LL(0xd20a95e6,0x3fb6cf8e),LL(0xbbeeccb1,0x0b10a596), + LL(0x0005acd3,0x06ceaa62),LL(0x47555053,0x09db6b28),LL(0x7e7d18d7,0x45d84a85),LL(0x229ad33e,0x33c28b02), LL(0x72e4f34c,0x1e5a6b52),LL(0xb64fa831,0x81eefbe6),LL(0x2aa209aa,0x4983b84a),LL(0x38d6a8d6,0x20777198), + LL(0x40096f25,0xbe993805),LL(0xec820131,0x900d4bdd),LL(0x2a993f9c,0x2170cfd3),LL(0x2dfe1007,0xa0e3d894), LL(0x0e7df109,0x600d0b5a),LL(0x47fde3dd,0xc904985a),LL(0xcb89816a,0x15597a84),LL(0xb9dfeb9e,0x8ac8b027), + LL(0x6450a179,0x5c9211bc),LL(0xf6333f95,0xd448a70a),LL(0x824e1d7f,0xe9c9a964),LL(0x15750ae4,0xc47d3f3c), LL(0xadcf9482,0x959f776b),LL(0xe741ceb3,0x00557ffe),LL(0x353d7898,0x8b69d3f6),LL(0x45cfa492,0x6b4d80d3), + LL(0x30c313da,0xc33ead78),LL(0x67eee139,0x86f96c3a),LL(0x08611b15,0x0c6675c7),LL(0x60620c27,0xf9ee695d), LL(0xd70c9258,0xb35d438c),LL(0xa5e7a4b1,0x1bc2b1e7),LL(0xef92f629,0x38d257f8),LL(0x79fd1eb0,0x090af73a), + LL(0xf59342e5,0x96ebd1f0),LL(0x4d053375,0xd4869362),LL(0x5fab54aa,0x7db504e2),LL(0x6e8e43fb,0x17c0801e), LL(0x136b1941,0xd3904d62),LL(0x28a43bd1,0x5932b753),LL(0xacb35572,0x551d895e),LL(0x1a6fdfbe,0x3f7a8a46), + LL(0xf7a2df83,0x9e3ea4fd),LL(0x64524d44,0x8b68b26b),LL(0x126aee21,0x74caeeab),LL(0x915d9e1c,0x590a00a5), LL(0x49b90eff,0x5ae2a6ab),LL(0x2df4fe51,0x74b4cb1e),LL(0x07fcb6ed,0x0306ed11),LL(0x502f5b30,0x564ebe2e), + LL(0x0c89e9ba,0x5a09a32e),LL(0x84f2466d,0x967f9dfb),LL(0x26a1a8a0,0x8b27416c),LL(0xc3158a18,0x1c21ef95), LL(0x2be23ae9,0xa7ee1ad8),LL(0x4daa1fcf,0x1f312d04),LL(0x44f9c7d7,0x6782f843),LL(0xe19e2681,0xb12ea2bf), + LL(0xd20578af,0xd2e43cbf),LL(0xbb5819b4,0x5566460a),LL(0xb658c03c,0x86f6c860),LL(0x62d42d82,0xc8d90309), LL(0xcb883ceb,0x7975a5f3),LL(0xdcded5a1,0xf6f5cf0f),LL(0xd3eb810b,0x25554fb1),LL(0xa596c7c6,0x3df7536b), + LL(0x83de31cd,0x255ca136),LL(0x7795eb96,0x7ac532ee),LL(0xb12bc235,0xfa9d83a9),LL(0x4b287a83,0x7df5d231), LL(0xb4f19fce,0xb2eaaaf6),LL(0x1a045f6a,0x7caabfb0),LL(0xb1449e6a,0x6d1b7f40),LL(0x12d22c82,0x24ae41da), + LL(0xc0a9d128,0xb0f7a0c3),LL(0xaed0d3bd,0x2adc34d3),LL(0x13e778e6,0x4ebf5778),LL(0xbb8476ba,0xd3b89bd0), LL(0x37413953,0xe09eb528),LL(0xd8ba3471,0x952b705c),LL(0x86a79c09,0xcaa81ade),LL(0x7e0e7b2a,0xc08eed3d), + LL(0xc80b4196,0x313fb103),LL(0x88c7ac81,0x25449ece),LL(0x24f16fa2,0xa6cb9ad3),LL(0x728a0c4b,0x4602c441), LL(0x5a000a9c,0xc3366f88),LL(0xef8778bd,0x146623e3),LL(0xf0773fdc,0x184ba0b1),LL(0xaecd8d63,0xe1d115ef), + LL(0xae165e6c,0x420d5473),LL(0x108d8575,0xefe137cd),LL(0x6fcff7d9,0x15378c57),LL(0xc1f5b601,0x49c48099), LL(0xf68473f1,0x1b0e3aee),LL(0xd320720d,0xa78329bb),LL(0xdca54cad,0x385e521b),LL(0x5c5b8d60,0x478ce06f), + LL(0xca7c4d4b,0x215d7d33),LL(0xa095366c,0x773f3ab3),LL(0x7afeeaa1,0x668e0d11),LL(0x4878d8a3,0x410cd205), LL(0xb2800646,0x2748fd98),LL(0x73a1dbef,0xf1183786),LL(0x7567ed3a,0xecc31bd2),LL(0x0ceb3873,0x775210df), + LL(0x9a8f42ff,0x2ea0c2bc),LL(0x4a1c029a,0x62974240),LL(0x9e4dd41b,0x5ee5f5f6),LL(0x2e110249,0x5b1bba80), LL(0x78da8016,0x5ac4eadb),LL(0x1809f79f,0xc2978780),LL(0xe3f8c0de,0x39d2dbce),LL(0x064d3ba9,0x7fb4b5fc), + LL(0xcd481ab4,0x038a736d),LL(0x396888eb,0xc4b15c5c),LL(0xd421f36f,0x459a896d),LL(0x47f54663,0x6058b90f), LL(0xafefebe5,0x0a5f6771),LL(0xa5b7c2ca,0x45c97ab2),LL(0x85139ca4,0x6d547af9),LL(0xa29d71cb,0x6db218de), + LL(0xfb956184,0xca6e0e7b),LL(0x1f660ac6,0x6682b6e3),LL(0x8b21bceb,0x3959e396),LL(0x632cf9c8,0x0459fd46), LL(0x74f296bb,0xc741250c),LL(0x990dbefa,0x29b9cacf),LL(0xfc35bdf7,0x5065d818),LL(0xa551dc04,0xeb8e9e1b), + LL(0x11befe9e,0x4f7d6f7d),LL(0x7478fdee,0xa88f1fce),LL(0xafa688e3,0x39b1e053),LL(0xe16847e0,0x562a0d6e), LL(0x34c26d14,0xf6044e4b),LL(0x7df61b90,0x5ebe8727),LL(0x6b5e5a39,0xa82a4de4),LL(0xfb9d296c,0xc916b0ba), + LL(0x2e1dc01e,0x029f1cb2),LL(0xfc429483,0x7699d92e),LL(0x154560f0,0xee0e425a),LL(0x787b6641,0x3f5cdfe6), LL(0xe5f6794e,0x726d87bb),LL(0x23aecad2,0x97d73588),LL(0x09ca351c,0x47f4f5b9),LL(0x57dc5e3b,0xd742ef4b), + LL(0x71411a86,0xccd2209d),LL(0x2223e6ce,0x94d57663),LL(0x66c7950c,0x228a7400),LL(0x54dd4e37,0x2d00ef6e), LL(0xd60f66be,0x9ea5daf3),LL(0x8aca724f,0x743c58a5),LL(0x44e38446,0x1f638406),LL(0x92ef6bb0,0x06314bb0), + LL(0xbb05589d,0xa7459e7f),LL(0xbfa913d7,0xc3a39592),LL(0xdf07b190,0x27dbabee),LL(0xd2ee62ff,0x1c8a2f33), LL(0xe31e8d63,0x60c8bcb8),LL(0xce410061,0xea1604d1),LL(0x3d7f7a98,0x55e8cfee),LL(0xebc64102,0x49efc316), + LL(0x41934914,0x04c86d8e),LL(0xab7facd4,0x26667c76),LL(0xa71a8916,0x319158db),LL(0x114fff43,0xb802942d), LL(0x8ce544f5,0x5efdef7b),LL(0x70e870c1,0xf531c718),LL(0x4d92071d,0x4b9a5f1b),LL(0x60cc48b6,0xbe767cf2), + LL(0x717381ea,0xbf389d37),LL(0x06bc5bcb,0xefd9e984),LL(0x67ff1110,0xcc8bc960),LL(0xb05612e4,0xd3414c0b), LL(0x927fad1a,0x084e5f05),LL(0x438e241f,0x999bd581),LL(0xfaa4fab8,0x0c917602),LL(0x95080657,0xda0520d2), + LL(0xce2f1af2,0x3160f928),LL(0x364f56e4,0x61186d84),LL(0x25fa68f0,0xe36a5fc0),LL(0x774c584b,0x9e6f66bd), LL(0x9ecb449a,0x2611bba4),LL(0xec5a0667,0xb1e0b341),LL(0x6cddb6c3,0x336de76d),LL(0x65a18f95,0x9668b5b3), + LL(0x7c3ec388,0x1ff6c81f),LL(0x40a8e2d0,0x53545b05),LL(0x14ae31d6,0x990a3cc5),LL(0x063a2995,0x769b4c26), LL(0x039e279f,0xcea238f4),LL(0x732fb38e,0xbfc5cfb9),LL(0x82fa05d8,0x99f5a33c),LL(0x69c42686,0x274dc741), + LL(0x76af2af7,0x193338ee),LL(0x6914ae66,0x0488c19f),LL(0x5fc58bf4,0x8d197f4e),LL(0xf0e61d4b,0x23de54df), LL(0x44a165e1,0x547dd09f),LL(0x1c2d5471,0x99878065),LL(0x39b311db,0xb2cabfad),LL(0x4b61a7eb,0x0aed63d9), + LL(0xbe8110ef,0x03713ac5),LL(0x50f989d3,0xaab1917d),LL(0x358fe8b0,0x0d80fe98),LL(0xa7a1f8e3,0xf6e874c5), LL(0xdeb42398,0x05650fd8),LL(0x1c44de73,0xbad3e085),LL(0x1c27f3c2,0x5369135f),LL(0xa7fc74ac,0x14bc04f8), + LL(0xb5dae291,0x18cbf622),LL(0x9356b88c,0xce290570),LL(0x39eba4e6,0x61bbb446),LL(0x980fee37,0xa79c291b), LL(0x19960cc6,0xd9f18006),LL(0x0ce98896,0xb0823f41),LL(0x1377df6f,0xf2bc612e),LL(0xc0b0e71c,0x1c07bded), + LL(0x37211fd5,0xffbf9841),LL(0x04a81410,0xbd704a6b),LL(0x6abf1c67,0x653cd2ee),LL(0x40681621,0x73ab8aa1), LL(0x271ada5c,0xc0bae4fd),LL(0xc46f189d,0xf567cae8),LL(0xa5535334,0xd183cb27),LL(0xe53c530c,0xcbf133f7), + LL(0xedd6a17e,0x32e53f78),LL(0xa2194fae,0x6ce6da9a),LL(0x58cd3586,0xa89b8054),LL(0x43b520a5,0x0037febb), LL(0x653e2c0b,0xbe67a2cf),LL(0x50301f52,0xc07a1ed1),LL(0xf5ea954f,0xf98b2b60),LL(0x7af6c499,0xfa6da95d), + LL(0xe3889cb1,0x44892091),LL(0xd45ae338,0x123fc555),LL(0x02a63128,0x2bc4a9ef),LL(0xa1dbb436,0xb72012c9), LL(0x556a0b46,0x8c75f7b3),LL(0x5b7608a4,0xe4c6f46c),LL(0x38fce20f,0xb36abf68),LL(0xbf6c21e1,0xb5a8e657), + LL(0xcecd5963,0x9ceaeefe),LL(0x6105fc29,0xe84f200d),LL(0x8591e346,0xc28981a9),LL(0x207001f1,0x0be4e931), LL(0x88616b18,0x31399d9d),LL(0x238c626e,0x3dac9f55),LL(0x65574274,0x0213fca7),LL(0x827aa853,0xa3560244), + LL(0x1ca99134,0x3ffbfeeb),LL(0xd07a2dac,0x0a4b56f6),LL(0x75417a6b,0x01795eca),LL(0x18a5fb22,0xe2a6dd9c), LL(0x8aca0cd8,0x13c97586),LL(0x7c323c52,0x3c2bb26e),LL(0xe38319bf,0xa3688cae),LL(0x4c88726a,0xe04b44b4), + LL(0xb0a88a4c,0xfed864d0),LL(0x6b1fa305,0x3e6cf152),LL(0x00e18e4a,0x8416b5f0),LL(0xfa4cd8f2,0x3a7603ca), LL(0x7ec750ef,0x8b04d5ff),LL(0x1c1ced05,0xe1867f9b),LL(0xdac2f8c1,0x87ffd0fb),LL(0x08b3cdca,0xc9ebe420), + LL(0x029c247e,0x5028a4fd),LL(0xa5ae3e76,0xd6225a43),LL(0xf970817e,0xfb3fa71c),LL(0x9ab4aef1,0x74216809), LL(0xa39c2f13,0xca81ee99),LL(0x86a97827,0xa8336e42),LL(0xb6489555,0xb75aff99),LL(0xe565435c,0x005b2338), + LL(0x524bdb34,0xbaee27bb),LL(0x82e47e71,0xbf22e1c9),LL(0x97491139,0x6ab1d712),LL(0x2cf4cbff,0xf420ce06), LL(0xb2b0c86a,0x9f96a2fc),LL(0xabeb7335,0x42765dd9),LL(0x45b7e044,0x7c223bb7),LL(0x1794e144,0xce3f9245), + LL(0xf3ee5c4e,0xa0a15b27),LL(0x54622215,0x1721c5bf),LL(0xada5a99c,0x0003fd16),LL(0xdbdccc7b,0x8e96dd56), LL(0x43f83748,0xd1abdc0b),LL(0x0f5ce74b,0x71cac4a6),LL(0xd46717ad,0xb8539aff),LL(0xb99661d9,0xeb65c589), + LL(0x85e89e17,0x66b4df3b),LL(0x6fc30672,0xc94fad66),LL(0x81d90df8,0xfd2aa80f),LL(0xbd8234c1,0xed73a163), LL(0x72eb45dd,0xe1a2c0b9),LL(0x902b5ca9,0x417e355d),LL(0xd5128812,0xa8f55aaa),LL(0x3c14cb79,0x4826b934), + LL(0x394d7a92,0xeae495e3),LL(0xb90faec0,0xcba23153),LL(0xf6d9d80c,0xd687c821),LL(0x8bff3082,0x951dada2), LL(0x701708ad,0x4e74f1f0),LL(0xdd2134f2,0xa6333cd1),LL(0x04665f7c,0xeea276cf),LL(0xae74c17a,0x527257fc), + LL(0xe51b53bd,0xeb3fd493),LL(0x69ec567e,0xae7807db),LL(0x5de15fd0,0xa50124aa),LL(0x1336f055,0x781bfe70), LL(0xd70a0dfc,0xb5729a74),LL(0x9f50c1a3,0x89da37f3),LL(0xcd8e6c1c,0x6e063297),LL(0x181d0271,0x17eb6ec1), + LL(0xe4e52a8c,0x36e7251a),LL(0x94705324,0x3acfe89b),LL(0xbc130c3b,0xaa94f06e),LL(0x309ae25a,0x01b5e44c), LL(0x0f61b55b,0xb72160f2),LL(0xe7bbc3f2,0xbef61953),LL(0x1bf138a1,0x96734d7a),LL(0x08c25135,0xdaa6186c), + LL(0xf34534a2,0xa3b031b2),LL(0xde46f581,0x44136619),LL(0x6d379647,0x4d0ed04b),LL(0xbb2b6735,0x4879d90d), LL(0x590156e0,0x8f7e031a),LL(0xf42bbc53,0x28428354),LL(0x5c5b791e,0x1cbed33c),LL(0x4cfc5562,0x17571645), + LL(0x7f76462f,0x8392350a),LL(0x0c216ccb,0x659ce7db),LL(0x047e35d5,0xe87a78b7),LL(0x6e0862d6,0x307c4861), LL(0xe70741bd,0xd444fb86),LL(0xfea1abe2,0x1138a886),LL(0x62b79c4f,0x4695397d),LL(0x003130ee,0x11aaf588), + LL(0x3a11712b,0x53bdda6d),LL(0x40fba3d2,0x30c384bd),LL(0x50ea38be,0x63039585),LL(0x3da9738a,0x7f110eca), LL(0x5b68c01e,0xbd701fc6),LL(0xcc48f38d,0xd23f3e8f),LL(0xf8b9bb65,0x6e2557eb),LL(0xa3dafc8f,0x29ceb4b6), + LL(0x4b6b7705,0x24659686),LL(0x4aca2b43,0x04143a8a),LL(0x975e06d8,0x3baed256),LL(0x3e834249,0x846fb3c9), LL(0x75f6770a,0x7890761e),LL(0x203c78fd,0x1187920e),LL(0x6b26281f,0x9b06c3a9),LL(0xa98215e1,0x3fe3dccd), + LL(0x4f33655e,0x099d7d7a),LL(0x662fb55a,0x1ba571e6),LL(0xcbc497f0,0x1a0d0147),LL(0x2862ff38,0xa94218ae), LL(0x5ce08047,0x1b0f919b),LL(0x2baf46cd,0x9a3ac37a),LL(0x8964cc68,0x76b7a03a),LL(0x4d3e1a40,0x5aed8c6d), + LL(0x7f034ff4,0x6607179c),LL(0x3781eac2,0xefb8fcd9),LL(0x7b022138,0xfa57f8a9),LL(0x56ab6039,0xc5bb9f1d), LL(0xe4d2ab7f,0xf9027e24),LL(0x77a9e364,0x3d67ad71),LL(0x1f7f487d,0xc579e70c),LL(0x2a7e6bd0,0x7fefc894), +}, +/* digit=35 base_pwr=2^245 */ +{ + LL(0xa45cfd31,0x2cb91662),LL(0x16f65cfe,0x09dd56d3),LL(0x14f3de51,0x983e005d),LL(0x210f64fc,0xb9dc05b0), LL(0x885eafe5,0x22790afd),LL(0x7444bdec,0xbd5213d3),LL(0x8987300a,0x289dca92),LL(0xb3960b76,0x69fb2ac2), + LL(0x9ae7540f,0xe3274886),LL(0x6131e921,0xd7386631),LL(0xf2a360c8,0x2e3d4fd8),LL(0x3d9d41e0,0xb20a59b6), LL(0x99082a34,0x72b67eae),LL(0x51819470,0xfad6aa7d),LL(0xa2d1d007,0x7c299b9a),LL(0x8100bed0,0xc1f841e0), + LL(0x43e90ebd,0x2c1f7d4c),LL(0x1fc72b07,0x58b78107),LL(0xaf94f402,0xda8503e1),LL(0x59f244b0,0xfbb724b7), LL(0xfcd8755a,0x2fcd2186),LL(0x868482b7,0x7407cdee),LL(0x349be3d5,0x4d62f578),LL(0xdcc6202c,0x4a012544), + LL(0x151ffc08,0xb8a414d2),LL(0x740d6b55,0xaa79acf0),LL(0xcdf472ab,0xeeab0104),LL(0xa3aa5f1d,0x5014a8c1), LL(0x33f13425,0x8c743405),LL(0x57eb54d4,0x2b776b49),LL(0x548a723b,0x3a0cc4ac),LL(0xc79fe63a,0x65aae6f3), + LL(0xee5e579c,0xe8b388f2),LL(0x991c03d4,0x31cc9373),LL(0x567bfa7c,0x53eed518),LL(0x67f985ed,0x267e282d), LL(0xb4763ea0,0xc435fd22),LL(0xe39b7703,0xead83837),LL(0x094ba5b4,0x806daad5),LL(0x45842672,0xa738a847), + LL(0x9984c4c2,0x99421b42),LL(0xd35c7bbd,0x1a3bce27),LL(0x3563b09e,0xe51ae6f6),LL(0x8d9c9fbf,0x8e67853b), LL(0x6b2100b5,0xca8784da),LL(0x98879bba,0xe89a24f7),LL(0xe286b039,0xe901b45c),LL(0xf50384bd,0x23dedbb8), + LL(0x5cbf7df2,0x4728cbdb),LL(0x6764329e,0xed274fdf),LL(0x642d199a,0xc2af1a07),LL(0x17a50e7e,0x5d665659), LL(0xfaa5eb82,0x7babf4bc),LL(0x99fe4026,0xd3bcfc67),LL(0x607d9f41,0xaa5d2648),LL(0x967efac1,0x7405c071), + LL(0xdfa782a4,0x79447ef9),LL(0x74cd9500,0x6dadc8e1),LL(0xdc38f762,0x0574020e),LL(0xe2ee7a14,0x17596d7e), LL(0x9e1f8adc,0x9ef75af7),LL(0xa4791da0,0x5ac5f216),LL(0x7b7b5d80,0x1583226b),LL(0xa21c26cc,0x59f3f053), + LL(0xf95e30d9,0xd80e7fdc),LL(0x0a3a3616,0xecf5041c),LL(0x03043fa6,0x50b93b8b),LL(0xae262ad6,0xa31a2aa4), LL(0xd63cd98d,0x1468b370),LL(0xdc07a367,0xfb89cc65),LL(0x4d47b59f,0x6cf1df6b),LL(0x1b6350fe,0xab451a99), + LL(0x8c124dff,0xeb74554d),LL(0x21be0be0,0x781a8c4d),LL(0xe3510068,0xfaacc154),LL(0xd6238265,0x16655d65), LL(0x0466134a,0xba46d27b),LL(0x3101e283,0x1a3f51b9),LL(0x096ec237,0xc08298a9),LL(0xc69cfb5b,0x46248627), + LL(0x81a0500f,0xf9e7a5a4),LL(0xbd2e03e7,0x92db27d5),LL(0x82899e3c,0x3dcce4f6),LL(0xf39a39c7,0x861f1797), LL(0x69dc8961,0x175b2430),LL(0xdc67953e,0x93d2a88e),LL(0x92d794d6,0xa40f3704),LL(0x3526eeaf,0x607019f0), + LL(0x22f37d65,0xf20e163b),LL(0x32cf180a,0x70fd00c8),LL(0x0b17244e,0xff1a97d2),LL(0xacedb33a,0x9a5a141b), LL(0xcc16bbb4,0xf03dd868),LL(0xa40e44e9,0x9b15372d),LL(0x15ac397f,0xd5ba6436),LL(0xc821f6b7,0xb1a886d4), + LL(0x4b7b4e21,0xbe3aacda),LL(0x66b56729,0xad9829fe),LL(0xd541cc1a,0x78965cef),LL(0x7d17631e,0x414bfea7), LL(0xc64dd621,0xf55835d9),LL(0xef644d93,0xa0ebf68b),LL(0xc8a40021,0x01d15340),LL(0x42b99aa0,0x00ae640d), + LL(0x6881e64f,0x92321ee2),LL(0x5267bdd3,0xaccab5c8),LL(0x5927647b,0x0c1f87ea),LL(0x162c6d86,0x0093d07e), LL(0x389c711d,0x49868df4),LL(0xc11e1f37,0xe68d69ae),LL(0xb4207246,0xa40e7aa8),LL(0xce832395,0x4ae8d126), + LL(0x86450cc0,0x5f9b128a),LL(0xc8ec07e6,0x88f76293),LL(0x179702b8,0x0762f293),LL(0x4910006d,0xb5696102), LL(0x35fe0505,0x3951291b),LL(0xce8d7056,0x70f75a5c),LL(0x2eb13369,0x4541beaf),LL(0xa643316c,0x7060a749), + LL(0x49c879a5,0xee16abd0),LL(0xa47ac42e,0x844874a7),LL(0x3c9c2326,0xee3f8a20),LL(0xdeaed33b,0x99a12054), LL(0x63b333ae,0x4814a15b),LL(0x9d923fa0,0xee9f28a5),LL(0x33b1b1ef,0x5b0cd250),LL(0x8346d928,0x3ccc39b9), + LL(0x002bec95,0xf5c1000e),LL(0xf63528c2,0x2ba2f18c),LL(0xcdcec15a,0x8102f6c8),LL(0xbb13d14a,0xab7effcd), LL(0xfcd3787c,0x183e0ba2),LL(0x2f4a7fc0,0xae70635e),LL(0x760bbc96,0x473ed37f),LL(0x8a8efb39,0xf0ea0acf), + LL(0x29b526a9,0x63cea36c),LL(0x9d03f3db,0xcdb31613),LL(0xd57cca8e,0xa3891096),LL(0xa14a8ffc,0x646418a9), LL(0x8075291f,0x10f8906b),LL(0x2c618bf6,0x8221d941),LL(0x8a5da4df,0x1dc1ae7a),LL(0x8a8cc8bc,0xb66b96e3), + LL(0xfe3551b4,0xe4da7e48),LL(0xad9b3827,0xe6891cc9),LL(0x6b37b99f,0xb266733f),LL(0xfd5d1219,0xfccce911), LL(0x7262b8cc,0xe5a47c4b),LL(0xe656af49,0x5d349caf),LL(0x7e04e48e,0x7a3a4a28),LL(0x80ea7d03,0x7c39a68e), + LL(0xbee0d434,0xf35d5e32),LL(0x0b536161,0x9651f3d9),LL(0x72cb370c,0x42634cc9),LL(0x406b3457,0xa7764026), LL(0x65d53d80,0xec7525bd),LL(0xadcc8b05,0xf44a1bca),LL(0xda112ddc,0x12ef8427),LL(0x20a0f78f,0x796a65b3), + LL(0x6bd5b0ab,0x12726e24),LL(0x8242fe07,0x9e441467),LL(0xde2bea52,0x4b52e276),LL(0x10422c2c,0x3a76b6b4), LL(0xb4e496b9,0x71f14945),LL(0xf36dce4f,0xd20f04b0),LL(0x2b310e90,0xa0e57d8d),LL(0x32ec8f38,0x59eb7737), + LL(0xaaf6619e,0x20a19834),LL(0x691a538e,0x633b94e8),LL(0x92cdf395,0xea1a8985),LL(0x4349b971,0xa3a01c57), LL(0x30822c59,0x0d65192a),LL(0xa522ae8c,0x93a5152d),LL(0x0e1aa4bc,0x5b09a7a3),LL(0x8d3b97a9,0xdd2588f3), + LL(0x5b62a3a5,0xafa1f62a),LL(0xa9ace9c5,0xbded10e6),LL(0xbf6e7fb2,0x9d03e061),LL(0x4b87177a,0x60c564a8), LL(0xc22c14c1,0x36be1308),LL(0x9425b6bb,0xeeda57e8),LL(0x36af38c2,0x5ddaae14),LL(0xecdc2911,0x1695464b), + LL(0x161e13e7,0x4b795e1d),LL(0x907e7069,0x0f9abc20),LL(0x54522fe7,0xfb3be618),LL(0x1938134e,0x9e2d0f37), LL(0xd292c6b0,0xb8dc7c36),LL(0xc1e77895,0xbafbf59c),LL(0x1b6c55f3,0x7d0132cd),LL(0xf609f087,0xefa02ed9), + LL(0x03519f9f,0x4bfe6aeb),LL(0xdab4c075,0x248e19a0),LL(0x69429f29,0x83ee803d),LL(0x8190ce56,0xdbbe31e2), LL(0x6b787a5d,0x3ba735d2),LL(0x1818070c,0xfa021185),LL(0xa3683cee,0x9b653000),LL(0xe9517ba2,0xfc3c7907), + LL(0x88d94f69,0x6521c92f),LL(0x7b52da8d,0x3950e9e8),LL(0x8ee64a88,0xadb81700),LL(0xf73994fe,0x8ccbfa3c), LL(0xb048e71e,0xb780ab12),LL(0xe2aeb607,0x52879e7b),LL(0x3237554f,0xef04b1ed),LL(0xe1d5a5ef,0xaeba6a96), + LL(0x266f7e93,0xedb58542),LL(0x5ea02852,0x9a1b8057),LL(0x5beb3fbd,0x1963c6f2),LL(0x1ad52473,0xf4183355), LL(0x6faed2f4,0xca772e9e),LL(0x3cf8fd1f,0x937eddd0),LL(0xc1d49dac,0xb3255376),LL(0xe670d3cc,0x549c2119), + LL(0x3b6cd199,0x10edbf39),LL(0x75644d6a,0xe9479223),LL(0xd6e8cc36,0x36cfba92),LL(0xfe00d80a,0xa37b1d91), LL(0xdeb5ef4a,0x3aadf918),LL(0xd3893df2,0x5bb2ca4d),LL(0x6569ab8b,0xa776000e),LL(0x1cf64905,0x4fb2308f), + LL(0x273731c2,0x04da4d09),LL(0x23c05518,0x1d4a61fe),LL(0x0d5206e5,0x201098a3),LL(0x06430c82,0xd9a7ad4e), LL(0x36f7f462,0x56df0d06),LL(0x44c81042,0x2c067f3d),LL(0xc453d28e,0x01193bc9),LL(0x45ce6e64,0xcdf5af5d), + LL(0x0f7d8d12,0x9992ce1a),LL(0x0e5e4542,0xa7c46a61),LL(0x057802ba,0x3fcc0647),LL(0xc7dccbe2,0xa78f73d8), LL(0xf138dc6d,0x67f37b94),LL(0x650a9002,0x89330885),LL(0x68aa24c7,0xf661caf2),LL(0xbf73c435,0x47732bcd), + LL(0x3b04622e,0xb9ba5f91),LL(0x477d420a,0x24265f73),LL(0x0d44cb89,0x5da6ddb0),LL(0x151fc40b,0x9f8cb8b6), LL(0x9b9f2613,0x81b6956b),LL(0xebb917df,0x37419206),LL(0x2bb7a709,0xdb9cfc16),LL(0xbacd3fb7,0x7a800aa3), + LL(0xd93f6e1a,0xf8ea9359),LL(0x3d41c118,0x729005d4),LL(0x7cb641af,0x4c293410),LL(0x895e8e78,0x6b2b4671), LL(0x5958fad3,0x2a1251d0),LL(0x78619fe4,0xb69bc2be),LL(0xd911d318,0xd74df34c),LL(0x15102704,0x5def8378), + LL(0x08268482,0xb19ea17a),LL(0x1c37e5d1,0x14591196),LL(0x7640df9c,0xe0e12d2e),LL(0x8c941274,0x8fd6bd4d), LL(0xdcd142b1,0xc3f9f120),LL(0x78dfe6b0,0x106c49ac),LL(0xcfd9b542,0x243c8e93),LL(0x0a2c5fe6,0x6758062d), + LL(0x15f2f638,0xee5a99e8),LL(0x13979ab6,0xb95b008d),LL(0xacfcca6a,0x7fd03105),LL(0xe4ced1b3,0x6af914a4), LL(0xa25f8627,0x8bef3d0f),LL(0xf9b2f931,0x21bae309),LL(0x2195a839,0xe85dee2b),LL(0xa3baeb25,0x46ad0ad9), + LL(0x022b62a9,0x6d8e40f8),LL(0x90b5cd33,0x4a6bbabf),LL(0xffa89bb2,0x53481e6b),LL(0x22003cc2,0xd674b3b3), LL(0x004a2aa6,0xc71a0a85),LL(0xb5404657,0x86df9697),LL(0xc74e80cc,0x407727f4),LL(0x950a7b08,0x39c13926), + LL(0xd74472a4,0x26bee75a),LL(0x2eb6f0d6,0xbf7c4ea0),LL(0x608bea48,0x689a5de5),LL(0x29d513f8,0x5b383892), LL(0xda457cf9,0x49fee2c2),LL(0x62d523d3,0x7fc0aee7),LL(0xb636a46e,0x5bf447de),LL(0x8194766e,0xda3efd98), + LL(0xd382756d,0xa77c3ad2),LL(0x0fa55315,0xc0eaa9de),LL(0xb1df90e3,0xe68d0a51),LL(0x01d1d8a7,0x0994e8c7), LL(0xa91bfed0,0x4f898bc3),LL(0xab6025df,0x1c2a3e46),LL(0x8b269271,0x37bd5c37),LL(0x8b97f0af,0x4e07f5ca), + LL(0x97923d14,0xe346b5aa),LL(0x9e0bd9c4,0xa8549f61),LL(0x40113a60,0x78e59d6b),LL(0xed8a0fc6,0xe3015fb2), LL(0x8b43288a,0xfc56a08f),LL(0xcae6c18a,0xcbdb8cae),LL(0x5f4423db,0xcb147c44),LL(0x10f361c1,0xa6aaa6c9), + LL(0x7caf702a,0x6be86c0c),LL(0x736f6dac,0x2188e83c),LL(0x59ba2af9,0x40b5ed25),LL(0xab8018c3,0x76211894), LL(0xf5b0b048,0x0c1c532f),LL(0xe3200521,0x7503aca9),LL(0xdfa7eb2d,0xb9325d85),LL(0x2edbb68f,0xe6c25a00), + LL(0x8c673c89,0xf9ff5867),LL(0x099c7bae,0x4925a046),LL(0xdbb1e1b6,0x0b3bf59a),LL(0x30ae084f,0xc7e9d9f2), LL(0x0fa1776f,0x70982396),LL(0x624c897b,0xb2e1b98f),LL(0x6c3534d5,0xa9a6407d),LL(0xa4dc3f63,0x5e22319b), + LL(0x2431dc01,0xc2f0bf3f),LL(0xc2cfb315,0x478f797d),LL(0x3b3ae0c5,0x6559f59c),LL(0xe18e74a8,0x7e95aa62), LL(0xd3fce912,0xf2a94006),LL(0xe1bd96ce,0x7f1b57a2),LL(0xa3d7b828,0x55028ad0),LL(0x4f09fe92,0xadae7e92), + LL(0x757b418f,0x2174c736),LL(0x3661d54d,0xd904ba43),LL(0x63218ecb,0x0281f912),LL(0xc50c8eb6,0x5fd03ba0), LL(0x896a5aea,0x29626906),LL(0xe55ee73f,0xab4d3f27),LL(0xedfd1991,0x3db1c43d),LL(0xa3f9c340,0x498cc31a), + LL(0x4fe75d33,0xa43bdec1),LL(0x66ae5d4f,0x5b067dfb),LL(0x464c8239,0x84581814),LL(0x503a52ea,0x2f10557f), LL(0xa10fbb90,0x21c4c180),LL(0xf79d5e02,0x33b191ee),LL(0xb499478e,0x6dee3881),LL(0xbfbd56fa,0x27dfef0b), + LL(0x28be2d62,0x671a3dd7),LL(0x050897ff,0x06f2f4c2),LL(0xb7c4587d,0xd92bdab6),LL(0xfd8d5160,0xd2253a16), LL(0xf1c098b1,0x64f6e4ae),LL(0x11ea7255,0x005a3939),LL(0xdab542e5,0x2ed4eb92),LL(0x50c5e874,0x26920bc1), + LL(0x5d0bc87c,0x93e8f58a),LL(0xb2b29b4b,0xaa4d313e),LL(0x01b2096f,0x3e175dec),LL(0x1cf31783,0x6c560972), LL(0x73b76f6b,0x9d41aca2),LL(0x5f1d4b12,0xa2454cf5),LL(0x65b35eea,0xa5615196),LL(0x70af4fde,0xf241e516), + LL(0x65061472,0x5255e91b),LL(0x5bdbb257,0x6ef98d2d),LL(0xc74c7b2c,0x0d1d1ab1),LL(0x2e9febde,0x9ffb9fdf), LL(0x6c50bf24,0x853f3b9f),LL(0x6fbd22bd,0x3d369594),LL(0xbcdad9a9,0x4d281126),LL(0xdc46ddc1,0x99eb62b6), + LL(0x4b10c402,0x5aa8c8b2),LL(0x473af61d,0x2e79f595),LL(0xce360f37,0x96153360),LL(0x66bc29dd,0x16dffe22), LL(0x1137f9c3,0x35536eb1),LL(0xe2a6a47a,0xd636ecad),LL(0xb499f840,0x83cdf214),LL(0xd247f18c,0x3642c77c), + LL(0x916ef527,0x4d906a2e),LL(0x293dc085,0xadeb94d0),LL(0x1491da3e,0x03a07801),LL(0x0b84d2eb,0x177dceae), LL(0x7b691e0c,0x61e5a3c1),LL(0xd172cea3,0x47d40bd7),LL(0x8ca76bce,0x7d0646ad),LL(0xc64d635f,0x90b030a9), + LL(0x97118df2,0x71eca8e7),LL(0x3ac9536b,0x2cd48f70),LL(0x89fb4d72,0x9ffd991d),LL(0xebf781fb,0xd49006bc), LL(0xd3e60da1,0x688af07f),LL(0x619063b7,0x5f74aa46),LL(0xa40d313f,0x44fcbeb3),LL(0x326faaa4,0x0ed5908b), + LL(0xf41ec05d,0xe836d537),LL(0x221b0c32,0x01eaf207),LL(0x72f8c904,0x1d6a0bb6),LL(0xdfd74641,0xa6ef58b2), LL(0x811bd6cb,0xbb855ceb),LL(0x05408eab,0x7b1c8b71),LL(0x4187fb7f,0xd24d709e),LL(0x8b30a9be,0x283d647d), + LL(0xf9f0d6e6,0x6d9d3793),LL(0xb1c06b19,0x02fc3ddb),LL(0x94d9abec,0x8ff86793),LL(0x24705873,0x1f20bba2), LL(0x0021b290,0x74eebc12),LL(0x35b6c157,0xd859521e),LL(0x431aea79,0x2201aa41),LL(0x90da1a75,0x79c1caaf), + LL(0x6e412a6a,0xcd6abab7),LL(0xb4c58b05,0x82399139),LL(0xa3b55538,0xdf416966),LL(0x679882d3,0x2b2d546f), LL(0xf9145657,0x17425cbc),LL(0xe1b8247e,0x3cc6735f),LL(0x57edd04c,0x13e50c56),LL(0x1b85b7cb,0xc8723137), + LL(0xdc0ab9d5,0x907b5b02),LL(0x4ab23b78,0x5617fb7f),LL(0xe8f449cd,0x7ae8ff03),LL(0x174e0e22,0x86d3ff17), LL(0xbf1e9f8d,0x22cb7f69),LL(0x0b0c62f0,0x12f0abbe),LL(0x537f658c,0xc8315981),LL(0xc467f2b4,0x43da2770), + LL(0x5b9e88ef,0x3ef9bb81),LL(0x3a8e51f2,0xb8526318),LL(0xf8d744ac,0x2e47cb7f),LL(0x510aaa7c,0x63d6dc16), LL(0xb40ccc41,0x54da7cdb),LL(0x402b2ad9,0xdecbe5fd),LL(0x34c8f225,0x14c6f15c),LL(0xc6559496,0x6d8b2342), + LL(0x66fea635,0xa4b72281),LL(0x22f248a8,0x55f5c27f),LL(0x0959cd01,0x3ced1483),LL(0xb53bdf42,0xcc6469db), LL(0x1e460645,0x2bb2346f),LL(0x9d7535e7,0x4d8573c6),LL(0x49cd2d68,0x988cddd5),LL(0xb9835538,0x785c4a70), + LL(0x1f6e396c,0xb08337b3),LL(0x49a782f3,0x6166b21e),LL(0x8ec9b974,0x1ac6198b),LL(0x0bb44d3d,0xee2e3446), LL(0x35039dd9,0xdb283740),LL(0x29f5c692,0x7c708f95),LL(0x98ddb466,0x8914cce0),LL(0xd446f3cf,0x8bb1b9f1), + LL(0xee0933a3,0xa9dea222),LL(0x4b26049e,0x2538bd43),LL(0xbdcafae2,0x18741aca),LL(0x16b0f4bb,0xe0f830f7), LL(0x902caefa,0x0479ec95),LL(0xdcda9e64,0x1f858937),LL(0x515c4089,0xe75b4f7b),LL(0x2eb91b51,0xb78afde4), + LL(0x18949935,0x1eebe3e9),LL(0xba092037,0xde8deaa9),LL(0x93609411,0xd43cf4ef),LL(0xc2d7b76e,0xe0fdb1e4), LL(0x4e34b4bd,0x1d3191a5),LL(0x9ccc4c26,0x106d92f1),LL(0x29a2a6d1,0x1a404ef6),LL(0xc598f481,0x3338bc9c), + LL(0xe3fcbf71,0x3945e39d),LL(0x9c89ab61,0x123b082c),LL(0x0f9f3c37,0xc7477f77),LL(0x7dbcc077,0x408c0c7a), LL(0x3654f98c,0x6c4d99f5),LL(0x05299a1a,0x276a007a),LL(0x23e2d7d0,0xabd4b8ea),LL(0x86017545,0xe05a5f3a), + LL(0xa11b03cd,0xde3b885c),LL(0x8df5d64e,0x46ef0755),LL(0xbf3f085d,0x112a49d6),LL(0x198ff32f,0xf6ebf441), LL(0x7feae481,0x581c00d8),LL(0xcfde5b2f,0xf2b43827),LL(0x9b7358f2,0x3ceb7f8f),LL(0x55fe7117,0x95761fbd), + LL(0xdc04773c,0x305836fa),LL(0xb3c4873c,0x66324504),LL(0x55b130de,0x5d878c1f),LL(0x8ad49a9b,0x96e9b28c), LL(0x76d70429,0xd1a707b8),LL(0xaa402e90,0xaff33f93),LL(0xedbfb28f,0x733d6256),LL(0xa75d2342,0x9e421a7c), + LL(0xc02e49c1,0xdf86b254),LL(0xb56d308a,0x6bb53867),LL(0x73f29561,0x771dde4b),LL(0x8bf28e5f,0x96eaf73e), LL(0x06fbb550,0x9b1ee6be),LL(0x97d4a4e8,0xe09fec77),LL(0xd5aa84fd,0x93bdcd60),LL(0xd457ab9c,0x3fa8d3a0), + LL(0xa0a2e52c,0x315b32b1),LL(0x3bbcb61d,0xe7e50b2d),LL(0x5e5c6e11,0x8a55cc0e),LL(0x961295ef,0xc2bfa998), LL(0x66e996d1,0x4a5ab3bb),LL(0x4886a421,0x22c42e4f),LL(0x4850e0a4,0xa0cdd364),LL(0xc64ed713,0x7682d38d), + LL(0xa2c539e4,0xe31575c2),LL(0xa756daf9,0x0bac5dcd),LL(0x91f55a12,0xe917cecf),LL(0xe96f6299,0x1e96433b), LL(0x3700d8fb,0xeec7c71c),LL(0xdc9b4444,0x9a1d2965),LL(0xcf74f19c,0x3d2c6970),LL(0xac5e0d6b,0x3b444c48), +}, +/* digit=36 base_pwr=2^252 */ +{ + LL(0x8ccb854c,0xe563cefd),LL(0x65b0c45a,0xf5452cdb),LL(0x9c37f743,0xb3c78769),LL(0x95d444ab,0x34e9d192), LL(0x52ff26b7,0x29347946),LL(0x9b94d642,0x70d6ecfa),LL(0xfdaffb8f,0x7d201858),LL(0x45dcdc71,0xc288719d), + LL(0x0728a2eb,0xc695469d),LL(0xc433d11c,0x7b46244e),LL(0xf106c08e,0x4a8b99ba),LL(0x63422083,0x7989794f), LL(0xd4fc5696,0x82218867),LL(0x3c79cdb8,0x6b021f28),LL(0xb26d5049,0x5ff7bbea),LL(0xa7261628,0xb78611ca), + LL(0x531313d7,0x5a75f961),LL(0x66dcdc9e,0x85a1f4db),LL(0x6460e991,0xae3026b9),LL(0x17ecf7cc,0x7d467bef), LL(0x05118708,0x8a0dbf67),LL(0xf3b2f1c9,0x54bfa368),LL(0xf2c0e4e0,0xa9fc9d5c),LL(0x5e93611b,0xa8c2ad11), + LL(0xaa1256bd,0x3ef1faf0),LL(0x9e4631de,0x0f224545),LL(0xde9c2676,0x69cb9800),LL(0x95782b24,0x26019816), LL(0xa66c0ccd,0x945c172c),LL(0xb440719a,0x6c25f635),LL(0xa49f681a,0x917d5dba),LL(0xb2dc5df4,0xc0cad047), + LL(0x5960ef1c,0xd45bcf4c),LL(0x8c6979d5,0xbabcb16d),LL(0xae9090d6,0x8e3be750),LL(0xac0eb728,0x9481d261), LL(0x0d6a7d46,0x46b436cd),LL(0x1f976501,0x6eb1a6a3),LL(0xdbe1064f,0x5984ffa2),LL(0xf809dc20,0xe6575fb1), + LL(0x4d974a81,0xf0426d80),LL(0x97a74be6,0x61304f0f),LL(0xa9486531,0x2346ff98),LL(0xf53d781a,0xa1242cca), LL(0x97355f15,0x482f03df),LL(0xbd6058cf,0xc607ed33),LL(0x68aefe28,0x03bc8cd4),LL(0x851307e4,0xa6e7de5a), + LL(0xc6af7d44,0x2c07df0f),LL(0xb15a9188,0x310b251f),LL(0xd3e15c2f,0xd42661ce),LL(0x1b4d8313,0x5198fd90), LL(0xda8368a1,0x7a6062cd),LL(0x5e9c2542,0x1a905d11),LL(0xdae37cee,0x1d752b70),LL(0x16bf84ca,0x3ed8c1a5), + LL(0xeecc2f22,0x5190fb0f),LL(0x698d8e60,0x3df210f3),LL(0xf5f3ce72,0xcce57d3a),LL(0x312b8fc6,0xb2fb6223), LL(0x71867c84,0x79947005),LL(0xbe139ebe,0x141cd92c),LL(0x5de7944e,0x415efc9e),LL(0x45821058,0xae9ee919), + LL(0x5bf363dc,0xd696e1d9),LL(0x8251449c,0x6a1bcfc0),LL(0xa5fa53e9,0xa1b82dff),LL(0xeef05378,0x6c56b5be), LL(0xc0e74dc3,0xaf9efe4c),LL(0xe5c1f1a0,0x3d9a7ae9),LL(0x2823c3e5,0x34b38577),LL(0x41fbabac,0x69f297dc), + LL(0xd74c5a65,0xf01aff98),LL(0x1951a915,0x97993104),LL(0x723096a6,0x8b211915),LL(0xa769ef1f,0xf85910c4), LL(0x8ddc0eb4,0x30cefb9e),LL(0xbb09607b,0xd5957eef),LL(0x2355b499,0x2e139c9c),LL(0xc1789013,0x5749531d), + LL(0x5475f2d2,0x1603ca64),LL(0x0a336508,0x57190e0e),LL(0xcea7d558,0x2203b703),LL(0xfb5503e3,0xf16eba4d), LL(0xb7344a98,0x62e2ce3d),LL(0x9a4efa7a,0xebf5b243),LL(0x1c914064,0x96212455),LL(0xbe5bbc07,0xd2c5e31c), + LL(0x06c30b28,0x2b5f2f77),LL(0xbc9823d3,0x0931841d),LL(0xadfa6fdb,0xd16fb08b),LL(0xd6fd441e,0x8892bae2), LL(0x2e576983,0x3fc64630),LL(0x07b05529,0x08c60fb6),LL(0x7afc1d4d,0x32b283b1),LL(0xa2f0e37f,0xc9c56965), + LL(0x4644e173,0x8e719178),LL(0xf88b43ff,0x4c2a11ec),LL(0x7d3ddbb3,0xb13644e6),LL(0xc3d8703c,0xd4746056), LL(0x55dca667,0x6611395f),LL(0x27c91d73,0x63596712),LL(0xea2ff489,0x4ca68a87),LL(0x337adc1d,0x2864a816), + LL(0x224d4f21,0x8aa830ae),LL(0x9f7845dc,0xda6c122e),LL(0xfb240793,0xb0c61ffc),LL(0xce8580e9,0xf4df6842), LL(0x0a990dc7,0x94372aaa),LL(0x5ce1aa24,0x42968cd3),LL(0x4df363a5,0x177c5ff0),LL(0x68c4546f,0xa8c3f737), + LL(0xbd21c524,0xc69750d5),LL(0x22a6c4ae,0xbf3b4857),LL(0xe2883a1d,0xcefcbb98),LL(0xae13f22b,0x6ffef743), LL(0x5defea01,0x6316ba60),LL(0x4ba63810,0x0a89e6a7),LL(0x15ab0e11,0x7f9af1de),LL(0x385911c9,0x6247ca15), + LL(0x32f9eaf5,0x6f7b1a6a),LL(0xacfc13dc,0x2c440f94),LL(0x66b18adf,0x2cf39bc5),LL(0x9f01533f,0xb9939fe8), LL(0x383a6450,0x031c4553),LL(0xf0102087,0x16d96ad3),LL(0x501f1360,0xcbd6fa95),LL(0x65f96c08,0x667d3ea0), + LL(0x68a1a680,0xa5a7cbfa),LL(0x42041db7,0xf131d779),LL(0xd85d377f,0xbefee3ac),LL(0x3b62dfa2,0x6d0ed6b7), LL(0x1baacfbd,0xef683f0f),LL(0xc976cebd,0xc586c4f2),LL(0x3b163339,0x3a4120dc),LL(0xc79e5c1f,0x9ac9b950), + LL(0xe294926a,0xaf1ff35f),LL(0xa2103168,0x2703bab8),LL(0x658a52bf,0xc645560a),LL(0xe466fd97,0x5ff3ccd9), LL(0x54867f14,0xe62fdc01),LL(0x9cdba39e,0x435ef950),LL(0x92828acc,0x2a7bbffd),LL(0xfe763981,0xe7538fdb), + LL(0x8bfe9773,0xedf45173),LL(0x471b8b9c,0xd187fa01),LL(0x78fa54df,0x34506c35),LL(0xc2767589,0x73cab9fd), LL(0x6726f305,0xf8f76c65),LL(0x8de332b2,0xea45012d),LL(0x87970e03,0xb746f40d),LL(0x1ba8fbd6,0xb2b2279a), + LL(0x21147dbc,0x79cdc610),LL(0x9939a3cc,0x738ef680),LL(0x8101bd8b,0xd66d6ce6),LL(0x09323caa,0x65612acb), LL(0x10310a29,0x6874b372),LL(0x5ee9ecfa,0x3cf30f0a),LL(0x8cfe1df8,0x4e1026ad),LL(0xd5989af5,0x75a153f7), + LL(0x8b8e0c49,0xc362ccee),LL(0xb533f3dd,0x8adfc0d2),LL(0xa109572e,0xe02ab03b),LL(0xfd3066ec,0x06efacdc), LL(0x3fa28700,0xf136a8ba),LL(0x308cceb9,0x48a1e987),LL(0x68253387,0xe8ee7c03),LL(0xc2b463c7,0x47febbe8), + LL(0x39a5c4d3,0x485195f2),LL(0xa26241ec,0xf42e000e),LL(0xcd05368d,0x08c64f90),LL(0x857cdbdb,0x46fbd381), LL(0x4c7e16ae,0xf711df8b),LL(0xe4edea42,0x95637e46),LL(0xad171465,0x2df8b206),LL(0x4bccedce,0xa31ea895), + LL(0x50743bb6,0x28dbcb77),LL(0xf9cf84b9,0x13d12f8e),LL(0xc8f7d408,0x39e3d3af),LL(0xeba591d4,0x5824117f), LL(0x1bead2d6,0xd8ef7c9a),LL(0xcaf53dd7,0x9003a559),LL(0x174cb9a9,0x33b2365c),LL(0xadb33afb,0x1149d080), + LL(0xaea9bd3c,0x55231d00),LL(0xfdf3f015,0x07e107c9),LL(0xec9d8fce,0xf535947d),LL(0xbba349a7,0x8b64ed8a), LL(0x049301df,0xdd5881fd),LL(0xe6490fd0,0xefac9c43),LL(0x73740a78,0xd9902852),LL(0x942c326c,0x6eef3724), + LL(0x5cfb3c8c,0x5671a6e9),LL(0x0ea29721,0x040aabd2),LL(0xeac8da18,0x24e92ca6),LL(0xa31170c3,0xc34d3d79), LL(0xb061e416,0xf81dd15f),LL(0x85f80af0,0xff7be70e),LL(0xade45cd4,0xa9faba4b),LL(0x505fddd4,0x42a6ab05), + LL(0x0a793534,0x17d5821d),LL(0xce0ade43,0x9e094e54),LL(0xc42cb4d2,0xa127fb6d),LL(0xdb12dc99,0x43865428), LL(0x59e3bfc1,0xb6b1b347),LL(0x1ec5b810,0x0b0076a9),LL(0xa6864982,0xbf2dd17a),LL(0x9d523c87,0x0c45947f), + LL(0x4c5dd59e,0x9f53372f),LL(0xca5ce09f,0x3d0ceaea),LL(0x7c0337fb,0xf3ff88e8),LL(0xfaa022c7,0xb4fa4593), LL(0xd65ea54d,0x575240a7),LL(0xadb92fb0,0xa4ec0a39),LL(0x79429eb1,0xc20e737c),LL(0x69addec4,0xcea931d1), + LL(0x3e09f46a,0x7a29011f),LL(0x0e578a5b,0x9c36865e),LL(0x71d805f4,0x8746ea51),LL(0xe12d3024,0xf024de85), LL(0x15a7f6be,0xc397b46c),LL(0x1b0580d7,0x612db6fb),LL(0xf736d087,0xe5342f76),LL(0x8c1e844c,0x65276853), + LL(0x113841a5,0xedf48adc),LL(0xe5c482f0,0xc21b67e1),LL(0x684a540b,0xe43b0138),LL(0xa5d4b266,0xc4f2782b), LL(0x397f3664,0x184e240c),LL(0x0d8788f8,0x968e89e7),LL(0x377e18bf,0xec3eba1a),LL(0x36002652,0x4d03cbbc), + LL(0x1005a953,0x21eedee7),LL(0x75ba987e,0xc178ddf1),LL(0xc4ba43f6,0xd0d577f6),LL(0x4d6f24fd,0x9486f46c), LL(0xc5421895,0x3d33c574),LL(0x842e52ab,0x5be6cb4c),LL(0x9debc9ff,0x3809690d),LL(0xa84a5b6f,0xe4b1c692), + LL(0xd7e18b57,0x58b966ad),LL(0x77c94715,0x7ff0b61e),LL(0xf06add82,0x0e295883),LL(0x65c7f5a4,0x7c3c04fd), LL(0x60223be5,0x4ea92660),LL(0x89262bfd,0x5d843a57),LL(0x36da11c0,0x35bf4aef),LL(0xaf859eb1,0xa6692f14), + LL(0xa12fdf41,0xca1fc13b),LL(0x8224f5d2,0xd798c04b),LL(0x1dd5872b,0x22f4594e),LL(0x1bddfda8,0xdee12df5), LL(0xed83420a,0x96473ff0),LL(0x8daa27f4,0xf41cf1c7),LL(0xaecefd8a,0x2772cd56),LL(0x4902b47f,0xd5ddaf18), + LL(0xc0798101,0xff77551f),LL(0x26946bda,0x8baa01d6),LL(0x100525f2,0xd0087e47),LL(0x4c0de308,0x521d6254), LL(0x9bbce049,0x4a0f45eb),LL(0xa6c6b96e,0x5ee33cbe),LL(0xd6a22345,0x9a6af4b7),LL(0x38b1b406,0x0d0d35e7), + LL(0xbbedc29b,0x9e71252d),LL(0xcad1455e,0x3aa70bb6),LL(0x42a1778c,0xa406fb7a),LL(0xf0897613,0xd94f9646), LL(0xf57f66c8,0x5256370f),LL(0x4432f679,0x95891e35),LL(0xbcb6b3d3,0x75d6423a),LL(0x2367483f,0x79d9ea01), + LL(0x9efb0473,0x1e36ccc6),LL(0xdfdc0cec,0x3e64b034),LL(0x028bb238,0x13bfd326),LL(0x209edd95,0x171e9d96), LL(0x07b22424,0xda258380),LL(0xd41b8023,0xe31e97f6),LL(0x7269cecd,0xdd4ed390),LL(0x12d5cec6,0x810fb3c8), + LL(0xbabeec88,0x2f956519),LL(0x455baf52,0xb0350c52),LL(0x48d5abf1,0xa7fb548a),LL(0xca5e2d9f,0xcb81bd0c), LL(0xa6d17b19,0xda5ecd39),LL(0x508e5149,0xd2588bab),LL(0xc3e23cfd,0x1a30cff5),LL(0xf89f8712,0x2dd398b4), + LL(0x5b304005,0x2a911800),LL(0xd9dece69,0xd091be7a),LL(0xf6cabc89,0x147e93da),LL(0x44935824,0x7eac2018), LL(0x32f5de9b,0xd4aaf2be),LL(0xd9396cd1,0xe302bc41),LL(0x2c069d1a,0x3c2794cf),LL(0xa9d433ae,0xf9197eaa), + LL(0x4445e8c2,0x98f822ef),LL(0x1383ece8,0xc578360e),LL(0x01869457,0xa5372c12),LL(0x787d6644,0x1c6ed00d), LL(0x86531814,0x77fb08cd),LL(0x63a70db8,0xeff6ee26),LL(0x80976119,0x980be153),LL(0xd69d60c5,0x534a09bd), + LL(0x759dba20,0x71a58b0c),LL(0x679c0b40,0x34d5f06c),LL(0xceed2f9f,0xdc0e7e5f),LL(0x48808edb,0xaaa5996e), LL(0xbcdd88e5,0x8ca96ff0),LL(0xc37c2b46,0x91b02d67),LL(0x95526319,0xbe4f3948),LL(0x89be56d1,0x4315c7f2), + LL(0xdc85bba1,0xa312a3c0),LL(0x431ca797,0x3328fa8e),LL(0x68fd219a,0x5438bf1c),LL(0x85837d74,0x98812c6f), LL(0xf8c96d49,0xe88c4913),LL(0xc2442aca,0xcc62e79c),LL(0x046655f1,0x4ef3c7d4),LL(0xdadab1ea,0x04a362ed), + LL(0x30a199cf,0x975e2f3c),LL(0x014a165a,0x831e04a9),LL(0xaa126719,0x1e1d3c53),LL(0x1bf707a0,0xc42661e0), LL(0xaa2da264,0x295b0738),LL(0x65d4ba34,0xb45f5ed8),LL(0x9f3938fa,0x27fb5a12),LL(0xcb26f86c,0x25fba614), + LL(0xcf3c1c4d,0x6bd41981),LL(0xa0dedafd,0xd6f9239c),LL(0xae55b97f,0x46882526),LL(0x81b628d4,0x8e6fa994), LL(0xdc0aa158,0xbdb314dd),LL(0x12ba2a17,0x35343678),LL(0x32e2e431,0xac018e83),LL(0xe65cc63e,0x43a64e35), + LL(0x0b6603ea,0x887f3a2a),LL(0x76b2673f,0xe015426c),LL(0x27edfe8a,0x59dc5530),LL(0x68d9ebf3,0xea9eacf3), LL(0xcc3e07ca,0x40301c8e),LL(0x0f57a2e6,0xd8cb9b5b),LL(0x60ec5864,0x542e6b52),LL(0x17f6affe,0xb8791dd6), + LL(0x798d9993,0x6735bd1c),LL(0xd5da393c,0x006d8b25),LL(0x49e6d0d2,0x1d675bdb),LL(0x8607f99e,0x331d9a10), LL(0x9dc4cd07,0x4ff8ab74),LL(0x64ea3192,0xa87d4ae1),LL(0x41196b5b,0xdde0d92e),LL(0xb2e010eb,0xa15ad47b), + LL(0xa5522a75,0x23e6003f),LL(0x84afa749,0xc6ef3f1b),LL(0x146d10a3,0x9a723f75),LL(0x119106b0,0x5fa99480), LL(0xc0013dba,0x01d500db),LL(0x548edbe0,0x10b30ada),LL(0xb04ffc6b,0xb2eb046e),LL(0x64f25ee2,0xa57088f3), + LL(0x83a068a3,0xc0c919c3),LL(0xfbde282f,0x8139559d),LL(0x9fec9a99,0x4e2b5d13),LL(0xfbefa7e6,0x53bad712), LL(0x2860bd4f,0xa6befe0d),LL(0x0011bd15,0x6ea0ae15),LL(0x2bce3779,0xc1ef3463),LL(0x5d742dbb,0xc09ecb30), + LL(0xf73db19d,0x29526afd),LL(0x6a029a40,0x7c02c905),LL(0xde5a48ba,0xa778460f),LL(0x77c105f6,0xda05993e), LL(0xc9ddece9,0xb6d599f9),LL(0x0cfc33ca,0x9f9df668),LL(0xa0aa67a8,0xdcd8ef4f),LL(0xc3f4d178,0x31277019), + LL(0x53e86ae1,0x98e05abf),LL(0x3850830d,0xc1dc4d90),LL(0xe06bc33c,0xbd7fd806),LL(0xacf1286f,0x1ac330d6), LL(0xe1588c1e,0x28ce2303),LL(0x1b7e9c19,0xdc25e54b),LL(0x4b7149f2,0x11e51e49),LL(0xb5c7fa25,0x551b8391), + LL(0x1bf69873,0xa2fc251c),LL(0x2aec1574,0x099b7b53),LL(0x7c53296a,0x9ff98156),LL(0xa2dc60de,0xaf3f8d08), LL(0x59b72d6c,0x18dd295c),LL(0xe75f44fc,0x165c9063),LL(0x427a0c55,0x9046ee7c),LL(0xc79ffdb3,0x317ea24d), + LL(0x0ef0447d,0x6835a315),LL(0x8068e7c7,0xb2b9c786),LL(0x0e646af5,0xe6352714),LL(0x442baaa0,0xc5554a91), LL(0x6d0ba1ea,0x671febc5),LL(0x0cf649ed,0x44f9ef7b),LL(0x0c1dac6b,0x4aa0cd61),LL(0x6e393e68,0x865f3c23), + LL(0xa71dee29,0xf6886bcd),LL(0xda44ffae,0x934b0455),LL(0x016d6039,0xda7621c4),LL(0x3ad35493,0xf36c41bf), LL(0xe5f6ab8d,0x9063135e),LL(0x47bdc0a8,0xb0e8eaba),LL(0x4c737cf3,0x62530616),LL(0x64f6b6cb,0x8046423e), + LL(0x7958e3dc,0x11e50ad7),LL(0xb57234ab,0x4dab4e16),LL(0xe0916210,0x6ccfe2c6),LL(0x80f49d40,0x4d5dbc3b), LL(0xef1b2b1b,0x2b8ff368),LL(0x752fea2a,0xf2afb326),LL(0x0246e36b,0xffa48ea7),LL(0x589b7444,0x3a4bae9b), + LL(0x6ff3efcf,0x80ff984a),LL(0x56b77b47,0x7af53f30),LL(0x9320cae6,0x1f1c33b0),LL(0x26fc4ad4,0xce1f1c48), LL(0xad350ee5,0x9cac662b),LL(0xe27a7dbd,0xf4c72fff),LL(0x703184e5,0xd766f986),LL(0x7c5b241e,0x36d3efd5), + LL(0xf7ff5804,0xd4d6e358),LL(0x8f5e0bf6,0xa832b302),LL(0x453d9a22,0x4b3d73f7),LL(0xdf938705,0xb4dae072), LL(0x92401620,0x6bff7b2e),LL(0x9bfa61cd,0x96b8494e),LL(0xb74dc1e5,0x4bcda341),LL(0xc19c393d,0x383fe3d2), + LL(0x077e8821,0xa375fb70),LL(0xc17eb9bc,0xea35e04b),LL(0x7c4dd076,0x941d21ba),LL(0x3d0c3d8a,0x916c0a59), LL(0x15b2cf47,0x2c1304e3),LL(0xd0c955c0,0x9233ebf3),LL(0x2b2fc935,0x77acdd07),LL(0xc04276bf,0xd71b6a7a), + LL(0xd2ee8d68,0x789ea49b),LL(0x0a84a920,0x89552b46),LL(0x1a4ea456,0xe629d5de),LL(0x22ddd405,0xddfefe87), LL(0x1cdb9e7b,0x3d56f697),LL(0xa8bf715b,0x95a861b0),LL(0x7896c993,0xb01248d6),LL(0x01a3085c,0x3c4e3d98), + LL(0x9085b360,0x674939e1),LL(0xb589a287,0xae67dea9),LL(0x2bfdcfc9,0xc563856f),LL(0x313b685d,0x62fa9a80), LL(0x7ad501d9,0x36ff33d9),LL(0x730ab349,0xf8bab4dd),LL(0xc46ba69d,0x18fd59f3),LL(0xe65278e9,0x81e08665), + LL(0xeb8a1e84,0x5a5e803f),LL(0x47243604,0x5b4eef35),LL(0x393d6cdf,0x0ee71ee0),LL(0xc3a9c6db,0xde4d9dea), LL(0x64466b53,0x0c14c376),LL(0x89e3b45e,0xc2ce9642),LL(0x54a2de21,0x6aa8012f),LL(0x29b6bc2c,0x519759c1), + LL(0xe4667322,0x17768527),LL(0xac83b2ea,0x09fdfe4d),LL(0x04a0d5f5,0xd422125d),LL(0x2b86b310,0x02e8ff96), LL(0xd7ee97af,0xf033628d),LL(0x7d72e0e6,0x778a846c),LL(0x882f63d6,0x06fde613),LL(0x8d434f14,0x9e258b0d), + LL(0xccdcd600,0x5cdda529),LL(0x033c4535,0x37038b38),LL(0x391c1d7d,0xd6a1d639),LL(0x31d4ce6b,0x4f6489e4), LL(0x5754e08c,0xd1b82f17),LL(0x75db7bd6,0x7df268ee),LL(0xad14dcfa,0x1e4a1202),LL(0xccfb9b77,0x7ab92ce2), + LL(0x23aef997,0x61388e03),LL(0x06440ce3,0x9981f5bf),LL(0xc67d0edd,0x8d7631da),LL(0xc0a93516,0xc6ea593f), LL(0xee841b38,0x064a06e0),LL(0x521ce83f,0x0d1d4f57),LL(0x70df2613,0xf7a0e0c3),LL(0x84c071ab,0x1506cccb), + LL(0x328565e9,0x42a138ec),LL(0xb8130d16,0xe16b4578),LL(0x45ba251a,0x0628ff22),LL(0x210e22e8,0x016a84ca), LL(0x94592d43,0x8ba14bb4),LL(0x785274a5,0xffee4308),LL(0x01354b75,0x01fc21ab),LL(0x7e424674,0xc37ce45f), + LL(0xa7fe2735,0x71e153af),LL(0xc307721f,0x000fcee9),LL(0x805b56e3,0x3b189004),LL(0x7f504d9b,0x2f1435aa), LL(0xa083bd72,0xd9aa1eba),LL(0x720ccf3d,0xf1145036),LL(0x4084fa32,0x95b29e27),LL(0x01f94f0c,0x8862d213), + LL(0x1510a406,0x23fc5ddf),LL(0xc9f0e98d,0x475a78f4),LL(0xe72843a0,0xb6d681c4),LL(0x4a00c5a6,0xa90af2a4), LL(0xa34f4412,0x95fc6d45),LL(0xe7f5d703,0x60f9c0e2),LL(0xad110925,0x2bc0642b),LL(0xbe24a4d5,0x79abfc10), +} +}; +#endif /* _DISABLE_ECP_SM2_HARDCODED_BP_TBL_ */ +#endif /* _IPP_DATA */ + + +IPP_OWN_DEFN (const cpPrecompAP*, gfpec_precom_sm2_fun, (void)) +{ + static cpPrecompAP t = { + /* w */ 7, + /* select function */ p256r1_select_ap_w7, + /* precomputed data */ (BNU_CHUNK_T*)precomputed_ec_sm2 + }; + return &t; +}; diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3ca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3ca.c new file mode 100644 index 0000000..d62e4eb --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3ca.c @@ -0,0 +1,56 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SM3 +// +// Contents: +// cpFinalizeSM3() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsm3stuff.h" + +IPP_OWN_DEFN (void, cpFinalizeSM3, (DigestSHA1 pHash, const Ipp8u* inpBuffer, int inpLen, Ipp64u processedMsgLen)) +{ + /* local buffer and it length */ + Ipp8u buffer[MBS_SM3*2]; + int bufferLen = inpLen < (MBS_SM3-(int)MLR_SM3)? MBS_SM3 : MBS_SM3*2; + + /* copy rest of message into internal buffer */ + CopyBlock(inpBuffer, buffer, inpLen); + + /* padd message */ + buffer[inpLen++] = 0x80; + PadBlock(0, buffer+inpLen, (cpSize)(bufferLen-inpLen-(int)MLR_SM3)); + + /* put processed message length in bits */ + processedMsgLen = ENDIANNESS64(processedMsgLen<<3); + ((Ipp64u*)(buffer+bufferLen))[-1] = processedMsgLen; + + /* copmplete hash computation */ + UpdateSM3(pHash, buffer, bufferLen, sm3_cnt); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3duplicate.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3duplicate.c new file mode 100644 index 0000000..e272622 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3duplicate.c @@ -0,0 +1,66 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SM3 +// +// Contents: +// ippsSM3Duplicate() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsSM3Duplicate +// +// Purpose: Clone SM3 state. +// +// Returns: Reason: +// ippStsNullPtrErr pSrcState == NULL +// pDstState == NULL +// ippStsContextMatchErr pSrcState->idCtx != idCtxSM3 +// pDstState->idCtx != idCtxSM3 +// ippStsNoErr no errors +// +// Parameters: +// pSrcState pointer to the source SM3 state +// pDstState pointer to the target SM3 state +// +// Note: +// pDstState may to be uninitialized by ippsSM3Init() +// +*F*/ +IPPFUN(IppStatus, ippsSM3Duplicate,(const IppsSM3State* pSrcState, IppsSM3State* pDstState)) +{ + /* test state pointers */ + IPP_BAD_PTR2_RET(pSrcState, pDstState); + IPP_BADARG_RET(!HASH_VALID_ID(pSrcState, idCtxSM3), ippStsContextMatchErr); + + /* copy state */ + CopyBlock(pSrcState, pDstState, sizeof(IppsSM3State)); + HASH_SET_ID(pDstState, idCtxSM3); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3final.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3final.c new file mode 100644 index 0000000..904c6b1 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3final.c @@ -0,0 +1,78 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SM3 +// +// Contents: +// ippsSM3Final() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsm3stuff.h" + +/*F* +// Name: ippsSM3Final +// +// Purpose: Stop message digesting and return digest. +// +// Returns: Reason: +// ippStsNullPtrErr pState == NULL +// pMD == NULL +// ippStsContextMatchErr pState->idCtx != idCtxSM3 +// ippStsNoErr no errors +// +// Parameters: +// pMD address of the output digest +// pState pointer to the SM3 state +// +*F*/ +IPPFUN(IppStatus, ippsSM3Final,(Ipp8u* pMD, IppsSM3State* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxSM3), ippStsContextMatchErr); + + /* test digest pointer */ + IPP_BAD_PTR1_RET(pMD); + + cpFinalizeSM3(HASH_VALUE(pState), HASH_BUFF(pState), HASH_BUFFIDX(pState), HASH_LENLO(pState)); + /* convert hash into big endian */ + ((Ipp32u*)pMD)[0] = ENDIANNESS32(HASH_VALUE(pState)[0]); + ((Ipp32u*)pMD)[1] = ENDIANNESS32(HASH_VALUE(pState)[1]); + ((Ipp32u*)pMD)[2] = ENDIANNESS32(HASH_VALUE(pState)[2]); + ((Ipp32u*)pMD)[3] = ENDIANNESS32(HASH_VALUE(pState)[3]); + ((Ipp32u*)pMD)[4] = ENDIANNESS32(HASH_VALUE(pState)[4]); + ((Ipp32u*)pMD)[5] = ENDIANNESS32(HASH_VALUE(pState)[5]); + ((Ipp32u*)pMD)[6] = ENDIANNESS32(HASH_VALUE(pState)[6]); + ((Ipp32u*)pMD)[7] = ENDIANNESS32(HASH_VALUE(pState)[7]); + + /* re-init hash value */ + HASH_BUFFIDX(pState) = 0; + HASH_LENLO(pState) = 0; + sm3_hashInit(HASH_VALUE(pState)); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3getsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3getsize.c new file mode 100644 index 0000000..0c8ffbf --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3getsize.c @@ -0,0 +1,56 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SM3 +// +// Contents: +// ippsSM3GetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsSM3GetSize +// +// Purpose: Returns size (bytes) of IppsSM3State state. +// +// Returns: Reason: +// ippStsNullPtrErr pSize == NULL +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to state size +// +*F*/ +IPPFUN(IppStatus, ippsSM3GetSize,(int* pSize)) +{ + /* test pointer */ + IPP_BAD_PTR1_RET(pSize); + + *pSize = sizeof(IppsSM3State); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3gettag.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3gettag.c new file mode 100644 index 0000000..be45f83 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3gettag.c @@ -0,0 +1,81 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SM3 +// +// Contents: +// ippsSM3GetTag() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsm3stuff.h" + +/*F* +// Name: ippsSM3GetTag +// +// Purpose: Compute digest based on current state. +// Note, that futher digest update is possible +// +// Returns: Reason: +// ippStsNullPtrErr pTag == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxSM3 +// ippStsLengthErr max_SHA_digestLen < tagLen <1 +// ippStsNoErr no errors +// +// Parameters: +// pTag address of the output digest +// tagLen length of digest +// pState pointer to the SHS state +// +*F*/ +IPPFUN(IppStatus, ippsSM3GetTag,(Ipp8u* pTag, Ipp32u tagLen, const IppsSM3State* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxSM3), ippStsContextMatchErr); + + /* test digest pointer */ + IPP_BAD_PTR1_RET(pTag); + IPP_BADARG_RET((tagLen<1)||(sizeof(DigestSM3)idCtx != idCtxSM3 +// ippStsNoErr no errors +// +// Parameters: +// pState pointer hash state +// pBuffer pointer to the destination buffer +// +*F*/ +IPPFUN(IppStatus, ippsSM3Pack,(const IppsSM3State* pState, Ipp8u* pBuffer)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pState, pBuffer); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxSM3), ippStsContextMatchErr); + + CopyBlock(pState, pBuffer, sizeof(IppsSM3State)); + IppsSM3State* pCopy = (IppsSM3State*)pBuffer; + HASH_RESET_ID(pCopy, idCtxSM3); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3stuff.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3stuff.h new file mode 100644 index 0000000..937687b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3stuff.h @@ -0,0 +1,96 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SM3 +// +// Contents: +// SM3 methods and constants +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +#if !defined _PCP_SM3_STUFF_H +#define _PCP_SM3_STUFF_H + +/* SM3 constants */ +static const Ipp32u sm3_iv[] = { + 0x7380166F, 0x4914B2B9, 0x172442D7, 0xDA8A0600, + 0xA96F30BC, 0x163138AA, 0xE38DEE4D, 0xB0FB0E4E}; + +static __ALIGN16 const Ipp32u sm3_cnt[] = { + 0x79CC4519,0xF3988A32,0xE7311465,0xCE6228CB,0x9CC45197,0x3988A32F,0x7311465E,0xE6228CBC, + 0xCC451979,0x988A32F3,0x311465E7,0x6228CBCE,0xC451979C,0x88A32F39,0x11465E73,0x228CBCE6, + 0x9D8A7A87,0x3B14F50F,0x7629EA1E,0xEC53D43C,0xD8A7A879,0xB14F50F3,0x629EA1E7,0xC53D43CE, + 0x8A7A879D,0x14F50F3B,0x29EA1E76,0x53D43CEC,0xA7A879D8,0x4F50F3B1,0x9EA1E762,0x3D43CEC5, + 0x7A879D8A,0xF50F3B14,0xEA1E7629,0xD43CEC53,0xA879D8A7,0x50F3B14F,0xA1E7629E,0x43CEC53D, + 0x879D8A7A,0x0F3B14F5,0x1E7629EA,0x3CEC53D4,0x79D8A7A8,0xF3B14F50,0xE7629EA1,0xCEC53D43, + 0x9D8A7A87,0x3B14F50F,0x7629EA1E,0xEC53D43C,0xD8A7A879,0xB14F50F3,0x629EA1E7,0xC53D43CE, + 0x8A7A879D,0x14F50F3B,0x29EA1E76,0x53D43CEC,0xA7A879D8,0x4F50F3B1,0x9EA1E762,0x3D43CEC5 +}; + +IPP_OWN_DEFN (static void, sm3_hashInit, (void* pHash)) +{ + /* setup initial digest */ + ((Ipp32u*)pHash)[0] = sm3_iv[0]; + ((Ipp32u*)pHash)[1] = sm3_iv[1]; + ((Ipp32u*)pHash)[2] = sm3_iv[2]; + ((Ipp32u*)pHash)[3] = sm3_iv[3]; + ((Ipp32u*)pHash)[4] = sm3_iv[4]; + ((Ipp32u*)pHash)[5] = sm3_iv[5]; + ((Ipp32u*)pHash)[6] = sm3_iv[6]; + ((Ipp32u*)pHash)[7] = sm3_iv[7]; +} + +IPP_OWN_DEFN (static void, sm3_hashUpdate, (void* pHash, const Ipp8u* pMsg, int msgLen)) +{ + UpdateSM3(pHash, pMsg, msgLen, sm3_cnt); +} + +IPP_OWN_DEFN (static void, sm3_hashOctString, (Ipp8u* pMD, void* pHashVal)) +{ + /* convert hash into big endian */ + ((Ipp32u*)pMD)[0] = ENDIANNESS32(((Ipp32u*)pHashVal)[0]); + ((Ipp32u*)pMD)[1] = ENDIANNESS32(((Ipp32u*)pHashVal)[1]); + ((Ipp32u*)pMD)[2] = ENDIANNESS32(((Ipp32u*)pHashVal)[2]); + ((Ipp32u*)pMD)[3] = ENDIANNESS32(((Ipp32u*)pHashVal)[3]); + ((Ipp32u*)pMD)[4] = ENDIANNESS32(((Ipp32u*)pHashVal)[4]); + ((Ipp32u*)pMD)[5] = ENDIANNESS32(((Ipp32u*)pHashVal)[5]); + ((Ipp32u*)pMD)[6] = ENDIANNESS32(((Ipp32u*)pHashVal)[6]); + ((Ipp32u*)pMD)[7] = ENDIANNESS32(((Ipp32u*)pHashVal)[7]); +} + +IPP_OWN_DEFN (static void, sm3_msgRep, (Ipp8u* pDst, Ipp64u lenLo, Ipp64u lenHi)) +{ + IPP_UNREFERENCED_PARAMETER(lenHi); + lenLo = ENDIANNESS64(lenLo<<3); + ((Ipp64u*)(pDst))[0] = lenLo; +} + +#define cpFinalizeSM3 OWNAPI(cpFinalizeSM3) + IPP_OWN_DECL (void, cpFinalizeSM3, (DigestSHA1 pHash, const Ipp8u* inpBuffer, int inpLen, Ipp64u processedMsgLen)) + +#endif /* #if !defined _PCP_SM3_STUFF_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3unpack.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3unpack.c new file mode 100644 index 0000000..07114e9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3unpack.c @@ -0,0 +1,58 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SM3 +// +// Contents: +// ippsSM3Unpack() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" + +/*F* +// Name: ippsSM3Unpack +// +// Purpose: Unpack buffer content into the initialized context. +// +// Returns: Reason: +// ippStsNullPtrErr pBuffer == NULL +// pState == NULL +// ippStsNoErr no errors +// +// Parameters: +// pBuffer pointer to the input buffer +// pState pointer hash state +// +*F*/ +IPPFUN(IppStatus, ippsSM3Unpack,(const Ipp8u* pBuffer, IppsSM3State* pState)) +{ + /* test pointers */ + IPP_BAD_PTR2_RET(pState, pBuffer); + + CopyBlock(pBuffer, pState, sizeof(IppsSM3State)); + HASH_SET_ID(pState, idCtxSM3); + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3update.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3update.c new file mode 100644 index 0000000..dfd3f24 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsm3update.c @@ -0,0 +1,113 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SM3 +// +// Contents: +// ippsSM3Update() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsm3stuff.h" + +/*F* +// Name: ippsSM3Update +// +// Purpose: Updates intermediate digest based on input stream. +// +// Returns: Reason: +// ippStsNullPtrErr pSrc == NULL +// pState == NULL +// ippStsContextMatchErr pState->idCtx != idCtxSM3 +// ippStsLengthErr len <0 +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the input stream +// len input stream length +// pState pointer to the SM3 state +// +*F*/ +IPPFUN(IppStatus, ippsSM3Update,(const Ipp8u* pSrc, int len, IppsSM3State* pState)) +{ + /* test state pointer and ID */ + IPP_BAD_PTR1_RET(pState); + IPP_BADARG_RET(!HASH_VALID_ID(pState, idCtxSM3), ippStsContextMatchErr); + + /* test input length */ + IPP_BADARG_RET((len<0), ippStsLengthErr); + /* test source pointer */ + IPP_BADARG_RET((len && !pSrc), ippStsNullPtrErr); + + /* + // handle non empty message + */ + if(len) { + int procLen; + + int idx = HASH_BUFFIDX(pState); + Ipp8u* pBuffer = HASH_BUFF(pState); + Ipp64u lenLo = HASH_LENLO(pState) + (Ipp64u)len; + + /* if non empty internal buffer filling */ + if(idx) { + /* copy from input stream to the internal buffer as match as possible */ + procLen = IPP_MIN(len, (MBS_SM3-idx)); + CopyBlock(pSrc, pBuffer+idx, procLen); + + /* update message pointer and length */ + idx += procLen; + pSrc += procLen; + len -= procLen; + + /* update digest if buffer full */ + if( MBS_SM3 == idx) { + UpdateSM3(HASH_VALUE(pState), pBuffer, MBS_SM3, sm3_cnt); + idx = 0; + } + } + + /* main message part processing */ + procLen = len & ~(MBS_SM3-1); + if(procLen) { + UpdateSM3(HASH_VALUE(pState), pSrc, procLen, sm3_cnt); + pSrc += procLen; + len -= procLen; + } + + /* store rest of message into the internal buffer */ + if(len) { + CopyBlock(pSrc, pBuffer, len); + idx += len; + } + + /* update length of processed message */ + HASH_LENLO(pState) = lenLo; + HASH_BUFFIDX(pState) = idx; + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4.h new file mode 100644 index 0000000..d40d8de --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4.h @@ -0,0 +1,185 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Internal Definitions and +// Internal SMS4 Function Prototypes +// +// +*/ + +#if !defined(_PCP_SMS4_H) +#define _PCP_SMS4_H + +#include "owncp.h" +#include "pcpmask_ct.h" + +/* SMS4 round keys number */ +#define SMS4_ROUND_KEYS_NUM (32) + +#if SMS4_ROUND_KEYS_NUM != 32 + #error SMS4_ROUND_KEYS_NUM must be equal 32 +#endif + +struct _cpSMS4 { + Ipp32u idCtx; /* SMS4 spec identifier */ + Ipp32u enc_rkeys[SMS4_ROUND_KEYS_NUM]; /* enc round keys */ + Ipp32u dec_rkeys[SMS4_ROUND_KEYS_NUM]; /* dec round keys */ +}; + +/* +// access macros +*/ +#define SMS4_SET_ID(ctx) ((ctx)->idCtx = (Ipp32u)idCtxSMS4 ^ (Ipp32u)IPP_UINT_PTR(ctx)) +#define SMS4_RK(ctx) ((ctx)->enc_rkeys) +#define SMS4_ERK(ctx) ((ctx)->enc_rkeys) +#define SMS4_DRK(ctx) ((ctx)->dec_rkeys) + +/* SMS4 data block size (bytes) */ +#define MBS_SMS4 (16) + +#if MBS_SMS4 != 16 + #error MBS_SMS4 must be equal 16 +#endif + +/* valid SMS4 context ID */ +#define VALID_SMS4_ID(ctx) ((((ctx)->idCtx) ^ (Ipp32u)IPP_UINT_PTR((ctx))) == (Ipp32u)idCtxSMS4) + +/* alignment of AES context */ +#define SMS4_ALIGNMENT (4) + +/* size of SMS4 context */ +__INLINE int cpSizeofCtx_SMS4(void) +{ + return sizeof(IppsSMS4Spec); +} + +/* SMS4 constants */ +extern const __ALIGN64 Ipp8u SMS4_Sbox[16*16]; +extern const Ipp32u SMS4_FK[4]; +extern const Ipp32u SMS4_CK[32]; + +////////////////////////////////////////////////////////////////////////////////////////////////////// +/* S-box substitution (endian dependent!) */ + +#include "pcpbnuimpl.h" +#define SELECTION_BITS ((sizeof(BNU_CHUNK_T)/sizeof(Ipp8u)) -1) + +__INLINE Ipp8u getSboxValue(Ipp8u x) +{ + BNU_CHUNK_T selection = 0; + const Ipp8u* SboxEntry = SMS4_Sbox; + + Ipp32u i; + for (i = 0; i> 8) & 0xFF) <<8); + y |= (Ipp32u)(getSboxValue((x>>16) & 0xFF) <<16); + y |= (Ipp32u)(getSboxValue((x>>24) & 0xFF) <<24); + return y; +} + +/* key expansion transformation: + - linear Linear + - mixer Mix (permutation T in the SMS4 standart phraseology) +*/ +__INLINE Ipp32u cpExpKeyLinear_SMS4(Ipp32u x) +{ + return x^ROL32(x,13)^ROL32(x,23); +} + +__INLINE Ipp32u cpExpKeyMix_SMS4(Ipp32u x) +{ + return cpExpKeyLinear_SMS4( cpSboxT_SMS4(x) ); +} + +/* cipher transformations: + - linear Linear + - mixer Mix (permutation T in the SMS4 standart phraseology) +*/ +__INLINE Ipp32u cpCipherLinear_SMS4(Ipp32u x) +{ + return x^ROL32(x,2)^ROL32(x,10)^ROL32(x,18)^ROL32(x,24); +} + +__INLINE Ipp32u cpCipherMix_SMS4(Ipp32u x) +{ + return cpCipherLinear_SMS4( cpSboxT_SMS4(x) ); +} +////////////////////////////////////////////////////////////////////////////////////////////// + + +#define cpSMS4_Cipher OWNAPI(cpSMS4_Cipher) + IPP_OWN_DECL (void, cpSMS4_Cipher, (Ipp8u* otxt, const Ipp8u* itxt, const Ipp32u* pRoundKeys)) + +#if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) +#define cpSMS4_SetRoundKeys_aesni OWNAPI(cpSMS4_SetRoundKeys_aesni) + IPP_OWN_DECL (void, cpSMS4_SetRoundKeys_aesni, (Ipp32u* pRounKey, const Ipp8u* pSecretKey)) + +#define cpSMS4_ECB_aesni_x1 OWNAPI(cpSMS4_ECB_aesni_x1) + IPP_OWN_DECL (void, cpSMS4_ECB_aesni_x1, (Ipp8u* pOut, const Ipp8u* pInp, const Ipp32u* pRKey)) +#define cpSMS4_ECB_aesni OWNAPI(cpSMS4_ECB_aesni) + IPP_OWN_DECL (int, cpSMS4_ECB_aesni, (Ipp8u* pDst, const Ipp8u* pSrc, int nLen, const Ipp32u* pRKey)) +#define cpSMS4_CBC_dec_aesni OWNAPI(cpSMS4_CBC_dec_aesni) + IPP_OWN_DECL (int, cpSMS4_CBC_dec_aesni, (Ipp8u* pDst, const Ipp8u* pSrc, int nLen, const Ipp32u* pRKey, Ipp8u* pIV)) +#define cpSMS4_CTR_aesni OWNAPI(cpSMS4_CTR_aesni) + IPP_OWN_DECL (int, cpSMS4_CTR_aesni, (Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, const Ipp8u* pCtrMask, Ipp8u* pCtr)) + +#if (_IPP>=_IPP_H9) || (_IPP32E>=_IPP32E_L9) +#define cpSMS4_ECB_aesni_x12 OWNAPI(cpSMS4_ECB_aesni_x12) + IPP_OWN_DECL (int, cpSMS4_ECB_aesni_x12, (Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey)) +#define cpSMS4_CBC_dec_aesni_x12 OWNAPI(cpSMS4_CBC_dec_aesni_x12) + IPP_OWN_DECL (int, cpSMS4_CBC_dec_aesni_x12, (Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, Ipp8u* pIV)) +#define cpSMS4_CTR_aesni_x4 OWNAPI(cpSMS4_CTR_aesni_x4) + IPP_OWN_DECL (int, cpSMS4_CTR_aesni_x4, (Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, const Ipp8u* pCtrMask, Ipp8u* pCtr)) + +#if (_IPP32E>=_IPP32E_K1) +#if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) + +#define cpSMS4_ECB_gfni_x1 OWNAPI(cpSMS4_ECB_gfni_x1) + IPP_OWN_DECL (void, cpSMS4_ECB_gfni_x1, (Ipp8u* pOut, const Ipp8u* pInp, const Ipp32u* pRKey)) +#define cpSMS4_ECB_gfni512 OWNAPI(cpSMS4_ECB_gfni512) + IPP_OWN_DECL (int, cpSMS4_ECB_gfni512, (Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey)) +#define cpSMS4_CBC_dec_gfni512 OWNAPI(cpSMS4_CBC_dec_gfni512) + IPP_OWN_DECL (int, cpSMS4_CBC_dec_gfni512, (Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, Ipp8u* pIV)) +#define cpSMS4_CTR_gfni512 OWNAPI(cpSMS4_CTR_gfni512) + IPP_OWN_DECL (int, cpSMS4_CTR_gfni512, (Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, const Ipp8u* pCtrMask, Ipp8u* pCtr)) +#define cpSMS4_CFB_dec_gfni512 OWNAPI(cpSMS4_CFB_dec_gfni512) + IPP_OWN_DECL (void, cpSMS4_CFB_dec_gfni512, (Ipp8u* pOut, const Ipp8u* pInp, int len, int cfbBlkSize, const Ipp32u* pRKey, Ipp8u* pIV)) + +#endif /* #if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) */ +#endif /* (_IPP32E>=_IPP32E_K1) */ + +#endif /* (_IPP>=_IPP_H9) || (_IPP32E>=_IPP32E_L9) */ + +#endif + +#define cpProcessSMS4_ctr OWNAPI(cpProcessSMS4_ctr) + IPP_OWN_DECL (IppStatus, cpProcessSMS4_ctr, (const Ipp8u* pSrc, Ipp8u* pDst, int dataLen, const IppsSMS4Spec* pCtx, Ipp8u* pCtrValue, int ctrNumBitSize)) + +#endif /* _PCP_SMS4_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_cbc_gfni.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_cbc_gfni.c new file mode 100644 index 0000000..044c451 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_cbc_gfni.c @@ -0,0 +1,1267 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 ECB decryption +// +// Contents: +// cpSMS4_ECB_gfni512() +// cpSMS4_CBC_dec_gfni512x48() +// cpSMS4_CBC_dec_gfni512x32() +// cpSMS4_CBC_dec_gfni512x16() +// cpSMS4_CBC_dec_gfni128x12() +// cpSMS4_CBC_dec_gfni128x8() +// cpSMS4_CBC_dec_gfni128x4() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpsms4.h" +#include "pcptool.h" + +#if (_IPP32E>=_IPP32E_K1) + +#if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) + +#include "pcpsms4_gfni.h" + +static +int cpSMS4_CBC_dec_gfni512x48(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, Ipp8u* pIV); +static +int cpSMS4_CBC_dec_gfni512x32(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, Ipp8u* pIV); +static +int cpSMS4_CBC_dec_gfni512x16(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, Ipp8u* pIV); +static +int cpSMS4_CBC_dec_gfni128x12(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, Ipp8u* pIV); +static +int cpSMS4_CBC_dec_gfni128x8(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, Ipp8u* pIV); +static +int cpSMS4_CBC_dec_gfni128x4(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, Ipp8u* pIV); + +/* +// 64*MBS_SMS4 bytes processing +*/ + +IPP_OWN_DEFN (int, cpSMS4_CBC_dec_gfni512, (Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, Ipp8u* pIV)) +{ + __ALIGN16 __m512i TMP[21]; + + __m128i IV = _mm_loadu_si128((__m128i*)(pIV)); // Non-secret data + + int processedLen = len - (len % (64 * MBS_SMS4)); + int n; + for (n = 0; n < processedLen; n += (64 * MBS_SMS4), pInp += (64 * MBS_SMS4), pOut += (64 * MBS_SMS4)) { + int itr; + + TMP[0] = _mm512_loadu_si512((__m512i*)(pInp)); + TMP[1] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 4)); + TMP[2] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 8)); + TMP[3] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 12)); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 16)); + TMP[1] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 20)); + TMP[2] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 24)); + TMP[3] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 28)); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 32)); + TMP[1] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 36)); + TMP[2] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 40)); + TMP[3] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 44)); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[12], TMP[13], TMP[14], TMP[15], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 48)); + TMP[1] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 52)); + TMP[2] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 56)); + TMP[3] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 60)); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[16], TMP[17], TMP[18], TMP[19], TMP[0], TMP[1], TMP[2], TMP[3]); + + for (itr = 0; itr < 8; itr++, pRKey += 4) { + /* initial xors */ + TMP[3] = TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[0]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[13]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[14]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[15]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[17]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[18]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[19]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + TMP[3] = sBox512(TMP[3]); + /* Sbox done, now L */ + TMP[4] = _mm512_xor_si512(_mm512_xor_si512(TMP[4], TMP[0]), L512(TMP[0])); + TMP[8] = _mm512_xor_si512(_mm512_xor_si512(TMP[8], TMP[1]), L512(TMP[1])); + TMP[12] = _mm512_xor_si512(_mm512_xor_si512(TMP[12], TMP[2]), L512(TMP[2])); + TMP[16] = _mm512_xor_si512(_mm512_xor_si512(TMP[16], TMP[3]), L512(TMP[3])); + + /* initial xors */ + TMP[3] = TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[1]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[14]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[15]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[12]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[18]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[19]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[16]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + TMP[3] = sBox512(TMP[3]); + /* Sbox done, now L */ + TMP[5] = _mm512_xor_si512(_mm512_xor_si512(TMP[5], TMP[0]), L512(TMP[0])); + TMP[9] = _mm512_xor_si512(_mm512_xor_si512(TMP[9], TMP[1]), L512(TMP[1])); + TMP[13] = _mm512_xor_si512(_mm512_xor_si512(TMP[13], TMP[2]), L512(TMP[2])); + TMP[17] = _mm512_xor_si512(_mm512_xor_si512(TMP[17], TMP[3]), L512(TMP[3])); + + /* initial xors */ + TMP[3] = TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[2]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[15]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[12]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[13]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[19]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[16]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[17]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + TMP[3] = sBox512(TMP[3]); + /* Sbox done, now L */ + TMP[6] = _mm512_xor_si512(_mm512_xor_si512(TMP[6], TMP[0]), L512(TMP[0])); + TMP[10] = _mm512_xor_si512(_mm512_xor_si512(TMP[10], TMP[1]), L512(TMP[1])); + TMP[14] = _mm512_xor_si512(_mm512_xor_si512(TMP[14], TMP[2]), L512(TMP[2])); + TMP[18] = _mm512_xor_si512(_mm512_xor_si512(TMP[18], TMP[3]), L512(TMP[3])); + + /* initial xors */ + TMP[3] = TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[3]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[12]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[13]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[14]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[16]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[17]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[18]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + TMP[3] = sBox512(TMP[3]); + /* Sbox done, now L */ + TMP[7] = _mm512_xor_si512(_mm512_xor_si512(TMP[7], TMP[0]), L512(TMP[0])); + TMP[11] = _mm512_xor_si512(_mm512_xor_si512(TMP[11], TMP[1]), L512(TMP[1])); + TMP[15] = _mm512_xor_si512(_mm512_xor_si512(TMP[15], TMP[2]), L512(TMP[2])); + TMP[19] = _mm512_xor_si512(_mm512_xor_si512(TMP[19], TMP[3]), L512(TMP[3])); + } + + pRKey -= 32; + + /* Prepare the first block for xor*/ + + TMP[20] = _mm512_loadu_si512((__m512i*)(pInp)); + TMP[20] = _mm512_alignr_epi64(TMP[20], TMP[20], 6); + TMP[20] = _mm512_inserti64x2(TMP[20], IV, 0); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + + TMP[4] = _mm512_xor_si512(TMP[0], TMP[20]); + TMP[5] = _mm512_xor_si512(TMP[1], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*3))); + TMP[6] = _mm512_xor_si512(TMP[2], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*7))); + TMP[7] = _mm512_xor_si512(TMP[3], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*11))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + + TMP[8] = _mm512_xor_si512(TMP[0], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*15))); + TMP[9] = _mm512_xor_si512(TMP[1], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*19))); + TMP[10] = _mm512_xor_si512(TMP[2], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*23))); + TMP[11] = _mm512_xor_si512(TMP[3], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*27))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[12], TMP[13], TMP[14], TMP[15]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + + TMP[12] = _mm512_xor_si512(TMP[0], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*31))); + TMP[13] = _mm512_xor_si512(TMP[1], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*35))); + TMP[14] = _mm512_xor_si512(TMP[2], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*39))); + TMP[15] = _mm512_xor_si512(TMP[3], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*43))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[16], TMP[17], TMP[18], TMP[19]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + + TMP[16] = _mm512_xor_si512(TMP[0], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*47))); + TMP[17] = _mm512_xor_si512(TMP[1], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*51))); + TMP[18] = _mm512_xor_si512(TMP[2], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*55))); + TMP[19] = _mm512_xor_si512(TMP[3], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*59))); + + IV = _mm_loadu_si128((__m128i*)(pInp+MBS_SMS4*63)); // Ciphertext, non-secret data + + _mm512_storeu_si512((__m512i*)(pOut), TMP[4]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 4), TMP[5]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 8), TMP[6]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 12), TMP[7]); + + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 16), TMP[8]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 20), TMP[9]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 24), TMP[10]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 28), TMP[11]); + + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 32), TMP[12]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 36), TMP[13]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 40), TMP[14]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 44), TMP[15]); + + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 48), TMP[16]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 52), TMP[17]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 56), TMP[18]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 60), TMP[19]); + } + + _mm_storeu_si128((__m128i*)(pIV), IV); + + /* clear secret data */ + for (unsigned int i = 0; i < sizeof(TMP) / sizeof(TMP[0]); ++i) { + TMP[i] = _mm512_setzero_si512(); //_mm512_xor_si512(TMP[i], TMP[i]); + } + + len -= processedLen; + if (len){ + if(len < 256){ + processedLen += cpSMS4_CBC_dec_gfni128x12(pOut, pInp, len, pRKey, pIV); + return processedLen; + } + processedLen += cpSMS4_CBC_dec_gfni512x48(pOut, pInp, len, pRKey, pIV); + } + + return processedLen; +} + +/* +// 48*MBS_SMS4 bytes processing +*/ + +static +int cpSMS4_CBC_dec_gfni512x48(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, Ipp8u* pIV) +{ + __ALIGN16 __m512i TMP[17]; + + __m128i IV = _mm_loadu_si128((__m128i*)(pIV)); // Non-secret data + + int processedLen = len - (len % (48 * MBS_SMS4)); + int n; + for (n = 0; n < processedLen; n += (48 * MBS_SMS4), pInp += (48 * MBS_SMS4), pOut += (48 * MBS_SMS4)) { + int itr; + + TMP[0] = _mm512_loadu_si512((__m512i*)(pInp)); + TMP[1] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 4)); + TMP[2] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 8)); + TMP[3] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 12)); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 16)); + TMP[1] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 20)); + TMP[2] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 24)); + TMP[3] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 28)); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 32)); + TMP[1] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 36)); + TMP[2] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 40)); + TMP[3] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 44)); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[12], TMP[13], TMP[14], TMP[15], TMP[0], TMP[1], TMP[2], TMP[3]); + + for (itr = 0; itr < 8; itr++, pRKey += 4) { + /* initial xors */ + TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[0]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[13]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[14]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[15]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + /* Sbox done, now L */ + TMP[4] = _mm512_xor_si512(_mm512_xor_si512(TMP[4], TMP[0]), L512(TMP[0])); + TMP[8] = _mm512_xor_si512(_mm512_xor_si512(TMP[8], TMP[1]), L512(TMP[1])); + TMP[12] = _mm512_xor_si512(_mm512_xor_si512(TMP[12], TMP[2]), L512(TMP[2])); + + /* initial xors */ + TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[1]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[14]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[15]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[12]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + /* Sbox done, now L */ + TMP[5] = _mm512_xor_si512(_mm512_xor_si512(TMP[5], TMP[0]), L512(TMP[0])); + TMP[9] = _mm512_xor_si512(_mm512_xor_si512(TMP[9], TMP[1]), L512(TMP[1])); + TMP[13] = _mm512_xor_si512(_mm512_xor_si512(TMP[13], TMP[2]), L512(TMP[2])); + + /* initial xors */ + TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[2]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[15]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[12]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[13]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + /* Sbox done, now L */ + TMP[6] = _mm512_xor_si512(_mm512_xor_si512(TMP[6], TMP[0]), L512(TMP[0])); + TMP[10] = _mm512_xor_si512(_mm512_xor_si512(TMP[10], TMP[1]), L512(TMP[1])); + TMP[14] = _mm512_xor_si512(_mm512_xor_si512(TMP[14], TMP[2]), L512(TMP[2])); + + /* initial xors */ + TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[3]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[12]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[13]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[14]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + /* Sbox done, now L */ + TMP[7] = _mm512_xor_si512(_mm512_xor_si512(TMP[7], TMP[0]), L512(TMP[0])); + TMP[11] = _mm512_xor_si512(_mm512_xor_si512(TMP[11], TMP[1]), L512(TMP[1])); + TMP[15] = _mm512_xor_si512(_mm512_xor_si512(TMP[15], TMP[2]), L512(TMP[2])); + } + + pRKey -= 32; + + /* Prepare the first block for xor*/ + + TMP[16] = _mm512_loadu_si512((__m512i*)(pInp)); + TMP[16] = _mm512_alignr_epi64(TMP[16], TMP[16], 6); + TMP[16] = _mm512_inserti64x2(TMP[16], IV, 0); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + + TMP[4] = _mm512_xor_si512(TMP[0], TMP[16]); + TMP[5] = _mm512_xor_si512(TMP[1], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*3))); + TMP[6] = _mm512_xor_si512(TMP[2], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*7))); + TMP[7] = _mm512_xor_si512(TMP[3], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*11))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + + TMP[8] = _mm512_xor_si512(TMP[0], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*15))); + TMP[9] = _mm512_xor_si512(TMP[1], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*19))); + TMP[10] = _mm512_xor_si512(TMP[2], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*23))); + TMP[11] = _mm512_xor_si512(TMP[3], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*27))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[12], TMP[13], TMP[14], TMP[15]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + + TMP[12] = _mm512_xor_si512(TMP[0], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*31))); + TMP[13] = _mm512_xor_si512(TMP[1], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*35))); + TMP[14] = _mm512_xor_si512(TMP[2], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*39))); + TMP[15] = _mm512_xor_si512(TMP[3], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*43))); + + IV = _mm_loadu_si128((__m128i*)(pInp+MBS_SMS4*47)); // Ciphertext, non-secret data + + _mm512_storeu_si512((__m512i*)(pOut), TMP[4]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 4), TMP[5]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 8), TMP[6]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 12), TMP[7]); + + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 16), TMP[8]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 20), TMP[9]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 24), TMP[10]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 28), TMP[11]); + + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 32), TMP[12]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 36), TMP[13]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 40), TMP[14]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 44), TMP[15]); + } + + _mm_storeu_si128((__m128i*)(pIV), IV); + + /* clear secret data */ + for (unsigned int i = 0; i < sizeof(TMP) / sizeof(TMP[0]); ++i) { + TMP[i] = _mm512_setzero_si512(); //_mm512_xor_si512(TMP[i], TMP[i]); + } + + len -= processedLen; + if (len) + processedLen += cpSMS4_CBC_dec_gfni512x32(pOut, pInp, len, pRKey, pIV); + + return processedLen; +} + +/* +// 32*MBS_SMS4 bytes processing +*/ + +static +int cpSMS4_CBC_dec_gfni512x32(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, Ipp8u* pIV) +{ + __ALIGN16 __m512i TMP[13]; + + __m128i IV = _mm_loadu_si128((__m128i*)(pIV)); // Non-secret data + + int processedLen = len - (len % (32 * MBS_SMS4)); + int n; + for (n = 0; n < processedLen; n += (32 * MBS_SMS4), pInp += (32 * MBS_SMS4), pOut += (32 * MBS_SMS4)) { + int itr; + + TMP[0] = _mm512_loadu_si512((__m512i*)(pInp)); + TMP[1] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 4)); + TMP[2] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 8)); + TMP[3] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 12)); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 16)); + TMP[1] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 20)); + TMP[2] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 24)); + TMP[3] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 28)); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], TMP[0], TMP[1], TMP[2], TMP[3]); + + for (itr = 0; itr < 8; itr++, pRKey += 4) { + /* initial xors */ + TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[0]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + /* Sbox done, now L */ + TMP[4] = _mm512_xor_si512(_mm512_xor_si512(TMP[4], TMP[0]), L512(TMP[0])); + TMP[8] = _mm512_xor_si512(_mm512_xor_si512(TMP[8], TMP[1]), L512(TMP[1])); + + /* initial xors */ + TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[1]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + /* Sbox done, now L */ + TMP[5] = _mm512_xor_si512(_mm512_xor_si512(TMP[5], TMP[0]), L512(TMP[0])); + TMP[9] = _mm512_xor_si512(_mm512_xor_si512(TMP[9], TMP[1]), L512(TMP[1])); + + /* initial xors */ + TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[2]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + /* Sbox done, now L */ + TMP[6] = _mm512_xor_si512(_mm512_xor_si512(TMP[6], TMP[0]), L512(TMP[0])); + TMP[10] = _mm512_xor_si512(_mm512_xor_si512(TMP[10], TMP[1]), L512(TMP[1])); + + /* initial xors */ + TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[3]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + /* Sbox done, now L */ + TMP[7] = _mm512_xor_si512(_mm512_xor_si512(TMP[7], TMP[0]), L512(TMP[0])); + TMP[11] = _mm512_xor_si512(_mm512_xor_si512(TMP[11], TMP[1]), L512(TMP[1])); + } + + pRKey -= 32; + + /* Prepare the first block for xor*/ + + TMP[12] = _mm512_loadu_si512((__m512i*)(pInp)); + TMP[12] = _mm512_alignr_epi64(TMP[12], TMP[12], 6); + TMP[12] = _mm512_inserti64x2(TMP[12], IV, 0); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + + TMP[4] = _mm512_xor_si512(TMP[0], TMP[12]); + TMP[5] = _mm512_xor_si512(TMP[1], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*3))); + TMP[6] = _mm512_xor_si512(TMP[2], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*7))); + TMP[7] = _mm512_xor_si512(TMP[3], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*11))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + + TMP[8] = _mm512_xor_si512(TMP[0], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*15))); + TMP[9] = _mm512_xor_si512(TMP[1], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*19))); + TMP[10] = _mm512_xor_si512(TMP[2], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*23))); + TMP[11] = _mm512_xor_si512(TMP[3], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*27))); + + IV = _mm_loadu_si128((__m128i*)(pInp+MBS_SMS4*31)); // Ciphertext, non-secret data + + _mm512_storeu_si512((__m512i*)(pOut), TMP[4]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 4), TMP[5]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 8), TMP[6]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 12), TMP[7]); + + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 16), TMP[8]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 20), TMP[9]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 24), TMP[10]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 28), TMP[11]); + } + + _mm_storeu_si128((__m128i*)(pIV), IV); + + /* clear secret data */ + for (unsigned int i = 0; i < sizeof(TMP) / sizeof(TMP[0]); ++i) { + TMP[i] = _mm512_setzero_si512(); //_mm512_xor_si512(TMP[i], TMP[i]); + } + + len -= processedLen; + if (len) + processedLen += cpSMS4_CBC_dec_gfni512x16(pOut, pInp, len, pRKey, pIV); + + return processedLen; +} + +/* +// 16*MBS_SMS4 bytes processing +*/ + +static +int cpSMS4_CBC_dec_gfni512x16(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, Ipp8u* pIV) +{ + __ALIGN16 __m512i TMP[9]; + + __m128i IV = _mm_loadu_si128((__m128i*)(pIV)); // Non-secret data + + int processedLen = len - (len % (16 * MBS_SMS4)); + int n; + for (n = 0; n < processedLen; n += (16 * MBS_SMS4), pInp += (16 * MBS_SMS4), pOut += (16 * MBS_SMS4)) { + int itr; + + TMP[0] = _mm512_loadu_si512((__m512i*)(pInp)); + TMP[1] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 4)); + TMP[2] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 8)); + TMP[3] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 12)); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + for (itr = 0; itr < 8; itr++, pRKey += 4) { + /* initial xors */ + TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[0]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + /* Sbox done, now L */ + TMP[4] = _mm512_xor_si512(_mm512_xor_si512(TMP[4], TMP[0]), L512(TMP[0])); + + /* initial xors */ + TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[1]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + /* Sbox done, now L */ + TMP[5] = _mm512_xor_si512(_mm512_xor_si512(TMP[5], TMP[0]), L512(TMP[0])); + + /* initial xors */ + TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[2]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + /* Sbox done, now L */ + TMP[6] = _mm512_xor_si512(_mm512_xor_si512(TMP[6], TMP[0]), L512(TMP[0])); + + /* initial xors */ + TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[3]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + /* Sbox done, now L */ + TMP[7] = _mm512_xor_si512(_mm512_xor_si512(TMP[7], TMP[0]), L512(TMP[0])); + } + + pRKey -= 32; + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + + TMP[8] = _mm512_loadu_si512((__m512i*)(pInp)); + TMP[8] = _mm512_alignr_epi64(TMP[8], TMP[8], 6); + TMP[8] = _mm512_inserti64x2(TMP[8], IV, 0); + + TMP[4] = _mm512_xor_si512(TMP[0], TMP[8]); + TMP[5] = _mm512_xor_si512(TMP[1], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*3))); + TMP[6] = _mm512_xor_si512(TMP[2], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*7))); + TMP[7] = _mm512_xor_si512(TMP[3], _mm512_loadu_si512((__m512i*)(pInp+MBS_SMS4*11))); + + IV = _mm_loadu_si128((__m128i*)(pInp+MBS_SMS4*15)); // Ciphertext, non-secret data + + _mm512_storeu_si512((__m512i*)(pOut), TMP[4]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 4), TMP[5]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 8), TMP[6]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 12), TMP[7]); + } + + _mm_storeu_si128((__m128i*)(pIV), IV); + + /* clear secret data */ + for (unsigned int i = 0; i < sizeof(TMP) / sizeof(TMP[0]); ++i) { + TMP[i] = _mm512_setzero_si512(); //_mm512_xor_si512(TMP[i], TMP[i]); + } + + len -= processedLen; + if (len) + processedLen += cpSMS4_CBC_dec_gfni128x12(pOut, pInp, len, pRKey, pIV); + + return processedLen; +} + +/* +// 12*MBS_SMS4 bytes processing +*/ + +static +int cpSMS4_CBC_dec_gfni128x12(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, Ipp8u* pIV) +{ + __ALIGN16 __m128i TMP[19]; + TMP[3] = _mm_loadu_si128((__m128i*)(pIV)); + + int processedLen = len -(len % (12*MBS_SMS4)); + int n; + for(n=0; n= 1920) */ + +#endif /* _IPP32E>=_IPP32E_K1 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ccmdecrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ccmdecrypt.c new file mode 100644 index 0000000..ea67531 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ccmdecrypt.c @@ -0,0 +1,188 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// ippsSMS4_CCMDecrypt() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpsms4authccm.h" +#include "pcptool.h" + +/*F* +// Name: ippsSMS4_CCMDecrypt +// +// Purpose: Decrypts data and updates authentication tag. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// pSrc == NULL +// pDst == NULL +// ippStsContextMatchErr !VALID_SMS4CCM_ID() +// ippStsLengthErr if exceed overall length of message is being processed +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the cipher text beffer +// pDst pointer to the plane text bubber +// len length of the bugger +// pCtx pointer to the CCM context +// +*F*/ + +/* +// +// NOTE +// +// We consider to not spend time for further optimizing of this algoritm because it is not widely using. +// There is two ways for further optimization: +// - Add parallel processing of CTR cipher. Performance advantages can be achieved by parallel processing of a big number of blocks. +// - Try to decreace the memory reading/writing operation number, eg combine two calls of one block encryption +// function to single call that procees two blocks with the same key. Performance advantages can be achieved by +// reducing of reading/writing operations number, because we do not need to read the key twice in single loop. +// +*/ + +IPPFUN(IppStatus, ippsSMS4_CCMDecrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int len, IppsSMS4_CCMState* pCtx)) +{ + /* test pCtx pointer */ + IPP_BAD_PTR1_RET(pCtx); + + /* test state ID */ + IPP_BADARG_RET(!VALID_SMS4CCM_ID(pCtx), ippStsContextMatchErr); + + /* test source/destination data */ + IPP_BAD_PTR2_RET(pSrc, pDst); + + /* test message length */ + IPP_BADARG_RET(len<0 || SMS4CCM_LENPRO(pCtx)+(Ipp64u)len >SMS4CCM_MSGLEN(pCtx), ippStsLengthErr); + + /* + // enctypt payload and update MAC + */ + if(len) { + /* SMS4 context */ + IppsSMS4Spec* pSMS4 = SMS4CCM_CIPHER(pCtx); + + /* buffer for secret data */ + __ALIGN16 Ipp32u TMP[3*(MBS_SMS4/sizeof(Ipp32u))+6]; + + /* + MAC size = MBS_SMS4/sizeof(Ipp32u) + CTR size = MBS_SMS4/sizeof(Ipp32u) + S size = MBS_SMS4/sizeof(Ipp32u) + flag size = 1 + qLen size = 1 + tmpLen size = 1 + counterVal size = 1 + counterEnc size = 2 + */ + + Ipp32u* MAC = TMP; + Ipp32u* CTR = TMP + MBS_SMS4/sizeof(Ipp32u); + Ipp32u* S = TMP + 2*(MBS_SMS4/sizeof(Ipp32u)); + Ipp32u* flag = TMP + 3*(MBS_SMS4/sizeof(Ipp32u)); + Ipp32u* qLen = TMP + 3*(MBS_SMS4/sizeof(Ipp32u)) + 1; + Ipp32u* tmpLen = TMP + 3*(MBS_SMS4/sizeof(Ipp32u)) + 2; + Ipp32u* counterVal = TMP + 3*(MBS_SMS4/sizeof(Ipp32u)) + 3; + Ipp32u* counterEnc = TMP + 3*(MBS_SMS4/sizeof(Ipp32u)) + 4; + + *flag = (Ipp32u)( SMS4CCM_LENPRO(pCtx) &(MBS_SMS4-1) ); + + /* extract from the state */ + CopyBlock16(SMS4CCM_MAC(pCtx), MAC); + CopyBlock16(SMS4CCM_CTR0(pCtx), CTR); + CopyBlock16(SMS4CCM_Si(pCtx), S); + *counterVal = SMS4CCM_COUNTER(pCtx); + + /* extract qLen */ + *qLen = (((Ipp8u*)CTR)[0] &0x7) +1; /* &0x7 just to fix KW issue */ + + if(*flag) { + *tmpLen = (Ipp32u)( IPP_MIN(len, MBS_SMS4-1) ); + XorBlock(pSrc, (Ipp8u*)S+*flag, pDst, (Ipp32s)(*tmpLen)); + + /* copy as much input as possible into the internal buffer*/ + CopyBlock(pDst, SMS4CCM_BLK(pCtx)+*flag, (cpSize)(*tmpLen)); + + /* update MAC */ + if(*flag+*tmpLen == MBS_SMS4) { + XorBlock16(MAC, SMS4CCM_BLK(pCtx), MAC); + cpSMS4_Cipher((Ipp8u*)MAC, (Ipp8u*)MAC, SMS4_RK(pSMS4)); + } + + SMS4CCM_LENPRO(pCtx) += *tmpLen; + pSrc += *tmpLen; + pDst += *tmpLen; + len -= *tmpLen; + } + + while(len >= MBS_SMS4) { + /* increment counter and format counter block */ + *counterVal+=1; + CopyBlock(CounterEnc(counterEnc, (Ipp32s)(*qLen), *counterVal), ((Ipp8u*)CTR)+MBS_SMS4-*qLen, (cpSize)(*qLen)); + /* encode counter block */ + cpSMS4_Cipher((Ipp8u*)S, (Ipp8u*)CTR, SMS4_RK(pSMS4)); + + /* store cipher text */ + XorBlock16(pSrc, S, pDst); + + /* update MAC */ + XorBlock16(MAC, pDst, MAC); + cpSMS4_Cipher((Ipp8u*)MAC, (Ipp8u*)MAC, SMS4_RK(pSMS4)); + + SMS4CCM_LENPRO(pCtx) += MBS_SMS4; + pSrc += MBS_SMS4; + pDst += MBS_SMS4; + len -= MBS_SMS4; + } + + if(len) { + /* increment counter and format counter block */ + *counterVal+=1; + CopyBlock(CounterEnc(counterEnc, (Ipp32s)(*qLen), *counterVal), ((Ipp8u*)CTR)+MBS_SMS4-*qLen, (cpSize)(*qLen)); + /* encode counter block */ + cpSMS4_Cipher((Ipp8u*)S, (Ipp8u*)CTR, SMS4_RK(pSMS4)); + + /* store cipher text */ + XorBlock(pSrc, S, pDst, len); + + /* workaround to avoid false positive stringop-overflow error on gcc10.1 and gcc11.1 */ + len = ( IPP_MIN(len, MBS_SMS4-1) ); + + /* store partial data block */ + CopyBlock(pDst, SMS4CCM_BLK(pCtx), len); + + SMS4CCM_LENPRO(pCtx) += (Ipp64u)len; + } + + /* update state */ + CopyBlock16(MAC, SMS4CCM_MAC(pCtx)); + CopyBlock16(S, SMS4CCM_Si(pCtx)); + SMS4CCM_COUNTER(pCtx) = *counterVal; + + /* clear secret data */ + PurgeBlock(TMP, sizeof(TMP)); + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ccmencrypt.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ccmencrypt.c new file mode 100644 index 0000000..a371278 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ccmencrypt.c @@ -0,0 +1,187 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives. Cryptography Primitives. +// +// Context: +// ippsSMS4_CCMEncrypt() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpsms4authccm.h" +#include "pcptool.h" + +/*F* +// Name: ippsSMS4_CCMEncrypt +// +// Purpose: Encrypts data and updates authentication tag. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx== NULL +// pSrc == NULL +// pDst == NULL +// ippStsContextMatchErr !VALID_SMS4CCM_ID() +// ippStsLengthErr if exceed overall length of message is being processed +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the plane text beffer +// pDst pointer to the cipher text bubber +// len length of the buffer +// pCtx pointer to the CCM context +// +*F*/ + +/* +// +// NOTE +// +// We consider to not spend time for further optimizing of this algoritm because it is not widely using. +// There is two ways for further optimization: +// - Add parallel processing of CTR cipher. Performance advantages can be achieved by parallel processing of a big number of blocks. +// - Try to decreace the memory reading/writing operation number, eg combine two calls of one block encryption +// function to single call that procees two blocks with the same key. Performance advantages can be achieved by +// reducing of reading/writing operations number, because we do not need to read the key twice in single loop. +// +*/ + +IPPFUN(IppStatus, ippsSMS4_CCMEncrypt,(const Ipp8u* pSrc, Ipp8u* pDst, int len, IppsSMS4_CCMState* pCtx)) +{ + /* test pCtx pointer */ + IPP_BAD_PTR1_RET(pCtx); + IPP_BADARG_RET(!VALID_SMS4CCM_ID(pCtx), ippStsContextMatchErr); + + /* test source/destination data */ + IPP_BAD_PTR2_RET(pSrc, pDst); + + /* test message length */ + IPP_BADARG_RET(len<0 || SMS4CCM_LENPRO(pCtx)+(Ipp64u)len >SMS4CCM_MSGLEN(pCtx), ippStsLengthErr); + + /* + // enctypt payload and update MAC + */ + if(len) { + /* SMS4 context */ + IppsSMS4Spec* pSMS4 = SMS4CCM_CIPHER(pCtx); + + /* buffer for secret data */ + __ALIGN16 Ipp32u TMP[3*(MBS_SMS4/sizeof(Ipp32u))+6]; + + /* + MAC size = MBS_SMS4/sizeof(Ipp32u) + CTR size = MBS_SMS4/sizeof(Ipp32u) + S size = MBS_SMS4/sizeof(Ipp32u) + flag size = 1 + qLen size = 1 + tmpLen size = 1 + counterVal size = 1 + counterEnc size = 2 + */ + + Ipp32u* MAC = TMP; + Ipp32u* CTR = TMP + MBS_SMS4/sizeof(Ipp32u); + Ipp32u* S = TMP + 2*(MBS_SMS4/sizeof(Ipp32u)); + Ipp32u* flag = TMP + 3*(MBS_SMS4/sizeof(Ipp32u)); + Ipp32u* qLen = TMP + 3*(MBS_SMS4/sizeof(Ipp32u)) + 1; + Ipp32u* tmpLen = TMP + 3*(MBS_SMS4/sizeof(Ipp32u)) + 2; + Ipp32u* counterVal = TMP + 3*(MBS_SMS4/sizeof(Ipp32u)) + 3; + Ipp32u* counterEnc = TMP + 3*(MBS_SMS4/sizeof(Ipp32u)) + 4; + + *flag = (Ipp32u)( SMS4CCM_LENPRO(pCtx) &(MBS_SMS4-1) ); + + /* extract from the state */ + CopyBlock16(SMS4CCM_MAC(pCtx), MAC); + CopyBlock16(SMS4CCM_CTR0(pCtx), CTR); + CopyBlock16(SMS4CCM_Si(pCtx), S); + *counterVal = SMS4CCM_COUNTER(pCtx); + + /* extract qLen */ + *qLen = (((Ipp8u*)CTR)[0] &0x7) +1; /* &0x7 just to fix KW issue */ + + if(*flag) { + *tmpLen = IPP_MIN((Ipp32u)len, MBS_SMS4-1); + + /* copy as much input as possible into the internal buffer*/ + CopyBlock(pSrc, SMS4CCM_BLK(pCtx)+*flag, (cpSize)(*tmpLen)); + + XorBlock(pSrc, (Ipp8u*)S+*flag, pDst, (cpSize)(*tmpLen)); + + /* update MAC */ + if(*flag+*tmpLen == MBS_SMS4) { + XorBlock16(MAC, SMS4CCM_BLK(pCtx), MAC); + cpSMS4_Cipher((Ipp8u*)MAC, (Ipp8u*)MAC, SMS4_RK(pSMS4)); + } + + SMS4CCM_LENPRO(pCtx) += (Ipp32u)*tmpLen; + pSrc += *tmpLen; + pDst += *tmpLen; + len -= *tmpLen; + } + + while(len >= MBS_SMS4) { + /* update MAC */ + XorBlock16(MAC, pSrc, MAC); + cpSMS4_Cipher((Ipp8u*)MAC, (Ipp8u*)MAC, SMS4_RK(pSMS4)); + + /* increment counter and format counter block */ + *counterVal+=1; + CopyBlock(CounterEnc(counterEnc, (Ipp32s)(*qLen), *counterVal), ((Ipp8u*)CTR)+MBS_SMS4-*qLen, (Ipp32s)(*qLen)); + /* encode counter block */ + cpSMS4_Cipher((Ipp8u*)S, (Ipp8u*)CTR, SMS4_RK(pSMS4)); + + /* store cipher text */ + XorBlock16(pSrc, S, pDst); + + SMS4CCM_LENPRO(pCtx) += MBS_SMS4; + pSrc += MBS_SMS4; + pDst += MBS_SMS4; + len -= MBS_SMS4; + } + + if(len) { + /* workaround to avoid false positive stringop-overflow error on gcc10.1 and gcc11.1 */ + len = ( IPP_MIN(len, MBS_SMS4-1) ); + + /* store partial data block */ + CopyBlock(pSrc, SMS4CCM_BLK(pCtx), len); + + /* increment counter and format counter block */ + *counterVal+=1; + CopyBlock(CounterEnc(counterEnc, (Ipp32s)(*qLen), *counterVal), ((Ipp8u*)CTR)+MBS_SMS4-*qLen, (Ipp32s)(*qLen)); + /* encode counter block */ + cpSMS4_Cipher((Ipp8u*)S, (Ipp8u*)CTR, SMS4_RK(pSMS4)); + + /* store cipher text */ + XorBlock(pSrc, S, pDst, len); + + SMS4CCM_LENPRO(pCtx) += (Ipp64u)len; + } + + /* update state */ + CopyBlock16(MAC, SMS4CCM_MAC(pCtx)); + CopyBlock16(S, SMS4CCM_Si(pCtx)); + SMS4CCM_COUNTER(pCtx) = *counterVal; + + /* clear secret data */ + PurgeBlock(TMP, sizeof(TMP)); + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ccmgetsize.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ccmgetsize.c new file mode 100644 index 0000000..ee30b10 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ccmgetsize.c @@ -0,0 +1,54 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Cryptography Primitive. +// SMS4-CCM implementation. +// +// Content: +// ippsSMS4_CCMGetSize() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpsms4authccm.h" +#include "pcptool.h" + +/*F* +// Name: ippsSMS4_CCMGetSize +// +// Purpose: Returns size of SMS4-CCM state (bytes). +// +// Returns: Reason: +// ippStsNullPtrErr pSzie == NULL +// ippStsNoErr no errors +// +// Parameters: +// pSize pointer to the size of CCM (in bytes) +// +*F*/ +IPPFUN(IppStatus, ippsSMS4_CCMGetSize,(int * pSize)) +{ + /* test size's pointer */ + IPP_BAD_PTR1_RET(pSize); + + *pSize = cpSizeofCtx_SMS4CCM(); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ccmgettag.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ccmgettag.c new file mode 100644 index 0000000..a031d3f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ccmgettag.c @@ -0,0 +1,100 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Cryptography Primitive. +// SMS4-CCM implementation. +// +// Content: +// ippsSMS4_CCMGetTag() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpsms4authccm.h" +#include "pcptool.h" + +/*F* +// Name: ippsSMS4_CCMGetTag +// +// Purpose: Compute message auth tag and return one. +// Note, that futher encryption/decryption and auth tag update is possible +// +// Returns: Reason: +// ippStsNullPtrErr pTag == NULL +// pCtx == NULL +// ippStsContextMatchErr !VALID_SMS4CCM_ID() +// ippStsLengthErr MBS_SMS4 < tagLen +// 1 > tagLen +// ippStsNoErr no errors +// +// Parameters: +// pTag pointer to the output authenticated tag +// tagLen requested length of the tag +// pCtx pointer to the CCM context +// +*F*/ +IPPFUN(IppStatus, ippsSMS4_CCMGetTag,(Ipp8u* pTag, int tagLen, const IppsSMS4_CCMState* pCtx)) +{ + /* test pCtx pointer */ + IPP_BAD_PTR1_RET(pCtx); + + /* test state ID */ + IPP_BADARG_RET(!VALID_SMS4CCM_ID(pCtx), ippStsContextMatchErr); + + /* test tag (pointer and length) */ + IPP_BAD_PTR1_RET(pTag); + IPP_BADARG_RET((Ipp32u)tagLen>SMS4CCM_TAGLEN(pCtx) || tagLen<1, ippStsLengthErr); + + { + __ALIGN16 Ipp32u TMP[2*(MBS_SMS4/sizeof(Ipp32u)) + 1]; + /* + MAC size = MBS_SMS4/sizeof(Ipp32u) + BLK size = MBS_SMS4/sizeof(Ipp32u) + flag size = 1 + */ + + Ipp32u* MAC = TMP; + Ipp32u* BLK = TMP + (MBS_SMS4/sizeof(Ipp32u)); + Ipp32u* flag = TMP + 2*(MBS_SMS4/sizeof(Ipp32u)); + + *flag = (Ipp32u)( SMS4CCM_LENPRO(pCtx) &(MBS_SMS4-1) ); + + CopyBlock16(SMS4CCM_MAC(pCtx), MAC); + + if(*flag) { + /* SMS4 context */ + IppsSMS4Spec* pSMS4 = SMS4CCM_CIPHER(pCtx); + + FillBlock16(0, NULL,BLK, 0); + CopyBlock(SMS4CCM_BLK(pCtx), (Ipp8u*)BLK, (cpSize)(*flag)); + + XorBlock16(MAC, BLK, MAC); + cpSMS4_Cipher((Ipp8u*)MAC, (Ipp8u*)MAC, SMS4_RK(pSMS4)); + + } + + XorBlock(MAC, SMS4CCM_S0(pCtx), pTag, tagLen); + + /* clear secret data */ + PurgeBlock(TMP, sizeof(TMP)); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ccminit.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ccminit.c new file mode 100644 index 0000000..8449731 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ccminit.c @@ -0,0 +1,71 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Cryptography Primitive. +// SMS4-CCM implementation. +// +// Content: +// ippsSMS4_CCMInit() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpsms4authccm.h" +#include "pcptool.h" + +/*F* +// Name: ippsSMS4_CCMInit +// +// Purpose: Init SMS4-CCM state. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// ippStsMemAllocErr size of buffer is not match fro operation +// ippStsLengthErr keyLen < 16 +// ippStsNoErr no errors +// +// Parameters: +// pKey pointer to the secret key +// keyLen length of secret key (in bytes) +// pCtx pointer to initialized as CCM context +// ctxSize available size (in bytes) of buffer above +// +*F*/ +IPPFUN(IppStatus, ippsSMS4_CCMInit,(const Ipp8u* pKey, int keyLen, + IppsSMS4_CCMState* pCtx, int ctxSize)) +{ + /* test pCtx pointer */ + IPP_BAD_PTR1_RET(pCtx); + + /* test available size of context buffer */ + IPP_BADARG_RET(ctxSize13), ippStsLengthErr); + + /* test AAD pointer if defined */ + IPP_BADARG_RET(adLen<0, ippStsLengthErr); + if(adLen) + IPP_BAD_PTR1_RET(pAD); + + /* init for new message */ + SMS4CCM_LENPRO(pCtx) = 0; + SMS4CCM_COUNTER(pCtx) = 0; + + { + /* setup encoder function */ + IppsSMS4Spec* pSMS4 = SMS4CCM_CIPHER(pCtx); + + __ALIGN16 Ipp32u TMP[4*(MBS_SMS4/sizeof(Ipp32u)) + 8]; + /* + MAC size = MBS_SMS4/sizeof(Ipp32u) + CTR size = MBS_SMS4/sizeof(Ipp32u) + block size = 2*MBS_SMS4/sizeof(Ipp32u) + qLen size = 1 + qLenEnc size = 1 + tagLenEnc size = 1 + payloadLen size = 2 + adLenEnc size = 3 + */ + + Ipp32u* MAC = TMP; + Ipp32u* CTR = TMP + (MBS_SMS4/sizeof(Ipp32u)); + Ipp32u* block = TMP + 2*(MBS_SMS4/sizeof(Ipp32u)); + Ipp32u* qLen = TMP + 4*(MBS_SMS4/sizeof(Ipp32u)); + Ipp32u* qLenEnc = TMP + 4*(MBS_SMS4/sizeof(Ipp32u)) + 1; + Ipp32u* tagLenEnc = TMP + 4*(MBS_SMS4/sizeof(Ipp32u)) + 2; + Ipp32u* payloadLen = TMP + 4*(MBS_SMS4/sizeof(Ipp32u)) + 3; + Ipp32u* adLenEnc = TMP + 4*(MBS_SMS4/sizeof(Ipp32u)) + 5; + + /* + // prepare the 1-st input block B0 and encode + */ + *qLen = (Ipp32u)( (MBS_SMS4-1) - ivLen); + *qLenEnc = *qLen-1; + + *tagLenEnc = (SMS4CCM_TAGLEN(pCtx)-2)>>1; + + *((Ipp64u*)payloadLen) = SMS4CCM_MSGLEN(pCtx); + + ((Ipp8u*)MAC)[0] = (Ipp8u)( (Ipp32u)((adLen!=0) <<6) + (*tagLenEnc<<3) + *qLenEnc); /* flags */ + #if (IPP_ENDIAN == IPP_LITTLE_ENDIAN) + MAC[2] = ENDIANNESS(IPP_HIDWORD(*((Ipp64u*)payloadLen))); + MAC[3] = ENDIANNESS(IPP_LODWORD(*((Ipp64u*)payloadLen))); + #else + MAC[2] = IPP_HIDWORD(*((Ipp64u*)payloadLen)); + MAC[3] = IPP_LODWORD(*((Ipp64u*)payloadLen)); + #endif + CopyBlock(pIV, ((Ipp8u*)MAC)+1, ivLen); + cpSMS4_Cipher((Ipp8u*)MAC, (Ipp8u*)MAC, SMS4_RK(pSMS4)); + + /* setup CTR0 */ + FillBlock16(0, NULL,CTR, 0); + ((Ipp8u*)CTR)[0] = (Ipp8u)(*qLenEnc); /* flags */ + CopyBlock(pIV, ((Ipp8u*)CTR)+1, ivLen); + CopyBlock16(CTR, SMS4CCM_CTR0(pCtx)); + + /* compute and store S0=ENC(CTR0) */ + cpSMS4_Cipher(SMS4CCM_S0(pCtx), (Ipp8u*)CTR, SMS4_RK(pSMS4)); + + /* + // update MAC by the AD + */ + if(adLen) { + /* encode length of associated data */ + Ipp8u* adLenEncPtr; + int adLenEncSize; + + #if (IPP_ENDIAN == IPP_LITTLE_ENDIAN) + adLenEnc[1] = ENDIANNESS(IPP_HIDWORD(adLen)); + adLenEnc[2] = ENDIANNESS(IPP_LODWORD(adLen)); + #else + adLenEnc[1] = IPP_HIDWORD(adLen); + adLenEnc[2] = IPP_LODWORD(adLen); + #endif + + if(adLen >= 0xFF00) { + adLenEncSize = 6; + #if (IPP_ENDIAN == IPP_LITTLE_ENDIAN) + adLenEnc[1] = 0xFEFFFFFF; + #else + adLenEnc[1] = 0xFFFFFFFE; + #endif + } + else { + adLenEncSize= 2; + } + adLenEncPtr = (Ipp8u*)adLenEnc+3*sizeof(Ipp32u)-adLenEncSize; + + /* prepare first formatted block of Header */ + CopyBlock(adLenEncPtr, block, adLenEncSize); + FillBlock16(0,pAD, (Ipp8u*)block+adLenEncSize, IPP_MIN((MBS_SMS4-adLenEncSize), adLen)); + + /* and update MAC */ + MAC[0] ^= block[0]; + MAC[1] ^= block[1]; + MAC[2] ^= block[2]; + MAC[3] ^= block[3]; + cpSMS4_Cipher((Ipp8u*)MAC, (Ipp8u*)MAC, SMS4_RK(pSMS4)); + + /* update MAC the by rest of addition data */ + if( (adLen+adLenEncSize) > MBS_SMS4 ) { + pAD += (MBS_SMS4-adLenEncSize); + adLen -= (MBS_SMS4-adLenEncSize); + while(adLen >= MBS_SMS4) { + CopyBlock16(pAD, block); + MAC[0] ^= block[0]; + MAC[1] ^= block[1]; + MAC[2] ^= block[2]; + MAC[3] ^= block[3]; + cpSMS4_Cipher((Ipp8u*)MAC, (Ipp8u*)MAC, SMS4_RK(pSMS4)); + + pAD += MBS_SMS4; + adLen -= MBS_SMS4; + } + + if(adLen) { + FillBlock16(0, pAD, block, (int)adLen); + MAC[0] ^= block[0]; + MAC[1] ^= block[1]; + MAC[2] ^= block[2]; + MAC[3] ^= block[3]; + cpSMS4_Cipher((Ipp8u*)MAC, (Ipp8u*)MAC, SMS4_RK(pSMS4)); + } + } + } + + SMS4CCM_COUNTER(pCtx) = 0; + CopyBlock16(MAC, SMS4CCM_MAC(pCtx)); + + /* clear secret data */ + PurgeBlock(TMP, sizeof(TMP)); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ccmtaglen.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ccmtaglen.c new file mode 100644 index 0000000..32aad50 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ccmtaglen.c @@ -0,0 +1,63 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Purpose: +// Cryptography Primitive. +// SMS4-CCM implementation. +// +// Content: +// ippsSMS4_CCMTagLen() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpsms4authccm.h" +#include "pcptool.h" + +/*F* +// Name: ippsSMS4_CCMTagLen +// +// Purpose: Setup length of the tag. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// ippStsContextMatchErr !VALID_SMS4CCM_ID() +// ippStsLengthErr MBS_SMS4 < tagLen || tagLen < 4 +// or odd value of tagLen +// ippStsNoErr no errors +// +// Parameters: +// tagLen length in bytes of the requested tag +// pCtx pointer to the CCM context +// +*F*/ +IPPFUN(IppStatus, ippsSMS4_CCMTagLen,(int tagLen, IppsSMS4_CCMState* pCtx)) +{ + /* test context */ + IPP_BAD_PTR1_RET(pCtx); + IPP_BADARG_RET(!VALID_SMS4CCM_ID(pCtx), ippStsContextMatchErr); + + /* test tag length */ + IPP_BADARG_RET(tagLen>MBS_SMS4 || tagLen<4 || tagLen&1, ippStsLengthErr); + + /* init for new message */ + SMS4CCM_TAGLEN(pCtx) = (Ipp32u)tagLen; + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_cfb_gfni.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_cfb_gfni.c new file mode 100644 index 0000000..3a16b13 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_cfb_gfni.c @@ -0,0 +1,1597 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* This software and the related documents are Intel copyrighted materials, and your use of them is governed by +* the express license under which they were provided to you ('License'). Unless the License provides otherwise, +* you may not use, modify, copy, publish, distribute, disclose or transmit this software or the related +* documents without Intel's prior written permission. +* This software and the related documents are provided as is, with no express or implied warranties, other than +* those that are expressly stated in the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 CFB decryption +// +// Contents: +// cpSMS4_CFB_gfni512() +// cpSMS4_CFB_gfni512x48() +// cpSMS4_CFB_gfni512x32() +// cpSMS4_CFB_gfni512x16() +// cpSMS4_CFB_gfni128x12() +// cpSMS4_CFB_gfni128x8() +// cpSMS4_CFB_gfni128x4() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpsms4.h" +#include "pcptool.h" + +#if (_IPP32E>=_IPP32E_K1) +#if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) + +#include "pcpsms4_gfni.h" + +static +void cpSMS4_CFB_dec_gfni512x48(Ipp8u* pDst, const Ipp8u* pSrc, int len, int cfbBlkSize, const Ipp32u* pRKey, Ipp8u* pIV); +static +void cpSMS4_CFB_dec_gfni512x32(Ipp8u* pDst, const Ipp8u* pSrc, int len, int cfbBlkSize, const Ipp32u* pRKey, Ipp8u* pIV); +static +void cpSMS4_CFB_dec_gfni512x16(Ipp8u* pDst, const Ipp8u* pSrc, int len, int cfbBlkSize, const Ipp32u* pRKey, Ipp8u* pIV); +static +void cpSMS4_CFB_dec_gfni128x12(Ipp8u* pDst, const Ipp8u* pSrc, int len, int cfbBlkSize, const Ipp32u* pRKey, Ipp8u* pIV); +static +void cpSMS4_CFB_dec_gfni128x8(Ipp8u* pDst, const Ipp8u* pSrc, int len, int cfbBlkSize, const Ipp32u* pRKey, Ipp8u* pIV); +static +void cpSMS4_CFB_dec_gfni128x4(Ipp8u* pDst, const Ipp8u* pSrc, int len, int cfbBlkSize, const Ipp32u* pRKey, Ipp8u* pIV); + +__FORCEINLINE Ipp64u broadcast_16to64(Ipp16u mask16) +{ + Ipp64u mask64 = (Ipp64u)mask16; + mask64 = (mask64 << 48) | (mask64 << 32) | (mask64 << 16) | mask64; + return mask64; +} + +__FORCEINLINE __m512i getInputBlocks(__m128i * const currentState, const __m512i * const pCipherBlocks, __mmask16 blocksCompressMask) +{ + /* extract 128-bit cipher blocks */ + __m128i c0 = _mm512_extracti64x2_epi64(*pCipherBlocks, 0); + __m128i c1 = _mm512_extracti64x2_epi64(*pCipherBlocks, 1); + __m128i c2 = _mm512_extracti64x2_epi64(*pCipherBlocks, 2); + __m128i c3 = _mm512_extracti64x2_epi64(*pCipherBlocks, 3); + + /* InpBlk_j = LSB_(b-s)(InpBlk_(j-1)) | C_(j-1); b = 128 bit, s = 8*cfbBlkSize bit */ + __m128i inpBlk0 = *currentState; + /* drop MSB bits (size = cfbBlkSize) from the inpBlk_i (by mask) and append c_(i-1) to LSB bits */ + __m128i inpBlk1 = _mm_mask_compress_epi8(c0, blocksCompressMask, inpBlk0); + __m128i inpBlk2 = _mm_mask_compress_epi8(c1, blocksCompressMask, inpBlk1); + __m128i inpBlk3 = _mm_mask_compress_epi8(c2, blocksCompressMask, inpBlk2); + + /* next InputBlock ready */ + *currentState = _mm_mask_compress_epi8(c3, blocksCompressMask, inpBlk3); + + /* inserts */ + __m512i inpBlk512 = _mm512_setzero_si512(); + inpBlk512 = _mm512_mask_broadcast_i64x2(inpBlk512, 0x03, inpBlk0); /* 0000 0011 */ + inpBlk512 = _mm512_mask_broadcast_i64x2(inpBlk512, 0x0C, inpBlk1); /* 0000 1100 */ + inpBlk512 = _mm512_mask_broadcast_i64x2(inpBlk512, 0x30, inpBlk2); /* 0011 0000 */ + inpBlk512 = _mm512_mask_broadcast_i64x2(inpBlk512, 0xC0, inpBlk3); /* 1100 0000 */ + + return inpBlk512; +} + +/* +// 64*cfbBlkSize bytes processing +*/ + +IPP_OWN_DEFN (void, cpSMS4_CFB_dec_gfni512, (Ipp8u* pDst, const Ipp8u* pSrc, int len, int cfbBlkSize, const Ipp32u* pRKey, Ipp8u* pIV)) +{ + __ALIGN16 __m512i TMP[24]; + + /* + // TMP[0..15] - decrypted text blocks + // TMP[16..19] - key + // TMP[20..23] - temp out + */ + + const int bytesPerLoad512 = 4 * cfbBlkSize; + const Ipp16u blocksCompressMask = (Ipp16u)(0xFFFF << cfbBlkSize); + + int processedLen = len - (len % (64 * cfbBlkSize)); + int blocks; + + /* load masks; allows to load 4 source blocks of cfbBlkSize each (in bytes) to LSB parts of 128-bit lanes in 512-bit register */ + __mmask64 kLsbMask64 = broadcast_16to64((Ipp16u)(0xFFFF << (16-cfbBlkSize))); + /* same mask to load in MSB parts */ + __mmask64 kMsbMask64 = broadcast_16to64(~blocksCompressMask); + + /* load IV */ + __m128i currentState = _mm_maskz_loadu_epi64(0x03 /* load 128-bit */, pIV); + + for (blocks = len / cfbBlkSize; blocks >= (64); blocks -= (64)) { + /* load cipher blocks to LSB parts of registers */ + __m512i ciphBytes0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 0 * bytesPerLoad512); + __m512i ciphBytes1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 1 * bytesPerLoad512); + __m512i ciphBytes2 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 2 * bytesPerLoad512); + __m512i ciphBytes3 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 3 * bytesPerLoad512); + + /* prepare InputBlocks for decryption */ + ciphBytes0 = getInputBlocks(¤tState, &ciphBytes0, blocksCompressMask); + ciphBytes1 = getInputBlocks(¤tState, &ciphBytes1, blocksCompressMask); + ciphBytes2 = getInputBlocks(¤tState, &ciphBytes2, blocksCompressMask); + ciphBytes3 = getInputBlocks(¤tState, &ciphBytes3, blocksCompressMask); + + ciphBytes0 = _mm512_shuffle_epi8(ciphBytes0, M512(swapBytes)); + ciphBytes1 = _mm512_shuffle_epi8(ciphBytes1, M512(swapBytes)); + ciphBytes2 = _mm512_shuffle_epi8(ciphBytes2, M512(swapBytes)); + ciphBytes3 = _mm512_shuffle_epi8(ciphBytes3, M512(swapBytes)); + TRANSPOSE_INP_512(TMP[0], TMP[1], TMP[2], TMP[3], ciphBytes0, ciphBytes1, ciphBytes2, ciphBytes3); + + /* load cipher blocks to LSB parts of registers */ + ciphBytes0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 4 * bytesPerLoad512); + ciphBytes1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 5 * bytesPerLoad512); + ciphBytes2 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 6 * bytesPerLoad512); + ciphBytes3 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 7 * bytesPerLoad512); + + /* prepare InputBlocks for decryption */ + ciphBytes0 = getInputBlocks(¤tState, &ciphBytes0, blocksCompressMask); + ciphBytes1 = getInputBlocks(¤tState, &ciphBytes1, blocksCompressMask); + ciphBytes2 = getInputBlocks(¤tState, &ciphBytes2, blocksCompressMask); + ciphBytes3 = getInputBlocks(¤tState, &ciphBytes3, blocksCompressMask); + + ciphBytes0 = _mm512_shuffle_epi8(ciphBytes0, M512(swapBytes)); + ciphBytes1 = _mm512_shuffle_epi8(ciphBytes1, M512(swapBytes)); + ciphBytes2 = _mm512_shuffle_epi8(ciphBytes2, M512(swapBytes)); + ciphBytes3 = _mm512_shuffle_epi8(ciphBytes3, M512(swapBytes)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], ciphBytes0, ciphBytes1, ciphBytes2, ciphBytes3); + + /* load cipher blocks to LSB parts of registers */ + ciphBytes0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 8 * bytesPerLoad512); + ciphBytes1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 9 * bytesPerLoad512); + ciphBytes2 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 10 * bytesPerLoad512); + ciphBytes3 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 11 * bytesPerLoad512); + + /* prepare InputBlocks for decryption */ + ciphBytes0 = getInputBlocks(¤tState, &ciphBytes0, blocksCompressMask); + ciphBytes1 = getInputBlocks(¤tState, &ciphBytes1, blocksCompressMask); + ciphBytes2 = getInputBlocks(¤tState, &ciphBytes2, blocksCompressMask); + ciphBytes3 = getInputBlocks(¤tState, &ciphBytes3, blocksCompressMask); + + ciphBytes0 = _mm512_shuffle_epi8(ciphBytes0, M512(swapBytes)); + ciphBytes1 = _mm512_shuffle_epi8(ciphBytes1, M512(swapBytes)); + ciphBytes2 = _mm512_shuffle_epi8(ciphBytes2, M512(swapBytes)); + ciphBytes3 = _mm512_shuffle_epi8(ciphBytes3, M512(swapBytes)); + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], ciphBytes0, ciphBytes1, ciphBytes2, ciphBytes3); + + /* load cipher blocks to LSB parts of registers */ + ciphBytes0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 12 * bytesPerLoad512); + ciphBytes1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 13 * bytesPerLoad512); + ciphBytes2 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 14 * bytesPerLoad512); + ciphBytes3 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 15 * bytesPerLoad512); + + /* prepare InputBlocks for decryption */ + ciphBytes0 = getInputBlocks(¤tState, &ciphBytes0, blocksCompressMask); + ciphBytes1 = getInputBlocks(¤tState, &ciphBytes1, blocksCompressMask); + ciphBytes2 = getInputBlocks(¤tState, &ciphBytes2, blocksCompressMask); + ciphBytes3 = getInputBlocks(¤tState, &ciphBytes3, blocksCompressMask); + + ciphBytes0 = _mm512_shuffle_epi8(ciphBytes0, M512(swapBytes)); + ciphBytes1 = _mm512_shuffle_epi8(ciphBytes1, M512(swapBytes)); + ciphBytes2 = _mm512_shuffle_epi8(ciphBytes2, M512(swapBytes)); + ciphBytes3 = _mm512_shuffle_epi8(ciphBytes3, M512(swapBytes)); + TRANSPOSE_INP_512(TMP[12], TMP[13], TMP[14], TMP[15], ciphBytes0, ciphBytes1, ciphBytes2, ciphBytes3); + + int itr; + for (itr = 0; itr < 8; itr++, pRKey += 4) { + /* initial xors */ + TMP[19] = TMP[18] = TMP[17] = TMP[16] = _mm512_set1_epi32((Ipp32s)pRKey[0]); + TMP[16] = _mm512_xor_si512(TMP[16], TMP[1] ); + TMP[16] = _mm512_xor_si512(TMP[16], TMP[2] ); + TMP[16] = _mm512_xor_si512(TMP[16], TMP[3] ); + TMP[17] = _mm512_xor_si512(TMP[17], TMP[5] ); + TMP[17] = _mm512_xor_si512(TMP[17], TMP[6] ); + TMP[17] = _mm512_xor_si512(TMP[17], TMP[7] ); + TMP[18] = _mm512_xor_si512(TMP[18], TMP[9] ); + TMP[18] = _mm512_xor_si512(TMP[18], TMP[10]); + TMP[18] = _mm512_xor_si512(TMP[18], TMP[11]); + TMP[19] = _mm512_xor_si512(TMP[19], TMP[13]); + TMP[19] = _mm512_xor_si512(TMP[19], TMP[14]); + TMP[19] = _mm512_xor_si512(TMP[19], TMP[15]); + /* Sbox */ + TMP[16] = sBox512(TMP[16]); + TMP[17] = sBox512(TMP[17]); + TMP[18] = sBox512(TMP[18]); + TMP[19] = sBox512(TMP[19]); + /* Sbox done, now L */ + TMP[0] = _mm512_xor_si512(_mm512_xor_si512(TMP[0], TMP[16]), L512(TMP[16])); + TMP[4] = _mm512_xor_si512(_mm512_xor_si512(TMP[4], TMP[17]), L512(TMP[17])); + TMP[8] = _mm512_xor_si512(_mm512_xor_si512(TMP[8], TMP[18]), L512(TMP[18])); + TMP[12] = _mm512_xor_si512(_mm512_xor_si512(TMP[12], TMP[19]), L512(TMP[19])); + + /* initial xors */ + TMP[19] = TMP[18] = TMP[17] = TMP[16] = _mm512_set1_epi32((Ipp32s)pRKey[1]); + TMP[16] = _mm512_xor_si512(TMP[16], TMP[2] ); + TMP[16] = _mm512_xor_si512(TMP[16], TMP[3] ); + TMP[16] = _mm512_xor_si512(TMP[16], TMP[0] ); + TMP[17] = _mm512_xor_si512(TMP[17], TMP[6] ); + TMP[17] = _mm512_xor_si512(TMP[17], TMP[7] ); + TMP[17] = _mm512_xor_si512(TMP[17], TMP[4] ); + TMP[18] = _mm512_xor_si512(TMP[18], TMP[10]); + TMP[18] = _mm512_xor_si512(TMP[18], TMP[11]); + TMP[18] = _mm512_xor_si512(TMP[18], TMP[8] ); + TMP[19] = _mm512_xor_si512(TMP[19], TMP[14]); + TMP[19] = _mm512_xor_si512(TMP[19], TMP[15]); + TMP[19] = _mm512_xor_si512(TMP[19], TMP[12]); + /* Sbox */ + TMP[16] = sBox512(TMP[16]); + TMP[17] = sBox512(TMP[17]); + TMP[18] = sBox512(TMP[18]); + TMP[19] = sBox512(TMP[19]); + /* Sbox done, now L */ + TMP[1] = _mm512_xor_si512(_mm512_xor_si512(TMP[1], TMP[16]), L512(TMP[16])); + TMP[5] = _mm512_xor_si512(_mm512_xor_si512(TMP[5], TMP[17]), L512(TMP[17])); + TMP[9] = _mm512_xor_si512(_mm512_xor_si512(TMP[9], TMP[18]), L512(TMP[18])); + TMP[13] = _mm512_xor_si512(_mm512_xor_si512(TMP[13], TMP[19]), L512(TMP[19])); + + /* initial xors */ + TMP[19] = TMP[18] = TMP[17] = TMP[16] = _mm512_set1_epi32((Ipp32s)pRKey[2]); + TMP[16] = _mm512_xor_si512(TMP[16], TMP[3] ); + TMP[16] = _mm512_xor_si512(TMP[16], TMP[0] ); + TMP[16] = _mm512_xor_si512(TMP[16], TMP[1] ); + TMP[17] = _mm512_xor_si512(TMP[17], TMP[7] ); + TMP[17] = _mm512_xor_si512(TMP[17], TMP[4] ); + TMP[17] = _mm512_xor_si512(TMP[17], TMP[5] ); + TMP[18] = _mm512_xor_si512(TMP[18], TMP[11]); + TMP[18] = _mm512_xor_si512(TMP[18], TMP[8] ); + TMP[18] = _mm512_xor_si512(TMP[18], TMP[9] ); + TMP[19] = _mm512_xor_si512(TMP[19], TMP[15]); + TMP[19] = _mm512_xor_si512(TMP[19], TMP[12]); + TMP[19] = _mm512_xor_si512(TMP[19], TMP[13]); + /* Sbox */ + TMP[16] = sBox512(TMP[16]); + TMP[17] = sBox512(TMP[17]); + TMP[18] = sBox512(TMP[18]); + TMP[19] = sBox512(TMP[19]); + /* Sbox done, now L */ + TMP[2] = _mm512_xor_si512(_mm512_xor_si512(TMP[2], TMP[16]), L512(TMP[16])); + TMP[6] = _mm512_xor_si512(_mm512_xor_si512(TMP[6], TMP[17]), L512(TMP[17])); + TMP[10] = _mm512_xor_si512(_mm512_xor_si512(TMP[10], TMP[18]), L512(TMP[18])); + TMP[14] = _mm512_xor_si512(_mm512_xor_si512(TMP[14], TMP[19]), L512(TMP[19])); + + /* initial xors */ + TMP[19] = TMP[18] = TMP[17] = TMP[16] = _mm512_set1_epi32((Ipp32s)pRKey[3]); + TMP[16] = _mm512_xor_si512(TMP[16], TMP[0] ); + TMP[16] = _mm512_xor_si512(TMP[16], TMP[1] ); + TMP[16] = _mm512_xor_si512(TMP[16], TMP[2] ); + TMP[17] = _mm512_xor_si512(TMP[17], TMP[4] ); + TMP[17] = _mm512_xor_si512(TMP[17], TMP[5] ); + TMP[17] = _mm512_xor_si512(TMP[17], TMP[6] ); + TMP[18] = _mm512_xor_si512(TMP[18], TMP[8] ); + TMP[18] = _mm512_xor_si512(TMP[18], TMP[9] ); + TMP[18] = _mm512_xor_si512(TMP[18], TMP[10]); + TMP[19] = _mm512_xor_si512(TMP[19], TMP[12]); + TMP[19] = _mm512_xor_si512(TMP[19], TMP[13]); + TMP[19] = _mm512_xor_si512(TMP[19], TMP[14]); + /* Sbox */ + TMP[16] = sBox512(TMP[16]); + TMP[17] = sBox512(TMP[17]); + TMP[18] = sBox512(TMP[18]); + TMP[19] = sBox512(TMP[19]); + /* Sbox done, now L */ + TMP[3] = _mm512_xor_si512(_mm512_xor_si512(TMP[3], TMP[16]), L512(TMP[16])); + TMP[7] = _mm512_xor_si512(_mm512_xor_si512(TMP[7], TMP[17]), L512(TMP[17])); + TMP[11] = _mm512_xor_si512(_mm512_xor_si512(TMP[11], TMP[18]), L512(TMP[18])); + TMP[15] = _mm512_xor_si512(_mm512_xor_si512(TMP[15], TMP[19]), L512(TMP[19])); + } + + pRKey -= 32; + + TRANSPOSE_OUT_512(TMP[20], TMP[21], TMP[22], TMP[23], TMP[0], TMP[1], TMP[2], TMP[3]); + TMP[20] = _mm512_shuffle_epi8(TMP[20], M512(swapBytes)); + TMP[21] = _mm512_shuffle_epi8(TMP[21], M512(swapBytes)); + TMP[22] = _mm512_shuffle_epi8(TMP[22], M512(swapBytes)); + TMP[23] = _mm512_shuffle_epi8(TMP[23], M512(swapBytes)); + + /* load cipher blocks to MSB parts of registers */ + ciphBytes0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 0 * bytesPerLoad512); + ciphBytes1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 1 * bytesPerLoad512); + ciphBytes2 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 2 * bytesPerLoad512); + ciphBytes3 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 3 * bytesPerLoad512); + + TMP[20] = _mm512_xor_si512(ciphBytes0, TMP[20]); + TMP[21] = _mm512_xor_si512(ciphBytes1, TMP[21]); + TMP[22] = _mm512_xor_si512(ciphBytes2, TMP[22]); + TMP[23] = _mm512_xor_si512(ciphBytes3, TMP[23]); + + _mm512_mask_compressstoreu_epi8(pDst + 0 * bytesPerLoad512, kMsbMask64, TMP[20]); + _mm512_mask_compressstoreu_epi8(pDst + 1 * bytesPerLoad512, kMsbMask64, TMP[21]); + _mm512_mask_compressstoreu_epi8(pDst + 2 * bytesPerLoad512, kMsbMask64, TMP[22]); + _mm512_mask_compressstoreu_epi8(pDst + 3 * bytesPerLoad512, kMsbMask64, TMP[23]); + + TRANSPOSE_OUT_512(TMP[20], TMP[21], TMP[22], TMP[23], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[20] = _mm512_shuffle_epi8(TMP[20], M512(swapBytes)); + TMP[21] = _mm512_shuffle_epi8(TMP[21], M512(swapBytes)); + TMP[22] = _mm512_shuffle_epi8(TMP[22], M512(swapBytes)); + TMP[23] = _mm512_shuffle_epi8(TMP[23], M512(swapBytes)); + + /* load cipher blocks to MSB parts of registers */ + ciphBytes0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 4 * bytesPerLoad512); + ciphBytes1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 5 * bytesPerLoad512); + ciphBytes2 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 6 * bytesPerLoad512); + ciphBytes3 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 7 * bytesPerLoad512); + + TMP[20] = _mm512_xor_si512(ciphBytes0, TMP[20]); + TMP[21] = _mm512_xor_si512(ciphBytes1, TMP[21]); + TMP[22] = _mm512_xor_si512(ciphBytes2, TMP[22]); + TMP[23] = _mm512_xor_si512(ciphBytes3, TMP[23]); + + _mm512_mask_compressstoreu_epi8(pDst + 4 * bytesPerLoad512, kMsbMask64, TMP[20]); + _mm512_mask_compressstoreu_epi8(pDst + 5 * bytesPerLoad512, kMsbMask64, TMP[21]); + _mm512_mask_compressstoreu_epi8(pDst + 6 * bytesPerLoad512, kMsbMask64, TMP[22]); + _mm512_mask_compressstoreu_epi8(pDst + 7 * bytesPerLoad512, kMsbMask64, TMP[23]); + + TRANSPOSE_OUT_512(TMP[20], TMP[21], TMP[22], TMP[23], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[20] = _mm512_shuffle_epi8(TMP[20], M512(swapBytes)); + TMP[21] = _mm512_shuffle_epi8(TMP[21], M512(swapBytes)); + TMP[22] = _mm512_shuffle_epi8(TMP[22], M512(swapBytes)); + TMP[23] = _mm512_shuffle_epi8(TMP[23], M512(swapBytes)); + + /* load cipher blocks to MSB parts of registers */ + ciphBytes0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 8 * bytesPerLoad512); + ciphBytes1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 9 * bytesPerLoad512); + ciphBytes2 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 10 * bytesPerLoad512); + ciphBytes3 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 11 * bytesPerLoad512); + + TMP[20] = _mm512_xor_si512(ciphBytes0, TMP[20]); + TMP[21] = _mm512_xor_si512(ciphBytes1, TMP[21]); + TMP[22] = _mm512_xor_si512(ciphBytes2, TMP[22]); + TMP[23] = _mm512_xor_si512(ciphBytes3, TMP[23]); + + _mm512_mask_compressstoreu_epi8(pDst + 8 * bytesPerLoad512, kMsbMask64, TMP[20]); + _mm512_mask_compressstoreu_epi8(pDst + 9 * bytesPerLoad512, kMsbMask64, TMP[21]); + _mm512_mask_compressstoreu_epi8(pDst + 10 * bytesPerLoad512, kMsbMask64, TMP[22]); + _mm512_mask_compressstoreu_epi8(pDst + 11 * bytesPerLoad512, kMsbMask64, TMP[23]); + + TRANSPOSE_OUT_512(TMP[20], TMP[21], TMP[22], TMP[23], TMP[12], TMP[13], TMP[14], TMP[15]); + TMP[20] = _mm512_shuffle_epi8(TMP[20], M512(swapBytes)); + TMP[21] = _mm512_shuffle_epi8(TMP[21], M512(swapBytes)); + TMP[22] = _mm512_shuffle_epi8(TMP[22], M512(swapBytes)); + TMP[23] = _mm512_shuffle_epi8(TMP[23], M512(swapBytes)); + + /* load cipher blocks to MSB parts of registers */ + ciphBytes0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 12 * bytesPerLoad512); + ciphBytes1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 13 * bytesPerLoad512); + ciphBytes2 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 14 * bytesPerLoad512); + ciphBytes3 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 15 * bytesPerLoad512); + + TMP[20] = _mm512_xor_si512(ciphBytes0, TMP[20]); + TMP[21] = _mm512_xor_si512(ciphBytes1, TMP[21]); + TMP[22] = _mm512_xor_si512(ciphBytes2, TMP[22]); + TMP[23] = _mm512_xor_si512(ciphBytes3, TMP[23]); + + _mm512_mask_compressstoreu_epi8(pDst + 12 * bytesPerLoad512, kMsbMask64, TMP[20]); + _mm512_mask_compressstoreu_epi8(pDst + 13 * bytesPerLoad512, kMsbMask64, TMP[21]); + _mm512_mask_compressstoreu_epi8(pDst + 14 * bytesPerLoad512, kMsbMask64, TMP[22]); + _mm512_mask_compressstoreu_epi8(pDst + 15 * bytesPerLoad512, kMsbMask64, TMP[23]); + + pSrc += 16 * bytesPerLoad512; + pDst += 16 * bytesPerLoad512; + } + + /* clear secret data */ + for(Ipp32u i = 0; i < sizeof(TMP)/sizeof(TMP[0]); i++){ + TMP[i] = _mm512_setzero_si512(); //_mm512_xor_si512(TMP[i],TMP[i]); + } + + len -= processedLen; + + if (len){ + _mm_storeu_si128((__m128i*)(pIV), currentState); + cpSMS4_CFB_dec_gfni512x48(pDst, pSrc, len, cfbBlkSize, pRKey, pIV); + } + +} + +/* +// 48*cfbBlkSize bytes processing +*/ + +static +void cpSMS4_CFB_dec_gfni512x48(Ipp8u* pDst, const Ipp8u* pSrc, int len, int cfbBlkSize, const Ipp32u* pRKey, Ipp8u* pIV) +{ + __ALIGN16 __m512i TMP[19]; + + /* + // TMP[0..11] - decrypted text blocks + // TMP[12..14] - key + // TMP[15..18] - temp out + */ + + const int bytesPerLoad512 = 4 * cfbBlkSize; + const Ipp16u blocksCompressMask = (Ipp16u)(0xFFFF << cfbBlkSize); + + int processedLen = len - (len % (48 * cfbBlkSize)); + int blocks; + + /* load masks; allows to load 4 source blocks of cfbBlkSize each (in bytes) to LSB parts of 128-bit lanes in 512-bit register */ + __mmask64 kLsbMask64 = broadcast_16to64((Ipp16u)(0xFFFF << (16-cfbBlkSize))); + /* same mask to load in MSB parts */ + __mmask64 kMsbMask64 = broadcast_16to64(~blocksCompressMask); + + /* load IV */ + __m128i currentState = _mm_maskz_loadu_epi64(0x03 /* load 128-bit */, pIV); + + for (blocks = len / cfbBlkSize; blocks >= (48); blocks -= (48)) { + /* load cipher blocks to LSB parts of registers */ + __m512i ciphBytes0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 0 * bytesPerLoad512); + __m512i ciphBytes1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 1 * bytesPerLoad512); + __m512i ciphBytes2 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 2 * bytesPerLoad512); + __m512i ciphBytes3 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 3 * bytesPerLoad512); + + /* prepare InputBlocks for decryption */ + ciphBytes0 = getInputBlocks(¤tState, &ciphBytes0, blocksCompressMask); + ciphBytes1 = getInputBlocks(¤tState, &ciphBytes1, blocksCompressMask); + ciphBytes2 = getInputBlocks(¤tState, &ciphBytes2, blocksCompressMask); + ciphBytes3 = getInputBlocks(¤tState, &ciphBytes3, blocksCompressMask); + + ciphBytes0 = _mm512_shuffle_epi8(ciphBytes0, M512(swapBytes)); + ciphBytes1 = _mm512_shuffle_epi8(ciphBytes1, M512(swapBytes)); + ciphBytes2 = _mm512_shuffle_epi8(ciphBytes2, M512(swapBytes)); + ciphBytes3 = _mm512_shuffle_epi8(ciphBytes3, M512(swapBytes)); + TRANSPOSE_INP_512(TMP[0], TMP[1], TMP[2], TMP[3], ciphBytes0, ciphBytes1, ciphBytes2, ciphBytes3); + + /* load cipher blocks to LSB parts of registers */ + ciphBytes0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 4 * bytesPerLoad512); + ciphBytes1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 5 * bytesPerLoad512); + ciphBytes2 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 6 * bytesPerLoad512); + ciphBytes3 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 7 * bytesPerLoad512); + + /* prepare InputBlocks for decryption */ + ciphBytes0 = getInputBlocks(¤tState, &ciphBytes0, blocksCompressMask); + ciphBytes1 = getInputBlocks(¤tState, &ciphBytes1, blocksCompressMask); + ciphBytes2 = getInputBlocks(¤tState, &ciphBytes2, blocksCompressMask); + ciphBytes3 = getInputBlocks(¤tState, &ciphBytes3, blocksCompressMask); + + ciphBytes0 = _mm512_shuffle_epi8(ciphBytes0, M512(swapBytes)); + ciphBytes1 = _mm512_shuffle_epi8(ciphBytes1, M512(swapBytes)); + ciphBytes2 = _mm512_shuffle_epi8(ciphBytes2, M512(swapBytes)); + ciphBytes3 = _mm512_shuffle_epi8(ciphBytes3, M512(swapBytes)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], ciphBytes0, ciphBytes1, ciphBytes2, ciphBytes3); + + /* load cipher blocks to LSB parts of registers */ + ciphBytes0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 8 * bytesPerLoad512); + ciphBytes1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 9 * bytesPerLoad512); + ciphBytes2 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 10 * bytesPerLoad512); + ciphBytes3 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 11 * bytesPerLoad512); + + /* prepare InputBlocks for decryption */ + ciphBytes0 = getInputBlocks(¤tState, &ciphBytes0, blocksCompressMask); + ciphBytes1 = getInputBlocks(¤tState, &ciphBytes1, blocksCompressMask); + ciphBytes2 = getInputBlocks(¤tState, &ciphBytes2, blocksCompressMask); + ciphBytes3 = getInputBlocks(¤tState, &ciphBytes3, blocksCompressMask); + + ciphBytes0 = _mm512_shuffle_epi8(ciphBytes0, M512(swapBytes)); + ciphBytes1 = _mm512_shuffle_epi8(ciphBytes1, M512(swapBytes)); + ciphBytes2 = _mm512_shuffle_epi8(ciphBytes2, M512(swapBytes)); + ciphBytes3 = _mm512_shuffle_epi8(ciphBytes3, M512(swapBytes)); + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], ciphBytes0, ciphBytes1, ciphBytes2, ciphBytes3); + + int itr; + for (itr = 0; itr < 8; itr++, pRKey += 4) { + /* initial xors */ + TMP[14] = TMP[13] = TMP[12] = _mm512_set1_epi32((Ipp32s)pRKey[0]); + TMP[12] = _mm512_xor_si512(TMP[12], TMP[1] ); + TMP[12] = _mm512_xor_si512(TMP[12], TMP[2] ); + TMP[12] = _mm512_xor_si512(TMP[12], TMP[3] ); + TMP[13] = _mm512_xor_si512(TMP[13], TMP[5] ); + TMP[13] = _mm512_xor_si512(TMP[13], TMP[6] ); + TMP[13] = _mm512_xor_si512(TMP[13], TMP[7] ); + TMP[14] = _mm512_xor_si512(TMP[14], TMP[9] ); + TMP[14] = _mm512_xor_si512(TMP[14], TMP[10]); + TMP[14] = _mm512_xor_si512(TMP[14], TMP[11]); + /* Sbox */ + TMP[12] = sBox512(TMP[12]); + TMP[13] = sBox512(TMP[13]); + TMP[14] = sBox512(TMP[14]); + /* Sbox done, now L */ + TMP[0] = _mm512_xor_si512(_mm512_xor_si512(TMP[0], TMP[12]), L512(TMP[12])); + TMP[4] = _mm512_xor_si512(_mm512_xor_si512(TMP[4], TMP[13]), L512(TMP[13])); + TMP[8] = _mm512_xor_si512(_mm512_xor_si512(TMP[8], TMP[14]), L512(TMP[14])); + + /* initial xors */ + TMP[14] = TMP[13] = TMP[12] = _mm512_set1_epi32((Ipp32s)pRKey[1]); + TMP[12] = _mm512_xor_si512(TMP[12], TMP[2] ); + TMP[12] = _mm512_xor_si512(TMP[12], TMP[3] ); + TMP[12] = _mm512_xor_si512(TMP[12], TMP[0] ); + TMP[13] = _mm512_xor_si512(TMP[13], TMP[6] ); + TMP[13] = _mm512_xor_si512(TMP[13], TMP[7] ); + TMP[13] = _mm512_xor_si512(TMP[13], TMP[4] ); + TMP[14] = _mm512_xor_si512(TMP[14], TMP[10]); + TMP[14] = _mm512_xor_si512(TMP[14], TMP[11]); + TMP[14] = _mm512_xor_si512(TMP[14], TMP[8] ); + /* Sbox */ + TMP[12] = sBox512(TMP[12]); + TMP[13] = sBox512(TMP[13]); + TMP[14] = sBox512(TMP[14]); + /* Sbox done, now L */ + TMP[1] = _mm512_xor_si512(_mm512_xor_si512(TMP[1], TMP[12]), L512(TMP[12])); + TMP[5] = _mm512_xor_si512(_mm512_xor_si512(TMP[5], TMP[13]), L512(TMP[13])); + TMP[9] = _mm512_xor_si512(_mm512_xor_si512(TMP[9], TMP[14]), L512(TMP[14])); + + /* initial xors */ + TMP[14] = TMP[13] = TMP[12] = _mm512_set1_epi32((Ipp32s)pRKey[2]); + TMP[12] = _mm512_xor_si512(TMP[12], TMP[3] ); + TMP[12] = _mm512_xor_si512(TMP[12], TMP[0] ); + TMP[12] = _mm512_xor_si512(TMP[12], TMP[1] ); + TMP[13] = _mm512_xor_si512(TMP[13], TMP[7] ); + TMP[13] = _mm512_xor_si512(TMP[13], TMP[4] ); + TMP[13] = _mm512_xor_si512(TMP[13], TMP[5] ); + TMP[14] = _mm512_xor_si512(TMP[14], TMP[11]); + TMP[14] = _mm512_xor_si512(TMP[14], TMP[8] ); + TMP[14] = _mm512_xor_si512(TMP[14], TMP[9] ); + /* Sbox */ + TMP[12] = sBox512(TMP[12]); + TMP[13] = sBox512(TMP[13]); + TMP[14] = sBox512(TMP[14]); + /* Sbox done, now L */ + TMP[2] = _mm512_xor_si512(_mm512_xor_si512(TMP[2], TMP[12]), L512(TMP[12])); + TMP[6] = _mm512_xor_si512(_mm512_xor_si512(TMP[6], TMP[13]), L512(TMP[13])); + TMP[10] = _mm512_xor_si512(_mm512_xor_si512(TMP[10], TMP[14]), L512(TMP[14])); + + /* initial xors */ + TMP[14] = TMP[13] = TMP[12] = _mm512_set1_epi32((Ipp32s)pRKey[3]); + TMP[12] = _mm512_xor_si512(TMP[12], TMP[0] ); + TMP[12] = _mm512_xor_si512(TMP[12], TMP[1] ); + TMP[12] = _mm512_xor_si512(TMP[12], TMP[2] ); + TMP[13] = _mm512_xor_si512(TMP[13], TMP[4] ); + TMP[13] = _mm512_xor_si512(TMP[13], TMP[5] ); + TMP[13] = _mm512_xor_si512(TMP[13], TMP[6] ); + TMP[14] = _mm512_xor_si512(TMP[14], TMP[8] ); + TMP[14] = _mm512_xor_si512(TMP[14], TMP[9] ); + TMP[14] = _mm512_xor_si512(TMP[14], TMP[10]); + /* Sbox */ + TMP[12] = sBox512(TMP[12]); + TMP[13] = sBox512(TMP[13]); + TMP[14] = sBox512(TMP[14]); + /* Sbox done, now L */ + TMP[3] = _mm512_xor_si512(_mm512_xor_si512(TMP[3], TMP[12]), L512(TMP[12])); + TMP[7] = _mm512_xor_si512(_mm512_xor_si512(TMP[7], TMP[13]), L512(TMP[13])); + TMP[11] = _mm512_xor_si512(_mm512_xor_si512(TMP[11], TMP[14]), L512(TMP[14])); + } + pRKey -= 32; + + TRANSPOSE_OUT_512(TMP[15], TMP[16], TMP[17], TMP[18], TMP[0], TMP[1], TMP[2], TMP[3]); + TMP[15] = _mm512_shuffle_epi8(TMP[15], M512(swapBytes)); + TMP[16] = _mm512_shuffle_epi8(TMP[16], M512(swapBytes)); + TMP[17] = _mm512_shuffle_epi8(TMP[17], M512(swapBytes)); + TMP[18] = _mm512_shuffle_epi8(TMP[18], M512(swapBytes)); + + /* load cipher blocks to MSB parts of registers */ + ciphBytes0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 0 * bytesPerLoad512); + ciphBytes1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 1 * bytesPerLoad512); + ciphBytes2 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 2 * bytesPerLoad512); + ciphBytes3 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 3 * bytesPerLoad512); + + TMP[15] = _mm512_xor_si512(ciphBytes0, TMP[15]); + TMP[16] = _mm512_xor_si512(ciphBytes1, TMP[16]); + TMP[17] = _mm512_xor_si512(ciphBytes2, TMP[17]); + TMP[18] = _mm512_xor_si512(ciphBytes3, TMP[18]); + + _mm512_mask_compressstoreu_epi8(pDst + 0 * bytesPerLoad512, kMsbMask64, TMP[15]); + _mm512_mask_compressstoreu_epi8(pDst + 1 * bytesPerLoad512, kMsbMask64, TMP[16]); + _mm512_mask_compressstoreu_epi8(pDst + 2 * bytesPerLoad512, kMsbMask64, TMP[17]); + _mm512_mask_compressstoreu_epi8(pDst + 3 * bytesPerLoad512, kMsbMask64, TMP[18]); + + TRANSPOSE_OUT_512(TMP[15], TMP[16], TMP[17], TMP[18], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[15] = _mm512_shuffle_epi8(TMP[15], M512(swapBytes)); + TMP[16] = _mm512_shuffle_epi8(TMP[16], M512(swapBytes)); + TMP[17] = _mm512_shuffle_epi8(TMP[17], M512(swapBytes)); + TMP[18] = _mm512_shuffle_epi8(TMP[18], M512(swapBytes)); + + /* load cipher blocks to MSB parts of registers */ + ciphBytes0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 4 * bytesPerLoad512); + ciphBytes1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 5 * bytesPerLoad512); + ciphBytes2 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 6 * bytesPerLoad512); + ciphBytes3 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 7 * bytesPerLoad512); + + TMP[15] = _mm512_xor_si512(ciphBytes0, TMP[15]); + TMP[16] = _mm512_xor_si512(ciphBytes1, TMP[16]); + TMP[17] = _mm512_xor_si512(ciphBytes2, TMP[17]); + TMP[18] = _mm512_xor_si512(ciphBytes3, TMP[18]); + + _mm512_mask_compressstoreu_epi8(pDst + 4 * bytesPerLoad512, kMsbMask64, TMP[15]); + _mm512_mask_compressstoreu_epi8(pDst + 5 * bytesPerLoad512, kMsbMask64, TMP[16]); + _mm512_mask_compressstoreu_epi8(pDst + 6 * bytesPerLoad512, kMsbMask64, TMP[17]); + _mm512_mask_compressstoreu_epi8(pDst + 7 * bytesPerLoad512, kMsbMask64, TMP[18]); + + TRANSPOSE_OUT_512(TMP[15], TMP[16], TMP[17], TMP[18], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[15] = _mm512_shuffle_epi8(TMP[15], M512(swapBytes)); + TMP[16] = _mm512_shuffle_epi8(TMP[16], M512(swapBytes)); + TMP[17] = _mm512_shuffle_epi8(TMP[17], M512(swapBytes)); + TMP[18] = _mm512_shuffle_epi8(TMP[18], M512(swapBytes)); + + /* load cipher blocks to MSB parts of registers */ + ciphBytes0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 8 * bytesPerLoad512); + ciphBytes1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 9 * bytesPerLoad512); + ciphBytes2 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 10 * bytesPerLoad512); + ciphBytes3 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 11 * bytesPerLoad512); + + TMP[15] = _mm512_xor_si512(ciphBytes0, TMP[15]); + TMP[16] = _mm512_xor_si512(ciphBytes1, TMP[16]); + TMP[17] = _mm512_xor_si512(ciphBytes2, TMP[17]); + TMP[18] = _mm512_xor_si512(ciphBytes3, TMP[18]); + + _mm512_mask_compressstoreu_epi8(pDst + 8 * bytesPerLoad512, kMsbMask64, TMP[15]); + _mm512_mask_compressstoreu_epi8(pDst + 9 * bytesPerLoad512, kMsbMask64, TMP[16]); + _mm512_mask_compressstoreu_epi8(pDst + 10 * bytesPerLoad512, kMsbMask64, TMP[17]); + _mm512_mask_compressstoreu_epi8(pDst + 11 * bytesPerLoad512, kMsbMask64, TMP[18]); + + pSrc += 12 * bytesPerLoad512; + pDst += 12 * bytesPerLoad512; + } + + /* clear secret data */ + for(Ipp32u i = 0; i < sizeof(TMP)/sizeof(TMP[0]); i++){ + TMP[i] = _mm512_setzero_si512(); //_mm512_xor_si512(TMP[i],TMP[i]); + } + + len -= processedLen; + + if (len){ + _mm_storeu_si128((__m128i*)(pIV), currentState); + cpSMS4_CFB_dec_gfni512x32(pDst, pSrc, len, cfbBlkSize, pRKey, pIV); + } + +} + +/* +// 32*cfbBlkSize bytes processing +*/ + +static +void cpSMS4_CFB_dec_gfni512x32(Ipp8u* pDst, const Ipp8u* pSrc, int len, int cfbBlkSize, const Ipp32u* pRKey, Ipp8u* pIV) +{ + __ALIGN16 __m512i TMP[15]; + + /* + // TMP[0..7] - decrypted text blocks + // TMP[8..9] - key + // TMP[10..14] - temp out + */ + + const int bytesPerLoad512 = 4 * cfbBlkSize; + const Ipp16u blocksCompressMask = (Ipp16u)(0xFFFF << cfbBlkSize); + + int processedLen = len - (len % (32 * cfbBlkSize)); + int blocks; + + /* load masks; allows to load 4 source blocks of cfbBlkSize each (in bytes) to LSB parts of 128-bit lanes in 512-bit register */ + __mmask64 kLsbMask64 = broadcast_16to64((Ipp16u)(0xFFFF << (16-cfbBlkSize))); + /* same mask to load in MSB parts */ + __mmask64 kMsbMask64 = broadcast_16to64(~blocksCompressMask); + + /* load IV */ + __m128i currentState = _mm_maskz_loadu_epi64(0x03 /* load 128-bit */, pIV); + + for (blocks = len / cfbBlkSize; blocks >= (32); blocks -= (32)) { + /* load cipher blocks to LSB parts of registers */ + __m512i ciphBytes0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 0 * bytesPerLoad512); + __m512i ciphBytes1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 1 * bytesPerLoad512); + __m512i ciphBytes2 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 2 * bytesPerLoad512); + __m512i ciphBytes3 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 3 * bytesPerLoad512); + + /* prepare InputBlocks for decryption */ + ciphBytes0 = getInputBlocks(¤tState, &ciphBytes0, blocksCompressMask); + ciphBytes1 = getInputBlocks(¤tState, &ciphBytes1, blocksCompressMask); + ciphBytes2 = getInputBlocks(¤tState, &ciphBytes2, blocksCompressMask); + ciphBytes3 = getInputBlocks(¤tState, &ciphBytes3, blocksCompressMask); + + ciphBytes0 = _mm512_shuffle_epi8(ciphBytes0, M512(swapBytes)); + ciphBytes1 = _mm512_shuffle_epi8(ciphBytes1, M512(swapBytes)); + ciphBytes2 = _mm512_shuffle_epi8(ciphBytes2, M512(swapBytes)); + ciphBytes3 = _mm512_shuffle_epi8(ciphBytes3, M512(swapBytes)); + TRANSPOSE_INP_512(TMP[0], TMP[1], TMP[2], TMP[3], ciphBytes0, ciphBytes1, ciphBytes2, ciphBytes3); + + /* load cipher blocks to LSB parts of registers */ + ciphBytes0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 4 * bytesPerLoad512); + ciphBytes1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 5 * bytesPerLoad512); + ciphBytes2 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 6 * bytesPerLoad512); + ciphBytes3 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 7 * bytesPerLoad512); + + /* prepare InputBlocks for decryption */ + ciphBytes0 = getInputBlocks(¤tState, &ciphBytes0, blocksCompressMask); + ciphBytes1 = getInputBlocks(¤tState, &ciphBytes1, blocksCompressMask); + ciphBytes2 = getInputBlocks(¤tState, &ciphBytes2, blocksCompressMask); + ciphBytes3 = getInputBlocks(¤tState, &ciphBytes3, blocksCompressMask); + + ciphBytes0 = _mm512_shuffle_epi8(ciphBytes0, M512(swapBytes)); + ciphBytes1 = _mm512_shuffle_epi8(ciphBytes1, M512(swapBytes)); + ciphBytes2 = _mm512_shuffle_epi8(ciphBytes2, M512(swapBytes)); + ciphBytes3 = _mm512_shuffle_epi8(ciphBytes3, M512(swapBytes)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], ciphBytes0, ciphBytes1, ciphBytes2, ciphBytes3); + + int itr; + for (itr = 0; itr < 8; itr++, pRKey += 4) { + /* initial xors */ + TMP[9] = TMP[8] = _mm512_set1_epi32((Ipp32s)pRKey[0]); + TMP[8] = _mm512_xor_si512(TMP[8], TMP[1] ); + TMP[8] = _mm512_xor_si512(TMP[8], TMP[2] ); + TMP[8] = _mm512_xor_si512(TMP[8], TMP[3] ); + TMP[9] = _mm512_xor_si512(TMP[9], TMP[5] ); + TMP[9] = _mm512_xor_si512(TMP[9], TMP[6] ); + TMP[9] = _mm512_xor_si512(TMP[9], TMP[7] ); + /* Sbox */ + TMP[8] = sBox512(TMP[8]); + TMP[9] = sBox512(TMP[9]); + /* Sbox done, now L */ + TMP[0] = _mm512_xor_si512(_mm512_xor_si512(TMP[0], TMP[8]), L512(TMP[8])); + TMP[4] = _mm512_xor_si512(_mm512_xor_si512(TMP[4], TMP[9]), L512(TMP[9])); + + /* initial xors */ + TMP[9] = TMP[8] = _mm512_set1_epi32((Ipp32s)pRKey[1]); + TMP[8] = _mm512_xor_si512(TMP[8], TMP[2] ); + TMP[8] = _mm512_xor_si512(TMP[8], TMP[3] ); + TMP[8] = _mm512_xor_si512(TMP[8], TMP[0] ); + TMP[9] = _mm512_xor_si512(TMP[9], TMP[6] ); + TMP[9] = _mm512_xor_si512(TMP[9], TMP[7] ); + TMP[9] = _mm512_xor_si512(TMP[9], TMP[4] ); + /* Sbox */ + TMP[8] = sBox512(TMP[8]); + TMP[9] = sBox512(TMP[9]); + /* Sbox done, now L */ + TMP[1] = _mm512_xor_si512(_mm512_xor_si512(TMP[1], TMP[8]), L512(TMP[8])); + TMP[5] = _mm512_xor_si512(_mm512_xor_si512(TMP[5], TMP[9]), L512(TMP[9])); + + /* initial xors */ + TMP[9] = TMP[8] = _mm512_set1_epi32((Ipp32s)pRKey[2]); + TMP[8] = _mm512_xor_si512(TMP[8], TMP[3] ); + TMP[8] = _mm512_xor_si512(TMP[8], TMP[0] ); + TMP[8] = _mm512_xor_si512(TMP[8], TMP[1] ); + TMP[9] = _mm512_xor_si512(TMP[9], TMP[7] ); + TMP[9] = _mm512_xor_si512(TMP[9], TMP[4] ); + TMP[9] = _mm512_xor_si512(TMP[9], TMP[5] ); + /* Sbox */ + TMP[8] = sBox512(TMP[8]); + TMP[9] = sBox512(TMP[9]); + /* Sbox done, now L */ + TMP[2] = _mm512_xor_si512(_mm512_xor_si512(TMP[2], TMP[8]), L512(TMP[8])); + TMP[6] = _mm512_xor_si512(_mm512_xor_si512(TMP[6], TMP[9]), L512(TMP[9])); + + /* initial xors */ + TMP[9] = TMP[8] = _mm512_set1_epi32((Ipp32s)pRKey[3]); + TMP[8] = _mm512_xor_si512(TMP[8], TMP[0] ); + TMP[8] = _mm512_xor_si512(TMP[8], TMP[1] ); + TMP[8] = _mm512_xor_si512(TMP[8], TMP[2] ); + TMP[9] = _mm512_xor_si512(TMP[9], TMP[4] ); + TMP[9] = _mm512_xor_si512(TMP[9], TMP[5] ); + TMP[9] = _mm512_xor_si512(TMP[9], TMP[6] ); + /* Sbox */ + TMP[8] = sBox512(TMP[8]); + TMP[9] = sBox512(TMP[9]); + /* Sbox done, now L */ + TMP[3] = _mm512_xor_si512(_mm512_xor_si512(TMP[3], TMP[8]), L512(TMP[8])); + TMP[7] = _mm512_xor_si512(_mm512_xor_si512(TMP[7], TMP[9]), L512(TMP[9])); + } + + pRKey -= 32; + + TRANSPOSE_OUT_512(TMP[10], TMP[11], TMP[12], TMP[13], TMP[0], TMP[1], TMP[2], TMP[3]); + TMP[10] = _mm512_shuffle_epi8(TMP[10], M512(swapBytes)); + TMP[11] = _mm512_shuffle_epi8(TMP[11], M512(swapBytes)); + TMP[12] = _mm512_shuffle_epi8(TMP[12], M512(swapBytes)); + TMP[13] = _mm512_shuffle_epi8(TMP[13], M512(swapBytes)); + + /* load cipher blocks to MSB parts of registers */ + ciphBytes0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 0 * bytesPerLoad512); + ciphBytes1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 1 * bytesPerLoad512); + ciphBytes2 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 2 * bytesPerLoad512); + ciphBytes3 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 3 * bytesPerLoad512); + + TMP[10] = _mm512_xor_si512(ciphBytes0, TMP[10]); + TMP[11] = _mm512_xor_si512(ciphBytes1, TMP[11]); + TMP[12] = _mm512_xor_si512(ciphBytes2, TMP[12]); + TMP[13] = _mm512_xor_si512(ciphBytes3, TMP[13]); + + _mm512_mask_compressstoreu_epi8(pDst + 0 * bytesPerLoad512, kMsbMask64, TMP[10]); + _mm512_mask_compressstoreu_epi8(pDst + 1 * bytesPerLoad512, kMsbMask64, TMP[11]); + _mm512_mask_compressstoreu_epi8(pDst + 2 * bytesPerLoad512, kMsbMask64, TMP[12]); + _mm512_mask_compressstoreu_epi8(pDst + 3 * bytesPerLoad512, kMsbMask64, TMP[13]); + + TRANSPOSE_OUT_512(TMP[10], TMP[11], TMP[12], TMP[13], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[10] = _mm512_shuffle_epi8(TMP[10], M512(swapBytes)); + TMP[11] = _mm512_shuffle_epi8(TMP[11], M512(swapBytes)); + TMP[12] = _mm512_shuffle_epi8(TMP[12], M512(swapBytes)); + TMP[13] = _mm512_shuffle_epi8(TMP[13], M512(swapBytes)); + + /* load cipher blocks to MSB parts of registers */ + ciphBytes0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 4 * bytesPerLoad512); + ciphBytes1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 5 * bytesPerLoad512); + ciphBytes2 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 6 * bytesPerLoad512); + ciphBytes3 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 7 * bytesPerLoad512); + + TMP[10] = _mm512_xor_si512(ciphBytes0, TMP[10]); + TMP[11] = _mm512_xor_si512(ciphBytes1, TMP[11]); + TMP[12] = _mm512_xor_si512(ciphBytes2, TMP[12]); + TMP[13] = _mm512_xor_si512(ciphBytes3, TMP[13]); + + _mm512_mask_compressstoreu_epi8(pDst + 4 * bytesPerLoad512, kMsbMask64, TMP[10]); + _mm512_mask_compressstoreu_epi8(pDst + 5 * bytesPerLoad512, kMsbMask64, TMP[11]); + _mm512_mask_compressstoreu_epi8(pDst + 6 * bytesPerLoad512, kMsbMask64, TMP[12]); + _mm512_mask_compressstoreu_epi8(pDst + 7 * bytesPerLoad512, kMsbMask64, TMP[13]); + + pSrc += 8 * bytesPerLoad512; + pDst += 8 * bytesPerLoad512; + + } + + /* clear secret data */ + for(Ipp32u i = 0; i < sizeof(TMP)/sizeof(TMP[0]); i++){ + TMP[i] = _mm512_setzero_si512(); //_mm512_xor_si512(TMP[i],TMP[i]); + } + + len -= processedLen; + + if (len){ + _mm_storeu_si128((__m128i*)(pIV), currentState); + cpSMS4_CFB_dec_gfni512x16(pDst, pSrc, len, cfbBlkSize, pRKey, pIV); + } + +} + +/* +// 16*cfbBlkSize bytes processing +*/ + +static +void cpSMS4_CFB_dec_gfni512x16(Ipp8u* pDst, const Ipp8u* pSrc, int len, int cfbBlkSize, const Ipp32u* pRKey, Ipp8u* pIV) +{ + __ALIGN16 __m512i TMP[9]; + + /* + // TMP[0..3] - decrypted text blocks + // TMP[4] - key + // TMP[5..8] - temp out + */ + + const int bytesPerLoad512 = 4 * cfbBlkSize; + const Ipp16u blocksCompressMask = (Ipp16u)(0xFFFF << cfbBlkSize); + + int processedLen = len - (len % (16 * cfbBlkSize)); + int blocks; + + /* load masks; allows to load 4 source blocks of cfbBlkSize each (in bytes) to LSB parts of 128-bit lanes in 512-bit register */ + __mmask64 kLsbMask64 = broadcast_16to64((Ipp16u)(0xFFFF << (16-cfbBlkSize))); + /* same mask to load in MSB parts */ + __mmask64 kMsbMask64 = broadcast_16to64(~blocksCompressMask); + + /* load IV */ + __m128i currentState = _mm_maskz_loadu_epi64(0x03 /* load 128-bit */, pIV); + + for (blocks = len / cfbBlkSize; blocks >= (16); blocks -= (16)) { + /* load cipher blocks to LSB parts of registers */ + __m512i ciphBytes0 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 0 * bytesPerLoad512); + __m512i ciphBytes1 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 1 * bytesPerLoad512); + __m512i ciphBytes2 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 2 * bytesPerLoad512); + __m512i ciphBytes3 = _mm512_maskz_expandloadu_epi8(kLsbMask64, pSrc + 3 * bytesPerLoad512); + + /* prepare InputBlocks for decryption */ + ciphBytes0 = getInputBlocks(¤tState, &ciphBytes0, blocksCompressMask); + ciphBytes1 = getInputBlocks(¤tState, &ciphBytes1, blocksCompressMask); + ciphBytes2 = getInputBlocks(¤tState, &ciphBytes2, blocksCompressMask); + ciphBytes3 = getInputBlocks(¤tState, &ciphBytes3, blocksCompressMask); + + ciphBytes0 = _mm512_shuffle_epi8(ciphBytes0, M512(swapBytes)); + ciphBytes1 = _mm512_shuffle_epi8(ciphBytes1, M512(swapBytes)); + ciphBytes2 = _mm512_shuffle_epi8(ciphBytes2, M512(swapBytes)); + ciphBytes3 = _mm512_shuffle_epi8(ciphBytes3, M512(swapBytes)); + + TRANSPOSE_INP_512(TMP[0], TMP[1], TMP[2], TMP[3], ciphBytes0, ciphBytes1, ciphBytes2, ciphBytes3); + int itr; + for (itr = 0; itr < 8; itr++, pRKey += 4) { + /* initial xors */ + TMP[4] = _mm512_set1_epi32((Ipp32s)pRKey[0]); + TMP[4] = _mm512_xor_si512(TMP[4], TMP[1]); + TMP[4] = _mm512_xor_si512(TMP[4], TMP[2]); + TMP[4] = _mm512_xor_si512(TMP[4], TMP[3]); + /* Sbox */ + TMP[4] = sBox512(TMP[4]); + /* Sbox done, now L */ + TMP[0] = _mm512_xor_si512(_mm512_xor_si512(TMP[0], TMP[4]), L512(TMP[4])); + + /* initial xors */ + TMP[4] = _mm512_set1_epi32((Ipp32s)pRKey[1]); + TMP[4] = _mm512_xor_si512(TMP[4], TMP[2]); + TMP[4] = _mm512_xor_si512(TMP[4], TMP[3]); + TMP[4] = _mm512_xor_si512(TMP[4], TMP[0]); + /* Sbox */ + TMP[4] = sBox512(TMP[4]); + /* Sbox done, now L */ + TMP[1] = _mm512_xor_si512(_mm512_xor_si512(TMP[1], TMP[4]), L512(TMP[4])); + + /* initial xors */ + TMP[4] = _mm512_set1_epi32((Ipp32s)pRKey[2]); + TMP[4] = _mm512_xor_si512(TMP[4], TMP[3]); + TMP[4] = _mm512_xor_si512(TMP[4], TMP[0]); + TMP[4] = _mm512_xor_si512(TMP[4], TMP[1]); + /* Sbox */ + TMP[4] = sBox512(TMP[4]); + /* Sbox done, now L */ + TMP[2] = _mm512_xor_si512(_mm512_xor_si512(TMP[2], TMP[4]), L512(TMP[4])); + + /* initial xors */ + TMP[4] = _mm512_set1_epi32((Ipp32s)pRKey[3]); + TMP[4] = _mm512_xor_si512(TMP[4], TMP[0]); + TMP[4] = _mm512_xor_si512(TMP[4], TMP[1]); + TMP[4] = _mm512_xor_si512(TMP[4], TMP[2]); + /* Sbox */ + TMP[4] = sBox512(TMP[4]); + /* Sbox done, now L */ + TMP[3] = _mm512_xor_si512(_mm512_xor_si512(TMP[3], TMP[4]), L512(TMP[4])); + } + + pRKey -= 32; + + TRANSPOSE_OUT_512(TMP[5], TMP[6], TMP[7], TMP[8], TMP[0], TMP[1], TMP[2], TMP[3]); + TMP[5] = _mm512_shuffle_epi8(TMP[5], M512(swapBytes)); + TMP[6] = _mm512_shuffle_epi8(TMP[6], M512(swapBytes)); + TMP[7] = _mm512_shuffle_epi8(TMP[7], M512(swapBytes)); + TMP[8] = _mm512_shuffle_epi8(TMP[8], M512(swapBytes)); + + /* load cipher blocks to MSB parts of registers */ + ciphBytes0 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 0 * bytesPerLoad512); + ciphBytes1 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 1 * bytesPerLoad512); + ciphBytes2 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 2 * bytesPerLoad512); + ciphBytes3 = _mm512_maskz_expandloadu_epi8(kMsbMask64, pSrc + 3 * bytesPerLoad512); + + TMP[5] = _mm512_xor_si512(ciphBytes0, TMP[5]); + TMP[6] = _mm512_xor_si512(ciphBytes1, TMP[6]); + TMP[7] = _mm512_xor_si512(ciphBytes2, TMP[7]); + TMP[8] = _mm512_xor_si512(ciphBytes3, TMP[8]); + + _mm512_mask_compressstoreu_epi8(pDst + 0 * bytesPerLoad512, kMsbMask64, TMP[5]); + _mm512_mask_compressstoreu_epi8(pDst + 1 * bytesPerLoad512, kMsbMask64, TMP[6]); + _mm512_mask_compressstoreu_epi8(pDst + 2 * bytesPerLoad512, kMsbMask64, TMP[7]); + _mm512_mask_compressstoreu_epi8(pDst + 3 * bytesPerLoad512, kMsbMask64, TMP[8]); + + pSrc += 4 * bytesPerLoad512; + pDst += 4 * bytesPerLoad512; + } + + /* clear secret data */ + for(Ipp32u i = 0; i < sizeof(TMP)/sizeof(TMP[0]); i++){ + TMP[i] = _mm512_setzero_si512(); //_mm512_xor_si512(TMP[i],TMP[i]); + } + + len -= processedLen; + + if (len){ + _mm_storeu_si128((__m128i*)(pIV), currentState); + cpSMS4_CFB_dec_gfni128x12(pDst, pSrc, len, cfbBlkSize, pRKey, pIV); + } + +} + +/* +// 12*cfbBlkSize bytes processing +*/ + +static +void cpSMS4_CFB_dec_gfni128x12(Ipp8u* pDst, const Ipp8u* pSrc, int len, int cfbBlkSize, const Ipp32u* pRKey, Ipp8u* pIV) +{ + __ALIGN16 __m128i TMP[15]; + + /* + // TMP[0..11] - decrypted text blocks + // TMP[12..14] - key + */ + + const Ipp16u blocksCompressMask = (Ipp16u)(0xFFFF << cfbBlkSize); + + int processedLen = len - (len % (12 * cfbBlkSize)); + int blocks; + + /* load masks; allows to load 4 source blocks of cfbBlkSize each (in bytes) to LSB parts of 128-bit lanes in 512-bit register */ + __mmask16 kLsbMask16 = (Ipp16u)(0xFFFF << (16-cfbBlkSize)); + /* same mask to load in MSB parts */ + __mmask16 kMsbMask16 = ~blocksCompressMask; + + /* load IV */ + __m128i IV128 = _mm_maskz_loadu_epi64(0x03 /* load 128-bit */, pIV); + + __m128i currentState = IV128; + + for (blocks = len / cfbBlkSize; blocks >= (12); blocks -= (12)) { + + /* load cipher blocks to LSB parts of registers */ + __m128i ciphBytes0 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 0 * cfbBlkSize); + __m128i ciphBytes1 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 1 * cfbBlkSize); + __m128i ciphBytes2 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 2 * cfbBlkSize); + __m128i ciphBytes3 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 3 * cfbBlkSize); + + /* InpBlk_j = LSB_(b-s)(InpBlk_(j-1)) | C_(j-1); b = 128 bit, s = 8*cfbBlkSize bit */ + TMP[0] = currentState; + /* drop MSB bits (size = cfbBlkSize) from the inpBlk_i (by mask) and append c_(i-1) to LSB bits */ + TMP[1] = _mm_mask_compress_epi8(ciphBytes0, blocksCompressMask, TMP[0]); + TMP[2] = _mm_mask_compress_epi8(ciphBytes1, blocksCompressMask, TMP[1]); + TMP[3] = _mm_mask_compress_epi8(ciphBytes2, blocksCompressMask, TMP[2]); + + /* next InputBlock ready */ + currentState = _mm_mask_compress_epi8(ciphBytes3, blocksCompressMask, TMP[3]); + + TMP[0] = _mm_shuffle_epi8(TMP[0], M128(swapBytes)); + TMP[1] = _mm_shuffle_epi8(TMP[1], M128(swapBytes)); + TMP[2] = _mm_shuffle_epi8(TMP[2], M128(swapBytes)); + TMP[3] = _mm_shuffle_epi8(TMP[3], M128(swapBytes)); + TRANSPOSE_INP_128(TMP[0], TMP[1], TMP[2], TMP[3], TMP[12]); /* TMP[12] - buffer */ + + /* load cipher blocks to LSB parts of registers */ + ciphBytes0 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 4 * cfbBlkSize); + ciphBytes1 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 5 * cfbBlkSize); + ciphBytes2 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 6 * cfbBlkSize); + ciphBytes3 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 7 * cfbBlkSize); + + /* InpBlk_j = LSB_(b-s)(InpBlk_(j-1)) | C_(j-1); b = 128 bit, s = 8*cfbBlkSize bit */ + TMP[4] = currentState; + /* drop MSB bits (size = cfbBlkSize) from the inpBlk_i (by mask) and append c_(i-1) to LSB bits */ + TMP[5] = _mm_mask_compress_epi8(ciphBytes0, blocksCompressMask, TMP[4]); + TMP[6] = _mm_mask_compress_epi8(ciphBytes1, blocksCompressMask, TMP[5]); + TMP[7] = _mm_mask_compress_epi8(ciphBytes2, blocksCompressMask, TMP[6]); + + /* next InputBlock ready */ + currentState = _mm_mask_compress_epi8(ciphBytes3, blocksCompressMask, TMP[7]); + + TMP[4] = _mm_shuffle_epi8(TMP[4], M128(swapBytes)); + TMP[5] = _mm_shuffle_epi8(TMP[5], M128(swapBytes)); + TMP[6] = _mm_shuffle_epi8(TMP[6], M128(swapBytes)); + TMP[7] = _mm_shuffle_epi8(TMP[7], M128(swapBytes)); + TRANSPOSE_INP_128(TMP[4], TMP[5], TMP[6], TMP[7], TMP[12]); /* TMP[12] - buffer */ + + /* load cipher blocks to LSB parts of registers */ + ciphBytes0 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 8 * cfbBlkSize); + ciphBytes1 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 9 * cfbBlkSize); + ciphBytes2 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 10 * cfbBlkSize); + ciphBytes3 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 11 * cfbBlkSize); + + /* InpBlk_j = LSB_(b-s)(InpBlk_(j-1)) | C_(j-1); b = 128 bit, s = 8*cfbBlkSize bit */ + TMP[8] = currentState; + /* drop MSB bits (size = cfbBlkSize) from the inpBlk_i (by mask) and append c_(i-1) to LSB bits */ + TMP[9] = _mm_mask_compress_epi8(ciphBytes0, blocksCompressMask, TMP[8] ); + TMP[10] = _mm_mask_compress_epi8(ciphBytes1, blocksCompressMask, TMP[9] ); + TMP[11] = _mm_mask_compress_epi8(ciphBytes2, blocksCompressMask, TMP[10]); + + /* next InputBlock ready */ + currentState = _mm_mask_compress_epi8(ciphBytes3, blocksCompressMask, TMP[11]); + + TMP[8] = _mm_shuffle_epi8(TMP[8], M128(swapBytes)); + TMP[9] = _mm_shuffle_epi8(TMP[9], M128(swapBytes)); + TMP[10] = _mm_shuffle_epi8(TMP[10], M128(swapBytes)); + TMP[11] = _mm_shuffle_epi8(TMP[11], M128(swapBytes)); + TRANSPOSE_INP_128(TMP[8], TMP[9], TMP[10], TMP[11], TMP[12]); /* TMP[12] - buffer */ + + + int itr; + for(itr=0; itr<8; itr++, pRKey+=4) { + /* initial xors */ + TMP[13] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[0]), 0); + TMP[14] = TMP[13]; + TMP[12] = TMP[13]; + TMP[12] = _mm_xor_si128(TMP[12], TMP[1] ); + TMP[12] = _mm_xor_si128(TMP[12], TMP[2] ); + TMP[12] = _mm_xor_si128(TMP[12], TMP[3] ); + TMP[13] = _mm_xor_si128(TMP[13], TMP[5] ); + TMP[13] = _mm_xor_si128(TMP[13], TMP[6] ); + TMP[13] = _mm_xor_si128(TMP[13], TMP[7] ); + TMP[14] = _mm_xor_si128(TMP[14], TMP[9] ); + TMP[14] = _mm_xor_si128(TMP[14], TMP[10]); + TMP[14] = _mm_xor_si128(TMP[14], TMP[11]); + /* Sbox */ + TMP[12] = sBox128(TMP[12]); + TMP[13] = sBox128(TMP[13]); + TMP[14] = sBox128(TMP[14]); + /* Sbox done, now L */ + TMP[0] = _mm_xor_si128(_mm_xor_si128(TMP[0], TMP[12]), L128(TMP[12])); + TMP[4] = _mm_xor_si128(_mm_xor_si128(TMP[4], TMP[13]), L128(TMP[13])); + TMP[8] = _mm_xor_si128(_mm_xor_si128(TMP[8], TMP[14]), L128(TMP[14])); + + /* initial xors */ + TMP[13] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[1]), 0); + TMP[14] = TMP[13]; + TMP[12] = TMP[13]; + TMP[12] = _mm_xor_si128(TMP[12], TMP[2] ); + TMP[12] = _mm_xor_si128(TMP[12], TMP[3] ); + TMP[12] = _mm_xor_si128(TMP[12], TMP[0] ); + TMP[13] = _mm_xor_si128(TMP[13], TMP[6] ); + TMP[13] = _mm_xor_si128(TMP[13], TMP[7] ); + TMP[13] = _mm_xor_si128(TMP[13], TMP[4] ); + TMP[14] = _mm_xor_si128(TMP[14], TMP[10]); + TMP[14] = _mm_xor_si128(TMP[14], TMP[11]); + TMP[14] = _mm_xor_si128(TMP[14], TMP[8] ); + /* Sbox */ + TMP[12] = sBox128(TMP[12]); + TMP[13] = sBox128(TMP[13]); + TMP[14] = sBox128(TMP[14]); + /* Sbox done, now L */ + TMP[1] = _mm_xor_si128(_mm_xor_si128(TMP[1], TMP[12]), L128(TMP[12])); + TMP[5] = _mm_xor_si128(_mm_xor_si128(TMP[5], TMP[13]), L128(TMP[13])); + TMP[9] = _mm_xor_si128(_mm_xor_si128(TMP[9], TMP[14]), L128(TMP[14])); + + /* initial xors */ + TMP[13] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[2]), 0); + TMP[14] = TMP[13]; + TMP[12] = TMP[13]; + TMP[12] = _mm_xor_si128(TMP[12], TMP[3] ); + TMP[12] = _mm_xor_si128(TMP[12], TMP[0] ); + TMP[12] = _mm_xor_si128(TMP[12], TMP[1] ); + TMP[13] = _mm_xor_si128(TMP[13], TMP[7] ); + TMP[13] = _mm_xor_si128(TMP[13], TMP[4] ); + TMP[13] = _mm_xor_si128(TMP[13], TMP[5] ); + TMP[14] = _mm_xor_si128(TMP[14], TMP[11]); + TMP[14] = _mm_xor_si128(TMP[14], TMP[8] ); + TMP[14] = _mm_xor_si128(TMP[14], TMP[9] ); + /* Sbox */ + TMP[12] = sBox128(TMP[12]); + TMP[13] = sBox128(TMP[13]); + TMP[14] = sBox128(TMP[14]); + /* Sbox done, now L */ + TMP[2] = _mm_xor_si128(_mm_xor_si128(TMP[2], TMP[12]), L128(TMP[12])); + TMP[6] = _mm_xor_si128(_mm_xor_si128(TMP[6], TMP[13]), L128(TMP[13])); + TMP[10] = _mm_xor_si128(_mm_xor_si128(TMP[10], TMP[14]), L128(TMP[14])); + + /* initial xors */ + TMP[13] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[3]), 0); + TMP[14] = TMP[13]; + TMP[12] = TMP[13]; + TMP[12] = _mm_xor_si128(TMP[12], TMP[0] ); + TMP[12] = _mm_xor_si128(TMP[12], TMP[1] ); + TMP[12] = _mm_xor_si128(TMP[12], TMP[2] ); + TMP[13] = _mm_xor_si128(TMP[13], TMP[4] ); + TMP[13] = _mm_xor_si128(TMP[13], TMP[5] ); + TMP[13] = _mm_xor_si128(TMP[13], TMP[6] ); + TMP[14] = _mm_xor_si128(TMP[14], TMP[8] ); + TMP[14] = _mm_xor_si128(TMP[14], TMP[9] ); + TMP[14] = _mm_xor_si128(TMP[14], TMP[10]); + /* Sbox */ + TMP[12] = sBox128(TMP[12]); + TMP[13] = sBox128(TMP[13]); + TMP[14] = sBox128(TMP[14]); + /* Sbox done, now L */ + TMP[3] = _mm_xor_si128(_mm_xor_si128(TMP[3], TMP[12]), L128(TMP[12])); + TMP[7] = _mm_xor_si128(_mm_xor_si128(TMP[7], TMP[13]), L128(TMP[13])); + TMP[11] = _mm_xor_si128(_mm_xor_si128(TMP[11], TMP[14]), L128(TMP[14])); + } + + pRKey -= 32; + + TRANSPOSE_OUT_128(TMP[0], TMP[1], TMP[2], TMP[3], TMP[12]); /* TMP[12] - buffer */ + /* Order of blocks is inverted */ + + TMP[3] = _mm_shuffle_epi8(TMP[3], M128(swapBytes)); + TMP[2] = _mm_shuffle_epi8(TMP[2], M128(swapBytes)); + TMP[1] = _mm_shuffle_epi8(TMP[1], M128(swapBytes)); + TMP[0] = _mm_shuffle_epi8(TMP[0], M128(swapBytes)); + + ciphBytes0 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 0 * cfbBlkSize); + ciphBytes1 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 1 * cfbBlkSize); + ciphBytes2 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 2 * cfbBlkSize); + ciphBytes3 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 3 * cfbBlkSize); + + TMP[3] = _mm_xor_si128(ciphBytes0, TMP[3]); + TMP[2] = _mm_xor_si128(ciphBytes1, TMP[2]); + TMP[1] = _mm_xor_si128(ciphBytes2, TMP[1]); + TMP[0] = _mm_xor_si128(ciphBytes3, TMP[0]); + + _mm_mask_compressstoreu_epi8(pDst + 0 * cfbBlkSize, kMsbMask16, TMP[3]); + _mm_mask_compressstoreu_epi8(pDst + 1 * cfbBlkSize, kMsbMask16, TMP[2]); + _mm_mask_compressstoreu_epi8(pDst + 2 * cfbBlkSize, kMsbMask16, TMP[1]); + _mm_mask_compressstoreu_epi8(pDst + 3 * cfbBlkSize, kMsbMask16, TMP[0]); + + TRANSPOSE_OUT_128(TMP[4], TMP[5], TMP[6], TMP[7], TMP[12]); /* TMP[12] - buffer */ + /* Order of blocks is inverted */ + + TMP[7] = _mm_shuffle_epi8(TMP[7], M128(swapBytes)); + TMP[6] = _mm_shuffle_epi8(TMP[6], M128(swapBytes)); + TMP[5] = _mm_shuffle_epi8(TMP[5], M128(swapBytes)); + TMP[4] = _mm_shuffle_epi8(TMP[4], M128(swapBytes)); + + ciphBytes0 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 4 * cfbBlkSize); + ciphBytes1 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 5 * cfbBlkSize); + ciphBytes2 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 6 * cfbBlkSize); + ciphBytes3 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 7 * cfbBlkSize); + + TMP[7] = _mm_xor_si128(ciphBytes0, TMP[7]); + TMP[6] = _mm_xor_si128(ciphBytes1, TMP[6]); + TMP[5] = _mm_xor_si128(ciphBytes2, TMP[5]); + TMP[4] = _mm_xor_si128(ciphBytes3, TMP[4]); + + _mm_mask_compressstoreu_epi8(pDst + 4 * cfbBlkSize, kMsbMask16, TMP[7]); + _mm_mask_compressstoreu_epi8(pDst + 5 * cfbBlkSize, kMsbMask16, TMP[6]); + _mm_mask_compressstoreu_epi8(pDst + 6 * cfbBlkSize, kMsbMask16, TMP[5]); + _mm_mask_compressstoreu_epi8(pDst + 7 * cfbBlkSize, kMsbMask16, TMP[4]); + + TRANSPOSE_OUT_128(TMP[8], TMP[9], TMP[10], TMP[11], TMP[12]); /* TMP[12] - buffer */ + /* Order of blocks is inverted */ + + TMP[11] = _mm_shuffle_epi8(TMP[11], M128(swapBytes)); + TMP[10] = _mm_shuffle_epi8(TMP[10], M128(swapBytes)); + TMP[9] = _mm_shuffle_epi8(TMP[9], M128(swapBytes)); + TMP[8] = _mm_shuffle_epi8(TMP[8], M128(swapBytes)); + + ciphBytes0 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 8 * cfbBlkSize); + ciphBytes1 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 9 * cfbBlkSize); + ciphBytes2 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 10 * cfbBlkSize); + ciphBytes3 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 11 * cfbBlkSize); + + TMP[11] = _mm_xor_si128(ciphBytes0, TMP[11]); + TMP[10] = _mm_xor_si128(ciphBytes1, TMP[10]); + TMP[9] = _mm_xor_si128(ciphBytes2, TMP[9]); + TMP[8] = _mm_xor_si128(ciphBytes3, TMP[8]); + + _mm_mask_compressstoreu_epi8(pDst + 8 * cfbBlkSize, kMsbMask16, TMP[11]); + _mm_mask_compressstoreu_epi8(pDst + 9 * cfbBlkSize, kMsbMask16, TMP[10]); + _mm_mask_compressstoreu_epi8(pDst + 10 * cfbBlkSize, kMsbMask16, TMP[9]); + _mm_mask_compressstoreu_epi8(pDst + 11 * cfbBlkSize, kMsbMask16, TMP[8]); + + pSrc += 12 * cfbBlkSize; + pDst += 12 * cfbBlkSize; + } + + /* clear secret data */ + for(Ipp32u i = 0; i < sizeof(TMP)/sizeof(TMP[0]); i++){ + TMP[i] = _mm_setzero_si128(); //_mm_xor_si128(TMP[i],TMP[i]); + } + + len -= processedLen; + + if (len){ + _mm_storeu_si128((__m128i*)(pIV), currentState); + cpSMS4_CFB_dec_gfni128x8(pDst, pSrc, len, cfbBlkSize, pRKey, pIV); + } +} + +/* +// 8*cfbBlkSize bytes processing +*/ + +static +void cpSMS4_CFB_dec_gfni128x8(Ipp8u* pDst, const Ipp8u* pSrc, int len, int cfbBlkSize, const Ipp32u* pRKey, Ipp8u* pIV) +{ + + __ALIGN16 __m128i TMP[10]; + + /* + // TMP[0..7] - decrypted text blocks + // TMP[8..9] - key + */ + + const Ipp16u blocksCompressMask = (Ipp16u)(0xFFFF << cfbBlkSize); + + int processedLen = len - (len % (8 * cfbBlkSize)); + int blocks; + + /* load masks; allows to load 4 source blocks of cfbBlkSize each (in bytes) to LSB parts of 128-bit lanes in 512-bit register */ + __mmask16 kLsbMask16 = (Ipp16u)(0xFFFF << (16-cfbBlkSize)); + /* same mask to load in MSB parts */ + __mmask16 kMsbMask16 = ~blocksCompressMask; + + /* load IV */ + __m128i currentState = _mm_maskz_loadu_epi64(0x03 /* load 128-bit */, pIV); + + for (blocks = len / cfbBlkSize; blocks >= (8); blocks -= (8)) { + + /* load cipher blocks to LSB parts of registers */ + __m128i ciphBytes0 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 0 * cfbBlkSize); + __m128i ciphBytes1 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 1 * cfbBlkSize); + __m128i ciphBytes2 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 2 * cfbBlkSize); + __m128i ciphBytes3 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 3 * cfbBlkSize); + + /* InpBlk_j = LSB_(b-s)(InpBlk_(j-1)) | C_(j-1); b = 128 bit, s = 8*cfbBlkSize bit */ + TMP[0] = currentState; + /* drop MSB bits (size = cfbBlkSize) from the inpBlk_i (by mask) and append c_(i-1) to LSB bits */ + TMP[1] = _mm_mask_compress_epi8(ciphBytes0, blocksCompressMask, TMP[0]); + TMP[2] = _mm_mask_compress_epi8(ciphBytes1, blocksCompressMask, TMP[1]); + TMP[3] = _mm_mask_compress_epi8(ciphBytes2, blocksCompressMask, TMP[2]); + + /* next InputBlock ready */ + currentState = _mm_mask_compress_epi8(ciphBytes3, blocksCompressMask, TMP[3]); + + TMP[0] = _mm_shuffle_epi8(TMP[0], M128(swapBytes)); + TMP[1] = _mm_shuffle_epi8(TMP[1], M128(swapBytes)); + TMP[2] = _mm_shuffle_epi8(TMP[2], M128(swapBytes)); + TMP[3] = _mm_shuffle_epi8(TMP[3], M128(swapBytes)); + TRANSPOSE_INP_128(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8]); /* TMP[8] - buffer */ + + /* load cipher blocks to LSB parts of registers */ + ciphBytes0 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 4 * cfbBlkSize); + ciphBytes1 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 5 * cfbBlkSize); + ciphBytes2 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 6 * cfbBlkSize); + ciphBytes3 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 7 * cfbBlkSize); + + /* InpBlk_j = LSB_(b-s)(InpBlk_(j-1)) | C_(j-1); b = 128 bit, s = 8*cfbBlkSize bit */ + TMP[4] = currentState; + /* drop MSB bits (size = cfbBlkSize) from the inpBlk_i (by mask) and append c_(i-1) to LSB bits */ + TMP[5] = _mm_mask_compress_epi8(ciphBytes0, blocksCompressMask, TMP[4]); + TMP[6] = _mm_mask_compress_epi8(ciphBytes1, blocksCompressMask, TMP[5]); + TMP[7] = _mm_mask_compress_epi8(ciphBytes2, blocksCompressMask, TMP[6]); + + /* next InputBlock ready */ + currentState = _mm_mask_compress_epi8(ciphBytes3, blocksCompressMask, TMP[7]); + + TMP[4] = _mm_shuffle_epi8(TMP[4], M128(swapBytes)); + TMP[5] = _mm_shuffle_epi8(TMP[5], M128(swapBytes)); + TMP[6] = _mm_shuffle_epi8(TMP[6], M128(swapBytes)); + TMP[7] = _mm_shuffle_epi8(TMP[7], M128(swapBytes)); + TRANSPOSE_INP_128(TMP[4], TMP[5], TMP[6], TMP[7], TMP[8]); /* TMP[8] - buffer */ + + int itr; + for(itr=0; itr<8; itr++, pRKey+=4) { + /* initial xors */ + TMP[9] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[0]), 0); + TMP[8] = TMP[9]; + TMP[8] = _mm_xor_si128(TMP[8], TMP[1] ); + TMP[8] = _mm_xor_si128(TMP[8], TMP[2] ); + TMP[8] = _mm_xor_si128(TMP[8], TMP[3] ); + TMP[9] = _mm_xor_si128(TMP[9], TMP[5] ); + TMP[9] = _mm_xor_si128(TMP[9], TMP[6] ); + TMP[9] = _mm_xor_si128(TMP[9], TMP[7] ); + /* Sbox */ + TMP[8] = sBox128(TMP[8]); + TMP[9] = sBox128(TMP[9]); + /* Sbox done, now L */ + TMP[0] = _mm_xor_si128(_mm_xor_si128(TMP[0], TMP[8]), L128(TMP[8])); + TMP[4] = _mm_xor_si128(_mm_xor_si128(TMP[4], TMP[9]), L128(TMP[9])); + + /* initial xors */ + TMP[9] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[1]), 0); + TMP[8] = TMP[9]; + TMP[8] = _mm_xor_si128(TMP[8], TMP[2] ); + TMP[8] = _mm_xor_si128(TMP[8], TMP[3] ); + TMP[8] = _mm_xor_si128(TMP[8], TMP[0] ); + TMP[9] = _mm_xor_si128(TMP[9], TMP[6] ); + TMP[9] = _mm_xor_si128(TMP[9], TMP[7] ); + TMP[9] = _mm_xor_si128(TMP[9], TMP[4] ); + /* Sbox */ + TMP[8] = sBox128(TMP[8]); + TMP[9] = sBox128(TMP[9]); + /* Sbox done, now L */ + TMP[1] = _mm_xor_si128(_mm_xor_si128(TMP[1], TMP[8]), L128(TMP[8])); + TMP[5] = _mm_xor_si128(_mm_xor_si128(TMP[5], TMP[9]), L128(TMP[9])); + + /* initial xors */ + TMP[9] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[2]), 0); + TMP[8] = TMP[9]; + TMP[8] = _mm_xor_si128(TMP[8], TMP[3] ); + TMP[8] = _mm_xor_si128(TMP[8], TMP[0] ); + TMP[8] = _mm_xor_si128(TMP[8], TMP[1] ); + TMP[9] = _mm_xor_si128(TMP[9], TMP[7] ); + TMP[9] = _mm_xor_si128(TMP[9], TMP[4] ); + TMP[9] = _mm_xor_si128(TMP[9], TMP[5] ); + /* Sbox */ + TMP[8] = sBox128(TMP[8]); + TMP[9] = sBox128(TMP[9]); + /* Sbox done, now L */ + TMP[2] = _mm_xor_si128(_mm_xor_si128(TMP[2], TMP[8]), L128(TMP[8])); + TMP[6] = _mm_xor_si128(_mm_xor_si128(TMP[6], TMP[9]), L128(TMP[9])); + + /* initial xors */ + TMP[9] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[3]), 0); + TMP[8] = TMP[9]; + TMP[8] = _mm_xor_si128(TMP[8], TMP[0] ); + TMP[8] = _mm_xor_si128(TMP[8], TMP[1] ); + TMP[8] = _mm_xor_si128(TMP[8], TMP[2] ); + TMP[9] = _mm_xor_si128(TMP[9], TMP[4] ); + TMP[9] = _mm_xor_si128(TMP[9], TMP[5] ); + TMP[9] = _mm_xor_si128(TMP[9], TMP[6] ); + /* Sbox */ + TMP[8] = sBox128(TMP[8]); + TMP[9] = sBox128(TMP[9]); + /* Sbox done, now L */ + TMP[3] = _mm_xor_si128(_mm_xor_si128(TMP[3], TMP[8]), L128(TMP[8])); + TMP[7] = _mm_xor_si128(_mm_xor_si128(TMP[7], TMP[9]), L128(TMP[9])); + } + + pRKey -= 32; + + TRANSPOSE_OUT_128(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8]); /* TMP[8] - buffer */ + /* Order of blocks is inverted */ + + TMP[3] = _mm_shuffle_epi8(TMP[3], M128(swapBytes)); + TMP[2] = _mm_shuffle_epi8(TMP[2], M128(swapBytes)); + TMP[1] = _mm_shuffle_epi8(TMP[1], M128(swapBytes)); + TMP[0] = _mm_shuffle_epi8(TMP[0], M128(swapBytes)); + + ciphBytes0 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 0 * cfbBlkSize); + ciphBytes1 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 1 * cfbBlkSize); + ciphBytes2 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 2 * cfbBlkSize); + ciphBytes3 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 3 * cfbBlkSize); + + TMP[3] = _mm_xor_si128(ciphBytes0, TMP[3]); + TMP[2] = _mm_xor_si128(ciphBytes1, TMP[2]); + TMP[1] = _mm_xor_si128(ciphBytes2, TMP[1]); + TMP[0] = _mm_xor_si128(ciphBytes3, TMP[0]); + + _mm_mask_compressstoreu_epi8(pDst + 0 * cfbBlkSize, kMsbMask16, TMP[3]); + _mm_mask_compressstoreu_epi8(pDst + 1 * cfbBlkSize, kMsbMask16, TMP[2]); + _mm_mask_compressstoreu_epi8(pDst + 2 * cfbBlkSize, kMsbMask16, TMP[1]); + _mm_mask_compressstoreu_epi8(pDst + 3 * cfbBlkSize, kMsbMask16, TMP[0]); + + TRANSPOSE_OUT_128(TMP[4], TMP[5], TMP[6], TMP[7], TMP[8]); /* TMP[8] - buffer */ + /* Order of blocks is inverted */ + + TMP[7] = _mm_shuffle_epi8(TMP[7], M128(swapBytes)); + TMP[6] = _mm_shuffle_epi8(TMP[6], M128(swapBytes)); + TMP[5] = _mm_shuffle_epi8(TMP[5], M128(swapBytes)); + TMP[4] = _mm_shuffle_epi8(TMP[4], M128(swapBytes)); + + ciphBytes0 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 4 * cfbBlkSize); + ciphBytes1 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 5 * cfbBlkSize); + ciphBytes2 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 6 * cfbBlkSize); + ciphBytes3 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 7 * cfbBlkSize); + + TMP[7] = _mm_xor_si128(ciphBytes0, TMP[7]); + TMP[6] = _mm_xor_si128(ciphBytes1, TMP[6]); + TMP[5] = _mm_xor_si128(ciphBytes2, TMP[5]); + TMP[4] = _mm_xor_si128(ciphBytes3, TMP[4]); + + _mm_mask_compressstoreu_epi8(pDst + 4 * cfbBlkSize, kMsbMask16, TMP[7]); + _mm_mask_compressstoreu_epi8(pDst + 5 * cfbBlkSize, kMsbMask16, TMP[6]); + _mm_mask_compressstoreu_epi8(pDst + 6 * cfbBlkSize, kMsbMask16, TMP[5]); + _mm_mask_compressstoreu_epi8(pDst + 7 * cfbBlkSize, kMsbMask16, TMP[4]); + + pSrc += 8 * cfbBlkSize; + pDst += 8 * cfbBlkSize; + } + + /* clear secret data */ + for(Ipp32u i = 0; i < sizeof(TMP)/sizeof(TMP[0]); i++){ + TMP[i] = _mm_setzero_si128(); //_mm_xor_si128(TMP[i],TMP[i]); + } + + len -= processedLen; + + if (len){ + _mm_storeu_si128((__m128i*)(pIV), currentState); + cpSMS4_CFB_dec_gfni128x4(pDst, pSrc, len, cfbBlkSize, pRKey, pIV); + } +} + +/* +// 4*cfbBlkSize bytes processing +*/ + +static +void cpSMS4_CFB_dec_gfni128x4(Ipp8u* pDst, const Ipp8u* pSrc, int len, int cfbBlkSize, const Ipp32u* pRKey, Ipp8u* pIV) +{ + __ALIGN16 __m128i TMP[5]; + + /* + // TMP[0..3] - decrypted text blocks + // TMP[4] - key + */ + + const Ipp16u blocksCompressMask = (Ipp16u)(0xFFFF << cfbBlkSize); + + int processedLen = len - (len % (4 * cfbBlkSize)); + int blocks; + + /* load masks; allows to load 4 source blocks of cfbBlkSize each (in bytes) to LSB parts of 128-bit lanes in 512-bit register */ + __mmask16 kLsbMask16 = (Ipp16u)(0xFFFF << (16-cfbBlkSize)); + /* same mask to load in MSB parts */ + __mmask16 kMsbMask16 = ~blocksCompressMask; + + /* load IV */ + __m128i currentState = _mm_maskz_loadu_epi64(0x03 /* load 128-bit */, pIV); + + for (blocks = len / cfbBlkSize; blocks >= (4); blocks -= (4)) { + + /* load cipher blocks to LSB parts of registers */ + __m128i ciphBytes0 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 0 * cfbBlkSize); + __m128i ciphBytes1 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 1 * cfbBlkSize); + __m128i ciphBytes2 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 2 * cfbBlkSize); + __m128i ciphBytes3 = _mm_maskz_expandloadu_epi8(kLsbMask16, pSrc + 3 * cfbBlkSize); + + /* InpBlk_j = LSB_(b-s)(InpBlk_(j-1)) | C_(j-1); b = 128 bit, s = 8*cfbBlkSize bit */ + TMP[0] = currentState; + /* drop MSB bits (size = cfbBlkSize) from the inpBlk_i (by mask) and append c_(i-1) to LSB bits */ + TMP[1] = _mm_mask_compress_epi8(ciphBytes0, blocksCompressMask, TMP[0]); + TMP[2] = _mm_mask_compress_epi8(ciphBytes1, blocksCompressMask, TMP[1]); + TMP[3] = _mm_mask_compress_epi8(ciphBytes2, blocksCompressMask, TMP[2]); + + /* next InputBlock ready */ + currentState = _mm_mask_compress_epi8(ciphBytes3, blocksCompressMask, TMP[3]); + + TMP[0] = _mm_shuffle_epi8(TMP[0], M128(swapBytes)); + TMP[1] = _mm_shuffle_epi8(TMP[1], M128(swapBytes)); + TMP[2] = _mm_shuffle_epi8(TMP[2], M128(swapBytes)); + TMP[3] = _mm_shuffle_epi8(TMP[3], M128(swapBytes)); + TRANSPOSE_INP_128(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4]); /* TMP[4] - buffer */ + + int itr; + for(itr=0; itr<8; itr++, pRKey+=4) { + /* initial xors */ + TMP[4] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[0]), 0); + TMP[4] = _mm_xor_si128(TMP[4], TMP[1] ); + TMP[4] = _mm_xor_si128(TMP[4], TMP[2] ); + TMP[4] = _mm_xor_si128(TMP[4], TMP[3] ); + /* Sbox */ + TMP[4] = sBox128(TMP[4]); + /* Sbox done, now L */ + TMP[0] = _mm_xor_si128(_mm_xor_si128(TMP[0], TMP[4]), L128(TMP[4])); + + /* initial xors */ + TMP[4] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[1]), 0); + TMP[4] = _mm_xor_si128(TMP[4], TMP[2] ); + TMP[4] = _mm_xor_si128(TMP[4], TMP[3] ); + TMP[4] = _mm_xor_si128(TMP[4], TMP[0] ); + /* Sbox */ + TMP[4] = sBox128(TMP[4]); + /* Sbox done, now L */ + TMP[1] = _mm_xor_si128(_mm_xor_si128(TMP[1], TMP[4]), L128(TMP[4])); + + /* initial xors */ + TMP[4] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[2]), 0); + TMP[4] = _mm_xor_si128(TMP[4], TMP[3] ); + TMP[4] = _mm_xor_si128(TMP[4], TMP[0] ); + TMP[4] = _mm_xor_si128(TMP[4], TMP[1] ); + /* Sbox */ + TMP[4] = sBox128(TMP[4]); + /* Sbox done, now L */ + TMP[2] = _mm_xor_si128(_mm_xor_si128(TMP[2], TMP[4]), L128(TMP[4])); + + /* initial xors */ + TMP[4] = _mm_shuffle_epi32(_mm_cvtsi32_si128((Ipp32s)pRKey[3]), 0); + TMP[4] = _mm_xor_si128(TMP[4], TMP[0] ); + TMP[4] = _mm_xor_si128(TMP[4], TMP[1] ); + TMP[4] = _mm_xor_si128(TMP[4], TMP[2] ); + /* Sbox */ + TMP[4] = sBox128(TMP[4]); + /* Sbox done, now L */ + TMP[3] = _mm_xor_si128(_mm_xor_si128(TMP[3], TMP[4]), L128(TMP[4])); + } + + pRKey -= 32; + + TRANSPOSE_OUT_128(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4]); /* TMP[4] - buffer */ + /* Order of blocks is inverted */ + + TMP[3] = _mm_shuffle_epi8(TMP[3], M128(swapBytes)); + TMP[2] = _mm_shuffle_epi8(TMP[2], M128(swapBytes)); + TMP[1] = _mm_shuffle_epi8(TMP[1], M128(swapBytes)); + TMP[0] = _mm_shuffle_epi8(TMP[0], M128(swapBytes)); + + ciphBytes0 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 0 * cfbBlkSize); + ciphBytes1 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 1 * cfbBlkSize); + ciphBytes2 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 2 * cfbBlkSize); + ciphBytes3 = _mm_maskz_expandloadu_epi8(kMsbMask16, pSrc + 3 * cfbBlkSize); + + TMP[3] = _mm_xor_si128(ciphBytes0, TMP[3]); + TMP[2] = _mm_xor_si128(ciphBytes1, TMP[2]); + TMP[1] = _mm_xor_si128(ciphBytes2, TMP[1]); + TMP[0] = _mm_xor_si128(ciphBytes3, TMP[0]); + + _mm_mask_compressstoreu_epi8(pDst + 0 * cfbBlkSize, kMsbMask16, TMP[3]); + _mm_mask_compressstoreu_epi8(pDst + 1 * cfbBlkSize, kMsbMask16, TMP[2]); + _mm_mask_compressstoreu_epi8(pDst + 2 * cfbBlkSize, kMsbMask16, TMP[1]); + _mm_mask_compressstoreu_epi8(pDst + 3 * cfbBlkSize, kMsbMask16, TMP[0]); + + pSrc += 4 * cfbBlkSize; + pDst += 4 * cfbBlkSize; + } + + /* clear secret data */ + for(Ipp32u i = 0; i < sizeof(TMP)/sizeof(TMP[0]); i++){ + TMP[i] = _mm_setzero_si128(); //_mm_xor_si128(TMP[i],TMP[i]); + } + + len -= processedLen; + + if (len){ + _mm_storeu_si128((__m128i*)(pIV), currentState); + } +} + +#endif /* #if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) */ +#endif /* _IPP32E>=_IPP32E_K1 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_cipher.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_cipher.c new file mode 100644 index 0000000..710abf4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_cipher.c @@ -0,0 +1,79 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 encryption/decryption +// +// Contents: +// cpSMS4_Cipher() +// +*/ + +#include "owncp.h" +#include "pcpsms4.h" +#include "pcptool.h" + + +//#include "owndefs.h" +static void cpSMS4_ECB_gpr_x1(Ipp8u* otxt, const Ipp8u* itxt, const Ipp32u* pRoundKeys) +{ + __ALIGN16 Ipp32u buff[4 + SMS4_ROUND_KEYS_NUM]; + buff[0] = HSTRING_TO_U32(itxt); + buff[1] = HSTRING_TO_U32(itxt+sizeof(Ipp32u)); + buff[2] = HSTRING_TO_U32(itxt+sizeof(Ipp32u)*2); + buff[3] = HSTRING_TO_U32(itxt+sizeof(Ipp32u)*3); + { + int nr; + for(nr=0; nr < SMS4_ROUND_KEYS_NUM; nr++) { + buff[4+nr] = buff[nr] ^ cpCipherMix_SMS4(buff[nr+1]^buff[nr+2]^buff[nr+3]^pRoundKeys[nr]); + } + } + U32_TO_HSTRING(otxt, buff[4 + SMS4_ROUND_KEYS_NUM - 1]); + U32_TO_HSTRING(otxt+sizeof(Ipp32u), buff[4 + SMS4_ROUND_KEYS_NUM - 2]); + U32_TO_HSTRING(otxt+sizeof(Ipp32u)*2, buff[4 + SMS4_ROUND_KEYS_NUM - 3]); + U32_TO_HSTRING(otxt+sizeof(Ipp32u)*3, buff[4 + SMS4_ROUND_KEYS_NUM - 4]); + + /* clear secret data */ + PurgeBlock(buff, sizeof(buff)); +} + +IPP_OWN_DEFN (void, cpSMS4_Cipher, (Ipp8u* otxt, const Ipp8u* itxt, const Ipp32u* pRoundKeys)) +{ + #if (_IPP32E>=_IPP32E_K1) + #if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) + if (IsFeatureEnabled(ippCPUID_AVX512GFNI)){ + cpSMS4_ECB_gfni_x1(otxt, itxt, pRoundKeys); + return; + } + else + #endif /* #if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) */ + #endif + #if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + if(IsFeatureEnabled(ippCPUID_AES) || IsFeatureEnabled(ippCPUID_AVX2VAES)){ + cpSMS4_ECB_aesni_x1(otxt, itxt, pRoundKeys); + return; + } + else + #endif + { + cpSMS4_ECB_gpr_x1(otxt, itxt, pRoundKeys); + return; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ctr_gfni.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ctr_gfni.c new file mode 100644 index 0000000..6a9cb7d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ctr_gfni.c @@ -0,0 +1,1427 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 ECB decryption +// +// Contents: +// cpSMS4_CTR_gfni512() +// cpSMS4_CTR_gfni512x48() +// cpSMS4_CTR_gfni512x32() +// cpSMS4_CTR_gfni512x16() +// cpSMS4_CTR_gfni128x12() +// cpSMS4_CTR_gfni128x8() +// cpSMS4_CTR_gfni128x4() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpsms4.h" +#include "pcptool.h" + +#if (_IPP32E>=_IPP32E_K1) + +#if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) + +#include "pcpsms4_gfni.h" + +static __ALIGN32 Ipp8u endiannes_swap[] = {12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3, + 12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3, + 12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3, + 12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3}; + +static __ALIGN32 Ipp8u endiannes[] = {15,14,13,12, 11,10,9,8, 7,6,5,4, 3,2,1,0, + 15,14,13,12, 11,10,9,8, 7,6,5,4, 3,2,1,0, + 15,14,13,12, 11,10,9,8, 7,6,5,4, 3,2,1,0, + 15,14,13,12, 11,10,9,8, 7,6,5,4, 3,2,1,0}; + +static __ALIGN16 Ipp8u first_inc[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; + +static __ALIGN16 Ipp8u next_inc[] = {4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; + +static __ALIGN16 Ipp8u one128[] = {1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; + +__INLINE __m512i inc512(__m512i x, Ipp8u* increment) +{ + __m512i t = _mm512_add_epi64(x, M512(increment)); + __mmask8 carryMask = _mm512_cmplt_epu64_mask(t, x); + carryMask = (__mmask8)(carryMask << 1); + t = _mm512_add_epi64(t, _mm512_mask_set1_epi64(_mm512_setzero_si512(), carryMask, 1)); + + return t; +} + +__INLINE __m128i inc128(__m128i x) +{ + __m128i t = _mm_add_epi64(x, M128(one128)); + x = _mm_cmpeq_epi64(t, _mm_setzero_si128()); + t = _mm_sub_epi64(t, _mm_slli_si128(x, sizeof(Ipp64u))); + return t; +} + +static +int cpSMS4_CTR_gfni512x48(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, const Ipp8u* pCtrMask, Ipp8u* pCtr); +static +int cpSMS4_CTR_gfni512x32(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, const Ipp8u* pCtrMask, Ipp8u* pCtr); +static +int cpSMS4_CTR_gfni512x16(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, const Ipp8u* pCtrMask, Ipp8u* pCtr); +static +int cpSMS4_CTR_gfni128x12(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, const Ipp8u* pCtrMask, Ipp8u* pCtr); +static +int cpSMS4_CTR_gfni128x8(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, const Ipp8u* pCtrMask, Ipp8u* pCtr); +static +int cpSMS4_ECB_gfni128x4(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, const Ipp8u* pCtrMask, Ipp8u* pCtr); + +/* +// 64*MBS_SMS4 bytes processing +*/ + +IPP_OWN_DEFN (int, cpSMS4_CTR_gfni512, (Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, const Ipp8u* pCtrMask, Ipp8u* pCtr)) +{ + int processedLen = len - (len % (64 * MBS_SMS4)); + int n; + + if(processedLen){ + + __ALIGN16 __m512i TMP[23]; + + // TMP[20] - ctr + // TMP[21] - ctrMask + // TMP[22] - ctrUnch + + TMP[20] = _mm512_broadcast_i64x2(_mm_loadu_si128((__m128i*)pCtr)); + TMP[21] = _mm512_broadcast_i64x2(_mm_loadu_si128((__m128i*)pCtrMask)); + + /* read string counter and convert to numerical */ + TMP[20] = _mm512_shuffle_epi8(TMP[20], M512(endiannes)); + + /* read string mask and convert to numerical */ + TMP[21] = _mm512_shuffle_epi8(TMP[21], M512(endiannes)); + + /* upchanged counter bits */ + TMP[22] = _mm512_andnot_si512(TMP[21], TMP[20]); + + /* first incremention */ + TMP[20] = inc512(TMP[20], first_inc); + + TMP[20] = _mm512_and_si512(TMP[21], TMP[20]); + + for (n = 0; n < processedLen; n += (64 * MBS_SMS4), pInp += (64 * MBS_SMS4), pOut += (64 * MBS_SMS4)) { + int itr; + + TMP[0] = TMP[20]; + TMP[1] = inc512(TMP[0], next_inc); + TMP[2] = inc512(TMP[1], next_inc); + TMP[3] = inc512(TMP[2], next_inc); + TMP[20] = inc512(TMP[3], next_inc); + + TMP[0] = _mm512_xor_si512(TMP[22], _mm512_and_si512(TMP[0], TMP[21])); + TMP[1] = _mm512_xor_si512(TMP[22], _mm512_and_si512(TMP[1], TMP[21])); + TMP[2] = _mm512_xor_si512(TMP[22], _mm512_and_si512(TMP[2], TMP[21])); + TMP[3] = _mm512_xor_si512(TMP[22], _mm512_and_si512(TMP[3], TMP[21])); + + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(endiannes_swap)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(endiannes_swap)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(endiannes_swap)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(endiannes_swap)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = TMP[20]; + TMP[1] = inc512(TMP[0], next_inc); + TMP[2] = inc512(TMP[1], next_inc); + TMP[3] = inc512(TMP[2], next_inc); + TMP[20] = inc512(TMP[3], next_inc); + + TMP[0] = _mm512_xor_si512(TMP[22], _mm512_and_si512(TMP[0], TMP[21])); + TMP[1] = _mm512_xor_si512(TMP[22], _mm512_and_si512(TMP[1], TMP[21])); + TMP[2] = _mm512_xor_si512(TMP[22], _mm512_and_si512(TMP[2], TMP[21])); + TMP[3] = _mm512_xor_si512(TMP[22], _mm512_and_si512(TMP[3], TMP[21])); + + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(endiannes_swap)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(endiannes_swap)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(endiannes_swap)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(endiannes_swap)); + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = TMP[20]; + TMP[1] = inc512(TMP[0], next_inc); + TMP[2] = inc512(TMP[1], next_inc); + TMP[3] = inc512(TMP[2], next_inc); + TMP[20] = inc512(TMP[3], next_inc); + + TMP[0] = _mm512_xor_si512(TMP[22], _mm512_and_si512(TMP[0], TMP[21])); + TMP[1] = _mm512_xor_si512(TMP[22], _mm512_and_si512(TMP[1], TMP[21])); + TMP[2] = _mm512_xor_si512(TMP[22], _mm512_and_si512(TMP[2], TMP[21])); + TMP[3] = _mm512_xor_si512(TMP[22], _mm512_and_si512(TMP[3], TMP[21])); + + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(endiannes_swap)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(endiannes_swap)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(endiannes_swap)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(endiannes_swap)); + TRANSPOSE_INP_512(TMP[12], TMP[13], TMP[14], TMP[15], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = TMP[20]; + TMP[1] = inc512(TMP[0], next_inc); + TMP[2] = inc512(TMP[1], next_inc); + TMP[3] = inc512(TMP[2], next_inc); + TMP[20] = inc512(TMP[3], next_inc); + + TMP[0] = _mm512_xor_si512(TMP[22], _mm512_and_si512(TMP[0], TMP[21])); + TMP[1] = _mm512_xor_si512(TMP[22], _mm512_and_si512(TMP[1], TMP[21])); + TMP[2] = _mm512_xor_si512(TMP[22], _mm512_and_si512(TMP[2], TMP[21])); + TMP[3] = _mm512_xor_si512(TMP[22], _mm512_and_si512(TMP[3], TMP[21])); + + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(endiannes_swap)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(endiannes_swap)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(endiannes_swap)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(endiannes_swap)); + TRANSPOSE_INP_512(TMP[16], TMP[17], TMP[18], TMP[19], TMP[0], TMP[1], TMP[2], TMP[3]); + + + for (itr = 0; itr < 8; itr++, pRKey += 4) { + /* initial xors */ + TMP[3] = TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[0]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[13]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[14]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[15]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[17]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[18]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[19]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + TMP[3] = sBox512(TMP[3]); + /* Sbox done, now L */ + TMP[4] = _mm512_xor_si512(_mm512_xor_si512(TMP[4], TMP[0]), L512(TMP[0])); + TMP[8] = _mm512_xor_si512(_mm512_xor_si512(TMP[8], TMP[1]), L512(TMP[1])); + TMP[12] = _mm512_xor_si512(_mm512_xor_si512(TMP[12], TMP[2]), L512(TMP[2])); + TMP[16] = _mm512_xor_si512(_mm512_xor_si512(TMP[16], TMP[3]), L512(TMP[3])); + + /* initial xors */ + TMP[3] = TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[1]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[14]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[15]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[12]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[18]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[19]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[16]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + TMP[3] = sBox512(TMP[3]); + /* Sbox done, now L */ + TMP[5] = _mm512_xor_si512(_mm512_xor_si512(TMP[5], TMP[0]), L512(TMP[0])); + TMP[9] = _mm512_xor_si512(_mm512_xor_si512(TMP[9], TMP[1]), L512(TMP[1])); + TMP[13] = _mm512_xor_si512(_mm512_xor_si512(TMP[13], TMP[2]), L512(TMP[2])); + TMP[17] = _mm512_xor_si512(_mm512_xor_si512(TMP[17], TMP[3]), L512(TMP[3])); + + /* initial xors */ + TMP[3] = TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[2]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[15]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[12]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[13]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[19]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[16]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[17]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + TMP[3] = sBox512(TMP[3]); + /* Sbox done, now L */ + TMP[6] = _mm512_xor_si512(_mm512_xor_si512(TMP[6], TMP[0]), L512(TMP[0])); + TMP[10] = _mm512_xor_si512(_mm512_xor_si512(TMP[10], TMP[1]), L512(TMP[1])); + TMP[14] = _mm512_xor_si512(_mm512_xor_si512(TMP[14], TMP[2]), L512(TMP[2])); + TMP[18] = _mm512_xor_si512(_mm512_xor_si512(TMP[18], TMP[3]), L512(TMP[3])); + + /* initial xors */ + TMP[3] = TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[3]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[12]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[13]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[14]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[16]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[17]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[18]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + TMP[3] = sBox512(TMP[3]); + /* Sbox done, now L */ + TMP[7] = _mm512_xor_si512(_mm512_xor_si512(TMP[7], TMP[0]), L512(TMP[0])); + TMP[11] = _mm512_xor_si512(_mm512_xor_si512(TMP[11], TMP[1]), L512(TMP[1])); + TMP[15] = _mm512_xor_si512(_mm512_xor_si512(TMP[15], TMP[2]), L512(TMP[2])); + TMP[19] = _mm512_xor_si512(_mm512_xor_si512(TMP[19], TMP[3]), L512(TMP[3])); + + } + + pRKey -= 32; + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(pOut), _mm512_xor_si512(TMP[0], _mm512_loadu_si512((__m512i*)(pInp)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 4), _mm512_xor_si512(TMP[1], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 4)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 8), _mm512_xor_si512(TMP[2], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 8)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 12), _mm512_xor_si512(TMP[3], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 12)))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 16), _mm512_xor_si512(TMP[0], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 16)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 20), _mm512_xor_si512(TMP[1], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 20)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 24), _mm512_xor_si512(TMP[2], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 24)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 28), _mm512_xor_si512(TMP[3], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 28)))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[12], TMP[13], TMP[14], TMP[15]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 32), _mm512_xor_si512(TMP[0], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 32)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 36), _mm512_xor_si512(TMP[1], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 36)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 40), _mm512_xor_si512(TMP[2], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 40)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 44), _mm512_xor_si512(TMP[3], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 44)))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[16], TMP[17], TMP[18], TMP[19]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 48), _mm512_xor_si512(TMP[0], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 48)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 52), _mm512_xor_si512(TMP[1], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 52)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 56), _mm512_xor_si512(TMP[2], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 56)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 60), _mm512_xor_si512(TMP[3], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 60)))); + + } + + /* Save counter */ + TMP[20] = _mm512_xor_si512(TMP[22], _mm512_and_si512(TMP[20], TMP[21])); + TMP[20] = _mm512_shuffle_epi8(TMP[20], M512(endiannes)); + _mm_storeu_si128((__m128i*)pCtr, _mm512_castsi512_si128(TMP[20])); + + /* clear secret data */ + for (unsigned int i = 0; i < sizeof(TMP) / sizeof(TMP[0]); ++i) { + TMP[i] = _mm512_xor_si512(TMP[i], TMP[i]); + } + + } + + len -= processedLen; + if (len) + processedLen += cpSMS4_CTR_gfni512x48(pOut, pInp, len, pRKey, pCtrMask, pCtr); + + return processedLen; +} + +/* +// 48*MBS_SMS4 bytes processing +*/ + +static +int cpSMS4_CTR_gfni512x48(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, const Ipp8u* pCtrMask, Ipp8u* pCtr) +{ + int processedLen = len - (len % (48 * MBS_SMS4)); + int n; + + if(processedLen){ + + __ALIGN16 __m512i TMP[19]; + + // TMP[16] - ctr + // TMP[17] - ctrMask + // TMP[18] - ctrUnch + + TMP[16] = _mm512_broadcast_i64x2(_mm_loadu_si128((__m128i*)pCtr)); + TMP[17] = _mm512_broadcast_i64x2(_mm_loadu_si128((__m128i*)pCtrMask)); + + /* read string counter and convert to numerical */ + TMP[16] = _mm512_shuffle_epi8(TMP[16], M512(endiannes)); + + /* read string mask and convert to numerical */ + TMP[17] = _mm512_shuffle_epi8(TMP[17], M512(endiannes)); + + /* upchanged counter bits */ + TMP[18] = _mm512_andnot_si512(TMP[17], TMP[16]); + + /* first incremention */ + TMP[16] = inc512(TMP[16], first_inc); + + TMP[16] = _mm512_and_si512(TMP[17], TMP[16]); + + for (n = 0; n < processedLen; n += (48 * MBS_SMS4), pInp += (48 * MBS_SMS4), pOut += (48 * MBS_SMS4)) { + int itr; + + TMP[0] = TMP[16]; + TMP[1] = inc512(TMP[0], next_inc); + TMP[2] = inc512(TMP[1], next_inc); + TMP[3] = inc512(TMP[2], next_inc); + TMP[16] = inc512(TMP[3], next_inc); + + TMP[0] = _mm512_xor_si512(TMP[18], _mm512_and_si512(TMP[0], TMP[17])); + TMP[1] = _mm512_xor_si512(TMP[18], _mm512_and_si512(TMP[1], TMP[17])); + TMP[2] = _mm512_xor_si512(TMP[18], _mm512_and_si512(TMP[2], TMP[17])); + TMP[3] = _mm512_xor_si512(TMP[18], _mm512_and_si512(TMP[3], TMP[17])); + + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(endiannes_swap)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(endiannes_swap)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(endiannes_swap)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(endiannes_swap)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = TMP[16]; + TMP[1] = inc512(TMP[0], next_inc); + TMP[2] = inc512(TMP[1], next_inc); + TMP[3] = inc512(TMP[2], next_inc); + TMP[16] = inc512(TMP[3], next_inc); + + TMP[0] = _mm512_xor_si512(TMP[18], _mm512_and_si512(TMP[0], TMP[17])); + TMP[1] = _mm512_xor_si512(TMP[18], _mm512_and_si512(TMP[1], TMP[17])); + TMP[2] = _mm512_xor_si512(TMP[18], _mm512_and_si512(TMP[2], TMP[17])); + TMP[3] = _mm512_xor_si512(TMP[18], _mm512_and_si512(TMP[3], TMP[17])); + + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(endiannes_swap)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(endiannes_swap)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(endiannes_swap)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(endiannes_swap)); + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = TMP[16]; + TMP[1] = inc512(TMP[0], next_inc); + TMP[2] = inc512(TMP[1], next_inc); + TMP[3] = inc512(TMP[2], next_inc); + TMP[16] = inc512(TMP[3], next_inc); + + TMP[0] = _mm512_xor_si512(TMP[18], _mm512_and_si512(TMP[0], TMP[17])); + TMP[1] = _mm512_xor_si512(TMP[18], _mm512_and_si512(TMP[1], TMP[17])); + TMP[2] = _mm512_xor_si512(TMP[18], _mm512_and_si512(TMP[2], TMP[17])); + TMP[3] = _mm512_xor_si512(TMP[18], _mm512_and_si512(TMP[3], TMP[17])); + + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(endiannes_swap)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(endiannes_swap)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(endiannes_swap)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(endiannes_swap)); + TRANSPOSE_INP_512(TMP[12], TMP[13], TMP[14], TMP[15], TMP[0], TMP[1], TMP[2], TMP[3]); + + for (itr = 0; itr < 8; itr++, pRKey += 4) { + /* initial xors */ + TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[0]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[13]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[14]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[15]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + /* Sbox done, now L */ + TMP[4] = _mm512_xor_si512(_mm512_xor_si512(TMP[4], TMP[0]), L512(TMP[0])); + TMP[8] = _mm512_xor_si512(_mm512_xor_si512(TMP[8], TMP[1]), L512(TMP[1])); + TMP[12] = _mm512_xor_si512(_mm512_xor_si512(TMP[12], TMP[2]), L512(TMP[2])); + + /* initial xors */ + TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[1]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[14]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[15]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[12]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + /* Sbox done, now L */ + TMP[5] = _mm512_xor_si512(_mm512_xor_si512(TMP[5], TMP[0]), L512(TMP[0])); + TMP[9] = _mm512_xor_si512(_mm512_xor_si512(TMP[9], TMP[1]), L512(TMP[1])); + TMP[13] = _mm512_xor_si512(_mm512_xor_si512(TMP[13], TMP[2]), L512(TMP[2])); + + /* initial xors */ + TMP[3] = TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[2]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[15]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[12]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[13]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + /* Sbox done, now L */ + TMP[6] = _mm512_xor_si512(_mm512_xor_si512(TMP[6], TMP[0]), L512(TMP[0])); + TMP[10] = _mm512_xor_si512(_mm512_xor_si512(TMP[10], TMP[1]), L512(TMP[1])); + TMP[14] = _mm512_xor_si512(_mm512_xor_si512(TMP[14], TMP[2]), L512(TMP[2])); + + /* initial xors */ + TMP[3] = TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[3]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[12]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[13]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[14]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + /* Sbox done, now L */ + TMP[7] = _mm512_xor_si512(_mm512_xor_si512(TMP[7], TMP[0]), L512(TMP[0])); + TMP[11] = _mm512_xor_si512(_mm512_xor_si512(TMP[11], TMP[1]), L512(TMP[1])); + TMP[15] = _mm512_xor_si512(_mm512_xor_si512(TMP[15], TMP[2]), L512(TMP[2])); + + } + + pRKey -= 32; + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(pOut), _mm512_xor_si512(TMP[0], _mm512_loadu_si512((__m512i*)(pInp)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 4), _mm512_xor_si512(TMP[1], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 4)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 8), _mm512_xor_si512(TMP[2], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 8)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 12), _mm512_xor_si512(TMP[3], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 12)))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 16), _mm512_xor_si512(TMP[0], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 16)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 20), _mm512_xor_si512(TMP[1], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 20)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 24), _mm512_xor_si512(TMP[2], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 24)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 28), _mm512_xor_si512(TMP[3], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 28)))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[12], TMP[13], TMP[14], TMP[15]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 32), _mm512_xor_si512(TMP[0], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 32)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 36), _mm512_xor_si512(TMP[1], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 36)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 40), _mm512_xor_si512(TMP[2], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 40)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 44), _mm512_xor_si512(TMP[3], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 44)))); + + } + + /* Save counter */ + TMP[16] = _mm512_xor_si512(TMP[18], _mm512_and_si512(TMP[16], TMP[17])); + TMP[16] = _mm512_shuffle_epi8(TMP[16], M512(endiannes)); + _mm_storeu_si128((__m128i*)pCtr, _mm512_castsi512_si128(TMP[16])); + + /* clear secret data */ + for (unsigned int i = 0; i < sizeof(TMP) / sizeof(TMP[0]); ++i) { + TMP[i] = _mm512_xor_si512(TMP[i], TMP[i]); + } + + } + + len -= processedLen; + if (len) + processedLen += cpSMS4_CTR_gfni512x32(pOut, pInp, len, pRKey, pCtrMask, pCtr); + + return processedLen; +} + +/* +// 32*MBS_SMS4 bytes processing +*/ + +static +int cpSMS4_CTR_gfni512x32(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, const Ipp8u* pCtrMask, Ipp8u* pCtr) +{ + int processedLen = len - (len % (32 * MBS_SMS4)); + int n; + + if(processedLen){ + + __ALIGN16 __m512i TMP[15]; + + // TMP[12] - ctr + // TMP[13] - ctrMask + // TMP[14] - ctrUnch + + TMP[12] = _mm512_broadcast_i64x2(_mm_loadu_si128((__m128i*)pCtr)); + TMP[13] = _mm512_broadcast_i64x2(_mm_loadu_si128((__m128i*)pCtrMask)); + + /* read string counter and convert to numerical */ + TMP[12] = _mm512_shuffle_epi8(TMP[12], M512(endiannes)); + + /* read string mask and convert to numerical */ + TMP[13] = _mm512_shuffle_epi8(TMP[13], M512(endiannes)); + + /* upchanged counter bits */ + TMP[14] = _mm512_andnot_si512(TMP[13], TMP[12]); + + /* first incremention */ + TMP[12] = inc512(TMP[12], first_inc); + + TMP[12] = _mm512_and_si512(TMP[13], TMP[12]); + + for (n = 0; n < processedLen; n += (32 * MBS_SMS4), pInp += (32 * MBS_SMS4), pOut += (32 * MBS_SMS4)) { + int itr; + + TMP[0] = TMP[12]; + TMP[1] = inc512(TMP[0], next_inc); + TMP[2] = inc512(TMP[1], next_inc); + TMP[3] = inc512(TMP[2], next_inc); + TMP[12] = inc512(TMP[3], next_inc); + + TMP[0] = _mm512_xor_si512(TMP[14], _mm512_and_si512(TMP[0], TMP[13])); + TMP[1] = _mm512_xor_si512(TMP[14], _mm512_and_si512(TMP[1], TMP[13])); + TMP[2] = _mm512_xor_si512(TMP[14], _mm512_and_si512(TMP[2], TMP[13])); + TMP[3] = _mm512_xor_si512(TMP[14], _mm512_and_si512(TMP[3], TMP[13])); + + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(endiannes_swap)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(endiannes_swap)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(endiannes_swap)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(endiannes_swap)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = TMP[12]; + TMP[1] = inc512(TMP[0], next_inc); + TMP[2] = inc512(TMP[1], next_inc); + TMP[3] = inc512(TMP[2], next_inc); + TMP[12] = inc512(TMP[3], next_inc); + + TMP[0] = _mm512_xor_si512(TMP[14], _mm512_and_si512(TMP[0], TMP[13])); + TMP[1] = _mm512_xor_si512(TMP[14], _mm512_and_si512(TMP[1], TMP[13])); + TMP[2] = _mm512_xor_si512(TMP[14], _mm512_and_si512(TMP[2], TMP[13])); + TMP[3] = _mm512_xor_si512(TMP[14], _mm512_and_si512(TMP[3], TMP[13])); + + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(endiannes_swap)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(endiannes_swap)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(endiannes_swap)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(endiannes_swap)); + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], TMP[0], TMP[1], TMP[2], TMP[3]); + + for (itr = 0; itr < 8; itr++, pRKey += 4) { + /* initial xors */ + TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[0]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + /* Sbox done, now L */ + TMP[4] = _mm512_xor_si512(_mm512_xor_si512(TMP[4], TMP[0]), L512(TMP[0])); + TMP[8] = _mm512_xor_si512(_mm512_xor_si512(TMP[8], TMP[1]), L512(TMP[1])); + + /* initial xors */ + TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[1]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + /* Sbox done, now L */ + TMP[5] = _mm512_xor_si512(_mm512_xor_si512(TMP[5], TMP[0]), L512(TMP[0])); + TMP[9] = _mm512_xor_si512(_mm512_xor_si512(TMP[9], TMP[1]), L512(TMP[1])); + + /* initial xors */ + TMP[3] = TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[2]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + /* Sbox done, now L */ + TMP[6] = _mm512_xor_si512(_mm512_xor_si512(TMP[6], TMP[0]), L512(TMP[0])); + TMP[10] = _mm512_xor_si512(_mm512_xor_si512(TMP[10], TMP[1]), L512(TMP[1])); + + /* initial xors */ + TMP[3] = TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[3]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + /* Sbox done, now L */ + TMP[7] = _mm512_xor_si512(_mm512_xor_si512(TMP[7], TMP[0]), L512(TMP[0])); + TMP[11] = _mm512_xor_si512(_mm512_xor_si512(TMP[11], TMP[1]), L512(TMP[1])); + + } + + pRKey -= 32; + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(pOut), _mm512_xor_si512(TMP[0], _mm512_loadu_si512((__m512i*)(pInp)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 4), _mm512_xor_si512(TMP[1], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 4)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 8), _mm512_xor_si512(TMP[2], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 8)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 12), _mm512_xor_si512(TMP[3], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 12)))); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 16), _mm512_xor_si512(TMP[0], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 16)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 20), _mm512_xor_si512(TMP[1], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 20)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 24), _mm512_xor_si512(TMP[2], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 24)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 28), _mm512_xor_si512(TMP[3], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 28)))); + + } + + /* Save counter */ + TMP[12] = _mm512_xor_si512(TMP[14], _mm512_and_si512(TMP[12], TMP[13])); + TMP[12] = _mm512_shuffle_epi8(TMP[12], M512(endiannes)); + _mm_storeu_si128((__m128i*)pCtr, _mm512_castsi512_si128(TMP[12])); + + /* clear secret data */ + for (unsigned int i = 0; i < sizeof(TMP) / sizeof(TMP[0]); ++i) { + TMP[i] = _mm512_xor_si512(TMP[i], TMP[i]); + } + + } + + len -= processedLen; + if (len) + processedLen += cpSMS4_CTR_gfni512x16(pOut, pInp, len, pRKey, pCtrMask, pCtr); + + return processedLen; +} + +/* +// 16*MBS_SMS4 bytes processing +*/ + +static +int cpSMS4_CTR_gfni512x16(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, const Ipp8u* pCtrMask, Ipp8u* pCtr) +{ + int processedLen = len - (len % (16 * MBS_SMS4)); + int n; + + if(processedLen){ + + __ALIGN16 __m512i TMP[11]; + + // TMP[8] - ctr + // TMP[9] - ctrMask + // TMP[10] - ctrUnch + + TMP[8] = _mm512_broadcast_i64x2(_mm_loadu_si128((__m128i*)pCtr)); + TMP[9] = _mm512_broadcast_i64x2(_mm_loadu_si128((__m128i*)pCtrMask)); + + /* read string counter and convert to numerical */ + TMP[8] = _mm512_shuffle_epi8(TMP[8], M512(endiannes)); + + /* read string mask and convert to numerical */ + TMP[9] = _mm512_shuffle_epi8(TMP[9], M512(endiannes)); + + /* upchanged counter bits */ + TMP[10] = _mm512_andnot_si512(TMP[9], TMP[8]); + + /* first incremention */ + TMP[8] = inc512(TMP[8], first_inc); + + TMP[8] = _mm512_and_si512(TMP[9], TMP[8]); + + for (n = 0; n < processedLen; n += (16 * MBS_SMS4), pInp += (16 * MBS_SMS4), pOut += (16 * MBS_SMS4)) { + int itr; + + TMP[0] = TMP[8]; + TMP[1] = inc512(TMP[0], next_inc); + TMP[2] = inc512(TMP[1], next_inc); + TMP[3] = inc512(TMP[2], next_inc); + TMP[8] = inc512(TMP[3], next_inc); + + TMP[0] = _mm512_xor_si512(TMP[10], _mm512_and_si512(TMP[0], TMP[9])); + TMP[1] = _mm512_xor_si512(TMP[10], _mm512_and_si512(TMP[1], TMP[9])); + TMP[2] = _mm512_xor_si512(TMP[10], _mm512_and_si512(TMP[2], TMP[9])); + TMP[3] = _mm512_xor_si512(TMP[10], _mm512_and_si512(TMP[3], TMP[9])); + + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(endiannes_swap)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(endiannes_swap)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(endiannes_swap)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(endiannes_swap)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + for (itr = 0; itr < 8; itr++, pRKey += 4) { + /* initial xors */ + TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[0]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + /* Sbox done, now L */ + TMP[4] = _mm512_xor_si512(_mm512_xor_si512(TMP[4], TMP[0]), L512(TMP[0])); + + /* initial xors */ + TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[1]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + /* Sbox done, now L */ + TMP[5] = _mm512_xor_si512(_mm512_xor_si512(TMP[5], TMP[0]), L512(TMP[0])); + + /* initial xors */ + TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[2]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + /* Sbox done, now L */ + TMP[6] = _mm512_xor_si512(_mm512_xor_si512(TMP[6], TMP[0]), L512(TMP[0])); + + /* initial xors */ + TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[3]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + /* Sbox done, now L */ + TMP[7] = _mm512_xor_si512(_mm512_xor_si512(TMP[7], TMP[0]), L512(TMP[0])); + } + + pRKey -= 32; + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + + _mm512_storeu_si512((__m512i*)(pOut), _mm512_xor_si512(TMP[0], _mm512_loadu_si512((__m512i*)(pInp)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 4), _mm512_xor_si512(TMP[1], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 4)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 8), _mm512_xor_si512(TMP[2], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 8)))); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 12), _mm512_xor_si512(TMP[3], _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 12)))); + } + + /* Save counter */ + TMP[8] = _mm512_xor_si512(TMP[10], _mm512_and_si512(TMP[8], TMP[9])); + TMP[8] = _mm512_shuffle_epi8(TMP[8], M512(endiannes)); + _mm_storeu_si128((__m128i*)pCtr, _mm512_castsi512_si128(TMP[8])); + + /* clear secret data */ + for (unsigned int i = 0; i < sizeof(TMP) / sizeof(TMP[0]); ++i) { + TMP[i] = _mm512_xor_si512(TMP[i], TMP[i]); + } + + } + + len -= processedLen; + if (len) + processedLen += cpSMS4_CTR_gfni128x12(pOut, pInp, len, pRKey, pCtrMask, pCtr); + + return processedLen; +} + +/* +// 12*MBS_SMS4 processing +*/ + +static +int cpSMS4_CTR_gfni128x12(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, const Ipp8u* pCtrMask, Ipp8u* pCtr) +{ + int processedLen = len - (len % (12 * MBS_SMS4)); + int n; + + if(processedLen){ + + __ALIGN16 __m128i TMP[22]; + + // TMP[15] - ctr + // TMP[16] - ctrMask + // TMP[17] - ctrUnch + + TMP[15] = _mm_loadu_si128((__m128i*)pCtr); + TMP[16] = _mm_loadu_si128((__m128i*)pCtrMask); + + TMP[16] = _mm_shuffle_epi8(TMP[16], M128(endiannes)); + TMP[15] = _mm_shuffle_epi8(TMP[15], M128(endiannes)); + TMP[17] = _mm_andnot_si128(TMP[16], TMP[15]); + + for(n=0; n= 1920) */ + +#endif /* _IPP32E>=_IPP32E_K1 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_decrypt_cbc.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_decrypt_cbc.c new file mode 100644 index 0000000..db773b0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_decrypt_cbc.c @@ -0,0 +1,104 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 encryption/decryption +// +// Contents: +// cpDecryptSMS4_cbc() +// +*/ + +#include "owncp.h" +#include "pcpsms4.h" +#include "pcptool.h" +#include "pcpsms4_decrypt_cbc.h" + +/*F* +// Name: cpDecryptSMS4_cbc +// +// Purpose: SMS4-CBC decryption. +// +// Parameters: +// pIV pointer to the initialization vector +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// dataLen input/output buffer length (in bytes) +// pCtx pointer to the SMS4 context +// +*F*/ +IPP_OWN_DEFN (void, cpDecryptSMS4_cbc, (const Ipp8u* pIV, const Ipp8u* pSrc, Ipp8u* pDst, int dataLen, const IppsSMS4Spec* pCtx)) +{ + const Ipp32u* pRoundKeys = SMS4_DRK(pCtx); + + __ALIGN16 Ipp32u TMP[2*(MBS_SMS4/sizeof(Ipp32u))]; + + /* + iv size = MBS_SMS4/sizeof(Ipp32u) + tmp size = MBS_SMS4/sizeof(Ipp32u) + */ + + Ipp32u* iv = TMP; + Ipp32u* tmp = TMP + (MBS_SMS4/sizeof(Ipp32u)); + + /* copy IV */ + CopyBlock16(pIV, iv); + + /* do decryption */ + #if (_IPP32E>=_IPP32E_K1) + #if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) + if (IsFeatureEnabled(ippCPUID_AVX512GFNI)) { + int processedLen = cpSMS4_CBC_dec_gfni512(pDst, pSrc, dataLen, pRoundKeys, (Ipp8u*)iv); + pSrc += processedLen; + pDst += processedLen; + dataLen -= processedLen; + } + else + #endif /* #if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) */ + #endif /* (_IPP32E>=_IPP32E_K1) */ + #if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + if(IsFeatureEnabled(ippCPUID_AES) || IsFeatureEnabled(ippCPUID_AVX2VAES)) { + int processedLen = cpSMS4_CBC_dec_aesni(pDst, pSrc, dataLen, pRoundKeys, (Ipp8u*)iv); + pSrc += processedLen; + pDst += processedLen; + dataLen -= processedLen; + } + #endif + + for(; dataLen>0; dataLen-=MBS_SMS4, pSrc+=MBS_SMS4, pDst+=MBS_SMS4) { + + cpSMS4_Cipher((Ipp8u*)tmp, (Ipp8u*)pSrc, pRoundKeys); + + tmp[0] ^= iv[0]; + tmp[1] ^= iv[1]; + tmp[2] ^= iv[2]; + tmp[3] ^= iv[3]; + + iv[0] = ((Ipp32u*)pSrc)[0]; + iv[1] = ((Ipp32u*)pSrc)[1]; + iv[2] = ((Ipp32u*)pSrc)[2]; + iv[3] = ((Ipp32u*)pSrc)[3]; + + CopyBlock16(tmp, pDst); + } + + /* clear secret data */ + PurgeBlock(TMP, sizeof(TMP)); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_decrypt_cbc.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_decrypt_cbc.h new file mode 100644 index 0000000..7db166a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_decrypt_cbc.h @@ -0,0 +1,37 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 encryption/decryption +// +// Contents: +// cpDecryptSMS4_cbc() +// +*/ + +#include "owncp.h" + +#if !defined _PCP_SMS4_DECRYPT_CBC_H +#define _PCP_SMS4_DECRYPT_CBC_H + +#define cpDecryptSMS4_cbc OWNAPI(cpDecryptSMS4_cbc) + IPP_OWN_DECL (void, cpDecryptSMS4_cbc, (const Ipp8u* pIV, const Ipp8u* pSrc, Ipp8u* pDst, int dataLen, const IppsSMS4Spec* pCtx)) + +#endif /* #if !defined _PCP_SMS4_DECRYPT_CBC_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ecb_gfni.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ecb_gfni.c new file mode 100644 index 0000000..193a912 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ecb_gfni.c @@ -0,0 +1,1170 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 ECB decryption +// +// Contents: +// cpSMS4_ECB_gfni512() +// cpSMS4_ECB_gfni512x48() +// cpSMS4_ECB_gfni512x32() +// cpSMS4_ECB_gfni512x16() +// cpSMS4_ECB_gfni128x12() +// cpSMS4_ECB_gfni128x8() +// cpSMS4_ECB_gfni128x4() +// cpSMS4_ECB_gfni128x4() +// cpSMS4_ECB_gfni128_tail() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpsms4.h" +#include "pcptool.h" + +#if (_IPP32E>=_IPP32E_K1) + +#if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) + +#include "pcpsms4_gfni.h" + + +static +int cpSMS4_ECB_gfni512x48(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey); +static +int cpSMS4_ECB_gfni512x32(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey); +static +int cpSMS4_ECB_gfni512x16(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey); +static +int cpSMS4_ECB_gfni128x12(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey); +static +int cpSMS4_ECB_gfni128x8(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey); +static +int cpSMS4_ECB_gfni128x4(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey); +static +int cpSMS4_ECB_gfni128_tail(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey); + +/* +// 64*MBS_SMS4 bytes processing +*/ + +IPP_OWN_DEFN (int, cpSMS4_ECB_gfni512, (Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey)) +{ + __ALIGN16 __m512i TMP[20]; + + int processedLen = len - (len % (64 * MBS_SMS4)); + int n; + for (n = 0; n < processedLen; n += (64 * MBS_SMS4), pInp += (64 * MBS_SMS4), pOut += (64 * MBS_SMS4)) { + int itr; + + TMP[0] = _mm512_loadu_si512((__m512i*)(pInp)); + TMP[1] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 4)); + TMP[2] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 8)); + TMP[3] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 12)); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 16)); + TMP[1] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 20)); + TMP[2] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 24)); + TMP[3] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 28)); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 32)); + TMP[1] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 36)); + TMP[2] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 40)); + TMP[3] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 44)); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[12], TMP[13], TMP[14], TMP[15], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 48)); + TMP[1] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 52)); + TMP[2] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 56)); + TMP[3] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 60)); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[16], TMP[17], TMP[18], TMP[19], TMP[0], TMP[1], TMP[2], TMP[3]); + + for (itr = 0; itr < 8; itr++, pRKey += 4) { + /* initial xors */ + TMP[3] = TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[0]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[13]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[14]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[15]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[17]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[18]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[19]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + TMP[3] = sBox512(TMP[3]); + /* Sbox done, now L */ + TMP[4] = _mm512_xor_si512(_mm512_xor_si512(TMP[4], TMP[0]), L512(TMP[0])); + TMP[8] = _mm512_xor_si512(_mm512_xor_si512(TMP[8], TMP[1]), L512(TMP[1])); + TMP[12] = _mm512_xor_si512(_mm512_xor_si512(TMP[12], TMP[2]), L512(TMP[2])); + TMP[16] = _mm512_xor_si512(_mm512_xor_si512(TMP[16], TMP[3]), L512(TMP[3])); + + /* initial xors */ + TMP[3] = TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[1]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[14]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[15]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[12]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[18]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[19]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[16]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + TMP[3] = sBox512(TMP[3]); + /* Sbox done, now L */ + TMP[5] = _mm512_xor_si512(_mm512_xor_si512(TMP[5], TMP[0]), L512(TMP[0])); + TMP[9] = _mm512_xor_si512(_mm512_xor_si512(TMP[9], TMP[1]), L512(TMP[1])); + TMP[13] = _mm512_xor_si512(_mm512_xor_si512(TMP[13], TMP[2]), L512(TMP[2])); + TMP[17] = _mm512_xor_si512(_mm512_xor_si512(TMP[17], TMP[3]), L512(TMP[3])); + + /* initial xors */ + TMP[3] = TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[2]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[15]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[12]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[13]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[19]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[16]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[17]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + TMP[3] = sBox512(TMP[3]); + /* Sbox done, now L */ + TMP[6] = _mm512_xor_si512(_mm512_xor_si512(TMP[6], TMP[0]), L512(TMP[0])); + TMP[10] = _mm512_xor_si512(_mm512_xor_si512(TMP[10], TMP[1]), L512(TMP[1])); + TMP[14] = _mm512_xor_si512(_mm512_xor_si512(TMP[14], TMP[2]), L512(TMP[2])); + TMP[18] = _mm512_xor_si512(_mm512_xor_si512(TMP[18], TMP[3]), L512(TMP[3])); + + /* initial xors */ + TMP[3] = TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[3]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[12]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[13]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[14]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[16]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[17]); + TMP[3] = _mm512_xor_si512(TMP[3], TMP[18]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + TMP[3] = sBox512(TMP[3]); + /* Sbox done, now L */ + TMP[7] = _mm512_xor_si512(_mm512_xor_si512(TMP[7], TMP[0]), L512(TMP[0])); + TMP[11] = _mm512_xor_si512(_mm512_xor_si512(TMP[11], TMP[1]), L512(TMP[1])); + TMP[15] = _mm512_xor_si512(_mm512_xor_si512(TMP[15], TMP[2]), L512(TMP[2])); + TMP[19] = _mm512_xor_si512(_mm512_xor_si512(TMP[19], TMP[3]), L512(TMP[3])); + } + + pRKey -= 32; + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(pOut), TMP[0]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 4), TMP[1]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 8), TMP[2]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 12), TMP[3]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 16), TMP[0]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 20), TMP[1]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 24), TMP[2]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 28), TMP[3]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[12], TMP[13], TMP[14], TMP[15]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 32), TMP[0]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 36), TMP[1]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 40), TMP[2]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 44), TMP[3]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[16], TMP[17], TMP[18], TMP[19]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 48), TMP[0]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 52), TMP[1]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 56), TMP[2]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 60), TMP[3]); + } + + /* clear secret data */ + for (unsigned int i = 0; i < sizeof(TMP) / sizeof(TMP[0]); ++i) { + TMP[i] = _mm512_setzero_si512(); //_mm512_xor_si512(TMP[i], TMP[i]); + } + + len -= processedLen; + if (len){ + if(len < 256){ + processedLen += cpSMS4_ECB_gfni128x12(pOut, pInp, len, pRKey); + return processedLen; + } + processedLen += cpSMS4_ECB_gfni512x48(pOut, pInp, len, pRKey); + } + + return processedLen; +} + +/* +// 48*MBS_SMS4 bytes processing +*/ + +static +int cpSMS4_ECB_gfni512x48(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey) +{ + __ALIGN16 __m512i TMP[16]; + + int processedLen = len - (len % (48 * MBS_SMS4)); + int n; + for (n = 0; n < processedLen; n += (48 * MBS_SMS4), pInp += (48 * MBS_SMS4), pOut += (48 * MBS_SMS4)) { + int itr; + + TMP[0] = _mm512_loadu_si512((__m512i*)(pInp)); + TMP[1] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 4)); + TMP[2] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 8)); + TMP[3] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 12)); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 16)); + TMP[1] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 20)); + TMP[2] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 24)); + TMP[3] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 28)); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 32)); + TMP[1] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 36)); + TMP[2] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 40)); + TMP[3] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 44)); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[12], TMP[13], TMP[14], TMP[15], TMP[0], TMP[1], TMP[2], TMP[3]); + + for (itr = 0; itr < 8; itr++, pRKey += 4) { + /* initial xors */ + TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[0]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[13]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[14]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[15]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + /* Sbox done, now L */ + TMP[4] = _mm512_xor_si512(_mm512_xor_si512(TMP[4], TMP[0]), L512(TMP[0])); + TMP[8] = _mm512_xor_si512(_mm512_xor_si512(TMP[8], TMP[1]), L512(TMP[1])); + TMP[12] = _mm512_xor_si512(_mm512_xor_si512(TMP[12], TMP[2]), L512(TMP[2])); + + /* initial xors */ + TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[1]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[14]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[15]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[12]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + /* Sbox done, now L */ + TMP[5] = _mm512_xor_si512(_mm512_xor_si512(TMP[5], TMP[0]), L512(TMP[0])); + TMP[9] = _mm512_xor_si512(_mm512_xor_si512(TMP[9], TMP[1]), L512(TMP[1])); + TMP[13] = _mm512_xor_si512(_mm512_xor_si512(TMP[13], TMP[2]), L512(TMP[2])); + + /* initial xors */ + TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[2]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[15]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[12]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[13]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + /* Sbox done, now L */ + TMP[6] = _mm512_xor_si512(_mm512_xor_si512(TMP[6], TMP[0]), L512(TMP[0])); + TMP[10] = _mm512_xor_si512(_mm512_xor_si512(TMP[10], TMP[1]), L512(TMP[1])); + TMP[14] = _mm512_xor_si512(_mm512_xor_si512(TMP[14], TMP[2]), L512(TMP[2])); + + /* initial xors */ + TMP[2] = TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[3]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[12]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[13]); + TMP[2] = _mm512_xor_si512(TMP[2], TMP[14]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + TMP[2] = sBox512(TMP[2]); + /* Sbox done, now L */ + TMP[7] = _mm512_xor_si512(_mm512_xor_si512(TMP[7], TMP[0]), L512(TMP[0])); + TMP[11] = _mm512_xor_si512(_mm512_xor_si512(TMP[11], TMP[1]), L512(TMP[1])); + TMP[15] = _mm512_xor_si512(_mm512_xor_si512(TMP[15], TMP[2]), L512(TMP[2])); + } + + pRKey -= 32; + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(pOut), TMP[0]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 4), TMP[1]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 8), TMP[2]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 12), TMP[3]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 16), TMP[0]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 20), TMP[1]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 24), TMP[2]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 28), TMP[3]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[12], TMP[13], TMP[14], TMP[15]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 32), TMP[0]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 36), TMP[1]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 40), TMP[2]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 44), TMP[3]); + } + + /* clear secret data */ + for (unsigned int i = 0; i < sizeof(TMP) / sizeof(TMP[0]); ++i) { + TMP[i] = _mm512_setzero_si512(); //_mm512_xor_si512(TMP[i], TMP[i]); + } + + len -= processedLen; + if (len) + processedLen += cpSMS4_ECB_gfni512x32(pOut, pInp, len, pRKey); + + return processedLen; +} + +/* +// 32*MBS_SMS4 bytes processing +*/ + +static +int cpSMS4_ECB_gfni512x32(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey) +{ + __ALIGN16 __m512i TMP[12]; + + int processedLen = len - (len % (32 * MBS_SMS4)); + int n; + for (n = 0; n < processedLen; n += (32 * MBS_SMS4), pInp += (32 * MBS_SMS4), pOut += (32 * MBS_SMS4)) { + int itr; + + TMP[0] = _mm512_loadu_si512((__m512i*)(pInp)); + TMP[1] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 4)); + TMP[2] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 8)); + TMP[3] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 12)); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + TMP[0] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 16)); + TMP[1] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 20)); + TMP[2] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 24)); + TMP[3] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 28)); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[8], TMP[9], TMP[10], TMP[11], TMP[0], TMP[1], TMP[2], TMP[3]); + + for (itr = 0; itr < 8; itr++, pRKey += 4) { + /* initial xors */ + TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[0]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + /* Sbox done, now L */ + TMP[4] = _mm512_xor_si512(_mm512_xor_si512(TMP[4], TMP[0]), L512(TMP[0])); + TMP[8] = _mm512_xor_si512(_mm512_xor_si512(TMP[8], TMP[1]), L512(TMP[1])); + + /* initial xors */ + TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[1]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + /* Sbox done, now L */ + TMP[5] = _mm512_xor_si512(_mm512_xor_si512(TMP[5], TMP[0]), L512(TMP[0])); + TMP[9] = _mm512_xor_si512(_mm512_xor_si512(TMP[9], TMP[1]), L512(TMP[1])); + + /* initial xors */ + TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[2]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[11]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + /* Sbox done, now L */ + TMP[6] = _mm512_xor_si512(_mm512_xor_si512(TMP[6], TMP[0]), L512(TMP[0])); + TMP[10] = _mm512_xor_si512(_mm512_xor_si512(TMP[10], TMP[1]), L512(TMP[1])); + + /* initial xors */ + TMP[1] = TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[3]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[8]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[9]); + TMP[1] = _mm512_xor_si512(TMP[1], TMP[10]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + TMP[1] = sBox512(TMP[1]); + /* Sbox done, now L */ + TMP[7] = _mm512_xor_si512(_mm512_xor_si512(TMP[7], TMP[0]), L512(TMP[0])); + TMP[11] = _mm512_xor_si512(_mm512_xor_si512(TMP[11], TMP[1]), L512(TMP[1])); + } + + pRKey -= 32; + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(pOut), TMP[0]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 4), TMP[1]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 8), TMP[2]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 12), TMP[3]); + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[8], TMP[9], TMP[10], TMP[11]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 16), TMP[0]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 20), TMP[1]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 24), TMP[2]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 28), TMP[3]); + } + + /* clear secret data */ + for (unsigned int i = 0; i < sizeof(TMP) / sizeof(TMP[0]); ++i) { + TMP[i] = _mm512_setzero_si512(); //_mm512_xor_si512(TMP[i], TMP[i]); + } + + len -= processedLen; + if (len) + processedLen += cpSMS4_ECB_gfni512x16(pOut, pInp, len, pRKey); + + return processedLen; +} + +/* +// 16*MBS_SMS4 bytes processing +*/ + +static +int cpSMS4_ECB_gfni512x16(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey) +{ + __ALIGN16 __m512i TMP[8]; + + int processedLen = len - (len % (16 * MBS_SMS4)); + int n; + for (n = 0; n < processedLen; n += (16 * MBS_SMS4), pInp += (16 * MBS_SMS4), pOut += (16 * MBS_SMS4)) { + int itr; + + TMP[0] = _mm512_loadu_si512((__m512i*)(pInp)); + TMP[1] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 4)); + TMP[2] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 8)); + TMP[3] = _mm512_loadu_si512((__m512i*)(pInp + MBS_SMS4 * 12)); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + TRANSPOSE_INP_512(TMP[4], TMP[5], TMP[6], TMP[7], TMP[0], TMP[1], TMP[2], TMP[3]); + + for (itr = 0; itr < 8; itr++, pRKey += 4) { + /* initial xors */ + TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[0]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + /* Sbox done, now L */ + TMP[4] = _mm512_xor_si512(_mm512_xor_si512(TMP[4], TMP[0]), L512(TMP[0])); + + /* initial xors */ + TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[1]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + /* Sbox done, now L */ + TMP[5] = _mm512_xor_si512(_mm512_xor_si512(TMP[5], TMP[0]), L512(TMP[0])); + + /* initial xors */ + TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[2]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[7]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + /* Sbox done, now L */ + TMP[6] = _mm512_xor_si512(_mm512_xor_si512(TMP[6], TMP[0]), L512(TMP[0])); + + /* initial xors */ + TMP[0] = _mm512_set1_epi32((Ipp32s)pRKey[3]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[4]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[5]); + TMP[0] = _mm512_xor_si512(TMP[0], TMP[6]); + /* Sbox */ + TMP[0] = sBox512(TMP[0]); + /* Sbox done, now L */ + TMP[7] = _mm512_xor_si512(_mm512_xor_si512(TMP[7], TMP[0]), L512(TMP[0])); + } + + pRKey -= 32; + + TRANSPOSE_OUT_512(TMP[0], TMP[1], TMP[2], TMP[3], TMP[4], TMP[5], TMP[6], TMP[7]); + TMP[0] = _mm512_shuffle_epi8(TMP[0], M512(swapBytes)); + TMP[1] = _mm512_shuffle_epi8(TMP[1], M512(swapBytes)); + TMP[2] = _mm512_shuffle_epi8(TMP[2], M512(swapBytes)); + TMP[3] = _mm512_shuffle_epi8(TMP[3], M512(swapBytes)); + _mm512_storeu_si512((__m512i*)(pOut), TMP[0]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 4), TMP[1]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 8), TMP[2]); + _mm512_storeu_si512((__m512i*)(pOut + MBS_SMS4 * 12), TMP[3]); + } + + /* clear secret data */ + for (unsigned int i = 0; i < sizeof(TMP) / sizeof(TMP[0]); ++i) { + TMP[i] = _mm512_setzero_si512(); //_mm512_xor_si512(TMP[i], TMP[i]); + } + + len -= processedLen; + if (len) + processedLen += cpSMS4_ECB_gfni128x12(pOut, pInp, len, pRKey); + + return processedLen; +} + +/* +// 12*MBS_SMS4 processing +*/ + +static +int cpSMS4_ECB_gfni128x12(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey) +{ + __ALIGN16 __m128i TMP[15]; + + int processedLen = len -(len % (12*MBS_SMS4)); + int n; + for(n=0; n= 1920) */ + +#endif /* _IPP32E>=_IPP32E_K1 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ecb_x1_aesni.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ecb_x1_aesni.c new file mode 100644 index 0000000..72c38e7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ecb_x1_aesni.c @@ -0,0 +1,114 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 ECB encryption/decryption +// +// Contents: +// cpSMS4_ECB_aesni_x1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpsms4.h" +#include "pcptool.h" + +#if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + +#include "pcpsms4_y8cn.h" + +/* +// 1*MBS_SMS4 processing +*/ + +IPP_OWN_DEFN (void, cpSMS4_ECB_aesni_x1, (Ipp8u* pOut, const Ipp8u* pInp, const Ipp32u* pRKey)) +{ + __ALIGN16 __m128i TMP[6]; + /* + TMP[0] = T + TMP[1] = K0 + TMP[2] = K1 + TMP[3] = K2 + TMP[4] = K3 + TMP[5] = key4 + */ + + TMP[1] = _mm_shuffle_epi8( _mm_cvtsi32_si128(((Ipp32s*)pInp)[0]), M128(swapBytes)); + TMP[2] = _mm_shuffle_epi8( _mm_cvtsi32_si128(((Ipp32s*)pInp)[1]), M128(swapBytes)); + TMP[3] = _mm_shuffle_epi8( _mm_cvtsi32_si128(((Ipp32s*)pInp)[2]), M128(swapBytes)); + TMP[4] = _mm_shuffle_epi8( _mm_cvtsi32_si128(((Ipp32s*)pInp)[3]), M128(swapBytes)); + + int itr; + for(itr=0; itr<8; itr++, pRKey+=4) { + TMP[5] = _mm_loadu_si128((__m128i*)pRKey); + /* initial xors */ + TMP[0] = _mm_shuffle_epi32(TMP[5], 0x00); /* broadcast(key4 TMP[0]) */ + TMP[0] = _mm_xor_si128(TMP[0], TMP[2]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[3]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[4]); + /* Sbox */ + TMP[0] = sBox(TMP[0]); + /* Sbox done, now L */ + TMP[1] = _mm_xor_si128(_mm_xor_si128(TMP[1], TMP[0]), L(TMP[0])); + + /* initial xors */ + TMP[0] = _mm_shuffle_epi32(TMP[5], 0x55); /* broadcast(key4 TMP[1]) */ + TMP[0] = _mm_xor_si128(TMP[0], TMP[3]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[4]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[1]); + /* Sbox */ + TMP[0] = sBox(TMP[0]); + /* Sbox done, now L */ + TMP[2] = _mm_xor_si128(_mm_xor_si128(TMP[2], TMP[0]), L(TMP[0])); + + /* initial xors */ + TMP[0] = _mm_shuffle_epi32(TMP[5], 0xAA); /* broadcast(key4 TMP[2]) */ + TMP[0] = _mm_xor_si128(TMP[0], TMP[4]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[1]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[2]); + /* Sbox */ + TMP[0] = sBox(TMP[0]); + /* Sbox done, now L */ + TMP[3] = _mm_xor_si128(_mm_xor_si128(TMP[3], TMP[0]), L(TMP[0])); + + /* initial xors */ + TMP[0] = _mm_shuffle_epi32(TMP[5], 0xFF); /* broadcast(key4 TMP[3]) */ + TMP[0] = _mm_xor_si128(TMP[0], TMP[1]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[2]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[3]); + /* Sbox */ + TMP[0] = sBox(TMP[0]); + /* Sbox done, now L */ + TMP[4] = _mm_xor_si128(_mm_xor_si128(TMP[4], TMP[0]), L(TMP[0])); + } + + ((Ipp32u*)(pOut))[0] = (Ipp32u)_mm_cvtsi128_si32(_mm_shuffle_epi8(TMP[4], M128(swapBytes))); + ((Ipp32u*)(pOut))[1] = (Ipp32u)_mm_cvtsi128_si32(_mm_shuffle_epi8(TMP[3], M128(swapBytes))); + ((Ipp32u*)(pOut))[2] = (Ipp32u)_mm_cvtsi128_si32(_mm_shuffle_epi8(TMP[2], M128(swapBytes))); + ((Ipp32u*)(pOut))[3] = (Ipp32u)_mm_cvtsi128_si32(_mm_shuffle_epi8(TMP[1], M128(swapBytes))); + + /* clear secret data */ + for(Ipp32u i = 0; i < sizeof(TMP)/sizeof(TMP[0]); i++){ + TMP[i] = _mm_xor_si128(TMP[i],TMP[i]); + } +} + +#endif /* _IPP_P8, _IPP32E_Y8 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ecb_x1_gfni.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ecb_x1_gfni.c new file mode 100644 index 0000000..16ab55e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_ecb_x1_gfni.c @@ -0,0 +1,119 @@ +/******************************************************************************* +* Copyright (C) 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 ECB encryption/decryption +// +// Contents: +// cpSMS4_ECB_gfni_x1() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpsms4.h" +#include "pcptool.h" + +#if (_IPP32E>=_IPP32E_K1) + +#if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) + +#include "pcpsms4_gfni.h" + +IPP_OWN_DEFN (void, cpSMS4_ECB_gfni_x1, (Ipp8u* pOut, const Ipp8u* pInp, const Ipp32u* pRKey)) +{ + __ALIGN16 __m128i TMP[6]; + /* + TMP[0] = T + TMP[1] = K0 + TMP[2] = K1 + TMP[3] = K2 + TMP[4] = K3 + TMP[5] = key4 + */ + + TMP[1] = _mm_loadu_si128((__m128i*)pInp); + TMP[1] = _mm_shuffle_epi8(TMP[1], M128(swapBytes)); + + TMP[2] = _mm_alignr_epi32(TMP[1], TMP[1], 1); + TMP[3] = _mm_alignr_epi32(TMP[1], TMP[1], 2); + TMP[4] = _mm_alignr_epi32(TMP[1], TMP[1], 3); + + int itr; + for(itr=0; itr<8; itr++, pRKey+=4) { + TMP[5] = _mm_loadu_si128((__m128i*)pRKey); + /* initial xors */ + TMP[0] = _mm_shuffle_epi32(TMP[5], 0x00); /* broadcast(key4 TMP[0]) */ + TMP[0] = _mm_xor_si128(TMP[0], TMP[2]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[3]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[4]); + /* Sbox */ + TMP[0] = sBox128(TMP[0]); + /* Sbox done, now L */ + TMP[1] = _mm_xor_si128(_mm_xor_si128(TMP[1], TMP[0]), L128(TMP[0])); + + /* initial xors */ + TMP[0] = _mm_shuffle_epi32(TMP[5], 0x55); /* broadcast(key4 TMP[1]) */ + TMP[0] = _mm_xor_si128(TMP[0], TMP[3]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[4]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[1]); + /* Sbox */ + TMP[0] = sBox128(TMP[0]); + /* Sbox done, now L */ + TMP[2] = _mm_xor_si128(_mm_xor_si128(TMP[2], TMP[0]), L128(TMP[0])); + + /* initial xors */ + TMP[0] = _mm_shuffle_epi32(TMP[5], 0xAA); /* broadcast(key4 TMP[2]) */ + TMP[0] = _mm_xor_si128(TMP[0], TMP[4]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[1]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[2]); + /* Sbox */ + TMP[0] = sBox128(TMP[0]); + /* Sbox done, now L */ + TMP[3] = _mm_xor_si128(_mm_xor_si128(TMP[3], TMP[0]), L128(TMP[0])); + + /* initial xors */ + TMP[0] = _mm_shuffle_epi32(TMP[5], 0xFF); /* broadcast(key4 TMP[3]) */ + TMP[0] = _mm_xor_si128(TMP[0], TMP[1]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[2]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[3]); + /* Sbox */ + TMP[0] = sBox128(TMP[0]); + /* Sbox done, now L */ + TMP[4] = _mm_xor_si128(_mm_xor_si128(TMP[4], TMP[0]), L128(TMP[0])); + } + + TMP[0] = _mm_unpacklo_epi32(TMP[2], TMP[1]); + TMP[5] = _mm_unpacklo_epi32(TMP[4], TMP[3]); + TMP[1] = _mm_unpacklo_epi64(TMP[5], TMP[0]); + + TMP[1] = _mm_shuffle_epi8(TMP[1], M128(swapBytes)); + + _mm_storeu_si128((__m128i*)pOut, TMP[1]); + + /* clear secret data */ + for(Ipp32u i = 0; i < sizeof(TMP)/sizeof(TMP[0]); i++){ + TMP[i] = _mm_xor_si128(TMP[i],TMP[i]); + } +} + +#endif /* #if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) */ + +#endif /* (_IPP32E>=_IPP32E_K1) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_encrypt_cbc.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_encrypt_cbc.c new file mode 100644 index 0000000..16e4778 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_encrypt_cbc.c @@ -0,0 +1,73 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 encryption/decryption +// +// Contents: +// cpEncryptSMS4_cbc() +// +*/ + +#include "owncp.h" +#include "pcpsms4.h" +#include "pcptool.h" +#include "pcpsms4_encrypt_cbc.h" + +/*F* +// +// Name: cpEncryptSMS4_cbc +// +// Purpose: SMS4-CBC encryption. +// +// Parameters: +// pIV pointer to the initialization vector +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// dataLen input/output buffer length (in bytes) +// pCtx pointer to the SMS4 context +// +*F*/ +IPP_OWN_DEFN (void, cpEncryptSMS4_cbc, (const Ipp8u* pIV, const Ipp8u* pSrc, Ipp8u* pDst, int dataLen, const IppsSMS4Spec* pCtx)) +{ + const Ipp32u* pRoundKeys = SMS4_RK(pCtx); + + /* read IV */ + __ALIGN16 Ipp32u iv[MBS_SMS4/sizeof(Ipp32u)]; + CopyBlock16(pIV, iv); + + /* do encryption */ + for(; dataLen>0; dataLen-=MBS_SMS4, pSrc+=MBS_SMS4, pDst+=MBS_SMS4) { + iv[0] ^= ((Ipp32u*)pSrc)[0]; + iv[1] ^= ((Ipp32u*)pSrc)[1]; + iv[2] ^= ((Ipp32u*)pSrc)[2]; + iv[3] ^= ((Ipp32u*)pSrc)[3]; + + cpSMS4_Cipher(pDst, (Ipp8u*)iv, pRoundKeys); + + iv[0] = ((Ipp32u*)pDst)[0]; + iv[1] = ((Ipp32u*)pDst)[1]; + iv[2] = ((Ipp32u*)pDst)[2]; + iv[3] = ((Ipp32u*)pDst)[3]; + } + + /* clear secret data */ + PurgeBlock(iv, sizeof(iv)); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_encrypt_cbc.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_encrypt_cbc.h new file mode 100644 index 0000000..70dc8ac --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_encrypt_cbc.h @@ -0,0 +1,37 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 encryption/decryption +// +// Contents: +// cpEncryptSMS4_cbc() +// +*/ + +#include "owncp.h" + +#if !defined _PCP_SMS4_ENCRYPT_CBC_H +#define _PCP_SMS4_ENCRYPT_CBC_H + +#define cpEncryptSMS4_cbc OWNAPI(cpEncryptSMS4_cbc) + IPP_OWN_DECL (void, cpEncryptSMS4_cbc, (const Ipp8u* pIV, const Ipp8u* pSrc, Ipp8u* pDst, int dataLen, const IppsSMS4Spec* pCtx)) + +#endif /* #if !defined _PCP_SMS4_ENCRYPT_CBC_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_gfni.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_gfni.h new file mode 100644 index 0000000..ce085c7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_gfni.h @@ -0,0 +1,302 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 encryption/decryption +// +// Contents: +// sBox() +// L() +// TRANSPOSE_INP() +// TRANSPOSE_OUT() +// +*/ + +#ifndef __SMS4_SBOX_GFNI512_H_ +#define __SMS4_SBOX_GFNI512_H_ + +#include "owndefs.h" +#include "owncp.h" + +#if (_IPP32E>=_IPP32E_K1) + +#if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) + +static __ALIGN64 Ipp8u swapBytes[] = { 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12, + 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12, + 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12, + 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12 }; +/* +// Not used in current pipeline +static __ALIGN32 Ipp8u permMask256[] = {0x00,0x00,0x00,0x00, 0x04,0x00,0x00,0x00, 0x01,0x00,0x00,0x00, 0x05,0x00,0x00,0x00, + 0x02,0x00,0x00,0x00, 0x06,0x00,0x00,0x00, 0x03,0x00,0x00,0x00, 0x07,0x00,0x00,0x00}; +*/ + +static __ALIGN64 Ipp8u permMask_in[] = {0,0x00,0x00,0x00, 4,0x00,0x00,0x00, 8,0x00,0x00,0x00, 12,0x00,0x00,0x00, + 1,0x00,0x00,0x00, 5,0x00,0x00,0x00, 9,0x00,0x00,0x00, 13,0x00,0x00,0x00, + 2,0x00,0x00,0x00, 6,0x00,0x00,0x00, 10,0x00,0x00,0x00, 14,0x00,0x00,0x00, + 3,0x00,0x00,0x00, 7,0x00,0x00,0x00, 11,0x00,0x00,0x00, 15,0x00,0x00,0x00 }; + +static __ALIGN64 Ipp8u permMask_out[] = {12,0x00,0x00,0x00, 8,0x00,0x00,0x00, 4,0x00,0x00,0x00, 0,0x00,0x00,0x00, + 13,0x00,0x00,0x00, 9,0x00,0x00,0x00, 5,0x00,0x00,0x00, 1,0x00,0x00,0x00, + 14,0x00,0x00,0x00, 10,0x00,0x00,0x00, 6,0x00,0x00,0x00, 2,0x00,0x00,0x00, + 15,0x00,0x00,0x00, 11,0x00,0x00,0x00, 7,0x00,0x00,0x00, 3,0x00,0x00,0x00}; + +static __ALIGN64 Ipp8u affineIn[] = { 0x52,0xBC,0x2D,0x02,0x9E,0x25,0xAC,0x34, 0x52,0xBC,0x2D,0x02,0x9E,0x25,0xAC,0x34, + 0x52,0xBC,0x2D,0x02,0x9E,0x25,0xAC,0x34, 0x52,0xBC,0x2D,0x02,0x9E,0x25,0xAC,0x34, + 0x52,0xBC,0x2D,0x02,0x9E,0x25,0xAC,0x34, 0x52,0xBC,0x2D,0x02,0x9E,0x25,0xAC,0x34, + 0x52,0xBC,0x2D,0x02,0x9E,0x25,0xAC,0x34, 0x52,0xBC,0x2D,0x02,0x9E,0x25,0xAC,0x34 }; +static __ALIGN64 Ipp8u affineOut[] = { 0x19,0x8b,0x6c,0x1e,0x51,0x8e,0x2d,0xd7, 0x19,0x8b,0x6c,0x1e,0x51,0x8e,0x2d,0xd7, + 0x19,0x8b,0x6c,0x1e,0x51,0x8e,0x2d,0xd7, 0x19,0x8b,0x6c,0x1e,0x51,0x8e,0x2d,0xd7, + 0x19,0x8b,0x6c,0x1e,0x51,0x8e,0x2d,0xd7, 0x19,0x8b,0x6c,0x1e,0x51,0x8e,0x2d,0xd7, + 0x19,0x8b,0x6c,0x1e,0x51,0x8e,0x2d,0xd7, 0x19,0x8b,0x6c,0x1e,0x51,0x8e,0x2d,0xd7 }; + +#define M512(mem) (*((__m512i*)(mem))) +#define M256(mem) (*((__m256i*)(mem))) +#define M128(mem) (*((__m128i*)(mem))) + +/* +// +// GF(256) is isomorfic. +// Encoding/decoding data of SM4 and AES are elements of GF(256). +// The difference in representation only. +// (It happend due to using different generating polynomials in SM4 and AES representations). +// Doing data conversion from SM4 to AES domain +// lets use AES specific intrinsics to perform less expensive SMS4 S-box computation. +// +// Original SMS4 S-box algorithm is converted to the following: +// +// - transform data from SMS4 representation to AES representation +// - compute S-box value +// - transform data back from AES representation to SMS4 representation +// +*/ + +/* +// sBox +*/ + +__FORCEINLINE __m512i sBox512(__m512i block) +{ + block = _mm512_gf2p8affine_epi64_epi8(block, M512(affineIn), 0x65); + block = _mm512_gf2p8affineinv_epi64_epi8(block, M512(affineOut), 0xd3); + return block; +} + +/* +// Not used in current pipeline +__FORCEINLINE __m256i sBox256(__m256i block) +{ + block = _mm256_gf2p8affine_epi64_epi8(block, M256(affineIn), 0x65); + block = _mm256_gf2p8affineinv_epi64_epi8(block, M256(affineOut), 0xd3); + return block; + +} +*/ + +__FORCEINLINE __m128i sBox128(__m128i block) +{ + block = _mm_gf2p8affine_epi64_epi8(block, M128(affineIn), 0x65); + block = _mm_gf2p8affineinv_epi64_epi8(block, M128(affineOut), 0xd3); + return block; + +} + +/* +// L +*/ + +__FORCEINLINE __m512i L512(__m512i x) +{ + __m512i rolled0 = _mm512_rol_epi32(x, 2); + __m512i rolled1 = _mm512_rol_epi32(x, 10); + __m512i temp = _mm512_xor_si512(rolled0, rolled1); + __m512i rolled2 = _mm512_rol_epi32(x, 18); + __m512i rolled3 = _mm512_rol_epi32(x, 24); + __m512i res = _mm512_ternarylogic_epi32(temp, rolled2, rolled3, 0x96); + return res; +} + +/* +// Not used in current pipeline +__FORCEINLINE __m256i L256(__m256i x) +{ + __m256i T = _mm256_xor_si256(_mm256_slli_epi32(x, 2), _mm256_srli_epi32(x,30)); + + T = _mm256_xor_si256(T, _mm256_slli_epi32 (x,10)); + T = _mm256_xor_si256(T, _mm256_srli_epi32 (x,22)); + + T = _mm256_xor_si256(T, _mm256_slli_epi32 (x,18)); + T = _mm256_xor_si256(T, _mm256_srli_epi32 (x,14)); + + T = _mm256_xor_si256(T, _mm256_slli_epi32 (x,24)); + T = _mm256_xor_si256(T, _mm256_srli_epi32 (x, 8)); + return T; +} +*/ + +__FORCEINLINE __m128i L128(__m128i x) +{ + __m128i rolled0 = _mm_rol_epi32(x, 2); + __m128i rolled1 = _mm_rol_epi32(x, 10); + __m128i temp = _mm_xor_si128(rolled1, rolled0); + __m128i rolled2 = _mm_rol_epi32(x, 18); + __m128i rolled3 = _mm_rol_epi32(x, 24); + __m128i res = _mm_ternarylogic_epi32(temp, rolled2, rolled3, 0x96); + return res; +} + + +/* +// TRANSPOSE_INP +*/ + +/* +// inp: T0, T1, T2, T3 +// out: K0, K1, K2, K3 +*/ +#define TRANSPOSE_INP_512(K0,K1,K2,K3, T0,T1,T2,T3) \ + K0 = _mm512_unpacklo_epi32(T0, T1); \ + K1 = _mm512_unpacklo_epi32(T2, T3); \ + K2 = _mm512_unpackhi_epi32(T0, T1); \ + K3 = _mm512_unpackhi_epi32(T2, T3); \ + \ + T0 = _mm512_unpacklo_epi64(K0, K1); \ + T1 = _mm512_unpacklo_epi64(K2, K3); \ + T2 = _mm512_unpackhi_epi64(K0, K1); \ + T3 = _mm512_unpackhi_epi64(K2, K3); \ + \ + K2 = _mm512_permutexvar_epi32(M512(permMask_in), T1); \ + K1 = _mm512_permutexvar_epi32(M512(permMask_in), T2); \ + K3 = _mm512_permutexvar_epi32(M512(permMask_in), T3); \ + K0 = _mm512_permutexvar_epi32(M512(permMask_in), T0) + +/* +// inp: T0, T1, T2, T3 +// out: K0, K1, K2, K3 +*/ + +/* +// Not used in current pipeline +#define TRANSPOSE_INP_256(K0,K1,K2,K3, T0,T1,T2,T3) \ + K0 = _mm256_unpacklo_epi32(T0, T1); \ + K1 = _mm256_unpacklo_epi32(T2, T3); \ + K2 = _mm256_unpackhi_epi32(T0, T1); \ + K3 = _mm256_unpackhi_epi32(T2, T3); \ + \ + T0 = _mm256_unpacklo_epi64(K0, K1); \ + T1 = _mm256_unpacklo_epi64(K2, K3); \ + T2 = _mm256_unpackhi_epi64(K0, K1); \ + T3 = _mm256_unpackhi_epi64(K2, K3); \ + \ + K2 = _mm256_permutevar8x32_epi32(T1, M256(permMask256)); \ + K1 = _mm256_permutevar8x32_epi32(T2, M256(permMask256)); \ + K3 = _mm256_permutevar8x32_epi32(T3, M256(permMask256)); \ + K0 = _mm256_permutevar8x32_epi32(T0, M256(permMask256)) +*/ + +#define TRANSPOSE_INP_128(K0,K1,K2,K3, T) \ + T = _mm_unpacklo_epi32(K0, K1); \ + K1 = _mm_unpackhi_epi32(K0, K1); \ + K0 = _mm_unpacklo_epi32(K2, K3); \ + K3 = _mm_unpackhi_epi32(K2, K3); \ + \ + K2 = _mm_unpacklo_epi64(K1, K3); \ + K3 = _mm_unpackhi_epi64(K1, K3); \ + K1 = _mm_unpackhi_epi64(T, K0); \ + K0 = _mm_unpacklo_epi64(T, K0) + +/* +// TRANSPOSE_OUT +*/ + +/* +// inp: K0, K1, K2, K3 +// out: T0, T1, T2, T3 +*/ + +#define TRANSPOSE_OUT_512(T0,T1,T2,T3, K0,K1,K2,K3) \ + T0 = _mm512_shuffle_i32x4(K0, K1, 0x44); \ + T1 = _mm512_shuffle_i32x4(K0, K1, 0xee); \ + T2 = _mm512_shuffle_i32x4(K2, K3, 0x44); \ + T3 = _mm512_shuffle_i32x4(K2, K3, 0xee); \ + \ + K0 = _mm512_shuffle_i32x4(T0, T2, 0x88); \ + K1 = _mm512_shuffle_i32x4(T0, T2, 0xdd); \ + K2 = _mm512_shuffle_i32x4(T1, T3, 0x88); \ + K3 = _mm512_shuffle_i32x4(T1, T3, 0xdd); \ + \ + K0 = _mm512_permutexvar_epi32(M512(permMask_out), K0);\ + K1 = _mm512_permutexvar_epi32(M512(permMask_out), K1);\ + K2 = _mm512_permutexvar_epi32(M512(permMask_out), K2);\ + K3 = _mm512_permutexvar_epi32(M512(permMask_out), K3);\ + \ +T0=K0,T1=K1,T2=K2,T3=K3 + + +/* +// inp: K0, K1, K2, K3 +// out: T0, T1, T2, T3 +*/ + +/* +// Not used in current pipeline +#define TRANSPOSE_OUT_256(T0,T1,T2,T3, K0,K1,K2,K3) \ + T0 = _mm256_unpacklo_epi32(K1, K0); \ + T1 = _mm256_unpacklo_epi32(K3, K2); \ + T2 = _mm256_unpackhi_epi32(K1, K0); \ + T3 = _mm256_unpackhi_epi32(K3, K2); \ + \ + K0 = _mm256_unpacklo_epi64(T1, T0); \ + K1 = _mm256_unpacklo_epi64(T3, T2); \ + K2 = _mm256_unpackhi_epi64(T1, T0); \ + K3 = _mm256_unpackhi_epi64(T3, T2); \ + \ + T0 = _mm256_permute2x128_si256(K0, K2, 0x20); \ + T1 = _mm256_permute2x128_si256(K1, K3, 0x20); \ + T2 = _mm256_permute2x128_si256(K0, K2, 0x31); \ + T3 = _mm256_permute2x128_si256(K1, K3, 0x31) +*/ + +#define TRANSPOSE_OUT_128(K0,K1,K2,K3, T) \ + T = _mm_unpacklo_epi32(K1, K0); \ + K0 = _mm_unpackhi_epi32(K1, K0); \ + K1 = _mm_unpacklo_epi32(K3, K2); \ + K3 = _mm_unpackhi_epi32(K3, K2); \ + \ + K2 = _mm_unpackhi_epi64(K1, T); \ + T = _mm_unpacklo_epi64(K1, T); \ + K1 = _mm_unpacklo_epi64(K3, K0); \ + K0 = _mm_unpackhi_epi64(K3, K0); \ + K3 = T + +//#define PR(X) printf("%08u %08u %08u %08u | %08u %08u %08u %08u | %08u %08u %08u %08u | %08u %08u %08u %08u\n",\ +// X.m512i_u32[0], X.m512i_u32[1], X.m512i_u32[2],\ +// X.m512i_u32[3], X.m512i_u32[4], X.m512i_u32[5],\ +// X.m512i_u32[6], X.m512i_u32[7], X.m512i_u32[8],\ +// X.m512i_u32[9], X.m512i_u32[10], X.m512i_u32[11],\ +// X.m512i_u32[12], X.m512i_u32[13], X.m512i_u32[14],\ +// X.m512i_u32[15]); + +#endif + +#endif /* if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) */ + +#endif /* __SMS4_SBOX_GFNI512_H_ */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_l9cn.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_l9cn.h new file mode 100644 index 0000000..8bf27d0 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_l9cn.h @@ -0,0 +1,186 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 encryption/decryption +// +// Contents: +// affine() +// sBox() +// L() +// TRANSPOSE_INP() +// TRANSPOSE_OUT() +// +*/ + +#if (_IPP>=_IPP_H9) || (_IPP32E>=_IPP32E_L9) + +#ifndef __SMS4_SBOX_L9_H_ +#define __SMS4_SBOX_L9_H_ + +#include "owndefs.h" +#include "owncp.h" + +static __ALIGN32 Ipp8u inpMaskLO[] = {0x65,0x41,0xfd,0xd9,0x0a,0x2e,0x92,0xb6,0x0f,0x2b,0x97,0xb3,0x60,0x44,0xf8,0xdc, + 0x65,0x41,0xfd,0xd9,0x0a,0x2e,0x92,0xb6,0x0f,0x2b,0x97,0xb3,0x60,0x44,0xf8,0xdc}; +static __ALIGN32 Ipp8u inpMaskHI[] = {0x00,0xc9,0x67,0xae,0x80,0x49,0xe7,0x2e,0x4a,0x83,0x2d,0xe4,0xca,0x03,0xad,0x64, + 0x00,0xc9,0x67,0xae,0x80,0x49,0xe7,0x2e,0x4a,0x83,0x2d,0xe4,0xca,0x03,0xad,0x64}; +static __ALIGN32 Ipp8u outMaskLO[] = {0xd3,0x59,0x38,0xb2,0xcc,0x46,0x27,0xad,0x36,0xbc,0xdd,0x57,0x29,0xa3,0xc2,0x48, + 0xd3,0x59,0x38,0xb2,0xcc,0x46,0x27,0xad,0x36,0xbc,0xdd,0x57,0x29,0xa3,0xc2,0x48}; +static __ALIGN32 Ipp8u outMaskHI[] = {0x00,0x50,0x14,0x44,0x89,0xd9,0x9d,0xcd,0xde,0x8e,0xca,0x9a,0x57,0x07,0x43,0x13, + 0x00,0x50,0x14,0x44,0x89,0xd9,0x9d,0xcd,0xde,0x8e,0xca,0x9a,0x57,0x07,0x43,0x13}; + +static __ALIGN32 Ipp8u maskSrows[] = {0x00,0x0d,0x0a,0x07,0x04,0x01,0x0e,0x0b,0x08,0x05,0x02,0x0f,0x0c,0x09,0x06,0x03, + 0x00,0x0d,0x0a,0x07,0x04,0x01,0x0e,0x0b,0x08,0x05,0x02,0x0f,0x0c,0x09,0x06,0x03}; + +static __ALIGN16 Ipp8u encKey[] = {0x63,0x63,0x63,0x63,0x63,0x63,0x63,0x63,0x63,0x63,0x63,0x63,0x63,0x63,0x63,0x63}; + +static __ALIGN32 Ipp8u lowBits4[] = {0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f, + 0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f}; + +static __ALIGN32 Ipp8u swapBytes[] = {3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12, + 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12}; + +static __ALIGN32 Ipp8u permMask[] = {0x00,0x00,0x00,0x00, 0x04,0x00,0x00,0x00, 0x01,0x00,0x00,0x00, 0x05,0x00,0x00,0x00, + 0x02,0x00,0x00,0x00, 0x06,0x00,0x00,0x00, 0x03,0x00,0x00,0x00, 0x07,0x00,0x00,0x00}; + +static __ALIGN32 Ipp8u affineIn[] = { 0x52,0xBC,0x2D,0x02,0x9E,0x25,0xAC,0x34, 0x52,0xBC,0x2D,0x02,0x9E,0x25,0xAC,0x34, + 0x52,0xBC,0x2D,0x02,0x9E,0x25,0xAC,0x34, 0x52,0xBC,0x2D,0x02,0x9E,0x25,0xAC,0x34 }; +static __ALIGN32 Ipp8u affineOut[] = { 0x19,0x8b,0x6c,0x1e,0x51,0x8e,0x2d,0xd7, 0x19,0x8b,0x6c,0x1e,0x51,0x8e,0x2d,0xd7, + 0x19,0x8b,0x6c,0x1e,0x51,0x8e,0x2d,0xd7, 0x19,0x8b,0x6c,0x1e,0x51,0x8e,0x2d,0xd7 }; + +#define M256(mem) (*((__m256i*)((Ipp8u*)(mem)))) +#define M128(mem) (*((__m128i*)((Ipp8u*)(mem)))) + +/* +// +// AES and SMS4 ciphers both based on composite field GF(2^8). +// This affine transformation transforms 16 bytes +// from SMS4 representation to AES representation or vise versa +// depending on passed masks. +// +*/ + +__FORCEINLINE __m256i affine(__m256i x, __m256i maskLO, __m256i maskHI) +{ + __m256i T1 = _mm256_and_si256(_mm256_srli_epi64(x, 4), M256(lowBits4)); + __m256i T0 = _mm256_and_si256(x, M256(lowBits4)); + T0 = _mm256_shuffle_epi8(maskLO, T0); + T1 = _mm256_shuffle_epi8(maskHI, T1); + return _mm256_xor_si256(T0, T1); +} + +__FORCEINLINE __m256i AES_ENC_LAST(__m256i x, __m128i key) +{ + __m128i t0 = _mm256_extracti128_si256(x, 0); + __m128i t1 = _mm256_extracti128_si256(x, 1); + t0 = _mm_aesenclast_si128(t0, key); + t1 = _mm_aesenclast_si128(t1, key); + x = _mm256_inserti128_si256(x, t0, 0); + x = _mm256_inserti128_si256(x, t1, 1); + return x; +} + +/* +// +// GF(256) is isomorfic. +// Encoding/decoding data of SM4 and AES are elements of GF(256). +// The difference in representation only. +// (It happend due to using different generating polynomials in SM4 and AES representations). +// Doing data conversion from SM4 to AES domain +// lets use AES specific intrinsics to perform less expensive SMS4 S-box computation. +// +// Original SMS4 S-box algorithm is converted to the following: +// +// - transform data from SMS4 representation to AES representation +// - compute S-box value using _mm_aesenclast_si128 with special key +// - re-shuffle data after _mm_aesenclast_si128 that shuffle it inside +// - transform data back from AES representation to SMS4 representation +// +*/ + +__FORCEINLINE __m256i sBox(__m256i block) +{ + block = affine(block, M256(inpMaskLO), M256(inpMaskHI)); + block = AES_ENC_LAST(block, M128(encKey)); + block = _mm256_shuffle_epi8(block, M256(maskSrows)); + block = affine(block, M256(outMaskLO), M256(outMaskHI)); + + return block; +} + +__FORCEINLINE __m256i L(__m256i x) +{ + __m256i T = _mm256_xor_si256(_mm256_slli_epi32(x, 2), _mm256_srli_epi32(x,30)); + + T = _mm256_xor_si256(T, _mm256_slli_epi32 (x,10)); + T = _mm256_xor_si256(T, _mm256_srli_epi32 (x,22)); + + T = _mm256_xor_si256(T, _mm256_slli_epi32 (x,18)); + T = _mm256_xor_si256(T, _mm256_srli_epi32 (x,14)); + + T = _mm256_xor_si256(T, _mm256_slli_epi32 (x,24)); + T = _mm256_xor_si256(T, _mm256_srli_epi32 (x, 8)); + return T; +} + +/* +// inp: T0, T1, T2, T3 +// out: K0, K1, K2, K3 +*/ +#define TRANSPOSE_INP(K0,K1,K2,K3, T0,T1,T2,T3) \ + K0 = _mm256_unpacklo_epi32(T0, T1); \ + K1 = _mm256_unpacklo_epi32(T2, T3); \ + K2 = _mm256_unpackhi_epi32(T0, T1); \ + K3 = _mm256_unpackhi_epi32(T2, T3); \ + \ + T0 = _mm256_unpacklo_epi64(K0, K1); \ + T1 = _mm256_unpacklo_epi64(K2, K3); \ + T2 = _mm256_unpackhi_epi64(K0, K1); \ + T3 = _mm256_unpackhi_epi64(K2, K3); \ + \ + K2 = _mm256_permutevar8x32_epi32(T1, M256(permMask)); \ + K1 = _mm256_permutevar8x32_epi32(T2, M256(permMask)); \ + K3 = _mm256_permutevar8x32_epi32(T3, M256(permMask)); \ + K0 = _mm256_permutevar8x32_epi32(T0, M256(permMask)) + +/* +// inp: K0, K1, K2, K3 +// out: T0, T1, T2, T3 +*/ +#define TRANSPOSE_OUT(T0,T1,T2,T3, K0,K1,K2,K3) \ + T0 = _mm256_unpacklo_epi32(K1, K0); \ + T1 = _mm256_unpacklo_epi32(K3, K2); \ + T2 = _mm256_unpackhi_epi32(K1, K0); \ + T3 = _mm256_unpackhi_epi32(K3, K2); \ + \ + K0 = _mm256_unpacklo_epi64(T1, T0); \ + K1 = _mm256_unpacklo_epi64(T3, T2); \ + K2 = _mm256_unpackhi_epi64(T1, T0); \ + K3 = _mm256_unpackhi_epi64(T3, T2); \ + \ + T0 = _mm256_permute2x128_si256(K0, K2, 0x20); \ + T1 = _mm256_permute2x128_si256(K1, K3, 0x20); \ + T2 = _mm256_permute2x128_si256(K0, K2, 0x31); \ + T3 = _mm256_permute2x128_si256(K1, K3, 0x31) + +#endif /* __SMS4_SBOX_L9_H_ */ + +#endif /* _IPP_G9, _IPP32E_L9 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_process_ctr.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_process_ctr.c new file mode 100644 index 0000000..b915ed5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_process_ctr.c @@ -0,0 +1,187 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 encryption/decryption +// +// Contents: +// cpProcessSMS4_ctr() +// +*/ + +#include "owncp.h" +#include "pcpsms4.h" +#include "pcptool.h" + +/* +// SMS4-CTR processing. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// pSrc == NULL +// pDst == NULL +// pCtrValue ==NULL +// ippStsContextMatchErr !VALID_SMS4_ID() +// ippStsLengthErr len <1 +// ippStsCTRSizeErr 128 < ctrNumBitSize < 1 +// ippStsCTRSizeErr data blocks number > 2^ctrNumBitSize +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// dataLen input/output buffer length (in bytes) +// pCtx pointer to rge SMS4 context +// pCtrValue pointer to the counter block +// ctrNumBitSize counter block size (bits) +// +// Note: +// counter will updated on return +// +*/ +IPP_OWN_DEFN (IppStatus, cpProcessSMS4_ctr, (const Ipp8u* pSrc, Ipp8u* pDst, int dataLen, const IppsSMS4Spec* pCtx, Ipp8u* pCtrValue, int ctrNumBitSize)) +{ + /* test context */ + IPP_BAD_PTR1_RET(pCtx); + /* test the context ID */ + IPP_BADARG_RET(!VALID_SMS4_ID(pCtx), ippStsContextMatchErr); + + /* test source, target and counter block pointers */ + IPP_BAD_PTR3_RET(pSrc, pDst, pCtrValue); + /* test stream length */ + IPP_BADARG_RET((dataLen<1), ippStsLengthErr); + + /* test counter block size */ + IPP_BADARG_RET(((MBS_SMS4*8)= 8 * sizeof(int) - 5 + // function can process data with any possible + // passed dataLen without counter overflow + */ + + int dataBlocksNum = dataLen >> 4; + if(dataLen & 15){ + dataBlocksNum++; + } + + IPP_BADARG_RET(dataBlocksNum > (1 << ctrNumBitSize), ippStsCTRSizeErr); + } + + { + __ALIGN16 Ipp8u TMP[2*MBS_SMS4+1]; + + /* + maskIV size = MBS_SMS4 + output size = MBS_SMS4 + counter size = MBS_SMS4 + maskValue size = 1 + */ + + Ipp8u* output = TMP; + Ipp8u* counter = TMP + MBS_SMS4; + + /* copy counter */ + CopyBlock16(pCtrValue, counter); + + /* do CTR processing */ + #if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + + Ipp8u* maskIV = TMP; + /* output is not used together with maskIV, so this is why buffer for both values is the same */ + Ipp8u* maskValue = TMP + 2*MBS_SMS4; + + if(dataLen>=4*MBS_SMS4) { + /* construct ctr mask */ + int n; + int maskPosition = (MBS_SMS4*8-ctrNumBitSize)/8; + *maskValue = (Ipp8u)(0xFF >> (MBS_SMS4*8-ctrNumBitSize)%8 ); + + for(n=0; n=_IPP32E_K1) + #if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) + if (IsFeatureEnabled(ippCPUID_AVX512GFNI)) { + processedLen = cpSMS4_CTR_gfni512(pDst, pSrc, dataLen, SMS4_RK(pCtx), maskIV, counter); + } + else + #endif /* #if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) */ + #endif /* (_IPP32E>=_IPP32E_K1) */ + if(IsFeatureEnabled(ippCPUID_AES) || IsFeatureEnabled(ippCPUID_AVX2VAES)) { + processedLen = cpSMS4_CTR_aesni(pDst, pSrc, dataLen, SMS4_RK(pCtx), maskIV, counter); + } + + pSrc += processedLen; + pDst += processedLen; + dataLen -= processedLen; + + } + #endif + + { + + /* block-by-block processing */ + while(dataLen>= MBS_SMS4) { + /* encrypt counter block */ + cpSMS4_Cipher((Ipp8u*)output, (Ipp8u*)counter, SMS4_RK(pCtx)); + /* compute ciphertext block */ + XorBlock16(pSrc, output, pDst); + /* increment counter block */ + StdIncrement(counter,MBS_SMS4*8, ctrNumBitSize); + + pSrc += MBS_SMS4; + pDst += MBS_SMS4; + dataLen -= MBS_SMS4; + } + + /* last data block processing */ + if(dataLen) { + /* encrypt counter block */ + cpSMS4_Cipher((Ipp8u*)output, (Ipp8u*)counter, SMS4_RK(pCtx)); + /* compute ciphertext block */ + XorBlock(pSrc, output, pDst,dataLen); + /* increment counter block */ + StdIncrement((Ipp8u*)counter,MBS_SMS4*8, ctrNumBitSize); + } + + } + + /* update counter */ + CopyBlock16(counter, pCtrValue); + + /* clear secret data */ + PurgeBlock(TMP, sizeof(TMP)); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_process_ofb8.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_process_ofb8.c new file mode 100644 index 0000000..97bd924 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_process_ofb8.c @@ -0,0 +1,81 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 encryption/decryption +// +// Contents: +// cpProcessSMS4_ofb8() +// +*/ + +#include "owncp.h" +#include "pcpsms4.h" +#include "pcptool.h" +#include "pcpsms4_process_ofb8.h" + +/* +// SMS4-OFB ecnryption/decryption +// +// Parameters: +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// dataLen input/output buffer length (in bytes) +// ofbBlkSize ofb block size (in bytes) +// pCtx pointer to the AES context +// pIV pointer to the initialization vector +*/ +IPP_OWN_DEFN (void, cpProcessSMS4_ofb8, (const Ipp8u *pSrc, Ipp8u *pDst, int dataLen, int ofbBlkSize, const IppsSMS4Spec* pCtx, Ipp8u* pIV)) +{ + __ALIGN16 Ipp32u tmpInpOut[2*MBS_SMS4/sizeof(Ipp32u)]; + + CopyBlock16(pIV, tmpInpOut); + + while(dataLen>=ofbBlkSize) { + /* block-by-block processing */ + cpSMS4_Cipher((Ipp8u*)tmpInpOut+MBS_SMS4, (Ipp8u*)tmpInpOut, SMS4_RK(pCtx)); + + /* store output and shift inpBuffer for the next OFB operation */ + if(ofbBlkSize==MBS_SMS4) { + ((Ipp32u*)pDst)[0] = tmpInpOut[0+MBS_SMS4/sizeof(Ipp32u)]^((Ipp32u*)pSrc)[0]; + ((Ipp32u*)pDst)[1] = tmpInpOut[1+MBS_SMS4/sizeof(Ipp32u)]^((Ipp32u*)pSrc)[1]; + ((Ipp32u*)pDst)[2] = tmpInpOut[2+MBS_SMS4/sizeof(Ipp32u)]^((Ipp32u*)pSrc)[2]; + ((Ipp32u*)pDst)[3] = tmpInpOut[3+MBS_SMS4/sizeof(Ipp32u)]^((Ipp32u*)pSrc)[3]; + tmpInpOut[0] = tmpInpOut[0+MBS_SMS4/sizeof(Ipp32u)]; + tmpInpOut[1] = tmpInpOut[1+MBS_SMS4/sizeof(Ipp32u)]; + tmpInpOut[2] = tmpInpOut[2+MBS_SMS4/sizeof(Ipp32u)]; + tmpInpOut[3] = tmpInpOut[3+MBS_SMS4/sizeof(Ipp32u)]; + } + else { + XorBlock(pSrc, tmpInpOut+MBS_SMS4/sizeof(Ipp32u), pDst, ofbBlkSize); + CopyBlock16((Ipp8u*)tmpInpOut+ofbBlkSize, tmpInpOut); + } + + pSrc += ofbBlkSize; + pDst += ofbBlkSize; + dataLen -= ofbBlkSize; + } + + /* update pIV */ + CopyBlock16((Ipp8u*)tmpInpOut, pIV); + + /* clear secret data */ + PurgeBlock(tmpInpOut, sizeof(tmpInpOut)); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_process_ofb8.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_process_ofb8.h new file mode 100644 index 0000000..5b5a615 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_process_ofb8.h @@ -0,0 +1,37 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 encryption/decryption +// +// Contents: +// cpProcessSMS4_ofb8() +// +*/ + +#include "owncp.h" + +#if !defined _PCP_SMS4_PROCESS_OFB8_H +#define _PCP_SMS4_PROCESS_OFB8_H + +#define cpProcessSMS4_ofb8 OWNAPI(cpProcessSMS4_ofb8) + IPP_OWN_DECL (void, cpProcessSMS4_ofb8, (const Ipp8u *pSrc, Ipp8u *pDst, int dataLen, int ofbBlkSize, const IppsSMS4Spec* pCtx, Ipp8u* pIV)) + +#endif /* #if !defined _PCP_SMS4_PROCESS_OFB8_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_y8cn.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_y8cn.h new file mode 100644 index 0000000..cb00754 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4_y8cn.h @@ -0,0 +1,161 @@ +/******************************************************************************* +* Copyright (C) 2019 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 encryption/decryption +// +// Contents: +// affine() +// sBox() +// L() +// TRANSPOSE_INP() +// TRANSPOSE_OUT() +// +*/ + +#if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + +#ifndef __SMS4_SBOX_Y8_H_ +#define __SMS4_SBOX_Y8_H_ + +#include "owndefs.h" +#include "owncp.h" + +static __ALIGN16 Ipp8u inpMaskLO[] = {0x65,0x41,0xfd,0xd9,0x0a,0x2e,0x92,0xb6,0x0f,0x2b,0x97,0xb3,0x60,0x44,0xf8,0xdc}; +static __ALIGN16 Ipp8u inpMaskHI[] = {0x00,0xc9,0x67,0xae,0x80,0x49,0xe7,0x2e,0x4a,0x83,0x2d,0xe4,0xca,0x03,0xad,0x64}; +static __ALIGN16 Ipp8u outMaskLO[] = {0xd3,0x59,0x38,0xb2,0xcc,0x46,0x27,0xad,0x36,0xbc,0xdd,0x57,0x29,0xa3,0xc2,0x48}; +static __ALIGN16 Ipp8u outMaskHI[] = {0x00,0x50,0x14,0x44,0x89,0xd9,0x9d,0xcd,0xde,0x8e,0xca,0x9a,0x57,0x07,0x43,0x13}; + +static __ALIGN16 Ipp8u encKey[] = {0x63,0x63,0x63,0x63,0x63,0x63,0x63,0x63,0x63,0x63,0x63,0x63,0x63,0x63,0x63,0x63}; +static __ALIGN16 Ipp8u maskSrows[] = {0x00,0x0d,0x0a,0x07,0x04,0x01,0x0e,0x0b,0x08,0x05,0x02,0x0f,0x0c,0x09,0x06,0x03}; + +static __ALIGN16 Ipp8u lowBits4[] = {0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f}; + +static __ALIGN16 Ipp8u swapBytes[] = {3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12}; + +static __ALIGN16 Ipp8u affineIn[] = { 0x52,0xBC,0x2D,0x02,0x9E,0x25,0xAC,0x34, 0x52,0xBC,0x2D,0x02,0x9E,0x25,0xAC,0x34 }; +static __ALIGN16 Ipp8u affineOut[] = { 0x19,0x8b,0x6c,0x1e,0x51,0x8e,0x2d,0xd7, 0x19,0x8b,0x6c,0x1e,0x51,0x8e,0x2d,0xd7 }; + +#define M128(mem) (*((__m128i*)((Ipp8u*)(mem)))) + +/* +// +// AES and SMS4 ciphers both based on composite field GF(2^8). +// This affine transformation transforms 16 bytes +// from SMS4 representation to AES representation or vise versa +// depending on passed masks. +// +*/ + +__FORCEINLINE __m128i affine(__m128i x, __m128i maskLO, __m128i maskHI) +{ + __m128i T1 = _mm_and_si128(_mm_srli_epi64(x, 4), M128(lowBits4)); + __m128i T0 = _mm_and_si128(x, M128(lowBits4)); + T0 = _mm_shuffle_epi8(maskLO, T0); + T1 = _mm_shuffle_epi8(maskHI, T1); + return _mm_xor_si128(T0, T1); +} + +/* +// +// GF(256) is isomorfic. +// Encoding/decoding data of SM4 and AES are elements of GF(256). +// The difference in representation only. +// (It happend due to using different generating polynomials in SM4 and AES representations). +// Doing data conversion from SM4 to AES domain +// lets use AES specific intrinsics to perform less expensive SMS4 S-box computation. +// +// Original SMS4 S-box algorithm is converted to the following: +// +// - transform data from SMS4 representation to AES representation +// - compute S-box value using _mm_aesenclast_si128 with special key +// - re-shuffle data after _mm_aesenclast_si128 that shuffle it inside +// - transform data back from AES representation to SMS4 representation +// +*/ + +__FORCEINLINE __m128i sBox(__m128i block) +{ + block = affine(block, M128(inpMaskLO), M128(inpMaskHI)); + block = _mm_aesenclast_si128(block, M128(encKey)); + block = _mm_shuffle_epi8(block, M128(maskSrows)); + block = affine(block, M128(outMaskLO), M128(outMaskHI)); + + return block; +} + +#if (_IPP==_IPP_I0) || (_IPP32E==_IPP32E_N0) +__FORCEINLINE __m128i L(__m128i x) +{ + __m128i T = _mm_slli_epi32(x, 2); + T = _mm_xor_si128(T, _mm_srli_epi32(x, 30)); + + T = _mm_xor_si128(T, _mm_slli_epi32(x, 10)); + T = _mm_xor_si128(T, _mm_srli_epi32(x, 22)); + + T = _mm_xor_si128(T, _mm_slli_epi32(x, 18)); + T = _mm_xor_si128(T, _mm_srli_epi32(x, 14)); + + T = _mm_xor_si128(T, _mm_slli_epi32(x, 24)); + T = _mm_xor_si128(T, _mm_srli_epi32(x, 8)); + return T; +} +#else +static __ALIGN16 Ipp8u ROL8[] = { 3,0,1,2, 7,4,5,6, 11,8,9,10, 15,12,13,14 }; +static __ALIGN16 Ipp8u ROL16[] = { 2,3,0,1, 6,7,4,5, 10,11,8,9, 14,15,12,13 }; +static __ALIGN16 Ipp8u ROL24[] = { 1,2,3,0, 5,6,7,4, 9,10,11,8, 13,14,15,12 }; + +__FORCEINLINE __m128i L(__m128i x) +{ + __m128i rol2 = _mm_xor_si128(_mm_slli_epi32(x, 2), _mm_srli_epi32(x, 30)); + __m128i rol24 = _mm_shuffle_epi8(x, M128(ROL24)); + __m128i rol10 = _mm_shuffle_epi8(rol2, M128(ROL8)); + __m128i rol18 = _mm_shuffle_epi8(rol2, M128(ROL16)); + __m128i R = _mm_xor_si128(rol24, _mm_xor_si128(rol18, _mm_xor_si128(rol2, rol10))); + return R; +} +#endif + +#define TRANSPOSE_INP(K0,K1,K2,K3, T) \ + T = _mm_unpacklo_epi32(K0, K1); \ + K1 = _mm_unpackhi_epi32(K0, K1); \ + K0 = _mm_unpacklo_epi32(K2, K3); \ + K3 = _mm_unpackhi_epi32(K2, K3); \ + \ + K2 = _mm_unpacklo_epi64(K1, K3); \ + K3 = _mm_unpackhi_epi64(K1, K3); \ + K1 = _mm_unpackhi_epi64(T, K0); \ + K0 = _mm_unpacklo_epi64(T, K0) + +#define TRANSPOSE_OUT(K0,K1,K2,K3, T) \ + T = _mm_unpacklo_epi32(K1, K0); \ + K0 = _mm_unpackhi_epi32(K1, K0); \ + K1 = _mm_unpacklo_epi32(K3, K2); \ + K3 = _mm_unpackhi_epi32(K3, K2); \ + \ + K2 = _mm_unpackhi_epi64(K1, T); \ + T = _mm_unpacklo_epi64(K1, T); \ + K1 = _mm_unpacklo_epi64(K3, K0); \ + K0 = _mm_unpackhi_epi64(K3, K0); \ + K3 = T + +#endif /* __SMS4_SBOX_Y8_H_ */ + +#endif /* _IPP_P8, _IPP32E_Y8 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4authccm.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4authccm.h new file mode 100644 index 0000000..f27c8f7 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4authccm.h @@ -0,0 +1,89 @@ +/******************************************************************************* +* Copyright (C) 2017 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Message Authentication Algorithm +// Internal Definitions and Internal Functions Prototypes +// +// +*/ + +#if !defined(_CP_SMS4_CCM_H) +#define _CP_SMS4_CCM_H + +#include "pcpsms4.h" + +struct _cpSMS4_CCM { + Ipp32u idCtx; /* CCM ID */ + + Ipp64u msgLen; /* length of message to be processed */ + Ipp64u lenProcessed; /* message length has been processed */ + Ipp32u tagLen; /* length of authentication tag */ + Ipp32u counterVal; /* currnt couter value */ + Ipp8u ctr0[MBS_SMS4]; /* counter value */ + Ipp8u s0[MBS_SMS4]; /* S0 = ENC(CTR0) content */ + Ipp8u si[MBS_SMS4]; /* Si = ENC(CTRi) content */ + Ipp8u blk[MBS_SMS4]; /* temporary data container */ + Ipp8u mac[MBS_SMS4]; /* current MAC value */ + + Ipp8u cipher[sizeof(IppsSMS4Spec)]; +}; + +/* alignment */ +#define SMS4CCM_ALIGNMENT ((int)(sizeof(void*))) + +/* +// access macros +*/ +#define SMS4CCM_SET_ID(stt) ((stt)->idCtx = (Ipp32u)idCtxAESCCM ^ (Ipp32u)IPP_UINT_PTR(stt)) +#define SMS4CCM_MSGLEN(stt) ((stt)->msgLen) +#define SMS4CCM_LENPRO(stt) ((stt)->lenProcessed) +#define SMS4CCM_TAGLEN(stt) ((stt)->tagLen) +#define SMS4CCM_COUNTER(stt) ((stt)->counterVal) +#define SMS4CCM_CTR0(stt) ((stt)->ctr0) +#define SMS4CCM_S0(stt) ((stt)->s0) +#define SMS4CCM_Si(stt) ((stt)->si) +#define SMS4CCM_BLK(stt) ((stt)->blk) +#define SMS4CCM_MAC(stt) ((stt)->mac) +#define SMS4CCM_CIPHER(stt) (IppsSMS4Spec*)(&((stt)->cipher)) + +/* valid context ID */ +#define VALID_SMS4CCM_ID(ctx) ((((ctx)->idCtx) ^ (Ipp32u)IPP_UINT_PTR((ctx))) == (Ipp32u)idCtxAESCCM) + +static int cpSizeofCtx_SMS4CCM(void) +{ + return sizeof(IppsSMS4_CCMState); +} + +/* Counter block formatter */ +static +Ipp8u* CounterEnc(Ipp32u* pBuffer, int fmt, Ipp64u counter) +{ + #if (IPP_ENDIAN == IPP_LITTLE_ENDIAN) + pBuffer[0] = ENDIANNESS(IPP_HIDWORD(counter)); + pBuffer[1] = ENDIANNESS(IPP_LODWORD(counter)); + #else + pBuffer[0] = IPP_HIDWORD(counter); + pBuffer[1] = IPP_LODWORD(counter); + #endif + return (Ipp8u*)pBuffer + 8 - fmt; +} + +#endif /* _CP_SMS4_CCM_H*/ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4cbcl9cn.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4cbcl9cn.c new file mode 100644 index 0000000..ded5ccf --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4cbcl9cn.c @@ -0,0 +1,239 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 CBC decryption +// +// Contents: +// cpSMS4_CBC_dec_aesni() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpsms4.h" + +#if (_IPP>=_IPP_H9) || (_IPP32E>=_IPP32E_L9) + +#include "pcpsms4_l9cn.h" + +IPP_OWN_DEFN (int, cpSMS4_CBC_dec_aesni, (Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, Ipp8u* pIV)) +{ + __ALIGN16 __m256i TMP[17]; + /* + TMP[ 0] = T0 + TMP[ 1] = T1 + TMP[ 2] = T2 + TMP[ 3] = T3 + TMP[ 4] = K0 + TMP[ 5] = K1 + TMP[ 6] = K2 + TMP[ 7] = K3 + TMP[ 8] = P0 + TMP[ 9] = P1 + TMP[10] = P2 + TMP[11] = P3 + TMP[12] = Q0 + TMP[13] = Q1 + TMP[14] = Q2 + TMP[15] = Q3 + TMP[16] = IV + */ + + TMP[16] = _mm256_castsi128_si256(_mm_loadu_si128((__m128i*)(pIV))); + + int processedLen = len -(len % (24*MBS_SMS4)); + int n; + for(n=0; n=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + +#include "pcpsms4_y8cn.h" + +/* +// 4*MBS_SMS4 processing +*/ + +static int cpSMS4_CBC_dec_aesni_x4(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, Ipp8u* pIV) +{ + __ALIGN16 __m128i TMP[9]; + /* + TMP[0] = T + TMP[1] = T0 + TMP[2] = T1 + TMP[3] = T2 + TMP[4] = T3 + TMP[5] = K0 + TMP[6] = K1 + TMP[7] = K2 + TMP[8] = K3 + */ + TMP[1] = _mm_loadu_si128((__m128i*)(pIV)); + + int processedLen = len & -(4*MBS_SMS4); + int n; + for(n=0; n=_IPP_H9) || (_IPP32E>=_IPP32E_L9) +IPP_OWN_DEFN (int, cpSMS4_CBC_dec_aesni_x12, (Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, Ipp8u* pIV)) +#else +IPP_OWN_DEFN (int, cpSMS4_CBC_dec_aesni, (Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, Ipp8u* pIV)) +#endif +{ + __ALIGN16 __m128i TMP[19]; + /* + TMP[ 0] = T + TMP[ 1] = U + TMP[ 2] = V + TMP[ 3] = T0 + TMP[ 4] = T1 + TMP[ 5] = T2 + TMP[ 6] = T3 + TMP[ 7] = K0 + TMP[ 8] = K1 + TMP[ 9] = K2 + TMP[10] = K3 + TMP[11] = P0 + TMP[12] = P1 + TMP[13] = P2 + TMP[14] = P3 + TMP[15] = Q0 + TMP[16] = Q1 + TMP[17] = Q2 + TMP[18] = Q3 + */ + TMP[3] = _mm_loadu_si128((__m128i*)(pIV)); + + int processedLen = len -(len % (12*MBS_SMS4)); + int n; + for(n=0; n=_IPP_H9) || (_IPP32E>=_IPP32E_L9) + +#include "pcpsms4_l9cn.h" + +static __ALIGN32 Ipp8u endiannes_swap[] = {12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3, + 12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3}; + +static __ALIGN32 Ipp8u endiannes[] = {15,14,13,12, 11,10,9,8, 7,6,5,4, 3,2,1,0, + 15,14,13,12, 11,10,9,8, 7,6,5,4, 3,2,1,0}; + +static __ALIGN32 Ipp8u two256[] = {2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; +static __ALIGN16 Ipp8u one256[] = {1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; + +__INLINE __m128i inc128(__m128i x) +{ + __m128i t = _mm_add_epi64(x, M128(one256)); + x = _mm_cmpeq_epi64(t, _mm_setzero_si128()); + t = _mm_sub_epi64(t, _mm_slli_si128(x, sizeof(Ipp64u))); + return t; +} + +__INLINE __m256i inc256(__m256i x) +{ + __m256i t = _mm256_add_epi64(x, M256(two256)); + x = _mm256_cmpeq_epi64(t, _mm256_setzero_si256()); + t = _mm256_sub_epi64(t, _mm256_slli_si256(x, sizeof(Ipp64u))); + return t; +} +__INLINE __m256i inc256_2(__m256i x) +{ + __m256i t = _mm256_add_epi64(x, M256(one256)); + x = _mm256_cmpeq_epi64(t, _mm256_setzero_si256()); + t = _mm256_sub_epi64(t, _mm256_slli_si256(x, sizeof(Ipp64u))); + + t = _mm256_add_epi64(t, M256(one256)); + x = _mm256_cmpeq_epi64(t, _mm256_setzero_si256()); + t = _mm256_sub_epi64(t, _mm256_slli_si256(x, sizeof(Ipp64u))); + return t; +} + +IPP_OWN_DEFN (int, cpSMS4_CTR_aesni, (Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, const Ipp8u* pCtrMask, Ipp8u* pCtr)) +{ + int processedLen = len & -(16*MBS_SMS4); + + if(processedLen) { + int n; + + __ALIGN16 __m256i TMP[16]; + /* + TMP[ 0] = T0 + TMP[ 1] = T1 + TMP[ 2] = T2 + TMP[ 3] = T3 + TMP[ 4] = K0 + TMP[ 5] = K1 + TMP[ 6] = K2 + TMP[ 7] = K3 + TMP[ 8] = P0 + TMP[ 9] = P1 + TMP[10] = P2 + TMP[11] = P3 + TMP[12] = ctr + TMP[13] = mask + TMP[14] = unch + */ + + /* read string counter and convert to numerical */ + TMP[12] = _mm256_shuffle_epi8(_mm256_castsi128_si256(_mm_loadu_si128((__m128i*)pCtr)), M256(endiannes)); + + /* read string mask and convert to numerical */ + TMP[13] = _mm256_shuffle_epi8(_mm256_castsi128_si256(_mm_loadu_si128((__m128i*)pCtrMask)), M256(endiannes)); + + /* upchanged counter bits */ + TMP[14] = _mm256_andnot_si256(TMP[13], TMP[12]); + + TMP[12] = _mm256_inserti128_si256(TMP[12], inc128(_mm256_castsi256_si128(TMP[12])), 1); + TMP[13]= _mm256_inserti128_si256(TMP[13], _mm256_castsi256_si128(TMP[13]), 1); + TMP[14]= _mm256_inserti128_si256(TMP[14], _mm256_castsi256_si128(TMP[14]), 1); + TMP[12] = _mm256_and_si256(TMP[12], TMP[13]); + + for(n=0; n=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + +#include "pcpsms4_y8cn.h" + +static __ALIGN16 Ipp8u one128[] = {1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; +static __ALIGN16 Ipp8u endiannes[] = {15,14,13,12, 11,10,9,8, 7,6,5,4, 3,2,1,0}; +static __ALIGN16 Ipp8u endiannes_swap[] = {12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3}; + +__INLINE __m128i inc128(__m128i x) +{ + __m128i t = _mm_add_epi64(x, M128(one128)); + x = _mm_cmpeq_epi64(t, _mm_setzero_si128()); + t = _mm_sub_epi64(t, _mm_slli_si128(x, sizeof(Ipp64u))); + return t; +} + +#if (_IPP>=_IPP_H9) || (_IPP32E>=_IPP32E_L9) +IPP_OWN_DEFN (int, cpSMS4_CTR_aesni_x4, (Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, const Ipp8u* pCtrMask, Ipp8u* pCtr)) +#else +IPP_OWN_DEFN (int, cpSMS4_CTR_aesni, (Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey, const Ipp8u* pCtrMask, Ipp8u* pCtr)) +#endif +{ + __ALIGN16 __m128i TMP[8]; + /* + TMP[ 0] = T + TMP[ 1] = K0 + TMP[ 2] = K1 + TMP[ 3] = K2 + TMP[ 4] = K3 + TMP[ 5] = ctrUnchanged + TMP[ 6] = ctrMask + TMP[ 7] = ctr + */ + + int processedLen = len & -(4*MBS_SMS4); + int n; + + TMP[6] = _mm_loadu_si128((__m128i*)pCtrMask); + TMP[7] = _mm_loadu_si128((__m128i*)pCtr); + + TMP[6] = _mm_shuffle_epi8(TMP[6], M128(endiannes)); + TMP[7] = _mm_shuffle_epi8(TMP[7], M128(endiannes)); + TMP[5] = _mm_andnot_si128(TMP[6], TMP[7]); + + for(n=0; ncfbBlkSize || cfbBlkSize>MBS_SMS4) +// ippStsUnderRunErr 0!=(len%cfbBlkSize) +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// len input/output buffer length (in bytes) +// cfbBlkSize CFB block size (in bytes) +// pCtx pointer to the SMS4 context +// pIV pointer to the initialization vector +// +*F*/ +static +void cpDecryptSMS4_cfb(const Ipp8u* pIV, + const Ipp8u* pSrc, Ipp8u* pDst, int nBlocks, int cfbBlkSize, + const IppsSMS4Spec* pCtx) +{ + __ALIGN16 Ipp32u TMP[3*MBS_SMS4/sizeof(Ipp32u)]; + + /* + tmpInp size = 2*MBS_SMS4/sizeof(Ipp32u) + tmpOut size = MBS_SMS4/sizeof(Ipp32u) + */ + + Ipp32u* tmpInp = TMP; + Ipp32u* tmpOut = TMP + 2*MBS_SMS4/sizeof(Ipp32u); + + /* read IV */ + CopyBlock16(pIV, tmpInp); + + /* decrypt data block-by-block of cfbLen each */ + while(nBlocks) { + /* decryption */ + cpSMS4_Cipher((Ipp8u*)tmpOut, (Ipp8u*)tmpInp, SMS4_RK(pCtx)); + + /* store output and put feedback into the input buffer (tmpInp) */ + if( cfbBlkSize==MBS_SMS4 && pSrc!=pDst) { + ((Ipp32u*)pDst)[0] = tmpOut[0]^((Ipp32u*)pSrc)[0]; + ((Ipp32u*)pDst)[1] = tmpOut[1]^((Ipp32u*)pSrc)[1]; + ((Ipp32u*)pDst)[2] = tmpOut[2]^((Ipp32u*)pSrc)[2]; + ((Ipp32u*)pDst)[3] = tmpOut[3]^((Ipp32u*)pSrc)[3]; + + tmpInp[0] = ((Ipp32u*)pSrc)[0]; + tmpInp[1] = ((Ipp32u*)pSrc)[1]; + tmpInp[2] = ((Ipp32u*)pSrc)[2]; + tmpInp[3] = ((Ipp32u*)pSrc)[3]; + } + else { + int n; + for(n=0; ncfbBlkSize) || (MBS_SMS4=_IPP32E_K1) + #if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) + if (IsFeatureEnabled(ippCPUID_AVX512GFNI)) { + + __ALIGN16 Ipp8u IV[MBS_SMS4]; + CopyBlock16(pIV, IV); + + if(len/cfbBlkSize >= 4) + { + cpSMS4_CFB_dec_gfni512(pDst, pSrc, len, cfbBlkSize, (Ipp32u*)SMS4_RK(pCtx), IV); /* pipeline */ + + int processedLen = len - (len % (4*cfbBlkSize)); + pSrc += processedLen; + pDst += processedLen; + len = len - processedLen; + } + if(len) + { + cpDecryptSMS4_cfb(IV, pSrc, pDst, len/cfbBlkSize, cfbBlkSize, pCtx); /* tail */ + } + } + else + #endif + #endif + { + cpDecryptSMS4_cfb(pIV, pSrc, pDst, len/cfbBlkSize, cfbBlkSize, pCtx); + } + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4decryptctr.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4decryptctr.c new file mode 100644 index 0000000..56808af --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4decryptctr.c @@ -0,0 +1,66 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 encryption/decryption +// +// Contents: +// ippsSMS4DecryptCTR() +// +*/ + +#include "owncp.h" +#include "pcpsms4.h" +#include "pcptool.h" + +/*F* +// Name: ippsSMS4DecryptCTR +// +// Purpose: SMS4-CRT encryption. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// pSrc == NULL +// pDst == NULL +// pCtrValue ==NULL +// ippStsContextMatchErr !VALID_SMS4_ID() +// ippStsLengthErr len <1 +// ippStsCTRSizeErr 128 < ctrNumBitSize < 1 +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// len input/output buffer length (in bytes) +// pCtx pointer to rge SMS4 context +// pCtrValue pointer to the counter block +// ctrNumBitSize counter block size (bits) +// +// Note: +// counter will updated on return +// +*F*/ + +IPPFUN(IppStatus, ippsSMS4DecryptCTR,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx, + Ipp8u* pCtrValue, int ctrNumBitSize)) +{ + return cpProcessSMS4_ctr(pSrc, pDst, len, pCtx, pCtrValue, ctrNumBitSize); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4decryptecb.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4decryptecb.c new file mode 100644 index 0000000..ed3b1d8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4decryptecb.c @@ -0,0 +1,95 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 encryption/decryption +// +// Contents: +// ippsSMS4DecryptECB() +// +*/ + +#include "owncp.h" +#include "pcpsms4.h" +#include "pcptool.h" + +/*F* +// Name: ippsSMS4DecryptECB +// +// Purpose: SMS4-ECB decryption. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// pSrc == NULL +// pDst == NULL +// ippStsContextMatchErr !VALID_SMS4_ID() +// ippStsLengthErr len <1 +// ippStsUnderRunErr 0!=(len%MBS_SMS4) +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// len input/output buffer length (in bytes) +// pCtx pointer to the SMS4 context +// +*F*/ +IPPFUN(IppStatus, ippsSMS4DecryptECB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsSMS4Spec* pCtx)) +{ + /* test context */ + IPP_BAD_PTR1_RET(pCtx); + /* test the context ID */ + IPP_BADARG_RET(!VALID_SMS4_ID(pCtx), ippStsContextMatchErr); + + /* test source and target buffer pointers */ + IPP_BAD_PTR2_RET(pSrc, pDst); + /* test stream length */ + IPP_BADARG_RET((len<1), ippStsLengthErr); + /* test stream integrity */ + IPP_BADARG_RET((len&(MBS_SMS4-1)), ippStsUnderRunErr); + + /* do encryption */ + #if (_IPP32E>=_IPP32E_K1) + #if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) + if (IsFeatureEnabled(ippCPUID_AVX512GFNI)) { + int processedLen = cpSMS4_ECB_gfni512(pDst, pSrc, len, SMS4_DRK(pCtx)); + pSrc += processedLen; + pDst += processedLen; + len -= processedLen; + } + else + #endif /* #if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920)) */ + #endif /* (_IPP32E>=_IPP32E_K1) */ + #if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + if(IsFeatureEnabled(ippCPUID_AES) || IsFeatureEnabled(ippCPUID_AVX2VAES)) { + int processedLen = cpSMS4_ECB_aesni(pDst, pSrc, len, SMS4_DRK(pCtx)); + pSrc += processedLen; + pDst += processedLen; + len -= processedLen; + } + else + #endif + + for(; len>0; len-=MBS_SMS4, pSrc+=MBS_SMS4, pDst+=MBS_SMS4) + cpSMS4_Cipher(pDst, pSrc, SMS4_DRK(pCtx)); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4decryptofb.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4decryptofb.c new file mode 100644 index 0000000..e68e667 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4decryptofb.c @@ -0,0 +1,78 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 encryption/decryption +// +// Contents: +// ippsSMS4DecryptOFB() +// +*/ + +#include "owncp.h" +#include "pcpsms4.h" +#include "pcptool.h" +#include "pcpsms4_process_ofb8.h" + +/*F* +// Name: ippsSMS4DecryptOFB +// +// Purpose: Decrypts byte data stream according to SMS4 in OFB mode. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// pSrc == NULL +// pDst == NULL +// pIV == NULL +// ippStsContextMatchErr !VALID_SMS4_ID() +// ippStsLengthErr len <1 +// ippStsOFBSizeErr (1>ofbBlkSize || ofbBlkSize>MBS_SMS4) +// ippStsUnderRunErr (len%ofbBlkSize) +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// len input/output buffer length (in bytes) +// ofbBlkSize OFB block size (in bytes) +// pCtx pointer to the SMS4 context +// pIV pointer to the initialization vector +*F*/ +IPPFUN(IppStatus, ippsSMS4DecryptOFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int ofbBlkSize, + const IppsSMS4Spec* pCtx, + Ipp8u* pIV)) +{ + /* test context */ + IPP_BAD_PTR1_RET(pCtx); + /* test the context ID */ + IPP_BADARG_RET(!VALID_SMS4_ID(pCtx), ippStsContextMatchErr); + + /* test source, target buffers and initialization pointers */ + IPP_BAD_PTR3_RET(pSrc, pIV, pDst); + /* test stream length */ + IPP_BADARG_RET((len<1), ippStsLengthErr); + /* test OFB value */ + IPP_BADARG_RET(((1>ofbBlkSize) || (MBS_SMS4=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + +#include "pcpsms4_y8cn.h" + +/* +// (1-3)*MBS_SMS4 processing +*/ + +static int cpSMS4_ECB_aesni_tail(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey) +{ + __ALIGN16 __m128i TMP[6]; + /* + TMP[0] = T + TMP[1] = K0 + TMP[2] = K1 + TMP[3] = K2 + TMP[4] = K3 + TMP[5] = key4 + */ + + TMP[2] = _mm_setzero_si128(); + TMP[3] = _mm_setzero_si128(); + TMP[4] = _mm_setzero_si128(); + + switch (len) { + case (3*MBS_SMS4): + TMP[3] = _mm_shuffle_epi8(_mm_loadu_si128((__m128i*)(pInp+2*MBS_SMS4)), M128(swapBytes)); + case (2*MBS_SMS4): + TMP[2] = _mm_shuffle_epi8(_mm_loadu_si128((__m128i*)(pInp+1*MBS_SMS4)), M128(swapBytes)); + case (1*MBS_SMS4): + TMP[1] = _mm_shuffle_epi8(_mm_loadu_si128((__m128i*)(pInp+0*MBS_SMS4)), M128(swapBytes)); + break; + default: return 0; + } + TRANSPOSE_INP(TMP[1],TMP[2],TMP[3],TMP[4], TMP[0]); + + { + int itr; + for(itr=0; itr<8; itr++, pRKey+=4) { + TMP[5] = _mm_loadu_si128((__m128i*)pRKey); + + /* initial xors */ + TMP[0] = _mm_shuffle_epi32(TMP[5], 0x00); /* broadcast(key4 TMP[0]) */ + TMP[0] = _mm_xor_si128(TMP[0], TMP[2]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[3]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[4]); + /* Sbox */ + TMP[0] = sBox(TMP[0]); + /* Sbox done, now L */ + TMP[1] = _mm_xor_si128(_mm_xor_si128(TMP[1], TMP[0]), L(TMP[0])); + + /* initial xors */ + TMP[0] = _mm_shuffle_epi32(TMP[5], 0x55); /* broadcast(key4 TMP[1]) */ + TMP[0] = _mm_xor_si128(TMP[0], TMP[3]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[4]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[1]); + /* Sbox */ + TMP[0] = sBox(TMP[0]); + /* Sbox done, now L */ + TMP[2] = _mm_xor_si128(_mm_xor_si128(TMP[2], TMP[0]), L(TMP[0])); + + /* initial xors */ + TMP[0] = _mm_shuffle_epi32(TMP[5], 0xAA); /* broadcast(key4 TMP[2]) */ + TMP[0] = _mm_xor_si128(TMP[0], TMP[4]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[1]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[2]); + /* Sbox */ + TMP[0] = sBox(TMP[0]); + /* Sbox done, now L */ + TMP[3] = _mm_xor_si128(_mm_xor_si128(TMP[3], TMP[0]), L(TMP[0])); + + /* initial xors */ + TMP[0] = _mm_shuffle_epi32(TMP[5], 0xFF); /* broadcast(key4 TMP[3]) */ + TMP[0] = _mm_xor_si128(TMP[0], TMP[1]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[2]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[3]); + /* Sbox */ + TMP[0] = sBox(TMP[0]); + /* Sbox done, now L */ + TMP[4] = _mm_xor_si128(_mm_xor_si128(TMP[4], TMP[0]), L(TMP[0])); + } + } + + TRANSPOSE_OUT(TMP[1],TMP[2],TMP[3],TMP[4], TMP[0]); + TMP[4] = _mm_shuffle_epi8(TMP[4], M128(swapBytes)); + TMP[3] = _mm_shuffle_epi8(TMP[3], M128(swapBytes)); + TMP[2] = _mm_shuffle_epi8(TMP[2], M128(swapBytes)); + TMP[1] = _mm_shuffle_epi8(TMP[1], M128(swapBytes)); + + switch (len) { + case (3*MBS_SMS4): + _mm_storeu_si128((__m128i*)(pOut+2*MBS_SMS4), TMP[2]); + case (2*MBS_SMS4): + _mm_storeu_si128((__m128i*)(pOut+1*MBS_SMS4), TMP[3]); + case (1*MBS_SMS4): + _mm_storeu_si128((__m128i*)(pOut+0*MBS_SMS4), TMP[4]); + break; + } + + /* clear secret data */ + for(Ipp32u i = 0; i < sizeof(TMP)/sizeof(TMP[0]); i++){ + TMP[i] = _mm_xor_si128(TMP[i],TMP[i]); + } + + return len; +} + +/* +// 4*MBS_SMS4 processing +*/ +static +int cpSMS4_ECB_aesni_x4(Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey) +{ + __ALIGN16 __m128i TMP[5]; + /* + TMP[0] = T + TMP[1] = K0 + TMP[2] = K1 + TMP[3] = K2 + TMP[4] = K3 + */ + int processedLen = len & -(4*MBS_SMS4); + int n; + for(n=0; n=_IPP_H9) || (_IPP32E>=_IPP32E_L9) +#define cpSMS4_ECB_aesni_x12 OWNAPI(cpSMS4_ECB_aesni_x12) + IPP_OWN_DECL (int, cpSMS4_ECB_aesni_x12, (Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey)) + IPP_OWN_DEFN (int, cpSMS4_ECB_aesni_x12, (Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey)) +#else +#define cpSMS4_ECB_aesni OWNAPI(cpSMS4_ECB_aesni) + IPP_OWN_DECL (int, cpSMS4_ECB_aesni, (Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey)) + IPP_OWN_DEFN (int, cpSMS4_ECB_aesni, (Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey)) +#endif +{ + __ALIGN16 __m128i TMP[15]; + /* + TMP[ 0] = T + TMP[ 1] = U + TMP[ 2] = V + TMP[ 3] = K0 + TMP[ 4] = K1 + TMP[ 5] = K2 + TMP[ 6] = K3 + TMP[ 7] = P0 + TMP[ 8] = P1 + TMP[ 9] = P2 + TMP[10] = P3 + TMP[11] = Q0 + TMP[12] = Q1 + TMP[13] = Q2 + TMP[14] = Q3 + */ + + int processedLen = len -(len % (12*MBS_SMS4)); + int n; + for(n=0; n=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + +#include "pcpsms4_y8cn.h" + +__INLINE __m128i Ltag(__m128i x) +{ + __m128i T = _mm_slli_epi32(x, 13); + T = _mm_xor_si128(T, _mm_srli_epi32 (x,19)); + T = _mm_xor_si128(T, _mm_slli_epi32 (x,23)); + T = _mm_xor_si128(T, _mm_srli_epi32 (x, 9)); + return T; +} + +/* +// compute round keys +*/ + +#define cpSMS4_SetRoundKeys_aesni OWNAPI(cpSMS4_SetRoundKeys_aesni) + IPP_OWN_DECL (void, cpSMS4_SetRoundKeys_aesni, (Ipp32u* pRoundKey, const Ipp8u* pSecretKey)) + +IPP_OWN_DEFN (void, cpSMS4_SetRoundKeys_aesni, (Ipp32u* pRoundKey, const Ipp8u* pSecretKey)) +{ + __ALIGN16 __m128i TMP[5]; + /* + TMP[0] = T + TMP[1] = K0 + TMP[2] = K1 + TMP[3] = K2 + TMP[4] = K3 + */ + TMP[1] = _mm_cvtsi32_si128((Ipp32s)(ENDIANNESS32(((Ipp32u*)pSecretKey)[0]) ^ SMS4_FK[0])); + TMP[2] = _mm_cvtsi32_si128((Ipp32s)(ENDIANNESS32(((Ipp32u*)pSecretKey)[1]) ^ SMS4_FK[1])); + TMP[3] = _mm_cvtsi32_si128((Ipp32s)(ENDIANNESS32(((Ipp32u*)pSecretKey)[2]) ^ SMS4_FK[2])); + TMP[4] = _mm_cvtsi32_si128((Ipp32s)(ENDIANNESS32(((Ipp32u*)pSecretKey)[3]) ^ SMS4_FK[3])); + + const Ipp32u* pCK = SMS4_CK; + + int itr; + for(itr=0; itr<8; itr++) { + /* initial xors */ + TMP[0] = _mm_cvtsi32_si128((Ipp32s)pCK[0]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[2]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[3]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[4]); + /* Sbox */ + TMP[0] = sBox(TMP[0]); + /* Sbox done, now Ltag */ + TMP[1] = _mm_xor_si128(_mm_xor_si128(TMP[1], TMP[0]), Ltag(TMP[0])); + pRoundKey[0] = (Ipp32u)_mm_cvtsi128_si32(TMP[1]); + + /* initial xors */ + TMP[0] = _mm_cvtsi32_si128((Ipp32s)pCK[1]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[3]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[4]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[1]); + /* Sbox */ + TMP[0] = sBox(TMP[0]); + /* Sbox done, now Ltag */ + TMP[2] = _mm_xor_si128(_mm_xor_si128(TMP[2], TMP[0]), Ltag(TMP[0])); + pRoundKey[1] = (Ipp32u)_mm_cvtsi128_si32(TMP[2]); + + /* initial xors */ + TMP[0] = _mm_cvtsi32_si128((Ipp32s)pCK[2]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[4]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[1]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[2]); + /* Sbox */ + TMP[0] = sBox(TMP[0]); + /* Sbox done, now Ltag */ + TMP[3] = _mm_xor_si128(_mm_xor_si128(TMP[3], TMP[0]), Ltag(TMP[0])); + pRoundKey[2] = (Ipp32u)_mm_cvtsi128_si32(TMP[3]); + + /* initial xors */ + TMP[0] = _mm_cvtsi32_si128((Ipp32s)pCK[3]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[1]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[2]); + TMP[0] = _mm_xor_si128(TMP[0], TMP[3]); + /* Sbox */ + TMP[0] = sBox(TMP[0]); + /* Sbox done, now Ltag */ + TMP[4] = _mm_xor_si128(_mm_xor_si128(TMP[4], TMP[0]), Ltag(TMP[0])); + pRoundKey[3] = (Ipp32u)_mm_cvtsi128_si32(TMP[4]); + + pCK += 4; + pRoundKey += 4; + } + + /* clear secret data */ + for(Ipp32u i = 0; i < sizeof(TMP)/sizeof(TMP[0]); i++){ + TMP[i] = _mm_xor_si128(TMP[i],TMP[i]); + } +} + +#endif /* _IPP_P8, _IPP32E_Y8 */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4ecbl9cn.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4ecbl9cn.c new file mode 100644 index 0000000..6314ba8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4ecbl9cn.c @@ -0,0 +1,223 @@ +/******************************************************************************* +* Copyright (C) 2014 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 ECB decryption +// +// Contents: +// cpSMS4_ECB_aesni() +// +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpsms4.h" +#include "pcptool.h" + +#if (_IPP>=_IPP_H9) || (_IPP32E>=_IPP32E_L9) + +#include "pcpsms4_l9cn.h" + +IPP_OWN_DEFN (int, cpSMS4_ECB_aesni, (Ipp8u* pOut, const Ipp8u* pInp, int len, const Ipp32u* pRKey)) +{ + __ALIGN16 __m256i TMP[16]; + /* + TMP[ 0] = T0 + TMP[ 1] = T1 + TMP[ 2] = T2 + TMP[ 3] = T3 + TMP[ 4] = K0 + TMP[ 5] = K1 + TMP[ 6] = K2 + TMP[ 7] = K3 + TMP[ 8] = P0 + TMP[ 9] = P1 + TMP[10] = P2 + TMP[11] = P3 + TMP[12] = Q0 + TMP[13] = Q1 + TMP[14] = Q2 + TMP[15] = Q3 + */ + + int processedLen = len -(len % (24*MBS_SMS4)); + int n; + for(n=0; ncfbBlkSize || cfbBlkSize>MBS_SMS4) +// ippStsUnderRunErr 0!=(len%cfbBlkSize) +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// len input/output buffer length (in bytes) +// cfbBlkSize CFB block size (in bytes) +// pCtx pointer to the SMS4 context +// pIV pointer to the initialization vector +// +*F*/ +IPPFUN(IppStatus, ippsSMS4EncryptCFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int cfbBlkSize, + const IppsSMS4Spec* pCtx, + const Ipp8u* pIV)) +{ + /* test context */ + IPP_BAD_PTR1_RET(pCtx); + /* test the context ID */ + IPP_BADARG_RET(!VALID_SMS4_ID(pCtx), ippStsContextMatchErr); + + /* test source, target buffers and initialization pointers */ + IPP_BAD_PTR3_RET(pSrc, pIV, pDst); + /* test stream length */ + IPP_BADARG_RET((len<1), ippStsLengthErr); + /* test CFB value */ + IPP_BADARG_RET(((1>cfbBlkSize) || (MBS_SMS4=cfbBlkSize) { + int n; + + /* encryption */ + cpSMS4_Cipher((Ipp8u*)tmpOut, (Ipp8u*)tmpInp, SMS4_RK(pCtx)); + + /* store output and put feedback into the input buffer (tmpInp) */ + if( cfbBlkSize==MBS_SMS4 && pSrc!=pDst) { + tmpInp[0] = ((Ipp32u*)pDst)[0] = tmpOut[0]^((Ipp32u*)pSrc)[0]; + tmpInp[1] = ((Ipp32u*)pDst)[1] = tmpOut[1]^((Ipp32u*)pSrc)[1]; + tmpInp[2] = ((Ipp32u*)pDst)[2] = tmpOut[2]^((Ipp32u*)pSrc)[2]; + tmpInp[3] = ((Ipp32u*)pDst)[3] = tmpOut[3]^((Ipp32u*)pSrc)[3]; + } + else { + for(n=0; n=_IPP32E_K1) + #if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) + if (IsFeatureEnabled(ippCPUID_AVX512GFNI)) { + int processedLen = cpSMS4_ECB_gfni512(pDst, pSrc, len, SMS4_ERK(pCtx)); + pSrc += processedLen; + pDst += processedLen; + len -= processedLen; + } + else + #endif /* #if defined (__INTEL_COMPILER) || defined (__INTEL_LLVM_COMPILER) || !defined (_MSC_VER) || (_MSC_VER >= 1920) */ + #endif /* (_IPP32E>=_IPP32E_K1) */ + #if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + if(IsFeatureEnabled(ippCPUID_AES) || IsFeatureEnabled(ippCPUID_AVX2VAES)) { + int processedLen = cpSMS4_ECB_aesni(pDst, pSrc, len, SMS4_ERK(pCtx)); + pSrc += processedLen; + pDst += processedLen; + len -= processedLen; + } + else + #endif + + for(; len>0; len-=MBS_SMS4, pSrc+=MBS_SMS4, pDst+=MBS_SMS4) + cpSMS4_Cipher(pDst, pSrc, SMS4_RK(pCtx)); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4encryptofb.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4encryptofb.c new file mode 100644 index 0000000..28c4f1e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpsms4encryptofb.c @@ -0,0 +1,78 @@ +/******************************************************************************* +* Copyright (C) 2013 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SMS4 encryption/decryption +// +// Contents: +// ippsSMS4EncryptOFB() +// +*/ + +#include "owncp.h" +#include "pcpsms4.h" +#include "pcptool.h" +#include "pcpsms4_process_ofb8.h" + +/*F* +// Name: ippsSMS4EncryptOFB +// +// Purpose: Encrypts byte data stream according to SMS4 in OFB mode. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx == NULL +// pSrc == NULL +// pDst == NULL +// pIV == NULL +// ippStsContextMatchErr !VALID_SMS4_ID() +// ippStsLengthErr len <1 +// ippStsOFBSizeErr (1>ofbBlkSize || ofbBlkSize>MBS_SMS4) +// ippStsUnderRunErr (len%ofbBlkSize) +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source data buffer +// pDst pointer to the target data buffer +// len input/output buffer length (in bytes) +// ofbBlkSize OFB block size (in bytes) +// pCtx pointer to the SMS4 context +// pIV pointer to the initialization vector +*F*/ +IPPFUN(IppStatus, ippsSMS4EncryptOFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int ofbBlkSize, + const IppsSMS4Spec* pCtx, + Ipp8u* pIV)) +{ + /* test context */ + IPP_BAD_PTR1_RET(pCtx); + /* test the context ID */ + IPP_BADARG_RET(!VALID_SMS4_ID(pCtx), ippStsContextMatchErr); + + /* test source, target buffers and initialization pointers */ + IPP_BAD_PTR3_RET(pSrc, pIV, pDst); + /* test stream length */ + IPP_BADARG_RET((len<1), ippStsLengthErr); + /* test OFB value */ + IPP_BADARG_RET(((1>ofbBlkSize) || (MBS_SMS4=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + if(IsFeatureEnabled(ippCPUID_AES) || IsFeatureEnabled(ippCPUID_AVX2VAES)) + cpSMS4_SetRoundKeys_aesni(SMS4_RK(pCtx), pSecretKey); + else + #endif + cpSMS4_SetRoundKeys(SMS4_RK(pCtx), pSecretKey); + + /* set deccryption round keys */ + { + int n; + for(n=0; n < SMS4_ROUND_KEYS_NUM; n++) { + SMS4_DRK(pCtx)[n] = SMS4_RK(pCtx)[SMS4_ROUND_KEYS_NUM - n-1]; + } + } + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcptdesctrca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcptdesctrca.c new file mode 100644 index 0000000..98eab5c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcptdesctrca.c @@ -0,0 +1,180 @@ +/******************************************************************************* +* Copyright (C) 2004 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Encrypt/Decrypt byte data stream according to TDES (CTR mode) +// +// Contents: +// ippsTDESEncryptCTR() +// ippsTDESDecryptCTR() +// +// +*/ + +#include "owndefs.h" + +#include "owncp.h" +#include "pcpdes.h" +#include "pcptool.h" + +static +IppStatus TDES_CTR(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsDESSpec* pCtx1, + const IppsDESSpec* pCtx2, + const IppsDESSpec* pCtx3, + Ipp8u* pCtrValue, int ctrNumBitSize) +{ + Ipp64u counter; + Ipp64u output; + + /* test contexts */ + IPP_BAD_PTR3_RET(pCtx1, pCtx2, pCtx3); + + IPP_BADARG_RET(!VALID_DES_ID(pCtx1), ippStsContextMatchErr); + IPP_BADARG_RET(!VALID_DES_ID(pCtx2), ippStsContextMatchErr); + IPP_BADARG_RET(!VALID_DES_ID(pCtx3), ippStsContextMatchErr); + /* test source, target and counter block pointers */ + IPP_BAD_PTR3_RET(pSrc, pDst, pCtrValue); + /* test stream length */ + IPP_BADARG_RET((len<1), ippStsLengthErr); + /* test counter block size */ + IPP_BADARG_RET(((MBS_DES*8)= MBS_DES) { + /* encrypt counter block */ + output = Cipher_DES(counter, DES_EKEYS(pCtx1), DESspbox); + output = Cipher_DES(output, DES_DKEYS(pCtx2), DESspbox); + output = Cipher_DES(output, DES_EKEYS(pCtx3), DESspbox); + /* compute ciphertext block */ + XorBlock8(pSrc, &output, pDst); + /* encrement counter block */ + StdIncrement((Ipp8u*)&counter,MBS_DES*8, ctrNumBitSize); + + pSrc += MBS_DES; + pDst += MBS_DES; + len -= MBS_DES; + } + /* + // encrypt last data block + */ + if(len) { + /* encrypt counter block */ + output = Cipher_DES(counter, DES_EKEYS(pCtx1), DESspbox); + output = Cipher_DES(output, DES_DKEYS(pCtx2), DESspbox); + output = Cipher_DES(output, DES_EKEYS(pCtx3), DESspbox); + /* compute ciphertext block */ + XorBlock(pSrc, &output, pDst,len); + /* encrement counter block */ + StdIncrement((Ipp8u*)&counter,MBS_DES*8, ctrNumBitSize); + } + + /* update counter */ + CopyBlock8(&counter, pCtrValue); + + return ippStsNoErr; +} + +/*F* +// Name: ippsTDESEncryptCTR +// +// Purpose: Encrypt byte data stream according to TDES in CTR mode. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx1 == NULL +// pCtx2 == NULL +// pCtx3 == NULL +// pSrc == NULL +// pDst == NULL +// pCtrValue ==NULL +// ippStsContextMatchErr pCtx1->idCtx != idCtxDES +// pCtx2->idCtx != idCtxDES +// pCtx3->idCtx != idCtxDES +// ippStsLengthErr len <1 +// ippStsCTRSizeErr 64 < ctrNumBitSize < 1 +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source data stream +// pDst pointer to the target data stream +// len plaintext stream length (bytes) +// pCtx1-pCtx3 DES contexts +// pCtrValue pointer to the counter block +// ctrNumBitSize counter block size (bits) +// +// Note: +// counter will updated on return +// +*F*/ + +IPPFUN(IppStatus, ippsTDESEncryptCTR,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsDESSpec* pCtx1, + const IppsDESSpec* pCtx2, + const IppsDESSpec* pCtx3, + Ipp8u* pCtrValue, int ctrNumBitSize)) +{ + return TDES_CTR(pSrc,pDst,len, pCtx1,pCtx2,pCtx3, pCtrValue,ctrNumBitSize); +} + +/*F* +// Name: ippsTDESDecryptCTR +// +// Purpose: Decrypt byte data stream according to TDES in CTR mode. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx1 == NULL +// pCtx2 == NULL +// pCtx3 == NULL +// pSrc == NULL +// pDst == NULL +// pCtrValue ==NULL +// ippStsContextMatchErr pCtx1->idCtx != idCtxDES +// pCtx2->idCtx != idCtxDES +// pCtx3->idCtx != idCtxDES +// ippStsLengthErr len <1 +// ippStsCTRSizeErr 64 < ctrNumBitSize < 1 +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source data stream +// pDst pointer to the target data stream +// len plaintext stream length (bytes) +// pCtx1-pCtx3 DES contexts +// pCtrValue pointer to the counter block +// ctrNumBitSize counter block size (bits) +// +// Note: +// counter will updated on return +// +*F*/ + +IPPFUN(IppStatus, ippsTDESDecryptCTR,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsDESSpec* pCtx1, + const IppsDESSpec* pCtx2, + const IppsDESSpec* pCtx3, + Ipp8u* pCtrValue, int ctrNumBitSize)) +{ + return TDES_CTR(pSrc,pDst,len, pCtx1,pCtx2,pCtx3, pCtrValue,ctrNumBitSize); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcptdesdecryptcbcca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcptdesdecryptcbcca.c new file mode 100644 index 0000000..3ae3242 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcptdesdecryptcbcca.c @@ -0,0 +1,129 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Decrypt byte data stream according to TDES (CBC mode) +// +// Contents: +// ippsTDESDecryptCBC() +// +// +*/ + +#include "owndefs.h" + +#include "owncp.h" +#include "pcpdes.h" +#include "pcptool.h" + + +/*F* +// Name: ippsTDESDecryptCBC +// +// Purpose: Decrypt byte data stream according to TDES in CBC mode. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx1 == NULL +// pCtx2 == NULL +// pCtx3 == NULL +// pSrc == NULL +// pDst == NULL +// ippStsContextMatchErr pCtx1->idCtx != idCtxDES +// pCtx2->idCtx != idCtxDES +// pCtx3->idCtx != idCtxDES +// ippStsLengthErr length <1 +// ippStsUnderRunErr (length&7) +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source byte data stream +// pDst pointer to the target byte data stream +// length plaintext stream length (bytes) +// pCtx1-3 pointers to the DES context +// pIV pointer to the init vector +// padding the padding scheme indicator +// +*F*/ +IPPFUN(IppStatus, ippsTDESDecryptCBC,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsDESSpec* pCtx1, + const IppsDESSpec* pCtx2, + const IppsDESSpec* pCtx3, + const Ipp8u* pIV, + IppsPadding padding)) +{ + /* test contexts */ + IPP_BAD_PTR3_RET(pCtx1, pCtx2, pCtx3); + + IPP_BADARG_RET(!VALID_DES_ID(pCtx1), ippStsContextMatchErr); + IPP_BADARG_RET(!VALID_DES_ID(pCtx2), ippStsContextMatchErr); + IPP_BADARG_RET(!VALID_DES_ID(pCtx3), ippStsContextMatchErr); + /* test source and destination pointers */ + IPP_BAD_PTR3_RET(pSrc, pDst, pIV); + /* test stream length */ + IPP_BADARG_RET((len<1), ippStsLengthErr); + + /* force ippPaddingNONE padding */ + if(ippPaddingNONE!=padding) + padding = ippPaddingNONE; + /* test stream integrity */ + //IPP_BADARG_RET(((length&7) && (ippPaddingNONE==padding)), ippStsUnderRunErr); + IPP_BADARG_RET((len&7), ippStsUnderRunErr); + + { + int nBlocks = len/MBS_DES; + + /* read IV */ + Ipp64u iv; + CopyBlock8(pIV, &iv); + + /* misaligned source and/or target */ + /*if( (IPP_UINT_PTR(pSrc) & 0x7) || (IPP_UINT_PTR(pDst) & 0x7) ) {*/ + if( (IPP_UINT_PTR(pSrc) & 0x7) || (IPP_UINT_PTR(pDst) & 0x7) || pSrc==pDst) { + int n; + for(n=0; nidCtx != idCtxDES +// pCtx2->idCtx != idCtxDES +// pCtx3->idCtx != idCtxDES +// ippStsLengthErr len <1 +// ippStsCFBSizeErr (1>cfbBlkSize || cfbBlkSize>MBS_DES) +// ippStsUnderRunErr (len%cfbBlkSize) +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source byte data stream +// pDst pointer to the target byte data stream +// len plaintext stream length (bytes) +// cfbBlkSize CFB block size (bytes) +// pCtx1-3 pointers to the DES context +// pIV pointer to the init vector +// padding the padding scheme indicator +// +*F*/ +IPPFUN(IppStatus, ippsTDESDecryptCFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int cfbBlkSize, + const IppsDESSpec* pCtx1, const IppsDESSpec* pCtx2, const IppsDESSpec* pCtx3, + const Ipp8u* pIV, + IppsPadding padding)) +{ + Ipp64u inpBuffer[2]; + Ipp64u outBuffer; + + /* test contexts */ + IPP_BAD_PTR3_RET(pCtx1, pCtx2, pCtx3); + + IPP_BADARG_RET(!VALID_DES_ID(pCtx1), ippStsContextMatchErr); + IPP_BADARG_RET(!VALID_DES_ID(pCtx2), ippStsContextMatchErr); + IPP_BADARG_RET(!VALID_DES_ID(pCtx3), ippStsContextMatchErr); + /* test source and destination pointers */ + IPP_BAD_PTR3_RET(pSrc, pDst, pIV); + /* test stream length */ + IPP_BADARG_RET((len<1), ippStsLengthErr); + /* test CFB value */ + IPP_BADARG_RET(((1>cfbBlkSize) || (MBS_DES=cfbBlkSize) { + int n; + + /* decryption */ + outBuffer = Cipher_DES(inpBuffer[0], DES_EKEYS(pCtx1), DESspbox); + outBuffer = Cipher_DES(outBuffer, DES_DKEYS(pCtx2), DESspbox); + outBuffer = Cipher_DES(outBuffer, DES_EKEYS(pCtx3), DESspbox); + + /* store output and put feedback into the buffer */ + for(n=0; nidCtx != idCtxDES +// pCtx2->idCtx != idCtxDES +// pCtx3->idCtx != idCtxDES +// ippStsLengthErr length < 1 +// ippStsUnderRunErr (length & 7) +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source byte data stream +// pDst pointer to the target byte data stream +// length plaintext stream length (bytes) +// pCtx1-3 pointers to the DES context +// padding the padding scheme indicator +// +*F*/ +IPPFUN(IppStatus, ippsTDESDecryptECB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsDESSpec* pCtx1, + const IppsDESSpec* pCtx2, + const IppsDESSpec* pCtx3, + IppsPadding padding)) +{ + /* test contexts */ + IPP_BAD_PTR3_RET(pCtx1, pCtx2, pCtx3); + + IPP_BADARG_RET(!VALID_DES_ID(pCtx1), ippStsContextMatchErr); + IPP_BADARG_RET(!VALID_DES_ID(pCtx2), ippStsContextMatchErr); + IPP_BADARG_RET(!VALID_DES_ID(pCtx3), ippStsContextMatchErr); + /* test source and destination pointers */ + IPP_BAD_PTR2_RET(pSrc, pDst); + /* test stream length */ + IPP_BADARG_RET((len<1), ippStsLengthErr); + + /* force ippPaddingNONE padding */ + if(ippPaddingNONE!=padding) + padding = ippPaddingNONE; + /* test stream integrity */ + IPP_BADARG_RET((len&7), ippStsUnderRunErr); + + { + int nBlocks = len/MBS_DES; + + /* misaligned source and/or target */ + if( (IPP_UINT_PTR(pSrc) & 0x7) || (IPP_UINT_PTR(pDst) & 0x7) ) { + int n; + for(n=0; nidCtx != idCtxDES +// pCtx2->idCtx != idCtxDES +// pCtx3->idCtx != idCtxDES +// ippStsLengthErr length <1 +// ippStsUnderRunErr (length & 7) +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source byte data stream +// pDst pointer to the target byte data stream +// length plaintext strem length (bytes) +// pCtx1-3 pointers to the DES context +// pIV pointer to the init vector +// padding the padding scheme indicator +// +*F*/ +IPPFUN(IppStatus, ippsTDESEncryptCBC,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsDESSpec* pCtx1, + const IppsDESSpec* pCtx2, + const IppsDESSpec* pCtx3, + const Ipp8u* pIV, + IppsPadding padding)) +{ + /* test contexts */ + IPP_BAD_PTR3_RET(pCtx1, pCtx2, pCtx3); + + IPP_BADARG_RET(!VALID_DES_ID(pCtx1), ippStsContextMatchErr); + IPP_BADARG_RET(!VALID_DES_ID(pCtx2), ippStsContextMatchErr); + IPP_BADARG_RET(!VALID_DES_ID(pCtx3), ippStsContextMatchErr); + /* test source and destination pointers */ + IPP_BAD_PTR3_RET(pSrc, pDst, pIV); + /* test stream length */ + IPP_BADARG_RET((len<1), ippStsLengthErr); + + /* force ippPaddingNONE padding */ + if(ippPaddingNONE!=padding) + padding = ippPaddingNONE; + /* test stream integrity */ + //IPP_BADARG_RET(((length&7) && (ippPaddingNONE==padding)), ippStsUnderRunErr); + IPP_BADARG_RET((len&7), ippStsUnderRunErr); + + { + int nBlocks = len/MBS_DES; + + /* read IV */ + Ipp64u iv; + CopyBlock8(pIV, &iv); + + /* misaligned source and/or target */ + /*if( (IPP_UINT_PTR(pSrc) & 0x7) || (IPP_UINT_PTR(pDst) & 0x7) ) {*/ + if( (IPP_UINT_PTR(pSrc) & 0x7) || (IPP_UINT_PTR(pDst) & 0x7) || pSrc==pDst) { + int n; + for(n=0; nidCtx != idCtxDES +// pCtx2->idCtx != idCtxDES +// pCtx3->idCtx != idCtxDES +// ippStsLengthErr len <1 +// ippStsCFBSizeErr (1>cfbBlkSize || cfbBlkSize>MBS_DES) +// ippStsUnderRunErr (len%cfbBlkSize) +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source byte data stream +// pDst pointer to the target byte data stream +// len plaintext stream length (bytes) +// cfbBlkSize CFB block size (bytes) +// pCtx1-3 pointers to the DES context +// pIV pointer to the init vector +// padding the padding scheme indicator +// +*F*/ +IPPFUN(IppStatus, ippsTDESEncryptCFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int cfbBlkSize, + const IppsDESSpec* pCtx1, const IppsDESSpec* pCtx2, const IppsDESSpec* pCtx3, + const Ipp8u* pIV, + IppsPadding padding)) +{ + Ipp64u inpBuffer; + Ipp64u outBuffer; + + /* test contexts */ + IPP_BAD_PTR3_RET(pCtx1, pCtx2, pCtx3); + + IPP_BADARG_RET(!VALID_DES_ID(pCtx1), ippStsContextMatchErr); + IPP_BADARG_RET(!VALID_DES_ID(pCtx2), ippStsContextMatchErr); + IPP_BADARG_RET(!VALID_DES_ID(pCtx3), ippStsContextMatchErr); + /* test source and destination pointers */ + IPP_BAD_PTR3_RET(pSrc, pDst, pIV); + /* test stream length */ + IPP_BADARG_RET((len<1), ippStsLengthErr); + /* test CFB value */ + IPP_BADARG_RET(((1>cfbBlkSize) || (MBS_DES=cfbBlkSize) { + int n; + + /* encryption */ + outBuffer = Cipher_DES(inpBuffer, DES_EKEYS(pCtx1), DESspbox); + outBuffer = Cipher_DES(outBuffer, DES_DKEYS(pCtx2), DESspbox); + outBuffer = Cipher_DES(outBuffer, DES_EKEYS(pCtx3), DESspbox); + + /* store output and put feedback into the buffer */ + for(n=0; nidCtx != idCtxDES +// pCtx2->idCtx != idCtxDES +// pCtx3->idCtx != idCtxDES +// ippStsLengthErr length <1 +// ippStsUnderRunErr (length&7) && (ippPaddingNONE==padding) +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source byte data stream +// pDst pointer to the target byte data stream +// length plaintext stream length (bytes) +// pCtx1-3 pointers to the DES context +// padding the padding scheme indicator +// +*F*/ +IPPFUN(IppStatus, ippsTDESEncryptECB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, + const IppsDESSpec* pCtx1, + const IppsDESSpec* pCtx2, + const IppsDESSpec* pCtx3, + IppsPadding padding)) +{ + /* test contexts */ + IPP_BAD_PTR3_RET(pCtx1, pCtx2, pCtx3); + + IPP_BADARG_RET(!VALID_DES_ID(pCtx1), ippStsContextMatchErr); + IPP_BADARG_RET(!VALID_DES_ID(pCtx2), ippStsContextMatchErr); + IPP_BADARG_RET(!VALID_DES_ID(pCtx3), ippStsContextMatchErr); + /* test source and destination pointers */ + IPP_BAD_PTR2_RET(pSrc, pDst); + /* test stream length */ + IPP_BADARG_RET((len<1), ippStsLengthErr); + + /* force ippPaddingNONE padding */ + if(ippPaddingNONE!=padding) + padding = ippPaddingNONE; + /* test stream integrity */ + //IPP_BADARG_RET(((length&7) && (ippPaddingNONE==padding)), ippStsUnderRunErr); + IPP_BADARG_RET((len&7), ippStsUnderRunErr); + + { + int nBlocks = len/MBS_DES; + + /* misaligned source and/or target */ + if( (IPP_UINT_PTR(pSrc) & 0x7) || (IPP_UINT_PTR(pDst) & 0x7) ) { + int n; + for(n=0; n=ofbBlkSize) { + /* block-by-block processing */ + outBuffer = Cipher_DES(inpBuffer, DES_EKEYS(pCtx1), DESspbox); + outBuffer = Cipher_DES(outBuffer, DES_DKEYS(pCtx2), DESspbox); + outBuffer = Cipher_DES(outBuffer, DES_EKEYS(pCtx3), DESspbox); + + /* store output */ + XorBlock(pSrc, &outBuffer, pDst, ofbBlkSize); + + /* shift inpBuffer for the next OFB operation */ + if(MBS_DES==ofbBlkSize) + inpBuffer = outBuffer; + else + #if (IPP_ENDIAN == IPP_BIG_ENDIAN) + inpBuffer = LSL64(inpBuffer, ofbBlkSize*8) + |LSR64(outBuffer, 64-ofbBlkSize*8); + #else + inpBuffer = LSR64(inpBuffer, ofbBlkSize*8) + |LSL64(outBuffer, 64-ofbBlkSize*8); + #endif + + pSrc += ofbBlkSize; + pDst += ofbBlkSize; + len -= ofbBlkSize; + } + + /* update pIV */ + CopyBlock8(&inpBuffer, pIV); +} + + +/*F* +// Name: ippsTDESEncryptOFB +// +// Purpose: Encrypts byte data stream according to TDES in OFB mode. +// +// Returns: Reason: +// ippStsNullPtrErr pCtx1== NULL +// pCtx2== NULL +// pCtx3== NULL +// pSrc == NULL +// pDst == NULL +// pIV == NULL +// ippStsContextMatchErr pCtx->idCtx != idCtxRijndael +// ippStsLengthErr len <1 +// ippStsOFBSizeErr (1>ofbBlkSize || ofbBlkSize>MBS_DES) +// ippStsUnderRunErr (len%ofbBlkSize) +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source data stream +// pDst pointer to the target data stream +// len plaintext stream length (bytes) +// ofbBlkSize OFB block size (bytes) +// pCtx1,.. DES contexts +// pIV pointer to the initialization vector +// +*F*/ +IPPFUN(IppStatus, ippsTDESEncryptOFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int ofbBlkSize, + const IppsDESSpec* pCtx1, + const IppsDESSpec* pCtx2, + const IppsDESSpec* pCtx3, + Ipp8u* pIV)) +{ + /* test contexts */ + IPP_BAD_PTR3_RET(pCtx1, pCtx2, pCtx3); + + /* test context validity */ + IPP_BADARG_RET(!VALID_DES_ID(pCtx1), ippStsContextMatchErr); + IPP_BADARG_RET(!VALID_DES_ID(pCtx2), ippStsContextMatchErr); + IPP_BADARG_RET(!VALID_DES_ID(pCtx3), ippStsContextMatchErr); + + /* test source and destination pointers */ + IPP_BAD_PTR3_RET(pSrc, pDst, pIV); + /* test stream length */ + IPP_BADARG_RET((len<1), ippStsLengthErr); + /* test OFB value */ + IPP_BADARG_RET(((1>ofbBlkSize) || (MBS_DESidCtx != idCtxRijndael +// ippStsLengthErr len <1 +// ippStsOFBSizeErr (1>ofbBlkSize || ofbBlkSize>MBS_DES) +// ippStsUnderRunErr (len%ofbBlkSize) +// ippStsNoErr no errors +// +// Parameters: +// pSrc pointer to the source data stream +// pDst pointer to the target data stream +// len plaintext stream length (bytes) +// ofbBlkSize OFB block size (bytes) +// pCtx1,.. DES contexts +// pIV pointer to the initialization vector +// +*F*/ + +IPPFUN(IppStatus, ippsTDESDecryptOFB,(const Ipp8u* pSrc, Ipp8u* pDst, int len, int ofbBlkSize, + const IppsDESSpec* pCtx1, + const IppsDESSpec* pCtx2, + const IppsDESSpec* pCtx3, + Ipp8u* pIV)) +{ + /* test contexts */ + IPP_BAD_PTR3_RET(pCtx1, pCtx2, pCtx3); + + /* test context validity */ + IPP_BADARG_RET(!VALID_DES_ID(pCtx1), ippStsContextMatchErr); + IPP_BADARG_RET(!VALID_DES_ID(pCtx2), ippStsContextMatchErr); + IPP_BADARG_RET(!VALID_DES_ID(pCtx3), ippStsContextMatchErr); + + /* test source and destination pointers */ + IPP_BAD_PTR3_RET(pSrc, pDst, pIV); + /* test stream length */ + IPP_BADARG_RET((len<1), ippStsLengthErr); + /* test OFB value */ + IPP_BADARG_RET(((1>ofbBlkSize) || (MBS_DES=_IPP_W7) || (_IPP32E>=_IPP32E_M7)) +__INLINE void PurgeBlock(void* pDst, int len) +{ + int n; + for(n=0; n> (blkBitSize -numSize)%8 ); + + int i; + Ipp32u carry = 1; + for(i=BITS2WORD8_SIZE(blkBitSize)-1; i>=0; i--) { + int d = maskPosition - i; + Ipp8u mask = (Ipp8u)(maskVal | cpIsMsb_ct((BNU_CHUNK_T)d)); + + Ipp32u x = pCounter[i] + carry; + Ipp8u y = pCounter[i]; + pCounter[i] = (Ipp8u)((y & ~mask) | (x & mask)); + + maskVal &= cpIsMsb_ct((BNU_CHUNK_T)d); + + carry = (x>>8) & 0x1; + } +} + +/* vb */ +__INLINE void ompStdIncrement64( void* pInitCtrVal, void* pCurrCtrVal, + int ctrNumBitSize, int n ) +{ + int k; + Ipp64u cntr; + Ipp64u temp; + Ipp64s item; + + #if( IPP_ENDIAN == IPP_LITTLE_ENDIAN ) + for( k = 0; k < 8; k++ ) + ( ( Ipp8u* )&cntr )[k] = ( ( Ipp8u* )pInitCtrVal )[7 - k]; + #else + for( k = 0; k < 8; k++ ) + ( ( Ipp8u* )&cntr )[k] = ( ( Ipp8u* )pInitCtrVal )[k]; + #endif + + if( ctrNumBitSize == 64 ) + { + cntr += ( Ipp64u )n; + } + else + { + Ipp64u mask = CONST_64(0xFFFFFFFFFFFFFFFF) >> ( 64 - ctrNumBitSize ); + Ipp64u save = cntr & ( ~mask ); + Ipp64u bndr = ( Ipp64u )1 << ctrNumBitSize; + + temp = cntr & mask; + cntr = temp + ( Ipp64u )n; + + if( cntr > bndr ) + { + item = ( Ipp64s )n - ( Ipp64s )( bndr - temp ); + + while( item > 0 ) + { + cntr = ( Ipp64u )item; + item -= ( Ipp64s )bndr; + } + } + + cntr = save | ( cntr & mask ); + } + + #if( IPP_ENDIAN == IPP_LITTLE_ENDIAN ) + for( k = 0; k < 8; k++ ) + ( ( Ipp8u* )pCurrCtrVal )[7 - k] = ( ( Ipp8u* )&cntr )[k]; + #else + for( k = 0; k < 8; k++ ) + ( ( Ipp8u* )pCurrCtrVal )[k] = ( ( Ipp8u* )&cntr )[k]; + #endif +} + + +/* vb */ +__INLINE void ompStdIncrement128( void* pInitCtrVal, void* pCurrCtrVal, + int ctrNumBitSize, int n ) +{ + int k; + Ipp64u low; + Ipp64u hgh; + Ipp64u flag; + Ipp64u mask = CONST_64(0xFFFFFFFFFFFFFFFF); + Ipp64u save = 0; + + #if( IPP_ENDIAN == IPP_LITTLE_ENDIAN ) + for( k = 0; k < 8; k++ ) + { + ( ( Ipp8u* )&low )[k] = ( ( Ipp8u* )pInitCtrVal )[15 - k]; + ( ( Ipp8u* )&hgh )[k] = ( ( Ipp8u* )pInitCtrVal )[7 - k]; + } + #else + for( k = 0; k < 8; k++ ) + { + ( ( Ipp8u* )&low )[k] = ( ( Ipp8u* )pInitCtrVal )[8 + k]; + ( ( Ipp8u* )&hgh )[k] = ( ( Ipp8u* )pInitCtrVal )[k]; + } + #endif + + if( ctrNumBitSize == 64 ) + { + low += ( Ipp64u )n; + } + else if( ctrNumBitSize < 64 ) + { + Ipp64u bndr; + Ipp64u cntr; + Ipp64s item; + + mask >>= ( 64 - ctrNumBitSize ); + save = low & ( ~mask ); + cntr = ( low & mask ) + ( Ipp64u )n; + + if( ctrNumBitSize < 31 ) + { + bndr = ( Ipp64u )1 << ctrNumBitSize; + + if( cntr > bndr ) + { + item = ( Ipp64s )( ( Ipp64s )n - ( ( Ipp64s )bndr - + ( Ipp64s )( low & mask ) ) ); + + while( item > 0 ) + { + cntr = ( Ipp64u )item; + item -= ( Ipp64s )bndr; + } + } + } + + low = save | ( cntr & mask ); + } + else + { + flag = ( low >> 63 ); + + if( ctrNumBitSize != 128 ) + { + mask >>= ( 128 - ctrNumBitSize ); + save = hgh & ( ~mask ); + hgh &= mask; + } + + low += ( Ipp64u )n; + + if( flag != ( low >> 63 ) ) hgh++; + + if( ctrNumBitSize != 128 ) + { + hgh = save | ( hgh & mask ); + } + } + + #if( IPP_ENDIAN == IPP_LITTLE_ENDIAN ) + for( k = 0; k < 8; k++ ) + { + ( ( Ipp8u* )pCurrCtrVal )[15 - k] = ( ( Ipp8u* )&low )[k]; + ( ( Ipp8u* )pCurrCtrVal )[7 - k] = ( ( Ipp8u* )&hgh )[k]; + } + #else + for( k = 0; k < 8; k++ ) + { + ( ( Ipp8u* )pCurrCtrVal )[8 + k] = ( ( Ipp8u* )&low )[k]; + ( ( Ipp8u* )pCurrCtrVal )[k] = ( ( Ipp8u* )&hgh )[k]; + } + #endif +} + +#if 0 +/* vb */ +__INLINE void ompStdIncrement192( void* pInitCtrVal, void* pCurrCtrVal, + int ctrNumBitSize, int n ) +{ + int k; + Ipp64u low; + Ipp64u mdl; + Ipp64u hgh; + Ipp64u flag; + Ipp64u mask = CONST_64(0xFFFFFFFFFFFFFFFF); + Ipp64u save; + + #if( IPP_ENDIAN == IPP_LITTLE_ENDIAN ) + for( k = 0; k < 8; k++ ) + { + ( ( Ipp8u* )&low )[k] = ( ( Ipp8u* )pInitCtrVal )[23 - k]; + ( ( Ipp8u* )&mdl )[k] = ( ( Ipp8u* )pInitCtrVal )[15 - k]; + ( ( Ipp8u* )&hgh )[k] = ( ( Ipp8u* )pInitCtrVal )[7 - k]; + } + #else + for( k = 0; k < 8; k++ ) + { + ( ( Ipp8u* )&low )[k] = ( ( Ipp8u* )pInitCtrVal )[16 + k]; + ( ( Ipp8u* )&mdl )[k] = ( ( Ipp8u* )pInitCtrVal )[8 + k]; + ( ( Ipp8u* )&hgh )[k] = ( ( Ipp8u* )pInitCtrVal )[k]; + } + #endif + + if( ctrNumBitSize == 64 ) + { + low += ( Ipp64u )n; + } + else if( ctrNumBitSize == 128 ) + { + flag = ( low >> 63 ); + low += ( Ipp64u )n; + if( flag != ( low >> 63 ) ) mdl++; + } + else if( ctrNumBitSize == 192 ) + { + flag = ( low >> 63 ); + low += ( Ipp64u )n; + + if( flag != ( low >> 63 ) ) + { + flag = ( mdl >> 63 ); + mdl++; + if( flag != ( mdl >> 63 ) ) hgh++; + } + } + else if( ctrNumBitSize < 64 ) + { + Ipp64u bndr; + Ipp64u cntr; + Ipp64s item; + + mask >>= ( 64 - ctrNumBitSize ); + save = low & ( ~mask ); + cntr = ( low & mask ) + ( Ipp64u )n; + + if( ctrNumBitSize < 31 ) + { + bndr = ( Ipp64u )1 << ctrNumBitSize; + + if( cntr > bndr ) + { + item = ( Ipp64s )( ( Ipp64s )n - ( ( Ipp64s )bndr - + ( Ipp64s )( low & mask ) ) ); + + while( item > 0 ) + { + cntr = ( Ipp64u )item; + item -= ( Ipp64s )bndr; + } + } + } + + low = save | ( cntr & mask ); + } + else if( ctrNumBitSize < 128 ) + { + flag = ( low >> 63 ); + mask >>= ( 128 - ctrNumBitSize ); + save = mdl & ( ~mask ); + mdl &= mask; + low += ( Ipp64u )n; + if( flag != ( low >> 63 ) ) mdl++; + mdl = save | ( mdl & mask ); + } + else + { + flag = ( low >> 63 ); + mask >>= ( 192 - ctrNumBitSize ); + save = hgh & ( ~mask ); + hgh &= mask; + low += ( Ipp64u )n; + + if( flag != ( low >> 63 ) ) + { + flag = ( mdl >> 63 ); + mdl++; + if( flag != ( mdl >> 63 ) ) hgh++; + } + + hgh = save | ( hgh & mask ); + } + + #if( IPP_ENDIAN == IPP_LITTLE_ENDIAN ) + for( k = 0; k < 8; k++ ) + { + ( ( Ipp8u* )pCurrCtrVal )[23 - k] = ( ( Ipp8u* )&low )[k]; + ( ( Ipp8u* )pCurrCtrVal )[15 - k] = ( ( Ipp8u* )&mdl )[k]; + ( ( Ipp8u* )pCurrCtrVal )[7 - k] = ( ( Ipp8u* )&hgh )[k]; + } + #else + for( k = 0; k < 8; k++ ) + { + ( ( Ipp8u* )pCurrCtrVal )[16 + k] = ( ( Ipp8u* )&low )[k]; + ( ( Ipp8u* )pCurrCtrVal )[8 + k] = ( ( Ipp8u* )&mdl )[k]; + ( ( Ipp8u* )pCurrCtrVal )[k] = ( ( Ipp8u* )&hgh )[k]; + } + #endif +} +#endif + +#if 0 +/* vb */ +__INLINE void ompStdIncrement256( void* pInitCtrVal, void* pCurrCtrVal, + int ctrNumBitSize, int n ) +{ + int k; + Ipp64u low; + Ipp64u mdl; + Ipp64u mdm; + Ipp64u hgh; + Ipp64u flag; + Ipp64u mask = CONST_64(0xFFFFFFFFFFFFFFFF); + Ipp64u save; + + #if( IPP_ENDIAN == IPP_LITTLE_ENDIAN ) + for( k = 0; k < 8; k++ ) + { + ( ( Ipp8u* )&low )[k] = ( ( Ipp8u* )pInitCtrVal )[31 - k]; + ( ( Ipp8u* )&mdl )[k] = ( ( Ipp8u* )pInitCtrVal )[23 - k]; + ( ( Ipp8u* )&mdm )[k] = ( ( Ipp8u* )pInitCtrVal )[15 - k]; + ( ( Ipp8u* )&hgh )[k] = ( ( Ipp8u* )pInitCtrVal )[7 - k]; + } + #else + for( k = 0; k < 8; k++ ) + { + ( ( Ipp8u* )&low )[k] = ( ( Ipp8u* )pInitCtrVal )[24 + k]; + ( ( Ipp8u* )&mdl )[k] = ( ( Ipp8u* )pInitCtrVal )[16 + k]; + ( ( Ipp8u* )&mdm )[k] = ( ( Ipp8u* )pInitCtrVal )[8 + k]; + ( ( Ipp8u* )&hgh )[k] = ( ( Ipp8u* )pInitCtrVal )[k]; + } + #endif + + if( ctrNumBitSize == 64 ) + { + low += ( Ipp64u )n; + } + else if( ctrNumBitSize == 128 ) + { + flag = ( low >> 63 ); + low += ( Ipp64u )n; + if( flag != ( low >> 63 ) ) mdl++; + } + else if( ctrNumBitSize == 192 ) + { + flag = ( low >> 63 ); + low += ( Ipp64u )n; + + if( flag != ( low >> 63 ) ) + { + flag = ( mdl >> 63 ); + mdl++; + if( flag != ( mdl >> 63 ) ) hgh++; + } + } + else if( ctrNumBitSize == 256 ) + { + flag = ( low >> 63 ); + low += ( Ipp64u )n; + + if( flag != ( low >> 63 ) ) + { + flag = ( mdl >> 63 ); + mdl++; + + if( flag != ( mdl >> 63 ) ) + { + flag = ( mdm >> 63 ); + mdm++; + if( flag != ( mdm >> 63 ) ) hgh++; + } + } + } + else if( ctrNumBitSize < 64 ) + { + Ipp64u bndr; + Ipp64u cntr; + Ipp64s item; + + mask >>= ( 64 - ctrNumBitSize ); + save = low & ( ~mask ); + cntr = ( low & mask ) + ( Ipp64u )n; + + if( ctrNumBitSize < 31 ) + { + bndr = ( Ipp64u )1 << ctrNumBitSize; + + if( cntr > bndr ) + { + item = ( Ipp64s )( ( Ipp64s )n - ( ( Ipp64s )bndr - + ( Ipp64s )( low & mask ) ) ); + + while( item > 0 ) + { + cntr = ( Ipp64u )item; + item -= ( Ipp64s )bndr; + } + } + } + + low = save | ( cntr & mask ); + } + else if( ctrNumBitSize < 128 ) + { + flag = ( low >> 63 ); + mask >>= ( 128 - ctrNumBitSize ); + save = mdl & ( ~mask ); + mdl &= mask; + low += ( Ipp64u )n; + if( flag != ( low >> 63 ) ) mdl++; + mdl = save | ( mdl & mask ); + } + else if( ctrNumBitSize < 192 ) + { + flag = ( low >> 63 ); + mask >>= ( 192 - ctrNumBitSize ); + save = mdm & ( ~mask ); + mdm &= mask; + low += ( Ipp64u )n; + + if( flag != ( low >> 63 ) ) + { + flag = ( mdl >> 63 ); + mdl++; + if( flag != ( mdl >> 63 ) ) mdm++; + } + + mdm = save | ( mdm & mask ); + } + else + { + flag = ( low >> 63 ); + mask >>= ( 256 - ctrNumBitSize ); + save = hgh & ( ~mask ); + hgh &= mask; + low += ( Ipp64u )n; + + if( flag != ( low >> 63 ) ) + { + flag = ( mdl >> 63 ); + mdl++; + + if( flag != ( mdl >> 63 ) ) + { + flag = ( mdm >> 63 ); + mdm++; + if( flag != ( mdm >> 63 ) ) hgh++; + } + } + + hgh = save | ( hgh & mask ); + } + + #if( IPP_ENDIAN == IPP_LITTLE_ENDIAN ) + for( k = 0; k < 8; k++ ) + { + ( ( Ipp8u* )pCurrCtrVal )[31 - k] = ( ( Ipp8u* )&low )[k]; + ( ( Ipp8u* )pCurrCtrVal )[23 - k] = ( ( Ipp8u* )&mdl )[k]; + ( ( Ipp8u* )pCurrCtrVal )[15 - k] = ( ( Ipp8u* )&mdm )[k]; + ( ( Ipp8u* )pCurrCtrVal )[7 - k] = ( ( Ipp8u* )&hgh )[k]; + } + #else + for( k = 0; k < 8; k++ ) + { + ( ( Ipp8u* )pCurrCtrVal )[24 + k] = ( ( Ipp8u* )&low )[k]; + ( ( Ipp8u* )pCurrCtrVal )[16 + k] = ( ( Ipp8u* )&mdl )[k]; + ( ( Ipp8u* )pCurrCtrVal )[8 + k] = ( ( Ipp8u* )&mdm )[k]; + ( ( Ipp8u* )pCurrCtrVal )[k] = ( ( Ipp8u* )&hgh )[k]; + } + #endif +} +#endif + +#endif /* _CP_TOOL_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcptrngenhwca.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcptrngenhwca.c new file mode 100644 index 0000000..5af412b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcptrngenhwca.c @@ -0,0 +1,202 @@ +/******************************************************************************* +* Copyright (C) 2015 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// TRNG Functions +// +// Contents: +// ippsTRNGenHW() +// ippsTRNGenHW_BN() +// +// +*/ + +#include "owndefs.h" + +#include "owncp.h" +#include "pcpbn.h" +#include "pcptool.h" + +#if ((_IPP>=_IPP_G9) || (_IPP32E>=_IPP32E_E9)) +static int cpSeed_hw_sample(BNU_CHUNK_T* pSample) +{ +#define LOCAL_COUNTER (320) /* this constant has been tuned manually */ + int n; + int success = 0; + for(n=0; n=_IPP32E_E9) +static int cpSeed_hw_sample32(Ipp32u* pSample) +{ +#define LOCAL_COUNTER (320) /* this constant has been tuned manually */ + int n; + int success = 0; + for(n=0; n=_IPP32E_E9) + if( bufLen%((Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(Ipp32u))) ) { + if( !cpSeed_hw_sample32(pRand)) { + return 0; + } + } + #endif + return 1; +} +#endif + +/*F* +// Name: ippsTRNGenRDSEED +// +// Purpose: Generates a true random bit sequence +// based on Intel® instruction RDSEED of the specified nBits length. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pRand +// +// ippStsLengthErr 1 > nBits +// +// ippStsNotSupportedModeErr unsupported rdrand instruction +// +// ippStsErr random bit sequence can't be generated +// +// ippStsNoErr no error +// +// Parameters: +// pRand pointer to the buffer +// nBits number of bits be requested +// pCtx pointer to the context +*F*/ +IPPFUN(IppStatus, ippsTRNGenRDSEED,(Ipp32u* pRand, int nBits, void* pCtx)) +{ + /* test PRNG buffer */ + IPP_BAD_PTR1_RET(pRand); + + /* test sizes */ + IPP_BADARG_RET(nBits< 1, ippStsLengthErr); + + IPP_UNREFERENCED_PARAMETER(pCtx); + + #if ((_IPP>=_IPP_G9) || (_IPP32E>=_IPP32E_E9)) + if( IsFeatureEnabled(ippCPUID_RDSEED) ) { + cpSize rndSize = BITS2WORD32_SIZE(nBits); + Ipp32u rndMask = MAKEMASK32(nBits); + + if(cpSeedHW_buffer(pRand, rndSize)) { + pRand[rndSize-1] &= rndMask; + return ippStsNoErr; + } + else + return ippStsErr; + } + /* unsupported rdrand instruction */ + else + #endif + IPP_ERROR_RET(ippStsNotSupportedModeErr); +} + + +/*F* +// Name: ippsTRNGenRDSEED_BN +// +// Purpose: Generates a true random big number +// based on Intel® instruction RDSEED of the specified nBits length. +// +// Returns: Reason: +// ippStsNullPtrErr NULL == pRand +// +// ippStsLengthErr 1 > nBits +// nBits > BN_ROOM(pRand) +// +// ippStsNotSupportedModeErr unsupported rdrand instruction +// +// ippStsErr random big number can't be generated +// +// ippStsNoErr no error +// +// Parameters: +// pRand pointer to the big number +// nBits number of bits be requested +// pCtx pointer to the context +*F*/ +IPPFUN(IppStatus, ippsTRNGenRDSEED_BN,(IppsBigNumState* pRand, int nBits, void* pCtx)) +{ + /* test random BN */ + IPP_BAD_PTR1_RET(pRand); + IPP_BADARG_RET(!BN_VALID_ID(pRand), ippStsContextMatchErr); + + /* test sizes */ + IPP_BADARG_RET(nBits< 1, ippStsLengthErr); + IPP_BADARG_RET(nBits> BN_ROOM(pRand)*BNU_CHUNK_BITS, ippStsLengthErr); + + IPP_UNREFERENCED_PARAMETER(pCtx); + + #if ((_IPP>=_IPP_G9) || (_IPP32E>=_IPP32E_E9)) + if( IsFeatureEnabled(ippCPUID_RDSEED) ) { + BNU_CHUNK_T* pRandBN = BN_NUMBER(pRand); + cpSize rndSize = BITS_BNU_CHUNK(nBits); + BNU_CHUNK_T rndMask = MASK_BNU_CHUNK(nBits); + + if(cpSeedHW_buffer((Ipp32u*)pRandBN, rndSize*(Ipp32s)(sizeof(BNU_CHUNK_T)/sizeof(Ipp32u)))) { + pRandBN[rndSize-1] &= rndMask; + + FIX_BNU(pRandBN, rndSize); + BN_SIZE(pRand) = rndSize; + BN_SIGN(pRand) = ippBigNumPOS; + + return ippStsNoErr; + } + else + return ippStsErr; + } + + /* unsupported rdrand instruction */ + else + #endif + IPP_ERROR_RET(ippStsNotSupportedModeErr); +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpvariant.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpvariant.h new file mode 100644 index 0000000..611d9f3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpvariant.h @@ -0,0 +1,419 @@ +/******************************************************************************* +* Copyright (C) 2005 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography) +// +// Purpose: +// Define ippCP variant +// +// +*/ + +#if !defined(_CP_VARIANT_H) +#define _CP_VARIANT_H + +/* +// set _AES_NI_ENABLING_ +*/ +#if defined _IPP_AES_NI_ + #if (_IPP_AES_NI_ == 0) + #define _AES_NI_ENABLING_ _FEATURE_OFF_ + #elif (_IPP_AES_NI_ == 1) + #define _AES_NI_ENABLING_ _FEATURE_ON_ + #else + #error Define _IPP_AES_NI_=0 or 1 or omit _IPP_AES_NI_ at all + #endif +#else + #if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + #define _AES_NI_ENABLING_ _FEATURE_TICKTOCK_ + #else + #define _AES_NI_ENABLING_ _FEATURE_OFF_ + #endif +#endif + +/* +// select AES safe implementation +*/ +#define _ALG_AES_SAFE_COMPACT_SBOX_ (1) +#define _ALG_AES_SAFE_COMPOSITE_GF_ (2) + +#if (_AES_NI_ENABLING_==_FEATURE_ON_) + #define _ALG_AES_SAFE_ _FEATURE_OFF_ +#else + #if (_IPP>=_IPP_V8) || (_IPP32E>=_IPP32E_U8) + #define _ALG_AES_SAFE_ _ALG_AES_SAFE_COMPOSITE_GF_ + #else + #define _ALG_AES_SAFE_ _ALG_AES_SAFE_COMPACT_SBOX_ + //#define _ALG_AES_SAFE_ _ALG_AES_SAFE_COMPOSITE_GF_ + #endif +#endif + +/* +// AES Noise +// Enable mitigation when dispatching available to +// Intel® Advanced Encryption Standard New Instructions (Intel® AES-NI) - _AES_NI_ENABLING_ != _FEATURE_OFF_ +// or vector extensions of Intel® AES-NI - _IPP32E >=_IPP32E_K1 +*/ +#ifndef IPP_AES_PROB_NOISE +#define IPP_AES_PROB_NOISE _FEATURE_ON_ +#endif + +#if (IPP_AES_PROB_NOISE == _FEATURE_ON_) + #if ((_AES_NI_ENABLING_ != _FEATURE_OFF_) || (_IPP32E >=_IPP32E_K1)) + #define _AES_PROB_NOISE _FEATURE_ON_ + #else + #define _AES_PROB_NOISE _FEATURE_OFF_ + #endif +#else + #define _AES_PROB_NOISE _FEATURE_OFF_ +#endif + +/* +// if there is no outside assignment +// set _SHA_NI_ENABLING_ based on CPU specification +*/ +#if !defined(_SHA_NI_ENABLING_) +#if (_IPP>=_IPP_P8) || (_IPP32E>=_IPP32E_Y8) + #define _SHA_NI_ENABLING_ _FEATURE_TICKTOCK_ +#else + #define _SHA_NI_ENABLING_ _FEATURE_OFF_ +#endif +#endif + +/* +// set/reset _ADCOX_NI_ENABLING_ +*/ +#if (_IPP32E>=_IPP32E_L9) + #if !defined(_ADCOX_NI_ENABLING_) + #define _ADCOX_NI_ENABLING_ _FEATURE_TICKTOCK_ + #endif +#else + #undef _ADCOX_NI_ENABLING_ + #define _ADCOX_NI_ENABLING_ _FEATURE_OFF_ +#endif + +/* +// Intel IPP Cryptography supports several hash algorithms by default: +// SHA-1 +// SHA-256 +// SHA-224 (or SHA256/224 by the FIPS180-4 classification) +// SHA-512 +// SHA-384 (or SHA512/384 by the FIPS180-4 classification) +// MD5 +// SM3 +// +// By default all hash algorithms are included in Intel IPP Cryptography. +// +// If one need excludes code of particular hash, just define +// suitable _DISABLE_ALG_XXX, where XXX name of the hash algorithm +// +*/ +#if !defined(_DISABLE_ALG_SHA1_) +#define _ENABLE_ALG_SHA1_ /* SHA1 on */ +#else +# undef _ENABLE_ALG_SHA1_ /* SHA1 off */ +#endif + +#if !defined(_DISABLE_ALG_SHA256_) +# define _ENABLE_ALG_SHA256_ /* SHA256 on */ +#else +# undef _ENABLE_ALG_SHA256_ /* SHA256 off */ +#endif + +#if !defined(_DISABLE_ALG_SHA224_) +# define _ENABLE_ALG_SHA224_ /* SHA224 on */ +#else +# undef _ENABLE_ALG_SHA224_ /* SHA224 off */ +#endif + +#if !defined(_DISABLE_ALG_SHA512_) +# define _ENABLE_ALG_SHA512_ /* SHA512 on */ +#else +# undef _ENABLE_ALG_SHA512_ /* SHA512 off */ +#endif + +#if !defined(_DISABLE_ALG_SHA384_) +# define _ENABLE_ALG_SHA384_ /* SHA384 on */ +#else +# undef _ENABLE_ALG_SHA384_ /* SHA384 off */ +#endif + +#if !defined(_DISABLE_ALG_SHA512_224_) +# define _ENABLE_ALG_SHA512_224_ /* SHA512/224 on */ +#else +# undef _ENABLE_ALG_SHA512_224_ /* SHA512/224 off */ +#endif + +#if !defined(_DISABLE_ALG_SHA512_256_) +# define _ENABLE_ALG_SHA512_256_ /* SHA512/256 on */ +#else +# undef _ENABLE_ALG_SHA512_256_ /* SHA512/256 off */ +#endif + +#if !defined(_DISABLE_ALG_MD5_) +# define _ENABLE_ALG_MD5_ /* MD5 on */ +#else +# undef _ENABLE_ALG_MD5_ /* MD5 off */ +#endif + +#if !defined(_DISABLE_ALG_SM3_) +# define _ENABLE_ALG_SM3_ /* SM3 on */ +#else +# undef _ENABLE_ALG_SM3_ /* SM3 off */ +#endif + +/* +// SHA1 plays especial role in Intel IPP Cryptography. +// Thus Intel IPP Cryptography random generator and +// therefore prime number generator are based on SHA1. +// So, do no exclude SHA1 from the active list of hash algorithms +*/ +#if defined(_DISABLE_ALG_SHA1_) +#undef _DISABLE_ALG_SHA1_ +#endif + +/* +// Because of performane reason hash algorithms are implemented in form +// of unroller cycle and therefore these implementations are big enough. +// Intel IPP Cryptography supports "compact" implementation of some basic hash algorithms: +// SHA-1 +// SHA-256 +// SHA-512 +// SM3 +// +// Define any +// _ALG_SHA1_COMPACT_ +// _ALG_SHA256_COMPACT_ +// _ALG_SHA512_COMPACT_ +// _ALG_SM3_COMPACT_ +// +// to select "compact" implementation of particular hash algorithm. +// Intel IPP Cryptography does not define "compact" implementation by default. +// +// Don't know what performance degradation leads "compact" +// in comparison with default Intel IPP Cryptography implementation. +// +// Note: the definition like _ALG_XXX_COMPACT_ has effect +// if and only if Intel IPP Cryptography instance is _PX or _MX +*/ +//#define _ALG_SHA1_COMPACT_ +//#define _ALG_SHA256_COMPACT_ +//#define _ALG_SHA512_COMPACT_ +//#define _ALG_SM3_COMPACT_ +//#undef _ALG_SHA1_COMPACT_ +//#undef _ALG_SHA256_COMPACT_ +//#undef _ALG_SHA512_COMPACT_ +//#undef _ALG_SM3_COMPACT_ + + +/* +// BN arithmetic: +// - do/don't use special implementation of sqr instead of usual multication +// - do/don't use Karatsuba multiplication alg +*/ +#define _USE_SQR_ /* use implementaton of sqr */ +#if !defined(_DISABLE_WINDOW_EXP_) + #define _USE_WINDOW_EXP_ /* use fixed window exponentiation */ +#endif + + +/* +// RSA: +// - do/don't use version 1 style mitigation of CBA +// - do/don't use own style mitigation of CBA +// - do/don't use Folding technique for RSA-1204 implementation +*/ +#define xUSE_VERSION1_CBA_MITIGATION_ /* not use (version 1) mitigation of CBA */ +#define _USE_IPP_OWN_CBA_MITIGATION_ /* use (own) mitigation of CBA */ +#define xUSE_FOLD_MONT512_ /* use folding technique in RSA-1024 case */ + + +/* +// Intel IPP Cryptography supports different implementation of NIST's (standard) EC over GF(0): +// P-128 (IppECCPStd128r1, IppECCPStd128r2) +// P-192 (IppECCPStd192r1) +// P-224 (IppECCPStd224r1) +// P-256 (IppECCPStd256r1) +// P-384 (IppECCPStd384r1) +// P-521 (IppECCPStd521r1) +// +// If one need replace the particular implementation by abritrary one +// assign _ECP_IMP_ARBIRTRARY_ to suitable symbol +// +// _ECP_IMPL_ARBIRTRARY_ means that implementtaion does not use any curve specific, +// provide the same (single) code for any type curve +// +// _ECP_IMPL_SPECIFIC_ means that implementation uses specific modular reduction +// based on prime structure; +// most of NIST's cures (p128, p192, p224, p256, p384, p521) are uses +// such kind of reduction procedure; +// in contrast with _ECP_IMPL_ARBIRTRARY_ and _ECP_IMPL_MFM_ +// this type of implementation uses point representation in REGULAR residual +// (not Montgometry!!) domain +// +// _ECP_IMPL_MFM_ means that implementation uses "Montgomary Friendly Modulus" (primes); +// p256 and sm2 are using such kind of optimization +*/ +#define _ECP_IMPL_NONE_ 0 +#define _ECP_IMPL_ARBIRTRARY_ 1 +#define _ECP_IMPL_SPECIFIC_ 2 +#define _ECP_IMPL_MFM_ 3 + +#if !defined(_ECP_112R1_) +#if !defined(_DISABLE_ECP_112R1_) +# define _ECP_112R1_ _ECP_IMPL_ARBIRTRARY_ +#else +# define _ECP_112R1_ _ECP_IMPL_NONE_ +#endif +#endif + +#if !defined(_ECP_112R2_) +#if !defined(_DISABLE_ECP_112R2_) +# define _ECP_112R2_ _ECP_IMPL_ARBIRTRARY_ +#else +# define _ECP_112R2_ _ECP_IMPL_NONE_ +#endif +#endif + +#if !defined(_ECP_160R1_) +#if !defined(_DISABLE_ECP_160R1_) +# define _ECP_160R1_ _ECP_IMPL_ARBIRTRARY_ +#else +# define _ECP_160R1_ _ECP_IMPL_NONE_ +#endif +#endif + +#if !defined(_ECP_160R2_) +#if !defined(_DISABLE_ECP_160R2_) +# define _ECP_160R2_ _ECP_IMPL_ARBIRTRARY_ +#else +# define _ECP_160R2_ _ECP_IMPL_NONE_ +#endif +#endif + +#if !defined(_ECP_128R1_) +#if !defined(_DISABLE_ECP_128R1_) +# define _ECP_128R1_ _ECP_IMPL_SPECIFIC_ +#else +# define _ECP_128R1_ _ECP_IMPL_NONE_ +#endif +#endif + +#if !defined(_ECP_128R2_) +#if !defined(_DISABLE_ECP_128R2_) +# define _ECP_128R2_ _ECP_IMPL_SPECIFIC_ +#else +# define _ECP_128R2_ _ECP_IMPL_NONE_ +#endif +#endif + +#if !defined(_ECP_192_) +#if !defined(_DISABLE_ECP_192_) +# if (_IPP32E >= _IPP32E_M7) || (_IPP >= _IPP_P8) +# define _ECP_192_ _ECP_IMPL_MFM_ +# else +# define _ECP_192_ _ECP_IMPL_SPECIFIC_ +# endif +#else +# define _ECP_192_ _ECP_IMPL_NONE_ +#endif +#endif + +#if !defined(_ECP_224_) +#if !defined(_DISABLE_ECP_224_) +# if (_IPP32E >= _IPP32E_M7) || (_IPP >= _IPP_P8) +# define _ECP_224_ _ECP_IMPL_MFM_ +# else +# define _ECP_224_ _ECP_IMPL_SPECIFIC_ +# endif +#else +# define _ECP_224_ _ECP_IMPL_NONE_ +#endif +#endif + +#if !defined(_ECP_256_) +#if !defined(_DISABLE_ECP_256_) +# if (_IPP32E >= _IPP32E_M7) || (_IPP >= _IPP_P8) +# define _ECP_256_ _ECP_IMPL_MFM_ +# else +# define _ECP_256_ _ECP_IMPL_SPECIFIC_ +# endif +#else +# define _ECP_256_ _ECP_IMPL_NONE_ +#endif +#endif + +#if !defined(_ECP_384_) +#if !defined(_DISABLE_ECP_384_) +# if (_IPP32E >= _IPP32E_M7) || (_IPP >= _IPP_P8) +# define _ECP_384_ _ECP_IMPL_MFM_ +# else +# define _ECP_384_ _ECP_IMPL_SPECIFIC_ +# endif +#else +# define _ECP_384_ _ECP_IMPL_NONE_ +#endif +#endif + +#if !defined(_ECP_521_) +#if !defined(_DISABLE_ECP_521_) +# if (_IPP32E >= _IPP32E_M7) || (_IPP >= _IPP_P8) +# define _ECP_521_ _ECP_IMPL_MFM_ +# else +# define _ECP_521_ _ECP_IMPL_SPECIFIC_ +# endif +#else +# define _ECP_521_ _ECP_IMPL_NONE_ +#endif +#endif + +#if !defined(_ECP_SM2_) +#if !defined(_DISABLE_ECP_SM2_) +# if (_IPP32E >= _IPP32E_M7) || (_IPP >= _IPP_P8) +# define _ECP_SM2_ _ECP_IMPL_MFM_ +# else +# define _ECP_SM2_ _ECP_IMPL_SPECIFIC_ +# endif +#else +# define _ECP_SM2_ _ECP_IMPL_NONE_ +#endif +#endif + +#if !defined(_ECP_BN_) +#if !defined(_DISABLE_ECP_BN_) +# define _ECP_BN_ _ECP_IMPL_ARBIRTRARY_ +#else +# define _ECP_BN_ _ECP_IMPL_NONE_ +#endif +#endif + +#if !defined(_DISABLE_ECP_GENERAL_) +# define _ECP_GENERAL_ _ECP_IMPL_ARBIRTRARY_ +#else +# define _ECP_GENERAL_ _ECP_IMPL_NONE_ +#endif + + +/* +// EC over GF(p): +// - do/don't use mitigation of CBA +*/ +#define _USE_ECCP_SSCM_ /* use SSCM ECCP */ + +#endif /* _CP_VARIANT_H */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpver.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpver.c new file mode 100644 index 0000000..130ee57 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpver.c @@ -0,0 +1,91 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel® Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +*/ + +#include "owndefs.h" +#include "ippcpdefs.h" +#include "owncp.h" +#include "pcpver.h" +#include "pcpname.h" + +#define GET_LIBRARY_NAME( cpu, is ) #cpu, IPP_LIB_SHORTNAME() " " is " (" #cpu ")" + +static const IppLibraryVersion ippcpLibVer = { + /* major, minor, update (ex-majorBuild) */ + BASE_VERSION(), +#if defined IPP_REVISION + IPP_REVISION, +#else + 0, +#endif /* IPP_REVISION */ + /* targetCpu[4] */ +#if ( _IPP_ARCH == _IPP_ARCH_IA32 ) || ( _IPP_ARCH == _IPP_ARCH_LP32 ) + #if ( _IPP == _IPP_M5 ) /* Intel® Quark(TM) processor - ia32 */ + GET_LIBRARY_NAME( m5, "586" ) + #elif ( _IPP == _IPP_H9 ) /* Intel® Advanced Vector Extensions 2 - ia32 */ + GET_LIBRARY_NAME( h9, "AVX2" ) + #elif ( _IPP == _IPP_G9 ) /* Intel® Advanced Vector Extensions - ia32 */ + GET_LIBRARY_NAME( g9, "AVX" ) + #elif ( _IPP == _IPP_P8 ) /* Intel® Streaming SIMD Extensions 4.2 (Intel® SSE4.2) - ia32 */ + GET_LIBRARY_NAME( p8, "SSE4.2" ) + #elif ( _IPP == _IPP_S8 ) /* Supplemental Streaming SIMD Extensions 3 + Intel® instruction MOVBE - ia32 */ + GET_LIBRARY_NAME( s8, "Atom" ) + #elif ( _IPP == _IPP_V8 ) /* Supplemental Streaming SIMD Extensions 3 - ia32 */ + GET_LIBRARY_NAME( v8, "SSSE3" ) + #elif ( _IPP == _IPP_W7 ) /* Intel® Streaming SIMD Extensions 2 - ia32 */ + GET_LIBRARY_NAME( w7, "SSE2" ) + #else + GET_LIBRARY_NAME( px, "PX" ) + #endif + +#elif ( _IPP_ARCH == _IPP_ARCH_EM64T ) || ( _IPP_ARCH == _IPP_ARCH_LP64 ) + #if ( _IPP32E == _IPP32E_K1 ) /* Intel® Advanced Vector Extensions 512 (formerly Icelake) - intel64 */ + GET_LIBRARY_NAME( k1, "AVX-512F/CD/BW/DQ/VL/SHA/VBMI/VBMI2/IFMA/GFNI/VAES/VCLMUL" ) + #elif ( _IPP32E == _IPP32E_K0 ) /* Intel® Advanced Vector Extensions 512 (formerly Skylake) - intel64 */ + GET_LIBRARY_NAME( k0, "AVX-512F/CD/BW/DQ/VL" ) + #elif ( _IPP32E == _IPP32E_N0 ) /* Intel® Advanced Vector Extensions 512 (formerly codenamed Knights Landing) - intel64 */ + GET_LIBRARY_NAME( n0, "AVX-512F/CD/ER/PF" ) + #elif ( _IPP32E == _IPP32E_E9 ) /* Intel® Advanced Vector Extensions - intel64 */ + GET_LIBRARY_NAME( e9, "AVX" ) + #elif ( _IPP32E == _IPP32E_L9 ) /* Intel® Advanced Vector Extensions 2 - intel64 */ + GET_LIBRARY_NAME( l9, "AVX2" ) + #elif ( _IPP32E == _IPP32E_Y8 ) /* Intel® Streaming SIMD Extensions 4.2 - intel64 */ + GET_LIBRARY_NAME( y8, "SSE4.2" ) + #elif ( _IPP32E == _IPP32E_N8 ) /* Supplemental Streaming SIMD Extensions 3 + Intel® instruction MOVBE - intel64 */ + GET_LIBRARY_NAME( n8, "Atom" ) + #elif ( _IPP32E == _IPP32E_U8 ) /* Supplemental Streaming SIMD Extensions 3 - intel64 */ + GET_LIBRARY_NAME( u8, "SSSE3" ) + #elif ( _IPP32E == _IPP32E_M7 ) /* Intel® Streaming SIMD Extensions 3 (Intel® SSE3) */ + GET_LIBRARY_NAME( m7, "SSE3" ) + #else + GET_LIBRARY_NAME( mx, "PX" ) + #endif +#endif + ,STR_VERSION() /* release Version */ + ,__DATE__ //BuildDate +}; + +IPPFUN( const IppLibraryVersion*, ippcpGetLibVersion, ( void )){ + return &ippcpLibVer; +}; + +/*////////////////////////// End of file "pcpver.c" ////////////////////////// */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpver.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpver.h new file mode 100644 index 0000000..835261a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpver.h @@ -0,0 +1,31 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// Intel(R) Integrated Performance Primitives +// Cryptographic Primitives (ippcp) +// +// Purpose: Describes the version of ippCP +// +*/ + + +#include "ippver.h" +#define BUILD() 1043 +#define VERSION() BASE_VERSION(),BUILD() + +/* ////////////////////////// End of file "pcpver.h" ///////////////////////// */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpver.rc b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpver.rc new file mode 100644 index 0000000..1269d80 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/pcpver.rc @@ -0,0 +1,30 @@ +/******************************************************************************* +* Copyright (C) 2002 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the 'License'); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an 'AS IS' BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions +* and limitations under the License. +* +*******************************************************************************/ + +/* +// +// Created: Tue Feb 5 17:02:02 2002 +// +*/ + + +#include "pcpver.h" +#include "pcpname.h" +#include "ippres.gen" + +/* ////////////////////////////// End of file /////////////////////////////// */ + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_arith_method_nsm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_arith_method_nsm2.c new file mode 100644 index 0000000..0f35705 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_arith_method_nsm2.c @@ -0,0 +1,50 @@ +/******************************************************************************* + * Copyright (C) 2017 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "sm2/ifma_arith_method_sm2.h" +#include "sm2/ifma_arith_psm2.h" +#include "sm2/ifma_arith_nsm2.h" + +IPP_OWN_DEFN(ifmaArithMethod*, gsArithGF_nsm2_avx512, (void)) { + static ifmaArithMethod m = { + /* import_to52 = */ fesm2_convert_radix64_radix52, + /* export_to64 = */ fesm2_convert_radix52_radix64, + /* encode = */ fesm2_to_mont_norder, + /* decode = */ fesm2_from_mont_norder, + /* mul = */ fesm2_mul_norder, + /* mul_dual = */ 0, + /* sqr = */ 0, + /* sqr_dual = */ 0, + /* norm = */ ifma_norm52, + /* norm_dual = */ ifma_norm52_dual, + /* lnorm = */ ifma_lnorm52, + /* lnorm_dual = */ ifma_lnorm52_dual, + /* add = */ fesm2_add_norder_norm, + /* sub = */ fesm2_sub_norder_norm, + /* neg = */ 0, + /* div2 = */ 0, + /* inv = */ fesm2_inv_norder_norm, + /* red = */ fesm2_fast_reduction_norder}; + + return &m; +} + +#endif /* #if (_IPP32E >= _IPP32E_K1) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_arith_method_psm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_arith_method_psm2.c new file mode 100644 index 0000000..91228be --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_arith_method_psm2.c @@ -0,0 +1,49 @@ +/******************************************************************************* + * Copyright (C) 2017 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "sm2/ifma_arith_method_sm2.h" +#include "sm2/ifma_arith_psm2.h" + +IPP_OWN_DEFN(ifmaArithMethod*, gsArithGF_psm2_avx512, (void)) { + static ifmaArithMethod m = { + /* import_to52 = */ fesm2_convert_radix64_radix52, + /* export_to64 = */ fesm2_convert_radix52_radix64, + /* encode = */ fesm2_to_mont, + /* decode = */ fesm2_from_mont, + /* mul = */ fesm2_mul, + /* mul_dual = */ fesm2_mul_dual, + /* sqr = */ fesm2_sqr, + /* sqr_dual = */ fesm2_sqr_dual, + /* norm = */ ifma_norm52, + /* norm_dual = */ ifma_norm52_dual, + /* lnorm = */ ifma_lnorm52, + /* lnorm_dual = */ ifma_lnorm52_dual, + /* add = */ 0, + /* sub = */ 0, + /* neg = */ fesm2_neg_norm, + /* div2 = */ fesm2_div2_norm, + /* inv = */ fesm2_inv_norm, + /* red = */ 0}; + + return &m; +} + +#endif /* #if (_IPP32E >= _IPP32E_K1) */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_arith_method_sm2.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_arith_method_sm2.h new file mode 100644 index 0000000..211e91b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_arith_method_sm2.h @@ -0,0 +1,38 @@ +/******************************************************************************* + * Copyright (C) 2017 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#ifndef IFMA_ARITH_METHOD_SM2_H_ +#define IFMA_ARITH_METHOD_SM2_H_ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "owncp.h" +#include "ecnist/ifma_alias_avx512.h" +#include "ecnist/ifma_arith_method.h" + +/* Pre-defined AVX512IFMA ISA based methods */ +#define gsArithGF_psm2_avx512 OWNAPI(gsArithGF_psm2_avx512) +IPP_OWN_DECL(ifmaArithMethod*, gsArithGF_psm2_avx512, (void)) + +#define gsArithGF_nsm2_avx512 OWNAPI(gsArithGF_nsm2_avx512) +IPP_OWN_DECL(ifmaArithMethod*, gsArithGF_nsm2_avx512, (void)) + +#endif /* #if (_IPP32E >= _IPP32E_K1) */ + +#endif /* #ifndef IFMA_ARITH_METHOD_SM2_H_ */ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_arith_nsm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_arith_nsm2.c new file mode 100644 index 0000000..e2e5b1c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_arith_nsm2.c @@ -0,0 +1,251 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "sm2/ifma_arith_nsm2.h" +#include "sm2/ifma_arith_psm2.h" +#include "sm2/ifma_defs_sm2.h" + +/* + * EC SM2 base point order + * in 2^52 radix + */ +static const __ALIGN64 Ipp64u nsm2_x1[PSM2_LEN52] = { + 0x000bf40939d54123, 0x0006b21c6052b53b, 0x000fffffff7203df, 0x000fffffffffffff, 0x0000fffffffeffff}; + +/* k0 = -( (1/nsm2) mod 2^DIGIT_SIZE_52 ) mod 2^DIGIT_SIZE_52 */ +static const Ipp64u nsm2_k0 = 0x000f9e8872350975; + +/* r = 2^((52*8)) mod nsm2 */ +static const __ALIGN64 Ipp64u nsm2_r[PSM2_LEN52] = { + 0x00062abedd000000, 0x00006cb726ecad3c, 0x00056c361b698a7e, 0x0000000008dfc209, 0x0000010000000000}; + +/* To Montgomery conversion constant + * rr = 2^((52*6)*2) mod nsm2 + */ +static const __ALIGN64 Ipp64u nsm2_rr[PSM2_LEN52] = { + 0x000643f5455b3830, 0x000c453935b521eb, 0x0009ec21d42b082e, 0x0004d74634238016, 0x0000ca6ea35cbdde}; + +static const __ALIGN64 Ipp64u ones[PSM2_LEN52] = { + 0x1, 0x0, 0x0, 0x0}; + +#define MULT_ROUND(R, A, B, IDX) \ + const fesm2 Bi##R##IDX = permutexvar_i8(idx##IDX, (B)); \ + const fesm2 amBiLo##R##IDX = madd52lo_i64(zero, (A), Bi##R##IDX); \ + fesm2 tr##R##IDX = madd52hi_i64(zero, (A), Bi##R##IDX); \ + { \ + /* low */ \ + (R) = add_i64((R), amBiLo##R##IDX); \ + const fesm2 R0 = permutexvar_i8(idx0, (R)); \ + const fesm2 u = madd52lo_i64(zero, R0, K0); /* u = R0 * K0 */ \ + tr##R##IDX = madd52hi_i64(tr##R##IDX, N, u); \ + (R) = madd52lo_i64((R), N, u); \ + /* shift */ \ + const fesm2 carryone = maskz_srai_i64(maskone, (R), DIGIT_SIZE_52); \ + tr##R##IDX = add_i64(tr##R##IDX, carryone); \ + (R) = maskz_permutexvar_i8(mask_sr64, idx_sr64, (R)); \ + /* hi */ \ + (R) = add_i64((R), tr##R##IDX); \ + } + +/* R = (A*B) - no normalization (in radix 2^52) */ +IPP_OWN_DEFN(fesm2, fesm2_mul_norder, (const fesm2 a, const fesm2 b)) { + const fesm2 N = FESM2_LOADU(nsm2_x1); /* p */ + const fesm2 K0 = set1_i64(nsm2_k0); + const fesm2 zero = setzero_i64(); + const mask8 maskone = 0x1; + /* index broadcast */ + const fesm2 idx0 = set_i64(REPL8(0x0706050403020100)); // 7, 6, 5, 4, 3, 2, 1, 0 + const fesm2 idx1 = set_i64(REPL8(0x0f0e0d0c0b0a0908)); // 15, 14, 13, 12, 11, 10, 9, 8 + const fesm2 idx2 = set_i64(REPL8(0x1716151413121110)); // 23, 22, 21, 20, 19, 18, 17, 16 + const fesm2 idx3 = set_i64(REPL8(0x1f1e1d1c1b1a1918)); // 31, 30, 29, 28, 27, 26, 25, 24 + const fesm2 idx4 = set_i64(REPL8(0x2726252423222120)); // 39, 38, 37, 36, 35, 34, 33, 32 + const fesm2 idx5 = set_i64(REPL8(0x2f2e2d2c2b2a2928)); // 47, 46, 45, 44, 43, 42, 41, 40 + + /* shift right 64 bit line [R >> 64] */ + const mask64 mask_sr64 = 0x00FFFFFFFFFFFFFF; + const fesm2 idx_sr64 = set_i64(0x0, // 0, 0, 0, 0, 0, 0, 0, 0 + 0x3f3e3d3c3b3a3938, // 63, 62, 61, 60, 59, 58, 57, 56 + 0x3736353433323130, // 55, 54, 53, 52, 51, 50, 49, 48 + 0x2f2e2d2c2b2a2928, // 47, 46, 45, 44, 43, 42, 41, 40 + 0x2726252423222120, // 39, 38, 37, 36, 35, 34, 33, 32 + 0x1f1e1d1c1b1a1918, // 31, 30, 29, 28, 27, 26, 25, 24 + 0x1716151413121110, // 23, 22, 21, 20, 19, 18, 17, 16 + 0x0f0e0d0c0b0a0908); // 15, 14, 13, 12, 11, 10, 9, 8 + + fesm2 r = setzero_i64(); + /* PSM2 + * m' mod b = 1 + * + * Algorithm + * a[] b[] - input data ((in radix 2^52)) m[] - module nsm2 + * + * when u = R[0]*m' mod b + * 1) R = R + a[] * b[i] (lo) + * 2) R = R + m[] * u (lo) + * 3) R = R >> 64 + * 4) R = R + a[] * b[i] (hi) + * 5) R = R + m[] * u (hi) + */ + /* one round = O(32) */ + MULT_ROUND(r, a, b, 0) + MULT_ROUND(r, a, b, 1) + MULT_ROUND(r, a, b, 2) + MULT_ROUND(r, a, b, 3) + MULT_ROUND(r, a, b, 4) + MULT_ROUND(r, a, b, 5) + + return r; +} + +IPP_OWN_DEFN(fesm2, fesm2_add_norder_norm, (const fesm2 a, const fesm2 b)) { + const fesm2 zero = setzero_i64(); + const fesm2 N = FESM2_LOADU(nsm2_x1); + + /* r = a + b */ + fesm2 r = add_i64(a, b); + r = ifma_lnorm52(r); + + /* t = r - N */ + fesm2 t = sub_i64(r, N); + t = ifma_norm52(t); + + /* lt = t < 0 */ + const mask8 lt = cmp_i64_mask(zero, srli_i64(t, DIGIT_SIZE_52 - 1), _MM_CMPINT_LT); + const mask8 mask = (mask8)((mask8)0 - ((lt >> 4) & 1)); + + /* maks != 0 ? a : r */ + r = mask_mov_i64(t, mask, r); + return r; +} +IPP_OWN_DEFN(fesm2, fesm2_sub_norder_norm, (const fesm2 a, const fesm2 b)) { + const fesm2 zero = setzero_i64(); + const fesm2 N = FESM2_LOADU(nsm2_x1); + /* r = a - b */ + fesm2 r = sub_i64(a, b); + r = ifma_norm52(r); + + /* t = r + M */ + fesm2 t = add_i64(r, N); + t = ifma_lnorm52(t); + + /* lt = r < 0 */ + const mask8 lt = cmp_i64_mask(zero, srli_i64(r, DIGIT_SIZE_52 - 1), _MM_CMPINT_LT); + const mask8 mask = (mask8)((mask8)0 - ((lt >> 4) & 1)); + /* mask != 0 ? t : r */ + r = mask_mov_i64(r, mask, t); + + return r; +} + +IPP_OWN_DEFN(fesm2, fesm2_fast_reduction_norder, (const fesm2 a)) { + const fesm2 zero = setzero_i64(); + const fesm2 N = FESM2_LOADU(nsm2_x1); + + fesm2 r = sub_i64(a, N); + r = ifma_norm52(r); + + const mask8 lt = cmp_i64_mask(zero, srli_i64(r, DIGIT_SIZE_52 - 1), _MM_CMPINT_LT); + const mask8 mask = (mask8)((mask8)0 - ((lt >> 4) & 1)); + + /* maks != 0 ? a : r */ + r = mask_mov_i64(r, mask, a); + return r; +} + +IPP_OWN_DEFN(fesm2, fesm2_to_mont_norder, (const fesm2 a)) { + const fesm2 RR = FESM2_LOADU(nsm2_rr); + const fesm2 r = fesm2_mul_norder(a, RR); + return ifma_lnorm52(r); +} + +IPP_OWN_DEFN(fesm2, fesm2_from_mont_norder, (const fesm2 a)) { + const fesm2 ONE = FESM2_LOADU(ones); + fesm2 r = fesm2_mul_norder(a, ONE); + r = ifma_lnorm52(r); + r = fesm2_fast_reduction_norder(r); + return r; +} + +__INLINE fesm2 mul_norder_norm(const fesm2 a, const fesm2 b) { + const fesm2 r = fesm2_mul_norder(a, b); + return ifma_lnorm52(r); +} + +__INLINE fesm2 sqr_norder_norm(const fesm2 a) { + const fesm2 r = fesm2_mul_norder(a, a); + return ifma_lnorm52(r); +} + +#define SIZE_TBL (16) + +#define sqr(R, A) (R) = sqr_norder_norm((A)) +#define mul(R, A, B) (R) = mul_norder_norm((A), (B)) + +IPP_OWN_DEFN(fesm2, fesm2_inv_norder_norm, (const fesm2 z)) { + + const fesm2 r_norder = FESM2_LOADU(nsm2_r); + int i; + + // pwr_z_Tbl[i][] = z^i, i=0,..,15 + __ALIGN64 fesm2 pwr_z_Tbl[SIZE_TBL]; + + pwr_z_Tbl[0] = r_norder; + pwr_z_Tbl[1] = z; + + for (i = 2; i < SIZE_TBL; i += 2) { + sqr(pwr_z_Tbl[i], pwr_z_Tbl[i / 2]); + mul(pwr_z_Tbl[i + 1], pwr_z_Tbl[i], z); + } + + // pwr = (nsm2-2) in big endian + const Ipp8u pwr[] = "\xFF\xFF\xFF\xFE\xFF\xFF\xFF\xFF" + "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" + "\x72\x03\xDF\x6B\x21\xC6\x05\x2B" + "\x53\xBB\xF4\x09\x39\xD5\x41\x21"; + // init r = 1 + fesm2 r = pwr_z_Tbl[0]; + + for (i = 0; i < 32; ++i) { + const int v = pwr[i]; + const int hi = (v >> 4) & 0xF; + const int lo = v & 0xF; + + sqr(r, r); + sqr(r, r); + sqr(r, r); + sqr(r, r); + if (hi) + mul(r, r, pwr_z_Tbl[hi]); + sqr(r, r); + sqr(r, r); + sqr(r, r); + sqr(r, r); + if (lo) + mul(r, r, pwr_z_Tbl[lo]); + } + return r; +} + +#undef SIZE_TBL +#undef sqr +#undef mul + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_arith_nsm2.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_arith_nsm2.h new file mode 100644 index 0000000..198a044 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_arith_nsm2.h @@ -0,0 +1,111 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ +#if !defined(_IFMA_ARITH_NSM2_H_) +#define _IFMA_ARITH_NSM2_H_ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "sm2/ifma_defs_sm2.h" + +/** + * \brief + * + * Montgomery multiplication + * + * a * b * R mod n, where R = 2^(6*52) mod n + * + * \param[in] a first value (in radix 2^52) + * \param[in] b second value (in radix 2^52) + * \return value no normalization + */ +IPP_OWN_DECL(fesm2, fesm2_mul_norder, (const fesm2 a, const fesm2 b)) + +/** + * \brief + * + * A + B (mod n) + * + * \param[in] a first value (in radix 2^52) + * \param[in] b second value (in radix 2^52) + * \return value (in radix 2^52) + */ +IPP_OWN_DECL(fesm2, fesm2_add_norder_norm, (const fesm2 a, const fesm2 b)) + +/** + * \brief + * + * A + B (mod n) + * + * \param[in] a first value (in radix 2^52) + * \param[in] b second value (in radix 2^52) + * \return value (in radix 2^52) + */ +IPP_OWN_DECL(fesm2, fesm2_sub_norder_norm, (const fesm2 a, const fesm2 b)) + +/** + * \brief + * + * Conversion to Montgomery domain modulo n (subgroup order). + * + * a * R mod n, where R = 2^(6*52) mod n + * + * \param[in] a value (in radix 2^52) + * \return value (in radix 2^52) + */ +IPP_OWN_DECL(fesm2, fesm2_to_mont_norder, (const fesm2 a)) + +/** + * \brief + * + * Reduction modulo n (subgroup order). + * + * (a >= n) ? a - n : a + * + * \param[in] a value (in radix 2^52) + * \return value (in radix 2^52) + */ +IPP_OWN_DECL(fesm2, fesm2_fast_reduction_norder, (const fesm2 a)) + +/** + * \brief + * + * Conversion from Montgomery domain modulo n (subgroup order). + * + * a mod n + * + * \param[in] a value (radix 2^52) + * \return value (radix 2^52) + */ +IPP_OWN_DECL(fesm2, fesm2_from_mont_norder, (const fesm2 a)) + +/** + * \brief + * + * Modular inverse modulo n (subgroup order). + * + * 1/z mod n + * + * \param[in] z value (in radix 2^52) + * \return value (in radix 2^52) + */ +IPP_OWN_DECL(fesm2, fesm2_inv_norder_norm, (const fesm2 z)) + +#endif // (_IPP32E >= _IPP32E_K1) + +#endif // _IFMA_ARITH_NSM2_H_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_arith_psm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_arith_psm2.c new file mode 100644 index 0000000..40033b3 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_arith_psm2.c @@ -0,0 +1,388 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "sm2/ifma_arith_psm2.h" +#include "sm2/ifma_defs_sm2.h" + +/* modulus psm2 = 2^256 - 2^224 - 2^96 + 2^64 - 1 */ +static const __ALIGN64 Ipp64u psm2_x1[PSM2_LEN52] = { + 0x000fffffffffffff, 0x000ff00000000fff, 0x000fffffffffffff, 0x000fffffffffffff, 0x0000fffffffeffff}; + +/* 4*p */ +static const __ALIGN64 Ipp64u psm2_x4[PSM2_LEN52] = { + 0x000ffffffffffffc, 0x000fc00000003fff, 0x000fffffffffffff, 0x000fffffffffffff, 0x0003fffffffbffff}; + +/* to Montgomery conversion constant + * rr = 2^((PSM2_LEN52*DIGIT_SIZE)*2) mod psm2 + */ +/* rr = 2^(52*6*2) mod psm2 */ +static const __ALIGN64 Ipp64u psm2_rr[PSM2_LEN52] = { + 0x0006000000070000, 0x000fffffd0000000, 0x00000400000003ff, 0x0000000000300000, 0x0000000800000003}; + +static const __ALIGN64 Ipp64u ones[PSM2_LEN52] = { + 0x1, 0x0, 0x0, 0x0, 0x0}; + + +/* R = (A/2) mod M */ +IPP_OWN_DEFN(fesm2, fesm2_div2_norm, (const fesm2 a)) { + const fesm2 M = FESM2_LOADU(psm2_x1); + const fesm2 zero = setzero_i64(); + const fesm2 one = set1_i64(1LL); + + const mask8 is_last_one = cmp_i64_mask(and_i64(a, one), zero, _MM_CMPINT_EQ); + const mask8 mask = (mask8)((is_last_one & 1) - 1); + + fesm2 r = mask_add_i64(a, mask, a, M); + r = ifma_lnorm52(r); + + /* 1-bit shift right */ + /* extract last bite + >> 64 */ + const mask64 mask_shift = 0xFFFFFFFF; + const fesm2 idx_shift = set_i64(0x0, // 0, 0, 0, 0, 0, 0, 0, 0 + 0x3f3e3d3c3b3a3938, // 63, 62, 61, 60, 59, 58, 57, 56 + 0x3736353433323130, // 55, 54, 53, 52, 51, 50, 49, 48 + 0x2f2e2d2c2b2a2928, // 47, 46, 45, 44, 43, 42, 41, 40 + 0x2726252423222120, // 39, 38, 37, 36, 35, 34, 33, 32 + 0x1f1e1d1c1b1a1918, // 31, 30, 29, 28, 27, 26, 25, 24 + 0x1716151413121110, // 23, 22, 21, 20, 19, 18, 17, 16 + 0x0f0e0d0c0b0a0908); // 15, 14, 13, 12, 11, 10, 9, 8 + + fesm2 shift_right = maskz_permutexvar_i8(mask_shift, idx_shift, and_i64(r, one)); + /* set last bit is first byte (52 radix) */ + shift_right = slli_i64(shift_right, DIGIT_SIZE_52 - 1); + /* join first new bite */ + r = srli_i64(r, 1); /* create slot by first bite 1111 -> 0111 */ + r = add_i64(r, shift_right); /* join first and other bite */ + + return r; +} + +IPP_OWN_DEFN(fesm2, fesm2_neg_norm, (const fesm2 a)) { + const fesm2 M4 = FESM2_LOADU(psm2_x4); + + /* a == 0 ? 0xFF : 0 */ + const mask8 mask_zero = FESM2_IS_ZERO(a); + + /* r = 4*p - a */ + fesm2 r = mask_sub_i64(a, (mask8)(~mask_zero), M4, a); + r = ifma_norm52(r); + return r; +} + +#define MULT_ROUND(R, A, B, IDX) \ + const fesm2 Bi##R##IDX = permutexvar_i8(idx##IDX, (B)); \ + const fesm2 amBiLo##R##IDX = madd52lo_i64(zero, (A), Bi##R##IDX); \ + fesm2 tr##R##IDX = madd52hi_i64(zero, (A), Bi##R##IDX); \ + { \ + /* low */ \ + (R) = add_i64((R), amBiLo##R##IDX); \ + const fesm2 u = permutexvar_i8(idx0, (R)); /* u = R0 * 1 */ \ + tr##R##IDX = madd52hi_i64(tr##R##IDX, M, u); \ + (R) = madd52lo_i64((R), M, u); \ + /* shift */ \ + const fesm2 carryone = maskz_srai_i64(maskone, (R), DIGIT_SIZE_52); \ + tr##R##IDX = add_i64(tr##R##IDX, carryone); \ + (R) = maskz_permutexvar_i8(mask_sr64, idx_sr64, (R)); \ + /* hi */ \ + (R) = add_i64((R), tr##R##IDX); \ + } + +/* R = (A*B) - no normalization (in radix 2^52) */ +IPP_OWN_DEFN(fesm2, fesm2_mul, (const fesm2 a, const fesm2 b)) { + const fesm2 M = FESM2_LOADU(psm2_x1); /* p */ + const fesm2 zero = setzero_i64(); + const mask8 maskone = 0x1; + /* index broadcast */ + const fesm2 idx0 = set_i64(REPL8(0x0706050403020100)); // 7, 6, 5, 4, 3, 2, 1, 0 + const fesm2 idx1 = set_i64(REPL8(0x0f0e0d0c0b0a0908)); // 15, 14, 13, 12, 11, 10, 9, 8 + const fesm2 idx2 = set_i64(REPL8(0x1716151413121110)); // 23, 22, 21, 20, 19, 18, 17, 16 + const fesm2 idx3 = set_i64(REPL8(0x1f1e1d1c1b1a1918)); // 31, 30, 29, 28, 27, 26, 25, 24 + const fesm2 idx4 = set_i64(REPL8(0x2726252423222120)); // 39, 38, 37, 36, 35, 34, 33, 32 + const fesm2 idx5 = set_i64(REPL8(0x2f2e2d2c2b2a2928)); // 47, 46, 45, 44, 43, 42, 41, 40 + + /* shift right 64 bit line [R >> 64] */ + const mask64 mask_sr64 = 0x00FFFFFFFFFFFFFF; + const fesm2 idx_sr64 = set_i64(0x0, // 0, 0, 0, 0, 0, 0, 0, 0 + 0x3f3e3d3c3b3a3938, // 63, 62, 61, 60, 59, 58, 57, 56 + 0x3736353433323130, // 55, 54, 53, 52, 51, 50, 49, 48 + 0x2f2e2d2c2b2a2928, // 47, 46, 45, 44, 43, 42, 41, 40 + 0x2726252423222120, // 39, 38, 37, 36, 35, 34, 33, 32 + 0x1f1e1d1c1b1a1918, // 31, 30, 29, 28, 27, 26, 25, 24 + 0x1716151413121110, // 23, 22, 21, 20, 19, 18, 17, 16 + 0x0f0e0d0c0b0a0908); // 15, 14, 13, 12, 11, 10, 9, 8 + + fesm2 r = setzero_i64(); + /* PSM2 + * m' mod b = 1 + * + * Algorithm + * a[] b[] - input data ((in radix 2^52)) m[] - module psm2 + * + * when u = R[0]*m' mod b + * 1) R = R + a[] * b[i] (lo) + * 2) R = R + m[] * u (lo) + * 3) R = R >> 64 + * 4) R = R + a[] * b[i] (hi) + * 5) R = R + m[] * u (hi) + */ + /* one round = O(32) */ + MULT_ROUND(r, a, b, 0) + MULT_ROUND(r, a, b, 1) + MULT_ROUND(r, a, b, 2) + MULT_ROUND(r, a, b, 3) + MULT_ROUND(r, a, b, 4) + MULT_ROUND(r, a, b, 5) + + return r; +} + +IPP_OWN_DEFN(void, fesm2_mul_dual, + (fesm2 pr1[], const fesm2 a1, const fesm2 b1, + fesm2 pr2[], const fesm2 a2, const fesm2 b2)) { + const fesm2 M = FESM2_LOADU(psm2_x1); /* p */ + const fesm2 zero = setzero_i64(); + const mask8 maskone = 0x1; + /* index broadcast */ + const fesm2 idx0 = set_i64(REPL8(0x0706050403020100)); // 7, 6, 5, 4, 3, 2, 1, 0 + const fesm2 idx1 = set_i64(REPL8(0x0f0e0d0c0b0a0908)); // 15, 14, 13, 12, 11, 10, 9, 8 + const fesm2 idx2 = set_i64(REPL8(0x1716151413121110)); // 23, 22, 21, 20, 19, 18, 17, 16 + const fesm2 idx3 = set_i64(REPL8(0x1f1e1d1c1b1a1918)); // 31, 30, 29, 28, 27, 26, 25, 24 + const fesm2 idx4 = set_i64(REPL8(0x2726252423222120)); // 39, 38, 37, 36, 35, 34, 33, 32 + const fesm2 idx5 = set_i64(REPL8(0x2f2e2d2c2b2a2928)); // 47, 46, 45, 44, 43, 42, 41, 40 + + /* shift right 64 bit line [R >> 64] */ + const mask64 mask_sr64 = 0x00FFFFFFFFFFFFFF; + const fesm2 idx_sr64 = set_i64(0x0, // 0, 0, 0, 0, 0, 0, 0, 0 + 0x3f3e3d3c3b3a3938, // 63, 62, 61, 60, 59, 58, 57, 56 + 0x3736353433323130, // 55, 54, 53, 52, 51, 50, 49, 48 + 0x2f2e2d2c2b2a2928, // 47, 46, 45, 44, 43, 42, 41, 40 + 0x2726252423222120, // 39, 38, 37, 36, 35, 34, 33, 32 + 0x1f1e1d1c1b1a1918, // 31, 30, 29, 28, 27, 26, 25, 24 + 0x1716151413121110, // 23, 22, 21, 20, 19, 18, 17, 16 + 0x0f0e0d0c0b0a0908); // 15, 14, 13, 12, 11, 10, 9, 8 + + fesm2 r1, r2; + r1 = r2 = setzero_i64(); + /* PSM2 + * m' mod b = 1 + * + * Algorithm + * a[] b[] - input data ((in radix 2^52)) m[] - module psm2 + * + * when u = R[0]*m' mod b + * 1) R = R + a[] * b[i] (lo) + * 2) R = R + m[] * u (lo) + * 3) R = R >> 64 + * 4) R = R + a[] * b[i] (hi) + * 5) R = R + m[] * u (hi) + */ + /* one round = O(32) */ + MULT_ROUND(r1, a1, b1, 0) + MULT_ROUND(r2, a2, b2, 0) + MULT_ROUND(r1, a1, b1, 1) + MULT_ROUND(r2, a2, b2, 1) + MULT_ROUND(r1, a1, b1, 2) + MULT_ROUND(r2, a2, b2, 2) + MULT_ROUND(r1, a1, b1, 3) + MULT_ROUND(r2, a2, b2, 3) + MULT_ROUND(r1, a1, b1, 4) + MULT_ROUND(r2, a2, b2, 4) + MULT_ROUND(r1, a1, b1, 5) + MULT_ROUND(r2, a2, b2, 5) + + *pr1 = r1; + *pr2 = r2; + return; +} + +IPP_OWN_DEFN(fesm2, fesm2_to_mont, (const fesm2 a)) { + const fesm2 RR = FESM2_LOADU(psm2_rr); + + fesm2 r = fesm2_mul(a, RR); + return ifma_lnorm52(r); +} + +static fesm2 fesm2_fast_reduction(const fesm2 a) { + const fesm2 M = FESM2_LOADU(psm2_x1); + const fesm2 zero = setzero_i64(); + + /* r = a - M */ + fesm2 r = sub_i64(a, M); + r = ifma_norm52(r); + + /* 1 < 0 */ + const mask8 lt = cmp_i64_mask(zero, srli_i64(r, DIGIT_SIZE_52 - 1), _MM_CMPINT_LT); + const mask8 mask = (mask8)((mask8)0 - ((lt >> 4) & 1)); + + /* maks != 0 ? a : r */ + r = mask_mov_i64(r, mask, a); + return r; +} + +IPP_OWN_DEFN(fesm2, fesm2_from_mont, (const fesm2 a)) { + const fesm2 ONE = FESM2_LOADU(ones); + + /* from mont */ + fesm2 r = fesm2_mul(a, ONE); + r = ifma_lnorm52(r); + r = fesm2_fast_reduction(r); + return r; +} + +__INLINE fesm2 fesm2_mul_norm(const fesm2 a, const fesm2 b) { + fesm2 r = fesm2_mul(a, b); + return ifma_lnorm52(r); +} + +__INLINE fesm2 fesm2_sqr_norm(const fesm2 a) { + fesm2 r = fesm2_sqr(a); + return ifma_lnorm52(r); +} + +#define mul(R, A, B) (R) = fesm2_mul_norm((A), (B)) +#define sqr(R, A) (R) = fesm2_sqr_norm((A)) +#define mul_dual(R1, A1, B1, R2, A2, B2) \ + fesm2_mul_dual(&(R1), (A1), (B1), &(R2), (A2), (B2)); \ + ifma_lnorm52_dual(&(R1), (R1), &(R2), (R2)); + +__INLINE fesm2 fesm2_sqr_ntimes(const fesm2 a, int n) { + fesm2 r = a; + for (; n > 0; --n) + sqr(r, r); + return r; +} + +#define sqr_ntimes(R, A, N) (R) = fesm2_sqr_ntimes((A), (N)) + +IPP_OWN_DEFN(fesm2, fesm2_inv_norm, (const fesm2 z)) { + + fesm2 tmp1, tmp2, D, E, F; + tmp1 = tmp2 = D = E = F = setzero_i64(); + fesm2 r; + r = setzero_i64(); + + sqr(tmp1, z); + mul(F, tmp1, z); /* F = z^3 */ + sqr_ntimes(tmp2, F, 2); /* tmp2 = z^0xC */ + /**/ + mul_dual(D, tmp2, z, /* D = z^0xD */ + E, tmp2, tmp1); /* E = z^0xE */ + mul(F, tmp2, F); /* F = z^0xF */ + /**/ + sqr_ntimes(tmp2, F, 4); /* tmp2 = z^0xF0 */ + /**/ + mul_dual(D, tmp2, D, /* D = z^0xFD */ + E, tmp2, E); /* E = z^0xFE */ + mul(F, tmp2, F); /* F = z^0xFF */ + /**/ + sqr_ntimes(tmp2, F, 8); /* tmp2 = z^0xFF00 */ + mul_dual(D, tmp2, D, /* D = z^0xFFFD */ + E, tmp2, E); /* E = z^0xFFFE */ + mul(F, tmp2, F); /* F = z^0xFFFF */ + /**/ + sqr_ntimes(tmp2, F, 16); /* tmp2 = z^0xFFFF0000 */ + mul_dual(D, tmp2, D, /* D = z^0xFFFFFFFD */ + E, tmp2, E); /* E = z^0xFFFFFFFE */ + mul(F, tmp2, F); /* F = z^0xFFFFFFFF */ + /**/ + /**/ + /* z ^ FFFFFFFE 00000000 */ + sqr_ntimes(r, E, 32); + /* z ^ FFFFFFFE FFFFFFFF */ + mul(r, r, F); + /* z ^ FFFFFFFE FFFFFFFF 00000000 */ + sqr_ntimes(r, r, 32); + /* z ^ FFFFFFFE FFFFFFFF FFFFFFFF */ + mul(r, r, F); + /* z ^ FFFFFFFE FFFFFFFF FFFFFFFF 00000000 */ + sqr_ntimes(r, r, 32); + /* z ^ FFFFFFFE FFFFFFFF FFFFFFFF FFFFFFFF */ + mul(r, r, F); + /* z ^ FFFFFFFE FFFFFFFF FFFFFFFF FFFFFFFF 00000000 */ + sqr_ntimes(r, r, 32); + /* z ^ FFFFFFFE FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF */ + mul(r, r, F); + /* z ^ FFFFFFFE FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF 00000000 00000000 */ + sqr_ntimes(r, r, 64); + /* z ^ FFFFFFFE FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF 00000000 FFFFFFFF */ + mul(r, r, F); + /* z ^ FFFFFFFE FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF 00000000 FFFFFFFF 00000000 */ + sqr_ntimes(r, r, 32); + /* z ^ FFFFFFFE FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF 00000000 FFFFFFFF FFFFFFFD */ + mul(r, r, D); + return r; +} + +IPP_OWN_DEFN(fesm2, fesm2_convert_radix64_radix52, (const Ipp64u* a)) { + /* load mask to register */ + const mask8 mask_load = 0x0F; + const fesm2 mask_rad52 = set1_i64(DIGIT_MASK_52); + /* set data */ + const fesm2 idx16 = set_i64(0x001f001f00170016, // 31, 31, 23, 22, + 0x0016001500140013, // 22, 21, 20, 19, + 0x0013001200110010, // 19, 18, 17, 16, + 0x0010000f000e000d, // 16, 15, 14, 13, + 0x000c000b000a0009, // 12, 11, 10, 9, + 0x0009000800070006, // 9, 8, 7, 6, + 0x0006000500040003, // 6, 5, 4, 3, + 0x0003000200010000); // 3, 2, 1, 0 + const fesm2 shift_right = set_i64(12LL, 8LL, 4LL, 0LL, 12LL, 8LL, 4LL, 0LL); + + fesm2 r = maskz_loadu_i64(mask_load, a); + r = permutexvar_i16(idx16, r); + r = srlv_i64(r, shift_right); + r = and_i64(mask_rad52, r); + return r; +} + +IPP_OWN_DEFN(void, fesm2_convert_radix52_radix64, (Ipp64u * out, const fesm2 a)) { + /* mask store */ + const mask8 mask_store = 0x0F; + const fesm2 shift_left = set_i64(4LL, 0LL, 4LL, 0LL, 4LL, 0LL, 4LL, 0LL); + const fesm2 idx_up8 = set_i64(0x3f3f3f3f3f3f3f3f, // {63,63,63,63,63,63,63,63} + 0x3f3f3f3f3e3d3c3b, // {63,63,63,63,62,61,60,59} + 0x3737363534333231, // {55,55,54,53,52,51,50,49} + 0x302e2d2c2b2a2928, // {48,46,45,44,43,42,41,40} + 0x1f1f1f1f1f1f1e1d, // {31,31,31,31,31,31,30,29} + 0x1717171716151413, // {23,23,23,23,22,21,20,19} + 0x0f0f0f0e0d0c0b0a, // {15,15,15,14,13,12,11,10} + 0x0706050403020100); // { 7, 6, 5, 4, 3, 2, 1, 0} + + const fesm2 idx_down8 = set_i64(0x3f3f3f3f3f3f3f3f, // {63,63,63,63,63,63,63,63} + 0x3f3f3f3f3f3f3f3f, // {63,63,63,63,63,63,63,63} + 0x3a39383737373737, // {58,57,56,55,55,55,55,55} + 0x2727272727272726, // {39,39,39,39,39,39,39,38} + 0x2524232221201f1f, // {37,36,35,34,33,32,31,31} + 0x1c1b1a1918171717, // {28,27,26,25,24,23,23,23} + 0x1211100f0f0f0f0f, // {18,17,16,15,15,15,15,15} + 0x0908070707070707); // { 9, 8, 7, 7, 7, 7, 7, 7} + + fesm2 r = a; + + r = sllv_i64(r, shift_left); + fesm2 T = permutexvar_i8(idx_up8, r); + r = permutexvar_i8(idx_down8, r); + r = or_i64(r, T); + mask_storeu_i64(out, mask_store, r); + return; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_arith_psm2.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_arith_psm2.h new file mode 100644 index 0000000..cd71a2f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_arith_psm2.h @@ -0,0 +1,175 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ +#if !defined(_IFMA_ARITH_PSM2_H_) +#define _IFMA_ARITH_PSM2_H_ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ecnist/ifma_alias_avx512.h" +#include "sm2/ifma_defs_sm2.h" +#include + + +/** + * \brief + * compute R = (-A) enhanced Montgomery (Gueron modification group operation) + * \param[in] a value (in radix 2^52) + * \return fesm2 value (in radix 2^52) + */ +IPP_OWN_DECL(fesm2, fesm2_neg_norm, (const fesm2 a)) + +/** + * \brief + * + * Montgomery multiplication + * + * a * b * r mod n, where r = 2^(6*52) mod n + * + * Note: final normalization to 2^52 radix is not performed and shall be + * handled separately. + * + * \param[in] a first value (in radix 2^52) + * \param[in] b second value (in radix 2^52) + * \return fesm2 not normalization value + */ +IPP_OWN_DECL(fesm2, fesm2_mul, (const fesm2 a, const fesm2 b)) + +/** + * \brief + * + * Montgomery multiplication + * + * a * a * r mod n, where r = 2^(6*52) mod n + * + * Note: final normalization to 2^52 radix is not performed and shall be + * handled separately. + * + * \param[in] a value (in radix 2^52) + * \return fesm2 not normalization value + */ +__INLINE IPP_OWN_DEFN(fesm2, fesm2_sqr, (const fesm2 a)) { + return fesm2_mul(a, a); +} + +/** + * \brief + * + * Dual Montgomery multiplication without final normalization to 2^52 radix. + * (two independent multiplications) + * + * a1 * b1 * R mod p + * a2 * b2 * R mod p, where R = 2^(6*52) mod p + * + * \param[out] pr1 ptr first value no normalization + * \param[in] a1 value first (in radix 2^52) + * \param[in] b1 value second (in radix 2^52) + * \param[out] pr2 ptr second value no normalization + * \param[in] a2 value first (in radix 2^52) + * \param[in] b2 value second (in radix 2^52) + */ +IPP_OWN_DECL(void, fesm2_mul_dual, (fesm2 pr1[], const fesm2 a1, const fesm2 b1, fesm2 pr2[], const fesm2 a2, const fesm2 b2)) + +/** + * \brief + * + * Dual Montgomery multiplication without final normalization to 2^52 radix. + * (two independent multiplications) + * + * a1 * a1 * R mod p + * a2 * a2 * R mod p, where R = 2^(6*52) mod p + * + * \param[out] pr1 ptr first value no normalization + * \param[in] a1 value (in radix 2^52) + * \param[out] pr2 ptr second value no normalization + * \param[in] a2 value (in radix 2^52) + */ +__INLINE IPP_OWN_DEFN(void, fesm2_sqr_dual, (fesm2 pr1[], const fesm2 a1, fesm2 pr2[], const fesm2 a2)) { + fesm2_mul_dual(pr1, a1, a1, pr2, a2, a2); + return; +} + +/* R = A + B */ +#define fesm2_add_no_red(A, B) add_i64((A), (B)) + +/* R = A - B */ +#define fesm2_sub_no_red(A, B) sub_i64((A), (B)) + +/** + * \brief + * R = A/2 + * \param[in] a value (in radix 2^52) + * \return fe384 value (in radix 2^52) + */ +IPP_OWN_DECL(fesm2, fesm2_div2_norm, (const fesm2 a)) + +/** + * \brief + * + * Modular inverse modulo p. + * + * 1/z mod p + * + * \param[in] z value (in radix 2^52) + * \return fesm2 value (in radix 2^52) + */ +IPP_OWN_DECL(fesm2, fesm2_inv_norm, (const fesm2 z)) + +/** + * \brief + * + * Conversion to Montgomery domain modulo p. + * + * a * r mod p, where r = 2^(6*52) mod p + * + * \param[in] a value (in radix 2^52) + * \return fesm2 value in Montgomery domain + */ +IPP_OWN_DECL(fesm2, fesm2_to_mont, (const fesm2 a)) + +/** + * \brief + * + * Conversion from Montgomery domain modulo p. + * + * a mod p + * + * \param[in] a value (in radix 2^52) + * \return fesm2 value from Montgomery + */ +IPP_OWN_DECL(fesm2, fesm2_from_mont, (const fesm2 a)) + +/** + * \brief + * convert radix 64 to (in radix 2^52) + * \param[in] arad64 ptr array size (6) + * \return fesm2 value in register 52 + */ +IPP_OWN_DECL(fesm2, fesm2_convert_radix64_radix52, (const Ipp64u* a)) + +/** + * \brief + * convert (in radix 2^52) to radix 64 + * \param[out] rrad64 ptr array size (6) + * \param[in] arad52 value (in radix 2^52) + */ +IPP_OWN_DECL(void, fesm2_convert_radix52_radix64, (Ipp64u * out, const fesm2 a)) + +#endif // (_IPP32E >= _IPP32E_K1) + +#endif // _IFMA_ARITH_PSM2_H_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_defs_sm2.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_defs_sm2.h new file mode 100644 index 0000000..0bbd8e2 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_defs_sm2.h @@ -0,0 +1,78 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ +#if !defined(_IFMA_DEFS_SM2_H_) +#define _IFMA_DEFS_SM2_H_ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "ecnist/ifma_alias_avx512.h" + +typedef m512 fesm2; + +#define DIGIT_MASK_52 (0xFFFFFFFFFFFFF) +#define DIGIT_SIZE_52 (52) +#define PSM2_LEN52 (5) +#define PSM2_LEN64 (4) + +#define REPL8(e) e, e, e, e, e, e, e, e + +/* from Montgomery conversion constant + * one + */ +static const __ALIGN64 Ipp64u PSM2_ONE52[PSM2_LEN52] = {0x1, 0x0, 0x0, 0x0, 0x0}; + +/* Montgomery(1) + * r = 2^(PSM2_LEN52*DIGIT_SIZE) mod psm2 + */ +/* r = 2^(52*6) mod psm2 */ +static const __ALIGN64 Ipp64u PSM2_R[PSM2_LEN52] = { + 0x0000000001000000, 0x000ffff000000010, 0x0000ffffffffffff, 0x0000000000000000, 0x0000010000000000}; + +/** + * \brief + * check is most significant bit + * \return mask8 + * 0xFF - is equal one + * 0x00 - is no equal one + */ +__INLINE mask8 sm2_is_msb(const mask8 a) { + return (mask8)((mask8)0 - (a >> 7)); +} + +/** + * \brief + * check is zero input value + * \param[in] value + * \return mask8 + * 0xFF - is zero value + * 0x00 - no equal zero + */ +__INLINE mask8 sm2_is_zero_i64(const m512 a) { + const mask8 mask = cmp_i64_mask(a, setzero_i64(), _MM_CMPINT_NE); + return sm2_is_msb((~mask & (mask - 1))); +} + +#define FESM2_LOADU(A) maskz_loadu_i64(0x1F, (A)) +#define FESM2_IS_ZERO(A) sm2_is_zero_i64((A)) +#define FESM2_CMP_MASK(A, B, ENUM_CMP) cmp_i64_mask((A), (B), (ENUM_CMP)) +#define FESM2_MASK_MOV(R, SRC, MASK, A) (R) = mask_mov_i64((SRC), (MASK), (A)) + +#endif // (_IPP32E >= _IPP32E_K1) + +#endif // _IFMA_DEFS_SM2_H_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ec_addpoint_sm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ec_addpoint_sm2.c new file mode 100644 index 0000000..22a3c9f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ec_addpoint_sm2.c @@ -0,0 +1,59 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpstuff.h" +#include "pcpgfpecstuff.h" + +#include "sm2/ifma_arith_method_sm2.h" +#include "sm2/ifma_ecpoint_sm2.h" + +IPP_OWN_DEFN(IppsGFpECPoint*, gfec_AddPoint_sm2_avx512, + (IppsGFpECPoint * pR, + const IppsGFpECPoint* pP, + const IppsGFpECPoint* pQ, + IppsGFpECState* pEC)) { + gsModEngine* pME = GFP_PMA(ECP_GFP(pEC)); + ifmaArithMethod* pmeth = (ifmaArithMethod*)GFP_METHOD_ALT(pME); + + __ALIGN64 PSM2_POINT_IFMA P, R; + + BNU_CHUNK_T* pPool = cpGFpGetPool(3, pME); + + recode_point_to_mont52(&P, ECP_POINT_DATA(pP), pPool, pmeth, pME); + + if (pP == pQ) { + gesm2_dbl(&R, &P); + } else { + __ALIGN64 PSM2_POINT_IFMA Q; + recode_point_to_mont52(&Q, ECP_POINT_DATA(pQ), pPool, pmeth, pME); + + gesm2_add(&R, &P, &Q); + } + + recode_point_to_mont64(pR, &R, pPool, pmeth, pME); + + cpGFpReleasePool(3, pME); + + ECP_POINT_FLAGS(pR) = gfec_IsPointAtInfinity(pR) ? 0 : ECP_FINITE_POINT; + return pR; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ec_dh_sm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ec_dh_sm2.c new file mode 100644 index 0000000..d15d807 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ec_dh_sm2.c @@ -0,0 +1,80 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpecstuff.h" +#include "pcpgfpstuff.h" + +#include "sm2/ifma_defs_sm2.h" +#include "sm2/ifma_ecpoint_sm2.h" +#include "sm2/ifma_arith_method_sm2.h" + +IPP_OWN_DEFN(int, gfec_SharedSecretDH_sm2_avx512, + (IppsGFpECPoint * pR, + const IppsGFpECPoint* pP, + const BNU_CHUNK_T* pScalar, + int scalarLen, + IppsGFpECState* pEC, + Ipp8u* pScratchBuffer)) { + FIX_BNU(pScalar, scalarLen); + { + IPP_UNREFERENCED_PARAMETER(pScratchBuffer); + + gsModEngine* pME = GFP_PMA(ECP_GFP(pEC)); + + const int orderBits = ECP_ORDBITSIZE(pEC); + const int orderLen = BITS_BNU_CHUNK(orderBits); + + ifmaArithMethod* pmeth = (ifmaArithMethod*)GFP_METHOD_ALT(pME); + + ifma_export from_radix52 = pmeth->export_to64; + + /* Mod engine (mod p) */ + ifma_decode p_from_mont = pmeth->decode; + + /* Copy scalar */ + BNU_CHUNK_T* pExtendedScalar = cpGFpGetPool(2, pME); + cpGFpElementCopyPad(pExtendedScalar, orderLen + 1, pScalar, scalarLen); + + __ALIGN64 PSM2_POINT_IFMA P52, R52; + BNU_CHUNK_T* pPool = cpGFpGetPool(3, pME); + + /* Convert point coordinates to a new Montgomery domain */ + recode_point_to_mont52(&P52, ECP_POINT_DATA(pP), pPool, pmeth, pME); + + gesm2_mul(&R52, &P52, (Ipp8u*)pExtendedScalar, orderBits); + + /* Check if the point - point to infinity */ + const mask8 is_zero_z = FESM2_IS_ZERO(R52.z); + int finite_point = ((mask8)0xFF != is_zero_z); + + /* Get X affine coordinate */ + gesm2_to_affine(/* x = */ &(R52.x), /* y = */ NULL, /* a = */ &R52); + + R52.x = p_from_mont(R52.x); + from_radix52(ECP_POINT_X(pR), R52.x); + + cpGFpReleasePool(5, pME); + + return finite_point; + } +} + +#endif // IPP32E >= _IPP32E_K1 diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ec_mulpoint_sm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ec_mulpoint_sm2.c new file mode 100644 index 0000000..45ca917 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ec_mulpoint_sm2.c @@ -0,0 +1,65 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpstuff.h" +#include "pcpgfpecstuff.h" + +#include "sm2/ifma_arith_method_sm2.h" +#include "sm2/ifma_ecpoint_sm2.h" + +IPP_OWN_DEFN(IppsGFpECPoint*, gfec_MulPoint_sm2_avx512, + (IppsGFpECPoint * pR, + const IppsGFpECPoint* pP, + const BNU_CHUNK_T* pScalar, + int scalarLen, + IppsGFpECState* pEC, + Ipp8u* pScratchBuffer)) { + IPP_UNREFERENCED_PARAMETER(pScratchBuffer); + + gsModEngine* pME = GFP_PMA(ECP_GFP(pEC)); + ifmaArithMethod* pmeth = (ifmaArithMethod*)GFP_METHOD_ALT(pME); + + const int orderBits = ECP_ORDBITSIZE(pEC); + const int orderLen = BITS_BNU_CHUNK(orderBits); + const int elemLen = GFP_PELEN(pME); + + BNU_CHUNK_T* pPool = cpGFpGetPool(5, pME); + BNU_CHUNK_T* pExtendedScalar = pPool; /* 2 pool elem to hold scalar */ + BNU_CHUNK_T* pPointPool = pPool + 2 * elemLen; /* 3 pool elem to to hold 3 point coordinates */ + + /* Copy scalar */ + cpGFpElementCopyPad(pExtendedScalar, orderLen + 1, pScalar, scalarLen); + + __ALIGN64 PSM2_POINT_IFMA P, R; + + recode_point_to_mont52(&P, ECP_POINT_DATA(pP), pPointPool, pmeth, pME); + + gesm2_mul(&R, &P, (Ipp8u*)pExtendedScalar, orderBits); + + recode_point_to_mont64(pR, &R, pPointPool, pmeth, pME); + + cpGFpReleasePool(5, pME); + + ECP_POINT_FLAGS(pR) = gfec_IsPointAtInfinity(pR) ? 0 : ECP_FINITE_POINT; + return pR; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ec_on_curve_sm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ec_on_curve_sm2.c new file mode 100644 index 0000000..7b36cb9 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ec_on_curve_sm2.c @@ -0,0 +1,45 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpecstuff.h" +#include "pcpgfpstuff.h" + +#include +#include + +IPP_OWN_DEFN(int, gfec_point_on_curve_sm2_avx512, (const IppsGFpECPoint *pPoint, IppsGFpECState *pEC)) +{ + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + ifmaArithMethod *pmeth = (ifmaArithMethod *)GFP_METHOD_ALT(pME); + + BNU_CHUNK_T *pPool = cpGFpGetPool(3, pME); + + __ALIGN64 PSM2_POINT_IFMA P; + + recode_point_to_mont52(&P, ECP_POINT_DATA(pPoint), pPool /* 3 elem */, pmeth, pME); + + const int onCurve = gesm2_is_on_curve(&P, /* use_jproj_coord = */ !IS_ECP_AFFINE_POINT(pPoint)); + + cpGFpReleasePool(3, pME); + return onCurve; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ec_pubkey_sm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ec_pubkey_sm2.c new file mode 100644 index 0000000..63b973b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ec_pubkey_sm2.c @@ -0,0 +1,75 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpstuff.h" +#include "pcpgfpecstuff.h" + +#include "sm2/ifma_ecpoint_sm2.h" +#include "sm2/ifma_arith_psm2.h" +#include "sm2/ifma_arith_method_sm2.h" + +IPP_OWN_DEFN(IppsGFpECPoint*, gfec_PubKey_sm2_avx512, + (IppsGFpECPoint * pR, + const BNU_CHUNK_T* pScalar, + int scalarLen, IppsGFpECState* pEC, + Ipp8u* pScratchBuffer)) { + FIX_BNU(pScalar, scalarLen); + { + IPP_UNREFERENCED_PARAMETER(pScratchBuffer); + + gsModEngine* pME = GFP_PMA(ECP_GFP(pEC)); + gsModEngine* nME = ECP_MONT_R(pEC); + + const int orderBits = ECP_ORDBITSIZE(pEC); + const int orderLen = BITS_BNU_CHUNK(orderBits); + + ifmaArithMethod* pmeth = (ifmaArithMethod*)GFP_METHOD_ALT(pME); + + BNU_CHUNK_T* pPool = cpGFpGetPool(5, nME); + BNU_CHUNK_T* pExtendedScalar = pPool; + BNU_CHUNK_T* pPointPool = pExtendedScalar + 2 * GFP_FELEN(pME); + + /* Copy scalar */ + cpGFpElementCopyPad(pExtendedScalar, orderLen + 1, pScalar, scalarLen); + + __ALIGN64 PSM2_POINT_IFMA R; + R.x = R.y = R.z = setzero_i64(); + + if (ECP_PREMULBP(pEC)) { + gesm2_mul_base(&R, (Ipp8u*)pExtendedScalar); + } else { + /* Convert base point to a new Montgomery domain */ + __ALIGN64 PSM2_POINT_IFMA G52; + recode_point_to_mont52(&G52, ECP_G(pEC), pPointPool /* 3 elem */, pmeth, pME); + + gesm2_mul(&R, &G52, (Ipp8u*)pExtendedScalar, orderBits); + } + + recode_point_to_mont64(pR, &R, pPointPool /* 3 elem */, pmeth, pME); + + cpGFpReleasePool(5, nME); + + ECP_POINT_FLAGS(pR) = gfec_IsPointAtInfinity(pR) ? 0 : ECP_FINITE_POINT; + return pR; + } +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ec_sign_sm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ec_sign_sm2.c new file mode 100644 index 0000000..0763d33 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ec_sign_sm2.c @@ -0,0 +1,165 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpecstuff.h" +#include "pcpgfpstuff.h" + +#include "sm2/ifma_defs_sm2.h" +#include "sm2/ifma_ecpoint_sm2.h" +#include "sm2/ifma_arith_method_sm2.h" + + +IPP_OWN_DEFN(IppStatus, gfec_Sign_sm2_avx512, + (const IppsBigNumState* pMsgDigest, + const IppsBigNumState* pRegPrivate, + IppsBigNumState* pEphPrivate, + IppsBigNumState* pSignR, IppsBigNumState* pSignS, + IppsGFpECState* pEC, + Ipp8u* pScratchBuffer)) { + IPP_UNREFERENCED_PARAMETER(pScratchBuffer); + + IppStatus sts = ippStsNoErr; + + gsModEngine* pME = GFP_PMA(ECP_GFP(pEC)); + gsModEngine* nME = ECP_MONT_R(pEC); + + const int orderBits = ECP_ORDBITSIZE(pEC); + const int orderLen = BITS_BNU_CHUNK(orderBits); + + ifmaArithMethod* pmeth = (ifmaArithMethod*)GFP_METHOD_ALT(pME); + ifmaArithMethod* nmeth = (ifmaArithMethod*)GFP_METHOD_ALT(nME); + + ifma_import to_radix52 = pmeth->import_to52; + ifma_export from_radix52 = pmeth->export_to64; + + /* Mod engine (mod p) */ + ifma_decode p_from_mont = pmeth->decode; + + /* Mod engine (mod n - subgroup order) */ + ifma_encode n_to_mont = nmeth->encode; + ifma_decode n_from_mont = nmeth->decode; + ifma_mul n_mul = nmeth->mul; + ifma_add n_add = nmeth->add; + ifma_add n_sub = nmeth->sub; + ifma_inv n_inv = nmeth->inv; + ifma_red n_red = nmeth->red; + + fesm2 sign_r, sign_s; + sign_r = sign_s = setzero_i64(); + + /* compute r-component + * 1) (x1,y1) = [s]G + * 2) r = (e + x1) mod n + */ + /* 1) (x1,y1) = [s]G */ + /* copy scalar */ + BNU_CHUNK_T* pExtendedScalar = cpGFpGetPool(2, pME); + cpGFpElementCopyPad(pExtendedScalar, orderLen + 1, BN_NUMBER(pEphPrivate), BN_SIZE(pEphPrivate)); + + __ALIGN64 PSM2_POINT_IFMA P; + + if (ECP_PREMULBP(pEC)) { + gesm2_mul_base(&P, (Ipp8u*)pExtendedScalar); + } else { + BNU_CHUNK_T* pPool = cpGFpGetPool(3, pME); + + /* Convert base point to a new Montgomery domain */ + __ALIGN64 PSM2_POINT_IFMA G52; + recode_point_to_mont52(&G52, ECP_G(pEC), pPool, pmeth, pME); + + gesm2_mul(&P, &G52, (Ipp8u*)pExtendedScalar, orderBits); + + cpGFpReleasePool(3, pME); + } + + /* extract affine P.x */ + gesm2_to_affine(/* x = */ &(P.x), /* y = */ NULL, /* a = */ &P); + P.x = p_from_mont(P.x); + P.x = n_red(P.x); + + /* 2) r = (e + x1) mod n */ + BNU_CHUNK_T* pTmp = cpGFpGetPool(1, pME); + fesm2 msg = setzero_i64(); + ZEXPAND_COPY_BNU(pTmp, orderLen, BN_NUMBER(pMsgDigest), BN_SIZE(pMsgDigest)); + msg = to_radix52((Ipp64u*)pTmp); + msg = n_red(msg); /* reduce just in case */ + + msg = n_to_mont(msg); + P.x = n_to_mont(P.x); + + sign_r = n_add(msg, P.x); + /* check r == 0 or r + k == n(0) */ + /* r == 0 */ + const mask8 sign_r_err_zeros = FESM2_IS_ZERO(sign_r); + /* r + k == 0 */ + /* extract k */ + fesm2 eph_key = setzero_i64(); + ZEXPAND_COPY_BNU(pTmp, orderLen, BN_NUMBER(pEphPrivate), BN_SIZE(pEphPrivate)); + eph_key = to_radix52((Ipp64u*)pTmp); + eph_key = n_to_mont(eph_key); + + fesm2 t; + t = n_add(eph_key, sign_r); + + const mask8 rpk_err_zeros = FESM2_IS_ZERO(t); + + if (((mask8)0xFF == sign_r_err_zeros) || ((mask8)0xFF == rpk_err_zeros)) + sts = ippStsEphemeralKeyErr; + + fesm2 reg_key = setzero_i64(); + ZEXPAND_COPY_BNU(pTmp, orderLen, BN_NUMBER(pRegPrivate), BN_SIZE(pRegPrivate)); + reg_key = to_radix52((Ipp64u*)pTmp); + + const fesm2 one = FESM2_LOADU(PSM2_ONE52); + + t = n_to_mont(one); + reg_key = n_to_mont(reg_key); + + sign_s = n_mul(sign_r, reg_key); /* sign_s = r * d */ + reg_key = n_add(reg_key, t); /* reg_key = 1 + d */ + reg_key = n_inv(reg_key); /* reg_key = (1 + d)^(-1) */ + sign_s = n_sub(eph_key, sign_s); /* sign_s = (k - r * d) */ + sign_s = n_mul(sign_s, reg_key); /* sign_s = (1 + d)^(-1) * (k - r * d) */ + + sign_s = n_from_mont(sign_s); + sign_r = n_from_mont(sign_r); + + const mask8 is_zero = (mask8)(FESM2_IS_ZERO(sign_r) | FESM2_IS_ZERO(sign_s)); + if ((mask8)0xFF == is_zero) + sts = ippStsEphemeralKeyErr; + + // return sign_s + from_radix52((Ipp64u*)pTmp, sign_s); + ZEXPAND_COPY_BNU(BN_NUMBER(pSignS), BN_SIZE(pSignS), pTmp, orderLen); + // return sign_r + from_radix52((Ipp64u*)pTmp, sign_r); + ZEXPAND_COPY_BNU(BN_NUMBER(pSignR), BN_SIZE(pSignR), pTmp, orderLen); + + /* clear secret data */ + clear_secrets(&eph_key, &(P.x), &t); + clear_secrets(&sign_r, &sign_s, ®_key); + + /* clear buffer */ + cpGFpReleasePool(3, pME); /* pExtendedScalar(2) + tmp(1) */ + return sts; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ec_verify_sm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ec_verify_sm2.c new file mode 100644 index 0000000..efb8606 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ec_verify_sm2.c @@ -0,0 +1,148 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpstuff.h" +#include "pcpgfpecstuff.h" + +#include "sm2/ifma_arith_method_sm2.h" +#include "sm2/ifma_ecpoint_sm2.h" + +IPP_OWN_DEFN(IppECResult, gfec_Verify_sm2_avx512, + (const IppsBigNumState* pMsgDigest, + const IppsGFpECPoint* pRegPublic, + const IppsBigNumState* pSignR, const IppsBigNumState* pSignS, + IppsGFpECState* pEC, + Ipp8u* pScratchBuffer)) { + IPP_UNREFERENCED_PARAMETER(pScratchBuffer); + + IppECResult verifyResult = ippECInvalidSignature; + + gsModEngine* pME = GFP_PMA(ECP_GFP(pEC)); + gsModEngine* nME = ECP_MONT_R(pEC); + + const int orderBits = ECP_ORDBITSIZE(pEC); + const int orderLen = BITS_BNU_CHUNK(orderBits); + const int elemLen = GFP_FELEN(pME); + + ifmaArithMethod* pmeth = (ifmaArithMethod*)GFP_METHOD_ALT(pME); + ifmaArithMethod* nmeth = (ifmaArithMethod*)GFP_METHOD_ALT(nME); + + ifma_import to_radix52 = pmeth->import_to52; + ifma_export from_radix52 = pmeth->export_to64; + + /* Mod engine (mod p) */ + ifma_decode p_from_mont = pmeth->decode; + + /* Mod engine (mod n - subgroup order) */ + ifma_encode n_to_mont = nmeth->encode; + ifma_decode n_from_mont = nmeth->decode; + ifma_add n_add = nmeth->add; + ifma_red n_red = nmeth->red; + + /* init message | sign_r | sing_s */ + fesm2 msg, sign_r, sign_s, t; + msg = sign_r = sign_s = t = setzero_i64(); + + /* buffer extract msg */ + BNU_CHUNK_T* pPool = cpGFpGetPool(3, pME); + BNU_CHUNK_T* pBufMsg = pPool; + BNU_CHUNK_T* pBufSignR = pPool + elemLen; + BNU_CHUNK_T* pBufSignS = pPool + 2 * elemLen; + + ZEXPAND_COPY_BNU(pBufMsg, orderLen, BN_NUMBER(pMsgDigest), BN_SIZE(pMsgDigest)); + ZEXPAND_COPY_BNU(pBufSignS, orderLen, BN_NUMBER(pSignS), BN_SIZE(pSignS)); + ZEXPAND_COPY_BNU(pBufSignR, orderLen, BN_NUMBER(pSignR), BN_SIZE(pSignR)); + + /* radix64 -> radix52 */ + msg = to_radix52((Ipp64u*)pBufMsg); + msg = n_red(msg); /* reduce just in case */ + sign_r = to_radix52((Ipp64u*)pBufSignR); + sign_s = to_radix52((Ipp64u*)pBufSignS); + + /* Convert public point to proper Montgomery domain and 2^52 radix */ + __ALIGN64 PSM2_POINT_IFMA P; + recode_point_to_mont52(&P, ECP_POINT_DATA(pRegPublic), pPool /* 3 elem */, pmeth, pME); + + /* compute t = (r + s) mod n */ + t = n_to_mont(sign_r); + sign_s = n_to_mont(sign_s); + + t = n_add(sign_s, t); + + t = n_from_mont(t); + /* check zeros t != 0 */ + const mask8 sign_err_mask = FESM2_IS_ZERO(t); + if ((mask8)0xFF == sign_err_mask) + verifyResult = ippECInvalidSignature; + + BNU_CHUNK_T* pExtendedS = cpGFpGetPool(2, pME); + BNU_CHUNK_T* pExtendedT = cpGFpGetPool(2, pME); + BNU_CHUNK_T* pTmp = cpGFpGetPool(1, pME); + + /* compute [s]G + t[P] */ + + /* copmute [s]G */ + __ALIGN64 PSM2_POINT_IFMA sG; + /* create s */ + cpGFpElementCopyPad(pExtendedS, orderLen + 1, pBufSignS, orderLen); + if (ECP_PREMULBP(pEC)) { + gesm2_mul_base(&sG, (Ipp8u*)pExtendedS); + } else { + /* Convert base point to a new Montgomery domain */ + __ALIGN64 PSM2_POINT_IFMA G52; + recode_point_to_mont52(&G52, ECP_G(pEC), pPool /* 3 elem */, pmeth, pME); + + gesm2_mul(&sG, &G52, (Ipp8u*)pExtendedS, orderBits); + } + + /* create t */ + from_radix52((Ipp64u*)pTmp, t); + cpGFpElementCopyPad(pExtendedT, orderLen + 1, pTmp, orderLen); + /* compute [t]P */ + gesm2_mul(&P, &P, (Ipp8u*)pExtendedT, orderBits); + /* compute [s]G + [t]P */ + gesm2_add(&P, &P, &sG); + + fesm2 sign_r_restore; + /* extract X affine coordinate */ + gesm2_to_affine(/* x = */ &sign_r_restore, /* y = */ NULL, &P); + sign_r_restore = p_from_mont(sign_r_restore); + /* x = x mod n */ + sign_r_restore = n_red(sign_r_restore); + + /* R = (e + x1) mod n */ + msg = n_to_mont(msg); + sign_r_restore = n_to_mont(sign_r_restore); + + sign_r_restore = n_add(msg, sign_r_restore); + + sign_r_restore = n_from_mont(sign_r_restore); + + const mask8 mask_ok = cmp_i64_mask(sign_r_restore, sign_r, _MM_CMPINT_EQ); + if ((mask8)0xFF == mask_ok) + verifyResult = ippECValid; + + cpGFpReleasePool(8, pME); + + return verifyResult; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ecpoint_sm2.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ecpoint_sm2.c new file mode 100644 index 0000000..c22874e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ecpoint_sm2.c @@ -0,0 +1,748 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpecstuff.h" + +#include "sm2/ifma_arith_psm2.h" +#include "sm2/ifma_defs_sm2.h" +#include "sm2/ifma_ecpoint_sm2.h" + +/* 2*p */ +static const __ALIGN64 Ipp64u psm2_x2[PSM2_LEN52] = { + 0x000ffffffffffffe, 0x000fe00000001fff, 0x000fffffffffffff, 0x000fffffffffffff, 0x0001fffffffdffff}; +/* 4*p */ +static const __ALIGN64 Ipp64u psm2_x4[PSM2_LEN52] = { + 0x000ffffffffffffc, 0x000fc00000003fff, 0x000fffffffffffff, 0x000fffffffffffff, 0x0003fffffffbffff}; +/* 6*p */ +static const __ALIGN64 Ipp64u psm2_x6[PSM2_LEN52] = { + 0x000ffffffffffffa, 0x000fa00000005fff, 0x000fffffffffffff, 0x000fffffffffffff, 0x0005fffffff9ffff}; +/* 8*p */ +static const __ALIGN64 Ipp64u psm2_x8[PSM2_LEN52] = { + 0x000ffffffffffff8, 0x000f800000007fff, 0x000fffffffffffff, 0x000fffffffffffff, 0x0007fffffff7ffff}; + +/* Mont(a) = a*r mod psm2, where r = 2^(6*52) mod psm2 */ +static const __ALIGN64 Ipp64u psm2_a[PSM2_LEN52] = { + 0x000ffffffcffffff, 0x000ff03000000fcf, 0x000cffffffffffff, 0x000fffffffffffff, 0x0000fcfffffeffff}; + +/* Mont(b) = b*r mod psm2, where r = 2^(6*52) mod psm2 */ +static const __ALIGN64 Ipp64u psm2_b[PSM2_LEN52] = { + 0x00040fe188de30c4, 0x00012da4d9019422, 0x000b0dc519344af3, 0x000a51c3c71cf379, 0x00005130aa45505e}; + + +#define add(R, A, B) (R) = fesm2_add_no_red((A), (B)) +#define sub(R, A, B) (R) = fesm2_sub_no_red((A), (B)) +#define mul(R, A, B) (R) = fesm2_mul((A), (B)) +#define sqr(R, A) (R) = fesm2_sqr((A)) +#define div2(R, A) (R) = fesm2_div2_norm((A)) +#define norm(R, A) (R) = ifma_norm52((A)) +#define lnorm(R, A) (R) = ifma_lnorm52((A)) +#define inv(R, A) (R) = fesm2_inv_norm((A)) +#define from_mont(R, A) (R) = fesm2_from_mont((A)) +/* duplicate mult/sqr/norm */ +#define mul_dual(R1, A1, B1, R2, A2, B2) fesm2_mul_dual(&(R1), (A1), (B1), &(R2), (A2), (B2)) +#define sqr_dual(R1, A1, R2, A2) fesm2_sqr_dual(&(R1), (A1), &(R2), (A2)) +#define norm_dual(R1, A1, R2, A2) ifma_norm52_dual(&(R1), (A1), &(R2), (A2)) +#define lnorm_dual(R1, A1, R2, A2) ifma_lnorm52_dual(&(R1), (A1), &(R2), (A2)) + +IPP_OWN_DEFN(void, gesm2_to_affine, (fesm2 prx[], fesm2 pry[], const PSM2_POINT_IFMA* a)) { + + fesm2 z1, z2, z3; + z1 = z2 = z3 = setzero_i64(); + + inv(z1, a->z); /* 1/z */ + sqr(z2, z1); /* (1/z)^2 */ + lnorm(z2, z2); /**/ + + /* x = x/z^2 */ + if (NULL != prx) { + mul(*prx, a->x, z2); /* x = x/z^2 */ + lnorm(*prx, *prx); /**/ + } + /* y = y/z^3 */ + if (NULL != pry) { + mul(z3, z1, z2); /* (1/z)^3 */ + lnorm(z3, z3); /**/ + mul(*pry, a->y, z3); /* y = y/z^3 */ + lnorm(*pry, *pry); /**/ + } + + return; +} + +IPP_OWN_DEFN(void, gesm2_dbl, (PSM2_POINT_IFMA * r, const PSM2_POINT_IFMA* p)) { + /* + * Algorithm (Gueron - Enhanced Montgomery Multiplication) + * l1 = 3x^2 + a*z^4 = (if sm2 a = -3) = 3*(x^2 - z^4) = 3*(x - z^2)*(x + z^2) + * z2 = 2*y*z + * l2 = 4*x*y^2 + * x2 = l1^2 - 2*l2 + * l3 = 8*y^4 + * y2 = l1*(l2 - x2) - l3 + * + * sum aripmetic: 8 mul; 9 add/sub; 1 div2. + */ + const fesm2* x1 = &p->x; + const fesm2* y1 = &p->y; + const fesm2* z1 = &p->z; + + fesm2 x2; + fesm2 y2; + fesm2 z2; + x2 = y2 = z2 = setzero_i64(); + + fesm2 T, U, V, A, B, H; + T = U = V = setzero_i64(); + A = B = H = setzero_i64(); + + const fesm2 M2 = FESM2_LOADU(psm2_x2); /* 2*p */ + const fesm2 M4 = FESM2_LOADU(psm2_x4); /* 4*p */ + const fesm2 M8 = FESM2_LOADU(psm2_x8); /* 8*p */ + + add(T, *y1, *y1); /* T = 2*y1 */ + lnorm(T, T); /**/ + /*=====*/ + sqr_dual(V, T, /* V = 4*y1^2 */ + U, *z1); /* U = z1^2 */ + /*=====*/ + sub(B, *x1, U); /* B = 2*p + x1 - z1^2 */ + add(B, B, M2); /**/ + add(U, *x1, U); /* U = x1 + z1^2 */ + /*=====*/ + /* normalization */ + lnorm_dual(V, V, /**/ + U, U); /**/ + norm(B, B); /**/ + /*=====*/ + mul_dual(A, V, *x1, /* A = 4*x1*y1^2 */ + B, B, U); /* B = (x1 - z1^2)*(x1 + z1^2) */ + /*=====*/ + add(x2, A, A); /* x2 = 8*x1*y1^2 (4p) */ + add(H, B, B); /**/ + add(B, B, H); /* B(l1) = 3*(x1 - z1^2)*(x1 + z1^2) */ + /*=====*/ + /* normalization */ + lnorm(B, B); /**/ + /*=====*/ + sqr_dual(U, B, /* U = l1^2 */ + y2, V); /* y2 = 16*y^2 */ + /*=====*/ + sub(x2, U, x2); /* x2 = 4*p + l1^2 - 2*l2 */ + add(x2, x2, M4); /**/ + div2(y2, y2); /**/ + /*=====*/ + sub(U, A, x2); /* U = 8*p + l2 - x2 */ + add(U, U, M8); /**/ + /*=====*/ + /* normalization */ + norm(U, U); /**/ + /*=====*/ + mul_dual(z2, T, *z1, /* z2 = 2*y1*z1 */ + U, U, B); /* U = B(l1)*(A(l2) - x2) */ + /*=====*/ + sub(y2, U, y2); /* y2 = 2*p + B(l1)*(A(l2) - x2) - y2(l3) */ + add(y2, y2, M2); /**/ + /*=====*/ + /* normalization */ + norm_dual(r->x, x2, /**/ + r->y, y2); /**/ + lnorm(r->z, z2); /**/ + return; +} + +IPP_OWN_DEFN(void, gesm2_add, (PSM2_POINT_IFMA * r, const PSM2_POINT_IFMA* p, const PSM2_POINT_IFMA* q)) { + /* + * Algorithm (Gueron - Enhanced Montgomery Multiplication) + * + * A = x1*z2^2 B = x2*z1^2 C = y1*z2^3 D = y2*z1^3 + * E = B - A F = D - C + * x3 = -E^3 - 2*A*E^2 + F^2 + * y3 = -C*E^3 + F*(A*E^2 - x3) + * z3 = z1*z2*E + */ + const fesm2* x1 = &p->x; + const fesm2* y1 = &p->y; + const fesm2* z1 = &p->z; + const mask8 p_is_inf = FESM2_IS_ZERO(p->z); + + const fesm2* x2 = &q->x; + const fesm2* y2 = &q->y; + const fesm2* z2 = &q->z; + const mask8 q_is_inf = FESM2_IS_ZERO(q->z); + + fesm2 x3; + fesm2 y3; + fesm2 z3; + x3 = y3 = z3 = setzero_i64(); + + const fesm2 M2 = FESM2_LOADU(psm2_x2); /* 2*p */ + const fesm2 M4 = FESM2_LOADU(psm2_x4); /* 4*p */ + const fesm2 M8 = FESM2_LOADU(psm2_x8); /* 8*p */ + + fesm2 U1, U2, S1, S2, H, R; + U1 = U2 = S1 = setzero_i64(); + S2 = H = R = setzero_i64(); + + mul_dual(S1, *y1, *z2, /* s1 = y1*z2 */ + U1, *z2, *z2); /* u1 = z2^2 */ + /*=====*/ + /* normalization */ + lnorm_dual(S1, S1, /**/ + U1, U1); /**/ + /*=====*/ + mul_dual(S2, *y2, *z1, /* s2 = y2*z1 */ + U2, *z1, *z1); /* u2 = z1^2 */ + /*=====*/ + /* normalization */ + lnorm_dual(S2, S2, /**/ + U2, U2); /**/ + /*=====*/ + mul_dual(S1, S1, U1, /* s1 = y1*z2^3 (C) */ + S2, S2, U2); /* s2 = y2*z1^3 (D) */ + /*=====*/ + /* normalization */ + lnorm_dual(S1, S1, /**/ + S2, S2); /* (need by correct compute F = D - C) */ + /*=====*/ + mul_dual(U1, *x1, U1, /* u1 = x1*z2^2 (A) */ + U2, *x2, U2); /* u2 = x2*z1^2 (B) */ + /*=====*/ + /* normalization */ + lnorm_dual(U1, U1, /**/ + U2, U2); /**/ + /*=====*/ + sub(R, S2, S1); /* r = D - C (F) */ + sub(H, U2, U1); /* h = B - A (E) */ + + /* checking the equality of X and Y coordinates (D - C == 0) and (B - A == 0) */ + const mask8 f_are_zero = FESM2_IS_ZERO(R); + const mask8 e_are_zero = FESM2_IS_ZERO(H); + const mask8 point_is_equal = ((e_are_zero & f_are_zero) & (~p_is_inf) & (~q_is_inf)); + + __ALIGN64 PSM2_POINT_IFMA r2; + r2.x = r2.y = r2.z = setzero_i64(); + if ((mask8)0xFF == point_is_equal) { + gesm2_dbl(&r2, p); + } + + add(R, R, M2); /**/ + add(H, H, M2); /**/ + /*=====*/ + /* normalization */ + norm_dual(R, R, /**/ + H, H); /**/ + /**/ + mul_dual(z3, *z1, *z2, /* z3 = z1*z2 */ + U2, H, H); /* u2 = E^2 */ + /*=====*/ + /* normalization */ + lnorm_dual(z3, z3, /**/ + U2, U2); /**/ + /**/ + mul_dual(z3, z3, H, /* z3 = (z1*z2)*E */ + S2, R, R); /* s2 = F^2 */ + mul(H, H, U2); /* h = E^3 */ + /*=====*/ + /* normalization */ + lnorm(H, H); /**/ + /*=====*/ + mul(U1, U1, U2); /* u1 = A*E^2 */ + sub(x3, S2, H); /* x3 = F^2 - E^3 */ + add(x3, x3, M2); /**/ + add(U2, U1, U1); /* u2 = 2*A*E^2 */ + mul(S1, S1, H); /* s1 = C*E^3 */ + sub(x3, x3, U2); /* x3 = (F^2 - E^3) -2*A*E^2 */ + add(x3, x3, M4); /**/ + /*=====*/ + sub(y3, U1, x3); /* y3 = A*E^2 - x3 */ + add(y3, y3, M8); /**/ + /*=====*/ + /* normalization */ + norm(y3, y3); /**/ + /**/ + mul(y3, y3, R); /* y3 = F*(A*E^2 - x3) */ + sub(y3, y3, S1); /* y3 = F*(A*E^2 - x3) - C*E^3 */ + add(y3, y3, M2); + + /* normalization */ + norm_dual(x3, x3, /**/ + y3, y3); /**/ + lnorm(z3, z3); /**/ + + /* T = p_is_inf ? q : T */ + FESM2_MASK_MOV(x3, x3, p_is_inf, *x2); + FESM2_MASK_MOV(y3, y3, p_is_inf, *y2); + FESM2_MASK_MOV(z3, z3, p_is_inf, *z2); + + /* T = q_is_inf ? p : T */ + FESM2_MASK_MOV(x3, x3, q_is_inf, *x1); + FESM2_MASK_MOV(y3, y3, q_is_inf, *y1); + FESM2_MASK_MOV(z3, z3, q_is_inf, *z1); + + /* r = point_is_equal ? r2 : T */ + FESM2_MASK_MOV(x3, x3, point_is_equal, r2.x); + FESM2_MASK_MOV(y3, y3, point_is_equal, r2.y); + FESM2_MASK_MOV(z3, z3, point_is_equal, r2.z); + + r->x = x3; + r->y = y3; + r->z = z3; + + return; +} + +IPP_OWN_DEFN(void, gesm2_add_affine, (PSM2_POINT_IFMA * r, const PSM2_POINT_IFMA* p, const PSM2_AFFINE_POINT_IFMA* q)) { + /* + * Algorithm (Gueron - Enhanced Montgomery Multiplication) + * + * A = x1 B = x2*z1^2 C = y1 D = y2*z1^3 + * E = B - A(x1) F = D - C(y1) + * x3 = -E^3 - 2*A(x1)*E^2 + F^2 + * y3 = -C(y1)*E^3 + F*(A(x1)*E^2 - x3) + * z3 = z1*E + */ + /* coordinates of p (jacobian projective) */ + const fesm2* x1 = &p->x; + const fesm2* y1 = &p->y; + const fesm2* z1 = &p->z; + const mask8 p_is_inf = FESM2_IS_ZERO(p->z); + + /* coodinate of q (affine) */ + const fesm2* x2 = &q->x; + const fesm2* y2 = &q->y; + const mask8 q_is_inf = (FESM2_IS_ZERO(q->x) & FESM2_IS_ZERO(q->y)); + + fesm2 x3; + fesm2 y3; + fesm2 z3; + x3 = y3 = z3 = setzero_i64(); + + const fesm2 M2 = FESM2_LOADU(psm2_x2); /* 2*p */ + const fesm2 M4 = FESM2_LOADU(psm2_x4); /* 4*p */ + const fesm2 M8 = FESM2_LOADU(psm2_x8); /* 8*p */ + + fesm2 U2, S2, H, R; + U2 = S2 = H = R = setzero_i64(); + + mul_dual(R, *z1, *z1, /* R = z1^2 */ + S2, *y2, *z1); /* S2 = y2*z1 */ + /*=====*/ + lnorm_dual(R, R, /**/ + S2, S2); /**/ + /*=====*/ + mul_dual(U2, *x2, R, /* U2 = x2*z1^2 (B) */ + S2, S2, R); /* S2 = y2*z1^3 (D) */ + /**/ + sub(H, U2, *x1); /* H = B - A (E) */ + add(H, H, M8); /**/ + sub(R, S2, *y1); /* R = D - C (F) */ + add(R, R, M4); /**/ + /*=====*/ + norm_dual(H, H, /**/ + R, R); /**/ + /**/ + mul(z3, H, *z1); /* z3 = z1*E */ + /**/ + sqr_dual(U2, H, /* U2 = E^2 */ + S2, R); /* S2 = F^2 */ + /**/ + lnorm(U2, U2); /**/ + /**/ + mul(H, H, U2); /* H = E^3 */ + /**/ + lnorm(H, H); /**/ + /**/ + mul_dual(U2, U2, *x1, /* U2 = A*E^2 */ + y3, H, *y1); /* y3 = C*E^3 */ + /**/ + add(x3, U2, U2); /* x2 = 2*A*E^2 */ + sub(x3, S2, x3); /* x3 = F^2 - 2*A*E^2 */ + add(x3, x3, M4); /**/ + sub(x3, x3, H); /* x3 = F^2 - 2*A*E^2 - E^3 */ + add(x3, x3, M2); /**/ + /**/ + sub(U2, U2, x3); /* U2 = A*E^2 - x3 */ + add(U2, U2, M8); /**/ + norm(U2, U2); /**/ + mul(U2, U2, R); /* U2 = F*(A*E^2 - x3) */ + sub(y3, U2, y3); /* y3 = F*(A*E^2 - x3) - C*E^2 */ + add(y3, y3, M2); /**/ + + /* normalization */ + norm_dual(x3, x3, /**/ + y3, y3); /**/ + lnorm(z3, z3); /**/ + + const fesm2 ONE = FESM2_LOADU(PSM2_R); + /* T = p_is_inf ? q : T */ + FESM2_MASK_MOV(x3, x3, p_is_inf, *x2); + FESM2_MASK_MOV(y3, y3, p_is_inf, *y2); + FESM2_MASK_MOV(z3, z3, p_is_inf, ONE); + + /* T = q_is_inf ? p : T */ + FESM2_MASK_MOV(x3, x3, q_is_inf, *x1); + FESM2_MASK_MOV(y3, y3, q_is_inf, *y1); + FESM2_MASK_MOV(z3, z3, q_is_inf, *z1); + + r->x = x3; + r->y = y3; + r->z = z3; + return; +} + +IPP_OWN_DEFN(int, gesm2_is_on_curve, (const PSM2_POINT_IFMA* p, const int use_jproj_coords)) { + /* + * Algorithm + * + * y^2 = x^3 + a*x + b (1) + * + * if input + * * Jacobian projection coordinate (x,y,z) - represent by (x/z^2,y/z^3,1) + * * Affine coodinate (x/z^2,y/z^3,z/z=1) + * + * mult formala (1) by z^6 + * + * y^2 = x^3 + a*x*z^4 + b*z^6 + */ + const fesm2 M6 = FESM2_LOADU(psm2_x6); + const fesm2 a = FESM2_LOADU(psm2_a); + const fesm2 b = FESM2_LOADU(psm2_b); + + fesm2 rh, Z4, Z6, tmp; + rh = Z4 = Z6 = tmp = setzero_i64(); + + sqr(rh, p->x); /* rh = x^2 */ + /* rh = x*(x^2 + a*z^4) + b*z^6 = x*(x^2 - 3*z^4) + b*z^6 */ + if (0 != use_jproj_coords) { + sqr(tmp, p->z); /* tmp = z^2 */ + lnorm(tmp, tmp); /**/ + /**/ + sqr(Z4, tmp); /* z4 = z^4 */ + lnorm(Z4, Z4); /**/ + mul(Z6, Z4, tmp); /* z6 = z^6 */ + lnorm(Z6, Z6); /**/ + /**/ + add(tmp, Z4, Z4); /* tmp = 2*z^4 */ + add(tmp, tmp, Z4); /* tmp = 3*z^4 */ + /**/ + sub(rh, rh, tmp); /* rh = x^2 - 3*z^4 */ + add(rh, rh, M6); /**/ + norm(rh, rh); /**/ + /**/ + mul_dual(rh, rh, p->x, /* rh = x*(x^2 - 3*z^4) */ + tmp, Z6, b); /* tmp = b*z^6 */ + /**/ + add(rh, rh, tmp); /* rh = x*(x^2 - 3*z^4) + b*z^6 */ + } + /* rh = x*(x^2 + a) + b */ + else { + add(rh, rh, a); /* rh = x^2 + a */ + lnorm(rh, rh); /**/ + mul(rh, rh, p->x); /* rh = x*(x^2 + a) */ + add(rh, rh, b); /* rh = x*(x^2 + a) + b */ + } + lnorm(rh, rh); /**/ + + /* rl = Y^2 */ + sqr(tmp, p->y); /* tmp = y^2 */ + lnorm(tmp, tmp); /**/ + + /* from mont */ + from_mont(tmp, tmp); + from_mont(rh, rh); + + const mask8 mask = FESM2_CMP_MASK(rh, tmp, _MM_CMPINT_EQ); + return (mask == 0xFF) ? 1 : 0; +} + +#undef add +#undef sub +#undef mul +#undef sqr +#undef div2 +#undef norm +#undef from_mont +#undef mul_dual +#undef sqr_dual +#undef norm_dual + +static __NOINLINE void clear_secret_context(Ipp16u* wval, + Ipp32s* chunk_no, Ipp32s* chunk_shift, + Ipp8u* sign, Ipp8u* digit, + PSM2_POINT_IFMA* R, PSM2_POINT_IFMA* H, + PSM2_AFFINE_POINT_IFMA* A) { + *wval = 0; + *chunk_no = 0; + *chunk_shift = 0; + *sign = 0; + *digit = 0; + + if (NULL != R) + (*R).x = (*R).y = (*R).z = setzero_i64(); + if (NULL != H) + (*H).x = (*H).y = (*H).z = setzero_i64(); + if (NULL != A) + (*A).x = (*A).y = setzero_i64(); + return; +} + +__INLINE mask8 is_eq_mask(const Ipp32s a, const Ipp32s b) { + const Ipp32s eq = a ^ b; + const Ipp32s v = ~eq & (eq - 1); + const Ipp32s msb = 0 - (v >> (sizeof(a) * 8 - 1)); + return (mask8)(0 - msb); +} + +#define WIN_SIZE (5) + +static void table_get_point(PSM2_POINT_IFMA* r, const Ipp32s digit, const PSM2_POINT_IFMA tbl[]) { + const Ipp32s idx = digit - 1; + + __ALIGN64 PSM2_POINT_IFMA R; + R.x = R.y = R.z = setzero_i64(); + + for (Ipp32s n = 0; n < (1 << (WIN_SIZE - 1)); ++n) { + const mask8 mask = is_eq_mask(n, idx); + + FESM2_MASK_MOV(R.x, R.x, mask, tbl[n].x); + FESM2_MASK_MOV(R.y, R.y, mask, tbl[n].y); + FESM2_MASK_MOV(R.z, R.z, mask, tbl[n].z); + } + + r->x = R.x; + r->y = R.y; + r->z = R.z; + return; +} + +#define dbl_point gesm2_dbl +#define add_point gesm2_add +#define neg_coord(R, A) (R) = fesm2_neg_norm((A)) +#define add_point_affine gesm2_add_affine + +/* r = n*P = (P + P + ... + P) */ +IPP_OWN_DEFN(void, gesm2_mul, (PSM2_POINT_IFMA * r, const PSM2_POINT_IFMA* p, const Ipp8u* pExtendedScalar, const int scalarBitSize)) { + /* default params */ + __ALIGN64 PSM2_POINT_IFMA tbl[(1 << (WIN_SIZE - 1))]; + + __ALIGN64 PSM2_POINT_IFMA R; + __ALIGN64 PSM2_POINT_IFMA H; + + fesm2 negHy; + negHy = setzero_i64(); + R.x = R.y = R.z = setzero_i64(); + H.x = H.y = H.z = setzero_i64(); + + /* compute tbl[] = [n]P, n = 1, ... , 2^(win_size - 1) + * tbl[2*n] = tbl[2*n - 1] + p + * tbl[2*n + 1] = [2]*tbl[n] + */ + /* tbl[0] = p */ + tbl[0].x = p->x; + tbl[0].y = p->y; + tbl[0].z = p->z; + + /* tbl[1] = [2]*p */ + dbl_point(/* r = */ (tbl + 1), /* a = */ p); + for (int n = 1; n < ((1 << (WIN_SIZE - 1)) / 2); ++n) { + add_point(/* r = */ (tbl + 2 * n), /* a = */ (tbl + 2 * n - 1), /* b = */ p); + dbl_point(/* r = */ (tbl + 2 * n + 1), /* a = */ (tbl + n)); + } + + Ipp16u wval; + Ipp8u digit, sign; + mask8 mask_neg; + const Ipp32s mask = ((1 << (WIN_SIZE + 1)) - 1); /* mask 0b111111 */ + Ipp32s bit = scalarBitSize - (scalarBitSize % WIN_SIZE); + + Ipp32s chunk_no = (bit - 1) / 8; + Ipp32s chunk_shift = (bit - 1) % 8; + + if (0 != bit) { + wval = *((Ipp16u*)(pExtendedScalar + chunk_no)); + wval = (Ipp16u)((wval >> chunk_shift) & mask); + } else { + wval = 0; + } + + booth_recode(/* sign = */ &sign, /* digit = */ &digit, /* in = */ (Ipp8u)wval, /* w = */ WIN_SIZE); + table_get_point(/* r = */ &R, /* digit = */ (Ipp32s)digit, /* tbl = */ tbl); + + for (bit -= WIN_SIZE; bit >= WIN_SIZE; bit -= WIN_SIZE) { + dbl_point(&R, &R); + dbl_point(&R, &R); + dbl_point(&R, &R); + dbl_point(&R, &R); +#if (WIN_SIZE == 5) + dbl_point(&R, &R); +#endif + + chunk_no = (bit - 1) / 8; + chunk_shift = (bit - 1) % 8; + + wval = *((Ipp16u*)(pExtendedScalar + chunk_no)); + wval = (Ipp16u)((wval >> chunk_shift) & mask); + + booth_recode(/* sign = */ &sign, /* digit = */ &digit, /* in = */ (Ipp8u)wval, /* w = */ WIN_SIZE); + table_get_point(/* r = */ &H, /* idx = */ (Ipp32s)digit, /* tbl = */ tbl); + + neg_coord(negHy, H.y); + + mask_neg = (mask8)(~(sign - 1)); + FESM2_MASK_MOV(H.y, H.y, mask_neg, negHy); + add_point(/* r = */ &R, /* a = */ &R, /* b = */ &H); + } + + /* last window */ + dbl_point(&R, &R); + dbl_point(&R, &R); + dbl_point(&R, &R); + dbl_point(&R, &R); +#if (WIN_SIZE == 5) + dbl_point(&R, &R); +#endif + + wval = *((Ipp16u*)(pExtendedScalar + 0)); + wval = (wval << 1) & mask; + + booth_recode(/* sign = */ &sign, /* digit = */ &digit, /* in = */ (Ipp8u)wval, /* w = */ WIN_SIZE); + table_get_point(/* r = */ &H, /* idx = */ (Ipp32s)digit, /* tbl = */ tbl); + + neg_coord(negHy, H.y); + + mask_neg = (mask8)(~(sign - 1)); + FESM2_MASK_MOV(H.y, H.y, mask_neg, negHy); + + add_point(/* r = */ &R, /* a = */ &R, /* b = */ &H); + + r->x = R.x; + r->y = R.y; + r->z = R.z; + + /* clear secret data */ + clear_secret_context(&wval, + &chunk_no, &chunk_shift, + &sign, &digit, + &R, &H, NULL); + return; +} + +#include "sm2/ifma_ecprecomp7_sm2.h" + +/* affine */ +#define BP_WIN_SIZE BASE_POINT_WIN_SIZE +#define BP_N_ENTRY BASE_POINT_N_ENTRY + +__INLINE void extract_point_affine(PSM2_AFFINE_POINT_IFMA* r, + const SINGLE_PSM2_AFFINE_POINT_IFMA* tbl, + const Ipp32s digit) { + const Ipp32s idx = digit - 1; + + __ALIGN64 PSM2_AFFINE_POINT_IFMA R; + R.x = R.y = setzero_i64(); + + for (Ipp32s n = 0; n < (1 << ((BP_WIN_SIZE)-1)); ++n, ++tbl) { + const mask8 mask = is_eq_mask(n, idx); + + FESM2_MASK_MOV(R.x, R.x, mask, FESM2_LOADU(tbl->x)); + FESM2_MASK_MOV(R.y, R.y, mask, FESM2_LOADU(tbl->y)); + } + + r->x = R.x; + r->y = R.y; + return; +} + +IPP_OWN_DEFN(void, gesm2_select_ap_w7_ifma, (BNU_CHUNK_T * pAffinePoint, const BNU_CHUNK_T* pTable, int index)) { + __ALIGN64 PSM2_AFFINE_POINT_IFMA ap; + + extract_point_affine(&ap, (SINGLE_PSM2_AFFINE_POINT_IFMA*)pTable, index); + + ap.x = fesm2_from_mont(ap.x); + ap.y = fesm2_from_mont(ap.y); + + fesm2_convert_radix52_radix64(pAffinePoint, ap.x); + fesm2_convert_radix52_radix64(pAffinePoint + PSM2_LEN64, ap.y); +} + +IPP_OWN_DEFN(void, gesm2_mul_base, (PSM2_POINT_IFMA * r, const Ipp8u* pExtendedScalar)) { + const SINGLE_PSM2_AFFINE_POINT_IFMA* tbl = &ifma_ec_sm2_bp_precomp[0][0]; + + __ALIGN64 PSM2_POINT_IFMA R; + __ALIGN64 PSM2_AFFINE_POINT_IFMA A; + fesm2 Ty; + R.x = R.y = R.z = setzero_i64(); + A.x = A.y = setzero_i64(); + Ty = setzero_i64(); + + Ipp16u wval; + Ipp8u digit, sign; + const Ipp32s mask = ((1 << (BP_WIN_SIZE + 1)) - 1); /* mask 0b11111 */ + Ipp32s bit = 0; + Ipp32s chunk_no, chunk_shift; + + wval = *((Ipp16u*)(pExtendedScalar + 0)); + wval = (Ipp16u)((wval << 1) & mask); + + booth_recode(&sign, &digit, (Ipp8u)wval, BP_WIN_SIZE); + extract_point_affine(&A, tbl, (Ipp32s)digit); + tbl += BP_N_ENTRY; + + /* A = sign == 1 ? -A : A */ + neg_coord(Ty, A.y); + mask8 mask_neg = (mask8)(~(sign - 1)); + FESM2_MASK_MOV(A.y, A.y, mask_neg, Ty); + + /* R += A */ + add_point_affine(&R, &R, &A); + + for (bit += BP_WIN_SIZE; bit <= 256; bit += BP_WIN_SIZE) { + chunk_no = (bit - 1) / 8; + chunk_shift = (bit - 1) % 8; + + wval = *((Ipp16u*)(pExtendedScalar + chunk_no)); + wval = (Ipp16u)((wval >> chunk_shift) & mask); + + booth_recode(&sign, &digit, (Ipp8u)wval, BP_WIN_SIZE); + extract_point_affine(&A, tbl, (Ipp32s)digit); + tbl += BP_N_ENTRY; + + /* A = sign == 1 ? -A : A */ + neg_coord(Ty, A.y); + mask_neg = (mask8)(~(sign - 1)); + FESM2_MASK_MOV(A.y, A.y, mask_neg, Ty); + + /* R += A */ + add_point_affine(&R, &R, &A); + } + + r->x = R.x; + r->y = R.y; + r->z = R.z; + + /* clear secret data */ + clear_secret_context(&wval, + &chunk_no, &chunk_shift, + &sign, &digit, + &R, NULL, &A); + return; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ecpoint_sm2.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ecpoint_sm2.h new file mode 100644 index 0000000..682fd3a --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ecpoint_sm2.h @@ -0,0 +1,193 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +/* Paper referenced in this header: + * + * [1] "Enhanced Montgomery Multiplication" DOI:10.1155/2008/583926 + * + */ + +#if !defined(_IFMA_ECPOINT_SM2_H_) +#define _IFMA_ECPOINT_SM2_H_ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "sm2/ifma_arith_psm2.h" +#include "pcpbnuimpl.h" + +/* SM2 point (x,y,z) */ +typedef struct PSM2_POINT_IFMA { + fesm2 x; + fesm2 y; + fesm2 z; +} PSM2_POINT_IFMA; + +/* SM2 affine point (x,y) */ +typedef struct PSM2_AFFINE_POINT_IFMA { + fesm2 x; + fesm2 y; +} PSM2_AFFINE_POINT_IFMA; + +/** + * \brief + * + * compute R = [pExtendedScalar]*P = (P + P + ... + P) + * + * \param[out] r point (in radix 2^52) + * \param[in] p point (in radix 2^52) + * \param[in] pExtendedScalar ptr Extended scalar + * \param[in] scalarBitSize size bits scalar + */ +IPP_OWN_DECL(void, gesm2_mul, (PSM2_POINT_IFMA * r, const PSM2_POINT_IFMA* p, const Ipp8u* pExtendedScalar, const int scalarBitSize)) + +/** + * \brief + * + * compute R = [pExtendedScalar]*BasePoint + * + * \param[out] r point (in radix 2^52) + * \param pExtendedScalar ptr scaler (length 256 bits) + */ +IPP_OWN_DECL(void, gesm2_mul_base, (PSM2_POINT_IFMA * r, const Ipp8u* pExtendedScalar)) + +/** + * \brief + * + * convert point to affine coordinate + * + * \param[out] r point affine coordinate + * \param[in] a point + */ +IPP_OWN_DECL(void, gesm2_to_affine, (fesm2 px[], fesm2 py[], const PSM2_POINT_IFMA* a)) + +/** + * \brief + * + * check point on curve SM2 + * + * \param[in] p point (in radix 2^52) + * \param[in] use_jproj_coords is Jacobian Projection coordinate or Affine Coordinate + * \return int 1 - is true | 0 - is false + */ +IPP_OWN_DECL(int, gesm2_is_on_curve, (const PSM2_POINT_IFMA* p, const int use_jproj_coords)) + +/** + * \brief + * + * compute double point SM2 (Enhanced Montgomery Algorithm) + * + * \param[out] r point + * \param[in] p value point (in radix 2^52) + */ +IPP_OWN_DECL(void, gesm2_dbl, (PSM2_POINT_IFMA * r, const PSM2_POINT_IFMA* p)) + +/** + * \brief + * + * compute add point SM2 (Enhanced Montgomery Algorithm) + * + * \param[out] r point + * \param[in] p first point (in radix 2^52) + * \param[in] q second point (in radix 2^52) + */ +IPP_OWN_DECL(void, gesm2_add, (PSM2_POINT_IFMA * r, const PSM2_POINT_IFMA* p, const PSM2_POINT_IFMA* q)) + +/** + * \brief + * + * compute add point affine SM2 (Enhanced Montgomery Algorithm) + * + * \param[out] r point + * \param[in] p first point (in radix 2^52) + * \param[in] q second affine point (in radix 2^52) + */ +IPP_OWN_DECL(void, gesm2_add_affine, (PSM2_POINT_IFMA * r, const PSM2_POINT_IFMA* p, const PSM2_AFFINE_POINT_IFMA* q)) + +/** + * \brief + * + * Extracts affine point from the precomputed table. + * + * \param[out] pAffinePoint array of x and y coordinates of affine point in 2^64 radix + * \param[in] pTable pointer to a precomputed table + * \param[in] index index of desired point in the table + */ +IPP_OWN_DECL(void, gesm2_select_ap_w7_ifma, (BNU_CHUNK_T * pAffinePoint, const BNU_CHUNK_T* pTable, int index)) + +#include "sm2/ifma_arith_method_sm2.h" +#include "gsmodstuff.h" +#include "pcpgfpstuff.h" +#include "pcpgfpecstuff.h" + +__INLINE void recode_point_to_mont52(PSM2_POINT_IFMA* pR, + const BNU_CHUNK_T* pP, + BNU_CHUNK_T* pPool, + ifmaArithMethod* method, + gsModEngine* pME) { + ifma_import to_radix52 = method->import_to52; + ifma_encode p_to_mont = method->encode; + + const int elemLen = GFP_FELEN(pME); + + BNU_CHUNK_T* pX = pPool; + BNU_CHUNK_T* pY = pPool + elemLen; + BNU_CHUNK_T* pZ = pPool + 2 * elemLen; + + GFP_METHOD(pME)->decode(pX, pP, pME); + GFP_METHOD(pME)->decode(pY, pP + elemLen, pME); + GFP_METHOD(pME)->decode(pZ, pP + 2 * elemLen, pME); + + pR->x = to_radix52((Ipp64u*)pX); + pR->y = to_radix52((Ipp64u*)pY); + pR->z = to_radix52((Ipp64u*)pZ); + + pR->x = p_to_mont(pR->x); + pR->y = p_to_mont(pR->y); + pR->z = p_to_mont(pR->z); +} + +__INLINE void recode_point_to_mont64(IppsGFpECPoint* pR, + PSM2_POINT_IFMA* pP, + BNU_CHUNK_T* pPool, + ifmaArithMethod* method, + gsModEngine* pME) { + ifma_export to_radix64 = method->export_to64; + ifma_decode p_from_mont = method->decode; + + const int elemLen = GFP_PELEN(pME); + BNU_CHUNK_T* pX = pPool; + BNU_CHUNK_T* pY = pPool + elemLen; + BNU_CHUNK_T* pZ = pPool + 2 * elemLen; + + pP->x = p_from_mont(pP->x); + pP->y = p_from_mont(pP->y); + pP->z = p_from_mont(pP->z); + + to_radix64((Ipp64u*)pX, pP->x); + to_radix64((Ipp64u*)pY, pP->y); + to_radix64((Ipp64u*)pZ, pP->z); + + GFP_METHOD(pME)->encode(ECP_POINT_X(pR), pX, pME); + GFP_METHOD(pME)->encode(ECP_POINT_Y(pR), pY, pME); + GFP_METHOD(pME)->encode(ECP_POINT_Z(pR), pZ, pME); +} + +#endif // (_IPP32E >= _IPP32E_K1) + +#endif // _IFMA_ECPOINT_SM2_H_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ecprecomp7_sm2.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ecprecomp7_sm2.h new file mode 100644 index 0000000..e0cb44e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_ecprecomp7_sm2.h @@ -0,0 +1,2536 @@ +/******************************************************************************* + * Copyright (C) 2021 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ +#if !defined(_IFMA_ECPRECOMP7_SM2_H_) +#define _IFMA_ECPRECOMP7_SM2_H_ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "pcpgfpecstuff.h" + +#include "sm2/ifma_defs_sm2.h" +#include "sm2/ifma_ecpoint_sm2.h" + +typedef struct SINGLE_PSM2_AFFINE_POINT_IFMA { + __ALIGN64 Ipp64u x[PSM2_LEN52]; + __ALIGN64 Ipp64u y[PSM2_LEN52]; +} SINGLE_PSM2_AFFINE_POINT_IFMA; + +#define BASE_POINT_WIN_SIZE (7) +#define BASE_POINT_N_ENTRY (1 << ((BASE_POINT_WIN_SIZE)-1)) + +extern const __ALIGN64 SINGLE_PSM2_AFFINE_POINT_IFMA ifma_ec_sm2_bp_precomp[37][BASE_POINT_N_ENTRY]; + +#if !defined(_DISABLE_ECP_SM2_HARDCODED_BP_TBL_) + +const __ALIGN64 SINGLE_PSM2_AFFINE_POINT_IFMA ifma_ec_sm2_bp_precomp[][BASE_POINT_N_ENTRY] = { +{ +/* digit=0 [{1,2,3,..,}]*([2^0]*G) */ + {{0x0001167a5f72d7b5,0x000c43181404d9e9,0x000e6dbd3aaf42f3,0x00024c3c33e7981e,0x0000654979a299ac}, {0x0003cd65d4e5a49b,0x0008384568872dd6,0x0008f0fffbcf430c,0x0002a48f8c1f5e57,0x00009272f196066e}}, + {{0x000d7e9c18d8240f,0x000aaa6e417d56a0,0x000a3b0e4550d595,0x00088cd2483bdc9b,0x0000832b6de84b57}, {0x00047e746600a032,0x000d7b7693b68c19,0x000859275c90d69f,0x0007157acccbd8d3,0x00005bc85177169b}}, + {{0x000019fd6c2ea279,0x000d0917cb6254a3,0x000a0b6907791b62,0x000eb4a0a2676528,0x0000d038c8d3308c}, {0x0008ad3478846130,0x000f770c6ce91902,0x000bb07902d865c4,0x000c13c97f40aa52,0x00002d50aa9bfa6d}}, + {{0x0000dc8e3b19766a,0x000581f7eeae6605,0x000039ddf04401d9,0x0009f9020487ea28,0x00001ca3fd26b4a0}, {0x000ffc31c595b91b,0x000797e67a87e986,0x000aec6fb58321c6,0x000cf70ed1de135e,0x0000a8e2c4f05632}}, + {{0x000f443ef3733cb4,0x000cb3ff75fcd620,0x000231240b46fe4f,0x000f5f4c515aa58f,0x0000acee1008bf1e}, {0x000b4152768f6bb9,0x0003d1943d948337,0x000e5ea1cf8e42ac,0x0002258efb30f4bb,0x0000df40ee234c36}}, + {{0x0005e74ca112a1cc,0x0001baaf82ba6ba8,0x0005e46613dca025,0x000e34db6b1274a2,0x0000e903fc536e5d}, {0x000568bc9800f516,0x000a645b374d537b,0x00007f00118f53d1,0x00030e9848fbf6d2,0x0000221155fdf8ec}}, + {{0x000bf492e195f07f,0x0001de5ebffe884b,0x0003c307b8018431,0x000fe18c50b89419,0x00000a1adeda69ac}, {0x000ba05a8f672532,0x0000d327ed0e66d2,0x000ea11a5c476cbf,0x000d99baf0e449a2,0x000006fdeeeb922b}}, + {{0x00015c2fc0eb2070,0x0007a5b1df1b6701,0x000a3428b0cdb816,0x00087df9e90be6f2,0x0000f4f38ed34143}, {0x00077fd5f39cf758,0x000c519eaa1d82ae,0x0001c9fb549748ab,0x0008f3382693b350,0x00003b68d86b6f9a}}, + {{0x0007d88740be6daa,0x000b882728e34867,0x000622fa36cfdf7c,0x0008669b48e5e049,0x000008f7936983b4}, {0x00022a6c24b493eb,0x0000a36e3e719ce8,0x0008bd29b04cc229,0x000a6e96963c563a,0x0000711254fc6553}}, + {{0x000e8237d906e272,0x00063b15d4a3f432,0x00085b93464a3fd4,0x00088728eecdb60d,0x000063eef3231b13}, {0x00031bbfda8cd29f,0x0004821c3dc4be1e,0x000a4a6263ce10f0,0x000dd716508c65d1,0x000015073445b5d7}}, + {{0x00083da169291613,0x0009c670b0bce154,0x00080eed9d6a0bfd,0x0005092677557c8c,0x00003202222cfd8a}, {0x00028398a7eae7b7,0x000180bd61f72bf5,0x0001683a9ec60b3b,0x0002c587d5dd8402,0x00006a86e659db9f}}, + {{0x00090ea13d89cc69,0x000255069e37462b,0x00018fe68620e964,0x0008d4f0789f3850,0x000031ef3cf56a9d}, {0x0006bd39ee352fe8,0x0000667570e4332b,0x000673b80d8ed70b,0x000385ffafa48d5c,0x0000a591652ce762}}, + {{0x000c80c9d04d2808,0x0005a9a32d8f64e3,0x000dc6453311458c,0x000bc2b2e7da2bdf,0x00002cb3a3b11de1}, {0x0005fa508acb12e2,0x0007d33fce2e44d4,0x0004849da0dcd4c6,0x000ec55f9aef09eb,0x0000eae191c31b26}}, + {{0x000f8babeb190812,0x0000a4e094c1dc30,0x000ced3824c7aa66,0x00074d0533f194a6,0x000002c062f92d6f}, {0x000f6cd2b110e9a8,0x00091ab321a1a47f,0x000bbc8af71c2e8f,0x0009b55adfe0c027,0x0000ebfab3c299b6}}, + {{0x000df89411540793,0x000923289e1ea4d0,0x000ac9ab62313957,0x000ddf2219c14e38,0x000002798dfc11cf}, {0x000c5c1e9ea4d4d6,0x000aa6fced9ad262,0x0008fa2e93406f8c,0x000885947d9be66a,0x000020846630e236}}, + {{0x000d01076ee1cd74,0x000cd5a581b2d20c,0x00093f74bd090be1,0x000f4639d7121d6b,0x0000c74662a9ce8e}, {0x000a4759c17b4394,0x000acd91ba3bb8a6,0x0000bf7a5d3f3801,0x0003c5c62b07e323,0x000082838a70c166}}, + {{0x000e33014346c3a9,0x000095088fa55ffb,0x000107c9d987c4a3,0x000f2b6584a31dd2,0x0000ff8f6b14a019}, {0x000da3bda2d72e1e,0x000953e595c8241b,0x00053ede9ccb4205,0x000cdc439cbf5da2,0x000018a52a7858cd}}, + {{0x000617c54f3a57d1,0x0009e6cda402d38f,0x000dcba397c6a7df,0x00017bfe48ed8eea,0x0000a909b54de6b4}, {0x000b7154885c6b51,0x00052ad21227839e,0x00060f8cff874f5d,0x0007e8e972e6a3a7,0x0000255f5e75e5e0}}, + {{0x0002d8e3f61e46a0,0x000ef8a682a8116e,0x00086b27e23bfa6a,0x0009e513f9b9e970,0x0000e27f60beec8c}, {0x0003246dbc6eb4e8,0x000255fe0d47ce4e,0x0003d805a7b4087e,0x0009af4f3ee12d06,0x0000dffd18b1c09a}}, + {{0x0008e3e09541188e,0x0006b0d2ff10201a,0x000abedec872090a,0x00069c20e3d876d4,0x000087ecb8de0f7b}, {0x000da6987805758b,0x00077a3929596b9a,0x00032632c2a7c1aa,0x0007d153d274c1de,0x000090621fde36d4}}, + {{0x000f01e019ea2472,0x000266cb630aa808,0x0000f49041d7deaa,0x0009350681611dea,0x00003d0a0dce821b}, {0x000abe5cc3f2bfb4,0x000d5418bd40f896,0x000e02ff3a27ab02,0x000e86399ce4ac99,0x000049d0896d001d}}, + {{0x000dbc0b894bb4f4,0x0002e20681014988,0x0001f9c1e48b7e8e,0x0006bcc8cbd44336,0x0000923926eb0c2c}, {0x000005bcfa05f5a4,0x000429e49d6299a7,0x000c78170090ecdb,0x0005f7927afb6fed,0x00000a0af6725bb3}}, + {{0x000f6ae59570ee69,0x000423cbdcc1639d,0x00074bb372d8a3e0,0x0006915c4c7c42e7,0x0000354c77faf967}, {0x0001bb84d9312a64,0x000a22b967527d11,0x0000bbd1e7d36ed8,0x0006f2599638cb07,0x000082f920f072ba}}, + {{0x000668304566ea9f,0x000c5a73b2b5ea29,0x0001b72e0dfb6f53,0x00068c65d19de7a9,0x0000b028d8339d06}, {0x000f529385702e1f,0x000b15aaaac89ff3,0x000446feccaf4f3e,0x000f06e44bc53bc2,0x00009f9c624cb5d9}}, + {{0x000e53aa7d49eafb,0x000cdac9d5e2826d,0x000fe4b7a2a6312c,0x0008cfdc13addc3d,0x0000b1d9952c6972}, {0x00071e68556c226a,0x000774683bb5d64a,0x000bb10c1848b985,0x0003d48c9d605474,0x000068340188b965}}, + {{0x00092bed74888d36,0x000cbce275c0fb58,0x000e72fe273b12ce,0x0002e80de91707e7,0x0000db762f73c5fb}, {0x0003605d1822de60,0x000264f60af612d9,0x000d43631ab3bd6b,0x0004629df73fd298,0x0000cfe82714262c}}, + {{0x000746a77aa308aa,0x0001a9c30737acc0,0x0002eb6e3ed4ce48,0x000f943a95bb54ee,0x00006010a8bdd232}, {0x000560e7e5cf332c,0x000be9efdeac35b0,0x000f8b7e87813782,0x0002181fb9421fe7,0x00000340a99e9269}}, + {{0x0007671e24561bb4,0x00000d8f762de2a3,0x00001886c3580ffe,0x0007956cd67f42a4,0x00002cf5abf25b97}, {0x000de5ed1dcc00e4,0x000478255fa3c47b,0x000bf3c02988a494,0x0007bba59141f96f,0x000072ac7d2040d2}}, + {{0x0003afcc938edfa0,0x000981cc61252576,0x000913c163700794,0x00002d5bdecb033c,0x0000e4e3c5ff4d82}, {0x0001d8f4668b50d7,0x0007cd2a51338b78,0x000edd0f0b7e280a,0x000f0bbe8dc91387,0x0000219bfbb5cbf8}}, + {{0x0007c9d827edc576,0x000d568cf9f5796f,0x0003c6b070fad2a5,0x000143c45bc8ac15,0x0000c8cc3708f7a1}, {0x000c780d9d193ed6,0x0007191b007b950e,0x000ab8ba8c708175,0x000ad5467340f445,0x000052592b31e902}}, + {{0x000802fb3eb3d841,0x000e04d04b1868a7,0x000982cd2e03b6ec,0x0001b9a36069a7ab,0x00006f98bd625e59}, {0x0004e2dfb27220ca,0x0007d60b11ba4ebf,0x000e42ff9a6e1785,0x000e36f3b18c612a,0x0000d89ac35c73c0}}, + {{0x00050e3e71fd4a23,0x000f7f7b071198b5,0x000ba19d01ea4efc,0x000f0a982d78af40,0x00007cfcfdcb0d03}, {0x0001c52fadb094b5,0x000950490fa2f5bb,0x0003b956e911832b,0x00047e33b5297cf4,0x0000601800188ccc}}, + {{0x00091c76dcf25dc1,0x00000084b5c6b0ec,0x0008cb89c5ee1953,0x000f206fba6c6844,0x00001903b72eb3c3}, {0x0009f9329b4a17d7,0x0005e366f36057c5,0x0007b0a7b8ab0858,0x000734400dfb20ce,0x0000db8842f3d9d8}}, + {{0x0003c17aa0e9a859,0x000f26b39a949a9e,0x000ca30e12c38332,0x000068df9575c1e1,0x00009178ba1bca31}, {0x0001ea1386258de0,0x000e8ea2a8ea6306,0x00082a7454b4a9cb,0x0000492d7ad24091,0x00003d94af9fb739}}, + {{0x000bd0a4c60f2724,0x000c9bd244c18bc3,0x000063be261d339f,0x00071ad908da622a,0x0000c90f49d55404}, {0x0005922faeaab30b,0x00008552e7fd4231,0x00008fb1810871af,0x000375b0f859ed94,0x000046581a19632a}}, + {{0x0002eab5669ebb2b,0x0004708bb2fef9b6,0x000fe7fc9aa83744,0x00090c3213ee8cc1,0x00003abf92585915}, {0x0009761c3595cedf,0x0006a92651244ab0,0x000f2886cba43860,0x000a8ac5d83b89ab,0x0000e8dc6841ec1d}}, + {{0x0009fed874184a69,0x000fdea655ea6c93,0x000d96c14ff60a53,0x000268939458e5ff,0x000084dc3eef11b9}, {0x00041b5b6de8e246,0x0007a0b14c452ccd,0x00027d8f16bd2bb6,0x0009b725bb028dc2,0x0000572ecdca9ee2}}, + {{0x000130b4de2895f9,0x0005f439cc75e202,0x0002c214bff765f8,0x0000651a5b628ccb,0x0000c56a9a48fe00}, {0x0004695e9be17d1b,0x00041f854faac0fd,0x0009ccfdbc214556,0x00069ae44fdcdb03,0x00007edcab584c41}}, + {{0x000e3590f168d2b1,0x000339e9acab36da,0x000f5f613429ffb6,0x00023e7d6983b7a2,0x00005c48a5be3e59}, {0x000d83804d41f76e,0x000c0e774de782a4,0x0004cbc3ab5f71cc,0x000ae8e01b15b054,0x00003bee4f4f1362}}, + {{0x000b014c73ee1325,0x00054d9da08a3283,0x00083663a292e2bc,0x000c2fa93b16ed9d,0x0000813ab0c77049}, {0x000434a3b15969fc,0x000b5b49bda3732d,0x0005c2f25b7357d1,0x000ce9480b6457d5,0x000006200abb3a4c}}, + {{0x00045b268a2411df,0x0005891671a913cc,0x0003cf1240da7743,0x000b948d5605bba5,0x00004b4552738e3d}, {0x000ea61903ca09a3,0x000f0db2771a4bd7,0x000aad4157e88d82,0x00061433957627a5,0x00004ab7fd350a66}}, + {{0x000e4964e7611bb9,0x000295adae74d432,0x000d8cac7948a4d6,0x000c3da9c7112bb3,0x0000028720a3c72b}, {0x000903eadc6af862,0x000266a7dbde62a9,0x000bbdd6677a6604,0x0007c5f888f772fc,0x0000717d5496d7ca}}, + {{0x000a1261ea8c6488,0x000ca2d55afe929b,0x0003b29c6c19e3fc,0x000283f973ccef5c,0x0000a10539a87a90}, {0x000b7937899af0f6,0x00050bf36f6aa1c7,0x00060f93ea46df3f,0x000a0668b373f1cd,0x0000948b79142119}}, + {{0x00008f8d46c0616e,0x0005754a372e43dd,0x000bc0c92bf53994,0x000a2a74df8df2cb,0x0000da1b4aba6922}, {0x000b4c696835e3a0,0x000565c909e53769,0x000987ddf078fecb,0x00008f50f3a8e25c,0x0000ab1518d04d06}}, + {{0x000cc1c4cbf2eada,0x0000b07cccf2a3b4,0x00073637b5075beb,0x0002cbb01aff4f89,0x0000cbdd0a13bdae}, {0x000af408e2bf7bff,0x000a5c0ca6d44b87,0x000590425f3ca33c,0x0002bced68c4d27e,0x0000860daa597e7f}}, + {{0x0006fe7e8c09b138,0x00085228df4d5e7c,0x000d0b002f39288d,0x0009113f99774834,0x0000f38034f3008a}, {0x000e75ecfbf60757,0x000c0515ee009c8c,0x0001dffbf3bde851,0x0003cd7b9f7b0540,0x00001d4b51f20fec}}, + {{0x000f73502b13dfd6,0x0000f6bfe31b4bdb,0x0004c8f0f3d24ad0,0x000df99852ba6e75,0x00007531d7f6850f}, {0x00076e3627f19150,0x000acfbce9c4ce08,0x00036c4b44a54a01,0x00043b9b4580491e,0x000069d8a672d2ae}}, + {{0x00006262266ce0ea,0x0007ceb468278ecf,0x0007246abb7f770e,0x00070c62b11c5a9f,0x00004d940c70f297}, {0x000a1dd93504b336,0x000c58ae56addebf,0x0002c8778a299d26,0x000826f34db89297,0x0000216c1d191d4f}}, + {{0x000d757d105035e9,0x0000892525eeeea3,0x000b8207d3ab24fe,0x000dd86c38699063,0x000048ba86d3e9a8}, {0x000543f2cf2834f5,0x000c4f1160cf0568,0x000be874b4e8fd83,0x00084d925a64595e,0x0000d3990d091ae1}}, + {{0x000e9af8e18b884c,0x00003c1132459043,0x000b64b2b57c9247,0x000641003b9843a4,0x00006be6b120ee1b}, {0x0008b29a50338313,0x0007739485c6ac24,0x00020bd7e43e2261,0x000cd2f42c7c7494,0x000007104b3c5bab}}, + {{0x00075df6aa322652,0x0008438ee19c58e7,0x000f0e751ca64953,0x000ee2cc1ceff8f3,0x0000007cbe08fafc}, {0x0005d3fe7d60d02b,0x000f86971b9f6cc4,0x000af3ff81584de1,0x000e38e96e8b86ad,0x0000c8a5afe65086}}, + {{0x000c0c2ef468013e,0x000447a6a3e240d9,0x000ce19f2d3f4f80,0x000f5b1d82e8fa8a,0x0000a47b981fb33a}, {0x0006f55f89eb0100,0x000573f986d93e93,0x00003f8e75c98dfb,0x000a00d1be6eb587,0x00006025306d77dc}}, + {{0x0005004985706114,0x0003a99c2255a574,0x00039ede58b41f12,0x00057962df076847,0x00004155be8eedab}, {0x0000393d7b59e2eb,0x0004af37b669d889,0x0001361a3ce556fd,0x000088c50176ce1a,0x0000ab255f00efdc}}, + {{0x000f2508973a0049,0x000e1a9999674ce0,0x000b7ed788a7049c,0x00076b0df4215b78,0x00003fc339599f2f}, {0x0008a181cc31d786,0x00004b008af5a09e,0x0006e20a280096e6,0x000aefa786ef0ad9,0x0000708a7fb34b8e}}, + {{0x000ecd84e8002ca5,0x0006e4c51e07c6d3,0x000605339c03f827,0x000e29d372b2ef6b,0x00009294965e1f2c}, {0x0008e0d1f90738b6,0x000ea3c21f6272c6,0x0002e51519ef8f60,0x000ea3fdf6d43936,0x00004958f256a93e}}, + {{0x0002ae43cded4986,0x0008c0786370d1e5,0x000d72bfe828c06d,0x000de1ca387f03fa,0x0000fe77288924ef}, {0x0005c10049922062,0x0001e1b0f661a1bf,0x00078e2792579aa8,0x000f50251dd874f6,0x0000e1fb6309e4fc}}, + {{0x000ccd7fbba38366,0x0006a2f5d13d7662,0x0000c2887037a707,0x000822e131ce6676,0x0000ab184d3f7174}, {0x0000417ce21feaac,0x000ba6783f71e187,0x000c16456fd30111,0x0002534b61beb5bb,0x000057298cddd196}}, + {{0x000938b88b271119,0x000cdf7ba43a3002,0x0008b28d3c70b643,0x000ff1e445a0b19e,0x0000e8d8708505a0}, {0x000b509b7bcaa8da,0x0004889cc6d3bc35,0x000c1b8dbbb4f0d8,0x00076ce646be6d56,0x000064c9e731d2c4}}, + {{0x00068a9aaae04721,0x00049c19bd83c3bb,0x000c97a89201474f,0x0007e36f61e60b19,0x00008e006d3601e3}, {0x000264283d8cd6ac,0x000a26d694677ca3,0x00050811d256561a,0x000f93bb86344083,0x0000be34e422eda5}}, + {{0x000f8047fec93faa,0x00073131b791a8e2,0x00020dd5c4c96fd6,0x000b598413ee227a,0x000003f52b6a9a3c}, {0x0000a3aa35db586e,0x000e80870eb165fc,0x000407776015d23f,0x00014524d484ac18,0x00007d7bd4527603}}, + {{0x0007d6e92e359c6f,0x000b32a6697b084d,0x00085cc819e78cf5,0x000e64343264911a,0x0000b8665b43083e}, {0x000987117e217dc3,0x000f6ea5f2ff0fb6,0x000d0aba23bc49fd,0x000e1da50005ab33,0x00006631cedcfd42}}, + {{0x0006861e633bad54,0x0009090870f777a7,0x000efcf8e7ed6758,0x00048f60795ddd3e,0x000066e4b07a455f}, {0x00057bcc2dcfcebf,0x0009631b8b2b3172,0x000c4e0e7bda1bfd,0x00084c79157e95b6,0x0000dfc62d4f9b9e}}, + {{0x000229bde0a25f6d,0x000c029e3fcf95af,0x000222ea492c862b,0x000ec26fcddc3afe,0x00003df0533d4aef}, {0x00093afe4a6dc67d,0x000928563cb37395,0x000db3baa6ee4ad4,0x0006c1219f41b8f6,0x0000aed543421b6f}}, + {{0x0003d9b4b1683537,0x00027fca4ce53d4f,0x000429ff213ff55b,0x000978dcef44d3df,0x0000c87265aec8bc}, {0x0008adaee633517a,0x0000f9e83600b2c5,0x00046726aded6663,0x000c242380cd02ed,0x00003bc51cfc7e49}}, +}, +{ +/* digit=1 [{1,2,3,..,}]*([2^7]*G) */ + {{0x0003f6205b429ade,0x0008bf1c0a4e32ae,0x00022c7d1370668b,0x0006f3f11ba4f98a,0x00003ef4e616f3e6}, {0x000eda7e2acd29f6,0x000cb5cda51a38f4,0x0007e42471db7ebf,0x0003a61d62c9b445,0x000045ac20585b53}}, + {{0x000e1431a0e449d7,0x000b00a18c82a340,0x0009ce6b623dbfa8,0x000726ce7b22b908,0x0000257d316c6584}, {0x000316fa44c65236,0x000abdf21e2c9a6a,0x000fa681abf2e073,0x00024b68464198dd,0x0000b20d3821b191}}, + {{0x00047be60cbe754a,0x000f718c602db135,0x00077d7fb8185623,0x0007f9aeed67c8ab,0x0000af29dafc6215}, {0x000785ec118bf7e4,0x000eb16620c35dba,0x000199925c15e48d,0x000c1ccda92fa080,0x000033aa09cb45cd}}, + {{0x0008bb37bbc8eaff,0x000c954d84c1ba41,0x00006d707a0a6cc6,0x00064269baff481a,0x00008b85ca90fd0e}, {0x000470e9650bf952,0x0004614637082390,0x000f1b05fe17e127,0x000e19ae44e27a58,0x0000d28ce5f3ca66}}, + {{0x00030cd9e7bba66b,0x0002a47d0f6c9f6f,0x000f98fa70856f4b,0x0006d7ae2af062ff,0x0000407848c30d3a}, {0x000ad179dbe5e183,0x00081bb63f0280dd,0x000e0af2bc5b9ef5,0x000814831cfa3aa4,0x000057f28a37fd0a}}, + {{0x000045cccc33ac82,0x00041300a514028b,0x000e69aa5701e6de,0x0005a99b5c0bae0a,0x0000570e393aab01}, {0x0008b5223fabe3c9,0x00035b4958105bd7,0x00093e95309ea81e,0x00079468e41e71fc,0x00006d2120c20644}}, + {{0x0008216623c8051a,0x000974d15e695a2b,0x0000ed4105279fa7,0x000b0efdc15ffcce,0x00008407c2795c45}, {0x000e93eb01f7bc66,0x000aabb5b8017b63,0x00053f362f01a15a,0x000e5aaeaf3866d1,0x00003fe8d5b0e2a0}}, + {{0x0007e4a6bab88bb5,0x000ac749feb21e8b,0x00068a9ac9915331,0x0005411d793c48f8,0x00004c41659fc83e}, {0x0006563f1f63710c,0x000b29b8e532f2ea,0x000f494986fd6c33,0x00009638f46578c7,0x0000487d3b7f6df0}}, + {{0x00099ffa74b0a131,0x00003c57e9ff5e74,0x00041fcef932469d,0x000943242ecc22d1,0x00006e8a53cc4b10}, {0x00053e70b02ce3d3,0x0000f7bfea4c6526,0x000f62519e5d6a4f,0x000cebfd7945f306,0x00003029e98b8415}}, + {{0x000328a51d2191fb,0x000aa9835ebb3353,0x0006ddec4b72ed65,0x000c31d009d2f054,0x00005adafdcd98fa}, {0x00033a5443f17c91,0x000464a519cce4e8,0x00062cd2554ecc5b,0x00047ccb6657f4ba,0x0000ea376f2b8440}}, + {{0x0008bc857ba2453a,0x000354b27eeba2be,0x0003efa1af80516d,0x0009be57220a5103,0x0000fe282dac550a}, {0x000b6dd98b001e61,0x0001d5dc16564000,0x000e0a4e87ba6cd5,0x0004d1decf33a87f,0x0000a1f0885f20c6}}, + {{0x0005611ee5405eb8,0x0008bb358c3e290d,0x0008f2d2f416d96a,0x000e3a8b7a6840bc,0x0000f3c09d0c0771}, {0x0001b7ebe5a31b29,0x000072918a3d173b,0x000c77a4349cc89d,0x000dea0ec6082dba,0x0000e943c639dfbb}}, + {{0x0009c5a35b857733,0x000b43af66b31410,0x000618bc27427626,0x000dd611df52cb1b,0x0000dea20d731f39}, {0x000a53863aaa59ac,0x000287d564c29a15,0x000e07486f27f2f6,0x000cc58c64e300dd,0x00008bb7806200b4}}, + {{0x000e6cf633beb827,0x0007541ee6485223,0x000f064e09c8e331,0x000479c4a2008653,0x000005672dd401d9}, {0x0002ef49575fc255,0x000fb1756ee1946b,0x000faaa78a620586,0x00078458d74c45db,0x0000286229ac59c2}}, + {{0x000f183d4950bd4e,0x0006829e2dd608d3,0x00024b281ebe2bef,0x0005d5ac1a775b95,0x0000ffcbf6fa1b5d}, {0x000c6f7a38b05932,0x0009a4d54018e45e,0x000735a43f67590e,0x0006365280498d29,0x000083d04586749b}}, + {{0x000a36193b229fdd,0x000d6a1f04e7a9d7,0x000b81137c112f74,0x0001d75822a9a12a,0x000092df75029907}, {0x000e19344b0a3b18,0x0004947df3b965bf,0x0003ac1801a2dec0,0x00014686de82c871,0x000005c6bebc40f7}}, + {{0x0003e45d0a32fcee,0x0005a7fa4a4a3aeb,0x0004023d27d3a94b,0x000a1159700a2516,0x0000f05f61efb8e4}, {0x00094cf5c5dd948a,0x000c6eb880b4b16a,0x000f504d1ef5ec3c,0x000cd2d4d99cf64d,0x0000e533ca08d4f2}}, + {{0x00022bc0f01f06e1,0x0001254579a83be0,0x000ba95a583c43a3,0x00031de06bd83e20,0x00002d83f0a341da}, {0x000a009dec653ef8,0x000a9def081681f6,0x00021b6942705a2a,0x00043026da1504f9,0x0000091f0e90b112}}, + {{0x000572cecfe7bf86,0x000ee8278a5f0508,0x0006508e5c6e542b,0x000b6af13a5bbd8f,0x0000f885cd1a4781}, {0x0008a541d84528e9,0x000447f8ada787a5,0x0006f2151fe114b5,0x000878e13175c476,0x0000495badfdb204}}, + {{0x000325396578ba7e,0x000f1d5b140820f9,0x000f5534905a956f,0x000b7ba771977a8d,0x00007dbdca2d6fc5}, {0x00060a74da50d8cb,0x0006557137368856,0x000e0eaff086fcbc,0x00033a7ba6c24de9,0x0000d539d80b8c55}}, + {{0x000e7fac53e5832a,0x000d1e7a73f7e66c,0x0009be7bb912cbb5,0x00008b9b169f23f1,0x00005bf337922a53}, {0x0002b5e9ce1f9bdc,0x0008af5eaec7f01a,0x000e559ad53c6d06,0x000d4b81bd375c03,0x000065d670d2c170}}, + {{0x000cf320046dae34,0x0003dc84af3ffc3d,0x0009110ca683092d,0x000d3d8dc44ef5dd,0x0000146ddfa12279}, {0x00091848ad843b2e,0x000e677ff21f464c,0x0008d7eea9091dcf,0x00044e760768c865,0x00001e2755ed17fc}}, + {{0x0008a20903a4deda,0x00053f20c8da65f4,0x000dc571a79e0955,0x000122dd7b74cc0e,0x00001f3b1489237b}, {0x00074eec705b4e7f,0x000b345d9736c54a,0x0000f9d3c9235647,0x0009c2ee431a3c33,0x0000a634b54e5906}}, + {{0x0008d3493802aedd,0x0007f6143720a39a,0x00079ce8e3d24e2a,0x000b745b17663a59,0x0000ba7c742c42b4}, {0x00008e23e6376d36,0x000eb442027cbdff,0x0004192ba0107cd9,0x000f3d3711d81bbe,0x00001954cdd874dd}}, + {{0x0008536eb6e977e0,0x00031c30a317f04a,0x000150b5cecb8ce9,0x0006ae3ac0b43e66,0x0000c0d72af52624}, {0x0005a628eb6b6cae,0x0002cf4928e1e625,0x00001afda0667e5a,0x0007f4265f3de227,0x0000cf11f93b8c0d}}, + {{0x0001910c3b106232,0x000b54c4e6c94ee9,0x000dbb97ce7d0b7e,0x000f00a7d4aefda2,0x00000a5704423357}, {0x00005307a4979c43,0x00049a5bc7522d2f,0x00044591f664b2c8,0x0004e020b15ed1b9,0x0000304591f7dbd2}}, + {{0x0002e01cb8acccaf,0x000db23b53a5142b,0x00077554ae226c41,0x000098d0174d7152,0x000044df36341e6f}, {0x000ed3679f629911,0x0008282c8aa57cdc,0x00046fbfad702a88,0x000e5b3ac8eb84b0,0x000001d5d3e16826}}, + {{0x000a64b76a2ec18e,0x00078c3aead2a764,0x000aeda893a50823,0x000054515bd98ad9,0x0000ae9054cb0356}, {0x000b64e7f5d86a85,0x00000fd39ede8e65,0x000fd4dbcfc6de80,0x0002a0a6b04984a9,0x00003c88762946cd}}, + {{0x0002fee193a647ca,0x000b670c4033ee6b,0x0001643aa02f7b4a,0x00087114e66f3146,0x0000e357bcb2874d}, {0x0008627cf129e3fd,0x000a292529e30056,0x000cf7e52e3bc05f,0x0005f1b5d8d50c49,0x0000864bce4c6e0f}}, + {{0x000ad7991fa05182,0x0004decea29a2582,0x000bb694041f373a,0x000e6989d82ab62b,0x0000931b87e7d76c}, {0x0005776952079845,0x0006358770640105,0x00085f9043a52233,0x000f00fdcb296a03,0x000059005bf4862b}}, + {{0x000cc11e5684c1b7,0x000e3f8f8bc1e41e,0x0008eb1d9286e66e,0x0000297b1812feac,0x000066f0a15d3e34}, {0x00096054a3178eff,0x0000bc29acc99c03,0x00023c25e83fcc7b,0x000147c408a07b0d,0x0000ddb2c5c07623}}, + {{0x000976011b1bc461,0x00060f0107f30719,0x0007818994a2a18f,0x0000776333ff85ee,0x00003e669ead7d8f}, {0x00054bbcc73bbbba,0x0000ed0af38e5638,0x00005b56c91358c0,0x000b3df90dff2123,0x0000265625f4bbba}}, + {{0x000db00f4b4772bf,0x000145e2b12b3c1c,0x0003c88be095cc1c,0x00090b61b8a4923a,0x0000344af5ccf9cd}, {0x0009dcc691662dd8,0x000962a4096193de,0x000145eaab6e7b80,0x000a2e933dee45c3,0x0000dda44baf48fb}}, + {{0x000d5d161871cf5e,0x000bf96b711c1622,0x0001b7c2140db6c3,0x0004fb357c8e120a,0x000091f47ad1dff2}, {0x0001508663c0622f,0x00026c2d6643c86f,0x000fb96dbf23d874,0x000f766581d547ca,0x0000da7eebc0a4d0}}, + {{0x000c00bde07e9500,0x00040dcbd4ee582e,0x000af33a5075386f,0x0006f9598d9f545b,0x0000ff588a64a467}, {0x000a474e6ea6e85c,0x0001ea51b156b584,0x000fb3e7504d1e38,0x00007f044e32f572,0x0000d5ab75d376b9}}, + {{0x000b47dc0dc20f02,0x00003326e2aa2b72,0x000a2cd6bec5fb3e,0x0007365f285de9ba,0x00008bcea35cebfd}, {0x0002605ff74f7a52,0x0003aafb4968c25b,0x000ac9d2a2b84e45,0x00004ff873ba7a11,0x000063946120e765}}, + {{0x000ea745b0f8807a,0x000e9d8587aa16bb,0x0001307a4103df44,0x000b78e0498dc118,0x00004e9d7e40f8a1}, {0x0008dcff7bc5f923,0x00065ad9cdec8732,0x0008b45d8a3f4963,0x000ca219f7b76513,0x0000644e1c93afd0}}, + {{0x000d04a8b55db72c,0x000f435dabbc4a91,0x0008fcbaeb67078e,0x000bbdfc9b2f2124,0x000019e43abb0f65}, {0x0003be91f3aae668,0x000f8a9736725587,0x00016e8f3bc10c21,0x00040ec4a74ee5e2,0x0000c0d08c02ae5a}}, + {{0x0006ce7146590851,0x00020187d3713eba,0x000d38a163074e6b,0x000cdbf349943c34,0x000053c9b5c6d74f}, {0x000e1b66247a7101,0x00069c95293b8e92,0x000d7308e5ba5fc1,0x000136280b9ebeb8,0x00008f80fc44a55d}}, + {{0x000b576e500ea398,0x000b82245cf3e7b2,0x0008f9cbc2bec6cc,0x0001d58f77345037,0x0000cdd0554e6eea}, {0x000cc7ccaae38d62,0x000534c4be5c6035,0x0000fc56156e4977,0x0004f1cf337aea39,0x0000ee6f49181e94}}, + {{0x00079d789c195287,0x00027477408790a6,0x0000930aceb86101,0x0006cbe2723b0edd,0x0000392b0905f00d}, {0x00088b89f8858b86,0x0007f3db1a0ba159,0x000ffe03a89fc719,0x0000201ba5da50ba,0x00006a863f2cd455}}, + {{0x0001a4387e58a913,0x000648acceb75521,0x0007521152b339f5,0x000d2dd874036bbb,0x000041757e920823}, {0x000a54437646ae42,0x00019047e92f959e,0x0006f40a3de71463,0x00075397ce58940c,0x0000ebb7ab93546a}}, + {{0x000a1b010e4aab26,0x000e3c5617a973ad,0x00044532982b8675,0x000a969681bed3c3,0x00007d4f1ae7cdba}, {0x0009ae9fffe1232f,0x000e65305444e20d,0x0006fe785b41f639,0x0004120e08eac5d0,0x0000aaa419560e8f}}, + {{0x000758b9c8293920,0x0008c6cbfbe97afe,0x000e690a1af49bfb,0x0004203981fcd743,0x00000b943f7aa23d}, {0x000f5a347a711abb,0x000b1d90df6603b8,0x000e12a3bbabc3a5,0x0008154f4b9ee0a5,0x0000c0badc5131ea}}, + {{0x000d8fbf079db722,0x0003ab9c82633610,0x000e51bbae11705a,0x0004eff476c3420f,0x0000c6baf02761a6}, {0x0009a0b4c2b16f08,0x00045aaf34d60563,0x0007d8efd5c37a31,0x0002406427c1b59a,0x00008f17aca33851}}, + {{0x00076f86426c3280,0x00065a8f5c36655c,0x000a66bb836ec14c,0x000111b0275c4c04,0x00005124ac1a1510}, {0x0005aa9d9a40b2c9,0x0002362d7acf7364,0x00073b575ab0d288,0x00008cff853212bd,0x0000e58faa0c1f04}}, + {{0x0001293d9c195932,0x000eef2d6ea5365d,0x00041ddd78506af1,0x0002081d18e3a2ee,0x0000175ed7253823}, {0x000cab1223717464,0x00081eaf69bff654,0x000231c48aeca4f3,0x000a6a4d3a8f27aa,0x00008dde1551daa6}}, + {{0x000901ca13309f4b,0x000cc1b25d3b7455,0x000e3ab715f6fdc0,0x00089282b9a04c15,0x0000c045df01cfee}, {0x000d75e0c168f76d,0x000e0171344f2fd0,0x0005435774ec19e9,0x00056bece16d3efb,0x0000be9d895f073b}}, + {{0x000a4a40e4749395,0x0003734cc0308f44,0x000ce52d8c15a3ea,0x000e8df3075e4632,0x00004bbedaed5c68}, {0x00006feb5915c0d9,0x000949c11a211e9d,0x00019a13bc8bd809,0x000ff23cfa204cd8,0x0000b0ba02c5fdd1}}, + {{0x00054deaaba5d8eb,0x000a6f6362a3502a,0x0007aa9e44e1e1d7,0x000cb2b299a780e5,0x000070536c702e08}, {0x00097282cef3adbd,0x000014bed566650a,0x000e5203d864d70e,0x000ddd1dc2014b85,0x00008605d670e497}}, + {{0x0000ad950d9dd8e2,0x0001a65713c77ad9,0x0003175ee249e6a4,0x00060f1892e877fa,0x0000ea9facbe9436}, {0x0009dfd972df6c3e,0x000cbedb2b8203e3,0x00058343fdd04e5e,0x0003d97ebfc1bf8d,0x00003e108caa72ac}}, + {{0x000b277b4f2aa16d,0x0008230545a5016c,0x0002e2769b2435ab,0x0002779964fdc84e,0x00002e12bff6b35c}, {0x0006dbbcbe4dd215,0x0009cf4e0ff812c7,0x0000e963f1c305f5,0x0005e69a0be36a64,0x00007762740fc054}}, + {{0x000c36a9138a7f68,0x0008470ee63806de,0x000cb7255647a569,0x0007fd1b252fccff,0x00007c9171f47258}, {0x00097ff34fc8093d,0x0008de7129a90bdd,0x0000a742e5f26310,0x00051c078b3f740a,0x00003fd65a14fa19}}, + {{0x000341e2c79fcc59,0x000d996d461221d7,0x000e9d32b3c02d65,0x000bf9b355c0b280,0x0000b04f743f16ea}, {0x000c8edc4bc3438f,0x00096016b5362987,0x00010d7d0315dd2e,0x000a6516e8c2661c,0x00009b5fb13a3567}}, + {{0x000012645f461429,0x00093cfe4e77bfe0,0x0000678940510b04,0x000c3c83e76847ae,0x0000aff2f45006a2}, {0x000568a45e9f4ac7,0x00003ec5016c2bdf,0x0000224802e6c323,0x0008a3781ba7f616,0x0000b608cbc8aa43}}, + {{0x0006e23886af7a22,0x0004d87701710104,0x000dd4329762ec47,0x000b0c85a529a29f,0x0000ca8eb75290d6}, {0x000944765a1f3112,0x0004526223465ae1,0x0008b3e5061d9db9,0x000c05c301f5a96a,0x00003eea325d3356}}, + {{0x000ea3fbc35ccbaf,0x0003d0980f8881d4,0x000742058b853b8f,0x000a93c1cd768f5a,0x0000aed3d20d113d}, {0x000f0613da6d40d8,0x000bfc5a47958e97,0x00078f3cffc06a56,0x000231aedfc6fba6,0x0000fe6d947b0cd3}}, + {{0x000959b0ed60b577,0x000f5ec0253c6f66,0x0008e5689b3f8d47,0x00064ea48fbc6376,0x000021581d29868b}, {0x0009abba118e8524,0x000b62a3b710d54d,0x00007ed07686fcbc,0x0002ac100d616b1a,0x00008f7e4edf5463}}, + {{0x000afd83d0bbed45,0x0009a4a423d92e83,0x0007efe97713bc6d,0x0002864fd29134cd,0x0000f105bf080515}, {0x00099d5c6362bc3b,0x000301adc72d227d,0x0007d741e5cbbe71,0x0003e8ec90a2b500,0x00002cb9449eed3c}}, + {{0x0007431dacc6cd8f,0x000327d0afc47ee7,0x0004cf1e41610068,0x00016c7df373dad7,0x00000042e51a0bb4}, {0x0000030244d5b5c0,0x00046436c0b767e7,0x000a283bcb615adb,0x0001f3281f151893,0x00002a5797383854}}, + {{0x000aa5a92ffbb0f3,0x000ddf4779f321c1,0x000d9726ba60bd3b,0x000141269fac0c6a,0x00009d33c89d1a6a}, {0x0001d981b6db934f,0x00097c858d13b960,0x00095003d1097326,0x000b1547c517a25b,0x000006db28fb4238}}, + {{0x00014be21fa8ba83,0x000723ee01b8bba9,0x00053235fc70e3a3,0x000854ee045851bf,0x000076061a1e3082}, {0x0000ed8989d0ade3,0x000e0475957e0684,0x0000b493f0327c6c,0x000c0eec5255cb9c,0x0000fb2ef57cd978}}, + {{0x0006e16fd686583d,0x00057ec048801ddf,0x00069f1cc520e3ff,0x000612fe4ef3e986,0x000004ce0c5522b7}, {0x00081124788f63ea,0x000409a1a13cbddd,0x000c67aaf921c06f,0x000d218fe7051bf6,0x00008b2ed76302fe}}, + {{0x000dfc779262f1ac,0x000f6f4b5df8fd32,0x000451c354acf03a,0x000b9cf87bbf3a89,0x0000202b08c85e13}, {0x000e0f0ccae32d7b,0x000d462b52f30d47,0x000ccba62aab00b8,0x0004cba64588a6eb,0x000032bdcd539457}}, +}, +{ +/* digit=2 [{1,2,3,..,}]*([2^14]*G) */ + {{0x000da81d1a2da736,0x00040521de498b84,0x0001c2d52e2068c9,0x00034dcbf6d4ec10,0x0000ea5880b663f8}, {0x0006df66b322ab71,0x000871f17ecb7ffc,0x0000f54419e4a898,0x00008b81cf1e389b,0x00007a05a6db6c73}}, + {{0x000d667d4f8e5a50,0x0005f26536b3c801,0x0004a16346a9cdbf,0x000ccbcf92ba9de0,0x0000fd5b3900eedf}, {0x00033c3069b93deb,0x000cf3b7ce8ad20c,0x000bd324a9a9bcab,0x000e5cfb2329a308,0x000095f2c48470ed}}, + {{0x0004bce015ccff87,0x000eb34d4a8f11bc,0x000721bc1cd438c9,0x0000827146cb5b70,0x0000a843732fe483}, {0x0001dd088c05eae6,0x0007da74df7bf31b,0x00093d301e85027f,0x0005ab37a147b28f,0x0000a2d230e7336c}}, + {{0x0007c5babcf7ea0d,0x000569819d685897,0x00078e0634866c49,0x0003879a283042ba,0x0000ab406b710c6e}, {0x000d69f7c9fd8080,0x000e1dc73087360a,0x000d64b5dfb2ca8d,0x00033828902da352,0x00009013008cdbe0}}, + {{0x000c1bee4cf1095b,0x000bd1666a4b713a,0x000b431e3d19fcc0,0x000938196d536753,0x00009b8e71621624}, {0x000a316451576162,0x000c76ddc87f3039,0x000e232d808e2b09,0x0006544735ee9f1f,0x000087a1d181f421}}, + {{0x000997691124b333,0x000d451eb882ca26,0x0000f886014ca42e,0x000749cbae97f712,0x000066796b2b5be8}, {0x00085f944d025c08,0x0007702f28a19d3f,0x000e61bdac26a8c0,0x000db2ea8aad567e,0x00008078a3f854ce}}, + {{0x00007ba56ac78d16,0x0009448f1d1d7ddb,0x000b3a1404a747c0,0x000254738fd1d565,0x00000fc34247734c}, {0x000dcc5f098ff4c6,0x000ef2299588fd1a,0x000526f2f3ca3291,0x000225dc6ae0385c,0x0000d29a013f2711}}, + {{0x000264880c2de156,0x000947380fee3995,0x0003cc64079cf1c7,0x0008da6ea4de174d,0x000003341f9f4354}, {0x000e8151f6e4ad1b,0x000bc54650c77ccc,0x000de287aa666044,0x000db0e21e16b6f4,0x0000efb7d194ac4c}}, + {{0x0007348efe215c9e,0x000a931c79ae7459,0x000bc87a3f8399b5,0x000a1bcc9091643d,0x0000c57a72438223}, {0x000cbcc9f0323ea7,0x0003f4633ae285d5,0x00075e694b36151f,0x000d0d5c1d1a3493,0x0000014dccf8288a}}, + {{0x00030900d2988954,0x0003e2d547c4b1d6,0x0003d769a258c9fd,0x00064ff7bf5754d0,0x00000bcff2e3a601}, {0x000995fcc5b48583,0x0002cf93957e8bcd,0x000d37ccadf3bf57,0x00031fc7dc53bffe,0x000071bb6556f9f8}}, + {{0x000f0306abec554b,0x000c8bc468067e3f,0x000d6238fc116a66,0x00069b8bd7796b94,0x0000a6558eced667}, {0x000af21c707b4feb,0x000a2aa0e68fd01b,0x0006f4d94f62eeee,0x0000ccb1d1af09d8,0x00003f1221391f32}}, + {{0x000eb563bf3376c9,0x0008f5d8ea00637f,0x000675ed8c7dbfa9,0x00051a85bac437f4,0x000037bf89c8189d}, {0x000ba46a66de87e9,0x000d24fe7e880662,0x000a387ef19a5076,0x0004953dbcdc6eb5,0x0000e2e1c4016e3e}}, + {{0x000f9b9bb42c0485,0x000ff2e8def6906c,0x000dacfee25d0f89,0x0001242fe1bc12ce,0x0000ba97a0a64232}, {0x0001c99c56cf201c,0x000d455c2ed2dbcd,0x000509787b117d3b,0x00095af8690b3a9b,0x0000417bd19de076}}, + {{0x000424aaeb3b91d4,0x000a998c13da2eb4,0x000c3d55be8eaf2e,0x000e03205c693bdb,0x0000fd5cf1c45d66}, {0x000d0f0a7ea7323d,0x0005a0440c0cb913,0x000e433cbef81c62,0x000ea78c3ad814d5,0x000070dbb0fed04a}}, + {{0x00090203a46ae630,0x0001c021ff3b323e,0x0008f647e46c84d1,0x00057306c16c8124,0x0000a93f9c553d27}, {0x0007068a29d80ae0,0x00066d1e92ca2f02,0x000d3c68f1fe6df3,0x000db0092823efe5,0x000043f0b8b11afd}}, + {{0x000ab7a126c29381,0x000d4f017773e3eb,0x00051c6ad9458693,0x0004188703c92ac9,0x00008d7d6ab30ed3}, {0x000fcc5e3f382814,0x0000f7ef5c71c3f7,0x0007acea6adea02a,0x000f418dc0bf1f23,0x0000b54ae9a11e38}}, + {{0x0002f3a2514a57f2,0x000755bd0ab98c69,0x000170e5611bc333,0x0001ad1373538876,0x0000a7a18cb1378a}, {0x0001b5efeee3ae1e,0x00054b58ddc3cceb,0x00036b1de1bda0d1,0x00035fe338e63f0a,0x000007514aeac9d8}}, + {{0x00012e9031f0a5ba,0x000cc72344d0cb10,0x000f9f81fac388f1,0x000819ec8de87114,0x000074f23945d20f}, {0x0003e7f085b71746,0x00040c6c0934c56a,0x00090588b97228a8,0x000b5a380052b408,0x0000fac86ae9306a}}, + {{0x000c5184d549deec,0x000d52b46bc9e417,0x000192f5462b2c21,0x000c8f8e355f7d23,0x0000cd177410648a}, {0x000d55e33cdefe21,0x000c7275af172a79,0x000eebe63c04f4ee,0x0007c713ff4aa0b7,0x00008d8307613e4b}}, + {{0x0004d3cd58e4d852,0x0006bab993f02b8b,0x00026693cfd3cf03,0x000af4e6d5408547,0x0000c2aabff8fa45}, {0x000a3c809ae43c50,0x000cf7a0f723e79e,0x00046edbbfbdccc1,0x000982a8cd0b6af1,0x00002b1e5a091517}}, + {{0x000fd07201deffac,0x00025809cadf872a,0x000cb98046995509,0x000ff21f4ff84d98,0x0000963d2bcbfe56}, {0x0002499093a6be69,0x000585326f80abc8,0x000f80d703b9d497,0x0001bfd4d58fc0e7,0x00004a60473ceb19}}, + {{0x000e5caaad4486b4,0x0005d1e24444b21e,0x000d3ed0c0b72819,0x000f982fc3cff018,0x00004ba520fa8898}, {0x000a25bbc397a8af,0x0002998611e20d76,0x000406e1ccb223d3,0x000b86f4e2652aa2,0x00009c27f8856e6a}}, + {{0x0009fc879466ceba,0x0001f18b2f8e875d,0x000210013a7160b9,0x000c7186305ed045,0x0000eb43ec41e200}, {0x0002cb32611aaefe,0x000c458e587ba1f1,0x0002c7fb38833b1b,0x0008800e363b5a76,0x00002467a594f1ce}}, + {{0x000404e6c71b8202,0x000e48bf41f295b4,0x0003bbd6407bf067,0x000844a0d45ccf07,0x0000dfc2ca06446e}, {0x0004dedc225205ce,0x00035847c15238fa,0x0001bcdcdbedad65,0x000bf3d178a82808,0x0000a8e3bc0a11ca}}, + {{0x0000c6ae5018d027,0x000a419015aee791,0x000bdfb1d84f73ff,0x000d4b9963b67b87,0x0000485df021db40}, {0x000ff3d64526fc0f,0x000809d05281ec5e,0x00034417b79ccca5,0x0008a68f1ced9e92,0x000084c11d3c4c8d}}, + {{0x000c1ee614663ebd,0x000eefc957140edb,0x0004cb8ebdd6ee14,0x000c319463886ccb,0x00003f1f77e17b13}, {0x0008c813d019b2ae,0x000ae3a496d29903,0x000b0b3353a94e2a,0x000af96c39382e3b,0x0000a718f3f170b3}}, + {{0x000c164496ab3d67,0x0006ea56a382dda6,0x000036dc6d5437ab,0x0008c51072a7b515,0x0000956b614be2c0}, {0x00073c805d2be046,0x000e2a99459dcad8,0x0005bf1a2e6a28d1,0x00042bc725dbc286,0x0000b972c00e8d03}}, + {{0x000aa44ce9e0496f,0x000a7518060ed141,0x000452f518ef97a7,0x000abccfd1dc35d3,0x000044605da47e0c}, {0x000f1fba3870f0ae,0x000ece7c99396b4b,0x000488733ee9559f,0x000e5da54cf3f91b,0x0000a28e5309a1c9}}, + {{0x0006d871fcfa1706,0x000e3b101284285a,0x0002d9406b0f302d,0x000121cb8a49debb,0x0000c65e2ed97a1e}, {0x0005b77a782ee91a,0x000a18fff5f1afba,0x0000590abfa71329,0x000f8f32227e83f1,0x000050dde9b93ce1}}, + {{0x0009607b794f0285,0x000f0c89ef49400c,0x00029e7554b7902b,0x000a03702d679b84,0x000027d77adc7fb0}, {0x000c9a2c835846f8,0x000d19300fc32289,0x000389f09dc87369,0x000449f520da060a,0x0000929d2a527a1a}}, + {{0x000d034270025e0c,0x0009dc6d5d8f5f46,0x0008b8bd0e581906,0x00031cd1e56e8bfe,0x0000ed5db268c8c6}, {0x000d7cfa60528665,0x0003499dd0753176,0x000a49d661bee9a9,0x00055f095fcec179,0x000078f295389204}}, + {{0x00010c7a09f07b72,0x00054a4f28f770b2,0x0000ebb86416fc49,0x000e678562cc3429,0x0000c146c66ed941}, {0x0006c78f44e0f9a9,0x000ed0e1730b01d2,0x000dd20b61e67d48,0x0004e94705fe8cc4,0x00004cb9e577978b}}, + {{0x0005b05d6914ffce,0x000bbcbc25e829c9,0x00064f5723a4258e,0x0009424439122bbc,0x000062a0f760a317}, {0x0008e2d3db0cdd63,0x0006e0a06d9aac8f,0x000ef51a43b76bd2,0x000e313b4e84259e,0x0000cd05a227ffe3}}, + {{0x0007a74f56626479,0x000ed3a968075def,0x0002063cf858014f,0x00060623e3b51455,0x0000d8a825e00c73}, {0x000803d1b3ff54bb,0x00057ca25942c21f,0x000b4932e5b4a102,0x0003a09d5ae52fa2,0x00001c461ac9803f}}, + {{0x000b28d11887efae,0x000ea168fa1b64e1,0x000b56a252fe844a,0x000dfae103e32b3d,0x000025b4f7dea5d3}, {0x000947e940ce1f1b,0x000ca48aa1767741,0x000a6ef9a8c24921,0x000739056310da94,0x0000c91e4f7cec0d}}, + {{0x000631d5545c61d7,0x000e9f1a5c65d6fa,0x00046c01c5cb2c3e,0x000b12160108765b,0x0000037e6b0239be}, {0x000f48f86beaf031,0x00003582003a33aa,0x0005101c310ecee8,0x000a3f604e6cbfb5,0x0000370d71300c7c}}, + {{0x0000b578f398f5c2,0x0002684aeddb0ff5,0x000a40bb38f9bc76,0x0008d0e81bf9c714,0x00005ffb3646b97a}, {0x0006781ae38091b4,0x0003bde34988e66e,0x00063bae09eddb66,0x0000f7f47a3c3027,0x0000045549bcc1b5}}, + {{0x000293017d41f805,0x00015b6e5b119f8a,0x0007fe0ad1dcef79,0x000dcb03d2f4f593,0x0000bc5de02b9ec2}, {0x000fe4bde4a50c54,0x00075fbe79e7001c,0x000f666fd4f9eba0,0x00036e2014942d8c,0x0000174ec1b3aeae}}, + {{0x000fca2c37292e34,0x000df0d50a803e6a,0x0009860576e3da8a,0x00016b82c8491f1a,0x0000f908c5eb3698}, {0x0007bce061481fee,0x0004ca0326e92580,0x0004ef03e64401e2,0x000e3e3b0b3335a2,0x00006a363fee2346}}, + {{0x0009631af51f3820,0x000310bcf577dfc0,0x0009a7fe9a4d5c89,0x0006e8e893b8e850,0x00004aa3176b18d3}, {0x000fe9e9998db107,0x00088de616ddcf17,0x0003cbbbcb14c557,0x00003faac8272dc8,0x0000f37f11c6d949}}, + {{0x000ba3b57ca7d9fb,0x000fd143f3a1d7db,0x000b58fe1aeb5b1f,0x0000b20f07391ec2,0x000053738f59c6e1}, {0x00071b4df4657720,0x00037c87c0c02e8f,0x00048a4dda850a8e,0x000c2bbecdb49b0f,0x00001e0e88fb7249}}, + {{0x00038cdac58335c3,0x000c66df86cb8aa1,0x000a5efe055d4c77,0x0008683ad1af4476,0x00001273e06eec8a}, {0x000a2ce980ef2516,0x000377fed539bad2,0x000293912c3a97cd,0x0005bf11b26fc0b7,0x00002701f2738b12}}, + {{0x0003ba8b2493b148,0x0007d81bf2ec5d3b,0x000ba5926cd37918,0x00003b04098e8707,0x0000993e2bf0b8df}, {0x000b9e789a25168f,0x00011110cc4fd0b4,0x000f73b82fcc0949,0x00025b8664697e9b,0x00008b52da34d004}}, + {{0x000c26b8aa6f79d5,0x000e067dcfb9770c,0x000edbc74258b6a8,0x0008a01b52d4c4ee,0x0000e4bcfdfaa83b}, {0x000fb6562627da7f,0x00009f556e31fea7,0x000812774e7e1710,0x00049a0c8d511a79,0x0000bbd260be01ec}}, + {{0x000722aab36249e1,0x0000a40db721fc3e,0x000026a00c5e1911,0x000d74549b846153,0x0000b622aadebdd0}, {0x000ca6686c5b5ea1,0x000292c7cbcce6e4,0x000fead6468bad0d,0x000bbc79632dbaa5,0x00008d702b3a959e}}, + {{0x000945ee8039f7f9,0x000e245d6a8c7bdc,0x00087d06e54cdf17,0x000783d6f1f6930d,0x0000551ed8a2334d}, {0x0008ab5b4cc0c954,0x000c9e84281ae650,0x0000185519e20a39,0x0007ef592eb5d96e,0x0000ec44157cc235}}, + {{0x000fa33716016f7e,0x000e46af2f5045d6,0x000a09af49a8fdb5,0x000e43c6a1cc8371,0x0000c277d0e603e7}, {0x00075a70fd1e4ccb,0x000139ebdaf3db97,0x000de03d23466af0,0x000a4a43866e1ef2,0x0000fdf225ba7f64}}, + {{0x000f14821c351edd,0x00021666bac77433,0x00028a88c1031a4f,0x00003eb20264c266,0x0000b50cc08ac4a3}, {0x000b467364f7c8ff,0x0009bc822547761a,0x000838d0cd7ef0bb,0x0004c66a947c25a3,0x0000d8d71347dba3}}, + {{0x000a2a39c57739c8,0x0003badbd20048ba,0x0002c2455087a0da,0x000600d3308eeb44,0x000035ab7e73bb70}, {0x00039893a9299ac7,0x000e1e5fae8a6704,0x00048b6bc37d45c8,0x0006000cec119be8,0x00002354d9fe710d}}, + {{0x00044539e0bf2fd6,0x0009b8e02b90f648,0x000d689709228660,0x0002c31506185413,0x000035c1a6a9b616}, {0x0002e4cfa6e7a7b9,0x000883f5b421e0bd,0x0006decae0701125,0x000ca3d924eb9b7e,0x00008973fbb6a139}}, + {{0x00084810bd80b472,0x000c9d87096bfb22,0x000213dd306a948b,0x000f10fc1ad42d95,0x00008adbd51369b8}, {0x0005716e1bc7e9cd,0x000e88f207a0a817,0x00081194cb775ec2,0x0000051502eda484,0x00001655fde650e6}}, + {{0x0002d8529ab17664,0x000febdd18efe4ef,0x0008f93d5bb9e7aa,0x0005bd5b6398956a,0x00003781ce087bdb}, {0x000adb3daf55c56c,0x00037ad269a5439a,0x000db01176c3f26d,0x0003e296872f7e7c,0x0000033a30fcd1df}}, + {{0x000bd6335284b84e,0x0008f43880ca626c,0x000925adb6bbaeb5,0x000a46b04a59bffa,0x0000f0ff8d1a11f0}, {0x000a185a6e7f82fb,0x000f677d6d0f0bf9,0x000fe783b2b1463c,0x0001df2e23eaa59a,0x0000740f8798259e}}, + {{0x000c722cfd41d7fc,0x00094d23f0bdd09a,0x000c4e6261ce5297,0x000c7c34230caad3,0x000087ee7f4ba1d9}, {0x000c5464bb31d104,0x000cc2b2971e1826,0x0006075a6c6d5093,0x00046ca459e99e4e,0x0000a65a1fb97994}}, + {{0x000c644d48d918fa,0x000a94d0bf8c23b4,0x0006cfb155d1a0ec,0x000071c4322d7bc1,0x00001f3bfe3fb2c5}, {0x000783f98fc491e3,0x000a7c0a39281925,0x0008235ca69e46fd,0x0008ae01fc9d169e,0x0000ca900548dc79}}, + {{0x000bda541349a39a,0x000c425a2a1aa23c,0x000962d9a09f1913,0x000462f7c27405d0,0x0000bde10d8a970c}, {0x000e683472ead4af,0x000cb9cfd08a2467,0x0009aeb8afaa4737,0x000ded0ac9c9a0d3,0x0000ce90da220b05}}, + {{0x0008f648f58227f8,0x0008836fa6d3184c,0x000e3ba47dcb6caa,0x000da665bec76370,0x0000b6a20603deac}, {0x000582c5e925265f,0x00046c9943a97cef,0x000568b96c2b5af9,0x000cc39e771b3bdd,0x0000841b94d6bf73}}, + {{0x000bf54149dde419,0x000a87dd1b4541dc,0x000e114d1eae42db,0x000618a1da2ae53e,0x0000d375cfe5fb7d}, {0x000c7151c5ec6dc1,0x0006021ec84fcd40,0x000c06e7f03abd0c,0x000e5abab6ac4a9f,0x000092cc4d88e993}}, + {{0x0004a400aa461c13,0x000fbb8169ed26cc,0x00035b80cc32eeed,0x000c0e53577fae15,0x0000c26d785c466c}, {0x00060512ceba4cd9,0x0004d5dbd6d77e63,0x0003d73e01830fe7,0x000f552a7971b6cb,0x0000ff6234aaf953}}, + {{0x000264d6ed5d6a7c,0x000e0ce67dac839d,0x0009760722497bcc,0x000fa7d6b7f1b9b1,0x0000412325c4dc51}, {0x0006aab96f5c6a1f,0x00010838ccd0083c,0x000477f4aed2a7fc,0x0003b9babdf5da39,0x0000b6954a4db685}}, + {{0x000b9720f5c665de,0x0007b936adb439b1,0x000f941b1f83f35f,0x000b862417afe4c2,0x0000bf88e1af3c59}, {0x0003fc188202f659,0x0002281f3bbbe4c4,0x0002a4f2f7105f4f,0x000558633d4e7b95,0x000028d27173eaf4}}, + {{0x00076bd78c722e88,0x0001551700e67635,0x00094f20827e73e4,0x000aef9bcfe57d6e,0x000097aad46ac9e8}, {0x000fb4432357b1de,0x0002f28bc7ebf10c,0x000ebcc31c6cab55,0x00039f2999933d7e,0x0000f5102b0ca303}}, + {{0x000a7e1a8ba693a5,0x000ba49ec02c239a,0x00067e4df2c98264,0x0004249f27435ecc,0x0000c02baa7777ec}, {0x000e994e304f2110,0x00083fbacc530a15,0x000fefab6e1cec97,0x000590dcbf287767,0x0000795c0592733f}}, + {{0x000984b1ef514e65,0x000ebd81411e2260,0x00032dfef40dafe3,0x00094956e4f8c6ae,0x0000136ba04fa012}, {0x0009ccee49a09296,0x00064c3b8e8e0968,0x000892b26d304e0f,0x000a112d29d432b7,0x00003ed12545e12b}}, +}, +{ +/* digit=3 [{1,2,3,..,}]*([2^21]*G) */ + {{0x000ce669fe8edc64,0x000977cc52c3285c,0x00000d67ed46f754,0x00061b80f9f0ec69,0x000012dbd13ccd38}, {0x000807b39e2a78af,0x000206aed104adae,0x0004dbd4a5d72c92,0x000027f3b3295361,0x0000693ac17f9250}}, + {{0x000f594afa4a4a7e,0x0000f5b4b990ac34,0x0006399e2ee44b8b,0x000b41f7bd0670ff,0x0000001c5d479361}, {0x000a8e6617658623,0x00048daa039ef32b,0x0006f48d6ebc6748,0x00037741a7683353,0x000042075593bdb8}}, + {{0x000094eefdd302c8,0x0009c460befb138c,0x00020963d227f072,0x000d142da3f15ca9,0x0000ba9e641f3b9d}, {0x000e401530c93b7d,0x00059744a3037da9,0x0003796f4979883e,0x00016795e7569395,0x00000c613cf9d65c}}, + {{0x000b20d1bb21d958,0x000fae479a31475e,0x000d97dd75179a2e,0x000145b32680eacd,0x00006b0a592db55e}, {0x00009e2a418bd004,0x0006a13ebcb0ebbf,0x0002fc81c9f8657b,0x000a128ce6f290ff,0x0000c2bf7d73e0f4}}, + {{0x00034ee8f7ca2841,0x000957847db5272e,0x00051343550a99c6,0x00027dce2b87e02c,0x000008e06a87c6fb}, {0x0003e4982cf177a8,0x0006fef4d4a55e9c,0x000544e47582945f,0x00026aa21056b319,0x0000bff078406896}}, + {{0x00002033f3eacba9,0x0001ecdd9087e29c,0x0005322e2493e6b9,0x000f6eadf4905da2,0x000016f929c8a045}, {0x00059f8488967f6f,0x0007be7575f06e12,0x000d9b6612dab69a,0x000892e6af43919b,0x00009097672f7684}}, + {{0x00083aac2d553230,0x000e2ff837639a05,0x0009af89299438f4,0x0004cfd3b1df4866,0x0000926c091d5ae1}, {0x000a5356cab1f151,0x00008a0b5e8d9e9f,0x0001ca41a7e71ebf,0x000839a4f0b024e7,0x000062a46f632c13}}, + {{0x000b21b7651b535d,0x000ed1ed8043c9bb,0x000ce1e1641850f3,0x000ddb9452c21474,0x00004ffeff4a6ad3}, {0x000dfe0c1b878ca7,0x0005170036f8309a,0x00092f2e778c4742,0x0002eb6f51aac2f0,0x00008b0fb8da7185}}, + {{0x000858780433abbd,0x00033774b30fce56,0x00038782098ecd9b,0x00099b6cee29ec01,0x00008890c1e6e660}, {0x000f67858c8ce86a,0x000d24efffce7d38,0x000387b8a9346a50,0x000711dcd7ef98b7,0x000069a697b171f9}}, + {{0x0003d469b1729fe4,0x0003a650ac09ae3b,0x0007f3ea0e2a6394,0x00028ea3015db36d,0x0000ff7f5572ee9d}, {0x000e9e34a5b7a36a,0x000b713b7a4042cc,0x000b2865c4506a15,0x000d73147264da3a,0x0000d28e88eb3d3c}}, + {{0x000c180b52f9cd55,0x00035dbd16da5551,0x0005671d63efc307,0x000d083bc2c9bef2,0x000077cd3b33b7bc}, {0x00016701d2c8f290,0x000af20f7a38f325,0x000c4fa1e8432fcd,0x000902477a71103f,0x0000c13307631946}}, + {{0x000a6de9599a5692,0x000b8cbb1dad4d5d,0x000317ddb4c6dcc3,0x000820def3a0ba2c,0x0000523c6f46afb3}, {0x000507a5b0666ce8,0x00009a4e67dd5fa6,0x0001f27b937bc419,0x000e17182d353469,0x000013aba876630c}}, + {{0x000e231e63f86155,0x0001ac0dd78b9590,0x00056fcf6a929cf7,0x000aff49a7e599c7,0x0000f2beae09c341}, {0x000c54d7d94bca5d,0x000460a94c47dce4,0x0002a0654f786ff2,0x000ef6bb318918dd,0x000043199fd3902e}}, + {{0x000672c94de6b79d,0x00065b645d1cb3c6,0x0006de9aae65c98a,0x00020c93993596a1,0x0000084850324b2b}, {0x0007470e2ed88727,0x00078a44113efdd4,0x000588cca7c8e729,0x00036e449a848413,0x000086370e30c8fe}}, + {{0x000cabc373c129c0,0x000b795474aefcee,0x00074d1801c4d4d6,0x0000903a019fb1d5,0x000090a24a4b2a71}, {0x000bcf6d4c7ea10b,0x00045a7dff3768f0,0x000f699905cf4cac,0x0004d37be1302ac8,0x000059f9ed7615c4}}, + {{0x0004954a8c5f8491,0x00036d6e05110de4,0x0001ebf0a61838fb,0x0003c617a5d05e90,0x0000931b4242abcb}, {0x00055e87db36946f,0x000d5c8c8d5ff47c,0x0001d34b9a502d72,0x000abe95f6140f42,0x0000eef1dfbb044d}}, + {{0x000af702fdfc658e,0x0009b5edcfe4f6b2,0x0003916044a0e71b,0x0009253d633d18f7,0x000067827258be5d}, {0x0002270c1591a40c,0x000aac7e34119c0c,0x0005718100a44dc3,0x0007de482d9958a4,0x0000936a6e932f8e}}, + {{0x00002d829e0de935,0x000d2b09417b0748,0x000bfb976a3c1581,0x000b232260ee014c,0x0000912559287882}, {0x000a3a2e63eb8e6a,0x000cdebf51672313,0x00079a1f51566a56,0x000a7c04a1f1a9ec,0x000070a867c7854d}}, + {{0x0008953d6fbc948f,0x000f59036928cf85,0x00092b62a61d026f,0x000b727b9d2522d8,0x0000d4740c141ba6}, {0x000e7258d3488637,0x000fa4f6dd639d25,0x000c32176d124824,0x000ef5dc0aae73c5,0x0000a6763062fc85}}, + {{0x000816180ac7b35c,0x0005671668988232,0x000293310c1fe0fe,0x000d24b7b7e35a0a,0x0000fbc7f7dfa8ee}, {0x00002f829b61fff0,0x0004af7a42839a1d,0x000ff28e0716b31b,0x0006c53c4262a0be,0x0000eeadb1a1ce5d}}, + {{0x00015f1a3129e10f,0x000af2b4e6c937c8,0x0002977a7267764a,0x00073c6a478a6c0d,0x0000cb95cba1c483}, {0x0009a10260d9eee0,0x00063e1488dd8e2c,0x000bef43bc3079ea,0x00087b3e5515d1e2,0x000027a54dfc6c17}}, + {{0x000bb05b171e260f,0x0000312f11202e54,0x000cbd9782e9eb34,0x0007071408c05d3f,0x00007b0ad44d5ffa}, {0x0008e0fdb6bdc2ae,0x00083bcf267acc86,0x000abae442a4c99c,0x000059d8f7232875,0x0000d3f091a78c65}}, + {{0x00025e9ecbca3bd5,0x0005b277501df3f8,0x00002e53470ab011,0x000c2c5c9f5f0879,0x00001e324f25d94a}, {0x0002c87b5f5c096e,0x0003690c467cfb66,0x00083478fe443b48,0x000ffa57d95db742,0x000084fcea7f7f1a}}, + {{0x000be1e8d816e34a,0x0005ed0373a139a1,0x0005503eb6c76567,0x000d72c0a0c0fe38,0x00000930e2800fd6}, {0x000a839aa5e6043c,0x0001feb1c5709e13,0x00063fa936978e73,0x0001cc67e25bec26,0x0000b4bdbcf0e7e5}}, + {{0x00063de507ea770d,0x000de8db50e250d4,0x0003df144efaa8e2,0x000948957b939cd8,0x0000f00ebd59ede6}, {0x00058f76309396a2,0x00070b764deb6457,0x0001c9d75c4c6c1f,0x0009d4bd7c9c7109,0x000094ab2f959459}}, + {{0x0008e62912bb0c69,0x0007715b88eb025b,0x0008b374d95a85ef,0x0008d1ee3692017d,0x0000e00546935947}, {0x00067e18f669c1a6,0x00031e71bb112d7b,0x000f45f7f358f9cd,0x0002a153c6281ddf,0x00009ce24d9faa21}}, + {{0x000f23a149bca110,0x0006b07752dadb87,0x00012f665e907d38,0x0000550047f8ad59,0x0000a6b2737c88e3}, {0x000df2eae0d2d96f,0x000d847b47f5dc0c,0x0008afaaf8da34d4,0x0000c0a53b08c91b,0x0000e0c5963803ea}}, + {{0x000040a69d08ead8,0x0005d6ad8eece93f,0x000e87e8da444df5,0x000e5f2b193b1fa2,0x00000036b7fac7e9}, {0x0004a15b7ebad244,0x00033aabbc15474c,0x000fda83a8c245cb,0x000c2ef6f58e5947,0x0000ab5619ed45b0}}, + {{0x0006744b238ad74a,0x00032c3f045f7417,0x000a0cd8d1dfa1dd,0x00031dc9d44dabde,0x000084c351aa16ef}, {0x00066dbbbd4b79cd,0x0004a96d5540a264,0x000654e5bf972fe2,0x000124ed39cff7e8,0x000006fad5584afd}}, + {{0x0009b54ae4ccf190,0x0006b5139c106a36,0x0000e8f52cd3cb71,0x000bf08f80ab4008,0x000063ecb5195105}, {0x000e88b3133f9c79,0x000eeb2da83f3530,0x00079992670c5be7,0x000c619c9d672568,0x00009c94c9c60e0b}}, + {{0x0002d3810e75db98,0x0005272729ef9b4f,0x0007ff48b8893550,0x00078145eb9f842d,0x0000b7c621d7385a}, {0x000e48f3bca9cf86,0x000b8618ffd026ad,0x000e625e5d56e39f,0x000ab106e697de51,0x0000a9125a0d4e5c}}, + {{0x000447e9cc831545,0x00039cab5ded44d9,0x000acfe1552fee6f,0x0002053a733d2726,0x0000d8fb9153c30d}, {0x000e7b29e73a90d1,0x000b934edb6a2c09,0x00051fd35387a394,0x000def08b496427e,0x00005f85a2688694}}, + {{0x000424e8f9ba0b93,0x0007cc5433ef7479,0x0002d758a3e8cabd,0x0002c570b4f000a1,0x0000648310308664}, {0x00091a20f1e32de3,0x00041f7fce3ececd,0x0006f7a6a4a18ef2,0x0007a40a6f273f95,0x00006d7bc3eb35a4}}, + {{0x000d62c3c0d878cc,0x000bd2f14f772133,0x000506fc0d074a1e,0x00053b069efcbcf3,0x000054da173b382d}, {0x0006a1152b4b6ab6,0x00077d56291d93b5,0x000356e41211a144,0x000cf742c41cc0cc,0x0000d44d93647521}}, + {{0x000f6db69902561b,0x0009788fc670cdc9,0x00011bc01617c072,0x0001039ee89bce70,0x0000cfeb4e30a91c}, {0x0008f453c12e47f9,0x0008ce538b6874d1,0x00013f0f3dd733fa,0x000df596b652acc4,0x0000070c70ded8f9}}, + {{0x0002577943ed1947,0x000c51f97d56ec8b,0x000635b321c1dce6,0x000f4c0548ac452f,0x0000108844a039c5}, {0x000d188c05a8f588,0x000c907ce36d8dc7,0x000454c24921477f,0x000444f341fa52c4,0x0000a2d7f2d74a60}}, + {{0x0004981202326049,0x0009f23af30ee110,0x000d64a57e4877b7,0x0008d66d3bca95de,0x0000aa95f1b4aaaf}, {0x000957488a1fa688,0x000248ced945f9d8,0x00053e4500e34b51,0x0004ec08199ff786,0x0000dc704075eeaf}}, + {{0x000432d7fec902b7,0x000dcb98903b6701,0x000d7b52267dcb60,0x000413111b403207,0x00001357bed41bd7}, {0x0003d756c1125098,0x000a7a9535cc50ee,0x00098d003854c0f0,0x0004962fbbd5ae57,0x00004841bca8d5b6}}, + {{0x000e30c76057eb58,0x00068ca7bfadc9c6,0x000b3aae0116da0a,0x0004e7debde602db,0x0000de60723a0a04}, {0x000c372afe5f7c62,0x00098fdeab5ccde3,0x000b5f5be690997e,0x0002ac9676df4a3c,0x000028ff6485238f}}, + {{0x000d7abe5b2e9d52,0x0002ffa22191deca,0x0001b9811b5540a1,0x000d459ea5db7dd7,0x00007563c3351050}, {0x000a75fab5e73768,0x00027ca2af62afa4,0x00063a4216da6470,0x000fd71b1066c376,0x0000728e0ecfca8f}}, + {{0x000cadd177056db7,0x0007112b5eaff3d4,0x0003bc769207538f,0x0007fbd0b226cdcd,0x0000d68a971cb7ae}, {0x000d3db0039f89bc,0x0001dce3018f0114,0x0000cd40c9ae9f18,0x000bb3ef18abf742,0x0000e4c4353ecb88}}, + {{0x000fdced3ed27960,0x000a90267c9be8cd,0x0001590535eb2ae4,0x0007701b0fc0c04a,0x00000a023e8bb8a0}, {0x00066947177447ea,0x000e06cde38d8e01,0x0001fff605dcc55c,0x000c26be22dda145,0x0000b32e709b81ae}}, + {{0x000518964e0c5fd9,0x00008a23f3ad0b05,0x00008aad44cb4866,0x000c136a024b9aaa,0x000067192f7f61c3}, {0x000aa96177525e8e,0x0006f03cf1735dcf,0x000d9114dd244865,0x000888ad108db4cf,0x0000aec1bfd08a5e}}, + {{0x000ab3806cce7241,0x000c1ce423887790,0x0000a059a1dfabf2,0x000a244b273397bb,0x00003d65f2a93933}, {0x000a60d1b06036f8,0x000cea8ca3b592d5,0x000964a2fa69559d,0x000e4b4a59647493,0x000017ee9471a84d}}, + {{0x00014f6deef40534,0x000a82e14d3d8207,0x00033d6449a055d1,0x000d431c43f7ed5f,0x00002ed0948cf9d7}, {0x0005e0389a99f1ac,0x000c7cbdce132937,0x0000397e12268f39,0x0003760a23c2f5d9,0x0000c935abac75ea}}, + {{0x00052b1e1073e1a5,0x000f84a0c1c1455f,0x000d942637ad068c,0x000010f8a702108f,0x00002c64ff56f96c}, {0x000b9a16d727e0b6,0x000acf06fe3379e5,0x0005d2bc212790e2,0x000afddcb18846f9,0x00004d030329bc97}}, + {{0x00000106b178893e,0x0007ae2c3659cf4c,0x0007128b98eaa6be,0x000d0f17c825e01d,0x0000fd2e94e765ff}, {0x000797e208c8a081,0x000d12668d49e74a,0x000cbfb9b2628f2c,0x000072932fe4d074,0x00002abae9bd73bf}}, + {{0x000793e3b6b83b30,0x00064f8714d7bcdf,0x0000861188ceb25a,0x000836b57f887559,0x0000aa70f4c39c97}, {0x000e5a6ccef975d5,0x00068bb3df26c700,0x000b1425b20bccac,0x0001beb2c7708823,0x0000c3e597f24036}}, + {{0x000920b4d475bedf,0x000601c20860a37a,0x0001a6c7768b0a63,0x0009437bdf506d72,0x0000de3d4aed0bf9}, {0x00051b6900b359cf,0x000cbfe3d2fe1e07,0x0002762542109347,0x0009d5d83f981349,0x00002085a33e208c}}, + {{0x000b427915cb2d9c,0x000fec8d33d25907,0x000a6b6f878fe209,0x0004da0927589135,0x00008ba9de943b1c}, {0x000a09fab0991dff,0x00080c1e31c86c33,0x0006792b45738377,0x00022fbfabbfabf8,0x00000b81945640ae}}, + {{0x0007aa27ff722866,0x00095500f6877413,0x0003805fcb9e8eeb,0x000097b2ddfcf0b7,0x0000906494cfde74}, {0x0001dad9278c86af,0x0001a53f73534dfe,0x00017c8e51d0b3ee,0x000e2264fe779fe2,0x0000f9399f5c2391}}, + {{0x000f2b9a899efa35,0x00044c5bebbb0c89,0x000f7a937ed79daf,0x000d8393645c3d73,0x000002e30297a15c}, {0x00000d05a8de6267,0x000f1c09869fb65a,0x00001a95369da17e,0x0001d8284f596a68,0x0000a54eb42f428d}}, + {{0x0007afd35e423eb1,0x000207737f624bca,0x0000589345b1b05d,0x000841ab44d9b3f6,0x0000856926f19438}, {0x00031d759bbd89f1,0x000491c56f96ea9e,0x000d5c66b3d428f8,0x0008f65ae85d04dd,0x0000fe7d32eb79df}}, + {{0x0006c9eef207e08f,0x0001e7e6e201fa79,0x00066ba39be6e809,0x000ae2ff6bec79f1,0x0000b616f14c0f5e}, {0x000a3da1546202dd,0x0000e77aab41895a,0x00093e7c9fe3e873,0x00005f4a80c70b54,0x0000548eeadf6f0a}}, + {{0x000cc1a29a108d50,0x0006ccd7dab0a262,0x000ec67c8390b012,0x0005dcb4c64bcf06,0x0000f259d55a3603}, {0x000d5ece299782c2,0x0007a0a1dc54ab27,0x000dd366a6679ec3,0x000a9f22b1d8fa56,0x00000463c1d0eece}}, + {{0x000e0e293cec1e79,0x000efc99ec7f6002,0x000ea6e4722d8500,0x000d2880f2620157,0x000054e88ff06240}, {0x0008fcb8d2e2a682,0x000bb41a29a94f41,0x000046598dbf7cca,0x000fc1c555e4fa70,0x0000e866a08c23df}}, + {{0x00055eaf0cd106ed,0x000eac0ffa583fe3,0x00075d5cb5db3a43,0x00068fdd48ba02fb,0x000000f80f6d4c38}, {0x000dffd39b9c5153,0x000091f21f2463a0,0x000a549ca0718f92,0x00093a5a7c8b59ed,0x000071141a9807b8}}, + {{0x0007357890dd4e38,0x000ad26368db2df2,0x000025d8495c27cf,0x000ff989f1617b78,0x000034d4937f3648}, {0x00079875fcdf6a41,0x0003bafc7add2319,0x00065054796a385e,0x00063f88333b6cfe,0x0000ba555751f971}}, + {{0x00044e669582dd15,0x000609ef7f52438b,0x00028c7c925d34d3,0x00088a39d0f09e7f,0x00001aa149662b55}, {0x000de74ef350ef69,0x00050efa2a33bc70,0x0004e9b4ca36537b,0x000f476a52619b28,0x0000e5cc6e2e16b7}}, + {{0x000740b19e406537,0x0001128987018afd,0x0008c9ef1e3cfcf5,0x000147c2788d343a,0x000000bd40c703bc}, {0x0006907b78203587,0x00054432615b4606,0x000a2fae0e3938e1,0x000420ac9fc4cd7e,0x000061a1a83680d7}}, + {{0x0001e1652f266287,0x0002072786662748,0x0004d6d5004b921b,0x0009798d1624a8c9,0x0000a6c2afe78ace}, {0x000b4716993415ad,0x0000b70c175d0b87,0x0003efa55ba75b9e,0x0009da493ef2cd19,0x0000967d0431319d}}, + {{0x0001bb3328972e05,0x00081d050273498c,0x000b43e91b4aea4e,0x0009d7c51b0b14e4,0x000036a831285442}, {0x00092962275aebf9,0x000ceea0fb01f9e0,0x000ea5a49dab5109,0x00062df927cf2354,0x0000f8845418f676}}, + {{0x000f39d3d76f2654,0x0008b5cef1752354,0x0002c38aeabb3b6c,0x00065abb0b5fd32d,0x00009a4a07ce1f25}, {0x0005648873cbfad2,0x000e64e5d115d4b3,0x000e40d2e3544b37,0x000823b6b1a1b049,0x0000420db940926a}}, + {{0x000a61e31101b401,0x000b74d28dad2d7c,0x0008f99381581fb1,0x0006d98a29dc712e,0x000046c5f42fff53}, {0x000011451423b3eb,0x0003c0d9f657b72f,0x0000ef292dbb3276,0x0002a7c4929e7744,0x0000ee6ba08f8841}}, +}, +{ +/* digit=4 [{1,2,3,..,}]*([2^28]*G) */ + {{0x000c50fdfbffcbfe,0x000a12ee6bc0d5f3,0x000d1eb16b2119f0,0x0003b21259ba1e92,0x000095afdef8cbeb}, {0x000584520b9e2c92,0x0008610feb591e96,0x000754802ab80276,0x00031ea8ecb90bdd,0x000002948477a4c6}}, + {{0x000fc6d460d9c206,0x00024b68998b6f02,0x000fc9a84a798355,0x0003bd99ba516a43,0x0000f1c752d2f2ce}, {0x000717f6e91c96f3,0x000ba1ff04750021,0x0000101ff724bb9d,0x0000f57875a3b062,0x00001911b2aa36f6}}, + {{0x000c9dbce96a690e,0x00020a9b23952efd,0x000ece1eda0be426,0x0003e42c87d6e797,0x00004250a5772078}, {0x000f792fd6216e65,0x00011a4a7752b22b,0x0004f0652e1c761c,0x0007d6395f3037f3,0x00000f9718cd45f8}}, + {{0x0004c7f57ef9e74f,0x0007af373f4ee836,0x000f3d49fd3a3dde,0x00003bd7ef68deb6,0x000023a2d231ebcd}, {0x000eca838c637b01,0x0002f973dfd6c576,0x0000dfe2ab1c805e,0x0005a94572e7fda6,0x00009401440f369a}}, + {{0x0007d74400024b40,0x0001f5b00db08719,0x0004509077de17a2,0x000354c8eb4c5624,0x000076dc0dfdda91}, {0x0004925d7e16e234,0x000e71fb7a5cd3e3,0x000628461ccda123,0x0001e761ca242827,0x000040a6ef696001}}, + {{0x000e553f6d124300,0x000275d95d682353,0x000d2b82b5b6dca7,0x000d63f3a355d098,0x0000ae0d057f7ca3}, {0x0001f9e4577d877b,0x000c3279477c834c,0x000c7e5cafae2064,0x000a2108248735aa,0x0000554c8dbd94eb}}, + {{0x000176969fce95fd,0x000241c5aa6dc450,0x000beb9be4d5ea44,0x0006fb48576985bc,0x0000dd0c88c86a87}, {0x0008fffa1635a744,0x00014b52e79efa78,0x0008c0387af837e9,0x000b1463f31ea88f,0x00003e02994385b3}}, + {{0x000adf82eec10655,0x0006da35576881ff,0x0001f3593b826072,0x0005479f5a9118fc,0x000081b7ed8887c5}, {0x0009f9ff322cf7ed,0x000a52f29f0216a2,0x000a1c6f5503a339,0x000087e4359a783f,0x00001f2f7550b63d}}, + {{0x000d2b7c76e70230,0x000cac6501adec50,0x000d15eb9ef1627f,0x0007ec71504ac4d0,0x0000e8e72360119c}, {0x000a6a690fe5719f,0x0000b2a6adf1eff9,0x0008001065e53180,0x000c3e7128f929d5,0x0000a8ed05310b1d}}, + {{0x0000184375155611,0x000680c6035cb9d5,0x0009cd9ebedaf7db,0x000025f68ebbaadd,0x00009b22d47c19e6}, {0x0009ed7e238ec156,0x00087ebb00fe3165,0x000f4fc1d9495e60,0x000a7fb7a835fe31,0x0000cbc5fbf249f5}}, + {{0x000ddaf091b1a54d,0x0005aa18a0a22a3e,0x0008c82e61dfde58,0x000ef6fe2926ccce,0x00006f8bf537ed18}, {0x000f182c887d4a0e,0x00001be42965f456,0x000e7c978127d1ce,0x000537d0a1d9e889,0x0000569de4dc738d}}, + {{0x000fa89a0e74e6d0,0x0003d52a6a5b2e22,0x0009d6028cb9cb9c,0x000868143cf11bcb,0x0000f98ceb8beb7c}, {0x0004f0bd89ceabf3,0x000d3055f6bc7d18,0x0000611f84ca8c91,0x0009ceb37e7fcea7,0x00002ab69d8e4537}}, + {{0x0006164be8ef810c,0x000c68f62e3a73d7,0x00042132ea0f220f,0x0009b931a8cfae76,0x0000120e0489c3ff}, {0x000056c882e544e9,0x000dba82d21f227a,0x0003f4d8ead76b96,0x000524458fba4ab8,0x0000af5f9fc5fd38}}, + {{0x00084efca06ca980,0x000f34de85805779,0x000729e45600bd35,0x000abe8d8f71256b,0x0000c757c5918e10}, {0x000d8a877f78c2e4,0x0008eba882fe2edb,0x000ff0a632610953,0x00000ba6c1759a7a,0x0000b96a1b55892e}}, + {{0x0007062382d80b8a,0x000852b7fcb37ae6,0x0000105dffe9b512,0x000554b3f092abdf,0x000049e87155c674}, {0x00073628f54f8dad,0x000eba81d77012b6,0x0001176a2e46bb7e,0x000f52d8011507b0,0x0000fc7de4c283a2}}, + {{0x0000a3774b40ba51,0x000d2e1cc3a9ee5d,0x000566b1a46d5ccf,0x00059f07988c4721,0x0000c80b4f9aef60}, {0x000f16c5f230e80a,0x000da03ec1812abb,0x00006315b4b346ed,0x000fedf2b2d744df,0x000097e8aa16c20a}}, + {{0x000d6e10aa7185cb,0x000017ccc3b062e6,0x00011d8ad8b90450,0x000f8e927aa95f45,0x0000358b43fc06a6}, {0x0001d5d7e36dcfc2,0x000fc29193e72912,0x00012443b6776098,0x0009b04e627380b4,0x0000fd059fca40f2}}, + {{0x00006b777ce58f01,0x000345cefa37e057,0x00050cf438f95f41,0x0008ee0abf1f45d9,0x0000d3ab82dfe23f}, {0x000dec2fdf1e1687,0x0003a9ada1e3f0b0,0x000621bc115b07d9,0x00091804b5c24dae,0x00005a536309e580}}, + {{0x0001a1f36d5a3e11,0x0008dc2f815e7fc9,0x000a6ab5b38987d7,0x00005ba64e516fa6,0x0000e35f790ff4a1}, {0x0000a6a16d83c088,0x0006ceefab1c6182,0x000449d91c98968f,0x0007bb24e4313214,0x0000f72073843b6f}}, + {{0x000bdb5454f13c39,0x000a22272f00d7ab,0x0002e178a51c5620,0x0007ffab23452bb1,0x000095fe77e2c7ec}, {0x0005269f7a15d9c8,0x00096a8c061d84bd,0x0004524b8f8eebd0,0x000df15fadf6f11d,0x000006922729d2f3}}, + {{0x0003db3cfadbe550,0x0007ea332972b6cb,0x000375df17c3205b,0x0003d308348750d8,0x0000156569ad36a1}, {0x0005c1e5a6730825,0x000935b88ef2de6e,0x000bf9f4cdfbcd4a,0x000c1aa110289f97,0x0000e8b4b218715f}}, + {{0x00047831dd65d665,0x00061c4f2c50ca7b,0x000e125f3749ad4c,0x000d6b6dedd6ee5b,0x00008f3560aa418f}, {0x0004cca576eb6ef0,0x0006bf234f5421e2,0x0002931e2453d90c,0x000a5a30ba21d2d2,0x0000f40aef62be5b}}, + {{0x000906e3dcb95765,0x0003952a2d0bb27c,0x00071e7a288f483b,0x0000b33851620f5e,0x000049133fb7e046}, {0x0006fedc5361d7ed,0x000097bdcdbabe80,0x000b57799cd31b0b,0x000f7bdbe9c25b69,0x00005ac7039c2b9c}}, + {{0x000b6b867e927375,0x000fa63b9d79dc5c,0x0006f3e1b3db6fd1,0x0007c0d5e633292b,0x00008252bcf8f282}, {0x000208df42a98b04,0x000321910210d248,0x000c2cc0bbf1cf7e,0x000fda998a5ce085,0x00004661e9931c1c}}, + {{0x0003ff64af040144,0x0000271a4dc2976c,0x000f713650222aaa,0x000a7b0407b2fb4a,0x0000de28d8759197}, {0x00046b6fabd1d7f3,0x000f4abb6a1aa945,0x0000e83a29065344,0x000c3009a559142d,0x00004e4b62383c59}}, + {{0x0009e0629876866b,0x000baa1e5ef979af,0x000344a68c0b5f09,0x000b007ac8563053,0x00009d8b747143ec}, {0x000042138fad570e,0x000f593e59ae3704,0x000886949e418ec7,0x00095cfd231c4091,0x0000cff10c404233}}, + {{0x0006585937313d7d,0x000c05513277642a,0x0003aea43c3c0b6e,0x000f848100f54c99,0x0000872c0c47c3fb}, {0x00097cc55eb26b25,0x0002361e834684ad,0x000dd6b60df38d56,0x000f85fa83e5f219,0x000080ec5b9b7a9b}}, + {{0x0008e57fdd7ade46,0x00059b24d6140c44,0x00002d217ca3f679,0x000cabad76479b8f,0x000088094d742238}, {0x0000f8a7e3617373,0x00035946e4869acc,0x000c11e58b0c0758,0x00047a937c7d9950,0x00006d8fa5650507}}, + {{0x0003e42674ed11ef,0x000404828d8b786a,0x000696d9b1c4a0b2,0x000a64d22d72fb93,0x000014d70fab6fd3}, {0x000db48af8bdb685,0x0000a04eb44f8247,0x000dd01bf9cf8243,0x0005d612508a37df,0x00003b03be491bcb}}, + {{0x00052e217a1d1fb3,0x000e03733ad7df0e,0x0003c7441a511a17,0x000e3fe73bb6fe95,0x0000c210edd71197}, {0x000e1c3b3b5b574a,0x000d784c56ae9c1b,0x0002719fec3c39ba,0x000ac883f91c7e72,0x0000596cbb0e72a7}}, + {{0x00087a7c47d16487,0x0007c27d1eee4af9,0x000efb643de140a8,0x000af19f4764d9f4,0x000015e713bb998e}, {0x000e86dcbb8a91af,0x00012fcbf8d52174,0x00004c90675aec42,0x000d6fee55cd1165,0x0000473a26dbefc9}}, + {{0x00036638b477ef6d,0x000889bef8c0d8f8,0x000f69f8ac86759b,0x0002c5990ba889fc,0x000077c230155958}, {0x000318fe4ed774b1,0x000943b1be4baa60,0x000ee1a405a3bdc0,0x000f93f1b2f417c0,0x0000d8aff7c79fd5}}, + {{0x000ab61ec541eea4,0x0001ddc9914833ea,0x000ddea233e07a02,0x00092d963c34f544,0x000044eff2fab1eb}, {0x000151dc44d86278,0x0003f0a1ebb24397,0x000d4968c888b6c7,0x000fdf950aa9dbd8,0x00009aceb5a6375e}}, + {{0x0008f88f4b8f4675,0x0003e1eb007bd2ce,0x00071e4c30d657d9,0x000e4cd8527b3006,0x0000e64c785f241a}, {0x00060c0a55cbc596,0x000f64194d67dbb6,0x000a74f7e445f24d,0x0008892b9bf8cc55,0x000087103038e47a}}, + {{0x000c4132cd9b183d,0x000612f382df8229,0x000787543d725563,0x0008dc0dafa6bf3a,0x000054b1d9938b00}, {0x000909776b50d022,0x000e79e83c7dbff4,0x000339f55e1b465e,0x000b542cad7a0e54,0x0000ba6a0a29e068}}, + {{0x000468725b4544cb,0x000d798fc46027b3,0x00037c0fef8d6693,0x0001f562e64ba62d,0x0000f4f40675932d}, {0x000d29eb4b2b5d30,0x00096664493a478f,0x000a8ba7c916705b,0x000ac523815da842,0x00002363b3a920b8}}, + {{0x0005576b1b9c4387,0x00012f0b382afb58,0x00090cf5e15b9217,0x000e25c7b242acfc,0x000019252c1e86b5}, {0x000d5ad8a3d0a645,0x000eeaf47eba0955,0x000f27b0e1492667,0x000c14fd914315df,0x00004f8a7c6076b0}}, + {{0x000807ac46701c33,0x000b8d6d492ad054,0x000c5c86829eeafe,0x0003db877d68abd2,0x0000fca131606089}, {0x00046fb59b42ee1f,0x0003ac322ebe5b5b,0x000567698797e76a,0x0003c7d91f0f74fd,0x0000c77ff2c96539}}, + {{0x0002670ed84701aa,0x0009919671d435d3,0x0006aa77f196a6d1,0x0009ff14771936f5,0x000045e8ec81d56d}, {0x0008ff7e385075a2,0x00053a57ae34859f,0x000496f4ac1269ed,0x000ab1e97fce260b,0x0000bc829794fbcc}}, + {{0x000727a64762c5b0,0x00023d5228f62cfc,0x000f9e44f2954c41,0x0006a900f293a819,0x00009324b17896cd}, {0x00027fe6fe89cd5b,0x000c3c1764b979f4,0x000d84375a0a4d5f,0x0007be72e67bd8fe,0x00007978b94bf0c7}}, + {{0x000014848f08e5f8,0x0000dc08a46d3f0d,0x000f2f51cd2de0cd,0x000f26442abb9c85,0x0000b1f0caf156bd}, {0x0001a678227aedd6,0x000363358fbe9f6c,0x000e8c01632b677f,0x00025feaa88fbae1,0x0000dd95784a0bf0}}, + {{0x00036ab62c9baf2f,0x000e3a0427cc9a23,0x000a9a0ba77b1f0f,0x000b2ba599b2ab1c,0x00006167aaef6883}, {0x0000834b4a11c2b3,0x0007e5fdfd466b81,0x000b1bdee8b8d4df,0x0001c6c0b819a31b,0x00006b8ce7712b3e}}, + {{0x000472ce5ee2aa98,0x000a7c8a447852c7,0x000e5e562934541d,0x000111e5956b46a2,0x0000c2def91e1de2}, {0x0004c70da2035d12,0x0009ecf24750f0ed,0x000774bd31fbde41,0x000bef51f1644a0a,0x000075f1a69ccb8d}}, + {{0x000a76901a375370,0x000534a3899464c4,0x000e1bd64649cb25,0x0008de028a8fda1f,0x0000afcf366ebe08}, {0x0003cc2fe33a0853,0x000c655169d3adba,0x0008bacebd18e2f2,0x0001722b708df59d,0x0000673842d3ffb5}}, + {{0x00084f5b5b4155b8,0x0003bee12769b41d,0x000d7c141a89a9b9,0x000f02731f3e90d4,0x0000f5807b7dec20}, {0x000e4d79f5f291c3,0x0007e1820075ebee,0x000d095ee31719b1,0x000e77b13e98787c,0x00009bacf15e8239}}, + {{0x00021e1ade0c9f13,0x0006baad0ce722d7,0x000d6be87abd4fcc,0x000228a81f7554c5,0x0000e992ec32bf51}, {0x000dfe81a7d3c300,0x00060efd94bfa623,0x0007abb915ac1f99,0x0001008b04e471a8,0x00006a6dfb158b1c}}, + {{0x000895918d9c587c,0x000b90d8c07b4267,0x000ab468e989eec5,0x00096f5f2fd3207a,0x00000c4fd8f7ddf2}, {0x0008b1cd86dd7ac0,0x000a8a8ab83f3bab,0x0005768923fb00d7,0x0001e0496024dfd8,0x000093c337f8280a}}, + {{0x000f2f33fb1ed546,0x000f5524877cacf3,0x0009dec562c094ff,0x0002af42d6955bb6,0x000051aac6f16c54}, {0x000f747f4566c18d,0x0000f500d0f779ed,0x0009bf4c72dfb961,0x000237be0f508927,0x00009d1af0421cb4}}, + {{0x000a30ddfb3c9567,0x00044273b206d755,0x000b8261642fa0c9,0x000999c462c3d5ec,0x0000e5cfff9087cb}, {0x0007f4092bba64cd,0x0003c030e85f29c6,0x0000dcbcd00ede6d,0x0001f5c5ab9995bb,0x0000d909fe94ac39}}, + {{0x0004393c8401d21e,0x0008419e515f55e0,0x000b70a2c970650a,0x000812aa1988dfb2,0x00004579ea7e0f77}, {0x00001ae0b356ff64,0x000a67b2f4f8d464,0x0004e4808c24fb2c,0x0003cc7049b70ecf,0x0000d988de6931b1}}, + {{0x0005205bc1daf6aa,0x000b46ea4ce6aedd,0x0001b36b71d5ade6,0x000c684feda79817,0x0000c1b97fd8fcc2}, {0x00015296e4680f82,0x00039c979c619a51,0x00019b0dc58bdb9c,0x000edbf5165245ca,0x0000dda5f64c3464}}, + {{0x0002da59de36c6d3,0x000dc41490e573d0,0x000197752e9c3c3d,0x000caf7729881db7,0x00005781ea0add88}, {0x000709293651cc16,0x000b2ea5e1c2adf9,0x000040381b05df36,0x000cd96512965ec0,0x0000cf761e9b4f6f}}, + {{0x00012f98f1b3e1b2,0x00055f5544270463,0x000c9e4a1d2b7326,0x00007925876fdd82,0x000074ea097a9061}, {0x0006e346721dc27b,0x000037acb54bd914,0x00011e8654a4af5d,0x000c7efafad16cfe,0x0000f0e1c71ca6c9}}, + {{0x00056bad8c8fb6bb,0x000d537ea4bef324,0x000e9fd62945dd49,0x000f97b5d567fef7,0x000086c5b4120ca3}, {0x0002cbc8de7435c4,0x000ef4e32422c90f,0x000915725631070d,0x00056753035060bb,0x0000ac9120611153}}, + {{0x0001ce2ea4b332e4,0x0008a88d39b03943,0x000be58c21e92546,0x000b8614d72a258c,0x0000d6dccab15931}, {0x0000f4d58db96687,0x0009b295553ee450,0x0000bd38d9439d07,0x000760d9baad6913,0x00000af15cbf9b45}}, + {{0x000e116715a75218,0x000ee37a35141d7c,0x000d96ee6c976ad7,0x00008be27249f510,0x0000dd9663c48776}, {0x000806b3a6abaec9,0x000974d53433737a,0x00034026f2f4f6f0,0x0003631542622247,0x000033f84d61ede9}}, + {{0x0005789811a3ae5f,0x000145466230de30,0x00035f272f8c7bcd,0x0008464d6e1bcc51,0x0000c0b21060c721}, {0x00056b9c81e33e1a,0x000ecf8a7f58e633,0x000b78291f065f2b,0x000e115598fa93ae,0x000008d4d5dc0246}}, + {{0x00023976b401d760,0x0006a4914be6b70b,0x000cd32b79cb8bde,0x0001ffab9c10c28c,0x000071ebaf4d93ce}, {0x00025fddbaeca2e7,0x0008b2280bebff24,0x000a3891662f455e,0x000cee7ab7537d72,0x00000da3f309f420}}, + {{0x000d2fa921a168a7,0x0002e4b7b670cfd8,0x0001d3d2b35868fd,0x000fd4925f10a54f,0x0000d3ce9bd12974}, {0x000176530f52e7e9,0x0005ca6a8cc32d2c,0x00019482b90bd81b,0x000cd9a2087e9855,0x0000804eb315a666}}, + {{0x0002197860bc9037,0x000d8b8123349561,0x0003876dc1f987c0,0x000583fb6e6ef179,0x00008cb21fa0cf30}, {0x000fc89c021518a8,0x0001fe3269ddefc5,0x000e6ecf600526c8,0x00082dc869beeb1a,0x0000ceed4cbeee27}}, + {{0x0000659ae15c0cff,0x000524432040fd35,0x000c95fbcb5411e2,0x00010634f1a87377,0x00003ae2a14e5951}, {0x000c01abc86960db,0x0007837753dd90d3,0x000b3a5cbd54cd6d,0x00025ee3849235af,0x00005aead12b08c4}}, + {{0x000b7d73502d7d8e,0x000d7454261868b3,0x000e46fbdbc13c21,0x000713844ea5cc30,0x000049be3239b611}, {0x0002b0432aea713a,0x000625e8db9e6204,0x000d1b612358a422,0x000bb2de468b3935,0x0000d73f40958c1c}}, + {{0x000ef0b57b47fd09,0x0005a747602a55e1,0x000ef3a6ec69c95a,0x000a9812736cac28,0x000000509d742fc3}, {0x000649cba516376b,0x000241310dde1462,0x000486dad6e304cf,0x00067a74ba5dd344,0x000076e637f5c43c}}, + {{0x00011c2158de7341,0x0007322124e7a4f5,0x00002187b24db413,0x0007c89cf37f3770,0x00005c2bbc1a94be}, {0x000c3a8a2a94c5d1,0x000019673c8091ee,0x000a670f1d766afb,0x000dcaf7cc3d5518,0x000067e6f9927685}}, +}, +{ +/* digit=5 [{1,2,3,..,}]*([2^35]*G) */ + {{0x00005700bcae739d,0x00002be037d24753,0x00015b109f3a7bb3,0x000e43f9da256a61,0x000093ad7b5eea05}, {0x0009fd0396309d30,0x00051affa8b57a31,0x0009e17000aa55be,0x000584057dd782f4,0x00008fc1c3299638}}, + {{0x000d93c48b3b2619,0x00080481131fe4d5,0x0004069419aa8668,0x0002802c3872251d,0x0000e0421b9f2df1}, {0x000bdc567515ba99,0x000aa60a7a56df82,0x000c254922dcad79,0x0003014dbea6f756,0x00003529c91e3e54}}, + {{0x000f4cead9303d8e,0x0007da2afde793dd,0x0006deeafd0c52d0,0x0002cee4e07a6bb6,0x0000805a8d0d1862}, {0x000bb34d90f85393,0x00081fd23389603e,0x000a4addd74321cb,0x000ce7a3c9e82e99,0x0000dd06c08d7e68}}, + {{0x0004e4b39f1a7fc3,0x000685bd324dc746,0x00077329f942afb5,0x0004c07e4ddb8995,0x0000a7edd72789c7}, {0x0005b61b34ef6322,0x0000a0c0708326c5,0x000423a1ac63d4b6,0x000e55cca0e0f401,0x00009daa63d66c01}}, + {{0x000d5fa96476c552,0x0001a9871f6885b5,0x0004ffaab3c7c9d1,0x000d4603f8de64d8,0x00007b2e031462b5}, {0x0008c9e92f0c5787,0x000de242f7202665,0x0003c82030726817,0x0006a6a3eaa87186,0x00003c95078f22e7}}, + {{0x00023916557f453e,0x0004ae26a44583ee,0x000fb5b0a3af5dde,0x0008c41782c4831c,0x0000ea56bb04a5ee}, {0x0000d169d9ff58e9,0x0007dac5db5ca14c,0x00004bf3faad365c,0x000cefa7842801b9,0x0000c473429c32cd}}, + {{0x000d3a6e10d62089,0x0002e7b29cd9afa4,0x000148cd51c8b2ae,0x000a75007bff9823,0x000091ea2803d58f}, {0x000a6e1d7cd54ba9,0x00012006683ca3d4,0x0006b0cc5578ac68,0x0005cbf6720ff656,0x0000c9b8e7ae81da}}, + {{0x0009db6d2b5b0c57,0x0000d45183dbc273,0x000c532d1cd3ebe1,0x000638b8aa1df7af,0x0000d1b32952af42}, {0x0005152b969da4ab,0x000ffc53c8ae6a5a,0x000f8a7ed96e444b,0x00021fa7d3dcb0a7,0x00009f0fef615ad7}}, + {{0x000b2a69f40c9752,0x000d9c51b7ec3baa,0x0003ed0013ffe154,0x000b0917e419d1a5,0x000087bc5ca8dc9f}, {0x0006dfe0e70f6230,0x000bdcab123ee058,0x0001e5900c76841c,0x0001e44fe0b74ae1,0x00003adc613ad53b}}, + {{0x00063c9f8355c127,0x0001540eb5653460,0x000b854039ff3d78,0x0008ae7c2d9a955b,0x00009ca4d64c814d}, {0x0001356a07c69e07,0x000e21ebd8080839,0x000a3b9495398a52,0x000846dcfe86a411,0x0000787df0a13b19}}, + {{0x00027c45c73a8764,0x0001414547d3af8f,0x000fb60a1bb40672,0x0004ae7a70f5fd99,0x000082531e392b1b}, {0x000da4c7c8966027,0x0005e83a98dbec18,0x000a67cc1dd3e148,0x000d57deb57d902c,0x0000f11b0aa60752}}, + {{0x0007d863d57f3414,0x000fe1509dc623ed,0x0008df4273550cea,0x00065f9b96e5fcb4,0x00009ae9885ba4ab}, {0x000ac350b383e5bd,0x000a0f46498bc168,0x000a58596f3b798e,0x000fe6aeb0539155,0x000054e1d4ea8874}}, + {{0x0003b275d29dc04d,0x0007d76747fcb13b,0x000306ea3b2fe9b6,0x000e902e3d618276,0x0000789aaa24ff3b}, {0x000d025585e1c78e,0x000596e4aecfcc94,0x0004f5b5dff9c8eb,0x000695d3ac036938,0x0000d2e6787c6ac0}}, + {{0x000ccd3ce60d460f,0x00007ea73fe2e166,0x000b13997331a00f,0x000bf84382342cbf,0x00007e70f6af3225}, {0x000335bae3648d46,0x0005c252c9be9d30,0x000b173ced6a5a93,0x00062cd16c887a31,0x00002a4f9fbbbf9f}}, + {{0x0005a47a979741b1,0x0009565bbf2dd4e9,0x000af96bbcbc82aa,0x00026bf44f9cd822,0x00004b2b149910a4}, {0x00050f9cdf45d19a,0x0001eda90005e40d,0x0000abd40b1a8e66,0x000244881e5b8a56,0x00000e73a53fe7c9}}, + {{0x0001dd32de445aa6,0x0001fc91cf2f9f3c,0x000cf76c57384160,0x0006f62abd1f13b2,0x000052e2ed7e3fc4}, {0x000d88787f35a985,0x000e2148e3be1e7a,0x0007b25528a99e57,0x00035b65281c2f4e,0x0000c1eb2107c6cd}}, + {{0x00035d5a8c2927bb,0x0008de6d2b0185a1,0x00024afffb21509f,0x00056b3b64117f50,0x000014162b8f15d3}, {0x00024513d05bd8b3,0x000434f9385172af,0x0001a1f98c4d653f,0x0005e9ef0660e5cd,0x0000273cbf1678a5}}, + {{0x0005911351a59103,0x000e04b05be68d91,0x0004a7bede129f38,0x000a1a51eec3037b,0x00009215fc231a08}, {0x000dd81523b72a23,0x000894715cc1a87c,0x00091f600ab80cef,0x0005a0ab3eca37dc,0x0000b2bc149ce4ad}}, + {{0x00068d2c36fd9f9f,0x0002973c5be71204,0x000e5a97f4bb36f9,0x000f1907757b63c9,0x0000fcc1fbc2fc33}, {0x000eb5cac213cead,0x0001c14e861adc28,0x000f903943fc7081,0x000da639df6e310f,0x000040d070b3e432}}, + {{0x000d7619d49860f4,0x000ea16ce8ca2f98,0x000f7a39c2b109d7,0x00055c8ae1632632,0x00003a28a5876314}, {0x000509cc3ed28df0,0x000cc59ccd51ee9a,0x000cdc1784ba5cb3,0x0005898cfd78caf0,0x00008e1987ffc9c4}}, + {{0x0006780551ed5362,0x0000162b85d37aab,0x00052c0a5dccf49a,0x0000008978e8fe28,0x00002aef7a358742}, {0x0003eaf9eb9304c7,0x0002bc6376029477,0x000c2b67f2b6f818,0x00045ae346f21474,0x0000533e9491e7ee}}, + {{0x000821aa8c6a2ac8,0x000818984232ecee,0x00008bcff77eddc2,0x000f1868ed1e27bd,0x000057062274e3c7}, {0x000f1f1e845f219e,0x0006638746018c4b,0x00079c20a4d39735,0x0003878465feda34,0x0000ad6855572cb6}}, + {{0x0009354b24d7ada1,0x0001567cc35e0245,0x000f79c525b386c1,0x00013a1c4fe295fd,0x0000734ae5333989}, {0x00049712b0342f42,0x000a478fead6dbc2,0x00041f2ad87713c3,0x00037135a752173d,0x0000b5a42c879097}}, + {{0x00066a43e3dbd58e,0x000729043504c8bd,0x0001dfe5e88ce125,0x000b8e4a214027f1,0x0000e0bfff41c734}, {0x000497d8ef9bff74,0x000dfbd8ec3dde6c,0x0009dae39fbaeced,0x000c615eeff4aecd,0x00000b575a8087f6}}, + {{0x0006858fe09d95af,0x000e54554313a7ae,0x0003adbfb75fad76,0x0006af4c0e89f84b,0x0000cbc199841374}, {0x000d3daf04bee219,0x0002f16c60fc3b73,0x0000bf406b089be7,0x0004df5aec57f12e,0x0000f4b714839375}}, + {{0x0001a7b34a992617,0x000262b67fd16035,0x000f4cded03bd337,0x000a495f4d446c82,0x0000de7f0ecd39ef}, {0x0004309127b7e470,0x000333eca6abb6e3,0x000e251b37d1836e,0x000e19b24cc67dda,0x00006986f3fce1fe}}, + {{0x00006b3c165ba83e,0x00040763b7168cd5,0x0005b2089849d3ee,0x000158864b408e50,0x0000d1e7c3c11ba3}, {0x000951b1022b0811,0x000a6ba9409ee8d0,0x00082031e38d13ee,0x000986a7af0f043e,0x0000cf5bbfcacd65}}, + {{0x0006ed33eb976593,0x000cb73bb5996a5d,0x000a8d40797d331a,0x0008195eab513d66,0x0000745011a8ad57}, {0x0009172aa1b56b55,0x000681e0477e446e,0x0004dd55b38c0636,0x00046c34beadb245,0x0000f622ee53fa27}}, + {{0x0007b0a3ab20cf47,0x0005b6ecbe5bf999,0x0009b3cdaea41415,0x000dddb1a4af7693,0x0000c87b4d02be99}, {0x00044162383848a6,0x0009bae35ed444fc,0x000b61b91b37a6bb,0x000c991cfe2a21c2,0x000005e9cfbed62a}}, + {{0x000a1a9489a1a5ec,0x000ce2cc85fbf558,0x000f2223a8aee3ca,0x000478fc82ab4b29,0x0000ce60dd26d1b6}, {0x000dc4d8f75b7e43,0x000c8e7ae5cb1be6,0x0009bc9a205da2e2,0x0000b9d8f07fbb11,0x000007c9c2797b7d}}, + {{0x000ba6fe16b74a37,0x000c9ef80b1e2a3a,0x0009b6ba80ac3749,0x000a58cf3522eba3,0x00004f2c5f7bc7fd}, {0x000ae382cb1abc5b,0x0005df37a1e6cd6d,0x0004f4dd227488b9,0x000587f8906c8701,0x0000c94e7603d61a}}, + {{0x000f31b12c8e946f,0x00041998676a0cd3,0x00012cccc0790516,0x000d05413f948da8,0x0000915d9bc125cc}, {0x000631c24a7ac56f,0x000cc7dbbc6a8f6e,0x000a263d88d9b845,0x000ac140e423d536,0x0000abcdfb4c1ede}}, + {{0x00036dfe301d6bcf,0x000976515927a0af,0x000260fcbb9fec72,0x000d7a1d042bcb4e,0x00003986716d1e65}, {0x0007a8b84e008f31,0x0001a9e0bbf32708,0x0003023c184225e5,0x000006bbd97e7f72,0x00007a8476f8c3f8}}, + {{0x000200faf183b044,0x00091921123920f6,0x000df3ccbb457e6d,0x0001c38fa6eba464,0x0000347d78c4d002}, {0x000b2f467c70d83f,0x000a9b0ee2ba361f,0x000def1984834b5b,0x0006757a81659a0a,0x00005cf0e1308d83}}, + {{0x00041ecde4f12ba0,0x00055e2ab6f60ccc,0x000400171094c76e,0x0001f83d982ca07e,0x0000bfd8343191ee}, {0x000f9f86ae6ad522,0x000bd01e24ac8cdd,0x000622bbda9e9819,0x000bfccf5e80a139,0x0000ea19704218e6}}, + {{0x000fb670adeb0531,0x000a2e1cb11d95f2,0x0002237733ba53d6,0x0002af7a4f5b85d2,0x0000d11b8b4df863}, {0x000f21077157d779,0x00029dae46ae076e,0x000ef1a942676f1f,0x000698d2955c21b4,0x0000366784d1b5f8}}, + {{0x000d37ab64d1b8a3,0x00048a552398fb88,0x0001d701426c97ce,0x00054721bdb66fec,0x0000bdc35f96e76d}, {0x000480bc0f59b636,0x000445d5741bd54e,0x00007a408f26f505,0x0003f9d9158c7ff6,0x0000ff89ff1faf90}}, + {{0x000bc2333e034b3a,0x00064dea255dc2e1,0x000153b1d74a371a,0x0009201c2dd539c2,0x00008464ef965378}, {0x0003be3e2d7ffc4f,0x000a0233bf14077a,0x0002bb433fcdcb11,0x000fb8623c1e9373,0x0000767ed200d1a8}}, + {{0x00082d0bd4f64295,0x0009f5eb0d7a40a1,0x000e40bed0984111,0x0000c3c234be2868,0x0000a55bb693492a}, {0x00003589e0e1c59a,0x000b79f78b50c3dd,0x0008e289bb712956,0x0000c85c99d84cfa,0x0000a5af77c7b795}}, + {{0x0006c6e73e758063,0x000aa4c0ca07be80,0x000a771d1a3c2b89,0x000f8e30bbd4d1f2,0x0000707591fefd22}, {0x000baf90e9d177b6,0x0002f80643449531,0x000fa11454675837,0x0002074957d79a89,0x000023d4bf71171b}}, + {{0x00012ff13201e572,0x0001c2d35ca8a962,0x0009211dfb91e1bb,0x000d6b83cbe76c57,0x00005b47bc9b1b75}, {0x000cf3690ccde132,0x000203bdc7c14ed2,0x0009be86fc27ebba,0x0004eba9b9fc557d,0x000081e537c1cfda}}, + {{0x000edfee74727d78,0x000980a22c07efda,0x000b10e358fad228,0x0005c368538ea7ce,0x0000b7ca5d8c2865}, {0x0006c1397b278758,0x000841d382b260a7,0x00040b29c25b9e45,0x00049b72f524c365,0x00001fffd1819be9}}, + {{0x000e1fe027315046,0x000e4a8e9ae005a4,0x000e29dad879e0cf,0x000713338832c69a,0x00003efed8d22203}, {0x000e9be0c541ee45,0x00043c0e163d6976,0x0006689793657bcd,0x0008f5b80673d362,0x0000c7348859135d}}, + {{0x00072bc3bd04c059,0x0008da4e3ff7f4d9,0x00070f0f1e03cebe,0x0004a82cb1df491d,0x0000cff4a142a9a8}, {0x0007a0aa0adfeeed,0x000d8ea5a8781ca7,0x000f8a9d70def5c5,0x0005f0e216119383,0x000056865c8536c9}}, + {{0x000c2b7335282880,0x000df2155af305ab,0x0003cc18f696a952,0x0003e94feda61624,0x0000074ce58fe838}, {0x000d1423175154af,0x000fe1ae0cacb704,0x0006b7521866fd89,0x000dc49d637a4c2f,0x00008dccfc32bf4d}}, + {{0x00058b746d6febb7,0x00056fdc8cde8233,0x000c5724a2fcaf62,0x0002c41d3479ab73,0x0000cd65f0ee0ad5}, {0x0004b4965d7be852,0x0006736dfdce5b87,0x00022e44ee30b9ff,0x000510f3c1fd2443,0x0000b17b0a94a0b8}}, + {{0x000ed9da5acaa517,0x000e2d9bc48b2b20,0x0009589a56d9ec6e,0x0000cd50699c9cf1,0x0000c0682f67c25b}, {0x000ad83acc2d9df4,0x000b491d3a194ec3,0x000f43e0aa5860a7,0x0008825d7c9356bf,0x0000ffd010d5ac2b}}, + {{0x000ad7d4257077af,0x0007d06b6e41f434,0x00081d274d0d97bc,0x000d46b97f058b8a,0x00008141f09a57ee}, {0x00059c2c95a92ba2,0x0004dbd7225bde57,0x0005ec9647f39a2e,0x000a46b029f14b8e,0x0000e8a3e62f0c3d}}, + {{0x000af97ab489033e,0x0005ffc2cbd90b39,0x000bc39c20e78840,0x000b48cd3c1d2d1a,0x000065f064cee3fc}, {0x000a3b79e03ee91a,0x000d739b8fffef5d,0x0008d7784f339ddf,0x000a02d5cc657cb5,0x00001fb78c5705a2}}, + {{0x000fe19eecc5e624,0x000306f9bcd832a1,0x000b535333e9171a,0x0000a14c3dd19f34,0x0000bfd509e3ec8f}, {0x000d131ff598e707,0x000efce0b7cd59b2,0x00008b2e08b38876,0x00028c0e783e18dd,0x0000a91d85b2dfd9}}, + {{0x0004382f883cfa65,0x00007fc08c3e7142,0x00031d2cf6a1e10f,0x000b73831dee9409,0x0000a725ec88ef3c}, {0x000e45138849f25c,0x000af48749f6cddb,0x000d14475f1cac00,0x0002cabd31e17b50,0x0000d48626e3da74}}, + {{0x000d0de0ff499960,0x0003fbfcf664ef0c,0x000452faffdd7772,0x0005d14882e7b0ee,0x0000b786850a0512}, {0x000390625f2d5fad,0x00097f518ec1a4a5,0x0002f3d039da3312,0x0001a9edc17472da,0x000096d8837d422c}}, + {{0x00098d1cd3290a63,0x0009e70dc26c34ec,0x000fb8080a9ca335,0x000e643ad8d71805,0x00009473d89368fb}, {0x0003a8f3f95413da,0x0006d4ba18760cd3,0x00015ad17f92f4f3,0x000ce2649cf662d6,0x00004e46841f615b}}, + {{0x000d33f7c64c592a,0x000b6794f7501394,0x000df640f443e759,0x0001115b48de8e41,0x0000a085e12aedd8}, {0x0008c0b7e2b229ef,0x000b6d02a9baf2c0,0x00022cc20ae9044c,0x000e8d6e6ca27b62,0x00005703e8b98911}}, + {{0x0002cefc1f02f736,0x000540270033b3e8,0x0003ceb47e84f986,0x000e14fc35ec1c12,0x0000f46085bdda1b}, {0x000cbc25542ce047,0x0005a07add73f5e2,0x000320971ec0822e,0x00035a98911e32d6,0x0000881d02f4e758}}, + {{0x000387b347fbdee2,0x0006458ebef829c1,0x000f7bdfa33f4d8a,0x0002883778b30bae,0x0000a57a86aa56f3}, {0x0001ee174ceb06c4,0x000b005d309fccfd,0x0004d56aaf05a383,0x0007b122e2a74e83,0x00003ae917835919}}, + {{0x000b2e078e41d080,0x000d82af788420d2,0x00062fb2b0bd6d30,0x0000d29e31188cbe,0x0000fb42bc94f44b}, {0x0003accbe6f8e771,0x0007513275c93443,0x000099c782740df3,0x000b459fad4b877a,0x000033181c9bb1c4}}, + {{0x000dde73fb0eda0e,0x0008f6af3e12a3b0,0x00070b40f538cf4b,0x000dfc9a8cad561c,0x00005c1801d14801}, {0x0002dc79588dc298,0x0009f476f2307348,0x00070463a39ee33f,0x00014bb90225bd9b,0x000010b40545d203}}, + {{0x00036dc7a7be1464,0x00010d064c152673,0x00019582bc150f70,0x000cb20deb2fe94d,0x00003c415ba4f212}, {0x000985f73eec9d69,0x000f765a6fccae60,0x00097f2f3c826fb7,0x00047dd07d59c9ae,0x0000707066e8983c}}, + {{0x00041a35ed2eefac,0x0003a0fbcccb63d2,0x000fd5d39ee7f02c,0x000d05a5130a3242,0x0000bbb31c95b0bc}, {0x000ca4ba534b4c07,0x000fbe96ffe723e0,0x000b5137627b8873,0x00053eb426e6c7a7,0x0000febe40fcfcba}}, + {{0x000346e0b45a707b,0x0009552613f225f5,0x000803741bd4eedf,0x000933cc8d49c2e1,0x0000a0a7a1802a2d}, {0x00015993942ef2d8,0x000a6714ab7c1ec9,0x00037e0ce22b32c3,0x000ffe9aeaab9bc3,0x000007042cafe458}}, + {{0x000b755d669d39d8,0x000c3024b484f11d,0x00034f49b01e2ce1,0x0008ed0468857e28,0x0000254a9385fdf0}, {0x00055e07a3d440f8,0x00042678ee665499,0x000d14e839ec6cdb,0x00015ef57c3c712e,0x0000ed2a33990852}}, + {{0x000957be96b408de,0x0001ebc7fea66f42,0x000ee42926b6ac46,0x0000b4e4cb455b9f,0x00007fb272c87861}, {0x0005ea64c175d75c,0x000e57ab709e7669,0x000f97d844601eea,0x000524f6c42043f9,0x0000257150f4b193}}, + {{0x00019b55359a6996,0x00066ea8746126f1,0x0003984c698d3b0b,0x000148d30ea757f0,0x0000a5a62e68d1b0}, {0x000b1ceb549a8926,0x000f6127020a2f62,0x000cb3fd49371a21,0x000f1e9c3dfc9693,0x0000f10737a457e3}}, +}, +{ +/* digit=6 [{1,2,3,..,}]*([2^42]*G) */ + {{0x0008da4ebfceed2f,0x000160ab87632830,0x000a7a776b62d8f6,0x0004161d81461a65,0x0000d2503d370ab1}, {0x000a3a28b12e059b,0x0006dba7a43ef9af,0x000eb449b66afd29,0x000b4f497099cf91,0x0000b740d849144f}}, + {{0x000d1cb14db150ae,0x000c2825db119cee,0x000e5fa06f3d8cd3,0x0001519236fc45ff,0x000017232f7c8181}, {0x0008b9e65214d82e,0x000a8d9bebd6dcfe,0x000c1e6f59806e97,0x000d0a1cae163880,0x0000454a197aa771}}, + {{0x000fb2e99ff222fe,0x0002f6cd6bc0c742,0x00011048d43a1261,0x0002bc6c0346ba52,0x00000874002068cc}, {0x000d17dfae243b95,0x000e8b373c9aa0e8,0x000e84c66747a444,0x0002560f7e0b8280,0x0000dcfd670f7595}}, + {{0x0002524b949e4535,0x00090f56a6cde629,0x00006c5056ca68db,0x00087837cf37f5e8,0x0000287aeba17f2c}, {0x000194d41a0f7a82,0x000aff261f9d588b,0x0006ee54936659f4,0x0003de1c86018de4,0x00005bb7bee891ac}}, + {{0x000dd4596bac0c33,0x000a79383e647f24,0x00062bfb0f5a73be,0x00082923eee62853,0x00009c04543340c6}, {0x0004c782a0e9f2c3,0x000e03645715f5f5,0x000fa691a3c1b9a6,0x000e7ab9adcc6de5,0x0000db83ee04ac86}}, + {{0x000d23467ed57309,0x000db8f9d25bed94,0x000dbe922092fe90,0x000764e90499c35e,0x0000cd06d43dbfbb}, {0x0000064077438554,0x0001c32ae4be0845,0x00061d2fcaebf091,0x000c4346a089d960,0x000055f71e1c5a73}}, + {{0x0002f96218c9d5a6,0x0001641dd16123d0,0x0008ab2e1526c4ed,0x0003e09ffc5e3edf,0x0000df1e41035c68}, {0x00077bcd67728b89,0x00047ee4e6273040,0x000cb877fdbfcf62,0x000eff21cb5a44a2,0x00003511953a16ce}}, + {{0x000409ffca73851f,0x000c3de29624b94e,0x0002a47c5c60b925,0x000e16665847cd08,0x00008e7f8ebd9097}, {0x000eeb9e98545750,0x0000ce814b1f1ed6,0x000b01ac8d35a5c3,0x00075837d8e25100,0x0000c3c5bd7d279b}}, + {{0x00076e1d283c6ce1,0x000f7174959c50ba,0x000dc4eaecd286d4,0x000634535982fee9,0x00009158cb09e7d4}, {0x00017b5f6cd33044,0x0002969dcb333eca,0x0005d0e40dd7519e,0x0006b515ebd579a2,0x000024e41aff93ab}}, + {{0x000040ff1375951b,0x000a1e839dda7393,0x0006bb352da54b3b,0x0002ae6c9597e201,0x0000d9dbd47572b3}, {0x00056e9021dda66f,0x000e6fe0123fc320,0x0007cca3903d3977,0x000ef0e68cdaccaf,0x0000bb4084bf66da}}, + {{0x000d2f201031c945,0x000b772e503d6e75,0x00042cdc17bd8ed3,0x0007777ce2a48632,0x00005c328a9c3027}, {0x00070c96d90b6fdf,0x0006762cff8288de,0x00001a6ce10f17ed,0x0004b60b2595dea4,0x0000d81fe47c8c90}}, + {{0x000c39102e4fe37e,0x0007b23e92ab7da2,0x000353e2a4aa27de,0x000cc7a3fc098e3d,0x00002536baff989e}, {0x000be55df78b163c,0x000cb15a4f900b15,0x000738e13e4be770,0x000cf45a8c0b1e35,0x0000d1c8b85e4c20}}, + {{0x000d8a6f91768025,0x000780bb92a70307,0x000ec75e9f0a4627,0x00018b3f31d5783a,0x0000ef5c08e724f6}, {0x00056b23247f3db8,0x00038ade392d9748,0x000a43483fdba1d5,0x000a5c1dc4f1a93d,0x0000b35fcbd53c35}}, + {{0x0006e5b5f09bd9de,0x0000bc9fb162290f,0x000034f2dbce94c6,0x000c8626fd489944,0x0000d0a4061eae30}, {0x000b78b3bd8a273d,0x0009de5ae80b923c,0x000eef67b29524b2,0x00014e750cd4e68d,0x000073f7f21962fc}}, + {{0x000877c7f92f06b5,0x000b95cee57a30e3,0x000ffcb286123ae9,0x00022d50fbf6547e,0x00004800675d1bc9}, {0x0009b9e818f59d1a,0x00077d2030cdc4a4,0x0003a9ed390ee53a,0x0001df3669085b25,0x0000f51fe98bf18b}}, + {{0x000ccbecccc94f0b,0x0008c99893a55b13,0x00041516d980406a,0x00012ad64445dc39,0x000003dcb08f3616}, {0x000905907009deb2,0x000ded15742863c2,0x0003a9de2612f089,0x0003f2f1e316164a,0x000095cdb1f35323}}, + {{0x00048f32683a1106,0x0003a7a5b3460224,0x000343e47d7bd6af,0x000126c091046994,0x00005de2a5b586f7}, {0x000254188f58af50,0x00060974a069758d,0x00091651f61f3bac,0x000a319e608f0298,0x000076b7f2c0358b}}, + {{0x00000d3f381e3ef3,0x000ceeb6c1dc33a4,0x0002adde75fa3959,0x0009beae4a15a271,0x0000d91f39a52916}, {0x0006e68167f8436b,0x000613d0010f1b08,0x0003bdc583de98b5,0x0008ced07b396a51,0x0000f5f1ccd47883}}, + {{0x00053474745b4111,0x0008d69ab64986d6,0x00028504062204bd,0x0000eca3a2c7a367,0x0000a183bd071463}, {0x000618d9a29310fd,0x000c586fd5dfd7f9,0x00026b628f55ea0d,0x000e9141be7be638,0x00009ab70426722c}}, + {{0x000f4c6b7f0b2b2e,0x000b35498ad70152,0x00093c54616a18ed,0x000e4887a4d5a30c,0x0000c8ce85111073}, {0x000b61623a4c2109,0x0007aaf6df31d435,0x000e6d28f6f0b661,0x00010c63d8f4fd58,0x0000a716f19ad118}}, + {{0x0009cc90fff48c76,0x0001a62b3ac032a0,0x000b5de11f98ce03,0x00066f2164aead02,0x0000da752ba1afdb}, {0x000d7ff20909c958,0x00089515a1f95adc,0x000675794b9fa634,0x000e20f2564f7acb,0x0000bcf3634218fa}}, + {{0x000e4e170e87d63e,0x00033b1ac864b298,0x000d6cac3ba652e6,0x0003f216dfe43c73,0x00000994b53a9850}, {0x000ef04ead3ef321,0x000f6d077d47a1c2,0x00091df581572494,0x0004bc458f8e24b3,0x0000c2de27be5104}}, + {{0x0004d336eb2b06dc,0x000d438af59f9816,0x00086b544e2a7565,0x0008feca007deaa8,0x000038b1c05044a7}, {0x000eacd2e1baa833,0x000ab3d6f9a39bf7,0x0005f9a26b4a4cd8,0x0007fe97e08d97c1,0x0000e07374b44b2d}}, + {{0x0004dd2e81908fd2,0x00053362700b61c4,0x00047b4115ede230,0x000e824902002b8e,0x0000fbcc6260def8}, {0x0003325697f653d2,0x0008966e57020f8d,0x0003416143080196,0x0004c4d2fd9b6e91,0x00007ade93e02f1e}}, + {{0x000ecd682caa18e1,0x000f70d2a0a391d9,0x0002000a80040e56,0x000857d371f72e8b,0x000053775b161201}, {0x000f90ea5eb901bc,0x0009364d5c28a36b,0x00005a3daaeff7bb,0x000133f544a030cc,0x0000c4cbbce12e0b}}, + {{0x000f8c9fcc7163d1,0x000572b83e473108,0x000bba1ec860c1e4,0x0001e09ceb4984ee,0x00008bbae4d3fd8e}, {0x0000e95aa9b6fff8,0x00013bf54498bb71,0x00098c0dc39f81a0,0x000f4d5f47a7767f,0x000053d12f97e872}}, + {{0x0000a32458403bc2,0x00082b711c5a4e92,0x000be5918eedb420,0x0007560f86f6b115,0x00009eef65546fcf}, {0x00081bcf351ff9e7,0x00039fdb3c33d733,0x0000221a9d2cfe38,0x0009978c093d8420,0x00008091d026a7bf}}, + {{0x00007fce2951dfdb,0x0009b16e29e1fefc,0x000e24c61a1bc2dc,0x00064ec20795b0f7,0x00006867bc74d613}, {0x000ea7544ff9e1a7,0x000863bcedd14dac,0x000776fbd052c35f,0x000040b845e5e89f,0x00008d3631aa2f1a}}, + {{0x00078cb5cf75d160,0x00082adcd531e19f,0x0001f334ec5d686b,0x0007bcfc67e75339,0x000083292fd5cb3c}, {0x000ca0f2c79e4395,0x000d7220b49dc04c,0x000d3bf945b11bd7,0x000dced7c2b83ea6,0x00005baf7714ff68}}, + {{0x0003508c06e840d9,0x000135b5a2b19028,0x000ee969d63b1a38,0x000161e84ec50423,0x0000f2fa14216578}, {0x0004bfe086722bf4,0x0003cf085472f45e,0x000dfda75e905f33,0x00079bdd9d624bac,0x0000fae3603f75ba}}, + {{0x000ee7df45826065,0x000159449fe4f3af,0x0001d996e565d3d7,0x000672368612ab1d,0x0000b960776c83e6}, {0x000d3ecc1939a703,0x0009f6c9772241da,0x00098eef9abac2b5,0x00032adbadd70bae,0x000062ee5f76ad58}}, + {{0x000fd21287e5c826,0x0009b0b6759542e3,0x00065f7682b53c33,0x000431f1b099e8c5,0x0000215359e247e6}, {0x000ecefb66ae71b0,0x000c047b9bdf08b0,0x0005d0993282cf6b,0x0001c428f51699c8,0x00006ac5644d7483}}, + {{0x0008a6bf88a12bc8,0x000cdb7063092912,0x0001649711d448cf,0x000792826cbe6242,0x0000b333087bceb8}, {0x0006b67ab7ffab12,0x00025a16516d0184,0x000c86a31293dd6e,0x000a457b718e848d,0x00005e930921509d}}, + {{0x00045277b9938fd7,0x000e086e2c20dff9,0x0007dce3453814e0,0x000e253ca6ba6c61,0x0000f538b78fd25f}, {0x0008e6179f3b912f,0x00078b639a80001e,0x000414bf793b6324,0x000bb8e28cac8f71,0x000031ad39adcade}}, + {{0x000a465f76d43f56,0x00090150ef7afe7d,0x000b2300b65a46dc,0x000e4e184f9a0661,0x0000561c2fc130d9}, {0x0000a143683c7212,0x0009a56843b2c208,0x0005135d6f7b6c2b,0x0000a908bec8776a,0x000034f7ab83b724}}, + {{0x0008c05b480c3297,0x0005a07fc36b2bc3,0x00097924fce51403,0x00007eaa739b3ab6,0x0000265fbf04966b}, {0x0002950b0f740005,0x0008cf8940d2564d,0x0005fd0c6b2dbfd1,0x000c6ce9a6b44dbe,0x000084a9579b81e7}}, + {{0x000e916369db82e0,0x00020510f751afcf,0x000eed18f269af1a,0x00013c1ce9324e71,0x00008683d69fc1e4}, {0x0004e5889623baec,0x0009ba4018b294b8,0x000e6bfb3c0b24e6,0x000bb0651dbdf225,0x000027ab085ca68d}}, + {{0x000f09a4ab6c3aaa,0x000de01cb3ba63c4,0x000e95f1d210e7b4,0x000f3a8d12cf38b4,0x000018d1e9f0e6ad}, {0x0005dfe6cbc23716,0x000aa7708956bd51,0x000a734e02a987f9,0x000198b26b6caa0c,0x000066d2bdfa41f7}}, + {{0x0009b8808e884a94,0x00056a949ba89d55,0x0006dd0557bc07d6,0x000df3f286c34b23,0x0000010cc2c61658}, {0x0003c84869304a8b,0x000c4e02538c802a,0x00014310ac59c7a8,0x0004e213a3299cbe,0x00008091b8c7dd4c}}, + {{0x00066d7d54bb7037,0x0009fc78a9ea064d,0x000c8c718ecb6d4e,0x000b29b0b388df47,0x00005ce278ce6360}, {0x000615c03685384f,0x0001493d593abf47,0x0007bd22554fbd0f,0x000cda5b2fbacf9b,0x00000be25106f6db}}, + {{0x000db548aff3dd66,0x0009fc3e222a2257,0x00021af9576355f5,0x00038fa46dacfcf1,0x0000a39c02de3477}, {0x000e5949738ae55b,0x0002b82b325aa2c3,0x000316cfaab2be69,0x000c3155c811feef,0x000094001c4354f9}}, + {{0x000e52c880cebefd,0x000fbac37dc45285,0x00047d8233836931,0x000393a52c077d39,0x0000f2cef1dcb976}, {0x0004fff652069879,0x000f14bd96afc0f1,0x000d7647c5393c35,0x000b655fa3d54814,0x0000b31085f87723}}, + {{0x000c64f583e1f402,0x000473a19323b48d,0x000d2fce8f8f79be,0x000cd0f61edb557d,0x00009ba886a6747e}, {0x000f6589946073be,0x00002e0d1b415ede,0x000135675a16d9b8,0x0002cee38fd0b032,0x0000a1d1db71309c}}, + {{0x0008a92ead7f3bdd,0x000ea3397552948c,0x000df72673192c3a,0x0006ba2de5b32e77,0x00009c0d4111e5a8}, {0x000bd04137e03344,0x00089b4162fd10fb,0x000ebd666ce66159,0x00064d877840eaad,0x0000381e39594fa2}}, + {{0x00000628e098c674,0x0009cdcbf9ca0680,0x000c9f9fe4c5a3c1,0x0000f7b96570ef82,0x0000049c1cd347b3}, {0x000fb672fa7243a5,0x000c2b46489362bf,0x0009a950e91645d7,0x000e29f1b1a58485,0x000003188482ca65}}, + {{0x000e8a5a08e6c7c8,0x000d1896d6be3d81,0x000632895b101d4c,0x00031a37b4f35e11,0x00007da1f2d7abe7}, {0x000576beeab98952,0x000748d0ccd0c23e,0x000b5c50c67fca2a,0x000936a558e370e5,0x0000985eec700203}}, + {{0x00037f6ba08ad439,0x0006926c8ad8398f,0x000df38a044b9452,0x0009463c48a42ddf,0x0000a860dd55a6b4}, {0x000d162812b222ed,0x000fd854816b2dad,0x000e65262e4eb660,0x0005a5b527b029ea,0x0000175bad33bdec}}, + {{0x0002c3da8cbbe229,0x000aa90abc735562,0x000bb0db89d0e690,0x0003454c1c36d6e8,0x000036f4e1df9931}, {0x0000ad2fb609c4b6,0x000745f01f277c0a,0x0004f07d4ce8634e,0x0007939a3356bbf7,0x00009fa9bbd0303c}}, + {{0x000834bdba6935df,0x00068b5bf0655c17,0x00009c8960d66e80,0x000b3b6460148a4f,0x0000f945f0f28aea}, {0x00008c5ad9f47c68,0x00079e9a1df4281a,0x000eb5d36cee9dce,0x000d177605462f15,0x0000bdcbfae25681}}, + {{0x0004c6b9e03db771,0x0001da5fbc75f102,0x000950b5e2ab475f,0x00073cbe4e444a5a,0x0000bd5871471640}, {0x000d74a1ca5ad868,0x00048ec67d0c8551,0x00000d3328df82c8,0x000e5f6f20f8912a,0x00002e0fe080c362}}, + {{0x0001c3d6d6d5d264,0x000b218027996557,0x0001970f15dafe9f,0x0000511096cf81f4,0x0000c9526de03ea1}, {0x0001e30dad4e3f37,0x0003aaa14b7782f7,0x0000539fa89532b3,0x000cc848b2ec61a4,0x00002b6f09d9ff2f}}, + {{0x000dfdd25cf29291,0x000dda0092de30ae,0x0000a4e132261624,0x0007de7dd4994fc6,0x00001c40532fba17}, {0x00018feb438b5b0b,0x0003e95815fe08be,0x000948087b006e4d,0x0005efbc2a9ef7a4,0x000009e6d587a5d6}}, + {{0x000969e4f9206d0f,0x00075a33642fbb5e,0x0001c4b5d8ec63ca,0x000a4b54ef559a14,0x00009b489e45d4b8}, {0x00079fd87810f9e3,0x00005eec7620bd39,0x000e3eca2d32cf90,0x000386a4932bc4fd,0x000080d2dfd12b4c}}, + {{0x0002bab6b693b018,0x0005396af6ef38e8,0x000153f766a56bc4,0x00059f4479f1c4a8,0x000065854adf0b46}, {0x000ca7e73f59716e,0x0006eb462570ba76,0x0001e66b58a3ff32,0x0007e313f7e7d714,0x000075a290729d59}}, + {{0x00062bf29147d07f,0x000b71338d29fb95,0x000a83b78c68c915,0x0008e5f7bbf7376c,0x00002d08e6ebdc2b}, {0x000022479e069a78,0x000529a2b2914757,0x00012616758d3d5f,0x00000b75f51a9a10,0x00005ca72d9581a1}}, + {{0x00056185d307f683,0x000a74cb99c3929c,0x000b06852fc4967f,0x00067199709504e7,0x000075f08dbc7f90}, {0x0007d8f5aec63809,0x00005c9d1e1a8854,0x000031b69a7b3391,0x000d26bf93901560,0x00004fd890aed914}}, + {{0x000e962e74fa42b4,0x0008b6eef166545e,0x000872fa7032fb4a,0x000897ad192d20e0,0x000094230f24ef1d}, {0x00071573ec958b1f,0x000ed4a294849e47,0x000b5bb10b0cb9ef,0x000c1bd82e65ac22,0x0000021aaa1f3222}}, + {{0x0003cf15c474b96e,0x0009e70fb75491a6,0x00055261629d3ef4,0x000ccabb565167c7,0x0000cc5624a31510}, {0x000841e7e25d97e2,0x000c7e403d770905,0x000be67912255f30,0x000c36beb19eb409,0x000000ba1f7537e3}}, + {{0x00022bdb04b10d27,0x000bf434239c52db,0x0008d4498f41ccfa,0x00035c025e14818a,0x0000e4cd009815c9}, {0x000292bfc4a245f8,0x00097364e9de7d60,0x000cb48c1de328ef,0x0007e4db08fe9d56,0x0000d6112b78e540}}, + {{0x000831ddebf9598e,0x000979bb92d0eac8,0x0000cfca46f5cab4,0x000b6015ad2f7772,0x0000d41fe36ef39c}, {0x0003a2ea9b69dc25,0x00042dff273b08cb,0x000f81461d1ffb14,0x000a4c36404a0c68,0x0000bd4f3b25d7e1}}, + {{0x000b52447a59ca38,0x000412904d355b9b,0x0009d306c9363867,0x00067f6caa01f11d,0x0000041bf277bf17}, {0x0009fef689c4169e,0x000e3dc4bfdf9266,0x0004eceb55ee1f01,0x0002944c74b71d64,0x000046d3fb3b5264}}, + {{0x000d89c6b8902588,0x0004a43fc1846231,0x0001866a88efa1d1,0x000fa61d29d42923,0x00004af39ae87686}, {0x000be51bae53ac07,0x0008423ad27f8d1c,0x0003aa54acb70da7,0x0007b8f235bf292c,0x00000305b9d0943e}}, + {{0x000514fe38c4f7de,0x000b829401fdb149,0x000db76997df5622,0x000f807d9447624a,0x00005ac0b00fbb71}, {0x000727b2d9749d3e,0x0001289dda4509a3,0x000629016714c594,0x000780af9268e465,0x00007f6db4424a1f}}, + {{0x0002aae6462c8c00,0x000054a3be95cf37,0x000ef28c0b873e69,0x000af8ca81887394,0x0000225ad865d446}, {0x000ef44f58dd3666,0x000173b419047734,0x0004dbd8a69723d0,0x000254b9684a7eec,0x00008b1aa366d4f9}}, +}, +{ +/* digit=7 [{1,2,3,..,}]*([2^49]*G) */ + {{0x0001aeed239f4556,0x00049d4ddf57cac1,0x0001d2a819808334,0x00000b6a5d2670cb,0x00000f2e4aea4ac9}, {0x000c8f8ccc05367c,0x0002818771212a6d,0x00085497ca404a57,0x000d31fe2406ea3f,0x000066d75813b43b}}, + {{0x000510a53ce5c73d,0x000fc461cc13581a,0x000c24ae8baba850,0x000a7dee213f3d39,0x0000f50e637950f1}, {0x000bbff69e5103db,0x000a85cb5f3beadb,0x000a70f73cee5aaa,0x000c947b2cc95a87,0x0000ba0e771ec061}}, + {{0x00040300d97bfe32,0x0006a033e7779a90,0x0006d553f95b3008,0x0007b47c5e9b61ac,0x0000021b54e152ca}, {0x000c553aa7a3a110,0x00046dd1c7ec0c1e,0x000577c34df1edd3,0x000d7165340112a0,0x0000d5fa7bfd90cc}}, + {{0x000f022a7558380e,0x0001ba7b90f5fdee,0x000e0da2bcb36cce,0x0007745bd4959204,0x0000dd2eac35ad01}, {0x0002c8521c0c5af3,0x0003ad91885450a0,0x000fb8b9f94c617a,0x0001771d86ee2b64,0x000058a7459f4852}}, + {{0x00087c696cede771,0x000a6d45b0f224af,0x000936b54c8b1b48,0x0008998378d19198,0x000043233bf94432}, {0x000fa637bed26bac,0x0005c18a17d7bb9a,0x00086ab22583df2c,0x000cd022a04cdac7,0x000073f0dad5030b}}, + {{0x0004aef507316508,0x0005e17245c6d9b5,0x000bfb309240aa80,0x000b0fce93d46ee3,0x0000cb28ae82deb4}, {0x00028ce7431d4cf7,0x000e6de34e2fd644,0x00027a4a239496d1,0x0005695b9f53f16f,0x000026a63f905548}}, + {{0x000eeda91a8f07be,0x00010b8144a4e27f,0x0005df4bc4b7698c,0x000c15c892204fe5,0x0000de3f93a307f0}, {0x000af35ec993e2fe,0x0004c71f0d310b53,0x000194dcb0a3be66,0x00024460840d4b9a,0x0000015279dfb876}}, + {{0x000dc97176f5fc32,0x00084c41f3ea9f74,0x000fcdfe91636818,0x000109e6007a66ed,0x0000d9337c74353b}, {0x0000aa28be36eb52,0x000f0a17aefcc5c9,0x00073205c1bdaa3f,0x000aa1cff6aa02da,0x0000e49689d28f4f}}, + {{0x000568b08146d249,0x00059da996f8b53d,0x000b5c2a3f70a11c,0x0005b73182b847ef,0x0000b7cbc8379d11}, {0x0001c6a0bb8a4a9a,0x000842be45b2e521,0x00032433b189ace9,0x000b9407f99f918b,0x000034002e73cc09}}, + {{0x0002b2c99ae289bb,0x0005a8aa4e3f5fdd,0x00024d4d0bab6201,0x00089e755c5401ca,0x0000536b9ff65767}, {0x000b91bfbcc62323,0x0004d65e1912fdcd,0x0003cb2f76182d10,0x0009c80f7ce8e668,0x0000cec3a67b10da}}, + {{0x000a8bf6d5778d03,0x000fbd801fe4d51f,0x00093d2ccba1f75b,0x000c28520bd7ebc8,0x00004d0d1b357aba}, {0x000f08f86aeb0018,0x000603f6c973af40,0x0004f47cd1006667,0x00016f77b47974d8,0x000047e82a24dcfe}}, + {{0x0005d1ddd12aa937,0x000401ffeda864d3,0x000e540c0684a675,0x000cbc4f9b990931,0x0000df6e60528b00}, {0x0001f5e0ad0b56b8,0x0006c70bafd39118,0x00050a6ae9d54b72,0x0004803e3bbc4682,0x0000b9184d9f5217}}, + {{0x000608dde1c5dff6,0x000b8a3467306f98,0x000e8e6eacb62013,0x000117376a795382,0x00002bc99d4c1ce7}, {0x000805d31cafc6c2,0x000ec33f2e2c8ccc,0x0007d9e547cce509,0x00073b8f1902fa58,0x0000c31805d21dfa}}, + {{0x000140b741887605,0x000490cd29bd1c0f,0x0007f308ee8d7d25,0x000275aecba30f74,0x0000780fa9561b0c}, {0x0007bfafab75c4b5,0x000f2bd8d7c718e3,0x00065faa7126ee22,0x00084992c17bc317,0x0000c7fdce2c0389}}, + {{0x0006e725b1a6606d,0x000e6576abca1b85,0x0003e60b5a32ce43,0x000b5e491da7eb02,0x0000f4529d1fd659}, {0x000f1cd618e6a1c1,0x0003d3b5f111ef6f,0x000aee2dc2902aa9,0x000819dbfeafe73b,0x00003af3de73bc78}}, + {{0x0008ffd479c62248,0x000654fd6beb4d68,0x000ebb327cc04add,0x00075d822732ea65,0x000044a14f2a186a}, {0x0002035bf8fb9958,0x00026d9f7591361a,0x000e0a6a62a3e29f,0x00023597cfb7041b,0x00007ddf6f52755e}}, + {{0x0001437de25e2c58,0x000a749e6e860bd4,0x0004848f10153cd7,0x00021532257cb78b,0x000082b7e982d724}, {0x000a791c09ebadfa,0x0009cc48599d4809,0x0007f5347df19150,0x0003790eb5e46213,0x0000eda7a00a335a}}, + {{0x000cb72c3119f7b8,0x0006335b4e574c18,0x000df0fee556561f,0x000efa728a3e48e9,0x00001de12a3aa369}, {0x0007bd62de5ff0d8,0x0009cb8e1debaebc,0x0000eeafa40c9382,0x00041ab5aef3f42a,0x0000f23bb1d172a9}}, + {{0x000007ee68dddd09,0x00041db7068d3bd9,0x000c45236df97f5e,0x000b27fec3b2979c,0x0000781fbd0bf529}, {0x00048429ebf0462b,0x00018b77dddaa659,0x00040b96a20bdeb6,0x0009b32be3321f34,0x0000677d50046519}}, + {{0x00093f5b430ae985,0x00063b607652d48b,0x0004cb9ee7647460,0x0008bed34ed097f7,0x000006e6a9aa175f}, {0x000dc50978b6e14f,0x0001b2110553d449,0x0005ca3c3412890a,0x0005f156e99a5a79,0x000027e794aaa77a}}, + {{0x000337bb4674f974,0x000faaf40bb50fe2,0x0001da97103f3a93,0x000f86ae5da89eac,0x00000d28934fa09e}, {0x000fe143abd373e0,0x0004d19798f1348e,0x0005faddb07be1f3,0x0000fce18709f123,0x0000a65d7f258bb4}}, + {{0x000ec2ea13bf8a9f,0x000fff94d1c2fc03,0x0004a5e758d72c01,0x000c2538fcb23537,0x0000519f25ad891e}, {0x000031644a896ea4,0x00095a8473f8d465,0x000e93c5c4dacb49,0x000ebe7a4d3ac4e1,0x00006a7515c98397}}, + {{0x00003531ef7c3513,0x000151a1adfcb1a2,0x0000fe90dea9dc2c,0x0003b8148c62d66d,0x0000b432f7beeb90}, {0x0009b4a3a3eb7360,0x000233afcef8ad73,0x000553d8625eca21,0x000dda60321e643d,0x00001b01349d116e}}, + {{0x00018c3569e6b0ee,0x00024b0d2fb5954c,0x000a0ecb65c4006a,0x000a62dc0f497e6e,0x0000b96b836c3a3f}, {0x00010ca5b00f3cf2,0x000d5a4d44403cb8,0x000ca83b4a082049,0x00032849181d8221,0x00007f7edd74f65f}}, + {{0x00010a1af882102e,0x0002f1c8b66b8d50,0x000cb8b5c86b5ba5,0x000aa20f54bd38af,0x00001096c5fa2608}, {0x00026d294ccad71f,0x000702638eb68c00,0x000abeba69f87807,0x0007937a69ba5105,0x00008964bbd724b0}}, + {{0x000ea9ea2bf0fdf7,0x000eb49d812be215,0x0005067fc7c83d3f,0x000764be2c803ded,0x00001fb9b50ab062}, {0x0006b70932d37f40,0x000d09395b17d0e8,0x0003a6022e7f0771,0x0000b27ee57052e5,0x0000c6cf1f16992b}}, + {{0x000add9e8bef59b2,0x000614c0c4be9be8,0x000e8fd1ff296e27,0x00093497783204fe,0x0000741476659093}, {0x000e8eb504b04abb,0x0001bcf379fb506d,0x000b9b91e1361604,0x00058b7cb22333c0,0x00002497bd48c840}}, + {{0x0004e4dfa0cd999d,0x0008b8d7ba3d9c29,0x000f6ce974ea95f3,0x000c95eb8a425f3a,0x0000f716a99f71ea}, {0x000edb51e52dc87b,0x000b25a37471e30f,0x000e4f019949d742,0x0001d40be1fa0a07,0x0000f2656c3c690d}}, + {{0x000f1a6bbf53c9fe,0x00054f2c7694ab45,0x0003232eb20d9b90,0x0006e109a9e361f4,0x0000bc91dee0ee17}, {0x000789bdbed0ccfa,0x0008288b3f1bee5f,0x00028f5d1ef0caa0,0x000ca42aeac3f01c,0x00006007b363bf84}}, + {{0x0006f0a64f93a622,0x000b3a5637862d42,0x000380c6ee69a504,0x000e6fa7057de947,0x0000433d4e4199f5}, {0x00023cf1f314c175,0x0009f1ff15bc3feb,0x0009fc8449252302,0x0000e595c6fae70e,0x00004481bce01d84}}, + {{0x000f51b3f1021673,0x00053d64db91443d,0x0008e54c650a1225,0x000220d157dc4221,0x0000a3341db93f5c}, {0x000f3d7a5bd331cf,0x00098ff4240a45b9,0x0007ceab2fcf47cf,0x00096f888d766253,0x0000ef3c60ed0577}}, + {{0x00097e010ff08ddf,0x000968240b00767f,0x0008602279b7acee,0x00099242875c872d,0x000019a7093413c4}, {0x00062417e2d75285,0x0005d70899ba18c1,0x00020ca0eb040adf,0x000bc187c981b514,0x00008b8cf4318d4f}}, + {{0x00080a709a32342a,0x0002965305d1f38d,0x000bf1feb4d161e5,0x0008904dcd385285,0x00009237da3aa1a2}, {0x000aa5a46cea858c,0x0009ce6d6f2234ac,0x000c28dcf76092ac,0x000e838a0ac0bd91,0x000030791072687a}}, + {{0x000accda4225c342,0x00029636e0fff56b,0x00021ccb1fb397b9,0x000dc741822beffb,0x0000d48cbd35cefc}, {0x0006d5bfe9ba3d6a,0x000225360b6d5563,0x000741b6001dfac4,0x0009b486b1b23b18,0x0000ef14d08ff4e9}}, + {{0x000f90e78c1f3bb0,0x0009b6f85de2028b,0x00037d91cf6b7783,0x000a3f03a27b6beb,0x000069ac8d736c15}, {0x0000c62e806bd30b,0x000056746a7ab9f6,0x0007ab1b6c245999,0x0000fe72037ecf0c,0x00008ca60c732f21}}, + {{0x000abe30741481cf,0x00040b18ce478cd1,0x000e7f8c34c982fe,0x000f0fe79cffb728,0x000074960aec39c4}, {0x00066aa9050b893e,0x0000122edd8925c8,0x00092c673fe40d52,0x0004b39f8c9033a8,0x0000ce1891820586}}, + {{0x00005206bf1ee33b,0x0000e4979d7a9a80,0x0009b650b8c12ac8,0x00054e4da9f88d79,0x00008aec7e264a80}, {0x000ae68f7b28d79c,0x000387cef16ca493,0x00089b6b0a4dc3e4,0x0003e40509431dcd,0x0000bbe744fec78a}}, + {{0x0005978ad410ef01,0x00014cc3515d61f0,0x000a46f70afcb4b4,0x00041970a01aa3fe,0x0000884ec45da55a}, {0x00063b35165dbca5,0x000cbc71ac33d0cd,0x000e4c38ef210420,0x000cf3a0f23b9fb7,0x00008858d7174d5f}}, + {{0x0000478139115bed,0x00093f433881438c,0x000447a7d8813dc2,0x0001188cb8732df2,0x0000f920508aa0d1}, {0x000bdd9b8248baca,0x00051a0fda74f21a,0x0004f3bd406ce881,0x000c164428836dd4,0x00000a2424cb008e}}, + {{0x00089b69fbbbca39,0x000b1669463edd19,0x000ccb304c3b438d,0x000a7733db961ef9,0x00006f2d432afb6d}, {0x000acc1b78bef59c,0x0000abcdba630ae8,0x000bcad61d6beb51,0x000b213bab1c81a0,0x0000f6c677215aa6}}, + {{0x000314509f77691d,0x000070083e4fdae3,0x000d7f1f5feb96d3,0x000a7d9f0c18c1cf,0x00001b7ddfe0651f}, {0x000db98fcc9de7a6,0x00069a529dcdc450,0x00097718843d4122,0x000d3e0ad913fe75,0x000005594f0670c8}}, + {{0x000e4506cab3068a,0x000d1d2494770a89,0x00030ec9b9955748,0x000846ab8cbf8946,0x00007c89daf9778a}, {0x000acec4f4606444,0x00078c364d73231b,0x000fe41c31286866,0x0005d3f1ce47c3d8,0x0000e365480c68c0}}, + {{0x0003e4160b45a121,0x0003baef5d6683e7,0x0004dd85a1e42102,0x00071f5e34c234f5,0x000091f9fbced850}, {0x0007767771328099,0x000d49be9c6b664d,0x000a38f38d829f06,0x000ec330eedef076,0x000026c48800340e}}, + {{0x000736dd0e1d6e43,0x0005022facfe3f4c,0x000191340b7da1b5,0x000d914f559d6ff2,0x00004e994ce30fd5}, {0x0002295702306bc8,0x00057ed0842bb003,0x0002112cacc1577d,0x000b4491fdafe49f,0x0000ad39b895860d}}, + {{0x0007eec37bfa3de4,0x000b16863791d4ec,0x00079f652f765dc5,0x000986d0c1bbc9c3,0x00002abf602df07d}, {0x0001e932edcf3171,0x000edb53f5fb229b,0x0009b60fc290f0be,0x00091a4acbc35e94,0x0000ba30ad68659e}}, + {{0x000dd8d9660b26f8,0x000350ca1223a663,0x00032ab3d4eba231,0x0002804991c779fe,0x000099f830e34a06}, {0x0000e71d3111ffed,0x000005ce14589dcb,0x000f365f377370cd,0x000623a24a8ccbf5,0x0000059fbf358ace}}, + {{0x0003a4098bbc3934,0x00004dcd090a5572,0x0005a7255ffc94fe,0x000185dfe7e5e6b6,0x0000bee85029ef3f}, {0x0002743db2177911,0x000d5e6bd9ce4510,0x000caa6d30f32aac,0x000f4f7b671a2e7f,0x00004304bbc2b68c}}, + {{0x000a04d9e64e4d35,0x0002ec1e1c9b0135,0x0000551455e12027,0x0002be716a3c508b,0x000085f4f1c5deaa}, {0x000fae6e86f4a15b,0x000864a7631005d0,0x000975559b3bb56e,0x000c45a95a184010,0x000086bd840e2d14}}, + {{0x0000c3fb351ec55d,0x000a716ba8612e75,0x000373c466979ea0,0x00032b4b9a776a63,0x00005a8b35671cd3}, {0x0002093dd7493ce9,0x00006ec41ba2b630,0x00026128f18e92b4,0x0006d1dd3049143e,0x00008405ec34615a}}, + {{0x0005e208a0356f29,0x000e456a9eac7617,0x0001f6a6f856d9b0,0x0008e436387a6c14,0x0000127cd283e4d8}, {0x00000f6a32221562,0x00012f58a6e1e3b9,0x000372ba690134c2,0x0000101938ae6152,0x0000a86d5af405d0}}, + {{0x000699670a64b1ab,0x0001fbf935745441,0x0002aba2b5b797e6,0x0007c9bbf9463769,0x0000dba14e270cb8}, {0x00013412230852bc,0x00022cfc37cc9c35,0x0006bb6af3ec8fc8,0x000ff9c5283e7b80,0x0000a3777b633ffa}}, + {{0x0004f15ce0554be8,0x000ecfa3ddcbe763,0x00055667be6039ae,0x0001a5485efcc93e,0x0000e042ceb13077}, {0x000e81d073599e34,0x000d6585f158bd8f,0x000f7f92cec7e3c7,0x000d61a31bb59a5b,0x000045f686d482c7}}, + {{0x0004eda272cf63fa,0x000636f1a9b8a218,0x000e1b6bb618ea24,0x000da926f4254f89,0x0000afeb6d402035}, {0x000c73357c46ecb5,0x000afe899f44ab86,0x0006183a62887e7c,0x0002c32e74795194,0x00001a51e3704cb7}}, + {{0x00044bb9b131bcf7,0x0006417f905f29da,0x000271aea17d6518,0x000225822f44b473,0x0000442a5c8529e6}, {0x00050fa2d6e447c8,0x0000dfcafbd2ec61,0x000f4ccd550401ca,0x0006a0f1350b7a4f,0x00001ec4502e2e21}}, + {{0x000bd15bf9662113,0x000335bcb1799bcd,0x00031ca94fd47034,0x000e1949d595cf1e,0x00001a8e934c2d2e}, {0x00034f38faa8ec78,0x000cbe16f46d9dc4,0x000bae7880239684,0x000b553e8a39b8ef,0x0000f6b6e14a3ec7}}, + {{0x000b937dc23032aa,0x0003d9c256e1f314,0x0009f15c9e65fc03,0x00090371034ea18b,0x00003c9a3c47dcaf}, {0x000484226b0c9336,0x0005daf367bd47e5,0x000896a1e101650b,0x00011d3fe8f106a5,0x000030553fa62618}}, + {{0x000620f5d69edeec,0x000d7a4eec8de791,0x000287a787907f50,0x0005ea68c4827aaa,0x00001a26fe08385c}, {0x0003a8e0e67d376e,0x000c9cb8e62ecdd8,0x0006423b3147defe,0x000bab0d5392e6ea,0x000093a79e3c20b0}}, + {{0x000e6ca8c201225a,0x000d361ddc9c9290,0x000842db93584608,0x000de2a6a6a74023,0x000052a66d55df1d}, {0x000ae8c9463e80db,0x000b580b54c73462,0x000d39a08482e062,0x000d4f4e9168d66b,0x0000432fa82c4408}}, + {{0x000cd6a07c166baa,0x000d0aab204c73a5,0x000f172b51bb0308,0x0000f224f298465e,0x0000bcb5fbbe0394}, {0x0003887f31da1be4,0x000fab42c7ea084b,0x000288810250c3d3,0x000243da8e6bf318,0x00006e11ea969a39}}, + {{0x00042beab44de79b,0x000d1bdf4e1ef56f,0x00092d95c84dc218,0x000ac84ec350fbab,0x000078c07677a9e8}, {0x0004fbd04103d16e,0x0008505240255fd8,0x000f4c2e83bf1669,0x0000f248fbf6016d,0x00003b3a59853adf}}, + {{0x0004d0bc9d68c082,0x00028f23db2991b1,0x0003cc0616167f9d,0x000e770931c7b4b1,0x0000da5ad5d2e0b8}, {0x000d6695dfec5d44,0x000981be910db292,0x000c7ac75239d091,0x000574e433511309,0x000082d8c29306e3}}, + {{0x0002be7195675921,0x000957bf6119ef73,0x000d96cb0c4d5c87,0x0006f8fa6a706d42,0x000021e1cb73d95a}, {0x00093b14fe798a38,0x000ad3f85e91623b,0x00033662ce9ac003,0x000caedbca9b36c8,0x000092ff41dd7e0c}}, + {{0x00076b51b75fec0d,0x000340db61af3e02,0x000abd899caf72bb,0x000472cd7bd7ae60,0x00007641cac436d9}, {0x000f1a178cf6f1cc,0x000000d23b7cffb0,0x000f77b796e5b6e4,0x000f33328af949d5,0x000010f779508865}}, + {{0x000eb802f32faab8,0x000060554f3b532e,0x0005ecce4b511538,0x000343924e7bc29d,0x0000c15966b6c62f}, {0x000f12a31d85149e,0x000702536a1b1333,0x0006deae7e597135,0x000a1cb884c1d3a3,0x00008b957c6d3bc7}}, +}, +{ +/* digit=8 [{1,2,3,..,}]*([2^56]*G) */ + {{0x000a0db802ad8e2a,0x000198c71fc20913,0x0001af4b76762aa1,0x000d0991a59e2221,0x0000724ce9039cad}, {0x000bf205e5876d57,0x000c736f68df81c5,0x000ab6ba1fb2636b,0x000ffe34868e7098,0x0000261df12d1037}}, + {{0x0003fb0efa0b1ffd,0x000a3fc568975624,0x000bdb4591f15f6b,0x000439654edf2cac,0x000003491ab55faf}, {0x000f3c88c630e16f,0x000e7a922cea29a0,0x00036b058547f279,0x00065d2986844536,0x0000159df225491f}}, + {{0x00016886aba2e444,0x0002ff5dc5081177,0x000694e142d2637f,0x000aae0245602bd1,0x000056094e9d2d91}, {0x000aac623d3970d5,0x000712ba9937c039,0x0003188e393e6d1a,0x000724cb6f2c4b0b,0x000051c7142d857d}}, + {{0x000a6c4126387d4f,0x0000d207fd54b018,0x0002d2c2fa82305e,0x0005af7f76e4d82a,0x00007991440733c6}, {0x0008b1ad2efa1bdb,0x000aa0154ec58403,0x000b501e885ff98a,0x000882de171aa384,0x00007914f6044142}}, + {{0x000ab01c6935b4f9,0x00003b6de76d230d,0x0004d5a6213a18df,0x0007963145617526,0x0000823e1682c1e6}, {0x0005e79ac9b872f3,0x0003ce7021be0c18,0x000c686947c500e0,0x0000a6319f616513,0x000045678a89295e}}, + {{0x0000c88440eaa722,0x000e347df565ba08,0x0000930155047bb6,0x0009d6475a78682d,0x0000591de4ffe082}, {0x000f7732b9087359,0x0001861be64be6ff,0x0005a48107286612,0x000bbfcd5daff943,0x0000410a05140a8b}}, + {{0x0003711d4a4e4fe6,0x0003fe30c63f29e9,0x000a2de989dabddd,0x000323866f1580ec,0x0000b9de281a175a}, {0x00007a398f7e9455,0x000debaed06f5308,0x0001392b17b31e3a,0x000de3ce5bf10e00,0x0000f2a8f37e8d28}}, + {{0x000eef460cea0d86,0x00080c089ea3e5c6,0x000d79ed4ba91ad4,0x000b168d4eb1f87a,0x00004c8dd11d9a52}, {0x0007631e210a2732,0x000522f1cfd893f3,0x000c313c40f24058,0x000538b4a53f3f1b,0x0000805ea5ac03cd}}, + {{0x000f2b4d9647871e,0x000698c6a70f2bf6,0x0000618fa5bbc040,0x000f35bdaacafdb1,0x00004d56a93f6a15}, {0x0002747db37534d4,0x000940549a7cacd3,0x0006eb9286c071e0,0x00086eb8426639ff,0x0000fb0a02399634}}, + {{0x000263cc8597555e,0x0000bcd376a4bf19,0x00078d8efc8cd230,0x000ecce815e792d7,0x0000672dbeb03db9}, {0x000ca483874e7dcc,0x00062391ae793e10,0x000b5a3cd1e05af9,0x00033f3361c5cc5e,0x00001ee94edad6df}}, + {{0x000e3a27b278aaf5,0x000acb57977526c7,0x0006911103ffb78e,0x000ed702ab4ad712,0x000051d4dd701893}, {0x00006061e1dd4f36,0x00024dbe375be15b,0x0000d19abf430996,0x00010fcc798197c9,0x0000dd790c374a6f}}, + {{0x000330e15cde34f8,0x000d285ca782df9c,0x00013e0f4ad9ca71,0x000b872d7e70e9f4,0x00009c965ed8166e}, {0x0006bfdf82982e7b,0x00029a67518e8d8e,0x000332d8a97d85f5,0x00001948637867f7,0x00005ff8016f7f28}}, + {{0x0004eddddb576a2e,0x000f99c20d415c51,0x00088bdcb245ef5e,0x000767739e179128,0x00007e8feed3b2d1}, {0x00060bf9c8d8bd74,0x0004fe343ddabcce,0x000534d9ae42b1a0,0x000cb78d550b1cb4,0x000062a4b409550d}}, + {{0x000c9aa9f5ec19fa,0x00036bab280e495f,0x000ec8135bd5fe5a,0x000257848b5c6e13,0x0000761ea009ba1c}, {0x000dd3c7a0d41127,0x000e4675e99a1705,0x00055926d841432c,0x000ebb9724b23770,0x000008204e2070fe}}, + {{0x0004bd37f61b0433,0x0007488b1baf6c6a,0x0001ff33dd611c9d,0x0003daff36d77f21,0x000012e12b486b0e}, {0x000d3d308ad3a01e,0x000cffbd52d205a0,0x000a3c9d1ddc263a,0x0008c49be3889951,0x0000c3cbb88dc5e4}}, + {{0x0008494d5ac38a60,0x000d1abd383b64d1,0x0008d5867de2630a,0x000e7c899c27a88d,0x0000cc4dc229cdf4}, {0x000e69248c51c51f,0x0001de98813939c0,0x000c9fc3540049af,0x0005acb02b92dd45,0x000015dbe915bf51}}, + {{0x00030100f6fd2798,0x000b5cc6b3f2de4c,0x0002cc9a1693c61a,0x0009d7cd8f059e65,0x00002cf62e1b8fd2}, {0x000b441e397cce3e,0x0004b8ee5b2a90b6,0x0001f1bf731ac8fe,0x00044622a0067d97,0x00007c2cb7412bf5}}, + {{0x0005ded4da08c914,0x000c57bc50d57705,0x000c17c0e5e74346,0x000674093e88cce1,0x00009d958b500cee}, {0x00011fd4c2ef91c2,0x0003d68c5fc39edf,0x000a3b166dbadb86,0x000c83e5a523a4dd,0x0000e1e7d5670101}}, + {{0x000e3abd49188ad4,0x00088dffda2a80be,0x000f3b05945c192d,0x0000c7b93d42d6a9,0x000002b90dd3fd2b}, {0x000346bd29b792b4,0x000c03609852e2b4,0x0009c492438fc4c5,0x000f90a6d5d4b916,0x0000d555a212879b}}, + {{0x000fdb47bd87a424,0x0008b5138112db69,0x0008d1d2f115354e,0x0005bc50db48e3da,0x0000aac2eb70e66f}, {0x00051bc6aba63664,0x000e262acc9ce72b,0x000438ba3085117e,0x000822be56bb230f,0x00006d5cca79124d}}, + {{0x00053f5fe3dc39cc,0x000b414d5f67bdc8,0x00046b348c10dd5e,0x000770a76e02dbd6,0x000009c3240fcb02}, {0x0008fb52645c9dfe,0x000c18c396fe8d53,0x00092dad0baa2238,0x00018629d37c8b0c,0x0000c60386360546}}, + {{0x0004a8de81e01bee,0x0007f8321592e20c,0x0006931b55bb7970,0x00076794a7e10024,0x00005ebf25e72a07}, {0x000483a0f5a9edca,0x0002bb646d9e6312,0x000e37073230f82d,0x0002517b5dddff68,0x000029ce765a98e3}}, + {{0x0000ae6e544c6ea6,0x000b57855552f6d8,0x000b8a5e25ebab2d,0x0003d955707f9c00,0x00007b357c483d08}, {0x0005361eff79a535,0x000156e0268e623e,0x000996462124b446,0x000a24e231e90f3c,0x0000bd6b1f150e05}}, + {{0x00040b77de486a19,0x000cf6c03fc07914,0x00036c6bfd63554f,0x000429473dad2c7a,0x0000929e308049f7}, {0x00070a9c2bbc1283,0x000621dd51016272,0x000eed7cf3413478,0x00025a8e63501e29,0x00003512397dbb12}}, + {{0x0006891e7084f0f2,0x000fd6a38da9e393,0x000256ae79ed2f3d,0x000d874a6b4d7cd2,0x00007695b9534b5e}, {0x0006e9d024eb60ec,0x000c40f326d7afea,0x000852c01e0e4684,0x00071b37bae69e1e,0x0000a388523fa1cc}}, + {{0x0007b4d81320b2e9,0x00042b4678e1317e,0x00031c45fcb6ea5c,0x000c542ee5785311,0x00009b995cc107ba}, {0x000eaebcec18878a,0x000a9849b1c2e018,0x000ad5438bae1123,0x0007805e180c0e02,0x00002ce8e4160ee9}}, + {{0x0003c9f0bf831b63,0x000e9bbb471a25f2,0x000ed4ce91f1e7dd,0x0008472d76956795,0x00005e67d84a19be}, {0x000eab76f5a65259,0x00060239051663e7,0x000a9f4ea42d2626,0x000b8a344f8f5d2c,0x0000bb99d60d6793}}, + {{0x0008053bb8d37d24,0x0000355c03827c18,0x000a8454066ad23b,0x000f6d341b9405ea,0x00009fdcc57978a0}, {0x0004e26d8aafb17b,0x000c4ec79ef02ddf,0x00008c46fe8c88b8,0x00016777aec8ea64,0x00005ea6fe832ac5}}, + {{0x0003b361878c3d3d,0x0007921447678781,0x0009b469595fe31b,0x00055924b0bcc3b2,0x00003e4b82a2e59e}, {0x0007f61727308214,0x000569f22b76dbd2,0x000233c6e2a1b59a,0x000e7aa7cea941b6,0x0000a51665729aa9}}, + {{0x000cc4b553fbe60d,0x000b1cc8f5672bc8,0x0004471a809c494b,0x000e7ff3633cdd40,0x000094b3115ecf90}, {0x000e7e954727f93d,0x000b292aa6dd5242,0x000798c3d8b2f132,0x0009e8fae560b438,0x00009604a24394ff}}, + {{0x00074df40c36a876,0x0008139bcd46117f,0x000f04c1dd9ab144,0x0007151706edf335,0x0000ca29fddf0c9f}, {0x00077943d7b52cc2,0x0005759e0b8ef052,0x000c4fbae923b66a,0x000b4312d8827f47,0x0000b2cac97b4362}}, + {{0x000ba3b3d8c333e3,0x00084249ddfc6f82,0x000412b21bbeeda4,0x0003f2feb78ebe73,0x00000f4a98190b90}, {0x0002b13fcaefd68e,0x000db2497727d8a1,0x00050310717e05a5,0x0009b4f4db117595,0x00002bad33ee056c}}, + {{0x000aa899669a94d0,0x0001eab7a00c5660,0x000041a733d2c8c1,0x0009450c1e253b75,0x000017bf47981a80}, {0x000f809d4fd21a7d,0x000be6533241ae79,0x000d829dd7bdcd33,0x000be78a25e57321,0x0000c09f94cc338d}}, + {{0x000f8d0a468848c5,0x00047b41a21d9ca1,0x000bc0e5b17c3f9a,0x00029374333d5705,0x000030912604ccc2}, {0x00008d0d6ceeefc1,0x0002a8c72cc8005f,0x000e6cd79d355262,0x00085d0e66445e88,0x0000b6ec8a4e04c2}}, + {{0x000dbc54066477ec,0x0000a8c5bea950c1,0x000684204e2e4f7e,0x00086fee8955629f,0x00009b1f71580e9d}, {0x00087fc2c0029688,0x00027479146bdc72,0x000667694cbc6782,0x000b7b2f7af3baef,0x0000f7f90385b159}}, + {{0x000bb32253f3a82d,0x000b3a639c9cb63d,0x000360890138bc41,0x00057b610a116196,0x0000ab4e04b9e339}, {0x000ae727cbd17ddf,0x000a2b1a48784582,0x0004ee4100db6d3b,0x0008cb38cff9eb42,0x000029b0c8dd23a8}}, + {{0x0005ed5e3f62c22d,0x00032b06f482bc86,0x000b2a80097544a1,0x000c7f76ed319dc3,0x000088eb3caa8336}, {0x000408416a3468e2,0x000cfcaa5c73b399,0x0002ccb5869caecf,0x000834622f2f7c10,0x00003ebea2e23bae}}, + {{0x000f634159c3f145,0x0001a314fd75ceb9,0x0009e502cefadb01,0x00082007b8010960,0x00008a2bf184a15c}, {0x000886d3ec835a82,0x00004bf7840e310a,0x000d8eaf18c9f355,0x000c84d39c35be1c,0x00003fa5437b08b0}}, + {{0x000e7c76ba48af4d,0x000a39c3a01a8fd9,0x000c62e23ddb4853,0x000913a6ec0c12f0,0x0000b9479a5d516f}, {0x000b51d7e32dbba9,0x0001abd8aa7d25f0,0x000afffda4d3a475,0x000270e67e9ae172,0x0000c1fa7bf9b1fb}}, + {{0x000931c17cdd8a73,0x000e0448a021a8ba,0x000920e03df7fecf,0x0001bcbd374eea3f,0x0000c62edf80f11e}, {0x0001ce6736088569,0x0003c0fe97f45d51,0x000ddb622fb70fe3,0x00042189193abf63,0x0000bace900bcb83}}, + {{0x000f81a878a56754,0x0004f75c479fee4a,0x0000afb5829b49ce,0x0005854186889ece,0x000032530cbd047d}, {0x000be7c429c1cf6a,0x000cc541d81ed275,0x000d999296d29cc7,0x00011ced6e04b532,0x00007aeadda8e71a}}, + {{0x0001699a94bbbc79,0x0000b3ed33d3b98c,0x00012db7668dbab9,0x0004d7e3a0ec5fbf,0x00002de225c570ce}, {0x0005c39338ab6850,0x000f612c245ee97d,0x0000222a78aa03b1,0x0001e27e4e3af28d,0x0000e34817cbf8ba}}, + {{0x0006fd6e8a4a3f4a,0x00070c75489636df,0x0002579d316b0e9f,0x000fe12b5fb3bc1f,0x0000fe57d4f72a95}, {0x0009a194ceff9331,0x000354f75a487036,0x0004549b046aab24,0x00032749a77772fd,0x0000f44e63c940f8}}, + {{0x000f73f8f359b8e8,0x0003626aa63db40c,0x0000ebb81b7bdea9,0x000ce970626f7e55,0x0000c692abaa0eb5}, {0x0009cffcdb9d6804,0x000f4c9bb068b774,0x0004adbce122a50e,0x000b00be247e3384,0x00009320ea728f86}}, + {{0x000300cb47746a83,0x000d765cce2bcb1c,0x000afbb2ba8b05a6,0x000c89050d56aac6,0x0000f10deb3e6496}, {0x000f6adeb5e3c3ca,0x00005b59f7d73f44,0x00034333fd8fef09,0x000cccc06179b892,0x0000d42424834343}}, + {{0x000cd7b1178680b4,0x00067205cb9300e2,0x00022f86af9a63f5,0x000dce7ea9cb6c53,0x00004736f3842b23}, {0x00053bf6f5d17905,0x0001af4b062f5ef8,0x000c8e412a224d5f,0x00097adffffe0026,0x00003f676531b201}}, + {{0x00046ade65aa0b35,0x000ab957bcc5a7c6,0x00021fd3aa044a8d,0x0002d742145aae3e,0x0000d79efcd7c6e3}, {0x0006479ffb6a9280,0x000fcff2758345e5,0x000cd168eaebbf6e,0x0001d311cb1cf204,0x0000d5fd7233e385}}, + {{0x00021f9564df392e,0x0000cdfbb4120dea,0x000697ef6773fc9e,0x000b33b64b26b1f1,0x00002ed2ba1b7d83}, {0x00069e6145eab824,0x0008a8dbc68bf47f,0x0005827d6b1acd9f,0x0009476929807897,0x0000cb8cab70e2ce}}, + {{0x000ed39da3052f0f,0x000d683870bcdc6f,0x000c41480005e7ed,0x000791569848974f,0x0000beac60351f97}, {0x00001799275b950a,0x000212527d574202,0x000173d93889bee7,0x000bea8685758a76,0x0000cfa1096d830e}}, + {{0x0006dbc7d5bcd224,0x00049799a90fe43e,0x000bf8e13f9e8a5d,0x000a40e9cc8d918c,0x0000f177852be5ed}, {0x000b278ab0123dd5,0x00010994e23c910c,0x00023635812c11f7,0x0002ee5d1ffddf6d,0x00000572865c55b3}}, + {{0x0007eec59607df89,0x000f6fc041bea2df,0x00025c8a86ba453c,0x0000ffdb8cd4fc82,0x000014b0c0ae3f77}, {0x000cb43d27069607,0x0001c526ab3aa02c,0x000c9d06bf4b1f88,0x0001c5df41cbbb56,0x00004c4747afd9b5}}, + {{0x000a373fd41c7efb,0x00008eee150edc52,0x00072492cd833b3f,0x000e6300f3ccb31b,0x0000df8c0cc0bc7d}, {0x000a3eb385d47b1a,0x0009bb3430739063,0x0004ff4afb3a8a1d,0x000739f02eb25348,0x0000cc1efa44f33c}}, + {{0x0000f4a61e1db8d0,0x000240c3ad0a8e2b,0x00003e752f6d15cf,0x000f1a4cb501fcec,0x0000253b9aa06d6b}, {0x00012854f610f356,0x000b9e867e0b0f0d,0x000cb4fae8d8d727,0x000e686ca560b0d7,0x0000fb6839787c32}}, + {{0x0008c30ba2a7b38c,0x0002c05a1e186e3f,0x000aa6ac9caa3aff,0x0002b57828d97c22,0x0000aa7fc12f8eff}, {0x000efe970785dc5d,0x0009dc8818f3ac30,0x0009c0b682ad0746,0x0008e4675851d1cb,0x00008a8897cf8c31}}, + {{0x0001252f65cf5ee5,0x00088f29ac5a604d,0x000ac9bbce3667bf,0x0006a0c6475eb601,0x00004bbc154f2060}, {0x0008776fe257543c,0x000a239d34de6a98,0x0009a87768ce62b2,0x000fab0fe16c0d6f,0x0000926673d22d5d}}, + {{0x000b2b3ad94a7b97,0x0008fed58b128f18,0x0008946706b30053,0x000479259ca13ce2,0x00006d23f69d840e}, {0x00025e4177e33430,0x0001bec8e41e4120,0x000d93e910ab22df,0x0009ade579d0c842,0x0000e273816addac}}, + {{0x00002ccc6fe7dff9,0x0009b502353dbed7,0x000ac7e295c845f3,0x000f70849f73ac4d,0x0000cd4776c48c6e}, {0x000669075bd0861c,0x000bd7cfe206c5c1,0x000ed7294a8a2fda,0x000631c9abfc3e62,0x0000ac1761b8c654}}, + {{0x00056155c59e3804,0x000fc6e8eea72563,0x000b382c5c09c5a2,0x0005601fb0724e41,0x0000a8641a0b7d43}, {0x0006e955b7a680f5,0x0009f5c576494068,0x0003be93435b2e37,0x00051a6a679fa13c,0x0000a5238a3c9d8f}}, + {{0x000957c5dad3b707,0x0006a860fe3b104b,0x0003bcab2013b711,0x000909be66aeae78,0x000014befe48deb9}, {0x0009073742c9ed9e,0x000a54923d3e39f5,0x000340d1ddd320c1,0x000b66ed03354572,0x0000f7d8e0dc288b}}, + {{0x000a09a0d72bae2a,0x000846530c949b89,0x000fc27421739535,0x000c46467b80b4a2,0x000049fd2b8b03fa}, {0x00047f3041803387,0x000a220b9b0be858,0x00019bcc545b9dc6,0x000a8ba2614c8a67,0x0000f019df8137ad}}, + {{0x00091858054e09c5,0x000179f271fdc8ca,0x000544ccb6eaaa79,0x000a8115523b3f72,0x00000e9b891c122a}, {0x000c7af09e9c2711,0x000a1a2a3a281d16,0x000a7927fc4edb0f,0x000f2cb10b6c326f,0x0000487e43472560}}, + {{0x000c0c37e2cff914,0x0006f180c765eabb,0x0003ddc4575d45d5,0x0008a0dd271d5c6a,0x00006393dbeb0d29}, {0x0008b4d194b3cc34,0x00073b2327d37a43,0x00016dc625fb4e30,0x000d1937f15f8154,0x0000fbc9f81b7d07}}, + {{0x000db8bec3a5d60d,0x000e4aa12f4eec94,0x00084204e9449ca2,0x00024cefe7cfc0d6,0x00000c8fbdf94de6}, {0x000813864f28dbf2,0x0004ab156ec6f5ad,0x0006721654073837,0x000938a6d1b62c4a,0x000068b432494983}}, + {{0x0007e91a84283e66,0x000bd6aeeb6f7400,0x0008a39d7929a015,0x0005543ca7019543,0x00009dac0d6ff2a3}, {0x00049431bd9902dd,0x0008243c9a6f36ef,0x000314d8d81ff65b,0x000423dd900e14c6,0x00000a4a34591300}}, +}, +{ +/* digit=9 [{1,2,3,..,}]*([2^63]*G) */ + {{0x000383af5664e7e2,0x000b6de6b7968a12,0x0009f4c5338a7675,0x000c15b898bae977,0x0000425a4bff82b8}, {0x00042e3b2dda3220,0x000b8c25241d990c,0x000f6aea17aabb56,0x0005ec6348df82e9,0x00006ce9658aa8d7}}, + {{0x00002787f882d1b1,0x000092838067fd23,0x000e893587ab9dde,0x00080ecb05c101f9,0x0000374fdfb598bc}, {0x000b09697248eb9b,0x0007e7ad923f4c84,0x000109184ce9bdb8,0x00013b128cf5cebe,0x0000ec24ef3b3889}}, + {{0x000944e7099b0bd8,0x000b1d537ca60adf,0x0008486e846852bf,0x00068d48e47fc8c7,0x00000d6c7de2929b}, {0x000987a8e27182cf,0x00074222fc3b99be,0x0007d0d1d309bede,0x00052f53cb0aad2d,0x0000f65f3dd24bca}}, + {{0x00020afe823378bd,0x000482dc3af323ad,0x000605e6877ccdc1,0x000ae077795ffeb9,0x0000845b442d429d}, {0x000ff1016818e3da,0x00066b078681d060,0x00031508ce71189e,0x000e406443b1e66f,0x0000623d9847ce12}}, + {{0x000cd6f7fd3f5519,0x00031e2aad87484a,0x00089654d2767308,0x0009ab132e31f05e,0x00006d3deb114a4e}, {0x000838481a653456,0x00035a31e914dead,0x000e292969158f5a,0x00028101fed010c0,0x000004a96f918ab8}}, + {{0x000bdfb703ba17af,0x0002cc4c9c4a5470,0x000b10c4a0a49397,0x0009166a2be76243,0x00000e3312fbbc81}, {0x000e0d4441ca1098,0x0007c4cf80735a8e,0x000250bc8597d46c,0x000ae9d302662bb7,0x00000b98758a9150}}, + {{0x00029cb244f2a00b,0x00002a5ca85559c2,0x0003d6204ed00e65,0x0003736e24ccaea9,0x00004d64bcfaa3e4}, {0x000b99a1975140f1,0x00011452dcfbdbb7,0x0003320cd47e6913,0x0008d1132c93a100,0x0000c845f62ed4f9}}, + {{0x0007c03d7c90a692,0x000ff6139f403262,0x000a916341661a22,0x000a50f3c701c63f,0x0000fc4a23f87ab0}, {0x000307b826279712,0x00040ecc37944070,0x000d90e024f1cfb4,0x000986d684936aed,0x00008b59b0818a0a}}, + {{0x0003cc56e8a58cfb,0x000ef645720b7ad2,0x0002b007e30ba2a1,0x000aa9db14cb4a09,0x0000d1f68b371a1c}, {0x000e1ce22bb53b31,0x000e06d532c443f4,0x000a47f15e80c1ce,0x0009bcc885a253d1,0x000061f7667f031a}}, + {{0x0001ec094d82f598,0x000e76868c8fc9f8,0x0006eb6e2aa85998,0x000f863c0b776229,0x00009ebfbdd5d6cf}, {0x000c395d97adaf8f,0x0009b131054a3d02,0x000c908ef314a2c3,0x000dc1eabe4e52a9,0x0000074223caa8fa}}, + {{0x000960a997f80b8f,0x0002574eeafbe347,0x00063a35458a6961,0x000959c7e7364dc5,0x0000b4cf05fd40e6}, {0x00049a6133af3f0b,0x000e62fe5ab6cc23,0x000fab20cf050c2f,0x000bba8ca2cf4997,0x0000e6736055bfbc}}, + {{0x000ef812191ed798,0x000315e7a6df324a,0x0005ddb2427d462b,0x000068f036fe19e2,0x00006eb983703d74}, {0x0008114f4f93c566,0x000e92bd193d9efe,0x000ee2b5a701a0ba,0x00003dff40a1ad85,0x000032ba327506a3}}, + {{0x000e85754a60d6f8,0x00078cd09cea9b87,0x000d99e0a120c995,0x0002e2c46af84652,0x0000cdebf2e2b109}, {0x00059f3155bc92d0,0x0008b8beca561249,0x000b156131d692d6,0x000046cab8309442,0x0000e1db93bb8bc3}}, + {{0x00091e5ece80aac8,0x0004c050f1825938,0x0006020f5726a25c,0x000a35ec48344664,0x00007fd9893709dd}, {0x000c9bc10425f548,0x000d2a56bcbf5675,0x000da5191679c8a3,0x000303a7e6e78063,0x0000f14860aa1e8d}}, + {{0x000bc50407ea0a28,0x00025007431b93e1,0x00055d0ffd5cf6dd,0x000aa71a55e8c0ac,0x0000f7f913113bea}, {0x000075d17988cb54,0x000b0f10950926bc,0x000b82f37b6b32bf,0x000e90e2937113ba,0x000081507f1a3652}}, + {{0x0001af7dc8c1da8f,0x0006951c6c9cc65d,0x00013a9509e4508a,0x0005bfe1ccb78920,0x00009fd6a67c65bc}, {0x0007a0698fe8bfa1,0x0005330e47adbc82,0x0008c7bae1477953,0x000e2a4028e9aefe,0x0000e977aa9f875a}}, + {{0x0003cf64d646bbcd,0x0000731b0f3ca569,0x000d322aebf578d1,0x0003eed7bffd65bc,0x000052a965205742}, {0x000bf947dfca9551,0x00086f7f3c853793,0x0007560c93ea9354,0x000c6925cdcceabd,0x00006d6f1ca191c5}}, + {{0x0007f133182d35c5,0x0000458e49b6d3bf,0x000ddcb73784e7a6,0x000a6aac37769247,0x0000fb26f76fabcf}, {0x0001ec3853bf14c5,0x000e7aa2a072ab68,0x000d7447c7e088fe,0x0000ff4d93e78cd8,0x00009fe9f418ede6}}, + {{0x000bd19d394d62fe,0x0008b38a0d8f64ab,0x000d938764136c16,0x000f4e72039b4ef6,0x0000849491679644}, {0x0009ba557055c06f,0x000dc7bdb007a9d4,0x000ba1745af2aec2,0x00015963a29318d7,0x00004187f01e40d3}}, + {{0x0003790c271585da,0x000ed5ad056f666e,0x000ee31740b9f498,0x000f8992c64d1268,0x000057c0938d8ebd}, {0x000298e2374bea71,0x0009768daf973b0c,0x000a2d513284e305,0x000c9eaf3e9e5fe2,0x00001291b298bd83}}, + {{0x00007ba27d3f2f53,0x0003b984e0b60811,0x0008eb875a3de64e,0x0004b33536bb951e,0x00007b9dec8b54be}, {0x00097920a0321af4,0x00038ad145d78b4c,0x000398b33b4205a3,0x000662f72f46d835,0x000081db115da0a1}}, + {{0x000d732092e35463,0x000b86781f2b173c,0x0003cc924b3fdc2a,0x0000dd4011f8eec0,0x0000d8fea672cc8d}, {0x000be33d826c6f6a,0x000a55623182076b,0x000d30071e7711f0,0x0002af15f3579d49,0x00000e7e0ed76e1b}}, + {{0x000d7abf4bb64eed,0x000c77dcee5b655c,0x000413063595edf1,0x0008c03bde3f3ddd,0x00008473cd3eaabc}, {0x00009fca27519070,0x0001a00dfd0763f0,0x00073571029941ca,0x000cb885e258bba0,0x00006ca17e26473d}}, + {{0x000a5610e0e8924f,0x000ff65b0e50d4e0,0x000400c48de61d42,0x0003ab903b10a4ed,0x0000007b9332ca3a}, {0x000f94cac8105ccf,0x000774a0a98a0b7d,0x0006b6f26564134a,0x0001f89637e2a236,0x000050ac987f5af1}}, + {{0x000a8d78638737d8,0x000c7cba5d96bf5f,0x000baf6465e23c9b,0x000bce6625837e83,0x00001f7b1951beb4}, {0x000cfde3643f8760,0x0007e0b1a62f4c1b,0x0009f918acc64067,0x000b46863c488c82,0x00007eeb70136714}}, + {{0x00038b733005eaec,0x00069bc862a3754b,0x00082e20a6c7da3d,0x00057108fdec85db,0x000022986577100b}, {0x00050a08db1555eb,0x00042a9d5da8256d,0x000025309b01beb5,0x000ddfa119befeff,0x0000030fdcce76ca}}, + {{0x0005d057caacf707,0x0003d6856c44b3cf,0x000bb66402bb6e67,0x0005d6fcf6d21aa4,0x000073ffa1ebc9a8}, {0x0007b52ffeaf8928,0x000b35e49a37db24,0x00070c9914f1e14d,0x0001ec5174b1f793,0x000068dbdc6a0007}}, + {{0x0002077a6effe60c,0x000e9ae84ebb9b28,0x000a90020b39c7e9,0x000c72996fc825a4,0x00009d2dd8f89811}, {0x000de4719ce198b8,0x0002073bd8ffd842,0x00091284580c3d61,0x0001c29d754578eb,0x0000e5723eaf004a}}, + {{0x000ea43768534f8b,0x00051abc588d36b7,0x00095d1fc7b73df0,0x00059713790bd820,0x00009bab617eb6a4}, {0x000918138f1334ab,0x000287a8ecff86f3,0x0007dc27cafd58e0,0x00090f2bea43a4a5,0x0000876309bff8e0}}, + {{0x00040a23110d738d,0x000d4d107a0cd20e,0x000fdfb6a9c754af,0x000106b9828ccb2c,0x000019c605440704}, {0x00026be98b9211c5,0x000da44d15bd9119,0x000db2afb42c11b6,0x000ef7dcddaea218,0x000006a2d4b20f0a}}, + {{0x0001358309571482,0x0001812df972a300,0x000a8cab38d53d41,0x00080cc32e21acfa,0x0000e6a672ed9fd1}, {0x000d45c4e36faaca,0x000be60297f9cf33,0x00070c34df7f4d7f,0x00001dcb91ebe024,0x0000d911f768257a}}, + {{0x0007255a2e0c5dd5,0x000d22bbbf673853,0x000ab32207a670b8,0x0008fe78ebe9b99c,0x0000fdde390be12b}, {0x000115f1b1cf88b4,0x000fc8cd1c7f9323,0x00039787a00aa2bd,0x00052384bc65bc2a,0x0000b21156bb8e1d}}, + {{0x0007c6bedf5eb992,0x000ccfb71e9c6a08,0x000a23eae9cb1f72,0x0007b145f4b20c36,0x0000cdd6cd364c1c}, {0x0002b96009e153bf,0x0000791667fd4b16,0x0006d6b0e682b36d,0x000dce91c1e9f97c,0x0000b3acb50ee02e}}, + {{0x000ffd490f51cbae,0x00013a606f79d85e,0x0009bf4dbe0ae081,0x000c1de9ba256a47,0x000044c5bde85698}, {0x0003bc53c7a38530,0x000bb62a6e6e5c80,0x000103b8eb6cc532,0x000dc7a5791e202a,0x0000707bc1728e28}}, + {{0x0000b00ef08771e0,0x0001e0a13ee687b0,0x0001b533b57773fb,0x0009a5ccaa4259f9,0x0000b79966f77917}, {0x0003e85beb00e9ce,0x000df3ff0394ced9,0x000c6bf70673db5a,0x0003f1d646c1a702,0x00007ad3795a15d2}}, + {{0x0009e4bc97bc7da4,0x000771d5ff2a522f,0x000985f8205c96c1,0x000f2cc4eaadd196,0x00002c57d9e6d752}, {0x000d1929233b3620,0x000e75373d1e4ac7,0x000b30d44f4b9421,0x0007bf6bda441140,0x0000bfb154cbcad3}}, + {{0x00070a4205b81469,0x0009a9885929e3d4,0x000ad5364994946f,0x000dfb6f3fb7423e,0x000003c1097c2e3a}, {0x000dc4ea3c0c73f6,0x0000a781ef8a533b,0x0008fa9cfdfb0cba,0x0000a169994d9edc,0x00004caf3b9da044}}, + {{0x0007bff77c009156,0x000b45f526c72d31,0x0009ca544fe44325,0x000af0abe60485f3,0x0000d0555feb6700}, {0x0009417bc4ccc145,0x0008e6c415362ec1,0x000df5ddfbceb3a1,0x000947ada3efdb22,0x00000c675f6c474d}}, + {{0x0001b02ff106393d,0x0009ab788c98851e,0x0004e8fd1bb09653,0x000a43a2d782d5da,0x00008c9f428680c1}, {0x000ad17045780bde,0x0001b95cdcc517cc,0x00061b333987fc28,0x00047c5787ae170a,0x0000f427985ca04d}}, + {{0x0008761292146075,0x000a0cf3aec278b9,0x000329f90f12cd22,0x000d72cb478d1718,0x000082d33765d490}, {0x000d6264ebe63b58,0x000388509017bfc0,0x00049f0b28500032,0x00044eb2df475953,0x000023dc015fc527}}, + {{0x0001a099d4037afa,0x000c7d2a4f51d503,0x0000b909ab658722,0x0002f75dc1387e94,0x000054c7cceb78bd}, {0x00084d3f96fe1299,0x00029004be5e603f,0x0009034366d312d5,0x00042265d0b7585f,0x00003b342c682c11}}, + {{0x000abe562108741f,0x00090edafcb0db97,0x00050a1e77625189,0x000612928f4ef0b5,0x000028436f533db9}, {0x00066a445e95a984,0x00061b7395817080,0x0008a14b66b45ca8,0x000fda5f298d70b3,0x00009d4a0366568d}}, + {{0x00099ff0ea5b0fba,0x000aa38e73001b62,0x00067dc4c57987d3,0x0006d5467cc55ce3,0x000040e6da53b93b}, {0x0009e490c345dc3c,0x00054155af276568,0x0009f77de77c956b,0x00064f912443ecdb,0x000013d1f2e35d2f}}, + {{0x00069dc8827b81ba,0x000ff2e3d5a36319,0x000f13ca81af30b3,0x000d3db2c866e0bb,0x00001728b7fa6e1d}, {0x0006d11ba4e53cde,0x0008c3f44d47dc80,0x000de776c71aa9a6,0x000377d5cb45efd5,0x000001fc46a09355}}, + {{0x000b7f02f609bf48,0x000d851c333890b5,0x0009646d5de532e0,0x00050dd82571cd4d,0x0000381d38f402a4}, {0x000a13f8fe5abf74,0x000e980cfbe80cee,0x000d8244e4e987e1,0x0002547265bb3560,0x00003bb71799807c}}, + {{0x000711e9d0717f73,0x00098bcbfceb725e,0x000af2faf4e210f9,0x000aaffcf4b3dad3,0x00001d7b1680a432}, {0x000a9fc79b356741,0x0009b13d5198f7a8,0x000714c9efebfbdc,0x000e9183ee381471,0x00008113de151b1d}}, + {{0x000b068e8dac101f,0x00082a77a77808b1,0x0005fe83ee2d68db,0x0009cd82fcc849c8,0x0000ced2c5678a6b}, {0x0002eaa12ebbd351,0x0006fe07a1b126f4,0x000c1b353de97cf6,0x000303776ab40f94,0x000029ea20475cdb}}, + {{0x0001c2d1586be5b9,0x000cc911b89d6385,0x00075acf780e9258,0x0000604f7cd421d7,0x0000cadd878a2dad}, {0x000218369356e650,0x0001634ebb7f5e99,0x000cda3a8202c34c,0x000abf90720028f5,0x0000fd4e963d3750}}, + {{0x0000f0db1730827b,0x000d644fc900fdd6,0x00037410dd37db24,0x000309b2d883f43c,0x00003acdbdd5534d}, {0x000703eac7e1a004,0x00004f5796e43be6,0x0008988176129b8d,0x00003dd7ccc20dde,0x00003c8ca33f4853}}, + {{0x000d0c1c7047f195,0x0002fc7ed2afa267,0x00088562adc95dd6,0x000098dd5add39ac,0x000066c878035e61}, {0x0004fefb47537004,0x000a5a743ef55e47,0x00047591d972f249,0x0009adda486006ed,0x00007c2df25f0a81}}, + {{0x00056ba80ce1e1cc,0x000e67cf1faba833,0x00089e7a051528a4,0x000c6753125898cd,0x0000607dcedb32bc}, {0x0009d23e8c6cd3f2,0x000ffcb09ace85e4,0x000b7d6c1505f623,0x000df6e74aae2597,0x0000001f7693e568}}, + {{0x0002f4fb58a08ff9,0x000b22b767ddd77a,0x00020e33dc68c924,0x0007b51b436b1d3d,0x000015859ad34f0d}, {0x000bb551c67e100c,0x00072572c5dcce84,0x000057987a70e414,0x0002b07c8616056a,0x0000bd0d7de543e1}}, + {{0x00091ff11bfcf2c7,0x000d9646c09265fc,0x0000510cfe52d918,0x0008aa8cca65d05a,0x00007afc5385da20}, {0x0005903d91d0fa38,0x00048cd25652dd29,0x000688d81a6042f6,0x0009e3be9a92ed90,0x00005df7150a5df5}}, + {{0x000edbae38271e0a,0x0008b5cc06bce54a,0x0001bfd7a369474c,0x000e44e3c9d39cd5,0x0000e2bf2cdfd46d}, {0x00082c92389f1ca3,0x0005c88dcb5d5b81,0x000a825a0e26bc70,0x00005671cf4bee96,0x00000b65e1f6a6a6}}, + {{0x000a928368d1a456,0x0000e11cb329d7af,0x00041e0414116b46,0x000a28451a71e385,0x000027ce6313e7a8}, {0x000e5e5782687675,0x000f0685af014510,0x000d3b826aac14a7,0x000bdffd100fad1e,0x000080ca2ea3195d}}, + {{0x000a01041994c103,0x000f47a314e14992,0x00008ef0842886db,0x000927a87388d695,0x0000b98c4ff30fce}, {0x00097d3fcc147238,0x000a8c29c45c7e2f,0x000f67220c5a2e9c,0x00037e0bd4619da3,0x0000189f33c99079}}, + {{0x0006fce6fd4eb1d2,0x00096d33c1253a1c,0x000e10ffc8333657,0x0002c4acbc619c17,0x00003e3f4c29d31b}, {0x00042a47b0149578,0x000ebc09d858af7b,0x000c49e700d501cc,0x000b440ce075301f,0x0000f231d5f803e1}}, + {{0x000ef3caffa4bdec,0x0002ac777c54035b,0x0003cd0bc68eba77,0x00045b2505bebe91,0x0000e4644ce7cd8b}, {0x00042777579dea15,0x000298fca9661f18,0x000590619d9f2264,0x00054720eabbb3fc,0x0000e6455a9be8fb}}, + {{0x000a577c0e475eea,0x000d6bc1436c96d9,0x00016236e94d021e,0x00008c00a8850a9b,0x00004b2d015d7f50}, {0x000c9dc4c6f4a38c,0x00028553656de8a8,0x000a90c362732682,0x0002bf3d15dd9142,0x000052c5e5dfbb42}}, + {{0x0000cbe5411c8dfd,0x0008a17b7a24b8c2,0x000b9b87fd8cb545,0x0005e6a11f9fc178,0x00006c1feb2d790c}, {0x000c58e0b0881f88,0x0003c321556f92f3,0x000a34013603d060,0x000d70aff166a927,0x0000d99e1e8e4319}}, + {{0x00020a329430fd34,0x00091b753e241ff6,0x000dc9b1f4a806fc,0x000873c09454c63f,0x000053aade33de2d}, {0x000c8ed7cf2adaa3,0x0009418f8a188df3,0x000e4faf13fb7d84,0x000bef84861cf96d,0x0000860acea4d6d0}}, + {{0x0001d18705dde70a,0x00011272a18859a9,0x00033f5014638a28,0x0009304080653c0b,0x00008c599a15b5b1}, {0x0001a5a4482a49b3,0x00092f456786cb4c,0x00064ba9b7822d73,0x00018755a97a72d7,0x0000b095c5fcf342}}, + {{0x00068b2d8952a5dc,0x0001384c32454373,0x0001e2922c3926d4,0x000aa3dca16b1fa7,0x000037a2bd72600f}, {0x0000a741ebdedfbf,0x000e23ed58f44104,0x0007baa3fe459532,0x00006a13286a4d62,0x00000256f65a8e61}}, + {{0x0009fdf904d40a4c,0x000a46face48942e,0x00021d60e882feb0,0x000dd0396add6456,0x0000900d2cd7fcc5}, {0x000a7b9be3cfc441,0x0003708a50df4de0,0x00082f25a62b3146,0x0008d616896ec237,0x0000efc52dd341af}}, +}, +{ +/* digit=10 [{1,2,3,..,}]*([2^70]*G) */ + {{0x00074cd669b25cd3,0x00058c67881c256b,0x000c2d070504e035,0x000ce9e47e328af1,0x0000334f9064ef41}, {0x000e26a1c2dcf305,0x0006722062f65cf3,0x000d10bc4005653d,0x000d490fca65ad22,0x000019179d890699}}, + {{0x00026e7838ac11e5,0x000e2a5a280f9fc4,0x0004127710bbdb6c,0x000c0919081aeb17,0x00009d3642b34a2e}, {0x0002ad9a69ff2662,0x000684c317570d7d,0x00000f6f3f188512,0x0005163b33b1c24b,0x0000d7b1d0e34234}}, + {{0x0009ff3fb1c1c89c,0x0005aabccb1a5742,0x0002d53a4eef7623,0x000053470aedff2d,0x00002e085185e01f}, {0x0003bc9f5fe3ea55,0x000e66ca1fc53dfa,0x000605f7ffc83fe6,0x000716623aa4d317,0x0000f64c0b56ddf9}}, + {{0x000674732b26ecaa,0x0006a9b93619c51b,0x0000bd8947d766ea,0x000f1f8da8cf9566,0x000039fdfa96c371}, {0x0008371eaa4eb5cf,0x000f1c2fe16a83bd,0x0006b6beb3c7b8ee,0x0003dfb56c848ff3,0x0000baf4cad5a849}}, + {{0x000b862c0fdd559d,0x0003472c11580706,0x0008b856520b2638,0x0006879e81a90197,0x000029294aa97e5e}, {0x0001e0f85a0008ae,0x000842f061b6095f,0x000af5f1784d787b,0x000532b513ddfaa0,0x000056317941cc65}}, + {{0x000d1d3a33d19d98,0x000d7df2ca8dcb4f,0x0007f4e6b5456b29,0x000f5ae912a132ae,0x00000f2c9a6887a5}, {0x000e27431535c2c9,0x00023aba1f2ffbd4,0x00049250f22df234,0x000935fba35d186e,0x0000929b2503d242}}, + {{0x000c9391c1b57c4b,0x000d954532e6f2ad,0x0004d08859d320c1,0x0004f4b4c12b6234,0x0000d868d83b4346}, {0x0000ea15c6c613ad,0x000238397325f0de,0x0000c2f9dc73b139,0x0004b7db504ce64c,0x0000dc963c4af5aa}}, + {{0x0005e02dca527597,0x000576587c548e0b,0x000cc2767c475cfc,0x000859476ce6a9bd,0x0000665715a85b0c}, {0x000b469bab32bb4d,0x000d0b6de7c31b91,0x0002327265da0fa4,0x00047cf1f57b245a,0x00006cda9501707b}}, + {{0x0008d572808b4fc0,0x00011694ede0e70b,0x000f8b71405a8e19,0x00015233c02d5353,0x0000ab4ce2e44ab2}, {0x000c47dfd222e80b,0x0008cc293dda614c,0x0002292f07c1d085,0x0006cbfeae2c20e7,0x000061ec173fa523}}, + {{0x000876bc63806f07,0x00024d66914b313a,0x0008170d851534d4,0x000b76faa20f3de5,0x0000aab05f61a766}, {0x000664f3ab471d0d,0x00011fdc64a701f2,0x000ea68223acb6e7,0x00013a23fabc7bb1,0x000090227c181a7d}}, + {{0x000ce1496c8edd99,0x000b6594d679e5a6,0x0002b74d6e54b8ba,0x0003a13ee780ee06,0x00000955d9961a01}, {0x000cd19a119ac211,0x00082f7e73061c51,0x0007bf37a3de8103,0x0005363b406e5945,0x00006f8f87f664ee}}, + {{0x000815967b132401,0x00064ecccb2576eb,0x0005d3b7d625290b,0x000ec5d6f358953e,0x0000a5226a7375ae}, {0x000abb11ef347a92,0x0007143ccb0f7b41,0x000c82f4ddf3c9db,0x0000dfd73dd924bb,0x000087da47b38012}}, + {{0x000aa601588458f9,0x0000975e5cf241d9,0x00060b80ec71bf1f,0x00029fcc1fb01f45,0x0000823f9d230d8f}, {0x000657c67a86fd8d,0x00034e709b0f413d,0x000642ec3286cfe5,0x000ba7caac701a18,0x0000fa3bb61f6852}}, + {{0x000801b0303fe8c1,0x000fbe0cde9a2307,0x000713051354d6fb,0x0006e735aff62229,0x000025ef6c00b818}, {0x000306588dd6cb80,0x000a99185c90f436,0x000ae928b4ff1484,0x0000b1c883be31ae,0x0000a9c05acf7be6}}, + {{0x000b28128cceb413,0x00082eab515e529c,0x000d8b8e887dd9a3,0x0002d5b9d67b20e6,0x00008b59180e89cb}, {0x00059e2abcbade3b,0x0006184d19e316ec,0x00044d27ee72fc4a,0x000c67731c05e18c,0x0000a8c4db3ec2d5}}, + {{0x00048bb35989f3d6,0x0009b3067fb4c320,0x0001a0ffbcfe88ff,0x000004e03be2c82f,0x0000fcbfa01af932}, {0x000f814e7e4cf8e6,0x00024ee9f240c53f,0x00086fc0004f7b88,0x0000ac34de98e74d,0x00007917ef356057}}, + {{0x000c6c4a11a11477,0x000d8b1cf6d0d720,0x000694efa1c1285e,0x000e0c1350e984be,0x000078cb9118b03f}, {0x000bf89c99dbbb29,0x0004fbfdfc88574b,0x000eee84278a2c53,0x0008ff0a649b25e5,0x00003d025e893279}}, + {{0x00037a391d54688c,0x000c7c592115f719,0x0007653bbcda0dcf,0x000a69f5817c7617,0x000029a05f07e803}, {0x000753fad30733d0,0x00072bd1849daff3,0x0002fca30fa2fd8a,0x0007feecaa47e99b,0x0000b0731727e7c5}}, + {{0x0005a21b38ecce44,0x0006e415c2abd742,0x0002c0e950133c0d,0x00000ea72b809264,0x00003015ecb9ba58}, {0x000360d2878775ae,0x000bc41c8e4f6d21,0x0003c10aa67f95a8,0x000900e292f6829e,0x0000a4ee43081d92}}, + {{0x000fe9693df562e6,0x0002fad17b8b0aa9,0x0003de3c362eb368,0x000d534ef25783ce,0x0000ebda4537f429}, {0x0003627aba553bd5,0x00097af716fc435e,0x00095a844547dbf0,0x00093eb262399131,0x0000588e07a230c3}}, + {{0x000af60d3202fb41,0x0004c8c7c74a1d4e,0x0009e5b39057f717,0x000f7cf49850c859,0x00008d151014d7b0}, {0x0008fa60b07a5144,0x000d9ab459ce85f3,0x00097edc90f40721,0x0000fbad27012797,0x00005754882b9450}}, + {{0x000dbfc225b16955,0x000deb78b51638a3,0x000458d99f09b93c,0x00061e32d72821e7,0x0000340498c56d86}, {0x0004bf05323cf76a,0x0002ff7ffc746f1f,0x0007d6ce5b44bb0a,0x000c88717795b069,0x000004ab2bd3e89f}}, + {{0x000605a2ca22e4c3,0x0008ac2d7351ad97,0x000eab2e88f2760d,0x00021fc887db7e2e,0x000029bc5ec3ebf3}, {0x00086560b7317f55,0x0004d18b91ce9d01,0x000b6f646f4fb8e4,0x0002a1695833078c,0x0000d6a694f08f66}}, + {{0x000c95e3b72c81be,0x000e8a03275a6b4a,0x000d6b09aa4f69a1,0x0002be1ee61ad29f,0x00002844ca4e17dd}, {0x000e76f8794eee2d,0x00087cd0bdb33924,0x00040e3f2ddfa70f,0x00072fa4a906265c,0x0000b96580f51259}}, + {{0x000023b688cad9cb,0x00076df8951487ec,0x00010c9cdd6ac550,0x000ae554e43a52b9,0x0000fef35226f5fe}, {0x000cbe7fdb2ed39f,0x000a0a5e1bf9b554,0x0009afefe7699601,0x000c50b5d7563445,0x0000010fb3404117}}, + {{0x000279f9137ac6ab,0x000249953bb50af7,0x0000826f6345477d,0x000865968a387bb2,0x000005a5cf531199}, {0x000c5268a915e259,0x0009720c4f4b65a7,0x000b7b99cd26125c,0x00058b59772469f6,0x0000853ae820ffe3}}, + {{0x0006b4b6ec89e3e9,0x000333d0ff1c8c17,0x00059b5dc3204606,0x0005662b172727c1,0x000080e1bebb1c71}, {0x00018894c4864ef8,0x000a9eca02e2c6e8,0x000cab3d09dad501,0x00060af51c7c848e,0x000047e4fa35bc16}}, + {{0x000570befacb28ea,0x0008743dc8f455c5,0x000f40096cf59c5c,0x0004b182c494694d,0x00003c0fc958c313}, {0x0003107f39793424,0x0000cce996b712e9,0x0000246006fd67c3,0x0007be905a6fad26,0x0000fbfc527a687c}}, + {{0x00095e1c222ce3ef,0x0005747315cd6f47,0x000423b7afe75d5e,0x00019bbc54314d23,0x0000ef8b8f77d09c}, {0x000803127214c810,0x00015ffb94e2fbe1,0x0006d06ac1164fef,0x000f30476727cb12,0x0000a8113857ebf7}}, + {{0x00087efddf19eaed,0x000e96b6e5f5f6d7,0x000ece472e89ecf0,0x000c927cc82cb9ae,0x00000feebacc040c}, {0x00099970ce1340b7,0x00074081fa310631,0x000f300ce340cba2,0x000a1055a8739566,0x0000fa0cef064603}}, + {{0x000082c59b5bc93f,0x000b65fc16a6d50a,0x0004bee2ba80b4cf,0x00016533174cf7dd,0x00006aceb571e5f7}, {0x000006d350847cb5,0x0008616d7f8911a7,0x0008b0f735565146,0x000122795d2872d6,0x00007b8ca3d3efb5}}, + {{0x0000a88e8800df98,0x00025448903b908a,0x0004349a5a375804,0x0005b13ac924efa4,0x0000a5e450fd59f3}, {0x000055716f999f92,0x000de9af288dca33,0x00069f12c2367b4d,0x000284d0897fc5ab,0x00004fc4fac80274}}, + {{0x000eb7eb35f2a934,0x000d353268becc03,0x0000a4174b282d2d,0x000f5ddd6356402d,0x0000e7ffa1bdee1a}, {0x000aaa4d2ca7e708,0x000704fc0f9760eb,0x000357060cd33f91,0x000d81e7eb1971de,0x00003d746cad3a0a}}, + {{0x0005e0ddb5450c74,0x0002a674a2e0aec2,0x000bc7d3441bdd25,0x00079a58f6e1b7da,0x00001d2cad53a42d}, {0x00076a79d2639d68,0x0003f32fc3bfb513,0x0002bfdf25a3776e,0x0005d91a51232583,0x00009aba866b1079}}, + {{0x0001534fdb96049f,0x000d73c129cc545a,0x000caf544a6dce8e,0x000d95f07d61f6ae,0x0000dcef4f010d17}, {0x0003b9847259e03f,0x000e08eaf94b5a31,0x0000a320a3862f66,0x000f35bfb4c3fc13,0x00003ce88af25ac5}}, + {{0x00084f5111ed5705,0x000b79be0976a8d9,0x0009ab22e7a6498e,0x000cda52cf1ba030,0x000013d88452929f}, {0x0006f21a33ad72d6,0x000e27b4947e3ec9,0x0000022021b0fa2a,0x0009c5f35557b5e4,0x0000f01ef8814f2f}}, + {{0x000c2442542799ce,0x000f9b3064143739,0x000f1d943ee0b9f4,0x000e26265a704416,0x0000e87d46c8a352}, {0x000a8e3615322bd7,0x0008190d24dbc6d6,0x0003d81377ed07b0,0x00056a2a112749e9,0x000094697b190dce}}, + {{0x0004cf425219eac2,0x0002cb6fb9207e1b,0x000cc82f73e79b1e,0x00073ae27f3c4a54,0x0000554f3a88ed84}, {0x0006de42a1700f2e,0x00006f9c40687f07,0x00093ba557a83e9c,0x0000a11fbe078a85,0x0000e7643032099a}}, + {{0x000f9abfe9907a6c,0x0007cc63f2dfda1c,0x000bac69ee3043cc,0x00041648aa637938,0x0000bbada2c4f2c0}, {0x0001495c9e6f03da,0x000fb7b4eda11e13,0x000b4954affd78b1,0x0009d34c115f6b0b,0x00000db881c07854}}, + {{0x00029e61ad8613f5,0x000c9ddb326c11c4,0x000869e2813f8b2e,0x0000b0117db8ee3a,0x0000dda75e5f5ffb}, {0x00073327c49cc58a,0x000d0c3da6be20e6,0x00070dd5c65cada3,0x000ef461959c8261,0x0000fa5cb2cf4faa}}, + {{0x000e000a34fd0c8a,0x00058901bb98ca77,0x0006eb2905cd2973,0x0003336dda93946a,0x0000934ccf97e513}, {0x000c1d692b201b8a,0x000f029af6321234,0x0008d9dfac9c7c2b,0x000f582d8679e67a,0x00008046ce69fc1a}}, + {{0x000ab8eb6e0336a9,0x0006ea229cba0232,0x000dedbcad54fcf3,0x000667a7a0cc9245,0x00000a2c42237703}, {0x000b601fbb80bfde,0x000a2c63fd8939e8,0x00049e20428365e9,0x0007aca1417dd6d7,0x00003f11af7e1f8b}}, + {{0x0002330b4c09a0de,0x000902a37b9c9a53,0x00036cda4e95cf6e,0x0005ec789015573d,0x000029deb51579af}, {0x00020a37b9ea8e07,0x000799397a517f13,0x000557b7511fa4f0,0x0004370b092cc886,0x00004fe2aea14013}}, + {{0x000b5c6a9db98dcc,0x000dd04a3aba71dd,0x0006608f14057254,0x0006b8b756359d19,0x0000c4455860c9f8}, {0x000a2e774f5cc99b,0x000300a46307126b,0x000302ddbaa681cb,0x000535fc96dbf608,0x0000c681d0076c06}}, + {{0x0004d84c9e62370a,0x000a69eb6fa1f9bb,0x0000ef588241feb6,0x000245740d6249c1,0x0000be2d5006f181}, {0x00041f73dee49528,0x000c00da95bad9d3,0x0000e049a1eb457a,0x0001bbb11190a486,0x0000110d97c60ffd}}, + {{0x0009877303f6c251,0x000a1c17dc10f282,0x000bb8dca73e7560,0x000a683b3ec4fd74,0x0000a9f75e1bb560}, {0x000919d9da696d60,0x000d217fedcd3ac2,0x0006adcd9d53df79,0x000215470c8bf19e,0x0000b0f430bfb8a1}}, + {{0x0004ec0bc6dcdfa8,0x00046693c98632a8,0x000bba59e9150cd8,0x00022c245639b8fc,0x00003d4bb7590c93}, {0x0004d6a04c6abc20,0x00066201cd556fb0,0x0001e5e4afc3d001,0x000ae07774cab315,0x00003d6c82521fe2}}, + {{0x000c32cbecb96f49,0x000a3005ab64ddcf,0x000baf06df924953,0x0005e8ae7d321d49,0x0000b2d331a8c32b}, {0x000d67cb123fdcbb,0x00088a617dcd4f59,0x000e94c2e5129661,0x0008ad46352e5f1a,0x0000d5565de6b48f}}, + {{0x000e6659b3560d39,0x000ffa229f90747e,0x00023180522e466d,0x0004d6c4a3f828c0,0x0000e455d80ec3ff}, {0x000e2eebf20d3aec,0x000c736f73d855fb,0x0002184b7e5febc1,0x000b11155c63fc28,0x0000a428c274cade}}, + {{0x0001c636e5373dbe,0x00086eb6f714bd17,0x000e1b6a90d860ea,0x00060dc93cbac42b,0x0000d34c45c4c0f5}, {0x000659cadd09b1eb,0x000ccdb20e3300d0,0x000141729d71e84b,0x00060e782510dc77,0x0000ec0565e653b0}}, + {{0x0009e4d6e92f36d7,0x000825cbda73f299,0x000665eeb18db8b5,0x00018f9cfee6e9a6,0x0000ca23779281ee}, {0x0005db4b1418ca14,0x00024447b1c7f54f,0x0004f104fcae5744,0x00000ab06a6080e1,0x00001054c45b9651}}, + {{0x00055d5f2446ec83,0x000467a9ebe2680a,0x000c77fc1761cda9,0x0005aa015446cd46,0x00001c27acc7da7f}, {0x00077703a89b1b78,0x000cd191b9b97518,0x00013cfd2ff9eac2,0x000e94ceb77c72d1,0x0000fb9cf6eb1afc}}, + {{0x000c0193744cd46e,0x0008b3ceea178d94,0x00014a628cfe494b,0x0009d6820be8421f,0x000094afe6f01bc6}, {0x000e28b4eda452b7,0x000538785cad4748,0x00037da88fc3c7e7,0x0009fc488a0a24ad,0x0000cc96f2897d92}}, + {{0x000f9f92b892fabe,0x000e8fb2e6bf1cfd,0x00099f05d11cf6a7,0x000c3b1597b79e80,0x0000a26740afb0a3}, {0x0003b35d8b31a83f,0x000396846e25cbc8,0x00082f6f006b2f50,0x000a658d89e03e35,0x0000db716db3d591}}, + {{0x000d177e1fa4ed0a,0x000dd5607e4daf0e,0x000573a2cc2358b6,0x0002b7ee7eca899a,0x00003a1628589e78}, {0x000f908d149527d0,0x0002262175b94b20,0x00013e200c0bcd10,0x0003975b06898fb3,0x00008491f71c53a0}}, + {{0x0002d700f31b0e00,0x00019237b591c539,0x0009fe9bccb246e6,0x000017bf39be6432,0x000028e4046f6380}, {0x00046c9c0d21521f,0x000fd73f252bbf1a,0x00043fdf02581ac2,0x000babf43b3c4272,0x00001a5ee9169a81}}, + {{0x000e5448aa1f7350,0x000f3edd6264a716,0x000253daee2205fe,0x000b4e7addb5be46,0x000055192f8f04cf}, {0x000b2643882ff890,0x000c409401a9d5ae,0x000413e0c8a16ec1,0x0002f7f8a4fe486c,0x000089973c8ecdb6}}, + {{0x000af3487f6b18cd,0x0008a3d07d2ec763,0x0000e5a1521e2961,0x000427104e65b30c,0x0000bdacad3f1ac8}, {0x000145358f37a6dd,0x000d004a44da735c,0x0009eebe68dd299b,0x0006b05a62c5ac63,0x00000ac76876f8b1}}, + {{0x0008acf5a8209bcb,0x000560df7a62c194,0x000c1fb14acfeda4,0x000df2c2224f5fad,0x00003404df8d5798}, {0x0009a77b70e54968,0x000520108a2364b0,0x00015fccaf6ebdf0,0x000566e36cd047b8,0x00005e46d74586d1}}, + {{0x000844d8e8ce5587,0x00092279b1356b90,0x00091add341ad4ae,0x000a7d2fa10f0f42,0x0000148c77380f8c}, {0x000496d6691d4cc8,0x000782bb8243fd56,0x000627db185c1156,0x000a5418645f44bd,0x0000d506007bdf79}}, + {{0x0000bd279fead68e,0x00056182e4e75be9,0x000dd1b6509c30b6,0x000ed8c47b692128,0x0000f5fe87abfda7}, {0x0003143dc3675e1a,0x000c502b0a119eb5,0x0004fe0aff458d3c,0x000a31f46a7b233d,0x00005acb343f0376}}, + {{0x0002d15eee0e0d34,0x0003d148684f5225,0x000cdbe9722216f4,0x000c5cce42f45a7d,0x0000e205d19ad51e}, {0x0000f64c68b5fa7c,0x000058e30f053866,0x000a6c9cb5f06a10,0x000f6f5287fecaf3,0x00001883d276db11}}, + {{0x000527672331c67e,0x000a87a64576176e,0x000c391a5c3662b4,0x0006c076cdba095c,0x0000f9220fdeba1e}, {0x000ccfc868b90cca,0x0008290a8b4bca9a,0x0000e92873faa03c,0x0009492e8d17378b,0x0000a47f8c64e176}}, + {{0x00028cb6de5ee8de,0x000cd5efcb0928d3,0x00077e0ccf78a9c3,0x0001c402efb29bef,0x0000459ba0ccaa24}, {0x00015eff8dc3bb71,0x0001926d930b21d2,0x000c2f8d6a0caf81,0x000f17b13725955f,0x00005d65bf7dfab7}}, +}, +{ +/* digit=11 [{1,2,3,..,}]*([2^77]*G) */ + {{0x000f9aa8775da137,0x000593a6332e3942,0x000acdfba04c12e0,0x00036116cc38b202,0x00008a8960a21793}, {0x0002d21251bfcee2,0x000d5d5e4ebe7fd4,0x000ecd2b4f06757e,0x0004179312209cb9,0x0000116a47ce63c7}}, + {{0x000fff36dfdd5049,0x000f7ce6110d0407,0x00098a9fc7cb9e1f,0x000926e424e2d9ea,0x0000835121b0ef1b}, {0x000d9bfa74da96b0,0x0002cd0f068f3fbe,0x000fb3442f781253,0x00047bf2a224f7f4,0x00003b3786a0183b}}, + {{0x000297122b1f3105,0x000638dba8324c9a,0x0007684493f6706f,0x0006825bd6946011,0x0000954fb43f082f}, {0x0009146aaba29f01,0x0004e9c6324aadf7,0x0008d8428f5d9954,0x00015d922efe9529,0x0000d292e4285a89}}, + {{0x000eda370bf670ce,0x000eff959124f0b7,0x000cac220c098413,0x000042e12faaaa26,0x0000895133539985}, {0x000e3efc5819a77d,0x00057c57c1941637,0x000d0cc10c5d3a74,0x0002ffae97c4601b,0x0000cc0b9439604c}}, + {{0x000eebc15ab85274,0x00019d13c980241c,0x0003a82e05366ff3,0x000eab4894fa90a8,0x0000f875d5053ef5}, {0x000f960988cc3ea5,0x000ead66986792b7,0x0002473220ad7ea6,0x000afad402fea6ab,0x0000112495728583}}, + {{0x000f4636a79fb44b,0x00090857f24b3d06,0x0000bd52a5b7cd91,0x00064839569b4ec8,0x0000d1b1ac7d83e4}, {0x000a69c7aac7f865,0x0006f8fb84cd4b3a,0x000b5e37812c149e,0x00091eb184db0903,0x000048acff136f66}}, + {{0x0001299f5efe435e,0x000b7f1400ac2880,0x00059719edb65124,0x000c51a678fab01d,0x0000165002ee9aa8}, {0x000bbf3f226f941d,0x00011de495d3deba,0x0007d94842b6a345,0x0003d2d9919c7098,0x00000d6590e37520}}, + {{0x0004ee9ebab5f9c3,0x000ee5aa2862cfa0,0x0000825883683453,0x0005a0745585d074,0x00009825b8191988}, {0x000e1d078fb65329,0x00028cc58f8e9d76,0x000caa6ecd94a45d,0x000e336d0acf4acd,0x0000622e82ce2bfc}}, + {{0x000008a44a2706aa,0x00041b1add588771,0x0007569a605cb923,0x000def6163a01430,0x0000a568614fb2eb}, {0x000f81a9ffce874d,0x0004c0a48585c4da,0x000233e6225a5f0a,0x000d30ea80e24f54,0x000012a0f8a98988}}, + {{0x000b7355cdc2f284,0x000d2a27b9adaef6,0x00093f3b59e1ed0c,0x000b746910956189,0x0000c77d5df71c8b}, {0x0001803652237193,0x000739b9537fbfa5,0x000a62d938c3a3e5,0x00058e9ebd4a1293,0x00008b4fbeec3446}}, + {{0x00052179751d0b7d,0x0000f901f8a82fe0,0x000136ce503fb389,0x000dba5498f02145,0x0000cb7cf9a661f9}, {0x0007016c2992806e,0x0001add3eb9fc4a7,0x000307dc5f99dbd8,0x0004c304da7f5ed0,0x00006fb20bdf58fc}}, + {{0x000325b563f62c26,0x00088e338763fd2e,0x00070b882d81d583,0x00086a60eec0789c,0x00000900a5d21f5d}, {0x000b2fb282b41ea8,0x00039ddb91517ee8,0x000f77556d931d68,0x000bf902f126e703,0x0000b19f00d7d3a7}}, + {{0x000110e849fc28b5,0x000b011096883b49,0x0007657f512d21a5,0x00011f76afc44bf5,0x0000746d7f32aaff}, {0x000037a74dc795a3,0x0002395a1c198937,0x000a8ef1664ed39e,0x000c26cdb84ee945,0x0000696b907f6d28}}, + {{0x0009d6f7600276fa,0x0006d3198ab29dd7,0x0005cb1d12f3e169,0x000f3329a912fc79,0x000030c403e7632d}, {0x0005a8f5ede17ddb,0x000859c12cf6284f,0x000881ae76605fac,0x0003ae8ff98c78aa,0x000023ec368ec733}}, + {{0x000130cefa1d5675,0x000a003664f802bf,0x0004bd36ef9058a3,0x000a3113c314fc7c,0x00002389880af8c5}, {0x0007ec1959838d48,0x000fb01ed3b95705,0x000f59b95a2c3c69,0x000e21c93ecb7a35,0x0000427f7b9b8f6f}}, + {{0x00004ae4288bb5a2,0x00037f3270264a85,0x00010b20b51cafa3,0x000a0e9e414efa30,0x0000157a33e62123}, {0x000d80dd6415cd8e,0x00020677cad8cac6,0x000c9fbc4f37610d,0x000d6dae155e1c85,0x00001d0a4ba60f48}}, + {{0x00067d8c0e279e91,0x0008245afbee6855,0x00078ddecf33ca01,0x0007946ed3edf19b,0x0000f617d59ab035}, {0x0007945ae089290f,0x0005fe0200f2c492,0x000bf24784884e8b,0x000928852fe9c760,0x0000ffcb8771fa43}}, + {{0x00066fd3da53da79,0x000b479eb0a7ecd0,0x000b2d549c3fc2d9,0x000b9c3ae4337ac7,0x0000ad4d200fc335}, {0x000f93463a303081,0x00096441398584c8,0x0008f0df55fc6eb7,0x0006d5481aed496a,0x000038658e97f52a}}, + {{0x0009033a9a9ec889,0x00029548d2a17999,0x000b47721fa1d736,0x0008158e33920d87,0x0000045c74a58e24}, {0x000307917a0bc1d9,0x000057635c548972,0x0000f6f88a032dce,0x000d42aa81563402,0x00004682835e5a05}}, + {{0x000ff28d04ebc165,0x000507686c2c4e6f,0x00063dcb51fce3da,0x000d9fe06896b6ea,0x00002e5ce05439c7}, {0x0009d2e6bdf7eace,0x0000afa52e93e883,0x000674e412921b90,0x000bce92640a5a16,0x00008b7c0dd9fa63}}, + {{0x00028f0d3d5e18ee,0x000c30590eb006cf,0x000aec8faba412fc,0x00093e1f0888bc96,0x0000616d1fdc8348}, {0x00044c101c5f2508,0x00006dc12d0c3c51,0x000280f9293afec0,0x00028d276ff0f7e6,0x00004cf82b9fbef0}}, + {{0x000644eb6ad3daea,0x000f9da617bed4fa,0x000591916a0aea40,0x0004a62c8bc21622,0x0000b14787fa4323}, {0x00029dbaddcfdc82,0x0007d2c3a56f1f85,0x000d278b64ee5284,0x0007a85b4f7a0572,0x00004776f3f249a9}}, + {{0x0009bdc6a9c5425d,0x000a74cb2e08037a,0x000c41e34d0fdfda,0x000259c1bfad07fa,0x0000ef71695d1f87}, {0x0003cd312e6c539f,0x000dfe79874c3b10,0x000e03e026c03074,0x000366d7932c1541,0x000049edec49dd5a}}, + {{0x00048153ce2a6ced,0x0005cb374595998b,0x0001d374c1a016c7,0x00056bdfd770e4d1,0x0000152c3e66158d}, {0x0004234660137887,0x000baf4ea70fc909,0x0007a4061d5c4126,0x000a704f10817eaf,0x0000e2581ebc91a6}}, + {{0x000b859e2c097385,0x00023910a6cd24e1,0x000b5f1edfe2b5b0,0x000a39c796e06e2c,0x000061b222422caf}, {0x000f6e990b5ae344,0x0000276106c10652,0x000091bb1af35abd,0x00082644cec3093f,0x00005d1cd36bc1c2}}, + {{0x000fc1520ced10f5,0x0003010001daedfa,0x000eaefb25ebd7e8,0x0001d7614a3f50ef,0x0000f52e9b2b27ec}, {0x000b038acc2e629a,0x00014e35ab802696,0x0004c8730d623772,0x000307b05e87064d,0x00000c762bd34c12}}, + {{0x000e308ddcbf0bd1,0x000b96a5dc0cf082,0x0006cc07a52337c2,0x000fe84ae58e4cfb,0x0000659e706d98c1}, {0x0002ec7dd17741b3,0x000486171f471006,0x0002a5bee3b6d6b9,0x000eb91bd92dd984,0x0000975826a09b31}}, + {{0x0003c9292ce02260,0x000d0dd17ee212fc,0x000bd9b2da2e3c4b,0x00035318456b64ac,0x0000a1961934d45b}, {0x0007febf09ab4caf,0x000002c812353920,0x000942e1afcfb9c6,0x000a778eaf0948fc,0x000043ec3002cc4d}}, + {{0x00064b180a723820,0x0001aa42c8b77ff8,0x000fb95a1ada2085,0x000241ea27c78812,0x0000c96b8e900b7d}, {0x000a8792d5ec82d4,0x0000509cd684235d,0x000785d18e35f6ca,0x000788a287dd65ad,0x0000e9f2d7e56386}}, + {{0x000b08a242d1b6d6,0x00041b0fd09c63e7,0x0000cea65281d8c2,0x000dd0e09cd34ca3,0x0000cac233c630e7}, {0x000013f31fa41372,0x00054e81b46f3496,0x0006319dc9ade00f,0x000f520220b1fc74,0x0000e2b1a530749c}}, + {{0x000606de55192498,0x000c5809c40aa334,0x00096632d3890f16,0x0007e02bad0a1796,0x00005d185bd38bc9}, {0x00077cab67b3412d,0x00046a2cdd3292a0,0x000c830a347d5c0a,0x0000aa324c123aca,0x0000685c38fa87b6}}, + {{0x00038414a19d3175,0x000e6a1704e8a59a,0x00040b43fbe48253,0x000ce06f982b9219,0x0000d5b7bcfbcfc2}, {0x000713f4576e6000,0x00031b8ba82c39f1,0x00005f1c4af7d65c,0x000c3034d34805c2,0x0000f4899f930218}}, + {{0x0009d84b49c30b6d,0x000c425a60175a5c,0x0000a9f29180d182,0x0006fad47d4cfa1a,0x0000b40b136882cc}, {0x000d6a7c603f3345,0x0001bae3d9f84dde,0x0002e1b29542ed45,0x000b004616436a00,0x000003fa1413eddf}}, + {{0x000cfbbd57abdae4,0x00020fc5888c8737,0x000a4476d73cd5c7,0x000218725a0deed8,0x00005bf9e51796ff}, {0x0002e189a1751dcb,0x0001228d9683dc50,0x00093baabf58f351,0x0000af8fdceae65b,0x0000cc7a48df0733}}, + {{0x0003e0d34f77c46d,0x000fbc6edb0f3c88,0x000068e756b0e5b0,0x000e78037099eec3,0x000038e0b3c5fb3d}, {0x000f6109612e790d,0x0001867cb2863958,0x000f3a5af9ead942,0x000e99527c7a360b,0x0000669299225f30}}, + {{0x000378d02aded289,0x0003c0423ea0efed,0x00076140c21c9274,0x000f78ad1a14b332,0x0000f0a890144187}, {0x00021349cb6cb96e,0x00056d9c9076e14a,0x000e9750cf7a0c30,0x000eb4b264c96979,0x0000fb23ad7c9708}}, + {{0x00082f88233a37da,0x000d23f23abb42d1,0x0004b3734df9d345,0x000d6521d54ea368,0x000059fae532d7b1}, {0x0007515524ef218e,0x00007a73ae6b69e7,0x00058efc608d1521,0x00074e26beb85e78,0x0000aca7c3b8b78d}}, + {{0x000c9e891767e9cc,0x000a6db0fc3a5dec,0x000195eb1598acc5,0x00005382d1cca30d,0x0000094de8c7a92f}, {0x000eb3338b3392a9,0x00013886c546dc13,0x0000d496c04db373,0x000adde62bf2b9c8,0x00002f299eed9137}}, + {{0x0008bd479af2b8a7,0x000ea590b94dddfb,0x0009b3b12d8c5034,0x000cf0e1ced752cf,0x0000b36722ea39b4}, {0x00069628a0877b3a,0x00015bad064c4d30,0x0008ac446518bb7f,0x0006b03992b46ea9,0x000065fc30503fde}}, + {{0x000bc2015b83fd15,0x000dc269799522b1,0x000b535f20b2820a,0x00069a5142759c76,0x00006f2ac98f9822}, {0x00083169e636983e,0x000e591e55c80622,0x0005a4241aa4d2a1,0x000591cfdd294197,0x00007b55447387c4}}, + {{0x0003498722f42f3f,0x000a59410f413e15,0x00032a6354667481,0x0002956d7b04af7f,0x00006a645082d4e9}, {0x0000da5be5d8d302,0x00093df7e7583fbf,0x000d7c2be3abd8e1,0x000a000194d9418a,0x00000149bc9eb3d2}}, + {{0x000f986a073e3517,0x000f9b73f9e38de2,0x000a58bf2571610c,0x000b0af920fb7224,0x00003d4b8dd4ddf2}, {0x00030e4ce112c73f,0x0003bf36e1c67f91,0x0003453afe2dc80e,0x0005ac201a013e91,0x0000c59ce10e30f2}}, + {{0x0003b1cdc7f8928d,0x000ebc41452df139,0x0007451137f6868d,0x00053ce9613ed210,0x0000649aaa42f694}, {0x000b7ba1a932b30b,0x000f2a4924b83721,0x00040f1f33138c74,0x0003dd58146c6cea,0x0000e7f4ae83e138}}, + {{0x00085f185870b6bf,0x000b92eed3fa9528,0x000d2b635e2b8998,0x0007568a94e9d333,0x00000526592b1491}, {0x0008aab3be513b25,0x00045303376d3cfa,0x0007c72c989eae8c,0x0001f30e65c1f409,0x00005b94d4c0e401}}, + {{0x0003d066fe100843,0x00074d0b7004bb28,0x000967116b79d60d,0x000cb60757a89246,0x0000029e71f8c407}, {0x00065bff0dfe7d8a,0x000f7f471826dacf,0x0005753bb87557ed,0x000d529de1afaaf7,0x00003c27f0415370}}, + {{0x000c10c40e0ad658,0x000d03d36ac61fc1,0x000143c6a50f0060,0x000b87a02cab6659,0x0000aa0ac941392a}, {0x00077a9d795672d8,0x0009caada373a423,0x000f29a04260b72e,0x0002e8475833e9de,0x0000035e64a7ec99}}, + {{0x0001b1e50eb884d8,0x000fb550dec7d443,0x00015188fa1435ef,0x000fed29334dd65c,0x0000cd2d5a342204}, {0x00054e8f171c4219,0x000d2a2a4537bac5,0x000bde4f44854e54,0x000cf4c3773aace3,0x0000a962593e8aa2}}, + {{0x000a77a80d8d4ad6,0x00075f34cbc4dbca,0x00009e3fe02abb2c,0x0008ca980524452e,0x00006411b8d00213}, {0x0001764627b7712a,0x00023948a2b5cb5a,0x000858785cea3d52,0x0002036e6e748680,0x00000042eb925ac7}}, + {{0x000751674ece9ae9,0x000a512b2c44e9c8,0x000e88fccc205168,0x0002c75cd9811607,0x0000f9001ef938ff}, {0x000ac81e5a48ef97,0x0002e0109b2bb3fd,0x0008c3273e7e9d5a,0x0007b2dcb6ff8d2a,0x0000cc1bdbc8ee70}}, + {{0x00056fa2bf280369,0x000a9e2e674b01be,0x000c71f07ded96d9,0x000002b55134dba0,0x0000dd1195c38794}, {0x000266f2f86721fc,0x000adda42ae0dc28,0x000d8561fa395767,0x0008fd2270c1219d,0x0000d0cfc1b86620}}, + {{0x000bf27baf63ff66,0x00009ebc66252bbd,0x00081af1cacfa286,0x0007812dbb29b31b,0x0000fb9396e63aab}, {0x000ada1d38323f82,0x000ad4f13aa595b7,0x0001d791ee24a41d,0x000c0b1e03dfd84e,0x00007cfadee3c277}}, + {{0x000f91b24c3fdcf9,0x000dc39512d9d6c2,0x00079ea474581c15,0x000b3037480385b4,0x0000d036170f05a4}, {0x000567ff9990c1cf,0x000c52022e9733fd,0x000e053c243f4d6a,0x000e8d640b9aa656,0x000085f487e4d5f7}}, + {{0x00091898d9f31562,0x000dfbe0453c7472,0x0009e6b7f7af420e,0x0004b20e2b916673,0x0000164bef06f4b6}, {0x000b80fc51993f6f,0x000c34d3d86975a3,0x000ff3c930f61aa4,0x000b4918539af658,0x000059a8d10c4494}}, + {{0x00066e89fdfd1ee2,0x000e2e6e602e0282,0x000d5c751cb890ad,0x0005b84b04a46bd2,0x0000ce8ef41b6ed0}, {0x000a5fa420cfae5a,0x000f8b6e315d4386,0x000903ae604fd4ff,0x0004b04a393c9fe1,0x0000cd608fd28510}}, + {{0x000b3e82dc9c8f33,0x00059d7a2a568260,0x000ec023ab8ab819,0x000b4672639387f7,0x00002043db1d435e}, {0x0003a6c41ed2e7cf,0x00005ab1014aa9f9,0x0007dc38d75705e1,0x0004e061bd047969,0x0000eb648e51640a}}, + {{0x000f0f57c547a961,0x000d990237650041,0x0004510d6c34d742,0x00063c4132ff339a,0x0000408528595bcf}, {0x000a71c3fe658de8,0x0001c3cf64c2960f,0x000e6a5c2ae5ee0f,0x000e11651433eea4,0x00003984cde74fca}}, + {{0x0000003ac6f58504,0x000341671c32ea49,0x0002b8921facbf78,0x0004fa3c67b6d380,0x0000873d3817bc08}, {0x0000d0e2edff3bef,0x000ceefbd4439499,0x0008b941acf60583,0x00068f3529a5a269,0x000098b50bd525cf}}, + {{0x000a25873e60025a,0x00053855a627b0b2,0x0005a4f0d708e8e4,0x000b258de1632834,0x00003056505bfd84}, {0x00090d0a530bf0e6,0x00043c0c25a53dab,0x000f5fa79e358944,0x000b501d65285c91,0x000077599220aadc}}, + {{0x000decde86b39648,0x0003ad7ad0f350fc,0x000fe462f1844e2c,0x0003d39cc52142ca,0x00007a06c317b146}, {0x000867e4dc14e8c7,0x000ff41bcd37d80a,0x00032c5b0675151c,0x000653c449373f36,0x00005a57dc103d6d}}, + {{0x0000a2a7c0db9bfc,0x00011c2811b663de,0x0007ec267d1c9049,0x000b23256ba8b300,0x0000534d306792e2}, {0x00026c1d52004b5a,0x0007a3f4dc127efd,0x000d195c762760cc,0x000221924b8c8b14,0x000026c7b7eb7f67}}, + {{0x0002f18ed29ebf14,0x00098c2b10a9d4ab,0x000a74882a5d041b,0x000d5bf9c651a0c2,0x000088a4278579c9}, {0x00051235c715721d,0x0000e9dced4d4ef7,0x000ee67bd5212bc6,0x000d93da86c0e051,0x00006c0ebc2e5127}}, + {{0x00070438f33a86ab,0x0006a9c27c7efe20,0x0005fcfb1eb8742d,0x000157f616499b3f,0x0000133c81b68e98}, {0x00010f66856c8633,0x000cda9e424d843f,0x000454d474e9ef7f,0x000d084addbbba56,0x0000f1d7dd5fa3c0}}, + {{0x000d7b41a5dc9f16,0x0009ef12f95203b7,0x00013189c2fc91e0,0x000aec2347a9cef1,0x00002719ff12516e}, {0x000d3b82a3b61af6,0x000df7f38b308736,0x000bf80445b12e06,0x0000c743ba9b2725,0x00005ba45f8b88ed}}, + {{0x000261f56f9e574d,0x000455263e563df4,0x000e18043b012748,0x0002a907208773a4,0x0000ddabf1b7aae1}, {0x000554210f1c434d,0x0002ff3445d34069,0x000445f11b6f9f3a,0x000dae4831b39805,0x0000edfbc2a2b1d5}}, +}, +{ +/* digit=12 [{1,2,3,..,}]*([2^84]*G) */ + {{0x000c554483b6753e,0x0006801db8f9d305,0x000886fd9162b06d,0x000476456139bb92,0x0000ea4a8915ea88}, {0x0008d47007a48524,0x000e4818a4c5c808,0x00015a22cdb69736,0x0005f81ca589aaf2,0x00001d936a1d6f7b}}, + {{0x000ab2dcbd28cd7d,0x000a1162cfc4011b,0x000cc1af050d89ce,0x000063b3e8402465,0x000083d0e91f2d65}, {0x000f946464985316,0x000f0b6261148253,0x00038d8cb0463eb2,0x000a2eaf46232053,0x0000d3acff868021}}, + {{0x000dd5140afb7993,0x000bd8f7f093f0f0,0x000c26e525fd4cce,0x0006ac8e6399e015,0x0000a1f765c42a88}, {0x000b1ca956564f4b,0x00094bcd0652818d,0x0009c185081d8880,0x0008442cde2c9ec7,0x000025a44064aecd}}, + {{0x000f5f559bf4a74e,0x00050b61dbdb8ccb,0x0007c812ac18b374,0x0005a3f24ba94aac,0x00004f5dad1bf4c0}, {0x000b7e537d97e833,0x0002de512f6471df,0x00071b5884053ab5,0x00043d89f56b8b9d,0x0000c58226e7c64c}}, + {{0x000f0f17bc532082,0x000fed58cc574ab4,0x000e10b8d86bc64e,0x0007d2b3542fc362,0x0000a757378d0112}, {0x0006ccd629ef57cc,0x000f7b5d9c080f6f,0x00088029c11069f6,0x0000bff2e914dc87,0x00000558c2f6e839}}, + {{0x000da9445a23fc09,0x000605eeb8c005a6,0x000fff46275f912d,0x00078b2667155c28,0x000052569c27476d}, {0x0000ed991150c753,0x0003d9531b755efa,0x00075e9f483c52f1,0x00031dab94d69b7c,0x0000df793c3e38af}}, + {{0x0005af0c0e6e3ad8,0x000fa2bd1fd197e3,0x00019540b108b614,0x0001cf02f81fa984,0x0000838602012d96}, {0x00015ecd5143acca,0x000e93ececd5fb07,0x0004ae6741960659,0x00087c273955714a,0x0000f440efd280a5}}, + {{0x000e7c32fd3348a4,0x000777077b4c8efe,0x000fe73e11c88faa,0x000b96ecece59c8d,0x000009715b464a77}, {0x000aa5d14df28d16,0x0005301318185b7c,0x0005350981bb9857,0x0009644ee837ada5,0x00006a9819ff80d1}}, + {{0x000a414f9b65e700,0x0008351d28b72fc0,0x000807cb5eaeea1e,0x0005c55f09163a37,0x000086eb6b36bd63}, {0x00044698cc2a4ea8,0x000466e5324dbb10,0x000664773c425160,0x0003514ef5e7b9d3,0x0000bfc6b8578654}}, + {{0x000f3a37768912e9,0x000fade19cf47741,0x0004368c97f7e0cd,0x000b78a509039326,0x0000e3961c3955f3}, {0x000d1b5709e33830,0x000bcb212af98fb2,0x00071b5c07cebb92,0x000dab556ee7dd40,0x000022c98a5e5d55}}, + {{0x0007c40f3d55f440,0x0005850ab4bcb8d7,0x0005dc2904c0ecd9,0x000fc094f9552e74,0x000065bde9a45b12}, {0x0001b213e07d2a05,0x000dac491152e816,0x000bbb468a73324f,0x00022eaf5017795a,0x00000333c55c0892}}, + {{0x0002fde94b3766cf,0x0007174a526e10e3,0x00046da5e56aac35,0x000ac2d0c1c51f5e,0x000035efc2472eb4}, {0x000da0a323be075b,0x000717ec17895a91,0x0008ad7c571d2b3c,0x000bf89dbcd26f41,0x00006d4343245d6e}}, + {{0x000dc2eac62fb9f5,0x00084e51c141aa70,0x0009798c2bb7e6c3,0x0002dbd11d4a05b6,0x000074f72cf5e639}, {0x000fd435dd15c827,0x000425e3562b0e0d,0x000886af23f9d33f,0x000b98eee1982216,0x00002d9339f90269}}, + {{0x00046ab0a3564848,0x000023087b4fdedc,0x000aa974dfa3f8b2,0x000c5683df032518,0x00008d37e3dd003e}, {0x000b497bcaf07eb2,0x000a80cfce97a4ae,0x000e8c971e5379fe,0x0009ba740afedac2,0x000087c1f6eafee3}}, + {{0x00083a26aac344f6,0x00074201b2856380,0x0007578c50361026,0x0005b40a8a67be24,0x0000605b8627889a}, {0x000aabd30a19439e,0x000ca53f88dd0bbf,0x0004a3ee818767c1,0x0006663ec91f08b6,0x0000bcc3862cf387}}, + {{0x00084ddfe58cda46,0x0003b276513902f5,0x00002aed28befe9b,0x00072ae88c3c7ae7,0x00004c8c2205288a}, {0x0001cb844e502e5a,0x0000d9ae644c5df9,0x000c1aa51015e574,0x000a6be30714bc7d,0x0000f41ed76cc10d}}, + {{0x00033bb53beeec19,0x00007eefd1662fa0,0x00011a9bf3b2cd1d,0x000d5a31a0c88f70,0x000032688031e363}, {0x00016c08ae3a3eca,0x00015b3281e6f092,0x0008a75e39d7d5a3,0x0000909972d7b786,0x0000c65def428795}}, + {{0x00005874e890e300,0x00041d95ddabcf98,0x000368cbf916f257,0x0002fcdd589ca455,0x0000f2d91af1373c}, {0x0001207d3f06d40e,0x000b5b5367021a75,0x000f49339cc99d98,0x000d3d50f580d82b,0x0000a7749f8b2793}}, + {{0x00086dac65846f84,0x000b8b184a18be90,0x000f50dfc9f9163e,0x000bdfa618f3c6e4,0x0000afe515af4887}, {0x000e16cfa60c51e7,0x0009ee647322d93d,0x000691d42a467932,0x0002e56863b1a893,0x0000c8721a2b5b90}}, + {{0x000ee5987897056a,0x0006fff7338cc84d,0x0003cd0a29f7d1a7,0x000f4c1eeec55391,0x00008625bca9aef8}, {0x000f0fe0533242d8,0x000ddebee7314e38,0x00034542be2dc785,0x000311e6d17be942,0x0000a2343b6a2094}}, + {{0x0004301cf477c5cd,0x0002ab6a3d1b7d53,0x00030cd8103b9cfb,0x00095e6e96f2ce03,0x000038fbea44936b}, {0x000b90a008acc54b,0x000a8904b3609016,0x000c686bd1acda20,0x0000e048114fd494,0x00005ab0f4d4d8f9}}, + {{0x000bdef90ead4f46,0x0007246d293b9ec5,0x000303573d9de112,0x000f45aadc6dc3d3,0x00001af5d644db85}, {0x000f4601e63bbd49,0x000a087a714b4af8,0x0000935870f2f324,0x000f960a6914e009,0x000012ad683418ae}}, + {{0x0003354792bcf5ae,0x00007f41be3c88c1,0x000c142b47e1eca6,0x000ddebdc8cbf3a6,0x00008e5a41a3acc1}, {0x000d1ff364d640ad,0x00071b7bbff3b08e,0x0009b246ca485c01,0x0001f1e241e73b86,0x00008915bc9ac6a8}}, + {{0x000cb418d14bdcd6,0x00009b819b92ec5f,0x000ecaff2142a74c,0x00019578fa6a421a,0x0000cda9a55a8acc}, {0x0006359363a78d5a,0x0000ebce5d6e8f57,0x0005c404f6dc243f,0x00056438b0c481d8,0x000021f1152ba645}}, + {{0x0003438f3e2fc0c3,0x0002778ec5d0bef6,0x0000bab2acfaf639,0x000a613040995c34,0x00009c33c69e3d8f}, {0x000a7896b97bf0fb,0x000d924183025e51,0x000de7e95ff5c9f9,0x000871ae0c7dd05b,0x000034587b2ddc5e}}, + {{0x000a79da8f271b1b,0x000381332a91ce35,0x00037e9d38ecec65,0x0005eaabd737de2e,0x0000827fb571e37a}, {0x0002f6167f95c9a9,0x000a0030b5abafa6,0x00086b6bf174705e,0x0009211f763fc83d,0x0000ded86cb103bd}}, + {{0x000f7cf342886a76,0x000e724e6bc2322c,0x000265932e8929ba,0x0005ba669f694ca2,0x0000767c25712d5f}, {0x000c9821f6a99f7f,0x0005f36eff86ec2b,0x0005ed3486fe21b9,0x00084100857e6f4b,0x00003ba09b776464}}, + {{0x000a956c6538d32b,0x000ce0ebaa99bb77,0x0006e54dfc273705,0x000ca26fc41d2862,0x0000ec59f9c31c58}, {0x0005dddbc05f1be4,0x000eabd6ef105f32,0x00011aa76a1b687f,0x000ce473e0cb49ae,0x0000f0197e08a6ca}}, + {{0x0006580b23d6fb8b,0x0002f22ce000c4dc,0x00088dc5c9518521,0x00070c22efb775fe,0x0000e08eaf386d96}, {0x0000f9e5bdd09ac2,0x0000337c16dba3d8,0x0000363489849e43,0x0001c3c9be67c37a,0x0000d611ceba3199}}, + {{0x0009d284701935ec,0x0004f56adc223b4e,0x000d1eee70422ff8,0x000e16a6f16e99eb,0x00005bde8bd944aa}, {0x000c5e0516c8252f,0x000bdb9bbaf3e0fa,0x000b35adb2dccf13,0x00031f937c0cb137,0x00004bea07b8f8ab}}, + {{0x000bf359cf0ad3be,0x000eb4ce9011d0d0,0x000eac2cdc3b6101,0x0008c802272d1fdc,0x0000ce74ad8a8b66}, {0x000e8fac38d80fc9,0x000537f10c1a619e,0x000432c10ce4a266,0x0002836eede88f40,0x0000bf1a37e07a03}}, + {{0x0002775c8d90f0f3,0x00029d5b535f043e,0x0009d75c2586f7e6,0x000648a06f699957,0x0000735f63f9f6b8}, {0x00040e7726af474e,0x000bd520fc28421c,0x000f156aa1f46680,0x000b48b41f5251e1,0x000071b7c6df2177}}, + {{0x0001cfc4cbfca31b,0x000f57ed1f1f84af,0x000d23dc50912828,0x000085f1b1048a1b,0x000002a9d0f5f8e3}, {0x000caf1497eea594,0x000546369358a96e,0x0007ce467f4e2dcf,0x000867022d181302,0x0000af96b3e41c15}}, + {{0x000872dc000aeb4a,0x000cc793dc01eb7b,0x0004d8c005622aa4,0x000edbf05a0f117e,0x0000cda49f07a17d}, {0x000c8c1f58a0f7cc,0x000e8d9b661613ce,0x000639e32b5f0a90,0x000ecad3e495fbb0,0x00009ea83415cbfa}}, + {{0x000d0c201f5ce32c,0x00016577157dbc72,0x00073d070c744357,0x0002426c39bbf87d,0x000042537dcae3af}, {0x00021ce869d0da00,0x0007377f351b6617,0x000d8fe45a48f43b,0x000cfafa73e6ae19,0x0000ed3189970de3}}, + {{0x000ae762be466e04,0x00021e7720f0ade5,0x0002843c06a19691,0x00013fc0033576f0,0x00004a129decc0c4}, {0x0008e8867fa59016,0x0002d5e9a56b0dc7,0x000237f5a534e8d8,0x000fa748cb4f9d00,0x00005d9983dbb9d4}}, + {{0x0007d12937513371,0x000403b7cbdf14b2,0x000bcfac2b7fb8e8,0x000d9866bfdce003,0x0000db5c06aff71a}, {0x00031018effa29e8,0x00004fbcf727d6a1,0x00022a8553be0ad9,0x000bf7969cfb64da,0x0000e985eacf416f}}, + {{0x0008e834cedc78a2,0x0005c97f5478967f,0x000c186a2a467c85,0x0009469dee8aa6cd,0x00006d837221b315}, {0x000629dbdec3f949,0x000478eadc28ad1d,0x000bcf7c9c16219c,0x000b53383eac76a4,0x0000c5ec8b33a0e5}}, + {{0x00007c0f0e12ca94,0x00018bf91307ba11,0x00071d9ab1a24d3f,0x0007747bf130fd05,0x000079c81e5f13b5}, {0x0005fa3e80924361,0x000c57f2b1fbeef3,0x000248ba50a359ac,0x000af11a98df62ec,0x0000363f004297c0}}, + {{0x00019cde83118148,0x000e64cb25d0fba2,0x000c4ad7b162ac09,0x00058e222b481ca1,0x00000cc2a53bbb20}, {0x0002508860462ab2,0x000b158a37864da9,0x00086722e4f10c86,0x000d14158a829423,0x0000e4fb785f8862}}, + {{0x00027d333d753d0e,0x00099a2600a68bd8,0x0002fa817222a7a3,0x000b8713381a4c58,0x00008f9cd717634e}, {0x0004b7b30e02de62,0x00075c3ea937f197,0x0002821c922ac428,0x000a3c40e3d9b177,0x000003323a2e093f}}, + {{0x000fc05a7e4d4e55,0x0008f1cefec7348f,0x000192a2b46ee68b,0x000353c77d99e99d,0x00009d818e97658d}, {0x000392d3e49f6104,0x000248caf3c8ec9a,0x0003023ca2d7b6c1,0x000b2c8a0f354f8b,0x000074921f13eab8}}, + {{0x000df76d07f39331,0x0003d350ece61581,0x00081c9e57d42e69,0x0000f9cb516a8c9e,0x0000d6335e12edb9}, {0x00090961506c4c3b,0x0005bc7192868ceb,0x0007bcc0497dec67,0x0002067339d0df2b,0x0000cdd06eb70f33}}, + {{0x000f7ed789462fbf,0x0007ef672b19b20b,0x000e33cd81a16b1f,0x000f12f89df2aa89,0x00005dea3af5b77f}, {0x000838649cd0da1e,0x000c5bfcd2a2c5be,0x000c32ff1d3c1e34,0x000f06839599cfa2,0x00007239d0120da8}}, + {{0x0000e69d0c2b5ec3,0x000b79db1e73bda6,0x0008b9c4a330f8b4,0x00035cddc5f73aa3,0x000006f789e69775}, {0x000bfed204c713d0,0x00093bf9b2e7bd72,0x000422f0f0282a2f,0x000a48b205e175a1,0x0000d820622b824f}}, + {{0x000e515a23d90057,0x00022d89b47f4c3f,0x0000548b05e86278,0x0000426dc052b635,0x00005bef95a0b629}, {0x000089b8f8a1e0dd,0x000c7a412f79fcbb,0x0002f0be65d47bba,0x000090db03dec5f6,0x000085fb777a8161}}, + {{0x0007aaf19ae92fdd,0x000b5b9b429c59f5,0x000ef392e01cd435,0x000166a7e522b3e3,0x0000ce727a8ae2ac}, {0x000a5bed9bc9a546,0x000c27594ab2b1ac,0x0008e070e66e8389,0x0007d19f80cafd14,0x00005a8fbc95dc3f}}, + {{0x000cc3bbb7c912f1,0x000bff31beae296c,0x000fa0357fab8a10,0x000dfefdef6e8a9e,0x00002f8fc87142a5}, {0x00042101a914d6cc,0x00008a1ca0911556,0x000418ad615b500e,0x0002d61e09ea4513,0x0000e1df01fe2923}}, + {{0x000e5cc99cf67fd2,0x000ce5fb36c081a0,0x0006e1396618f51d,0x00059123402398bd,0x00009ecfd4e3d980}, {0x000e341bff7f018d,0x000c1f6950d1ea13,0x00091e965892d500,0x0004e189b9fe1910,0x000072842d82d9d7}}, + {{0x000cf638f90465fb,0x000755cc7173d4dc,0x000b1dada810c125,0x00004d9dfedee32d,0x0000d16f02f61958}, {0x000aa8044614721e,0x000015fccdbc214b,0x00003a0873eb45a7,0x000b58c3b0135086,0x0000180d15fc5f69}}, + {{0x000a048c4901672a,0x000b2340b7ff98dd,0x000caf804fb759b4,0x00075f5cad446960,0x0000319c7e6dceb2}, {0x000b09071211838a,0x000d7c9c2382267d,0x000bf6ab01dd36cd,0x0008da89262c392f,0x000032c1ea49aea1}}, + {{0x000deaa5ca6ee65f,0x0004c1800a9c59a9,0x000680184e358c6c,0x0008d4ad86de1415,0x00000b504795d3b3}, {0x000795b139b52e54,0x000ee69f5a0fce3c,0x000b814ff6a9423d,0x000cb742a66e5645,0x0000032d10e19448}}, + {{0x00072569564d217a,0x000429fd0889deee,0x0004319cdca4ef8f,0x000f3e3b6ac95065,0x0000f32517aa4f71}, {0x000b21b117693d29,0x000183555154e948,0x0005077940c71c7c,0x000c0754bdaf53be,0x0000cc60c2cdf8eb}}, + {{0x000b43f7dce0b1b4,0x0009c72824968563,0x00065de307f4a3b5,0x000b7842c8a92f9e,0x00002c0e25912368}, {0x0004adf586027566,0x0004f78ce7e8eca4,0x000a94b38a6951fd,0x0008f5eaddabfb1b,0x0000f4735e8fe842}}, + {{0x0002dc3c7e725420,0x000d3f88fcc2867c,0x000ecf5483ea2af7,0x000783793f2f306b,0x000045bad8bfd57e}, {0x0002ca4b36818218,0x000835b27feb5f63,0x000453187115ad6a,0x000070eab7ca7b7a,0x0000719ffe4782c5}}, + {{0x0000593929af0d40,0x000e5664a3d91ab9,0x0001e9eb544e57b3,0x000d2f624b4757ae,0x000032347a13f3b4}, {0x000d1da32f451911,0x000266acc7aa2e45,0x00091a4d237ee7c0,0x000dc21ed6afba7a,0x000010b7c3ccaa69}}, + {{0x0001d7c699cfaf4f,0x0007ca6c922936b5,0x000449a44405d498,0x000f2f24d9456a6c,0x00003f6e5ffdbb14}, {0x00037c452dfb4b57,0x000926a4b56d7971,0x000a9f7ea115ed0a,0x000d5afaa27d8ea2,0x00003adc089d4a10}}, + {{0x000d5a0f6239d756,0x00018c596b9e37b8,0x0003af1688d17ce9,0x0005a37b04b307bf,0x00007aa9f6053a7f}, {0x00074342f5e14850,0x000fbae0c42ca37b,0x0000e1ec491136fe,0x0000eb4e610854f1,0x0000f0ebf7930ca3}}, + {{0x0007b6e75a63ab1b,0x0001d7cbd366e9fe,0x000fda58c355abe3,0x00062d68e49464a8,0x0000bcb1280d5e57}, {0x000777d197806999,0x000a9ecf9715de06,0x000f6afd589c8aee,0x0004af4e62d3ff71,0x00006b88426b88bc}}, + {{0x00074d1d902ef8eb,0x000cdb0df8d05362,0x000e73dab4f8bd22,0x000917798e1bb263,0x0000747ec50844e3}, {0x00053692d52a05db,0x000f6ca8d9b29727,0x000f02477bc17e87,0x000f3a1726632e3b,0x00003ce08d4ab050}}, + {{0x00048d2262daaba8,0x000aa837e39f0f7c,0x0009320f1da31454,0x000ea364915f9041,0x00002e8921d77b9b}, {0x000423a958563826,0x000a3497921fbf0a,0x0000461a0e93409b,0x000ec5b86536c442,0x0000775d4ee0d067}}, + {{0x0004ed2cae04d7ad,0x000171ad7cb36930,0x000d2d311bc08ed7,0x000f695e13619e36,0x0000d2e0eeab2d2e}, {0x000ee6772c62de91,0x000baa1b52e55366,0x00004bce2a244410,0x00054fc30c34b0bc,0x000005c73893a90c}}, + {{0x000cccf4a1a0cca9,0x000722bdc2035d63,0x00057bd07cf346bb,0x000f316c79ae4fd5,0x0000ac63d30daba2}, {0x000ff39d838e4cd7,0x000204741c2e1f2e,0x00069d840c3461d8,0x000a84505593db6d,0x0000741dc28c944a}}, + {{0x0000876a8b27f44b,0x0008bb19870aa30b,0x000f8e9ec31cc2ec,0x0004c4e60731153a,0x0000c55b52b0033d}, {0x00089b1ed9e1929b,0x000e314409183cd1,0x000e19b9a4f6719c,0x0007dd490b7456b3,0x00001a8b428feb17}}, +}, +{ +/* digit=13 [{1,2,3,..,}]*([2^91]*G) */ + {{0x00023862936ccd46,0x000c9f3bb8a5bfc7,0x000ca23157985ce2,0x00081bc9abd0a0d9,0x0000af5b9895adc3}, {0x0005faaee4832a45,0x000d59f64358253c,0x00046da055a2f27c,0x000f568dc06afc75,0x00002ba011d7eeb2}}, + {{0x00055fc1cf25ca24,0x00020e1d348934fe,0x0002536cb0ab57ca,0x000233ee745fbe8e,0x00003e16abce68d9}, {0x000e1c0522422409,0x000a5979e6309e7a,0x00066b196881b4f7,0x000c1edac5fd108b,0x0000142c6726f6ee}}, + {{0x000c2137390618df,0x000be970f27a0fee,0x0002f4c961a17241,0x000f8a198a977f70,0x0000195907ddcf67}, {0x000f379501c8471f,0x00059ed6d515c9a1,0x000bb63a198210a5,0x0001b9124b243403,0x0000ed746bd02b9c}}, + {{0x0006e74ded67841b,0x0004cc8fd37f409b,0x0002803b1af55986,0x000c3bf000b91399,0x00007ceb09bcb37f}, {0x000f8d459310a1cb,0x0007593dba573a77,0x000ed294ac4fff32,0x00092052bf2f7fbb,0x0000fe022e7f7a32}}, + {{0x000c7a6ac040f2fb,0x00055a54b073122b,0x0003e7acd6ce751a,0x000107d66c28518c,0x00003aeca2c1f0ec}, {0x00033bbbdafae2cc,0x0009f8fa7af36884,0x000dd647c01943c3,0x00009696d6c87579,0x00001c20d14c8db3}}, + {{0x000a61a0c68f6032,0x0006a9123a83491a,0x000bd0c5a9dc2dd5,0x000db59a99946a29,0x000081228cdb3c43}, {0x000cd2ebd0599303,0x000f84227c8ed033,0x0009053e94a3bae2,0x00069f434a9ed1b1,0x00002474f03f1410}}, + {{0x000126dbaaed16c2,0x0009899463a3470c,0x000eb820a2fb87f6,0x00066fd977557dc9,0x0000a9950295b8de}, {0x000646dbf257d07b,0x000e7a4e19ec84a4,0x0006a7eaee3f42b8,0x00025c05ee05cfcd,0x000057200c018815}}, + {{0x000d1bc968cdc161,0x0009f62c0803d083,0x0007eca12854bc1a,0x0008683a5785328d,0x000052551f11c77b}, {0x0005bb2ee4edb660,0x0002b2b5a388ab4d,0x0004ac3bf9559546,0x0003650232668dd4,0x0000b1b0d76d1f17}}, + {{0x00004047d03bfb04,0x000267ecceeb6096,0x00091101feb19ab6,0x00010ef35c6beedb,0x00003595d5f45735}, {0x0004e8ad3b49694d,0x000206d570226401,0x0007f45f06b32f3d,0x000807cfb7433dee,0x0000c15e03794afd}}, + {{0x000e7a6211af3d16,0x000c95e02ef4989f,0x000bbdf2c96b6185,0x000b8ad9fd10ece4,0x00003291adf91780}, {0x0003b209aa3794c1,0x0008e8ddaca35358,0x00074558fbaca474,0x00087d9bbf1f277d,0x0000f61a073f7b35}}, + {{0x000d8adb85ee3f83,0x000ebd78bd46cfa0,0x00011af45006aa9f,0x000c03cb789b856f,0x000038718b3de225}, {0x000b87d218ea96d0,0x0008296dd64f240a,0x000a55d762afba77,0x0000c106f6b48540,0x00001637a1067f29}}, + {{0x000fe903a2ddee0b,0x000c2147304ab5b6,0x000bce46578683be,0x000529212a7403d4,0x00000a0c11ee7628}, {0x0002eeb8c98226d4,0x000acd7eace9cfb7,0x0007ed1f6bef5b8a,0x000c572a94bfb860,0x0000fb6e6fd9f827}}, + {{0x000129845c7d39d8,0x000c08fc8d104a0a,0x00091c5df9c60f44,0x000c84addd4d5bba,0x0000117fecbad4ea}, {0x000e5b606f925139,0x00040714a55c0cef,0x000e33820c14e7e6,0x0000982da6e9998e,0x0000df370032e21c}}, + {{0x000f523358513503,0x0001fd64adfb3997,0x0006889ad5e8f89b,0x000efd415f1bcdca,0x00000dc5193d322a}, {0x0006d6646cc33bcf,0x000bc53d4982b948,0x0008f8e8fa9ce266,0x00054acf5820c0d5,0x0000307c2eb0ed58}}, + {{0x000613f329e775c2,0x00074e8d77c0eda9,0x000f3edcede373e6,0x000303ea32e7c5f4,0x0000ace4b9dce770}, {0x000d62a55c54561d,0x0004408d9c3361f9,0x0005d18ac4c51c97,0x00082e094177a277,0x00004c611cafa47c}}, + {{0x000f298db093d970,0x000d8fc659552b85,0x0004088b59c6d6fc,0x00017f0a2d06747c,0x000086eba0701cbd}, {0x0003e9934facc02a,0x000b5a33252cc71a,0x000700d0a64d5a86,0x000a74776c01c506,0x00000d58cf6a3492}}, + {{0x000e24841127be4d,0x0006109c01c62f85,0x0009d1e24f0d7950,0x000f9906c5f8998d,0x0000d14baa86e1ba}, {0x0006495cd2b7045c,0x00079b660d67c95c,0x000b91fd538bc207,0x000698f1bfde9d01,0x000069d46f82ef5f}}, + {{0x00054cb598765316,0x0003c6ee2235eecf,0x0001d05682fcdb79,0x0000cf6e6100f9ff,0x0000dc6262cd1fb3}, {0x0002721130ab0425,0x000fd66b83432c4b,0x00068d04d701f922,0x000c56e867326ed1,0x0000fa2450fdaf47}}, + {{0x000e18b2b572c025,0x0007aabc185629d1,0x00011115af0a8c3d,0x00059e153c5e496b,0x00006571350e5f65}, {0x0000804bcbbda06e,0x000ba4057dba77c0,0x0005a19dc3818e66,0x00007db6d31ee3f2,0x00008435030e7d57}}, + {{0x00046d989cf9775b,0x00008bbdbfa6a9e1,0x0004e3b35a8bc449,0x000972cffc49e7a3,0x000053ed79153d43}, {0x0008b080e14ff257,0x000ace79b7fe22f9,0x00059f0939c733aa,0x0000aeb72dd048de,0x00002619ceb8b79a}}, + {{0x0001e4f65ecf7496,0x000ae8ee251022a3,0x000790a0b8ce447b,0x000e5b7dc58d2831,0x00004c7c03b41e33}, {0x0008b362b8088d3b,0x000cbcb87bd9c6d1,0x000cb48e67cf5c74,0x00069f85efc38c53,0x00003fa36fb3ee0e}}, + {{0x00043ceed8105788,0x00090734b644bac6,0x00036bb8ea2711c0,0x000b3810041eb459,0x000093c69a1b4b83}, {0x00084710540e3d01,0x000d719e03fd4f77,0x0005e622babf321d,0x0007b11922ccf87d,0x000096e359a83944}}, + {{0x000fc5fc73da9b5d,0x000b1595fc9ab291,0x0005a95066904c34,0x00015959fe032bdc,0x0000b4b92b11f12d}, {0x00034add676f593d,0x000519f1f0786c3b,0x000697738b1c970a,0x0009584fb4963dc9,0x00001084921d81d4}}, + {{0x000b5da661dacece,0x000f6b0cd23bd2f4,0x0001cb076a96cfbf,0x000ee755e6ddc3ce,0x00006cb5574c3f61}, {0x0003a2123b371d00,0x00074a4a55ca4cdf,0x000b8d605f7131d5,0x00082c751d8514b8,0x0000089e6493eacc}}, + {{0x0001884e243e59de,0x0008f13be7d12276,0x00090809ee25d4f7,0x00092c13560a056d,0x0000cbd0fd7048aa}, {0x000c7cf11455b365,0x000880bcf192c1c8,0x000ee71e5bebb543,0x0001694d7bdb0baa,0x00000e46da8bb58a}}, + {{0x000ef46c771c87f8,0x000f7f262a6e438d,0x00050eb2f306408e,0x0009c80d5cc4d430,0x0000385ce31c91e8}, {0x000059abc1a67c19,0x0002e387ea4cab0c,0x0003e03c8b9bbf51,0x0005c84742501cfa,0x0000918e418e3d1d}}, + {{0x00022bbef09486e0,0x00010f5c122211d7,0x000b2de7adf0502a,0x00049b5ef8799064,0x0000cd147024c20b}, {0x00090d67f2d70c08,0x0001cc8fbaf0cafb,0x0001547cd4a1a267,0x000146a52fc4ab4f,0x0000fe8ec3c3592c}}, + {{0x0007c5d6d28ee3f0,0x000c1f5e25a2d30a,0x0005a067148ab16a,0x000b0e3e550cbb99,0x0000cdbad202b2a5}, {0x00093875ccf1d8a9,0x0005756ac2a6bcdb,0x000720061b8c60e4,0x0005ffd19c95c274,0x00006cf02b509a1c}}, + {{0x00091e12337f16c0,0x000c7393588f049a,0x000c7530b5c0408c,0x000e8d167beb1eb4,0x0000bb3b2a2771f5}, {0x0000d4b2f53140be,0x0001dd263d66660e,0x000799cccb650354,0x000900827c699d37,0x00001ecf8af970a8}}, + {{0x000b005909514a74,0x000ecbe20fc8996c,0x0008fe96ae8999fe,0x00053c02484ea215,0x000081ece97b1b2e}, {0x000b646f679d3fed,0x0005b8da2343f2fe,0x0006cabdc877b5b6,0x0000406a1d29230e,0x0000aedcbd3b6bca}}, + {{0x00043b54f4b39c27,0x000d249a5720a07d,0x0002cb9000f597f6,0x0003ab0ac7a0dc36,0x0000fc9c13cdf3e6}, {0x000c0eb22fd82d5c,0x0003b8a81a1aedd4,0x000364a7c0e3f20e,0x00046fbf635e9939,0x0000c199b407168e}}, + {{0x000014cc44058054,0x0004050153cddf02,0x000dba3274172b05,0x000f0217ddb34198,0x000092ae38e923a0}, {0x000179277af42fa1,0x000c90d55c033ee8,0x0000a9f8725ae26f,0x000fd2debf1b8171,0x0000ce7e032b9a99}}, + {{0x000a44861b8554c2,0x0000a3a94aeb35be,0x0003e71fba83aa07,0x0001d959d8dc768f,0x0000eb913a4c91a2}, {0x000e183a28b0b33c,0x00088becfd84e460,0x000c958afe20d5c9,0x0004b5d299882015,0x0000649765aefe9f}}, + {{0x000230fd8633e864,0x0001f08ee9e87f37,0x000e6f193bf5b4b2,0x000cde4fcb7532ce,0x0000104b82017ce0}, {0x000132f59550b517,0x0009ea49ea36c092,0x00091a39ec7cc727,0x000b70bfe7e175f2,0x000021a2ab8b1372}}, + {{0x000eab9e1005ec21,0x000f75e979b06cba,0x000c6a4d8a9df400,0x000a67cc9a2af77f,0x000035cd470f6031}, {0x00096537b57815e9,0x00097e2741ed2894,0x0009a6fac39520fe,0x00003d6cc06b9003,0x0000eee76b0b2e06}}, + {{0x000f7b48f6edb04b,0x0005a929e3038237,0x000f7857536e7673,0x000901ecac573596,0x00004eb199d3fae8}, {0x000d3cac11a43ab4,0x0000d55fddc2f291,0x0007aa2635d37480,0x000ab944155f4673,0x000043238379f1ee}}, + {{0x000de044927b3f1d,0x000241bc54ae6068,0x0001082ef75dcd95,0x000d7d5dae8d8aba,0x0000eb60878f5e1f}, {0x000c905f7e55fe92,0x0001e240bd8a7772,0x000081d2630467e4,0x00059a6904f58040,0x0000ecaa62c1a4d8}}, + {{0x000e9f68575333da,0x000f00ecfae86bca,0x0009f8c83e13f2bc,0x000c5c3e371a2ec9,0x000037a85cb421a7}, {0x0005b5a329bbed02,0x00045419c8fd74f1,0x0009823120e28310,0x0007e3cdd91afbd7,0x0000839b1b682506}}, + {{0x0006e9d3ccc341ba,0x0002150f316b25f3,0x00080a38e4c7c7e3,0x0003c01f7e4d50a7,0x0000c3ea0d40f57c}, {0x0001c7b30f55321e,0x000e7d20d94bdaea,0x000a029fadf45df8,0x00097ee167f6b768,0x00002aa6cc9459c7}}, + {{0x000c17a91f1ba0e5,0x0009787593f0d015,0x000f9ce851544d29,0x0000a7d6fd2eac07,0x00006e42a2d03161}, {0x00047f0cf39d2dc2,0x000e394ce9aef618,0x000c79ad7bb38167,0x0007952cfa312fd5,0x00006e9508b6606b}}, + {{0x0004a9337b6501f3,0x00038724b1274086,0x0004b9735e20280a,0x0005a7b1685a2355,0x000049e5df053c39}, {0x0003d8976fcc51f6,0x000f211285be64a1,0x000a5a1dce61af27,0x000534f6c0f84385,0x0000330e4e187016}}, + {{0x000e341cd1abcafb,0x0009212e247ca60e,0x000d11cf4928ec27,0x00068275c242a86e,0x0000701b7f29cc07}, {0x000113f0cb2d0c5e,0x000a0e15728ff118,0x000ce069dd37a910,0x000e33fbaf062392,0x0000da9bf4139bfe}}, + {{0x000d893ecc8a3843,0x000aee3422fc03c4,0x000bc5cc45125f34,0x0003fa03d850b732,0x0000d9db328e612b}, {0x00093a198c9fdc42,0x0008d59f8b1b219f,0x000cf49c0b2febe9,0x00075e07a055d8a4,0x000070db58a14b84}}, + {{0x0001b49f9f74da4b,0x00079c277b30b6ae,0x0007a713eb7573ce,0x0004bdf55e41397d,0x0000c4c19bd84865}, {0x000931c026a86997,0x000de2bca5cf171e,0x000656b2300e38bb,0x000d5f65e8930d8b,0x0000ceba324ca80b}}, + {{0x0000f392145ceee1,0x000cec5519c0f83f,0x000ccf35a80091ac,0x0003f61fdd589fd8,0x0000aeb32670044b}, {0x000c35391cc68921,0x0007d7a227dab6a6,0x000a75a4bbb2c563,0x0004940e0441f13d,0x00000acf8254d41e}}, + {{0x000b7c2475bc8687,0x000275d5d3d4a8e7,0x000998b4c20aa04c,0x000a5a3f2c0804e0,0x0000f72faa30ab18}, {0x000cd37890ebf796,0x0000909c43568186,0x000ac20ea2015287,0x0002acdab79ef3d0,0x00008e23170d603d}}, + {{0x0001b7430856e702,0x000e326886326e48,0x000c9d6b951f2e59,0x0004074d31e623cd,0x0000e0b17d35a649}, {0x000830e9a2c52388,0x000c4af19a13dffa,0x0006e6405881b51b,0x000b8fb31e1b7bc8,0x0000bc0283606a6b}}, + {{0x000cd1399b0fbf20,0x0003ae7bfaaa7476,0x000515068398a2ea,0x000ccf4a67748a0f,0x000052c9dbaa0de7}, {0x000b107df0bab755,0x0000bf5cd043bd9d,0x000ca0aad0f46f5d,0x000122d95fa8e659,0x00004b88235175a9}}, + {{0x00063c7b5bcf7a13,0x000e4a0fcad630a2,0x0002e5101cfd5a29,0x0009bffc7f24c1e2,0x00003d1eff35a181}, {0x00005715a33e4b08,0x000243d6669d649c,0x000cac16fe982911,0x000ca0d08374e968,0x00008326799f5651}}, + {{0x0005bba6afc5ab95,0x000f1153dd46fad0,0x0005a6b8ed84540e,0x00031d26cc136a2a,0x00007c4b8712243d}, {0x00093b3f4bd9ae24,0x00099ed80b28abfd,0x000a5a784db8f876,0x0006abdfb9bb8a47,0x00007c9037ec1306}}, + {{0x0008517c194a6c3b,0x000ff8b4e080bad5,0x00095ef1ab25163b,0x000fd601840b3f70,0x0000f57e65fa20a0}, {0x000761f8718a4635,0x000026a37b1da3f5,0x0007d62b14f1d622,0x000a97dd8bb7dfe2,0x0000b65d15c8388c}}, + {{0x000fb630f9f3c36b,0x0003514f1507f908,0x000709730fc5a09d,0x0001a551491ecda3,0x0000eace9768d59b}, {0x0005fa0c62ba40c9,0x00057a73f9217bef,0x0000e19c42c546db,0x0005238d4079ca10,0x0000ca0c26f55313}}, + {{0x000ef817c3b2618c,0x000516c3678b7d65,0x000af4eab31e6d46,0x0005e838846fbf7e,0x0000cf31a0360563}, {0x000d8d7395cabd57,0x0003bfdfda36daaf,0x000e52df3d83d332,0x000c0c667a2f6684,0x0000e1a16cbedcfb}}, + {{0x0009566376fcec40,0x0004537333dd7cc5,0x00051d3c0941a4af,0x0000dbfb72206ff2,0x0000e6496b4df3a2}, {0x000340da7a61207b,0x000de24d61e2c6b0,0x000afa106554c6de,0x000514a942394f93,0x0000959becd8bae7}}, + {{0x000e7c288392edd9,0x000e09e87173c5d5,0x000badf27f43f861,0x0000875f56cbbdc6,0x000059011eb4dd0b}, {0x000b4176f01cea75,0x000d52c23a0d6c81,0x000f0b5d771b4b78,0x000415b0fd29a184,0x00006d4d765d5cff}}, + {{0x0004fbf6e5ab1c10,0x000c5e16117aed34,0x000068ab71d61e70,0x000d5853f76aac64,0x000041afad97ee21}, {0x000f2590f3380ea0,0x0000057fe06e1782,0x000ac009aab5f995,0x0008523a89866ffc,0x000010ad3182779e}}, + {{0x000edf9d8e340aa5,0x00013c432c9525be,0x000c5b0938e52347,0x000f7f2b547dc9ce,0x0000337edbcd4751}, {0x0005f8d089a8d34f,0x000849d94da27c6b,0x00021344d3091355,0x00019ae8dccbb520,0x00003db4afef5bb0}}, + {{0x000221483572979c,0x000f488d38f240cc,0x00001d197443feef,0x0007cba0ff6fdd65,0x0000c8ed1a1e245f}, {0x00043ac345ea9f35,0x0007ed4c13607cff,0x000ee8dc785c5171,0x000ce453e3b03a22,0x00000d9cd4fd626e}}, + {{0x0000e350ac33d68a,0x000cd316d7ea5a59,0x000d601e37ae8df2,0x000a613a4ef1b40d,0x0000b969a55dfd9c}, {0x000b7d820baae471,0x000789e5546e638d,0x000adea90d131722,0x000588b4bafc01bb,0x000020136f521961}}, + {{0x000719f9325b2400,0x0005ba8ad2510bfb,0x000c18046ed327c0,0x000dae7c4bb581f8,0x0000f5c43bf2821c}, {0x00064d2b8967b272,0x000ff2fc977093bf,0x000e43bcf14f6a54,0x000c6150374246de,0x0000edc1c38954d2}}, + {{0x000d28e4579a1c60,0x000a23ad0bc91684,0x000159efe0e89838,0x00013c18b907beb2,0x0000f982933523bf}, {0x0005a3d80fcaa3fd,0x000e71da29914720,0x000260190947110d,0x00044d6527935a8d,0x0000b8cdec4b3dda}}, + {{0x00078460ca8fa191,0x000f5396fa5eb6d6,0x000f8451a2349560,0x00008834935f4c4c,0x00009300128571d5}, {0x000dab31e4ba5a79,0x00014b60c848233f,0x0007e000ae8f45d3,0x000156c4a44930f5,0x0000e9ee0a99bbad}}, + {{0x000d5403bff252a4,0x0003c2c93f8b1649,0x0002ae1a111254e6,0x0008a652e0510e35,0x0000025872deaf32}, {0x000f49fcd577ca3c,0x000a66df694207c9,0x00088c106ad78c7e,0x000eacacb6deef66,0x0000ae91691d5244}}, + {{0x000a1a6556a90c69,0x000f247842e0528e,0x00055287f1f99c51,0x0000834918eed56d,0x0000addf95032778}, {0x0001770d1ffbb4c6,0x0006ee9a94d1ca36,0x000a5c9aeb4f8224,0x0002c9a6b1ea0ae2,0x0000815b08430763}}, +}, +{ +/* digit=14 [{1,2,3,..,}]*([2^98]*G) */ + {{0x000dc06264c6edf0,0x000ee76ada9f8a5a,0x00093a29ac0d4b89,0x000bd589805d8eab,0x0000a3f857d466c7}, {0x0008cdc0af4d7692,0x0003a4587e2c7fe4,0x000068fb7ac8249d,0x000cf84bb97b33ee,0x000025f782d1e221}}, + {{0x0007b344d830d611,0x00090706b372f099,0x0007c5d82081ecfb,0x00062e5f1d371461,0x0000630e68f8309b}, {0x000027ecab0d0e7b,0x0008eeffeefc38fb,0x0005a3f00f139e51,0x0006235555687383,0x0000000539130fe6}}, + {{0x000a7d994abc3339,0x000eada42542f0dd,0x000d2774877be20f,0x000d3e01fd92532d,0x000045b905b58c4b}, {0x000f4f6018764c26,0x000b906b0e9ad737,0x0005a7ddd213dad2,0x000979cb47e139ca,0x00007e5ac0e28093}}, + {{0x0005bb244b1a7745,0x0004d3be15ecec3f,0x0006b92215c50b54,0x0005c2343f4af69d,0x000061a0fb33d6b8}, {0x000996c8fb6c5f4c,0x000f7c15d9d2bd46,0x0001d8db7e469c42,0x000997e76e202eb9,0x000079e3e7b012a6}}, + {{0x0004e3dba4f34c34,0x00037e53915042c8,0x0002eb606a8d77ef,0x0004eacde90607b9,0x00001bf86bfff708}, {0x000dcfe17dc51185,0x000be051f4755508,0x00005d090cce6cc7,0x000d86e74d036824,0x000003082999813d}}, + {{0x0008952a021a9b5f,0x000accd265a08868,0x0006f5e0a1762020,0x0001ec077c5bca7f,0x000051a150e7a0c8}, {0x000c4dedb17071d6,0x00091d7d3367b9a8,0x000cb4a59fc13868,0x000144624e29dd68,0x00004aa6d75f039a}}, + {{0x000a3fa97e6e0640,0x000d7477c822e5cb,0x0006d71cfa74fbe6,0x0003b6e7f42fe5a5,0x0000e15e3b313245}, {0x0003747e55840168,0x0000f0b46d9f18e2,0x00012b245efd94b3,0x000e0e8bb7ed74b5,0x000071a6634049eb}}, + {{0x000afafb42e29add,0x00018d9ad91573de,0x0009c54ad6e3aa89,0x0007703a1f57b2de,0x00004626948ca4b6}, {0x000abbb815e6ca50,0x00035b40f7f54940,0x000a08f33a48dad8,0x00084e31e14799ef,0x000095a109d11471}}, + {{0x00002547eca94470,0x0004a32c671602cc,0x00040116a86806e6,0x000e8db86a0dfb11,0x0000467b5c6238db}, {0x00065c7b3957d267,0x000579251baa175c,0x00083f53398cd355,0x00066bef5f51a807,0x0000599690598dba}}, + {{0x000a1e066f254055,0x000e6ae221534d5b,0x0008d3bb91428e24,0x000b46795bd47970,0x000023f090e0e98d}, {0x000018f58c20dcc8,0x00068a6f249e38a2,0x000915d4db72af54,0x000729b41ac98972,0x00001f3ec32bf5b1}}, + {{0x000c30853828ecc8,0x00003ae49031a5b0,0x00016d35710f0304,0x000166de6aa30f87,0x00000b38f0b6be3c}, {0x000422386ba22348,0x000d2a6b4a859201,0x00069a2f618226b7,0x000dabb6b9065ae6,0x0000a1c3854e2ba7}}, + {{0x00086a14823ea47f,0x0000554a1c55f1c1,0x000a974aa782f893,0x0008ae49b3fe8e67,0x0000129a2c97602f}, {0x000d903dd777718a,0x000e76223c4293a4,0x0009a1637ef4efe8,0x0000b2f996c38d78,0x0000dfebadce6272}}, + {{0x000d950e92810589,0x000774dfa90b0136,0x000039faf2b175ce,0x000e185a260a83db,0x0000561f8e4b6c93}, {0x000ebf0e2fe28b50,0x00034acbe16edd05,0x000320d07af8414a,0x0004430be5fad27f,0x00000ee98f0868da}}, + {{0x000cae8c6791947b,0x00007996853c4c82,0x00019a0f47f2509f,0x00004935af953959,0x000027cd71f58bb6}, {0x00083fd8bc62211c,0x0009bfefe89d61da,0x000ddddf3c87eab0,0x00047e03a5e0f6f5,0x00008b3c456c5505}}, + {{0x000137df36f714f4,0x000582323ed21aa9,0x000cc68386a00b41,0x000bbd594ecffcce,0x0000cecf5b089abd}, {0x000983eaea96fcfe,0x000b24d12f6ae33a,0x00053fa16977b26c,0x0005befbcf594bb7,0x000088d8f7495883}}, + {{0x0007b8acedb9388e,0x000400efbaa8ca48,0x0000a06b17cd0588,0x000a7400dc760aec,0x000015cdfa858e76}, {0x000e7f3b8048b0e3,0x000ffae832778b42,0x0006b52cfbc15e64,0x0000d578158f0142,0x0000af7e47c0508f}}, + {{0x0008eb22823149ed,0x0009745fbd4f13e8,0x00047180965d7326,0x000b40df9c9b9a8f,0x0000c9f0be3c6954}, {0x000a28f9b361f46a,0x000c072e07a5a9d2,0x0000f2ed0ac3a2dc,0x000392aaeebda466,0x0000336c3e9cacdf}}, + {{0x000d1d43afceea08,0x000616a9cef394de,0x00024432a5bde0a6,0x00043191915f59c4,0x00002f62d7e1fcac}, {0x000fa75ea0b786d5,0x000a12c9de645b54,0x000d5287986117cc,0x0002a68bb817b967,0x00001ceae3eadae0}}, + {{0x000b7ef5b2d3b2f1,0x000000070d048432,0x000c6c03e4e7e07b,0x000c7ff417809ed3,0x000025fe6da4c479}, {0x000e17e440a00118,0x000671aa9008cffe,0x000c32ed6a56c499,0x000f577463eea607,0x0000831162ce69d7}}, + {{0x000e4805943376fa,0x000b1eca3608d8c9,0x000ed34782092af8,0x000ccd599e1ef20e,0x00008d7322137ad2}, {0x000f359d5c8c0d56,0x00038757adcde109,0x000514301b31564f,0x0003bb5fcb028045,0x000004f028be60c4}}, + {{0x00058323cd637177,0x00028200d4cf1cd6,0x00046e38835c4220,0x000d0b6ca712117f,0x000024a80f8a1923}, {0x0009bb310add934b,0x0009b0984f44f959,0x000603516ff5e5c7,0x000bbee13c21257e,0x000090eda9f674c5}}, + {{0x00005458fc6dd126,0x0000947af627b5ab,0x0008f966ca449c6d,0x000b0cc003e78cf1,0x00008256aa596830}, {0x000a0e079d35b70a,0x0009940a5f36cb04,0x0000c3f2d26877d1,0x000730ed00d82660,0x0000ab865b6523ff}}, + {{0x0009177f2d61f6a6,0x00066af811bf4a06,0x0003b1c247bd378c,0x00062465ea93b39f,0x00009171cdde60f3}, {0x00095252b216763b,0x000749e6bd882dfc,0x000ff92142ce99f9,0x0000524b31478981,0x00009b67920e8098}}, + {{0x000f5eab873e5982,0x00002b05e8702523,0x0009e3071bab7eec,0x00049dcb63dd8dd0,0x000065a0588f39b8}, {0x000f19f79a322046,0x000b6ebf2f4ddf97,0x0002dafc0af5691a,0x000d2c5d445ac1d5,0x00000c9570bf9541}}, + {{0x0008c920c8a7eca8,0x00034d2fb0311be7,0x0001335f7aa6a1a4,0x0005a8062bc8fbb6,0x0000934c72c41287}, {0x00035614cdaa4860,0x000b9d9596f7a99a,0x0008ec39c47104fc,0x000fd127b936a44b,0x0000930bd26ecf89}}, + {{0x00031bd38f4959f4,0x000209f43efcc5ff,0x000222f36c2b77df,0x0005f4ea5ceeacfd,0x0000dad55f834b8f}, {0x000abbbfbb9bf288,0x00085dc8869ed851,0x000c867e63c03f7e,0x0008257f770d1a05,0x0000ee1f000730c6}}, + {{0x000b8711de100539,0x0006ad95dfe207b6,0x0006cb512680143a,0x000f506e2c18fddb,0x0000bce3ca0378e8}, {0x0004eb8130d57fba,0x000fa5840b8715bb,0x000c84823eaae39c,0x000b26629e9d3145,0x00003031a8c8c1af}}, + {{0x000fec3fb7df9633,0x000f793017b5239e,0x000d417edfbe62de,0x0003a2bd4ed6d43f,0x00005cfa222922a4}, {0x000cd26d9111ed55,0x000332b5b95f6de9,0x00094dce34d4e23d,0x000ea1652aaebfcb,0x0000a74a2880fcc4}}, + {{0x000b85e6b698c60f,0x0006bd05bb9009a9,0x00080f4cd386aba1,0x000414fc79987378,0x000022a41cd0ade5}, {0x000386b37f02931d,0x000144ed3b36adc8,0x000831113fcbf024,0x00062655c85d148e,0x0000abf8c2abb579}}, + {{0x0008a280beb3d98e,0x00022e4dceaef7f5,0x00019e2cd5a2c67e,0x000d97b0a1fc4ced,0x0000ccb55ec7430e}, {0x000927ea8db4dbb2,0x00054287f738760d,0x0009e0694ad868fc,0x0009547719050c59,0x0000b8bd54ad1b07}}, + {{0x000b919915296235,0x000b9440eecccfab,0x0000df1d4f7f9db7,0x000fd803b66d2b79,0x000021f810f61fb8}, {0x000374ab0babbfca,0x000b1da810b6567a,0x00007019ae31b3a4,0x000e6dc732421879,0x0000aee1fd5d3d9e}}, + {{0x0003dca799b1e8c7,0x000498f64369d75b,0x0009b0643f842b8f,0x0001b609c435b8d7,0x00009e18a4dc6dd7}, {0x00093f286cd43de1,0x00000522ebfb1d7b,0x000981ef58ef5f5c,0x0009a7c511d1c979,0x0000079e84ba8d07}}, + {{0x000e22bd3c256aa8,0x0002fd4c94bbe5d5,0x000ad29c666de35f,0x00035e3b865874b6,0x0000224e3c128523}, {0x000d5f4d614945e5,0x0007845bb7831b45,0x00099e134cb89357,0x0005ec39f9f5f9ca,0x00002f887fca2984}}, + {{0x000ef0e4412d59f4,0x00035ee9036e0f4d,0x000ddd0c48d53893,0x0000fc472a1ce74a,0x0000a17b9b8404fa}, {0x0003f18afac2a705,0x000d43fe74db5f3c,0x000c2267174d55ce,0x0000895e0f14e108,0x00005dd34a549d92}}, + {{0x00026468b946678b,0x000bad393b4640d4,0x0009805afa144f1f,0x0005586d58b328cd,0x0000b5a443e9ccf4}, {0x00058637fc0a22d1,0x000e069b47544588,0x00019ddb88467c90,0x000d6df432509399,0x0000e17ce7f02ef7}}, + {{0x00091620e2b9010e,0x0003b1a55e8e98b3,0x000c2c0459f0e8a3,0x0001e1dcfff42fc9,0x0000dd65a82c1053}, {0x000a151f4aca9977,0x000c1b7e25a6edab,0x00038c1f361d8bb1,0x00000cd30261823d,0x000058c4491cd6bb}}, + {{0x0001b0cb59285f27,0x0000f4ffe44d5c38,0x0003e8dac0335ca3,0x00035f7325d84a82,0x00005079b33341fc}, {0x00012eba3b28d558,0x0004bcc94ac65271,0x000e2e85b998a96e,0x000224df60868980,0x000070d7cdb23165}}, + {{0x0004ee5b665a2bf1,0x000dc11697de8321,0x000e9fb2b99d4d00,0x000b6248a3b6b7cc,0x0000ebb83a1c6e31}, {0x0000e7f18e605d33,0x0006d595cf04d7a0,0x0008c209f036abd3,0x000f7b5e7c03f37f,0x0000202f91c41e8d}}, + {{0x0005fed58160235b,0x0004f5fcdab49aa8,0x000ad6f5b0d663cd,0x00046955fcc1cc31,0x000055aa92af2c90}, {0x0003cc5ae6b06c50,0x00057eaea58e748e,0x00014924cb8d73af,0x000859aefc2fbb85,0x0000055f1f24388c}}, + {{0x000b0ed88e1a7efb,0x000e7ceefbfc0fce,0x0006d857cea82848,0x000cba7abdcd838d,0x0000ec1dae8cead4}, {0x00000a669ceda0f7,0x00011d3126dfc250,0x000f4b9ca2e7c1e0,0x0009b84cbfd34e07,0x000040e4bcfea9b1}}, + {{0x0003f728819ea128,0x000b06e16cf36161,0x0002919408897f08,0x000e528858cfd6a6,0x000058155bc25fd9}, {0x000ae2986292c0c6,0x000167af291c84aa,0x000e79cb6ec4f1f0,0x000c81fd358f1993,0x0000ce475e3b0ba1}}, + {{0x000f785374d12d02,0x00012d98caadc70a,0x000d42f2a8babbf5,0x0006c2fa867fbae1,0x000053850d2b0795}, {0x000def812567bf3a,0x000d771344b51fe6,0x0006e684f2a6edf9,0x000c5e0a73f6690e,0x0000e282dbcf75a6}}, + {{0x0006d9be5836ecb2,0x00020a13c0defb0b,0x000e52387a145e68,0x000f89ba1fac2f4a,0x000009824d57a58e}, {0x000dae8413ed668b,0x000ade3adc47d3d5,0x00080a12938ac94a,0x0001e632ee79ec52,0x0000a4e43332753f}}, + {{0x0003b2a5bc9d9932,0x000a0b017d90e7c9,0x0008d33771f5faba,0x0001f007e829cfb3,0x0000a6756330d4e3}, {0x00071776f7f71d2a,0x0006552d8f758b90,0x0004bfebc559f745,0x000c49858fe49424,0x000017dcb740493d}}, + {{0x00036a95aead4696,0x0008faa7b0da3af4,0x000525e90dec1412,0x0008746bab082157,0x00007f8eaf99f609}, {0x0009610fdfd3120b,0x000a43ab469f1ec9,0x0009ad77bd19d496,0x000fd9a9a5d8758d,0x0000da7732d94025}}, + {{0x000ed2476d66cd25,0x000cfb4dafef99b7,0x000305d194636025,0x000c900c1346a4a9,0x0000bf0491ebcdab}, {0x00076b6079e2c2ca,0x0008b0b6c5b084b1,0x000dc7edc14533fe,0x0006f61b6cefcf0f,0x00003ee45c6b478a}}, + {{0x00092f1d816eabc1,0x000722ec5d199636,0x000d057ce2867339,0x000d5e74d26433eb,0x0000e5c102c41805}, {0x000cee82359f79ac,0x0007e6aa805964ea,0x000dc458eb929320,0x00072db27bd3f9bb,0x0000da2bf8fef359}}, + {{0x0002a01a0a8db27a,0x000a64fac978817d,0x00003993bedf7365,0x000f0423b74a2535,0x0000333bccf9f3ef}, {0x00020cdf3fdfc059,0x00040b069a38f55f,0x000d5dce6c8fed38,0x0006a5e2563db264,0x000065193986cacc}}, + {{0x0008add2e2366c41,0x0007d5b50c4b1c42,0x000492b8899d7804,0x0004015c2e9b786c,0x0000f1dfde4b6b08}, {0x000d8bfe14c5f775,0x000829260c708e76,0x0005519b1411e966,0x000b9c856fae8b5d,0x000022c66ad8a1ce}}, + {{0x000b24afec4eb474,0x0009ba269084ce0e,0x000929f95b4dc777,0x000103abb56c569f,0x000039c57b19bb9e}, {0x0009031d0f495421,0x000062aa769f5f16,0x0007f07c0d92f631,0x000de2bef7db92f6,0x0000309aec890e63}}, + {{0x000cd3221ad58754,0x0002a8fe3bfffc16,0x0007b21f1823e90d,0x000f9fd19412d4c9,0x00004e8e63528a95}, {0x000e5d23d1dae12b,0x000a643cacc7fff1,0x000061101366f4ce,0x0007da91960c70f9,0x0000b2e33c5a3b2c}}, + {{0x0000f0bc9de883e4,0x000832bc530773e8,0x000d6df7a905063c,0x000a3c143b5a28fe,0x0000e8befcc34662}, {0x0007800fa2c481ae,0x0000406435301c79,0x000f58b02d2a187a,0x00023349b0fc5eaf,0x000039152a95f548}}, + {{0x000eef8fa126aaf1,0x00086d7f5322c9ce,0x000e1277f0f736ba,0x00079b9f64d938c5,0x00003a4c8e990669}, {0x000b0604ed199f47,0x000bb8abe33c1b87,0x0009c5ec77b8ec2a,0x00068db6e60ca7ff,0x0000e2cbee08542a}}, + {{0x00097c18666cd322,0x00082e95d7c8b9ef,0x00093771f688714b,0x00001533c086020f,0x000059d210c8cad6}, {0x000146ca0174b83e,0x00073cf08052cb68,0x0003514a6f0317c9,0x000ee2740ce05a48,0x0000abd0c44aba33}}, + {{0x0007d09b4e2c6759,0x00045d92c0b09bb7,0x0009f5fadb7aa0a8,0x0006eb2aa8468c02,0x0000370e2b02e3bd}, {0x00010c5a05206771,0x0008d7564df22bf1,0x000eb5eadfc154a9,0x00080f2066bbcc22,0x00004986aca010b1}}, + {{0x000304ad640228e4,0x000ac74a48b12e47,0x000d9f11db8146e2,0x0005ae640f64bb3f,0x0000acaf40843993}, {0x000ef32dfa787e99,0x000d6ba59fa31a9d,0x0002fd7827169dbc,0x0007a251119b89c6,0x0000bb57c8e0e96e}}, + {{0x000a64d08043f76e,0x000da608af1ec1fc,0x0006d54e3eb16ab7,0x000343e63bacf520,0x0000468f9a504a0b}, {0x0005095ccc8689fe,0x000dbfc0d9ad075a,0x0007a7114b6bd402,0x0007cde5f2d4add3,0x0000ebcb63ead038}}, + {{0x000f2b16e394b9b4,0x00089a6e75339f88,0x0008cab78c7f95ab,0x000401234f291dc7,0x0000822d72a3dc49}, {0x000c53f4bd8392a2,0x0002d49a448fe0fe,0x000a59bac7adae0d,0x00085a96b5bb8fca,0x0000dd8cc17c987b}}, + {{0x0007f9fe72d2861b,0x000fbe6490d8b0f2,0x000f57650932916c,0x0009a9e1d65e06ec,0x0000368b49073e56}, {0x0000a09ec18478ba,0x000abce485c1be25,0x0009c2fd23e9d0cb,0x00070b0b05a40e0a,0x0000c0ac2a141ed4}}, + {{0x000b7f57b5a811f1,0x00003a332880bf5a,0x000f6ed2c331b72a,0x000c931b13881ef1,0x0000c494779169fb}, {0x000ff8589c026ea0,0x0007bf5b9f354528,0x0006516d66583a3c,0x000f0465c5c1aa93,0x0000533827189b9f}}, + {{0x0005d47b7457ae62,0x000e4dc51a1c0b6e,0x000a10b057309d30,0x000b80ea82422056,0x00006b7b1f3b4fd2}, {0x0008248dfe28471d,0x000da74653e6332f,0x000c48220afb600a,0x0002c84e00411034,0x00009c3060c6f37c}}, + {{0x0005a2209fb690bc,0x0001a6a994eb0794,0x00095af9bd66e6b6,0x000d3705c195b4b5,0x000080dddd3881f8}, {0x000d168d5bdbac0a,0x0004b0d2f72778b2,0x000fe8afb40cf8e5,0x0002a796b74d05e5,0x0000b591f243ce3f}}, + {{0x00024db163540742,0x000b30cc20fd6460,0x00051c51aa345835,0x0005a09a17bb71d1,0x0000277fb5139649}, {0x000d2c2542771796,0x0004d1f708634ac1,0x000d7e0ba72c1b8a,0x000e347e4f17f54b,0x00005fbc77bc2e9a}}, + {{0x000e083726268248,0x0003e07229767fec,0x00046e11f43b98b1,0x000a62eb12d4a2ea,0x00004a6f742ff8c1}, {0x000d16b330001ec8,0x0003aebe7cce5d32,0x000632924719bb7c,0x00039e6ff7ca5c15,0x0000f3e8d249e5fe}}, +}, +{ +/* digit=15 [{1,2,3,..,}]*([2^105]*G) */ + {{0x000f55e0ced7400f,0x0003959d5ee8a6a9,0x000d3d46dd0617bd,0x000c3976364d4eee,0x00005d6544ad3ddd}, {0x0007dfe795845bb1,0x00083b8c79468842,0x000dc535a3e15078,0x000da895f764d5c3,0x0000ff1d2743a1cf}}, + {{0x0003913423373fef,0x00099abec687f2cd,0x000c07192b34ea8e,0x0003509d102ac466,0x0000f4fa4a09c2d6}, {0x000f87ba71f45a5c,0x0001a2d1230874fd,0x000228058781f14e,0x000e8a4b4499f2d0,0x0000de5d3cffbca3}}, + {{0x0002e2e7392ec41e,0x000f0600583807b9,0x00044309ceb93a1b,0x0001bd4299d5f2dc,0x0000d341d792fd4f}, {0x000f7da2004fe859,0x000e680815368116,0x0000f675be924d2d,0x00063913a6fbfc35,0x00005c62b2743092}}, + {{0x0008a0d964e634ff,0x00016c532fb1c163,0x0001f4d76384f515,0x000adb344233bc23,0x000006bf9a0792cb}, {0x0007491415365c1b,0x000c3dd1431327e5,0x00020f4fe2b3068d,0x00070d4734ccd16b,0x000095b95b5bf075}}, + {{0x000761a38bb70b8b,0x0002166f263692b7,0x000d87e0b6acdec9,0x000d4767e1fd967e,0x0000224a8eaf9729}, {0x000363860e37a986,0x0008c4b2e8807f28,0x0004c4d7e510f286,0x000ce1a869283ca4,0x0000c3368915a32c}}, + {{0x000ff0d6cc67c43e,0x0004588de301c896,0x000096bf10683593,0x0005e94798800c86,0x00006f62ea26dac2}, {0x000a1e2efe6138ff,0x0008a3848069990f,0x000443db897dd7cb,0x00054d1aac07ca40,0x0000b2c088eca6ed}}, + {{0x000757ff5dacb5df,0x0003409dfa108aaf,0x000d1d3c5b1d8fa4,0x00021c63739d04cd,0x000002d31bfad6af}, {0x000e35f347dbf5cd,0x0006989679f1e20e,0x000e1a052e17f4f1,0x0005c7e7fe010b19,0x0000b14dd7d01c8e}}, + {{0x0007e840f9e4040a,0x0000a26798b65abf,0x000aa3d54ac179a9,0x000acbb1e5e42083,0x0000903524ea794a}, {0x00076ce040ccabc9,0x000befaca5bfabb7,0x000925f5fab22e60,0x000bb22783ab4b8b,0x00005d8af2205673}}, + {{0x000f399058b20522,0x0000dd5d61d0c0af,0x0002ded111c639bc,0x000df9d3a3f3688f,0x0000b2d2d7fb44a6}, {0x000af40a6c2a4094,0x000fe11fdb14a199,0x000a1ab6062e9058,0x0007ef709be94066,0x0000fa64f9da0087}}, + {{0x000b4a8964e772ad,0x000159d89cc553ab,0x000c8bf365c823e9,0x000a41337be0bf3d,0x000033f0eb462664}, {0x0008667a8e33616b,0x000bb9985acdac62,0x0003608bdf3a9d74,0x0005f0bcd80283fd,0x0000986dd5d601fa}}, + {{0x0009de4efcc85c15,0x00096f139013d950,0x00057dacdd1eab4d,0x0006ecb65bd6678a,0x000055db869cf233}, {0x0001dd30ad5ab142,0x000ee6314b871bd5,0x00033a0d66fdc7ae,0x0005777e6357c61e,0x000026c5eda33aa4}}, + {{0x0006dc5456e8775c,0x00046e911188ec30,0x00090025c7708950,0x000a20b40e8e0032,0x0000e507b2640d0c}, {0x000d535e705092a7,0x0006166bd3b5bcb5,0x0004cd250c59127e,0x0005550d02d21554,0x0000e39d008a8c05}}, + {{0x000a050a62fa45aa,0x000e1007a06593d1,0x000091014c6dbd3d,0x0004f5b6d0706a49,0x0000f6d6e35f5c39}, {0x0003b19cd3f850d4,0x000b5e7d85a52f32,0x0009e78ab4f434d5,0x000e32db2b9c0723,0x0000b10cf992d5a3}}, + {{0x000f592e7efa4378,0x0000fd0f0290dce3,0x000a14ca9b7d8b33,0x00085bdcf8aefff1,0x00004f13e355b1ab}, {0x00008a0f61a817bf,0x0009c5549bdd0f62,0x0009a0730387141b,0x000a93fd602598a9,0x0000cd8cdef9e264}}, + {{0x00004ae9a503381f,0x000a859248c8366d,0x000f3a32b04731ef,0x00078d74accfc96a,0x000061c0b6905f8e}, {0x0000ebd5f59d1b15,0x000331d6846375ff,0x00082480114edc5a,0x00058cfd89817663,0x00006f41ecdeb28a}}, + {{0x000aba7b5341a587,0x000bbbdbc220d5f2,0x00002e2f859f0298,0x000c85b9d62c9cff,0x00004cb6beaa83a0}, {0x0009ed02cf8d6bf1,0x0009bdb17547e9ad,0x0003397b4afb4753,0x00091a79f573a86a,0x0000919e0860dce7}}, + {{0x000d88918e2f47d3,0x000f6264e8e698b4,0x00076fd1ccd75cde,0x000411bc9ae6e276,0x0000d08e3e91b76a}, {0x0004a951353f00ca,0x000e856c3a8b5bcc,0x000143eb164cf5f9,0x000249e23f45b2a5,0x000001ab36c56983}}, + {{0x00042bf7db15975d,0x0005d84f2911ff78,0x000a7eef9c1ef2df,0x0009ce3c9dcfc71d,0x0000f79ee6983307}, {0x000ec3041bc41cb1,0x0004a53e33bf1432,0x0004e1053cb62983,0x0003a7aa20e23f98,0x0000b8294d01f0c0}}, + {{0x00048d0372d37ea5,0x0007626c450c99bb,0x0008f91caa38c50f,0x000b126bffd090f3,0x0000f3229a1aac39}, {0x000fd864b7a8296a,0x00036564a892a1ff,0x000a8d75bf858c5d,0x000d9ad955b9a736,0x0000da662d7fc17a}}, + {{0x0006ba795e751920,0x000846f147608e81,0x000ad1441e7d892d,0x00005e448a0cfc30,0x00005a4d96d983e1}, {0x00004229da172130,0x000ea801e09a0949,0x000fe63f9fc0e627,0x0001c8d681971a10,0x00004012304767fd}}, + {{0x000a19b17530f16f,0x00043fcb789432ba,0x0001dd7acbc549a0,0x0002eb708cbe384a,0x00006369e0348ead}, {0x000a6090c1b89596,0x0001c1598ed95616,0x000f9fbf0c2ef829,0x000f07a5a10b751f,0x000083266ab6634f}}, + {{0x00006753208ff59b,0x00024fc44d274298,0x000b8da4cc5e633c,0x0001eedd9e76d668,0x0000984dc35fc919}, {0x000742fdc2fb0b16,0x0003f1a37ea501bc,0x000ccd5c8f47f7dc,0x0004a12079fcbe1a,0x0000d15e05379840}}, + {{0x00012f5c7b57a458,0x000171b1fea8f655,0x000933ccf5a81d61,0x000369e7fda656f5,0x0000c0ab37cf5bb4}, {0x000b16333f34c676,0x000fff92c2932e05,0x000f70718aea2f68,0x0006ad4391ab6d2d,0x0000b621d11d77e2}}, + {{0x000ee9cb342f4650,0x0009e9f60819a72c,0x000640fbba78926d,0x000eda1a2390b4ef,0x0000ca57e589451f}, {0x0008eb97ffa2d64f,0x0005596f4451997f,0x0004125bb6a63afb,0x0002c6b2bdf70c17,0x00006643535dcda9}}, + {{0x0003cdda9cd82b35,0x000509e3611c4e81,0x0003f16e05c37f8a,0x000b18efdffa6c50,0x0000bf1aed211a9c}, {0x00082a34eed8ee8a,0x0000444ef3243f53,0x000e9936c0eb0c00,0x000a9ca6ba337bc5,0x00009e6106f21806}}, + {{0x00025aa6380f2af4,0x0005491d9d165dff,0x0008eb846ca78f20,0x0003a9f6e0d46b69,0x0000928b01e0eb89}, {0x0000f19d4744bdd2,0x0007232b1a35b4a7,0x000fcfede71fc522,0x000c5bdc58c9372c,0x00001f4e89d71396}}, + {{0x000b414c8ec0e785,0x000a1033d145c6c2,0x000e07a15d9a7bd5,0x0008686748b7db2f,0x000064f63f44f8f7}, {0x0007c5bd8808eb44,0x0009724642e6865c,0x00059e076e717c8b,0x0002161fb2086420,0x000012f7226e4689}}, + {{0x000812cbca321347,0x000e632740ba2570,0x0001db927a9e88dc,0x00052b0aba4374ba,0x000031227b96461e}, {0x000cd5dc06746bbe,0x0000cf772208a93f,0x00002a73b5a86ada,0x0008c08bbb02ac6e,0x0000f0196a56dc06}}, + {{0x000d114588509806,0x000fb66505d56d8e,0x0005d67e3ade8c1e,0x000119098b390daf,0x000036106afbab3f}, {0x00018cd4db085aa0,0x00060de60f8f3920,0x0008384555865e57,0x000c8b560c8a7794,0x0000bb865e6fb6e4}}, + {{0x0009a2695dcb03be,0x00082dce5033ad58,0x0009607c5c6bfee0,0x00022ae3df25d171,0x00000969faa0ee1b}, {0x000ba9a77472ec5a,0x00093546739c439c,0x000ea5d5c7a4c0b8,0x000e1e00c8fe66ae,0x00009ce280523997}}, + {{0x0005978b851b77b5,0x000283c18a02a8b6,0x0009ba7aa0364b0a,0x000f94a6337a27cd,0x000046af7665f865}, {0x000ca9cbf97d5b16,0x0002237045beff1c,0x00083538584b0821,0x0002838dc37d07a3,0x00008db8ece96033}}, + {{0x000c181d162aea79,0x000e82e60793d69b,0x0007258aefe6f5c3,0x000ff59bc53c6eba,0x0000fb553ca68c8c}, {0x000a8ffea309a6f3,0x0000b9a56441bca8,0x000e2c97bc141056,0x0001715ae34273f8,0x0000514b5e4a2efa}}, + {{0x000c99927b057ad6,0x0006c8df3e109788,0x00042efcbb33a466,0x000e21d25a75adbb,0x0000e29ea59779da}, {0x000921fdb43a94d1,0x0000339d1820fc2c,0x000ca4ed5d01956e,0x0007bd7de58e797b,0x00001e6e6b666ea0}}, + {{0x000db2987fced620,0x0009ec446ac47b5f,0x0004a043cfcbd482,0x00041a6a0c7495f3,0x000012af53d2f54a}, {0x0006ad3c90c6bfe5,0x0009e9382b437e60,0x000b911675112180,0x0001fe21b90ecd21,0x00003c0b3c579d34}}, + {{0x00027b6e3c50668c,0x0006d6d8d96df5a4,0x000da74b2f2f21c7,0x0004a2b1e238a1ad,0x000010d7b0c82fc3}, {0x0003512ea402316a,0x000e879853dbc21c,0x000ce88096dab781,0x0007d667b34f9207,0x00004b1eb299bb04}}, + {{0x000d118c309e65a1,0x000fabfb4147a551,0x0002cb43addff781,0x00012d35f7873d69,0x000027471214c7b1}, {0x000d72dac4e3e7ed,0x0007353a272c9f63,0x00092a6b9af262dd,0x000220bf901e7534,0x00007a7efb0540e6}}, + {{0x000b0ea863d4195d,0x0005368db7c846e3,0x00005939496972ef,0x000a5e8c21297c56,0x000067666c5eaca1}, {0x00025d8b0dad395b,0x000865268b8fdc43,0x00006bcfdef1f33c,0x000a4774da449704,0x0000a1358f89d014}}, + {{0x00029a88642c8c87,0x0008c8e1555b9104,0x00039a9aba22bed1,0x000fc7e5fe7cb866,0x00000202dfe03ad7}, {0x0008d27cd6bd32a5,0x000cf32a6b477ed5,0x000dfd6a90e47d00,0x000f4011739d9316,0x000009017911ff40}}, + {{0x000b591cfe2d8435,0x000da2d587d5570f,0x000368c95c0226cf,0x00024b0fb7a4d410,0x0000747cde3537b4}, {0x000b950273cff74b,0x000e13c9336c4563,0x000c3960d19448b9,0x0005749ac83ef9fe,0x00006bda1afb9c11}}, + {{0x000fd3c13fb0e65d,0x000fa286d2d8dee1,0x000429aebef8efe9,0x000941fc4bebd474,0x0000003c2cbf1f69}, {0x000b443a5eb895a6,0x000b471ff243422f,0x000bd3be819ff465,0x0001c38b9d452cce,0x00005d9b85ed78f9}}, + {{0x000375126cded2da,0x00007e9257a8c6bc,0x000dee102a30f31b,0x0008fe55ab4546ff,0x00000dd53a3fae81}, {0x0005cbb5a649fc4a,0x000817a77abd2632,0x000e1e9abc600dd8,0x00067b95baef2e23,0x0000de3e4fb2653e}}, + {{0x000383df4f38b019,0x0004818f9a6a720b,0x0002d4e1a9ca0f4f,0x00035b6d14a33e46,0x000044289f623629}, {0x00007a387d0c040d,0x000b1fa4bbe11901,0x000d31de4afc1507,0x000d61847236f98d,0x000092524f93ecbb}}, + {{0x000133e59049afec,0x0005726f65d7ad5c,0x000c20ba29b6689a,0x000fe847a4377af2,0x00000527d1b09af8}, {0x00040fab9f632b99,0x000399c6fd2087c9,0x000492e8aa07a977,0x0006bb6bf24157f1,0x0000127f701cf38b}}, + {{0x000a5446b8fad411,0x00094c04141b6e5b,0x0000c0c227c20ebf,0x000d5a04ab83fce1,0x0000bcb816fe010c}, {0x0003b1f39b167b8b,0x000508ecf57d0da8,0x000f43aa56b9dd56,0x00056a8cd6338fd7,0x0000bbded8d9e446}}, + {{0x0007cc07c0803c55,0x000f213fe5473232,0x000a3cfe69566daf,0x000b4b379c15809b,0x00005eb247e4e96e}, {0x0008c0ee69a81c78,0x000ca494920f6e7f,0x0000bf99b798df73,0x000a22034f1f6531,0x000097e6aeccea96}}, + {{0x000a954927e7800d,0x000a826d0a072de2,0x000d54515c3845d4,0x0009d7df7dea11cb,0x0000bd898c3c179c}, {0x0002dadc7f97ca3e,0x000c0dc928e98ba2,0x00044da48b29b42f,0x000aa74a564df536,0x0000634de334dbe8}}, + {{0x000fefbdd58c6c26,0x000505d10919ecdd,0x00022059482fdceb,0x000567341aff5332,0x0000c224f55a4389}, {0x000cb2becafcf4b7,0x00068c9356cee825,0x0005bf8155b2741c,0x00073337ed23e3fa,0x0000db5fb517c0d7}}, + {{0x000bd96248ab8040,0x0009230d301d5d87,0x00070e14ad7011c1,0x0003d21ef8438468,0x00006554bc48fe6a}, {0x00060d553e3bc93d,0x000d4998ea4a6e08,0x0008cdd7522b6e14,0x00008239e466a48c,0x0000715f93d875c7}}, + {{0x000ebb47ef45b374,0x0007b2b58c3731da,0x000e00a0e3d0dc78,0x000bc2a199fdb8b8,0x0000bfed2128b73b}, {0x000e635d70a51a82,0x000ba44f2975b044,0x000dad6e7aecd152,0x000a50eeccc149a8,0x0000aa60be904d47}}, + {{0x00057e2a9da811f5,0x000d8f3097464c41,0x000484e775d2c457,0x000cd6e150802622,0x0000d436e5ccb8e0}, {0x000b7da9c87674cf,0x000a0e9207076833,0x000861876e15b63c,0x0000a8cf27564749,0x00003acadb457197}}, + {{0x000409f6aa3e6037,0x000e9cd0f03103a4,0x0003670a08cb8a11,0x00029d97e7af5af4,0x000033d7696897ea}, {0x000513f584bf83dc,0x00003556fed046f4,0x000673b14b6edf6d,0x00085fcef8b7c83a,0x0000b1cd839a4b63}}, + {{0x000185d2c4242479,0x00097a295f96baea,0x000a2a8c7280ed03,0x0007ef7fc1f8550a,0x0000d696d88afdda}, {0x000ad885a9e3c7bd,0x000370ee8f9c98dd,0x000bda7835528be5,0x0009132edb484473,0x000039c54d3c3c08}}, + {{0x000279ed119371f3,0x00001fce787efaaf,0x000c87c24f135eb6,0x0009d979f366c07b,0x000041ca760045fc}, {0x000acb26b190eed1,0x000fcaffe42e29c7,0x000d6d3ab87c3da8,0x0008c050fea0c3b1,0x0000f921a3b9c2b3}}, + {{0x00019080b28583c8,0x00084271f1c9cd2f,0x000d88b4d5344abf,0x0006e6a1fa410880,0x00009780cad6113a}, {0x0008fccf32dc4852,0x000c09cdb1201d95,0x0009f592a499a670,0x00020202a5504704,0x0000b368bdb51e7b}}, + {{0x000bf0d270faaadb,0x000e467299302de8,0x000d5b29c9761e25,0x00038c9a2a3ca3e2,0x0000cc9ce0564bca}, {0x000feb5640283cae,0x000933c755cd2702,0x000ab454778cdcf5,0x0000f1802df412fa,0x0000a3fa943ac400}}, + {{0x000b9788e6e633fd,0x000544369ee142c4,0x00015ff93aad8109,0x000247c388415df8,0x0000bda8886b6aad}, {0x0001ae09bbfc6389,0x0003224a5c85ae86,0x000c8b0d806937c2,0x00033767015db086,0x00001093bc829ba2}}, + {{0x0006738d95956325,0x0006c14fe2f45374,0x000738d03b49c9e4,0x000f4f514a895f38,0x00004126445d21c6}, {0x0001bdd19accfc3f,0x000c75923b83aa21,0x000d4420ccc62233,0x0008570259ea62dc,0x0000ac5e9fd95e6a}}, + {{0x000f02f1e3020f7e,0x000abf391444e199,0x000d33d014595d4d,0x000243d1b275d094,0x00009fe416192225}, {0x000927e9313cc8ae,0x00007cbfad2bd631,0x0007d6f47c38a96a,0x000212ca8293dcf8,0x00009e934e1b8edc}}, + {{0x0007e895fac16188,0x000c48ddace03666,0x000ea05367de0677,0x000beff7720f6b94,0x0000f0cfc3ab04dc}, {0x000ffaae7332b4aa,0x000376aed4df1fee,0x000d7d9139fe5560,0x000cec33e0ed75b6,0x0000b793cbe3ee57}}, + {{0x00004d1db2df89a2,0x000fe1bb790eea99,0x0002e037ee87f71a,0x0001a43ef529bd8a,0x00003f57ccee309a}, {0x000e652d001535bf,0x000531e580c75add,0x000537db16a02484,0x00016ec744e0c9ec,0x000031da47f5dbff}}, + {{0x0004c7e6c2a25fa1,0x000d2f543aa6ac56,0x00040245d6d44b13,0x000d4cfd0af04138,0x00008a3e6f94f447}, {0x00053a1887a0b642,0x0004e78ebfe0b50a,0x000560e378ad1f19,0x000ddca374718976,0x000076c2fad52831}}, + {{0x00033c240fe437af,0x0004155b36bc99b8,0x0001d74ac6678611,0x000a5b24b825c503,0x0000dc5a9fb55609}, {0x000f6da55d8b0a12,0x00077b276db4ac76,0x0008fc93aacf912a,0x0005dd0cd8b799c4,0x000026948bba026c}}, + {{0x000e5dc6a8543fd2,0x000c1dd380d7e3a3,0x000f3ef8413c4a39,0x0001dfdffc537730,0x00006f878d18b4d0}, {0x00050e41bd35c986,0x000f502371f32ab5,0x000f1a88844ef8ee,0x00013ba15e21b99a,0x000064aa063b9517}}, + {{0x0000dc7a8ef6fb13,0x00029eb3893a9f55,0x000be61b7ffb9975,0x00084bb486f2c400,0x00007200d4bf41dd}, {0x000b00036c8f5e50,0x000f666addc17762,0x000741de8eaa75d7,0x0006e2beadda312e,0x00000cf20707c3ce}}, +}, +{ +/* digit=16 [{1,2,3,..,}]*([2^112]*G) */ + {{0x000263e88928ae78,0x0005b4e82be78b48,0x000573a10668411d,0x0006003ba7a1e67f,0x000055ce3d847374}, {0x0003d7e7e0444965,0x0003b5779daedf79,0x0006fec0025603ce,0x0005537019312576,0x000058c17835ab3a}}, + {{0x000f3801bf458e27,0x000267db440dcbaa,0x0009ae263c36f812,0x0007e4c9536223b8,0x0000a3cc7df0233d}, {0x000b1cd94cbeb372,0x00039f4807f8b9d0,0x000c597324185b52,0x000edcac170724de,0x0000d2110bf132b7}}, + {{0x0006829bfc72b079,0x0001d7df36f44abc,0x00071f67bd4d4b36,0x000dfe954c4d7c53,0x0000ce1626d4ce63}, {0x0001b137ff8ce06d,0x00074664f8dd7874,0x000300b6012fe268,0x000838c0a9ba2942,0x0000ccd61cd61418}}, + {{0x000561feaf4022a9,0x00054750df9b560d,0x0009e8873c0034a5,0x000babe5354bb528,0x00001434cb13b39b}, {0x000661885c701bf4,0x000024ea8f5c6d71,0x00087a94fb5f8163,0x0001f69289cb22b9,0x0000fc615e56fb16}}, + {{0x000183c12ef9b892,0x000bca4be6ce2dbc,0x00034f0eb5247e9b,0x0009319bfc5e8667,0x0000d7c61ce40520}, {0x000391e7aecc2dc4,0x000502b7dbd4b402,0x000c8e5c4c4cfbd9,0x0009e83613bcab0b,0x0000346c5d1041e3}}, + {{0x00061fe951496001,0x000940d172cbd2f2,0x0007178c0c43ab46,0x000faf9dccdec82f,0x0000adca0a729af1}, {0x000d70cf6f648cdd,0x00020f732870a2e6,0x000edafaf9b5c25d,0x000106ce0db1e405,0x0000d951a4245711}}, + {{0x00083ffb1021d71b,0x0002312ad996176a,0x000fe8f4b2e72ec8,0x000a2d26d3a1eb2f,0x0000d0da0974fbb1}, {0x0007b253b22acbc9,0x0008e75e121ea26e,0x0001d0d96ac731e4,0x0002d71a94473e82,0x00001f7b3a36202f}}, + {{0x00038deafeaad237,0x000df374aa4943cb,0x00049341a28d1f25,0x000da0001e16e457,0x00005d5a690e8563}, {0x00039cc322cd1a1e,0x0000e69f3d0f0287,0x0001f380debb0624,0x000965b31a6ca354,0x0000e9cbcd994718}}, + {{0x000c6922682f05da,0x00017d91dac21c30,0x000d9a935d39a514,0x0001318bfcdaa873,0x000086e4fe5534c0}, {0x000794e1a75fbff5,0x000536f457094f58,0x00038cd634ba889d,0x000e8fdf2b2976a5,0x00008ca32ea075ce}}, + {{0x000d5a656febc1de,0x0001902e98f903df,0x000f488a40aafbb9,0x0000728ee7bae61c,0x00004af95a5c8955}, {0x000e37e156e3a046,0x0004accd7ad577a5,0x000b8bf99c4f1997,0x000c3dc8efc06d64,0x00009535e542d847}}, + {{0x000c071894059e33,0x000e16afeee5055d,0x000d7c4e58da0f59,0x000f92484480872e,0x0000d26d813b9d7f}, {0x000db103b383d063,0x0003a77d69385adf,0x000f5cdf0dbbbe77,0x00044de79f617d94,0x000045aa4efb1858}}, + {{0x00024757beec731d,0x000827d28fdb9e94,0x000c2342a93d6395,0x000aa543b7350b9d,0x00003ff4ae312e55}, {0x0007f066a93f5d0f,0x000b7bb2a8d24025,0x0004453ce9313341,0x00018ac52053dcf8,0x000034eb883f74c1}}, + {{0x000eaef662e4d0d3,0x000059a256134cb7,0x00039f0720131fce,0x000403b640ec2ad1,0x0000bf5af241564f}, {0x000e90b3a2dc95fd,0x00011936e70af541,0x000cb4d905df5c12,0x000eec1d816ac3c6,0x0000f7bdb6871a08}}, + {{0x000a5314edacbf64,0x000cea476e8d2cbb,0x000329c989b5d217,0x00006210a624d6bd,0x0000117009f54cea}, {0x000f3c08ecd2255b,0x00003a9d114b6cd3,0x000eb5279e0c672a,0x000350f0cf81ab63,0x00007bd811141a39}}, + {{0x000e5ad10c0f9e13,0x000ca384048e1e66,0x0002e8128b54f642,0x0003a4179217c0e0,0x00007023624a2bbf}, {0x000303ed950a4a7b,0x000f2ab587314f2f,0x000579025c85df88,0x000da0e847c3dd28,0x0000ad36a42e0005}}, + {{0x000f3942e984ec00,0x00069b01d25ffd3e,0x00052e58aeb37752,0x000648244408f942,0x000050a44776d1a5}, {0x000a111b2f83e4d2,0x000a55031541fd7c,0x00010e5caab3d571,0x0008ea6a31cb7862,0x00004f23739f63cb}}, + {{0x00034d5aa351e7fa,0x000767b6bdceee2f,0x000c32c404785e1e,0x0005856ee43745dd,0x00009ac4c31ca9a2}, {0x000d00120efac828,0x000cbe4c18a7102f,0x000f056f5a74ffbe,0x000556b56adf5d31,0x000034b8508f4fca}}, + {{0x00072bd8e4e38d94,0x00066bc1a50e3f67,0x000eccca9fcbfd5d,0x000094357266d3bf,0x00005d1ab86354c8}, {0x000e84f97431deeb,0x000b71de91ac3fca,0x000c099c7cfa4e84,0x000438463acc3dc9,0x00005a3b751b8423}}, + {{0x000ae45e109ee44a,0x000869005ea9c422,0x0008dc7eefa037b1,0x0007cb207cd5ca5c,0x000002e590572ee6}, {0x000d6f711e0c867a,0x000b67004688c5c8,0x000ac0175a217761,0x000d346dca0af57f,0x0000834eafd30443}}, + {{0x000031a244eb62a0,0x0006e2bc5ae2ffe1,0x000da78702a1ecdb,0x0007d86a6091a100,0x0000efb1036654d2}, {0x00050d8f92e2f743,0x000f3b2dd2ae5d9c,0x00085081efcb5469,0x0009bb2381877fc6,0x0000a09d4c8315e0}}, + {{0x000d2bc606eecd87,0x000ce811461f32c0,0x000d2a5d24c90b85,0x000778f04514b56d,0x00009c7f1fec2e61}, {0x000670973b59d05b,0x0009bb1567fe8aea,0x0008152ae455eabf,0x00049b39a03b8257,0x000076e8321067c3}}, + {{0x000293372ff91a46,0x0006b775c576ea7b,0x000a3cde27e9c747,0x000bf429095d14f4,0x0000218f90c2db67}, {0x0008acca306563ab,0x00024d4b59a65276,0x000cf8c68ef2c300,0x0008a593a9f027a1,0x0000fb08f6575918}}, + {{0x00001bb71891edaf,0x00064a5d6ffe31b4,0x0002b148e3202aca,0x000672cd924679ac,0x0000f9aae98a214e}, {0x0005dab10356c4c1,0x000a2de53c8e3a1a,0x000fe283ee091971,0x000d484dfedf9266,0x0000a8b4a0624aae}}, + {{0x000cafe59e1c69e4,0x000ec2f66c9b2eab,0x0004aa0e4ce53351,0x0005f826078a7569,0x0000640f3228ba1a}, {0x000925c496579dc0,0x00038c97b76be7ee,0x00049904eb6310b9,0x000d1abde130dd9e,0x00007e9c5c47ec9a}}, + {{0x000747bf4a461c92,0x000604cac505b161,0x00039e37325036a3,0x000e3d40936c8564,0x0000d8a870eb02c7}, {0x000b9d2f973fb7f5,0x000c1a615f5d3397,0x000a0a4c943f3401,0x0002887b120a3cc0,0x0000f101a1ac6662}}, + {{0x0004a3d4fee19353,0x00015d011130b451,0x0005ac2c73b7caa8,0x000becf5b0c09c90,0x00003f82f1440fe4}, {0x000b4ab74f14c961,0x000bba7943306bd1,0x000b78631291e728,0x0009cb5c97bf6fe9,0x00001c908352f56f}}, + {{0x000a431a92b58b20,0x000f68a51d86d62b,0x000861d00fcae533,0x0000dd371413a4a8,0x0000cf49037f8657}, {0x000eaf4c3d1f3eb4,0x000521e2f9631faf,0x000851b77e971282,0x000ea6acb91a72c4,0x0000206da3cce7c9}}, + {{0x000d754a64d62c61,0x000ee0355ea5a1a9,0x000e8ba7266cd17c,0x000a54959bf91b62,0x000070bfd9a18d33}, {0x0008a1b0be98472a,0x00030fe418fb02b7,0x00052676e46d3013,0x0006d3b58f556cbb,0x000078fb2237e0f1}}, + {{0x0008653a9f8bb2b0,0x000e96befa270587,0x0006dad18832ede3,0x00068e53b0c689cc,0x0000db79f085c9af}, {0x0001c7d93d122b99,0x000db352a7a11671,0x000c8f1e726becff,0x0001d858067471da,0x00007d1cc378fead}}, + {{0x0007f002486685af,0x000e214080b0257d,0x000cb1b8970c4633,0x000a75ed72c06edc,0x0000da0218c79771}, {0x000f32f5200f7c7b,0x0003a69a8df75540,0x000d5f6e2ddffd2b,0x000fe2ec9a3ffb8f,0x00007df766b5d4e0}}, + {{0x0004d27b3dd34f56,0x0007de6fca2c5512,0x000787744861e22c,0x00006ce2f3d6847d,0x00001ec88437590b}, {0x000c281f52950ec0,0x0007bd6b5c7cab50,0x0003c2642b1c400e,0x0003890397e8e47e,0x0000cf7d2df12202}}, + {{0x0003266882874e03,0x000a89e81d0194cc,0x000549a334f700e7,0x000ebbbe4d500db4,0x0000433030eb4bf5}, {0x000d4c6fbdcd9292,0x000d720ac8474589,0x0001ef26b9eb5ed5,0x000acee19219c5ec,0x000001a7fc35fdee}}, + {{0x000f286f78c7ae3f,0x0000f6fe1ca839e0,0x0005678a2b39fbed,0x000a8e7ba6e825c9,0x00007d7bcb5335db}, {0x00060809ff657696,0x00043fe88e0ad6ff,0x000de58633c659b1,0x000251056cdae14b,0x0000b0b64689f024}}, + {{0x00008e21178a099f,0x0009afd88898f531,0x0006270b71af681a,0x000926359a6e55cd,0x0000e1e802206d9d}, {0x0000c10a39d0de19,0x00011dfc0b3fc87e,0x00058de46f676bbf,0x000ebc32f304138e,0x0000fd44af065751}}, + {{0x000837bfd2c4dafe,0x000f55cb3e99d194,0x000e83b433ea9169,0x000d019bf4f952b7,0x00005a04c25b21a2}, {0x000bf1e7dcbb4739,0x000c3be7dffaeeaf,0x000b4829d1ec04ec,0x000909474be22c14,0x00007e4c57a57cd7}}, + {{0x00061cc613abb95e,0x000cb58aa468c130,0x0000103c600933a2,0x000a98fc90deaeda,0x0000ce472cea4bfd}, {0x000ffac971727969,0x000fb1241fe002c2,0x00094fc710ac9f69,0x000df476c5db7755,0x00008ee99ab46283}}, + {{0x000da7a9e6fa679a,0x00050244f9571a74,0x0002bfdb99609f5f,0x000a0ca351106f8b,0x0000726ad35b71e8}, {0x0001399059b1506c,0x000d5f4c5d58d7c0,0x0004e983e0eec7d9,0x00024d47eab45931,0x000073b2a5db146b}}, + {{0x00053cd8ac3fd6e5,0x000911ae915c1c68,0x00035e452713cfe5,0x000bf9b5b88ed861,0x0000768dd4358000}, {0x00055d4bf4bc4280,0x000cfe3ceae694b1,0x00039860a90b33a4,0x00044533c50782b7,0x0000e0258fbe654f}}, + {{0x000483703eb7f60b,0x0007d6d369a806b8,0x00012af7a09ec39b,0x00037d6f136e5aac,0x0000dd8a6565513c}, {0x000255d417353feb,0x0007cebf91f97a59,0x000d5caba59e9564,0x00024a85f33bf2bd,0x0000a4be8c2c5c77}}, + {{0x000796c75120b49d,0x0008a90e9b928b9a,0x0008f95c1622af0a,0x000ede13d8b3fc2c,0x00003c265423eb9e}, {0x000ebc9a15bd7812,0x000ae6b2d10435bf,0x00011101cea171ae,0x000e63bfb9273095,0x00006dd7cd57f8d1}}, + {{0x000198e5d68cb6c0,0x000c190df284c844,0x0007e36c27d05eea,0x000e6fa431aa9dc0,0x00003b61254ff61e}, {0x0001a8e2b8663dba,0x000587bc144f1d1f,0x0005a87f2ae85cbf,0x0000fb374e6f7b1b,0x00008c121dbc747f}}, + {{0x0004c7bb9087d92a,0x0007c0e6a9bc8960,0x000c69ec2a7a547c,0x000ec22623c94a7a,0x0000525b055cda91}, {0x000b110ac44d3bf8,0x000f1422b1652063,0x000642f08b288324,0x000c8d48a8d2d517,0x000012612181f2e2}}, + {{0x0008cc568e4c47a0,0x00034c3e8a1d2cf8,0x000b8406ce8cac5c,0x00032c23bc082944,0x0000bb19f797a36a}, {0x000f360a586e48de,0x0006e44743247522,0x000e06255c5208aa,0x0004e8f858612ae7,0x0000b4ea39d737ef}}, + {{0x000bc460db94e995,0x00005080649d163b,0x00096d07ee0617ef,0x00093fde45ac2df3,0x00009614177e10ff}, {0x000dbd16ee571c89,0x000a36b028cc1b9b,0x0003821897ca0e3d,0x0005b2397d51cfd3,0x00005ff63bbd9c6e}}, + {{0x000776420861ada2,0x0005c1493e967a64,0x000d92b3fba264b6,0x00026a2e174e2760,0x00001c08d1309e77}, {0x0003add07c55bcbf,0x000e21a28772cc02,0x0006ed92bc89eea5,0x0002ab9f631e159e,0x0000237c626df26a}}, + {{0x000eb0be483e60b1,0x0008a8144da94b61,0x0007e3c9442ec74a,0x0008d5659b08da90,0x00003a62569751ae}, {0x000a60e8fe99eac3,0x000c83f07f5cddae,0x000b6a2c4f4d939e,0x000c3686516e4f8f,0x0000ce14006ae498}}, + {{0x0008be5037f0aef1,0x000400d9612b3b49,0x00000f5494e4a5f1,0x0004b350178c1bec,0x00002194ca390f0b}, {0x000371e991e9b508,0x00066f5513d3e57d,0x000bfed44a8a1cd6,0x000d0caaae2503dd,0x0000e2217e3ec2d1}}, + {{0x000de10dc8a838bc,0x000cb4db84f09e5f,0x000fc31b998d83b9,0x000d314cdae49c2c,0x0000e17fbf4482ca}, {0x0003d5685bc44cd4,0x00011b173b41f973,0x000b6c609b97d020,0x000fa8129cec8ae7,0x00008eb425aa9285}}, + {{0x0000d499bbee7027,0x000794e7726f36ed,0x000f7e50f1106588,0x000372016e8a20bb,0x00007cad5d314d87}, {0x000dd1a9c4d420b0,0x0002e0853dfe07df,0x000883f4657bcb1d,0x000e264db07a30b7,0x0000d54953a43577}}, + {{0x000b7bac5d80fad0,0x00044b2255967200,0x0006673244b80849,0x000399b432f19ed6,0x00007a899f7c61f9}, {0x000ec5ef6da900c1,0x000f95e8e63010d3,0x000ee7310e72fe29,0x0002467c0da3b206,0x0000d1381a884ad6}}, + {{0x000647f72ba142a2,0x000510a8a8617f04,0x0008c5b890377f95,0x0000384970fca731,0x00001b0e8d4dc56f}, {0x00069cdcd6ed4bd9,0x000dc638ead2b445,0x000d215fcc875e7a,0x0003c6dc06beb69a,0x00000645a027babb}}, + {{0x000b0431983cd6ed,0x00007bf2175a80d4,0x000798d7519a205d,0x000fa59086f5341d,0x00007f0d0b4ab66e}, {0x0001892cc0405eac,0x000cbf6a8145fae0,0x000fa60ef8231d5f,0x000d9bcd49a89231,0x0000ddf2a97a89fc}}, + {{0x000d2ffecb831b3c,0x0006c64be18ccf99,0x000b95a73a3566b9,0x0006ade17af67343,0x0000bb14c802d2dd}, {0x000046d9f8cb107c,0x000510d859fadfd1,0x000c8a0784b62dc2,0x000c0a6ad339c802,0x0000cd11269bffaf}}, + {{0x000468da94f038ab,0x000bb71bb5963514,0x00006cfbbfe6dae2,0x000e61154fc00490,0x0000bc8ae6a43e13}, {0x00044f06569f3f31,0x0006aa775eff10e0,0x00034b8704d0054d,0x0006b4251e6260c2,0x0000ef2a9d02c95e}}, + {{0x000ff430a65cf5cf,0x000e5b64c8a24667,0x000eac3ffcec204e,0x000777251bae8db8,0x0000c0889a098bf9}, {0x000fce13803c0161,0x000401026ff03bad,0x000f5e552f8dc21d,0x00091ae48fc57a54,0x0000609011dcbb57}}, + {{0x0002643d816f6738,0x000c3753f6e86736,0x000879b02d8cb868,0x000c7e594838c395,0x000067972ca999f1}, {0x0007d34dfd7c1088,0x0004614e333a9fad,0x00058a876d7b6624,0x000108e3cfac8612,0x000025823b4b5af0}}, + {{0x000268f6a19bb9b3,0x00070a8958a2b4db,0x000e8549f04fad60,0x000d9a69a06eac4e,0x0000d03548e51195}, {0x0004f596dcd373eb,0x00006334b296340e,0x000d58c42a009c79,0x000881171c453a68,0x00001b31a3104480}}, + {{0x000638d6478b071d,0x000d506ebaf4c8e1,0x000f38ac3c0decc2,0x000ef39e647b5165,0x0000428075e5d414}, {0x00028b3e3fe5a0cb,0x0009c6c2309c09a7,0x00009c0e65089762,0x0000d298398ccc4d,0x00001a2ece4161ce}}, + {{0x0004ed542000ffef,0x0006547c39491113,0x00093d2fae58456c,0x000070c2b32044ae,0x0000c506a795a7b1}, {0x0000510c08f5f461,0x0004b53bcfd62e25,0x00072129bad0f10f,0x000fe39c16f090ce,0x00003107a2433ad4}}, + {{0x00094ca6b8261d16,0x00015268752c1c63,0x000960928360a4b3,0x0000531849d69193,0x0000f72583b238ad}, {0x000fb502a8493e40,0x0006bd817731e757,0x000993764b0c236c,0x000d4f08682dde88,0x000027590fa24c64}}, + {{0x0004af2f50460797,0x000d37f2b0ce2a1e,0x0009a8ba740bbb10,0x0008100c3fd4bc4d,0x00002775b5aff598}, {0x000144f7f34177e8,0x000d1f57ccfb628e,0x000e818c9097dd78,0x000458b5e547accb,0x0000c02a026db11c}}, + {{0x000ce91cac08c3d1,0x0002b2736bca265d,0x00089001c395d92d,0x0000fcf505b6519e,0x00008c614d9cb52d}, {0x000ab8803efd70d8,0x0007c604446b154a,0x000619c8c4d96a6e,0x00064ae8fc9320b6,0x000068e0fb54c0f2}}, + {{0x0000cefcf4e70ec5,0x0004e2cf688d23cd,0x0008a11035bb79a6,0x00062874c5aaedce,0x000057759d2b1417}, {0x0001d559128402a1,0x00048367bea79cef,0x00096c617f84a0eb,0x0000d3fd7070c334,0x0000553c0472599b}}, + {{0x000d20f989b9e29c,0x00099deae2b2ff99,0x0005ec2b4b690d6e,0x0008df9c55072b82,0x0000bbf457111cd8}, {0x0009822f0aef2354,0x00029207a036f4ad,0x000ed9f8978a6431,0x0006e9dcb5c728b1,0x0000b9e20713e8f7}}, +}, +{ +/* digit=17 [{1,2,3,..,}]*([2^119]*G) */ + {{0x00041fa6a39c7f6a,0x0009eb5589baab08,0x000627ba2093d7e4,0x0005a1f11df73ccf,0x0000a3a9614f842b}, {0x0003036faf95407a,0x000a6b88f15492a3,0x000ef2aa8c72f6ed,0x000797929383c9bd,0x00000d4b788e861e}}, + {{0x00028e853253b32c,0x000764a3d3e50baf,0x000523564b25a2db,0x000cc7f385d655ff,0x0000d11a0255d54c}, {0x0005cc1bf209a125,0x000965f5e1df998c,0x000023464fa162cf,0x0009a40a48610028,0x0000fd3f9f50b2cb}}, + {{0x000a5228e90b176f,0x0000c1c3bcbc9702,0x0005ae8e770765a1,0x000f8b56923d8d2e,0x00008e74adba5e4c}, {0x0008ac1d36d9dbc1,0x0000bb6bcfb20110,0x000fa9d7efcf1d34,0x000aaad4625764cd,0x0000a06c7fa8fafb}}, + {{0x0006be82919499b2,0x0005b2e0e8754c2a,0x000ae54ff9e14e89,0x000853a30ef155b2,0x0000ecea0175c713}, {0x000d2365ed695a76,0x0009c344487f8ecb,0x000cbd5f7dd9d9ec,0x00093f8a2fa3c895,0x0000b4000583e162}}, + {{0x0006dfb3c80537c1,0x000928b656a8e73c,0x0001a720c007d242,0x000f0f7f2f75140d,0x0000307b44bb2faf}, {0x0001a93776043d2a,0x000131bbd300fe21,0x00012b8b0d581890,0x000bbec7aa05b0c8,0x0000c0d90f4dd862}}, + {{0x0001ca3ee253969f,0x0009e918dfe27423,0x0006aa687b67bc7c,0x000dacd55607b710,0x000066e8c14a024b}, {0x000f5d052e5ba875,0x0000afcc0c31ef9e,0x000cda3636263f60,0x000872630cbc5ef3,0x00001148ad0d4f22}}, + {{0x000777b4aa7e5304,0x0003615e1bdfba24,0x000858bacb14b766,0x000b945ec88b6fcd,0x0000bd20cb06635a}, {0x0006f5bdaadf1ec6,0x000c7d8267bddc7d,0x0008015bc00a297f,0x000366ce5cbef6fb,0x0000bef7be68218b}}, + {{0x000a98c52873cedd,0x000a4dce333a9777,0x000c49337107cb38,0x000d4e5d090eebee,0x00009c1f6657ee06}, {0x000ec426251c702c,0x000182c9b521b4df,0x00075e02a1270247,0x000e54af1cdaf2a3,0x0000dc7ffbf6f46f}}, + {{0x000522738eddba89,0x000f79659871f8b8,0x0003d82764c79ac3,0x0001c2fc21f3b82c,0x0000caedd9348e18}, {0x0001c424c016898b,0x000369f16a2211bf,0x000d0ee361012838,0x000e2f5b3b65c786,0x0000dbd3e2f44b64}}, + {{0x000ade03aff4bb02,0x0008b4529f4e17b0,0x00043bbb7fdbc450,0x000dd77af503eac2,0x00002cee72b9a511}, {0x00047cc71bb2a4b8,0x000c42f41d52bb5a,0x0003a786fb0e56c3,0x000b64068993ab40,0x00002c6197dabf9f}}, + {{0x00037d63972795aa,0x0009bbdc815efc80,0x0007ab6e71ef405c,0x000a874747c3c51e,0x0000f1d220edee78}, {0x000211ed3c96c1b7,0x0001bc179709211a,0x0000e0198510be77,0x000ea2a633d65d9e,0x0000562f78442b7d}}, + {{0x0007889df799db5f,0x000ba012c603f759,0x000ca505b101e3f2,0x0008490b743f6b4f,0x000042757e975d7c}, {0x00039e82875f4bb1,0x000f8ff53a5da9f0,0x000ec746f6aa587c,0x000474838f092ff9,0x00002758b9877567}}, + {{0x000825e7bf6ef16d,0x000e3b0e3eed4f65,0x00082804995e95a3,0x0009ac66e3d6ef1e,0x0000140e9150990d}, {0x00048379d22a5ca1,0x00006beaef803d57,0x000d94eb0997a2d9,0x0006df77b8617ffd,0x00005a68f3e15466}}, + {{0x00044939e5337133,0x000022bfa31f90af,0x0003a6b0a68785ce,0x000614dc6b2e1ac3,0x00004b2ee53c3f17}, {0x0009ceacff767259,0x000500352afe92bc,0x000b09c4726df2fa,0x0009aa62ed60a55b,0x0000d183787441ab}}, + {{0x0005c96dda71597f,0x000683e67ec813b4,0x000d724183cd30fd,0x0006536b8f8530d6,0x0000632c3ac24270}, {0x000b011fe29506ef,0x00057d43147ea463,0x0002a70211a49273,0x0008988dfb701702,0x0000debd588711ec}}, + {{0x00090744b027acb8,0x000909729b2dd4cf,0x000c784e4eef7dfd,0x0006feaaf145f925,0x0000ec26936eb073}, {0x0003622990de2b9d,0x0000059c4c7d2565,0x000f374f6997d858,0x00015e6db4239a6c,0x000061e73252bdd5}}, + {{0x0000c24d522fbc0b,0x000a942bb73d7298,0x0001ead460b4f626,0x0001ddb380775ed4,0x000081f005390776}, {0x000d2a88d7ed3d9a,0x000a953ef7cd9a17,0x0008cb51593a12b6,0x00041c930e0d8397,0x0000396686f3a8cd}}, + {{0x000a7aca5295f85c,0x000d1715f010ada4,0x000137d62ed99476,0x000a2c1fcf92691a,0x0000965ba19a5dad}, {0x0006e96b9c400652,0x000a0f797bf078b5,0x00027bfa01d46a27,0x00088481a98f9ead,0x0000e828cd3a93a3}}, + {{0x0008afbdbd10a431,0x000d106c891c4914,0x000c9e0e75d3c212,0x000d14e69985bb06,0x000006751cf618c4}, {0x000a3dc9130b9b81,0x000c1201b63586c7,0x000247844a994f46,0x0000e8518f9b96f4,0x0000552e79eb2973}}, + {{0x000a7cf36346748c,0x0004f02b5ab08498,0x000a67c8add8490b,0x0004083da6e9932a,0x000000ef8174e556}, {0x000ebdb7e8317689,0x0009dcee991652fa,0x000b17155928e7e7,0x000bfb43e0c0dcf1,0x000061d9f54d10b7}}, + {{0x000fea920dc3621a,0x0004c5a9ef80adda,0x000bf625b196feb3,0x000c75a487e45e47,0x00001586634f3500}, {0x00078e658ad24f8b,0x000b7ec508d6416e,0x00086e5d355ff7df,0x000160bf2a2a10f6,0x0000e97f906d9cd7}}, + {{0x000cf2ecce6a83cb,0x0002141a1def5419,0x000b08f6b415a133,0x0003ee9d14d27344,0x000001759dafa67d}, {0x000806af6fec05ea,0x000d7b011a04ca90,0x0001b07594714b39,0x0003766edb734998,0x000077fdd00849e4}}, + {{0x000869459607c4c7,0x000c4a2823d2a191,0x000f0c346c25108b,0x0007d785158a6d3b,0x00007e489373cae3}, {0x000a7f304840c842,0x000a08f43f2912a2,0x0002fbe1729e4bb0,0x00028709c4f280f9,0x0000201f93611efb}}, + {{0x000ad0bda7974af4,0x0005389c05296416,0x0004cb76abda5ecf,0x00088f85f754c7c2,0x000047c39eefff6a}, {0x000ef4343c9b4a94,0x000d4bb7a27b96c8,0x000f48abe786978c,0x000ca9cd7614727c,0x000061d39a4e9882}}, + {{0x000b2bcdba7c5bef,0x0002040e82c7d98c,0x0003da87f7a20776,0x000de1523702befb,0x00001de9d096e046}, {0x0008b008f178210e,0x0000448332b989b3,0x00042b84ffde25ce,0x00070d3efeacb710,0x0000f18da37a2d17}}, + {{0x00060e26735666d2,0x000fc57f5610acbb,0x0007546d4a7366dc,0x0003a833bb5f11b2,0x00001d815c609e9c}, {0x00095cb0437ffbfa,0x0000948f5b486c09,0x0006bd916f7bcf31,0x0000b87395b09b27,0x00004e8d8c9a4070}}, + {{0x00041de5bdbd9371,0x0007be38711fa84d,0x0006f2cb401fcc83,0x0000193a4baf92d1,0x00005c9fa811626e}, {0x0007ce2ec2720c70,0x0008351012112d54,0x000741181f3df95f,0x000fb101caa1421f,0x0000a6d145391ef7}}, + {{0x0003e76f52e737b3,0x0008b153cd4c597a,0x000b1f2cb690af03,0x000fe5aac80ff54e,0x0000f1fde4870f82}, {0x0001d9fc80aa5815,0x000701e0254a9b91,0x00008d9e03b39021,0x00058cb9d9791e0e,0x0000755cc5bcb5bc}}, + {{0x0009e0c2ce860434,0x000fad99789d9347,0x000ca723a383cfe4,0x000ab93713011097,0x000015d43c8d3294}, {0x00089eb941c9f1c0,0x00077e0a78126ce0,0x000da428594a8b44,0x00045995df8d73b1,0x0000085b5acc6ba9}}, + {{0x0007d41d3b9f2be9,0x0001e317380a4f50,0x000743cd69a185b7,0x000a3cb0ef1dd23f,0x0000c6f878d3ac81}, {0x0002defd67e4c7a9,0x000b3b6178e5ad5a,0x0008fa9c2073e098,0x000e3f845fb5d329,0x00000ed787e92ecd}}, + {{0x000c2092f308c0e2,0x000cf372518753d3,0x00059ce271c67fa0,0x000b1d3d864e401a,0x000067229588778a}, {0x0009ac48d524d467,0x00099d0876b9e6ac,0x000365b626974faa,0x000f2027749d4cf2,0x000070d784766bdb}}, + {{0x0005d1d7ca4afe69,0x000f7a9e7f152ce1,0x0003ff356ee399b9,0x000662e7d64a6c2b,0x00004e83beaf5730}, {0x000c85f020b5cda9,0x0001bec166277cc4,0x000cfd24f3afda30,0x00076a744e4b56c9,0x0000a9916a5eedfe}}, + {{0x000fa6272bc5acf5,0x000cefa642bb8d4c,0x00068f5594a97456,0x000df80bedbcfd29,0x000069874eb72838}, {0x0008a12ad07adf76,0x000a250132b7f40a,0x000e03fac5809168,0x000a22c9ab9c544f,0x0000df51bc0f4dd0}}, + {{0x000ad692517a2b9d,0x000b734402ced5e2,0x0000f7b228d8914c,0x0009398f05425fa0,0x0000ec8d276b76ec}, {0x00069d5463a061d6,0x000728f1acfb036d,0x000cd88a2d6ddbd4,0x000ba149b9378346,0x00002bbbfad499d0}}, + {{0x00010c3a4809f81d,0x000743c8ba6258e1,0x000de0fe62140ab3,0x00009b8609de61a8,0x0000981e461acd1e}, {0x0003461543c36db0,0x0000f993771d6b06,0x00033281a273ba0d,0x000c77143e6ff6e1,0x000035a642c2fc19}}, + {{0x000ef8af2b81260e,0x0007770b18ec5710,0x000a1ea80ad37192,0x0004707b4022fecc,0x0000fe51b5f7bf41}, {0x00093d72d28d2eae,0x0009b90c140f34a6,0x000335e00449e2e1,0x000091fe11c41f75,0x0000eaa50b2c1865}}, + {{0x000cfccee288755a,0x00076204618d3307,0x000b666fc9acda01,0x0009c38ee7071f98,0x00006f3fade76154}, {0x000c58c20c01f25e,0x000a2f9c9a157e82,0x0008d7ef9691b1e0,0x00034c9b3ea053c1,0x0000fe72dbc256d6}}, + {{0x0005764654f25ab6,0x0009ae7b52161674,0x000bc1c4b22e7a1d,0x000c905ff14471fa,0x00008d2388cb4775}, {0x00068e06e8a88b79,0x0007d25e7385d977,0x000b4997f422d547,0x00018f9575d9ad31,0x00008a642eb9c009}}, + {{0x000635f43433b57d,0x000665e0a64cd543,0x000f7611b6157565,0x000371fe20970719,0x00008f08a7576fb0}, {0x000f528e7c19e126,0x000d86a273b686fd,0x000ecb6a9eb7491c,0x00099d7cc31d126b,0x00006d5b3b32c1c2}}, + {{0x000d0b2e25cc9404,0x0007ea222455a916,0x00089fc82928a4da,0x000ac5c3a8de8988,0x000064f26e535f2e}, {0x0009e29fd63260f0,0x000dd3e24fc71457,0x000bae4202864593,0x0003c6895e5601a6,0x0000c5a1f6069911}}, + {{0x000dfcc37d0c2daa,0x00007dfc9cca7ffc,0x000b7d28f4112fdc,0x0000060424fc5c4e,0x0000dc27dc84402c}, {0x0005cf0f7a1c6f9c,0x00072632a0fbbeb0,0x0001285f3c9bf405,0x00052cb7ba5b1373,0x000024a1d802dd1c}}, + {{0x00005e8e5d9c7c3d,0x00038f7b6bf10095,0x00089e65c94ef08c,0x000d589c3fc80399,0x0000512ab462c8e0}, {0x0006248da5cee075,0x0004b0bf71366122,0x00009b19f788483f,0x00068461ae7d4670,0x0000e3240047ed4d}}, + {{0x0004a8045830ec3f,0x000ff27347bdebd9,0x000eb6a8bec329d4,0x000a3bc9be726d25,0x0000afd3ea7f8f65}, {0x000de1f94e8ec819,0x0000fd5786e2d77e,0x0008c8d2f48236de,0x00088e979e9d9bc8,0x00004177d03e4855}}, + {{0x000a920680b0b442,0x000dd67952d80308,0x0003db335984f1ab,0x000ab57e1ee014c2,0x0000efa311d1fa2b}, {0x000fa774c0978f94,0x000d6ef134fea6a1,0x0003be2502663d93,0x000589a91cb57c89,0x0000e81b2b61a07d}}, + {{0x000445556f590009,0x0001d606164fbd7f,0x00055c8b39ec5161,0x0007b36417bf0954,0x0000bfb7b6cc3e60}, {0x0007ef44ff8e3e17,0x000ee02f7cb58de9,0x000726935b92e359,0x000ec248d2c5ba66,0x00008f73b6ca1537}}, + {{0x000ea9a101c27ebd,0x00088450bc2d762c,0x000a6b7821eaee2f,0x000e7617f29336dc,0x0000bc610f11b9ee}, {0x00020c50b2e18605,0x0009a3531a079c62,0x0003c6f4589a8346,0x0002726dc650c838,0x00002cf4bee72e9b}}, + {{0x0005ef53014f895c,0x000dd9551573a2a6,0x000147dcafddcafa,0x000427c62e1a2646,0x00006be9a834c331}, {0x00031b08c02f0e7f,0x000609f1f2c5658d,0x0009db9bc2cb949f,0x00041b03603bb15e,0x0000294d7c597e01}}, + {{0x000f7cbdd4b39e30,0x0007591b53bf0d2c,0x00051085bc0a78af,0x000d6563b36998a2,0x00008f29a516edf9}, {0x0002df96b9af9b9a,0x000aceaf18ac1998,0x000cb38bd6224ff1,0x0009b595e64776cd,0x000086b39f9d62a5}}, + {{0x0006cc6d42576447,0x000663d1585e0963,0x000718b6eb780a69,0x0008a786d3112c4f,0x000050513009055f}, {0x0001c1f94b567f55,0x0008f04cad491658,0x00005055750428c6,0x0005942f8ebe9db1,0x0000bff399ae72d9}}, + {{0x000b37679ef9c071,0x0007c79b043a42e2,0x0001c11d543e3065,0x000090850df636db,0x0000cf5e687e4648}, {0x0004641c28d6f0e4,0x0009d91286f41e3b,0x0005a6a95105c1d3,0x00040a84b6eed5a4,0x00004f9c5c40b1f7}}, + {{0x000635917a1fc0e6,0x000de448fa1d343e,0x000d04e3e582a0ed,0x000b5be78e3f3810,0x0000e69322253204}, {0x0003f2d8fdb2f68b,0x000dd150d2fb477b,0x000e74fd46b26937,0x000832fb96b093bb,0x0000fd4236540d29}}, + {{0x000627c7da59bb55,0x0004a420a8a1e312,0x000b69e2db5a517b,0x000f174190a03a06,0x000034c0dc213d8b}, {0x0007341656c0a5bc,0x0004601ccf3374cd,0x000728ffe8da09bf,0x0006f93d580e29f4,0x000070d257e9d4a4}}, + {{0x0007cb0ef9a7de8e,0x000c7be468b6982a,0x00013cbb99b936ba,0x0006099f3a374002,0x0000d5e01e4045c3}, {0x000fd2a477e5592a,0x0008ab788c4a8e5a,0x00062949464a88f7,0x000f822e6ed3ab7b,0x000049956ac60d9d}}, + {{0x000632ac60494b85,0x000755001958b3c9,0x00092a6b084635a0,0x0005d4a6261cd6ca,0x0000c5c5a09d09d8}, {0x0003c778bbdfbed1,0x0000571017535054,0x0007f0364db303b2,0x000211c2f39c2c8c,0x0000b2228378b171}}, + {{0x0001b82327cbef8e,0x0006712292a1eb20,0x000d0aa8149ef800,0x00044484bd3c30a6,0x00001bc4794e2151}, {0x00007c4a40a45ab2,0x00072ef2f8c60537,0x000ab01c42dd4a68,0x0004c68445e92c6b,0x00009930cf73325b}}, + {{0x000e0f195743160d,0x0005e665920d09ed,0x00037949c34c1646,0x0001563924baaef5,0x0000864d65b629cd}, {0x000d0645d626af27,0x000b919ee5e27e75,0x000bcf4e3b28ade6,0x00072b450f7afe7d,0x00002fc16bb41c90}}, + {{0x0001f5474504517d,0x0007449fd75747aa,0x000d80fd4dd4e899,0x0000fb1503a0575a,0x00000bcf27aa872b}, {0x0001562ee6845a9b,0x0003adcbda3beec7,0x000e87f54cea9040,0x000d9aa4b0fd3e40,0x000055424363272e}}, + {{0x00000c0c822a63bb,0x0004adabf48f17a0,0x000c2629e0a4d95c,0x0005564c47efec06,0x0000fbcf94b6fd09}, {0x0007de8757b55d12,0x000113777841f5cc,0x0003e33bdeb58cf8,0x000534cfd0dd9945,0x000091a701a6191e}}, + {{0x00034fc2d6c998c3,0x000f5f1753a09b2d,0x000707feecebe5aa,0x00069ae9d032ee76,0x0000b6aa01f1d59f}, {0x000215a8ac67882a,0x0004aa4451eab1bb,0x0003598726a6d1c4,0x000ec7ca810ae0fb,0x0000c1072b2ff7a7}}, + {{0x00062dfef6c9d3f9,0x00089e8be45575b3,0x000c3679a444f608,0x00071da3d3cefd86,0x00009c6b4fe674c5}, {0x000df41f2400d99b,0x000e475f7d54f051,0x000134e5aa9dfed3,0x000b7685cd7bf7e3,0x0000b240168e9d1c}}, + {{0x000f0d440f9f2318,0x000c0a943da7cf17,0x0009cb3de587af78,0x00062e9e6652099c,0x0000b949476ce142}, {0x0005db129e890250,0x000a0753258e9fa5,0x0002708f8e51d5ee,0x00097e68246e57ab,0x00009d5a84444486}}, + {{0x000537887ec45f50,0x0004ae322bdb29f2,0x000adb167eb07940,0x0007ad30b3e3e19d,0x0000cd3891451308}, {0x000e460bdfbaf9e1,0x0006182f1a4179d4,0x00095e491e6878d6,0x00021d1b6dcd2c5e,0x0000e017c1488203}}, + {{0x00015e8bff7115fc,0x000daac297105666,0x000baadbd6bf7eec,0x000495ef121aecb3,0x00004b4a97a0b0f0}, {0x000019c825163c75,0x00095c56e3c4a123,0x000b6668b677a76b,0x00092bc127011ec5,0x000076ef43ce65f6}}, + {{0x00079e1d0309f4a9,0x000d470e28f20534,0x000381c00c0cf6f4,0x000f7d93eb4f59fb,0x00008542677c1440}, {0x000f92b296bd38fb,0x0002503a89b42228,0x00027708a23f058b,0x0006defa6cc068e8,0x0000ed74c5c5e932}}, +}, +{ +/* digit=18 [{1,2,3,..,}]*([2^126]*G) */ + {{0x000bda36cacdb2cf,0x000910b10ef58eee,0x000d29b4c26a98fa,0x0003e3520f48c766,0x000080d336a84be4}, {0x000adf4bead7d2a2,0x0007e7f8e550baac,0x00016417827824fb,0x00050949f19c4c62,0x0000c4b8ae7907f7}}, + {{0x000429a647025b7c,0x000f6df2b6d14a77,0x000609d41bea840f,0x000070b3276b7b67,0x0000d649bb3d432a}, {0x00057149509e594e,0x0009f596b64efcab,0x000a64e95827d522,0x000fd96c407e119a,0x0000c8006ba42564}}, + {{0x0001f1f6379bea52,0x0003f11b63e85354,0x000bd1b7caedef57,0x00000814fa04ef2b,0x00000ba83d7a9136}, {0x000975bc42555421,0x0003bef8e88beb41,0x0009541ba9f5aed0,0x00002dcdb28a4ddb,0x0000c066be89e1f7}}, + {{0x0009e6504120719c,0x000a1d96a1faf50c,0x000f5b3b23b7d22a,0x0009b4c352bf7130,0x0000978872929651}, {0x000356cf46ef6d36,0x000d4b97e2b6d316,0x000745689d64bc86,0x000562e5f781a12f,0x00008e0f6cd43d40}}, + {{0x0004a418781db583,0x000ddb124e3c2a76,0x00081c4f1f2539f9,0x000d86b17c619fe1,0x000024aecc3f2e6f}, {0x000536d45da54e56,0x00008208d6bf93b5,0x000bbdebc0443405,0x00079e4ec013e263,0x0000aed5ad15e7e7}}, + {{0x00005bb45d491a70,0x0004f423f59393d2,0x0008aca8510e837a,0x00078a4b50263ba4,0x00007b7f90bb8d6b}, {0x0003e303142c350f,0x000579c8254cdd54,0x00007552b0caff31,0x000f3c7248599467,0x0000b09e2c65643a}}, + {{0x0000623413605a2a,0x000e240c98eef219,0x000f738bd35a0ab1,0x0009cd49fd931436,0x00004b15c7d1bfb8}, {0x000dbe8e750cc8c8,0x000b2cf64775adee,0x000319f210d69428,0x0004c23bb86bcc34,0x00000692bf71bc31}}, + {{0x000dfb3eb9b38812,0x00060a94cb32edd2,0x0002674cf6680001,0x0004a91b08a68240,0x0000a3f5713fd970}, {0x000b550f67fb936a,0x00009cdf975dc67d,0x000d4f8ab994e401,0x00062714fffa287d,0x000076cbd2a37974}}, + {{0x000e793d6134da7d,0x000bfbaad67700c8,0x000044bc3a9b1458,0x00077a29db52c0a4,0x0000ae12105ecf19}, {0x0002c2d93e614751,0x000a3490a4cf1ae9,0x000abd69904507c2,0x0008d306a123b893,0x0000b8500fd782db}}, + {{0x000abf3c48af855b,0x0003d34d16708b4f,0x000a6ed07a282725,0x00019f730cb8f53d,0x00004ed0b7fb72b8}, {0x000816f3f3cbd30d,0x000d59260df36d85,0x0005cd9529f3ed7d,0x000918b8a442833c,0x000032c62276ad4e}}, + {{0x000505323819ad11,0x000a66c1746fd93f,0x0004515701dbeb19,0x000ac25d76041aa0,0x000039541e46492f}, {0x00082e4f5acecbe4,0x00047f003d36ac5c,0x00043a011a75a08d,0x000a029ab7481052,0x0000a07bbb938ff3}}, + {{0x000313ef5b5203b0,0x000154281306d809,0x0007c4b68fff9649,0x000164f75079e9fa,0x0000ddc7de06c72a}, {0x000a43bee64870b2,0x00058d075707ead1,0x0007b284d06638d0,0x00026466a4426597,0x0000eaeec425f7aa}}, + {{0x0009ce770a7c3145,0x000bb6337494e900,0x000a3e657770528c,0x00046d72c5c2351c,0x00002af88081301c}, {0x000002b6736a4987,0x000c03a730f22f8e,0x000a208d941c9745,0x000aed32506797d7,0x000039e0f2030605}}, + {{0x0003d4dfd808ef40,0x0004a58fcc7a1a47,0x000d19ae0ad86492,0x000cfe1443bf900b,0x0000040e88ad8cdc}, {0x000fb32cf6f58d39,0x0004562194b4f5d3,0x000375f3350db919,0x0008ebda78332dc7,0x00005e8bc45b952d}}, + {{0x00037937dd5f4dc8,0x000ad87dee916fc5,0x0000f9f838cdd726,0x00035bd06c293d17,0x000082c987f7d4fb}, {0x000e6e2af0881b65,0x000cecc3a59895a3,0x000db383e0d7d33a,0x00049900fdc70f9b,0x0000f53161d8db37}}, + {{0x00019a6df48dd005,0x00078c392aee7d99,0x00047da22270b610,0x0004292dad3f50d6,0x00004f9084de0936}, {0x0005ceb1e6d4f40f,0x00031c3e122362e4,0x000da2cb1ca24344,0x00089b025e58689c,0x0000ccbfb734d9f3}}, + {{0x00056e296b1d2a7c,0x00059aa411c51ff2,0x000534da9f87d206,0x000e1eef18a860a5,0x0000cafe63ce745f}, {0x000de8dab4468f3a,0x0000fb7e0c6a3275,0x0006193be6dadc7b,0x000ba599d3ced6ee,0x0000dbb9b3d4d796}}, + {{0x000f56c3e083f9ab,0x0000f603effd3017,0x0009c1d0099ea92c,0x0007d4e4fbe0dbe4,0x000058f3d2157ba5}, {0x000ca441fd54b954,0x0001acfab07e949c,0x0004453173fafb5c,0x0003e4c363c1d941,0x00009a533ea85b8d}}, + {{0x0006b73590d7896f,0x000cf513e5a6272a,0x00050b6b74115ef2,0x0008e802ec0c8d3c,0x000049dfcec6abc6}, {0x00026d644b778478,0x000145e3ef68d48e,0x000f9b1877d7f703,0x000ca9888630059d,0x0000e9f78f370136}}, + {{0x000cde2c3f33edbe,0x000ec5924d163d97,0x000758c8e3452e6c,0x00032d47700eb684,0x000001d15f0a9f82}, {0x000418cc0818999e,0x0007ac6d0ed810c4,0x000b5e011052bb51,0x0000cd43a902bc60,0x00002a9eeb8b57ff}}, + {{0x0009cbbf986df839,0x000414b85d3a71d7,0x00005e069c77aa0d,0x000f8f79dbf4a89a,0x00001526933b2c78}, {0x000844b710bed445,0x000cab49b00f7c1c,0x000718415d8c3d66,0x000865e5b99f683e,0x0000f111505bec98}}, + {{0x000b317b5fec109d,0x000ee60dd74ac94c,0x00063fc3e8bcf0db,0x0005aea594aafcc1,0x00002fd718e53283}, {0x0006bfa2ad47da58,0x00065d79c62c0537,0x00022ffb34140d60,0x000316bdfd12c941,0x00008ff86a718219}}, + {{0x00005b5677be2211,0x000de539062c2e91,0x000ee1b283c1d085,0x0000ba39a96ec9f8,0x00005bef7b5be408}, {0x000d5fcee8efdc4c,0x000c2f4b6689630b,0x000821b7000e829a,0x0006ac1efeb571f2,0x0000900fa02078a3}}, + {{0x000ac0bb76286c73,0x000d00f0c3a98fed,0x00087e24013a68d6,0x000322ae9cfa421f,0x00001657a067cc6f}, {0x0002945e67c00f12,0x00008ced22dd80a8,0x0005f0311374391e,0x000812c96c16f73e,0x00008be3307fc625}}, + {{0x0000d24be55cf092,0x000bdccaaa5ce5da,0x0008f0f8dc643eb3,0x0000f68b247fa097,0x0000c0f660a3692d}, {0x00041339e5f44cd6,0x00038483a65d698e,0x00067c073941df7a,0x000ed7f027b59c3c,0x00000823924cbd81}}, + {{0x000e36f7b744a49f,0x0001e8cea2841e6e,0x000a884d4e78ac9b,0x0002006211afc386,0x0000bc9bb9210e86}, {0x0001811b825aa7d6,0x000fb8213b25406e,0x00056e2bd62d5f8f,0x000b0d142d4390b7,0x000075f859df9893}}, + {{0x00057cfe953f406d,0x0006157c97bac651,0x00024b6d0bf2ff3c,0x0000073279a7e31e,0x0000cd7666235cc9}, {0x0005f46b2d76ca4b,0x000b0c3acdad6d7e,0x000d9e9e19354299,0x0000b8f783e65fb7,0x00000c641307f969}}, + {{0x000c59b6284008a8,0x000ad0ccb64fab61,0x000697bfa47d385d,0x0000a004d91fef33,0x0000201b805d7b1a}, {0x000b4cdbd7634bf8,0x000a8b1db7c95182,0x000a7b2b0dc8df9b,0x0004e30b652cca38,0x00009346424294cf}}, + {{0x000967ec6fb1f7d1,0x00013f8595e9532f,0x000760d79b0e83e4,0x0005cb6f0edab69a,0x000015764aef0bc4}, {0x0009ea00e9a799c7,0x000ac5b817fb15d5,0x000ad401b2f89864,0x0007afaa02dcaefc,0x00008c0d626305da}}, + {{0x000f2627ea780449,0x0004d0e16afee339,0x0001b1273fed7f14,0x000b2aaf8ae2dddc,0x00008e33f95fb10b}, {0x0003ee06234667b5,0x000f276fd3a5674e,0x000130d66a2b11a9,0x00010566bac7e07b,0x00002e0bb494f27c}}, + {{0x00054f5c6635026b,0x000a5a2c274a495c,0x0003d78d5e70c783,0x000d415596f0ba22,0x0000bea114aab96c}, {0x0006e4a26ec6994e,0x0003684f55dc21ce,0x000cb57be6c33dcb,0x000313516f6d6dbd,0x00008f669b41b988}}, + {{0x000c339ae3dc8edc,0x000f91873e0a1cd8,0x0006e937184cc542,0x000c8d374985e854,0x000064db8f80da1e}, {0x000fd63fc2e60274,0x000092f545cfdc42,0x000f97827e622ee9,0x00065dad12fb4d99,0x00009f63d4cea6ee}}, + {{0x0001d6efcdf88357,0x0008d95feecaa92e,0x00070e0938b6f590,0x000a9f0529925a20,0x0000cc42d8fc5d2e}, {0x0008c449b39c8e3e,0x00018b9ff21e615e,0x000c5b91c41f334c,0x0002004dc2fadbb9,0x00006d8605de3771}}, + {{0x0008cef52c6038cb,0x000595e25e5d6385,0x000a8e39d52faa35,0x00038504ffcbe219,0x00000875f8ef9618}, {0x0009872453c11381,0x0005c29f2eb78994,0x00004e6c20daf621,0x0005430dcf55501e,0x0000288ec97e36c0}}, + {{0x000c030591d26efd,0x0004d6d48ef7dce4,0x0009fd5d0405d2b4,0x0009ebbe289fc630,0x0000f97f0f01043c}, {0x0000b590022dc8ea,0x0004f976d4fbb359,0x000cecbf9a4644fc,0x0003207c99203ebf,0x0000edece41acc46}}, + {{0x000a362d721e9089,0x000a8925abc2aab2,0x0005350debe2e21e,0x00009fc91b3baa47,0x000048d66950c56c}, {0x000efeca03b29fb8,0x000f6bfda542da62,0x000139636634ba83,0x000c501c74caf2cd,0x0000b76d82e0f5b3}}, + {{0x000f6a56da8987c3,0x000e1aae6fe463a2,0x00059cd2c45e1dbf,0x000f0def678e8651,0x0000cea256fdc1ed}, {0x0003034d6ed1ba9c,0x000d0bf5e4dac73e,0x00084887837e23c3,0x00093ecd182d4808,0x00009476a1eec1f3}}, + {{0x00052dc77d7fb769,0x00013ff38f95cc7b,0x000733f795b3e21d,0x00090f612de561ca,0x00000f59a5455061}, {0x0007d48864196b53,0x000db4e3286957a1,0x00022d8d9ef59a96,0x000546011073136b,0x0000f1697c656115}}, + {{0x000317c1f8fc48ed,0x0006c0c0118eb24c,0x0000b3a095d3b7f5,0x000bbb608213550f,0x00005d5ad61f1fac}, {0x000a62d9583b13f2,0x0003577ecca5129c,0x0008c4140f3beafb,0x000b94480bfb33f0,0x00003f0a96a83959}}, + {{0x00031c65de0895cf,0x0006825228776fba,0x000cc88cce49fb2c,0x000d50e32472dd56,0x0000e65850b7a0d3}, {0x0002d9997abc2bc7,0x000247d47511ebbf,0x0000c31ba0b5abb4,0x000ba45f83b7816c,0x0000b1293a82eebf}}, + {{0x00014867763f2007,0x000c9a238ab540ca,0x0002810892f80e56,0x0000ae04b954b5a9,0x000009638ba155c2}, {0x00045f04b72df059,0x0007c79932f148e2,0x000f4ad95262dd9a,0x00058706f74ca482,0x00007911b864b69c}}, + {{0x0000b9c737140708,0x000667edd421d9f3,0x0008968685c81d98,0x0002788a4eac735f,0x0000023a157c9f57}, {0x000b4b003c82278b,0x000a81472fd167f3,0x000e0926bf38ac74,0x000e573126c462f5,0x00001ca6838518a1}}, + {{0x0001d85ac1d95c07,0x00033d583ca23351,0x0005118833877919,0x0002e080183f6e60,0x0000ff39e0469cbc}, {0x0002e04f226307cc,0x000f4bacaf9a2435,0x00074e2a51232d0b,0x0004f77d72b628f0,0x0000acec834b62bf}}, + {{0x0004545c8ad66709,0x000cfc688d8046a9,0x0004edfdc8d75925,0x000f05ae813c9e06,0x0000f09d9f9bb086}, {0x00062ca1db3df5b7,0x000c5b5ffcee14c9,0x00098918be83ad5a,0x000ac894b0684569,0x0000a8eed798d0b3}}, + {{0x0007970ef932b7f4,0x000f2ef627b27d5d,0x000b1f232a660b84,0x00047a67e2544348,0x0000f46ef3c81556}, {0x000ae5e50254dbfc,0x0000dbc8b8f53a6f,0x000f81f9760130dd,0x00056daa15ae29e5,0x0000531c6e3d8872}}, + {{0x0003fc5aec293c17,0x000fae2a8b3062bd,0x000d294391c65007,0x000ba8bc7a567060,0x0000e892b188f41d}, {0x000fea5c9cbb5a66,0x000a06ddbfb94b09,0x000a553dda4501d5,0x000c2b5f030fbe8d,0x00009f35d7661397}}, + {{0x00053c7d6c32ad49,0x000e202b6f62edae,0x0009918ca0e11d30,0x00064c3b7cc78f3d,0x0000aeb2ed04daba}, {0x000308348ade515d,0x000a42f73385ada9,0x000c1e0d66b4b6db,0x0008698732c86253,0x000004118fd191f3}}, + {{0x000796becef5c499,0x0007cd9c3e671366,0x0006808dd995c7ed,0x000e24412560f80a,0x000092c05fbf5a45}, {0x0001d3fb0df606cc,0x000bfd721b8febb4,0x00095b3096c03c24,0x0004db6d2624db4d,0x0000abed17dbcf2b}}, + {{0x000c4df41f71eab0,0x000a65e016a9a3c4,0x000dcf582ca058e1,0x0007f71ec4c968da,0x000026cfe1cad268}, {0x00096c924f46e504,0x0004491a46f19726,0x00070b7c43b16a8b,0x000da051694c0dab,0x00004ee2a129241e}}, + {{0x000900a5814e9247,0x0005613cf2606c1c,0x0005cccca1791d81,0x00048d0077d293ce,0x00006bc874a43ebc}, {0x000037e17cc1b585,0x0001d8d11bf83c3b,0x0005987d393c01a5,0x000228591bf583aa,0x000025efdb1260d8}}, + {{0x0003d91512d96c5e,0x0007e4b4ad0fa8de,0x0007b0bb68b040c8,0x00043c9dabe6dfa3,0x00000d7ee9b62348}, {0x0006da6bed5911cd,0x0009a6ece7ff38f6,0x000e5dd83880a808,0x00053087b04129e5,0x00001bbc6c30ba0f}}, + {{0x000e84dbadaa5f56,0x000cd377815fe20e,0x0008c62698e27faa,0x000347e02c37a9b4,0x00008e9bf569f39b}, {0x0007b248d318a6ae,0x000474a21df44e51,0x00031719dd246130,0x00074f8d75f0927c,0x0000b89236c1ad86}}, + {{0x00017308e1e58e7c,0x000867db13b9dde7,0x000306f021259e44,0x0001a4349c8e56e3,0x0000b578ecfaafa1}, {0x00084505fc0f3955,0x000c9cf62fa54df9,0x00036b0d416db73b,0x0005d79d27a78dac,0x00009d0ac179612c}}, + {{0x0004b0085e82cbc7,0x000abb85d10839db,0x0005f3bffdf52af9,0x00079078e4cbea13,0x00006cbba6b97a8a}, {0x000b82422c5fa00d,0x000a3ab275192d04,0x000e62fb0c96cace,0x00026b9b5e0fdcda,0x000028851c8af90b}}, + {{0x00012449f8c83d3e,0x00021b484ffb3484,0x0002a03c330d56a3,0x000de8f64f1a4e29,0x0000968c4493bb60}, {0x000b81bd33e1a3ce,0x000aa34a593e8cb6,0x0001cdb8a73950dd,0x000c5aca2dd94ae8,0x00002196c60ad005}}, + {{0x0002ecd9e6a86c14,0x0005bf4b4d550dfa,0x000030370fc52488,0x00002d4d8f6fdb09,0x000086c76907c7c6}, {0x00017c32a9829592,0x00024cd68ba39a26,0x0004748f380234c0,0x0006b496816425bd,0x0000b82c805a4841}}, + {{0x0008d667d5dc4d15,0x00099cf2f678940d,0x000694f978881dfb,0x000dbc7192ed98d2,0x0000de896634885e}, {0x0009ddbb3cc587e7,0x000d4b22b2d17299,0x000c81311033e3ff,0x0003b651ec3fe79f,0x0000aae471a0e640}}, + {{0x0004cd3ea6e3d753,0x00060a108f5907d1,0x00041859d6c895b2,0x0006353d871a05a2,0x00000971096c4c9a}, {0x000fa44e4d592627,0x000e35522b5be21a,0x0002456c5b24ea58,0x000fdf656200f0b2,0x000002a5a001b93e}}, + {{0x000a930b41d968d7,0x0005f45cda83c150,0x00048023ce9f4ff1,0x000af875c0255a23,0x0000618af75ec0f1}, {0x0004f16f88f7d864,0x000f11b9e30e6555,0x0008bf8f34a212db,0x000a4894a74095b8,0x0000f73c218673af}}, + {{0x000719474c355ac3,0x0007767feed9b317,0x0001495f898e9455,0x00044f2e1c716c37,0x00008656ae1b3716}, {0x0001a34aa65dcb6a,0x000d0b1c27a68be4,0x000536b1b2ce4de7,0x000a67af11085552,0x00009a395ffaad98}}, + {{0x000d3f4d7c754474,0x000c22c337ab180f,0x00072abe02a8ff15,0x0003fc10970c2e93,0x00003c652e722cfb}, {0x000c02f3684913d4,0x000297055bcefbca,0x00000984f91a0cab,0x0006a13ef0aec851,0x000083ec8cccc61c}}, + {{0x000a2ec8681d7400,0x000eb2e1e27bf54d,0x00021ca664bfbd0b,0x0009eecb3fb586d7,0x0000407b0b572888}, {0x0009fa49e7df0d49,0x000c8db83e5cf139,0x0008d62a8d255ab6,0x00064e972c4b357c,0x0000b7d17a711fc4}}, + {{0x000bb7a820c7747c,0x0008591aae71b534,0x0000ffe0e9e5c12a,0x0006d406fec50176,0x0000f79f9725d57d}, {0x00087fd706dc3021,0x00069d9f06684b74,0x00056d2a20079b4b,0x000da7fcbf83e575,0x0000880dacfe1d20}}, + {{0x00058e9c302ba867,0x00031efb88acbef5,0x000b99aabe4819fa,0x00022ba1871c5dd5,0x00009777c5931fe9}, {0x00099616dc64a369,0x0002003ba21a955b,0x000169185be37b58,0x0004c6afe824b374,0x0000bfc77522de74}}, +}, +{ +/* digit=19 [{1,2,3,..,}]*([2^133]*G) */ + {{0x000a727255b1faf7,0x000452ecd6f25da6,0x0004ecafd5a5a652,0x000bad00768839c1,0x0000a47db65e8b03}, {0x0008ff333559d9cb,0x000966d9582a7941,0x000861751aab0822,0x00061a8937485277,0x0000e70755598baa}}, + {{0x000d0b3a9d6e707a,0x000deb1e6b972802,0x000a9cdc816ee683,0x000300c2c0420274,0x0000882a6225668e}, {0x0005839e9d3e56bd,0x0003671e54dd8b9a,0x0004f99975d7c4b5,0x0007ae4fc6ef3c37,0x0000fc5b94dbc8c6}}, + {{0x000a6cf4e2db7b3d,0x0005cf35c5c3f019,0x000bdb6f316a364f,0x0006e1221c2e18a4,0x0000dc4b2200d39e}, {0x00078f1c8e821915,0x00009b22f686d8fb,0x000a305a94c1ab12,0x000cd236717c6df2,0x0000b14404a2f1a7}}, + {{0x000bac5f488d5a8b,0x000732f43799d874,0x000877268650340b,0x0003966fe4b48921,0x000034a8a84a9b2d}, {0x0000e4682408320c,0x000cb0f0f018a4e6,0x000fcb65b28d1028,0x000898348a10c5df,0x0000188d1de6f0a9}}, + {{0x000e203166a36c7b,0x000b4b6efd58b491,0x000b8cf6095cca77,0x00087b5779dd6452,0x00003dee080aa1e6}, {0x000516401adcd8a2,0x000c863c78749a85,0x0004b4b58a717ce5,0x000b1c6419ecb4d4,0x00001c953dfc5f7f}}, + {{0x0005a4caf12ba860,0x0000c62e1874ce63,0x000fde52b08d1a6b,0x000a7df06d9f7da7,0x00009e538c0d777f}, {0x000d8554da32762d,0x000fb42b270d4ac9,0x0006c0faa9638fdf,0x000554ad39e13597,0x000052af20a0ab43}}, + {{0x000de3d347ef0da1,0x0000e86b94de0d5d,0x00081ede6978f54a,0x00004b7554d28aca,0x000093f67d8e3d7e}, {0x000d6d4b2accb9b0,0x000eaac44bfff953,0x000334bb519d0743,0x0004597cb4782f84,0x0000e289c03cc10b}}, + {{0x000d7c4e6d753278,0x00056ab83d3f5734,0x0003ee15022d081a,0x000b4954cbe1d2ec,0x0000296520fe0c21}, {0x000b30c9f320aecd,0x00021106ddcb4b09,0x000f1b1848a3530d,0x000247c734a80168,0x0000496e464291ce}}, + {{0x00090a90c5f1581d,0x00021f437e8ccdee,0x00069fdd47f1c872,0x000a9c2e4357299f,0x00002d0483696306}, {0x000b5fd8d746c503,0x000c8bd661cb171d,0x0002cd43b1cc7a71,0x0001118c6769457e,0x0000150b880fe3d7}}, + {{0x0000f541d58b53b7,0x0006631c29ba2789,0x0002be7fa10f036c,0x0004b70db1abcfa3,0x00008eb4ca6712a3}, {0x0007134fe95a5dc8,0x00009bfdd4921a07,0x0004074f4fc33414,0x000344776f51e869,0x000039fcc742b48d}}, + {{0x000b7d00d6409a81,0x000dd67668b16ac9,0x000adde6b0cd175b,0x0003b8db64a49847,0x000077b648118844}, {0x000b3a9b85df3c57,0x0002bc406a3f3c24,0x000c75b3c764621e,0x00010e175b8c2793,0x0000b0bdce77a0e4}}, + {{0x0004813b51c2e64e,0x00005742d51835f0,0x000ecc30fadd8650,0x000bedf50410b627,0x000021e7232979d7}, {0x0005b9c764997bfb,0x000b0468d42cf2f8,0x00060163f7fd4379,0x000bda4d0bb60197,0x0000a925ae795940}}, + {{0x000c27028b18d58d,0x0009810f017ccd8a,0x000df69987d79344,0x000c3b7f59b76dce,0x0000dae304e49afd}, {0x0006d38b645dc0df,0x0006de4e4718ba52,0x000f7e2d51a37fea,0x000a96abfb95fdb4,0x00001485263258e0}}, + {{0x0007d52ce0614c9e,0x0004e699b20813ee,0x0006fd82d87f7d47,0x0004dd26820af588,0x00005ed613674375}, {0x000d90e6407c8b58,0x00045ffbeb5f5d2b,0x000b95218b07597c,0x000bce1f2697c7b4,0x0000512e42cb5fed}}, + {{0x0007ba3ac56e0291,0x00063e80caf76d16,0x000402f16cd63473,0x00035e2bb89abbd8,0x0000bbb3d5dfbc8e}, {0x0009ac52a39e9a6b,0x000961b06920b768,0x0007fad592b5ca40,0x000305260906fb21,0x0000df04971052ab}}, + {{0x0009f76f719e0c19,0x0009462eab4521c4,0x000093d14459bb4f,0x0009e86bfea8af42,0x0000c845c0599acb}, {0x000619f255078fa3,0x00078ac8483e27a2,0x0002f48e15d94779,0x0006657197c26f1c,0x00001b0bbf55242a}}, + {{0x00058c998e87eec5,0x0009d5bf41a5e976,0x00045718c33c7d9d,0x00008841dd908702,0x000048d0cab1bd20}, {0x00064a0500d4afb9,0x000806c82de8d3f0,0x000fd2028c970708,0x0006951017002380,0x00009d99f573ff71}}, + {{0x0005b58a79b2ebda,0x00028acfec229986,0x000fb08398ff8388,0x0006debd733c9ea8,0x0000aafb0b851b1a}, {0x000791b3602750dd,0x000f1eb59cdb2015,0x0005a8c5e25286fd,0x000fcdbccb6aa395,0x0000b0031f1a9f9e}}, + {{0x00036c2ab12c2213,0x00029a74288bef01,0x000813cfd8d67fb8,0x000edc144e8dbaa8,0x00001e994264cc9f}, {0x000d9b6095487f1f,0x000fbce801322d42,0x000d781e3f5b5610,0x000a5f163843cfa3,0x000018ad797760f0}}, + {{0x000104980838bd3e,0x0001d4b775d53e56,0x00032e1224b44db2,0x000b72ba0d374dfa,0x00002d1e993def8e}, {0x0000a19b53d7c05c,0x0001f3fcbd21bd3d,0x00040bbe6188a3aa,0x000cde0596e5ba59,0x00001c6391029e21}}, + {{0x0002b014fcbe1856,0x000b95e13a2b28d5,0x0006d0d4b135f7ef,0x00088914fd801d5a,0x000039be473b9886}, {0x0005da0db71ae0b1,0x00021d1359895df5,0x000fdaa6f9a87424,0x00086a4b6a38e436,0x00004b52ba6a46aa}}, + {{0x00012f2761b2f3c5,0x000c6614fc53bbdb,0x00042cf5cddc872d,0x000f394469eb926a,0x00000be240fe2dd0}, {0x000ba88505bde5bc,0x000093955a18ec7e,0x000d0aa3265bd9f6,0x0008cf2aba473ef4,0x00005612ed21f6df}}, + {{0x000c255f6facee1e,0x00050647746dc2e6,0x0003ba4f655a3120,0x0008bc9b08c0899c,0x0000ebed9359dd89}, {0x000b9c2f4be6d27a,0x0000379fa923f100,0x000ecd610db4fb74,0x000f5c0a7070baae,0x0000e5703880295d}}, + {{0x0002792b86361989,0x000c70e0ea1d1795,0x000cee4b2945e4cc,0x000e49191f5928e0,0x00005a092513973c}, {0x0001517aa0eec014,0x00022752e7d55c3b,0x00099eab1d2d4b0f,0x000a05485931759e,0x0000052fe15cd96c}}, + {{0x00079adbeed33d6a,0x000d9bb0b38172df,0x00014e338c2cf0c2,0x000c9072e49e377e,0x00006fbab6774105}, {0x0005c6b6657d9f8b,0x0007430c8a4f90d4,0x000273d11fc0a286,0x0004aa737c3f3d03,0x000085bc56c4d2aa}}, + {{0x0002379568227447,0x000d5a076a384854,0x000c19f6f86ca3f6,0x0001602451c729ff,0x00008f792dda5ecc}, {0x000f7755d8802213,0x000bd503649e95d2,0x0004c555e4ab7e6c,0x00014024efaf7d22,0x00008f2776245832}}, + {{0x0003c0c7fc837d12,0x0000c7efb2ca73af,0x000f96a119684707,0x00064f56fc1c5a8f,0x0000616d7204226b}, {0x0000d16d3a7388e9,0x00086882de5f323a,0x000e0dd9dbd81663,0x000c0961f0e26e4a,0x000063cf17e2495b}}, + {{0x0008a4eeed230bed,0x0003d0d557870091,0x000ee3dfbcc65b1b,0x000d4ec313899e1b,0x000034cc4615e318}, {0x00033c138a95f8fe,0x000a9bbf13a3a311,0x0008ef1d5a245a64,0x000b9c9d0bbf5c1a,0x0000d9889cd87380}}, + {{0x0006bf10834f02c6,0x0006e01e457cc3a7,0x000006873fbea818,0x00084c35cf9b69b4,0x0000f15793516087}, {0x000fb13cc05f8ada,0x000cdff5bcccc418,0x0007c49b2d54eb80,0x0008d54143bb4a8d,0x0000da62cdad2bc2}}, + {{0x0003138c57e0d795,0x000edaf3c2b5f7c2,0x00093802977df609,0x000f578d94d77af8,0x000044741cd9805e}, {0x000fe570c9322596,0x000592415a9bc8ab,0x0005e9bd3fe94d08,0x000c8e55ad18e986,0x000069c5461d43cd}}, + {{0x000d798d927c8bd1,0x000814dbe62b28e3,0x00071e6bacb5f66e,0x000791daf2930488,0x00002760f363ab82}, {0x000fb66661da3b86,0x000fc1f4f6b755e3,0x0003fd1f289bbecb,0x0009b11762446e5b,0x0000e4bbf14d6d68}}, + {{0x0004365872cbe5af,0x00013550dcea306a,0x000e8ef29eb2b51f,0x0000e37d100ab735,0x00008b0e721f1781}, {0x0003dfa6b34d3b08,0x000b96fbd55f946d,0x0007e3d70db70874,0x000d6f1c8c13ccdd,0x000067a70e52d644}}, + {{0x000ff9a5bce55c35,0x0000f532d926f394,0x0002cb267ae6b423,0x000cfe98bafbbea3,0x0000a47e9ec49a68}, {0x0001fac4aeef377b,0x00065088b8ca39aa,0x000fcb1b33346e68,0x0003f37c4595cf03,0x0000553a9a1f2973}}, + {{0x000db5078afd98f1,0x0009a5b1d544b457,0x0003cc22ac17ad50,0x00013f9c1f670349,0x0000b99d9c60b42d}, {0x00070f5e725ed33e,0x000ff4e76c5e3634,0x0009bb5aa3712ced,0x00077c075d63270e,0x0000feb5464092f1}}, + {{0x000266f5a6fae80b,0x000e77cf1d11cf5c,0x000c25058047f7d4,0x000d72477c80fbc1,0x00002ae841fc8624}, {0x000298af8ec94027,0x000f332b3001e607,0x00057ef418781d9a,0x0006e8f3cd58c10e,0x0000ae32ae0e9391}}, + {{0x000e957ce0b54fb1,0x0006972fc16c8d59,0x0009e1ac6a5ec5c0,0x0003cc0e2a509f44,0x0000ae99269d43b5}, {0x000dcfe7f8edbf2d,0x000c4c38a98ab0de,0x000f2b7fe3eb2afb,0x000251d41c2c671f,0x0000eed98b1d120d}}, + {{0x0007bb64c21224ff,0x000ff9321251834b,0x000b9cc1597d501f,0x000a592604354ccf,0x0000558105b4b952}, {0x000e794b05479262,0x00063092655e2220,0x000e9a375065befc,0x0002a52d2ca9bacf,0x000070e81eda0f95}}, + {{0x000c09aee59dc5b6,0x0006cb27787722db,0x0005983dff716c05,0x000b4410ab2999dc,0x0000ae99f605e39d}, {0x000a294b8a218144,0x000f9bdcdec057d9,0x00013071db4aa486,0x000cb97a73b9e06e,0x0000df58e4a10434}}, + {{0x00068854ccb5cd86,0x00099dfa38302192,0x000c6d69b04ec80a,0x00057f0b4854370f,0x0000695534ea04ee}, {0x000848f83b02064b,0x0002521616e17746,0x00016cdfc7d6dc14,0x000e2fc170a55382,0x0000c5b8c572de15}}, + {{0x00031c02e59f130f,0x00069247772d5603,0x00074879039b6cd8,0x0005614566fae8b3,0x000048c68435d8dc}, {0x000c560826cc061b,0x000ed2785591eb7c,0x000662c22886c967,0x00001f926b7e0d13,0x0000a566862f57c4}}, + {{0x000008cc2d2b80fc,0x000f5cb30fd3cfcf,0x0005075763e6f872,0x000b507b8d542c1a,0x000005ea27771e0b}, {0x000c23881dac761e,0x000cabc5e2d0bbca,0x0002d7913f2786aa,0x000fbe4390fd746f,0x00000c3ae903f765}}, + {{0x000fbd330058a1d6,0x00002ef54fb2b3c9,0x0008118044f97ba9,0x000b32b660df7427,0x0000e5c26ccf676b}, {0x000d3b470f7e68e0,0x000e721fa57e439d,0x0001dcdffd33db06,0x000ac717c7489a8f,0x00007ed817eec38a}}, + {{0x0002b4c45f41a24e,0x00055384c01b0f07,0x00027702295b3c4e,0x00021305b5164d72,0x0000213182bcadfc}, {0x000702c3c59da1dc,0x0000a16ef0a14df7,0x000d1c39e1032dcc,0x000e831d50a0d7c0,0x0000644edd83072a}}, + {{0x000e10ecfc435d32,0x000d54284c66de7f,0x000050600e3840da,0x000645f7f5805c2f,0x0000742262a04570}, {0x000079ecb500097e,0x000b82599044767d,0x00048fcfba3571b0,0x0002688ed95a5d75,0x000064270f1e0c8c}}, + {{0x000a352dc9346c12,0x00023d6f6fd90c8d,0x000b10eb8580e708,0x0001c595b7b0f310,0x00007adb908efd25}, {0x000fbfa7696a0663,0x000ea1f8a2cafde0,0x000feb5f1555b2d3,0x000b2d6e3ef80a48,0x0000afa35e99343f}}, + {{0x00009298d4b6dbfb,0x000061e47941a985,0x0001fd13f88f9bba,0x000f2b1b2d1d0d09,0x0000fb0457f5ddd7}, {0x00012be21119e4a3,0x0008bebb0f558503,0x0002cd850fffa065,0x00088482acdbcb76,0x000045182c46b168}}, + {{0x000a84723ad876d1,0x0000f44279fae932,0x0000d0bc65997267,0x00005d8a11945700,0x0000641726724e50}, {0x000b9a05034241c4,0x0003db865000b9ef,0x0003bf2b3fd17345,0x000b98f1f94024dc,0x000095ee6f7d1d97}}, + {{0x00026356f22e6e7c,0x00024c3361929d42,0x000551b739519480,0x0006ba8fcc02a12b,0x0000c866d3c65741}, {0x00082ef71dd1d595,0x00088888e5b5dac9,0x0002351988936e99,0x000912104b788baa,0x00005b38d17ef74e}}, + {{0x00078e44dd47b196,0x0006e3a7433dd3ac,0x0000e11737fb1f59,0x0000f52fec79569a,0x0000143a6b04cb9e}, {0x0005dfaa4b44dc9f,0x0002f6a3f86504ff,0x000e7730114be502,0x0005c09e9f83c1f6,0x0000a40c0dddb1a6}}, + {{0x000ad7a5d9a7c749,0x000bd0b3e979591a,0x000b18984b0c8837,0x000c930f85a5824b,0x0000523198d420b4}, {0x0008c2966f8c5ecd,0x000c7529d2eed63b,0x0007a1cec89e0293,0x000ce2ee43e4b31b,0x00002b2eb6888152}}, + {{0x0004dd204d849aaa,0x000d7aac3dc6fc74,0x0005bdf821345352,0x000d21fafcb0f7ae,0x0000c9bb14c6adc8}, {0x0007dbb7c46a380c,0x0002f91f52fe0303,0x000095c182afa9ec,0x00053300cfb28879,0x0000759b435956ca}}, + {{0x000ec5f3969f68e0,0x000700e5f0e3ef78,0x000faae20f28d8af,0x0009c666fe212050,0x000038ae7b9b88f0}, {0x000157ba6e527edb,0x0008a283e2353b72,0x00056665a3b00c20,0x000fb0165eb77eee,0x00007f6601175858}}, + {{0x0001cfbcf87353f5,0x0001b45b7fba97e9,0x000c88529186f293,0x00017cb9b7e1b550,0x00008d032cbbc13e}, {0x0008803b380a2bc6,0x000ce0e1eebebc7f,0x0000b3070c817171,0x000219f2106f56d1,0x000043c2b31388af}}, + {{0x000e260a60f350a6,0x0006b02cac1de1ec,0x00028ab680baa38e,0x00084fa849c9cc42,0x00001ef5b54b9710}, {0x000d93de4657ad0f,0x0005315d84295f24,0x000d5350e7f70518,0x000b1b81c02b900a,0x000086bacb6bfb11}}, + {{0x000b24f6599254b3,0x0002db150c28450e,0x0002523f2e8c805a,0x00016e26bd712d3a,0x0000cb4ef15dd141}, {0x000099a2e6f10b03,0x00002209f4af5024,0x0009230a1a79e9f8,0x00021b395212446f,0x0000a9513044d617}}, + {{0x000a8350780042bb,0x000fb7e23271391c,0x000a21d82290ac12,0x000018afb2dc3763,0x0000debc97de697c}, {0x0004834481d067c4,0x000bdc1916ceca62,0x00074c52ab470d9c,0x00057cbf28f87f59,0x0000397d4d3c4093}}, + {{0x000666ce7e64dcc5,0x000bfd2765fa9474,0x0009f08475f3c2aa,0x0004f9d5e1f47951,0x0000ac986961d2f1}, {0x000df6492b43447a,0x0007e6aa220eae5f,0x000687d74899951c,0x000190b730db55ed,0x0000fbcbbde88500}}, + {{0x000a91cf73f6d56d,0x000dd4ac318013b0,0x0002e920d4b81d0e,0x000af89bc5c79628,0x000009d7f0a3bdb6}, {0x000654d7b2096ba9,0x0001bac0128551c6,0x00000c2c28355efe,0x000725460cf1f377,0x000031708465e5f7}}, + {{0x0004c76c4fe9380a,0x000c3951a5ba03ba,0x0009390e3a0d1f19,0x000afc40881217e1,0x00004df0a88e83a7}, {0x000384ea5baa9628,0x000bb158f86192b3,0x0008e81ff877e966,0x000d6c002eb4c264,0x00006978687377e4}}, + {{0x0001f239971de4e9,0x00066b14c2bc0eae,0x000eec2d481fcc62,0x000e4ad4346cda42,0x0000812e9df26d7a}, {0x0005f2aa6c2765b2,0x000c0d4e6727b197,0x000cf9203e0c7cc3,0x0006a71e0008e593,0x0000cc328562e6d8}}, + {{0x00041b3f24c2e3f2,0x0000057bc18560ae,0x000dab87a402cc26,0x0004fc026500b9d8,0x000039eb743203ec}, {0x000cad3bc772a9a3,0x0003237fe16c560b,0x0001e4e176f45fb2,0x00086a0c35cb9a4d,0x0000b15ebf1dbd99}}, + {{0x00084d9c6bca11af,0x000efba4998ed17e,0x00030f688bbf5028,0x0003d90bfa347c4a,0x00003511c41bb988}, {0x000e04455e4a4408,0x0008eb1d2233b9f9,0x0004641f70bb7f2a,0x000bdb4bed7797bb,0x0000525831ae528e}}, + {{0x000feb573b5f15b2,0x000c92dc8b2b488e,0x0008915cb16e2b26,0x000bc6409f7cf294,0x0000643f5d55e347}, {0x0009c716263c840c,0x00099899481ae3bc,0x00030c337aa831fb,0x0008390b15f01b4c,0x0000ccb40e5e4096}}, + {{0x00034dc70a22059a,0x00095bf4dd319094,0x000ae3a347a94846,0x00022943c512ea17,0x0000f04ad0dee788}, {0x00057b3916c3e773,0x0005eba57fff134d,0x00040dab29c7a71f,0x00005e7267cf24ed,0x0000d39e69d8fe29}}, +}, +{ +/* digit=20 [{1,2,3,..,}]*([2^140]*G) */ + {{0x0008a58ed016cd2c,0x0009dc5bbdaedb58,0x0009d19885238728,0x000afe3665291aaf,0x0000f811206507f0}, {0x000a1acb3e875c10,0x000b5e9738520bc9,0x000b92c017b2fc48,0x000d82b6aee9ccf1,0x000062c002883ebe}}, + {{0x0000bfc1cb150958,0x000d323c4b7d62aa,0x00022080f25a592e,0x0002c2d0f11983fc,0x0000ee53be304d89}, {0x00037069d21f2963,0x0003e06e21bb46bf,0x00016283512c9c3c,0x000a67545bb2988e,0x00005bfe9d31adad}}, + {{0x000ffe7d4dc02b42,0x000fa98dc963d65b,0x000b49d8292da276,0x000c09ae3c49b3e9,0x0000d5431f45032f}, {0x0000664a7193cbb1,0x0005b74ae8346136,0x000506bf40d18877,0x0008baea2b86bf65,0x0000caf5ff39d91c}}, + {{0x000a3c28239094f1,0x0002a7547af8d168,0x0006f05d9a001ef2,0x00048b8e617195d7,0x000046b964d7a1f4}, {0x00087515a695697f,0x000f6e306cc6952f,0x000023ac96fca948,0x0000113b6adf1cd3,0x0000162f07c0aff3}}, + {{0x00048e3747fee076,0x0006b0d9898348ee,0x000bb6470ff98685,0x000533883290ae3e,0x00002a3d0c700bbf}, {0x0000cabf7bd10d8a,0x000bc597ff059eda,0x000be93bc3ae7d83,0x000241f3fb57e695,0x000038751d9c581b}}, + {{0x0009f4d266fe37d0,0x0001760efc6d5220,0x0008adf5ade25d2e,0x000f03a0868e028b,0x00009d6b114bf7a8}, {0x000ea7aedf8eb0cf,0x0009fb1cec06b1ff,0x00062e007b93aad0,0x000ddc47d356e3a3,0x0000f4c08726f1f8}}, + {{0x0002fe66ccc2faa5,0x000e00587d9fa66a,0x000e403eece74ae7,0x000a225f516ff2ba,0x0000d7e840f4377d}, {0x000c2e2512283ec5,0x000ed6d8436ba415,0x0002293851324017,0x00074b2665c85d3d,0x000061f518ef05c9}}, + {{0x0001472c11f2e3f8,0x000f1235e29595f7,0x000cb9e5a4678386,0x0000d17bcc861c54,0x0000988eac405b68}, {0x00035f6f32a0837b,0x000a0526e17e28b0,0x0008d388e1404513,0x000f50161698c9fe,0x0000290d6d02d161}}, + {{0x000e406349253af7,0x000d0541fb134eda,0x0003250ff90e5eba,0x000f014ee0bc4f4a,0x0000642c5dcc7c88}, {0x00087e04f81b7760,0x0007fe70876a68d8,0x0001713f35bc975d,0x0004f149dd30b653,0x000069332df09855}}, + {{0x000ae7c2d9437a12,0x000c29850a20044f,0x00048fd99d2889e0,0x00019802a1d48fe5,0x0000f30505df8965}, {0x00029dc332e26e50,0x000aaad107e92aff,0x0008fd1562aaff01,0x0002079447e8e0e0,0x0000a12a48f5f9b6}}, + {{0x000d35f7339a4cc6,0x0003ce58997c960a,0x00043b09663f8920,0x00086a9cabcc6fe5,0x000039df7b26f9bb}, {0x00004ebedc941e64,0x000a6c597d5e1f73,0x0005265ba7702fac,0x00039a977b7848e9,0x0000c0f2844cca29}}, + {{0x000dbd0c20287ea4,0x0007f00fcae2255a,0x0006723143b1f47c,0x000fdba1ca3b9a4e,0x0000a61bac73eb36}, {0x000d0243021dac3d,0x000f566be9665cd8,0x0007af8bea21805e,0x000fc6df478b1ca3,0x0000ee9bcbadfe87}}, + {{0x00079ee748becd5c,0x000fb2987ffe5418,0x0007a1ffc02d6eee,0x00056fea13c0fed1,0x0000c469e61b060f}, {0x000a5fdafddb3776,0x000368241f08c069,0x000473ca9d3dc589,0x000a42ebca40d7ce,0x0000bb3880b5b1fe}}, + {{0x0003d7e9a039d299,0x000ba8ffe689e1ce,0x000a4b476b07cf06,0x0005849486dae5df,0x00008b49ee4dfe4c}, {0x00043c12bbee57f5,0x000913e24f4393b8,0x000ffe1e8a443ce5,0x000d0a6060aa2a71,0x0000c42edaeca67e}}, + {{0x0009179cd2d899c7,0x00018a17e501d737,0x0001ed55b3c10df5,0x00048ca04cecde6c,0x00000472a0b8057c}, {0x000052a555cd31c5,0x0005e9616214105a,0x0003eccda90583ec,0x000ea0cb4bacb30c,0x00008fd7c1a7540e}}, + {{0x000f42db00490cd2,0x0000c875327c51bf,0x0008e6a820cf37cd,0x000a81bb95829eb0,0x00001100b078ae33}, {0x00022e00511257b1,0x0005f34ada3e41b7,0x0003409ef3752978,0x0008b1f12815fe3e,0x00000e75bd15cb42}}, + {{0x0006d6b5522c8d1e,0x00020c4a6e6885b0,0x0000674f8eb11a13,0x000d2924af2210e0,0x0000da23f4f5a620}, {0x000ca47360f31777,0x0003d3885edeca1f,0x00023929c46c52bd,0x000db6297cea00a9,0x000049714f519cd2}}, + {{0x00015a866630391b,0x00035087f3dd975b,0x000f2793a140b12b,0x0006fe1693367845,0x00002d5263bccea2}, {0x0006ef773fa9b4cd,0x000b902c1b552332,0x0001ac526ad1de38,0x00006620625db61e,0x000003a7023f0df2}}, + {{0x000651d3e64ea2e3,0x0006a54af43e7a32,0x0003ae4c9330df34,0x000236a6234eb5e1,0x0000759979693586}, {0x0008b45a9d54dd05,0x000e8978baa36a6c,0x0000ebaa91e3142f,0x00008dac63becce7,0x000003546ca0617c}}, + {{0x00094d5565a427ca,0x0007ad634538fdc5,0x0000250b2a522b76,0x00055a43ef3e4b9f,0x000011bf92e19348}, {0x00072023ecc683ce,0x000ea22ed8cf7efc,0x0003fc8983d51051,0x00075d723fc11811,0x0000633cf0cc488f}}, + {{0x000ba977892ce284,0x000b4cf71171d65a,0x000030c443d4463c,0x000202ca09b1099f,0x000056bf294a1a66}, {0x000a54aeda67248d,0x000d88b53799a85a,0x000ecc75a25eefd9,0x000ddc8babc0388b,0x0000a2fbe4a79e00}}, + {{0x00095a2a1a8636b1,0x000233d7f690fd6d,0x0009ce730bdac772,0x0003be8fed0f34c3,0x000049bee0dc1db5}, {0x00000cdcd095ded2,0x000d8b21e3539d6f,0x000e3d0a33e83d8f,0x00022450c4084ba9,0x00008d82e03dfafd}}, + {{0x00051869548ddcc5,0x00070778339d1ea2,0x00035f5f50d6f915,0x000e57fa01f58a4d,0x0000841779dd56a5}, {0x000f9ee31d9a47c9,0x000d12d602675ece,0x0009553f46c32b16,0x000eacdcd3c358d4,0x00001f6870b61f3f}}, + {{0x000628f163be6450,0x0005cd9c95d766bb,0x000f6a04ec28e610,0x000c81dc5fc2b063,0x0000b71137ccd020}, {0x00075183bdc05d4a,0x00016d6bde53c0ba,0x0004d7c49a68f235,0x0008d99b6b877a74,0x0000336e49572d6b}}, + {{0x0007024b4657059e,0x00054cef3d11f446,0x0003050457ac1b21,0x0002913d9da01ad5,0x0000b9bcb5753d0c}, {0x0007f76845c2dce5,0x00038068a539d060,0x00087fda517cafe7,0x0006476a3fa020c8,0x00005d290857bd93}}, + {{0x000092acd18e938c,0x0000f8464b918049,0x0004df7e752475a5,0x0002fde898e40329,0x0000287032729a40}, {0x000569905ec6b4ba,0x0002cfe346da592c,0x0007a24d3ae26557,0x0001e988692d9424,0x0000d3469d8df997}}, + {{0x000c0c46de1a94d9,0x0001e6f60e7509ba,0x0001814113ff2984,0x00011dee43d289c6,0x000082b07bdc2bd3}, {0x000f143c2c8cc0b0,0x00062d458fcb4868,0x000b290b3fd69d9a,0x000c0ecd23f4d8da,0x00000b05a52925f0}}, + {{0x000d9a283c536a93,0x000606d6de13d945,0x0003f7737b671559,0x0009ba66c6cc19e4,0x00003fc1b1d79963}, {0x000562a67776f95b,0x000cad8332c80b59,0x000985a2553fa97e,0x0007a0d727ab42a4,0x0000e8b82ed15b64}}, + {{0x00038e15452a1658,0x0008ec07d0a74207,0x0007878830171954,0x00059bac99ed3f63,0x00009f0f97c7517f}, {0x0001965ce020f57a,0x000438613db8102d,0x000c51ab225ad31a,0x0005e2bff0e582c6,0x00005ddc35e3ae14}}, + {{0x000fd23340796b5e,0x000244ac37f7a6da,0x000dbe23ff187911,0x0002ebb7c0ba278b,0x0000966a8a7ef6ac}, {0x0000656ed69c1c7b,0x000dd0d9607f7b6d,0x000ad9bd90340e14,0x000f7e0e40027a74,0x00000579cf7ce74c}}, + {{0x000a4a6a443029db,0x000d35ade4c87e84,0x00062de4100e89ee,0x00097010cdaa8575,0x0000847c3b75ba72}, {0x0001ce999ab945cf,0x00008688e09a9735,0x000d23c79201d508,0x0006dc217c5b0e19,0x000019d1d199e64f}}, + {{0x0002e32a7846845b,0x0008812fb2a589f1,0x000e90b88698136a,0x000814075eeda86f,0x0000f51b3d5754fb}, {0x00051260155595b0,0x0001d15da3c67d16,0x000140e2d64ddbfd,0x000dc7d8b5f5f20c,0x0000f851ed143a53}}, + {{0x000a956daaf5fc5b,0x000e309911eb23d1,0x000ec8c6b1504ee5,0x000ae12b96b7e04c,0x0000b90457102eb7}, {0x000f92df9e505e56,0x000f4a99a48ae2ee,0x0009af17826cba30,0x000679b8bbbc3531,0x0000173fb3989c78}}, + {{0x0008c28d0d5a2453,0x0001b72c3ae14d66,0x000253ce4f0708c7,0x00051484000a3ec7,0x00009309c4f55650}, {0x0002432cca3eca7f,0x000ca9872fce7dcd,0x0004a7f072317326,0x000e46d7d6868a46,0x00002b388e557070}}, + {{0x0000a1cb9e317b0e,0x0007686d711ded9e,0x000795851809a425,0x0008f153b3f1818e,0x0000d86c68fea802}, {0x0006abf4b7386739,0x00066a8acbc7e643,0x000a1ca448e39d09,0x000f6b81ecba6cba,0x0000b3922a40e4a8}}, + {{0x0002eda200d56537,0x000c11407fd1b8c8,0x00026245193f49e0,0x000b4f9680d4a834,0x000087c24bcc7dc9}, {0x0007394081d392c6,0x0004a065b74266bd,0x000b68e1b771b55c,0x000ad0c13127dd7b,0x00004fda7e5cd833}}, + {{0x0001b8779c1d0b47,0x0005ec5490f634d9,0x0001f8d01b7ced9f,0x000168ad06fe10a1,0x0000d38933d35353}, {0x00083efd496b859f,0x000068aafd56bb9b,0x0000ab4c451c3d2d,0x0001760199f69b45,0x00004f69ed69d503}}, + {{0x000d12f2dc71d6ed,0x00042873d131f911,0x0002f868f327b8c0,0x000917defbabf904,0x0000be4cf0dfeea9}, {0x0003a085c409e88a,0x0009aeb4bfae3400,0x00086f86c0eb81fc,0x000f9a5b91f6c11f,0x0000e0a0c02204ca}}, + {{0x0002d3ae9ccb3990,0x000a386ea321f4cd,0x000d286fce5d33ba,0x000d3cebb04af8c4,0x0000fed10ee266ab}, {0x000e7bc2e8e31050,0x00057520aee8c544,0x000ce0a339acd068,0x00016a6f230a6ff1,0x000075e1ddd84516}}, + {{0x000487cfa067d791,0x000998485f2e74ba,0x00090c71ee89da30,0x00025626b84584f9,0x00009454c2a51886}, {0x0004c5dd0d1b619d,0x000adc58a1b4f365,0x0006270fe416156b,0x00048f3ee0862486,0x00004e631cb6096a}}, + {{0x000d441dc42369c4,0x000382251bd526ee,0x000e37dea0d84bc8,0x000584bd68a92370,0x00002e0d68345dda}, {0x0007c227fd8dee47,0x00048d0a275abded,0x000c3b0af42e5ba5,0x000cc19c02809fe4,0x0000921ff2645404}}, + {{0x000de9f9684495a6,0x0009aeb8877a37c7,0x000d90721210b3aa,0x000e4725d8d1ea9f,0x0000a2fa64d2282a}, {0x000cf23abf5b2492,0x000693d5625863c4,0x000f2269b836d47c,0x0009574447f14d0f,0x00008a199e166c12}}, + {{0x000f1b41c43f8b0a,0x000c5c92e76aa8ef,0x00075e03be67ce10,0x000c96b9463a7ff1,0x0000c9d06afed35c}, {0x000b46fd3cde2980,0x000abe85c9d9c413,0x00090aace2da8075,0x000e0512bec57f1b,0x000053813aed6f3c}}, + {{0x0002d5a447d4f496,0x000f4cc8b20d45ec,0x000418aa14176c8a,0x00046f0dbfab3afe,0x00007f1b0c286c98}, {0x000399890aae6ec0,0x000cfc7435ee3b17,0x0004406fd3b6b369,0x00040be2ba195cf0,0x0000499740616294}}, + {{0x0000732badc26dd7,0x00095818d5141997,0x000bb5614607b7d7,0x0006880fb44d9c8a,0x00002af102cfc840}, {0x000ac0001b8ab28c,0x00095c267369f525,0x0005973685ec0685,0x000bfbaa025f9a67,0x00003dc0b77f0b1d}}, + {{0x000e6447298e5722,0x0009fd0c41f0b8d2,0x00063b53dd21c481,0x000fbb57d4fc248c,0x00007b8a669230e5}, {0x000412928c65556d,0x000525fd21fda1cb,0x0009148934fd899b,0x000cb3873e258005,0x000018bdfa0f4f09}}, + {{0x000257a9696db0da,0x000c6e1ebe6be7ab,0x0000ca7d82e3d900,0x00097095c69765ff,0x0000d64c3bbe074c}, {0x0004df32910ad555,0x000467cc458a6d2a,0x0000c4ac0377e6db,0x000f1093e2b3b04b,0x0000049f7467348f}}, + {{0x000169e23b854731,0x000c33e4abe5efac,0x000fd974f020f438,0x000a0585fe459ecf,0x0000dc81273d088c}, {0x0008b457450c4b5f,0x000085537b7106ee,0x0004064b559c8970,0x0008ed1e0bbc258c,0x0000c9e438969c32}}, + {{0x00002b120b62bc55,0x0000bc38a5440b51,0x000dd5bf16b581d6,0x00073e09f25725b0,0x0000e14d4eb778a2}, {0x00040298b8f7908c,0x000f4748986c1100,0x00095285cc66b0b9,0x000f606450bf5cf8,0x00008763246fb0b2}}, + {{0x000225d3ce00fd73,0x000562eb7a0a5fde,0x000b4dfc8355830b,0x0001747a548d6584,0x00007847c613610e}, {0x0003b60593ab7bd1,0x0006627b0acd38b0,0x000dddee8f6a141d,0x0008febdfc7a5ff4,0x00005caefa8d4b00}}, + {{0x00062bce0697db84,0x000364744ceaf19b,0x00056d73ed4ae9f0,0x0002e794e38b0f08,0x00001b958bddc911}, {0x000e012417427134,0x000459f114d48028,0x0006a3400cfc75c0,0x00058727f66fc710,0x0000756a6e62a482}}, + {{0x000219fe0b28a503,0x0006d50aecc17ee2,0x000b2435a97443af,0x00001d69d66a83c3,0x00000f96b615a755}, {0x00081a62fb7e0fb7,0x000a7c62b16f41b8,0x0005def12d53a09a,0x0001ef4471c81af9,0x00008288a92a8022}}, + {{0x000c8e03ffcc6378,0x00014bd82b7e98ab,0x000d883a9ce80865,0x000c6a7adb20e161,0x00008a1096cab53b}, {0x000e056288dce6c5,0x000cd762fbd62fcf,0x00047ac13d2142e1,0x000347c471a47759,0x00003efed5dffd1a}}, + {{0x00071f160d9b4fc7,0x00068ccde2a4414d,0x00076c3b61dd617f,0x000e354a6e4e01a4,0x0000394adf687f7f}, {0x000ace232376e7ce,0x0002f140cec9de0d,0x00033c69dda8f7d0,0x00032704299d10d8,0x00007b0ca9ef9df4}}, + {{0x0006175bbdd36444,0x000a4a9f510cbe77,0x000b49c3438fe5f8,0x000aa894881ab300,0x000081360b1cd545}, {0x0007e5993bcd0e48,0x0006045c198c176b,0x000dafcbc3541e85,0x00011a7fe785de3f,0x00001e918704e8ed}}, + {{0x0001b16510a4e3db,0x000cec6d1df534ba,0x0003cf6dff71bd1e,0x00007410def81561,0x0000e0dc1013565e}, {0x000f21a48f9585d6,0x000c6f1ecde37262,0x0000bf75a4803ebc,0x000ff11f11cb9818,0x000073d6d772e7c1}}, + {{0x00059115abc00ba2,0x000cb26283346405,0x0008740e0a87969c,0x000d13385833ae70,0x00005dac93f5d8ac}, {0x000474e1140c01a2,0x0006c16df16ac717,0x000f357675f37a81,0x000ca0828679abbc,0x0000ea1c1b3e843a}}, + {{0x000e6660da9a2e5c,0x0000a16aaba8f9a2,0x0000b0f1059a100e,0x0009bff504cde526,0x0000976e00939024}, {0x00047a5c49758124,0x00088fcebc829f02,0x0001304805038f43,0x0006e00b76e9a157,0x0000a4de4f473e09}}, + {{0x00004366e61d6724,0x0008876389533579,0x000a479bbe7f7414,0x0009e747e452682b,0x0000eaa1b7c1fe86}, {0x000039271078dc74,0x000ee3d381ab86d8,0x0006994cd10f6bc7,0x0008a6b5d9fbe311,0x0000aea87c1a2a42}}, + {{0x000ab7ba68c31022,0x0004f457d6aca783,0x000dbd1c13359695,0x0006f46d44d935d3,0x0000ddba6dd9dbfc}, {0x0009cd2d080265cd,0x000149797b3be503,0x0002299249e3b52b,0x0007bc0e30ffc768,0x0000209779d3d5e1}}, + {{0x000f9ce2a49386cb,0x00068985f70cff66,0x000479f3b00cc1ad,0x000fbdbdf292f316,0x0000fe7032ecf9f6}, {0x00019d7e849d0bd1,0x000249cfa039b34e,0x00027fcb6b7edb80,0x0008ff327d2e6371,0x0000d2b6d3758911}}, + {{0x000242405e3c60ec,0x000e43c80b9566b8,0x00010ea9d9b080a3,0x000e2687f432ff7b,0x0000ed8dcda9fac5}, {0x0009b28d87f8b2a0,0x00043ca1256e54e6,0x000a8cd15700f2af,0x00073f0b2cecd0fa,0x0000ed3ff9133f72}}, + {{0x0004550dc5a3b02e,0x000f5412342343c5,0x0006db191c975ce2,0x000a20f3bd209a9c,0x0000ef49682be20f}, {0x0006351ac6c47b86,0x000ab813a62241b8,0x00015ecebd6574ce,0x000f1186751398fa,0x000028d91f9cbecd}}, + {{0x000f6978b74df5f8,0x0001129ebc5501ab,0x000b5901ba0c64cb,0x000d8e622dcdc072,0x0000ebedc46638d2}, {0x000a78bab0638bbc,0x000e9c428abdd0a9,0x0008a8ed5636732d,0x000dc7c12d1f0e06,0x0000c1be2ebfb1cb}}, +}, +{ +/* digit=21 [{1,2,3,..,}]*([2^147]*G) */ + {{0x00041c6aa03286f9,0x0003770815b0f7b0,0x000348c284d38d56,0x000cd5c0fbd016dd,0x0000353258e98148}, {0x0002a661efd9b9ba,0x0008dd5eaf283913,0x000595edc99487af,0x00040c2b55829810,0x00008f3adaa1cb10}}, + {{0x000dd15cc4abcc6d,0x0007acc11a85a0ea,0x000e38c7cfb75800,0x0004e9a4844f7d00,0x00002e87f21868ef}, {0x0006dbfdafb2737d,0x0007c78ecbcb6bf4,0x000c3294b6571d8c,0x000d2861c088a49b,0x0000659cbbd569a8}}, + {{0x0009b01652619368,0x00097cc76f780883,0x000ad592edff2cc9,0x000558427112d204,0x000052f6c13013db}, {0x000b41e1bc721e44,0x00022128ce489bbe,0x000c29c730081266,0x0009efd2b1381a99,0x00009dda36961899}}, + {{0x000a8efe8dfc0aac,0x0009cb206c5feff9,0x000e08f69e7003e0,0x00088bdcea678c07,0x000046cbfbe7e7fd}, {0x00081f0181edacd6,0x0007c93bfad4cf46,0x000166e29a349916,0x0004fc2da8f82140,0x00007573537b65ff}}, + {{0x000bebd8693c0ce2,0x00010dc55e193bc7,0x0000e0dde6b99ca8,0x000d3d417badbf94,0x00003a2ff7257f49}, {0x00060a0c7bbfddb5,0x00081c1118dfd409,0x000b93b5a839109c,0x000a3b4702087141,0x0000534d9835ff20}}, + {{0x0004d062b05c59a2,0x000f5879c926f51c,0x000c6face7f545ea,0x000d0985a415dd36,0x000001497b737669}, {0x0007660300e35aab,0x00062195020d6f9d,0x000f8f2766a25988,0x00016e4edb995fe9,0x0000e827d9aa3b54}}, + {{0x000d3fcce93a152b,0x00024b91cc4b50f5,0x000d575f2dc97aea,0x00082a9bea228491,0x00000b0f559c3e87}, {0x00059372ba709ae1,0x0003193c52d9a904,0x00023ee401efcc7e,0x000302d47c1d80aa,0x00002c8592c8fc92}}, + {{0x0009708665ae5753,0x0005c56a03be8192,0x000d0ebf2fcec8fd,0x000acbb56af2db74,0x000082e5523113ae}, {0x0003f42e6dc35a1d,0x000e073f338c297b,0x00031eda91747ee5,0x000685f205a68c1a,0x00005a4916714ce7}}, + {{0x0008f1dbcd1c229e,0x00061773f6f1844d,0x000b29bcde833251,0x0009f684f808b930,0x0000a5e1fcfc8609}, {0x0009762c3a5334b6,0x000547b852bd1766,0x0001805b9b973ca7,0x00035282b13c8977,0x00005ad0b2362ab1}}, + {{0x0005af1b45cdc26c,0x0000c675c4635dcc,0x000b44dcd5bf1c0a,0x00064f5a3a7131e4,0x00004f42e03a4878}, {0x0009d8e6279a73f4,0x00083cfb4946f6e0,0x000537fe046fcce8,0x000d988539949b2f,0x0000ac6abd6d779b}}, + {{0x0005d5a30f177232,0x000e2df357dbfc95,0x00019bdcb3d911f3,0x00080abc003ca457,0x00000224695fee54}, {0x00070ac4186c55f7,0x000970319fe286f7,0x000f87d49b139392,0x00060915c89b2e48,0x000098096c2f2b08}}, + {{0x0006538c7cd583fa,0x0007504a26b85672,0x000751b6cd45e59d,0x0007acf63fefec94,0x00006952679a5609}, {0x0005299959d65ea3,0x000bf04521f25d19,0x000bd9b1bd07cbc0,0x00070fc519431095,0x00000659b6ab7a79}}, + {{0x0007da91e66c9574,0x000db441e739dd4f,0x000fb78e063f2620,0x0009ce36edde855e,0x000048530862d43d}, {0x000d326b7f93428b,0x00051bd23ff3de67,0x0002465efd4d3577,0x000e73c428efd466,0x000015e8b1961a78}}, + {{0x0002b67847e6ed6e,0x000e4c4145b4f3a8,0x000f47bea8e5ec4b,0x000709c101bbbe13,0x00009d64c85e6167}, {0x000258b265c6bb03,0x00052603cb62b641,0x000250984d93deb8,0x000184b5dac1f036,0x0000af93396d0909}}, + {{0x000454335c724255,0x0003604c60816548,0x0001f152c66dc7e0,0x0009f43ccde48316,0x000077cb80034588}, {0x0001b1a2abd9c810,0x00068976064aced9,0x00035e3f6e20d90a,0x000233aa7895ed69,0x00000b721fe71914}}, + {{0x000d5d2047e6bce3,0x000c5f2809e64a06,0x0004d29f57d77556,0x0005ec248cb3669c,0x00008e7d6893f712}, {0x000931b877a115c3,0x000d470ee473cb67,0x000837d7a680ceaa,0x0004ffd6fb968f2c,0x00003fd042aa3584}}, + {{0x00064b623466f8c0,0x0008b9ac46b1538c,0x000eba98de2d82a3,0x000aec359395681d,0x0000ec5727adf754}, {0x0001f29793223a53,0x0004f341230b56ae,0x000acb4a49d60698,0x000a48020fcf8ecc,0x00007f03e722fa1c}}, + {{0x000ffae3070a4cd2,0x0006c399579ec252,0x0009c2399abb8b5b,0x00058f54502d33d1,0x0000d2a871f20c1f}, {0x0005a7fcf9b1e2ad,0x00026c80ecd64f25,0x00094c286464daa8,0x0007fc8c2edbd839,0x000055ea342cc28d}}, + {{0x000e3f38fa4d2ebb,0x000a628413f152d1,0x00063d684664336d,0x00099a6d07b3bf0e,0x00002d0a21e1fc6a}, {0x000404c2235d7455,0x0000aeab84670292,0x000ec318fdc3d200,0x0005e4ed594202e0,0x0000b11acb9c1288}}, + {{0x00046804810695a3,0x000de9ba648df863,0x00051c17067ea77a,0x00015a8eb8039779,0x0000789adb2007f6}, {0x00094251845852f8,0x000597dd3f5a4cf4,0x000e0ca98f3213e5,0x000cdfe4111dea4b,0x000092e80e45f558}}, + {{0x000a77ccd1cefee3,0x000ada8387b76cec,0x000d53bdf4c487a2,0x0008553f0ee08fb0,0x0000ecf034e8072f}, {0x000d2cf4f5b884c3,0x000eecc4f5c4d598,0x000c4b8e274becd4,0x000fe18c1cf74be6,0x000033bf7648e6a5}}, + {{0x0003309c7f4956c1,0x0004bbd8072c2e72,0x0007386ff557979f,0x000400bd4c7d2735,0x0000d3097df1694f}, {0x0005419b24eddf2b,0x000fdec834dd8149,0x00008eb07ebf4b08,0x000944251b925cec,0x0000cf5bd49c0873}}, + {{0x000cc2b7fb7521cb,0x000cfca0448c3e29,0x0000c79198983000,0x000fa291122a2253,0x0000d819e4970366}, {0x000f14ca4bc473dc,0x0009d3f09a14b90a,0x0007cbc4041d07a8,0x000c0822fd840600,0x0000cc4468dc6484}}, + {{0x0008e2d3ac836bc5,0x000fa9f064bbcc21,0x00031a1b07517bc7,0x000b9cf6d996bf89,0x00007baa282710c0}, {0x00003689803b8618,0x000b9bc669415358,0x000927d51b0e7c4e,0x000917e22409c259,0x0000bd3b557da0b6}}, + {{0x000c21b01b14908b,0x000de4a8a31f4d7d,0x0009e8787bcbe4d4,0x0009e9ec73c5ca1a,0x000065165667e1be}, {0x0005566f103c7aad,0x00068820ea33101b,0x000afa05ad1acf04,0x000445b6f0613b7c,0x00003aa0c574081c}}, + {{0x00087a329c8ed384,0x00092d62bcbb14af,0x0004e3df6cf9f17d,0x0004809f2aca4e9d,0x0000ccfeca144a51}, {0x00041c1043c54414,0x000eaa7c807bc51f,0x000cfbed8d9c0b13,0x000f8173166f05f5,0x0000edbf62fa73e0}}, + {{0x000fa8831a8af89d,0x0007f4021d8aff5c,0x000dbeefd7a8d79a,0x0003ee6c7dc72f5b,0x00004336006bff0b}, {0x000e58807631b0f9,0x0003b6ca73f467c0,0x0002ef220dd8f222,0x0008681b33098d75,0x0000a8d5cb00940a}}, + {{0x0003a4465384338c,0x0002212dac4e37c8,0x0001527f99ca46a7,0x0007381b651de843,0x0000c4920dfc4a73}, {0x000defe279009365,0x00039c537acb711e,0x000d3684b5f0845f,0x000b186a2879fe49,0x000022890953f9ea}}, + {{0x0001888a21d8062f,0x000e6963bb268fdc,0x000fec2167faac6c,0x0000b12e8fefd495,0x0000fde1d1fbfc68}, {0x0008c3a8ac6e9251,0x000f165cd203fb4c,0x000da224da3bcfee,0x0009c76f7dcac95f,0x0000d2bc8ba06270}}, + {{0x00089f169709f64c,0x000ac1b2dac6c136,0x000f04db98b2b71f,0x0003d6ff19b52803,0x0000984ae8728ccb}, {0x000b07419ce76448,0x0006d40a72dd7560,0x000ce49974f769d2,0x000dc4f4b5d8a990,0x0000484a9d5d3a20}}, + {{0x000d0288e3a46587,0x000ff69f35c7af94,0x000ef3387d56412e,0x00038a84fec04a52,0x0000600f11d74aa5}, {0x000b58c2b1ee7fb6,0x0002d3e464c1a9a6,0x000491727fa92dac,0x000d338d06f7e0e9,0x0000c00602b9dd0d}}, + {{0x00025ec2a0a97736,0x000894c54fcb0993,0x000c79b1272a0fd3,0x0001dcc6d7533abd,0x000003f49dcc0749}, {0x0002691f11601ac4,0x0004f081e7ff4653,0x000af4c3b359a421,0x00031b5ba1c105b4,0x00008a6b864d6305}}, + {{0x000fd57dfc2a81d6,0x000625b39bc719c1,0x000bd95af6db2f85,0x0003bed53048b27c,0x00005b3082cf3fe5}, {0x00075165c86fc86c,0x0002759020878e09,0x000c8d6462169c4e,0x000a7ec795ff10df,0x0000740ea7b723b5}}, + {{0x000c734b07d5d331,0x000c0fc9ac292c4e,0x000275f2fa5774f9,0x0005b64cb6519644,0x00000932f708d9dd}, {0x00099e5189916537,0x000dab579fa96e9e,0x0003d581a498ab6c,0x0008466030186ad2,0x0000487b0b95ef84}}, + {{0x00035b4387096d4f,0x0003df6e71b2ea65,0x000937553df75f29,0x00010b31d49965f7,0x00006c0c5a717d4f}, {0x0008a025671103ec,0x000e123bb180d8a5,0x000b43b65a239855,0x000f61fa25b52a09,0x0000f771fc550da3}}, + {{0x0001111d075e5253,0x0004096fe18d6290,0x000c240dc07656fa,0x000abcb58d552e84,0x0000e3c9762d4f9e}, {0x0005b0b103502350,0x0009f566340da097,0x0004d4fa8636d51a,0x0005251761de667f,0x0000e20349d77617}}, + {{0x000804e04fc602d9,0x000ef28a89d0ded4,0x00086f1cd8ffccc5,0x000cc149f28d2957,0x00001630741537d4}, {0x000f2ba70b640180,0x000b39ff8919aa63,0x000ddf64f6350c9c,0x000ed9819793e411,0x00009e24759c0d79}}, + {{0x0000f19f4fd5dffc,0x0001ceff4bc9c7a1,0x0008f73cf0116b0c,0x00061f4e478201cb,0x0000d21b82382518}, {0x0000765f0c491b5b,0x000013baad4faf40,0x000300e32e5639f7,0x000d8d4ad69ae26b,0x000039328a470b33}}, + {{0x000c84a820e49c2a,0x000632dbefb4207c,0x0003a58a5ed4c55f,0x000382bac82b1f88,0x0000b15eb7c77391}, {0x000f20e1f1265f7c,0x00061058e88c9ab7,0x0006384f1cf8b862,0x0003879fb6321663,0x0000cb9aeb634518}}, + {{0x0008bbf43ae6df99,0x000e2d6ab4528524,0x0005334bce88a401,0x0008055ec2fa9a47,0x0000d4dca87805f4}, {0x0006ced09a787cb1,0x000d0999c03d5db5,0x000b45d8fbfd1741,0x000babb6f0173a87,0x0000a2096bb2323f}}, + {{0x000a21bedb16d4bd,0x000e6b75d1d16e09,0x0006bc64045b4bcf,0x00039d063765236c,0x00000a23f6f637d9}, {0x00074751ceb98088,0x000dba263631ffca,0x0005c24d5fa3b792,0x000afdf1a7fd6b25,0x000059077c5a10c5}}, + {{0x000da27ddfed9085,0x000e4268009b9ec1,0x0004b5d55add639a,0x000c645ca7d97f79,0x0000f7c88b1f440c}, {0x000dd0e67085ac15,0x00091a87c0cc8497,0x000956ebd13e13f9,0x000f43f208267e50,0x0000950c4ee3dc87}}, + {{0x000767687730a906,0x00086fc6d094b775,0x000902ef6b5667e3,0x0004940ab659cccb,0x00002c5c7ca26500}, {0x0007b7f151927dcb,0x0003251daf1587c1,0x000e13028bcd4fe6,0x000bec56ed9812dc,0x00002b1a165bd153}}, + {{0x000922b16d41f3ed,0x0003171dcd2ee6c0,0x000fb901db1d6685,0x0003a9ae9e9a3b18,0x0000fa1c572a62e6}, {0x000eddd1acbe3647,0x00057c3b62c81e6b,0x00095ef93b30ae10,0x0007c242fa98c96d,0x00004313bd27d329}}, + {{0x000590d655a44d1d,0x0007a791be183844,0x0003d3938484d868,0x000d526b8e7d0675,0x000025aa4d1a7d94}, {0x0002fc193234941e,0x000c53450cb67e66,0x000dd6fd063acd11,0x000fb3683bb3f0a6,0x0000017b88e1cd67}}, + {{0x000dfe137bc4a4d3,0x000a6f70d1220567,0x0005d43015bd4b27,0x000073c03a5810f2,0x000038319b9cc364}, {0x000e596aa8cf3d1b,0x000eff7eb1c9ff82,0x000b8e2dfd434646,0x000515518b703e5d,0x00000fd92c1335d7}}, + {{0x000f2ed01b9068a2,0x00003e0f5e11999b,0x00081ad391d14a72,0x0000ec9e891cb005,0x000034d551dfbd27}, {0x000e3e94269c7442,0x0002bdf2d623f98c,0x0000ac69fcba079f,0x0006b8e6df04439d,0x000094022be24a1e}}, + {{0x000624f1ce0f1e4d,0x00066318ff1e2b3e,0x000cf68fd9152d17,0x00026fa5d91b3a87,0x0000450cb4e504c4}, {0x0006852e3c9f1ef5,0x000539195d3f5568,0x000858b621b8887c,0x0002f2742dea5594,0x00008e75022bf2f9}}, + {{0x000b07bf5bdb8d74,0x0005adfca4449042,0x0002e1d1ead50f78,0x000484f785f4721a,0x000049993fef7c9f}, {0x000c8c0c7eeaa416,0x000b7f465774cab3,0x000a78325aa244d0,0x0000243470b1e003,0x0000822e9e020093}}, + {{0x000c845fa39fbc60,0x0002293ec4ec1d40,0x0006379a019debff,0x000755eda9d51a99,0x00009f94dc7743ec}, {0x00010b3843c61a6b,0x0009f974867d68ea,0x0005dbef26fadbc5,0x0008f2b3e875258e,0x00006cb8dd761e1a}}, + {{0x0000ad3ad5e6e384,0x000e291dd98cff89,0x000718231c43f57a,0x0006f21945dfe8c7,0x000061970b97a133}, {0x00082cfd677dce23,0x000f52f49f824e05,0x000b7784bdd02efb,0x000b85cc1c90ef12,0x000014d0c33b781f}}, + {{0x00060c791130be5d,0x000a3b5dc85a2ad9,0x00089ae96957ee66,0x000500bf4143b0b4,0x0000de968fbf16d4}, {0x000480887ad52fa6,0x0000d040ca0e6706,0x0001fc876898a967,0x00096b92364ae69e,0x0000dedc76278467}}, + {{0x0003de2fb18ca7df,0x000864975906c34c,0x0008c99c77a4177c,0x000744f9db29337a,0x0000758b714ec06b}, {0x0000287f5082885b,0x0009f9b49e575a58,0x0009f2005dcaeb2c,0x00057c4f90329a9b,0x0000e7c828178ce3}}, + {{0x000070cd2e84852f,0x000127c1c04293d3,0x00082b797b024206,0x000ae76c1b1ad1a5,0x0000f75b079c03e1}, {0x000e4e6207b73a21,0x0008751ccacd3daf,0x000a230c168e33ef,0x0003bebb75b39a80,0x0000ce273b967045}}, + {{0x000e4cf7ea2bf1e2,0x00002aee47c0de34,0x000ad4d02064992f,0x000d96aed6f23dc7,0x00000258e1fb9565}, {0x000d4dca99aff602,0x000578622da2f6bd,0x00013f2c376d4555,0x0004856bed645115,0x000023c2392fbfc8}}, + {{0x000c65742561338a,0x0000f335e54959ea,0x000e714b02e99bd9,0x000531924d4cd35a,0x00001199bd5dd95a}, {0x0009f7a238d15ffe,0x000e09222be15441,0x000374b05d41934b,0x00061df1d9a1aed2,0x00005508ba021f85}}, + {{0x000279df95030f56,0x0000f9ea1ab0d177,0x000f99992a351c3b,0x00064e9220c44b6d,0x00003455043e8fbd}, {0x0005de277b82a5bc,0x000ac3b68530cd38,0x000ca086694c0653,0x00063ce81e602ff2,0x0000ef40f405268b}}, + {{0x000b7c3a8d2bbe19,0x0001bb32fc4ebdbc,0x000caf9eea029867,0x00095c27a58c50c3,0x0000c0ab3cd9d1bf}, {0x000e9607ac0e8a6e,0x000fc1426abde891,0x00074904b992b549,0x000b98a3dd1a72aa,0x00003c2e33afdede}}, + {{0x0000a5f93a4e96de,0x000c85e5eafc25af,0x0005a62bdb7ac424,0x000558424b99e578,0x00003c24c7e4a4ea}, {0x00094a38275f9857,0x000c9c7fa4275e32,0x00081251fc7ef065,0x000800bc303dc5a0,0x00001f0537c463a9}}, + {{0x0009bd0faab7ac3d,0x000e80f256cd6560,0x0001cdf6644f220a,0x000341d90e8fe916,0x00002d221d5ca965}, {0x00019a678ae67ba6,0x00030f33b841d032,0x000d5af5d16ce5f4,0x00070b06fcd2214d,0x00007fcb30a9c986}}, + {{0x000c6d4993412281,0x000466c0976b7162,0x000e48195e68d8b7,0x000a7c3a400a9f5d,0x00003f11324f1feb}, {0x00068a9ee5214a73,0x000895b4a3f14392,0x00035066970be472,0x000438b2cae53e6e,0x00000e849079c7c8}}, + {{0x000ab4705ac23184,0x0007e535509d183e,0x0001a9d696cd87c5,0x0003178c8f88a2f8,0x0000610bf6f3760b}, {0x000d814c4dca8a05,0x00010f8ceb1157a8,0x000537398b47d20c,0x000424fa8d6446e9,0x00001fe7a476cec4}}, + {{0x00072bb2d03ec2ff,0x00025023cd075ac0,0x000d7ecf72f0f3d6,0x000efca8d8dd2d56,0x0000a3d270a63597}, {0x0005bdb8fa49c984,0x0003984320dca40c,0x0004211099e72bd4,0x0003af1f17bff561,0x000078129a3f3005}}, + {{0x000cb54f682dd291,0x000a10b13967c1cd,0x0009cda2fbf3fcfc,0x00038cfb00e857a7,0x00003a87c6a0cf82}, {0x0002489b23f1d3cd,0x0009ca41c9084179,0x000f0d3e98ef9e8e,0x0007bab2cdc4e833,0x0000e159ac26b23d}}, +}, +{ +/* digit=22 [{1,2,3,..,}]*([2^154]*G) */ + {{0x000b73dbe429e6f4,0x000c3138ffc05f82,0x000a3b67f590e285,0x0007bcb900484c1a,0x0000aec77b3d8941}, {0x000e442032e0e61f,0x00070b271f75c0ad,0x000b1798d01dbbbd,0x0007e3958d61f8ee,0x00007c8bd0d03ca0}}, + {{0x000e6d0844c0e6f2,0x000c3284a1e580fb,0x000933260fbe850c,0x000f1d49fa978af8,0x0000f631a40a5bff}, {0x000151afce47898b,0x000d97771637ce9d,0x00029ce3be231f37,0x0001bb143714824a,0x00008614e5ed8aaf}}, + {{0x000e80c351d61ae9,0x00034e58cbd057c7,0x000018bc8ecfdb95,0x000c2ca5a58c2283,0x00007d50125d76ea}, {0x000465a381106373,0x00041f1dd517dfc6,0x000184218d7ec5e4,0x000ea6cc1e142bb3,0x0000006ce0efce95}}, + {{0x00003b993a36842c,0x0004ff423a275d4d,0x000ea1fe70c60a7c,0x0001025151148bb1,0x0000be81326d9e64}, {0x0008d837d64990b2,0x000d2db8c89e7194,0x000692e72baae2d5,0x000e8c0dd17618c3,0x00004b6ea9b40d6f}}, + {{0x000fd692bf95567f,0x00053ebad159ba68,0x000340f3a53fec60,0x000ad6552431579d,0x0000df4e7d133f2e}, {0x000144d67c90b92c,0x000167158f063ae2,0x000abd10013a5c86,0x000729b8ee2bc967,0x0000b9abf1b96668}}, + {{0x000f140c73562945,0x000cfe3add38ebf2,0x000df7dbc2a2e566,0x000476e0e3f9dbd8,0x0000ae24bb41e6fc}, {0x0002d58819c1c23f,0x000517b2f5b645b3,0x000e863c8b271901,0x00069def3f6a2be5,0x00008f784e4c8202}}, + {{0x0005b1d21ec38f34,0x000d1346ef526b08,0x0001aaf39a530ac7,0x000336aa7792bc04,0x0000759ab5765fb1}, {0x000d665388e67c77,0x000e31ff335430b3,0x0002f4df0b037e8b,0x000bb1010d23084e,0x000085bacaea9944}}, + {{0x000e2601dd188f66,0x0009e8c268296001,0x0001a9eeecbb5766,0x000b9f7ae1f2b5d0,0x00004069ef9b9ba6}, {0x000798a2fcf1d940,0x00063d59f8620844,0x0009af2a19d2299b,0x00026abee6e38778,0x00001f7192df4bae}}, + {{0x00053eb08c850a38,0x000ec6ad45b295c2,0x000029b17c77b7ec,0x00066724f2c9dfb6,0x0000f4bbf5043f22}, {0x00027dfbadc1cfc0,0x000c9c9b43c55b3e,0x000f3904abe353eb,0x00010196a71e4ce1,0x0000960958fa8fb7}}, + {{0x0002fce77ea0ed70,0x000f5ed5ec66f089,0x000f00db8587d743,0x000085766c95169c,0x00005b6bf27de7cc}, {0x0003e7f9cb37cbcf,0x000f9ff50ac46924,0x000c0bf82abe04a4,0x000e20c506429d30,0x0000cf6603411d09}}, + {{0x00079d7ccf998928,0x00053aec1803601b,0x00023b4297648463,0x000aff355968bc95,0x0000d5d4c0854553}, {0x00057ac873f512b4,0x000cd320d16e1b4c,0x0004ed91186b08c1,0x0003e4608228cf7f,0x00004c70e3dd005d}}, + {{0x00088f1944156ff7,0x00011e971438cb5e,0x000058fa1b18ee73,0x000cdcb89e85f304,0x00006e2f51891388}, {0x000232f34ba5cffe,0x00003843e83b0cce,0x00097c9aa7eaba9a,0x000ece9610dcd10b,0x00009f5f0d594daf}}, + {{0x000986810095f72f,0x0001ac29a7c24844,0x00032e78ed086800,0x00098f626c35c5a8,0x0000d83403c23bba}, {0x000a7f5c20ac35fa,0x000d88d0bbc7266d,0x000e582bac8b42d0,0x000935657786ea28,0x0000bf7fdda51e43}}, + {{0x000de66539c57709,0x00029ab48f57c79a,0x00053b7f7c92d81d,0x0008ccd6929d04ad,0x0000d6805ffa633a}, {0x000c653e3c30e224,0x000a25014af38c1a,0x0000fb9056ff7e72,0x00036e84863461cf,0x0000feff997e9cd2}}, + {{0x0008f4fbfe1bb87e,0x000b234bf5009643,0x0003e08355419bfe,0x00059cb34567fe1c,0x00007cd632f8cfe3}, {0x000b06cf3a3127e3,0x0004a9a97e2d7d25,0x0007d0094b3c4c3d,0x000715572ac1af4e,0x000097c70eb31cd4}}, + {{0x000a30dbbe55bebb,0x000acbe0f0b73fc0,0x000b583ca72ac528,0x00058a26c3397a15,0x0000690f6fc7ed03}, {0x000c62882b679996,0x000a4e77a75d9acb,0x00038d76d9db5245,0x000eece8f87c7f73,0x0000595862fdb1d5}}, + {{0x0004badd8661917d,0x0005007f84c3acbd,0x00040fa3d59ba026,0x0002ed184e937c66,0x00001369d0f0a780}, {0x000fd94d28839e74,0x00021cd14e0c49ef,0x0007442dff19b151,0x0001044685338c05,0x0000b46081f8392f}}, + {{0x00011d53d358e352,0x000ad7cfd455b082,0x000bac2fd8bbbf06,0x000acc8cd30a233d,0x0000d70c7b7f7357}, {0x00026832fec7aa71,0x0005d1d2ead1c40d,0x000b7a761f51ab79,0x000816ec07f749be,0x0000c5ae5e6830a9}}, + {{0x00099bbb81316b37,0x000eee7327744649,0x00036c6dd3fa17e1,0x000cf40d417b7a4b,0x0000cdaf38243743}, {0x00092c6085af8552,0x000765fbc2a4eb0a,0x000a3aabcb9c1f75,0x000715c991d08c04,0x000063b26b5053f4}}, + {{0x000869e8f8653478,0x000498cb16b98bce,0x000bd2b92786668b,0x00070a1e52fd90fb,0x0000fcf025fb7145}, {0x0003105fe53d4b78,0x000c7d1f08cb8199,0x000e90aa136b8d03,0x000adab87b877995,0x0000ff5b8a7416f5}}, + {{0x0006181e6127b8f3,0x000e26986975931a,0x00050fa780fda309,0x0004c9cbf2d51ead,0x00000b52ace1f172}, {0x0007cae0f0e4d834,0x000d5d6c4773705a,0x0003f0301bc79516,0x00008598de8bea90,0x000050e748954ca6}}, + {{0x000313f2b32b8c07,0x0008ad46a7a7864a,0x00071aba203009b8,0x00019786bab2c270,0x00009eea2825dfae}, {0x00022bea3c6feb06,0x00067e299123604c,0x000aa7d5ba6e907b,0x000bc50302538712,0x000018eb27f3228f}}, + {{0x0005c9df43643466,0x000f22a16dea2da5,0x000026ad0fac5dd5,0x0003677c40f2faab,0x0000af1895ff0f0d}, {0x0005cc7072a95ee8,0x0002231880e3d2a1,0x0003245ac109f07c,0x000a3343e7d98a81,0x000053adddb5139a}}, + {{0x000da35f08f5eeaa,0x000b0c8141361e77,0x000274120e93a89c,0x000d2380e4856041,0x00001a55e742ddff}, {0x00047074615459da,0x000a3cae4a32bc77,0x0002bc1e180630de,0x0008995de64b3ee8,0x000066bb84bd1667}}, + {{0x000fde701d4ff7a0,0x0000c53f426d150c,0x000b25dbb8593360,0x000302331d9505e7,0x0000ed1a8e859f94}, {0x000ba4e5e0f86f6b,0x0001e557b53b35d7,0x0006087ef1317fc7,0x00069a1f34ef53a5,0x000023a1a44d68a2}}, + {{0x0002686d46f85357,0x000f2b2aa5adc9c6,0x0002bcd92779e24e,0x000ac2af4f9ea4cb,0x00005036fdea645c}, {0x0009945ac233c5b5,0x00042cd0ab16222f,0x00096a9d0c598bc8,0x000b25254749518e,0x00006f6d06bcc429}}, + {{0x000e2d17828d97f6,0x000a6cb1d21e5e54,0x0002876e02980d80,0x000fdd11debb54d5,0x000002f74e3fc7a2}, {0x000a25518c05c3ef,0x0004a101d6408d80,0x000ed1c406f00af2,0x00036f5c1f6158d6,0x0000932ec7aec045}}, + {{0x0000dd90c8eacd14,0x00051c277d39ced7,0x00037b995a86eabb,0x00069c2d692f5dc3,0x0000aaa65fcbd3f3}, {0x000fb8dac3487ba7,0x000fd9030bc0445e,0x0005b00d8a303975,0x0005ebfc43a8d1f7,0x0000cf7e4b1a80b0}}, + {{0x000e7d3babe2bed5,0x00008ed9aa788262,0x0009ae8cf61b9b73,0x000a2cb8859a7ead,0x000009dd0bff2a08}, {0x00089563be5f62a1,0x000c60b33a1ce2ba,0x000eb11c00b2efd6,0x00076537bfd95b86,0x00001760e857be9a}}, + {{0x000df2f2e4d68602,0x0005e7f162bab259,0x0008073b4332202d,0x00097922f1857cec,0x00001ada3db5ada4}, {0x000feabac21545b5,0x00044b8366fdd7cb,0x000823597e0a8ab9,0x0001b19ca5ad3fd4,0x0000556946280dc4}}, + {{0x00068bc679af7df3,0x0004d09bd4c65b7c,0x000c3d5c8bb79f15,0x000a8eb5a55faae6,0x00000e146d53e240}, {0x000bb5275fa8a745,0x000f9bc67f8fc6e7,0x0004d7d55f437e48,0x000162aaaac77964,0x00008f7b8a4e25c7}}, + {{0x000addb9871528de,0x000efb8f6e9085a4,0x0002b3dce56c416d,0x0009f9352c94a7f5,0x0000cb572963cc8d}, {0x00073bb6f154d061,0x00094ceb155e9479,0x000479f043bcfa01,0x00094fef2ebc01a0,0x00007c190148abea}}, + {{0x0007288c1fd2f6a9,0x0007ccb5f2f0c85d,0x000cd8ddf6f0d1dc,0x000d6d1b46cbab39,0x0000a99e4c52cc37}, {0x0006b442e61e355b,0x000805cf7f7f0461,0x00062ca1fbf6d2d2,0x000a6685af0aca3a,0x000059f9cee65508}}, + {{0x0009830a1370ba94,0x00066f630ed4cc2a,0x000c49043c9c70f2,0x000cda45b0a9a339,0x0000981cac0d3fd4}, {0x000fd2b1537c0568,0x000d2df567bfdb10,0x000f06ee7b678713,0x000035476bfd49ac,0x00007e163ef8dffc}}, + {{0x000a43558915672f,0x000c1ac1fda86ae8,0x00060c9a1cd95a10,0x000104f8c2f7429f,0x00008f5ad3637888}, {0x000c8fcf90898e9a,0x000df624e89346a2,0x000c1fea87fac86a,0x0004b83fd83bf0ac,0x000084646a23ba36}}, + {{0x000f4fb4cc23cc0e,0x000d0fdfd051e92b,0x000f292128680d8b,0x0001f9e822e278e0,0x00007d4d9e59445c}, {0x00015c16f300247b,0x00007aa90db5c335,0x000342a9caef4b57,0x0003e1dcae37a135,0x0000333cb5dfa28f}}, + {{0x000acedb3b302867,0x0007a112c1fd206e,0x000c8ec52d01b8c9,0x000b94e452325cab,0x00009f884b8bdf35}, {0x00097e89f70cf04a,0x000d749d1ae5ecdb,0x00044351436bbabf,0x0008abe4530f2f42,0x0000bb3ffc216d9c}}, + {{0x00001be0aef6af88,0x0003999c0a70d435,0x0003287b2e21580a,0x000a5b6ddd29cc9d,0x0000458091850c10}, {0x0003a83b67e26fc0,0x000ea4f022126008,0x000f9f5f013525d0,0x000f6daa2f518f8a,0x000004d78d00bb76}}, + {{0x00007dce35d51eb2,0x000bd3f986bf114d,0x0006bae418c11481,0x000fad9aa9bf9aed,0x000077c4aa6b8996}, {0x000c111fbadb653d,0x000475bd860ae81c,0x00037b28ab206311,0x000dced3249e29fe,0x0000f13e34b76224}}, + {{0x0000d8d46c6cabee,0x000f1556df1e311a,0x000bc9a29620950f,0x00069f3a256b625a,0x0000845ca024ae37}, {0x000bde60adf24a88,0x000bb4cd5c340731,0x000458b3da58fa82,0x000fbd43156083f6,0x0000e80538fd3b33}}, + {{0x000d5c8a4f806766,0x000e48d81dc67831,0x000430a5bacb70bf,0x000d86bda3b12c13,0x0000e602fe106d61}, {0x00026aa4d9ab33e0,0x000174ef7d6aae6e,0x00040016264165cb,0x000be577ac555a74,0x0000910d31ed147a}}, + {{0x00081f76af0426f7,0x0008410d604b45b2,0x0009a1d45d6fd752,0x000806aa4fbd4908,0x0000c1de40f80886}, {0x000bf8d15847181c,0x000703c99e3ba93d,0x000125e520380a85,0x0007d15781da897b,0x00000c67ed4697fc}}, + {{0x00028041ae87db81,0x0009887b3bc50e3e,0x0003fd20d364a165,0x000aadeb000f0f28,0x000038d839993e13}, {0x00085d6a8b8661a7,0x000b34e820a8623b,0x000eab235775fc13,0x0003ba5f2815b147,0x0000d88868130267}}, + {{0x000d1e00b2843c2b,0x0004df08cfc65f61,0x0008c6f049ce3d0e,0x000d175a3aa4f905,0x00009ff1be643e16}, {0x000c16adf8587d83,0x000242fe78408aef,0x00047747318730e3,0x000fe910dd08eac8,0x0000bef97b69315d}}, + {{0x00099942d00d801c,0x00081d20b58aab1d,0x00083cc49dd642b9,0x000a89b8d67ee603,0x0000d28c6d5ee521}, {0x000f3f9e69075ff4,0x0001d35323720055,0x0007768d68b51dbf,0x0001e7c55669a92c,0x0000298cf352d79d}}, + {{0x000f2548f7c0a953,0x00011d0960107f3e,0x000e0af7bb667c89,0x00064816f354cac5,0x0000d50791b116b8}, {0x0006848e4fd86c6e,0x000c9b7eb17c143a,0x000b947b4f8a167c,0x00060e30ba6169c4,0x0000133e463aa3a4}}, + {{0x00029106d1d6a7ce,0x000ba4385d93bdd2,0x000ee80a9f04f720,0x000132b787a4ec14,0x00004ea21e5863a8}, {0x0000364500d59b88,0x00060f60934a7413,0x000ef1d84166c20c,0x00031d1fab6365d3,0x000088cd5fbaa114}}, + {{0x000d32fedbd19814,0x000221d0419272e7,0x000c4ec059041e52,0x0004de9d2dbd541b,0x0000443d10dc8775}, {0x00078b0d3f94e824,0x000ad40784f9f399,0x000fcf6218b57c8e,0x000b5e55522304d1,0x000047969a949e6f}}, + {{0x00011391108d6518,0x000f28d0956f98d4,0x00002d8f8d2ccf71,0x000b41502b1a4d7c,0x0000df02a4b0a6b6}, {0x000be826697a7400,0x000bbaede068213b,0x000c698316391575,0x000947ceb3b39955,0x000092ee9cb03fb9}}, + {{0x000acdeec1fc6432,0x0002bf85fe71a639,0x000483fcd152b2d9,0x0006a076d90c3e2e,0x0000f765abd75a8f}, {0x000eaa6605b6e935,0x0005d4661347735e,0x000b692ca9771efb,0x000033f6b4f4f2c0,0x00009e887ec099c1}}, + {{0x0000f5f11e1df448,0x00062268eb0e4b3b,0x0007263a3032513a,0x000cc00b41244642,0x0000522b093085dd}, {0x000a4316959d64c4,0x000aece72aeebcb6,0x000a57e25bca4137,0x000a307ad4f570ca,0x0000ea827aee1273}}, + {{0x0001c403871a258f,0x0003c3e40bd72b4f,0x0002aef142a3cf29,0x000c0c94b6f5d42b,0x0000c868d6bb0915}, {0x000fdf303a147183,0x00075f9ac83d925c,0x00078cfb4a2974c2,0x000cc66f8a5e4142,0x0000ffea4657d7cf}}, + {{0x0003d2857b0fb80c,0x00071f7a356b4ff2,0x00028236aa740396,0x00073184ebce593e,0x0000eeb59035372a}, {0x000a4ef592e84599,0x00099e31e3885ac7,0x000dca9f931251e7,0x000405a7e073ded2,0x0000ffaa7091f0ca}}, + {{0x000993353be2eb86,0x000523cb41aea060,0x000f52305f75d323,0x0002da1c19ae132a,0x000005aaaff1356a}, {0x0000b58337d86433,0x000ea1fc9f91fef1,0x0002a47dd5fb33df,0x0009e296da6bdb2b,0x0000d30051c46b7e}}, + {{0x0001396d37931aa4,0x00042fe59d13f6e2,0x0003436e3bc1ea61,0x00095019fd93e242,0x00008077ea00eefa}, {0x000d18f2e5c1305e,0x000ee6a4b029fad4,0x0007af0a58115f19,0x0006774913164569,0x0000f85f4ccc0bf6}}, + {{0x000abfabc6473ac9,0x00076f8f0f80e13e,0x000ca19fb9dad550,0x0000add3a04f0b57,0x0000371e049416e8}, {0x00061632d80ea152,0x000278e8a04dac3d,0x000220d15df178e2,0x00019959c39ef8ff,0x000054bc5557480c}}, + {{0x000faf5f45158e7a,0x000d1f1c1ad0350b,0x000f15bb4fef27d8,0x000e3fc4f652ed78,0x0000166e75983350}, {0x0002cfe2e4cbe8e4,0x0008a56c51d904f9,0x000ec83d02ce2901,0x000b1cbafabfbd51,0x00001176ec3b0b9e}}, + {{0x00085b4bca3d230c,0x000a63179d4ba4da,0x000a166590810b39,0x000b36088f040c03,0x0000773154019362}, {0x0004e8f53d96a2e5,0x000b3ce80126aff0,0x000eca232cc5998f,0x00000d61e47aa73e,0x0000dd3cdf403670}}, + {{0x00055f69d0a7a33b,0x000d34ce0392f3dc,0x000bee0d7944608b,0x0005132e2a2b5516,0x0000cb2be45b0370}, {0x000fb9ef66b067f6,0x000160aba278abd1,0x0007f0556104063a,0x000b4519469ecf88,0x0000d00218f6f0a9}}, + {{0x000775dcbbf023f3,0x00045ef40d8ecd2b,0x000eee1c75c73796,0x00033027bf36185b,0x000082791736f63d}, {0x000d829fc7d15216,0x0003028b4561845d,0x000fbbee37cd94d7,0x00092fad0aff8b0a,0x0000180e25ae0f03}}, + {{0x000f7e62927d7a9a,0x000c839904e36315,0x0005df42f47d356d,0x0009290393e21cd2,0x0000466787630e0d}, {0x00025ee6ced93469,0x000f010090774044,0x0007a7592fd8fdb2,0x000a03dd870e7a31,0x0000e332fd788110}}, + {{0x0009edb975ec2c09,0x0005175735d65d1b,0x00064bc88e72b8be,0x000ef80b4c4c003f,0x00003be7738bb2d9}, {0x0006e0fd72ac494f,0x000ecf9a7a5fdcc7,0x000b1e5a2028d854,0x00097874b4af8b2c,0x0000b21fce5d8966}}, + {{0x00092d9ee6a8ebf2,0x000f26040d80ea6a,0x000852efe4c46437,0x0005037006c5176e,0x0000866683ba0c93}, {0x000bfd0134476305,0x00040e03c13b63bd,0x0008885347ace166,0x0003339702e75cc7,0x0000f159cdfbc73a}}, + {{0x0002a9170ad6789d,0x000d2234dc9ad48f,0x000f2a7d5e25f59e,0x0007a488b843c73e,0x000027897d1cc561}, {0x0000fd3365d34202,0x00012bb48f85ef77,0x00069764c4ade689,0x000d9404b74787a3,0x000055e4a1bf0653}}, +}, +{ +/* digit=23 [{1,2,3,..,}]*([2^161]*G) */ + {{0x000e715ce388849c,0x0009bb7a2d2af7f9,0x000661db13a1607c,0x000108dedcca8040,0x0000b1a7d04a99e3}, {0x0003a8e4d163dfdc,0x0001783f0e3121ce,0x0001fa2958fee10c,0x000e438b956bab30,0x0000a007e991f63f}}, + {{0x000c2bbff739fa94,0x000c017ea854bdf5,0x0007a64f91d3bfa6,0x0000fdc41fb9d90e,0x000083aa0d7f2c19}, {0x00010d24d367c028,0x0004597010d57ebc,0x00032c5579b43146,0x00072bbd8b596396,0x0000ddf287538ee7}}, + {{0x000d002aead503ac,0x000702bbfd247478,0x0000b804d9f4d7e3,0x0002ca633f318949,0x00003505e7867a6d}, {0x000e5d7dbff73d80,0x000335f9e633fe97,0x000ec9aa42cc3a3e,0x00034df252891ec7,0x00004900747ed888}}, + {{0x000d54830481b038,0x000e2747a4cb9b56,0x000c55970078ce04,0x00066e74e17ce4fa,0x0000ea3c99c7fdac}, {0x00047874a0836e33,0x0008462862478e13,0x000af3763a660e33,0x000f8efb8318cfbd,0x0000993ecdd67d59}}, + {{0x000143d1036960c6,0x0001ffe5607a28b5,0x000be0da767df772,0x000db340e2fc542c,0x00007bc3d3e1614e}, {0x0000a909a4228534,0x000df6d0a4402b66,0x00036e7c5f663676,0x000900fa2bc94c05,0x0000d5caab23f5a3}}, + {{0x000c9afe79e70193,0x000d0fec85a37b5e,0x000a360abb585419,0x000883bc10053d78,0x000097415c9102d7}, {0x0000686637dc9a27,0x0004c4b95f42fa66,0x000cd93361d4b524,0x00042e32721d0b6a,0x00007db0d3bdbc70}}, + {{0x000bffcda8b06d6c,0x000713d8d2427048,0x000cd16bb5d30666,0x000c2197309cf437,0x0000a11a6f46e8f0}, {0x0008bc17a71c4e20,0x000a92d41371975b,0x0008eb0f5349e195,0x0002ce43cc3800ca,0x000041f3bf72ccf0}}, + {{0x0001daeea3ed0905,0x000e1bd5e3c53fbf,0x000c1100344d6a8d,0x000e5e4699602597,0x000060ea92b03fab}, {0x000e180e331a3fe2,0x00080a41fd2b20f8,0x000cdb94814472c1,0x0006c8ec624980f6,0x00000f649cdb7533}}, + {{0x0004ac13e9d9a85e,0x000ac7ff9c45703b,0x000329fbf150d4b2,0x0001fe9bfe98b07d,0x00006ef7b3a05b5a}, {0x000f985fb40d4e51,0x00028cdb30c4b3f3,0x00069b998490bd1f,0x000ee22e2266c121,0x00005d8c7fd5c464}}, + {{0x00017c9deb217fde,0x0000d74da8211aad,0x0001d8f1f679025c,0x00046eae0861688b,0x00003593982918fa}, {0x0009d1bfbf4ff519,0x00072a742f2a69e4,0x000bc2636c0fae9c,0x00046b33d049c63c,0x0000197ec842285d}}, + {{0x000624c23eaa251f,0x00071318ca3c4e9a,0x00095832234ba387,0x000151f4edec7eb9,0x000040bfde516f7b}, {0x0008fa54043404cb,0x000a18aa5c602ed4,0x000ae3268e047506,0x000d7eab79eb8def,0x0000d3b0628dd6e8}}, + {{0x000c4cbd66e8fe68,0x000dc69d604cf338,0x00037b6dd97e28fd,0x00035b246f37d007,0x00000272666649ae}, {0x00093d37810cf944,0x000a459f78ac9a38,0x000ce165aabf2ebc,0x000d640453946fae,0x00008fa3bf971e87}}, + {{0x000b48a8dba5c4e8,0x0002279adf9d8dc3,0x0000634c4f3162b0,0x000b99b53c9ff728,0x000007d5d4dfaa42}, {0x000ff5483cd41f20,0x000f99d21f8d8aa4,0x00081bda1da27bb5,0x00014e9a023684cb,0x0000f7ca6a09c875}}, + {{0x000567e48aaf12a4,0x00072c6ad9edf914,0x000cbc02cdd34885,0x000546ad9d3126ff,0x0000421b423fa8d3}, {0x00006150c5383199,0x00071eb71f6ae471,0x000ba9109ff4978e,0x0004c14e33185f0d,0x00001b2adaf1a25b}}, + {{0x00022a2429dd8dec,0x0007cd18b13b0a91,0x000b0ceea09a2c07,0x0003e4cfd65c5719,0x0000c9bfb6cc8897}, {0x000ab8bb6da6f448,0x000361113f5c0f75,0x0004bcd4ce388639,0x0009b3c84e7da532,0x0000c1fd90d251ff}}, + {{0x000043aa2a095bb7,0x000f9b9f786a3956,0x0002db051d57f81a,0x000149260ef94740,0x00007bf6371cac51}, {0x0001f2f5c58d7ca4,0x00040f7f7b148a02,0x0007371672527386,0x0004b3f38f333829,0x0000dd7a77b276e8}}, + {{0x0005457a7eca1884,0x000cbe9cc41123f9,0x000136109bbb2c81,0x000d931b767fb526,0x00007e27c293ef66}, {0x0003f2073a6659c9,0x000dc1c5bb8bcb6e,0x0009084018ee6a36,0x000ecce3dcbff9b0,0x00008f3df8af2dc0}}, + {{0x000d83214c8380e8,0x000a3ec7ffe0ad3c,0x0000a2a618aea404,0x0000c47d0790d1b1,0x00000273e67906f3}, {0x00015287e729a739,0x000c7793bec7e17d,0x000acf922b6820b6,0x000254244b10a6d9,0x000075cf6a5b4cab}}, + {{0x000e81a727320b70,0x0002224f04c96aed,0x00023666fc18bf83,0x000b00b1675a97e2,0x00003640ad09f382}, {0x000efc15509f9819,0x000644de6fa11fe7,0x0008a4478b13a59f,0x000e660ba6778ad8,0x0000f50b5ac51bf3}}, + {{0x000335a6248572ee,0x000fe809fb5fe516,0x0009687ddec290cd,0x0005b5dde9196a7c,0x00005c7d5573c17b}, {0x000ca4c51c772072,0x000a292ca0ca7c47,0x00061427ea87854a,0x0003e4b62b22808a,0x0000c0e5c44caaf5}}, + {{0x000cad5d48a4d850,0x00038d63ffac15ef,0x000132f45987e66b,0x00026ccf724889b5,0x000098deb93b1efa}, {0x000c62a168e65f44,0x000179a62036dfda,0x0008c6dbc9a4ca00,0x0000127fb88e937e,0x000082fd3c4bc09a}}, + {{0x000f3ad4813e9d65,0x000bea963f775c93,0x000f6998a46925b6,0x00042bbe8efab45a,0x000029a003ea7de6}, {0x00043182741135fa,0x000c3b74e1a693fe,0x000868769efe7061,0x0001612e436da60a,0x0000de122bdce426}}, + {{0x000e61a87ddb5720,0x000452750c03edbd,0x000b38089eb41621,0x00024498574681e0,0x00004444c5108b2d}, {0x00036ed4405c72c4,0x000be6b96571a817,0x000f46d03728b4d6,0x000d22955878f06c,0x00002f2165d6e0b3}}, + {{0x00094e84458cd24f,0x0003bef3dc9a70f5,0x000015cc27ea5fec,0x0005fb57e2473ea9,0x00002182214585b4}, {0x000ebd801e198aee,0x00041dd30f306bbd,0x0008a039acb01cbb,0x0008fcba8a783d98,0x00006a26d47437cf}}, + {{0x00036d49031103bd,0x000fe7923cfb91e1,0x0009eb93161c2874,0x000b42decc591c88,0x0000df51011dba39}, {0x000990ab10e9def1,0x000b6e7f06951c4f,0x00063898370d2788,0x0006086393208c48,0x0000fe1f66e971ef}}, + {{0x000b1d4d77c20846,0x0003b33e6323e50a,0x000985e8c9dc50c0,0x000486a05634b919,0x0000032a15357882}, {0x00036b5f08065726,0x0001971634554a52,0x000f5fabff87414c,0x0006926e37e37906,0x000043513f70d8fb}}, + {{0x000739d209dee688,0x00006a1bf22b1a5c,0x0000f4d7e2ebc0fb,0x0006423d3abeed81,0x000036c96818fb4e}, {0x000d76faf0aa49ac,0x000c17dc5923cdc1,0x000780973c293b0a,0x0007d05101c6532d,0x000094adf9339dbb}}, + {{0x00046923a1ecf415,0x000a0f9ea7f71cc9,0x0005779649d48858,0x0002e729830f9f5a,0x0000a791f0968938}, {0x000b5938d2d48da3,0x000e4a63cc188eb4,0x000352d09bbdb586,0x00085e41ea2ca55d,0x0000adb9408e7ccc}}, + {{0x0000274ff274c238,0x000cf4ed75e4ccfb,0x000a58613ca6532f,0x0002f08974ef32d2,0x00009439f94f5580}, {0x000e9bd5e20f70f3,0x0008a3a320b0c62e,0x0009802b6b371846,0x000eb1e8d6efb074,0x0000fa8cacacc87a}}, + {{0x0007eae23823fcbb,0x0007a6db12378b1f,0x0005b04d466a2702,0x00074dc0e7967d3f,0x0000380c34854b55}, {0x0003c2ba3855dc42,0x00079c231fffe07b,0x000441be0c2f630c,0x000d9ac7ff5f1f15,0x000079fc2752ae5b}}, + {{0x000d5c7dfb3b7db6,0x000f2ca2b44387bf,0x000dd902aca2989f,0x000944ef359e06f4,0x000056de67cc5e30}, {0x000a7f7dc035890b,0x000b9b3ae32060de,0x000a3a327bb2950a,0x000ad059416d8521,0x00005f50358a117c}}, + {{0x0004d697d518207c,0x0008e8767b105e78,0x00004f1169c01bf5,0x00094aab5136a0ca,0x00007aff0662c2f3}, {0x0008afdcb5efc86f,0x000e9d607a6dc3be,0x0006fbaec814892d,0x000876bc381a1cec,0x0000804cfbddb73e}}, + {{0x000450c84a71f470,0x0001720cbb4694c3,0x00056165c8b36095,0x0003d3e5828b276c,0x000045bbc6744164}, {0x00051eb64e2708b8,0x000f53a1d23ebe35,0x000cf38009d06145,0x00098675fca31411,0x00002fe56c9b4d77}}, + {{0x00089e8111b8d201,0x00028297bce0d27e,0x0006a7f74f80e76c,0x000d806d8e7591d1,0x00003296fccf5a57}, {0x000b45c6234aea5e,0x0006dbb033b176cc,0x00001216b62e1303,0x00059592a07ef8c3,0x00008ded78bdb4a6}}, + {{0x000ba2653b58b0d9,0x000642b1c2d703cd,0x0000554cb792e88c,0x000b255ed4d6bdaf,0x00007f42217534e2}, {0x000de0e9cf5e143c,0x000f7f941fa77e41,0x000e9891ce2c2bc0,0x000551a51e978228,0x0000c49845f6fd9d}}, + {{0x000072da6b911236,0x000b00c92cb9f1c0,0x00011381534f49ec,0x000547bce7c215c9,0x0000dae67dc8e0ed}, {0x0001fe5736a8717c,0x0003fdb486b086b4,0x0005017a01d7b53a,0x000770705d0f3b3b,0x0000be31d50dc547}}, + {{0x00031863932c2b1f,0x0009f501ea21d51e,0x0007b61c6b454bba,0x0007c33cb217df73,0x0000437c6550aee4}, {0x00044ddc161b29fc,0x000c0745a7be39af,0x0008b4e070c24b6d,0x0009ad9f92882f93,0x000012fd81676997}}, + {{0x00027b5c52289a6d,0x00032826de7fbaeb,0x00017117cc11f3cc,0x00032957907378e9,0x000000bca115722b}, {0x000ac722f5f70b15,0x000c503d941e10b4,0x00062a5c2c8da21e,0x000534a9a3154dff,0x000008a1f7b230d8}}, + {{0x0003ceff9e7ddb51,0x000d455c86afb70d,0x0007400b6fcc1c34,0x00025964e9b9b522,0x0000c5cc14c5bb83}, {0x000dab9215029d5f,0x00095e492977ca58,0x000c4a20aa8d4a22,0x00079b19ad26e547,0x0000d91ae79427ea}}, + {{0x000785990e90367d,0x000a07e301b6cf61,0x000f636872d8e93b,0x000ab534875194e9,0x00009e9353e6c1d6}, {0x0003bbdf1baa0db0,0x000c20a4cb66f8ea,0x000ab37b0f01de77,0x00087161e429924e,0x00006efdcdf4befb}}, + {{0x000058743128a14e,0x000e066c3cd7d9ca,0x000e7e113a4a62ae,0x0000fbffb51b4a5f,0x0000cea959c32023}, {0x000a40cdef43eec5,0x000504a425d60020,0x000bb9d80185d21f,0x000572b5f2b10cbf,0x0000666c56168409}}, + {{0x0000890f3b3930eb,0x0001779057c83ef0,0x0005d2430da067a2,0x000527201bd76b42,0x00007acf20f1cd5e}, {0x000b427b09507e58,0x0007d0ae9fe8ebe9,0x0001249697373a3b,0x00059d9893406aaa,0x0000e193839ab695}}, + {{0x0003e416acc4f41a,0x000c0e4846f77700,0x0002367dc52fe292,0x0003b7ea2b9fb2b4,0x00004b818d575942}, {0x0001acd23a6740da,0x0002e5fd8bd4aaaa,0x000dbddefe835656,0x000f8fa01421ea02,0x00000ea7a8551c0e}}, + {{0x0004fb87bb00fd80,0x00047858e175eccf,0x00085edcf225c4cf,0x000c38fd395e90d8,0x0000c586da7163c3}, {0x000add010694ccaa,0x000a11f48044f949,0x0005c948f8de7c64,0x0009280114c685fa,0x0000715d4a32473d}}, + {{0x000e9f8ae08c8338,0x000c24ac457f9136,0x0007263471950b1c,0x000630b176b52349,0x0000e8f83579b684}, {0x000f9d951a40fda3,0x00089ef984f41fe3,0x000cddfe9a94991d,0x0002c27fae887b02,0x0000f0c63c32ebc6}}, + {{0x0008af107ba4d8d5,0x0003fbb10a2c242d,0x000f5b66318b0abf,0x000ce7ec492bfd61,0x00000fb5641a0bfd}, {0x0004113b1af6e684,0x0004855508208921,0x00033ccaf19c703d,0x000f8a48d57f4293,0x0000d1b2b012f3aa}}, + {{0x0007bfe9a941e18d,0x00055afc95f9e3ce,0x000005028fe66b94,0x000660d24af86ca7,0x0000eaa16abe3583}, {0x00042b18956071d8,0x00069c5ee733a2e0,0x000c62790d044917,0x000243845bb68654,0x00003a6cb05a0b5e}}, + {{0x00023c0963370038,0x0004f22524f47160,0x0002b7ff5d5eaddf,0x0008db0d442eb80f,0x0000556e77aff7df}, {0x00075e94c9c29a9e,0x000f494e0b218d01,0x000e6d2ba70c6541,0x000cd9f24ffb8b01,0x0000c6ad519d7d87}}, + {{0x0006285b576c16f8,0x0001de9045f059de,0x00073148979af3b5,0x0001d1be76385156,0x0000309c88e12487}, {0x000c2eaef778ff77,0x0009e31d2d017dc4,0x000396fd24eca05e,0x000f6feb4f3a3681,0x0000a6b0a015343c}}, + {{0x000eb84a8ff46d92,0x000e9fb21b2cd2a3,0x0007ea66290a8efa,0x000bf97c2fe9112a,0x0000c3722457ee2b}, {0x0004b94de41e2c1a,0x00070bb0903edd96,0x00012b5c31d350a5,0x000a101dae61431c,0x000045a6d6340610}}, + {{0x0004e955e16f4b09,0x00069c988f51727f,0x0004df57c7386200,0x000bdbe4bbb1fa5e,0x000074b4ca3ca24e}, {0x000dcfef41e02f97,0x000d622895da0cf7,0x000400bcaaaab9ed,0x000e8d05fbb139e2,0x00009606bc9991cf}}, + {{0x00007c5d049d81b1,0x000af8afecbf4e38,0x0009913b9af5ff22,0x0007bf2c6b5e2220,0x0000094dcf820643}, {0x000323f6bb2b4620,0x0005b4b00c0dd155,0x00004087e0e046ba,0x000c8dca3773450b,0x0000a3ee21010695}}, + {{0x000e23bf4f40ccea,0x000d69f16f378bb4,0x00081b75cffb5728,0x0001f674fcd6adee,0x0000975281c93b18}, {0x0001fa9b52577c5b,0x0000ddc2d2be26f4,0x0001deac33380425,0x0005e5008eaa09d5,0x0000fdd6b2fd7884}}, + {{0x0005aeef4710b606,0x0009620131b1a6ac,0x000ed6b51ad3a398,0x0001f7326c04601c,0x00004fed7c15b6f3}, {0x00051ae4e75e2291,0x000b2b77e46a8eb4,0x00024864f5396c6b,0x000a307fe44bd15a,0x0000fddf9a7fb162}}, + {{0x00004463e73b8ec4,0x000f55cec5c6d935,0x000affcd58b06c2e,0x00044e00ad788e57,0x0000872bf18a0fd9}, {0x0008d14e5d71e574,0x000d2c3030bc88b1,0x0005dddd1ce58ea7,0x00057d1f9dc7c5a3,0x00004d1818d515b6}}, + {{0x000ddb8bf4b54748,0x00023305f0daf6ed,0x00052038139d4af5,0x000efb7d652772de,0x0000069ae58d16c6}, {0x000bf73dfb68fd92,0x0003fde5dff8e66d,0x0005f967c0e4a7d2,0x00069cea388c3b52,0x000000d338cf626d}}, + {{0x00045c714519e589,0x000e0913096476e5,0x0000d02d7cf59be2,0x0008386476786752,0x0000a99633c4e597}, {0x0000911b961f3964,0x000589bdb3f722c1,0x00040b0284c204f6,0x000310c471f04659,0x0000b8f29400b771}}, + {{0x0007580908012844,0x0006c059281b1ac5,0x0001c35df7e30017,0x000950b68a678586,0x0000423082ebc843}, {0x0003aee90a09598e,0x000a9913987d1e31,0x0002a51b3792cb2b,0x000d1a18355dcf9d,0x00002891458b1327}}, + {{0x00059e6c45add90c,0x00035578dc6b43e3,0x0002ce37e6ba1e8c,0x0002e4a6505c2877,0x0000b821b953f4ae}, {0x00022256fceb3969,0x000fbf38ecb4e108,0x000eef42aaee7a3b,0x00075dab0d35094d,0x000025f891427625}}, + {{0x000bb97d0edfa1ce,0x000e46b5d78101cf,0x00056ac5655ba12e,0x0004dc4b371feede,0x0000daa9c221667c}, {0x000b374aafd9ed41,0x000ac3a4679a8a4c,0x00000de55fba1126,0x000b1bafaeade880,0x00003e746cdf818e}}, + {{0x000990ec0f88d5f8,0x000288b4c7c62b24,0x00084d7118dfe4c9,0x0009fca983eed9ba,0x0000a1b21a93ca27}, {0x0008872895a60682,0x000024e18c538587,0x000ca73ba0accfc1,0x00000837dd0e2487,0x00008513be562083}}, + {{0x00003f947b6ea260,0x0001bb7a4729c734,0x00032750426849a0,0x0004026b72cf4cd4,0x0000c0fffc1c5d6f}, {0x00085c96173719aa,0x000c05910bfce43c,0x000acef18e018793,0x00032e0f069d5faa,0x00009fae2afd5c69}}, + {{0x000acf01ed8bd7cd,0x0000263477c55128,0x00078d6112535697,0x000b9055421fe02d,0x0000f1a20e6ea246}, {0x0005ae7d073d4a89,0x000bd63778b84373,0x000bc7a37abb5f3e,0x00009b5483301a33,0x000036ae26c63a8e}}, + {{0x000061c767eff576,0x000dd9ea79a1ccc2,0x0003437b2eba650c,0x000cfcac1b11574d,0x0000cfb31a0d11b8}, {0x00054f89f42bfdbc,0x000cc2703661fe7c,0x00070d58a8d1569e,0x000bba5a9527776b,0x0000161cf8545e19}}, +}, +{ +/* digit=24 [{1,2,3,..,}]*([2^168]*G) */ + {{0x000c25e9f5170aa7,0x0001f8f5338bd76c,0x00040201a3c0073f,0x00012ffbf6fcc231,0x000009be87d49f50}, {0x000bfe953dc74903,0x0003d8653153dc6e,0x0000eb06e85facca,0x0003077c47d716a6,0x00009f6514c80bee}}, + {{0x0007b09654c48577,0x0004fdb0f5200d71,0x0004e6cb015e94bb,0x000f88dcf3519dc6,0x000059d275f2ebdb}, {0x000ec0fd11aeb819,0x000c8be9fb8ae833,0x0007d57ad2893bce,0x000e52f8ada66e74,0x00002d1bbca1a1e3}}, + {{0x000d4246631e6f4c,0x00052539fdfdb824,0x000b8f0507304674,0x00065e29c10ef5e6,0x00006b779bb7066a}, {0x0005a5380f5abd81,0x000523a56e993fe2,0x000c373f30122a40,0x000a4ff3a52697bc,0x0000de214925f8e6}}, + {{0x000a957eb8abd51b,0x0001bb0ab69abf50,0x000e7252ad43d460,0x000fe24207ecc4bb,0x0000964c4fe8348f}, {0x00091eb0668c06ef,0x00037f512c2d1c55,0x000cc3a429333eb0,0x00061923cd82147c,0x00008b93a9d6c59e}}, + {{0x000cc48773e35954,0x000e214b22340fa4,0x0007830d996f413a,0x000907b196a2bc70,0x000037ecd06850c4}, {0x00039bfe3ef35b95,0x000b82f5d6ce3ace,0x000d52cc302fad75,0x000d0d41e6e89210,0x000090aa6d71bfce}}, + {{0x000dc3c8f4aa5889,0x0003a50cbedadb5e,0x0007a52111e6d592,0x000a80177e437ee8,0x0000df8d7c441860}, {0x000eda8cf92436a5,0x000296bc8560bcf4,0x0009734d2066ad91,0x000059756ccdd882,0x000043cd11c08558}}, + {{0x000077ab273751fb,0x000c255f1d1bdcb6,0x000d46eee885b68f,0x0006411656e793e5,0x0000bcc61e8b1ba0}, {0x0004299f88dfd877,0x00072838b3a069d4,0x00011a67cda79c15,0x00041b6a28140e4e,0x000038faae273c96}}, + {{0x000cc732728468c7,0x000bdfa9a3bd9569,0x000d743f442bfed7,0x0002dd74b6f74702,0x00001de0fb3c4843}, {0x000e7d2fa94e43b9,0x000e83587eed4ca6,0x00099758a506eec9,0x000192a4a5292aa3,0x000006038c3c85a0}}, + {{0x000e8448ca36b091,0x000724026db2dc96,0x000a6f0c367dc2bd,0x000865e5b7051140,0x00009f8943f82b64}, {0x0005ff5fc9a5ec8e,0x0007001fcd7a9b31,0x000237cd7ebe7ebb,0x00005c2a5148967d,0x0000e997301ffa6b}}, + {{0x0004c665b9eac65a,0x00052ba5a915a9f2,0x00066278cc6bdeae,0x000b249b5e33760c,0x0000ebc09e8ac7b1}, {0x000caea5a6199620,0x000be60abe7d7c0b,0x00088044342ec817,0x0000a54835970666,0x000050d8ef2ff279}}, + {{0x0006c27846f75d05,0x0002cbb8971b0591,0x000fef98918d6613,0x000d3bb003639aad,0x00002a37323294ad}, {0x0004bdefdbf09029,0x0000fea38bc50670,0x00075dd7401061a2,0x000972a86fb9b7f3,0x00002a9ef8a399aa}}, + {{0x0004bb6ac1207afa,0x000e9580f1b1a4df,0x0009301d506bb9e3,0x0003ddb25c740d34,0x000024a77e6b0e29}, {0x0009997ebbefc02a,0x0000f9bee7b2e05d,0x000c3f6c1d1e9c60,0x000930fe3607022d,0x0000b29cca36dccc}}, + {{0x000a13edfc3232c8,0x000071af81e3d653,0x00093d6ba80325a6,0x000faf2033e68207,0x00001c7c20a162eb}, {0x0003995532aaaf64,0x000c4aecfc68ed5a,0x00098196ea438ecb,0x000c5c5fff04de2b,0x0000069d497969cf}}, + {{0x00016cb6ab4f0071,0x0002ebe7c2a951b2,0x0001507db4c9e3b8,0x0000e1bf7e75c535,0x00006b09f4ba63d2}, {0x0005b11f97734d28,0x0006dc581c376458,0x000cec0864a46ba7,0x0001fc733dcc712f,0x0000aa53b9688ed5}}, + {{0x000da0620844de29,0x0008013f258339b4,0x00080097b1ef8c8a,0x000716541e0eaa04,0x0000c5dfd5748615}, {0x00091ec9a14f247b,0x000b9fd915415a11,0x00018caef9331112,0x00097f0fe5bf0968,0x0000636405ec0ca8}}, + {{0x000dd20ccee2cb48,0x000695b1e68483ba,0x00039267fb005c14,0x000991f285b3686f,0x0000596c8811740c}, {0x0000039d01b2c886,0x0000066b542dfb66,0x00090ad7b941b21f,0x000fb42690c453ab,0x00002b8741e90fa7}}, + {{0x000af8194dadc627,0x0001edf51bfe1834,0x0009bd348776705b,0x0001ad70ee1dce5c,0x0000b26f64d4ae19}, {0x000a67e9b9267406,0x000dbf7a60d3b5cb,0x000bbaec7f815e13,0x000b9534b3ab92f3,0x0000d60d991417fd}}, + {{0x000ce9cfb15625e8,0x000fc4db106913f8,0x000b2c9cd8cedab6,0x000a59e395243e6e,0x0000434f60bd1431}, {0x00012f01f634f2cd,0x000c40ae9ddbe0ed,0x000b866c6a73238f,0x00081e4ad4d13cf2,0x0000375a9d4fdb5d}}, + {{0x000cffc53160c73c,0x00007539809f73a4,0x0005b23b923b3241,0x000b5195c30c96c5,0x000094fdb86cfd44}, {0x0009dbb264a47a78,0x0004d93efb5e99fe,0x000e9a68933cc2c1,0x000ffe8e878bb037,0x0000f7d2e44d123a}}, + {{0x0007e32ebbba9682,0x000be1ca85ab0b5d,0x000c9697a03f4a69,0x0000a0e9870318d2,0x0000e8015aabeaf5}, {0x0008c0ba7e1dfd93,0x000f56bc29921b79,0x0001d1608f5ae798,0x0000ee85a0922135,0x00007f26d995d2fb}}, + {{0x00089e5230cc1b43,0x000be3c41cfda991,0x000292450b3d3c1d,0x000dd89d5989e496,0x0000cfc1ecde9524}, {0x000d6a36c7493761,0x00031f1678e9dfa7,0x000a563232985073,0x000479ad7e48450c,0x0000ac60c6fc5646}}, + {{0x000e435ab0cf52fb,0x000756ea25b6a16d,0x000034b17c2186b4,0x0001f2fc7036334a,0x0000bd1c9eafb0ed}, {0x0004b6d249b21b42,0x000c8600552f96ed,0x000bfc49b2fc4e76,0x000ab86179a640ee,0x0000742b6f2cd85f}}, + {{0x000836e93b510939,0x00034ecdc5de9094,0x000cfec954c59e8e,0x000659f87fcc53cf,0x0000a4568a7a9057}, {0x000c03e5826f6ff5,0x0002f83faeae31ed,0x000b8b111d7a6557,0x0001ce6cb4bee145,0x0000e343b357add3}}, + {{0x0007124356c03ca5,0x0005e9256c013890,0x0003b390b3f0ccfa,0x00096e541ed7e668,0x00005b7692759858}, {0x000db7904f1906ec,0x0009f798fee14430,0x000c762aa8e0adfd,0x0009c408beb39255,0x000089f982ccad3d}}, + {{0x00079b689457b765,0x000fea0f5b956815,0x0003aae2e20e7df7,0x00036deeee3b323c,0x0000b98cee329d35}, {0x000cece61a5f45d6,0x00042e78c9cbc2e3,0x000ecf95735d7b06,0x00009e205e0aa2ca,0x0000e5839c503b44}}, + {{0x0008f9208fcdecb3,0x00061b66dbf41f69,0x000a1af91975c299,0x000495d23b37428d,0x00003d9b8eb27748}, {0x000c26ec038dd208,0x0007733cdf67d14c,0x000f975e452454ca,0x000c01fabd49971b,0x0000982b9f5186b5}}, + {{0x00082d71ed519447,0x000dd9063242dd6c,0x00086537a94dbe4e,0x0007e5159224fc27,0x000080fe5639583f}, {0x000c13fce88adb1a,0x000bc57f2f24f1ae,0x000fa63cf0fdf9e6,0x000b6d90fd8fafda,0x000044733d44e643}}, + {{0x0001fe8821de4787,0x000ad971ee6fb14a,0x000eeebab19b43ff,0x000fd6931813e718,0x0000f9d90866ab81}, {0x000ab8c25b81d6ec,0x00085bd040950086,0x000b02351a4be98f,0x000418cdeb6e4418,0x0000e57c7e8aa473}}, + {{0x00026d051d0b0c81,0x000c03f43bb4e055,0x000108d9671ce09f,0x0001db9508e74a7b,0x00001f1327927748}, {0x000b0010c84e7c59,0x000517a0dad7005a,0x000eefc16dacdfff,0x0009472e6bfbd9e3,0x0000daaed22249a2}}, + {{0x00068df75e5be29d,0x000385fff9a99479,0x000bb571140db0d9,0x00046304b0b4cfd7,0x00001addb26ae24b}, {0x000581856fb91014,0x0001696d3fdf03ca,0x0009d9e027eb9076,0x0002876031e89681,0x00003e9950d59c6e}}, + {{0x0008caae157977b0,0x000674349dcf9d6a,0x000b699a60e8b54c,0x00076b8f3c4ad763,0x00005cc559c43b1b}, {0x000211ff4c8d557b,0x000e78905358fe5c,0x00083936f8309b2c,0x0004af64acf5051f,0x0000481e7e0c6fec}}, + {{0x00053fe33c602daf,0x0004cf85cb7ed0f4,0x000d1adf62c3095c,0x0002834573d068b4,0x00002d0fd0243c96}, {0x000b5a83acc5f984,0x000951b8242871ac,0x00042785559a72b1,0x000383e0b9e6f513,0x0000eef0713ace69}}, + {{0x0001642514077632,0x000b7843397d07f7,0x0004342ab2e8b92a,0x000548eb3dea716a,0x0000fd46ebf66d9f}, {0x000108bf0b8a44a2,0x000444151a9a68e5,0x0006fbd93700306f,0x000a840cbdf94002,0x0000c577ee629894}}, + {{0x0000bca8d5a234cb,0x0002ada976341eed,0x0000d8f5b9ffe4f4,0x000614ea32e31bba,0x00009640fc08e2ba}, {0x000bd3091a3cc172,0x000dd6813f60e5d6,0x0000955ef6102e8d,0x000c318e716035a8,0x00007083188700d7}}, + {{0x00015232afdb4a80,0x0002497d796b320f,0x000fcb99991c4603,0x0008a8e8ee7f7f71,0x000086a97ca03731}, {0x0008e6620f196d18,0x000bfb677ecebb7a,0x00084316a2c3361d,0x00098a5a92e2000f,0x0000ad5b02777783}}, + {{0x000dd0efcada3ed5,0x000bc43a9054d4be,0x000db0e54123501a,0x000e0f955966efef,0x0000e31aa4eeecf5}, {0x00049cefca5df228,0x0001b845465e6d45,0x000595d6a9287939,0x0005be1879d8875b,0x000050e3ad18c18e}}, + {{0x0000ae732271801c,0x0005b803d1ed6ccd,0x000be65e61856a37,0x0006cdb05d43079f,0x0000757438954a07}, {0x000ff113986411c7,0x00005e8c6b438f05,0x00082dcb49c55eac,0x00095ff8773437d6,0x0000797c7879b91a}}, + {{0x000418f19e79e5c2,0x000898e184288dfc,0x000811bdb644c310,0x00017641a8142536,0x0000ec41eefe56df}, {0x00005b4e27fb6bc1,0x000f8e054fba1edc,0x00089e0e1650e5e9,0x00095692ab6f4529,0x0000fbcc505276d3}}, + {{0x000281ca2dfe6dc8,0x000110de712c653e,0x000ee928158404b6,0x000cf6d7e6105cd0,0x0000d388140e6953}, {0x000ed39f0617f89f,0x00067a61a5286d8c,0x0000ab74f3f1e213,0x00006e8ba2227632,0x00002cba3b64a144}}, + {{0x00019bb3ef88b573,0x0009c9454c264310,0x000e75b084b05970,0x00090f0042faa6a0,0x0000c42b966c283b}, {0x00069924227e2345,0x00081e764b1d0c29,0x00085621fbc826d9,0x0005ac4deb50a990,0x00003a0c83012d01}}, + {{0x000eedb8289fe19d,0x000455d8703a0f39,0x00069318dd303312,0x000f22bc207eb1d1,0x00009a1b36be080e}, {0x000a07cf89baf3ea,0x000d625b41a9e3cb,0x000bfa4f0e365656,0x000ce1cc88025770,0x000065eacea3656e}}, + {{0x0005c19ef45d6b91,0x0003ff6ee7a1c188,0x00095b614626c796,0x00008cdf716625be,0x0000905d59e52e68}, {0x000e8445d0e57d94,0x00010289d5e86e38,0x000258e2d5292144,0x000960edda6c726d,0x000078f149d15866}}, + {{0x0003b373eeb3be5d,0x000008cdfd007c91,0x00020d539c7e74db,0x000a226c08d0199f,0x0000f14535325914}, {0x000f5b02288d3d99,0x000c450de2506ea7,0x000a273bce6ad41d,0x000bdbc8e36df7c0,0x00001ad366534f40}}, + {{0x000602ba827d0dcb,0x00091adbf560c3f3,0x00013a340f90f764,0x00093eb8c55e59f6,0x000080ec3bdb9d66}, {0x000fa139bc659726,0x000f737f930cd576,0x00052013011430c2,0x0003652c9fa7479f,0x00006624def0a0e1}}, + {{0x0009772769b19129,0x0009d1ada7cf07e3,0x000c9a259ebf722e,0x000cc4241ca31714,0x0000b844dffaaf8c}, {0x000a55fb9723c092,0x000761a9a645934e,0x0000f493a79cd872,0x000410ce7fa8a9c6,0x0000992cbc50218e}}, + {{0x0009ec972907a41a,0x0003f2fe9e468bbf,0x000807c9a4ff51da,0x0000ea9411ccb7b4,0x0000798a299aa8f4}, {0x000bb39dfa6de2ba,0x000d102a830388a3,0x0009be6602815a52,0x00097420a96dc3e2,0x0000698523b7ae24}}, + {{0x000e5e2695331daf,0x000aa511993e978f,0x000890d5eea027e9,0x000380801eac0409,0x0000079a353c8f13}, {0x000d2aab807ac122,0x0003153b7b007d3e,0x000c8146f9dc100f,0x000de36f0b42a3f9,0x0000837fb01d7d39}}, + {{0x000bad9ce4ff7c82,0x0001f47732d9a89b,0x000ee5faee22e6f7,0x000d5ea181969613,0x0000e30cc88547dd}, {0x000c36cde1dae9e9,0x0006bc67c475249f,0x000dfb408d12c773,0x0000395cb3b91c3c,0x0000d1c1cf946d61}}, + {{0x00040f67eafffb16,0x000b5075d48b2a76,0x0002c7c72f3112e3,0x000256a2a08a64cb,0x00007bb6c8e3ab45}, {0x000cdb3da1c5cb8b,0x0003c150935de71c,0x00071dd0d002611f,0x0001db99d4d16d56,0x000038a095f1bbf4}}, + {{0x000d0fa367f54448,0x0003146d9fd69644,0x00003ace9de42457,0x000b4368540bef28,0x00007344421ec2ec}, {0x00080473886c71dd,0x0002dbd4c0da1c7f,0x000e1b1b8096a337,0x000b47b29c3663ce,0x00009b850a4fe4c5}}, + {{0x000e92e2711f6c64,0x000b96297ee64980,0x0000a0287ce98611,0x000f59621aee88a6,0x00003a13d7736ee6}, {0x00042a0c754f85e6,0x000e90b3dd1a56d1,0x000bc903c6539c68,0x000a0c394962bd7b,0x00000c0dac7c0d84}}, + {{0x00006ed5888a2297,0x000751181c6bd971,0x00001fa29f66bddf,0x0003dc751ce5d2c6,0x000030a703678c28}, {0x0001544a52f8cedf,0x000be7386c005d73,0x000b3a1020b312a8,0x000964a3e7d61e3b,0x0000cd0a1f2cc51e}}, + {{0x000bbb7730637d22,0x000c0ee69ffebac1,0x000752c78cf96785,0x0003758cc83733f2,0x0000a885c698de85}, {0x00081b51bc9dd850,0x000b4da7bf040293,0x00006ffb160bfa60,0x0008090cb031432f,0x0000a5961de3acd8}}, + {{0x000544e3c5d8a4f3,0x000d695380f1fb71,0x0001dd2550cdfc3a,0x00008c41c6df5c9a,0x000015751273680e}, {0x000829ee56b6eed1,0x0008736e5950fa14,0x00084ee6a21b66ec,0x00068cf12d853922,0x0000aacb8733b362}}, + {{0x000744efd56dcf02,0x000d5ea49439ff91,0x000ae04d42849b3c,0x0006a13b6e556b78,0x000068bec5e02e90}, {0x000aa440bca4d3c9,0x00061e186837ca14,0x000a14da69cfee5a,0x000e96cde0f100bd,0x0000a67e8a765191}}, + {{0x000e0aa93f9d4461,0x0008116ba5c9f703,0x000eec78f1b1b2fe,0x00027eda47d1be72,0x0000185448619cd4}, {0x0002b1e0e6c02d92,0x0005c61712615570,0x0007a197ef949c1d,0x000c916bf10a070e,0x0000fd96d47b2de6}}, + {{0x000688afda1eadac,0x0005b4195e38ca30,0x000eef15dd8a8720,0x00033df5b11f7a9b,0x00009f2a47d71132}, {0x00023ac090f6dd40,0x0002214691c3f9a8,0x0005cbd3a69343c1,0x00055aad1e8ad7b1,0x000084d228267ea4}}, + {{0x00086fe50c54cbcb,0x000f653d0a7793ff,0x000c8c956c1fb89e,0x000283d2e92f9f9a,0x0000056a14c921b1}, {0x00089485a04b6501,0x00077ea06a75ec6e,0x000a3c4a6d6534de,0x0002a89696e721dd,0x00007fda5cefe58f}}, + {{0x000c5318d0e40d7e,0x000f6a538eee8279,0x0004e48a6eb9f6e1,0x000da4bcb94b8cff,0x0000c4899eed700f}, {0x0002a357b45aee63,0x00099a0087e74e39,0x0001e767f55766e7,0x0000e932c5c64a26,0x00002c5fc6626169}}, + {{0x000c2d890c562791,0x0001c4070cc2c8dd,0x00054def1b70c1f0,0x000d5c30fdf7b1ee,0x0000dbfe11c8f13f}, {0x00065c130ac48a93,0x0003ce26b496f369,0x000fd989f121c642,0x00072c727a8a1055,0x00003a348bb24777}}, + {{0x000dcd25a190461a,0x000a106657730c58,0x000f65f30110341f,0x00052183383995bd,0x0000425c381d25fe}, {0x0004adfd6e549001,0x00041290a9292ea2,0x000b76df784221ca,0x0001819281717876,0x00003c6286226714}}, + {{0x000956ce903181f9,0x000756047fd2cff5,0x00082b850c30f551,0x000016a192cb7cc1,0x0000164246ae1ae0}, {0x000fc004f248d82d,0x000270a5afc4b0e4,0x000bbe52be2cd809,0x000bcc98f97e693a,0x000059d87c21fa20}}, + {{0x00041646e13ab52e,0x00054c259320b618,0x0000a293fa0cd0bf,0x00055b81537d75a9,0x000051de543bb104}, {0x00001de5fec9a0eb,0x000afab3f9905655,0x000b5b3263792733,0x0000f68755608613,0x00000d4e6f054c47}}, + {{0x000710267c542a45,0x00079aa6faea8981,0x000ab176cd6156ab,0x000dab6a3f451e57,0x0000377dae7b0730}, {0x00017a92f3c84900,0x0003ce118d488c05,0x00076348955e451d,0x0001227de12fc278,0x0000a2604acecd69}}, +}, +{ +/* digit=25 [{1,2,3,..,}]*([2^175]*G) */ + {{0x0000f07eb83bd033,0x000c42728bc15e5b,0x0007503622eb0cfa,0x000572855689e6d5,0x0000ffe17a0fa82e}, {0x000666a2a34814bf,0x000cdc30d17ce550,0x0008d4ebe4148a9a,0x000ee9bc0fe3fc26,0x0000237a4fb4ebee}}, + {{0x0001c57cbb6521da,0x0007e013ad088fb8,0x000e523b034683c9,0x0003fd973cba032f,0x0000bdd5593a2694}, {0x0002b0ee317fe3b8,0x000b0cf3aa3671bd,0x00053dd74d87c697,0x000950259f03a8c0,0x0000bd48cf81120b}}, + {{0x000c2d85f00903b9,0x00023d280c52e198,0x00003f2253a05cf2,0x000053ca5bce1e21,0x000095953ca7afa1}, {0x0007eaf045a97253,0x000b61ef1e17ab18,0x0001cc69a1ea96f0,0x000d886265e99974,0x00009d16dd8088db}}, + {{0x000070d24ea591be,0x000ddb1395a861ea,0x000c708bf3a531a9,0x00045cf5cab52096,0x0000f8dd80a0756c}, {0x00018008dc9efd3b,0x0008df4b56e64d3e,0x00050e16e3f81f4e,0x0005b29b73276fb7,0x00003164beeb41b9}}, + {{0x0008e3a4e414233d,0x0007c6b84bc799a9,0x000bf981390050aa,0x0001b5245d5f66b0,0x00008ade8eaa35b9}, {0x000dd15f5c095ff5,0x000754ae9f8d06a9,0x0001d477cbbf72a4,0x00022cbd33b02ff2,0x0000807cdb0901cb}}, + {{0x000ee8912650bd3e,0x00053f782a39217e,0x000618997e9887ca,0x00025a0184c0779b,0x0000f717203cdc98}, {0x0002d521bdde7363,0x000c28fb3a2da14f,0x00001a0c2ef900e1,0x000e1af7a79f2377,0x0000b5d67fcc63cb}}, + {{0x0002eb021df78d51,0x000b081c9f307643,0x0006e5c92268a6e3,0x000e68d0b6f1d616,0x0000780b514a1250}, {0x000363c9b4b5ca38,0x000c14058ae8d88f,0x000abcfae84c017a,0x00001c021a267bf1,0x0000352d6c69a7e8}}, + {{0x0000ce578ee0421a,0x000ce61cdee800a5,0x00088829e74b485e,0x0004993b7a9f52f5,0x000021992713481a}, {0x000d96b4c61e5ad5,0x000c4dfe48a2ed65,0x0001425d4368e4fb,0x00016f34820cdfc5,0x0000a61d206e3b36}}, + {{0x000e3ec504534afc,0x00004ec7955c88a4,0x00050fed6537707e,0x000f6d10700e71b5,0x00000877ca57aa69}, {0x0002364134c7dfdb,0x0005855b263b3933,0x000dd5682153885c,0x000150675240b632,0x00005d29c6405791}}, + {{0x000b0d18666db72a,0x000f2d06582741e7,0x0001793ca3db5107,0x00077db29e38bab0,0x000084e11850de3a}, {0x000c8c2f41c8d79b,0x000501d15f801de0,0x000e2fa4edb21191,0x00055b0e59608ebc,0x0000e2af30fb06fa}}, + {{0x000365bea852e986,0x00023d23f18d3454,0x0002a0a0c61de9a0,0x000db69a0b114ba2,0x00001a64b5ee88ce}, {0x0007873ef9d06036,0x000df11ed48a5e35,0x00071a5b115f7948,0x00021de1c334adcb,0x0000265cf2f68415}}, + {{0x000cad85cb2c08f1,0x00013050f5bebd73,0x000c923a2320bf0c,0x000fbc656f4c2f1f,0x0000b53d151eb2a0}, {0x000b47255d36eef0,0x000a7b1171c22301,0x000724d1552a8c41,0x0001e7407c769b93,0x0000a4c8f8e5501e}}, + {{0x000ea34e3d43a020,0x000a28fe86195962,0x00015f931be0edab,0x000b1a5b51a1a2e1,0x00008f3b12193b36}, {0x000f4bbef5440b0a,0x000163cb97e2f791,0x0009344115a98d55,0x00030bfa810b1d2c,0x00009ac76ae97e64}}, + {{0x000a89f24dbe5bbf,0x00027581706f6f1e,0x000b2064b8a59052,0x000c28e5a7101b9d,0x00005819dae01ddd}, {0x0008e03d02d3f29a,0x000fbf2a1752e0db,0x000e534102f0d5bc,0x00036cd64e0d1f26,0x0000e99c7c9e4cab}}, + {{0x000adc1b52d2c43f,0x00033f467ac3b66e,0x000ff7454896c1f4,0x0005b725476ddb98,0x000076bf8f2486b1}, {0x000fd017adb3aa43,0x000cb44c8927398a,0x00058f3e9553da51,0x0009000230e271cb,0x0000539e7977c345}}, + {{0x000fad0b915022c3,0x0002106d7a523ebc,0x000335d2c80fc99c,0x000c9cb1584ffa37,0x0000e03e0f554705}, {0x00006adb6d93260c,0x00003a928e28cd3a,0x000cb2640092877a,0x0007e59d702481ee,0x000096acaf7a1b0d}}, + {{0x00043348d46add3e,0x00017b9f7c368909,0x000a1f1374adebe3,0x000cf21b998ac50b,0x00006e1a007a85e5}, {0x000fc1c2509d7d1e,0x0004e813db920aac,0x00064a506bbbc35e,0x0005c8293e917e2c,0x0000a0658874c8bf}}, + {{0x0003c2cd968d2bbe,0x000cd4c6eb2327bc,0x000bc548ced0417b,0x000e8d485d4af2f9,0x00004dacd98b166f}, {0x000b9e5cc5ded1dc,0x000ae2a98ba23580,0x000492a536faaf10,0x000e70e356736920,0x0000e510a11666b8}}, + {{0x000f3f1dc1e8b2dc,0x00066500e64a04a3,0x00036c76bbd3bb86,0x0008bbbf79e266ec,0x000021afeb04482e}, {0x00077f89864546f7,0x000725833277b7ee,0x000c09d42161b114,0x0003b6320fd7a5fb,0x00008b814a9efdb6}}, + {{0x000a001441991ea3,0x000ddf08d40e88f7,0x000233055e0b07ef,0x0001ccea6feaab02,0x0000bc15f20a5108}, {0x00095e60f38e9c6b,0x000c2ae91436b227,0x000f0513590543c5,0x000e03be4e76138c,0x00005328faf22271}}, + {{0x000a10d536481292,0x000410288d8eb4c5,0x00019c657d7affd4,0x0008693fb147884e,0x000050afa8840f2a}, {0x0004ae9998b069e2,0x000cb30712223137,0x000e91fd044e1f54,0x000424e7c6c23e5b,0x000007d112a47f0f}}, + {{0x0000f60ce2868df5,0x00049dc327eeea11,0x00046812a36db546,0x00080d91bf0f5a1e,0x000048f22bd3d877}, {0x000dfe5037a0b7e2,0x0007ecb51339b1cb,0x00071e2f0fd2d522,0x0001f8676a49b5a4,0x0000c769f1a6e725}}, + {{0x0002bf88dc2f0a70,0x000a3e68dc50f43a,0x0006cb84e98ae70a,0x000c6a2dab22067b,0x0000b91bfae77104}, {0x000ca4c4e1a302aa,0x0007c10be63cbe8f,0x000d88a1c0a2bd81,0x0005edd219abbb19,0x000010f5e51606e8}}, + {{0x000a2ad1f09fbd80,0x000db1f885fd8bea,0x0004e03fbf5c1af0,0x000c71abe93ec293,0x000068d6944c156d}, {0x0008f5795e2c39d0,0x0007de2a058c1fb2,0x000f391b7a363cec,0x000c511d56812b80,0x0000f1912e543c50}}, + {{0x000fa1bea2fc7856,0x000facf2d79b50ca,0x000b4dc605ee792c,0x000e37949b168ca7,0x0000f9f755e580ab}, {0x000214f4f0dfe3d3,0x000cb8f668d68881,0x00035d50ad3e2ab5,0x000d53349af8b5fb,0x0000eaf332242029}}, + {{0x000b3c0c5daf0c95,0x0000abd9ca770b96,0x0007491231340717,0x000c014e3753afa8,0x0000c1c50ae93cc7}, {0x00019a912d2fa053,0x000518344b1ceaad,0x000ee6c426f62582,0x0001b96bc3e144fc,0x00001b86a8ec07a6}}, + {{0x000f7a88ba02f836,0x00073dda3e59c0b9,0x0009cccb19acdf4b,0x00089607ecb07e22,0x0000e179a7fe03f0}, {0x000d066aa749105e,0x000cc15f2c1af78e,0x00014bae07f26f5c,0x00089e31dede1a78,0x00007ba2b8802bf3}}, + {{0x0008fa37e8bbf903,0x0002caeac4c26d3b,0x00054f9cd941b0e0,0x000695d6897bed90,0x0000879936aa651b}, {0x000a4948166b0278,0x0006106510d0399a,0x0006d4927293dbad,0x000dc9d16600bdd4,0x00002fd5ac433045}}, + {{0x000ea7b9f63a3889,0x00052557fcca4d89,0x000f2529e5e6ada2,0x0008362a0cdc0208,0x00008c8b9774d48c}, {0x00017a89e8d0e96d,0x000e14b6920ed486,0x00029494b799f387,0x000de743c654d3c3,0x0000a2f31f9adfaf}}, + {{0x0007183488f904f9,0x000017bbed9c088a,0x000762e0db3b3f4c,0x000a3f18431c2c9e,0x0000d188af79e181}, {0x000747e667c0d1bd,0x000e968e2a6500ae,0x000769759bb4bffe,0x00055beab9f86803,0x0000262d46060ae5}}, + {{0x0000f4053aa9a78f,0x000f79c35e544089,0x000fd839d2c910e7,0x000055db8e7ca105,0x0000db97ad2b5831}, {0x000eaadbd3c4f7a6,0x00049ef7c107421b,0x000f045e1644c17b,0x0004119453602525,0x0000e5c41998aa2b}}, + {{0x000ae5b06add8234,0x000ece621122e9d0,0x0005aa6bbb8095fb,0x0004f8f151cbaa93,0x0000c30736f0d3f2}, {0x0004bb0b2c0934c3,0x0007af112376af19,0x00016cc005f81abf,0x0002b33e6640b099,0x000054837f425912}}, + {{0x00060e6aff8da2bf,0x0007b2b372a89876,0x0003629cb60a2cfb,0x00061c0c2676d696,0x0000a734b56f2086}, {0x00098761fa9a1967,0x000560b82b8f4fd4,0x000c50fbf8a331bb,0x00006ab7b7096885,0x0000ee75dde39230}}, + {{0x00076dffc96c21b4,0x00063017c3d1bee4,0x000d9abd38be81d2,0x000f32f709fcddee,0x0000acb956a0808a}, {0x000b9e25e66f3281,0x00033831c8a599f0,0x0007eb06721417f1,0x0005cd4833b0079e,0x0000493a266fb576}}, + {{0x0004baa2d5668523,0x00003689fc40b0ae,0x000279c28d2c7f3e,0x0005629534d2937e,0x000098a4d84f2ecf}, {0x00035f14cb929a91,0x00061f8963790393,0x00009e7b7b25a5d5,0x0001016ac69aab90,0x00003d893e043064}}, + {{0x0006b786a36bceb5,0x000f6c58efedb4f1,0x0004457a463ce9bd,0x000b72d69b1127cc,0x0000fe789e83b3a4}, {0x00017c51f7e29b87,0x0001531fdf820296,0x000f2588967ed30b,0x000ed5a3df57f4a1,0x0000ce6d4a94d6a8}}, + {{0x00021eb676b5af75,0x0006aeb5c3a18291,0x00043eaf051ebc58,0x0004ef8bd71c578c,0x000001fbd625ad3c}, {0x00032c1d4663f519,0x00045aeb159107de,0x0007a554a8e64c3b,0x000cc9cbc678ad96,0x000020a01fc96657}}, + {{0x000be89bb43950a1,0x0005817b762b4266,0x00047f7019afa059,0x0003ca4a2c7f75fd,0x00003dbf549fffe8}, {0x00097451ad446c62,0x00064f101758f41a,0x00094ed44c88f148,0x000c5d31f7dab03b,0x000007789b3fa42d}}, + {{0x000b763df689270c,0x000edece73483d7e,0x000aee476fae171e,0x0006c30996252a7b,0x000068c6d9b71408}, {0x000802d40be0a180,0x000279f7214afa5c,0x000a50baf4c90054,0x0006601236076d44,0x0000f1a4df9f611b}}, + {{0x000a1d871f3ad971,0x0001471f93998a18,0x0007cb6b4089cfe6,0x00046e2888680be6,0x0000a4a0b67496c8}, {0x00051504d9720c99,0x000b82f1dec98c6e,0x000bc6615e19ef35,0x000cbff674933ffd,0x000029e4752286b2}}, + {{0x00010cf8006801ed,0x000ff3f96b824680,0x000ece9035f8b7c1,0x0004370ed08d2e0f,0x00008e43d7f5270a}, {0x0007758e8c30cba1,0x000a4c5e382f556d,0x000374942bed58f8,0x000f7b4d390b896a,0x00003148cdc67c81}}, + {{0x0003a9cd5c44392f,0x000e47bdfe21ed29,0x000491c1481a25a7,0x0006fe1cd7dae300,0x0000b9d3868cc4ee}, {0x0002068ada03a6a0,0x0000340f5762442b,0x000e0b53a208f677,0x000762b2ee05976c,0x000047fee704c66e}}, + {{0x00045be28bb6cc47,0x000bf9a6e2427822,0x0006b0e18c349f9d,0x000d413cbad70f27,0x00005db7ba2cd50e}, {0x0006fa76a8688440,0x000f17cc272cd775,0x0000a9aacfd31874,0x00081d50e472af63,0x000031b1bf3d98d8}}, + {{0x0003b11d230fb125,0x0006dafeaf84ff7c,0x000c87a66c21c356,0x000c83a7fe395ccd,0x00009e0055334e3f}, {0x000b297d762bcd8f,0x0004090b1e777788,0x0005f7ff73f79031,0x000b978916e41f31,0x0000d6a86b093da1}}, + {{0x000049a7d1255d7e,0x0008c8b12a9250ae,0x000341efc83f0f58,0x000003e1e125c570,0x0000f75675c0a287}, {0x000daf8b1258cf93,0x000cabe2242a1697,0x000503d8cea55b42,0x0002e7bc47653393,0x000010c33e17e789}}, + {{0x00005ea79f286771,0x000e27569d6a75ce,0x00081e4e2f605135,0x000742a515ec1123,0x0000af60690f9618}, {0x000edaa92529d0d8,0x000dd00e97be79b0,0x0002108f6ae93abe,0x0000519721405467,0x0000ef5c7cd11264}}, + {{0x000d7e85074cf812,0x0000fdaef4fbe1a1,0x00029585e7cd8874,0x000bf4683db12a45,0x0000f263cdb933fd}, {0x0000daef9cecfd73,0x0002d1cb1ad9ff75,0x0001e2010333c1bd,0x000602f2eb13e6e4,0x00008c434479fa32}}, + {{0x000f364154e53c02,0x000b2c50b3e176ec,0x000b8ac60e89a87b,0x000d69c11fc58ebe,0x000070424c27f1ba}, {0x0003d753d21bb139,0x000b6450de0ee6cb,0x00061e4b1f84aa55,0x0003f32665545bec,0x000084a236952f13}}, + {{0x0003559faf384825,0x00038d591b30f3b7,0x00072436cb28e627,0x0002d24feb572f5d,0x0000a8b72daa2f39}, {0x0003eb59a42e3589,0x000e94348dc3c258,0x000373f8afa9d0e6,0x00065df154badfc5,0x0000c01d9c6026b4}}, + {{0x000a5d00987d7af1,0x00054d16fb7aa127,0x000a5390ebddde5e,0x000638bdd6eb95cc,0x0000ad6055b0a46b}, {0x000e6fcc176765e6,0x000a926d17614afd,0x0005b60ce8f86b23,0x000cb7cce2306fd5,0x00003641982b357a}}, + {{0x00025cfb6810afc0,0x0000d2ca6813e5c2,0x000d4b0af3ff9d91,0x00060ae0e7a3e4a7,0x0000f245dbc069b1}, {0x000feb26a5cb6ec8,0x000bccbcb43885f0,0x000e47433ac2cdbc,0x000d82e118d811be,0x0000f794b01a4d8c}}, + {{0x000b7da95b4aaf8b,0x000e80a2c2af911a,0x000a345252c2ffde,0x000120078e3c033d,0x000043c12ac705de}, {0x000a2b459a347a22,0x0008ec984c45ab7f,0x0004f78db43cb528,0x000713cad280f6bc,0x000082a8e65b4562}}, + {{0x0005a98aa0ade635,0x0007b2a62c54a937,0x00023803b526bfcf,0x000b028e8fa38d03,0x0000e116fb100b08}, {0x00099fca2c266e2a,0x000cb41473a5abfe,0x000cbd4a3edc82fc,0x000ee0a832b0f682,0x0000c6da25c35e5c}}, + {{0x00004550befd3bc3,0x000b40a480954444,0x00050861335df122,0x0002e8d27b00d3c2,0x0000e5cf22108602}, {0x00032b19ede4f65d,0x0005402e3b45b297,0x00081c377a4b3990,0x000db8602b61ae64,0x0000853d2136c902}}, + {{0x00041daa3fade2f3,0x000c4f5787b60696,0x0000017c50c733ce,0x000cae7f1ea7c369,0x0000822da6a5138f}, {0x000278eeafc2f192,0x000da7b7d5aa7a6f,0x00077a2da02fd70e,0x000a24eb85681571,0x00004c69d2e8b3fc}}, + {{0x00061ccef0bb0ac3,0x0005ae0f16b6abe3,0x0006987ba1d88953,0x0000e4764662ef0f,0x0000fed054617265}, {0x000889078fd300ed,0x000dc644c4935004,0x000723fc14e7aeb9,0x00001edc7f36fff0,0x00004b9f194f4d3c}}, + {{0x0002cce358b66aac,0x00016e2bf8e318a5,0x00084f0df71d65c7,0x000699f185f6f605,0x0000d06b9aba0d6f}, {0x0005205b7b70371e,0x0001c3cc258b70b1,0x00064ef797aab8d5,0x000ae313d81500be,0x000091b0d1e2d796}}, + {{0x000c14085cd77a2e,0x00055b024ee5e095,0x000a1a0b278a5dd8,0x0005b23d4b412cf3,0x0000f1bb25ca44f8}, {0x000da53ecc55a1cc,0x00040b943b5047b6,0x00089e5806f90cad,0x000403b58fa8ea96,0x00002429e57eaa6d}}, + {{0x000fdb4fc9660aab,0x000ae4b79d49d8f8,0x000e3da0cdf3ff94,0x00095a287b418943,0x00000220fb3bf163}, {0x000a22b7dbf3af75,0x00010e4436d28b1b,0x000a5068eb344ae3,0x0005e62736f5e1f3,0x0000f7041a241e82}}, + {{0x000ddbeb3f0ad80e,0x00075c157deb17ed,0x00083992e4d16850,0x00000290b495d7d2,0x0000c100310a5d3f}, {0x000f68ee053ebdad,0x000e0742709e626d,0x000d131a85412994,0x00030749a376ce4a,0x00006015a407e49b}}, + {{0x00032582654c7441,0x00053280089731ac,0x000bb5cd371832d1,0x0001aaba1ddd1d3b,0x0000ce419eac3358}, {0x0009570f5151ed5a,0x00076d6327900c1f,0x000c1968aa5e7f59,0x0006745c1da0190e,0x0000929c46f7b7b8}}, + {{0x000957b50de1fb7f,0x000cd8dfe690465c,0x000125b716eecbf0,0x0006b08cc2cd7fde,0x0000cd58f093a3bc}, {0x0007ffe391cfca39,0x0001ad8536a941f1,0x00047fec75582f2a,0x000ad3a9a7bfe386,0x0000e968cdb6aa35}}, + {{0x0003c22a58371af3,0x0009c11c7d557ef0,0x0007d54999e44d4f,0x000d7878f8030b46,0x00009bd72fb1023c}, {0x000507350d5a894e,0x0000975cf9c22941,0x0007ee69d55c81fa,0x000da33294738695,0x0000dcd116f20c82}}, + {{0x0003d73ce3e731f6,0x00067ef5c65f87ca,0x000cdad304e237bb,0x000b03dfef249e4c,0x0000d7ab5f93a8aa}, {0x000687e71c7835b9,0x000ada6bfddeaf3c,0x000b7e2419bf388b,0x0008a2c3f86d5779,0x0000dbf47353ff02}}, +}, +{ +/* digit=26 [{1,2,3,..,}]*([2^182]*G) */ + {{0x000d03b214a50ffc,0x000ef4d64a402d36,0x000aa27d487e7e1e,0x000960971f296c17,0x00001a46349f6f07}, {0x0006c65b2e7b77fc,0x000e7b3f579f668a,0x00003a0809d29fb8,0x0002f59cccc9b480,0x00008b96ae5ed56d}}, + {{0x0005623d2692dce7,0x000bb1439ee234b8,0x000abb782d3d06e6,0x000ad2f5b24c26bb,0x0000e1e5daf85aee}, {0x000f6b36e620b2be,0x000a6ea10ef984fc,0x000e9c789089a1c0,0x0008b74663659974,0x0000964db299747a}}, + {{0x0000e9e78a6de0fd,0x000e2abdcd0cfd27,0x00058ced92b387f3,0x000ff3a326731463,0x0000d2b37033bfd7}, {0x000a11f5037eb879,0x000c93128308bcfd,0x0002cecfc87b981c,0x000a5ce342d6130b,0x0000f944d511fb1c}}, + {{0x000f4a7a5b2673fb,0x0002ae68fae4404a,0x000f3366ba0604c7,0x0000b42cf803c9c0,0x0000ee8762def9c5}, {0x00068b0cc0c37ee0,0x0009620fc01bad69,0x00028d071cb3006d,0x000f8e637903ab4b,0x0000248af8b7ed90}}, + {{0x0004cdaba9bab8b8,0x000b4adf23c61405,0x000ea4e3112b10ca,0x000cff8f40c789d1,0x0000f3193286f762}, {0x000bc201e35954b8,0x0000e44d9cd0d7e3,0x0002dddc5389fd0d,0x000726e698571e6e,0x00003a64f0002bc3}}, + {{0x0007fb0822f35da0,0x000f09896ddacf82,0x00029979bd5bf27e,0x000381fe1269fc42,0x000052db83e4e95b}, {0x00050a861613e2cc,0x0001b06477bfb26e,0x000ecf598588df85,0x000b11c80fd01053,0x0000225a35dddd00}}, + {{0x00008399fa0ca14f,0x0009af16e80f4a00,0x000377c5e70b4acb,0x000c7c3b7a59ec93,0x000030724a30aaf5}, {0x00045d067262f443,0x0004b25982507749,0x000bd81b974e4f49,0x000a4d9132316fb9,0x00009a156b7e230d}}, + {{0x00040413b3024eea,0x0005d69b55098af7,0x0002a1af66a8f315,0x0008239a731304b0,0x0000e91a74955492}, {0x0003e0c7ee6a3f82,0x0008e0bd9c79b191,0x000152ebf5971c24,0x0000a198183e438f,0x0000d267b51648c7}}, + {{0x0009e678da935547,0x000192f9368350a5,0x00092c6744306b26,0x0001c83d266e0e8e,0x0000ff9bbcdce63d}, {0x0006b94ad0d71138,0x0007aa419046b6e4,0x0003f4797c423b94,0x0002636860d19aed,0x0000349f4cb07f5a}}, + {{0x0003e6b28a2b8001,0x000d21d65179ca2e,0x0001871c7a0e4390,0x000d93c4cf04dd6a,0x00003d99bc265487}, {0x0004e3cb86057988,0x000c2fe15fb3216b,0x000560cc19a197d1,0x0004a67b74e959eb,0x0000f4e1b585517c}}, + {{0x0004cc98e585076f,0x000e80aad228c051,0x0004600361200736,0x0006fca66598227e,0x0000a461711c06d4}, {0x00065a584be71d79,0x000f1f684055450f,0x0003eef672a8e1c0,0x0009f9a749b821fe,0x0000188a93a6ca82}}, + {{0x000d9a6edbbf0204,0x0007fde9666ee420,0x000b10356afb0a5c,0x0004d7dfc83b0c37,0x00000e52e0312799}, {0x0009e4ce359cd84b,0x000bf43f9183ab0f,0x00099c7e0393de76,0x0004427f846ebac9,0x0000439eeb2f8904}}, + {{0x000fce31fa271e8f,0x00085dcd6ab53339,0x000bf17ca52896f3,0x000c00ceed5c8afa,0x0000aead04245383}, {0x000a90776147dd2a,0x0001155288ca6668,0x000a7aac8449f137,0x0004f1a2e10e32a3,0x0000c8394d4d3c0c}}, + {{0x000f50c692933c5a,0x000c78ee682abe36,0x000b3d9a6c4ffe44,0x000be5c7b7730907,0x000000a137e49709}, {0x0001aca96183079e,0x000f9f04811611ad,0x000e37c96ce57770,0x000dec262fa1d23f,0x0000e281f8c1077d}}, + {{0x00025dacbfdb5d2b,0x00008995387e4bda,0x0006607c2701b7ab,0x00021e573e7f7184,0x00007a3aa3d01161}, {0x0000e9867cd0a313,0x000894b24ebf17cb,0x00086622314b0868,0x000340b1fc343aaa,0x0000480efbe0da3c}}, + {{0x0001cf09163daf1a,0x000e3feb1c836f08,0x0005a4a98decbf2d,0x0000d647b2ceaa70,0x00006d8310bc54fc}, {0x000b4c2600d63727,0x0003e4d5e8071ebd,0x000666689b444f65,0x000194168bb5dd62,0x00000c6b36071bc8}}, + {{0x0003c4390dc638a6,0x00084ed1e9bfb767,0x0003234c2e027266,0x0007342738b46e6c,0x0000f7959e2d7589}, {0x000dd45f4888ca02,0x000eb5127527806a,0x000e9bae4eab90bd,0x000a8b2d4fa5ffe4,0x000087b16d15c5a1}}, + {{0x000095b3fc633a8a,0x0007316bca0fc94f,0x000d8b2668cd3581,0x0008cbfa5aa5e026,0x0000c8c21a873920}, {0x0003e4ecb61679d4,0x000355a4fcc28c11,0x000ca6257920c25f,0x000b82d2d14b872a,0x000011540c35e1d0}}, + {{0x0006e26c82c45233,0x0001a997670789d5,0x0002a43f81b41944,0x000726ab8025355b,0x00003f01d8297cfb}, {0x00048146db204a06,0x000c7e02913ffa0e,0x0009ca312337ee0e,0x0008727e2840e097,0x0000fc4d0f49b524}}, + {{0x000b8a12308e33da,0x000337ec4d8beb7b,0x000c1659146b745d,0x000de150304503d7,0x0000d024f01e4921}, {0x000a6c823f9984d7,0x000eea03c4fda33f,0x000cc3e9c09105f7,0x000b318d61bb4a13,0x00006e743847f379}}, + {{0x0002cd374b837d25,0x0006c2ce00b7c947,0x00043b58db8cf662,0x000ecfef51327052,0x0000ac22adc4bf4f}, {0x000d1d45993f14ab,0x000739554cb38af5,0x000a81d9c426f8b4,0x000c7f0bb7e92096,0x00009053648a82fe}}, + {{0x000ed373accc4794,0x0003f4fa67ee2977,0x0000311becb2042d,0x000b24d99c7a2ceb,0x000036169432637f}, {0x000893691b72a9eb,0x000ecc61e9fc1546,0x000f6ea558e349b4,0x000779af664fd7fe,0x0000108ee2c5323d}}, + {{0x000ae27d9b8b43d8,0x000f3dcc55d2272f,0x000fb53844004b23,0x0004b76bc98d5c82,0x0000ce7f298d8e2d}, {0x000e2a94d1a64a33,0x00061562e50f2006,0x000029eca89ef15b,0x000ee81c47d9d4c0,0x00001a9fd03172d1}}, + {{0x000e1f968fcfc9eb,0x00006954531ab241,0x0008e861c066d0b4,0x000dc8c192eba4ff,0x0000ce2adf88d121}, {0x000aad0af982f095,0x000e37d4476b8527,0x000cf5d53ffa76e5,0x0009468c298e73ad,0x00005bffed75e1d3}}, + {{0x0007b7598b4f4c18,0x0008c456357c5e26,0x00015f31642f0a26,0x000bc5a0a3a625d5,0x0000d18b2d13a25f}, {0x00020409ad1c3378,0x000465836e75f72c,0x0002c5ec514b90a3,0x00016a126f9a9b39,0x0000d930609e4c2e}}, + {{0x000c5bc32c3d376d,0x000ee0c900c8573f,0x000eba03819e1ab3,0x0001b7ec2fecc561,0x00001001e880c624}, {0x00029ce1174fb908,0x000642285b2c2478,0x000e27cd432e3da1,0x00017bfd02b9a902,0x00007f7b6a8b6d20}}, + {{0x000b1e4ecc5b0dae,0x0005c2c0e62e9f5a,0x0006c9b1bc80f2c5,0x0006a81d4c1fe7be,0x0000c726d37bcccb}, {0x0001e5d17cfa0d17,0x00024f9bf26ab169,0x000ae21bd95ea4e3,0x000c96246db5808d,0x0000a655e7c27c6b}}, + {{0x000c7fa80a03a60a,0x0006bec2b0bd1212,0x0005bf438beb7af3,0x000a43da5be37048,0x000068eef804296a}, {0x0008674e2d968fc6,0x000f0425a5c48998,0x0003907aaaec9eee,0x00004751bd315af3,0x0000485845c77215}}, + {{0x0002d3539f63c362,0x0004eb689b184334,0x000fbfda21a18b97,0x0003b8d5ec44e740,0x0000b3ef041e5ea3}, {0x000172b46d952e28,0x000695f9dd9c7154,0x000c09cc503a2dad,0x000a80b8da9fe4df,0x00006bc6f98f03ef}}, + {{0x000aeb710af227b1,0x00042752ebd12219,0x0007f5abe6fddce3,0x000db478c999a0d0,0x0000464bcddc627b}, {0x00057116a55b2f45,0x000c46f39dc73fc6,0x000bc331ade611c7,0x00096a74bc1efafa,0x0000696914d1a572}}, + {{0x0005d9abb772a0a7,0x0002a585878f4775,0x000a12aabfef29ac,0x00077327bdcdc5cb,0x00006acb1b0fff47}, {0x000942a009622cf5,0x00060793cbe33a99,0x0004e6319018ba81,0x000462893e1ab4e1,0x0000067a7a34b6f7}}, + {{0x000c5d6a01c29e20,0x000c5755b9592a39,0x00003bd60271462e,0x000f05b7ed4bed82,0x00002c26351eea6e}, {0x00034a423d7cf134,0x000f08beb438aa4e,0x0009c47c4106b0b3,0x000affb4d68d1672,0x000093c8daf9d600}}, + {{0x000ba868113c9a1d,0x0000338cd1fd0610,0x00063c596a184b17,0x000121f53902bc06,0x00002a0f700a0162}, {0x000d41437a1108ab,0x00028b89ad88ec92,0x0000900636b15cde,0x0005693beb3a30fe,0x0000f596d95269aa}}, + {{0x000383992de6a0ea,0x0001c0eadf49d191,0x000b6249f08b65ce,0x0001bf26ab78aa37,0x000036b6d694e14f}, {0x000cd0f12f4691f3,0x000c7064c2be3f58,0x0006959713f90dad,0x0006d890ca9925ff,0x0000501135076e91}}, + {{0x0007b0cd9dce526c,0x000768dafc03b308,0x000d34455379120c,0x000a2d9703f89047,0x0000693de4cd17fe}, {0x000b57df03438758,0x000d3febe5d03f63,0x00006a2f5070a0ea,0x0005caa95fc81462,0x0000a1d0ef60cf4f}}, + {{0x000faed6a35b31e6,0x00027b0d2c666a29,0x000c0459970c0221,0x000d9efff00dd62e,0x00009fd870cfce88}, {0x00072489fe1ebe36,0x00056c3b41919584,0x00047468a39ca80d,0x000ed62caf69e96b,0x0000f68b71d9b183}}, + {{0x000f69c3fcce6d5b,0x000cf30e7ccdbcc6,0x0008023f7571be87,0x0006101b76ba1de8,0x00008ff4180c3051}, {0x000651f6c7323796,0x000d92757ac444f2,0x000e84bd2228de69,0x000d6650d8cb40a0,0x00009a11dcee05e3}}, + {{0x000a9ef3a388cd6d,0x00079a08b3f94c88,0x0004ed893c32273c,0x000eaa93278afa0a,0x0000804db1067191}, {0x0003ed02eeb1eaca,0x000aa269e85917ec,0x000b98ab4063caf8,0x000a9141573324dc,0x00003c41f6a74303}}, + {{0x0000a2c274fafa67,0x0007661f730e3668,0x0002382fe8bfcaf3,0x00019134ffd8ed37,0x0000935125943aa4}, {0x000244800946cd0c,0x000df0e459918728,0x00032302f204b042,0x0007899c8ebafedf,0x0000be5328a81cb7}}, + {{0x0006d0031fcee36f,0x00055e0a95cad3b7,0x00011f2abc910fa9,0x0005739da8677699,0x0000b7ac64a83b18}, {0x0005369151002784,0x0009d0563f5b5d1b,0x0007eef50df8b5bf,0x0008efc54918844c,0x00005ba66117f7e1}}, + {{0x00017aaebb23a9a3,0x000fad1195b0c4f6,0x000a08f41d8d5df0,0x00015b6a2f309dff,0x0000436a93489d4e}, {0x0002a63bf9a141ab,0x000fe97001b45cb3,0x0004f5b5f37347b5,0x000d0a7ffc6ce12a,0x00009b94f9ad54b9}}, + {{0x0006d07205aef3fc,0x0005231a57be8a89,0x0005a94a39f4563e,0x0003e9dea91ed89a,0x0000f63dabb48ec7}, {0x00024642fba4d373,0x000230b44974c808,0x0004d50af40a7979,0x00079137f5088ce2,0x00000609fca476c9}}, + {{0x000e8b2818c83ff4,0x0002f8faff6baa52,0x000d294722da303e,0x000f9977a2d4c556,0x0000cf6d037dbbd3}, {0x0003f7b469280f8b,0x0005bfc44a6f2f63,0x000adb24e051b615,0x000224ead0ac57d2,0x0000adcd2c71b8b8}}, + {{0x000f8bf1604c0532,0x000c14fadeb8bf40,0x0000e2ae1f370c05,0x0003afc120728f88,0x00001b8f66baa365}, {0x00075ab77c2209cf,0x000d8c32043fe2aa,0x0002c82e849b56c7,0x00012fd42038c184,0x0000fe0f8955a57b}}, + {{0x000d850f732fb7fa,0x000609faf4d7ae20,0x000f7f1924417e70,0x000349fc42eeeeb6,0x0000a6a934517190}, {0x00018b7cf247e02d,0x000f500de589351b,0x0009687358eba05b,0x0009fc5f5739e627,0x0000ddef5f556d65}}, + {{0x000774d30f504541,0x000f79d101da68e4,0x0005271bce5362f0,0x0001e6faa152f180,0x0000b838e945b4b6}, {0x0004064dd6323f11,0x0005ebb09f685488,0x000d8d4d8dcd854a,0x0006b7868850b02b,0x00009b7ff60aa093}}, + {{0x00094c377476a6c8,0x000f8bd879dbc613,0x00067c19587966a7,0x0002fdcaec3c6f44,0x0000f61ae7f6b03a}, {0x000e1fad21564449,0x0002c030664334ba,0x000c37cd03a60116,0x0003c2589ba1879d,0x0000869a09a56a80}}, + {{0x000ca054eff5af2f,0x000c2afeb00c909e,0x00030409f4f95238,0x00042f33519755ec,0x0000a031194f7bc2}, {0x0002d9d860515c64,0x000282b7c41b8262,0x000d58ae84b75001,0x00033911d917c2ec,0x0000382de3015dad}}, + {{0x000b6f92039287b2,0x000424f610648a17,0x000f670c6adaa03e,0x0005ebcc4155c96c,0x00008a428fdd5ded}, {0x000dc8bef29ae895,0x000feb853aa156db,0x000d58735fdd8c1d,0x0008ef18dbe67090,0x00005223ae90f12f}}, + {{0x000da0a305e720fa,0x000a9efc2a6ba83f,0x0009bb0d2d9ee4f0,0x000f424f84963425,0x000028fb31a147c9}, {0x0007a129692f9dca,0x0002c69d8dbf3308,0x0004070a2d33c240,0x000e53497ed09a9d,0x000056c1a642f857}}, + {{0x000fea20ff9207e7,0x000cdcba65125a16,0x00026ad2a6c19e9f,0x000dc711f4cf52db,0x000097fefb6bc4e8}, {0x000a94c4f27e69b5,0x000570e2212029fe,0x00031e5f3eb7feca,0x0005eb519c7da3b5,0x00007bb88afb91d0}}, + {{0x000a7012729021cb,0x000f25e2a3025023,0x000308907678c8f1,0x0003c3d53c45e204,0x0000ef510928bb4b}, {0x0002e7148c22ff2d,0x0008709b3aeae6c4,0x0007356abf6dcc9d,0x0005788d1b63fe8c,0x00009e378edf3940}}, + {{0x000ddeb357d98af5,0x0001e85a4558478c,0x00080a0cbd7e0c6a,0x0001dd3ddac79b65,0x0000056d8d40a6df}, {0x00091ba3a685470f,0x000a930cac5404b8,0x00005c3833644e17,0x0003a3e93f00d595,0x00008ff0d37e4337}}, + {{0x000882736c20485a,0x00008eebd8ed436b,0x00008765d62ca0ee,0x000eb404c1e6a63c,0x00001f51c3d2f4c5}, {0x0001dead0e0240b4,0x0005dc01cee53aaf,0x000ff1626340c23c,0x000eeab99ab3c4fb,0x00001c9f03e31c0c}}, + {{0x000ae756a6906fad,0x00063f4f9ceccc4a,0x000097866d772f72,0x00049ade04e7abe4,0x0000daaa2d4ec655}, {0x000f24569a5e1b2d,0x000cf346e0eedaf7,0x000077bb20e192c6,0x0002e1123812eec0,0x000043676cdfbb9b}}, + {{0x00058d8a85049646,0x000da101751a9416,0x0004052f48e7403b,0x00083e451d1f0c3c,0x000099b66e6f5e43}, {0x00099f641750313c,0x00052d7eb737c5b3,0x000781bc1f3ec6ea,0x0000d8f06b519c58,0x0000207bc899b604}}, + {{0x0008f806abe9cc03,0x0000a9a929601fb3,0x00009bbbf082d14e,0x0002f8208e30f142,0x0000554af3ba8279}, {0x0007f63f5b712db0,0x000f5e6bec6bc64b,0x00064a6500c0f15b,0x000c133fac18ffe8,0x0000426e543866c0}}, + {{0x0009d4ef17dca9b7,0x000a30ed49144714,0x000b7c7c7ff66bc6,0x00050b6f85c5627e,0x00001b5beb731ddf}, {0x0006b5a5059abd38,0x000239a4d95c865e,0x000fcce6a4320b87,0x000f54de8482cded,0x00004902e9c46a14}}, + {{0x000b7856664976a7,0x00078969c15376b1,0x00040ec0985a2ed7,0x000c001d865d85f0,0x0000f36fd193022a}, {0x0005392113e30f9c,0x0009f74dd0ed247a,0x0005828f69efb3ea,0x000a340b38e8b204,0x0000360f51bc06b3}}, + {{0x00080676229e895d,0x000b73902cd1daa3,0x00033ab334e9238b,0x000898af084a55e4,0x0000ad12d95a215d}, {0x0009537f56338b7b,0x0006b9ccc450a222,0x00005543a3bcfda1,0x000c3e65d49dd006,0x00007bfd6c4ca559}}, + {{0x0004e97c95dade65,0x000a255cb184c7af,0x0001913f02135561,0x000dacc6ec185432,0x0000c67073eac645}, {0x0006dbb0217a30c4,0x000ff3ddda1e0927,0x0001ccae8f3399c2,0x000457eca9825683,0x00009678071934aa}}, + {{0x0001b26feb7180b0,0x0003efd985904aad,0x000b49ff165faabd,0x0001d381a30fea11,0x0000c5c8356509ff}, {0x00001e0aa66f02a5,0x000536ad9cf84113,0x000abbdc628e871a,0x000d020376aaac57,0x0000a1378acc9219}}, + {{0x000d95e7668e901f,0x000060472569120d,0x000fc85db23dc73c,0x000aa6886846b97d,0x0000132a04ded861}, {0x000d5373fb18ede0,0x0004a57df5ac83b6,0x000da2aea92385a7,0x00022238fd2cc831,0x00002fe0eb0e035e}}, + {{0x00024cd9846e4d56,0x000a4f069d729131,0x0001f60df8dab9fc,0x000503b97c211a7a,0x000006e35fac3fa2}, {0x0003c474bc694434,0x00075c5dfd89e498,0x00042a47aa5934ee,0x000302b62af4f70a,0x00001298623271cc}}, +}, +{ +/* digit=27 [{1,2,3,..,}]*([2^189]*G) */ + {{0x0001b9cf1fff1383,0x0002e979256fd16d,0x000024e5cabb7109,0x000174971f93d488,0x0000f4a50d323ce2}, {0x00069687a5e60329,0x00035725ef1c291b,0x000d78bd756a79e4,0x000d4bb336a131fa,0x0000a42d2de19780}}, + {{0x0000770b1fd77c42,0x00073832340c46a0,0x0008fa3e971dd6ac,0x00055cfef494e03a,0x0000a4bd01ea78ff}, {0x000549569ec3b61c,0x0009fab41b9cb0b0,0x00085d069aee9d31,0x0005df668e37395d,0x0000222a186cd468}}, + {{0x0009034839af8af4,0x00043f6fb866f9e9,0x000b3d4bd58c9dea,0x000f06eb6869dd3b,0x000070c089100bba}, {0x00084c12d0fa30f3,0x00063a684ef0b7b3,0x000c5e098f27fa26,0x00008ba673334a74,0x0000febfadaaed70}}, + {{0x0004aabd136b85c6,0x000e5936fc61535e,0x000d5983d0ba7b67,0x000ed94d56bc23ae,0x0000fafa92939d12}, {0x0005ed1e9964a345,0x0001980b9f13e261,0x000c2d91983b5b76,0x0007686f1cf68c47,0x000002a1721de2ca}}, + {{0x0003d97f0a373569,0x0009541867b5c615,0x0002be69e5b1a6cf,0x000d318011508202,0x0000616a3fc9c362}, {0x000f8fb5bc1d92d7,0x000014a7ecae4920,0x000a07c0dbe70008,0x000136174fb9a821,0x00004a51fb854142}}, + {{0x00094501f646ae21,0x00093a3042a9aa65,0x000c3d31b80e7d9c,0x000f2a8addb71e1f,0x0000e4b00409743c}, {0x000cfea457fca237,0x000419bd5f99c642,0x000c9c750bb14dcf,0x000201a965cbd91a,0x000003a84781aab2}}, + {{0x000fc80219d626fa,0x0003948b0d1fdda0,0x0006db88bb4e6b39,0x000ff4f7b88d70b8,0x00003574172195ac}, {0x000d5e56f3d1172c,0x0002cead1829e0f5,0x000565f4996ea68d,0x000bab6b497081b3,0x0000f563edef0941}}, + {{0x00052442b55502c1,0x00010eecb291d39d,0x0005128fee4d7b18,0x0007125122549f56,0x0000056059074287}, {0x000a0909597c6015,0x00008af2773cc6e7,0x000b42443316a926,0x0006dec8c5b194f0,0x000077ea1102d890}}, + {{0x00069a576212f358,0x000506a394e93535,0x0009b22ea1ceb350,0x0000677ead07678a,0x00005b082f491db8}, {0x0004a81bf9ec1ad2,0x00071aee7ba0c916,0x0004e80a1337bc58,0x000abdbd5c577480,0x00000fc252a5dd59}}, + {{0x000a054c1e13f877,0x0003dcc6b450cabd,0x000d3144e7b6b2ae,0x000ed4c39ca22c31,0x0000f9567c38732a}, {0x00084e2f2344d0aa,0x000ec5ad8d221d7b,0x000e1a873a63bb2f,0x0009666a6747d67c,0x0000358844479c6b}}, + {{0x0007d2edba4a511e,0x000f2f775e00973e,0x0006cd11f44770c8,0x000bfe41e571cfc7,0x000075b695fd8542}, {0x00006e872fba8b1e,0x000e6a6534ab0dc0,0x000f0febaa07564a,0x00029bcfde2d885e,0x000025ff3bd3db13}}, + {{0x000f42c3d41c8e40,0x00027896206e317b,0x000581e8807e38f9,0x00072dbb1ab849ed,0x00008c003afb741e}, {0x000b05a75a716b03,0x000c35acc098339f,0x0006bb409efb624e,0x00044ea061fe93db,0x0000639c1c4896a9}}, + {{0x0005df2305ee4c95,0x000fb06813dbe2b0,0x000f5f222f613984,0x000e07b842f57781,0x000029d555fdb379}, {0x00085c8602f378f1,0x00078c23e6a3833d,0x000b5d0c4a1b72cb,0x000d5feb45473e34,0x0000db9aac6b4203}}, + {{0x0004b65361be7b6a,0x0002f2fb6b49ee6b,0x000913c64c9b2f8f,0x000d30a7c4338ec9,0x0000f69135ef57dd}, {0x000249afd7bdd274,0x000d7c213f9526e6,0x000247857a28573d,0x0005a9aaa2885378,0x0000cb194be632fd}}, + {{0x000c83d371b3f5e8,0x000cb5a067330a04,0x0001dc3a2fef33cc,0x000b187549f824ec,0x000091fb8cc00b56}, {0x0005b558729aa3fe,0x000dcc9c90c35486,0x000ebaa7ccca0c6a,0x00040c03f91930e5,0x0000d41e9d7170af}}, + {{0x000369c517263725,0x000f5b9c4d8f25e2,0x000af08d2fb833d3,0x000dfff4079aaa6e,0x00007b1dbc5eff3e}, {0x00095ef86114ccdf,0x000218447b952ff6,0x00028ac9591e2d52,0x00061a5f9065e821,0x00005b37299373e7}}, + {{0x0006de0bbde89cb9,0x000178894e0dfd48,0x00099f1f42306512,0x0005ae303a17d3a6,0x0000e98f7a864629}, {0x000854fd72db1c36,0x00018599b7d0f8ec,0x00067d94864ff03b,0x000a9a11c827deba,0x0000e7fc586d6b42}}, + {{0x0003905c60a37318,0x00020e672390fffd,0x000cc82bc8f9ef8c,0x0009dd202cb0dc17,0x00004b4757605370}, {0x000bd76d355ffa78,0x00062640c3b569ce,0x000d7be663352441,0x000fa5cc00063e7a,0x000041ec6b307eff}}, + {{0x0003445ff1d4f48d,0x00019ad186af8290,0x0003ec32546962ff,0x000c74bee7a65ffd,0x0000f3c5bfad6d70}, {0x0002af7a0d407d12,0x0003b49a55ee5914,0x000e67fde3d1641a,0x00017386bd678d4a,0x00006a7789b10f86}}, + {{0x000279671048d213,0x000fbf93af1c754a,0x0005ef03eed832a3,0x000166c4c086a7ec,0x00002ce472c668aa}, {0x00083e191dfa1fef,0x0001f9f62e555da2,0x00033e09245f9681,0x000834d7128c2232,0x00005980492cfaa4}}, + {{0x00005ed385c6f58d,0x0001e940322f855b,0x000abd1bdfe12344,0x000560254323ce5d,0x00003f3272b8e138}, {0x00091b1129ec9aef,0x000937dae4ab0926,0x0003b89972cd1064,0x000feb302abe5ff3,0x0000ac9ab092ccdb}}, + {{0x000ab8834fcbe7cb,0x00079a36285466b2,0x00021763ddf9573a,0x00087f23dd49e9d1,0x0000960a7dc34467}, {0x00024b2027d5087d,0x0008789865adccc4,0x0009d56dc06f79cb,0x0001e483a7a6349e,0x0000439526bf4c6c}}, + {{0x000850573c100bd9,0x000e92a154270a32,0x0002c03764947229,0x0001b3abcb8d85d4,0x0000c6b864e12fb5}, {0x00099a63a0948b0b,0x0003d11e7737521b,0x000f11ee54cc22bd,0x000c0f3897c28fe9,0x0000887a86105810}}, + {{0x000e64b613b901f0,0x00020b63d0cf3e5a,0x0000509aed92b157,0x000a4af71b397af8,0x00000f70a508ac7d}, {0x000443b477982ac4,0x0007bfcf80070119,0x0000ee5d52ca6ab5,0x0000fb76fea256a7,0x00001425f8a18955}}, + {{0x0002520e8f5fdd9a,0x0004415b44379a62,0x00013a5cfb245513,0x0007ff26da58b6c6,0x0000c62c15ae2094}, {0x000a325e4261f9e4,0x0006f038d32e2ef0,0x000ef3fc6abc70f6,0x000e30d93ea52afe,0x0000eebb6abb94e6}}, + {{0x0000247702ed7bce,0x00034f5464e44269,0x0009e8b128e5bb85,0x00080926518e624b,0x00002be136277a21}, {0x000471e16b34ec34,0x00012108d0ce4dbc,0x00009af5e6abb920,0x0002e754d17d1efb,0x00008a48cc1afe3e}}, + {{0x000e0422139c98e7,0x00020a9fb3a64484,0x000e10ede0bcc79d,0x0005a6dc2fa5ba99,0x0000ef0d52b0fc6c}, {0x00063d55b1d96541,0x0009f6d4c86785fe,0x0000eb9acfac3d5f,0x000572a8f2469101,0x0000dcb464d4523f}}, + {{0x0004f69a1b9f6cfb,0x000ce213f5056161,0x0008b7d48946501e,0x000149f12d547fe0,0x0000445a285c7f6a}, {0x000c4823f1a2ea23,0x0001385a5394c778,0x0001c1b9366acedc,0x000ab1c3ad0d81be,0x0000a11359b3675e}}, + {{0x000fad96e9cd5888,0x000b7a1e29d6289d,0x000c4b2381c4c5f0,0x000230ec1a276eae,0x0000dbfa685019cc}, {0x000f4922efdee4bc,0x00075cf61a1c6547,0x000edaf4fae86ec1,0x000804a8d53bbe31,0x0000fd5e6fb848cf}}, + {{0x000990f9e61819db,0x000a087bf284d04e,0x00046cc595c29ec5,0x000d285ade5df61f,0x00004427bdb3cd77}, {0x000aa52009715323,0x00064ee140eb5a6f,0x0007110a230db79a,0x0005beabf7fc498a,0x00005a15127fb48e}}, + {{0x0007b770b5f6a321,0x000f362ee591d72d,0x000bf83136cbb8cd,0x00084a2589241c8a,0x00004ee648e03895}, {0x000bb8cbdefa6e1f,0x00067c4e78df9a78,0x0002e73ca978976c,0x0001b28eef0426bd,0x0000de8fdba87229}}, + {{0x000066e356b5ba28,0x00042d6df6f58037,0x00011deaf6527ec0,0x000997dd3caa6655,0x0000dea9d72a1583}, {0x00071e5d50786fae,0x000acede2fe32d7f,0x0001c5ebbc758eac,0x00052ae508ead45d,0x0000105b61fc2cd2}}, + {{0x000a424820880205,0x000253ccc5077e5f,0x000f8c3fe20dc632,0x00062cc0c59b343a,0x0000155e23d52734}, {0x000ac65688011857,0x0001b16ae629218e,0x000d6756f09ff09a,0x0005fb7b7fd5fd37,0x000039a7040ebf80}}, + {{0x0003f9fc60cc69b0,0x000dc05b085eb973,0x0006e237d7c4ee9f,0x000c1aa988f662dd,0x00001d5d34fe8760}, {0x00031c971d36ad9a,0x00021af6097f9cda,0x000a112b99050a85,0x000760dd5b19501f,0x00004b8f2b2a3525}}, + {{0x00023daebb9c4933,0x00067f5f980754fd,0x00044b040f42405c,0x000cf1365bb83ea9,0x0000607651d34cdf}, {0x000cd8bdf6af7f24,0x0009f667d24992b6,0x000fdb2cba8a3b43,0x000216082779e2aa,0x000088707ba1c834}}, + {{0x000625373378cf7a,0x0006fbf97109acf9,0x0002a22d6e706614,0x0003b2b54dde5fc2,0x00007e3dc99d87cb}, {0x000e1cd88864bc14,0x00094f0b89418e8a,0x000a64fc194c02ee,0x0006b75b5f046f5a,0x0000bd2c8c6b65ac}}, + {{0x0002c8cf2c1c71b8,0x00034ba2d46ae80e,0x0005f7bfd38ee996,0x000b7c07f17b75fa,0x0000c9cda0385190}, {0x000ab3a305fb5758,0x000f9bedaa65e5b5,0x0002fd3ce5ea38ea,0x000693fb2d8b1c01,0x00000024b4599946}}, + {{0x000b3720ed0dfde1,0x000d03618777a3f3,0x0001f1f7a5cb0edf,0x000e6286b75f35df,0x0000463e61ef6663}, {0x000b207d0b79891f,0x000bc03f51f18bec,0x00045012b8dadaea,0x0000ea4f5238c4d9,0x0000e7374d260339}}, + {{0x000b9cdebf558d6e,0x000a7fe8cd19b8fc,0x000466552cced26d,0x0007ab4a23901802,0x00007cffa718628b}, {0x000bbebacd192e54,0x000631e137725179,0x0003765188e7a414,0x000a907dbaadfd47,0x00002b4be3e6d8da}}, + {{0x00094dd9e1077a92,0x000ff54da5dfa25d,0x000275244547d0a6,0x000af9942339905e,0x00001db13a32ed62}, {0x0009e8b2ff2ea6f0,0x000ccdb7c81f268b,0x000c0b9228169c5b,0x00091f33ef28ab69,0x000063fd42c2ea96}}, + {{0x00026bdfdc8fe7b7,0x00006cf1c07e97d3,0x000cadbd9a6a9283,0x0002d6bf51c2ba05,0x0000b8cb99a356d3}, {0x000ae7bb14928177,0x000b0e246da65b16,0x0004b026ae4c5b8e,0x000f61a66781bfd3,0x0000dbdfa283ad1e}}, + {{0x00056863c55d6206,0x0008826bbed476aa,0x0000f81fe2cd68ff,0x00028ab98204a548,0x0000057156e622a3}, {0x0005fd89b91e90fe,0x0003deeafbbf9bdc,0x00098a9587ab8014,0x0002738fcf98e3f5,0x0000b981ec89c6f5}}, + {{0x000b316827a46307,0x000a300d18683649,0x000f2fa071726038,0x000d3794466aabaa,0x00003b21867de806}, {0x000d2944b3ec91e9,0x000fdf4baecce9b4,0x000c417b42945b46,0x000aa79fdb7f35bb,0x00003a9f897a4e64}}, + {{0x0005a34d2eebeba1,0x000e6afd5bc4437f,0x000b456c7dd1b3d2,0x000b4153b76695e9,0x00003307619aebbc}, {0x000956bb3e38fdd6,0x0000426ed87648e5,0x0005f711ecd8c046,0x000079d6d63de859,0x0000dbf4b9f107da}}, + {{0x0007e175ead15d39,0x000561b4f3f83c33,0x00011b177a1402ec,0x000a58752be2a08d,0x00000da4d4e2fe92}, {0x000e6e8276401658,0x00032d6c4199ae4a,0x0001faba2e498fdd,0x00014b1af4332ef3,0x00005ba35a81c223}}, + {{0x000649050769b34f,0x00083bfbb56c5761,0x00021ee2541366b4,0x0008704cdec9bb64,0x0000d7945159c0d3}, {0x0008bafaf13e0aea,0x00036e13e82d5eb7,0x000339761f9d8d47,0x0009eca4591e67dd,0x0000d96c07568a95}}, + {{0x0003d5fd2029df76,0x000a5fb33c0e0f3c,0x0001ff5fc68759f4,0x00051f621da14b63,0x00003caa71b11ccb}, {0x0007c30a7e617c2a,0x0009b711ffee663b,0x000759b3ba8a1266,0x0007f0dc13775b68,0x00007e1383f3a56c}}, + {{0x0002e53bb23c1e0e,0x00092de4f81808c0,0x000027ce8893cbad,0x0000f3ec1a375761,0x000021cb492d2c4f}, {0x000aa3245f82dbb4,0x00069cced42a6e8c,0x000b6d15012948dc,0x000fb237260c2eac,0x0000ea4c9c423931}}, + {{0x000296fd70720795,0x000229b349f0d757,0x000372c7ecdb2d87,0x0000ab683986a30a,0x00008eebfefdf917}, {0x0007b20676f23d23,0x0001b68216600cd4,0x000d60209e8a6542,0x00056e7d5a86ec0d,0x00000dcc35fe04cd}}, + {{0x0003bd54c9a586b6,0x000b8854c7485039,0x0005a40f4ec5f1bd,0x000289a4e41a1f76,0x000049dafe50ee7f}, {0x000fd6e34d712730,0x00053fcc6922c67f,0x0006083c963c85d5,0x000ce0b410b01af9,0x0000bcb42a6ffe90}}, + {{0x000b767ca282a4d0,0x00020bd980476c87,0x000bfbe1aabbbaaf,0x000b88e6d7f7cc73,0x0000b94422c7d82d}, {0x0008f139b5e18b15,0x000af37294afe4be,0x00004cff3264b3dc,0x000c73112544433a,0x00005a6007937b5b}}, + {{0x0004af8e5428d95b,0x00010191ac710f35,0x0007cbedd6b7a565,0x000b814d7e43f856,0x00001c5e2eb7d556}, {0x000fcff49a2a7cf6,0x00072f558310e77c,0x00038dd2bc421df3,0x000a70595301fb38,0x000070f7e2a9b2d2}}, + {{0x00078d66e9dbffa4,0x000fd8c7ca048a86,0x0007e3eda5c2456e,0x000dfc3b87b2b93c,0x00008309da1e8c6d}, {0x000d753512b919a2,0x000af02b49c6907b,0x00097e34d80b0b22,0x000116caf4066684,0x0000a329ce0d5c8e}}, + {{0x00018e79cbc3d261,0x0005c013dcb21798,0x0002b229899226c7,0x000ec7a427d40e6e,0x00009be63696ee9d}, {0x0002ca736795eca8,0x000ab6fbe78de343,0x000bd2519b13a095,0x000f556bca40cafe,0x0000db7209b1f081}}, + {{0x00073c09004f67ac,0x000b29108a8f67d1,0x000d371b9f9fcb0c,0x0007a067e6fb34ba,0x00007b352a5eff65}, {0x0004bcafc8686e3b,0x0009f001e6a411ec,0x000e8fb4225c8102,0x0006a982156a2ca9,0x0000d80803a38166}}, + {{0x000da2e4dabb9727,0x000355d6f6b308ba,0x000f1255aae1bef5,0x0001611794716339,0x00007b4e604aae5b}, {0x000972e788159151,0x000fa3d326703740,0x00098bc1191d5969,0x00078a0d4a378bf7,0x00006a59fa849d24}}, + {{0x0000fa58dc098923,0x000a1856964a0842,0x0006b9932f4e037c,0x000814ca2cabcd97,0x000078dcf9cd7aba}, {0x000a197dc3eebe13,0x00042e556e9988d9,0x000157e412bf69e5,0x0009df4f48dcf986,0x00003aded6912caa}}, + {{0x00084e2f15edb89d,0x000ca9cc6e956412,0x00012680b1917ab9,0x0006783f51439caf,0x00005d29d2d26dd3}, {0x000ab0638623d5b5,0x000fed329f9f8df8,0x000792dc163a5fd0,0x000655955e82993f,0x000069608a1e7223}}, + {{0x00050665c639e317,0x0007cd3a0be23b31,0x000d42c79903f0b3,0x0002c444001375bb,0x0000929d6d962097}, {0x000772643d922771,0x000edf18c8218c66,0x000ac87392e06af9,0x0002719c1204c4eb,0x0000d70ae05eaaac}}, + {{0x000b98d73f6132c3,0x0002d06ff3aa0dea,0x000114d3850749fa,0x0007044ffa876d9b,0x0000f45872e2fb66}, {0x00016ce3d2f5b6fd,0x0001dbce4a5c1e39,0x00023420be2346fc,0x000ad2073526e3aa,0x0000ec42bf82e3b4}}, + {{0x000a0e0400250d20,0x00073477185ee8db,0x00076add835f002a,0x000b3b5382a489fd,0x0000bee501d59ba8}, {0x000f808fc2a3269f,0x0005a1a324483855,0x0004b0561e253e07,0x0006877e92305edd,0x0000fe56709ccee3}}, + {{0x000c69fd4d6e9ddc,0x0001efe17ab5ebee,0x000e8772fcc9e90c,0x00075476218c92d3,0x000034d8fe876b89}, {0x0009a53714b3d5de,0x000392d52e8cffab,0x000f9a84a7982c75,0x000969339421caf1,0x0000be235f473bd7}}, + {{0x0004db392168c345,0x000a7d3a892289ad,0x0001c36832b48c85,0x000ed9a7ee4e9c5f,0x0000c04e75e82317}, {0x0001be0108fab1b2,0x000295b25288526c,0x000465a6858de779,0x00095b9a8e5cb36a,0x000089a16a0405f4}}, + {{0x0004dfd0a2e212af,0x00036b8548f44a8d,0x0000661b76948703,0x0005acf11a3d8f18,0x00001c00c194d622}, {0x000d5c1f47b27588,0x0009b03b8c767f30,0x000cb8a097b7b205,0x000c759ac3acc6de,0x0000805e35066db9}}, +}, +{ +/* digit=28 [{1,2,3,..,}]*([2^196]*G) */ + {{0x000c38a2a1d9a703,0x00081b56c40e7912,0x000dd8e69ee22588,0x0002cd3934da3f86,0x00009ea1c46f7d16}, {0x000bbb8bb266d6f5,0x0008f24907c529d6,0x000f29a7ebd75276,0x0002d5547c914e08,0x00001c5ef3df3449}}, + {{0x000a76cc5116d7af,0x000eadafba1a14c1,0x00024709cc081b48,0x00081c7e2a466abb,0x00008b57f0616a3e}, {0x000f956ec6c161e4,0x00000fb95d4f3aeb,0x00097cb468593666,0x000bf252520b3955,0x0000ed97cbc6ae5f}}, + {{0x00086b91a9173051,0x0002f80999c69a5e,0x000310070a4efd46,0x000f45d8957ef5bd,0x00007ce05888474d}, {0x000f95d561b01a26,0x0009d0c4e3c33152,0x0006abee75a4662f,0x00001c4bce657471,0x00009c915c04da63}}, + {{0x0003fced8aa40761,0x000b87e4ac77d3d7,0x0005ac612e5b0c4c,0x000de0d7711889b3,0x0000d21f2a009398}, {0x00066281b56378cd,0x00067c47fa3c2881,0x000dd18a00054474,0x00087c02c8db20bd,0x0000cff165d7702b}}, + {{0x000b9efc0d760cf6,0x00050c278d0980c9,0x00007feafd3862e1,0x000aa3eb68afab38,0x0000647da2895fe4}, {0x000f70e5bbb84fbf,0x000d1eef05b4940e,0x0004b7c6fbbfd8cf,0x000db93150f3a4b6,0x0000b3e4f8d0efd5}}, + {{0x0005488239f8e105,0x00014cfadc9e08bb,0x0008dc975fdc87d6,0x000be3c9a3d0798e,0x000016550e4b2cc2}, {0x00023a4e3abd5d4c,0x000b906da2001e44,0x000060bc93fedeb4,0x000093dc27fd9104,0x00004cd7ca88713b}}, + {{0x000fbf542fa80e44,0x0005f10928bc0a3f,0x0006293cef2e8b64,0x000a898509b32c4e,0x0000b22619841e62}, {0x0007f5cec0217b2b,0x0001d20d8159ff53,0x0004247eb95b2102,0x000f96a510270434,0x000007a5eaff6c63}}, + {{0x0004d031f8bb00dc,0x000b5e02bac4ddc4,0x0000aed50c213344,0x000e05b337dca4b1,0x0000c86350c6fcad}, {0x000a47ddf9ce7cad,0x000feacff71799c2,0x000bb0df8204bce3,0x000e5f8261e40571,0x0000e0c599f1adf0}}, + {{0x0001b1bb94f507fc,0x000b0e3b80fd0544,0x000d3472c146d60f,0x00084fae8223732d,0x0000101a1e7f36e2}, {0x000e5e9e34bd1816,0x000f545cc7b24119,0x000d9ddc782a4857,0x0003d7ca719e13d3,0x0000d9522fd316f2}}, + {{0x0007bed302860d5b,0x000b5abd9969e2fc,0x0005b5076fa90a70,0x000ea613dd5e38e8,0x0000f8c38f07a470}, {0x0003fd5e928fd844,0x0008d2c777f94dbe,0x000e5e8ee0421bb0,0x00090b66f1472441,0x0000b3088fcb535a}}, + {{0x000e1ae7289c2895,0x00025afe82ac1535,0x000bdff2cd902878,0x000f4ef1b8d241de,0x00004fb71fcc98fa}, {0x0004222f68f44d53,0x0002c233b26fdb58,0x00075ebee32eda92,0x000808e057bccbe0,0x0000e4465cfeab5b}}, + {{0x000dfdcd833d339a,0x00020e7a389e4f2b,0x000d23683025a3a7,0x00035da3027d310d,0x0000fdf534e8caad}, {0x000c3671c7931bc6,0x000295770ef2045c,0x0000c060bf62b8ca,0x0000be1c0672ffae,0x00003930a309702d}}, + {{0x0001923fdb3e29eb,0x0001b817d72ab678,0x000b46b1a5f7f642,0x00070769c8c3e584,0x0000d0238e461de5}, {0x000c2ab9f826b572,0x000076be5f5577c6,0x00026baf39420c61,0x000bfa2c745c1765,0x00008bee5c5cf98c}}, + {{0x000b34e1ffbd4f2a,0x000747c36e6384e6,0x000188caf2580fc4,0x000021a1cc113e97,0x0000166f801bb0a2}, {0x000ee689dc2b83c4,0x000fc7170038b4d4,0x00020d913b44f23a,0x0006f4916eaa0f95,0x0000a37abc373fb5}}, + {{0x0008e478edbc9edb,0x000b6cdbf59f53be,0x0007bc01ef6645bc,0x00030d7b078b0e6d,0x000048412e9378f1}, {0x0002f15c532138a7,0x000dc14eed557fb9,0x000c77fae6fb3f15,0x0001ec558fe1463a,0x0000f318b95338a1}}, + {{0x000b7438e5468e71,0x000caedffb29bb1f,0x0002ff19da1a8652,0x000de068d72097f2,0x00007621c88c8098}, {0x00078d98bb33eb05,0x000422f044eced99,0x000e8c4b19e1266e,0x000800261d9ad034,0x00000d3bbc0defd4}}, + {{0x000b6e60f54419ae,0x000b6eca8a0201c6,0x0007bd20f02e3c59,0x000f848a270d2b43,0x00006c057044b677}, {0x00047dc47384cc81,0x0003e307e3f7012d,0x000f9a650ee8e3f4,0x0009500539cdb1a6,0x000072f13befc47a}}, + {{0x0007e2ac62f29ecc,0x000c7076b6aa4df4,0x0008c07b37559318,0x00062001974448d9,0x00008a44c6309b81}, {0x000e2de6bcf14f8a,0x000f4990ab063f22,0x0004697a9bbec1c2,0x00030cecea565c48,0x0000fd3da1c5e5d8}}, + {{0x000aabb0b18a6f6d,0x000e3c002fa1b7dc,0x000818bf3e956799,0x000cb0a09f9d8ac6,0x00004c37af29089f}, {0x000975ede76d2691,0x0000edc449b75047,0x00054acd30477815,0x000fff298f1059ac,0x00007ac93c57a2ee}}, + {{0x000b8d64baf52f42,0x000b09a3292f23a0,0x000456c047f3f84b,0x000fc353ae21c75e,0x0000110967ba5ea7}, {0x000db604a4420675,0x0000bc0332e07024,0x000f18635c4ebf20,0x000ecf2e0f788c83,0x000050d2e58fb836}}, + {{0x000c9678e5433b08,0x00070837189e593c,0x000f524cd351331b,0x000d2130907177b5,0x000027851eca7500}, {0x0004709b248eca72,0x000a87322fbb36f3,0x0009e73d0e54945e,0x00023d52f940920a,0x00005827d8bfa825}}, + {{0x000b019bac0eecc0,0x000cc707f8790281,0x0004e858045722f8,0x000538e7e32b5f38,0x0000c291610a5aee}, {0x000c51ee868e43d1,0x0000b754959a25fb,0x000f424af9289eaf,0x00069ecaf0bb54a6,0x0000923b39262f9a}}, + {{0x000eb037fabec5a2,0x000f5113b5f773df,0x00015a54ebc1d2d4,0x000181f233f0f51b,0x0000fe807bffeb66}, {0x00046d05b62ce294,0x00098d67cf9c5ed1,0x000a8fc4e5cef5ef,0x000aadad520b5c8e,0x00007039215094ef}}, + {{0x000a1db4a22af952,0x000cb9edece623c4,0x000366394458ac70,0x0001dfb057f7a0ae,0x00001fe7a1fbcf33}, {0x000d7cf370fe2a14,0x0005732e78fb4925,0x00055e87c37f393a,0x00011bffd64640e5,0x0000ec4a5ddbe8c0}}, + {{0x0004d4d3a17880f7,0x000daa8f972e8527,0x000d1a6e2ec1bea4,0x000c27636300963b,0x0000efe1ddeeeed6}, {0x0004f3e8c69f40b8,0x0000b1029582dbbf,0x000af32882bd8807,0x0000a6c9827573a8,0x00007105abfc154b}}, + {{0x0002a5c62b9097eb,0x0008d4989c3a69bd,0x000c39b3112f1d05,0x0000d73bfed8eb5d,0x0000a4dd7028190d}, {0x000df21e7c9ea92c,0x000cb9a2427202b9,0x00055d8cf2e18cf8,0x00000d9225aa33bc,0x000093d58b32465e}}, + {{0x00056d7e3c7e81fc,0x00063f455beb34d8,0x00041bac0617ff16,0x000d390004aab0d3,0x000066a594a7db36}, {0x000674ef21314c13,0x000cd499799bff44,0x000674e5b127989b,0x0006f3d0eac3e533,0x0000475033eb386f}}, + {{0x0003be03f488b066,0x000afe3e2458f80e,0x000cbb0ca9c1df72,0x00032d854724a756,0x0000f294d9ced004}, {0x00099d4fed7a8bf1,0x0000a3d4dc031bbf,0x00098fe4b4fa6535,0x000d3d62a156b316,0x000061c892ad39b5}}, + {{0x00057062dd51b0e3,0x000fb0b6ede2e44a,0x0009b84b4bb2e66b,0x0003892b535e65bf,0x0000f04bbbd4cfae}, {0x000e254b0efecf74,0x000c7b425efa6f99,0x0006fc1223cb5014,0x000ed3775dda3b47,0x00007d91e4176c7b}}, + {{0x0001e11238f17189,0x00055477448462ca,0x00076d1ca907db73,0x000957e211b110df,0x00009f00d6031218}, {0x0008148bbf09b400,0x00081c3bb9e89771,0x0008b25f9e1f66b9,0x0003658ee958d873,0x00008ea2bc754377}}, + {{0x00040ec2f8b18d2b,0x0005720588c7874c,0x00012febf68a4ccf,0x0000ad74cc5bc416,0x0000c0cd6c894bb8}, {0x000a5770732df031,0x0009df7b0dec4b86,0x000bf3c40616b170,0x000ad41d43d29a1c,0x0000eb2874f42570}}, + {{0x00075fff53df8b53,0x00080ad93ea9fdeb,0x00094201518d77bf,0x0007e5c4c968a760,0x0000623567d0a637}, {0x000b5f86a4738b2d,0x000027189c03c512,0x00071bad6653470f,0x0007c87146d1d64f,0x0000a1a59a4469af}}, + {{0x000b514e53719c74,0x0004263a91d246bf,0x00026ec908be45b4,0x00047bf0842913d7,0x0000b483a7ba011e}, {0x000411dadfaf9c39,0x0005f75f1a475043,0x0000622a4242b2a6,0x00076430694e65b7,0x0000cb0de0487bd5}}, + {{0x000a48d97cccb4f8,0x000039d242ee958b,0x0007a2d265fc7e92,0x0004f9b0346ee66d,0x0000ae6d9eccb1ec}, {0x00071cb876b50704,0x000b3a208b80528c,0x000e466e83bea7e3,0x000f09d38293ce87,0x0000f7bc81782076}}, + {{0x000cc15e3a833811,0x00027d31aa50385b,0x000d0e82633ec551,0x000ec7922987876d,0x000070d03a64f875}, {0x000cfaf5650d15f6,0x0003473ac211140c,0x000c98877e0f73f0,0x000c08a75c5c482d,0x0000622d09e823c7}}, + {{0x0000b7e8cbf9ba79,0x000cd6e710efc7c9,0x00077ce47b6bd4ab,0x000960be8ff09b54,0x00004e8f8dbc6cc5}, {0x00021368a1283c7b,0x000f518b32eb7e5c,0x000cbf5b162623fa,0x000005dbef4d2e43,0x000088b15fa01ecb}}, + {{0x0000073cbb4b6ec4,0x000b0a7babe5c5c5,0x000fe549f4ecc335,0x0008c322d1e8f812,0x0000c49af93d1581}, {0x0002b2e39f0417cf,0x0003d9808a14c3e1,0x00017386e20adee0,0x000690f7522d1fa4,0x0000be41fb5f632d}}, + {{0x000d671707701933,0x000297bc3160d8f6,0x000cd76c64b8a53c,0x000dd64e0e6d9406,0x000034d74fce765d}, {0x000daf8269a471c5,0x00075ff07d0520f0,0x0007f32fb264506f,0x000b4a0a95e8f527,0x00009abfb62dfbd9}}, + {{0x00005b5fd8e98e12,0x000e14ad70cc06de,0x0008fa21ed10a09e,0x000d8d84fd47afe8,0x0000d6f824d23bd3}, {0x000baae0dd9c557e,0x000ad65982a3dbde,0x00009c77a82ec43d,0x0005bf745e0fa843,0x00003f0eeb6c0fc7}}, + {{0x00088e490854a78d,0x00018930eadc94c0,0x0006910aa320ffac,0x0000a278a57e2f16,0x000051dd5b2d1423}, {0x00005547cd73a6e7,0x000cfd1d02fc869d,0x000428efc74b6a1e,0x0001e7c08394445e,0x00004579b6ee7a03}}, + {{0x00024c9bcf01037b,0x0009474916aabed7,0x0000d51d09748fda,0x000cd6db5cd430b8,0x0000dfdeb4fdd00d}, {0x000893bdeed91a80,0x000f120778a2597c,0x000058a2e837a8e0,0x000cbf18735f2428,0x000099a77bb6fe13}}, + {{0x000ab7d29b0fc1d2,0x0000c2ff82ad4240,0x0003e8b1824be74b,0x000b66033eda9291,0x0000639f82a1d05b}, {0x0000f2c91ad5de88,0x0006ee9ddf6690a1,0x0005fc7fbf5d854f,0x00072ac4ded94a95,0x00002682d9b714a2}}, + {{0x0001a874ed7acebe,0x00029408c1b46ebe,0x00096d405f800d1d,0x000d48657b969e8f,0x000025248132720c}, {0x000113432ff0aab9,0x000ed7506e97a2de,0x000f3ff61e19447d,0x0008868795e67865,0x0000dd308ae491e6}}, + {{0x0000c4982db65ca2,0x0007cd8b3cfa5d15,0x0008b3c34d779cee,0x00007ce56f66df9b,0x00002bad9190373e}, {0x0007dacf82c7cd70,0x000170cd5e21faae,0x000d737aa745c2ed,0x0003683c666088dd,0x00002a7ff3c66133}}, + {{0x000a25b51b2364ab,0x000cca23ad8f56c0,0x000cc5060f4037d8,0x0002d7b40bd50e15,0x00003c6b05849780}, {0x000dfb5580a54f98,0x00017736f3f4da3c,0x000fd025f36ffc28,0x0008e648f7012618,0x0000d358c70b39f6}}, + {{0x00058b4c855a6e77,0x00057c0c522b55f1,0x000aab00625e8fee,0x000e46856ddaf929,0x0000358453361f0a}, {0x0009303af2d55212,0x0002aa5a6f6c0d2d,0x000148546bae13b5,0x0008b24d4e4ab630,0x0000d7472f84840d}}, + {{0x00053970acdb5e8f,0x0007fc269b0f06da,0x000ed8404fc5b791,0x0007e86e017875d8,0x0000ae584763a508}, {0x0008f45064b6f293,0x00033c845a65aaf0,0x0002b4eb7bd3ce36,0x000130fa6c359a38,0x0000a8bd9f84a685}}, + {{0x0008edd0663f8f91,0x000dfd59df5ea31a,0x000b723f079a56bf,0x0009327b892c2263,0x0000ca95b5e3d558}, {0x000ff18c4e8a781b,0x0001e49ef80c48df,0x000240760a2bb746,0x0007c06223d5cf77,0x000078f428a9e459}}, + {{0x0007912ca1754a1b,0x00092318062eb864,0x000974e01c1c5675,0x00077e15df7992f8,0x0000a5205ad4cc39}, {0x0007e259fb6cbd53,0x00069eb726a354b4,0x000e567fe8f54419,0x0009e27f6dd9bb81,0x00001e04658e6f27}}, + {{0x0003571b3d512de4,0x0003b7ec1208dbee,0x00021dbe93f6b7c2,0x000e0305001dba00,0x00002cde309eec3b}, {0x0000e38dd838d4f1,0x0005ed3f3ec2efea,0x000695c674c80611,0x000b53a586d725c7,0x000023cf980c45f2}}, + {{0x000b294c00e93609,0x00036582b06d7012,0x0002b6731864a65d,0x00038b0faad9a9f5,0x0000f77e45dc3d84}, {0x00049e1125dd970b,0x0006521b5e3f5cef,0x00026ca225bd9b00,0x000e53d27026153a,0x0000ffc6effc2de9}}, + {{0x00049dac9eda55e0,0x0008ed551742ae1e,0x000db16b7071ad42,0x0007e519f35180de,0x0000acf03adba5a9}, {0x000674fe200f7515,0x000041ad6226b24d,0x000bc185532514d9,0x000f57098d181f26,0x00003a47ab5d1829}}, + {{0x00065694a7f9b78f,0x000f181f805c903a,0x000293f8c82ead90,0x000dc417c53de726,0x00002878d513c8f0}, {0x0005fb44bef65dc1,0x000c159a08f75e63,0x000e766741f4f5d0,0x000925d04127a52d,0x00000eabd36a8ede}}, + {{0x000b38bd3f9e6f35,0x000a429a8af3f5ad,0x000b3d4c94dd8b95,0x0005081f01c01e1e,0x0000a43459cea12f}, {0x000266c93a596f15,0x000fbb27f18ef36e,0x0005dc2d06428594,0x000875dbb1d59fd7,0x0000123eea204f7f}}, + {{0x000b96fae7ffdf8b,0x0002c722fe26e092,0x0005b25bf727e271,0x00079b4b3ece9d22,0x000029e084fabe5d}, {0x000cd0f3353903c6,0x0000436a938bfe7c,0x00097acfe3f67b99,0x000bce9cd2d1a3c7,0x00006253f0aaad9f}}, + {{0x00044bbce9995733,0x000f073624473dd5,0x00033abee446da98,0x000978d07a005203,0x0000caeaa7309d7f}, {0x000150d1bf2bcd41,0x000ccb53f84c0b65,0x000ac6215464c51e,0x000609e277528a4e,0x000098cc59ce3166}}, + {{0x000efcad4a7da4d5,0x0008301bbee31186,0x000b3db7feb894f7,0x000f1f8ace71e7f1,0x0000112c9354a5eb}, {0x0003b627286dce6e,0x00095a30713afd8d,0x0001c6f0a0610827,0x000d93fc4c531160,0x0000964d3161c3a4}}, + {{0x000ed7017f788ea0,0x00094674852856bf,0x000c96919c78850c,0x000d11c15e8987ef,0x00008ec3fd406441}, {0x0001d9a8b2134698,0x000beec4889b6406,0x0003f1ad5d6f0bd8,0x000240648f309015,0x0000e5b581f2cb82}}, + {{0x00027926c5189337,0x0002741b168b71e7,0x000a4a599fdcd605,0x0007cba7afaa4cd6,0x00002940e9635844}, {0x000a7689dae16a8a,0x000dba4b641f4ffb,0x0002ff98dda84226,0x000dd3217fd968b2,0x0000584b3c0e2f5c}}, + {{0x00031b6de16fdb99,0x0008582a440f2355,0x0001c6e1855a3304,0x0005300eec11c310,0x0000d21585a61848}, {0x00011b3ef7a3b94e,0x000505e21369d2ce,0x0004b130e8bfbb57,0x000a18b7f8363266,0x000093672301cecb}}, + {{0x00006d08ad2b8a84,0x000174564b2255b8,0x00072579bbdec57c,0x0009ce6932d9aefc,0x00002be0f18664b2}, {0x000379820c03f35b,0x000b81cc51d23ac4,0x0003b05894b88c59,0x0003d5999a808624,0x00005debcd1e1590}}, + {{0x00063183e156632d,0x00095cc68bbb24c7,0x0001bdedfb33c7bb,0x000fc8798ec08681,0x00004d45061bb8c2}, {0x00096921fd5268a1,0x0006fb5a73e1eaff,0x000c44fe9ef420da,0x0002d2e7d1a2a7ff,0x000099f6ec68c8bf}}, + {{0x000b091537d50e00,0x0004c4d8240e0105,0x00088289db80bb3d,0x000f2516e62e6a1d,0x000049b3d353af60}, {0x0006c1e8a9308ea7,0x000a834375ce9c5d,0x000bb4bf8bdb7520,0x0000568aac390a23,0x00000b0ddd236d48}}, + {{0x00072da34e3add6e,0x000a25b562a221be,0x000f893f7463701f,0x0001b78a85dc4a41,0x0000b8945c02e7af}, {0x0004515392a4b278,0x0007224cf9eb4248,0x000a36d27223dc97,0x0000469b1c26a8fb,0x0000c8a286157f29}}, +}, +{ +/* digit=29 [{1,2,3,..,}]*([2^203]*G) */ + {{0x0003d44e194a9fdf,0x0002786ca99f031c,0x000ad1c1ca8706b5,0x00016f6144a3f729,0x00007cd191d19930}, {0x000032e9850c5a09,0x00064ac49359a6c2,0x000556cf735b82bf,0x000ee923f58e8bf0,0x00004134d7aa14f9}}, + {{0x000d9bb86ddaba10,0x000af2ff8d7f7ad8,0x000598bba183ded6,0x0009e01bf6f62c72,0x0000b0e9cfa9acd8}, {0x000fa485b832c346,0x000511c3faaf2d1e,0x000b424d4c7077aa,0x000d2f6f4b824a1c,0x0000e6cb10d909b1}}, + {{0x0002f3ece47545bf,0x000cc82281bab5ca,0x000fef90fb8bf71c,0x000bee7b6c1c7445,0x00009f6c799f8590}, {0x000043420d1a0c68,0x00050384ca2526ca,0x0006d804758be80b,0x0004f8d0335e99b1,0x00007f6cdd65f76d}}, + {{0x0009dee69ba8813b,0x000aa6bce2ade252,0x000b5e21e2f41814,0x000311e74086449d,0x0000fd363bf4202e}, {0x000eef53cd037a91,0x000f421162d3c6c0,0x000c1bfd296d4eed,0x0006d0ff1ba38cc4,0x00007b7b0d0ae516}}, + {{0x000118e549825790,0x00087608d2b36a6a,0x000db2f0bc20e5a2,0x000100d9275dfc96,0x0000fde447ee7b6c}, {0x00028286516d65bb,0x00018776f794bb5d,0x00073d44dfb1675b,0x0007d7116902d37d,0x0000b303e3549334}}, + {{0x0002b20e7355edc0,0x00029cd368ead897,0x0008b57463d27897,0x0009c3165bfa78bc,0x00002fe2446e3da6}, {0x000b89e933220b48,0x000cf81af9f1c16e,0x000c693a260f9ee9,0x000a0fd40f000040,0x000052f4665ea9b0}}, + {{0x000e8f27a45c0327,0x000e86dd8982d9c4,0x000e56fb7618c159,0x00082ae92801f788,0x0000cb0bbd114f00}, {0x0002622a7b7bdd1f,0x0001cde8a1787ac9,0x00079d3f0007ae73,0x0002f8c21efea529,0x000059193889e5c5}}, + {{0x000d6baee189bb56,0x000e053f037f48a7,0x000fad7013164603,0x00005fdb161acf55,0x0000e17a5d95e63f}, {0x00019ffe09ff7228,0x0008cace368cf35f,0x00091ae861843311,0x0004e6206e370ead,0x0000ddd9c9e96ad8}}, + {{0x000c45cd3f314ad9,0x000d67364744703a,0x000b4a453c3af51f,0x000b9543050e4e10,0x0000fc178295f043}, {0x0007775ac574c8d3,0x000779f0b9acb4d8,0x0005b7ce50c8b03d,0x000b17132e5af1fb,0x0000c9a4f064d508}}, + {{0x000f522e5ae3e277,0x00035c35e812e660,0x00042cb0bdb0d2f5,0x00094342c735bbee,0x0000907ea8901582}, {0x000ec0462e80ee09,0x0004686fb3008f47,0x00006f827e223116,0x0009550860a2ce31,0x00000a75951e7b8d}}, + {{0x000f70c1ae342323,0x00018ad9615f80b9,0x000cdb22b183eae2,0x0007f44fe13030aa,0x0000096480bb3134}, {0x00051223b16a6387,0x0009942d6ff78220,0x0007ab9f39c0104a,0x000222d1b2b20f04,0x0000e09b38b19739}}, + {{0x00015cdd08fa5ad2,0x0009abe3f073baa9,0x000000d8e4981763,0x000177db88ef6f94,0x00003eaac824bcb0}, {0x000eec6acf802868,0x000ce8144efca222,0x000da7040d7d51ac,0x000194f0b2b6495c,0x0000cabd0cde140b}}, + {{0x00017df57e74652b,0x000f4607f76a84d1,0x000a479ff1f3d544,0x00007f04801c4412,0x0000698c2293de3c}, {0x0007faa80f480970,0x000345e6a405407c,0x000b9c66fdd33168,0x000aa0cb4a878f56,0x0000967b41c670b8}}, + {{0x000e336c7c6f0782,0x00096d368d37636b,0x0007cb94d59ddda6,0x000066012877b72b,0x00001a597b4c1bdb}, {0x0001a77f61e01063,0x000941d2f7f6a765,0x000adc00b4c447fc,0x00091ec195fdceab,0x0000225175f8fbf6}}, + {{0x00092cdef9b9f47a,0x0006904c0879edbb,0x0006347c48b0eaac,0x000a434c1967d357,0x0000a127c7caa1a9}, {0x0008363c59d19935,0x0001d39d521da939,0x0003198ee98fb3d9,0x00029ecfd6c18cdc,0x0000c77256b8e3c5}}, + {{0x0004c3f4f7c6da11,0x00061ad5689997f7,0x00062877229a2a25,0x0007088588d5eba7,0x000058d5cc44dfa4}, {0x00035cc9a95c4417,0x0004bc289fdf6e40,0x00019b3a507f5383,0x000ce233b96c0c50,0x00000c852a997bcf}}, + {{0x00074ab6e99c27db,0x000700add209d2de,0x0003682485035083,0x0001df8625e5b7c7,0x00000b17bc10dbf5}, {0x000a3800fdb2a3d9,0x000dcddadf173751,0x00052591af594942,0x0009ba307d31c514,0x00002993a31f60d9}}, + {{0x000f10830427f20c,0x000f3b328a7fe500,0x0008f3641b8a00c1,0x000ddb433bdf6377,0x00009d207a3e0f16}, {0x000ad2c986b8e3d4,0x0000a50419f77145,0x0002215e5dba85f9,0x00036043616de9a7,0x0000890c85022c4d}}, + {{0x000c79fb017c6831,0x0007a3ac08d0d0c0,0x000029dfa2a4a427,0x000ad7d78feae29b,0x00008d9adfa5f678}, {0x0009c5a65d795f42,0x000b9846b9bb1ee6,0x00058ffe0f45ce60,0x000e91a8c1a88f3e,0x0000be10bcc58190}}, + {{0x0005faab9590cb5e,0x0001342fabcea57b,0x0003e7b2710e9804,0x0002cc1450419041,0x000074d1a8f02914}, {0x00094b897325bd19,0x00008ae2d222f310,0x000d2bd77ff05aef,0x000980085cf5bd1e,0x0000b0fe23af35c4}}, + {{0x0004fcb8b197112f,0x000f08c2d9a81289,0x000d49a846e64b4e,0x0007aa5f20e1a8fa,0x00007cbd1a7ba201}, {0x00064a4a4bc89168,0x0001fad822dd3cda,0x0005d3c95cf37a38,0x00042789c1ac2c12,0x0000ff1fe3c1d92e}}, + {{0x00092a5b1d033025,0x000fa30de8fdc44d,0x000a4e718fcef607,0x0004fbc816269dfb,0x0000bf7617c683f1}, {0x000de6a2dba301dc,0x000825272ede6fb6,0x000b46c3e8670aae,0x000f2ae811ace7dc,0x000059fa6d27c8ae}}, + {{0x0002c9a0ef6d99fa,0x0009458f7d2ffabd,0x000a81b1810c0ec1,0x0000355e7c18df1c,0x0000c49e954b5556}, {0x0006c6fcf3518d17,0x000b3430cd0f57ce,0x000ed76dc6ca44be,0x000fa9be95d0a2cb,0x0000228db9607d58}}, + {{0x00029d5eb1852497,0x000c231b03be4c4d,0x000b6be41e03aa73,0x0000c8bd817ca00c,0x000045736b555b18}, {0x0002467e0d2667c6,0x000112c70e0b53ce,0x000ce135f6da78ac,0x00097cce5c18f0ce,0x0000e522cc53caf7}}, + {{0x0002eccb8b0b2cc8,0x0001b5ccf40f2f60,0x0004aebd14979660,0x000e7c210c1a93ee,0x00006c082ccfbf4d}, {0x0005a29cf6241ec3,0x00051c57365898d3,0x000c5cea4126aa24,0x000fb5f29b7ff954,0x000035d47dc88418}}, + {{0x000e379fec34dd00,0x000af10f04b2da8c,0x000244ba7b67c67d,0x000b291e6c1fc84c,0x000064b8ffb56a4b}, {0x000f0c726a01b425,0x0000ab4e63c20a03,0x000daa4bfc15dee4,0x000b2ba598f42985,0x00000412dc5ee9a6}}, + {{0x0008e2b0cbea7f60,0x0008aa38f3dd0462,0x000bbefab03aae77,0x000928ca9669f559,0x0000980eab114d68}, {0x00060ecf7e0890f4,0x000add678a21193d,0x000e75ce55e32dff,0x000b93e865c3741f,0x0000c6b7dadc3ac3}}, + {{0x0000ebc9c7982a25,0x000c291e4d112cd9,0x00016c66bcb45331,0x000f8b46194b0c7f,0x0000dfc74610f6eb}, {0x000d1481dd4169f3,0x00038b5f60af605b,0x00009d46e7986887,0x000bb629eb8a4bc3,0x00004d4d57fb25fa}}, + {{0x00045dd030229a4b,0x000f628bf05c5a04,0x000c8ae68bb98061,0x000b9637143f5451,0x000055db14aaf091}, {0x000c500b3ec39fe8,0x0009d418e00ae4c8,0x000f274dda387c10,0x00044d258b0c3190,0x000082d86c4dad00}}, + {{0x000c6b74352596ac,0x000bbda2c19cc12f,0x000372be7ac616b8,0x00068ef8dd414a4a,0x0000a5c3507d3814}, {0x0005c60076424b1e,0x0008521878c8f96f,0x000720621f0ecd3b,0x00092f64b81e74eb,0x0000538ab09b9a9e}}, + {{0x000147af5c6b6fc3,0x0009d626e2d99fe0,0x000e4efc5d3b7322,0x00023ead978b1f26,0x0000f06e93347735}, {0x000048486ca8b31c,0x00068bde8b51ab05,0x00034d7e9ebce61e,0x000b3f2f10632b10,0x000093e107c4e9cc}}, + {{0x000a93461fccf159,0x000ac594ab715187,0x00023895d730753c,0x000cc5a17e2ac232,0x0000ef8cc49c9735}, {0x000ae251f43e9bef,0x000f1381286e7f14,0x000ece1bd4bbc7ec,0x000c24d2741d2d9d,0x00004d288de19a8e}}, + {{0x000485ef01499626,0x0001422c7178e581,0x00079cbd8df6a745,0x0008dba5d0e4b656,0x000073799e051574}, {0x00030e5c106aa63a,0x0007bb8bf1bfe2fb,0x00051e7a789d99b2,0x000550c0acf7e0c2,0x00003f634d522030}}, + {{0x000bba47947ece86,0x0009f8484a1cb45b,0x0001b504f55b8127,0x0005f5297e6d1f72,0x0000ecb79c9e7225}, {0x000a586963b03fbe,0x0009a9a594524654,0x000722ee4a3b403a,0x000ab40a3e93989d,0x0000606778e918ca}}, + {{0x0009e70fcdb5c523,0x0009777edeab081f,0x000b6321e92e6fc6,0x0007d56dca6cbab0,0x0000481a9a365925}, {0x000c9c830d7147e8,0x000d7ed3c882eb18,0x0002ffe1754187e3,0x000ba41b2d1d263d,0x0000d9341cfa1024}}, + {{0x000c73485db43e65,0x00072de631cdb39c,0x000dddde68822312,0x0009f1cca53521df,0x00008999320f891a}, {0x000dc155336fa529,0x00035bc4fb3d5300,0x000d60db809e08ae,0x000611c315d50e15,0x00009de175044122}}, + {{0x000fa9194d351ac8,0x00074be1275acbad,0x000d047459aa08d5,0x0006b7f0a2a4d8f4,0x0000b07256b61364}, {0x000b6c992e60611d,0x000e5798e15351c8,0x0005f15a3b1f6a0e,0x000ebc7ea67466ba,0x0000be447e50b869}}, + {{0x0005d7c17fc06372,0x000ab2ce49cccaab,0x000161cd0f703ffe,0x0006f3af701e3650,0x0000a22686632fac}, {0x0004ffc699f8f077,0x000cbff2b120fb16,0x000f1296557865ec,0x000e28bb4519d52c,0x000031c60ef40cec}}, + {{0x00058f18eab52d1d,0x000560d708260815,0x000f6f11289d112b,0x0003447238d0bb08,0x0000ec9f217619aa}, {0x0009a34620f0a1e1,0x0008b55172a98dc1,0x0000ef9ddda42dee,0x000e7f1fef171e88,0x00001dc69d22f38b}}, + {{0x00081cbe5c6b0ffc,0x000a70f127a082a1,0x00036c3fcc49c7b5,0x00032943bcf4908c,0x00006403b86cd8db}, {0x000d0a77264c886d,0x000074c46e8308b8,0x0002b835f05f0d1b,0x000920f60b800c29,0x000049222a687aa8}}, + {{0x000576f2d8ff940d,0x0008af6e9ad4ff42,0x0001881bf2265dc0,0x0006ff7cb9f24d1e,0x000088c9c79e4b0a}, {0x00073d5b18e1c9f9,0x00006a4d2af38012,0x000cf1d6666875b3,0x0001d3988b576672,0x000045dbe0545fff}}, + {{0x000abeb891f1ec6b,0x000fd53801df03ff,0x000b2923fddd0a86,0x000d0d90338c4e04,0x000093b7a1c89855}, {0x0007c6d30f5cdae0,0x000fec5d2630aa27,0x00063d74b39cc985,0x000af9d1a3f8d39f,0x00004b3d5f66a46b}}, + {{0x000c9f671d7b1d51,0x000a34d571ffd097,0x000e017c1332810b,0x000ac856d2de4856,0x0000b5ac59d08ba6}, {0x000c9395a299f7d6,0x000550364c6c872d,0x00073ca853f70ad0,0x000d64e23185ac4e,0x0000a0dc786f885e}}, + {{0x000365ca2f6cc753,0x0001950b7492d550,0x00003bfbde327978,0x000b87d29fe0d9cb,0x0000746555bd48ce}, {0x000c895bc1abe035,0x0003dbcf3d569c94,0x000bf1343778745b,0x000ef3214b6486b3,0x000047aa6736af22}}, + {{0x000a2fcbbaf2c939,0x000d9fb3b552fd25,0x000b6e203f0b0875,0x00062eb1f0689b22,0x00002dad5d7a6b6d}, {0x000e02be2a7f8910,0x000f4be85299fed4,0x000459f8ce325e28,0x000f4d4865b62c76,0x0000b1af905e5737}}, + {{0x00052120e85c6041,0x0001ba10be26a162,0x000d804fd3ec775c,0x0002eaa61440fb52,0x0000068ea63e38d2}, {0x000b7fc5f961a053,0x000cc635146d1c90,0x000e33657bdda345,0x00085ee9c9428d6b,0x0000446275189867}}, + {{0x000784a3c5468e85,0x00047984ba27c370,0x0004066d7e5a8330,0x00061c6d875b96cd,0x00009ef1394bc3b5}, {0x0008a9fa2de76220,0x000e636f54a420bd,0x0002ea4587bdbb30,0x000e5caf449dd5f0,0x000010d9580bf257}}, + {{0x000bc0df5663cc09,0x0008ce45bea99fdc,0x00066c8aadefcd5e,0x000c717280653351,0x000013d1b8e2e804}, {0x000e2ee1bcaa98d5,0x000a89c86e998159,0x0007ba8b8fd3241b,0x000cda83ec96ed7d,0x0000cbcf8e5c4088}}, + {{0x00013b8639ee43e4,0x0001cd5b01220ed1,0x0003036576d0f097,0x00062825f0577f4c,0x00008ef888dd7597}, {0x0002f120ef9824af,0x000bdfedb88169eb,0x00096098c26c3113,0x000fb30d47829f28,0x0000b9a090863684}}, + {{0x000e1d9cbc3fde8e,0x00004d7bb98ff2fe,0x000f525f9a307331,0x0005b55bcd46bc61,0x0000018644ffde13}, {0x0006b02f72aa32c6,0x00064f4c36a1c7cd,0x0000fd885a8f4dfe,0x00087bd6ea8c9b1e,0x000074cf3eb4034c}}, + {{0x00073bd32b3ce74a,0x000ae1cd16e13987,0x000ae6b331a3b6f6,0x00046de18f795e62,0x00001b268c2eac8f}, {0x00069500b796bcb9,0x00084b0bba811e96,0x000f2ccf64d07f55,0x00041c6d9453b627,0x0000ab5a6b96d1e4}}, + {{0x0006ec3a747b5540,0x000399f0d80c1e58,0x0007fdb082affb22,0x000eb0c210e996e8,0x0000e95e547a6d00}, {0x000c9c0049c9c5cf,0x000e0f520325da72,0x0005ebfd1d560466,0x00076497d952c276,0x000045f0b3696836}}, + {{0x00008a22bc43724c,0x0003c41e774eef64,0x0003a4dfeb9d0e74,0x00004b006555b0ab,0x00005d1666eb0348}, {0x000248fd93a49062,0x000be3c4df2ca530,0x000998ee3e168071,0x000d38f9f0133243,0x000048b66cacb6bb}}, + {{0x00089d9f89790db3,0x0005f889a66e42ae,0x00077b13eaca57e7,0x000143092c735596,0x000008aacec1a523}, {0x0001e6516c69967d,0x0006f9a3b6e5c1e8,0x0009f29dfbf864b7,0x000799dd8c521037,0x000071100f81f8ea}}, + {{0x00095a4465926123,0x000cefa1e2ba3fe7,0x00026ef78e5b31e1,0x000a78581fbbd306,0x0000cae68ad41793}, {0x000c18c2eb0d08ca,0x0003ef8c811ae66b,0x000e2252da9bb52c,0x0008a2c5369b8118,0x0000bd6b4123893e}}, + {{0x00098fbeb198837b,0x0007e591d932f152,0x000a139813f9b9d2,0x0001b9a3399f77e4,0x0000a1298c3d05c0}, {0x00022a6bfb15d6ef,0x0001033da01cb0e1,0x0002fb9fda76e2c7,0x000f5fbc3f0ec949,0x00006c65aa22897f}}, + {{0x000c71c17acd4295,0x0001011c393d2328,0x000286245219bdd6,0x000cba4b683f2657,0x0000f802506bd066}, {0x000b96620f24420c,0x000fa4428c9d74ac,0x000aacb236ad94a3,0x0002c648f91ae4f5,0x0000fa1ce59741a2}}, + {{0x0003b3ce11cb6f2d,0x0004d34dc8b8ff58,0x0002fd7cc9dfed18,0x000f048bfd16a8b9,0x00002ef21c9d0bcb}, {0x000be2a7a86c43c4,0x000ab9dfeed3f183,0x00013c43d69b6205,0x0008db23413a7f78,0x00001d39c7d32aa1}}, + {{0x0005b43063f50fcd,0x000194e360c3747b,0x0003b425a290fd4f,0x000535e867f57238,0x000025fd6b026625}, {0x000adc9d6e1191c5,0x00026e3cd549d280,0x000ec1539adc8332,0x0000cdb1ae29236a,0x0000810251d55eab}}, + {{0x00029cb4887c43a9,0x000bf0ecbf8060fe,0x0000dc5c04dd638d,0x0005a407f09b063c,0x0000c90890142afc}, {0x000c7f52d04bee9f,0x0006e697fab88589,0x000126de25d7c826,0x000402657a0ae9f1,0x0000e4761b55036e}}, + {{0x00025a23ef5de284,0x00092f812e80c46a,0x0003fc24edd7ca58,0x000462dfde4b0b1d,0x0000cd2f0f0bb018}, {0x00086a72d1cc82af,0x000c2a483fcf4d9f,0x000519f9a53040b2,0x00086b7b2c6bb96c,0x000049adc46ab823}}, + {{0x0008d0ffa742f123,0x000a2df6bdd73b6b,0x000a1cc8e1a0bf73,0x000c4aaf283aa1df,0x00001ce68bd7aa7a}, {0x000282e1f607c5f2,0x000ef137e8b5ad86,0x00037413093d7e59,0x000744f3271f905b,0x00008c6b4eb4ee8c}}, + {{0x0005cb112391f8f0,0x0008641ea3f94138,0x000a010f48cd20ed,0x00039d7c395dd04d,0x00005c716598493c}, {0x000e3a42b6e9113f,0x00069cfdaf0139c0,0x0008a7df12e8d24d,0x00074ace4e33416a,0x0000674561bb3fb8}}, + {{0x0002c06cde003641,0x0007882475bc642a,0x0000572001a99da3,0x000608dd4b142d6f,0x000093c30c54e876}, {0x0005a56a6f3399b1,0x000625608bc9ef4a,0x000c52c2c93c2f89,0x0002135ade60f126,0x000004d9ce986570}}, +}, +{ +/* digit=30 [{1,2,3,..,}]*([2^210]*G) */ + {{0x0005587744fc9b0d,0x0002235a186fab45,0x00014d4716a6ca8e,0x0008e21b61f724f3,0x0000f3f72b99a2c5}, {0x00080e1a9312fb07,0x0007d9cef9613e14,0x0003e5490ac148ab,0x0003955a1c207543,0x0000f96356794b0f}}, + {{0x000819be29f68267,0x00009fe4deb38664,0x00050b6e73cb6078,0x000558aacd6e06b1,0x0000279daf8f7d63}, {0x00018e07f25095c8,0x0008331cfc9f5f26,0x000782972feb9f2c,0x000658f802fca94e,0x000038c50f53e7be}}, + {{0x0003d4ddbebe9fdc,0x000143df4bab03d1,0x000400f0c967a672,0x000eb663cc254819,0x00004477f6a2dd2c}, {0x000ee0d4c3d37c69,0x00009f15d2633c34,0x000643fc0eb2b41e,0x00009b2d010432e1,0x000030aa45907c25}}, + {{0x000ec65be9e92878,0x000c55c6b87a3bca,0x000db423a40a8edd,0x000575f207c56c36,0x0000de77bbae2005}, {0x000580d56553246c,0x0002d79f3040d372,0x000398bd3d4cafaf,0x000cf3e0535cea88,0x0000d8835ad63a58}}, + {{0x000938515a35e548,0x000bad5845ecfedf,0x00085e8461dfb3c4,0x000503cd8b2d66fc,0x0000a93de3d98e57}, {0x000f822006f36a34,0x000a47a6f9bfc29e,0x0009551254d51965,0x000048cb979c84c0,0x00001df6019a10a8}}, + {{0x00078cab73289d88,0x000b9d9fa4701c6b,0x0003396e135bd0c6,0x000974c909a06d9c,0x000019ac99ac8f6b}, {0x000ceba50d80dbc5,0x0001303b634316a2,0x000b684d771cc7e9,0x000516e058c5049e,0x0000292ba043dbc8}}, + {{0x00038fbb6d7a003c,0x0005cf09193bfc73,0x0009d8f07894acea,0x000acadc9f445791,0x0000d19f61fbcab1}, {0x000ee3f4a227ca59,0x000da7f63d2a427b,0x000791f62bd40826,0x00024a161e806c1e,0x0000add1e908c47f}}, + {{0x0009ab29ea7ebd27,0x0005e96d975a66cf,0x00089209baa3ca4d,0x00014f686c002e33,0x00002fc9f0d6de64}, {0x0008c324e1c3942b,0x0000f90ac257b88c,0x000d3ac5c6940150,0x000a881ab7db6e14,0x000041cfa9e1e973}}, + {{0x000cd33ae6949b59,0x000c4ebf33ea027e,0x000599a62bed515f,0x00080a74262307f2,0x0000658216a1a59c}, {0x00032e776bfdb8c8,0x00017f7f1e742947,0x000e4d1cd2135990,0x0004b293f38844b0,0x00007a0099785d21}}, + {{0x00037890b4606073,0x000bb550c322b536,0x0004ba1bce567534,0x0009e06d2e436e29,0x0000a4c94f2a27ce}, {0x000138a6812a277f,0x000f29b99d6648ce,0x000a581647b50956,0x000333ece9cb729d,0x000027d4916a975c}}, + {{0x0008ffdfd9dd8995,0x000722772d432015,0x0006c364a3ab59c6,0x00070a5bf413de34,0x0000d46f8062e931}, {0x000b960a7a4ea0ea,0x000695d0db76048e,0x000e021533582f16,0x000878c9174f8cee,0x00005573b093f78c}}, + {{0x0008ee06f4cb971e,0x000eb1dfdb17fcb4,0x00055a1b0f4066e7,0x0004c7cd2cec053a,0x0000ebdb6c1c4266}, {0x000350374bfd9193,0x0007524863bd2f5f,0x000948b9fb7ad01f,0x0008761103a17657,0x000010efd69bf34f}}, + {{0x000e391227b7b918,0x000179b0e8190ad9,0x0003188ad2f3b5ab,0x000db428f6ac1ae1,0x0000ac2d65e8d7e1}, {0x0005fad05741fe5d,0x000766f6f06a7260,0x000ce0f46f458b8d,0x000fce97f099d97c,0x0000ce699ff7d01f}}, + {{0x000d627bbbfea21b,0x00065bc86f5b59c9,0x000cbb800712f816,0x00030ace738f6336,0x00006b1e3676ef63}, {0x000629b9767990f6,0x0002ea49f8676b78,0x000441c77086b44f,0x00050133cf0e09be,0x0000f290415611c6}}, + {{0x0008e4736e5a00cc,0x00089440d1089750,0x0001a583a42edd9e,0x00093fd719bfc55b,0x0000b56d2dda07ce}, {0x000d6fc706e0a7f0,0x00032881de026303,0x0005d65f6c087882,0x00067db008ec67a8,0x000014533f4dc722}}, + {{0x0002f3d088db12bd,0x000fcf97a9b2b8b5,0x00019e14ff0d94a7,0x0001362fa0525b50,0x00003579f0be0bb6}, {0x000f2ad6651cd253,0x000f8099f9cb7e45,0x000375844682073c,0x000b76f4525fe726,0x0000a75ebc0af507}}, + {{0x000f75b1f2819cef,0x000f19e282729c41,0x000e04765719be03,0x00083fadad0e2c38,0x0000a88c8f8e6bca}, {0x00095351371d8ffa,0x000a01a80f584579,0x0002533ae7490891,0x00012b5f33fb816d,0x0000bbdbc03aae64}}, + {{0x0009a60af672be3e,0x000fbb1bf99b39df,0x0003def3e27b2b86,0x000698ab5d32dc62,0x00000c7f112674f8}, {0x0006c0939385470a,0x000d3c072ad3f6a4,0x000e841d647176dd,0x00050201733adfd1,0x00008ba67afea17f}}, + {{0x00061ae2b3427567,0x000dea4fd3f60fe6,0x000698aad50aa6e4,0x000b8d08d2ad5882,0x00000e1b9baf7513}, {0x000488bb1d898d7e,0x0006ce8c0168ad39,0x00091a4ad9a1c7e1,0x000819b619ef164f,0x000047df7d2d85b7}}, + {{0x000feab634bed6f8,0x0009357dce07ffc6,0x0001f22c07bc045d,0x00020284bf369c39,0x000000ec5355d927}, {0x0004492e1ef8e017,0x000bebc2c0cb9bef,0x0007f47cb55429b9,0x0001452791fca0d9,0x0000446d21780520}}, + {{0x0007bbe224fcd24c,0x00073d7bf6169471,0x000ac5c21a5a8b44,0x00039b7699434475,0x0000c24ec0229fa1}, {0x000ef056abea3468,0x0001b5631170484a,0x00014686e31baa0f,0x0008aa6711643949,0x0000ab5fe36c56bc}}, + {{0x0000fc5e6fc64867,0x0007fa43f868a21a,0x0003a88ba36f8db7,0x0002e70eb5ad6bdf,0x0000871d27ec8c07}, {0x0000eb5d0d0dfac6,0x00046fcb36133cf6,0x000b30f407d7c7a9,0x00000d6528aeb587,0x0000c3c8285253b9}}, + {{0x00054b828518597c,0x0008972f858fb628,0x00099871d157ba17,0x0007e3f4033329ca,0x00007518282b4088}, {0x0008335b1c0daefa,0x000b28c6fa72dbe1,0x00044e1cdd57d324,0x000b9e9bd8eb94d6,0x000090a27421fd2e}}, + {{0x0004aa7033c1794b,0x000b58d101a5a260,0x0009e75d4582f913,0x000c67d11d071709,0x00000c26db9ab4c4}, {0x000e2fd553fa1538,0x00092583731d4527,0x000ab1e032a78837,0x000565e5708eb33c,0x0000c48e233e93b4}}, + {{0x0000f3606cd3c7e1,0x000f9cf3f1a25ffb,0x0003c6509d6f19f6,0x000fc6ce3b57e1f2,0x0000c8aa53c29394}, {0x00013193e35bc970,0x00071c65a63e7410,0x00003fc0f1d93591,0x000631583a7d1998,0x000061d7972661e4}}, + {{0x000f3b0ec646b6fc,0x000ab80b5bc4d4b2,0x000e871c1dc2998c,0x0006f2466508817e,0x0000fbc22289a212}, {0x000d2a00a06285a6,0x000173fe65578efc,0x0004a98cb75bf247,0x0008fac92d989d9c,0x0000948f4932b9f7}}, + {{0x000845cf66cfa8d6,0x000482a7592908e7,0x000938c783adbd0a,0x0003fdf2c40e9dc3,0x000081189439bf8c}, {0x0001dfbde2120af3,0x000ec8f666c61cac,0x000049635e3b8104,0x000b570d910b372a,0x00003b0a13fe3b6a}}, + {{0x000bc798fcd4c386,0x000e2d3e989b0ec2,0x000354b0ebfb4e64,0x000ad6b241e9e539,0x0000e64b4499f9ea}, {0x000824a5c0b286bc,0x00095daf822450de,0x000b5e3cda89f6da,0x0002c25d832cbc77,0x00009df72511f703}}, + {{0x000d12e444a0e491,0x000990d76c78030c,0x000c2baffeec0fbc,0x000f8c635cc28c64,0x0000eed641f9bf3a}, {0x000c2a90ac6e705a,0x000a41dfdafd6d8f,0x000c5e6d0fab169c,0x0009e24840dfe5bb,0x0000261cb58c5df6}}, + {{0x00068c7c8f041f5d,0x000195a9be95746c,0x000e714fd26f00e2,0x00086a2fa5e86163,0x00005265628346d5}, {0x000d2f1ae615f3b1,0x0005881ec186cd66,0x0001f8ebdd09686d,0x00062642c2578874,0x0000b8d0ec7aa3ef}}, + {{0x00058a6e2678b76a,0x000c3decc2e12e45,0x000400578f26b5fe,0x000e9c64934cdf4a,0x0000ac4e70084d31}, {0x0005269ba3707c6c,0x0000ea75e26ae313,0x00076aec64c7ef65,0x000cede81346c1e0,0x0000e7007b498e3f}}, + {{0x000ea93372944b60,0x00032e7f0407c535,0x0004662e884f3908,0x000ff3b4dba2af07,0x00002dd0c0bfba19}, {0x000a01101857f415,0x00002e9c6bfcd3bb,0x000b305efdfbac49,0x000f445da2626ecc,0x0000907070059481}}, + {{0x0007b22d683b3a49,0x0009bcf9beabef21,0x000c7562a038a49c,0x000910aa13a5841f,0x0000c9389d6162bd}, {0x00075090b585818a,0x0009316773fa9595,0x0009848a8d0f9dd8,0x0001f651555c4021,0x0000d05a8d89f69f}}, + {{0x0004d24ad962db65,0x000543238c6a727e,0x00075965517dee3d,0x000a85f8def34db8,0x00000d3a88c6d24b}, {0x00034a623ebab5d8,0x000bf7b417eac935,0x00037ce6949c24d6,0x000036e934f11b54,0x000018002a7a5455}}, + {{0x0003123c18986e41,0x0004271e4976ebd1,0x0009925168a8da06,0x0004b18aa1334900,0x00002665721f0fa9}, {0x000b0ff43133deb8,0x00000a9c1b9fdf83,0x0006c154d3516d4b,0x000d735850f7ed9b,0x0000e079bec8fd08}}, + {{0x00069c12865882fc,0x0008d9957a5ac327,0x00093aac73f8192c,0x000ceac6fcd6d246,0x000006af48a5b01e}, {0x000026027353e4aa,0x000e1c3a62c87965,0x0006a866c4b5c0d4,0x00057ef67206ffc3,0x0000f866c2bb479c}}, + {{0x0006024ef491d5db,0x0004e2704aefb3f8,0x00074dc8b5853c6d,0x00063b3be06adb19,0x00000a163a242c3d}, {0x000472d0e212765c,0x000adbffde000bf9,0x00095dfc6114a063,0x0002ffa142ae6b29,0x00003767dae16b26}}, + {{0x0004392a32caaba8,0x00067d4e7317e8fe,0x00062cd25f60018a,0x0002429928340b9d,0x0000d019c13c0075}, {0x0005df10a659460f,0x000ede055af81262,0x000fbaec0f8c8158,0x0004dd990e40671f,0x0000f1d47ce0cdde}}, + {{0x000b86d5de825fda,0x000827ad5ef1c31e,0x00017d3651481ad4,0x0006f11c7f12dd40,0x000023ccbabe1ee2}, {0x0006b90aaec673e6,0x000becfdce1eaa36,0x0004ffdf0bd56245,0x000f51bd73c118ef,0x0000fc7f3b398c76}}, + {{0x000b507845c7e712,0x00083994ff8becf0,0x000b099529c1c424,0x0000bc950938df6d,0x0000f211f0736bc4}, {0x00067c661de70ba3,0x0001d734eec5882f,0x0008914f01a0144a,0x000a3c5faebc04d3,0x0000aa86b02d5b26}}, + {{0x00019c31bec876f4,0x0001988ea72cfa1a,0x000ee8601d01485d,0x00046183d3520a82,0x000047ab80a29f96}, {0x0004251df44392bf,0x000a29983d364de2,0x000a88da2e7b5ba4,0x000954f4e59b9a0f,0x000037e6c24257f1}}, + {{0x000e6dfb09a4ace1,0x00047c55d4e26d05,0x000e515e1f45c89b,0x00043f7bd69ae93f,0x00005479b4aa9e09}, {0x000ba9bbf74ce4ec,0x000f332dccae40b7,0x00003a3676d0e1de,0x00089f9a692179b6,0x00006ac8d0990a7d}}, + {{0x000b612be9f087c7,0x0005d2b73fa0575c,0x0002ca2ba11c214a,0x000ed841f6a65480,0x00000b110025c00f}, {0x000d80afd4f314e3,0x000f3c64e352cab2,0x000798d050b2af3d,0x0004185885404c7a,0x000023d177bde8cb}}, + {{0x000178806542640b,0x000d88c52d25f44d,0x000446d579176f30,0x000397051da7626c,0x00008886aafbab41}, {0x00015912cf9cea49,0x0006e21fa90bb0ab,0x000908b221aaa808,0x0003a7333bfb3941,0x0000ff4d7efda834}}, + {{0x0006e39f3414f11f,0x0009d29f664e12c5,0x000500f05843165b,0x0007ffcb61460ce2,0x0000626de1e0448f}, {0x000c9f888fd60e9b,0x000814b7c85bdd05,0x000503d082652999,0x000f69928a0fbae7,0x000079af9aa5c51b}}, + {{0x000b748853dc369e,0x0001d5c46a9c7c80,0x0004943a99974554,0x0007adda66e0232a,0x0000290429a60883}, {0x000995ec151d853c,0x0008ff3a1822fa7f,0x0006eaede0e6bbab,0x000cba88badae027,0x0000c256d3997a8b}}, + {{0x000e2a41663b8554,0x00092366f160ca96,0x0001c123d905387b,0x00034e0adcd7509c,0x00007b1f6e4adc08}, {0x000b82f6a17de8df,0x00016127f77fa75a,0x0004d788321ad9a0,0x000a97ae85ad92ca,0x0000b54c99e120fb}}, + {{0x000bffe361a46388,0x000fa4c0605193de,0x0000fcdd0adb800a,0x000f5618356c1336,0x0000436032ec11de}, {0x00053a068b16e75a,0x0007a5a6a6730ec0,0x0001a5ba4dbd79ac,0x0005fe6729055356,0x00003a575af0acad}}, + {{0x000fdc35bf0c9f33,0x000da45b9b398957,0x000ad7beef5e41db,0x0006432a7c818563,0x00008c2e1c2b7db9}, {0x000b00db46177027,0x0003e2b378f87bf1,0x000a0f28a308889e,0x00097ba082187936,0x0000acef7ee2ea51}}, + {{0x0009390c918d6270,0x000362caac635c08,0x00015b8376799de3,0x000eaa821a885535,0x0000cabcf406953c}, {0x000d04fc7648475d,0x00098dc7d882d0c2,0x000aa0b4c0a7dc6d,0x000adf01bdcf234e,0x00000e2626b268f4}}, + {{0x000220825ed35afa,0x0009576f7cccbe00,0x0003a8d37c11d3dc,0x000bd7a0701b842c,0x00007dc318423ed8}, {0x000dc6851e23a270,0x0001482baddbc17b,0x000ef80594dcf07c,0x000635f48b62fcd6,0x00004ba3f7faffe9}}, + {{0x000df3379d5aaa09,0x00053e95a9719715,0x00094035e04096ab,0x000371086a53ef79,0x0000479c7b8f06c0}, {0x0006071ec85431de,0x000b4d8ac75c33e6,0x000b91fc3cf2421d,0x0000c5a713565a6e,0x000093f6b5095293}}, + {{0x00053433c213411b,0x000f19421558a32f,0x000cb3419d7893da,0x0003e94fca316dfb,0x0000374884a86a35}, {0x00018637e8891bd7,0x000dda699e3631dd,0x00099f03059dff7f,0x000c475d060ebfba,0x0000bdd0e28de233}}, + {{0x0008625a553631ce,0x000a9796b7c66042,0x00069e1a8d149b49,0x000b9d24bf15c71d,0x0000499b36421998}, {0x000c53bf713ec42e,0x0004b77890566781,0x000528324cc47d1a,0x000e99ee4752d162,0x0000b9b11a1afa93}}, + {{0x00081a0ffb5c9c9c,0x000e055f0aae4142,0x000f67533c055cb5,0x000ba1661d64b05f,0x0000d37a37649d4f}, {0x000fbbba5ad491dd,0x000cf8282ecf7198,0x0005ccb6e6b04c81,0x0007ebe65a66949f,0x0000c80d786e0584}}, + {{0x0008203f4211c7b1,0x0002f22609747204,0x00046dc191e8d62d,0x00019cebf4ef9152,0x00009163ffa00eb0}, {0x00076e6fe036a5d9,0x000f17c8eb9e7d4d,0x000b94224aca7e98,0x0007dafc52c4dd07,0x00008fcefbaee74b}}, + {{0x000f5f295afbe525,0x000dabfbf6f56e65,0x00029dd9bee154ba,0x00075fc7e43056b1,0x0000c6703f05bb43}, {0x000575770ca61480,0x00034ab150f37095,0x000ceacefada8fcc,0x000f68dcc0b53bd8,0x0000e9010313dffa}}, + {{0x0003c71f5c8dd248,0x00064afb2df6f44b,0x000c4e22d86f05fd,0x0006bdb2e524154b,0x0000823ebb857629}, {0x00050ddefbd1fc06,0x000d57e3a6573398,0x000e9fd87a05db65,0x00097c5a82bfea97,0x000012602f1dc978}}, + {{0x000d464401ebe000,0x000aae7249d91866,0x00036dc90713e6e6,0x0005568974870842,0x0000783ee0137c6c}, {0x00074662eccf7759,0x000d7f6b891712b7,0x000f43eca62ac342,0x000072ff0edd30ee,0x00008a7880f860aa}}, + {{0x0006cebf5ed71c29,0x0009efe22291ed90,0x000b496537c3d7bd,0x00085a93bb5041ed,0x0000dfedb68b74e5}, {0x00072c8d5958c8f8,0x000afb5d5f31f91d,0x000f4d874ee28bf5,0x0004f7e7a69fd006,0x0000529fdae49ca5}}, + {{0x0004deb5cf2cc3c6,0x0008c9d887dc815c,0x000d6a375e22f1ae,0x00076773ad520b03,0x0000de108610ca0d}, {0x0004e8cb47e14977,0x000ad0aea66d4a4a,0x000783042dd19814,0x0008d9e54672563b,0x0000b890128daa2e}}, + {{0x000808410ee3d23c,0x000392a615b14640,0x000d2683f464306a,0x000bd3ab01e405c2,0x0000e70e8511df3b}, {0x000fdca6a9e839a1,0x000c9403fb89e6fe,0x0006a9fa129439dd,0x0006d65462b965b3,0x0000f3fd2b9110f6}}, + {{0x000afdf466dca9a6,0x000ca27896b1bc08,0x00026b7aac5ebb91,0x000b7ee6800bd153,0x000059f83a9b154b}, {0x000160df796189f5,0x00088ae54694e160,0x000d66289e9c55c3,0x000b00a2b43c3c19,0x00003413ff683fcd}}, + {{0x000aad2f31e88186,0x0004568332d1f745,0x00085e5b8099a8b9,0x0006911240d922f2,0x0000f5f060aefdca}, {0x0008a8cc9a6632e2,0x00009a8ad40a2cac,0x000b2ec9b7267e5f,0x00067797ad93d424,0x000056880826d350}}, +}, +{ +/* digit=31 [{1,2,3,..,}]*([2^217]*G) */ + {{0x000c5b31b84523e2,0x000be6e4ef99ea3b,0x000a69918305e9d7,0x000f226013092787,0x0000aeef51df4898}, {0x000935d577020efe,0x00047c06a751ec81,0x0009737700fb9a01,0x0008f210ddae1671,0x000078a0d0f4c882}}, + {{0x000d5e8ccd993818,0x0000454160028f27,0x00035c9e5ac962fe,0x0003961bd348325d,0x0000f7888f9d08c4}, {0x00028e7cc2ce2849,0x000864e2a7ab864d,0x00004f4743155567,0x000056ef40757409,0x0000de9ba7203fc3}}, + {{0x0005c0ee77817ffe,0x000841cea362486f,0x0007dd9962de3465,0x00073b6d58d6ef81,0x000092b6a9f738f6}, {0x0008e609ec8a67f5,0x000d874bf95d4493,0x000f082668cc88ea,0x00018e51a53b6999,0x0000c44968fda017}}, + {{0x0006d9a9c9c7ecd6,0x0004277106c7b597,0x0006ce73e23fd318,0x00098a67c30e1c70,0x0000f0d0fe2ac602}, {0x00005926a46c71a3,0x0006512eff4774d6,0x00068ac6f253f036,0x00014e045f24b5e9,0x0000148b466e0cd7}}, + {{0x000fcfcd84a929d3,0x0002ad54a2149937,0x00096084028ac0c1,0x000a98eb3a256be9,0x0000a028f7a712e1}, {0x0003ed11da5648c6,0x000765175dac0cf5,0x0007e6793730e016,0x0006e12887741adb,0x00007b90747e694e}}, + {{0x00095060b7f3155e,0x000baaf4df36ae23,0x000b773beedb6ad9,0x000e9434c85948c6,0x0000285db363d9d0}, {0x000ba7a7d1c9fea8,0x000fd444e8a8f514,0x0001079dfc491f23,0x000423e35a941ff6,0x000029feac4a3859}}, + {{0x000f263a7b127166,0x000ffda74297f963,0x0000d25cbda01050,0x000eb7ffa92a1540,0x0000d4cfd6140a62}, {0x000ec796b38c0f5d,0x000b3301612e7e11,0x00088f375bb456a6,0x000f1d2e639ccef5,0x000023c5b7ec9322}}, + {{0x0000d0706eeee3b3,0x0002383bb3c41af5,0x000789d57412b36c,0x0004f76f161f4348,0x00009d54f9fa323c}, {0x00098713416de57e,0x000f289880622fb7,0x000ffa44b6065378,0x000ae646228a95c0,0x0000e77ebc11e012}}, + {{0x0002e057a7667ee3,0x000980621f16ec01,0x000dc6a70ce7de5d,0x0000ab25dd09f571,0x00006ba631c46be3}, {0x000a3ceb0468f0d6,0x0005cca0a4b4f979,0x0006814d43b03db2,0x000c037bf72fa0df,0x000070b2e22306cc}}, + {{0x0006e4060e56d291,0x00073f9eac2a60a7,0x000609574bf57884,0x000850711a75007e,0x00004257875312d3}, {0x000515eddc745f56,0x0008085a3ecc7070,0x0000354544ed84c9,0x000a74204234e39c,0x000012c06b29876a}}, + {{0x0003f156c03f4661,0x00016f589034ebea,0x000b6ec7176a4552,0x000ff259e041df5b,0x00004ed99b1037dd}, {0x00086d386ad2e060,0x000fad2ab5238d4b,0x00011e6c46846865,0x0002b3a450957a0a,0x0000c5d260143d1e}}, + {{0x0008e38152258344,0x000fef5032e4a93c,0x00001424b43ecac7,0x0006743325458b93,0x00007f95a9785eba}, {0x000a205186d9025c,0x000035a7fe558fef,0x000a3c8776c0b58d,0x0009496073e77a5c,0x0000b53d4427a2a0}}, + {{0x000aa0fa4a2e211c,0x000f72cc7932a003,0x000c29fd7e905873,0x0007a0116f12125d,0x0000e442886f9b48}, {0x00007e055af80e44,0x000c596df90f2dc5,0x000d6c18109d73f5,0x0001df729ebcf653,0x00004160bd446d9c}}, + {{0x000fd2d788ba6520,0x000d76ff5738c9bb,0x0007fb3242d9cc46,0x0003339c8d8920af,0x00005c508f078998}, {0x0009e3679f4ab8a7,0x000e2634f4ee182c,0x0009bfae6cc835e1,0x00046a9f81c34f84,0x00003d0468bc1916}}, + {{0x0003b15660f59ef7,0x00088c12554eef23,0x0009330a3a321c51,0x000b3626b05480ad,0x0000c022c30d43d6}, {0x0000661046f68e00,0x000e80e8e9ae5d65,0x000b4697993ae49c,0x000eb1d32afb6dc9,0x00003f033a44cbae}}, + {{0x00087e5163cbc0a8,0x00006ae98f3ee679,0x0007b68448c3e107,0x00085d27aa4bdafa,0x0000aae2dbd0b3f1}, {0x000ac1ef01e8f3c8,0x000854cda9282656,0x00092a068b13fa74,0x0009658322dae1bb,0x0000f651015f4306}}, + {{0x000161d5feff15b2,0x00050345887b8e42,0x0007f685e1f8bca9,0x000a5e069e055b08,0x000014e3f1706090}, {0x0005bacd466e93c2,0x00024cc6b95b9719,0x00056a22d1b25609,0x000c93c6c0448103,0x000081192897bb0a}}, + {{0x00036a0c08a594ea,0x0003b66986efe95b,0x0002f86466896b35,0x000998b3227efd50,0x0000227f683c30e5}, {0x000dc9ff42ea5c5c,0x000e34f0f1d46fea,0x00047ca2cad6837e,0x000856ab3a67ba93,0x0000f0eb2cce452f}}, + {{0x000a1589e0b82fbe,0x000de449e007aa58,0x000d14a4ee633c68,0x000e61af8fef7e89,0x0000f60872f433ac}, {0x000016d64afd1619,0x00060de62c698aca,0x000269bb3d24f6f1,0x0006b7f138fd2d41,0x0000d75248556ff5}}, + {{0x0005206689b92bee,0x000843322d9c7cc9,0x0009915cd7756249,0x0003f02dfc1956ca,0x00005236e7bf69e6}, {0x000a95adca139b5d,0x0008994070b778a3,0x000f73bdbd4254b2,0x0000abac251dcf19,0x00001d28ad0b0466}}, + {{0x000a1cb8a87fdd06,0x00090886a6941375,0x00048f1903f9324c,0x000cf3376989c09c,0x00008b982f180de2}, {0x000716ea40012c9a,0x0001f5874d1db96a,0x000ec9471af0aa5e,0x000a1ab118cd78d2,0x0000f9537952f8cf}}, + {{0x0000b7eee1e1bf2f,0x000297cd743a214a,0x0006a906ae3eb720,0x000f876830a1a414,0x0000eec17838e9e3}, {0x000c737251493343,0x00025cba6f7cb12f,0x00039b703deff11a,0x0000f4e3aceabbfd,0x00002ffa04f9bc74}}, + {{0x0008f06e2aff6bdd,0x0009c7332d94e0e2,0x000f1e13c0a81628,0x0000ff353b4e659f,0x0000c3f048b4016c}, {0x0002e01399fc2da4,0x000c7348679e5b47,0x000abca0e8e7b659,0x00038dd0e20e577d,0x0000dd894b007fae}}, + {{0x000396824f4dbc09,0x000ef06eb9cd7555,0x0000b18acd3df1a5,0x000c18b8dac35ee4,0x00001c7903921397}, {0x0001731faf0f10d2,0x0002d760c1326c4a,0x000f6f5ffea89051,0x000fc39c0f9ced36,0x0000da37d490e5c2}}, + {{0x00023c3b69935ff7,0x0001b3e28de35919,0x000607e0e5dba168,0x000e0a1c7a1ab695,0x0000bc842d58c6bb}, {0x00050d68e758fca3,0x0000ca556db21bc5,0x00047dbec3ec857a,0x00078a4f47c22922,0x00009b45db21fb1f}}, + {{0x000d2b6ed3921dae,0x000414a6a41de350,0x000e1bfe1f418863,0x000a7247d4440aa1,0x0000b202e69be99b}, {0x00035efe5f191a91,0x000303646d9b68c0,0x0007fafb6834e5cd,0x000afd13b66152d2,0x0000e96ff34fa402}}, + {{0x000b50e619a7cc77,0x0000dee84fe2d25c,0x0007ba3f798fa246,0x0003a2051c8c69ea,0x00001559f9755949}, {0x000191005085716a,0x00048db5ae91d058,0x0007c9a5df811350,0x0001e3e76081f3fe,0x0000a0f5614b8aeb}}, + {{0x00018f0a36ef4ceb,0x000febaebf8d4eae,0x000e5c11972c983d,0x000173cfd8c7fe1f,0x0000ff0b6553b58d}, {0x0009033ebd26a6ee,0x0009ca7741c57b63,0x0006d6d0466228cd,0x000813a88d56d595,0x0000001ac00370c6}}, + {{0x0005d49926257fb8,0x00077f587dd695d7,0x00072592f34e8e90,0x00023c28f9cfa11c,0x00008d9950a9e218}, {0x00075ebe481c5af1,0x00050e45ff723481,0x000f61f5abfd66c4,0x000a368b3c547be6,0x00004be914ed594c}}, + {{0x000d4cb73f9b6f7d,0x000e6906174ac89e,0x0004880c64f0fe98,0x00036ab58f236e04,0x0000e2d38bcdc3ab}, {0x000079f3c7af69a5,0x000d95f71ef374c3,0x00041a7447c72579,0x0002667971715ac4,0x0000cb5f998473fa}}, + {{0x00092f2379f7fce1,0x000b91a877864bd9,0x000364c15e47690f,0x000ea25416b9dd02,0x0000a7502960f53b}, {0x00054d4ece71149c,0x00063a28d0321ca5,0x00027b740cac9de6,0x00064ca0b521c06f,0x000059d3a7108c18}}, + {{0x0005870ea005a99e,0x000ded4e493a6467,0x0008b6342b1f047f,0x000374e92a439f17,0x0000a8cf07331051}, {0x0000b513642fc657,0x000dc5e958827812,0x0009b568996f4efd,0x000bfc8f6ef537ee,0x00006743642f3440}}, + {{0x000e3950009e8bd3,0x000279e9228e1db5,0x0007ea75e0e087a5,0x000f0b16adfbb8b1,0x000019ff54645b70}, {0x000dbac58096735c,0x000bb470fce96e38,0x00074907665c92ce,0x0003152b53c9925c,0x00007fc121cadb2c}}, + {{0x0002ee6e219e439c,0x0008fe4475fcfc8f,0x00032583de9c8ba6,0x000c3513fd771433,0x00001a001f040451}, {0x000cd5f7de1cf983,0x00085bb184294f07,0x000f814357b1fe00,0x0001ccad465687a2,0x00004f0c50019ffd}}, + {{0x00030d60cc283637,0x0007dd26a89da97e,0x000f66c26f3bdc24,0x00028716946a9f2d,0x000049a732f2b2fe}, {0x000eef2bd92b8614,0x000b8f3412a393ac,0x000dd110fa90e595,0x0008b581363eb9d0,0x000070f21b629f74}}, + {{0x000fad5927308e4f,0x00030bb54d64004a,0x000a41ebe0fe1a31,0x000560cebeee51f9,0x0000dac11edecab6}, {0x0001e3b630efedfc,0x00077c2beb530f9d,0x0005792e9f5dc319,0x000a71d1a7ea1706,0x00007af663c8c1cd}}, + {{0x000da7bfbd9bb256,0x00088f98890312d0,0x0009548cec55316c,0x000574630b21bcd3,0x000068afb84e9c85}, {0x0009deff6c589987,0x0008e55e9876e7b2,0x000294123dbb7769,0x000712eb8772a9b1,0x0000839395ed1e6f}}, + {{0x000c0a30eb328e86,0x000d5f3a638185aa,0x000c9547dffb413a,0x000f6fae6f8615ab,0x0000e8a6e4701dce}, {0x000b06cc1508bb6c,0x0001b02dbc1f68a9,0x000300f8c0971c4b,0x000fcc371a6d703e,0x0000572fdf15650b}}, + {{0x00014db965071922,0x0003fd3a06369215,0x00011156ac525f3f,0x00027cdb39a981f8,0x00009ff3e0ddcbaf}, {0x0009eefc2c911a83,0x000dae8564210d12,0x000e48efd1688b3d,0x0001eba6dd9308c0,0x00001e6c71545583}}, + {{0x000239152df58a36,0x0008b88e7897dc94,0x000725ab0da0e680,0x0005712ab70e052f,0x00002dfafe3d4212}, {0x0000c2b547371a4b,0x000f4634997de154,0x000cf959ac418e8c,0x0002b4c8fd0e263e,0x000002ca92f3a644}}, + {{0x000805c84e906013,0x000919f4512d7d81,0x000b903bdb4e8554,0x000102020b9b4abd,0x0000a2f04651f18c}, {0x0006d4fa96d8846c,0x00065f0a7d84a420,0x000a4f5352b5e62c,0x000aebcd087cf50a,0x000089c1c7665b9e}}, + {{0x0008e8c15f4490e1,0x000c711806b5723b,0x000f3dfcc9e3ac69,0x0003592e9cdffd09,0x0000676c5f59ab6b}, {0x000e90769fc0122e,0x000ab513ee51387a,0x0006265ef72c716a,0x000395f648e85a3a,0x0000f4445a0a69c7}}, + {{0x00018151ce7faf96,0x0000532c5b17ec8e,0x00077e27fb8ef2dc,0x0009410a4e7962b7,0x0000c7bebe0dfb90}, {0x0005b36791119157,0x000f9791b658b9e3,0x000be736d2983a2e,0x000e11010136896a,0x0000c94c6654ec0f}}, + {{0x000406ad44de0c1b,0x000c823978422f9c,0x0006f3621ed79654,0x0001a45243e9cb05,0x00005a526aaf5259}, {0x0006e9a9952c4041,0x000855d7a5dd0ef8,0x000dfa8b751cd3ea,0x00025c3d5a28b82f,0x00002a085cddda12}}, + {{0x000574c792dd046d,0x000f5db7e04c0de0,0x0002d73f5299ae27,0x000f2bc6f076a4c8,0x0000f2221a703b2e}, {0x0002417d54d5b7b6,0x000fc839da3fd624,0x00044fe6c06c3727,0x000aaebb622fca7a,0x00001614329d0960}}, + {{0x000325aa1dbc71da,0x0003a3f618cb10ed,0x000260f367d91c3c,0x00030ea528ea5e8e,0x0000e7f55131cb6b}, {0x0004e5c73aafc584,0x00007632d1f4e563,0x0003a9ada2a5f580,0x0003d5c9f78e98cd,0x00002612d9f9a844}}, + {{0x0003160cc8cf665b,0x000e9e2cfb82d281,0x000edb5756ac3c06,0x000e86af8cadc78b,0x000071b8a8a9cc1f}, {0x0003f77d6e5686cf,0x0005209f60fd7801,0x0008985728540dc7,0x0000e66d1006fb9d,0x0000dc1d2d11771e}}, + {{0x0004b1a3a9ebb91f,0x00079e23c8e26301,0x0003f0318faa7688,0x000cb05ca5649a17,0x0000ebcaa67af4d9}, {0x0002c158208295bd,0x000371886d2c6ec3,0x000e776fbf3d9c03,0x000b7362ad550edf,0x00001334c9a2dcf7}}, + {{0x0005cdabf52ae483,0x000f0023da388f9f,0x000ea4f0b62843e1,0x0000cd6ef53468be,0x0000740e9f1ab937}, {0x00048cac670d194c,0x00029ef042cd432f,0x0004ca10792b2164,0x0001b35141393fa9,0x0000aa5fc8cb5d0c}}, + {{0x0000b35727eb78be,0x0002fb568e8e3d15,0x0001e26a3e2ebfea,0x0002fe87e6eee810,0x00002e0c18aae6ca}, {0x000ae2e653b7bf2a,0x000aaae347811b25,0x000b752f14d7c1f6,0x000db47d5881de85,0x0000780051b01dcf}}, + {{0x000c69e25aa2f590,0x000fcfeff037dd97,0x0006795b3afeec1a,0x000072a7aa5584f8,0x0000bd60ddffb0ab}, {0x0003d1d63579ef96,0x00038bdcb0ce53e6,0x0004cf4f0c7d0452,0x000c8c0caec14839,0x0000f6a443c4bfe1}}, + {{0x000aec68e12bb566,0x000cadd2bf62c823,0x000164fe059a6763,0x0005e5b4c9c33897,0x0000c530539c6f12}, {0x00069d17e4ca16f3,0x00092006835fcdad,0x000446dd2c99910f,0x0006bb216844cce3,0x000072cda8fd616a}}, + {{0x000043b5b6bfe4cb,0x000c16005febbe5c,0x0009968d097918c5,0x000f4e4a462373a4,0x0000b41b04486447}, {0x000539244c050f62,0x000a32282ebb4b0f,0x000cce069df1c278,0x00019d3afcac56b9,0x0000055958d3f6cc}}, + {{0x0005beb93e6a98a7,0x0001afe915c2b32c,0x00073fe92bbdb3df,0x000079b9f6b3c836,0x00000e14fe20db48}, {0x0007c1023143cf73,0x000f8f3cb4626a39,0x0004f700663c36e3,0x00032fd7c3f3a263,0x0000681de4ac3a90}}, + {{0x000e480c11b6a9ee,0x000d18da2ed97ee4,0x0008b99486776294,0x000a1dcdde1ed3b2,0x0000d4858b9d272d}, {0x000b8bde14ee45e7,0x000f52dae44f1c3c,0x00050b513d37586f,0x00076d444fd07f6a,0x0000af995ed1d2ea}}, + {{0x00044d16e12de73c,0x0008fa09833ad676,0x000cd6eff8c6eec8,0x000211ef8af11042,0x0000dde13acc2a1a}, {0x000e1eb4ffd0ff5c,0x0007d068a68888ad,0x00042fbf6b645660,0x000e072b8fa57096,0x0000277b6578737e}}, + {{0x0005f4f9aa856e82,0x000f2619d7efce68,0x0003de1a8a355582,0x000a02db631aaf40,0x0000ab82cdeda7bf}, {0x000e072b3caaef56,0x000a8199f1d82785,0x000d0fbdc36aed81,0x000cf338edd76909,0x0000ee323f0115ec}}, + {{0x00091002ce826ecb,0x0005070eb620e97c,0x00030a866284171d,0x00087fc5ea32117f,0x0000e6dc3918a37d}, {0x00077c1f1c8587bf,0x0006106dfc0658fb,0x000423e93d2aa347,0x000d9893fef10a37,0x0000e7353a966099}}, + {{0x000cd1f4fed94fd1,0x000122df66a25b2d,0x000312cbb38fa84c,0x00087fa9de6e108d,0x0000e58b82e39071}, {0x00062712709f1afb,0x0008220f16bf0299,0x0000e9cc7fc74306,0x000d421bdb073052,0x0000513ee8efcfab}}, + {{0x00068279bad9b185,0x000b327666c7d175,0x0008c9c1dd36184d,0x0004cd676a4d1d70,0x000008a102f5f44f}, {0x000c7971cffbe4aa,0x0003a2729dcd0b70,0x000906aa5389c9b7,0x000ba57ecfcc1302,0x0000e349023971ac}}, + {{0x000f9aa0de1a61d9,0x0001f5fa5b318663,0x00043909cd0eb9de,0x00070dd7cd579618,0x0000a3eba0c2f73f}, {0x0001563d342860ad,0x00037b084a171284,0x000349978d04890b,0x000779882982abae,0x0000299f32178cc3}}, + {{0x0009f8c49c28aac3,0x00063d222aa77172,0x000103c13f3dda6b,0x000d9a62596af21a,0x00005e568bfd037b}, {0x0001e0a3cdd62251,0x0008c79442a934a4,0x0005966bddbc1924,0x000d996eb83bb296,0x0000d90edb6dce94}}, + {{0x00040493f8013851,0x000ede0d970de34f,0x0009faf7a3b27f40,0x00096ad59ba58757,0x000057262f97764c}, {0x000fd035bf8844cc,0x0005a298a25a667c,0x000fe862aa4f48dd,0x000cfe02fa51099d,0x000012af89f023a9}}, + {{0x000031e9fdea2db1,0x000832d1378f3a95,0x000a5138980ba7d5,0x00069a4bbdba0178,0x000013224e7af549}, {0x000465bcc967c9e7,0x0002d4159c9f2428,0x0007c9b27802478e,0x0008fa77cdc3d6ab,0x00005eb845feb91c}}, +}, +{ +/* digit=32 [{1,2,3,..,}]*([2^224]*G) */ + {{0x000b2d9543778c7e,0x0000f184ad17cd35,0x0009a46cd9fa03ad,0x000d6d473d03630b,0x0000d62ab38692bf}, {0x0004c753b746abb8,0x000a9805cc3c45f4,0x000dcf780bd4f09d,0x0008b45e59befc01,0x0000428ee8930818}}, + {{0x000e7e4fe75c81d7,0x00043a055bf3f186,0x00035ef774abdc47,0x000c9255708754bd,0x0000dc4ba227334a}, {0x0008b967e563c5e5,0x0005ffd12d0380e3,0x000770a428f47d00,0x000757a882b69d41,0x00001a9a96d95f73}}, + {{0x000a784242f916eb,0x0003881ba08bc794,0x0005598b45b14359,0x000d754b78024dc8,0x0000f1f6b99a6382}, {0x000032cfc2cfa484,0x000c8e929e3f547d,0x00059d8829cc674e,0x0003a46aa7f2ecfc,0x0000788320c644f6}}, + {{0x000719cf322c469b,0x000508f3f43cf975,0x000e62459967e01c,0x000174d43d2e7f3d,0x00008d9e6307095b}, {0x00021eba643504f0,0x000978c05daa5177,0x000a1e00ca592f59,0x000eacc0e1e26416,0x00003636fec99769}}, + {{0x000f92768319d8df,0x00052450c06acb5c,0x000f8507a4008c6c,0x0003b281d931f721,0x0000415627b3750a}, {0x000f1a6fba4408d2,0x00097d431af170c9,0x000e56bd0b53c06d,0x00067f2c806e5c24,0x00008d4461687aa1}}, + {{0x000c802e66289036,0x000a3e8fef25bf13,0x000531667a83b699,0x00062916e69a9032,0x0000326f5c93428d}, {0x00025675e2faf33d,0x000b9c7672692bbf,0x000dc358ce94e379,0x000e142af942870d,0x00008672198b68a5}}, + {{0x000f91306ebdde85,0x000deccb55825166,0x000e2bed929a65cb,0x00089e0295a92cc2,0x0000e1f181df2c17}, {0x000223e7b6db2362,0x0001fcafee6e4f52,0x000d6487f3719ea2,0x0007b3ae4fb532be,0x000028642271114a}}, + {{0x0002cedb269ab580,0x000d60cc26c5261c,0x000b8d89fae34d79,0x0001b3b61a336289,0x000038143332f363}, {0x0002e8df47ebad1e,0x000adec248835328,0x000adff3c2c72144,0x00017c5c7e5f1fb4,0x00000b437c44a55d}}, + {{0x000f096ee9f20f71,0x0002602065c66848,0x000be0ba3d5ed400,0x000170008711a9db,0x0000c675cabb9082}, {0x00001a5382a25585,0x0009146dcc0627aa,0x000b9e8cd2497f62,0x000ce9d3863516a6,0x0000450d5c077a1d}}, + {{0x000bb204eb1ee993,0x0000248697876b8e,0x0009b153201c458b,0x00061176db7875fa,0x0000a7194d5e99bd}, {0x000677cfea8c0c4a,0x000a3b5e4b4ebf64,0x000a40a1a68a382e,0x00017da42f53941e,0x0000cf37a3fd9a99}}, + {{0x000f89d531aa521b,0x000d2cd651bbe08a,0x000e4807dc9acf2e,0x00064b6fd48bb09e,0x00000c23a0ed988d}, {0x000accaacd9c0465,0x000107b9102820de,0x0002f26b1b74087c,0x00024fc03e2a1a8c,0x0000e96ef37c79f5}}, + {{0x000728e8507807fb,0x0000036e51fa4423,0x000c53ff715475b8,0x000abd5504dd5344,0x0000af2d60b21d24}, {0x000bcbc1aee6b633,0x0005403e9323219e,0x000c98408f908e14,0x000b388b548ad17f,0x000090118c4e2230}}, + {{0x000dee15b199bc28,0x000c3ec9bdce59de,0x00065ca517f3fca5,0x0008ae7e6b17138e,0x0000784fd240f6aa}, {0x000b34895517b55d,0x000b5fbb4bca8fc7,0x0009e51560b1461d,0x00084b390e438efc,0x00008237a0c71050}}, + {{0x000f4029482f6db1,0x000118db548d0404,0x000c18f0bbf747da,0x0001ad66134afd23,0x000094dcd37be9fe}, {0x00063b0dee6fdae6,0x00067ddfb415ac18,0x00080f9c5669ba44,0x0006d1752f950c9f,0x0000804af7be1120}}, + {{0x0002ac943502b506,0x0004a8db8753f15a,0x00091c2ab67133b3,0x000a77d46c93c8cb,0x0000eb11fc244912}, {0x0007ed2e8e4fb729,0x000c26141a84a0bb,0x00028a4201f1077b,0x00044796a9fb7240,0x00008a44d90c3a6b}}, + {{0x000b1148a2a41ffc,0x000c4865b95ebce8,0x000339dcdf6479e2,0x0005ca788dbd8b54,0x00002ea6baafdd0b}, {0x000edbe9b9fed913,0x000b6fe6b2206245,0x0001105f7b61f901,0x000d311b9cd2149f,0x0000736ac5f8b6fe}}, + {{0x000559ba097b695f,0x000aa4c95ed7fd2b,0x000e2778f8711990,0x000cdc8cbf812ce5,0x000005cbb9ac407c}, {0x00027bd9059f0688,0x00032d37e6550791,0x000b95e9ffc012b5,0x00026ca112f8f653,0x000081248e6edfeb}}, + {{0x000fd09736f5292d,0x000fed6e1546c027,0x0005355f15af57c6,0x0000dd4d11a6061f,0x0000478b38244148}, {0x0002a6999d7e8576,0x0007e0506d5152b7,0x0002a801e7ac65ff,0x0006595b43ed6866,0x00009951fde545e7}}, + {{0x0001a071295d542e,0x000a51a0cc328460,0x0007e1fe3156a9fc,0x000d31bc6de53c42,0x000060f47cd2c597}, {0x000fe89df3012d41,0x0002da5a61ccecc0,0x000fcc1cfbeb7505,0x0003e0edb233e400,0x00001296a606f2cc}}, + {{0x000af7da8d3682b4,0x0007212de42eeccd,0x000e95389b488c0b,0x00018a5aeeda5a59,0x0000474193abe0b7}, {0x000ee97fbc642a3d,0x000cfc9230f0fde2,0x000e131dd8901307,0x000d7c11c9bdb27c,0x0000d12ac316724b}}, + {{0x000d17162e8e5a89,0x000bc66e49efb223,0x00032c9d77f37db3,0x00008043efc71fee,0x00006af3cf6922ce}, {0x0008b9edfb4d41be,0x000844d25cc9d80e,0x000b65b33958a078,0x00079779756de346,0x00001047c9633e6c}}, + {{0x0001564562861678,0x0007624f5cf93034,0x00042ddd963d4ea7,0x000eaa5452277ebc,0x0000f778004d947a}, {0x0001d5def2c56adf,0x0003ae9887f18ecf,0x000a1bf7ebc72b29,0x00097380809dd58f,0x000017a4ef6dc806}}, + {{0x000ffbc9689eda66,0x000883cbbaca1e51,0x000bd0b4d8274f68,0x000fe6bc2914046c,0x0000ec4eebe6b263}, {0x0005ffb314eea5f2,0x000725a4dacce9fb,0x000c953fe6a21a13,0x000ec703932270ec,0x0000c0a405e7c67e}}, + {{0x000c2104134b7fa7,0x000cb3ca29958ec1,0x0005393c52936dfd,0x0004c4da1702d97f,0x0000076b85955721}, {0x0004e97b47458ce3,0x000b0255e87b6495,0x000978b25d48c2ed,0x00049992e3869a33,0x0000244683c71001}}, + {{0x000bbb5e9143e442,0x00055db9d859c5c0,0x000375cb2b951190,0x0001b1733a3e81f9,0x0000ac5b03ec9ddb}, {0x0007d302948b0cd7,0x0004f9a6575ac87e,0x0007e4790e2f1bd4,0x000630d239465cc1,0x0000495665c00aab}}, + {{0x00003d0220dd5b89,0x00050373b8bcb287,0x000b0ad2922a78e9,0x000846833d2f1554,0x000074a397b3fb03}, {0x000d9918bc837c91,0x000769a8126c6a24,0x000f1b5a8b117dd9,0x0001af8d5aafba53,0x0000dd131159eb82}}, + {{0x000a397f380ea2a5,0x000e34cd48b2b67c,0x0008c1b9089a0492,0x0000b7cfb2598235,0x00000f8994fbca51}, {0x0001ef0d4dc6c624,0x0009298199b2a1ec,0x00087db3d9e0b8b3,0x00099bbc93b64cd2,0x0000cd843c51f5fb}}, + {{0x000f2327e09ec79d,0x00048f5c71fc1d62,0x000a82286325e75b,0x000fe92ccbe59c8d,0x00006fa414a59f54}, {0x000724008dcda821,0x00010b18f8339015,0x000cf347b8cc0b11,0x000128d6947544c4,0x0000ee012db1fa6c}}, + {{0x00050c20403a0188,0x000194e8aeedbe60,0x0004a13ba2ac81b4,0x0004e49f3507ba09,0x0000a84889026e6f}, {0x0007b081fb782f23,0x00008c896e27b256,0x000d7b64ff3aa055,0x0008e9a6a881dd34,0x000087a29a866dca}}, + {{0x000f6328d9d29d96,0x000d08db211bfd9b,0x000196c59440e6de,0x0006dfc8e0e709a0,0x0000f9e5f4bbfa4c}, {0x000465c50215dae0,0x0005eb2929551f20,0x000717e82209b62c,0x0005f3c4562e90e6,0x0000e285dee28eb8}}, + {{0x000205c0600232bd,0x000431f8827f3763,0x000527014aa51d8a,0x000c1fa849aa3f41,0x00008ba1d2ff2e5d}, {0x0001bc02ae95f211,0x000374b719625b66,0x000a5dfbf82172e2,0x000b0210053bc018,0x0000a5c238000900}}, + {{0x0006fb6e9e7d0596,0x000bb48d628f76a4,0x000537258d83215f,0x0000496c09409290,0x000066b4a0c8e820}, {0x0001553dd1a5b840,0x00079e8bc4811584,0x00014f392f62ee96,0x0004d21271a3266c,0x0000c58bc7ed96c9}}, + {{0x000f14924e235001,0x0007aac142e06662,0x000c167087c97cfa,0x000e77976870a24d,0x000066e483cc4979}, {0x000f427bdff1294c,0x000acb0b96b81737,0x0003562de53d5748,0x00004ef798c420eb,0x0000348b0d67a82f}}, + {{0x00034268181809e0,0x000330cf8b408ce8,0x000667cd7f1007cc,0x00096db7b5dbc432,0x00004a5fdb5ab287}, {0x0000a0bbf904ec7c,0x0004515938c30e2e,0x0000e23f53a28bcd,0x0009f9b087f747b9,0x00007ca540d88594}}, + {{0x0000d0129cd59a23,0x00046c9cc596ba69,0x0006da59b804d17f,0x0003a7fcd9be29ff,0x00009391f4a28708}, {0x0002481b5b5fd7cc,0x0002560774cebe2d,0x000e9d1e0bc3a862,0x000a4e0a9a5290cb,0x00009ae842e67c86}}, + {{0x00070bee0bc209ae,0x0001fa68d56bcc57,0x000336ccfc8acb05,0x00056142ff984b5a,0x00002395e2c8eaf0}, {0x000ee62df7bd9c4d,0x000e990aead7b27a,0x0008b9905f304af8,0x00087890cb620c4e,0x00008a6a079e2dad}}, + {{0x000c8dcd3073a282,0x000eccbae3857ecd,0x0006a1bbf1f16623,0x000a2b699d780d31,0x0000a776063eab23}, {0x00035739c18802d9,0x0008be6cd5f5211c,0x0009c468c8a0afc2,0x00066879d94b3d64,0x000009f3b2c364ec}}, + {{0x000a623675caa390,0x00028de287f93701,0x00038ba2ae9ab86e,0x0007f403801b55dd,0x00005f51848e944f}, {0x00047ca0ec88095e,0x000876d34449537b,0x000f35de03bacc95,0x00060e05b1b4f1ff,0x0000e43d1be7522d}}, + {{0x00091191b64ddcb0,0x000c0794106d2195,0x0003f8ccc8b5d196,0x000c12c9dc943645,0x000075d1ab7bf58a}, {0x000f0de89450cd20,0x0002aef9f6ef6324,0x000c1a78a35e6afc,0x000a26e7aaebd3f9,0x00004364f4e1540f}}, + {{0x000647708e12eadb,0x000e541c5a2c7a67,0x00065c0fd50feaec,0x00094ee408014a21,0x000031b912b28fcf}, {0x000e4a198d6c20a2,0x00014241bc066c87,0x0003cfb9a4b52b25,0x000955038acdc322,0x000092e10576361d}}, + {{0x000de1ccb3d8e2fa,0x0007927ebdcc0d1a,0x0002337815f56cdd,0x0000d85fffd1121b,0x00003cbd45e59c0c}, {0x0007f3a600093df9,0x0008803285131a3e,0x000bc6bc473d5036,0x000a13a8c656b44d,0x00000f99f5a3c8cd}}, + {{0x000e2b91357917d6,0x0004af49c788ab9d,0x0009b75be9a83bb4,0x000609521c19f260,0x00006e8679e9c4c9}, {0x0009644a604a9c50,0x0006a166441a0a62,0x000f865732bd6b62,0x0004a024589222df,0x00009ff82ef1d308}}, + {{0x000eb11759f3013a,0x000c301c5150e323,0x000fc70dfc2b0fa7,0x0005d3b88ecf6620,0x000062d12a272f04}, {0x000692973b38bb4e,0x000d47714e446b89,0x0004821687518ba7,0x000a53ba0fa24858,0x0000f49c5dc1b4cd}}, + {{0x00074d983ef0560e,0x00040d3c61a6ebd3,0x00049367c38c1271,0x000f0b2401895ccb,0x000001ef869f5737}, {0x000614a7234ff845,0x000dea374efd0e7d,0x000f6ee62c289cdf,0x0009ae62ef2db27b,0x000014784cee48aa}}, + {{0x000159129c7218f2,0x000a4b541372a190,0x000bd9f602675cae,0x0004cd6bce2b878e,0x00002512bc39576b}, {0x000dce0a8803d993,0x000c6cb44bb6fca3,0x000f6629ead7ed05,0x000912b0c7932084,0x00004002232bf471}}, + {{0x00001d2621033b58,0x0001f8fc3dd965c4,0x000ffb247e571eb2,0x000a0595cb93a225,0x00006a8adbe4947a}, {0x00051a6245eb6e91,0x000576eef743cb9c,0x00071f3cc875e4bd,0x000d32e4f4b68e10,0x0000cfc7a753fcc9}}, + {{0x0004b5ad8454309c,0x0006a96c32757c42,0x0002adc838288f2e,0x000629309a27d6c0,0x0000768fa5916f5e}, {0x000ffe6b0e8c2f5b,0x00052be31993a968,0x00095c2da86b50da,0x0002e92416e8bccf,0x0000ead84d4d2c31}}, + {{0x0004170fe1d6eaee,0x0000fc0d00121e46,0x000299ad5553d9d8,0x000da5c3949da0b4,0x00003391fdbfb09d}, {0x000007f4de911dcb,0x000013813852ad9b,0x00059598b370cba1,0x000b9b0b5de1bad6,0x00005d49170081f0}}, + {{0x0006b63bfed067e9,0x000222470ef49d4e,0x000e7714fcfc954f,0x0003504782951149,0x0000a36523156374}, {0x000bffff641c505c,0x000a9e6f797f68f4,0x0005b45bf19da0eb,0x00014b7ce6bed3d2,0x0000cf9f5296d91a}}, + {{0x0009f207b21eb218,0x00028586cffb7a0b,0x0008f3fbd0c3918c,0x00078201c70c1850,0x0000fb503e59c258}, {0x000cfbaddcbe4dc7,0x000f3120734c0d9c,0x000bd2bfeb7c7b4f,0x0001a34a1db6fa8d,0x00005381611bf031}}, + {{0x0000a263cc6eff32,0x0002da89d5f5188f,0x000998b48a9b7274,0x000a6a1ab7e090ce,0x0000c9c108427eed}, {0x000f32ca05971eb5,0x000453079a84fa1b,0x000b8e4604f377cc,0x000af86c83784d7c,0x00008f9d8ae61637}}, + {{0x0006bab37064a7ba,0x000afe0450e00b42,0x000d6875d28c4234,0x000c62f083b2f98b,0x00000c3d4377d759}, {0x00076976a45017b2,0x000880119d2b1ae5,0x0002e89d560ca989,0x00017efbe57b32f3,0x0000d217b1d4d05a}}, + {{0x0001246609a703d2,0x0008581f5ff0012b,0x000ed431681c4d33,0x000889e339cc9e61,0x0000c41d7af37992}, {0x00040e40ee5add10,0x00035ce326c02808,0x00011acb7ee358e6,0x000753b73e148f47,0x0000b41cc70c70ed}}, + {{0x0001e8a27fc5bb80,0x00023f9ce2773a00,0x000f5d4d0de3c75a,0x000ae0e67462e5ab,0x00005f356d1edad7}, {0x0005eb5f99aa27f3,0x000299fa1850b2d6,0x000ed719ea950e23,0x000f6d30fbc3a134,0x0000c607bc2d868a}}, + {{0x000e0dfc812789b9,0x000bf86fbb02a48d,0x0004c91b7d673e7b,0x0009d220bd69b769,0x00007aa28a6d3b5b}, {0x000c81d68bb3e89c,0x000a9675845e91f9,0x00021b13aa03e749,0x000266a5bef7793d,0x00004d522c8aac9d}}, + {{0x000314f0f6a1c7c8,0x0008a2a54dd0d604,0x00034a4a739d70d0,0x000841041cd0a4f0,0x00004a610e679a6e}, {0x0004259d51ad71d0,0x00042afded7b4144,0x000ec9b81308724e,0x000c0385af63d00e,0x000069d0017809f1}}, + {{0x000cc7faf31bc33c,0x000170b86ea12ac6,0x000118a6da045fa1,0x00064c601bb95001,0x00000823d360ce62}, {0x000d0907a77ed7bb,0x000c5e207baeca4a,0x00060d6267b9621f,0x0000becb62bef413,0x0000d67d9cdb9a0f}}, + {{0x000944561b2ae48d,0x0007606ca32aa2ce,0x000a0f576a73ae7e,0x0001f59d79acd4f0,0x0000f91e47274349}, {0x000f58ed0786fa06,0x00003b6fb73155cd,0x0008196d63d3af60,0x0004a1abf0728a90,0x0000cc301947a07d}}, + {{0x000173fcda66688d,0x000968f54078a044,0x000b892a6e4f3e8d,0x0000a6e9a8aa3f76,0x000003f2358c3edd}, {0x000c73ddba515a86,0x00031571066fc332,0x0009a52dd8494f4f,0x000596f40df54b23,0x000033c2a77c2737}}, + {{0x000ccc6a38469a7b,0x0000dfeabba0419f,0x00010b4aef255f07,0x0006f5fc0d85c473,0x0000844714199f30}, {0x00044a1800f79726,0x00003e748495b202,0x000fe708b728cd1d,0x00039fdb9df876a1,0x000044dfaeed2e69}}, + {{0x0009efb3b1befb52,0x000dd3db9ff2d119,0x000d54531b9666f3,0x000ef6ae6ecaa250,0x00000a5b3d55478d}, {0x00017d0cbe53c8f4,0x000f93025e9358c1,0x000f7b33ecb21d3e,0x0009e51f5b49200c,0x0000a7b0480bbb55}}, + {{0x000419c661687e70,0x000df62ce653f542,0x000907c00a126d6e,0x00080f9eed2326cd,0x0000422a8722b335}, {0x000534595c60a63b,0x0001ede5c24ec59f,0x00040fac2dc4f284,0x000b07e5a5d36fcf,0x000093bb56108a49}}, + {{0x000250012f4ef06f,0x0009f321814949f5,0x000d1cc017a50bad,0x0007969417282b08,0x000037ff4efb3e13}, {0x00043412190a4c29,0x0004b018dc035a37,0x0002185201cb14ca,0x00062453f92f5aeb,0x00006f2a7e6de00e}}, + {{0x000cc31c7ee87f99,0x0004ff0f8aafbabc,0x000a827098c656c9,0x000768469f364b3b,0x0000447f273ba78e}, {0x00012a02b0c76f80,0x000531f409e3b92c,0x0003182511cfed50,0x000d99f57fb201e9,0x000056a73dfa1e29}}, +}, +{ +/* digit=33 [{1,2,3,..,}]*([2^231]*G) */ + {{0x000a2dbe215c3b8b,0x000f884ce45689c6,0x0001d2f99ea9d0b4,0x000bd1cc1cb2174b,0x000007f8af7769d6}, {0x000eccb56ca3c064,0x000c39f4aea85da5,0x000dfcc2e6466598,0x000634cd8c935e85,0x00006ec507784fa1}}, + {{0x0009bc96b496a337,0x0001a1b2a3f800a5,0x000d4849519c3461,0x000db18fc2569c8b,0x0000b3cfcd94befa}, {0x000b4eb10b12d26e,0x000c1e97882a2059,0x00063f99dce2a002,0x000c447d77efe3aa,0x0000db216f26ed6a}}, + {{0x00018c63ece662c0,0x000a493d31f196f3,0x000906fc16932e12,0x00049057ef719d82,0x000018862ee36f81}, {0x00038809d700b8c4,0x0007c005edd01de2,0x000ff477e0932f9a,0x000e87c6433b3745,0x000003ad9c08d54a}}, + {{0x000c11886bc24c43,0x000e560aecce0485,0x000f9b2977cba2fd,0x000e25b0e545401b,0x000025a455e793cd}, {0x00031a786e57de70,0x0007942d10e0c90b,0x000728cff7ca8ade,0x000d9169618b125a,0x00006ddeb0245afa}}, + {{0x000527e462b3815e,0x000471040d6d3d81,0x0006a10bf201ce2e,0x00030111185978fa,0x000050eb91c129a9}, {0x000e282a5d067477,0x00069217051858a0,0x0004001b5ab39043,0x000546a05205fa47,0x000092822e494ea6}}, + {{0x0000adf6b5dfc7f2,0x00013e0797f2d1f2,0x000d16de2c2918ec,0x000bf5a4b2e613de,0x00009b8761499478}, {0x000f016f7b3256e4,0x0003de3e31def4dd,0x0009e833ac658d9e,0x0002df9cf95488f0,0x00007b7761d27f55}}, + {{0x0005841c4e740ce0,0x000b177b99bc2a6d,0x000904ce22809135,0x00082f03a03fc8cf,0x0000705c0bf36b5b}, {0x000056724d3e873d,0x000a122406826f04,0x000e68bc339873d5,0x000fe7ba392091ad,0x00008baee7844c64}}, + {{0x000929675be6922d,0x0001213a70da1b6f,0x0009a47bf6932638,0x000303b2b11e85f7,0x000061c91b9a2552}, {0x000f571053bef378,0x0008b2051d390a9a,0x0007f02ef5fb0b6e,0x00041047f4682ca1,0x00009401362dc590}}, + {{0x0007de12cad37a4f,0x00067f0d1f0a5333,0x000e96610c120cd7,0x0006283c745aaf2c,0x0000076bfca61a10}, {0x000798e91fe1dac9,0x0001ad2dd163c294,0x000e1f0a0c8c6588,0x000935de3e18d763,0x0000ba965532ca62}}, + {{0x0007623440b9b877,0x000a956d4db76d55,0x00020f4c8385ebbf,0x000d33d8735cdb8e,0x000016f0b0a75dfe}, {0x00036c0ba0e7092d,0x000d6dfad54aad69,0x00060683455d3376,0x0007bf127965ffcb,0x0000681b61bb1f63}}, + {{0x0009fcb83e9c7da1,0x000095ad4e66dedd,0x000319f0c0213304,0x000440bc0916b195,0x0000b5e953298188}, {0x0004bdc725dd35fc,0x0002eb60236de83f,0x000f3212cba2f5b6,0x0006b84c9c629414,0x000096bd8d34a8c1}}, + {{0x0004d6dc80f87d6a,0x000f7d8a2163bc94,0x000ceda80be2312d,0x000b2aa43026be75,0x00002edd166aede9}, {0x000c606ca8c0666c,0x00010c2bcbd751bc,0x000382e1aeb1ed71,0x000c659561279121,0x000056fe8e73cf6c}}, + {{0x000adbcbc1c3b15f,0x00046039a4313266,0x0002f2dea642ad5a,0x0007adadcfbaf904,0x00007547c5c21508}, {0x00015c16ec80761e,0x0003fa9e8435733a,0x000d36a8d8a3a115,0x000a4ddd0a6919aa,0x00009e7377d0e427}}, + {{0x0006e39e2d28ea84,0x000e750985b22c70,0x00029d6b3707d572,0x0000eff5dcca6438,0x0000aefa25e7768d}, {0x00017dff52661252,0x0005251d4caff7af,0x000902575f409232,0x000c7c7a8e670e6c,0x00005774dda3abf1}}, + {{0x00001285e5eb464b,0x000cecbc4c7510fd,0x00036c854df0124e,0x0000a49a1c719cf6,0x0000cc1c52a5ebe0}, {0x00070df7350f9229,0x0001991f61845b72,0x000192d67dac63a8,0x000be321af84188b,0x0000748d9575b564}}, + {{0x000b281c9947edae,0x000fa609d25cf20c,0x000adaa68d459e53,0x000b86b0a79588cb,0x000097fec12d5258}, {0x000b88cb27d7e1dc,0x00050c5b8023206c,0x0006b92572ba3d39,0x000d3a3da394ded9,0x000058c9b408ddbc}}, + {{0x0005a17512d500c9,0x000bf90baf002f44,0x00044b074849f6d7,0x0009a667b85738a4,0x00001d723bfeecaf}, {0x00019a4f16591f2b,0x0002d5b54c67aba9,0x0003cb20eafab933,0x00052d84b720e282,0x000036c1b31c4a53}}, + {{0x000dc48359ae88b3,0x00095b94925a7673,0x000d28c38c7da6d7,0x000449b9611f43d1,0x0000507e76bb8c39}, {0x000a4109b143d327,0x00085897b4d64670,0x000ec856db407a04,0x000c3391b35a7c2b,0x00004729f04e48e2}}, + {{0x0000e287d444576f,0x000fa46c0fcedfc1,0x000c4bc495541641,0x000dc4c61a898da2,0x0000c49474474950}, {0x0005d34a86e99ff6,0x000dde84f0eb279c,0x0008b517448eb4a2,0x00020f168c48d9af,0x0000d2e2947e6a6c}}, + {{0x000d4b7fc5ddff94,0x0008b8cd5d76c4ee,0x000e6ec7533c62e9,0x000b64d05167db0e,0x0000723b4547dadf}, {0x0003255992390795,0x000cf88d2ea48646,0x0007f30d8f37975a,0x00001460943feac1,0x0000681ac9cf176a}}, + {{0x000b8b6daae521b7,0x000adbdf2be35556,0x000f2c0e8441695d,0x00052433bb1bc3fe,0x0000a6eeb8381b5f}, {0x0008b6d0a7d65ee1,0x0008d0ed4f2aa649,0x0009d15a7352ac47,0x0008209e26a67a00,0x0000eabbbcca5492}}, + {{0x000ce07d1a83db8c,0x0001217601ca6e80,0x000bf1f026fe55be,0x00067c6a6a9b77f3,0x00002da4da482151}, {0x00050ce0660bb9c8,0x000c647dbdd30a3f,0x0000ce6acc0badf2,0x000af2afe2cb4eb2,0x0000df11e844ac4c}}, + {{0x000bcbea50bf7b2c,0x000d6fa5e1ea7e76,0x0005104e4d183277,0x0003530de5287e95,0x000061dbeae917cc}, {0x000361797e3c36a8,0x0006a4c11975d8d7,0x00056ffe587eac28,0x000ec4cf96fe62e6,0x0000ff2a60a5ece8}}, + {{0x0005b7d6b5c17565,0x000e725336dfb23f,0x000f699eb5967d95,0x0001d71cc4a1282c,0x00006f2d1e5dbace}, {0x000cc2260eba8909,0x000a041a0642f2b6,0x0009abfc6cbfaebf,0x000997d9347b4051,0x00006c695a44b98a}}, + {{0x0001ae8291d465ef,0x000694bf89326d90,0x0001c482f1ffe9f3,0x000d2382bad4525e,0x0000168c52db0bcd}, {0x0006017568b375d4,0x00089290b50e580a,0x000f160a15743a92,0x0005551201b0040b,0x0000581e3cdcd77d}}, + {{0x000874ca4cd68d8e,0x000f8c4484600f4f,0x0005c73324d85176,0x000fecbac7c97f28,0x000079aa93211324}, {0x00075824d0274253,0x00064d8b475d6b62,0x000eaa57375d9aa1,0x0000f362be914bdd,0x00004995d6eeabfd}}, + {{0x0009820f41846ba3,0x0000604085981174,0x000978a94fd1bcdf,0x000d0ba2fed0a907,0x0000ba1d2eda2efd}, {0x0008512656b5739f,0x000594918c39e3e4,0x0000b6224e9c896a,0x000d5bbb1875d792,0x0000b93012187315}}, + {{0x00019ed5ba8931b4,0x0005acd90608de3c,0x000b33a89ff0f3a9,0x00089e41298c1df6,0x00000292d30c3fde}, {0x000d7df4a273bfe2,0x000087b011c26348,0x000a2dc24b8b9f1f,0x0000ad39d9d45852,0x000067daaf3663ad}}, + {{0x000dd476f0bcde8e,0x00074f1aef494e4f,0x0004bfcbf6a3568b,0x0004f71520bf6fc8,0x00000306160f678b}, {0x000e43f9114c874a,0x0006a66a6723e3a0,0x000fa916c94626c1,0x000e1a4c6dbb2fcb,0x0000a2a7851b6164}}, + {{0x000f6c17b91cb2fb,0x000022d25723d046,0x000c4a5d56108b74,0x000423bab6def6ce,0x0000c2cabcf84f05}, {0x0009b3aeef06e341,0x00032ae65511bd33,0x0009815f724c639f,0x00015a5666c8aa1e,0x00004c097c5786e0}}, + {{0x000b7ddec96a31f1,0x000bb6126e7d0171,0x0003107612ab4881,0x000f991937b21ce8,0x0000c2f1836cf5e4}, {0x000f324d669b052c,0x000bbf0d0024898d,0x000fedf68f963603,0x000e5446be060625,0x00008eaa3a89c4b0}}, + {{0x0003578addae3db2,0x0001256b65bb4a56,0x000f89bb28f673ae,0x000ce0f3897587f4,0x0000e7b5627e000b}, {0x000373d4ebc646ed,0x000c26c01b67d1af,0x00046e755320c491,0x000df2ed183b1a53,0x00003dc80714e3ef}}, + {{0x0001d181707854b5,0x000e045ed6440d70,0x0008141226b65beb,0x0005fac44ea8ce1b,0x00004d31afc2b255}, {0x0000f1d4fd98ed23,0x0000361122a75ad2,0x000dba8c53597134,0x000fd9b5173a0d9d,0x00005c6b22beb557}}, + {{0x0002ee43acde4fa2,0x0003572b4de2188c,0x0009e491b7321151,0x000935ef340f8681,0x00003ae38901b2ff}, {0x0001fe8fde235367,0x0003c3ac4d9e99a9,0x0002c2b7c3253510,0x000130bcc3a87d5a,0x000018cf5dc1d05b}}, + {{0x000b0ea388337482,0x0005bc0ab3d81a29,0x000f26100c68ffe3,0x000c3d24c4d4108f,0x0000f14a32ccd1dd}, {0x0000a315097789d0,0x000b428d4c71c703,0x000a782fabac5dc1,0x00014d3bba4799bd,0x0000e5c67c1d3f71}}, + {{0x000dee9bee66b26b,0x00040993141604b7,0x000f8818745348d7,0x000064a5cf2b66d8,0x000035fa570e7a60}, {0x000e3b2e90a9f6a4,0x000a93d618844b8b,0x000e1533c3136043,0x0004b406fc07f2f6,0x0000e68110e4d8d2}}, + {{0x0009c06a197144e0,0x000a0ebd90e8998b,0x0004637e982d59c0,0x000346ad1dcaf45b,0x000059c29fb15ffe}, {0x000e68bd0de11ce1,0x0000ef9123f1e4ad,0x000809e2e7028807,0x0001a38b44fb792e,0x000021fdeb0bf5a4}}, + {{0x000b525dfbbe18f4,0x0006ce6bebd7328a,0x0005014218d893eb,0x0007f27415906d1b,0x0000683c30795a82}, {0x0003859154db22e4,0x0003c459a6b28d30,0x000d54f8a9fdddf8,0x000441f5e0dcb8f5,0x0000b323984a4542}}, + {{0x00020890ea27494a,0x000e1e35ace9abf1,0x0005949a7637aefb,0x00092a65ed6aed2e,0x00009fb6d9b1e31c}, {0x000c850fe43da55b,0x000c7a2aebb47154,0x0005a1f29c49139d,0x000f8c8a3b6609e8,0x0000c0e447c73747}}, + {{0x0003b83c899e16c9,0x0007290a4c485b88,0x0005c11ac3264056,0x000b31c23fdf0557,0x0000d6294d704452}, {0x0005adda6fd01b3a,0x000ed638d76cebf3,0x000bc2a591640d40,0x00090bcb6beb8873,0x00005774079636cf}}, + {{0x000d07a4df05112a,0x000c75ef10b3ed9e,0x00000dc454558b1b,0x00068ee57b1c4e11,0x0000c05303af0df1}, {0x000ff70dbf770c49,0x00078abb8add6239,0x00051783cf1da24a,0x00085717000ea909,0x000083c07443aa20}}, + {{0x000e669c9aa38ff1,0x000cdf9b6663ec67,0x000fbaf822c76f14,0x000d8f2e05f459fa,0x000081bfa0e72146}, {0x00031960fccc9403,0x000079ddbab69572,0x000c0f236afff64a,0x000c6b8a2dc2b77e,0x0000cd9b9db0072c}}, + {{0x000f50005e38e681,0x00011d2a53bfce83,0x000d3127de4fb9f7,0x000a7389040d2691,0x000090991b1df4f9}, {0x000f4d487fb49c70,0x000acbf3cd2d4b92,0x0004a8e9f3453a0d,0x00056284948bf480,0x0000bf9b42b8763c}}, + {{0x00085bea7295b89f,0x00032c67fdbe5c4a,0x000dfa99ff9b5214,0x0003dcf7dd419d4b,0x000053b1b1f00c14}, {0x000fc48e64a06587,0x000c86e4c0bd1189,0x000f07b904597445,0x00012361c88b11d0,0x0000de6046f7d84d}}, + {{0x00087bbacc585b5c,0x00054110d0cc505c,0x000cc9394ff5fa6b,0x000013753c1362a9,0x0000f747b69989a6}, {0x0001acdff683da86,0x00085c9ade093e40,0x0009346cc256f729,0x0003da9907d05802,0x000068d87f52e4e5}}, + {{0x00072320892f7939,0x000ecfb65e03e668,0x00005b60e8867af8,0x0006db6855960ac2,0x00007cfe82946f8d}, {0x000126504e895a59,0x00082f9ab1ae37c7,0x0003475b8072a97d,0x0009b271cf682e37,0x0000a001e427da0c}}, + {{0x000f56aa1c2ac6a2,0x000b0056bb5e8607,0x000840d5e7b5a2f6,0x0002a8575fce01b3,0x0000bd0d02a2a11f}, {0x00020acf5e461b5d,0x000143f8b7882cb9,0x0005684f429c8f84,0x000b43261bec02c9,0x0000976e4c74bcf7}}, + {{0x0008818531ad28e4,0x0004e2d377207b87,0x0004885027f26899,0x000503f289ead51e,0x000098cca8ee68f8}, {0x000c970b5c5d77dc,0x000080459981f27b,0x000b87fa1544bbcc,0x00070d2411cca590,0x000054ea81cf6071}}, + {{0x000086e61ca59dc9,0x00021cd8c326fa3c,0x000811dadebe13e0,0x00041b5d83482a0d,0x0000a49369b21466}, {0x000c7cd8304d967f,0x000e9364df0d6130,0x000343b0e27f449a,0x000d17ab76d37fee,0x00008a1d0ed05ee5}}, + {{0x00046cadc6205653,0x000ab65343bf3181,0x000556549569c932,0x0006bd0af34c44f4,0x0000ee209046d270}, {0x0000c0ef40c6247e,0x000dedf91b009463,0x000bb38be3f102c0,0x000064d340ea43f7,0x0000e2d7c507278a}}, + {{0x0001fc4894efce0e,0x000d16f705c791bb,0x00001c6f5dcb8966,0x000e5e4eff50cb1f,0x000096a8482ad3cb}, {0x000dec26d0cfd830,0x0003040d53e317a4,0x0007f8e26336612c,0x000559bd95bdd1d6,0x0000ccd1e540626e}}, + {{0x00068bdee681a9fd,0x00019176d44b5bc5,0x00088a36d1aaa6b0,0x000c7bf9e56290f9,0x0000b5d955ba2779}, {0x00025fcd134d4965,0x0005ff8f9c5b8014,0x00085dfc847e63e2,0x0001a0634eecfe39,0x00008b0e4909de31}}, + {{0x000397d6afe3edfb,0x000ead213cf0c484,0x000fe129db20b9ba,0x0004aa03eebcca16,0x0000ac8b193bcbab}, {0x000d00ec766789a2,0x00007b641c6280af,0x00012d36a671a1b6,0x000788d44e1010d6,0x0000e906a8a2cc4f}}, + {{0x00085346f1857b14,0x0003410a4089af78,0x000c2398d5cc5098,0x000b93ca8a5f67b0,0x00004695f3af907d}, {0x000f016bdb075823,0x00067ed27633c156,0x000a99381e81d34c,0x000d88bd4282e4c4,0x00008536f5808ba6}}, + {{0x00099ab2afcd854b,0x0007d8eda94411de,0x000c8a192d747cf5,0x000c2d246295bece,0x00001424f4c40f48}, {0x0001c25b3f186f40,0x0003eb32b7394006,0x000ddbfc3b2985f6,0x000d7fda0dd1d8fa,0x00009df5df39c2fc}}, + {{0x000736cb9995a1c8,0x0003f161f3b15ad2,0x00016b96e4f367b2,0x000c18470c49cfa2,0x00001b9d55b9d130}, {0x0005a5f78a9f0c67,0x00021aa433e2085d,0x0000dcf549740986,0x000faed0c9af8ba4,0x00009380b92d46a9}}, + {{0x0002c88c54018551,0x00029c0673e5a8f4,0x00054155f79a8500,0x0007f49c76fa0d73,0x0000e0e7821cfcaf}, {0x000a0f6255d3014f,0x0001c980f9fa8f28,0x000cc0b0ed7bc620,0x0000f657b694eb76,0x00003dc3e7fd436b}}, + {{0x000900f0e2ab22a4,0x0007485db521beeb,0x000e657e4f9fc7b6,0x000645a1ee59ad38,0x00005db32d0f9721}, {0x000237fe16ed3e0e,0x000bbb7178cab7d5,0x0001c17f2278f133,0x000c4cba85c30874,0x00007993168c23c7}}, + {{0x00019588e7f16445,0x0007acc30bbdcc18,0x0000d00fb3765dae,0x000e56c47a4fe57d,0x0000540123b0db2d}, {0x00027a35c65a51ed,0x000209b84b2313a2,0x0007634784e94bf1,0x0001a877d76a3143,0x0000a49efebe4d8e}}, + {{0x000c9dbf562fa402,0x00091c7548693347,0x0001d08b33389499,0x00003924b7a2ed2c,0x000073ffa2df5839}, {0x0004bddada7df426,0x0007a35d8ec44b5a,0x000d0629a0a53add,0x000c22216b46cf5c,0x0000156ae4a410f7}}, + {{0x000684c43a0908b4,0x0008578d18268934,0x0008e8668505170f,0x000d80851c09a959,0x000081f6e0c2f121}, {0x000e6b0e93141606,0x0003960731d37a3f,0x000fc8e4b7861d75,0x000aec6eb173ec4d,0x0000839de40cf2b4}}, + {{0x0009da66960a1d07,0x00027d771fe97d30,0x000fac3ad619dafb,0x000baf9f688aba90,0x0000f8d73eab08be}, {0x0004a8a35306c347,0x0002ece26a385f6e,0x0005297e499a6cb6,0x0000e956434c7490,0x0000f8fac1dfe93b}}, + {{0x0000533de36faba0,0x00016d1b717153e6,0x000e7b6a46cd71d1,0x000b0d8454ddf26c,0x0000d2dfeda7af6e}, {0x000a49d8f229e942,0x000aeebbb5a51c05,0x00026ea87b89faf8,0x00040f65455941df,0x0000270c9e42b6c2}}, + {{0x000fccb792331f1f,0x000bbd5f4310303e,0x0005487a9aa841df,0x000687415985039d,0x00000722f292f10a}, {0x0002b63fa69dbac1,0x00055c77189d6d9a,0x0001b8b7eb40e9fd,0x0007b2ad14152df7,0x000026e00b8e2fe4}}, +}, +{ +/* digit=34 [{1,2,3,..,}]*([2^238]*G) */ + {{0x000164b77ff9c88f,0x000a2ef7eb4901c8,0x0004be8651e6510a,0x0003146b0427bac9,0x0000014ba0639f38}, {0x000079d912a4b87c,0x00053718addb5b8b,0x0009350d61f10a85,0x0008cd383ef5b3aa,0x0000b3050be022e5}}, + {{0x0004372d2fb6aded,0x000801214c59b7ee,0x0009a805676222c1,0x000393e75f878b61,0x0000bc0e9d60d556}, {0x000dcc68fb3aa122,0x0008198ce83ad58a,0x000c586f80dd8a70,0x000608da5fa2a4de,0x00005c1fbffecc79}}, + {{0x0000cd3410794975,0x00002f17f97e1c7c,0x000edb80180e62e3,0x000677d857c7cf22,0x00003c5dc338930a}, {0x000919ff266e4f08,0x000d7ce8b10eb218,0x000a415a98781060,0x00067eb9c65da11c,0x00000a82892a18eb}}, + {{0x000e99ea38b2555e,0x000c2d0f78b90aed,0x000f899f908f42d0,0x00045b85d2246e6f,0x00006affd17a8110}, {0x0007243c95adc81b,0x0005ff1b55034f43,0x00020858fc587c33,0x0001203202a22342,0x000031c328f6ed23}}, + {{0x00094131d1a510a8,0x000fb492e59d9b18,0x000d46eeacaad4c2,0x0001e0f8cfbc62bf,0x00002d9b92bdc756}, {0x00054b85d74c27e5,0x0007ec6f99df2367,0x000ce9d06698647c,0x00058b504025de05,0x00006aaf1cab6c68}}, + {{0x000994dd11fe6a9d,0x0008eba83bfa0e00,0x0001b31f14d4500d,0x00041e927422e49b,0x00000c3d5fda89be}, {0x000bfc9eaa562805,0x0000baca1c2c7a32,0x0000a870fecfd05a,0x000e1425337b4e23,0x0000170a512337a8}}, + {{0x000ce57b66ce8b33,0x00017e3f3e322882,0x000fd4d1b4d897aa,0x000a30a6df75816f,0x00007c5429d1c726}, {0x00001a389212e06f,0x0008462c37e1fc25,0x000bf5f267708a70,0x000088314bab6b3b,0x00003e8cf2215d7b}}, + {{0x0001ca59294b3b6f,0x0008266971b412b0,0x0003fc23a009e882,0x00035452a7149569,0x00005836d205ffc3}, {0x00041dbb019e639c,0x000004f228ef2720,0x000c74f69a1c8850,0x0009110f82baac62,0x00006c6dc96e97f4}}, + {{0x000cebdedd438eb2,0x00067491a855ec0d,0x000b83f8e144f459,0x000c0ddacb21f3c6,0x000017a832750eba}, {0x0002281cf11b5e08,0x000c27889bbd621d,0x0000eb5a495a0f12,0x0004fca13a692b6a,0x00009360a8e55b15}}, + {{0x000c8e9a166a355e,0x00013d1882ac523b,0x000719f73df1639b,0x000eec37a772fb6a,0x00000b5f4f23d476}, {0x000f7e07224ee886,0x00021d0a2a679ee5,0x00027786b2648c3d,0x00080ad189601c14,0x00005554b996f704}}, + {{0x00085c70ed8b7a07,0x00080ff5ac69721f,0x000229dbea53ae19,0x000124751f671465,0x000026c04728ff80}, {0x00077fee956c748e,0x0005e60b02d0294d,0x000be3040a2ee31b,0x000b41001c513f66,0x0000124ec995bdb9}}, + {{0x0002a05eef749b88,0x000bb345f0296e45,0x000ca0decb1b42ac,0x000a45c34c8832a2,0x00000ad9447c3da3}, {0x000bd418f40be07d,0x0004656d8d611385,0x000b4a01a333c0e0,0x0005b7da85289120,0x0000d53f63e89842}}, + {{0x000b1030f7484975,0x00072638cc84ef88,0x000da3c65c73ce22,0x000f0fcae17f1301,0x00006f3f5a252d97}, {0x000c09e996d5598f,0x000a57310512997d,0x0002aa7f9e694253,0x000b70e1bfdb23ac,0x0000064f9d6804f2}}, + {{0x000e6573eacaea0b,0x000e59b058bd265c,0x00018847df779bb5,0x000c3b678c0d39ce,0x0000ca9ee0828efd}, {0x000a29bc7409894f,0x000c8845e22607a3,0x000bdea795bbeea4,0x00059242b80706da,0x00007c709434bcc0}}, + {{0x0008ffb240e3f455,0x0008556a6fe6a748,0x0003f5486e2d2ded,0x00044844798dbb44,0x000001050e8f96eb}, {0x0002f32ccee8cca7,0x0005765b412f7202,0x000a163954aedf33,0x0000057cca4d0d83,0x0000e78e9b71d3f0}}, + {{0x000ffd0f38bd9188,0x0009b09114aa4975,0x000b8b6035798f34,0x000fb588ff0f5489,0x0000206e2219ddcb}, {0x000fcf8f821d742a,0x000dc1fbd6f34da9,0x000702c041ed9195,0x0009dde18d25563a,0x0000b89d7575171b}}, + {{0x00061fbbe6d6db62,0x0006e63bc763d31c,0x000286d321f07966,0x000195bc5e14c35c,0x00001600cfa9dd08}, {0x000c3c815ed8598f,0x0006fed767ed31a9,0x000fa58d2625008e,0x000a2a33e8005f67,0x0000a1b81d42e593}}, + {{0x000e39db02b697a5,0x00078d8dcf8a266d,0x000715a9abbdf72f,0x000d03033cd5e0cf,0x00003393e3d87890}, {0x000b10a596c6ff72,0x000ea2c1ffb438b0,0x0000785544e805be,0x0000a95e6d16d794,0x00004806b6418ed2}}, + {{0x0003c28b02565d5e,0x0007d5fa9a84ed33,0x00087b17db552969,0x0007d18d709db6b2,0x0000409c35a8857e}, {0x0000777198594e1a,0x00013ba1996d94c2,0x0006d6c719c95731,0x000a209aa81eefbe,0x00006ea2d1d24a2a}}, + {{0x0000e3d894cee1e8,0x000477071278725a,0x000e8d65d9c5ec97,0x000993f9c900d4bd,0x00009bf052b7d32a}, {0x000ac8b02844a89b,0x000f631c9d556098,0x000ad2c6940519ec,0x00089816ac904985,0x0000c65a021584cb}}, + {{0x00047d3f3cd9f24a,0x0001c7f8a5e5779c,0x000bbab07ed17207,0x0004e1d7fd448a70,0x000021c3bbf36482}, {0x000b4d80d3b11d25,0x0001b97fcb26f826,0x000f528f4f86db6f,0x0003d789800557ff,0x0000663c86f8f635}}, + {{0x0009ee695e5a5075,0x000b919d6729edaf,0x000b61dd4a9723a0,0x000611b1586f96c3,0x00008566b6eac708}, {0x00090af73a830815,0x000615254047d580,0x0007aef29bec2d5a,0x00092f6291bc2b1e,0x0000eabbda6cf8ef}}, + {{0x0007c0801e864ec4,0x00015d26f447ee51,0x000264c5b394057a,0x000ab54aad486936,0x00001a0403c8e25f}, {0x000f7a8a4659ea6a,0x0002d1bb980af413,0x0003681ec617ee00,0x000b355725932b75,0x000004af07f35eac}}, + {{0x00090a00a5ea679e,0x00042580d3b41835,0x000bbd5c4dea2f9c,0x0006aee218b68b26,0x0000c25f328cab12}, {0x00064ebe2ea67e19,0x000017ca33af5ff5,0x000e8443bc7fab12,0x000fcb6ed74b4cb1,0x00005ea985061107}}, + {{0x000c21ef95df3779,0x0002c982d5270ba1,0x000ba11436031d1f,0x000a1a8a0967f9df,0x0000ae6a5eba6c26}, {0x00012ea2c092ccc9,0x0004117991571e9b,0x0004fed8c28f898c,0x000f9c7d71f312d0,0x000041fa4fc14344}}, + {{0x0008d9030a2bad30,0x00069b5a65848afc,0x000b84311cbe35b8,0x00058c03c5566460,0x00008cb2a3f860b6}, {0x000df7536be38e1b,0x0006d87e7fa21eb3,0x00001ad6290d1f0c,0x000eb810bf6f5cf1,0x00003208e36ab1d3}}, + {{0x000df5d231c91e4c,0x0001b04babfe5cd7,0x000ef58bbdc77085,0x0002bc2357ac532e,0x0000b5c3bbcfa9b1}, {0x0004ae41da37806e,0x000d71c7d7131ce2,0x00003eb2a144c5bc,0x000449e6a7caabfb,0x00005ca49bed40b1}}, + {{0x0003b89bd18f3d12,0x00016f2316cbf28d,0x000482896f8e6c7c,0x000e778e62adc34d,0x00008bddfc697813}, {0x00008eed3e3e9d68,0x0002fe9f8a3d153c,0x000d994921af5ead,0x000a79c09952b705,0x000069094582de86}}, + {{0x000602c441b88cd0,0x000bcc20f7e71964,0x000ececa70c2a3c9,0x000f16fa225449ec,0x00008d5f586ad324}, {0x0001d115f0909ea3,0x000fc97c961679ce,0x0004d1588ead7203,0x000773fdc146623e,0x000053a8ea43b1f0}}, + {{0x0009c4809a0bba36,0x00009d9a25c286c4,0x000d5a52060f0403,0x000cff7d9efe137c,0x00009b20f1c2576f}, {0x00078ce06fa3e86d,0x000c77f529c06f14,0x000c1aad527c7769,0x000a54cada78329b,0x0000cfdc46bf1bdc}}, + {{0x00010cd2058985aa,0x000552e40f6a34b4,0x0003e1a2087169d6,0x000feeaa1773f3ab,0x0000a8f013b7117a}, {0x00075210df843d49,0x00035b92e42bd467,0x0006eaf3ecce3434,0x00067ed3af118378,0x000053710064d275}}, + {{0x000b1bba80892cbc,0x000c43c116286ff5,0x0000a537bd1a5cb1,0x0004dd41b6297424,0x0000c9e812b1f69e}, {0x000fb4b5fc8601f1,0x00025def2d88f167,0x000097bead9b6112,0x000f8c0dec297878,0x0000a5bfd4cccee3}}, + {{0x000058b90fa84dff,0x000b95e24fa1bb46,0x000c99c141fa4b7f,0x00021f36fc4b15c5,0x000072ede8886dd4}, {0x000db218df104f8a,0x000d8929fa061e56,0x00031369dba8acfc,0x000139ca445c97ab,0x0000aa7da404f985}}, + {{0x000459fd466786f7,0x0000835940e6a840,0x000323c0080d2d9b,0x00021bceb6682b6e,0x00000ea0e0da968b}, {0x000b8e9e1c90e07a,0x00000efe4121cbbe,0x0000849c5d166c93,0x00035bdf729b9cad,0x000020e1465218fc}}, + {{0x00062a0d6f379255,0x000b70dda2ca99e5,0x000ecaa30b5d30e5,0x000a688e3a88f1fc,0x00004f71443553af}, {0x000916b0bbc4b3da,0x000778f700e9314c,0x0008470ccc4bf1a1,0x0005e5a395ebe872,0x0000286cde27e46b}}, + {{0x000f5cdfe6b7d846,0x00082cb76457a1e3,0x000f3b9f74697b1a,0x0004560f07699d92,0x000028a5e6885a15}, {0x000742ef4c2f1f4d,0x000e56fb6d72c4ed,0x0008faf1ba1dca49,0x000ca351c97d7358,0x000087771442b909}}, + {{0x000d00ef6e81de3d,0x0006e2eef62dd862,0x00034f24d63d21af,0x000c7950c94d5766,0x0000a5a468b10066}, {0x0006314bb09920b7,0x00046433ceeafbe0,0x000590fbbe003195,0x000e38446743c58a,0x000060b8843b0644}}, + {{0x000c8a2f33ef7892,0x000014bcb8cc69d1,0x0002dc33430b7a34,0x00007b190c3a3959,0x00003317543deedf}, {0x0009efc31735b604,0x000fda1ad6889634,0x00021830c3784c8e,0x0007f7a98ea1604d,0x0000198b9ed3ee3d}}, + {{0x000802942dc95293,0x0006c607840b614b,0x0007638241011618,0x0001a891626667c7,0x000070fae3ebdba7}, {0x000e767cf31f42c5,0x00037886da27ff5b,0x00092f5eedb3bfca,0x00092071df531c71,0x0000a96add241b4d}}, + {{0x0003414c0c83975e,0x000af2aeddc23ead,0x0004d9fda7d76f8e,0x000ff1110efd9e98,0x0000f15023276067}, {0x000a0520d36f0d27,0x00065322372861ad,0x00021d9344f19d56,0x000a4fab8999bd58,0x00002a7b9e9d02fa}}, + {{0x000e6f66be15bbbf,0x000516ab8735bf29,0x0004d4bebda1a8ad,0x000fa68f061186d8,0x000009f9261ec025}, {0x000668b5b3fc0a45,0x0004af0a2c0ff9a9,0x000282c2bc1a8bb3,0x000ddb6c3b1e0b34,0x0000492f782c6d6c}}, + {{0x00069b4c267cd575,0x000f0f8ff694e887,0x0005b7442ef62630,0x000ae31d653545b0,0x0000bc15dfb1c514}, {0x00074dc7419111ed,0x0005eb2728c3a9f2,0x00099a7d7ad03866,0x000fa05d8bfc5cfb,0x0000c82b07903c82}}, + {{0x0003de54e014c472,0x000560e61eab8f72,0x000f8cf303460a19,0x000c58bf40488c19,0x00002ba1ddf14e5f}, {0x000aed63d9564f0b,0x00077c5ee525ae10,0x0005271ab84a9fdf,0x000b311db9987806,0x0000c50919caad39}}, + {{0x0006e874c69e8a6d,0x00032ff1ff6a3eff,0x000e47e1fe98ab13,0x0008fe8b0aab1917,0x0000a9ac0b6b9835}, {0x0004bc04f8bcb879,0x00083e021fbaa981,0x00053100e36bad61,0x00027f3c2bad3e08,0x0000a510218c5f1c}}, + {{0x00079c291c3fac17,0x000e406762ecb91a,0x00013af2e1a7b0db,0x000eba4e6ce29057,0x000053a167cb4639}, {0x000c07bdeddcb8a5,0x00067183cdd67c61,0x000128f146849aa2,0x00077df6fb0823f4,0x00000acf75062e13}}, + {{0x0003ab8aa1b413a0,0x000ad9f830d7fd57,0x000b78539eb24027,0x000bf1c67bd704a6,0x0000c3195072ee6a}, {0x000bf133f8b12d87,0x000380475ed535cc,0x000990604c95a5f7,0x000535334f567cae,0x00000582b15227a5}}, + {{0x000037febb43ed1f,0x0005fbda9e9827e0,0x000aa2514e69769a,0x000cd35866ce6da9,0x000060ec889f5458}, {0x000a6da95e75646d,0x0006670efd9bf0bf,0x00024a9dc8b0395e,0x000ea954fc07a1ed,0x0000f86eef9860f5}}, + {{0x00072012ca58fbc7,0x000d4c78a8cd5b1b,0x00068b7af601e664,0x000a63128123fc55,0x00000084c070ef02}, {0x0005a8e658751508,0x000195ae0550346b,0x000d111eeefc4be2,0x000fce20fe4c6f46,0x00003a287fc76838}}, + {{0x000be4e9312c54eb,0x000f0cda2786e630,0x000d6ceae55abd5a,0x00091e346e84f200,0x000022eede6ca985}, {0x000356024525d0aa,0x00045586290c118a,0x0005c6e264b2b3b4,0x0005742743dac9f5,0x00009827e4a6a765}}, + {{0x0002a6dd9cfb4cd8,0x000f94e215cb934e,0x0007b3210b4858a1,0x000417a6b0a4b56f,0x0000befcc636ca75}, {0x00004b44b52cd3b7,0x000e7d15df655d8e,0x000f5c7d81066051,0x0008319bf3c2bb26,0x00001fd03c43aee3}}, + {{0x000a7603cb34c2dc,0x0003d057be5ae4c3,0x0002a595a6d0f925,0x000e18e4a3e6cf15,0x0000bdb8d991f000}, {0x0009ebe420d29fb1,0x000a2deac279fefc,0x000be608d12593b8,0x000c2f8c1e1867f9,0x0000eb5a9f81fbda}}, + {{0x000421680a0ed616,0x00052f2f3c60e7e7,0x000419cfa67feadd,0x00070817ed6225a4,0x0000fc0a15bd1cf9}, {0x00005b2338e5c066,0x0003160bddbc9130,0x000287049b60afe7,0x000489555a8336e4,0x0000959d1b6599b6}}, + {{0x000420ce0721159a,0x000f3b431364134f,0x000a77054c77e7e2,0x000491139bf22e1c,0x0000068bc7711297}, {0x000e3f9245e5d473,0x00083b6ccdc556ac,0x000a7a2b057ab72b,0x000b7e04442765dd,0x00008a61f6aeb745}}, + {{0x000e96dd576a73a9,0x00026d0897ab34e8,0x000fe2f8ff6c7c7e,0x000a5a99c1721c5b,0x0000d26a77a616ad}, {0x000b65c58aa4fc27,0x0003d809efc1048e,0x0006fac2acd58b42,0x0006717ad71cac4a,0x0000645d4fc1ffd4}}, + {{0x000d73a164aaf5d6,0x00013d6daf2c817e,0x00075d36a7d62437,0x000d90df8c94fad6,0x000026a8207e0f81}, {0x000826b934843b84,0x0008b84eeafc1dd4,0x000dd85215de1db7,0x000128812417e355,0x0000ae2d30deaad5}}, + {{0x00051dada3211cde,0x000c64018309c929,0x00044e2d5c6376e3,0x000d9d80ccba2315,0x000025f7a4a621f6}, {0x00027257fd00e719,0x000b2f36f2fefad5,0x00022f938ceefce9,0x000665f7ca6333cd,0x000077ef898fcf04}}, + {{0x00081bfe708b52ee,0x000c42359c865bd7,0x000be20854eefe76,0x000e15fd0ae7807d,0x0000c6305412aa5d}, {0x0007eb6ec1300871,0x0009cb3a7019cfc1,0x0003b73c3064cd8f,0x0008e6c1c89da37f,0x0000329e0ea397cd}}, + {{0x0001b5e44c3250c6,0x00006ceb294648c0,0x000b962637706782,0x000130c3b3acfe89,0x0000a6dce5b66ebc}, {0x000aa6186ce36869,0x000b1852bf94c5bd,0x0004c261dc5ebfe3,0x000f138a1bef6195,0x0000a279dbb67a1b}}, + {{0x000879d90e03a540,0x00098a4ef9ff4a24,0x000a26c0ce8f5edb,0x0003796474413661,0x00004350b4104b6d}, {0x000757164564536b,0x00057d4f4adebe01,0x00050b82d298dc7a,0x0005b791e2842835,0x0000a781123e3c5c}}, + {{0x00007c48619e84ab,0x00097a8e0f19b2f3,0x000b3c9db52cf19a,0x0007e35d5659ce7d,0x00003886ff23b704}, {0x0001aaf58811dc26,0x0002bfed52b1bbd1,0x0007104ca16ad476,0x000b79c4f1138a88,0x00007658715f7d62}}, + {{0x000f110ecabcba82,0x0004da27d56ef2b7,0x000dc00cb29c9167,0x000ea38be30c384b,0x0000551fbe178550}, {0x0009ceb4b6cda9b1,0x0001c0f8dbf0f1e2,0x000ff617a844614b,0x000b9bb65d23f3e8,0x0000463bcf08ebf8}}, + {{0x00046fb3c9c2f2f6,0x000d8bc887881058,0x000acf39df0c62e8,0x0005e06d804143a8,0x000012fea1c85697}, {0x000fe3dccde965f2,0x0008b508c90850a3,0x000e602055cb2212,0x00026281f1187920,0x0000af846cb5a96b}}, + {{0x00094218aed1a517,0x0007bcb7d8e4e5ea,0x00070f71ce083200,0x000c497f01ba571e,0x0000e6ebb21847cb}, {0x000aed8c6da82ba6,0x000ab2db4b4da475,0x000a869cd33a684d,0x00064cc689a3ac37,0x0000ae1ee3463a89}}, + {{0x0005bb9f1e1c66ff,0x000777e629c50f4c,0x0009fd3d89dfbcb2,0x000022138efb8fcd,0x00005816bef7a97b}, {0x000fefc894aa6e34,0x000e9903a64777f7,0x0001f799abf92380,0x0007f487d3d67ad7,0x0000656fe81b0c1f}}, +}, +{ +/* digit=35 [{1,2,3,..,}]*([2^245]*G) */ + {{0x0009dc05b0daeb6a,0x0007ab1c9719331b,0x0003d0d262ae4dc8,0x000f3de5109dd56d,0x0000ad73296a5d14}, {0x0009fb2ac31d9136,0x000163a6acd79e56,0x0003de3fe8aed60f,0x00087300abd5213d,0x000039462f009289}}, + {{0x00020a59b6efa79b,0x00089cfab3fb90fb,0x0002133c42d820c4,0x000a360c8d738663,0x0000971de4ead8f2}, {0x0001f841e142f900,0x0003ccd560f2a34c,0x000e1379d650f3b7,0x000d1d007fad6aa7,0x0000b1bf229b9aa2}}, + {{0x000bb724b855a969,0x000c193ee3fa5bdf,0x00081b7e4fbe8611,0x00094f40258b7810,0x000069302e6ce1af}, {0x000a01254526c745,0x0004141d611305a4,0x000ed085a7fc0c93,0x0009be3d57407cde,0x000071742a3a7834}}, + {{0x000014a8c1f3bf07,0x00073102160f5085,0x0000c42214175c4e,0x000f472abaa79acf,0x0000dfe26a0804cd}, {0x0005aae6f42d4acd,0x0001a1106a667256,0x0009bd963bc85414,0x0008a723b2b776b4,0x00002e675791ac54}}, + {{0x00067e282d8e77ae,0x0000ec55fe6a99c2,0x0003bf9a2c0250ad,0x0007bfa7c31cc937,0x00001ae266831856}, {0x000738a847ecbcce,0x00022dac7b970a0a,0x00088ad41f4b09ba,0x0004ba5b4ead8383,0x0000ba6d2a78d509}}, + {{0x000e67853c1c0424,0x000ba067d80a0c28,0x000861c400f926de,0x00063b09e1a3bce2,0x0000fc011f0af635}, {0x0003dedbb918e260,0x0000921523ea0b52,0x0007bc667773bf8b,0x00086b039e89a24f,0x00007701e4145ce2}}, + {{0x000d665659750b64,0x000d981e7b419f25,0x000fc4ca88f75ecd,0x0002d199aed274fd,0x0000d837ba7e0764}, {0x000405c0720a84bb,0x000ef4af02130827,0x00080e040098122a,0x0007d9f41d3bcfc6,0x000033b4e1e14860}}, + {{0x0007596d7efa47e7,0x000f87ae55f9ba41,0x00018c27027f5c32,0x00038f7626dadc8e,0x000092ffbbe90edc}, {0x0009f3f053fc1017,0x00081a3a20f73dc5,0x0006fe6d0df44113,0x0007b5d805ac5f21,0x0000201193396b7b}}, + {{0x00031a2aa5514055,0x000aa37a81ddbd9a,0x000cad5460bb8634,0x000043fa6ecf5041,0x00007ba1f9908b03}, {0x000b451a99c6a86b,0x00003d70f946e8da,0x0006874cbe002fcc,0x00047b59ffb89cc6,0x000098339a4a6b4d}}, + {{0x0006655d65ec88df,0x000d6e79f896eff1,0x000d38236946c197,0x000510068781a8c4,0x0000cbe735a054e3}, {0x00062486280cc181,0x000cd52f7a4924a4,0x0009772668ab80e3,0x0006ec2371a3f51b,0x000083cd4419a909}}, + {{0x00061f179879b951,0x000df0c07e6ff0f8,0x0006434d1b7fed81,0x000899e3c92db27d,0x00005fb78635f682}, {0x00007019f0959708,0x000123fd44581616,0x000f3cd7af2e4c82,0x000d794d693d2a88,0x0000a039a63f0492}}, + {{0x000a5a141c4747c7,0x000c91edbabb6659,0x0008cd292c269efb,0x00017244e70fd00c,0x00005746625ed20b}, {0x0001a886d579ca7d,0x000ce93524c3eb4b,0x000e55b6cbbeb85f,0x000ac397f9b15372,0x00008d4f84e13615}}, + {{0x00014bfea7be6361,0x0000f328d17ed214,0x000ea80165d13b52,0x00041cc1aad9829f,0x0000c636f9bdefd5}, {0x0000ae640d4367fe,0x000cfcc82e5d8210,0x000bf012b1a13811,0x000a40021a0ebf68,0x0000ad45395140c8}}, + {{0x000093d07e16c03e,0x0008c6451c1a84f0,0x000852fb8e51a85e,0x00027647baccab5c,0x00000422dfc5ea59}, {0x000ae8d127196bf4,0x000b0cd1f307d1d4,0x000f0c06f05e1809,0x000207246e68d69a,0x0000bcbd7a6ea8b4}}, + {{0x0005696102fe7961,0x000128787cbabc0b,0x00047e5568e8a8ab,0x0009702b888f7629,0x00007005dc539317}, {0x000060a74a16a3d8,0x00059d11f5a2d057,0x000d3eee179fdf94,0x000b1336970f75a5,0x0000b65be596af2e}}, + {{0x0009a12055784ff3,0x0007e7ad17886a59,0x00083e1be483ccc5,0x0009c2326844874a,0x000091668f7d203c}, {0x000ccc39b9c01312,0x00079a1a3a021ae3,0x0005da5e7959cb5b,0x000b1b1efee9f28a,0x0000e21b1fe45033}}, + {{0x000b7effce6692d1,0x000d13f99991b95a,0x000da1b42890b0d4,0x000cec15a2ba2f18,0x000018e795c7c8cd}, {0x0000ea0ad07b7906,0x00006d2815a727cf,0x000f20348a8fa2cd,0x0000bbc96ae70635,0x000009c2b7d97f76}}, + {{0x00046418aa05aea8,0x00032c224067ea96,0x000401680c850519,0x0007cca8ecdb3161,0x0000a6a937b896d5}, {0x00066b96e440f85f,0x00058873f7cca1fb,0x0001e2cd22d99b85,0x0005da4df8221d94,0x0000a05eba0d7a8a}}, + {{0x000ccce912fa29fb,0x0009036040b56b4f,0x000aaa682139e237,0x00037b99fe6891cc,0x00002cac906e3f6b}, {0x000c39a68efd2423,0x000f8bc753e95cc7,0x0000629055d8668e,0x00004e48e5d349cb,0x000092775e6d287e}}, + {{0x0007764026e7e174,0x000920bd6ff6034a,0x0009b2c9a18833c8,0x000cb370c9651f3d,0x00007e2a44c0c972}, {0x00096a65b39a0b5d,0x0001d09cbc9e0807,0x000b2736f0b90d16,0x000112ddcf44a1bc,0x000042acfae127da}}, + {{0x000a76b6b44ab8e2,0x0009970211cceab3,0x0007bcb9b4bb22b4,0x0002bea529e44146,0x0000e0960bc476de}, {0x0009eb77378cd806,0x000d80e280c90b95,0x00014d594586a4dd,0x000310e90d20f04b,0x0000702dbd838d2b}}, + {{0x0003a01c57e6e9d5,0x00050dcc40c8c9ea,0x00090cba6fe563eb,0x000cdf395633b94e,0x0000c9d1045e8592}, {0x000d2588f46a6120,0x000b035c6210c59d,0x000e8248377f9aa0,0x0001aa4bc93a5152,0x00009dc56ac7a30e}}, + {{0x0000c564a8ac4c7c,0x0000d81af1627a56,0x00070a724e6dfb29,0x0006e7fb2bded10e,0x00002349505c61bf}, {0x000695464c03716f,0x0003bbcbebaa5c11,0x0008aabafd07239a,0x000af38c2eeda57e,0x00005d614c1d1436}}, + {{0x000e2d0f37b76522,0x00070e55eb8f1e79,0x00012eab7fa064b1,0x000522fe70f9abc2,0x000086b2a1081854}, {0x000fa02edae5aa1f,0x0006c5bece8a7b0e,0x000db187a76faee6,0x0006c55f3bafbf59,0x00006262ab51cd1b}}, + {{0x000bbe31e35d4f00,0x0003907a6029f9fd,0x0001b672f257cd8f,0x000429f29248e19a,0x000039e13d803d69}, {0x000c3c7908e58df4,0x000b0c985ea865df,0x00061454801424f8,0x000683ceefa02118,0x0000ab80f32400a3}}, + {{0x000ccbfa3d84058f,0x0005df204d3c0698,0x0009081ed4ca5c5b,0x000e64a883950e9e,0x00003c31bda6008e}, {0x000eba6a97909010,0x000507b1fb8d71ea,0x000c9169209e9956,0x00037554f52879e7,0x0000877f94c1ed32}}, + {{0x00041833560eed57,0x000a8ec17822793f,0x000852b85ba8088a,0x000eb3fbd9a1b805,0x0000c928511df25b}, {0x00049c211a3b0cf4,0x000018434a1def45,0x000091951e39b0e8,0x000d49dac937eddd,0x0000e6ee324776c1}}, + {{0x00037b1d92a17bf5,0x00096a699f0dc99a,0x000418df6afc0eee,0x000e8cc36e947922,0x00009cd84baf92d6}, {0x000fb2308f6ca879,0x0004189720d764a4,0x000e233b6e8157a4,0x00069ab8b5bb2ca4,0x000095141e790e65}}, + {{0x0009a7ad4edfeab9,0x00058ba474c78c2d,0x000efd6802660b1d,0x0005206e51d4a61f,0x0000d0fffb51a30d}, {0x000df5af5e13c41d,0x0007aa82333d762c,0x000e12bdbf9f9cad,0x00053d28e2c067f3,0x0000c214dd58c9c4}}, + {{0x00078f73d96f6c3f,0x0009940a0114e12a,0x0001b5edb91b616f,0x0007802baa7c46a6,0x0000bbaf38454705}, {0x0007732bce06e6f0,0x0003fc6ea51ec6d4,0x0005ac7dbbd02767,0x000aa24c78933088,0x000003fd48baf268}}, + {{0x000f8cb8b6b4ac7c,0x00022da8657e62e9,0x0003e709fac0ceda,0x00044cb8924265f7,0x0000c2125359b00d}, {0x000a800aa4354d4a,0x000d4c76651dc137,0x0007663922833c83,0x000b7a7093741920,0x00005c10ea46162b}}, + {{0x000b2b4671f489d4,0x00020e7e4b59a1a6,0x0004a86d078a8249,0x000b641af729005d,0x0000ea40b308107c}, {0x000def837872ffaa,0x0007857e65950d35,0x000ed651235c3f22,0x00011d318b69bc2b,0x00007d4a4d9d4cd9}}, + {{0x000fd6bd4e1c6acf,0x000b32bebbbb5828,0x0006ac0ea31f3e32,0x00040df9c1459119,0x0000c2fd4bfc2e76}, {0x000758062d718466,0x00050f36b4cdcb16,0x000ce037ecddce26,0x000d9b542106c49a,0x00001395c0f493cf}}, + {{0x000af914a54fc7e6,0x0006b42c62b10386,0x000d7e90af5bd329,0x000fcca6ab95b008,0x000058cf981705ac}, {0x0006ad0ad9ea67f5,0x0002735b7f791274,0x000a4060040b2faa,0x00095a83921bae30,0x0000ffd2c5e32b21}}, + {{0x000674b3b3f874f0,0x0007d4409b672a9d,0x0000672a80e68f8e,0x000a89bb24a6bbac,0x0000764bbd0e6bff}, {0x0009c13926cecbb4,0x000855e317e76a63,0x0007ef017f7e5c24,0x0004e80cc86df969,0x00002f0f42dbf4c7}}, + {{0x000b383892850d4c,0x000fac8523726a45,0x000089ef29685093,0x0008bea48bf7c4ea,0x00008aeda7a9e560}, {0x000a3efd995bd374,0x00059297e7208f9d,0x00083d14216bcb93,0x00036a46e7fc0aee,0x000007b7c7bbdeb6}}, + {{0x000994e8c70b66c1,0x000130bc81bb46d0,0x000e193a3bdca94e,0x000df90e3c0eaa9d,0x00006ef1f3cb51b1}, {0x000e07f5cad99fe6,0x0007bf8cf7c18d04,0x0006f9681ba9db21,0x0002692711c2a3e4,0x00007a115d42378b}}, + {{0x0003015fb3d08b6f,0x000c4f6c706ce14e,0x0002810d3977d0d0,0x000113a60a8549f6,0x00007a49710c6b40}, {0x0006aaa6c9b79e08,0x00001c5d3a5208aa,0x000f719168540d4a,0x0004423dbcbdb8ca,0x00008b82b284445f}}, + {{0x000621189521a131,0x00084775b0e3f2a7,0x000ce99086411768,0x000ba2af92188e83,0x00005862571e2559}, {0x0006c25a01159e10,0x000092ee012a048e,0x000ac9e25f213af8,0x000a7eb2d7503aca,0x000090ced06d85df}}, + {{0x0007e9d9f2f897e2,0x000607493cf5a89c,0x0006d18655a12aad,0x000b1e1b64925a04,0x00004203d3d79adb}, {0x000e22319c02fe70,0x00061fa0ca3076f5,0x000fc06ebb171574,0x0003534d5b2e1b98,0x0000ffaca4b07d6c}}, + {{0x000e95aa6360241f,0x00033dbc40dbd017,0x000e41655d78a47f,0x0003ae0c5478f797,0x00000bc57e149c3b}, {0x000dae7e92fcb87d,0x0003e73d7446c12a,0x00038f6c156141b3,0x000d7b8287f1b57a,0x00002551bb07d0a3}}, + {{0x000fd03ba124dcca,0x0005595509e778f5,0x0003963210ede681,0x000218ecbd904ba4,0x000057275ec31263}, {0x00098cc31aed8686,0x0002beb9be3d4ea4,0x00082eebaa59cd5c,0x000fd1991ab4d3f2,0x00005b2b384a3ded}}, + {{0x000f10557f7f4aa8,0x0003141d09cb5332,0x000b95beb2cef476,0x0004c82395b067df,0x00006a03a2c01446}, {0x0007dfef0be79d46,0x0001874b97275902,0x000f1f7d4d0de182,0x00099478e33b191e,0x000006558b7e81b4}}, + {{0x0002253a17cfb28b,0x0008ebf590ba262d,0x0002d72dd21664a7,0x000c4587d06f2f4c,0x000078a8de65b6b7}, {0x0006920bc17757f4,0x000cced7a68a4b12,0x0009387c7e16b5bc,0x000b542e5005a393,0x000035a62cdf92da}}, + {{0x000c560972894920,0x0000c17d3c2a87c6,0x000f1f08a4bdb0dc,0x000b2096faa4d313,0x0000f5c7607dec01}, {0x000241e51762f134,0x000fb8b10c63b6bf,0x0006515f30290df0,0x000b35eeaa2454cf,0x0000f60852859665}}, + {{0x000ffb9fdfce9b8b,0x000d43b966a89729,0x000dfbd7523680f5,0x0004c7b2c6ef98d2,0x0000bddbb8a5b1c7}, {0x0009eb62b7763240,0x00018e7f61e7f249,0x000509a885746186,0x000dad9a93d36959,0x000078c35a5126bc}}, + {{0x0006dffe227d9c28,0x000f28fcd749c021,0x00055e1af43fc164,0x000360f372e79f59,0x00000013b15b60ce}, {0x000642c77d088ab9,0x000603408ad40c33,0x000e18e96bf7079b,0x00099f840d636eca,0x0000098c58ab14b4}}, + {{0x00077dceae2302a1,0x0003c806e6c54271,0x000040bb8f335915,0x00091da3eadeb94d,0x00009926a3190114}, {0x0000b030aa56fd94,0x0000717246b8a0c9,0x00086222ff4d2833,0x000a76bce47d40bd,0x000009d403daad8c}}, + {{0x00049006bdc08788,0x0002a29d68a05f2d,0x00010f595a285de4,0x000fb4d722cd48f7,0x0000b96085211d89}, {0x000ed5908b41453b,0x0009af492a0d2a10,0x00067065f4429afa,0x0000d313f5f74aa4,0x00002f8641f9b3a4}}, + {{0x0006ef58b386c69e,0x0001a846d58225da,0x0007c90a64e5c80e,0x000f8c90401eaf20,0x0000f4a430a9b672}, {0x00083d647db36e0e,0x000066dcdadc8cb2,0x00012d7df32946b6,0x00087fb7f7b1c8b7,0x00003c85bb7e9e41}}, + {{0x000f20bba2439114,0x0008ff1b65fc2e61,0x000bd0e126bb920d,0x000d9abec02fc3dd,0x000015d3897b9394}, {0x0009c1cab00a9be5,0x000d661f585cd907,0x000eaf788c0705c8,0x0001aea79d859521,0x0000252c9d8f4143}}, + {{0x000b2d546f92c5d7,0x0003d47db7b536a2,0x0009dff2df753503,0x000b555388239913,0x00004372074066a3}, {0x0008723137e3f7e9,0x0001485151c6d57c,0x0000aa2a55b532c8,0x000edd04c3cc6736,0x000002f7dcf55657}}, + {{0x0006d3ff179e220d,0x00068eb3de8acd58,0x000fd1863a8fa7c9,0x000f449cd5617fb7,0x00003a190b0c03e8}, {0x0003da277108421a,0x00071f8b6dc858d4,0x000e4ee68a60e733,0x0007f658c12f0abb,0x000025d073738153}}, + {{0x0003d6dc16b4e186,0x000656aa6bd02ef6,0x00089e652e089004,0x000d744acb852631,0x000092e329517ff8}, {0x000d8b234333e0b7,0x0001098802c15416,0x000dadb64e1c1b30,0x000c8f225decbe5f,0x0000d948a7a85c34}}, + {{0x000c6469dc81a049,0x00001a4e55e5d35c,0x000fef56b28459f3,0x00059cd0155f5c27,0x00001ebe8d5d8309}, {0x00085c4a7131df9f,0x00088fdec6667457,0x000715d18057e535,0x000cd2d684d8573c,0x0000a9ca6c7cd549}}, + {{0x000e2e3446f9e281,0x000846c258bb86ce,0x000f37d5b739bc37,0x000c9b9746166b21,0x00008414a89a8b8e}, {0x000bb1b9f25ff8ad,0x0002a4dd50af0d98,0x0005b5a78084af6f,0x000ddb4667c708f9,0x0000c1e90d79e098}}, + {{0x0000f830f7f7a925,0x000962af6600ea3e,0x00042c1e3595c08f,0x000cafae22538bd4,0x0000b3101d3fcabd}, {0x00078afde4e64419,0x00007b0a9e895fab,0x000894659c483333,0x0005c40891f85893,0x000036cd9f687b51}}, + {{0x0000fdb1e5a3d569,0x0009b0374bf3035e,0x000a9b06d21be1c3,0x000609411de8deaa,0x00005478125def93}, {0x000338bc9cf8d1b1,0x0008608556303bd3,0x0001d00508c2e2ca,0x000a2a6d1106d92f,0x00001e1311fff629}}, + {{0x00008c0c7abe48cc,0x000a32325b3f3714,0x000cdd15b7dbb702,0x0009f3c37123b082,0x0000f285904b770f}, {0x00005a5f3b665bd4,0x0000eb9cff9258ce,0x000ae583f954f24f,0x000e2d7d0276a007,0x00008112308cea23}}, + {{0x0006ebf442107be7,0x0007b1a909f1ccdf,0x000684e1ca8ff7cb,0x0003f085d46ef075,0x00007121a630d6bf}, {0x0005761fbdeb7490,0x000711a947654819,0x000865547aecae1a,0x0007358f2f2b4382,0x0000d528600f8f9b}}, + {{0x0006e9b28d21be4d,0x000d16dba462a3c9,0x00054aae39c8bb2c,0x000b130de6632450,0x0000287f45d91f55}, {0x000e421a7d459f3d,0x0002a3b3137c7299,0x00044882490d7904,0x000bfb28faff33f9,0x0000bfb8dc9f56ed}}, + {{0x0006eaf73f22dd85,0x00040159d50c4c19,0x00084c5827c96b79,0x000f295616bb5386,0x00009e99fb634b73}, {0x000fa8d3a114007f,0x000921cf2fb36503,0x0007d77d78896f76,0x000aa84fde09fec7,0x00003da7be4c60d5}}, + {{0x0002bfa99958d23f,0x000c81847d0a62cc,0x000dfe7c5fb5c76d,0x0005c6e11e7e50b2,0x000088e3280b0e5e}, {0x000682d38e3cd1aa,0x0008a2d2a17ecd17,0x000fbf0977af10a9,0x00050e0a422c42e4,0x0000a1dd9f7d6448}}, + {{0x000e96433c0805a5,0x000d7869abf94e41,0x000dc5ed1e35cc84,0x000f55a120bac5dc,0x0000d5f11d73cf91}, {0x000b444c48e7a259,0x000d3d34f5e7ffb3,0x000617df908d9b25,0x00074f19c9a1d296,0x0000b424cec270cf}}, +}, +{ +/* digit=36 [{1,2,3,..,}]*([2^252]*G) */ + {{0x0004e9d192cabe16,0x000136ac20d6f4c3,0x000b9a9a95ed7b38,0x00037f743f5452cd,0x00003e7e859d699c}, {0x000288719e08654e,0x00055a84a99d8b7c,0x000b5e1d47df6f11,0x000affb8f70d6ecf,0x00000f85856658fd}}, + {{0x000989794fdccb99,0x000664d2a5d09eb7,0x000f3dbd4a6c29d7,0x00006c08e7b46244,0x0000d3275732baf1}, {0x00078611cb5eac27,0x0009d9c76502f96b,0x0008f3ffdf832947,0x0006d50496b021f2,0x0000f3bea3e2eab2}}, + {{0x000d467bef953373,0x000f071bddfa0d77,0x000be423588d7262,0x00060e99185a1f4d,0x0000bc436399b964}, {0x0008c2ad1207560e,0x0002054fdbb7908a,0x00099c759edae8a1,0x000c0e4e054bfa36,0x00002db152ab5cf2}}, + {{0x0006019816bb79c3,0x00025d9ee9893bd2,0x0005c447c9f4d46a,0x0009c26760f22454,0x00003b25455b00de}, {0x0000cad04873a72e,0x00074e432c4decdc,0x0006750b41e24738,0x0009f681a6c25f63,0x00003d05248bbaa4}}, + {{0x000481d262409089,0x00085ea18d0661c9,0x000e20eb4c37806a,0x0009090d6babcb16,0x00008acecc7050ae}, {0x0006575fb2de613b,0x000121a2f094246e,0x000405eec4b33ebe,0x000e1064f6eb1a6a,0x0000d337e63aa2db}}, + {{0x0001242ccb9661a4,0x000e4b4b735a681a,0x000038cb78b1e57f,0x00048653161304f1,0x0000e5b9a8a398a9}, {0x0006e7de5b2bfae6,0x0000b846b3a7915a,0x000464483729cd42,0x000aefe28c607ed3,0x00003f2fb772d468}}, + {{0x000198fd906ce680,0x000617f59c8fd445,0x000002f38f184755,0x000e15c2f310b252,0x0000a4410ce1ced3}, {0x000ed8c1a5559846,0x000e72884eb22a13,0x00019d74e6e7911f,0x000e37cee1a905d1,0x00006f730d7170da}}, + {{0x0002fb6223e426f1,0x00089ec0aa53e22b,0x00041c88f08382bc,0x000f3ce723df210f,0x0000eab10c6e3af5}, {0x000e9ee919f420f9,0x0007feb7d658384a,0x000d6cb287d7bf16,0x000e7944e141cd92,0x000072357ff59e5d}}, + {{0x000c56b5bf5b4709,0x000351a00ac5adc6,0x0000eea7fa5bc587,0x000fa53e96a1bcfc,0x000037fcff36ffa5}, {0x0009f297dcabee43,0x000a97014f90ac36,0x000a4fb4897cf19a,0x00023c3e53d9a7ae,0x000088e0a1c87728}}, + {{0x00085910c59fc2ff,0x000edd337895b65f,0x000511aab9da9784,0x0003096a69799310,0x0000e52ae4181572}, {0x000749531e18c1e3,0x0008b80751a2bb45,0x00001252b398f247,0x00055b499d5957ef,0x00003146d57f9c23}}, + {{0x00016eba4eecc3be,0x000ce1567b234d2f,0x000efba21f561158,0x000a7d55857190e0,0x0000320ec77503ce}, {0x0002c5e31d91219f,0x0008a202612ab98d,0x00046d14dd97213e,0x000914064ebf5b24,0x0000252742c3551c}}, + {{0x000892bae35f8fff,0x0007393a7330c288,0x000e452adeb6025c,0x000fa6fdb0931841,0x00000230ffaf8bad}, {0x0009c569666cb64c,0x00028c9c1a11d83c,0x0006d175be8ee2b7,0x000fc1d4d08c60fb,0x0000e59f68cfb17a}}, + {{0x0004746057984cd0,0x0000120adf81173d,0x000dccffa456524a,0x0003ddbb34c2a11e,0x000094498314e67d}, {0x000864a8165bdf84,0x0001548f9fd22672,0x0002502dc589998c,0x0002ff4896359671,0x000033a8860e87ea}}, + {{0x0004df6843c364e9,0x000b16a5ee86621f,0x000f9457ae1f592d,0x000240793da6c122,0x00002d742b08fcfb}, {0x0008c3f73811884b,0x0007e71f910c2c7a,0x000405a5a15bfcfb,0x000f363a542968cd,0x0000a72904aaf04d}}, + {{0x000ffef7441e12e9,0x00042919f0edc246,0x000792a5bbf274ab,0x000883a1dbf3b485,0x00006fed0fa498e2}, {0x000247ca159aa0db,0x000cb4ac34f0f016,0x0007adee02259b6f,0x000ab0e110a89e6a,0x0000df1a3bccde15}}, + {{0x0009939fe95894f3,0x0006d80da64f7f5b,0x0005668fb3c50e7c,0x000b18adf2c440f9,0x00002885888ec566}, {0x00067d3ea0cc76aa,0x000b0b26bc3ba506,0x0004568d5f276915,0x0001f136016d96ad,0x0000a9984da49550}}, + {{0x000d0ed6b7a871b6,0x000ab42c02ff0806,0x0009af12f46ee10a,0x0005d377ff131d77,0x00005a677099acd8}, {0x000ac9b951626815,0x0009abdb942babd9,0x00036440880eb706,0x000163339c586c4f,0x0000709ca935dc3b}}, + {{0x000ff3ccda445aca,0x000f0859e39c86a5,0x00090203fe429386,0x0008a52bf2703bab,0x0000720aa0200a65}, {0x0007538fdce5c9c9,0x00015246ebcb614e,0x0001842f337ae4a6,0x000828acc435ef95,0x00005e104588fd92}}, + {{0x0003cab9fe36412f,0x000c67555bd68737,0x0001bae6459ab06a,0x000fa54dfd187fa0,0x0000876a919b3578}, {0x0002b2279ace5b23,0x00067ca98cbd005b,0x000e40955a4d14a0,0x000970e03ea45012,0x00007185a2170d87}}, + {{0x0005612acb6e9367,0x0000244b28116bc6,0x0000fe9ace978300,0x00001bd8b738ef68,0x0000764500d3e681}, {0x0005a153f84b39ee,0x0004d79c4f71c297,0x000ad48b40f23e0d,0x000fe1df83cf30f0,0x0000ed994a14ad8c}}, + {{0x0006efacdd042013,0x0003311876df9490,0x0002bc23a0bac093,0x00009572e8adfc0d,0x0000c9e44ac33ba1}, {0x0007febbe90ab31f,0x0000bd134ef68004,0x0007788b8aa2b3eb,0x00025338748a1e98,0x0000b0f3a19b0368}}, + {{0x0006fbd381cc78af,0x00071706d2d15d34,0x000ee95e156dcdce,0x00005368df42e000,0x00005cd53efe90cd}, {0x00031ea895eeeb96,0x000ccf55d9280aea,0x0007880c92d842de,0x00017146595637e4,0x0000641ce44806ad}}, + {{0x000824118043c9a3,0x0005cf70caa98b65,0x000f51f396391481,0x000f7d40813d12f8,0x0000547dad76afc8}, {0x000149d080befd0b,0x000b7195cedc7d61,0x0009dc3f0e5886a2,0x0004cb9a99003a55,0x00007bf2af415c17}}, + {{0x000b64ed8b470837,0x000667567a1863c8,0x000a8958dda010c6,0x0009d8fce07e107c,0x0000333c3dcb7dec}, {0x000eef3725031b69,0x000b3d8017798df6,0x0004553846f57184,0x000740a78efac9c4,0x000091dcab915273}}, + {{0x00034d3d7a665eae,0x000176ef69c8e8cc,0x0002d1efd49af983,0x000c8da18040aabd,0x00003d8b47daa6ea}, {0x0002a6ab05930688,0x000ae5a1d5b5c164,0x000ec89eb5f6487d,0x000e45cd4ff7be70,0x0000da3d01424bad}}, + {{0x00038654291e9930,0x0005df3ebe005344,0x00051191326bf2e8,0x0002cb4d29e094e5,0x0000c2bfc12b6dc4}, {0x000c45947fa997d1,0x000efc7b04beec10,0x00092b0b4c905403,0x0008649820b0076a,0x00000768c5a27aa6}}, + {{0x0004fa4594af9a68,0x000599a9cc36d9eb,0x000b7f57263399f3,0x0000337fb3d0ceae,0x00005ca399f0e87c}, {0x000ea931d2385710,0x0001ed59e07954dc,0x000a7c626181c100,0x000429eb1a4ec0a3,0x000096fa65837c79}}, + {{0x000024de86d1520e,0x00030986cb7e66af,0x000efe7c68e15b56,0x000d805f49c36865,0x0000ab5898f85171}, {0x0005276853f145ec,0x000381824620abe6,0x000b802ce92b4fb6,0x00036d087612db6f,0x0000a0d67a1b76f7}}, + {{0x0004f2782c6ac72a,0x0003cafa67117a5c,0x0002aab6fb1c93c9,0x0004a540bc21b67e,0x0000934f022b3868}, {0x000d03cbbc8303f2,0x0004a4fb67b44644,0x00075a8b54b44e4e,0x0007e18bf968e89e,0x00000f6f42ac1a37}}, + {{0x000486f46ce1f619,0x000037a2e0f90539,0x00020a418cea6f5e,0x000ba43f6c178ddf,0x00006ab2cb90f6c4}, {0x0004b1c6938cfc22,0x00020e13845f695e,0x000d68e0193de57e,0x000ebc9ff5be6cb4,0x000002c5058b0d9d}}, + {{0x000c3c04fde203fa,0x0005baff5dd91577,0x000ef4054c12be81,0x0006add827ff0b61,0x0000a1f02d5283f0}, {0x0006692f1555eecd,0x000c44b0a336ee5a,0x00082f8f5b11fe2e,0x000da11c05d843a5,0x0000c68bae17ef36}}, + {{0x000ee12df5fabf2b,0x000be45a670b441d,0x000c610623c7e5fd,0x000d5872bd798c04,0x00009e1db3844e1d}, {0x0005ddaf191ee063,0x000f3d7cea2df0ad,0x00086387d70cdf49,0x000cefd8af41cf1c,0x00009846533056ae}}, + {{0x00021d62549e2b45,0x00037cb224e3c015,0x000678b1ce2f4b85,0x0000525f28baa01d,0x00005d6e33c34710}, {0x000d0d35e745bee9,0x000f90455fdf7490,0x000eb3d3ef5582c0,0x000a223455ee33cb,0x0000ede029ddb7d6}}, + {{0x00094f9647c9d90c,0x0009ae5f214b69bd,0x0007a420dba58efa,0x000a1778c3aa70bb,0x00005b6de0077a42}, {0x0009d9ea019d4132,0x0007f0e583e34c87,0x0005be0ce07a75bd,0x000b6b3d395891e3,0x0000411317743abc}}, + {{0x00071e9d9637bd7b,0x000aa30673d89731,0x0004f6faaa823ed5,0x0008bb2383e64b03,0x00002b4b7d4e2602}, {0x00010fb3c893e582,0x00051b773cca2248,0x0007552b33ebecfb,0x00069cecde31e97f,0x00008f7134559072}}, + {{0x000b81bd0d95dfea,0x000920c24df0288c,0x000310dd6c5ef9f3,0x000d5abf1b0350c5,0x0000ad3ddb3e8a48}, {0x000dd398b526731f,0x0005384805e5c192,0x000b7e61e9fed2fe,0x000e23cfdd2588ba,0x0000c740a3eef5c3}}, + {{0x000eac2018c33f78,0x0006fe797f0c8057,0x000b588aee816f24,0x000cabc89d091be7,0x00003cd7be0bdaf6}, {0x0009197eaba2edb2,0x000261290082c9bf,0x0002d252eb7c7e7f,0x000069d1ae302bc4,0x000059df1546cf2c}}, + {{0x000c6ed00d94ec36,0x00088e1af59b2c21,0x000e2ff2bcf61175,0x000869457c578360,0x0000523a23621201}, {0x00034a09be29e76a,0x000690f5c6bae145,0x0006b6f117764e98,0x000976119eff6ee2,0x000083c1f34b5380}}, + {{0x000aa5996ef32628,0x000199d82779220a,0x000d1241a4aeba26,0x000ed2f9f34d5f06,0x00004acf34a65fce}, {0x000315c7f2ccd41e,0x000c5fdf0096ae54,0x00080691f3391667,0x00052631991b02d6,0x0000c48b23574895}}, + {{0x0008812c701e04a9,0x0002050be8112a19,0x000edb9dd4072896,0x000fd219a3328fa8,0x0000e4723d681c68}, {0x0004a362eddf7e14,0x000fa26194b59490,0x000cc6e78db8c366,0x0006655f1cc62e79,0x0000d82e71dbd404}}, + {{0x00042661e0e01d69,0x000365b508430cfc,0x0009c570783ab355,0x000126719831e04a,0x000080fe3aa553aa}, {0x0005fba614f1229e,0x000ff23b90b04642,0x00088bd06048f481,0x0003938fab45f5ed,0x000081191df8129f}}, + {{0x000e6fa9951025d2,0x00041ecbf164a4d8,0x000d2f4e8491ed8a,0x00055b97fd6f9239,0x00006956adf726ae}, {0x0003a64e362a0314,0x000daa7b2078d584,0x00085660784da40f,0x000e2e4313534367,0x000074d604a28332}}, + {{0x000a9eacf4537898,0x0002535b7ed6beae,0x000d61511432f159,0x000edfe8ae015426,0x0000e7ad54ed3027}, {0x0008791dd6d06fcd,0x000cbb7fbce3acab,0x000bc7d0c0bc5826,0x000ec5864d8cb9b5,0x0000d5249e385260}}, + {{0x00031d9a10b92593,0x000b60bc06806933,0x000608f7d34ced3d,0x000e6d0d2006d8b2,0x0000aed68ceedb49}, {0x00015ad47c543ae5,0x000bbf84989e807a,0x00020645060e02d8,0x000196b5ba87d4ae,0x000068321bbe2e41}}, + {{0x000fa99480713a9b,0x00006bf34178f755,0x000be4593bc93577,0x0006d10a3c6ef3f1,0x0000310bacda7514}, {0x00057088f40a62e7,0x0005ee7b59e56baa,0x000af9ff64d366c7,0x0004ffc6b10b30ad,0x0000d6bd4deb6eb0}}, + {{0x0003bad7134faa7e,0x000c0b033f5eaa35,0x000e4f98ff42bcb8,0x000ec9a998139559,0x0000f99dd5db139f}, {0x00009ecb311e12f8,0x0002adc0a4dc54fc,0x0005c0b088460433,0x000ce37796ea0ae1,0x0000ece0022c632b}}, + {{0x000a05993f51c69f,0x00070bea577129dd,0x00064408337ea113,0x0005a48ba7c02c90,0x000035f93ee50fde}, {0x0001277019f51c41,0x0006adfd4c1abe93,0x00083e23a3e47aca,0x000aa67a89f9df66,0x000092d1f5304fa0}}, + {{0x000ac330d6c7b459,0x00082e88c3411e11,0x00005313b3e445d1,0x0006bc33cc1dc4d9,0x00004685343106e0}, {0x00051b83920ae37d,0x0001c71d6750f1e5,0x000b709a1faade96,0x0007149f2dc25e54,0x0000b71cc89b494b}}, + {{0x000f3f8d09521bed,0x0008512c9daab73a,0x0003da2ba27d45d8,0x00053296a099b7b5,0x0000e7f2156e567c}, {0x00017ea24df91ea0,0x000270e60988d6c3,0x000418dde749e07d,0x0007a0c55165c906,0x00000189658e7c42}}, + {{0x0005554a920980f5,0x0004d83056f4f7dc,0x000745be3258ac61,0x000646af5b2b9c78,0x000032efb61c140e}, {0x00065f3c23f4987a,0x00029a1787327ea8,0x000b93558610d559,0x0001dac6b44f9ef7,0x00008c3f3947610c}}, + {{0x00036c41c02e3f96,0x000c00d78de5829f,0x0006cdb1416e315b,0x0006d6039934b045,0x00005408b5b7c401}, {0x000046423ee53cf9,0x000ca2000b9b28d8,0x000ac80402e6f559,0x000737cf3b0e8eab,0x00000a478fff164c}}, + {{0x000d5dbc3bce5259,0x000a79bab068adc4,0x000702cff0e692d9,0x0009162104dab4e1,0x00007c3b223bc6e0}, {0x000a4bae9b92e722,0x00066cd5c34091b3,0x0006af7b98c5842b,0x00046e36bf2afb32,0x0000e0928bb0a702}}, + {{0x000e1f1c48f51b67,0x000e3017ad888cfc,0x000124d6978fa7fb,0x00020cae67af53f3,0x00001d14379ab093}, {0x0006d3efd5b32f13,0x0008955fa05fbe53,0x0000194e6d931907,0x0003184e5f4c7300,0x0000f48a960c8670}}, + {{0x0004dae073946e67,0x00069e56390f104b,0x00034438ec69b46a,0x0003d9a22a832b30,0x000078dfabdaf745}, {0x00083fe3d2f9dc1d,0x000b45b9863f9203,0x000ed43a45a02d9b,0x0004dc1e596b8494,0x00001045a9c041b7}}, + {{0x00016c0a59ce7847,0x0003816390641219,0x000c52eac415e082,0x0004dd076ea35e04,0x0000e4629568ba7c}, {0x00071b6a7b975de1,0x0007b677e54ee47d,0x0004a7e4c03aec55,0x0002fc9359233ebf,0x00003b0f0abe072b}}, + {{0x000dfefe8800dcd2,0x0007813d211bb68d,0x0006e883a7a79b7c,0x0004ea45689552b4,0x00008de706a7de1a}, {0x000c4e3d983df145,0x000fdfedeea597b3,0x0000e50daef33ef9,0x00096c99395a861b,0x0000f4ee038dd678}}, + {{0x0002fa9a80943602,0x000a160fc4fb1606,0x000a18843d079884,0x000fdcfc9ae67dea,0x0000de5999876f2b}, {0x0001e086666832ff,0x000ac7312a202d98,0x000df4eb39af1d51,0x0006ba69df8bab4d,0x00004f813058f3c4}}, + {{0x000e4d9deba1f764,0x00046544992ba84d,0x00062571d3ef1e08,0x0003d6cdf5b4eef3,0x0000c6b0de82e039}, {0x00019759c17b4e15,0x0007eb4e8f856535,0x0002db7b0e1f35cb,0x000a2de21c2ce964,0x0000ede5f6162f54}}, + {{0x0002e8ff962e6fb2,0x0003791b5f6c1220,0x000daf6cb28042fd,0x000a0d5f509fdfe4,0x0000a70291c45d04}, {0x000e258b0e2b68da,0x000b17fac85bdaf9,0x000d1b986bf47d76,0x0002f63d6778a846,0x0000223266c01388}}, + {{0x000f6489e4813958,0x00073454ba37e004,0x000852a0cf198eb2,0x0001c1d7d37038b3,0x00005057db2e3939}, {0x000ab92ce347b4c8,0x000ca340fa0188c7,0x000ef094a8b99eb3,0x00014dcfa7df268e,0x00005a65feda02ad}}, + {{0x0006ea594087938e,0x000c2c29c1b6b97c,0x000fcd2e662321e1,0x0007d0edd9981f5b,0x0000571509bfdac6}, {0x000506cccb99c73e,0x000781554bcdd381,0x00076723b50a8b0a,0x000df26130d1d4f5,0x00007791681ec370}}, + {{0x00016a84ca2278a7,0x0005b22100cbee90,0x0008b97d91e063af,0x000ba251ae16b457,0x0000b228a1a62245}, {0x00037ce46041bf2a,0x0009154529a0343c,0x00093bcf590509e3,0x000354b75ffee430,0x0000d443bb4bab01}}, + {{0x000f1435aaae6483,0x000a104f999a4352,0x0009f21ba7c9f131,0x0005b56e3000fcee,0x000045e97d130480}, {0x000862d2138a5c21,0x0006da716279c728,0x0006fa6fa150dba3,0x00084fa32f114503,0x000020200ebf2740}}, + {{0x00090af2a4f30bb8,0x000233a2204ec06a,0x000572fbdc316dfd,0x0002843a0475a78f,0x00004ba9e239c4e7}, {0x0009abfc1137d0a0,0x00011346b7ea4127,0x000361a1d3145421,0x00011092560f9c0e,0x0000e66391042bad}}, +} +}; + +#endif //(_DISABLE_ECP_SM2_HARDCODED_BP_TBL_) + +IPP_OWN_DEFN(const cpPrecompAP *, gfpec_precom_sm2_radix52_fun, (void)) { + static cpPrecompAP t = { + /* w = */ BASE_POINT_WIN_SIZE, + /* select_function = */ gesm2_select_ap_w7_ifma, + /* precomputed_data = */ (BNU_CHUNK_T *)ifma_ec_sm2_bp_precomp + }; + return &t; +} + +#endif //(_IPP32E >= _IPP32E_K1) + +#endif // _IFMA_ECPRECOMP7_SM2_H_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_sm2_key_exchange_shared_key.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_sm2_key_exchange_shared_key.c new file mode 100644 index 0000000..ff33a13 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/ifma_sm2_key_exchange_shared_key.c @@ -0,0 +1,264 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" + +#if (_IPP32E >= _IPP32E_K1) + +#include "sm2/sm2_stuff.h" + +#include "sm2/ifma_defs_sm2.h" +#include "sm2/ifma_ecpoint_sm2.h" +#include "sm2/ifma_arith_method_sm2.h" + + +/* clang-format off */ +__INLINE void ifma_sm2_set_affine_point_radix52(PSM2_POINT_IFMA *rp, + const BNU_CHUNK_T *x, const BNU_CHUNK_T *y, + ifmaArithMethod *method) +/* clang-format on */ +{ + ifma_import to_radix52 = method->import_to52; + ifma_encode p_to_mont = method->encode; + + rp->x = to_radix52(x); + rp->y = to_radix52(y); + + rp->x = p_to_mont(rp->x); + rp->y = p_to_mont(rp->y); + rp->z = FESM2_LOADU(PSM2_R); + + return; +} + +/* clang-format off */ +__INLINE void ifma_sm2_get_affine(BNU_CHUNK_T *x, BNU_CHUNK_T *y, + const PSM2_POINT_IFMA* p, + ifmaArithMethod* method) +/* clang-format on */ +{ + /* extract affine coordinate */ + fesm2 loc_x, loc_y; + gesm2_to_affine(/* x = */ &loc_x, /* y = */ &loc_y, /* p = */ p); + + ifma_export to_radix64 = method->export_to64; + ifma_decode p_from_mont = method->decode; + + loc_x = p_from_mont(loc_x); + loc_y = p_from_mont(loc_y); + + to_radix64((Ipp64u *)x, loc_x); + to_radix64((Ipp64u *)y, loc_y); + + return; +} + +/* clang-format off */ +IPP_OWN_DEFN(IppStatus, gfec_key_exchange_sm2_shared_key_avx512, (Ipp8u* pSharedKey, const int sharedKeySize, + Ipp8u* pSSelf, + const IppsBigNumState* pPrvKey, + IppsBigNumState* pEphPrvKey, + IppsGFpECKeyExchangeSM2State *pKE, Ipp8u* pScratchBuffer)) +/* clang-format on */ +{ + IPP_UNREFERENCED_PARAMETER(pScratchBuffer); + + const IppsKeyExchangeRoleSM2 role = EC_SM2_KEY_EXCH_ROLE(pKE); + + const IppsGFpECPoint *pPubKey = (ippKESM2Requester == role) ? EC_SM2_KEY_EXCH_PUB_KEY_USER_B(pKE) : EC_SM2_KEY_EXCH_PUB_KEY_USER_A(pKE); + const IppsGFpECPoint *pSelfEphPubKey = (ippKESM2Requester == role) ? EC_SM2_KEY_EXCH_EPH_PUB_KEY_USER_A(pKE) : EC_SM2_KEY_EXCH_EPH_PUB_KEY_USER_B(pKE); + const Ipp8u firstNum = (ippKESM2Requester == role) ? 0x03 : 0x02; + + IppStatus sts = ippStsNoErr; + + IppsGFpECState *pEC = EC_SM2_KEY_EXCH_EC(pKE); + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + gsModEngine *nME = ECP_MONT_R(pEC); + + const int elemBits = GFP_FEBITLEN(pME); /* size Bits */ + const int elemBytes = (elemBits + 7) / 8; /* size Bytes */ + const int elemSize = GFP_FELEN(pME); /* size BNU_CHUNK */ + + ifmaArithMethod *pmeth = (ifmaArithMethod *)GFP_METHOD_ALT(pME); + ifmaArithMethod *nmeth = (ifmaArithMethod *)GFP_METHOD_ALT(nME); + + ifma_import to_radix52 = pmeth->import_to52; + ifma_export from_radix52 = pmeth->export_to64; + + /* Mod engine (mod n - subgroup order) */ + ifma_encode n_to_mont = nmeth->encode; + ifma_decode n_from_mont = nmeth->decode; + ifma_mul n_mul = nmeth->mul; + ifma_add n_add = nmeth->add; + + /* check that Ephemeral Public and Private Keys are related (R(a/b)=r(a/b)G) */ + /* Ephemeral Private Key -> r(a/b) */ + IppsGFpECPoint pTmpPoint; + cpEcGFpInitPoint(&pTmpPoint, cpEcGFpGetPool(1, pEC), 0, pEC); + BNU_CHUNK_T *pDataEphPrv = BN_NUMBER(pEphPrvKey); + gfec_PubKey_sm2_avx512(&pTmpPoint, pDataEphPrv, BN_SIZE(pEphPrvKey), pEC, pScratchBuffer); // r(a/b)*G + int result = gfec_ComparePoint(&pTmpPoint, pSelfEphPubKey, pEC); // R(a/b) == r(a/b)G + cpEcGFpReleasePool(1, pEC); + IPP_BADARG_RET(!result, ippStsEphemeralKeyErr); + + /* create buffer data (it needes further use compute tmp_p) + * -> SM3( x(u/v)(0) || Za(1) || Zb(2) || xa(3) || ya(4) || xb(5) || yb(6) ) + */ + BNU_CHUNK_T *pDataBuff = cpGFpGetPool(7, pME); + BNU_CHUNK_T *xuv = pDataBuff; /* x(u/v)(0) */ + BNU_CHUNK_T *za = pDataBuff + 1 * elemSize; /* Za(1) */ + BNU_CHUNK_T *zb = pDataBuff + 2 * elemSize; /* Za(1) */ + BNU_CHUNK_T *xa = pDataBuff + 3 * elemSize; /* xa(3) */ + BNU_CHUNK_T *ya = pDataBuff + 4 * elemSize; /* ya(4) */ + BNU_CHUNK_T *xb = pDataBuff + 5 * elemSize; /* xb(5) */ + BNU_CHUNK_T *yb = pDataBuff + 6 * elemSize; /* yb(6) */ + + PSM2_POINT_IFMA Ra; /* ephemeral public key User A */ + PSM2_POINT_IFMA Rb; /* ephemeral public key User B */ + PSM2_POINT_IFMA P; /* public key User (A/B) */ + + /* convert to Montgomery IFMA SM2 */ + /* extract -> xa | ya | xb | yb */ + BNU_CHUNK_T *_xa_ = za; /* use Za(1) */ + /* 2) x(a/b)` = 2^w + (x(a/b) & (2^w – 1)) */ + /* point to affine coordinate */ + cpSM2KE_get_affine_ext_euclid(/* x = */ xa, /* y = */ ya, /* p = */ EC_SM2_KEY_EXCH_EPH_PUB_KEY_USER_A(pKE), /* pEC = */ pEC); + /* set affine coordinate radix 52 */ + ifma_sm2_set_affine_point_radix52(/* rp = */ &Ra, /* x= */ xa, /* y = */ ya, pmeth); + /* reduction x = 2^w + ( x & (2^w - 1) ) */ + cpSM2KE_reduction_x2w(_xa_, xa, pEC); + /* to big endian */ + cpSM2KE_xy_to_BE(xa, ya, pEC); + + BNU_CHUNK_T *_xb_ = zb; /* use Zb(2) */ + /* 4) x(b/a)` = 2^w + ( x(b/a) & (2^w – 1) ) */ + cpSM2KE_get_affine_ext_euclid(/* x = */ xb, /* y = */ yb, /* p = */ EC_SM2_KEY_EXCH_EPH_PUB_KEY_USER_B(pKE), /* pEC = */ pEC); + /* set affine coordinate radix 52 */ + ifma_sm2_set_affine_point_radix52(/* rp = */ &Rb, /* x = */ xb, /* y = */ yb, pmeth); + /* reduction x = 2^w + ( x & (2^w - 1) ) */ + cpSM2KE_reduction_x2w(_xb_, xb, pEC); + /* to big endian */ + cpSM2KE_xy_to_BE(xb, yb, pEC); + + BNU_CHUNK_T *pPool = cpGFpGetPool(3, pME); + recode_point_to_mont52(&P, ECP_POINT_DATA(pPubKey), pPool /* 3 elem */, pmeth, pME); + cpGFpReleasePool(3, pME); /* pPool(3) */ + + /* extract to role */ + BNU_CHUNK_T *_xf_ = (ippKESM2Requester == role) ? _xa_ : _xb_; + BNU_CHUNK_T *_xs_ = (ippKESM2Requester == role) ? _xb_ : _xa_; + PSM2_POINT_IFMA *R = (ippKESM2Requester == role) ? &Rb : &Ra; + + /* 3) t(a/b) = ( d(a/b) + x(a/b)`*r(a/b) ) mod n */ + fesm2 d, r, x1; + d = r = x1 = setzero_i64(); + /* preparation d(a/b) */ + d = to_radix52((const Ipp64u *)BN_NUMBER(pPrvKey)); + d = n_to_mont(d); + /* preparation r(a/b) */ + r = to_radix52((const Ipp64u *)pDataEphPrv); + r = n_to_mont(r); + /* preparation x(a/b)` */ + x1 = to_radix52((const Ipp64u *)_xf_); + x1 = n_to_mont(x1); + /* compute */ + r = n_mul(x1, r); /* r = x(a/b)`r(a/b) */ + d = n_add(d, r); /* t = ( d(a/b) + x(a/b)`*r(a/b) ) */ + /* t -> convert form Montgomery */ + d = n_from_mont(d); + + /* 5) U/V = [h*t(a/b)]( P(b/a) + [x(b/a)`]R(b/a) ) = ( x(u/v), y(u/v) ) */ + const int scalarBits = ((elemBits + 1) / 2); + gesm2_mul(R, R, (const Ipp8u *)_xs_, scalarBits); /* R = [x(b/a)`]R(b/a) */ + gesm2_add(&P, &P, R); /* P = P(b/a) + [x(b/a)`]R(b/a) */ + BNU_CHUNK_T *pScalar = xuv; /* use buffer x(u/v)(0) and Za(1) */ + from_radix52((Ipp64u *)pScalar, d); + pScalar[elemSize] = 0u; /* extended scalar - add 0 -> R = [scalar]P */ + gesm2_mul(&P, &P, (const Ipp8u *)pScalar, elemBits); /* P = [h*t(a/b)]( P(b/a) + [x(b/a)`]R(b/a) ) */ + + /* check U/V == 0 */ + const mask8 is_zero = (mask8)(FESM2_IS_ZERO(P.x) & FESM2_IS_ZERO(P.y) & FESM2_IS_ZERO(P.y)); + if ((mask8)0xFF == is_zero) + sts = ippStsEphemeralKeyErr; + + /* extract x(u/v) ,y(u/v) */ + ifma_sm2_get_affine(/* x = */ EC_SM2_KEY_EXCH_POINT_X(pKE), + /* y = */ EC_SM2_KEY_EXCH_POINT_Y(pKE), + /* p = */ &P, + pmeth); + cpSM2KE_xy_to_BE(/* x = */ EC_SM2_KEY_EXCH_POINT_X(pKE), /* y = */ EC_SM2_KEY_EXCH_POINT_Y(pKE), pEC); + + /* Precompute Hash */ + Ipp8u *pBuff = (Ipp8u *)pDataBuff; + /* tmp_p = SM3( x(u/v) || Za || Zb || xa || ya || xb || yb ) */ + /* copy x(u/v)(0) */ + cpSM2_CopyBlock(xuv, (const Ipp8u *)EC_SM2_KEY_EXCH_POINT_X(pKE), elemBytes); + /* copy Za(1) */ + cpSM2_CopyBlock(za, EC_SM2_KEY_EXCH_USER_ID_HASH_USER_A(pKE), IPP_SM3_DIGEST_BYTESIZE); + /* copy Zb(2) */ + cpSM2_CopyBlock(zb, EC_SM2_KEY_EXCH_USER_ID_HASH_USER_B(pKE), IPP_SM3_DIGEST_BYTESIZE); + /* xa || ya || xb || yb - is exists */ + + const int sizeS_SM3 = elemBytes + 2 * IPP_SM3_DIGEST_BYTESIZE + 4 * elemBytes; /* 224 bytes */ + + /* copy to next operation */ + cpSM2KE_compute_hash_SM3(EC_SM2_KEY_EXCH_PRECOM_HASH(pKE), pBuff, sizeS_SM3); + + if ((NULL != pSSelf) && (ippStsNoErr == sts)) { + /* 6) S(a/b) = SM3( 0x0(3/2) || y(u/v) || tmp_p ) */ + /* copy 0x0(3/2) */ + pBuff[0] = firstNum; + /* copy y(u/v) */ + cpSM2_CopyBlock(pBuff + 1, (const Ipp8u *)EC_SM2_KEY_EXCH_POINT_Y(pKE), elemBytes); + /* copy tmp_p */ + cpSM2_CopyBlock(pBuff + 1 + elemBytes, EC_SM2_KEY_EXCH_PRECOM_HASH(pKE), IPP_SM3_DIGEST_BYTESIZE); + + const int sizeSab_SM3 = 1 + elemBytes + IPP_SM3_DIGEST_BYTESIZE; /* 65 bytes */ + cpSM2KE_compute_hash_SM3(pSSelf, pBuff, sizeSab_SM3); + } + + /* compute KDF */ + /* 7) K(a/b) = KDF(x(u/v) || y(u/v) || Za || Zb, klen) */ + /* copy x(u/v)(0)*/ + cpSM2_CopyBlock(xuv, (const Ipp8u *)EC_SM2_KEY_EXCH_POINT_X(pKE), elemBytes); /* slot x(u/v)(0) */ + /* copy y(u/v)(1) */ + cpSM2_CopyBlock(za, (const Ipp8u *)EC_SM2_KEY_EXCH_POINT_Y(pKE), elemBytes); /* slot Za(1) */ + /* copy Za(2) */ + cpSM2_CopyBlock(zb, EC_SM2_KEY_EXCH_USER_ID_HASH_USER_A(pKE), IPP_SM3_DIGEST_BYTESIZE); /* slot Zb(2) */ + /* copy Zb(3) */ + cpSM2_CopyBlock(xa, EC_SM2_KEY_EXCH_USER_ID_HASH_USER_B(pKE), IPP_SM3_DIGEST_BYTESIZE); /* slot xa(3) */ + + const int sizeKab_KDF = 2 * elemBytes + 2 * IPP_SM3_DIGEST_BYTESIZE; /* 128 bytes */ + + /* set pointer start */ + pBuff = (Ipp8u *)pDataBuff; + + /* compute KDF */ + KDF_sm3(pSharedKey, sharedKeySize, (const Ipp8u *)pBuff, sizeKab_KDF); + + cpGFpReleasePool(7, pME); + /* clear Ephemeral Private Key */ + cpBN_zero(pEphPrvKey); + /* clear secret register */ + PurgeBlock(&r, sizeof(r)); + PurgeBlock(&d, sizeof(d)); + + return sts; +} + +#endif // (_IPP32E >= _IPP32E_K1) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_decrypt_ext.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_decrypt_ext.c new file mode 100644 index 0000000..94c5272 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_decrypt_ext.c @@ -0,0 +1,186 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + +#include "owncp.h" +#include "owndefs.h" +#include "sm2/sm2_stuff.h" + +#define CHECK_PRIVATE_KEY(KEY) \ + IPP_BAD_PTR1_RET((KEY)) \ + IPP_BADARG_RET(!BN_VALID_ID((KEY)), ippStsContextMatchErr) \ + IPP_BADARG_RET(BN_NEGATIVE((KEY)), ippStsInvalidPrivateKey) \ + /* test if 0 < pPrivateA < Order */ \ + IPP_BADARG_RET(0 == gfec_CheckPrivateKey((KEY), pEC), ippStsInvalidPrivateKey) + +/** + * @brief + * Decryption message text based SM2 elliptic curve + * Implemenation based on standart: + * GM/T 0003.4-2012 SM2 + * Public key cryptographic algorithm SM2 based on elliptic curves + * Part 4: Public key encryption algorithm + * @param [out] pOut message decryption + * @param [in] maxOutLen available message size buffer + * @param [out] pOutSize message size fill container + * 0 - if the function ends with an error, + * other (>0) - if the function ends with OK + * @param [in] pInp pointer cipher text + * @param [in] inpLen cipher size + * @param [in] pPrvKey private key + * @param [in] pEC context elliptic curve + * @param [in] pScratchBuffer supported buffer + * @return + * ippStsNoErr - successful + * ippStsNullPtrErr - if pOut | pOutSize | pInp | pPrvKey | pEC | pScratchBuffer is NULL + * ippStsContextMatchErr - if pEC no valid ID | pEC no exists SUBGROUP + * ippStsNotSupportedModeErr - if pEC no supported (n|p) modulus arithmetic + * ippStsBadArgErr - if + * * maxOutLen < 0 + * * inpLen < (len(PC) + 2*len(x) + len(SM3)) + * ippStsInvalidPrivateKey - if test is failed 0 < pPrvKey < Order + * + */ +/* clang-format off */ +IPPFUN(IppStatus, ippsGFpECDecryptSM2_Ext, (Ipp8u *pOut, int maxOutLen, + int *pOutSize, + const Ipp8u *pInp, int inpLen, + const IppsBigNumState *pPrvKey, + IppsGFpECState *pEC, Ipp8u *pScratchBuffer)) +/* clang-format on */ +{ + + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + IPP_BADARG_RET(!ECP_SUBGROUP(pEC), ippStsContextMatchErr); + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); /* base P */ + IPP_BADARG_RET(1 < GFP_EXTDEGREE(pME), ippStsNotSupportedModeErr); + gsModEngine *nME = ECP_MONT_R(pEC); /* base N */ + IPP_BADARG_RET(1 < GFP_EXTDEGREE(nME), ippStsNotSupportedModeErr); + + const int elemBits = GFP_FEBITLEN(pME); /* size Bits */ + const int elemBytes = (elemBits + 7) / 8; /* size Bytes */ + const int elemSize = GFP_FELEN(pME); /* size BNU_CHUNK */ + + /* buffer */ + IPP_BAD_PTR1_RET(pScratchBuffer) + + /* check cipher text */ + IPP_BAD_PTR1_RET(pInp) + /* size */ + const int ciph_PC_size = 1; + const int ciph_xy_size = 2 * (Ipp32s)sizeof(BNU_CHUNK_T) * elemSize; + const int ciph_hash_size = IPP_SM3_DIGEST_BYTESIZE; + const int min_ciph_size = ciph_PC_size + ciph_xy_size + ciph_hash_size; + IPP_BADARG_RET(!(inpLen >= min_ciph_size), ippStsOutOfRangeErr) + + /* message */ + IPP_BAD_PTR1_RET(pOut) + /* the cipher text condition guarantees a positive or zero maxOutLen */ + const int ciph_msg_size = inpLen - min_ciph_size; + IPP_BADARG_RET(!(maxOutLen >= ciph_msg_size), ippStsOutOfRangeErr) + + /* out message size */ + IPP_BAD_PTR1_RET(pOutSize) + /* zeros */ + *pOutSize = 0; + + /* private key */ + CHECK_PRIVATE_KEY(pPrvKey) + + { + IppStatus sts = ippStsInvalidPoint; // pointer input if invalid -> status out + int finitPoint = 0; + + const Ipp8u *pC1_PC = pInp; + const Ipp8u *pC1_xy = pC1_PC + ciph_PC_size; + const Ipp8u *pC3 = pC1_xy + ciph_xy_size; + const Ipp8u *pC2 = pC3 + ciph_hash_size; + + IppsGFpECPoint R; + cpEcGFpInitPoint(&R, cpEcGFpGetPool(1, pEC), 0, pEC); + /* step1: extract C1 from C */ + BNU_CHUNK_T *pBuff = cpGFpGetPool(2, pME); + BNU_CHUNK_T *pC1_x = pBuff; + BNU_CHUNK_T *pC1_y = pC1_x + elemSize; + + /* copy coordinate point */ + COPY_BNU(pC1_x, (BNU_CHUNK_T *)(pC1_xy), elemSize); + COPY_BNU(pC1_y, (BNU_CHUNK_T *)(pC1_xy + elemBytes), elemSize); + /* convert big endian -> little endian */ + cpSM2KE_reverse_inplace((Ipp8u *)pC1_x, elemBytes); + cpSM2KE_reverse_inplace((Ipp8u *)pC1_y, elemBytes); + /* convert to Montgomery */ + GFP_METHOD(pME)->encode(pC1_x, pC1_x, pME); + GFP_METHOD(pME)->encode(pC1_y, pC1_y, pME); + + /* step 2 (standart) - check valid input coordinate x|y */ + finitPoint = gfec_SetPoint(ECP_POINT_DATA(&R), pC1_x, pC1_y, pEC); + ECP_POINT_FLAGS(&R) = finitPoint ? (ECP_AFFINE_POINT | ECP_FINITE_POINT) : 0; + + if (finitPoint && (0 != gfec_IsPointOnCurve(&R, pEC))) { + sts = ippStsNoErr; + /* step 3 (standart): [db]C1 = (x2,y2) */ + ippsGFpECMulPoint(&R, pPrvKey, &R, pEC, pScratchBuffer); + + BNU_CHUNK_T *pX = pBuff; + BNU_CHUNK_T *pY = pX + elemSize; + + gfec_GetPoint(pX, pY, &R, pEC); + GFP_METHOD(pME)->decode(pX, pX, pME); + GFP_METHOD(pME)->decode(pY, pY, pME); + + cpSM2KE_reverse_inplace((Ipp8u *)pX, elemBytes); + cpSM2KE_reverse_inplace((Ipp8u *)pY, elemBytes); + + /* step 4 (standart): t = KDF(x2 || y2, klen) */ + KDF_sm3(pOut, ciph_msg_size, (Ipp8u *)pBuff, 2 * elemBytes); + + /* step 5 (standart): M` = C2 (x) t */ + for (int i = 0; i < ciph_msg_size; ++i) { + pOut[i] = pOut[i] ^ pC2[i]; + } + + /* step 6 (standart): u = Hash(x2 || M` || y2) */ + static IppsHashState_rmf ctx; + + Ipp8u u[IPP_SM3_DIGEST_BYTESIZE]; + ippsHashInit_rmf(&ctx, ippsHashMethod_SM3()); + /* x2 */ + ippsHashUpdate_rmf((Ipp8u *)pX, elemBytes, &ctx); + /* M */ + ippsHashUpdate_rmf(pOut, ciph_msg_size, &ctx); + /* y2 */ + ippsHashUpdate_rmf((Ipp8u *)pY, elemBytes, &ctx); + /* C3 */ + ippsHashFinal_rmf(u, &ctx); + + if (0 == EquBlock(u, pC3, IPP_SM3_DIGEST_BYTESIZE)) { + PurgeBlock(pOut, ciph_msg_size); + } else { + *pOutSize = ciph_msg_size; + } + + PurgeBlock(u, IPP_SM3_DIGEST_BYTESIZE); + } + + cpGFpReleasePool(2, pME); + cpEcGFpReleasePool(1, pEC); /* release R from the pool */ + return sts; + } +} + +#undef CHECK_PRIVATE_KEY diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_decrypt_ext_dec_msg_size.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_decrypt_ext_dec_msg_size.c new file mode 100644 index 0000000..995b02f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_decrypt_ext_dec_msg_size.c @@ -0,0 +1,65 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + +#include "owncp.h" +#include "owndefs.h" +#include "sm2/sm2_stuff.h" + +/** + * @brief ippsGFpECDecryptSM2_Ext_DecMsgSize + * Get Message Size data by Decrypt SM2 + * Implemenation based on standart: + * GM/T 0003.4-2012 SM2 + * Public key cryptographic algorithm SM2 based on elliptic curves + * Part 4: Public key encryption algorithm + * @param [in] pEC Context Elliptic Curve + * @param [in] ctMsgSize cipher size + * @param [out] pSize size allocation bytes by Key Exchange Context + * @return + * ippStsNoErr - successful + * ippStsNullPtrErr - if pEC or pSize is NULL + * ippStsContextMatchErr - if pEC no valid ID or no exists SUBGROUP + * ippStsBagArgErr - if cipher < 0 or ctMsgSize - (C1 + C3) < 0 + */ +IPPFUN(IppStatus, ippsGFpECDecryptSM2_Ext_DecMsgSize, (const IppsGFpECState *pEC, int ctMsgSize, int *pSize)) +{ + /* check Context Elliptic Curve */ + IPP_BAD_PTR2_RET(pEC, pSize); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + IPP_BADARG_RET(!ECP_SUBGROUP(pEC), ippStsContextMatchErr); + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); /* base P */ + IPP_BADARG_RET(1 < GFP_EXTDEGREE(pME), ippStsNotSupportedModeErr); + gsModEngine *nME = ECP_MONT_R(pEC); /* base N */ + IPP_BADARG_RET(1 < GFP_EXTDEGREE(nME), ippStsNotSupportedModeErr); + + const int elemSize = GFP_FELEN(pME); /* size BNU_CHUNK */ + + /* check chiper size */ + IPP_BADARG_RET(!(ctMsgSize >= 0), ippStsOutOfRangeErr) + const int ciph_PC_size = 1; + const int ciph_xy_size = 2 * (Ipp32s)sizeof(BNU_CHUNK_T) * elemSize; + const int ciph_hash_size = IPP_SM3_DIGEST_BYTESIZE; + + const int size = ctMsgSize - (ciph_PC_size + ciph_xy_size + ciph_hash_size); + + /* if size < 0 -> pSize = 0 + call ippStsBadArgErr */ + *pSize = 0; + IPP_BADARG_RET(!(size >= 0), ippStsOutOfRangeErr) + + *pSize = size; + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_encrypt_ext.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_encrypt_ext.c new file mode 100644 index 0000000..ab5949c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_encrypt_ext.c @@ -0,0 +1,199 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + +#include "owncp.h" +#include "owndefs.h" +#include "sm2/sm2_stuff.h" + +#define CHECK_PUBLIC_KEY(KEY) \ + IPP_BAD_PTR1_RET((KEY)) \ + IPP_BADARG_RET(!ECP_POINT_VALID_ID((KEY)), ippStsContextMatchErr) \ + IPP_BADARG_RET(ECP_POINT_FELEN((KEY)) != GFP_FELEN(pME), ippStsOutOfRangeErr) \ + IPP_BADARG_RET(0 == gfec_IsPointOnCurve((KEY), pEC), ippStsInvalidPoint) + +#define CHECK_PRIVATE_KEY(KEY) \ + IPP_BAD_PTR1_RET((KEY)) \ + IPP_BADARG_RET(!BN_VALID_ID((KEY)), ippStsContextMatchErr) \ + IPP_BADARG_RET(BN_NEGATIVE((KEY)), ippStsInvalidPrivateKey) \ + /* test if 0 < pPrivateA < Order */ \ + IPP_BADARG_RET(0 == gfec_CheckPrivateKey((KEY), pEC), ippStsInvalidPrivateKey) + +/** + * @brief + * Encryption message text based SM2 elliptic curve + * Implemenation based on standart: + * GM/T 0003.4-2012 SM2 + * Public key cryptographic algorithm SM2 based on elliptic curves + * Part 4: Public key encryption algorithm + * @param [out] pOut pointer output cipher text + * @param [in] maxOutLen available cipher size container + * @param [out] pOutSize size of the ciphertext filled in the container + * 0 - if the function ends with an error, + * other (>0) - if the function ends with OK + * @param [in] pInp pointer message encrypt + * @param [in] inpLen message size encrypt + * @param [in] pPublicKey public key + * @param [in out] pEhpPublicKey (!) will be cleared at the end (!) Ephemeral Public Key + * @param [in out] pEphPrvKey (!) will be cleared at the end (!) Ephemeral Private Key + * @param [in] pEC context elliptic curve + * @param [in] pScratchBuffer supported buffer + * @return + * ippStsNoErr - successful + * ippStsNullPtrErr - if + * pEC | pScratchBuffer | pInp | pOut | pOutSize | pPublicKey | pEphPublicKey | pEphPrvKey + * is NULL + * ippStsContextMatchErr - if pEC no valid ID | pEC no exists SUBGROUP + * ippStsNotSupportedModeErr - if pEC no supported (n|p) modulus arithmetic + * ippStsBadArgErr - if + * * inpLen < 0 + * * maxOutLen < (len(PC) + 2*len(x) + len(SM3)) + * ippStsInvalidPrivateKey - if test is failed 0 < pEphPrvKey < Order + * ippStsInvalidPoint - if no in curve Pub Key and Ephemeral Public Key + */ +/* clang-format off */ +IPPFUN(IppStatus, ippsGFpECEncryptSM2_Ext, (Ipp8u *pOut, int maxOutLen, + int *pOutSize, + const Ipp8u *pInp, int inpLen, + const IppsGFpECPoint *pPublicKey, + IppsGFpECPoint *pEphPublicKey, IppsBigNumState *pEphPrvKey, + IppsGFpECState *pEC, Ipp8u *pScratchBuffer)) +/* clang-format on */ +{ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + IPP_BADARG_RET(!ECP_SUBGROUP(pEC), ippStsContextMatchErr); + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); /* base P */ + IPP_BADARG_RET(1 < GFP_EXTDEGREE(pME), ippStsNotSupportedModeErr); + gsModEngine *nME = ECP_MONT_R(pEC); /* base N */ + IPP_BADARG_RET(1 < GFP_EXTDEGREE(nME), ippStsNotSupportedModeErr); + + const int elemBits = GFP_FEBITLEN(pME); /* size Bits */ + const int elemBytes = (elemBits + 7) / 8; /* size Bytes */ + const int elemSize = GFP_FELEN(pME); /* size BNU_CHUNK */ + + /* buffer */ + IPP_BAD_PTR1_RET(pScratchBuffer) + + /* check msg size */ + IPP_BAD_PTR1_RET(pInp) + /* check size */ + IPP_BADARG_RET(!(inpLen >= 0), ippStsOutOfRangeErr) + + /* chiper text */ + IPP_BAD_PTR2_RET(pOut, pOutSize) + /* init zeros out cipher text */ + *pOutSize = 0; + /* check size */ + const int ciph_PC_size = 1; + const int ciph_xy_size = (Ipp32s)sizeof(BNU_CHUNK_T) * elemSize * 2; + const int ciph_hash_size = IPP_SM3_DIGEST_BYTESIZE; + const int ciph_msg_size = inpLen; + IPP_BADARG_RET(!(maxOutLen >= (ciph_PC_size + ciph_xy_size + ciph_msg_size + ciph_hash_size)), ippStsOutOfRangeErr) + + /* check Public | Private key */ + /* private */ + CHECK_PRIVATE_KEY(pEphPrvKey) + /* public */ + /* step 3: S = [h]Pb -> if Pb valid point and h != 0 -> S valid */ + CHECK_PUBLIC_KEY(pPublicKey) + CHECK_PUBLIC_KEY(pEphPublicKey) + + /* check that user's Ephemeral Public and Private Keys are related (C1=[k]G) */ + IppsGFpECPoint pTmpPoint; + cpEcGFpInitPoint(&pTmpPoint, cpEcGFpGetPool(1, pEC), 0, pEC); + ippsGFpECPublicKey(pEphPrvKey, &pTmpPoint, pEC, pScratchBuffer); // k*G + int result = gfec_ComparePoint(&pTmpPoint, pEphPublicKey, pEC); // C1 == [k]G ? + cpEcGFpReleasePool(1, pEC); // Release pool before the possible exit + IPP_BADARG_RET(!result, ippStsEphemeralKeyErr); + + { + IppStatus sts = ippStsNoErr; + /* init pointer C = C1 || C2 || C3 */ + Ipp8u *pC1_PC = pOut; + Ipp8u *pC1_xy = pC1_PC + ciph_PC_size; + Ipp8u *pC3 = pC1_xy + ciph_xy_size; + Ipp8u *pC2 = pC3 + ciph_hash_size; + + /* point elliptic curve */ + IppsGFpECPoint R; + cpEcGFpInitPoint(&R, cpEcGFpGetPool(1, pEC), 0, pEC); + BNU_CHUNK_T *pX = NULL; + BNU_CHUNK_T *pY = NULL; + + /* step 2: C1 = PC || x1 || y1 */ + pC1_PC[0] = 0x04; + + pX = (BNU_CHUNK_T *)(pC1_xy); + pY = pX + elemSize; + + gfec_GetPoint(pX, pY, pEphPublicKey, pEC); + GFP_METHOD(pME)->decode(pX, pX, pME); + GFP_METHOD(pME)->decode(pY, pY, pME); + cpSM2KE_reverse_inplace((Ipp8u *)pX, elemBytes); + cpSM2KE_reverse_inplace((Ipp8u *)pY, elemBytes); + + /* step 4: [k]Pb = (x2,y2) */ + ippsGFpECMulPoint(pPublicKey, pEphPrvKey, &R, pEC, pScratchBuffer); + /* step 5: t = KDF(x2||y2, klen) */ + BNU_CHUNK_T *pBuff = cpGFpGetPool(2, pME); + + pX = pBuff; + pY = pX + elemSize; + + gfec_GetPoint(pX, pY, &R, pEC); + GFP_METHOD(pME)->decode(pX, pX, pME); + GFP_METHOD(pME)->decode(pY, pY, pME); + + cpSM2KE_reverse_inplace((Ipp8u *)pX, elemBytes); + cpSM2KE_reverse_inplace((Ipp8u *)pY, elemBytes); + + KDF_sm3(pC2, ciph_msg_size, (Ipp8u *)pBuff, 2 * elemBytes); + + /* step 6: C2 = M (x) t */ + for (int i = 0; i < ciph_msg_size; ++i) { + pC2[i] = pC2[i] ^ pInp[i]; + } + + /* step 7: C3 = Hash(x2 || M || y2) */ + static IppsHashState_rmf ctx; + + ippsHashInit_rmf(&ctx, ippsHashMethod_SM3()); + /* x2 */ + ippsHashUpdate_rmf((Ipp8u *)pX, elemBytes, &ctx); + /* M */ + ippsHashUpdate_rmf(pInp, ciph_msg_size, &ctx); + /* y2 */ + ippsHashUpdate_rmf((Ipp8u *)pY, elemBytes, &ctx); + /* C3 */ + ippsHashFinal_rmf(pC3, &ctx); + + /* set output size data */ + if (sts == ippStsNoErr) { + *pOutSize = ciph_PC_size + ciph_xy_size + ciph_msg_size + ciph_hash_size; + } + /* zeros Public | Private Key */ + cpBN_zero(pEphPrvKey); + gfec_SetPointAtInfinity(pEphPublicKey); + + cpGFpReleasePool(2, pME); + cpEcGFpReleasePool(1, pEC); /* release R from the pool */ + return sts; + } +} + +#undef CHECK_PRIVATE_KEY +#undef CHECK_PUBLIC_KEY diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_encrypt_ext_enc_msg_size.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_encrypt_ext_enc_msg_size.c new file mode 100644 index 0000000..2d153fc --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_encrypt_ext_enc_msg_size.c @@ -0,0 +1,68 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + +#include "owncp.h" +#include "owndefs.h" +#include "sm2/sm2_stuff.h" + +/** + * @brief ippsGFpECEncryptSM2_Ext_EncMsgSize + * Implemenation based on standart: + * GM/T 0003.4-2012 SM2 + * Public key cryptographic algorithm SM2 based on elliptic curves + * Part 4: Public key encryption algorithm + * Get Cipher Size data by Encrypt SM2 + * @param [in] pEC Context Elliptic Curve + * @param [in] ctMsgSize chipher text message size + * @param [out] pSize size allocation bytes by Key Exchange Context + * @return + * ippStsNoErr - successful + * ippStsNullPtrErr - if pEC or pSize is NULL + * ippStsContextMatchErr - if pEC no valid ID or no exists SUBGROUP + * ippStsBagArgErr - if message size < 0 + */ +IPPFUN(IppStatus, ippsGFpECEncryptSM2_Ext_EncMsgSize, (const IppsGFpECState *pEC, int ctMsgSize, int *pSize)) +{ + /* check Context Elliptic Curve */ + IPP_BAD_PTR2_RET(pEC, pSize); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + IPP_BADARG_RET(!ECP_SUBGROUP(pEC), ippStsContextMatchErr); + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); /* base P */ + IPP_BADARG_RET(1 < GFP_EXTDEGREE(pME), ippStsNotSupportedModeErr); + gsModEngine *nME = ECP_MONT_R(pEC); /* base N */ + IPP_BADARG_RET(1 < GFP_EXTDEGREE(nME), ippStsNotSupportedModeErr); + + const int elemSize = GFP_FELEN(pME); /* size BNU_CHUNK */ + + /* check message size */ + IPP_BADARG_RET(!(ctMsgSize >= 0), ippStsOutOfRangeErr) + + { + const int ciph_PC_size = 1; + const int ciph_xy_size = 2 * (Ipp32s)sizeof(BNU_CHUNK_T) * elemSize; + const int ciph_hash_size = IPP_SM3_DIGEST_BYTESIZE; + const int ciph_msg_size = ctMsgSize; + + const int size = ciph_PC_size /* PC */ + + ciph_xy_size /* Point (x,y) */ + + ciph_hash_size /* SM3 */ + + ciph_msg_size; /* message size */ + + *pSize = size; + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_key_exchange_confirmation.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_key_exchange_confirmation.c new file mode 100644 index 0000000..cb71039 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_key_exchange_confirmation.c @@ -0,0 +1,116 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owncp.h" +#include "owndefs.h" +#include "sm2/sm2_key_exchange_method.h" +#include "sm2/sm2_stuff.h" + +/** + * @brief ippsGFpECKeyExchangeSM2_Confirm + * this step Optional in protocol SM2 Key Exchange - Confirmation Peer data + * see standart: + * [GBT.32918.3-2016] Public Key cryptographic algorithm SM2 based on elliptic curves + * Part 3: Key exchange protocol + * 6.2 Process of key exchange protocol + * stack compute[standart link]: + * [user A| user B] + * 8) S(1/2) = SM3( 0x0(2/3) || y(u/v) || tmp_p ) [step 9| step 10] + * when tmp_p - precomute in SharedKey step + * @param [in] pSPeer S(b/a) peer data confirmation + * @param [out] pStatus status confirmation + * 1 is succsessfull or 0 is bad confirmation + * @param [in] pKE constrex Key Excange + * @return + * ippStsNoErr - successful + * ippStsNullPtrErr - if pEC | pKE | pSPeer | pStatus is NULL + * ippStsContextMatchErr - if pEC no valid ID | pEC no exists SUBGROUP + * ippStsNotSupportedModeErr - if pEC no supported (n|p) modulus arithmetic + * ippStsRangeErr - if BitSize(pEC) < IPP_SM3_DIGEST_BITSIZE + * ippStsBadArgErr - if role(pKE) no equal ippKESM2Requester|ippKESM2Responder + */ +/* clang-format off */ +IPPFUN(IppStatus, ippsGFpECKeyExchangeSM2_Confirm, (const Ipp8u pSPeer[IPP_SM3_DIGEST_BYTESIZE], + int* pStatus, + IppsGFpECKeyExchangeSM2State* pKE)) +/* clang-format on */ +{ + /* check Key Exchange */ + IPP_BAD_PTR1_RET(pKE); + /* check id */ + IPP_BADARG_RET(!EC_SM2_KEY_EXCH_VALID_ID(pKE), ippStsContextMatchErr); + /* check User Role */ + const IppsKeyExchangeRoleSM2 role = EC_SM2_KEY_EXCH_ROLE(pKE); + IPP_BADARG_RET(((ippKESM2Requester != role) && (ippKESM2Responder != role)), ippStsBadArgErr); + + /* check Elliptic Curve */ + IppsGFpECState *pEC = EC_SM2_KEY_EXCH_EC(pKE); + + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + IPP_BADARG_RET(!ECP_SUBGROUP(pEC), ippStsContextMatchErr); + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); /* base P */ + IPP_BADARG_RET(1 < GFP_EXTDEGREE(pME), ippStsNotSupportedModeErr); + gsModEngine *nME = ECP_MONT_R(pEC); /* base N */ + IPP_BADARG_RET(1 < GFP_EXTDEGREE(nME), ippStsNotSupportedModeErr); + /* check bitsize >= SM3_DIGSET */ + IPP_BADARG_RET(!(ECP_ORDBITSIZE(pEC) >= IPP_SM3_DIGEST_BITSIZE), ippStsRangeErr); + + /* check call Setup */ + /* check init Public Key | Ephemeral Public Key */ + IPP_BADARG_RET( NULL == EC_SM2_KEY_EXCH_PUB_KEY_USER_A(pKE) || + NULL == EC_SM2_KEY_EXCH_PUB_KEY_USER_B(pKE) || + NULL == EC_SM2_KEY_EXCH_EPH_PUB_KEY_USER_A(pKE) || + NULL == EC_SM2_KEY_EXCH_EPH_PUB_KEY_USER_B(pKE), ippStsContextMatchErr) + + /* check pSPeer */ + IPP_BAD_PTR1_RET(pSPeer); + + /* check status */ + IPP_BAD_PTR1_RET(pStatus); + + + const Ipp8u firstNum = (ippKESM2Requester == role) ? 0x02 : 0x03; + + { + /* extract data Elliptic Curve */ + const int elemBits = GFP_FEBITLEN(pME); /* size Bits */ + const int elemBytes = (elemBits + 7) / 8; /* size Bytes */ + + /* buffer */ + BNU_CHUNK_T *pDataBuff = cpGFpGetPool(3, pME); + Ipp8u *pBuff = (Ipp8u *)pDataBuff; + + /* 8) S(1/2) = SM3( 0x0(2/3) || y(u/v) || tmp_p ) */ + /* copy 0x0(2/3) */ + pBuff[0] = firstNum; /* set first number (role) */ + /* copy y(u/v) */ + cpSM2_CopyBlock(pBuff + 1, (const Ipp8u *)EC_SM2_KEY_EXCH_POINT_Y(pKE), elemBytes); + /* copy Precomp Hash */ + cpSM2_CopyBlock(pBuff + 1 + elemBytes, (const Ipp8u *)EC_SM2_KEY_EXCH_PRECOM_HASH(pKE), IPP_SM3_DIGEST_BYTESIZE); + + const int sizeSab_SM3 = 1 + elemBytes + IPP_SM3_DIGEST_BYTESIZE; + cpSM2KE_compute_hash_SM3(pBuff, pBuff, sizeSab_SM3); + + /* pS == S(1/2) */ + *pStatus = EquBlock((Ipp8u *)pBuff, pSPeer, IPP_SM3_DIGEST_BYTESIZE) ? 1 : 0; + + cpGFpReleasePool(3, pME); + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_key_exchange_get_size.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_key_exchange_get_size.c new file mode 100644 index 0000000..871cf9c --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_key_exchange_get_size.c @@ -0,0 +1,58 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owncp.h" +#include "owndefs.h" +#include "sm2/sm2_key_exchange_method.h" +#include "sm2/sm2_stuff.h" + +/** + * @brief ippsGFpECKeyExchangeSM2_GetSize + * Get Size allocation bytes by Key Exchange Conntext + * @param [in] pEC Context Elliptic Curve + * @param [out] pSize size allocation bytes by Key Exchange Context + * @return + * ippStsNoErr - successful + * ippStsNullPtrErr - if pEC or pSize is NULL + * ippStsContextMatchErr - if pEC no valid ID or no exists SUBGROUP + */ +IPPFUN(IppStatus, ippsGFpECKeyExchangeSM2_GetSize, (const IppsGFpECState *pEC, int *pSize)) +{ + + IPP_BAD_PTR2_RET(pEC, pSize); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + IPP_BADARG_RET(!ECP_SUBGROUP(pEC), ippStsContextMatchErr); + + { + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); /* base P */ + const int elemSize = GFP_FELEN(pME); /* size BNU_CHUNK */ + + const int size = (Ipp32s)sizeof(IppsGFpECKeyExchangeSM2State) /* Key Exchange SM2 struct */ + + (Ipp32s)sizeof(IppsGFpECPoint) + 3 * (Ipp32s)sizeof(BNU_CHUNK_T) * elemSize /* User A Public Key + Data */ + + (Ipp32s)sizeof(IppsGFpECPoint) + 3 * (Ipp32s)sizeof(BNU_CHUNK_T) * elemSize /* User B Public Key + Data */ + + (Ipp32s)sizeof(IppsGFpECPoint) + 3 * (Ipp32s)sizeof(BNU_CHUNK_T) * elemSize /* User A Ephemeral Public Key + Data */ + + (Ipp32s)sizeof(IppsGFpECPoint) + 3 * (Ipp32s)sizeof(BNU_CHUNK_T) * elemSize /* User B Ephemeral Public Key + Data */ + + (Ipp32s)sizeof(Ipp8u) * IPP_SM3_DIGEST_BYTESIZE /* Za [User A] */ + + (Ipp32s)sizeof(Ipp8u) * IPP_SM3_DIGEST_BYTESIZE /* Zb [User B] */ + + (Ipp32s)sizeof(Ipp8u) * IPP_SM3_DIGEST_BYTESIZE /* Precompute Hash */ + + (Ipp32s)sizeof(BNU_CHUNK_T) * elemSize /* [U/V].x */ + + (Ipp32s)sizeof(BNU_CHUNK_T) * elemSize; /* [U/V].y */ + + *pSize = size; + return ippStsNoErr; + } +} \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_key_exchange_init.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_key_exchange_init.c new file mode 100644 index 0000000..12d0243 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_key_exchange_init.c @@ -0,0 +1,108 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owncp.h" +#include "owndefs.h" +#include "sm2/sm2_key_exchange_method.h" +#include "sm2/sm2_stuff.h" + +/** + * @brief ippsGFpECKeyExchangeSM2_Init + * initial Key Exchange Context + * see standart: + * [GBT.32918.3-2016] Public Key cryptographic algorithm SM2 based on elliptic curves + * Part 3: Key exchange protocol + * @param [out] pKE context Key Exchange + * @param [in] role User(A|B) role (ippKESM2Requester|ippKESM2Responder) + * @param [in] pEC conext Elliptic Curve + * @return + * ippStsNoErr - successful + * ippStsNullPtrErr - if pEC | pKE is NULL + * ippStsContextMatchErr - if pEC no valid ID or no exists SUBGROUP + * ippStsNotSupportedModeErr - if pEC no supported n or p modulus arithmetic + * ippStsRangeErr - if BitSize(pEC) < IPP_SM3_DIGEST_BITSIZE + * ippStsBadArgErr - if role no equal ippKESM2Requester|ippKESM2Responder + */ +IPPFUN(IppStatus, ippsGFpECKeyExchangeSM2_Init, (IppsGFpECKeyExchangeSM2State * pKE, IppsKeyExchangeRoleSM2 role, IppsGFpECState *pEC)) +{ + /* check Elliptic Curve */ + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + IPP_BADARG_RET(!ECP_SUBGROUP(pEC), ippStsContextMatchErr); + /* check support Mode arithmetic */ + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); /* base P */ + IPP_BADARG_RET(1 < GFP_EXTDEGREE(pME), ippStsNotSupportedModeErr); + gsModEngine *nME = ECP_MONT_R(pEC); /* base N */ + IPP_BADARG_RET(1 < GFP_EXTDEGREE(nME), ippStsNotSupportedModeErr); + /* check bitsize >= SM3_DIGSET */ + IPP_BADARG_RET(!(ECP_ORDBITSIZE(pEC) >= IPP_SM3_DIGEST_BITSIZE), ippStsRangeErr); + + /* check Key Exchange */ + IPP_BAD_PTR1_RET(pKE); + + /* check User Role */ + IPP_BADARG_RET(((ippKESM2Requester != role) && (ippKESM2Responder != role)), ippStsBadArgErr); + + { + const int elemSize = GFP_FELEN(pME); /* size BNU_CHUNK */ + /* set id */ + EC_SM2_KEY_EXCH_SET_ID(pKE); + + /* set role [User A| User B] */ + EC_SM2_KEY_EXCH_ROLE(pKE) = role; + + /* set context EC */ + EC_SM2_KEY_EXCH_EC(pKE) = pEC; + + /* set zero pointer to public key */ + /* public key */ + EC_SM2_KEY_EXCH_PUB_KEY_USER_A(pKE) = NULL; + EC_SM2_KEY_EXCH_PUB_KEY_USER_B(pKE) = NULL; + /* ephemera key */ + EC_SM2_KEY_EXCH_EPH_PUB_KEY_USER_A(pKE) = NULL; + EC_SM2_KEY_EXCH_EPH_PUB_KEY_USER_B(pKE) = NULL; + + Ipp8u *ptr = (Ipp8u *)(pKE); + ptr += (Ipp32s)sizeof(IppsGFpECKeyExchangeSM2State); + + /* skip Public Key Space */ + ptr += 4 * ((Ipp32s)sizeof(IppsGFpECPoint) + 3 * (Ipp32s)sizeof(BNU_CHUNK_T) * elemSize); + + /* set Za (user id hash) */ + EC_SM2_KEY_EXCH_USER_ID_HASH_USER_A(pKE) = ptr; + PurgeBlock(ptr, IPP_SM3_DIGEST_BYTESIZE); + ptr += IPP_SM3_DIGEST_BYTESIZE; + + /* set Zb (user id hash) */ + EC_SM2_KEY_EXCH_USER_ID_HASH_USER_B(pKE) = ptr; + PurgeBlock(ptr, IPP_SM3_DIGEST_BYTESIZE); + ptr += IPP_SM3_DIGEST_BYTESIZE; + + /* set Precompute Hash */ + EC_SM2_KEY_EXCH_PRECOM_HASH(pKE) = ptr; + PurgeBlock(ptr, IPP_SM3_DIGEST_BYTESIZE); + ptr += IPP_SM3_DIGEST_BYTESIZE; + + /* set Point(X,Y) */ + EC_SM2_KEY_EXCH_POINT_PTR(pKE) = (BNU_CHUNK_T *)ptr; + + ZEXPAND_BNU(EC_SM2_KEY_EXCH_POINT_X(pKE), 0, elemSize); + ZEXPAND_BNU(EC_SM2_KEY_EXCH_POINT_Y(pKE), 0, elemSize); + + return ippStsNoErr; + } +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_key_exchange_method.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_key_exchange_method.h new file mode 100644 index 0000000..29655d5 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_key_exchange_method.h @@ -0,0 +1,63 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ +#ifndef _SM2_KEY_EXCHANGE_METHOD_H_ +#define _SM2_KEY_EXCHANGE_METHOD_H_ + +#include "owncp.h" +#include "pcpbnuimpl.h" +#include "pcpgfpecstuff.h" + +/* +// EC over GF(p) Key Exchange SM2 context +*/ +typedef struct _GFpECKeyExchangeSM2 { + Ipp32u idCtx; /* context identifier */ + IppsKeyExchangeRoleSM2 role; /* role User A(request) or B(response) */ + IppsGFpECState *pEC; /* Elliptic Curve data context */ + IppsGFpECPoint *pPubKeyReqA; /* Pa[User A] - Public Key */ + IppsGFpECPoint *pPubKeyRespB; /* Pb[User B] - Public Key */ + IppsGFpECPoint *pEphPubKeyReqA; /* Ra[User A] - Ephemeral Public Key */ + IppsGFpECPoint *pEphPubKeyRespB; /* Rb[User B] - Ephemeral Public Key */ + Ipp8u *pZReqA; /* Za[User A] - User ID Hash */ + Ipp8u *pZRespB; /* Zb[User B] - User ID Hash */ + Ipp8u *pPrecHash; /* Precompute Hash - SM3( x(u/v) || Za || Zb || xa || ya || xb || yb ) */ + BNU_CHUNK_T *pPointXYBigEndian; /* U/V */ +} GFpECKeyExchangeSM2; + + +/* +// Context Access Macros +*/ +/* clang-format off */ +#define EC_SM2_KEY_EXCH_ID(ctx) ((ctx)->idCtx) +#define EC_SM2_KEY_EXCH_SET_ID(ctx) (EC_SM2_KEY_EXCH_ID(ctx) = (Ipp32u)idCtxGFPECKE ^ (Ipp32u)IPP_UINT_PTR(ctx)) +#define EC_SM2_KEY_EXCH_VALID_ID(ctx) ((EC_SM2_KEY_EXCH_ID(ctx) ^ (Ipp32u)IPP_UINT_PTR(ctx)) == (Ipp32u)idCtxGFPECKE) +#define EC_SM2_KEY_EXCH_ROLE(ctx) ((ctx)->role) +#define EC_SM2_KEY_EXCH_EC(ctx) ((ctx)->pEC) +#define EC_SM2_KEY_EXCH_PUB_KEY_USER_A(ctx) ((ctx)->pPubKeyReqA) +#define EC_SM2_KEY_EXCH_PUB_KEY_USER_B(ctx) ((ctx)->pPubKeyRespB) +#define EC_SM2_KEY_EXCH_EPH_PUB_KEY_USER_A(ctx) ((ctx)->pEphPubKeyReqA) +#define EC_SM2_KEY_EXCH_EPH_PUB_KEY_USER_B(ctx) ((ctx)->pEphPubKeyRespB) +#define EC_SM2_KEY_EXCH_USER_ID_HASH_USER_A(ctx) ((ctx)->pZReqA) +#define EC_SM2_KEY_EXCH_USER_ID_HASH_USER_B(ctx) ((ctx)->pZRespB) +#define EC_SM2_KEY_EXCH_PRECOM_HASH(ctx) ((ctx)->pPrecHash) +#define EC_SM2_KEY_EXCH_POINT_PTR(ctx) ((ctx)->pPointXYBigEndian) +#define EC_SM2_KEY_EXCH_POINT_X(ctx) ((ctx)->pPointXYBigEndian) +#define EC_SM2_KEY_EXCH_POINT_Y(ctx) ((ctx)->pPointXYBigEndian + GFP_FELEN(GFP_PMA(ECP_GFP(EC_SM2_KEY_EXCH_EC(ctx)))) ) +/* clang-format on */ + +#endif // _SM2_KEY_EXCHANGE_METHOD_H_ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_key_exchange_setup.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_key_exchange_setup.c new file mode 100644 index 0000000..10b229e --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_key_exchange_setup.c @@ -0,0 +1,145 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owncp.h" +#include "owndefs.h" +#include "pcpgfpecstuff.h" +#include "sm2/sm2_key_exchange_method.h" +#include "sm2/sm2_stuff.h" + +#define CHECK_PUBLIC_KEY(KEY) \ + IPP_BAD_PTR1_RET((KEY)) \ + IPP_BADARG_RET(!ECP_POINT_VALID_ID((KEY)), ippStsContextMatchErr) \ + IPP_BADARG_RET(ECP_POINT_FELEN((KEY)) != GFP_FELEN(pME), ippStsOutOfRangeErr) \ + IPP_BADARG_RET(0 == gfec_IsPointOnCurve((KEY), pEC), ippStsInvalidPoint) + + +/** + * @brief ippsGFpECKeyExchangeSM2_Setup + * setup Key Exchange Context (add Za | Zb | Pa | Pb | Ra | Rb) + * see standart: + * [GBT.32918.3-2016] Public Key cryptographic algorithm SM2 based on elliptic curves + * Part 3: Key exchange protocol + * @param [in] pZSelf (self) user ID hash Z (copy) + * @param [in] pZPeer (peer) user ID hash Z (copy) + * @param [in] pPublicKeySelf (self) public key (copy) + * @param [in] pPublicKeyPeer (peer) public key (copy) + * @param [in] pEphPublicKeySelf (self) ephemeral public key (copy) + * @param [in] pEphPublicKeyPeer (peer) ephemeral public key (copy) + * @param [out] pKE constext + * @return + * ippStsNoErr - successful + * ippStsNullPtrErr - if pEC | pKE is NULL + * ippStsContextMatchErr - if pEC no valid ID | pEC no exists SUBGROUP + * ippStsNotSupportedModeErr - if pEC no supported (n|p) modulus arithmetic + * ippStsRangeErr - if BitSize(pEC) < IPP_SM3_DIGEST_BITSIZE + * ippStsBadArgErr - if role(pKE) no equal ippKESM2Requester|ippKESM2Responder + * ippStsInvalidPoint - if no in curve Pub Key (User A|B) | Eph Pub Key (User A|B) + */ +/* clang-format off */ +IPPFUN(IppStatus, ippsGFpECKeyExchangeSM2_Setup, (const Ipp8u pZSelf[IPP_SM3_DIGEST_BYTESIZE], + const Ipp8u pZPeer[IPP_SM3_DIGEST_BYTESIZE], + const IppsGFpECPoint *pPublicKeySelf, + const IppsGFpECPoint *pPublicKeyPeer, + const IppsGFpECPoint *pEphPublicKeySelf, + const IppsGFpECPoint *pEphPublicKeyPeer, + IppsGFpECKeyExchangeSM2State *pKE)) +/* clang-format on */ +{ + /* check Key Exchange */ + IPP_BAD_PTR1_RET(pKE); + /* check id */ + IPP_BADARG_RET(!EC_SM2_KEY_EXCH_VALID_ID(pKE), ippStsContextMatchErr); + /* check User Role */ + const IppsKeyExchangeRoleSM2 role = EC_SM2_KEY_EXCH_ROLE(pKE); + IPP_BADARG_RET(((ippKESM2Requester != role) && (ippKESM2Responder != role)), ippStsBadArgErr); + + /* check Za | Zb */ + IPP_BAD_PTR2_RET(pZSelf, pZPeer); + + /* check Elliptic Curve */ + IppsGFpECState *pEC = EC_SM2_KEY_EXCH_EC(pKE); + + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + IPP_BADARG_RET(!ECP_SUBGROUP(pEC), ippStsContextMatchErr); + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); /* base P */ + IPP_BADARG_RET(1 < GFP_EXTDEGREE(pME), ippStsNotSupportedModeErr); + gsModEngine *nME = ECP_MONT_R(pEC); /* base N */ + IPP_BADARG_RET(1 < GFP_EXTDEGREE(nME), ippStsNotSupportedModeErr); + /* check bitsize >= SM3_DIGSET */ + IPP_BADARG_RET(!(ECP_ORDBITSIZE(pEC) >= IPP_SM3_DIGEST_BITSIZE), ippStsRangeErr); + + /* check data User A */ + /* public and ephemeral public key */ + CHECK_PUBLIC_KEY(pPublicKeySelf) + CHECK_PUBLIC_KEY(pEphPublicKeySelf) + + /* check data User B */ + /* public and ephemeral public key */ + CHECK_PUBLIC_KEY(pPublicKeyPeer) + CHECK_PUBLIC_KEY(pEphPublicKeyPeer) + + { + const int elemSize = GFP_FELEN(pME); /* size BNU_CHUNK */ + + /* init pointer copy User A|B */ + Ipp8u *pCopyZSelf = (ippKESM2Requester == role) ? EC_SM2_KEY_EXCH_USER_ID_HASH_USER_A(pKE) : EC_SM2_KEY_EXCH_USER_ID_HASH_USER_B(pKE); + Ipp8u *pCopyZPeer = (ippKESM2Requester == role) ? EC_SM2_KEY_EXCH_USER_ID_HASH_USER_B(pKE) : EC_SM2_KEY_EXCH_USER_ID_HASH_USER_A(pKE); + /* copy Za and Zb */ + cpSM2_CopyBlock(pCopyZSelf, pZSelf, IPP_SM3_DIGEST_BYTESIZE); + cpSM2_CopyBlock(pCopyZPeer, pZPeer, IPP_SM3_DIGEST_BYTESIZE); + + /* init ptr pointer */ + /* User A */ + const IppsGFpECPoint *loc_pub_key_user_a = (ippKESM2Requester == role) ? pPublicKeySelf : pPublicKeyPeer; + const IppsGFpECPoint *loc_eph_pub_key_user_a = (ippKESM2Requester == role) ? pEphPublicKeySelf : pEphPublicKeyPeer; + /* User B */ + const IppsGFpECPoint *loc_pub_key_user_b = (ippKESM2Requester == role) ? pPublicKeyPeer : pPublicKeySelf; + const IppsGFpECPoint *loc_eph_pub_key_user_b = (ippKESM2Requester == role) ? pEphPublicKeyPeer : pEphPublicKeySelf; + /* Copy Pointer User A|B */ + const Ipp32s size_struct_point = (Ipp32s)sizeof(IppsGFpECPoint); + const Ipp32s size_data_point = 3 * (Ipp32s)sizeof(BNU_CHUNK_T) * elemSize; + + Ipp8u *ptr = (Ipp8u *)(pKE) + sizeof(IppsGFpECKeyExchangeSM2State); + + /* Public key User A */ + EC_SM2_KEY_EXCH_PUB_KEY_USER_A(pKE) = (IppsGFpECPoint *)ptr; + ptr += size_struct_point; + cpSM2KE_CopyPointData(EC_SM2_KEY_EXCH_PUB_KEY_USER_A(pKE), (BNU_CHUNK_T *)ptr, loc_pub_key_user_a, pEC); + ptr += size_data_point; + /* Ephemeral Public Key User A */ + EC_SM2_KEY_EXCH_EPH_PUB_KEY_USER_A(pKE) = (IppsGFpECPoint *)ptr; + ptr += size_struct_point; + cpSM2KE_CopyPointData(EC_SM2_KEY_EXCH_EPH_PUB_KEY_USER_A(pKE), (BNU_CHUNK_T *)ptr, loc_eph_pub_key_user_a, pEC); + ptr += size_data_point; + /* Public key User B */ + EC_SM2_KEY_EXCH_PUB_KEY_USER_B(pKE) = (IppsGFpECPoint *)ptr; + ptr += size_struct_point; + cpSM2KE_CopyPointData(EC_SM2_KEY_EXCH_PUB_KEY_USER_B(pKE), (BNU_CHUNK_T *)ptr, loc_pub_key_user_b, pEC); + ptr += size_data_point; + /* Ephemeral Public Key User B */ + EC_SM2_KEY_EXCH_EPH_PUB_KEY_USER_B(pKE) = (IppsGFpECPoint *)ptr; + ptr += size_struct_point; + cpSM2KE_CopyPointData(EC_SM2_KEY_EXCH_EPH_PUB_KEY_USER_B(pKE), (BNU_CHUNK_T *)ptr, loc_eph_pub_key_user_b, pEC); + + return ippStsNoErr; + } +} + +#undef CHECK_PUBLIC_KEY diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_key_exchange_shared_key.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_key_exchange_shared_key.c new file mode 100644 index 0000000..c805f30 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_key_exchange_shared_key.c @@ -0,0 +1,310 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owncp.h" +#include "owndefs.h" +#include "sm2/sm2_key_exchange_method.h" +#include "sm2/sm2_stuff.h" + +#define CHECK_PRIVATE_KEY(KEY) \ + IPP_BAD_PTR1_RET((KEY)) \ + IPP_BADARG_RET(!BN_VALID_ID((KEY)), ippStsContextMatchErr) \ + IPP_BADARG_RET(BN_NEGATIVE((KEY)), ippStsInvalidPrivateKey) \ + /* test if 0 < pPrivateA < Order */ \ + IPP_BADARG_RET(0 == gfec_CheckPrivateKey((KEY), pEC), ippStsInvalidPrivateKey) + +/** + * @brief ippsGFpECKeyExchangeSM2_SharedKey + * compute x(u/v) | y(u/v) | precomHash | shared key + * see standart: + * [GBT.32918.3-2016] Public Key cryptographic algorithm SM2 based on elliptic curves + * Part 3: Key exchange protocol + * 6.2 Process of key exchange protocol + * stack compute[standart link]: + * [user A| user B] + * 2) x(a/b)` = 2^w + (x(a/b) & (2^w – 1)) [step 4| step 3] + * 3) t(a/b) = (d(a/b) + x(a/b)`*r(a/b) ) mod n [step 5| step 4] + * 4) x(b/a)` = 2^w + ( x(b/a) & (2^w – 1) ) [step 6| step 5] + * 5) U/V = [h*t(a/b)]( P(b/a) + [x(b/a)`]R(b/a) ) = ( x(u/v), y(u/v) ) [step 7| step 6] + * tmp_p = SM3( x(u/v) || Za || Zb || xa || ya || xb || yb ) + * 6) S(a/b) = SM3( 0x0(3/2) || y(u/v) || tmp_p ) [step 10| step 8] + * 7) K(a/b) = KDF(x(u/v) || y(u/v) || Za || Zb, klen) [step 8| step 7] + * @param [out] pSharedKey shared key [K(a/b) = KDF(x(u/v) || y(u/v) || Za || Zb, klen)] + * @param [in] sharedKeySize length shared key + * @param [out] pSSelf (may be == NULL) self data confirmation + * @param [in] pPrvKey private key + * @param [in] pEphPrvKey ephemeral private key (!) will be cleared at the end if function successfully competed (!) + * @param [out] pKE context Key Exchange (save x(u/v) | y(u/v) | precompHash ) + * @param [in] pScratchBuffer supported buffer + * @return + * ippStsNoErr - successful + * ippStsNullPtrErr - if pEC | pKE | pSharedKey | pPrvKey | PEphPrvKey | pSB is NULL + * ippStsContextMatchErr - if pEC no valid ID | pEC no exists SUBGROUP + * ippStsNotSupportedModeErr - if pEC no supported (n|p) modulus arithmetic + * ippStsRangeErr - if BitSize(pEC) < IPP_SM3_DIGEST_BITSIZE + * ippStsBadArgErr - if role(pKE) no equal ippKESM2Requester|ippKESM2Responder or sharedKeySize <= 0 + * ippStsInvalidPrivateKey - if test is failed 0 < pPrvKey|pEphPrvKey < Order + * ippStsEphemeralKeyErr - if test is failed pEphPrvKey == pEphPublicKeySelf*G or if calculated U(V) is an + * infinity point, U/V = [h*t(a/b)]( P(b/a) + [x(b/a)`]R(b/a) ) = ( x(u/v), y(u/v) ) + */ +/* clang-format off */ +IPPFUN(IppStatus, ippsGFpECKeyExchangeSM2_SharedKey, (Ipp8u* pSharedKey, int sharedKeySize, + Ipp8u* pSSelf, + const IppsBigNumState* pPrvKey, + IppsBigNumState* pEphPrvKey, + IppsGFpECKeyExchangeSM2State *pKE, Ipp8u* pScratchBuffer)) +/* clang-format on */ +{ + /* check Key Exchange */ + IPP_BAD_PTR1_RET(pKE); + /* check id */ + IPP_BADARG_RET(!EC_SM2_KEY_EXCH_VALID_ID(pKE), ippStsContextMatchErr); + /* check User Role */ + const IppsKeyExchangeRoleSM2 role = EC_SM2_KEY_EXCH_ROLE(pKE); + IPP_BADARG_RET(((ippKESM2Requester != role) && (ippKESM2Responder != role)), ippStsBadArgErr); + + /* check Elliptic Curve */ + IppsGFpECState *pEC = EC_SM2_KEY_EXCH_EC(pKE); + + IPP_BAD_PTR1_RET(pEC); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + IPP_BADARG_RET(!ECP_SUBGROUP(pEC), ippStsContextMatchErr); + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); /* base P */ + IPP_BADARG_RET(1 < GFP_EXTDEGREE(pME), ippStsNotSupportedModeErr); + gsModEngine *nME = ECP_MONT_R(pEC); /* base N */ + IPP_BADARG_RET(1 < GFP_EXTDEGREE(nME), ippStsNotSupportedModeErr); + /* check bitsize >= SM3_DIGSET */ + IPP_BADARG_RET(!(ECP_ORDBITSIZE(pEC) >= IPP_SM3_DIGEST_BITSIZE), ippStsRangeErr); + + /* check call Setup */ + /* check init Public Key | Ephemeral Public Key */ + IPP_BADARG_RET( NULL == EC_SM2_KEY_EXCH_PUB_KEY_USER_A(pKE) || + NULL == EC_SM2_KEY_EXCH_PUB_KEY_USER_B(pKE) || + NULL == EC_SM2_KEY_EXCH_EPH_PUB_KEY_USER_A(pKE) || + NULL == EC_SM2_KEY_EXCH_EPH_PUB_KEY_USER_B(pKE), ippStsContextMatchErr) + + /* check Shared Key */ + IPP_BAD_PTR1_RET(pSharedKey); + /* [GBT.32918.3-2016] Public Key cryptographic algorithm SM2 based on elliptic curves + * Part 3: Key exchange protocol + * 5.4.3 Key derivation function + * (0 < sharedKeySize) - (+) check + * (sharedKeySize < (2^32 - 1)*log2(n) ) - NO NEED if use INT input data type + */ + IPP_BADARG_RET(!(sharedKeySize > 0), ippStsOutOfRangeErr); + + /* check buffer */ + IPP_BAD_PTR1_RET(pScratchBuffer); + + /* check Private Key */ + CHECK_PRIVATE_KEY(pPrvKey) + + /* check Ephemeral Private Key */ + CHECK_PRIVATE_KEY(pEphPrvKey) + +#if (_IPP32E >= _IPP32E_K1) + if (IsFeatureEnabled(ippCPUID_AVX512IFMA) && (cpID_PrimeTPM_SM2 == ECP_MODULUS_ID(pEC))) { + const IppStatus sts = gfec_key_exchange_sm2_shared_key_avx512(pSharedKey, sharedKeySize, pSSelf, pPrvKey, pEphPrvKey, pKE, pScratchBuffer); + return sts; + } +#endif // (_IPP32E >= _IPP32E_K1) + { + IppStatus sts = ippStsNoErr; + + /* setup Public | Ephemeral Public */ + const IppsGFpECPoint *pPubKey = (ippKESM2Requester == role) ? EC_SM2_KEY_EXCH_PUB_KEY_USER_B(pKE) : EC_SM2_KEY_EXCH_PUB_KEY_USER_A(pKE); + const IppsGFpECPoint *pEphPubKey = (ippKESM2Requester == role) ? EC_SM2_KEY_EXCH_EPH_PUB_KEY_USER_B(pKE) : EC_SM2_KEY_EXCH_EPH_PUB_KEY_USER_A(pKE); + const IppsGFpECPoint *pSelfEphPubKey = (ippKESM2Requester == role) ? EC_SM2_KEY_EXCH_EPH_PUB_KEY_USER_A(pKE) : EC_SM2_KEY_EXCH_EPH_PUB_KEY_USER_B(pKE); + const Ipp8u firstNum = (ippKESM2Requester == role) ? 0x03 : 0x02; + + /* check that Ephemeral Public and Private Keys are related (R(a/b)=r(a/b)G) */ + IppsGFpECPoint pTmpPoint; + cpEcGFpInitPoint(&pTmpPoint, cpEcGFpGetPool(1, pEC), 0, pEC); + BNU_CHUNK_T *pDataEphPrv = BN_NUMBER(pEphPrvKey); // Ephemeral Private Key -> r(a/b) + gfec_MulBasePoint(&pTmpPoint, pDataEphPrv, BN_SIZE(pEphPrvKey), pEC, pScratchBuffer); // r(a/b)*G + int result = gfec_ComparePoint(&pTmpPoint, pSelfEphPubKey, pEC); // R(a/b) == r(a/b)G ? + cpEcGFpReleasePool(1, pEC); // Release pool before the possible exit + IPP_BADARG_RET(!result, ippStsEphemeralKeyErr); + + /* extract data Elliptic Curve */ + BNU_CHUNK_T *pOrder = MOD_MODULUS(nME); /* data oreder */ + const int ordLen = MOD_LEN(nME); /* len order */ + + const int elemBits = GFP_FEBITLEN(pME); /* size Bits */ + const int elemBytes = (elemBits + 7) / 8; /* size Bytes */ + const int elemSize = GFP_FELEN(pME); /* size BNU_CHUNK */ + + /* create buffer data (it needes further use compute tmp_p) + * -> SM3( x(u/v)(0) || Za(1) || Zb(2) || xa(3) || ya(4) || xb(5) || yb(6) ) + */ + BNU_CHUNK_T *pDataBuff = cpGFpGetPool(7, pME); + BNU_CHUNK_T *_pXY_ = cpEcGFpGetPool(1, pEC); /* elem 3 */ + /* 2) x(a/b)` = 2^w + (x(a/b) & (2^w – 1)) */ + BNU_CHUNK_T *xa = pDataBuff + 3 * elemSize; /* xa(3) */ + BNU_CHUNK_T *ya = pDataBuff + 4 * elemSize; /* ya(4) */ + BNU_CHUNK_T *_xa_ = _pXY_; + /* point to affine coordinate */ + cpSM2KE_get_affine_ext_euclid(/* x = */ xa, /* y = */ ya, /* p = */ EC_SM2_KEY_EXCH_EPH_PUB_KEY_USER_A(pKE), /* pEC = */ pEC); + /* reduction x = 2^w + ( x & (2^w - 1) ) */ + cpSM2KE_reduction_x2w(_xa_, xa, pEC); + /* to big endian */ + cpSM2KE_xy_to_BE(xa, ya, pEC); + /* 4) x(b/a)` = 2^w + ( x(b/a) & (2^w – 1) ) */ + BNU_CHUNK_T *xb = pDataBuff + 5 * elemSize; /* xb(5) */ + BNU_CHUNK_T *yb = pDataBuff + 6 * elemSize; /* yb(6) */ + BNU_CHUNK_T *_xb_ = _pXY_ + elemSize; + /* point to affine coordinate */ + cpSM2KE_get_affine_ext_euclid(/* x = */ xb, /* y = */ yb, /* p = */ EC_SM2_KEY_EXCH_EPH_PUB_KEY_USER_B(pKE), /* pEC = */ pEC); + /* reduction x = 2^w + ( x & (2^w - 1) ) */ + cpSM2KE_reduction_x2w(_xb_, xb, pEC); + /* to big endian */ + cpSM2KE_xy_to_BE(xb, yb, pEC); + + BNU_CHUNK_T *_xf_ = (ippKESM2Requester == role) ? _xa_ : _xb_; + BNU_CHUNK_T *_xs_ = (ippKESM2Requester == role) ? _xb_ : _xa_; + + /* 3) t(a/b) = (d(a/b) + x(a/b)`*r(a/b) ) mod n */ + BNU_CHUNK_T *t = pDataBuff + elemSize; /* temporary use slot Za(1) */ + /* Private key -> d(a/b) */ + BNU_CHUNK_T *pTmpPrv = pDataBuff; /* temporary use slot x(u/v)(0) */ + cpGFpElementCopy(pTmpPrv, BN_NUMBER(pPrvKey), elemSize); + GFP_METHOD(nME)->encode(pTmpPrv, pTmpPrv, nME); + /* Ephemeral Private Key -> r(a/b) */ + GFP_METHOD(nME)->encode(pDataEphPrv, pDataEphPrv, nME); + /* x(a/b)` */ + GFP_METHOD(nME)->encode(_xf_, _xf_, nME); + + GFP_METHOD(nME)->mul(pDataEphPrv, pDataEphPrv, _xf_, nME); /* x(a/b)`*r(a/b) mod n */ + BNU_CHUNK_T *pBuffTmp = pDataBuff + 2 * elemSize; /* temporary use slot Zb(2) */ + cpModAdd_BNU(t, pDataEphPrv, pTmpPrv, pOrder, ordLen, pBuffTmp); /* (d(a/b) + x(a/b)`*r(a/b) ) mod n */ + + /* 5) U/V = [h*t(a/b)]( P(b/a) + [x(b/a)`]R(b/a) ) = ( x(u/v), y(u/v) ) */ + /* [x(b/a)`]R(b/a) */ + const int scalarBits = ((elemBits + 1) / 2); + cpEcGFpInitPoint(&pTmpPoint, cpEcGFpGetPool(1, pEC), 0, pEC); + gfec_point_mul(ECP_POINT_X(&pTmpPoint), ECP_POINT_X(pEphPubKey), (Ipp8u *)_xs_, scalarBits, pEC, pScratchBuffer); + /* P(b/a) + [x(b/a)`]R(b/a) */ + gfec_point_add(ECP_POINT_X(&pTmpPoint), ECP_POINT_X(&pTmpPoint), ECP_POINT_X(pPubKey), pEC); + + /* h*t(a/b) */ + BNU_CHUNK_T *pCofactor = ECP_COFACTOR(pEC); + const int cofactorLen = cpGFpElementLen(pCofactor, elemSize); + + if (!GFP_IS_ONE(pCofactor, cofactorLen)) { + cpMontMul_BNU_EX(t, t, elemSize, pCofactor, cofactorLen, nME); + } + GFP_METHOD(nME)->decode(t, t, nME); + /* we can do this because slot Zb(2) not Use Now */ + cpGFpElementCopyPad(t, elemSize + 1, t, elemSize); + /* U/V = [h*t(a/b)]( P(b/a) + [x(b/a)`]R(b/a) ) = ( x(u/v), y(u/v) ) */ + gfec_point_mul(ECP_POINT_X(&pTmpPoint), ECP_POINT_X(&pTmpPoint), (Ipp8u *)t, elemBits, pEC, pScratchBuffer); + /* check U/V == 0 */ + ECP_POINT_FLAGS(&pTmpPoint) = gfec_IsPointAtInfinity(&pTmpPoint) ? 0 : ECP_FINITE_POINT; + if(ECP_POINT_FLAGS(&pTmpPoint) == 0) { + sts = ippStsEphemeralKeyErr; + } + + /* extract Affine Coordinate X|Y */ + /* save x(u/v), y(u/v) */ + BNU_CHUNK_T *x_af = EC_SM2_KEY_EXCH_POINT_X(pKE); + BNU_CHUNK_T *y_af = EC_SM2_KEY_EXCH_POINT_Y(pKE); + /* point to affine coordinate */ + cpSM2KE_get_affine_ext_euclid(/* x = */ x_af, /* y = */ y_af, /* p = */ &pTmpPoint, /* pEC = */ pEC); + /* to big endian */ + cpSM2KE_xy_to_BE(x_af, y_af, pEC); + + /* Precompute Hash */ + Ipp8u *pBuff = (Ipp8u *)pDataBuff; + /* tmp_p = SM3( x(u/v) || Za || Zb || xa || ya || xb || yb ) */ + /* copy x(u/v)(0) */ + cpSM2_CopyBlock(pBuff, (const Ipp8u *)x_af, elemBytes); + pBuff += elemBytes; + /* copy Za(1) */ + cpSM2_CopyBlock(pBuff, EC_SM2_KEY_EXCH_USER_ID_HASH_USER_A(pKE), IPP_SM3_DIGEST_BYTESIZE); + pBuff += IPP_SM3_DIGEST_BYTESIZE; + /* copy Zb(2) */ + cpSM2_CopyBlock(pBuff, EC_SM2_KEY_EXCH_USER_ID_HASH_USER_B(pKE), IPP_SM3_DIGEST_BYTESIZE); + pBuff += IPP_SM3_DIGEST_BYTESIZE; + /* recopy xa */ + cpSM2_CopyBlock(pBuff, (const Ipp8u *)xa, elemBytes); + pBuff += elemBytes; + /* recopy ya */ + cpSM2_CopyBlock(pBuff, (const Ipp8u *)ya, elemBytes); + pBuff += elemBytes; + /* recopy xb */ + cpSM2_CopyBlock(pBuff, (const Ipp8u *)xb, elemBytes); + pBuff += elemBytes; + /* recopy yb */ + cpSM2_CopyBlock(pBuff, (const Ipp8u *)yb, elemBytes); + + /* set pointer start */ + pBuff = (Ipp8u *)pDataBuff; + + const int sizeS_SM3 = elemBytes + 2 * IPP_SM3_DIGEST_BYTESIZE + 4 * elemBytes; + + /* copy to next operation */ + cpSM2KE_compute_hash_SM3(EC_SM2_KEY_EXCH_PRECOM_HASH(pKE), pBuff, sizeS_SM3); + + /* check U/V == 0 and S == NULL */ + if ((NULL != pSSelf) && IS_ECP_FINITE_POINT(&pTmpPoint)) { + /* 6) S(a/b) = SM3( 0x0(3/2) || y(u/v) || tmp_p ) */ + /* copy 0x0(3/2) */ + pBuff[0] = firstNum; + /* copy y(u/v) */ + cpSM2_CopyBlock(pBuff + 1, (const Ipp8u *)y_af, elemBytes); + /* copy tmp_p */ + cpSM2_CopyBlock(pBuff + 1 + elemBytes, EC_SM2_KEY_EXCH_PRECOM_HASH(pKE), IPP_SM3_DIGEST_BYTESIZE); + + const int sizeSab_SM3 = 1 + elemBytes + IPP_SM3_DIGEST_BYTESIZE; + cpSM2KE_compute_hash_SM3(pSSelf, pBuff, sizeSab_SM3); + } + + /* compute KDF */ + /* 7) K(a/b) = KDF(x(u/v) || y(u/v) || Za || Zb, klen) */ + /* copy x(u/v)(0)*/ + cpSM2_CopyBlock(pBuff, (const Ipp8u *)EC_SM2_KEY_EXCH_POINT_X(pKE), elemBytes); + pBuff += elemBytes; + /* copy y(u/v)(1) */ + cpSM2_CopyBlock(pBuff, (const Ipp8u *)EC_SM2_KEY_EXCH_POINT_Y(pKE), elemBytes); + pBuff += elemBytes; + /* copy Za(2) */ + cpSM2_CopyBlock(pBuff, EC_SM2_KEY_EXCH_USER_ID_HASH_USER_A(pKE), IPP_SM3_DIGEST_BYTESIZE); + pBuff += IPP_SM3_DIGEST_BYTESIZE; + /* copy Zb(3) */ + cpSM2_CopyBlock(pBuff, EC_SM2_KEY_EXCH_USER_ID_HASH_USER_B(pKE), IPP_SM3_DIGEST_BYTESIZE); + + const int sizeKab_KDF = 2 * elemBytes + 2 * IPP_SM3_DIGEST_BYTESIZE; + + /* set pointer start */ + pBuff = (Ipp8u *)pDataBuff; + + /* compute KDF */ + KDF_sm3(pSharedKey, sharedKeySize, (const Ipp8u *)pBuff, sizeKab_KDF); + + /* free buffers */ + cpEcGFpReleasePool(2, pEC); + cpGFpReleasePool(7, pME); + + /* clear Ephemeral Private Key */ + cpBN_zero(pEphPrvKey); + + return sts; + } +} + +#undef CHECK_PRIVATE_KEY diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_message_representation.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_message_representation.c new file mode 100644 index 0000000..779e6af --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_message_representation.c @@ -0,0 +1,107 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// Digesting message according to SM3 +// +// Contents: +// ippsGFpECMessageRepresentationSM2() +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpsm3stuff.h" +#include "pcpgfpstuff.h" +#include "pcpgfpecstuff.h" +#include "gsmodstuff.h" +#include "sm2/sm2_stuff.h" + + +/* clang-format off */ +IPPFUN(IppStatus, ippsGFpECMessageRepresentationSM2, (IppsBigNumState * pMsgDigest, + const Ipp8u* pMsg, int msgLen, + const Ipp8u* pUserID, int userIDLen, + const IppsGFpECPoint* pRegPublic, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +/* clang-format on */ +{ + IppsGFpState *pGF; + gsModEngine *pGFE; + + /* check curve data */ + IPP_BAD_PTR1_RET(pEC); + IPP_BAD_PTR1_RET(pScratchBuffer); + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr); + IPP_BADARG_RET(!ECP_SUBGROUP(pEC), ippStsContextMatchErr); + + /* check Message */ + IPP_BAD_PTR1_RET(pMsg); + /* check border (msgLen > 0) */ + IPP_BADARG_RET(!(msgLen > 0), ippStsOutOfRangeErr); + + /* check message digest */ + IPP_BAD_PTR1_RET(pMsgDigest); + IPP_BADARG_RET(!BN_VALID_ID(pMsgDigest), ippStsContextMatchErr); + /* make sure bitsize(pMsgDigest) <= bitsize(order) */ + IPP_BADARG_RET(!(cpBN_bitsize(pMsgDigest) <= ECP_ORDBITSIZE(pEC)), ippStsMessageErr); + + /* check User ID */ + IPP_BAD_PTR1_RET(pUserID); + /* check border (userIDLen > 0) */ + IPP_BADARG_RET(!(userIDLen > 0), ippStsOutOfRangeErr); + + pGF = ECP_GFP(pEC); + pGFE = GFP_PMA(pGF); + IPP_BADARG_RET(1 < GFP_EXTDEGREE(pGFE), ippStsNotSupportedModeErr); + + /* check Public Key */ + IPP_BAD_PTR1_RET(pRegPublic); + IPP_BADARG_RET(!ECP_POINT_VALID_ID(pRegPublic), ippStsContextMatchErr); + IPP_BADARG_RET(ECP_POINT_FELEN(pRegPublic) != GFP_FELEN(pGFE), ippStsOutOfRangeErr); + + Ipp8u Za[IPP_SM3_DIGEST_BYTESIZE]; + /* compute Za = SM3( ENTL || ID || a || b || xG || yG || xA || yA ) */ + const IppStatus sts = ippsGFpECUserIDHashSM2((Ipp8u *)Za, pUserID, userIDLen, pRegPublic, pEC, pScratchBuffer); + if(ippStsNoErr != sts){ + return sts; + } + + /* e = SM3(Za || M) */ + static IppsHashState_rmf ctx; + + ippsHashInit_rmf(&ctx, ippsHashMethod_SM3()); + /* Za */ + ippsHashUpdate_rmf(Za, sizeof(Za), &ctx); + /* M */ + ippsHashUpdate_rmf(pMsg, msgLen, &ctx); + + /* final */ + ippsHashFinal_rmf((Ipp8u *)(BN_NUMBER(pMsgDigest)), &ctx); + BN_SIGN(pMsgDigest) = ippBigNumPOS; + + /* clear stack data */ + PurgeBlock(Za, sizeof(Za)); + + return ippStsNoErr; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_stuff.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_stuff.c new file mode 100644 index 0000000..2707912 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_stuff.c @@ -0,0 +1,179 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" +#include "owncp.h" +#include "gsmodstuff.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcpsm3stuff.h" +#include "pcptool.h" + +#include "sm2/sm2_stuff.h" + +/** + * @brief + * compute Za digest = SM3( ENTL || ID || a || b || xG || yG || xA || yA ) + * @param [out] pZa_digest (Za) output digest Za + * @param [in] p_user_id (ID) data user ID + * @param [in] user_id_len (ENTL) length user ID + * @param [in] elem_len length element bytes + * @param [in] a (a) coefficient a of Elliptic Curve + * @param [in] b (b) coefficient b of Elliptic Curve + * @param [in] Gx (Gx) base point coordinate X of Elliptic Curve + * @param [in] Gy (Gy) base point coordinate Y of Elliptic Curve + * @param [in] pub_key_x (xA) public point coordinate X + * @param [in] pub_key_y (yA) public point coordinate Y + */ +/* clang-format off */ +IPP_OWN_DEFN(IppStatus, computeZa_user_id_hash_sm2, (Ipp8u * pZa_digest, + const Ipp8u *p_user_id, const int user_id_len, + const int elem_len, + const Ipp8u *a, const Ipp8u *b, + const Ipp8u *Gx, const Ipp8u *Gy, + const Ipp8u *pub_key_x, const Ipp8u *pub_key_y)) +/* clang-format on */ +{ + /* check pointer Za Digest | user id */ + IPP_BAD_PTR2_RET(pZa_digest, p_user_id); + /* check border (user_id_len > 0) | (elem_len > 0) */ + IPP_BADARG_RET(!(user_id_len > 0) || !(elem_len > 0), ippStsBadArgErr); + /* check (user_id_len*8 <= 0xFFFF) ~ (user_id_len <= 0x1FFF) for two bytes overflow. + user_id_len*8 operation will be executed in algorithm's flow */ + IPP_BADARG_RET(user_id_len > 0x1FFF, ippStsBadArgErr); + /* param curve: a, b, Gx, Gy */ + IPP_BAD_PTR2_RET(a, b); + IPP_BAD_PTR2_RET(Gx, Gy); + /* Public X|Y */ + IPP_BAD_PTR2_RET(pub_key_x, pub_key_y) + + static IppsHashState_rmf ctx; + + ippsHashInit_rmf(&ctx, ippsHashMethod_SM3()); + + /* compute Za = SM3( ENTL || ID || a || b || xG || yG || xA || yA ) */ + /* ENLT */ + const Ipp16u entl = ((user_id_len * 8) & 0xFFFF); + Ipp8u ENTL[sizeof(Ipp16u)]; + ENTL[0] = (Ipp8u)(entl >> 8); + ENTL[1] = (Ipp8u)(entl & 0xFF); + ippsHashUpdate_rmf(ENTL, sizeof(ENTL), &ctx); + /* ID */ + ippsHashUpdate_rmf(p_user_id, user_id_len, &ctx); + /* a */ + ippsHashUpdate_rmf(a, elem_len, &ctx); + /* b */ + ippsHashUpdate_rmf(b, elem_len, &ctx); + /* Gx */ + ippsHashUpdate_rmf(Gx, elem_len, &ctx); + /* Gy */ + ippsHashUpdate_rmf(Gy, elem_len, &ctx); + /* Px */ + ippsHashUpdate_rmf(pub_key_x, elem_len, &ctx); + /* Py */ + ippsHashUpdate_rmf(pub_key_y, elem_len, &ctx); + + /* final */ + ippsHashFinal_rmf(pZa_digest, &ctx); + + /* clear stack data */ + PurgeBlock(ENTL, sizeof(ENTL)); + + return ippStsNoErr; +} + +#define SIZE_CT (4) + +__INLINE void convert_ct_to_big_endian(Ipp8u pCt[SIZE_CT], const Ipp32u ct) +{ + pCt[0] = (Ipp8u)(ct >> 24); + pCt[1] = (Ipp8u)(ct >> 16); + pCt[2] = (Ipp8u)(ct >> 8); + pCt[3] = (Ipp8u)(ct); + return; +} + +/** + * @brief compute KDF base by SM3 hash + * @param [out] pKDF pointer output KDF + * @param [in] kdf_len length KDF + * @param [in] pZ data Z to create KDF + * @param [in] z_len length Z data + * bound kdf_len in standart: + * + */ +IPP_OWN_DEFN(IppStatus, KDF_sm3, (Ipp8u * pKDF, int kdf_len, const Ipp8u *pZ, const int z_len)) +{ + /* check pointer input */ + IPP_BAD_PTR2_RET(pKDF, pZ); + /* [GBT.32918.3-2016] Public Key cryptographic algorithm SM2 based on elliptic curves Part 3: Key exchange protocol + * 5.4.3 Key derivation function + * kdf_len < (2^32 - 1) * 2^5 = 2^37 - 32 [bytes] + * BUT if input INT type - NO NEED check this border + */ + /* check border (kdf_len >= 0) */ + IPP_BADARG_RET(!(kdf_len >= 0), ippStsBadArgErr); + /* check border (z_len > 0) */ + IPP_BADARG_RET(!(z_len > 0), ippStsBadArgErr); + + /* if kdf > 0 */ + if (kdf_len > 0) { + /* buffer */ + Ipp8u buff[IPP_SM3_DIGEST_BYTESIZE]; + + // step (a) + Ipp8u pCt[SIZE_CT]; + + const Ipp32u n = (Ipp32u)((kdf_len + IPP_SM3_DIGEST_BYTESIZE - 1) / IPP_SM3_DIGEST_BYTESIZE) + 1u; /* add 1 - loop start 1 */ + + /* init copy output len */ + int num_copy = IPP_SM3_DIGEST_BYTESIZE; + + static IppsHashState_rmf ctx; + ippsHashInit_rmf(&ctx, ippsHashMethod_SM3()); + + /* compute length K = Ha1 || Ha2 || ... */ + // step (b) + for (Ipp32u i = 1u; i < n; ++i) { + // step (b.1) -> Hai = H(Z || ct) + /* Z */ + ippsHashUpdate_rmf(pZ, z_len, &ctx); + /* ct */ + convert_ct_to_big_endian(pCt, i); + ippsHashUpdate_rmf(pCt, sizeof(pCt), &ctx); + /* auto init of end function - no need call in start loop */ + ippsHashFinal_rmf(buff, &ctx); + /* copy result */ + if ((i == n - 1u) && (0 != kdf_len % IPP_SM3_DIGEST_BYTESIZE)) { + num_copy = kdf_len % IPP_SM3_DIGEST_BYTESIZE; + } + cpSM2_CopyBlock(pKDF, buff, num_copy); + + /* update copy next reult */ + pKDF += num_copy; + kdf_len -= num_copy; + } + + /* clear stack data */ + PurgeBlock(buff, sizeof(buff)); + PurgeBlock(pCt, sizeof(pCt)); + } + + return ippStsNoErr; +} + +#undef SIZE_CT diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_stuff.h b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_stuff.h new file mode 100644 index 0000000..83d0602 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_stuff.h @@ -0,0 +1,181 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +/* +// +// Purpose: +// Cryptography Primitive. +// SM2 support function +// +// Contents: +// SM2 methods and constants +// +*/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcphash.h" +#include "pcphash_rmf.h" +#include "pcptool.h" +#include "pcpeccp.h" +#include "pcpgfpecstuff.h" +#include "pcpsm3stuff.h" +#include "sm2/sm2_key_exchange_method.h" + +#if !defined _SM2_STUFF_H +#define _SM2_STUFF_H + +#define cpSM2_CopyBlock(DST, SRC, SIZE) CopyBlock((SRC), (DST), (SIZE)) + +/** + * @brief + * reverse inplace array ( Little Endian to Big Endian data ) + * @param[in out] arr array data + * @param[in] len length array + */ +__INLINE void cpSM2KE_reverse_inplace(Ipp8u *arr, const int len) +{ +#define SWAPXOR(x, y) \ + (x) ^= (y); \ + (y) ^= (x); \ + (x) ^= (y); + + for (int i = 0; i < len / 2; ++i) { + SWAPXOR(arr[i], arr[len - 1 - i]) + } + return; +#undef SWAPXOR +} + +/** + * @brief cpSM2KE_CopyPointData + * init and copy Pointer + * @param[out] r point in Elliptic Curve + * @param[in] data buffer init pointer data + * @param[in] p point copy + * @param[in] pEC context Elliptic Curve + */ +__INLINE void cpSM2KE_CopyPointData(IppsGFpECPoint *r, BNU_CHUNK_T *data, const IppsGFpECPoint *p, const IppsGFpECState *pEC) +{ + ECP_POINT_SET_ID(r); + cpEcGFpInitPoint(r, data, ECP_POINT_FLAGS(p), pEC); + gfec_CopyPoint(r, p, ECP_POINT_FELEN(p)); + return; +} + +/** + * @brief + * reduction for the SM2 Key Exchange standard + * x` = 2^w + (x & (2^w – 1)) + * when + * w = log2(n)/2 - 1, n - number bytes order + * @param[out] r reduction value x` + * @param[in] a value x + * @param[in] pEC context Elliptic Curve + */ +__INLINE void cpSM2KE_reduction_x2w(BNU_CHUNK_T *r, const BNU_CHUNK_T *a, const IppsGFpECState *pEC) +{ + const gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + + const int elemBits = GFP_FEBITLEN(pME); /* size Bits */ + const int elemSize = GFP_FELEN(pME); /* size BNU_CHUNK */ + /* compute w = [log2(n)/2 - 1] */ + const int w = ((elemBits + 1) / 2 - 1); + + /* compute copy BNU_CHUNK */ + const int num_copy_bc = (w + (BNU_CHUNK_BITS - 1)) / BNU_CHUNK_BITS; + const int num_bit_shift = (w - (num_copy_bc - 1) * BNU_CHUNK_BITS); + const BNU_CHUNK_T vadd = (BNU_CHUNK_T)(1ULL << num_bit_shift); + const BNU_CHUNK_T mask = (BNU_CHUNK_T)(vadd - 1); + + ZEXPAND_COPY_BNU(r, elemSize, a, num_copy_bc); + r[num_copy_bc - 1] = (r[num_copy_bc - 1] & mask) + vadd; + return; +} + +/* clang-format off */ +__INLINE void cpSM2KE_get_affine_ext_euclid(BNU_CHUNK_T *x, BNU_CHUNK_T *y, + const IppsGFpECPoint *p, + IppsGFpECState *pEC) +/* clang-format on */ +{ + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + + gfec_GetPoint(x, y, p, pEC); + GFP_METHOD(pME)->decode(x, x, pME); + GFP_METHOD(pME)->decode(y, y, pME); + return; +} + +__INLINE void cpSM2KE_xy_to_BE(BNU_CHUNK_T *x, BNU_CHUNK_T *y, const IppsGFpECState *pEC) +{ + const gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + + const int elemBits = GFP_FEBITLEN(pME); /* size Bits */ + const int elemBytes = (elemBits + 7) / 8; /* size Bytes */ + + cpSM2KE_reverse_inplace((Ipp8u *)x, elemBytes); + cpSM2KE_reverse_inplace((Ipp8u *)y, elemBytes); + return; +} + +/** + * @brief + * comput SM3 hash by message + * @param[out] r hash SM3 compute + * @param[in] a hashing an array data + * @param[in] numBytes numers bytes + */ +__INLINE void cpSM2KE_compute_hash_SM3(Ipp8u *r, const Ipp8u *a, const int numBytes) +{ + static IppsHashState_rmf ctx; + + /* init */ + ippsHashInit_rmf(&ctx, ippsHashMethod_SM3()); + /* update hash */ + ippsHashUpdate_rmf(a, numBytes, &ctx); + /* final */ + ippsHashFinal_rmf(r, &ctx); + return; +} + +/* clang-format off */ +/* compute Za digest = SM3( ENTL || ID || a || b || xG || yG || xA || yA ) */ +#define computeZa_user_id_hash_sm2 OWNAPI(computeZa_user_id_hash_sm2) +IPP_OWN_DECL(IppStatus, computeZa_user_id_hash_sm2, (Ipp8u * pZa_digest, + const Ipp8u* p_user_id, const int user_id_len, + const int elem_len, + const Ipp8u* a, const Ipp8u* b, + const Ipp8u* Gx, const Ipp8u* Gy, + const Ipp8u* pub_key_x, const Ipp8u* pub_key_y)) + +/* KDF sm3 */ +#define KDF_sm3 OWNAPI(KDF_sm3) +IPP_OWN_DECL(IppStatus, KDF_sm3, (Ipp8u * pKDF, int kdf_len, + const Ipp8u* pZ, const int z_len)) + +/* IFMA optimization SM2 Key Exchange Shared Key */ +#define gfec_key_exchange_sm2_shared_key_avx512 OWNAPI(gfec_key_exchange_sm2_shared_key_avx512) + +IPP_OWN_DECL(IppStatus, gfec_key_exchange_sm2_shared_key_avx512, (Ipp8u* pSharedKey, const int sharedKeySize, + Ipp8u* pSSelf, + const IppsBigNumState* pPrvKey, + IppsBigNumState* pEphPrvKey, + IppsGFpECKeyExchangeSM2State *pKE, Ipp8u* pScratchBuffer)) +/* clang-format on */ + +#endif // _SM2_STUFF_H diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_user_id_hash.c b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_user_id_hash.c new file mode 100644 index 0000000..5a79006 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/sources/ippcp/sm2/sm2_user_id_hash.c @@ -0,0 +1,125 @@ +/******************************************************************************* + * Copyright (C) 2022 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * + *******************************************************************************/ + +#include "owndefs.h" +#include "owncp.h" +#include "pcpgfpstuff.h" +#include "pcpgfpecstuff.h" +#include "sm2/sm2_stuff.h" + + +/** + * @brief ippsGFpECUserIDHashSM2 + * compute digest Za = SM3( ENTL||IDA||a||b||Gx||Gy||Px||Py) + * @param [out] pZaDigest digest Za + * @param [in] pUserID user ID data + * @param [in] userIDLen length user ID data + * @param [in] pPublicKey public key + * @param [in] pEC context Elliptic Curve + * @param [in] pScratchBuffer supported buffer + * @return + * ippStsNoErr - successful + * ippStsNullPtrErr - if pEC | pUserID | pPublicKey is NULL + * ippStsContextMatchErr - if pEC no valid ID | pEC | ID pPublic Key no exists SUBGROUP + * ippStsNotSupportedModeErr - if pEC no supported (n|p) modulus arithmetic + * ippStsInvalidPoint - if no in curve Pub Key + * ippStsBadArgErr - if userIDLen <= 0 or the number representing the userIDLen in bits + * exceed two bytes (user_id_len*8 > 0xFFFF) + * ippStsOutOfRangeErr - if userIDLen <= 0 or public key length is out of range + */ +/* clang-format off */ +IPPFUN(IppStatus, ippsGFpECUserIDHashSM2, (Ipp8u * pZaDigest, + const Ipp8u* pUserID, int userIDLen, + const IppsGFpECPoint* pPublicKey, + IppsGFpECState* pEC, Ipp8u* pScratchBuffer)) +/* clang-format on */ +{ + + /* check curve data */ + IPP_BAD_PTR1_RET(pEC) + IPP_BAD_PTR1_RET(pScratchBuffer) + IPP_BADARG_RET(!VALID_ECP_ID(pEC), ippStsContextMatchErr) + IPP_BADARG_RET(!ECP_SUBGROUP(pEC), ippStsContextMatchErr) + + gsModEngine *pME = GFP_PMA(ECP_GFP(pEC)); + IPP_BADARG_RET(1 < GFP_EXTDEGREE(pME), ippStsNotSupportedModeErr) + + /* check Za return */ + IPP_BAD_PTR1_RET(pZaDigest) + + /* check User ID */ + IPP_BAD_PTR1_RET(pUserID) + /* check border (userIDLen > 0) */ + IPP_BADARG_RET(!(userIDLen > 0), ippStsOutOfRangeErr); + + /* check Public Key */ + IPP_BAD_PTR1_RET(pPublicKey) + IPP_BADARG_RET(!ECP_POINT_VALID_ID(pPublicKey), ippStsContextMatchErr) + IPP_BADARG_RET(ECP_POINT_FELEN(pPublicKey) != GFP_FELEN(pME), ippStsOutOfRangeErr) + IPP_BADARG_RET(0 == gfec_IsPointOnCurve(pPublicKey, pEC), ippStsInvalidPoint); + + IppStatus ret = ippStsNoErr; + + const int elemBits = GFP_FEBITLEN(pME); /* size Bits */ + const int elemBytes = (elemBits + 7) / 8; /* size Bytes */ + const int elemLen = GFP_FELEN(pME); /* size BNU_CHUNK */ + + BNU_CHUNK_T *pDataBuff = cpGFpGetPool(6, pME); + /* param curve : a,b,Gx,Gy */ + BNU_CHUNK_T *pA = pDataBuff; + BNU_CHUNK_T *pB = pDataBuff + 1 * elemLen; + BNU_CHUNK_T *pGx = pDataBuff + 2 * elemLen; + BNU_CHUNK_T *pGy = pDataBuff + 3 * elemLen; + /* Public X | Y */ + BNU_CHUNK_T *pX = pDataBuff + 4 * elemLen; + BNU_CHUNK_T *pY = pDataBuff + 5 * elemLen; + + /* get affine coordinate X | Y */ + gfec_GetPoint(/* pX = */ pX, /* pY = */ pY, /* pPoint = */ pPublicKey, /* pEC = */ pEC); + + /* decode */ + GFP_METHOD(pME)->decode(pX, pX, pME); + GFP_METHOD(pME)->decode(pY, pY, pME); + GFP_METHOD(pME)->decode(pA, ECP_A(pEC), pME); + GFP_METHOD(pME)->decode(pB, ECP_B(pEC), pME); + GFP_METHOD(pME)->decode(pGx, ECP_G(pEC), pME); + GFP_METHOD(pME)->decode(pGy, ECP_G(pEC) + elemLen, pME); + + /* convert Little Endian to Big Endian */ + cpSM2KE_reverse_inplace((Ipp8u *)pX, elemBytes); /* X */ + cpSM2KE_reverse_inplace((Ipp8u *)pY, elemBytes); /* Y */ + cpSM2KE_reverse_inplace((Ipp8u *)pA, elemBytes); /* a */ + cpSM2KE_reverse_inplace((Ipp8u *)pB, elemBytes); /* b */ + cpSM2KE_reverse_inplace((Ipp8u *)pGx, elemBytes); /* Gx */ + cpSM2KE_reverse_inplace((Ipp8u *)pGy, elemBytes); /* Gy */ + + /* create hash */ + ret = computeZa_user_id_hash_sm2(pZaDigest, + /* p_user_id = */ pUserID, + /* user_id_len = */ userIDLen, + /* elem_len = */ elemBytes, + /* a = */ (Ipp8u *)pA, + /* b = */ (Ipp8u *)pB, + /* Gx = */ (Ipp8u *)pGx, + /* Gy = */ (Ipp8u *)pGy, + /* px = */ (Ipp8u *)pX, + /* py = */ (Ipp8u *)pY); + + cpGFpReleasePool(6, pME); + + return ret; +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/gui/app.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/gui/app.py new file mode 100644 index 0000000..269a182 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/gui/app.py @@ -0,0 +1,118 @@ +""" +Copyright 2018-2021 Intel Corporation. + +This software and the related documents are Intel copyrighted materials, and +your use of them is governed by the express license under which they were +provided to you (License). Unless the License provides otherwise, you may not +use, modify, copy, publish, distribute, disclose or transmit this software or +the related documents without Intel's prior written permission. + +This software and the related documents are provided as is, with no express +or implied warranties, other than those that are expressly stated in the +License. + +License: +http://software.intel.com/en-us/articles/intel-sample-source-code-license-agr +eement/ +""" +import os +import json + +from PyQt5.QtCore import QEvent, Qt +from PyQt5.QtGui import QIcon +from PyQt5.QtWidgets import QMainWindow, QWidget, QGridLayout, QFileDialog, QMessageBox, QDesktopWidget + +from tool import utils +from gui.selection_panel import SelectionPanel +from gui.custom_functions_panel import CustomFunctionsPanel +from gui.settings_panel import SettingsPanel +from gui.controller import Controller + + +class MainAppWindow(QMainWindow): + def __init__(self): + super(MainAppWindow, self).__init__() + self.setWindowIcon(QIcon('icon.ico')) + self.setWindowTitle('Intel(R) Integrated Performance Primitives Custom Library Tool') + + project_menu = self.menuBar() + + open_action = project_menu.addAction('Open project') + save_action = project_menu.addAction('Save project') + save_as_action = project_menu.addAction('Save project as...') + + self.settings_panel = SettingsPanel() + self.selection_panel = SelectionPanel(self.settings_panel) + self.custom_functions_panel = CustomFunctionsPanel(self.settings_panel) + + self.controller = Controller(self, + self.settings_panel, + self.selection_panel, + self.custom_functions_panel) + + widget = QWidget() + self.setCentralWidget(widget) + + layout = QGridLayout() + layout.addWidget(self.settings_panel, 0, 0, 1, 3) + layout.addWidget(self.selection_panel, 1, 0) + layout.addWidget(self.controller, 1, 1, Qt.AlignCenter) + layout.addWidget(self.custom_functions_panel, 1, 2) + widget.setLayout(layout) + + open_action.triggered.connect(self.on_open) + save_action.triggered.connect(self.on_save) + save_as_action.triggered.connect(self.on_save_as) + + self.project = '' + self.show() + + def on_open(self): + while True: + extension = 'CLT project (*' + utils.PROJECT_EXTENSION + ')' + chosen_path = QFileDialog.getOpenFileName(self, + 'Open project', + '', + extension, + options=QFileDialog.DontResolveSymlinks)[0] + if not chosen_path: + return + elif os.path.islink(chosen_path): + QMessageBox.information(self, 'ERROR!', 'Please, select not a symlink') + continue + else: + self.project = chosen_path + break + + with open(self.project, 'r') as project_file: + configs = json.load(project_file) + self.controller.set_configs(configs) + + def on_save(self): + return self.on_save_as(self.project) + + def on_save_as(self, project): + self.controller.get_selected_configs() + + if not project: + extension = 'CLT project (*' + utils.PROJECT_EXTENSION + ')' + project = QFileDialog.getSaveFileName(self, 'Save project as...', '', extension)[0] + if not project: + return + else: + self.project = project + + if utils.CONFIGS[utils.PACKAGE]: + utils.CONFIGS[utils.PACKAGE] = utils.CONFIGS[utils.PACKAGE].root + + with open(project, 'w') as project_file: + json.dump(utils.CONFIGS, project_file) + + def event(self, e): + if e.type() == QEvent.Show: + qt_rectangle = self.frameGeometry() + center_point = QDesktopWidget().availableGeometry().center() + qt_rectangle.moveCenter(center_point) + self.move(qt_rectangle.topLeft()) + + return super().event(e) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/gui/controller.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/gui/controller.py new file mode 100644 index 0000000..996f656 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/gui/controller.py @@ -0,0 +1,204 @@ +""" +Copyright 2018-2021 Intel Corporation. + +This software and the related documents are Intel copyrighted materials, and +your use of them is governed by the express license under which they were +provided to you (License). Unless the License provides otherwise, you may not +use, modify, copy, publish, distribute, disclose or transmit this software or +the related documents without Intel's prior written permission. + +This software and the related documents are provided as is, with no express +or implied warranties, other than those that are expressly stated in the +License. + +License: +http://software.intel.com/en-us/articles/intel-sample-source-code-license-agr +eement/ +""" +import os + +from PyQt5.QtWidgets import QWidget, QPushButton, QVBoxLayout, QFileDialog, QMessageBox, QLabel, QCheckBox + +import tool +from tool import utils +from tool.core import build, generate_script + + +class Controller(QWidget): + def change_state(function): + def wrapper(self, *args): + function(self, *args) + self.check_current_state() + + return wrapper + + def __init__(self, parent, settings, left_side_menu, right_side_menu): + super().__init__() + self.parent = parent + self.settings = settings + self.left = left_side_menu + self.right = right_side_menu + + self.to_right = QPushButton('>>') + self.to_left = QPushButton('<<') + + layout = QVBoxLayout() + layout.addWidget(self.to_right) + layout.addWidget(self.to_left) + self.setLayout(layout) + + self.status_message = QLabel() + self.parent.statusBar().addWidget(self.status_message) + + self.to_right.pressed.connect(self.on_right_press) + self.to_left.pressed.connect(self.on_left_press) + self.settings.package_changed.connect(self.check_current_state) + self.left.autobuild.connect(self.autobuild) + self.right.save_script.connect(self.save_build_script) + + self.settings.init_settings() + + @change_state + def on_right_press(self): + function = self.left.functions_list.currentItem() + if function: + self.left.remove_function(function.text()) + self.right.add_function(function.text()) + + @change_state + def on_left_press(self): + function = self.right.functions_list.currentItem() + if function: + self.right.remove_function(function.text()) + self.left.add_function(function.text()) + + def check_current_state(self): + state = self.get_current_state() + + if state == utils.ENABLE_GENERATION_RULES: + self.enable_generation(True) + self.status_message.setText("Ready to build custom library") + else: + self.enable_generation(False) + differences = dict(utils.ENABLE_GENERATION_RULES.items() - state.items()) + self.status_message.setText("Select " + sorted(differences, key=len)[0]) + + def enable_generation(self, flag): + self.left.autobuild_button.setEnabled(flag) + self.right.save_script_button.setEnabled(flag) + + def autobuild(self): + self.get_selected_configs() + + extension = 'Dynamic library (*' + utils.DYNAMIC_LIB_EXTENSION[utils.HOST_SYSTEM] + ')' + chosen_path = QFileDialog.getSaveFileName(self, + 'Save custom library as...', + utils.CONFIGS[utils.CUSTOM_LIBRARY_NAME], + extension)[0] + if not chosen_path: + return + else: + utils.CONFIGS[utils.CUSTOM_LIBRARY_NAME] = os.path.basename(os.path.splitext(chosen_path)[0]) + utils.CONFIGS[utils.OUTPUT_PATH] = os.path.dirname(chosen_path) + + self.parent.setDisabled(True) + QMessageBox.information(self, + 'Build', + 'Building will start after this window is closed. ' + 'Please, wait until process is done.') + success = build() + QMessageBox.information(self, + 'Success' if success else 'Failure', + 'Build completed!' if success else 'Build failed!') + self.parent.setDisabled(False) + + def save_build_script(self): + self.get_selected_configs() + + extension = 'Script (*' + utils.BATCH_EXTENSIONS[utils.HOST_SYSTEM] + ')' + script_path = QFileDialog.getSaveFileName(self, + 'Save build script as...', + utils.CONFIGS[utils.BUILD_SCRIPT_NAME], + extension)[0] + if not script_path: + return + else: + utils.CONFIGS[utils.BUILD_SCRIPT_NAME] = os.path.basename(os.path.splitext(script_path)[0] + + utils.BATCH_EXTENSIONS[utils.HOST_SYSTEM]) + utils.CONFIGS[utils.OUTPUT_PATH] = os.path.dirname(script_path) + + success = generate_script() + QMessageBox.information(self, + 'Success' if success else 'Failure', + 'Generation completed!' if success else 'Generation failed!') + + def get_current_state(self): + return {utils.HAVE_PACKAGE : not self.settings.package.broken, + utils.HAVE_FUNCTIONS : bool(self.right.functions_list.count())} + + def get_selected_configs(self): + """ + Collecting all user-specified information about future dynamic library into dictionary + """ + if not self.settings.package.broken: + custom_library_name = (self.right.lib_name.text() if self.right.lib_name.text() else + utils.CONFIGS[utils.CUSTOM_LIBRARY_NAME]) + functions_list = [] + for i in range(self.right.functions_list.count()): + functions_list.append(self.right.functions_list.item(i).text()) + + architecture = (utils.IA32 if self.settings.ia32.isChecked() else utils.INTEL64) + thread_mode = (utils.SINGLE_THREADED if self.settings.single_threaded.isChecked() else utils.MULTI_THREADED) + + if self.settings.tbb.isChecked(): + tl_type = utils.TBB + elif self.settings.omp.isChecked(): + tl_type = utils.OPENMP + else: + tl_type = '' + + custom_cpu_set = [self.settings.get_formatted_button_name(cpu) + for cpu in self.settings.custom_dispatch.findChildren(QCheckBox) if cpu.isChecked()] + + utils.set_configs_dict(package=self.settings.package, + functions_list=functions_list, + architecture=architecture, + thread_mode=thread_mode, + threading_layer_type=tl_type, + custom_library_name=custom_library_name, + custom_cpu_set=custom_cpu_set) + + @change_state + def set_configs(self, configs): + self.settings.package = tool.package.Package(configs[utils.PACKAGE]) + self.settings.init_settings() + + if configs[utils.ARCHITECTURE] == utils.IA32: + self.settings.ia32.setChecked(True) + if configs[utils.ARCHITECTURE] == utils.INTEL64: + self.settings.intel64.setChecked(True) + + if configs[utils.THREAD_MODE] == utils.SINGLE_THREADED: + self.settings.single_threaded.setChecked(True) + if configs[utils.THREAD_MODE] == utils.MULTI_THREADED: + self.settings.multi_threaded.setChecked(True) + + if configs[utils.THREADING_LAYER] == utils.TBB: + self.settings.tbb.setChecked(True) + if configs[utils.THREADING_LAYER] == utils.OPENMP: + self.settings.omp.setChecked(True) + + if configs[utils.CUSTOM_CPU_SET]: + self.settings.custom_dispatch.setChecked(True) + self.settings.on_switch_custom_dispatch() + for cpu in self.settings.custom_dispatch.findChildren(QCheckBox): + cpu_name = self.settings.get_formatted_button_name(cpu) + if cpu_name in configs[utils.CUSTOM_CPU_SET]: + cpu.setChecked(True) + + self.right.lib_name.setText(configs[utils.CUSTOM_LIBRARY_NAME]) + + self.right.reset() + for function in configs[utils.FUNCTIONS_LIST]: + self.left.remove_function(function) + self.right.add_function(function) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/gui/custom_functions_panel.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/gui/custom_functions_panel.py new file mode 100644 index 0000000..bcba751 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/gui/custom_functions_panel.py @@ -0,0 +1,67 @@ +""" +Copyright 2018-2021 Intel Corporation. + +This software and the related documents are Intel copyrighted materials, and +your use of them is governed by the express license under which they were +provided to you (License). Unless the License provides otherwise, you may not +use, modify, copy, publish, distribute, disclose or transmit this software or +the related documents without Intel's prior written permission. + +This software and the related documents are provided as is, with no express +or implied warranties, other than those that are expressly stated in the +License. + +License: +http://software.intel.com/en-us/articles/intel-sample-source-code-license-agr +eement/ +""" +from PyQt5 import QtCore +from PyQt5.QtWidgets import QWidget, QLineEdit, QListWidget, QPushButton, QVBoxLayout, QListWidgetItem + + +class CustomFunctionsPanel(QWidget): + save_script = QtCore.pyqtSignal() + + def __init__(self, settings): + super().__init__() + self.settings = settings + + # Initializing GUI elements + self.lib_name = QLineEdit() + self.functions_list = QListWidget() + self.save_script_button = QPushButton('Save build script') + + # Preparing elements by giving initial values and etc + self.lib_name.setPlaceholderText('Custom library name...') + + # Setting all widgets in their places + layout = QVBoxLayout() + layout.addWidget(self.lib_name) + layout.addWidget(self.functions_list) + layout.addWidget(self.save_script_button) + self.setLayout(layout) + + self.save_script_button.clicked.connect(self.on_save_script) + self.settings.package_changed.connect(self.reset) + + def on_save_script(self): + self.save_script.emit() + + def reset(self): + self.functions_list.clear() + + def add_function(self, function): + """ + Adds new function to required list + + :param function: name if function + """ + self.functions_list.addItem(QListWidgetItem(function)) + + def remove_function(self, function): + """ + Removes function from left list + """ + item = self.functions_list.findItems(function, QtCore.Qt.MatchExactly) + if item: + self.functions_list.takeItem(self.functions_list.row(item[0])) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/gui/selection_panel.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/gui/selection_panel.py new file mode 100644 index 0000000..f2b214b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/gui/selection_panel.py @@ -0,0 +1,133 @@ +""" +Copyright 2018-2021 Intel Corporation. + +This software and the related documents are Intel copyrighted materials, and +your use of them is governed by the express license under which they were +provided to you (License). Unless the License provides otherwise, you may not +use, modify, copy, publish, distribute, disclose or transmit this software or +the related documents without Intel's prior written permission. + +This software and the related documents are provided as is, with no express +or implied warranties, other than those that are expressly stated in the +License. + +License: +http://software.intel.com/en-us/articles/intel-sample-source-code-license-agr +eement/ +""" +import copy + +from PyQt5 import QtCore +from PyQt5.QtWidgets import QWidget, QComboBox, QLineEdit, QListWidget, QPushButton, QVBoxLayout, QListWidgetItem + +from tool import utils + + +class SelectionPanel(QWidget): + autobuild = QtCore.pyqtSignal() + + def __init__(self, settings): + super().__init__() + self.settings = settings + + # Initializing GUI elements + self.domains_list = QComboBox(self) + self.search = QLineEdit(self) + self.functions_list = QListWidget(self) + self.autobuild_button = QPushButton('Autobuild') + + # Preparing elements by giving initial values, etc + self.setMinimumHeight(500) + self.search.setPlaceholderText('Search...') + + # Setting all widgets in their places + layout = QVBoxLayout() + layout.addWidget(self.domains_list) + layout.addWidget(self.search) + layout.addWidget(self.functions_list) + layout.addWidget(self.autobuild_button) + self.setLayout(layout) + + self.domains_list.activated[str].connect(self.on_select_domain) + self.search.textEdited.connect(self.on_search) + self.autobuild_button.clicked.connect(self.on_autobuild) + self.settings.package_changed.connect(self.init_selection_panel) + self.settings.tl_changed.connect(self.refresh) + + def init_selection_panel(self): + if not self.settings.package.broken: + self.functions_dict = copy.deepcopy(self.settings.package.functions) + self.search.setEnabled(True) + self.refresh() + else: + self.search.setEnabled(False) + self.reset() + + def refresh(self): + self.domains_type = (self.settings.package.type if not self.settings.tl_group.isChecked() + else utils.THREADING_LAYER) + domains_list = self.functions_dict[self.domains_type].keys() + self.set_widget_items(self.domains_list, domains_list) + self.on_select_domain() + + def on_select_domain(self): + self.current_domain = self.domains_list.currentText() + self.on_search(self.search.text()) + + def on_search(self, search_request): + self.set_widget_items(self.functions_list, + [entry for entry in self.functions_dict[self.domains_type][self.current_domain] + if search_request.upper() in entry.upper()]) + + def on_autobuild(self): + self.autobuild.emit() + + def set_widget_items(self, widget, items): + """ + Adds items to widget + :param widget: widget + :param items: list of strings + """ + widget.clear() + widget.addItems(items) + + def reset(self): + self.domains_list.clear() + self.functions_list.clear() + + def add_function(self, function): + """ + Adds new function to required list + + :param function: name if function + """ + domain_type, domain, index = self.find_function(function) + self.functions_dict[domain_type][domain].insert(index, function) + if domain == self.current_domain: + self.functions_list.insertItem(index, QListWidgetItem(function)) + self.on_search(self.search.text()) + + def remove_function(self, function): + """ + Removes function from left list + """ + domain_type, domain, index = self.find_function(function) + self.functions_dict[domain_type][domain].remove(function) + if self.current_domain == domain: + item = self.functions_list.findItems(function, QtCore.Qt.MatchExactly) + if item: + self.functions_list.takeItem(self.functions_list.row(item[0])) + + def find_function(self, function_name): + previous_domain = '' + initial_functions_dict = self.settings.package.functions + + for domain_type, domain, function in utils.walk_dict(initial_functions_dict): + if domain != previous_domain: + index = 0 + + if function_name == function: + return domain_type, domain, index + elif function in self.functions_dict[domain_type][domain]: + previous_domain = domain + index = self.functions_dict[domain_type][domain].index(function) + 1 diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/gui/settings_panel.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/gui/settings_panel.py new file mode 100644 index 0000000..86f1274 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/gui/settings_panel.py @@ -0,0 +1,204 @@ +""" +Copyright 2018-2021 Intel Corporation. + +This software and the related documents are Intel copyrighted materials, and +your use of them is governed by the express license under which they were +provided to you (License). Unless the License provides otherwise, you may not +use, modify, copy, publish, distribute, disclose or transmit this software or +the related documents without Intel's prior written permission. + +This software and the related documents are provided as is, with no express +or implied warranties, other than those that are expressly stated in the +License. + +License: +http://software.intel.com/en-us/articles/intel-sample-source-code-license-agr +eement/ +""" +import re + +from PyQt5 import QtCore +from PyQt5.QtWidgets import QWidget, QLabel, QPushButton, QGroupBox, QRadioButton, QCheckBox, QGridLayout, \ + QVBoxLayout, QFileDialog, QMessageBox + +import tool +from tool import utils + + +class SettingsPanel(QWidget): + package_changed = QtCore.pyqtSignal() + tl_changed = QtCore.pyqtSignal() + + def __init__(self): + super().__init__() + + # Initializing GUI elements + self.current_package = QLabel() + self.select_package = QPushButton('Select package') + + self.settings = QGroupBox('Settings') + + self.arch_group = QGroupBox('Architecture') + self.ia32 = QRadioButton('IA32') + self.intel64 = QRadioButton('Intel(R) 64') + + self.thread_group = QGroupBox('Thread mode') + self.single_threaded = QRadioButton('Single-threaded') + self.multi_threaded = QRadioButton('Multi-threaded') + + self.tl_group = QGroupBox('Threading layer') + self.tbb = QRadioButton('TBB') + self.omp = QRadioButton('OpenMP') + + self.custom_dispatch = QGroupBox('Custom dispatcher') + self.sse2 = QCheckBox('SSE2') + self.sse3 = QCheckBox('SSE3') + self.ssse3 = QCheckBox('SSSE3') + self.sse42 = QCheckBox('SSE4.2') + self.avx = QCheckBox('AVX') + self.avx2 = QCheckBox('AVX2') + self.avx512f = QCheckBox('AVX512F') + self.avx512bw = QCheckBox('AVX512BW') + self.avx512ifma = QCheckBox('AVX512IFMA') + + self.tl_group.setCheckable(True) + self.custom_dispatch.setCheckable(True) + + self.select_package.setFixedWidth(130) + + # Setting all widgets in their places + layout = QGridLayout() + layout.addWidget(self.select_package, 0, 0) + layout.addWidget(self.current_package, 0, 1) + layout.addWidget(self.settings, 1, 0, 1, 3) + self.setLayout(layout) + + settings_layout = QGridLayout() + settings_layout.addWidget(self.arch_group, 0, 0) + settings_layout.addWidget(self.thread_group, 0, 1) + settings_layout.addWidget(self.tl_group, 0, 2) + settings_layout.addWidget(self.custom_dispatch, 0, 3) + self.settings.setLayout(settings_layout) + + arch_layout = QVBoxLayout() + arch_layout.addWidget(self.intel64) + arch_layout.addWidget(self.ia32) + self.arch_group.setLayout(arch_layout) + + thread_layout = QVBoxLayout() + thread_layout.addWidget(self.single_threaded) + thread_layout.addWidget(self.multi_threaded) + self.thread_group.setLayout(thread_layout) + + tl_layout = QVBoxLayout() + tl_layout.addWidget(self.tbb) + tl_layout.addWidget(self.omp) + self.tl_group.setLayout(tl_layout) + + custom_dispatch_layout = QGridLayout() + + custom_dispatch_layout.addWidget(self.sse2, 1, 0) + custom_dispatch_layout.addWidget(self.sse3, 1, 1) + custom_dispatch_layout.addWidget(self.ssse3, 1, 2) + custom_dispatch_layout.addWidget(self.sse42, 1, 3) + + custom_dispatch_layout.addWidget(self.avx, 2, 0) + custom_dispatch_layout.addWidget(self.avx2, 2, 1) + custom_dispatch_layout.addWidget(self.avx512f, 2, 2) + custom_dispatch_layout.addWidget(self.avx512bw, 2, 3) + custom_dispatch_layout.addWidget(self.avx512ifma, 2, 4) + + self.custom_dispatch.setLayout(custom_dispatch_layout) + + self.select_package.clicked.connect(self.on_select_package) + self.ia32.toggled.connect(lambda checked: checked and self.on_switch_arch()) + self.intel64.toggled.connect(lambda checked: checked and self.on_switch_arch()) + self.tl_group.clicked.connect(self.on_switch_tl) + self.custom_dispatch.clicked.connect(self.on_switch_custom_dispatch) + + root = tool.package.get_package_path() + self.package = tool.package.Package(root) + + def init_settings(self): + self.current_package.setText('Current package: ' + self.package.name) + self.disable_widgets() + + if not self.package.broken: + arch_flags = {arch: any([self.package.features[arch][thread] for thread in utils.THREAD_MODES]) + for arch in utils.ARCHITECTURES} + self.refresh_group(self.arch_group, flags_dict=arch_flags) + + self.package_changed.emit() + + def on_switch_arch(self): + self.current_arch = (utils.IA32 if self.ia32.isChecked() else utils.INTEL64) + + self.refresh_group(self.thread_group, flags_dict=self.package.features[self.current_arch]) + self.refresh_group(self.tl_group, flags_dict=self.package.features[self.current_arch]) + self.on_switch_custom_dispatch() + + def on_switch_tl(self): + self.refresh_group(self.tl_group, flags_dict=self.package.features[self.current_arch]) + self.tl_changed.emit() + + def on_switch_custom_dispatch(self): + self.refresh_group(self.custom_dispatch, + search_list=utils.SUPPORTED_CPUS[self.package.type][self.current_arch][utils.HOST_SYSTEM]) + + def on_select_package(self): + while True: + chosen_path = QFileDialog.getExistingDirectory(self, 'Select package') + if not chosen_path: + break + + package = tool.package.Package(chosen_path) + if package.broken: + QMessageBox.information(self, 'ERROR!', package.error_message) + else: + self.package = package + self.init_settings() + break + + def refresh_group(self, group, flags_dict={}, search_list=[]): + group.setDisabled(True) + + for button in group.findChildren(QRadioButton) + group.findChildren(QCheckBox): + button_name = self.get_formatted_button_name(button) + + if flags_dict and flags_dict[button_name] or button_name in search_list: + group.setEnabled(True) + if not group.isCheckable() or group.isCheckable() and group.isChecked(): + button.setEnabled(True) + continue + + button.setChecked(False) + button.setEnabled(False) + + self.check_group(group) + + def check_group(self, group): + if not any([button.isChecked() for button in group.findChildren(QRadioButton)]): + for radiobutton in group.findChildren(QRadioButton): + if radiobutton.isEnabled(): + radiobutton.setChecked(True) + break + + for checkbox in group.findChildren(QCheckBox): + checkbox.setChecked(checkbox.isEnabled()) + + def disable_widgets(self): + for group in self.settings.findChildren(QGroupBox): + for button in group.findChildren(QRadioButton): + button.setAutoExclusive(False) + button.setChecked(False) + button.setAutoExclusive(True) + + for button in group.findChildren(QCheckBox): + button.setChecked(False) + + group.setChecked(False) + group.setEnabled(False) + + def get_formatted_button_name(self, button): + button_name = button.text().replace('(R)', '') + return re.sub('[^\w-]', '', button_name.lower()) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/icon.ico b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/icon.ico new file mode 100644 index 0000000..53c57c4 Binary files /dev/null and b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/icon.ico differ diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/main.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/main.py new file mode 100644 index 0000000..5f67165 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/main.py @@ -0,0 +1,166 @@ +""" +Copyright 2018-2021 Intel Corporation. + +This software and the related documents are Intel copyrighted materials, and +your use of them is governed by the express license under which they were +provided to you (License). Unless the License provides otherwise, you may not +use, modify, copy, publish, distribute, disclose or transmit this software or +the related documents without Intel's prior written permission. + +This software and the related documents are provided as is, with no express +or implied warranties, other than those that are expressly stated in the +License. + +License: +http://software.intel.com/en-us/articles/intel-sample-source-code-license-agr +eement/ +""" +import sys +import os +from argparse import ArgumentParser, RawTextHelpFormatter + +import tool.package +from tool import utils +from tool.core import build, generate_script + + +if __name__ == '__main__': + utils.set_host_system() + + args_parser = ArgumentParser(formatter_class=RawTextHelpFormatter) + args_parser.add_argument('-c', '--console', + help='Turns on console version.\n' + 'Custom Library Tool is running in GUI mode by default', + action='store_true') + + console_args = args_parser.add_argument_group('Console mode options') + console_args.add_argument('-g', '--generate', + help='Generate build script without building custom dynamic library', + action='store_true') + console_args.add_argument('-n', '--name', + help='Name of custom dynamic library', + default=utils.CONFIGS[utils.CUSTOM_LIBRARY_NAME]) + console_args.add_argument('-p', '--output_path', + help='Path to output directory', + default=utils.CONFIGS[utils.OUTPUT_PATH]) + console_args.add_argument('-root', + help='Path to specified ' + utils.PACKAGE_NAME[utils.IPP] + + '\nor ' + utils.PACKAGE_NAME[utils.IPPCP] + ' package') + console_args.add_argument('-f', '--functions', + help='Functions that has to be in dynamic library (appendable)', + nargs='+', + metavar='FUNCTION') + console_args.add_argument('-ff', '--functions_file', + help='Load custom functions list from text file') + console_args.add_argument('-arch', '--architecture', + help='Target architecture', + choices=utils.ARCHITECTURES, + default='intel64') + console_args.add_argument('-mt', '--multi-threaded', + help='Build multi-threaded dynamic library', + action='store_true') + console_args.add_argument('-tl', '--threading_layer_type', + help='Build dynamic library with selected threading layer type', + choices=utils.TL_TYPES) + console_args.add_argument('-d', '--custom_dispatcher', + help='Build dynamic library with custom dispatcher.\n' + 'Set of CPUs can be any combination of the following:\n' + + utils.PACKAGE_NAME[utils.IPP] + ':\n' + '\tIA32 architecture - ' + ' '.join(utils.SUPPORTED_CPUS[utils.IPP] + [utils.IA32] + [utils.HOST_SYSTEM]) + '\n' + + '\tIntel 64 architecture - ' + ' '.join(utils.SUPPORTED_CPUS[utils.IPP] + [utils.INTEL64] + [utils.HOST_SYSTEM]) + '\n' + + utils.PACKAGE_NAME[utils.IPPCP] + ':\n' + '\tIA32 architecture - ' + ' '.join(utils.SUPPORTED_CPUS[utils.IPPCP] + [utils.IA32] + [utils.HOST_SYSTEM]) + '\n' + + '\tIntel 64 architecture - ' + ' '.join(utils.SUPPORTED_CPUS[utils.IPPCP] + [utils.INTEL64] + [utils.HOST_SYSTEM]), + nargs='+', + metavar='CPU') + console_args.add_argument('--prefix', + help='Rename functions in custom dispatcher with specified prefix', + default='') + + args = args_parser.parse_args() + + if args.console: + print('Intel(R) Integrated Performance Primitives Custom Library Tool console version is on...') + + functions_list = [] + if args.functions_file: + with open(args.functions_file, 'r') as functions_file: + functions_list += map(lambda x: x.replace('\n', ''), functions_file.readlines()) + if args.functions: + functions_list += args.functions + if not functions_list: + sys.exit("Please, specify functions that has to be in dynamic library by using -f or -ff options") + + if args.root: + root = args.root + if not os.path.exists(root): + sys.exit("Error: specified package path " + args.root + " doesn't exist") + else: + root = tool.package.get_package_path() + if not os.path.exists(root): + sys.exit("Error: cannot find " + utils.PACKAGE_NAME[utils.IPP] + ' or ' + + utils.PACKAGE_NAME[utils.IPPCP] + " package. " + "Please, specify IPPROOT or IPPCRYPTOROOT by using -root option") + + package = tool.package.Package(root) + print('Current package: ' + package.name) + + architecture = args.architecture + thread_mode = (utils.MULTI_THREADED if args.multi_threaded else utils.SINGLE_THREADED) + threading_layer_type = args.threading_layer_type + + error = package.errors[architecture][thread_mode] + if error: + sys.exit('Error: ' + error) + if threading_layer_type: + error = package.errors[architecture][threading_layer_type] + if error: + sys.exit('Error: ' + error) + + custom_cpu_set = [] + if args.custom_dispatcher: + for cpu in args.custom_dispatcher: + if cpu not in utils.SUPPORTED_CPUS[package.type][architecture][utils.HOST_SYSTEM]: + sys.exit("Error: " + cpu + " isn't supported for " + utils.PACKAGE_NAME[package.type] + ' ' + + utils.HOST_SYSTEM + ' ' + architecture) + custom_cpu_set = args.custom_dispatcher + prefix = args.prefix + + custom_library_name = args.name + output_path = os.path.abspath(args.output_path) + if not os.path.exists(output_path): + os.makedirs(output_path) + + utils.set_configs_dict(package=package, + functions_list=functions_list, + architecture=architecture, + thread_mode=thread_mode, + threading_layer_type=threading_layer_type, + custom_library_name=custom_library_name, + output_path=output_path, + custom_cpu_set=custom_cpu_set, + prefix=prefix) + + if args.generate: + success = generate_script() + print('Generation', 'completed!' if success else 'failed!') + else: + success = build() + + if not success: + exit(1) + else: + from PyQt5.QtWidgets import QApplication + from gui.app import MainAppWindow + + app = QApplication(sys.argv) + ex = MainAppWindow() + sys.exit(app.exec_()) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/requirements.txt b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/requirements.txt new file mode 100644 index 0000000..be36e37 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/requirements.txt @@ -0,0 +1,2 @@ +PyQt5==5.15.9 + diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/tests/functions_tests.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/tests/functions_tests.py new file mode 100644 index 0000000..86f4a60 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/tests/functions_tests.py @@ -0,0 +1,86 @@ +""" +Copyright 2019-2021 Intel Corporation. + +This software and the related documents are Intel copyrighted materials, and +your use of them is governed by the express license under which they were +provided to you (License). Unless the License provides otherwise, you may not +use, modify, copy, publish, distribute, disclose or transmit this software or +the related documents without Intel's prior written permission. + +This software and the related documents are provided as is, with no express +or implied warranties, other than those that are expressly stated in the +License. + +License: +http://software.intel.com/en-us/articles/intel-sample-source-code-license-agr +eement/ +""" +import os +import shutil +import unittest +from sys import platform + +import tests.utils +from tool.core import generate_script, build + + +class FunctionalityTests(unittest.TestCase): + functions = ['ippccGetLibVersion', + 'ippdcGetLibVersion', + 'ippchGetLibVersion', + 'ippsGetLibVersion'] + domains = ['ippcc', + 'ippdc', + 'ippch', + 'ipps', + 'ippcore', + 'ippvm'] + + def generator_assertion(self): + self.assertTrue(os.path.exists('./tmp'), 'Temporary folder does not exist') + self.assertTrue(os.path.exists('./tmp/main.c'), 'main.c file was not generated') + self.assertTrue(os.path.exists('./tmp/' + tests.utils.EXPORT_FILES[tests.utils.HOST_SYSTEM]), + 'Export file was not generated') + self.assertTrue(os.path.exists('./tmp/' + + tests.utils.BUILD_SCRIPT[tests.utils.INTEL64][tests.utils.HOST_SYSTEM]), + 'Build script was not generated') + + @classmethod + def setUpClass(cls): + if platform == "linux" or platform == "linux2": + tests.utils.HOST_SYSTEM = tests.utils.LINUX + elif platform == "darwin": + tests.utils.HOST_SYSTEM = tests.utils.MACOSX + elif platform == "win32": + tests.utils.HOST_SYSTEM = tests.utils.WINDOWS + + @classmethod + def tearDown(self): + if os.path.exists(tests.utils.TEMPORARY_FOLDER): + shutil.rmtree(tests.utils.TEMPORARY_FOLDER, ignore_errors=True) + + def test_generation(self): + generate_script(tests.utils.HOST_SYSTEM, + tests.utils.HOST_SYSTEM, + self.functions, + tests.utils.TEMPORARY_FOLDER, + 'tmp_dll', + self.domains, + tests.utils.INTEL64, + False) + self.generator_assertion() + + def test_build(self): + self.assertTrue(build(tests.utils.HOST_SYSTEM, + tests.utils.HOST_SYSTEM, + self.functions, + os.path.abspath(tests.utils.TEMPORARY_FOLDER), + 'tmp_dll', + self.domains, + tests.utils.INTEL64, + False, + os.environ['COMPILERS_AND_LIBRARIES'])) + + +if __name__ == '__main__': + unittest.main() diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/tests/utils.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/tests/utils.py new file mode 100644 index 0000000..ea3289b --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/tests/utils.py @@ -0,0 +1,58 @@ +""" +Copyright 2019-2021 Intel Corporation. + +This software and the related documents are Intel copyrighted materials, and +your use of them is governed by the express license under which they were +provided to you (License). Unless the License provides otherwise, you may not +use, modify, copy, publish, distribute, disclose or transmit this software or +the related documents without Intel's prior written permission. + +This software and the related documents are provided as is, with no express +or implied warranties, other than those that are expressly stated in the +License. + +License: +http://software.intel.com/en-us/articles/intel-sample-source-code-license-agr +eement/ +""" +HOST_SYSTEM = '' + +WINDOWS = 'Windows' +LINUX = 'Linux' +MACOSX = 'MacOSX' + +TEMPORARY_FOLDER = './tmp' + +INTEL64 = 'intel64' +IA32 = 'ia32' + +BUILD_SCRIPT = { + INTEL64: { + WINDOWS: 'intel64.bat', + LINUX: 'intel64.sh', + MACOSX: 'intel64.sh' + }, + IA32: { + WINDOWS: 'ia32.bat', + LINUX: 'ia32.sh', + MACOSX: 'ia32.sh' + } +} + +EXPORT_FILES = { + WINDOWS: 'export.def', + LINUX: 'export.def', + MACOSX: 'export.lib-export' +} + +LIBRARIES_EXTENSIONS = { + WINDOWS: '.dll', + LINUX: '.so', + MACOSX: '.dy' +} + +LIBRARIES_PREFIX = { + WINDOWS: '', + LINUX: 'lib', + MACOSX: 'lib' +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/tool/core.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/tool/core.py new file mode 100644 index 0000000..f6a323d --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/tool/core.py @@ -0,0 +1,89 @@ +""" +Copyright 2018-2021 Intel Corporation. + +This software and the related documents are Intel copyrighted materials, and +your use of them is governed by the express license under which they were +provided to you (License). Unless the License provides otherwise, you may not +use, modify, copy, publish, distribute, disclose or transmit this software or +the related documents without Intel's prior written permission. + +This software and the related documents are provided as is, with no express +or implied warranties, other than those that are expressly stated in the +License. + +License: +http://software.intel.com/en-us/articles/intel-sample-source-code-license-agr +eement/ +""" + +import os +from subprocess import call # nosec + +from tool import utils +from tool.generators import main_file_generator, EXPORT_GENERATORS, build_script_generator, custom_dispatcher_generator, \ + rename_header_generator + + +def generate_script(): + """ + Generates build script + """ + host = utils.HOST_SYSTEM + configs = utils.CONFIGS + + package = configs[utils.PACKAGE] + functions_list = configs[utils.FUNCTIONS_LIST] + output_path = configs[utils.OUTPUT_PATH] + + if not os.path.exists(output_path): + os.makedirs(output_path) + + with open(os.path.join(output_path, utils.MAIN_FILE_NAME), 'w') as main_file: + main_file.write(main_file_generator()) + + if configs[utils.CUSTOM_CPU_SET]: + disp_folder = os.path.join(output_path, 'custom_dispatcher', configs[utils.ARCHITECTURE]) + if not os.path.exists(disp_folder): + os.makedirs(disp_folder) + + functions_with_custom_disp = [] + for function in functions_list: + if function not in package.functions_without_dispatcher: + disp_file_path = os.path.join(disp_folder, + utils.CUSTOM_DISPATCHER_FILE_NAME.format(function=function)) + functions_with_custom_disp.append(function) + + with open(disp_file_path, 'w') as disp_file: + disp_file.write(custom_dispatcher_generator(function)) + + if configs[utils.PREFIX]: + with open(os.path.join(disp_folder, utils.RENAME_HEADER_NAME), 'w') as rename_header: + rename_header.write(rename_header_generator(functions_with_custom_disp)) + + functions_list = [configs[utils.PREFIX] + func if func in functions_with_custom_disp else func + for func in functions_list] + + with open(os.path.join(output_path, utils.EXPORT_FILE[host]), 'w') as export_file: + EXPORT_GENERATORS[host](export_file, functions_list) + + script_path = os.path.join(output_path, configs[utils.BUILD_SCRIPT_NAME]) + with open(script_path, 'w') as build_script: + build_script.write(build_script_generator()) + os.chmod(script_path, 0o744) + + return os.path.exists(script_path) + + +def build(): + """ + Builds dynamic library + + :return: True if build was successful and False in the opposite case + """ + success = generate_script() + if not success: + return False + + error = call([os.path.join(utils.CONFIGS[utils.OUTPUT_PATH], + utils.CONFIGS[utils.BUILD_SCRIPT_NAME])]) + return False if error else True diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/tool/generators.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/tool/generators.py new file mode 100644 index 0000000..078c733 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/tool/generators.py @@ -0,0 +1,224 @@ +""" +Copyright 2018-2021 Intel Corporation. + +This software and the related documents are Intel copyrighted materials, and +your use of them is governed by the express license under which they were +provided to you (License). Unless the License provides otherwise, you may not +use, modify, copy, publish, distribute, disclose or transmit this software or +the related documents without Intel's prior written permission. + +This software and the related documents are provided as is, with no express +or implied warranties, other than those that are expressly stated in the +License. + +License: +http://software.intel.com/en-us/articles/intel-sample-source-code-license-agr +eement/ +""" +from tool import utils +from tool.generators_utils import * + + +def main_file_generator(): + return MAIN_FILE[utils.HOST_SYSTEM].format(package_type=utils.CONFIGS[PACKAGE].type.lower()) + + +def create_windows_export_file(export_file, functions_list): + """ + Creates export file for Windows + + :param export_file: object that is returned by open function + :param functions_list: list of functions for dynamic library + """ + export_file.write('EXPORTS\n\n') + export_file.writelines(map(lambda x: x + '\n', functions_list)) + + +def create_linux_export_file(export_file, functions_list): + """ + Creates Linux export file + + :param export_file: + :param functions_list: + :return: + """ + export_file.writelines(map(lambda x: 'EXTERN(' + x + ')\n', functions_list)) + export_file.write('\nVERSION\n' + '{\n' + ' {\n' + ' global:\n' + + ''.join( + map(lambda x: ' ' + x + ';\n', functions_list)) + + ' local:* ;\n' + ' };\n' + '}\n') + + +def create_macosx_export_file(export_file, functions_list): + """ + Creates MacOSX export file + + :param export_file: object that is returned by open function + :param functions_list: list of functions for dynamic library + """ + export_file.writelines(map(lambda x: '_' + x + '\n', functions_list)) + + +EXPORT_GENERATORS = { + WINDOWS: create_windows_export_file, + LINUX: create_linux_export_file, + MACOSX: create_macosx_export_file +} + + +def custom_dispatcher_generator(function): + package = utils.CONFIGS[PACKAGE] + arch = utils.CONFIGS[ARCHITECTURE] + include_lines = INCLUDE_STR.format(header_name=package.type.lower() + '.h') + + dispatcher = '' + dispatcher += func_dispatcher_generator(arch, function) + + ippe = utils.DOMAINS[IPP]['ippe'] + if ippe in package.functions[IPP].keys() and \ + function in package.functions[IPP][ippe]: + include_lines += INCLUDE_STR.format(header_name='ippe.h') + + return CUSTOM_DISPATCHER_FILE.format(include_lines=include_lines, + architecture=ARCHITECTURE_DEFINE[arch], + features=FEATURES[arch], + dispatcher=dispatcher) + + +def rename_header_generator(functions_list): + package = utils.CONFIGS[PACKAGE] + prefix = utils.CONFIGS[PREFIX] + + defs_header = ('defs.h' if package.type == IPP or package.type == IPPCP else 'base.h') + content = INCLUDE_STR.format(header_name=package.type.lower() + defs_header) + for function in functions_list: + declaration = package.declarations[function].replace(function, prefix + function) + content += RENAME_FORMAT.format(declaration=declaration, + function=function, + prefix=prefix) + + return content + + +def func_dispatcher_generator(arch, function): + package_type = utils.CONFIGS[PACKAGE].type + declarations = utils.CONFIGS[PACKAGE].declarations[function] + ippfun = declarations.replace('IPPAPI', 'IPPFUN').replace(function, utils.CONFIGS[PREFIX] + function) + + args = utils.get_match(utils.FUNCTION_NAME_REGEX, declarations, 'args').split(',') + args = [utils.get_match(utils.ARGUMENT_REGEX, arg, 'arg') for arg in args] + args = ', '.join(args) + + ippapi = '' + dispatching_scheme = '' + for cpu in CPUID.keys(): + if cpu not in utils.CONFIGS[CUSTOM_CPU_SET]: + continue + + cpuid = CPUID[cpu] + cpu_code = CPU[cpu][arch] + + function_with_cpu_code = cpu_code + '_' + function + ippapi += declarations.replace(function, function_with_cpu_code) + '\n' + + dispatching_scheme += DISPATCHING_SCHEME_FORMAT.format(cpuid=cpuid, + function=function_with_cpu_code, + args=args) + + ret_type = utils.get_match(utils.FUNCTION_NAME_REGEX, declarations, 'ret_type') + ret_value = get_dict_value(RETURN_VALUES, ret_type) + + dispatching_scheme += ' return ' + ret_value + ';\n' + if not ret_value: + dispatching_scheme = dispatching_scheme.replace('return', '') + + return FUNCTION_DISPATCHER.format(ippapi=ippapi, + ippfun=ippfun, + package_type=package_type.lower(), + dispatching_scheme=dispatching_scheme) + + +def build_script_generator(): + """ + Generates script for building custom dynamic library + :return: String that represents script + """ + host = utils.HOST_SYSTEM + configs = utils.CONFIGS + + package = configs[PACKAGE] + output_path = configs[OUTPUT_PATH] + + arch = configs[ARCHITECTURE] + thread = configs[THREAD_MODE] + tl = configs[THREADING_LAYER] + + root_type = (IPPROOT if package.type == IPP else IPPCRYPTOROOT) + + if package.env_script: + force_flag = '' + if 'setvars' in package.env_script: + force_flag = '--force' + + env_commands = CALL_ENV_SCRIPT_COMMAND[host].format(env_script=package.env_script, + arch=arch, + force_flag=force_flag) + else: + env_commands = SET_ENV_COMMAND[host].format(env_var=root_type, + path=package.root) + if ADDITIONAL_ENV[host]: + env_commands += '\n' + ADDITIONAL_ENV[host] + + compiler = COMPILERS[host] + + cmp_flags = COMPILERS_FLAGS[host][arch] + if tl == OPENMP and host == WINDOWS: + cmp_flags += ' /openmp' + + c_files = MAIN_FILE_NAME + if configs[utils.CUSTOM_CPU_SET]: + c_files += ' ' + os.path.join('custom_dispatcher', arch, '*.c') + + compile_command = COMPILE_COMMAND_FORMAT[host].format(compiler=compiler, + cmp_flags=cmp_flags, + root_type=root_type, + c_files=c_files) + + linker = LINKERS[host] + link_flags = LINKER_FLAGS[host][arch] + custom_library = LIB_PREFIX[host] + configs[CUSTOM_LIBRARY_NAME] + export_file = EXPORT_FILE[host] + + ipp_libraries = package.libraries[arch][thread] + if tl: + ipp_libraries = package.libraries[arch][tl] + ipp_libraries + ipp_libraries = [lib.replace(package.root, ENV_VAR[host].format(env_var=root_type)) + for lib in ipp_libraries] + ipp_libraries = ' '.join('"{0}"'.format(lib) for lib in ipp_libraries) + + exp_libs = EXP_LIBS[host][thread] + if tl and EXP_LIBS[host][tl] not in exp_libs: + exp_libs += ' ' + EXP_LIBS[host][tl] + + sys_libs_path = SYS_LIBS_PATH[host][arch] + + link_command = LINK_COMMAND_FORMAT[host].format(linker=linker, + link_flags=link_flags, + custom_library=custom_library, + export_file=export_file, + ipp_libraries=ipp_libraries, + exp_libs=exp_libs, + sys_libs_path=sys_libs_path,) + + return BUILD_SCRIPT[host].format(architecture=arch, + threading=thread.lower(), + output_path=output_path, + custom_library=custom_library, + env_commands=env_commands, + compile_command=compile_command, + link_command=link_command) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/tool/generators_utils.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/tool/generators_utils.py new file mode 100644 index 0000000..3efda35 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/tool/generators_utils.py @@ -0,0 +1,326 @@ +""" +Copyright 2018-2021 Intel Corporation. + +This software and the related documents are Intel copyrighted materials, and +your use of them is governed by the express license under which they were +provided to you (License). Unless the License provides otherwise, you may not +use, modify, copy, publish, distribute, disclose or transmit this software or +the related documents without Intel's prior written permission. + +This software and the related documents are provided as is, with no express +or implied warranties, other than those that are expressly stated in the +License. + +License: +http://software.intel.com/en-us/articles/intel-sample-source-code-license-agr +eement/ +""" + +from tool.utils import * + +ENV_VAR = { + WINDOWS : '%{env_var}%', + LINUX : '${{{env_var}}}', + MACOSX : '${{{env_var}}}' +} + +CALL_ENV_SCRIPT_COMMAND = { + WINDOWS : 'call "{env_script}" {arch} {force_flag}', + LINUX : 'source "{env_script}" -arch {arch} {force_flag}', + MACOSX : 'source "{env_script}" -arch {arch} {force_flag}' +} + +SET_ENV_COMMAND = { + WINDOWS : 'set "{env_var}={path}"', + LINUX : 'export "{env_var}={path}"', + MACOSX : 'export "{env_var}={path}"' +} + +ADDITIONAL_ENV = { + WINDOWS : '', + LINUX : 'export LIBRARY_PATH=$LD_LIBRARY_PATH:$LIBRARY_PATH', + MACOSX : 'export LIBRARY_PATH=$DYLD_LIBRARY_PATH:$LIBRARY_PATH' +} + +COMPILERS = { + WINDOWS : 'cl.exe', + LINUX : 'g++', + MACOSX : 'clang' +} + +LINKERS = { + WINDOWS : 'link.exe', + LINUX : 'g++', + MACOSX : 'clang' +} + +COMPILERS_FLAGS = { + WINDOWS: { + INTEL64 : '/c /MP /MT /GS /sdl /O2', + IA32 : '/c /MP /MT /GS /sdl /O2' + }, + LINUX: { + INTEL64: '-c -m64 -fPIC -fPIE -fstack-protector-strong ' + + '-fstack-protector -O2 -D_FORTIFY_SOURCE=2 ' + + '-Wformat -Wformat-security', + IA32: '-c -m32 -fPIC -fPIE -fstack-protector-strong ' + + '-fstack-protector -O2 -D_FORTIFY_SOURCE=2 ' + + '-Wformat -Wformat-security' + }, + MACOSX: { + INTEL64 : '-c -m64', + IA32 : '-c -m32' + } +} + +LINKER_FLAGS = { + WINDOWS: { + INTEL64 : '/MACHINE:X64 /NXCompat /DynamicBase ', + IA32 : '/MACHINE:X86 /SafeSEH /NXCompat /DynamicBase' + }, + LINUX: { + INTEL64 : '-z noexecstack -z relro -z now', + IA32 : '-m32 -z noexecstack -z relro -z now' + }, + MACOSX: { + INTEL64: '-dynamiclib -single_module ' + + '-flat_namespace -headerpad_max_install_names ' + + '-current_version 2017.0 -compatibility_version 2017.0', + IA32: '-dynamical -single_module ' + + '-flat_namespace -headerpad_max_install_names ' + + '-current_version 2017.0 -compatibility_version 2017.0' + } +} + +COMPILE_COMMAND_FORMAT = { + WINDOWS: '{compiler} {cmp_flags} ' + '/I "%{root_type}%\\include" ' + '{c_files}', + LINUX: '{compiler} {cmp_flags}' + ' -I "${root_type}/include" ' + '{c_files}', + MACOSX: '{compiler} {cmp_flags} ' + '-I "${root_type}/include" ' + '{c_files}' +} + +LINK_COMMAND_FORMAT = { + WINDOWS: '{linker} /DLL {link_flags} /VERBOSE:SAFESEH ' + '/DEF:"{export_file}" *.obj ' + '/OUT:"{custom_library}.dll" /IMPLIB:"{custom_library}.lib" ' + '{ipp_libraries} ' + '{exp_libs}', + LINUX: '{linker} -shared {link_flags} ' + '"{export_file}" *.o ' + '-o "{custom_library}.so" ' + '{ipp_libraries} ' + '-L"{sys_libs_path}" -lc -lm {exp_libs}', + MACOSX: '{linker} {link_flags} ' + '-install_name @rpath/{custom_library}.dylib ' + '-o "{custom_library}.dylib" ' + '-exported_symbols_list "{export_file}" *.o ' + '{ipp_libraries} ' + '-lgcc_s.1 -lm {exp_libs}' +} + +SYS_LIBS_PATH = { + WINDOWS: { + INTEL64 : '', + IA32 : '', + }, + LINUX: { + INTEL64 : '$SYSROOT/lib64', + IA32 : '$SYSROOT/lib' + }, + MACOSX: { + INTEL64 : '', + IA32 : '' + } +} + +EXP_LIBS = { + WINDOWS: { + SINGLE_THREADED : '', + MULTI_THREADED : 'libiomp5md.lib', + TBB : 'tbb.lib', + OPENMP : 'libiomp5md.lib' + }, + LINUX: { + SINGLE_THREADED : '', + MULTI_THREADED : '-liomp5', + TBB : '-ltbb', + OPENMP : '-liomp5' + }, + MACOSX: { + SINGLE_THREADED : '', + MULTI_THREADED : '-liomp5', + TBB : '"${TBBROOT}/lib/libtbb.dylib"', + OPENMP : '-liomp5' + } +} + +MAIN_FILE = { + WINDOWS: "#include \n" + "#include \"{package_type}.h\"\n\n" + "int WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved)\n" + "{{\n" + " switch (fdwReason)\n" + " {{\n" + " case DLL_PROCESS_ATTACH:\n" + " {package_type}Init(); break;\n" + " case DLL_THREAD_ATTACH: break;\n" + " case DLL_THREAD_DETACH: break;\n" + " case DLL_PROCESS_DETACH: break;\n" + " default: break;\n" + " }}\n" + " return 1;\n" + " UNREFERENCED_PARAMETER(hinstDLL);\n" + " UNREFERENCED_PARAMETER(lpvReserved);\n" + "}}\n", + LINUX: "#include \"{package_type}.h\"\n\n" + "int _init(void)\n" + "{{\n" + " {package_type}Init();\n" + " return 1;\n" + "}}\n\n" + "void _fini(void)\n" + "{{\n" + "}}\n", + MACOSX: "#include \"{package_type}.h\"\n\n" + "__attribute__((constructor)) void initializer( void )\n" + "{{\n" + " static int initialized = 0;\n" + " if (!initialized)\n" + " {{\n" + " initialized = 1;\n" + " }}\n\n" + " {package_type}Init();\n" + " return;\n" + "}}\n\n" + "__attribute__((destructor)) void destructor()\n" + "{{\n" + "}}\n" +} + +CUSTOM_DISPATCHER_FILE = '{include_lines}\n'\ + '#ifndef IPP_CALL\n' \ + '#define IPP_CALL IPP_STDCALL\n' \ + '#endif\n'\ + '#define IPPFUN(type,name,arg) extern type IPP_CALL name arg\n\n'\ + '#ifndef NULL\n'\ + '#ifdef __cplusplus\n'\ + '#define NULL 0\n'\ + '#else\n'\ + '#define NULL ((void *)0)\n'\ + '#endif\n'\ + '#endif\n\n'\ + '{architecture}\n'\ + '{features}\n'\ + '#ifdef __cplusplus\n'\ + 'extern "C" {{\n'\ + '#endif\n\n'\ + '{dispatcher}'\ + '#ifdef __cplusplus\n'\ + '}}\n'\ + '#endif\n\n'\ + '#endif\n' + +RENAME_FORMAT = '\n\n{declaration}\n' \ + '#define {function} {prefix}{function}' + +INCLUDE_STR = '#include "{header_name}"\n' + +ARCHITECTURE_DEFINE = { + IA32 : '#if !defined (_M_AMD64) && !defined (__x86_64__)', + INTEL64 : '#if defined (_M_AMD64) || defined (__x86_64__)' +} + +FEATURES = { + IA32: '', + INTEL64: '\n#define AVX3I_FEATURES ( ippCPUID_SHA|ippCPUID_AVX512VBMI|' + 'ippCPUID_AVX512VBMI2|ippCPUID_AVX512IFMA|ippCPUID_AVX512GFNI|' + 'ippCPUID_AVX512VAES|ippCPUID_AVX512VCLMUL )\n' + '#define AVX3X_FEATURES ( ippCPUID_AVX512F|ippCPUID_AVX512CD|' + 'ippCPUID_AVX512VL|ippCPUID_AVX512BW|ippCPUID_AVX512DQ )\n' + '#define AVX3M_FEATURES ( ippCPUID_AVX512F|ippCPUID_AVX512CD|' + 'ippCPUID_AVX512PF|ippCPUID_AVX512ER )\n' +} + +FUNCTION_DISPATCHER = '{ippapi}\n'\ + '{ippfun}\n'\ + '{{\n'\ + ' Ipp64u _features;\n'\ + ' _features = {package_type}GetEnabledCpuFeatures();\n\n'\ + '{dispatching_scheme}'\ + '}}\n\n' + +DISPATCHING_SCHEME_FORMAT = ' if( {cpuid} == ( _features & {cpuid} )) {{\n'\ + ' return {function}( {args} );\n'\ + ' }} else \n' + +RETURN_VALUES = { + 'IppStatus' : 'ippStsCpuNotSupportedErr', + 'IppiRect' : '(IppiRect) { IPP_MIN_32S / 2, IPP_MIN_32S / 2, ' + 'IPP_MAX_32S, IPP_MAX_32S }', + 'void' : '', + 'default' : 'NULL' +} + +BUILD_SCRIPT = { + WINDOWS: ':: Generates {threading} dynamic library ' + + 'for {architecture} architecture\n' + + '@echo off\n' + + 'set "OUTPUT_PATH={output_path}"\n' + + 'if not exist %OUTPUT_PATH% mkdir %OUTPUT_PATH%\n' + + 'if exist "{custom_library}.dll" del "{custom_library}.dll"\n\n' + + 'setlocal\n' + + '{env_commands}\n' + + 'cd /d %OUTPUT_PATH%\n' + + '{compile_command}\n' + + '{link_command}\n' + + 'endlocal\n\n' + + 'if %ERRORLEVEL%==0 (\n' + + ' echo Build completed!\n' + + ' del /s /q /f *.obj > nul\n' + + ' exit /b 0\n' + + ') else (\n' + + ' echo Build failed!\n' + + ' exit /b 1\n' + + ')', + LINUX: '#!/bin/bash\n' + + '# Generates {threading} dynamic library ' + + 'for {architecture} architecture\n' + + 'OUTPUT_PATH="{output_path}"\n' + + 'mkdir -p $OUTPUT_PATH\n' + + 'cd $OUTPUT_PATH\n\n' + + 'rm -rf "{custom_library}.so"\n\n' + + '{env_commands}\n' + + '{compile_command}\n' + + '{link_command}\n' + + 'if [ $? == 0 ]; then\n' + + ' echo Build completed!\n' + + ' rm -rf *.o\n' + + ' exit 0\n' + + 'else\n' + + ' echo Build failed!\n' + + ' exit 1\n' + + 'fi', + MACOSX: '#!/bin/bash\n' + + '# Generates {threading} dynamic library ' + + 'for {architecture} architecture\n' + + 'OUTPUT_PATH="{output_path}"\n' + + 'mkdir -p $OUTPUT_PATH\n' + + 'cd $OUTPUT_PATH\n\n' + + 'rm -rf "{custom_library}.dylib"\n\n' + + '{env_commands}\n' + + '{compile_command}\n' + + '{link_command}\n\n' + + 'if [ $? == 0 ]; then\n' + + ' echo Build completed!\n' + + ' rm -rf *.o\n' + + ' exit 0\n' + + 'else\n' + + ' echo Build failed!\n' + + ' exit 1\n' + + 'fi' +} diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/tool/package.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/tool/package.py new file mode 100644 index 0000000..da1f96f --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/tool/package.py @@ -0,0 +1,231 @@ +""" +Copyright 2018-2021 Intel Corporation. + +This software and the related documents are Intel copyrighted materials, and +your use of them is governed by the express license under which they were +provided to you (License). Unless the License provides otherwise, you may not +use, modify, copy, publish, distribute, disclose or transmit this software or +the related documents without Intel's prior written permission. + +This software and the related documents are provided as is, with no express +or implied warranties, other than those that are expressly stated in the +License. + +License: +http://software.intel.com/en-us/articles/intel-sample-source-code-license-agr +eement/ +""" +import os +from collections import defaultdict + +from tool import utils + + +class Package: + def __init__(self, path): + self.root = path + self.headers_dir = os.path.join(self.root, 'include') + self.libraries_dir = os.path.join(self.root, 'lib') + + self.headers = utils.nested_dict_init() + self.functions = utils.nested_dict_init() + self.declarations = defaultdict() + + self.libraries = utils.nested_dict_init() + self.features = utils.nested_dict_init() + self.errors = utils.nested_dict_init() + + self.functions_without_dispatcher = [] + + self.set_type() + self.set_env_script() + + self.set_headers_functions_declarations_dicts() + self.set_libraries_features_errors_dicts(self.type, utils.THREAD_MODES) + self.set_libraries_features_errors_dicts(utils.THREADING_LAYER, utils.TL_TYPES) + + self.package_validation() + self.set_name() + + def set_type(self): + header = os.path.join(self.headers_dir, 'ippcp.h') + self.type = (utils.IPP if not os.path.exists(header) else utils.IPPCP) + + def set_name(self): + self.name = 'None' + version = self.get_version() + + if not self.broken: + self.name = utils.PACKAGE_NAME[self.type] + ' Version ' + version + + def get_version(self): + version = '' + header = os.path.join(self.headers_dir, 'ippversion.h') + + for line in utils.get_lines_from_file(header): + version = utils.get_match(utils.VERSION_REGEX, line, 'ver') + if version: + break + if not version: + version = 'None' + + version = self.parse_version_string(version, header) + + return version + + def parse_version_string(self, version, header): + while True: + macros = utils.get_match(utils.STR_MACROS_REGEX, version, 'macros') + if not macros: + break + + macros_value = '' + for line in utils.get_lines_from_file(header): + if macros in line: + macros_value = line.split(macros, 1)[1].strip() + break + + version = version.replace('STR(' + macros + ')', macros_value) + + while True: + c_string = utils.get_match(utils.C_STRING_REGEX, version, 'string') + if not c_string: + break + + c_string_value = utils.get_match(utils.C_STRING_VALUE_REGEX, c_string, 'value') + version = version.replace(c_string, c_string_value) + + return version + + def set_env_script(self): + paths_to_search = [] + batch_extension = utils.BATCH_EXTENSIONS[utils.HOST_SYSTEM] + + components_install_dir = utils.get_match(utils.COMPONENTS_INSTALL_DIR_REGEX, self.root, 'path') + if components_install_dir: + paths_to_search.append(os.path.join(components_install_dir, 'setvars' + batch_extension)) + paths_to_search.append(os.path.join(components_install_dir, 'bin', 'compilervars' + batch_extension)) + paths_to_search.append(os.path.join(self.root, 'bin', self.type.lower() + 'vars' + batch_extension)) + paths_to_search.append(os.path.join(self.root, 'env', 'vars' + batch_extension)) + + self.env_script = utils.get_first_existing_path_in_list(paths_to_search) + + def set_headers_functions_declarations_dicts(self): + if not os.path.exists(self.headers_dir): + return + + headers = [header for header in os.listdir(self.headers_dir) if '.h' in header] + + for header in headers: + domain_type, domain = self.get_type_and_domain(header) + if not domain: + continue + + functions_list = [] + incomplete_function = '' + for line in utils.get_lines_from_file(os.path.join(self.headers_dir, header)): + if incomplete_function: + self.declarations[incomplete_function] += ' ' + " ".join(line.split()) + incomplete_function = self.get_function_if_incomplete(incomplete_function) + continue + + function = utils.get_match(utils.FUNCTION_NAME_REGEX, line, 'function_name') + if not function: + continue + functions_list.append(function) + self.declarations[function] = " ".join(line.split()) + incomplete_function = self.get_function_if_incomplete(function) + + if not functions_list: + continue + + self.headers[domain_type][domain] = header + + domain_name = utils.DOMAINS[domain_type][domain] + if domain_name not in self.functions[domain_type]: + self.functions[domain_type][domain_name] = functions_list + else: + self.functions[domain_type][domain_name] += functions_list + + if header == 'ippcpdefs.h' or \ + domain_type == utils.THREADING_LAYER or \ + domain == 'ippcore': + self.functions_without_dispatcher += functions_list + + def set_libraries_features_errors_dicts(self, domains_type, thread_types): + host = utils.HOST_SYSTEM + + for arch in utils.ARCHITECTURES: + for thread_type in thread_types: + self.libraries[arch][thread_type] = [] + path_to_libraries = self.get_path_to_libraries(arch, thread_type) + + for domain in utils.DOMAINS[domains_type].keys(): + lib_name = utils.LIB_PREFIX[host] + \ + domain + \ + utils.STATIC_LIB_POSTFIX[host] + \ + utils.LIB_POSTFIX[thread_type] + \ + utils.STATIC_LIB_EXTENSION[host] + + lib_path = os.path.join(path_to_libraries, + lib_name) + + if os.path.exists(lib_path): + self.libraries[arch][thread_type].append(lib_path) + elif not domain == 'ippe': + self.libraries[arch][thread_type].clear() + break + + self.features[arch][thread_type] = bool(self.libraries[arch][thread_type]) + self.errors[arch][thread_type] = self.check_headers_and_libs(arch, domains_type, thread_type) + + def package_validation(self): + self.broken = True + self.error_message = self.errors[utils.INTEL64][utils.SINGLE_THREADED] + + for arch in utils.ARCHITECTURES: + for thread in utils.THREAD_MODES: + if self.features[arch][thread]: + self.broken = False + return + + def get_type_and_domain(self, header): + domain_type = (self.type if '_tl' not in header else utils.THREADING_LAYER) + for domain in utils.DOMAINS[domain_type].keys(): + if domain in header: + return domain_type, domain + return '', '' + + def get_function_if_incomplete(self, function): + if self.declarations[function].count('(') != self.declarations[function].count(')'): + return function + return '' + + def get_path_to_libraries(self, arch, thread_type): + paths_to_search = [os.path.join(self.libraries_dir, arch), + os.path.join(self.libraries_dir, arch + '_' + utils.HOST_SYSTEM[:3].lower()), + self.libraries_dir] + paths_to_search = [utils.PATH_TO_LIBRARIES[thread_type].format(libs_arch_dir=path) + for path in paths_to_search] + + return utils.get_first_existing_path_in_list(paths_to_search) + + def check_headers_and_libs(self, arch, domains_type, thread_type): + for domain in utils.DOMAINS[domains_type].keys(): + if domains_type not in self.headers.keys() or \ + (domain not in self.headers[domains_type].keys() and not domain == 'ippe'): + return "Broken package - cannot find header files for " + domains_type + " functions" + + if not self.libraries[arch][thread_type]: + return "Cannot find " + thread_type + " libraries for " + arch + " architecture" + + return '' + + +def get_package_path(): + current_path = os.path.realpath(__file__) + paths_to_search = [utils.get_match(utils.PATH_TO_PACKAGE_REGEX, current_path, 'path'), + utils.get_env(utils.IPPROOT), + utils.get_env(utils.IPPCRYPTOROOT)] + + return utils.get_first_existing_path_in_list(paths_to_search) diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/tool/utils.py b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/tool/utils.py new file mode 100644 index 0000000..33e5793 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/library/src/tools/ipp_custom_library_tool_python/tool/utils.py @@ -0,0 +1,348 @@ +""" +Copyright 2018-2021 Intel Corporation. + +This software and the related documents are Intel copyrighted materials, and +your use of them is governed by the express license under which they were +provided to you (License). Unless the License provides otherwise, you may not +use, modify, copy, publish, distribute, disclose or transmit this software or +the related documents without Intel's prior written permission. + +This software and the related documents are provided as is, with no express +or implied warranties, other than those that are expressly stated in the +License. + +License: +http://software.intel.com/en-us/articles/intel-sample-source-code-license-agr +eement/ +""" +import os +import sys +import re +from platform import system +from collections import defaultdict + +WINDOWS = 'Windows' +LINUX = 'Linux' +MACOSX = 'MacOS' +HOST_SYSTEM = None +SUPPORTED_SYSTEMS = [WINDOWS, LINUX, MACOSX] + +IA32 = 'ia32' +INTEL64 = 'intel64' +ARCHITECTURES = [IA32, INTEL64] + +SINGLE_THREADED = 'single-threaded' +MULTI_THREADED = 'multi-threaded' +THREAD_MODES = [SINGLE_THREADED, MULTI_THREADED] + +TBB = 'tbb' +OPENMP = 'openmp' +TL_TYPES = [TBB, OPENMP] + +PATH_TO_PACKAGE_REGEX = '(?P.*)\Wtools\W.*' +COMPONENTS_INSTALL_DIR_REGEX = '(?P.*)\Wipp.*' +VERSION_REGEX = '.*VERSION_STR\s*(?P.*)\s*' +STR_MACROS_REGEX = '.*STR\((?P\S*)\).*' +C_STRING_REGEX = '.*(\S|^)(?P\s*".*"\s*)(\S|$).*' +C_STRING_VALUE_REGEX = '.*"(?P.*)".*' +FUNCTION_NAME_REGEX = 'IPPAPI\s*\(\s*(?P.*?)\s*,' \ + '\s*(?P\S*)\s*,' \ + '\s*\(?(?P.*?)\s*\)?\s*\)?\s*$' +ARGUMENT_REGEX = '.*\W*\w+\W*\s+\W*(?P[^\W\d]+\w*)\W*?' + +CUSTOM_LIBRARY_NAME = 'Custom library name' +BUILD_SCRIPT_NAME = 'Build script name' +OUTPUT_PATH = 'Output path' +FUNCTIONS_LIST = 'Functions list' +PACKAGE = 'Package' +ARCHITECTURE = 'Architecture' +THREAD_MODE = 'Thread mode' +THREADING_LAYER = 'Threading layer' +CUSTOM_CPU_SET = 'Custom CPU set' +PREFIX = 'Prefix' + +CONFIGS = { + CUSTOM_LIBRARY_NAME : 'custom_library', + BUILD_SCRIPT_NAME : '', + OUTPUT_PATH : '.', + FUNCTIONS_LIST : [], + PACKAGE : '', + ARCHITECTURE : '', + THREAD_MODE : '', + THREADING_LAYER : '', + CUSTOM_CPU_SET : [], + PREFIX : '' +} + +IPP = 'IPP' +IPPCP = 'IPPCP' + +IPPROOT = 'IPPROOT' +IPPCRYPTOROOT = 'IPPCRYPTOROOT' + +PACKAGE_NAME = { + IPP : 'Intel(R) Integrated Performance Primitives', + IPPCP : 'Intel(R) Integrated Performance Primitives Cryptography' +} + +DOMAINS = { + IPP: { + 'ippcc' : 'Color Conversion', + 'ippch' : 'String Operations', + 'ippcv' : 'Computer Vision', + 'ippdc' : 'Data Compression', + 'ippe' : 'Embedded Functionality', + 'ippi' : 'Image Processing', + 'ipps' : 'Signal Processing', + 'ippvm' : 'Vector Math', + 'ippcore' : 'Core' + }, + IPPCP: { + 'ippcp' : 'Cryptography' + }, + THREADING_LAYER: { + 'ippcc' : 'Color Conversion TL', + 'ippcv' : 'Computer Vision TL', + 'ippi' : 'Image Processing TL', + 'ippcore' : 'Core TL' + } +} + +SSE2 = 'sse2' +SSE3 = 'sse3' +SSSE3 = 'ssse3' +SSE42 = 'sse42' +AVX = 'avx' +AVX2 = 'avx2' +AVX512F = 'avx512f' +AVX512BW = 'avx512bw' +AVX512IFMA = 'avx512ifma' + +CPU = { + SSE2: { + IA32 : 'w7', + INTEL64 : '' + }, + SSE3: { + IA32 : '', + INTEL64 : 'm7' + }, + SSSE3: { + IA32 : 's8', + INTEL64 : 'n8' + }, + SSE42: { + IA32 : 'p8', + INTEL64 : 'y8' + }, + AVX: { + IA32 : 'g9', + INTEL64 : 'e9' + }, + AVX2: { + IA32 : 'h9', + INTEL64 : 'l9' + }, + AVX512F: { + IA32 : '', + INTEL64 : 'n0' + }, + AVX512BW: { + IA32 : '', + INTEL64 : 'k0' + }, + AVX512IFMA: { + IA32 : '', + INTEL64 : 'k1' + } +} + +SUPPORTED_CPUS = { + IPP: { + IA32: { + WINDOWS : [SSE2, SSSE3, SSE42, AVX, AVX2], + LINUX : [SSE2, SSSE3, SSE42, AVX, AVX2], + MACOSX : [] + }, + INTEL64: { + WINDOWS : [SSE3, SSSE3, SSE42, AVX, AVX2, AVX512BW], + LINUX : [SSE3, SSSE3, SSE42, AVX, AVX2, AVX512F, AVX512BW], + MACOSX : [SSE42, AVX, AVX2, AVX512BW] + } + }, + IPPCP: { + IA32: { + WINDOWS : [SSE2, SSSE3, SSE42, AVX, AVX2], + LINUX : [SSE2, SSSE3, SSE42, AVX, AVX2], + MACOSX : [] + }, + INTEL64: { + WINDOWS : [SSE3, SSSE3, SSE42, AVX, AVX2, AVX512BW, AVX512IFMA], + LINUX : [SSE3, SSSE3, SSE42, AVX, AVX2, AVX512F, AVX512BW, AVX512IFMA], + MACOSX : [SSE42, AVX, AVX2, AVX512BW, AVX512IFMA] + } + } +} + +CPUID = { + AVX512IFMA : 'AVX3I_FEATURES', + AVX512BW : 'AVX3X_FEATURES', + AVX512F : 'AVX3M_FEATURES', + AVX2 : 'ippCPUID_AVX2', + AVX : 'ippCPUID_AVX', + SSE42 : 'ippCPUID_SSE42', + SSSE3 : 'ippCPUID_SSSE3', + SSE3 : 'ippCPUID_SSE3', + SSE2 : 'ippCPUID_SSE2', +} + +PATH_TO_LIBRARIES = { + SINGLE_THREADED : '{libs_arch_dir}', + MULTI_THREADED : '{libs_arch_dir}/threaded', + TBB : '{libs_arch_dir}/tl/' + TBB, + OPENMP : '{libs_arch_dir}/tl/' + OPENMP +} + +LIB_PREFIX = { + WINDOWS : '', + LINUX : 'lib', + MACOSX : 'lib' +} + +LIB_POSTFIX = { + SINGLE_THREADED : '', + MULTI_THREADED : '', + TBB: '_tl_tbb', + OPENMP: '_tl_omp' +} + +STATIC_LIB_POSTFIX = { + WINDOWS : 'mt', + LINUX : '', + MACOSX : '' +} + +STATIC_LIB_EXTENSION = { + WINDOWS : '.lib', + LINUX : '.a', + MACOSX : '.a' +} + +DYNAMIC_LIB_EXTENSION = { + WINDOWS : '.dll', + LINUX : '.so', + MACOSX : '.dylib' +} + +BUILD_SCRIPT_NAME_FORMAT = { + WINDOWS : 'build_{name}_{arch}.bat', + LINUX : 'build_{name}_{arch}.sh', + MACOSX : 'build_{name}_{arch}.sh' +} + +BATCH_EXTENSIONS = { + WINDOWS : '.bat', + LINUX : '.sh', + MACOSX : '.sh' +} + +MAIN_FILE_NAME = 'main.c' +CUSTOM_DISPATCHER_FILE_NAME = '{function}.c' +RENAME_HEADER_NAME = 'rename.h' + +EXPORT_FILE = { + WINDOWS : 'export.def', + LINUX : 'export.def', + MACOSX : 'export.lib-export' +} + +PROJECT_EXTENSION = '.cltproj' + +HAVE_PACKAGE = 'package...' +HAVE_FUNCTIONS = 'functions that has to be in dynamic library... ' + +ENABLE_GENERATION_RULES = { + HAVE_PACKAGE : True, + HAVE_FUNCTIONS : True, +} + + +def set_host_system(): + host_system = system() + + if host_system == 'Darwin': + host_system = MACOSX + if host_system not in SUPPORTED_SYSTEMS: + sys.exit("Error: Intel(R) Integrated Performance Primitives Custom Library Tool isn't supported for OS " + + host_system) + + global HOST_SYSTEM + HOST_SYSTEM = host_system + + +def set_configs_dict(package, + functions_list, + architecture, + thread_mode, + threading_layer_type, + custom_library_name=CONFIGS[CUSTOM_LIBRARY_NAME], + build_script_name=CONFIGS[BUILD_SCRIPT_NAME], + output_path=CONFIGS[OUTPUT_PATH], + custom_cpu_set=CONFIGS[CUSTOM_CPU_SET], + prefix=CONFIGS[PREFIX]): + CONFIGS[CUSTOM_LIBRARY_NAME] = custom_library_name + CONFIGS[OUTPUT_PATH] = output_path + CONFIGS[FUNCTIONS_LIST] = functions_list + CONFIGS[PACKAGE] = package + CONFIGS[ARCHITECTURE] = architecture + CONFIGS[THREAD_MODE] = thread_mode + CONFIGS[THREADING_LAYER] = threading_layer_type + CONFIGS[CUSTOM_CPU_SET] = custom_cpu_set + CONFIGS[PREFIX] = prefix + + if not build_script_name: + build_script_name = BUILD_SCRIPT_NAME_FORMAT[HOST_SYSTEM].format(name=custom_library_name, arch=architecture) + CONFIGS[BUILD_SCRIPT_NAME] = build_script_name + + +def get_first_existing_path_in_list(paths_list): + for path in paths_list: + if os.path.exists(path): + return path + return '' + + +def get_lines_from_file(file_path): + if os.path.exists(file_path): + with open(file_path, 'r') as file: + return file.readlines() + else: + return [] + + +def get_env(env_var): + return os.environ[env_var] if os.getenv(env_var) and os.path.exists(os.environ[env_var]) else '' + + +def get_match(regex, string, group): + return re.match(regex, string).group(group) if re.compile(regex).match(string) else '' + + +def nested_dict_init(): + return defaultdict(lambda: defaultdict()) + + +def walk_dict(dictionary): + for key, value in dictionary.items(): + if type(value) == type(dict()) or type(value) == type(defaultdict()): + for entire in walk_dict(value): + yield (key,) + entire + elif type(value) == type(list()): + for elem in value: + yield key, elem + else: + yield key, value + + +def get_dict_value(dictionary, key): + return dictionary[key] if key in dictionary.keys() else dictionary['default'] diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/security.md b/librocksdb-sys/rocksdb/plugin/ippcp/security.md new file mode 100644 index 0000000..cb59eb8 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/security.md @@ -0,0 +1,5 @@ +# Security Policy +Intel is committed to rapidly addressing security vulnerabilities affecting our customers and providing clear guidance on the solution, impact, severity and mitigation. + +## Reporting a Vulnerability +Please report any security vulnerabilities in this project utilizing the guidelines [here](https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html). diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/tests/CMakeLists.txt b/librocksdb-sys/rocksdb/plugin/ippcp/tests/CMakeLists.txt new file mode 100644 index 0000000..a573a58 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/tests/CMakeLists.txt @@ -0,0 +1,61 @@ +# Copyright (C) 2022 Intel Corporation + +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.15) + +project(ippcp_encryptor_test VERSION 0.0.1) + +option(COVERAGE "Enable test coverage report" ON) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +set(ippcp_encryptor_test_CMAKE_EXE_LINKER_FLAGS "-u ippcp_reg") +add_executable(ippcp_encryptor_test ../ippcp_provider.cc ippcp_encryptor_test.cc) + + +if(NOT DEFINED IPPCRYPTOROOT) + find_package(ippcp REQUIRED) + if(ippcp_FOUND) + message(STATUS "Found ippcp: ${ippcp_DIR}") + target_link_libraries(ippcp_encryptor_test ippcp::ippcp) + endif() +else() + message(STATUS "Using IPPCRYPTOROOT: ${IPPCRYPTOROOT}") + include_directories(${IPPCRYPTOROOT}/include) + target_link_libraries(ippcp_encryptor_test ippcp) +endif() + +if(NOT DEFINED ROCKSDB_PATH) + find_package(RocksDB REQUIRED) + if(RocksDB_FOUND) + message(STATUS "Found RocksDB: ${RocksDB_DIR}") + target_link_libraries(ippcp_encryptor_test rocksdb) + endif() +elseif(DEFINED ROCKSDB_PATH) + message(STATUS "Using ROCKSDB_PATH: ${ROCKSDB_PATH}") + include_directories(${ROCKSDB_PATH} ${ROCKSDB_PATH}/include) + target_link_directories(ippcp_encryptor_test PUBLIC ${ROCKSDB_PATH}) + target_link_libraries(ippcp_encryptor_test rocksdb) +endif() + +find_package(GTest REQUIRED) +target_link_libraries(ippcp_encryptor_test gtest) + +add_compile_definitions(ROCKSDB_PLATFORM_POSIX) +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-rtti") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti") +if(COVERAGE) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --coverage") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage") +endif() + +add_custom_target(run + COMMAND ./ippcp_encryptor_test + DEPENDS ippcp_encryptor_test +) + +add_custom_target(coverage + COMMAND lcov --directory . --capture --output-file ippcp_encryptor_test.info && genhtml -o html ippcp_encryptor_test.info +) \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/plugin/ippcp/tests/ippcp_encryptor_test.cc b/librocksdb-sys/rocksdb/plugin/ippcp/tests/ippcp_encryptor_test.cc new file mode 100644 index 0000000..3686ea4 --- /dev/null +++ b/librocksdb-sys/rocksdb/plugin/ippcp/tests/ippcp_encryptor_test.cc @@ -0,0 +1,430 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// Copyright (c) 2020 Intel Corporation +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "../ippcp_provider.h" +#include + +namespace ROCKSDB_NAMESPACE +{ + + TEST(IppcpBasicTests, LoadIppcpProvider) + { + std::string IPPCP = IppcpProvider::kName(); + std::shared_ptr provider; + Status s = EncryptionProvider::CreateFromString(ConfigOptions(), IPPCP, &provider); + ASSERT_TRUE(s.ok()); + ASSERT_NE(provider, nullptr); + ASSERT_EQ(provider->Name(), IPPCP); + std::string cipher_key; + cipher_key.assign("a6d2ae2816157e2b3c4fcf098815f7x1"); + s = provider->AddCipher("", cipher_key.c_str(), cipher_key.length(), false); + ASSERT_TRUE(s.ok()) << s.ToString(); + ; + } + + TEST(IppcpBasicTests, TestAddKeys) + { + std::string IPPCP = IppcpProvider::kName(); + std::shared_ptr provider; + Status s = EncryptionProvider::CreateFromString(ConfigOptions(), IPPCP, &provider); + ASSERT_TRUE(s.ok()) << s.ToString(); + ; + ASSERT_NE(provider, nullptr); + std::string cipher_key; + cipher_key.assign("a6d2ae2816157e2b3c4fcf098815f7x2"); + s = provider->AddCipher("", cipher_key.c_str(), cipher_key.length(), false); + ASSERT_TRUE(s.ok()) << s.ToString(); + + provider.reset(); + s = EncryptionProvider::CreateFromString(ConfigOptions(), IPPCP, &provider); + ASSERT_TRUE(s.ok()) << s.ToString(); + ; + ASSERT_NE(provider, nullptr); + cipher_key.assign("a6d2ae2816157e2beeeeeeee"); + s = provider->AddCipher("", cipher_key.c_str(), cipher_key.length(), false); + ASSERT_TRUE(s.ok()) << s.ToString(); + + provider.reset(); + cipher_key.assign("a6d2ae2816157e21"); + s = EncryptionProvider::CreateFromString(ConfigOptions(), IPPCP, &provider); + ASSERT_TRUE(s.ok()) << s.ToString(); + ASSERT_NE(provider, nullptr); + s = provider->AddCipher("", cipher_key.c_str(), cipher_key.length(), false); + ASSERT_TRUE(s.ok()) << s.ToString(); + } + + TEST(IppcpBasicTests, TestIncorrectKeyLength) + { + std::string IPPCP = IppcpProvider::kName(); + std::shared_ptr provider; + Status s = EncryptionProvider::CreateFromString(ConfigOptions(), IPPCP, &provider); + ASSERT_TRUE(s.ok()); + ASSERT_NE(provider, nullptr); + std::string cipher_key; + + // empty encryption key + cipher_key.assign(""); + s = provider->AddCipher("", cipher_key.c_str(), cipher_key.length(), false); + ASSERT_TRUE(s.IsInvalidArgument()) << s.ToString(); + + // incoorect encryption key length + cipher_key.assign("a6d2ae2"); + s = provider->AddCipher("", cipher_key.c_str(), cipher_key.length(), false); + ASSERT_TRUE(s.IsInvalidArgument()) << s.ToString(); + } + + TEST(IppcpBasicTests, TestAddingMultipleKeys) + { + std::string IPPCP = IppcpProvider::kName(); + std::shared_ptr provider; + Status s = EncryptionProvider::CreateFromString(ConfigOptions(), IPPCP, &provider); + ASSERT_TRUE(s.ok()); + ASSERT_NE(provider, nullptr); + std::string cipher_key; + // correct encryption key + cipher_key.assign("a6d2ae2816157e21"); + s = provider->AddCipher("", cipher_key.c_str(), cipher_key.length(), false); + ASSERT_TRUE(s.ok()) << s.ToString(); + + // adding multiple cipher/encryption keys not allowed + cipher_key.assign("a6d2ae281wwwwddd22222213"); + s = provider->AddCipher("", cipher_key.c_str(), cipher_key.length(), false); + ASSERT_TRUE(s.IsInvalidArgument()) << s.ToString(); + } + + TEST(IppcpEncryptionTests, CounterBlkTests) + { + std::string IPPCP = IppcpProvider::kName(); + std::shared_ptr provider; + // creating ipp provider and setting cipher key + Status s = EncryptionProvider::CreateFromString(ConfigOptions(), IPPCP, &provider); + ASSERT_TRUE(s.ok()); + ASSERT_NE(provider, nullptr); + std::string cipher_key; + cipher_key.assign("a6d2ae2816157e2b3c4fcf098815f7x2"); + s = provider->AddCipher("", cipher_key.c_str(), cipher_key.length(), false); + ASSERT_TRUE(s.ok()) << s.ToString(); + // initilizing prefix which sets the 128 initVector data memmber + // the first 8 bytes will be used for counter + size_t prefixLen = 16; // minimum size of prefix is 16(blockSize) + uint8_t ctr[] = {0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; + Slice prefix((char *)ctr, prefixLen); + + std::unique_ptr stream; + const EnvOptions options; + // creating cipher stream object to perform encryption and decryption + s = provider->CreateCipherStream("", options, prefix, &stream); + ASSERT_TRUE(s.ok()) << s.ToString(); + + std::string input1, input2, input3, plainTxt; + uint64_t offset = 0; // offset where from we need to perform encryption/decryption + plainTxt = ""; + input1.assign("1 input for CounterBlk hellooo0 "); + input2.assign("2 input for CounterBlk hellooo0 "); + input3.assign("3 input for CounterBlk helloo0 "); + // concatenate the strings and encrypt them + plainTxt = input1 + input2 + input3; + s = stream->Encrypt(offset, (char *)plainTxt.c_str(), plainTxt.length()); // does in place encryption so plainTxt will be encrypted now + s = stream->Decrypt(offset, (char *)plainTxt.c_str(), plainTxt.length()); // in .place decryption + ASSERT_EQ(input1 + input2 + input3, plainTxt) << " both are strings are same after decryption!!"; + } + + /* + This test checks wraparound condition for counter.The plugin code uses 64 bit intrinsic _mm_add_epi64 for addition as index is 64bits. + plugin counter for all ff -> (ff ff ff ff ff ff ff ff 0 0 0 0 0 0 0 0) and (ff ff ff ff ff ff ff ff 0 0 0 0 0 0 0 1) so on + if the kCounterLen passed to ipp lib is 128 then it use all 128 bits for addition which means counter created + by plugin and ipp code will differ as it will rollover to all 0. + if all FF counter is passed to ipp then new counter created ->() 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 ),( 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2) etc + + To fix this issue the counter addition bit length needs to be same in both plugin and ipp lib code + so kCounterLen needs to be 64 bits. + + This test will fail if kCounterLen is 128 + */ + TEST(IppcpEncryptionTests, CounterBlkOverFlowTests) + { + std::string IPPCP = IppcpProvider::kName(); + std::shared_ptr provider; + // creating ipp provider and setting cipher key + Status s = EncryptionProvider::CreateFromString(ConfigOptions(), IPPCP, &provider); + ASSERT_TRUE(s.ok()); + ASSERT_NE(provider, nullptr); + std::string cipher_key; + cipher_key.assign("a6d2ae2816157e2b3c4fcf098815f7x2"); + s = provider->AddCipher("", cipher_key.c_str(), cipher_key.length(), false); + ASSERT_TRUE(s.ok()) << s.ToString(); + + // creating prefix which sets the 128 initVector data memmber + size_t prefixLen = 16; // minimum size of prefix is 16 + // setting prefix/counter to all ff's to check the overflow + uint8_t ctr[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; + Slice prefix((char *)ctr, prefixLen); + // creating cipher stream object to perform encryption and decryption + std::unique_ptr stream; + const EnvOptions options; + s = provider->CreateCipherStream("", options, prefix, &stream); + ASSERT_TRUE(s.ok()) << s.ToString(); + + // creating string each of 16 byte(blocksize) for encryption + std::string str1, str2, str3; + str1.assign("1111111111111111"); + str2.assign("2222222222222222"); + str3.assign("3333333333333333"); + + std::string encryptedString = ""; + encryptedString += str1; + encryptedString += str2; + encryptedString += str3; + // encrypted all the strings in one go.Here ipp lib will create counter block for 2nd and 3rd string block + s = stream->Encrypt(0, (char *)encryptedString.c_str(), encryptedString.length()); + std::string cipherTxt = encryptedString.substr(str1.length()); + // decrypt the encrypted string from str2 onwards i.e from block 2 onwards + s = stream->Decrypt(str1.length(), (char *)cipherTxt.c_str(), cipherTxt.length()); + // the decrypted string should match the str2 + str3 + ASSERT_EQ((str2 + str3), cipherTxt) << " both are strings are same after decryption!!"; + ASSERT_TRUE(s.ok()) << s.ToString(); + } + + /* + This test encrypts the input data and then decrypts it. Decrypted data should match the input for success. + This Matches RocksDB Encryption API flow. + */ + TEST(IppcpEncryptionTests, EncryptDecryptTest) + { + std::string IPPCP = IppcpProvider::kName(); + std::shared_ptr provider; + Status s = EncryptionProvider::CreateFromString(ConfigOptions(), IPPCP, &provider); + ASSERT_TRUE(s.ok()); + ASSERT_NE(provider, nullptr); + + std::string cipher_key; + cipher_key.assign("a6d2ae2816157e2b3c4fcf098815f7x2"); + s = provider->AddCipher("", cipher_key.c_str(), cipher_key.length(), false); + ASSERT_TRUE(s.ok()) << s.ToString(); + + size_t prefixLen = provider->GetPrefixLength(); + ASSERT_GT(prefixLen, 0); + char *buf = (char *)malloc(prefixLen); + ASSERT_NE(buf, nullptr); + std::unique_ptr stream; + const EnvOptions options; + s = provider->CreateNewPrefix("", buf, prefixLen); + ASSERT_TRUE(s.ok()) << s.ToString(); + Slice prefix(buf, prefixLen); + + s = provider->CreateCipherStream("", options, prefix, &stream); + ASSERT_TRUE(s.ok()) << s.ToString(); + + std::string input, plainTxt; + uint64_t offset = prefixLen; + input.assign("test ippcp crypto"); + plainTxt = input; // input becomes cipher txt in below API. + s = stream->Encrypt(offset, (char *)input.c_str(), input.length()); // does in place encryption + ASSERT_TRUE(s.ok()) << s.ToString(); + s = stream->Decrypt(offset, (char *)input.c_str(), input.length()); + ASSERT_EQ(plainTxt, input) << " both are strings are same after decryption!!"; + free(buf); + } + + /* + This test encrypts the multple input data and then decrypts it in one go. + Decrypted data should match the combined input for success. + This is to test the random decryption functionality. + */ + + TEST(IppcpEncryptionTests, RandomDecryptionTests) + { + std::string IPPCP = IppcpProvider::kName(); + std::shared_ptr provider; + Status s = EncryptionProvider::CreateFromString(ConfigOptions(), IPPCP, &provider); + ASSERT_TRUE(s.ok()); + ASSERT_NE(provider, nullptr); + + std::string cipher_key; + cipher_key.assign("a6d2ae2816157e2b3c4fcf098815f7x2"); + s = provider->AddCipher("", cipher_key.c_str(), cipher_key.length(), false); + ASSERT_TRUE(s.ok()) << s.ToString(); + + size_t prefixLen = provider->GetPrefixLength(); + ASSERT_GT(prefixLen, 0); + char *buf = (char *)malloc(prefixLen); + ASSERT_NE(buf, nullptr); + s = provider->CreateNewPrefix("", buf, prefixLen); + ASSERT_TRUE(s.ok()) << s.ToString(); + Slice prefix(buf, prefixLen); + + std::unique_ptr stream; + const EnvOptions options; + s = provider->CreateCipherStream("", options, prefix, &stream); + ASSERT_TRUE(s.ok()) << s.ToString(); + + std::string input1, plainTxt, cipherTxt; + uint64_t offset = prefixLen; + + input1.assign("1 input for encryption hellooo0 "); + plainTxt = input1; + s = stream->Encrypt(offset, (char *)input1.c_str(), input1.length()); // does in place encryption + ASSERT_TRUE(s.ok()) << s.ToString(); + cipherTxt = input1; + offset += input1.length(); + + std::string input2; + input2.assign("2 input for encryption hellooo0 "); + plainTxt += input2; + s = stream->Encrypt(offset, (char *)input2.c_str(), input2.length()); // does in place encryption + ASSERT_TRUE(s.ok()) << s.ToString(); + cipherTxt += input2; + offset += input2.length(); + + std::string input3; + input3.assign("3 input for encryption helloo0 "); + plainTxt += input3; + s = stream->Encrypt(offset, (char *)input3.c_str(), input3.length()); // does in place encryption + ASSERT_TRUE(s.ok()) << s.ToString(); + cipherTxt += input3; + // decrypt the all the input string in one go. + s = stream->Decrypt(prefixLen, (char *)cipherTxt.c_str(), cipherTxt.length()); + + ASSERT_EQ(plainTxt, cipherTxt) << " both are strings are same after decryption!!"; + free(buf); + } + + TEST(IppcpEncryptionTests, EncryptDecryptWithDifferentKeys) + { + std::string IPPCP = IppcpProvider::kName(); + std::shared_ptr provider; + Status s = EncryptionProvider::CreateFromString(ConfigOptions(), IPPCP, &provider); + ASSERT_TRUE(s.ok()); + ASSERT_NE(provider, nullptr); + + std::string cipher_key; + cipher_key.assign("a6d2ae2816157e2b3c4fcf098815f7x2"); + s = provider->AddCipher("", cipher_key.c_str(), cipher_key.length(), false); + ASSERT_TRUE(s.ok()) << s.ToString(); + + size_t prefixLen = provider->GetPrefixLength(); + ASSERT_GT(prefixLen, 0); + char *buf = (char *)malloc(prefixLen); + ASSERT_NE(buf, nullptr); + std::unique_ptr stream; + const EnvOptions options; + s = provider->CreateNewPrefix("", buf, prefixLen); + ASSERT_TRUE(s.ok()) << s.ToString(); + Slice prefix(buf, prefixLen); + s = provider->CreateCipherStream("", options, prefix, &stream); + ASSERT_TRUE(s.ok()) << s.ToString(); + + std::string input, plainTxt, cipherTxt; + uint64_t offset = prefixLen; + + input.assign("test ippcp crypto"); + plainTxt = input; + + s = stream->Encrypt(offset, (char *)input.c_str(), input.length()); // does in place encryption + ASSERT_TRUE(s.ok()) << s.ToString(); + cipherTxt = input; // encrypted txt + + provider.reset(); + s = EncryptionProvider::CreateFromString(ConfigOptions(), IPPCP, &provider); + ASSERT_TRUE(s.ok()) << s.ToString(); + ; + ASSERT_NE(provider, nullptr); + + // change the key + cipher_key.assign("a6d2ae2816157e2b"); + s = provider->AddCipher("", cipher_key.c_str(), cipher_key.length(), false); + ASSERT_TRUE(s.ok()) << s.ToString(); + s = stream->Decrypt(offset, (char *)cipherTxt.c_str(), input.length()); + ASSERT_TRUE(s.ok()) << s.ToString(); + ASSERT_NE(plainTxt, cipherTxt) << " both are strings are same after decryption!!"; + free(buf); + } + + struct TestParam + { + TestParam(std::string _cipher_desc, std::string _cipher_key, std::string _plainTxt = "") : cipher_desc(_cipher_desc), cipher_key(_cipher_key), plainTxt(_plainTxt) {} + + std::string cipher_desc; + std::string cipher_key; + std::string plainTxt; + std::string GetOpts() + { + return "cipher_desc = " + cipher_desc + "; cipher_key = " + cipher_key + "; cipher_size = " + std::to_string(cipher_key.length()) + "; plaintxt = " + plainTxt; + } + }; + + class IppcpProviderTest : public testing::TestWithParam> + { + public: + static void SetUpTestSuite() + { + ObjectLibrary::Default()->AddFactory( + IppcpProvider::kName(), + [](const std::string & /* uri */, std::unique_ptr *f, + std::string * /* errmsg */) + { + *f = IppcpProvider::CreateProvider(); + return f->get(); + }); + } + void SetUp() override + { + TestParam test_param(std::get<0>(GetParam()), std::get<1>(GetParam()), std::get<2>(GetParam())); + ConfigOptions config_options; + Status s = EncryptionProvider::CreateFromString(config_options, IppcpProvider::kName(), &provider); + } + std::shared_ptr provider; + const EnvOptions soptions_; + }; + + TEST_P(IppcpProviderTest, EncryptDecrypt) + { + TestParam test_param(std::get<0>(GetParam()), std::get<1>(GetParam()), std::get<2>(GetParam())); + Status s = provider->AddCipher(test_param.cipher_desc, (char *)test_param.cipher_key.c_str(), test_param.cipher_key.length(), false); + ASSERT_TRUE(s.ok()) << s.ToString(); + + size_t prefixLen = 16; // minimum size of prefix is 16(blockSize) + uint8_t ctr[] = {0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; + Slice prefix((char *)ctr, prefixLen); + + std::unique_ptr stream; + s = provider->CreateCipherStream("", soptions_, prefix, &stream); + ASSERT_TRUE(s.ok()) << s.ToString(); + + std::string input = test_param.plainTxt; + s = stream->Encrypt(0, (char *)input.c_str(), input.length()); + ASSERT_TRUE(s.ok()) << s.ToString(); + s = stream->Decrypt(0, (char *)input.c_str(), input.length()); + ASSERT_TRUE(s.ok()) << s.ToString(); + ASSERT_TRUE(test_param.plainTxt == input) << " both are strings are same after decryption!!"; + } + + // working but uses cartesian product + INSTANTIATE_TEST_SUITE_P(IppcpProviderTestInstance, + IppcpProviderTest, + testing::Combine(testing::Values("ippcp_test_aes"), // key description + testing::Values("a6d2ae2816157e2b3c4fcf098815f7xb", "a6d2ae2816157e2334512345", "a6d2ae2816157e23"), // encryption key // offset for encryption and decryption + testing::Values("Hello world", "Helloooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo worldddddddddddddddddddddddddddddddd 111111111111111111111111111111111111111111111111111111111111111"))); // plain text to encrypt + +} // end of namespace + +int main(int argc, char *argv[]) +{ + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/port/README b/librocksdb-sys/rocksdb/port/README new file mode 100644 index 0000000..422563e --- /dev/null +++ b/librocksdb-sys/rocksdb/port/README @@ -0,0 +1,10 @@ +This directory contains interfaces and implementations that isolate the +rest of the package from platform details. + +Code in the rest of the package includes "port.h" from this directory. +"port.h" in turn includes a platform specific "port_.h" file +that provides the platform specific implementation. + +See port_posix.h for an example of what must be provided in a platform +specific header file. + diff --git a/librocksdb-sys/rocksdb/port/jemalloc_helper.h b/librocksdb-sys/rocksdb/port/jemalloc_helper.h new file mode 100644 index 0000000..f085f62 --- /dev/null +++ b/librocksdb-sys/rocksdb/port/jemalloc_helper.h @@ -0,0 +1,107 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#if defined(__clang__) && defined(__GLIBC__) +// glibc's `posix_memalign()` declaration specifies `throw()` while clang's +// declaration does not. There is a hack in clang to make its re-declaration +// compatible with glibc's if they are declared consecutively. That hack breaks +// if yet another `posix_memalign()` declaration comes between glibc's and +// clang's declarations. Include "mm_malloc.h" here ensures glibc's and clang's +// declarations both come before "jemalloc.h"'s `posix_memalign()` declaration. +// +// This problem could also be avoided if "jemalloc.h"'s `posix_memalign()` +// declaration did not specify `throw()` when built with clang. +#include +#endif + +#ifdef ROCKSDB_JEMALLOC +#ifdef __FreeBSD__ +#include +#define JEMALLOC_USABLE_SIZE_CONST const +#else +#define JEMALLOC_MANGLE +#include +#endif + +#ifndef JEMALLOC_CXX_THROW +#define JEMALLOC_CXX_THROW +#endif + +#if defined(OS_WIN) && defined(_MSC_VER) + +// MSVC does not have weak symbol support. As long as ROCKSDB_JEMALLOC is +// defined, Jemalloc memory allocator is used. +static inline bool HasJemalloc() { return true; } + +#else + +// definitions for compatibility with older versions of jemalloc +#if !defined(JEMALLOC_ALLOCATOR) +#define JEMALLOC_ALLOCATOR +#endif +#if !defined(JEMALLOC_RESTRICT_RETURN) +#define JEMALLOC_RESTRICT_RETURN +#endif +#if !defined(JEMALLOC_NOTHROW) +#define JEMALLOC_NOTHROW JEMALLOC_ATTR(nothrow) +#endif +#if !defined(JEMALLOC_ALLOC_SIZE) +#ifdef JEMALLOC_HAVE_ATTR_ALLOC_SIZE +#define JEMALLOC_ALLOC_SIZE(s) JEMALLOC_ATTR(alloc_size(s)) +#else +#define JEMALLOC_ALLOC_SIZE(s) +#endif +#endif + +// Declare non-standard jemalloc APIs as weak symbols. We can null-check these +// symbols to detect whether jemalloc is linked with the binary. +extern "C" JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN void JEMALLOC_NOTHROW * +mallocx(size_t, int) JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) + __attribute__((__weak__)); +extern "C" JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN void JEMALLOC_NOTHROW * +rallocx(void *, size_t, int) JEMALLOC_ALLOC_SIZE(2) __attribute__((__weak__)); +extern "C" size_t JEMALLOC_NOTHROW xallocx(void *, size_t, size_t, int) + __attribute__((__weak__)); +extern "C" size_t JEMALLOC_NOTHROW sallocx(const void *, int) + JEMALLOC_ATTR(pure) __attribute__((__weak__)); +extern "C" void JEMALLOC_NOTHROW dallocx(void *, int) __attribute__((__weak__)); +extern "C" void JEMALLOC_NOTHROW sdallocx(void *, size_t, int) + __attribute__((__weak__)); +extern "C" size_t JEMALLOC_NOTHROW nallocx(size_t, int) JEMALLOC_ATTR(pure) + __attribute__((__weak__)); +extern "C" int JEMALLOC_NOTHROW mallctl(const char *, void *, size_t *, void *, + size_t) __attribute__((__weak__)); +extern "C" int JEMALLOC_NOTHROW mallctlnametomib(const char *, size_t *, + size_t *) + __attribute__((__weak__)); +extern "C" int JEMALLOC_NOTHROW mallctlbymib(const size_t *, size_t, void *, + size_t *, void *, size_t) + __attribute__((__weak__)); +extern "C" void JEMALLOC_NOTHROW +malloc_stats_print(void (*)(void *, const char *), void *, const char *) + __attribute__((__weak__)); +extern "C" size_t JEMALLOC_NOTHROW +malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *) JEMALLOC_CXX_THROW + __attribute__((__weak__)); + +// Check if Jemalloc is linked with the binary. Note the main program might be +// using a different memory allocator even this method return true. +// It is loosely based on folly::usingJEMalloc(), minus the check that actually +// allocate memory and see if it is through jemalloc, to handle the dlopen() +// case: +// https://github.com/facebook/folly/blob/76cf8b5841fb33137cfbf8b224f0226437c855bc/folly/memory/Malloc.h#L147 +static inline bool HasJemalloc() { + return mallocx != nullptr && rallocx != nullptr && xallocx != nullptr && + sallocx != nullptr && dallocx != nullptr && sdallocx != nullptr && + nallocx != nullptr && mallctl != nullptr && + mallctlnametomib != nullptr && mallctlbymib != nullptr && + malloc_stats_print != nullptr && malloc_usable_size != nullptr; +} + +#endif + +#endif // ROCKSDB_JEMALLOC diff --git a/librocksdb-sys/rocksdb/port/lang.h b/librocksdb-sys/rocksdb/port/lang.h new file mode 100644 index 0000000..a4201ca --- /dev/null +++ b/librocksdb-sys/rocksdb/port/lang.h @@ -0,0 +1,97 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#ifndef FALLTHROUGH_INTENDED +#if defined(__clang__) +#define FALLTHROUGH_INTENDED [[clang::fallthrough]] +#elif defined(__GNUC__) && __GNUC__ >= 7 +#define FALLTHROUGH_INTENDED [[gnu::fallthrough]] +#else +#define FALLTHROUGH_INTENDED \ + do { \ + } while (0) +#endif +#endif + +#define DECLARE_DEFAULT_MOVES(Name) \ + Name(Name&&) noexcept = default; \ + Name& operator=(Name&&) = default + +// ASAN (Address sanitizer) + +#if defined(__clang__) +#if defined(__has_feature) +#if __has_feature(address_sanitizer) +#define MUST_FREE_HEAP_ALLOCATIONS 1 +#endif // __has_feature(address_sanitizer) +#endif // defined(__has_feature) +#else // __clang__ +#ifdef __SANITIZE_ADDRESS__ +#define MUST_FREE_HEAP_ALLOCATIONS 1 +#endif // __SANITIZE_ADDRESS__ +#endif // __clang__ + +#ifdef ROCKSDB_VALGRIND_RUN +#define MUST_FREE_HEAP_ALLOCATIONS 1 +#endif // ROCKSDB_VALGRIND_RUN + +// Coding guidelines say to avoid static objects with non-trivial destructors, +// because it's easy to cause trouble (UB) in static destruction. This +// macro makes it easier to define static objects that are normally never +// destructed, except are destructed when running under ASAN. This should +// avoid unexpected, unnecessary destruction behavior in production. +// Note that constructor arguments can be provided as in +// STATIC_AVOID_DESTRUCTION(Foo, foo)(arg1, arg2); +#ifdef MUST_FREE_HEAP_ALLOCATIONS +#define STATIC_AVOID_DESTRUCTION(Type, name) static Type name +constexpr bool kMustFreeHeapAllocations = true; +#else +#define STATIC_AVOID_DESTRUCTION(Type, name) static Type& name = *new Type +constexpr bool kMustFreeHeapAllocations = false; +#endif + +// TSAN (Thread sanitizer) + +// For simplicity, standardize on the GCC define +#if defined(__clang__) +#if defined(__has_feature) && __has_feature(thread_sanitizer) +#define __SANITIZE_THREAD__ 1 +#endif // __has_feature(thread_sanitizer) +#endif // __clang__ + +#ifdef __SANITIZE_THREAD__ +#define TSAN_SUPPRESSION __attribute__((no_sanitize("thread"))) +#else +#define TSAN_SUPPRESSION +#endif // TSAN_SUPPRESSION + +// Compile-time CPU feature testing compatibility +// +// A way to be extra sure these defines have been included. +#define ASSERT_FEATURE_COMPAT_HEADER() /* empty */ + +// MSVC doesn't support the same defines that gcc and clang provide +// but does some like __AVX__. Here we can infer some features from others. +#ifdef __AVX__ +#define __SSE4_2__ 1 +#define __PCLMUL__ 1 +#endif // __AVX__ + +// A way to disable PCLMUL +#ifdef NO_PCLMUL +#undef __PCLMUL__ +#endif + +// popcnt is generally implied by SSE4.2 +#if defined(__SSE4_2__) +#define __POPCNT__ 1 +#endif + +// A way to disable POPCNT +#ifdef NO_POPCNT +#undef __POPCNT__ +#endif diff --git a/librocksdb-sys/rocksdb/port/likely.h b/librocksdb-sys/rocksdb/port/likely.h new file mode 100644 index 0000000..0bd90d7 --- /dev/null +++ b/librocksdb-sys/rocksdb/port/likely.h @@ -0,0 +1,18 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#if defined(__GNUC__) && __GNUC__ >= 4 +#define LIKELY(x) (__builtin_expect((x), 1)) +#define UNLIKELY(x) (__builtin_expect((x), 0)) +#else +#define LIKELY(x) (x) +#define UNLIKELY(x) (x) +#endif diff --git a/librocksdb-sys/rocksdb/port/malloc.h b/librocksdb-sys/rocksdb/port/malloc.h new file mode 100644 index 0000000..f973263 --- /dev/null +++ b/librocksdb-sys/rocksdb/port/malloc.h @@ -0,0 +1,17 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once + +#ifdef ROCKSDB_MALLOC_USABLE_SIZE +#ifdef OS_FREEBSD +#include +#else +#include +#endif // OS_FREEBSD +#endif // ROCKSDB_MALLOC_USABLE_SIZE diff --git a/librocksdb-sys/rocksdb/port/mmap.cc b/librocksdb-sys/rocksdb/port/mmap.cc new file mode 100644 index 0000000..36e8f32 --- /dev/null +++ b/librocksdb-sys/rocksdb/port/mmap.cc @@ -0,0 +1,98 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "port/mmap.h" + +#include +#include +#include +#include +#include + +#include "util/hash.h" + +namespace ROCKSDB_NAMESPACE { + +MemMapping::~MemMapping() { +#ifdef OS_WIN + if (addr_ != nullptr) { + (void)::UnmapViewOfFile(addr_); + } + if (page_file_handle_ != NULL) { + (void)::CloseHandle(page_file_handle_); + } +#else // OS_WIN -> !OS_WIN + if (addr_ != nullptr) { + auto status = munmap(addr_, length_); + assert(status == 0); + if (status != 0) { + // TODO: handle error? + } + } +#endif // OS_WIN +} + +MemMapping::MemMapping(MemMapping&& other) noexcept { + *this = std::move(other); +} + +MemMapping& MemMapping::operator=(MemMapping&& other) noexcept { + if (&other == this) { + return *this; + } + this->~MemMapping(); + std::memcpy(this, &other, sizeof(*this)); + new (&other) MemMapping(); + return *this; +} + +MemMapping MemMapping::AllocateAnonymous(size_t length, bool huge) { + MemMapping mm; + mm.length_ = length; + assert(mm.addr_ == nullptr); + if (length == 0) { + // OK to leave addr as nullptr + return mm; + } + int huge_flag = 0; +#ifdef OS_WIN + if (huge) { +#ifdef FILE_MAP_LARGE_PAGES + huge_flag = FILE_MAP_LARGE_PAGES; +#endif // FILE_MAP_LARGE_PAGES + } + mm.page_file_handle_ = ::CreateFileMapping( + INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE | SEC_COMMIT, + Upper32of64(length), Lower32of64(length), nullptr); + if (mm.page_file_handle_ == NULL) { + // Failure + return mm; + } + mm.addr_ = ::MapViewOfFile(mm.page_file_handle_, FILE_MAP_WRITE | huge_flag, + 0, 0, length); +#else // OS_WIN -> !OS_WIN + if (huge) { +#ifdef MAP_HUGETLB + huge_flag = MAP_HUGETLB; +#endif // MAP_HUGE_TLB + } + mm.addr_ = mmap(nullptr, length, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | huge_flag, -1, 0); + if (mm.addr_ == MAP_FAILED) { + mm.addr_ = nullptr; + } +#endif // OS_WIN + return mm; +} + +MemMapping MemMapping::AllocateHuge(size_t length) { + return AllocateAnonymous(length, /*huge*/ true); +} + +MemMapping MemMapping::AllocateLazyZeroed(size_t length) { + return AllocateAnonymous(length, /*huge*/ false); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/port/mmap.h b/librocksdb-sys/rocksdb/port/mmap.h new file mode 100644 index 0000000..7342a13 --- /dev/null +++ b/librocksdb-sys/rocksdb/port/mmap.h @@ -0,0 +1,70 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#ifdef OS_WIN +#include "port/win/port_win.h" +// ^^^ For proper/safe inclusion of windows.h. Must come first. +#include +#else +#include +#endif // OS_WIN + +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +// An RAII wrapper for mmaped memory +class MemMapping { + public: + static constexpr bool kHugePageSupported = +#if defined(MAP_HUGETLB) || defined(FILE_MAP_LARGE_PAGES) + true; +#else + false; +#endif + + // Allocate memory requesting to be backed by huge pages + static MemMapping AllocateHuge(size_t length); + + // Allocate memory that is only lazily mapped to resident memory and + // guaranteed to be zero-initialized. Note that some platforms like + // Linux allow memory over-commit, where only the used portion of memory + // matters, while other platforms require enough swap space (page file) to + // back the full mapping. + static MemMapping AllocateLazyZeroed(size_t length); + + // No copies + MemMapping(const MemMapping&) = delete; + MemMapping& operator=(const MemMapping&) = delete; + // Move + MemMapping(MemMapping&&) noexcept; + MemMapping& operator=(MemMapping&&) noexcept; + + // Releases the mapping + ~MemMapping(); + + inline void* Get() const { return addr_; } + inline size_t Length() const { return length_; } + + private: + MemMapping() {} + + // The mapped memory, or nullptr on failure / not supported + void* addr_ = nullptr; + // The known usable number of bytes starting at that address + size_t length_ = 0; + +#ifdef OS_WIN + HANDLE page_file_handle_ = NULL; +#endif // OS_WIN + + static MemMapping AllocateAnonymous(size_t length, bool huge); +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/port/port.h b/librocksdb-sys/rocksdb/port/port.h new file mode 100644 index 0000000..13aa56d --- /dev/null +++ b/librocksdb-sys/rocksdb/port/port.h @@ -0,0 +1,21 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include + +// Include the appropriate platform specific file below. If you are +// porting to a new platform, see "port_example.h" for documentation +// of what the new port_.h file must provide. +#if defined(ROCKSDB_PLATFORM_POSIX) +#include "port/port_posix.h" +#elif defined(OS_WIN) +#include "port/win/port_win.h" +#endif diff --git a/librocksdb-sys/rocksdb/port/port_dirent.h b/librocksdb-sys/rocksdb/port/port_dirent.h new file mode 100644 index 0000000..2b23e2f --- /dev/null +++ b/librocksdb-sys/rocksdb/port/port_dirent.h @@ -0,0 +1,44 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// See port_example.h for documentation for the following types/functions. + +#pragma once + +#ifdef ROCKSDB_PLATFORM_POSIX +#include +#include +#elif defined(OS_WIN) + +namespace ROCKSDB_NAMESPACE { +namespace port { + +struct dirent { + char d_name[_MAX_PATH]; /* filename */ +}; + +struct DIR; + +DIR* opendir(const char* name); + +dirent* readdir(DIR* dirp); + +int closedir(DIR* dirp); + +} // namespace port + +using port::closedir; +using port::DIR; +using port::dirent; +using port::opendir; +using port::readdir; + +} // namespace ROCKSDB_NAMESPACE + +#endif // OS_WIN diff --git a/librocksdb-sys/rocksdb/port/port_example.h b/librocksdb-sys/rocksdb/port/port_example.h new file mode 100644 index 0000000..794149a --- /dev/null +++ b/librocksdb-sys/rocksdb/port/port_example.h @@ -0,0 +1,101 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// This file contains the specification, but not the implementations, +// of the types/operations/etc. that should be defined by a platform +// specific port_.h file. Use this file as a reference for +// how to port this package to a new platform. + +#pragma once + +namespace ROCKSDB_NAMESPACE { +namespace port { + +// TODO(jorlow): Many of these belong more in the environment class rather than +// here. We should try moving them and see if it affects perf. + +// The following boolean constant must be true on a little-endian machine +// and false otherwise. +static const bool kLittleEndian = true /* or some other expression */; + +// ------------------ Threading ------------------- + +// A Mutex represents an exclusive lock. +class Mutex { + public: + Mutex(); + ~Mutex(); + + // Lock the mutex. Waits until other lockers have exited. + // Will deadlock if the mutex is already locked by this thread. + void Lock(); + + // Unlock the mutex. + // REQUIRES: This mutex was locked by this thread. + void Unlock(); + + // Optionally crash if this thread does not hold this mutex. + // The implementation must be fast, especially if NDEBUG is + // defined. The implementation is allowed to skip all checks. + void AssertHeld(); +}; + +class CondVar { + public: + explicit CondVar(Mutex* mu); + ~CondVar(); + + // Atomically release *mu and block on this condition variable until + // either a call to SignalAll(), or a call to Signal() that picks + // this thread to wakeup. + // REQUIRES: this thread holds *mu + void Wait(); + + // If there are some threads waiting, wake up at least one of them. + void Signal(); + + // Wake up all waiting threads. + void SignallAll(); +}; + +// Thread-safe initialization. +// Used as follows: +// static port::OnceType init_control = LEVELDB_ONCE_INIT; +// static void Initializer() { ... do something ...; } +// ... +// port::InitOnce(&init_control, &Initializer); +using OnceType = intptr_t; +#define LEVELDB_ONCE_INIT 0 +extern void InitOnce(port::OnceType*, void (*initializer)()); + +// ------------------ Compression ------------------- + +// Store the snappy compression of "input[0,input_length-1]" in *output. +// Returns false if snappy is not supported by this port. +extern bool Snappy_Compress(const char* input, size_t input_length, + std::string* output); + +// If input[0,input_length-1] looks like a valid snappy compressed +// buffer, store the size of the uncompressed data in *result and +// return true. Else return false. +extern bool Snappy_GetUncompressedLength(const char* input, size_t length, + size_t* result); + +// Attempt to snappy uncompress input[0,input_length-1] into *output. +// Returns true if successful, false if the input is invalid lightweight +// compressed data. +// +// REQUIRES: at least the first "n" bytes of output[] must be writable +// where "n" is the result of a successful call to +// Snappy_GetUncompressedLength. +extern bool Snappy_Uncompress(const char* input_data, size_t input_length, + char* output); + +} // namespace port +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/port/port_posix.cc b/librocksdb-sys/rocksdb/port/port_posix.cc new file mode 100644 index 0000000..3872293 --- /dev/null +++ b/librocksdb-sys/rocksdb/port/port_posix.cc @@ -0,0 +1,300 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#if !defined(OS_WIN) + +#include "port/port_posix.h" + +#include +#if defined(__i386__) || defined(__x86_64__) +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +// We want to give users opportunity to default all the mutexes to adaptive if +// not specified otherwise. This enables a quick way to conduct various +// performance related experiements. +// +// NB! Support for adaptive mutexes is turned on by definining +// ROCKSDB_PTHREAD_ADAPTIVE_MUTEX during the compilation. If you use RocksDB +// build environment then this happens automatically; otherwise it's up to the +// consumer to define the identifier. +#ifdef ROCKSDB_DEFAULT_TO_ADAPTIVE_MUTEX +extern const bool kDefaultToAdaptiveMutex = true; +#else +extern const bool kDefaultToAdaptiveMutex = false; +#endif + +namespace port { + +static int PthreadCall(const char* label, int result) { + if (result != 0 && result != ETIMEDOUT && result != EBUSY) { + fprintf(stderr, "pthread %s: %s\n", label, errnoStr(result).c_str()); + abort(); + } + return result; +} + +Mutex::Mutex(bool adaptive) { + (void)adaptive; +#ifdef ROCKSDB_PTHREAD_ADAPTIVE_MUTEX + if (!adaptive) { + PthreadCall("init mutex", pthread_mutex_init(&mu_, nullptr)); + } else { + pthread_mutexattr_t mutex_attr; + PthreadCall("init mutex attr", pthread_mutexattr_init(&mutex_attr)); + PthreadCall("set mutex attr", pthread_mutexattr_settype( + &mutex_attr, PTHREAD_MUTEX_ADAPTIVE_NP)); + PthreadCall("init mutex", pthread_mutex_init(&mu_, &mutex_attr)); + PthreadCall("destroy mutex attr", pthread_mutexattr_destroy(&mutex_attr)); + } +#else + PthreadCall("init mutex", pthread_mutex_init(&mu_, nullptr)); +#endif // ROCKSDB_PTHREAD_ADAPTIVE_MUTEX +} + +Mutex::~Mutex() { PthreadCall("destroy mutex", pthread_mutex_destroy(&mu_)); } + +void Mutex::Lock() { + PthreadCall("lock", pthread_mutex_lock(&mu_)); +#ifndef NDEBUG + locked_ = true; +#endif +} + +void Mutex::Unlock() { +#ifndef NDEBUG + locked_ = false; +#endif + PthreadCall("unlock", pthread_mutex_unlock(&mu_)); +} + +bool Mutex::TryLock() { + bool ret = PthreadCall("trylock", pthread_mutex_trylock(&mu_)) == 0; +#ifndef NDEBUG + if (ret) { + locked_ = true; + } +#endif + return ret; +} + +void Mutex::AssertHeld() { +#ifndef NDEBUG + assert(locked_); +#endif +} + +CondVar::CondVar(Mutex* mu) : mu_(mu) { + PthreadCall("init cv", pthread_cond_init(&cv_, nullptr)); +} + +CondVar::~CondVar() { PthreadCall("destroy cv", pthread_cond_destroy(&cv_)); } + +void CondVar::Wait() { +#ifndef NDEBUG + mu_->locked_ = false; +#endif + PthreadCall("wait", pthread_cond_wait(&cv_, &mu_->mu_)); +#ifndef NDEBUG + mu_->locked_ = true; +#endif +} + +bool CondVar::TimedWait(uint64_t abs_time_us) { + struct timespec ts; + ts.tv_sec = static_cast(abs_time_us / 1000000); + ts.tv_nsec = static_cast((abs_time_us % 1000000) * 1000); + +#ifndef NDEBUG + mu_->locked_ = false; +#endif + int err = pthread_cond_timedwait(&cv_, &mu_->mu_, &ts); +#ifndef NDEBUG + mu_->locked_ = true; +#endif + if (err == ETIMEDOUT) { + return true; + } + if (err != 0) { + PthreadCall("timedwait", err); + } + return false; +} + +void CondVar::Signal() { PthreadCall("signal", pthread_cond_signal(&cv_)); } + +void CondVar::SignalAll() { + PthreadCall("broadcast", pthread_cond_broadcast(&cv_)); +} + +RWMutex::RWMutex() { + PthreadCall("init mutex", pthread_rwlock_init(&mu_, nullptr)); +} + +RWMutex::~RWMutex() { + PthreadCall("destroy mutex", pthread_rwlock_destroy(&mu_)); +} + +void RWMutex::ReadLock() { + PthreadCall("read lock", pthread_rwlock_rdlock(&mu_)); +} + +void RWMutex::WriteLock() { + PthreadCall("write lock", pthread_rwlock_wrlock(&mu_)); +} + +void RWMutex::ReadUnlock() { + PthreadCall("read unlock", pthread_rwlock_unlock(&mu_)); +} + +void RWMutex::WriteUnlock() { + PthreadCall("write unlock", pthread_rwlock_unlock(&mu_)); +} + +int PhysicalCoreID() { +#if defined(ROCKSDB_SCHED_GETCPU_PRESENT) && defined(__x86_64__) && \ + (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 22)) + // sched_getcpu uses VDSO getcpu() syscall since 2.22. I believe Linux offers + // VDSO support only on x86_64. This is the fastest/preferred method if + // available. + int cpuno = sched_getcpu(); + if (cpuno < 0) { + return -1; + } + return cpuno; +#elif defined(__x86_64__) || defined(__i386__) + // clang/gcc both provide cpuid.h, which defines __get_cpuid(), for x86_64 and + // i386. + unsigned eax, ebx = 0, ecx, edx; + if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx)) { + return -1; + } + return ebx >> 24; +#else + // give up, the caller can generate a random number or something. + return -1; +#endif +} + +void InitOnce(OnceType* once, void (*initializer)()) { + PthreadCall("once", pthread_once(once, initializer)); +} + +void Crash(const std::string& srcfile, int srcline) { + fprintf(stdout, "Crashing at %s:%d\n", srcfile.c_str(), srcline); + fflush(stdout); + kill(getpid(), SIGTERM); +} + +int GetMaxOpenFiles() { +#if defined(RLIMIT_NOFILE) + struct rlimit no_files_limit; + if (getrlimit(RLIMIT_NOFILE, &no_files_limit) != 0) { + return -1; + } + // protect against overflow + if (static_cast(no_files_limit.rlim_cur) >= + static_cast(std::numeric_limits::max())) { + return std::numeric_limits::max(); + } + return static_cast(no_files_limit.rlim_cur); +#endif + return -1; +} + +void* cacheline_aligned_alloc(size_t size) { +#if __GNUC__ < 5 && defined(__SANITIZE_ADDRESS__) + return malloc(size); +#elif (_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600 || defined(__APPLE__)) + void* m; + errno = posix_memalign(&m, CACHE_LINE_SIZE, size); + return errno ? nullptr : m; +#else + return malloc(size); +#endif +} + +void cacheline_aligned_free(void* memblock) { free(memblock); } + +static size_t GetPageSize() { +#if defined(OS_LINUX) || defined(_SC_PAGESIZE) + long v = sysconf(_SC_PAGESIZE); + if (v >= 1024) { + return static_cast(v); + } +#endif + // Default assume 4KB + return 4U * 1024U; +} + +const size_t kPageSize = GetPageSize(); + +void SetCpuPriority(ThreadId id, CpuPriority priority) { +#ifdef OS_LINUX + sched_param param; + param.sched_priority = 0; + switch (priority) { + case CpuPriority::kHigh: + sched_setscheduler(id, SCHED_OTHER, ¶m); + setpriority(PRIO_PROCESS, id, -20); + break; + case CpuPriority::kNormal: + sched_setscheduler(id, SCHED_OTHER, ¶m); + setpriority(PRIO_PROCESS, id, 0); + break; + case CpuPriority::kLow: + sched_setscheduler(id, SCHED_OTHER, ¶m); + setpriority(PRIO_PROCESS, id, 19); + break; + case CpuPriority::kIdle: + sched_setscheduler(id, SCHED_IDLE, ¶m); + break; + default: + assert(false); + } +#else + (void)id; + (void)priority; +#endif +} + +int64_t GetProcessID() { return getpid(); } + +bool GenerateRfcUuid(std::string* output) { + output->clear(); + std::ifstream f("/proc/sys/kernel/random/uuid"); + std::getline(f, /*&*/ *output); + if (output->size() == 36) { + return true; + } else { + output->clear(); + return false; + } +} + +} // namespace port +} // namespace ROCKSDB_NAMESPACE + +#endif diff --git a/librocksdb-sys/rocksdb/port/port_posix.h b/librocksdb-sys/rocksdb/port/port_posix.h new file mode 100644 index 0000000..cdb256a --- /dev/null +++ b/librocksdb-sys/rocksdb/port/port_posix.h @@ -0,0 +1,243 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// See port_example.h for documentation for the following types/functions. + +#pragma once + +#include + +#include "rocksdb/port_defs.h" +#include "rocksdb/rocksdb_namespace.h" + +// size_t printf formatting named in the manner of C99 standard formatting +// strings such as PRIu64 +// in fact, we could use that one +#define ROCKSDB_PRIszt "zu" + +#define __declspec(S) + +#undef PLATFORM_IS_LITTLE_ENDIAN +#if defined(OS_MACOSX) +#include +#if defined(__DARWIN_LITTLE_ENDIAN) && defined(__DARWIN_BYTE_ORDER) +#define PLATFORM_IS_LITTLE_ENDIAN \ + (__DARWIN_BYTE_ORDER == __DARWIN_LITTLE_ENDIAN) +#endif +#elif defined(OS_SOLARIS) +#include +#ifdef _LITTLE_ENDIAN +#define PLATFORM_IS_LITTLE_ENDIAN true +#else +#define PLATFORM_IS_LITTLE_ENDIAN false +#endif +#include +#elif defined(OS_AIX) +#include +#include +#define PLATFORM_IS_LITTLE_ENDIAN (BYTE_ORDER == LITTLE_ENDIAN) +#include +#elif defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || \ + defined(OS_DRAGONFLYBSD) || defined(OS_ANDROID) +#include +#include +#define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN) +#else +#include +#endif +#include +#include +#include + +#include +#include + +#ifndef PLATFORM_IS_LITTLE_ENDIAN +#define PLATFORM_IS_LITTLE_ENDIAN (__BYTE_ORDER == __LITTLE_ENDIAN) +#endif + +#if defined(OS_MACOSX) || defined(OS_SOLARIS) || defined(OS_FREEBSD) || \ + defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) || \ + defined(OS_ANDROID) || defined(CYGWIN) || defined(OS_AIX) +// Use fread/fwrite/fflush on platforms without _unlocked variants +#define fread_unlocked fread +#define fwrite_unlocked fwrite +#define fflush_unlocked fflush +#endif + +#if defined(OS_MACOSX) || defined(OS_FREEBSD) || defined(OS_OPENBSD) || \ + defined(OS_DRAGONFLYBSD) +// Use fsync() on platforms without fdatasync() +#define fdatasync fsync +#endif + +#if defined(OS_ANDROID) && __ANDROID_API__ < 9 +// fdatasync() was only introduced in API level 9 on Android. Use fsync() +// when targeting older platforms. +#define fdatasync fsync +#endif + +namespace ROCKSDB_NAMESPACE { + +extern const bool kDefaultToAdaptiveMutex; + +namespace port { +constexpr bool kLittleEndian = PLATFORM_IS_LITTLE_ENDIAN; +#undef PLATFORM_IS_LITTLE_ENDIAN + +class CondVar; + +class Mutex { + public: + static const char* kName() { return "pthread_mutex_t"; } + + explicit Mutex(bool adaptive = kDefaultToAdaptiveMutex); + // No copying + Mutex(const Mutex&) = delete; + void operator=(const Mutex&) = delete; + + ~Mutex(); + + void Lock(); + void Unlock(); + + bool TryLock(); + + // this will assert if the mutex is not locked + // it does NOT verify that mutex is held by a calling thread + void AssertHeld(); + + // Also implement std Lockable + inline void lock() { Lock(); } + inline void unlock() { Unlock(); } + inline bool try_lock() { return TryLock(); } + + private: + friend class CondVar; + pthread_mutex_t mu_; +#ifndef NDEBUG + bool locked_ = false; +#endif +}; + +class RWMutex { + public: + RWMutex(); + // No copying allowed + RWMutex(const RWMutex&) = delete; + void operator=(const RWMutex&) = delete; + + ~RWMutex(); + + void ReadLock(); + void WriteLock(); + void ReadUnlock(); + void WriteUnlock(); + void AssertHeld() {} + + private: + pthread_rwlock_t mu_; // the underlying platform mutex +}; + +class CondVar { + public: + explicit CondVar(Mutex* mu); + ~CondVar(); + void Wait(); + // Timed condition wait. Returns true if timeout occurred. + bool TimedWait(uint64_t abs_time_us); + void Signal(); + void SignalAll(); + + private: + pthread_cond_t cv_; + Mutex* mu_; +}; + +using Thread = std::thread; + +static inline void AsmVolatilePause() { +#if defined(__i386__) || defined(__x86_64__) + asm volatile("pause"); +#elif defined(__aarch64__) + asm volatile("isb"); +#elif defined(__powerpc64__) + asm volatile("or 27,27,27"); +#elif defined(__loongarch64) + asm volatile("dbar 0"); +#endif + // it's okay for other platforms to be no-ops +} + +// Returns -1 if not available on this platform +extern int PhysicalCoreID(); + +using OnceType = pthread_once_t; +#define LEVELDB_ONCE_INIT PTHREAD_ONCE_INIT +extern void InitOnce(OnceType* once, void (*initializer)()); + +#ifndef CACHE_LINE_SIZE +// To test behavior with non-native cache line size, e.g. for +// Bloom filters, set TEST_CACHE_LINE_SIZE to the desired test size. +// This disables ALIGN_AS to keep it from failing compilation. +#ifdef TEST_CACHE_LINE_SIZE +#define CACHE_LINE_SIZE TEST_CACHE_LINE_SIZE +#define ALIGN_AS(n) /*empty*/ +#else +#if defined(__s390__) +#if defined(__GNUC__) && __GNUC__ < 7 +#define CACHE_LINE_SIZE 64U +#else +#define CACHE_LINE_SIZE 256U +#endif +#elif defined(__powerpc__) || defined(__aarch64__) +#define CACHE_LINE_SIZE 128U +#else +#define CACHE_LINE_SIZE 64U +#endif +#define ALIGN_AS(n) alignas(n) +#endif +#endif + +static_assert((CACHE_LINE_SIZE & (CACHE_LINE_SIZE - 1)) == 0, + "Cache line size must be a power of 2 number of bytes"); + +extern void* cacheline_aligned_alloc(size_t size); + +extern void cacheline_aligned_free(void* memblock); + +#if defined(__aarch64__) +// __builtin_prefetch(..., 1) turns into a prefetch into prfm pldl3keep. On +// arm64 we want this as close to the core as possible to turn it into a +// L1 prefetech unless locality == 0 in which case it will be turned into a +// non-temporal prefetch +#define PREFETCH(addr, rw, locality) \ + __builtin_prefetch(addr, rw, locality >= 1 ? 3 : locality) +#else +#define PREFETCH(addr, rw, locality) __builtin_prefetch(addr, rw, locality) +#endif + +extern void Crash(const std::string& srcfile, int srcline); + +extern int GetMaxOpenFiles(); + +extern const size_t kPageSize; + +using ThreadId = pid_t; + +extern void SetCpuPriority(ThreadId id, CpuPriority priority); + +int64_t GetProcessID(); + +// Uses platform APIs to generate a 36-character RFC-4122 UUID. Returns +// true on success or false on failure. +bool GenerateRfcUuid(std::string* output); + +} // namespace port +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/port/stack_trace.cc b/librocksdb-sys/rocksdb/port/stack_trace.cc new file mode 100644 index 0000000..1ccf9d8 --- /dev/null +++ b/librocksdb-sys/rocksdb/port/stack_trace.cc @@ -0,0 +1,351 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#include "port/stack_trace.h" + +#if !(defined(ROCKSDB_BACKTRACE) || defined(OS_MACOSX)) || defined(CYGWIN) || \ + defined(OS_SOLARIS) || defined(OS_WIN) + +// noop + +namespace ROCKSDB_NAMESPACE { +namespace port { +void InstallStackTraceHandler() {} +void PrintStack(int /*first_frames_to_skip*/) {} +void PrintAndFreeStack(void* /*callstack*/, int /*num_frames*/) {} +void* SaveStack(int* /*num_frames*/, int /*first_frames_to_skip*/) { + return nullptr; +} +} // namespace port +} // namespace ROCKSDB_NAMESPACE + +#else + +#include +#include +#include +#include +#include +#include +#include + +#ifdef OS_OPENBSD +#include +#include +#endif // OS_OPENBSD +#ifdef OS_FREEBSD +#include +#endif // OS_FREEBSD +#ifdef OS_LINUX +#include +#include +#include +#if __GLIBC__ < 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ < 30) +#include +#define gettid() syscall(SYS_gettid) +#endif // GLIBC version +#endif // OS_LINUX + +#include "port/lang.h" + +namespace ROCKSDB_NAMESPACE { +namespace port { + +namespace { + +#if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_GNU_KFREEBSD) +const char* GetExecutableName() { + static char name[1024]; + +#if defined(OS_FREEBSD) + int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1}; + size_t namesz = sizeof(name); + + auto ret = sysctl(mib, 4, name, &namesz, nullptr, 0); + if (-1 == ret) { + return nullptr; + } else { + return name; + } +#elif defined(OS_OPENBSD) + int mib[4] = {CTL_KERN, KERN_PROC_ARGS, getpid(), KERN_PROC_ARGV}; + size_t namesz = sizeof(name); + char* bin[namesz]; + + auto ret = sysctl(mib, 4, bin, &namesz, nullptr, 0); + if (-1 == ret) { + return nullptr; + } else { + return bin[0]; + } +#else + char link[1024]; + snprintf(link, sizeof(link), "/proc/%d/exe", getpid()); + auto read = readlink(link, name, sizeof(name) - 1); + if (-1 == read) { + return nullptr; + } else { + name[read] = 0; + return name; + } +#endif +} + +void PrintStackTraceLine(const char* symbol, void* frame) { + static const char* executable = GetExecutableName(); + if (symbol) { + fprintf(stderr, "%s ", symbol); + } + if (executable) { + // out source to addr2line, for the address translation + const int kLineMax = 256; + char cmd[kLineMax]; + snprintf(cmd, kLineMax, "addr2line %p -e %s -f -C 2>&1", frame, executable); + auto f = popen(cmd, "r"); + if (f) { + char line[kLineMax]; + while (fgets(line, sizeof(line), f)) { + line[strlen(line) - 1] = 0; // remove newline + fprintf(stderr, "%s\t", line); + } + pclose(f); + } + } else { + fprintf(stderr, " %p", frame); + } + + fprintf(stderr, "\n"); +} +#elif defined(OS_MACOSX) + +void PrintStackTraceLine(const char* symbol, void* frame) { + static int pid = getpid(); + // out source to atos, for the address translation + const int kLineMax = 256; + char cmd[kLineMax]; + snprintf(cmd, kLineMax, "xcrun atos %p -p %d 2>&1", frame, pid); + auto f = popen(cmd, "r"); + if (f) { + char line[kLineMax]; + while (fgets(line, sizeof(line), f)) { + line[strlen(line) - 1] = 0; // remove newline + fprintf(stderr, "%s\t", line); + } + pclose(f); + } else if (symbol) { + fprintf(stderr, "%s ", symbol); + } + + fprintf(stderr, "\n"); +} + +#endif + +const char* GetLldbScriptSelectThread(long long tid) { + // NOTE: called from a signal handler, so no heap allocation + static char script[80]; + snprintf(script, sizeof(script), + "script -l python -- lldb.process.SetSelectedThreadByID(%lld)", tid); + return script; +} + +} // namespace + +void PrintStack(void* frames[], int num_frames) { + auto symbols = backtrace_symbols(frames, num_frames); + + for (int i = 0; i < num_frames; ++i) { + fprintf(stderr, "#%-2d ", i); + PrintStackTraceLine((symbols != nullptr) ? symbols[i] : nullptr, frames[i]); + } + free(symbols); +} + +void PrintStack(int first_frames_to_skip) { + // Default to getting stack traces with GDB, at least on Linux where we + // know how to attach to a particular thread. + // + // * Address space layout randomization (ASLR) interferes with getting good + // stack information from backtrace+addr2line. This is more likely to show + // up with LIB_MODE=shared builds (when kernel.randomize_va_space >= 1) + // but can also show up with LIB_MODE=static builds ((when + // kernel.randomize_va_space == 2). + // * It doesn't appear easy to detect when ASLR is in use. + // * With DEBUG_LEVEL < 2, backtrace() can skip frames that are not skipped + // in GDB. + // + // LLDB also available as an option + bool lldb_stack_trace = getenv("ROCKSDB_LLDB_STACK") != nullptr; +#if defined(OS_LINUX) + // Default true, override with ROCKSDB_BACKTRACE_STACK=1 + bool gdb_stack_trace = + !lldb_stack_trace && getenv("ROCKSDB_BACKTRACE_STACK") == nullptr; +#else + // Default false, override with ROCKSDB_GDB_STACK=1 + bool gdb_stack_trace = getenv("ROCKSDB_GDB_STACK") != nullptr; +#endif + // Also support invoking interactive debugger on stack trace, with this + // envvar set to non-empty + char* debug_env = getenv("ROCKSDB_DEBUG"); + bool debug = debug_env != nullptr && strlen(debug_env) > 0; + + if (lldb_stack_trace || gdb_stack_trace || debug) { + // Allow ouside debugger to attach, even with Yama security restrictions +#ifdef PR_SET_PTRACER_ANY + (void)prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0); +#endif + // Try to invoke GDB, either for stack trace or debugging. + long long attach_pid = getpid(); + // NOTE: we're in a signal handler, so no heap allocation + char attach_pid_str[20]; + snprintf(attach_pid_str, sizeof(attach_pid_str), "%lld", attach_pid); + + // `gdb -p PID` seems to always attach to main thread, but `gdb -p TID` + // seems to be able to attach to a particular thread in a process, which + // makes sense as the main thread TID == PID of the process. + // But I haven't found that gdb capability documented anywhere, so leave + // a back door to attach to main thread. + long long gdb_attach_id = attach_pid; + // Save current thread id before fork + long long attach_tid = 0; +#ifdef OS_LINUX + attach_tid = gettid(); + if (getenv("ROCKSDB_DEBUG_USE_PID") == nullptr) { + gdb_attach_id = attach_tid; + } +#endif + + char gdb_attach_id_str[20]; + snprintf(gdb_attach_id_str, sizeof(gdb_attach_id_str), "%lld", + gdb_attach_id); + + pid_t child_pid = fork(); + if (child_pid == 0) { + // child process + if (debug) { + if (strcmp(debug_env, "lldb") == 0) { + fprintf(stderr, "Invoking LLDB for debugging (ROCKSDB_DEBUG=%s)...\n", + debug_env); + execlp(/*cmd in PATH*/ "lldb", /*arg0*/ "lldb", "-p", attach_pid_str, + /*"-Q",*/ "-o", GetLldbScriptSelectThread(attach_tid), + (char*)nullptr); + return; + } else { + fprintf(stderr, "Invoking GDB for debugging (ROCKSDB_DEBUG=%s)...\n", + debug_env); + execlp(/*cmd in PATH*/ "gdb", /*arg0*/ "gdb", "-p", gdb_attach_id_str, + (char*)nullptr); + return; + } + } else { + // Redirect child stdout to original stderr + dup2(2, 1); + // No child stdin (don't use pager) + close(0); + if (lldb_stack_trace) { + fprintf(stderr, "Invoking LLDB for stack trace...\n"); + + // Skip top ~8 frames here in PrintStack + auto bt_in_lldb = + "script -l python -- for f in lldb.thread.frames[8:]: print(f)"; + execlp(/*cmd in PATH*/ "lldb", /*arg0*/ "lldb", "-p", attach_pid_str, + "-b", "-Q", "-o", GetLldbScriptSelectThread(attach_tid), "-o", + bt_in_lldb, (char*)nullptr); + } else { + // gdb_stack_trace + fprintf(stderr, "Invoking GDB for stack trace...\n"); + + // Skip top ~4 frames here in PrintStack + // See https://stackoverflow.com/q/40991943/454544 + auto bt_in_gdb = + "frame apply level 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 " + "21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 " + "42 43 44 -q frame"; + // -n : Loading config files can apparently cause failures with the + // other options here. + // -batch : non-interactive; suppress banners as much as possible + execlp(/*cmd in PATH*/ "gdb", /*arg0*/ "gdb", "-n", "-batch", "-p", + gdb_attach_id_str, "-ex", bt_in_gdb, (char*)nullptr); + } + return; + } + } else { + // parent process; wait for child + int wstatus; + waitpid(child_pid, &wstatus, 0); + if (WIFEXITED(wstatus) && WEXITSTATUS(wstatus) == EXIT_SUCCESS) { + // Good + return; + } + } + fprintf(stderr, "GDB failed; falling back on backtrace+addr2line...\n"); + } + + const int kMaxFrames = 100; + void* frames[kMaxFrames]; + + int num_frames = (int) backtrace(frames, kMaxFrames); + PrintStack(&frames[first_frames_to_skip], num_frames - first_frames_to_skip); +} + +void PrintAndFreeStack(void* callstack, int num_frames) { + PrintStack(static_cast(callstack), num_frames); + free(callstack); +} + +void* SaveStack(int* num_frames, int first_frames_to_skip) { + const int kMaxFrames = 100; + void* frames[kMaxFrames]; + + int count = (int) backtrace(frames, kMaxFrames); + *num_frames = count - first_frames_to_skip; + void* callstack = malloc(sizeof(void*) * *num_frames); + memcpy(callstack, &frames[first_frames_to_skip], sizeof(void*) * *num_frames); + return callstack; +} + +static void StackTraceHandler(int sig) { + // reset to default handler + signal(sig, SIG_DFL); + fprintf(stderr, "Received signal %d (%s)\n", sig, strsignal(sig)); + // skip the top three signal handler related frames + PrintStack(3); + + // Efforts to fix or suppress TSAN warnings "signal-unsafe call inside of + // a signal" have failed, so just warn the user about them. +#ifdef __SANITIZE_THREAD__ + fprintf(stderr, + "==> NOTE: any above warnings about \"signal-unsafe call\" are\n" + "==> ignorable, as they are expected when generating a stack\n" + "==> trace because of a signal under TSAN. Consider why the\n" + "==> signal was generated to begin with, and the stack trace\n" + "==> in the TSAN warning can be useful for that. (The stack\n" + "==> trace printed by the signal handler is likely obscured\n" + "==> by TSAN output.)\n"); +#endif + + // re-signal to default handler (so we still get core dump if needed...) + raise(sig); +} + +void InstallStackTraceHandler() { + // just use the plain old signal as it's simple and sufficient + // for this use case + signal(SIGILL, StackTraceHandler); + signal(SIGSEGV, StackTraceHandler); + signal(SIGBUS, StackTraceHandler); + signal(SIGABRT, StackTraceHandler); + // Allow ouside debugger to attach, even with Yama security restrictions. + // This is needed even outside of PrintStack() so that external mechanisms + // can dump stacks if they suspect that a test has hung. +#ifdef PR_SET_PTRACER_ANY + (void)prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0); +#endif +} + +} // namespace port +} // namespace ROCKSDB_NAMESPACE + +#endif diff --git a/librocksdb-sys/rocksdb/port/stack_trace.h b/librocksdb-sys/rocksdb/port/stack_trace.h new file mode 100644 index 0000000..5b3bf93 --- /dev/null +++ b/librocksdb-sys/rocksdb/port/stack_trace.h @@ -0,0 +1,31 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#pragma once + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { +namespace port { + +// Install a signal handler to print callstack on the following signals: +// SIGILL SIGSEGV SIGBUS SIGABRT +// And also (Linux ony for now) overrides security settings to allow outside +// processes to attach to this one as a debugger. ONLY USE FOR NON-SECURITY +// CRITICAL PROCESSES such as unit tests or benchmarking tools. +// Currently supports only some POSIX implementations. No-op otherwise. +void InstallStackTraceHandler(); + +// Prints stack, skips skip_first_frames frames +void PrintStack(int first_frames_to_skip = 0); + +// Prints the given callstack +void PrintAndFreeStack(void* callstack, int num_frames); + +// Save the current callstack +void* SaveStack(int* num_frame, int first_frames_to_skip = 0); + +} // namespace port +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/port/sys_time.h b/librocksdb-sys/rocksdb/port/sys_time.h new file mode 100644 index 0000000..f213752 --- /dev/null +++ b/librocksdb-sys/rocksdb/port/sys_time.h @@ -0,0 +1,63 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +// This file is a portable substitute for sys/time.h which does not exist on +// Windows + +#pragma once + +#include "rocksdb/rocksdb_namespace.h" + +#if defined(OS_WIN) && (defined(_MSC_VER) || defined(__MINGW32__)) + +#include + +namespace ROCKSDB_NAMESPACE { + +namespace port { + +struct TimeVal { + long tv_sec; + long tv_usec; +}; + +void GetTimeOfDay(TimeVal* tv, struct timezone* tz); + +inline struct tm* LocalTimeR(const time_t* timep, struct tm* result) { + errno_t ret = localtime_s(result, timep); + return (ret == 0) ? result : NULL; +} + +} // namespace port + +} // namespace ROCKSDB_NAMESPACE + +#else +#include +#include + +namespace ROCKSDB_NAMESPACE { + +namespace port { + +using TimeVal = struct timeval; + +inline void GetTimeOfDay(TimeVal* tv, struct timezone* tz) { + gettimeofday(tv, tz); +} + +inline struct tm* LocalTimeR(const time_t* timep, struct tm* result) { + return localtime_r(timep, result); +} + +} // namespace port + +} // namespace ROCKSDB_NAMESPACE + +#endif diff --git a/librocksdb-sys/rocksdb/port/util_logger.h b/librocksdb-sys/rocksdb/port/util_logger.h new file mode 100644 index 0000000..ce7e3a9 --- /dev/null +++ b/librocksdb-sys/rocksdb/port/util_logger.h @@ -0,0 +1,18 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +// Include the appropriate platform specific file below. If you are +// porting to a new platform, see "port_example.h" for documentation +// of what the new port_.h file must provide. + +#if defined(OS_WIN) +#include "port/win/win_logger.h" +#endif diff --git a/librocksdb-sys/rocksdb/port/win/env_default.cc b/librocksdb-sys/rocksdb/port/win/env_default.cc new file mode 100644 index 0000000..48853f2 --- /dev/null +++ b/librocksdb-sys/rocksdb/port/win/env_default.cc @@ -0,0 +1,45 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#if defined(OS_WIN) + +#include + +#include "port/win/env_win.h" +#include "rocksdb/env.h" +#include "test_util/sync_point.h" +#include "util/compression_context_cache.h" +#include "util/thread_local.h" + +namespace ROCKSDB_NAMESPACE { +namespace port { + +// We choose not to destroy the env because joining the threads from the +// system loader +// which destroys the statics (same as from DLLMain) creates a system loader +// dead-lock. +// in this manner any remaining threads are terminated OK. +namespace { +std::once_flag winenv_once_flag; +Env* envptr; +}; // namespace +} // namespace port + +Env* Env::Default() { + ThreadLocalPtr::InitSingletons(); + CompressionContextCache::InitSingleton(); + INIT_SYNC_POINT_SINGLETONS(); + std::call_once(port::winenv_once_flag, + []() { port::envptr = new port::WinEnv(); }); + return port::envptr; +} + +} // namespace ROCKSDB_NAMESPACE + +#endif diff --git a/librocksdb-sys/rocksdb/port/win/env_win.cc b/librocksdb-sys/rocksdb/port/win/env_win.cc new file mode 100644 index 0000000..2262eb5 --- /dev/null +++ b/librocksdb-sys/rocksdb/port/win/env_win.cc @@ -0,0 +1,1437 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#if defined(OS_WIN) + +#include "port/win/env_win.h" + +#include // _rmdir, _mkdir, _getcwd +#include +#include // _access +#include // for uuid generation +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "monitoring/iostats_context_imp.h" +#include "monitoring/thread_status_updater.h" +#include "monitoring/thread_status_util.h" +#include "port/lang.h" +#include "port/port.h" +#include "port/port_dirent.h" +#include "port/win/io_win.h" +#include "port/win/win_logger.h" +#include "rocksdb/env.h" +#include "rocksdb/slice.h" +#include "strsafe.h" +#include "util/string_util.h" + +// Undefine the functions windows might use (again)... +#undef GetCurrentTime +#undef DeleteFile +#undef LoadLibrary + +namespace ROCKSDB_NAMESPACE { + +ThreadStatusUpdater* CreateThreadStatusUpdater() { + return new ThreadStatusUpdater(); +} + +namespace { + +// Sector size used when physical sector size cannot be obtained from device. +static const size_t kSectorSize = 512; + +// RAII helpers for HANDLEs +const auto CloseHandleFunc = [](HANDLE h) { ::CloseHandle(h); }; +using UniqueCloseHandlePtr = std::unique_ptr; + +const auto FindCloseFunc = [](HANDLE h) { ::FindClose(h); }; +using UniqueFindClosePtr = std::unique_ptr; + +void WinthreadCall(const char* label, std::error_code result) { + if (0 != result.value()) { + fprintf(stderr, "Winthread %s: %s\n", label, + errnoStr(result.value()).c_str()); + abort(); + } +} + +} // namespace + +namespace port { +WinClock::WinClock() + : perf_counter_frequency_(0), + nano_seconds_per_period_(0), + GetSystemTimePreciseAsFileTime_(NULL) { + { + LARGE_INTEGER qpf; + BOOL ret __attribute__((__unused__)); + ret = QueryPerformanceFrequency(&qpf); + assert(ret == TRUE); + perf_counter_frequency_ = qpf.QuadPart; + + if (std::nano::den % perf_counter_frequency_ == 0) { + nano_seconds_per_period_ = std::nano::den / perf_counter_frequency_; + } + } + + HMODULE module = GetModuleHandle("kernel32.dll"); + if (module != NULL) { + GetSystemTimePreciseAsFileTime_ = (FnGetSystemTimePreciseAsFileTime)( + void*)GetProcAddress(module, "GetSystemTimePreciseAsFileTime"); + } +} + +void WinClock::SleepForMicroseconds(int micros) { + std::this_thread::sleep_for(std::chrono::microseconds(micros)); +} + +std::string WinClock::TimeToString(uint64_t secondsSince1970) { + std::string result; + + const time_t seconds = secondsSince1970; + const int maxsize = 64; + + struct tm t; + errno_t ret = localtime_s(&t, &seconds); + + if (ret) { + result = std::to_string(seconds); + } else { + result.resize(maxsize); + char* p = &result[0]; + + int len = + snprintf(p, maxsize, "%04d/%02d/%02d-%02d:%02d:%02d ", t.tm_year + 1900, + t.tm_mon + 1, t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec); + assert(len > 0); + + result.resize(len); + } + + return result; +} + +uint64_t WinClock::NowMicros() { + if (GetSystemTimePreciseAsFileTime_ != NULL) { + // all std::chrono clocks on windows proved to return + // values that may repeat that is not good enough for some uses. + const int64_t c_UnixEpochStartTicks = 116444736000000000LL; + const int64_t c_FtToMicroSec = 10; + + // This interface needs to return system time and not + // just any microseconds because it is often used as an argument + // to TimedWait() on condition variable + FILETIME ftSystemTime; + GetSystemTimePreciseAsFileTime_(&ftSystemTime); + + LARGE_INTEGER li; + li.LowPart = ftSystemTime.dwLowDateTime; + li.HighPart = ftSystemTime.dwHighDateTime; + // Subtract unix epoch start + li.QuadPart -= c_UnixEpochStartTicks; + // Convert to microsecs + li.QuadPart /= c_FtToMicroSec; + return li.QuadPart; + } + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); +} + +uint64_t WinClock::NowNanos() { + if (nano_seconds_per_period_ != 0) { + // all std::chrono clocks on windows have the same resolution that is only + // good enough for microseconds but not nanoseconds + // On Windows 8 and Windows 2012 Server + // GetSystemTimePreciseAsFileTime(¤t_time) can be used + LARGE_INTEGER li; + QueryPerformanceCounter(&li); + // Convert performance counter to nanoseconds by precomputed ratio. + // Directly multiply nano::den with li.QuadPart causes overflow. + // Only do this when nano::den is divisible by perf_counter_frequency_, + // which most likely is the case in reality. If it's not, fall back to + // high_resolution_clock, which may be less precise under old compilers. + li.QuadPart *= nano_seconds_per_period_; + return li.QuadPart; + } + return std::chrono::duration_cast( + std::chrono::high_resolution_clock::now().time_since_epoch()) + .count(); +} + +Status WinClock::GetCurrentTime(int64_t* unix_time) { + time_t time = std::time(nullptr); + if (time == (time_t)(-1)) { + return Status::NotSupported("Failed to get time"); + } + + *unix_time = time; + return Status::OK(); +} + +WinFileSystem::WinFileSystem(const std::shared_ptr& clock) + : clock_(clock), page_size_(4 * 1024), allocation_granularity_(page_size_) { + SYSTEM_INFO sinfo; + GetSystemInfo(&sinfo); + + page_size_ = sinfo.dwPageSize; + allocation_granularity_ = sinfo.dwAllocationGranularity; +} + +const std::shared_ptr& WinFileSystem::Default() { + STATIC_AVOID_DESTRUCTION(std::shared_ptr, fs) + (std::make_shared(WinClock::Default())); + return fs; +} + +WinEnvIO::WinEnvIO(Env* hosted_env) : hosted_env_(hosted_env) {} + +WinEnvIO::~WinEnvIO() {} + +IOStatus WinFileSystem::DeleteFile(const std::string& fname, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + IOStatus result; + + BOOL ret = RX_DeleteFile(RX_FN(fname).c_str()); + + if (!ret) { + auto lastError = GetLastError(); + result = IOErrorFromWindowsError("Failed to delete: " + fname, lastError); + } + + return result; +} + +IOStatus WinFileSystem::Truncate(const std::string& fname, size_t size, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + IOStatus s; + int result = ROCKSDB_NAMESPACE::port::Truncate(fname, size); + if (result != 0) { + s = IOError("Failed to truncate: " + fname, errno); + } + return s; +} + +IOStatus WinFileSystem::NewSequentialFile( + const std::string& fname, const FileOptions& options, + std::unique_ptr* result, IODebugContext* /*dbg*/) { + IOStatus s; + + result->reset(); + + // Corruption test needs to rename and delete files of these kind + // while they are still open with another handle. For that reason we + // allow share_write and delete(allows rename). + HANDLE hFile = INVALID_HANDLE_VALUE; + + DWORD fileFlags = FILE_ATTRIBUTE_READONLY; + + if (options.use_direct_reads && !options.use_mmap_reads) { + fileFlags |= FILE_FLAG_NO_BUFFERING; + } + + { + IOSTATS_TIMER_GUARD(open_nanos); + hFile = RX_CreateFile( + RX_FN(fname).c_str(), GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, // Original fopen mode is "rb" + fileFlags, NULL); + } + + if (INVALID_HANDLE_VALUE == hFile) { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError("Failed to open NewSequentialFile" + fname, + lastError); + } else { + result->reset(new WinSequentialFile(fname, hFile, options)); + } + return s; +} + +IOStatus WinFileSystem::NewRandomAccessFile( + const std::string& fname, const FileOptions& options, + std::unique_ptr* result, IODebugContext* dbg) { + result->reset(); + IOStatus s; + + // Open the file for read-only random access + // Random access is to disable read-ahead as the system reads too much data + DWORD fileFlags = FILE_ATTRIBUTE_READONLY; + + if (options.use_direct_reads && !options.use_mmap_reads) { + fileFlags |= FILE_FLAG_NO_BUFFERING; + } else { + fileFlags |= FILE_FLAG_RANDOM_ACCESS; + } + + /// Shared access is necessary for corruption test to pass + // almost all tests would work with a possible exception of fault_injection + HANDLE hFile = 0; + { + IOSTATS_TIMER_GUARD(open_nanos); + hFile = + RX_CreateFile(RX_FN(fname).c_str(), GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, OPEN_EXISTING, fileFlags, NULL); + } + + if (INVALID_HANDLE_VALUE == hFile) { + auto lastError = GetLastError(); + return IOErrorFromWindowsError( + "NewRandomAccessFile failed to Create/Open: " + fname, lastError); + } + + UniqueCloseHandlePtr fileGuard(hFile, CloseHandleFunc); + + // CAUTION! This will map the entire file into the process address space. + // Not recommended for 32-bit platforms. + if (options.use_mmap_reads) { + uint64_t fileSize; + + s = GetFileSize(fname, IOOptions(), &fileSize, dbg); + + if (s.ok()) { + // Will not map empty files + if (fileSize == 0) { + return IOError("NewRandomAccessFile failed to map empty file: " + fname, + EINVAL); + } + + HANDLE hMap = RX_CreateFileMapping(hFile, NULL, PAGE_READONLY, + 0, // At its present length + 0, + NULL); // Mapping name + + if (!hMap) { + auto lastError = GetLastError(); + return IOErrorFromWindowsError( + "Failed to create file mapping for NewRandomAccessFile: " + fname, + lastError); + } + + UniqueCloseHandlePtr mapGuard(hMap, CloseHandleFunc); + + const void* mapped_region = + MapViewOfFileEx(hMap, FILE_MAP_READ, + 0, // High DWORD of access start + 0, // Low DWORD + static_cast(fileSize), + NULL); // Let the OS choose the mapping + + if (!mapped_region) { + auto lastError = GetLastError(); + return IOErrorFromWindowsError( + "Failed to MapViewOfFile for NewRandomAccessFile: " + fname, + lastError); + } + + result->reset(new WinMmapReadableFile(fname, hFile, hMap, mapped_region, + static_cast(fileSize))); + + mapGuard.release(); + fileGuard.release(); + } + } else { + result->reset(new WinRandomAccessFile(fname, hFile, page_size_, options)); + fileGuard.release(); + } + return s; +} + +IOStatus WinFileSystem::OpenWritableFile( + const std::string& fname, const FileOptions& options, + std::unique_ptr* result, bool reopen) { + const size_t c_BufferCapacity = 64 * 1024; + + EnvOptions local_options(options); + + result->reset(); + IOStatus s; + + DWORD fileFlags = FILE_ATTRIBUTE_NORMAL; + + if (local_options.use_direct_writes && !local_options.use_mmap_writes) { + fileFlags = FILE_FLAG_NO_BUFFERING | FILE_FLAG_WRITE_THROUGH; + } + + // Desired access. We are want to write only here but if we want to memory + // map + // the file then there is no write only mode so we have to create it + // Read/Write + // However, MapViewOfFile specifies only Write only + DWORD desired_access = GENERIC_WRITE; + DWORD shared_mode = FILE_SHARE_READ; + + if (local_options.use_mmap_writes) { + desired_access |= GENERIC_READ; + } else { + // Adding this solely for tests to pass (fault_injection_test, + // wal_manager_test). + shared_mode |= (FILE_SHARE_WRITE | FILE_SHARE_DELETE); + } + + // This will always truncate the file + DWORD creation_disposition = CREATE_ALWAYS; + if (reopen) { + creation_disposition = OPEN_ALWAYS; + } + + HANDLE hFile = 0; + { + IOSTATS_TIMER_GUARD(open_nanos); + hFile = RX_CreateFile( + RX_FN(fname).c_str(), + desired_access, // Access desired + shared_mode, + NULL, // Security attributes + // Posix env says (reopen) ? (O_CREATE | O_APPEND) : O_CREAT | O_TRUNC + creation_disposition, + fileFlags, // Flags + NULL); // Template File + } + + if (INVALID_HANDLE_VALUE == hFile) { + auto lastError = GetLastError(); + return IOErrorFromWindowsError( + "Failed to create a NewWritableFile: " + fname, lastError); + } + + // We will start writing at the end, appending + if (reopen) { + LARGE_INTEGER zero_move; + zero_move.QuadPart = 0; + BOOL ret = SetFilePointerEx(hFile, zero_move, NULL, FILE_END); + if (!ret) { + auto lastError = GetLastError(); + return IOErrorFromWindowsError( + "Failed to create a ReopenWritableFile move to the end: " + fname, + lastError); + } + } + + if (options.use_mmap_writes) { + // We usually do not use mmmapping on SSD and thus we pass memory + // page_size + result->reset(new WinMmapFile(fname, hFile, page_size_, + allocation_granularity_, local_options)); + } else { + // Here we want the buffer allocation to be aligned by the SSD page size + // and to be a multiple of it + result->reset(new WinWritableFile(fname, hFile, GetPageSize(), + c_BufferCapacity, local_options)); + } + return s; +} + +IOStatus WinFileSystem::NewWritableFile(const std::string& fname, + const FileOptions& options, + std::unique_ptr* result, + IODebugContext* /*dbg*/) { + return OpenWritableFile(fname, options, result, false); +} + +IOStatus WinFileSystem::ReopenWritableFile( + const std::string& fname, const FileOptions& options, + std::unique_ptr* result, IODebugContext* /*dbg*/) { + return OpenWritableFile(fname, options, result, true); +} + +IOStatus WinFileSystem::NewRandomRWFile(const std::string& fname, + const FileOptions& options, + std::unique_ptr* result, + IODebugContext* /*dbg*/) { + IOStatus s; + + // Open the file for read-only random access + // Random access is to disable read-ahead as the system reads too much data + DWORD desired_access = GENERIC_READ | GENERIC_WRITE; + DWORD shared_mode = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE; + DWORD creation_disposition = OPEN_EXISTING; // Fail if file does not exist + DWORD file_flags = FILE_FLAG_RANDOM_ACCESS; + + if (options.use_direct_reads && options.use_direct_writes) { + file_flags |= FILE_FLAG_NO_BUFFERING; + } + + /// Shared access is necessary for corruption test to pass + // almost all tests would work with a possible exception of fault_injection + HANDLE hFile = 0; + { + IOSTATS_TIMER_GUARD(open_nanos); + hFile = RX_CreateFile(RX_FN(fname).c_str(), desired_access, shared_mode, + NULL, // Security attributes + creation_disposition, file_flags, NULL); + } + + if (INVALID_HANDLE_VALUE == hFile) { + auto lastError = GetLastError(); + return IOErrorFromWindowsError( + "NewRandomRWFile failed to Create/Open: " + fname, lastError); + } + + UniqueCloseHandlePtr fileGuard(hFile, CloseHandleFunc); + result->reset(new WinRandomRWFile(fname, hFile, GetPageSize(), options)); + fileGuard.release(); + + return s; +} + +IOStatus WinFileSystem::NewMemoryMappedFileBuffer( + const std::string& fname, std::unique_ptr* result) { + IOStatus s; + result->reset(); + + DWORD fileFlags = FILE_ATTRIBUTE_READONLY; + + HANDLE hFile = INVALID_HANDLE_VALUE; + { + IOSTATS_TIMER_GUARD(open_nanos); + hFile = RX_CreateFile( + RX_FN(fname).c_str(), GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, // Open only if it exists + fileFlags, NULL); + } + + if (INVALID_HANDLE_VALUE == hFile) { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError( + "Failed to open NewMemoryMappedFileBuffer: " + fname, lastError); + return s; + } + UniqueCloseHandlePtr fileGuard(hFile, CloseHandleFunc); + + uint64_t fileSize = 0; + s = GetFileSize(fname, IOOptions(), &fileSize, nullptr); + if (!s.ok()) { + return s; + } + // Will not map empty files + if (fileSize == 0) { + return IOStatus::NotSupported( + "NewMemoryMappedFileBuffer can not map zero length files: " + fname); + } + + // size_t is 32-bit with 32-bit builds + if (fileSize > std::numeric_limits::max()) { + return IOStatus::NotSupported( + "The specified file size does not fit into 32-bit memory addressing: " + + fname); + } + + HANDLE hMap = RX_CreateFileMapping(hFile, NULL, PAGE_READWRITE, + 0, // Whole file at its present length + 0, + NULL); // Mapping name + + if (!hMap) { + auto lastError = GetLastError(); + return IOErrorFromWindowsError( + "Failed to create file mapping for: " + fname, lastError); + } + UniqueCloseHandlePtr mapGuard(hMap, CloseHandleFunc); + + void* base = MapViewOfFileEx(hMap, FILE_MAP_WRITE, + 0, // High DWORD of access start + 0, // Low DWORD + static_cast(fileSize), + NULL); // Let the OS choose the mapping + + if (!base) { + auto lastError = GetLastError(); + return IOErrorFromWindowsError( + "Failed to MapViewOfFile for NewMemoryMappedFileBuffer: " + fname, + lastError); + } + + result->reset(new WinMemoryMappedBuffer(hFile, hMap, base, + static_cast(fileSize))); + + mapGuard.release(); + fileGuard.release(); + + return s; +} + +IOStatus WinFileSystem::NewDirectory(const std::string& name, + const IOOptions& /*options*/, + std::unique_ptr* result, + IODebugContext* /*dbg*/) { + IOStatus s; + // Must be nullptr on failure + result->reset(); + + if (!DirExists(name)) { + s = IOErrorFromWindowsError("open folder: " + name, ERROR_DIRECTORY); + return s; + } + + HANDLE handle = INVALID_HANDLE_VALUE; + // 0 - for access means read metadata + { + IOSTATS_TIMER_GUARD(open_nanos); + handle = RX_CreateFile( + RX_FN(name).c_str(), 0, + FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, // make opening folders possible + NULL); + } + + if (INVALID_HANDLE_VALUE == handle) { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError("open folder: " + name, lastError); + return s; + } + + result->reset(new WinDirectory(name, handle)); + + return s; +} + +IOStatus WinFileSystem::FileExists(const std::string& fname, + const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + IOStatus s; + // TODO: This does not follow symbolic links at this point + // which is consistent with _access() impl on windows + // but can be added + WIN32_FILE_ATTRIBUTE_DATA attrs; + if (FALSE == RX_GetFileAttributesEx(RX_FN(fname).c_str(), + GetFileExInfoStandard, &attrs)) { + auto lastError = GetLastError(); + switch (lastError) { + case ERROR_ACCESS_DENIED: + case ERROR_NOT_FOUND: + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + s = IOStatus::NotFound(); + break; + default: + s = IOErrorFromWindowsError("Unexpected error for: " + fname, + lastError); + break; + } + } + return s; +} + +IOStatus WinFileSystem::GetChildren(const std::string& dir, + const IOOptions& /*opts*/, + std::vector* result, + IODebugContext* /*dbg*/) { + IOStatus status; + result->clear(); + + RX_WIN32_FIND_DATA data; + memset(&data, 0, sizeof(data)); + std::string pattern(dir); + pattern.append("\\").append("*"); + + HANDLE handle = + RX_FindFirstFileEx(RX_FN(pattern).c_str(), + // Do not want alternative name + FindExInfoBasic, &data, FindExSearchNameMatch, + NULL, // lpSearchFilter + 0); + + if (handle == INVALID_HANDLE_VALUE) { + auto lastError = GetLastError(); + switch (lastError) { + case ERROR_NOT_FOUND: + case ERROR_ACCESS_DENIED: + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + status = IOStatus::NotFound(); + break; + default: + status = IOErrorFromWindowsError("Failed to GetChhildren for: " + dir, + lastError); + } + return status; + } + + UniqueFindClosePtr fc(handle, FindCloseFunc); + + // For safety + data.cFileName[MAX_PATH - 1] = 0; + + while (true) { + // filter out '.' and '..' directory entries + // which appear only on some platforms + const bool ignore = + ((data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) && + (RX_FNCMP(data.cFileName, ".") == 0 || + RX_FNCMP(data.cFileName, "..") == 0); + if (!ignore) { + auto x = RX_FILESTRING(data.cFileName, RX_FNLEN(data.cFileName)); + result->push_back(FN_TO_RX(x)); + } + + BOOL ret = -RX_FindNextFile(handle, &data); + // If the function fails the return value is zero + // and non-zero otherwise. Not TRUE or FALSE. + if (ret == FALSE) { + // Posix does not care why we stopped + break; + } + data.cFileName[MAX_PATH - 1] = 0; + } + return status; +} + +IOStatus WinFileSystem::CreateDir(const std::string& name, + const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + IOStatus result; + BOOL ret = RX_CreateDirectory(RX_FN(name).c_str(), NULL); + if (!ret) { + auto lastError = GetLastError(); + result = IOErrorFromWindowsError("Failed to create a directory: " + name, + lastError); + } + + return result; +} + +IOStatus WinFileSystem::CreateDirIfMissing(const std::string& name, + const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + IOStatus result; + + if (DirExists(name)) { + return result; + } + + BOOL ret = RX_CreateDirectory(RX_FN(name).c_str(), NULL); + if (!ret) { + auto lastError = GetLastError(); + if (lastError != ERROR_ALREADY_EXISTS) { + result = IOErrorFromWindowsError("Failed to create a directory: " + name, + lastError); + } else { + result = IOStatus::IOError(name + ": exists but is not a directory"); + } + } + return result; +} + +IOStatus WinFileSystem::DeleteDir(const std::string& name, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + IOStatus result; + BOOL ret = RX_RemoveDirectory(RX_FN(name).c_str()); + if (!ret) { + auto lastError = GetLastError(); + result = + IOErrorFromWindowsError("Failed to remove dir: " + name, lastError); + } + return result; +} + +IOStatus WinFileSystem::GetFileSize(const std::string& fname, + const IOOptions& /*opts*/, uint64_t* size, + IODebugContext* /*dbg*/) { + IOStatus s; + + WIN32_FILE_ATTRIBUTE_DATA attrs; + if (RX_GetFileAttributesEx(RX_FN(fname).c_str(), GetFileExInfoStandard, + &attrs)) { + ULARGE_INTEGER file_size; + file_size.HighPart = attrs.nFileSizeHigh; + file_size.LowPart = attrs.nFileSizeLow; + *size = file_size.QuadPart; + } else { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError("Can not get size for: " + fname, lastError); + } + return s; +} + +uint64_t WinFileSystem::FileTimeToUnixTime(const FILETIME& ftTime) { + const uint64_t c_FileTimePerSecond = 10000000U; + // UNIX epoch starts on 1970-01-01T00:00:00Z + // Windows FILETIME starts on 1601-01-01T00:00:00Z + // Therefore, we need to subtract the below number of seconds from + // the seconds that we obtain from FILETIME with an obvious loss of + // precision + const uint64_t c_SecondBeforeUnixEpoch = 11644473600U; + + ULARGE_INTEGER li; + li.HighPart = ftTime.dwHighDateTime; + li.LowPart = ftTime.dwLowDateTime; + + uint64_t result = + (li.QuadPart / c_FileTimePerSecond) - c_SecondBeforeUnixEpoch; + return result; +} + +IOStatus WinFileSystem::GetFileModificationTime(const std::string& fname, + const IOOptions& /*opts*/, + uint64_t* file_mtime, + IODebugContext* /*dbg*/) { + IOStatus s; + + WIN32_FILE_ATTRIBUTE_DATA attrs; + if (RX_GetFileAttributesEx(RX_FN(fname).c_str(), GetFileExInfoStandard, + &attrs)) { + *file_mtime = FileTimeToUnixTime(attrs.ftLastWriteTime); + } else { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError( + "Can not get file modification time for: " + fname, lastError); + *file_mtime = 0; + } + + return s; +} + +IOStatus WinFileSystem::RenameFile(const std::string& src, + const std::string& target, + const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + IOStatus result; + + // rename() is not capable of replacing the existing file as on Linux + // so use OS API directly + if (!RX_MoveFileEx(RX_FN(src).c_str(), RX_FN(target).c_str(), + MOVEFILE_REPLACE_EXISTING)) { + DWORD lastError = GetLastError(); + + std::string text("Failed to rename: "); + text.append(src).append(" to: ").append(target); + + result = IOErrorFromWindowsError(text, lastError); + } + + return result; +} + +IOStatus WinFileSystem::LinkFile(const std::string& src, + const std::string& target, + const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + IOStatus result; + + if (!RX_CreateHardLink(RX_FN(target).c_str(), RX_FN(src).c_str(), NULL)) { + DWORD lastError = GetLastError(); + if (lastError == ERROR_NOT_SAME_DEVICE) { + return IOStatus::NotSupported("No cross FS links allowed"); + } + + std::string text("Failed to link: "); + text.append(src).append(" to: ").append(target); + + result = IOErrorFromWindowsError(text, lastError); + } + + return result; +} + +IOStatus WinFileSystem::NumFileLinks(const std::string& fname, + const IOOptions& /*opts*/, uint64_t* count, + IODebugContext* /*dbg*/) { + IOStatus s; + HANDLE handle = + RX_CreateFile(RX_FN(fname).c_str(), 0, + FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, + NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + + if (INVALID_HANDLE_VALUE == handle) { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError("NumFileLinks: " + fname, lastError); + return s; + } + UniqueCloseHandlePtr handle_guard(handle, CloseHandleFunc); + FILE_STANDARD_INFO standard_info; + if (0 != GetFileInformationByHandleEx(handle, FileStandardInfo, + &standard_info, + sizeof(standard_info))) { + *count = standard_info.NumberOfLinks; + } else { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError("GetFileInformationByHandleEx: " + fname, + lastError); + } + return s; +} + +IOStatus WinFileSystem::AreFilesSame(const std::string& first, + const std::string& second, + const IOOptions& /*opts*/, bool* res, + IODebugContext* /*dbg*/) { +// For MinGW builds +#if (_WIN32_WINNT == _WIN32_WINNT_VISTA) + IOStatus s = IOStatus::NotSupported(); +#else + assert(res != nullptr); + IOStatus s; + if (res == nullptr) { + s = IOStatus::InvalidArgument("res"); + return s; + } + + // 0 - for access means read metadata + HANDLE file_1 = RX_CreateFile( + RX_FN(first).c_str(), 0, + FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, // make opening folders possible + NULL); + + if (INVALID_HANDLE_VALUE == file_1) { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError("open file: " + first, lastError); + return s; + } + UniqueCloseHandlePtr g_1(file_1, CloseHandleFunc); + + HANDLE file_2 = RX_CreateFile( + RX_FN(second).c_str(), 0, + FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, // make opening folders possible + NULL); + + if (INVALID_HANDLE_VALUE == file_2) { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError("open file: " + second, lastError); + return s; + } + UniqueCloseHandlePtr g_2(file_2, CloseHandleFunc); + + FILE_ID_INFO FileInfo_1; + BOOL result = GetFileInformationByHandleEx(file_1, FileIdInfo, &FileInfo_1, + sizeof(FileInfo_1)); + + if (!result) { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError("stat file: " + first, lastError); + return s; + } + + FILE_ID_INFO FileInfo_2; + result = GetFileInformationByHandleEx(file_2, FileIdInfo, &FileInfo_2, + sizeof(FileInfo_2)); + + if (!result) { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError("stat file: " + second, lastError); + return s; + } + + if (FileInfo_1.VolumeSerialNumber == FileInfo_2.VolumeSerialNumber) { + *res = + (0 == memcmp(FileInfo_1.FileId.Identifier, FileInfo_2.FileId.Identifier, + sizeof(FileInfo_1.FileId.Identifier))); + } else { + *res = false; + } +#endif + return s; +} + +IOStatus WinFileSystem::LockFile(const std::string& lockFname, + const IOOptions& /*opts*/, FileLock** lock, + IODebugContext* /*dbg*/) { + assert(lock != nullptr); + + *lock = NULL; + IOStatus result; + + // No-sharing, this is a LOCK file + const DWORD ExclusiveAccessON = 0; + + // Obtain exclusive access to the LOCK file + // Previously, instead of NORMAL attr we set DELETE on close and that worked + // well except with fault_injection test that insists on deleting it. + HANDLE hFile = 0; + { + IOSTATS_TIMER_GUARD(open_nanos); + hFile = RX_CreateFile(RX_FN(lockFname).c_str(), + (GENERIC_READ | GENERIC_WRITE), ExclusiveAccessON, + NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + } + + if (INVALID_HANDLE_VALUE == hFile) { + auto lastError = GetLastError(); + result = IOErrorFromWindowsError("Failed to create lock file: " + lockFname, + lastError); + } else { + *lock = new WinFileLock(hFile); + } + + return result; +} + +IOStatus WinFileSystem::UnlockFile(FileLock* lock, const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) { + IOStatus result; + + assert(lock != nullptr); + + delete lock; + + return result; +} + +IOStatus WinFileSystem::GetTestDirectory(const IOOptions& opts, + std::string* result, + IODebugContext* dbg) { + std::string output; + + const char* env = getenv("TEST_TMPDIR"); + if (env && env[0] != '\0') { + output = env; + } else { + env = getenv("TMP"); + + if (env && env[0] != '\0') { + output = env; + } else { + output = "c:\\tmp"; + } + } + CreateDir(output, opts, dbg); + + output.append("\\testrocksdb-"); + output.append(std::to_string(GetCurrentProcessId())); + + CreateDir(output, opts, dbg); + + output.swap(*result); + + return IOStatus::OK(); +} + +IOStatus WinFileSystem::NewLogger(const std::string& fname, + const IOOptions& /*opts*/, + std::shared_ptr* result, + IODebugContext* /*dbg*/) { + IOStatus s; + + result->reset(); + + HANDLE hFile = 0; + { + IOSTATS_TIMER_GUARD(open_nanos); + hFile = RX_CreateFile( + RX_FN(fname).c_str(), GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_DELETE, // In RocksDb log files are + // renamed and deleted before + // they are closed. This enables + // doing so. + NULL, + CREATE_ALWAYS, // Original fopen mode is "w" + FILE_ATTRIBUTE_NORMAL, NULL); + } + + if (INVALID_HANDLE_VALUE == hFile) { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError("Failed to open LogFile" + fname, lastError); + } else { + { + // With log files we want to set the true creation time as of now + // because the system + // for some reason caches the attributes of the previous file that just + // been renamed from + // this name so auto_roll_logger_test fails + FILETIME ft; + GetSystemTimeAsFileTime(&ft); + // Set creation, last access and last write time to the same value + SetFileTime(hFile, &ft, &ft, &ft); + } + result->reset(new WinLogger(&WinEnvThreads::gettid, clock_.get(), hFile)); + } + return s; +} + +IOStatus WinFileSystem::IsDirectory(const std::string& path, + const IOOptions& /*opts*/, bool* is_dir, + IODebugContext* /*dbg*/) { + BOOL ret = RX_PathIsDirectory(RX_FN(path).c_str()); + if (is_dir) { + *is_dir = ret ? true : false; + } + return IOStatus::OK(); +} + +Status WinEnvIO::GetHostName(char* name, uint64_t len) { + Status s; + DWORD nSize = static_cast( + std::min(len, std::numeric_limits::max())); + + if (!::GetComputerNameA(name, &nSize)) { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError("GetHostName", lastError); + } else { + name[nSize] = 0; + } + + return s; +} + +IOStatus WinFileSystem::GetAbsolutePath(const std::string& db_path, + const IOOptions& /*options*/, + std::string* output_path, + IODebugContext* dbg) { + // Check if we already have an absolute path + // For test compatibility we will consider starting slash as an + // absolute path + if ((!db_path.empty() && (db_path[0] == '\\' || db_path[0] == '/')) || + !RX_PathIsRelative(RX_FN(db_path).c_str())) { + *output_path = db_path; + return IOStatus::OK(); + } + + RX_FILESTRING result; + result.resize(MAX_PATH); + + // Hopefully no changes the current directory while we do this + // however _getcwd also suffers from the same limitation + DWORD len = RX_GetCurrentDirectory(MAX_PATH, &result[0]); + if (len == 0) { + auto lastError = GetLastError(); + return IOErrorFromWindowsError("Failed to get current working directory", + lastError); + } + + result.resize(len); + std::string res = FN_TO_RX(result); + + res.swap(*output_path); + return IOStatus::OK(); +} + +IOStatus WinFileSystem::GetFreeSpace(const std::string& path, + const IOOptions& /*options*/, + uint64_t* diskfree, + IODebugContext* /*dbg*/) { + assert(diskfree != nullptr); + ULARGE_INTEGER freeBytes; + BOOL f = RX_GetDiskFreeSpaceEx(RX_FN(path).c_str(), &freeBytes, NULL, NULL); + if (f) { + *diskfree = freeBytes.QuadPart; + return IOStatus::OK(); + } else { + DWORD lastError = GetLastError(); + return IOErrorFromWindowsError("Failed to get free space: " + path, + lastError); + } +} + +FileOptions WinFileSystem::OptimizeForLogWrite( + const FileOptions& file_options, const DBOptions& db_options) const { + FileOptions optimized(file_options); + // These two the same as default optimizations + optimized.bytes_per_sync = db_options.wal_bytes_per_sync; + optimized.writable_file_max_buffer_size = + db_options.writable_file_max_buffer_size; + + // This adversely affects %999 on windows + optimized.use_mmap_writes = false; + // Direct writes will produce a huge perf impact on + // Windows. Pre-allocate space for WAL. + optimized.use_direct_writes = false; + return optimized; +} + +FileOptions WinFileSystem::OptimizeForManifestWrite( + const FileOptions& options) const { + FileOptions optimized(options); + optimized.use_mmap_writes = false; + optimized.use_direct_reads = false; + return optimized; +} + +FileOptions WinFileSystem::OptimizeForManifestRead( + const FileOptions& file_options) const { + FileOptions optimized(file_options); + optimized.use_mmap_writes = false; + optimized.use_direct_reads = false; + return optimized; +} + +// Returns true iff the named directory exists and is a directory. +bool WinFileSystem::DirExists(const std::string& dname) { + WIN32_FILE_ATTRIBUTE_DATA attrs; + if (RX_GetFileAttributesEx(RX_FN(dname).c_str(), GetFileExInfoStandard, + &attrs)) { + return 0 != (attrs.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY); + } + return false; +} + +size_t WinFileSystem::GetSectorSize(const std::string& fname) { + size_t sector_size = kSectorSize; + + // obtain device handle + char devicename[7] = "\\\\.\\"; + int erresult = 0; + if (RX_PathIsRelative(RX_FN(fname).c_str())) { + RX_FILESTRING rx_current_dir; + rx_current_dir.resize(MAX_PATH); + DWORD len = RX_GetCurrentDirectory(MAX_PATH, &rx_current_dir[0]); + if (len == 0) { + return sector_size; + } + rx_current_dir.resize(len); + std::string current_dir = FN_TO_RX(rx_current_dir); + erresult = + strncat_s(devicename, sizeof(devicename), current_dir.c_str(), 2); + } else { + erresult = strncat_s(devicename, sizeof(devicename), fname.c_str(), 2); + } + + if (erresult) { + assert(false); + return sector_size; + } + + HANDLE hDevice = CreateFile(devicename, 0, 0, nullptr, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + + if (hDevice == INVALID_HANDLE_VALUE) { + return sector_size; + } + + STORAGE_PROPERTY_QUERY spropertyquery; + spropertyquery.PropertyId = StorageAccessAlignmentProperty; + spropertyquery.QueryType = PropertyStandardQuery; + + BYTE output_buffer[sizeof(STORAGE_ACCESS_ALIGNMENT_DESCRIPTOR)]; + DWORD output_bytes = 0; + + BOOL ret = DeviceIoControl( + hDevice, IOCTL_STORAGE_QUERY_PROPERTY, &spropertyquery, + sizeof(spropertyquery), output_buffer, + sizeof(STORAGE_ACCESS_ALIGNMENT_DESCRIPTOR), &output_bytes, nullptr); + + if (ret) { + sector_size = ((STORAGE_ACCESS_ALIGNMENT_DESCRIPTOR*)output_buffer) + ->BytesPerLogicalSector; + } else { + // many devices do not support StorageProcessAlignmentProperty. Any failure + // here and we fall back to logical alignment + + DISK_GEOMETRY_EX geometry = {0}; + ret = DeviceIoControl(hDevice, IOCTL_DISK_GET_DRIVE_GEOMETRY, nullptr, 0, + &geometry, sizeof(geometry), &output_bytes, nullptr); + if (ret) { + sector_size = geometry.Geometry.BytesPerSector; + } + } + + if (hDevice != INVALID_HANDLE_VALUE) { + CloseHandle(hDevice); + } + + return sector_size; +} + +//////////////////////////////////////////////////////////////////////// +// WinEnvThreads + +WinEnvThreads::WinEnvThreads(Env* hosted_env) + : hosted_env_(hosted_env), thread_pools_(Env::Priority::TOTAL) { + for (int pool_id = 0; pool_id < Env::Priority::TOTAL; ++pool_id) { + thread_pools_[pool_id].SetThreadPriority( + static_cast(pool_id)); + // This allows later initializing the thread-local-env of each thread. + thread_pools_[pool_id].SetHostEnv(hosted_env); + } +} + +WinEnvThreads::~WinEnvThreads() { + WaitForJoin(); + + for (auto& thpool : thread_pools_) { + thpool.JoinAllThreads(); + } +} + +void WinEnvThreads::Schedule(void (*function)(void*), void* arg, + Env::Priority pri, void* tag, + void (*unschedFunction)(void* arg)) { + assert(pri >= Env::Priority::BOTTOM && pri <= Env::Priority::HIGH); + thread_pools_[pri].Schedule(function, arg, tag, unschedFunction); +} + +int WinEnvThreads::UnSchedule(void* arg, Env::Priority pri) { + return thread_pools_[pri].UnSchedule(arg); +} + +namespace { + +struct StartThreadState { + void (*user_function)(void*); + void* arg; +}; + +void* StartThreadWrapper(void* arg) { + std::unique_ptr state( + reinterpret_cast(arg)); + state->user_function(state->arg); + return nullptr; +} + +} // namespace + +void WinEnvThreads::StartThread(void (*function)(void* arg), void* arg) { + std::unique_ptr state(new StartThreadState); + state->user_function = function; + state->arg = arg; + try { + Thread th(&StartThreadWrapper, state.get()); + state.release(); + + std::lock_guard lg(mu_); + threads_to_join_.push_back(std::move(th)); + + } catch (const std::system_error& ex) { + WinthreadCall("start thread", ex.code()); + } +} + +void WinEnvThreads::WaitForJoin() { + for (auto& th : threads_to_join_) { + th.join(); + } + threads_to_join_.clear(); +} + +unsigned int WinEnvThreads::GetThreadPoolQueueLen(Env::Priority pri) const { + assert(pri >= Env::Priority::BOTTOM && pri <= Env::Priority::HIGH); + return thread_pools_[pri].GetQueueLen(); +} + +int WinEnvThreads::ReserveThreads(int threads_to_reserved, Env::Priority pri) { + assert(pri >= Env::Priority::BOTTOM && pri <= Env::Priority::HIGH); + return thread_pools_[pri].ReserveThreads(threads_to_reserved); +} + +int WinEnvThreads::ReleaseThreads(int threads_to_released, Env::Priority pri) { + assert(pri >= Env::Priority::BOTTOM && pri <= Env::Priority::HIGH); + return thread_pools_[pri].ReleaseThreads(threads_to_released); +} + +uint64_t WinEnvThreads::gettid() { + uint64_t thread_id = GetCurrentThreadId(); + return thread_id; +} + +uint64_t WinEnvThreads::GetThreadID() const { return gettid(); } + +void WinEnvThreads::SetBackgroundThreads(int num, Env::Priority pri) { + assert(pri >= Env::Priority::BOTTOM && pri <= Env::Priority::HIGH); + thread_pools_[pri].SetBackgroundThreads(num); +} + +int WinEnvThreads::GetBackgroundThreads(Env::Priority pri) { + assert(pri >= Env::Priority::BOTTOM && pri <= Env::Priority::HIGH); + return thread_pools_[pri].GetBackgroundThreads(); +} + +void WinEnvThreads::IncBackgroundThreadsIfNeeded(int num, Env::Priority pri) { + assert(pri >= Env::Priority::BOTTOM && pri <= Env::Priority::HIGH); + thread_pools_[pri].IncBackgroundThreadsIfNeeded(num); +} + +///////////////////////////////////////////////////////////////////////// +// WinEnv + +WinEnv::WinEnv() + : CompositeEnv(WinFileSystem::Default(), WinClock::Default()), + winenv_io_(this), + winenv_threads_(this) { + // Protected member of the base class + thread_status_updater_ = CreateThreadStatusUpdater(); +} + +WinEnv::~WinEnv() { + // All threads must be joined before the deletion of + // thread_status_updater_. + delete thread_status_updater_; +} + +Status WinEnv::GetThreadList(std::vector* thread_list) { + assert(thread_status_updater_); + return thread_status_updater_->GetThreadList(thread_list); +} + +Status WinEnv::GetHostName(char* name, uint64_t len) { + return winenv_io_.GetHostName(name, len); +} + +void WinEnv::Schedule(void (*function)(void*), void* arg, Env::Priority pri, + void* tag, void (*unschedFunction)(void* arg)) { + return winenv_threads_.Schedule(function, arg, pri, tag, unschedFunction); +} + +int WinEnv::UnSchedule(void* arg, Env::Priority pri) { + return winenv_threads_.UnSchedule(arg, pri); +} + +void WinEnv::StartThread(void (*function)(void* arg), void* arg) { + return winenv_threads_.StartThread(function, arg); +} + +void WinEnv::WaitForJoin() { return winenv_threads_.WaitForJoin(); } + +unsigned int WinEnv::GetThreadPoolQueueLen(Env::Priority pri) const { + return winenv_threads_.GetThreadPoolQueueLen(pri); +} +int WinEnv::ReserveThreads(int threads_to_reserved, Env::Priority pri) { + return winenv_threads_.ReserveThreads(threads_to_reserved, pri); +} + +int WinEnv::ReleaseThreads(int threads_to_released, Env::Priority pri) { + return winenv_threads_.ReleaseThreads(threads_to_released, pri); +} + +uint64_t WinEnv::GetThreadID() const { return winenv_threads_.GetThreadID(); } + +// Allow increasing the number of worker threads. +void WinEnv::SetBackgroundThreads(int num, Env::Priority pri) { + return winenv_threads_.SetBackgroundThreads(num, pri); +} + +int WinEnv::GetBackgroundThreads(Env::Priority pri) { + return winenv_threads_.GetBackgroundThreads(pri); +} + +void WinEnv::IncBackgroundThreadsIfNeeded(int num, Env::Priority pri) { + return winenv_threads_.IncBackgroundThreadsIfNeeded(num, pri); +} + +} // namespace port + +std::shared_ptr FileSystem::Default() { + return port::WinFileSystem::Default(); +} + +const std::shared_ptr& SystemClock::Default() { + STATIC_AVOID_DESTRUCTION(std::shared_ptr, clock) + (std::make_shared()); + return clock; +} +} // namespace ROCKSDB_NAMESPACE + +#endif diff --git a/librocksdb-sys/rocksdb/port/win/env_win.h b/librocksdb-sys/rocksdb/port/win/env_win.h new file mode 100644 index 0000000..b6482ba --- /dev/null +++ b/librocksdb-sys/rocksdb/port/win/env_win.h @@ -0,0 +1,305 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// An Env is an interface used by the rocksdb implementation to access +// operating system functionality like the filesystem etc. Callers +// may wish to provide a custom Env object when opening a database to +// get fine gain control; e.g., to rate limit file system operations. +// +// All Env implementations are safe for concurrent access from +// multiple threads without any external synchronization. + +#pragma once +#include +#include + +#include +#include +#include + +#include "env/composite_env_wrapper.h" +#include "port/port.h" +#include "rocksdb/env.h" +#include "rocksdb/file_system.h" +#include "rocksdb/system_clock.h" +#include "util/threadpool_imp.h" + +#undef GetCurrentTime +#undef DeleteFile +#undef LoadLibrary + +namespace ROCKSDB_NAMESPACE { +namespace port { + +// Currently not designed for inheritance but rather a replacement +class WinEnvThreads { + public: + explicit WinEnvThreads(Env* hosted_env); + + ~WinEnvThreads(); + + WinEnvThreads(const WinEnvThreads&) = delete; + WinEnvThreads& operator=(const WinEnvThreads&) = delete; + + void Schedule(void (*function)(void*), void* arg, Env::Priority pri, + void* tag, void (*unschedFunction)(void* arg)); + + int UnSchedule(void* arg, Env::Priority pri); + + void StartThread(void (*function)(void* arg), void* arg); + + void WaitForJoin(); + + unsigned int GetThreadPoolQueueLen(Env::Priority pri) const; + + int ReserveThreads(int threads_to_be_reserved, Env::Priority pri); + + int ReleaseThreads(int threads_to_be_released, Env::Priority pri); + + static uint64_t gettid(); + + uint64_t GetThreadID() const; + + // Allow increasing the number of worker threads. + void SetBackgroundThreads(int num, Env::Priority pri); + int GetBackgroundThreads(Env::Priority pri); + + void IncBackgroundThreadsIfNeeded(int num, Env::Priority pri); + + private: + Env* hosted_env_; + mutable std::mutex mu_; + std::vector thread_pools_; + std::vector threads_to_join_; +}; + +class WinClock : public SystemClock { + public: + WinClock(); + virtual ~WinClock() {} + + static const char* kClassName() { return "WindowsClock"; } + const char* Name() const override { return kDefaultName(); } + const char* NickName() const override { return kClassName(); } + + uint64_t NowMicros() override; + + uint64_t NowNanos() override; + + // 0 indicates not supported + uint64_t CPUMicros() override { return 0; } + void SleepForMicroseconds(int micros) override; + + Status GetCurrentTime(int64_t* unix_time) override; + // Converts seconds-since-Jan-01-1970 to a printable string + std::string TimeToString(uint64_t time) override; + + uint64_t GetPerfCounterFrequency() const { return perf_counter_frequency_; } + + private: + using FnGetSystemTimePreciseAsFileTime = VOID(WINAPI*)(LPFILETIME); + + uint64_t perf_counter_frequency_; + uint64_t nano_seconds_per_period_; + FnGetSystemTimePreciseAsFileTime GetSystemTimePreciseAsFileTime_; +}; + +class WinFileSystem : public FileSystem { + public: + static const std::shared_ptr& Default(); + WinFileSystem(const std::shared_ptr& clock); + ~WinFileSystem() {} + static const char* kClassName() { return "WinFS"; } + const char* Name() const override { return kClassName(); } + const char* NickName() const override { return kDefaultName(); } + + static size_t GetSectorSize(const std::string& fname); + size_t GetPageSize() const { return page_size_; } + size_t GetAllocationGranularity() const { return allocation_granularity_; } + + IOStatus DeleteFile(const std::string& fname, const IOOptions& options, + IODebugContext* dbg) override; + + // Truncate the named file to the specified size. + IOStatus Truncate(const std::string& /*fname*/, size_t /*size*/, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override; + IOStatus NewSequentialFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus NewRandomAccessFile(const std::string& fname, + const FileOptions& options, + std::unique_ptr* result, + IODebugContext* /*dbg*/) override; + IOStatus NewWritableFile(const std::string& f, const FileOptions& file_opts, + std::unique_ptr* r, + IODebugContext* dbg) override; + IOStatus ReopenWritableFile(const std::string& fname, + const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus NewRandomRWFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + IOStatus NewMemoryMappedFileBuffer( + const std::string& fname, + std::unique_ptr* result) override; + + IOStatus NewDirectory(const std::string& name, const IOOptions& io_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + IOStatus FileExists(const std::string& f, const IOOptions& io_opts, + IODebugContext* dbg) override; + IOStatus GetChildren(const std::string& dir, const IOOptions& io_opts, + std::vector* r, + IODebugContext* dbg) override; + IOStatus CreateDir(const std::string& dirname, const IOOptions& options, + IODebugContext* dbg) override; + + // Creates directory if missing. Return Ok if it exists, or successful in + // Creating. + IOStatus CreateDirIfMissing(const std::string& dirname, + const IOOptions& options, + IODebugContext* dbg) override; + + // Delete the specified directory. + IOStatus DeleteDir(const std::string& dirname, const IOOptions& options, + IODebugContext* dbg) override; + // Store the size of fname in *file_size. + IOStatus GetFileSize(const std::string& fname, const IOOptions& options, + uint64_t* file_size, IODebugContext* dbg) override; + // Store the last modification time of fname in *file_mtime. + IOStatus GetFileModificationTime(const std::string& fname, + const IOOptions& options, + uint64_t* file_mtime, + IODebugContext* dbg) override; + // Rename file src to target. + IOStatus RenameFile(const std::string& src, const std::string& target, + const IOOptions& options, IODebugContext* dbg) override; + + // Hard Link file src to target. + IOStatus LinkFile(const std::string& /*src*/, const std::string& /*target*/, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override; + IOStatus NumFileLinks(const std::string& /*fname*/, + const IOOptions& /*options*/, uint64_t* /*count*/, + IODebugContext* /*dbg*/) override; + IOStatus AreFilesSame(const std::string& /*first*/, + const std::string& /*second*/, + const IOOptions& /*options*/, bool* /*res*/, + IODebugContext* /*dbg*/) override; + IOStatus LockFile(const std::string& fname, const IOOptions& options, + FileLock** lock, IODebugContext* dbg) override; + IOStatus UnlockFile(FileLock* lock, const IOOptions& options, + IODebugContext* dbg) override; + IOStatus GetTestDirectory(const IOOptions& options, std::string* path, + IODebugContext* dbg) override; + + // Create and returns a default logger (an instance of EnvLogger) for storing + // informational messages. Derived classes can override to provide custom + // logger. + IOStatus NewLogger(const std::string& fname, const IOOptions& io_opts, + std::shared_ptr* result, + IODebugContext* dbg) override; + // Get full directory name for this db. + IOStatus GetAbsolutePath(const std::string& db_path, const IOOptions& options, + std::string* output_path, + IODebugContext* dbg) override; + IOStatus IsDirectory(const std::string& /*path*/, const IOOptions& options, + bool* is_dir, IODebugContext* /*dgb*/) override; + // This seems to clash with a macro on Windows, so #undef it here +#undef GetFreeSpace + IOStatus GetFreeSpace(const std::string& /*path*/, + const IOOptions& /*options*/, uint64_t* /*diskfree*/, + IODebugContext* /*dbg*/) override; + FileOptions OptimizeForLogWrite(const FileOptions& file_options, + const DBOptions& db_options) const override; + FileOptions OptimizeForManifestRead( + const FileOptions& file_options) const override; + FileOptions OptimizeForManifestWrite( + const FileOptions& file_options) const override; + void SupportedOps(int64_t& supported_ops) override { supported_ops = 0; } + + protected: + static uint64_t FileTimeToUnixTime(const FILETIME& ftTime); + // Returns true iff the named directory exists and is a directory. + + virtual bool DirExists(const std::string& dname); + // Helper for NewWritable and ReopenWritableFile + virtual IOStatus OpenWritableFile(const std::string& fname, + const FileOptions& options, + std::unique_ptr* result, + bool reopen); + + private: + std::shared_ptr clock_; + size_t page_size_; + size_t allocation_granularity_; +}; + +// Designed for inheritance so can be re-used +// but certain parts replaced +class WinEnvIO { + public: + explicit WinEnvIO(Env* hosted_env); + + virtual ~WinEnvIO(); + + virtual Status GetHostName(char* name, uint64_t len); + + private: + Env* hosted_env_; +}; + +class WinEnv : public CompositeEnv { + public: + WinEnv(); + + ~WinEnv(); + static const char* kClassName() { return "WinEnv"; } + const char* Name() const override { return kClassName(); } + const char* NickName() const override { return kDefaultName(); } + + Status GetHostName(char* name, uint64_t len) override; + + Status GetThreadList(std::vector* thread_list) override; + + void Schedule(void (*function)(void*), void* arg, Env::Priority pri, + void* tag, void (*unschedFunction)(void* arg)) override; + + int UnSchedule(void* arg, Env::Priority pri) override; + + void StartThread(void (*function)(void* arg), void* arg) override; + + void WaitForJoin() override; + + unsigned int GetThreadPoolQueueLen(Env::Priority pri) const override; + + int ReserveThreads(int threads_to_be_reserved, Env::Priority pri) override; + + int ReleaseThreads(int threads_to_be_released, Env::Priority pri) override; + + uint64_t GetThreadID() const override; + + // Allow increasing the number of worker threads. + void SetBackgroundThreads(int num, Env::Priority pri) override; + int GetBackgroundThreads(Env::Priority pri) override; + + void IncBackgroundThreadsIfNeeded(int num, Env::Priority pri) override; + + private: + WinEnvIO winenv_io_; + WinEnvThreads winenv_threads_; +}; + +} // namespace port +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/port/win/io_win.cc b/librocksdb-sys/rocksdb/port/win/io_win.cc new file mode 100644 index 0000000..4fa7355 --- /dev/null +++ b/librocksdb-sys/rocksdb/port/win/io_win.cc @@ -0,0 +1,1101 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#if defined(OS_WIN) + +#include "port/win/io_win.h" + +#include "env_win.h" +#include "monitoring/iostats_context_imp.h" +#include "test_util/sync_point.h" +#include "util/aligned_buffer.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { +namespace port { + +/* + * DirectIOHelper + */ +namespace { + +const size_t kSectorSize = 512; + +inline bool IsPowerOfTwo(const size_t alignment) { + return ((alignment) & (alignment - 1)) == 0; +} + +inline bool IsAligned(size_t alignment, const void* ptr) { + return ((uintptr_t(ptr)) & (alignment - 1)) == 0; +} +} // namespace + +std::string GetWindowsErrSz(DWORD err) { + std::string Err; + LPSTR lpMsgBuf = nullptr; + FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, err, + 0, // Default language + reinterpret_cast(&lpMsgBuf), 0, NULL); + + if (lpMsgBuf) { + Err = lpMsgBuf; + LocalFree(lpMsgBuf); + } + return Err; +} + +// We preserve the original name of this interface to denote the original idea +// behind it. +// All reads happen by a specified offset and pwrite interface does not change +// the position of the file pointer. Judging from the man page and errno it does +// execute +// lseek atomically to return the position of the file back where it was. +// WriteFile() does not +// have this capability. Therefore, for both pread and pwrite the pointer is +// advanced to the next position +// which is fine for writes because they are (should be) sequential. +// Because all the reads/writes happen by the specified offset, the caller in +// theory should not +// rely on the current file offset. +IOStatus pwrite(const WinFileData* file_data, const Slice& data, + uint64_t offset, size_t& bytes_written) { + IOStatus s; + bytes_written = 0; + + size_t num_bytes = data.size(); + if (num_bytes > std::numeric_limits::max()) { + // May happen in 64-bit builds where size_t is 64-bits but + // long is still 32-bit, but that's the API here at the moment + return IOStatus::InvalidArgument( + "num_bytes is too large for a single write: " + file_data->GetName()); + } + + OVERLAPPED overlapped = {0}; + ULARGE_INTEGER offsetUnion; + offsetUnion.QuadPart = offset; + + overlapped.Offset = offsetUnion.LowPart; + overlapped.OffsetHigh = offsetUnion.HighPart; + + DWORD bytesWritten = 0; + + if (FALSE == WriteFile(file_data->GetFileHandle(), data.data(), + static_cast(num_bytes), &bytesWritten, + &overlapped)) { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError("WriteFile failed: " + file_data->GetName(), + lastError); + } else { + bytes_written = bytesWritten; + } + + return s; +} + +// See comments for pwrite above +IOStatus pread(const WinFileData* file_data, char* src, size_t num_bytes, + uint64_t offset, size_t& bytes_read) { + IOStatus s; + bytes_read = 0; + + if (num_bytes > std::numeric_limits::max()) { + return IOStatus::InvalidArgument( + "num_bytes is too large for a single read: " + file_data->GetName()); + } + + OVERLAPPED overlapped = {0}; + ULARGE_INTEGER offsetUnion; + offsetUnion.QuadPart = offset; + + overlapped.Offset = offsetUnion.LowPart; + overlapped.OffsetHigh = offsetUnion.HighPart; + + DWORD bytesRead = 0; + + if (FALSE == ReadFile(file_data->GetFileHandle(), src, + static_cast(num_bytes), &bytesRead, + &overlapped)) { + auto lastError = GetLastError(); + // EOF is OK with zero bytes read + if (lastError != ERROR_HANDLE_EOF) { + s = IOErrorFromWindowsError("ReadFile failed: " + file_data->GetName(), + lastError); + } + } else { + bytes_read = bytesRead; + } + + return s; +} + +// SetFileInformationByHandle() is capable of fast pre-allocates. +// However, this does not change the file end position unless the file is +// truncated and the pre-allocated space is not considered filled with zeros. +IOStatus fallocate(const std::string& filename, HANDLE hFile, + uint64_t to_size) { + IOStatus status; + + FILE_ALLOCATION_INFO alloc_info; + alloc_info.AllocationSize.QuadPart = to_size; + + if (!SetFileInformationByHandle(hFile, FileAllocationInfo, &alloc_info, + sizeof(FILE_ALLOCATION_INFO))) { + auto lastError = GetLastError(); + status = IOErrorFromWindowsError( + "Failed to pre-allocate space: " + filename, lastError); + } + + return status; +} + +IOStatus ftruncate(const std::string& filename, HANDLE hFile, uint64_t toSize) { + IOStatus status; + + FILE_END_OF_FILE_INFO end_of_file; + end_of_file.EndOfFile.QuadPart = toSize; + + if (!SetFileInformationByHandle(hFile, FileEndOfFileInfo, &end_of_file, + sizeof(FILE_END_OF_FILE_INFO))) { + auto lastError = GetLastError(); + status = IOErrorFromWindowsError("Failed to Set end of file: " + filename, + lastError); + } + + return status; +} + +size_t GetUniqueIdFromFile(HANDLE /*hFile*/, char* /*id*/, + size_t /*max_size*/) { + // Returning 0 is safe as it causes the table reader to generate a unique ID. + // This is suboptimal for performance as it prevents multiple table readers + // for the same file from sharing cached blocks. For example, if users have + // a low value for `max_open_files`, there can be many table readers opened + // for the same file. + // + // TODO: this is a temporarily solution as it is safe but not optimal for + // performance. For more details see discussion in + // https://github.com/facebook/rocksdb/pull/5844. + return 0; +} + +WinFileData::WinFileData(const std::string& filename, HANDLE hFile, + bool direct_io) + : filename_(filename), + hFile_(hFile), + use_direct_io_(direct_io), + sector_size_(WinFileSystem::GetSectorSize(filename)) {} + +bool WinFileData::IsSectorAligned(const size_t off) const { + return (off & (sector_size_ - 1)) == 0; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// WinMmapReadableFile + +WinMmapReadableFile::WinMmapReadableFile(const std::string& fileName, + HANDLE hFile, HANDLE hMap, + const void* mapped_region, + size_t length) + : WinFileData(fileName, hFile, false /* use_direct_io */), + hMap_(hMap), + mapped_region_(mapped_region), + length_(length) {} + +WinMmapReadableFile::~WinMmapReadableFile() { + BOOL ret __attribute__((__unused__)); + ret = ::UnmapViewOfFile(mapped_region_); + assert(ret); + + ret = ::CloseHandle(hMap_); + assert(ret); +} + +IOStatus WinMmapReadableFile::Read(uint64_t offset, size_t n, + const IOOptions& /*options*/, Slice* result, + char* scratch, + IODebugContext* /*dbg*/) const { + IOStatus s; + + if (offset > length_) { + *result = Slice(); + return IOError(filename_, EINVAL); + } else if (offset + n > length_) { + n = length_ - static_cast(offset); + } + *result = Slice(reinterpret_cast(mapped_region_) + offset, n); + return s; +} + +IOStatus WinMmapReadableFile::InvalidateCache(size_t offset, size_t length) { + return IOStatus::OK(); +} + +size_t WinMmapReadableFile::GetUniqueId(char* id, size_t max_size) const { + return GetUniqueIdFromFile(hFile_, id, max_size); +} + +/////////////////////////////////////////////////////////////////////////////// +/// WinMmapFile + +// Can only truncate or reserve to a sector size aligned if +// used on files that are opened with Unbuffered I/O +IOStatus WinMmapFile::TruncateFile(uint64_t toSize) { + return ftruncate(filename_, hFile_, toSize); +} + +IOStatus WinMmapFile::UnmapCurrentRegion() { + IOStatus status; + + if (mapped_begin_ != nullptr) { + if (!::UnmapViewOfFile(mapped_begin_)) { + status = IOErrorFromWindowsError( + "Failed to unmap file view: " + filename_, GetLastError()); + } + + // Move on to the next portion of the file + file_offset_ += view_size_; + + // UnmapView automatically sends data to disk but not the metadata + // which is good and provides some equivalent of fdatasync() on Linux + // therefore, we donot need separate flag for metadata + mapped_begin_ = nullptr; + mapped_end_ = nullptr; + dst_ = nullptr; + + last_sync_ = nullptr; + pending_sync_ = false; + } + + return status; +} + +IOStatus WinMmapFile::MapNewRegion(const IOOptions& options, + IODebugContext* dbg) { + IOStatus status; + + assert(mapped_begin_ == nullptr); + + size_t minDiskSize = static_cast(file_offset_) + view_size_; + + if (minDiskSize > reserved_size_) { + status = Allocate(file_offset_, view_size_, options, dbg); + if (!status.ok()) { + return status; + } + } + + // Need to remap + if (hMap_ == NULL || reserved_size_ > mapping_size_) { + if (hMap_ != NULL) { + // Unmap the previous one + BOOL ret __attribute__((__unused__)); + ret = ::CloseHandle(hMap_); + assert(ret); + hMap_ = NULL; + } + + ULARGE_INTEGER mappingSize; + mappingSize.QuadPart = reserved_size_; + + hMap_ = CreateFileMappingA( + hFile_, + NULL, // Security attributes + PAGE_READWRITE, // There is not a write only mode for mapping + mappingSize.HighPart, // Enable mapping the whole file but the actual + // amount mapped is determined by MapViewOfFile + mappingSize.LowPart, + NULL); // Mapping name + + if (NULL == hMap_) { + return IOErrorFromWindowsError( + "WindowsMmapFile failed to create file mapping for: " + filename_, + GetLastError()); + } + + mapping_size_ = reserved_size_; + } + + ULARGE_INTEGER offset; + offset.QuadPart = file_offset_; + + // View must begin at the granularity aligned offset + mapped_begin_ = reinterpret_cast( + MapViewOfFileEx(hMap_, FILE_MAP_WRITE, offset.HighPart, offset.LowPart, + view_size_, NULL)); + + if (!mapped_begin_) { + status = IOErrorFromWindowsError( + "WindowsMmapFile failed to map file view: " + filename_, + GetLastError()); + } else { + mapped_end_ = mapped_begin_ + view_size_; + dst_ = mapped_begin_; + last_sync_ = mapped_begin_; + pending_sync_ = false; + } + return status; +} + +IOStatus WinMmapFile::PreallocateInternal(uint64_t spaceToReserve) { + return fallocate(filename_, hFile_, spaceToReserve); +} + +WinMmapFile::WinMmapFile(const std::string& fname, HANDLE hFile, + size_t page_size, size_t allocation_granularity, + const FileOptions& options) + : WinFileData(fname, hFile, false), + FSWritableFile(options), + hMap_(NULL), + page_size_(page_size), + allocation_granularity_(allocation_granularity), + reserved_size_(0), + mapping_size_(0), + view_size_(0), + mapped_begin_(nullptr), + mapped_end_(nullptr), + dst_(nullptr), + last_sync_(nullptr), + file_offset_(0), + pending_sync_(false) { + // Allocation granularity must be obtained from GetSystemInfo() and must be + // a power of two. + assert(allocation_granularity > 0); + assert((allocation_granularity & (allocation_granularity - 1)) == 0); + + assert(page_size > 0); + assert((page_size & (page_size - 1)) == 0); + + // Only for memory mapped writes + assert(options.use_mmap_writes); + + // View size must be both the multiple of allocation_granularity AND the + // page size and the granularity is usually a multiple of a page size. + const size_t viewSize = + 32 * 1024; // 32Kb similar to the Windows File Cache in buffered mode + view_size_ = Roundup(viewSize, allocation_granularity_); +} + +WinMmapFile::~WinMmapFile() { + if (hFile_) { + this->Close(IOOptions(), nullptr); + } +} + +IOStatus WinMmapFile::Append(const Slice& data, const IOOptions& options, + IODebugContext* dbg) { + const char* src = data.data(); + size_t left = data.size(); + + while (left > 0) { + assert(mapped_begin_ <= dst_); + size_t avail = mapped_end_ - dst_; + + if (avail == 0) { + IOStatus s = UnmapCurrentRegion(); + if (s.ok()) { + s = MapNewRegion(options, dbg); + } + + if (!s.ok()) { + return s; + } + } else { + size_t n = std::min(left, avail); + memcpy(dst_, src, n); + dst_ += n; + src += n; + left -= n; + pending_sync_ = true; + } + } + + // Now make sure that the last partial page is padded with zeros if needed + size_t bytesToPad = Roundup(size_t(dst_), page_size_) - size_t(dst_); + if (bytesToPad > 0) { + memset(dst_, 0, bytesToPad); + } + + return IOStatus::OK(); +} + +// Means Close() will properly take care of truncate +// and it does not need any additional information +IOStatus WinMmapFile::Truncate(uint64_t size, const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + return IOStatus::OK(); +} + +IOStatus WinMmapFile::Close(const IOOptions& options, IODebugContext* dbg) { + IOStatus s; + + assert(NULL != hFile_); + + // We truncate to the precise size so no + // uninitialized data at the end. SetEndOfFile + // which we use does not write zeros and it is good. + uint64_t targetSize = GetFileSize(options, dbg); + + if (mapped_begin_ != nullptr) { + // Sync before unmapping to make sure everything + // is on disk and there is not a lazy writing + // so we are deterministic with the tests + Sync(options, dbg); + s = UnmapCurrentRegion(); + } + + if (NULL != hMap_) { + BOOL ret = ::CloseHandle(hMap_); + if (!ret && s.ok()) { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError( + "Failed to Close mapping for file: " + filename_, lastError); + } + + hMap_ = NULL; + } + + if (hFile_ != NULL) { + TruncateFile(targetSize); + + BOOL ret = ::CloseHandle(hFile_); + hFile_ = NULL; + + if (!ret && s.ok()) { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError( + "Failed to close file map handle: " + filename_, lastError); + } + } + + return s; +} + +IOStatus WinMmapFile::Flush(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + return IOStatus::OK(); +} + +// Flush only data +IOStatus WinMmapFile::Sync(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + IOStatus s; + + // Some writes occurred since last sync + if (dst_ > last_sync_) { + assert(mapped_begin_); + assert(dst_); + assert(dst_ > mapped_begin_); + assert(dst_ < mapped_end_); + + size_t page_begin = + TruncateToPageBoundary(page_size_, last_sync_ - mapped_begin_); + size_t page_end = + TruncateToPageBoundary(page_size_, dst_ - mapped_begin_ - 1); + + // Flush only the amount of that is a multiple of pages + if (!::FlushViewOfFile(mapped_begin_ + page_begin, + (page_end - page_begin) + page_size_)) { + s = IOErrorFromWindowsError("Failed to FlushViewOfFile: " + filename_, + GetLastError()); + } else { + last_sync_ = dst_; + } + } + + return s; +} + +/** + * Flush data as well as metadata to stable storage. + */ +IOStatus WinMmapFile::Fsync(const IOOptions& options, IODebugContext* dbg) { + IOStatus s = Sync(options, dbg); + + // Flush metadata + if (s.ok() && pending_sync_) { + if (!::FlushFileBuffers(hFile_)) { + s = IOErrorFromWindowsError("Failed to FlushFileBuffers: " + filename_, + GetLastError()); + } + pending_sync_ = false; + } + + return s; +} + +/** + * Get the size of valid data in the file. This will not match the + * size that is returned from the filesystem because we use mmap + * to extend file by map_size every time. + */ +uint64_t WinMmapFile::GetFileSize(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + size_t used = dst_ - mapped_begin_; + return file_offset_ + used; +} + +IOStatus WinMmapFile::InvalidateCache(size_t offset, size_t length) { + return IOStatus::OK(); +} + +IOStatus WinMmapFile::Allocate(uint64_t offset, uint64_t len, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + IOStatus status; + TEST_KILL_RANDOM("WinMmapFile::Allocate"); + + // Make sure that we reserve an aligned amount of space + // since the reservation block size is driven outside so we want + // to check if we are ok with reservation here + size_t spaceToReserve = + Roundup(static_cast(offset + len), view_size_); + // Nothing to do + if (spaceToReserve <= reserved_size_) { + return status; + } + + IOSTATS_TIMER_GUARD(allocate_nanos); + status = PreallocateInternal(spaceToReserve); + if (status.ok()) { + reserved_size_ = spaceToReserve; + } + return status; +} + +size_t WinMmapFile::GetUniqueId(char* id, size_t max_size) const { + return GetUniqueIdFromFile(hFile_, id, max_size); +} + +////////////////////////////////////////////////////////////////////////////////// +// WinSequentialFile + +WinSequentialFile::WinSequentialFile(const std::string& fname, HANDLE f, + const FileOptions& options) + : WinFileData(fname, f, options.use_direct_reads) {} + +WinSequentialFile::~WinSequentialFile() { + assert(hFile_ != INVALID_HANDLE_VALUE); +} + +IOStatus WinSequentialFile::Read(size_t n, const IOOptions& /*opts*/, + Slice* result, char* scratch, + IODebugContext* /*dbg*/) { + IOStatus s; + size_t r = 0; + + assert(result != nullptr); + if (WinFileData::use_direct_io()) { + return IOStatus::NotSupported("Read() does not support direct_io"); + } + + // Windows ReadFile API accepts a DWORD. + // While it is possible to read in a loop if n is too big + // it is an unlikely case. + if (n > std::numeric_limits::max()) { + return IOStatus::InvalidArgument("n is too big for a single ReadFile: " + + filename_); + } + + DWORD bytesToRead = + static_cast(n); // cast is safe due to the check above + DWORD bytesRead = 0; + BOOL ret = ReadFile(hFile_, scratch, bytesToRead, &bytesRead, NULL); + if (ret != FALSE) { + r = bytesRead; + } else { + auto lastError = GetLastError(); + if (lastError != ERROR_HANDLE_EOF) { + s = IOErrorFromWindowsError("ReadFile failed: " + filename_, lastError); + } + } + + *result = Slice(scratch, r); + return s; +} + +IOStatus WinSequentialFile::PositionedReadInternal(char* src, size_t numBytes, + uint64_t offset, + size_t& bytes_read) const { + return pread(this, src, numBytes, offset, bytes_read); +} + +IOStatus WinSequentialFile::PositionedRead(uint64_t offset, size_t n, + const IOOptions& /*opts*/, + Slice* result, char* scratch, + IODebugContext* /*dbg*/) { + if (!WinFileData::use_direct_io()) { + return IOStatus::NotSupported("This function is only used for direct_io"); + } + + assert(IsSectorAligned(static_cast(offset))); + assert(IsSectorAligned(static_cast(n))); + + size_t bytes_read = 0; // out param + IOStatus s = PositionedReadInternal(scratch, static_cast(n), offset, + bytes_read); + *result = Slice(scratch, bytes_read); + return s; +} + +IOStatus WinSequentialFile::Skip(uint64_t n) { + // Can't handle more than signed max as SetFilePointerEx accepts a signed + // 64-bit integer. As such it is a highly unlikley case to have n so large. + if (n > static_cast(std::numeric_limits::max())) { + return IOStatus::InvalidArgument( + "n is too large for a single SetFilePointerEx() call" + filename_); + } + + LARGE_INTEGER li; + li.QuadPart = static_cast(n); // cast is safe due to the check + // above + BOOL ret = SetFilePointerEx(hFile_, li, NULL, FILE_CURRENT); + if (ret == FALSE) { + auto lastError = GetLastError(); + return IOErrorFromWindowsError("Skip SetFilePointerEx():" + filename_, + lastError); + } + return IOStatus::OK(); +} + +IOStatus WinSequentialFile::InvalidateCache(size_t offset, size_t length) { + return IOStatus::OK(); +} + +////////////////////////////////////////////////////////////////////////////////////////////////// +/// WinRandomAccessBase + +inline IOStatus WinRandomAccessImpl::PositionedReadInternal( + char* src, size_t numBytes, uint64_t offset, size_t& bytes_read) const { + return pread(file_base_, src, numBytes, offset, bytes_read); +} + +inline WinRandomAccessImpl::WinRandomAccessImpl(WinFileData* file_base, + size_t alignment, + const FileOptions& options) + : file_base_(file_base), + alignment_(std::max(alignment, file_base->GetSectorSize())) { + assert(!options.use_mmap_reads); +} + +inline IOStatus WinRandomAccessImpl::ReadImpl(uint64_t offset, size_t n, + Slice* result, + char* scratch) const { + // Check buffer alignment + if (file_base_->use_direct_io()) { + assert(file_base_->IsSectorAligned(static_cast(offset))); + assert(IsAligned(alignment_, scratch)); + } + + if (n == 0) { + *result = Slice(scratch, 0); + return IOStatus::OK(); + } + + size_t bytes_read = 0; + IOStatus s = PositionedReadInternal(scratch, n, offset, bytes_read); + *result = Slice(scratch, bytes_read); + return s; +} + +/////////////////////////////////////////////////////////////////////////////////////////////////// +/// WinRandomAccessFile + +WinRandomAccessFile::WinRandomAccessFile(const std::string& fname, HANDLE hFile, + size_t alignment, + const FileOptions& options) + : WinFileData(fname, hFile, options.use_direct_reads), + WinRandomAccessImpl(this, alignment, options) {} + +WinRandomAccessFile::~WinRandomAccessFile() {} + +IOStatus WinRandomAccessFile::Read(uint64_t offset, size_t n, + const IOOptions& /*options*/, Slice* result, + char* scratch, + IODebugContext* /*dbg*/) const { + return ReadImpl(offset, n, result, scratch); +} + +IOStatus WinRandomAccessFile::InvalidateCache(size_t offset, size_t length) { + return IOStatus::OK(); +} + +size_t WinRandomAccessFile::GetUniqueId(char* id, size_t max_size) const { + return GetUniqueIdFromFile(GetFileHandle(), id, max_size); +} + +size_t WinRandomAccessFile::GetRequiredBufferAlignment() const { + return GetAlignment(); +} + +///////////////////////////////////////////////////////////////////////////// +// WinWritableImpl +// + +inline IOStatus WinWritableImpl::PreallocateInternal(uint64_t spaceToReserve) { + return fallocate(file_data_->GetName(), file_data_->GetFileHandle(), + spaceToReserve); +} + +inline WinWritableImpl::WinWritableImpl(WinFileData* file_data, + size_t alignment) + : file_data_(file_data), + alignment_(std::max(alignment, file_data->GetSectorSize())), + next_write_offset_(0), + reservedsize_(0) { + // Query current position in case ReopenWritableFile is called + // This position is only important for buffered writes + // for unbuffered writes we explicitely specify the position. + LARGE_INTEGER zero_move; + zero_move.QuadPart = 0; // Do not move + LARGE_INTEGER pos; + pos.QuadPart = 0; + BOOL ret = SetFilePointerEx(file_data_->GetFileHandle(), zero_move, &pos, + FILE_CURRENT); + // Querying no supped to fail + if (ret != 0) { + next_write_offset_ = pos.QuadPart; + } else { + assert(false); + } +} + +inline IOStatus WinWritableImpl::AppendImpl(const Slice& data) { + IOStatus s; + + if (data.size() > std::numeric_limits::max()) { + return IOStatus::InvalidArgument("data is too long for a single write" + + file_data_->GetName()); + } + + size_t bytes_written = 0; // out param + + if (file_data_->use_direct_io()) { + // With no offset specified we are appending + // to the end of the file + assert(file_data_->IsSectorAligned(next_write_offset_)); + assert(file_data_->IsSectorAligned(data.size())); + assert(IsAligned(static_cast(GetAlignment()), data.data())); + s = pwrite(file_data_, data, next_write_offset_, bytes_written); + } else { + DWORD bytesWritten = 0; + if (!WriteFile(file_data_->GetFileHandle(), data.data(), + static_cast(data.size()), &bytesWritten, NULL)) { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError( + "Failed to WriteFile: " + file_data_->GetName(), lastError); + } else { + bytes_written = bytesWritten; + } + } + + if (s.ok()) { + if (bytes_written == data.size()) { + // This matters for direct_io cases where + // we rely on the fact that next_write_offset_ + // is sector aligned + next_write_offset_ += bytes_written; + } else { + s = IOStatus::IOError("Failed to write all bytes: " + + file_data_->GetName()); + } + } + + return s; +} + +inline IOStatus WinWritableImpl::PositionedAppendImpl(const Slice& data, + uint64_t offset) { + if (file_data_->use_direct_io()) { + assert(file_data_->IsSectorAligned(static_cast(offset))); + assert(file_data_->IsSectorAligned(data.size())); + assert(IsAligned(static_cast(GetAlignment()), data.data())); + } + + size_t bytes_written = 0; + IOStatus s = pwrite(file_data_, data, offset, bytes_written); + + if (s.ok()) { + if (bytes_written == data.size()) { + // For sequential write this would be simple + // size extension by data.size() + uint64_t write_end = offset + bytes_written; + if (write_end >= next_write_offset_) { + next_write_offset_ = write_end; + } + } else { + s = IOStatus::IOError("Failed to write all of the requested data: " + + file_data_->GetName()); + } + } + return s; +} + +inline IOStatus WinWritableImpl::TruncateImpl(uint64_t size) { + // It is tempting to check for the size for sector alignment + // but truncation may come at the end and there is not a requirement + // for this to be sector aligned so long as we do not attempt to write + // after that. The interface docs state that the behavior is undefined + // in that case. + IOStatus s = + ftruncate(file_data_->GetName(), file_data_->GetFileHandle(), size); + + if (s.ok()) { + next_write_offset_ = size; + } + return s; +} + +inline IOStatus WinWritableImpl::CloseImpl() { + IOStatus s; + + auto hFile = file_data_->GetFileHandle(); + assert(INVALID_HANDLE_VALUE != hFile); + + if (!::FlushFileBuffers(hFile)) { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError( + "FlushFileBuffers failed at Close() for: " + file_data_->GetName(), + lastError); + } + + if (!file_data_->CloseFile() && s.ok()) { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError( + "CloseHandle failed for: " + file_data_->GetName(), lastError); + } + return s; +} + +inline IOStatus WinWritableImpl::SyncImpl(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + IOStatus s; + if (!::FlushFileBuffers(file_data_->GetFileHandle())) { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError( + "FlushFileBuffers failed at Sync() for: " + file_data_->GetName(), + lastError); + } + return s; +} + +inline IOStatus WinWritableImpl::AllocateImpl(uint64_t offset, uint64_t len) { + IOStatus status; + TEST_KILL_RANDOM("WinWritableFile::Allocate"); + + // Make sure that we reserve an aligned amount of space + // since the reservation block size is driven outside so we want + // to check if we are ok with reservation here + size_t spaceToReserve = Roundup(static_cast(offset + len), + static_cast(alignment_)); + // Nothing to do + if (spaceToReserve <= reservedsize_) { + return status; + } + + IOSTATS_TIMER_GUARD(allocate_nanos); + status = PreallocateInternal(spaceToReserve); + if (status.ok()) { + reservedsize_ = spaceToReserve; + } + return status; +} + +//////////////////////////////////////////////////////////////////////////////// +/// WinWritableFile + +WinWritableFile::WinWritableFile(const std::string& fname, HANDLE hFile, + size_t alignment, size_t /* capacity */, + const FileOptions& options) + : WinFileData(fname, hFile, options.use_direct_writes), + WinWritableImpl(this, alignment), + FSWritableFile(options) { + assert(!options.use_mmap_writes); +} + +WinWritableFile::~WinWritableFile() {} + +// Indicates if the class makes use of direct I/O +bool WinWritableFile::use_direct_io() const { + return WinFileData::use_direct_io(); +} + +size_t WinWritableFile::GetRequiredBufferAlignment() const { + return static_cast(GetAlignment()); +} + +IOStatus WinWritableFile::Append(const Slice& data, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + return AppendImpl(data); +} + +IOStatus WinWritableFile::PositionedAppend(const Slice& data, uint64_t offset, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + return PositionedAppendImpl(data, offset); +} + +// Need to implement this so the file is truncated correctly +// when buffered and unbuffered mode +IOStatus WinWritableFile::Truncate(uint64_t size, const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + return TruncateImpl(size); +} + +IOStatus WinWritableFile::Close(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + return CloseImpl(); +} + +// write out the cached data to the OS cache +// This is now taken care of the WritableFileWriter +IOStatus WinWritableFile::Flush(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + return IOStatus::OK(); +} + +IOStatus WinWritableFile::Sync(const IOOptions& options, IODebugContext* dbg) { + return SyncImpl(options, dbg); +} + +IOStatus WinWritableFile::Fsync(const IOOptions& options, IODebugContext* dbg) { + return SyncImpl(options, dbg); +} + +bool WinWritableFile::IsSyncThreadSafe() const { return true; } + +uint64_t WinWritableFile::GetFileSize(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + return GetFileNextWriteOffset(); +} + +IOStatus WinWritableFile::Allocate(uint64_t offset, uint64_t len, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + return AllocateImpl(offset, len); +} + +size_t WinWritableFile::GetUniqueId(char* id, size_t max_size) const { + return GetUniqueIdFromFile(GetFileHandle(), id, max_size); +} + +///////////////////////////////////////////////////////////////////////// +/// WinRandomRWFile + +WinRandomRWFile::WinRandomRWFile(const std::string& fname, HANDLE hFile, + size_t alignment, const FileOptions& options) + : WinFileData(fname, hFile, + options.use_direct_reads && options.use_direct_writes), + WinRandomAccessImpl(this, alignment, options), + WinWritableImpl(this, alignment) {} + +bool WinRandomRWFile::use_direct_io() const { + return WinFileData::use_direct_io(); +} + +size_t WinRandomRWFile::GetRequiredBufferAlignment() const { + assert(WinRandomAccessImpl::GetAlignment() == + WinWritableImpl::GetAlignment()); + return static_cast(WinRandomAccessImpl::GetAlignment()); +} + +IOStatus WinRandomRWFile::Write(uint64_t offset, const Slice& data, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + return PositionedAppendImpl(data, offset); +} + +IOStatus WinRandomRWFile::Read(uint64_t offset, size_t n, + const IOOptions& /*options*/, Slice* result, + char* scratch, IODebugContext* /*dbg*/) const { + return ReadImpl(offset, n, result, scratch); +} + +IOStatus WinRandomRWFile::Flush(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + return IOStatus::OK(); +} + +IOStatus WinRandomRWFile::Sync(const IOOptions& options, IODebugContext* dbg) { + return SyncImpl(options, dbg); +} + +IOStatus WinRandomRWFile::Close(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + return CloseImpl(); +} + +////////////////////////////////////////////////////////////////////////// +/// WinMemoryMappedBufer +WinMemoryMappedBuffer::~WinMemoryMappedBuffer() { + BOOL ret +#if defined(_MSC_VER) + = FALSE; +#else + __attribute__((__unused__)); +#endif + if (base_ != nullptr) { + ret = ::UnmapViewOfFile(base_); + assert(ret); + base_ = nullptr; + } + if (map_handle_ != NULL && map_handle_ != INVALID_HANDLE_VALUE) { + ret = ::CloseHandle(map_handle_); + assert(ret); + map_handle_ = NULL; + } + if (file_handle_ != NULL && file_handle_ != INVALID_HANDLE_VALUE) { + ret = ::CloseHandle(file_handle_); + assert(ret); + file_handle_ = NULL; + } +} + +////////////////////////////////////////////////////////////////////////// +/// WinDirectory + +IOStatus WinDirectory::Fsync(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + return IOStatus::OK(); +} + +IOStatus WinDirectory::Close(const IOOptions& /*options*/, + IODebugContext* /*dbg*/) { + IOStatus s = IOStatus::OK(); + BOOL ret __attribute__((__unused__)); + if (handle_ != INVALID_HANDLE_VALUE) { + ret = ::CloseHandle(handle_); + if (!ret) { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError("Directory closes failed for : " + GetName(), + lastError); + } + handle_ = NULL; + } + return s; +} + +size_t WinDirectory::GetUniqueId(char* id, size_t max_size) const { + return GetUniqueIdFromFile(handle_, id, max_size); +} +////////////////////////////////////////////////////////////////////////// +/// WinFileLock + +WinFileLock::~WinFileLock() { + BOOL ret __attribute__((__unused__)); + ret = ::CloseHandle(hFile_); + assert(ret); +} + +} // namespace port +} // namespace ROCKSDB_NAMESPACE + +#endif diff --git a/librocksdb-sys/rocksdb/port/win/io_win.h b/librocksdb-sys/rocksdb/port/win/io_win.h new file mode 100644 index 0000000..a4fee83 --- /dev/null +++ b/librocksdb-sys/rocksdb/port/win/io_win.h @@ -0,0 +1,508 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once + +#include +#include + +#include +#include + +#include "rocksdb/file_system.h" +#include "rocksdb/status.h" +#include "util/aligned_buffer.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { +namespace port { + +std::string GetWindowsErrSz(DWORD err); + +inline IOStatus IOErrorFromWindowsError(const std::string& context, DWORD err) { + return ((err == ERROR_HANDLE_DISK_FULL) || (err == ERROR_DISK_FULL)) + ? IOStatus::NoSpace(context, GetWindowsErrSz(err)) + : ((err == ERROR_FILE_NOT_FOUND) || (err == ERROR_PATH_NOT_FOUND)) + ? IOStatus::PathNotFound(context, GetWindowsErrSz(err)) + : IOStatus::IOError(context, GetWindowsErrSz(err)); +} + +inline IOStatus IOErrorFromLastWindowsError(const std::string& context) { + return IOErrorFromWindowsError(context, GetLastError()); +} + +inline IOStatus IOError(const std::string& context, int err_number) { + return (err_number == ENOSPC) + ? IOStatus::NoSpace(context, errnoStr(err_number).c_str()) + : (err_number == ENOENT) + ? IOStatus::PathNotFound(context, errnoStr(err_number).c_str()) + : IOStatus::IOError(context, errnoStr(err_number).c_str()); +} + +class WinFileData; + +IOStatus pwrite(const WinFileData* file_data, const Slice& data, + uint64_t offset, size_t& bytes_written); + +IOStatus pread(const WinFileData* file_data, char* src, size_t num_bytes, + uint64_t offset, size_t& bytes_read); + +IOStatus fallocate(const std::string& filename, HANDLE hFile, uint64_t to_size); + +IOStatus ftruncate(const std::string& filename, HANDLE hFile, uint64_t toSize); + +size_t GetUniqueIdFromFile(HANDLE hFile, char* id, size_t max_size); + +class WinFileData { + protected: + const std::string filename_; + HANDLE hFile_; + // If true, the I/O issued would be direct I/O which the buffer + // will need to be aligned (not sure there is a guarantee that the buffer + // passed in is aligned). + const bool use_direct_io_; + const size_t sector_size_; + + public: + // We want this class be usable both for inheritance (prive + // or protected) and for containment so __ctor and __dtor public + WinFileData(const std::string& filename, HANDLE hFile, bool direct_io); + + virtual ~WinFileData() { this->CloseFile(); } + + bool CloseFile() { + bool result = true; + + if (hFile_ != NULL && hFile_ != INVALID_HANDLE_VALUE) { + result = ::CloseHandle(hFile_); + assert(result); + hFile_ = NULL; + } + return result; + } + + const std::string& GetName() const { return filename_; } + + HANDLE GetFileHandle() const { return hFile_; } + + bool use_direct_io() const { return use_direct_io_; } + + size_t GetSectorSize() const { return sector_size_; } + + bool IsSectorAligned(const size_t off) const; + + WinFileData(const WinFileData&) = delete; + WinFileData& operator=(const WinFileData&) = delete; +}; + +class WinSequentialFile : protected WinFileData, public FSSequentialFile { + // Override for behavior change when creating a custom env + virtual IOStatus PositionedReadInternal(char* src, size_t numBytes, + uint64_t offset, + size_t& bytes_read) const; + + public: + WinSequentialFile(const std::string& fname, HANDLE f, + const FileOptions& options); + + ~WinSequentialFile(); + + WinSequentialFile(const WinSequentialFile&) = delete; + WinSequentialFile& operator=(const WinSequentialFile&) = delete; + + IOStatus Read(size_t n, const IOOptions& options, Slice* result, + char* scratch, IODebugContext* dbg) override; + IOStatus PositionedRead(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) override; + + IOStatus Skip(uint64_t n) override; + + IOStatus InvalidateCache(size_t offset, size_t length) override; + + virtual bool use_direct_io() const override { + return WinFileData::use_direct_io(); + } +}; + +// mmap() based random-access +class WinMmapReadableFile : private WinFileData, public FSRandomAccessFile { + HANDLE hMap_; + + const void* mapped_region_; + const size_t length_; + + public: + // mapped_region_[0,length-1] contains the mmapped contents of the file. + WinMmapReadableFile(const std::string& fileName, HANDLE hFile, HANDLE hMap, + const void* mapped_region, size_t length); + + ~WinMmapReadableFile(); + + WinMmapReadableFile(const WinMmapReadableFile&) = delete; + WinMmapReadableFile& operator=(const WinMmapReadableFile&) = delete; + + IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const override; + + virtual IOStatus InvalidateCache(size_t offset, size_t length) override; + + virtual size_t GetUniqueId(char* id, size_t max_size) const override; +}; + +// We preallocate and use memcpy to append new +// data to the file. This is safe since we either properly close the +// file before reading from it, or for log files, the reading code +// knows enough to skip zero suffixes. +class WinMmapFile : private WinFileData, public FSWritableFile { + private: + HANDLE hMap_; + + const size_t page_size_; // We flush the mapping view in page_size + // increments. We may decide if this is a memory + // page size or SSD page size + const size_t + allocation_granularity_; // View must start at such a granularity + + size_t reserved_size_; // Preallocated size + + size_t mapping_size_; // The max size of the mapping object + // we want to guess the final file size to minimize the remapping + size_t view_size_; // How much memory to map into a view at a time + + char* mapped_begin_; // Must begin at the file offset that is aligned with + // allocation_granularity_ + char* mapped_end_; + char* dst_; // Where to write next (in range [mapped_begin_,mapped_end_]) + char* last_sync_; // Where have we synced up to + + uint64_t file_offset_; // Offset of mapped_begin_ in file + + // Do we have unsynced writes? + bool pending_sync_; + + // Can only truncate or reserve to a sector size aligned if + // used on files that are opened with Unbuffered I/O + IOStatus TruncateFile(uint64_t toSize); + + IOStatus UnmapCurrentRegion(); + + IOStatus MapNewRegion(const IOOptions& options, IODebugContext* dbg); + + virtual IOStatus PreallocateInternal(uint64_t spaceToReserve); + + public: + WinMmapFile(const std::string& fname, HANDLE hFile, size_t page_size, + size_t allocation_granularity, const FileOptions& options); + + ~WinMmapFile(); + + WinMmapFile(const WinMmapFile&) = delete; + WinMmapFile& operator=(const WinMmapFile&) = delete; + + IOStatus Append(const Slice& data, const IOOptions& options, + IODebugContext* dbg) override; + IOStatus Append(const Slice& data, const IOOptions& opts, + const DataVerificationInfo& /* verification_info */, + IODebugContext* dbg) override { + return Append(data, opts, dbg); + } + + // Means Close() will properly take care of truncate + // and it does not need any additional information + IOStatus Truncate(uint64_t size, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus Close(const IOOptions& options, IODebugContext* dbg) override; + + IOStatus Flush(const IOOptions& options, IODebugContext* dbg) override; + + // Flush only data + IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override; + + /** + * Flush data as well as metadata to stable storage. + */ + IOStatus Fsync(const IOOptions& options, IODebugContext* dbg) override; + + /** + * Get the size of valid data in the file. This will not match the + * size that is returned from the filesystem because we use mmap + * to extend file by map_size every time. + */ + uint64_t GetFileSize(const IOOptions& options, IODebugContext* dbg) override; + + IOStatus InvalidateCache(size_t offset, size_t length) override; + + IOStatus Allocate(uint64_t offset, uint64_t len, const IOOptions& options, + IODebugContext* dbg) override; + + virtual size_t GetUniqueId(char* id, size_t max_size) const override; +}; + +class WinRandomAccessImpl { + protected: + WinFileData* file_base_; + size_t alignment_; + + // Override for behavior change when creating a custom env + virtual IOStatus PositionedReadInternal(char* src, size_t numBytes, + uint64_t offset, + size_t& bytes_read) const; + + WinRandomAccessImpl(WinFileData* file_base, size_t alignment, + const FileOptions& options); + + virtual ~WinRandomAccessImpl() {} + + IOStatus ReadImpl(uint64_t offset, size_t n, Slice* result, + char* scratch) const; + + size_t GetAlignment() const { return alignment_; } + + public: + WinRandomAccessImpl(const WinRandomAccessImpl&) = delete; + WinRandomAccessImpl& operator=(const WinRandomAccessImpl&) = delete; +}; + +// pread() based random-access +class WinRandomAccessFile + : private WinFileData, + protected WinRandomAccessImpl, // Want to be able to override + // PositionedReadInternal + public FSRandomAccessFile { + public: + WinRandomAccessFile(const std::string& fname, HANDLE hFile, size_t alignment, + const FileOptions& options); + + ~WinRandomAccessFile(); + + IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const override; + + virtual size_t GetUniqueId(char* id, size_t max_size) const override; + + virtual bool use_direct_io() const override { + return WinFileData::use_direct_io(); + } + + IOStatus InvalidateCache(size_t offset, size_t length) override; + + virtual size_t GetRequiredBufferAlignment() const override; +}; + +// This is a sequential write class. It has been mimicked (as others) after +// the original Posix class. We add support for unbuffered I/O on windows as +// well +// we utilize the original buffer as an alignment buffer to write directly to +// file with no buffering. +// No buffering requires that the provided buffer is aligned to the physical +// sector size (SSD page size) and +// that all SetFilePointer() operations to occur with such an alignment. +// We thus always write in sector/page size increments to the drive and leave +// the tail for the next write OR for Close() at which point we pad with zeros. +// No padding is required for +// buffered access. +class WinWritableImpl { + protected: + WinFileData* file_data_; + const uint64_t alignment_; + uint64_t + next_write_offset_; // Needed because Windows does not support O_APPEND + uint64_t reservedsize_; // how far we have reserved space + + virtual IOStatus PreallocateInternal(uint64_t spaceToReserve); + + WinWritableImpl(WinFileData* file_data, size_t alignment); + + ~WinWritableImpl() {} + + uint64_t GetAlignment() const { return alignment_; } + + IOStatus AppendImpl(const Slice& data); + + // Requires that the data is aligned as specified by + // GetRequiredBufferAlignment() + IOStatus PositionedAppendImpl(const Slice& data, uint64_t offset); + + IOStatus TruncateImpl(uint64_t size); + + IOStatus CloseImpl(); + + IOStatus SyncImpl(const IOOptions& options, IODebugContext* dbg); + + uint64_t GetFileNextWriteOffset() { + // Double accounting now here with WritableFileWriter + // and this size will be wrong when unbuffered access is used + // but tests implement their own writable files and do not use + // WritableFileWrapper + // so we need to squeeze a square peg through + // a round hole here. + return next_write_offset_; + } + + IOStatus AllocateImpl(uint64_t offset, uint64_t len); + + public: + WinWritableImpl(const WinWritableImpl&) = delete; + WinWritableImpl& operator=(const WinWritableImpl&) = delete; +}; + +class WinWritableFile : private WinFileData, + protected WinWritableImpl, + public FSWritableFile { + public: + WinWritableFile(const std::string& fname, HANDLE hFile, size_t alignment, + size_t capacity, const FileOptions& options); + + ~WinWritableFile(); + + IOStatus Append(const Slice& data, const IOOptions& options, + IODebugContext* dbg) override; + IOStatus Append(const Slice& data, const IOOptions& opts, + const DataVerificationInfo& /* verification_info */, + IODebugContext* dbg) override { + return Append(data, opts, dbg); + } + + // Requires that the data is aligned as specified by + // GetRequiredBufferAlignment() + IOStatus PositionedAppend(const Slice& data, uint64_t offset, + const IOOptions& options, + IODebugContext* dbg) override; + IOStatus PositionedAppend(const Slice& data, uint64_t offset, + const IOOptions& opts, + const DataVerificationInfo& /* verification_info */, + IODebugContext* dbg) override { + return PositionedAppend(data, offset, opts, dbg); + } + + // Need to implement this so the file is truncated correctly + // when buffered and unbuffered mode + IOStatus Truncate(uint64_t size, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus Close(const IOOptions& options, IODebugContext* dbg) override; + + // write out the cached data to the OS cache + // This is now taken care of the WritableFileWriter + IOStatus Flush(const IOOptions& options, IODebugContext* dbg) override; + + IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override; + + IOStatus Fsync(const IOOptions& options, IODebugContext* dbg) override; + + virtual bool IsSyncThreadSafe() const override; + + // Indicates if the class makes use of direct I/O + // Use PositionedAppend + virtual bool use_direct_io() const override; + + virtual size_t GetRequiredBufferAlignment() const override; + + uint64_t GetFileSize(const IOOptions& options, IODebugContext* dbg) override; + + IOStatus Allocate(uint64_t offset, uint64_t len, const IOOptions& options, + IODebugContext* dbg) override; + + virtual size_t GetUniqueId(char* id, size_t max_size) const override; +}; + +class WinRandomRWFile : private WinFileData, + protected WinRandomAccessImpl, + protected WinWritableImpl, + public FSRandomRWFile { + public: + WinRandomRWFile(const std::string& fname, HANDLE hFile, size_t alignment, + const FileOptions& options); + + ~WinRandomRWFile() {} + + // Indicates if the class makes use of direct I/O + // If false you must pass aligned buffer to Write() + virtual bool use_direct_io() const override; + + // Use the returned alignment value to allocate aligned + // buffer for Write() when use_direct_io() returns true + virtual size_t GetRequiredBufferAlignment() const override; + + // Write bytes in `data` at offset `offset`, Returns Status::OK() on success. + // Pass aligned buffer when use_direct_io() returns true. + IOStatus Write(uint64_t offset, const Slice& data, const IOOptions& options, + IODebugContext* dbg) override; + + // Read up to `n` bytes starting from offset `offset` and store them in + // result, provided `scratch` size should be at least `n`. + // Returns Status::OK() on success. + IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const override; + + IOStatus Flush(const IOOptions& options, IODebugContext* dbg) override; + + IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override; + + IOStatus Fsync(const IOOptions& options, IODebugContext* dbg) override { + return Sync(options, dbg); + } + + IOStatus Close(const IOOptions& options, IODebugContext* dbg) override; +}; + +class WinMemoryMappedBuffer : public MemoryMappedFileBuffer { + private: + HANDLE file_handle_; + HANDLE map_handle_; + + public: + WinMemoryMappedBuffer(HANDLE file_handle, HANDLE map_handle, void* base, + size_t size) + : MemoryMappedFileBuffer(base, size), + file_handle_(file_handle), + map_handle_(map_handle) {} + ~WinMemoryMappedBuffer() override; +}; + +class WinDirectory : public FSDirectory { + const std::string filename_; + HANDLE handle_; + + public: + explicit WinDirectory(const std::string& filename, HANDLE h) noexcept + : filename_(filename), handle_(h) { + assert(handle_ != INVALID_HANDLE_VALUE); + } + ~WinDirectory() { + if (handle_ != NULL) { + IOStatus s = WinDirectory::Close(IOOptions(), nullptr); + s.PermitUncheckedError(); + } + } + const std::string& GetName() const { return filename_; } + IOStatus Fsync(const IOOptions& options, IODebugContext* dbg) override; + IOStatus Close(const IOOptions& options, IODebugContext* dbg) override; + + size_t GetUniqueId(char* id, size_t max_size) const override; +}; + +class WinFileLock : public FileLock { + public: + explicit WinFileLock(HANDLE hFile) : hFile_(hFile) { + assert(hFile != NULL); + assert(hFile != INVALID_HANDLE_VALUE); + } + + ~WinFileLock(); + + private: + HANDLE hFile_; +}; +} // namespace port +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/port/win/port_win.cc b/librocksdb-sys/rocksdb/port/win/port_win.cc new file mode 100644 index 0000000..37e8f65 --- /dev/null +++ b/librocksdb-sys/rocksdb/port/win/port_win.cc @@ -0,0 +1,303 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#if defined(OS_WIN) + +#include "port/win/port_win.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "port/port_dirent.h" +#include "port/sys_time.h" + +#ifdef ROCKSDB_WINDOWS_UTF8_FILENAMES +// utf8 <-> utf16 +#include +#include +#include +#endif + +#include "logging/logging.h" + +namespace ROCKSDB_NAMESPACE { + +extern const bool kDefaultToAdaptiveMutex = false; + +namespace port { + +#ifdef ROCKSDB_WINDOWS_UTF8_FILENAMES +std::string utf16_to_utf8(const std::wstring& utf16) { + std::wstring_convert, wchar_t> convert; + return convert.to_bytes(utf16); +} + +std::wstring utf8_to_utf16(const std::string& utf8) { + std::wstring_convert> converter; + return converter.from_bytes(utf8); +} +#endif + +void GetTimeOfDay(TimeVal* tv, struct timezone* /* tz */) { + std::chrono::microseconds usNow( + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch())); + + std::chrono::seconds secNow( + std::chrono::duration_cast(usNow)); + + tv->tv_sec = static_cast(secNow.count()); + tv->tv_usec = static_cast( + usNow.count() - + std::chrono::duration_cast(secNow).count()); +} + +Mutex::~Mutex() {} + +CondVar::~CondVar() {} + +void CondVar::Wait() { + // Caller must ensure that mutex is held prior to calling this method + std::unique_lock lk(mu_->getLock(), std::adopt_lock); +#ifndef NDEBUG + mu_->locked_ = false; +#endif + cv_.wait(lk); +#ifndef NDEBUG + mu_->locked_ = true; +#endif + // Release ownership of the lock as we don't want it to be unlocked when + // it goes out of scope (as we adopted the lock and didn't lock it ourselves) + lk.release(); +} + +bool CondVar::TimedWait(uint64_t abs_time_us) { + // MSVC++ library implements wait_until in terms of wait_for so + // we need to convert absolute wait into relative wait. + std::chrono::microseconds usAbsTime(abs_time_us); + + std::chrono::microseconds usNow( + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch())); + std::chrono::microseconds relTimeUs = (usAbsTime > usNow) + ? (usAbsTime - usNow) + : std::chrono::microseconds::zero(); + + // Caller must ensure that mutex is held prior to calling this method + std::unique_lock lk(mu_->getLock(), std::adopt_lock); + + // Work around https://github.com/microsoft/STL/issues/369 +#if defined(_MSC_VER) && \ + (!defined(_MSVC_STL_UPDATE) || _MSVC_STL_UPDATE < 202008L) + if (relTimeUs == std::chrono::microseconds::zero()) { + lk.unlock(); + lk.lock(); + } +#endif +#ifndef NDEBUG + mu_->locked_ = false; +#endif + std::cv_status cvStatus = cv_.wait_for(lk, relTimeUs); +#ifndef NDEBUG + mu_->locked_ = true; +#endif + // Release ownership of the lock as we don't want it to be unlocked when + // it goes out of scope (as we adopted the lock and didn't lock it ourselves) + lk.release(); + + if (cvStatus == std::cv_status::timeout) { + return true; + } + + return false; +} + +void CondVar::Signal() { cv_.notify_one(); } + +void CondVar::SignalAll() { cv_.notify_all(); } + +int PhysicalCoreID() { return GetCurrentProcessorNumber(); } + +void InitOnce(OnceType* once, void (*initializer)()) { + std::call_once(once->flag_, initializer); +} + +// Private structure, exposed only by pointer +struct DIR { + HANDLE handle_; + bool firstread_; + RX_WIN32_FIND_DATA data_; + dirent entry_; + + DIR() : handle_(INVALID_HANDLE_VALUE), firstread_(true) {} + + DIR(const DIR&) = delete; + DIR& operator=(const DIR&) = delete; + + ~DIR() { + if (INVALID_HANDLE_VALUE != handle_) { + ::FindClose(handle_); + } + } +}; + +DIR* opendir(const char* name) { + if (!name || *name == 0) { + errno = ENOENT; + return nullptr; + } + + std::string pattern(name); + pattern.append("\\").append("*"); + + std::unique_ptr dir(new DIR); + + dir->handle_ = + RX_FindFirstFileEx(RX_FN(pattern).c_str(), + FindExInfoBasic, // Do not want alternative name + &dir->data_, FindExSearchNameMatch, + NULL, // lpSearchFilter + 0); + + if (dir->handle_ == INVALID_HANDLE_VALUE) { + return nullptr; + } + + RX_FILESTRING x(dir->data_.cFileName, RX_FNLEN(dir->data_.cFileName)); + strcpy_s(dir->entry_.d_name, sizeof(dir->entry_.d_name), FN_TO_RX(x).c_str()); + + return dir.release(); +} + +struct dirent* readdir(DIR* dirp) { + if (!dirp || dirp->handle_ == INVALID_HANDLE_VALUE) { + errno = EBADF; + return nullptr; + } + + if (dirp->firstread_) { + dirp->firstread_ = false; + return &dirp->entry_; + } + + auto ret = RX_FindNextFile(dirp->handle_, &dirp->data_); + + if (ret == 0) { + return nullptr; + } + + RX_FILESTRING x(dirp->data_.cFileName, RX_FNLEN(dirp->data_.cFileName)); + strcpy_s(dirp->entry_.d_name, sizeof(dirp->entry_.d_name), + FN_TO_RX(x).c_str()); + + return &dirp->entry_; +} + +int closedir(DIR* dirp) { + delete dirp; + return 0; +} + +int truncate(const char* path, int64_t length) { + if (path == nullptr) { + errno = EFAULT; + return -1; + } + return ROCKSDB_NAMESPACE::port::Truncate(path, length); +} + +int Truncate(std::string path, int64_t len) { + if (len < 0) { + errno = EINVAL; + return -1; + } + + HANDLE hFile = + RX_CreateFile(RX_FN(path).c_str(), GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, // Security attrs + OPEN_EXISTING, // Truncate existing file only + FILE_ATTRIBUTE_NORMAL, NULL); + + if (INVALID_HANDLE_VALUE == hFile) { + auto lastError = GetLastError(); + if (lastError == ERROR_FILE_NOT_FOUND) { + errno = ENOENT; + } else if (lastError == ERROR_ACCESS_DENIED) { + errno = EACCES; + } else { + errno = EIO; + } + return -1; + } + + int result = 0; + FILE_END_OF_FILE_INFO end_of_file; + end_of_file.EndOfFile.QuadPart = len; + + if (!SetFileInformationByHandle(hFile, FileEndOfFileInfo, &end_of_file, + sizeof(FILE_END_OF_FILE_INFO))) { + errno = EIO; + result = -1; + } + + CloseHandle(hFile); + return result; +} + +void Crash(const std::string& srcfile, int srcline) { + fprintf(stdout, "Crashing at %s:%d\n", srcfile.c_str(), srcline); + fflush(stdout); + abort(); +} + +int GetMaxOpenFiles() { return -1; } + +// Assume 4KB page size +const size_t kPageSize = 4U * 1024U; + +void SetCpuPriority(ThreadId id, CpuPriority priority) { + // Not supported + (void)id; + (void)priority; +} + +int64_t GetProcessID() { return GetCurrentProcessId(); } + +bool GenerateRfcUuid(std::string* output) { + UUID uuid; + UuidCreateSequential(&uuid); + + RPC_CSTR rpc_str; + auto status = UuidToStringA(&uuid, &rpc_str); + if (status != RPC_S_OK) { + return false; + } + + // rpc_str is nul-terminated + *output = reinterpret_cast(rpc_str); + + status = RpcStringFreeA(&rpc_str); + assert(status == RPC_S_OK); + + return true; +} + +} // namespace port +} // namespace ROCKSDB_NAMESPACE + +#endif diff --git a/librocksdb-sys/rocksdb/port/win/port_win.h b/librocksdb-sys/rocksdb/port/win/port_win.h new file mode 100644 index 0000000..4d9883b --- /dev/null +++ b/librocksdb-sys/rocksdb/port/win/port_win.h @@ -0,0 +1,379 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// See port_example.h for documentation for the following types/functions. + +#pragma once + +// Always want minimum headers +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif + +#include +//^^ should be included first before other system lib +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "port/win/win_thread.h" +#include "rocksdb/port_defs.h" + +#undef min +#undef max +#undef DeleteFile +#undef GetCurrentTime + +#ifndef strcasecmp +#define strcasecmp _stricmp +#endif + +#undef GetCurrentTime +#undef DeleteFile + +#ifndef _SSIZE_T_DEFINED +using ssize_t = SSIZE_T; +#endif + +// size_t printf formatting named in the manner of C99 standard formatting +// strings such as PRIu64 +// in fact, we could use that one +#ifndef ROCKSDB_PRIszt +#define ROCKSDB_PRIszt "Iu" +#endif + +#ifdef _MSC_VER +#define __attribute__(A) + +#endif + +namespace ROCKSDB_NAMESPACE { + +#define PREFETCH(addr, rw, locality) + +extern const bool kDefaultToAdaptiveMutex; + +namespace port { + +// "Windows is designed to run on little-endian computer architectures." +// https://docs.microsoft.com/en-us/windows/win32/sysinfo/registry-value-types +constexpr bool kLittleEndian = true; +#undef PLATFORM_IS_LITTLE_ENDIAN + +class CondVar; + +class Mutex { + public: + static const char* kName() { return "std::mutex"; } + + explicit Mutex(bool IGNORED_adaptive = kDefaultToAdaptiveMutex) +#ifndef NDEBUG + : locked_(false) +#endif + { + (void)IGNORED_adaptive; + } + + ~Mutex(); + + void Lock() { + mutex_.lock(); +#ifndef NDEBUG + locked_ = true; +#endif + } + + void Unlock() { +#ifndef NDEBUG + locked_ = false; +#endif + mutex_.unlock(); + } + + bool TryLock() { + bool ret = mutex_.try_lock(); +#ifndef NDEBUG + if (ret) { + locked_ = true; + } +#endif + return ret; + } + + // this will assert if the mutex is not locked + // it does NOT verify that mutex is held by a calling thread + void AssertHeld() { +#ifndef NDEBUG + assert(locked_); +#endif + } + + // Also implement std Lockable + inline void lock() { Lock(); } + inline void unlock() { Unlock(); } + inline bool try_lock() { return TryLock(); } + + // Mutex is move only with lock ownership transfer + Mutex(const Mutex&) = delete; + void operator=(const Mutex&) = delete; + + private: + friend class CondVar; + + std::mutex& getLock() { return mutex_; } + + std::mutex mutex_; +#ifndef NDEBUG + bool locked_; +#endif +}; + +class RWMutex { + public: + RWMutex() { InitializeSRWLock(&srwLock_); } + // No copying allowed + RWMutex(const RWMutex&) = delete; + void operator=(const RWMutex&) = delete; + + void ReadLock() { AcquireSRWLockShared(&srwLock_); } + + void WriteLock() { AcquireSRWLockExclusive(&srwLock_); } + + void ReadUnlock() { ReleaseSRWLockShared(&srwLock_); } + + void WriteUnlock() { ReleaseSRWLockExclusive(&srwLock_); } + + // Empty as in POSIX + void AssertHeld() {} + + private: + SRWLOCK srwLock_; +}; + +class CondVar { + public: + explicit CondVar(Mutex* mu) : mu_(mu) {} + + ~CondVar(); + void Wait(); + bool TimedWait(uint64_t expiration_time); + void Signal(); + void SignalAll(); + + // Condition var is not copy/move constructible + CondVar(const CondVar&) = delete; + CondVar& operator=(const CondVar&) = delete; + + CondVar(CondVar&&) = delete; + CondVar& operator=(CondVar&&) = delete; + + private: + std::condition_variable cv_; + Mutex* mu_; +}; + +#ifdef _POSIX_THREADS +using Thread = std::thread; +#else +// Wrapper around the platform efficient +// or otherwise preferrable implementation +using Thread = WindowsThread; +#endif + +// OnceInit type helps emulate +// Posix semantics with initialization +// adopted in the project +struct OnceType { + struct Init {}; + + OnceType() {} + OnceType(const Init&) {} + OnceType(const OnceType&) = delete; + OnceType& operator=(const OnceType&) = delete; + + std::once_flag flag_; +}; + +#define LEVELDB_ONCE_INIT port::OnceType::Init() +extern void InitOnce(OnceType* once, void (*initializer)()); + +#ifndef CACHE_LINE_SIZE +#define CACHE_LINE_SIZE 64U +#endif + +#ifdef ROCKSDB_JEMALLOC +// Separate inlines so they can be replaced if needed +void* jemalloc_aligned_alloc(size_t size, size_t alignment) noexcept; +void jemalloc_aligned_free(void* p) noexcept; +#endif + +inline void* cacheline_aligned_alloc(size_t size) { +#ifdef ROCKSDB_JEMALLOC + return jemalloc_aligned_alloc(size, CACHE_LINE_SIZE); +#else + return _aligned_malloc(size, CACHE_LINE_SIZE); +#endif +} + +inline void cacheline_aligned_free(void* memblock) { +#ifdef ROCKSDB_JEMALLOC + jemalloc_aligned_free(memblock); +#else + _aligned_free(memblock); +#endif +} + +extern const size_t kPageSize; + +// Part of C++11 +#define ALIGN_AS(n) alignas(n) + +static inline void AsmVolatilePause() { +#if defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) || defined(_M_ARM) + YieldProcessor(); +#endif + // it would be nice to get "wfe" on ARM here +} + +extern int PhysicalCoreID(); + +// For Thread Local Storage abstraction +using pthread_key_t = DWORD; + +inline int pthread_key_create(pthread_key_t* key, void (*destructor)(void*)) { + // Not used + (void)destructor; + + pthread_key_t k = TlsAlloc(); + if (TLS_OUT_OF_INDEXES == k) { + return ENOMEM; + } + + *key = k; + return 0; +} + +inline int pthread_key_delete(pthread_key_t key) { + if (!TlsFree(key)) { + return EINVAL; + } + return 0; +} + +inline int pthread_setspecific(pthread_key_t key, const void* value) { + if (!TlsSetValue(key, const_cast(value))) { + return ENOMEM; + } + return 0; +} + +inline void* pthread_getspecific(pthread_key_t key) { + void* result = TlsGetValue(key); + if (!result) { + if (GetLastError() != ERROR_SUCCESS) { + errno = EINVAL; + } else { + errno = NOERROR; + } + } + return result; +} + +// UNIX equiv although errno numbers will be off +// using C-runtime to implement. Note, this does not +// feel space with zeros in case the file is extended. +int truncate(const char* path, int64_t length); +int Truncate(std::string path, int64_t length); +void Crash(const std::string& srcfile, int srcline); +extern int GetMaxOpenFiles(); +std::string utf16_to_utf8(const std::wstring& utf16); +std::wstring utf8_to_utf16(const std::string& utf8); + +using ThreadId = int; + +extern void SetCpuPriority(ThreadId id, CpuPriority priority); + +int64_t GetProcessID(); + +// Uses platform APIs to generate a 36-character RFC-4122 UUID. Returns +// true on success or false on failure. +bool GenerateRfcUuid(std::string* output); + +} // namespace port + +#ifdef ROCKSDB_WINDOWS_UTF8_FILENAMES + +#define RX_FILESTRING std::wstring +#define RX_FN(a) ROCKSDB_NAMESPACE::port::utf8_to_utf16(a) +#define FN_TO_RX(a) ROCKSDB_NAMESPACE::port::utf16_to_utf8(a) +#define RX_FNCMP(a, b) ::wcscmp(a, RX_FN(b).c_str()) +#define RX_FNLEN(a) ::wcslen(a) + +#define RX_DeleteFile DeleteFileW +#define RX_CreateFile CreateFileW +#define RX_CreateFileMapping CreateFileMappingW +#define RX_GetFileAttributesEx GetFileAttributesExW +#define RX_FindFirstFileEx FindFirstFileExW +#define RX_FindNextFile FindNextFileW +#define RX_WIN32_FIND_DATA WIN32_FIND_DATAW +#define RX_CreateDirectory CreateDirectoryW +#define RX_RemoveDirectory RemoveDirectoryW +#define RX_GetFileAttributesEx GetFileAttributesExW +#define RX_MoveFileEx MoveFileExW +#define RX_CreateHardLink CreateHardLinkW +#define RX_PathIsRelative PathIsRelativeW +#define RX_GetCurrentDirectory GetCurrentDirectoryW +#define RX_GetDiskFreeSpaceEx GetDiskFreeSpaceExW +#define RX_PathIsDirectory PathIsDirectoryW + +#else + +#define RX_FILESTRING std::string +#define RX_FN(a) a +#define FN_TO_RX(a) a +#define RX_FNCMP(a, b) strcmp(a, b) +#define RX_FNLEN(a) strlen(a) + +#define RX_DeleteFile DeleteFileA +#define RX_CreateFile CreateFileA +#define RX_CreateFileMapping CreateFileMappingA +#define RX_GetFileAttributesEx GetFileAttributesExA +#define RX_FindFirstFileEx FindFirstFileExA +#define RX_CreateDirectory CreateDirectoryA +#define RX_FindNextFile FindNextFileA +#define RX_WIN32_FIND_DATA WIN32_FIND_DATAA +#define RX_CreateDirectory CreateDirectoryA +#define RX_RemoveDirectory RemoveDirectoryA +#define RX_GetFileAttributesEx GetFileAttributesExA +#define RX_MoveFileEx MoveFileExA +#define RX_CreateHardLink CreateHardLinkA +#define RX_PathIsRelative PathIsRelativeA +#define RX_GetCurrentDirectory GetCurrentDirectoryA +#define RX_GetDiskFreeSpaceEx GetDiskFreeSpaceExA +#define RX_PathIsDirectory PathIsDirectoryA + +#endif + +using port::pthread_getspecific; +using port::pthread_key_create; +using port::pthread_key_delete; +using port::pthread_key_t; +using port::pthread_setspecific; +using port::truncate; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/port/win/win_jemalloc.cc b/librocksdb-sys/rocksdb/port/win/win_jemalloc.cc new file mode 100644 index 0000000..cf38f55 --- /dev/null +++ b/librocksdb-sys/rocksdb/port/win/win_jemalloc.cc @@ -0,0 +1,80 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#if defined(OS_WIN) + +#ifndef ROCKSDB_JEMALLOC +#error This file can only be part of jemalloc aware build +#endif + +#include + +#include "jemalloc/jemalloc.h" +#include "port/win/port_win.h" + +#if defined(ZSTD) && defined(ZSTD_STATIC_LINKING_ONLY) +#include +#if (ZSTD_VERSION_NUMBER >= 500) +namespace ROCKSDB_NAMESPACE { +namespace port { +void* JemallocAllocateForZSTD(void* /* opaque */, size_t size) { + return je_malloc(size); +} +void JemallocDeallocateForZSTD(void* /* opaque */, void* address) { + je_free(address); +} +ZSTD_customMem GetJeZstdAllocationOverrides() { + return {JemallocAllocateForZSTD, JemallocDeallocateForZSTD, nullptr}; +} +} // namespace port +} // namespace ROCKSDB_NAMESPACE +#endif // (ZSTD_VERSION_NUMBER >= 500) +#endif // defined(ZSTD) defined(ZSTD_STATIC_LINKING_ONLY) + +// Global operators to be replaced by a linker when this file is +// a part of the build + +namespace ROCKSDB_NAMESPACE { +namespace port { +void* jemalloc_aligned_alloc(size_t size, size_t alignment) noexcept { + return je_aligned_alloc(alignment, size); +} +void jemalloc_aligned_free(void* p) noexcept { je_free(p); } +} // namespace port +} // namespace ROCKSDB_NAMESPACE + +void* operator new(size_t size) { + void* p = je_malloc(size); + if (!p) { + throw std::bad_alloc(); + } + return p; +} + +void* operator new[](size_t size) { + void* p = je_malloc(size); + if (!p) { + throw std::bad_alloc(); + } + return p; +} + +void operator delete(void* p) { + if (p) { + je_free(p); + } +} + +void operator delete[](void* p) { + if (p) { + je_free(p); + } +} + +#endif diff --git a/librocksdb-sys/rocksdb/port/win/win_logger.cc b/librocksdb-sys/rocksdb/port/win/win_logger.cc new file mode 100644 index 0000000..072ea41 --- /dev/null +++ b/librocksdb-sys/rocksdb/port/win/win_logger.cc @@ -0,0 +1,192 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Logger implementation that can be shared by all environments +// where enough posix functionality is available. + +#if defined(OS_WIN) + +#include "port/win/win_logger.h" + +#include +#include +#include + +#include +#include + +#include "monitoring/iostats_context_imp.h" +#include "port/sys_time.h" +#include "port/win/env_win.h" +#include "port/win/io_win.h" +#include "rocksdb/env.h" +#include "rocksdb/system_clock.h" + +namespace ROCKSDB_NAMESPACE { + +namespace port { + +WinLogger::WinLogger(uint64_t (*gettid)(), SystemClock* clock, HANDLE file, + const InfoLogLevel log_level) + : Logger(log_level), + file_(file), + gettid_(gettid), + log_size_(0), + last_flush_micros_(0), + clock_(clock), + flush_pending_(false) { + assert(file_ != NULL); + assert(file_ != INVALID_HANDLE_VALUE); +} + +void WinLogger::DebugWriter(const char* str, int len) { + assert(file_ != INVALID_HANDLE_VALUE); + DWORD bytesWritten = 0; + BOOL ret = WriteFile(file_, str, len, &bytesWritten, NULL); + if (ret == FALSE) { + std::string errSz = GetWindowsErrSz(GetLastError()); + fprintf(stderr, "%s", errSz.c_str()); + } +} + +WinLogger::~WinLogger() { CloseInternal().PermitUncheckedError(); } + +Status WinLogger::CloseImpl() { return CloseInternal(); } + +Status WinLogger::CloseInternal() { + Status s; + if (INVALID_HANDLE_VALUE != file_) { + BOOL ret = FlushFileBuffers(file_); + if (ret == 0) { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError("Failed to flush LOG on Close() ", lastError); + } + ret = CloseHandle(file_); + // On error the return value is zero + if (ret == 0 && s.ok()) { + auto lastError = GetLastError(); + s = IOErrorFromWindowsError("Failed to flush LOG on Close() ", lastError); + } + file_ = INVALID_HANDLE_VALUE; + closed_ = true; + } + return s; +} + +void WinLogger::Flush() { + assert(file_ != INVALID_HANDLE_VALUE); + if (flush_pending_) { + flush_pending_ = false; + // With Windows API writes go to OS buffers directly so no fflush needed + // unlike with C runtime API. We don't flush all the way to disk + // for perf reasons. + } + + last_flush_micros_ = clock_->NowMicros(); +} + +void WinLogger::Logv(const char* format, va_list ap) { + IOSTATS_TIMER_GUARD(logger_nanos); + assert(file_ != INVALID_HANDLE_VALUE); + + const uint64_t thread_id = (*gettid_)(); + + // We try twice: the first time with a fixed-size stack allocated buffer, + // and the second time with a much larger dynamically allocated buffer. + char buffer[500]; + std::unique_ptr largeBuffer; + for (int iter = 0; iter < 2; ++iter) { + char* base; + int bufsize; + if (iter == 0) { + bufsize = sizeof(buffer); + base = buffer; + } else { + bufsize = 30000; + largeBuffer.reset(new char[bufsize]); + base = largeBuffer.get(); + } + + char* p = base; + char* limit = base + bufsize; + + port::TimeVal now_tv; + port::GetTimeOfDay(&now_tv, nullptr); + const time_t seconds = now_tv.tv_sec; + struct tm t; + localtime_s(&t, &seconds); + p += snprintf(p, limit - p, "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx ", + t.tm_year + 1900, t.tm_mon + 1, t.tm_mday, t.tm_hour, + t.tm_min, t.tm_sec, static_cast(now_tv.tv_usec), + static_cast(thread_id)); + + // Print the message + if (p < limit) { + va_list backup_ap; + va_copy(backup_ap, ap); + int done = vsnprintf(p, limit - p, format, backup_ap); + if (done > 0) { + p += done; + } else { + continue; + } + va_end(backup_ap); + } + + // Truncate to available space if necessary + if (p >= limit) { + if (iter == 0) { + continue; // Try again with larger buffer + } else { + p = limit - 1; + } + } + + // Add newline if necessary + if (p == base || p[-1] != '\n') { + *p++ = '\n'; + } + + assert(p <= limit); + const size_t write_size = p - base; + + DWORD bytesWritten = 0; + BOOL ret = WriteFile(file_, base, static_cast(write_size), + &bytesWritten, NULL); + if (ret == FALSE) { + std::string errSz = GetWindowsErrSz(GetLastError()); + fprintf(stderr, "%s", errSz.c_str()); + } + + flush_pending_ = true; + assert((bytesWritten == write_size) || (ret == FALSE)); + if (bytesWritten > 0) { + log_size_ += write_size; + } + + uint64_t now_micros = + static_cast(now_tv.tv_sec) * 1000000 + now_tv.tv_usec; + if (now_micros - last_flush_micros_ >= flush_every_seconds_ * 1000000) { + flush_pending_ = false; + // With Windows API writes go to OS buffers directly so no fflush needed + // unlike with C runtime API. We don't flush all the way to disk + // for perf reasons. + last_flush_micros_ = now_micros; + } + break; + } +} + +size_t WinLogger::GetLogFileSize() const { return log_size_; } + +} // namespace port + +} // namespace ROCKSDB_NAMESPACE + +#endif diff --git a/librocksdb-sys/rocksdb/port/win/win_logger.h b/librocksdb-sys/rocksdb/port/win/win_logger.h new file mode 100644 index 0000000..1ca4610 --- /dev/null +++ b/librocksdb-sys/rocksdb/port/win/win_logger.h @@ -0,0 +1,64 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Logger implementation that can be shared by all environments +// where enough posix functionality is available. + +#pragma once + +#include +#include + +#include +#include + +#include "rocksdb/env.h" + +namespace ROCKSDB_NAMESPACE { +class SystemClock; + +namespace port { +class WinLogger : public ROCKSDB_NAMESPACE::Logger { + public: + WinLogger(uint64_t (*gettid)(), SystemClock* clock, HANDLE file, + const InfoLogLevel log_level = InfoLogLevel::ERROR_LEVEL); + + virtual ~WinLogger(); + + WinLogger(const WinLogger&) = delete; + + WinLogger& operator=(const WinLogger&) = delete; + + void Flush() override; + + using ROCKSDB_NAMESPACE::Logger::Logv; + void Logv(const char* format, va_list ap) override; + + size_t GetLogFileSize() const override; + + void DebugWriter(const char* str, int len); + + protected: + Status CloseImpl() override; + + private: + HANDLE file_; + uint64_t (*gettid_)(); // Return the thread id for the current thread + std::atomic_size_t log_size_; + std::atomic_uint_fast64_t last_flush_micros_; + SystemClock* clock_; + bool flush_pending_; + + Status CloseInternal(); + + const static uint64_t flush_every_seconds_ = 5; +}; +} // namespace port + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/port/win/win_thread.cc b/librocksdb-sys/rocksdb/port/win/win_thread.cc new file mode 100644 index 0000000..3c82e73 --- /dev/null +++ b/librocksdb-sys/rocksdb/port/win/win_thread.cc @@ -0,0 +1,170 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#if defined(OS_WIN) +// Most Mingw builds support std::thread only when using posix threads. +// In that case, some of these functions will be unavailable. +// Note that we're using either WindowsThread or std::thread, depending on +// which one is available. +#ifndef _POSIX_THREADS + +#include "port/win/win_thread.h" + +#include +#include // __beginthreadex +#include + +#include +#include +#include + +namespace ROCKSDB_NAMESPACE { +namespace port { + +struct WindowsThread::Data { + std::function func_; + uintptr_t handle_; + + Data(std::function&& func) : func_(std::move(func)), handle_(0) {} + + Data(const Data&) = delete; + Data& operator=(const Data&) = delete; + + static unsigned int __stdcall ThreadProc(void* arg); +}; + +void WindowsThread::Init(std::function&& func) { + data_ = std::make_shared(std::move(func)); + // We create another instance of std::shared_ptr to get an additional ref + // since we may detach and destroy this instance before the threadproc + // may start to run. We choose to allocate this additional ref on the heap + // so we do not need to synchronize and allow this thread to proceed + std::unique_ptr> th_data( + new std::shared_ptr(data_)); + + data_->handle_ = _beginthreadex(NULL, + 0, // stack size + &Data::ThreadProc, th_data.get(), + 0, // init flag + &th_id_); + + if (data_->handle_ == 0) { + throw std::system_error( + std::make_error_code(std::errc::resource_unavailable_try_again), + "Unable to create a thread"); + } + th_data.release(); +} + +WindowsThread::WindowsThread() : data_(nullptr), th_id_(0) {} + +WindowsThread::~WindowsThread() { + // Must be joined or detached + // before destruction. + // This is the same as std::thread + if (data_) { + if (joinable()) { + assert(false); + std::terminate(); + } + data_.reset(); + } +} + +WindowsThread::WindowsThread(WindowsThread&& o) noexcept : WindowsThread() { + *this = std::move(o); +} + +WindowsThread& WindowsThread::operator=(WindowsThread&& o) noexcept { + if (joinable()) { + assert(false); + std::terminate(); + } + + data_ = std::move(o.data_); + + // Per spec both instances will have the same id + th_id_ = o.th_id_; + + return *this; +} + +bool WindowsThread::joinable() const { return (data_ && data_->handle_ != 0); } + +WindowsThread::native_handle_type WindowsThread::native_handle() const { + return reinterpret_cast(data_->handle_); +} + +unsigned WindowsThread::hardware_concurrency() { + return std::thread::hardware_concurrency(); +} + +void WindowsThread::join() { + if (!joinable()) { + assert(false); + throw std::system_error(std::make_error_code(std::errc::invalid_argument), + "Thread is no longer joinable"); + } + + if (GetThreadId(GetCurrentThread()) == th_id_) { + assert(false); + throw std::system_error( + std::make_error_code(std::errc::resource_deadlock_would_occur), + "Can not join itself"); + } + + auto ret = + WaitForSingleObject(reinterpret_cast(data_->handle_), INFINITE); + if (ret != WAIT_OBJECT_0) { + auto lastError = GetLastError(); + assert(false); + throw std::system_error(static_cast(lastError), std::system_category(), + "WaitForSingleObjectFailed: thread join"); + } + + BOOL rc +#if defined(_MSC_VER) + = FALSE; +#else + __attribute__((__unused__)); +#endif + rc = CloseHandle(reinterpret_cast(data_->handle_)); + assert(rc != 0); + data_->handle_ = 0; +} + +bool WindowsThread::detach() { + if (!joinable()) { + assert(false); + throw std::system_error(std::make_error_code(std::errc::invalid_argument), + "Thread is no longer available"); + } + + BOOL ret = CloseHandle(reinterpret_cast(data_->handle_)); + data_->handle_ = 0; + + return (ret != 0); +} + +void WindowsThread::swap(WindowsThread& o) { + data_.swap(o.data_); + std::swap(th_id_, o.th_id_); +} + +unsigned int __stdcall WindowsThread::Data::ThreadProc(void* arg) { + auto ptr = reinterpret_cast*>(arg); + std::unique_ptr> data(ptr); + (*data)->func_(); + return 0; +} +} // namespace port +} // namespace ROCKSDB_NAMESPACE + +#endif // !_POSIX_THREADS +#endif // OS_WIN diff --git a/librocksdb-sys/rocksdb/port/win/win_thread.h b/librocksdb-sys/rocksdb/port/win/win_thread.h new file mode 100644 index 0000000..916033b --- /dev/null +++ b/librocksdb-sys/rocksdb/port/win/win_thread.h @@ -0,0 +1,117 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#ifndef _POSIX_THREADS + +#include +#include +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { +namespace port { + +// This class is a replacement for std::thread +// 2 reasons we do not like std::thread: +// -- is that it dynamically allocates its internals that are automatically +// freed when the thread terminates and not on the destruction of the +// object. This makes it difficult to control the source of memory +// allocation +// - This implements Pimpl so we can easily replace the guts of the +// object in our private version if necessary. +class WindowsThread { + struct Data; + + std::shared_ptr data_; + unsigned int th_id_; + + void Init(std::function&&); + + public: + using native_handle_type = void*; + + // Construct with no thread + WindowsThread(); + + // Template constructor + // + // This templated constructor accomplishes several things + // + // - Allows the class as whole to be not a template + // + // - take "universal" references to support both _lvalues and _rvalues + // + // - because this constructor is a catchall case in many respects it + // may prevent us from using both the default __ctor, the move __ctor. + // Also it may circumvent copy __ctor deletion. To work around this + // we make sure this one has at least one argument and eliminate + // it from the overload selection when WindowsThread is the first + // argument. + // + // - construct with Fx(Ax...) with a variable number of types/arguments. + // + // - Gathers together the callable object with its arguments and constructs + // a single callable entity + // + // - Makes use of std::function to convert it to a specification-template + // dependent type that both checks the signature conformance to ensure + // that all of the necessary arguments are provided and allows pimpl + // implementation. + template ::type, WindowsThread>::value>::type> + explicit WindowsThread(Fn&& fx, Args&&... ax) : WindowsThread() { + // Use binder to create a single callable entity + auto binder = std::bind(std::forward(fx), std::forward(ax)...); + // Use std::function to take advantage of the type erasure + // so we can still hide implementation within pimpl + // This also makes sure that the binder signature is compliant + std::function target = binder; + + Init(std::move(target)); + } + + ~WindowsThread(); + + WindowsThread(const WindowsThread&) = delete; + + WindowsThread& operator=(const WindowsThread&) = delete; + + WindowsThread(WindowsThread&&) noexcept; + + WindowsThread& operator=(WindowsThread&&) noexcept; + + bool joinable() const; + + unsigned int get_id() const { return th_id_; } + + native_handle_type native_handle() const; + + static unsigned hardware_concurrency(); + + void join(); + + bool detach(); + + void swap(WindowsThread&); +}; +} // namespace port +} // namespace ROCKSDB_NAMESPACE + +namespace std { +inline void swap(ROCKSDB_NAMESPACE::port::WindowsThread& th1, + ROCKSDB_NAMESPACE::port::WindowsThread& th2) { + th1.swap(th2); +} +} // namespace std + +#endif // !_POSIX_THREADS diff --git a/librocksdb-sys/rocksdb/port/win/xpress_win.cc b/librocksdb-sys/rocksdb/port/win/xpress_win.cc new file mode 100644 index 0000000..21904d5 --- /dev/null +++ b/librocksdb-sys/rocksdb/port/win/xpress_win.cc @@ -0,0 +1,210 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#if defined(OS_WIN) + +#include "port/win/xpress_win.h" + +#include + +#include +#include +#include +#include + +#ifdef XPRESS + +// Put this under ifdef so windows systems w/o this +// can still build +#include + +namespace ROCKSDB_NAMESPACE { +namespace port { +namespace xpress { + +// Helpers +namespace { + +auto CloseCompressorFun = [](void* h) { + if (NULL != h) { + ::CloseCompressor(reinterpret_cast(h)); + } +}; + +auto CloseDecompressorFun = [](void* h) { + if (NULL != h) { + ::CloseDecompressor(reinterpret_cast(h)); + } +}; +} // namespace + +bool Compress(const char* input, size_t length, std::string* output) { + assert(input != nullptr); + assert(output != nullptr); + + if (length == 0) { + output->clear(); + return true; + } + + COMPRESS_ALLOCATION_ROUTINES* allocRoutinesPtr = nullptr; + + COMPRESSOR_HANDLE compressor = NULL; + + BOOL success = + CreateCompressor(COMPRESS_ALGORITHM_XPRESS, // Compression Algorithm + allocRoutinesPtr, // Optional allocation routine + &compressor); // Handle + + if (!success) { +#ifdef _DEBUG + std::cerr << "XPRESS: Failed to create Compressor LastError: " + << GetLastError() << std::endl; +#endif + return false; + } + + std::unique_ptr compressorGuard( + compressor, CloseCompressorFun); + + SIZE_T compressedBufferSize = 0; + + // Query compressed buffer size. + success = ::Compress(compressor, // Compressor Handle + const_cast(input), // Input buffer + length, // Uncompressed data size + NULL, // Compressed Buffer + 0, // Compressed Buffer size + &compressedBufferSize); // Compressed Data size + + if (!success) { + auto lastError = GetLastError(); + + if (lastError != ERROR_INSUFFICIENT_BUFFER) { +#ifdef _DEBUG + std::cerr + << "XPRESS: Failed to estimate compressed buffer size LastError " + << lastError << std::endl; +#endif + return false; + } + } + + assert(compressedBufferSize > 0); + + std::string result; + result.resize(compressedBufferSize); + + SIZE_T compressedDataSize = 0; + + // Compress + success = ::Compress(compressor, // Compressor Handle + const_cast(input), // Input buffer + length, // Uncompressed data size + &result[0], // Compressed Buffer + compressedBufferSize, // Compressed Buffer size + &compressedDataSize); // Compressed Data size + + if (!success) { +#ifdef _DEBUG + std::cerr << "XPRESS: Failed to compress LastError " << GetLastError() + << std::endl; +#endif + return false; + } + + result.resize(compressedDataSize); + output->swap(result); + + return true; +} + +char* Decompress(const char* input_data, size_t input_length, + size_t* uncompressed_size) { + assert(input_data != nullptr); + assert(uncompressed_size != nullptr); + + if (input_length == 0) { + return nullptr; + } + + COMPRESS_ALLOCATION_ROUTINES* allocRoutinesPtr = nullptr; + + DECOMPRESSOR_HANDLE decompressor = NULL; + + BOOL success = + CreateDecompressor(COMPRESS_ALGORITHM_XPRESS, // Compression Algorithm + allocRoutinesPtr, // Optional allocation routine + &decompressor); // Handle + + if (!success) { +#ifdef _DEBUG + std::cerr << "XPRESS: Failed to create Decompressor LastError " + << GetLastError() << std::endl; +#endif + return nullptr; + } + + std::unique_ptr compressorGuard( + decompressor, CloseDecompressorFun); + + SIZE_T decompressedBufferSize = 0; + + success = ::Decompress(decompressor, // Compressor Handle + const_cast(input_data), // Compressed data + input_length, // Compressed data size + NULL, // Buffer set to NULL + 0, // Buffer size set to 0 + &decompressedBufferSize); // Decompressed Data size + + if (!success) { + auto lastError = GetLastError(); + + if (lastError != ERROR_INSUFFICIENT_BUFFER) { +#ifdef _DEBUG + std::cerr + << "XPRESS: Failed to estimate decompressed buffer size LastError " + << lastError << std::endl; +#endif + return nullptr; + } + } + + assert(decompressedBufferSize > 0); + + // The callers are deallocating using delete[] + // thus we must allocate with new[] + std::unique_ptr outputBuffer(new char[decompressedBufferSize]); + + SIZE_T decompressedDataSize = 0; + + success = ::Decompress(decompressor, const_cast(input_data), + input_length, outputBuffer.get(), + decompressedBufferSize, &decompressedDataSize); + + if (!success) { +#ifdef _DEBUG + std::cerr << "XPRESS: Failed to decompress LastError " << GetLastError() + << std::endl; +#endif + return nullptr; + } + + *uncompressed_size = decompressedDataSize; + + // Return the raw buffer to the caller supporting the tradition + return outputBuffer.release(); +} +} // namespace xpress +} // namespace port +} // namespace ROCKSDB_NAMESPACE + +#endif + +#endif diff --git a/librocksdb-sys/rocksdb/port/win/xpress_win.h b/librocksdb-sys/rocksdb/port/win/xpress_win.h new file mode 100644 index 0000000..187adff --- /dev/null +++ b/librocksdb-sys/rocksdb/port/win/xpress_win.h @@ -0,0 +1,26 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { +namespace port { +namespace xpress { + +bool Compress(const char* input, size_t length, std::string* output); + +char* Decompress(const char* input_data, size_t input_length, + size_t* uncompressed_size); +} // namespace xpress +} // namespace port +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/port/xpress.h b/librocksdb-sys/rocksdb/port/xpress.h new file mode 100644 index 0000000..457025f --- /dev/null +++ b/librocksdb-sys/rocksdb/port/xpress.h @@ -0,0 +1,17 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +// Xpress on Windows is implemeted using Win API +#if defined(ROCKSDB_PLATFORM_POSIX) +#error "Xpress compression not implemented" +#elif defined(OS_WIN) +#include "port/win/xpress_win.h" +#endif diff --git a/librocksdb-sys/rocksdb/rocksdb.pc.in b/librocksdb-sys/rocksdb/rocksdb.pc.in new file mode 100644 index 0000000..5217a45 --- /dev/null +++ b/librocksdb-sys/rocksdb/rocksdb.pc.in @@ -0,0 +1,10 @@ +prefix="@CMAKE_INSTALL_PREFIX@" +includedir="${prefix}/@CMAKE_INSTALL_INCLUDEDIR@" +libdir="${prefix}/@CMAKE_INSTALL_LIBDIR@" + +Name: @PROJECT_NAME@ +Description: @PROJECT_DESCRIPTION@ +URL: @PROJECT_HOMEPAGE_URL@ +Version: @PROJECT_VERSION@ +Cflags: -I"${includedir}" +Libs: -L"${libdir}" -lrocksdb diff --git a/librocksdb-sys/rocksdb/src.mk b/librocksdb-sys/rocksdb/src.mk new file mode 100644 index 0000000..7d2663b --- /dev/null +++ b/librocksdb-sys/rocksdb/src.mk @@ -0,0 +1,716 @@ +# These are the sources from which librocksdb.a is built: +LIB_SOURCES = \ + cache/cache.cc \ + cache/cache_entry_roles.cc \ + cache/cache_key.cc \ + cache/cache_helpers.cc \ + cache/cache_reservation_manager.cc \ + cache/charged_cache.cc \ + cache/clock_cache.cc \ + cache/lru_cache.cc \ + cache/compressed_secondary_cache.cc \ + cache/secondary_cache.cc \ + cache/secondary_cache_adapter.cc \ + cache/sharded_cache.cc \ + db/arena_wrapped_db_iter.cc \ + db/blob/blob_contents.cc \ + db/blob/blob_fetcher.cc \ + db/blob/blob_file_addition.cc \ + db/blob/blob_file_builder.cc \ + db/blob/blob_file_cache.cc \ + db/blob/blob_file_garbage.cc \ + db/blob/blob_file_meta.cc \ + db/blob/blob_file_reader.cc \ + db/blob/blob_garbage_meter.cc \ + db/blob/blob_log_format.cc \ + db/blob/blob_log_sequential_reader.cc \ + db/blob/blob_log_writer.cc \ + db/blob/blob_source.cc \ + db/blob/prefetch_buffer_collection.cc \ + db/builder.cc \ + db/c.cc \ + db/column_family.cc \ + db/compaction/compaction.cc \ + db/compaction/compaction_iterator.cc \ + db/compaction/compaction_job.cc \ + db/compaction/compaction_picker.cc \ + db/compaction/compaction_picker_fifo.cc \ + db/compaction/compaction_picker_level.cc \ + db/compaction/compaction_picker_universal.cc \ + db/compaction/compaction_service_job.cc \ + db/compaction/compaction_state.cc \ + db/compaction/compaction_outputs.cc \ + db/compaction/sst_partitioner.cc \ + db/compaction/subcompaction_state.cc \ + db/convenience.cc \ + db/db_filesnapshot.cc \ + db/db_impl/compacted_db_impl.cc \ + db/db_impl/db_impl.cc \ + db/db_impl/db_impl_compaction_flush.cc \ + db/db_impl/db_impl_debug.cc \ + db/db_impl/db_impl_experimental.cc \ + db/db_impl/db_impl_files.cc \ + db/db_impl/db_impl_open.cc \ + db/db_impl/db_impl_readonly.cc \ + db/db_impl/db_impl_secondary.cc \ + db/db_impl/db_impl_write.cc \ + db/db_info_dumper.cc \ + db/db_iter.cc \ + db/dbformat.cc \ + db/error_handler.cc \ + db/event_helpers.cc \ + db/experimental.cc \ + db/external_sst_file_ingestion_job.cc \ + db/file_indexer.cc \ + db/flush_job.cc \ + db/flush_scheduler.cc \ + db/forward_iterator.cc \ + db/import_column_family_job.cc \ + db/internal_stats.cc \ + db/logs_with_prep_tracker.cc \ + db/log_reader.cc \ + db/log_writer.cc \ + db/malloc_stats.cc \ + db/memtable.cc \ + db/memtable_list.cc \ + db/merge_helper.cc \ + db/merge_operator.cc \ + db/output_validator.cc \ + db/periodic_task_scheduler.cc \ + db/range_del_aggregator.cc \ + db/range_tombstone_fragmenter.cc \ + db/repair.cc \ + db/seqno_to_time_mapping.cc \ + db/snapshot_impl.cc \ + db/table_cache.cc \ + db/table_properties_collector.cc \ + db/transaction_log_impl.cc \ + db/trim_history_scheduler.cc \ + db/version_builder.cc \ + db/version_edit.cc \ + db/version_edit_handler.cc \ + db/version_set.cc \ + db/wal_edit.cc \ + db/wal_manager.cc \ + db/wide/wide_column_serialization.cc \ + db/wide/wide_columns.cc \ + db/write_batch.cc \ + db/write_batch_base.cc \ + db/write_controller.cc \ + db/write_stall_stats.cc \ + db/write_thread.cc \ + env/composite_env.cc \ + env/env.cc \ + env/env_chroot.cc \ + env/env_encryption.cc \ + env/env_posix.cc \ + env/file_system.cc \ + env/fs_posix.cc \ + env/fs_remap.cc \ + env/file_system_tracer.cc \ + env/io_posix.cc \ + env/mock_env.cc \ + env/unique_id_gen.cc \ + file/delete_scheduler.cc \ + file/file_prefetch_buffer.cc \ + file/file_util.cc \ + file/filename.cc \ + file/line_file_reader.cc \ + file/random_access_file_reader.cc \ + file/read_write_util.cc \ + file/readahead_raf.cc \ + file/sequence_file_reader.cc \ + file/sst_file_manager_impl.cc \ + file/writable_file_writer.cc \ + logging/auto_roll_logger.cc \ + logging/event_logger.cc \ + logging/log_buffer.cc \ + memory/arena.cc \ + memory/concurrent_arena.cc \ + memory/jemalloc_nodump_allocator.cc \ + memory/memkind_kmem_allocator.cc \ + memory/memory_allocator.cc \ + memtable/alloc_tracker.cc \ + memtable/hash_linklist_rep.cc \ + memtable/hash_skiplist_rep.cc \ + memtable/skiplistrep.cc \ + memtable/vectorrep.cc \ + memtable/write_buffer_manager.cc \ + monitoring/histogram.cc \ + monitoring/histogram_windowing.cc \ + monitoring/in_memory_stats_history.cc \ + monitoring/instrumented_mutex.cc \ + monitoring/iostats_context.cc \ + monitoring/perf_context.cc \ + monitoring/perf_level.cc \ + monitoring/persistent_stats_history.cc \ + monitoring/statistics.cc \ + monitoring/thread_status_impl.cc \ + monitoring/thread_status_updater.cc \ + monitoring/thread_status_updater_debug.cc \ + monitoring/thread_status_util.cc \ + monitoring/thread_status_util_debug.cc \ + options/cf_options.cc \ + options/configurable.cc \ + options/customizable.cc \ + options/db_options.cc \ + options/options.cc \ + options/options_helper.cc \ + options/options_parser.cc \ + port/mmap.cc \ + port/port_posix.cc \ + port/win/env_default.cc \ + port/win/env_win.cc \ + port/win/io_win.cc \ + port/win/port_win.cc \ + port/win/win_logger.cc \ + port/win/win_thread.cc \ + port/stack_trace.cc \ + table/adaptive/adaptive_table_factory.cc \ + table/block_based/binary_search_index_reader.cc \ + table/block_based/block.cc \ + table/block_based/block_based_table_builder.cc \ + table/block_based/block_based_table_factory.cc \ + table/block_based/block_based_table_iterator.cc \ + table/block_based/block_based_table_reader.cc \ + table/block_based/block_builder.cc \ + table/block_based/block_cache.cc \ + table/block_based/block_prefetcher.cc \ + table/block_based/block_prefix_index.cc \ + table/block_based/data_block_hash_index.cc \ + table/block_based/data_block_footer.cc \ + table/block_based/filter_block_reader_common.cc \ + table/block_based/filter_policy.cc \ + table/block_based/flush_block_policy.cc \ + table/block_based/full_filter_block.cc \ + table/block_based/hash_index_reader.cc \ + table/block_based/index_builder.cc \ + table/block_based/index_reader_common.cc \ + table/block_based/parsed_full_filter_block.cc \ + table/block_based/partitioned_filter_block.cc \ + table/block_based/partitioned_index_iterator.cc \ + table/block_based/partitioned_index_reader.cc \ + table/block_based/reader_common.cc \ + table/block_based/uncompression_dict_reader.cc \ + table/block_fetcher.cc \ + table/cuckoo/cuckoo_table_builder.cc \ + table/cuckoo/cuckoo_table_factory.cc \ + table/cuckoo/cuckoo_table_reader.cc \ + table/format.cc \ + table/get_context.cc \ + table/iterator.cc \ + table/merging_iterator.cc \ + table/compaction_merging_iterator.cc \ + table/meta_blocks.cc \ + table/persistent_cache_helper.cc \ + table/plain/plain_table_bloom.cc \ + table/plain/plain_table_builder.cc \ + table/plain/plain_table_factory.cc \ + table/plain/plain_table_index.cc \ + table/plain/plain_table_key_coding.cc \ + table/plain/plain_table_reader.cc \ + table/sst_file_dumper.cc \ + table/sst_file_reader.cc \ + table/sst_file_writer.cc \ + table/table_factory.cc \ + table/table_properties.cc \ + table/two_level_iterator.cc \ + table/unique_id.cc \ + test_util/sync_point.cc \ + test_util/sync_point_impl.cc \ + test_util/transaction_test_util.cc \ + tools/dump/db_dump_tool.cc \ + trace_replay/trace_record_handler.cc \ + trace_replay/trace_record_result.cc \ + trace_replay/trace_record.cc \ + trace_replay/trace_replay.cc \ + trace_replay/block_cache_tracer.cc \ + trace_replay/io_tracer.cc \ + util/async_file_reader.cc \ + util/build_version.cc \ + util/cleanable.cc \ + util/coding.cc \ + util/compaction_job_stats_impl.cc \ + util/comparator.cc \ + util/compression.cc \ + util/compression_context_cache.cc \ + util/concurrent_task_limiter_impl.cc \ + util/crc32c.cc \ + util/crc32c_arm64.cc \ + util/data_structure.cc \ + util/dynamic_bloom.cc \ + util/hash.cc \ + util/murmurhash.cc \ + util/random.cc \ + util/rate_limiter.cc \ + util/ribbon_config.cc \ + util/slice.cc \ + util/file_checksum_helper.cc \ + util/status.cc \ + util/stderr_logger.cc \ + util/string_util.cc \ + util/thread_local.cc \ + util/threadpool_imp.cc \ + util/udt_util.cc \ + util/write_batch_util.cc \ + util/xxhash.cc \ + utilities/agg_merge/agg_merge.cc \ + utilities/backup/backup_engine.cc \ + utilities/blob_db/blob_compaction_filter.cc \ + utilities/blob_db/blob_db.cc \ + utilities/blob_db/blob_db_impl.cc \ + utilities/blob_db/blob_db_impl_filesnapshot.cc \ + utilities/blob_db/blob_file.cc \ + utilities/cache_dump_load.cc \ + utilities/cache_dump_load_impl.cc \ + utilities/cassandra/cassandra_compaction_filter.cc \ + utilities/cassandra/format.cc \ + utilities/cassandra/merge_operator.cc \ + utilities/checkpoint/checkpoint_impl.cc \ + utilities/compaction_filters.cc \ + utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc \ + utilities/convenience/info_log_finder.cc \ + utilities/counted_fs.cc \ + utilities/debug.cc \ + utilities/env_mirror.cc \ + utilities/env_timed.cc \ + utilities/fault_injection_env.cc \ + utilities/fault_injection_fs.cc \ + utilities/fault_injection_secondary_cache.cc \ + utilities/leveldb_options/leveldb_options.cc \ + utilities/memory/memory_util.cc \ + utilities/merge_operators.cc \ + utilities/merge_operators/max.cc \ + utilities/merge_operators/put.cc \ + utilities/merge_operators/sortlist.cc \ + utilities/merge_operators/string_append/stringappend.cc \ + utilities/merge_operators/string_append/stringappend2.cc \ + utilities/merge_operators/uint64add.cc \ + utilities/merge_operators/bytesxor.cc \ + utilities/object_registry.cc \ + utilities/option_change_migration/option_change_migration.cc \ + utilities/options/options_util.cc \ + utilities/persistent_cache/block_cache_tier.cc \ + utilities/persistent_cache/block_cache_tier_file.cc \ + utilities/persistent_cache/block_cache_tier_metadata.cc \ + utilities/persistent_cache/persistent_cache_tier.cc \ + utilities/persistent_cache/volatile_tier_impl.cc \ + utilities/simulator_cache/cache_simulator.cc \ + utilities/simulator_cache/sim_cache.cc \ + utilities/table_properties_collectors/compact_on_deletion_collector.cc \ + utilities/trace/file_trace_reader_writer.cc \ + utilities/trace/replayer_impl.cc \ + utilities/transactions/lock/lock_manager.cc \ + utilities/transactions/lock/point/point_lock_tracker.cc \ + utilities/transactions/lock/point/point_lock_manager.cc \ + utilities/transactions/optimistic_transaction.cc \ + utilities/transactions/optimistic_transaction_db_impl.cc \ + utilities/transactions/pessimistic_transaction.cc \ + utilities/transactions/pessimistic_transaction_db.cc \ + utilities/transactions/snapshot_checker.cc \ + utilities/transactions/transaction_base.cc \ + utilities/transactions/transaction_db_mutex_impl.cc \ + utilities/transactions/transaction_util.cc \ + utilities/transactions/write_prepared_txn.cc \ + utilities/transactions/write_prepared_txn_db.cc \ + utilities/transactions/write_unprepared_txn.cc \ + utilities/transactions/write_unprepared_txn_db.cc \ + utilities/ttl/db_ttl_impl.cc \ + utilities/wal_filter.cc \ + utilities/write_batch_with_index/write_batch_with_index.cc \ + utilities/write_batch_with_index/write_batch_with_index_internal.cc \ + +ifeq (,$(shell $(CXX) -fsyntax-only -maltivec -xc /dev/null 2>&1)) +LIB_SOURCES_ASM =\ + util/crc32c_ppc_asm.S +LIB_SOURCES_C = \ + util/crc32c_ppc.c +else +LIB_SOURCES_ASM = +LIB_SOURCES_C = +endif + +RANGE_TREE_SOURCES =\ + utilities/transactions/lock/range/range_tree/lib/locktree/concurrent_tree.cc \ + utilities/transactions/lock/range/range_tree/lib/locktree/keyrange.cc \ + utilities/transactions/lock/range/range_tree/lib/locktree/lock_request.cc \ + utilities/transactions/lock/range/range_tree/lib/locktree/locktree.cc \ + utilities/transactions/lock/range/range_tree/lib/locktree/manager.cc \ + utilities/transactions/lock/range/range_tree/lib/locktree/range_buffer.cc \ + utilities/transactions/lock/range/range_tree/lib/locktree/treenode.cc \ + utilities/transactions/lock/range/range_tree/lib/locktree/txnid_set.cc \ + utilities/transactions/lock/range/range_tree/lib/locktree/wfg.cc \ + utilities/transactions/lock/range/range_tree/lib/standalone_port.cc \ + utilities/transactions/lock/range/range_tree/lib/util/dbt.cc \ + utilities/transactions/lock/range/range_tree/lib/util/memarena.cc \ + utilities/transactions/lock/range/range_tree/range_tree_lock_manager.cc \ + utilities/transactions/lock/range/range_tree/range_tree_lock_tracker.cc + +TOOL_LIB_SOURCES = \ + tools/io_tracer_parser_tool.cc \ + tools/ldb_cmd.cc \ + tools/ldb_tool.cc \ + tools/sst_dump_tool.cc \ + utilities/blob_db/blob_dump_tool.cc \ + +ANALYZER_LIB_SOURCES = \ + tools/block_cache_analyzer/block_cache_trace_analyzer.cc \ + tools/trace_analyzer_tool.cc \ + +MOCK_LIB_SOURCES = \ + table/mock_table.cc \ + +BENCH_LIB_SOURCES = \ + tools/db_bench_tool.cc \ + tools/simulated_hybrid_file_system.cc \ + +CACHE_BENCH_LIB_SOURCES = \ + cache/cache_bench_tool.cc \ + +STRESS_LIB_SOURCES = \ + db_stress_tool/batched_ops_stress.cc \ + db_stress_tool/cf_consistency_stress.cc \ + db_stress_tool/db_stress_common.cc \ + db_stress_tool/db_stress_driver.cc \ + db_stress_tool/db_stress_gflags.cc \ + db_stress_tool/db_stress_listener.cc \ + db_stress_tool/db_stress_shared_state.cc \ + db_stress_tool/db_stress_stat.cc \ + db_stress_tool/db_stress_test_base.cc \ + db_stress_tool/db_stress_tool.cc \ + db_stress_tool/expected_state.cc \ + db_stress_tool/expected_value.cc \ + db_stress_tool/no_batched_ops_stress.cc \ + db_stress_tool/multi_ops_txns_stress.cc \ + +TEST_LIB_SOURCES = \ + db/db_test_util.cc \ + db/db_with_timestamp_test_util.cc \ + test_util/mock_time_env.cc \ + test_util/secondary_cache_test_util.cc \ + test_util/testharness.cc \ + test_util/testutil.cc \ + utilities/agg_merge/test_agg_merge.cc \ + utilities/cassandra/test_utils.cc \ + +FOLLY_SOURCES = \ + $(FOLLY_DIR)/folly/container/detail/F14Table.cpp \ + $(FOLLY_DIR)/folly/detail/Futex.cpp \ + $(FOLLY_DIR)/folly/lang/SafeAssert.cpp \ + $(FOLLY_DIR)/folly/lang/ToAscii.cpp \ + $(FOLLY_DIR)/folly/ScopeGuard.cpp \ + $(FOLLY_DIR)/folly/synchronization/AtomicNotification.cpp \ + $(FOLLY_DIR)/folly/synchronization/DistributedMutex.cpp \ + $(FOLLY_DIR)/folly/synchronization/ParkingLot.cpp \ + +TOOLS_MAIN_SOURCES = \ + db_stress_tool/db_stress.cc \ + tools/blob_dump.cc \ + tools/block_cache_analyzer/block_cache_trace_analyzer_tool.cc \ + tools/db_repl_stress.cc \ + tools/db_sanity_test.cc \ + tools/ldb.cc \ + tools/io_tracer_parser.cc \ + tools/sst_dump.cc \ + tools/write_stress.cc \ + tools/dump/rocksdb_dump.cc \ + tools/dump/rocksdb_undump.cc \ + tools/trace_analyzer.cc \ + tools/io_tracer_parser_tool.cc \ + +BENCH_MAIN_SOURCES = \ + cache/cache_bench.cc \ + db/range_del_aggregator_bench.cc \ + memtable/memtablerep_bench.cc \ + table/table_reader_bench.cc \ + tools/db_bench.cc \ + util/filter_bench.cc \ + utilities/persistent_cache/persistent_cache_bench.cc \ + #util/log_write_bench.cc \ + +TEST_MAIN_SOURCES = \ + cache/cache_test.cc \ + cache/cache_reservation_manager_test.cc \ + cache/lru_cache_test.cc \ + cache/compressed_secondary_cache_test.cc \ + db/blob/blob_counting_iterator_test.cc \ + db/blob/blob_file_addition_test.cc \ + db/blob/blob_file_builder_test.cc \ + db/blob/blob_file_cache_test.cc \ + db/blob/blob_file_garbage_test.cc \ + db/blob/blob_file_reader_test.cc \ + db/blob/blob_garbage_meter_test.cc \ + db/blob/blob_source_test.cc \ + db/blob/db_blob_basic_test.cc \ + db/blob/db_blob_compaction_test.cc \ + db/blob/db_blob_corruption_test.cc \ + db/blob/db_blob_index_test.cc \ + db/column_family_test.cc \ + db/compact_files_test.cc \ + db/compaction/clipping_iterator_test.cc \ + db/compaction/compaction_iterator_test.cc \ + db/compaction/compaction_job_test.cc \ + db/compaction/compaction_job_stats_test.cc \ + db/compaction/compaction_picker_test.cc \ + db/compaction/compaction_service_test.cc \ + db/compaction/tiered_compaction_test.cc \ + db/comparator_db_test.cc \ + db/corruption_test.cc \ + db/cuckoo_table_db_test.cc \ + db/db_basic_test.cc \ + db/db_block_cache_test.cc \ + db/db_bloom_filter_test.cc \ + db/db_compaction_filter_test.cc \ + db/db_compaction_test.cc \ + db/db_clip_test.cc \ + db/db_dynamic_level_test.cc \ + db/db_encryption_test.cc \ + db/db_flush_test.cc \ + db/db_readonly_with_timestamp_test.cc \ + db/db_with_timestamp_basic_test.cc \ + db/import_column_family_test.cc \ + db/db_inplace_update_test.cc \ + db/db_io_failure_test.cc \ + db/db_iter_test.cc \ + db/db_iter_stress_test.cc \ + db/db_iterator_test.cc \ + db/db_kv_checksum_test.cc \ + db/db_log_iter_test.cc \ + db/db_memtable_test.cc \ + db/db_merge_operator_test.cc \ + db/db_merge_operand_test.cc \ + db/db_options_test.cc \ + db/db_properties_test.cc \ + db/db_range_del_test.cc \ + db/db_rate_limiter_test.cc \ + db/db_secondary_test.cc \ + db/db_sst_test.cc \ + db/db_statistics_test.cc \ + db/db_table_properties_test.cc \ + db/db_tailing_iter_test.cc \ + db/db_test.cc \ + db/db_test2.cc \ + db/db_logical_block_size_cache_test.cc \ + db/db_universal_compaction_test.cc \ + db/db_wal_test.cc \ + db/db_with_timestamp_compaction_test.cc \ + db/db_write_buffer_manager_test.cc \ + db/db_write_test.cc \ + db/dbformat_test.cc \ + db/deletefile_test.cc \ + db/error_handler_fs_test.cc \ + db/external_sst_file_basic_test.cc \ + db/external_sst_file_test.cc \ + db/fault_injection_test.cc \ + db/file_indexer_test.cc \ + db/filename_test.cc \ + db/flush_job_test.cc \ + db/listener_test.cc \ + db/log_test.cc \ + db/manual_compaction_test.cc \ + db/memtable_list_test.cc \ + db/merge_helper_test.cc \ + db/merge_test.cc \ + db/obsolete_files_test.cc \ + db/options_file_test.cc \ + db/perf_context_test.cc \ + db/periodic_task_scheduler_test.cc \ + db/plain_table_db_test.cc \ + db/prefix_test.cc \ + db/repair_test.cc \ + db/range_del_aggregator_test.cc \ + db/range_tombstone_fragmenter_test.cc \ + db/seqno_time_test.cc \ + db/table_properties_collector_test.cc \ + db/version_builder_test.cc \ + db/version_edit_test.cc \ + db/version_set_test.cc \ + db/wal_manager_test.cc \ + db/wide/db_wide_basic_test.cc \ + db/wide/wide_column_serialization_test.cc \ + db/write_batch_test.cc \ + db/write_callback_test.cc \ + db/write_controller_test.cc \ + env/env_basic_test.cc \ + env/env_test.cc \ + env/io_posix_test.cc \ + env/mock_env_test.cc \ + file/delete_scheduler_test.cc \ + file/prefetch_test.cc \ + file/random_access_file_reader_test.cc \ + logging/auto_roll_logger_test.cc \ + logging/env_logger_test.cc \ + logging/event_logger_test.cc \ + memory/arena_test.cc \ + memory/memory_allocator_test.cc \ + memtable/inlineskiplist_test.cc \ + memtable/skiplist_test.cc \ + memtable/write_buffer_manager_test.cc \ + monitoring/histogram_test.cc \ + monitoring/iostats_context_test.cc \ + monitoring/statistics_test.cc \ + monitoring/stats_history_test.cc \ + options/configurable_test.cc \ + options/customizable_test.cc \ + options/options_settable_test.cc \ + options/options_test.cc \ + table/block_based/block_based_table_reader_test.cc \ + table/block_based/block_test.cc \ + table/block_based/data_block_hash_index_test.cc \ + table/block_based/full_filter_block_test.cc \ + table/block_based/partitioned_filter_block_test.cc \ + table/cleanable_test.cc \ + table/cuckoo/cuckoo_table_builder_test.cc \ + table/cuckoo/cuckoo_table_reader_test.cc \ + table/merger_test.cc \ + table/sst_file_reader_test.cc \ + table/table_test.cc \ + table/block_fetcher_test.cc \ + test_util/testutil_test.cc \ + tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc \ + tools/io_tracer_parser_test.cc \ + tools/ldb_cmd_test.cc \ + tools/reduce_levels_test.cc \ + tools/sst_dump_test.cc \ + tools/trace_analyzer_test.cc \ + trace_replay/block_cache_tracer_test.cc \ + trace_replay/io_tracer_test.cc \ + util/autovector_test.cc \ + util/bloom_test.cc \ + util/coding_test.cc \ + util/crc32c_test.cc \ + util/defer_test.cc \ + util/dynamic_bloom_test.cc \ + util/filelock_test.cc \ + util/file_reader_writer_test.cc \ + util/hash_test.cc \ + util/heap_test.cc \ + util/random_test.cc \ + util/rate_limiter_test.cc \ + util/repeatable_thread_test.cc \ + util/ribbon_test.cc \ + util/slice_test.cc \ + util/slice_transform_test.cc \ + util/timer_queue_test.cc \ + util/timer_test.cc \ + util/thread_list_test.cc \ + util/thread_local_test.cc \ + util/udt_util_test.cc \ + util/work_queue_test.cc \ + utilities/agg_merge/agg_merge_test.cc \ + utilities/backup/backup_engine_test.cc \ + utilities/blob_db/blob_db_test.cc \ + utilities/cassandra/cassandra_format_test.cc \ + utilities/cassandra/cassandra_functional_test.cc \ + utilities/cassandra/cassandra_row_merge_test.cc \ + utilities/cassandra/cassandra_serialize_test.cc \ + utilities/checkpoint/checkpoint_test.cc \ + utilities/env_timed_test.cc \ + utilities/memory/memory_test.cc \ + utilities/merge_operators/string_append/stringappend_test.cc \ + utilities/object_registry_test.cc \ + utilities/option_change_migration/option_change_migration_test.cc \ + utilities/options/options_util_test.cc \ + utilities/persistent_cache/hash_table_test.cc \ + utilities/persistent_cache/persistent_cache_test.cc \ + utilities/simulator_cache/cache_simulator_test.cc \ + utilities/simulator_cache/sim_cache_test.cc \ + utilities/table_properties_collectors/compact_on_deletion_collector_test.cc \ + utilities/transactions/optimistic_transaction_test.cc \ + utilities/transactions/lock/range/range_locking_test.cc \ + utilities/transactions/transaction_test.cc \ + utilities/transactions/lock/point/point_lock_manager_test.cc \ + utilities/transactions/write_prepared_transaction_test.cc \ + utilities/transactions/write_unprepared_transaction_test.cc \ + utilities/transactions/write_committed_transaction_ts_test.cc \ + utilities/transactions/timestamped_snapshot_test.cc \ + utilities/ttl/ttl_test.cc \ + utilities/util_merge_operators_test.cc \ + utilities/write_batch_with_index/write_batch_with_index_test.cc \ + +TEST_MAIN_SOURCES_C = \ + db/c_test.c \ + +MICROBENCH_SOURCES = \ + microbench/ribbon_bench.cc \ + microbench/db_basic_bench.cc \ + +JNI_NATIVE_SOURCES = \ + java/rocksjni/backupenginejni.cc \ + java/rocksjni/backup_engine_options.cc \ + java/rocksjni/checkpoint.cc \ + java/rocksjni/clock_cache.cc \ + java/rocksjni/cache.cc \ + java/rocksjni/columnfamilyhandle.cc \ + java/rocksjni/compact_range_options.cc \ + java/rocksjni/compaction_filter.cc \ + java/rocksjni/compaction_filter_factory.cc \ + java/rocksjni/compaction_filter_factory_jnicallback.cc \ + java/rocksjni/compaction_job_info.cc \ + java/rocksjni/compaction_job_stats.cc \ + java/rocksjni/compaction_options.cc \ + java/rocksjni/compaction_options_fifo.cc \ + java/rocksjni/compaction_options_universal.cc \ + java/rocksjni/comparator.cc \ + java/rocksjni/comparatorjnicallback.cc \ + java/rocksjni/compression_options.cc \ + java/rocksjni/concurrent_task_limiter.cc \ + java/rocksjni/config_options.cc \ + java/rocksjni/env.cc \ + java/rocksjni/env_options.cc \ + java/rocksjni/event_listener.cc \ + java/rocksjni/event_listener_jnicallback.cc \ + java/rocksjni/ingest_external_file_options.cc \ + java/rocksjni/filter.cc \ + java/rocksjni/iterator.cc \ + java/rocksjni/jnicallback.cc \ + java/rocksjni/loggerjnicallback.cc \ + java/rocksjni/lru_cache.cc \ + java/rocksjni/memtablejni.cc \ + java/rocksjni/memory_util.cc \ + java/rocksjni/merge_operator.cc \ + java/rocksjni/native_comparator_wrapper_test.cc \ + java/rocksjni/optimistic_transaction_db.cc \ + java/rocksjni/optimistic_transaction_options.cc \ + java/rocksjni/options.cc \ + java/rocksjni/options_util.cc \ + java/rocksjni/persistent_cache.cc \ + java/rocksjni/ratelimiterjni.cc \ + java/rocksjni/remove_emptyvalue_compactionfilterjni.cc \ + java/rocksjni/cassandra_compactionfilterjni.cc \ + java/rocksjni/cassandra_value_operator.cc \ + java/rocksjni/restorejni.cc \ + java/rocksjni/rocks_callback_object.cc \ + java/rocksjni/rocksjni.cc \ + java/rocksjni/rocksdb_exception_test.cc \ + java/rocksjni/slice.cc \ + java/rocksjni/snapshot.cc \ + java/rocksjni/sst_file_manager.cc \ + java/rocksjni/sst_file_writerjni.cc \ + java/rocksjni/sst_file_readerjni.cc \ + java/rocksjni/sst_file_reader_iterator.cc \ + java/rocksjni/sst_partitioner.cc \ + java/rocksjni/statistics.cc \ + java/rocksjni/statisticsjni.cc \ + java/rocksjni/table.cc \ + java/rocksjni/table_filter.cc \ + java/rocksjni/table_filter_jnicallback.cc \ + java/rocksjni/thread_status.cc \ + java/rocksjni/trace_writer.cc \ + java/rocksjni/trace_writer_jnicallback.cc \ + java/rocksjni/transaction.cc \ + java/rocksjni/transaction_db.cc \ + java/rocksjni/transaction_options.cc \ + java/rocksjni/transaction_db_options.cc \ + java/rocksjni/transaction_log.cc \ + java/rocksjni/transaction_notifier.cc \ + java/rocksjni/transaction_notifier_jnicallback.cc \ + java/rocksjni/ttl.cc \ + java/rocksjni/testable_event_listener.cc \ + java/rocksjni/wal_filter.cc \ + java/rocksjni/wal_filter_jnicallback.cc \ + java/rocksjni/write_batch.cc \ + java/rocksjni/writebatchhandlerjnicallback.cc \ + java/rocksjni/write_batch_test.cc \ + java/rocksjni/write_batch_with_index.cc \ + java/rocksjni/write_buffer_manager.cc diff --git a/librocksdb-sys/rocksdb/table/adaptive/adaptive_table_factory.cc b/librocksdb-sys/rocksdb/table/adaptive/adaptive_table_factory.cc new file mode 100644 index 0000000..5a573ca --- /dev/null +++ b/librocksdb-sys/rocksdb/table/adaptive/adaptive_table_factory.cc @@ -0,0 +1,125 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "table/adaptive/adaptive_table_factory.h" + +#include "port/port.h" +#include "table/format.h" +#include "table/table_builder.h" + +namespace ROCKSDB_NAMESPACE { + +AdaptiveTableFactory::AdaptiveTableFactory( + std::shared_ptr table_factory_to_write, + std::shared_ptr block_based_table_factory, + std::shared_ptr plain_table_factory, + std::shared_ptr cuckoo_table_factory) + : table_factory_to_write_(table_factory_to_write), + block_based_table_factory_(block_based_table_factory), + plain_table_factory_(plain_table_factory), + cuckoo_table_factory_(cuckoo_table_factory) { + if (!plain_table_factory_) { + plain_table_factory_.reset(NewPlainTableFactory()); + } + if (!block_based_table_factory_) { + block_based_table_factory_.reset(NewBlockBasedTableFactory()); + } + if (!cuckoo_table_factory_) { + cuckoo_table_factory_.reset(NewCuckooTableFactory()); + } + if (!table_factory_to_write_) { + table_factory_to_write_ = block_based_table_factory_; + } +} + +extern const uint64_t kPlainTableMagicNumber; +extern const uint64_t kLegacyPlainTableMagicNumber; +extern const uint64_t kBlockBasedTableMagicNumber; +extern const uint64_t kLegacyBlockBasedTableMagicNumber; +extern const uint64_t kCuckooTableMagicNumber; + +Status AdaptiveTableFactory::NewTableReader( + const ReadOptions& ro, const TableReaderOptions& table_reader_options, + std::unique_ptr&& file, uint64_t file_size, + std::unique_ptr* table, + bool prefetch_index_and_filter_in_cache) const { + Footer footer; + IOOptions opts; + auto s = + ReadFooterFromFile(opts, file.get(), *table_reader_options.ioptions.fs, + nullptr /* prefetch_buffer */, file_size, &footer); + if (!s.ok()) { + return s; + } + if (footer.table_magic_number() == kPlainTableMagicNumber || + footer.table_magic_number() == kLegacyPlainTableMagicNumber) { + return plain_table_factory_->NewTableReader( + table_reader_options, std::move(file), file_size, table); + } else if (footer.table_magic_number() == kBlockBasedTableMagicNumber || + footer.table_magic_number() == kLegacyBlockBasedTableMagicNumber) { + return block_based_table_factory_->NewTableReader( + ro, table_reader_options, std::move(file), file_size, table, + prefetch_index_and_filter_in_cache); + } else if (footer.table_magic_number() == kCuckooTableMagicNumber) { + return cuckoo_table_factory_->NewTableReader( + table_reader_options, std::move(file), file_size, table); + } else { + return Status::NotSupported("Unidentified table format"); + } +} + +TableBuilder* AdaptiveTableFactory::NewTableBuilder( + const TableBuilderOptions& table_builder_options, + WritableFileWriter* file) const { + return table_factory_to_write_->NewTableBuilder(table_builder_options, file); +} + +std::string AdaptiveTableFactory::GetPrintableOptions() const { + std::string ret; + ret.reserve(20000); + const int kBufferSize = 200; + char buffer[kBufferSize]; + + if (table_factory_to_write_) { + snprintf(buffer, kBufferSize, " write factory (%s) options:\n%s\n", + (table_factory_to_write_->Name() ? table_factory_to_write_->Name() + : ""), + table_factory_to_write_->GetPrintableOptions().c_str()); + ret.append(buffer); + } + if (plain_table_factory_) { + snprintf(buffer, kBufferSize, " %s options:\n%s\n", + plain_table_factory_->Name() ? plain_table_factory_->Name() : "", + plain_table_factory_->GetPrintableOptions().c_str()); + ret.append(buffer); + } + if (block_based_table_factory_) { + snprintf( + buffer, kBufferSize, " %s options:\n%s\n", + (block_based_table_factory_->Name() ? block_based_table_factory_->Name() + : ""), + block_based_table_factory_->GetPrintableOptions().c_str()); + ret.append(buffer); + } + if (cuckoo_table_factory_) { + snprintf(buffer, kBufferSize, " %s options:\n%s\n", + cuckoo_table_factory_->Name() ? cuckoo_table_factory_->Name() : "", + cuckoo_table_factory_->GetPrintableOptions().c_str()); + ret.append(buffer); + } + return ret; +} + +extern TableFactory* NewAdaptiveTableFactory( + std::shared_ptr table_factory_to_write, + std::shared_ptr block_based_table_factory, + std::shared_ptr plain_table_factory, + std::shared_ptr cuckoo_table_factory) { + return new AdaptiveTableFactory(table_factory_to_write, + block_based_table_factory, + plain_table_factory, cuckoo_table_factory); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/adaptive/adaptive_table_factory.h b/librocksdb-sys/rocksdb/table/adaptive/adaptive_table_factory.h new file mode 100644 index 0000000..55c8bca --- /dev/null +++ b/librocksdb-sys/rocksdb/table/adaptive/adaptive_table_factory.h @@ -0,0 +1,56 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + + +#include + +#include "rocksdb/options.h" +#include "rocksdb/table.h" + +namespace ROCKSDB_NAMESPACE { + +struct EnvOptions; + +class Status; +class RandomAccessFile; +class WritableFile; +class Table; +class TableBuilder; + +class AdaptiveTableFactory : public TableFactory { + public: + ~AdaptiveTableFactory() {} + + explicit AdaptiveTableFactory( + std::shared_ptr table_factory_to_write, + std::shared_ptr block_based_table_factory, + std::shared_ptr plain_table_factory, + std::shared_ptr cuckoo_table_factory); + + const char* Name() const override { return "AdaptiveTableFactory"; } + + using TableFactory::NewTableReader; + Status NewTableReader( + const ReadOptions& ro, const TableReaderOptions& table_reader_options, + std::unique_ptr&& file, uint64_t file_size, + std::unique_ptr* table, + bool prefetch_index_and_filter_in_cache = true) const override; + + TableBuilder* NewTableBuilder( + const TableBuilderOptions& table_builder_options, + WritableFileWriter* file) const override; + + std::string GetPrintableOptions() const override; + + private: + std::shared_ptr table_factory_to_write_; + std::shared_ptr block_based_table_factory_; + std::shared_ptr plain_table_factory_; + std::shared_ptr cuckoo_table_factory_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/binary_search_index_reader.cc b/librocksdb-sys/rocksdb/table/block_based/binary_search_index_reader.cc new file mode 100644 index 0000000..50e2ca8 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/binary_search_index_reader.cc @@ -0,0 +1,74 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "table/block_based/binary_search_index_reader.h" + +namespace ROCKSDB_NAMESPACE { +Status BinarySearchIndexReader::Create( + const BlockBasedTable* table, const ReadOptions& ro, + FilePrefetchBuffer* prefetch_buffer, bool use_cache, bool prefetch, + bool pin, BlockCacheLookupContext* lookup_context, + std::unique_ptr* index_reader) { + assert(table != nullptr); + assert(table->get_rep()); + assert(!pin || prefetch); + assert(index_reader != nullptr); + + CachableEntry index_block; + if (prefetch || !use_cache) { + const Status s = + ReadIndexBlock(table, prefetch_buffer, ro, use_cache, + /*get_context=*/nullptr, lookup_context, &index_block); + if (!s.ok()) { + return s; + } + + if (use_cache && !pin) { + index_block.Reset(); + } + } + + index_reader->reset( + new BinarySearchIndexReader(table, std::move(index_block))); + + return Status::OK(); +} + +InternalIteratorBase* BinarySearchIndexReader::NewIterator( + const ReadOptions& read_options, bool /* disable_prefix_seek */, + IndexBlockIter* iter, GetContext* get_context, + BlockCacheLookupContext* lookup_context) { + const BlockBasedTable::Rep* rep = table()->get_rep(); + const bool no_io = (read_options.read_tier == kBlockCacheTier); + CachableEntry index_block; + const Status s = GetOrReadIndexBlock(no_io, get_context, lookup_context, + &index_block, read_options); + if (!s.ok()) { + if (iter != nullptr) { + iter->Invalidate(s); + return iter; + } + + return NewErrorInternalIterator(s); + } + + Statistics* kNullStats = nullptr; + // We don't return pinned data from index blocks, so no need + // to set `block_contents_pinned`. + auto it = index_block.GetValue()->NewIndexIterator( + internal_comparator()->user_comparator(), + rep->get_global_seqno(BlockType::kIndex), iter, kNullStats, true, + index_has_first_key(), index_key_includes_seq(), index_value_is_full(), + false /* block_contents_pinned */, user_defined_timestamps_persisted()); + + assert(it != nullptr); + index_block.TransferTo(it); + + return it; +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/binary_search_index_reader.h b/librocksdb-sys/rocksdb/table/block_based/binary_search_index_reader.h new file mode 100644 index 0000000..d4a611e --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/binary_search_index_reader.h @@ -0,0 +1,48 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once +#include "table/block_based/index_reader_common.h" + +namespace ROCKSDB_NAMESPACE { +// Index that allows binary search lookup for the first key of each block. +// This class can be viewed as a thin wrapper for `Block` class which already +// supports binary search. +class BinarySearchIndexReader : public BlockBasedTable::IndexReaderCommon { + public: + // Read index from the file and create an intance for + // `BinarySearchIndexReader`. + // On success, index_reader will be populated; otherwise it will remain + // unmodified. + static Status Create(const BlockBasedTable* table, const ReadOptions& ro, + FilePrefetchBuffer* prefetch_buffer, bool use_cache, + bool prefetch, bool pin, + BlockCacheLookupContext* lookup_context, + std::unique_ptr* index_reader); + + InternalIteratorBase* NewIterator( + const ReadOptions& read_options, bool /* disable_prefix_seek */, + IndexBlockIter* iter, GetContext* get_context, + BlockCacheLookupContext* lookup_context) override; + + size_t ApproximateMemoryUsage() const override { + size_t usage = ApproximateIndexBlockMemoryUsage(); +#ifdef ROCKSDB_MALLOC_USABLE_SIZE + usage += malloc_usable_size(const_cast(this)); +#else + usage += sizeof(*this); +#endif // ROCKSDB_MALLOC_USABLE_SIZE + return usage; + } + + private: + BinarySearchIndexReader(const BlockBasedTable* t, + CachableEntry&& index_block) + : IndexReaderCommon(t, std::move(index_block)) {} +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/block.cc b/librocksdb-sys/rocksdb/table/block_based/block.cc new file mode 100644 index 0000000..13e3397 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block.cc @@ -0,0 +1,1314 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Decodes the blocks generated by block_builder.cc. + +#include "table/block_based/block.h" + +#include +#include +#include +#include + +#include "monitoring/perf_context_imp.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/comparator.h" +#include "table/block_based/block_prefix_index.h" +#include "table/block_based/data_block_footer.h" +#include "table/format.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { + +// Helper routine: decode the next block entry starting at "p", +// storing the number of shared key bytes, non_shared key bytes, +// and the length of the value in "*shared", "*non_shared", and +// "*value_length", respectively. Will not dereference past "limit". +// +// If any errors are detected, returns nullptr. Otherwise, returns a +// pointer to the key delta (just past the three decoded values). +struct DecodeEntry { + inline const char* operator()(const char* p, const char* limit, + uint32_t* shared, uint32_t* non_shared, + uint32_t* value_length) { + // We need 2 bytes for shared and non_shared size. We also need one more + // byte either for value size or the actual value in case of value delta + // encoding. + assert(limit - p >= 3); + *shared = reinterpret_cast(p)[0]; + *non_shared = reinterpret_cast(p)[1]; + *value_length = reinterpret_cast(p)[2]; + if ((*shared | *non_shared | *value_length) < 128) { + // Fast path: all three values are encoded in one byte each + p += 3; + } else { + if ((p = GetVarint32Ptr(p, limit, shared)) == nullptr) return nullptr; + if ((p = GetVarint32Ptr(p, limit, non_shared)) == nullptr) return nullptr; + if ((p = GetVarint32Ptr(p, limit, value_length)) == nullptr) { + return nullptr; + } + } + + // Using an assert in place of "return null" since we should not pay the + // cost of checking for corruption on every single key decoding + assert(!(static_cast(limit - p) < (*non_shared + *value_length))); + return p; + } +}; + +// Helper routine: similar to DecodeEntry but does not have assertions. +// Instead, returns nullptr so that caller can detect and report failure. +struct CheckAndDecodeEntry { + inline const char* operator()(const char* p, const char* limit, + uint32_t* shared, uint32_t* non_shared, + uint32_t* value_length) { + // We need 2 bytes for shared and non_shared size. We also need one more + // byte either for value size or the actual value in case of value delta + // encoding. + if (limit - p < 3) { + return nullptr; + } + *shared = reinterpret_cast(p)[0]; + *non_shared = reinterpret_cast(p)[1]; + *value_length = reinterpret_cast(p)[2]; + if ((*shared | *non_shared | *value_length) < 128) { + // Fast path: all three values are encoded in one byte each + p += 3; + } else { + if ((p = GetVarint32Ptr(p, limit, shared)) == nullptr) return nullptr; + if ((p = GetVarint32Ptr(p, limit, non_shared)) == nullptr) return nullptr; + if ((p = GetVarint32Ptr(p, limit, value_length)) == nullptr) { + return nullptr; + } + } + + if (static_cast(limit - p) < (*non_shared + *value_length)) { + return nullptr; + } + return p; + } +}; + +struct DecodeKey { + inline const char* operator()(const char* p, const char* limit, + uint32_t* shared, uint32_t* non_shared) { + uint32_t value_length; + return DecodeEntry()(p, limit, shared, non_shared, &value_length); + } +}; + +// In format_version 4, which is used by index blocks, the value size is not +// encoded before the entry, as the value is known to be the handle with the +// known size. +struct DecodeKeyV4 { + inline const char* operator()(const char* p, const char* limit, + uint32_t* shared, uint32_t* non_shared) { + // We need 2 bytes for shared and non_shared size. We also need one more + // byte either for value size or the actual value in case of value delta + // encoding. + if (limit - p < 3) return nullptr; + *shared = reinterpret_cast(p)[0]; + *non_shared = reinterpret_cast(p)[1]; + if ((*shared | *non_shared) < 128) { + // Fast path: all three values are encoded in one byte each + p += 2; + } else { + if ((p = GetVarint32Ptr(p, limit, shared)) == nullptr) return nullptr; + if ((p = GetVarint32Ptr(p, limit, non_shared)) == nullptr) return nullptr; + } + return p; + } +}; + +struct DecodeEntryV4 { + inline const char* operator()(const char* p, const char* limit, + uint32_t* shared, uint32_t* non_shared, + uint32_t* value_length) { + assert(value_length); + + *value_length = 0; + return DecodeKeyV4()(p, limit, shared, non_shared); + } +}; + +void DataBlockIter::NextImpl() { +#ifndef NDEBUG + if (TEST_Corrupt_Callback("DataBlockIter::NextImpl")) return; +#endif + bool is_shared = false; + ParseNextDataKey(&is_shared); + ++cur_entry_idx_; +} + +void MetaBlockIter::NextImpl() { + bool is_shared = false; + ParseNextKey(&is_shared); + ++cur_entry_idx_; +} + +void IndexBlockIter::NextImpl() { + ParseNextIndexKey(); + ++cur_entry_idx_; +} + +void IndexBlockIter::PrevImpl() { + assert(Valid()); + // Scan backwards to a restart point before current_ + const uint32_t original = current_; + while (GetRestartPoint(restart_index_) >= original) { + if (restart_index_ == 0) { + // No more entries + current_ = restarts_; + restart_index_ = num_restarts_; + return; + } + restart_index_--; + } + SeekToRestartPoint(restart_index_); + // Loop until end of current entry hits the start of original entry + while (ParseNextIndexKey() && NextEntryOffset() < original) { + } + --cur_entry_idx_; +} + +void MetaBlockIter::PrevImpl() { + assert(Valid()); + // Scan backwards to a restart point before current_ + const uint32_t original = current_; + while (GetRestartPoint(restart_index_) >= original) { + if (restart_index_ == 0) { + // No more entries + current_ = restarts_; + restart_index_ = num_restarts_; + return; + } + restart_index_--; + } + SeekToRestartPoint(restart_index_); + bool is_shared = false; + // Loop until end of current entry hits the start of original entry + while (ParseNextKey(&is_shared) && + NextEntryOffset() < original) { + } + --cur_entry_idx_; +} + +// Similar to IndexBlockIter::PrevImpl but also caches the prev entries +void DataBlockIter::PrevImpl() { + assert(Valid()); + + assert(prev_entries_idx_ == -1 || + static_cast(prev_entries_idx_) < prev_entries_.size()); + --cur_entry_idx_; + // Check if we can use cached prev_entries_ + if (prev_entries_idx_ > 0 && + prev_entries_[prev_entries_idx_].offset == current_) { + // Read cached CachedPrevEntry + prev_entries_idx_--; + const CachedPrevEntry& current_prev_entry = + prev_entries_[prev_entries_idx_]; + + const char* key_ptr = nullptr; + bool raw_key_cached; + if (current_prev_entry.key_ptr != nullptr) { + // The key is not delta encoded and stored in the data block + key_ptr = current_prev_entry.key_ptr; + raw_key_cached = false; + } else { + // The key is delta encoded and stored in prev_entries_keys_buff_ + key_ptr = prev_entries_keys_buff_.data() + current_prev_entry.key_offset; + raw_key_cached = true; + } + const Slice current_key(key_ptr, current_prev_entry.key_size); + + current_ = current_prev_entry.offset; + // TODO(ajkr): the copy when `raw_key_cached` is done here for convenience, + // not necessity. It is convenient since this class treats keys as pinned + // when `raw_key_` points to an outside buffer. So we cannot allow + // `raw_key_` point into Prev cache as it is a transient outside buffer + // (i.e., keys in it are not actually pinned). + raw_key_.SetKey(current_key, raw_key_cached /* copy */); + value_ = current_prev_entry.value; + + return; + } + + // Clear prev entries cache + prev_entries_idx_ = -1; + prev_entries_.clear(); + prev_entries_keys_buff_.clear(); + + // Scan backwards to a restart point before current_ + const uint32_t original = current_; + while (GetRestartPoint(restart_index_) >= original) { + if (restart_index_ == 0) { + // No more entries + current_ = restarts_; + restart_index_ = num_restarts_; + return; + } + restart_index_--; + } + + SeekToRestartPoint(restart_index_); + + do { + bool is_shared = false; + if (!ParseNextDataKey(&is_shared)) { + break; + } + Slice current_key = raw_key_.GetKey(); + + if (raw_key_.IsKeyPinned()) { + // The key is not delta encoded + prev_entries_.emplace_back(current_, current_key.data(), 0, + current_key.size(), value()); + } else { + // The key is delta encoded, cache decoded key in buffer + size_t new_key_offset = prev_entries_keys_buff_.size(); + prev_entries_keys_buff_.append(current_key.data(), current_key.size()); + + prev_entries_.emplace_back(current_, nullptr, new_key_offset, + current_key.size(), value()); + } + // Loop until end of current entry hits the start of original entry + } while (NextEntryOffset() < original); + prev_entries_idx_ = static_cast(prev_entries_.size()) - 1; +} + +void DataBlockIter::SeekImpl(const Slice& target) { + Slice seek_key = target; + PERF_TIMER_GUARD(block_seek_nanos); + if (data_ == nullptr) { // Not init yet + return; + } + uint32_t index = 0; + bool skip_linear_scan = false; + bool ok = BinarySeek(seek_key, &index, &skip_linear_scan); + + if (!ok) { + return; + } + FindKeyAfterBinarySeek(seek_key, index, skip_linear_scan); +} + +void MetaBlockIter::SeekImpl(const Slice& target) { + Slice seek_key = target; + PERF_TIMER_GUARD(block_seek_nanos); + if (data_ == nullptr) { // Not init yet + return; + } + uint32_t index = 0; + bool skip_linear_scan = false; + bool ok = BinarySeek(seek_key, &index, &skip_linear_scan); + + if (!ok) { + return; + } + FindKeyAfterBinarySeek(seek_key, index, skip_linear_scan); +} + +// Optimized Seek for point lookup for an internal key `target` +// target = "seek_user_key @ type | seqno". +// +// For any type other than kTypeValue, kTypeDeletion, kTypeSingleDeletion, +// kTypeBlobIndex, kTypeWideColumnEntity or kTypeMerge, this function behaves +// identically to Seek(). +// +// For any type in kTypeValue, kTypeDeletion, kTypeSingleDeletion, +// kTypeBlobIndex, kTypeWideColumnEntity, or kTypeMerge: +// +// If the return value is FALSE, iter location is undefined, and it means: +// 1) there is no key in this block falling into the range: +// ["seek_user_key @ type | seqno", "seek_user_key @ kTypeDeletion | 0"], +// inclusive; AND +// 2) the last key of this block has a greater user_key from seek_user_key +// +// If the return value is TRUE, iter location has two possibilities: +// 1) If iter is valid, it is set to a location as if set by SeekImpl(target). +// In this case, it points to the first key with a larger user_key or a +// matching user_key with a seqno no greater than the seeking seqno. +// 2) If the iter is invalid, it means that either all the user_key is less +// than the seek_user_key, or the block ends with a matching user_key but +// with a smaller [ type | seqno ] (i.e. a larger seqno, or the same seqno +// but larger type). +bool DataBlockIter::SeekForGetImpl(const Slice& target) { + Slice target_user_key = ExtractUserKey(target); + uint32_t map_offset = restarts_ + num_restarts_ * sizeof(uint32_t); + uint8_t entry = + data_block_hash_index_->Lookup(data_, map_offset, target_user_key); + + if (entry == kCollision) { + // HashSeek not effective, falling back + SeekImpl(target); + return true; + } + + if (entry == kNoEntry) { + // Even if we cannot find the user_key in this block, the result may + // exist in the next block. Consider this example: + // + // Block N: [aab@100, ... , app@120] + // boundary key: axy@50 (we make minimal assumption about a boundary key) + // Block N+1: [axy@10, ... ] + // + // If seek_key = axy@60, the search will start from Block N. + // Even if the user_key is not found in the hash map, the caller still + // have to continue searching the next block. + // + // In this case, we pretend the key is in the last restart interval. + // The while-loop below will search the last restart interval for the + // key. It will stop at the first key that is larger than the seek_key, + // or to the end of the block if no one is larger. + entry = static_cast(num_restarts_ - 1); + } + + uint32_t restart_index = entry; + + // check if the key is in the restart_interval + assert(restart_index < num_restarts_); + SeekToRestartPoint(restart_index); + current_ = GetRestartPoint(restart_index); + cur_entry_idx_ = + static_cast(restart_index * block_restart_interval_) - 1; + + uint32_t limit = restarts_; + if (restart_index + 1 < num_restarts_) { + limit = GetRestartPoint(restart_index + 1); + } + while (current_ < limit) { + ++cur_entry_idx_; + bool shared; + // Here we only linear seek the target key inside the restart interval. + // If a key does not exist inside a restart interval, we avoid + // further searching the block content across restart interval boundary. + // + // TODO(fwu): check the left and right boundary of the restart interval + // to avoid linear seek a target key that is out of range. + if (!ParseNextDataKey(&shared) || CompareCurrentKey(target) >= 0) { + // we stop at the first potential matching user key. + break; + } + // If the loop exits due to CompareCurrentKey(target) >= 0, then current key + // exists, and its checksum verification will be done in UpdateKey() called + // in SeekForGet(). + // TODO(cbi): If this loop exits with current_ == restart_, per key-value + // checksum will not be verified in UpdateKey() since Valid() + // will return false. + } + + if (current_ == restarts_) { + // Search reaches to the end of the block. There are three possibilities: + // 1) there is only one user_key match in the block (otherwise collision). + // the matching user_key resides in the last restart interval, and it + // is the last key of the restart interval and of the block as well. + // ParseNextKey() skipped it as its [ type | seqno ] is smaller. + // + // 2) The seek_key is not found in the HashIndex Lookup(), i.e. kNoEntry, + // AND all existing user_keys in the restart interval are smaller than + // seek_user_key. + // + // 3) The seek_key is a false positive and happens to be hashed to the + // last restart interval, AND all existing user_keys in the restart + // interval are smaller than seek_user_key. + // + // The result may exist in the next block each case, so we return true. + return true; + } + + if (icmp_->user_comparator()->Compare(raw_key_.GetUserKey(), + target_user_key) != 0) { + // the key is not in this block and cannot be at the next block either. + return false; + } + + // Here we are conservative and only support a limited set of cases + ValueType value_type = ExtractValueType(raw_key_.GetInternalKey()); + if (value_type != ValueType::kTypeValue && + value_type != ValueType::kTypeDeletion && + value_type != ValueType::kTypeMerge && + value_type != ValueType::kTypeSingleDeletion && + value_type != ValueType::kTypeBlobIndex && + value_type != ValueType::kTypeWideColumnEntity) { + SeekImpl(target); + } + + // Result found, and the iter is correctly set. + return true; +} + +void IndexBlockIter::SeekImpl(const Slice& target) { +#ifndef NDEBUG + if (TEST_Corrupt_Callback("IndexBlockIter::SeekImpl")) return; +#endif + TEST_SYNC_POINT("IndexBlockIter::Seek:0"); + PERF_TIMER_GUARD(block_seek_nanos); + if (data_ == nullptr) { // Not init yet + return; + } + Slice seek_key = target; + if (raw_key_.IsUserKey()) { + seek_key = ExtractUserKey(target); + } + status_ = Status::OK(); + uint32_t index = 0; + bool skip_linear_scan = false; + bool ok = false; + if (prefix_index_) { + bool prefix_may_exist = true; + ok = PrefixSeek(target, &index, &prefix_may_exist); + if (!prefix_may_exist) { + // This is to let the caller to distinguish between non-existing prefix, + // and when key is larger than the last key, which both set Valid() to + // false. + current_ = restarts_; + status_ = Status::NotFound(); + } + // restart interval must be one when hash search is enabled so the binary + // search simply lands at the right place. + skip_linear_scan = true; + } else if (value_delta_encoded_) { + ok = BinarySeek(seek_key, &index, &skip_linear_scan); + } else { + ok = BinarySeek(seek_key, &index, &skip_linear_scan); + } + + if (!ok) { + return; + } + FindKeyAfterBinarySeek(seek_key, index, skip_linear_scan); +} + +void DataBlockIter::SeekForPrevImpl(const Slice& target) { + PERF_TIMER_GUARD(block_seek_nanos); + Slice seek_key = target; + if (data_ == nullptr) { // Not init yet + return; + } + uint32_t index = 0; + bool skip_linear_scan = false; + bool ok = BinarySeek(seek_key, &index, &skip_linear_scan); + + if (!ok) { + return; + } + FindKeyAfterBinarySeek(seek_key, index, skip_linear_scan); + + if (!Valid()) { + if (status_.ok()) { + SeekToLastImpl(); + } + } else { + while (Valid() && CompareCurrentKey(seek_key) > 0) { + PrevImpl(); + } + } +} + +void MetaBlockIter::SeekForPrevImpl(const Slice& target) { + PERF_TIMER_GUARD(block_seek_nanos); + Slice seek_key = target; + if (data_ == nullptr) { // Not init yet + return; + } + uint32_t index = 0; + bool skip_linear_scan = false; + bool ok = BinarySeek(seek_key, &index, &skip_linear_scan); + + if (!ok) { + return; + } + FindKeyAfterBinarySeek(seek_key, index, skip_linear_scan); + + if (!Valid()) { + if (status_.ok()) { + SeekToLastImpl(); + } + } else { + while (Valid() && CompareCurrentKey(seek_key) > 0) { + PrevImpl(); + } + } +} + +void DataBlockIter::SeekToFirstImpl() { + if (data_ == nullptr) { // Not init yet + return; + } + SeekToRestartPoint(0); + bool is_shared = false; + ParseNextDataKey(&is_shared); + cur_entry_idx_ = 0; +} + +void MetaBlockIter::SeekToFirstImpl() { + if (data_ == nullptr) { // Not init yet + return; + } + SeekToRestartPoint(0); + bool is_shared = false; + ParseNextKey(&is_shared); + cur_entry_idx_ = 0; +} + +void IndexBlockIter::SeekToFirstImpl() { +#ifndef NDEBUG + if (TEST_Corrupt_Callback("IndexBlockIter::SeekToFirstImpl")) return; +#endif + if (data_ == nullptr) { // Not init yet + return; + } + status_ = Status::OK(); + SeekToRestartPoint(0); + ParseNextIndexKey(); + cur_entry_idx_ = 0; +} + +void DataBlockIter::SeekToLastImpl() { + if (data_ == nullptr) { // Not init yet + return; + } + SeekToRestartPoint(num_restarts_ - 1); + bool is_shared = false; + cur_entry_idx_ = (num_restarts_ - 1) * block_restart_interval_; + while (ParseNextDataKey(&is_shared) && NextEntryOffset() < restarts_) { + // Keep skipping + ++cur_entry_idx_; + } +} + +void MetaBlockIter::SeekToLastImpl() { + if (data_ == nullptr) { // Not init yet + return; + } + SeekToRestartPoint(num_restarts_ - 1); + bool is_shared = false; + assert(num_restarts_ >= 1); + cur_entry_idx_ = + static_cast((num_restarts_ - 1) * block_restart_interval_); + while (ParseNextKey(&is_shared) && + NextEntryOffset() < restarts_) { + // Will probably never reach here since restart_interval is always 1 + ++cur_entry_idx_; + } +} + +void IndexBlockIter::SeekToLastImpl() { + if (data_ == nullptr) { // Not init yet + return; + } + status_ = Status::OK(); + SeekToRestartPoint(num_restarts_ - 1); + cur_entry_idx_ = (num_restarts_ - 1) * block_restart_interval_; + while (ParseNextIndexKey() && NextEntryOffset() < restarts_) { + ++cur_entry_idx_; + } +} + +template +template +bool BlockIter::ParseNextKey(bool* is_shared) { + current_ = NextEntryOffset(); + const char* p = data_ + current_; + const char* limit = data_ + restarts_; // Restarts come right after data + + if (p >= limit) { + // No more entries to return. Mark as invalid. + current_ = restarts_; + restart_index_ = num_restarts_; + return false; + } + // Decode next entry + uint32_t shared, non_shared, value_length; + p = DecodeEntryFunc()(p, limit, &shared, &non_shared, &value_length); + if (p == nullptr || raw_key_.Size() < shared) { + CorruptionError(); + return false; + } else { + if (shared == 0) { + *is_shared = false; + // If this key doesn't share any bytes with prev key, and no min timestamp + // needs to be padded to the key, then we don't need to decode it and + // can use its address in the block directly (no copy). + UpdateRawKeyAndMaybePadMinTimestamp(Slice(p, non_shared)); + } else { + // This key share `shared` bytes with prev key, we need to decode it + *is_shared = true; + // If user-defined timestamp is stripped from user key before keys are + // delta encoded, the decoded key consisting of the shared and non shared + // bytes do not have user-defined timestamp yet. We need to pad min + // timestamp to it. + if (pad_min_timestamp_) { + raw_key_.TrimAppendWithTimestamp(shared, p, non_shared, ts_sz_); + } else { + raw_key_.TrimAppend(shared, p, non_shared); + } + } + value_ = Slice(p + non_shared, value_length); + if (shared == 0) { + while (restart_index_ + 1 < num_restarts_ && + GetRestartPoint(restart_index_ + 1) < current_) { + ++restart_index_; + } + } + // else we are in the middle of a restart interval and the restart_index_ + // thus has not changed + return true; + } +} + +bool DataBlockIter::ParseNextDataKey(bool* is_shared) { + if (ParseNextKey(is_shared)) { +#ifndef NDEBUG + if (global_seqno_ != kDisableGlobalSequenceNumber) { + // If we are reading a file with a global sequence number we should + // expect that all encoded sequence numbers are zeros and any value + // type is kTypeValue, kTypeMerge, kTypeDeletion, + // kTypeDeletionWithTimestamp, or kTypeRangeDeletion. + uint64_t packed = ExtractInternalKeyFooter(raw_key_.GetKey()); + SequenceNumber seqno; + ValueType value_type; + UnPackSequenceAndType(packed, &seqno, &value_type); + assert(value_type == ValueType::kTypeValue || + value_type == ValueType::kTypeMerge || + value_type == ValueType::kTypeDeletion || + value_type == ValueType::kTypeDeletionWithTimestamp || + value_type == ValueType::kTypeRangeDeletion); + assert(seqno == 0); + } +#endif // NDEBUG + return true; + } else { + return false; + } +} + +bool IndexBlockIter::ParseNextIndexKey() { + bool is_shared = false; + bool ok = (value_delta_encoded_) ? ParseNextKey(&is_shared) + : ParseNextKey(&is_shared); + if (ok) { + if (value_delta_encoded_ || global_seqno_state_ != nullptr || + pad_min_timestamp_) { + DecodeCurrentValue(is_shared); + } + } + return ok; +} + +// The format: +// restart_point 0: k, v (off, sz), k, v (delta-sz), ..., k, v (delta-sz) +// restart_point 1: k, v (off, sz), k, v (delta-sz), ..., k, v (delta-sz) +// ... +// restart_point n-1: k, v (off, sz), k, v (delta-sz), ..., k, v (delta-sz) +// where, k is key, v is value, and its encoding is in parentheses. +// The format of each key is (shared_size, non_shared_size, shared, non_shared) +// The format of each value, i.e., block handle, is (offset, size) whenever the +// is_shared is false, which included the first entry in each restart point. +// Otherwise, the format is delta-size = the size of current block - the size o +// last block. +void IndexBlockIter::DecodeCurrentValue(bool is_shared) { + Slice v(value_.data(), data_ + restarts_ - value_.data()); + // Delta encoding is used if `shared` != 0. + Status decode_s __attribute__((__unused__)) = decoded_value_.DecodeFrom( + &v, have_first_key_, + (value_delta_encoded_ && is_shared) ? &decoded_value_.handle : nullptr); + assert(decode_s.ok()); + value_ = Slice(value_.data(), v.data() - value_.data()); + + if (global_seqno_state_ != nullptr) { + // Overwrite sequence number the same way as in DataBlockIter. + + IterKey& first_internal_key = global_seqno_state_->first_internal_key; + first_internal_key.SetInternalKey(decoded_value_.first_internal_key, + /* copy */ true); + + assert(GetInternalKeySeqno(first_internal_key.GetInternalKey()) == 0); + + ValueType value_type = ExtractValueType(first_internal_key.GetKey()); + assert(value_type == ValueType::kTypeValue || + value_type == ValueType::kTypeMerge || + value_type == ValueType::kTypeDeletion || + value_type == ValueType::kTypeRangeDeletion); + + first_internal_key.UpdateInternalKey(global_seqno_state_->global_seqno, + value_type); + decoded_value_.first_internal_key = first_internal_key.GetKey(); + } + if (pad_min_timestamp_ && !decoded_value_.first_internal_key.empty()) { + first_internal_key_with_ts_.clear(); + PadInternalKeyWithMinTimestamp(&first_internal_key_with_ts_, + decoded_value_.first_internal_key, ts_sz_); + decoded_value_.first_internal_key = first_internal_key_with_ts_; + } +} + +template +void BlockIter::FindKeyAfterBinarySeek(const Slice& target, + uint32_t index, + bool skip_linear_scan) { + // SeekToRestartPoint() only does the lookup in the restart block. We need + // to follow it up with NextImpl() to position the iterator at the restart + // key. + SeekToRestartPoint(index); + cur_entry_idx_ = static_cast(index * block_restart_interval_) - 1; + NextImpl(); + + if (!skip_linear_scan) { + // Linear search (within restart block) for first key >= target + uint32_t max_offset; + if (index + 1 < num_restarts_) { + // We are in a non-last restart interval. Since `BinarySeek()` guarantees + // the next restart key is strictly greater than `target`, we can + // terminate upon reaching it without any additional key comparison. + max_offset = GetRestartPoint(index + 1); + } else { + // We are in the last restart interval. The while-loop will terminate by + // `Valid()` returning false upon advancing past the block's last key. + max_offset = std::numeric_limits::max(); + } + while (true) { + NextImpl(); + if (!Valid()) { + // TODO(cbi): per key-value checksum will not be verified in UpdateKey() + // since Valid() will returns false. + break; + } + if (current_ == max_offset) { + assert(CompareCurrentKey(target) > 0); + break; + } else if (CompareCurrentKey(target) >= 0) { + break; + } + } + } +} + +// Binary searches in restart array to find the starting restart point for the +// linear scan, and stores it in `*index`. Assumes restart array does not +// contain duplicate keys. It is guaranteed that the restart key at `*index + 1` +// is strictly greater than `target` or does not exist (this can be used to +// elide a comparison when linear scan reaches all the way to the next restart +// key). Furthermore, `*skip_linear_scan` is set to indicate whether the +// `*index`th restart key is the final result so that key does not need to be +// compared again later. +template +template +bool BlockIter::BinarySeek(const Slice& target, uint32_t* index, + bool* skip_linear_scan) { + if (restarts_ == 0) { + // SST files dedicated to range tombstones are written with index blocks + // that have no keys while also having `num_restarts_ == 1`. This would + // cause a problem for `BinarySeek()` as it'd try to access the first key + // which does not exist. We identify such blocks by the offset at which + // their restarts are stored, and return false to prevent any attempted + // key accesses. + return false; + } + + *skip_linear_scan = false; + // Loop invariants: + // - Restart key at index `left` is less than or equal to the target key. The + // sentinel index `-1` is considered to have a key that is less than all + // keys. + // - Any restart keys after index `right` are strictly greater than the target + // key. + int64_t left = -1, right = num_restarts_ - 1; + while (left != right) { + // The `mid` is computed by rounding up so it lands in (`left`, `right`]. + int64_t mid = left + (right - left + 1) / 2; + uint32_t region_offset = GetRestartPoint(static_cast(mid)); + uint32_t shared, non_shared; + const char* key_ptr = DecodeKeyFunc()( + data_ + region_offset, data_ + restarts_, &shared, &non_shared); + if (key_ptr == nullptr || (shared != 0)) { + CorruptionError(); + return false; + } + Slice mid_key(key_ptr, non_shared); + UpdateRawKeyAndMaybePadMinTimestamp(mid_key); + int cmp = CompareCurrentKey(target); + if (cmp < 0) { + // Key at "mid" is smaller than "target". Therefore all + // blocks before "mid" are uninteresting. + left = mid; + } else if (cmp > 0) { + // Key at "mid" is >= "target". Therefore all blocks at or + // after "mid" are uninteresting. + right = mid - 1; + } else { + *skip_linear_scan = true; + left = right = mid; + } + } + + if (left == -1) { + // All keys in the block were strictly greater than `target`. So the very + // first key in the block is the final seek result. + *skip_linear_scan = true; + *index = 0; + } else { + *index = static_cast(left); + } + return true; +} + +// Compare target key and the block key of the block of `block_index`. +// Return -1 if error. +int IndexBlockIter::CompareBlockKey(uint32_t block_index, const Slice& target) { + uint32_t region_offset = GetRestartPoint(block_index); + uint32_t shared, non_shared; + const char* key_ptr = + value_delta_encoded_ + ? DecodeKeyV4()(data_ + region_offset, data_ + restarts_, &shared, + &non_shared) + : DecodeKey()(data_ + region_offset, data_ + restarts_, &shared, + &non_shared); + if (key_ptr == nullptr || (shared != 0)) { + CorruptionError(); + return 1; // Return target is smaller + } + Slice block_key(key_ptr, non_shared); + UpdateRawKeyAndMaybePadMinTimestamp(block_key); + return CompareCurrentKey(target); +} + +// Binary search in block_ids to find the first block +// with a key >= target +bool IndexBlockIter::BinaryBlockIndexSeek(const Slice& target, + uint32_t* block_ids, uint32_t left, + uint32_t right, uint32_t* index, + bool* prefix_may_exist) { + assert(left <= right); + assert(index); + assert(prefix_may_exist); + *prefix_may_exist = true; + uint32_t left_bound = left; + + while (left <= right) { + uint32_t mid = (right + left) / 2; + + int cmp = CompareBlockKey(block_ids[mid], target); + if (!status_.ok()) { + return false; + } + if (cmp < 0) { + // Key at "target" is larger than "mid". Therefore all + // blocks before or at "mid" are uninteresting. + left = mid + 1; + } else { + // Key at "target" is <= "mid". Therefore all blocks + // after "mid" are uninteresting. + // If there is only one block left, we found it. + if (left == right) break; + right = mid; + } + } + + if (left == right) { + // In one of the two following cases: + // (1) left is the first one of block_ids + // (2) there is a gap of blocks between block of `left` and `left-1`. + // we can further distinguish the case of key in the block or key not + // existing, by comparing the target key and the key of the previous + // block to the left of the block found. + if (block_ids[left] > 0 && + (left == left_bound || block_ids[left - 1] != block_ids[left] - 1) && + CompareBlockKey(block_ids[left] - 1, target) > 0) { + current_ = restarts_; + *prefix_may_exist = false; + return false; + } + + *index = block_ids[left]; + return true; + } else { + assert(left > right); + + // If the next block key is larger than seek key, it is possible that + // no key shares the prefix with `target`, or all keys with the same + // prefix as `target` are smaller than prefix. In the latter case, + // we are mandated to set the position the same as the total order. + // In the latter case, either: + // (1) `target` falls into the range of the next block. In this case, + // we can place the iterator to the next block, or + // (2) `target` is larger than all block keys. In this case we can + // keep the iterator invalidate without setting `prefix_may_exist` + // to false. + // We might sometimes end up with setting the total order position + // while there is no key sharing the prefix as `target`, but it + // still follows the contract. + uint32_t right_index = block_ids[right]; + assert(right_index + 1 <= num_restarts_); + if (right_index + 1 < num_restarts_) { + if (CompareBlockKey(right_index + 1, target) >= 0) { + *index = right_index + 1; + return true; + } else { + // We have to set the flag here because we are not positioning + // the iterator to the total order position. + *prefix_may_exist = false; + } + } + + // Mark iterator invalid + current_ = restarts_; + return false; + } +} + +bool IndexBlockIter::PrefixSeek(const Slice& target, uint32_t* index, + bool* prefix_may_exist) { + assert(index); + assert(prefix_may_exist); + assert(prefix_index_); + *prefix_may_exist = true; + Slice seek_key = target; + if (raw_key_.IsUserKey()) { + seek_key = ExtractUserKey(target); + } + uint32_t* block_ids = nullptr; + uint32_t num_blocks = prefix_index_->GetBlocks(target, &block_ids); + + if (num_blocks == 0) { + current_ = restarts_; + *prefix_may_exist = false; + return false; + } else { + assert(block_ids); + return BinaryBlockIndexSeek(seek_key, block_ids, 0, num_blocks - 1, index, + prefix_may_exist); + } +} + +uint32_t Block::NumRestarts() const { + assert(size_ >= 2 * sizeof(uint32_t)); + uint32_t block_footer = DecodeFixed32(data_ + size_ - sizeof(uint32_t)); + uint32_t num_restarts = block_footer; + if (size_ > kMaxBlockSizeSupportedByHashIndex) { + // In BlockBuilder, we have ensured a block with HashIndex is less than + // kMaxBlockSizeSupportedByHashIndex (64KiB). + // + // Therefore, if we encounter a block with a size > 64KiB, the block + // cannot have HashIndex. So the footer will directly interpreted as + // num_restarts. + // + // Such check is for backward compatibility. We can ensure legacy block + // with a vary large num_restarts i.e. >= 0x80000000 can be interpreted + // correctly as no HashIndex even if the MSB of num_restarts is set. + return num_restarts; + } + BlockBasedTableOptions::DataBlockIndexType index_type; + UnPackIndexTypeAndNumRestarts(block_footer, &index_type, &num_restarts); + return num_restarts; +} + +BlockBasedTableOptions::DataBlockIndexType Block::IndexType() const { + assert(size_ >= 2 * sizeof(uint32_t)); + if (size_ > kMaxBlockSizeSupportedByHashIndex) { + // The check is for the same reason as that in NumRestarts() + return BlockBasedTableOptions::kDataBlockBinarySearch; + } + uint32_t block_footer = DecodeFixed32(data_ + size_ - sizeof(uint32_t)); + uint32_t num_restarts = block_footer; + BlockBasedTableOptions::DataBlockIndexType index_type; + UnPackIndexTypeAndNumRestarts(block_footer, &index_type, &num_restarts); + return index_type; +} + +Block::~Block() { + // This sync point can be re-enabled if RocksDB can control the + // initialization order of any/all static options created by the user. + // TEST_SYNC_POINT("Block::~Block"); + delete[] kv_checksum_; +} + +Block::Block(BlockContents&& contents, size_t read_amp_bytes_per_bit, + Statistics* statistics) + : contents_(std::move(contents)), + data_(contents_.data.data()), + size_(contents_.data.size()), + restart_offset_(0), + num_restarts_(0) { + TEST_SYNC_POINT("Block::Block:0"); + if (size_ < sizeof(uint32_t)) { + size_ = 0; // Error marker + } else { + // Should only decode restart points for uncompressed blocks + num_restarts_ = NumRestarts(); + switch (IndexType()) { + case BlockBasedTableOptions::kDataBlockBinarySearch: + restart_offset_ = static_cast(size_) - + (1 + num_restarts_) * sizeof(uint32_t); + if (restart_offset_ > size_ - sizeof(uint32_t)) { + // The size is too small for NumRestarts() and therefore + // restart_offset_ wrapped around. + size_ = 0; + } + break; + case BlockBasedTableOptions::kDataBlockBinaryAndHash: + if (size_ < sizeof(uint32_t) /* block footer */ + + sizeof(uint16_t) /* NUM_BUCK */) { + size_ = 0; + break; + } + + uint16_t map_offset; + data_block_hash_index_.Initialize( + data_, static_cast(size_ - sizeof(uint32_t)), /*chop off + NUM_RESTARTS*/ + &map_offset); + + restart_offset_ = map_offset - num_restarts_ * sizeof(uint32_t); + + if (restart_offset_ > map_offset) { + // map_offset is too small for NumRestarts() and + // therefore restart_offset_ wrapped around. + size_ = 0; + break; + } + break; + default: + size_ = 0; // Error marker + } + } + if (read_amp_bytes_per_bit != 0 && statistics && size_ != 0) { + read_amp_bitmap_.reset(new BlockReadAmpBitmap( + restart_offset_, read_amp_bytes_per_bit, statistics)); + } +} + +void Block::InitializeDataBlockProtectionInfo(uint8_t protection_bytes_per_key, + const Comparator* raw_ucmp) { + protection_bytes_per_key_ = 0; + if (protection_bytes_per_key > 0 && num_restarts_ > 0) { + // NewDataIterator() is called with protection_bytes_per_key_ = 0. + // This is intended since checksum is not constructed yet. + // + // We do not know global_seqno yet, so checksum computation and + // verification all assume global_seqno = 0. + // TODO(yuzhangyu): handle the implication of padding timestamp for kv + // protection. + std::unique_ptr iter{NewDataIterator( + raw_ucmp, kDisableGlobalSequenceNumber, nullptr /* iter */, + nullptr /* stats */, true /* block_contents_pinned */, + true /* user_defined_timestamps_persisted */)}; + if (iter->status().ok()) { + block_restart_interval_ = iter->GetRestartInterval(); + } + uint32_t num_keys = 0; + if (iter->status().ok()) { + num_keys = iter->NumberOfKeys(block_restart_interval_); + } + if (iter->status().ok()) { + checksum_size_ = num_keys * protection_bytes_per_key; + kv_checksum_ = new char[(size_t)checksum_size_]; + size_t i = 0; + iter->SeekToFirst(); + while (iter->Valid()) { + GenerateKVChecksum(kv_checksum_ + i, protection_bytes_per_key, + iter->key(), iter->value()); + iter->Next(); + i += protection_bytes_per_key; + } + assert(!iter->status().ok() || i == num_keys * protection_bytes_per_key); + } + if (!iter->status().ok()) { + size_ = 0; // Error marker + return; + } + protection_bytes_per_key_ = protection_bytes_per_key; + } +} + +void Block::InitializeIndexBlockProtectionInfo(uint8_t protection_bytes_per_key, + const Comparator* raw_ucmp, + bool value_is_full, + bool index_has_first_key) { + protection_bytes_per_key_ = 0; + if (num_restarts_ > 0 && protection_bytes_per_key > 0) { + // Note that `global_seqno` and `key_includes_seq` are hardcoded here. They + // do not impact how the index block is parsed. During checksum + // construction/verification, we use the entire key buffer from + // raw_key_.GetKey() returned by iter->key() as the `key` part of key-value + // checksum, and the content of this buffer do not change for different + // values of `global_seqno` or `key_includes_seq`. + // TODO(yuzhangyu): handle the implication of padding timestamp for kv + // protection. + std::unique_ptr iter{NewIndexIterator( + raw_ucmp, kDisableGlobalSequenceNumber /* global_seqno */, nullptr, + nullptr /* Statistics */, true /* total_order_seek */, + index_has_first_key /* have_first_key */, false /* key_includes_seq */, + value_is_full, true /* block_contents_pinned */, + true /* user_defined_timestamps_persisted*/, + nullptr /* prefix_index */)}; + if (iter->status().ok()) { + block_restart_interval_ = iter->GetRestartInterval(); + } + uint32_t num_keys = 0; + if (iter->status().ok()) { + num_keys = iter->NumberOfKeys(block_restart_interval_); + } + if (iter->status().ok()) { + checksum_size_ = num_keys * protection_bytes_per_key; + kv_checksum_ = new char[(size_t)checksum_size_]; + iter->SeekToFirst(); + size_t i = 0; + while (iter->Valid()) { + GenerateKVChecksum(kv_checksum_ + i, protection_bytes_per_key, + iter->key(), iter->raw_value()); + iter->Next(); + i += protection_bytes_per_key; + } + assert(!iter->status().ok() || i == num_keys * protection_bytes_per_key); + } + if (!iter->status().ok()) { + size_ = 0; // Error marker + return; + } + protection_bytes_per_key_ = protection_bytes_per_key; + } +} + +void Block::InitializeMetaIndexBlockProtectionInfo( + uint8_t protection_bytes_per_key) { + protection_bytes_per_key_ = 0; + if (num_restarts_ > 0 && protection_bytes_per_key > 0) { + std::unique_ptr iter{ + NewMetaIterator(true /* block_contents_pinned */)}; + if (iter->status().ok()) { + block_restart_interval_ = iter->GetRestartInterval(); + } + uint32_t num_keys = 0; + if (iter->status().ok()) { + num_keys = iter->NumberOfKeys(block_restart_interval_); + } + if (iter->status().ok()) { + checksum_size_ = num_keys * protection_bytes_per_key; + kv_checksum_ = new char[(size_t)checksum_size_]; + iter->SeekToFirst(); + size_t i = 0; + while (iter->Valid()) { + GenerateKVChecksum(kv_checksum_ + i, protection_bytes_per_key, + iter->key(), iter->value()); + iter->Next(); + i += protection_bytes_per_key; + } + assert(!iter->status().ok() || i == num_keys * protection_bytes_per_key); + } + if (!iter->status().ok()) { + size_ = 0; // Error marker + return; + } + protection_bytes_per_key_ = protection_bytes_per_key; + } +} + +MetaBlockIter* Block::NewMetaIterator(bool block_contents_pinned) { + MetaBlockIter* iter = new MetaBlockIter(); + if (size_ < 2 * sizeof(uint32_t)) { + iter->Invalidate(Status::Corruption("bad block contents")); + return iter; + } else if (num_restarts_ == 0) { + // Empty block. + iter->Invalidate(Status::OK()); + } else { + iter->Initialize(data_, restart_offset_, num_restarts_, + block_contents_pinned, protection_bytes_per_key_, + kv_checksum_, block_restart_interval_); + } + return iter; +} + +DataBlockIter* Block::NewDataIterator(const Comparator* raw_ucmp, + SequenceNumber global_seqno, + DataBlockIter* iter, Statistics* stats, + bool block_contents_pinned, + bool user_defined_timestamps_persisted) { + DataBlockIter* ret_iter; + if (iter != nullptr) { + ret_iter = iter; + } else { + ret_iter = new DataBlockIter; + } + if (size_ < 2 * sizeof(uint32_t)) { + ret_iter->Invalidate(Status::Corruption("bad block contents")); + return ret_iter; + } + if (num_restarts_ == 0) { + // Empty block. + ret_iter->Invalidate(Status::OK()); + return ret_iter; + } else { + ret_iter->Initialize( + raw_ucmp, data_, restart_offset_, num_restarts_, global_seqno, + read_amp_bitmap_.get(), block_contents_pinned, + user_defined_timestamps_persisted, + data_block_hash_index_.Valid() ? &data_block_hash_index_ : nullptr, + protection_bytes_per_key_, kv_checksum_, block_restart_interval_); + if (read_amp_bitmap_) { + if (read_amp_bitmap_->GetStatistics() != stats) { + // DB changed the Statistics pointer, we need to notify read_amp_bitmap_ + read_amp_bitmap_->SetStatistics(stats); + } + } + } + + return ret_iter; +} + +IndexBlockIter* Block::NewIndexIterator( + const Comparator* raw_ucmp, SequenceNumber global_seqno, + IndexBlockIter* iter, Statistics* /*stats*/, bool total_order_seek, + bool have_first_key, bool key_includes_seq, bool value_is_full, + bool block_contents_pinned, bool user_defined_timestamps_persisted, + BlockPrefixIndex* prefix_index) { + IndexBlockIter* ret_iter; + if (iter != nullptr) { + ret_iter = iter; + } else { + ret_iter = new IndexBlockIter; + } + if (size_ < 2 * sizeof(uint32_t)) { + ret_iter->Invalidate(Status::Corruption("bad block contents")); + return ret_iter; + } + if (num_restarts_ == 0) { + // Empty block. + ret_iter->Invalidate(Status::OK()); + return ret_iter; + } else { + BlockPrefixIndex* prefix_index_ptr = + total_order_seek ? nullptr : prefix_index; + ret_iter->Initialize( + raw_ucmp, data_, restart_offset_, num_restarts_, global_seqno, + prefix_index_ptr, have_first_key, key_includes_seq, value_is_full, + block_contents_pinned, user_defined_timestamps_persisted, + protection_bytes_per_key_, kv_checksum_, block_restart_interval_); + } + + return ret_iter; +} + +size_t Block::ApproximateMemoryUsage() const { + size_t usage = usable_size(); +#ifdef ROCKSDB_MALLOC_USABLE_SIZE + usage += malloc_usable_size((void*)this); +#else + usage += sizeof(*this); +#endif // ROCKSDB_MALLOC_USABLE_SIZE + if (read_amp_bitmap_) { + usage += read_amp_bitmap_->ApproximateMemoryUsage(); + } + usage += checksum_size_; + return usage; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/block.h b/librocksdb-sys/rocksdb/table/block_based/block.h new file mode 100644 index 0000000..dcd83aa --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block.h @@ -0,0 +1,969 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include +#include + +#include +#include + +#include "db/kv_checksum.h" +#include "db/pinned_iterators_manager.h" +#include "port/malloc.h" +#include "rocksdb/advanced_cache.h" +#include "rocksdb/iterator.h" +#include "rocksdb/options.h" +#include "rocksdb/statistics.h" +#include "rocksdb/table.h" +#include "table/block_based/block_prefix_index.h" +#include "table/block_based/data_block_hash_index.h" +#include "table/format.h" +#include "table/internal_iterator.h" +#include "test_util/sync_point.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +struct BlockContents; +class Comparator; +template +class BlockIter; +class DataBlockIter; +class IndexBlockIter; +class MetaBlockIter; +class BlockPrefixIndex; + +// BlockReadAmpBitmap is a bitmap that map the ROCKSDB_NAMESPACE::Block data +// bytes to a bitmap with ratio bytes_per_bit. Whenever we access a range of +// bytes in the Block we update the bitmap and increment +// READ_AMP_ESTIMATE_USEFUL_BYTES. +class BlockReadAmpBitmap { + public: + explicit BlockReadAmpBitmap(size_t block_size, size_t bytes_per_bit, + Statistics* statistics) + : bitmap_(nullptr), + bytes_per_bit_pow_(0), + statistics_(statistics), + rnd_(Random::GetTLSInstance()->Uniform( + static_cast(bytes_per_bit))) { + TEST_SYNC_POINT_CALLBACK("BlockReadAmpBitmap:rnd", &rnd_); + assert(block_size > 0 && bytes_per_bit > 0); + + // convert bytes_per_bit to be a power of 2 + while (bytes_per_bit >>= 1) { + bytes_per_bit_pow_++; + } + + // num_bits_needed = ceil(block_size / bytes_per_bit) + size_t num_bits_needed = ((block_size - 1) >> bytes_per_bit_pow_) + 1; + assert(num_bits_needed > 0); + + // bitmap_size = ceil(num_bits_needed / kBitsPerEntry) + size_t bitmap_size = (num_bits_needed - 1) / kBitsPerEntry + 1; + + // Create bitmap and set all the bits to 0 + bitmap_ = new std::atomic[bitmap_size](); + + RecordTick(GetStatistics(), READ_AMP_TOTAL_READ_BYTES, block_size); + } + + ~BlockReadAmpBitmap() { delete[] bitmap_; } + + void Mark(uint32_t start_offset, uint32_t end_offset) { + assert(end_offset >= start_offset); + // Index of first bit in mask + uint32_t start_bit = + (start_offset + (1 << bytes_per_bit_pow_) - rnd_ - 1) >> + bytes_per_bit_pow_; + // Index of last bit in mask + 1 + uint32_t exclusive_end_bit = + (end_offset + (1 << bytes_per_bit_pow_) - rnd_) >> bytes_per_bit_pow_; + if (start_bit >= exclusive_end_bit) { + return; + } + assert(exclusive_end_bit > 0); + + if (GetAndSet(start_bit) == 0) { + uint32_t new_useful_bytes = (exclusive_end_bit - start_bit) + << bytes_per_bit_pow_; + RecordTick(GetStatistics(), READ_AMP_ESTIMATE_USEFUL_BYTES, + new_useful_bytes); + } + } + + Statistics* GetStatistics() { + return statistics_.load(std::memory_order_relaxed); + } + + void SetStatistics(Statistics* stats) { statistics_.store(stats); } + + uint32_t GetBytesPerBit() { return 1 << bytes_per_bit_pow_; } + + size_t ApproximateMemoryUsage() const { +#ifdef ROCKSDB_MALLOC_USABLE_SIZE + return malloc_usable_size((void*)this); +#endif // ROCKSDB_MALLOC_USABLE_SIZE + return sizeof(*this); + } + + private: + // Get the current value of bit at `bit_idx` and set it to 1 + inline bool GetAndSet(uint32_t bit_idx) { + const uint32_t byte_idx = bit_idx / kBitsPerEntry; + const uint32_t bit_mask = 1 << (bit_idx % kBitsPerEntry); + + return bitmap_[byte_idx].fetch_or(bit_mask, std::memory_order_relaxed) & + bit_mask; + } + + const uint32_t kBytesPersEntry = sizeof(uint32_t); // 4 bytes + const uint32_t kBitsPerEntry = kBytesPersEntry * 8; // 32 bits + + // Bitmap used to record the bytes that we read, use atomic to protect + // against multiple threads updating the same bit + std::atomic* bitmap_; + // (1 << bytes_per_bit_pow_) is bytes_per_bit. Use power of 2 to optimize + // muliplication and division + uint8_t bytes_per_bit_pow_; + // Pointer to DB Statistics object, Since this bitmap may outlive the DB + // this pointer maybe invalid, but the DB will update it to a valid pointer + // by using SetStatistics() before calling Mark() + std::atomic statistics_; + uint32_t rnd_; +}; + +// class Block is the uncompressed and "parsed" form for blocks containing +// key-value pairs. (See BlockContents comments for more on terminology.) +// This includes the in-memory representation of data blocks, index blocks +// (including partitions), range deletion blocks, properties blocks, metaindex +// blocks, as well as the top level of the partitioned filter structure (which +// is actually an index of the filter partitions). It is NOT suitable for +// compressed blocks in general, filter blocks/partitions, or compression +// dictionaries. +// +// See https://github.com/facebook/rocksdb/wiki/Rocksdb-BlockBasedTable-Format +// for details of the format and the various block types. +// +// TODO: Rename to ParsedKvBlock? +class Block { + public: + // Initialize the block with the specified contents. + explicit Block(BlockContents&& contents, size_t read_amp_bytes_per_bit = 0, + Statistics* statistics = nullptr); + // No copying allowed + Block(const Block&) = delete; + void operator=(const Block&) = delete; + + ~Block(); + + size_t size() const { return size_; } + const char* data() const { return data_; } + // The additional memory space taken by the block data. + size_t usable_size() const { return contents_.usable_size(); } + uint32_t NumRestarts() const; + bool own_bytes() const { return contents_.own_bytes(); } + + BlockBasedTableOptions::DataBlockIndexType IndexType() const; + + // raw_ucmp is a raw (i.e., not wrapped by `UserComparatorWrapper`) user key + // comparator. + // + // If iter is null, return new Iterator + // If iter is not null, update this one and return it as Iterator* + // + // Updates read_amp_bitmap_ if it is not nullptr. + // + // If `block_contents_pinned` is true, the caller will guarantee that when + // the cleanup functions are transferred from the iterator to other + // classes, e.g. PinnableSlice, the pointer to the bytes will still be + // valid. Either the iterator holds cache handle or ownership of some resource + // and release them in a release function, or caller is sure that the data + // will not go away (for example, it's from mmapped file which will not be + // closed). + // + // `user_defined_timestamps_persisted` controls whether a min timestamp is + // padded while key is being parsed from the block. + // + // NOTE: for the hash based lookup, if a key prefix doesn't match any key, + // the iterator will simply be set as "invalid", rather than returning + // the key that is just pass the target key. + DataBlockIter* NewDataIterator(const Comparator* raw_ucmp, + SequenceNumber global_seqno, + DataBlockIter* iter = nullptr, + Statistics* stats = nullptr, + bool block_contents_pinned = false, + bool user_defined_timestamps_persisted = true); + + // Returns an MetaBlockIter for iterating over blocks containing metadata + // (like Properties blocks). Unlike data blocks, the keys for these blocks + // do not contain sequence numbers, do not use a user-define comparator, and + // do not track read amplification/statistics. Additionally, MetaBlocks will + // not assert if the block is formatted improperly. + // + // If `block_contents_pinned` is true, the caller will guarantee that when + // the cleanup functions are transferred from the iterator to other + // classes, e.g. PinnableSlice, the pointer to the bytes will still be + // valid. Either the iterator holds cache handle or ownership of some resource + // and release them in a release function, or caller is sure that the data + // will not go away (for example, it's from mmapped file which will not be + // closed). + MetaBlockIter* NewMetaIterator(bool block_contents_pinned = false); + + // raw_ucmp is a raw (i.e., not wrapped by `UserComparatorWrapper`) user key + // comparator. + // + // key_includes_seq, default true, means that the keys are in internal key + // format. + // value_is_full, default true, means that no delta encoding is + // applied to values. + // + // If `prefix_index` is not nullptr this block will do hash lookup for the key + // prefix. If total_order_seek is true, prefix_index_ is ignored. + // + // `have_first_key` controls whether IndexValue will contain + // first_internal_key. It affects data serialization format, so the same value + // have_first_key must be used when writing and reading index. + // It is determined by IndexType property of the table. + // `user_defined_timestamps_persisted` controls whether a min timestamp is + // padded while key is being parsed from the block. + IndexBlockIter* NewIndexIterator( + const Comparator* raw_ucmp, SequenceNumber global_seqno, + IndexBlockIter* iter, Statistics* stats, bool total_order_seek, + bool have_first_key, bool key_includes_seq, bool value_is_full, + bool block_contents_pinned = false, + bool user_defined_timestamps_persisted = true, + BlockPrefixIndex* prefix_index = nullptr); + + // Report an approximation of how much memory has been used. + size_t ApproximateMemoryUsage() const; + + // For TypedCacheInterface + const Slice& ContentSlice() const { return contents_.data; } + + // Initializes per key-value checksum protection. + // After this method is called, each DataBlockIterator returned + // by NewDataIterator will verify per key-value checksum for any key it read. + void InitializeDataBlockProtectionInfo(uint8_t protection_bytes_per_key, + const Comparator* raw_ucmp); + + // Initializes per key-value checksum protection. + // After this method is called, each IndexBlockIterator returned + // by NewIndexIterator will verify per key-value checksum for any key it read. + // value_is_full and index_has_first_key are needed to be able to parse + // the index block content and construct checksums. + void InitializeIndexBlockProtectionInfo(uint8_t protection_bytes_per_key, + const Comparator* raw_ucmp, + bool value_is_full, + bool index_has_first_key); + + // Initializes per key-value checksum protection. + // After this method is called, each MetaBlockIter returned + // by NewMetaIterator will verify per key-value checksum for any key it read. + void InitializeMetaIndexBlockProtectionInfo(uint8_t protection_bytes_per_key); + + static void GenerateKVChecksum(char* checksum_ptr, uint8_t checksum_len, + const Slice& key, const Slice& value) { + ProtectionInfo64().ProtectKV(key, value).Encode(checksum_len, checksum_ptr); + } + + const char* TEST_GetKVChecksum() const { return kv_checksum_; } + + private: + BlockContents contents_; + const char* data_; // contents_.data.data() + size_t size_; // contents_.data.size() + uint32_t restart_offset_; // Offset in data_ of restart array + uint32_t num_restarts_; + std::unique_ptr read_amp_bitmap_; + char* kv_checksum_{nullptr}; + uint32_t checksum_size_{0}; + // Used by block iterators to calculate current key index within a block + uint32_t block_restart_interval_{0}; + uint8_t protection_bytes_per_key_{0}; + DataBlockHashIndex data_block_hash_index_; +}; + +// A `BlockIter` iterates over the entries in a `Block`'s data buffer. The +// format of this data buffer is an uncompressed, sorted sequence of key-value +// pairs (see `Block` API for more details). +// +// Notably, the keys may either be in internal key format or user key format. +// Subclasses are responsible for configuring the key format. +// +// `BlockIter` intends to provide final overrides for all of +// `InternalIteratorBase` functions that can move the iterator. It does +// this to guarantee `UpdateKey()` is called exactly once after each key +// movement potentially visible to users. In this step, the key is prepared +// (e.g., serialized if global seqno is in effect) so it can be returned +// immediately when the user asks for it via calling `key() const`. +// +// For its subclasses, it provides protected variants of the above-mentioned +// final-overridden methods. They are named with the "Impl" suffix, e.g., +// `Seek()` logic would be implemented by subclasses in `SeekImpl()`. These +// "Impl" functions are responsible for positioning `raw_key_` but not +// invoking `UpdateKey()`. +// +// Per key-value checksum is enabled if relevant states are passed in during +// `InitializeBase()`. The checksum verification is done in each call to +// UpdateKey() for the current key. Each subclass is responsible for keeping +// track of cur_entry_idx_, the index of the current key within the block. +// BlockIter uses this index to get the corresponding checksum for current key. +// Additional checksum verification may be done in subclasses if they read keys +// other than the key being processed in UpdateKey(). +template +class BlockIter : public InternalIteratorBase { + public: + // Makes Valid() return false, status() return `s`, and Seek()/Prev()/etc do + // nothing. Calls cleanup functions. + virtual void Invalidate(const Status& s) { + // Assert that the BlockIter is never deleted while Pinning is Enabled. + assert(!pinned_iters_mgr_ || !pinned_iters_mgr_->PinningEnabled()); + + data_ = nullptr; + current_ = restarts_; + status_ = s; + + // Call cleanup callbacks. + Cleanable::Reset(); + } + + bool Valid() const override { + // When status_ is not ok, iter should be invalid. + assert(status_.ok() || current_ >= restarts_); + return current_ < restarts_; + } + + virtual void SeekToFirst() override final { +#ifndef NDEBUG + if (TEST_Corrupt_Callback("BlockIter::SeekToFirst")) return; +#endif + SeekToFirstImpl(); + UpdateKey(); + } + + virtual void SeekToLast() override final { + SeekToLastImpl(); + UpdateKey(); + } + + virtual void Seek(const Slice& target) override final { + SeekImpl(target); + UpdateKey(); + } + + virtual void SeekForPrev(const Slice& target) override final { + SeekForPrevImpl(target); + UpdateKey(); + } + + virtual void Next() override final { + NextImpl(); + UpdateKey(); + } + + virtual bool NextAndGetResult(IterateResult* result) override final { + // This does not need to call `UpdateKey()` as the parent class only has + // access to the `UpdateKey()`-invoking functions. + return InternalIteratorBase::NextAndGetResult(result); + } + + virtual void Prev() override final { + PrevImpl(); + UpdateKey(); + } + + Status status() const override { return status_; } + + Slice key() const override { + assert(Valid()); + return key_; + } + +#ifndef NDEBUG + ~BlockIter() override { + // Assert that the BlockIter is never deleted while Pinning is Enabled. + assert(!pinned_iters_mgr_ || + (pinned_iters_mgr_ && !pinned_iters_mgr_->PinningEnabled())); + status_.PermitUncheckedError(); + } + + void SetPinnedItersMgr(PinnedIteratorsManager* pinned_iters_mgr) override { + pinned_iters_mgr_ = pinned_iters_mgr; + } + + PinnedIteratorsManager* pinned_iters_mgr_ = nullptr; + + bool TEST_Corrupt_Callback(const std::string& sync_point) { + bool corrupt = false; + TEST_SYNC_POINT_CALLBACK(sync_point, static_cast(&corrupt)); + + if (corrupt) { + CorruptionError(); + } + return corrupt; + } +#endif + + bool IsKeyPinned() const override { + return block_contents_pinned_ && key_pinned_; + } + + bool IsValuePinned() const override { return block_contents_pinned_; } + + size_t TEST_CurrentEntrySize() { return NextEntryOffset() - current_; } + + uint32_t ValueOffset() const { + return static_cast(value_.data() - data_); + } + + void SetCacheHandle(Cache::Handle* handle) { cache_handle_ = handle; } + + Cache::Handle* cache_handle() { return cache_handle_; } + + protected: + std::unique_ptr icmp_; + const char* data_; // underlying block contents + uint32_t num_restarts_; // Number of uint32_t entries in restart array + + // Index of restart block in which current_ or current_-1 falls + uint32_t restart_index_; + uint32_t restarts_; // Offset of restart array (list of fixed32) + // current_ is offset in data_ of current entry. >= restarts_ if !Valid + uint32_t current_; + // Raw key from block. + IterKey raw_key_; + // Buffer for key data when global seqno assignment is enabled. + IterKey key_buf_; + Slice value_; + Status status_; + // Key to be exposed to users. + Slice key_; + SequenceNumber global_seqno_; + // Size of the user-defined timestamp. + size_t ts_sz_ = 0; + // If user-defined timestamp is enabled but not persisted. A min timestamp + // will be padded to the key during key parsing where it applies. Such as when + // parsing keys from data block, index block, parsing the first internal + // key from IndexValue entry. Min timestamp padding is different for when + // `raw_key_` is a user key vs is an internal key. + // + // This only applies to data block and index blocks including index block for + // data blocks, index block for partitioned filter blocks, index block for + // partitioned index blocks. In summary, this only applies to block whose key + // are real user keys or internal keys created from user keys. + bool pad_min_timestamp_; + + // Per key-value checksum related states + const char* kv_checksum_; + int32_t cur_entry_idx_; + uint32_t block_restart_interval_; + uint8_t protection_bytes_per_key_; + + bool key_pinned_; + // Whether the block data is guaranteed to outlive this iterator, and + // as long as the cleanup functions are transferred to another class, + // e.g. PinnableSlice, the pointer to the bytes will still be valid. + bool block_contents_pinned_; + + virtual void SeekToFirstImpl() = 0; + virtual void SeekToLastImpl() = 0; + virtual void SeekImpl(const Slice& target) = 0; + virtual void SeekForPrevImpl(const Slice& target) = 0; + virtual void NextImpl() = 0; + virtual void PrevImpl() = 0; + + // Returns the restart interval of this block. + // Returns 0 if num_restarts_ <= 1 or if the BlockIter is not initialized. + virtual uint32_t GetRestartInterval() { + if (num_restarts_ <= 1 || data_ == nullptr) { + return 0; + } + SeekToFirstImpl(); + uint32_t end_index = GetRestartPoint(1); + uint32_t count = 1; + while (NextEntryOffset() < end_index && status_.ok()) { + assert(Valid()); + NextImpl(); + ++count; + } + return count; + } + + // Returns the number of keys in this block. + virtual uint32_t NumberOfKeys(uint32_t block_restart_interval) { + if (num_restarts_ == 0 || data_ == nullptr) { + return 0; + } + uint32_t count = (num_restarts_ - 1) * block_restart_interval; + // Add number of keys from the last restart interval + SeekToRestartPoint(num_restarts_ - 1); + while (NextEntryOffset() < restarts_ && status_.ok()) { + NextImpl(); + ++count; + } + return count; + } + + // Stores whether the current key has a shared bytes with prev key in + // *is_shared. + // Sets raw_key_, value_ to the current parsed key and value. + // Sets restart_index_ to point to the restart interval that contains + // the current key. + template + inline bool ParseNextKey(bool* is_shared); + + // protection_bytes_per_key, kv_checksum, and block_restart_interval + // are needed only for per kv checksum verification. + void InitializeBase(const Comparator* raw_ucmp, const char* data, + uint32_t restarts, uint32_t num_restarts, + SequenceNumber global_seqno, bool block_contents_pinned, + bool user_defined_timestamp_persisted, + + uint8_t protection_bytes_per_key, const char* kv_checksum, + uint32_t block_restart_interval) { + assert(data_ == nullptr); // Ensure it is called only once + assert(num_restarts > 0); // Ensure the param is valid + + icmp_ = std::make_unique(raw_ucmp); + data_ = data; + restarts_ = restarts; + num_restarts_ = num_restarts; + current_ = restarts_; + restart_index_ = num_restarts_; + global_seqno_ = global_seqno; + if (raw_ucmp != nullptr) { + ts_sz_ = raw_ucmp->timestamp_size(); + } + pad_min_timestamp_ = ts_sz_ > 0 && !user_defined_timestamp_persisted; + block_contents_pinned_ = block_contents_pinned; + cache_handle_ = nullptr; + cur_entry_idx_ = -1; + protection_bytes_per_key_ = protection_bytes_per_key; + kv_checksum_ = kv_checksum; + block_restart_interval_ = block_restart_interval; + // Checksum related states are either all 0/nullptr or all non-zero. + // One exception is when num_restarts == 0, block_restart_interval can be 0 + // since we are not able to compute it. + assert((protection_bytes_per_key == 0 && kv_checksum == nullptr) || + (protection_bytes_per_key > 0 && kv_checksum != nullptr && + (block_restart_interval > 0 || num_restarts == 1))); + } + + void CorruptionError(const std::string& error_msg = "bad entry in block") { + current_ = restarts_; + restart_index_ = num_restarts_; + status_ = Status::Corruption(error_msg); + raw_key_.Clear(); + value_.clear(); + } + + void PerKVChecksumCorruptionError() { + std::string error_msg{ + "Corrupted block entry: per key-value checksum verification " + "failed."}; + error_msg.append(" Offset: " + std::to_string(current_) + "."); + error_msg.append(" Entry index: " + std::to_string(cur_entry_idx_) + "."); + CorruptionError(error_msg); + } + + void UpdateRawKeyAndMaybePadMinTimestamp(const Slice& key) { + if (pad_min_timestamp_) { + std::string buf; + if (raw_key_.IsUserKey()) { + AppendKeyWithMinTimestamp(&buf, key, ts_sz_); + } else { + PadInternalKeyWithMinTimestamp(&buf, key, ts_sz_); + } + raw_key_.SetKey(buf, true /* copy */); + } else { + raw_key_.SetKey(key, false /* copy */); + } + } + + // Must be called every time a key is found that needs to be returned to user, + // and may be called when no key is found (as a no-op). Updates `key_`, + // `key_buf_`, and `key_pinned_` with info about the found key. + // Per key-value checksum verification is done if available for the key to be + // returned. Iterator is invalidated with corruption status if checksum + // verification fails. + void UpdateKey() { + key_buf_.Clear(); + if (!Valid()) { + return; + } + if (raw_key_.IsUserKey()) { + assert(global_seqno_ == kDisableGlobalSequenceNumber); + key_ = raw_key_.GetUserKey(); + key_pinned_ = raw_key_.IsKeyPinned(); + } else if (global_seqno_ == kDisableGlobalSequenceNumber) { + key_ = raw_key_.GetInternalKey(); + key_pinned_ = raw_key_.IsKeyPinned(); + } else { + key_buf_.SetInternalKey(raw_key_.GetUserKey(), global_seqno_, + ExtractValueType(raw_key_.GetInternalKey())); + key_ = key_buf_.GetInternalKey(); + key_pinned_ = false; + } + TEST_SYNC_POINT_CALLBACK("BlockIter::UpdateKey::value", + (void*)value_.data()); + TEST_SYNC_POINT_CALLBACK("Block::VerifyChecksum::checksum_len", + &protection_bytes_per_key_); + if (protection_bytes_per_key_ > 0) { + if (!ProtectionInfo64() + .ProtectKV(raw_key_.GetKey(), value_) + .Verify( + protection_bytes_per_key_, + kv_checksum_ + protection_bytes_per_key_ * cur_entry_idx_)) { + PerKVChecksumCorruptionError(); + } + } + } + + // Returns the result of `Comparator::Compare()`, where the appropriate + // comparator is used for the block contents, the LHS argument is the current + // key with global seqno applied, and the RHS argument is `other`. + int CompareCurrentKey(const Slice& other) { + if (raw_key_.IsUserKey()) { + assert(global_seqno_ == kDisableGlobalSequenceNumber); + return icmp_->user_comparator()->Compare(raw_key_.GetUserKey(), other); + } else if (global_seqno_ == kDisableGlobalSequenceNumber) { + return icmp_->Compare(raw_key_.GetInternalKey(), other); + } + return icmp_->Compare(raw_key_.GetInternalKey(), global_seqno_, other, + kDisableGlobalSequenceNumber); + } + + private: + // Store the cache handle, if the block is cached. We need this since the + // only other place the handle is stored is as an argument to the Cleanable + // function callback, which is hard to retrieve. When multiple value + // PinnableSlices reference the block, they need the cache handle in order + // to bump up the ref count + Cache::Handle* cache_handle_; + + public: + // Return the offset in data_ just past the end of the current entry. + inline uint32_t NextEntryOffset() const { + // NOTE: We don't support blocks bigger than 2GB + return static_cast((value_.data() + value_.size()) - data_); + } + + uint32_t GetRestartPoint(uint32_t index) const { + assert(index < num_restarts_); + return DecodeFixed32(data_ + restarts_ + index * sizeof(uint32_t)); + } + + void SeekToRestartPoint(uint32_t index) { + raw_key_.Clear(); + restart_index_ = index; + // current_ will be fixed by ParseNextKey(); + + // ParseNextKey() starts at the end of value_, so set value_ accordingly + uint32_t offset = GetRestartPoint(index); + value_ = Slice(data_ + offset, 0); + } + + protected: + template + inline bool BinarySeek(const Slice& target, uint32_t* index, + bool* is_index_key_result); + + // Find the first key in restart interval `index` that is >= `target`. + // If there is no such key, iterator is positioned at the first key in + // restart interval `index + 1`. + // If is_index_key_result is true, it positions the iterator at the first key + // in this restart interval. + // Per key-value checksum verification is done for all keys scanned + // up to but not including the last key (the key that current_ points to + // when this function returns). This key's checksum is verified in + // UpdateKey(). + void FindKeyAfterBinarySeek(const Slice& target, uint32_t index, + bool is_index_key_result); +}; + +class DataBlockIter final : public BlockIter { + public: + DataBlockIter() + : BlockIter(), read_amp_bitmap_(nullptr), last_bitmap_offset_(0) {} + void Initialize(const Comparator* raw_ucmp, const char* data, + uint32_t restarts, uint32_t num_restarts, + SequenceNumber global_seqno, + BlockReadAmpBitmap* read_amp_bitmap, + bool block_contents_pinned, + bool user_defined_timestamps_persisted, + DataBlockHashIndex* data_block_hash_index, + uint8_t protection_bytes_per_key, const char* kv_checksum, + uint32_t block_restart_interval) { + InitializeBase(raw_ucmp, data, restarts, num_restarts, global_seqno, + block_contents_pinned, user_defined_timestamps_persisted, + protection_bytes_per_key, kv_checksum, + block_restart_interval); + raw_key_.SetIsUserKey(false); + read_amp_bitmap_ = read_amp_bitmap; + last_bitmap_offset_ = current_ + 1; + data_block_hash_index_ = data_block_hash_index; + } + + Slice value() const override { + assert(Valid()); + if (read_amp_bitmap_ && current_ < restarts_ && + current_ != last_bitmap_offset_) { + read_amp_bitmap_->Mark(current_ /* current entry offset */, + NextEntryOffset() - 1); + last_bitmap_offset_ = current_; + } + return value_; + } + + // Returns if `target` may exist. + inline bool SeekForGet(const Slice& target) { +#ifndef NDEBUG + if (TEST_Corrupt_Callback("DataBlockIter::SeekForGet")) return true; +#endif + if (!data_block_hash_index_) { + SeekImpl(target); + UpdateKey(); + return true; + } + bool res = SeekForGetImpl(target); + UpdateKey(); + return res; + } + + void Invalidate(const Status& s) override { + BlockIter::Invalidate(s); + // Clear prev entries cache. + prev_entries_keys_buff_.clear(); + prev_entries_.clear(); + prev_entries_idx_ = -1; + } + + protected: + friend Block; + inline bool ParseNextDataKey(bool* is_shared); + void SeekToFirstImpl() override; + void SeekToLastImpl() override; + void SeekImpl(const Slice& target) override; + void SeekForPrevImpl(const Slice& target) override; + void NextImpl() override; + void PrevImpl() override; + + private: + // read-amp bitmap + BlockReadAmpBitmap* read_amp_bitmap_; + // last `current_` value we report to read-amp bitmp + mutable uint32_t last_bitmap_offset_; + struct CachedPrevEntry { + explicit CachedPrevEntry(uint32_t _offset, const char* _key_ptr, + size_t _key_offset, size_t _key_size, Slice _value) + : offset(_offset), + key_ptr(_key_ptr), + key_offset(_key_offset), + key_size(_key_size), + value(_value) {} + + // offset of entry in block + uint32_t offset; + // Pointer to key data in block (nullptr if key is delta-encoded) + const char* key_ptr; + // offset of key in prev_entries_keys_buff_ (0 if key_ptr is not nullptr) + size_t key_offset; + // size of key + size_t key_size; + // value slice pointing to data in block + Slice value; + }; + std::string prev_entries_keys_buff_; + std::vector prev_entries_; + int32_t prev_entries_idx_ = -1; + + DataBlockHashIndex* data_block_hash_index_; + + bool SeekForGetImpl(const Slice& target); +}; + +// Iterator over MetaBlocks. MetaBlocks are similar to Data Blocks and +// are used to store Properties associated with table. +// Meta blocks always store user keys (no sequence number) and always +// use the BytewiseComparator. Additionally, MetaBlock accesses are +// not recorded in the Statistics or for Read-Amplification. +class MetaBlockIter final : public BlockIter { + public: + MetaBlockIter() : BlockIter() { raw_key_.SetIsUserKey(true); } + void Initialize(const char* data, uint32_t restarts, uint32_t num_restarts, + bool block_contents_pinned, uint8_t protection_bytes_per_key, + const char* kv_checksum, uint32_t block_restart_interval) { + // Initializes the iterator with a BytewiseComparator and + // the raw key being a user key. + InitializeBase(BytewiseComparator(), data, restarts, num_restarts, + kDisableGlobalSequenceNumber, block_contents_pinned, + /* user_defined_timestamps_persisted */ true, + protection_bytes_per_key, kv_checksum, + block_restart_interval); + raw_key_.SetIsUserKey(true); + } + + Slice value() const override { + assert(Valid()); + return value_; + } + + protected: + friend Block; + void SeekToFirstImpl() override; + void SeekToLastImpl() override; + void SeekImpl(const Slice& target) override; + void SeekForPrevImpl(const Slice& target) override; + void NextImpl() override; + void PrevImpl() override; + // Meta index block's restart interval is always 1. See + // MetaIndexBuilder::MetaIndexBuilder() for hard-coded restart interval. + uint32_t GetRestartInterval() override { return 1; } + uint32_t NumberOfKeys(uint32_t) override { return num_restarts_; } +}; + +class IndexBlockIter final : public BlockIter { + public: + IndexBlockIter() : BlockIter(), prefix_index_(nullptr) {} + + // key_includes_seq, default true, means that the keys are in internal key + // format. + // value_is_full, default true, means that no delta encoding is + // applied to values. + void Initialize(const Comparator* raw_ucmp, const char* data, + uint32_t restarts, uint32_t num_restarts, + SequenceNumber global_seqno, BlockPrefixIndex* prefix_index, + bool have_first_key, bool key_includes_seq, + bool value_is_full, bool block_contents_pinned, + bool user_defined_timestamps_persisted, + uint8_t protection_bytes_per_key, const char* kv_checksum, + uint32_t block_restart_interval) { + InitializeBase(raw_ucmp, data, restarts, num_restarts, + kDisableGlobalSequenceNumber, block_contents_pinned, + user_defined_timestamps_persisted, protection_bytes_per_key, + kv_checksum, block_restart_interval); + raw_key_.SetIsUserKey(!key_includes_seq); + prefix_index_ = prefix_index; + value_delta_encoded_ = !value_is_full; + have_first_key_ = have_first_key; + if (have_first_key_ && global_seqno != kDisableGlobalSequenceNumber) { + global_seqno_state_.reset(new GlobalSeqnoState(global_seqno)); + } else { + global_seqno_state_.reset(); + } + } + + Slice user_key() const override { + assert(Valid()); + return raw_key_.GetUserKey(); + } + + IndexValue value() const override { + assert(Valid()); + if (value_delta_encoded_ || global_seqno_state_ != nullptr || + pad_min_timestamp_) { + return decoded_value_; + } else { + IndexValue entry; + Slice v = value_; + Status decode_s __attribute__((__unused__)) = + entry.DecodeFrom(&v, have_first_key_, nullptr); + assert(decode_s.ok()); + return entry; + } + } + + Slice raw_value() const { + assert(Valid()); + return value_; + } + + bool IsValuePinned() const override { + return global_seqno_state_ != nullptr ? false : BlockIter::IsValuePinned(); + } + + protected: + friend Block; + // IndexBlockIter follows a different contract for prefix iterator + // from data iterators. + // If prefix of the seek key `target` exists in the file, it must + // return the same result as total order seek. + // If the prefix of `target` doesn't exist in the file, it can either + // return the result of total order seek, or set both of Valid() = false + // and status() = NotFound(). + void SeekImpl(const Slice& target) override; + + void SeekForPrevImpl(const Slice&) override { + assert(false); + current_ = restarts_; + restart_index_ = num_restarts_; + status_ = Status::InvalidArgument( + "RocksDB internal error: should never call SeekForPrev() on index " + "blocks"); + raw_key_.Clear(); + value_.clear(); + } + + void PrevImpl() override; + void NextImpl() override; + void SeekToFirstImpl() override; + void SeekToLastImpl() override; + + private: + bool value_delta_encoded_; + bool have_first_key_; // value includes first_internal_key + BlockPrefixIndex* prefix_index_; + // Whether the value is delta encoded. In that case the value is assumed to be + // BlockHandle. The first value in each restart interval is the full encoded + // BlockHandle; the restart of encoded size part of the BlockHandle. The + // offset of delta encoded BlockHandles is computed by adding the size of + // previous delta encoded values in the same restart interval to the offset of + // the first value in that restart interval. + IndexValue decoded_value_; + + // When sequence number overwriting is enabled, this struct contains the seqno + // to overwrite with, and current first_internal_key with overwritten seqno. + // This is rarely used, so we put it behind a pointer and only allocate when + // needed. + struct GlobalSeqnoState { + // First internal key according to current index entry, but with sequence + // number overwritten to global_seqno. + IterKey first_internal_key; + SequenceNumber global_seqno; + + explicit GlobalSeqnoState(SequenceNumber seqno) : global_seqno(seqno) {} + }; + + std::unique_ptr global_seqno_state_; + + // Buffers the `first_internal_key` referred by `decoded_value_` when + // `pad_min_timestamp_` is true. + std::string first_internal_key_with_ts_; + + // Set *prefix_may_exist to false if no key possibly share the same prefix + // as `target`. If not set, the result position should be the same as total + // order Seek. + bool PrefixSeek(const Slice& target, uint32_t* index, bool* prefix_may_exist); + // Set *prefix_may_exist to false if no key can possibly share the same + // prefix as `target`. If not set, the result position should be the same + // as total order seek. + bool BinaryBlockIndexSeek(const Slice& target, uint32_t* block_ids, + uint32_t left, uint32_t right, uint32_t* index, + bool* prefix_may_exist); + inline int CompareBlockKey(uint32_t block_index, const Slice& target); + + inline bool ParseNextIndexKey(); + + // When value_delta_encoded_ is enabled it decodes the value which is assumed + // to be BlockHandle and put it to decoded_value_ + inline void DecodeCurrentValue(bool is_shared); +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/block_based_table_builder.cc b/librocksdb-sys/rocksdb/table/block_based/block_based_table_builder.cc new file mode 100644 index 0000000..26e071a --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block_based_table_builder.cc @@ -0,0 +1,2095 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "table/block_based/block_based_table_builder.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "block_cache.h" +#include "cache/cache_entry_roles.h" +#include "cache/cache_helpers.h" +#include "cache/cache_key.h" +#include "cache/cache_reservation_manager.h" +#include "db/dbformat.h" +#include "index_builder.h" +#include "logging/logging.h" +#include "memory/memory_allocator_impl.h" +#include "rocksdb/cache.h" +#include "rocksdb/comparator.h" +#include "rocksdb/env.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/flush_block_policy.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/table.h" +#include "rocksdb/types.h" +#include "table/block_based/block.h" +#include "table/block_based/block_based_table_factory.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/block_based/block_builder.h" +#include "table/block_based/filter_block.h" +#include "table/block_based/filter_policy_internal.h" +#include "table/block_based/full_filter_block.h" +#include "table/block_based/partitioned_filter_block.h" +#include "table/format.h" +#include "table/meta_blocks.h" +#include "table/table_builder.h" +#include "util/coding.h" +#include "util/compression.h" +#include "util/stop_watch.h" +#include "util/string_util.h" +#include "util/work_queue.h" + +namespace ROCKSDB_NAMESPACE { + +extern const std::string kHashIndexPrefixesBlock; +extern const std::string kHashIndexPrefixesMetadataBlock; + +// Without anonymous namespace here, we fail the warning -Wmissing-prototypes +namespace { + +constexpr size_t kBlockTrailerSize = BlockBasedTable::kBlockTrailerSize; + +// Create a filter block builder based on its type. +FilterBlockBuilder* CreateFilterBlockBuilder( + const ImmutableCFOptions& /*opt*/, const MutableCFOptions& mopt, + const FilterBuildingContext& context, + const bool use_delta_encoding_for_index_values, + PartitionedIndexBuilder* const p_index_builder, size_t ts_sz, + const bool persist_user_defined_timestamps) { + const BlockBasedTableOptions& table_opt = context.table_options; + assert(table_opt.filter_policy); // precondition + + FilterBitsBuilder* filter_bits_builder = + BloomFilterPolicy::GetBuilderFromContext(context); + if (filter_bits_builder == nullptr) { + return nullptr; + } else { + if (table_opt.partition_filters) { + assert(p_index_builder != nullptr); + // Since after partition cut request from filter builder it takes time + // until index builder actully cuts the partition, until the end of a + // data block potentially with many keys, we take the lower bound as + // partition size. + assert(table_opt.block_size_deviation <= 100); + auto partition_size = + static_cast(((table_opt.metadata_block_size * + (100 - table_opt.block_size_deviation)) + + 99) / + 100); + partition_size = std::max(partition_size, static_cast(1)); + return new PartitionedFilterBlockBuilder( + mopt.prefix_extractor.get(), table_opt.whole_key_filtering, + filter_bits_builder, table_opt.index_block_restart_interval, + use_delta_encoding_for_index_values, p_index_builder, partition_size, + ts_sz, persist_user_defined_timestamps); + } else { + return new FullFilterBlockBuilder(mopt.prefix_extractor.get(), + table_opt.whole_key_filtering, + filter_bits_builder); + } + } +} + +bool GoodCompressionRatio(size_t compressed_size, size_t uncomp_size, + int max_compressed_bytes_per_kb) { + // For efficiency, avoid floating point and division + return compressed_size <= + (static_cast(max_compressed_bytes_per_kb) * uncomp_size) >> + 10; +} + +} // namespace + +// format_version is the block format as defined in include/rocksdb/table.h +Slice CompressBlock(const Slice& uncompressed_data, const CompressionInfo& info, + CompressionType* type, uint32_t format_version, + bool allow_sample, std::string* compressed_output, + std::string* sampled_output_fast, + std::string* sampled_output_slow) { + assert(type); + assert(compressed_output); + assert(compressed_output->empty()); + + // If requested, we sample one in every N block with a + // fast and slow compression algorithm and report the stats. + // The users can use these stats to decide if it is worthwhile + // enabling compression and they also get a hint about which + // compression algorithm wil be beneficial. + if (allow_sample && info.SampleForCompression() && + Random::GetTLSInstance()->OneIn( + static_cast(info.SampleForCompression()))) { + // Sampling with a fast compression algorithm + if (sampled_output_fast && (LZ4_Supported() || Snappy_Supported())) { + CompressionType c = + LZ4_Supported() ? kLZ4Compression : kSnappyCompression; + CompressionContext context(c); + CompressionOptions options; + CompressionInfo info_tmp(options, context, + CompressionDict::GetEmptyDict(), c, + info.SampleForCompression()); + + CompressData(uncompressed_data, info_tmp, + GetCompressFormatForVersion(format_version), + sampled_output_fast); + } + + // Sampling with a slow but high-compression algorithm + if (sampled_output_slow && (ZSTD_Supported() || Zlib_Supported())) { + CompressionType c = ZSTD_Supported() ? kZSTD : kZlibCompression; + CompressionContext context(c); + CompressionOptions options; + CompressionInfo info_tmp(options, context, + CompressionDict::GetEmptyDict(), c, + info.SampleForCompression()); + + CompressData(uncompressed_data, info_tmp, + GetCompressFormatForVersion(format_version), + sampled_output_slow); + } + } + + int max_compressed_bytes_per_kb = info.options().max_compressed_bytes_per_kb; + if (info.type() == kNoCompression || max_compressed_bytes_per_kb <= 0) { + *type = kNoCompression; + return uncompressed_data; + } + + // Actually compress the data; if the compression method is not supported, + // or the compression fails etc., just fall back to uncompressed + if (!CompressData(uncompressed_data, info, + GetCompressFormatForVersion(format_version), + compressed_output)) { + *type = kNoCompression; + return uncompressed_data; + } + + // Check the compression ratio; if it's not good enough, just fall back to + // uncompressed + if (!GoodCompressionRatio(compressed_output->size(), uncompressed_data.size(), + max_compressed_bytes_per_kb)) { + *type = kNoCompression; + return uncompressed_data; + } + + *type = info.type(); + return *compressed_output; +} + +// kBlockBasedTableMagicNumber was picked by running +// echo rocksdb.table.block_based | sha1sum +// and taking the leading 64 bits. +// Please note that kBlockBasedTableMagicNumber may also be accessed by other +// .cc files +// for that reason we declare it extern in the header but to get the space +// allocated +// it must be not extern in one place. +const uint64_t kBlockBasedTableMagicNumber = 0x88e241b785f4cff7ull; +// We also support reading and writing legacy block based table format (for +// backwards compatibility) +const uint64_t kLegacyBlockBasedTableMagicNumber = 0xdb4775248b80fb57ull; + +// A collector that collects properties of interest to block-based table. +// For now this class looks heavy-weight since we only write one additional +// property. +// But in the foreseeable future, we will add more and more properties that are +// specific to block-based table. +class BlockBasedTableBuilder::BlockBasedTablePropertiesCollector + : public IntTblPropCollector { + public: + explicit BlockBasedTablePropertiesCollector( + BlockBasedTableOptions::IndexType index_type, bool whole_key_filtering, + bool prefix_filtering) + : index_type_(index_type), + whole_key_filtering_(whole_key_filtering), + prefix_filtering_(prefix_filtering) {} + + Status InternalAdd(const Slice& /*key*/, const Slice& /*value*/, + uint64_t /*file_size*/) override { + // Intentionally left blank. Have no interest in collecting stats for + // individual key/value pairs. + return Status::OK(); + } + + virtual void BlockAdd(uint64_t /* block_uncomp_bytes */, + uint64_t /* block_compressed_bytes_fast */, + uint64_t /* block_compressed_bytes_slow */) override { + // Intentionally left blank. No interest in collecting stats for + // blocks. + return; + } + + Status Finish(UserCollectedProperties* properties) override { + std::string val; + PutFixed32(&val, static_cast(index_type_)); + properties->insert({BlockBasedTablePropertyNames::kIndexType, val}); + properties->insert({BlockBasedTablePropertyNames::kWholeKeyFiltering, + whole_key_filtering_ ? kPropTrue : kPropFalse}); + properties->insert({BlockBasedTablePropertyNames::kPrefixFiltering, + prefix_filtering_ ? kPropTrue : kPropFalse}); + return Status::OK(); + } + + // The name of the properties collector can be used for debugging purpose. + const char* Name() const override { + return "BlockBasedTablePropertiesCollector"; + } + + UserCollectedProperties GetReadableProperties() const override { + // Intentionally left blank. + return UserCollectedProperties(); + } + + private: + BlockBasedTableOptions::IndexType index_type_; + bool whole_key_filtering_; + bool prefix_filtering_; +}; + +struct BlockBasedTableBuilder::Rep { + const ImmutableOptions ioptions; + // BEGIN from MutableCFOptions + std::shared_ptr prefix_extractor; + // END from MutableCFOptions + const BlockBasedTableOptions table_options; + const InternalKeyComparator& internal_comparator; + // Size in bytes for the user-defined timestamps. + size_t ts_sz; + // When `ts_sz` > 0 and this flag is false, the user-defined timestamp in the + // user key will be stripped when creating the block based table. This + // stripping happens for all user keys, including the keys in data block, + // index block for data block, index block for index block (if index type is + // `kTwoLevelIndexSearch`), index for filter blocks (if using partitioned + // filters), the `first_internal_key` in `IndexValue`, the `end_key` for range + // deletion entries. + // As long as the user keys are sorted when added via `Add` API, their logic + // ordering won't change after timestamps are stripped. However, for each user + // key to be logically equivalent before and after timestamp is stripped, the + // user key should contain the minimum timestamp. + bool persist_user_defined_timestamps; + WritableFileWriter* file; + std::atomic offset; + size_t alignment; + BlockBuilder data_block; + // Buffers uncompressed data blocks to replay later. Needed when + // compression dictionary is enabled so we can finalize the dictionary before + // compressing any data blocks. + std::vector data_block_buffers; + BlockBuilder range_del_block; + + InternalKeySliceTransform internal_prefix_transform; + std::unique_ptr index_builder; + PartitionedIndexBuilder* p_index_builder_ = nullptr; + + std::string last_key; + const Slice* first_key_in_next_block = nullptr; + CompressionType compression_type; + uint64_t sample_for_compression; + std::atomic compressible_input_data_bytes; + std::atomic uncompressible_input_data_bytes; + std::atomic sampled_input_data_bytes; + std::atomic sampled_output_slow_data_bytes; + std::atomic sampled_output_fast_data_bytes; + CompressionOptions compression_opts; + std::unique_ptr compression_dict; + std::vector> compression_ctxs; + std::vector> verify_ctxs; + std::unique_ptr verify_dict; + + size_t data_begin_offset = 0; + + TableProperties props; + + // States of the builder. + // + // - `kBuffered`: This is the initial state where zero or more data blocks are + // accumulated uncompressed in-memory. From this state, call + // `EnterUnbuffered()` to finalize the compression dictionary if enabled, + // compress/write out any buffered blocks, and proceed to the `kUnbuffered` + // state. + // + // - `kUnbuffered`: This is the state when compression dictionary is finalized + // either because it wasn't enabled in the first place or it's been created + // from sampling previously buffered data. In this state, blocks are simply + // compressed/written out as they fill up. From this state, call `Finish()` + // to complete the file (write meta-blocks, etc.), or `Abandon()` to delete + // the partially created file. + // + // - `kClosed`: This indicates either `Finish()` or `Abandon()` has been + // called, so the table builder is no longer usable. We must be in this + // state by the time the destructor runs. + enum class State { + kBuffered, + kUnbuffered, + kClosed, + }; + State state; + // `kBuffered` state is allowed only as long as the buffering of uncompressed + // data blocks (see `data_block_buffers`) does not exceed `buffer_limit`. + uint64_t buffer_limit; + std::shared_ptr + compression_dict_buffer_cache_res_mgr; + const bool use_delta_encoding_for_index_values; + std::unique_ptr filter_builder; + OffsetableCacheKey base_cache_key; + const TableFileCreationReason reason; + + BlockHandle pending_handle; // Handle to add to index block + + std::string compressed_output; + std::unique_ptr flush_block_policy; + + std::vector> table_properties_collectors; + + std::unique_ptr pc_rep; + BlockCreateContext create_context; + + // The size of the "tail" part of a SST file. "Tail" refers to + // all blocks after data blocks till the end of the SST file. + uint64_t tail_size; + + // See class Footer + uint32_t base_context_checksum; + + uint64_t get_offset() { return offset.load(std::memory_order_relaxed); } + void set_offset(uint64_t o) { offset.store(o, std::memory_order_relaxed); } + + bool IsParallelCompressionEnabled() const { + return compression_opts.parallel_threads > 1; + } + + Status GetStatus() { + // We need to make modifications of status visible when status_ok is set + // to false, and this is ensured by status_mutex, so no special memory + // order for status_ok is required. + if (status_ok.load(std::memory_order_relaxed)) { + return Status::OK(); + } else { + return CopyStatus(); + } + } + + Status CopyStatus() { + std::lock_guard lock(status_mutex); + return status; + } + + IOStatus GetIOStatus() { + // We need to make modifications of io_status visible when status_ok is set + // to false, and this is ensured by io_status_mutex, so no special memory + // order for io_status_ok is required. + if (io_status_ok.load(std::memory_order_relaxed)) { +#ifdef ROCKSDB_ASSERT_STATUS_CHECKED // Avoid unnecessary lock acquisition + auto ios = CopyIOStatus(); + ios.PermitUncheckedError(); + // Assume no races in unit tests + assert(ios.ok()); +#endif // ROCKSDB_ASSERT_STATUS_CHECKED + return IOStatus::OK(); + } else { + return CopyIOStatus(); + } + } + + IOStatus CopyIOStatus() { + std::lock_guard lock(io_status_mutex); + return io_status; + } + + // Never erase an existing status that is not OK. + void SetStatus(Status s) { + if (!s.ok() && status_ok.load(std::memory_order_relaxed)) { + // Locking is an overkill for non compression_opts.parallel_threads + // case but since it's unlikely that s is not OK, we take this cost + // to be simplicity. + std::lock_guard lock(status_mutex); + status = s; + status_ok.store(false, std::memory_order_relaxed); + } + } + + // Never erase an existing I/O status that is not OK. + // Calling this will also SetStatus(ios) + void SetIOStatus(IOStatus ios) { + if (!ios.ok() && io_status_ok.load(std::memory_order_relaxed)) { + // Locking is an overkill for non compression_opts.parallel_threads + // case but since it's unlikely that s is not OK, we take this cost + // to be simplicity. + std::lock_guard lock(io_status_mutex); + io_status = ios; + io_status_ok.store(false, std::memory_order_relaxed); + } + SetStatus(ios); + } + + Rep(const BlockBasedTableOptions& table_opt, const TableBuilderOptions& tbo, + WritableFileWriter* f) + : ioptions(tbo.ioptions), + prefix_extractor(tbo.moptions.prefix_extractor), + table_options(table_opt), + internal_comparator(tbo.internal_comparator), + ts_sz(tbo.internal_comparator.user_comparator()->timestamp_size()), + persist_user_defined_timestamps( + tbo.ioptions.persist_user_defined_timestamps), + file(f), + offset(0), + alignment(table_options.block_align + ? std::min(static_cast(table_options.block_size), + kDefaultPageSize) + : 0), + data_block(table_options.block_restart_interval, + table_options.use_delta_encoding, + false /* use_value_delta_encoding */, + tbo.internal_comparator.user_comparator() + ->CanKeysWithDifferentByteContentsBeEqual() + ? BlockBasedTableOptions::kDataBlockBinarySearch + : table_options.data_block_index_type, + table_options.data_block_hash_table_util_ratio, ts_sz, + persist_user_defined_timestamps), + range_del_block( + 1 /* block_restart_interval */, true /* use_delta_encoding */, + false /* use_value_delta_encoding */, + BlockBasedTableOptions::kDataBlockBinarySearch /* index_type */, + 0.75 /* data_block_hash_table_util_ratio */, ts_sz, + persist_user_defined_timestamps), + internal_prefix_transform(prefix_extractor.get()), + compression_type(tbo.compression_type), + sample_for_compression(tbo.moptions.sample_for_compression), + compressible_input_data_bytes(0), + uncompressible_input_data_bytes(0), + sampled_input_data_bytes(0), + sampled_output_slow_data_bytes(0), + sampled_output_fast_data_bytes(0), + compression_opts(tbo.compression_opts), + compression_dict(), + compression_ctxs(tbo.compression_opts.parallel_threads), + verify_ctxs(tbo.compression_opts.parallel_threads), + verify_dict(), + state((tbo.compression_opts.max_dict_bytes > 0) ? State::kBuffered + : State::kUnbuffered), + use_delta_encoding_for_index_values(table_opt.format_version >= 4 && + !table_opt.block_align), + reason(tbo.reason), + flush_block_policy( + table_options.flush_block_policy_factory->NewFlushBlockPolicy( + table_options, data_block)), + create_context(&table_options, ioptions.stats, + compression_type == kZSTD || + compression_type == kZSTDNotFinalCompression, + tbo.moptions.block_protection_bytes_per_key, + tbo.internal_comparator.user_comparator(), + !use_delta_encoding_for_index_values, + table_opt.index_type == + BlockBasedTableOptions::kBinarySearchWithFirstKey), + tail_size(0), + status_ok(true), + io_status_ok(true) { + if (tbo.target_file_size == 0) { + buffer_limit = compression_opts.max_dict_buffer_bytes; + } else if (compression_opts.max_dict_buffer_bytes == 0) { + buffer_limit = tbo.target_file_size; + } else { + buffer_limit = std::min(tbo.target_file_size, + compression_opts.max_dict_buffer_bytes); + } + + const auto compress_dict_build_buffer_charged = + table_options.cache_usage_options.options_overrides + .at(CacheEntryRole::kCompressionDictionaryBuildingBuffer) + .charged; + if (table_options.block_cache && + (compress_dict_build_buffer_charged == + CacheEntryRoleOptions::Decision::kEnabled || + compress_dict_build_buffer_charged == + CacheEntryRoleOptions::Decision::kFallback)) { + compression_dict_buffer_cache_res_mgr = + std::make_shared>( + table_options.block_cache); + } else { + compression_dict_buffer_cache_res_mgr = nullptr; + } + + for (uint32_t i = 0; i < compression_opts.parallel_threads; i++) { + compression_ctxs[i].reset(new CompressionContext(compression_type)); + } + if (table_options.index_type == + BlockBasedTableOptions::kTwoLevelIndexSearch) { + p_index_builder_ = PartitionedIndexBuilder::CreateIndexBuilder( + &internal_comparator, use_delta_encoding_for_index_values, + table_options, ts_sz, persist_user_defined_timestamps); + index_builder.reset(p_index_builder_); + } else { + index_builder.reset(IndexBuilder::CreateIndexBuilder( + table_options.index_type, &internal_comparator, + &this->internal_prefix_transform, use_delta_encoding_for_index_values, + table_options, ts_sz, persist_user_defined_timestamps)); + } + if (ioptions.optimize_filters_for_hits && tbo.is_bottommost) { + // Apply optimize_filters_for_hits setting here when applicable by + // skipping filter generation + filter_builder.reset(); + } else if (tbo.skip_filters) { + // For SstFileWriter skip_filters + filter_builder.reset(); + } else if (!table_options.filter_policy) { + // Null filter_policy -> no filter + filter_builder.reset(); + } else { + FilterBuildingContext filter_context(table_options); + + filter_context.info_log = ioptions.logger; + filter_context.column_family_name = tbo.column_family_name; + filter_context.reason = reason; + + // Only populate other fields if known to be in LSM rather than + // generating external SST file + if (reason != TableFileCreationReason::kMisc) { + filter_context.compaction_style = ioptions.compaction_style; + filter_context.num_levels = ioptions.num_levels; + filter_context.level_at_creation = tbo.level_at_creation; + filter_context.is_bottommost = tbo.is_bottommost; + assert(filter_context.level_at_creation < filter_context.num_levels); + } + + filter_builder.reset(CreateFilterBlockBuilder( + ioptions, tbo.moptions, filter_context, + use_delta_encoding_for_index_values, p_index_builder_, ts_sz, + persist_user_defined_timestamps)); + } + + assert(tbo.int_tbl_prop_collector_factories); + for (auto& factory : *tbo.int_tbl_prop_collector_factories) { + assert(factory); + + table_properties_collectors.emplace_back( + factory->CreateIntTblPropCollector(tbo.column_family_id, + tbo.level_at_creation)); + } + table_properties_collectors.emplace_back( + new BlockBasedTablePropertiesCollector( + table_options.index_type, table_options.whole_key_filtering, + prefix_extractor != nullptr)); + if (ts_sz > 0 && persist_user_defined_timestamps) { + table_properties_collectors.emplace_back( + new TimestampTablePropertiesCollector( + tbo.internal_comparator.user_comparator())); + } + if (table_options.verify_compression) { + for (uint32_t i = 0; i < compression_opts.parallel_threads; i++) { + verify_ctxs[i].reset(new UncompressionContext(compression_type)); + } + } + + // These are only needed for populating table properties + props.column_family_id = tbo.column_family_id; + props.column_family_name = tbo.column_family_name; + props.oldest_key_time = tbo.oldest_key_time; + props.file_creation_time = tbo.file_creation_time; + props.orig_file_number = tbo.cur_file_num; + props.db_id = tbo.db_id; + props.db_session_id = tbo.db_session_id; + props.db_host_id = ioptions.db_host_id; + if (!ReifyDbHostIdProperty(ioptions.env, &props.db_host_id).ok()) { + ROCKS_LOG_INFO(ioptions.logger, "db_host_id property will not be set"); + } + + if (FormatVersionUsesContextChecksum(table_options.format_version)) { + // Must be non-zero and semi- or quasi-random + // TODO: ideally guaranteed different for related files (e.g. use file + // number and db_session, for benefit of SstFileWriter) + do { + base_context_checksum = Random::GetTLSInstance()->Next(); + } while (UNLIKELY(base_context_checksum == 0)); + } else { + base_context_checksum = 0; + } + } + + Rep(const Rep&) = delete; + Rep& operator=(const Rep&) = delete; + + private: + // Synchronize status & io_status accesses across threads from main thread, + // compression thread and write thread in parallel compression. + std::mutex status_mutex; + std::atomic status_ok; + Status status; + std::mutex io_status_mutex; + std::atomic io_status_ok; + IOStatus io_status; +}; + +struct BlockBasedTableBuilder::ParallelCompressionRep { + // Keys is a wrapper of vector of strings avoiding + // releasing string memories during vector clear() + // in order to save memory allocation overhead + class Keys { + public: + Keys() : keys_(kKeysInitSize), size_(0) {} + void PushBack(const Slice& key) { + if (size_ == keys_.size()) { + keys_.emplace_back(key.data(), key.size()); + } else { + keys_[size_].assign(key.data(), key.size()); + } + size_++; + } + void SwapAssign(std::vector& keys) { + size_ = keys.size(); + std::swap(keys_, keys); + } + void Clear() { size_ = 0; } + size_t Size() { return size_; } + std::string& Back() { return keys_[size_ - 1]; } + std::string& operator[](size_t idx) { + assert(idx < size_); + return keys_[idx]; + } + + private: + const size_t kKeysInitSize = 32; + std::vector keys_; + size_t size_; + }; + std::unique_ptr curr_block_keys; + + class BlockRepSlot; + + // BlockRep instances are fetched from and recycled to + // block_rep_pool during parallel compression. + struct BlockRep { + Slice contents; + Slice compressed_contents; + std::unique_ptr data; + std::unique_ptr compressed_data; + CompressionType compression_type; + std::unique_ptr first_key_in_next_block; + std::unique_ptr keys; + std::unique_ptr slot; + Status status; + }; + // Use a vector of BlockRep as a buffer for a determined number + // of BlockRep structures. All data referenced by pointers in + // BlockRep will be freed when this vector is destructed. + using BlockRepBuffer = std::vector; + BlockRepBuffer block_rep_buf; + // Use a thread-safe queue for concurrent access from block + // building thread and writer thread. + using BlockRepPool = WorkQueue; + BlockRepPool block_rep_pool; + + // Use BlockRepSlot to keep block order in write thread. + // slot_ will pass references to BlockRep + class BlockRepSlot { + public: + BlockRepSlot() : slot_(1) {} + template + void Fill(T&& rep) { + slot_.push(std::forward(rep)); + }; + void Take(BlockRep*& rep) { slot_.pop(rep); } + + private: + // slot_ will pass references to BlockRep in block_rep_buf, + // and those references are always valid before the destruction of + // block_rep_buf. + WorkQueue slot_; + }; + + // Compression queue will pass references to BlockRep in block_rep_buf, + // and those references are always valid before the destruction of + // block_rep_buf. + using CompressQueue = WorkQueue; + CompressQueue compress_queue; + std::vector compress_thread_pool; + + // Write queue will pass references to BlockRep::slot in block_rep_buf, + // and those references are always valid before the corresponding + // BlockRep::slot is destructed, which is before the destruction of + // block_rep_buf. + using WriteQueue = WorkQueue; + WriteQueue write_queue; + std::unique_ptr write_thread; + + // Estimate output file size when parallel compression is enabled. This is + // necessary because compression & flush are no longer synchronized, + // and BlockBasedTableBuilder::FileSize() is no longer accurate. + // memory_order_relaxed suffices because accurate statistics is not required. + class FileSizeEstimator { + public: + explicit FileSizeEstimator() + : uncomp_bytes_compressed(0), + uncomp_bytes_curr_block(0), + uncomp_bytes_curr_block_set(false), + uncomp_bytes_inflight(0), + blocks_inflight(0), + curr_compression_ratio(0), + estimated_file_size(0) {} + + // Estimate file size when a block is about to be emitted to + // compression thread + void EmitBlock(uint64_t uncomp_block_size, uint64_t curr_file_size) { + uint64_t new_uncomp_bytes_inflight = + uncomp_bytes_inflight.fetch_add(uncomp_block_size, + std::memory_order_relaxed) + + uncomp_block_size; + + uint64_t new_blocks_inflight = + blocks_inflight.fetch_add(1, std::memory_order_relaxed) + 1; + + estimated_file_size.store( + curr_file_size + + static_cast( + static_cast(new_uncomp_bytes_inflight) * + curr_compression_ratio.load(std::memory_order_relaxed)) + + new_blocks_inflight * kBlockTrailerSize, + std::memory_order_relaxed); + } + + // Estimate file size when a block is already reaped from + // compression thread + void ReapBlock(uint64_t compressed_block_size, uint64_t curr_file_size) { + assert(uncomp_bytes_curr_block_set); + + uint64_t new_uncomp_bytes_compressed = + uncomp_bytes_compressed + uncomp_bytes_curr_block; + assert(new_uncomp_bytes_compressed > 0); + + curr_compression_ratio.store( + (curr_compression_ratio.load(std::memory_order_relaxed) * + uncomp_bytes_compressed + + compressed_block_size) / + static_cast(new_uncomp_bytes_compressed), + std::memory_order_relaxed); + uncomp_bytes_compressed = new_uncomp_bytes_compressed; + + uint64_t new_uncomp_bytes_inflight = + uncomp_bytes_inflight.fetch_sub(uncomp_bytes_curr_block, + std::memory_order_relaxed) - + uncomp_bytes_curr_block; + + uint64_t new_blocks_inflight = + blocks_inflight.fetch_sub(1, std::memory_order_relaxed) - 1; + + estimated_file_size.store( + curr_file_size + + static_cast( + static_cast(new_uncomp_bytes_inflight) * + curr_compression_ratio.load(std::memory_order_relaxed)) + + new_blocks_inflight * kBlockTrailerSize, + std::memory_order_relaxed); + + uncomp_bytes_curr_block_set = false; + } + + void SetEstimatedFileSize(uint64_t size) { + estimated_file_size.store(size, std::memory_order_relaxed); + } + + uint64_t GetEstimatedFileSize() { + return estimated_file_size.load(std::memory_order_relaxed); + } + + void SetCurrBlockUncompSize(uint64_t size) { + uncomp_bytes_curr_block = size; + uncomp_bytes_curr_block_set = true; + } + + private: + // Input bytes compressed so far. + uint64_t uncomp_bytes_compressed; + // Size of current block being appended. + uint64_t uncomp_bytes_curr_block; + // Whether uncomp_bytes_curr_block has been set for next + // ReapBlock call. + bool uncomp_bytes_curr_block_set; + // Input bytes under compression and not appended yet. + std::atomic uncomp_bytes_inflight; + // Number of blocks under compression and not appended yet. + std::atomic blocks_inflight; + // Current compression ratio, maintained by BGWorkWriteMaybeCompressedBlock. + std::atomic curr_compression_ratio; + // Estimated SST file size. + std::atomic estimated_file_size; + }; + FileSizeEstimator file_size_estimator; + + // Facilities used for waiting first block completion. Need to Wait for + // the completion of first block compression and flush to get a non-zero + // compression ratio. + std::atomic first_block_processed; + std::condition_variable first_block_cond; + std::mutex first_block_mutex; + + explicit ParallelCompressionRep(uint32_t parallel_threads) + : curr_block_keys(new Keys()), + block_rep_buf(parallel_threads), + block_rep_pool(parallel_threads), + compress_queue(parallel_threads), + write_queue(parallel_threads), + first_block_processed(false) { + for (uint32_t i = 0; i < parallel_threads; i++) { + block_rep_buf[i].contents = Slice(); + block_rep_buf[i].compressed_contents = Slice(); + block_rep_buf[i].data.reset(new std::string()); + block_rep_buf[i].compressed_data.reset(new std::string()); + block_rep_buf[i].compression_type = CompressionType(); + block_rep_buf[i].first_key_in_next_block.reset(new std::string()); + block_rep_buf[i].keys.reset(new Keys()); + block_rep_buf[i].slot.reset(new BlockRepSlot()); + block_rep_buf[i].status = Status::OK(); + block_rep_pool.push(&block_rep_buf[i]); + } + } + + ~ParallelCompressionRep() { block_rep_pool.finish(); } + + // Make a block prepared to be emitted to compression thread + // Used in non-buffered mode + BlockRep* PrepareBlock(CompressionType compression_type, + const Slice* first_key_in_next_block, + BlockBuilder* data_block) { + BlockRep* block_rep = + PrepareBlockInternal(compression_type, first_key_in_next_block); + assert(block_rep != nullptr); + data_block->SwapAndReset(*(block_rep->data)); + block_rep->contents = *(block_rep->data); + std::swap(block_rep->keys, curr_block_keys); + curr_block_keys->Clear(); + return block_rep; + } + + // Used in EnterUnbuffered + BlockRep* PrepareBlock(CompressionType compression_type, + const Slice* first_key_in_next_block, + std::string* data_block, + std::vector* keys) { + BlockRep* block_rep = + PrepareBlockInternal(compression_type, first_key_in_next_block); + assert(block_rep != nullptr); + std::swap(*(block_rep->data), *data_block); + block_rep->contents = *(block_rep->data); + block_rep->keys->SwapAssign(*keys); + return block_rep; + } + + // Emit a block to compression thread + void EmitBlock(BlockRep* block_rep) { + assert(block_rep != nullptr); + assert(block_rep->status.ok()); + if (!write_queue.push(block_rep->slot.get())) { + return; + } + if (!compress_queue.push(block_rep)) { + return; + } + + if (!first_block_processed.load(std::memory_order_relaxed)) { + std::unique_lock lock(first_block_mutex); + first_block_cond.wait(lock, [this] { + return first_block_processed.load(std::memory_order_relaxed); + }); + } + } + + // Reap a block from compression thread + void ReapBlock(BlockRep* block_rep) { + assert(block_rep != nullptr); + block_rep->compressed_data->clear(); + block_rep_pool.push(block_rep); + + if (!first_block_processed.load(std::memory_order_relaxed)) { + std::lock_guard lock(first_block_mutex); + first_block_processed.store(true, std::memory_order_relaxed); + first_block_cond.notify_one(); + } + } + + private: + BlockRep* PrepareBlockInternal(CompressionType compression_type, + const Slice* first_key_in_next_block) { + BlockRep* block_rep = nullptr; + block_rep_pool.pop(block_rep); + assert(block_rep != nullptr); + + assert(block_rep->data); + + block_rep->compression_type = compression_type; + + if (first_key_in_next_block == nullptr) { + block_rep->first_key_in_next_block.reset(nullptr); + } else { + block_rep->first_key_in_next_block->assign( + first_key_in_next_block->data(), first_key_in_next_block->size()); + } + + return block_rep; + } +}; + +BlockBasedTableBuilder::BlockBasedTableBuilder( + const BlockBasedTableOptions& table_options, const TableBuilderOptions& tbo, + WritableFileWriter* file) { + BlockBasedTableOptions sanitized_table_options(table_options); + if (sanitized_table_options.format_version == 0 && + sanitized_table_options.checksum != kCRC32c) { + ROCKS_LOG_WARN( + tbo.ioptions.logger, + "Silently converting format_version to 1 because checksum is " + "non-default"); + // silently convert format_version to 1 to keep consistent with current + // behavior + sanitized_table_options.format_version = 1; + } + auto ucmp = tbo.internal_comparator.user_comparator(); + assert(ucmp); + (void)ucmp; // avoids unused variable error. + rep_ = new Rep(sanitized_table_options, tbo, file); + + TEST_SYNC_POINT_CALLBACK( + "BlockBasedTableBuilder::BlockBasedTableBuilder:PreSetupBaseCacheKey", + const_cast(&rep_->props)); + + BlockBasedTable::SetupBaseCacheKey(&rep_->props, tbo.db_session_id, + tbo.cur_file_num, &rep_->base_cache_key); + + if (rep_->IsParallelCompressionEnabled()) { + StartParallelCompression(); + } +} + +BlockBasedTableBuilder::~BlockBasedTableBuilder() { + // Catch errors where caller forgot to call Finish() + assert(rep_->state == Rep::State::kClosed); + delete rep_; +} + +void BlockBasedTableBuilder::Add(const Slice& key, const Slice& value) { + Rep* r = rep_; + assert(rep_->state != Rep::State::kClosed); + if (!ok()) return; + ValueType value_type = ExtractValueType(key); + if (IsValueType(value_type)) { +#ifndef NDEBUG + if (r->props.num_entries > r->props.num_range_deletions) { + assert(r->internal_comparator.Compare(key, Slice(r->last_key)) > 0); + } +#endif // !NDEBUG + + auto should_flush = r->flush_block_policy->Update(key, value); + if (should_flush) { + assert(!r->data_block.empty()); + r->first_key_in_next_block = &key; + Flush(); + if (r->state == Rep::State::kBuffered) { + bool exceeds_buffer_limit = + (r->buffer_limit != 0 && r->data_begin_offset > r->buffer_limit); + bool exceeds_global_block_cache_limit = false; + + // Increase cache charging for the last buffered data block + // only if the block is not going to be unbuffered immediately + // and there exists a cache reservation manager + if (!exceeds_buffer_limit && + r->compression_dict_buffer_cache_res_mgr != nullptr) { + Status s = + r->compression_dict_buffer_cache_res_mgr->UpdateCacheReservation( + r->data_begin_offset); + exceeds_global_block_cache_limit = s.IsMemoryLimit(); + } + + if (exceeds_buffer_limit || exceeds_global_block_cache_limit) { + EnterUnbuffered(); + } + } + + // Add item to index block. + // We do not emit the index entry for a block until we have seen the + // first key for the next data block. This allows us to use shorter + // keys in the index block. For example, consider a block boundary + // between the keys "the quick brown fox" and "the who". We can use + // "the r" as the key for the index block entry since it is >= all + // entries in the first block and < all entries in subsequent + // blocks. + if (ok() && r->state == Rep::State::kUnbuffered) { + if (r->IsParallelCompressionEnabled()) { + r->pc_rep->curr_block_keys->Clear(); + } else { + r->index_builder->AddIndexEntry(&r->last_key, &key, + r->pending_handle); + } + } + } + + // Note: PartitionedFilterBlockBuilder requires key being added to filter + // builder after being added to index builder. + if (r->state == Rep::State::kUnbuffered) { + if (r->IsParallelCompressionEnabled()) { + r->pc_rep->curr_block_keys->PushBack(key); + } else { + if (r->filter_builder != nullptr) { + r->filter_builder->Add( + ExtractUserKeyAndStripTimestamp(key, r->ts_sz)); + } + } + } + + r->data_block.AddWithLastKey(key, value, r->last_key); + r->last_key.assign(key.data(), key.size()); + if (r->state == Rep::State::kBuffered) { + // Buffered keys will be replayed from data_block_buffers during + // `Finish()` once compression dictionary has been finalized. + } else { + if (!r->IsParallelCompressionEnabled()) { + r->index_builder->OnKeyAdded(key); + } + } + // TODO offset passed in is not accurate for parallel compression case + NotifyCollectTableCollectorsOnAdd(key, value, r->get_offset(), + r->table_properties_collectors, + r->ioptions.logger); + + } else if (value_type == kTypeRangeDeletion) { + // TODO(yuzhangyu): handle range deletion entries for UDT in memtable only. + r->range_del_block.Add(key, value); + // TODO offset passed in is not accurate for parallel compression case + NotifyCollectTableCollectorsOnAdd(key, value, r->get_offset(), + r->table_properties_collectors, + r->ioptions.logger); + } else { + assert(false); + } + + r->props.num_entries++; + r->props.raw_key_size += key.size(); + if (!r->persist_user_defined_timestamps) { + r->props.raw_key_size -= r->ts_sz; + } + r->props.raw_value_size += value.size(); + if (value_type == kTypeDeletion || value_type == kTypeSingleDeletion || + value_type == kTypeDeletionWithTimestamp) { + r->props.num_deletions++; + } else if (value_type == kTypeRangeDeletion) { + r->props.num_deletions++; + r->props.num_range_deletions++; + } else if (value_type == kTypeMerge) { + r->props.num_merge_operands++; + } +} + +void BlockBasedTableBuilder::Flush() { + Rep* r = rep_; + assert(rep_->state != Rep::State::kClosed); + if (!ok()) return; + if (r->data_block.empty()) return; + if (r->IsParallelCompressionEnabled() && + r->state == Rep::State::kUnbuffered) { + r->data_block.Finish(); + ParallelCompressionRep::BlockRep* block_rep = r->pc_rep->PrepareBlock( + r->compression_type, r->first_key_in_next_block, &(r->data_block)); + assert(block_rep != nullptr); + r->pc_rep->file_size_estimator.EmitBlock(block_rep->data->size(), + r->get_offset()); + r->pc_rep->EmitBlock(block_rep); + } else { + WriteBlock(&r->data_block, &r->pending_handle, BlockType::kData); + } +} + +void BlockBasedTableBuilder::WriteBlock(BlockBuilder* block, + BlockHandle* handle, + BlockType block_type) { + block->Finish(); + std::string uncompressed_block_data; + uncompressed_block_data.reserve(rep_->table_options.block_size); + block->SwapAndReset(uncompressed_block_data); + if (rep_->state == Rep::State::kBuffered) { + assert(block_type == BlockType::kData); + rep_->data_block_buffers.emplace_back(std::move(uncompressed_block_data)); + rep_->data_begin_offset += rep_->data_block_buffers.back().size(); + return; + } + WriteBlock(uncompressed_block_data, handle, block_type); +} + +void BlockBasedTableBuilder::WriteBlock(const Slice& uncompressed_block_data, + BlockHandle* handle, + BlockType block_type) { + Rep* r = rep_; + assert(r->state == Rep::State::kUnbuffered); + Slice block_contents; + CompressionType type; + Status compress_status; + bool is_data_block = block_type == BlockType::kData; + CompressAndVerifyBlock(uncompressed_block_data, is_data_block, + *(r->compression_ctxs[0]), r->verify_ctxs[0].get(), + &(r->compressed_output), &(block_contents), &type, + &compress_status); + r->SetStatus(compress_status); + if (!ok()) { + return; + } + + WriteMaybeCompressedBlock(block_contents, type, handle, block_type, + &uncompressed_block_data); + r->compressed_output.clear(); + if (is_data_block) { + r->props.data_size = r->get_offset(); + ++r->props.num_data_blocks; + } +} + +void BlockBasedTableBuilder::BGWorkCompression( + const CompressionContext& compression_ctx, + UncompressionContext* verify_ctx) { + ParallelCompressionRep::BlockRep* block_rep = nullptr; + while (rep_->pc_rep->compress_queue.pop(block_rep)) { + assert(block_rep != nullptr); + CompressAndVerifyBlock(block_rep->contents, true, /* is_data_block*/ + compression_ctx, verify_ctx, + block_rep->compressed_data.get(), + &block_rep->compressed_contents, + &(block_rep->compression_type), &block_rep->status); + block_rep->slot->Fill(block_rep); + } +} + +void BlockBasedTableBuilder::CompressAndVerifyBlock( + const Slice& uncompressed_block_data, bool is_data_block, + const CompressionContext& compression_ctx, UncompressionContext* verify_ctx, + std::string* compressed_output, Slice* block_contents, + CompressionType* type, Status* out_status) { + Rep* r = rep_; + bool is_status_ok = ok(); + if (!r->IsParallelCompressionEnabled()) { + assert(is_status_ok); + } + + if (is_status_ok && uncompressed_block_data.size() < kCompressionSizeLimit) { + StopWatchNano timer( + r->ioptions.clock, + ShouldReportDetailedTime(r->ioptions.env, r->ioptions.stats)); + + if (is_data_block) { + r->compressible_input_data_bytes.fetch_add(uncompressed_block_data.size(), + std::memory_order_relaxed); + } + const CompressionDict* compression_dict; + if (!is_data_block || r->compression_dict == nullptr) { + compression_dict = &CompressionDict::GetEmptyDict(); + } else { + compression_dict = r->compression_dict.get(); + } + assert(compression_dict != nullptr); + CompressionInfo compression_info(r->compression_opts, compression_ctx, + *compression_dict, r->compression_type, + r->sample_for_compression); + + std::string sampled_output_fast; + std::string sampled_output_slow; + *block_contents = CompressBlock( + uncompressed_block_data, compression_info, type, + r->table_options.format_version, is_data_block /* allow_sample */, + compressed_output, &sampled_output_fast, &sampled_output_slow); + + if (sampled_output_slow.size() > 0 || sampled_output_fast.size() > 0) { + // Currently compression sampling is only enabled for data block. + assert(is_data_block); + r->sampled_input_data_bytes.fetch_add(uncompressed_block_data.size(), + std::memory_order_relaxed); + r->sampled_output_slow_data_bytes.fetch_add(sampled_output_slow.size(), + std::memory_order_relaxed); + r->sampled_output_fast_data_bytes.fetch_add(sampled_output_fast.size(), + std::memory_order_relaxed); + } + // notify collectors on block add + NotifyCollectTableCollectorsOnBlockAdd( + r->table_properties_collectors, uncompressed_block_data.size(), + sampled_output_fast.size(), sampled_output_slow.size()); + + // Some of the compression algorithms are known to be unreliable. If + // the verify_compression flag is set then try to de-compress the + // compressed data and compare to the input. + if (*type != kNoCompression && r->table_options.verify_compression) { + // Retrieve the uncompressed contents into a new buffer + const UncompressionDict* verify_dict; + if (!is_data_block || r->verify_dict == nullptr) { + verify_dict = &UncompressionDict::GetEmptyDict(); + } else { + verify_dict = r->verify_dict.get(); + } + assert(verify_dict != nullptr); + BlockContents contents; + UncompressionInfo uncompression_info(*verify_ctx, *verify_dict, + r->compression_type); + Status uncompress_status = UncompressBlockData( + uncompression_info, block_contents->data(), block_contents->size(), + &contents, r->table_options.format_version, r->ioptions); + + if (uncompress_status.ok()) { + bool data_match = contents.data.compare(uncompressed_block_data) == 0; + if (!data_match) { + // The result of the compression was invalid. abort. + const char* const msg = + "Decompressed block did not match pre-compression block"; + ROCKS_LOG_ERROR(r->ioptions.logger, "%s", msg); + *out_status = Status::Corruption(msg); + *type = kNoCompression; + } + } else { + // Decompression reported an error. abort. + *out_status = Status::Corruption(std::string("Could not decompress: ") + + uncompress_status.getState()); + *type = kNoCompression; + } + } + if (timer.IsStarted()) { + RecordTimeToHistogram(r->ioptions.stats, COMPRESSION_TIMES_NANOS, + timer.ElapsedNanos()); + } + } else { + // Status is not OK, or block is too big to be compressed. + if (is_data_block) { + r->uncompressible_input_data_bytes.fetch_add( + uncompressed_block_data.size(), std::memory_order_relaxed); + } + *type = kNoCompression; + } + if (is_data_block) { + r->uncompressible_input_data_bytes.fetch_add(kBlockTrailerSize, + std::memory_order_relaxed); + } + + // Abort compression if the block is too big, or did not pass + // verification. + if (*type == kNoCompression) { + *block_contents = uncompressed_block_data; + bool compression_attempted = !compressed_output->empty(); + RecordTick(r->ioptions.stats, compression_attempted + ? NUMBER_BLOCK_COMPRESSION_REJECTED + : NUMBER_BLOCK_COMPRESSION_BYPASSED); + RecordTick(r->ioptions.stats, + compression_attempted ? BYTES_COMPRESSION_REJECTED + : BYTES_COMPRESSION_BYPASSED, + uncompressed_block_data.size()); + } else { + RecordTick(r->ioptions.stats, NUMBER_BLOCK_COMPRESSED); + RecordTick(r->ioptions.stats, BYTES_COMPRESSED_FROM, + uncompressed_block_data.size()); + RecordTick(r->ioptions.stats, BYTES_COMPRESSED_TO, + compressed_output->size()); + } +} + +void BlockBasedTableBuilder::WriteMaybeCompressedBlock( + const Slice& block_contents, CompressionType comp_type, BlockHandle* handle, + BlockType block_type, const Slice* uncompressed_block_data) { + // File format contains a sequence of blocks where each block has: + // block_data: uint8[n] + // compression_type: uint8 + // checksum: uint32 + Rep* r = rep_; + bool is_data_block = block_type == BlockType::kData; + // Old, misleading name of this function: WriteRawBlock + StopWatch sw(r->ioptions.clock, r->ioptions.stats, WRITE_RAW_BLOCK_MICROS); + const uint64_t offset = r->get_offset(); + handle->set_offset(offset); + handle->set_size(block_contents.size()); + assert(status().ok()); + assert(io_status().ok()); + if (uncompressed_block_data == nullptr) { + uncompressed_block_data = &block_contents; + assert(comp_type == kNoCompression); + } + + { + IOStatus io_s = r->file->Append(block_contents); + if (!io_s.ok()) { + r->SetIOStatus(io_s); + return; + } + } + + std::array trailer; + trailer[0] = comp_type; + uint32_t checksum = ComputeBuiltinChecksumWithLastByte( + r->table_options.checksum, block_contents.data(), block_contents.size(), + /*last_byte*/ comp_type); + checksum += ChecksumModifierForContext(r->base_context_checksum, offset); + + if (block_type == BlockType::kFilter) { + Status s = r->filter_builder->MaybePostVerifyFilter(block_contents); + if (!s.ok()) { + r->SetStatus(s); + return; + } + } + + EncodeFixed32(trailer.data() + 1, checksum); + TEST_SYNC_POINT_CALLBACK( + "BlockBasedTableBuilder::WriteMaybeCompressedBlock:TamperWithChecksum", + trailer.data()); + { + IOStatus io_s = r->file->Append(Slice(trailer.data(), trailer.size())); + if (!io_s.ok()) { + r->SetIOStatus(io_s); + return; + } + } + + { + bool warm_cache; + switch (r->table_options.prepopulate_block_cache) { + case BlockBasedTableOptions::PrepopulateBlockCache::kFlushOnly: + warm_cache = (r->reason == TableFileCreationReason::kFlush); + break; + case BlockBasedTableOptions::PrepopulateBlockCache::kDisable: + warm_cache = false; + break; + default: + // missing case + assert(false); + warm_cache = false; + } + if (warm_cache) { + Status s = InsertBlockInCacheHelper(*uncompressed_block_data, handle, + block_type); + if (!s.ok()) { + r->SetStatus(s); + return; + } + } + } + + r->set_offset(r->get_offset() + block_contents.size() + kBlockTrailerSize); + if (r->table_options.block_align && is_data_block) { + size_t pad_bytes = + (r->alignment - + ((block_contents.size() + kBlockTrailerSize) & (r->alignment - 1))) & + (r->alignment - 1); + IOStatus io_s = r->file->Pad(pad_bytes); + if (io_s.ok()) { + r->set_offset(r->get_offset() + pad_bytes); + } else { + r->SetIOStatus(io_s); + return; + } + } + + if (r->IsParallelCompressionEnabled()) { + if (is_data_block) { + r->pc_rep->file_size_estimator.ReapBlock(block_contents.size(), + r->get_offset()); + } else { + r->pc_rep->file_size_estimator.SetEstimatedFileSize(r->get_offset()); + } + } +} + +void BlockBasedTableBuilder::BGWorkWriteMaybeCompressedBlock() { + Rep* r = rep_; + ParallelCompressionRep::BlockRepSlot* slot = nullptr; + ParallelCompressionRep::BlockRep* block_rep = nullptr; + while (r->pc_rep->write_queue.pop(slot)) { + assert(slot != nullptr); + slot->Take(block_rep); + assert(block_rep != nullptr); + if (!block_rep->status.ok()) { + r->SetStatus(block_rep->status); + // Reap block so that blocked Flush() can finish + // if there is one, and Flush() will notice !ok() next time. + block_rep->status = Status::OK(); + r->pc_rep->ReapBlock(block_rep); + continue; + } + + for (size_t i = 0; i < block_rep->keys->Size(); i++) { + auto& key = (*block_rep->keys)[i]; + if (r->filter_builder != nullptr) { + r->filter_builder->Add(ExtractUserKeyAndStripTimestamp(key, r->ts_sz)); + } + r->index_builder->OnKeyAdded(key); + } + + r->pc_rep->file_size_estimator.SetCurrBlockUncompSize( + block_rep->data->size()); + WriteMaybeCompressedBlock(block_rep->compressed_contents, + block_rep->compression_type, &r->pending_handle, + BlockType::kData, &block_rep->contents); + if (!ok()) { + break; + } + + r->props.data_size = r->get_offset(); + ++r->props.num_data_blocks; + + if (block_rep->first_key_in_next_block == nullptr) { + r->index_builder->AddIndexEntry(&(block_rep->keys->Back()), nullptr, + r->pending_handle); + } else { + Slice first_key_in_next_block = + Slice(*block_rep->first_key_in_next_block); + r->index_builder->AddIndexEntry(&(block_rep->keys->Back()), + &first_key_in_next_block, + r->pending_handle); + } + + r->pc_rep->ReapBlock(block_rep); + } +} + +void BlockBasedTableBuilder::StartParallelCompression() { + rep_->pc_rep.reset( + new ParallelCompressionRep(rep_->compression_opts.parallel_threads)); + rep_->pc_rep->compress_thread_pool.reserve( + rep_->compression_opts.parallel_threads); + for (uint32_t i = 0; i < rep_->compression_opts.parallel_threads; i++) { + rep_->pc_rep->compress_thread_pool.emplace_back([this, i] { + BGWorkCompression(*(rep_->compression_ctxs[i]), + rep_->verify_ctxs[i].get()); + }); + } + rep_->pc_rep->write_thread.reset( + new port::Thread([this] { BGWorkWriteMaybeCompressedBlock(); })); +} + +void BlockBasedTableBuilder::StopParallelCompression() { + rep_->pc_rep->compress_queue.finish(); + for (auto& thread : rep_->pc_rep->compress_thread_pool) { + thread.join(); + } + rep_->pc_rep->write_queue.finish(); + rep_->pc_rep->write_thread->join(); +} + +Status BlockBasedTableBuilder::status() const { return rep_->GetStatus(); } + +IOStatus BlockBasedTableBuilder::io_status() const { + return rep_->GetIOStatus(); +} + +Status BlockBasedTableBuilder::InsertBlockInCacheHelper( + const Slice& block_contents, const BlockHandle* handle, + BlockType block_type) { + + Cache* block_cache = rep_->table_options.block_cache.get(); + Status s; + auto helper = + GetCacheItemHelper(block_type, rep_->ioptions.lowest_used_cache_tier); + if (block_cache && helper && helper->create_cb) { + CacheKey key = BlockBasedTable::GetCacheKey(rep_->base_cache_key, *handle); + size_t charge; + s = WarmInCache(block_cache, key.AsSlice(), block_contents, + &rep_->create_context, helper, Cache::Priority::LOW, + &charge); + + if (s.ok()) { + BlockBasedTable::UpdateCacheInsertionMetrics( + block_type, nullptr /*get_context*/, charge, s.IsOkOverwritten(), + rep_->ioptions.stats); + } else { + RecordTick(rep_->ioptions.stats, BLOCK_CACHE_ADD_FAILURES); + } + } + return s; +} + +void BlockBasedTableBuilder::WriteFilterBlock( + MetaIndexBuilder* meta_index_builder) { + if (rep_->filter_builder == nullptr || rep_->filter_builder->IsEmpty()) { + // No filter block needed + return; + } + BlockHandle filter_block_handle; + bool is_partitioned_filter = rep_->table_options.partition_filters; + if (ok()) { + rep_->props.num_filter_entries += + rep_->filter_builder->EstimateEntriesAdded(); + Status s = Status::Incomplete(); + while (ok() && s.IsIncomplete()) { + // filter_data is used to store the transferred filter data payload from + // FilterBlockBuilder and deallocate the payload by going out of scope. + // Otherwise, the payload will unnecessarily remain until + // BlockBasedTableBuilder is deallocated. + // + // See FilterBlockBuilder::Finish() for more on the difference in + // transferred filter data payload among different FilterBlockBuilder + // subtypes. + std::unique_ptr filter_data; + Slice filter_content = + rep_->filter_builder->Finish(filter_block_handle, &s, &filter_data); + + assert(s.ok() || s.IsIncomplete() || s.IsCorruption()); + if (s.IsCorruption()) { + rep_->SetStatus(s); + break; + } + + rep_->props.filter_size += filter_content.size(); + + BlockType btype = is_partitioned_filter && /* last */ s.ok() + ? BlockType::kFilterPartitionIndex + : BlockType::kFilter; + WriteMaybeCompressedBlock(filter_content, kNoCompression, + &filter_block_handle, btype); + } + rep_->filter_builder->ResetFilterBitsBuilder(); + } + if (ok()) { + // Add mapping from ".Name" to location + // of filter data. + std::string key; + key = is_partitioned_filter ? BlockBasedTable::kPartitionedFilterBlockPrefix + : BlockBasedTable::kFullFilterBlockPrefix; + key.append(rep_->table_options.filter_policy->CompatibilityName()); + meta_index_builder->Add(key, filter_block_handle); + } +} + +void BlockBasedTableBuilder::WriteIndexBlock( + MetaIndexBuilder* meta_index_builder, BlockHandle* index_block_handle) { + if (!ok()) { + return; + } + IndexBuilder::IndexBlocks index_blocks; + auto index_builder_status = rep_->index_builder->Finish(&index_blocks); + if (index_builder_status.IsIncomplete()) { + // We we have more than one index partition then meta_blocks are not + // supported for the index. Currently meta_blocks are used only by + // HashIndexBuilder which is not multi-partition. + assert(index_blocks.meta_blocks.empty()); + } else if (ok() && !index_builder_status.ok()) { + rep_->SetStatus(index_builder_status); + } + if (ok()) { + for (const auto& item : index_blocks.meta_blocks) { + BlockHandle block_handle; + WriteBlock(item.second, &block_handle, BlockType::kIndex); + if (!ok()) { + break; + } + meta_index_builder->Add(item.first, block_handle); + } + } + if (ok()) { + if (rep_->table_options.enable_index_compression) { + WriteBlock(index_blocks.index_block_contents, index_block_handle, + BlockType::kIndex); + } else { + WriteMaybeCompressedBlock(index_blocks.index_block_contents, + kNoCompression, index_block_handle, + BlockType::kIndex); + } + } + // If there are more index partitions, finish them and write them out + if (index_builder_status.IsIncomplete()) { + bool index_building_finished = false; + while (ok() && !index_building_finished) { + Status s = + rep_->index_builder->Finish(&index_blocks, *index_block_handle); + if (s.ok()) { + index_building_finished = true; + } else if (s.IsIncomplete()) { + // More partitioned index after this one + assert(!index_building_finished); + } else { + // Error + rep_->SetStatus(s); + return; + } + + if (rep_->table_options.enable_index_compression) { + WriteBlock(index_blocks.index_block_contents, index_block_handle, + BlockType::kIndex); + } else { + WriteMaybeCompressedBlock(index_blocks.index_block_contents, + kNoCompression, index_block_handle, + BlockType::kIndex); + } + // The last index_block_handle will be for the partition index block + } + } + // If success and need to record in metaindex rather than footer... + if (!FormatVersionUsesIndexHandleInFooter( + rep_->table_options.format_version)) { + meta_index_builder->Add(kIndexBlockName, *index_block_handle); + } +} + +void BlockBasedTableBuilder::WritePropertiesBlock( + MetaIndexBuilder* meta_index_builder) { + BlockHandle properties_block_handle; + if (ok()) { + PropertyBlockBuilder property_block_builder; + rep_->props.filter_policy_name = + rep_->table_options.filter_policy != nullptr + ? rep_->table_options.filter_policy->Name() + : ""; + rep_->props.index_size = + rep_->index_builder->IndexSize() + kBlockTrailerSize; + rep_->props.comparator_name = rep_->ioptions.user_comparator != nullptr + ? rep_->ioptions.user_comparator->Name() + : "nullptr"; + rep_->props.merge_operator_name = + rep_->ioptions.merge_operator != nullptr + ? rep_->ioptions.merge_operator->Name() + : "nullptr"; + rep_->props.compression_name = + CompressionTypeToString(rep_->compression_type); + rep_->props.compression_options = + CompressionOptionsToString(rep_->compression_opts); + rep_->props.prefix_extractor_name = + rep_->prefix_extractor ? rep_->prefix_extractor->AsString() : "nullptr"; + std::string property_collectors_names = "["; + for (size_t i = 0; + i < rep_->ioptions.table_properties_collector_factories.size(); ++i) { + if (i != 0) { + property_collectors_names += ","; + } + property_collectors_names += + rep_->ioptions.table_properties_collector_factories[i]->Name(); + } + property_collectors_names += "]"; + rep_->props.property_collectors_names = property_collectors_names; + if (rep_->table_options.index_type == + BlockBasedTableOptions::kTwoLevelIndexSearch) { + assert(rep_->p_index_builder_ != nullptr); + rep_->props.index_partitions = rep_->p_index_builder_->NumPartitions(); + rep_->props.top_level_index_size = + rep_->p_index_builder_->TopLevelIndexSize(rep_->offset); + } + rep_->props.index_key_is_user_key = + !rep_->index_builder->seperator_is_key_plus_seq(); + rep_->props.index_value_is_delta_encoded = + rep_->use_delta_encoding_for_index_values; + if (rep_->sampled_input_data_bytes > 0) { + rep_->props.slow_compression_estimated_data_size = static_cast( + static_cast(rep_->sampled_output_slow_data_bytes) / + rep_->sampled_input_data_bytes * + rep_->compressible_input_data_bytes + + rep_->uncompressible_input_data_bytes + 0.5); + rep_->props.fast_compression_estimated_data_size = static_cast( + static_cast(rep_->sampled_output_fast_data_bytes) / + rep_->sampled_input_data_bytes * + rep_->compressible_input_data_bytes + + rep_->uncompressible_input_data_bytes + 0.5); + } else if (rep_->sample_for_compression > 0) { + // We tried to sample but none were found. Assume worst-case (compression + // ratio 1.0) so data is complete and aggregatable. + rep_->props.slow_compression_estimated_data_size = + rep_->compressible_input_data_bytes + + rep_->uncompressible_input_data_bytes; + rep_->props.fast_compression_estimated_data_size = + rep_->compressible_input_data_bytes + + rep_->uncompressible_input_data_bytes; + } + rep_->props.user_defined_timestamps_persisted = + rep_->persist_user_defined_timestamps; + + // Add basic properties + property_block_builder.AddTableProperty(rep_->props); + + // Add use collected properties + NotifyCollectTableCollectorsOnFinish(rep_->table_properties_collectors, + rep_->ioptions.logger, + &property_block_builder); + + Slice block_data = property_block_builder.Finish(); + TEST_SYNC_POINT_CALLBACK( + "BlockBasedTableBuilder::WritePropertiesBlock:BlockData", &block_data); + WriteMaybeCompressedBlock(block_data, kNoCompression, + &properties_block_handle, BlockType::kProperties); + } + if (ok()) { +#ifndef NDEBUG + { + uint64_t props_block_offset = properties_block_handle.offset(); + uint64_t props_block_size = properties_block_handle.size(); + TEST_SYNC_POINT_CALLBACK( + "BlockBasedTableBuilder::WritePropertiesBlock:GetPropsBlockOffset", + &props_block_offset); + TEST_SYNC_POINT_CALLBACK( + "BlockBasedTableBuilder::WritePropertiesBlock:GetPropsBlockSize", + &props_block_size); + } +#endif // !NDEBUG + + const std::string* properties_block_meta = &kPropertiesBlockName; + TEST_SYNC_POINT_CALLBACK( + "BlockBasedTableBuilder::WritePropertiesBlock:Meta", + &properties_block_meta); + meta_index_builder->Add(*properties_block_meta, properties_block_handle); + } +} + +void BlockBasedTableBuilder::WriteCompressionDictBlock( + MetaIndexBuilder* meta_index_builder) { + if (rep_->compression_dict != nullptr && + rep_->compression_dict->GetRawDict().size()) { + BlockHandle compression_dict_block_handle; + if (ok()) { + WriteMaybeCompressedBlock(rep_->compression_dict->GetRawDict(), + kNoCompression, &compression_dict_block_handle, + BlockType::kCompressionDictionary); +#ifndef NDEBUG + Slice compression_dict = rep_->compression_dict->GetRawDict(); + TEST_SYNC_POINT_CALLBACK( + "BlockBasedTableBuilder::WriteCompressionDictBlock:RawDict", + &compression_dict); +#endif // NDEBUG + } + if (ok()) { + meta_index_builder->Add(kCompressionDictBlockName, + compression_dict_block_handle); + } + } +} + +void BlockBasedTableBuilder::WriteRangeDelBlock( + MetaIndexBuilder* meta_index_builder) { + if (ok() && !rep_->range_del_block.empty()) { + BlockHandle range_del_block_handle; + WriteMaybeCompressedBlock(rep_->range_del_block.Finish(), kNoCompression, + &range_del_block_handle, + BlockType::kRangeDeletion); + meta_index_builder->Add(kRangeDelBlockName, range_del_block_handle); + } +} + +void BlockBasedTableBuilder::WriteFooter(BlockHandle& metaindex_block_handle, + BlockHandle& index_block_handle) { + assert(ok()); + Rep* r = rep_; + // this is guaranteed by BlockBasedTableBuilder's constructor + assert(r->table_options.checksum == kCRC32c || + r->table_options.format_version != 0); + FooterBuilder footer; + Status s = footer.Build(kBlockBasedTableMagicNumber, + r->table_options.format_version, r->get_offset(), + r->table_options.checksum, metaindex_block_handle, + index_block_handle, r->base_context_checksum); + if (!s.ok()) { + r->SetStatus(s); + return; + } + IOStatus ios = r->file->Append(footer.GetSlice()); + if (ios.ok()) { + r->set_offset(r->get_offset() + footer.GetSlice().size()); + } else { + r->SetIOStatus(ios); + } +} + +void BlockBasedTableBuilder::EnterUnbuffered() { + Rep* r = rep_; + assert(r->state == Rep::State::kBuffered); + r->state = Rep::State::kUnbuffered; + const size_t kSampleBytes = r->compression_opts.zstd_max_train_bytes > 0 + ? r->compression_opts.zstd_max_train_bytes + : r->compression_opts.max_dict_bytes; + const size_t kNumBlocksBuffered = r->data_block_buffers.size(); + if (kNumBlocksBuffered == 0) { + // The below code is neither safe nor necessary for handling zero data + // blocks. + return; + } + + // Abstract algebra teaches us that a finite cyclic group (such as the + // additive group of integers modulo N) can be generated by a number that is + // coprime with N. Since N is variable (number of buffered data blocks), we + // must then pick a prime number in order to guarantee coprimeness with any N. + // + // One downside of this approach is the spread will be poor when + // `kPrimeGeneratorRemainder` is close to zero or close to + // `kNumBlocksBuffered`. + // + // Picked a random number between one and one trillion and then chose the + // next prime number greater than or equal to it. + const uint64_t kPrimeGenerator = 545055921143ull; + // Can avoid repeated division by just adding the remainder repeatedly. + const size_t kPrimeGeneratorRemainder = static_cast( + kPrimeGenerator % static_cast(kNumBlocksBuffered)); + const size_t kInitSampleIdx = kNumBlocksBuffered / 2; + + std::string compression_dict_samples; + std::vector compression_dict_sample_lens; + size_t buffer_idx = kInitSampleIdx; + for (size_t i = 0; + i < kNumBlocksBuffered && compression_dict_samples.size() < kSampleBytes; + ++i) { + size_t copy_len = std::min(kSampleBytes - compression_dict_samples.size(), + r->data_block_buffers[buffer_idx].size()); + compression_dict_samples.append(r->data_block_buffers[buffer_idx], 0, + copy_len); + compression_dict_sample_lens.emplace_back(copy_len); + + buffer_idx += kPrimeGeneratorRemainder; + if (buffer_idx >= kNumBlocksBuffered) { + buffer_idx -= kNumBlocksBuffered; + } + } + + // final data block flushed, now we can generate dictionary from the samples. + // OK if compression_dict_samples is empty, we'll just get empty dictionary. + std::string dict; + if (r->compression_opts.zstd_max_train_bytes > 0) { + if (r->compression_opts.use_zstd_dict_trainer) { + dict = ZSTD_TrainDictionary(compression_dict_samples, + compression_dict_sample_lens, + r->compression_opts.max_dict_bytes); + } else { + dict = ZSTD_FinalizeDictionary( + compression_dict_samples, compression_dict_sample_lens, + r->compression_opts.max_dict_bytes, r->compression_opts.level); + } + } else { + dict = std::move(compression_dict_samples); + } + r->compression_dict.reset(new CompressionDict(dict, r->compression_type, + r->compression_opts.level)); + r->verify_dict.reset(new UncompressionDict( + dict, r->compression_type == kZSTD || + r->compression_type == kZSTDNotFinalCompression)); + + auto get_iterator_for_block = [&r](size_t i) { + auto& data_block = r->data_block_buffers[i]; + assert(!data_block.empty()); + + Block reader{BlockContents{data_block}}; + DataBlockIter* iter = reader.NewDataIterator( + r->internal_comparator.user_comparator(), kDisableGlobalSequenceNumber, + nullptr /* iter */, nullptr /* stats */, + false /* block_contents_pinned */, r->persist_user_defined_timestamps); + + iter->SeekToFirst(); + assert(iter->Valid()); + return std::unique_ptr(iter); + }; + + std::unique_ptr iter = nullptr, next_block_iter = nullptr; + + for (size_t i = 0; ok() && i < r->data_block_buffers.size(); ++i) { + if (iter == nullptr) { + iter = get_iterator_for_block(i); + assert(iter != nullptr); + }; + + if (i + 1 < r->data_block_buffers.size()) { + next_block_iter = get_iterator_for_block(i + 1); + } + + auto& data_block = r->data_block_buffers[i]; + if (r->IsParallelCompressionEnabled()) { + Slice first_key_in_next_block; + const Slice* first_key_in_next_block_ptr = &first_key_in_next_block; + if (i + 1 < r->data_block_buffers.size()) { + assert(next_block_iter != nullptr); + first_key_in_next_block = next_block_iter->key(); + } else { + first_key_in_next_block_ptr = r->first_key_in_next_block; + } + + std::vector keys; + for (; iter->Valid(); iter->Next()) { + keys.emplace_back(iter->key().ToString()); + } + + ParallelCompressionRep::BlockRep* block_rep = r->pc_rep->PrepareBlock( + r->compression_type, first_key_in_next_block_ptr, &data_block, &keys); + + assert(block_rep != nullptr); + r->pc_rep->file_size_estimator.EmitBlock(block_rep->data->size(), + r->get_offset()); + r->pc_rep->EmitBlock(block_rep); + } else { + for (; iter->Valid(); iter->Next()) { + Slice key = iter->key(); + if (r->filter_builder != nullptr) { + r->filter_builder->Add( + ExtractUserKeyAndStripTimestamp(key, r->ts_sz)); + } + r->index_builder->OnKeyAdded(key); + } + WriteBlock(Slice(data_block), &r->pending_handle, BlockType::kData); + if (ok() && i + 1 < r->data_block_buffers.size()) { + assert(next_block_iter != nullptr); + Slice first_key_in_next_block = next_block_iter->key(); + + Slice* first_key_in_next_block_ptr = &first_key_in_next_block; + + iter->SeekToLast(); + std::string last_key = iter->key().ToString(); + r->index_builder->AddIndexEntry(&last_key, first_key_in_next_block_ptr, + r->pending_handle); + } + } + std::swap(iter, next_block_iter); + } + r->data_block_buffers.clear(); + r->data_begin_offset = 0; + // Release all reserved cache for data block buffers + if (r->compression_dict_buffer_cache_res_mgr != nullptr) { + Status s = r->compression_dict_buffer_cache_res_mgr->UpdateCacheReservation( + r->data_begin_offset); + s.PermitUncheckedError(); + } +} + +Status BlockBasedTableBuilder::Finish() { + Rep* r = rep_; + assert(r->state != Rep::State::kClosed); + bool empty_data_block = r->data_block.empty(); + r->first_key_in_next_block = nullptr; + Flush(); + if (r->state == Rep::State::kBuffered) { + EnterUnbuffered(); + } + if (r->IsParallelCompressionEnabled()) { + StopParallelCompression(); +#ifndef NDEBUG + for (const auto& br : r->pc_rep->block_rep_buf) { + assert(br.status.ok()); + } +#endif // !NDEBUG + } else { + // To make sure properties block is able to keep the accurate size of index + // block, we will finish writing all index entries first. + if (ok() && !empty_data_block) { + r->index_builder->AddIndexEntry( + &r->last_key, nullptr /* no next data block */, r->pending_handle); + } + } + + r->props.tail_start_offset = r->offset; + + // Write meta blocks, metaindex block and footer in the following order. + // 1. [meta block: filter] + // 2. [meta block: index] + // 3. [meta block: compression dictionary] + // 4. [meta block: range deletion tombstone] + // 5. [meta block: properties] + // 6. [metaindex block] + // 7. Footer + BlockHandle metaindex_block_handle, index_block_handle; + MetaIndexBuilder meta_index_builder; + WriteFilterBlock(&meta_index_builder); + WriteIndexBlock(&meta_index_builder, &index_block_handle); + WriteCompressionDictBlock(&meta_index_builder); + WriteRangeDelBlock(&meta_index_builder); + WritePropertiesBlock(&meta_index_builder); + if (ok()) { + // flush the meta index block + WriteMaybeCompressedBlock(meta_index_builder.Finish(), kNoCompression, + &metaindex_block_handle, BlockType::kMetaIndex); + } + if (ok()) { + WriteFooter(metaindex_block_handle, index_block_handle); + } + r->state = Rep::State::kClosed; + r->tail_size = r->offset - r->props.tail_start_offset; + + Status ret_status = r->CopyStatus(); + IOStatus ios = r->GetIOStatus(); + if (!ios.ok() && ret_status.ok()) { + // Let io_status supersede ok status (otherwise status takes precedennce) + ret_status = ios; + } + return ret_status; +} + +void BlockBasedTableBuilder::Abandon() { + assert(rep_->state != Rep::State::kClosed); + if (rep_->IsParallelCompressionEnabled()) { + StopParallelCompression(); + } + rep_->state = Rep::State::kClosed; +#ifdef ROCKSDB_ASSERT_STATUS_CHECKED // Avoid unnecessary lock acquisition + rep_->CopyStatus().PermitUncheckedError(); + rep_->CopyIOStatus().PermitUncheckedError(); +#endif // ROCKSDB_ASSERT_STATUS_CHECKED +} + +uint64_t BlockBasedTableBuilder::NumEntries() const { + return rep_->props.num_entries; +} + +bool BlockBasedTableBuilder::IsEmpty() const { + return rep_->props.num_entries == 0 && rep_->props.num_range_deletions == 0; +} + +uint64_t BlockBasedTableBuilder::FileSize() const { return rep_->offset; } + +uint64_t BlockBasedTableBuilder::EstimatedFileSize() const { + if (rep_->IsParallelCompressionEnabled()) { + // Use compression ratio so far and inflight uncompressed bytes to estimate + // final SST size. + return rep_->pc_rep->file_size_estimator.GetEstimatedFileSize(); + } else { + return FileSize(); + } +} + +uint64_t BlockBasedTableBuilder::GetTailSize() const { return rep_->tail_size; } + +bool BlockBasedTableBuilder::NeedCompact() const { + for (const auto& collector : rep_->table_properties_collectors) { + if (collector->NeedCompact()) { + return true; + } + } + return false; +} + +TableProperties BlockBasedTableBuilder::GetTableProperties() const { + TableProperties ret = rep_->props; + for (const auto& collector : rep_->table_properties_collectors) { + for (const auto& prop : collector->GetReadableProperties()) { + ret.readable_properties.insert(prop); + } + collector->Finish(&ret.user_collected_properties).PermitUncheckedError(); + } + return ret; +} + +std::string BlockBasedTableBuilder::GetFileChecksum() const { + if (rep_->file != nullptr) { + return rep_->file->GetFileChecksum(); + } else { + return kUnknownFileChecksum; + } +} + +const char* BlockBasedTableBuilder::GetFileChecksumFuncName() const { + if (rep_->file != nullptr) { + return rep_->file->GetFileChecksumFuncName(); + } else { + return kUnknownFileChecksumFuncName; + } +} +void BlockBasedTableBuilder::SetSeqnoTimeTableProperties( + const std::string& encoded_seqno_to_time_mapping, + uint64_t oldest_ancestor_time) { + rep_->props.seqno_to_time_mapping = encoded_seqno_to_time_mapping; + rep_->props.creation_time = oldest_ancestor_time; +} + +const std::string BlockBasedTable::kObsoleteFilterBlockPrefix = "filter."; +const std::string BlockBasedTable::kFullFilterBlockPrefix = "fullfilter."; +const std::string BlockBasedTable::kPartitionedFilterBlockPrefix = + "partitionedfilter."; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/block_based_table_builder.h b/librocksdb-sys/rocksdb/table/block_based/block_based_table_builder.h new file mode 100644 index 0000000..3949474 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block_based_table_builder.h @@ -0,0 +1,209 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include + +#include +#include +#include +#include +#include + +#include "db/version_edit.h" +#include "rocksdb/flush_block_policy.h" +#include "rocksdb/listener.h" +#include "rocksdb/options.h" +#include "rocksdb/status.h" +#include "rocksdb/table.h" +#include "table/meta_blocks.h" +#include "table/table_builder.h" +#include "util/compression.h" + +namespace ROCKSDB_NAMESPACE { + +class BlockBuilder; +class BlockHandle; +class WritableFile; +struct BlockBasedTableOptions; + +extern const uint64_t kBlockBasedTableMagicNumber; +extern const uint64_t kLegacyBlockBasedTableMagicNumber; + +class BlockBasedTableBuilder : public TableBuilder { + public: + // Create a builder that will store the contents of the table it is + // building in *file. Does not close the file. It is up to the + // caller to close the file after calling Finish(). + BlockBasedTableBuilder(const BlockBasedTableOptions& table_options, + const TableBuilderOptions& table_builder_options, + WritableFileWriter* file); + + // No copying allowed + BlockBasedTableBuilder(const BlockBasedTableBuilder&) = delete; + BlockBasedTableBuilder& operator=(const BlockBasedTableBuilder&) = delete; + + // REQUIRES: Either Finish() or Abandon() has been called. + ~BlockBasedTableBuilder(); + + // Add key,value to the table being constructed. + // REQUIRES: Unless key has type kTypeRangeDeletion, key is after any + // previously added non-kTypeRangeDeletion key according to + // comparator. + // REQUIRES: Finish(), Abandon() have not been called + void Add(const Slice& key, const Slice& value) override; + + // Return non-ok iff some error has been detected. + Status status() const override; + + // Return non-ok iff some error happens during IO. + IOStatus io_status() const override; + + // Finish building the table. Stops using the file passed to the + // constructor after this function returns. + // REQUIRES: Finish(), Abandon() have not been called + Status Finish() override; + + // Indicate that the contents of this builder should be abandoned. Stops + // using the file passed to the constructor after this function returns. + // If the caller is not going to call Finish(), it must call Abandon() + // before destroying this builder. + // REQUIRES: Finish(), Abandon() have not been called + void Abandon() override; + + // Number of calls to Add() so far. + uint64_t NumEntries() const override; + + bool IsEmpty() const override; + + // Size of the file generated so far. If invoked after a successful + // Finish() call, returns the size of the final generated file. + uint64_t FileSize() const override; + + // Estimated size of the file generated so far. This is used when + // FileSize() cannot estimate final SST size, e.g. parallel compression + // is enabled. + uint64_t EstimatedFileSize() const override; + + // Get the size of the "tail" part of a SST file. "Tail" refers to + // all blocks after data blocks till the end of the SST file. + uint64_t GetTailSize() const override; + + bool NeedCompact() const override; + + // Get table properties + TableProperties GetTableProperties() const override; + + // Get file checksum + std::string GetFileChecksum() const override; + + // Get file checksum function name + const char* GetFileChecksumFuncName() const override; + + void SetSeqnoTimeTableProperties( + const std::string& encoded_seqno_to_time_mapping, + uint64_t oldest_ancestor_time) override; + + private: + bool ok() const { return status().ok(); } + + // Transition state from buffered to unbuffered. See `Rep::State` API comment + // for details of the states. + // REQUIRES: `rep_->state == kBuffered` + void EnterUnbuffered(); + + // Call block's Finish() method and then + // - in buffered mode, buffer the uncompressed block contents. + // - in unbuffered mode, write the compressed block contents to file. + void WriteBlock(BlockBuilder* block, BlockHandle* handle, + BlockType blocktype); + + // Compress and write block content to the file. + void WriteBlock(const Slice& block_contents, BlockHandle* handle, + BlockType block_type); + // Directly write data to the file. + void WriteMaybeCompressedBlock( + const Slice& block_contents, CompressionType, BlockHandle* handle, + BlockType block_type, const Slice* uncompressed_block_data = nullptr); + + void SetupCacheKeyPrefix(const TableBuilderOptions& tbo); + + template + Status InsertBlockInCache(const Slice& block_contents, + const BlockHandle* handle, BlockType block_type); + + Status InsertBlockInCacheHelper(const Slice& block_contents, + const BlockHandle* handle, + BlockType block_type); + + Status InsertBlockInCompressedCache(const Slice& block_contents, + const CompressionType type, + const BlockHandle* handle); + + void WriteFilterBlock(MetaIndexBuilder* meta_index_builder); + void WriteIndexBlock(MetaIndexBuilder* meta_index_builder, + BlockHandle* index_block_handle); + void WritePropertiesBlock(MetaIndexBuilder* meta_index_builder); + void WriteCompressionDictBlock(MetaIndexBuilder* meta_index_builder); + void WriteRangeDelBlock(MetaIndexBuilder* meta_index_builder); + void WriteFooter(BlockHandle& metaindex_block_handle, + BlockHandle& index_block_handle); + + struct Rep; + class BlockBasedTablePropertiesCollectorFactory; + class BlockBasedTablePropertiesCollector; + Rep* rep_; + + struct ParallelCompressionRep; + + // Advanced operation: flush any buffered key/value pairs to file. + // Can be used to ensure that two adjacent entries never live in + // the same data block. Most clients should not need to use this method. + // REQUIRES: Finish(), Abandon() have not been called + void Flush(); + + // Some compression libraries fail when the uncompressed size is bigger than + // int. If uncompressed size is bigger than kCompressionSizeLimit, don't + // compress it + const uint64_t kCompressionSizeLimit = std::numeric_limits::max(); + + // Get blocks from mem-table walking thread, compress them and + // pass them to the write thread. Used in parallel compression mode only + void BGWorkCompression(const CompressionContext& compression_ctx, + UncompressionContext* verify_ctx); + + // Given uncompressed block content, try to compress it and return result and + // compression type + void CompressAndVerifyBlock(const Slice& uncompressed_block_data, + bool is_data_block, + const CompressionContext& compression_ctx, + UncompressionContext* verify_ctx, + std::string* compressed_output, + Slice* result_block_contents, + CompressionType* result_compression_type, + Status* out_status); + + // Get compressed blocks from BGWorkCompression and write them into SST + void BGWorkWriteMaybeCompressedBlock(); + + // Initialize parallel compression context and + // start BGWorkCompression and BGWorkWriteMaybeCompressedBlock threads + void StartParallelCompression(); + + // Stop BGWorkCompression and BGWorkWriteMaybeCompressedBlock threads + void StopParallelCompression(); +}; + +Slice CompressBlock(const Slice& uncompressed_data, const CompressionInfo& info, + CompressionType* type, uint32_t format_version, + bool do_sample, std::string* compressed_output, + std::string* sampled_output_fast, + std::string* sampled_output_slow); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/block_based_table_factory.cc b/librocksdb-sys/rocksdb/table/block_based/block_based_table_factory.cc new file mode 100644 index 0000000..4ef5da4 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block_based_table_factory.cc @@ -0,0 +1,963 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "table/block_based/block_based_table_factory.h" + +#include + +#include +#include +#include + +#include "cache/cache_entry_roles.h" +#include "cache/cache_reservation_manager.h" +#include "logging/logging.h" +#include "options/options_helper.h" +#include "port/port.h" +#include "rocksdb/cache.h" +#include "rocksdb/convenience.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/flush_block_policy.h" +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/table.h" +#include "rocksdb/utilities/options_type.h" +#include "table/block_based/block_based_table_builder.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/format.h" +#include "util/mutexlock.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +void TailPrefetchStats::RecordEffectiveSize(size_t len) { + MutexLock l(&mutex_); + if (num_records_ < kNumTracked) { + num_records_++; + } + records_[next_++] = len; + if (next_ == kNumTracked) { + next_ = 0; + } +} + +size_t TailPrefetchStats::GetSuggestedPrefetchSize() { + std::vector sorted; + { + MutexLock l(&mutex_); + + if (num_records_ == 0) { + return 0; + } + sorted.assign(records_, records_ + num_records_); + } + + // Of the historic size, we find the maximum one that satisifis the condtiion + // that if prefetching all, less than 1/8 will be wasted. + std::sort(sorted.begin(), sorted.end()); + + // Assuming we have 5 data points, and after sorting it looks like this: + // + // +---+ + // +---+ | | + // | | | | + // | | | | + // | | | | + // | | | | + // +---+ | | | | + // | | | | | | + // +---+ | | | | | | + // | | | | | | | | + // +---+ | | | | | | | | + // | | | | | | | | | | + // | | | | | | | | | | + // | | | | | | | | | | + // | | | | | | | | | | + // | | | | | | | | | | + // +---+ +---+ +---+ +---+ +---+ + // + // and we use every of the value as a candidate, and estimate how much we + // wasted, compared to read. For example, when we use the 3rd record + // as candidate. This area is what we read: + // +---+ + // +---+ | | + // | | | | + // | | | | + // | | | | + // | | | | + // *** *** *** ***+ *** *** *** *** ** + // * | | | | | | + // +---+ | | | | | * + // * | | | | | | | | + // +---+ | | | | | | | * + // * | | | | X | | | | | + // | | | | | | | | | * + // * | | | | | | | | | + // | | | | | | | | | * + // * | | | | | | | | | + // *** *** ***-*** ***--*** ***--*** +**** + // which is (size of the record) X (number of records). + // + // While wasted is this area: + // +---+ + // +---+ | | + // | | | | + // | | | | + // | | | | + // | | | | + // *** *** *** ****---+ | | | | + // * * | | | | | + // * *-*** *** | | | | | + // * * | | | | | | | + // *--** *** | | | | | | | + // | | | | | X | | | | | + // | | | | | | | | | | + // | | | | | | | | | | + // | | | | | | | | | | + // | | | | | | | | | | + // +---+ +---+ +---+ +---+ +---+ + // + // Which can be calculated iteratively. + // The difference between wasted using 4th and 3rd record, will + // be following area: + // +---+ + // +--+ +-+ ++ +-+ +-+ +---+ | | + // + xxxxxxxxxxxxxxxxxxxxxxxx | | | | + // xxxxxxxxxxxxxxxxxxxxxxxx | | | | + // + xxxxxxxxxxxxxxxxxxxxxxxx | | | | + // | xxxxxxxxxxxxxxxxxxxxxxxx | | | | + // +-+ +-+ +-+ ++ +---+ +--+ | | | + // | | | | | | | + // +---+ ++ | | | | | | + // | | | | | | X | | | + // +---+ ++ | | | | | | | | + // | | | | | | | | | | + // | | | | | | | | | | + // | | | | | | | | | | + // | | | | | | | | | | + // | | | | | | | | | | + // +---+ +---+ +---+ +---+ +---+ + // + // which will be the size difference between 4th and 3rd record, + // times 3, which is number of records before the 4th. + // Here we assume that all data within the prefetch range will be useful. In + // reality, it may not be the case when a partial block is inside the range, + // or there are data in the middle that is not read. We ignore those cases + // for simplicity. + assert(!sorted.empty()); + size_t prev_size = sorted[0]; + size_t max_qualified_size = sorted[0]; + size_t wasted = 0; + for (size_t i = 1; i < sorted.size(); i++) { + size_t read = sorted[i] * sorted.size(); + wasted += (sorted[i] - prev_size) * i; + if (wasted <= read / 8) { + max_qualified_size = sorted[i]; + } + prev_size = sorted[i]; + } + const size_t kMaxPrefetchSize = 512 * 1024; // Never exceed 512KB + return std::min(kMaxPrefetchSize, max_qualified_size); +} + + +const std::string kOptNameMetadataCacheOpts = "metadata_cache_options"; + +static std::unordered_map + pinning_tier_type_string_map = { + {"kFallback", PinningTier::kFallback}, + {"kNone", PinningTier::kNone}, + {"kFlushedAndSimilar", PinningTier::kFlushedAndSimilar}, + {"kAll", PinningTier::kAll}}; + +static std::unordered_map + block_base_table_index_type_string_map = { + {"kBinarySearch", BlockBasedTableOptions::IndexType::kBinarySearch}, + {"kHashSearch", BlockBasedTableOptions::IndexType::kHashSearch}, + {"kTwoLevelIndexSearch", + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch}, + {"kBinarySearchWithFirstKey", + BlockBasedTableOptions::IndexType::kBinarySearchWithFirstKey}}; + +static std::unordered_map + block_base_table_data_block_index_type_string_map = { + {"kDataBlockBinarySearch", + BlockBasedTableOptions::DataBlockIndexType::kDataBlockBinarySearch}, + {"kDataBlockBinaryAndHash", + BlockBasedTableOptions::DataBlockIndexType::kDataBlockBinaryAndHash}}; + +static std::unordered_map + block_base_table_index_shortening_mode_string_map = { + {"kNoShortening", + BlockBasedTableOptions::IndexShorteningMode::kNoShortening}, + {"kShortenSeparators", + BlockBasedTableOptions::IndexShorteningMode::kShortenSeparators}, + {"kShortenSeparatorsAndSuccessor", + BlockBasedTableOptions::IndexShorteningMode:: + kShortenSeparatorsAndSuccessor}}; + +static std::unordered_map + metadata_cache_options_type_info = { + {"top_level_index_pinning", + OptionTypeInfo::Enum( + offsetof(struct MetadataCacheOptions, top_level_index_pinning), + &pinning_tier_type_string_map)}, + {"partition_pinning", + OptionTypeInfo::Enum( + offsetof(struct MetadataCacheOptions, partition_pinning), + &pinning_tier_type_string_map)}, + {"unpartitioned_pinning", + OptionTypeInfo::Enum( + offsetof(struct MetadataCacheOptions, unpartitioned_pinning), + &pinning_tier_type_string_map)}}; + +static std::unordered_map + block_base_table_prepopulate_block_cache_string_map = { + {"kDisable", BlockBasedTableOptions::PrepopulateBlockCache::kDisable}, + {"kFlushOnly", + BlockBasedTableOptions::PrepopulateBlockCache::kFlushOnly}}; + + +static std::unordered_map + block_based_table_type_info = { + /* currently not supported + std::shared_ptr block_cache = nullptr; + CacheUsageOptions cache_usage_options; + */ + {"flush_block_policy_factory", + OptionTypeInfo::AsCustomSharedPtr( + offsetof(struct BlockBasedTableOptions, + flush_block_policy_factory), + OptionVerificationType::kByName, OptionTypeFlags::kCompareNever)}, + {"cache_index_and_filter_blocks", + {offsetof(struct BlockBasedTableOptions, + cache_index_and_filter_blocks), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"cache_index_and_filter_blocks_with_high_priority", + {offsetof(struct BlockBasedTableOptions, + cache_index_and_filter_blocks_with_high_priority), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"pin_l0_filter_and_index_blocks_in_cache", + {offsetof(struct BlockBasedTableOptions, + pin_l0_filter_and_index_blocks_in_cache), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"index_type", OptionTypeInfo::Enum( + offsetof(struct BlockBasedTableOptions, index_type), + &block_base_table_index_type_string_map)}, + {"hash_index_allow_collision", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"data_block_index_type", + OptionTypeInfo::Enum( + offsetof(struct BlockBasedTableOptions, data_block_index_type), + &block_base_table_data_block_index_type_string_map)}, + {"index_shortening", + OptionTypeInfo::Enum( + offsetof(struct BlockBasedTableOptions, index_shortening), + &block_base_table_index_shortening_mode_string_map)}, + {"data_block_hash_table_util_ratio", + {offsetof(struct BlockBasedTableOptions, + data_block_hash_table_util_ratio), + OptionType::kDouble, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"checksum", + {offsetof(struct BlockBasedTableOptions, checksum), + OptionType::kChecksumType, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"no_block_cache", + {offsetof(struct BlockBasedTableOptions, no_block_cache), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"block_size", + {offsetof(struct BlockBasedTableOptions, block_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"block_size_deviation", + {offsetof(struct BlockBasedTableOptions, block_size_deviation), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"block_restart_interval", + {offsetof(struct BlockBasedTableOptions, block_restart_interval), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"index_block_restart_interval", + {offsetof(struct BlockBasedTableOptions, index_block_restart_interval), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"index_per_partition", + {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"metadata_block_size", + {offsetof(struct BlockBasedTableOptions, metadata_block_size), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"partition_filters", + {offsetof(struct BlockBasedTableOptions, partition_filters), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"optimize_filters_for_memory", + {offsetof(struct BlockBasedTableOptions, optimize_filters_for_memory), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"filter_policy", + OptionTypeInfo::AsCustomSharedPtr( + offsetof(struct BlockBasedTableOptions, filter_policy), + OptionVerificationType::kByNameAllowFromNull, + OptionTypeFlags::kNone)}, + {"whole_key_filtering", + {offsetof(struct BlockBasedTableOptions, whole_key_filtering), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"detect_filter_construct_corruption", + {offsetof(struct BlockBasedTableOptions, + detect_filter_construct_corruption), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"reserve_table_builder_memory", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"reserve_table_reader_memory", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"skip_table_builder_flush", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"format_version", + {offsetof(struct BlockBasedTableOptions, format_version), + OptionType::kUInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"verify_compression", + {offsetof(struct BlockBasedTableOptions, verify_compression), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"read_amp_bytes_per_bit", + {offsetof(struct BlockBasedTableOptions, read_amp_bytes_per_bit), + OptionType::kUInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, + [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + const std::string& value, void* addr) { + // A workaround to fix a bug in 6.10, 6.11, 6.12, 6.13 + // and 6.14. The bug will write out 8 bytes to OPTIONS file from the + // starting address of BlockBasedTableOptions.read_amp_bytes_per_bit + // which is actually a uint32. Consequently, the value of + // read_amp_bytes_per_bit written in the OPTIONS file is wrong. + // From 6.15, RocksDB will try to parse the read_amp_bytes_per_bit + // from OPTIONS file as a uint32. To be able to load OPTIONS file + // generated by affected releases before the fix, we need to + // manually parse read_amp_bytes_per_bit with this special hack. + uint64_t read_amp_bytes_per_bit = ParseUint64(value); + *(static_cast(addr)) = + static_cast(read_amp_bytes_per_bit); + return Status::OK(); + }}}, + {"enable_index_compression", + {offsetof(struct BlockBasedTableOptions, enable_index_compression), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"block_align", + {offsetof(struct BlockBasedTableOptions, block_align), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"pin_top_level_index_and_filter", + {offsetof(struct BlockBasedTableOptions, + pin_top_level_index_and_filter), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {kOptNameMetadataCacheOpts, + OptionTypeInfo::Struct( + kOptNameMetadataCacheOpts, &metadata_cache_options_type_info, + offsetof(struct BlockBasedTableOptions, metadata_cache_options), + OptionVerificationType::kNormal, OptionTypeFlags::kNone)}, + {"block_cache", + {offsetof(struct BlockBasedTableOptions, block_cache), + OptionType::kUnknown, OptionVerificationType::kNormal, + (OptionTypeFlags::kCompareNever | OptionTypeFlags::kDontSerialize), + // Parses the input value as a Cache + [](const ConfigOptions& opts, const std::string&, + const std::string& value, void* addr) { + auto* cache = static_cast*>(addr); + return Cache::CreateFromString(opts, value, cache); + }}}, + {"block_cache_compressed", + {0, OptionType::kUnknown, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"max_auto_readahead_size", + {offsetof(struct BlockBasedTableOptions, max_auto_readahead_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"prepopulate_block_cache", + OptionTypeInfo::Enum( + offsetof(struct BlockBasedTableOptions, prepopulate_block_cache), + &block_base_table_prepopulate_block_cache_string_map, + OptionTypeFlags::kMutable)}, + {"initial_auto_readahead_size", + {offsetof(struct BlockBasedTableOptions, initial_auto_readahead_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"num_file_reads_for_auto_readahead", + {offsetof(struct BlockBasedTableOptions, + num_file_reads_for_auto_readahead), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + +}; + +// TODO(myabandeh): We should return an error instead of silently changing the +// options +BlockBasedTableFactory::BlockBasedTableFactory( + const BlockBasedTableOptions& _table_options) + : table_options_(_table_options) { + InitializeOptions(); + RegisterOptions(&table_options_, &block_based_table_type_info); + + const auto table_reader_charged = + table_options_.cache_usage_options.options_overrides + .at(CacheEntryRole::kBlockBasedTableReader) + .charged; + if (table_options_.block_cache && + table_reader_charged == CacheEntryRoleOptions::Decision::kEnabled) { + table_reader_cache_res_mgr_.reset(new ConcurrentCacheReservationManager( + std::make_shared>( + table_options_.block_cache))); + } +} + +void BlockBasedTableFactory::InitializeOptions() { + if (table_options_.flush_block_policy_factory == nullptr) { + table_options_.flush_block_policy_factory.reset( + new FlushBlockBySizePolicyFactory()); + } + if (table_options_.no_block_cache) { + table_options_.block_cache.reset(); + } else if (table_options_.block_cache == nullptr) { + LRUCacheOptions co; + // 32MB, the recommended minimum size for 64 shards, to reduce contention + co.capacity = 32 << 20; + table_options_.block_cache = NewLRUCache(co); + } + if (table_options_.block_size_deviation < 0 || + table_options_.block_size_deviation > 100) { + table_options_.block_size_deviation = 0; + } + if (table_options_.block_restart_interval < 1) { + table_options_.block_restart_interval = 1; + } + if (table_options_.index_block_restart_interval < 1) { + table_options_.index_block_restart_interval = 1; + } + if (table_options_.index_type == BlockBasedTableOptions::kHashSearch && + table_options_.index_block_restart_interval != 1) { + // Currently kHashSearch is incompatible with + // index_block_restart_interval > 1 + table_options_.index_block_restart_interval = 1; + } + if (table_options_.partition_filters && + table_options_.index_type != + BlockBasedTableOptions::kTwoLevelIndexSearch) { + // We do not support partitioned filters without partitioning indexes + table_options_.partition_filters = false; + } + auto& options_overrides = + table_options_.cache_usage_options.options_overrides; + const auto options = table_options_.cache_usage_options.options; + for (std::uint32_t i = 0; i < kNumCacheEntryRoles; ++i) { + CacheEntryRole role = static_cast(i); + auto options_overrides_iter = options_overrides.find(role); + if (options_overrides_iter == options_overrides.end()) { + options_overrides.insert({role, options}); + } else if (options_overrides_iter->second.charged == + CacheEntryRoleOptions::Decision::kFallback) { + options_overrides_iter->second.charged = options.charged; + } + } +} + +Status BlockBasedTableFactory::PrepareOptions(const ConfigOptions& opts) { + InitializeOptions(); + return TableFactory::PrepareOptions(opts); +} + +namespace { +// Different cache kinds use the same keys for physically different values, so +// they must not share an underlying key space with each other. +Status CheckCacheOptionCompatibility(const BlockBasedTableOptions& bbto) { + int cache_count = (bbto.block_cache != nullptr) + + (bbto.persistent_cache != nullptr); + if (cache_count <= 1) { + // Nothing to share / overlap + return Status::OK(); + } + + // More complex test of shared key space, in case the instances are wrappers + // for some shared underlying cache. + static Cache::CacheItemHelper kHelper{CacheEntryRole::kMisc}; + CacheKey sentinel_key = CacheKey::CreateUniqueForProcessLifetime(); + struct SentinelValue { + explicit SentinelValue(char _c) : c(_c) {} + char c; + }; + static SentinelValue kRegularBlockCacheMarker{'b'}; + static char kPersistentCacheMarker{'p'}; + if (bbto.block_cache) { + bbto.block_cache + ->Insert(sentinel_key.AsSlice(), &kRegularBlockCacheMarker, &kHelper, 1) + .PermitUncheckedError(); + } + if (bbto.persistent_cache) { + // Note: persistent cache copies the data, not keeping the pointer + bbto.persistent_cache + ->Insert(sentinel_key.AsSlice(), &kPersistentCacheMarker, 1) + .PermitUncheckedError(); + } + // If we get something different from what we inserted, that indicates + // dangerously overlapping key spaces. + if (bbto.block_cache) { + auto handle = bbto.block_cache->Lookup(sentinel_key.AsSlice()); + if (handle) { + auto v = static_cast(bbto.block_cache->Value(handle)); + char c = v->c; + bbto.block_cache->Release(handle); + if (c == kPersistentCacheMarker) { + return Status::InvalidArgument( + "block_cache and persistent_cache share the same key space, " + "which is not supported"); + } else if (v != &kRegularBlockCacheMarker) { + return Status::Corruption("Unexpected mutation to block_cache"); + } + } + } + + if (bbto.persistent_cache) { + std::unique_ptr data; + size_t size = 0; + bbto.persistent_cache->Lookup(sentinel_key.AsSlice(), &data, &size) + .PermitUncheckedError(); + if (data && size > 0) { + if (data[0] == kRegularBlockCacheMarker.c) { + return Status::InvalidArgument( + "persistent_cache and block_cache share the same key space, " + "which is not supported"); + } else if (data[0] != kPersistentCacheMarker) { + return Status::Corruption("Unexpected mutation to persistent_cache"); + } + } + } + return Status::OK(); +} + +} // namespace + +Status BlockBasedTableFactory::NewTableReader( + const ReadOptions& ro, const TableReaderOptions& table_reader_options, + std::unique_ptr&& file, uint64_t file_size, + std::unique_ptr* table_reader, + bool prefetch_index_and_filter_in_cache) const { + return BlockBasedTable::Open( + ro, table_reader_options.ioptions, table_reader_options.env_options, + table_options_, table_reader_options.internal_comparator, std::move(file), + file_size, table_reader_options.block_protection_bytes_per_key, + table_reader, table_reader_options.tail_size, table_reader_cache_res_mgr_, + table_reader_options.prefix_extractor, prefetch_index_and_filter_in_cache, + table_reader_options.skip_filters, table_reader_options.level, + table_reader_options.immortal, table_reader_options.largest_seqno, + table_reader_options.force_direct_prefetch, &tail_prefetch_stats_, + table_reader_options.block_cache_tracer, + table_reader_options.max_file_size_for_l0_meta_pin, + table_reader_options.cur_db_session_id, table_reader_options.cur_file_num, + table_reader_options.unique_id, + table_reader_options.user_defined_timestamps_persisted); +} + +TableBuilder* BlockBasedTableFactory::NewTableBuilder( + const TableBuilderOptions& table_builder_options, + WritableFileWriter* file) const { + return new BlockBasedTableBuilder(table_options_, table_builder_options, + file); +} + +Status BlockBasedTableFactory::ValidateOptions( + const DBOptions& db_opts, const ColumnFamilyOptions& cf_opts) const { + if (table_options_.index_type == BlockBasedTableOptions::kHashSearch && + cf_opts.prefix_extractor == nullptr) { + return Status::InvalidArgument( + "Hash index is specified for block-based " + "table, but prefix_extractor is not given"); + } + if (table_options_.cache_index_and_filter_blocks && + table_options_.no_block_cache) { + return Status::InvalidArgument( + "Enable cache_index_and_filter_blocks, " + ", but block cache is disabled"); + } + if (table_options_.pin_l0_filter_and_index_blocks_in_cache && + table_options_.no_block_cache) { + return Status::InvalidArgument( + "Enable pin_l0_filter_and_index_blocks_in_cache, " + ", but block cache is disabled"); + } + if (!IsSupportedFormatVersion(table_options_.format_version)) { + return Status::InvalidArgument( + "Unsupported BlockBasedTable format_version. Please check " + "include/rocksdb/table.h for more info"); + } + if (table_options_.block_align && (cf_opts.compression != kNoCompression)) { + return Status::InvalidArgument( + "Enable block_align, but compression " + "enabled"); + } + if (table_options_.block_align && + (table_options_.block_size & (table_options_.block_size - 1))) { + return Status::InvalidArgument( + "Block alignment requested but block size is not a power of 2"); + } + if (table_options_.block_size > std::numeric_limits::max()) { + return Status::InvalidArgument( + "block size exceeds maximum number (4GiB) allowed"); + } + if (table_options_.data_block_index_type == + BlockBasedTableOptions::kDataBlockBinaryAndHash && + table_options_.data_block_hash_table_util_ratio <= 0) { + return Status::InvalidArgument( + "data_block_hash_table_util_ratio should be greater than 0 when " + "data_block_index_type is set to kDataBlockBinaryAndHash"); + } + if (db_opts.unordered_write && cf_opts.max_successive_merges > 0) { + // TODO(myabandeh): support it + return Status::InvalidArgument( + "max_successive_merges larger than 0 is currently inconsistent with " + "unordered_write"); + } + const auto& options_overrides = + table_options_.cache_usage_options.options_overrides; + for (auto options_overrides_iter = options_overrides.cbegin(); + options_overrides_iter != options_overrides.cend(); + ++options_overrides_iter) { + const CacheEntryRole role = options_overrides_iter->first; + const CacheEntryRoleOptions options = options_overrides_iter->second; + static const std::set kMemoryChargingSupported = { + CacheEntryRole::kCompressionDictionaryBuildingBuffer, + CacheEntryRole::kFilterConstruction, + CacheEntryRole::kBlockBasedTableReader, CacheEntryRole::kFileMetadata, + CacheEntryRole::kBlobCache}; + if (options.charged != CacheEntryRoleOptions::Decision::kFallback && + kMemoryChargingSupported.count(role) == 0) { + return Status::NotSupported( + "Enable/Disable CacheEntryRoleOptions::charged" + " for CacheEntryRole " + + kCacheEntryRoleToCamelString[static_cast(role)] + + " is not supported"); + } + if (table_options_.no_block_cache && + options.charged == CacheEntryRoleOptions::Decision::kEnabled) { + return Status::InvalidArgument( + "Enable CacheEntryRoleOptions::charged" + " for CacheEntryRole " + + kCacheEntryRoleToCamelString[static_cast(role)] + + " but block cache is disabled"); + } + if (role == CacheEntryRole::kBlobCache && + options.charged == CacheEntryRoleOptions::Decision::kEnabled) { + if (cf_opts.blob_cache == nullptr) { + return Status::InvalidArgument( + "Enable CacheEntryRoleOptions::charged" + " for CacheEntryRole " + + kCacheEntryRoleToCamelString[static_cast(role)] + + " but blob cache is not configured"); + } + if (table_options_.no_block_cache) { + return Status::InvalidArgument( + "Enable CacheEntryRoleOptions::charged" + " for CacheEntryRole " + + kCacheEntryRoleToCamelString[static_cast(role)] + + " but block cache is disabled"); + } + if (table_options_.block_cache == cf_opts.blob_cache) { + return Status::InvalidArgument( + "Enable CacheEntryRoleOptions::charged" + " for CacheEntryRole " + + kCacheEntryRoleToCamelString[static_cast(role)] + + " but blob cache is the same as block cache"); + } + if (cf_opts.blob_cache->GetCapacity() > + table_options_.block_cache->GetCapacity()) { + return Status::InvalidArgument( + "Enable CacheEntryRoleOptions::charged" + " for CacheEntryRole " + + kCacheEntryRoleToCamelString[static_cast(role)] + + " but blob cache capacity is larger than block cache capacity"); + } + } + } + { + Status s = CheckCacheOptionCompatibility(table_options_); + if (!s.ok()) { + return s; + } + } + std::string garbage; + if (!SerializeEnum(checksum_type_string_map, + table_options_.checksum, &garbage)) { + return Status::InvalidArgument( + "Unrecognized ChecksumType for checksum: " + + std::to_string(static_cast(table_options_.checksum))); + } + return TableFactory::ValidateOptions(db_opts, cf_opts); +} + +std::string BlockBasedTableFactory::GetPrintableOptions() const { + std::string ret; + ret.reserve(20000); + const int kBufferSize = 200; + char buffer[kBufferSize]; + + snprintf(buffer, kBufferSize, " flush_block_policy_factory: %s (%p)\n", + table_options_.flush_block_policy_factory->Name(), + static_cast(table_options_.flush_block_policy_factory.get())); + ret.append(buffer); + snprintf(buffer, kBufferSize, " cache_index_and_filter_blocks: %d\n", + table_options_.cache_index_and_filter_blocks); + ret.append(buffer); + snprintf(buffer, kBufferSize, + " cache_index_and_filter_blocks_with_high_priority: %d\n", + table_options_.cache_index_and_filter_blocks_with_high_priority); + ret.append(buffer); + snprintf(buffer, kBufferSize, + " pin_l0_filter_and_index_blocks_in_cache: %d\n", + table_options_.pin_l0_filter_and_index_blocks_in_cache); + ret.append(buffer); + snprintf(buffer, kBufferSize, " pin_top_level_index_and_filter: %d\n", + table_options_.pin_top_level_index_and_filter); + ret.append(buffer); + snprintf(buffer, kBufferSize, " index_type: %d\n", + table_options_.index_type); + ret.append(buffer); + snprintf(buffer, kBufferSize, " data_block_index_type: %d\n", + table_options_.data_block_index_type); + ret.append(buffer); + snprintf(buffer, kBufferSize, " index_shortening: %d\n", + static_cast(table_options_.index_shortening)); + ret.append(buffer); + snprintf(buffer, kBufferSize, " data_block_hash_table_util_ratio: %lf\n", + table_options_.data_block_hash_table_util_ratio); + ret.append(buffer); + snprintf(buffer, kBufferSize, " checksum: %d\n", table_options_.checksum); + ret.append(buffer); + snprintf(buffer, kBufferSize, " no_block_cache: %d\n", + table_options_.no_block_cache); + ret.append(buffer); + snprintf(buffer, kBufferSize, " block_cache: %p\n", + static_cast(table_options_.block_cache.get())); + ret.append(buffer); + if (table_options_.block_cache) { + const char* block_cache_name = table_options_.block_cache->Name(); + if (block_cache_name != nullptr) { + snprintf(buffer, kBufferSize, " block_cache_name: %s\n", + block_cache_name); + ret.append(buffer); + } + ret.append(" block_cache_options:\n"); + ret.append(table_options_.block_cache->GetPrintableOptions()); + } + snprintf(buffer, kBufferSize, " persistent_cache: %p\n", + static_cast(table_options_.persistent_cache.get())); + ret.append(buffer); + if (table_options_.persistent_cache) { + snprintf(buffer, kBufferSize, " persistent_cache_options:\n"); + ret.append(buffer); + ret.append(table_options_.persistent_cache->GetPrintableOptions()); + } + snprintf(buffer, kBufferSize, " block_size: %" PRIu64 "\n", + table_options_.block_size); + ret.append(buffer); + snprintf(buffer, kBufferSize, " block_size_deviation: %d\n", + table_options_.block_size_deviation); + ret.append(buffer); + snprintf(buffer, kBufferSize, " block_restart_interval: %d\n", + table_options_.block_restart_interval); + ret.append(buffer); + snprintf(buffer, kBufferSize, " index_block_restart_interval: %d\n", + table_options_.index_block_restart_interval); + ret.append(buffer); + snprintf(buffer, kBufferSize, " metadata_block_size: %" PRIu64 "\n", + table_options_.metadata_block_size); + ret.append(buffer); + snprintf(buffer, kBufferSize, " partition_filters: %d\n", + table_options_.partition_filters); + ret.append(buffer); + snprintf(buffer, kBufferSize, " use_delta_encoding: %d\n", + table_options_.use_delta_encoding); + ret.append(buffer); + snprintf(buffer, kBufferSize, " filter_policy: %s\n", + table_options_.filter_policy == nullptr + ? "nullptr" + : table_options_.filter_policy->Name()); + ret.append(buffer); + snprintf(buffer, kBufferSize, " whole_key_filtering: %d\n", + table_options_.whole_key_filtering); + ret.append(buffer); + snprintf(buffer, kBufferSize, " verify_compression: %d\n", + table_options_.verify_compression); + ret.append(buffer); + snprintf(buffer, kBufferSize, " read_amp_bytes_per_bit: %d\n", + table_options_.read_amp_bytes_per_bit); + ret.append(buffer); + snprintf(buffer, kBufferSize, " format_version: %d\n", + table_options_.format_version); + ret.append(buffer); + snprintf(buffer, kBufferSize, " enable_index_compression: %d\n", + table_options_.enable_index_compression); + ret.append(buffer); + snprintf(buffer, kBufferSize, " block_align: %d\n", + table_options_.block_align); + ret.append(buffer); + snprintf(buffer, kBufferSize, + " max_auto_readahead_size: %" ROCKSDB_PRIszt "\n", + table_options_.max_auto_readahead_size); + ret.append(buffer); + snprintf(buffer, kBufferSize, " prepopulate_block_cache: %d\n", + static_cast(table_options_.prepopulate_block_cache)); + ret.append(buffer); + snprintf(buffer, kBufferSize, + " initial_auto_readahead_size: %" ROCKSDB_PRIszt "\n", + table_options_.initial_auto_readahead_size); + ret.append(buffer); + snprintf(buffer, kBufferSize, + " num_file_reads_for_auto_readahead: %" PRIu64 "\n", + table_options_.num_file_reads_for_auto_readahead); + ret.append(buffer); + return ret; +} + +const void* BlockBasedTableFactory::GetOptionsPtr( + const std::string& name) const { + if (name == kBlockCacheOpts()) { + if (table_options_.no_block_cache) { + return nullptr; + } else { + return table_options_.block_cache.get(); + } + } else { + return TableFactory::GetOptionsPtr(name); + } +} + +// Take a default BlockBasedTableOptions "table_options" in addition to a +// map "opts_map" of option name to option value to construct the new +// BlockBasedTableOptions "new_table_options". +// +// Below are the instructions of how to config some non-primitive-typed +// options in BlockBasedTableOptions: +// +// * filter_policy: +// We currently only support the following FilterPolicy in the convenience +// functions: +// - BloomFilter: use "bloomfilter:[bits_per_key]:[use_block_based_builder]" +// to specify BloomFilter. The above string is equivalent to calling +// NewBloomFilterPolicy(bits_per_key, use_block_based_builder). +// [Example]: +// - Pass {"filter_policy", "bloomfilter:4:true"} in +// GetBlockBasedTableOptionsFromMap to use a BloomFilter with 4-bits +// per key and use_block_based_builder enabled. +// +// * block_cache / block_cache_compressed: +// We currently only support LRU cache in the GetOptions API. The LRU +// cache can be set by directly specifying its size. +// [Example]: +// - Passing {"block_cache", "1M"} in GetBlockBasedTableOptionsFromMap is +// equivalent to setting block_cache using NewLRUCache(1024 * 1024). +// +// @param table_options the default options of the output "new_table_options". +// @param opts_map an option name to value map for specifying how +// "new_table_options" should be set. +// @param new_table_options the resulting options based on "table_options" +// with the change specified in "opts_map". +// @param input_strings_escaped when set to true, each escaped characters +// prefixed by '\' in the values of the opts_map will be further converted +// back to the raw string before assigning to the associated options. +// @param ignore_unknown_options when set to true, unknown options are ignored +// instead of resulting in an unknown-option error. +// @return Status::OK() on success. Otherwise, a non-ok status indicating +// error will be returned, and "new_table_options" will be set to +// "table_options". +Status BlockBasedTableFactory::ParseOption(const ConfigOptions& config_options, + const OptionTypeInfo& opt_info, + const std::string& opt_name, + const std::string& opt_value, + void* opt_ptr) { + Status status = TableFactory::ParseOption(config_options, opt_info, opt_name, + opt_value, opt_ptr); + if (config_options.input_strings_escaped && !status.ok()) { // Got an error + // !input_strings_escaped indicates the old API, where everything is + // parsable. + if (opt_info.IsByName()) { + status = Status::OK(); + } + } + return status; +} + +Status GetBlockBasedTableOptionsFromString( + const ConfigOptions& config_options, + const BlockBasedTableOptions& table_options, const std::string& opts_str, + BlockBasedTableOptions* new_table_options) { + std::unordered_map opts_map; + Status s = StringToMap(opts_str, &opts_map); + if (!s.ok()) { + return s; + } + s = GetBlockBasedTableOptionsFromMap(config_options, table_options, opts_map, + new_table_options); + // Translate any errors (NotFound, NotSupported, to InvalidArgument + if (s.ok() || s.IsInvalidArgument()) { + return s; + } else { + return Status::InvalidArgument(s.getState()); + } +} + +Status GetBlockBasedTableOptionsFromMap( + const ConfigOptions& config_options, + const BlockBasedTableOptions& table_options, + const std::unordered_map& opts_map, + BlockBasedTableOptions* new_table_options) { + assert(new_table_options); + BlockBasedTableFactory bbtf(table_options); + Status s = bbtf.ConfigureFromMap(config_options, opts_map); + if (s.ok()) { + *new_table_options = *(bbtf.GetOptions()); + } else { + *new_table_options = table_options; + } + return s; +} + +TableFactory* NewBlockBasedTableFactory( + const BlockBasedTableOptions& _table_options) { + return new BlockBasedTableFactory(_table_options); +} + +const std::string BlockBasedTablePropertyNames::kIndexType = + "rocksdb.block.based.table.index.type"; +const std::string BlockBasedTablePropertyNames::kWholeKeyFiltering = + "rocksdb.block.based.table.whole.key.filtering"; +const std::string BlockBasedTablePropertyNames::kPrefixFiltering = + "rocksdb.block.based.table.prefix.filtering"; +const std::string kHashIndexPrefixesBlock = "rocksdb.hashindex.prefixes"; +const std::string kHashIndexPrefixesMetadataBlock = + "rocksdb.hashindex.metadata"; +const std::string kPropTrue = "1"; +const std::string kPropFalse = "0"; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/block_based_table_factory.h b/librocksdb-sys/rocksdb/table/block_based/block_based_table_factory.h new file mode 100644 index 0000000..1f78769 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block_based_table_factory.h @@ -0,0 +1,102 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include + +#include +#include + +#include "cache/cache_reservation_manager.h" +#include "port/port.h" +#include "rocksdb/flush_block_policy.h" +#include "rocksdb/table.h" + +namespace ROCKSDB_NAMESPACE { +struct ColumnFamilyOptions; +struct ConfigOptions; +struct DBOptions; +struct EnvOptions; + +class BlockBasedTableBuilder; +class RandomAccessFileReader; +class WritableFileWriter; + +// TODO: deprecate this class as it can be replaced with +// `FileMetaData::tail_size` +// +// A class used to track actual bytes written from the tail in the recent SST +// file opens, and provide a suggestion for following open. +class TailPrefetchStats { + public: + void RecordEffectiveSize(size_t len); + // 0 indicates no information to determine. + size_t GetSuggestedPrefetchSize(); + + private: + const static size_t kNumTracked = 32; + size_t records_[kNumTracked]; + port::Mutex mutex_; + size_t next_ = 0; + size_t num_records_ = 0; +}; + +class BlockBasedTableFactory : public TableFactory { + public: + explicit BlockBasedTableFactory( + const BlockBasedTableOptions& table_options = BlockBasedTableOptions()); + + ~BlockBasedTableFactory() {} + + // Method to allow CheckedCast to work for this class + static const char* kClassName() { return kBlockBasedTableName(); } + + const char* Name() const override { return kBlockBasedTableName(); } + + using TableFactory::NewTableReader; + Status NewTableReader( + const ReadOptions& ro, const TableReaderOptions& table_reader_options, + std::unique_ptr&& file, uint64_t file_size, + std::unique_ptr* table_reader, + bool prefetch_index_and_filter_in_cache = true) const override; + + TableBuilder* NewTableBuilder( + const TableBuilderOptions& table_builder_options, + WritableFileWriter* file) const override; + + // Valdates the specified DB Options. + Status ValidateOptions(const DBOptions& db_opts, + const ColumnFamilyOptions& cf_opts) const override; + Status PrepareOptions(const ConfigOptions& opts) override; + + std::string GetPrintableOptions() const override; + + bool IsDeleteRangeSupported() const override { return true; } + + TailPrefetchStats* tail_prefetch_stats() { return &tail_prefetch_stats_; } + + protected: + const void* GetOptionsPtr(const std::string& name) const override; + Status ParseOption(const ConfigOptions& config_options, + const OptionTypeInfo& opt_info, + const std::string& opt_name, const std::string& opt_value, + void* opt_ptr) override; + void InitializeOptions(); + + private: + BlockBasedTableOptions table_options_; + std::shared_ptr table_reader_cache_res_mgr_; + mutable TailPrefetchStats tail_prefetch_stats_; +}; + +extern const std::string kHashIndexPrefixesBlock; +extern const std::string kHashIndexPrefixesMetadataBlock; +extern const std::string kPropTrue; +extern const std::string kPropFalse; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/block_based_table_iterator.cc b/librocksdb-sys/rocksdb/table/block_based/block_based_table_iterator.cc new file mode 100644 index 0000000..57744a5 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block_based_table_iterator.cc @@ -0,0 +1,500 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "table/block_based/block_based_table_iterator.h" + +namespace ROCKSDB_NAMESPACE { + +void BlockBasedTableIterator::SeekToFirst() { SeekImpl(nullptr, false); } + +void BlockBasedTableIterator::Seek(const Slice& target) { + SeekImpl(&target, true); +} + +void BlockBasedTableIterator::SeekImpl(const Slice* target, + bool async_prefetch) { + bool is_first_pass = true; + if (async_read_in_progress_) { + AsyncInitDataBlock(false); + is_first_pass = false; + } + + is_out_of_bound_ = false; + is_at_first_key_from_index_ = false; + seek_stat_state_ = kNone; + bool filter_checked = false; + if (target && + !CheckPrefixMayMatch(*target, IterDirection::kForward, &filter_checked)) { + ResetDataIter(); + RecordTick(table_->GetStatistics(), is_last_level_ + ? LAST_LEVEL_SEEK_FILTERED + : NON_LAST_LEVEL_SEEK_FILTERED); + return; + } + if (filter_checked) { + seek_stat_state_ = kFilterUsed; + RecordTick(table_->GetStatistics(), is_last_level_ + ? LAST_LEVEL_SEEK_FILTER_MATCH + : NON_LAST_LEVEL_SEEK_FILTER_MATCH); + } + + bool need_seek_index = true; + if (block_iter_points_to_real_block_ && block_iter_.Valid()) { + // Reseek. + prev_block_offset_ = index_iter_->value().handle.offset(); + + if (target) { + // We can avoid an index seek if: + // 1. The new seek key is larger than the current key + // 2. The new seek key is within the upper bound of the block + // Since we don't necessarily know the internal key for either + // the current key or the upper bound, we check user keys and + // exclude the equality case. Considering internal keys can + // improve for the boundary cases, but it would complicate the + // code. + if (user_comparator_.Compare(ExtractUserKey(*target), + block_iter_.user_key()) > 0 && + user_comparator_.Compare(ExtractUserKey(*target), + index_iter_->user_key()) < 0) { + need_seek_index = false; + } + } + } + + if (need_seek_index) { + if (target) { + index_iter_->Seek(*target); + } else { + index_iter_->SeekToFirst(); + } + + if (!index_iter_->Valid()) { + ResetDataIter(); + return; + } + } + + IndexValue v = index_iter_->value(); + const bool same_block = block_iter_points_to_real_block_ && + v.handle.offset() == prev_block_offset_; + + if (!v.first_internal_key.empty() && !same_block && + (!target || icomp_.Compare(*target, v.first_internal_key) <= 0) && + allow_unprepared_value_) { + // Index contains the first key of the block, and it's >= target. + // We can defer reading the block. + is_at_first_key_from_index_ = true; + // ResetDataIter() will invalidate block_iter_. Thus, there is no need to + // call CheckDataBlockWithinUpperBound() to check for iterate_upper_bound + // as that will be done later when the data block is actually read. + ResetDataIter(); + } else { + // Need to use the data block. + if (!same_block) { + if (read_options_.async_io && async_prefetch) { + if (is_first_pass) { + AsyncInitDataBlock(is_first_pass); + } + if (async_read_in_progress_) { + // Status::TryAgain indicates asynchronous request for retrieval of + // data blocks has been submitted. So it should return at this point + // and Seek should be called again to retrieve the requested block and + // execute the remaining code. + return; + } + } else { + InitDataBlock(); + } + } else { + // When the user does a reseek, the iterate_upper_bound might have + // changed. CheckDataBlockWithinUpperBound() needs to be called + // explicitly if the reseek ends up in the same data block. + // If the reseek ends up in a different block, InitDataBlock() will do + // the iterator upper bound check. + CheckDataBlockWithinUpperBound(); + } + + if (target) { + block_iter_.Seek(*target); + } else { + block_iter_.SeekToFirst(); + } + FindKeyForward(); + } + + CheckOutOfBound(); + + if (target) { + assert(!Valid() || icomp_.Compare(*target, key()) <= 0); + } +} + +void BlockBasedTableIterator::SeekForPrev(const Slice& target) { + is_out_of_bound_ = false; + is_at_first_key_from_index_ = false; + seek_stat_state_ = kNone; + bool filter_checked = false; + // For now totally disable prefix seek in auto prefix mode because we don't + // have logic + if (!CheckPrefixMayMatch(target, IterDirection::kBackward, &filter_checked)) { + ResetDataIter(); + RecordTick(table_->GetStatistics(), is_last_level_ + ? LAST_LEVEL_SEEK_FILTERED + : NON_LAST_LEVEL_SEEK_FILTERED); + return; + } + if (filter_checked) { + seek_stat_state_ = kFilterUsed; + RecordTick(table_->GetStatistics(), is_last_level_ + ? LAST_LEVEL_SEEK_FILTER_MATCH + : NON_LAST_LEVEL_SEEK_FILTER_MATCH); + } + + SavePrevIndexValue(); + + // Call Seek() rather than SeekForPrev() in the index block, because the + // target data block will likely to contain the position for `target`, the + // same as Seek(), rather than than before. + // For example, if we have three data blocks, each containing two keys: + // [2, 4] [6, 8] [10, 12] + // (the keys in the index block would be [4, 8, 12]) + // and the user calls SeekForPrev(7), we need to go to the second block, + // just like if they call Seek(7). + // The only case where the block is difference is when they seek to a position + // in the boundary. For example, if they SeekForPrev(5), we should go to the + // first block, rather than the second. However, we don't have the information + // to distinguish the two unless we read the second block. In this case, we'll + // end up with reading two blocks. + index_iter_->Seek(target); + + if (!index_iter_->Valid()) { + auto seek_status = index_iter_->status(); + // Check for IO error + if (!seek_status.IsNotFound() && !seek_status.ok()) { + ResetDataIter(); + return; + } + + // With prefix index, Seek() returns NotFound if the prefix doesn't exist + if (seek_status.IsNotFound()) { + // Any key less than the target is fine for prefix seek + ResetDataIter(); + return; + } else { + index_iter_->SeekToLast(); + } + // Check for IO error + if (!index_iter_->Valid()) { + ResetDataIter(); + return; + } + } + + InitDataBlock(); + + block_iter_.SeekForPrev(target); + + FindKeyBackward(); + CheckDataBlockWithinUpperBound(); + assert(!block_iter_.Valid() || + icomp_.Compare(target, block_iter_.key()) >= 0); +} + +void BlockBasedTableIterator::SeekToLast() { + is_out_of_bound_ = false; + is_at_first_key_from_index_ = false; + seek_stat_state_ = kNone; + SavePrevIndexValue(); + index_iter_->SeekToLast(); + if (!index_iter_->Valid()) { + ResetDataIter(); + return; + } + InitDataBlock(); + block_iter_.SeekToLast(); + FindKeyBackward(); + CheckDataBlockWithinUpperBound(); +} + +void BlockBasedTableIterator::Next() { + if (is_at_first_key_from_index_ && !MaterializeCurrentBlock()) { + return; + } + assert(block_iter_points_to_real_block_); + block_iter_.Next(); + FindKeyForward(); + CheckOutOfBound(); +} + +bool BlockBasedTableIterator::NextAndGetResult(IterateResult* result) { + Next(); + bool is_valid = Valid(); + if (is_valid) { + result->key = key(); + result->bound_check_result = UpperBoundCheckResult(); + result->value_prepared = !is_at_first_key_from_index_; + } + return is_valid; +} + +void BlockBasedTableIterator::Prev() { + if (is_at_first_key_from_index_) { + is_at_first_key_from_index_ = false; + + index_iter_->Prev(); + if (!index_iter_->Valid()) { + return; + } + + InitDataBlock(); + block_iter_.SeekToLast(); + } else { + assert(block_iter_points_to_real_block_); + block_iter_.Prev(); + } + + FindKeyBackward(); +} + +void BlockBasedTableIterator::InitDataBlock() { + BlockHandle data_block_handle = index_iter_->value().handle; + if (!block_iter_points_to_real_block_ || + data_block_handle.offset() != prev_block_offset_ || + // if previous attempt of reading the block missed cache, try again + block_iter_.status().IsIncomplete()) { + if (block_iter_points_to_real_block_) { + ResetDataIter(); + } + auto* rep = table_->get_rep(); + + bool is_for_compaction = + lookup_context_.caller == TableReaderCaller::kCompaction; + // Prefetch additional data for range scans (iterators). + // Implicit auto readahead: + // Enabled after 2 sequential IOs when ReadOptions.readahead_size == 0. + // Explicit user requested readahead: + // Enabled from the very first IO when ReadOptions.readahead_size is set. + block_prefetcher_.PrefetchIfNeeded( + rep, data_block_handle, read_options_.readahead_size, is_for_compaction, + /*no_sequential_checking=*/false, read_options_.rate_limiter_priority); + Status s; + table_->NewDataBlockIterator( + read_options_, data_block_handle, &block_iter_, BlockType::kData, + /*get_context=*/nullptr, &lookup_context_, + block_prefetcher_.prefetch_buffer(), + /*for_compaction=*/is_for_compaction, /*async_read=*/false, s); + block_iter_points_to_real_block_ = true; + CheckDataBlockWithinUpperBound(); + if (!is_for_compaction && + (seek_stat_state_ & kDataBlockReadSinceLastSeek) == 0) { + RecordTick(table_->GetStatistics(), is_last_level_ + ? LAST_LEVEL_SEEK_DATA + : NON_LAST_LEVEL_SEEK_DATA); + seek_stat_state_ = static_cast( + seek_stat_state_ | kDataBlockReadSinceLastSeek | kReportOnUseful); + } + } +} + +void BlockBasedTableIterator::AsyncInitDataBlock(bool is_first_pass) { + BlockHandle data_block_handle = index_iter_->value().handle; + bool is_for_compaction = + lookup_context_.caller == TableReaderCaller::kCompaction; + if (is_first_pass) { + if (!block_iter_points_to_real_block_ || + data_block_handle.offset() != prev_block_offset_ || + // if previous attempt of reading the block missed cache, try again + block_iter_.status().IsIncomplete()) { + if (block_iter_points_to_real_block_) { + ResetDataIter(); + } + auto* rep = table_->get_rep(); + // Prefetch additional data for range scans (iterators). + // Implicit auto readahead: + // Enabled after 2 sequential IOs when ReadOptions.readahead_size == 0. + // Explicit user requested readahead: + // Enabled from the very first IO when ReadOptions.readahead_size is + // set. + // In case of async_io with Implicit readahead, block_prefetcher_ will + // always the create the prefetch buffer by setting no_sequential_checking + // = true. + block_prefetcher_.PrefetchIfNeeded( + rep, data_block_handle, read_options_.readahead_size, + is_for_compaction, /*no_sequential_checking=*/read_options_.async_io, + read_options_.rate_limiter_priority); + + Status s; + table_->NewDataBlockIterator( + read_options_, data_block_handle, &block_iter_, BlockType::kData, + /*get_context=*/nullptr, &lookup_context_, + block_prefetcher_.prefetch_buffer(), + /*for_compaction=*/is_for_compaction, /*async_read=*/true, s); + + if (s.IsTryAgain()) { + async_read_in_progress_ = true; + return; + } + } + } else { + // Second pass will call the Poll to get the data block which has been + // requested asynchronously. + Status s; + table_->NewDataBlockIterator( + read_options_, data_block_handle, &block_iter_, BlockType::kData, + /*get_context=*/nullptr, &lookup_context_, + block_prefetcher_.prefetch_buffer(), + /*for_compaction=*/is_for_compaction, /*async_read=*/false, s); + } + block_iter_points_to_real_block_ = true; + CheckDataBlockWithinUpperBound(); + + if (!is_for_compaction && + (seek_stat_state_ & kDataBlockReadSinceLastSeek) == 0) { + RecordTick(table_->GetStatistics(), is_last_level_ + ? LAST_LEVEL_SEEK_DATA + : NON_LAST_LEVEL_SEEK_DATA); + seek_stat_state_ = static_cast( + seek_stat_state_ | kDataBlockReadSinceLastSeek | kReportOnUseful); + } + async_read_in_progress_ = false; +} + +bool BlockBasedTableIterator::MaterializeCurrentBlock() { + assert(is_at_first_key_from_index_); + assert(!block_iter_points_to_real_block_); + assert(index_iter_->Valid()); + + is_at_first_key_from_index_ = false; + InitDataBlock(); + assert(block_iter_points_to_real_block_); + + if (!block_iter_.status().ok()) { + return false; + } + + block_iter_.SeekToFirst(); + + if (!block_iter_.Valid() || + icomp_.Compare(block_iter_.key(), + index_iter_->value().first_internal_key) != 0) { + block_iter_.Invalidate(Status::Corruption( + "first key in index doesn't match first key in block")); + return false; + } + + return true; +} + +void BlockBasedTableIterator::FindKeyForward() { + // This method's code is kept short to make it likely to be inlined. + + assert(!is_out_of_bound_); + assert(block_iter_points_to_real_block_); + + if (!block_iter_.Valid()) { + // This is the only call site of FindBlockForward(), but it's extracted into + // a separate method to keep FindKeyForward() short and likely to be + // inlined. When transitioning to a different block, we call + // FindBlockForward(), which is much longer and is probably not inlined. + FindBlockForward(); + } else { + // This is the fast path that avoids a function call. + } +} + +void BlockBasedTableIterator::FindBlockForward() { + // TODO the while loop inherits from two-level-iterator. We don't know + // whether a block can be empty so it can be replaced by an "if". + do { + if (!block_iter_.status().ok()) { + return; + } + // Whether next data block is out of upper bound, if there is one. + const bool next_block_is_out_of_bound = + read_options_.iterate_upper_bound != nullptr && + block_iter_points_to_real_block_ && + block_upper_bound_check_ == BlockUpperBound::kUpperBoundInCurBlock; + assert(!next_block_is_out_of_bound || + user_comparator_.CompareWithoutTimestamp( + *read_options_.iterate_upper_bound, /*a_has_ts=*/false, + index_iter_->user_key(), /*b_has_ts=*/true) <= 0); + ResetDataIter(); + index_iter_->Next(); + if (next_block_is_out_of_bound) { + // The next block is out of bound. No need to read it. + TEST_SYNC_POINT_CALLBACK("BlockBasedTableIterator:out_of_bound", nullptr); + // We need to make sure this is not the last data block before setting + // is_out_of_bound_, since the index key for the last data block can be + // larger than smallest key of the next file on the same level. + if (index_iter_->Valid()) { + is_out_of_bound_ = true; + } + return; + } + + if (!index_iter_->Valid()) { + return; + } + + IndexValue v = index_iter_->value(); + + if (!v.first_internal_key.empty() && allow_unprepared_value_) { + // Index contains the first key of the block. Defer reading the block. + is_at_first_key_from_index_ = true; + return; + } + + InitDataBlock(); + block_iter_.SeekToFirst(); + } while (!block_iter_.Valid()); +} + +void BlockBasedTableIterator::FindKeyBackward() { + while (!block_iter_.Valid()) { + if (!block_iter_.status().ok()) { + return; + } + + ResetDataIter(); + index_iter_->Prev(); + + if (index_iter_->Valid()) { + InitDataBlock(); + block_iter_.SeekToLast(); + } else { + return; + } + } + + // We could have check lower bound here too, but we opt not to do it for + // code simplicity. +} + +void BlockBasedTableIterator::CheckOutOfBound() { + if (read_options_.iterate_upper_bound != nullptr && + block_upper_bound_check_ != BlockUpperBound::kUpperBoundBeyondCurBlock && + Valid()) { + is_out_of_bound_ = + user_comparator_.CompareWithoutTimestamp( + *read_options_.iterate_upper_bound, /*a_has_ts=*/false, user_key(), + /*b_has_ts=*/true) <= 0; + } +} + +void BlockBasedTableIterator::CheckDataBlockWithinUpperBound() { + if (read_options_.iterate_upper_bound != nullptr && + block_iter_points_to_real_block_) { + block_upper_bound_check_ = (user_comparator_.CompareWithoutTimestamp( + *read_options_.iterate_upper_bound, + /*a_has_ts=*/false, index_iter_->user_key(), + /*b_has_ts=*/true) > 0) + ? BlockUpperBound::kUpperBoundBeyondCurBlock + : BlockUpperBound::kUpperBoundInCurBlock; + } +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/block_based_table_iterator.h b/librocksdb-sys/rocksdb/table/block_based/block_based_table_iterator.h new file mode 100644 index 0000000..6ea53f3 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block_based_table_iterator.h @@ -0,0 +1,310 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once +#include "table/block_based/block_based_table_reader.h" +#include "table/block_based/block_based_table_reader_impl.h" +#include "table/block_based/block_prefetcher.h" +#include "table/block_based/reader_common.h" + +namespace ROCKSDB_NAMESPACE { +// Iterates over the contents of BlockBasedTable. +class BlockBasedTableIterator : public InternalIteratorBase { + // compaction_readahead_size: its value will only be used if for_compaction = + // true + // @param read_options Must outlive this iterator. + public: + BlockBasedTableIterator( + const BlockBasedTable* table, const ReadOptions& read_options, + const InternalKeyComparator& icomp, + std::unique_ptr>&& index_iter, + bool check_filter, bool need_upper_bound_check, + const SliceTransform* prefix_extractor, TableReaderCaller caller, + size_t compaction_readahead_size = 0, bool allow_unprepared_value = false) + : index_iter_(std::move(index_iter)), + table_(table), + read_options_(read_options), + icomp_(icomp), + user_comparator_(icomp.user_comparator()), + pinned_iters_mgr_(nullptr), + prefix_extractor_(prefix_extractor), + lookup_context_(caller), + block_prefetcher_( + compaction_readahead_size, + table_->get_rep()->table_options.initial_auto_readahead_size), + allow_unprepared_value_(allow_unprepared_value), + block_iter_points_to_real_block_(false), + check_filter_(check_filter), + need_upper_bound_check_(need_upper_bound_check), + async_read_in_progress_(false), + is_last_level_(table->IsLastLevel()) {} + + ~BlockBasedTableIterator() {} + + void Seek(const Slice& target) override; + void SeekForPrev(const Slice& target) override; + void SeekToFirst() override; + void SeekToLast() override; + void Next() final override; + bool NextAndGetResult(IterateResult* result) override; + void Prev() override; + bool Valid() const override { + return !is_out_of_bound_ && + (is_at_first_key_from_index_ || + (block_iter_points_to_real_block_ && block_iter_.Valid())); + } + Slice key() const override { + assert(Valid()); + if (is_at_first_key_from_index_) { + return index_iter_->value().first_internal_key; + } else { + return block_iter_.key(); + } + } + Slice user_key() const override { + assert(Valid()); + if (is_at_first_key_from_index_) { + return ExtractUserKey(index_iter_->value().first_internal_key); + } else { + return block_iter_.user_key(); + } + } + bool PrepareValue() override { + assert(Valid()); + + if (!is_at_first_key_from_index_) { + return true; + } + + return const_cast(this) + ->MaterializeCurrentBlock(); + } + Slice value() const override { + // PrepareValue() must have been called. + assert(!is_at_first_key_from_index_); + assert(Valid()); + + if (seek_stat_state_ & kReportOnUseful) { + bool filter_used = (seek_stat_state_ & kFilterUsed) != 0; + RecordTick( + table_->GetStatistics(), + filter_used + ? (is_last_level_ ? LAST_LEVEL_SEEK_DATA_USEFUL_FILTER_MATCH + : NON_LAST_LEVEL_SEEK_DATA_USEFUL_FILTER_MATCH) + : (is_last_level_ ? LAST_LEVEL_SEEK_DATA_USEFUL_NO_FILTER + : NON_LAST_LEVEL_SEEK_DATA_USEFUL_NO_FILTER)); + seek_stat_state_ = kDataBlockReadSinceLastSeek; + } + + return block_iter_.value(); + } + Status status() const override { + // Prefix index set status to NotFound when the prefix does not exist + if (!index_iter_->status().ok() && !index_iter_->status().IsNotFound()) { + return index_iter_->status(); + } else if (block_iter_points_to_real_block_) { + return block_iter_.status(); + } else if (async_read_in_progress_) { + return Status::TryAgain(); + } else { + return Status::OK(); + } + } + + inline IterBoundCheck UpperBoundCheckResult() override { + if (is_out_of_bound_) { + return IterBoundCheck::kOutOfBound; + } else if (block_upper_bound_check_ == + BlockUpperBound::kUpperBoundBeyondCurBlock) { + assert(!is_out_of_bound_); + return IterBoundCheck::kInbound; + } else { + return IterBoundCheck::kUnknown; + } + } + + void SetPinnedItersMgr(PinnedIteratorsManager* pinned_iters_mgr) override { + pinned_iters_mgr_ = pinned_iters_mgr; + } + bool IsKeyPinned() const override { + // Our key comes either from block_iter_'s current key + // or index_iter_'s current *value*. + return pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled() && + ((is_at_first_key_from_index_ && index_iter_->IsValuePinned()) || + (block_iter_points_to_real_block_ && block_iter_.IsKeyPinned())); + } + bool IsValuePinned() const override { + assert(!is_at_first_key_from_index_); + assert(Valid()); + + // BlockIter::IsValuePinned() is always true. No need to check + return pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled() && + block_iter_points_to_real_block_; + } + + void ResetDataIter() { + if (block_iter_points_to_real_block_) { + if (pinned_iters_mgr_ != nullptr && pinned_iters_mgr_->PinningEnabled()) { + block_iter_.DelegateCleanupsTo(pinned_iters_mgr_); + } + block_iter_.Invalidate(Status::OK()); + block_iter_points_to_real_block_ = false; + } + block_upper_bound_check_ = BlockUpperBound::kUnknown; + } + + void SavePrevIndexValue() { + if (block_iter_points_to_real_block_) { + // Reseek. If they end up with the same data block, we shouldn't re-fetch + // the same data block. + prev_block_offset_ = index_iter_->value().handle.offset(); + } + } + + void GetReadaheadState(ReadaheadFileInfo* readahead_file_info) override { + if (block_prefetcher_.prefetch_buffer() != nullptr && + read_options_.adaptive_readahead) { + block_prefetcher_.prefetch_buffer()->GetReadaheadState( + &(readahead_file_info->data_block_readahead_info)); + if (index_iter_) { + index_iter_->GetReadaheadState(readahead_file_info); + } + } + } + + void SetReadaheadState(ReadaheadFileInfo* readahead_file_info) override { + if (read_options_.adaptive_readahead) { + block_prefetcher_.SetReadaheadState( + &(readahead_file_info->data_block_readahead_info)); + if (index_iter_) { + index_iter_->SetReadaheadState(readahead_file_info); + } + } + } + + std::unique_ptr> index_iter_; + + private: + enum class IterDirection { + kForward, + kBackward, + }; + // This enum indicates whether the upper bound falls into current block + // or beyond. + // +-------------+ + // | cur block | <-- (1) + // +-------------+ + // <-- (2) + // --- --- + // <-- (3) + // +-------------+ + // | next block | <-- (4) + // ...... + // + // When the block is smaller than , kUpperBoundInCurBlock + // is the value to use. The examples are (1) or (2) in the graph. It means + // all keys in the next block or beyond will be out of bound. Keys within + // the current block may or may not be out of bound. + // When the block is larger or equal to , + // kUpperBoundBeyondCurBlock is to be used. The examples are (3) and (4) + // in the graph. It means that all keys in the current block is within the + // upper bound and keys in the next block may or may not be within the uppder + // bound. + // If the boundary key hasn't been checked against the upper bound, + // kUnknown can be used. + enum class BlockUpperBound : uint8_t { + kUpperBoundInCurBlock, + kUpperBoundBeyondCurBlock, + kUnknown, + }; + + // State bits for collecting stats on seeks and whether they returned useful + // results. + enum SeekStatState : uint8_t { + kNone = 0, + // Most recent seek checked prefix filter (or similar future feature) + kFilterUsed = 1 << 0, + // Already recorded that a data block was accessed since the last seek. + kDataBlockReadSinceLastSeek = 1 << 1, + // Have not yet recorded that a value() was accessed. + kReportOnUseful = 1 << 2, + }; + + const BlockBasedTable* table_; + const ReadOptions& read_options_; + const InternalKeyComparator& icomp_; + UserComparatorWrapper user_comparator_; + PinnedIteratorsManager* pinned_iters_mgr_; + DataBlockIter block_iter_; + const SliceTransform* prefix_extractor_; + uint64_t prev_block_offset_ = std::numeric_limits::max(); + BlockCacheLookupContext lookup_context_; + + BlockPrefetcher block_prefetcher_; + + const bool allow_unprepared_value_; + // True if block_iter_ is initialized and points to the same block + // as index iterator. + bool block_iter_points_to_real_block_; + // See InternalIteratorBase::IsOutOfBound(). + bool is_out_of_bound_ = false; + // How current data block's boundary key with the next block is compared with + // iterate upper bound. + BlockUpperBound block_upper_bound_check_ = BlockUpperBound::kUnknown; + // True if we're standing at the first key of a block, and we haven't loaded + // that block yet. A call to PrepareValue() will trigger loading the block. + bool is_at_first_key_from_index_ = false; + bool check_filter_; + // TODO(Zhongyi): pick a better name + bool need_upper_bound_check_; + + bool async_read_in_progress_; + + mutable SeekStatState seek_stat_state_ = SeekStatState::kNone; + bool is_last_level_; + + // If `target` is null, seek to first. + void SeekImpl(const Slice* target, bool async_prefetch); + + void InitDataBlock(); + void AsyncInitDataBlock(bool is_first_pass); + bool MaterializeCurrentBlock(); + void FindKeyForward(); + void FindBlockForward(); + void FindKeyBackward(); + void CheckOutOfBound(); + + // Check if data block is fully within iterate_upper_bound. + // + // Note MyRocks may update iterate bounds between seek. To workaround it, + // we need to check and update data_block_within_upper_bound_ accordingly. + void CheckDataBlockWithinUpperBound(); + + bool CheckPrefixMayMatch(const Slice& ikey, IterDirection direction, + bool* filter_checked) { + if (need_upper_bound_check_ && direction == IterDirection::kBackward) { + // Upper bound check isn't sufficient for backward direction to + // guarantee the same result as total order, so disable prefix + // check. + return true; + } + if (check_filter_ && + !table_->PrefixRangeMayMatch(ikey, read_options_, prefix_extractor_, + need_upper_bound_check_, &lookup_context_, + filter_checked)) { + // TODO remember the iterator is invalidated because of prefix + // match. This can avoid the upper level file iterator to falsely + // believe the position is the end of the SST file and move to + // the first key of the next file. + ResetDataIter(); + return false; + } + return true; + } +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/block_based_table_reader.cc b/librocksdb-sys/rocksdb/table/block_based/block_based_table_reader.cc new file mode 100644 index 0000000..129f878 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block_based_table_reader.cc @@ -0,0 +1,3058 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "table/block_based/block_based_table_reader.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "block_cache.h" +#include "cache/cache_entry_roles.h" +#include "cache/cache_key.h" +#include "db/compaction/compaction_picker.h" +#include "db/dbformat.h" +#include "db/pinned_iterators_manager.h" +#include "file/file_prefetch_buffer.h" +#include "file/file_util.h" +#include "file/random_access_file_reader.h" +#include "logging/logging.h" +#include "monitoring/perf_context_imp.h" +#include "parsed_full_filter_block.h" +#include "port/lang.h" +#include "rocksdb/cache.h" +#include "rocksdb/comparator.h" +#include "rocksdb/convenience.h" +#include "rocksdb/env.h" +#include "rocksdb/file_system.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/iterator.h" +#include "rocksdb/options.h" +#include "rocksdb/snapshot.h" +#include "rocksdb/statistics.h" +#include "rocksdb/system_clock.h" +#include "rocksdb/table.h" +#include "rocksdb/table_properties.h" +#include "rocksdb/trace_record.h" +#include "table/block_based/binary_search_index_reader.h" +#include "table/block_based/block.h" +#include "table/block_based/block_based_table_factory.h" +#include "table/block_based/block_based_table_iterator.h" +#include "table/block_based/block_prefix_index.h" +#include "table/block_based/block_type.h" +#include "table/block_based/filter_block.h" +#include "table/block_based/filter_policy_internal.h" +#include "table/block_based/full_filter_block.h" +#include "table/block_based/hash_index_reader.h" +#include "table/block_based/partitioned_filter_block.h" +#include "table/block_based/partitioned_index_reader.h" +#include "table/block_fetcher.h" +#include "table/format.h" +#include "table/get_context.h" +#include "table/internal_iterator.h" +#include "table/meta_blocks.h" +#include "table/multiget_context.h" +#include "table/persistent_cache_helper.h" +#include "table/persistent_cache_options.h" +#include "table/sst_file_writer_collectors.h" +#include "table/two_level_iterator.h" +#include "test_util/sync_point.h" +#include "util/coding.h" +#include "util/crc32c.h" +#include "util/stop_watch.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { +namespace { + +CacheAllocationPtr CopyBufferToHeap(MemoryAllocator* allocator, Slice& buf) { + CacheAllocationPtr heap_buf; + heap_buf = AllocateBlock(buf.size(), allocator); + memcpy(heap_buf.get(), buf.data(), buf.size()); + return heap_buf; +} +} // namespace + +// Explicitly instantiate templates for each "blocklike" type we use (and +// before implicit specialization). +// This makes it possible to keep the template definitions in the .cc file. +#define INSTANTIATE_RETRIEVE_BLOCK(T) \ + template Status BlockBasedTable::RetrieveBlock( \ + FilePrefetchBuffer * prefetch_buffer, const ReadOptions& ro, \ + const BlockHandle& handle, const UncompressionDict& uncompression_dict, \ + CachableEntry* out_parsed_block, GetContext* get_context, \ + BlockCacheLookupContext* lookup_context, bool for_compaction, \ + bool use_cache, bool async_read) const; + +INSTANTIATE_RETRIEVE_BLOCK(ParsedFullFilterBlock); +INSTANTIATE_RETRIEVE_BLOCK(UncompressionDict); +INSTANTIATE_RETRIEVE_BLOCK(Block_kData); +INSTANTIATE_RETRIEVE_BLOCK(Block_kIndex); +INSTANTIATE_RETRIEVE_BLOCK(Block_kFilterPartitionIndex); +INSTANTIATE_RETRIEVE_BLOCK(Block_kRangeDeletion); +INSTANTIATE_RETRIEVE_BLOCK(Block_kMetaIndex); + +} // namespace ROCKSDB_NAMESPACE + +// Generate the regular and coroutine versions of some methods by +// including block_based_table_reader_sync_and_async.h twice +// Macros in the header will expand differently based on whether +// WITH_COROUTINES or WITHOUT_COROUTINES is defined +// clang-format off +#define WITHOUT_COROUTINES +#include "table/block_based/block_based_table_reader_sync_and_async.h" +#undef WITHOUT_COROUTINES +#define WITH_COROUTINES +#include "table/block_based/block_based_table_reader_sync_and_async.h" +#undef WITH_COROUTINES +// clang-format on + +namespace ROCKSDB_NAMESPACE { + +extern const uint64_t kBlockBasedTableMagicNumber; +extern const std::string kHashIndexPrefixesBlock; +extern const std::string kHashIndexPrefixesMetadataBlock; + +BlockBasedTable::~BlockBasedTable() { delete rep_; } + +namespace { +// Read the block identified by "handle" from "file". +// The only relevant option is options.verify_checksums for now. +// On failure return non-OK. +// On success fill *result and return OK - caller owns *result +// @param uncompression_dict Data for presetting the compression library's +// dictionary. +template +Status ReadAndParseBlockFromFile( + RandomAccessFileReader* file, FilePrefetchBuffer* prefetch_buffer, + const Footer& footer, const ReadOptions& options, const BlockHandle& handle, + std::unique_ptr* result, const ImmutableOptions& ioptions, + BlockCreateContext& create_context, bool maybe_compressed, + const UncompressionDict& uncompression_dict, + const PersistentCacheOptions& cache_options, + MemoryAllocator* memory_allocator, bool for_compaction, bool async_read) { + assert(result); + + BlockContents contents; + BlockFetcher block_fetcher( + file, prefetch_buffer, footer, options, handle, &contents, ioptions, + /*do_uncompress*/ maybe_compressed, maybe_compressed, + TBlocklike::kBlockType, uncompression_dict, cache_options, + memory_allocator, nullptr, for_compaction); + Status s; + // If prefetch_buffer is not allocated, it will fallback to synchronous + // reading of block contents. + if (async_read && prefetch_buffer != nullptr) { + s = block_fetcher.ReadAsyncBlockContents(); + if (!s.ok()) { + return s; + } + } else { + s = block_fetcher.ReadBlockContents(); + } + if (s.ok()) { + create_context.Create(result, std::move(contents)); + } + return s; +} + +// For hash based index, return false if table_properties->prefix_extractor_name +// and prefix_extractor both exist and match, otherwise true. +inline bool PrefixExtractorChangedHelper( + const TableProperties* table_properties, + const SliceTransform* prefix_extractor) { + // BlockBasedTableOptions::kHashSearch requires prefix_extractor to be set. + // Turn off hash index in prefix_extractor is not set; if prefix_extractor + // is set but prefix_extractor_block is not set, also disable hash index + if (prefix_extractor == nullptr || table_properties == nullptr || + table_properties->prefix_extractor_name.empty()) { + return true; + } + + // prefix_extractor and prefix_extractor_block are both non-empty + if (table_properties->prefix_extractor_name != prefix_extractor->AsString()) { + return true; + } else { + return false; + } +} + +template +uint32_t GetBlockNumRestarts(const TBlocklike& block) { + if constexpr (std::is_convertible_v) { + const Block& b = block; + return b.NumRestarts(); + } else { + return 0; + } +} + +} // namespace + +void BlockBasedTable::UpdateCacheHitMetrics(BlockType block_type, + GetContext* get_context, + size_t usage) const { + Statistics* const statistics = rep_->ioptions.stats; + + PERF_COUNTER_ADD(block_cache_hit_count, 1); + PERF_COUNTER_BY_LEVEL_ADD(block_cache_hit_count, 1, + static_cast(rep_->level)); + + if (get_context) { + ++get_context->get_context_stats_.num_cache_hit; + get_context->get_context_stats_.num_cache_bytes_read += usage; + } else { + RecordTick(statistics, BLOCK_CACHE_HIT); + RecordTick(statistics, BLOCK_CACHE_BYTES_READ, usage); + } + + switch (block_type) { + case BlockType::kFilter: + case BlockType::kFilterPartitionIndex: + PERF_COUNTER_ADD(block_cache_filter_hit_count, 1); + + if (get_context) { + ++get_context->get_context_stats_.num_cache_filter_hit; + } else { + RecordTick(statistics, BLOCK_CACHE_FILTER_HIT); + } + break; + + case BlockType::kCompressionDictionary: + // TODO: introduce perf counter for compression dictionary hit count + if (get_context) { + ++get_context->get_context_stats_.num_cache_compression_dict_hit; + } else { + RecordTick(statistics, BLOCK_CACHE_COMPRESSION_DICT_HIT); + } + break; + + case BlockType::kIndex: + PERF_COUNTER_ADD(block_cache_index_hit_count, 1); + + if (get_context) { + ++get_context->get_context_stats_.num_cache_index_hit; + } else { + RecordTick(statistics, BLOCK_CACHE_INDEX_HIT); + } + break; + + default: + // TODO: introduce dedicated tickers/statistics/counters + // for range tombstones + if (get_context) { + ++get_context->get_context_stats_.num_cache_data_hit; + } else { + RecordTick(statistics, BLOCK_CACHE_DATA_HIT); + } + break; + } +} + +void BlockBasedTable::UpdateCacheMissMetrics(BlockType block_type, + GetContext* get_context) const { + Statistics* const statistics = rep_->ioptions.stats; + + // TODO: introduce aggregate (not per-level) block cache miss count + PERF_COUNTER_BY_LEVEL_ADD(block_cache_miss_count, 1, + static_cast(rep_->level)); + + if (get_context) { + ++get_context->get_context_stats_.num_cache_miss; + } else { + RecordTick(statistics, BLOCK_CACHE_MISS); + } + + // TODO: introduce perf counters for misses per block type + switch (block_type) { + case BlockType::kFilter: + case BlockType::kFilterPartitionIndex: + if (get_context) { + ++get_context->get_context_stats_.num_cache_filter_miss; + } else { + RecordTick(statistics, BLOCK_CACHE_FILTER_MISS); + } + break; + + case BlockType::kCompressionDictionary: + if (get_context) { + ++get_context->get_context_stats_.num_cache_compression_dict_miss; + } else { + RecordTick(statistics, BLOCK_CACHE_COMPRESSION_DICT_MISS); + } + break; + + case BlockType::kIndex: + if (get_context) { + ++get_context->get_context_stats_.num_cache_index_miss; + } else { + RecordTick(statistics, BLOCK_CACHE_INDEX_MISS); + } + break; + + default: + // TODO: introduce dedicated tickers/statistics/counters + // for range tombstones + if (get_context) { + ++get_context->get_context_stats_.num_cache_data_miss; + } else { + RecordTick(statistics, BLOCK_CACHE_DATA_MISS); + } + break; + } +} + +void BlockBasedTable::UpdateCacheInsertionMetrics( + BlockType block_type, GetContext* get_context, size_t usage, bool redundant, + Statistics* const statistics) { + // TODO: introduce perf counters for block cache insertions + if (get_context) { + ++get_context->get_context_stats_.num_cache_add; + if (redundant) { + ++get_context->get_context_stats_.num_cache_add_redundant; + } + get_context->get_context_stats_.num_cache_bytes_write += usage; + } else { + RecordTick(statistics, BLOCK_CACHE_ADD); + if (redundant) { + RecordTick(statistics, BLOCK_CACHE_ADD_REDUNDANT); + } + RecordTick(statistics, BLOCK_CACHE_BYTES_WRITE, usage); + } + + switch (block_type) { + case BlockType::kFilter: + case BlockType::kFilterPartitionIndex: + if (get_context) { + ++get_context->get_context_stats_.num_cache_filter_add; + if (redundant) { + ++get_context->get_context_stats_.num_cache_filter_add_redundant; + } + get_context->get_context_stats_.num_cache_filter_bytes_insert += usage; + } else { + RecordTick(statistics, BLOCK_CACHE_FILTER_ADD); + if (redundant) { + RecordTick(statistics, BLOCK_CACHE_FILTER_ADD_REDUNDANT); + } + RecordTick(statistics, BLOCK_CACHE_FILTER_BYTES_INSERT, usage); + } + break; + + case BlockType::kCompressionDictionary: + if (get_context) { + ++get_context->get_context_stats_.num_cache_compression_dict_add; + if (redundant) { + ++get_context->get_context_stats_ + .num_cache_compression_dict_add_redundant; + } + get_context->get_context_stats_ + .num_cache_compression_dict_bytes_insert += usage; + } else { + RecordTick(statistics, BLOCK_CACHE_COMPRESSION_DICT_ADD); + if (redundant) { + RecordTick(statistics, BLOCK_CACHE_COMPRESSION_DICT_ADD_REDUNDANT); + } + RecordTick(statistics, BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT, + usage); + } + break; + + case BlockType::kIndex: + if (get_context) { + ++get_context->get_context_stats_.num_cache_index_add; + if (redundant) { + ++get_context->get_context_stats_.num_cache_index_add_redundant; + } + get_context->get_context_stats_.num_cache_index_bytes_insert += usage; + } else { + RecordTick(statistics, BLOCK_CACHE_INDEX_ADD); + if (redundant) { + RecordTick(statistics, BLOCK_CACHE_INDEX_ADD_REDUNDANT); + } + RecordTick(statistics, BLOCK_CACHE_INDEX_BYTES_INSERT, usage); + } + break; + + default: + // TODO: introduce dedicated tickers/statistics/counters + // for range tombstones + if (get_context) { + ++get_context->get_context_stats_.num_cache_data_add; + if (redundant) { + ++get_context->get_context_stats_.num_cache_data_add_redundant; + } + get_context->get_context_stats_.num_cache_data_bytes_insert += usage; + } else { + RecordTick(statistics, BLOCK_CACHE_DATA_ADD); + if (redundant) { + RecordTick(statistics, BLOCK_CACHE_DATA_ADD_REDUNDANT); + } + RecordTick(statistics, BLOCK_CACHE_DATA_BYTES_INSERT, usage); + } + break; + } +} + +namespace { +// Return True if table_properties has `user_prop_name` has a `true` value +// or it doesn't contain this property (for backward compatible). +bool IsFeatureSupported(const TableProperties& table_properties, + const std::string& user_prop_name, Logger* info_log) { + auto& props = table_properties.user_collected_properties; + auto pos = props.find(user_prop_name); + // Older version doesn't have this value set. Skip this check. + if (pos != props.end()) { + if (pos->second == kPropFalse) { + return false; + } else if (pos->second != kPropTrue) { + ROCKS_LOG_WARN(info_log, "Property %s has invalidate value %s", + user_prop_name.c_str(), pos->second.c_str()); + } + } + return true; +} + +// Caller has to ensure seqno is not nullptr. +Status GetGlobalSequenceNumber(const TableProperties& table_properties, + SequenceNumber largest_seqno, + SequenceNumber* seqno) { + const auto& props = table_properties.user_collected_properties; + const auto version_pos = props.find(ExternalSstFilePropertyNames::kVersion); + const auto seqno_pos = props.find(ExternalSstFilePropertyNames::kGlobalSeqno); + + *seqno = kDisableGlobalSequenceNumber; + if (version_pos == props.end()) { + if (seqno_pos != props.end()) { + std::array msg_buf; + // This is not an external sst file, global_seqno is not supported. + snprintf( + msg_buf.data(), msg_buf.max_size(), + "A non-external sst file have global seqno property with value %s", + seqno_pos->second.c_str()); + return Status::Corruption(msg_buf.data()); + } + return Status::OK(); + } + + uint32_t version = DecodeFixed32(version_pos->second.c_str()); + if (version < 2) { + if (seqno_pos != props.end() || version != 1) { + std::array msg_buf; + // This is a v1 external sst file, global_seqno is not supported. + snprintf(msg_buf.data(), msg_buf.max_size(), + "An external sst file with version %u have global seqno " + "property with value %s", + version, seqno_pos->second.c_str()); + return Status::Corruption(msg_buf.data()); + } + return Status::OK(); + } + + // Since we have a plan to deprecate global_seqno, we do not return failure + // if seqno_pos == props.end(). We rely on version_pos to detect whether the + // SST is external. + SequenceNumber global_seqno(0); + if (seqno_pos != props.end()) { + global_seqno = DecodeFixed64(seqno_pos->second.c_str()); + } + // SstTableReader open table reader with kMaxSequenceNumber as largest_seqno + // to denote it is unknown. + if (largest_seqno < kMaxSequenceNumber) { + if (global_seqno == 0) { + global_seqno = largest_seqno; + } + if (global_seqno != largest_seqno) { + std::array msg_buf; + snprintf( + msg_buf.data(), msg_buf.max_size(), + "An external sst file with version %u have global seqno property " + "with value %s, while largest seqno in the file is %llu", + version, seqno_pos->second.c_str(), + static_cast(largest_seqno)); + return Status::Corruption(msg_buf.data()); + } + } + *seqno = global_seqno; + + if (global_seqno > kMaxSequenceNumber) { + std::array msg_buf; + snprintf(msg_buf.data(), msg_buf.max_size(), + "An external sst file with version %u have global seqno property " + "with value %llu, which is greater than kMaxSequenceNumber", + version, static_cast(global_seqno)); + return Status::Corruption(msg_buf.data()); + } + + return Status::OK(); +} +} // namespace + +void BlockBasedTable::SetupBaseCacheKey(const TableProperties* properties, + const std::string& cur_db_session_id, + uint64_t cur_file_number, + OffsetableCacheKey* out_base_cache_key, + bool* out_is_stable) { + // Use a stable cache key if sufficient data is in table properties + std::string db_session_id; + uint64_t file_num; + std::string db_id; + if (properties && !properties->db_session_id.empty() && + properties->orig_file_number > 0) { + // (Newer SST file case) + // We must have both properties to get a stable unique id because + // CreateColumnFamilyWithImport or IngestExternalFiles can change the + // file numbers on a file. + db_session_id = properties->db_session_id; + file_num = properties->orig_file_number; + // Less critical, populated in earlier release than above + db_id = properties->db_id; + if (out_is_stable) { + *out_is_stable = true; + } + } else { + // (Old SST file case) + // We use (unique) cache keys based on current identifiers. These are at + // least stable across table file close and re-open, but not across + // different DBs nor DB close and re-open. + db_session_id = cur_db_session_id; + file_num = cur_file_number; + // Plumbing through the DB ID to here would be annoying, and of limited + // value because of the case of VersionSet::Recover opening some table + // files and later setting the DB ID. So we just rely on uniqueness + // level provided by session ID. + db_id = "unknown"; + if (out_is_stable) { + *out_is_stable = false; + } + } + + // Too many tests to update to get these working + // assert(file_num > 0); + // assert(!db_session_id.empty()); + // assert(!db_id.empty()); + + // Minimum block size is 5 bytes; therefore we can trim off two lower bits + // from offsets. See GetCacheKey. + *out_base_cache_key = OffsetableCacheKey(db_id, db_session_id, file_num); +} + +CacheKey BlockBasedTable::GetCacheKey(const OffsetableCacheKey& base_cache_key, + const BlockHandle& handle) { + // Minimum block size is 5 bytes; therefore we can trim off two lower bits + // from offet. + return base_cache_key.WithOffset(handle.offset() >> 2); +} + +Status BlockBasedTable::Open( + const ReadOptions& read_options, const ImmutableOptions& ioptions, + const EnvOptions& env_options, const BlockBasedTableOptions& table_options, + const InternalKeyComparator& internal_comparator, + std::unique_ptr&& file, uint64_t file_size, + uint8_t block_protection_bytes_per_key, + std::unique_ptr* table_reader, uint64_t tail_size, + std::shared_ptr table_reader_cache_res_mgr, + const std::shared_ptr& prefix_extractor, + const bool prefetch_index_and_filter_in_cache, const bool skip_filters, + const int level, const bool immortal_table, + const SequenceNumber largest_seqno, const bool force_direct_prefetch, + TailPrefetchStats* tail_prefetch_stats, + BlockCacheTracer* const block_cache_tracer, + size_t max_file_size_for_l0_meta_pin, const std::string& cur_db_session_id, + uint64_t cur_file_num, UniqueId64x2 expected_unique_id, + const bool user_defined_timestamps_persisted) { + table_reader->reset(); + + Status s; + Footer footer; + std::unique_ptr prefetch_buffer; + + // From read_options, retain deadline, io_timeout, rate_limiter_priority, and + // verify_checksums. In future, we may retain more options. + ReadOptions ro; + ro.deadline = read_options.deadline; + ro.io_timeout = read_options.io_timeout; + ro.rate_limiter_priority = read_options.rate_limiter_priority; + ro.verify_checksums = read_options.verify_checksums; + ro.io_activity = read_options.io_activity; + + // prefetch both index and filters, down to all partitions + const bool prefetch_all = prefetch_index_and_filter_in_cache || level == 0; + const bool preload_all = !table_options.cache_index_and_filter_blocks; + + if (!ioptions.allow_mmap_reads) { + s = PrefetchTail(ro, file.get(), file_size, force_direct_prefetch, + tail_prefetch_stats, prefetch_all, preload_all, + &prefetch_buffer, ioptions.stats, tail_size, + ioptions.logger); + // Return error in prefetch path to users. + if (!s.ok()) { + return s; + } + } else { + // Should not prefetch for mmap mode. + prefetch_buffer.reset(new FilePrefetchBuffer( + 0 /* readahead_size */, 0 /* max_readahead_size */, false /* enable */, + true /* track_min_offset */)); + } + + // Read in the following order: + // 1. Footer + // 2. [metaindex block] + // 3. [meta block: properties] + // 4. [meta block: range deletion tombstone] + // 5. [meta block: compression dictionary] + // 6. [meta block: index] + // 7. [meta block: filter] + IOOptions opts; + s = file->PrepareIOOptions(ro, opts); + if (s.ok()) { + s = ReadFooterFromFile(opts, file.get(), *ioptions.fs, + prefetch_buffer.get(), file_size, &footer, + kBlockBasedTableMagicNumber); + } + if (!s.ok()) { + return s; + } + if (!IsSupportedFormatVersion(footer.format_version())) { + return Status::Corruption( + "Unknown Footer version. Maybe this file was created with newer " + "version of RocksDB?"); + } + + BlockCacheLookupContext lookup_context{TableReaderCaller::kPrefetch}; + Rep* rep = new BlockBasedTable::Rep( + ioptions, env_options, table_options, internal_comparator, skip_filters, + file_size, level, immortal_table, user_defined_timestamps_persisted); + rep->file = std::move(file); + rep->footer = footer; + + // For fully portable/stable cache keys, we need to read the properties + // block before setting up cache keys. TODO: consider setting up a bootstrap + // cache key for PersistentCache to use for metaindex and properties blocks. + rep->persistent_cache_options = PersistentCacheOptions(); + + // Meta-blocks are not dictionary compressed. Explicitly set the dictionary + // handle to null, otherwise it may be seen as uninitialized during the below + // meta-block reads. + rep->compression_dict_handle = BlockHandle::NullBlockHandle(); + + rep->create_context.protection_bytes_per_key = block_protection_bytes_per_key; + // Read metaindex + std::unique_ptr new_table( + new BlockBasedTable(rep, block_cache_tracer)); + std::unique_ptr metaindex; + std::unique_ptr metaindex_iter; + s = new_table->ReadMetaIndexBlock(ro, prefetch_buffer.get(), &metaindex, + &metaindex_iter); + if (!s.ok()) { + return s; + } + + // Populates table_properties and some fields that depend on it, + // such as index_type. + s = new_table->ReadPropertiesBlock(ro, prefetch_buffer.get(), + metaindex_iter.get(), largest_seqno); + if (!s.ok()) { + return s; + } + + // Populate BlockCreateContext + bool blocks_definitely_zstd_compressed = + rep->table_properties && + (rep->table_properties->compression_name == + CompressionTypeToString(kZSTD) || + rep->table_properties->compression_name == + CompressionTypeToString(kZSTDNotFinalCompression)); + rep->create_context = BlockCreateContext( + &rep->table_options, rep->ioptions.stats, + blocks_definitely_zstd_compressed, block_protection_bytes_per_key, + rep->internal_comparator.user_comparator(), rep->index_value_is_full, + rep->index_has_first_key); + + // Check expected unique id if provided + if (expected_unique_id != kNullUniqueId64x2) { + auto props = rep->table_properties; + if (!props) { + return Status::Corruption("Missing table properties on file " + + std::to_string(cur_file_num) + + " with known unique ID"); + } + UniqueId64x2 actual_unique_id{}; + s = GetSstInternalUniqueId(props->db_id, props->db_session_id, + props->orig_file_number, &actual_unique_id, + /*force*/ true); + assert(s.ok()); // because force=true + if (expected_unique_id != actual_unique_id) { + return Status::Corruption( + "Mismatch in unique ID on table file " + + std::to_string(cur_file_num) + + ". Expected: " + InternalUniqueIdToHumanString(&expected_unique_id) + + " Actual: " + InternalUniqueIdToHumanString(&actual_unique_id)); + } + TEST_SYNC_POINT_CALLBACK("BlockBasedTable::Open::PassedVerifyUniqueId", + &actual_unique_id); + } else { + TEST_SYNC_POINT_CALLBACK("BlockBasedTable::Open::SkippedVerifyUniqueId", + nullptr); + if (ioptions.verify_sst_unique_id_in_manifest && ioptions.logger) { + // A crude but isolated way of reporting unverified files. This should not + // be an ongoing concern so doesn't deserve a place in Statistics IMHO. + static std::atomic unverified_count{0}; + auto prev_count = + unverified_count.fetch_add(1, std::memory_order_relaxed); + if (prev_count == 0) { + ROCKS_LOG_WARN( + ioptions.logger, + "At least one SST file opened without unique ID to verify: %" PRIu64 + ".sst", + cur_file_num); + } else if (prev_count % 1000 == 0) { + ROCKS_LOG_WARN( + ioptions.logger, + "Another ~1000 SST files opened without unique ID to verify"); + } + } + } + + // Set up prefix extracto as needed + bool force_null_table_prefix_extractor = false; + TEST_SYNC_POINT_CALLBACK( + "BlockBasedTable::Open::ForceNullTablePrefixExtractor", + &force_null_table_prefix_extractor); + if (force_null_table_prefix_extractor) { + assert(!rep->table_prefix_extractor); + } else if (!PrefixExtractorChangedHelper(rep->table_properties.get(), + prefix_extractor.get())) { + // Establish fast path for unchanged prefix_extractor + rep->table_prefix_extractor = prefix_extractor; + } else { + // Current prefix_extractor doesn't match table + if (rep->table_properties) { + //**TODO: If/When the DBOptions has a registry in it, the ConfigOptions + // will need to use it + ConfigOptions config_options; + Status st = SliceTransform::CreateFromString( + config_options, rep->table_properties->prefix_extractor_name, + &(rep->table_prefix_extractor)); + if (!st.ok()) { + //**TODO: Should this be error be returned or swallowed? + ROCKS_LOG_ERROR(rep->ioptions.logger, + "Failed to create prefix extractor[%s]: %s", + rep->table_properties->prefix_extractor_name.c_str(), + st.ToString().c_str()); + } + } + } + + // With properties loaded, we can set up portable/stable cache keys + SetupBaseCacheKey(rep->table_properties.get(), cur_db_session_id, + cur_file_num, &rep->base_cache_key); + + rep->persistent_cache_options = + PersistentCacheOptions(rep->table_options.persistent_cache, + rep->base_cache_key, rep->ioptions.stats); + + // TODO(yuzhangyu): handle range deletion entries for UDT in memtable only. + s = new_table->ReadRangeDelBlock(ro, prefetch_buffer.get(), + metaindex_iter.get(), internal_comparator, + &lookup_context); + if (!s.ok()) { + return s; + } + rep->verify_checksum_set_on_open = ro.verify_checksums; + s = new_table->PrefetchIndexAndFilterBlocks( + ro, prefetch_buffer.get(), metaindex_iter.get(), new_table.get(), + prefetch_all, table_options, level, file_size, + max_file_size_for_l0_meta_pin, &lookup_context); + + if (s.ok()) { + // Update tail prefetch stats + assert(prefetch_buffer.get() != nullptr); + if (tail_prefetch_stats != nullptr) { + assert(prefetch_buffer->min_offset_read() < file_size); + tail_prefetch_stats->RecordEffectiveSize( + static_cast(file_size) - prefetch_buffer->min_offset_read()); + } + } + + if (s.ok() && table_reader_cache_res_mgr) { + std::size_t mem_usage = new_table->ApproximateMemoryUsage(); + s = table_reader_cache_res_mgr->MakeCacheReservation( + mem_usage, &(rep->table_reader_cache_res_handle)); + if (s.IsMemoryLimit()) { + s = Status::MemoryLimit( + "Can't allocate " + + kCacheEntryRoleToCamelString[static_cast( + CacheEntryRole::kBlockBasedTableReader)] + + " due to memory limit based on " + "cache capacity for memory allocation"); + } + } + + if (s.ok()) { + *table_reader = std::move(new_table); + } + return s; +} + +Status BlockBasedTable::PrefetchTail( + const ReadOptions& ro, RandomAccessFileReader* file, uint64_t file_size, + bool force_direct_prefetch, TailPrefetchStats* tail_prefetch_stats, + const bool prefetch_all, const bool preload_all, + std::unique_ptr* prefetch_buffer, Statistics* stats, + uint64_t tail_size, Logger* const logger) { + assert(tail_size <= file_size); + + size_t tail_prefetch_size = 0; + if (tail_size != 0) { + tail_prefetch_size = tail_size; + } else { + if (tail_prefetch_stats != nullptr) { + // Multiple threads may get a 0 (no history) when running in parallel, + // but it will get cleared after the first of them finishes. + tail_prefetch_size = tail_prefetch_stats->GetSuggestedPrefetchSize(); + } + if (tail_prefetch_size == 0) { + // Before read footer, readahead backwards to prefetch data. Do more + // readahead if we're going to read index/filter. + // TODO: This may incorrectly select small readahead in case partitioned + // index/filter is enabled and top-level partition pinning is enabled. + // That's because we need to issue readahead before we read the + // properties, at which point we don't yet know the index type. + tail_prefetch_size = prefetch_all || preload_all ? 512 * 1024 : 4 * 1024; + + ROCKS_LOG_WARN(logger, + "Tail prefetch size %zu is calculated based on heuristics", + tail_prefetch_size); + } else { + ROCKS_LOG_WARN( + logger, + "Tail prefetch size %zu is calculated based on TailPrefetchStats", + tail_prefetch_size); + } + } + size_t prefetch_off; + size_t prefetch_len; + if (file_size < tail_prefetch_size) { + prefetch_off = 0; + prefetch_len = static_cast(file_size); + } else { + prefetch_off = static_cast(file_size - tail_prefetch_size); + prefetch_len = tail_prefetch_size; + } + +#ifndef NDEBUG + std::pair prefetch_off_len_pair = {&prefetch_off, + &prefetch_len}; + TEST_SYNC_POINT_CALLBACK("BlockBasedTable::Open::TailPrefetchLen", + &prefetch_off_len_pair); +#endif // NDEBUG + + // Try file system prefetch + if (!file->use_direct_io() && !force_direct_prefetch) { + if (!file->Prefetch(prefetch_off, prefetch_len, ro.rate_limiter_priority) + .IsNotSupported()) { + prefetch_buffer->reset(new FilePrefetchBuffer( + 0 /* readahead_size */, 0 /* max_readahead_size */, + false /* enable */, true /* track_min_offset */)); + return Status::OK(); + } + } + + // Use `FilePrefetchBuffer` + prefetch_buffer->reset(new FilePrefetchBuffer( + 0 /* readahead_size */, 0 /* max_readahead_size */, true /* enable */, + true /* track_min_offset */, false /* implicit_auto_readahead */, + 0 /* num_file_reads */, 0 /* num_file_reads_for_auto_readahead */, + nullptr /* fs */, nullptr /* clock */, stats, + FilePrefetchBufferUsage::kTableOpenPrefetchTail)); + + IOOptions opts; + Status s = file->PrepareIOOptions(ro, opts); + if (s.ok()) { + s = (*prefetch_buffer) + ->Prefetch(opts, file, prefetch_off, prefetch_len, + ro.rate_limiter_priority); + } + return s; +} + +Status BlockBasedTable::ReadPropertiesBlock( + const ReadOptions& ro, FilePrefetchBuffer* prefetch_buffer, + InternalIterator* meta_iter, const SequenceNumber largest_seqno) { + Status s; + BlockHandle handle; + s = FindOptionalMetaBlock(meta_iter, kPropertiesBlockName, &handle); + + if (!s.ok()) { + ROCKS_LOG_WARN(rep_->ioptions.logger, + "Error when seeking to properties block from file: %s", + s.ToString().c_str()); + } else if (!handle.IsNull()) { + s = meta_iter->status(); + std::unique_ptr table_properties; + if (s.ok()) { + s = ReadTablePropertiesHelper( + ro, handle, rep_->file.get(), prefetch_buffer, rep_->footer, + rep_->ioptions, &table_properties, nullptr /* memory_allocator */); + } + IGNORE_STATUS_IF_ERROR(s); + + if (!s.ok()) { + ROCKS_LOG_WARN(rep_->ioptions.logger, + "Encountered error while reading data from properties " + "block %s", + s.ToString().c_str()); + } else { + assert(table_properties != nullptr); + rep_->table_properties = std::move(table_properties); + rep_->blocks_maybe_compressed = + rep_->table_properties->compression_name != + CompressionTypeToString(kNoCompression); + } + } else { + ROCKS_LOG_ERROR(rep_->ioptions.logger, + "Cannot find Properties block from file."); + } + + // Read the table properties, if provided. + if (rep_->table_properties) { + rep_->whole_key_filtering &= + IsFeatureSupported(*(rep_->table_properties), + BlockBasedTablePropertyNames::kWholeKeyFiltering, + rep_->ioptions.logger); + rep_->prefix_filtering &= IsFeatureSupported( + *(rep_->table_properties), + BlockBasedTablePropertyNames::kPrefixFiltering, rep_->ioptions.logger); + + rep_->index_key_includes_seq = + rep_->table_properties->index_key_is_user_key == 0; + rep_->index_value_is_full = + rep_->table_properties->index_value_is_delta_encoded == 0; + + // Update index_type with the true type. + // If table properties don't contain index type, we assume that the table + // is in very old format and has kBinarySearch index type. + auto& props = rep_->table_properties->user_collected_properties; + auto index_type_pos = props.find(BlockBasedTablePropertyNames::kIndexType); + if (index_type_pos != props.end()) { + rep_->index_type = static_cast( + DecodeFixed32(index_type_pos->second.c_str())); + } + auto min_ts_pos = props.find("rocksdb.timestamp_min"); + if (min_ts_pos != props.end()) { + rep_->min_timestamp = Slice(min_ts_pos->second); + } + auto max_ts_pos = props.find("rocksdb.timestamp_max"); + if (max_ts_pos != props.end()) { + rep_->max_timestamp = Slice(max_ts_pos->second); + } + + rep_->index_has_first_key = + rep_->index_type == BlockBasedTableOptions::kBinarySearchWithFirstKey; + + s = GetGlobalSequenceNumber(*(rep_->table_properties), largest_seqno, + &(rep_->global_seqno)); + if (!s.ok()) { + ROCKS_LOG_ERROR(rep_->ioptions.logger, "%s", s.ToString().c_str()); + } + } + return s; +} + +Status BlockBasedTable::ReadRangeDelBlock( + const ReadOptions& read_options, FilePrefetchBuffer* prefetch_buffer, + InternalIterator* meta_iter, + const InternalKeyComparator& internal_comparator, + BlockCacheLookupContext* lookup_context) { + Status s; + BlockHandle range_del_handle; + s = FindOptionalMetaBlock(meta_iter, kRangeDelBlockName, &range_del_handle); + if (!s.ok()) { + ROCKS_LOG_WARN( + rep_->ioptions.logger, + "Error when seeking to range delete tombstones block from file: %s", + s.ToString().c_str()); + } else if (!range_del_handle.IsNull()) { + Status tmp_status; + std::unique_ptr iter(NewDataBlockIterator( + read_options, range_del_handle, + /*input_iter=*/nullptr, BlockType::kRangeDeletion, + /*get_context=*/nullptr, lookup_context, prefetch_buffer, + /*for_compaction= */ false, /*async_read= */ false, tmp_status)); + assert(iter != nullptr); + s = iter->status(); + if (!s.ok()) { + ROCKS_LOG_WARN( + rep_->ioptions.logger, + "Encountered error while reading data from range del block %s", + s.ToString().c_str()); + IGNORE_STATUS_IF_ERROR(s); + } else { + rep_->fragmented_range_dels = + std::make_shared(std::move(iter), + internal_comparator); + } + } + return s; +} + +Status BlockBasedTable::PrefetchIndexAndFilterBlocks( + const ReadOptions& ro, FilePrefetchBuffer* prefetch_buffer, + InternalIterator* meta_iter, BlockBasedTable* new_table, bool prefetch_all, + const BlockBasedTableOptions& table_options, const int level, + size_t file_size, size_t max_file_size_for_l0_meta_pin, + BlockCacheLookupContext* lookup_context) { + // Find filter handle and filter type + if (rep_->filter_policy) { + auto name = rep_->filter_policy->CompatibilityName(); + bool builtin_compatible = + strcmp(name, BuiltinFilterPolicy::kCompatibilityName()) == 0; + + for (const auto& [filter_type, prefix] : + {std::make_pair(Rep::FilterType::kFullFilter, kFullFilterBlockPrefix), + std::make_pair(Rep::FilterType::kPartitionedFilter, + kPartitionedFilterBlockPrefix), + std::make_pair(Rep::FilterType::kNoFilter, + kObsoleteFilterBlockPrefix)}) { + if (builtin_compatible) { + // This code is only here to deal with a hiccup in early 7.0.x where + // there was an unintentional name change in the SST files metadata. + // It should be OK to remove this in the future (late 2022) and just + // have the 'else' code. + // NOTE: the test:: names below are likely not needed but included + // out of caution + static const std::unordered_set kBuiltinNameAndAliases = { + BuiltinFilterPolicy::kCompatibilityName(), + test::LegacyBloomFilterPolicy::kClassName(), + test::FastLocalBloomFilterPolicy::kClassName(), + test::Standard128RibbonFilterPolicy::kClassName(), + "rocksdb.internal.DeprecatedBlockBasedBloomFilter", + BloomFilterPolicy::kClassName(), + RibbonFilterPolicy::kClassName(), + }; + + // For efficiency, do a prefix seek and see if the first match is + // good. + meta_iter->Seek(prefix); + if (meta_iter->status().ok() && meta_iter->Valid()) { + Slice key = meta_iter->key(); + if (key.starts_with(prefix)) { + key.remove_prefix(prefix.size()); + if (kBuiltinNameAndAliases.find(key.ToString()) != + kBuiltinNameAndAliases.end()) { + Slice v = meta_iter->value(); + Status s = rep_->filter_handle.DecodeFrom(&v); + if (s.ok()) { + rep_->filter_type = filter_type; + if (filter_type == Rep::FilterType::kNoFilter) { + ROCKS_LOG_WARN(rep_->ioptions.logger, + "Detected obsolete filter type in %s. Read " + "performance might suffer until DB is fully " + "re-compacted.", + rep_->file->file_name().c_str()); + } + break; + } + } + } + } + } else { + std::string filter_block_key = prefix + name; + if (FindMetaBlock(meta_iter, filter_block_key, &rep_->filter_handle) + .ok()) { + rep_->filter_type = filter_type; + if (filter_type == Rep::FilterType::kNoFilter) { + ROCKS_LOG_WARN( + rep_->ioptions.logger, + "Detected obsolete filter type in %s. Read performance might " + "suffer until DB is fully re-compacted.", + rep_->file->file_name().c_str()); + } + break; + } + } + } + } + // Partition filters cannot be enabled without partition indexes + assert(rep_->filter_type != Rep::FilterType::kPartitionedFilter || + rep_->index_type == BlockBasedTableOptions::kTwoLevelIndexSearch); + + // Find compression dictionary handle + Status s = FindOptionalMetaBlock(meta_iter, kCompressionDictBlockName, + &rep_->compression_dict_handle); + if (!s.ok()) { + return s; + } + + BlockBasedTableOptions::IndexType index_type = rep_->index_type; + + const bool use_cache = table_options.cache_index_and_filter_blocks; + + const bool maybe_flushed = + level == 0 && file_size <= max_file_size_for_l0_meta_pin; + std::function is_pinned = + [maybe_flushed, &is_pinned](PinningTier pinning_tier, + PinningTier fallback_pinning_tier) { + // Fallback to fallback would lead to infinite recursion. Disallow it. + assert(fallback_pinning_tier != PinningTier::kFallback); + + switch (pinning_tier) { + case PinningTier::kFallback: + return is_pinned(fallback_pinning_tier, + PinningTier::kNone /* fallback_pinning_tier */); + case PinningTier::kNone: + return false; + case PinningTier::kFlushedAndSimilar: + return maybe_flushed; + case PinningTier::kAll: + return true; + }; + + // In GCC, this is needed to suppress `control reaches end of non-void + // function [-Werror=return-type]`. + assert(false); + return false; + }; + const bool pin_top_level_index = is_pinned( + table_options.metadata_cache_options.top_level_index_pinning, + table_options.pin_top_level_index_and_filter ? PinningTier::kAll + : PinningTier::kNone); + const bool pin_partition = + is_pinned(table_options.metadata_cache_options.partition_pinning, + table_options.pin_l0_filter_and_index_blocks_in_cache + ? PinningTier::kFlushedAndSimilar + : PinningTier::kNone); + const bool pin_unpartitioned = + is_pinned(table_options.metadata_cache_options.unpartitioned_pinning, + table_options.pin_l0_filter_and_index_blocks_in_cache + ? PinningTier::kFlushedAndSimilar + : PinningTier::kNone); + + // pin the first level of index + const bool pin_index = + index_type == BlockBasedTableOptions::kTwoLevelIndexSearch + ? pin_top_level_index + : pin_unpartitioned; + // prefetch the first level of index + // WART: this might be redundant (unnecessary cache hit) if !pin_index, + // depending on prepopulate_block_cache option + const bool prefetch_index = prefetch_all || pin_index; + + std::unique_ptr index_reader; + s = new_table->CreateIndexReader(ro, prefetch_buffer, meta_iter, use_cache, + prefetch_index, pin_index, lookup_context, + &index_reader); + if (!s.ok()) { + return s; + } + + rep_->index_reader = std::move(index_reader); + + // The partitions of partitioned index are always stored in cache. They + // are hence follow the configuration for pin and prefetch regardless of + // the value of cache_index_and_filter_blocks + if (prefetch_all || pin_partition) { + s = rep_->index_reader->CacheDependencies(ro, pin_partition, + prefetch_buffer); + } + if (!s.ok()) { + return s; + } + + // pin the first level of filter + const bool pin_filter = + rep_->filter_type == Rep::FilterType::kPartitionedFilter + ? pin_top_level_index + : pin_unpartitioned; + // prefetch the first level of filter + // WART: this might be redundant (unnecessary cache hit) if !pin_filter, + // depending on prepopulate_block_cache option + const bool prefetch_filter = prefetch_all || pin_filter; + + if (rep_->filter_policy) { + auto filter = new_table->CreateFilterBlockReader( + ro, prefetch_buffer, use_cache, prefetch_filter, pin_filter, + lookup_context); + + if (filter) { + // Refer to the comment above about paritioned indexes always being cached + if (prefetch_all || pin_partition) { + s = filter->CacheDependencies(ro, pin_partition, prefetch_buffer); + if (!s.ok()) { + return s; + } + } + rep_->filter = std::move(filter); + } + } + + if (!rep_->compression_dict_handle.IsNull()) { + std::unique_ptr uncompression_dict_reader; + s = UncompressionDictReader::Create( + this, ro, prefetch_buffer, use_cache, prefetch_all || pin_unpartitioned, + pin_unpartitioned, lookup_context, &uncompression_dict_reader); + if (!s.ok()) { + return s; + } + + rep_->uncompression_dict_reader = std::move(uncompression_dict_reader); + } + + assert(s.ok()); + return s; +} + +void BlockBasedTable::SetupForCompaction() {} + +std::shared_ptr BlockBasedTable::GetTableProperties() + const { + return rep_->table_properties; +} + +size_t BlockBasedTable::ApproximateMemoryUsage() const { + size_t usage = 0; + if (rep_) { + usage += rep_->ApproximateMemoryUsage(); + } else { + return usage; + } + if (rep_->filter) { + usage += rep_->filter->ApproximateMemoryUsage(); + } + if (rep_->index_reader) { + usage += rep_->index_reader->ApproximateMemoryUsage(); + } + if (rep_->uncompression_dict_reader) { + usage += rep_->uncompression_dict_reader->ApproximateMemoryUsage(); + } + if (rep_->table_properties) { + usage += rep_->table_properties->ApproximateMemoryUsage(); + } + return usage; +} + +// Load the meta-index-block from the file. On success, return the loaded +// metaindex +// block and its iterator. +Status BlockBasedTable::ReadMetaIndexBlock( + const ReadOptions& ro, FilePrefetchBuffer* prefetch_buffer, + std::unique_ptr* metaindex_block, + std::unique_ptr* iter) { + // TODO(sanjay): Skip this if footer.metaindex_handle() size indicates + // it is an empty block. + std::unique_ptr metaindex; + Status s = ReadAndParseBlockFromFile( + rep_->file.get(), prefetch_buffer, rep_->footer, ro, + rep_->footer.metaindex_handle(), &metaindex, rep_->ioptions, + rep_->create_context, true /*maybe_compressed*/, + UncompressionDict::GetEmptyDict(), rep_->persistent_cache_options, + GetMemoryAllocator(rep_->table_options), false /* for_compaction */, + false /* async_read */); + + if (!s.ok()) { + ROCKS_LOG_ERROR(rep_->ioptions.logger, + "Encountered error while reading data from properties" + " block %s", + s.ToString().c_str()); + return s; + } + + *metaindex_block = std::move(metaindex); + // meta block uses bytewise comparator. + iter->reset(metaindex_block->get()->NewMetaIterator()); + return Status::OK(); +} + +template +Cache::Priority BlockBasedTable::GetCachePriority() const { + // Here we treat the legacy name "...index_and_filter_blocks..." to mean all + // metadata blocks that might go into block cache, EXCEPT only those needed + // for the read path (Get, etc.). TableProperties should not be needed on the + // read path (prefix extractor setting is an O(1) size special case that we + // are working not to require from TableProperties), so it is not given + // high-priority treatment if it should go into BlockCache. + if constexpr (TBlocklike::kBlockType == BlockType::kData || + TBlocklike::kBlockType == BlockType::kProperties) { + return Cache::Priority::LOW; + } else if (rep_->table_options + .cache_index_and_filter_blocks_with_high_priority) { + return Cache::Priority::HIGH; + } else { + return Cache::Priority::LOW; + } +} + +template +WithBlocklikeCheck BlockBasedTable::GetDataBlockFromCache( + const Slice& cache_key, BlockCacheInterface block_cache, + CachableEntry* out_parsed_block, + GetContext* get_context) const { + assert(out_parsed_block); + assert(out_parsed_block->IsEmpty()); + + Status s; + Statistics* statistics = rep_->ioptions.statistics.get(); + + // Lookup uncompressed cache first + if (block_cache) { + assert(!cache_key.empty()); + auto cache_handle = block_cache.LookupFull( + cache_key, &rep_->create_context, GetCachePriority(), + statistics, rep_->ioptions.lowest_used_cache_tier); + + // Avoid updating metrics here if the handle is not complete yet. This + // happens with MultiGet and secondary cache. So update the metrics only + // if its a miss, or a hit and value is ready + if (!cache_handle) { + UpdateCacheMissMetrics(TBlocklike::kBlockType, get_context); + } else { + TBlocklike* value = block_cache.Value(cache_handle); + if (value) { + UpdateCacheHitMetrics(TBlocklike::kBlockType, get_context, + block_cache.get()->GetUsage(cache_handle)); + } + out_parsed_block->SetCachedValue(value, block_cache.get(), cache_handle); + return s; + } + } + + // If not found, search from the compressed block cache. + assert(out_parsed_block->IsEmpty()); + + return s; +} + +template +WithBlocklikeCheck BlockBasedTable::PutDataBlockToCache( + const Slice& cache_key, BlockCacheInterface block_cache, + CachableEntry* out_parsed_block, BlockContents&& block_contents, + CompressionType block_comp_type, + const UncompressionDict& uncompression_dict, + MemoryAllocator* memory_allocator, GetContext* get_context) const { + const ImmutableOptions& ioptions = rep_->ioptions; + const uint32_t format_version = rep_->table_options.format_version; + assert(out_parsed_block); + assert(out_parsed_block->IsEmpty()); + + Status s; + Statistics* statistics = ioptions.stats; + + std::unique_ptr block_holder; + if (block_comp_type != kNoCompression) { + // Retrieve the uncompressed contents into a new buffer + BlockContents uncompressed_block_contents; + UncompressionContext context(block_comp_type); + UncompressionInfo info(context, uncompression_dict, block_comp_type); + s = UncompressBlockData(info, block_contents.data.data(), + block_contents.data.size(), + &uncompressed_block_contents, format_version, + ioptions, memory_allocator); + if (!s.ok()) { + return s; + } + rep_->create_context.Create(&block_holder, + std::move(uncompressed_block_contents)); + } else { + rep_->create_context.Create(&block_holder, std::move(block_contents)); + } + + // insert into uncompressed block cache + if (block_cache && block_holder->own_bytes()) { + size_t charge = block_holder->ApproximateMemoryUsage(); + BlockCacheTypedHandle* cache_handle = nullptr; + s = block_cache.InsertFull(cache_key, block_holder.get(), charge, + &cache_handle, GetCachePriority(), + rep_->ioptions.lowest_used_cache_tier); + + if (s.ok()) { + assert(cache_handle != nullptr); + out_parsed_block->SetCachedValue(block_holder.release(), + block_cache.get(), cache_handle); + + UpdateCacheInsertionMetrics(TBlocklike::kBlockType, get_context, charge, + s.IsOkOverwritten(), rep_->ioptions.stats); + } else { + RecordTick(statistics, BLOCK_CACHE_ADD_FAILURES); + } + } else { + out_parsed_block->SetOwnedValue(std::move(block_holder)); + } + + return s; +} + +std::unique_ptr BlockBasedTable::CreateFilterBlockReader( + const ReadOptions& ro, FilePrefetchBuffer* prefetch_buffer, bool use_cache, + bool prefetch, bool pin, BlockCacheLookupContext* lookup_context) { + auto& rep = rep_; + auto filter_type = rep->filter_type; + if (filter_type == Rep::FilterType::kNoFilter) { + return std::unique_ptr(); + } + + assert(rep->filter_policy); + + switch (filter_type) { + case Rep::FilterType::kPartitionedFilter: + return PartitionedFilterBlockReader::Create( + this, ro, prefetch_buffer, use_cache, prefetch, pin, lookup_context); + + case Rep::FilterType::kFullFilter: + return FullFilterBlockReader::Create(this, ro, prefetch_buffer, use_cache, + prefetch, pin, lookup_context); + + default: + // filter_type is either kNoFilter (exited the function at the first if), + // or it must be covered in this switch block + assert(false); + return std::unique_ptr(); + } +} + +// disable_prefix_seek should be set to true when prefix_extractor found in SST +// differs from the one in mutable_cf_options and index type is HashBasedIndex +InternalIteratorBase* BlockBasedTable::NewIndexIterator( + const ReadOptions& read_options, bool disable_prefix_seek, + IndexBlockIter* input_iter, GetContext* get_context, + BlockCacheLookupContext* lookup_context) const { + assert(rep_ != nullptr); + assert(rep_->index_reader != nullptr); + + // We don't return pinned data from index blocks, so no need + // to set `block_contents_pinned`. + return rep_->index_reader->NewIterator(read_options, disable_prefix_seek, + input_iter, get_context, + lookup_context); +} + +// TODO? +template <> +DataBlockIter* BlockBasedTable::InitBlockIterator( + const Rep* rep, Block* block, BlockType block_type, + DataBlockIter* input_iter, bool block_contents_pinned) { + return block->NewDataIterator(rep->internal_comparator.user_comparator(), + rep->get_global_seqno(block_type), input_iter, + rep->ioptions.stats, block_contents_pinned, + rep->user_defined_timestamps_persisted); +} + +// TODO? +template <> +IndexBlockIter* BlockBasedTable::InitBlockIterator( + const Rep* rep, Block* block, BlockType block_type, + IndexBlockIter* input_iter, bool block_contents_pinned) { + return block->NewIndexIterator( + rep->internal_comparator.user_comparator(), + rep->get_global_seqno(block_type), input_iter, rep->ioptions.stats, + /* total_order_seek */ true, rep->index_has_first_key, + rep->index_key_includes_seq, rep->index_value_is_full, + block_contents_pinned, rep->user_defined_timestamps_persisted); +} + +// If contents is nullptr, this function looks up the block caches for the +// data block referenced by handle, and read the block from disk if necessary. +// If contents is non-null, it skips the cache lookup and disk read, since +// the caller has already read it. In both cases, if ro.fill_cache is true, +// it inserts the block into the block cache. +template +WithBlocklikeCheck +BlockBasedTable::MaybeReadBlockAndLoadToCache( + FilePrefetchBuffer* prefetch_buffer, const ReadOptions& ro, + const BlockHandle& handle, const UncompressionDict& uncompression_dict, + bool for_compaction, CachableEntry* out_parsed_block, + GetContext* get_context, BlockCacheLookupContext* lookup_context, + BlockContents* contents, bool async_read) const { + assert(out_parsed_block != nullptr); + const bool no_io = (ro.read_tier == kBlockCacheTier); + BlockCacheInterface block_cache{ + rep_->table_options.block_cache.get()}; + + // First, try to get the block from the cache + // + // If either block cache is enabled, we'll try to read from it. + Status s; + CacheKey key_data; + Slice key; + bool is_cache_hit = false; + if (block_cache) { + // create key for block cache + key_data = GetCacheKey(rep_->base_cache_key, handle); + key = key_data.AsSlice(); + + if (!contents) { + s = GetDataBlockFromCache(key, block_cache, out_parsed_block, + get_context); + // Value could still be null at this point, so check the cache handle + // and update the read pattern for prefetching + if (out_parsed_block->GetValue() || out_parsed_block->GetCacheHandle()) { + // TODO(haoyu): Differentiate cache hit on uncompressed block cache and + // compressed block cache. + is_cache_hit = true; + if (prefetch_buffer) { + // Update the block details so that PrefetchBuffer can use the read + // pattern to determine if reads are sequential or not for + // prefetching. It should also take in account blocks read from cache. + prefetch_buffer->UpdateReadPattern( + handle.offset(), BlockSizeWithTrailer(handle), + ro.adaptive_readahead /*decrease_readahead_size*/); + } + } + } + + // Can't find the block from the cache. If I/O is allowed, read from the + // file. + if (out_parsed_block->GetValue() == nullptr && + out_parsed_block->GetCacheHandle() == nullptr && !no_io && + ro.fill_cache) { + Statistics* statistics = rep_->ioptions.stats; + const bool maybe_compressed = + TBlocklike::kBlockType != BlockType::kFilter && + TBlocklike::kBlockType != BlockType::kCompressionDictionary && + rep_->blocks_maybe_compressed; + const bool do_uncompress = maybe_compressed; + CompressionType contents_comp_type; + // Maybe serialized or uncompressed + BlockContents tmp_contents; + if (!contents) { + Histograms histogram = for_compaction ? READ_BLOCK_COMPACTION_MICROS + : READ_BLOCK_GET_MICROS; + StopWatch sw(rep_->ioptions.clock, statistics, histogram); + BlockFetcher block_fetcher( + rep_->file.get(), prefetch_buffer, rep_->footer, ro, handle, + &tmp_contents, rep_->ioptions, do_uncompress, maybe_compressed, + TBlocklike::kBlockType, uncompression_dict, + rep_->persistent_cache_options, + GetMemoryAllocator(rep_->table_options), + /*allocator=*/nullptr); + + // If prefetch_buffer is not allocated, it will fallback to synchronous + // reading of block contents. + if (async_read && prefetch_buffer != nullptr) { + s = block_fetcher.ReadAsyncBlockContents(); + if (!s.ok()) { + return s; + } + } else { + s = block_fetcher.ReadBlockContents(); + } + + contents_comp_type = block_fetcher.get_compression_type(); + contents = &tmp_contents; + if (get_context) { + switch (TBlocklike::kBlockType) { + case BlockType::kIndex: + ++get_context->get_context_stats_.num_index_read; + break; + case BlockType::kFilter: + case BlockType::kFilterPartitionIndex: + ++get_context->get_context_stats_.num_filter_read; + break; + default: + break; + } + } + } else { + contents_comp_type = GetBlockCompressionType(*contents); + } + + if (s.ok()) { + // If filling cache is allowed and a cache is configured, try to put the + // block to the cache. + s = PutDataBlockToCache( + key, block_cache, out_parsed_block, std::move(*contents), + contents_comp_type, uncompression_dict, + GetMemoryAllocator(rep_->table_options), get_context); + } + } + } + + // TODO: optimize so that lookup_context != nullptr implies the others + if (block_cache_tracer_ && block_cache_tracer_->is_tracing_enabled() && + lookup_context) { + SaveLookupContextOrTraceRecord( + key, is_cache_hit, ro, out_parsed_block->GetValue(), lookup_context); + } + + assert(s.ok() || out_parsed_block->GetValue() == nullptr); + return s; +} + +template +WithBlocklikeCheck +BlockBasedTable::SaveLookupContextOrTraceRecord( + const Slice& block_key, bool is_cache_hit, const ReadOptions& ro, + const TBlocklike* parsed_block_value, + BlockCacheLookupContext* lookup_context) const { + assert(lookup_context); + size_t usage = 0; + uint64_t nkeys = 0; + if (parsed_block_value) { + // Approximate the number of keys in the block using restarts. + int interval = rep_->table_options.block_restart_interval; + nkeys = interval * GetBlockNumRestarts(*parsed_block_value); + // On average, the last restart should be just over half utilized. + // Specifically, 1..N should be N/2 + 0.5. For example, 7 -> 4, 8 -> 4.5. + // Use the get_id to alternate between rounding up vs. down. + if (nkeys > 0) { + bool rounding = static_cast(lookup_context->get_id) & 1; + nkeys -= (interval - rounding) / 2; + } + usage = parsed_block_value->ApproximateMemoryUsage(); + } + TraceType trace_block_type = TraceType::kTraceMax; + switch (TBlocklike::kBlockType) { + case BlockType::kData: + trace_block_type = TraceType::kBlockTraceDataBlock; + break; + case BlockType::kFilter: + case BlockType::kFilterPartitionIndex: + trace_block_type = TraceType::kBlockTraceFilterBlock; + break; + case BlockType::kCompressionDictionary: + trace_block_type = TraceType::kBlockTraceUncompressionDictBlock; + break; + case BlockType::kRangeDeletion: + trace_block_type = TraceType::kBlockTraceRangeDeletionBlock; + break; + case BlockType::kIndex: + trace_block_type = TraceType::kBlockTraceIndexBlock; + break; + default: + // This cannot happen. + assert(false); + break; + } + const bool no_io = ro.read_tier == kBlockCacheTier; + bool no_insert = no_io || !ro.fill_cache; + if (BlockCacheTraceHelper::IsGetOrMultiGetOnDataBlock( + trace_block_type, lookup_context->caller)) { + // Make a copy of the block key here since it will be logged later. + lookup_context->FillLookupContext(is_cache_hit, no_insert, trace_block_type, + /*block_size=*/usage, + block_key.ToString(), nkeys); + + // Defer logging the access to Get() and MultiGet() to trace additional + // information, e.g., referenced_key + } else { + // Avoid making copy of block_key if it doesn't need to be saved in + // BlockCacheLookupContext + lookup_context->FillLookupContext(is_cache_hit, no_insert, trace_block_type, + /*block_size=*/usage, + /*block_key=*/{}, nkeys); + + // Fill in default values for irrelevant/unknown fields + FinishTraceRecord(*lookup_context, block_key, + lookup_context->referenced_key, + /*does_referenced_key_exist*/ false, + /*referenced_data_size*/ 0); + } +} + +void BlockBasedTable::FinishTraceRecord( + const BlockCacheLookupContext& lookup_context, const Slice& block_key, + const Slice& referenced_key, bool does_referenced_key_exist, + uint64_t referenced_data_size) const { + // Avoid making copy of referenced_key if it doesn't need to be saved in + // BlockCacheLookupContext + BlockCacheTraceRecord access_record( + rep_->ioptions.clock->NowMicros(), + /*block_key=*/"", lookup_context.block_type, lookup_context.block_size, + rep_->cf_id_for_tracing(), + /*cf_name=*/"", rep_->level_for_tracing(), rep_->sst_number_for_tracing(), + lookup_context.caller, lookup_context.is_cache_hit, + lookup_context.no_insert, lookup_context.get_id, + lookup_context.get_from_user_specified_snapshot, + /*referenced_key=*/"", referenced_data_size, + lookup_context.num_keys_in_block, does_referenced_key_exist); + // TODO: Should handle status here? + block_cache_tracer_ + ->WriteBlockAccess(access_record, block_key, rep_->cf_name_for_tracing(), + referenced_key) + .PermitUncheckedError(); +} + +template +WithBlocklikeCheck BlockBasedTable::RetrieveBlock( + FilePrefetchBuffer* prefetch_buffer, const ReadOptions& ro, + const BlockHandle& handle, const UncompressionDict& uncompression_dict, + CachableEntry* out_parsed_block, GetContext* get_context, + BlockCacheLookupContext* lookup_context, bool for_compaction, + bool use_cache, bool async_read) const { + assert(out_parsed_block); + assert(out_parsed_block->IsEmpty()); + + Status s; + if (use_cache) { + s = MaybeReadBlockAndLoadToCache( + prefetch_buffer, ro, handle, uncompression_dict, for_compaction, + out_parsed_block, get_context, lookup_context, + /*contents=*/nullptr, async_read); + + if (!s.ok()) { + return s; + } + + if (out_parsed_block->GetValue() != nullptr || + out_parsed_block->GetCacheHandle() != nullptr) { + assert(s.ok()); + return s; + } + } + + assert(out_parsed_block->IsEmpty()); + + const bool no_io = ro.read_tier == kBlockCacheTier; + if (no_io) { + return Status::Incomplete("no blocking io"); + } + + const bool maybe_compressed = + TBlocklike::kBlockType != BlockType::kFilter && + TBlocklike::kBlockType != BlockType::kCompressionDictionary && + rep_->blocks_maybe_compressed; + std::unique_ptr block; + + { + Histograms histogram = + for_compaction ? READ_BLOCK_COMPACTION_MICROS : READ_BLOCK_GET_MICROS; + StopWatch sw(rep_->ioptions.clock, rep_->ioptions.stats, histogram); + s = ReadAndParseBlockFromFile( + rep_->file.get(), prefetch_buffer, rep_->footer, ro, handle, &block, + rep_->ioptions, rep_->create_context, maybe_compressed, + uncompression_dict, rep_->persistent_cache_options, + GetMemoryAllocator(rep_->table_options), for_compaction, async_read); + + if (get_context) { + switch (TBlocklike::kBlockType) { + case BlockType::kIndex: + ++(get_context->get_context_stats_.num_index_read); + break; + case BlockType::kFilter: + case BlockType::kFilterPartitionIndex: + ++(get_context->get_context_stats_.num_filter_read); + break; + default: + break; + } + } + } + + if (!s.ok()) { + return s; + } + + out_parsed_block->SetOwnedValue(std::move(block)); + + assert(s.ok()); + return s; +} + +BlockBasedTable::PartitionedIndexIteratorState::PartitionedIndexIteratorState( + const BlockBasedTable* table, + UnorderedMap>* block_map) + : table_(table), block_map_(block_map) {} + +InternalIteratorBase* +BlockBasedTable::PartitionedIndexIteratorState::NewSecondaryIterator( + const BlockHandle& handle) { + // Return a block iterator on the index partition + auto block = block_map_->find(handle.offset()); + // block_map_ must be exhaustive + if (block == block_map_->end()) { + assert(false); + // Signal problem to caller + return nullptr; + } + const Rep* rep = table_->get_rep(); + assert(rep); + + Statistics* kNullStats = nullptr; + // We don't return pinned data from index blocks, so no need + // to set `block_contents_pinned`. + return block->second.GetValue()->NewIndexIterator( + rep->internal_comparator.user_comparator(), + rep->get_global_seqno(BlockType::kIndex), nullptr, kNullStats, true, + rep->index_has_first_key, rep->index_key_includes_seq, + rep->index_value_is_full); +} + +// This will be broken if the user specifies an unusual implementation +// of Options.comparator, or if the user specifies an unusual +// definition of prefixes in BlockBasedTableOptions.filter_policy. +// In particular, we require the following three properties: +// +// 1) key.starts_with(prefix(key)) +// 2) Compare(prefix(key), key) <= 0. +// 3) If Compare(key1, key2) <= 0, then Compare(prefix(key1), prefix(key2)) <= 0 +// +// If read_options.read_tier == kBlockCacheTier, this method will do no I/O and +// will return true if the filter block is not in memory and not found in block +// cache. +// +// REQUIRES: this method shouldn't be called while the DB lock is held. +bool BlockBasedTable::PrefixRangeMayMatch( + const Slice& internal_key, const ReadOptions& read_options, + const SliceTransform* options_prefix_extractor, + const bool need_upper_bound_check, BlockCacheLookupContext* lookup_context, + bool* filter_checked) const { + if (!rep_->filter_policy) { + return true; + } + + const SliceTransform* prefix_extractor; + + if (rep_->table_prefix_extractor == nullptr) { + if (need_upper_bound_check) { + return true; + } + prefix_extractor = options_prefix_extractor; + } else { + prefix_extractor = rep_->table_prefix_extractor.get(); + } + auto ts_sz = rep_->internal_comparator.user_comparator()->timestamp_size(); + auto user_key_without_ts = + ExtractUserKeyAndStripTimestamp(internal_key, ts_sz); + if (!prefix_extractor->InDomain(user_key_without_ts)) { + return true; + } + + bool may_match = true; + + FilterBlockReader* const filter = rep_->filter.get(); + *filter_checked = false; + if (filter != nullptr) { + const bool no_io = read_options.read_tier == kBlockCacheTier; + + const Slice* const const_ikey_ptr = &internal_key; + may_match = filter->RangeMayExist( + read_options.iterate_upper_bound, user_key_without_ts, prefix_extractor, + rep_->internal_comparator.user_comparator(), const_ikey_ptr, + filter_checked, need_upper_bound_check, no_io, lookup_context, + read_options); + } + + return may_match; +} + +bool BlockBasedTable::PrefixExtractorChanged( + const SliceTransform* prefix_extractor) const { + if (prefix_extractor == nullptr) { + return true; + } else if (prefix_extractor == rep_->table_prefix_extractor.get()) { + return false; + } else { + return PrefixExtractorChangedHelper(rep_->table_properties.get(), + prefix_extractor); + } +} + +Statistics* BlockBasedTable::GetStatistics() const { + return rep_->ioptions.stats; +} +bool BlockBasedTable::IsLastLevel() const { + return rep_->level == rep_->ioptions.num_levels - 1; +} + +InternalIterator* BlockBasedTable::NewIterator( + const ReadOptions& read_options, const SliceTransform* prefix_extractor, + Arena* arena, bool skip_filters, TableReaderCaller caller, + size_t compaction_readahead_size, bool allow_unprepared_value) { + BlockCacheLookupContext lookup_context{caller}; + bool need_upper_bound_check = + read_options.auto_prefix_mode || PrefixExtractorChanged(prefix_extractor); + std::unique_ptr> index_iter(NewIndexIterator( + read_options, + /*disable_prefix_seek=*/need_upper_bound_check && + rep_->index_type == BlockBasedTableOptions::kHashSearch, + /*input_iter=*/nullptr, /*get_context=*/nullptr, &lookup_context)); + if (arena == nullptr) { + return new BlockBasedTableIterator( + this, read_options, rep_->internal_comparator, std::move(index_iter), + !skip_filters && !read_options.total_order_seek && + prefix_extractor != nullptr, + need_upper_bound_check, prefix_extractor, caller, + compaction_readahead_size, allow_unprepared_value); + } else { + auto* mem = arena->AllocateAligned(sizeof(BlockBasedTableIterator)); + return new (mem) BlockBasedTableIterator( + this, read_options, rep_->internal_comparator, std::move(index_iter), + !skip_filters && !read_options.total_order_seek && + prefix_extractor != nullptr, + need_upper_bound_check, prefix_extractor, caller, + compaction_readahead_size, allow_unprepared_value); + } +} + +FragmentedRangeTombstoneIterator* BlockBasedTable::NewRangeTombstoneIterator( + const ReadOptions& read_options) { + if (rep_->fragmented_range_dels == nullptr) { + return nullptr; + } + SequenceNumber snapshot = kMaxSequenceNumber; + if (read_options.snapshot != nullptr) { + snapshot = read_options.snapshot->GetSequenceNumber(); + } + return new FragmentedRangeTombstoneIterator(rep_->fragmented_range_dels, + rep_->internal_comparator, + snapshot, read_options.timestamp); +} + +bool BlockBasedTable::FullFilterKeyMayMatch( + FilterBlockReader* filter, const Slice& internal_key, const bool no_io, + const SliceTransform* prefix_extractor, GetContext* get_context, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) const { + if (filter == nullptr) { + return true; + } + Slice user_key = ExtractUserKey(internal_key); + const Slice* const const_ikey_ptr = &internal_key; + bool may_match = true; + size_t ts_sz = rep_->internal_comparator.user_comparator()->timestamp_size(); + Slice user_key_without_ts = StripTimestampFromUserKey(user_key, ts_sz); + if (rep_->whole_key_filtering) { + may_match = filter->KeyMayMatch(user_key_without_ts, no_io, const_ikey_ptr, + get_context, lookup_context, read_options); + if (may_match) { + RecordTick(rep_->ioptions.stats, BLOOM_FILTER_FULL_POSITIVE); + PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_full_positive, 1, rep_->level); + } else { + RecordTick(rep_->ioptions.stats, BLOOM_FILTER_USEFUL); + PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_useful, 1, rep_->level); + } + } else if (!PrefixExtractorChanged(prefix_extractor) && + prefix_extractor->InDomain(user_key_without_ts)) { + // FIXME ^^^: there should be no reason for Get() to depend on current + // prefix_extractor at all. It should always use table_prefix_extractor. + may_match = filter->PrefixMayMatch( + prefix_extractor->Transform(user_key_without_ts), no_io, const_ikey_ptr, + get_context, lookup_context, read_options); + RecordTick(rep_->ioptions.stats, BLOOM_FILTER_PREFIX_CHECKED); + if (may_match) { + // Includes prefix stats + PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_full_positive, 1, rep_->level); + } else { + RecordTick(rep_->ioptions.stats, BLOOM_FILTER_PREFIX_USEFUL); + // Includes prefix stats + PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_useful, 1, rep_->level); + } + } + return may_match; +} + +void BlockBasedTable::FullFilterKeysMayMatch( + FilterBlockReader* filter, MultiGetRange* range, const bool no_io, + const SliceTransform* prefix_extractor, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) const { + if (filter == nullptr) { + return; + } + uint64_t before_keys = range->KeysLeft(); + assert(before_keys > 0); // Caller should ensure + if (rep_->whole_key_filtering) { + filter->KeysMayMatch(range, no_io, lookup_context, read_options); + uint64_t after_keys = range->KeysLeft(); + if (after_keys) { + RecordTick(rep_->ioptions.stats, BLOOM_FILTER_FULL_POSITIVE, after_keys); + PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_full_positive, after_keys, + rep_->level); + } + uint64_t filtered_keys = before_keys - after_keys; + if (filtered_keys) { + RecordTick(rep_->ioptions.stats, BLOOM_FILTER_USEFUL, filtered_keys); + PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_useful, filtered_keys, + rep_->level); + } + } else if (!PrefixExtractorChanged(prefix_extractor)) { + // FIXME ^^^: there should be no reason for MultiGet() to depend on current + // prefix_extractor at all. It should always use table_prefix_extractor. + filter->PrefixesMayMatch(range, prefix_extractor, false, lookup_context, + read_options); + RecordTick(rep_->ioptions.stats, BLOOM_FILTER_PREFIX_CHECKED, before_keys); + uint64_t after_keys = range->KeysLeft(); + if (after_keys) { + // Includes prefix stats + PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_full_positive, after_keys, + rep_->level); + } + uint64_t filtered_keys = before_keys - after_keys; + if (filtered_keys) { + RecordTick(rep_->ioptions.stats, BLOOM_FILTER_PREFIX_USEFUL, + filtered_keys); + // Includes prefix stats + PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_useful, filtered_keys, + rep_->level); + } + } +} + +Status BlockBasedTable::ApproximateKeyAnchors(const ReadOptions& read_options, + std::vector& anchors) { + // We iterator the whole index block here. More efficient implementation + // is possible if we push this operation into IndexReader. For example, we + // can directly sample from restart block entries in the index block and + // only read keys needed. Here we take a simple solution. Performance is + // likely not to be a problem. We are compacting the whole file, so all + // keys will be read out anyway. An extra read to index block might be + // a small share of the overhead. We can try to optimize if needed. + // + // `CacheDependencies()` brings all the blocks into cache using one I/O. That + // way the full index scan usually finds the index data it is looking for in + // cache rather than doing an I/O for each "dependency" (partition). + Status s = rep_->index_reader->CacheDependencies( + read_options, false /* pin */, nullptr /* prefetch_buffer */); + if (!s.ok()) { + return s; + } + + IndexBlockIter iiter_on_stack; + auto iiter = NewIndexIterator( + read_options, /*disable_prefix_seek=*/false, &iiter_on_stack, + /*get_context=*/nullptr, /*lookup_context=*/nullptr); + std::unique_ptr> iiter_unique_ptr; + if (iiter != &iiter_on_stack) { + iiter_unique_ptr.reset(iiter); + } + + // If needed the threshold could be more adaptive. For example, it can be + // based on size, so that a larger will be sampled to more partitions than a + // smaller file. The size might also need to be passed in by the caller based + // on total compaction size. + const uint64_t kMaxNumAnchors = uint64_t{128}; + uint64_t num_blocks = this->GetTableProperties()->num_data_blocks; + uint64_t num_blocks_per_anchor = num_blocks / kMaxNumAnchors; + if (num_blocks_per_anchor == 0) { + num_blocks_per_anchor = 1; + } + + uint64_t count = 0; + std::string last_key; + uint64_t range_size = 0; + uint64_t prev_offset = 0; + for (iiter->SeekToFirst(); iiter->Valid(); iiter->Next()) { + const BlockHandle& bh = iiter->value().handle; + range_size += bh.offset() + bh.size() - prev_offset; + prev_offset = bh.offset() + bh.size(); + if (++count % num_blocks_per_anchor == 0) { + count = 0; + anchors.emplace_back(iiter->user_key(), range_size); + range_size = 0; + } else { + last_key = iiter->user_key().ToString(); + } + } + if (count != 0) { + anchors.emplace_back(last_key, range_size); + } + return Status::OK(); +} + +bool BlockBasedTable::TimestampMayMatch(const ReadOptions& read_options) const { + if (read_options.timestamp != nullptr && !rep_->min_timestamp.empty()) { + RecordTick(rep_->ioptions.stats, TIMESTAMP_FILTER_TABLE_CHECKED); + auto read_ts = read_options.timestamp; + auto comparator = rep_->internal_comparator.user_comparator(); + if (comparator->CompareTimestamp(*read_ts, rep_->min_timestamp) < 0) { + RecordTick(rep_->ioptions.stats, TIMESTAMP_FILTER_TABLE_FILTERED); + return false; + } + } + return true; +} + +Status BlockBasedTable::Get(const ReadOptions& read_options, const Slice& key, + GetContext* get_context, + const SliceTransform* prefix_extractor, + bool skip_filters) { + // Similar to Bloom filter !may_match + // If timestamp is beyond the range of the table, skip + if (!TimestampMayMatch(read_options)) { + return Status::OK(); + } + assert(key.size() >= 8); // key must be internal key + assert(get_context != nullptr); + Status s; + const bool no_io = read_options.read_tier == kBlockCacheTier; + + FilterBlockReader* const filter = + !skip_filters ? rep_->filter.get() : nullptr; + + // First check the full filter + // If full filter not useful, Then go into each block + uint64_t tracing_get_id = get_context->get_tracing_get_id(); + BlockCacheLookupContext lookup_context{ + TableReaderCaller::kUserGet, tracing_get_id, + /*get_from_user_specified_snapshot=*/read_options.snapshot != nullptr}; + if (block_cache_tracer_ && block_cache_tracer_->is_tracing_enabled()) { + // Trace the key since it contains both user key and sequence number. + lookup_context.referenced_key = key.ToString(); + lookup_context.get_from_user_specified_snapshot = + read_options.snapshot != nullptr; + } + TEST_SYNC_POINT("BlockBasedTable::Get:BeforeFilterMatch"); + const bool may_match = + FullFilterKeyMayMatch(filter, key, no_io, prefix_extractor, get_context, + &lookup_context, read_options); + TEST_SYNC_POINT("BlockBasedTable::Get:AfterFilterMatch"); + if (may_match) { + IndexBlockIter iiter_on_stack; + // if prefix_extractor found in block differs from options, disable + // BlockPrefixIndex. Only do this check when index_type is kHashSearch. + bool need_upper_bound_check = false; + if (rep_->index_type == BlockBasedTableOptions::kHashSearch) { + need_upper_bound_check = PrefixExtractorChanged(prefix_extractor); + } + auto iiter = + NewIndexIterator(read_options, need_upper_bound_check, &iiter_on_stack, + get_context, &lookup_context); + std::unique_ptr> iiter_unique_ptr; + if (iiter != &iiter_on_stack) { + iiter_unique_ptr.reset(iiter); + } + + size_t ts_sz = + rep_->internal_comparator.user_comparator()->timestamp_size(); + bool matched = false; // if such user key matched a key in SST + bool done = false; + for (iiter->Seek(key); iiter->Valid() && !done; iiter->Next()) { + IndexValue v = iiter->value(); + + if (!v.first_internal_key.empty() && !skip_filters && + UserComparatorWrapper(rep_->internal_comparator.user_comparator()) + .CompareWithoutTimestamp( + ExtractUserKey(key), + ExtractUserKey(v.first_internal_key)) < 0) { + // The requested key falls between highest key in previous block and + // lowest key in current block. + break; + } + + BlockCacheLookupContext lookup_data_block_context{ + TableReaderCaller::kUserGet, tracing_get_id, + /*get_from_user_specified_snapshot=*/read_options.snapshot != + nullptr}; + bool does_referenced_key_exist = false; + DataBlockIter biter; + uint64_t referenced_data_size = 0; + Status tmp_status; + NewDataBlockIterator( + read_options, v.handle, &biter, BlockType::kData, get_context, + &lookup_data_block_context, /*prefetch_buffer=*/nullptr, + /*for_compaction=*/false, /*async_read=*/false, tmp_status); + + if (no_io && biter.status().IsIncomplete()) { + // couldn't get block from block_cache + // Update Saver.state to Found because we are only looking for + // whether we can guarantee the key is not there when "no_io" is set + get_context->MarkKeyMayExist(); + s = biter.status(); + break; + } + if (!biter.status().ok()) { + s = biter.status(); + break; + } + + bool may_exist = biter.SeekForGet(key); + // If user-specified timestamp is supported, we cannot end the search + // just because hash index lookup indicates the key+ts does not exist. + if (!may_exist && ts_sz == 0) { + // HashSeek cannot find the key this block and the the iter is not + // the end of the block, i.e. cannot be in the following blocks + // either. In this case, the seek_key cannot be found, so we break + // from the top level for-loop. + done = true; + } else { + // Call the *saver function on each entry/block until it returns false + for (; biter.Valid(); biter.Next()) { + ParsedInternalKey parsed_key; + Status pik_status = ParseInternalKey( + biter.key(), &parsed_key, false /* log_err_key */); // TODO + if (!pik_status.ok()) { + s = pik_status; + } + + if (!get_context->SaveValue( + parsed_key, biter.value(), &matched, + biter.IsValuePinned() ? &biter : nullptr)) { + if (get_context->State() == GetContext::GetState::kFound) { + does_referenced_key_exist = true; + referenced_data_size = biter.key().size() + biter.value().size(); + } + done = true; + break; + } + } + s = biter.status(); + if (!s.ok()) { + break; + } + } + // Write the block cache access record. + if (block_cache_tracer_ && block_cache_tracer_->is_tracing_enabled()) { + // Avoid making copy of block_key, cf_name, and referenced_key when + // constructing the access record. + Slice referenced_key; + if (does_referenced_key_exist) { + referenced_key = biter.key(); + } else { + referenced_key = key; + } + FinishTraceRecord(lookup_data_block_context, + lookup_data_block_context.block_key, referenced_key, + does_referenced_key_exist, referenced_data_size); + } + + if (done) { + // Avoid the extra Next which is expensive in two-level indexes + break; + } + } + if (matched && filter != nullptr) { + if (rep_->whole_key_filtering) { + RecordTick(rep_->ioptions.stats, BLOOM_FILTER_FULL_TRUE_POSITIVE); + } else { + RecordTick(rep_->ioptions.stats, BLOOM_FILTER_PREFIX_TRUE_POSITIVE); + } + // Includes prefix stats + PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_full_true_positive, 1, + rep_->level); + } + + if (s.ok() && !iiter->status().IsNotFound()) { + s = iiter->status(); + } + } + + return s; +} + +Status BlockBasedTable::MultiGetFilter(const ReadOptions& read_options, + const SliceTransform* prefix_extractor, + MultiGetRange* mget_range) { + if (mget_range->empty()) { + // Caller should ensure non-empty (performance bug) + assert(false); + return Status::OK(); // Nothing to do + } + + FilterBlockReader* const filter = rep_->filter.get(); + if (!filter) { + return Status::OK(); + } + + // First check the full filter + // If full filter not useful, Then go into each block + const bool no_io = read_options.read_tier == kBlockCacheTier; + uint64_t tracing_mget_id = BlockCacheTraceHelper::kReservedGetId; + if (mget_range->begin()->get_context) { + tracing_mget_id = mget_range->begin()->get_context->get_tracing_get_id(); + } + BlockCacheLookupContext lookup_context{ + TableReaderCaller::kUserMultiGet, tracing_mget_id, + /*_get_from_user_specified_snapshot=*/read_options.snapshot != nullptr}; + FullFilterKeysMayMatch(filter, mget_range, no_io, prefix_extractor, + &lookup_context, read_options); + + return Status::OK(); +} + +Status BlockBasedTable::Prefetch(const ReadOptions& read_options, + const Slice* const begin, + const Slice* const end) { + auto& comparator = rep_->internal_comparator; + UserComparatorWrapper user_comparator(comparator.user_comparator()); + // pre-condition + if (begin && end && comparator.Compare(*begin, *end) > 0) { + return Status::InvalidArgument(*begin, *end); + } + BlockCacheLookupContext lookup_context{TableReaderCaller::kPrefetch}; + IndexBlockIter iiter_on_stack; + auto iiter = NewIndexIterator(read_options, /*need_upper_bound_check=*/false, + &iiter_on_stack, /*get_context=*/nullptr, + &lookup_context); + std::unique_ptr> iiter_unique_ptr; + if (iiter != &iiter_on_stack) { + iiter_unique_ptr = std::unique_ptr>(iiter); + } + + if (!iiter->status().ok()) { + // error opening index iterator + return iiter->status(); + } + + // indicates if we are on the last page that need to be pre-fetched + bool prefetching_boundary_page = false; + + for (begin ? iiter->Seek(*begin) : iiter->SeekToFirst(); iiter->Valid(); + iiter->Next()) { + BlockHandle block_handle = iiter->value().handle; + const bool is_user_key = !rep_->index_key_includes_seq; + if (end && + ((!is_user_key && comparator.Compare(iiter->key(), *end) >= 0) || + (is_user_key && + user_comparator.Compare(iiter->key(), ExtractUserKey(*end)) >= 0))) { + if (prefetching_boundary_page) { + break; + } + + // The index entry represents the last key in the data block. + // We should load this page into memory as well, but no more + prefetching_boundary_page = true; + } + + // Load the block specified by the block_handle into the block cache + DataBlockIter biter; + Status tmp_status; + NewDataBlockIterator( + read_options, block_handle, &biter, /*type=*/BlockType::kData, + /*get_context=*/nullptr, &lookup_context, + /*prefetch_buffer=*/nullptr, /*for_compaction=*/false, + /*async_read=*/false, tmp_status); + + if (!biter.status().ok()) { + // there was an unexpected error while pre-fetching + return biter.status(); + } + } + + return Status::OK(); +} + +Status BlockBasedTable::VerifyChecksum(const ReadOptions& read_options, + TableReaderCaller caller) { + Status s; + // Check Meta blocks + std::unique_ptr metaindex; + std::unique_ptr metaindex_iter; + s = ReadMetaIndexBlock(read_options, nullptr /* prefetch buffer */, + &metaindex, &metaindex_iter); + if (s.ok()) { + s = VerifyChecksumInMetaBlocks(read_options, metaindex_iter.get()); + if (!s.ok()) { + return s; + } + } else { + return s; + } + // Check Data blocks + IndexBlockIter iiter_on_stack; + BlockCacheLookupContext context{caller}; + InternalIteratorBase* iiter = NewIndexIterator( + read_options, /*disable_prefix_seek=*/false, &iiter_on_stack, + /*get_context=*/nullptr, &context); + std::unique_ptr> iiter_unique_ptr; + if (iiter != &iiter_on_stack) { + iiter_unique_ptr = std::unique_ptr>(iiter); + } + if (!iiter->status().ok()) { + // error opening index iterator + return iiter->status(); + } + s = VerifyChecksumInBlocks(read_options, iiter); + return s; +} + +Status BlockBasedTable::VerifyChecksumInBlocks( + const ReadOptions& read_options, + InternalIteratorBase* index_iter) { + Status s; + // We are scanning the whole file, so no need to do exponential + // increasing of the buffer size. + size_t readahead_size = (read_options.readahead_size != 0) + ? read_options.readahead_size + : rep_->table_options.max_auto_readahead_size; + // FilePrefetchBuffer doesn't work in mmap mode and readahead is not + // needed there. + FilePrefetchBuffer prefetch_buffer( + readahead_size /* readahead_size */, + readahead_size /* max_readahead_size */, + !rep_->ioptions.allow_mmap_reads /* enable */); + + for (index_iter->SeekToFirst(); index_iter->Valid(); index_iter->Next()) { + s = index_iter->status(); + if (!s.ok()) { + break; + } + BlockHandle handle = index_iter->value().handle; + BlockContents contents; + BlockFetcher block_fetcher( + rep_->file.get(), &prefetch_buffer, rep_->footer, read_options, handle, + &contents, rep_->ioptions, false /* decompress */, + false /*maybe_compressed*/, BlockType::kData, + UncompressionDict::GetEmptyDict(), rep_->persistent_cache_options); + s = block_fetcher.ReadBlockContents(); + if (!s.ok()) { + break; + } + } + if (s.ok()) { + // In the case of two level indexes, we would have exited the above loop + // by checking index_iter->Valid(), but Valid() might have returned false + // due to an IO error. So check the index_iter status + s = index_iter->status(); + } + return s; +} + +BlockType BlockBasedTable::GetBlockTypeForMetaBlockByName( + const Slice& meta_block_name) { + if (meta_block_name.starts_with(kFullFilterBlockPrefix)) { + return BlockType::kFilter; + } + + if (meta_block_name.starts_with(kPartitionedFilterBlockPrefix)) { + return BlockType::kFilterPartitionIndex; + } + + if (meta_block_name == kPropertiesBlockName) { + return BlockType::kProperties; + } + + if (meta_block_name == kCompressionDictBlockName) { + return BlockType::kCompressionDictionary; + } + + if (meta_block_name == kRangeDelBlockName) { + return BlockType::kRangeDeletion; + } + + if (meta_block_name == kHashIndexPrefixesBlock) { + return BlockType::kHashIndexPrefixes; + } + + if (meta_block_name == kHashIndexPrefixesMetadataBlock) { + return BlockType::kHashIndexMetadata; + } + + if (meta_block_name == kIndexBlockName) { + return BlockType::kIndex; + } + + if (meta_block_name.starts_with(kObsoleteFilterBlockPrefix)) { + // Obsolete but possible in old files + return BlockType::kInvalid; + } + + assert(false); + return BlockType::kInvalid; +} + +Status BlockBasedTable::VerifyChecksumInMetaBlocks( + const ReadOptions& read_options, InternalIteratorBase* index_iter) { + Status s; + for (index_iter->SeekToFirst(); index_iter->Valid(); index_iter->Next()) { + s = index_iter->status(); + if (!s.ok()) { + break; + } + BlockHandle handle; + Slice input = index_iter->value(); + s = handle.DecodeFrom(&input); + if (!s.ok()) { + break; + } + BlockContents contents; + const Slice meta_block_name = index_iter->key(); + if (meta_block_name == kPropertiesBlockName) { + // Unfortunate special handling for properties block checksum w/ + // global seqno + std::unique_ptr table_properties; + s = ReadTablePropertiesHelper(read_options, handle, rep_->file.get(), + nullptr /* prefetch_buffer */, rep_->footer, + rep_->ioptions, &table_properties, + nullptr /* memory_allocator */); + } else if (rep_->verify_checksum_set_on_open && + meta_block_name == kIndexBlockName) { + // WART: For now, to maintain similar I/O behavior as before + // format_version=6, we skip verifying index block checksum--but only + // if it was checked on open. + } else { + // FIXME? Need to verify checksums of index and filter partitions? + s = BlockFetcher( + rep_->file.get(), nullptr /* prefetch buffer */, rep_->footer, + read_options, handle, &contents, rep_->ioptions, + false /* decompress */, false /*maybe_compressed*/, + GetBlockTypeForMetaBlockByName(meta_block_name), + UncompressionDict::GetEmptyDict(), rep_->persistent_cache_options) + .ReadBlockContents(); + } + if (!s.ok()) { + break; + } + } + return s; +} + +bool BlockBasedTable::TEST_BlockInCache(const BlockHandle& handle) const { + assert(rep_ != nullptr); + + Cache* const cache = rep_->table_options.block_cache.get(); + if (cache == nullptr) { + return false; + } + + CacheKey key = GetCacheKey(rep_->base_cache_key, handle); + + Cache::Handle* const cache_handle = cache->Lookup(key.AsSlice()); + if (cache_handle == nullptr) { + return false; + } + + cache->Release(cache_handle); + + return true; +} + +bool BlockBasedTable::TEST_KeyInCache(const ReadOptions& options, + const Slice& key) { + std::unique_ptr> iiter(NewIndexIterator( + options, /*need_upper_bound_check=*/false, /*input_iter=*/nullptr, + /*get_context=*/nullptr, /*lookup_context=*/nullptr)); + iiter->Seek(key); + assert(iiter->Valid()); + + return TEST_BlockInCache(iiter->value().handle); +} + +// REQUIRES: The following fields of rep_ should have already been populated: +// 1. file +// 2. index_handle, +// 3. options +// 4. internal_comparator +// 5. index_type +Status BlockBasedTable::CreateIndexReader( + const ReadOptions& ro, FilePrefetchBuffer* prefetch_buffer, + InternalIterator* meta_iter, bool use_cache, bool prefetch, bool pin, + BlockCacheLookupContext* lookup_context, + std::unique_ptr* index_reader) { + if (FormatVersionUsesIndexHandleInFooter(rep_->footer.format_version())) { + rep_->index_handle = rep_->footer.index_handle(); + } else { + Status s = FindMetaBlock(meta_iter, kIndexBlockName, &rep_->index_handle); + if (!s.ok()) { + return s; + } + } + + switch (rep_->index_type) { + case BlockBasedTableOptions::kTwoLevelIndexSearch: { + return PartitionIndexReader::Create(this, ro, prefetch_buffer, use_cache, + prefetch, pin, lookup_context, + index_reader); + } + case BlockBasedTableOptions::kBinarySearch: + FALLTHROUGH_INTENDED; + case BlockBasedTableOptions::kBinarySearchWithFirstKey: { + return BinarySearchIndexReader::Create(this, ro, prefetch_buffer, + use_cache, prefetch, pin, + lookup_context, index_reader); + } + case BlockBasedTableOptions::kHashSearch: { + if (!rep_->table_prefix_extractor) { + ROCKS_LOG_WARN(rep_->ioptions.logger, + "Missing prefix extractor for hash index. Fall back to" + " binary search index."); + return BinarySearchIndexReader::Create(this, ro, prefetch_buffer, + use_cache, prefetch, pin, + lookup_context, index_reader); + } else { + return HashIndexReader::Create(this, ro, prefetch_buffer, meta_iter, + use_cache, prefetch, pin, lookup_context, + index_reader); + } + } + default: { + std::string error_message = + "Unrecognized index type: " + std::to_string(rep_->index_type); + return Status::InvalidArgument(error_message.c_str()); + } + } +} + +uint64_t BlockBasedTable::ApproximateDataOffsetOf( + const InternalIteratorBase& index_iter, + uint64_t data_size) const { + assert(index_iter.status().ok()); + if (index_iter.Valid()) { + BlockHandle handle = index_iter.value().handle; + return handle.offset(); + } else { + // The iterator is past the last key in the file. + return data_size; + } +} + +uint64_t BlockBasedTable::GetApproximateDataSize() { + // Should be in table properties unless super old version + if (rep_->table_properties) { + return rep_->table_properties->data_size; + } + // Fall back to rough estimate from footer + return rep_->footer.metaindex_handle().offset(); +} + +uint64_t BlockBasedTable::ApproximateOffsetOf(const ReadOptions& read_options, + const Slice& key, + TableReaderCaller caller) { + uint64_t data_size = GetApproximateDataSize(); + if (UNLIKELY(data_size == 0)) { + // Hmm. Let's just split in half to avoid skewing one way or another, + // since we don't know whether we're operating on lower bound or + // upper bound. + return rep_->file_size / 2; + } + + BlockCacheLookupContext context(caller); + IndexBlockIter iiter_on_stack; + ReadOptions ro; + ro.total_order_seek = true; + ro.io_activity = read_options.io_activity; + auto index_iter = + NewIndexIterator(ro, /*disable_prefix_seek=*/true, + /*input_iter=*/&iiter_on_stack, /*get_context=*/nullptr, + /*lookup_context=*/&context); + std::unique_ptr> iiter_unique_ptr; + if (index_iter != &iiter_on_stack) { + iiter_unique_ptr.reset(index_iter); + } + + index_iter->Seek(key); + uint64_t offset; + if (index_iter->status().ok()) { + offset = ApproximateDataOffsetOf(*index_iter, data_size); + } else { + // Split in half to avoid skewing one way or another, + // since we don't know whether we're operating on lower bound or + // upper bound. + return rep_->file_size / 2; + } + + // Pro-rate file metadata (incl filters) size-proportionally across data + // blocks. + double size_ratio = + static_cast(offset) / static_cast(data_size); + return static_cast(size_ratio * + static_cast(rep_->file_size)); +} + +uint64_t BlockBasedTable::ApproximateSize(const ReadOptions& read_options, + const Slice& start, const Slice& end, + TableReaderCaller caller) { + assert(rep_->internal_comparator.Compare(start, end) <= 0); + + uint64_t data_size = GetApproximateDataSize(); + if (UNLIKELY(data_size == 0)) { + // Hmm. Assume whole file is involved, since we have lower and upper + // bound. This likely skews the estimate if we consider that this function + // is typically called with `[start, end]` fully contained in the file's + // key-range. + return rep_->file_size; + } + + BlockCacheLookupContext context(caller); + IndexBlockIter iiter_on_stack; + ReadOptions ro; + ro.total_order_seek = true; + ro.io_activity = read_options.io_activity; + auto index_iter = + NewIndexIterator(ro, /*disable_prefix_seek=*/true, + /*input_iter=*/&iiter_on_stack, /*get_context=*/nullptr, + /*lookup_context=*/&context); + std::unique_ptr> iiter_unique_ptr; + if (index_iter != &iiter_on_stack) { + iiter_unique_ptr.reset(index_iter); + } + + index_iter->Seek(start); + uint64_t start_offset; + if (index_iter->status().ok()) { + start_offset = ApproximateDataOffsetOf(*index_iter, data_size); + } else { + // Assume file is involved from the start. This likely skews the estimate + // but is consistent with the above error handling. + start_offset = 0; + } + + index_iter->Seek(end); + uint64_t end_offset; + if (index_iter->status().ok()) { + end_offset = ApproximateDataOffsetOf(*index_iter, data_size); + } else { + // Assume file is involved until the end. This likely skews the estimate + // but is consistent with the above error handling. + end_offset = data_size; + } + + assert(end_offset >= start_offset); + // Pro-rate file metadata (incl filters) size-proportionally across data + // blocks. + double size_ratio = static_cast(end_offset - start_offset) / + static_cast(data_size); + return static_cast(size_ratio * + static_cast(rep_->file_size)); +} + +bool BlockBasedTable::TEST_FilterBlockInCache() const { + assert(rep_ != nullptr); + return rep_->filter_type != Rep::FilterType::kNoFilter && + TEST_BlockInCache(rep_->filter_handle); +} + +bool BlockBasedTable::TEST_IndexBlockInCache() const { + assert(rep_ != nullptr); + + return TEST_BlockInCache(rep_->index_handle); +} + +Status BlockBasedTable::GetKVPairsFromDataBlocks( + const ReadOptions& read_options, std::vector* kv_pair_blocks) { + std::unique_ptr> blockhandles_iter( + NewIndexIterator(read_options, /*need_upper_bound_check=*/false, + /*input_iter=*/nullptr, /*get_context=*/nullptr, + /*lookup_contex=*/nullptr)); + + Status s = blockhandles_iter->status(); + if (!s.ok()) { + // Cannot read Index Block + return s; + } + + for (blockhandles_iter->SeekToFirst(); blockhandles_iter->Valid(); + blockhandles_iter->Next()) { + s = blockhandles_iter->status(); + + if (!s.ok()) { + break; + } + + std::unique_ptr datablock_iter; + Status tmp_status; + datablock_iter.reset(NewDataBlockIterator( + read_options, blockhandles_iter->value().handle, + /*input_iter=*/nullptr, /*type=*/BlockType::kData, + /*get_context=*/nullptr, /*lookup_context=*/nullptr, + /*prefetch_buffer=*/nullptr, /*for_compaction=*/false, + /*async_read=*/false, tmp_status)); + s = datablock_iter->status(); + + if (!s.ok()) { + // Error reading the block - Skipped + continue; + } + + KVPairBlock kv_pair_block; + for (datablock_iter->SeekToFirst(); datablock_iter->Valid(); + datablock_iter->Next()) { + s = datablock_iter->status(); + if (!s.ok()) { + // Error reading the block - Skipped + break; + } + const Slice& key = datablock_iter->key(); + const Slice& value = datablock_iter->value(); + std::string key_copy = std::string(key.data(), key.size()); + std::string value_copy = std::string(value.data(), value.size()); + + kv_pair_block.push_back( + std::make_pair(std::move(key_copy), std::move(value_copy))); + } + kv_pair_blocks->push_back(std::move(kv_pair_block)); + } + return Status::OK(); +} + +Status BlockBasedTable::DumpTable(WritableFile* out_file) { + WritableFileStringStreamAdapter out_file_wrapper(out_file); + std::ostream out_stream(&out_file_wrapper); + // Output Footer + out_stream << "Footer Details:\n" + "--------------------------------------\n"; + out_stream << " " << rep_->footer.ToString() << "\n"; + + // Output MetaIndex + out_stream << "Metaindex Details:\n" + "--------------------------------------\n"; + std::unique_ptr metaindex; + std::unique_ptr metaindex_iter; + // TODO: plumb Env::IOActivity + const ReadOptions ro; + Status s = ReadMetaIndexBlock(ro, nullptr /* prefetch_buffer */, &metaindex, + &metaindex_iter); + if (s.ok()) { + for (metaindex_iter->SeekToFirst(); metaindex_iter->Valid(); + metaindex_iter->Next()) { + s = metaindex_iter->status(); + if (!s.ok()) { + return s; + } + if (metaindex_iter->key() == kPropertiesBlockName) { + out_stream << " Properties block handle: " + << metaindex_iter->value().ToString(true) << "\n"; + } else if (metaindex_iter->key() == kCompressionDictBlockName) { + out_stream << " Compression dictionary block handle: " + << metaindex_iter->value().ToString(true) << "\n"; + } else if (strstr(metaindex_iter->key().ToString().c_str(), + "filter.rocksdb.") != nullptr) { + out_stream << " Filter block handle: " + << metaindex_iter->value().ToString(true) << "\n"; + } else if (metaindex_iter->key() == kRangeDelBlockName) { + out_stream << " Range deletion block handle: " + << metaindex_iter->value().ToString(true) << "\n"; + } + } + out_stream << "\n"; + } else { + return s; + } + + // Output TableProperties + const ROCKSDB_NAMESPACE::TableProperties* table_properties; + table_properties = rep_->table_properties.get(); + + if (table_properties != nullptr) { + out_stream << "Table Properties:\n" + "--------------------------------------\n"; + out_stream << " " << table_properties->ToString("\n ", ": ") << "\n"; + } + + if (rep_->filter) { + out_stream << "Filter Details:\n" + "--------------------------------------\n"; + out_stream << " " << rep_->filter->ToString() << "\n"; + } + + // Output Index block + s = DumpIndexBlock(out_stream); + if (!s.ok()) { + return s; + } + + // Output compression dictionary + if (rep_->uncompression_dict_reader) { + CachableEntry uncompression_dict; + s = rep_->uncompression_dict_reader->GetOrReadUncompressionDictionary( + nullptr /* prefetch_buffer */, ro, false /* no_io */, + false, /* verify_checksums */ + nullptr /* get_context */, nullptr /* lookup_context */, + &uncompression_dict); + if (!s.ok()) { + return s; + } + + assert(uncompression_dict.GetValue()); + + const Slice& raw_dict = uncompression_dict.GetValue()->GetRawDict(); + out_stream << "Compression Dictionary:\n" + "--------------------------------------\n"; + out_stream << " size (bytes): " << raw_dict.size() << "\n\n"; + out_stream << " HEX " << raw_dict.ToString(true) << "\n\n"; + } + + // Output range deletions block + auto* range_del_iter = NewRangeTombstoneIterator(ro); + if (range_del_iter != nullptr) { + range_del_iter->SeekToFirst(); + if (range_del_iter->Valid()) { + out_stream << "Range deletions:\n" + "--------------------------------------\n"; + for (; range_del_iter->Valid(); range_del_iter->Next()) { + DumpKeyValue(range_del_iter->key(), range_del_iter->value(), + out_stream); + } + out_stream << "\n"; + } + delete range_del_iter; + } + // Output Data blocks + s = DumpDataBlocks(out_stream); + + if (!s.ok()) { + return s; + } + + if (!out_stream.good()) { + return Status::IOError("Failed to write to output file"); + } + return Status::OK(); +} + +Status BlockBasedTable::DumpIndexBlock(std::ostream& out_stream) { + out_stream << "Index Details:\n" + "--------------------------------------\n"; + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + std::unique_ptr> blockhandles_iter( + NewIndexIterator(read_options, /*need_upper_bound_check=*/false, + /*input_iter=*/nullptr, /*get_context=*/nullptr, + /*lookup_contex=*/nullptr)); + Status s = blockhandles_iter->status(); + if (!s.ok()) { + out_stream << "Can not read Index Block \n\n"; + return s; + } + + out_stream << " Block key hex dump: Data block handle\n"; + out_stream << " Block key ascii\n\n"; + for (blockhandles_iter->SeekToFirst(); blockhandles_iter->Valid(); + blockhandles_iter->Next()) { + s = blockhandles_iter->status(); + if (!s.ok()) { + break; + } + Slice key = blockhandles_iter->key(); + Slice user_key; + InternalKey ikey; + if (!rep_->index_key_includes_seq) { + user_key = key; + } else { + ikey.DecodeFrom(key); + user_key = ikey.user_key(); + } + + out_stream << " HEX " << user_key.ToString(true) << ": " + << blockhandles_iter->value().ToString(true, + rep_->index_has_first_key) + << " offset " << blockhandles_iter->value().handle.offset() + << " size " << blockhandles_iter->value().handle.size() << "\n"; + + std::string str_key = user_key.ToString(); + std::string res_key(""); + char cspace = ' '; + for (size_t i = 0; i < str_key.size(); i++) { + res_key.append(&str_key[i], 1); + res_key.append(1, cspace); + } + out_stream << " ASCII " << res_key << "\n"; + out_stream << " ------\n"; + } + out_stream << "\n"; + return Status::OK(); +} + +Status BlockBasedTable::DumpDataBlocks(std::ostream& out_stream) { + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + std::unique_ptr> blockhandles_iter( + NewIndexIterator(read_options, /*need_upper_bound_check=*/false, + /*input_iter=*/nullptr, /*get_context=*/nullptr, + /*lookup_contex=*/nullptr)); + Status s = blockhandles_iter->status(); + if (!s.ok()) { + out_stream << "Can not read Index Block \n\n"; + return s; + } + + uint64_t datablock_size_min = std::numeric_limits::max(); + uint64_t datablock_size_max = 0; + uint64_t datablock_size_sum = 0; + + size_t block_id = 1; + for (blockhandles_iter->SeekToFirst(); blockhandles_iter->Valid(); + block_id++, blockhandles_iter->Next()) { + s = blockhandles_iter->status(); + if (!s.ok()) { + break; + } + + BlockHandle bh = blockhandles_iter->value().handle; + uint64_t datablock_size = bh.size(); + datablock_size_min = std::min(datablock_size_min, datablock_size); + datablock_size_max = std::max(datablock_size_max, datablock_size); + datablock_size_sum += datablock_size; + + out_stream << "Data Block # " << block_id << " @ " + << blockhandles_iter->value().handle.ToString(true) << "\n"; + out_stream << "--------------------------------------\n"; + + std::unique_ptr datablock_iter; + Status tmp_status; + datablock_iter.reset(NewDataBlockIterator( + read_options, blockhandles_iter->value().handle, + /*input_iter=*/nullptr, /*type=*/BlockType::kData, + /*get_context=*/nullptr, /*lookup_context=*/nullptr, + /*prefetch_buffer=*/nullptr, /*for_compaction=*/false, + /*async_read=*/false, tmp_status)); + s = datablock_iter->status(); + + if (!s.ok()) { + out_stream << "Error reading the block - Skipped \n\n"; + continue; + } + + for (datablock_iter->SeekToFirst(); datablock_iter->Valid(); + datablock_iter->Next()) { + s = datablock_iter->status(); + if (!s.ok()) { + out_stream << "Error reading the block - Skipped \n"; + break; + } + DumpKeyValue(datablock_iter->key(), datablock_iter->value(), out_stream); + } + out_stream << "\n"; + } + + uint64_t num_datablocks = block_id - 1; + if (num_datablocks) { + double datablock_size_avg = + static_cast(datablock_size_sum) / num_datablocks; + out_stream << "Data Block Summary:\n"; + out_stream << "--------------------------------------\n"; + out_stream << " # data blocks: " << num_datablocks << "\n"; + out_stream << " min data block size: " << datablock_size_min << "\n"; + out_stream << " max data block size: " << datablock_size_max << "\n"; + out_stream << " avg data block size: " + << std::to_string(datablock_size_avg) << "\n"; + } + + return Status::OK(); +} + +void BlockBasedTable::DumpKeyValue(const Slice& key, const Slice& value, + std::ostream& out_stream) { + InternalKey ikey; + ikey.DecodeFrom(key); + + out_stream << " HEX " << ikey.user_key().ToString(true) << ": " + << value.ToString(true) << "\n"; + + std::string str_key = ikey.user_key().ToString(); + std::string str_value = value.ToString(); + std::string res_key(""), res_value(""); + char cspace = ' '; + for (size_t i = 0; i < str_key.size(); i++) { + if (str_key[i] == '\0') { + res_key.append("\\0", 2); + } else { + res_key.append(&str_key[i], 1); + } + res_key.append(1, cspace); + } + for (size_t i = 0; i < str_value.size(); i++) { + if (str_value[i] == '\0') { + res_value.append("\\0", 2); + } else { + res_value.append(&str_value[i], 1); + } + res_value.append(1, cspace); + } + + out_stream << " ASCII " << res_key << ": " << res_value << "\n"; + out_stream << " ------\n"; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/block_based_table_reader.h b/librocksdb-sys/rocksdb/table/block_based/block_based_table_reader.h new file mode 100644 index 0000000..0b5fe1c --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block_based_table_reader.h @@ -0,0 +1,764 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include +#include + +#include "cache/cache_entry_roles.h" +#include "cache/cache_key.h" +#include "cache/cache_reservation_manager.h" +#include "db/range_tombstone_fragmenter.h" +#include "file/filename.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/table_properties.h" +#include "table/block_based/block.h" +#include "table/block_based/block_based_table_factory.h" +#include "table/block_based/block_cache.h" +#include "table/block_based/block_type.h" +#include "table/block_based/cachable_entry.h" +#include "table/block_based/filter_block.h" +#include "table/block_based/uncompression_dict_reader.h" +#include "table/format.h" +#include "table/persistent_cache_options.h" +#include "table/table_properties_internal.h" +#include "table/table_reader.h" +#include "table/two_level_iterator.h" +#include "trace_replay/block_cache_tracer.h" +#include "util/coro_utils.h" +#include "util/hash_containers.h" + +namespace ROCKSDB_NAMESPACE { + +class Cache; +class FilterBlockReader; +class FullFilterBlockReader; +class Footer; +class InternalKeyComparator; +class Iterator; +class FSRandomAccessFile; +class TableCache; +class TableReader; +class WritableFile; +struct BlockBasedTableOptions; +struct EnvOptions; +struct ReadOptions; +class GetContext; + +using KVPairBlock = std::vector>; + +// Reader class for BlockBasedTable format. +// For the format of BlockBasedTable refer to +// https://github.com/facebook/rocksdb/wiki/Rocksdb-BlockBasedTable-Format. +// This is the default table type. Data is chucked into fixed size blocks and +// each block in-turn stores entries. When storing data, we can compress and/or +// encode data efficiently within a block, which often results in a much smaller +// data size compared with the raw data size. As for the record retrieval, we'll +// first locate the block where target record may reside, then read the block to +// memory, and finally search that record within the block. Of course, to avoid +// frequent reads of the same block, we introduced the block cache to keep the +// loaded blocks in the memory. +class BlockBasedTable : public TableReader { + public: + static const std::string kObsoleteFilterBlockPrefix; + static const std::string kFullFilterBlockPrefix; + static const std::string kPartitionedFilterBlockPrefix; + + // 1-byte compression type + 32-bit checksum + static constexpr size_t kBlockTrailerSize = 5; + + // Attempt to open the table that is stored in bytes [0..file_size) + // of "file", and read the metadata entries necessary to allow + // retrieving data from the table. + // + // If successful, returns ok and sets "*table_reader" to the newly opened + // table. The client should delete "*table_reader" when no longer needed. + // If there was an error while initializing the table, sets "*table_reader" + // to nullptr and returns a non-ok status. + // + // @param file must remain live while this Table is in use. + // @param prefetch_index_and_filter_in_cache can be used to disable + // prefetching of + // index and filter blocks into block cache at startup + // @param skip_filters Disables loading/accessing the filter block. Overrides + // prefetch_index_and_filter_in_cache, so filter will be skipped if both + // are set. + // @param force_direct_prefetch if true, always prefetching to RocksDB + // buffer, rather than calling RandomAccessFile::Prefetch(). + static Status Open( + const ReadOptions& ro, const ImmutableOptions& ioptions, + const EnvOptions& env_options, + const BlockBasedTableOptions& table_options, + const InternalKeyComparator& internal_key_comparator, + std::unique_ptr&& file, uint64_t file_size, + uint8_t block_protection_bytes_per_key, + std::unique_ptr* table_reader, uint64_t tail_size, + std::shared_ptr table_reader_cache_res_mgr = + nullptr, + const std::shared_ptr& prefix_extractor = nullptr, + bool prefetch_index_and_filter_in_cache = true, bool skip_filters = false, + int level = -1, const bool immortal_table = false, + const SequenceNumber largest_seqno = 0, + bool force_direct_prefetch = false, + TailPrefetchStats* tail_prefetch_stats = nullptr, + BlockCacheTracer* const block_cache_tracer = nullptr, + size_t max_file_size_for_l0_meta_pin = 0, + const std::string& cur_db_session_id = "", uint64_t cur_file_num = 0, + UniqueId64x2 expected_unique_id = {}, + const bool user_defined_timestamps_persisted = true); + + bool PrefixRangeMayMatch(const Slice& internal_key, + const ReadOptions& read_options, + const SliceTransform* options_prefix_extractor, + const bool need_upper_bound_check, + BlockCacheLookupContext* lookup_context, + bool* filter_checked) const; + + // Returns a new iterator over the table contents. + // The result of NewIterator() is initially invalid (caller must + // call one of the Seek methods on the iterator before using it). + // @param read_options Must outlive the returned iterator. + // @param skip_filters Disables loading/accessing the filter block + // compaction_readahead_size: its value will only be used if caller = + // kCompaction. + InternalIterator* NewIterator(const ReadOptions&, + const SliceTransform* prefix_extractor, + Arena* arena, bool skip_filters, + TableReaderCaller caller, + size_t compaction_readahead_size = 0, + bool allow_unprepared_value = false) override; + + FragmentedRangeTombstoneIterator* NewRangeTombstoneIterator( + const ReadOptions& read_options) override; + + // @param skip_filters Disables loading/accessing the filter block + Status Get(const ReadOptions& readOptions, const Slice& key, + GetContext* get_context, const SliceTransform* prefix_extractor, + bool skip_filters = false) override; + + Status MultiGetFilter(const ReadOptions& read_options, + const SliceTransform* prefix_extractor, + MultiGetRange* mget_range) override; + + DECLARE_SYNC_AND_ASYNC_OVERRIDE(void, MultiGet, + const ReadOptions& readOptions, + const MultiGetContext::Range* mget_range, + const SliceTransform* prefix_extractor, + bool skip_filters = false); + + // Pre-fetch the disk blocks that correspond to the key range specified by + // (kbegin, kend). The call will return error status in the event of + // IO or iteration error. + Status Prefetch(const ReadOptions& read_options, const Slice* begin, + const Slice* end) override; + + // Given a key, return an approximate byte offset in the file where + // the data for that key begins (or would begin if the key were + // present in the file). The returned value is in terms of file + // bytes, and so includes effects like compression of the underlying data. + // E.g., the approximate offset of the last key in the table will + // be close to the file length. + uint64_t ApproximateOffsetOf(const ReadOptions& read_options, + const Slice& key, + TableReaderCaller caller) override; + + // Given start and end keys, return the approximate data size in the file + // between the keys. The returned value is in terms of file bytes, and so + // includes effects like compression of the underlying data. + // The start key must not be greater than the end key. + uint64_t ApproximateSize(const ReadOptions& read_options, const Slice& start, + const Slice& end, TableReaderCaller caller) override; + + Status ApproximateKeyAnchors(const ReadOptions& read_options, + std::vector& anchors) override; + + bool TEST_BlockInCache(const BlockHandle& handle) const; + + // Returns true if the block for the specified key is in cache. + // REQUIRES: key is in this table && block cache enabled + bool TEST_KeyInCache(const ReadOptions& options, const Slice& key); + + // Set up the table for Compaction. Might change some parameters with + // posix_fadvise + void SetupForCompaction() override; + + std::shared_ptr GetTableProperties() const override; + + size_t ApproximateMemoryUsage() const override; + + // convert SST file to a human readable form + Status DumpTable(WritableFile* out_file) override; + + Status VerifyChecksum(const ReadOptions& readOptions, + TableReaderCaller caller) override; + + ~BlockBasedTable(); + + bool TEST_FilterBlockInCache() const; + bool TEST_IndexBlockInCache() const; + + // IndexReader is the interface that provides the functionality for index + // access. + class IndexReader { + public: + virtual ~IndexReader() = default; + + // Create an iterator for index access. If iter is null, then a new object + // is created on the heap, and the callee will have the ownership. + // If a non-null iter is passed in, it will be used, and the returned value + // is either the same as iter or a new on-heap object that + // wraps the passed iter. In the latter case the return value points + // to a different object then iter, and the callee has the ownership of the + // returned object. + virtual InternalIteratorBase* NewIterator( + const ReadOptions& read_options, bool disable_prefix_seek, + IndexBlockIter* iter, GetContext* get_context, + BlockCacheLookupContext* lookup_context) = 0; + + // Report an approximation of how much memory has been used other than + // memory that was allocated in block cache. + virtual size_t ApproximateMemoryUsage() const = 0; + // Cache the dependencies of the index reader (e.g. the partitions + // of a partitioned index). + virtual Status CacheDependencies( + const ReadOptions& /*ro*/, bool /* pin */, + FilePrefetchBuffer* /* tail_prefetch_buffer */) { + return Status::OK(); + } + }; + + class IndexReaderCommon; + + static void SetupBaseCacheKey(const TableProperties* properties, + const std::string& cur_db_session_id, + uint64_t cur_file_number, + OffsetableCacheKey* out_base_cache_key, + bool* out_is_stable = nullptr); + + static CacheKey GetCacheKey(const OffsetableCacheKey& base_cache_key, + const BlockHandle& handle); + + static void UpdateCacheInsertionMetrics(BlockType block_type, + GetContext* get_context, size_t usage, + bool redundant, + Statistics* const statistics); + + Statistics* GetStatistics() const; + bool IsLastLevel() const; + + // Get the size to read from storage for a BlockHandle. size_t because we + // are about to load into memory. + static inline size_t BlockSizeWithTrailer(const BlockHandle& handle) { + return static_cast(handle.size() + kBlockTrailerSize); + } + + // It is the caller's responsibility to make sure that this is called with + // block-based table serialized block contents, which contains the compression + // byte in the trailer after `block_size`. + static inline CompressionType GetBlockCompressionType(const char* block_data, + size_t block_size) { + return static_cast(block_data[block_size]); + } + static inline CompressionType GetBlockCompressionType( + const BlockContents& contents) { + assert(contents.has_trailer); + return GetBlockCompressionType(contents.data.data(), contents.data.size()); + } + + // Retrieve all key value pairs from data blocks in the table. + // The key retrieved are internal keys. + Status GetKVPairsFromDataBlocks(const ReadOptions& read_options, + std::vector* kv_pair_blocks); + + struct Rep; + + Rep* get_rep() { return rep_; } + const Rep* get_rep() const { return rep_; } + + // input_iter: if it is not null, update this one and return it as Iterator + template + TBlockIter* NewDataBlockIterator(const ReadOptions& ro, + const BlockHandle& block_handle, + TBlockIter* input_iter, BlockType block_type, + GetContext* get_context, + BlockCacheLookupContext* lookup_context, + FilePrefetchBuffer* prefetch_buffer, + bool for_compaction, bool async_read, + Status& s) const; + + // input_iter: if it is not null, update this one and return it as Iterator + template + TBlockIter* NewDataBlockIterator(const ReadOptions& ro, + CachableEntry& block, + TBlockIter* input_iter, Status s) const; + + class PartitionedIndexIteratorState; + + template + friend class FilterBlockReaderCommon; + + friend class PartitionIndexReader; + + friend class UncompressionDictReader; + + protected: + Rep* rep_; + explicit BlockBasedTable(Rep* rep, BlockCacheTracer* const block_cache_tracer) + : rep_(rep), block_cache_tracer_(block_cache_tracer) {} + // No copying allowed + explicit BlockBasedTable(const TableReader&) = delete; + void operator=(const TableReader&) = delete; + + private: + friend class MockedBlockBasedTable; + friend class BlockBasedTableReaderTestVerifyChecksum_ChecksumMismatch_Test; + BlockCacheTracer* const block_cache_tracer_; + + void UpdateCacheHitMetrics(BlockType block_type, GetContext* get_context, + size_t usage) const; + void UpdateCacheMissMetrics(BlockType block_type, + GetContext* get_context) const; + + // Either Block::NewDataIterator() or Block::NewIndexIterator(). + template + static TBlockIter* InitBlockIterator(const Rep* rep, Block* block, + BlockType block_type, + TBlockIter* input_iter, + bool block_contents_pinned); + + // If block cache enabled (compressed or uncompressed), looks for the block + // identified by handle in (1) uncompressed cache, (2) compressed cache, and + // then (3) file. If found, inserts into the cache(s) that were searched + // unsuccessfully (e.g., if found in file, will add to both uncompressed and + // compressed caches if they're enabled). + // + // @param block_entry value is set to the uncompressed block if found. If + // in uncompressed block cache, also sets cache_handle to reference that + // block. + template + WithBlocklikeCheck MaybeReadBlockAndLoadToCache( + FilePrefetchBuffer* prefetch_buffer, const ReadOptions& ro, + const BlockHandle& handle, const UncompressionDict& uncompression_dict, + bool for_compaction, CachableEntry* block_entry, + GetContext* get_context, BlockCacheLookupContext* lookup_context, + BlockContents* contents, bool async_read) const; + + // Similar to the above, with one crucial difference: it will retrieve the + // block from the file even if there are no caches configured (assuming the + // read options allow I/O). + template + WithBlocklikeCheck RetrieveBlock( + FilePrefetchBuffer* prefetch_buffer, const ReadOptions& ro, + const BlockHandle& handle, const UncompressionDict& uncompression_dict, + CachableEntry* block_entry, GetContext* get_context, + BlockCacheLookupContext* lookup_context, bool for_compaction, + bool use_cache, bool async_read) const; + + template + WithBlocklikeCheck SaveLookupContextOrTraceRecord( + const Slice& block_key, bool is_cache_hit, const ReadOptions& ro, + const TBlocklike* parsed_block_value, + BlockCacheLookupContext* lookup_context) const; + + void FinishTraceRecord(const BlockCacheLookupContext& lookup_context, + const Slice& block_key, const Slice& referenced_key, + bool does_referenced_key_exist, + uint64_t referenced_data_size) const; + + DECLARE_SYNC_AND_ASYNC_CONST( + void, RetrieveMultipleBlocks, const ReadOptions& options, + const MultiGetRange* batch, + const autovector* handles, + Status* statuses, CachableEntry* results, char* scratch, + const UncompressionDict& uncompression_dict, bool use_fs_scratch); + + // Get the iterator from the index reader. + // + // If input_iter is not set, return a new Iterator. + // If input_iter is set, try to update it and return it as Iterator. + // However note that in some cases the returned iterator may be different + // from input_iter. In such case the returned iterator should be freed. + // + // Note: ErrorIterator with Status::Incomplete shall be returned if all the + // following conditions are met: + // 1. We enabled table_options.cache_index_and_filter_blocks. + // 2. index is not present in block cache. + // 3. We disallowed any io to be performed, that is, read_options == + // kBlockCacheTier + InternalIteratorBase* NewIndexIterator( + const ReadOptions& read_options, bool need_upper_bound_check, + IndexBlockIter* input_iter, GetContext* get_context, + BlockCacheLookupContext* lookup_context) const; + + template + Cache::Priority GetCachePriority() const; + + // Read block cache from block caches (if set): block_cache. + // On success, Status::OK with be returned and @block will be populated with + // pointer to the block as well as its block handle. + // @param uncompression_dict Data for presetting the compression library's + // dictionary. + template + WithBlocklikeCheck GetDataBlockFromCache( + const Slice& cache_key, BlockCacheInterface block_cache, + CachableEntry* block, GetContext* get_context) const; + + // Put a maybe compressed block to the corresponding block caches. + // This method will perform decompression against block_contents if needed + // and then populate the block caches. + // On success, Status::OK will be returned; also @block will be populated with + // uncompressed block and its cache handle. + // + // Allocated memory managed by block_contents will be transferred to + // PutDataBlockToCache(). After the call, the object will be invalid. + // @param uncompression_dict Data for presetting the compression library's + // dictionary. + template + WithBlocklikeCheck PutDataBlockToCache( + const Slice& cache_key, BlockCacheInterface block_cache, + CachableEntry* cached_block, BlockContents&& block_contents, + CompressionType block_comp_type, + const UncompressionDict& uncompression_dict, + MemoryAllocator* memory_allocator, GetContext* get_context) const; + + // Calls (*handle_result)(arg, ...) repeatedly, starting with the entry found + // after a call to Seek(key), until handle_result returns false. + // May not make such a call if filter policy says that key is not present. + friend class TableCache; + friend class BlockBasedTableBuilder; + + // Create a index reader based on the index type stored in the table. + // Optionally, user can pass a preloaded meta_index_iter for the index that + // need to access extra meta blocks for index construction. This parameter + // helps avoid re-reading meta index block if caller already created one. + Status CreateIndexReader(const ReadOptions& ro, + FilePrefetchBuffer* prefetch_buffer, + InternalIterator* preloaded_meta_index_iter, + bool use_cache, bool prefetch, bool pin, + BlockCacheLookupContext* lookup_context, + std::unique_ptr* index_reader); + + bool FullFilterKeyMayMatch(FilterBlockReader* filter, const Slice& user_key, + const bool no_io, + const SliceTransform* prefix_extractor, + GetContext* get_context, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) const; + + void FullFilterKeysMayMatch(FilterBlockReader* filter, MultiGetRange* range, + const bool no_io, + const SliceTransform* prefix_extractor, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) const; + + // If force_direct_prefetch is true, always prefetching to RocksDB + // buffer, rather than calling RandomAccessFile::Prefetch(). + static Status PrefetchTail( + const ReadOptions& ro, RandomAccessFileReader* file, uint64_t file_size, + bool force_direct_prefetch, TailPrefetchStats* tail_prefetch_stats, + const bool prefetch_all, const bool preload_all, + std::unique_ptr* prefetch_buffer, Statistics* stats, + uint64_t tail_size, Logger* const logger); + Status ReadMetaIndexBlock(const ReadOptions& ro, + FilePrefetchBuffer* prefetch_buffer, + std::unique_ptr* metaindex_block, + std::unique_ptr* iter); + Status ReadPropertiesBlock(const ReadOptions& ro, + FilePrefetchBuffer* prefetch_buffer, + InternalIterator* meta_iter, + const SequenceNumber largest_seqno); + Status ReadRangeDelBlock(const ReadOptions& ro, + FilePrefetchBuffer* prefetch_buffer, + InternalIterator* meta_iter, + const InternalKeyComparator& internal_comparator, + BlockCacheLookupContext* lookup_context); + Status PrefetchIndexAndFilterBlocks( + const ReadOptions& ro, FilePrefetchBuffer* prefetch_buffer, + InternalIterator* meta_iter, BlockBasedTable* new_table, + bool prefetch_all, const BlockBasedTableOptions& table_options, + const int level, size_t file_size, size_t max_file_size_for_l0_meta_pin, + BlockCacheLookupContext* lookup_context); + + static BlockType GetBlockTypeForMetaBlockByName(const Slice& meta_block_name); + + Status VerifyChecksumInMetaBlocks(const ReadOptions& read_options, + InternalIteratorBase* index_iter); + Status VerifyChecksumInBlocks(const ReadOptions& read_options, + InternalIteratorBase* index_iter); + + // Create the filter from the filter block. + std::unique_ptr CreateFilterBlockReader( + const ReadOptions& ro, FilePrefetchBuffer* prefetch_buffer, + bool use_cache, bool prefetch, bool pin, + BlockCacheLookupContext* lookup_context); + + // Size of all data blocks, maybe approximate + uint64_t GetApproximateDataSize(); + + // Given an iterator return its offset in data block section of file. + uint64_t ApproximateDataOffsetOf( + const InternalIteratorBase& index_iter, + uint64_t data_size) const; + + // Helper functions for DumpTable() + Status DumpIndexBlock(std::ostream& out_stream); + Status DumpDataBlocks(std::ostream& out_stream); + void DumpKeyValue(const Slice& key, const Slice& value, + std::ostream& out_stream); + + // Returns false if prefix_extractor exists and is compatible with that used + // in building the table file, otherwise true. + bool PrefixExtractorChanged(const SliceTransform* prefix_extractor) const; + + bool TimestampMayMatch(const ReadOptions& read_options) const; + + // A cumulative data block file read in MultiGet lower than this size will + // use a stack buffer + static constexpr size_t kMultiGetReadStackBufSize = 8192; + + friend class PartitionedFilterBlockReader; + friend class PartitionedFilterBlockTest; + friend class DBBasicTest_MultiGetIOBufferOverrun_Test; +}; + +// Maintaining state of a two-level iteration on a partitioned index structure. +class BlockBasedTable::PartitionedIndexIteratorState + : public TwoLevelIteratorState { + public: + PartitionedIndexIteratorState( + const BlockBasedTable* table, + UnorderedMap>* block_map); + InternalIteratorBase* NewSecondaryIterator( + const BlockHandle& index_value) override; + + private: + // Don't own table_ + const BlockBasedTable* table_; + UnorderedMap>* block_map_; +}; + +// Stores all the properties associated with a BlockBasedTable. +// These are immutable. +struct BlockBasedTable::Rep { + Rep(const ImmutableOptions& _ioptions, const EnvOptions& _env_options, + const BlockBasedTableOptions& _table_opt, + const InternalKeyComparator& _internal_comparator, bool skip_filters, + uint64_t _file_size, int _level, const bool _immortal_table, + const bool _user_defined_timestamps_persisted = true) + : ioptions(_ioptions), + env_options(_env_options), + table_options(_table_opt), + filter_policy(skip_filters ? nullptr : _table_opt.filter_policy.get()), + internal_comparator(_internal_comparator), + filter_type(FilterType::kNoFilter), + index_type(BlockBasedTableOptions::IndexType::kBinarySearch), + whole_key_filtering(_table_opt.whole_key_filtering), + prefix_filtering(true), + global_seqno(kDisableGlobalSequenceNumber), + file_size(_file_size), + level(_level), + immortal_table(_immortal_table), + user_defined_timestamps_persisted(_user_defined_timestamps_persisted) {} + ~Rep() { status.PermitUncheckedError(); } + const ImmutableOptions& ioptions; + const EnvOptions& env_options; + const BlockBasedTableOptions table_options; + const FilterPolicy* const filter_policy; + const InternalKeyComparator& internal_comparator; + Status status; + std::unique_ptr file; + OffsetableCacheKey base_cache_key; + PersistentCacheOptions persistent_cache_options; + + // Footer contains the fixed table information + Footer footer; + + std::unique_ptr index_reader; + std::unique_ptr filter; + std::unique_ptr uncompression_dict_reader; + + enum class FilterType { + kNoFilter, + kFullFilter, + kPartitionedFilter, + }; + FilterType filter_type; + BlockHandle filter_handle; + BlockHandle compression_dict_handle; + + std::shared_ptr table_properties; + BlockHandle index_handle; + BlockBasedTableOptions::IndexType index_type; + bool whole_key_filtering; + bool prefix_filtering; + std::shared_ptr table_prefix_extractor; + + std::shared_ptr fragmented_range_dels; + + // FIXME + // If true, data blocks in this file are definitely ZSTD compressed. If false + // they might not be. When false we skip creating a ZSTD digested + // uncompression dictionary. Even if we get a false negative, things should + // still work, just not as quickly. + BlockCreateContext create_context; + + // If global_seqno is used, all Keys in this file will have the same + // seqno with value `global_seqno`. + // + // A value of kDisableGlobalSequenceNumber means that this feature is disabled + // and every key have it's own seqno. + SequenceNumber global_seqno; + + // Size of the table file on disk + uint64_t file_size; + + // the level when the table is opened, could potentially change when trivial + // move is involved + int level; + + // the timestamp range of table + // Points into memory owned by TableProperties. This would need to change if + // TableProperties become subject to cache eviction. + Slice min_timestamp; + Slice max_timestamp; + + // If false, blocks in this file are definitely all uncompressed. Knowing this + // before reading individual blocks enables certain optimizations. + bool blocks_maybe_compressed = true; + + // These describe how index is encoded. + bool index_has_first_key = false; + bool index_key_includes_seq = true; + bool index_value_is_full = true; + + // Whether block checksums in metadata blocks were verified on open. + // This is only to mostly maintain current dubious behavior of VerifyChecksum + // with respect to index blocks, but only when the checksum was previously + // verified. + bool verify_checksum_set_on_open = false; + + const bool immortal_table; + // Whether the user key contains user-defined timestamps. If this is false and + // the running user comparator has a non-zero timestamp size, a min timestamp + // of this size will be padded to each user key while parsing blocks whenever + // it applies. This includes the keys in data block, index block for data + // block, top-level index for index partitions (if index type is + // `kTwoLevelIndexSearch`), top-level index for filter partitions (if using + // partitioned filters), the `first_internal_key` in `IndexValue`, the + // `end_key` for range deletion entries. + const bool user_defined_timestamps_persisted; + + std::unique_ptr + table_reader_cache_res_handle = nullptr; + + SequenceNumber get_global_seqno(BlockType block_type) const { + return (block_type == BlockType::kFilterPartitionIndex || + block_type == BlockType::kCompressionDictionary) + ? kDisableGlobalSequenceNumber + : global_seqno; + } + + uint64_t cf_id_for_tracing() const { + return table_properties + ? table_properties->column_family_id + : ROCKSDB_NAMESPACE::TablePropertiesCollectorFactory::Context:: + kUnknownColumnFamily; + } + + Slice cf_name_for_tracing() const { + return table_properties ? table_properties->column_family_name + : BlockCacheTraceHelper::kUnknownColumnFamilyName; + } + + uint32_t level_for_tracing() const { return level >= 0 ? level : UINT32_MAX; } + + uint64_t sst_number_for_tracing() const { + return file ? TableFileNameToNumber(file->file_name()) : UINT64_MAX; + } + void CreateFilePrefetchBuffer( + size_t readahead_size, size_t max_readahead_size, + std::unique_ptr* fpb, bool implicit_auto_readahead, + uint64_t num_file_reads, + uint64_t num_file_reads_for_auto_readahead) const { + fpb->reset(new FilePrefetchBuffer( + readahead_size, max_readahead_size, + !ioptions.allow_mmap_reads /* enable */, false /* track_min_offset */, + implicit_auto_readahead, num_file_reads, + num_file_reads_for_auto_readahead, ioptions.fs.get(), ioptions.clock, + ioptions.stats)); + } + + void CreateFilePrefetchBufferIfNotExists( + size_t readahead_size, size_t max_readahead_size, + std::unique_ptr* fpb, bool implicit_auto_readahead, + uint64_t num_file_reads, + uint64_t num_file_reads_for_auto_readahead) const { + if (!(*fpb)) { + CreateFilePrefetchBuffer(readahead_size, max_readahead_size, fpb, + implicit_auto_readahead, num_file_reads, + num_file_reads_for_auto_readahead); + } + } + + std::size_t ApproximateMemoryUsage() const { + std::size_t usage = 0; +#ifdef ROCKSDB_MALLOC_USABLE_SIZE + usage += malloc_usable_size(const_cast(this)); +#else + usage += sizeof(*this); +#endif // ROCKSDB_MALLOC_USABLE_SIZE + return usage; + } +}; + +// This is an adapter class for `WritableFile` to be used for `std::ostream`. +// The adapter wraps a `WritableFile`, which can be passed to a `std::ostream` +// constructor for storing streaming data. +// Note: +// * This adapter doesn't provide any buffering, each write is forwarded to +// `WritableFile->Append()` directly. +// * For a failed write, the user needs to check the status by `ostream.good()` +class WritableFileStringStreamAdapter : public std::stringbuf { + public: + explicit WritableFileStringStreamAdapter(WritableFile* writable_file) + : file_(writable_file) {} + + // Override overflow() to handle `sputc()`. There are cases that will not go + // through `xsputn()` e.g. `std::endl` or an unsigned long long is written by + // `os.put()` directly and will call `sputc()` By internal implementation: + // int_type __CLR_OR_THIS_CALL sputc(_Elem _Ch) { // put a character + // return 0 < _Pnavail() ? _Traits::to_int_type(*_Pninc() = _Ch) : + // overflow(_Traits::to_int_type(_Ch)); + // } + // As we explicitly disabled buffering (_Pnavail() is always 0), every write, + // not captured by xsputn(), becomes an overflow here. + int overflow(int ch = EOF) override { + if (ch != EOF) { + Status s = file_->Append(Slice((char*)&ch, 1)); + if (s.ok()) { + return ch; + } + } + return EOF; + } + + std::streamsize xsputn(char const* p, std::streamsize n) override { + Status s = file_->Append(Slice(p, n)); + if (!s.ok()) { + return 0; + } + return n; + } + + private: + WritableFile* file_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/block_based_table_reader_impl.h b/librocksdb-sys/rocksdb/table/block_based/block_based_table_reader_impl.h new file mode 100644 index 0000000..801b461 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block_based_table_reader_impl.h @@ -0,0 +1,203 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once +#include + +#include "block.h" +#include "block_cache.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/block_based/reader_common.h" + +// The file contains some member functions of BlockBasedTable that +// cannot be implemented in block_based_table_reader.cc because +// it's called by other files (e.g. block_based_iterator.h) and +// are templates. + +namespace ROCKSDB_NAMESPACE { +namespace { +using IterPlaceholderCacheInterface = + PlaceholderCacheInterface; + +template +struct IterTraits {}; + +template <> +struct IterTraits { + using IterBlocklike = Block_kData; +}; + +template <> +struct IterTraits { + using IterBlocklike = Block_kIndex; +}; + +} // namespace + +// Convert an index iterator value (i.e., an encoded BlockHandle) +// into an iterator over the contents of the corresponding block. +// If input_iter is null, new a iterator +// If input_iter is not null, update this iter and return it +template +TBlockIter* BlockBasedTable::NewDataBlockIterator( + const ReadOptions& ro, const BlockHandle& handle, TBlockIter* input_iter, + BlockType block_type, GetContext* get_context, + BlockCacheLookupContext* lookup_context, + FilePrefetchBuffer* prefetch_buffer, bool for_compaction, bool async_read, + Status& s) const { + using IterBlocklike = typename IterTraits::IterBlocklike; + PERF_TIMER_GUARD(new_table_block_iter_nanos); + + TBlockIter* iter = input_iter != nullptr ? input_iter : new TBlockIter; + if (!s.ok()) { + iter->Invalidate(s); + return iter; + } + + CachableEntry block; + if (rep_->uncompression_dict_reader && block_type == BlockType::kData) { + CachableEntry uncompression_dict; + const bool no_io = (ro.read_tier == kBlockCacheTier); + // For async scans, don't use the prefetch buffer since an async prefetch + // might already be under way and this would invalidate it. Also, the + // uncompression dict is typically at the end of the file and would + // most likely break the sequentiality of the access pattern. + s = rep_->uncompression_dict_reader->GetOrReadUncompressionDictionary( + ro.async_io ? nullptr : prefetch_buffer, ro, no_io, ro.verify_checksums, + get_context, lookup_context, &uncompression_dict); + if (!s.ok()) { + iter->Invalidate(s); + return iter; + } + const UncompressionDict& dict = uncompression_dict.GetValue() + ? *uncompression_dict.GetValue() + : UncompressionDict::GetEmptyDict(); + s = RetrieveBlock(prefetch_buffer, ro, handle, dict, + &block.As(), get_context, lookup_context, + for_compaction, + /* use_cache */ true, async_read); + } else { + s = RetrieveBlock( + prefetch_buffer, ro, handle, UncompressionDict::GetEmptyDict(), + &block.As(), get_context, lookup_context, for_compaction, + /* use_cache */ true, async_read); + } + + if (s.IsTryAgain() && async_read) { + return iter; + } + + if (!s.ok()) { + assert(block.IsEmpty()); + iter->Invalidate(s); + return iter; + } + + assert(block.GetValue() != nullptr); + + // Block contents are pinned and it is still pinned after the iterator + // is destroyed as long as cleanup functions are moved to another object, + // when: + // 1. block cache handle is set to be released in cleanup function, or + // 2. it's pointing to immortal source. If own_bytes is true then we are + // not reading data from the original source, whether immortal or not. + // Otherwise, the block is pinned iff the source is immortal. + const bool block_contents_pinned = + block.IsCached() || + (!block.GetValue()->own_bytes() && rep_->immortal_table); + iter = InitBlockIterator(rep_, block.GetValue(), block_type, iter, + block_contents_pinned); + + if (!block.IsCached()) { + if (!ro.fill_cache) { + IterPlaceholderCacheInterface block_cache{ + rep_->table_options.block_cache.get()}; + if (block_cache) { + // insert a dummy record to block cache to track the memory usage + Cache::Handle* cache_handle = nullptr; + CacheKey key = + CacheKey::CreateUniqueForCacheLifetime(block_cache.get()); + s = block_cache.Insert(key.AsSlice(), + block.GetValue()->ApproximateMemoryUsage(), + &cache_handle); + + if (s.ok()) { + assert(cache_handle != nullptr); + iter->RegisterCleanup(&ForceReleaseCachedEntry, block_cache.get(), + cache_handle); + } + } + } + } else { + iter->SetCacheHandle(block.GetCacheHandle()); + } + + block.TransferTo(iter); + + return iter; +} + +// Convert an uncompressed data block (i.e CachableEntry) +// into an iterator over the contents of the corresponding block. +// If input_iter is null, new a iterator +// If input_iter is not null, update this iter and return it +template +TBlockIter* BlockBasedTable::NewDataBlockIterator(const ReadOptions& ro, + CachableEntry& block, + TBlockIter* input_iter, + Status s) const { + PERF_TIMER_GUARD(new_table_block_iter_nanos); + + TBlockIter* iter = input_iter != nullptr ? input_iter : new TBlockIter; + if (!s.ok()) { + iter->Invalidate(s); + return iter; + } + + assert(block.GetValue() != nullptr); + // Block contents are pinned and it is still pinned after the iterator + // is destroyed as long as cleanup functions are moved to another object, + // when: + // 1. block cache handle is set to be released in cleanup function, or + // 2. it's pointing to immortal source. If own_bytes is true then we are + // not reading data from the original source, whether immortal or not. + // Otherwise, the block is pinned iff the source is immortal. + const bool block_contents_pinned = + block.IsCached() || + (!block.GetValue()->own_bytes() && rep_->immortal_table); + iter = InitBlockIterator(rep_, block.GetValue(), BlockType::kData, + iter, block_contents_pinned); + + if (!block.IsCached()) { + if (!ro.fill_cache) { + IterPlaceholderCacheInterface block_cache{ + rep_->table_options.block_cache.get()}; + if (block_cache) { + // insert a dummy record to block cache to track the memory usage + Cache::Handle* cache_handle = nullptr; + CacheKey key = + CacheKey::CreateUniqueForCacheLifetime(block_cache.get()); + s = block_cache.Insert(key.AsSlice(), + block.GetValue()->ApproximateMemoryUsage(), + &cache_handle); + + if (s.ok()) { + assert(cache_handle != nullptr); + iter->RegisterCleanup(&ForceReleaseCachedEntry, block_cache.get(), + cache_handle); + } + } + } + } else { + iter->SetCacheHandle(block.GetCacheHandle()); + } + + block.TransferTo(iter); + return iter; +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/block_based_table_reader_sync_and_async.h b/librocksdb-sys/rocksdb/table/block_based/block_based_table_reader_sync_and_async.h new file mode 100644 index 0000000..43af02f --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block_based_table_reader_sync_and_async.h @@ -0,0 +1,781 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "util/async_file_reader.h" +#include "util/coro_utils.h" + +#if defined(WITHOUT_COROUTINES) || \ + (defined(USE_COROUTINES) && defined(WITH_COROUTINES)) + +namespace ROCKSDB_NAMESPACE { + +// This function reads multiple data blocks from disk using Env::MultiRead() +// and optionally inserts them into the block cache. It uses the scratch +// buffer provided by the caller, which is contiguous. If scratch is a nullptr +// it allocates a separate buffer for each block. Typically, if the blocks +// need to be uncompressed and there is no compressed block cache, callers +// can allocate a temporary scratch buffer in order to minimize memory +// allocations. +// If options.fill_cache is true, it inserts the blocks into cache. If its +// false and scratch is non-null and the blocks are uncompressed, it copies +// the buffers to heap. In any case, the CachableEntry returned will +// own the data bytes. +// If compression is enabled and also there is no compressed block cache, +// the adjacent blocks are read out in one IO (combined read) +// batch - A MultiGetRange with only those keys with unique data blocks not +// found in cache +// handles - A vector of block handles. Some of them me be NULL handles +// scratch - An optional contiguous buffer to read compressed blocks into +DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks) +(const ReadOptions& options, const MultiGetRange* batch, + const autovector* handles, + Status* statuses, CachableEntry* results, char* scratch, + const UncompressionDict& uncompression_dict, bool use_fs_scratch) const { + RandomAccessFileReader* file = rep_->file.get(); + const Footer& footer = rep_->footer; + const ImmutableOptions& ioptions = rep_->ioptions; + size_t read_amp_bytes_per_bit = rep_->table_options.read_amp_bytes_per_bit; + MemoryAllocator* memory_allocator = GetMemoryAllocator(rep_->table_options); + + if (ioptions.allow_mmap_reads) { + size_t idx_in_batch = 0; + for (auto mget_iter = batch->begin(); mget_iter != batch->end(); + ++mget_iter, ++idx_in_batch) { + const BlockHandle& handle = (*handles)[idx_in_batch]; + if (handle.IsNull()) { + continue; + } + + // XXX: use_cache=true means double cache query? + statuses[idx_in_batch] = + RetrieveBlock(nullptr, options, handle, uncompression_dict, + &results[idx_in_batch].As(), + mget_iter->get_context, /* lookup_context */ nullptr, + /* for_compaction */ false, /* use_cache */ true, + /* async_read */ false); + } + assert(idx_in_batch == handles->size()); + CO_RETURN; + } + + // In direct IO mode, blocks share the direct io buffer. + // Otherwise, blocks share the scratch buffer. + const bool use_shared_buffer = file->use_direct_io() || scratch != nullptr; + + autovector read_reqs; + size_t buf_offset = 0; + size_t idx_in_batch = 0; + + uint64_t prev_offset = 0; + size_t prev_len = 0; + autovector req_idx_for_block; + autovector req_offset_for_block; + for (auto mget_iter = batch->begin(); mget_iter != batch->end(); + ++mget_iter, ++idx_in_batch) { + const BlockHandle& handle = (*handles)[idx_in_batch]; + if (handle.IsNull()) { + continue; + } + + size_t prev_end = static_cast(prev_offset) + prev_len; + + // If current block is adjacent to the previous one, at the same time, + // compression is enabled and there is no compressed cache, we combine + // the two block read as one. + // We don't combine block reads here in direct IO mode, because when doing + // direct IO read, the block requests will be realigned and merged when + // necessary. + if ((use_shared_buffer || use_fs_scratch) && !file->use_direct_io() && + prev_end == handle.offset()) { + req_offset_for_block.emplace_back(prev_len); + prev_len += BlockSizeWithTrailer(handle); + } else { + // No compression or current block and previous one is not adjacent: + // Step 1, create a new request for previous blocks + if (prev_len != 0) { + FSReadRequest req; + req.offset = prev_offset; + req.len = prev_len; + if (file->use_direct_io() || use_fs_scratch) { + req.scratch = nullptr; + } else if (use_shared_buffer) { + req.scratch = scratch + buf_offset; + buf_offset += req.len; + } else { + req.scratch = new char[req.len]; + } + read_reqs.emplace_back(std::move(req)); + } + + // Step 2, remember the previous block info + prev_offset = handle.offset(); + prev_len = BlockSizeWithTrailer(handle); + req_offset_for_block.emplace_back(0); + } + req_idx_for_block.emplace_back(read_reqs.size()); + + PERF_COUNTER_ADD(block_read_count, 1); + PERF_COUNTER_ADD(block_read_byte, BlockSizeWithTrailer(handle)); + } + // Handle the last block and process the pending last request + if (prev_len != 0) { + FSReadRequest req; + req.offset = prev_offset; + req.len = prev_len; + if (file->use_direct_io() || use_fs_scratch) { + req.scratch = nullptr; + } else if (use_shared_buffer) { + req.scratch = scratch + buf_offset; + } else { + req.scratch = new char[req.len]; + } + read_reqs.emplace_back(std::move(req)); + } + + AlignedBuf direct_io_buf; + { + IOOptions opts; + IOStatus s = file->PrepareIOOptions(options, opts); + if (s.ok()) { +#if defined(WITH_COROUTINES) + if (file->use_direct_io()) { +#endif // WITH_COROUTINES + s = file->MultiRead(opts, &read_reqs[0], read_reqs.size(), + &direct_io_buf, options.rate_limiter_priority); +#if defined(WITH_COROUTINES) + } else { + co_await batch->context()->reader().MultiReadAsync( + file, opts, &read_reqs[0], read_reqs.size(), &direct_io_buf); + } +#endif // WITH_COROUTINES + } + if (!s.ok()) { + // Discard all the results in this batch if there is any time out + // or overall MultiRead error + for (FSReadRequest& req : read_reqs) { + req.status = s; + } + } + } + + idx_in_batch = 0; + size_t valid_batch_idx = 0; + for (auto mget_iter = batch->begin(); mget_iter != batch->end(); + ++mget_iter, ++idx_in_batch) { + const BlockHandle& handle = (*handles)[idx_in_batch]; + + if (handle.IsNull()) { + continue; + } + + assert(valid_batch_idx < req_idx_for_block.size()); + assert(valid_batch_idx < req_offset_for_block.size()); + assert(req_idx_for_block[valid_batch_idx] < read_reqs.size()); + size_t& req_idx = req_idx_for_block[valid_batch_idx]; + size_t& req_offset = req_offset_for_block[valid_batch_idx]; + valid_batch_idx++; + FSReadRequest& req = read_reqs[req_idx]; + Status s = req.status; + if (s.ok()) { + if ((req.result.size() != req.len) || + (req_offset + BlockSizeWithTrailer(handle) > req.result.size())) { + s = Status::Corruption("truncated block read from " + + rep_->file->file_name() + " offset " + + std::to_string(handle.offset()) + ", expected " + + std::to_string(req.len) + " bytes, got " + + std::to_string(req.result.size())); + } + } + + BlockContents serialized_block; + if (s.ok()) { + if (use_fs_scratch) { + serialized_block = + BlockContents(Slice(req.result.data() + req_offset, handle.size())); + } else if (!use_shared_buffer) { + // We allocated a buffer for this block. Give ownership of it to + // BlockContents so it can free the memory + assert(req.result.data() == req.scratch); + assert(req.result.size() == BlockSizeWithTrailer(handle)); + assert(req_offset == 0); + serialized_block = + BlockContents(std::unique_ptr(req.scratch), handle.size()); + } else { + // We used the scratch buffer or direct io buffer + // which are shared by the blocks. + // serialized_block does not have the ownership. + serialized_block = + BlockContents(Slice(req.result.data() + req_offset, handle.size())); + } +#ifndef NDEBUG + serialized_block.has_trailer = true; +#endif + + if (options.verify_checksums) { + PERF_TIMER_GUARD(block_checksum_time); + const char* data = req.result.data(); + // Since the scratch might be shared, the offset of the data block in + // the buffer might not be 0. req.result.data() only point to the + // begin address of each read request, we need to add the offset + // in each read request. Checksum is stored in the block trailer, + // beyond the payload size. + s = VerifyBlockChecksum(footer, data + req_offset, handle.size(), + rep_->file->file_name(), handle.offset()); + TEST_SYNC_POINT_CALLBACK("RetrieveMultipleBlocks:VerifyChecksum", &s); + } + } else if (!use_shared_buffer) { + // Free the allocated scratch buffer. + delete[] req.scratch; + } + + if (s.ok()) { + // When the blocks share the same underlying buffer (scratch or direct io + // buffer), we may need to manually copy the block into heap if the + // serialized block has to be inserted into a cache. That falls into the + // following cases - + // 1. serialized block is not compressed, it needs to be inserted into + // the uncompressed block cache if there is one + // 2. If the serialized block is compressed, it needs to be inserted + // into the compressed block cache if there is one + // + // In all other cases, the serialized block is either uncompressed into a + // heap buffer or there is no cache at all. + CompressionType compression_type = + GetBlockCompressionType(serialized_block); + if ((use_fs_scratch || use_shared_buffer) && + compression_type == kNoCompression) { + Slice serialized = + Slice(req.result.data() + req_offset, BlockSizeWithTrailer(handle)); + serialized_block = BlockContents( + CopyBufferToHeap(GetMemoryAllocator(rep_->table_options), + serialized), + handle.size()); +#ifndef NDEBUG + serialized_block.has_trailer = true; +#endif + } + } + + if (s.ok()) { + if (options.fill_cache) { + CachableEntry* block_entry = &results[idx_in_batch]; + // MaybeReadBlockAndLoadToCache will insert into the block caches if + // necessary. Since we're passing the serialized block contents, it + // will avoid looking up the block cache + s = MaybeReadBlockAndLoadToCache( + nullptr, options, handle, uncompression_dict, + /*for_compaction=*/false, block_entry, mget_iter->get_context, + /*lookup_context=*/nullptr, &serialized_block, + /*async_read=*/false); + + // block_entry value could be null if no block cache is present, i.e + // BlockBasedTableOptions::no_block_cache is true and no compressed + // block cache is configured. In that case, fall + // through and set up the block explicitly + if (block_entry->GetValue() != nullptr) { + s.PermitUncheckedError(); + continue; + } + } + + CompressionType compression_type = + GetBlockCompressionType(serialized_block); + BlockContents contents; + if (compression_type != kNoCompression) { + UncompressionContext context(compression_type); + UncompressionInfo info(context, uncompression_dict, compression_type); + s = UncompressSerializedBlock( + info, req.result.data() + req_offset, handle.size(), &contents, + footer.format_version(), rep_->ioptions, memory_allocator); + } else { + // There are two cases here: + // 1) caller uses the shared buffer (scratch or direct io buffer); + // 2) we use the requst buffer. + // If scratch buffer or direct io buffer is used, we ensure that + // all serialized blocks are copyed to the heap as single blocks. If + // scratch buffer is not used, we also have no combined read, so the + // serialized block can be used directly. + contents = std::move(serialized_block); + } + if (s.ok()) { + results[idx_in_batch].SetOwnedValue(std::make_unique( + std::move(contents), read_amp_bytes_per_bit, ioptions.stats)); + } + } + statuses[idx_in_batch] = s; + } + + if (use_fs_scratch) { + // Free the allocated scratch buffer by fs here as read requests might have + // been combined into one. + for (FSReadRequest& req : read_reqs) { + if (req.fs_scratch != nullptr) { + req.fs_scratch.reset(); + req.fs_scratch = nullptr; + } + } + } +} + +using MultiGetRange = MultiGetContext::Range; +DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::MultiGet) +(const ReadOptions& read_options, const MultiGetRange* mget_range, + const SliceTransform* prefix_extractor, bool skip_filters) { + if (mget_range->empty()) { + // Caller should ensure non-empty (performance bug) + assert(false); + CO_RETURN; // Nothing to do + } + + FilterBlockReader* const filter = + !skip_filters ? rep_->filter.get() : nullptr; + MultiGetRange sst_file_range(*mget_range, mget_range->begin(), + mget_range->end()); + + // First check the full filter + // If full filter not useful, Then go into each block + const bool no_io = read_options.read_tier == kBlockCacheTier; + uint64_t tracing_mget_id = BlockCacheTraceHelper::kReservedGetId; + if (sst_file_range.begin()->get_context) { + tracing_mget_id = sst_file_range.begin()->get_context->get_tracing_get_id(); + } + // TODO: need more than one lookup_context here to track individual filter + // and index partition hits and misses. + BlockCacheLookupContext metadata_lookup_context{ + TableReaderCaller::kUserMultiGet, tracing_mget_id, + /*_get_from_user_specified_snapshot=*/read_options.snapshot != nullptr}; + FullFilterKeysMayMatch(filter, &sst_file_range, no_io, prefix_extractor, + &metadata_lookup_context, read_options); + + if (!sst_file_range.empty()) { + IndexBlockIter iiter_on_stack; + // if prefix_extractor found in block differs from options, disable + // BlockPrefixIndex. Only do this check when index_type is kHashSearch. + bool need_upper_bound_check = false; + if (rep_->index_type == BlockBasedTableOptions::kHashSearch) { + need_upper_bound_check = PrefixExtractorChanged(prefix_extractor); + } + auto iiter = NewIndexIterator( + read_options, need_upper_bound_check, &iiter_on_stack, + sst_file_range.begin()->get_context, &metadata_lookup_context); + std::unique_ptr> iiter_unique_ptr; + if (iiter != &iiter_on_stack) { + iiter_unique_ptr.reset(iiter); + } + + uint64_t prev_offset = std::numeric_limits::max(); + autovector block_handles; + std::array, MultiGetContext::MAX_BATCH_SIZE> + results; + std::array statuses; + // Empty data_lookup_contexts means "unused," when block cache tracing is + // disabled. (Limited options as element type is not default contructible.) + std::vector data_lookup_contexts; + MultiGetContext::Mask reused_mask = 0; + char stack_buf[kMultiGetReadStackBufSize]; + std::unique_ptr block_buf; + if (block_cache_tracer_ && block_cache_tracer_->is_tracing_enabled()) { + // Awkward because BlockCacheLookupContext is not CopyAssignable + data_lookup_contexts.reserve(MultiGetContext::MAX_BATCH_SIZE); + for (size_t i = 0; i < MultiGetContext::MAX_BATCH_SIZE; ++i) { + data_lookup_contexts.push_back(metadata_lookup_context); + } + } + { + MultiGetRange data_block_range(sst_file_range, sst_file_range.begin(), + sst_file_range.end()); + CachableEntry uncompression_dict; + Status uncompression_dict_status; + uncompression_dict_status.PermitUncheckedError(); + bool uncompression_dict_inited = false; + size_t total_len = 0; + + // GetContext for any key will do, as the stats will be aggregated + // anyway + GetContext* get_context = sst_file_range.begin()->get_context; + + { + using BCI = BlockCacheInterface; + BCI block_cache{rep_->table_options.block_cache.get()}; + std::array + async_handles; + std::array cache_keys; + size_t cache_lookup_count = 0; + + for (auto miter = data_block_range.begin(); + miter != data_block_range.end(); ++miter) { + const Slice& key = miter->ikey; + iiter->Seek(miter->ikey); + + IndexValue v; + if (iiter->Valid()) { + v = iiter->value(); + } + if (!iiter->Valid() || + (!v.first_internal_key.empty() && !skip_filters && + UserComparatorWrapper( + rep_->internal_comparator.user_comparator()) + .CompareWithoutTimestamp( + ExtractUserKey(key), + ExtractUserKey(v.first_internal_key)) < 0)) { + // The requested key falls between highest key in previous block and + // lowest key in current block. + if (!iiter->status().IsNotFound()) { + *(miter->s) = iiter->status(); + } + data_block_range.SkipKey(miter); + sst_file_range.SkipKey(miter); + continue; + } + + if (!uncompression_dict_inited && rep_->uncompression_dict_reader) { + uncompression_dict_status = + rep_->uncompression_dict_reader + ->GetOrReadUncompressionDictionary( + nullptr /* prefetch_buffer */, read_options, no_io, + read_options.verify_checksums, get_context, + &metadata_lookup_context, &uncompression_dict); + uncompression_dict_inited = true; + } + + if (!uncompression_dict_status.ok()) { + assert(!uncompression_dict_status.IsNotFound()); + *(miter->s) = uncompression_dict_status; + data_block_range.SkipKey(miter); + sst_file_range.SkipKey(miter); + continue; + } + + if (v.handle.offset() == prev_offset) { + // This key can reuse the previous block (later on). + // Mark previous as "reused" + reused_mask |= MultiGetContext::Mask{1} + << (block_handles.size() - 1); + // Use null handle to indicate this one reuses same block as + // previous. + block_handles.emplace_back(BlockHandle::NullBlockHandle()); + continue; + } + prev_offset = v.handle.offset(); + block_handles.emplace_back(v.handle); + + if (block_cache) { + // Lookup the cache for the given data block referenced by an index + // iterator value (i.e BlockHandle). If it exists in the cache, + // initialize block to the contents of the data block. + + // An async version of MaybeReadBlockAndLoadToCache / + // GetDataBlockFromCache + BCI::TypedAsyncLookupHandle& async_handle = + async_handles[cache_lookup_count]; + cache_keys[cache_lookup_count] = + GetCacheKey(rep_->base_cache_key, v.handle); + async_handle.key = cache_keys[cache_lookup_count].AsSlice(); + // NB: StartAsyncLookupFull populates async_handle.helper + async_handle.create_context = &rep_->create_context; + async_handle.priority = GetCachePriority(); + async_handle.stats = rep_->ioptions.statistics.get(); + + block_cache.StartAsyncLookupFull( + async_handle, rep_->ioptions.lowest_used_cache_tier); + ++cache_lookup_count; + // TODO: stats? + } + } + + if (block_cache) { + block_cache.get()->WaitAll(&async_handles[0], cache_lookup_count); + } + size_t lookup_idx = 0; + for (size_t i = 0; i < block_handles.size(); ++i) { + // If this block was a success or failure or not needed because + // the corresponding key is in the same block as a prior key, skip + if (block_handles[i] == BlockHandle::NullBlockHandle()) { + continue; + } + if (!block_cache) { + total_len += BlockSizeWithTrailer(block_handles[i]); + } else { + BCI::TypedHandle* h = async_handles[lookup_idx].Result(); + if (h) { + // Cache hit + results[i].SetCachedValue(block_cache.Value(h), block_cache.get(), + h); + // Don't need to fetch + block_handles[i] = BlockHandle::NullBlockHandle(); + UpdateCacheHitMetrics(BlockType::kData, get_context, + block_cache.get()->GetUsage(h)); + } else { + // Cache miss + total_len += BlockSizeWithTrailer(block_handles[i]); + UpdateCacheMissMetrics(BlockType::kData, get_context); + } + if (!data_lookup_contexts.empty()) { + // Populate cache key before it's discarded + data_lookup_contexts[i].block_key = + async_handles[lookup_idx].key.ToString(); + } + ++lookup_idx; + } + } + assert(lookup_idx == cache_lookup_count); + } + + if (total_len) { + char* scratch = nullptr; + bool use_fs_scratch = false; + const UncompressionDict& dict = uncompression_dict.GetValue() + ? *uncompression_dict.GetValue() + : UncompressionDict::GetEmptyDict(); + assert(uncompression_dict_inited || !rep_->uncompression_dict_reader); + assert(uncompression_dict_status.ok()); + + if (!rep_->file->use_direct_io()) { + if (CheckFSFeatureSupport(rep_->ioptions.fs.get(), + FSSupportedOps::kFSBuffer)) { + use_fs_scratch = true; + } + } + + // If using direct IO, then scratch is not used, so keep it nullptr. + // If the blocks need to be uncompressed and we don't need the + // compressed blocks, then we can use a contiguous block of + // memory to read in all the blocks as it will be temporary + // storage + // 1. If blocks are compressed and compressed block cache is there, + // alloc heap bufs + // 2. If blocks are uncompressed, alloc heap bufs + // 3. If blocks are compressed and no compressed block cache, use + // stack buf + if (!use_fs_scratch && !rep_->file->use_direct_io() && + rep_->blocks_maybe_compressed) { + if (total_len <= kMultiGetReadStackBufSize) { + scratch = stack_buf; + } else { + scratch = new char[total_len]; + block_buf.reset(scratch); + } + } + CO_AWAIT(RetrieveMultipleBlocks) + (read_options, &data_block_range, &block_handles, &statuses[0], + &results[0], scratch, dict, use_fs_scratch); + if (get_context) { + ++(get_context->get_context_stats_.num_sst_read); + } + } + } + + DataBlockIter first_biter; + DataBlockIter next_biter; + size_t idx_in_batch = 0; + SharedCleanablePtr shared_cleanable; + for (auto miter = sst_file_range.begin(); miter != sst_file_range.end(); + ++miter) { + Status s; + GetContext* get_context = miter->get_context; + const Slice& key = miter->ikey; + bool matched = false; // if such user key matched a key in SST + bool done = false; + bool first_block = true; + do { + DataBlockIter* biter = nullptr; + uint64_t referenced_data_size = 0; + Block_kData* parsed_block_value = nullptr; + bool reusing_prev_block; + bool later_reused; + bool does_referenced_key_exist = false; + bool handle_present = false; + BlockCacheLookupContext* lookup_data_block_context = + data_lookup_contexts.empty() ? nullptr + : &data_lookup_contexts[idx_in_batch]; + if (first_block) { + handle_present = !block_handles[idx_in_batch].IsNull(); + parsed_block_value = results[idx_in_batch].GetValue(); + if (handle_present || parsed_block_value) { + first_biter.Invalidate(Status::OK()); + NewDataBlockIterator( + read_options, results[idx_in_batch].As(), &first_biter, + statuses[idx_in_batch]); + reusing_prev_block = false; + } else { + // If handle is null and result is empty, then the status is never + // set, which should be the initial value: ok(). + assert(statuses[idx_in_batch].ok()); + reusing_prev_block = true; + } + biter = &first_biter; + later_reused = + (reused_mask & (MultiGetContext::Mask{1} << idx_in_batch)) != 0; + idx_in_batch++; + } else { + IndexValue v = iiter->value(); + if (!v.first_internal_key.empty() && !skip_filters && + UserComparatorWrapper(rep_->internal_comparator.user_comparator()) + .CompareWithoutTimestamp( + ExtractUserKey(key), + ExtractUserKey(v.first_internal_key)) < 0) { + // The requested key falls between highest key in previous block and + // lowest key in current block. + break; + } + + next_biter.Invalidate(Status::OK()); + Status tmp_s; + NewDataBlockIterator( + read_options, iiter->value().handle, &next_biter, + BlockType::kData, get_context, lookup_data_block_context, + /* prefetch_buffer= */ nullptr, /* for_compaction = */ false, + /*async_read = */ false, tmp_s); + biter = &next_biter; + reusing_prev_block = false; + later_reused = false; + } + + if (read_options.read_tier == kBlockCacheTier && + biter->status().IsIncomplete()) { + // couldn't get block from block_cache + // Update Saver.state to Found because we are only looking for + // whether we can guarantee the key is not there when "no_io" is set + get_context->MarkKeyMayExist(); + break; + } + if (!biter->status().ok()) { + s = biter->status(); + break; + } + + // Reusing blocks complicates pinning/Cleanable, because the cache + // entry referenced by biter can only be released once all returned + // pinned values are released. This code previously did an extra + // block_cache Ref for each reuse, but that unnecessarily increases + // block cache contention. Instead we can use a variant of shared_ptr + // to release in block cache only once. + // + // Although the biter loop below might SaveValue multiple times for + // merges, just one value_pinner suffices, as MultiGet will merge + // the operands before returning to the API user. + Cleanable* value_pinner; + if (biter->IsValuePinned()) { + if (reusing_prev_block) { + // Note that we don't yet know if the MultiGet results will need + // to pin this block, so we might wrap a block for sharing and + // still end up with 1 (or 0) pinning ref. Not ideal but OK. + // + // Here we avoid adding redundant cleanups if we didn't end up + // delegating the cleanup from last time around. + if (!biter->HasCleanups()) { + assert(shared_cleanable.get()); + if (later_reused) { + shared_cleanable.RegisterCopyWith(biter); + } else { + shared_cleanable.MoveAsCleanupTo(biter); + } + } + } else if (later_reused) { + assert(biter->HasCleanups()); + // Make the existing cleanups on `biter` sharable: + shared_cleanable.Allocate(); + // Move existing `biter` cleanup(s) to `shared_cleanable` + biter->DelegateCleanupsTo(&*shared_cleanable); + // Reference `shared_cleanable` as new cleanup for `biter` + shared_cleanable.RegisterCopyWith(biter); + } + assert(biter->HasCleanups()); + value_pinner = biter; + } else { + value_pinner = nullptr; + } + + bool may_exist = biter->SeekForGet(key); + if (!may_exist) { + // HashSeek cannot find the key this block and the the iter is not + // the end of the block, i.e. cannot be in the following blocks + // either. In this case, the seek_key cannot be found, so we break + // from the top level for-loop. + break; + } + + // Call the *saver function on each entry/block until it returns false + for (; biter->Valid(); biter->Next()) { + ParsedInternalKey parsed_key; + Status pik_status = ParseInternalKey( + biter->key(), &parsed_key, false /* log_err_key */); // TODO + if (!pik_status.ok()) { + s = pik_status; + } + if (!get_context->SaveValue(parsed_key, biter->value(), &matched, + value_pinner)) { + if (get_context->State() == GetContext::GetState::kFound) { + does_referenced_key_exist = true; + referenced_data_size = + biter->key().size() + biter->value().size(); + } + done = true; + break; + } + s = biter->status(); + } + // Write the block cache access. + // XXX: There appear to be 'break' statements above that bypass this + // writing of the block cache trace record + if (lookup_data_block_context && !reusing_prev_block && first_block) { + Slice referenced_key; + if (does_referenced_key_exist) { + referenced_key = biter->key(); + } else { + referenced_key = key; + } + + // block_key is self-assigned here (previously assigned from + // cache_keys / async_handles, now out of scope) + SaveLookupContextOrTraceRecord(lookup_data_block_context->block_key, + /*is_cache_hit=*/!handle_present, + read_options, parsed_block_value, + lookup_data_block_context); + FinishTraceRecord( + *lookup_data_block_context, lookup_data_block_context->block_key, + referenced_key, does_referenced_key_exist, referenced_data_size); + } + s = biter->status(); + if (done) { + // Avoid the extra Next which is expensive in two-level indexes + break; + } + if (first_block) { + iiter->Seek(key); + if (!iiter->Valid()) { + break; + } + } + first_block = false; + iiter->Next(); + } while (iiter->Valid()); + + if (matched && filter != nullptr) { + if (rep_->whole_key_filtering) { + RecordTick(rep_->ioptions.stats, BLOOM_FILTER_FULL_TRUE_POSITIVE); + } else { + RecordTick(rep_->ioptions.stats, BLOOM_FILTER_PREFIX_TRUE_POSITIVE); + } + // Includes prefix stats + PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_full_true_positive, 1, + rep_->level); + } + if (s.ok() && !iiter->status().IsNotFound()) { + s = iiter->status(); + } + *(miter->s) = s; + } +#ifdef ROCKSDB_ASSERT_STATUS_CHECKED + // Not sure why we need to do it. Should investigate more. + for (auto& st : statuses) { + st.PermitUncheckedError(); + } +#endif // ROCKSDB_ASSERT_STATUS_CHECKED + } +} +} // namespace ROCKSDB_NAMESPACE +#endif diff --git a/librocksdb-sys/rocksdb/table/block_based/block_based_table_reader_test.cc b/librocksdb-sys/rocksdb/table/block_based/block_based_table_reader_test.cc new file mode 100644 index 0000000..2aaf505 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block_based_table_reader_test.cc @@ -0,0 +1,735 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "table/block_based/block_based_table_reader.h" + +#include +#include +#include + +#include "cache/cache_reservation_manager.h" +#include "db/db_test_util.h" +#include "db/table_properties_collector.h" +#include "file/file_util.h" +#include "options/options_helper.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/compression_type.h" +#include "rocksdb/db.h" +#include "rocksdb/file_system.h" +#include "table/block_based/block_based_table_builder.h" +#include "table/block_based/block_based_table_factory.h" +#include "table/block_based/partitioned_index_iterator.h" +#include "table/format.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +class BlockBasedTableReaderBaseTest : public testing::Test { + protected: + // Prepare key-value pairs to occupy multiple blocks. + // Each value is 256B, every 16 pairs constitute 1 block. + // If mixed_with_human_readable_string_value == true, + // then adjacent blocks contain values with different compression + // complexity: human readable strings are easier to compress than random + // strings. + static std::map GenerateKVMap( + int num_block = 100, bool mixed_with_human_readable_string_value = false, + size_t ts_sz = 0) { + std::map kv; + + Random rnd(101); + uint32_t key = 0; + for (int block = 0; block < num_block; block++) { + for (int i = 0; i < 16; i++) { + char k[9] = {0}; + // Internal key is constructed directly from this key, + // and internal key size is required to be >= 8 bytes, + // so use %08u as the format string. + snprintf(k, sizeof(k), "%08u", key); + std::string v; + if (mixed_with_human_readable_string_value) { + v = (block % 2) ? rnd.HumanReadableString(256) + : rnd.RandomString(256); + } else { + v = rnd.RandomString(256); + } + if (ts_sz > 0) { + std::string user_key; + AppendKeyWithMinTimestamp(&user_key, std::string(k), ts_sz); + kv[user_key] = v; + } else { + kv[std::string(k)] = v; + } + key++; + } + } + return kv; + } + + void SetUp() override { + SetupSyncPointsToMockDirectIO(); + test_dir_ = test::PerThreadDBPath("block_based_table_reader_test"); + env_ = Env::Default(); + fs_ = FileSystem::Default(); + ASSERT_OK(fs_->CreateDir(test_dir_, IOOptions(), nullptr)); + ConfigureTableFactory(); + } + + virtual void ConfigureTableFactory() = 0; + + void TearDown() override { EXPECT_OK(DestroyDir(env_, test_dir_)); } + + // Creates a table with the specificied key value pairs (kv). + void CreateTable(const std::string& table_name, + const ImmutableOptions& ioptions, + const CompressionType& compression_type, + const std::map& kv, + uint32_t compression_parallel_threads = 1, + uint32_t compression_dict_bytes = 0) { + std::unique_ptr writer; + NewFileWriter(table_name, &writer); + + InternalKeyComparator comparator(ioptions.user_comparator); + ColumnFamilyOptions cf_options; + cf_options.prefix_extractor = options_.prefix_extractor; + MutableCFOptions moptions(cf_options); + CompressionOptions compression_opts; + compression_opts.parallel_threads = compression_parallel_threads; + // Enable compression dictionary and set a buffering limit that is the same + // as each block's size. + compression_opts.max_dict_bytes = compression_dict_bytes; + compression_opts.max_dict_buffer_bytes = compression_dict_bytes; + IntTblPropCollectorFactories factories; + std::unique_ptr table_builder( + options_.table_factory->NewTableBuilder( + TableBuilderOptions(ioptions, moptions, comparator, &factories, + compression_type, compression_opts, + 0 /* column_family_id */, + kDefaultColumnFamilyName, -1 /* level */), + writer.get())); + + // Build table. + for (auto it = kv.begin(); it != kv.end(); it++) { + std::string k = ToInternalKey(it->first); + std::string v = it->second; + table_builder->Add(k, v); + } + ASSERT_OK(table_builder->Finish()); + } + + void NewBlockBasedTableReader(const FileOptions& foptions, + const ImmutableOptions& ioptions, + const InternalKeyComparator& comparator, + const std::string& table_name, + std::unique_ptr* table, + bool prefetch_index_and_filter_in_cache = true, + Status* status = nullptr, + bool user_defined_timestamps_persisted = true) { + const MutableCFOptions moptions(options_); + TableReaderOptions table_reader_options = TableReaderOptions( + ioptions, moptions.prefix_extractor, EnvOptions(), comparator, + 0 /* block_protection_bytes_per_key */, false /* _skip_filters */, + false /* _immortal */, false /* _force_direct_prefetch */, + -1 /* _level */, nullptr /* _block_cache_tracer */, + 0 /* _max_file_size_for_l0_meta_pin */, "" /* _cur_db_session_id */, + 0 /* _cur_file_num */, {} /* _unique_id */, 0 /* _largest_seqno */, + 0 /* _tail_size */, user_defined_timestamps_persisted); + + std::unique_ptr file; + NewFileReader(table_name, foptions, &file); + + uint64_t file_size = 0; + ASSERT_OK(env_->GetFileSize(Path(table_name), &file_size)); + + ReadOptions read_opts; + read_opts.verify_checksums = true; + std::unique_ptr general_table; + Status s = options_.table_factory->NewTableReader( + read_opts, table_reader_options, std::move(file), file_size, + &general_table, prefetch_index_and_filter_in_cache); + + if (s.ok()) { + table->reset(reinterpret_cast(general_table.release())); + } + + if (status) { + *status = s; + } + } + + std::string Path(const std::string& fname) { return test_dir_ + "/" + fname; } + + std::string test_dir_; + Env* env_; + std::shared_ptr fs_; + Options options_; + + std::string ToInternalKey(const std::string& key) { + InternalKey internal_key(key, 0, ValueType::kTypeValue); + return internal_key.Encode().ToString(); + } + + private: + void WriteToFile(const std::string& content, const std::string& filename) { + std::unique_ptr f; + ASSERT_OK(fs_->NewWritableFile(Path(filename), FileOptions(), &f, nullptr)); + ASSERT_OK(f->Append(content, IOOptions(), nullptr)); + ASSERT_OK(f->Close(IOOptions(), nullptr)); + } + + void NewFileWriter(const std::string& filename, + std::unique_ptr* writer) { + std::string path = Path(filename); + EnvOptions env_options; + FileOptions foptions; + std::unique_ptr file; + ASSERT_OK(fs_->NewWritableFile(path, foptions, &file, nullptr)); + writer->reset(new WritableFileWriter(std::move(file), path, env_options)); + } + + void NewFileReader(const std::string& filename, const FileOptions& opt, + std::unique_ptr* reader) { + std::string path = Path(filename); + std::unique_ptr f; + ASSERT_OK(fs_->NewRandomAccessFile(path, opt, &f, nullptr)); + reader->reset(new RandomAccessFileReader(std::move(f), path, + env_->GetSystemClock().get())); + } +}; + +// Param 1: compression type +// Param 2: whether to use direct reads +// Param 3: Block Based Table Index type +// Param 4: BBTO no_block_cache option +// Param 5: test mode for the user-defined timestamp feature +// Param 6: number of parallel compression threads +// Param 7: CompressionOptions.max_dict_bytes and +// CompressionOptions.max_dict_buffer_bytes to enable/disable +// compression dictionary. +class BlockBasedTableReaderTest + : public BlockBasedTableReaderBaseTest, + public testing::WithParamInterface> { + protected: + void SetUp() override { + compression_type_ = std::get<0>(GetParam()); + use_direct_reads_ = std::get<1>(GetParam()); + test::UserDefinedTimestampTestMode udt_test_mode = std::get<4>(GetParam()); + udt_enabled_ = test::IsUDTEnabled(udt_test_mode); + persist_udt_ = test::ShouldPersistUDT(udt_test_mode); + compression_parallel_threads_ = std::get<5>(GetParam()); + compression_dict_bytes_ = std::get<6>(GetParam()); + BlockBasedTableReaderBaseTest::SetUp(); + } + + void ConfigureTableFactory() override { + BlockBasedTableOptions opts; + opts.index_type = std::get<2>(GetParam()); + opts.no_block_cache = std::get<3>(GetParam()); + opts.filter_policy.reset(NewBloomFilterPolicy(10, false)); + opts.partition_filters = + opts.index_type == + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + options_.table_factory.reset( + static_cast(NewBlockBasedTableFactory(opts))); + options_.prefix_extractor = + std::shared_ptr(NewFixedPrefixTransform(3)); + } + + CompressionType compression_type_; + bool use_direct_reads_; + bool udt_enabled_; + bool persist_udt_; + uint32_t compression_parallel_threads_; + uint32_t compression_dict_bytes_; +}; + +// Tests MultiGet in both direct IO and non-direct IO mode. +// The keys should be in cache after MultiGet. +TEST_P(BlockBasedTableReaderTest, MultiGet) { + Options options; + ReadOptions read_opts; + std::string dummy_ts(sizeof(uint64_t), '\0'); + Slice read_timestamp = dummy_ts; + if (udt_enabled_) { + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + read_opts.timestamp = &read_timestamp; + } + options.persist_user_defined_timestamps = persist_udt_; + size_t ts_sz = options.comparator->timestamp_size(); + std::map kv = + BlockBasedTableReaderBaseTest::GenerateKVMap( + 100 /* num_block */, + true /* mixed_with_human_readable_string_value */, ts_sz); + + // Prepare keys, values, and statuses for MultiGet. + autovector keys; + autovector keys_without_timestamps; + autovector values; + autovector statuses; + { + const int step = + static_cast(kv.size()) / MultiGetContext::MAX_BATCH_SIZE; + auto it = kv.begin(); + for (int i = 0; i < MultiGetContext::MAX_BATCH_SIZE; i++) { + keys.emplace_back(it->first); + if (ts_sz > 0) { + Slice ukey_without_ts = StripTimestampFromUserKey(it->first, ts_sz); + keys_without_timestamps.push_back(ukey_without_ts); + } else { + keys_without_timestamps.emplace_back(it->first); + } + values.emplace_back(); + statuses.emplace_back(); + std::advance(it, step); + } + } + + std::string table_name = "BlockBasedTableReaderTest_MultiGet" + + CompressionTypeToString(compression_type_); + + ImmutableOptions ioptions(options); + CreateTable(table_name, ioptions, compression_type_, kv, + compression_parallel_threads_, compression_dict_bytes_); + + std::unique_ptr table; + FileOptions foptions; + foptions.use_direct_reads = use_direct_reads_; + InternalKeyComparator comparator(options.comparator); + NewBlockBasedTableReader(foptions, ioptions, comparator, table_name, &table, + true /* bool prefetch_index_and_filter_in_cache */, + nullptr /* status */, persist_udt_); + + ASSERT_OK( + table->VerifyChecksum(read_opts, TableReaderCaller::kUserVerifyChecksum)); + + // Ensure that keys are not in cache before MultiGet. + for (auto& key : keys) { + std::string ikey = ToInternalKey(key.ToString()); + ASSERT_FALSE(table->TEST_KeyInCache(read_opts, ikey)); + } + + // Prepare MultiGetContext. + autovector get_context; + autovector key_context; + autovector sorted_keys; + for (size_t i = 0; i < keys.size(); ++i) { + get_context.emplace_back(options.comparator, nullptr, nullptr, nullptr, + GetContext::kNotFound, keys[i], &values[i], + nullptr, nullptr, nullptr, nullptr, + true /* do_merge */, nullptr, nullptr, nullptr, + nullptr, nullptr, nullptr); + key_context.emplace_back(nullptr, keys_without_timestamps[i], &values[i], + nullptr, nullptr, &statuses.back()); + key_context.back().get_context = &get_context.back(); + } + for (auto& key_ctx : key_context) { + sorted_keys.emplace_back(&key_ctx); + } + MultiGetContext ctx(&sorted_keys, 0, sorted_keys.size(), 0, read_opts, + fs_.get(), nullptr); + + // Execute MultiGet. + MultiGetContext::Range range = ctx.GetMultiGetRange(); + PerfContext* perf_ctx = get_perf_context(); + perf_ctx->Reset(); + table->MultiGet(read_opts, &range, nullptr); + + ASSERT_GE(perf_ctx->block_read_count - perf_ctx->index_block_read_count - + perf_ctx->filter_block_read_count - + perf_ctx->compression_dict_block_read_count, + 1); + ASSERT_GE(perf_ctx->block_read_byte, 1); + + for (const Status& status : statuses) { + ASSERT_OK(status); + } + // Check that keys are in cache after MultiGet. + for (size_t i = 0; i < keys.size(); i++) { + std::string ikey = ToInternalKey(keys[i].ToString()); + ASSERT_TRUE(table->TEST_KeyInCache(read_opts, ikey)); + ASSERT_EQ(values[i].ToString(), kv[keys[i].ToString()]); + } +} + +TEST_P(BlockBasedTableReaderTest, NewIterator) { + Options options; + ReadOptions read_opts; + std::string dummy_ts(sizeof(uint64_t), '\0'); + Slice read_timestamp = dummy_ts; + if (udt_enabled_) { + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + read_opts.timestamp = &read_timestamp; + } + options.persist_user_defined_timestamps = persist_udt_; + size_t ts_sz = options.comparator->timestamp_size(); + std::map kv = + BlockBasedTableReaderBaseTest::GenerateKVMap( + 100 /* num_block */, + true /* mixed_with_human_readable_string_value */, ts_sz); + + std::string table_name = "BlockBasedTableReaderTest_NewIterator" + + CompressionTypeToString(compression_type_); + + ImmutableOptions ioptions(options); + CreateTable(table_name, ioptions, compression_type_, kv, + compression_parallel_threads_, compression_dict_bytes_); + + std::unique_ptr table; + FileOptions foptions; + foptions.use_direct_reads = use_direct_reads_; + InternalKeyComparator comparator(options.comparator); + NewBlockBasedTableReader(foptions, ioptions, comparator, table_name, &table, + true /* bool prefetch_index_and_filter_in_cache */, + nullptr /* status */, persist_udt_); + ASSERT_OK( + table->VerifyChecksum(read_opts, TableReaderCaller::kUserVerifyChecksum)); + + std::unique_ptr iter; + iter.reset(table->NewIterator( + read_opts, options_.prefix_extractor.get(), /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized)); + + // Test forward scan. + ASSERT_TRUE(!iter->Valid()); + iter->SeekToFirst(); + ASSERT_OK(iter->status()); + for (auto kv_iter = kv.begin(); kv_iter != kv.end(); kv_iter++) { + std::string ikey = ToInternalKey(kv_iter->first); + ASSERT_EQ(iter->key().ToString(), ikey); + ASSERT_EQ(iter->value().ToString(), kv_iter->second); + iter->Next(); + ASSERT_OK(iter->status()); + } + ASSERT_TRUE(!iter->Valid()); + ASSERT_OK(iter->status()); + + // Test backward scan. + iter->SeekToLast(); + ASSERT_OK(iter->status()); + for (auto kv_iter = kv.rbegin(); kv_iter != kv.rend(); kv_iter++) { + std::string ikey = ToInternalKey(kv_iter->first); + ASSERT_EQ(iter->key().ToString(), ikey); + ASSERT_EQ(iter->value().ToString(), kv_iter->second); + iter->Prev(); + ASSERT_OK(iter->status()); + } + ASSERT_TRUE(!iter->Valid()); + ASSERT_OK(iter->status()); +} + +class ChargeTableReaderTest + : public BlockBasedTableReaderBaseTest, + public testing::WithParamInterface< + CacheEntryRoleOptions::Decision /* charge_table_reader_mem */> { + protected: + static std::size_t CalculateMaxTableReaderNumBeforeCacheFull( + std::size_t cache_capacity, std::size_t approx_table_reader_mem) { + // To make calculation easier for testing + assert(cache_capacity % CacheReservationManagerImpl< + CacheEntryRole::kBlockBasedTableReader>:: + GetDummyEntrySize() == + 0 && + cache_capacity >= 2 * CacheReservationManagerImpl< + CacheEntryRole::kBlockBasedTableReader>:: + GetDummyEntrySize()); + + // We need to subtract 1 for max_num_dummy_entry to account for dummy + // entries' overhead, assumed the overhead is no greater than 1 dummy entry + // size + std::size_t max_num_dummy_entry = + (size_t)std::floor(( + 1.0 * cache_capacity / + CacheReservationManagerImpl< + CacheEntryRole::kBlockBasedTableReader>::GetDummyEntrySize())) - + 1; + std::size_t cache_capacity_rounded_to_dummy_entry_multiples = + max_num_dummy_entry * + CacheReservationManagerImpl< + CacheEntryRole::kBlockBasedTableReader>::GetDummyEntrySize(); + std::size_t max_table_reader_num_capped = static_cast( + std::floor(1.0 * cache_capacity_rounded_to_dummy_entry_multiples / + approx_table_reader_mem)); + + return max_table_reader_num_capped; + } + + void SetUp() override { + // To cache and re-use the same kv map and compression type in the test + // suite for elimiating variance caused by these two factors + kv_ = BlockBasedTableReaderBaseTest::GenerateKVMap(); + compression_type_ = CompressionType::kNoCompression; + + table_reader_charge_tracking_cache_ = std::make_shared< + TargetCacheChargeTrackingCache< + CacheEntryRole::kBlockBasedTableReader>>((NewLRUCache( + 4 * CacheReservationManagerImpl< + CacheEntryRole::kBlockBasedTableReader>::GetDummyEntrySize(), + 0 /* num_shard_bits */, true /* strict_capacity_limit */))); + + // To ApproximateTableReaderMem() without being affected by + // the feature of charging its memory, we turn off the feature + charge_table_reader_ = CacheEntryRoleOptions::Decision::kDisabled; + BlockBasedTableReaderBaseTest::SetUp(); + approx_table_reader_mem_ = ApproximateTableReaderMem(); + + // Now we condtionally turn on the feature to test + charge_table_reader_ = GetParam(); + ConfigureTableFactory(); + } + + void ConfigureTableFactory() override { + BlockBasedTableOptions table_options; + table_options.cache_usage_options.options_overrides.insert( + {CacheEntryRole::kBlockBasedTableReader, + {/*.charged = */ charge_table_reader_}}); + table_options.block_cache = table_reader_charge_tracking_cache_; + + table_options.cache_index_and_filter_blocks = false; + table_options.filter_policy.reset(NewBloomFilterPolicy(10, false)); + table_options.partition_filters = true; + table_options.index_type = BlockBasedTableOptions::kTwoLevelIndexSearch; + + options_.table_factory.reset(NewBlockBasedTableFactory(table_options)); + } + + CacheEntryRoleOptions::Decision charge_table_reader_; + std::shared_ptr< + TargetCacheChargeTrackingCache> + table_reader_charge_tracking_cache_; + std::size_t approx_table_reader_mem_; + std::map kv_; + CompressionType compression_type_; + + private: + std::size_t ApproximateTableReaderMem() { + std::size_t approx_table_reader_mem = 0; + + std::string table_name = "table_for_approx_table_reader_mem"; + ImmutableOptions ioptions(options_); + CreateTable(table_name, ioptions, compression_type_, kv_); + + std::unique_ptr table; + Status s; + NewBlockBasedTableReader( + FileOptions(), ImmutableOptions(options_), + InternalKeyComparator(options_.comparator), table_name, &table, + false /* prefetch_index_and_filter_in_cache */, &s); + assert(s.ok()); + + approx_table_reader_mem = table->ApproximateMemoryUsage(); + assert(approx_table_reader_mem > 0); + return approx_table_reader_mem; + } +}; + +INSTANTIATE_TEST_CASE_P( + ChargeTableReaderTest, ChargeTableReaderTest, + ::testing::Values(CacheEntryRoleOptions::Decision::kEnabled, + CacheEntryRoleOptions::Decision::kDisabled)); + +TEST_P(ChargeTableReaderTest, Basic) { + const std::size_t max_table_reader_num_capped = + ChargeTableReaderTest::CalculateMaxTableReaderNumBeforeCacheFull( + table_reader_charge_tracking_cache_->GetCapacity(), + approx_table_reader_mem_); + + // Acceptable estimtation errors coming from + // 1. overstimate max_table_reader_num_capped due to # dummy entries is high + // and results in metadata charge overhead greater than 1 dummy entry size + // (violating our assumption in calculating max_table_reader_num_capped) + // 2. overestimate/underestimate max_table_reader_num_capped due to the gap + // between ApproximateTableReaderMem() and actual table reader mem + std::size_t max_table_reader_num_capped_upper_bound = + (std::size_t)(max_table_reader_num_capped * 1.05); + std::size_t max_table_reader_num_capped_lower_bound = + (std::size_t)(max_table_reader_num_capped * 0.95); + std::size_t max_table_reader_num_uncapped = + (std::size_t)(max_table_reader_num_capped * 1.1); + ASSERT_GT(max_table_reader_num_uncapped, + max_table_reader_num_capped_upper_bound) + << "We need `max_table_reader_num_uncapped` > " + "`max_table_reader_num_capped_upper_bound` to differentiate cases " + "between " + "charge_table_reader_ == kDisabled and == kEnabled)"; + + Status s = Status::OK(); + std::size_t opened_table_reader_num = 0; + std::string table_name; + std::vector> tables; + ImmutableOptions ioptions(options_); + // Keep creating BlockBasedTableReader till hiting the memory limit based on + // cache capacity and creation fails (when charge_table_reader_ == + // kEnabled) or reaching a specfied big number of table readers (when + // charge_table_reader_ == kDisabled) + while (s.ok() && opened_table_reader_num < max_table_reader_num_uncapped) { + table_name = "table_" + std::to_string(opened_table_reader_num); + CreateTable(table_name, ioptions, compression_type_, kv_); + tables.push_back(std::unique_ptr()); + NewBlockBasedTableReader( + FileOptions(), ImmutableOptions(options_), + InternalKeyComparator(options_.comparator), table_name, &tables.back(), + false /* prefetch_index_and_filter_in_cache */, &s); + if (s.ok()) { + ++opened_table_reader_num; + } + } + + if (charge_table_reader_ == CacheEntryRoleOptions::Decision::kEnabled) { + EXPECT_TRUE(s.IsMemoryLimit()) << "s: " << s.ToString(); + EXPECT_TRUE(s.ToString().find( + kCacheEntryRoleToCamelString[static_cast( + CacheEntryRole::kBlockBasedTableReader)]) != + std::string::npos); + EXPECT_TRUE(s.ToString().find("memory limit based on cache capacity") != + std::string::npos); + + EXPECT_GE(opened_table_reader_num, max_table_reader_num_capped_lower_bound); + EXPECT_LE(opened_table_reader_num, max_table_reader_num_capped_upper_bound); + + std::size_t updated_max_table_reader_num_capped = + ChargeTableReaderTest::CalculateMaxTableReaderNumBeforeCacheFull( + table_reader_charge_tracking_cache_->GetCapacity() / 2, + approx_table_reader_mem_); + + // Keep deleting BlockBasedTableReader to lower down memory usage from the + // memory limit to make the next creation succeeds + while (opened_table_reader_num >= updated_max_table_reader_num_capped) { + tables.pop_back(); + --opened_table_reader_num; + } + table_name = "table_for_successful_table_reader_open"; + CreateTable(table_name, ioptions, compression_type_, kv_); + tables.push_back(std::unique_ptr()); + NewBlockBasedTableReader( + FileOptions(), ImmutableOptions(options_), + InternalKeyComparator(options_.comparator), table_name, &tables.back(), + false /* prefetch_index_and_filter_in_cache */, &s); + EXPECT_TRUE(s.ok()) << s.ToString(); + + tables.clear(); + EXPECT_EQ(table_reader_charge_tracking_cache_->GetCacheCharge(), 0); + } else { + EXPECT_TRUE(s.ok() && + opened_table_reader_num == max_table_reader_num_uncapped) + << "s: " << s.ToString() << " opened_table_reader_num: " + << std::to_string(opened_table_reader_num); + EXPECT_EQ(table_reader_charge_tracking_cache_->GetCacheCharge(), 0); + } +} + +class BlockBasedTableReaderTestVerifyChecksum + : public BlockBasedTableReaderTest { + public: + BlockBasedTableReaderTestVerifyChecksum() : BlockBasedTableReaderTest() {} +}; + +TEST_P(BlockBasedTableReaderTestVerifyChecksum, ChecksumMismatch) { + Options options; + ReadOptions read_opts; + std::string dummy_ts(sizeof(uint64_t), '\0'); + Slice read_timestamp = dummy_ts; + if (udt_enabled_) { + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + read_opts.timestamp = &read_timestamp; + } + options.persist_user_defined_timestamps = persist_udt_; + size_t ts_sz = options.comparator->timestamp_size(); + std::map kv = + BlockBasedTableReaderBaseTest::GenerateKVMap( + 800 /* num_block */, + false /* mixed_with_human_readable_string_value=*/, ts_sz); + + options.statistics = CreateDBStatistics(); + ImmutableOptions ioptions(options); + std::string table_name = + "BlockBasedTableReaderTest" + CompressionTypeToString(compression_type_); + CreateTable(table_name, ioptions, compression_type_, kv, + compression_parallel_threads_, compression_dict_bytes_); + + std::unique_ptr table; + FileOptions foptions; + foptions.use_direct_reads = use_direct_reads_; + InternalKeyComparator comparator(options.comparator); + NewBlockBasedTableReader(foptions, ioptions, comparator, table_name, &table, + true /* bool prefetch_index_and_filter_in_cache */, + nullptr /* status */, persist_udt_); + + // Use the top level iterator to find the offset/size of the first + // 2nd level index block and corrupt the block + IndexBlockIter iiter_on_stack; + BlockCacheLookupContext context{TableReaderCaller::kUserVerifyChecksum}; + InternalIteratorBase* iiter = table->NewIndexIterator( + read_opts, /*need_upper_bound_check=*/false, &iiter_on_stack, + /*get_context=*/nullptr, &context); + std::unique_ptr> iiter_unique_ptr; + if (iiter != &iiter_on_stack) { + iiter_unique_ptr = std::unique_ptr>(iiter); + } + ASSERT_OK(iiter->status()); + iiter->SeekToFirst(); + BlockHandle handle = static_cast(iiter) + ->index_iter_->value() + .handle; + table.reset(); + + // Corrupt the block pointed to by handle + ASSERT_OK(test::CorruptFile(options.env, Path(table_name), + static_cast(handle.offset()), 128)); + + NewBlockBasedTableReader(foptions, ioptions, comparator, table_name, &table, + true /* bool prefetch_index_and_filter_in_cache */, + nullptr /* status */, persist_udt_); + ASSERT_EQ(0, + options.statistics->getTickerCount(BLOCK_CHECKSUM_MISMATCH_COUNT)); + Status s = + table->VerifyChecksum(read_opts, TableReaderCaller::kUserVerifyChecksum); + ASSERT_EQ(1, + options.statistics->getTickerCount(BLOCK_CHECKSUM_MISMATCH_COUNT)); + ASSERT_EQ(s.code(), Status::kCorruption); +} + +// Param 1: compression type +// Param 2: whether to use direct reads +// Param 3: Block Based Table Index type, partitioned filters are also enabled +// when index type is kTwoLevelIndexSearch +// Param 4: BBTO no_block_cache option +// Param 5: test mode for the user-defined timestamp feature +// Param 6: number of parallel compression threads +// Param 7: CompressionOptions.max_dict_bytes and +// CompressionOptions.max_dict_buffer_bytes. This enable/disables +// compression dictionary. +INSTANTIATE_TEST_CASE_P( + BlockBasedTableReaderTest, BlockBasedTableReaderTest, + ::testing::Combine( + ::testing::ValuesIn(GetSupportedCompressions()), ::testing::Bool(), + ::testing::Values( + BlockBasedTableOptions::IndexType::kBinarySearch, + BlockBasedTableOptions::IndexType::kHashSearch, + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch, + BlockBasedTableOptions::IndexType::kBinarySearchWithFirstKey), + ::testing::Values(false), ::testing::ValuesIn(test::GetUDTTestModes()), + ::testing::Values(1, 2), ::testing::Values(0, 4096))); +INSTANTIATE_TEST_CASE_P( + VerifyChecksum, BlockBasedTableReaderTestVerifyChecksum, + ::testing::Combine( + ::testing::ValuesIn(GetSupportedCompressions()), + ::testing::Values(false), + ::testing::Values( + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch), + ::testing::Values(true), ::testing::ValuesIn(test::GetUDTTestModes()), + ::testing::Values(1, 2), ::testing::Values(0))); + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/table/block_based/block_builder.cc b/librocksdb-sys/rocksdb/table/block_based/block_builder.cc new file mode 100644 index 0000000..877df81 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block_builder.cc @@ -0,0 +1,267 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// BlockBuilder generates blocks where keys are prefix-compressed: +// +// When we store a key, we drop the prefix shared with the previous +// string. This helps reduce the space requirement significantly. +// Furthermore, once every K keys, we do not apply the prefix +// compression and store the entire key. We call this a "restart +// point". The tail end of the block stores the offsets of all of the +// restart points, and can be used to do a binary search when looking +// for a particular key. Values are stored as-is (without compression) +// immediately following the corresponding key. +// +// An entry for a particular key-value pair has the form: +// shared_bytes: varint32 +// unshared_bytes: varint32 +// value_length: varint32 +// key_delta: char[unshared_bytes] +// value: char[value_length] +// shared_bytes == 0 for restart points. +// +// The trailer of the block has the form: +// restarts: uint32[num_restarts] +// num_restarts: uint32 +// restarts[i] contains the offset within the block of the ith restart point. + +#include "table/block_based/block_builder.h" + +#include + +#include + +#include "db/dbformat.h" +#include "rocksdb/comparator.h" +#include "table/block_based/data_block_footer.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { + +BlockBuilder::BlockBuilder( + int block_restart_interval, bool use_delta_encoding, + bool use_value_delta_encoding, + BlockBasedTableOptions::DataBlockIndexType index_type, + double data_block_hash_table_util_ratio, size_t ts_sz, + bool persist_user_defined_timestamps, bool is_user_key) + : block_restart_interval_(block_restart_interval), + use_delta_encoding_(use_delta_encoding), + use_value_delta_encoding_(use_value_delta_encoding), + strip_ts_sz_(persist_user_defined_timestamps ? 0 : ts_sz), + is_user_key_(is_user_key), + restarts_(1, 0), // First restart point is at offset 0 + counter_(0), + finished_(false) { + switch (index_type) { + case BlockBasedTableOptions::kDataBlockBinarySearch: + break; + case BlockBasedTableOptions::kDataBlockBinaryAndHash: + data_block_hash_index_builder_.Initialize( + data_block_hash_table_util_ratio); + break; + default: + assert(0); + } + assert(block_restart_interval_ >= 1); + estimate_ = sizeof(uint32_t) + sizeof(uint32_t); +} + +void BlockBuilder::Reset() { + buffer_.clear(); + restarts_.resize(1); // First restart point is at offset 0 + assert(restarts_[0] == 0); + estimate_ = sizeof(uint32_t) + sizeof(uint32_t); + counter_ = 0; + finished_ = false; + last_key_.clear(); + if (data_block_hash_index_builder_.Valid()) { + data_block_hash_index_builder_.Reset(); + } +#ifndef NDEBUG + add_with_last_key_called_ = false; +#endif +} + +void BlockBuilder::SwapAndReset(std::string& buffer) { + std::swap(buffer_, buffer); + Reset(); +} + +size_t BlockBuilder::EstimateSizeAfterKV(const Slice& key, + const Slice& value) const { + size_t estimate = CurrentSizeEstimate(); + // Note: this is an imprecise estimate as it accounts for the whole key size + // instead of non-shared key size. + estimate += key.size(); + if (strip_ts_sz_ > 0) { + estimate -= strip_ts_sz_; + } + // In value delta encoding we estimate the value delta size as half the full + // value size since only the size field of block handle is encoded. + estimate += + !use_value_delta_encoding_ || (counter_ >= block_restart_interval_) + ? value.size() + : value.size() / 2; + + if (counter_ >= block_restart_interval_) { + estimate += sizeof(uint32_t); // a new restart entry. + } + + estimate += sizeof(int32_t); // varint for shared prefix length. + // Note: this is an imprecise estimate as we will have to encoded size, one + // for shared key and one for non-shared key. + estimate += VarintLength(key.size()); // varint for key length. + if (!use_value_delta_encoding_ || (counter_ >= block_restart_interval_)) { + estimate += VarintLength(value.size()); // varint for value length. + } + + return estimate; +} + +Slice BlockBuilder::Finish() { + // Append restart array + for (size_t i = 0; i < restarts_.size(); i++) { + PutFixed32(&buffer_, restarts_[i]); + } + + uint32_t num_restarts = static_cast(restarts_.size()); + BlockBasedTableOptions::DataBlockIndexType index_type = + BlockBasedTableOptions::kDataBlockBinarySearch; + if (data_block_hash_index_builder_.Valid() && + CurrentSizeEstimate() <= kMaxBlockSizeSupportedByHashIndex) { + data_block_hash_index_builder_.Finish(buffer_); + index_type = BlockBasedTableOptions::kDataBlockBinaryAndHash; + } + + // footer is a packed format of data_block_index_type and num_restarts + uint32_t block_footer = PackIndexTypeAndNumRestarts(index_type, num_restarts); + + PutFixed32(&buffer_, block_footer); + finished_ = true; + return Slice(buffer_); +} + +void BlockBuilder::Add(const Slice& key, const Slice& value, + const Slice* const delta_value) { + // Ensure no unsafe mixing of Add and AddWithLastKey + assert(!add_with_last_key_called_); + + AddWithLastKeyImpl(key, value, last_key_, delta_value, buffer_.size()); + if (use_delta_encoding_) { + // Update state + // We used to just copy the changed data, but it appears to be + // faster to just copy the whole thing. + last_key_.assign(key.data(), key.size()); + } +} + +void BlockBuilder::AddWithLastKey(const Slice& key, const Slice& value, + const Slice& last_key_param, + const Slice* const delta_value) { + // Ensure no unsafe mixing of Add and AddWithLastKey + assert(last_key_.empty()); +#ifndef NDEBUG + add_with_last_key_called_ = false; +#endif + + // Here we make sure to use an empty `last_key` on first call after creation + // or Reset. This is more convenient for the caller and we can be more + // clever inside BlockBuilder. On this hot code path, we want to avoid + // conditional jumps like `buffer_.empty() ? ... : ...` so we can use a + // fast arithmetic operation instead, with an assertion to be sure our logic + // is sound. + size_t buffer_size = buffer_.size(); + size_t last_key_size = last_key_param.size(); + assert(buffer_size == 0 || buffer_size >= last_key_size - strip_ts_sz_); + + Slice last_key(last_key_param.data(), last_key_size * (buffer_size > 0)); + + AddWithLastKeyImpl(key, value, last_key, delta_value, buffer_size); +} + +inline void BlockBuilder::AddWithLastKeyImpl(const Slice& key, + const Slice& value, + const Slice& last_key, + const Slice* const delta_value, + size_t buffer_size) { + assert(!finished_); + assert(counter_ <= block_restart_interval_); + assert(!use_value_delta_encoding_ || delta_value); + std::string key_buf; + std::string last_key_buf; + const Slice key_to_persist = MaybeStripTimestampFromKey(&key_buf, key); + // For delta key encoding, the first key in each restart interval doesn't have + // a last key to share bytes with. + const Slice last_key_persisted = + last_key.size() == 0 + ? last_key + : MaybeStripTimestampFromKey(&last_key_buf, last_key); + size_t shared = 0; // number of bytes shared with prev key + if (counter_ >= block_restart_interval_) { + // Restart compression + restarts_.push_back(static_cast(buffer_size)); + estimate_ += sizeof(uint32_t); + counter_ = 0; + } else if (use_delta_encoding_) { + // See how much sharing to do with previous string + shared = key_to_persist.difference_offset(last_key_persisted); + } + + const size_t non_shared = key_to_persist.size() - shared; + + if (use_value_delta_encoding_) { + // Add "" to buffer_ + PutVarint32Varint32(&buffer_, static_cast(shared), + static_cast(non_shared)); + } else { + // Add "" to buffer_ + PutVarint32Varint32Varint32(&buffer_, static_cast(shared), + static_cast(non_shared), + static_cast(value.size())); + } + + // Add string delta to buffer_ followed by value + buffer_.append(key_to_persist.data() + shared, non_shared); + // Use value delta encoding only when the key has shared bytes. This would + // simplify the decoding, where it can figure which decoding to use simply by + // looking at the shared bytes size. + if (shared != 0 && use_value_delta_encoding_) { + buffer_.append(delta_value->data(), delta_value->size()); + } else { + buffer_.append(value.data(), value.size()); + } + + // TODO(yuzhangyu): make user defined timestamp work with block hash index. + if (data_block_hash_index_builder_.Valid()) { + // Only data blocks should be using `kDataBlockBinaryAndHash` index type. + // And data blocks should always be built with internal keys instead of + // user keys. + assert(!is_user_key_); + data_block_hash_index_builder_.Add(ExtractUserKey(key), + restarts_.size() - 1); + } + + counter_++; + estimate_ += buffer_.size() - buffer_size; +} + +const Slice BlockBuilder::MaybeStripTimestampFromKey(std::string* key_buf, + const Slice& key) { + Slice stripped_key = key; + if (strip_ts_sz_ > 0) { + if (is_user_key_) { + stripped_key.remove_suffix(strip_ts_sz_); + } else { + StripTimestampFromInternalKey(key_buf, key, strip_ts_sz_); + stripped_key = *key_buf; + } + } + return stripped_key; +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/block_builder.h b/librocksdb-sys/rocksdb/table/block_based/block_builder.h new file mode 100644 index 0000000..f167470 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block_builder.h @@ -0,0 +1,128 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include + +#include + +#include "rocksdb/slice.h" +#include "rocksdb/table.h" +#include "table/block_based/data_block_hash_index.h" + +namespace ROCKSDB_NAMESPACE { + +class BlockBuilder { + public: + BlockBuilder(const BlockBuilder&) = delete; + void operator=(const BlockBuilder&) = delete; + + explicit BlockBuilder(int block_restart_interval, + bool use_delta_encoding = true, + bool use_value_delta_encoding = false, + BlockBasedTableOptions::DataBlockIndexType index_type = + BlockBasedTableOptions::kDataBlockBinarySearch, + double data_block_hash_table_util_ratio = 0.75, + size_t ts_sz = 0, + bool persist_user_defined_timestamps = true, + bool is_user_key = false); + + // Reset the contents as if the BlockBuilder was just constructed. + void Reset(); + + // Swap the contents in BlockBuilder with buffer, then reset the BlockBuilder. + void SwapAndReset(std::string& buffer); + + // REQUIRES: Finish() has not been called since the last call to Reset(). + // REQUIRES: Unless a range tombstone block, key is larger than any previously + // added key + // DO NOT mix with AddWithLastKey() between Resets. For efficiency, use + // AddWithLastKey() in contexts where previous added key is already known + // and delta encoding might be used. + void Add(const Slice& key, const Slice& value, + const Slice* const delta_value = nullptr); + + // A faster version of Add() if the previous key is already known for all + // Add()s. + // REQUIRES: Finish() has not been called since the last call to Reset(). + // REQUIRES: Unless a range tombstone block, key is larger than any previously + // added key + // REQUIRES: if AddWithLastKey has been called since last Reset(), last_key + // is the key from most recent AddWithLastKey. (For convenience, last_key + // is ignored on first call after creation or Reset().) + // DO NOT mix with Add() between Resets. + void AddWithLastKey(const Slice& key, const Slice& value, + const Slice& last_key, + const Slice* const delta_value = nullptr); + + // Finish building the block and return a slice that refers to the + // block contents. The returned slice will remain valid for the + // lifetime of this builder or until Reset() is called. + Slice Finish(); + + // Returns an estimate of the current (uncompressed) size of the block + // we are building. + inline size_t CurrentSizeEstimate() const { + return estimate_ + (data_block_hash_index_builder_.Valid() + ? data_block_hash_index_builder_.EstimateSize() + : 0); + } + + // Returns an estimated block size after appending key and value. + size_t EstimateSizeAfterKV(const Slice& key, const Slice& value) const; + + // Return true iff no entries have been added since the last Reset() + bool empty() const { return buffer_.empty(); } + + private: + inline void AddWithLastKeyImpl(const Slice& key, const Slice& value, + const Slice& last_key, + const Slice* const delta_value, + size_t buffer_size); + + inline const Slice MaybeStripTimestampFromKey(std::string* key_buf, + const Slice& key); + + const int block_restart_interval_; + // TODO(myabandeh): put it into a separate IndexBlockBuilder + const bool use_delta_encoding_; + // Refer to BlockIter::DecodeCurrentValue for format of delta encoded values + const bool use_value_delta_encoding_; + // Size in bytes for the user-defined timestamp to strip in a user key. + // This is non-zero if there is user-defined timestamp in the user key and it + // should not be persisted. + const size_t strip_ts_sz_; + // Whether the keys provided to build this block are user keys. If not, + // the keys are internal keys. This will affect how timestamp stripping is + // done for the key if `persisted_user_defined_timestamps_` is false and + // `ts_sz_` is non-zero. + // The timestamp stripping only applies to the keys added to the block. If the + // value contains user defined timestamp that needed to be stripped too, such + // as the `first_internal_key` in an `IndexValue` for an index block, the + // value part for a range deletion entry, their timestamp should be stripped + // before calling `BlockBuilder::Add`. + // Timestamp stripping only applies to data block and index blocks including + // index block for data blocks, index block for partitioned filter blocks, + // index block for partitioned index blocks. In summary, this only applies to + // block whose key are real user keys or internal keys created from user keys. + const bool is_user_key_; + + std::string buffer_; // Destination buffer + std::vector restarts_; // Restart points + size_t estimate_; + int counter_; // Number of entries emitted since restart + bool finished_; // Has Finish() been called? + std::string last_key_; + DataBlockHashIndexBuilder data_block_hash_index_builder_; +#ifndef NDEBUG + bool add_with_last_key_called_ = false; +#endif +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/block_cache.cc b/librocksdb-sys/rocksdb/table/block_based/block_cache.cc new file mode 100644 index 0000000..a252899 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block_cache.cc @@ -0,0 +1,106 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "table/block_based/block_cache.h" + +namespace ROCKSDB_NAMESPACE { + +void BlockCreateContext::Create(std::unique_ptr* parsed_out, + BlockContents&& block) { + parsed_out->reset(new Block_kData( + std::move(block), table_options->read_amp_bytes_per_bit, statistics)); + parsed_out->get()->InitializeDataBlockProtectionInfo(protection_bytes_per_key, + raw_ucmp); +} +void BlockCreateContext::Create(std::unique_ptr* parsed_out, + BlockContents&& block) { + parsed_out->reset(new Block_kIndex(std::move(block), + /*read_amp_bytes_per_bit*/ 0, statistics)); + parsed_out->get()->InitializeIndexBlockProtectionInfo( + protection_bytes_per_key, raw_ucmp, index_value_is_full, + index_has_first_key); +} +void BlockCreateContext::Create( + std::unique_ptr* parsed_out, + BlockContents&& block) { + parsed_out->reset(new Block_kFilterPartitionIndex( + std::move(block), /*read_amp_bytes_per_bit*/ 0, statistics)); + parsed_out->get()->InitializeIndexBlockProtectionInfo( + protection_bytes_per_key, raw_ucmp, index_value_is_full, + index_has_first_key); +} +void BlockCreateContext::Create( + std::unique_ptr* parsed_out, BlockContents&& block) { + parsed_out->reset(new Block_kRangeDeletion( + std::move(block), /*read_amp_bytes_per_bit*/ 0, statistics)); +} +void BlockCreateContext::Create(std::unique_ptr* parsed_out, + BlockContents&& block) { + parsed_out->reset(new Block_kMetaIndex( + std::move(block), /*read_amp_bytes_per_bit*/ 0, statistics)); + parsed_out->get()->InitializeMetaIndexBlockProtectionInfo( + protection_bytes_per_key); +} + +void BlockCreateContext::Create( + std::unique_ptr* parsed_out, BlockContents&& block) { + parsed_out->reset(new ParsedFullFilterBlock( + table_options->filter_policy.get(), std::move(block))); +} + +void BlockCreateContext::Create(std::unique_ptr* parsed_out, + BlockContents&& block) { + parsed_out->reset(new UncompressionDict( + block.data, std::move(block.allocation), using_zstd)); +} + +namespace { +// For getting SecondaryCache-compatible helpers from a BlockType. This is +// useful for accessing block cache in untyped contexts, such as for generic +// cache warming in table builder. +const std::array(BlockType::kInvalid) + 1> + kCacheItemFullHelperForBlockType{{ + BlockCacheInterface::GetFullHelper(), + BlockCacheInterface::GetFullHelper(), + BlockCacheInterface::GetFullHelper(), + nullptr, // kProperties + BlockCacheInterface::GetFullHelper(), + BlockCacheInterface::GetFullHelper(), + nullptr, // kHashIndexPrefixes + nullptr, // kHashIndexMetadata + nullptr, // kMetaIndex (not yet stored in block cache) + BlockCacheInterface::GetFullHelper(), + nullptr, // kInvalid + }}; + +// For getting basic helpers from a BlockType (no SecondaryCache support) +const std::array(BlockType::kInvalid) + 1> + kCacheItemBasicHelperForBlockType{{ + BlockCacheInterface::GetBasicHelper(), + BlockCacheInterface::GetBasicHelper(), + BlockCacheInterface::GetBasicHelper(), + nullptr, // kProperties + BlockCacheInterface::GetBasicHelper(), + BlockCacheInterface::GetBasicHelper(), + nullptr, // kHashIndexPrefixes + nullptr, // kHashIndexMetadata + nullptr, // kMetaIndex (not yet stored in block cache) + BlockCacheInterface::GetBasicHelper(), + nullptr, // kInvalid + }}; +} // namespace + +const Cache::CacheItemHelper* GetCacheItemHelper( + BlockType block_type, CacheTier lowest_used_cache_tier) { + if (lowest_used_cache_tier == CacheTier::kNonVolatileBlockTier) { + return kCacheItemFullHelperForBlockType[static_cast(block_type)]; + } else { + return kCacheItemBasicHelperForBlockType[static_cast(block_type)]; + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/block_cache.h b/librocksdb-sys/rocksdb/table/block_based/block_cache.h new file mode 100644 index 0000000..00eafac --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block_cache.h @@ -0,0 +1,140 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +// Code supporting block cache (Cache) access for block-based table, based on +// the convenient APIs in typed_cache.h + +#pragma once + +#include + +#include "cache/typed_cache.h" +#include "port/lang.h" +#include "table/block_based/block.h" +#include "table/block_based/block_type.h" +#include "table/block_based/parsed_full_filter_block.h" +#include "table/format.h" + +namespace ROCKSDB_NAMESPACE { + +// Metaprogramming wrappers for Block, to give each type a single role when +// used with FullTypedCacheInterface. +// (NOTE: previous attempts to create actual derived classes of Block with +// virtual calls resulted in performance regression) + +class Block_kData : public Block { + public: + using Block::Block; + + static constexpr CacheEntryRole kCacheEntryRole = CacheEntryRole::kDataBlock; + static constexpr BlockType kBlockType = BlockType::kData; +}; + +class Block_kIndex : public Block { + public: + using Block::Block; + + static constexpr CacheEntryRole kCacheEntryRole = CacheEntryRole::kIndexBlock; + static constexpr BlockType kBlockType = BlockType::kIndex; +}; + +class Block_kFilterPartitionIndex : public Block { + public: + using Block::Block; + + static constexpr CacheEntryRole kCacheEntryRole = + CacheEntryRole::kFilterMetaBlock; + static constexpr BlockType kBlockType = BlockType::kFilterPartitionIndex; +}; + +class Block_kRangeDeletion : public Block { + public: + using Block::Block; + + static constexpr CacheEntryRole kCacheEntryRole = CacheEntryRole::kOtherBlock; + static constexpr BlockType kBlockType = BlockType::kRangeDeletion; +}; + +// Useful for creating the Block even though meta index blocks are not +// yet stored in block cache +class Block_kMetaIndex : public Block { + public: + using Block::Block; + + static constexpr CacheEntryRole kCacheEntryRole = CacheEntryRole::kOtherBlock; + static constexpr BlockType kBlockType = BlockType::kMetaIndex; +}; + +struct BlockCreateContext : public Cache::CreateContext { + BlockCreateContext() {} + BlockCreateContext(const BlockBasedTableOptions* _table_options, + Statistics* _statistics, bool _using_zstd, + uint8_t _protection_bytes_per_key, + const Comparator* _raw_ucmp, + bool _index_value_is_full = false, + bool _index_has_first_key = false) + : table_options(_table_options), + statistics(_statistics), + using_zstd(_using_zstd), + protection_bytes_per_key(_protection_bytes_per_key), + raw_ucmp(_raw_ucmp), + index_value_is_full(_index_value_is_full), + index_has_first_key(_index_has_first_key) {} + + const BlockBasedTableOptions* table_options = nullptr; + Statistics* statistics = nullptr; + bool using_zstd = false; + uint8_t protection_bytes_per_key = 0; + const Comparator* raw_ucmp = nullptr; + bool index_value_is_full; + bool index_has_first_key; + + // For TypedCacheInterface + template + inline void Create(std::unique_ptr* parsed_out, + size_t* charge_out, const Slice& data, + MemoryAllocator* alloc) { + Create(parsed_out, + BlockContents(AllocateAndCopyBlock(data, alloc), data.size())); + *charge_out = parsed_out->get()->ApproximateMemoryUsage(); + } + + void Create(std::unique_ptr* parsed_out, BlockContents&& block); + void Create(std::unique_ptr* parsed_out, BlockContents&& block); + void Create(std::unique_ptr* parsed_out, + BlockContents&& block); + void Create(std::unique_ptr* parsed_out, + BlockContents&& block); + void Create(std::unique_ptr* parsed_out, + BlockContents&& block); + void Create(std::unique_ptr* parsed_out, + BlockContents&& block); + void Create(std::unique_ptr* parsed_out, + BlockContents&& block); +}; + +// Convenient cache interface to use for block_cache, with support for +// SecondaryCache. +template +using BlockCacheInterface = + FullTypedCacheInterface; + +// Shortcut name for cache handles under BlockCacheInterface +template +using BlockCacheTypedHandle = + typename BlockCacheInterface::TypedHandle; + +// Selects the right helper based on BlockType and CacheTier +const Cache::CacheItemHelper* GetCacheItemHelper( + BlockType block_type, + CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier); + +// For SFINAE check that a type is "blocklike" with a kCacheEntryRole member. +// Can get difficult compiler/linker errors without a good check like this. +template +using WithBlocklikeCheck = std::enable_if_t< + TBlocklike::kCacheEntryRole == CacheEntryRole::kMisc || true, TUse>; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/block_prefetcher.cc b/librocksdb-sys/rocksdb/table/block_based/block_prefetcher.cc new file mode 100644 index 0000000..1734e76 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block_prefetcher.cc @@ -0,0 +1,137 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "table/block_based/block_prefetcher.h" + +#include "rocksdb/file_system.h" +#include "table/block_based/block_based_table_reader.h" + +namespace ROCKSDB_NAMESPACE { +void BlockPrefetcher::PrefetchIfNeeded( + const BlockBasedTable::Rep* rep, const BlockHandle& handle, + const size_t readahead_size, bool is_for_compaction, + const bool no_sequential_checking, + const Env::IOPriority rate_limiter_priority) { + const size_t len = BlockBasedTable::BlockSizeWithTrailer(handle); + const size_t offset = handle.offset(); + + if (is_for_compaction) { + if (!rep->file->use_direct_io()) { + // If FS supports prefetching (readahead_limit_ will be non zero in that + // case) and current block exists in prefetch buffer then return. + if (offset + len <= readahead_limit_) { + return; + } + Status s = rep->file->Prefetch(offset, len + compaction_readahead_size_, + rate_limiter_priority); + if (s.ok()) { + readahead_limit_ = offset + len + compaction_readahead_size_; + return; + } + } + // If FS prefetch is not supported, fall back to use internal prefetch + // buffer. Discarding other return status of Prefetch calls intentionally, + // as we can fallback to reading from disk if Prefetch fails. + // + // num_file_reads is used by FilePrefetchBuffer only when + // implicit_auto_readahead is set. + rep->CreateFilePrefetchBufferIfNotExists( + compaction_readahead_size_, compaction_readahead_size_, + &prefetch_buffer_, /*implicit_auto_readahead=*/false, + /*num_file_reads=*/0, /*num_file_reads_for_auto_readahead=*/0); + return; + } + + // Explicit user requested readahead. + if (readahead_size > 0) { + rep->CreateFilePrefetchBufferIfNotExists( + readahead_size, readahead_size, &prefetch_buffer_, + /*implicit_auto_readahead=*/false, /*num_file_reads=*/0, + /*num_file_reads_for_auto_readahead=*/0); + return; + } + + // Implicit readahead. + + // If max_auto_readahead_size is set to be 0 by user, no data will be + // prefetched. + size_t max_auto_readahead_size = rep->table_options.max_auto_readahead_size; + if (max_auto_readahead_size == 0 || initial_auto_readahead_size_ == 0) { + return; + } + + if (initial_auto_readahead_size_ > max_auto_readahead_size) { + initial_auto_readahead_size_ = max_auto_readahead_size; + } + + // In case of no_sequential_checking, it will skip the num_file_reads_ and + // will always creates the FilePrefetchBuffer. + if (no_sequential_checking) { + rep->CreateFilePrefetchBufferIfNotExists( + initial_auto_readahead_size_, max_auto_readahead_size, + &prefetch_buffer_, /*implicit_auto_readahead=*/true, + /*num_file_reads=*/0, + rep->table_options.num_file_reads_for_auto_readahead); + return; + } + + // If FS supports prefetching (readahead_limit_ will be non zero in that case) + // and current block exists in prefetch buffer then return. + if (offset + len <= readahead_limit_) { + UpdateReadPattern(offset, len); + return; + } + + if (!IsBlockSequential(offset)) { + UpdateReadPattern(offset, len); + ResetValues(rep->table_options.initial_auto_readahead_size); + return; + } + UpdateReadPattern(offset, len); + + // Implicit auto readahead, which will be enabled if the number of reads + // reached `table_options.num_file_reads_for_auto_readahead` (default: 2) and + // scans are sequential. + num_file_reads_++; + if (num_file_reads_ <= rep->table_options.num_file_reads_for_auto_readahead) { + return; + } + + if (rep->file->use_direct_io()) { + rep->CreateFilePrefetchBufferIfNotExists( + initial_auto_readahead_size_, max_auto_readahead_size, + &prefetch_buffer_, /*implicit_auto_readahead=*/true, num_file_reads_, + rep->table_options.num_file_reads_for_auto_readahead); + return; + } + + if (readahead_size_ > max_auto_readahead_size) { + readahead_size_ = max_auto_readahead_size; + } + + // If prefetch is not supported, fall back to use internal prefetch buffer. + // Discarding other return status of Prefetch calls intentionally, as + // we can fallback to reading from disk if Prefetch fails. + Status s = rep->file->Prefetch( + handle.offset(), + BlockBasedTable::BlockSizeWithTrailer(handle) + readahead_size_, + rate_limiter_priority); + if (s.IsNotSupported()) { + rep->CreateFilePrefetchBufferIfNotExists( + initial_auto_readahead_size_, max_auto_readahead_size, + &prefetch_buffer_, /*implicit_auto_readahead=*/true, num_file_reads_, + rep->table_options.num_file_reads_for_auto_readahead); + return; + } + + readahead_limit_ = offset + len + readahead_size_; + // Keep exponentially increasing readahead size until + // max_auto_readahead_size. + readahead_size_ = std::min(max_auto_readahead_size, readahead_size_ * 2); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/block_prefetcher.h b/librocksdb-sys/rocksdb/table/block_based/block_prefetcher.h new file mode 100644 index 0000000..774bf85 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block_prefetcher.h @@ -0,0 +1,73 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once +#include "table/block_based/block_based_table_reader.h" + +namespace ROCKSDB_NAMESPACE { +class BlockPrefetcher { + public: + explicit BlockPrefetcher(size_t compaction_readahead_size, + size_t initial_auto_readahead_size) + : compaction_readahead_size_(compaction_readahead_size), + readahead_size_(initial_auto_readahead_size), + initial_auto_readahead_size_(initial_auto_readahead_size) {} + + void PrefetchIfNeeded(const BlockBasedTable::Rep* rep, + const BlockHandle& handle, size_t readahead_size, + bool is_for_compaction, + const bool no_sequential_checking, + Env::IOPriority rate_limiter_priority); + FilePrefetchBuffer* prefetch_buffer() { return prefetch_buffer_.get(); } + + void UpdateReadPattern(const uint64_t& offset, const size_t& len) { + prev_offset_ = offset; + prev_len_ = len; + } + + bool IsBlockSequential(const uint64_t& offset) { + return (prev_len_ == 0 || (prev_offset_ + prev_len_ == offset)); + } + + void ResetValues(size_t initial_auto_readahead_size) { + num_file_reads_ = 1; + // Since initial_auto_readahead_size_ can be different from + // the value passed to BlockBasedTableOptions.initial_auto_readahead_size in + // case of adaptive_readahead, so fallback the readahead_size_ to that value + // in case of reset. + initial_auto_readahead_size_ = initial_auto_readahead_size; + readahead_size_ = initial_auto_readahead_size_; + readahead_limit_ = 0; + return; + } + + void SetReadaheadState(ReadaheadFileInfo::ReadaheadInfo* readahead_info) { + num_file_reads_ = readahead_info->num_file_reads; + initial_auto_readahead_size_ = readahead_info->readahead_size; + TEST_SYNC_POINT_CALLBACK("BlockPrefetcher::SetReadaheadState", + &initial_auto_readahead_size_); + } + + private: + // Readahead size used in compaction, its value is used only if + // lookup_context_.caller = kCompaction. + size_t compaction_readahead_size_; + + // readahead_size_ is used in non-compaction read if underlying FS supports + // prefetching. + size_t readahead_size_; + size_t readahead_limit_ = 0; + // initial_auto_readahead_size_ is used in non-compaction read if RocksDB uses + // internal prefetch buffer. + uint64_t initial_auto_readahead_size_; + uint64_t num_file_reads_ = 0; + uint64_t prev_offset_ = 0; + size_t prev_len_ = 0; + std::unique_ptr prefetch_buffer_; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/block_prefix_index.cc b/librocksdb-sys/rocksdb/table/block_based/block_prefix_index.cc new file mode 100644 index 0000000..c83701d --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block_prefix_index.cc @@ -0,0 +1,226 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "table/block_based/block_prefix_index.h" + +#include + +#include "memory/arena.h" +#include "rocksdb/comparator.h" +#include "rocksdb/slice.h" +#include "rocksdb/slice_transform.h" +#include "util/coding.h" +#include "util/hash.h" + +namespace ROCKSDB_NAMESPACE { + +inline uint32_t Hash(const Slice& s) { + return ROCKSDB_NAMESPACE::Hash(s.data(), s.size(), 0); +} + +inline uint32_t PrefixToBucket(const Slice& prefix, uint32_t num_buckets) { + return Hash(prefix) % num_buckets; +} + +// The prefix block index is simply a bucket array, with each entry pointing to +// the blocks that span the prefixes hashed to this bucket. +// +// To reduce memory footprint, if there is only one block per bucket, the entry +// stores the block id directly. If there are more than one blocks per bucket, +// because of hash collision or a single prefix spanning multiple blocks, +// the entry points to an array of block ids. The block array is an array of +// uint32_t's. The first uint32_t indicates the total number of blocks, followed +// by the block ids. +// +// To differentiate the two cases, the high order bit of the entry indicates +// whether it is a 'pointer' into a separate block array. +// 0x7FFFFFFF is reserved for empty bucket. + +const uint32_t kNoneBlock = 0x7FFFFFFF; +const uint32_t kBlockArrayMask = 0x80000000; + +inline bool IsNone(uint32_t block_id) { return block_id == kNoneBlock; } + +inline bool IsBlockId(uint32_t block_id) { + return (block_id & kBlockArrayMask) == 0; +} + +inline uint32_t DecodeIndex(uint32_t block_id) { + uint32_t index = block_id ^ kBlockArrayMask; + assert(index < kBlockArrayMask); + return index; +} + +inline uint32_t EncodeIndex(uint32_t index) { + assert(index < kBlockArrayMask); + return index | kBlockArrayMask; +} + +// temporary storage for prefix information during index building +struct PrefixRecord { + Slice prefix; + uint32_t start_block; + uint32_t end_block; + uint32_t num_blocks; + PrefixRecord* next; +}; + +class BlockPrefixIndex::Builder { + public: + void Add(const Slice& key_prefix, uint32_t start_block, uint32_t num_blocks) { + PrefixRecord* record = reinterpret_cast( + arena_.AllocateAligned(sizeof(PrefixRecord))); + record->prefix = key_prefix; + record->start_block = start_block; + record->end_block = start_block + num_blocks - 1; + record->num_blocks = num_blocks; + prefixes_.push_back(record); + } + + BlockPrefixIndex* Finish(const SliceTransform* prefix_extractor) { + // For now, use roughly 1:1 prefix to bucket ratio. + uint32_t num_buckets = static_cast(prefixes_.size()) + 1; + + // Collect prefix records that hash to the same bucket, into a single + // linklist. + std::vector prefixes_per_bucket(num_buckets, nullptr); + std::vector num_blocks_per_bucket(num_buckets, 0); + for (PrefixRecord* current : prefixes_) { + uint32_t bucket = PrefixToBucket(current->prefix, num_buckets); + // merge the prefix block span if the first block of this prefix is + // connected to the last block of the previous prefix. + PrefixRecord* prev = prefixes_per_bucket[bucket]; + if (prev) { + assert(current->start_block >= prev->end_block); + auto distance = current->start_block - prev->end_block; + if (distance <= 1) { + prev->end_block = current->end_block; + prev->num_blocks = prev->end_block - prev->start_block + 1; + num_blocks_per_bucket[bucket] += (current->num_blocks + distance - 1); + continue; + } + } + current->next = prev; + prefixes_per_bucket[bucket] = current; + num_blocks_per_bucket[bucket] += current->num_blocks; + } + + // Calculate the block array buffer size + uint32_t total_block_array_entries = 0; + for (uint32_t i = 0; i < num_buckets; i++) { + uint32_t num_blocks = num_blocks_per_bucket[i]; + if (num_blocks > 1) { + total_block_array_entries += (num_blocks + 1); + } + } + + // Populate the final prefix block index + uint32_t* block_array_buffer = new uint32_t[total_block_array_entries]; + uint32_t* buckets = new uint32_t[num_buckets]; + uint32_t offset = 0; + for (uint32_t i = 0; i < num_buckets; i++) { + uint32_t num_blocks = num_blocks_per_bucket[i]; + if (num_blocks == 0) { + assert(prefixes_per_bucket[i] == nullptr); + buckets[i] = kNoneBlock; + } else if (num_blocks == 1) { + assert(prefixes_per_bucket[i] != nullptr); + assert(prefixes_per_bucket[i]->next == nullptr); + buckets[i] = prefixes_per_bucket[i]->start_block; + } else { + assert(total_block_array_entries > 0); + assert(prefixes_per_bucket[i] != nullptr); + buckets[i] = EncodeIndex(offset); + block_array_buffer[offset] = num_blocks; + uint32_t* last_block = &block_array_buffer[offset + num_blocks]; + auto current = prefixes_per_bucket[i]; + // populate block ids from largest to smallest + while (current != nullptr) { + for (uint32_t iter = 0; iter < current->num_blocks; iter++) { + *last_block = current->end_block - iter; + last_block--; + } + current = current->next; + } + assert(last_block == &block_array_buffer[offset]); + offset += (num_blocks + 1); + } + } + + assert(offset == total_block_array_entries); + + return new BlockPrefixIndex(prefix_extractor, num_buckets, buckets, + total_block_array_entries, block_array_buffer); + } + + private: + std::vector prefixes_; + Arena arena_; +}; + +Status BlockPrefixIndex::Create(const SliceTransform* prefix_extractor, + const Slice& prefixes, const Slice& prefix_meta, + BlockPrefixIndex** prefix_index) { + uint64_t pos = 0; + auto meta_pos = prefix_meta; + Status s; + Builder builder; + + while (!meta_pos.empty()) { + uint32_t prefix_size = 0; + uint32_t entry_index = 0; + uint32_t num_blocks = 0; + if (!GetVarint32(&meta_pos, &prefix_size) || + !GetVarint32(&meta_pos, &entry_index) || + !GetVarint32(&meta_pos, &num_blocks)) { + s = Status::Corruption( + "Corrupted prefix meta block: unable to read from it."); + break; + } + if (pos + prefix_size > prefixes.size()) { + s = Status::Corruption( + "Corrupted prefix meta block: size inconsistency."); + break; + } + Slice prefix(prefixes.data() + pos, prefix_size); + builder.Add(prefix, entry_index, num_blocks); + + pos += prefix_size; + } + + if (s.ok() && pos != prefixes.size()) { + s = Status::Corruption("Corrupted prefix meta block"); + } + + if (s.ok()) { + *prefix_index = builder.Finish(prefix_extractor); + } + + return s; +} + +uint32_t BlockPrefixIndex::GetBlocks(const Slice& key, uint32_t** blocks) { + Slice prefix = internal_prefix_extractor_.Transform(key); + + uint32_t bucket = PrefixToBucket(prefix, num_buckets_); + uint32_t block_id = buckets_[bucket]; + + if (IsNone(block_id)) { + return 0; + } else if (IsBlockId(block_id)) { + *blocks = &buckets_[bucket]; + return 1; + } else { + uint32_t index = DecodeIndex(block_id); + assert(index < num_block_array_buffer_entries_); + *blocks = &block_array_buffer_[index + 1]; + uint32_t num_blocks = block_array_buffer_[index]; + assert(num_blocks > 1); + assert(index + num_blocks < num_block_array_buffer_entries_); + return num_blocks; + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/block_prefix_index.h b/librocksdb-sys/rocksdb/table/block_based/block_prefix_index.h new file mode 100644 index 0000000..4db8e2c --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block_prefix_index.h @@ -0,0 +1,70 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include + +#include "db/dbformat.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +class Comparator; +class Iterator; +class Slice; +class SliceTransform; + +// Build a hash-based index to speed up the lookup for "index block". +// BlockHashIndex accepts a key and, if found, returns its restart index within +// that index block. +class BlockPrefixIndex { + public: + // Maps a key to a list of data blocks that could potentially contain + // the key, based on the prefix. + // Returns the total number of relevant blocks, 0 means the key does + // not exist. + uint32_t GetBlocks(const Slice& key, uint32_t** blocks); + + size_t ApproximateMemoryUsage() const { + return sizeof(BlockPrefixIndex) + + (num_block_array_buffer_entries_ + num_buckets_) * sizeof(uint32_t); + } + + // Create hash index by reading from the metadata blocks. + // Note: table reader (caller) is responsible for keeping shared_ptr to + // underlying prefix extractor + // @params prefixes: a sequence of prefixes. + // @params prefix_meta: contains the "metadata" to of the prefixes. + static Status Create(const SliceTransform* hash_key_extractor, + const Slice& prefixes, const Slice& prefix_meta, + BlockPrefixIndex** prefix_index); + + ~BlockPrefixIndex() { + delete[] buckets_; + delete[] block_array_buffer_; + } + + private: + class Builder; + friend Builder; + + BlockPrefixIndex(const SliceTransform* prefix_extractor, uint32_t num_buckets, + uint32_t* buckets, uint32_t num_block_array_buffer_entries, + uint32_t* block_array_buffer) + : internal_prefix_extractor_(prefix_extractor), + num_buckets_(num_buckets), + num_block_array_buffer_entries_(num_block_array_buffer_entries), + buckets_(buckets), + block_array_buffer_(block_array_buffer) {} + + InternalKeySliceTransform internal_prefix_extractor_; + + uint32_t num_buckets_; + uint32_t num_block_array_buffer_entries_; + uint32_t* buckets_; + uint32_t* block_array_buffer_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/block_test.cc b/librocksdb-sys/rocksdb/table/block_based/block_test.cc new file mode 100644 index 0000000..3264371 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block_test.cc @@ -0,0 +1,1683 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#include "table/block_based/block.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include "db/db_test_util.h" +#include "db/dbformat.h" +#include "db/memtable.h" +#include "db/write_batch_internal.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/iterator.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/table.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/block_based/block_builder.h" +#include "table/format.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +std::string GenerateInternalKey(int primary_key, int secondary_key, + int padding_size, Random *rnd, + size_t ts_sz = 0) { + char buf[50]; + char *p = &buf[0]; + snprintf(buf, sizeof(buf), "%6d%4d", primary_key, secondary_key); + std::string k(p); + if (padding_size) { + k += rnd->RandomString(padding_size); + } + AppendInternalKeyFooter(&k, 0 /* seqno */, kTypeValue); + std::string key_with_ts; + if (ts_sz > 0) { + PadInternalKeyWithMinTimestamp(&key_with_ts, k, ts_sz); + return key_with_ts; + } + + return k; +} + +// Generate random key value pairs. +// The generated key will be sorted. You can tune the parameters to generated +// different kinds of test key/value pairs for different scenario. +void GenerateRandomKVs(std::vector *keys, + std::vector *values, const int from, + const int len, const int step = 1, + const int padding_size = 0, + const int keys_share_prefix = 1, size_t ts_sz = 0) { + Random rnd(302); + + // generate different prefix + for (int i = from; i < from + len; i += step) { + // generating keys that shares the prefix + for (int j = 0; j < keys_share_prefix; ++j) { + // `DataBlockIter` assumes it reads only internal keys. + keys->emplace_back(GenerateInternalKey(i, j, padding_size, &rnd, ts_sz)); + + // 100 bytes values + values->emplace_back(rnd.RandomString(100)); + } + } +} + +// Test Param 1): key use delta encoding. +// Test Param 2): user-defined timestamp test mode. +// Test Param 3): data block index type. +class BlockTest : public testing::Test, + public testing::WithParamInterface< + std::tuple> { + public: + bool keyUseDeltaEncoding() const { return std::get<0>(GetParam()); } + bool isUDTEnabled() const { + return test::IsUDTEnabled(std::get<1>(GetParam())); + } + bool shouldPersistUDT() const { + return test::ShouldPersistUDT(std::get<1>(GetParam())); + } + + BlockBasedTableOptions::DataBlockIndexType dataBlockIndexType() const { + return std::get<2>(GetParam()); + } +}; + +// block test +TEST_P(BlockTest, SimpleTest) { + Random rnd(301); + Options options = Options(); + if (isUDTEnabled()) { + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + } + size_t ts_sz = options.comparator->timestamp_size(); + + std::vector keys; + std::vector values; + BlockBasedTableOptions::DataBlockIndexType index_type = + isUDTEnabled() ? BlockBasedTableOptions::kDataBlockBinarySearch + : dataBlockIndexType(); + BlockBuilder builder(16, keyUseDeltaEncoding(), + false /* use_value_delta_encoding */, index_type, + 0.75 /* data_block_hash_table_util_ratio */, ts_sz, + shouldPersistUDT(), false /* is_user_key */); + int num_records = 100000; + + GenerateRandomKVs(&keys, &values, 0, num_records, 1 /* step */, + 0 /* padding_size */, 1 /* keys_share_prefix */, ts_sz); + // add a bunch of records to a block + for (int i = 0; i < num_records; i++) { + builder.Add(keys[i], values[i]); + } + + // read serialized contents of the block + Slice rawblock = builder.Finish(); + + // create block reader + BlockContents contents; + contents.data = rawblock; + Block reader(std::move(contents)); + + // read contents of block sequentially + int count = 0; + InternalIterator *iter = reader.NewDataIterator( + options.comparator, kDisableGlobalSequenceNumber, nullptr /* iter */, + nullptr /* stats */, false /* block_contents_pinned */, + shouldPersistUDT()); + for (iter->SeekToFirst(); iter->Valid(); count++, iter->Next()) { + // read kv from block + Slice k = iter->key(); + Slice v = iter->value(); + + // compare with lookaside array + ASSERT_EQ(k.ToString().compare(keys[count]), 0); + ASSERT_EQ(v.ToString().compare(values[count]), 0); + } + delete iter; + + // read block contents randomly + iter = reader.NewDataIterator( + options.comparator, kDisableGlobalSequenceNumber, nullptr /* iter */, + nullptr /* stats */, false /* block_contents_pinned */, + shouldPersistUDT()); + for (int i = 0; i < num_records; i++) { + // find a random key in the lookaside array + int index = rnd.Uniform(num_records); + Slice k(keys[index]); + + // search in block for this key + iter->Seek(k); + ASSERT_TRUE(iter->Valid()); + Slice v = iter->value(); + ASSERT_EQ(v.ToString().compare(values[index]), 0); + } + delete iter; +} + +// return the block contents +BlockContents GetBlockContents( + std::unique_ptr *builder, + const std::vector &keys, + const std::vector &values, bool key_use_delta_encoding, + size_t ts_sz, bool should_persist_udt, const int /*prefix_group_size*/ = 1, + BlockBasedTableOptions::DataBlockIndexType dblock_index_type = + BlockBasedTableOptions::DataBlockIndexType::kDataBlockBinarySearch) { + builder->reset( + new BlockBuilder(1 /* restart interval */, key_use_delta_encoding, + false /* use_value_delta_encoding */, dblock_index_type, + 0.75 /* data_block_hash_table_util_ratio */, ts_sz, + should_persist_udt, false /* is_user_key */)); + + // Add only half of the keys + for (size_t i = 0; i < keys.size(); ++i) { + (*builder)->Add(keys[i], values[i]); + } + Slice rawblock = (*builder)->Finish(); + + BlockContents contents; + contents.data = rawblock; + + return contents; +} + +void CheckBlockContents(BlockContents contents, const int max_key, + const std::vector &keys, + const std::vector &values, + bool is_udt_enabled, bool should_persist_udt) { + const size_t prefix_size = 6; + // create block reader + BlockContents contents_ref(contents.data); + Block reader1(std::move(contents)); + Block reader2(std::move(contents_ref)); + + std::unique_ptr prefix_extractor( + NewFixedPrefixTransform(prefix_size)); + + std::unique_ptr regular_iter(reader2.NewDataIterator( + is_udt_enabled ? test::BytewiseComparatorWithU64TsWrapper() + : BytewiseComparator(), + kDisableGlobalSequenceNumber, nullptr /* iter */, nullptr /* stats */, + false /* block_contents_pinned */, should_persist_udt)); + + // Seek existent keys + for (size_t i = 0; i < keys.size(); i++) { + regular_iter->Seek(keys[i]); + ASSERT_OK(regular_iter->status()); + ASSERT_TRUE(regular_iter->Valid()); + + Slice v = regular_iter->value(); + ASSERT_EQ(v.ToString().compare(values[i]), 0); + } + + // Seek non-existent keys. + // For hash index, if no key with a given prefix is not found, iterator will + // simply be set as invalid; whereas the binary search based iterator will + // return the one that is closest. + for (int i = 1; i < max_key - 1; i += 2) { + // `DataBlockIter` assumes its APIs receive only internal keys. + auto key = GenerateInternalKey(i, 0, 0, nullptr, + is_udt_enabled ? 8 : 0 /* ts_sz */); + regular_iter->Seek(key); + ASSERT_TRUE(regular_iter->Valid()); + } +} + +// In this test case, no two key share same prefix. +TEST_P(BlockTest, SimpleIndexHash) { + const int kMaxKey = 100000; + size_t ts_sz = isUDTEnabled() ? 8 : 0; + std::vector keys; + std::vector values; + GenerateRandomKVs(&keys, &values, 0 /* first key id */, + kMaxKey /* last key id */, 2 /* step */, + 8 /* padding size (8 bytes randomly generated suffix) */, + 1 /* keys_share_prefix */, ts_sz); + + std::unique_ptr builder; + + auto contents = GetBlockContents( + &builder, keys, values, keyUseDeltaEncoding(), ts_sz, shouldPersistUDT(), + 1 /* prefix_group_size */, + isUDTEnabled() + ? BlockBasedTableOptions::DataBlockIndexType::kDataBlockBinarySearch + : dataBlockIndexType()); + + CheckBlockContents(std::move(contents), kMaxKey, keys, values, isUDTEnabled(), + shouldPersistUDT()); +} + +TEST_P(BlockTest, IndexHashWithSharedPrefix) { + const int kMaxKey = 100000; + // for each prefix, there will be 5 keys starts with it. + const int kPrefixGroup = 5; + size_t ts_sz = isUDTEnabled() ? 8 : 0; + std::vector keys; + std::vector values; + // Generate keys with same prefix. + GenerateRandomKVs(&keys, &values, 0, // first key id + kMaxKey, // last key id + 2 /* step */, + 10 /* padding size (8 bytes randomly generated suffix) */, + kPrefixGroup /* keys_share_prefix */, ts_sz); + + std::unique_ptr builder; + + auto contents = GetBlockContents( + &builder, keys, values, keyUseDeltaEncoding(), isUDTEnabled(), + shouldPersistUDT(), kPrefixGroup, + isUDTEnabled() + ? BlockBasedTableOptions::DataBlockIndexType::kDataBlockBinarySearch + : dataBlockIndexType()); + + CheckBlockContents(std::move(contents), kMaxKey, keys, values, isUDTEnabled(), + shouldPersistUDT()); +} + +// Param 0: key use delta encoding +// Param 1: user-defined timestamp test mode +// Param 2: data block index type. User-defined timestamp feature is not +// compatible with `kDataBlockBinaryAndHash` data block index type because the +// user comparator doesn't provide a `CanKeysWithDifferentByteContentsBeEqual` +// override. This combination is disabled. +INSTANTIATE_TEST_CASE_P( + P, BlockTest, + ::testing::Combine( + ::testing::Bool(), ::testing::ValuesIn(test::GetUDTTestModes()), + ::testing::Values( + BlockBasedTableOptions::DataBlockIndexType::kDataBlockBinarySearch, + BlockBasedTableOptions::DataBlockIndexType:: + kDataBlockBinaryAndHash))); + +// A slow and accurate version of BlockReadAmpBitmap that simply store +// all the marked ranges in a set. +class BlockReadAmpBitmapSlowAndAccurate { + public: + void Mark(size_t start_offset, size_t end_offset) { + assert(end_offset >= start_offset); + marked_ranges_.emplace(end_offset, start_offset); + } + + void ResetCheckSequence() { iter_valid_ = false; } + + // Return true if any byte in this range was Marked + // This does linear search from the previous position. When calling + // multiple times, `offset` needs to be incremental to get correct results. + // Call ResetCheckSequence() to reset it. + bool IsPinMarked(size_t offset) { + if (iter_valid_) { + // Has existing iterator, try linear search from + // the iterator. + for (int i = 0; i < 64; i++) { + if (offset < iter_->second) { + return false; + } + if (offset <= iter_->first) { + return true; + } + + iter_++; + if (iter_ == marked_ranges_.end()) { + iter_valid_ = false; + return false; + } + } + } + // Initial call or have linear searched too many times. + // Do binary search. + iter_ = marked_ranges_.lower_bound( + std::make_pair(offset, static_cast(0))); + if (iter_ == marked_ranges_.end()) { + iter_valid_ = false; + return false; + } + iter_valid_ = true; + return offset <= iter_->first && offset >= iter_->second; + } + + private: + std::set> marked_ranges_; + std::set>::iterator iter_; + bool iter_valid_ = false; +}; + +TEST_F(BlockTest, BlockReadAmpBitmap) { + uint32_t pin_offset = 0; + SyncPoint::GetInstance()->SetCallBack( + "BlockReadAmpBitmap:rnd", [&pin_offset](void *arg) { + pin_offset = *(static_cast(arg)); + }); + SyncPoint::GetInstance()->EnableProcessing(); + std::vector block_sizes = { + 1, // 1 byte + 32, // 32 bytes + 61, // 61 bytes + 64, // 64 bytes + 512, // 0.5 KB + 1024, // 1 KB + 1024 * 4, // 4 KB + 1024 * 10, // 10 KB + 1024 * 50, // 50 KB + 1024 * 1024 * 4, // 5 MB + 777, + 124653, + }; + const size_t kBytesPerBit = 64; + + Random rnd(301); + for (size_t block_size : block_sizes) { + std::shared_ptr stats = ROCKSDB_NAMESPACE::CreateDBStatistics(); + BlockReadAmpBitmap read_amp_bitmap(block_size, kBytesPerBit, stats.get()); + BlockReadAmpBitmapSlowAndAccurate read_amp_slow_and_accurate; + + size_t needed_bits = (block_size / kBytesPerBit); + if (block_size % kBytesPerBit != 0) { + needed_bits++; + } + + ASSERT_EQ(stats->getTickerCount(READ_AMP_TOTAL_READ_BYTES), block_size); + + // Generate some random entries + std::vector random_entry_offsets; + for (int i = 0; i < 1000; i++) { + random_entry_offsets.push_back(rnd.Next() % block_size); + } + std::sort(random_entry_offsets.begin(), random_entry_offsets.end()); + auto it = + std::unique(random_entry_offsets.begin(), random_entry_offsets.end()); + random_entry_offsets.resize( + std::distance(random_entry_offsets.begin(), it)); + + std::vector> random_entries; + for (size_t i = 0; i < random_entry_offsets.size(); i++) { + size_t entry_start = random_entry_offsets[i]; + size_t entry_end; + if (i + 1 < random_entry_offsets.size()) { + entry_end = random_entry_offsets[i + 1] - 1; + } else { + entry_end = block_size - 1; + } + random_entries.emplace_back(entry_start, entry_end); + } + + for (size_t i = 0; i < random_entries.size(); i++) { + read_amp_slow_and_accurate.ResetCheckSequence(); + auto ¤t_entry = random_entries[rnd.Next() % random_entries.size()]; + + read_amp_bitmap.Mark(static_cast(current_entry.first), + static_cast(current_entry.second)); + read_amp_slow_and_accurate.Mark(current_entry.first, + current_entry.second); + + size_t total_bits = 0; + for (size_t bit_idx = 0; bit_idx < needed_bits; bit_idx++) { + total_bits += read_amp_slow_and_accurate.IsPinMarked( + bit_idx * kBytesPerBit + pin_offset); + } + size_t expected_estimate_useful = total_bits * kBytesPerBit; + size_t got_estimate_useful = + stats->getTickerCount(READ_AMP_ESTIMATE_USEFUL_BYTES); + ASSERT_EQ(expected_estimate_useful, got_estimate_useful); + } + } + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(BlockTest, BlockWithReadAmpBitmap) { + Random rnd(301); + Options options = Options(); + + std::vector keys; + std::vector values; + BlockBuilder builder(16); + int num_records = 10000; + + GenerateRandomKVs(&keys, &values, 0, num_records, 1 /* step */); + // add a bunch of records to a block + for (int i = 0; i < num_records; i++) { + builder.Add(keys[i], values[i]); + } + + Slice rawblock = builder.Finish(); + const size_t kBytesPerBit = 8; + + // Read the block sequentially using Next() + { + std::shared_ptr stats = ROCKSDB_NAMESPACE::CreateDBStatistics(); + + // create block reader + BlockContents contents; + contents.data = rawblock; + Block reader(std::move(contents), kBytesPerBit, stats.get()); + + // read contents of block sequentially + size_t read_bytes = 0; + DataBlockIter *iter = reader.NewDataIterator( + options.comparator, kDisableGlobalSequenceNumber, nullptr, stats.get()); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + iter->value(); + read_bytes += iter->TEST_CurrentEntrySize(); + + double semi_acc_read_amp = + static_cast(read_bytes) / rawblock.size(); + double read_amp = static_cast(stats->getTickerCount( + READ_AMP_ESTIMATE_USEFUL_BYTES)) / + stats->getTickerCount(READ_AMP_TOTAL_READ_BYTES); + + // Error in read amplification will be less than 1% if we are reading + // sequentially + double error_pct = fabs(semi_acc_read_amp - read_amp) * 100; + EXPECT_LT(error_pct, 1); + } + + delete iter; + } + + // Read the block sequentially using Seek() + { + std::shared_ptr stats = ROCKSDB_NAMESPACE::CreateDBStatistics(); + + // create block reader + BlockContents contents; + contents.data = rawblock; + Block reader(std::move(contents), kBytesPerBit, stats.get()); + + size_t read_bytes = 0; + DataBlockIter *iter = reader.NewDataIterator( + options.comparator, kDisableGlobalSequenceNumber, nullptr, stats.get()); + for (int i = 0; i < num_records; i++) { + Slice k(keys[i]); + + // search in block for this key + iter->Seek(k); + iter->value(); + read_bytes += iter->TEST_CurrentEntrySize(); + + double semi_acc_read_amp = + static_cast(read_bytes) / rawblock.size(); + double read_amp = static_cast(stats->getTickerCount( + READ_AMP_ESTIMATE_USEFUL_BYTES)) / + stats->getTickerCount(READ_AMP_TOTAL_READ_BYTES); + + // Error in read amplification will be less than 1% if we are reading + // sequentially + double error_pct = fabs(semi_acc_read_amp - read_amp) * 100; + EXPECT_LT(error_pct, 1); + } + delete iter; + } + + // Read the block randomly + { + std::shared_ptr stats = ROCKSDB_NAMESPACE::CreateDBStatistics(); + + // create block reader + BlockContents contents; + contents.data = rawblock; + Block reader(std::move(contents), kBytesPerBit, stats.get()); + + size_t read_bytes = 0; + DataBlockIter *iter = reader.NewDataIterator( + options.comparator, kDisableGlobalSequenceNumber, nullptr, stats.get()); + std::unordered_set read_keys; + for (int i = 0; i < num_records; i++) { + int index = rnd.Uniform(num_records); + Slice k(keys[index]); + + iter->Seek(k); + iter->value(); + if (read_keys.find(index) == read_keys.end()) { + read_keys.insert(index); + read_bytes += iter->TEST_CurrentEntrySize(); + } + + double semi_acc_read_amp = + static_cast(read_bytes) / rawblock.size(); + double read_amp = static_cast(stats->getTickerCount( + READ_AMP_ESTIMATE_USEFUL_BYTES)) / + stats->getTickerCount(READ_AMP_TOTAL_READ_BYTES); + + double error_pct = fabs(semi_acc_read_amp - read_amp) * 100; + // Error in read amplification will be less than 2% if we are reading + // randomly + EXPECT_LT(error_pct, 2); + } + delete iter; + } +} + +TEST_F(BlockTest, ReadAmpBitmapPow2) { + std::shared_ptr stats = ROCKSDB_NAMESPACE::CreateDBStatistics(); + ASSERT_EQ(BlockReadAmpBitmap(100, 1, stats.get()).GetBytesPerBit(), 1u); + ASSERT_EQ(BlockReadAmpBitmap(100, 2, stats.get()).GetBytesPerBit(), 2u); + ASSERT_EQ(BlockReadAmpBitmap(100, 4, stats.get()).GetBytesPerBit(), 4u); + ASSERT_EQ(BlockReadAmpBitmap(100, 8, stats.get()).GetBytesPerBit(), 8u); + ASSERT_EQ(BlockReadAmpBitmap(100, 16, stats.get()).GetBytesPerBit(), 16u); + ASSERT_EQ(BlockReadAmpBitmap(100, 32, stats.get()).GetBytesPerBit(), 32u); + + ASSERT_EQ(BlockReadAmpBitmap(100, 3, stats.get()).GetBytesPerBit(), 2u); + ASSERT_EQ(BlockReadAmpBitmap(100, 7, stats.get()).GetBytesPerBit(), 4u); + ASSERT_EQ(BlockReadAmpBitmap(100, 11, stats.get()).GetBytesPerBit(), 8u); + ASSERT_EQ(BlockReadAmpBitmap(100, 17, stats.get()).GetBytesPerBit(), 16u); + ASSERT_EQ(BlockReadAmpBitmap(100, 33, stats.get()).GetBytesPerBit(), 32u); + ASSERT_EQ(BlockReadAmpBitmap(100, 35, stats.get()).GetBytesPerBit(), 32u); +} + +class IndexBlockTest + : public testing::Test, + public testing::WithParamInterface< + std::tuple> { + public: + IndexBlockTest() = default; + + bool keyIncludesSeq() const { return std::get<0>(GetParam()); } + bool useValueDeltaEncoding() const { return std::get<1>(GetParam()); } + bool includeFirstKey() const { return std::get<2>(GetParam()); } + bool isUDTEnabled() const { + return test::IsUDTEnabled(std::get<3>(GetParam())); + } + bool shouldPersistUDT() const { + return test::ShouldPersistUDT(std::get<3>(GetParam())); + } +}; + +// Similar to GenerateRandomKVs but for index block contents. +void GenerateRandomIndexEntries(std::vector *separators, + std::vector *block_handles, + std::vector *first_keys, + const int len, size_t ts_sz = 0, + bool zero_seqno = false) { + Random rnd(42); + + // For each of `len` blocks, we need to generate a first and last key. + // Let's generate n*2 random keys, sort them, group into consecutive pairs. + std::set keys; + while ((int)keys.size() < len * 2) { + // Keys need to be at least 8 bytes long to look like internal keys. + std::string new_key = test::RandomKey(&rnd, 12); + if (zero_seqno) { + AppendInternalKeyFooter(&new_key, 0 /* seqno */, kTypeValue); + } + if (ts_sz > 0) { + std::string key; + PadInternalKeyWithMinTimestamp(&key, new_key, ts_sz); + keys.insert(std::move(key)); + } else { + keys.insert(std::move(new_key)); + } + } + + uint64_t offset = 0; + for (auto it = keys.begin(); it != keys.end();) { + first_keys->emplace_back(*it++); + separators->emplace_back(*it++); + uint64_t size = rnd.Uniform(1024 * 16); + BlockHandle handle(offset, size); + offset += size + BlockBasedTable::kBlockTrailerSize; + block_handles->emplace_back(handle); + } +} + +TEST_P(IndexBlockTest, IndexValueEncodingTest) { + Random rnd(301); + Options options = Options(); + if (isUDTEnabled()) { + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + } + size_t ts_sz = options.comparator->timestamp_size(); + + std::vector separators; + std::vector block_handles; + std::vector first_keys; + const bool kUseDeltaEncoding = true; + BlockBuilder builder(16, kUseDeltaEncoding, useValueDeltaEncoding(), + BlockBasedTableOptions::kDataBlockBinarySearch, + 0.75 /* data_block_hash_table_util_ratio */, ts_sz, + shouldPersistUDT(), !keyIncludesSeq()); + + int num_records = 100; + + GenerateRandomIndexEntries(&separators, &block_handles, &first_keys, + num_records, ts_sz, false /* zero_seqno */); + BlockHandle last_encoded_handle; + for (int i = 0; i < num_records; i++) { + std::string first_key_to_persist_buf; + Slice first_internal_key = first_keys[i]; + if (ts_sz > 0 && !shouldPersistUDT()) { + StripTimestampFromInternalKey(&first_key_to_persist_buf, first_keys[i], + ts_sz); + first_internal_key = first_key_to_persist_buf; + } + IndexValue entry(block_handles[i], first_internal_key); + std::string encoded_entry; + std::string delta_encoded_entry; + entry.EncodeTo(&encoded_entry, includeFirstKey(), nullptr); + if (useValueDeltaEncoding() && i > 0) { + entry.EncodeTo(&delta_encoded_entry, includeFirstKey(), + &last_encoded_handle); + } + last_encoded_handle = entry.handle; + const Slice delta_encoded_entry_slice(delta_encoded_entry); + + if (keyIncludesSeq()) { + builder.Add(separators[i], encoded_entry, &delta_encoded_entry_slice); + } else { + const Slice user_key = ExtractUserKey(separators[i]); + builder.Add(user_key, encoded_entry, &delta_encoded_entry_slice); + } + } + + // read serialized contents of the block + Slice rawblock = builder.Finish(); + + // create block reader + BlockContents contents; + contents.data = rawblock; + Block reader(std::move(contents)); + + const bool kTotalOrderSeek = true; + IndexBlockIter *kNullIter = nullptr; + Statistics *kNullStats = nullptr; + // read contents of block sequentially + InternalIteratorBase *iter = reader.NewIndexIterator( + options.comparator, kDisableGlobalSequenceNumber, kNullIter, kNullStats, + kTotalOrderSeek, includeFirstKey(), keyIncludesSeq(), + !useValueDeltaEncoding(), false /* block_contents_pinned */, + shouldPersistUDT()); + iter->SeekToFirst(); + for (int index = 0; index < num_records; ++index) { + ASSERT_TRUE(iter->Valid()); + + Slice k = iter->key(); + IndexValue v = iter->value(); + + if (keyIncludesSeq()) { + EXPECT_EQ(separators[index], k.ToString()); + } else { + const Slice user_key = ExtractUserKey(separators[index]); + EXPECT_EQ(user_key, k); + } + EXPECT_EQ(block_handles[index].offset(), v.handle.offset()); + EXPECT_EQ(block_handles[index].size(), v.handle.size()); + EXPECT_EQ(includeFirstKey() ? first_keys[index] : "", + v.first_internal_key.ToString()); + + iter->Next(); + } + delete iter; + + // read block contents randomly + iter = reader.NewIndexIterator( + options.comparator, kDisableGlobalSequenceNumber, kNullIter, kNullStats, + kTotalOrderSeek, includeFirstKey(), keyIncludesSeq(), + !useValueDeltaEncoding(), false /* block_contents_pinned */, + shouldPersistUDT()); + for (int i = 0; i < num_records * 2; i++) { + // find a random key in the lookaside array + int index = rnd.Uniform(num_records); + Slice k(separators[index]); + + // search in block for this key + iter->Seek(k); + ASSERT_TRUE(iter->Valid()); + IndexValue v = iter->value(); + if (keyIncludesSeq()) { + EXPECT_EQ(separators[index], iter->key().ToString()); + } else { + const Slice user_key = ExtractUserKey(separators[index]); + EXPECT_EQ(user_key, iter->key()); + } + EXPECT_EQ(block_handles[index].offset(), v.handle.offset()); + EXPECT_EQ(block_handles[index].size(), v.handle.size()); + EXPECT_EQ(includeFirstKey() ? first_keys[index] : "", + v.first_internal_key.ToString()); + } + delete iter; +} + +// Param 0: key includes sequence number (whether to use user key or internal +// key as key entry in index block). +// Param 1: use value delta encoding +// Param 2: include first key +// Param 3: user-defined timestamp test mode +INSTANTIATE_TEST_CASE_P( + P, IndexBlockTest, + ::testing::Combine(::testing::Bool(), ::testing::Bool(), ::testing::Bool(), + ::testing::ValuesIn(test::GetUDTTestModes()))); + +class BlockPerKVChecksumTest : public DBTestBase { + public: + BlockPerKVChecksumTest() + : DBTestBase("block_per_kv_checksum", /*env_do_fsync=*/false) {} + + template + void TestIterateForward(std::unique_ptr &biter, + size_t &verification_count) { + while (biter->Valid()) { + verification_count = 0; + biter->Next(); + if (biter->Valid()) { + ASSERT_GE(verification_count, 1); + } + } + } + + template + void TestIterateBackward(std::unique_ptr &biter, + size_t &verification_count) { + while (biter->Valid()) { + verification_count = 0; + biter->Prev(); + if (biter->Valid()) { + ASSERT_GE(verification_count, 1); + } + } + } + + template + void TestSeekToFirst(std::unique_ptr &biter, + size_t &verification_count) { + verification_count = 0; + biter->SeekToFirst(); + ASSERT_GE(verification_count, 1); + TestIterateForward(biter, verification_count); + } + + template + void TestSeekToLast(std::unique_ptr &biter, + size_t &verification_count) { + verification_count = 0; + biter->SeekToLast(); + ASSERT_GE(verification_count, 1); + TestIterateBackward(biter, verification_count); + } + + template + void TestSeekForPrev(std::unique_ptr &biter, + size_t &verification_count, std::string k) { + verification_count = 0; + biter->SeekForPrev(k); + ASSERT_GE(verification_count, 1); + TestIterateBackward(biter, verification_count); + } + + template + void TestSeek(std::unique_ptr &biter, size_t &verification_count, + std::string k) { + verification_count = 0; + biter->Seek(k); + ASSERT_GE(verification_count, 1); + TestIterateForward(biter, verification_count); + } + + bool VerifyChecksum(uint32_t checksum_len, const char *checksum_ptr, + const Slice &key, const Slice &val) { + if (!checksum_len) { + return checksum_ptr == nullptr; + } + return ProtectionInfo64().ProtectKV(key, val).Verify( + static_cast(checksum_len), checksum_ptr); + } +}; + +TEST_F(BlockPerKVChecksumTest, EmptyBlock) { + // Tests that empty block code path is not broken by per kv checksum. + BlockBuilder builder( + 16 /* block_restart_interval */, true /* use_delta_encoding */, + false /* use_value_delta_encoding */, + BlockBasedTableOptions::DataBlockIndexType::kDataBlockBinarySearch); + Slice raw_block = builder.Finish(); + BlockContents contents; + contents.data = raw_block; + + std::unique_ptr data_block; + Options options = Options(); + BlockBasedTableOptions tbo; + uint8_t protection_bytes_per_key = 8; + BlockCreateContext create_context{ + &tbo, nullptr /* statistics */, false /* using_zstd */, + protection_bytes_per_key, options.comparator}; + create_context.Create(&data_block, std::move(contents)); + std::unique_ptr biter{data_block->NewDataIterator( + options.comparator, kDisableGlobalSequenceNumber)}; + biter->SeekToFirst(); + ASSERT_FALSE(biter->Valid()); + ASSERT_OK(biter->status()); + Random rnd(33); + biter->SeekForGet(GenerateInternalKey(1, 1, 10, &rnd)); + ASSERT_FALSE(biter->Valid()); + ASSERT_OK(biter->status()); + biter->SeekToLast(); + ASSERT_FALSE(biter->Valid()); + ASSERT_OK(biter->status()); + biter->Seek(GenerateInternalKey(1, 1, 10, &rnd)); + ASSERT_FALSE(biter->Valid()); + ASSERT_OK(biter->status()); + biter->SeekForPrev(GenerateInternalKey(1, 1, 10, &rnd)); + ASSERT_FALSE(biter->Valid()); + ASSERT_OK(biter->status()); +} + +TEST_F(BlockPerKVChecksumTest, UnsupportedOptionValue) { + Options options = Options(); + options.block_protection_bytes_per_key = 128; + Destroy(options); + ASSERT_TRUE(TryReopen(options).IsNotSupported()); +} + +TEST_F(BlockPerKVChecksumTest, InitializeProtectionInfo) { + // Make sure that the checksum construction code path does not break + // when the block is itself already corrupted. + Options options = Options(); + BlockBasedTableOptions tbo; + uint8_t protection_bytes_per_key = 8; + BlockCreateContext create_context{ + &tbo, nullptr /* statistics */, false /* using_zstd */, + protection_bytes_per_key, options.comparator}; + + { + std::string invalid_content = "1"; + Slice raw_block = invalid_content; + BlockContents contents; + contents.data = raw_block; + std::unique_ptr data_block; + create_context.Create(&data_block, std::move(contents)); + std::unique_ptr iter{data_block->NewDataIterator( + options.comparator, kDisableGlobalSequenceNumber)}; + ASSERT_TRUE(iter->status().IsCorruption()); + } + { + std::string invalid_content = "1"; + Slice raw_block = invalid_content; + BlockContents contents; + contents.data = raw_block; + std::unique_ptr index_block; + create_context.Create(&index_block, std::move(contents)); + std::unique_ptr iter{index_block->NewIndexIterator( + options.comparator, kDisableGlobalSequenceNumber, nullptr, nullptr, + true, false, true, true)}; + ASSERT_TRUE(iter->status().IsCorruption()); + } + { + std::string invalid_content = "1"; + Slice raw_block = invalid_content; + BlockContents contents; + contents.data = raw_block; + std::unique_ptr meta_block; + create_context.Create(&meta_block, std::move(contents)); + std::unique_ptr iter{meta_block->NewMetaIterator(true)}; + ASSERT_TRUE(iter->status().IsCorruption()); + } +} + +TEST_F(BlockPerKVChecksumTest, ApproximateMemory) { + // Tests that ApproximateMemoryUsage() includes memory used by block kv + // checksum. + const int kNumRecords = 20; + std::vector keys; + std::vector values; + GenerateRandomKVs(&keys, &values, 0, kNumRecords, 1 /* step */, + 24 /* padding_size */); + std::unique_ptr builder; + auto generate_block_content = [&]() { + builder = std::make_unique(16 /* restart_interval */); + for (int i = 0; i < kNumRecords; ++i) { + builder->Add(keys[i], values[i]); + } + Slice raw_block = builder->Finish(); + BlockContents contents; + contents.data = raw_block; + return contents; + }; + + Options options = Options(); + BlockBasedTableOptions tbo; + uint8_t protection_bytes_per_key = 8; + BlockCreateContext with_checksum_create_context{ + &tbo, + nullptr /* statistics */, + false /* using_zstd */, + protection_bytes_per_key, + options.comparator, + true /* index_value_is_full */}; + BlockCreateContext create_context{ + &tbo, nullptr /* statistics */, false /* using_zstd */, + 0, options.comparator, true /* index_value_is_full */}; + + { + std::unique_ptr data_block; + create_context.Create(&data_block, generate_block_content()); + size_t block_memory = data_block->ApproximateMemoryUsage(); + std::unique_ptr with_checksum_data_block; + with_checksum_create_context.Create(&with_checksum_data_block, + generate_block_content()); + ASSERT_GT(with_checksum_data_block->ApproximateMemoryUsage() - block_memory, + 100); + } + + { + std::unique_ptr meta_block; + create_context.Create(&meta_block, generate_block_content()); + size_t block_memory = meta_block->ApproximateMemoryUsage(); + std::unique_ptr with_checksum_meta_block; + with_checksum_create_context.Create(&with_checksum_meta_block, + generate_block_content()); + // Rough comparison to avoid flaky test due to memory allocation alignment. + ASSERT_GT(with_checksum_meta_block->ApproximateMemoryUsage() - block_memory, + 100); + } + + { + // Index block has different contents. + std::vector separators; + std::vector block_handles; + std::vector first_keys; + GenerateRandomIndexEntries(&separators, &block_handles, &first_keys, + kNumRecords); + auto generate_index_content = [&]() { + builder = std::make_unique(16 /* restart_interval */); + BlockHandle last_encoded_handle; + for (int i = 0; i < kNumRecords; ++i) { + IndexValue entry(block_handles[i], first_keys[i]); + std::string encoded_entry; + std::string delta_encoded_entry; + entry.EncodeTo(&encoded_entry, false, nullptr); + last_encoded_handle = entry.handle; + const Slice delta_encoded_entry_slice(delta_encoded_entry); + builder->Add(separators[i], encoded_entry, &delta_encoded_entry_slice); + } + Slice raw_block = builder->Finish(); + BlockContents contents; + contents.data = raw_block; + return contents; + }; + + std::unique_ptr index_block; + create_context.Create(&index_block, generate_index_content()); + size_t block_memory = index_block->ApproximateMemoryUsage(); + std::unique_ptr with_checksum_index_block; + with_checksum_create_context.Create(&with_checksum_index_block, + generate_index_content()); + ASSERT_GT( + with_checksum_index_block->ApproximateMemoryUsage() - block_memory, + 100); + } +} + +std::string GetDataBlockIndexTypeStr( + BlockBasedTableOptions::DataBlockIndexType t) { + return t == BlockBasedTableOptions::DataBlockIndexType::kDataBlockBinarySearch + ? "BinarySearch" + : "BinaryAndHash"; +} + +class DataBlockKVChecksumTest + : public BlockPerKVChecksumTest, + public testing::WithParamInterface> { + public: + DataBlockKVChecksumTest() = default; + + BlockBasedTableOptions::DataBlockIndexType GetDataBlockIndexType() const { + return std::get<0>(GetParam()); + } + uint8_t GetChecksumLen() const { return std::get<1>(GetParam()); } + uint32_t GetRestartInterval() const { return std::get<2>(GetParam()); } + bool GetUseDeltaEncoding() const { return std::get<3>(GetParam()); } + + std::unique_ptr GenerateDataBlock( + std::vector &keys, std::vector &values, + int num_record) { + BlockBasedTableOptions tbo; + BlockCreateContext create_context{&tbo, nullptr /* statistics */, + false /* using_zstd */, GetChecksumLen(), + Options().comparator}; + builder_ = std::make_unique( + static_cast(GetRestartInterval()), + GetUseDeltaEncoding() /* use_delta_encoding */, + false /* use_value_delta_encoding */, GetDataBlockIndexType()); + for (int i = 0; i < num_record; i++) { + builder_->Add(keys[i], values[i]); + } + Slice raw_block = builder_->Finish(); + BlockContents contents; + contents.data = raw_block; + std::unique_ptr data_block; + create_context.Create(&data_block, std::move(contents)); + return data_block; + } + + std::unique_ptr builder_; +}; + +INSTANTIATE_TEST_CASE_P( + P, DataBlockKVChecksumTest, + ::testing::Combine( + ::testing::Values( + BlockBasedTableOptions::DataBlockIndexType::kDataBlockBinarySearch, + BlockBasedTableOptions::DataBlockIndexType:: + kDataBlockBinaryAndHash), + ::testing::Values(0, 1, 2, 4, 8) /* protection_bytes_per_key */, + ::testing::Values(1, 2, 3, 8, 16) /* restart_interval */, + ::testing::Values(false, true)) /* delta_encoding */, + [](const testing::TestParamInfo> + &args) { + std::ostringstream oss; + oss << GetDataBlockIndexTypeStr(std::get<0>(args.param)) + << "ProtectionPerKey" << std::to_string(std::get<1>(args.param)) + << "RestartInterval" << std::to_string(std::get<2>(args.param)) + << "DeltaEncode" << std::to_string(std::get<3>(args.param)); + return oss.str(); + }); + +TEST_P(DataBlockKVChecksumTest, ChecksumConstructionAndVerification) { + uint8_t protection_bytes_per_key = GetChecksumLen(); + std::vector num_restart_intervals = {1, 16}; + for (const auto num_restart_interval : num_restart_intervals) { + const int kNumRecords = + num_restart_interval * static_cast(GetRestartInterval()); + std::vector keys; + std::vector values; + GenerateRandomKVs(&keys, &values, 0, kNumRecords + 1, 1 /* step */, + 24 /* padding_size */); + SyncPoint::GetInstance()->DisableProcessing(); + std::unique_ptr data_block = + GenerateDataBlock(keys, values, kNumRecords); + + const char *checksum_ptr = data_block->TEST_GetKVChecksum(); + // Check checksum of correct length is generated + for (int i = 0; i < kNumRecords; i++) { + ASSERT_TRUE(VerifyChecksum(protection_bytes_per_key, + checksum_ptr + i * protection_bytes_per_key, + keys[i], values[i])); + } + std::vector seqnos{kDisableGlobalSequenceNumber, 0}; + + // Could just use a boolean flag. Use a counter here just to keep open the + // possibility of checking the exact number of verifications in the future. + size_t verification_count = 0; + // The SyncPoint is placed before checking checksum_len == 0 in + // Block::VerifyChecksum(). So verification count is incremented even with + // protection_bytes_per_key = 0. No actual checksum computation is done in + // that case (see Block::VerifyChecksum()). + SyncPoint::GetInstance()->SetCallBack( + "Block::VerifyChecksum::checksum_len", + [&verification_count, protection_bytes_per_key](void *checksum_len) { + ASSERT_EQ((*static_cast(checksum_len)), + protection_bytes_per_key); + ++verification_count; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + for (const auto seqno : seqnos) { + std::unique_ptr biter{ + data_block->NewDataIterator(Options().comparator, seqno)}; + + // SeekForGet() some key that does not exist + biter->SeekForGet(keys[kNumRecords]); + TestIterateForward(biter, verification_count); + + verification_count = 0; + biter->SeekForGet(keys[kNumRecords / 2]); + ASSERT_GE(verification_count, 1); + TestIterateForward(biter, verification_count); + + TestSeekToFirst(biter, verification_count); + TestSeekToLast(biter, verification_count); + TestSeekForPrev(biter, verification_count, keys[kNumRecords / 2]); + TestSeek(biter, verification_count, keys[kNumRecords / 2]); + } + } +} + +class IndexBlockKVChecksumTest + : public BlockPerKVChecksumTest, + public testing::WithParamInterface< + std::tuple> { + public: + IndexBlockKVChecksumTest() = default; + + BlockBasedTableOptions::DataBlockIndexType GetDataBlockIndexType() const { + return std::get<0>(GetParam()); + } + uint8_t GetChecksumLen() const { return std::get<1>(GetParam()); } + uint32_t GetRestartInterval() const { return std::get<2>(GetParam()); } + bool UseValueDeltaEncoding() const { return std::get<3>(GetParam()); } + bool IncludeFirstKey() const { return std::get<4>(GetParam()); } + + std::unique_ptr GenerateIndexBlock( + std::vector &separators, + std::vector &block_handles, + std::vector &first_keys, int num_record) { + Options options = Options(); + BlockBasedTableOptions tbo; + uint8_t protection_bytes_per_key = GetChecksumLen(); + BlockCreateContext create_context{ + &tbo, + nullptr /* statistics */, + false /* _using_zstd */, + protection_bytes_per_key, + options.comparator, + !UseValueDeltaEncoding() /* value_is_full */, + IncludeFirstKey()}; + builder_ = std::make_unique( + static_cast(GetRestartInterval()), true /* use_delta_encoding */, + UseValueDeltaEncoding() /* use_value_delta_encoding */, + GetDataBlockIndexType()); + BlockHandle last_encoded_handle; + for (int i = 0; i < num_record; i++) { + IndexValue entry(block_handles[i], first_keys[i]); + std::string encoded_entry; + std::string delta_encoded_entry; + entry.EncodeTo(&encoded_entry, IncludeFirstKey(), nullptr); + if (UseValueDeltaEncoding() && i > 0) { + entry.EncodeTo(&delta_encoded_entry, IncludeFirstKey(), + &last_encoded_handle); + } + + last_encoded_handle = entry.handle; + const Slice delta_encoded_entry_slice(delta_encoded_entry); + builder_->Add(separators[i], encoded_entry, &delta_encoded_entry_slice); + } + // read serialized contents of the block + Slice raw_block = builder_->Finish(); + // create block reader + BlockContents contents; + contents.data = raw_block; + std::unique_ptr index_block; + + create_context.Create(&index_block, std::move(contents)); + return index_block; + } + + std::unique_ptr builder_; +}; + +INSTANTIATE_TEST_CASE_P( + P, IndexBlockKVChecksumTest, + ::testing::Combine( + ::testing::Values( + BlockBasedTableOptions::DataBlockIndexType::kDataBlockBinarySearch, + BlockBasedTableOptions::DataBlockIndexType:: + kDataBlockBinaryAndHash), + ::testing::Values(0, 1, 2, 4, 8), ::testing::Values(1, 3, 8, 16), + ::testing::Values(true, false), ::testing::Values(true, false)), + [](const testing::TestParamInfo< + std::tuple> &args) { + std::ostringstream oss; + oss << GetDataBlockIndexTypeStr(std::get<0>(args.param)) << "ProtBytes" + << std::to_string(std::get<1>(args.param)) << "RestartInterval" + << std::to_string(std::get<2>(args.param)) << "ValueDeltaEncode" + << std::to_string(std::get<3>(args.param)) << "IncludeFirstKey" + << std::to_string(std::get<4>(args.param)); + return oss.str(); + }); + +TEST_P(IndexBlockKVChecksumTest, ChecksumConstructionAndVerification) { + Options options = Options(); + uint8_t protection_bytes_per_key = GetChecksumLen(); + std::vector num_restart_intervals = {1, 16}; + std::vector seqnos{kDisableGlobalSequenceNumber, 10001}; + + for (const auto num_restart_interval : num_restart_intervals) { + const int kNumRecords = + num_restart_interval * static_cast(GetRestartInterval()); + for (const auto seqno : seqnos) { + std::vector separators; + std::vector block_handles; + std::vector first_keys; + GenerateRandomIndexEntries(&separators, &block_handles, &first_keys, + kNumRecords, 0 /* ts_sz */, + seqno != kDisableGlobalSequenceNumber); + SyncPoint::GetInstance()->DisableProcessing(); + std::unique_ptr index_block = GenerateIndexBlock( + separators, block_handles, first_keys, kNumRecords); + IndexBlockIter *kNullIter = nullptr; + Statistics *kNullStats = nullptr; + // read contents of block sequentially + std::unique_ptr biter{index_block->NewIndexIterator( + options.comparator, seqno, kNullIter, kNullStats, + true /* total_order_seek */, IncludeFirstKey() /* have_first_key */, + true /* key_includes_seq */, + !UseValueDeltaEncoding() /* value_is_full */, + true /* block_contents_pinned*/, + true /* user_defined_timestamps_persisted */, + nullptr /* prefix_index */)}; + biter->SeekToFirst(); + const char *checksum_ptr = index_block->TEST_GetKVChecksum(); + // Check checksum of correct length is generated + for (int i = 0; i < kNumRecords; i++) { + // Obtaining the actual content written as value to index block is not + // trivial: delta-encoded value is only persisted when not at block + // restart point and that keys share some byte (see more in + // BlockBuilder::AddWithLastKeyImpl()). So here we just do verification + // using value from iterator unlike tests for DataBlockIter or + // MetaBlockIter. + ASSERT_TRUE(VerifyChecksum(protection_bytes_per_key, checksum_ptr, + biter->key(), biter->raw_value())); + } + + size_t verification_count = 0; + // The SyncPoint is placed before checking checksum_len == 0 in + // Block::VerifyChecksum(). To make the testing code below simpler and not + // having to differentiate 0 vs non-0 checksum_len, we do an explicit + // assert checking on checksum_len here. + SyncPoint::GetInstance()->SetCallBack( + "Block::VerifyChecksum::checksum_len", + [&verification_count, protection_bytes_per_key](void *checksum_len) { + ASSERT_EQ((*static_cast(checksum_len)), + protection_bytes_per_key); + ++verification_count; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + TestSeekToFirst(biter, verification_count); + TestSeekToLast(biter, verification_count); + TestSeek(biter, verification_count, first_keys[kNumRecords / 2]); + } + } +} + +class MetaIndexBlockKVChecksumTest + : public BlockPerKVChecksumTest, + public testing::WithParamInterface< + uint8_t /* block_protection_bytes_per_key */> { + public: + MetaIndexBlockKVChecksumTest() = default; + uint8_t GetChecksumLen() const { return GetParam(); } + uint32_t GetRestartInterval() const { return 1; } + + std::unique_ptr GenerateMetaIndexBlock( + std::vector &keys, std::vector &values, + int num_record) { + Options options = Options(); + BlockBasedTableOptions tbo; + uint8_t protection_bytes_per_key = GetChecksumLen(); + BlockCreateContext create_context{ + &tbo, nullptr /* statistics */, false /* using_zstd */, + protection_bytes_per_key, options.comparator}; + builder_ = + std::make_unique(static_cast(GetRestartInterval())); + // add a bunch of records to a block + for (int i = 0; i < num_record; i++) { + builder_->Add(keys[i], values[i]); + } + Slice raw_block = builder_->Finish(); + BlockContents contents; + contents.data = raw_block; + std::unique_ptr meta_block; + create_context.Create(&meta_block, std::move(contents)); + return meta_block; + } + + std::unique_ptr builder_; +}; + +INSTANTIATE_TEST_CASE_P(P, MetaIndexBlockKVChecksumTest, + ::testing::Values(0, 1, 2, 4, 8), + [](const testing::TestParamInfo &args) { + std::ostringstream oss; + oss << "ProtBytes" << std::to_string(args.param); + return oss.str(); + }); + +TEST_P(MetaIndexBlockKVChecksumTest, ChecksumConstructionAndVerification) { + Options options = Options(); + BlockBasedTableOptions tbo; + uint8_t protection_bytes_per_key = GetChecksumLen(); + BlockCreateContext create_context{ + &tbo, nullptr /* statistics */, false /* using_zstd */, + protection_bytes_per_key, options.comparator}; + std::vector num_restart_intervals = {1, 16}; + for (const auto num_restart_interval : num_restart_intervals) { + const int kNumRecords = num_restart_interval * GetRestartInterval(); + std::vector keys; + std::vector values; + GenerateRandomKVs(&keys, &values, 0, kNumRecords + 1, 1 /* step */, + 24 /* padding_size */); + SyncPoint::GetInstance()->DisableProcessing(); + std::unique_ptr meta_block = + GenerateMetaIndexBlock(keys, values, kNumRecords); + const char *checksum_ptr = meta_block->TEST_GetKVChecksum(); + // Check checksum of correct length is generated + for (int i = 0; i < kNumRecords; i++) { + ASSERT_TRUE(VerifyChecksum(protection_bytes_per_key, + checksum_ptr + i * protection_bytes_per_key, + keys[i], values[i])); + } + + size_t verification_count = 0; + // The SyncPoint is placed before checking checksum_len == 0 in + // Block::VerifyChecksum(). To make the testing code below simpler and not + // having to differentiate 0 vs non-0 checksum_len, we do an explicit assert + // checking on checksum_len here. + SyncPoint::GetInstance()->SetCallBack( + "Block::VerifyChecksum::checksum_len", + [&verification_count, protection_bytes_per_key](void *checksum_len) { + ASSERT_EQ((*static_cast(checksum_len)), + protection_bytes_per_key); + ++verification_count; + }); + SyncPoint::GetInstance()->EnableProcessing(); + // Check that block iterator does checksum verification + std::unique_ptr biter{ + meta_block->NewMetaIterator(true /* block_contents_pinned */)}; + TestSeekToFirst(biter, verification_count); + TestSeekToLast(biter, verification_count); + TestSeek(biter, verification_count, keys[kNumRecords / 2]); + TestSeekForPrev(biter, verification_count, keys[kNumRecords / 2]); + } +} + +class DataBlockKVChecksumCorruptionTest : public DataBlockKVChecksumTest { + public: + DataBlockKVChecksumCorruptionTest() = default; + + std::unique_ptr GenerateDataBlockIter( + std::vector &keys, std::vector &values, + int num_record) { + // During Block construction, we may create block iter to initialize per kv + // checksum. Disable syncpoint that may be created for block iter methods. + SyncPoint::GetInstance()->DisableProcessing(); + block_ = GenerateDataBlock(keys, values, num_record); + std::unique_ptr biter{block_->NewDataIterator( + Options().comparator, kDisableGlobalSequenceNumber)}; + SyncPoint::GetInstance()->EnableProcessing(); + return biter; + } + + protected: + std::unique_ptr block_; +}; + +TEST_P(DataBlockKVChecksumCorruptionTest, CorruptEntry) { + std::vector num_restart_intervals = {1, 3}; + for (const auto num_restart_interval : num_restart_intervals) { + const int kNumRecords = + num_restart_interval * static_cast(GetRestartInterval()); + std::vector keys; + std::vector values; + GenerateRandomKVs(&keys, &values, 0, kNumRecords + 1, 1 /* step */, + 24 /* padding_size */); + SyncPoint::GetInstance()->SetCallBack( + "BlockIter::UpdateKey::value", [](void *arg) { + char *value = static_cast(arg); + // values generated by GenerateRandomKVs are of length 100 + ++value[10]; + }); + + // Purely for reducing the number of lines of code. + typedef std::unique_ptr IterPtr; + typedef void(IterAPI)(IterPtr & iter, std::string &); + + std::string seek_key = keys[kNumRecords / 2]; + auto test_seek = [&](IterAPI iter_api) { + IterPtr biter = GenerateDataBlockIter(keys, values, kNumRecords); + ASSERT_OK(biter->status()); + iter_api(biter, seek_key); + ASSERT_FALSE(biter->Valid()); + ASSERT_TRUE(biter->status().IsCorruption()); + }; + + test_seek([](IterPtr &iter, std::string &) { iter->SeekToFirst(); }); + test_seek([](IterPtr &iter, std::string &) { iter->SeekToLast(); }); + test_seek([](IterPtr &iter, std::string &k) { iter->Seek(k); }); + test_seek([](IterPtr &iter, std::string &k) { iter->SeekForPrev(k); }); + test_seek([](IterPtr &iter, std::string &k) { iter->SeekForGet(k); }); + + typedef void (DataBlockIter::*IterStepAPI)(); + auto test_step = [&](IterStepAPI iter_api, std::string &k) { + IterPtr biter = GenerateDataBlockIter(keys, values, kNumRecords); + SyncPoint::GetInstance()->DisableProcessing(); + biter->Seek(k); + ASSERT_TRUE(biter->Valid()); + ASSERT_OK(biter->status()); + SyncPoint::GetInstance()->EnableProcessing(); + std::invoke(iter_api, biter); + ASSERT_FALSE(biter->Valid()); + ASSERT_TRUE(biter->status().IsCorruption()); + }; + + if (kNumRecords > 1) { + test_step(&DataBlockIter::Prev, seek_key); + test_step(&DataBlockIter::Next, seek_key); + } + } +} + +INSTANTIATE_TEST_CASE_P( + P, DataBlockKVChecksumCorruptionTest, + ::testing::Combine( + ::testing::Values( + BlockBasedTableOptions::DataBlockIndexType::kDataBlockBinarySearch, + BlockBasedTableOptions::DataBlockIndexType:: + kDataBlockBinaryAndHash), + ::testing::Values(4, 8) /* block_protection_bytes_per_key */, + ::testing::Values(1, 3, 8, 16) /* restart_interval */, + ::testing::Values(false, true)), + [](const testing::TestParamInfo> + &args) { + std::ostringstream oss; + oss << GetDataBlockIndexTypeStr(std::get<0>(args.param)) << "ProtBytes" + << std::to_string(std::get<1>(args.param)) << "RestartInterval" + << std::to_string(std::get<2>(args.param)) << "DeltaEncode" + << std::to_string(std::get<3>(args.param)); + return oss.str(); + }); + +class IndexBlockKVChecksumCorruptionTest : public IndexBlockKVChecksumTest { + public: + IndexBlockKVChecksumCorruptionTest() = default; + + std::unique_ptr GenerateIndexBlockIter( + std::vector &separators, + std::vector &block_handles, + std::vector &first_keys, int num_record, + SequenceNumber seqno) { + SyncPoint::GetInstance()->DisableProcessing(); + block_ = + GenerateIndexBlock(separators, block_handles, first_keys, num_record); + std::unique_ptr biter{block_->NewIndexIterator( + Options().comparator, seqno, nullptr, nullptr, + true /* total_order_seek */, IncludeFirstKey() /* have_first_key */, + true /* key_includes_seq */, + !UseValueDeltaEncoding() /* value_is_full */, + true /* block_contents_pinned */, + true /* user_defined_timestamps_persisted */, + nullptr /* prefix_index */)}; + SyncPoint::GetInstance()->EnableProcessing(); + return biter; + } + + protected: + std::unique_ptr block_; +}; + +INSTANTIATE_TEST_CASE_P( + P, IndexBlockKVChecksumCorruptionTest, + ::testing::Combine( + ::testing::Values( + BlockBasedTableOptions::DataBlockIndexType::kDataBlockBinarySearch, + BlockBasedTableOptions::DataBlockIndexType:: + kDataBlockBinaryAndHash), + ::testing::Values(4, 8) /* block_protection_bytes_per_key */, + ::testing::Values(1, 3, 8, 16) /* restart_interval */, + ::testing::Values(true, false), ::testing::Values(true, false)), + [](const testing::TestParamInfo< + std::tuple> &args) { + std::ostringstream oss; + oss << GetDataBlockIndexTypeStr(std::get<0>(args.param)) << "ProtBytes" + << std::to_string(std::get<1>(args.param)) << "RestartInterval" + << std::to_string(std::get<2>(args.param)) << "ValueDeltaEncode" + << std::to_string(std::get<3>(args.param)) << "IncludeFirstKey" + << std::to_string(std::get<4>(args.param)); + return oss.str(); + }); + +TEST_P(IndexBlockKVChecksumCorruptionTest, CorruptEntry) { + std::vector num_restart_intervals = {1, 3}; + std::vector seqnos{kDisableGlobalSequenceNumber, 10001}; + + for (const auto num_restart_interval : num_restart_intervals) { + const int kNumRecords = + num_restart_interval * static_cast(GetRestartInterval()); + for (const auto seqno : seqnos) { + std::vector separators; + std::vector block_handles; + std::vector first_keys; + GenerateRandomIndexEntries(&separators, &block_handles, &first_keys, + kNumRecords, 0 /* ts_sz */, + seqno != kDisableGlobalSequenceNumber); + SyncPoint::GetInstance()->SetCallBack( + "BlockIter::UpdateKey::value", [](void *arg) { + char *value = static_cast(arg); + // value can be delta-encoded with different lengths, so we corrupt + // first bytes here to be safe + ++value[0]; + }); + + typedef std::unique_ptr IterPtr; + typedef void(IterAPI)(IterPtr & iter, std::string &); + std::string seek_key = first_keys[kNumRecords / 2]; + auto test_seek = [&](IterAPI iter_api) { + std::unique_ptr biter = GenerateIndexBlockIter( + separators, block_handles, first_keys, kNumRecords, seqno); + ASSERT_OK(biter->status()); + iter_api(biter, seek_key); + ASSERT_FALSE(biter->Valid()); + ASSERT_TRUE(biter->status().IsCorruption()); + }; + test_seek([](IterPtr &iter, std::string &) { iter->SeekToFirst(); }); + test_seek([](IterPtr &iter, std::string &) { iter->SeekToLast(); }); + test_seek([](IterPtr &iter, std::string &k) { iter->Seek(k); }); + + typedef void (IndexBlockIter::*IterStepAPI)(); + auto test_step = [&](IterStepAPI iter_api, std::string &k) { + std::unique_ptr biter = GenerateIndexBlockIter( + separators, block_handles, first_keys, kNumRecords, seqno); + SyncPoint::GetInstance()->DisableProcessing(); + biter->Seek(k); + ASSERT_TRUE(biter->Valid()); + ASSERT_OK(biter->status()); + SyncPoint::GetInstance()->EnableProcessing(); + std::invoke(iter_api, biter); + ASSERT_FALSE(biter->Valid()); + ASSERT_TRUE(biter->status().IsCorruption()); + }; + if (kNumRecords > 1) { + test_step(&IndexBlockIter::Prev, seek_key); + test_step(&IndexBlockIter::Next, seek_key); + } + } + } +} + +class MetaIndexBlockKVChecksumCorruptionTest + : public MetaIndexBlockKVChecksumTest { + public: + MetaIndexBlockKVChecksumCorruptionTest() = default; + + std::unique_ptr GenerateMetaIndexBlockIter( + std::vector &keys, std::vector &values, + int num_record) { + SyncPoint::GetInstance()->DisableProcessing(); + block_ = GenerateMetaIndexBlock(keys, values, num_record); + std::unique_ptr biter{ + block_->NewMetaIterator(true /* block_contents_pinned */)}; + SyncPoint::GetInstance()->EnableProcessing(); + return biter; + } + + protected: + std::unique_ptr block_; +}; + +INSTANTIATE_TEST_CASE_P( + P, MetaIndexBlockKVChecksumCorruptionTest, + ::testing::Values(4, 8) /* block_protection_bytes_per_key */, + [](const testing::TestParamInfo &args) { + std::ostringstream oss; + oss << "ProtBytes" << std::to_string(args.param); + return oss.str(); + }); + +TEST_P(MetaIndexBlockKVChecksumCorruptionTest, CorruptEntry) { + Options options = Options(); + std::vector num_restart_intervals = {1, 3}; + for (const auto num_restart_interval : num_restart_intervals) { + const int kNumRecords = + num_restart_interval * static_cast(GetRestartInterval()); + std::vector keys; + std::vector values; + GenerateRandomKVs(&keys, &values, 0, kNumRecords + 1, 1 /* step */, + 24 /* padding_size */); + SyncPoint::GetInstance()->SetCallBack( + "BlockIter::UpdateKey::value", [](void *arg) { + char *value = static_cast(arg); + // values generated by GenerateRandomKVs are of length 100 + ++value[10]; + }); + + typedef std::unique_ptr IterPtr; + typedef void(IterAPI)(IterPtr & iter, std::string &); + typedef void (MetaBlockIter::*IterStepAPI)(); + std::string seek_key = keys[kNumRecords / 2]; + auto test_seek = [&](IterAPI iter_api) { + IterPtr biter = GenerateMetaIndexBlockIter(keys, values, kNumRecords); + ASSERT_OK(biter->status()); + iter_api(biter, seek_key); + ASSERT_FALSE(biter->Valid()); + ASSERT_TRUE(biter->status().IsCorruption()); + }; + + test_seek([](IterPtr &iter, std::string &) { iter->SeekToFirst(); }); + test_seek([](IterPtr &iter, std::string &) { iter->SeekToLast(); }); + test_seek([](IterPtr &iter, std::string &k) { iter->Seek(k); }); + test_seek([](IterPtr &iter, std::string &k) { iter->SeekForPrev(k); }); + + auto test_step = [&](IterStepAPI iter_api, const std::string &k) { + IterPtr biter = GenerateMetaIndexBlockIter(keys, values, kNumRecords); + SyncPoint::GetInstance()->DisableProcessing(); + biter->Seek(k); + ASSERT_TRUE(biter->Valid()); + ASSERT_OK(biter->status()); + SyncPoint::GetInstance()->EnableProcessing(); + std::invoke(iter_api, biter); + ASSERT_FALSE(biter->Valid()); + ASSERT_TRUE(biter->status().IsCorruption()); + }; + + if (kNumRecords > 1) { + test_step(&MetaBlockIter::Prev, seek_key); + test_step(&MetaBlockIter::Next, seek_key); + } + } +} +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char **argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/table/block_based/block_type.h b/librocksdb-sys/rocksdb/table/block_based/block_type.h new file mode 100644 index 0000000..a9d6a1a --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/block_type.h @@ -0,0 +1,34 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +// Represents the types of blocks used in the block based table format. +// See https://github.com/facebook/rocksdb/wiki/Rocksdb-BlockBasedTable-Format +// for details. +// For code sanity, BlockType should imply a specific TBlocklike for +// BlocklikeTraits. +enum class BlockType : uint8_t { + kData, + kFilter, // for second level partitioned filters and full filters + kFilterPartitionIndex, // for top-level index of filter partitions + kProperties, + kCompressionDictionary, + kRangeDeletion, + kHashIndexPrefixes, + kHashIndexMetadata, + kMetaIndex, + kIndex, + // Note: keep kInvalid the last value when adding new enum values. + kInvalid +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/cachable_entry.h b/librocksdb-sys/rocksdb/table/block_based/cachable_entry.h new file mode 100644 index 0000000..3cd1bb8 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/cachable_entry.h @@ -0,0 +1,244 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include +#include + +#include "port/likely.h" +#include "rocksdb/advanced_cache.h" +#include "rocksdb/cleanable.h" + +namespace ROCKSDB_NAMESPACE { + +// CachableEntry is a handle to an object that may or may not be in the block +// cache. It is used in a variety of ways: +// +// 1) It may refer to an object in the block cache. In this case, cache_ and +// cache_handle_ are not nullptr, and the cache handle has to be released when +// the CachableEntry is destroyed (the lifecycle of the cached object, on the +// other hand, is managed by the cache itself). +// 2) It may uniquely own the (non-cached) object it refers to (examples include +// a block read directly from file, or uncompressed blocks when there is a +// compressed block cache but no uncompressed block cache). In such cases, the +// object has to be destroyed when the CachableEntry is destroyed. +// 3) It may point to an object (cached or not) without owning it. In this case, +// no action is needed when the CachableEntry is destroyed. +// 4) Sometimes, management of a cached or owned object (see #1 and #2 above) +// is transferred to some other object. This is used for instance with iterators +// (where cleanup is performed using a chain of cleanup functions, +// see Cleanable). +// +// Because of #1 and #2 above, copying a CachableEntry is not safe (and thus not +// allowed); hence, this is a move-only type, where a move transfers the +// management responsibilities, and leaves the source object in an empty state. + +template +class CachableEntry { + public: + CachableEntry() = default; + + CachableEntry(T* value, Cache* cache, Cache::Handle* cache_handle, + bool own_value) + : value_(value), + cache_(cache), + cache_handle_(cache_handle), + own_value_(own_value) { + assert(value_ != nullptr || + (cache_ == nullptr && cache_handle_ == nullptr && !own_value_)); + assert(!!cache_ == !!cache_handle_); + assert(!cache_handle_ || !own_value_); + } + + CachableEntry(const CachableEntry&) = delete; + CachableEntry& operator=(const CachableEntry&) = delete; + + CachableEntry(CachableEntry&& rhs) noexcept + : value_(rhs.value_), + cache_(rhs.cache_), + cache_handle_(rhs.cache_handle_), + own_value_(rhs.own_value_) { + assert(value_ != nullptr || + (cache_ == nullptr && cache_handle_ == nullptr && !own_value_)); + assert(!!cache_ == !!cache_handle_); + assert(!cache_handle_ || !own_value_); + + rhs.ResetFields(); + } + + CachableEntry& operator=(CachableEntry&& rhs) noexcept { + if (UNLIKELY(this == &rhs)) { + return *this; + } + + ReleaseResource(); + + value_ = rhs.value_; + cache_ = rhs.cache_; + cache_handle_ = rhs.cache_handle_; + own_value_ = rhs.own_value_; + + assert(value_ != nullptr || + (cache_ == nullptr && cache_handle_ == nullptr && !own_value_)); + assert(!!cache_ == !!cache_handle_); + assert(!cache_handle_ || !own_value_); + + rhs.ResetFields(); + + return *this; + } + + ~CachableEntry() { ReleaseResource(); } + + bool IsEmpty() const { + return value_ == nullptr && cache_ == nullptr && cache_handle_ == nullptr && + !own_value_; + } + + bool IsCached() const { + assert(!!cache_ == !!cache_handle_); + + return cache_handle_ != nullptr; + } + + T* GetValue() const { return value_; } + Cache* GetCache() const { return cache_; } + Cache::Handle* GetCacheHandle() const { return cache_handle_; } + bool GetOwnValue() const { return own_value_; } + + void Reset() { + ReleaseResource(); + ResetFields(); + } + + void TransferTo(Cleanable* cleanable) { + if (cleanable) { + if (cache_handle_ != nullptr) { + assert(cache_ != nullptr); + cleanable->RegisterCleanup(&ReleaseCacheHandle, cache_, cache_handle_); + } else if (own_value_) { + cleanable->RegisterCleanup(&DeleteValue, value_, nullptr); + } + } + + ResetFields(); + } + + void SetOwnedValue(std::unique_ptr&& value) { + assert(value.get() != nullptr); + + if (UNLIKELY(value_ == value.get() && own_value_)) { + assert(cache_ == nullptr && cache_handle_ == nullptr); + return; + } + + Reset(); + + value_ = value.release(); + own_value_ = true; + } + + void SetUnownedValue(T* value) { + assert(value != nullptr); + + if (UNLIKELY(value_ == value && cache_ == nullptr && + cache_handle_ == nullptr && !own_value_)) { + return; + } + + Reset(); + + value_ = value; + assert(!own_value_); + } + + void SetCachedValue(T* value, Cache* cache, Cache::Handle* cache_handle) { + assert(cache != nullptr); + assert(cache_handle != nullptr); + + if (UNLIKELY(value_ == value && cache_ == cache && + cache_handle_ == cache_handle && !own_value_)) { + return; + } + + Reset(); + + value_ = value; + cache_ = cache; + cache_handle_ = cache_handle; + assert(!own_value_); + } + + // Since this class is essentially an elaborate pointer, it's sometimes + // useful to be able to upcast or downcast the base type of the pointer, + // especially when interacting with typed_cache.h. + template + std::enable_if_t || + std::is_base_of_v), + /* Actual return type */ + CachableEntry&> + As() { + CachableEntry* result_ptr = + reinterpret_cast*>(this); + // Ensure no weirdness in template instantiations + assert(static_cast(&this->value_) == + static_cast(&result_ptr->value_)); + assert(&this->cache_handle_ == &result_ptr->cache_handle_); + // This function depends on no arithmetic involved in the pointer + // conversion, which is not statically checkable. + assert(static_cast(this->value_) == + static_cast(result_ptr->value_)); + return *result_ptr; + } + + private: + void ReleaseResource() noexcept { + if (LIKELY(cache_handle_ != nullptr)) { + assert(cache_ != nullptr); + cache_->Release(cache_handle_); + } else if (own_value_) { + delete value_; + } + } + + void ResetFields() noexcept { + value_ = nullptr; + cache_ = nullptr; + cache_handle_ = nullptr; + own_value_ = false; + } + + static void ReleaseCacheHandle(void* arg1, void* arg2) { + Cache* const cache = static_cast(arg1); + assert(cache); + + Cache::Handle* const cache_handle = static_cast(arg2); + assert(cache_handle); + + cache->Release(cache_handle); + } + + static void DeleteValue(void* arg1, void* /* arg2 */) { + delete static_cast(arg1); + } + + private: + // Have to be your own best friend + template + friend class CachableEntry; + + T* value_ = nullptr; + Cache* cache_ = nullptr; + Cache::Handle* cache_handle_ = nullptr; + bool own_value_ = false; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/data_block_footer.cc b/librocksdb-sys/rocksdb/table/block_based/data_block_footer.cc new file mode 100644 index 0000000..5d5d8ed --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/data_block_footer.cc @@ -0,0 +1,59 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "table/block_based/data_block_footer.h" + +#include "rocksdb/table.h" + +namespace ROCKSDB_NAMESPACE { + +const int kDataBlockIndexTypeBitShift = 31; + +// 0x7FFFFFFF +const uint32_t kMaxNumRestarts = (1u << kDataBlockIndexTypeBitShift) - 1u; + +// 0x7FFFFFFF +const uint32_t kNumRestartsMask = (1u << kDataBlockIndexTypeBitShift) - 1u; + +uint32_t PackIndexTypeAndNumRestarts( + BlockBasedTableOptions::DataBlockIndexType index_type, + uint32_t num_restarts) { + if (num_restarts > kMaxNumRestarts) { + assert(0); // mute travis "unused" warning + } + + uint32_t block_footer = num_restarts; + if (index_type == BlockBasedTableOptions::kDataBlockBinaryAndHash) { + block_footer |= 1u << kDataBlockIndexTypeBitShift; + } else if (index_type != BlockBasedTableOptions::kDataBlockBinarySearch) { + assert(0); + } + + return block_footer; +} + +void UnPackIndexTypeAndNumRestarts( + uint32_t block_footer, + BlockBasedTableOptions::DataBlockIndexType* index_type, + uint32_t* num_restarts) { + if (index_type) { + if (block_footer & 1u << kDataBlockIndexTypeBitShift) { + *index_type = BlockBasedTableOptions::kDataBlockBinaryAndHash; + } else { + *index_type = BlockBasedTableOptions::kDataBlockBinarySearch; + } + } + + if (num_restarts) { + *num_restarts = block_footer & kNumRestartsMask; + assert(*num_restarts <= kMaxNumRestarts); + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/data_block_footer.h b/librocksdb-sys/rocksdb/table/block_based/data_block_footer.h new file mode 100644 index 0000000..c1cfd47 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/data_block_footer.h @@ -0,0 +1,25 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include "rocksdb/table.h" + +namespace ROCKSDB_NAMESPACE { + +uint32_t PackIndexTypeAndNumRestarts( + BlockBasedTableOptions::DataBlockIndexType index_type, + uint32_t num_restarts); + +void UnPackIndexTypeAndNumRestarts( + uint32_t block_footer, + BlockBasedTableOptions::DataBlockIndexType* index_type, + uint32_t* num_restarts); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/data_block_hash_index.cc b/librocksdb-sys/rocksdb/table/block_based/data_block_hash_index.cc new file mode 100644 index 0000000..c579dcc --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/data_block_hash_index.cc @@ -0,0 +1,94 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#include "table/block_based/data_block_hash_index.h" + +#include +#include + +#include "rocksdb/slice.h" +#include "util/coding.h" +#include "util/hash.h" + +namespace ROCKSDB_NAMESPACE { + +void DataBlockHashIndexBuilder::Add(const Slice& key, + const size_t restart_index) { + assert(Valid()); + if (restart_index > kMaxRestartSupportedByHashIndex) { + valid_ = false; + return; + } + + uint32_t hash_value = GetSliceHash(key); + hash_and_restart_pairs_.emplace_back(hash_value, + static_cast(restart_index)); + estimated_num_buckets_ += bucket_per_key_; +} + +void DataBlockHashIndexBuilder::Finish(std::string& buffer) { + assert(Valid()); + uint16_t num_buckets = static_cast(estimated_num_buckets_); + + if (num_buckets == 0) { + num_buckets = 1; // sanity check + } + + // The build-in hash cannot well distribute strings when into different + // buckets when num_buckets is power of two, resulting in high hash + // collision. + // We made the num_buckets to be odd to avoid this issue. + num_buckets |= 1; + + std::vector buckets(num_buckets, kNoEntry); + // write the restart_index array + for (auto& entry : hash_and_restart_pairs_) { + uint32_t hash_value = entry.first; + uint8_t restart_index = entry.second; + uint16_t buck_idx = static_cast(hash_value % num_buckets); + if (buckets[buck_idx] == kNoEntry) { + buckets[buck_idx] = restart_index; + } else if (buckets[buck_idx] != restart_index) { + // same bucket cannot store two different restart_index, mark collision + buckets[buck_idx] = kCollision; + } + } + + for (uint8_t restart_index : buckets) { + buffer.append( + const_cast(reinterpret_cast(&restart_index)), + sizeof(restart_index)); + } + + // write NUM_BUCK + PutFixed16(&buffer, num_buckets); + + assert(buffer.size() <= kMaxBlockSizeSupportedByHashIndex); +} + +void DataBlockHashIndexBuilder::Reset() { + estimated_num_buckets_ = 0; + valid_ = true; + hash_and_restart_pairs_.clear(); +} + +void DataBlockHashIndex::Initialize(const char* data, uint16_t size, + uint16_t* map_offset) { + assert(size >= sizeof(uint16_t)); // NUM_BUCKETS + num_buckets_ = DecodeFixed16(data + size - sizeof(uint16_t)); + assert(num_buckets_ > 0); + assert(size > num_buckets_ * sizeof(uint8_t)); + *map_offset = static_cast(size - sizeof(uint16_t) - + num_buckets_ * sizeof(uint8_t)); +} + +uint8_t DataBlockHashIndex::Lookup(const char* data, uint32_t map_offset, + const Slice& key) const { + uint32_t hash_value = GetSliceHash(key); + uint16_t idx = static_cast(hash_value % num_buckets_); + const char* bucket_table = data + map_offset; + return static_cast(*(bucket_table + idx * sizeof(uint8_t))); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/data_block_hash_index.h b/librocksdb-sys/rocksdb/table/block_based/data_block_hash_index.h new file mode 100644 index 0000000..3215221 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/data_block_hash_index.h @@ -0,0 +1,137 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include + +#include "rocksdb/slice.h" + +namespace ROCKSDB_NAMESPACE { +// This is an experimental feature aiming to reduce the CPU utilization of +// point-lookup within a data-block. It is only used in data blocks, and not +// in meta-data blocks or per-table index blocks. +// +// It only used to support BlockBasedTable::Get(). +// +// A serialized hash index is appended to the data-block. The new block data +// format is as follows: +// +// DATA_BLOCK: [RI RI RI ... RI RI_IDX HASH_IDX FOOTER] +// +// RI: Restart Interval (the same as the default data-block format) +// RI_IDX: Restart Interval index (the same as the default data-block format) +// HASH_IDX: The new data-block hash index feature. +// FOOTER: A 32bit block footer, which is the NUM_RESTARTS with the MSB as +// the flag indicating if this hash index is in use. Note that +// given a data block < 32KB, the MSB is never used. So we can +// borrow the MSB as the hash index flag. Therefore, this format is +// compatible with the legacy data-blocks with num_restarts < 32768, +// as the MSB is 0. +// +// The format of the data-block hash index is as follows: +// +// HASH_IDX: [B B B ... B NUM_BUCK] +// +// B: bucket, an array of restart index. Each buckets is uint8_t. +// NUM_BUCK: Number of buckets, which is the length of the bucket array. +// +// We reserve two special flag: +// kNoEntry=255, +// kCollision=254. +// +// Therefore, the max number of restarts this hash index can supoport is 253. +// +// Buckets are initialized to be kNoEntry. +// +// When storing a key in the hash index, the key is first hashed to a bucket. +// If there the bucket is empty (kNoEntry), the restart index is stored in +// the bucket. If there is already a restart index there, we will update the +// existing restart index to a collision marker (kCollision). If the +// the bucket is already marked as collision, we do not store the restart +// index either. +// +// During query process, a key is first hashed to a bucket. Then we examine if +// the buckets store nothing (kNoEntry) or the bucket had a collision +// (kCollision). If either of those happens, we get the restart index of +// the key and will directly go to the restart interval to search the key. +// +// Note that we only support blocks with #restart_interval < 254. If a block +// has more restart interval than that, hash index will not be create for it. + +const uint8_t kNoEntry = 255; +const uint8_t kCollision = 254; +const uint8_t kMaxRestartSupportedByHashIndex = 253; + +// Because we use uint16_t address, we only support block no more than 64KB +const size_t kMaxBlockSizeSupportedByHashIndex = 1u << 16; +const double kDefaultUtilRatio = 0.75; + +class DataBlockHashIndexBuilder { + public: + DataBlockHashIndexBuilder() + : bucket_per_key_(-1 /*uninitialized marker*/), + estimated_num_buckets_(0), + valid_(false) {} + + void Initialize(double util_ratio) { + if (util_ratio <= 0) { + util_ratio = kDefaultUtilRatio; // sanity check + } + bucket_per_key_ = 1 / util_ratio; + valid_ = true; + } + + inline bool Valid() const { return valid_ && bucket_per_key_ > 0; } + void Add(const Slice& key, const size_t restart_index); + void Finish(std::string& buffer); + void Reset(); + inline size_t EstimateSize() const { + uint16_t estimated_num_buckets = + static_cast(estimated_num_buckets_); + + // Maching the num_buckets number in DataBlockHashIndexBuilder::Finish. + estimated_num_buckets |= 1; + + return sizeof(uint16_t) + + static_cast(estimated_num_buckets * sizeof(uint8_t)); + } + + private: + double bucket_per_key_; // is the multiplicative inverse of util_ratio_ + double estimated_num_buckets_; + + // Now the only usage for `valid_` is to mark false when the inserted + // restart_index is larger than supported. In this case HashIndex is not + // appended to the block content. + bool valid_; + + std::vector> hash_and_restart_pairs_; + friend class DataBlockHashIndex_DataBlockHashTestSmall_Test; +}; + +class DataBlockHashIndex { + public: + DataBlockHashIndex() : num_buckets_(0) {} + + void Initialize(const char* data, uint16_t size, uint16_t* map_offset); + + uint8_t Lookup(const char* data, uint32_t map_offset, const Slice& key) const; + + inline bool Valid() { return num_buckets_ != 0; } + + private: + // To make the serialized hash index compact and to save the space overhead, + // here all the data fields persisted in the block are in uint16 format. + // We find that a uint16 is large enough to index every offset of a 64KiB + // block. + // So in other words, DataBlockHashIndex does not support block size equal + // or greater then 64KiB. + uint16_t num_buckets_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/data_block_hash_index_test.cc b/librocksdb-sys/rocksdb/table/block_based/data_block_hash_index_test.cc new file mode 100644 index 0000000..2841b27 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/data_block_hash_index_test.cc @@ -0,0 +1,718 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "table/block_based/data_block_hash_index.h" + +#include +#include +#include + +#include "db/table_properties_collector.h" +#include "rocksdb/slice.h" +#include "table/block_based/block.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/block_based/block_builder.h" +#include "table/get_context.h" +#include "table/table_builder.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +bool SearchForOffset(DataBlockHashIndex& index, const char* data, + uint16_t map_offset, const Slice& key, + uint8_t& restart_point) { + uint8_t entry = index.Lookup(data, map_offset, key); + if (entry == kCollision) { + return true; + } + + if (entry == kNoEntry) { + return false; + } + + return entry == restart_point; +} + +std::string GenerateKey(int primary_key, int secondary_key, int padding_size, + Random* rnd) { + char buf[50]; + char* p = &buf[0]; + snprintf(buf, sizeof(buf), "%6d%4d", primary_key, secondary_key); + std::string k(p); + if (padding_size) { + k += rnd->RandomString(padding_size); + } + + return k; +} + +// Generate random key value pairs. +// The generated key will be sorted. You can tune the parameters to generated +// different kinds of test key/value pairs for different scenario. +void GenerateRandomKVs(std::vector* keys, + std::vector* values, const int from, + const int len, const int step = 1, + const int padding_size = 0, + const int keys_share_prefix = 1) { + Random rnd(302); + + // generate different prefix + for (int i = from; i < from + len; i += step) { + // generating keys that shares the prefix + for (int j = 0; j < keys_share_prefix; ++j) { + keys->emplace_back(GenerateKey(i, j, padding_size, &rnd)); + + // 100 bytes values + values->emplace_back(rnd.RandomString(100)); + } + } +} + +TEST(DataBlockHashIndex, DataBlockHashTestSmall) { + DataBlockHashIndexBuilder builder; + builder.Initialize(0.75 /*util_ratio*/); + for (int j = 0; j < 5; j++) { + for (uint8_t i = 0; i < 2 + j; i++) { + std::string key("key" + std::to_string(i)); + uint8_t restart_point = i; + builder.Add(key, restart_point); + } + + size_t estimated_size = builder.EstimateSize(); + + std::string buffer("fake"), buffer2; + size_t original_size = buffer.size(); + estimated_size += original_size; + builder.Finish(buffer); + + ASSERT_EQ(buffer.size(), estimated_size); + + buffer2 = buffer; // test for the correctness of relative offset + + Slice s(buffer2); + DataBlockHashIndex index; + uint16_t map_offset; + index.Initialize(s.data(), static_cast(s.size()), &map_offset); + + // the additional hash map should start at the end of the buffer + ASSERT_EQ(original_size, map_offset); + for (uint8_t i = 0; i < 2; i++) { + std::string key("key" + std::to_string(i)); + uint8_t restart_point = i; + ASSERT_TRUE( + SearchForOffset(index, s.data(), map_offset, key, restart_point)); + } + builder.Reset(); + } +} + +TEST(DataBlockHashIndex, DataBlockHashTest) { + // bucket_num = 200, #keys = 100. 50% utilization + DataBlockHashIndexBuilder builder; + builder.Initialize(0.75 /*util_ratio*/); + + for (uint8_t i = 0; i < 100; i++) { + std::string key("key" + std::to_string(i)); + uint8_t restart_point = i; + builder.Add(key, restart_point); + } + + size_t estimated_size = builder.EstimateSize(); + + std::string buffer("fake content"), buffer2; + size_t original_size = buffer.size(); + estimated_size += original_size; + builder.Finish(buffer); + + ASSERT_EQ(buffer.size(), estimated_size); + + buffer2 = buffer; // test for the correctness of relative offset + + Slice s(buffer2); + DataBlockHashIndex index; + uint16_t map_offset; + index.Initialize(s.data(), static_cast(s.size()), &map_offset); + + // the additional hash map should start at the end of the buffer + ASSERT_EQ(original_size, map_offset); + for (uint8_t i = 0; i < 100; i++) { + std::string key("key" + std::to_string(i)); + uint8_t restart_point = i; + ASSERT_TRUE( + SearchForOffset(index, s.data(), map_offset, key, restart_point)); + } +} + +TEST(DataBlockHashIndex, DataBlockHashTestCollision) { + // bucket_num = 2. There will be intense hash collisions + DataBlockHashIndexBuilder builder; + builder.Initialize(0.75 /*util_ratio*/); + + for (uint8_t i = 0; i < 100; i++) { + std::string key("key" + std::to_string(i)); + uint8_t restart_point = i; + builder.Add(key, restart_point); + } + + size_t estimated_size = builder.EstimateSize(); + + std::string buffer("some other fake content to take up space"), buffer2; + size_t original_size = buffer.size(); + estimated_size += original_size; + builder.Finish(buffer); + + ASSERT_EQ(buffer.size(), estimated_size); + + buffer2 = buffer; // test for the correctness of relative offset + + Slice s(buffer2); + DataBlockHashIndex index; + uint16_t map_offset; + index.Initialize(s.data(), static_cast(s.size()), &map_offset); + + // the additional hash map should start at the end of the buffer + ASSERT_EQ(original_size, map_offset); + for (uint8_t i = 0; i < 100; i++) { + std::string key("key" + std::to_string(i)); + uint8_t restart_point = i; + ASSERT_TRUE( + SearchForOffset(index, s.data(), map_offset, key, restart_point)); + } +} + +TEST(DataBlockHashIndex, DataBlockHashTestLarge) { + DataBlockHashIndexBuilder builder; + builder.Initialize(0.75 /*util_ratio*/); + std::unordered_map m; + + for (uint8_t i = 0; i < 100; i++) { + if (i % 2) { + continue; // leave half of the keys out + } + std::string key = "key" + std::to_string(i); + uint8_t restart_point = i; + builder.Add(key, restart_point); + m[key] = restart_point; + } + + size_t estimated_size = builder.EstimateSize(); + + std::string buffer("filling stuff"), buffer2; + size_t original_size = buffer.size(); + estimated_size += original_size; + builder.Finish(buffer); + + ASSERT_EQ(buffer.size(), estimated_size); + + buffer2 = buffer; // test for the correctness of relative offset + + Slice s(buffer2); + DataBlockHashIndex index; + uint16_t map_offset; + index.Initialize(s.data(), static_cast(s.size()), &map_offset); + + // the additional hash map should start at the end of the buffer + ASSERT_EQ(original_size, map_offset); + for (uint8_t i = 0; i < 100; i++) { + std::string key = "key" + std::to_string(i); + uint8_t restart_point = i; + if (m.count(key)) { + ASSERT_TRUE(m[key] == restart_point); + ASSERT_TRUE( + SearchForOffset(index, s.data(), map_offset, key, restart_point)); + } else { + // we allow false positve, so don't test the nonexisting keys. + // when false positive happens, the search will continue to the + // restart intervals to see if the key really exist. + } + } +} + +TEST(DataBlockHashIndex, RestartIndexExceedMax) { + DataBlockHashIndexBuilder builder; + builder.Initialize(0.75 /*util_ratio*/); + std::unordered_map m; + + for (uint8_t i = 0; i <= 253; i++) { + std::string key = "key" + std::to_string(i); + uint8_t restart_point = i; + builder.Add(key, restart_point); + } + ASSERT_TRUE(builder.Valid()); + + builder.Reset(); + + for (uint8_t i = 0; i <= 254; i++) { + std::string key = "key" + std::to_string(i); + uint8_t restart_point = i; + builder.Add(key, restart_point); + } + + ASSERT_FALSE(builder.Valid()); + + builder.Reset(); + ASSERT_TRUE(builder.Valid()); +} + +TEST(DataBlockHashIndex, BlockRestartIndexExceedMax) { + Options options = Options(); + + BlockBuilder builder(1 /* block_restart_interval */, + true /* use_delta_encoding */, + false /* use_value_delta_encoding */, + BlockBasedTableOptions::kDataBlockBinaryAndHash); + + // #restarts <= 253. HashIndex is valid + for (int i = 0; i <= 253; i++) { + std::string ukey = "key" + std::to_string(i); + InternalKey ikey(ukey, 0, kTypeValue); + builder.Add(ikey.Encode().ToString(), "value"); + } + + { + // read serialized contents of the block + Slice rawblock = builder.Finish(); + + // create block reader + BlockContents contents; + contents.data = rawblock; + Block reader(std::move(contents)); + + ASSERT_EQ(reader.IndexType(), + BlockBasedTableOptions::kDataBlockBinaryAndHash); + } + + builder.Reset(); + + // #restarts > 253. HashIndex is not used + for (int i = 0; i <= 254; i++) { + std::string ukey = "key" + std::to_string(i); + InternalKey ikey(ukey, 0, kTypeValue); + builder.Add(ikey.Encode().ToString(), "value"); + } + + { + // read serialized contents of the block + Slice rawblock = builder.Finish(); + + // create block reader + BlockContents contents; + contents.data = rawblock; + Block reader(std::move(contents)); + + ASSERT_EQ(reader.IndexType(), + BlockBasedTableOptions::kDataBlockBinarySearch); + } +} + +TEST(DataBlockHashIndex, BlockSizeExceedMax) { + Options options = Options(); + std::string ukey(10, 'k'); + InternalKey ikey(ukey, 0, kTypeValue); + + BlockBuilder builder(1 /* block_restart_interval */, + false /* use_delta_encoding */, + false /* use_value_delta_encoding */, + BlockBasedTableOptions::kDataBlockBinaryAndHash); + + { + // insert a large value. The block size plus HashIndex is 65536. + std::string value(65502, 'v'); + + builder.Add(ikey.Encode().ToString(), value); + + // read serialized contents of the block + Slice rawblock = builder.Finish(); + ASSERT_LE(rawblock.size(), kMaxBlockSizeSupportedByHashIndex); + std::cerr << "block size: " << rawblock.size() << std::endl; + + // create block reader + BlockContents contents; + contents.data = rawblock; + Block reader(std::move(contents)); + + ASSERT_EQ(reader.IndexType(), + BlockBasedTableOptions::kDataBlockBinaryAndHash); + } + + builder.Reset(); + + { + // insert a large value. The block size plus HashIndex would be 65537. + // This excceed the max block size supported by HashIndex (65536). + // So when build finishes HashIndex will not be created for the block. + std::string value(65503, 'v'); + + builder.Add(ikey.Encode().ToString(), value); + + // read serialized contents of the block + Slice rawblock = builder.Finish(); + ASSERT_LE(rawblock.size(), kMaxBlockSizeSupportedByHashIndex); + std::cerr << "block size: " << rawblock.size() << std::endl; + + // create block reader + BlockContents contents; + contents.data = rawblock; + Block reader(std::move(contents)); + + // the index type have fallen back to binary when build finish. + ASSERT_EQ(reader.IndexType(), + BlockBasedTableOptions::kDataBlockBinarySearch); + } +} + +TEST(DataBlockHashIndex, BlockTestSingleKey) { + Options options = Options(); + + BlockBuilder builder(16 /* block_restart_interval */, + true /* use_delta_encoding */, + false /* use_value_delta_encoding */, + BlockBasedTableOptions::kDataBlockBinaryAndHash); + + std::string ukey("gopher"); + std::string value("gold"); + InternalKey ikey(ukey, 10, kTypeValue); + builder.Add(ikey.Encode().ToString(), value /*value*/); + + // read serialized contents of the block + Slice rawblock = builder.Finish(); + + // create block reader + BlockContents contents; + contents.data = rawblock; + Block reader(std::move(contents)); + + const InternalKeyComparator icmp(BytewiseComparator()); + auto iter = reader.NewDataIterator(icmp.user_comparator(), + kDisableGlobalSequenceNumber); + bool may_exist; + // search in block for the key just inserted + { + InternalKey seek_ikey(ukey, 10, kValueTypeForSeek); + may_exist = iter->SeekForGet(seek_ikey.Encode().ToString()); + ASSERT_TRUE(may_exist); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ( + options.comparator->Compare(iter->key(), ikey.Encode().ToString()), 0); + ASSERT_EQ(iter->value(), value); + } + + // search in block for the existing ukey, but with higher seqno + { + InternalKey seek_ikey(ukey, 20, kValueTypeForSeek); + + // HashIndex should be able to set the iter correctly + may_exist = iter->SeekForGet(seek_ikey.Encode().ToString()); + ASSERT_TRUE(may_exist); + ASSERT_TRUE(iter->Valid()); + + // user key should match + ASSERT_EQ(options.comparator->Compare(ExtractUserKey(iter->key()), ukey), + 0); + + // seek_key seqno number should be greater than that of iter result + ASSERT_GT(GetInternalKeySeqno(seek_ikey.Encode()), + GetInternalKeySeqno(iter->key())); + + ASSERT_EQ(iter->value(), value); + } + + // Search in block for the existing ukey, but with lower seqno + // in this case, hash can find the only occurrence of the user_key, but + // ParseNextDataKey() will skip it as it does not have a older seqno. + // In this case, GetForSeek() is effective to locate the user_key, and + // iter->Valid() == false indicates that we've reached to the end of + // the block and the caller should continue searching the next block. + { + InternalKey seek_ikey(ukey, 5, kValueTypeForSeek); + may_exist = iter->SeekForGet(seek_ikey.Encode().ToString()); + ASSERT_TRUE(may_exist); + ASSERT_FALSE(iter->Valid()); // should have reached to the end of block + } + + delete iter; +} + +TEST(DataBlockHashIndex, BlockTestLarge) { + Random rnd(1019); + Options options = Options(); + std::vector keys; + std::vector values; + + BlockBuilder builder(16 /* block_restart_interval */, + true /* use_delta_encoding */, + false /* use_value_delta_encoding */, + BlockBasedTableOptions::kDataBlockBinaryAndHash); + int num_records = 500; + + GenerateRandomKVs(&keys, &values, 0, num_records); + + // Generate keys. Adding a trailing "1" to indicate existent keys. + // Later will Seeking for keys with a trailing "0" to test seeking + // non-existent keys. + for (int i = 0; i < num_records; i++) { + std::string ukey(keys[i] + "1" /* existing key marker */); + InternalKey ikey(ukey, 0, kTypeValue); + builder.Add(ikey.Encode().ToString(), values[i]); + } + + // read serialized contents of the block + Slice rawblock = builder.Finish(); + + // create block reader + BlockContents contents; + contents.data = rawblock; + Block reader(std::move(contents)); + const InternalKeyComparator icmp(BytewiseComparator()); + + // random seek existent keys + for (int i = 0; i < num_records; i++) { + auto iter = reader.NewDataIterator(icmp.user_comparator(), + kDisableGlobalSequenceNumber); + // find a random key in the lookaside array + int index = rnd.Uniform(num_records); + std::string ukey(keys[index] + "1" /* existing key marker */); + InternalKey ikey(ukey, 0, kTypeValue); + + // search in block for this key + bool may_exist = iter->SeekForGet(ikey.Encode().ToString()); + ASSERT_TRUE(may_exist); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(values[index], iter->value()); + + delete iter; + } + + // random seek non-existent user keys + // In this case A), the user_key cannot be found in HashIndex. The key may + // exist in the next block. So the iter is set invalidated to tell the + // caller to search the next block. This test case belongs to this case A). + // + // Note that for non-existent keys, there is possibility of false positive, + // i.e. the key is still hashed into some restart interval. + // Two additional possible outcome: + // B) linear seek the restart interval and not found, the iter stops at the + // starting of the next restart interval. The key does not exist + // anywhere. + // C) linear seek the restart interval and not found, the iter stops at the + // the end of the block, i.e. restarts_. The key may exist in the next + // block. + // So these combinations are possible when searching non-existent user_key: + // + // case# may_exist iter->Valid() + // A true false + // B false true + // C true false + + for (int i = 0; i < num_records; i++) { + auto iter = reader.NewDataIterator(icmp.user_comparator(), + kDisableGlobalSequenceNumber); + // find a random key in the lookaside array + int index = rnd.Uniform(num_records); + std::string ukey(keys[index] + "0" /* non-existing key marker */); + InternalKey ikey(ukey, 0, kTypeValue); + + // search in block for this key + bool may_exist = iter->SeekForGet(ikey.Encode().ToString()); + if (!may_exist) { + ASSERT_TRUE(iter->Valid()); + } + if (!iter->Valid()) { + ASSERT_TRUE(may_exist); + } + + delete iter; + } +} + +// helper routine for DataBlockHashIndex.BlockBoundary +void TestBoundary(InternalKey& ik1, std::string& v1, InternalKey& ik2, + std::string& v2, InternalKey& seek_ikey, + GetContext& get_context, Options& options) { + std::unique_ptr file_writer; + std::unique_ptr file_reader; + std::unique_ptr table_reader; + int level_ = -1; + + std::vector keys; + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + const InternalKeyComparator internal_comparator(options.comparator); + + EnvOptions soptions; + + soptions.use_mmap_reads = ioptions.allow_mmap_reads; + test::StringSink* sink = new test::StringSink(); + std::unique_ptr f(sink); + file_writer.reset( + new WritableFileWriter(std::move(f), "" /* don't care */, FileOptions())); + std::unique_ptr builder; + IntTblPropCollectorFactories int_tbl_prop_collector_factories; + std::string column_family_name; + builder.reset(ioptions.table_factory->NewTableBuilder( + TableBuilderOptions( + ioptions, moptions, internal_comparator, + &int_tbl_prop_collector_factories, options.compression, + CompressionOptions(), + TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, + column_family_name, level_), + file_writer.get())); + + builder->Add(ik1.Encode().ToString(), v1); + builder->Add(ik2.Encode().ToString(), v2); + EXPECT_TRUE(builder->status().ok()); + + Status s = builder->Finish(); + ASSERT_OK(file_writer->Flush()); + EXPECT_TRUE(s.ok()) << s.ToString(); + + EXPECT_EQ(sink->contents().size(), builder->FileSize()); + + // Open the table + test::StringSource* source = new test::StringSource( + sink->contents(), 0 /*uniq_id*/, ioptions.allow_mmap_reads); + std::unique_ptr file(source); + file_reader.reset(new RandomAccessFileReader(std::move(file), "test")); + const bool kSkipFilters = true; + const bool kImmortal = true; + ASSERT_OK(ioptions.table_factory->NewTableReader( + TableReaderOptions(ioptions, moptions.prefix_extractor, soptions, + internal_comparator, + 0 /* block_protection_bytes_per_key */, !kSkipFilters, + !kImmortal, level_), + std::move(file_reader), sink->contents().size(), &table_reader)); + // Search using Get() + ReadOptions ro; + + ASSERT_OK(table_reader->Get(ro, seek_ikey.Encode().ToString(), &get_context, + moptions.prefix_extractor.get())); +} + +TEST(DataBlockHashIndex, BlockBoundary) { + BlockBasedTableOptions table_options; + table_options.data_block_index_type = + BlockBasedTableOptions::kDataBlockBinaryAndHash; + table_options.block_restart_interval = 1; + table_options.block_size = 4096; + + Options options; + options.comparator = BytewiseComparator(); + + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + // insert two large k/v pair. Given that the block_size is 4096, one k/v + // pair will take up one block. + // [ k1/v1 ][ k2/v2 ] + // [ Block N ][ Block N+1 ] + + { + // [ "aab"@100 ][ "axy"@10 ] + // | Block N ][ Block N+1 ] + // seek for "axy"@60 + std::string uk1("aab"); + InternalKey ik1(uk1, 100, kTypeValue); + std::string v1(4100, '1'); // large value + + std::string uk2("axy"); + InternalKey ik2(uk2, 10, kTypeValue); + std::string v2(4100, '2'); // large value + + PinnableSlice value; + std::string seek_ukey("axy"); + InternalKey seek_ikey(seek_ukey, 60, kTypeValue); + GetContext get_context(options.comparator, nullptr, nullptr, nullptr, + GetContext::kNotFound, seek_ukey, &value, nullptr, + nullptr, nullptr, true, nullptr, nullptr); + + TestBoundary(ik1, v1, ik2, v2, seek_ikey, get_context, options); + ASSERT_EQ(get_context.State(), GetContext::kFound); + ASSERT_EQ(value, v2); + value.Reset(); + } + + { + // [ "axy"@100 ][ "axy"@10 ] + // | Block N ][ Block N+1 ] + // seek for "axy"@60 + std::string uk1("axy"); + InternalKey ik1(uk1, 100, kTypeValue); + std::string v1(4100, '1'); // large value + + std::string uk2("axy"); + InternalKey ik2(uk2, 10, kTypeValue); + std::string v2(4100, '2'); // large value + + PinnableSlice value; + std::string seek_ukey("axy"); + InternalKey seek_ikey(seek_ukey, 60, kTypeValue); + GetContext get_context(options.comparator, nullptr, nullptr, nullptr, + GetContext::kNotFound, seek_ukey, &value, nullptr, + nullptr, nullptr, true, nullptr, nullptr); + + TestBoundary(ik1, v1, ik2, v2, seek_ikey, get_context, options); + ASSERT_EQ(get_context.State(), GetContext::kFound); + ASSERT_EQ(value, v2); + value.Reset(); + } + + { + // [ "axy"@100 ][ "axy"@10 ] + // | Block N ][ Block N+1 ] + // seek for "axy"@120 + std::string uk1("axy"); + InternalKey ik1(uk1, 100, kTypeValue); + std::string v1(4100, '1'); // large value + + std::string uk2("axy"); + InternalKey ik2(uk2, 10, kTypeValue); + std::string v2(4100, '2'); // large value + + PinnableSlice value; + std::string seek_ukey("axy"); + InternalKey seek_ikey(seek_ukey, 120, kTypeValue); + GetContext get_context(options.comparator, nullptr, nullptr, nullptr, + GetContext::kNotFound, seek_ukey, &value, nullptr, + nullptr, nullptr, true, nullptr, nullptr); + + TestBoundary(ik1, v1, ik2, v2, seek_ikey, get_context, options); + ASSERT_EQ(get_context.State(), GetContext::kFound); + ASSERT_EQ(value, v1); + value.Reset(); + } + + { + // [ "axy"@100 ][ "axy"@10 ] + // | Block N ][ Block N+1 ] + // seek for "axy"@5 + std::string uk1("axy"); + InternalKey ik1(uk1, 100, kTypeValue); + std::string v1(4100, '1'); // large value + + std::string uk2("axy"); + InternalKey ik2(uk2, 10, kTypeValue); + std::string v2(4100, '2'); // large value + + PinnableSlice value; + std::string seek_ukey("axy"); + InternalKey seek_ikey(seek_ukey, 5, kTypeValue); + GetContext get_context(options.comparator, nullptr, nullptr, nullptr, + GetContext::kNotFound, seek_ukey, &value, nullptr, + nullptr, nullptr, true, nullptr, nullptr); + + TestBoundary(ik1, v1, ik2, v2, seek_ikey, get_context, options); + ASSERT_EQ(get_context.State(), GetContext::kNotFound); + value.Reset(); + } +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/table/block_based/filter_block.h b/librocksdb-sys/rocksdb/table/block_based/filter_block.h new file mode 100644 index 0000000..b14858c --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/filter_block.h @@ -0,0 +1,183 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// A filter block is stored near the end of a Table file. It contains +// filters (e.g., bloom filters) for all data blocks in the table combined +// into a single filter block. + +#pragma once + +#include +#include + +#include +#include +#include + +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/table.h" +#include "table/format.h" +#include "table/multiget_context.h" +#include "trace_replay/block_cache_tracer.h" +#include "util/hash.h" + +namespace ROCKSDB_NAMESPACE { + +const uint64_t kNotValid = ULLONG_MAX; +class FilterPolicy; + +class GetContext; +using MultiGetRange = MultiGetContext::Range; + +// A FilterBlockBuilder is used to construct all of the filters for a +// particular Table. It generates a single string which is stored as +// a special block in the Table, or partitioned into smaller filters. +// +// The sequence of calls to FilterBlockBuilder must match the regexp: +// Add* Finish +class FilterBlockBuilder { + public: + explicit FilterBlockBuilder() {} + // No copying allowed + FilterBlockBuilder(const FilterBlockBuilder&) = delete; + void operator=(const FilterBlockBuilder&) = delete; + + virtual ~FilterBlockBuilder() {} + + virtual void Add( + const Slice& key_without_ts) = 0; // Add a key to current filter + virtual bool IsEmpty() const = 0; // Empty == none added + // For reporting stats on how many entries the builder considered unique + virtual size_t EstimateEntriesAdded() = 0; + Slice Finish() { // Generate Filter + const BlockHandle empty_handle; + Status dont_care_status; + auto ret = Finish(empty_handle, &dont_care_status); + assert(dont_care_status.ok()); + return ret; + } + // If filter_data is not nullptr, Finish() may transfer ownership of + // underlying filter data to the caller, so that it can be freed as soon as + // possible. BlockBasedFilterBlock will ignore this parameter. + // + virtual Slice Finish( + const BlockHandle& tmp /* only used in PartitionedFilterBlock as + last_partition_block_handle */ + , + Status* status, std::unique_ptr* filter_data = nullptr) = 0; + + // This is called when finishes using the FilterBitsBuilder + // in order to release memory usage and cache charge + // associated with it timely + virtual void ResetFilterBitsBuilder() {} + + // To optionally post-verify the filter returned from + // FilterBlockBuilder::Finish. + // Return Status::OK() if skipped. + virtual Status MaybePostVerifyFilter(const Slice& /* filter_content */) { + return Status::OK(); + } +}; + +// A FilterBlockReader is used to parse filter from SST table. +// KeyMayMatch and PrefixMayMatch would trigger filter checking +// +// BlockBased/Full FilterBlock would be called in the same way. +class FilterBlockReader { + public: + FilterBlockReader() = default; + virtual ~FilterBlockReader() = default; + + FilterBlockReader(const FilterBlockReader&) = delete; + FilterBlockReader& operator=(const FilterBlockReader&) = delete; + + /** + * If no_io is set, then it returns true if it cannot answer the query without + * reading data from disk. This is used in PartitionedFilterBlockReader to + * avoid reading partitions that are not in block cache already + * + * Normally filters are built on only the user keys and the InternalKey is not + * needed for a query. The index in PartitionedFilterBlockReader however is + * built upon InternalKey and must be provided via const_ikey_ptr when running + * queries. + */ + virtual bool KeyMayMatch(const Slice& key, const bool no_io, + const Slice* const const_ikey_ptr, + GetContext* get_context, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) = 0; + + virtual void KeysMayMatch(MultiGetRange* range, const bool no_io, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) { + for (auto iter = range->begin(); iter != range->end(); ++iter) { + const Slice ukey_without_ts = iter->ukey_without_ts; + const Slice ikey = iter->ikey; + GetContext* const get_context = iter->get_context; + if (!KeyMayMatch(ukey_without_ts, no_io, &ikey, get_context, + lookup_context, read_options)) { + range->SkipKey(iter); + } + } + } + + /** + * no_io and const_ikey_ptr here means the same as in KeyMayMatch + */ + virtual bool PrefixMayMatch(const Slice& prefix, const bool no_io, + const Slice* const const_ikey_ptr, + GetContext* get_context, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) = 0; + + virtual void PrefixesMayMatch(MultiGetRange* range, + const SliceTransform* prefix_extractor, + const bool no_io, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) { + for (auto iter = range->begin(); iter != range->end(); ++iter) { + const Slice ukey_without_ts = iter->ukey_without_ts; + const Slice ikey = iter->ikey; + GetContext* const get_context = iter->get_context; + if (prefix_extractor->InDomain(ukey_without_ts) && + !PrefixMayMatch(prefix_extractor->Transform(ukey_without_ts), no_io, + &ikey, get_context, lookup_context, read_options)) { + range->SkipKey(iter); + } + } + } + + virtual size_t ApproximateMemoryUsage() const = 0; + + // convert this object to a human readable form + virtual std::string ToString() const { + std::string error_msg("Unsupported filter \n"); + return error_msg; + } + + virtual Status CacheDependencies( + const ReadOptions& /*ro*/, bool /*pin*/, + FilePrefetchBuffer* /* tail_prefetch_buffer */) { + return Status::OK(); + } + + virtual bool RangeMayExist(const Slice* /*iterate_upper_bound*/, + const Slice& user_key_without_ts, + const SliceTransform* prefix_extractor, + const Comparator* /*comparator*/, + const Slice* const const_ikey_ptr, + bool* filter_checked, bool need_upper_bound_check, + bool no_io, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) = 0; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/filter_block_reader_common.cc b/librocksdb-sys/rocksdb/table/block_based/filter_block_reader_common.cc new file mode 100644 index 0000000..32f800d --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/filter_block_reader_common.cc @@ -0,0 +1,163 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#include "table/block_based/filter_block_reader_common.h" + +#include "block_cache.h" +#include "monitoring/perf_context_imp.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/block_based/parsed_full_filter_block.h" + +namespace ROCKSDB_NAMESPACE { + +template +Status FilterBlockReaderCommon::ReadFilterBlock( + const BlockBasedTable* table, FilePrefetchBuffer* prefetch_buffer, + const ReadOptions& read_options, bool use_cache, GetContext* get_context, + BlockCacheLookupContext* lookup_context, + CachableEntry* filter_block) { + PERF_TIMER_GUARD(read_filter_block_nanos); + + assert(table); + assert(filter_block); + assert(filter_block->IsEmpty()); + + const BlockBasedTable::Rep* const rep = table->get_rep(); + assert(rep); + + const Status s = + table->RetrieveBlock(prefetch_buffer, read_options, rep->filter_handle, + UncompressionDict::GetEmptyDict(), filter_block, + get_context, lookup_context, + /* for_compaction */ false, use_cache, + /* async_read */ false); + + return s; +} + +template +const SliceTransform* +FilterBlockReaderCommon::table_prefix_extractor() const { + assert(table_); + + const BlockBasedTable::Rep* const rep = table_->get_rep(); + assert(rep); + + return rep->prefix_filtering ? rep->table_prefix_extractor.get() : nullptr; +} + +template +bool FilterBlockReaderCommon::whole_key_filtering() const { + assert(table_); + assert(table_->get_rep()); + + return table_->get_rep()->whole_key_filtering; +} + +template +bool FilterBlockReaderCommon::cache_filter_blocks() const { + assert(table_); + assert(table_->get_rep()); + + return table_->get_rep()->table_options.cache_index_and_filter_blocks; +} + +template +Status FilterBlockReaderCommon::GetOrReadFilterBlock( + bool no_io, GetContext* get_context, + BlockCacheLookupContext* lookup_context, + CachableEntry* filter_block, + const ReadOptions& read_options) const { + assert(filter_block); + + if (!filter_block_.IsEmpty()) { + filter_block->SetUnownedValue(filter_block_.GetValue()); + return Status::OK(); + } + + ReadOptions ro = read_options; + if (no_io) { + ro.read_tier = kBlockCacheTier; + } + + return ReadFilterBlock(table_, nullptr /* prefetch_buffer */, ro, + cache_filter_blocks(), get_context, lookup_context, + filter_block); +} + +template +size_t FilterBlockReaderCommon::ApproximateFilterBlockMemoryUsage() + const { + assert(!filter_block_.GetOwnValue() || filter_block_.GetValue() != nullptr); + return filter_block_.GetOwnValue() + ? filter_block_.GetValue()->ApproximateMemoryUsage() + : 0; +} + +template +bool FilterBlockReaderCommon::RangeMayExist( + const Slice* iterate_upper_bound, const Slice& user_key_without_ts, + const SliceTransform* prefix_extractor, const Comparator* comparator, + const Slice* const const_ikey_ptr, bool* filter_checked, + bool need_upper_bound_check, bool no_io, + BlockCacheLookupContext* lookup_context, const ReadOptions& read_options) { + if (!prefix_extractor || !prefix_extractor->InDomain(user_key_without_ts)) { + *filter_checked = false; + return true; + } + Slice prefix = prefix_extractor->Transform(user_key_without_ts); + if (need_upper_bound_check && + !IsFilterCompatible(iterate_upper_bound, prefix, comparator)) { + *filter_checked = false; + return true; + } else { + *filter_checked = true; + return PrefixMayMatch(prefix, no_io, const_ikey_ptr, + /* get_context */ nullptr, lookup_context, + read_options); + } +} + +template +bool FilterBlockReaderCommon::IsFilterCompatible( + const Slice* iterate_upper_bound, const Slice& prefix, + const Comparator* comparator) const { + // Try to reuse the bloom filter in the SST table if prefix_extractor in + // mutable_cf_options has changed. If range [user_key, upper_bound) all + // share the same prefix then we may still be able to use the bloom filter. + const SliceTransform* const prefix_extractor = table_prefix_extractor(); + if (iterate_upper_bound != nullptr && prefix_extractor) { + if (!prefix_extractor->InDomain(*iterate_upper_bound)) { + return false; + } + Slice upper_bound_xform = prefix_extractor->Transform(*iterate_upper_bound); + // first check if user_key and upper_bound all share the same prefix + if (comparator->CompareWithoutTimestamp(prefix, false, upper_bound_xform, + false) != 0) { + // second check if user_key's prefix is the immediate predecessor of + // upper_bound and have the same length. If so, we know for sure all + // keys in the range [user_key, upper_bound) share the same prefix. + // Also need to make sure upper_bound are full length to ensure + // correctness + if (!full_length_enabled_ || + iterate_upper_bound->size() != prefix_extractor_full_length_ || + !comparator->IsSameLengthImmediateSuccessor(prefix, + *iterate_upper_bound)) { + return false; + } + } + return true; + } else { + return false; + } +} + +// Explicitly instantiate templates for both "blocklike" types we use. +// This makes it possible to keep the template definitions in the .cc file. +template class FilterBlockReaderCommon; +template class FilterBlockReaderCommon; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/filter_block_reader_common.h b/librocksdb-sys/rocksdb/table/block_based/filter_block_reader_common.h new file mode 100644 index 0000000..62335b3 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/filter_block_reader_common.h @@ -0,0 +1,76 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#pragma once + +#include + +#include "table/block_based/cachable_entry.h" +#include "table/block_based/filter_block.h" + +namespace ROCKSDB_NAMESPACE { + +class BlockBasedTable; +class FilePrefetchBuffer; + +// Encapsulates common functionality for the various filter block reader +// implementations. Provides access to the filter block regardless of whether +// it is owned by the reader or stored in the cache, or whether it is pinned +// in the cache or not. +template +class FilterBlockReaderCommon : public FilterBlockReader { + public: + FilterBlockReaderCommon(const BlockBasedTable* t, + CachableEntry&& filter_block) + : table_(t), filter_block_(std::move(filter_block)) { + assert(table_); + const SliceTransform* const prefix_extractor = table_prefix_extractor(); + if (prefix_extractor) { + full_length_enabled_ = + prefix_extractor->FullLengthEnabled(&prefix_extractor_full_length_); + } + } + + bool RangeMayExist(const Slice* iterate_upper_bound, const Slice& user_key, + const SliceTransform* prefix_extractor, + const Comparator* comparator, + const Slice* const const_ikey_ptr, bool* filter_checked, + bool need_upper_bound_check, bool no_io, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) override; + + protected: + static Status ReadFilterBlock(const BlockBasedTable* table, + FilePrefetchBuffer* prefetch_buffer, + const ReadOptions& read_options, bool use_cache, + GetContext* get_context, + BlockCacheLookupContext* lookup_context, + CachableEntry* filter_block); + + const BlockBasedTable* table() const { return table_; } + const SliceTransform* table_prefix_extractor() const; + bool whole_key_filtering() const; + bool cache_filter_blocks() const; + + Status GetOrReadFilterBlock(bool no_io, GetContext* get_context, + BlockCacheLookupContext* lookup_context, + CachableEntry* filter_block, + const ReadOptions& read_options) const; + + size_t ApproximateFilterBlockMemoryUsage() const; + + private: + bool IsFilterCompatible(const Slice* iterate_upper_bound, const Slice& prefix, + const Comparator* comparator) const; + + private: + const BlockBasedTable* table_; + CachableEntry filter_block_; + size_t prefix_extractor_full_length_ = 0; + bool full_length_enabled_ = false; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/filter_policy.cc b/librocksdb-sys/rocksdb/table/block_based/filter_policy.cc new file mode 100644 index 0000000..36f3b16 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/filter_policy.cc @@ -0,0 +1,1966 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "rocksdb/filter_policy.h" + +#include +#include +#include +#include +#include +#include + +#include "cache/cache_entry_roles.h" +#include "cache/cache_reservation_manager.h" +#include "logging/logging.h" +#include "port/lang.h" +#include "rocksdb/convenience.h" +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/slice.h" +#include "rocksdb/utilities/object_registry.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/block_based/filter_policy_internal.h" +#include "table/block_based/full_filter_block.h" +#include "util/bloom_impl.h" +#include "util/coding.h" +#include "util/hash.h" +#include "util/math.h" +#include "util/ribbon_config.h" +#include "util/ribbon_impl.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { + +// Metadata trailer size for built-in filters. (This is separate from +// block-based table block trailer.) +// +// Originally this was 1 byte for num_probes and 4 bytes for number of +// cache lines in the Bloom filter, but now the first trailer byte is +// usually an implementation marker and remaining 4 bytes have various +// meanings. +static constexpr uint32_t kMetadataLen = 5; + +Slice FinishAlwaysFalse(std::unique_ptr* /*buf*/) { + // Missing metadata, treated as zero entries + return Slice(nullptr, 0); +} + +Slice FinishAlwaysTrue(std::unique_ptr* /*buf*/) { + return Slice("\0\0\0\0\0\0", 6); +} + +// Base class for filter builders using the XXH3 preview hash, +// also known as Hash64 or GetSliceHash64. +class XXPH3FilterBitsBuilder : public BuiltinFilterBitsBuilder { + public: + explicit XXPH3FilterBitsBuilder( + std::atomic* aggregate_rounding_balance, + std::shared_ptr cache_res_mgr, + bool detect_filter_construct_corruption) + : aggregate_rounding_balance_(aggregate_rounding_balance), + cache_res_mgr_(cache_res_mgr), + detect_filter_construct_corruption_( + detect_filter_construct_corruption) {} + + ~XXPH3FilterBitsBuilder() override {} + + virtual void AddKey(const Slice& key) override { + uint64_t hash = GetSliceHash64(key); + // Especially with prefixes, it is common to have repetition, + // though only adjacent repetition, which we want to immediately + // recognize and collapse for estimating true filter space + // requirements. + if (hash_entries_info_.entries.empty() || + hash != hash_entries_info_.entries.back()) { + if (detect_filter_construct_corruption_) { + hash_entries_info_.xor_checksum ^= hash; + } + hash_entries_info_.entries.push_back(hash); + if (cache_res_mgr_ && + // Traditional rounding to whole bucket size + ((hash_entries_info_.entries.size() % + kUint64tHashEntryCacheResBucketSize) == + kUint64tHashEntryCacheResBucketSize / 2)) { + hash_entries_info_.cache_res_bucket_handles.emplace_back(nullptr); + Status s = cache_res_mgr_->MakeCacheReservation( + kUint64tHashEntryCacheResBucketSize * sizeof(hash), + &hash_entries_info_.cache_res_bucket_handles.back()); + s.PermitUncheckedError(); + } + } + } + + virtual size_t EstimateEntriesAdded() override { + return hash_entries_info_.entries.size(); + } + + virtual Status MaybePostVerify(const Slice& filter_content) override; + + protected: + static constexpr uint32_t kMetadataLen = 5; + + // Number of hash entries to accumulate before charging their memory usage to + // the cache when cache charging is available + static const std::size_t kUint64tHashEntryCacheResBucketSize = + CacheReservationManagerImpl< + CacheEntryRole::kFilterConstruction>::GetDummyEntrySize() / + sizeof(uint64_t); + + // For delegating between XXPH3FilterBitsBuilders + void SwapEntriesWith(XXPH3FilterBitsBuilder* other) { + assert(other != nullptr); + hash_entries_info_.Swap(&(other->hash_entries_info_)); + } + + void ResetEntries() { hash_entries_info_.Reset(); } + + virtual size_t RoundDownUsableSpace(size_t available_size) = 0; + + // To choose size using malloc_usable_size, we have to actually allocate. + size_t AllocateMaybeRounding(size_t target_len_with_metadata, + size_t num_entries, + std::unique_ptr* buf) { + // Return value set to a default; overwritten in some cases + size_t rv = target_len_with_metadata; +#ifdef ROCKSDB_MALLOC_USABLE_SIZE + if (aggregate_rounding_balance_ != nullptr) { + // Do optimize_filters_for_memory, using malloc_usable_size. + // Approach: try to keep FP rate balance better than or on + // target (negative aggregate_rounding_balance_). We can then select a + // lower bound filter size (within reasonable limits) that gets us as + // close to on target as possible. We request allocation for that filter + // size and use malloc_usable_size to "round up" to the actual + // allocation size. + + // Although it can be considered bad practice to use malloc_usable_size + // to access an object beyond its original size, this approach should be + // quite general: working for all allocators that properly support + // malloc_usable_size. + + // Race condition on balance is OK because it can only cause temporary + // skew in rounding up vs. rounding down, as long as updates are atomic + // and relative. + int64_t balance = aggregate_rounding_balance_->load(); + + double target_fp_rate = + EstimatedFpRate(num_entries, target_len_with_metadata); + double rv_fp_rate = target_fp_rate; + + if (balance < 0) { + // See formula for BloomFilterPolicy::aggregate_rounding_balance_ + double for_balance_fp_rate = + -balance / double{0x100000000} + target_fp_rate; + + // To simplify, we just try a few modified smaller sizes. This also + // caps how much we vary filter size vs. target, to avoid outlier + // behavior from excessive variance. + size_t target_len = target_len_with_metadata - kMetadataLen; + assert(target_len < target_len_with_metadata); // check underflow + for (uint64_t maybe_len_rough : + {uint64_t{3} * target_len / 4, uint64_t{13} * target_len / 16, + uint64_t{7} * target_len / 8, uint64_t{15} * target_len / 16}) { + size_t maybe_len_with_metadata = + RoundDownUsableSpace(maybe_len_rough + kMetadataLen); + double maybe_fp_rate = + EstimatedFpRate(num_entries, maybe_len_with_metadata); + if (maybe_fp_rate <= for_balance_fp_rate) { + rv = maybe_len_with_metadata; + rv_fp_rate = maybe_fp_rate; + break; + } + } + } + + // Filter blocks are loaded into block cache with their block trailer. + // We need to make sure that's accounted for in choosing a + // fragmentation-friendly size. + const size_t kExtraPadding = BlockBasedTable::kBlockTrailerSize; + size_t requested = rv + kExtraPadding; + + // Allocate and get usable size + buf->reset(new char[requested]); + size_t usable = malloc_usable_size(buf->get()); + + if (usable - usable / 4 > requested) { + // Ratio greater than 4/3 is too much for utilizing, if it's + // not a buggy or mislinked malloc_usable_size implementation. + // Non-linearity of FP rates with bits/key means rapidly + // diminishing returns in overall accuracy for additional + // storage on disk. + // Nothing to do, except assert that the result is accurate about + // the usable size. (Assignment never used.) + assert(((*buf)[usable - 1] = 'x')); + } else if (usable > requested) { + rv = RoundDownUsableSpace(usable - kExtraPadding); + assert(rv <= usable - kExtraPadding); + rv_fp_rate = EstimatedFpRate(num_entries, rv); + } else { + // Too small means bad malloc_usable_size + assert(usable == requested); + } + memset(buf->get(), 0, rv); + + // Update balance + int64_t diff = static_cast((rv_fp_rate - target_fp_rate) * + double{0x100000000}); + *aggregate_rounding_balance_ += diff; + } else { + buf->reset(new char[rv]()); + } +#else + (void)num_entries; + buf->reset(new char[rv]()); +#endif // ROCKSDB_MALLOC_USABLE_SIZE + return rv; + } + + // TODO: Ideally we want to verify the hash entry + // as it is added to the filter and eliminate this function + // for speeding up and leaving fewer spaces for undetected memory/CPU + // corruption. For Ribbon Filter, it's bit harder. + // Possible solution: + // pass a custom iterator that tracks the xor checksum as + // it iterates to ResetAndFindSeedToSolve + Status MaybeVerifyHashEntriesChecksum() { + if (!detect_filter_construct_corruption_) { + return Status::OK(); + } + + uint64_t actual_hash_entries_xor_checksum = 0; + for (uint64_t h : hash_entries_info_.entries) { + actual_hash_entries_xor_checksum ^= h; + } + + if (actual_hash_entries_xor_checksum == hash_entries_info_.xor_checksum) { + return Status::OK(); + } else { + // Since these hash entries are corrupted and they will not be used + // anymore, we can reset them and release memory. + ResetEntries(); + return Status::Corruption("Filter's hash entries checksum mismatched"); + } + } + + // See BloomFilterPolicy::aggregate_rounding_balance_. If nullptr, + // always "round up" like historic behavior. + std::atomic* aggregate_rounding_balance_; + + // For reserving memory used in (new) Bloom and Ribbon Filter construction + std::shared_ptr cache_res_mgr_; + + // For managing cache charge for final filter in (new) Bloom and Ribbon + // Filter construction + std::deque> + final_filter_cache_res_handles_; + + bool detect_filter_construct_corruption_; + + struct HashEntriesInfo { + // A deque avoids unnecessary copying of already-saved values + // and has near-minimal peak memory use. + std::deque entries; + + // If cache_res_mgr_ != nullptr, + // it manages cache charge for buckets of hash entries in (new) Bloom + // or Ribbon Filter construction. + // Otherwise, it is empty. + std::deque> + cache_res_bucket_handles; + + // If detect_filter_construct_corruption_ == true, + // it records the xor checksum of hash entries. + // Otherwise, it is 0. + uint64_t xor_checksum = 0; + + void Swap(HashEntriesInfo* other) { + assert(other != nullptr); + std::swap(entries, other->entries); + std::swap(cache_res_bucket_handles, other->cache_res_bucket_handles); + std::swap(xor_checksum, other->xor_checksum); + } + + void Reset() { + entries.clear(); + cache_res_bucket_handles.clear(); + xor_checksum = 0; + } + }; + + HashEntriesInfo hash_entries_info_; +}; + +// #################### FastLocalBloom implementation ################## // +// ############## also known as format_version=5 Bloom filter ########## // + +// See description in FastLocalBloomImpl +class FastLocalBloomBitsBuilder : public XXPH3FilterBitsBuilder { + public: + // Non-null aggregate_rounding_balance implies optimize_filters_for_memory + explicit FastLocalBloomBitsBuilder( + const int millibits_per_key, + std::atomic* aggregate_rounding_balance, + std::shared_ptr cache_res_mgr, + bool detect_filter_construct_corruption) + : XXPH3FilterBitsBuilder(aggregate_rounding_balance, cache_res_mgr, + detect_filter_construct_corruption), + millibits_per_key_(millibits_per_key) { + assert(millibits_per_key >= 1000); + } + + // No Copy allowed + FastLocalBloomBitsBuilder(const FastLocalBloomBitsBuilder&) = delete; + void operator=(const FastLocalBloomBitsBuilder&) = delete; + + ~FastLocalBloomBitsBuilder() override {} + + using FilterBitsBuilder::Finish; + + virtual Slice Finish(std::unique_ptr* buf) override { + return Finish(buf, nullptr); + } + + virtual Slice Finish(std::unique_ptr* buf, + Status* status) override { + size_t num_entries = hash_entries_info_.entries.size(); + size_t len_with_metadata = CalculateSpace(num_entries); + + std::unique_ptr mutable_buf; + std::unique_ptr + final_filter_cache_res_handle; + len_with_metadata = + AllocateMaybeRounding(len_with_metadata, num_entries, &mutable_buf); + // Cache charging for mutable_buf + if (cache_res_mgr_) { + Status s = cache_res_mgr_->MakeCacheReservation( + len_with_metadata * sizeof(char), &final_filter_cache_res_handle); + s.PermitUncheckedError(); + } + + assert(mutable_buf); + assert(len_with_metadata >= kMetadataLen); + + // Max size supported by implementation + assert(len_with_metadata <= 0xffffffffU); + + // Compute num_probes after any rounding / adjustments + int num_probes = GetNumProbes(num_entries, len_with_metadata); + + uint32_t len = static_cast(len_with_metadata - kMetadataLen); + if (len > 0) { + TEST_SYNC_POINT_CALLBACK( + "XXPH3FilterBitsBuilder::Finish::" + "TamperHashEntries", + &hash_entries_info_.entries); + AddAllEntries(mutable_buf.get(), len, num_probes); + Status verify_hash_entries_checksum_status = + MaybeVerifyHashEntriesChecksum(); + if (!verify_hash_entries_checksum_status.ok()) { + if (status) { + *status = verify_hash_entries_checksum_status; + } + return FinishAlwaysTrue(buf); + } + } + + bool keep_entries_for_postverify = detect_filter_construct_corruption_; + if (!keep_entries_for_postverify) { + ResetEntries(); + } + + // See BloomFilterPolicy::GetBloomBitsReader re: metadata + // -1 = Marker for newer Bloom implementations + mutable_buf[len] = static_cast(-1); + // 0 = Marker for this sub-implementation + mutable_buf[len + 1] = static_cast(0); + // num_probes (and 0 in upper bits for 64-byte block size) + mutable_buf[len + 2] = static_cast(num_probes); + // rest of metadata stays zero + + auto TEST_arg_pair __attribute__((__unused__)) = + std::make_pair(&mutable_buf, len_with_metadata); + TEST_SYNC_POINT_CALLBACK("XXPH3FilterBitsBuilder::Finish::TamperFilter", + &TEST_arg_pair); + + Slice rv(mutable_buf.get(), len_with_metadata); + *buf = std::move(mutable_buf); + final_filter_cache_res_handles_.push_back( + std::move(final_filter_cache_res_handle)); + if (status) { + *status = Status::OK(); + } + return rv; + } + + size_t ApproximateNumEntries(size_t bytes) override { + size_t bytes_no_meta = + bytes >= kMetadataLen ? RoundDownUsableSpace(bytes) - kMetadataLen : 0; + return static_cast(uint64_t{8000} * bytes_no_meta / + millibits_per_key_); + } + + size_t CalculateSpace(size_t num_entries) override { + // If not for cache line blocks in the filter, what would the target + // length in bytes be? + size_t raw_target_len = static_cast( + (uint64_t{num_entries} * millibits_per_key_ + 7999) / 8000); + + if (raw_target_len >= size_t{0xffffffc0}) { + // Max supported for this data structure implementation + raw_target_len = size_t{0xffffffc0}; + } + + // Round up to nearest multiple of 64 (block size). This adjustment is + // used for target FP rate only so that we don't receive complaints about + // lower FP rate vs. historic Bloom filter behavior. + return ((raw_target_len + 63) & ~size_t{63}) + kMetadataLen; + } + + double EstimatedFpRate(size_t keys, size_t len_with_metadata) override { + int num_probes = GetNumProbes(keys, len_with_metadata); + return FastLocalBloomImpl::EstimatedFpRate( + keys, len_with_metadata - kMetadataLen, num_probes, /*hash bits*/ 64); + } + + protected: + size_t RoundDownUsableSpace(size_t available_size) override { + size_t rv = available_size - kMetadataLen; + + if (rv >= size_t{0xffffffc0}) { + // Max supported for this data structure implementation + rv = size_t{0xffffffc0}; + } + + // round down to multiple of 64 (block size) + rv &= ~size_t{63}; + + return rv + kMetadataLen; + } + + private: + // Compute num_probes after any rounding / adjustments + int GetNumProbes(size_t keys, size_t len_with_metadata) { + uint64_t millibits = uint64_t{len_with_metadata - kMetadataLen} * 8000; + int actual_millibits_per_key = + static_cast(millibits / std::max(keys, size_t{1})); + // BEGIN XXX/TODO(peterd): preserving old/default behavior for now to + // minimize unit test churn. Remove this some time. + if (!aggregate_rounding_balance_) { + actual_millibits_per_key = millibits_per_key_; + } + // END XXX/TODO + return FastLocalBloomImpl::ChooseNumProbes(actual_millibits_per_key); + } + + void AddAllEntries(char* data, uint32_t len, int num_probes) { + // Simple version without prefetching: + // + // for (auto h : hash_entries_info_.entries) { + // FastLocalBloomImpl::AddHash(Lower32of64(h), Upper32of64(h), len, + // num_probes, data); + // } + + const size_t num_entries = hash_entries_info_.entries.size(); + constexpr size_t kBufferMask = 7; + static_assert(((kBufferMask + 1) & kBufferMask) == 0, + "Must be power of 2 minus 1"); + + std::array hashes; + std::array byte_offsets; + + // Prime the buffer + size_t i = 0; + std::deque::iterator hash_entries_it = + hash_entries_info_.entries.begin(); + for (; i <= kBufferMask && i < num_entries; ++i) { + uint64_t h = *hash_entries_it; + FastLocalBloomImpl::PrepareHash(Lower32of64(h), len, data, + /*out*/ &byte_offsets[i]); + hashes[i] = Upper32of64(h); + ++hash_entries_it; + } + + // Process and buffer + for (; i < num_entries; ++i) { + uint32_t& hash_ref = hashes[i & kBufferMask]; + uint32_t& byte_offset_ref = byte_offsets[i & kBufferMask]; + // Process (add) + FastLocalBloomImpl::AddHashPrepared(hash_ref, num_probes, + data + byte_offset_ref); + // And buffer + uint64_t h = *hash_entries_it; + FastLocalBloomImpl::PrepareHash(Lower32of64(h), len, data, + /*out*/ &byte_offset_ref); + hash_ref = Upper32of64(h); + ++hash_entries_it; + } + + // Finish processing + for (i = 0; i <= kBufferMask && i < num_entries; ++i) { + FastLocalBloomImpl::AddHashPrepared(hashes[i], num_probes, + data + byte_offsets[i]); + } + } + + // Target allocation per added key, in thousandths of a bit. + int millibits_per_key_; +}; + +// See description in FastLocalBloomImpl +class FastLocalBloomBitsReader : public BuiltinFilterBitsReader { + public: + FastLocalBloomBitsReader(const char* data, int num_probes, uint32_t len_bytes) + : data_(data), num_probes_(num_probes), len_bytes_(len_bytes) {} + + // No Copy allowed + FastLocalBloomBitsReader(const FastLocalBloomBitsReader&) = delete; + void operator=(const FastLocalBloomBitsReader&) = delete; + + ~FastLocalBloomBitsReader() override {} + + bool MayMatch(const Slice& key) override { + uint64_t h = GetSliceHash64(key); + uint32_t byte_offset; + FastLocalBloomImpl::PrepareHash(Lower32of64(h), len_bytes_, data_, + /*out*/ &byte_offset); + return FastLocalBloomImpl::HashMayMatchPrepared(Upper32of64(h), num_probes_, + data_ + byte_offset); + } + + virtual void MayMatch(int num_keys, Slice** keys, bool* may_match) override { + std::array hashes; + std::array byte_offsets; + for (int i = 0; i < num_keys; ++i) { + uint64_t h = GetSliceHash64(*keys[i]); + FastLocalBloomImpl::PrepareHash(Lower32of64(h), len_bytes_, data_, + /*out*/ &byte_offsets[i]); + hashes[i] = Upper32of64(h); + } + for (int i = 0; i < num_keys; ++i) { + may_match[i] = FastLocalBloomImpl::HashMayMatchPrepared( + hashes[i], num_probes_, data_ + byte_offsets[i]); + } + } + + bool HashMayMatch(const uint64_t h) override { + return FastLocalBloomImpl::HashMayMatch(Lower32of64(h), Upper32of64(h), + len_bytes_, num_probes_, data_); + } + + private: + const char* data_; + const int num_probes_; + const uint32_t len_bytes_; +}; + +// ##################### Ribbon filter implementation ################### // + +// Implements concept RehasherTypesAndSettings in ribbon_impl.h +struct Standard128RibbonRehasherTypesAndSettings { + // These are schema-critical. Any change almost certainly changes + // underlying data. + static constexpr bool kIsFilter = true; + static constexpr bool kHomogeneous = false; + static constexpr bool kFirstCoeffAlwaysOne = true; + static constexpr bool kUseSmash = false; + using CoeffRow = ROCKSDB_NAMESPACE::Unsigned128; + using Hash = uint64_t; + using Seed = uint32_t; + // Changing these doesn't necessarily change underlying data, + // but might affect supported scalability of those dimensions. + using Index = uint32_t; + using ResultRow = uint32_t; + // Save a conditional in Ribbon queries + static constexpr bool kAllowZeroStarts = false; +}; + +using Standard128RibbonTypesAndSettings = + ribbon::StandardRehasherAdapter; + +class Standard128RibbonBitsBuilder : public XXPH3FilterBitsBuilder { + public: + explicit Standard128RibbonBitsBuilder( + double desired_one_in_fp_rate, int bloom_millibits_per_key, + std::atomic* aggregate_rounding_balance, + std::shared_ptr cache_res_mgr, + bool detect_filter_construct_corruption, Logger* info_log) + : XXPH3FilterBitsBuilder(aggregate_rounding_balance, cache_res_mgr, + detect_filter_construct_corruption), + desired_one_in_fp_rate_(desired_one_in_fp_rate), + info_log_(info_log), + bloom_fallback_(bloom_millibits_per_key, aggregate_rounding_balance, + cache_res_mgr, detect_filter_construct_corruption) { + assert(desired_one_in_fp_rate >= 1.0); + } + + // No Copy allowed + Standard128RibbonBitsBuilder(const Standard128RibbonBitsBuilder&) = delete; + void operator=(const Standard128RibbonBitsBuilder&) = delete; + + ~Standard128RibbonBitsBuilder() override {} + + using FilterBitsBuilder::Finish; + + virtual Slice Finish(std::unique_ptr* buf) override { + return Finish(buf, nullptr); + } + + virtual Slice Finish(std::unique_ptr* buf, + Status* status) override { + if (hash_entries_info_.entries.size() > kMaxRibbonEntries) { + ROCKS_LOG_WARN( + info_log_, "Too many keys for Ribbon filter: %llu", + static_cast(hash_entries_info_.entries.size())); + SwapEntriesWith(&bloom_fallback_); + assert(hash_entries_info_.entries.empty()); + return bloom_fallback_.Finish(buf, status); + } + if (hash_entries_info_.entries.size() == 0) { + // Save a conditional in Ribbon queries by using alternate reader + // for zero entries added. + if (status) { + *status = Status::OK(); + } + return FinishAlwaysFalse(buf); + } + uint32_t num_entries = + static_cast(hash_entries_info_.entries.size()); + uint32_t num_slots; + size_t len_with_metadata; + + CalculateSpaceAndSlots(num_entries, &len_with_metadata, &num_slots); + + // Bloom fall-back indicator + if (num_slots == 0) { + SwapEntriesWith(&bloom_fallback_); + assert(hash_entries_info_.entries.empty()); + return bloom_fallback_.Finish(buf, status); + } + + uint32_t entropy = 0; + if (!hash_entries_info_.entries.empty()) { + entropy = Lower32of64(hash_entries_info_.entries.front()); + } + + BandingType banding; + std::size_t bytes_banding = ribbon::StandardBanding< + Standard128RibbonTypesAndSettings>::EstimateMemoryUsage(num_slots); + Status status_banding_cache_res = Status::OK(); + + // Cache charging for banding + std::unique_ptr + banding_res_handle; + if (cache_res_mgr_) { + status_banding_cache_res = cache_res_mgr_->MakeCacheReservation( + bytes_banding, &banding_res_handle); + } + + if (status_banding_cache_res.IsMemoryLimit()) { + ROCKS_LOG_WARN(info_log_, + "Cache charging for Ribbon filter banding failed due " + "to cache full"); + SwapEntriesWith(&bloom_fallback_); + assert(hash_entries_info_.entries.empty()); + // Release cache for banding since the banding won't be allocated + banding_res_handle.reset(); + return bloom_fallback_.Finish(buf, status); + } + + TEST_SYNC_POINT_CALLBACK( + "XXPH3FilterBitsBuilder::Finish::" + "TamperHashEntries", + &hash_entries_info_.entries); + + bool success = banding.ResetAndFindSeedToSolve( + num_slots, hash_entries_info_.entries.begin(), + hash_entries_info_.entries.end(), + /*starting seed*/ entropy & 255, /*seed mask*/ 255); + if (!success) { + ROCKS_LOG_WARN( + info_log_, "Too many re-seeds (256) for Ribbon filter, %llu / %llu", + static_cast(hash_entries_info_.entries.size()), + static_cast(num_slots)); + SwapEntriesWith(&bloom_fallback_); + assert(hash_entries_info_.entries.empty()); + return bloom_fallback_.Finish(buf, status); + } + + Status verify_hash_entries_checksum_status = + MaybeVerifyHashEntriesChecksum(); + if (!verify_hash_entries_checksum_status.ok()) { + ROCKS_LOG_WARN(info_log_, "Verify hash entries checksum error: %s", + verify_hash_entries_checksum_status.getState()); + if (status) { + *status = verify_hash_entries_checksum_status; + } + return FinishAlwaysTrue(buf); + } + + bool keep_entries_for_postverify = detect_filter_construct_corruption_; + if (!keep_entries_for_postverify) { + ResetEntries(); + } + + uint32_t seed = banding.GetOrdinalSeed(); + assert(seed < 256); + + std::unique_ptr mutable_buf; + std::unique_ptr + final_filter_cache_res_handle; + len_with_metadata = + AllocateMaybeRounding(len_with_metadata, num_entries, &mutable_buf); + // Cache charging for mutable_buf + if (cache_res_mgr_) { + Status s = cache_res_mgr_->MakeCacheReservation( + len_with_metadata * sizeof(char), &final_filter_cache_res_handle); + s.PermitUncheckedError(); + } + + SolnType soln(mutable_buf.get(), len_with_metadata); + soln.BackSubstFrom(banding); + uint32_t num_blocks = soln.GetNumBlocks(); + // This should be guaranteed: + // num_entries < 2^30 + // => (overhead_factor < 2.0) + // num_entries * overhead_factor == num_slots < 2^31 + // => (num_blocks = num_slots / 128) + // num_blocks < 2^24 + assert(num_blocks < 0x1000000U); + + // See BloomFilterPolicy::GetBloomBitsReader re: metadata + // -2 = Marker for Standard128 Ribbon + mutable_buf[len_with_metadata - 5] = static_cast(-2); + // Hash seed + mutable_buf[len_with_metadata - 4] = static_cast(seed); + // Number of blocks, in 24 bits + // (Along with bytes, we can derive other settings) + mutable_buf[len_with_metadata - 3] = static_cast(num_blocks & 255); + mutable_buf[len_with_metadata - 2] = + static_cast((num_blocks >> 8) & 255); + mutable_buf[len_with_metadata - 1] = + static_cast((num_blocks >> 16) & 255); + + auto TEST_arg_pair __attribute__((__unused__)) = + std::make_pair(&mutable_buf, len_with_metadata); + TEST_SYNC_POINT_CALLBACK("XXPH3FilterBitsBuilder::Finish::TamperFilter", + &TEST_arg_pair); + + Slice rv(mutable_buf.get(), len_with_metadata); + *buf = std::move(mutable_buf); + final_filter_cache_res_handles_.push_back( + std::move(final_filter_cache_res_handle)); + if (status) { + *status = Status::OK(); + } + return rv; + } + + // Setting num_slots to 0 means "fall back on Bloom filter." + // And note this implementation does not support num_entries or num_slots + // beyond uint32_t; see kMaxRibbonEntries. + void CalculateSpaceAndSlots(size_t num_entries, + size_t* target_len_with_metadata, + uint32_t* num_slots) { + if (num_entries > kMaxRibbonEntries) { + // More entries than supported by this Ribbon + *num_slots = 0; // use Bloom + *target_len_with_metadata = bloom_fallback_.CalculateSpace(num_entries); + return; + } + uint32_t entropy = 0; + if (!hash_entries_info_.entries.empty()) { + entropy = Upper32of64(hash_entries_info_.entries.front()); + } + + *num_slots = NumEntriesToNumSlots(static_cast(num_entries)); + *target_len_with_metadata = + SolnType::GetBytesForOneInFpRate(*num_slots, desired_one_in_fp_rate_, + /*rounding*/ entropy) + + kMetadataLen; + + // Consider possible Bloom fallback for small filters + if (*num_slots < 1024) { + size_t bloom = bloom_fallback_.CalculateSpace(num_entries); + if (bloom < *target_len_with_metadata) { + *num_slots = 0; // use Bloom + *target_len_with_metadata = bloom; + return; + } + } + } + + size_t CalculateSpace(size_t num_entries) override { + if (num_entries == 0) { + // See FinishAlwaysFalse + return 0; + } + size_t target_len_with_metadata; + uint32_t num_slots; + CalculateSpaceAndSlots(num_entries, &target_len_with_metadata, &num_slots); + (void)num_slots; + return target_len_with_metadata; + } + + // This is a somewhat ugly but reasonably fast and reasonably accurate + // reversal of CalculateSpace. + size_t ApproximateNumEntries(size_t bytes) override { + size_t len_no_metadata = + RoundDownUsableSpace(std::max(bytes, size_t{kMetadataLen})) - + kMetadataLen; + + if (!(desired_one_in_fp_rate_ > 1.0)) { + // Effectively asking for 100% FP rate, or NaN etc. + // Note that NaN is neither < 1.0 nor > 1.0 + return kMaxRibbonEntries; + } + + // Find a slight under-estimate for actual average bits per slot + double min_real_bits_per_slot; + if (desired_one_in_fp_rate_ >= 1.0 + std::numeric_limits::max()) { + // Max of 32 solution columns (result bits) + min_real_bits_per_slot = 32.0; + } else { + // Account for mix of b and b+1 solution columns being slightly + // suboptimal vs. ideal log2(1/fp_rate) bits. + uint32_t rounded = static_cast(desired_one_in_fp_rate_); + int upper_bits_per_key = 1 + FloorLog2(rounded); + double fp_rate_for_upper = std::pow(2.0, -upper_bits_per_key); + double portion_lower = + (1.0 / desired_one_in_fp_rate_ - fp_rate_for_upper) / + fp_rate_for_upper; + min_real_bits_per_slot = upper_bits_per_key - portion_lower; + assert(min_real_bits_per_slot > 0.0); + assert(min_real_bits_per_slot <= 32.0); + } + + // An overestimate, but this should only be O(1) slots away from truth. + double max_slots = len_no_metadata * 8.0 / min_real_bits_per_slot; + + // Let's not bother accounting for overflow to Bloom filter + // (Includes NaN case) + if (!(max_slots < ConfigHelper::GetNumSlots(kMaxRibbonEntries))) { + return kMaxRibbonEntries; + } + + // Set up for short iteration + uint32_t slots = static_cast(max_slots); + slots = SolnType::RoundUpNumSlots(slots); + + // Assert that we have a valid upper bound on slots + assert(SolnType::GetBytesForOneInFpRate( + SolnType::RoundUpNumSlots(slots + 1), desired_one_in_fp_rate_, + /*rounding*/ 0) > len_no_metadata); + + // Iterate up to a few times to rather precisely account for small effects + for (int i = 0; slots > 0; ++i) { + size_t reqd_bytes = + SolnType::GetBytesForOneInFpRate(slots, desired_one_in_fp_rate_, + /*rounding*/ 0); + if (reqd_bytes <= len_no_metadata) { + break; // done + } + if (i >= 2) { + // should have been enough iterations + assert(false); + break; + } + slots = SolnType::RoundDownNumSlots(slots - 1); + } + + uint32_t num_entries = ConfigHelper::GetNumToAdd(slots); + + // Consider possible Bloom fallback for small filters + if (slots < 1024) { + size_t bloom = bloom_fallback_.ApproximateNumEntries(bytes); + if (bloom > num_entries) { + return bloom; + } else { + return num_entries; + } + } else { + return std::min(num_entries, kMaxRibbonEntries); + } + } + + double EstimatedFpRate(size_t num_entries, + size_t len_with_metadata) override { + if (num_entries > kMaxRibbonEntries) { + // More entries than supported by this Ribbon + return bloom_fallback_.EstimatedFpRate(num_entries, len_with_metadata); + } + uint32_t num_slots = + NumEntriesToNumSlots(static_cast(num_entries)); + SolnType fake_soln(nullptr, len_with_metadata); + fake_soln.ConfigureForNumSlots(num_slots); + return fake_soln.ExpectedFpRate(); + } + + Status MaybePostVerify(const Slice& filter_content) override { + bool fall_back = (bloom_fallback_.EstimateEntriesAdded() > 0); + return fall_back ? bloom_fallback_.MaybePostVerify(filter_content) + : XXPH3FilterBitsBuilder::MaybePostVerify(filter_content); + } + + protected: + size_t RoundDownUsableSpace(size_t available_size) override { + size_t rv = available_size - kMetadataLen; + + // round down to multiple of 16 (segment size) + rv &= ~size_t{15}; + + return rv + kMetadataLen; + } + + private: + using TS = Standard128RibbonTypesAndSettings; + using SolnType = ribbon::SerializableInterleavedSolution; + using BandingType = ribbon::StandardBanding; + using ConfigHelper = ribbon::BandingConfigHelper1TS; + + static uint32_t NumEntriesToNumSlots(uint32_t num_entries) { + uint32_t num_slots1 = ConfigHelper::GetNumSlots(num_entries); + return SolnType::RoundUpNumSlots(num_slots1); + } + + // Approximate num_entries to ensure number of bytes fits in 32 bits, which + // is not an inherent limitation but does ensure somewhat graceful Bloom + // fallback for crazy high number of entries, since the Bloom implementation + // does not support number of bytes bigger than fits in 32 bits. This is + // within an order of magnitude of implementation limit on num_slots + // fitting in 32 bits, and even closer for num_blocks fitting in 24 bits + // (for filter metadata). + static constexpr uint32_t kMaxRibbonEntries = 950000000; // ~ 1 billion + + // A desired value for 1/fp_rate. For example, 100 -> 1% fp rate. + double desired_one_in_fp_rate_; + + // For warnings, or can be nullptr + Logger* info_log_; + + // For falling back on Bloom filter in some exceptional cases and + // very small filter cases + FastLocalBloomBitsBuilder bloom_fallback_; +}; + +// for the linker, at least with DEBUG_LEVEL=2 +constexpr uint32_t Standard128RibbonBitsBuilder::kMaxRibbonEntries; + +class Standard128RibbonBitsReader : public BuiltinFilterBitsReader { + public: + Standard128RibbonBitsReader(const char* data, size_t len_bytes, + uint32_t num_blocks, uint32_t seed) + : soln_(const_cast(data), len_bytes) { + soln_.ConfigureForNumBlocks(num_blocks); + hasher_.SetOrdinalSeed(seed); + } + + // No Copy allowed + Standard128RibbonBitsReader(const Standard128RibbonBitsReader&) = delete; + void operator=(const Standard128RibbonBitsReader&) = delete; + + ~Standard128RibbonBitsReader() override {} + + bool MayMatch(const Slice& key) override { + uint64_t h = GetSliceHash64(key); + return soln_.FilterQuery(h, hasher_); + } + + virtual void MayMatch(int num_keys, Slice** keys, bool* may_match) override { + struct SavedData { + uint64_t seeded_hash; + uint32_t segment_num; + uint32_t num_columns; + uint32_t start_bits; + }; + std::array saved; + for (int i = 0; i < num_keys; ++i) { + ribbon::InterleavedPrepareQuery( + GetSliceHash64(*keys[i]), hasher_, soln_, &saved[i].seeded_hash, + &saved[i].segment_num, &saved[i].num_columns, &saved[i].start_bits); + } + for (int i = 0; i < num_keys; ++i) { + may_match[i] = ribbon::InterleavedFilterQuery( + saved[i].seeded_hash, saved[i].segment_num, saved[i].num_columns, + saved[i].start_bits, hasher_, soln_); + } + } + + bool HashMayMatch(const uint64_t h) override { + return soln_.FilterQuery(h, hasher_); + } + + private: + using TS = Standard128RibbonTypesAndSettings; + ribbon::SerializableInterleavedSolution soln_; + ribbon::StandardHasher hasher_; +}; + +// ##################### Legacy Bloom implementation ################### // + +using LegacyBloomImpl = LegacyLocalityBloomImpl; + +class LegacyBloomBitsBuilder : public BuiltinFilterBitsBuilder { + public: + explicit LegacyBloomBitsBuilder(const int bits_per_key, Logger* info_log); + + // No Copy allowed + LegacyBloomBitsBuilder(const LegacyBloomBitsBuilder&) = delete; + void operator=(const LegacyBloomBitsBuilder&) = delete; + + ~LegacyBloomBitsBuilder() override; + + void AddKey(const Slice& key) override; + + virtual size_t EstimateEntriesAdded() override { + return hash_entries_.size(); + } + + using FilterBitsBuilder::Finish; + + Slice Finish(std::unique_ptr* buf) override; + + size_t CalculateSpace(size_t num_entries) override { + uint32_t dont_care1; + uint32_t dont_care2; + return CalculateSpace(num_entries, &dont_care1, &dont_care2); + } + + double EstimatedFpRate(size_t keys, size_t bytes) override { + return LegacyBloomImpl::EstimatedFpRate(keys, bytes - kMetadataLen, + num_probes_); + } + + size_t ApproximateNumEntries(size_t bytes) override; + + private: + int bits_per_key_; + int num_probes_; + std::vector hash_entries_; + Logger* info_log_; + + // Get totalbits that optimized for cpu cache line + uint32_t GetTotalBitsForLocality(uint32_t total_bits); + + // Reserve space for new filter + char* ReserveSpace(size_t num_entries, uint32_t* total_bits, + uint32_t* num_lines); + + // Implementation-specific variant of public CalculateSpace + uint32_t CalculateSpace(size_t num_entries, uint32_t* total_bits, + uint32_t* num_lines); + + // Assuming single threaded access to this function. + void AddHash(uint32_t h, char* data, uint32_t num_lines, uint32_t total_bits); +}; + +LegacyBloomBitsBuilder::LegacyBloomBitsBuilder(const int bits_per_key, + Logger* info_log) + : bits_per_key_(bits_per_key), + num_probes_(LegacyNoLocalityBloomImpl::ChooseNumProbes(bits_per_key_)), + info_log_(info_log) { + assert(bits_per_key_); +} + +LegacyBloomBitsBuilder::~LegacyBloomBitsBuilder() {} + +void LegacyBloomBitsBuilder::AddKey(const Slice& key) { + uint32_t hash = BloomHash(key); + if (hash_entries_.size() == 0 || hash != hash_entries_.back()) { + hash_entries_.push_back(hash); + } +} + +Slice LegacyBloomBitsBuilder::Finish(std::unique_ptr* buf) { + uint32_t total_bits, num_lines; + size_t num_entries = hash_entries_.size(); + char* data = + ReserveSpace(static_cast(num_entries), &total_bits, &num_lines); + assert(data); + + if (total_bits != 0 && num_lines != 0) { + for (auto h : hash_entries_) { + AddHash(h, data, num_lines, total_bits); + } + + // Check for excessive entries for 32-bit hash function + if (num_entries >= /* minimum of 3 million */ 3000000U) { + // More specifically, we can detect that the 32-bit hash function + // is causing significant increase in FP rate by comparing current + // estimated FP rate to what we would get with a normal number of + // keys at same memory ratio. + double est_fp_rate = LegacyBloomImpl::EstimatedFpRate( + num_entries, total_bits / 8, num_probes_); + double vs_fp_rate = LegacyBloomImpl::EstimatedFpRate( + 1U << 16, (1U << 16) * bits_per_key_ / 8, num_probes_); + + if (est_fp_rate >= 1.50 * vs_fp_rate) { + // For more details, see + // https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter + ROCKS_LOG_WARN( + info_log_, + "Using legacy SST/BBT Bloom filter with excessive key count " + "(%.1fM @ %dbpk), causing estimated %.1fx higher filter FP rate. " + "Consider using new Bloom with format_version>=5, smaller SST " + "file size, or partitioned filters.", + num_entries / 1000000.0, bits_per_key_, est_fp_rate / vs_fp_rate); + } + } + } + // See BloomFilterPolicy::GetFilterBitsReader for metadata + data[total_bits / 8] = static_cast(num_probes_); + EncodeFixed32(data + total_bits / 8 + 1, static_cast(num_lines)); + + const char* const_data = data; + buf->reset(const_data); + hash_entries_.clear(); + + return Slice(data, total_bits / 8 + kMetadataLen); +} + +size_t LegacyBloomBitsBuilder::ApproximateNumEntries(size_t bytes) { + assert(bits_per_key_); + assert(bytes > 0); + + uint64_t total_bits_tmp = bytes * 8; + // total bits, including temporary computations, cannot exceed 2^32 + // for compatibility + total_bits_tmp = std::min(total_bits_tmp, uint64_t{0xffff0000}); + + uint32_t high = static_cast(total_bits_tmp) / + static_cast(bits_per_key_) + + 1; + uint32_t low = 1; + uint32_t n = high; + for (; n >= low; n--) { + if (CalculateSpace(n) <= bytes) { + break; + } + } + return n; +} + +uint32_t LegacyBloomBitsBuilder::GetTotalBitsForLocality(uint32_t total_bits) { + uint32_t num_lines = + (total_bits + CACHE_LINE_SIZE * 8 - 1) / (CACHE_LINE_SIZE * 8); + + // Make num_lines an odd number to make sure more bits are involved + // when determining which block. + if (num_lines % 2 == 0) { + num_lines++; + } + return num_lines * (CACHE_LINE_SIZE * 8); +} + +uint32_t LegacyBloomBitsBuilder::CalculateSpace(size_t num_entries, + uint32_t* total_bits, + uint32_t* num_lines) { + assert(bits_per_key_); + if (num_entries != 0) { + size_t total_bits_tmp = num_entries * bits_per_key_; + // total bits, including temporary computations, cannot exceed 2^32 + // for compatibility + total_bits_tmp = std::min(total_bits_tmp, size_t{0xffff0000}); + + *total_bits = + GetTotalBitsForLocality(static_cast(total_bits_tmp)); + *num_lines = *total_bits / (CACHE_LINE_SIZE * 8); + assert(*total_bits > 0 && *total_bits % 8 == 0); + } else { + // filter is empty, just leave space for metadata + *total_bits = 0; + *num_lines = 0; + } + + // Reserve space for Filter + uint32_t sz = *total_bits / 8; + sz += kMetadataLen; // 4 bytes for num_lines, 1 byte for num_probes + return sz; +} + +char* LegacyBloomBitsBuilder::ReserveSpace(size_t num_entries, + uint32_t* total_bits, + uint32_t* num_lines) { + uint32_t sz = CalculateSpace(num_entries, total_bits, num_lines); + char* data = new char[sz]; + memset(data, 0, sz); + return data; +} + +inline void LegacyBloomBitsBuilder::AddHash(uint32_t h, char* data, + uint32_t num_lines, + uint32_t total_bits) { +#ifdef NDEBUG + static_cast(total_bits); +#endif + assert(num_lines > 0 && total_bits > 0); + + LegacyBloomImpl::AddHash(h, num_lines, num_probes_, data, + ConstexprFloorLog2(CACHE_LINE_SIZE)); +} + +class LegacyBloomBitsReader : public BuiltinFilterBitsReader { + public: + LegacyBloomBitsReader(const char* data, int num_probes, uint32_t num_lines, + uint32_t log2_cache_line_size) + : data_(data), + num_probes_(num_probes), + num_lines_(num_lines), + log2_cache_line_size_(log2_cache_line_size) {} + + // No Copy allowed + LegacyBloomBitsReader(const LegacyBloomBitsReader&) = delete; + void operator=(const LegacyBloomBitsReader&) = delete; + + ~LegacyBloomBitsReader() override {} + + // "contents" contains the data built by a preceding call to + // FilterBitsBuilder::Finish. MayMatch must return true if the key was + // passed to FilterBitsBuilder::AddKey. This method may return true or false + // if the key was not on the list, but it should aim to return false with a + // high probability. + bool MayMatch(const Slice& key) override { + uint32_t hash = BloomHash(key); + uint32_t byte_offset; + LegacyBloomImpl::PrepareHashMayMatch( + hash, num_lines_, data_, /*out*/ &byte_offset, log2_cache_line_size_); + return LegacyBloomImpl::HashMayMatchPrepared( + hash, num_probes_, data_ + byte_offset, log2_cache_line_size_); + } + + virtual void MayMatch(int num_keys, Slice** keys, bool* may_match) override { + std::array hashes; + std::array byte_offsets; + for (int i = 0; i < num_keys; ++i) { + hashes[i] = BloomHash(*keys[i]); + LegacyBloomImpl::PrepareHashMayMatch(hashes[i], num_lines_, data_, + /*out*/ &byte_offsets[i], + log2_cache_line_size_); + } + for (int i = 0; i < num_keys; ++i) { + may_match[i] = LegacyBloomImpl::HashMayMatchPrepared( + hashes[i], num_probes_, data_ + byte_offsets[i], + log2_cache_line_size_); + } + } + + bool HashMayMatch(const uint64_t /* h */) override { return false; } + + private: + const char* data_; + const int num_probes_; + const uint32_t num_lines_; + const uint32_t log2_cache_line_size_; +}; + +class AlwaysTrueFilter : public BuiltinFilterBitsReader { + public: + bool MayMatch(const Slice&) override { return true; } + using FilterBitsReader::MayMatch; // inherit overload + bool HashMayMatch(const uint64_t) override { return true; } + using BuiltinFilterBitsReader::HashMayMatch; // inherit overload +}; + +class AlwaysFalseFilter : public BuiltinFilterBitsReader { + public: + bool MayMatch(const Slice&) override { return false; } + using FilterBitsReader::MayMatch; // inherit overload + bool HashMayMatch(const uint64_t) override { return false; } + using BuiltinFilterBitsReader::HashMayMatch; // inherit overload +}; + +Status XXPH3FilterBitsBuilder::MaybePostVerify(const Slice& filter_content) { + Status s = Status::OK(); + + if (!detect_filter_construct_corruption_) { + return s; + } + + std::unique_ptr bits_reader( + BuiltinFilterPolicy::GetBuiltinFilterBitsReader(filter_content)); + + for (uint64_t h : hash_entries_info_.entries) { + // The current approach will not detect corruption from XXPH3Filter to + // AlwaysTrueFilter, which can lead to performance cost later due to + // AlwaysTrueFilter not filtering anything. But this cost is acceptable + // given the extra implementation complixity to detect such case. + bool may_match = bits_reader->HashMayMatch(h); + if (!may_match) { + s = Status::Corruption("Corrupted filter content"); + break; + } + } + + ResetEntries(); + return s; +} +} // namespace + +const char* BuiltinFilterPolicy::kClassName() { + return "rocksdb.internal.BuiltinFilter"; +} + +bool BuiltinFilterPolicy::IsInstanceOf(const std::string& name) const { + if (name == kClassName()) { + return true; + } else { + return FilterPolicy::IsInstanceOf(name); + } +} + +static const char* kBuiltinFilterMetadataName = "rocksdb.BuiltinBloomFilter"; + +const char* BuiltinFilterPolicy::kCompatibilityName() { + return kBuiltinFilterMetadataName; +} + +const char* BuiltinFilterPolicy::CompatibilityName() const { + return kBuiltinFilterMetadataName; +} + +BloomLikeFilterPolicy::BloomLikeFilterPolicy(double bits_per_key) + : warned_(false), aggregate_rounding_balance_(0) { + // Sanitize bits_per_key + if (bits_per_key < 0.5) { + // Round down to no filter + bits_per_key = 0; + } else if (bits_per_key < 1.0) { + // Minimum 1 bit per key (equiv) when creating filter + bits_per_key = 1.0; + } else if (!(bits_per_key < 100.0)) { // including NaN + bits_per_key = 100.0; + } + + // Includes a nudge toward rounding up, to ensure on all platforms + // that doubles specified with three decimal digits after the decimal + // point are interpreted accurately. + millibits_per_key_ = static_cast(bits_per_key * 1000.0 + 0.500001); + + // For now configure Ribbon filter to match Bloom FP rate and save + // memory. (Ribbon bits per key will be ~30% less than Bloom bits per key + // for same FP rate.) + desired_one_in_fp_rate_ = + 1.0 / BloomMath::CacheLocalFpRate( + bits_per_key, + FastLocalBloomImpl::ChooseNumProbes(millibits_per_key_), + /*cache_line_bits*/ 512); + + // For better or worse, this is a rounding up of a nudged rounding up, + // e.g. 7.4999999999999 will round up to 8, but that provides more + // predictability against small arithmetic errors in floating point. + whole_bits_per_key_ = (millibits_per_key_ + 500) / 1000; +} + +BloomLikeFilterPolicy::~BloomLikeFilterPolicy() {} +const char* BloomLikeFilterPolicy::kClassName() { + return "rocksdb.internal.BloomLikeFilter"; +} + +bool BloomLikeFilterPolicy::IsInstanceOf(const std::string& name) const { + if (name == kClassName()) { + return true; + } else { + return BuiltinFilterPolicy::IsInstanceOf(name); + } +} + +const char* ReadOnlyBuiltinFilterPolicy::kClassName() { + return kBuiltinFilterMetadataName; +} + +std::string BloomLikeFilterPolicy::GetId() const { + return Name() + GetBitsPerKeySuffix(); +} + +BloomFilterPolicy::BloomFilterPolicy(double bits_per_key) + : BloomLikeFilterPolicy(bits_per_key) {} + +FilterBitsBuilder* BloomFilterPolicy::GetBuilderWithContext( + const FilterBuildingContext& context) const { + if (GetMillibitsPerKey() == 0) { + // "No filter" special case + return nullptr; + } else if (context.table_options.format_version < 5) { + return GetLegacyBloomBuilderWithContext(context); + } else { + return GetFastLocalBloomBuilderWithContext(context); + } +} + +const char* BloomFilterPolicy::kClassName() { return "bloomfilter"; } +const char* BloomFilterPolicy::kNickName() { return "rocksdb.BloomFilter"; } + +std::string BloomFilterPolicy::GetId() const { + // Including ":false" for better forward-compatibility with 6.29 and earlier + // which required a boolean `use_block_based_builder` parameter + return BloomLikeFilterPolicy::GetId() + ":false"; +} + +FilterBitsBuilder* BloomLikeFilterPolicy::GetFastLocalBloomBuilderWithContext( + const FilterBuildingContext& context) const { + bool offm = context.table_options.optimize_filters_for_memory; + const auto options_overrides_iter = + context.table_options.cache_usage_options.options_overrides.find( + CacheEntryRole::kFilterConstruction); + const auto filter_construction_charged = + options_overrides_iter != + context.table_options.cache_usage_options.options_overrides.end() + ? options_overrides_iter->second.charged + : context.table_options.cache_usage_options.options.charged; + + std::shared_ptr cache_res_mgr; + if (context.table_options.block_cache && + filter_construction_charged == + CacheEntryRoleOptions::Decision::kEnabled) { + cache_res_mgr = std::make_shared< + CacheReservationManagerImpl>( + context.table_options.block_cache); + } + return new FastLocalBloomBitsBuilder( + millibits_per_key_, offm ? &aggregate_rounding_balance_ : nullptr, + cache_res_mgr, context.table_options.detect_filter_construct_corruption); +} + +FilterBitsBuilder* BloomLikeFilterPolicy::GetLegacyBloomBuilderWithContext( + const FilterBuildingContext& context) const { + if (whole_bits_per_key_ >= 14 && context.info_log && + !warned_.load(std::memory_order_relaxed)) { + warned_ = true; + const char* adjective; + if (whole_bits_per_key_ >= 20) { + adjective = "Dramatic"; + } else { + adjective = "Significant"; + } + // For more details, see + // https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter + ROCKS_LOG_WARN(context.info_log, + "Using legacy Bloom filter with high (%d) bits/key. " + "%s filter space and/or accuracy improvement is available " + "with format_version>=5.", + whole_bits_per_key_, adjective); + } + return new LegacyBloomBitsBuilder(whole_bits_per_key_, context.info_log); +} + +FilterBitsBuilder* +BloomLikeFilterPolicy::GetStandard128RibbonBuilderWithContext( + const FilterBuildingContext& context) const { + // FIXME: code duplication with GetFastLocalBloomBuilderWithContext + bool offm = context.table_options.optimize_filters_for_memory; + const auto options_overrides_iter = + context.table_options.cache_usage_options.options_overrides.find( + CacheEntryRole::kFilterConstruction); + const auto filter_construction_charged = + options_overrides_iter != + context.table_options.cache_usage_options.options_overrides.end() + ? options_overrides_iter->second.charged + : context.table_options.cache_usage_options.options.charged; + + std::shared_ptr cache_res_mgr; + if (context.table_options.block_cache && + filter_construction_charged == + CacheEntryRoleOptions::Decision::kEnabled) { + cache_res_mgr = std::make_shared< + CacheReservationManagerImpl>( + context.table_options.block_cache); + } + return new Standard128RibbonBitsBuilder( + desired_one_in_fp_rate_, millibits_per_key_, + offm ? &aggregate_rounding_balance_ : nullptr, cache_res_mgr, + context.table_options.detect_filter_construct_corruption, + context.info_log); +} + +std::string BloomLikeFilterPolicy::GetBitsPerKeySuffix() const { + std::string rv = ":" + std::to_string(millibits_per_key_ / 1000); + int frac = millibits_per_key_ % 1000; + if (frac > 0) { + rv.push_back('.'); + rv.push_back(static_cast('0' + (frac / 100))); + frac %= 100; + if (frac > 0) { + rv.push_back(static_cast('0' + (frac / 10))); + frac %= 10; + if (frac > 0) { + rv.push_back(static_cast('0' + frac)); + } + } + } + return rv; +} + +FilterBitsBuilder* BuiltinFilterPolicy::GetBuilderFromContext( + const FilterBuildingContext& context) { + if (context.table_options.filter_policy) { + return context.table_options.filter_policy->GetBuilderWithContext(context); + } else { + return nullptr; + } +} + +// For testing only, but always constructable with internal names +namespace test { + +const char* LegacyBloomFilterPolicy::kClassName() { + return "rocksdb.internal.LegacyBloomFilter"; +} + +FilterBitsBuilder* LegacyBloomFilterPolicy::GetBuilderWithContext( + const FilterBuildingContext& context) const { + if (GetMillibitsPerKey() == 0) { + // "No filter" special case + return nullptr; + } + return GetLegacyBloomBuilderWithContext(context); +} + +const char* FastLocalBloomFilterPolicy::kClassName() { + return "rocksdb.internal.FastLocalBloomFilter"; +} + +FilterBitsBuilder* FastLocalBloomFilterPolicy::GetBuilderWithContext( + const FilterBuildingContext& context) const { + if (GetMillibitsPerKey() == 0) { + // "No filter" special case + return nullptr; + } + return GetFastLocalBloomBuilderWithContext(context); +} + +const char* Standard128RibbonFilterPolicy::kClassName() { + return "rocksdb.internal.Standard128RibbonFilter"; +} + +FilterBitsBuilder* Standard128RibbonFilterPolicy::GetBuilderWithContext( + const FilterBuildingContext& context) const { + if (GetMillibitsPerKey() == 0) { + // "No filter" special case + return nullptr; + } + return GetStandard128RibbonBuilderWithContext(context); +} + +} // namespace test + +BuiltinFilterBitsReader* BuiltinFilterPolicy::GetBuiltinFilterBitsReader( + const Slice& contents) { + uint32_t len_with_meta = static_cast(contents.size()); + if (len_with_meta <= kMetadataLen) { + // filter is empty or broken. Treat like zero keys added. + return new AlwaysFalseFilter(); + } + + // Legacy Bloom filter data: + // 0 +-----------------------------------+ + // | Raw Bloom filter data | + // | ... | + // len +-----------------------------------+ + // | byte for num_probes or | + // | marker for new implementations | + // len+1 +-----------------------------------+ + // | four bytes for number of cache | + // | lines | + // len_with_meta +-----------------------------------+ + + int8_t raw_num_probes = + static_cast(contents.data()[len_with_meta - kMetadataLen]); + // NB: *num_probes > 30 and < 128 probably have not been used, because of + // BloomFilterPolicy::initialize, unless directly calling + // LegacyBloomBitsBuilder as an API, but we are leaving those cases in + // limbo with LegacyBloomBitsReader for now. + + if (raw_num_probes < 1) { + // Note: < 0 (or unsigned > 127) indicate special new implementations + // (or reserved for future use) + switch (raw_num_probes) { + case 0: + // Treat as zero probes (always FP) + return new AlwaysTrueFilter(); + case -1: + // Marker for newer Bloom implementations + return GetBloomBitsReader(contents); + case -2: + // Marker for Ribbon implementations + return GetRibbonBitsReader(contents); + default: + // Reserved (treat as zero probes, always FP, for now) + return new AlwaysTrueFilter(); + } + } + // else attempt decode for LegacyBloomBitsReader + + int num_probes = raw_num_probes; + assert(num_probes >= 1); + assert(num_probes <= 127); + + uint32_t len = len_with_meta - kMetadataLen; + assert(len > 0); + + uint32_t num_lines = DecodeFixed32(contents.data() + len_with_meta - 4); + uint32_t log2_cache_line_size; + + if (num_lines * CACHE_LINE_SIZE == len) { + // Common case + log2_cache_line_size = ConstexprFloorLog2(CACHE_LINE_SIZE); + } else if (num_lines == 0 || len % num_lines != 0) { + // Invalid (no solution to num_lines * x == len) + // Treat as zero probes (always FP) for now. + return new AlwaysTrueFilter(); + } else { + // Determine the non-native cache line size (from another system) + log2_cache_line_size = 0; + while ((num_lines << log2_cache_line_size) < len) { + ++log2_cache_line_size; + } + if ((num_lines << log2_cache_line_size) != len) { + // Invalid (block size not a power of two) + // Treat as zero probes (always FP) for now. + return new AlwaysTrueFilter(); + } + } + // if not early return + return new LegacyBloomBitsReader(contents.data(), num_probes, num_lines, + log2_cache_line_size); +} + +// Read metadata to determine what kind of FilterBitsReader is needed +// and return a new one. +FilterBitsReader* BuiltinFilterPolicy::GetFilterBitsReader( + const Slice& contents) const { + return BuiltinFilterPolicy::GetBuiltinFilterBitsReader(contents); +} + +BuiltinFilterBitsReader* BuiltinFilterPolicy::GetRibbonBitsReader( + const Slice& contents) { + uint32_t len_with_meta = static_cast(contents.size()); + uint32_t len = len_with_meta - kMetadataLen; + + assert(len > 0); // precondition + + uint32_t seed = static_cast(contents.data()[len + 1]); + uint32_t num_blocks = static_cast(contents.data()[len + 2]); + num_blocks |= static_cast(contents.data()[len + 3]) << 8; + num_blocks |= static_cast(contents.data()[len + 4]) << 16; + if (num_blocks < 2) { + // Not supported + // num_blocks == 1 is not used because num_starts == 1 is problematic + // for the hashing scheme. num_blocks == 0 is unused because there's + // already a concise encoding of an "always false" filter. + // Return something safe: + return new AlwaysTrueFilter(); + } + return new Standard128RibbonBitsReader(contents.data(), len, num_blocks, + seed); +} + +// For newer Bloom filter implementations +BuiltinFilterBitsReader* BuiltinFilterPolicy::GetBloomBitsReader( + const Slice& contents) { + uint32_t len_with_meta = static_cast(contents.size()); + uint32_t len = len_with_meta - kMetadataLen; + + assert(len > 0); // precondition + + // New Bloom filter data: + // 0 +-----------------------------------+ + // | Raw Bloom filter data | + // | ... | + // len +-----------------------------------+ + // | char{-1} byte -> new Bloom filter | + // len+1 +-----------------------------------+ + // | byte for subimplementation | + // | 0: FastLocalBloom | + // | other: reserved | + // len+2 +-----------------------------------+ + // | byte for block_and_probes | + // | 0 in top 3 bits -> 6 -> 64-byte | + // | reserved: | + // | 1 in top 3 bits -> 7 -> 128-byte| + // | 2 in top 3 bits -> 8 -> 256-byte| + // | ... | + // | num_probes in bottom 5 bits, | + // | except 0 and 31 reserved | + // len+3 +-----------------------------------+ + // | two bytes reserved | + // | possibly for hash seed | + // len_with_meta +-----------------------------------+ + + // Read more metadata (see above) + char sub_impl_val = contents.data()[len_with_meta - 4]; + char block_and_probes = contents.data()[len_with_meta - 3]; + int log2_block_bytes = ((block_and_probes >> 5) & 7) + 6; + + int num_probes = (block_and_probes & 31); + if (num_probes < 1 || num_probes > 30) { + // Reserved / future safe + return new AlwaysTrueFilter(); + } + + uint16_t rest = DecodeFixed16(contents.data() + len_with_meta - 2); + if (rest != 0) { + // Reserved, possibly for hash seed + // Future safe + return new AlwaysTrueFilter(); + } + + if (sub_impl_val == 0) { // FastLocalBloom + if (log2_block_bytes == 6) { // Only block size supported for now + return new FastLocalBloomBitsReader(contents.data(), num_probes, len); + } + } + // otherwise + // Reserved / future safe + return new AlwaysTrueFilter(); +} + +const FilterPolicy* NewBloomFilterPolicy(double bits_per_key, + bool /*use_block_based_builder*/) { + // NOTE: use_block_based_builder now ignored so block-based filter is no + // longer accessible in public API. + return new BloomFilterPolicy(bits_per_key); +} + +RibbonFilterPolicy::RibbonFilterPolicy(double bloom_equivalent_bits_per_key, + int bloom_before_level) + : BloomLikeFilterPolicy(bloom_equivalent_bits_per_key), + bloom_before_level_(bloom_before_level) {} + +FilterBitsBuilder* RibbonFilterPolicy::GetBuilderWithContext( + const FilterBuildingContext& context) const { + if (GetMillibitsPerKey() == 0) { + // "No filter" special case + return nullptr; + } + // Treat unknown same as bottommost + int levelish = INT_MAX; + + switch (context.compaction_style) { + case kCompactionStyleLevel: + case kCompactionStyleUniversal: { + if (context.reason == TableFileCreationReason::kFlush) { + // Treat flush as level -1 + assert(context.level_at_creation == 0); + levelish = -1; + } else if (context.level_at_creation == -1) { + // Unknown level + assert(levelish == INT_MAX); + } else { + levelish = context.level_at_creation; + } + break; + } + case kCompactionStyleFIFO: + case kCompactionStyleNone: + // Treat as bottommost + assert(levelish == INT_MAX); + break; + } + if (levelish < bloom_before_level_) { + return GetFastLocalBloomBuilderWithContext(context); + } else { + return GetStandard128RibbonBuilderWithContext(context); + } +} + +const char* RibbonFilterPolicy::kClassName() { return "ribbonfilter"; } +const char* RibbonFilterPolicy::kNickName() { return "rocksdb.RibbonFilter"; } + +std::string RibbonFilterPolicy::GetId() const { + return BloomLikeFilterPolicy::GetId() + ":" + + std::to_string(bloom_before_level_); +} + +const FilterPolicy* NewRibbonFilterPolicy(double bloom_equivalent_bits_per_key, + int bloom_before_level) { + return new RibbonFilterPolicy(bloom_equivalent_bits_per_key, + bloom_before_level); +} + +FilterBuildingContext::FilterBuildingContext( + const BlockBasedTableOptions& _table_options) + : table_options(_table_options) {} + +FilterPolicy::~FilterPolicy() {} + +std::shared_ptr BloomLikeFilterPolicy::Create( + const std::string& name, double bits_per_key) { + if (name == test::LegacyBloomFilterPolicy::kClassName()) { + return std::make_shared(bits_per_key); + } else if (name == test::FastLocalBloomFilterPolicy::kClassName()) { + return std::make_shared(bits_per_key); + } else if (name == test::Standard128RibbonFilterPolicy::kClassName()) { + return std::make_shared(bits_per_key); + } else if (name == BloomFilterPolicy::kClassName()) { + // For testing + return std::make_shared(bits_per_key); + } else if (name == RibbonFilterPolicy::kClassName()) { + // For testing + return std::make_shared(bits_per_key, + /*bloom_before_level*/ 0); + } else { + return nullptr; + } +} + +namespace { +static ObjectLibrary::PatternEntry FilterPatternEntryWithBits( + const char* name) { + return ObjectLibrary::PatternEntry(name, false).AddNumber(":", false); +} + +template +T* NewBuiltinFilterPolicyWithBits(const std::string& uri) { + const std::vector vals = StringSplit(uri, ':'); + double bits_per_key = ParseDouble(vals[1]); + return new T(bits_per_key); +} +static int RegisterBuiltinFilterPolicies(ObjectLibrary& library, + const std::string& /*arg*/) { + library.AddFactory( + ReadOnlyBuiltinFilterPolicy::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new ReadOnlyBuiltinFilterPolicy()); + return guard->get(); + }); + + library.AddFactory( + FilterPatternEntryWithBits(BloomFilterPolicy::kClassName()) + .AnotherName(BloomFilterPolicy::kNickName()), + [](const std::string& uri, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(NewBuiltinFilterPolicyWithBits(uri)); + return guard->get(); + }); + library.AddFactory( + FilterPatternEntryWithBits(BloomFilterPolicy::kClassName()) + .AnotherName(BloomFilterPolicy::kNickName()) + .AddSuffix(":false"), + [](const std::string& uri, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(NewBuiltinFilterPolicyWithBits(uri)); + return guard->get(); + }); + library.AddFactory( + FilterPatternEntryWithBits(BloomFilterPolicy::kClassName()) + .AnotherName(BloomFilterPolicy::kNickName()) + .AddSuffix(":true"), + [](const std::string& uri, std::unique_ptr* guard, + std::string* /* errmsg */) { + const std::vector vals = StringSplit(uri, ':'); + double bits_per_key = ParseDouble(vals[1]); + // NOTE: This case previously configured the deprecated block-based + // filter, but old ways of configuring that now map to full filter. We + // defer to the corresponding API to ensure consistency in case that + // change is reverted. + guard->reset(NewBloomFilterPolicy(bits_per_key, true)); + return guard->get(); + }); + library.AddFactory( + FilterPatternEntryWithBits(RibbonFilterPolicy::kClassName()) + .AnotherName(RibbonFilterPolicy::kNickName()), + [](const std::string& uri, std::unique_ptr* guard, + std::string* /* errmsg */) { + const std::vector vals = StringSplit(uri, ':'); + double bits_per_key = ParseDouble(vals[1]); + guard->reset(NewRibbonFilterPolicy(bits_per_key)); + return guard->get(); + }); + library.AddFactory( + FilterPatternEntryWithBits(RibbonFilterPolicy::kClassName()) + .AnotherName(RibbonFilterPolicy::kNickName()) + .AddNumber(":", true), + [](const std::string& uri, std::unique_ptr* guard, + std::string* /* errmsg */) { + const std::vector vals = StringSplit(uri, ':'); + double bits_per_key = ParseDouble(vals[1]); + int bloom_before_level = ParseInt(vals[2]); + guard->reset(NewRibbonFilterPolicy(bits_per_key, bloom_before_level)); + return guard->get(); + }); + library.AddFactory( + FilterPatternEntryWithBits(test::LegacyBloomFilterPolicy::kClassName()), + [](const std::string& uri, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset( + NewBuiltinFilterPolicyWithBits(uri)); + return guard->get(); + }); + library.AddFactory( + FilterPatternEntryWithBits( + test::FastLocalBloomFilterPolicy::kClassName()), + [](const std::string& uri, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset( + NewBuiltinFilterPolicyWithBits( + uri)); + return guard->get(); + }); + library.AddFactory( + FilterPatternEntryWithBits( + test::Standard128RibbonFilterPolicy::kClassName()), + [](const std::string& uri, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset( + NewBuiltinFilterPolicyWithBits( + uri)); + return guard->get(); + }); + size_t num_types; + return static_cast(library.GetFactoryCount(&num_types)); +} +} // namespace + +Status FilterPolicy::CreateFromString( + const ConfigOptions& options, const std::string& value, + std::shared_ptr* policy) { + if (value == kNullptrString || value.empty()) { + policy->reset(); + return Status::OK(); + } else if (value == ReadOnlyBuiltinFilterPolicy::kClassName()) { + *policy = std::make_shared(); + return Status::OK(); + } + + std::string id; + std::unordered_map opt_map; + Status status = + Customizable::GetOptionsMap(options, policy->get(), value, &id, &opt_map); + if (!status.ok()) { // GetOptionsMap failed + return status; + } else if (id.empty()) { // We have no Id but have options. Not good + return Status::NotSupported("Cannot reset object ", id); + } else { + static std::once_flag loaded; + std::call_once(loaded, [&]() { + RegisterBuiltinFilterPolicies(*(ObjectLibrary::Default().get()), ""); + }); + status = options.registry->NewSharedObject(id, policy); + } + if (options.ignore_unsupported_options && status.IsNotSupported()) { + return Status::OK(); + } else if (status.ok()) { + status = Customizable::ConfigureNewObject( + options, const_cast(policy->get()), opt_map); + } + return status; +} + +const std::vector& BloomLikeFilterPolicy::GetAllFixedImpls() { + STATIC_AVOID_DESTRUCTION(std::vector, impls){ + // Match filter_bench -impl=x ordering + test::LegacyBloomFilterPolicy::kClassName(), + test::FastLocalBloomFilterPolicy::kClassName(), + test::Standard128RibbonFilterPolicy::kClassName(), + }; + return impls; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/filter_policy_internal.h b/librocksdb-sys/rocksdb/table/block_based/filter_policy_internal.h new file mode 100644 index 0000000..9bc3a24 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/filter_policy_internal.h @@ -0,0 +1,340 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include +#include +#include +#include + +#include "rocksdb/filter_policy.h" +#include "rocksdb/table.h" + +namespace ROCKSDB_NAMESPACE { + +// A class that takes a bunch of keys, then generates filter +class FilterBitsBuilder { + public: + virtual ~FilterBitsBuilder() {} + + // Add a key (or prefix) to the filter. Typically, a builder will keep + // a set of 64-bit key hashes and only build the filter in Finish + // when the final number of keys is known. Keys are added in sorted order + // and duplicated keys are possible, so typically, the builder will + // only add this key if its hash is different from the most recently + // added. + virtual void AddKey(const Slice& key) = 0; + + // Called by RocksDB before Finish to populate + // TableProperties::num_filter_entries, so should represent the + // number of unique keys (and/or prefixes) added, but does not have + // to be exact. `return 0;` may be used to conspicuously indicate "unknown". + virtual size_t EstimateEntriesAdded() = 0; + + // Generate the filter using the keys that are added + // The return value of this function would be the filter bits, + // The ownership of actual data is set to buf + virtual Slice Finish(std::unique_ptr* buf) = 0; + + // Similar to Finish(std::unique_ptr* buf), except that + // for a non-null status pointer argument, it will point to + // Status::Corruption() when there is any corruption during filter + // construction or Status::OK() otherwise. + // + // WARNING: do not use a filter resulted from a corrupted construction + // TODO: refactor this to have a better signature, consolidate + virtual Slice Finish(std::unique_ptr* buf, + Status* /* status */) { + return Finish(buf); + } + + // Verify the filter returned from calling FilterBitsBuilder::Finish. + // The function returns Status::Corruption() if there is any corruption in the + // constructed filter or Status::OK() otherwise. + // + // Implementations should normally consult + // FilterBuildingContext::table_options.detect_filter_construct_corruption + // to determine whether to perform verification or to skip by returning + // Status::OK(). The decision is left to the FilterBitsBuilder so that + // verification prerequisites before PostVerify can be skipped when not + // configured. + // + // RocksDB internal will always call MaybePostVerify() on the filter after + // it is returned from calling FilterBitsBuilder::Finish + // except for FilterBitsBuilder::Finish resulting a corruption + // status, which indicates the filter is already in a corrupted state and + // there is no need to post-verify + virtual Status MaybePostVerify(const Slice& /* filter_content */) { + return Status::OK(); + } + + // Approximate the number of keys that can be added and generate a filter + // <= the specified number of bytes. Callers (including RocksDB) should + // only use this result for optimizing performance and not as a guarantee. + virtual size_t ApproximateNumEntries(size_t bytes) = 0; +}; + +// A class that checks if a key can be in filter +// It should be initialized by Slice generated by BitsBuilder +class FilterBitsReader { + public: + virtual ~FilterBitsReader() {} + + // Check if the entry match the bits in filter + virtual bool MayMatch(const Slice& entry) = 0; + + // Check if an array of entries match the bits in filter + virtual void MayMatch(int num_keys, Slice** keys, bool* may_match) { + for (int i = 0; i < num_keys; ++i) { + may_match[i] = MayMatch(*keys[i]); + } + } +}; + +// Exposes any extra information needed for testing built-in +// FilterBitsBuilders +class BuiltinFilterBitsBuilder : public FilterBitsBuilder { + public: + // Calculate number of bytes needed for a new filter, including + // metadata. Passing the result to ApproximateNumEntries should + // (ideally, usually) return >= the num_entry passed in. + // When optimize_filters_for_memory is enabled, this function + // is not authoritative but represents a target size that should + // be close to the average size. + virtual size_t CalculateSpace(size_t num_entries) = 0; + + // Returns an estimate of the FP rate of the returned filter if + // `num_entries` keys are added and the filter returned by Finish + // is `bytes` bytes. + virtual double EstimatedFpRate(size_t num_entries, size_t bytes) = 0; +}; + +// Base class for RocksDB built-in filter reader with +// extra useful functionalities for inernal. +class BuiltinFilterBitsReader : public FilterBitsReader { + public: + // Check if the hash of the entry match the bits in filter + virtual bool HashMayMatch(const uint64_t /* h */) { return true; } +}; + +// Base class for RocksDB built-in filter policies. This provides the +// ability to read all kinds of built-in filters (so that old filters can +// be used even when you change between built-in policies). +class BuiltinFilterPolicy : public FilterPolicy { + public: // overrides + // Read metadata to determine what kind of FilterBitsReader is needed + // and return a new one. This must successfully process any filter data + // generated by a built-in FilterBitsBuilder, regardless of the impl + // chosen for this BloomFilterPolicy. + FilterBitsReader* GetFilterBitsReader(const Slice& contents) const override; + static const char* kClassName(); + bool IsInstanceOf(const std::string& id) const override; + // All variants of BuiltinFilterPolicy can read each others filters. + const char* CompatibilityName() const override; + static const char* kCompatibilityName(); + + public: // new + // An internal function for the implementation of + // BuiltinFilterBitsReader::GetFilterBitsReader without requiring an instance + // or working around potential virtual overrides. + static BuiltinFilterBitsReader* GetBuiltinFilterBitsReader( + const Slice& contents); + + // Returns a new FilterBitsBuilder from the filter_policy in + // table_options of a context, or nullptr if not applicable. + // (An internal convenience function to save boilerplate.) + static FilterBitsBuilder* GetBuilderFromContext(const FilterBuildingContext&); + + private: + // For Bloom filter implementation(s) + static BuiltinFilterBitsReader* GetBloomBitsReader(const Slice& contents); + + // For Ribbon filter implementation(s) + static BuiltinFilterBitsReader* GetRibbonBitsReader(const Slice& contents); +}; + +// A "read only" filter policy used for backward compatibility with old +// OPTIONS files, which did not specifying a Bloom configuration, just +// "rocksdb.BuiltinBloomFilter". Although this can read existing filters, +// this policy does not build new filters, so new SST files generated +// under the policy will get no filters (like nullptr FilterPolicy). +// This class is considered internal API and subject to change. +class ReadOnlyBuiltinFilterPolicy : public BuiltinFilterPolicy { + public: + const char* Name() const override { return kClassName(); } + static const char* kClassName(); + + // Does not write filters. + FilterBitsBuilder* GetBuilderWithContext( + const FilterBuildingContext&) const override { + return nullptr; + } +}; + +// RocksDB built-in filter policy for Bloom or Bloom-like filters including +// Ribbon filters. +// This class is considered internal API and subject to change. +// See NewBloomFilterPolicy and NewRibbonFilterPolicy. +class BloomLikeFilterPolicy : public BuiltinFilterPolicy { + public: + explicit BloomLikeFilterPolicy(double bits_per_key); + + ~BloomLikeFilterPolicy() override; + static const char* kClassName(); + bool IsInstanceOf(const std::string& id) const override; + + std::string GetId() const override; + + // Essentially for testing only: configured millibits/key + int GetMillibitsPerKey() const { return millibits_per_key_; } + // Essentially for testing only: legacy whole bits/key + int GetWholeBitsPerKey() const { return whole_bits_per_key_; } + + // All the different underlying implementations that a BloomLikeFilterPolicy + // might use, as a configuration string name for a testing mode for + // "always use this implementation." Only appropriate for unit tests. + static const std::vector& GetAllFixedImpls(); + + // Convenience function for creating by name for fixed impls + static std::shared_ptr Create(const std::string& name, + double bits_per_key); + + protected: + // Some implementations used by aggregating policies + FilterBitsBuilder* GetLegacyBloomBuilderWithContext( + const FilterBuildingContext& context) const; + FilterBitsBuilder* GetFastLocalBloomBuilderWithContext( + const FilterBuildingContext& context) const; + FilterBitsBuilder* GetStandard128RibbonBuilderWithContext( + const FilterBuildingContext& context) const; + + std::string GetBitsPerKeySuffix() const; + + private: + // Bits per key settings are for configuring Bloom filters. + + // Newer filters support fractional bits per key. For predictable behavior + // of 0.001-precision values across floating point implementations, we + // round to thousandths of a bit (on average) per key. + int millibits_per_key_; + + // Older filters round to whole number bits per key. (There *should* be no + // compatibility issue with fractional bits per key, but preserving old + // behavior with format_version < 5 just in case.) + int whole_bits_per_key_; + + // For configuring Ribbon filter: a desired value for 1/fp_rate. For + // example, 100 -> 1% fp rate. + double desired_one_in_fp_rate_; + + // Whether relevant warnings have been logged already. (Remember so we + // only report once per BloomFilterPolicy instance, to keep the noise down.) + mutable std::atomic warned_; + + // State for implementing optimize_filters_for_memory. Essentially, this + // tracks a surplus or deficit in total FP rate of filters generated by + // builders under this policy vs. what would have been generated without + // optimize_filters_for_memory. + // + // To avoid floating point weirdness, the actual value is + // Sum over all generated filters f: + // (predicted_fp_rate(f) - predicted_fp_rate(f|o_f_f_m=false)) * 2^32 + mutable std::atomic aggregate_rounding_balance_; +}; + +// For NewBloomFilterPolicy +// +// This is a user-facing policy that automatically choose between +// LegacyBloom and FastLocalBloom based on context at build time, +// including compatibility with format_version. +class BloomFilterPolicy : public BloomLikeFilterPolicy { + public: + explicit BloomFilterPolicy(double bits_per_key); + + // To use this function, call BuiltinFilterPolicy::GetBuilderFromContext(). + // + // Neither the context nor any objects therein should be saved beyond + // the call to this function, unless it's shared_ptr. + FilterBitsBuilder* GetBuilderWithContext( + const FilterBuildingContext&) const override; + + static const char* kClassName(); + const char* Name() const override { return kClassName(); } + static const char* kNickName(); + const char* NickName() const override { return kNickName(); } + std::string GetId() const override; +}; + +// For NewRibbonFilterPolicy +// +// This is a user-facing policy that chooses between Standard128Ribbon +// and FastLocalBloom based on context at build time (LSM level and other +// factors in extreme cases). +class RibbonFilterPolicy : public BloomLikeFilterPolicy { + public: + explicit RibbonFilterPolicy(double bloom_equivalent_bits_per_key, + int bloom_before_level); + + FilterBitsBuilder* GetBuilderWithContext( + const FilterBuildingContext&) const override; + + int GetBloomBeforeLevel() const { return bloom_before_level_; } + + static const char* kClassName(); + const char* Name() const override { return kClassName(); } + static const char* kNickName(); + const char* NickName() const override { return kNickName(); } + std::string GetId() const override; + + private: + const int bloom_before_level_; +}; + +// For testing only, but always constructable with internal names +namespace test { + +class LegacyBloomFilterPolicy : public BloomLikeFilterPolicy { + public: + explicit LegacyBloomFilterPolicy(double bits_per_key) + : BloomLikeFilterPolicy(bits_per_key) {} + + FilterBitsBuilder* GetBuilderWithContext( + const FilterBuildingContext& context) const override; + + static const char* kClassName(); + const char* Name() const override { return kClassName(); } +}; + +class FastLocalBloomFilterPolicy : public BloomLikeFilterPolicy { + public: + explicit FastLocalBloomFilterPolicy(double bits_per_key) + : BloomLikeFilterPolicy(bits_per_key) {} + + FilterBitsBuilder* GetBuilderWithContext( + const FilterBuildingContext& context) const override; + + static const char* kClassName(); + const char* Name() const override { return kClassName(); } +}; + +class Standard128RibbonFilterPolicy : public BloomLikeFilterPolicy { + public: + explicit Standard128RibbonFilterPolicy(double bloom_equiv_bits_per_key) + : BloomLikeFilterPolicy(bloom_equiv_bits_per_key) {} + + FilterBitsBuilder* GetBuilderWithContext( + const FilterBuildingContext& context) const override; + + static const char* kClassName(); + const char* Name() const override { return kClassName(); } +}; + +} // namespace test + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/flush_block_policy.cc b/librocksdb-sys/rocksdb/table/block_based/flush_block_policy.cc new file mode 100644 index 0000000..d5cc310 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/flush_block_policy.cc @@ -0,0 +1,132 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "rocksdb/flush_block_policy.h" + +#include +#include + +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "rocksdb/utilities/customizable_util.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/block_based/block_builder.h" +#include "table/block_based/flush_block_policy_impl.h" +#include "table/format.h" + +namespace ROCKSDB_NAMESPACE { + +// Flush block by size +class FlushBlockBySizePolicy : public FlushBlockPolicy { + public: + // @params block_size: Approximate size of user data packed per + // block. + // @params block_size_deviation: This is used to close a block before it + // reaches the configured + FlushBlockBySizePolicy(const uint64_t block_size, + const uint64_t block_size_deviation, const bool align, + const BlockBuilder& data_block_builder) + : block_size_(block_size), + block_size_deviation_limit_( + ((block_size * (100 - block_size_deviation)) + 99) / 100), + align_(align), + data_block_builder_(data_block_builder) {} + + bool Update(const Slice& key, const Slice& value) override { + // it makes no sense to flush when the data block is empty + if (data_block_builder_.empty()) { + return false; + } + + auto curr_size = data_block_builder_.CurrentSizeEstimate(); + + // Do flush if one of the below two conditions is true: + // 1) if the current estimated size already exceeds the block size, + // 2) block_size_deviation is set and the estimated size after appending + // the kv will exceed the block size and the current size is under the + // the deviation. + return curr_size >= block_size_ || BlockAlmostFull(key, value); + } + + private: + bool BlockAlmostFull(const Slice& key, const Slice& value) const { + if (block_size_deviation_limit_ == 0) { + return false; + } + + const auto curr_size = data_block_builder_.CurrentSizeEstimate(); + auto estimated_size_after = + data_block_builder_.EstimateSizeAfterKV(key, value); + + if (align_) { + estimated_size_after += BlockBasedTable::kBlockTrailerSize; + return estimated_size_after > block_size_; + } + + return estimated_size_after > block_size_ && + curr_size > block_size_deviation_limit_; + } + + const uint64_t block_size_; + const uint64_t block_size_deviation_limit_; + const bool align_; + const BlockBuilder& data_block_builder_; +}; + +FlushBlockPolicy* FlushBlockBySizePolicyFactory::NewFlushBlockPolicy( + const BlockBasedTableOptions& table_options, + const BlockBuilder& data_block_builder) const { + return new FlushBlockBySizePolicy( + table_options.block_size, table_options.block_size_deviation, + table_options.block_align, data_block_builder); +} + +FlushBlockPolicy* FlushBlockBySizePolicyFactory::NewFlushBlockPolicy( + const uint64_t size, const int deviation, + const BlockBuilder& data_block_builder) { + return new FlushBlockBySizePolicy(size, deviation, false, data_block_builder); +} + +static int RegisterFlushBlockPolicyFactories(ObjectLibrary& library, + const std::string& /*arg*/) { + library.AddFactory( + FlushBlockBySizePolicyFactory::kClassName(), + [](const std::string& /*uri*/, + std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new FlushBlockBySizePolicyFactory()); + return guard->get(); + }); + library.AddFactory( + FlushBlockEveryKeyPolicyFactory::kClassName(), + [](const std::string& /*uri*/, + std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new FlushBlockEveryKeyPolicyFactory()); + return guard->get(); + }); + return 2; +} + +FlushBlockBySizePolicyFactory::FlushBlockBySizePolicyFactory() + : FlushBlockPolicyFactory() {} + +Status FlushBlockPolicyFactory::CreateFromString( + const ConfigOptions& config_options, const std::string& value, + std::shared_ptr* factory) { + static std::once_flag once; + std::call_once(once, [&]() { + RegisterFlushBlockPolicyFactories(*(ObjectLibrary::Default().get()), ""); + }); + + if (value.empty()) { + factory->reset(new FlushBlockBySizePolicyFactory()); + return Status::OK(); + } else { + return LoadSharedObject(config_options, value, + factory); + } +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/flush_block_policy_impl.h b/librocksdb-sys/rocksdb/table/block_based/flush_block_policy_impl.h new file mode 100644 index 0000000..4f79682 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/flush_block_policy_impl.h @@ -0,0 +1,40 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "rocksdb/flush_block_policy.h" + +namespace ROCKSDB_NAMESPACE { + +// FlushBlockEveryKeyPolicy currently used only in tests. + +class FlushBlockEveryKeyPolicy : public FlushBlockPolicy { + public: + bool Update(const Slice& /*key*/, const Slice& /*value*/) override { + if (!start_) { + start_ = true; + return false; + } + return true; + } + + private: + bool start_ = false; +}; + +class FlushBlockEveryKeyPolicyFactory : public FlushBlockPolicyFactory { + public: + explicit FlushBlockEveryKeyPolicyFactory() {} + + static const char* kClassName() { return "FlushBlockEveryKeyPolicyFactory"; } + const char* Name() const override { return kClassName(); } + + FlushBlockPolicy* NewFlushBlockPolicy( + const BlockBasedTableOptions& /*table_options*/, + const BlockBuilder& /*data_block_builder*/) const override { + return new FlushBlockEveryKeyPolicy; + } +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/full_filter_block.cc b/librocksdb-sys/rocksdb/table/block_based/full_filter_block.cc new file mode 100644 index 0000000..60ff7c4 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/full_filter_block.cc @@ -0,0 +1,290 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "table/block_based/full_filter_block.h" + +#include + +#include "block_type.h" +#include "monitoring/perf_context_imp.h" +#include "port/malloc.h" +#include "port/port.h" +#include "rocksdb/filter_policy.h" +#include "table/block_based/block_based_table_reader.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { + +FullFilterBlockBuilder::FullFilterBlockBuilder( + const SliceTransform* _prefix_extractor, bool whole_key_filtering, + FilterBitsBuilder* filter_bits_builder) + : prefix_extractor_(_prefix_extractor), + whole_key_filtering_(whole_key_filtering), + last_whole_key_recorded_(false), + last_prefix_recorded_(false), + last_key_in_domain_(false), + any_added_(false) { + assert(filter_bits_builder != nullptr); + filter_bits_builder_.reset(filter_bits_builder); +} + +size_t FullFilterBlockBuilder::EstimateEntriesAdded() { + return filter_bits_builder_->EstimateEntriesAdded(); +} + +void FullFilterBlockBuilder::Add(const Slice& key_without_ts) { + const bool add_prefix = + prefix_extractor_ && prefix_extractor_->InDomain(key_without_ts); + + if (!last_prefix_recorded_ && last_key_in_domain_) { + // We can reach here when a new filter partition starts in partitioned + // filter. The last prefix in the previous partition should be added if + // necessary regardless of key_without_ts, to support prefix SeekForPrev. + AddKey(last_prefix_str_); + last_prefix_recorded_ = true; + } + + if (whole_key_filtering_) { + if (!add_prefix) { + AddKey(key_without_ts); + } else { + // if both whole_key and prefix are added to bloom then we will have whole + // key_without_ts and prefix addition being interleaved and thus cannot + // rely on the bits builder to properly detect the duplicates by comparing + // with the last item. + Slice last_whole_key = Slice(last_whole_key_str_); + if (!last_whole_key_recorded_ || + last_whole_key.compare(key_without_ts) != 0) { + AddKey(key_without_ts); + last_whole_key_recorded_ = true; + last_whole_key_str_.assign(key_without_ts.data(), + key_without_ts.size()); + } + } + } + if (add_prefix) { + last_key_in_domain_ = true; + AddPrefix(key_without_ts); + } else { + last_key_in_domain_ = false; + } +} + +// Add key to filter if needed +inline void FullFilterBlockBuilder::AddKey(const Slice& key) { + filter_bits_builder_->AddKey(key); + any_added_ = true; +} + +// Add prefix to filter if needed +void FullFilterBlockBuilder::AddPrefix(const Slice& key) { + assert(prefix_extractor_ && prefix_extractor_->InDomain(key)); + Slice prefix = prefix_extractor_->Transform(key); + if (whole_key_filtering_) { + // if both whole_key and prefix are added to bloom then we will have whole + // key and prefix addition being interleaved and thus cannot rely on the + // bits builder to properly detect the duplicates by comparing with the last + // item. + Slice last_prefix = Slice(last_prefix_str_); + if (!last_prefix_recorded_ || last_prefix.compare(prefix) != 0) { + AddKey(prefix); + last_prefix_recorded_ = true; + last_prefix_str_.assign(prefix.data(), prefix.size()); + } + } else { + AddKey(prefix); + } +} + +void FullFilterBlockBuilder::Reset() { + last_whole_key_recorded_ = false; + last_prefix_recorded_ = false; +} + +Slice FullFilterBlockBuilder::Finish( + const BlockHandle& /*tmp*/, Status* status, + std::unique_ptr* filter_data) { + Reset(); + // In this impl we ignore BlockHandle + *status = Status::OK(); + if (any_added_) { + any_added_ = false; + Slice filter_content = filter_bits_builder_->Finish( + filter_data ? filter_data : &filter_data_, status); + return filter_content; + } + return Slice(); +} + +FullFilterBlockReader::FullFilterBlockReader( + const BlockBasedTable* t, + CachableEntry&& filter_block) + : FilterBlockReaderCommon(t, std::move(filter_block)) {} + +bool FullFilterBlockReader::KeyMayMatch(const Slice& key, const bool no_io, + const Slice* const /*const_ikey_ptr*/, + GetContext* get_context, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) { + if (!whole_key_filtering()) { + return true; + } + return MayMatch(key, no_io, get_context, lookup_context, read_options); +} + +std::unique_ptr FullFilterBlockReader::Create( + const BlockBasedTable* table, const ReadOptions& ro, + FilePrefetchBuffer* prefetch_buffer, bool use_cache, bool prefetch, + bool pin, BlockCacheLookupContext* lookup_context) { + assert(table); + assert(table->get_rep()); + assert(!pin || prefetch); + + CachableEntry filter_block; + if (prefetch || !use_cache) { + const Status s = ReadFilterBlock(table, prefetch_buffer, ro, use_cache, + nullptr /* get_context */, lookup_context, + &filter_block); + if (!s.ok()) { + IGNORE_STATUS_IF_ERROR(s); + return std::unique_ptr(); + } + + if (use_cache && !pin) { + filter_block.Reset(); + } + } + + return std::unique_ptr( + new FullFilterBlockReader(table, std::move(filter_block))); +} + +bool FullFilterBlockReader::PrefixMayMatch( + const Slice& prefix, const bool no_io, + const Slice* const /*const_ikey_ptr*/, GetContext* get_context, + BlockCacheLookupContext* lookup_context, const ReadOptions& read_options) { + return MayMatch(prefix, no_io, get_context, lookup_context, read_options); +} + +bool FullFilterBlockReader::MayMatch(const Slice& entry, bool no_io, + GetContext* get_context, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) const { + CachableEntry filter_block; + + const Status s = GetOrReadFilterBlock(no_io, get_context, lookup_context, + &filter_block, read_options); + if (!s.ok()) { + IGNORE_STATUS_IF_ERROR(s); + return true; + } + + assert(filter_block.GetValue()); + + FilterBitsReader* const filter_bits_reader = + filter_block.GetValue()->filter_bits_reader(); + + if (filter_bits_reader) { + if (filter_bits_reader->MayMatch(entry)) { + PERF_COUNTER_ADD(bloom_sst_hit_count, 1); + return true; + } else { + PERF_COUNTER_ADD(bloom_sst_miss_count, 1); + return false; + } + } + return true; +} + +void FullFilterBlockReader::KeysMayMatch( + MultiGetRange* range, const bool no_io, + BlockCacheLookupContext* lookup_context, const ReadOptions& read_options) { + if (!whole_key_filtering()) { + // Simply return. Don't skip any key - consider all keys as likely to be + // present + return; + } + MayMatch(range, no_io, nullptr, lookup_context, read_options); +} + +void FullFilterBlockReader::PrefixesMayMatch( + MultiGetRange* range, const SliceTransform* prefix_extractor, + const bool no_io, BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) { + MayMatch(range, no_io, prefix_extractor, lookup_context, read_options); +} + +void FullFilterBlockReader::MayMatch(MultiGetRange* range, bool no_io, + const SliceTransform* prefix_extractor, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) const { + CachableEntry filter_block; + + const Status s = + GetOrReadFilterBlock(no_io, range->begin()->get_context, lookup_context, + &filter_block, read_options); + if (!s.ok()) { + IGNORE_STATUS_IF_ERROR(s); + return; + } + + assert(filter_block.GetValue()); + + FilterBitsReader* const filter_bits_reader = + filter_block.GetValue()->filter_bits_reader(); + + if (!filter_bits_reader) { + return; + } + + // We need to use an array instead of autovector for may_match since + // &may_match[0] doesn't work for autovector (compiler error). So + // declare both keys and may_match as arrays, which is also slightly less + // expensive compared to autovector + std::array keys; + std::array may_match = {{true}}; + autovector prefixes; + int num_keys = 0; + MultiGetRange filter_range(*range, range->begin(), range->end()); + for (auto iter = filter_range.begin(); iter != filter_range.end(); ++iter) { + if (!prefix_extractor) { + keys[num_keys++] = &iter->ukey_without_ts; + } else if (prefix_extractor->InDomain(iter->ukey_without_ts)) { + prefixes.emplace_back(prefix_extractor->Transform(iter->ukey_without_ts)); + keys[num_keys++] = &prefixes.back(); + } else { + filter_range.SkipKey(iter); + } + } + + filter_bits_reader->MayMatch(num_keys, &keys[0], &may_match[0]); + + int i = 0; + for (auto iter = filter_range.begin(); iter != filter_range.end(); ++iter) { + if (!may_match[i]) { + // Update original MultiGet range to skip this key. The filter_range + // was temporarily used just to skip keys not in prefix_extractor domain + range->SkipKey(iter); + PERF_COUNTER_ADD(bloom_sst_miss_count, 1); + } else { + // PERF_COUNTER_ADD(bloom_sst_hit_count, 1); + PerfContext* perf_ctx = get_perf_context(); + perf_ctx->bloom_sst_hit_count++; + } + ++i; + } +} + +size_t FullFilterBlockReader::ApproximateMemoryUsage() const { + size_t usage = ApproximateFilterBlockMemoryUsage(); +#ifdef ROCKSDB_MALLOC_USABLE_SIZE + usage += malloc_usable_size(const_cast(this)); +#else + usage += sizeof(*this); +#endif // ROCKSDB_MALLOC_USABLE_SIZE + return usage; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/full_filter_block.h b/librocksdb-sys/rocksdb/table/block_based/full_filter_block.h new file mode 100644 index 0000000..7b0890d --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/full_filter_block.h @@ -0,0 +1,147 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include + +#include +#include +#include + +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "rocksdb/slice_transform.h" +#include "table/block_based/filter_block_reader_common.h" +#include "table/block_based/filter_policy_internal.h" +#include "table/block_based/parsed_full_filter_block.h" +#include "util/hash.h" + +namespace ROCKSDB_NAMESPACE { + +class FilterPolicy; +class FilterBitsBuilder; +class FilterBitsReader; + +// A FullFilterBlockBuilder is used to construct a full filter for a +// particular Table. It generates a single string which is stored as +// a special block in the Table. +// The format of full filter block is: +// +----------------------------------------------------------------+ +// | full filter for all keys in sst file | +// +----------------------------------------------------------------+ +// The full filter can be very large. At the end of it, we put +// num_probes: how many hash functions are used in bloom filter +// +class FullFilterBlockBuilder : public FilterBlockBuilder { + public: + explicit FullFilterBlockBuilder(const SliceTransform* prefix_extractor, + bool whole_key_filtering, + FilterBitsBuilder* filter_bits_builder); + // No copying allowed + FullFilterBlockBuilder(const FullFilterBlockBuilder&) = delete; + void operator=(const FullFilterBlockBuilder&) = delete; + + // bits_builder is created in filter_policy, it should be passed in here + // directly. and be deleted here + ~FullFilterBlockBuilder() {} + + virtual void Add(const Slice& key_without_ts) override; + virtual bool IsEmpty() const override { return !any_added_; } + virtual size_t EstimateEntriesAdded() override; + virtual Slice Finish( + const BlockHandle& tmp, Status* status, + std::unique_ptr* filter_data = nullptr) override; + using FilterBlockBuilder::Finish; + + virtual void ResetFilterBitsBuilder() override { + filter_bits_builder_.reset(); + } + + virtual Status MaybePostVerifyFilter(const Slice& filter_content) override { + return filter_bits_builder_->MaybePostVerify(filter_content); + } + + protected: + virtual void AddKey(const Slice& key); + std::unique_ptr filter_bits_builder_; + virtual void Reset(); + void AddPrefix(const Slice& key); + const SliceTransform* prefix_extractor() { return prefix_extractor_; } + const std::string& last_prefix_str() const { return last_prefix_str_; } + + private: + // important: all of these might point to invalid addresses + // at the time of destruction of this filter block. destructor + // should NOT dereference them. + const SliceTransform* prefix_extractor_; + bool whole_key_filtering_; + bool last_whole_key_recorded_; + std::string last_whole_key_str_; + bool last_prefix_recorded_; + std::string last_prefix_str_; + // Whether prefix_extractor_->InDomain(last_whole_key_) is true. + // Used in partitioned filters so that the last prefix from the previous + // filter partition will be added to the current partition if + // last_key_in_domain_ is true, regardless of the current key. + bool last_key_in_domain_; + bool any_added_; + std::unique_ptr filter_data_; +}; + +// A FilterBlockReader is used to parse filter from SST table. +// KeyMayMatch and PrefixMayMatch would trigger filter checking +class FullFilterBlockReader + : public FilterBlockReaderCommon { + public: + FullFilterBlockReader(const BlockBasedTable* t, + CachableEntry&& filter_block); + + static std::unique_ptr Create( + const BlockBasedTable* table, const ReadOptions& ro, + FilePrefetchBuffer* prefetch_buffer, bool use_cache, bool prefetch, + bool pin, BlockCacheLookupContext* lookup_context); + + bool KeyMayMatch(const Slice& key, const bool no_io, + const Slice* const const_ikey_ptr, GetContext* get_context, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) override; + + bool PrefixMayMatch(const Slice& prefix, const bool no_io, + const Slice* const const_ikey_ptr, + GetContext* get_context, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) override; + + void KeysMayMatch(MultiGetRange* range, const bool no_io, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) override; + // Used in partitioned filter code + void KeysMayMatch2(MultiGetRange* range, + const SliceTransform* /*prefix_extractor*/, + const bool no_io, BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) { + KeysMayMatch(range, no_io, lookup_context, read_options); + } + + void PrefixesMayMatch(MultiGetRange* range, + const SliceTransform* prefix_extractor, + const bool no_io, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) override; + size_t ApproximateMemoryUsage() const override; + + private: + bool MayMatch(const Slice& entry, bool no_io, GetContext* get_context, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) const; + void MayMatch(MultiGetRange* range, bool no_io, + const SliceTransform* prefix_extractor, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) const; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/full_filter_block_test.cc b/librocksdb-sys/rocksdb/table/block_based/full_filter_block_test.cc new file mode 100644 index 0000000..0268b7b --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/full_filter_block_test.cc @@ -0,0 +1,323 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "table/block_based/full_filter_block.h" + +#include + +#include "rocksdb/filter_policy.h" +#include "rocksdb/status.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/block_based/filter_policy_internal.h" +#include "table/block_based/mock_block_based_table.h" +#include "table/format.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/coding.h" +#include "util/hash.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +class TestFilterBitsBuilder : public FilterBitsBuilder { + public: + explicit TestFilterBitsBuilder() {} + + // Add Key to filter + void AddKey(const Slice& key) override { + hash_entries_.push_back(Hash(key.data(), key.size(), 1)); + } + + using FilterBitsBuilder::Finish; + + // Generate the filter using the keys that are added + Slice Finish(std::unique_ptr* buf) override { + uint32_t len = static_cast(hash_entries_.size()) * 4; + char* data = new char[len]; + for (size_t i = 0; i < hash_entries_.size(); i++) { + EncodeFixed32(data + i * 4, hash_entries_[i]); + } + const char* const_data = data; + buf->reset(const_data); + return Slice(data, len); + } + + size_t EstimateEntriesAdded() override { return hash_entries_.size(); } + + size_t ApproximateNumEntries(size_t bytes) override { return bytes / 4; } + + private: + std::vector hash_entries_; +}; + +class MockBlockBasedTable : public BlockBasedTable { + public: + explicit MockBlockBasedTable(Rep* rep) + : BlockBasedTable(rep, nullptr /* block_cache_tracer */) {} +}; + +class TestFilterBitsReader : public FilterBitsReader { + public: + explicit TestFilterBitsReader(const Slice& contents) + : data_(contents.data()), len_(static_cast(contents.size())) {} + + // Silence compiler warning about overloaded virtual + using FilterBitsReader::MayMatch; + bool MayMatch(const Slice& entry) override { + uint32_t h = Hash(entry.data(), entry.size(), 1); + for (size_t i = 0; i + 4 <= len_; i += 4) { + if (h == DecodeFixed32(data_ + i)) { + return true; + } + } + return false; + } + + private: + const char* data_; + uint32_t len_; +}; + +class TestHashFilter : public FilterPolicy { + public: + const char* Name() const override { return "TestHashFilter"; } + const char* CompatibilityName() const override { return Name(); } + + FilterBitsBuilder* GetBuilderWithContext( + const FilterBuildingContext&) const override { + return new TestFilterBitsBuilder(); + } + + FilterBitsReader* GetFilterBitsReader(const Slice& contents) const override { + return new TestFilterBitsReader(contents); + } +}; + +class PluginFullFilterBlockTest : public mock::MockBlockBasedTableTester, + public testing::Test { + public: + PluginFullFilterBlockTest() + : mock::MockBlockBasedTableTester(new TestHashFilter) {} +}; + +TEST_F(PluginFullFilterBlockTest, PluginEmptyBuilder) { + FullFilterBlockBuilder builder(nullptr, true, GetBuilder()); + Slice slice = builder.Finish(); + ASSERT_EQ("", EscapeString(slice)); + + CachableEntry block( + new ParsedFullFilterBlock(table_options_.filter_policy.get(), + BlockContents(slice)), + nullptr /* cache */, nullptr /* cache_handle */, true /* own_value */); + + FullFilterBlockReader reader(table_.get(), std::move(block)); + // Remain same symantic with blockbased filter + ASSERT_TRUE(reader.KeyMayMatch("foo", + /*no_io=*/false, /*const_ikey_ptr=*/nullptr, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, ReadOptions())); +} + +TEST_F(PluginFullFilterBlockTest, PluginSingleChunk) { + FullFilterBlockBuilder builder(nullptr, true, GetBuilder()); + builder.Add("foo"); + builder.Add("bar"); + builder.Add("box"); + builder.Add("box"); + builder.Add("hello"); + Slice slice = builder.Finish(); + + CachableEntry block( + new ParsedFullFilterBlock(table_options_.filter_policy.get(), + BlockContents(slice)), + nullptr /* cache */, nullptr /* cache_handle */, true /* own_value */); + + FullFilterBlockReader reader(table_.get(), std::move(block)); + ASSERT_TRUE(reader.KeyMayMatch("foo", + /*no_io=*/false, /*const_ikey_ptr=*/nullptr, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, ReadOptions())); + ASSERT_TRUE(reader.KeyMayMatch("bar", + /*no_io=*/false, /*const_ikey_ptr=*/nullptr, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, ReadOptions())); + ASSERT_TRUE(reader.KeyMayMatch("box", + /*no_io=*/false, /*const_ikey_ptr=*/nullptr, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, ReadOptions())); + ASSERT_TRUE(reader.KeyMayMatch("hello", + /*no_io=*/false, /*const_ikey_ptr=*/nullptr, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, ReadOptions())); + ASSERT_TRUE(reader.KeyMayMatch("foo", + /*no_io=*/false, /*const_ikey_ptr=*/nullptr, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, ReadOptions())); + ASSERT_TRUE(!reader.KeyMayMatch("missing", + /*no_io=*/false, /*const_ikey_ptr=*/nullptr, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, ReadOptions())); + ASSERT_TRUE(!reader.KeyMayMatch("other", + /*no_io=*/false, /*const_ikey_ptr=*/nullptr, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, ReadOptions())); +} + +class FullFilterBlockTest : public mock::MockBlockBasedTableTester, + public testing::Test { + public: + FullFilterBlockTest() + : mock::MockBlockBasedTableTester(NewBloomFilterPolicy(10, false)) {} +}; + +TEST_F(FullFilterBlockTest, EmptyBuilder) { + FullFilterBlockBuilder builder(nullptr, true, GetBuilder()); + Slice slice = builder.Finish(); + ASSERT_EQ("", EscapeString(slice)); + + CachableEntry block( + new ParsedFullFilterBlock(table_options_.filter_policy.get(), + BlockContents(slice)), + nullptr /* cache */, nullptr /* cache_handle */, true /* own_value */); + + FullFilterBlockReader reader(table_.get(), std::move(block)); + // Remain same symantic with blockbased filter + ASSERT_TRUE(reader.KeyMayMatch("foo", + /*no_io=*/false, /*const_ikey_ptr=*/nullptr, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, ReadOptions())); +} + +class CountUniqueFilterBitsBuilderWrapper : public FilterBitsBuilder { + std::unique_ptr b_; + std::set uniq_; + + public: + explicit CountUniqueFilterBitsBuilderWrapper(FilterBitsBuilder* b) : b_(b) {} + + ~CountUniqueFilterBitsBuilderWrapper() override {} + + void AddKey(const Slice& key) override { + b_->AddKey(key); + uniq_.insert(key.ToString()); + } + + using FilterBitsBuilder::Finish; + + Slice Finish(std::unique_ptr* buf) override { + Slice rv = b_->Finish(buf); + Status s_dont_care = b_->MaybePostVerify(rv); + s_dont_care.PermitUncheckedError(); + uniq_.clear(); + return rv; + } + + size_t EstimateEntriesAdded() override { return b_->EstimateEntriesAdded(); } + + size_t ApproximateNumEntries(size_t bytes) override { + return b_->ApproximateNumEntries(bytes); + } + + size_t CountUnique() { return uniq_.size(); } +}; + +TEST_F(FullFilterBlockTest, DuplicateEntries) { + { // empty prefixes + std::unique_ptr prefix_extractor( + NewFixedPrefixTransform(0)); + auto bits_builder = new CountUniqueFilterBitsBuilderWrapper(GetBuilder()); + const bool WHOLE_KEY = true; + FullFilterBlockBuilder builder(prefix_extractor.get(), WHOLE_KEY, + bits_builder); + ASSERT_EQ(0, bits_builder->CountUnique()); + // adds key and empty prefix; both abstractions count them + builder.Add("key1"); + ASSERT_EQ(2, bits_builder->CountUnique()); + // Add different key (unique) and also empty prefix (not unique). + // From here in this test, it's immaterial whether the block builder + // can count unique keys. + builder.Add("key2"); + ASSERT_EQ(3, bits_builder->CountUnique()); + // Empty key -> nothing unique + builder.Add(""); + ASSERT_EQ(3, bits_builder->CountUnique()); + } + + // mix of empty and non-empty + std::unique_ptr prefix_extractor( + NewFixedPrefixTransform(7)); + auto bits_builder = new CountUniqueFilterBitsBuilderWrapper(GetBuilder()); + const bool WHOLE_KEY = true; + FullFilterBlockBuilder builder(prefix_extractor.get(), WHOLE_KEY, + bits_builder); + builder.Add(""); // test with empty key too + builder.Add("prefix1key1"); + builder.Add("prefix1key1"); + builder.Add("prefix1key2"); + builder.Add("prefix1key3"); + builder.Add("prefix2key4"); + // 1 empty, 2 non-empty prefixes, and 4 non-empty keys + ASSERT_EQ(1 + 2 + 4, bits_builder->CountUnique()); +} + +TEST_F(FullFilterBlockTest, SingleChunk) { + FullFilterBlockBuilder builder(nullptr, true, GetBuilder()); + ASSERT_TRUE(builder.IsEmpty()); + builder.Add("foo"); + ASSERT_FALSE(builder.IsEmpty()); + builder.Add("bar"); + builder.Add("box"); + builder.Add("box"); + builder.Add("hello"); + // "box" only counts once + ASSERT_EQ(4, builder.EstimateEntriesAdded()); + ASSERT_FALSE(builder.IsEmpty()); + Status s; + Slice slice = builder.Finish(BlockHandle(), &s); + ASSERT_OK(s); + + CachableEntry block( + new ParsedFullFilterBlock(table_options_.filter_policy.get(), + BlockContents(slice)), + nullptr /* cache */, nullptr /* cache_handle */, true /* own_value */); + + FullFilterBlockReader reader(table_.get(), std::move(block)); + ASSERT_TRUE(reader.KeyMayMatch("foo", + /*no_io=*/false, /*const_ikey_ptr=*/nullptr, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, ReadOptions())); + ASSERT_TRUE(reader.KeyMayMatch("bar", + /*no_io=*/false, /*const_ikey_ptr=*/nullptr, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, ReadOptions())); + ASSERT_TRUE(reader.KeyMayMatch("box", + /*no_io=*/false, /*const_ikey_ptr=*/nullptr, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, ReadOptions())); + ASSERT_TRUE(reader.KeyMayMatch("hello", + /*no_io=*/false, /*const_ikey_ptr=*/nullptr, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, ReadOptions())); + ASSERT_TRUE(reader.KeyMayMatch("foo", + /*no_io=*/false, /*const_ikey_ptr=*/nullptr, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, ReadOptions())); + ASSERT_TRUE(!reader.KeyMayMatch("missing", + /*no_io=*/false, /*const_ikey_ptr=*/nullptr, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, ReadOptions())); + ASSERT_TRUE(!reader.KeyMayMatch("other", + /*no_io=*/false, /*const_ikey_ptr=*/nullptr, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, ReadOptions())); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/table/block_based/hash_index_reader.cc b/librocksdb-sys/rocksdb/table/block_based/hash_index_reader.cc new file mode 100644 index 0000000..5b710a7 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/hash_index_reader.cc @@ -0,0 +1,147 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "table/block_based/hash_index_reader.h" + +#include "table/block_fetcher.h" +#include "table/meta_blocks.h" + +namespace ROCKSDB_NAMESPACE { +Status HashIndexReader::Create(const BlockBasedTable* table, + const ReadOptions& ro, + FilePrefetchBuffer* prefetch_buffer, + InternalIterator* meta_index_iter, + bool use_cache, bool prefetch, bool pin, + BlockCacheLookupContext* lookup_context, + std::unique_ptr* index_reader) { + assert(table != nullptr); + assert(index_reader != nullptr); + assert(!pin || prefetch); + + const BlockBasedTable::Rep* rep = table->get_rep(); + assert(rep != nullptr); + + CachableEntry index_block; + if (prefetch || !use_cache) { + const Status s = + ReadIndexBlock(table, prefetch_buffer, ro, use_cache, + /*get_context=*/nullptr, lookup_context, &index_block); + if (!s.ok()) { + return s; + } + + if (use_cache && !pin) { + index_block.Reset(); + } + } + + // Note, failure to create prefix hash index does not need to be a + // hard error. We can still fall back to the original binary search index. + // So, Create will succeed regardless, from this point on. + + index_reader->reset(new HashIndexReader(table, std::move(index_block))); + + // Get prefixes block + BlockHandle prefixes_handle; + Status s = + FindMetaBlock(meta_index_iter, kHashIndexPrefixesBlock, &prefixes_handle); + if (!s.ok()) { + // TODO: log error + return Status::OK(); + } + + // Get index metadata block + BlockHandle prefixes_meta_handle; + s = FindMetaBlock(meta_index_iter, kHashIndexPrefixesMetadataBlock, + &prefixes_meta_handle); + if (!s.ok()) { + // TODO: log error + return Status::OK(); + } + + RandomAccessFileReader* const file = rep->file.get(); + const Footer& footer = rep->footer; + const ImmutableOptions& ioptions = rep->ioptions; + const PersistentCacheOptions& cache_options = rep->persistent_cache_options; + MemoryAllocator* const memory_allocator = + GetMemoryAllocator(rep->table_options); + + // Read contents for the blocks + BlockContents prefixes_contents; + BlockFetcher prefixes_block_fetcher( + file, prefetch_buffer, footer, ro, prefixes_handle, &prefixes_contents, + ioptions, true /*decompress*/, true /*maybe_compressed*/, + BlockType::kHashIndexPrefixes, UncompressionDict::GetEmptyDict(), + cache_options, memory_allocator); + s = prefixes_block_fetcher.ReadBlockContents(); + if (!s.ok()) { + return s; + } + BlockContents prefixes_meta_contents; + BlockFetcher prefixes_meta_block_fetcher( + file, prefetch_buffer, footer, ro, prefixes_meta_handle, + &prefixes_meta_contents, ioptions, true /*decompress*/, + true /*maybe_compressed*/, BlockType::kHashIndexMetadata, + UncompressionDict::GetEmptyDict(), cache_options, memory_allocator); + s = prefixes_meta_block_fetcher.ReadBlockContents(); + if (!s.ok()) { + // TODO: log error + return Status::OK(); + } + + BlockPrefixIndex* prefix_index = nullptr; + assert(rep->table_prefix_extractor); + s = BlockPrefixIndex::Create(rep->table_prefix_extractor.get(), + prefixes_contents.data, + prefixes_meta_contents.data, &prefix_index); + // TODO: log error + if (s.ok()) { + HashIndexReader* const hash_index_reader = + static_cast(index_reader->get()); + hash_index_reader->prefix_index_.reset(prefix_index); + } + + return Status::OK(); +} + +InternalIteratorBase* HashIndexReader::NewIterator( + const ReadOptions& read_options, bool disable_prefix_seek, + IndexBlockIter* iter, GetContext* get_context, + BlockCacheLookupContext* lookup_context) { + const BlockBasedTable::Rep* rep = table()->get_rep(); + const bool no_io = (read_options.read_tier == kBlockCacheTier); + CachableEntry index_block; + const Status s = GetOrReadIndexBlock(no_io, get_context, lookup_context, + &index_block, read_options); + if (!s.ok()) { + if (iter != nullptr) { + iter->Invalidate(s); + return iter; + } + + return NewErrorInternalIterator(s); + } + + Statistics* kNullStats = nullptr; + const bool total_order_seek = + read_options.total_order_seek || disable_prefix_seek; + // We don't return pinned data from index blocks, so no need + // to set `block_contents_pinned`. + auto it = index_block.GetValue()->NewIndexIterator( + internal_comparator()->user_comparator(), + rep->get_global_seqno(BlockType::kIndex), iter, kNullStats, + total_order_seek, index_has_first_key(), index_key_includes_seq(), + index_value_is_full(), false /* block_contents_pinned */, + user_defined_timestamps_persisted(), prefix_index_.get()); + + assert(it != nullptr); + index_block.TransferTo(it); + + return it; +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/hash_index_reader.h b/librocksdb-sys/rocksdb/table/block_based/hash_index_reader.h new file mode 100644 index 0000000..9037efc --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/hash_index_reader.h @@ -0,0 +1,49 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once + +#include "table/block_based/index_reader_common.h" + +namespace ROCKSDB_NAMESPACE { +// Index that leverages an internal hash table to quicken the lookup for a given +// key. +class HashIndexReader : public BlockBasedTable::IndexReaderCommon { + public: + static Status Create(const BlockBasedTable* table, const ReadOptions& ro, + FilePrefetchBuffer* prefetch_buffer, + InternalIterator* meta_index_iter, bool use_cache, + bool prefetch, bool pin, + BlockCacheLookupContext* lookup_context, + std::unique_ptr* index_reader); + + InternalIteratorBase* NewIterator( + const ReadOptions& read_options, bool disable_prefix_seek, + IndexBlockIter* iter, GetContext* get_context, + BlockCacheLookupContext* lookup_context) override; + + size_t ApproximateMemoryUsage() const override { + size_t usage = ApproximateIndexBlockMemoryUsage(); +#ifdef ROCKSDB_MALLOC_USABLE_SIZE + usage += malloc_usable_size(const_cast(this)); +#else + if (prefix_index_) { + usage += prefix_index_->ApproximateMemoryUsage(); + } + usage += sizeof(*this); +#endif // ROCKSDB_MALLOC_USABLE_SIZE + return usage; + } + + private: + HashIndexReader(const BlockBasedTable* t, CachableEntry&& index_block) + : IndexReaderCommon(t, std::move(index_block)) {} + + std::unique_ptr prefix_index_; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/index_builder.cc b/librocksdb-sys/rocksdb/table/block_based/index_builder.cc new file mode 100644 index 0000000..a9e02a2 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/index_builder.cc @@ -0,0 +1,297 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "table/block_based/index_builder.h" + +#include + +#include +#include +#include + +#include "db/dbformat.h" +#include "rocksdb/comparator.h" +#include "rocksdb/flush_block_policy.h" +#include "table/block_based/partitioned_filter_block.h" +#include "table/format.h" + +namespace ROCKSDB_NAMESPACE { + +// Create a index builder based on its type. +IndexBuilder* IndexBuilder::CreateIndexBuilder( + BlockBasedTableOptions::IndexType index_type, + const InternalKeyComparator* comparator, + const InternalKeySliceTransform* int_key_slice_transform, + const bool use_value_delta_encoding, + const BlockBasedTableOptions& table_opt, size_t ts_sz, + const bool persist_user_defined_timestamps) { + IndexBuilder* result = nullptr; + switch (index_type) { + case BlockBasedTableOptions::kBinarySearch: { + result = new ShortenedIndexBuilder( + comparator, table_opt.index_block_restart_interval, + table_opt.format_version, use_value_delta_encoding, + table_opt.index_shortening, /* include_first_key */ false, ts_sz, + persist_user_defined_timestamps); + break; + } + case BlockBasedTableOptions::kHashSearch: { + // Currently kHashSearch is incompatible with index_block_restart_interval + // > 1 + assert(table_opt.index_block_restart_interval == 1); + result = new HashIndexBuilder( + comparator, int_key_slice_transform, + table_opt.index_block_restart_interval, table_opt.format_version, + use_value_delta_encoding, table_opt.index_shortening, ts_sz, + persist_user_defined_timestamps); + break; + } + case BlockBasedTableOptions::kTwoLevelIndexSearch: { + result = PartitionedIndexBuilder::CreateIndexBuilder( + comparator, use_value_delta_encoding, table_opt, ts_sz, + persist_user_defined_timestamps); + break; + } + case BlockBasedTableOptions::kBinarySearchWithFirstKey: { + result = new ShortenedIndexBuilder( + comparator, table_opt.index_block_restart_interval, + table_opt.format_version, use_value_delta_encoding, + table_opt.index_shortening, /* include_first_key */ true, ts_sz, + persist_user_defined_timestamps); + break; + } + default: { + assert(!"Do not recognize the index type "); + break; + } + } + return result; +} + +void ShortenedIndexBuilder::FindShortestInternalKeySeparator( + const Comparator& comparator, std::string* start, const Slice& limit) { + // Attempt to shorten the user portion of the key + Slice user_start = ExtractUserKey(*start); + Slice user_limit = ExtractUserKey(limit); + std::string tmp(user_start.data(), user_start.size()); + comparator.FindShortestSeparator(&tmp, user_limit); + if (tmp.size() <= user_start.size() && + comparator.Compare(user_start, tmp) < 0) { + // User key has become shorter physically, but larger logically. + // Tack on the earliest possible number to the shortened user key. + PutFixed64(&tmp, + PackSequenceAndType(kMaxSequenceNumber, kValueTypeForSeek)); + assert(InternalKeyComparator(&comparator).Compare(*start, tmp) < 0); + assert(InternalKeyComparator(&comparator).Compare(tmp, limit) < 0); + start->swap(tmp); + } +} + +void ShortenedIndexBuilder::FindShortInternalKeySuccessor( + const Comparator& comparator, std::string* key) { + Slice user_key = ExtractUserKey(*key); + std::string tmp(user_key.data(), user_key.size()); + comparator.FindShortSuccessor(&tmp); + if (tmp.size() <= user_key.size() && comparator.Compare(user_key, tmp) < 0) { + // User key has become shorter physically, but larger logically. + // Tack on the earliest possible number to the shortened user key. + PutFixed64(&tmp, + PackSequenceAndType(kMaxSequenceNumber, kValueTypeForSeek)); + assert(InternalKeyComparator(&comparator).Compare(*key, tmp) < 0); + key->swap(tmp); + } +} + +PartitionedIndexBuilder* PartitionedIndexBuilder::CreateIndexBuilder( + const InternalKeyComparator* comparator, + const bool use_value_delta_encoding, + const BlockBasedTableOptions& table_opt, size_t ts_sz, + const bool persist_user_defined_timestamps) { + return new PartitionedIndexBuilder(comparator, table_opt, + use_value_delta_encoding, ts_sz, + persist_user_defined_timestamps); +} + +PartitionedIndexBuilder::PartitionedIndexBuilder( + const InternalKeyComparator* comparator, + const BlockBasedTableOptions& table_opt, + const bool use_value_delta_encoding, size_t ts_sz, + const bool persist_user_defined_timestamps) + : IndexBuilder(comparator, ts_sz, persist_user_defined_timestamps), + index_block_builder_( + table_opt.index_block_restart_interval, true /*use_delta_encoding*/, + use_value_delta_encoding, + BlockBasedTableOptions::kDataBlockBinarySearch /* index_type */, + 0.75 /* data_block_hash_table_util_ratio */, ts_sz, + persist_user_defined_timestamps, false /* is_user_key */), + index_block_builder_without_seq_( + table_opt.index_block_restart_interval, true /*use_delta_encoding*/, + use_value_delta_encoding, + BlockBasedTableOptions::kDataBlockBinarySearch /* index_type */, + 0.75 /* data_block_hash_table_util_ratio */, ts_sz, + persist_user_defined_timestamps, true /* is_user_key */), + sub_index_builder_(nullptr), + table_opt_(table_opt), + // We start by false. After each partition we revise the value based on + // what the sub_index_builder has decided. If the feature is disabled + // entirely, this will be set to true after switching the first + // sub_index_builder. Otherwise, it could be set to true even one of the + // sub_index_builders could not safely exclude seq from the keys, then it + // wil be enforced on all sub_index_builders on ::Finish. + seperator_is_key_plus_seq_(false), + use_value_delta_encoding_(use_value_delta_encoding) {} + +PartitionedIndexBuilder::~PartitionedIndexBuilder() { + delete sub_index_builder_; +} + +void PartitionedIndexBuilder::MakeNewSubIndexBuilder() { + assert(sub_index_builder_ == nullptr); + sub_index_builder_ = new ShortenedIndexBuilder( + comparator_, table_opt_.index_block_restart_interval, + table_opt_.format_version, use_value_delta_encoding_, + table_opt_.index_shortening, /* include_first_key */ false, ts_sz_, + persist_user_defined_timestamps_); + + // Set sub_index_builder_->seperator_is_key_plus_seq_ to true if + // seperator_is_key_plus_seq_ is true (internal-key mode) (set to false by + // default on Creation) so that flush policy can point to + // sub_index_builder_->index_block_builder_ + if (seperator_is_key_plus_seq_) { + sub_index_builder_->seperator_is_key_plus_seq_ = true; + } + + flush_policy_.reset(FlushBlockBySizePolicyFactory::NewFlushBlockPolicy( + table_opt_.metadata_block_size, table_opt_.block_size_deviation, + // Note: this is sub-optimal since sub_index_builder_ could later reset + // seperator_is_key_plus_seq_ but the probability of that is low. + sub_index_builder_->seperator_is_key_plus_seq_ + ? sub_index_builder_->index_block_builder_ + : sub_index_builder_->index_block_builder_without_seq_)); + partition_cut_requested_ = false; +} + +void PartitionedIndexBuilder::RequestPartitionCut() { + partition_cut_requested_ = true; +} + +void PartitionedIndexBuilder::AddIndexEntry( + std::string* last_key_in_current_block, + const Slice* first_key_in_next_block, const BlockHandle& block_handle) { + // Note: to avoid two consecuitive flush in the same method call, we do not + // check flush policy when adding the last key + if (UNLIKELY(first_key_in_next_block == nullptr)) { // no more keys + if (sub_index_builder_ == nullptr) { + MakeNewSubIndexBuilder(); + } + sub_index_builder_->AddIndexEntry(last_key_in_current_block, + first_key_in_next_block, block_handle); + if (!seperator_is_key_plus_seq_ && + sub_index_builder_->seperator_is_key_plus_seq_) { + // then we need to apply it to all sub-index builders and reset + // flush_policy to point to Block Builder of sub_index_builder_ that store + // internal keys. + seperator_is_key_plus_seq_ = true; + flush_policy_.reset(FlushBlockBySizePolicyFactory::NewFlushBlockPolicy( + table_opt_.metadata_block_size, table_opt_.block_size_deviation, + sub_index_builder_->index_block_builder_)); + } + sub_index_last_key_ = std::string(*last_key_in_current_block); + entries_.push_back( + {sub_index_last_key_, + std::unique_ptr(sub_index_builder_)}); + sub_index_builder_ = nullptr; + cut_filter_block = true; + } else { + // apply flush policy only to non-empty sub_index_builder_ + if (sub_index_builder_ != nullptr) { + std::string handle_encoding; + block_handle.EncodeTo(&handle_encoding); + bool do_flush = + partition_cut_requested_ || + flush_policy_->Update(*last_key_in_current_block, handle_encoding); + if (do_flush) { + entries_.push_back( + {sub_index_last_key_, + std::unique_ptr(sub_index_builder_)}); + cut_filter_block = true; + sub_index_builder_ = nullptr; + } + } + if (sub_index_builder_ == nullptr) { + MakeNewSubIndexBuilder(); + } + sub_index_builder_->AddIndexEntry(last_key_in_current_block, + first_key_in_next_block, block_handle); + sub_index_last_key_ = std::string(*last_key_in_current_block); + if (!seperator_is_key_plus_seq_ && + sub_index_builder_->seperator_is_key_plus_seq_) { + // then we need to apply it to all sub-index builders and reset + // flush_policy to point to Block Builder of sub_index_builder_ that store + // internal keys. + seperator_is_key_plus_seq_ = true; + flush_policy_.reset(FlushBlockBySizePolicyFactory::NewFlushBlockPolicy( + table_opt_.metadata_block_size, table_opt_.block_size_deviation, + sub_index_builder_->index_block_builder_)); + } + } +} + +Status PartitionedIndexBuilder::Finish( + IndexBlocks* index_blocks, const BlockHandle& last_partition_block_handle) { + if (partition_cnt_ == 0) { + partition_cnt_ = entries_.size(); + } + // It must be set to null after last key is added + assert(sub_index_builder_ == nullptr); + if (finishing_indexes == true) { + Entry& last_entry = entries_.front(); + std::string handle_encoding; + last_partition_block_handle.EncodeTo(&handle_encoding); + std::string handle_delta_encoding; + PutVarsignedint64( + &handle_delta_encoding, + last_partition_block_handle.size() - last_encoded_handle_.size()); + last_encoded_handle_ = last_partition_block_handle; + const Slice handle_delta_encoding_slice(handle_delta_encoding); + index_block_builder_.Add(last_entry.key, handle_encoding, + &handle_delta_encoding_slice); + if (!seperator_is_key_plus_seq_) { + index_block_builder_without_seq_.Add(ExtractUserKey(last_entry.key), + handle_encoding, + &handle_delta_encoding_slice); + } + entries_.pop_front(); + } + // If there is no sub_index left, then return the 2nd level index. + if (UNLIKELY(entries_.empty())) { + if (seperator_is_key_plus_seq_) { + index_blocks->index_block_contents = index_block_builder_.Finish(); + } else { + index_blocks->index_block_contents = + index_block_builder_without_seq_.Finish(); + } + top_level_index_size_ = index_blocks->index_block_contents.size(); + index_size_ += top_level_index_size_; + return Status::OK(); + } else { + // Finish the next partition index in line and Incomplete() to indicate we + // expect more calls to Finish + Entry& entry = entries_.front(); + // Apply the policy to all sub-indexes + entry.value->seperator_is_key_plus_seq_ = seperator_is_key_plus_seq_; + auto s = entry.value->Finish(index_blocks); + index_size_ += index_blocks->index_block_contents.size(); + finishing_indexes = true; + return s.ok() ? Status::Incomplete() : s; + } +} + +size_t PartitionedIndexBuilder::NumPartitions() const { return partition_cnt_; } +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/index_builder.h b/librocksdb-sys/rocksdb/table/block_based/index_builder.h new file mode 100644 index 0000000..5693554 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/index_builder.h @@ -0,0 +1,494 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include +#include +#include +#include + +#include "db/dbformat.h" +#include "rocksdb/comparator.h" +#include "table/block_based/block_based_table_factory.h" +#include "table/block_based/block_builder.h" +#include "table/format.h" + +namespace ROCKSDB_NAMESPACE { +// The interface for building index. +// Instruction for adding a new concrete IndexBuilder: +// 1. Create a subclass instantiated from IndexBuilder. +// 2. Add a new entry associated with that subclass in TableOptions::IndexType. +// 3. Add a create function for the new subclass in CreateIndexBuilder. +// Note: we can devise more advanced design to simplify the process for adding +// new subclass, which will, on the other hand, increase the code complexity and +// catch unwanted attention from readers. Given that we won't add/change +// indexes frequently, it makes sense to just embrace a more straightforward +// design that just works. +class IndexBuilder { + public: + static IndexBuilder* CreateIndexBuilder( + BlockBasedTableOptions::IndexType index_type, + const ROCKSDB_NAMESPACE::InternalKeyComparator* comparator, + const InternalKeySliceTransform* int_key_slice_transform, + bool use_value_delta_encoding, const BlockBasedTableOptions& table_opt, + size_t ts_sz, bool persist_user_defined_timestamps); + + // Index builder will construct a set of blocks which contain: + // 1. One primary index block. + // 2. (Optional) a set of metablocks that contains the metadata of the + // primary index. + struct IndexBlocks { + Slice index_block_contents; + std::unordered_map meta_blocks; + }; + IndexBuilder(const InternalKeyComparator* comparator, size_t ts_sz, + bool persist_user_defined_timestamps) + : comparator_(comparator), + ts_sz_(ts_sz), + persist_user_defined_timestamps_(persist_user_defined_timestamps) {} + + virtual ~IndexBuilder() = default; + + // Add a new index entry to index block. + // To allow further optimization, we provide `last_key_in_current_block` and + // `first_key_in_next_block`, based on which the specific implementation can + // determine the best index key to be used for the index block. + // Called before the OnKeyAdded() call for first_key_in_next_block. + // @last_key_in_current_block: this parameter maybe overridden with the value + // "substitute key". + // @first_key_in_next_block: it will be nullptr if the entry being added is + // the last one in the table + // + // REQUIRES: Finish() has not yet been called. + virtual void AddIndexEntry(std::string* last_key_in_current_block, + const Slice* first_key_in_next_block, + const BlockHandle& block_handle) = 0; + + // This method will be called whenever a key is added. The subclasses may + // override OnKeyAdded() if they need to collect additional information. + virtual void OnKeyAdded(const Slice& /*key*/) {} + + // Inform the index builder that all entries has been written. Block builder + // may therefore perform any operation required for block finalization. + // + // REQUIRES: Finish() has not yet been called. + inline Status Finish(IndexBlocks* index_blocks) { + // Throw away the changes to last_partition_block_handle. It has no effect + // on the first call to Finish anyway. + BlockHandle last_partition_block_handle; + return Finish(index_blocks, last_partition_block_handle); + } + + // This override of Finish can be utilized to build the 2nd level index in + // PartitionIndexBuilder. + // + // index_blocks will be filled with the resulting index data. If the return + // value is Status::InComplete() then it means that the index is partitioned + // and the callee should keep calling Finish until Status::OK() is returned. + // In that case, last_partition_block_handle is pointer to the block written + // with the result of the last call to Finish. This can be utilized to build + // the second level index pointing to each block of partitioned indexes. The + // last call to Finish() that returns Status::OK() populates index_blocks with + // the 2nd level index content. + virtual Status Finish(IndexBlocks* index_blocks, + const BlockHandle& last_partition_block_handle) = 0; + + // Get the size for index block. Must be called after ::Finish. + virtual size_t IndexSize() const = 0; + + virtual bool seperator_is_key_plus_seq() { return true; } + + protected: + const InternalKeyComparator* comparator_; + // Size of user-defined timestamp in bytes. + size_t ts_sz_; + // Whether user-defined timestamp in the user key should be persisted when + // creating index block. If this flag is false, user-defined timestamp will + // be stripped from user key for each index entry, and the + // `first_internal_key` in `IndexValue` if it's included. + bool persist_user_defined_timestamps_; + // Set after ::Finish is called + size_t index_size_ = 0; +}; + +// This index builder builds space-efficient index block. +// +// Optimizations: +// 1. Made block's `block_restart_interval` to be 1, which will avoid linear +// search when doing index lookup (can be disabled by setting +// index_block_restart_interval). +// 2. Shorten the key length for index block. Other than honestly using the +// last key in the data block as the index key, we instead find a shortest +// substitute key that serves the same function. +class ShortenedIndexBuilder : public IndexBuilder { + public: + ShortenedIndexBuilder( + const InternalKeyComparator* comparator, + const int index_block_restart_interval, const uint32_t format_version, + const bool use_value_delta_encoding, + BlockBasedTableOptions::IndexShorteningMode shortening_mode, + bool include_first_key, size_t ts_sz, + const bool persist_user_defined_timestamps) + : IndexBuilder(comparator, ts_sz, persist_user_defined_timestamps), + index_block_builder_( + index_block_restart_interval, true /*use_delta_encoding*/, + use_value_delta_encoding, + BlockBasedTableOptions::kDataBlockBinarySearch /* index_type */, + 0.75 /* data_block_hash_table_util_ratio */, ts_sz, + persist_user_defined_timestamps, false /* is_user_key */), + index_block_builder_without_seq_( + index_block_restart_interval, true /*use_delta_encoding*/, + use_value_delta_encoding, + BlockBasedTableOptions::kDataBlockBinarySearch /* index_type */, + 0.75 /* data_block_hash_table_util_ratio */, ts_sz, + persist_user_defined_timestamps, true /* is_user_key */), + use_value_delta_encoding_(use_value_delta_encoding), + include_first_key_(include_first_key), + shortening_mode_(shortening_mode) { + // Making the default true will disable the feature for old versions + seperator_is_key_plus_seq_ = (format_version <= 2); + } + + void OnKeyAdded(const Slice& key) override { + if (include_first_key_ && current_block_first_internal_key_.empty()) { + current_block_first_internal_key_.assign(key.data(), key.size()); + } + } + + void AddIndexEntry(std::string* last_key_in_current_block, + const Slice* first_key_in_next_block, + const BlockHandle& block_handle) override { + if (first_key_in_next_block != nullptr) { + if (shortening_mode_ != + BlockBasedTableOptions::IndexShorteningMode::kNoShortening) { + FindShortestInternalKeySeparator(*comparator_->user_comparator(), + last_key_in_current_block, + *first_key_in_next_block); + } + if (!seperator_is_key_plus_seq_ && + comparator_->user_comparator()->Compare( + ExtractUserKey(*last_key_in_current_block), + ExtractUserKey(*first_key_in_next_block)) == 0) { + seperator_is_key_plus_seq_ = true; + } + } else { + if (shortening_mode_ == BlockBasedTableOptions::IndexShorteningMode:: + kShortenSeparatorsAndSuccessor) { + FindShortInternalKeySuccessor(*comparator_->user_comparator(), + last_key_in_current_block); + } + } + auto sep = Slice(*last_key_in_current_block); + + assert(!include_first_key_ || !current_block_first_internal_key_.empty()); + // When UDT should not be persisted, the index block builders take care of + // stripping UDT from the key, for the first internal key contained in the + // IndexValue, we need to explicitly do the stripping here before passing + // it to the block builders. + std::string first_internal_key_buf; + Slice first_internal_key = current_block_first_internal_key_; + if (!current_block_first_internal_key_.empty() && ts_sz_ > 0 && + !persist_user_defined_timestamps_) { + StripTimestampFromInternalKey(&first_internal_key_buf, + current_block_first_internal_key_, ts_sz_); + first_internal_key = first_internal_key_buf; + } + IndexValue entry(block_handle, first_internal_key); + std::string encoded_entry; + std::string delta_encoded_entry; + entry.EncodeTo(&encoded_entry, include_first_key_, nullptr); + if (use_value_delta_encoding_ && !last_encoded_handle_.IsNull()) { + entry.EncodeTo(&delta_encoded_entry, include_first_key_, + &last_encoded_handle_); + } else { + // If it's the first block, or delta encoding is disabled, + // BlockBuilder::Add() below won't use delta-encoded slice. + } + last_encoded_handle_ = block_handle; + const Slice delta_encoded_entry_slice(delta_encoded_entry); + + // TODO(yuzhangyu): fix this when "FindShortInternalKeySuccessor" + // optimization is available. + // Timestamp aware comparator currently doesn't provide override for + // "FindShortInternalKeySuccessor" optimization. So the actual + // last key in current block is used as the key for indexing the current + // block. As a result, when UDTs should not be persisted, it's safe to strip + // away the UDT from key in index block as data block does the same thing. + // What are the implications if a "FindShortInternalKeySuccessor" + // optimization is provided. + index_block_builder_.Add(sep, encoded_entry, &delta_encoded_entry_slice); + if (!seperator_is_key_plus_seq_) { + index_block_builder_without_seq_.Add(ExtractUserKey(sep), encoded_entry, + &delta_encoded_entry_slice); + } + + current_block_first_internal_key_.clear(); + } + + using IndexBuilder::Finish; + Status Finish(IndexBlocks* index_blocks, + const BlockHandle& /*last_partition_block_handle*/) override { + if (seperator_is_key_plus_seq_) { + index_blocks->index_block_contents = index_block_builder_.Finish(); + } else { + index_blocks->index_block_contents = + index_block_builder_without_seq_.Finish(); + } + index_size_ = index_blocks->index_block_contents.size(); + return Status::OK(); + } + + size_t IndexSize() const override { return index_size_; } + + bool seperator_is_key_plus_seq() override { + return seperator_is_key_plus_seq_; + } + + // Changes *key to a short string >= *key. + // + static void FindShortestInternalKeySeparator(const Comparator& comparator, + std::string* start, + const Slice& limit); + + static void FindShortInternalKeySuccessor(const Comparator& comparator, + std::string* key); + + friend class PartitionedIndexBuilder; + + private: + BlockBuilder index_block_builder_; + BlockBuilder index_block_builder_without_seq_; + const bool use_value_delta_encoding_; + bool seperator_is_key_plus_seq_; + const bool include_first_key_; + BlockBasedTableOptions::IndexShorteningMode shortening_mode_; + BlockHandle last_encoded_handle_ = BlockHandle::NullBlockHandle(); + std::string current_block_first_internal_key_; +}; + +// HashIndexBuilder contains a binary-searchable primary index and the +// metadata for secondary hash index construction. +// The metadata for hash index consists two parts: +// - a metablock that compactly contains a sequence of prefixes. All prefixes +// are stored consectively without any metadata (like, prefix sizes) being +// stored, which is kept in the other metablock. +// - a metablock contains the metadata of the prefixes, including prefix size, +// restart index and number of block it spans. The format looks like: +// +// +-----------------+---------------------------+---------------------+ +// <=prefix 1 +// | length: 4 bytes | restart interval: 4 bytes | num-blocks: 4 bytes | +// +-----------------+---------------------------+---------------------+ +// <=prefix 2 +// | length: 4 bytes | restart interval: 4 bytes | num-blocks: 4 bytes | +// +-----------------+---------------------------+---------------------+ +// | | +// | .... | +// | | +// +-----------------+---------------------------+---------------------+ +// <=prefix n +// | length: 4 bytes | restart interval: 4 bytes | num-blocks: 4 bytes | +// +-----------------+---------------------------+---------------------+ +// +// The reason of separating these two metablocks is to enable the efficiently +// reuse the first metablock during hash index construction without unnecessary +// data copy or small heap allocations for prefixes. +class HashIndexBuilder : public IndexBuilder { + public: + HashIndexBuilder(const InternalKeyComparator* comparator, + const SliceTransform* hash_key_extractor, + int index_block_restart_interval, int format_version, + bool use_value_delta_encoding, + BlockBasedTableOptions::IndexShorteningMode shortening_mode, + size_t ts_sz, const bool persist_user_defined_timestamps) + : IndexBuilder(comparator, ts_sz, persist_user_defined_timestamps), + primary_index_builder_(comparator, index_block_restart_interval, + format_version, use_value_delta_encoding, + shortening_mode, /* include_first_key */ false, + ts_sz, persist_user_defined_timestamps), + hash_key_extractor_(hash_key_extractor) {} + + void AddIndexEntry(std::string* last_key_in_current_block, + const Slice* first_key_in_next_block, + const BlockHandle& block_handle) override { + ++current_restart_index_; + primary_index_builder_.AddIndexEntry(last_key_in_current_block, + first_key_in_next_block, block_handle); + } + + void OnKeyAdded(const Slice& key) override { + auto key_prefix = hash_key_extractor_->Transform(key); + bool is_first_entry = pending_block_num_ == 0; + + // Keys may share the prefix + if (is_first_entry || pending_entry_prefix_ != key_prefix) { + if (!is_first_entry) { + FlushPendingPrefix(); + } + + // need a hard copy otherwise the underlying data changes all the time. + // TODO(kailiu) std::to_string() is expensive. We may speed up can avoid + // data copy. + pending_entry_prefix_ = key_prefix.ToString(); + pending_block_num_ = 1; + pending_entry_index_ = static_cast(current_restart_index_); + } else { + // entry number increments when keys share the prefix reside in + // different data blocks. + auto last_restart_index = pending_entry_index_ + pending_block_num_ - 1; + assert(last_restart_index <= current_restart_index_); + if (last_restart_index != current_restart_index_) { + ++pending_block_num_; + } + } + } + + Status Finish(IndexBlocks* index_blocks, + const BlockHandle& last_partition_block_handle) override { + if (pending_block_num_ != 0) { + FlushPendingPrefix(); + } + Status s = primary_index_builder_.Finish(index_blocks, + last_partition_block_handle); + index_blocks->meta_blocks.insert( + {kHashIndexPrefixesBlock.c_str(), prefix_block_}); + index_blocks->meta_blocks.insert( + {kHashIndexPrefixesMetadataBlock.c_str(), prefix_meta_block_}); + return s; + } + + size_t IndexSize() const override { + return primary_index_builder_.IndexSize() + prefix_block_.size() + + prefix_meta_block_.size(); + } + + bool seperator_is_key_plus_seq() override { + return primary_index_builder_.seperator_is_key_plus_seq(); + } + + private: + void FlushPendingPrefix() { + prefix_block_.append(pending_entry_prefix_.data(), + pending_entry_prefix_.size()); + PutVarint32Varint32Varint32( + &prefix_meta_block_, + static_cast(pending_entry_prefix_.size()), + pending_entry_index_, pending_block_num_); + } + + ShortenedIndexBuilder primary_index_builder_; + const SliceTransform* hash_key_extractor_; + + // stores a sequence of prefixes + std::string prefix_block_; + // stores the metadata of prefixes + std::string prefix_meta_block_; + + // The following 3 variables keeps unflushed prefix and its metadata. + // The details of block_num and entry_index can be found in + // "block_hash_index.{h,cc}" + uint32_t pending_block_num_ = 0; + uint32_t pending_entry_index_ = 0; + std::string pending_entry_prefix_; + + uint64_t current_restart_index_ = 0; +}; + +/** + * IndexBuilder for two-level indexing. Internally it creates a new index for + * each partition and Finish then in order when Finish is called on it + * continiously until Status::OK() is returned. + * + * The format on the disk would be I I I I I I IP where I is block containing a + * partition of indexes built using ShortenedIndexBuilder and IP is a block + * containing a secondary index on the partitions, built using + * ShortenedIndexBuilder. + */ +class PartitionedIndexBuilder : public IndexBuilder { + public: + static PartitionedIndexBuilder* CreateIndexBuilder( + const ROCKSDB_NAMESPACE::InternalKeyComparator* comparator, + bool use_value_delta_encoding, const BlockBasedTableOptions& table_opt, + size_t ts_sz, bool persist_user_defined_timestamps); + + PartitionedIndexBuilder(const InternalKeyComparator* comparator, + const BlockBasedTableOptions& table_opt, + bool use_value_delta_encoding, size_t ts_sz, + bool persist_user_defined_timestamps); + + ~PartitionedIndexBuilder() override; + + void AddIndexEntry(std::string* last_key_in_current_block, + const Slice* first_key_in_next_block, + const BlockHandle& block_handle) override; + + Status Finish(IndexBlocks* index_blocks, + const BlockHandle& last_partition_block_handle) override; + + size_t IndexSize() const override { return index_size_; } + size_t TopLevelIndexSize(uint64_t) const { return top_level_index_size_; } + size_t NumPartitions() const; + + inline bool ShouldCutFilterBlock() { + // Current policy is to align the partitions of index and filters + if (cut_filter_block) { + cut_filter_block = false; + return true; + } + return false; + } + + std::string& GetPartitionKey() { return sub_index_last_key_; } + + // Called when an external entity (such as filter partition builder) request + // cutting the next partition + void RequestPartitionCut(); + + bool seperator_is_key_plus_seq() override { + return seperator_is_key_plus_seq_; + } + + bool get_use_value_delta_encoding() const { + return use_value_delta_encoding_; + } + + private: + // Set after ::Finish is called + size_t top_level_index_size_ = 0; + // Set after ::Finish is called + size_t partition_cnt_ = 0; + + void MakeNewSubIndexBuilder(); + + struct Entry { + std::string key; + std::unique_ptr value; + }; + std::list entries_; // list of partitioned indexes and their keys + BlockBuilder index_block_builder_; // top-level index builder + BlockBuilder index_block_builder_without_seq_; // same for user keys + // the active partition index builder + ShortenedIndexBuilder* sub_index_builder_; + // the last key in the active partition index builder + std::string sub_index_last_key_; + std::unique_ptr flush_policy_; + // true if Finish is called once but not complete yet. + bool finishing_indexes = false; + const BlockBasedTableOptions& table_opt_; + bool seperator_is_key_plus_seq_; + bool use_value_delta_encoding_; + // true if an external entity (such as filter partition builder) request + // cutting the next partition + bool partition_cut_requested_ = true; + // true if it should cut the next filter partition block + bool cut_filter_block = false; + BlockHandle last_encoded_handle_; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/index_reader_common.cc b/librocksdb-sys/rocksdb/table/block_based/index_reader_common.cc new file mode 100644 index 0000000..e68be2a --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/index_reader_common.cc @@ -0,0 +1,57 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "table/block_based/index_reader_common.h" + +#include "block_cache.h" + +namespace ROCKSDB_NAMESPACE { +Status BlockBasedTable::IndexReaderCommon::ReadIndexBlock( + const BlockBasedTable* table, FilePrefetchBuffer* prefetch_buffer, + const ReadOptions& read_options, bool use_cache, GetContext* get_context, + BlockCacheLookupContext* lookup_context, + CachableEntry* index_block) { + PERF_TIMER_GUARD(read_index_block_nanos); + + assert(table != nullptr); + assert(index_block != nullptr); + assert(index_block->IsEmpty()); + + const Rep* const rep = table->get_rep(); + assert(rep != nullptr); + + const Status s = table->RetrieveBlock( + prefetch_buffer, read_options, rep->index_handle, + UncompressionDict::GetEmptyDict(), &index_block->As(), + get_context, lookup_context, /* for_compaction */ false, use_cache, + /* async_read */ false); + + return s; +} + +Status BlockBasedTable::IndexReaderCommon::GetOrReadIndexBlock( + bool no_io, GetContext* get_context, + BlockCacheLookupContext* lookup_context, CachableEntry* index_block, + const ReadOptions& ro) const { + assert(index_block != nullptr); + + if (!index_block_.IsEmpty()) { + index_block->SetUnownedValue(index_block_.GetValue()); + return Status::OK(); + } + + ReadOptions read_options = ro; + if (no_io) { + read_options.read_tier = kBlockCacheTier; + } + + return ReadIndexBlock(table_, /*prefetch_buffer=*/nullptr, read_options, + cache_index_blocks(), get_context, lookup_context, + index_block); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/index_reader_common.h b/librocksdb-sys/rocksdb/table/block_based/index_reader_common.h new file mode 100644 index 0000000..1aa7cbb --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/index_reader_common.h @@ -0,0 +1,91 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once + +#include "table/block_based/block_based_table_reader.h" +#include "table/block_based/reader_common.h" + +namespace ROCKSDB_NAMESPACE { +// Encapsulates common functionality for the various index reader +// implementations. Provides access to the index block regardless of whether +// it is owned by the reader or stored in the cache, or whether it is pinned +// in the cache or not. +class BlockBasedTable::IndexReaderCommon : public BlockBasedTable::IndexReader { + public: + IndexReaderCommon(const BlockBasedTable* t, + CachableEntry&& index_block) + : table_(t), index_block_(std::move(index_block)) { + assert(table_ != nullptr); + } + + protected: + static Status ReadIndexBlock(const BlockBasedTable* table, + FilePrefetchBuffer* prefetch_buffer, + const ReadOptions& read_options, bool use_cache, + GetContext* get_context, + BlockCacheLookupContext* lookup_context, + CachableEntry* index_block); + + const BlockBasedTable* table() const { return table_; } + + const InternalKeyComparator* internal_comparator() const { + assert(table_ != nullptr); + assert(table_->get_rep() != nullptr); + + return &table_->get_rep()->internal_comparator; + } + + bool index_has_first_key() const { + assert(table_ != nullptr); + assert(table_->get_rep() != nullptr); + return table_->get_rep()->index_has_first_key; + } + + bool index_key_includes_seq() const { + assert(table_ != nullptr); + assert(table_->get_rep() != nullptr); + return table_->get_rep()->index_key_includes_seq; + } + + bool index_value_is_full() const { + assert(table_ != nullptr); + assert(table_->get_rep() != nullptr); + return table_->get_rep()->index_value_is_full; + } + + bool cache_index_blocks() const { + assert(table_ != nullptr); + assert(table_->get_rep() != nullptr); + return table_->get_rep()->table_options.cache_index_and_filter_blocks; + } + + bool user_defined_timestamps_persisted() const { + assert(table_ != nullptr); + assert(table_->get_rep() != nullptr); + return table_->get_rep()->user_defined_timestamps_persisted; + } + + Status GetOrReadIndexBlock(bool no_io, GetContext* get_context, + BlockCacheLookupContext* lookup_context, + CachableEntry* index_block, + const ReadOptions& read_options) const; + + size_t ApproximateIndexBlockMemoryUsage() const { + assert(!index_block_.GetOwnValue() || index_block_.GetValue() != nullptr); + return index_block_.GetOwnValue() + ? index_block_.GetValue()->ApproximateMemoryUsage() + : 0; + } + + private: + const BlockBasedTable* table_; + CachableEntry index_block_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/mock_block_based_table.h b/librocksdb-sys/rocksdb/table/block_based/mock_block_based_table.h new file mode 100644 index 0000000..13f3dfa --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/mock_block_based_table.h @@ -0,0 +1,62 @@ +// Copyright (c) 2019-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include + +#include "rocksdb/filter_policy.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/block_based/filter_policy_internal.h" + +namespace ROCKSDB_NAMESPACE { +namespace mock { + +class MockBlockBasedTable : public BlockBasedTable { + public: + explicit MockBlockBasedTable(Rep* rep) + : BlockBasedTable(rep, nullptr /* block_cache_tracer */) {} +}; + +class MockBlockBasedTableTester { + static constexpr int kMockLevel = 0; + + public: + Options options_; + ImmutableOptions ioptions_; + EnvOptions env_options_; + BlockBasedTableOptions table_options_; + InternalKeyComparator icomp_; + std::unique_ptr table_; + + explicit MockBlockBasedTableTester(const FilterPolicy* filter_policy) + : MockBlockBasedTableTester( + std::shared_ptr(filter_policy)){}; + + explicit MockBlockBasedTableTester( + std::shared_ptr filter_policy) + : ioptions_(options_), + env_options_(options_), + icomp_(options_.comparator) { + table_options_.filter_policy = std::move(filter_policy); + + constexpr bool skip_filters = false; + constexpr bool immortal_table = false; + table_.reset(new MockBlockBasedTable(new BlockBasedTable::Rep( + ioptions_, env_options_, table_options_, icomp_, skip_filters, + 12345 /*file_size*/, kMockLevel, immortal_table))); + } + + FilterBitsBuilder* GetBuilder() const { + FilterBuildingContext context(table_options_); + context.column_family_name = "mock_cf"; + context.compaction_style = ioptions_.compaction_style; + context.level_at_creation = kMockLevel; + context.info_log = ioptions_.logger; + return BloomFilterPolicy::GetBuilderFromContext(context); + } +}; + +} // namespace mock +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/parsed_full_filter_block.cc b/librocksdb-sys/rocksdb/table/block_based/parsed_full_filter_block.cc new file mode 100644 index 0000000..9184a48 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/parsed_full_filter_block.cc @@ -0,0 +1,23 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#include "table/block_based/parsed_full_filter_block.h" + +#include "table/block_based/filter_policy_internal.h" + +namespace ROCKSDB_NAMESPACE { + +ParsedFullFilterBlock::ParsedFullFilterBlock(const FilterPolicy* filter_policy, + BlockContents&& contents) + : block_contents_(std::move(contents)), + filter_bits_reader_( + !block_contents_.data.empty() + ? filter_policy->GetFilterBitsReader(block_contents_.data) + : nullptr) {} + +ParsedFullFilterBlock::~ParsedFullFilterBlock() = default; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/parsed_full_filter_block.h b/librocksdb-sys/rocksdb/table/block_based/parsed_full_filter_block.h new file mode 100644 index 0000000..8d81868 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/parsed_full_filter_block.h @@ -0,0 +1,47 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "table/block_based/block_type.h" +#include "table/format.h" + +namespace ROCKSDB_NAMESPACE { + +class FilterBitsReader; +class FilterPolicy; + +// The sharable/cachable part of the full filter. +class ParsedFullFilterBlock { + public: + ParsedFullFilterBlock(const FilterPolicy* filter_policy, + BlockContents&& contents); + ~ParsedFullFilterBlock(); + + FilterBitsReader* filter_bits_reader() const { + return filter_bits_reader_.get(); + } + + // TODO: consider memory usage of the FilterBitsReader + size_t ApproximateMemoryUsage() const { + return block_contents_.ApproximateMemoryUsage(); + } + + bool own_bytes() const { return block_contents_.own_bytes(); } + + // For TypedCacheInterface + const Slice& ContentSlice() const { return block_contents_.data; } + static constexpr CacheEntryRole kCacheEntryRole = + CacheEntryRole::kFilterBlock; + static constexpr BlockType kBlockType = BlockType::kFilter; + + private: + BlockContents block_contents_; + std::unique_ptr filter_bits_reader_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/partitioned_filter_block.cc b/librocksdb-sys/rocksdb/table/block_based/partitioned_filter_block.cc new file mode 100644 index 0000000..faddaeb --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/partitioned_filter_block.cc @@ -0,0 +1,569 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "table/block_based/partitioned_filter_block.h" + +#include + +#include "block_cache.h" +#include "block_type.h" +#include "file/random_access_file_reader.h" +#include "logging/logging.h" +#include "monitoring/perf_context_imp.h" +#include "port/malloc.h" +#include "port/port.h" +#include "rocksdb/filter_policy.h" +#include "table/block_based/block.h" +#include "table/block_based/block_based_table_reader.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { + +PartitionedFilterBlockBuilder::PartitionedFilterBlockBuilder( + const SliceTransform* _prefix_extractor, bool whole_key_filtering, + FilterBitsBuilder* filter_bits_builder, int index_block_restart_interval, + const bool use_value_delta_encoding, + PartitionedIndexBuilder* const p_index_builder, + const uint32_t partition_size, size_t ts_sz, + const bool persist_user_defined_timestamps) + : FullFilterBlockBuilder(_prefix_extractor, whole_key_filtering, + filter_bits_builder), + index_on_filter_block_builder_( + index_block_restart_interval, true /*use_delta_encoding*/, + use_value_delta_encoding, + BlockBasedTableOptions::kDataBlockBinarySearch /* index_type */, + 0.75 /* data_block_hash_table_util_ratio */, ts_sz, + persist_user_defined_timestamps, false /* is_user_key */), + index_on_filter_block_builder_without_seq_( + index_block_restart_interval, true /*use_delta_encoding*/, + use_value_delta_encoding, + BlockBasedTableOptions::kDataBlockBinarySearch /* index_type */, + 0.75 /* data_block_hash_table_util_ratio */, ts_sz, + persist_user_defined_timestamps, true /* is_user_key */), + p_index_builder_(p_index_builder), + keys_added_to_partition_(0), + total_added_in_built_(0) { + keys_per_partition_ = static_cast( + filter_bits_builder_->ApproximateNumEntries(partition_size)); + if (keys_per_partition_ < 1) { + // partition_size (minus buffer, ~10%) might be smaller than minimum + // filter size, sometimes based on cache line size. Try to find that + // minimum size without CalculateSpace (not necessarily available). + uint32_t larger = std::max(partition_size + 4, uint32_t{16}); + for (;;) { + keys_per_partition_ = static_cast( + filter_bits_builder_->ApproximateNumEntries(larger)); + if (keys_per_partition_ >= 1) { + break; + } + larger += larger / 4; + if (larger > 100000) { + // might be a broken implementation. substitute something reasonable: + // 1 key / byte. + keys_per_partition_ = partition_size; + break; + } + } + } +} + +PartitionedFilterBlockBuilder::~PartitionedFilterBlockBuilder() { + partitioned_filters_construction_status_.PermitUncheckedError(); +} + +void PartitionedFilterBlockBuilder::MaybeCutAFilterBlock( + const Slice* next_key) { + // Use == to send the request only once + if (keys_added_to_partition_ == keys_per_partition_) { + // Currently only index builder is in charge of cutting a partition. We keep + // requesting until it is granted. + p_index_builder_->RequestPartitionCut(); + } + if (!p_index_builder_->ShouldCutFilterBlock()) { + return; + } + + // Add the prefix of the next key before finishing the partition without + // updating last_prefix_str_. This hack, fixes a bug with format_verison=3 + // where seeking for the prefix would lead us to the previous partition. + const bool maybe_add_prefix = + next_key && prefix_extractor() && prefix_extractor()->InDomain(*next_key); + if (maybe_add_prefix) { + const Slice next_key_prefix = prefix_extractor()->Transform(*next_key); + if (next_key_prefix.compare(last_prefix_str()) != 0) { + AddKey(next_key_prefix); + } + } + + total_added_in_built_ += filter_bits_builder_->EstimateEntriesAdded(); + std::unique_ptr filter_data; + Status filter_construction_status = Status::OK(); + Slice filter = + filter_bits_builder_->Finish(&filter_data, &filter_construction_status); + if (filter_construction_status.ok()) { + filter_construction_status = filter_bits_builder_->MaybePostVerify(filter); + } + std::string& index_key = p_index_builder_->GetPartitionKey(); + filters.push_back({index_key, std::move(filter_data), filter}); + if (!filter_construction_status.ok() && + partitioned_filters_construction_status_.ok()) { + partitioned_filters_construction_status_ = filter_construction_status; + } + keys_added_to_partition_ = 0; + Reset(); +} + +void PartitionedFilterBlockBuilder::Add(const Slice& key) { + MaybeCutAFilterBlock(&key); + FullFilterBlockBuilder::Add(key); +} + +void PartitionedFilterBlockBuilder::AddKey(const Slice& key) { + FullFilterBlockBuilder::AddKey(key); + keys_added_to_partition_++; +} + +size_t PartitionedFilterBlockBuilder::EstimateEntriesAdded() { + return total_added_in_built_ + filter_bits_builder_->EstimateEntriesAdded(); +} + +Slice PartitionedFilterBlockBuilder::Finish( + const BlockHandle& last_partition_block_handle, Status* status, + std::unique_ptr* filter_data) { + if (finishing_filters == true) { + // Record the handle of the last written filter block in the index + std::string handle_encoding; + last_partition_block_handle.EncodeTo(&handle_encoding); + std::string handle_delta_encoding; + PutVarsignedint64( + &handle_delta_encoding, + last_partition_block_handle.size() - last_encoded_handle_.size()); + last_encoded_handle_ = last_partition_block_handle; + const Slice handle_delta_encoding_slice(handle_delta_encoding); + index_on_filter_block_builder_.Add(last_filter_entry_key, handle_encoding, + &handle_delta_encoding_slice); + if (!p_index_builder_->seperator_is_key_plus_seq()) { + index_on_filter_block_builder_without_seq_.Add( + ExtractUserKey(last_filter_entry_key), handle_encoding, + &handle_delta_encoding_slice); + } + } else { + MaybeCutAFilterBlock(nullptr); + } + + if (!partitioned_filters_construction_status_.ok()) { + *status = partitioned_filters_construction_status_; + return Slice(); + } + + // If there is no filter partition left, then return the index on filter + // partitions + if (UNLIKELY(filters.empty())) { + *status = Status::OK(); + last_filter_data.reset(); + if (finishing_filters) { + // Simplest to just add them all at the end + total_added_in_built_ = 0; + if (p_index_builder_->seperator_is_key_plus_seq()) { + return index_on_filter_block_builder_.Finish(); + } else { + return index_on_filter_block_builder_without_seq_.Finish(); + } + } else { + // This is the rare case where no key was added to the filter + return Slice(); + } + } else { + // Return the next filter partition in line and set Incomplete() status to + // indicate we expect more calls to Finish + *status = Status::Incomplete(); + finishing_filters = true; + + last_filter_entry_key = filters.front().key; + Slice filter = filters.front().filter; + last_filter_data = std::move(filters.front().filter_data); + if (filter_data != nullptr) { + *filter_data = std::move(last_filter_data); + } + filters.pop_front(); + return filter; + } +} + +PartitionedFilterBlockReader::PartitionedFilterBlockReader( + const BlockBasedTable* t, + CachableEntry&& filter_block) + : FilterBlockReaderCommon(t, std::move(filter_block)) {} + +std::unique_ptr PartitionedFilterBlockReader::Create( + const BlockBasedTable* table, const ReadOptions& ro, + FilePrefetchBuffer* prefetch_buffer, bool use_cache, bool prefetch, + bool pin, BlockCacheLookupContext* lookup_context) { + assert(table); + assert(table->get_rep()); + assert(!pin || prefetch); + + CachableEntry filter_block; + if (prefetch || !use_cache) { + const Status s = ReadFilterBlock(table, prefetch_buffer, ro, use_cache, + nullptr /* get_context */, lookup_context, + &filter_block); + if (!s.ok()) { + IGNORE_STATUS_IF_ERROR(s); + return std::unique_ptr(); + } + + if (use_cache && !pin) { + filter_block.Reset(); + } + } + + return std::unique_ptr( + new PartitionedFilterBlockReader(table, std::move(filter_block))); +} + +bool PartitionedFilterBlockReader::KeyMayMatch( + const Slice& key, const bool no_io, const Slice* const const_ikey_ptr, + GetContext* get_context, BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) { + assert(const_ikey_ptr != nullptr); + if (!whole_key_filtering()) { + return true; + } + + return MayMatch(key, no_io, const_ikey_ptr, get_context, lookup_context, + read_options, &FullFilterBlockReader::KeyMayMatch); +} + +void PartitionedFilterBlockReader::KeysMayMatch( + MultiGetRange* range, const bool no_io, + BlockCacheLookupContext* lookup_context, const ReadOptions& read_options) { + if (!whole_key_filtering()) { + return; // Any/all may match + } + + MayMatch(range, nullptr, no_io, lookup_context, read_options, + &FullFilterBlockReader::KeysMayMatch2); +} + +bool PartitionedFilterBlockReader::PrefixMayMatch( + const Slice& prefix, const bool no_io, const Slice* const const_ikey_ptr, + GetContext* get_context, BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) { + assert(const_ikey_ptr != nullptr); + return MayMatch(prefix, no_io, const_ikey_ptr, get_context, lookup_context, + read_options, &FullFilterBlockReader::PrefixMayMatch); +} + +void PartitionedFilterBlockReader::PrefixesMayMatch( + MultiGetRange* range, const SliceTransform* prefix_extractor, + const bool no_io, BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) { + assert(prefix_extractor); + MayMatch(range, prefix_extractor, no_io, lookup_context, read_options, + &FullFilterBlockReader::PrefixesMayMatch); +} + +BlockHandle PartitionedFilterBlockReader::GetFilterPartitionHandle( + const CachableEntry& filter_block, + const Slice& entry) const { + IndexBlockIter iter; + const InternalKeyComparator* const comparator = internal_comparator(); + Statistics* kNullStats = nullptr; + filter_block.GetValue()->NewIndexIterator( + comparator->user_comparator(), + table()->get_rep()->get_global_seqno(BlockType::kFilterPartitionIndex), + &iter, kNullStats, true /* total_order_seek */, + false /* have_first_key */, index_key_includes_seq(), + index_value_is_full(), false /* block_contents_pinned */, + user_defined_timestamps_persisted()); + iter.Seek(entry); + if (UNLIKELY(!iter.Valid())) { + // entry is larger than all the keys. However its prefix might still be + // present in the last partition. If this is called by PrefixMayMatch this + // is necessary for correct behavior. Otherwise it is unnecessary but safe. + // Assuming this is an unlikely case for full key search, the performance + // overhead should be negligible. + iter.SeekToLast(); + } + assert(iter.Valid()); + BlockHandle fltr_blk_handle = iter.value().handle; + return fltr_blk_handle; +} + +Status PartitionedFilterBlockReader::GetFilterPartitionBlock( + FilePrefetchBuffer* prefetch_buffer, const BlockHandle& fltr_blk_handle, + bool no_io, GetContext* get_context, + BlockCacheLookupContext* lookup_context, const ReadOptions& _read_options, + CachableEntry* filter_block) const { + assert(table()); + assert(filter_block); + assert(filter_block->IsEmpty()); + + if (!filter_map_.empty()) { + auto iter = filter_map_.find(fltr_blk_handle.offset()); + // This is a possible scenario since block cache might not have had space + // for the partition + if (iter != filter_map_.end()) { + filter_block->SetUnownedValue(iter->second.GetValue()); + return Status::OK(); + } + } + + ReadOptions read_options = _read_options; + if (no_io) { + read_options.read_tier = kBlockCacheTier; + } + + const Status s = + table()->RetrieveBlock(prefetch_buffer, read_options, fltr_blk_handle, + UncompressionDict::GetEmptyDict(), filter_block, + get_context, lookup_context, + /* for_compaction */ false, /* use_cache */ true, + /* async_read */ false); + + return s; +} + +bool PartitionedFilterBlockReader::MayMatch( + const Slice& slice, bool no_io, const Slice* const_ikey_ptr, + GetContext* get_context, BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options, FilterFunction filter_function) const { + CachableEntry filter_block; + Status s = GetOrReadFilterBlock(no_io, get_context, lookup_context, + &filter_block, read_options); + if (UNLIKELY(!s.ok())) { + IGNORE_STATUS_IF_ERROR(s); + return true; + } + + if (UNLIKELY(filter_block.GetValue()->size() == 0)) { + return true; + } + + auto filter_handle = GetFilterPartitionHandle(filter_block, *const_ikey_ptr); + if (UNLIKELY(filter_handle.size() == 0)) { // key is out of range + return false; + } + + CachableEntry filter_partition_block; + s = GetFilterPartitionBlock(nullptr /* prefetch_buffer */, filter_handle, + no_io, get_context, lookup_context, read_options, + &filter_partition_block); + if (UNLIKELY(!s.ok())) { + IGNORE_STATUS_IF_ERROR(s); + return true; + } + + FullFilterBlockReader filter_partition(table(), + std::move(filter_partition_block)); + return (filter_partition.*filter_function)( + slice, no_io, const_ikey_ptr, get_context, lookup_context, read_options); +} + +void PartitionedFilterBlockReader::MayMatch( + MultiGetRange* range, const SliceTransform* prefix_extractor, bool no_io, + BlockCacheLookupContext* lookup_context, const ReadOptions& read_options, + FilterManyFunction filter_function) const { + CachableEntry filter_block; + Status s = GetOrReadFilterBlock(no_io, range->begin()->get_context, + lookup_context, &filter_block, read_options); + if (UNLIKELY(!s.ok())) { + IGNORE_STATUS_IF_ERROR(s); + return; // Any/all may match + } + + if (UNLIKELY(filter_block.GetValue()->size() == 0)) { + return; // Any/all may match + } + + auto start_iter_same_handle = range->begin(); + BlockHandle prev_filter_handle = BlockHandle::NullBlockHandle(); + + // For all keys mapping to same partition (must be adjacent in sorted order) + // share block cache lookup and use full filter multiget on the partition + // filter. + for (auto iter = start_iter_same_handle; iter != range->end(); ++iter) { + // TODO: re-use one top-level index iterator + BlockHandle this_filter_handle = + GetFilterPartitionHandle(filter_block, iter->ikey); + if (!prev_filter_handle.IsNull() && + this_filter_handle != prev_filter_handle) { + MultiGetRange subrange(*range, start_iter_same_handle, iter); + MayMatchPartition(&subrange, prefix_extractor, prev_filter_handle, no_io, + lookup_context, read_options, filter_function); + range->AddSkipsFrom(subrange); + start_iter_same_handle = iter; + } + if (UNLIKELY(this_filter_handle.size() == 0)) { // key is out of range + // Not reachable with current behavior of GetFilterPartitionHandle + assert(false); + range->SkipKey(iter); + prev_filter_handle = BlockHandle::NullBlockHandle(); + } else { + prev_filter_handle = this_filter_handle; + } + } + if (!prev_filter_handle.IsNull()) { + MultiGetRange subrange(*range, start_iter_same_handle, range->end()); + MayMatchPartition(&subrange, prefix_extractor, prev_filter_handle, no_io, + lookup_context, read_options, filter_function); + range->AddSkipsFrom(subrange); + } +} + +void PartitionedFilterBlockReader::MayMatchPartition( + MultiGetRange* range, const SliceTransform* prefix_extractor, + BlockHandle filter_handle, bool no_io, + BlockCacheLookupContext* lookup_context, const ReadOptions& read_options, + FilterManyFunction filter_function) const { + CachableEntry filter_partition_block; + Status s = GetFilterPartitionBlock( + nullptr /* prefetch_buffer */, filter_handle, no_io, + range->begin()->get_context, lookup_context, read_options, + &filter_partition_block); + if (UNLIKELY(!s.ok())) { + IGNORE_STATUS_IF_ERROR(s); + return; // Any/all may match + } + + FullFilterBlockReader filter_partition(table(), + std::move(filter_partition_block)); + (filter_partition.*filter_function)(range, prefix_extractor, no_io, + lookup_context, read_options); +} + +size_t PartitionedFilterBlockReader::ApproximateMemoryUsage() const { + size_t usage = ApproximateFilterBlockMemoryUsage(); +#ifdef ROCKSDB_MALLOC_USABLE_SIZE + usage += malloc_usable_size(const_cast(this)); +#else + usage += sizeof(*this); +#endif // ROCKSDB_MALLOC_USABLE_SIZE + return usage; + // TODO(myabandeh): better estimation for filter_map_ size +} + +// TODO(myabandeh): merge this with the same function in IndexReader +Status PartitionedFilterBlockReader::CacheDependencies( + const ReadOptions& ro, bool pin, FilePrefetchBuffer* tail_prefetch_buffer) { + assert(table()); + + const BlockBasedTable::Rep* const rep = table()->get_rep(); + assert(rep); + + BlockCacheLookupContext lookup_context{TableReaderCaller::kPrefetch}; + + CachableEntry filter_block; + + Status s = GetOrReadFilterBlock(false /* no_io */, nullptr /* get_context */, + &lookup_context, &filter_block, ro); + if (!s.ok()) { + ROCKS_LOG_ERROR(rep->ioptions.logger, + "Error retrieving top-level filter block while trying to " + "cache filter partitions: %s", + s.ToString().c_str()); + return s; + } + + // Before read partitions, prefetch them to avoid lots of IOs + assert(filter_block.GetValue()); + + IndexBlockIter biter; + const InternalKeyComparator* const comparator = internal_comparator(); + Statistics* kNullStats = nullptr; + filter_block.GetValue()->NewIndexIterator( + comparator->user_comparator(), + rep->get_global_seqno(BlockType::kFilterPartitionIndex), &biter, + kNullStats, true /* total_order_seek */, false /* have_first_key */, + index_key_includes_seq(), index_value_is_full(), + false /* block_contents_pinned */, user_defined_timestamps_persisted()); + // Index partitions are assumed to be consecuitive. Prefetch them all. + // Read the first block offset + biter.SeekToFirst(); + BlockHandle handle = biter.value().handle; + uint64_t prefetch_off = handle.offset(); + + // Read the last block's offset + biter.SeekToLast(); + handle = biter.value().handle; + uint64_t last_off = + handle.offset() + handle.size() + BlockBasedTable::kBlockTrailerSize; + uint64_t prefetch_len = last_off - prefetch_off; + std::unique_ptr prefetch_buffer; + if (tail_prefetch_buffer == nullptr || !tail_prefetch_buffer->Enabled() || + tail_prefetch_buffer->GetPrefetchOffset() > prefetch_off) { + rep->CreateFilePrefetchBuffer( + 0, 0, &prefetch_buffer, false /* Implicit autoreadahead */, + 0 /*num_reads_*/, 0 /*num_file_reads_for_auto_readahead*/); + + IOOptions opts; + s = rep->file->PrepareIOOptions(ro, opts); + if (s.ok()) { + s = prefetch_buffer->Prefetch(opts, rep->file.get(), prefetch_off, + static_cast(prefetch_len), + ro.rate_limiter_priority); + } + if (!s.ok()) { + return s; + } + } + // After prefetch, read the partitions one by one + for (biter.SeekToFirst(); biter.Valid(); biter.Next()) { + handle = biter.value().handle; + + CachableEntry block; + // TODO: Support counter batch update for partitioned index and + // filter blocks + s = table()->MaybeReadBlockAndLoadToCache( + prefetch_buffer ? prefetch_buffer.get() : tail_prefetch_buffer, ro, + handle, UncompressionDict::GetEmptyDict(), + /* for_compaction */ false, &block, nullptr /* get_context */, + &lookup_context, nullptr /* contents */, false); + if (!s.ok()) { + return s; + } + assert(s.ok() || block.GetValue() == nullptr); + + if (block.GetValue() != nullptr) { + if (block.IsCached()) { + if (pin) { + filter_map_[handle.offset()] = std::move(block); + } + } + } + } + return biter.status(); +} + +const InternalKeyComparator* PartitionedFilterBlockReader::internal_comparator() + const { + assert(table()); + assert(table()->get_rep()); + + return &table()->get_rep()->internal_comparator; +} + +bool PartitionedFilterBlockReader::index_key_includes_seq() const { + assert(table()); + assert(table()->get_rep()); + + return table()->get_rep()->index_key_includes_seq; +} + +bool PartitionedFilterBlockReader::index_value_is_full() const { + assert(table()); + assert(table()->get_rep()); + + return table()->get_rep()->index_value_is_full; +} + +bool PartitionedFilterBlockReader::user_defined_timestamps_persisted() const { + assert(table()); + assert(table()->get_rep()); + + return table()->get_rep()->user_defined_timestamps_persisted; +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/partitioned_filter_block.h b/librocksdb-sys/rocksdb/table/block_based/partitioned_filter_block.h new file mode 100644 index 0000000..817fe94 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/partitioned_filter_block.h @@ -0,0 +1,184 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include +#include + +#include "block_cache.h" +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "rocksdb/slice_transform.h" +#include "table/block_based/block.h" +#include "table/block_based/filter_block_reader_common.h" +#include "table/block_based/full_filter_block.h" +#include "table/block_based/index_builder.h" +#include "util/autovector.h" +#include "util/hash_containers.h" + +namespace ROCKSDB_NAMESPACE { +class InternalKeyComparator; + +class PartitionedFilterBlockBuilder : public FullFilterBlockBuilder { + public: + explicit PartitionedFilterBlockBuilder( + const SliceTransform* prefix_extractor, bool whole_key_filtering, + FilterBitsBuilder* filter_bits_builder, int index_block_restart_interval, + const bool use_value_delta_encoding, + PartitionedIndexBuilder* const p_index_builder, + const uint32_t partition_size, size_t ts_sz, + const bool persist_user_defined_timestamps); + + virtual ~PartitionedFilterBlockBuilder(); + + void AddKey(const Slice& key) override; + void Add(const Slice& key) override; + size_t EstimateEntriesAdded() override; + + virtual Slice Finish( + const BlockHandle& last_partition_block_handle, Status* status, + std::unique_ptr* filter_data = nullptr) override; + + virtual void ResetFilterBitsBuilder() override { + // Previously constructed partitioned filters by + // this to-be-reset FiterBitsBuilder can also be + // cleared + filters.clear(); + FullFilterBlockBuilder::ResetFilterBitsBuilder(); + } + + // For PartitionFilter, optional post-verifing the filter is done + // as part of PartitionFilterBlockBuilder::Finish + // to avoid implementation complexity of doing it elsewhere. + // Therefore we are skipping it in here. + virtual Status MaybePostVerifyFilter( + const Slice& /* filter_content */) override { + return Status::OK(); + } + + private: + // Filter data + BlockBuilder index_on_filter_block_builder_; // top-level index builder + BlockBuilder + index_on_filter_block_builder_without_seq_; // same for user keys + struct FilterEntry { + std::string key; + std::unique_ptr filter_data; + Slice filter; + }; + std::deque filters; // list of partitioned filters and keys used + // in building the index + + // Set to the first non-okay status if any of the filter + // partitions experiences construction error. + // If partitioned_filters_construction_status_ is non-okay, + // then the whole partitioned filters should not be used. + Status partitioned_filters_construction_status_; + std::string last_filter_entry_key; + std::unique_ptr last_filter_data; + std::unique_ptr value; + bool finishing_filters = + false; // true if Finish is called once but not complete yet. + // The policy of when cut a filter block and Finish it + void MaybeCutAFilterBlock(const Slice* next_key); + // Currently we keep the same number of partitions for filters and indexes. + // This would allow for some potentioal optimizations in future. If such + // optimizations did not realize we can use different number of partitions and + // eliminate p_index_builder_ + PartitionedIndexBuilder* const p_index_builder_; + // The desired number of keys per partition + uint32_t keys_per_partition_; + // The number of keys added to the last partition so far + uint32_t keys_added_to_partition_; + // According to the bits builders, how many keys/prefixes added + // in all the filters we have fully built + uint64_t total_added_in_built_; + BlockHandle last_encoded_handle_; +}; + +class PartitionedFilterBlockReader + : public FilterBlockReaderCommon { + public: + PartitionedFilterBlockReader( + const BlockBasedTable* t, + CachableEntry&& filter_block); + + static std::unique_ptr Create( + const BlockBasedTable* table, const ReadOptions& ro, + FilePrefetchBuffer* prefetch_buffer, bool use_cache, bool prefetch, + bool pin, BlockCacheLookupContext* lookup_context); + + bool KeyMayMatch(const Slice& key, const bool no_io, + const Slice* const const_ikey_ptr, GetContext* get_context, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) override; + void KeysMayMatch(MultiGetRange* range, const bool no_io, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) override; + + bool PrefixMayMatch(const Slice& prefix, const bool no_io, + const Slice* const const_ikey_ptr, + GetContext* get_context, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) override; + void PrefixesMayMatch(MultiGetRange* range, + const SliceTransform* prefix_extractor, + const bool no_io, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options) override; + + size_t ApproximateMemoryUsage() const override; + + private: + BlockHandle GetFilterPartitionHandle( + const CachableEntry& filter_block, + const Slice& entry) const; + Status GetFilterPartitionBlock( + FilePrefetchBuffer* prefetch_buffer, const BlockHandle& handle, + bool no_io, GetContext* get_context, + BlockCacheLookupContext* lookup_context, const ReadOptions& read_options, + CachableEntry* filter_block) const; + + using FilterFunction = bool (FullFilterBlockReader::*)( + const Slice& slice, const bool no_io, const Slice* const const_ikey_ptr, + GetContext* get_context, BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options); + bool MayMatch(const Slice& slice, bool no_io, const Slice* const_ikey_ptr, + GetContext* get_context, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options, + FilterFunction filter_function) const; + using FilterManyFunction = void (FullFilterBlockReader::*)( + MultiGetRange* range, const SliceTransform* prefix_extractor, + const bool no_io, BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options); + void MayMatch(MultiGetRange* range, const SliceTransform* prefix_extractor, + bool no_io, BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options, + FilterManyFunction filter_function) const; + void MayMatchPartition(MultiGetRange* range, + const SliceTransform* prefix_extractor, + BlockHandle filter_handle, bool no_io, + BlockCacheLookupContext* lookup_context, + const ReadOptions& read_options, + FilterManyFunction filter_function) const; + Status CacheDependencies(const ReadOptions& ro, bool pin, + FilePrefetchBuffer* tail_prefetch_buffer) override; + + const InternalKeyComparator* internal_comparator() const; + bool index_key_includes_seq() const; + bool index_value_is_full() const; + bool user_defined_timestamps_persisted() const; + + protected: + // For partition blocks pinned in cache. Can be a subset of blocks + // in case some fail insertion on attempt to pin. + UnorderedMap> filter_map_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/partitioned_filter_block_test.cc b/librocksdb-sys/rocksdb/table/block_based/partitioned_filter_block_test.cc new file mode 100644 index 0000000..1d6e2fc --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/partitioned_filter_block_test.cc @@ -0,0 +1,480 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "table/block_based/partitioned_filter_block.h" + +#include + +#include "block_cache.h" +#include "index_builder.h" +#include "rocksdb/filter_policy.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/block_based/filter_policy_internal.h" +#include "table/format.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/coding.h" +#include "util/hash.h" + +namespace ROCKSDB_NAMESPACE { + +std::map blooms; + +class MockedBlockBasedTable : public BlockBasedTable { + public: + MockedBlockBasedTable(Rep* rep, PartitionedIndexBuilder* pib) + : BlockBasedTable(rep, /*block_cache_tracer=*/nullptr) { + // Initialize what Open normally does as much as necessary for the test + rep->index_key_includes_seq = pib->seperator_is_key_plus_seq(); + rep->index_value_is_full = !pib->get_use_value_delta_encoding(); + } +}; + +class MyPartitionedFilterBlockReader : public PartitionedFilterBlockReader { + public: + MyPartitionedFilterBlockReader(BlockBasedTable* t, + CachableEntry&& filter_block) + : PartitionedFilterBlockReader( + t, std::move(filter_block.As())) { + for (const auto& pair : blooms) { + const uint64_t offset = pair.first; + const std::string& bloom = pair.second; + + assert(t); + assert(t->get_rep()); + CachableEntry block( + new ParsedFullFilterBlock( + t->get_rep()->table_options.filter_policy.get(), + BlockContents(Slice(bloom))), + nullptr /* cache */, nullptr /* cache_handle */, + true /* own_value */); + filter_map_[offset] = std::move(block); + } + } +}; + +class PartitionedFilterBlockTest + : public testing::Test, + virtual public ::testing::WithParamInterface< + std::tuple> { + public: + Options options_; + ImmutableOptions ioptions_; + EnvOptions env_options_; + BlockBasedTableOptions table_options_; + InternalKeyComparator icomp_; + std::unique_ptr table_; + std::shared_ptr cache_; + int bits_per_key_; + size_t ts_sz_; + bool user_defined_timestamps_persisted_; + + PartitionedFilterBlockTest() : bits_per_key_(10) { + auto udt_test_mode = std::get<1>(GetParam()); + if (test::IsUDTEnabled(udt_test_mode)) { + options_.comparator = test::BytewiseComparatorWithU64TsWrapper(); + } + ts_sz_ = options_.comparator->timestamp_size(); + user_defined_timestamps_persisted_ = test::ShouldPersistUDT(udt_test_mode); + icomp_ = InternalKeyComparator(options_.comparator); + env_options_ = EnvOptions(options_); + ioptions_ = ImmutableOptions(options_); + table_options_.filter_policy.reset( + NewBloomFilterPolicy(bits_per_key_, false)); + table_options_.format_version = std::get<0>(GetParam()); + table_options_.index_block_restart_interval = 3; + } + + ~PartitionedFilterBlockTest() override {} + + static constexpr int kKeyNum = 4; + static constexpr int kMissingKeyNum = 2; + const std::string keys_without_ts[kKeyNum] = {"afoo", "bar", "box", "hello"}; + const std::string missing_keys_without_ts[kMissingKeyNum] = {"missing", + "other"}; + + std::vector PrepareKeys(const std::string* orig_keys, + int number_of_keys) { + std::vector user_keys; + if (ts_sz_ == 0) { + user_keys.assign(orig_keys, orig_keys + number_of_keys); + } else { + for (int i = 0; i < number_of_keys; i++) { + std::string key_with_ts; + AppendKeyWithMinTimestamp(&key_with_ts, orig_keys[i], ts_sz_); + user_keys.push_back(std::move(key_with_ts)); + } + } + return user_keys; + } + + uint64_t MaxIndexSize() { + uint64_t max_key_size = 0; + for (int i = 0; i < kKeyNum; i++) { + // If UDT is enabled, the size of each key would be increased by a + // timestamp size. + max_key_size = std::max( + max_key_size, static_cast(keys_without_ts[i].size()) + + ts_sz_ * sizeof(static_cast(0))); + } + uint64_t max_index_size = kKeyNum * (max_key_size + 8 /*handle*/); + return max_index_size; + } + + uint64_t MaxFilterSize() { + // General, rough over-approximation + return kKeyNum * bits_per_key_ + (CACHE_LINE_SIZE * 8 + /*metadata*/ 5); + } + + uint64_t last_offset = 10; + BlockHandle Write(const Slice& slice) { + BlockHandle bh(last_offset + 1, slice.size()); + blooms[bh.offset()] = slice.ToString(); + last_offset += bh.size(); + return bh; + } + + PartitionedIndexBuilder* NewIndexBuilder() { + const bool kValueDeltaEncoded = true; + return PartitionedIndexBuilder::CreateIndexBuilder( + &icomp_, !kValueDeltaEncoded, table_options_, ts_sz_, + user_defined_timestamps_persisted_); + } + + PartitionedFilterBlockBuilder* NewBuilder( + PartitionedIndexBuilder* const p_index_builder, + const SliceTransform* prefix_extractor = nullptr) { + assert(table_options_.block_size_deviation <= 100); + auto partition_size = + static_cast(((table_options_.metadata_block_size * + (100 - table_options_.block_size_deviation)) + + 99) / + 100); + partition_size = std::max(partition_size, static_cast(1)); + const bool kValueDeltaEncoded = true; + return new PartitionedFilterBlockBuilder( + prefix_extractor, table_options_.whole_key_filtering, + BloomFilterPolicy::GetBuilderFromContext( + FilterBuildingContext(table_options_)), + table_options_.index_block_restart_interval, !kValueDeltaEncoded, + p_index_builder, partition_size, ts_sz_, + user_defined_timestamps_persisted_); + } + + PartitionedFilterBlockReader* NewReader( + PartitionedFilterBlockBuilder* builder, PartitionedIndexBuilder* pib) { + BlockHandle bh; + Status status; + Slice slice; + std::unique_ptr filter_data; + do { + slice = builder->Finish(bh, &status, &filter_data); + bh = Write(slice); + } while (status.IsIncomplete()); + + constexpr bool skip_filters = false; + constexpr uint64_t file_size = 12345; + constexpr int level = 0; + constexpr bool immortal_table = false; + table_.reset(new MockedBlockBasedTable( + new BlockBasedTable::Rep(ioptions_, env_options_, table_options_, + icomp_, skip_filters, file_size, level, + immortal_table, + user_defined_timestamps_persisted_), + pib)); + BlockContents contents(slice); + CachableEntry block( + new Block(std::move(contents), 0 /* read_amp_bytes_per_bit */, nullptr), + nullptr /* cache */, nullptr /* cache_handle */, true /* own_value */); + auto reader = + new MyPartitionedFilterBlockReader(table_.get(), std::move(block)); + return reader; + } + + void VerifyReader(PartitionedFilterBlockBuilder* builder, + PartitionedIndexBuilder* pib, bool empty = false) { + std::unique_ptr reader( + NewReader(builder, pib)); + // Querying added keys + const bool no_io = true; + std::vector keys = PrepareKeys(keys_without_ts, kKeyNum); + for (auto key : keys) { + auto ikey = InternalKey(key, 0, ValueType::kTypeValue); + const Slice ikey_slice = Slice(*ikey.rep()); + ASSERT_TRUE(reader->KeyMayMatch( + StripTimestampFromUserKey(key, ts_sz_), !no_io, &ikey_slice, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, ReadOptions())); + } + { + // querying a key twice + auto ikey = InternalKey(keys[0], 0, ValueType::kTypeValue); + const Slice ikey_slice = Slice(*ikey.rep()); + ASSERT_TRUE(reader->KeyMayMatch( + StripTimestampFromUserKey(keys[0], ts_sz_), !no_io, &ikey_slice, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, ReadOptions())); + } + // querying missing keys + std::vector missing_keys = + PrepareKeys(missing_keys_without_ts, kMissingKeyNum); + for (auto key : missing_keys) { + auto ikey = InternalKey(key, 0, ValueType::kTypeValue); + const Slice ikey_slice = Slice(*ikey.rep()); + if (empty) { + ASSERT_TRUE(reader->KeyMayMatch( + StripTimestampFromUserKey(key, ts_sz_), !no_io, &ikey_slice, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, ReadOptions())); + } else { + // assuming a good hash function + ASSERT_FALSE(reader->KeyMayMatch( + StripTimestampFromUserKey(key, ts_sz_), !no_io, &ikey_slice, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, ReadOptions())); + } + } + } + + int TestBlockPerKey() { + std::unique_ptr pib(NewIndexBuilder()); + std::unique_ptr builder( + NewBuilder(pib.get())); + int i = 0; + std::vector keys = PrepareKeys(keys_without_ts, kKeyNum); + builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_)); + CutABlock(pib.get(), keys[i], keys[i + 1]); + i++; + builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_)); + CutABlock(pib.get(), keys[i], keys[i + 1]); + i++; + builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_)); + builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_)); + CutABlock(pib.get(), keys[i], keys[i + 1]); + i++; + builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_)); + CutABlock(pib.get(), keys[i]); + + VerifyReader(builder.get(), pib.get()); + return CountNumOfIndexPartitions(pib.get()); + } + + void TestBlockPerTwoKeys(const SliceTransform* prefix_extractor = nullptr) { + std::unique_ptr pib(NewIndexBuilder()); + std::unique_ptr builder( + NewBuilder(pib.get(), prefix_extractor)); + std::vector keys = PrepareKeys(keys_without_ts, kKeyNum); + int i = 0; + builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_)); + i++; + builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_)); + CutABlock(pib.get(), keys[i], keys[i + 1]); + i++; + builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_)); + builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_)); + i++; + builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_)); + CutABlock(pib.get(), keys[i]); + + VerifyReader(builder.get(), pib.get(), prefix_extractor); + } + + void TestBlockPerAllKeys() { + std::unique_ptr pib(NewIndexBuilder()); + std::unique_ptr builder( + NewBuilder(pib.get())); + std::vector keys = PrepareKeys(keys_without_ts, kKeyNum); + int i = 0; + builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_)); + i++; + builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_)); + i++; + builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_)); + builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_)); + i++; + builder->Add(StripTimestampFromUserKey(keys[i], ts_sz_)); + CutABlock(pib.get(), keys[i]); + + VerifyReader(builder.get(), pib.get()); + } + + void CutABlock(PartitionedIndexBuilder* builder, + const std::string& user_key) { + // Assuming a block is cut, add an entry to the index + std::string key = + std::string(*InternalKey(user_key, 0, ValueType::kTypeValue).rep()); + BlockHandle dont_care_block_handle(1, 1); + builder->AddIndexEntry(&key, nullptr, dont_care_block_handle); + } + + void CutABlock(PartitionedIndexBuilder* builder, const std::string& user_key, + const std::string& next_user_key) { + // Assuming a block is cut, add an entry to the index + std::string key = + std::string(*InternalKey(user_key, 0, ValueType::kTypeValue).rep()); + std::string next_key = std::string( + *InternalKey(next_user_key, 0, ValueType::kTypeValue).rep()); + BlockHandle dont_care_block_handle(1, 1); + Slice slice = Slice(next_key.data(), next_key.size()); + builder->AddIndexEntry(&key, &slice, dont_care_block_handle); + } + + int CountNumOfIndexPartitions(PartitionedIndexBuilder* builder) { + IndexBuilder::IndexBlocks dont_care_ib; + BlockHandle dont_care_bh(10, 10); + Status s; + int cnt = 0; + do { + s = builder->Finish(&dont_care_ib, dont_care_bh); + cnt++; + } while (s.IsIncomplete()); + return cnt - 1; // 1 is 2nd level index + } +}; + +// Format versions potentially intersting to partitioning +INSTANTIATE_TEST_CASE_P( + FormatVersions, PartitionedFilterBlockTest, + testing::Combine(testing::ValuesIn(std::set{ + 2, 3, 4, test::kDefaultFormatVersion, + kLatestFormatVersion}), + testing::ValuesIn(test::GetUDTTestModes()))); + +TEST_P(PartitionedFilterBlockTest, EmptyBuilder) { + std::unique_ptr pib(NewIndexBuilder()); + std::unique_ptr builder(NewBuilder(pib.get())); + const bool empty = true; + VerifyReader(builder.get(), pib.get(), empty); +} + +TEST_P(PartitionedFilterBlockTest, OneBlock) { + uint64_t max_index_size = MaxIndexSize(); + for (uint64_t i = 1; i < max_index_size + 1; i++) { + table_options_.metadata_block_size = i; + TestBlockPerAllKeys(); + } +} + +TEST_P(PartitionedFilterBlockTest, TwoBlocksPerKey) { + uint64_t max_index_size = MaxIndexSize(); + for (uint64_t i = 1; i < max_index_size + 1; i++) { + table_options_.metadata_block_size = i; + TestBlockPerTwoKeys(); + } +} + +// This reproduces the bug that a prefix is the same among multiple consecutive +// blocks but the bug would add it only to the first block. +TEST_P(PartitionedFilterBlockTest, SamePrefixInMultipleBlocks) { + // some small number to cause partition cuts + table_options_.metadata_block_size = 1; + std::unique_ptr prefix_extractor( + ROCKSDB_NAMESPACE::NewFixedPrefixTransform(1)); + std::unique_ptr pib(NewIndexBuilder()); + std::unique_ptr builder( + NewBuilder(pib.get(), prefix_extractor.get())); + const std::string pkeys_without_ts[3] = {"p-key10", "p-key20", "p-key30"}; + std::vector pkeys = + PrepareKeys(pkeys_without_ts, 3 /* number_of_keys */); + builder->Add(StripTimestampFromUserKey(pkeys[0], ts_sz_)); + CutABlock(pib.get(), pkeys[0], pkeys[1]); + builder->Add(StripTimestampFromUserKey(pkeys[1], ts_sz_)); + CutABlock(pib.get(), pkeys[1], pkeys[2]); + builder->Add(StripTimestampFromUserKey(pkeys[2], ts_sz_)); + CutABlock(pib.get(), pkeys[2]); + std::unique_ptr reader( + NewReader(builder.get(), pib.get())); + for (auto key : pkeys) { + auto ikey = InternalKey(key, 0, ValueType::kTypeValue); + const Slice ikey_slice = Slice(*ikey.rep()); + ASSERT_TRUE(reader->PrefixMayMatch(prefix_extractor->Transform(key), + /*no_io=*/false, &ikey_slice, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, + ReadOptions())); + } + // Non-existent keys but with the same prefix + const std::string pnonkeys_without_ts[4] = {"p-key9", "p-key11", "p-key21", + "p-key31"}; + std::vector pnonkeys = + PrepareKeys(pnonkeys_without_ts, 4 /* number_of_keys */); + for (auto key : pnonkeys) { + auto ikey = InternalKey(key, 0, ValueType::kTypeValue); + const Slice ikey_slice = Slice(*ikey.rep()); + ASSERT_TRUE(reader->PrefixMayMatch(prefix_extractor->Transform(key), + /*no_io=*/false, &ikey_slice, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, + ReadOptions())); + } +} + +// This reproduces the bug in format_version=3 that the seeking the prefix will +// lead us to the partition before the one that has filter for the prefix. +TEST_P(PartitionedFilterBlockTest, PrefixInWrongPartitionBug) { + // some small number to cause partition cuts + table_options_.metadata_block_size = 1; + std::unique_ptr prefix_extractor( + ROCKSDB_NAMESPACE::NewFixedPrefixTransform(2)); + std::unique_ptr pib(NewIndexBuilder()); + std::unique_ptr builder( + NewBuilder(pib.get(), prefix_extractor.get())); + // In the bug, searching for prefix "p3" on an index with format version 3, + // will give the key "p3" and the partition of the keys that are <= p3, i.e., + // p2-keys, where the filter for prefix "p3" does not exist. + const std::string pkeys_without_ts[] = {"p1-key1", "p2-key2", "p3-key3", + "p4-key3", "p5-key3"}; + std::vector pkeys = + PrepareKeys(pkeys_without_ts, 5 /* number_of_keys */); + builder->Add(StripTimestampFromUserKey(pkeys[0], ts_sz_)); + CutABlock(pib.get(), pkeys[0], pkeys[1]); + builder->Add(StripTimestampFromUserKey(pkeys[1], ts_sz_)); + CutABlock(pib.get(), pkeys[1], pkeys[2]); + builder->Add(StripTimestampFromUserKey(pkeys[2], ts_sz_)); + CutABlock(pib.get(), pkeys[2], pkeys[3]); + builder->Add(StripTimestampFromUserKey(pkeys[3], ts_sz_)); + CutABlock(pib.get(), pkeys[3], pkeys[4]); + builder->Add(StripTimestampFromUserKey(pkeys[4], ts_sz_)); + CutABlock(pib.get(), pkeys[4]); + std::unique_ptr reader( + NewReader(builder.get(), pib.get())); + for (auto key : pkeys) { + auto prefix = prefix_extractor->Transform(key); + auto ikey = InternalKey(key, 0, ValueType::kTypeValue); + const Slice ikey_slice = Slice(*ikey.rep()); + ASSERT_TRUE(reader->PrefixMayMatch(prefix, + /*no_io=*/false, &ikey_slice, + /*get_context=*/nullptr, + /*lookup_context=*/nullptr, + ReadOptions())); + } +} + +TEST_P(PartitionedFilterBlockTest, OneBlockPerKey) { + uint64_t max_index_size = MaxIndexSize(); + for (uint64_t i = 1; i < max_index_size + 1; i++) { + table_options_.metadata_block_size = i; + TestBlockPerKey(); + } +} + +TEST_P(PartitionedFilterBlockTest, PartitionCount) { + table_options_.metadata_block_size = + std::max(MaxIndexSize(), MaxFilterSize()); + int partitions = TestBlockPerKey(); + ASSERT_EQ(partitions, 1); + // A low number ensures cutting a block after each key + table_options_.metadata_block_size = 1; + partitions = TestBlockPerKey(); + ASSERT_EQ(partitions, kKeyNum - 1 /* last two keys make one flush */); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/librocksdb-sys/rocksdb/table/block_based/partitioned_index_iterator.cc b/librocksdb-sys/rocksdb/table/block_based/partitioned_index_iterator.cc new file mode 100644 index 0000000..b9bc215 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/partitioned_index_iterator.cc @@ -0,0 +1,163 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "table/block_based/partitioned_index_iterator.h" + +namespace ROCKSDB_NAMESPACE { +void PartitionedIndexIterator::Seek(const Slice& target) { SeekImpl(&target); } + +void PartitionedIndexIterator::SeekToFirst() { SeekImpl(nullptr); } + +void PartitionedIndexIterator::SeekImpl(const Slice* target) { + SavePrevIndexValue(); + + if (target) { + index_iter_->Seek(*target); + } else { + index_iter_->SeekToFirst(); + } + + if (!index_iter_->Valid()) { + ResetPartitionedIndexIter(); + return; + } + + InitPartitionedIndexBlock(); + + if (target) { + block_iter_.Seek(*target); + } else { + block_iter_.SeekToFirst(); + } + FindKeyForward(); + + // We could check upper bound here, but that would be too complicated + // and checking index upper bound is less useful than for data blocks. + + if (target) { + assert(!Valid() || (table_->get_rep()->index_key_includes_seq + ? (icomp_.Compare(*target, key()) <= 0) + : (user_comparator_.Compare(ExtractUserKey(*target), + key()) <= 0))); + } +} + +void PartitionedIndexIterator::SeekToLast() { + SavePrevIndexValue(); + index_iter_->SeekToLast(); + if (!index_iter_->Valid()) { + ResetPartitionedIndexIter(); + return; + } + InitPartitionedIndexBlock(); + block_iter_.SeekToLast(); + FindKeyBackward(); +} + +void PartitionedIndexIterator::Next() { + assert(block_iter_points_to_real_block_); + block_iter_.Next(); + FindKeyForward(); +} + +void PartitionedIndexIterator::Prev() { + assert(block_iter_points_to_real_block_); + block_iter_.Prev(); + + FindKeyBackward(); +} + +void PartitionedIndexIterator::InitPartitionedIndexBlock() { + BlockHandle partitioned_index_handle = index_iter_->value().handle; + if (!block_iter_points_to_real_block_ || + partitioned_index_handle.offset() != prev_block_offset_ || + // if previous attempt of reading the block missed cache, try again + block_iter_.status().IsIncomplete()) { + if (block_iter_points_to_real_block_) { + ResetPartitionedIndexIter(); + } + auto* rep = table_->get_rep(); + bool is_for_compaction = + lookup_context_.caller == TableReaderCaller::kCompaction; + // Prefetch additional data for range scans (iterators). + // Implicit auto readahead: + // Enabled after 2 sequential IOs when ReadOptions.readahead_size == 0. + // Explicit user requested readahead: + // Enabled from the very first IO when ReadOptions.readahead_size is set. + block_prefetcher_.PrefetchIfNeeded( + rep, partitioned_index_handle, read_options_.readahead_size, + is_for_compaction, /*no_sequential_checking=*/false, + read_options_.rate_limiter_priority); + Status s; + table_->NewDataBlockIterator( + read_options_, partitioned_index_handle, &block_iter_, + BlockType::kIndex, + /*get_context=*/nullptr, &lookup_context_, + block_prefetcher_.prefetch_buffer(), + /*for_compaction=*/is_for_compaction, /*async_read=*/false, s); + block_iter_points_to_real_block_ = true; + // We could check upper bound here but it is complicated to reason about + // upper bound in index iterator. On the other than, in large scans, index + // iterators are moved much less frequently compared to data blocks. So + // the upper bound check is skipped for simplicity. + } +} + +void PartitionedIndexIterator::FindKeyForward() { + // This method's code is kept short to make it likely to be inlined. + + assert(block_iter_points_to_real_block_); + + if (!block_iter_.Valid()) { + // This is the only call site of FindBlockForward(), but it's extracted into + // a separate method to keep FindKeyForward() short and likely to be + // inlined. When transitioning to a different block, we call + // FindBlockForward(), which is much longer and is probably not inlined. + FindBlockForward(); + } else { + // This is the fast path that avoids a function call. + } +} + +void PartitionedIndexIterator::FindBlockForward() { + // TODO the while loop inherits from two-level-iterator. We don't know + // whether a block can be empty so it can be replaced by an "if". + do { + if (!block_iter_.status().ok()) { + return; + } + ResetPartitionedIndexIter(); + index_iter_->Next(); + + if (!index_iter_->Valid()) { + return; + } + + InitPartitionedIndexBlock(); + block_iter_.SeekToFirst(); + } while (!block_iter_.Valid()); +} + +void PartitionedIndexIterator::FindKeyBackward() { + while (!block_iter_.Valid()) { + if (!block_iter_.status().ok()) { + return; + } + + ResetPartitionedIndexIter(); + index_iter_->Prev(); + + if (index_iter_->Valid()) { + InitPartitionedIndexBlock(); + block_iter_.SeekToLast(); + } else { + return; + } + } +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/partitioned_index_iterator.h b/librocksdb-sys/rocksdb/table/block_based/partitioned_index_iterator.h new file mode 100644 index 0000000..6412fe2 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/partitioned_index_iterator.h @@ -0,0 +1,160 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once +#include "table/block_based/block_based_table_reader.h" +#include "table/block_based/block_based_table_reader_impl.h" +#include "table/block_based/block_prefetcher.h" +#include "table/block_based/reader_common.h" + +namespace ROCKSDB_NAMESPACE { +// Iterator that iterates over partitioned index. +// Some upper and lower bound tricks played in block based table iterators +// could be played here, but it's too complicated to reason about index +// keys with upper or lower bound, so we skip it for simplicity. +class PartitionedIndexIterator : public InternalIteratorBase { + // compaction_readahead_size: its value will only be used if for_compaction = + // true + public: + PartitionedIndexIterator( + const BlockBasedTable* table, const ReadOptions& read_options, + const InternalKeyComparator& icomp, + std::unique_ptr>&& index_iter, + TableReaderCaller caller, size_t compaction_readahead_size = 0) + : index_iter_(std::move(index_iter)), + table_(table), + read_options_(read_options), +#ifndef NDEBUG + icomp_(icomp), +#endif + user_comparator_(icomp.user_comparator()), + block_iter_points_to_real_block_(false), + lookup_context_(caller), + block_prefetcher_( + compaction_readahead_size, + table_->get_rep()->table_options.initial_auto_readahead_size) { + } + + ~PartitionedIndexIterator() override {} + + void Seek(const Slice& target) override; + void SeekForPrev(const Slice&) override { + // Shouldn't be called. + assert(false); + } + void SeekToFirst() override; + void SeekToLast() override; + void Next() final override; + bool NextAndGetResult(IterateResult*) override { + assert(false); + return false; + } + void Prev() override; + bool Valid() const override { + return block_iter_points_to_real_block_ && block_iter_.Valid(); + } + Slice key() const override { + assert(Valid()); + return block_iter_.key(); + } + Slice user_key() const override { + assert(Valid()); + return block_iter_.user_key(); + } + IndexValue value() const override { + assert(Valid()); + return block_iter_.value(); + } + Status status() const override { + // Prefix index set status to NotFound when the prefix does not exist + if (!index_iter_->status().ok() && !index_iter_->status().IsNotFound()) { + return index_iter_->status(); + } else if (block_iter_points_to_real_block_) { + return block_iter_.status(); + } else { + return Status::OK(); + } + } + inline IterBoundCheck UpperBoundCheckResult() override { + // Shouldn't be called. + assert(false); + return IterBoundCheck::kUnknown; + } + void SetPinnedItersMgr(PinnedIteratorsManager*) override { + // Shouldn't be called. + assert(false); + } + bool IsKeyPinned() const override { + // Shouldn't be called. + assert(false); + return false; + } + bool IsValuePinned() const override { + // Shouldn't be called. + assert(false); + return false; + } + + void ResetPartitionedIndexIter() { + if (block_iter_points_to_real_block_) { + block_iter_.Invalidate(Status::OK()); + block_iter_points_to_real_block_ = false; + } + } + + void SavePrevIndexValue() { + if (block_iter_points_to_real_block_) { + // Reseek. If they end up with the same data block, we shouldn't re-fetch + // the same data block. + prev_block_offset_ = index_iter_->value().handle.offset(); + } + } + + void GetReadaheadState(ReadaheadFileInfo* readahead_file_info) override { + if (block_prefetcher_.prefetch_buffer() != nullptr && + read_options_.adaptive_readahead) { + block_prefetcher_.prefetch_buffer()->GetReadaheadState( + &(readahead_file_info->index_block_readahead_info)); + } + } + + void SetReadaheadState(ReadaheadFileInfo* readahead_file_info) override { + if (read_options_.adaptive_readahead) { + block_prefetcher_.SetReadaheadState( + &(readahead_file_info->index_block_readahead_info)); + } + } + + std::unique_ptr> index_iter_; + + private: + friend class BlockBasedTableReaderTestVerifyChecksum_ChecksumMismatch_Test; + const BlockBasedTable* table_; + const ReadOptions read_options_; +#ifndef NDEBUG + const InternalKeyComparator& icomp_; +#endif + UserComparatorWrapper user_comparator_; + IndexBlockIter block_iter_; + + // True if block_iter_ is initialized and points to the same block + // as index iterator. + bool block_iter_points_to_real_block_; + uint64_t prev_block_offset_ = std::numeric_limits::max(); + BlockCacheLookupContext lookup_context_; + BlockPrefetcher block_prefetcher_; + + // If `target` is null, seek to first. + void SeekImpl(const Slice* target); + + void InitPartitionedIndexBlock(); + void FindKeyForward(); + void FindBlockForward(); + void FindKeyBackward(); +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/partitioned_index_reader.cc b/librocksdb-sys/rocksdb/table/block_based/partitioned_index_reader.cc new file mode 100644 index 0000000..d1c5591 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/partitioned_index_reader.cc @@ -0,0 +1,227 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "table/block_based/partitioned_index_reader.h" + +#include "block_cache.h" +#include "file/random_access_file_reader.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/block_based/partitioned_index_iterator.h" + +namespace ROCKSDB_NAMESPACE { +Status PartitionIndexReader::Create( + const BlockBasedTable* table, const ReadOptions& ro, + FilePrefetchBuffer* prefetch_buffer, bool use_cache, bool prefetch, + bool pin, BlockCacheLookupContext* lookup_context, + std::unique_ptr* index_reader) { + assert(table != nullptr); + assert(table->get_rep()); + assert(!pin || prefetch); + assert(index_reader != nullptr); + + CachableEntry index_block; + if (prefetch || !use_cache) { + const Status s = + ReadIndexBlock(table, prefetch_buffer, ro, use_cache, + /*get_context=*/nullptr, lookup_context, &index_block); + if (!s.ok()) { + return s; + } + + if (use_cache && !pin) { + index_block.Reset(); + } + } + + index_reader->reset(new PartitionIndexReader(table, std::move(index_block))); + + return Status::OK(); +} + +InternalIteratorBase* PartitionIndexReader::NewIterator( + const ReadOptions& read_options, bool /* disable_prefix_seek */, + IndexBlockIter* iter, GetContext* get_context, + BlockCacheLookupContext* lookup_context) { + const bool no_io = (read_options.read_tier == kBlockCacheTier); + CachableEntry index_block; + const Status s = GetOrReadIndexBlock(no_io, get_context, lookup_context, + &index_block, read_options); + if (!s.ok()) { + if (iter != nullptr) { + iter->Invalidate(s); + return iter; + } + + return NewErrorInternalIterator(s); + } + + const BlockBasedTable::Rep* rep = table()->rep_; + InternalIteratorBase* it = nullptr; + + Statistics* kNullStats = nullptr; + // Filters are already checked before seeking the index + if (!partition_map_.empty()) { + // We don't return pinned data from index blocks, so no need + // to set `block_contents_pinned`. + it = NewTwoLevelIterator( + new BlockBasedTable::PartitionedIndexIteratorState(table(), + &partition_map_), + index_block.GetValue()->NewIndexIterator( + internal_comparator()->user_comparator(), + rep->get_global_seqno(BlockType::kIndex), nullptr, kNullStats, true, + index_has_first_key(), index_key_includes_seq(), + index_value_is_full(), false /* block_contents_pinned */, + user_defined_timestamps_persisted())); + } else { + ReadOptions ro; + ro.fill_cache = read_options.fill_cache; + ro.deadline = read_options.deadline; + ro.io_timeout = read_options.io_timeout; + ro.adaptive_readahead = read_options.adaptive_readahead; + ro.async_io = read_options.async_io; + ro.rate_limiter_priority = read_options.rate_limiter_priority; + ro.verify_checksums = read_options.verify_checksums; + ro.io_activity = read_options.io_activity; + + // We don't return pinned data from index blocks, so no need + // to set `block_contents_pinned`. + std::unique_ptr> index_iter( + index_block.GetValue()->NewIndexIterator( + internal_comparator()->user_comparator(), + rep->get_global_seqno(BlockType::kIndex), nullptr, kNullStats, true, + index_has_first_key(), index_key_includes_seq(), + index_value_is_full(), false /* block_contents_pinned */, + user_defined_timestamps_persisted())); + + it = new PartitionedIndexIterator( + table(), ro, *internal_comparator(), std::move(index_iter), + lookup_context ? lookup_context->caller + : TableReaderCaller::kUncategorized); + } + + assert(it != nullptr); + index_block.TransferTo(it); + + return it; + + // TODO(myabandeh): Update TwoLevelIterator to be able to make use of + // on-stack BlockIter while the state is on heap. Currentlly it assumes + // the first level iter is always on heap and will attempt to delete it + // in its destructor. +} +Status PartitionIndexReader::CacheDependencies( + const ReadOptions& ro, bool pin, FilePrefetchBuffer* tail_prefetch_buffer) { + if (!partition_map_.empty()) { + // The dependencies are already cached since `partition_map_` is filled in + // an all-or-nothing manner. + return Status::OK(); + } + // Before read partitions, prefetch them to avoid lots of IOs + BlockCacheLookupContext lookup_context{TableReaderCaller::kPrefetch}; + const BlockBasedTable::Rep* rep = table()->rep_; + IndexBlockIter biter; + BlockHandle handle; + Statistics* kNullStats = nullptr; + + CachableEntry index_block; + { + Status s = GetOrReadIndexBlock(false /* no_io */, nullptr /* get_context */, + &lookup_context, &index_block, ro); + if (!s.ok()) { + return s; + } + } + + // We don't return pinned data from index blocks, so no need + // to set `block_contents_pinned`. + index_block.GetValue()->NewIndexIterator( + internal_comparator()->user_comparator(), + rep->get_global_seqno(BlockType::kIndex), &biter, kNullStats, true, + index_has_first_key(), index_key_includes_seq(), index_value_is_full(), + false /* block_contents_pinned */, user_defined_timestamps_persisted()); + // Index partitions are assumed to be consecuitive. Prefetch them all. + // Read the first block offset + biter.SeekToFirst(); + if (!biter.Valid()) { + // Empty index. + return biter.status(); + } + handle = biter.value().handle; + uint64_t prefetch_off = handle.offset(); + + // Read the last block's offset + biter.SeekToLast(); + if (!biter.Valid()) { + // Empty index. + return biter.status(); + } + handle = biter.value().handle; + uint64_t last_off = + handle.offset() + BlockBasedTable::BlockSizeWithTrailer(handle); + uint64_t prefetch_len = last_off - prefetch_off; + std::unique_ptr prefetch_buffer; + if (tail_prefetch_buffer == nullptr || !tail_prefetch_buffer->Enabled() || + tail_prefetch_buffer->GetPrefetchOffset() > prefetch_off) { + rep->CreateFilePrefetchBuffer( + 0, 0, &prefetch_buffer, false /*Implicit auto readahead*/, + 0 /*num_reads_*/, 0 /*num_file_reads_for_auto_readahead*/); + IOOptions opts; + { + Status s = rep->file->PrepareIOOptions(ro, opts); + if (s.ok()) { + s = prefetch_buffer->Prefetch(opts, rep->file.get(), prefetch_off, + static_cast(prefetch_len), + ro.rate_limiter_priority); + } + if (!s.ok()) { + return s; + } + } + } + // For saving "all or nothing" to partition_map_ + UnorderedMap> map_in_progress; + + // After prefetch, read the partitions one by one + biter.SeekToFirst(); + size_t partition_count = 0; + for (; biter.Valid(); biter.Next()) { + handle = biter.value().handle; + CachableEntry block; + ++partition_count; + // TODO: Support counter batch update for partitioned index and + // filter blocks + Status s = table()->MaybeReadBlockAndLoadToCache( + prefetch_buffer ? prefetch_buffer.get() : tail_prefetch_buffer, ro, + handle, UncompressionDict::GetEmptyDict(), + /*for_compaction=*/false, &block.As(), + /*get_context=*/nullptr, &lookup_context, /*contents=*/nullptr, + /*async_read=*/false); + + if (!s.ok()) { + return s; + } + if (block.GetValue() != nullptr) { + // Might need to "pin" some mmap-read blocks (GetOwnValue) if some + // partitions are successfully compressed (cached) and some are not + // compressed (mmap eligible) + if (block.IsCached() || block.GetOwnValue()) { + if (pin) { + map_in_progress[handle.offset()] = std::move(block); + } + } + } + } + Status s = biter.status(); + // Save (pin) them only if everything checks out + if (map_in_progress.size() == partition_count && s.ok()) { + std::swap(partition_map_, map_in_progress); + } + return s; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/partitioned_index_reader.h b/librocksdb-sys/rocksdb/table/block_based/partitioned_index_reader.h new file mode 100644 index 0000000..9482fd6 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/partitioned_index_reader.h @@ -0,0 +1,56 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once +#include "table/block_based/index_reader_common.h" +#include "util/hash_containers.h" + +namespace ROCKSDB_NAMESPACE { +// Index that allows binary search lookup in a two-level index structure. +class PartitionIndexReader : public BlockBasedTable::IndexReaderCommon { + public: + // Read the partition index from the file and create an instance for + // `PartitionIndexReader`. + // On success, index_reader will be populated; otherwise it will remain + // unmodified. + static Status Create(const BlockBasedTable* table, const ReadOptions& ro, + FilePrefetchBuffer* prefetch_buffer, bool use_cache, + bool prefetch, bool pin, + BlockCacheLookupContext* lookup_context, + std::unique_ptr* index_reader); + + // return a two-level iterator: first level is on the partition index + InternalIteratorBase* NewIterator( + const ReadOptions& read_options, bool /* disable_prefix_seek */, + IndexBlockIter* iter, GetContext* get_context, + BlockCacheLookupContext* lookup_context) override; + + Status CacheDependencies(const ReadOptions& ro, bool pin, + FilePrefetchBuffer* tail_prefetch_buffer) override; + size_t ApproximateMemoryUsage() const override { + size_t usage = ApproximateIndexBlockMemoryUsage(); +#ifdef ROCKSDB_MALLOC_USABLE_SIZE + usage += malloc_usable_size(const_cast(this)); +#else + usage += sizeof(*this); +#endif // ROCKSDB_MALLOC_USABLE_SIZE + // TODO(myabandeh): more accurate estimate of partition_map_ mem usage + return usage; + } + + private: + PartitionIndexReader(const BlockBasedTable* t, + CachableEntry&& index_block) + : IndexReaderCommon(t, std::move(index_block)) {} + + // For partition blocks pinned in cache. This is expected to be "all or + // none" so that !partition_map_.empty() can use an iterator expecting + // all partitions to be saved here. + UnorderedMap> partition_map_; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/reader_common.cc b/librocksdb-sys/rocksdb/table/block_based/reader_common.cc new file mode 100644 index 0000000..7d0c97c --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/reader_common.cc @@ -0,0 +1,64 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "table/block_based/reader_common.h" + +#include "monitoring/perf_context_imp.h" +#include "rocksdb/table.h" +#include "table/format.h" +#include "util/coding.h" +#include "util/crc32c.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { +void ForceReleaseCachedEntry(void* arg, void* h) { + Cache* cache = reinterpret_cast(arg); + Cache::Handle* handle = reinterpret_cast(h); + cache->Release(handle, true /* erase_if_last_ref */); +} + +// WART: this is specific to block-based table +Status VerifyBlockChecksum(const Footer& footer, const char* data, + size_t block_size, const std::string& file_name, + uint64_t offset) { + PERF_TIMER_GUARD(block_checksum_time); + + assert(footer.GetBlockTrailerSize() == 5); + ChecksumType type = footer.checksum_type(); + + // After block_size bytes is compression type (1 byte), which is part of + // the checksummed section. + size_t len = block_size + 1; + // And then the stored checksum value (4 bytes). + uint32_t stored = DecodeFixed32(data + len); + + uint32_t computed = ComputeBuiltinChecksum(type, data, len); + + // Unapply context to 'stored' rather than apply to 'computed, for people + // who might look for reference crc value in error message + uint32_t modifier = + ChecksumModifierForContext(footer.base_context_checksum(), offset); + stored -= modifier; + + if (stored == computed) { + return Status::OK(); + } else { + // Unmask for people who might look for reference crc value + if (type == kCRC32c) { + stored = crc32c::Unmask(stored); + computed = crc32c::Unmask(computed); + } + return Status::Corruption( + "block checksum mismatch: stored" + + std::string(modifier ? "(context removed)" : "") + " = " + + std::to_string(stored) + ", computed = " + std::to_string(computed) + + ", type = " + std::to_string(type) + " in " + file_name + " offset " + + std::to_string(offset) + " size " + std::to_string(block_size)); + } +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/reader_common.h b/librocksdb-sys/rocksdb/table/block_based/reader_common.h new file mode 100644 index 0000000..08c2a75 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/reader_common.h @@ -0,0 +1,37 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once + +#include "rocksdb/advanced_cache.h" +#include "rocksdb/table.h" + +namespace ROCKSDB_NAMESPACE { +class Footer; + +// Release the cached entry and decrement its ref count. +extern void ForceReleaseCachedEntry(void* arg, void* h); + +inline MemoryAllocator* GetMemoryAllocator( + const BlockBasedTableOptions& table_options) { + return table_options.block_cache.get() + ? table_options.block_cache->memory_allocator() + : nullptr; +} + +// Assumes block has a trailer past `data + block_size` as in format.h. +// `file_name` provided for generating diagnostic message in returned status. +// `offset` might be required for proper verification (also used for message). +// +// Returns Status::OK() on checksum match, or Status::Corruption() on checksum +// mismatch. +extern Status VerifyBlockChecksum(const Footer& footer, const char* data, + size_t block_size, + const std::string& file_name, + uint64_t offset); +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/uncompression_dict_reader.cc b/librocksdb-sys/rocksdb/table/block_based/uncompression_dict_reader.cc new file mode 100644 index 0000000..4ac442b --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/uncompression_dict_reader.cc @@ -0,0 +1,126 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#include "table/block_based/uncompression_dict_reader.h" + +#include "logging/logging.h" +#include "monitoring/perf_context_imp.h" +#include "table/block_based/block_based_table_reader.h" +#include "util/compression.h" + +namespace ROCKSDB_NAMESPACE { + +Status UncompressionDictReader::Create( + const BlockBasedTable* table, const ReadOptions& ro, + FilePrefetchBuffer* prefetch_buffer, bool use_cache, bool prefetch, + bool pin, BlockCacheLookupContext* lookup_context, + std::unique_ptr* uncompression_dict_reader) { + assert(table); + assert(table->get_rep()); + assert(!pin || prefetch); + assert(uncompression_dict_reader); + + CachableEntry uncompression_dict; + if (prefetch || !use_cache) { + const Status s = ReadUncompressionDictionary( + table, prefetch_buffer, ro, use_cache, nullptr /* get_context */, + lookup_context, &uncompression_dict); + if (!s.ok()) { + return s; + } + + if (use_cache && !pin) { + uncompression_dict.Reset(); + } + } + + uncompression_dict_reader->reset( + new UncompressionDictReader(table, std::move(uncompression_dict))); + + return Status::OK(); +} + +Status UncompressionDictReader::ReadUncompressionDictionary( + const BlockBasedTable* table, FilePrefetchBuffer* prefetch_buffer, + const ReadOptions& read_options, bool use_cache, GetContext* get_context, + BlockCacheLookupContext* lookup_context, + CachableEntry* uncompression_dict) { + // TODO: add perf counter for compression dictionary read time + + assert(table); + assert(uncompression_dict); + assert(uncompression_dict->IsEmpty()); + + const BlockBasedTable::Rep* const rep = table->get_rep(); + assert(rep); + assert(!rep->compression_dict_handle.IsNull()); + + const Status s = table->RetrieveBlock( + prefetch_buffer, read_options, rep->compression_dict_handle, + UncompressionDict::GetEmptyDict(), uncompression_dict, get_context, + lookup_context, + /* for_compaction */ false, use_cache, + /* async_read */ false); + + if (!s.ok()) { + ROCKS_LOG_WARN( + rep->ioptions.logger, + "Encountered error while reading data from compression dictionary " + "block %s", + s.ToString().c_str()); + } + + return s; +} + +Status UncompressionDictReader::GetOrReadUncompressionDictionary( + FilePrefetchBuffer* prefetch_buffer, const ReadOptions& ro, bool no_io, + bool verify_checksums, GetContext* get_context, + BlockCacheLookupContext* lookup_context, + CachableEntry* uncompression_dict) const { + assert(uncompression_dict); + + if (!uncompression_dict_.IsEmpty()) { + uncompression_dict->SetUnownedValue(uncompression_dict_.GetValue()); + return Status::OK(); + } + + ReadOptions read_options; + if (no_io) { + read_options.read_tier = kBlockCacheTier; + } + read_options.verify_checksums = verify_checksums; + read_options.io_activity = ro.io_activity; + + return ReadUncompressionDictionary(table_, prefetch_buffer, read_options, + cache_dictionary_blocks(), get_context, + lookup_context, uncompression_dict); +} + +size_t UncompressionDictReader::ApproximateMemoryUsage() const { + assert(!uncompression_dict_.GetOwnValue() || + uncompression_dict_.GetValue() != nullptr); + size_t usage = uncompression_dict_.GetOwnValue() + ? uncompression_dict_.GetValue()->ApproximateMemoryUsage() + : 0; + +#ifdef ROCKSDB_MALLOC_USABLE_SIZE + usage += malloc_usable_size(const_cast(this)); +#else + usage += sizeof(*this); +#endif // ROCKSDB_MALLOC_USABLE_SIZE + + return usage; +} + +bool UncompressionDictReader::cache_dictionary_blocks() const { + assert(table_); + assert(table_->get_rep()); + + return table_->get_rep()->table_options.cache_index_and_filter_blocks; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_based/uncompression_dict_reader.h b/librocksdb-sys/rocksdb/table/block_based/uncompression_dict_reader.h new file mode 100644 index 0000000..c78800d --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_based/uncompression_dict_reader.h @@ -0,0 +1,61 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#pragma once + +#include + +#include "table/block_based/cachable_entry.h" +#include "table/format.h" + +namespace ROCKSDB_NAMESPACE { + +class BlockBasedTable; +struct BlockCacheLookupContext; +class FilePrefetchBuffer; +class GetContext; +struct ReadOptions; +struct UncompressionDict; + +// Provides access to the uncompression dictionary regardless of whether +// it is owned by the reader or stored in the cache, or whether it is pinned +// in the cache or not. +class UncompressionDictReader { + public: + static Status Create( + const BlockBasedTable* table, const ReadOptions& ro, + FilePrefetchBuffer* prefetch_buffer, bool use_cache, bool prefetch, + bool pin, BlockCacheLookupContext* lookup_context, + std::unique_ptr* uncompression_dict_reader); + + Status GetOrReadUncompressionDictionary( + FilePrefetchBuffer* prefetch_buffer, const ReadOptions& ro, bool no_io, + bool verify_checksums, GetContext* get_context, + BlockCacheLookupContext* lookup_context, + CachableEntry* uncompression_dict) const; + + size_t ApproximateMemoryUsage() const; + + private: + UncompressionDictReader(const BlockBasedTable* t, + CachableEntry&& uncompression_dict) + : table_(t), uncompression_dict_(std::move(uncompression_dict)) { + assert(table_); + } + + bool cache_dictionary_blocks() const; + + static Status ReadUncompressionDictionary( + const BlockBasedTable* table, FilePrefetchBuffer* prefetch_buffer, + const ReadOptions& read_options, bool use_cache, GetContext* get_context, + BlockCacheLookupContext* lookup_context, + CachableEntry* uncompression_dict); + + const BlockBasedTable* table_; + CachableEntry uncompression_dict_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_fetcher.cc b/librocksdb-sys/rocksdb/table/block_fetcher.cc new file mode 100644 index 0000000..412ac4b --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_fetcher.cc @@ -0,0 +1,405 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "table/block_fetcher.h" + +#include +#include +#include + +#include "logging/logging.h" +#include "memory/memory_allocator_impl.h" +#include "monitoring/perf_context_imp.h" +#include "rocksdb/compression_type.h" +#include "rocksdb/env.h" +#include "table/block_based/block.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/block_based/block_type.h" +#include "table/block_based/reader_common.h" +#include "table/format.h" +#include "table/persistent_cache_helper.h" +#include "util/compression.h" +#include "util/stop_watch.h" + +namespace ROCKSDB_NAMESPACE { + +inline void BlockFetcher::ProcessTrailerIfPresent() { + if (footer_.GetBlockTrailerSize() > 0) { + assert(footer_.GetBlockTrailerSize() == BlockBasedTable::kBlockTrailerSize); + if (read_options_.verify_checksums) { + io_status_ = status_to_io_status( + VerifyBlockChecksum(footer_, slice_.data(), block_size_, + file_->file_name(), handle_.offset())); + RecordTick(ioptions_.stats, BLOCK_CHECKSUM_COMPUTE_COUNT); + if (!io_status_.ok()) { + assert(io_status_.IsCorruption()); + RecordTick(ioptions_.stats, BLOCK_CHECKSUM_MISMATCH_COUNT); + } + } + compression_type_ = + BlockBasedTable::GetBlockCompressionType(slice_.data(), block_size_); + } else { + // E.g. plain table or cuckoo table + compression_type_ = kNoCompression; + } +} + +inline bool BlockFetcher::TryGetUncompressBlockFromPersistentCache() { + if (cache_options_.persistent_cache && + !cache_options_.persistent_cache->IsCompressed()) { + Status status = PersistentCacheHelper::LookupUncompressed( + cache_options_, handle_, contents_); + if (status.ok()) { + // uncompressed page is found for the block handle + return true; + } else { + // uncompressed page is not found + if (ioptions_.logger && !status.IsNotFound()) { + assert(!status.ok()); + ROCKS_LOG_INFO(ioptions_.logger, + "Error reading from persistent cache. %s", + status.ToString().c_str()); + } + } + } + return false; +} + +inline bool BlockFetcher::TryGetFromPrefetchBuffer() { + if (prefetch_buffer_ != nullptr) { + IOOptions opts; + IOStatus io_s = file_->PrepareIOOptions(read_options_, opts); + if (io_s.ok()) { + bool read_from_prefetch_buffer = false; + if (read_options_.async_io && !for_compaction_) { + read_from_prefetch_buffer = prefetch_buffer_->TryReadFromCacheAsync( + opts, file_, handle_.offset(), block_size_with_trailer_, &slice_, + &io_s, read_options_.rate_limiter_priority); + } else { + read_from_prefetch_buffer = prefetch_buffer_->TryReadFromCache( + opts, file_, handle_.offset(), block_size_with_trailer_, &slice_, + &io_s, read_options_.rate_limiter_priority, for_compaction_); + } + if (read_from_prefetch_buffer) { + ProcessTrailerIfPresent(); + if (!io_status_.ok()) { + return true; + } + got_from_prefetch_buffer_ = true; + used_buf_ = const_cast(slice_.data()); + } + } + if (!io_s.ok()) { + io_status_ = io_s; + return true; + } + } + return got_from_prefetch_buffer_; +} + +inline bool BlockFetcher::TryGetSerializedBlockFromPersistentCache() { + if (cache_options_.persistent_cache && + cache_options_.persistent_cache->IsCompressed()) { + std::unique_ptr buf; + io_status_ = status_to_io_status(PersistentCacheHelper::LookupSerialized( + cache_options_, handle_, &buf, block_size_with_trailer_)); + if (io_status_.ok()) { + heap_buf_ = CacheAllocationPtr(buf.release()); + used_buf_ = heap_buf_.get(); + slice_ = Slice(heap_buf_.get(), block_size_); + ProcessTrailerIfPresent(); + return true; + } else if (!io_status_.IsNotFound() && ioptions_.logger) { + assert(!io_status_.ok()); + ROCKS_LOG_INFO(ioptions_.logger, + "Error reading from persistent cache. %s", + io_status_.ToString().c_str()); + } + } + return false; +} + +inline void BlockFetcher::PrepareBufferForBlockFromFile() { + // cache miss read from device + if ((do_uncompress_ || ioptions_.allow_mmap_reads) && + block_size_with_trailer_ < kDefaultStackBufferSize) { + // If we've got a small enough chunk of data, read it in to the + // trivially allocated stack buffer instead of needing a full malloc() + // + // `GetBlockContents()` cannot return this data as its lifetime is tied to + // this `BlockFetcher`'s lifetime. That is fine because this is only used + // in cases where we do not expect the `GetBlockContents()` result to be the + // same buffer we are assigning here. If we guess incorrectly, there will be + // a heap allocation and memcpy in `GetBlockContents()` to obtain the final + // result. Considering we are eliding a heap allocation here by using the + // stack buffer, the cost of guessing incorrectly here is one extra memcpy. + // + // When `do_uncompress_` is true, we expect the uncompression step will + // allocate heap memory for the final result. However this expectation will + // be wrong if the block turns out to already be uncompressed, which we + // won't know for sure until after reading it. + // + // When `ioptions_.allow_mmap_reads` is true, we do not expect the file + // reader to use the scratch buffer at all, but instead return a pointer + // into the mapped memory. This expectation will be wrong when using a + // file reader that does not implement mmap reads properly. + used_buf_ = &stack_buf_[0]; + } else if (maybe_compressed_ && !do_uncompress_) { + compressed_buf_ = + AllocateBlock(block_size_with_trailer_, memory_allocator_compressed_); + used_buf_ = compressed_buf_.get(); + } else { + heap_buf_ = AllocateBlock(block_size_with_trailer_, memory_allocator_); + used_buf_ = heap_buf_.get(); + } +} + +inline void BlockFetcher::InsertCompressedBlockToPersistentCacheIfNeeded() { + if (io_status_.ok() && read_options_.fill_cache && + cache_options_.persistent_cache && + cache_options_.persistent_cache->IsCompressed()) { + PersistentCacheHelper::InsertSerialized(cache_options_, handle_, used_buf_, + block_size_with_trailer_); + } +} + +inline void BlockFetcher::InsertUncompressedBlockToPersistentCacheIfNeeded() { + if (io_status_.ok() && !got_from_prefetch_buffer_ && + read_options_.fill_cache && cache_options_.persistent_cache && + !cache_options_.persistent_cache->IsCompressed()) { + // insert to uncompressed cache + PersistentCacheHelper::InsertUncompressed(cache_options_, handle_, + *contents_); + } +} + +inline void BlockFetcher::CopyBufferToHeapBuf() { + assert(used_buf_ != heap_buf_.get()); + heap_buf_ = AllocateBlock(block_size_with_trailer_, memory_allocator_); + memcpy(heap_buf_.get(), used_buf_, block_size_with_trailer_); +#ifndef NDEBUG + num_heap_buf_memcpy_++; +#endif +} + +inline void BlockFetcher::CopyBufferToCompressedBuf() { + assert(used_buf_ != compressed_buf_.get()); + compressed_buf_ = + AllocateBlock(block_size_with_trailer_, memory_allocator_compressed_); + memcpy(compressed_buf_.get(), used_buf_, block_size_with_trailer_); +#ifndef NDEBUG + num_compressed_buf_memcpy_++; +#endif +} + +// Entering this method means the block is not compressed or do not need to be +// uncompressed. The block can be in one of the following buffers: +// 1. prefetch buffer if prefetch is enabled and the block is prefetched before +// 2. stack_buf_ if block size is smaller than the stack_buf_ size and block +// is not compressed +// 3. heap_buf_ if the block is not compressed +// 4. compressed_buf_ if the block is compressed +// 5. direct_io_buf_ if direct IO is enabled +// After this method, if the block is compressed, it should be in +// compressed_buf_, otherwise should be in heap_buf_. +inline void BlockFetcher::GetBlockContents() { + if (slice_.data() != used_buf_) { + // the slice content is not the buffer provided + *contents_ = BlockContents(Slice(slice_.data(), block_size_)); + } else { + // page can be either uncompressed or compressed, the buffer either stack + // or heap provided. Refer to https://github.com/facebook/rocksdb/pull/4096 + if (got_from_prefetch_buffer_ || used_buf_ == &stack_buf_[0]) { + CopyBufferToHeapBuf(); + } else if (used_buf_ == compressed_buf_.get()) { + if (compression_type_ == kNoCompression && + memory_allocator_ != memory_allocator_compressed_) { + CopyBufferToHeapBuf(); + } else { + heap_buf_ = std::move(compressed_buf_); + } + } else if (direct_io_buf_.get() != nullptr) { + if (compression_type_ == kNoCompression) { + CopyBufferToHeapBuf(); + } else { + CopyBufferToCompressedBuf(); + heap_buf_ = std::move(compressed_buf_); + } + } + *contents_ = BlockContents(std::move(heap_buf_), block_size_); + } +#ifndef NDEBUG + contents_->has_trailer = footer_.GetBlockTrailerSize() > 0; +#endif +} + +IOStatus BlockFetcher::ReadBlockContents() { + if (TryGetUncompressBlockFromPersistentCache()) { + compression_type_ = kNoCompression; +#ifndef NDEBUG + contents_->has_trailer = footer_.GetBlockTrailerSize() > 0; +#endif // NDEBUG + return IOStatus::OK(); + } + if (TryGetFromPrefetchBuffer()) { + if (!io_status_.ok()) { + return io_status_; + } + } else if (!TryGetSerializedBlockFromPersistentCache()) { + IOOptions opts; + io_status_ = file_->PrepareIOOptions(read_options_, opts); + // Actual file read + if (io_status_.ok()) { + if (file_->use_direct_io()) { + PERF_TIMER_GUARD(block_read_time); + PERF_CPU_TIMER_GUARD(block_read_cpu_time, nullptr); + io_status_ = file_->Read( + opts, handle_.offset(), block_size_with_trailer_, &slice_, nullptr, + &direct_io_buf_, read_options_.rate_limiter_priority); + PERF_COUNTER_ADD(block_read_count, 1); + used_buf_ = const_cast(slice_.data()); + } else { + PrepareBufferForBlockFromFile(); + PERF_TIMER_GUARD(block_read_time); + PERF_CPU_TIMER_GUARD(block_read_cpu_time, nullptr); + io_status_ = file_->Read(opts, handle_.offset(), + block_size_with_trailer_, &slice_, used_buf_, + nullptr, read_options_.rate_limiter_priority); + PERF_COUNTER_ADD(block_read_count, 1); +#ifndef NDEBUG + if (slice_.data() == &stack_buf_[0]) { + num_stack_buf_memcpy_++; + } else if (slice_.data() == heap_buf_.get()) { + num_heap_buf_memcpy_++; + } else if (slice_.data() == compressed_buf_.get()) { + num_compressed_buf_memcpy_++; + } +#endif + } + } + + // TODO: introduce dedicated perf counter for range tombstones + switch (block_type_) { + case BlockType::kFilter: + case BlockType::kFilterPartitionIndex: + PERF_COUNTER_ADD(filter_block_read_count, 1); + break; + + case BlockType::kCompressionDictionary: + PERF_COUNTER_ADD(compression_dict_block_read_count, 1); + break; + + case BlockType::kIndex: + PERF_COUNTER_ADD(index_block_read_count, 1); + break; + + // Nothing to do here as we don't have counters for the other types. + default: + break; + } + + PERF_COUNTER_ADD(block_read_byte, block_size_with_trailer_); + if (!io_status_.ok()) { + return io_status_; + } + + if (slice_.size() != block_size_with_trailer_) { + return IOStatus::Corruption( + "truncated block read from " + file_->file_name() + " offset " + + std::to_string(handle_.offset()) + ", expected " + + std::to_string(block_size_with_trailer_) + " bytes, got " + + std::to_string(slice_.size())); + } + + ProcessTrailerIfPresent(); + if (io_status_.ok()) { + InsertCompressedBlockToPersistentCacheIfNeeded(); + } else { + return io_status_; + } + } + + if (do_uncompress_ && compression_type_ != kNoCompression) { + PERF_TIMER_GUARD(block_decompress_time); + // compressed page, uncompress, update cache + UncompressionContext context(compression_type_); + UncompressionInfo info(context, uncompression_dict_, compression_type_); + io_status_ = status_to_io_status(UncompressSerializedBlock( + info, slice_.data(), block_size_, contents_, footer_.format_version(), + ioptions_, memory_allocator_)); +#ifndef NDEBUG + num_heap_buf_memcpy_++; +#endif + compression_type_ = kNoCompression; + } else { + GetBlockContents(); + } + + InsertUncompressedBlockToPersistentCacheIfNeeded(); + + return io_status_; +} + +IOStatus BlockFetcher::ReadAsyncBlockContents() { + if (TryGetUncompressBlockFromPersistentCache()) { + compression_type_ = kNoCompression; +#ifndef NDEBUG + contents_->has_trailer = footer_.GetBlockTrailerSize() > 0; +#endif // NDEBUG + return IOStatus::OK(); + } else if (!TryGetSerializedBlockFromPersistentCache()) { + assert(prefetch_buffer_ != nullptr); + if (!for_compaction_) { + IOOptions opts; + IOStatus io_s = file_->PrepareIOOptions(read_options_, opts); + if (!io_s.ok()) { + return io_s; + } + io_s = status_to_io_status(prefetch_buffer_->PrefetchAsync( + opts, file_, handle_.offset(), block_size_with_trailer_, &slice_)); + if (io_s.IsTryAgain()) { + return io_s; + } + if (io_s.ok()) { + // Data Block is already in prefetch. + got_from_prefetch_buffer_ = true; + ProcessTrailerIfPresent(); + if (!io_status_.ok()) { + return io_status_; + } + used_buf_ = const_cast(slice_.data()); + + if (do_uncompress_ && compression_type_ != kNoCompression) { + PERF_TIMER_GUARD(block_decompress_time); + // compressed page, uncompress, update cache + UncompressionContext context(compression_type_); + UncompressionInfo info(context, uncompression_dict_, + compression_type_); + io_status_ = status_to_io_status(UncompressSerializedBlock( + info, slice_.data(), block_size_, contents_, + footer_.format_version(), ioptions_, memory_allocator_)); +#ifndef NDEBUG + num_heap_buf_memcpy_++; +#endif + compression_type_ = kNoCompression; + } else { + GetBlockContents(); + } + InsertUncompressedBlockToPersistentCacheIfNeeded(); + return io_status_; + } + } + // Fallback to sequential reading of data blocks in case of io_s returns + // error or for_compaction_is true. + return ReadBlockContents(); + } + return io_status_; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_fetcher.h b/librocksdb-sys/rocksdb/table/block_fetcher.h new file mode 100644 index 0000000..da6c352 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_fetcher.h @@ -0,0 +1,142 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include "memory/memory_allocator_impl.h" +#include "table/block_based/block.h" +#include "table/block_based/block_type.h" +#include "table/format.h" +#include "table/persistent_cache_options.h" + +namespace ROCKSDB_NAMESPACE { + +// Retrieves a single block of a given file. Utilizes the prefetch buffer and/or +// persistent cache provided (if any) to try to avoid reading from the file +// directly. Note that both the prefetch buffer and the persistent cache are +// optional; also, note that the persistent cache may be configured to store +// either compressed or uncompressed blocks. +// +// If the retrieved block is compressed and the do_uncompress flag is set, +// BlockFetcher uncompresses the block (using the uncompression dictionary, +// if provided, to prime the compression algorithm), and returns the resulting +// uncompressed block data. Otherwise, it returns the original block. +// +// Two read options affect the behavior of BlockFetcher: if verify_checksums is +// true, the checksum of the (original) block is checked; if fill_cache is true, +// the block is added to the persistent cache if needed. +// +// Memory for uncompressed and compressed blocks is allocated as needed +// using memory_allocator and memory_allocator_compressed, respectively +// (if provided; otherwise, the default allocator is used). + +class BlockFetcher { + public: + BlockFetcher(RandomAccessFileReader* file, + FilePrefetchBuffer* prefetch_buffer, + const Footer& footer /* ref retained */, + const ReadOptions& read_options, + const BlockHandle& handle /* ref retained */, + BlockContents* contents, + const ImmutableOptions& ioptions /* ref retained */, + bool do_uncompress, bool maybe_compressed, BlockType block_type, + const UncompressionDict& uncompression_dict /* ref retained */, + const PersistentCacheOptions& cache_options /* ref retained */, + MemoryAllocator* memory_allocator = nullptr, + MemoryAllocator* memory_allocator_compressed = nullptr, + bool for_compaction = false) + : file_(file), + prefetch_buffer_(prefetch_buffer), + footer_(footer), + read_options_(read_options), + handle_(handle), + contents_(contents), + ioptions_(ioptions), + do_uncompress_(do_uncompress), + maybe_compressed_(maybe_compressed), + block_type_(block_type), + block_size_(static_cast(handle_.size())), + block_size_with_trailer_(block_size_ + footer.GetBlockTrailerSize()), + uncompression_dict_(uncompression_dict), + cache_options_(cache_options), + memory_allocator_(memory_allocator), + memory_allocator_compressed_(memory_allocator_compressed), + for_compaction_(for_compaction) { + io_status_.PermitUncheckedError(); // TODO(AR) can we improve on this? + } + + IOStatus ReadBlockContents(); + IOStatus ReadAsyncBlockContents(); + + inline CompressionType get_compression_type() const { + return compression_type_; + } + inline size_t GetBlockSizeWithTrailer() const { + return block_size_with_trailer_; + } + +#ifndef NDEBUG + int TEST_GetNumStackBufMemcpy() const { return num_stack_buf_memcpy_; } + int TEST_GetNumHeapBufMemcpy() const { return num_heap_buf_memcpy_; } + int TEST_GetNumCompressedBufMemcpy() const { + return num_compressed_buf_memcpy_; + } + +#endif + private: +#ifndef NDEBUG + int num_stack_buf_memcpy_ = 0; + int num_heap_buf_memcpy_ = 0; + int num_compressed_buf_memcpy_ = 0; + +#endif + static const uint32_t kDefaultStackBufferSize = 5000; + + RandomAccessFileReader* file_; + FilePrefetchBuffer* prefetch_buffer_; + const Footer& footer_; + const ReadOptions read_options_; + const BlockHandle& handle_; + BlockContents* contents_; + const ImmutableOptions& ioptions_; + const bool do_uncompress_; + const bool maybe_compressed_; + const BlockType block_type_; + const size_t block_size_; + const size_t block_size_with_trailer_; + const UncompressionDict& uncompression_dict_; + const PersistentCacheOptions& cache_options_; + MemoryAllocator* memory_allocator_; + MemoryAllocator* memory_allocator_compressed_; + IOStatus io_status_; + Slice slice_; + char* used_buf_ = nullptr; + AlignedBuf direct_io_buf_; + CacheAllocationPtr heap_buf_; + CacheAllocationPtr compressed_buf_; + char stack_buf_[kDefaultStackBufferSize]; + bool got_from_prefetch_buffer_ = false; + CompressionType compression_type_; + bool for_compaction_ = false; + + // return true if found + bool TryGetUncompressBlockFromPersistentCache(); + // return true if found + bool TryGetFromPrefetchBuffer(); + bool TryGetSerializedBlockFromPersistentCache(); + void PrepareBufferForBlockFromFile(); + // Copy content from used_buf_ to new heap_buf_. + void CopyBufferToHeapBuf(); + // Copy content from used_buf_ to new compressed_buf_. + void CopyBufferToCompressedBuf(); + void GetBlockContents(); + void InsertCompressedBlockToPersistentCacheIfNeeded(); + void InsertUncompressedBlockToPersistentCacheIfNeeded(); + void ProcessTrailerIfPresent(); +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/block_fetcher_test.cc b/librocksdb-sys/rocksdb/table/block_fetcher_test.cc new file mode 100644 index 0000000..1810981 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/block_fetcher_test.cc @@ -0,0 +1,523 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "table/block_fetcher.h" + +#include "db/table_properties_collector.h" +#include "file/file_util.h" +#include "options/options_helper.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/db.h" +#include "rocksdb/file_system.h" +#include "table/block_based/binary_search_index_reader.h" +#include "table/block_based/block_based_table_builder.h" +#include "table/block_based/block_based_table_factory.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/format.h" +#include "test_util/testharness.h" +#include "utilities/memory_allocators.h" + +namespace ROCKSDB_NAMESPACE { +namespace { +struct MemcpyStats { + int num_stack_buf_memcpy; + int num_heap_buf_memcpy; + int num_compressed_buf_memcpy; +}; + +struct BufAllocationStats { + int num_heap_buf_allocations; + int num_compressed_buf_allocations; +}; + +struct TestStats { + MemcpyStats memcpy_stats; + BufAllocationStats buf_allocation_stats; +}; + +class BlockFetcherTest : public testing::Test { + public: + enum class Mode { + kBufferedRead = 0, + kBufferedMmap, + kDirectRead, + kNumModes, + }; + // use NumModes as array size to avoid "size of array '...' has non-integral + // type" errors. + const static int NumModes = static_cast(Mode::kNumModes); + + protected: + void SetUp() override { + SetupSyncPointsToMockDirectIO(); + test_dir_ = test::PerThreadDBPath("block_fetcher_test"); + env_ = Env::Default(); + fs_ = FileSystem::Default(); + ASSERT_OK(fs_->CreateDir(test_dir_, IOOptions(), nullptr)); + } + + void TearDown() override { EXPECT_OK(DestroyDir(env_, test_dir_)); } + + void AssertSameBlock(const std::string& block1, const std::string& block2) { + ASSERT_EQ(block1, block2); + } + + // Creates a table with kv pairs (i, i) where i ranges from 0 to 9, inclusive. + void CreateTable(const std::string& table_name, + const CompressionType& compression_type) { + std::unique_ptr writer; + NewFileWriter(table_name, &writer); + + // Create table builder. + ImmutableOptions ioptions(options_); + InternalKeyComparator comparator(options_.comparator); + ColumnFamilyOptions cf_options(options_); + MutableCFOptions moptions(cf_options); + IntTblPropCollectorFactories factories; + std::unique_ptr table_builder(table_factory_.NewTableBuilder( + TableBuilderOptions(ioptions, moptions, comparator, &factories, + compression_type, CompressionOptions(), + 0 /* column_family_id */, kDefaultColumnFamilyName, + -1 /* level */), + writer.get())); + + // Build table. + for (int i = 0; i < 9; i++) { + std::string key = ToInternalKey(std::to_string(i)); + // Append "00000000" to string value to enhance compression ratio + std::string value = "00000000" + std::to_string(i); + table_builder->Add(key, value); + } + ASSERT_OK(table_builder->Finish()); + } + + void FetchIndexBlock(const std::string& table_name, + CountedMemoryAllocator* heap_buf_allocator, + CountedMemoryAllocator* compressed_buf_allocator, + MemcpyStats* memcpy_stats, BlockContents* index_block, + std::string* result) { + FileOptions fopt(options_); + std::unique_ptr file; + NewFileReader(table_name, fopt, &file); + + // Get handle of the index block. + Footer footer; + ReadFooter(file.get(), &footer); + const BlockHandle& index_handle = footer.index_handle(); + // FIXME: index handle will need to come from metaindex for + // format_version >= 6 when that becomes the default + ASSERT_FALSE(index_handle.IsNull()); + + CompressionType compression_type; + FetchBlock(file.get(), index_handle, BlockType::kIndex, + false /* compressed */, false /* do_uncompress */, + heap_buf_allocator, compressed_buf_allocator, index_block, + memcpy_stats, &compression_type); + ASSERT_EQ(compression_type, CompressionType::kNoCompression); + result->assign(index_block->data.ToString()); + } + + // Fetches the first data block in both direct IO and non-direct IO mode. + // + // compressed: whether the data blocks are compressed; + // do_uncompress: whether the data blocks should be uncompressed on fetching. + // compression_type: the expected compression type. + // + // Expects: + // Block contents are the same. + // Bufferr allocation and memory copy statistics are expected. + void TestFetchDataBlock( + const std::string& table_name_prefix, bool compressed, bool do_uncompress, + std::array expected_stats_by_mode) { + for (CompressionType compression_type : GetSupportedCompressions()) { + bool do_compress = compression_type != kNoCompression; + if (compressed != do_compress) continue; + std::string compression_type_str = + CompressionTypeToString(compression_type); + + std::string table_name = table_name_prefix + compression_type_str; + CreateTable(table_name, compression_type); + + CompressionType expected_compression_type_after_fetch = + (compressed && !do_uncompress) ? compression_type : kNoCompression; + + BlockContents blocks[NumModes]; + std::string block_datas[NumModes]; + MemcpyStats memcpy_stats[NumModes]; + CountedMemoryAllocator heap_buf_allocators[NumModes]; + CountedMemoryAllocator compressed_buf_allocators[NumModes]; + for (int i = 0; i < NumModes; ++i) { + SetMode(static_cast(i)); + FetchFirstDataBlock(table_name, compressed, do_uncompress, + expected_compression_type_after_fetch, + &heap_buf_allocators[i], + &compressed_buf_allocators[i], &blocks[i], + &block_datas[i], &memcpy_stats[i]); + } + + for (int i = 0; i < NumModes - 1; ++i) { + AssertSameBlock(block_datas[i], block_datas[i + 1]); + } + + // Check memcpy and buffer allocation statistics. + for (int i = 0; i < NumModes; ++i) { + const TestStats& expected_stats = expected_stats_by_mode[i]; + + ASSERT_EQ(memcpy_stats[i].num_stack_buf_memcpy, + expected_stats.memcpy_stats.num_stack_buf_memcpy); + ASSERT_EQ(memcpy_stats[i].num_heap_buf_memcpy, + expected_stats.memcpy_stats.num_heap_buf_memcpy); + ASSERT_EQ(memcpy_stats[i].num_compressed_buf_memcpy, + expected_stats.memcpy_stats.num_compressed_buf_memcpy); + + if (kXpressCompression == compression_type) { + // XPRESS allocates memory internally, thus does not support for + // custom allocator verification + continue; + } else { + ASSERT_EQ( + heap_buf_allocators[i].GetNumAllocations(), + expected_stats.buf_allocation_stats.num_heap_buf_allocations); + ASSERT_EQ(compressed_buf_allocators[i].GetNumAllocations(), + expected_stats.buf_allocation_stats + .num_compressed_buf_allocations); + + // The allocated buffers are not deallocated until + // the block content is deleted. + ASSERT_EQ(heap_buf_allocators[i].GetNumDeallocations(), 0); + ASSERT_EQ(compressed_buf_allocators[i].GetNumDeallocations(), 0); + blocks[i].allocation.reset(); + ASSERT_EQ( + heap_buf_allocators[i].GetNumDeallocations(), + expected_stats.buf_allocation_stats.num_heap_buf_allocations); + ASSERT_EQ(compressed_buf_allocators[i].GetNumDeallocations(), + expected_stats.buf_allocation_stats + .num_compressed_buf_allocations); + } + } + } + } + + void SetMode(Mode mode) { + switch (mode) { + case Mode::kBufferedRead: + options_.use_direct_reads = false; + options_.allow_mmap_reads = false; + break; + case Mode::kBufferedMmap: + options_.use_direct_reads = false; + options_.allow_mmap_reads = true; + break; + case Mode::kDirectRead: + options_.use_direct_reads = true; + options_.allow_mmap_reads = false; + break; + case Mode::kNumModes: + assert(false); + } + } + + private: + std::string test_dir_; + Env* env_; + std::shared_ptr fs_; + BlockBasedTableFactory table_factory_; + Options options_; + + std::string Path(const std::string& fname) { return test_dir_ + "/" + fname; } + + void WriteToFile(const std::string& content, const std::string& filename) { + std::unique_ptr f; + ASSERT_OK(fs_->NewWritableFile(Path(filename), FileOptions(), &f, nullptr)); + ASSERT_OK(f->Append(content, IOOptions(), nullptr)); + ASSERT_OK(f->Close(IOOptions(), nullptr)); + } + + void NewFileWriter(const std::string& filename, + std::unique_ptr* writer) { + std::string path = Path(filename); + FileOptions file_options; + ASSERT_OK(WritableFileWriter::Create(env_->GetFileSystem(), path, + file_options, writer, nullptr)); + } + + void NewFileReader(const std::string& filename, const FileOptions& opt, + std::unique_ptr* reader) { + std::string path = Path(filename); + std::unique_ptr f; + ASSERT_OK(fs_->NewRandomAccessFile(path, opt, &f, nullptr)); + reader->reset(new RandomAccessFileReader(std::move(f), path, + env_->GetSystemClock().get())); + } + + void NewTableReader(const ImmutableOptions& ioptions, + const FileOptions& foptions, + const InternalKeyComparator& comparator, + const std::string& table_name, + std::unique_ptr* table) { + std::unique_ptr file; + NewFileReader(table_name, foptions, &file); + + uint64_t file_size = 0; + ASSERT_OK(env_->GetFileSize(Path(table_name), &file_size)); + + std::unique_ptr table_reader; + ReadOptions ro; + const auto* table_options = + table_factory_.GetOptions(); + ASSERT_NE(table_options, nullptr); + ASSERT_OK(BlockBasedTable::Open(ro, ioptions, EnvOptions(), *table_options, + comparator, std::move(file), file_size, + 0 /* block_protection_bytes_per_key */, + &table_reader, 0 /* tail_size */)); + + table->reset(reinterpret_cast(table_reader.release())); + } + + std::string ToInternalKey(const std::string& key) { + InternalKey internal_key(key, 0, ValueType::kTypeValue); + return internal_key.Encode().ToString(); + } + + void ReadFooter(RandomAccessFileReader* file, Footer* footer) { + uint64_t file_size = 0; + ASSERT_OK(env_->GetFileSize(file->file_name(), &file_size)); + IOOptions opts; + ASSERT_OK(ReadFooterFromFile(opts, file, *fs_, + nullptr /* prefetch_buffer */, file_size, + footer, kBlockBasedTableMagicNumber)); + } + + // NOTE: compression_type returns the compression type of the fetched block + // contents, so if the block is fetched and uncompressed, then it's + // kNoCompression. + void FetchBlock(RandomAccessFileReader* file, const BlockHandle& block, + BlockType block_type, bool compressed, bool do_uncompress, + MemoryAllocator* heap_buf_allocator, + MemoryAllocator* compressed_buf_allocator, + BlockContents* contents, MemcpyStats* stats, + CompressionType* compresstion_type) { + ImmutableOptions ioptions(options_); + ReadOptions roptions; + PersistentCacheOptions persistent_cache_options; + Footer footer; + ReadFooter(file, &footer); + std::unique_ptr fetcher(new BlockFetcher( + file, nullptr /* prefetch_buffer */, footer, roptions, block, contents, + ioptions, do_uncompress, compressed, block_type, + UncompressionDict::GetEmptyDict(), persistent_cache_options, + heap_buf_allocator, compressed_buf_allocator)); + + ASSERT_OK(fetcher->ReadBlockContents()); + + stats->num_stack_buf_memcpy = fetcher->TEST_GetNumStackBufMemcpy(); + stats->num_heap_buf_memcpy = fetcher->TEST_GetNumHeapBufMemcpy(); + stats->num_compressed_buf_memcpy = + fetcher->TEST_GetNumCompressedBufMemcpy(); + + *compresstion_type = fetcher->get_compression_type(); + } + + // NOTE: expected_compression_type is the expected compression + // type of the fetched block content, if the block is uncompressed, + // then the expected compression type is kNoCompression. + void FetchFirstDataBlock(const std::string& table_name, bool compressed, + bool do_uncompress, + CompressionType expected_compression_type, + MemoryAllocator* heap_buf_allocator, + MemoryAllocator* compressed_buf_allocator, + BlockContents* block, std::string* result, + MemcpyStats* memcpy_stats) { + ImmutableOptions ioptions(options_); + InternalKeyComparator comparator(options_.comparator); + FileOptions foptions(options_); + + // Get block handle for the first data block. + std::unique_ptr table; + NewTableReader(ioptions, foptions, comparator, table_name, &table); + + std::unique_ptr index_reader; + ReadOptions ro; + ASSERT_OK(BinarySearchIndexReader::Create( + table.get(), ro, nullptr /* prefetch_buffer */, false /* use_cache */, + false /* prefetch */, false /* pin */, nullptr /* lookup_context */, + &index_reader)); + + std::unique_ptr> iter( + index_reader->NewIterator( + ReadOptions(), false /* disable_prefix_seek */, nullptr /* iter */, + nullptr /* get_context */, nullptr /* lookup_context */)); + ASSERT_OK(iter->status()); + iter->SeekToFirst(); + BlockHandle first_block_handle = iter->value().handle; + + // Fetch first data block. + std::unique_ptr file; + NewFileReader(table_name, foptions, &file); + CompressionType compression_type; + FetchBlock(file.get(), first_block_handle, BlockType::kData, compressed, + do_uncompress, heap_buf_allocator, compressed_buf_allocator, + block, memcpy_stats, &compression_type); + ASSERT_EQ(compression_type, expected_compression_type); + result->assign(block->data.ToString()); + } +}; + +// Skip the following tests in lite mode since direct I/O is unsupported. + +// Fetch index block under both direct IO and non-direct IO. +// Expects: +// the index block contents are the same for both read modes. +TEST_F(BlockFetcherTest, FetchIndexBlock) { + for (CompressionType compression : GetSupportedCompressions()) { + std::string table_name = + "FetchIndexBlock" + CompressionTypeToString(compression); + CreateTable(table_name, compression); + + CountedMemoryAllocator allocator; + MemcpyStats memcpy_stats; + BlockContents indexes[NumModes]; + std::string index_datas[NumModes]; + for (int i = 0; i < NumModes; ++i) { + SetMode(static_cast(i)); + FetchIndexBlock(table_name, &allocator, &allocator, &memcpy_stats, + &indexes[i], &index_datas[i]); + } + for (int i = 0; i < NumModes - 1; ++i) { + AssertSameBlock(index_datas[i], index_datas[i + 1]); + } + } +} + +// Data blocks are not compressed, +// fetch data block under direct IO, mmap IO,and non-direct IO. +// Expects: +// 1. in non-direct IO mode, allocate a heap buffer and memcpy the block +// into the buffer; +// 2. in direct IO mode, allocate a heap buffer and memcpy from the +// direct IO buffer to the heap buffer. +TEST_F(BlockFetcherTest, FetchUncompressedDataBlock) { + TestStats expected_non_mmap_stats = { + { + 0 /* num_stack_buf_memcpy */, + 1 /* num_heap_buf_memcpy */, + 0 /* num_compressed_buf_memcpy */, + }, + { + 1 /* num_heap_buf_allocations */, + 0 /* num_compressed_buf_allocations */, + }}; + TestStats expected_mmap_stats = {{ + 0 /* num_stack_buf_memcpy */, + 0 /* num_heap_buf_memcpy */, + 0 /* num_compressed_buf_memcpy */, + }, + { + 0 /* num_heap_buf_allocations */, + 0 /* num_compressed_buf_allocations */, + }}; + std::array expected_stats_by_mode{{ + expected_non_mmap_stats /* kBufferedRead */, + expected_mmap_stats /* kBufferedMmap */, + expected_non_mmap_stats /* kDirectRead */, + }}; + TestFetchDataBlock("FetchUncompressedDataBlock", false, false, + expected_stats_by_mode); +} + +// Data blocks are compressed, +// fetch data block under both direct IO and non-direct IO, +// but do not uncompress. +// Expects: +// 1. in non-direct IO mode, allocate a compressed buffer and memcpy the block +// into the buffer; +// 2. in direct IO mode, allocate a compressed buffer and memcpy from the +// direct IO buffer to the compressed buffer. +TEST_F(BlockFetcherTest, FetchCompressedDataBlock) { + TestStats expected_non_mmap_stats = { + { + 0 /* num_stack_buf_memcpy */, + 0 /* num_heap_buf_memcpy */, + 1 /* num_compressed_buf_memcpy */, + }, + { + 0 /* num_heap_buf_allocations */, + 1 /* num_compressed_buf_allocations */, + }}; + TestStats expected_mmap_stats = {{ + 0 /* num_stack_buf_memcpy */, + 0 /* num_heap_buf_memcpy */, + 0 /* num_compressed_buf_memcpy */, + }, + { + 0 /* num_heap_buf_allocations */, + 0 /* num_compressed_buf_allocations */, + }}; + std::array expected_stats_by_mode{{ + expected_non_mmap_stats /* kBufferedRead */, + expected_mmap_stats /* kBufferedMmap */, + expected_non_mmap_stats /* kDirectRead */, + }}; + TestFetchDataBlock("FetchCompressedDataBlock", true, false, + expected_stats_by_mode); +} + +// Data blocks are compressed, +// fetch and uncompress data block under both direct IO and non-direct IO. +// Expects: +// 1. in non-direct IO mode, since the block is small, so it's first memcpyed +// to the stack buffer, then a heap buffer is allocated and the block is +// uncompressed into the heap. +// 2. in direct IO mode mode, allocate a heap buffer, then directly uncompress +// and memcpy from the direct IO buffer to the heap buffer. +TEST_F(BlockFetcherTest, FetchAndUncompressCompressedDataBlock) { + TestStats expected_buffered_read_stats = { + { + 1 /* num_stack_buf_memcpy */, + 1 /* num_heap_buf_memcpy */, + 0 /* num_compressed_buf_memcpy */, + }, + { + 1 /* num_heap_buf_allocations */, + 0 /* num_compressed_buf_allocations */, + }}; + TestStats expected_mmap_stats = {{ + 0 /* num_stack_buf_memcpy */, + 1 /* num_heap_buf_memcpy */, + 0 /* num_compressed_buf_memcpy */, + }, + { + 1 /* num_heap_buf_allocations */, + 0 /* num_compressed_buf_allocations */, + }}; + TestStats expected_direct_read_stats = { + { + 0 /* num_stack_buf_memcpy */, + 1 /* num_heap_buf_memcpy */, + 0 /* num_compressed_buf_memcpy */, + }, + { + 1 /* num_heap_buf_allocations */, + 0 /* num_compressed_buf_allocations */, + }}; + std::array expected_stats_by_mode{{ + expected_buffered_read_stats, + expected_mmap_stats, + expected_direct_read_stats, + }}; + TestFetchDataBlock("FetchAndUncompressCompressedDataBlock", true, true, + expected_stats_by_mode); +} + + +} // namespace +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/table/cleanable_test.cc b/librocksdb-sys/rocksdb/table/cleanable_test.cc new file mode 100644 index 0000000..b58eb7d --- /dev/null +++ b/librocksdb-sys/rocksdb/table/cleanable_test.cc @@ -0,0 +1,390 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "rocksdb/cleanable.h" + +#include + +#include + +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/iostats_context.h" +#include "rocksdb/perf_context.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" + +namespace ROCKSDB_NAMESPACE { + +class CleanableTest : public testing::Test {}; + +// Use this to keep track of the cleanups that were actually performed +void Multiplier(void* arg1, void* arg2) { + int* res = reinterpret_cast(arg1); + int* num = reinterpret_cast(arg2); + *res *= *num; +} + +// the first Cleanup is on stack and the rest on heap, so test with both cases +TEST_F(CleanableTest, Register) { + int n2 = 2, n3 = 3; + int res = 1; + { Cleanable c1; } + // ~Cleanable + ASSERT_EQ(1, res); + + res = 1; + { + Cleanable c1; + c1.RegisterCleanup(Multiplier, &res, &n2); // res = 2; + } + // ~Cleanable + ASSERT_EQ(2, res); + + res = 1; + { + Cleanable c1; + c1.RegisterCleanup(Multiplier, &res, &n2); // res = 2; + c1.RegisterCleanup(Multiplier, &res, &n3); // res = 2 * 3; + } + // ~Cleanable + ASSERT_EQ(6, res); + + // Test the Reset does cleanup + res = 1; + { + Cleanable c1; + c1.RegisterCleanup(Multiplier, &res, &n2); // res = 2; + c1.RegisterCleanup(Multiplier, &res, &n3); // res = 2 * 3; + c1.Reset(); + ASSERT_EQ(6, res); + } + // ~Cleanable + ASSERT_EQ(6, res); + + // Test Clenable is usable after Reset + res = 1; + { + Cleanable c1; + c1.RegisterCleanup(Multiplier, &res, &n2); // res = 2; + c1.Reset(); + ASSERT_EQ(2, res); + c1.RegisterCleanup(Multiplier, &res, &n3); // res = 2 * 3; + } + // ~Cleanable + ASSERT_EQ(6, res); +} + +// the first Cleanup is on stack and the rest on heap, +// so test all the combinations of them +TEST_F(CleanableTest, Delegation) { + int n2 = 2, n3 = 3, n5 = 5, n7 = 7; + int res = 1; + { + Cleanable c2; + { + Cleanable c1; + c1.RegisterCleanup(Multiplier, &res, &n2); // res = 2; + c1.DelegateCleanupsTo(&c2); + } + // ~Cleanable + ASSERT_EQ(1, res); + } + // ~Cleanable + ASSERT_EQ(2, res); + + res = 1; + { + Cleanable c2; + { + Cleanable c1; + c1.DelegateCleanupsTo(&c2); + } + // ~Cleanable + ASSERT_EQ(1, res); + } + // ~Cleanable + ASSERT_EQ(1, res); + + res = 1; + { + Cleanable c2; + { + Cleanable c1; + c1.RegisterCleanup(Multiplier, &res, &n2); // res = 2; + c1.RegisterCleanup(Multiplier, &res, &n3); // res = 2 * 3; + c1.DelegateCleanupsTo(&c2); + } + // ~Cleanable + ASSERT_EQ(1, res); + } + // ~Cleanable + ASSERT_EQ(6, res); + + res = 1; + { + Cleanable c2; + c2.RegisterCleanup(Multiplier, &res, &n5); // res = 5; + { + Cleanable c1; + c1.RegisterCleanup(Multiplier, &res, &n2); // res = 2; + c1.RegisterCleanup(Multiplier, &res, &n3); // res = 2 * 3; + c1.DelegateCleanupsTo(&c2); // res = 2 * 3 * 5; + } + // ~Cleanable + ASSERT_EQ(1, res); + } + // ~Cleanable + ASSERT_EQ(30, res); + + res = 1; + { + Cleanable c2; + c2.RegisterCleanup(Multiplier, &res, &n5); // res = 5; + c2.RegisterCleanup(Multiplier, &res, &n7); // res = 5 * 7; + { + Cleanable c1; + c1.RegisterCleanup(Multiplier, &res, &n2); // res = 2; + c1.RegisterCleanup(Multiplier, &res, &n3); // res = 2 * 3; + c1.DelegateCleanupsTo(&c2); // res = 2 * 3 * 5 * 7; + } + // ~Cleanable + ASSERT_EQ(1, res); + } + // ~Cleanable + ASSERT_EQ(210, res); + + res = 1; + { + Cleanable c2; + c2.RegisterCleanup(Multiplier, &res, &n5); // res = 5; + c2.RegisterCleanup(Multiplier, &res, &n7); // res = 5 * 7; + { + Cleanable c1; + c1.RegisterCleanup(Multiplier, &res, &n2); // res = 2; + c1.DelegateCleanupsTo(&c2); // res = 2 * 5 * 7; + } + // ~Cleanable + ASSERT_EQ(1, res); + } + // ~Cleanable + ASSERT_EQ(70, res); + + res = 1; + { + Cleanable c2; + c2.RegisterCleanup(Multiplier, &res, &n5); // res = 5; + c2.RegisterCleanup(Multiplier, &res, &n7); // res = 5 * 7; + { + Cleanable c1; + c1.DelegateCleanupsTo(&c2); // res = 5 * 7; + } + // ~Cleanable + ASSERT_EQ(1, res); + } + // ~Cleanable + ASSERT_EQ(35, res); + + res = 1; + { + Cleanable c2; + c2.RegisterCleanup(Multiplier, &res, &n5); // res = 5; + { + Cleanable c1; + c1.DelegateCleanupsTo(&c2); // res = 5; + } + // ~Cleanable + ASSERT_EQ(1, res); + } + // ~Cleanable + ASSERT_EQ(5, res); +} + +static void ReleaseStringHeap(void* s, void*) { + delete reinterpret_cast(s); +} + +class PinnableSlice4Test : public PinnableSlice { + public: + void TestStringIsRegistered(std::string* s) { + ASSERT_TRUE(cleanup_.function == ReleaseStringHeap); + ASSERT_EQ(cleanup_.arg1, s); + ASSERT_EQ(cleanup_.arg2, nullptr); + ASSERT_EQ(cleanup_.next, nullptr); + } +}; + +// Putting the PinnableSlice tests here due to similarity to Cleanable tests +TEST_F(CleanableTest, PinnableSlice) { + int n2 = 2; + int res = 1; + const std::string const_str = "123"; + + { + res = 1; + PinnableSlice4Test value; + Slice slice(const_str); + value.PinSlice(slice, Multiplier, &res, &n2); + std::string str; + str.assign(value.data(), value.size()); + ASSERT_EQ(const_str, str); + } + // ~Cleanable + ASSERT_EQ(2, res); + + { + res = 1; + PinnableSlice4Test value; + Slice slice(const_str); + { + Cleanable c1; + c1.RegisterCleanup(Multiplier, &res, &n2); // res = 2; + value.PinSlice(slice, &c1); + } + // ~Cleanable + ASSERT_EQ(1, res); // cleanups must have be delegated to value + std::string str; + str.assign(value.data(), value.size()); + ASSERT_EQ(const_str, str); + } + // ~Cleanable + ASSERT_EQ(2, res); + + { + PinnableSlice4Test value; + Slice slice(const_str); + value.PinSelf(slice); + std::string str; + str.assign(value.data(), value.size()); + ASSERT_EQ(const_str, str); + } + + { + PinnableSlice4Test value; + std::string* self_str_ptr = value.GetSelf(); + self_str_ptr->assign(const_str); + value.PinSelf(); + std::string str; + str.assign(value.data(), value.size()); + ASSERT_EQ(const_str, str); + } +} + +static void Decrement(void* intptr, void*) { --*static_cast(intptr); } + +// Allow unit testing moved-from data +template +void MarkInitializedForClangAnalyze(T& t) { + // No net effect, but confuse analyzer. (Published advice doesn't work.) + char* p = reinterpret_cast(&t); + std::swap(*p, *p); +} + +TEST_F(CleanableTest, SharedWrapCleanables) { + int val = 5; + Cleanable c1, c2; + c1.RegisterCleanup(&Decrement, &val, nullptr); + c1.RegisterCleanup(&Decrement, &val, nullptr); + ASSERT_TRUE(c1.HasCleanups()); + ASSERT_FALSE(c2.HasCleanups()); + + SharedCleanablePtr scp1; + ASSERT_EQ(scp1.get(), nullptr); + + // No-ops + scp1.RegisterCopyWith(&c2); + scp1.MoveAsCleanupTo(&c2); + + ASSERT_FALSE(c2.HasCleanups()); + c2.RegisterCleanup(&Decrement, &val, nullptr); + c2.RegisterCleanup(&Decrement, &val, nullptr); + c2.RegisterCleanup(&Decrement, &val, nullptr); + + scp1.Allocate(); + ASSERT_NE(scp1.get(), nullptr); + ASSERT_FALSE(scp1->HasCleanups()); + + // Copy ctor (alias scp2 = scp1) + SharedCleanablePtr scp2{scp1}; + ASSERT_EQ(scp1.get(), scp2.get()); + + c1.DelegateCleanupsTo(&*scp1); + ASSERT_TRUE(scp1->HasCleanups()); + ASSERT_TRUE(scp2->HasCleanups()); + ASSERT_FALSE(c1.HasCleanups()); + + SharedCleanablePtr scp3; + ASSERT_EQ(scp3.get(), nullptr); + + // Copy operator (alias scp3 = scp2 = scp1) + scp3 = scp2; + + // Make scp2 point elsewhere + scp2.Allocate(); + c2.DelegateCleanupsTo(&*scp2); + + ASSERT_EQ(val, 5); + // Move operator, invoke old c2 cleanups + scp2 = std::move(scp1); + ASSERT_EQ(val, 2); + MarkInitializedForClangAnalyze(scp1); + ASSERT_EQ(scp1.get(), nullptr); + + // Move ctor + { + SharedCleanablePtr scp4{std::move(scp3)}; + MarkInitializedForClangAnalyze(scp3); + ASSERT_EQ(scp3.get(), nullptr); + ASSERT_EQ(scp4.get(), scp2.get()); + + scp2.Reset(); + ASSERT_EQ(val, 2); + // invoke old c1 cleanups + } + ASSERT_EQ(val, 0); +} + +TEST_F(CleanableTest, CleanableWrapShared) { + int val = 5; + SharedCleanablePtr scp1, scp2; + scp1.Allocate(); + scp1->RegisterCleanup(&Decrement, &val, nullptr); + scp1->RegisterCleanup(&Decrement, &val, nullptr); + + scp2.Allocate(); + scp2->RegisterCleanup(&Decrement, &val, nullptr); + scp2->RegisterCleanup(&Decrement, &val, nullptr); + scp2->RegisterCleanup(&Decrement, &val, nullptr); + + { + Cleanable c1; + { + Cleanable c2, c3; + scp1.RegisterCopyWith(&c1); + scp1.MoveAsCleanupTo(&c2); + ASSERT_TRUE(c1.HasCleanups()); + ASSERT_TRUE(c2.HasCleanups()); + ASSERT_EQ(scp1.get(), nullptr); + scp2.MoveAsCleanupTo(&c3); + ASSERT_TRUE(c3.HasCleanups()); + ASSERT_EQ(scp2.get(), nullptr); + c2.Reset(); + ASSERT_FALSE(c2.HasCleanups()); + ASSERT_EQ(val, 5); + // invoke cleanups from scp2 + } + ASSERT_EQ(val, 2); + // invoke cleanups from scp1 + } + ASSERT_EQ(val, 0); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/table/compaction_merging_iterator.cc b/librocksdb-sys/rocksdb/table/compaction_merging_iterator.cc new file mode 100644 index 0000000..8a5c452 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/compaction_merging_iterator.cc @@ -0,0 +1,370 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#include "table/compaction_merging_iterator.h" + +namespace ROCKSDB_NAMESPACE { +class CompactionMergingIterator : public InternalIterator { + public: + CompactionMergingIterator( + const InternalKeyComparator* comparator, InternalIterator** children, + int n, bool is_arena_mode, + std::vector< + std::pair> + range_tombstones) + : is_arena_mode_(is_arena_mode), + comparator_(comparator), + current_(nullptr), + minHeap_(CompactionHeapItemComparator(comparator_)), + pinned_iters_mgr_(nullptr) { + children_.resize(n); + for (int i = 0; i < n; i++) { + children_[i].level = i; + children_[i].iter.Set(children[i]); + assert(children_[i].type == HeapItem::ITERATOR); + } + assert(range_tombstones.size() == static_cast(n)); + for (auto& p : range_tombstones) { + range_tombstone_iters_.push_back(p.first); + } + pinned_heap_item_.resize(n); + for (int i = 0; i < n; ++i) { + if (range_tombstones[i].second) { + // for LevelIterator + *range_tombstones[i].second = &range_tombstone_iters_[i]; + } + pinned_heap_item_[i].level = i; + pinned_heap_item_[i].type = HeapItem::DELETE_RANGE_START; + } + } + + void considerStatus(const Status& s) { + if (!s.ok() && status_.ok()) { + status_ = s; + } + } + + ~CompactionMergingIterator() override { + // TODO: use unique_ptr for range_tombstone_iters_ + for (auto child : range_tombstone_iters_) { + delete child; + } + + for (auto& child : children_) { + child.iter.DeleteIter(is_arena_mode_); + } + status_.PermitUncheckedError(); + } + + bool Valid() const override { return current_ != nullptr && status_.ok(); } + + Status status() const override { return status_; } + + void SeekToFirst() override; + + void Seek(const Slice& target) override; + + void Next() override; + + Slice key() const override { + assert(Valid()); + return current_->key(); + } + + Slice value() const override { + assert(Valid()); + if (LIKELY(current_->type == HeapItem::ITERATOR)) { + return current_->iter.value(); + } else { + return dummy_tombstone_val; + } + } + + // Here we simply relay MayBeOutOfLowerBound/MayBeOutOfUpperBound result + // from current child iterator. Potentially as long as one of child iterator + // report out of bound is not possible, we know current key is within bound. + bool MayBeOutOfLowerBound() override { + assert(Valid()); + return current_->type == HeapItem::DELETE_RANGE_START || + current_->iter.MayBeOutOfLowerBound(); + } + + IterBoundCheck UpperBoundCheckResult() override { + assert(Valid()); + return current_->type == HeapItem::DELETE_RANGE_START + ? IterBoundCheck::kUnknown + : current_->iter.UpperBoundCheckResult(); + } + + void SetPinnedItersMgr(PinnedIteratorsManager* pinned_iters_mgr) override { + pinned_iters_mgr_ = pinned_iters_mgr; + for (auto& child : children_) { + child.iter.SetPinnedItersMgr(pinned_iters_mgr); + } + } + + bool IsDeleteRangeSentinelKey() const override { + assert(Valid()); + return current_->type == HeapItem::DELETE_RANGE_START; + } + + // Compaction uses the above subset of InternalIterator interface. + void SeekToLast() override { assert(false); } + + void SeekForPrev(const Slice&) override { assert(false); } + + void Prev() override { assert(false); } + + bool NextAndGetResult(IterateResult*) override { + assert(false); + return false; + } + + bool IsKeyPinned() const override { + assert(false); + return false; + } + + bool IsValuePinned() const override { + assert(false); + return false; + } + + bool PrepareValue() override { + assert(false); + return false; + } + + private: + struct HeapItem { + HeapItem() = default; + + IteratorWrapper iter; + size_t level = 0; + std::string tombstone_str; + enum Type { ITERATOR, DELETE_RANGE_START }; + Type type = ITERATOR; + + explicit HeapItem(size_t _level, InternalIteratorBase* _iter) + : level(_level), type(Type::ITERATOR) { + iter.Set(_iter); + } + + void SetTombstoneForCompaction(const ParsedInternalKey&& pik) { + tombstone_str.clear(); + AppendInternalKey(&tombstone_str, pik); + } + + [[nodiscard]] Slice key() const { + return type == ITERATOR ? iter.key() : tombstone_str; + } + }; + + class CompactionHeapItemComparator { + public: + explicit CompactionHeapItemComparator( + const InternalKeyComparator* comparator) + : comparator_(comparator) {} + + bool operator()(HeapItem* a, HeapItem* b) const { + int r = comparator_->Compare(a->key(), b->key()); + // For each file, we assume all range tombstone start keys come before + // its file boundary sentinel key (file's meta.largest key). + // In the case when meta.smallest = meta.largest and range tombstone start + // key is truncated at meta.smallest, the start key will have op_type = + // kMaxValid to make it smaller (see TruncatedRangeDelIterator + // constructor). The following assertion validates this assumption. + assert(a->type == b->type || r != 0); + return r > 0; + } + + private: + const InternalKeyComparator* comparator_; + }; + + using CompactionMinHeap = BinaryHeap; + bool is_arena_mode_; + const InternalKeyComparator* comparator_; + // HeapItem for all child point iterators. + std::vector children_; + // HeapItem for range tombstones. pinned_heap_item_[i] corresponds to the + // current range tombstone from range_tombstone_iters_[i]. + std::vector pinned_heap_item_; + // range_tombstone_iters_[i] contains range tombstones in the sorted run that + // corresponds to children_[i]. range_tombstone_iters_[i] == + // nullptr means the sorted run of children_[i] does not have range + // tombstones (or the current SSTable does not have range tombstones in the + // case of LevelIterator). + std::vector range_tombstone_iters_; + // Used as value for range tombstone keys + std::string dummy_tombstone_val{}; + + // Skip file boundary sentinel keys. + void FindNextVisibleKey(); + + // top of minHeap_ + HeapItem* current_; + // If any of the children have non-ok status, this is one of them. + Status status_; + CompactionMinHeap minHeap_; + PinnedIteratorsManager* pinned_iters_mgr_; + // Process a child that is not in the min heap. + // If valid, add to the min heap. Otherwise, check status. + void AddToMinHeapOrCheckStatus(HeapItem*); + + HeapItem* CurrentForward() const { + return !minHeap_.empty() ? minHeap_.top() : nullptr; + } + + void InsertRangeTombstoneAtLevel(size_t level) { + if (range_tombstone_iters_[level]->Valid()) { + pinned_heap_item_[level].SetTombstoneForCompaction( + range_tombstone_iters_[level]->start_key()); + minHeap_.push(&pinned_heap_item_[level]); + } + } +}; + +void CompactionMergingIterator::SeekToFirst() { + minHeap_.clear(); + status_ = Status::OK(); + for (auto& child : children_) { + child.iter.SeekToFirst(); + AddToMinHeapOrCheckStatus(&child); + } + + for (size_t i = 0; i < range_tombstone_iters_.size(); ++i) { + if (range_tombstone_iters_[i]) { + range_tombstone_iters_[i]->SeekToFirst(); + InsertRangeTombstoneAtLevel(i); + } + } + + FindNextVisibleKey(); + current_ = CurrentForward(); +} + +void CompactionMergingIterator::Seek(const Slice& target) { + minHeap_.clear(); + status_ = Status::OK(); + for (auto& child : children_) { + child.iter.Seek(target); + AddToMinHeapOrCheckStatus(&child); + } + + ParsedInternalKey pik; + ParseInternalKey(target, &pik, false /* log_err_key */) + .PermitUncheckedError(); + for (size_t i = 0; i < range_tombstone_iters_.size(); ++i) { + if (range_tombstone_iters_[i]) { + range_tombstone_iters_[i]->Seek(pik.user_key); + // For compaction, output keys should all be after seek target. + while (range_tombstone_iters_[i]->Valid() && + comparator_->Compare(range_tombstone_iters_[i]->start_key(), pik) < + 0) { + range_tombstone_iters_[i]->Next(); + } + InsertRangeTombstoneAtLevel(i); + } + } + + FindNextVisibleKey(); + current_ = CurrentForward(); +} + +void CompactionMergingIterator::Next() { + assert(Valid()); + // For the heap modifications below to be correct, current_ must be the + // current top of the heap. + assert(current_ == CurrentForward()); + // as the current points to the current record. move the iterator forward. + if (current_->type == HeapItem::ITERATOR) { + current_->iter.Next(); + if (current_->iter.Valid()) { + // current is still valid after the Next() call above. Call + // replace_top() to restore the heap property. When the same child + // iterator yields a sequence of keys, this is cheap. + assert(current_->iter.status().ok()); + minHeap_.replace_top(current_); + } else { + // current stopped being valid, remove it from the heap. + considerStatus(current_->iter.status()); + minHeap_.pop(); + } + } else { + assert(current_->type == HeapItem::DELETE_RANGE_START); + size_t level = current_->level; + assert(range_tombstone_iters_[level]); + range_tombstone_iters_[level]->Next(); + if (range_tombstone_iters_[level]->Valid()) { + pinned_heap_item_[level].SetTombstoneForCompaction( + range_tombstone_iters_[level]->start_key()); + minHeap_.replace_top(&pinned_heap_item_[level]); + } else { + minHeap_.pop(); + } + } + FindNextVisibleKey(); + current_ = CurrentForward(); +} + +void CompactionMergingIterator::FindNextVisibleKey() { + while (!minHeap_.empty()) { + HeapItem* current = minHeap_.top(); + // IsDeleteRangeSentinelKey() here means file boundary sentinel keys. + if (current->type != HeapItem::ITERATOR || + !current->iter.IsDeleteRangeSentinelKey()) { + return; + } + // range tombstone start keys from the same SSTable should have been + // exhausted + assert(!range_tombstone_iters_[current->level] || + !range_tombstone_iters_[current->level]->Valid()); + // current->iter is a LevelIterator, and it enters a new SST file in the + // Next() call here. + current->iter.Next(); + if (current->iter.Valid()) { + assert(current->iter.status().ok()); + minHeap_.replace_top(current); + } else { + minHeap_.pop(); + } + if (range_tombstone_iters_[current->level]) { + InsertRangeTombstoneAtLevel(current->level); + } + } +} + +void CompactionMergingIterator::AddToMinHeapOrCheckStatus(HeapItem* child) { + if (child->iter.Valid()) { + assert(child->iter.status().ok()); + minHeap_.push(child); + } else { + considerStatus(child->iter.status()); + } +} + +InternalIterator* NewCompactionMergingIterator( + const InternalKeyComparator* comparator, InternalIterator** children, int n, + std::vector>& range_tombstone_iters, + Arena* arena) { + assert(n >= 0); + if (n == 0) { + return NewEmptyInternalIterator(arena); + } else { + if (arena == nullptr) { + return new CompactionMergingIterator(comparator, children, n, + false /* is_arena_mode */, + range_tombstone_iters); + } else { + auto mem = arena->AllocateAligned(sizeof(CompactionMergingIterator)); + return new (mem) CompactionMergingIterator(comparator, children, n, + true /* is_arena_mode */, + range_tombstone_iters); + } + } +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/compaction_merging_iterator.h b/librocksdb-sys/rocksdb/table/compaction_merging_iterator.h new file mode 100644 index 0000000..e3fd779 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/compaction_merging_iterator.h @@ -0,0 +1,44 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "db/range_del_aggregator.h" +#include "rocksdb/slice.h" +#include "rocksdb/types.h" +#include "table/merging_iterator.h" + +namespace ROCKSDB_NAMESPACE { + +/* + * This is a simplified version of MergingIterator and is specifically used for + * compaction. It merges the input `children` iterators into a sorted stream of + * keys. Range tombstone start keys are also emitted to prevent oversize + * compactions. For example, consider an L1 file with content [a, b), y, z, + * where [a, b) is a range tombstone and y and z are point keys. This could + * cause an oversize compaction as it can overlap with a wide range of key space + * in L2. + * + * CompactionMergingIterator emits range tombstone start keys from each LSM + * level's range tombstone iterator, and for each range tombstone + * [start,end)@seqno, the key will be start@seqno with op_type + * kTypeRangeDeletion unless truncated at file boundary (see detail in + * TruncatedRangeDelIterator::start_key()). + * + * Caller should use CompactionMergingIterator::IsDeleteRangeSentinelKey() to + * check if the current key is a range tombstone key. + * TODO(cbi): IsDeleteRangeSentinelKey() is used for two kinds of keys at + * different layers: file boundary and range tombstone keys. Separate them into + * two APIs for clarity. + */ +class CompactionMergingIterator; + +InternalIterator* NewCompactionMergingIterator( + const InternalKeyComparator* comparator, InternalIterator** children, int n, + std::vector>& range_tombstone_iters, + Arena* arena = nullptr); +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_builder.cc b/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_builder.cc new file mode 100644 index 0000000..0cf6834 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_builder.cc @@ -0,0 +1,555 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "table/cuckoo/cuckoo_table_builder.h" + +#include + +#include +#include +#include +#include + +#include "db/dbformat.h" +#include "file/writable_file_writer.h" +#include "rocksdb/env.h" +#include "rocksdb/table.h" +#include "table/block_based/block_builder.h" +#include "table/cuckoo/cuckoo_table_factory.h" +#include "table/format.h" +#include "table/meta_blocks.h" +#include "util/autovector.h" +#include "util/random.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { +const std::string CuckooTablePropertyNames::kEmptyKey = + "rocksdb.cuckoo.bucket.empty.key"; +const std::string CuckooTablePropertyNames::kNumHashFunc = + "rocksdb.cuckoo.hash.num"; +const std::string CuckooTablePropertyNames::kHashTableSize = + "rocksdb.cuckoo.hash.size"; +const std::string CuckooTablePropertyNames::kValueLength = + "rocksdb.cuckoo.value.length"; +const std::string CuckooTablePropertyNames::kIsLastLevel = + "rocksdb.cuckoo.file.islastlevel"; +const std::string CuckooTablePropertyNames::kCuckooBlockSize = + "rocksdb.cuckoo.hash.cuckooblocksize"; +const std::string CuckooTablePropertyNames::kIdentityAsFirstHash = + "rocksdb.cuckoo.hash.identityfirst"; +const std::string CuckooTablePropertyNames::kUseModuleHash = + "rocksdb.cuckoo.hash.usemodule"; +const std::string CuckooTablePropertyNames::kUserKeyLength = + "rocksdb.cuckoo.hash.userkeylength"; + +// Obtained by running echo rocksdb.table.cuckoo | sha1sum +extern const uint64_t kCuckooTableMagicNumber = 0x926789d0c5f17873ull; + +CuckooTableBuilder::CuckooTableBuilder( + WritableFileWriter* file, double max_hash_table_ratio, + uint32_t max_num_hash_table, uint32_t max_search_depth, + const Comparator* user_comparator, uint32_t cuckoo_block_size, + bool use_module_hash, bool identity_as_first_hash, + uint64_t (*get_slice_hash)(const Slice&, uint32_t, uint64_t), + uint32_t column_family_id, const std::string& column_family_name, + const std::string& db_id, const std::string& db_session_id, + uint64_t file_number) + : num_hash_func_(2), + file_(file), + max_hash_table_ratio_(max_hash_table_ratio), + max_num_hash_func_(max_num_hash_table), + max_search_depth_(max_search_depth), + cuckoo_block_size_(std::max(1U, cuckoo_block_size)), + hash_table_size_(use_module_hash ? 0 : 2), + is_last_level_file_(false), + has_seen_first_key_(false), + has_seen_first_value_(false), + key_size_(0), + value_size_(0), + num_entries_(0), + num_values_(0), + ucomp_(user_comparator), + use_module_hash_(use_module_hash), + identity_as_first_hash_(identity_as_first_hash), + get_slice_hash_(get_slice_hash), + closed_(false) { + // Data is in a huge block. + properties_.num_data_blocks = 1; + properties_.index_size = 0; + properties_.filter_size = 0; + properties_.column_family_id = column_family_id; + properties_.column_family_name = column_family_name; + properties_.db_id = db_id; + properties_.db_session_id = db_session_id; + properties_.orig_file_number = file_number; + status_.PermitUncheckedError(); + io_status_.PermitUncheckedError(); +} + +void CuckooTableBuilder::Add(const Slice& key, const Slice& value) { + if (num_entries_ >= kMaxVectorIdx - 1) { + status_ = Status::NotSupported("Number of keys in a file must be < 2^32-1"); + return; + } + ParsedInternalKey ikey; + Status pik_status = + ParseInternalKey(key, &ikey, false /* log_err_key */); // TODO + if (!pik_status.ok()) { + status_ = Status::Corruption("Unable to parse key into internal key. ", + pik_status.getState()); + return; + } + if (ikey.type != kTypeDeletion && ikey.type != kTypeValue) { + status_ = Status::NotSupported("Unsupported key type " + + std::to_string(ikey.type)); + return; + } + + // Determine if we can ignore the sequence number and value type from + // internal keys by looking at sequence number from first key. We assume + // that if first key has a zero sequence number, then all the remaining + // keys will have zero seq. no. + if (!has_seen_first_key_) { + is_last_level_file_ = ikey.sequence == 0; + has_seen_first_key_ = true; + smallest_user_key_.assign(ikey.user_key.data(), ikey.user_key.size()); + largest_user_key_.assign(ikey.user_key.data(), ikey.user_key.size()); + key_size_ = is_last_level_file_ ? ikey.user_key.size() : key.size(); + } + if (key_size_ != (is_last_level_file_ ? ikey.user_key.size() : key.size())) { + status_ = Status::NotSupported("all keys have to be the same size"); + return; + } + + if (ikey.type == kTypeValue) { + if (!has_seen_first_value_) { + has_seen_first_value_ = true; + value_size_ = value.size(); + } + if (value_size_ != value.size()) { + status_ = Status::NotSupported("all values have to be the same size"); + return; + } + + if (is_last_level_file_) { + kvs_.append(ikey.user_key.data(), ikey.user_key.size()); + } else { + kvs_.append(key.data(), key.size()); + } + kvs_.append(value.data(), value.size()); + ++num_values_; + } else { + if (is_last_level_file_) { + deleted_keys_.append(ikey.user_key.data(), ikey.user_key.size()); + } else { + deleted_keys_.append(key.data(), key.size()); + } + } + ++num_entries_; + + // In order to fill the empty buckets in the hash table, we identify a + // key which is not used so far (unused_user_key). We determine this by + // maintaining smallest and largest keys inserted so far in bytewise order + // and use them to find a key outside this range in Finish() operation. + // Note that this strategy is independent of user comparator used here. + if (ikey.user_key.compare(smallest_user_key_) < 0) { + smallest_user_key_.assign(ikey.user_key.data(), ikey.user_key.size()); + } else if (ikey.user_key.compare(largest_user_key_) > 0) { + largest_user_key_.assign(ikey.user_key.data(), ikey.user_key.size()); + } + if (!use_module_hash_) { + if (hash_table_size_ < num_entries_ / max_hash_table_ratio_) { + hash_table_size_ *= 2; + } + } +} + +bool CuckooTableBuilder::IsDeletedKey(uint64_t idx) const { + assert(closed_); + return idx >= num_values_; +} + +Slice CuckooTableBuilder::GetKey(uint64_t idx) const { + assert(closed_); + if (IsDeletedKey(idx)) { + return Slice( + &deleted_keys_[static_cast((idx - num_values_) * key_size_)], + static_cast(key_size_)); + } + return Slice(&kvs_[static_cast(idx * (key_size_ + value_size_))], + static_cast(key_size_)); +} + +Slice CuckooTableBuilder::GetUserKey(uint64_t idx) const { + assert(closed_); + return is_last_level_file_ ? GetKey(idx) : ExtractUserKey(GetKey(idx)); +} + +Slice CuckooTableBuilder::GetValue(uint64_t idx) const { + assert(closed_); + if (IsDeletedKey(idx)) { + static std::string empty_value(static_cast(value_size_), 'a'); + return Slice(empty_value); + } + return Slice( + &kvs_[static_cast(idx * (key_size_ + value_size_) + key_size_)], + static_cast(value_size_)); +} + +Status CuckooTableBuilder::MakeHashTable(std::vector* buckets) { + buckets->resize( + static_cast(hash_table_size_ + cuckoo_block_size_ - 1)); + uint32_t make_space_for_key_call_id = 0; + for (uint32_t vector_idx = 0; vector_idx < num_entries_; vector_idx++) { + uint64_t bucket_id = 0; + bool bucket_found = false; + autovector hash_vals; + Slice user_key = GetUserKey(vector_idx); + for (uint32_t hash_cnt = 0; hash_cnt < num_hash_func_ && !bucket_found; + ++hash_cnt) { + uint64_t hash_val = + CuckooHash(user_key, hash_cnt, use_module_hash_, hash_table_size_, + identity_as_first_hash_, get_slice_hash_); + // If there is a collision, check next cuckoo_block_size_ locations for + // empty locations. While checking, if we reach end of the hash table, + // stop searching and proceed for next hash function. + for (uint32_t block_idx = 0; block_idx < cuckoo_block_size_; + ++block_idx, ++hash_val) { + if ((*buckets)[static_cast(hash_val)].vector_idx == + kMaxVectorIdx) { + bucket_id = hash_val; + bucket_found = true; + break; + } else { + if (ucomp_->Compare( + user_key, GetUserKey((*buckets)[static_cast(hash_val)] + .vector_idx)) == 0) { + return Status::NotSupported("Same key is being inserted again."); + } + hash_vals.push_back(hash_val); + } + } + } + while (!bucket_found && + !MakeSpaceForKey(hash_vals, ++make_space_for_key_call_id, buckets, + &bucket_id)) { + // Rehash by increashing number of hash tables. + if (num_hash_func_ >= max_num_hash_func_) { + return Status::NotSupported("Too many collisions. Unable to hash."); + } + // We don't really need to rehash the entire table because old hashes are + // still valid and we only increased the number of hash functions. + uint64_t hash_val = CuckooHash(user_key, num_hash_func_, use_module_hash_, + hash_table_size_, identity_as_first_hash_, + get_slice_hash_); + ++num_hash_func_; + for (uint32_t block_idx = 0; block_idx < cuckoo_block_size_; + ++block_idx, ++hash_val) { + if ((*buckets)[static_cast(hash_val)].vector_idx == + kMaxVectorIdx) { + bucket_found = true; + bucket_id = hash_val; + break; + } else { + hash_vals.push_back(hash_val); + } + } + } + (*buckets)[static_cast(bucket_id)].vector_idx = vector_idx; + } + return Status::OK(); +} + +Status CuckooTableBuilder::Finish() { + assert(!closed_); + closed_ = true; + std::vector buckets; + std::string unused_bucket; + if (num_entries_ > 0) { + // Calculate the real hash size if module hash is enabled. + if (use_module_hash_) { + hash_table_size_ = + static_cast(num_entries_ / max_hash_table_ratio_); + } + status_ = MakeHashTable(&buckets); + if (!status_.ok()) { + return status_; + } + // Determine unused_user_key to fill empty buckets. + std::string unused_user_key = smallest_user_key_; + int curr_pos = static_cast(unused_user_key.size()) - 1; + while (curr_pos >= 0) { + --unused_user_key[curr_pos]; + if (Slice(unused_user_key).compare(smallest_user_key_) < 0) { + break; + } + --curr_pos; + } + if (curr_pos < 0) { + // Try using the largest key to identify an unused key. + unused_user_key = largest_user_key_; + curr_pos = static_cast(unused_user_key.size()) - 1; + while (curr_pos >= 0) { + ++unused_user_key[curr_pos]; + if (Slice(unused_user_key).compare(largest_user_key_) > 0) { + break; + } + --curr_pos; + } + } + if (curr_pos < 0) { + return Status::Corruption("Unable to find unused key"); + } + if (is_last_level_file_) { + unused_bucket = unused_user_key; + } else { + ParsedInternalKey ikey(unused_user_key, 0, kTypeValue); + AppendInternalKey(&unused_bucket, ikey); + } + } + properties_.num_entries = num_entries_; + properties_.num_deletions = num_entries_ - num_values_; + properties_.fixed_key_len = key_size_; + properties_.user_collected_properties[CuckooTablePropertyNames::kValueLength] + .assign(reinterpret_cast(&value_size_), sizeof(value_size_)); + + uint64_t bucket_size = key_size_ + value_size_; + unused_bucket.resize(static_cast(bucket_size), 'a'); + // Write the table. + uint32_t num_added = 0; + for (auto& bucket : buckets) { + if (bucket.vector_idx == kMaxVectorIdx) { + io_status_ = file_->Append(Slice(unused_bucket)); + } else { + ++num_added; + io_status_ = file_->Append(GetKey(bucket.vector_idx)); + if (io_status_.ok()) { + if (value_size_ > 0) { + io_status_ = file_->Append(GetValue(bucket.vector_idx)); + } + } + } + if (!io_status_.ok()) { + status_ = io_status_; + return status_; + } + } + assert(num_added == NumEntries()); + properties_.raw_key_size = num_added * properties_.fixed_key_len; + properties_.raw_value_size = num_added * value_size_; + + uint64_t offset = buckets.size() * bucket_size; + properties_.data_size = offset; + unused_bucket.resize(static_cast(properties_.fixed_key_len)); + properties_.user_collected_properties[CuckooTablePropertyNames::kEmptyKey] = + unused_bucket; + properties_.user_collected_properties[CuckooTablePropertyNames::kNumHashFunc] + .assign(reinterpret_cast(&num_hash_func_), sizeof(num_hash_func_)); + + properties_ + .user_collected_properties[CuckooTablePropertyNames::kHashTableSize] + .assign(reinterpret_cast(&hash_table_size_), + sizeof(hash_table_size_)); + properties_.user_collected_properties[CuckooTablePropertyNames::kIsLastLevel] + .assign(reinterpret_cast(&is_last_level_file_), + sizeof(is_last_level_file_)); + properties_ + .user_collected_properties[CuckooTablePropertyNames::kCuckooBlockSize] + .assign(reinterpret_cast(&cuckoo_block_size_), + sizeof(cuckoo_block_size_)); + properties_ + .user_collected_properties[CuckooTablePropertyNames::kIdentityAsFirstHash] + .assign(reinterpret_cast(&identity_as_first_hash_), + sizeof(identity_as_first_hash_)); + properties_ + .user_collected_properties[CuckooTablePropertyNames::kUseModuleHash] + .assign(reinterpret_cast(&use_module_hash_), + sizeof(use_module_hash_)); + uint32_t user_key_len = static_cast(smallest_user_key_.size()); + properties_ + .user_collected_properties[CuckooTablePropertyNames::kUserKeyLength] + .assign(reinterpret_cast(&user_key_len), + sizeof(user_key_len)); + + // Write meta blocks. + MetaIndexBuilder meta_index_builder; + PropertyBlockBuilder property_block_builder; + + property_block_builder.AddTableProperty(properties_); + property_block_builder.Add(properties_.user_collected_properties); + Slice property_block = property_block_builder.Finish(); + BlockHandle property_block_handle; + property_block_handle.set_offset(offset); + property_block_handle.set_size(property_block.size()); + io_status_ = file_->Append(property_block); + offset += property_block.size(); + if (!io_status_.ok()) { + status_ = io_status_; + return status_; + } + + meta_index_builder.Add(kPropertiesBlockName, property_block_handle); + Slice meta_index_block = meta_index_builder.Finish(); + + BlockHandle meta_index_block_handle; + meta_index_block_handle.set_offset(offset); + meta_index_block_handle.set_size(meta_index_block.size()); + io_status_ = file_->Append(meta_index_block); + if (!io_status_.ok()) { + status_ = io_status_; + return status_; + } + + FooterBuilder footer; + Status s = footer.Build(kCuckooTableMagicNumber, /* format_version */ 1, + offset, kNoChecksum, meta_index_block_handle); + if (!s.ok()) { + status_ = s; + return status_; + } + io_status_ = file_->Append(footer.GetSlice()); + status_ = io_status_; + return status_; +} + +void CuckooTableBuilder::Abandon() { + assert(!closed_); + closed_ = true; +} + +uint64_t CuckooTableBuilder::NumEntries() const { return num_entries_; } + +uint64_t CuckooTableBuilder::FileSize() const { + if (closed_) { + return file_->GetFileSize(); + } else if (num_entries_ == 0) { + return 0; + } + + if (use_module_hash_) { + return static_cast((key_size_ + value_size_) * num_entries_ / + max_hash_table_ratio_); + } else { + // Account for buckets being a power of two. + // As elements are added, file size remains constant for a while and + // doubles its size. Since compaction algorithm stops adding elements + // only after it exceeds the file limit, we account for the extra element + // being added here. + uint64_t expected_hash_table_size = hash_table_size_; + if (expected_hash_table_size < (num_entries_ + 1) / max_hash_table_ratio_) { + expected_hash_table_size *= 2; + } + return (key_size_ + value_size_) * expected_hash_table_size - 1; + } +} + +// This method is invoked when there is no place to insert the target key. +// It searches for a set of elements that can be moved to accommodate target +// key. The search is a BFS graph traversal with first level (hash_vals) +// being all the buckets target key could go to. +// Then, from each node (curr_node), we find all the buckets that curr_node +// could go to. They form the children of curr_node in the tree. +// We continue the traversal until we find an empty bucket, in which case, we +// move all elements along the path from first level to this empty bucket, to +// make space for target key which is inserted at first level (*bucket_id). +// If tree depth exceedes max depth, we return false indicating failure. +bool CuckooTableBuilder::MakeSpaceForKey( + const autovector& hash_vals, + const uint32_t make_space_for_key_call_id, + std::vector* buckets, uint64_t* bucket_id) { + struct CuckooNode { + uint64_t bucket_id; + uint32_t depth; + uint32_t parent_pos; + CuckooNode(uint64_t _bucket_id, uint32_t _depth, int _parent_pos) + : bucket_id(_bucket_id), depth(_depth), parent_pos(_parent_pos) {} + }; + // This is BFS search tree that is stored simply as a vector. + // Each node stores the index of parent node in the vector. + std::vector tree; + // We want to identify already visited buckets in the current method call so + // that we don't add same buckets again for exploration in the tree. + // We do this by maintaining a count of current method call in + // make_space_for_key_call_id, which acts as a unique id for this invocation + // of the method. We store this number into the nodes that we explore in + // current method call. + // It is unlikely for the increment operation to overflow because the maximum + // no. of times this will be called is <= max_num_hash_func_ + num_entries_. + for (uint32_t hash_cnt = 0; hash_cnt < num_hash_func_; ++hash_cnt) { + uint64_t bid = hash_vals[hash_cnt]; + (*buckets)[static_cast(bid)].make_space_for_key_call_id = + make_space_for_key_call_id; + tree.push_back(CuckooNode(bid, 0, 0)); + } + bool null_found = false; + uint32_t curr_pos = 0; + while (!null_found && curr_pos < tree.size()) { + CuckooNode& curr_node = tree[curr_pos]; + uint32_t curr_depth = curr_node.depth; + if (curr_depth >= max_search_depth_) { + break; + } + CuckooBucket& curr_bucket = + (*buckets)[static_cast(curr_node.bucket_id)]; + for (uint32_t hash_cnt = 0; hash_cnt < num_hash_func_ && !null_found; + ++hash_cnt) { + uint64_t child_bucket_id = CuckooHash( + GetUserKey(curr_bucket.vector_idx), hash_cnt, use_module_hash_, + hash_table_size_, identity_as_first_hash_, get_slice_hash_); + // Iterate inside Cuckoo Block. + for (uint32_t block_idx = 0; block_idx < cuckoo_block_size_; + ++block_idx, ++child_bucket_id) { + if ((*buckets)[static_cast(child_bucket_id)] + .make_space_for_key_call_id == make_space_for_key_call_id) { + continue; + } + (*buckets)[static_cast(child_bucket_id)] + .make_space_for_key_call_id = make_space_for_key_call_id; + tree.push_back(CuckooNode(child_bucket_id, curr_depth + 1, curr_pos)); + if ((*buckets)[static_cast(child_bucket_id)].vector_idx == + kMaxVectorIdx) { + null_found = true; + break; + } + } + } + ++curr_pos; + } + + if (null_found) { + // There is an empty node in tree.back(). Now, traverse the path from this + // empty node to top of the tree and at every node in the path, replace + // child with the parent. Stop when first level is reached in the tree + // (happens when 0 <= bucket_to_replace_pos < num_hash_func_) and return + // this location in first level for target key to be inserted. + uint32_t bucket_to_replace_pos = static_cast(tree.size()) - 1; + while (bucket_to_replace_pos >= num_hash_func_) { + CuckooNode& curr_node = tree[bucket_to_replace_pos]; + (*buckets)[static_cast(curr_node.bucket_id)] = + (*buckets)[static_cast(tree[curr_node.parent_pos].bucket_id)]; + bucket_to_replace_pos = curr_node.parent_pos; + } + *bucket_id = tree[bucket_to_replace_pos].bucket_id; + } + return null_found; +} + +std::string CuckooTableBuilder::GetFileChecksum() const { + if (file_ != nullptr) { + return file_->GetFileChecksum(); + } else { + return kUnknownFileChecksum; + } +} + +const char* CuckooTableBuilder::GetFileChecksumFuncName() const { + if (file_ != nullptr) { + return file_->GetFileChecksumFuncName(); + } else { + return kUnknownFileChecksumFuncName; + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_builder.h b/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_builder.h new file mode 100644 index 0000000..3a19dd6 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_builder.h @@ -0,0 +1,136 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once +#include + +#include +#include +#include +#include + +#include "db/version_edit.h" +#include "port/port.h" +#include "rocksdb/status.h" +#include "rocksdb/table.h" +#include "rocksdb/table_properties.h" +#include "table/table_builder.h" +#include "util/autovector.h" + +namespace ROCKSDB_NAMESPACE { + +class CuckooTableBuilder : public TableBuilder { + public: + CuckooTableBuilder( + WritableFileWriter* file, double max_hash_table_ratio, + uint32_t max_num_hash_func, uint32_t max_search_depth, + const Comparator* user_comparator, uint32_t cuckoo_block_size, + bool use_module_hash, bool identity_as_first_hash, + uint64_t (*get_slice_hash)(const Slice&, uint32_t, uint64_t), + uint32_t column_family_id, const std::string& column_family_name, + const std::string& db_id = "", const std::string& db_session_id = "", + uint64_t file_number = 0); + // No copying allowed + CuckooTableBuilder(const CuckooTableBuilder&) = delete; + void operator=(const CuckooTableBuilder&) = delete; + + // REQUIRES: Either Finish() or Abandon() has been called. + ~CuckooTableBuilder() {} + + // Add key,value to the table being constructed. + // REQUIRES: key is after any previously added key according to comparator. + // REQUIRES: Finish(), Abandon() have not been called + void Add(const Slice& key, const Slice& value) override; + + // Return non-ok iff some error has been detected. + Status status() const override { return status_; } + + // Return non-ok iff some error happens during IO. + IOStatus io_status() const override { return io_status_; } + + // Finish building the table. Stops using the file passed to the + // constructor after this function returns. + // REQUIRES: Finish(), Abandon() have not been called + Status Finish() override; + + // Indicate that the contents of this builder should be abandoned. Stops + // using the file passed to the constructor after this function returns. + // If the caller is not going to call Finish(), it must call Abandon() + // before destroying this builder. + // REQUIRES: Finish(), Abandon() have not been called + void Abandon() override; + + // Number of calls to Add() so far. + uint64_t NumEntries() const override; + + // Size of the file generated so far. If invoked after a successful + // Finish() call, returns the size of the final generated file. + uint64_t FileSize() const override; + + TableProperties GetTableProperties() const override { return properties_; } + + // Get file checksum + std::string GetFileChecksum() const override; + + // Get file checksum function name + const char* GetFileChecksumFuncName() const override; + + private: + struct CuckooBucket { + CuckooBucket() : vector_idx(kMaxVectorIdx), make_space_for_key_call_id(0) {} + uint32_t vector_idx; + // This number will not exceed kvs_.size() + max_num_hash_func_. + // We assume number of items is <= 2^32. + uint32_t make_space_for_key_call_id; + }; + static const uint32_t kMaxVectorIdx = std::numeric_limits::max(); + + bool MakeSpaceForKey(const autovector& hash_vals, + const uint32_t call_id, + std::vector* buckets, uint64_t* bucket_id); + Status MakeHashTable(std::vector* buckets); + + inline bool IsDeletedKey(uint64_t idx) const; + inline Slice GetKey(uint64_t idx) const; + inline Slice GetUserKey(uint64_t idx) const; + inline Slice GetValue(uint64_t idx) const; + + uint32_t num_hash_func_; + WritableFileWriter* file_; + const double max_hash_table_ratio_; + const uint32_t max_num_hash_func_; + const uint32_t max_search_depth_; + const uint32_t cuckoo_block_size_; + uint64_t hash_table_size_; + bool is_last_level_file_; + bool has_seen_first_key_; + bool has_seen_first_value_; + uint64_t key_size_; + uint64_t value_size_; + // A list of fixed-size key-value pairs concatenating into a string. + // Use GetKey(), GetUserKey(), and GetValue() to retrieve a specific + // key / value given an index + std::string kvs_; + std::string deleted_keys_; + // Number of key-value pairs stored in kvs_ + number of deleted keys + uint64_t num_entries_; + // Number of keys that contain value (non-deletion op) + uint64_t num_values_; + Status status_; + IOStatus io_status_; + TableProperties properties_; + const Comparator* ucomp_; + bool use_module_hash_; + bool identity_as_first_hash_; + uint64_t (*get_slice_hash_)(const Slice& s, uint32_t index, + uint64_t max_num_buckets); + std::string largest_user_key_ = ""; + std::string smallest_user_key_ = ""; + + bool closed_; // Either Finish() or Abandon() has been called. +}; + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_builder_test.cc b/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_builder_test.cc new file mode 100644 index 0000000..ceddbf3 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_builder_test.cc @@ -0,0 +1,631 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + + +#include "table/cuckoo/cuckoo_table_builder.h" + +#include +#include +#include +#include + +#include "file/random_access_file_reader.h" +#include "file/writable_file_writer.h" +#include "rocksdb/db.h" +#include "rocksdb/file_system.h" +#include "table/meta_blocks.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" + +namespace ROCKSDB_NAMESPACE { +extern const uint64_t kCuckooTableMagicNumber; + +namespace { +std::unordered_map> hash_map; + +uint64_t GetSliceHash(const Slice& s, uint32_t index, + uint64_t /*max_num_buckets*/) { + return hash_map[s.ToString()][index]; +} +} // namespace + +class CuckooBuilderTest : public testing::Test { + public: + CuckooBuilderTest() { + env_ = Env::Default(); + Options options; + options.allow_mmap_reads = true; + file_options_ = FileOptions(options); + } + + void CheckFileContents(const std::vector& keys, + const std::vector& values, + const std::vector& expected_locations, + std::string expected_unused_bucket, + uint64_t expected_table_size, + uint32_t expected_num_hash_func, + bool expected_is_last_level, + uint32_t expected_cuckoo_block_size = 1) { + uint64_t num_deletions = 0; + for (const auto& key : keys) { + ParsedInternalKey parsed; + Status pik_status = + ParseInternalKey(key, &parsed, true /* log_err_key */); + if (pik_status.ok() && parsed.type == kTypeDeletion) { + num_deletions++; + } + } + // Read file + uint64_t read_file_size; + ASSERT_OK(env_->GetFileSize(fname, &read_file_size)); + std::unique_ptr file_reader; + ASSERT_OK(RandomAccessFileReader::Create( + env_->GetFileSystem(), fname, file_options_, &file_reader, nullptr)); + + Options options; + options.allow_mmap_reads = true; + ImmutableOptions ioptions(options); + + // Assert Table Properties. + std::unique_ptr props; + const ReadOptions read_options; + ASSERT_OK(ReadTableProperties(file_reader.get(), read_file_size, + kCuckooTableMagicNumber, ioptions, + read_options, &props)); + // Check unused bucket. + std::string unused_key = + props->user_collected_properties[CuckooTablePropertyNames::kEmptyKey]; + ASSERT_EQ(expected_unused_bucket.substr(0, props->fixed_key_len), + unused_key); + + uint64_t value_len_found = *reinterpret_cast( + props->user_collected_properties[CuckooTablePropertyNames::kValueLength] + .data()); + ASSERT_EQ(values.empty() ? 0 : values[0].size(), value_len_found); + ASSERT_EQ(props->raw_value_size, values.size() * value_len_found); + const uint64_t table_size = *reinterpret_cast( + props + ->user_collected_properties + [CuckooTablePropertyNames::kHashTableSize] + .data()); + ASSERT_EQ(expected_table_size, table_size); + const uint32_t num_hash_func_found = *reinterpret_cast( + props->user_collected_properties[CuckooTablePropertyNames::kNumHashFunc] + .data()); + ASSERT_EQ(expected_num_hash_func, num_hash_func_found); + const uint32_t cuckoo_block_size = *reinterpret_cast( + props + ->user_collected_properties + [CuckooTablePropertyNames::kCuckooBlockSize] + .data()); + ASSERT_EQ(expected_cuckoo_block_size, cuckoo_block_size); + const bool is_last_level_found = *reinterpret_cast( + props->user_collected_properties[CuckooTablePropertyNames::kIsLastLevel] + .data()); + ASSERT_EQ(expected_is_last_level, is_last_level_found); + + ASSERT_EQ(props->num_entries, keys.size()); + ASSERT_EQ(props->num_deletions, num_deletions); + ASSERT_EQ(props->fixed_key_len, keys.empty() ? 0 : keys[0].size()); + ASSERT_EQ(props->data_size, + expected_unused_bucket.size() * + (expected_table_size + expected_cuckoo_block_size - 1)); + ASSERT_EQ(props->raw_key_size, keys.size() * props->fixed_key_len); + ASSERT_EQ(props->column_family_id, 0); + ASSERT_EQ(props->column_family_name, kDefaultColumnFamilyName); + + // Check contents of the bucket. + std::vector keys_found(keys.size(), false); + size_t bucket_size = expected_unused_bucket.size(); + for (uint32_t i = 0; i + 1 < table_size + cuckoo_block_size; ++i) { + Slice read_slice; + ASSERT_OK(file_reader->Read(IOOptions(), i * bucket_size, bucket_size, + &read_slice, nullptr, nullptr, + Env::IO_TOTAL /* rate_limiter_priority */)); + size_t key_idx = + std::find(expected_locations.begin(), expected_locations.end(), i) - + expected_locations.begin(); + if (key_idx == keys.size()) { + // i is not one of the expected locations. Empty bucket. + if (read_slice.data() == nullptr) { + ASSERT_EQ(0, expected_unused_bucket.size()); + } else { + ASSERT_EQ(read_slice.compare(expected_unused_bucket), 0); + } + } else { + keys_found[key_idx] = true; + ASSERT_EQ(read_slice.compare(keys[key_idx] + values[key_idx]), 0); + } + } + for (auto key_found : keys_found) { + // Check that all keys wereReader found. + ASSERT_TRUE(key_found); + } + } + + std::string GetInternalKey(Slice user_key, bool zero_seqno, + ValueType type = kTypeValue) { + IterKey ikey; + ikey.SetInternalKey(user_key, zero_seqno ? 0 : 1000, type); + return ikey.GetInternalKey().ToString(); + } + + uint64_t NextPowOf2(uint64_t num) { + uint64_t n = 2; + while (n <= num) { + n *= 2; + } + return n; + } + + uint64_t GetExpectedTableSize(uint64_t num) { + return NextPowOf2(static_cast(num / kHashTableRatio)); + } + + Env* env_; + FileOptions file_options_; + std::string fname; + const double kHashTableRatio = 0.9; +}; + +TEST_F(CuckooBuilderTest, SuccessWithEmptyFile) { + std::unique_ptr writable_file; + fname = test::PerThreadDBPath("EmptyFile"); + std::unique_ptr file_writer; + ASSERT_OK(WritableFileWriter::Create(env_->GetFileSystem(), fname, + file_options_, &file_writer, nullptr)); + CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, 4, 100, + BytewiseComparator(), 1, false, false, + GetSliceHash, 0 /* column_family_id */, + kDefaultColumnFamilyName); + ASSERT_OK(builder.status()); + ASSERT_EQ(0UL, builder.FileSize()); + ASSERT_OK(builder.Finish()); + ASSERT_OK(file_writer->Close()); + CheckFileContents({}, {}, {}, "", 2, 2, false); +} + +TEST_F(CuckooBuilderTest, WriteSuccessNoCollisionFullKey) { + for (auto type : {kTypeValue, kTypeDeletion}) { + uint32_t num_hash_fun = 4; + std::vector user_keys = {"key01", "key02", "key03", "key04"}; + std::vector values; + if (type == kTypeValue) { + values = {"v01", "v02", "v03", "v04"}; + } else { + values = {"", "", "", ""}; + } + // Need to have a temporary variable here as VS compiler does not currently + // support operator= with initializer_list as a parameter + std::unordered_map> hm = { + {user_keys[0], {0, 1, 2, 3}}, + {user_keys[1], {1, 2, 3, 4}}, + {user_keys[2], {2, 3, 4, 5}}, + {user_keys[3], {3, 4, 5, 6}}}; + hash_map = std::move(hm); + + std::vector expected_locations = {0, 1, 2, 3}; + std::vector keys; + for (auto& user_key : user_keys) { + keys.push_back(GetInternalKey(user_key, false, type)); + } + uint64_t expected_table_size = GetExpectedTableSize(keys.size()); + + fname = test::PerThreadDBPath("NoCollisionFullKey"); + std::unique_ptr file_writer; + ASSERT_OK(WritableFileWriter::Create(env_->GetFileSystem(), fname, + file_options_, &file_writer, nullptr)); + CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun, + 100, BytewiseComparator(), 1, false, false, + GetSliceHash, 0 /* column_family_id */, + kDefaultColumnFamilyName); + ASSERT_OK(builder.status()); + for (uint32_t i = 0; i < user_keys.size(); i++) { + builder.Add(Slice(keys[i]), Slice(values[i])); + ASSERT_EQ(builder.NumEntries(), i + 1); + ASSERT_OK(builder.status()); + } + size_t bucket_size = keys[0].size() + values[0].size(); + ASSERT_EQ(expected_table_size * bucket_size - 1, builder.FileSize()); + ASSERT_OK(builder.Finish()); + ASSERT_OK(file_writer->Close()); + ASSERT_LE(expected_table_size * bucket_size, builder.FileSize()); + + std::string expected_unused_bucket = GetInternalKey("key00", true); + expected_unused_bucket += std::string(values[0].size(), 'a'); + CheckFileContents(keys, values, expected_locations, expected_unused_bucket, + expected_table_size, 2, false); + } +} + +TEST_F(CuckooBuilderTest, WriteSuccessWithCollisionFullKey) { + uint32_t num_hash_fun = 4; + std::vector user_keys = {"key01", "key02", "key03", "key04"}; + std::vector values = {"v01", "v02", "v03", "v04"}; + // Need to have a temporary variable here as VS compiler does not currently + // support operator= with initializer_list as a parameter + std::unordered_map> hm = { + {user_keys[0], {0, 1, 2, 3}}, + {user_keys[1], {0, 1, 2, 3}}, + {user_keys[2], {0, 1, 2, 3}}, + {user_keys[3], {0, 1, 2, 3}}, + }; + hash_map = std::move(hm); + + std::vector expected_locations = {0, 1, 2, 3}; + std::vector keys; + for (auto& user_key : user_keys) { + keys.push_back(GetInternalKey(user_key, false)); + } + uint64_t expected_table_size = GetExpectedTableSize(keys.size()); + + fname = test::PerThreadDBPath("WithCollisionFullKey"); + std::unique_ptr file_writer; + ASSERT_OK(WritableFileWriter::Create(env_->GetFileSystem(), fname, + file_options_, &file_writer, nullptr)); + CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun, + 100, BytewiseComparator(), 1, false, false, + GetSliceHash, 0 /* column_family_id */, + kDefaultColumnFamilyName); + ASSERT_OK(builder.status()); + for (uint32_t i = 0; i < user_keys.size(); i++) { + builder.Add(Slice(keys[i]), Slice(values[i])); + ASSERT_EQ(builder.NumEntries(), i + 1); + ASSERT_OK(builder.status()); + } + size_t bucket_size = keys[0].size() + values[0].size(); + ASSERT_EQ(expected_table_size * bucket_size - 1, builder.FileSize()); + ASSERT_OK(builder.Finish()); + ASSERT_OK(file_writer->Close()); + ASSERT_LE(expected_table_size * bucket_size, builder.FileSize()); + + std::string expected_unused_bucket = GetInternalKey("key00", true); + expected_unused_bucket += std::string(values[0].size(), 'a'); + CheckFileContents(keys, values, expected_locations, expected_unused_bucket, + expected_table_size, 4, false); +} + +TEST_F(CuckooBuilderTest, WriteSuccessWithCollisionAndCuckooBlock) { + uint32_t num_hash_fun = 4; + std::vector user_keys = {"key01", "key02", "key03", "key04"}; + std::vector values = {"v01", "v02", "v03", "v04"}; + // Need to have a temporary variable here as VS compiler does not currently + // support operator= with initializer_list as a parameter + std::unordered_map> hm = { + {user_keys[0], {0, 1, 2, 3}}, + {user_keys[1], {0, 1, 2, 3}}, + {user_keys[2], {0, 1, 2, 3}}, + {user_keys[3], {0, 1, 2, 3}}, + }; + hash_map = std::move(hm); + + std::vector expected_locations = {0, 1, 2, 3}; + std::vector keys; + for (auto& user_key : user_keys) { + keys.push_back(GetInternalKey(user_key, false)); + } + uint64_t expected_table_size = GetExpectedTableSize(keys.size()); + + std::unique_ptr file_writer; + uint32_t cuckoo_block_size = 2; + fname = test::PerThreadDBPath("WithCollisionFullKey2"); + ASSERT_OK(WritableFileWriter::Create(env_->GetFileSystem(), fname, + file_options_, &file_writer, nullptr)); + CuckooTableBuilder builder( + file_writer.get(), kHashTableRatio, num_hash_fun, 100, + BytewiseComparator(), cuckoo_block_size, false, false, GetSliceHash, + 0 /* column_family_id */, kDefaultColumnFamilyName); + ASSERT_OK(builder.status()); + for (uint32_t i = 0; i < user_keys.size(); i++) { + builder.Add(Slice(keys[i]), Slice(values[i])); + ASSERT_EQ(builder.NumEntries(), i + 1); + ASSERT_OK(builder.status()); + } + size_t bucket_size = keys[0].size() + values[0].size(); + ASSERT_EQ(expected_table_size * bucket_size - 1, builder.FileSize()); + ASSERT_OK(builder.Finish()); + ASSERT_OK(file_writer->Close()); + ASSERT_LE(expected_table_size * bucket_size, builder.FileSize()); + + std::string expected_unused_bucket = GetInternalKey("key00", true); + expected_unused_bucket += std::string(values[0].size(), 'a'); + CheckFileContents(keys, values, expected_locations, expected_unused_bucket, + expected_table_size, 3, false, cuckoo_block_size); +} + +TEST_F(CuckooBuilderTest, WithCollisionPathFullKey) { + // Have two hash functions. Insert elements with overlapping hashes. + // Finally insert an element with hash value somewhere in the middle + // so that it displaces all the elements after that. + uint32_t num_hash_fun = 2; + std::vector user_keys = {"key01", "key02", "key03", "key04", + "key05"}; + std::vector values = {"v01", "v02", "v03", "v04", "v05"}; + // Need to have a temporary variable here as VS compiler does not currently + // support operator= with initializer_list as a parameter + std::unordered_map> hm = { + {user_keys[0], {0, 1}}, {user_keys[1], {1, 2}}, {user_keys[2], {2, 3}}, + {user_keys[3], {3, 4}}, {user_keys[4], {0, 2}}, + }; + hash_map = std::move(hm); + + std::vector expected_locations = {0, 1, 3, 4, 2}; + std::vector keys; + for (auto& user_key : user_keys) { + keys.push_back(GetInternalKey(user_key, false)); + } + uint64_t expected_table_size = GetExpectedTableSize(keys.size()); + + std::unique_ptr file_writer; + fname = test::PerThreadDBPath("WithCollisionPathFullKey"); + ASSERT_OK(WritableFileWriter::Create(env_->GetFileSystem(), fname, + file_options_, &file_writer, nullptr)); + CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun, + 100, BytewiseComparator(), 1, false, false, + GetSliceHash, 0 /* column_family_id */, + kDefaultColumnFamilyName); + ASSERT_OK(builder.status()); + for (uint32_t i = 0; i < user_keys.size(); i++) { + builder.Add(Slice(keys[i]), Slice(values[i])); + ASSERT_EQ(builder.NumEntries(), i + 1); + ASSERT_OK(builder.status()); + } + size_t bucket_size = keys[0].size() + values[0].size(); + ASSERT_EQ(expected_table_size * bucket_size - 1, builder.FileSize()); + ASSERT_OK(builder.Finish()); + ASSERT_OK(file_writer->Close()); + ASSERT_LE(expected_table_size * bucket_size, builder.FileSize()); + + std::string expected_unused_bucket = GetInternalKey("key00", true); + expected_unused_bucket += std::string(values[0].size(), 'a'); + CheckFileContents(keys, values, expected_locations, expected_unused_bucket, + expected_table_size, 2, false); +} + +TEST_F(CuckooBuilderTest, WithCollisionPathFullKeyAndCuckooBlock) { + uint32_t num_hash_fun = 2; + std::vector user_keys = {"key01", "key02", "key03", "key04", + "key05"}; + std::vector values = {"v01", "v02", "v03", "v04", "v05"}; + // Need to have a temporary variable here as VS compiler does not currently + // support operator= with initializer_list as a parameter + std::unordered_map> hm = { + {user_keys[0], {0, 1}}, {user_keys[1], {1, 2}}, {user_keys[2], {3, 4}}, + {user_keys[3], {4, 5}}, {user_keys[4], {0, 3}}, + }; + hash_map = std::move(hm); + + std::vector expected_locations = {2, 1, 3, 4, 0}; + std::vector keys; + for (auto& user_key : user_keys) { + keys.push_back(GetInternalKey(user_key, false)); + } + uint64_t expected_table_size = GetExpectedTableSize(keys.size()); + + std::unique_ptr file_writer; + fname = test::PerThreadDBPath("WithCollisionPathFullKeyAndCuckooBlock"); + ASSERT_OK(WritableFileWriter::Create(env_->GetFileSystem(), fname, + file_options_, &file_writer, nullptr)); + CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun, + 100, BytewiseComparator(), 2, false, false, + GetSliceHash, 0 /* column_family_id */, + kDefaultColumnFamilyName); + ASSERT_OK(builder.status()); + for (uint32_t i = 0; i < user_keys.size(); i++) { + builder.Add(Slice(keys[i]), Slice(values[i])); + ASSERT_EQ(builder.NumEntries(), i + 1); + ASSERT_OK(builder.status()); + } + size_t bucket_size = keys[0].size() + values[0].size(); + ASSERT_EQ(expected_table_size * bucket_size - 1, builder.FileSize()); + ASSERT_OK(builder.Finish()); + ASSERT_OK(file_writer->Close()); + ASSERT_LE(expected_table_size * bucket_size, builder.FileSize()); + + std::string expected_unused_bucket = GetInternalKey("key00", true); + expected_unused_bucket += std::string(values[0].size(), 'a'); + CheckFileContents(keys, values, expected_locations, expected_unused_bucket, + expected_table_size, 2, false, 2); +} + +TEST_F(CuckooBuilderTest, WriteSuccessNoCollisionUserKey) { + uint32_t num_hash_fun = 4; + std::vector user_keys = {"key01", "key02", "key03", "key04"}; + std::vector values = {"v01", "v02", "v03", "v04"}; + // Need to have a temporary variable here as VS compiler does not currently + // support operator= with initializer_list as a parameter + std::unordered_map> hm = { + {user_keys[0], {0, 1, 2, 3}}, + {user_keys[1], {1, 2, 3, 4}}, + {user_keys[2], {2, 3, 4, 5}}, + {user_keys[3], {3, 4, 5, 6}}}; + hash_map = std::move(hm); + + std::vector expected_locations = {0, 1, 2, 3}; + uint64_t expected_table_size = GetExpectedTableSize(user_keys.size()); + + std::unique_ptr file_writer; + fname = test::PerThreadDBPath("NoCollisionUserKey"); + ASSERT_OK(WritableFileWriter::Create(env_->GetFileSystem(), fname, + file_options_, &file_writer, nullptr)); + + CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun, + 100, BytewiseComparator(), 1, false, false, + GetSliceHash, 0 /* column_family_id */, + kDefaultColumnFamilyName); + ASSERT_OK(builder.status()); + for (uint32_t i = 0; i < user_keys.size(); i++) { + builder.Add(Slice(GetInternalKey(user_keys[i], true)), Slice(values[i])); + ASSERT_EQ(builder.NumEntries(), i + 1); + ASSERT_OK(builder.status()); + } + size_t bucket_size = user_keys[0].size() + values[0].size(); + ASSERT_EQ(expected_table_size * bucket_size - 1, builder.FileSize()); + ASSERT_OK(builder.Finish()); + ASSERT_OK(file_writer->Close()); + ASSERT_LE(expected_table_size * bucket_size, builder.FileSize()); + + std::string expected_unused_bucket = "key00"; + expected_unused_bucket += std::string(values[0].size(), 'a'); + CheckFileContents(user_keys, values, expected_locations, + expected_unused_bucket, expected_table_size, 2, true); +} + +TEST_F(CuckooBuilderTest, WriteSuccessWithCollisionUserKey) { + uint32_t num_hash_fun = 4; + std::vector user_keys = {"key01", "key02", "key03", "key04"}; + std::vector values = {"v01", "v02", "v03", "v04"}; + // Need to have a temporary variable here as VS compiler does not currently + // support operator= with initializer_list as a parameter + std::unordered_map> hm = { + {user_keys[0], {0, 1, 2, 3}}, + {user_keys[1], {0, 1, 2, 3}}, + {user_keys[2], {0, 1, 2, 3}}, + {user_keys[3], {0, 1, 2, 3}}, + }; + hash_map = std::move(hm); + + std::vector expected_locations = {0, 1, 2, 3}; + uint64_t expected_table_size = GetExpectedTableSize(user_keys.size()); + + std::unique_ptr file_writer; + fname = test::PerThreadDBPath("WithCollisionUserKey"); + ASSERT_OK(WritableFileWriter::Create(env_->GetFileSystem(), fname, + file_options_, &file_writer, nullptr)); + + CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun, + 100, BytewiseComparator(), 1, false, false, + GetSliceHash, 0 /* column_family_id */, + kDefaultColumnFamilyName); + ASSERT_OK(builder.status()); + for (uint32_t i = 0; i < user_keys.size(); i++) { + builder.Add(Slice(GetInternalKey(user_keys[i], true)), Slice(values[i])); + ASSERT_EQ(builder.NumEntries(), i + 1); + ASSERT_OK(builder.status()); + } + size_t bucket_size = user_keys[0].size() + values[0].size(); + ASSERT_EQ(expected_table_size * bucket_size - 1, builder.FileSize()); + ASSERT_OK(builder.Finish()); + ASSERT_OK(file_writer->Close()); + ASSERT_LE(expected_table_size * bucket_size, builder.FileSize()); + + std::string expected_unused_bucket = "key00"; + expected_unused_bucket += std::string(values[0].size(), 'a'); + CheckFileContents(user_keys, values, expected_locations, + expected_unused_bucket, expected_table_size, 4, true); +} + +TEST_F(CuckooBuilderTest, WithCollisionPathUserKey) { + uint32_t num_hash_fun = 2; + std::vector user_keys = {"key01", "key02", "key03", "key04", + "key05"}; + std::vector values = {"v01", "v02", "v03", "v04", "v05"}; + // Need to have a temporary variable here as VS compiler does not currently + // support operator= with initializer_list as a parameter + std::unordered_map> hm = { + {user_keys[0], {0, 1}}, {user_keys[1], {1, 2}}, {user_keys[2], {2, 3}}, + {user_keys[3], {3, 4}}, {user_keys[4], {0, 2}}, + }; + hash_map = std::move(hm); + + std::vector expected_locations = {0, 1, 3, 4, 2}; + uint64_t expected_table_size = GetExpectedTableSize(user_keys.size()); + + std::unique_ptr file_writer; + fname = test::PerThreadDBPath("WithCollisionPathUserKey"); + ASSERT_OK(WritableFileWriter::Create(env_->GetFileSystem(), fname, + file_options_, &file_writer, nullptr)); + + CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun, + 2, BytewiseComparator(), 1, false, false, + GetSliceHash, 0 /* column_family_id */, + kDefaultColumnFamilyName); + ASSERT_OK(builder.status()); + for (uint32_t i = 0; i < user_keys.size(); i++) { + builder.Add(Slice(GetInternalKey(user_keys[i], true)), Slice(values[i])); + ASSERT_EQ(builder.NumEntries(), i + 1); + ASSERT_OK(builder.status()); + } + size_t bucket_size = user_keys[0].size() + values[0].size(); + ASSERT_EQ(expected_table_size * bucket_size - 1, builder.FileSize()); + ASSERT_OK(builder.Finish()); + ASSERT_OK(file_writer->Close()); + ASSERT_LE(expected_table_size * bucket_size, builder.FileSize()); + + std::string expected_unused_bucket = "key00"; + expected_unused_bucket += std::string(values[0].size(), 'a'); + CheckFileContents(user_keys, values, expected_locations, + expected_unused_bucket, expected_table_size, 2, true); +} + +TEST_F(CuckooBuilderTest, FailWhenCollisionPathTooLong) { + // Have two hash functions. Insert elements with overlapping hashes. + // Finally try inserting an element with hash value somewhere in the middle + // and it should fail because the no. of elements to displace is too high. + uint32_t num_hash_fun = 2; + std::vector user_keys = {"key01", "key02", "key03", "key04", + "key05"}; + // Need to have a temporary variable here as VS compiler does not currently + // support operator= with initializer_list as a parameter + std::unordered_map> hm = { + {user_keys[0], {0, 1}}, {user_keys[1], {1, 2}}, {user_keys[2], {2, 3}}, + {user_keys[3], {3, 4}}, {user_keys[4], {0, 1}}, + }; + hash_map = std::move(hm); + + std::unique_ptr file_writer; + fname = test::PerThreadDBPath("WithCollisionPathUserKey"); + ASSERT_OK(WritableFileWriter::Create(env_->GetFileSystem(), fname, + file_options_, &file_writer, nullptr)); + CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun, + 2, BytewiseComparator(), 1, false, false, + GetSliceHash, 0 /* column_family_id */, + kDefaultColumnFamilyName); + ASSERT_OK(builder.status()); + for (uint32_t i = 0; i < user_keys.size(); i++) { + builder.Add(Slice(GetInternalKey(user_keys[i], false)), Slice("value")); + ASSERT_EQ(builder.NumEntries(), i + 1); + ASSERT_OK(builder.status()); + } + ASSERT_TRUE(builder.Finish().IsNotSupported()); + ASSERT_OK(file_writer->Close()); +} + +TEST_F(CuckooBuilderTest, FailWhenSameKeyInserted) { + // Need to have a temporary variable here as VS compiler does not currently + // support operator= with initializer_list as a parameter + std::unordered_map> hm = { + {"repeatedkey", {0, 1, 2, 3}}}; + hash_map = std::move(hm); + uint32_t num_hash_fun = 4; + std::string user_key = "repeatedkey"; + + std::unique_ptr file_writer; + fname = test::PerThreadDBPath("FailWhenSameKeyInserted"); + ASSERT_OK(WritableFileWriter::Create(env_->GetFileSystem(), fname, + file_options_, &file_writer, nullptr)); + CuckooTableBuilder builder(file_writer.get(), kHashTableRatio, num_hash_fun, + 100, BytewiseComparator(), 1, false, false, + GetSliceHash, 0 /* column_family_id */, + kDefaultColumnFamilyName); + ASSERT_OK(builder.status()); + + builder.Add(Slice(GetInternalKey(user_key, false)), Slice("value1")); + ASSERT_EQ(builder.NumEntries(), 1u); + ASSERT_OK(builder.status()); + builder.Add(Slice(GetInternalKey(user_key, true)), Slice("value2")); + ASSERT_EQ(builder.NumEntries(), 2u); + ASSERT_OK(builder.status()); + + ASSERT_TRUE(builder.Finish().IsNotSupported()); + ASSERT_OK(file_writer->Close()); +} +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_factory.cc b/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_factory.cc new file mode 100644 index 0000000..774e002 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_factory.cc @@ -0,0 +1,100 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "table/cuckoo/cuckoo_table_factory.h" + +#include "db/dbformat.h" +#include "options/configurable_helper.h" +#include "rocksdb/utilities/options_type.h" +#include "table/cuckoo/cuckoo_table_builder.h" +#include "table/cuckoo/cuckoo_table_reader.h" + +namespace ROCKSDB_NAMESPACE { + +Status CuckooTableFactory::NewTableReader( + const ReadOptions& /*ro*/, const TableReaderOptions& table_reader_options, + std::unique_ptr&& file, uint64_t file_size, + std::unique_ptr* table, + bool /*prefetch_index_and_filter_in_cache*/) const { + std::unique_ptr new_reader(new CuckooTableReader( + table_reader_options.ioptions, std::move(file), file_size, + table_reader_options.internal_comparator.user_comparator(), nullptr)); + Status s = new_reader->status(); + if (s.ok()) { + *table = std::move(new_reader); + } + return s; +} + +TableBuilder* CuckooTableFactory::NewTableBuilder( + const TableBuilderOptions& table_builder_options, + WritableFileWriter* file) const { + // TODO: change builder to take the option struct + return new CuckooTableBuilder( + file, table_options_.hash_table_ratio, 64, + table_options_.max_search_depth, + table_builder_options.internal_comparator.user_comparator(), + table_options_.cuckoo_block_size, table_options_.use_module_hash, + table_options_.identity_as_first_hash, nullptr /* get_slice_hash */, + table_builder_options.column_family_id, + table_builder_options.column_family_name, table_builder_options.db_id, + table_builder_options.db_session_id, table_builder_options.cur_file_num); +} + +std::string CuckooTableFactory::GetPrintableOptions() const { + std::string ret; + ret.reserve(2000); + const int kBufferSize = 200; + char buffer[kBufferSize]; + + snprintf(buffer, kBufferSize, " hash_table_ratio: %lf\n", + table_options_.hash_table_ratio); + ret.append(buffer); + snprintf(buffer, kBufferSize, " max_search_depth: %u\n", + table_options_.max_search_depth); + ret.append(buffer); + snprintf(buffer, kBufferSize, " cuckoo_block_size: %u\n", + table_options_.cuckoo_block_size); + ret.append(buffer); + snprintf(buffer, kBufferSize, " identity_as_first_hash: %d\n", + table_options_.identity_as_first_hash); + ret.append(buffer); + return ret; +} + +static std::unordered_map cuckoo_table_type_info = + { + {"hash_table_ratio", + {offsetof(struct CuckooTableOptions, hash_table_ratio), + OptionType::kDouble, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"max_search_depth", + {offsetof(struct CuckooTableOptions, max_search_depth), + OptionType::kUInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"cuckoo_block_size", + {offsetof(struct CuckooTableOptions, cuckoo_block_size), + OptionType::kUInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"identity_as_first_hash", + {offsetof(struct CuckooTableOptions, identity_as_first_hash), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"use_module_hash", + {offsetof(struct CuckooTableOptions, use_module_hash), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, +}; + +CuckooTableFactory::CuckooTableFactory(const CuckooTableOptions& table_options) + : table_options_(table_options) { + RegisterOptions(&table_options_, &cuckoo_table_type_info); +} + +TableFactory* NewCuckooTableFactory(const CuckooTableOptions& table_options) { + return new CuckooTableFactory(table_options); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_factory.h b/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_factory.h new file mode 100644 index 0000000..7132cec --- /dev/null +++ b/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_factory.h @@ -0,0 +1,80 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "rocksdb/options.h" +#include "rocksdb/table.h" +#include "util/murmurhash.h" + +namespace ROCKSDB_NAMESPACE { + +const uint32_t kCuckooMurmurSeedMultiplier = 816922183; +static inline uint64_t CuckooHash( + const Slice& user_key, uint32_t hash_cnt, bool use_module_hash, + uint64_t table_size_, bool identity_as_first_hash, + uint64_t (*get_slice_hash)(const Slice&, uint32_t, uint64_t)) { +#if !defined NDEBUG || defined OS_WIN + // This part is used only in unit tests but we have to keep it for Windows + // build as we run test in both debug and release modes under Windows. + if (get_slice_hash != nullptr) { + return get_slice_hash(user_key, hash_cnt, table_size_); + } +#else + (void)get_slice_hash; +#endif + + uint64_t value = 0; + if (hash_cnt == 0 && identity_as_first_hash) { + value = (*reinterpret_cast(user_key.data())); + } else { + value = MurmurHash(user_key.data(), static_cast(user_key.size()), + kCuckooMurmurSeedMultiplier * hash_cnt); + } + if (use_module_hash) { + return value % table_size_; + } else { + return value & (table_size_ - 1); + } +} + +// Cuckoo Table is designed for applications that require fast point lookups +// but not fast range scans. +// +// Some assumptions: +// - Key length and Value length are fixed. +// - Does not support Snapshot. +// - Does not support Merge operations. +// - Does not support prefix bloom filters. +class CuckooTableFactory : public TableFactory { + public: + explicit CuckooTableFactory( + const CuckooTableOptions& table_option = CuckooTableOptions()); + ~CuckooTableFactory() {} + + // Method to allow CheckedCast to work for this class + static const char* kClassName() { return kCuckooTableName(); } + const char* Name() const override { return kCuckooTableName(); } + + using TableFactory::NewTableReader; + Status NewTableReader( + const ReadOptions& ro, const TableReaderOptions& table_reader_options, + std::unique_ptr&& file, uint64_t file_size, + std::unique_ptr* table, + bool prefetch_index_and_filter_in_cache = true) const override; + + TableBuilder* NewTableBuilder( + const TableBuilderOptions& table_builder_options, + WritableFileWriter* file) const override; + + std::string GetPrintableOptions() const override; + + private: + CuckooTableOptions table_options_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_reader.cc b/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_reader.cc new file mode 100644 index 0000000..d647619 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_reader.cc @@ -0,0 +1,412 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "table/cuckoo/cuckoo_table_reader.h" + +#include +#include +#include +#include +#include + +#include "memory/arena.h" +#include "options/cf_options.h" +#include "rocksdb/iterator.h" +#include "rocksdb/table.h" +#include "table/cuckoo/cuckoo_table_factory.h" +#include "table/get_context.h" +#include "table/internal_iterator.h" +#include "table/meta_blocks.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { +namespace { +const uint64_t CACHE_LINE_MASK = ~((uint64_t)CACHE_LINE_SIZE - 1); +const uint32_t kInvalidIndex = std::numeric_limits::max(); +} // namespace + +extern const uint64_t kCuckooTableMagicNumber; + +CuckooTableReader::CuckooTableReader( + const ImmutableOptions& ioptions, + std::unique_ptr&& file, uint64_t file_size, + const Comparator* comparator, + uint64_t (*get_slice_hash)(const Slice&, uint32_t, uint64_t)) + : file_(std::move(file)), + is_last_level_(false), + identity_as_first_hash_(false), + use_module_hash_(false), + num_hash_func_(0), + unused_key_(""), + key_length_(0), + user_key_length_(0), + value_length_(0), + bucket_length_(0), + cuckoo_block_size_(0), + cuckoo_block_bytes_minus_one_(0), + table_size_(0), + ucomp_(comparator), + get_slice_hash_(get_slice_hash) { + if (!ioptions.allow_mmap_reads) { + status_ = Status::InvalidArgument("File is not mmaped"); + return; + } + { + std::unique_ptr props; + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + status_ = + ReadTableProperties(file_.get(), file_size, kCuckooTableMagicNumber, + ioptions, read_options, &props); + if (!status_.ok()) { + return; + } + table_props_ = std::move(props); + } + auto& user_props = table_props_->user_collected_properties; + auto hash_funs = user_props.find(CuckooTablePropertyNames::kNumHashFunc); + if (hash_funs == user_props.end()) { + status_ = Status::Corruption("Number of hash functions not found"); + return; + } + num_hash_func_ = *reinterpret_cast(hash_funs->second.data()); + auto unused_key = user_props.find(CuckooTablePropertyNames::kEmptyKey); + if (unused_key == user_props.end()) { + status_ = Status::Corruption("Empty bucket value not found"); + return; + } + unused_key_ = unused_key->second; + + key_length_ = static_cast(table_props_->fixed_key_len); + auto user_key_len = user_props.find(CuckooTablePropertyNames::kUserKeyLength); + if (user_key_len == user_props.end()) { + status_ = Status::Corruption("User key length not found"); + return; + } + user_key_length_ = + *reinterpret_cast(user_key_len->second.data()); + + auto value_length = user_props.find(CuckooTablePropertyNames::kValueLength); + if (value_length == user_props.end()) { + status_ = Status::Corruption("Value length not found"); + return; + } + value_length_ = + *reinterpret_cast(value_length->second.data()); + bucket_length_ = key_length_ + value_length_; + + auto hash_table_size = + user_props.find(CuckooTablePropertyNames::kHashTableSize); + if (hash_table_size == user_props.end()) { + status_ = Status::Corruption("Hash table size not found"); + return; + } + table_size_ = + *reinterpret_cast(hash_table_size->second.data()); + + auto is_last_level = user_props.find(CuckooTablePropertyNames::kIsLastLevel); + if (is_last_level == user_props.end()) { + status_ = Status::Corruption("Is last level not found"); + return; + } + is_last_level_ = *reinterpret_cast(is_last_level->second.data()); + + auto identity_as_first_hash = + user_props.find(CuckooTablePropertyNames::kIdentityAsFirstHash); + if (identity_as_first_hash == user_props.end()) { + status_ = Status::Corruption("identity as first hash not found"); + return; + } + identity_as_first_hash_ = + *reinterpret_cast(identity_as_first_hash->second.data()); + + auto use_module_hash = + user_props.find(CuckooTablePropertyNames::kUseModuleHash); + if (use_module_hash == user_props.end()) { + status_ = Status::Corruption("hash type is not found"); + return; + } + use_module_hash_ = + *reinterpret_cast(use_module_hash->second.data()); + auto cuckoo_block_size = + user_props.find(CuckooTablePropertyNames::kCuckooBlockSize); + if (cuckoo_block_size == user_props.end()) { + status_ = Status::Corruption("Cuckoo block size not found"); + return; + } + cuckoo_block_size_ = + *reinterpret_cast(cuckoo_block_size->second.data()); + cuckoo_block_bytes_minus_one_ = cuckoo_block_size_ * bucket_length_ - 1; + // TODO: rate limit reads of whole cuckoo tables. + status_ = + file_->Read(IOOptions(), 0, static_cast(file_size), &file_data_, + nullptr, nullptr, Env::IO_TOTAL /* rate_limiter_priority */); +} + +Status CuckooTableReader::Get(const ReadOptions& /*readOptions*/, + const Slice& key, GetContext* get_context, + const SliceTransform* /* prefix_extractor */, + bool /*skip_filters*/) { + assert(key.size() == key_length_ + (is_last_level_ ? 8 : 0)); + Slice user_key = ExtractUserKey(key); + for (uint32_t hash_cnt = 0; hash_cnt < num_hash_func_; ++hash_cnt) { + uint64_t offset = + bucket_length_ * CuckooHash(user_key, hash_cnt, use_module_hash_, + table_size_, identity_as_first_hash_, + get_slice_hash_); + const char* bucket = &file_data_.data()[offset]; + for (uint32_t block_idx = 0; block_idx < cuckoo_block_size_; + ++block_idx, bucket += bucket_length_) { + if (ucomp_->Equal(Slice(unused_key_.data(), user_key.size()), + Slice(bucket, user_key.size()))) { + return Status::OK(); + } + // Here, we compare only the user key part as we support only one entry + // per user key and we don't support snapshot. + if (ucomp_->Equal(user_key, Slice(bucket, user_key.size()))) { + Slice value(bucket + key_length_, value_length_); + if (is_last_level_) { + // Sequence number is not stored at the last level, so we will use + // kMaxSequenceNumber since it is unknown. This could cause some + // transactions to fail to lock a key due to known sequence number. + // However, it is expected for anyone to use a CuckooTable in a + // TransactionDB. + get_context->SaveValue(value, kMaxSequenceNumber); + } else { + Slice full_key(bucket, key_length_); + ParsedInternalKey found_ikey; + Status s = ParseInternalKey(full_key, &found_ikey, + false /* log_err_key */); // TODO + if (!s.ok()) return s; + bool dont_care __attribute__((__unused__)); + get_context->SaveValue(found_ikey, value, &dont_care); + } + // We don't support merge operations. So, we return here. + return Status::OK(); + } + } + } + return Status::OK(); +} + +void CuckooTableReader::Prepare(const Slice& key) { + // Prefetch the first Cuckoo Block. + Slice user_key = ExtractUserKey(key); + uint64_t addr = + reinterpret_cast(file_data_.data()) + + bucket_length_ * CuckooHash(user_key, 0, use_module_hash_, table_size_, + identity_as_first_hash_, nullptr); + uint64_t end_addr = addr + cuckoo_block_bytes_minus_one_; + for (addr &= CACHE_LINE_MASK; addr < end_addr; addr += CACHE_LINE_SIZE) { + PREFETCH(reinterpret_cast(addr), 0, 3); + } +} + +class CuckooTableIterator : public InternalIterator { + public: + explicit CuckooTableIterator(CuckooTableReader* reader); + // No copying allowed + CuckooTableIterator(const CuckooTableIterator&) = delete; + void operator=(const Iterator&) = delete; + ~CuckooTableIterator() override {} + bool Valid() const override; + void SeekToFirst() override; + void SeekToLast() override; + void Seek(const Slice& target) override; + void SeekForPrev(const Slice& target) override; + void Next() override; + void Prev() override; + Slice key() const override; + Slice value() const override; + Status status() const override { return Status::OK(); } + void InitIfNeeded(); + + private: + struct BucketComparator { + BucketComparator(const Slice& file_data, const Comparator* ucomp, + uint32_t bucket_len, uint32_t user_key_len, + const Slice& target = Slice()) + : file_data_(file_data), + ucomp_(ucomp), + bucket_len_(bucket_len), + user_key_len_(user_key_len), + target_(target) {} + bool operator()(const uint32_t first, const uint32_t second) const { + const char* first_bucket = (first == kInvalidIndex) + ? target_.data() + : &file_data_.data()[first * bucket_len_]; + const char* second_bucket = + (second == kInvalidIndex) ? target_.data() + : &file_data_.data()[second * bucket_len_]; + return ucomp_->Compare(Slice(first_bucket, user_key_len_), + Slice(second_bucket, user_key_len_)) < 0; + } + + private: + const Slice file_data_; + const Comparator* ucomp_; + const uint32_t bucket_len_; + const uint32_t user_key_len_; + const Slice target_; + }; + + const BucketComparator bucket_comparator_; + void PrepareKVAtCurrIdx(); + CuckooTableReader* reader_; + bool initialized_; + // Contains a map of keys to bucket_id sorted in key order. + std::vector sorted_bucket_ids_; + // We assume that the number of items can be stored in uint32 (4 Billion). + uint32_t curr_key_idx_; + Slice curr_value_; + IterKey curr_key_; +}; + +CuckooTableIterator::CuckooTableIterator(CuckooTableReader* reader) + : bucket_comparator_(reader->file_data_, reader->ucomp_, + reader->bucket_length_, reader->user_key_length_), + reader_(reader), + initialized_(false), + curr_key_idx_(kInvalidIndex) { + sorted_bucket_ids_.clear(); + curr_value_.clear(); + curr_key_.Clear(); +} + +void CuckooTableIterator::InitIfNeeded() { + if (initialized_) { + return; + } + sorted_bucket_ids_.reserve( + static_cast(reader_->GetTableProperties()->num_entries)); + uint64_t num_buckets = reader_->table_size_ + reader_->cuckoo_block_size_ - 1; + assert(num_buckets < kInvalidIndex); + const char* bucket = reader_->file_data_.data(); + for (uint32_t bucket_id = 0; bucket_id < num_buckets; ++bucket_id) { + if (Slice(bucket, reader_->key_length_) != Slice(reader_->unused_key_)) { + sorted_bucket_ids_.push_back(bucket_id); + } + bucket += reader_->bucket_length_; + } + assert(sorted_bucket_ids_.size() == + reader_->GetTableProperties()->num_entries); + std::sort(sorted_bucket_ids_.begin(), sorted_bucket_ids_.end(), + bucket_comparator_); + curr_key_idx_ = kInvalidIndex; + initialized_ = true; +} + +void CuckooTableIterator::SeekToFirst() { + InitIfNeeded(); + curr_key_idx_ = 0; + PrepareKVAtCurrIdx(); +} + +void CuckooTableIterator::SeekToLast() { + InitIfNeeded(); + curr_key_idx_ = static_cast(sorted_bucket_ids_.size()) - 1; + PrepareKVAtCurrIdx(); +} + +void CuckooTableIterator::Seek(const Slice& target) { + InitIfNeeded(); + const BucketComparator seek_comparator( + reader_->file_data_, reader_->ucomp_, reader_->bucket_length_, + reader_->user_key_length_, ExtractUserKey(target)); + auto seek_it = + std::lower_bound(sorted_bucket_ids_.begin(), sorted_bucket_ids_.end(), + kInvalidIndex, seek_comparator); + curr_key_idx_ = + static_cast(std::distance(sorted_bucket_ids_.begin(), seek_it)); + PrepareKVAtCurrIdx(); +} + +void CuckooTableIterator::SeekForPrev(const Slice& /*target*/) { + // Not supported + assert(false); +} + +bool CuckooTableIterator::Valid() const { + return curr_key_idx_ < sorted_bucket_ids_.size(); +} + +void CuckooTableIterator::PrepareKVAtCurrIdx() { + if (!Valid()) { + curr_value_.clear(); + curr_key_.Clear(); + return; + } + uint32_t id = sorted_bucket_ids_[curr_key_idx_]; + const char* offset = + reader_->file_data_.data() + id * reader_->bucket_length_; + if (reader_->is_last_level_) { + // Always return internal key. + curr_key_.SetInternalKey(Slice(offset, reader_->user_key_length_), 0, + kTypeValue); + } else { + curr_key_.SetInternalKey(Slice(offset, reader_->key_length_)); + } + curr_value_ = Slice(offset + reader_->key_length_, reader_->value_length_); +} + +void CuckooTableIterator::Next() { + if (!Valid()) { + curr_value_.clear(); + curr_key_.Clear(); + return; + } + ++curr_key_idx_; + PrepareKVAtCurrIdx(); +} + +void CuckooTableIterator::Prev() { + if (curr_key_idx_ == 0) { + curr_key_idx_ = static_cast(sorted_bucket_ids_.size()); + } + if (!Valid()) { + curr_value_.clear(); + curr_key_.Clear(); + return; + } + --curr_key_idx_; + PrepareKVAtCurrIdx(); +} + +Slice CuckooTableIterator::key() const { + assert(Valid()); + return curr_key_.GetInternalKey(); +} + +Slice CuckooTableIterator::value() const { + assert(Valid()); + return curr_value_; +} + +InternalIterator* CuckooTableReader::NewIterator( + const ReadOptions& /*read_options*/, + const SliceTransform* /* prefix_extractor */, Arena* arena, + bool /*skip_filters*/, TableReaderCaller /*caller*/, + size_t /*compaction_readahead_size*/, bool /* allow_unprepared_value */) { + if (!status().ok()) { + return NewErrorInternalIterator( + Status::Corruption("CuckooTableReader status is not okay."), arena); + } + CuckooTableIterator* iter; + if (arena == nullptr) { + iter = new CuckooTableIterator(this); + } else { + auto iter_mem = arena->AllocateAligned(sizeof(CuckooTableIterator)); + iter = new (iter_mem) CuckooTableIterator(this); + } + return iter; +} + +size_t CuckooTableReader::ApproximateMemoryUsage() const { return 0; } + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_reader.h b/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_reader.h new file mode 100644 index 0000000..d17011e --- /dev/null +++ b/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_reader.h @@ -0,0 +1,100 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include +#include +#include +#include + +#include "file/random_access_file_reader.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "table/table_reader.h" + +namespace ROCKSDB_NAMESPACE { + +class Arena; +class TableReader; +struct ImmutableOptions; + +class CuckooTableReader : public TableReader { + public: + CuckooTableReader(const ImmutableOptions& ioptions, + std::unique_ptr&& file, + uint64_t file_size, const Comparator* user_comparator, + uint64_t (*get_slice_hash)(const Slice&, uint32_t, + uint64_t)); + ~CuckooTableReader() {} + + std::shared_ptr GetTableProperties() const override { + return table_props_; + } + + Status status() const { return status_; } + + Status Get(const ReadOptions& readOptions, const Slice& key, + GetContext* get_context, const SliceTransform* prefix_extractor, + bool skip_filters = false) override; + + // Returns a new iterator over table contents + // compaction_readahead_size: its value will only be used if for_compaction = + // true + InternalIterator* NewIterator(const ReadOptions&, + const SliceTransform* prefix_extractor, + Arena* arena, bool skip_filters, + TableReaderCaller caller, + size_t compaction_readahead_size = 0, + bool allow_unprepared_value = false) override; + void Prepare(const Slice& target) override; + + // Report an approximation of how much memory has been used. + size_t ApproximateMemoryUsage() const override; + + // Following methods are not implemented for Cuckoo Table Reader + uint64_t ApproximateOffsetOf(const ReadOptions& /*read_options*/, + const Slice& /*key*/, + TableReaderCaller /*caller*/) override { + return 0; + } + + uint64_t ApproximateSize(const ReadOptions& /* read_options */, + const Slice& /*start*/, const Slice& /*end*/, + TableReaderCaller /*caller*/) override { + return 0; + } + + void SetupForCompaction() override {} + // End of methods not implemented. + + private: + friend class CuckooTableIterator; + void LoadAllKeys(std::vector>* key_to_bucket_id); + std::unique_ptr file_; + Slice file_data_; + bool is_last_level_; + bool identity_as_first_hash_; + bool use_module_hash_; + std::shared_ptr table_props_; + Status status_; + uint32_t num_hash_func_; + std::string unused_key_; + uint32_t key_length_; + uint32_t user_key_length_; + uint32_t value_length_; + uint32_t bucket_length_; + uint32_t cuckoo_block_size_; + uint32_t cuckoo_block_bytes_minus_one_; + uint64_t table_size_; + const Comparator* ucomp_; + uint64_t (*get_slice_hash_)(const Slice& s, uint32_t index, + uint64_t max_num_buckets); +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_reader_test.cc b/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_reader_test.cc new file mode 100644 index 0000000..e83baa1 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/cuckoo/cuckoo_table_reader_test.cc @@ -0,0 +1,574 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + + +#ifndef GFLAGS +#include +int main() { + fprintf(stderr, "Please install gflags to run this test... Skipping...\n"); + return 0; +} +#else + +#include +#include +#include +#include + +#include "memory/arena.h" +#include "rocksdb/db.h" +#include "table/cuckoo/cuckoo_table_builder.h" +#include "table/cuckoo/cuckoo_table_factory.h" +#include "table/cuckoo/cuckoo_table_reader.h" +#include "table/get_context.h" +#include "table/meta_blocks.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/gflags_compat.h" +#include "util/random.h" +#include "util/string_util.h" + +using GFLAGS_NAMESPACE::ParseCommandLineFlags; + +DEFINE_string(file_dir, "", + "Directory where the files will be created" + " for benchmark. Added for using tmpfs."); +DEFINE_bool(enable_perf, false, "Run Benchmark Tests too."); +DEFINE_bool(write, false, + "Should write new values to file in performance tests?"); +DEFINE_bool(identity_as_first_hash, true, "use identity as first hash"); + +namespace ROCKSDB_NAMESPACE { + +namespace { +const uint32_t kNumHashFunc = 10; +// Methods, variables related to Hash functions. +std::unordered_map > hash_map; + +void AddHashLookups(const std::string& s, uint64_t bucket_id, + uint32_t num_hash_fun) { + std::vector v; + for (uint32_t i = 0; i < num_hash_fun; i++) { + v.push_back(bucket_id + i); + } + hash_map[s] = v; +} + +uint64_t GetSliceHash(const Slice& s, uint32_t index, + uint64_t /*max_num_buckets*/) { + return hash_map[s.ToString()][index]; +} +} // namespace + +class CuckooReaderTest : public testing::Test { + public: + using testing::Test::SetUp; + + CuckooReaderTest() { + options.allow_mmap_reads = true; + env = options.env; + file_options = FileOptions(options); + } + + void SetUp(int num) { + num_items = num; + hash_map.clear(); + keys.clear(); + keys.resize(num_items); + user_keys.clear(); + user_keys.resize(num_items); + values.clear(); + values.resize(num_items); + } + + std::string NumToStr(int64_t i) { + return std::string(reinterpret_cast(&i), sizeof(i)); + } + + void CreateCuckooFileAndCheckReader( + const Comparator* ucomp = BytewiseComparator()) { + std::unique_ptr file_writer; + ASSERT_OK(WritableFileWriter::Create(env->GetFileSystem(), fname, + file_options, &file_writer, nullptr)); + CuckooTableBuilder builder( + file_writer.get(), 0.9, kNumHashFunc, 100, ucomp, 2, false, false, + GetSliceHash, 0 /* column_family_id */, kDefaultColumnFamilyName); + ASSERT_OK(builder.status()); + for (uint32_t key_idx = 0; key_idx < num_items; ++key_idx) { + builder.Add(Slice(keys[key_idx]), Slice(values[key_idx])); + ASSERT_OK(builder.status()); + ASSERT_EQ(builder.NumEntries(), key_idx + 1); + } + ASSERT_OK(builder.Finish()); + ASSERT_EQ(num_items, builder.NumEntries()); + file_size = builder.FileSize(); + ASSERT_OK(file_writer->Close()); + + // Check reader now. + std::unique_ptr file_reader; + ASSERT_OK(RandomAccessFileReader::Create( + env->GetFileSystem(), fname, file_options, &file_reader, nullptr)); + const ImmutableOptions ioptions(options); + CuckooTableReader reader(ioptions, std::move(file_reader), file_size, ucomp, + GetSliceHash); + ASSERT_OK(reader.status()); + // Assume no merge/deletion + for (uint32_t i = 0; i < num_items; ++i) { + PinnableSlice value; + GetContext get_context(ucomp, nullptr, nullptr, nullptr, + GetContext::kNotFound, Slice(user_keys[i]), &value, + nullptr, nullptr, nullptr, nullptr, true, nullptr, + nullptr); + ASSERT_OK( + reader.Get(ReadOptions(), Slice(keys[i]), &get_context, nullptr)); + ASSERT_STREQ(values[i].c_str(), value.data()); + } + } + void UpdateKeys(bool with_zero_seqno) { + for (uint32_t i = 0; i < num_items; i++) { + ParsedInternalKey ikey(user_keys[i], with_zero_seqno ? 0 : i + 1000, + kTypeValue); + keys[i].clear(); + AppendInternalKey(&keys[i], ikey); + } + } + + void CheckIterator(const Comparator* ucomp = BytewiseComparator()) { + std::unique_ptr file_reader; + ASSERT_OK(RandomAccessFileReader::Create( + env->GetFileSystem(), fname, file_options, &file_reader, nullptr)); + const ImmutableOptions ioptions(options); + CuckooTableReader reader(ioptions, std::move(file_reader), file_size, ucomp, + GetSliceHash); + ASSERT_OK(reader.status()); + InternalIterator* it = reader.NewIterator( + ReadOptions(), /*prefix_extractor=*/nullptr, /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized); + ASSERT_OK(it->status()); + ASSERT_TRUE(!it->Valid()); + it->SeekToFirst(); + int cnt = 0; + while (it->Valid()) { + ASSERT_OK(it->status()); + ASSERT_TRUE(Slice(keys[cnt]) == it->key()); + ASSERT_TRUE(Slice(values[cnt]) == it->value()); + ++cnt; + it->Next(); + } + ASSERT_EQ(static_cast(cnt), num_items); + + it->SeekToLast(); + cnt = static_cast(num_items) - 1; + ASSERT_TRUE(it->Valid()); + while (it->Valid()) { + ASSERT_OK(it->status()); + ASSERT_TRUE(Slice(keys[cnt]) == it->key()); + ASSERT_TRUE(Slice(values[cnt]) == it->value()); + --cnt; + it->Prev(); + } + ASSERT_EQ(cnt, -1); + + cnt = static_cast(num_items) / 2; + it->Seek(keys[cnt]); + while (it->Valid()) { + ASSERT_OK(it->status()); + ASSERT_TRUE(Slice(keys[cnt]) == it->key()); + ASSERT_TRUE(Slice(values[cnt]) == it->value()); + ++cnt; + it->Next(); + } + ASSERT_EQ(static_cast(cnt), num_items); + delete it; + + Arena arena; + it = reader.NewIterator(ReadOptions(), /*prefix_extractor=*/nullptr, &arena, + /*skip_filters=*/false, + TableReaderCaller::kUncategorized); + ASSERT_OK(it->status()); + ASSERT_TRUE(!it->Valid()); + it->Seek(keys[num_items / 2]); + ASSERT_TRUE(it->Valid()); + ASSERT_OK(it->status()); + ASSERT_TRUE(keys[num_items / 2] == it->key()); + ASSERT_TRUE(values[num_items / 2] == it->value()); + ASSERT_OK(it->status()); + it->~InternalIterator(); + } + + std::vector keys; + std::vector user_keys; + std::vector values; + uint64_t num_items; + std::string fname; + uint64_t file_size; + Options options; + Env* env; + FileOptions file_options; +}; + +TEST_F(CuckooReaderTest, FileNotMmaped) { + options.allow_mmap_reads = false; + ImmutableOptions ioptions(options); + CuckooTableReader reader(ioptions, nullptr, 0, nullptr, nullptr); + ASSERT_TRUE(reader.status().IsInvalidArgument()); + ASSERT_STREQ("File is not mmaped", reader.status().getState()); +} + +TEST_F(CuckooReaderTest, WhenKeyExists) { + SetUp(kNumHashFunc); + fname = test::PerThreadDBPath("CuckooReader_WhenKeyExists"); + for (uint64_t i = 0; i < num_items; i++) { + user_keys[i] = "key" + NumToStr(i); + ParsedInternalKey ikey(user_keys[i], i + 1000, kTypeValue); + AppendInternalKey(&keys[i], ikey); + values[i] = "value" + NumToStr(i); + // Give disjoint hash values. + AddHashLookups(user_keys[i], i, kNumHashFunc); + } + CreateCuckooFileAndCheckReader(); + // Last level file. + UpdateKeys(true); + CreateCuckooFileAndCheckReader(); + // Test with collision. Make all hash values collide. + hash_map.clear(); + for (uint32_t i = 0; i < num_items; i++) { + AddHashLookups(user_keys[i], 0, kNumHashFunc); + } + UpdateKeys(false); + CreateCuckooFileAndCheckReader(); + // Last level file. + UpdateKeys(true); + CreateCuckooFileAndCheckReader(); +} + +TEST_F(CuckooReaderTest, WhenKeyExistsWithUint64Comparator) { + SetUp(kNumHashFunc); + fname = test::PerThreadDBPath("CuckooReaderUint64_WhenKeyExists"); + for (uint64_t i = 0; i < num_items; i++) { + user_keys[i].resize(8); + memcpy(&user_keys[i][0], static_cast(&i), 8); + ParsedInternalKey ikey(user_keys[i], i + 1000, kTypeValue); + AppendInternalKey(&keys[i], ikey); + values[i] = "value" + NumToStr(i); + // Give disjoint hash values. + AddHashLookups(user_keys[i], i, kNumHashFunc); + } + CreateCuckooFileAndCheckReader(test::Uint64Comparator()); + // Last level file. + UpdateKeys(true); + CreateCuckooFileAndCheckReader(test::Uint64Comparator()); + // Test with collision. Make all hash values collide. + hash_map.clear(); + for (uint32_t i = 0; i < num_items; i++) { + AddHashLookups(user_keys[i], 0, kNumHashFunc); + } + UpdateKeys(false); + CreateCuckooFileAndCheckReader(test::Uint64Comparator()); + // Last level file. + UpdateKeys(true); + CreateCuckooFileAndCheckReader(test::Uint64Comparator()); +} + +TEST_F(CuckooReaderTest, CheckIterator) { + SetUp(2 * kNumHashFunc); + fname = test::PerThreadDBPath("CuckooReader_CheckIterator"); + for (uint64_t i = 0; i < num_items; i++) { + user_keys[i] = "key" + NumToStr(i); + ParsedInternalKey ikey(user_keys[i], 1000, kTypeValue); + AppendInternalKey(&keys[i], ikey); + values[i] = "value" + NumToStr(i); + // Give disjoint hash values, in reverse order. + AddHashLookups(user_keys[i], num_items - i - 1, kNumHashFunc); + } + CreateCuckooFileAndCheckReader(); + CheckIterator(); + // Last level file. + UpdateKeys(true); + CreateCuckooFileAndCheckReader(); + CheckIterator(); +} + +TEST_F(CuckooReaderTest, CheckIteratorUint64) { + SetUp(2 * kNumHashFunc); + fname = test::PerThreadDBPath("CuckooReader_CheckIterator"); + for (uint64_t i = 0; i < num_items; i++) { + user_keys[i].resize(8); + memcpy(&user_keys[i][0], static_cast(&i), 8); + ParsedInternalKey ikey(user_keys[i], 1000, kTypeValue); + AppendInternalKey(&keys[i], ikey); + values[i] = "value" + NumToStr(i); + // Give disjoint hash values, in reverse order. + AddHashLookups(user_keys[i], num_items - i - 1, kNumHashFunc); + } + CreateCuckooFileAndCheckReader(test::Uint64Comparator()); + CheckIterator(test::Uint64Comparator()); + // Last level file. + UpdateKeys(true); + CreateCuckooFileAndCheckReader(test::Uint64Comparator()); + CheckIterator(test::Uint64Comparator()); +} + +TEST_F(CuckooReaderTest, WhenKeyNotFound) { + // Add keys with colliding hash values. + SetUp(kNumHashFunc); + fname = test::PerThreadDBPath("CuckooReader_WhenKeyNotFound"); + for (uint64_t i = 0; i < num_items; i++) { + user_keys[i] = "key" + NumToStr(i); + ParsedInternalKey ikey(user_keys[i], i + 1000, kTypeValue); + AppendInternalKey(&keys[i], ikey); + values[i] = "value" + NumToStr(i); + // Make all hash values collide. + AddHashLookups(user_keys[i], 0, kNumHashFunc); + } + auto* ucmp = BytewiseComparator(); + CreateCuckooFileAndCheckReader(); + + std::unique_ptr file_reader; + ASSERT_OK(RandomAccessFileReader::Create( + env->GetFileSystem(), fname, file_options, &file_reader, nullptr)); + + const ImmutableOptions ioptions(options); + CuckooTableReader reader(ioptions, std::move(file_reader), file_size, ucmp, + GetSliceHash); + ASSERT_OK(reader.status()); + // Search for a key with colliding hash values. + std::string not_found_user_key = "key" + NumToStr(num_items); + std::string not_found_key; + AddHashLookups(not_found_user_key, 0, kNumHashFunc); + ParsedInternalKey ikey(not_found_user_key, 1000, kTypeValue); + AppendInternalKey(¬_found_key, ikey); + PinnableSlice value; + GetContext get_context(ucmp, nullptr, nullptr, nullptr, GetContext::kNotFound, + Slice(not_found_key), &value, nullptr, nullptr, + nullptr, nullptr, true, nullptr, nullptr); + ASSERT_OK( + reader.Get(ReadOptions(), Slice(not_found_key), &get_context, nullptr)); + ASSERT_TRUE(value.empty()); + ASSERT_OK(reader.status()); + // Search for a key with an independent hash value. + std::string not_found_user_key2 = "key" + NumToStr(num_items + 1); + AddHashLookups(not_found_user_key2, kNumHashFunc, kNumHashFunc); + ParsedInternalKey ikey2(not_found_user_key2, 1000, kTypeValue); + std::string not_found_key2; + AppendInternalKey(¬_found_key2, ikey2); + value.Reset(); + GetContext get_context2(ucmp, nullptr, nullptr, nullptr, + GetContext::kNotFound, Slice(not_found_key2), &value, + nullptr, nullptr, nullptr, nullptr, true, nullptr, + nullptr); + ASSERT_OK( + reader.Get(ReadOptions(), Slice(not_found_key2), &get_context2, nullptr)); + ASSERT_TRUE(value.empty()); + ASSERT_OK(reader.status()); + + // Test read when key is unused key. + std::string unused_key = + reader.GetTableProperties()->user_collected_properties.at( + CuckooTablePropertyNames::kEmptyKey); + // Add hash values that map to empty buckets. + AddHashLookups(ExtractUserKey(unused_key).ToString(), kNumHashFunc, + kNumHashFunc); + value.Reset(); + GetContext get_context3( + ucmp, nullptr, nullptr, nullptr, GetContext::kNotFound, Slice(unused_key), + &value, nullptr, nullptr, nullptr, nullptr, true, nullptr, nullptr); + ASSERT_OK( + reader.Get(ReadOptions(), Slice(unused_key), &get_context3, nullptr)); + ASSERT_TRUE(value.empty()); + ASSERT_OK(reader.status()); +} + +// Performance tests +namespace { +void GetKeys(uint64_t num, std::vector* keys) { + keys->clear(); + IterKey k; + k.SetInternalKey("", 0, kTypeValue); + std::string internal_key_suffix = k.GetInternalKey().ToString(); + ASSERT_EQ(static_cast(8), internal_key_suffix.size()); + for (uint64_t key_idx = 0; key_idx < num; ++key_idx) { + uint64_t value = 2 * key_idx; + std::string new_key(reinterpret_cast(&value), sizeof(value)); + new_key += internal_key_suffix; + keys->push_back(new_key); + } +} + +std::string GetFileName(uint64_t num) { + if (FLAGS_file_dir.empty()) { + FLAGS_file_dir = test::TmpDir(); + } + return test::PerThreadDBPath(FLAGS_file_dir, "cuckoo_read_benchmark") + + std::to_string(num / 1000000) + "Mkeys"; +} + +// Create last level file as we are interested in measuring performance of +// last level file only. +void WriteFile(const std::vector& keys, const uint64_t num, + double hash_ratio) { + Options options; + options.allow_mmap_reads = true; + const auto& fs = options.env->GetFileSystem(); + FileOptions file_options(options); + std::string fname = GetFileName(num); + + std::unique_ptr file_writer; + ASSERT_OK(WritableFileWriter::Create(fs, fname, file_options, &file_writer, + nullptr)); + CuckooTableBuilder builder( + file_writer.get(), hash_ratio, 64, 1000, test::Uint64Comparator(), 5, + false, FLAGS_identity_as_first_hash, nullptr, 0 /* column_family_id */, + kDefaultColumnFamilyName); + ASSERT_OK(builder.status()); + for (uint64_t key_idx = 0; key_idx < num; ++key_idx) { + // Value is just a part of key. + builder.Add(Slice(keys[key_idx]), Slice(&keys[key_idx][0], 4)); + ASSERT_EQ(builder.NumEntries(), key_idx + 1); + ASSERT_OK(builder.status()); + } + ASSERT_OK(builder.Finish()); + ASSERT_EQ(num, builder.NumEntries()); + ASSERT_OK(file_writer->Close()); + + uint64_t file_size; + ASSERT_OK( + fs->GetFileSize(fname, file_options.io_options, &file_size, nullptr)); + std::unique_ptr file_reader; + ASSERT_OK(RandomAccessFileReader::Create(fs, fname, file_options, + &file_reader, nullptr)); + + const ImmutableOptions ioptions(options); + CuckooTableReader reader(ioptions, std::move(file_reader), file_size, + test::Uint64Comparator(), nullptr); + ASSERT_OK(reader.status()); + ReadOptions r_options; + PinnableSlice value; + // Assume only the fast path is triggered + GetContext get_context(nullptr, nullptr, nullptr, nullptr, + GetContext::kNotFound, Slice(), &value, nullptr, + nullptr, nullptr, true, nullptr, nullptr); + for (uint64_t i = 0; i < num; ++i) { + value.Reset(); + value.clear(); + ASSERT_OK(reader.Get(r_options, Slice(keys[i]), &get_context, nullptr)); + ASSERT_TRUE(Slice(keys[i]) == Slice(&keys[i][0], 4)); + } +} + +void ReadKeys(uint64_t num, uint32_t batch_size) { + Options options; + options.allow_mmap_reads = true; + Env* env = options.env; + const auto& fs = options.env->GetFileSystem(); + FileOptions file_options(options); + std::string fname = GetFileName(num); + + uint64_t file_size; + ASSERT_OK( + fs->GetFileSize(fname, file_options.io_options, &file_size, nullptr)); + std::unique_ptr file_reader; + ASSERT_OK(RandomAccessFileReader::Create(fs, fname, file_options, + &file_reader, nullptr)); + + const ImmutableOptions ioptions(options); + CuckooTableReader reader(ioptions, std::move(file_reader), file_size, + test::Uint64Comparator(), nullptr); + ASSERT_OK(reader.status()); + const UserCollectedProperties user_props = + reader.GetTableProperties()->user_collected_properties; + const uint32_t num_hash_fun = *reinterpret_cast( + user_props.at(CuckooTablePropertyNames::kNumHashFunc).data()); + const uint64_t table_size = *reinterpret_cast( + user_props.at(CuckooTablePropertyNames::kHashTableSize).data()); + fprintf(stderr, + "With %" PRIu64 + " items, utilization is %.2f%%, number of" + " hash functions: %u.\n", + num, num * 100.0 / (table_size), num_hash_fun); + ReadOptions r_options; + + std::vector keys; + keys.reserve(num); + for (uint64_t i = 0; i < num; ++i) { + keys.push_back(2 * i); + } + RandomShuffle(keys.begin(), keys.end()); + + PinnableSlice value; + // Assume only the fast path is triggered + GetContext get_context(nullptr, nullptr, nullptr, nullptr, + GetContext::kNotFound, Slice(), &value, nullptr, + nullptr, nullptr, true, nullptr, nullptr); + uint64_t start_time = env->NowMicros(); + if (batch_size > 0) { + for (uint64_t i = 0; i < num; i += batch_size) { + for (uint64_t j = i; j < i + batch_size && j < num; ++j) { + reader.Prepare(Slice(reinterpret_cast(&keys[j]), 16)); + } + for (uint64_t j = i; j < i + batch_size && j < num; ++j) { + reader.Get(r_options, Slice(reinterpret_cast(&keys[j]), 16), + &get_context, nullptr); + } + } + } else { + for (uint64_t i = 0; i < num; i++) { + reader.Get(r_options, Slice(reinterpret_cast(&keys[i]), 16), + &get_context, nullptr); + } + } + float time_per_op = (env->NowMicros() - start_time) * 1.0f / num; + fprintf(stderr, + "Time taken per op is %.3fus (%.1f Mqps) with batch size of %u\n", + time_per_op, 1.0 / time_per_op, batch_size); +} +} // namespace. + +TEST_F(CuckooReaderTest, TestReadPerformance) { + if (!FLAGS_enable_perf) { + return; + } + double hash_ratio = 0.95; + // These numbers are chosen to have a hash utilization % close to + // 0.9, 0.75, 0.6 and 0.5 respectively. + // They all create 128 M buckets. + std::vector nums = {120 * 1024 * 1024, 100 * 1024 * 1024, + 80 * 1024 * 1024, 70 * 1024 * 1024}; +#ifndef NDEBUG + fprintf( + stdout, + "WARNING: Not compiled with DNDEBUG. Performance tests may be slow.\n"); +#endif + for (uint64_t num : nums) { + if (FLAGS_write || + Env::Default()->FileExists(GetFileName(num)).IsNotFound()) { + std::vector all_keys; + GetKeys(num, &all_keys); + WriteFile(all_keys, num, hash_ratio); + } + ReadKeys(num, 0); + ReadKeys(num, 10); + ReadKeys(num, 25); + ReadKeys(num, 50); + ReadKeys(num, 100); + fprintf(stderr, "\n"); + } +} +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + if (ROCKSDB_NAMESPACE::port::kLittleEndian) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + ParseCommandLineFlags(&argc, &argv, true); + return RUN_ALL_TESTS(); + } else { + fprintf(stderr, "SKIPPED as Cuckoo table doesn't support Big Endian\n"); + return 0; + } +} + +#endif // GFLAGS. + diff --git a/librocksdb-sys/rocksdb/table/format.cc b/librocksdb-sys/rocksdb/table/format.cc new file mode 100644 index 0000000..8825384 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/format.cc @@ -0,0 +1,711 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "table/format.h" + +#include +#include +#include + +#include "block_fetcher.h" +#include "file/random_access_file_reader.h" +#include "memory/memory_allocator_impl.h" +#include "monitoring/perf_context_imp.h" +#include "monitoring/statistics_impl.h" +#include "options/options_helper.h" +#include "port/likely.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "rocksdb/table.h" +#include "table/block_based/block.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/persistent_cache_helper.h" +#include "unique_id_impl.h" +#include "util/cast_util.h" +#include "util/coding.h" +#include "util/compression.h" +#include "util/crc32c.h" +#include "util/hash.h" +#include "util/stop_watch.h" +#include "util/string_util.h" +#include "util/xxhash.h" + +namespace ROCKSDB_NAMESPACE { + +extern const uint64_t kLegacyBlockBasedTableMagicNumber; +extern const uint64_t kBlockBasedTableMagicNumber; + +extern const uint64_t kLegacyPlainTableMagicNumber; +extern const uint64_t kPlainTableMagicNumber; +const char* kHostnameForDbHostId = "__hostname__"; + +bool ShouldReportDetailedTime(Env* env, Statistics* stats) { + return env != nullptr && stats != nullptr && + stats->get_stats_level() > kExceptDetailedTimers; +} + +void BlockHandle::EncodeTo(std::string* dst) const { + // Sanity check that all fields have been set + assert(offset_ != ~uint64_t{0}); + assert(size_ != ~uint64_t{0}); + PutVarint64Varint64(dst, offset_, size_); +} + +char* BlockHandle::EncodeTo(char* dst) const { + // Sanity check that all fields have been set + assert(offset_ != ~uint64_t{0}); + assert(size_ != ~uint64_t{0}); + char* cur = EncodeVarint64(dst, offset_); + cur = EncodeVarint64(cur, size_); + return cur; +} + +Status BlockHandle::DecodeFrom(Slice* input) { + if (GetVarint64(input, &offset_) && GetVarint64(input, &size_)) { + return Status::OK(); + } else { + // reset in case failure after partially decoding + offset_ = 0; + size_ = 0; + return Status::Corruption("bad block handle"); + } +} + +Status BlockHandle::DecodeSizeFrom(uint64_t _offset, Slice* input) { + if (GetVarint64(input, &size_)) { + offset_ = _offset; + return Status::OK(); + } else { + // reset in case failure after partially decoding + offset_ = 0; + size_ = 0; + return Status::Corruption("bad block handle"); + } +} + +// Return a string that contains the copy of handle. +std::string BlockHandle::ToString(bool hex) const { + std::string handle_str; + EncodeTo(&handle_str); + if (hex) { + return Slice(handle_str).ToString(true); + } else { + return handle_str; + } +} + +const BlockHandle BlockHandle::kNullBlockHandle(0, 0); + +void IndexValue::EncodeTo(std::string* dst, bool have_first_key, + const BlockHandle* previous_handle) const { + if (previous_handle) { + // WART: this is specific to Block-based table + assert(handle.offset() == previous_handle->offset() + + previous_handle->size() + + BlockBasedTable::kBlockTrailerSize); + PutVarsignedint64(dst, handle.size() - previous_handle->size()); + } else { + handle.EncodeTo(dst); + } + assert(dst->size() != 0); + + if (have_first_key) { + PutLengthPrefixedSlice(dst, first_internal_key); + } +} + +Status IndexValue::DecodeFrom(Slice* input, bool have_first_key, + const BlockHandle* previous_handle) { + if (previous_handle) { + int64_t delta; + if (!GetVarsignedint64(input, &delta)) { + return Status::Corruption("bad delta-encoded index value"); + } + // WART: this is specific to Block-based table + handle = BlockHandle(previous_handle->offset() + previous_handle->size() + + BlockBasedTable::kBlockTrailerSize, + previous_handle->size() + delta); + } else { + Status s = handle.DecodeFrom(input); + if (!s.ok()) { + return s; + } + } + + if (!have_first_key) { + first_internal_key = Slice(); + } else if (!GetLengthPrefixedSlice(input, &first_internal_key)) { + return Status::Corruption("bad first key in block info"); + } + + return Status::OK(); +} + +std::string IndexValue::ToString(bool hex, bool have_first_key) const { + std::string s; + EncodeTo(&s, have_first_key, nullptr); + if (hex) { + return Slice(s).ToString(true); + } else { + return s; + } +} + +namespace { +inline bool IsLegacyFooterFormat(uint64_t magic_number) { + return magic_number == kLegacyBlockBasedTableMagicNumber || + magic_number == kLegacyPlainTableMagicNumber; +} +inline uint64_t UpconvertLegacyFooterFormat(uint64_t magic_number) { + if (magic_number == kLegacyBlockBasedTableMagicNumber) { + return kBlockBasedTableMagicNumber; + } + if (magic_number == kLegacyPlainTableMagicNumber) { + return kPlainTableMagicNumber; + } + assert(false); + return magic_number; +} +inline uint64_t DownconvertToLegacyFooterFormat(uint64_t magic_number) { + if (magic_number == kBlockBasedTableMagicNumber) { + return kLegacyBlockBasedTableMagicNumber; + } + if (magic_number == kPlainTableMagicNumber) { + return kLegacyPlainTableMagicNumber; + } + assert(false); + return magic_number; +} +inline uint8_t BlockTrailerSizeForMagicNumber(uint64_t magic_number) { + if (magic_number == kBlockBasedTableMagicNumber || + magic_number == kLegacyBlockBasedTableMagicNumber) { + return static_cast(BlockBasedTable::kBlockTrailerSize); + } else { + return 0; + } +} + +// Footer format, in three parts: +// * Part1 +// -> format_version == 0 (inferred from legacy magic number) +// (0 bytes) +// -> format_version >= 1 +// checksum type (char, 1 byte) +// * Part2 +// -> format_version <= 5 +// metaindex handle (varint64 offset, varint64 size) +// index handle (varint64 offset, varint64 size) +// for part2 size = 2 * BlockHandle::kMaxEncodedLength = 40 +// - This padding is unchecked/ignored +// -> format_version >= 6 +// extended magic number (4 bytes) = 0x3e 0x00 0x7a 0x00 +// - Also surely invalid (size 0) handles if interpreted as older version +// - (Helps ensure a corrupted format_version doesn't get us far with no +// footer checksum.) +// footer_checksum (uint32LE, 4 bytes) +// - Checksum of above checksum type of whole footer, with this field +// set to all zeros. +// base_context_checksum (uint32LE, 4 bytes) +// metaindex block size (uint32LE, 4 bytes) +// - Assumed to be immediately before footer, < 4GB +// (24 bytes, reserved for future use) +// - Brings part2 size also to 40 bytes +// - Checked that last eight bytes == 0, so reserved for a future +// incompatible feature (but under format_version=6) +// * Part3 +// -> format_version == 0 (inferred from legacy magic number) +// legacy magic number (8 bytes) +// -> format_version >= 1 (inferred from NOT legacy magic number) +// format_version (uint32LE, 4 bytes), also called "footer version" +// newer magic number (8 bytes) +const std::array kExtendedMagic{{0x3e, 0x00, 0x7a, 0x00}}; +constexpr size_t kFooterPart2Size = 2 * BlockHandle::kMaxEncodedLength; +} // namespace + +Status FooterBuilder::Build(uint64_t magic_number, uint32_t format_version, + uint64_t footer_offset, ChecksumType checksum_type, + const BlockHandle& metaindex_handle, + const BlockHandle& index_handle, + uint32_t base_context_checksum) { + assert(magic_number != Footer::kNullTableMagicNumber); + assert(IsSupportedFormatVersion(format_version)); + + char* part2; + char* part3; + if (format_version > 0) { + slice_ = Slice(data_.data(), Footer::kNewVersionsEncodedLength); + // Generate parts 1 and 3 + char* cur = data_.data(); + // Part 1 + *(cur++) = checksum_type; + // Part 2 + part2 = cur; + // Skip over part 2 for now + cur += kFooterPart2Size; + // Part 3 + part3 = cur; + EncodeFixed32(cur, format_version); + cur += 4; + EncodeFixed64(cur, magic_number); + assert(cur + 8 == slice_.data() + slice_.size()); + } else { + slice_ = Slice(data_.data(), Footer::kVersion0EncodedLength); + // Legacy SST files use kCRC32c checksum but it's not stored in footer. + assert(checksum_type == kNoChecksum || checksum_type == kCRC32c); + // Generate part 3 (part 1 empty, skip part 2 for now) + part2 = data_.data(); + part3 = part2 + kFooterPart2Size; + char* cur = part3; + // Use legacy magic numbers to indicate format_version=0, for + // compatibility. No other cases should use format_version=0. + EncodeFixed64(cur, DownconvertToLegacyFooterFormat(magic_number)); + assert(cur + 8 == slice_.data() + slice_.size()); + } + + if (format_version >= 6) { + if (BlockTrailerSizeForMagicNumber(magic_number) != 0) { + // base context checksum required for table formats with block checksums + assert(base_context_checksum != 0); + assert(ChecksumModifierForContext(base_context_checksum, 0) != 0); + } else { + // base context checksum not used + assert(base_context_checksum == 0); + assert(ChecksumModifierForContext(base_context_checksum, 0) == 0); + } + + // Start populating Part 2 + char* cur = data_.data() + /* part 1 size */ 1; + // Set extended magic of part2 + std::copy(kExtendedMagic.begin(), kExtendedMagic.end(), cur); + cur += kExtendedMagic.size(); + // Fill checksum data with zeros (for later computing checksum) + char* checksum_data = cur; + EncodeFixed32(cur, 0); + cur += 4; + // Save base context checksum + EncodeFixed32(cur, base_context_checksum); + cur += 4; + // Compute and save metaindex size + uint32_t metaindex_size = static_cast(metaindex_handle.size()); + if (metaindex_size != metaindex_handle.size()) { + return Status::NotSupported("Metaindex block size > 4GB"); + } + // Metaindex must be adjacent to footer + assert(metaindex_size == 0 || + metaindex_handle.offset() + metaindex_handle.size() == + footer_offset - BlockTrailerSizeForMagicNumber(magic_number)); + EncodeFixed32(cur, metaindex_size); + cur += 4; + + // Zero pad remainder (for future use) + std::fill_n(cur, 24U, char{0}); + assert(cur + 24 == part3); + + // Compute checksum, add context + uint32_t checksum = ComputeBuiltinChecksum( + checksum_type, data_.data(), Footer::kNewVersionsEncodedLength); + checksum += + ChecksumModifierForContext(base_context_checksum, footer_offset); + // Store it + EncodeFixed32(checksum_data, checksum); + } else { + // Base context checksum not used + assert(!FormatVersionUsesContextChecksum(format_version)); + // Should be left empty + assert(base_context_checksum == 0); + assert(ChecksumModifierForContext(base_context_checksum, 0) == 0); + + // Populate all of part 2 + char* cur = part2; + cur = metaindex_handle.EncodeTo(cur); + cur = index_handle.EncodeTo(cur); + // Zero pad remainder + std::fill(cur, part3, char{0}); + } + return Status::OK(); +} + +Status Footer::DecodeFrom(Slice input, uint64_t input_offset, + uint64_t enforce_table_magic_number) { + // Only decode to unused Footer + assert(table_magic_number_ == kNullTableMagicNumber); + assert(input != nullptr); + assert(input.size() >= kMinEncodedLength); + + const char* magic_ptr = input.data() + input.size() - kMagicNumberLengthByte; + uint64_t magic = DecodeFixed64(magic_ptr); + + // We check for legacy formats here and silently upconvert them + bool legacy = IsLegacyFooterFormat(magic); + if (legacy) { + magic = UpconvertLegacyFooterFormat(magic); + } + if (enforce_table_magic_number != 0 && enforce_table_magic_number != magic) { + return Status::Corruption("Bad table magic number: expected " + + std::to_string(enforce_table_magic_number) + + ", found " + std::to_string(magic)); + } + table_magic_number_ = magic; + block_trailer_size_ = BlockTrailerSizeForMagicNumber(magic); + + // Parse Part3 + const char* part3_ptr = magic_ptr; + uint32_t computed_checksum = 0; + uint64_t footer_offset = 0; + if (legacy) { + // The size is already asserted to be at least kMinEncodedLength + // at the beginning of the function + input.remove_prefix(input.size() - kVersion0EncodedLength); + format_version_ = 0 /* legacy */; + checksum_type_ = kCRC32c; + } else { + part3_ptr = magic_ptr - 4; + format_version_ = DecodeFixed32(part3_ptr); + if (UNLIKELY(!IsSupportedFormatVersion(format_version_))) { + return Status::Corruption("Corrupt or unsupported format_version: " + + std::to_string(format_version_)); + } + // All known format versions >= 1 occupy exactly this many bytes. + if (UNLIKELY(input.size() < kNewVersionsEncodedLength)) { + return Status::Corruption("Input is too short to be an SST file"); + } + uint64_t adjustment = input.size() - kNewVersionsEncodedLength; + input.remove_prefix(adjustment); + footer_offset = input_offset + adjustment; + + // Parse Part1 + char chksum = input.data()[0]; + checksum_type_ = lossless_cast(chksum); + if (UNLIKELY(!IsSupportedChecksumType(checksum_type()))) { + return Status::Corruption("Corrupt or unsupported checksum type: " + + std::to_string(lossless_cast(chksum))); + } + // This is the most convenient place to compute the checksum + if (checksum_type_ != kNoChecksum && format_version_ >= 6) { + std::array copy_without_checksum; + std::copy_n(input.data(), kNewVersionsEncodedLength, + ©_without_checksum[0]); + EncodeFixed32(©_without_checksum[5], 0); // Clear embedded checksum + computed_checksum = + ComputeBuiltinChecksum(checksum_type(), copy_without_checksum.data(), + kNewVersionsEncodedLength); + } + // Consume checksum type field + input.remove_prefix(1); + } + + // Parse Part2 + if (format_version_ >= 6) { + Slice ext_magic(input.data(), 4); + if (UNLIKELY(ext_magic.compare(Slice(kExtendedMagic.data(), + kExtendedMagic.size())) != 0)) { + return Status::Corruption("Bad extended magic number: 0x" + + ext_magic.ToString(/*hex*/ true)); + } + input.remove_prefix(4); + uint32_t stored_checksum = 0, metaindex_size = 0; + bool success; + success = GetFixed32(&input, &stored_checksum); + assert(success); + success = GetFixed32(&input, &base_context_checksum_); + assert(success); + if (UNLIKELY(ChecksumModifierForContext(base_context_checksum_, 0) == 0)) { + return Status::Corruption("Invalid base context checksum"); + } + computed_checksum += + ChecksumModifierForContext(base_context_checksum_, footer_offset); + if (UNLIKELY(computed_checksum != stored_checksum)) { + return Status::Corruption("Footer at " + std::to_string(footer_offset) + + " checksum mismatch"); + } + success = GetFixed32(&input, &metaindex_size); + assert(success); + (void)success; + uint64_t metaindex_end = footer_offset - GetBlockTrailerSize(); + metaindex_handle_ = + BlockHandle(metaindex_end - metaindex_size, metaindex_size); + + // Mark unpopulated + index_handle_ = BlockHandle::NullBlockHandle(); + + // 16 bytes of unchecked reserved padding + input.remove_prefix(16U); + + // 8 bytes of checked reserved padding (expected to be zero unless using a + // future feature). + uint64_t reserved = 0; + success = GetFixed64(&input, &reserved); + assert(success); + if (UNLIKELY(reserved != 0)) { + return Status::NotSupported( + "File uses a future feature not supported in this version"); + } + // End of part 2 + assert(input.data() == part3_ptr); + } else { + // format_version_ < 6 + Status result = metaindex_handle_.DecodeFrom(&input); + if (result.ok()) { + result = index_handle_.DecodeFrom(&input); + } + if (!result.ok()) { + return result; + } + // Padding in part2 is ignored + } + return Status::OK(); +} + +std::string Footer::ToString() const { + std::string result; + result.reserve(1024); + + bool legacy = IsLegacyFooterFormat(table_magic_number_); + if (legacy) { + result.append("metaindex handle: " + metaindex_handle_.ToString() + "\n "); + result.append("index handle: " + index_handle_.ToString() + "\n "); + result.append("table_magic_number: " + std::to_string(table_magic_number_) + + "\n "); + } else { + result.append("metaindex handle: " + metaindex_handle_.ToString() + "\n "); + result.append("index handle: " + index_handle_.ToString() + "\n "); + result.append("table_magic_number: " + std::to_string(table_magic_number_) + + "\n "); + result.append("format version: " + std::to_string(format_version_) + + "\n "); + } + return result; +} + +Status ReadFooterFromFile(const IOOptions& opts, RandomAccessFileReader* file, + FileSystem& fs, FilePrefetchBuffer* prefetch_buffer, + uint64_t file_size, Footer* footer, + uint64_t enforce_table_magic_number) { + if (file_size < Footer::kMinEncodedLength) { + return Status::Corruption("file is too short (" + + std::to_string(file_size) + + " bytes) to be an " + "sstable: " + + file->file_name()); + } + + std::string footer_buf; + AlignedBuf internal_buf; + Slice footer_input; + uint64_t read_offset = (file_size > Footer::kMaxEncodedLength) + ? file_size - Footer::kMaxEncodedLength + : 0; + Status s; + // TODO: Need to pass appropriate deadline to TryReadFromCache(). Right now, + // there is no readahead for point lookups, so TryReadFromCache will fail if + // the required data is not in the prefetch buffer. Once deadline is enabled + // for iterator, TryReadFromCache might do a readahead. Revisit to see if we + // need to pass a timeout at that point + // TODO: rate limit footer reads. + if (prefetch_buffer == nullptr || + !prefetch_buffer->TryReadFromCache( + opts, file, read_offset, Footer::kMaxEncodedLength, &footer_input, + nullptr, opts.rate_limiter_priority)) { + if (file->use_direct_io()) { + s = file->Read(opts, read_offset, Footer::kMaxEncodedLength, + &footer_input, nullptr, &internal_buf, + opts.rate_limiter_priority); + } else { + footer_buf.reserve(Footer::kMaxEncodedLength); + s = file->Read(opts, read_offset, Footer::kMaxEncodedLength, + &footer_input, &footer_buf[0], nullptr, + opts.rate_limiter_priority); + } + if (!s.ok()) return s; + } + + // Check that we actually read the whole footer from the file. It may be + // that size isn't correct. + if (footer_input.size() < Footer::kMinEncodedLength) { + uint64_t size_on_disk = 0; + if (fs.GetFileSize(file->file_name(), IOOptions(), &size_on_disk, nullptr) + .ok()) { + // Similar to CheckConsistency message, but not completely sure the + // expected size always came from manifest. + return Status::Corruption("Sst file size mismatch: " + file->file_name() + + ". Expected " + std::to_string(file_size) + + ", actual size " + + std::to_string(size_on_disk) + "\n"); + } else { + return Status::Corruption( + "Missing SST footer data in file " + file->file_name() + + " File too short? Expected size: " + std::to_string(file_size)); + } + } + + s = footer->DecodeFrom(footer_input, read_offset, enforce_table_magic_number); + if (!s.ok()) { + s = Status::CopyAppendMessage(s, " in ", file->file_name()); + return s; + } + return Status::OK(); +} + +namespace { +// Custom handling for the last byte of a block, to avoid invoking streaming +// API to get an effective block checksum. This function is its own inverse +// because it uses xor. +inline uint32_t ModifyChecksumForLastByte(uint32_t checksum, char last_byte) { + // This strategy bears some resemblance to extending a CRC checksum by one + // more byte, except we don't need to re-mix the input checksum as long as + // we do this step only once (per checksum). + const uint32_t kRandomPrime = 0x6b9083d9; + return checksum ^ lossless_cast(last_byte) * kRandomPrime; +} +} // namespace + +uint32_t ComputeBuiltinChecksum(ChecksumType type, const char* data, + size_t data_size) { + switch (type) { + case kCRC32c: + return crc32c::Mask(crc32c::Value(data, data_size)); + case kxxHash: + return XXH32(data, data_size, /*seed*/ 0); + case kxxHash64: + return Lower32of64(XXH64(data, data_size, /*seed*/ 0)); + case kXXH3: { + if (data_size == 0) { + // Special case because of special handling for last byte, not + // present in this case. Can be any value different from other + // small input size checksums. + return 0; + } else { + // See corresponding code in ComputeBuiltinChecksumWithLastByte + uint32_t v = Lower32of64(XXH3_64bits(data, data_size - 1)); + return ModifyChecksumForLastByte(v, data[data_size - 1]); + } + } + default: // including kNoChecksum + return 0; + } +} + +uint32_t ComputeBuiltinChecksumWithLastByte(ChecksumType type, const char* data, + size_t data_size, char last_byte) { + switch (type) { + case kCRC32c: { + uint32_t crc = crc32c::Value(data, data_size); + // Extend to cover last byte (compression type) + crc = crc32c::Extend(crc, &last_byte, 1); + return crc32c::Mask(crc); + } + case kxxHash: { + XXH32_state_t* const state = XXH32_createState(); + XXH32_reset(state, 0); + XXH32_update(state, data, data_size); + // Extend to cover last byte (compression type) + XXH32_update(state, &last_byte, 1); + uint32_t v = XXH32_digest(state); + XXH32_freeState(state); + return v; + } + case kxxHash64: { + XXH64_state_t* const state = XXH64_createState(); + XXH64_reset(state, 0); + XXH64_update(state, data, data_size); + // Extend to cover last byte (compression type) + XXH64_update(state, &last_byte, 1); + uint32_t v = Lower32of64(XXH64_digest(state)); + XXH64_freeState(state); + return v; + } + case kXXH3: { + // XXH3 is a complicated hash function that is extremely fast on + // contiguous input, but that makes its streaming support rather + // complex. It is worth custom handling of the last byte (`type`) + // in order to avoid allocating a large state object and bringing + // that code complexity into CPU working set. + uint32_t v = Lower32of64(XXH3_64bits(data, data_size)); + return ModifyChecksumForLastByte(v, last_byte); + } + default: // including kNoChecksum + return 0; + } +} + +Status UncompressBlockData(const UncompressionInfo& uncompression_info, + const char* data, size_t size, + BlockContents* out_contents, uint32_t format_version, + const ImmutableOptions& ioptions, + MemoryAllocator* allocator) { + Status ret = Status::OK(); + + assert(uncompression_info.type() != kNoCompression && + "Invalid compression type"); + + StopWatchNano timer(ioptions.clock, + ShouldReportDetailedTime(ioptions.env, ioptions.stats)); + size_t uncompressed_size = 0; + CacheAllocationPtr ubuf = + UncompressData(uncompression_info, data, size, &uncompressed_size, + GetCompressFormatForVersion(format_version), allocator); + if (!ubuf) { + if (!CompressionTypeSupported(uncompression_info.type())) { + return Status::NotSupported( + "Unsupported compression method for this build", + CompressionTypeToString(uncompression_info.type())); + } else { + return Status::Corruption( + "Corrupted compressed block contents", + CompressionTypeToString(uncompression_info.type())); + } + } + + *out_contents = BlockContents(std::move(ubuf), uncompressed_size); + + if (ShouldReportDetailedTime(ioptions.env, ioptions.stats)) { + RecordTimeToHistogram(ioptions.stats, DECOMPRESSION_TIMES_NANOS, + timer.ElapsedNanos()); + } + RecordTick(ioptions.stats, BYTES_DECOMPRESSED_FROM, size); + RecordTick(ioptions.stats, BYTES_DECOMPRESSED_TO, out_contents->data.size()); + RecordTick(ioptions.stats, NUMBER_BLOCK_DECOMPRESSED); + + TEST_SYNC_POINT_CALLBACK("UncompressBlockData:TamperWithReturnValue", + static_cast(&ret)); + TEST_SYNC_POINT_CALLBACK( + "UncompressBlockData:" + "TamperWithDecompressionOutput", + static_cast(out_contents)); + + return ret; +} + +Status UncompressSerializedBlock(const UncompressionInfo& uncompression_info, + const char* data, size_t size, + BlockContents* out_contents, + uint32_t format_version, + const ImmutableOptions& ioptions, + MemoryAllocator* allocator) { + assert(data[size] != kNoCompression); + assert(data[size] == static_cast(uncompression_info.type())); + return UncompressBlockData(uncompression_info, data, size, out_contents, + format_version, ioptions, allocator); +} + +// Replace the contents of db_host_id with the actual hostname, if db_host_id +// matches the keyword kHostnameForDbHostId +Status ReifyDbHostIdProperty(Env* env, std::string* db_host_id) { + assert(db_host_id); + if (*db_host_id == kHostnameForDbHostId) { + Status s = env->GetHostNameString(db_host_id); + if (!s.ok()) { + db_host_id->clear(); + } + return s; + } + + return Status::OK(); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/format.h b/librocksdb-sys/rocksdb/table/format.h new file mode 100644 index 0000000..7367538 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/format.h @@ -0,0 +1,431 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include +#include +#include + +#include "file/file_prefetch_buffer.h" +#include "file/random_access_file_reader.h" +#include "memory/memory_allocator_impl.h" +#include "options/cf_options.h" +#include "port/malloc.h" +#include "port/port.h" // noexcept +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "rocksdb/table.h" +#include "util/hash.h" + +namespace ROCKSDB_NAMESPACE { + +class RandomAccessFile; +struct ReadOptions; + +bool ShouldReportDetailedTime(Env* env, Statistics* stats); + +// the length of the magic number in bytes. +constexpr uint32_t kMagicNumberLengthByte = 8; + +// BlockHandle is a pointer to the extent of a file that stores a data +// block or a meta block. +class BlockHandle { + public: + // Creates a block handle with special values indicating "uninitialized," + // distinct from the "null" block handle. + BlockHandle(); + BlockHandle(uint64_t offset, uint64_t size); + + // The offset of the block in the file. + uint64_t offset() const { return offset_; } + void set_offset(uint64_t _offset) { offset_ = _offset; } + + // The size of the stored block + uint64_t size() const { return size_; } + void set_size(uint64_t _size) { size_ = _size; } + + void EncodeTo(std::string* dst) const; + char* EncodeTo(char* dst) const; + Status DecodeFrom(Slice* input); + Status DecodeSizeFrom(uint64_t offset, Slice* input); + + // Return a string that contains the copy of handle. + std::string ToString(bool hex = true) const; + + // if the block handle's offset and size are both "0", we will view it + // as a null block handle that points to no where. + bool IsNull() const { return offset_ == 0 && size_ == 0; } + + static const BlockHandle& NullBlockHandle() { return kNullBlockHandle; } + + // Maximum encoding length of a BlockHandle + static constexpr uint32_t kMaxEncodedLength = 2 * kMaxVarint64Length; + + inline bool operator==(const BlockHandle& rhs) const { + return offset_ == rhs.offset_ && size_ == rhs.size_; + } + inline bool operator!=(const BlockHandle& rhs) const { + return !(*this == rhs); + } + + private: + uint64_t offset_; + uint64_t size_; + + static const BlockHandle kNullBlockHandle; +}; + +// Value in block-based table file index. +// +// The index entry for block n is: y -> h, [x], +// where: y is some key between the last key of block n (inclusive) and the +// first key of block n+1 (exclusive); h is BlockHandle pointing to block n; +// x, if present, is the first key of block n (unshortened). +// This struct represents the "h, [x]" part. +struct IndexValue { + BlockHandle handle; + // Empty means unknown. + Slice first_internal_key; + + IndexValue() = default; + IndexValue(BlockHandle _handle, Slice _first_internal_key) + : handle(_handle), first_internal_key(_first_internal_key) {} + + // have_first_key indicates whether the `first_internal_key` is used. + // If previous_handle is not null, delta encoding is used; + // in this case, the two handles must point to consecutive blocks: + // handle.offset() == + // previous_handle->offset() + previous_handle->size() + kBlockTrailerSize + void EncodeTo(std::string* dst, bool have_first_key, + const BlockHandle* previous_handle) const; + Status DecodeFrom(Slice* input, bool have_first_key, + const BlockHandle* previous_handle); + + std::string ToString(bool hex, bool have_first_key) const; +}; + +// Given a file's base_context_checksum and an offset of a block within that +// file, choose a 32-bit value that is as unique as possible. This value will +// be added to the standard checksum to get a checksum "with context," or can +// be subtracted to "remove" context. Returns zero (no modifier) if feature is +// disabled with base_context_checksum == 0. +inline uint32_t ChecksumModifierForContext(uint32_t base_context_checksum, + uint64_t offset) { + // To disable on base_context_checksum == 0, we could write + // `if (base_context_checksum == 0) return 0;` but benchmarking shows + // measurable performance penalty vs. this: compute the modifier + // unconditionally and use an "all or nothing" bit mask to enable + // or disable. + uint32_t all_or_nothing = uint32_t{0} - (base_context_checksum != 0); + + // Desired properties: + // (call this function f(b, o) where b = base and o = offset) + // 1. Fast + // 2. f(b1, o) == f(b2, o) iff b1 == b2 + // (Perfectly preserve base entropy) + // 3. f(b, o1) == f(b, o2) only if o1 == o2 or |o1-o2| >= 4 billion + // (Guaranteed uniqueness for nearby offsets) + // 3. f(b, o + j * 2**32) == f(b, o + k * 2**32) only if j == k + // (Upper bits matter, and *aligned* misplacement fails check) + // 4. f(b1, o) == f(b2, o + x) then preferably not + // f(b1, o + y) == f(b2, o + x + y) + // (Avoid linearly correlated matches) + // 5. f(b, o) == 0 depends on both b and o + // (No predictable overlap with non-context checksums) + uint32_t modifier = + base_context_checksum ^ (Lower32of64(offset) + Upper32of64(offset)); + + return modifier & all_or_nothing; +} + +inline uint32_t GetCompressFormatForVersion(uint32_t format_version) { + // As of format_version 2, we encode compressed block with + // compress_format_version == 2. Before that, the version is 1. + // DO NOT CHANGE THIS FUNCTION, it affects disk format + return format_version >= 2 ? 2 : 1; +} + +constexpr uint32_t kLatestFormatVersion = 6; + +inline bool IsSupportedFormatVersion(uint32_t version) { + return version <= kLatestFormatVersion; +} + +// Same as having a unique id in footer. +inline bool FormatVersionUsesContextChecksum(uint32_t version) { + return version >= 6; +} + +inline bool FormatVersionUsesIndexHandleInFooter(uint32_t version) { + return version < 6; +} + +// Footer encapsulates the fixed information stored at the tail end of every +// SST file. In general, it should only include things that cannot go +// elsewhere under the metaindex block. For example, checksum_type is +// required for verifying metaindex block checksum (when applicable), but +// index block handle can easily go in metaindex block. See also FooterBuilder +// below. +class Footer { + public: + // Create empty. Populate using DecodeFrom. + Footer() {} + + // Deserialize a footer (populate fields) from `input` and check for various + // corruptions. `input_offset` is the offset within the target file of + // `input` buffer, which is needed for verifying format_version >= 6 footer. + // If enforce_table_magic_number != 0, will return corruption if table magic + // number is not equal to enforce_table_magic_number. + Status DecodeFrom(Slice input, uint64_t input_offset, + uint64_t enforce_table_magic_number = 0); + + // Table magic number identifies file as RocksDB SST file and which kind of + // SST format is use. + uint64_t table_magic_number() const { return table_magic_number_; } + + // A version (footer and more) within a kind of SST. (It would add more + // unnecessary complexity to separate footer versions and + // BBTO::format_version.) + uint32_t format_version() const { return format_version_; } + + // See ChecksumModifierForContext() + uint32_t base_context_checksum() const { return base_context_checksum_; } + + // Block handle for metaindex block. + const BlockHandle& metaindex_handle() const { return metaindex_handle_; } + + // Block handle for (top-level) index block. + // TODO? remove from this struct and only read on decode for legacy cases + const BlockHandle& index_handle() const { return index_handle_; } + + // Checksum type used in the file, including footer for format version >= 6. + ChecksumType checksum_type() const { + return static_cast(checksum_type_); + } + + // Block trailer size used by file with this footer (e.g. 5 for block-based + // table and 0 for plain table). This is inferred from magic number so + // not in the serialized form. + inline size_t GetBlockTrailerSize() const { return block_trailer_size_; } + + // Convert this object to a human readable form + std::string ToString() const; + + // Encoded lengths of Footers. Bytes for serialized Footer will always be + // >= kMinEncodedLength and <= kMaxEncodedLength. + // + // Footer version 0 (legacy) will always occupy exactly this many bytes. + // It consists of two block handles, padding, and a magic number. + static constexpr uint32_t kVersion0EncodedLength = + 2 * BlockHandle::kMaxEncodedLength + kMagicNumberLengthByte; + static constexpr uint32_t kMinEncodedLength = kVersion0EncodedLength; + + // Footer of versions 1 and higher will always occupy exactly this many + // bytes. It originally consisted of the checksum type, two block handles, + // padding (to maximum handle encoding size), a format version number, and a + // magic number. + static constexpr uint32_t kNewVersionsEncodedLength = + 1 + 2 * BlockHandle::kMaxEncodedLength + 4 + kMagicNumberLengthByte; + static constexpr uint32_t kMaxEncodedLength = kNewVersionsEncodedLength; + + static constexpr uint64_t kNullTableMagicNumber = 0; + + static constexpr uint32_t kInvalidFormatVersion = 0xffffffffU; + + private: + static constexpr int kInvalidChecksumType = + (1 << (sizeof(ChecksumType) * 8)) | kNoChecksum; + + uint64_t table_magic_number_ = kNullTableMagicNumber; + uint32_t format_version_ = kInvalidFormatVersion; + uint32_t base_context_checksum_ = 0; + BlockHandle metaindex_handle_; + BlockHandle index_handle_; + int checksum_type_ = kInvalidChecksumType; + uint8_t block_trailer_size_ = 0; +}; + +// Builder for Footer +class FooterBuilder { + public: + // Run builder in inputs. This is a single step with lots of parameters for + // efficiency (based on perf testing). + // * table_magic_number identifies file as RocksDB SST file and which kind of + // SST format is use. + // * format_version is a version for the footer and can also apply to other + // aspects of the SST file (see BlockBasedTableOptions::format_version). + // NOTE: To save complexity in the caller, when format_version == 0 and + // there is a corresponding legacy magic number to the one specified, the + // legacy magic number will be written for forward compatibility. + // * footer_offset is the file offset where the footer will be written + // (for future use). + // * checksum_type is for formats using block checksums. + // * index_handle is optional for some SST kinds and (for caller convenience) + // ignored when format_version >= 6. (Must be added to metaindex in that + // case.) + // * unique_id must be specified if format_vesion >= 6 and SST uses block + // checksums with context. Otherwise, auto-generated if format_vesion >= 6. + Status Build(uint64_t table_magic_number, uint32_t format_version, + uint64_t footer_offset, ChecksumType checksum_type, + const BlockHandle& metaindex_handle, + const BlockHandle& index_handle = BlockHandle::NullBlockHandle(), + uint32_t base_context_checksum = 0); + + // After Builder, get a Slice for the serialized Footer, backed by this + // FooterBuilder. + const Slice& GetSlice() const { + assert(slice_.size()); + return slice_; + } + + private: + Slice slice_; + std::array data_; +}; + +// Read the footer from file +// If enforce_table_magic_number != 0, ReadFooterFromFile() will return +// corruption if table_magic number is not equal to enforce_table_magic_number +Status ReadFooterFromFile(const IOOptions& opts, RandomAccessFileReader* file, + FileSystem& fs, FilePrefetchBuffer* prefetch_buffer, + uint64_t file_size, Footer* footer, + uint64_t enforce_table_magic_number = 0); + +// Computes a checksum using the given ChecksumType. Sometimes we need to +// include one more input byte logically at the end but not part of the main +// data buffer. If data_size >= 1, then +// ComputeBuiltinChecksum(type, data, size) +// == +// ComputeBuiltinChecksumWithLastByte(type, data, size - 1, data[size - 1]) +uint32_t ComputeBuiltinChecksum(ChecksumType type, const char* data, + size_t size); +uint32_t ComputeBuiltinChecksumWithLastByte(ChecksumType type, const char* data, + size_t size, char last_byte); + +// Represents the contents of a block read from an SST file. Depending on how +// it's created, it may or may not own the actual block bytes. As an example, +// BlockContents objects representing data read from mmapped files only point +// into the mmapped region. Depending on context, it might be a serialized +// (potentially compressed) block, including a trailer beyond `size`, or an +// uncompressed block. +// +// Please try to use this terminology when dealing with blocks: +// * "Serialized block" - bytes that go into storage. For block-based table +// (usually the case) this includes the block trailer. Here the `size` does +// not include the trailer, but other places in code might include the trailer +// in the size. +// * "Maybe compressed block" - like a serialized block, but without the +// trailer (or no promise of including a trailer). Must be accompanied by a +// CompressionType in some other variable or field. +// * "Uncompressed block" - "payload" bytes that are either stored with no +// compression, used as input to compression function, or result of +// decompression function. +// * "Parsed block" - an in-memory form of a block in block cache, as it is +// used by the table reader. Different C++ types are used depending on the +// block type (see block_cache.h). Only trivially parsable block types +// use BlockContents as the parsed form. +// +struct BlockContents { + // Points to block payload (without trailer) + Slice data; + CacheAllocationPtr allocation; + +#ifndef NDEBUG + // Whether there is a known trailer after what is pointed to by `data`. + // See BlockBasedTable::GetCompressionType. + bool has_trailer = false; +#endif // NDEBUG + + BlockContents() {} + + // Does not take ownership of the underlying data bytes. + BlockContents(const Slice& _data) : data(_data) {} + + // Takes ownership of the underlying data bytes. + BlockContents(CacheAllocationPtr&& _data, size_t _size) + : data(_data.get(), _size), allocation(std::move(_data)) {} + + // Takes ownership of the underlying data bytes. + BlockContents(std::unique_ptr&& _data, size_t _size) + : data(_data.get(), _size) { + allocation.reset(_data.release()); + } + + // Returns whether the object has ownership of the underlying data bytes. + bool own_bytes() const { return allocation.get() != nullptr; } + + // The additional memory space taken by the block data. + size_t usable_size() const { + if (allocation.get() != nullptr) { + auto allocator = allocation.get_deleter().allocator; + if (allocator) { + return allocator->UsableSize(allocation.get(), data.size()); + } +#ifdef ROCKSDB_MALLOC_USABLE_SIZE + return malloc_usable_size(allocation.get()); +#else + return data.size(); +#endif // ROCKSDB_MALLOC_USABLE_SIZE + } else { + return 0; // no extra memory is occupied by the data + } + } + + size_t ApproximateMemoryUsage() const { + return usable_size() + sizeof(*this); + } + + BlockContents(BlockContents&& other) noexcept { *this = std::move(other); } + + BlockContents& operator=(BlockContents&& other) { + data = std::move(other.data); + allocation = std::move(other.allocation); +#ifndef NDEBUG + has_trailer = other.has_trailer; +#endif // NDEBUG + return *this; + } +}; + +// The `data` points to serialized block contents read in from file, which +// must be compressed and include a trailer beyond `size`. A new buffer is +// allocated with the given allocator (or default) and the uncompressed +// contents are returned in `out_contents`. +// format_version is as defined in include/rocksdb/table.h, which is +// used to determine compression format version. +Status UncompressSerializedBlock(const UncompressionInfo& info, + const char* data, size_t size, + BlockContents* out_contents, + uint32_t format_version, + const ImmutableOptions& ioptions, + MemoryAllocator* allocator = nullptr); + +// This is a variant of UncompressSerializedBlock that does not expect a +// block trailer beyond `size`. (CompressionType is taken from `info`.) +Status UncompressBlockData(const UncompressionInfo& info, const char* data, + size_t size, BlockContents* out_contents, + uint32_t format_version, + const ImmutableOptions& ioptions, + MemoryAllocator* allocator = nullptr); + +// Replace db_host_id contents with the real hostname if necessary +Status ReifyDbHostIdProperty(Env* env, std::string* db_host_id); + +// Implementation details follow. Clients should ignore, + +// TODO(andrewkr): we should prefer one way of representing a null/uninitialized +// BlockHandle. Currently we use zeros for null and use negation-of-zeros for +// uninitialized. +inline BlockHandle::BlockHandle() : BlockHandle(~uint64_t{0}, ~uint64_t{0}) {} + +inline BlockHandle::BlockHandle(uint64_t _offset, uint64_t _size) + : offset_(_offset), size_(_size) {} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/get_context.cc b/librocksdb-sys/rocksdb/table/get_context.cc new file mode 100644 index 0000000..8f5cd75 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/get_context.cc @@ -0,0 +1,616 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "table/get_context.h" + +#include "db/blob//blob_fetcher.h" +#include "db/merge_helper.h" +#include "db/pinned_iterators_manager.h" +#include "db/read_callback.h" +#include "db/wide/wide_column_serialization.h" +#include "monitoring/file_read_sample.h" +#include "monitoring/perf_context_imp.h" +#include "monitoring/statistics_impl.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/statistics.h" +#include "rocksdb/system_clock.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { + +void appendToReplayLog(std::string* replay_log, ValueType type, Slice value) { + if (replay_log) { + if (replay_log->empty()) { + // Optimization: in the common case of only one operation in the + // log, we allocate the exact amount of space needed. + replay_log->reserve(1 + VarintLength(value.size()) + value.size()); + } + replay_log->push_back(type); + PutLengthPrefixedSlice(replay_log, value); + } +} + +} // namespace + +GetContext::GetContext( + const Comparator* ucmp, const MergeOperator* merge_operator, Logger* logger, + Statistics* statistics, GetState init_state, const Slice& user_key, + PinnableSlice* pinnable_val, PinnableWideColumns* columns, + std::string* timestamp, bool* value_found, MergeContext* merge_context, + bool do_merge, SequenceNumber* _max_covering_tombstone_seq, + SystemClock* clock, SequenceNumber* seq, + PinnedIteratorsManager* _pinned_iters_mgr, ReadCallback* callback, + bool* is_blob_index, uint64_t tracing_get_id, BlobFetcher* blob_fetcher) + : ucmp_(ucmp), + merge_operator_(merge_operator), + logger_(logger), + statistics_(statistics), + state_(init_state), + user_key_(user_key), + pinnable_val_(pinnable_val), + columns_(columns), + timestamp_(timestamp), + value_found_(value_found), + merge_context_(merge_context), + max_covering_tombstone_seq_(_max_covering_tombstone_seq), + clock_(clock), + seq_(seq), + replay_log_(nullptr), + pinned_iters_mgr_(_pinned_iters_mgr), + callback_(callback), + do_merge_(do_merge), + is_blob_index_(is_blob_index), + tracing_get_id_(tracing_get_id), + blob_fetcher_(blob_fetcher) { + if (seq_) { + *seq_ = kMaxSequenceNumber; + } + sample_ = should_sample_file_read(); +} + +GetContext::GetContext(const Comparator* ucmp, + const MergeOperator* merge_operator, Logger* logger, + Statistics* statistics, GetState init_state, + const Slice& user_key, PinnableSlice* pinnable_val, + PinnableWideColumns* columns, bool* value_found, + MergeContext* merge_context, bool do_merge, + SequenceNumber* _max_covering_tombstone_seq, + SystemClock* clock, SequenceNumber* seq, + PinnedIteratorsManager* _pinned_iters_mgr, + ReadCallback* callback, bool* is_blob_index, + uint64_t tracing_get_id, BlobFetcher* blob_fetcher) + : GetContext(ucmp, merge_operator, logger, statistics, init_state, user_key, + pinnable_val, columns, /*timestamp=*/nullptr, value_found, + merge_context, do_merge, _max_covering_tombstone_seq, clock, + seq, _pinned_iters_mgr, callback, is_blob_index, + tracing_get_id, blob_fetcher) {} + +// Called from TableCache::Get and Table::Get when file/block in which +// key may exist are not there in TableCache/BlockCache respectively. In this +// case we can't guarantee that key does not exist and are not permitted to do +// IO to be certain.Set the status=kFound and value_found=false to let the +// caller know that key may exist but is not there in memory +void GetContext::MarkKeyMayExist() { + state_ = kFound; + if (value_found_ != nullptr) { + *value_found_ = false; + } +} + +void GetContext::SaveValue(const Slice& value, SequenceNumber /*seq*/) { + assert(state_ == kNotFound); + appendToReplayLog(replay_log_, kTypeValue, value); + + state_ = kFound; + if (LIKELY(pinnable_val_ != nullptr)) { + pinnable_val_->PinSelf(value); + } +} + +void GetContext::ReportCounters() { + if (get_context_stats_.num_cache_hit > 0) { + RecordTick(statistics_, BLOCK_CACHE_HIT, get_context_stats_.num_cache_hit); + } + if (get_context_stats_.num_cache_index_hit > 0) { + RecordTick(statistics_, BLOCK_CACHE_INDEX_HIT, + get_context_stats_.num_cache_index_hit); + } + if (get_context_stats_.num_cache_data_hit > 0) { + RecordTick(statistics_, BLOCK_CACHE_DATA_HIT, + get_context_stats_.num_cache_data_hit); + } + if (get_context_stats_.num_cache_filter_hit > 0) { + RecordTick(statistics_, BLOCK_CACHE_FILTER_HIT, + get_context_stats_.num_cache_filter_hit); + } + if (get_context_stats_.num_cache_compression_dict_hit > 0) { + RecordTick(statistics_, BLOCK_CACHE_COMPRESSION_DICT_HIT, + get_context_stats_.num_cache_compression_dict_hit); + } + if (get_context_stats_.num_cache_index_miss > 0) { + RecordTick(statistics_, BLOCK_CACHE_INDEX_MISS, + get_context_stats_.num_cache_index_miss); + } + if (get_context_stats_.num_cache_filter_miss > 0) { + RecordTick(statistics_, BLOCK_CACHE_FILTER_MISS, + get_context_stats_.num_cache_filter_miss); + } + if (get_context_stats_.num_cache_data_miss > 0) { + RecordTick(statistics_, BLOCK_CACHE_DATA_MISS, + get_context_stats_.num_cache_data_miss); + } + if (get_context_stats_.num_cache_compression_dict_miss > 0) { + RecordTick(statistics_, BLOCK_CACHE_COMPRESSION_DICT_MISS, + get_context_stats_.num_cache_compression_dict_miss); + } + if (get_context_stats_.num_cache_bytes_read > 0) { + RecordTick(statistics_, BLOCK_CACHE_BYTES_READ, + get_context_stats_.num_cache_bytes_read); + } + if (get_context_stats_.num_cache_miss > 0) { + RecordTick(statistics_, BLOCK_CACHE_MISS, + get_context_stats_.num_cache_miss); + } + if (get_context_stats_.num_cache_add > 0) { + RecordTick(statistics_, BLOCK_CACHE_ADD, get_context_stats_.num_cache_add); + } + if (get_context_stats_.num_cache_add_redundant > 0) { + RecordTick(statistics_, BLOCK_CACHE_ADD_REDUNDANT, + get_context_stats_.num_cache_add_redundant); + } + if (get_context_stats_.num_cache_bytes_write > 0) { + RecordTick(statistics_, BLOCK_CACHE_BYTES_WRITE, + get_context_stats_.num_cache_bytes_write); + } + if (get_context_stats_.num_cache_index_add > 0) { + RecordTick(statistics_, BLOCK_CACHE_INDEX_ADD, + get_context_stats_.num_cache_index_add); + } + if (get_context_stats_.num_cache_index_add_redundant > 0) { + RecordTick(statistics_, BLOCK_CACHE_INDEX_ADD_REDUNDANT, + get_context_stats_.num_cache_index_add_redundant); + } + if (get_context_stats_.num_cache_index_bytes_insert > 0) { + RecordTick(statistics_, BLOCK_CACHE_INDEX_BYTES_INSERT, + get_context_stats_.num_cache_index_bytes_insert); + } + if (get_context_stats_.num_cache_data_add > 0) { + RecordTick(statistics_, BLOCK_CACHE_DATA_ADD, + get_context_stats_.num_cache_data_add); + } + if (get_context_stats_.num_cache_data_add_redundant > 0) { + RecordTick(statistics_, BLOCK_CACHE_DATA_ADD_REDUNDANT, + get_context_stats_.num_cache_data_add_redundant); + } + if (get_context_stats_.num_cache_data_bytes_insert > 0) { + RecordTick(statistics_, BLOCK_CACHE_DATA_BYTES_INSERT, + get_context_stats_.num_cache_data_bytes_insert); + } + if (get_context_stats_.num_cache_filter_add > 0) { + RecordTick(statistics_, BLOCK_CACHE_FILTER_ADD, + get_context_stats_.num_cache_filter_add); + } + if (get_context_stats_.num_cache_filter_add_redundant > 0) { + RecordTick(statistics_, BLOCK_CACHE_FILTER_ADD_REDUNDANT, + get_context_stats_.num_cache_filter_add_redundant); + } + if (get_context_stats_.num_cache_filter_bytes_insert > 0) { + RecordTick(statistics_, BLOCK_CACHE_FILTER_BYTES_INSERT, + get_context_stats_.num_cache_filter_bytes_insert); + } + if (get_context_stats_.num_cache_compression_dict_add > 0) { + RecordTick(statistics_, BLOCK_CACHE_COMPRESSION_DICT_ADD, + get_context_stats_.num_cache_compression_dict_add); + } + if (get_context_stats_.num_cache_compression_dict_add_redundant > 0) { + RecordTick(statistics_, BLOCK_CACHE_COMPRESSION_DICT_ADD_REDUNDANT, + get_context_stats_.num_cache_compression_dict_add_redundant); + } + if (get_context_stats_.num_cache_compression_dict_bytes_insert > 0) { + RecordTick(statistics_, BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT, + get_context_stats_.num_cache_compression_dict_bytes_insert); + } +} + +bool GetContext::SaveValue(const ParsedInternalKey& parsed_key, + const Slice& value, bool* matched, + Cleanable* value_pinner) { + assert(matched); + assert((state_ != kMerge && parsed_key.type != kTypeMerge) || + merge_context_ != nullptr); + if (ucmp_->EqualWithoutTimestamp(parsed_key.user_key, user_key_)) { + *matched = true; + // If the value is not in the snapshot, skip it + if (!CheckCallback(parsed_key.sequence)) { + return true; // to continue to the next seq + } + + appendToReplayLog(replay_log_, parsed_key.type, value); + + if (seq_ != nullptr) { + // Set the sequence number if it is uninitialized + if (*seq_ == kMaxSequenceNumber) { + *seq_ = parsed_key.sequence; + } + if (max_covering_tombstone_seq_) { + *seq_ = std::max(*seq_, *max_covering_tombstone_seq_); + } + } + + size_t ts_sz = ucmp_->timestamp_size(); + if (ts_sz > 0 && timestamp_ != nullptr) { + if (!timestamp_->empty()) { + assert(ts_sz == timestamp_->size()); + // `timestamp` can be set before `SaveValue` is ever called + // when max_covering_tombstone_seq_ was set. + // If this key has a higher sequence number than range tombstone, + // then timestamp should be updated. `ts_from_rangetombstone_` is + // set to false afterwards so that only the key with highest seqno + // updates the timestamp. + if (ts_from_rangetombstone_) { + assert(max_covering_tombstone_seq_); + if (parsed_key.sequence > *max_covering_tombstone_seq_) { + Slice ts = ExtractTimestampFromUserKey(parsed_key.user_key, ts_sz); + timestamp_->assign(ts.data(), ts.size()); + ts_from_rangetombstone_ = false; + } + } + } + // TODO optimize for small size ts + const std::string kMaxTs(ts_sz, '\xff'); + if (timestamp_->empty() || + ucmp_->CompareTimestamp(*timestamp_, kMaxTs) == 0) { + Slice ts = ExtractTimestampFromUserKey(parsed_key.user_key, ts_sz); + timestamp_->assign(ts.data(), ts.size()); + } + } + + auto type = parsed_key.type; + // Key matches. Process it + if ((type == kTypeValue || type == kTypeMerge || type == kTypeBlobIndex || + type == kTypeWideColumnEntity || type == kTypeDeletion || + type == kTypeDeletionWithTimestamp || type == kTypeSingleDeletion) && + max_covering_tombstone_seq_ != nullptr && + *max_covering_tombstone_seq_ > parsed_key.sequence) { + // Note that deletion types are also considered, this is for the case + // when we need to return timestamp to user. If a range tombstone has a + // higher seqno than point tombstone, its timestamp should be returned. + type = kTypeRangeDeletion; + } + switch (type) { + case kTypeValue: + case kTypeBlobIndex: + case kTypeWideColumnEntity: + assert(state_ == kNotFound || state_ == kMerge); + if (type == kTypeBlobIndex) { + if (is_blob_index_ == nullptr) { + // Blob value not supported. Stop. + state_ = kUnexpectedBlobIndex; + return false; + } + } + + if (is_blob_index_ != nullptr) { + *is_blob_index_ = (type == kTypeBlobIndex); + } + + if (kNotFound == state_) { + state_ = kFound; + if (do_merge_) { + if (type == kTypeBlobIndex && ucmp_->timestamp_size() != 0) { + ukey_with_ts_found_.PinSelf(parsed_key.user_key); + } + if (LIKELY(pinnable_val_ != nullptr)) { + Slice value_to_use = value; + + if (type == kTypeWideColumnEntity) { + Slice value_copy = value; + + if (!WideColumnSerialization::GetValueOfDefaultColumn( + value_copy, value_to_use) + .ok()) { + state_ = kCorrupt; + return false; + } + } + + if (LIKELY(value_pinner != nullptr)) { + // If the backing resources for the value are provided, pin them + pinnable_val_->PinSlice(value_to_use, value_pinner); + } else { + TEST_SYNC_POINT_CALLBACK("GetContext::SaveValue::PinSelf", + this); + // Otherwise copy the value + pinnable_val_->PinSelf(value_to_use); + } + } else if (columns_ != nullptr) { + if (type == kTypeWideColumnEntity) { + if (!columns_->SetWideColumnValue(value, value_pinner).ok()) { + state_ = kCorrupt; + return false; + } + } else { + columns_->SetPlainValue(value, value_pinner); + } + } + } else { + // It means this function is called as part of DB GetMergeOperands + // API and the current value should be part of + // merge_context_->operand_list + if (type == kTypeBlobIndex) { + PinnableSlice pin_val; + if (GetBlobValue(parsed_key.user_key, value, &pin_val) == false) { + return false; + } + Slice blob_value(pin_val); + push_operand(blob_value, nullptr); + } else if (type == kTypeWideColumnEntity) { + Slice value_copy = value; + Slice value_of_default; + + if (!WideColumnSerialization::GetValueOfDefaultColumn( + value_copy, value_of_default) + .ok()) { + state_ = kCorrupt; + return false; + } + + push_operand(value_of_default, value_pinner); + } else { + assert(type == kTypeValue); + push_operand(value, value_pinner); + } + } + } else if (kMerge == state_) { + assert(merge_operator_ != nullptr); + if (type == kTypeBlobIndex) { + PinnableSlice pin_val; + if (GetBlobValue(parsed_key.user_key, value, &pin_val) == false) { + return false; + } + Slice blob_value(pin_val); + state_ = kFound; + if (do_merge_) { + Merge(&blob_value); + } else { + // It means this function is called as part of DB GetMergeOperands + // API and the current value should be part of + // merge_context_->operand_list + push_operand(blob_value, nullptr); + } + } else if (type == kTypeWideColumnEntity) { + state_ = kFound; + + if (do_merge_) { + MergeWithEntity(value); + } else { + // It means this function is called as part of DB GetMergeOperands + // API and the current value should be part of + // merge_context_->operand_list + Slice value_copy = value; + Slice value_of_default; + + if (!WideColumnSerialization::GetValueOfDefaultColumn( + value_copy, value_of_default) + .ok()) { + state_ = kCorrupt; + return false; + } + + push_operand(value_of_default, value_pinner); + } + } else { + assert(type == kTypeValue); + + state_ = kFound; + if (do_merge_) { + Merge(&value); + } else { + // It means this function is called as part of DB GetMergeOperands + // API and the current value should be part of + // merge_context_->operand_list + push_operand(value, value_pinner); + } + } + } + return false; + + case kTypeDeletion: + case kTypeDeletionWithTimestamp: + case kTypeSingleDeletion: + case kTypeRangeDeletion: + // TODO(noetzli): Verify correctness once merge of single-deletes + // is supported + assert(state_ == kNotFound || state_ == kMerge); + if (kNotFound == state_) { + state_ = kDeleted; + } else if (kMerge == state_) { + state_ = kFound; + if (do_merge_) { + Merge(nullptr); + } + // If do_merge_ = false then the current value shouldn't be part of + // merge_context_->operand_list + } + return false; + + case kTypeMerge: + assert(state_ == kNotFound || state_ == kMerge); + state_ = kMerge; + // value_pinner is not set from plain_table_reader.cc for example. + push_operand(value, value_pinner); + PERF_COUNTER_ADD(internal_merge_point_lookup_count, 1); + + if (do_merge_ && merge_operator_ != nullptr && + merge_operator_->ShouldMerge( + merge_context_->GetOperandsDirectionBackward())) { + state_ = kFound; + Merge(nullptr); + return false; + } + return true; + + default: + assert(false); + break; + } + } + + // state_ could be Corrupt, merge or notfound + return false; +} + +void GetContext::Merge(const Slice* value) { + assert(do_merge_); + assert(!pinnable_val_ || !columns_); + + std::string result; + // `op_failure_scope` (an output parameter) is not provided (set to nullptr) + // since a failure must be propagated regardless of its value. + const Status s = MergeHelper::TimedFullMerge( + merge_operator_, user_key_, value, merge_context_->GetOperands(), &result, + logger_, statistics_, clock_, /* result_operand */ nullptr, + /* update_num_ops_stats */ true, + /* op_failure_scope */ nullptr); + if (!s.ok()) { + if (s.subcode() == Status::SubCode::kMergeOperatorFailed) { + state_ = kMergeOperatorFailed; + } else { + state_ = kCorrupt; + } + return; + } + + if (LIKELY(pinnable_val_ != nullptr)) { + *(pinnable_val_->GetSelf()) = std::move(result); + pinnable_val_->PinSelf(); + return; + } + + assert(columns_); + columns_->SetPlainValue(std::move(result)); +} + +void GetContext::MergeWithEntity(Slice entity) { + assert(do_merge_); + assert(!pinnable_val_ || !columns_); + + if (LIKELY(pinnable_val_ != nullptr)) { + Slice value_of_default; + + { + const Status s = WideColumnSerialization::GetValueOfDefaultColumn( + entity, value_of_default); + if (!s.ok()) { + state_ = kCorrupt; + return; + } + } + + { + // `op_failure_scope` (an output parameter) is not provided (set to + // nullptr) since a failure must be propagated regardless of its value. + const Status s = MergeHelper::TimedFullMerge( + merge_operator_, user_key_, &value_of_default, + merge_context_->GetOperands(), pinnable_val_->GetSelf(), logger_, + statistics_, clock_, /* result_operand */ nullptr, + /* update_num_ops_stats */ true, + /* op_failure_scope */ nullptr); + if (!s.ok()) { + if (s.subcode() == Status::SubCode::kMergeOperatorFailed) { + state_ = kMergeOperatorFailed; + } else { + state_ = kCorrupt; + } + return; + } + } + + pinnable_val_->PinSelf(); + return; + } + + std::string result; + + { + // `op_failure_scope` (an output parameter) is not provided (set to nullptr) + // since a failure must be propagated regardless of its value. + const Status s = MergeHelper::TimedFullMergeWithEntity( + merge_operator_, user_key_, entity, merge_context_->GetOperands(), + &result, logger_, statistics_, clock_, /* update_num_ops_stats */ true, + /* op_failure_scope */ nullptr); + if (!s.ok()) { + if (s.subcode() == Status::SubCode::kMergeOperatorFailed) { + state_ = kMergeOperatorFailed; + } else { + state_ = kCorrupt; + } + return; + } + } + + { + assert(columns_); + const Status s = columns_->SetWideColumnValue(std::move(result)); + if (!s.ok()) { + state_ = kCorrupt; + return; + } + } +} + +bool GetContext::GetBlobValue(const Slice& user_key, const Slice& blob_index, + PinnableSlice* blob_value) { + constexpr FilePrefetchBuffer* prefetch_buffer = nullptr; + constexpr uint64_t* bytes_read = nullptr; + + Status status = blob_fetcher_->FetchBlob( + user_key, blob_index, prefetch_buffer, blob_value, bytes_read); + if (!status.ok()) { + if (status.IsIncomplete()) { + // FIXME: this code is not covered by unit tests + MarkKeyMayExist(); + return false; + } + state_ = kCorrupt; + return false; + } + *is_blob_index_ = false; + return true; +} + +void GetContext::push_operand(const Slice& value, Cleanable* value_pinner) { + // TODO(yanqin) preserve timestamps information in merge_context + if (pinned_iters_mgr() && pinned_iters_mgr()->PinningEnabled() && + value_pinner != nullptr) { + value_pinner->DelegateCleanupsTo(pinned_iters_mgr()); + merge_context_->PushOperand(value, true /*value_pinned*/); + } else { + merge_context_->PushOperand(value, false); + } +} + +void replayGetContextLog(const Slice& replay_log, const Slice& user_key, + GetContext* get_context, Cleanable* value_pinner) { + Slice s = replay_log; + while (s.size()) { + auto type = static_cast(*s.data()); + s.remove_prefix(1); + Slice value; + bool ret = GetLengthPrefixedSlice(&s, &value); + assert(ret); + (void)ret; + + bool dont_care __attribute__((__unused__)); + // Since SequenceNumber is not stored and unknown, we will use + // kMaxSequenceNumber. + get_context->SaveValue( + ParsedInternalKey(user_key, kMaxSequenceNumber, type), value, + &dont_care, value_pinner); + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/get_context.h b/librocksdb-sys/rocksdb/table/get_context.h new file mode 100644 index 0000000..528cd14 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/get_context.h @@ -0,0 +1,245 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once +#include + +#include "db/read_callback.h" +#include "rocksdb/types.h" + +namespace ROCKSDB_NAMESPACE { +class BlobFetcher; +class Comparator; +class Logger; +class MergeContext; +class MergeOperator; +class PinnableWideColumns; +class PinnedIteratorsManager; +class Statistics; +class SystemClock; +struct ParsedInternalKey; + +// Data structure for accumulating statistics during a point lookup. At the +// end of the point lookup, the corresponding ticker stats are updated. This +// avoids the overhead of frequent ticker stats updates +struct GetContextStats { + uint64_t num_cache_hit = 0; + uint64_t num_cache_index_hit = 0; + uint64_t num_cache_data_hit = 0; + uint64_t num_cache_filter_hit = 0; + uint64_t num_cache_compression_dict_hit = 0; + uint64_t num_cache_index_miss = 0; + uint64_t num_cache_filter_miss = 0; + uint64_t num_cache_data_miss = 0; + uint64_t num_cache_compression_dict_miss = 0; + uint64_t num_cache_bytes_read = 0; + uint64_t num_cache_miss = 0; + uint64_t num_cache_add = 0; + uint64_t num_cache_add_redundant = 0; + uint64_t num_cache_bytes_write = 0; + uint64_t num_cache_index_add = 0; + uint64_t num_cache_index_add_redundant = 0; + uint64_t num_cache_index_bytes_insert = 0; + uint64_t num_cache_data_add = 0; + uint64_t num_cache_data_add_redundant = 0; + uint64_t num_cache_data_bytes_insert = 0; + uint64_t num_cache_filter_add = 0; + uint64_t num_cache_filter_add_redundant = 0; + uint64_t num_cache_filter_bytes_insert = 0; + uint64_t num_cache_compression_dict_add = 0; + uint64_t num_cache_compression_dict_add_redundant = 0; + uint64_t num_cache_compression_dict_bytes_insert = 0; + // MultiGet stats. + uint64_t num_filter_read = 0; + uint64_t num_index_read = 0; + uint64_t num_sst_read = 0; +}; + +// A class to hold context about a point lookup, such as pointer to value +// slice, key, merge context etc, as well as the current state of the +// lookup. Any user using GetContext to track the lookup result must call +// SaveValue() whenever the internal key is found. This can happen +// repeatedly in case of merge operands. In case the key may exist with +// high probability, but IO is required to confirm and the user doesn't allow +// it, MarkKeyMayExist() must be called instead of SaveValue(). +class GetContext { + public: + // Current state of the point lookup. All except kNotFound and kMerge are + // terminal states + enum GetState { + kNotFound, + kFound, + kDeleted, + kCorrupt, + kMerge, // saver contains the current merge result (the operands) + kUnexpectedBlobIndex, + kMergeOperatorFailed, + }; + GetContextStats get_context_stats_; + + // Constructor + // @param value Holds the value corresponding to user_key. If its nullptr + // then return all merge operands corresponding to user_key + // via merge_context + // @param value_found If non-nullptr, set to false if key may be present + // but we can't be certain because we cannot do IO + // @param max_covering_tombstone_seq Pointer to highest sequence number of + // range deletion covering the key. When an internal key + // is found with smaller sequence number, the lookup + // terminates + // @param seq If non-nullptr, the sequence number of the found key will be + // saved here + // @param callback Pointer to ReadCallback to perform additional checks + // for visibility of a key + // @param is_blob_index If non-nullptr, will be used to indicate if a found + // key is of type blob index + // @param do_merge True if value associated with user_key has to be returned + // and false if all the merge operands associated with user_key has to be + // returned. Id do_merge=false then all the merge operands are stored in + // merge_context and they are never merged. The value pointer is untouched. + GetContext(const Comparator* ucmp, const MergeOperator* merge_operator, + Logger* logger, Statistics* statistics, GetState init_state, + const Slice& user_key, PinnableSlice* value, + PinnableWideColumns* columns, bool* value_found, + MergeContext* merge_context, bool do_merge, + SequenceNumber* max_covering_tombstone_seq, SystemClock* clock, + SequenceNumber* seq = nullptr, + PinnedIteratorsManager* _pinned_iters_mgr = nullptr, + ReadCallback* callback = nullptr, bool* is_blob_index = nullptr, + uint64_t tracing_get_id = 0, BlobFetcher* blob_fetcher = nullptr); + GetContext(const Comparator* ucmp, const MergeOperator* merge_operator, + Logger* logger, Statistics* statistics, GetState init_state, + const Slice& user_key, PinnableSlice* value, + PinnableWideColumns* columns, std::string* timestamp, + bool* value_found, MergeContext* merge_context, bool do_merge, + SequenceNumber* max_covering_tombstone_seq, SystemClock* clock, + SequenceNumber* seq = nullptr, + PinnedIteratorsManager* _pinned_iters_mgr = nullptr, + ReadCallback* callback = nullptr, bool* is_blob_index = nullptr, + uint64_t tracing_get_id = 0, BlobFetcher* blob_fetcher = nullptr); + + GetContext() = delete; + + // This can be called to indicate that a key may be present, but cannot be + // confirmed due to IO not allowed + void MarkKeyMayExist(); + + // Records this key, value, and any meta-data (such as sequence number and + // state) into this GetContext. + // + // If the parsed_key matches the user key that we are looking for, sets + // matched to true. + // + // Returns True if more keys need to be read (due to merges) or + // False if the complete value has been found. + bool SaveValue(const ParsedInternalKey& parsed_key, const Slice& value, + bool* matched, Cleanable* value_pinner = nullptr); + + // Simplified version of the previous function. Should only be used when we + // know that the operation is a Put. + void SaveValue(const Slice& value, SequenceNumber seq); + + GetState State() const { return state_; } + + SequenceNumber* max_covering_tombstone_seq() { + return max_covering_tombstone_seq_; + } + + bool NeedTimestamp() { return timestamp_ != nullptr; } + + void SetTimestampFromRangeTombstone(const Slice& timestamp) { + assert(timestamp_); + timestamp_->assign(timestamp.data(), timestamp.size()); + ts_from_rangetombstone_ = true; + } + + PinnedIteratorsManager* pinned_iters_mgr() { return pinned_iters_mgr_; } + + // If a non-null string is passed, all the SaveValue calls will be + // logged into the string. The operations can then be replayed on + // another GetContext with replayGetContextLog. + void SetReplayLog(std::string* replay_log) { replay_log_ = replay_log; } + + // Do we need to fetch the SequenceNumber for this key? + bool NeedToReadSequence() const { return (seq_ != nullptr); } + + bool sample() const { return sample_; } + + bool CheckCallback(SequenceNumber seq) { + if (callback_) { + return callback_->IsVisible(seq); + } + return true; + } + + void ReportCounters(); + + bool has_callback() const { return callback_ != nullptr; } + + const Slice& ukey_to_get_blob_value() const { + if (!ukey_with_ts_found_.empty()) { + return ukey_with_ts_found_; + } else { + return user_key_; + } + } + + uint64_t get_tracing_get_id() const { return tracing_get_id_; } + + void push_operand(const Slice& value, Cleanable* value_pinner); + + private: + void Merge(const Slice* value); + void MergeWithEntity(Slice entity); + bool GetBlobValue(const Slice& user_key, const Slice& blob_index, + PinnableSlice* blob_value); + + const Comparator* ucmp_; + const MergeOperator* merge_operator_; + // the merge operations encountered; + Logger* logger_; + Statistics* statistics_; + + GetState state_; + Slice user_key_; + // When a blob index is found with the user key containing timestamp, + // this copies the corresponding user key on record in the sst file + // and is later used for blob verification. + PinnableSlice ukey_with_ts_found_; + PinnableSlice* pinnable_val_; + PinnableWideColumns* columns_; + std::string* timestamp_; + bool ts_from_rangetombstone_{false}; + bool* value_found_; // Is value set correctly? Used by KeyMayExist + MergeContext* merge_context_; + SequenceNumber* max_covering_tombstone_seq_; + SystemClock* clock_; + // If a key is found, seq_ will be set to the SequenceNumber of most recent + // write to the key or kMaxSequenceNumber if unknown + SequenceNumber* seq_; + std::string* replay_log_; + // Used to temporarily pin blocks when state_ == GetContext::kMerge + PinnedIteratorsManager* pinned_iters_mgr_; + ReadCallback* callback_; + bool sample_; + // Value is true if it's called as part of DB Get API and false if it's + // called as part of DB GetMergeOperands API. When it's false merge operators + // are never merged. + bool do_merge_; + bool* is_blob_index_; + // Used for block cache tracing only. A tracing get id uniquely identifies a + // Get or a MultiGet. + const uint64_t tracing_get_id_; + BlobFetcher* blob_fetcher_; +}; + +// Call this to replay a log and bring the get_context up to date. The replay +// log must have been created by another GetContext object, whose replay log +// must have been set by calling GetContext::SetReplayLog(). +void replayGetContextLog(const Slice& replay_log, const Slice& user_key, + GetContext* get_context, + Cleanable* value_pinner = nullptr); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/internal_iterator.h b/librocksdb-sys/rocksdb/table/internal_iterator.h new file mode 100644 index 0000000..8015ed6 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/internal_iterator.h @@ -0,0 +1,224 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#pragma once + +#include + +#include "db/dbformat.h" +#include "file/readahead_file_info.h" +#include "rocksdb/comparator.h" +#include "rocksdb/iterator.h" +#include "rocksdb/status.h" +#include "table/format.h" + +namespace ROCKSDB_NAMESPACE { + +class PinnedIteratorsManager; + +enum class IterBoundCheck : char { + kUnknown = 0, + kOutOfBound, + kInbound, +}; + +struct IterateResult { + Slice key; + IterBoundCheck bound_check_result = IterBoundCheck::kUnknown; + // If false, PrepareValue() needs to be called before value(). + bool value_prepared = true; +}; + +template +class InternalIteratorBase : public Cleanable { + public: + InternalIteratorBase() {} + + // No copying allowed + InternalIteratorBase(const InternalIteratorBase&) = delete; + InternalIteratorBase& operator=(const InternalIteratorBase&) = delete; + + virtual ~InternalIteratorBase() {} + + // An iterator is either positioned at a key/value pair, or + // not valid. This method returns true iff the iterator is valid. + // Always returns false if !status().ok(). + virtual bool Valid() const = 0; + + // Position at the first key in the source. The iterator is Valid() + // after this call iff the source is not empty. + virtual void SeekToFirst() = 0; + + // Position at the last key in the source. The iterator is + // Valid() after this call iff the source is not empty. + virtual void SeekToLast() = 0; + + // Position at the first key in the source that at or past target + // The iterator is Valid() after this call iff the source contains + // an entry that comes at or past target. + // All Seek*() methods clear any error status() that the iterator had prior to + // the call; after the seek, status() indicates only the error (if any) that + // happened during the seek, not any past errors. + // 'target' contains user timestamp if timestamp is enabled. + virtual void Seek(const Slice& target) = 0; + + // Position at the first key in the source that at or before target + // The iterator is Valid() after this call iff the source contains + // an entry that comes at or before target. + virtual void SeekForPrev(const Slice& target) = 0; + + // Moves to the next entry in the source. After this call, Valid() is + // true iff the iterator was not positioned at the last entry in the source. + // REQUIRES: Valid() + virtual void Next() = 0; + + // Moves to the next entry in the source, and return result. Iterator + // implementation should override this method to help methods inline better, + // or when UpperBoundCheckResult() is non-trivial. + // REQUIRES: Valid() + virtual bool NextAndGetResult(IterateResult* result) { + Next(); + bool is_valid = Valid(); + if (is_valid) { + result->key = key(); + // Default may_be_out_of_upper_bound to true to avoid unnecessary virtual + // call. If an implementation has non-trivial UpperBoundCheckResult(), + // it should also override NextAndGetResult(). + result->bound_check_result = IterBoundCheck::kUnknown; + result->value_prepared = false; + assert(UpperBoundCheckResult() != IterBoundCheck::kOutOfBound); + } + return is_valid; + } + + // Moves to the previous entry in the source. After this call, Valid() is + // true iff the iterator was not positioned at the first entry in source. + // REQUIRES: Valid() + virtual void Prev() = 0; + + // Return the key for the current entry. The underlying storage for + // the returned slice is valid only until the next modification of + // the iterator. + // REQUIRES: Valid() + virtual Slice key() const = 0; + + // Return user key for the current entry. + // REQUIRES: Valid() + virtual Slice user_key() const { return ExtractUserKey(key()); } + + // Return the value for the current entry. The underlying storage for + // the returned slice is valid only until the next modification of + // the iterator. + // REQUIRES: Valid() + // REQUIRES: PrepareValue() has been called if needed (see PrepareValue()). + virtual TValue value() const = 0; + + // If an error has occurred, return it. Else return an ok status. + // If non-blocking IO is requested and this operation cannot be + // satisfied without doing some IO, then this returns Status::Incomplete(). + virtual Status status() const = 0; + + // For some types of iterators, sometimes Seek()/Next()/SeekForPrev()/etc may + // load key but not value (to avoid the IO cost of reading the value from disk + // if it won't be not needed). This method loads the value in such situation. + // + // Needs to be called before value() at least once after each iterator + // movement (except if IterateResult::value_prepared = true), for iterators + // created with allow_unprepared_value = true. + // + // Returns false if an error occurred; in this case Valid() is also changed + // to false, and status() is changed to non-ok. + // REQUIRES: Valid() + virtual bool PrepareValue() { return true; } + + // Keys return from this iterator can be smaller than iterate_lower_bound. + virtual bool MayBeOutOfLowerBound() { return true; } + + // If the iterator has checked the key against iterate_upper_bound, returns + // the result here. The function can be used by user of the iterator to skip + // their own checks. If Valid() = true, IterBoundCheck::kUnknown is always + // a valid value. If Valid() = false, IterBoundCheck::kOutOfBound indicates + // that the iterator is filtered out by upper bound checks. + virtual IterBoundCheck UpperBoundCheckResult() { + return IterBoundCheck::kUnknown; + } + + // Pass the PinnedIteratorsManager to the Iterator, most Iterators don't + // communicate with PinnedIteratorsManager so default implementation is no-op + // but for Iterators that need to communicate with PinnedIteratorsManager + // they will implement this function and use the passed pointer to communicate + // with PinnedIteratorsManager. + virtual void SetPinnedItersMgr(PinnedIteratorsManager* /*pinned_iters_mgr*/) { + } + + // If true, this means that the Slice returned by key() is valid as long as + // PinnedIteratorsManager::ReleasePinnedData is not called and the + // Iterator is not deleted. + // + // IsKeyPinned() is guaranteed to always return true if + // - Iterator is created with ReadOptions::pin_data = true + // - DB tables were created with BlockBasedTableOptions::use_delta_encoding + // set to false. + virtual bool IsKeyPinned() const { return false; } + + // If true, this means that the Slice returned by value() is valid as long as + // PinnedIteratorsManager::ReleasePinnedData is not called and the + // Iterator is not deleted. + // REQUIRES: Same as for value(). + virtual bool IsValuePinned() const { return false; } + + virtual Status GetProperty(std::string /*prop_name*/, std::string* /*prop*/) { + return Status::NotSupported(""); + } + + // When iterator moves from one file to another file at same level, new file's + // readahead state (details of last block read) is updated with previous + // file's readahead state. This way internal readahead_size of Prefetch Buffer + // doesn't start from scratch and can fall back to 8KB with no prefetch if + // reads are not sequential. + // + // Default implementation is no-op and its implemented by iterators. + virtual void GetReadaheadState(ReadaheadFileInfo* /*readahead_file_info*/) {} + + // Default implementation is no-op and its implemented by iterators. + virtual void SetReadaheadState(ReadaheadFileInfo* /*readahead_file_info*/) {} + + // When used under merging iterator, LevelIterator treats file boundaries + // as sentinel keys to prevent it from moving to next SST file before range + // tombstones in the current SST file are no longer needed. This method makes + // it cheap to check if the current key is a sentinel key. This should only be + // used by MergingIterator and LevelIterator for now. + virtual bool IsDeleteRangeSentinelKey() const { return false; } + + protected: + void SeekForPrevImpl(const Slice& target, const CompareInterface* cmp) { + Seek(target); + if (!Valid()) { + SeekToLast(); + } + while (Valid() && cmp->Compare(target, key()) < 0) { + Prev(); + } + } +}; + +using InternalIterator = InternalIteratorBase; + +// Return an empty iterator (yields nothing). +template +extern InternalIteratorBase* NewEmptyInternalIterator(); + +// Return an empty iterator with the specified status. +template +extern InternalIteratorBase* NewErrorInternalIterator( + const Status& status); + +// Return an empty iterator with the specified status, allocated arena. +template +extern InternalIteratorBase* NewErrorInternalIterator( + const Status& status, Arena* arena); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/iter_heap.h b/librocksdb-sys/rocksdb/table/iter_heap.h new file mode 100644 index 0000000..6ad94be --- /dev/null +++ b/librocksdb-sys/rocksdb/table/iter_heap.h @@ -0,0 +1,44 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#pragma once + +#include "db/dbformat.h" +#include "table/iterator_wrapper.h" + +namespace ROCKSDB_NAMESPACE { + +// When used with std::priority_queue, this comparison functor puts the +// iterator with the max/largest key on top. +class MaxIteratorComparator { + public: + MaxIteratorComparator(const InternalKeyComparator* comparator) + : comparator_(comparator) {} + + bool operator()(IteratorWrapper* a, IteratorWrapper* b) const { + return comparator_->Compare(a->key(), b->key()) < 0; + } + + private: + const InternalKeyComparator* comparator_; +}; + +// When used with std::priority_queue, this comparison functor puts the +// iterator with the min/smallest key on top. +class MinIteratorComparator { + public: + MinIteratorComparator(const InternalKeyComparator* comparator) + : comparator_(comparator) {} + + bool operator()(IteratorWrapper* a, IteratorWrapper* b) const { + return comparator_->Compare(a->key(), b->key()) > 0; + } + + private: + const InternalKeyComparator* comparator_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/iterator.cc b/librocksdb-sys/rocksdb/table/iterator.cc new file mode 100644 index 0000000..14e280a --- /dev/null +++ b/librocksdb-sys/rocksdb/table/iterator.cc @@ -0,0 +1,130 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "rocksdb/iterator.h" + +#include "memory/arena.h" +#include "table/internal_iterator.h" +#include "table/iterator_wrapper.h" + +namespace ROCKSDB_NAMESPACE { + +Status Iterator::GetProperty(std::string prop_name, std::string* prop) { + if (prop == nullptr) { + return Status::InvalidArgument("prop is nullptr"); + } + if (prop_name == "rocksdb.iterator.is-key-pinned") { + *prop = "0"; + return Status::OK(); + } + return Status::InvalidArgument("Unidentified property."); +} + +namespace { +class EmptyIterator : public Iterator { + public: + explicit EmptyIterator(const Status& s) : status_(s) {} + bool Valid() const override { return false; } + void Seek(const Slice& /*target*/) override {} + void SeekForPrev(const Slice& /*target*/) override {} + void SeekToFirst() override {} + void SeekToLast() override {} + void Next() override { assert(false); } + void Prev() override { assert(false); } + Slice key() const override { + assert(false); + return Slice(); + } + Slice value() const override { + assert(false); + return Slice(); + } + Status status() const override { return status_; } + + private: + Status status_; +}; + +template +class EmptyInternalIterator : public InternalIteratorBase { + public: + explicit EmptyInternalIterator(const Status& s) : status_(s) {} + bool Valid() const override { return false; } + void Seek(const Slice& /*target*/) override {} + void SeekForPrev(const Slice& /*target*/) override {} + void SeekToFirst() override {} + void SeekToLast() override {} + void Next() override { assert(false); } + void Prev() override { assert(false); } + Slice key() const override { + assert(false); + return Slice(); + } + TValue value() const override { + assert(false); + return TValue(); + } + Status status() const override { return status_; } + + private: + Status status_; +}; +} // namespace + +Iterator* NewEmptyIterator() { return new EmptyIterator(Status::OK()); } + +Iterator* NewErrorIterator(const Status& status) { + return new EmptyIterator(status); +} + +template +InternalIteratorBase* NewErrorInternalIterator(const Status& status) { + return new EmptyInternalIterator(status); +} +template InternalIteratorBase* NewErrorInternalIterator( + const Status& status); +template InternalIteratorBase* NewErrorInternalIterator( + const Status& status); + +template +InternalIteratorBase* NewErrorInternalIterator(const Status& status, + Arena* arena) { + if (arena == nullptr) { + return NewErrorInternalIterator(status); + } else { + auto mem = arena->AllocateAligned(sizeof(EmptyInternalIterator)); + return new (mem) EmptyInternalIterator(status); + } +} +template InternalIteratorBase* NewErrorInternalIterator( + const Status& status, Arena* arena); +template InternalIteratorBase* NewErrorInternalIterator( + const Status& status, Arena* arena); + +template +InternalIteratorBase* NewEmptyInternalIterator() { + return new EmptyInternalIterator(Status::OK()); +} +template InternalIteratorBase* NewEmptyInternalIterator(); +template InternalIteratorBase* NewEmptyInternalIterator(); + +template +InternalIteratorBase* NewEmptyInternalIterator(Arena* arena) { + if (arena == nullptr) { + return NewEmptyInternalIterator(); + } else { + auto mem = arena->AllocateAligned(sizeof(EmptyInternalIterator)); + return new (mem) EmptyInternalIterator(Status::OK()); + } +} +template InternalIteratorBase* NewEmptyInternalIterator( + Arena* arena); +template InternalIteratorBase* NewEmptyInternalIterator(Arena* arena); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/iterator_wrapper.h b/librocksdb-sys/rocksdb/table/iterator_wrapper.h new file mode 100644 index 0000000..17abef4 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/iterator_wrapper.h @@ -0,0 +1,190 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include + +#include "table/internal_iterator.h" +#include "test_util/sync_point.h" + +namespace ROCKSDB_NAMESPACE { + +// A internal wrapper class with an interface similar to Iterator that caches +// the valid() and key() results for an underlying iterator. +// This can help avoid virtual function calls and also gives better +// cache locality. +template +class IteratorWrapperBase { + public: + IteratorWrapperBase() : iter_(nullptr), valid_(false) {} + explicit IteratorWrapperBase(InternalIteratorBase* _iter) + : iter_(nullptr) { + Set(_iter); + } + ~IteratorWrapperBase() {} + InternalIteratorBase* iter() const { return iter_; } + + // Set the underlying Iterator to _iter and return + // previous underlying Iterator. + InternalIteratorBase* Set(InternalIteratorBase* _iter) { + InternalIteratorBase* old_iter = iter_; + + iter_ = _iter; + if (iter_ == nullptr) { + valid_ = false; + } else { + Update(); + } + return old_iter; + } + + void DeleteIter(bool is_arena_mode) { + if (iter_) { + if (!is_arena_mode) { + delete iter_; + } else { + iter_->~InternalIteratorBase(); + } + } + } + + // Iterator interface methods + bool Valid() const { return valid_; } + Slice key() const { + assert(Valid()); + return result_.key; + } + TValue value() const { + assert(Valid()); + return iter_->value(); + } + // Methods below require iter() != nullptr + Status status() const { + assert(iter_); + return iter_->status(); + } + bool PrepareValue() { + assert(Valid()); + if (result_.value_prepared) { + return true; + } + if (iter_->PrepareValue()) { + result_.value_prepared = true; + return true; + } + + assert(!iter_->Valid()); + valid_ = false; + return false; + } + void Next() { + assert(iter_); + valid_ = iter_->NextAndGetResult(&result_); + assert(!valid_ || iter_->status().ok()); + } + bool NextAndGetResult(IterateResult* result) { + assert(iter_); + valid_ = iter_->NextAndGetResult(&result_); + *result = result_; + assert(!valid_ || iter_->status().ok()); + return valid_; + } + void Prev() { + assert(iter_); + iter_->Prev(); + Update(); + } + void Seek(const Slice& k) { + assert(iter_); + iter_->Seek(k); + Update(); + } + void SeekForPrev(const Slice& k) { + assert(iter_); + iter_->SeekForPrev(k); + Update(); + } + void SeekToFirst() { + assert(iter_); + iter_->SeekToFirst(); + Update(); + } + void SeekToLast() { + assert(iter_); + iter_->SeekToLast(); + Update(); + } + + bool MayBeOutOfLowerBound() { + assert(Valid()); + return iter_->MayBeOutOfLowerBound(); + } + + IterBoundCheck UpperBoundCheckResult() { + assert(Valid()); + return result_.bound_check_result; + } + + void SetPinnedItersMgr(PinnedIteratorsManager* pinned_iters_mgr) { + assert(iter_); + iter_->SetPinnedItersMgr(pinned_iters_mgr); + } + bool IsKeyPinned() const { + assert(Valid()); + return iter_->IsKeyPinned(); + } + bool IsValuePinned() const { + assert(Valid()); + return iter_->IsValuePinned(); + } + + bool IsValuePrepared() const { return result_.value_prepared; } + + Slice user_key() const { + assert(Valid()); + return iter_->user_key(); + } + + void UpdateReadaheadState(InternalIteratorBase* old_iter) { + if (old_iter && iter_) { + ReadaheadFileInfo readahead_file_info; + old_iter->GetReadaheadState(&readahead_file_info); + iter_->SetReadaheadState(&readahead_file_info); + } + } + + bool IsDeleteRangeSentinelKey() const { + return iter_->IsDeleteRangeSentinelKey(); + } + + private: + void Update() { + valid_ = iter_->Valid(); + if (valid_) { + assert(iter_->status().ok()); + result_.key = iter_->key(); + result_.bound_check_result = IterBoundCheck::kUnknown; + result_.value_prepared = false; + } + } + + InternalIteratorBase* iter_; + IterateResult result_; + bool valid_; +}; + +using IteratorWrapper = IteratorWrapperBase; + +class Arena; +// Return an empty iterator (yields nothing) allocated from arena. +template +extern InternalIteratorBase* NewEmptyInternalIterator(Arena* arena); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/merger_test.cc b/librocksdb-sys/rocksdb/table/merger_test.cc new file mode 100644 index 0000000..71dc798 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/merger_test.cc @@ -0,0 +1,182 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include +#include + +#include "table/merging_iterator.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/random.h" +#include "util/vector_iterator.h" + +namespace ROCKSDB_NAMESPACE { + +class MergerTest : public testing::Test { + public: + MergerTest() + : icomp_(BytewiseComparator()), + rnd_(3), + merging_iterator_(nullptr), + single_iterator_(nullptr) {} + ~MergerTest() override = default; + std::vector GenerateStrings(size_t len, int string_len) { + std::vector ret; + + for (size_t i = 0; i < len; ++i) { + InternalKey ik(rnd_.HumanReadableString(string_len), 0, + ValueType::kTypeValue); + ret.push_back(ik.Encode().ToString(false)); + } + return ret; + } + + void AssertEquivalence() { + auto a = merging_iterator_.get(); + auto b = single_iterator_.get(); + if (!a->Valid()) { + ASSERT_TRUE(!b->Valid()); + } else { + ASSERT_TRUE(b->Valid()); + ASSERT_EQ(b->key().ToString(), a->key().ToString()); + ASSERT_EQ(b->value().ToString(), a->value().ToString()); + } + } + + void SeekToRandom() { + InternalKey ik(rnd_.HumanReadableString(5), 0, ValueType::kTypeValue); + Seek(ik.Encode().ToString(false)); + } + + void Seek(std::string target) { + merging_iterator_->Seek(target); + single_iterator_->Seek(target); + } + + void SeekToFirst() { + merging_iterator_->SeekToFirst(); + single_iterator_->SeekToFirst(); + } + + void SeekToLast() { + merging_iterator_->SeekToLast(); + single_iterator_->SeekToLast(); + } + + void Next(int times) { + for (int i = 0; i < times && merging_iterator_->Valid(); ++i) { + AssertEquivalence(); + merging_iterator_->Next(); + single_iterator_->Next(); + } + AssertEquivalence(); + } + + void Prev(int times) { + for (int i = 0; i < times && merging_iterator_->Valid(); ++i) { + AssertEquivalence(); + merging_iterator_->Prev(); + single_iterator_->Prev(); + } + AssertEquivalence(); + } + + void NextAndPrev(int times) { + for (int i = 0; i < times && merging_iterator_->Valid(); ++i) { + AssertEquivalence(); + if (rnd_.OneIn(2)) { + merging_iterator_->Prev(); + single_iterator_->Prev(); + } else { + merging_iterator_->Next(); + single_iterator_->Next(); + } + } + AssertEquivalence(); + } + + void Generate(size_t num_iterators, size_t strings_per_iterator, + int letters_per_string) { + std::vector small_iterators; + for (size_t i = 0; i < num_iterators; ++i) { + auto strings = GenerateStrings(strings_per_iterator, letters_per_string); + small_iterators.push_back(new VectorIterator(strings, strings, &icomp_)); + all_keys_.insert(all_keys_.end(), strings.begin(), strings.end()); + } + + merging_iterator_.reset( + NewMergingIterator(&icomp_, &small_iterators[0], + static_cast(small_iterators.size()))); + single_iterator_.reset(new VectorIterator(all_keys_, all_keys_, &icomp_)); + } + + InternalKeyComparator icomp_; + Random rnd_; + std::unique_ptr merging_iterator_; + std::unique_ptr single_iterator_; + std::vector all_keys_; +}; + +TEST_F(MergerTest, SeekToRandomNextTest) { + Generate(1000, 50, 50); + for (int i = 0; i < 10; ++i) { + SeekToRandom(); + AssertEquivalence(); + Next(50000); + } +} + +TEST_F(MergerTest, SeekToRandomNextSmallStringsTest) { + Generate(1000, 50, 2); + for (int i = 0; i < 10; ++i) { + SeekToRandom(); + AssertEquivalence(); + Next(50000); + } +} + +TEST_F(MergerTest, SeekToRandomPrevTest) { + Generate(1000, 50, 50); + for (int i = 0; i < 10; ++i) { + SeekToRandom(); + AssertEquivalence(); + Prev(50000); + } +} + +TEST_F(MergerTest, SeekToRandomRandomTest) { + Generate(200, 50, 50); + for (int i = 0; i < 3; ++i) { + SeekToRandom(); + AssertEquivalence(); + NextAndPrev(5000); + } +} + +TEST_F(MergerTest, SeekToFirstTest) { + Generate(1000, 50, 50); + for (int i = 0; i < 10; ++i) { + SeekToFirst(); + AssertEquivalence(); + Next(50000); + } +} + +TEST_F(MergerTest, SeekToLastTest) { + Generate(1000, 50, 50); + for (int i = 0; i < 10; ++i) { + SeekToLast(); + AssertEquivalence(); + Prev(50000); + } +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/table/merging_iterator.cc b/librocksdb-sys/rocksdb/table/merging_iterator.cc new file mode 100644 index 0000000..0fa3fcd --- /dev/null +++ b/librocksdb-sys/rocksdb/table/merging_iterator.cc @@ -0,0 +1,1725 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "table/merging_iterator.h" + +#include "db/arena_wrapped_db_iter.h" + +namespace ROCKSDB_NAMESPACE { +// MergingIterator uses a min/max heap to combine data from point iterators. +// Range tombstones can be added and keys covered by range tombstones will be +// skipped. +// +// The following are implementation details and can be ignored by user. +// For merging iterator to process range tombstones, it treats the start and end +// keys of a range tombstone as two keys and put them into minHeap_ or maxHeap_ +// together with regular point keys. Each range tombstone is active only within +// its internal key range [start_key, end_key). An `active_` set is used to +// track levels that have an active range tombstone. Take forward scanning +// for example. Level j is in active_ if its current range tombstone has its +// start_key popped from minHeap_ and its end_key in minHeap_. If the top of +// minHeap_ is a point key from level L, we can determine if the point key is +// covered by any range tombstone by checking if there is an l <= L in active_. +// The case of l == L also involves checking range tombstone's sequence number. +// +// The following (non-exhaustive) list of invariants are maintained by +// MergingIterator during forward scanning. After each InternalIterator API, +// i.e., Seek*() and Next(), and FindNextVisibleKey(), if minHeap_ is not empty: +// (1) minHeap_.top().type == ITERATOR +// (2) minHeap_.top()->key() is not covered by any range tombstone. +// +// After each call to SeekImpl() in addition to the functions mentioned above: +// (3) For all level i and j <= i, range_tombstone_iters_[j].prev.end_key() < +// children_[i].iter.key(). That is, range_tombstone_iters_[j] is at or before +// the first range tombstone from level j with end_key() > +// children_[i].iter.key(). +// (4) For all level i and j <= i, if j in active_, then +// range_tombstone_iters_[j]->start_key() < children_[i].iter.key(). +// - When range_tombstone_iters_[j] is !Valid(), we consider its `prev` to be +// the last range tombstone from that range tombstone iterator. +// - When referring to range tombstone start/end keys, assume it is the value of +// HeapItem::tombstone_pik. This value has op_type = kMaxValid, which makes +// range tombstone keys have distinct values from point keys. +// +// Applicable class variables have their own (forward scanning) invariants +// listed in the comments above their definition. +class MergingIterator : public InternalIterator { + public: + MergingIterator(const InternalKeyComparator* comparator, + InternalIterator** children, int n, bool is_arena_mode, + bool prefix_seek_mode, + const Slice* iterate_upper_bound = nullptr) + : is_arena_mode_(is_arena_mode), + prefix_seek_mode_(prefix_seek_mode), + direction_(kForward), + comparator_(comparator), + current_(nullptr), + minHeap_(MinHeapItemComparator(comparator_)), + pinned_iters_mgr_(nullptr), + iterate_upper_bound_(iterate_upper_bound) { + children_.resize(n); + for (int i = 0; i < n; i++) { + children_[i].level = i; + children_[i].iter.Set(children[i]); + } + } + + void considerStatus(Status s) { + if (!s.ok() && status_.ok()) { + status_ = s; + } + } + + virtual void AddIterator(InternalIterator* iter) { + children_.emplace_back(children_.size(), iter); + if (pinned_iters_mgr_) { + iter->SetPinnedItersMgr(pinned_iters_mgr_); + } + // Invalidate to ensure `Seek*()` is called to construct the heaps before + // use. + current_ = nullptr; + } + + // There must be either no range tombstone iterator or the same number of + // range tombstone iterators as point iterators after all iters are added. + // The i-th added range tombstone iterator and the i-th point iterator + // must point to the same LSM level. + // Merging iterator takes ownership of `iter` and is responsible for freeing + // it. One exception to this is when a LevelIterator moves to a different SST + // file or when Iterator::Refresh() is called, the range tombstone iterator + // could be updated. In that case, this merging iterator is only responsible + // for freeing the new range tombstone iterator that it has pointers to in + // range_tombstone_iters_. + void AddRangeTombstoneIterator(TruncatedRangeDelIterator* iter) { + range_tombstone_iters_.emplace_back(iter); + } + + // Called by MergingIteratorBuilder when all point iterators and range + // tombstone iterators are added. Initializes HeapItems for range tombstone + // iterators. + void Finish() { + if (!range_tombstone_iters_.empty()) { + assert(range_tombstone_iters_.size() == children_.size()); + pinned_heap_item_.resize(range_tombstone_iters_.size()); + for (size_t i = 0; i < range_tombstone_iters_.size(); ++i) { + pinned_heap_item_[i].level = i; + // Range tombstone end key is exclusive. If a point internal key has the + // same user key and sequence number as the start or end key of a range + // tombstone, the order will be start < end key < internal key with the + // following op_type change. This is helpful to ensure keys popped from + // heap are in expected order since range tombstone start/end keys will + // be distinct from point internal keys. Strictly speaking, this is only + // needed for tombstone end points that are truncated in + // TruncatedRangeDelIterator since untruncated tombstone end points + // always have kMaxSequenceNumber and kTypeRangeDeletion (see + // TruncatedRangeDelIterator::start_key()/end_key()). + pinned_heap_item_[i].tombstone_pik.type = kTypeMaxValid; + } + } + } + + ~MergingIterator() override { + for (auto child : range_tombstone_iters_) { + delete child; + } + + for (auto& child : children_) { + child.iter.DeleteIter(is_arena_mode_); + } + status_.PermitUncheckedError(); + } + + bool Valid() const override { return current_ != nullptr && status_.ok(); } + + Status status() const override { return status_; } + + // Add range_tombstone_iters_[level] into min heap. + // Updates active_ if the end key of a range tombstone is inserted. + // pinned_heap_items_[level].type is updated based on `start_key`. + // + // If range_tombstone_iters_[level] is after iterate_upper_bound_, + // it is removed from the heap. + // @param start_key specifies which end point of the range tombstone to add. + void InsertRangeTombstoneToMinHeap(size_t level, bool start_key = true, + bool replace_top = false) { + assert(!range_tombstone_iters_.empty() && + range_tombstone_iters_[level]->Valid()); + // Maintains Invariant(phi) + if (start_key) { + pinned_heap_item_[level].type = HeapItem::Type::DELETE_RANGE_START; + ParsedInternalKey pik = range_tombstone_iters_[level]->start_key(); + // iterate_upper_bound does not have timestamp + if (iterate_upper_bound_ && + comparator_->user_comparator()->CompareWithoutTimestamp( + pik.user_key, true /* a_has_ts */, *iterate_upper_bound_, + false /* b_has_ts */) >= 0) { + if (replace_top) { + // replace_top implies this range tombstone iterator is still in + // minHeap_ and at the top. + minHeap_.pop(); + } + return; + } + pinned_heap_item_[level].SetTombstoneKey(std::move(pik)); + // Checks Invariant(active_) + assert(active_.count(level) == 0); + } else { + // allow end key to go over upper bound (if present) since start key is + // before upper bound and the range tombstone could still cover a + // range before upper bound. + // Maintains Invariant(active_) + pinned_heap_item_[level].SetTombstoneKey( + range_tombstone_iters_[level]->end_key()); + pinned_heap_item_[level].type = HeapItem::Type::DELETE_RANGE_END; + active_.insert(level); + } + if (replace_top) { + minHeap_.replace_top(&pinned_heap_item_[level]); + } else { + minHeap_.push(&pinned_heap_item_[level]); + } + } + + // Add range_tombstone_iters_[level] into max heap. + // Updates active_ if the start key of a range tombstone is inserted. + // @param end_key specifies which end point of the range tombstone to add. + void InsertRangeTombstoneToMaxHeap(size_t level, bool end_key = true, + bool replace_top = false) { + assert(!range_tombstone_iters_.empty() && + range_tombstone_iters_[level]->Valid()); + if (end_key) { + pinned_heap_item_[level].SetTombstoneKey( + range_tombstone_iters_[level]->end_key()); + pinned_heap_item_[level].type = HeapItem::Type::DELETE_RANGE_END; + assert(active_.count(level) == 0); + } else { + pinned_heap_item_[level].SetTombstoneKey( + range_tombstone_iters_[level]->start_key()); + pinned_heap_item_[level].type = HeapItem::Type::DELETE_RANGE_START; + active_.insert(level); + } + if (replace_top) { + maxHeap_->replace_top(&pinned_heap_item_[level]); + } else { + maxHeap_->push(&pinned_heap_item_[level]); + } + } + + // Remove HeapItems from top of minHeap_ that are of type DELETE_RANGE_START + // until minHeap_ is empty or the top of the minHeap_ is not of type + // DELETE_RANGE_START. Each such item means a range tombstone becomes active, + // so `active_` is updated accordingly. + void PopDeleteRangeStart() { + while (!minHeap_.empty() && + minHeap_.top()->type == HeapItem::Type::DELETE_RANGE_START) { + TEST_SYNC_POINT_CALLBACK("MergeIterator::PopDeleteRangeStart", nullptr); + // Invariant(rti) holds since + // range_tombstone_iters_[minHeap_.top()->level] is still valid, and + // parameter `replace_top` is set to true here to ensure only one such + // HeapItem is in minHeap_. + InsertRangeTombstoneToMinHeap( + minHeap_.top()->level, false /* start_key */, true /* replace_top */); + } + } + + // Remove HeapItems from top of maxHeap_ that are of type DELETE_RANGE_END + // until maxHeap_ is empty or the top of the maxHeap_ is not of type + // DELETE_RANGE_END. Each such item means a range tombstone becomes active, + // so `active_` is updated accordingly. + void PopDeleteRangeEnd() { + while (!maxHeap_->empty() && + maxHeap_->top()->type == HeapItem::Type::DELETE_RANGE_END) { + // insert start key of this range tombstone and updates active_ + InsertRangeTombstoneToMaxHeap(maxHeap_->top()->level, false /* end_key */, + true /* replace_top */); + } + } + + void SeekToFirst() override { + ClearHeaps(); + status_ = Status::OK(); + for (auto& child : children_) { + child.iter.SeekToFirst(); + AddToMinHeapOrCheckStatus(&child); + } + + for (size_t i = 0; i < range_tombstone_iters_.size(); ++i) { + if (range_tombstone_iters_[i]) { + range_tombstone_iters_[i]->SeekToFirst(); + if (range_tombstone_iters_[i]->Valid()) { + // It is possible to be invalid due to snapshots. + InsertRangeTombstoneToMinHeap(i); + } + } + } + FindNextVisibleKey(); + direction_ = kForward; + current_ = CurrentForward(); + } + + void SeekToLast() override { + ClearHeaps(); + InitMaxHeap(); + status_ = Status::OK(); + for (auto& child : children_) { + child.iter.SeekToLast(); + AddToMaxHeapOrCheckStatus(&child); + } + + for (size_t i = 0; i < range_tombstone_iters_.size(); ++i) { + if (range_tombstone_iters_[i]) { + range_tombstone_iters_[i]->SeekToLast(); + if (range_tombstone_iters_[i]->Valid()) { + // It is possible to be invalid due to snapshots. + InsertRangeTombstoneToMaxHeap(i); + } + } + } + FindPrevVisibleKey(); + direction_ = kReverse; + current_ = CurrentReverse(); + } + + // Position this merging iterator at the first key >= target (internal key). + // If range tombstones are present, keys covered by range tombstones are + // skipped, and this merging iter points to the first non-range-deleted key >= + // target after Seek(). If !Valid() and status().ok() then this iterator + // reaches the end. + // + // If range tombstones are present, cascading seeks may be called (an + // optimization adapted from Pebble https://github.com/cockroachdb/pebble). + // Roughly, if there is a range tombstone [start, end) that covers the + // target user key at level L, then this range tombstone must cover the range + // [target key, end) in all levels > L. So for all levels > L, we can pretend + // the target key is `end`. This optimization is applied at each level and + // hence the name "cascading seek". + void Seek(const Slice& target) override { + // Define LevelNextVisible(i, k) to be the first key >= k in level i that is + // not covered by any range tombstone. + // After SeekImpl(target, 0), invariants (3) and (4) hold. + // For all level i, target <= children_[i].iter.key() <= LevelNextVisible(i, + // target). By the contract of FindNextVisibleKey(), Invariants (1)-(4) + // holds after this call, and minHeap_.top().iter points to the + // first key >= target among children_ that is not covered by any range + // tombstone. + SeekImpl(target); + FindNextVisibleKey(); + + direction_ = kForward; + { + PERF_TIMER_GUARD(seek_min_heap_time); + current_ = CurrentForward(); + } + } + + void SeekForPrev(const Slice& target) override { + assert(range_tombstone_iters_.empty() || + range_tombstone_iters_.size() == children_.size()); + SeekForPrevImpl(target); + FindPrevVisibleKey(); + + direction_ = kReverse; + { + PERF_TIMER_GUARD(seek_max_heap_time); + current_ = CurrentReverse(); + } + } + + void Next() override { + assert(Valid()); + // Ensure that all children are positioned after key(). + // If we are moving in the forward direction, it is already + // true for all the non-current children since current_ is + // the smallest child and key() == current_->key(). + if (direction_ != kForward) { + // The loop advanced all non-current children to be > key() so current_ + // should still be strictly the smallest key. + SwitchToForward(); + } + + // For the heap modifications below to be correct, current_ must be the + // current top of the heap. + assert(current_ == CurrentForward()); + // as the current points to the current record. move the iterator forward. + current_->Next(); + if (current_->Valid()) { + // current is still valid after the Next() call above. Call + // replace_top() to restore the heap property. When the same child + // iterator yields a sequence of keys, this is cheap. + assert(current_->status().ok()); + minHeap_.replace_top(minHeap_.top()); + } else { + // current stopped being valid, remove it from the heap. + considerStatus(current_->status()); + minHeap_.pop(); + } + // Invariants (3) and (4) hold when after advancing current_. + // Let k be the smallest key among children_[i].iter.key(). + // k <= children_[i].iter.key() <= LevelNextVisible(i, k) holds for all + // level i. After FindNextVisible(), Invariants (1)-(4) hold and + // minHeap_.top()->key() is the first key >= k from any children_ that is + // not covered by any range tombstone. + FindNextVisibleKey(); + current_ = CurrentForward(); + } + + bool NextAndGetResult(IterateResult* result) override { + Next(); + bool is_valid = Valid(); + if (is_valid) { + result->key = key(); + result->bound_check_result = UpperBoundCheckResult(); + result->value_prepared = current_->IsValuePrepared(); + } + return is_valid; + } + + void Prev() override { + assert(Valid()); + // Ensure that all children are positioned before key(). + // If we are moving in the reverse direction, it is already + // true for all the non-current children since current_ is + // the largest child and key() == current_->key(). + if (direction_ != kReverse) { + // Otherwise, retreat the non-current children. We retreat current_ + // just after the if-block. + SwitchToBackward(); + } + + // For the heap modifications below to be correct, current_ must be the + // current top of the heap. + assert(current_ == CurrentReverse()); + current_->Prev(); + if (current_->Valid()) { + // current is still valid after the Prev() call above. Call + // replace_top() to restore the heap property. When the same child + // iterator yields a sequence of keys, this is cheap. + assert(current_->status().ok()); + maxHeap_->replace_top(maxHeap_->top()); + } else { + // current stopped being valid, remove it from the heap. + considerStatus(current_->status()); + maxHeap_->pop(); + } + FindPrevVisibleKey(); + current_ = CurrentReverse(); + } + + Slice key() const override { + assert(Valid()); + return current_->key(); + } + + Slice value() const override { + assert(Valid()); + return current_->value(); + } + + bool PrepareValue() override { + assert(Valid()); + if (current_->PrepareValue()) { + return true; + } + + considerStatus(current_->status()); + assert(!status_.ok()); + return false; + } + + // Here we simply relay MayBeOutOfLowerBound/MayBeOutOfUpperBound result + // from current child iterator. Potentially as long as one of child iterator + // report out of bound is not possible, we know current key is within bound. + bool MayBeOutOfLowerBound() override { + assert(Valid()); + return current_->MayBeOutOfLowerBound(); + } + + IterBoundCheck UpperBoundCheckResult() override { + assert(Valid()); + return current_->UpperBoundCheckResult(); + } + + void SetPinnedItersMgr(PinnedIteratorsManager* pinned_iters_mgr) override { + pinned_iters_mgr_ = pinned_iters_mgr; + for (auto& child : children_) { + child.iter.SetPinnedItersMgr(pinned_iters_mgr); + } + } + + bool IsKeyPinned() const override { + assert(Valid()); + return pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled() && + current_->IsKeyPinned(); + } + + bool IsValuePinned() const override { + assert(Valid()); + return pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled() && + current_->IsValuePinned(); + } + + private: + // Represents an element in the min/max heap. Each HeapItem corresponds to a + // point iterator or a range tombstone iterator, differentiated by + // HeapItem::type. + struct HeapItem { + HeapItem() = default; + + // corresponding point iterator + IteratorWrapper iter; + size_t level = 0; + // corresponding range tombstone iterator's start or end key value + // depending on value of `type`. + ParsedInternalKey tombstone_pik; + // Will be overwritten before use, initialize here so compiler does not + // complain. + enum class Type { ITERATOR, DELETE_RANGE_START, DELETE_RANGE_END }; + Type type = Type::ITERATOR; + + explicit HeapItem(size_t _level, InternalIteratorBase* _iter) + : level(_level), type(Type::ITERATOR) { + iter.Set(_iter); + } + + void SetTombstoneKey(ParsedInternalKey&& pik) { + // op_type is already initialized in MergingIterator::Finish(). + tombstone_pik.user_key = pik.user_key; + tombstone_pik.sequence = pik.sequence; + } + }; + + class MinHeapItemComparator { + public: + explicit MinHeapItemComparator(const InternalKeyComparator* comparator) + : comparator_(comparator) {} + + bool operator()(HeapItem* a, HeapItem* b) const { + if (LIKELY(a->type == HeapItem::Type::ITERATOR)) { + if (LIKELY(b->type == HeapItem::Type::ITERATOR)) { + return comparator_->Compare(a->iter.key(), b->iter.key()) > 0; + } else { + return comparator_->Compare(a->iter.key(), b->tombstone_pik) > 0; + } + } else { + if (LIKELY(b->type == HeapItem::Type::ITERATOR)) { + return comparator_->Compare(a->tombstone_pik, b->iter.key()) > 0; + } else { + return comparator_->Compare(a->tombstone_pik, b->tombstone_pik) > 0; + } + } + } + + private: + const InternalKeyComparator* comparator_; + }; + + class MaxHeapItemComparator { + public: + explicit MaxHeapItemComparator(const InternalKeyComparator* comparator) + : comparator_(comparator) {} + + bool operator()(HeapItem* a, HeapItem* b) const { + if (LIKELY(a->type == HeapItem::Type::ITERATOR)) { + if (LIKELY(b->type == HeapItem::Type::ITERATOR)) { + return comparator_->Compare(a->iter.key(), b->iter.key()) < 0; + } else { + return comparator_->Compare(a->iter.key(), b->tombstone_pik) < 0; + } + } else { + if (LIKELY(b->type == HeapItem::Type::ITERATOR)) { + return comparator_->Compare(a->tombstone_pik, b->iter.key()) < 0; + } else { + return comparator_->Compare(a->tombstone_pik, b->tombstone_pik) < 0; + } + } + } + + private: + const InternalKeyComparator* comparator_; + }; + + using MergerMinIterHeap = BinaryHeap; + using MergerMaxIterHeap = BinaryHeap; + + friend class MergeIteratorBuilder; + // Clears heaps for both directions, used when changing direction or seeking + void ClearHeaps(bool clear_active = true); + // Ensures that maxHeap_ is initialized when starting to go in the reverse + // direction + void InitMaxHeap(); + // Advance this merging iterator until the current key (minHeap_.top()) is + // from a point iterator and is not covered by any range tombstone, + // or that there is no more keys (heap is empty). SeekImpl() may be called + // to seek to the end of a range tombstone as an optimization. + void FindNextVisibleKey(); + void FindPrevVisibleKey(); + + // Advance this merging iterators to the first key >= `target` for all + // components from levels >= starting_level. All iterators before + // starting_level are untouched. + // + // @param range_tombstone_reseek Whether target is some range tombstone + // end, i.e., whether this SeekImpl() call is a part of a "cascading seek". + // This is used only for recoding relevant perf_context. + void SeekImpl(const Slice& target, size_t starting_level = 0, + bool range_tombstone_reseek = false); + + // Seek to fist key <= target key (internal key) for + // children_[starting_level:]. + void SeekForPrevImpl(const Slice& target, size_t starting_level = 0, + bool range_tombstone_reseek = false); + + bool is_arena_mode_; + bool prefix_seek_mode_; + // Which direction is the iterator moving? + enum Direction : uint8_t { kForward, kReverse }; + Direction direction_; + const InternalKeyComparator* comparator_; + // HeapItem for all child point iterators. + // Invariant(children_): children_[i] is in minHeap_ iff + // children_[i].iter.Valid(), and at most one children_[i] is in minHeap_. + // TODO: We could use an autovector with a larger reserved size. + std::vector children_; + // HeapItem for range tombstone start and end keys. + // pinned_heap_item_[i] corresponds to range_tombstone_iters_[i]. + // Invariant(phi): If range_tombstone_iters_[i]->Valid(), + // pinned_heap_item_[i].tombstone_pik is equal to + // range_tombstone_iters_[i]->start_key() when + // pinned_heap_item_[i].type is DELETE_RANGE_START and + // range_tombstone_iters_[i]->end_key() when + // pinned_heap_item_[i].type is DELETE_RANGE_END (ignoring op_type which is + // kMaxValid for all pinned_heap_item_.tombstone_pik). + // pinned_heap_item_[i].type is either DELETE_RANGE_START or DELETE_RANGE_END. + std::vector pinned_heap_item_; + // range_tombstone_iters_[i] contains range tombstones in the sorted run that + // corresponds to children_[i]. range_tombstone_iters_.empty() means not + // handling range tombstones in merging iterator. range_tombstone_iters_[i] == + // nullptr means the sorted run of children_[i] does not have range + // tombstones. + // Invariant(rti): pinned_heap_item_[i] is in minHeap_ iff + // range_tombstone_iters_[i]->Valid() and at most one pinned_heap_item_[i] is + // in minHeap_. + std::vector range_tombstone_iters_; + + // Levels (indices into range_tombstone_iters_/children_ ) that currently have + // "active" range tombstones. See comments above MergingIterator for meaning + // of "active". + // Invariant(active_): i is in active_ iff range_tombstone_iters_[i]->Valid() + // and pinned_heap_item_[i].type == DELETE_RANGE_END. + std::set active_; + + bool SkipNextDeleted(); + + bool SkipPrevDeleted(); + + // Invariant: at the end of each InternalIterator API, + // current_ points to minHeap_.top().iter (maxHeap_ if backward scanning) + // or nullptr if no child iterator is valid. + // This follows from that current_ = CurrentForward()/CurrentReverse() is + // called at the end of each InternalIterator API. + IteratorWrapper* current_; + // If any of the children have non-ok status, this is one of them. + Status status_; + // Invariant: min heap property is maintained (parent is always <= child). + // This holds by using only BinaryHeap APIs to modify heap. One + // exception is to modify heap top item directly (by caller iter->Next()), and + // it should be followed by a call to replace_top() or pop(). + MergerMinIterHeap minHeap_; + + // Max heap is used for reverse iteration, which is way less common than + // forward. Lazily initialize it to save memory. + std::unique_ptr maxHeap_; + PinnedIteratorsManager* pinned_iters_mgr_; + + // Used to bound range tombstones. For point keys, DBIter and SSTable iterator + // take care of boundary checking. + const Slice* iterate_upper_bound_; + + // In forward direction, process a child that is not in the min heap. + // If valid, add to the min heap. Otherwise, check status. + void AddToMinHeapOrCheckStatus(HeapItem*); + + // In backward direction, process a child that is not in the max heap. + // If valid, add to the min heap. Otherwise, check status. + void AddToMaxHeapOrCheckStatus(HeapItem*); + + void SwitchToForward(); + + // Switch the direction from forward to backward without changing the + // position. Iterator should still be valid. + void SwitchToBackward(); + + IteratorWrapper* CurrentForward() const { + assert(direction_ == kForward); + assert(minHeap_.empty() || + minHeap_.top()->type == HeapItem::Type::ITERATOR); + return !minHeap_.empty() ? &minHeap_.top()->iter : nullptr; + } + + IteratorWrapper* CurrentReverse() const { + assert(direction_ == kReverse); + assert(maxHeap_); + assert(maxHeap_->empty() || + maxHeap_->top()->type == HeapItem::Type::ITERATOR); + return !maxHeap_->empty() ? &maxHeap_->top()->iter : nullptr; + } +}; + +// Pre-condition: +// - Invariants (3) and (4) hold for i < starting_level +// - For i < starting_level, range_tombstone_iters_[i].prev.end_key() < +// `target`. +// - For i < starting_level, if i in active_, then +// range_tombstone_iters_[i]->start_key() < `target`. +// +// Post-condition: +// - Invariants (3) and (4) hold for all level i. +// - (*) target <= children_[i].iter.key() <= LevelNextVisible(i, target) +// for i >= starting_level +// - (**) target < pinned_heap_item_[i].tombstone_pik if +// range_tombstone_iters_[i].Valid() for i >= starting_level +// +// Proof sketch: +// Invariant (3) holds for all level i. +// For j <= i < starting_level, it follows from Pre-condition that (3) holds +// and that SeekImpl(-, starting_level) does not update children_[i] or +// range_tombstone_iters_[j]. +// For j < starting_level and i >= starting_level, it follows from +// - Pre-condition that range_tombstone_iters_[j].prev.end_key() < `target` +// - range_tombstone_iters_[j] is not updated in SeekImpl(), and +// - children_[i].iter.Seek(current_search_key) is called with +// current_search_key >= target (shown below). +// When current_search_key is updated, it is updated to some +// range_tombstone_iter->end_key() after +// range_tombstone_iter->SeekInternalKey(current_search_key) was called. So +// current_search_key increases if updated and >= target. +// For starting_level <= j <= i: +// children_[i].iter.Seek(k1) and range_tombstone_iters_[j]->SeekInternalKey(k2) +// are called in SeekImpl(). Seek(k1) positions children_[i] at the first key >= +// k1 from level i. SeekInternalKey(k2) positions range_tombstone_iters_[j] at +// the first range tombstone from level j with end_key() > k2. It suffices to +// show that k1 >= k2. Since k1 and k2 are values of current_search_key where +// k1 = k2 or k1 is value of a later current_search_key than k2, so k1 >= k2. +// +// Invariant (4) holds for all level >= 0. +// By Pre-condition Invariant (4) holds for i < starting_level. +// Since children_[i], range_tombstone_iters_[i] and contents of active_ for +// i < starting_level do not change (4) holds for j <= i < starting_level. +// By Pre-condition: for all j < starting_level, if j in active_, then +// range_tombstone_iters_[j]->start_key() < target. For i >= starting_level, +// children_[i].iter.Seek(k) is called for k >= target. So +// children_[i].iter.key() >= target > range_tombstone_iters_[j]->start_key() +// for j < starting_level and i >= starting_level. So invariant (4) holds for +// j < starting_level and i >= starting_level. +// For starting_level <= j <= i, j is added to active_ only if +// - range_tombstone_iters_[j]->SeekInternalKey(k1) was called +// - range_tombstone_iters_[j]->start_key() <= k1 +// Since children_[i].iter.Seek(k2) is called for some k2 >= k1 and for all +// starting_level <= j <= i, (4) also holds for all starting_level <= j <= i. +// +// Post-condition (*): target <= children_[i].iter.key() <= LevelNextVisible(i, +// target) for i >= starting_level. +// target <= children_[i].iter.key() follows from that Seek() is called on some +// current_search_key >= target for children_[i].iter. If current_search_key +// is updated from k1 to k2 when level = i, we show that the range [k1, k2) is +// not visible for children_[j] for any j > i. When current_search_key is +// updated from k1 to k2, +// - range_tombstone_iters_[i]->SeekInternalKey(k1) was called +// - range_tombstone_iters_[i]->Valid() +// - range_tombstone_iters_[i]->start_key().user_key <= k1.user_key +// - k2 = range_tombstone_iters_[i]->end_key() +// We assume that range_tombstone_iters_[i]->start_key() has a higher sequence +// number compared to any key from levels > i that has the same user key. So no +// point key from levels > i in range [k1, k2) is visible. So +// children_[i].iter.key() <= LevelNextVisible(i, target). +// +// Post-condition (**) target < pinned_heap_item_[i].tombstone_pik for i >= +// starting_level if range_tombstone_iters_[i].Valid(). This follows from that +// SeekInternalKey() being called for each range_tombstone_iters_ with some key +// >= `target` and that we pick start/end key that is > `target` to insert to +// minHeap_. +void MergingIterator::SeekImpl(const Slice& target, size_t starting_level, + bool range_tombstone_reseek) { + // active range tombstones before `starting_level` remain active + ClearHeaps(false /* clear_active */); + ParsedInternalKey pik; + if (!range_tombstone_iters_.empty()) { + // pik is only used in InsertRangeTombstoneToMinHeap(). + ParseInternalKey(target, &pik, false).PermitUncheckedError(); + } + + // TODO: perhaps we could save some upheap cost by add all child iters first + // and then do a single heapify. + // Invariant(children_) for level < starting_level + for (size_t level = 0; level < starting_level; ++level) { + PERF_TIMER_GUARD(seek_min_heap_time); + AddToMinHeapOrCheckStatus(&children_[level]); + } + if (!range_tombstone_iters_.empty()) { + // Add range tombstones from levels < starting_level. We can insert from + // pinned_heap_item_ for the following reasons: + // - pinned_heap_item_[level] is in minHeap_ iff + // range_tombstone_iters[level]->Valid(). + // - If `level` is in active_, then range_tombstone_iters_[level]->Valid() + // and pinned_heap_item_[level] is of type RANGE_DELETION_END. + for (size_t level = 0; level < starting_level; ++level) { + // Restores Invariants(rti), (phi) and (active_) for level < + // starting_level + if (range_tombstone_iters_[level] && + range_tombstone_iters_[level]->Valid()) { + // use an iterator on active_ if performance becomes an issue here + if (active_.count(level) > 0) { + assert(pinned_heap_item_[level].type == + HeapItem::Type::DELETE_RANGE_END); + // if it was active, then start key must be within upper_bound, + // so we can add to minHeap_ directly. + minHeap_.push(&pinned_heap_item_[level]); + } else { + assert(pinned_heap_item_[level].type == + HeapItem::Type::DELETE_RANGE_START); + // this takes care of checking iterate_upper_bound, but with an extra + // key comparison if range_tombstone_iters_[level] was already out of + // bound. Consider using a new HeapItem type or some flag to remember + // boundary checking result. + InsertRangeTombstoneToMinHeap(level); + } + } else { + assert(!active_.count(level)); + } + } + // levels >= starting_level will be reseeked below, so clearing their active + // state here. + active_.erase(active_.lower_bound(starting_level), active_.end()); + } + + status_ = Status::OK(); + IterKey current_search_key; + current_search_key.SetInternalKey(target, false /* copy */); + // Seek target might change to some range tombstone end key, so + // we need to remember them for async requests. + // (level, target) pairs + autovector> prefetched_target; + for (auto level = starting_level; level < children_.size(); ++level) { + { + PERF_TIMER_GUARD(seek_child_seek_time); + children_[level].iter.Seek(current_search_key.GetInternalKey()); + } + + PERF_COUNTER_ADD(seek_child_seek_count, 1); + + if (!range_tombstone_iters_.empty()) { + if (range_tombstone_reseek) { + // This seek is to some range tombstone end key. + // Should only happen when there are range tombstones. + PERF_COUNTER_ADD(internal_range_del_reseek_count, 1); + } + if (children_[level].iter.status().IsTryAgain()) { + prefetched_target.emplace_back( + level, current_search_key.GetInternalKey().ToString()); + } + auto range_tombstone_iter = range_tombstone_iters_[level]; + if (range_tombstone_iter) { + range_tombstone_iter->SeekInternalKey( + current_search_key.GetInternalKey()); + // Invariants (rti) and (phi) + if (range_tombstone_iter->Valid()) { + // If range tombstone starts after `current_search_key`, + // we should insert start key to heap as the range tombstone is not + // active yet. + InsertRangeTombstoneToMinHeap( + level, comparator_->Compare(range_tombstone_iter->start_key(), + pik) > 0 /* start_key */); + // current_search_key < end_key guaranteed by the SeekInternalKey() + // and Valid() calls above. Here we only need to compare user_key + // since if target.user_key == + // range_tombstone_iter->start_key().user_key and target < + // range_tombstone_iter->start_key(), no older level would have any + // key in range [target, range_tombstone_iter->start_key()], so no + // keys in range [target, range_tombstone_iter->end_key()) from older + // level would be visible. So it is safe to seek to + // range_tombstone_iter->end_key(). + // + // TODO: range_tombstone_iter->Seek() finds the max covering + // sequence number, can make it cheaper by not looking for max. + if (comparator_->user_comparator()->Compare( + range_tombstone_iter->start_key().user_key, + current_search_key.GetUserKey()) <= 0) { + range_tombstone_reseek = true; + // Note that for prefix seek case, it is possible that the prefix + // is not the same as the original target, it should not affect + // correctness. Besides, in most cases, range tombstone start and + // end key should have the same prefix? + current_search_key.SetInternalKey(range_tombstone_iter->end_key()); + } + } + } + } + // child.iter.status() is set to Status::TryAgain indicating asynchronous + // request for retrieval of data blocks has been submitted. So it should + // return at this point and Seek should be called again to retrieve the + // requested block and add the child to min heap. + if (children_[level].iter.status().IsTryAgain()) { + continue; + } + { + // Strictly, we timed slightly more than min heap operation, + // but these operations are very cheap. + PERF_TIMER_GUARD(seek_min_heap_time); + AddToMinHeapOrCheckStatus(&children_[level]); + } + } + + if (range_tombstone_iters_.empty()) { + for (auto& child : children_) { + if (child.iter.status().IsTryAgain()) { + child.iter.Seek(target); + { + PERF_TIMER_GUARD(seek_min_heap_time); + AddToMinHeapOrCheckStatus(&child); + } + PERF_COUNTER_ADD(number_async_seek, 1); + } + } + } else { + for (auto& prefetch : prefetched_target) { + // (level, target) pairs + children_[prefetch.first].iter.Seek(prefetch.second); + { + PERF_TIMER_GUARD(seek_min_heap_time); + AddToMinHeapOrCheckStatus(&children_[prefetch.first]); + } + PERF_COUNTER_ADD(number_async_seek, 1); + } + } +} + +// Returns true iff the current key (min heap top) should not be returned +// to user (of the merging iterator). This can be because the current key +// is deleted by some range tombstone, the current key is some fake file +// boundary sentinel key, or the current key is an end point of a range +// tombstone. Advance the iterator at heap top if needed. Heap order is restored +// and `active_` is updated accordingly. +// See FindNextVisibleKey() for more detail on internal implementation +// of advancing child iters. +// When false is returned, if minHeap is not empty, then minHeap_.top().type +// == ITERATOR +// +// REQUIRES: +// - min heap is currently not empty, and iter is in kForward direction. +// - minHeap_ top is not DELETE_RANGE_START (so that `active_` is current). +bool MergingIterator::SkipNextDeleted() { + // 3 types of keys: + // - point key + // - file boundary sentinel keys + // - range deletion end key + auto current = minHeap_.top(); + if (current->type == HeapItem::Type::DELETE_RANGE_END) { + // Invariant(active_): range_tombstone_iters_[current->level] is about to + // become !Valid() or that its start key is going to be added to minHeap_. + active_.erase(current->level); + assert(range_tombstone_iters_[current->level] && + range_tombstone_iters_[current->level]->Valid()); + range_tombstone_iters_[current->level]->Next(); + // Maintain Invariants (rti) and (phi) + if (range_tombstone_iters_[current->level]->Valid()) { + InsertRangeTombstoneToMinHeap(current->level, true /* start_key */, + true /* replace_top */); + } else { + minHeap_.pop(); + } + return true /* current key deleted */; + } + if (current->iter.IsDeleteRangeSentinelKey()) { + // If the file boundary is defined by a range deletion, the range + // tombstone's end key must come before this sentinel key (see op_type in + // SetTombstoneKey()). + assert(ExtractValueType(current->iter.key()) != kTypeRangeDeletion || + active_.count(current->level) == 0); + // When entering a new file, range tombstone iter from the old file is + // freed, but the last key from that range tombstone iter may still be in + // the heap. We need to ensure the data underlying its corresponding key + // Slice is still alive. We do so by popping the range tombstone key from + // heap before calling iter->Next(). Technically, this change is not needed: + // if there is a range tombstone end key that is after file boundary + // sentinel key in minHeap_, the range tombstone end key must have been + // truncated at file boundary. The underlying data of the range tombstone + // end key Slice is the SST file's largest internal key stored as file + // metadata in Version. However, since there are too many implicit + // assumptions made, it is safer to just ensure range tombstone iter is + // still alive. + minHeap_.pop(); + // Remove last SST file's range tombstone end key if there is one. + // This means file boundary is before range tombstone end key, + // which could happen when a range tombstone and a user key + // straddle two SST files. Note that in TruncatedRangeDelIterator + // constructor, parsed_largest.sequence is decremented 1 in this case. + // Maintains Invariant(rti) that at most one + // pinned_heap_item_[current->level] is in minHeap_. + if (range_tombstone_iters_[current->level] && + range_tombstone_iters_[current->level]->Valid()) { + if (!minHeap_.empty() && minHeap_.top()->level == current->level) { + assert(minHeap_.top()->type == HeapItem::Type::DELETE_RANGE_END); + minHeap_.pop(); + // Invariant(active_): we are about to enter a new SST file with new + // range_tombstone_iters[current->level]. Either it is !Valid() or its + // start key is going to be added to minHeap_. + active_.erase(current->level); + } else { + // range tombstone is still valid, but it is not on heap. + // This should only happen if the range tombstone is over iterator + // upper bound. + assert(iterate_upper_bound_ && + comparator_->user_comparator()->CompareWithoutTimestamp( + range_tombstone_iters_[current->level]->start_key().user_key, + true /* a_has_ts */, *iterate_upper_bound_, + false /* b_has_ts */) >= 0); + } + } + // LevelIterator enters a new SST file + current->iter.Next(); + // Invariant(children_): current is popped from heap and added back only if + // it is valid + if (current->iter.Valid()) { + assert(current->iter.status().ok()); + minHeap_.push(current); + } + // Invariants (rti) and (phi) + if (range_tombstone_iters_[current->level] && + range_tombstone_iters_[current->level]->Valid()) { + InsertRangeTombstoneToMinHeap(current->level); + } + return true /* current key deleted */; + } + assert(current->type == HeapItem::Type::ITERATOR); + // Point key case: check active_ for range tombstone coverage. + ParsedInternalKey pik; + ParseInternalKey(current->iter.key(), &pik, false).PermitUncheckedError(); + if (!active_.empty()) { + auto i = *active_.begin(); + if (i < current->level) { + // range tombstone is from a newer level, definitely covers + assert(comparator_->Compare(range_tombstone_iters_[i]->start_key(), + pik) <= 0); + assert(comparator_->Compare(pik, range_tombstone_iters_[i]->end_key()) < + 0); + std::string target; + AppendInternalKey(&target, range_tombstone_iters_[i]->end_key()); + SeekImpl(target, current->level, true); + return true /* current key deleted */; + } else if (i == current->level) { + // range tombstone is from the same level as current, check sequence + // number. By `active_` we know current key is between start key and end + // key. + assert(comparator_->Compare(range_tombstone_iters_[i]->start_key(), + pik) <= 0); + assert(comparator_->Compare(pik, range_tombstone_iters_[i]->end_key()) < + 0); + if (pik.sequence < range_tombstone_iters_[current->level]->seq()) { + // covered by range tombstone + current->iter.Next(); + // Invariant (children_) + if (current->iter.Valid()) { + minHeap_.replace_top(current); + } else { + minHeap_.pop(); + } + return true /* current key deleted */; + } else { + return false /* current key not deleted */; + } + } else { + return false /* current key not deleted */; + // range tombstone from an older sorted run with current key < end key. + // current key is not deleted and the older sorted run will have its range + // tombstone updated when the range tombstone's end key are popped from + // minHeap_. + } + } + // we can reach here only if active_ is empty + assert(active_.empty()); + assert(minHeap_.top()->type == HeapItem::Type::ITERATOR); + return false /* current key not deleted */; +} + +void MergingIterator::SeekForPrevImpl(const Slice& target, + size_t starting_level, + bool range_tombstone_reseek) { + // active range tombstones before `starting_level` remain active + ClearHeaps(false /* clear_active */); + InitMaxHeap(); + ParsedInternalKey pik; + if (!range_tombstone_iters_.empty()) { + ParseInternalKey(target, &pik, false).PermitUncheckedError(); + } + for (size_t level = 0; level < starting_level; ++level) { + PERF_TIMER_GUARD(seek_max_heap_time); + AddToMaxHeapOrCheckStatus(&children_[level]); + } + if (!range_tombstone_iters_.empty()) { + // Add range tombstones before starting_level. + for (size_t level = 0; level < starting_level; ++level) { + if (range_tombstone_iters_[level] && + range_tombstone_iters_[level]->Valid()) { + assert(static_cast(active_.count(level)) == + (pinned_heap_item_[level].type == + HeapItem::Type::DELETE_RANGE_START)); + maxHeap_->push(&pinned_heap_item_[level]); + } else { + assert(!active_.count(level)); + } + } + // levels >= starting_level will be reseeked below, + active_.erase(active_.lower_bound(starting_level), active_.end()); + } + + status_ = Status::OK(); + IterKey current_search_key; + current_search_key.SetInternalKey(target, false /* copy */); + // Seek target might change to some range tombstone end key, so + // we need to remember them for async requests. + // (level, target) pairs + autovector> prefetched_target; + for (auto level = starting_level; level < children_.size(); ++level) { + { + PERF_TIMER_GUARD(seek_child_seek_time); + children_[level].iter.SeekForPrev(current_search_key.GetInternalKey()); + } + + PERF_COUNTER_ADD(seek_child_seek_count, 1); + + if (!range_tombstone_iters_.empty()) { + if (range_tombstone_reseek) { + // This seek is to some range tombstone end key. + // Should only happen when there are range tombstones. + PERF_COUNTER_ADD(internal_range_del_reseek_count, 1); + } + if (children_[level].iter.status().IsTryAgain()) { + prefetched_target.emplace_back( + level, current_search_key.GetInternalKey().ToString()); + } + auto range_tombstone_iter = range_tombstone_iters_[level]; + if (range_tombstone_iter) { + range_tombstone_iter->SeekForPrev(current_search_key.GetUserKey()); + if (range_tombstone_iter->Valid()) { + InsertRangeTombstoneToMaxHeap( + level, comparator_->Compare(range_tombstone_iter->end_key(), + pik) <= 0 /* end_key */); + // start key <= current_search_key guaranteed by the Seek() call above + // Only interested in user key coverage since older sorted runs must + // have smaller sequence numbers than this tombstone. + if (comparator_->user_comparator()->Compare( + current_search_key.GetUserKey(), + range_tombstone_iter->end_key().user_key) < 0) { + range_tombstone_reseek = true; + current_search_key.SetInternalKey( + range_tombstone_iter->start_key().user_key, kMaxSequenceNumber, + kValueTypeForSeekForPrev); + } + } + } + } + // child.iter.status() is set to Status::TryAgain indicating asynchronous + // request for retrieval of data blocks has been submitted. So it should + // return at this point and Seek should be called again to retrieve the + // requested block and add the child to min heap. + if (children_[level].iter.status().IsTryAgain()) { + continue; + } + { + // Strictly, we timed slightly more than min heap operation, + // but these operations are very cheap. + PERF_TIMER_GUARD(seek_max_heap_time); + AddToMaxHeapOrCheckStatus(&children_[level]); + } + } + + if (range_tombstone_iters_.empty()) { + for (auto& child : children_) { + if (child.iter.status().IsTryAgain()) { + child.iter.SeekForPrev(target); + { + PERF_TIMER_GUARD(seek_min_heap_time); + AddToMaxHeapOrCheckStatus(&child); + } + PERF_COUNTER_ADD(number_async_seek, 1); + } + } + } else { + for (auto& prefetch : prefetched_target) { + // (level, target) pairs + children_[prefetch.first].iter.SeekForPrev(prefetch.second); + { + PERF_TIMER_GUARD(seek_max_heap_time); + AddToMaxHeapOrCheckStatus(&children_[prefetch.first]); + } + PERF_COUNTER_ADD(number_async_seek, 1); + } + } +} + +// See more in comments above SkipNextDeleted(). +// REQUIRES: +// - max heap is currently not empty, and iter is in kReverse direction. +// - maxHeap_ top is not DELETE_RANGE_END (so that `active_` is current). +bool MergingIterator::SkipPrevDeleted() { + // 3 types of keys: + // - point key + // - file boundary sentinel keys + // - range deletion start key + auto current = maxHeap_->top(); + if (current->type == HeapItem::Type::DELETE_RANGE_START) { + active_.erase(current->level); + assert(range_tombstone_iters_[current->level] && + range_tombstone_iters_[current->level]->Valid()); + range_tombstone_iters_[current->level]->Prev(); + if (range_tombstone_iters_[current->level]->Valid()) { + InsertRangeTombstoneToMaxHeap(current->level, true /* end_key */, + true /* replace_top */); + } else { + maxHeap_->pop(); + } + return true /* current key deleted */; + } + if (current->iter.IsDeleteRangeSentinelKey()) { + // LevelIterator enters a new SST file + maxHeap_->pop(); + // Remove last SST file's range tombstone key if there is one. + if (!maxHeap_->empty() && maxHeap_->top()->level == current->level && + maxHeap_->top()->type == HeapItem::Type::DELETE_RANGE_START) { + maxHeap_->pop(); + active_.erase(current->level); + } + current->iter.Prev(); + if (current->iter.Valid()) { + assert(current->iter.status().ok()); + maxHeap_->push(current); + } + + if (range_tombstone_iters_[current->level] && + range_tombstone_iters_[current->level]->Valid()) { + InsertRangeTombstoneToMaxHeap(current->level); + } + return true /* current key deleted */; + } + assert(current->type == HeapItem::Type::ITERATOR); + // Point key case: check active_ for range tombstone coverage. + ParsedInternalKey pik; + ParseInternalKey(current->iter.key(), &pik, false).PermitUncheckedError(); + if (!active_.empty()) { + auto i = *active_.begin(); + if (i < current->level) { + // range tombstone is from a newer level, definitely covers + assert(comparator_->Compare(range_tombstone_iters_[i]->start_key(), + pik) <= 0); + assert(comparator_->Compare(pik, range_tombstone_iters_[i]->end_key()) < + 0); + std::string target; + AppendInternalKey(&target, range_tombstone_iters_[i]->start_key()); + // This is different from SkipNextDeleted() which does reseek at sorted + // runs >= level (instead of i+1 here). With min heap, if level L is at + // top of the heap, then levels level L's + // current internal key, which means levels level) { + // By `active_` we know current key is between start key and end key. + assert(comparator_->Compare(range_tombstone_iters_[i]->start_key(), + pik) <= 0); + assert(comparator_->Compare(pik, range_tombstone_iters_[i]->end_key()) < + 0); + if (pik.sequence < range_tombstone_iters_[current->level]->seq()) { + current->iter.Prev(); + if (current->iter.Valid()) { + maxHeap_->replace_top(current); + } else { + maxHeap_->pop(); + } + return true /* current key deleted */; + } else { + return false /* current key not deleted */; + } + } else { + return false /* current key not deleted */; + } + } + + assert(active_.empty()); + assert(maxHeap_->top()->type == HeapItem::Type::ITERATOR); + return false /* current key not deleted */; +} + +void MergingIterator::AddToMinHeapOrCheckStatus(HeapItem* child) { + // Invariant(children_) + if (child->iter.Valid()) { + assert(child->iter.status().ok()); + minHeap_.push(child); + } else { + considerStatus(child->iter.status()); + } +} + +void MergingIterator::AddToMaxHeapOrCheckStatus(HeapItem* child) { + if (child->iter.Valid()) { + assert(child->iter.status().ok()); + maxHeap_->push(child); + } else { + considerStatus(child->iter.status()); + } +} + +// Advance all non current_ child to > current_.key(). +// We advance current_ after the this function call as it does not require +// Seek(). +// Advance all range tombstones iters, including the one corresponding to +// current_, to the first tombstone with end_key > current_.key(). +// TODO: potentially do cascading seek here too +// TODO: show that invariants hold +void MergingIterator::SwitchToForward() { + ClearHeaps(); + Slice target = key(); + for (auto& child : children_) { + if (&child.iter != current_) { + child.iter.Seek(target); + // child.iter.status() is set to Status::TryAgain indicating asynchronous + // request for retrieval of data blocks has been submitted. So it should + // return at this point and Seek should be called again to retrieve the + // requested block and add the child to min heap. + if (child.iter.status() == Status::TryAgain()) { + continue; + } + if (child.iter.Valid() && comparator_->Equal(target, child.iter.key())) { + assert(child.iter.status().ok()); + child.iter.Next(); + } + } + AddToMinHeapOrCheckStatus(&child); + } + + for (auto& child : children_) { + if (child.iter.status() == Status::TryAgain()) { + child.iter.Seek(target); + if (child.iter.Valid() && comparator_->Equal(target, child.iter.key())) { + assert(child.iter.status().ok()); + child.iter.Next(); + } + AddToMinHeapOrCheckStatus(&child); + } + } + + // Current range tombstone iter also needs to seek for the following case: + // Previous direction is backward, so range tombstone iter may point to a + // tombstone before current_. If there is no such tombstone, then the range + // tombstone iter is !Valid(). Need to reseek here to make it valid again. + if (!range_tombstone_iters_.empty()) { + ParsedInternalKey pik; + ParseInternalKey(target, &pik, false /* log_err_key */) + .PermitUncheckedError(); + for (size_t i = 0; i < range_tombstone_iters_.size(); ++i) { + auto iter = range_tombstone_iters_[i]; + if (iter) { + iter->Seek(pik.user_key); + // The while loop is needed as the Seek() call above is only for user + // key. We could have a range tombstone with end_key covering user_key, + // but still is smaller than target. This happens when the range + // tombstone is truncated at iter.largest_. + while (iter->Valid() && + comparator_->Compare(iter->end_key(), pik) <= 0) { + iter->Next(); + } + if (range_tombstone_iters_[i]->Valid()) { + InsertRangeTombstoneToMinHeap( + i, comparator_->Compare(range_tombstone_iters_[i]->start_key(), + pik) > 0 /* start_key */); + } + } + } + } + + direction_ = kForward; + assert(current_ == CurrentForward()); +} + +// Advance all range tombstones iters, including the one corresponding to +// current_, to the first tombstone with start_key <= current_.key(). +void MergingIterator::SwitchToBackward() { + ClearHeaps(); + InitMaxHeap(); + Slice target = key(); + for (auto& child : children_) { + if (&child.iter != current_) { + child.iter.SeekForPrev(target); + TEST_SYNC_POINT_CALLBACK("MergeIterator::Prev:BeforePrev", &child); + if (child.iter.Valid() && comparator_->Equal(target, child.iter.key())) { + assert(child.iter.status().ok()); + child.iter.Prev(); + } + } + AddToMaxHeapOrCheckStatus(&child); + } + + ParsedInternalKey pik; + ParseInternalKey(target, &pik, false /* log_err_key */) + .PermitUncheckedError(); + for (size_t i = 0; i < range_tombstone_iters_.size(); ++i) { + auto iter = range_tombstone_iters_[i]; + if (iter) { + iter->SeekForPrev(pik.user_key); + // Since the SeekForPrev() call above is only for user key, + // we may end up with some range tombstone with start key having the + // same user key at current_, but with a smaller sequence number. This + // makes current_ not at maxHeap_ top for the CurrentReverse() call + // below. If there is a range tombstone start key with the same user + // key and the same sequence number as current_.key(), it will be fine as + // in InsertRangeTombstoneToMaxHeap() we change op_type to be the smallest + // op_type. + while (iter->Valid() && + comparator_->Compare(iter->start_key(), pik) > 0) { + iter->Prev(); + } + if (iter->Valid()) { + InsertRangeTombstoneToMaxHeap( + i, comparator_->Compare(range_tombstone_iters_[i]->end_key(), + pik) <= 0 /* end_key */); + } + } + } + + direction_ = kReverse; + if (!prefix_seek_mode_) { + // Note that we don't do assert(current_ == CurrentReverse()) here + // because it is possible to have some keys larger than the seek-key + // inserted between Seek() and SeekToLast(), which makes current_ not + // equal to CurrentReverse(). + current_ = CurrentReverse(); + } + assert(current_ == CurrentReverse()); +} + +void MergingIterator::ClearHeaps(bool clear_active) { + minHeap_.clear(); + if (maxHeap_) { + maxHeap_->clear(); + } + if (clear_active) { + active_.clear(); + } +} + +void MergingIterator::InitMaxHeap() { + if (!maxHeap_) { + maxHeap_ = + std::make_unique(MaxHeapItemComparator(comparator_)); + } +} + +// Assume there is a next key that is not covered by range tombstone. +// Pre-condition: +// - Invariants (3) and (4) +// - There is some k where k <= children_[i].iter.key() <= LevelNextVisible(i, +// k) for all levels i (LevelNextVisible() defined in Seek()). +// +// Define NextVisible(k) to be the first key >= k from among children_ that +// is not covered by any range tombstone. +// Post-condition: +// - Invariants (1)-(4) hold +// - (*): minHeap_->top()->key() == NextVisible(k) +// +// Loop invariants: +// - Invariants (3) and (4) +// - (*): k <= children_[i].iter.key() <= LevelNextVisible(i, k) +// +// Progress: minHeap_.top()->key() is non-decreasing and strictly increases in +// a finite number of iterations. +// TODO: it is possible to call SeekImpl(k2) after SeekImpl(k1) with +// k2 < k1 in the same FindNextVisibleKey(). For example, l1 has a range +// tombstone [2,3) and l2 has a range tombstone [1, 4). Point key 1 from l5 +// triggers SeekImpl(4 /* target */, 5). Then point key 2 from l3 triggers +// SeekImpl(3 /* target */, 3). +// Ideally we should only move iterators forward in SeekImpl(), and the +// progress condition can be made simpler: iterator only moves forward. +// +// Proof sketch: +// Post-condition: +// Invariant (1) holds when this method returns: +// Ignoring the empty minHeap_ case, there are two cases: +// Case 1: active_ is empty and !minHeap_.top()->iter.IsDeleteRangeSentinelKey() +// By invariants (rti) and (active_), active_ being empty means if a +// pinned_heap_item_[i] is in minHeap_, it has type DELETE_RANGE_START. Note +// that PopDeleteRangeStart() was called right before the while loop condition, +// so minHeap_.top() is not of type DELETE_RANGE_START. So minHeap_.top() must +// be of type ITERATOR. +// Case 2: SkipNextDeleted() returns false. The method returns false only when +// minHeap_.top().type == ITERATOR. +// +// Invariant (2) holds when this method returns: +// From Invariant (1), minHeap_.top().type == ITERATOR. Suppose it is +// children_[i] for some i. Suppose that children_[i].iter.key() is covered by +// some range tombstone. This means there is a j <= i and a range tombstone from +// level j with start_key() < children_[i].iter.key() < end_key(). +// - If range_tombstone_iters_[j]->Valid(), by Invariants (rti) and (phi), +// pinned_heap_item_[j] is in minHeap_, and pinned_heap_item_[j].tombstone_pik +// is either start or end key of this range tombstone. If +// pinned_heap_item_[j].tombstone_pik < children_[i].iter.key(), it would be at +// top of minHeap_ which would contradict Invariant (1). So +// pinned_heap_item_[j].tombstone_pik > children_[i].iter.key(). +// By Invariant (3), range_tombstone_iters_[j].prev.end_key() < +// children_[i].iter.key(). We assume that in each level, range tombstones +// cover non-overlapping ranges. So range_tombstone_iters_[j] is at +// the range tombstone with start_key() < children_[i].iter.key() < end_key() +// and has its end_key() in minHeap_. By Invariants (phi) and (active_), +// j is in active_. From while loop condition, SkipNextDeleted() must have +// returned false for this method to return. +// - If j < i, then SeekImpl(range_tombstone_iters_[j']->end_key(), i) +// was called for some j' < i and j' in active_. Note that since j' is in +// active_, pinned_heap_item_[j'] is in minHeap_ and has tombstone_pik = +// range_tombstone_iters_[j']->end_key(). So +// range_tombstone_iters_[j']->end_key() must be larger than +// children_[i].iter.key() to not be at top of minHeap_. This means after +// SeekImpl(), children_[i] would be at a key > children_[i].iter.key() +// -- contradiction. +// - If j == i, children_[i]->Next() would have been called and children_[i] +// would be at a key > children_[i].iter.key() -- contradiction. +// - If !range_tombstone_iters_[j]->Valid(). Then range_tombstone_iters_[j] +// points to an SST file with all range tombstones from that file exhausted. +// The file must come before the file containing the first +// range tombstone with start_key() < children_[i].iter.key() < end_key(). +// Assume files from same level have non-overlapping ranges, the current file's +// meta.largest is less than children_[i].iter.key(). So the file boundary key, +// which has value meta.largest must have been popped from minHeap_ before +// children_[i].iter.key(). So range_tombstone_iters_[j] would not point to +// this SST file -- contradiction. +// So it is impossible for children_[i].iter.key() to be covered by a range +// tombstone. +// +// Post-condition (*) holds when the function returns: +// From loop invariant (*) that k <= children_[i].iter.key() <= +// LevelNextVisible(i, k) and Invariant (2) above, when the function returns, +// minHeap_.top()->key() is the smallest LevelNextVisible(i, k) among all levels +// i. This is equal to NextVisible(k). +// +// Invariant (3) holds after each iteration: +// PopDeleteRangeStart() does not change range tombstone position. +// In SkipNextDeleted(): +// - If DELETE_RANGE_END is popped from minHeap_, it means the range +// tombstone's end key is < all other point keys, so it is safe to advance to +// next range tombstone. +// - If file boundary is popped (current->iter.IsDeleteRangeSentinelKey()), +// we assume that file's last range tombstone's +// end_key <= file boundary key < all other point keys. So it is safe to +// move to the first range tombstone in the next SST file. +// - If children_[i]->Next() is called, then it is fine as it is advancing a +// point iterator. +// - If SeekImpl(target, l) is called, then (3) follows from SeekImpl()'s +// post-condition if its pre-condition holds. First pre-condition follows +// from loop invariant where Invariant (3) holds for all levels i. +// Now we should second pre-condition holds. Since Invariant (3) holds for +// all i, we have for all j <= l, range_tombstone_iters_[j].prev.end_key() +// < children_[l].iter.key(). `target` is the value of +// range_tombstone_iters_[j'].end_key() for some j' < l and j' in active_. +// By Invariant (active_) and (rti), pinned_heap_item_[j'] is in minHeap_ and +// pinned_heap_item_[j'].tombstone_pik = range_tombstone_iters_[j'].end_key(). +// This end_key must be larger than children_[l].key() since it was not at top +// of minHeap_. So for all levels j <= l, +// range_tombstone_iters_[j].prev.end_key() < children_[l].iter.key() < target +// +// Invariant (4) holds after each iteration: +// A level i is inserted into active_ during calls to PopDeleteRangeStart(). +// In that case, range_tombstone_iters_[i].start_key() < all point keys +// by heap property and the assumption that point keys and range tombstone keys +// are distinct. +// If SeekImpl(target, l) is called, then there is a range_tombstone_iters_[j] +// where target = range_tombstone_iters_[j]->end_key() and children_[l]->key() +// < target. By loop invariants, (3) and (4) holds for levels. +// Since target > children_[l]->key(), it also holds that for j < l, +// range_tombstone_iters_[j].prev.end_key() < target and that if j in active_, +// range_tombstone_iters_[i]->start_key() < target. So all pre-conditions of +// SeekImpl(target, l) holds, and (4) follow from its post-condition. +// All other places either in this function either advance point iterators +// or remove some level from active_, so (4) still holds. +// +// Look Invariant (*): for all level i, k <= children_[i] <= LevelNextVisible(i, +// k). +// k <= children_[i] follows from loop `progress` condition. +// Consider when children_[i] is changed for any i. It is through +// children_[i].iter.Next() or SeekImpl() in SkipNextDeleted(). +// If children_[i].iter.Next() is called, there is a range tombstone from level +// i where tombstone seqno > children_[i].iter.key()'s seqno and i in active_. +// By Invariant (4), tombstone's start_key < children_[i].iter.key(). By +// invariants (active_), (phi), and (rti), tombstone's end_key is in minHeap_ +// and that children_[i].iter.key() < end_key. So children_[i].iter.key() is +// not visible, and it is safe to call Next(). +// If SeekImpl(target, l) is called, by its contract, when SeekImpl() returns, +// target <= children_[i]->key() <= LevelNextVisible(i, target) for i >= l, +// and children_[end_key() for some j < i and j is in active_. +// By Invariant (4), range_tombstone_iters_[j]->start_key() < +// children_[i].iter.key() for all i >= l. So for each level i >= l, the range +// [children_[i].iter.key(), target) is not visible. So after SeekImpl(), +// children_[i].iter.key() <= LevelNextVisible(i, target) <= +// LevelNextVisible(i, k). +// +// `Progress` holds for each iteration: +// Very sloppy intuition: +// - in PopDeleteRangeStart(): the value of a pinned_heap_item_.tombstone_pik_ +// is updated from the start key to the end key of the same range tombstone. +// We assume that start key <= end key for the same range tombstone. +// - in SkipNextDeleted() +// - If the top of heap is DELETE_RANGE_END, the range tombstone is advanced +// and the relevant pinned_heap_item_.tombstone_pik is increased or popped +// from minHeap_. +// - If the top of heap is a file boundary key, then both point iter and +// range tombstone iter are advanced to the next file. +// - If the top of heap is ITERATOR and current->iter.Next() is called, it +// moves to a larger point key. +// - If the top of heap is ITERATOR and SeekImpl(k, l) is called, then all +// iterators from levels >= l are advanced to some key >= k by its contract. +// And top of minHeap_ before SeekImpl(k, l) was less than k. +// There are special cases where different heap items have the same key, +// e.g. when two range tombstone end keys share the same value). In +// these cases, iterators are being advanced, so the minimum key should increase +// in a finite number of steps. +inline void MergingIterator::FindNextVisibleKey() { + PopDeleteRangeStart(); + // PopDeleteRangeStart() implies heap top is not DELETE_RANGE_START + // active_ being empty implies no DELETE_RANGE_END in heap. + // So minHeap_->top() must be of type ITERATOR. + while ( + !minHeap_.empty() && + (!active_.empty() || minHeap_.top()->iter.IsDeleteRangeSentinelKey()) && + SkipNextDeleted()) { + PopDeleteRangeStart(); + } + // Checks Invariant (1) + assert(minHeap_.empty() || minHeap_.top()->type == HeapItem::Type::ITERATOR); +} + +inline void MergingIterator::FindPrevVisibleKey() { + PopDeleteRangeEnd(); + // PopDeleteRangeEnd() implies heap top is not DELETE_RANGE_END + // active_ being empty implies no DELETE_RANGE_START in heap. + // So maxHeap_->top() must be of type ITERATOR. + while ( + !maxHeap_->empty() && + (!active_.empty() || maxHeap_->top()->iter.IsDeleteRangeSentinelKey()) && + SkipPrevDeleted()) { + PopDeleteRangeEnd(); + } +} + +InternalIterator* NewMergingIterator(const InternalKeyComparator* cmp, + InternalIterator** list, int n, + Arena* arena, bool prefix_seek_mode) { + assert(n >= 0); + if (n == 0) { + return NewEmptyInternalIterator(arena); + } else if (n == 1) { + return list[0]; + } else { + if (arena == nullptr) { + return new MergingIterator(cmp, list, n, false, prefix_seek_mode); + } else { + auto mem = arena->AllocateAligned(sizeof(MergingIterator)); + return new (mem) MergingIterator(cmp, list, n, true, prefix_seek_mode); + } + } +} + +MergeIteratorBuilder::MergeIteratorBuilder( + const InternalKeyComparator* comparator, Arena* a, bool prefix_seek_mode, + const Slice* iterate_upper_bound) + : first_iter(nullptr), use_merging_iter(false), arena(a) { + auto mem = arena->AllocateAligned(sizeof(MergingIterator)); + merge_iter = new (mem) MergingIterator(comparator, nullptr, 0, true, + prefix_seek_mode, iterate_upper_bound); +} + +MergeIteratorBuilder::~MergeIteratorBuilder() { + if (first_iter != nullptr) { + first_iter->~InternalIterator(); + } + if (merge_iter != nullptr) { + merge_iter->~MergingIterator(); + } +} + +void MergeIteratorBuilder::AddIterator(InternalIterator* iter) { + if (!use_merging_iter && first_iter != nullptr) { + merge_iter->AddIterator(first_iter); + use_merging_iter = true; + first_iter = nullptr; + } + if (use_merging_iter) { + merge_iter->AddIterator(iter); + } else { + first_iter = iter; + } +} + +void MergeIteratorBuilder::AddPointAndTombstoneIterator( + InternalIterator* point_iter, TruncatedRangeDelIterator* tombstone_iter, + TruncatedRangeDelIterator*** tombstone_iter_ptr) { + // tombstone_iter_ptr != nullptr means point_iter is a LevelIterator. + bool add_range_tombstone = tombstone_iter || + !merge_iter->range_tombstone_iters_.empty() || + tombstone_iter_ptr; + if (!use_merging_iter && (add_range_tombstone || first_iter)) { + use_merging_iter = true; + if (first_iter) { + merge_iter->AddIterator(first_iter); + first_iter = nullptr; + } + } + if (use_merging_iter) { + merge_iter->AddIterator(point_iter); + if (add_range_tombstone) { + // If there was a gap, fill in nullptr as empty range tombstone iterators. + while (merge_iter->range_tombstone_iters_.size() < + merge_iter->children_.size() - 1) { + merge_iter->AddRangeTombstoneIterator(nullptr); + } + merge_iter->AddRangeTombstoneIterator(tombstone_iter); + } + + if (tombstone_iter_ptr) { + // This is needed instead of setting to &range_tombstone_iters_[i] + // directly here since the memory address of range_tombstone_iters_[i] + // might change during vector resizing. + range_del_iter_ptrs_.emplace_back( + merge_iter->range_tombstone_iters_.size() - 1, tombstone_iter_ptr); + } + } else { + first_iter = point_iter; + } +} + +InternalIterator* MergeIteratorBuilder::Finish(ArenaWrappedDBIter* db_iter) { + InternalIterator* ret = nullptr; + if (!use_merging_iter) { + ret = first_iter; + first_iter = nullptr; + } else { + for (auto& p : range_del_iter_ptrs_) { + *(p.second) = &(merge_iter->range_tombstone_iters_[p.first]); + } + if (db_iter && !merge_iter->range_tombstone_iters_.empty()) { + // memtable is always the first level + db_iter->SetMemtableRangetombstoneIter( + &merge_iter->range_tombstone_iters_.front()); + } + merge_iter->Finish(); + ret = merge_iter; + merge_iter = nullptr; + } + return ret; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/merging_iterator.h b/librocksdb-sys/rocksdb/table/merging_iterator.h new file mode 100644 index 0000000..562a4e5 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/merging_iterator.h @@ -0,0 +1,97 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include "db/range_del_aggregator.h" +#include "rocksdb/slice.h" +#include "rocksdb/types.h" +#include "table/iterator_wrapper.h" + +namespace ROCKSDB_NAMESPACE { + +class Arena; +class ArenaWrappedDBIter; +class InternalKeyComparator; + +template +class InternalIteratorBase; +using InternalIterator = InternalIteratorBase; + +// Return an iterator that provides the union of the data in +// children[0,n-1]. Takes ownership of the child iterators and +// will delete them when the result iterator is deleted. +// +// The result does no duplicate suppression. I.e., if a particular +// key is present in K child iterators, it will be yielded K times. +// +// REQUIRES: n >= 0 +extern InternalIterator* NewMergingIterator( + const InternalKeyComparator* comparator, InternalIterator** children, int n, + Arena* arena = nullptr, bool prefix_seek_mode = false); + +// The iterator returned by NewMergingIterator() and +// MergeIteratorBuilder::Finish(). MergingIterator handles the merging of data +// from different point and/or range tombstone iterators. +class MergingIterator; + +// A builder class to for an iterator that provides the union of data +// of input iterators. Two APIs are provided to add input iterators. User should +// only call one of them exclusively depending on if range tombstone should be +// processed. +class MergeIteratorBuilder { + public: + // comparator: the comparator used in merging comparator + // arena: where the merging iterator needs to be allocated from. + explicit MergeIteratorBuilder(const InternalKeyComparator* comparator, + Arena* arena, bool prefix_seek_mode = false, + const Slice* iterate_upper_bound = nullptr); + ~MergeIteratorBuilder(); + + // Add point key iterator `iter` to the merging iterator. + void AddIterator(InternalIterator* iter); + + // Add a point key iterator and a range tombstone iterator. + // `tombstone_iter_ptr` should and only be set by LevelIterator. + // *tombstone_iter_ptr will be set to where the merging iterator stores + // `tombstone_iter` when MergeIteratorBuilder::Finish() is called. This is + // used by LevelIterator to update range tombstone iters when switching to a + // different SST file. If a single point iterator with a nullptr range + // tombstone iterator is provided, and the point iterator is not a level + // iterator, then this builder will return the point iterator directly, + // instead of creating a merging iterator on top of it. Internally, if all + // point iterators are not LevelIterator, then range tombstone iterator is + // only added to the merging iter if there is a non-null `tombstone_iter`. + void AddPointAndTombstoneIterator( + InternalIterator* point_iter, TruncatedRangeDelIterator* tombstone_iter, + TruncatedRangeDelIterator*** tombstone_iter_ptr = nullptr); + + // Get arena used to build the merging iterator. It is called one a child + // iterator needs to be allocated. + Arena* GetArena() { return arena; } + + // Return the result merging iterator. + // If db_iter is not nullptr, then db_iter->SetMemtableRangetombstoneIter() + // will be called with pointer to where the merging iterator + // stores the memtable range tombstone iterator. + // This is used for DB iterator to refresh memtable range tombstones. + InternalIterator* Finish(ArenaWrappedDBIter* db_iter = nullptr); + + private: + MergingIterator* merge_iter; + InternalIterator* first_iter; + bool use_merging_iter; + Arena* arena; + // Used to set LevelIterator.range_tombstone_iter_. + // See AddRangeTombstoneIterator() implementation for more detail. + std::vector> + range_del_iter_ptrs_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/meta_blocks.cc b/librocksdb-sys/rocksdb/table/meta_blocks.cc new file mode 100644 index 0000000..b82b596 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/meta_blocks.cc @@ -0,0 +1,572 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#include "table/meta_blocks.h" + +#include +#include + +#include "block_fetcher.h" +#include "db/table_properties_collector.h" +#include "file/random_access_file_reader.h" +#include "logging/logging.h" +#include "rocksdb/options.h" +#include "rocksdb/table.h" +#include "rocksdb/table_properties.h" +#include "table/block_based/block.h" +#include "table/block_based/reader_common.h" +#include "table/format.h" +#include "table/internal_iterator.h" +#include "table/persistent_cache_helper.h" +#include "table/sst_file_writer_collectors.h" +#include "table/table_properties_internal.h" +#include "test_util/sync_point.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { + +const std::string kPropertiesBlockName = "rocksdb.properties"; +// NB: only used with format_version >= 6 +const std::string kIndexBlockName = "rocksdb.index"; +// Old property block name for backward compatibility +const std::string kPropertiesBlockOldName = "rocksdb.stats"; +const std::string kCompressionDictBlockName = "rocksdb.compression_dict"; +const std::string kRangeDelBlockName = "rocksdb.range_del"; + +MetaIndexBuilder::MetaIndexBuilder() + : meta_index_block_(new BlockBuilder(1 /* restart interval */)) {} + +void MetaIndexBuilder::Add(const std::string& key, const BlockHandle& handle) { + std::string handle_encoding; + handle.EncodeTo(&handle_encoding); + meta_block_handles_.insert({key, handle_encoding}); +} + +Slice MetaIndexBuilder::Finish() { + for (const auto& metablock : meta_block_handles_) { + meta_index_block_->Add(metablock.first, metablock.second); + } + return meta_index_block_->Finish(); +} + +// Property block will be read sequentially and cached in a heap located +// object, so there's no need for restart points. Thus we set the restart +// interval to infinity to save space. +PropertyBlockBuilder::PropertyBlockBuilder() + : properties_block_(new BlockBuilder( + std::numeric_limits::max() /* restart interval */)) {} + +void PropertyBlockBuilder::Add(const std::string& name, + const std::string& val) { + props_.insert({name, val}); +} + +void PropertyBlockBuilder::Add(const std::string& name, uint64_t val) { + assert(props_.find(name) == props_.end()); + + std::string dst; + PutVarint64(&dst, val); + + Add(name, dst); +} + +void PropertyBlockBuilder::Add( + const UserCollectedProperties& user_collected_properties) { + for (const auto& prop : user_collected_properties) { + Add(prop.first, prop.second); + } +} + +void PropertyBlockBuilder::AddTableProperty(const TableProperties& props) { + TEST_SYNC_POINT_CALLBACK("PropertyBlockBuilder::AddTableProperty:Start", + const_cast(&props)); + + Add(TablePropertiesNames::kOriginalFileNumber, props.orig_file_number); + Add(TablePropertiesNames::kRawKeySize, props.raw_key_size); + Add(TablePropertiesNames::kRawValueSize, props.raw_value_size); + Add(TablePropertiesNames::kDataSize, props.data_size); + Add(TablePropertiesNames::kIndexSize, props.index_size); + if (props.index_partitions != 0) { + Add(TablePropertiesNames::kIndexPartitions, props.index_partitions); + Add(TablePropertiesNames::kTopLevelIndexSize, props.top_level_index_size); + } + Add(TablePropertiesNames::kIndexKeyIsUserKey, props.index_key_is_user_key); + Add(TablePropertiesNames::kIndexValueIsDeltaEncoded, + props.index_value_is_delta_encoded); + Add(TablePropertiesNames::kNumEntries, props.num_entries); + Add(TablePropertiesNames::kNumFilterEntries, props.num_filter_entries); + Add(TablePropertiesNames::kDeletedKeys, props.num_deletions); + Add(TablePropertiesNames::kMergeOperands, props.num_merge_operands); + Add(TablePropertiesNames::kNumRangeDeletions, props.num_range_deletions); + Add(TablePropertiesNames::kNumDataBlocks, props.num_data_blocks); + Add(TablePropertiesNames::kFilterSize, props.filter_size); + Add(TablePropertiesNames::kFormatVersion, props.format_version); + Add(TablePropertiesNames::kFixedKeyLen, props.fixed_key_len); + Add(TablePropertiesNames::kColumnFamilyId, props.column_family_id); + Add(TablePropertiesNames::kCreationTime, props.creation_time); + Add(TablePropertiesNames::kOldestKeyTime, props.oldest_key_time); + if (props.file_creation_time > 0) { + Add(TablePropertiesNames::kFileCreationTime, props.file_creation_time); + } + if (props.slow_compression_estimated_data_size > 0) { + Add(TablePropertiesNames::kSlowCompressionEstimatedDataSize, + props.slow_compression_estimated_data_size); + } + if (props.fast_compression_estimated_data_size > 0) { + Add(TablePropertiesNames::kFastCompressionEstimatedDataSize, + props.fast_compression_estimated_data_size); + } + Add(TablePropertiesNames::kTailStartOffset, props.tail_start_offset); + if (props.user_defined_timestamps_persisted == 0) { + Add(TablePropertiesNames::kUserDefinedTimestampsPersisted, + props.user_defined_timestamps_persisted); + } + if (!props.db_id.empty()) { + Add(TablePropertiesNames::kDbId, props.db_id); + } + if (!props.db_session_id.empty()) { + Add(TablePropertiesNames::kDbSessionId, props.db_session_id); + } + if (!props.db_host_id.empty()) { + Add(TablePropertiesNames::kDbHostId, props.db_host_id); + } + + if (!props.filter_policy_name.empty()) { + Add(TablePropertiesNames::kFilterPolicy, props.filter_policy_name); + } + if (!props.comparator_name.empty()) { + Add(TablePropertiesNames::kComparator, props.comparator_name); + } + + if (!props.merge_operator_name.empty()) { + Add(TablePropertiesNames::kMergeOperator, props.merge_operator_name); + } + if (!props.prefix_extractor_name.empty()) { + Add(TablePropertiesNames::kPrefixExtractorName, + props.prefix_extractor_name); + } + if (!props.property_collectors_names.empty()) { + Add(TablePropertiesNames::kPropertyCollectors, + props.property_collectors_names); + } + if (!props.column_family_name.empty()) { + Add(TablePropertiesNames::kColumnFamilyName, props.column_family_name); + } + + if (!props.compression_name.empty()) { + Add(TablePropertiesNames::kCompression, props.compression_name); + } + if (!props.compression_options.empty()) { + Add(TablePropertiesNames::kCompressionOptions, props.compression_options); + } + if (!props.seqno_to_time_mapping.empty()) { + Add(TablePropertiesNames::kSequenceNumberTimeMapping, + props.seqno_to_time_mapping); + } +} + +Slice PropertyBlockBuilder::Finish() { + for (const auto& prop : props_) { + properties_block_->Add(prop.first, prop.second); + } + + return properties_block_->Finish(); +} + +void LogPropertiesCollectionError(Logger* info_log, const std::string& method, + const std::string& name) { + assert(method == "Add" || method == "Finish"); + + std::string msg = + "Encountered error when calling TablePropertiesCollector::" + method + + "() with collector name: " + name; + ROCKS_LOG_ERROR(info_log, "%s", msg.c_str()); +} + +bool NotifyCollectTableCollectorsOnAdd( + const Slice& key, const Slice& value, uint64_t file_size, + const std::vector>& collectors, + Logger* info_log) { + bool all_succeeded = true; + for (auto& collector : collectors) { + Status s = collector->InternalAdd(key, value, file_size); + all_succeeded = all_succeeded && s.ok(); + if (!s.ok()) { + LogPropertiesCollectionError(info_log, "Add" /* method */, + collector->Name()); + } + } + return all_succeeded; +} + +void NotifyCollectTableCollectorsOnBlockAdd( + const std::vector>& collectors, + const uint64_t block_uncomp_bytes, + const uint64_t block_compressed_bytes_fast, + const uint64_t block_compressed_bytes_slow) { + for (auto& collector : collectors) { + collector->BlockAdd(block_uncomp_bytes, block_compressed_bytes_fast, + block_compressed_bytes_slow); + } +} + +bool NotifyCollectTableCollectorsOnFinish( + const std::vector>& collectors, + Logger* info_log, PropertyBlockBuilder* builder) { + bool all_succeeded = true; + for (auto& collector : collectors) { + UserCollectedProperties user_collected_properties; + Status s = collector->Finish(&user_collected_properties); + + all_succeeded = all_succeeded && s.ok(); + if (!s.ok()) { + LogPropertiesCollectionError(info_log, "Finish" /* method */, + collector->Name()); + } else { + builder->Add(user_collected_properties); + } + } + + return all_succeeded; +} + +// FIXME: should be a parameter for reading table properties to use persistent +// cache? +Status ReadTablePropertiesHelper( + const ReadOptions& ro, const BlockHandle& handle, + RandomAccessFileReader* file, FilePrefetchBuffer* prefetch_buffer, + const Footer& footer, const ImmutableOptions& ioptions, + std::unique_ptr* table_properties, + MemoryAllocator* memory_allocator) { + assert(table_properties); + + // If this is an external SST file ingested with write_global_seqno set to + // true, then we expect the checksum mismatch because checksum was written + // by SstFileWriter, but its global seqno in the properties block may have + // been changed during ingestion. For this reason, we initially read + // and process without checksum verification, then later try checksum + // verification so that if it fails, we can copy to a temporary buffer with + // global seqno set to its original value, i.e. 0, and attempt checksum + // verification again. + ReadOptions modified_ro = ro; + modified_ro.verify_checksums = false; + BlockContents block_contents; + BlockFetcher block_fetcher(file, prefetch_buffer, footer, modified_ro, handle, + &block_contents, ioptions, false /* decompress */, + false /*maybe_compressed*/, BlockType::kProperties, + UncompressionDict::GetEmptyDict(), + PersistentCacheOptions::kEmpty, memory_allocator); + Status s = block_fetcher.ReadBlockContents(); + if (!s.ok()) { + return s; + } + + // Unfortunately, Block::size() might not equal block_contents.data.size(), + // and Block hides block_contents + uint64_t block_size = block_contents.data.size(); + Block properties_block(std::move(block_contents)); + std::unique_ptr iter(properties_block.NewMetaIterator()); + + std::unique_ptr new_table_properties{new TableProperties}; + // All pre-defined properties of type uint64_t + std::unordered_map predefined_uint64_properties = { + {TablePropertiesNames::kOriginalFileNumber, + &new_table_properties->orig_file_number}, + {TablePropertiesNames::kDataSize, &new_table_properties->data_size}, + {TablePropertiesNames::kIndexSize, &new_table_properties->index_size}, + {TablePropertiesNames::kIndexPartitions, + &new_table_properties->index_partitions}, + {TablePropertiesNames::kTopLevelIndexSize, + &new_table_properties->top_level_index_size}, + {TablePropertiesNames::kIndexKeyIsUserKey, + &new_table_properties->index_key_is_user_key}, + {TablePropertiesNames::kIndexValueIsDeltaEncoded, + &new_table_properties->index_value_is_delta_encoded}, + {TablePropertiesNames::kFilterSize, &new_table_properties->filter_size}, + {TablePropertiesNames::kRawKeySize, &new_table_properties->raw_key_size}, + {TablePropertiesNames::kRawValueSize, + &new_table_properties->raw_value_size}, + {TablePropertiesNames::kNumDataBlocks, + &new_table_properties->num_data_blocks}, + {TablePropertiesNames::kNumEntries, &new_table_properties->num_entries}, + {TablePropertiesNames::kNumFilterEntries, + &new_table_properties->num_filter_entries}, + {TablePropertiesNames::kDeletedKeys, + &new_table_properties->num_deletions}, + {TablePropertiesNames::kMergeOperands, + &new_table_properties->num_merge_operands}, + {TablePropertiesNames::kNumRangeDeletions, + &new_table_properties->num_range_deletions}, + {TablePropertiesNames::kFormatVersion, + &new_table_properties->format_version}, + {TablePropertiesNames::kFixedKeyLen, + &new_table_properties->fixed_key_len}, + {TablePropertiesNames::kColumnFamilyId, + &new_table_properties->column_family_id}, + {TablePropertiesNames::kCreationTime, + &new_table_properties->creation_time}, + {TablePropertiesNames::kOldestKeyTime, + &new_table_properties->oldest_key_time}, + {TablePropertiesNames::kFileCreationTime, + &new_table_properties->file_creation_time}, + {TablePropertiesNames::kSlowCompressionEstimatedDataSize, + &new_table_properties->slow_compression_estimated_data_size}, + {TablePropertiesNames::kFastCompressionEstimatedDataSize, + &new_table_properties->fast_compression_estimated_data_size}, + {TablePropertiesNames::kTailStartOffset, + &new_table_properties->tail_start_offset}, + {TablePropertiesNames::kUserDefinedTimestampsPersisted, + &new_table_properties->user_defined_timestamps_persisted}, + }; + + std::string last_key; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + s = iter->status(); + if (!s.ok()) { + break; + } + + auto key = iter->key().ToString(); + // properties block should be strictly sorted with no duplicate key. + if (!last_key.empty() && + BytewiseComparator()->Compare(key, last_key) <= 0) { + s = Status::Corruption("properties unsorted"); + break; + } + last_key = key; + + auto raw_val = iter->value(); + auto pos = predefined_uint64_properties.find(key); + + if (key == ExternalSstFilePropertyNames::kGlobalSeqno) { + new_table_properties->external_sst_file_global_seqno_offset = + handle.offset() + iter->ValueOffset(); + } + + if (pos != predefined_uint64_properties.end()) { + if (key == TablePropertiesNames::kDeletedKeys || + key == TablePropertiesNames::kMergeOperands) { + // Insert in user-collected properties for API backwards compatibility + new_table_properties->user_collected_properties.insert( + {key, raw_val.ToString()}); + } + // handle predefined rocksdb properties + uint64_t val; + if (!GetVarint64(&raw_val, &val)) { + // skip malformed value + auto error_msg = + "Detect malformed value in properties meta-block:" + "\tkey: " + + key + "\tval: " + raw_val.ToString(); + ROCKS_LOG_ERROR(ioptions.logger, "%s", error_msg.c_str()); + continue; + } + *(pos->second) = val; + } else if (key == TablePropertiesNames::kDbId) { + new_table_properties->db_id = raw_val.ToString(); + } else if (key == TablePropertiesNames::kDbSessionId) { + new_table_properties->db_session_id = raw_val.ToString(); + } else if (key == TablePropertiesNames::kDbHostId) { + new_table_properties->db_host_id = raw_val.ToString(); + } else if (key == TablePropertiesNames::kFilterPolicy) { + new_table_properties->filter_policy_name = raw_val.ToString(); + } else if (key == TablePropertiesNames::kColumnFamilyName) { + new_table_properties->column_family_name = raw_val.ToString(); + } else if (key == TablePropertiesNames::kComparator) { + new_table_properties->comparator_name = raw_val.ToString(); + } else if (key == TablePropertiesNames::kMergeOperator) { + new_table_properties->merge_operator_name = raw_val.ToString(); + } else if (key == TablePropertiesNames::kPrefixExtractorName) { + new_table_properties->prefix_extractor_name = raw_val.ToString(); + } else if (key == TablePropertiesNames::kPropertyCollectors) { + new_table_properties->property_collectors_names = raw_val.ToString(); + } else if (key == TablePropertiesNames::kCompression) { + new_table_properties->compression_name = raw_val.ToString(); + } else if (key == TablePropertiesNames::kCompressionOptions) { + new_table_properties->compression_options = raw_val.ToString(); + } else if (key == TablePropertiesNames::kSequenceNumberTimeMapping) { + new_table_properties->seqno_to_time_mapping = raw_val.ToString(); + } else { + // handle user-collected properties + new_table_properties->user_collected_properties.insert( + {key, raw_val.ToString()}); + } + } + + // Modified version of BlockFetcher checksum verification + // (See write_global_seqno comment above) + if (s.ok() && footer.GetBlockTrailerSize() > 0) { + s = VerifyBlockChecksum(footer, properties_block.data(), block_size, + file->file_name(), handle.offset()); + if (s.IsCorruption()) { + if (new_table_properties->external_sst_file_global_seqno_offset != 0) { + std::string tmp_buf(properties_block.data(), + block_fetcher.GetBlockSizeWithTrailer()); + uint64_t global_seqno_offset = + new_table_properties->external_sst_file_global_seqno_offset - + handle.offset(); + EncodeFixed64(&tmp_buf[static_cast(global_seqno_offset)], 0); + s = VerifyBlockChecksum(footer, tmp_buf.data(), block_size, + file->file_name(), handle.offset()); + } + } + } + + if (s.ok()) { + *table_properties = std::move(new_table_properties); + } + + return s; +} + +Status ReadTableProperties(RandomAccessFileReader* file, uint64_t file_size, + uint64_t table_magic_number, + const ImmutableOptions& ioptions, + const ReadOptions& read_options, + std::unique_ptr* properties, + MemoryAllocator* memory_allocator, + FilePrefetchBuffer* prefetch_buffer) { + BlockHandle block_handle; + Footer footer; + Status s = + FindMetaBlockInFile(file, file_size, table_magic_number, ioptions, + read_options, kPropertiesBlockName, &block_handle, + memory_allocator, prefetch_buffer, &footer); + if (!s.ok()) { + return s; + } + + if (!block_handle.IsNull()) { + s = ReadTablePropertiesHelper(read_options, block_handle, file, + prefetch_buffer, footer, ioptions, properties, + memory_allocator); + } else { + s = Status::NotFound(); + } + return s; +} + +Status FindOptionalMetaBlock(InternalIterator* meta_index_iter, + const std::string& meta_block_name, + BlockHandle* block_handle) { + assert(block_handle != nullptr); + meta_index_iter->Seek(meta_block_name); + if (meta_index_iter->status().ok()) { + if (meta_index_iter->Valid() && meta_index_iter->key() == meta_block_name) { + Slice v = meta_index_iter->value(); + return block_handle->DecodeFrom(&v); + } else if (meta_block_name == kPropertiesBlockName) { + // Have to try old name for compatibility + meta_index_iter->Seek(kPropertiesBlockOldName); + if (meta_index_iter->status().ok() && meta_index_iter->Valid() && + meta_index_iter->key() == kPropertiesBlockOldName) { + Slice v = meta_index_iter->value(); + return block_handle->DecodeFrom(&v); + } + } + } + // else + *block_handle = BlockHandle::NullBlockHandle(); + return meta_index_iter->status(); +} + +Status FindMetaBlock(InternalIterator* meta_index_iter, + const std::string& meta_block_name, + BlockHandle* block_handle) { + Status s = + FindOptionalMetaBlock(meta_index_iter, meta_block_name, block_handle); + if (s.ok() && block_handle->IsNull()) { + return Status::Corruption("Cannot find the meta block", meta_block_name); + } else { + return s; + } +} + +Status ReadMetaIndexBlockInFile(RandomAccessFileReader* file, + uint64_t file_size, uint64_t table_magic_number, + const ImmutableOptions& ioptions, + const ReadOptions& read_options, + BlockContents* metaindex_contents, + MemoryAllocator* memory_allocator, + FilePrefetchBuffer* prefetch_buffer, + Footer* footer_out) { + Footer footer; + IOOptions opts; + Status s; + s = file->PrepareIOOptions(read_options, opts); + if (!s.ok()) { + return s; + } + s = ReadFooterFromFile(opts, file, *ioptions.fs, prefetch_buffer, file_size, + &footer, table_magic_number); + if (!s.ok()) { + return s; + } + if (footer_out) { + *footer_out = footer; + } + + auto metaindex_handle = footer.metaindex_handle(); + return BlockFetcher(file, prefetch_buffer, footer, read_options, + metaindex_handle, metaindex_contents, ioptions, + false /* do decompression */, false /*maybe_compressed*/, + BlockType::kMetaIndex, UncompressionDict::GetEmptyDict(), + PersistentCacheOptions::kEmpty, memory_allocator) + .ReadBlockContents(); +} + +Status FindMetaBlockInFile( + RandomAccessFileReader* file, uint64_t file_size, + uint64_t table_magic_number, const ImmutableOptions& ioptions, + const ReadOptions& read_options, const std::string& meta_block_name, + BlockHandle* block_handle, MemoryAllocator* memory_allocator, + FilePrefetchBuffer* prefetch_buffer, Footer* footer_out) { + BlockContents metaindex_contents; + auto s = ReadMetaIndexBlockInFile( + file, file_size, table_magic_number, ioptions, read_options, + &metaindex_contents, memory_allocator, prefetch_buffer, footer_out); + if (!s.ok()) { + return s; + } + // meta blocks are never compressed. Need to add uncompress logic if we are to + // compress it. + Block metaindex_block(std::move(metaindex_contents)); + + std::unique_ptr meta_iter; + meta_iter.reset(metaindex_block.NewMetaIterator()); + + return FindMetaBlock(meta_iter.get(), meta_block_name, block_handle); +} + +Status ReadMetaBlock(RandomAccessFileReader* file, + FilePrefetchBuffer* prefetch_buffer, uint64_t file_size, + uint64_t table_magic_number, + const ImmutableOptions& ioptions, + const ReadOptions& read_options, + const std::string& meta_block_name, BlockType block_type, + BlockContents* contents, + MemoryAllocator* memory_allocator) { + // TableProperties requires special handling because of checksum issues. + // Call ReadTableProperties instead for that case. + assert(block_type != BlockType::kProperties); + + BlockHandle block_handle; + Footer footer; + Status status = + FindMetaBlockInFile(file, file_size, table_magic_number, ioptions, + read_options, meta_block_name, &block_handle, + memory_allocator, prefetch_buffer, &footer); + if (!status.ok()) { + return status; + } + + return BlockFetcher(file, prefetch_buffer, footer, read_options, block_handle, + contents, ioptions, false /* decompress */, + false /*maybe_compressed*/, block_type, + UncompressionDict::GetEmptyDict(), + PersistentCacheOptions::kEmpty, memory_allocator) + .ReadBlockContents(); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/meta_blocks.h b/librocksdb-sys/rocksdb/table/meta_blocks.h new file mode 100644 index 0000000..1ed9cf2 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/meta_blocks.h @@ -0,0 +1,173 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include +#include +#include +#include + +#include "db/builder.h" +#include "db/table_properties_collector.h" +#include "rocksdb/comparator.h" +#include "rocksdb/memory_allocator.h" +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "table/block_based/block_builder.h" +#include "table/block_based/block_type.h" +#include "table/format.h" +#include "util/kv_map.h" + +namespace ROCKSDB_NAMESPACE { + +class BlockBuilder; +class BlockHandle; +class Env; +class Footer; +class Logger; +class RandomAccessFile; +struct TableProperties; + +// Meta block names for metaindex +extern const std::string kPropertiesBlockName; +extern const std::string kIndexBlockName; +extern const std::string kPropertiesBlockOldName; +extern const std::string kCompressionDictBlockName; +extern const std::string kRangeDelBlockName; + +class MetaIndexBuilder { + public: + MetaIndexBuilder(const MetaIndexBuilder&) = delete; + MetaIndexBuilder& operator=(const MetaIndexBuilder&) = delete; + + MetaIndexBuilder(); + void Add(const std::string& key, const BlockHandle& handle); + + // Write all the added key/value pairs to the block and return the contents + // of the block. + Slice Finish(); + + private: + // store the sorted key/handle of the metablocks. + stl_wrappers::KVMap meta_block_handles_; + std::unique_ptr meta_index_block_; +}; + +class PropertyBlockBuilder { + public: + PropertyBlockBuilder(const PropertyBlockBuilder&) = delete; + PropertyBlockBuilder& operator=(const PropertyBlockBuilder&) = delete; + + PropertyBlockBuilder(); + + void AddTableProperty(const TableProperties& props); + void Add(const std::string& key, uint64_t value); + void Add(const std::string& key, const std::string& value); + void Add(const UserCollectedProperties& user_collected_properties); + + // Write all the added entries to the block and return the block contents + Slice Finish(); + + private: + std::unique_ptr properties_block_; + stl_wrappers::KVMap props_; +}; + +// Were we encounter any error occurs during user-defined statistics collection, +// we'll write the warning message to info log. +void LogPropertiesCollectionError(Logger* info_log, const std::string& method, + const std::string& name); + +// Utility functions help table builder to trigger batch events for user +// defined property collectors. +// Return value indicates if there is any error occurred; if error occurred, +// the warning message will be logged. +// NotifyCollectTableCollectorsOnAdd() triggers the `Add` event for all +// property collectors. +bool NotifyCollectTableCollectorsOnAdd( + const Slice& key, const Slice& value, uint64_t file_size, + const std::vector>& collectors, + Logger* info_log); + +void NotifyCollectTableCollectorsOnBlockAdd( + const std::vector>& collectors, + uint64_t block_uncomp_bytes, uint64_t block_compressed_bytes_fast, + uint64_t block_compressed_bytes_slow); + +// NotifyCollectTableCollectorsOnFinish() triggers the `Finish` event for all +// property collectors. The collected properties will be added to `builder`. +bool NotifyCollectTableCollectorsOnFinish( + const std::vector>& collectors, + Logger* info_log, PropertyBlockBuilder* builder); + +// Read table properties from a file using known BlockHandle. +// @returns a status to indicate if the operation succeeded. On success, +// *table_properties will point to a heap-allocated TableProperties +// object, otherwise value of `table_properties` will not be modified. +Status ReadTablePropertiesHelper( + const ReadOptions& ro, const BlockHandle& handle, + RandomAccessFileReader* file, FilePrefetchBuffer* prefetch_buffer, + const Footer& footer, const ImmutableOptions& ioptions, + std::unique_ptr* table_properties, + MemoryAllocator* memory_allocator = nullptr); + +// Read table properties from the properties block of a plain table. +// @returns a status to indicate if the operation succeeded. On success, +// *table_properties will point to a heap-allocated TableProperties +// object, otherwise value of `table_properties` will not be modified. +Status ReadTableProperties(RandomAccessFileReader* file, uint64_t file_size, + uint64_t table_magic_number, + const ImmutableOptions& ioptions, + const ReadOptions& read_options, + std::unique_ptr* properties, + MemoryAllocator* memory_allocator = nullptr, + FilePrefetchBuffer* prefetch_buffer = nullptr); + +// Find the meta block from the meta index block. Returns OK and +// block_handle->IsNull() if not found. +Status FindOptionalMetaBlock(InternalIterator* meta_index_iter, + const std::string& meta_block_name, + BlockHandle* block_handle); + +// Find the meta block from the meta index block. Returns Corruption if not +// found. +Status FindMetaBlock(InternalIterator* meta_index_iter, + const std::string& meta_block_name, + BlockHandle* block_handle); + +// Find the meta block +Status FindMetaBlockInFile(RandomAccessFileReader* file, uint64_t file_size, + uint64_t table_magic_number, + const ImmutableOptions& ioptions, + const ReadOptions& read_options, + const std::string& meta_block_name, + BlockHandle* block_handle, + MemoryAllocator* memory_allocator = nullptr, + FilePrefetchBuffer* prefetch_buffer = nullptr, + Footer* footer_out = nullptr); + +// Read meta block contents +Status ReadMetaIndexBlockInFile(RandomAccessFileReader* file, + uint64_t file_size, uint64_t table_magic_number, + const ImmutableOptions& ioptions, + const ReadOptions& read_options, + BlockContents* block_contents, + MemoryAllocator* memory_allocator = nullptr, + FilePrefetchBuffer* prefetch_buffer = nullptr, + Footer* footer_out = nullptr); + +// Read the specified meta block with name meta_block_name +// from `file` and initialize `contents` with contents of this block. +// Return Status::OK in case of success. +Status ReadMetaBlock(RandomAccessFileReader* file, + FilePrefetchBuffer* prefetch_buffer, uint64_t file_size, + uint64_t table_magic_number, + const ImmutableOptions& ioptions, + const ReadOptions& read_options, + const std::string& meta_block_name, BlockType block_type, + BlockContents* contents, + MemoryAllocator* memory_allocator = nullptr); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/mock_table.cc b/librocksdb-sys/rocksdb/table/mock_table.cc new file mode 100644 index 0000000..d6229ef --- /dev/null +++ b/librocksdb-sys/rocksdb/table/mock_table.cc @@ -0,0 +1,352 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "table/mock_table.h" + +#include "db/dbformat.h" +#include "env/composite_env_wrapper.h" +#include "file/random_access_file_reader.h" +#include "port/port.h" +#include "rocksdb/table_properties.h" +#include "table/get_context.h" +#include "util/coding.h" + +namespace ROCKSDB_NAMESPACE { +namespace mock { + +KVVector MakeMockFile(std::initializer_list l) { return KVVector(l); } + +void SortKVVector(KVVector* kv_vector, const Comparator* ucmp) { + InternalKeyComparator icmp(ucmp); + std::sort(kv_vector->begin(), kv_vector->end(), + [icmp](KVPair a, KVPair b) -> bool { + return icmp.Compare(a.first, b.first) < 0; + }); +} + +class MockTableReader : public TableReader { + public: + explicit MockTableReader(const KVVector& table) : table_(table) {} + + InternalIterator* NewIterator(const ReadOptions&, + const SliceTransform* prefix_extractor, + Arena* arena, bool skip_filters, + TableReaderCaller caller, + size_t compaction_readahead_size = 0, + bool allow_unprepared_value = false) override; + + Status Get(const ReadOptions& readOptions, const Slice& key, + GetContext* get_context, const SliceTransform* prefix_extractor, + bool skip_filters = false) override; + + uint64_t ApproximateOffsetOf(const ReadOptions& /*read_options*/, + const Slice& /*key*/, + TableReaderCaller /*caller*/) override { + return 0; + } + + uint64_t ApproximateSize(const ReadOptions& /*read_options*/, + const Slice& /*start*/, const Slice& /*end*/, + TableReaderCaller /*caller*/) override { + return 0; + } + + size_t ApproximateMemoryUsage() const override { return 0; } + + void SetupForCompaction() override {} + + std::shared_ptr GetTableProperties() const override; + + ~MockTableReader() {} + + private: + const KVVector& table_; +}; + +class MockTableIterator : public InternalIterator { + public: + explicit MockTableIterator(const KVVector& table) : table_(table) { + itr_ = table_.end(); + } + + bool Valid() const override { return itr_ != table_.end(); } + + void SeekToFirst() override { itr_ = table_.begin(); } + + void SeekToLast() override { + itr_ = table_.end(); + --itr_; + } + + void Seek(const Slice& target) override { + KVPair target_pair(target.ToString(), ""); + InternalKeyComparator icmp(BytewiseComparator()); + itr_ = std::lower_bound(table_.begin(), table_.end(), target_pair, + [icmp](KVPair a, KVPair b) -> bool { + return icmp.Compare(a.first, b.first) < 0; + }); + } + + void SeekForPrev(const Slice& target) override { + KVPair target_pair(target.ToString(), ""); + InternalKeyComparator icmp(BytewiseComparator()); + itr_ = std::upper_bound(table_.begin(), table_.end(), target_pair, + [icmp](KVPair a, KVPair b) -> bool { + return icmp.Compare(a.first, b.first) < 0; + }); + Prev(); + } + + void Next() override { ++itr_; } + + void Prev() override { + if (itr_ == table_.begin()) { + itr_ = table_.end(); + } else { + --itr_; + } + } + + Slice key() const override { return Slice(itr_->first); } + + Slice value() const override { return Slice(itr_->second); } + + Status status() const override { return Status::OK(); } + + private: + const KVVector& table_; + KVVector::const_iterator itr_; +}; + +class MockTableBuilder : public TableBuilder { + public: + MockTableBuilder(uint32_t id, MockTableFileSystem* file_system, + MockTableFactory::MockCorruptionMode corrupt_mode = + MockTableFactory::kCorruptNone, + size_t key_value_size = 1) + : id_(id), + file_system_(file_system), + corrupt_mode_(corrupt_mode), + key_value_size_(key_value_size) { + table_ = MakeMockFile({}); + } + + // REQUIRES: Either Finish() or Abandon() has been called. + ~MockTableBuilder() {} + + // Add key,value to the table being constructed. + // REQUIRES: key is after any previously added key according to comparator. + // REQUIRES: Finish(), Abandon() have not been called + void Add(const Slice& key, const Slice& value) override { + if (corrupt_mode_ == MockTableFactory::kCorruptValue) { + // Corrupt the value + table_.push_back({key.ToString(), value.ToString() + " "}); + corrupt_mode_ = MockTableFactory::kCorruptNone; + } else if (corrupt_mode_ == MockTableFactory::kCorruptKey) { + table_.push_back({key.ToString() + " ", value.ToString()}); + corrupt_mode_ = MockTableFactory::kCorruptNone; + } else if (corrupt_mode_ == MockTableFactory::kCorruptReorderKey) { + if (prev_key_.empty()) { + prev_key_ = key.ToString(); + prev_value_ = value.ToString(); + } else { + table_.push_back({key.ToString(), value.ToString()}); + table_.push_back({prev_key_, prev_value_}); + corrupt_mode_ = MockTableFactory::kCorruptNone; + } + } else { + table_.push_back({key.ToString(), value.ToString()}); + } + } + + // Return non-ok iff some error has been detected. + Status status() const override { return Status::OK(); } + + // Return non-ok iff some error happens during IO. + IOStatus io_status() const override { return IOStatus::OK(); } + + Status Finish() override { + MutexLock lock_guard(&file_system_->mutex); + file_system_->files.insert({id_, table_}); + return Status::OK(); + } + + void Abandon() override {} + + uint64_t NumEntries() const override { return table_.size(); } + + uint64_t FileSize() const override { return table_.size() * key_value_size_; } + + TableProperties GetTableProperties() const override { + return TableProperties(); + } + + // Get file checksum + std::string GetFileChecksum() const override { return kUnknownFileChecksum; } + // Get file checksum function name + const char* GetFileChecksumFuncName() const override { + return kUnknownFileChecksumFuncName; + } + + private: + uint32_t id_; + std::string prev_key_; + std::string prev_value_; + MockTableFileSystem* file_system_; + int corrupt_mode_; + KVVector table_; + size_t key_value_size_; +}; + +InternalIterator* MockTableReader::NewIterator( + const ReadOptions&, const SliceTransform* /* prefix_extractor */, + Arena* /*arena*/, bool /*skip_filters*/, TableReaderCaller /*caller*/, + size_t /*compaction_readahead_size*/, bool /* allow_unprepared_value */) { + return new MockTableIterator(table_); +} + +Status MockTableReader::Get(const ReadOptions&, const Slice& key, + GetContext* get_context, + const SliceTransform* /*prefix_extractor*/, + bool /*skip_filters*/) { + std::unique_ptr iter(new MockTableIterator(table_)); + for (iter->Seek(key); iter->Valid(); iter->Next()) { + ParsedInternalKey parsed_key; + Status pik_status = + ParseInternalKey(iter->key(), &parsed_key, true /* log_err_key */); + if (!pik_status.ok()) { + return pik_status; + } + + bool dont_care __attribute__((__unused__)); + if (!get_context->SaveValue(parsed_key, iter->value(), &dont_care)) { + break; + } + } + return Status::OK(); +} + +std::shared_ptr MockTableReader::GetTableProperties() + const { + TableProperties* tp = new TableProperties(); + tp->num_entries = table_.size(); + tp->num_range_deletions = 0; + tp->raw_key_size = 1; + tp->raw_value_size = 1; + + return std::shared_ptr(tp); +} + +MockTableFactory::MockTableFactory() + : next_id_(1), corrupt_mode_(MockTableFactory::kCorruptNone) {} + +Status MockTableFactory::NewTableReader( + const ReadOptions& /*ro*/, + const TableReaderOptions& /*table_reader_options*/, + std::unique_ptr&& file, uint64_t /*file_size*/, + std::unique_ptr* table_reader, + bool /*prefetch_index_and_filter_in_cache*/) const { + uint32_t id; + Status s = GetIDFromFile(file.get(), &id); + if (!s.ok()) { + return s; + } + + MutexLock lock_guard(&file_system_.mutex); + + auto it = file_system_.files.find(id); + if (it == file_system_.files.end()) { + return Status::IOError("Mock file not found"); + } + + table_reader->reset(new MockTableReader(it->second)); + + return Status::OK(); +} + +TableBuilder* MockTableFactory::NewTableBuilder( + const TableBuilderOptions& /*table_builder_options*/, + WritableFileWriter* file) const { + uint32_t id; + Status s = GetAndWriteNextID(file, &id); + assert(s.ok()); + + return new MockTableBuilder(id, &file_system_, corrupt_mode_, + key_value_size_); +} + +Status MockTableFactory::CreateMockTable(Env* env, const std::string& fname, + KVVector file_contents) { + std::unique_ptr file_writer; + Status s = WritableFileWriter::Create(env->GetFileSystem(), fname, + FileOptions(), &file_writer, nullptr); + if (!s.ok()) { + return s; + } + uint32_t id; + s = GetAndWriteNextID(file_writer.get(), &id); + if (s.ok()) { + file_system_.files.insert({id, std::move(file_contents)}); + } + return s; +} + +Status MockTableFactory::GetAndWriteNextID(WritableFileWriter* file, + uint32_t* next_id) const { + *next_id = next_id_.fetch_add(1); + char buf[4]; + EncodeFixed32(buf, *next_id); + return file->Append(Slice(buf, 4)); +} + +Status MockTableFactory::GetIDFromFile(RandomAccessFileReader* file, + uint32_t* id) const { + char buf[4]; + Slice result; + Status s = file->Read(IOOptions(), 0, 4, &result, buf, nullptr, + Env::IO_TOTAL /* rate_limiter_priority */); + assert(result.size() == 4); + *id = DecodeFixed32(buf); + return s; +} + +void MockTableFactory::AssertSingleFile(const KVVector& file_contents) { + ASSERT_EQ(file_system_.files.size(), 1U); + ASSERT_EQ(file_contents, file_system_.files.begin()->second); +} + +void MockTableFactory::AssertLatestFiles( + const std::vector& files_contents) { + ASSERT_GE(file_system_.files.size(), files_contents.size()); + auto it = file_system_.files.rbegin(); + for (auto expect = files_contents.rbegin(); expect != files_contents.rend(); + expect++, it++) { + ASSERT_TRUE(it != file_system_.files.rend()); + if (*expect != it->second) { + std::cout << "Wrong content! Content of file, expect:" << std::endl; + for (const auto& kv : *expect) { + ParsedInternalKey ikey; + std::string key, value; + std::tie(key, value) = kv; + ASSERT_OK(ParseInternalKey(Slice(key), &ikey, true /* log_err_key */)); + std::cout << ikey.DebugString(true, false) << " -> " << value + << std::endl; + } + std::cout << "actual:" << std::endl; + for (const auto& kv : it->second) { + ParsedInternalKey ikey; + std::string key, value; + std::tie(key, value) = kv; + ASSERT_OK(ParseInternalKey(Slice(key), &ikey, true /* log_err_key */)); + std::cout << ikey.DebugString(true, false) << " -> " << value + << std::endl; + } + FAIL(); + } + } +} + +} // namespace mock +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/mock_table.h b/librocksdb-sys/rocksdb/table/mock_table.h new file mode 100644 index 0000000..e4850d0 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/mock_table.h @@ -0,0 +1,94 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "db/version_edit.h" +#include "port/port.h" +#include "rocksdb/comparator.h" +#include "rocksdb/io_status.h" +#include "rocksdb/table.h" +#include "table/internal_iterator.h" +#include "table/table_builder.h" +#include "table/table_reader.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/kv_map.h" +#include "util/mutexlock.h" + +namespace ROCKSDB_NAMESPACE { +namespace mock { +using KVPair = std::pair; +using KVVector = std::vector; + +KVVector MakeMockFile(std::initializer_list l = {}); +void SortKVVector(KVVector* kv_vector, + const Comparator* ucmp = BytewiseComparator()); + +struct MockTableFileSystem { + port::Mutex mutex; + std::map files; +}; + +class MockTableFactory : public TableFactory { + public: + enum MockCorruptionMode { + kCorruptNone, + kCorruptKey, + kCorruptValue, + kCorruptReorderKey, + }; + + MockTableFactory(); + static const char* kClassName() { return "MockTable"; } + const char* Name() const override { return kClassName(); } + using TableFactory::NewTableReader; + Status NewTableReader( + const ReadOptions& ro, const TableReaderOptions& table_reader_options, + std::unique_ptr&& file, uint64_t file_size, + std::unique_ptr* table_reader, + bool prefetch_index_and_filter_in_cache = true) const override; + TableBuilder* NewTableBuilder( + const TableBuilderOptions& table_builder_options, + WritableFileWriter* file) const override; + + // This function will directly create mock table instead of going through + // MockTableBuilder. file_contents has to have a format of . Those key-value pairs will then be inserted into the mock table. + Status CreateMockTable(Env* env, const std::string& fname, + KVVector file_contents); + + virtual std::string GetPrintableOptions() const override { + return std::string(); + } + + void SetCorruptionMode(MockCorruptionMode mode) { corrupt_mode_ = mode; } + + void SetKeyValueSize(size_t size) { key_value_size_ = size; } + // This function will assert that only a single file exists and that the + // contents are equal to file_contents + void AssertSingleFile(const KVVector& file_contents); + void AssertLatestFiles(const std::vector& files_contents); + + private: + Status GetAndWriteNextID(WritableFileWriter* file, uint32_t* id) const; + Status GetIDFromFile(RandomAccessFileReader* file, uint32_t* id) const; + + mutable MockTableFileSystem file_system_; + mutable std::atomic next_id_; + MockCorruptionMode corrupt_mode_; + + size_t key_value_size_ = 1; +}; + +} // namespace mock +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/multiget_context.h b/librocksdb-sys/rocksdb/table/multiget_context.h new file mode 100644 index 0000000..54af226 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/multiget_context.h @@ -0,0 +1,405 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once +#include +#include +#include + +#include "db/dbformat.h" +#include "db/lookup_key.h" +#include "db/merge_context.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "rocksdb/statistics.h" +#include "rocksdb/types.h" +#include "util/async_file_reader.h" +#include "util/autovector.h" +#include "util/math.h" +#include "util/single_thread_executor.h" + +namespace ROCKSDB_NAMESPACE { +class GetContext; +class PinnableWideColumns; + +struct KeyContext { + const Slice* key; + LookupKey* lkey; + Slice ukey_with_ts; + Slice ukey_without_ts; + Slice ikey; + ColumnFamilyHandle* column_family; + Status* s; + MergeContext merge_context; + SequenceNumber max_covering_tombstone_seq; + bool key_exists; + bool is_blob_index; + void* cb_arg; + PinnableSlice* value; + PinnableWideColumns* columns; + std::string* timestamp; + GetContext* get_context; + + KeyContext(ColumnFamilyHandle* col_family, const Slice& user_key, + PinnableSlice* val, PinnableWideColumns* cols, std::string* ts, + Status* stat) + : key(&user_key), + lkey(nullptr), + column_family(col_family), + s(stat), + max_covering_tombstone_seq(0), + key_exists(false), + is_blob_index(false), + cb_arg(nullptr), + value(val), + columns(cols), + timestamp(ts), + get_context(nullptr) {} +}; + +// The MultiGetContext class is a container for the sorted list of keys that +// we need to lookup in a batch. Its main purpose is to make batch execution +// easier by allowing various stages of the MultiGet lookups to operate on +// subsets of keys, potentially non-contiguous. In order to accomplish this, +// it defines the following classes - +// +// MultiGetContext::Range +// MultiGetContext::Range::Iterator +// MultiGetContext::Range::IteratorWrapper +// +// Here is an example of how this can be used - +// +// { +// MultiGetContext ctx(...); +// MultiGetContext::Range range = ctx.GetMultiGetRange(); +// +// // Iterate to determine some subset of the keys +// MultiGetContext::Range::Iterator start = range.begin(); +// MultiGetContext::Range::Iterator end = ...; +// +// // Make a new range with a subset of keys +// MultiGetContext::Range subrange(range, start, end); +// +// // Define an auxillary vector, if needed, to hold additional data for +// // each key +// std::array aux; +// +// // Iterate over the subrange and the auxillary vector simultaneously +// MultiGetContext::Range::Iterator iter = subrange.begin(); +// for (; iter != subrange.end(); ++iter) { +// KeyContext& key = *iter; +// Foo& aux_key = aux_iter[iter.index()]; +// ... +// } +// } +class MultiGetContext { + public: + // Limit the number of keys in a batch to this number. Benchmarks show that + // there is negligible benefit for batches exceeding this. Keeping this < 32 + // simplifies iteration, as well as reduces the amount of stack allocations + // that need to be performed + static const int MAX_BATCH_SIZE = 32; + + // A bitmask of at least MAX_BATCH_SIZE - 1 bits, so that + // Mask{1} << MAX_BATCH_SIZE is well defined + using Mask = uint64_t; + static_assert(MAX_BATCH_SIZE < sizeof(Mask) * 8); + + MultiGetContext(autovector* sorted_keys, + size_t begin, size_t num_keys, SequenceNumber snapshot, + const ReadOptions& read_opts, FileSystem* fs, + Statistics* stats) + : num_keys_(num_keys), + value_mask_(0), + value_size_(0), + lookup_key_ptr_(reinterpret_cast(lookup_key_stack_buf)) +#if USE_COROUTINES + , + reader_(fs, stats), + executor_(reader_) +#endif // USE_COROUTINES + { + (void)fs; + (void)stats; + assert(num_keys <= MAX_BATCH_SIZE); + if (num_keys > MAX_LOOKUP_KEYS_ON_STACK) { + lookup_key_heap_buf.reset(new char[sizeof(LookupKey) * num_keys]); + lookup_key_ptr_ = reinterpret_cast(lookup_key_heap_buf.get()); + } + + for (size_t iter = 0; iter != num_keys_; ++iter) { + // autovector may not be contiguous storage, so make a copy + sorted_keys_[iter] = (*sorted_keys)[begin + iter]; + sorted_keys_[iter]->lkey = new (&lookup_key_ptr_[iter]) + LookupKey(*sorted_keys_[iter]->key, snapshot, read_opts.timestamp); + sorted_keys_[iter]->ukey_with_ts = sorted_keys_[iter]->lkey->user_key(); + sorted_keys_[iter]->ukey_without_ts = StripTimestampFromUserKey( + sorted_keys_[iter]->lkey->user_key(), + read_opts.timestamp == nullptr ? 0 : read_opts.timestamp->size()); + sorted_keys_[iter]->ikey = sorted_keys_[iter]->lkey->internal_key(); + sorted_keys_[iter]->timestamp = (*sorted_keys)[begin + iter]->timestamp; + sorted_keys_[iter]->get_context = + (*sorted_keys)[begin + iter]->get_context; + } + } + + ~MultiGetContext() { + for (size_t i = 0; i < num_keys_; ++i) { + lookup_key_ptr_[i].~LookupKey(); + } + } + +#if USE_COROUTINES + SingleThreadExecutor& executor() { return executor_; } + + AsyncFileReader& reader() { return reader_; } +#endif // USE_COROUTINES + + private: + static const int MAX_LOOKUP_KEYS_ON_STACK = 16; + alignas( + alignof(LookupKey)) char lookup_key_stack_buf[sizeof(LookupKey) * + MAX_LOOKUP_KEYS_ON_STACK]; + std::array sorted_keys_; + size_t num_keys_; + Mask value_mask_; + uint64_t value_size_; + std::unique_ptr lookup_key_heap_buf; + LookupKey* lookup_key_ptr_; +#if USE_COROUTINES + AsyncFileReader reader_; + SingleThreadExecutor executor_; +#endif // USE_COROUTINES + + public: + // MultiGetContext::Range - Specifies a range of keys, by start and end index, + // from the parent MultiGetContext. Each range contains a bit vector that + // indicates whether the corresponding keys need to be processed or skipped. + // A Range object can be copy constructed, and the new object inherits the + // original Range's bit vector. This is useful for progressively skipping + // keys as the lookup goes through various stages. For example, when looking + // up keys in the same SST file, a Range is created excluding keys not + // belonging to that file. A new Range is then copy constructed and individual + // keys are skipped based on bloom filter lookup. + class Range { + public: + // MultiGetContext::Range::Iterator - A forward iterator that iterates over + // non-skippable keys in a Range, as well as keys whose final value has been + // found. The latter is tracked by MultiGetContext::value_mask_ + class Iterator { + public: + // -- iterator traits + using self_type = Iterator; + using value_type = KeyContext; + using reference = KeyContext&; + using pointer = KeyContext*; + using difference_type = int; + using iterator_category = std::forward_iterator_tag; + + Iterator(const Range* range, size_t idx) + : range_(range), ctx_(range->ctx_), index_(idx) { + while (index_ < range_->end_ && + (Mask{1} << index_) & + (range_->ctx_->value_mask_ | range_->skip_mask_ | + range_->invalid_mask_)) + index_++; + } + + Iterator(const Iterator&) = default; + + Iterator(const Iterator& other, const Range* range) + : range_(range), ctx_(other.ctx_), index_(other.index_) { + assert(range->ctx_ == other.ctx_); + } + Iterator& operator=(const Iterator&) = default; + + Iterator& operator++() { + while (++index_ < range_->end_ && + (Mask{1} << index_) & + (range_->ctx_->value_mask_ | range_->skip_mask_ | + range_->invalid_mask_)) + ; + return *this; + } + + bool operator==(Iterator other) const { + assert(range_->ctx_ == other.range_->ctx_); + return index_ == other.index_; + } + + bool operator!=(Iterator other) const { + assert(range_->ctx_ == other.range_->ctx_); + return index_ != other.index_; + } + + KeyContext& operator*() { + assert(index_ < range_->end_ && index_ >= range_->start_); + return *(ctx_->sorted_keys_[index_]); + } + + KeyContext* operator->() { + assert(index_ < range_->end_ && index_ >= range_->start_); + return ctx_->sorted_keys_[index_]; + } + + size_t index() { return index_; } + + private: + friend Range; + const Range* range_; + const MultiGetContext* ctx_; + size_t index_; + }; + + Range(const Range& mget_range, const Iterator& first, + const Iterator& last) { + ctx_ = mget_range.ctx_; + if (first == last) { + // This means create an empty range based on mget_range. So just + // set start_ and and_ to the same value + start_ = mget_range.start_; + end_ = start_; + } else { + start_ = first.index_; + end_ = last.index_; + } + skip_mask_ = mget_range.skip_mask_; + invalid_mask_ = mget_range.invalid_mask_; + assert(start_ < 64); + assert(end_ < 64); + } + + Range() = default; + + Iterator begin() const { return Iterator(this, start_); } + + Iterator end() const { return Iterator(this, end_); } + + bool empty() const { return RemainingMask() == 0; } + + void SkipIndex(size_t index) { skip_mask_ |= Mask{1} << index; } + + void SkipKey(const Iterator& iter) { SkipIndex(iter.index_); } + + bool IsKeySkipped(const Iterator& iter) const { + return skip_mask_ & (Mask{1} << iter.index_); + } + + // Update the value_mask_ in MultiGetContext so its + // immediately reflected in all the Range Iterators + void MarkKeyDone(Iterator& iter) { + ctx_->value_mask_ |= (Mask{1} << iter.index_); + } + + bool CheckKeyDone(Iterator& iter) const { + return ctx_->value_mask_ & (Mask{1} << iter.index_); + } + + uint64_t KeysLeft() const { return BitsSetToOne(RemainingMask()); } + + void AddSkipsFrom(const Range& other) { + assert(ctx_ == other.ctx_); + skip_mask_ |= other.skip_mask_; + } + + uint64_t GetValueSize() { return ctx_->value_size_; } + + void AddValueSize(uint64_t value_size) { ctx_->value_size_ += value_size; } + + MultiGetContext* context() const { return ctx_; } + + Range Suffix(const Range& other) const { + size_t other_last = other.FindLastRemaining(); + size_t my_last = FindLastRemaining(); + + if (my_last > other_last) { + return Range(*this, Iterator(this, other_last), + Iterator(this, my_last)); + } else { + return Range(*this, begin(), begin()); + } + } + + // The += operator expands the number of keys in this range. The expansion + // is always to the right, i.e start of the additional range >= end of + // current range. There should be no overlap. Any skipped keys in rhs are + // marked as invalid in the invalid_mask_. + Range& operator+=(const Range& rhs) { + assert(rhs.start_ >= end_); + // Check for non-overlapping ranges and adjust invalid_mask_ accordingly + if (end_ < rhs.start_) { + invalid_mask_ |= RangeMask(end_, rhs.start_); + skip_mask_ |= RangeMask(end_, rhs.start_); + } + start_ = std::min(start_, rhs.start_); + end_ = std::max(end_, rhs.end_); + skip_mask_ |= rhs.skip_mask_ & RangeMask(rhs.start_, rhs.end_); + invalid_mask_ |= (rhs.invalid_mask_ | rhs.skip_mask_) & + RangeMask(rhs.start_, rhs.end_); + assert(start_ < 64); + assert(end_ < 64); + return *this; + } + + // The -= operator removes keys from this range. The removed keys should + // come from a range completely overlapping the current range. The removed + // keys are marked invalid in the invalid_mask_. + Range& operator-=(const Range& rhs) { + assert(start_ <= rhs.start_ && end_ >= rhs.end_); + skip_mask_ |= (~rhs.skip_mask_ | rhs.invalid_mask_) & + RangeMask(rhs.start_, rhs.end_); + invalid_mask_ |= (~rhs.skip_mask_ | rhs.invalid_mask_) & + RangeMask(rhs.start_, rhs.end_); + return *this; + } + + // Return a complement of the current range + Range operator~() { + Range res = *this; + res.skip_mask_ = ~skip_mask_ & RangeMask(start_, end_); + return res; + } + + private: + friend MultiGetContext; + MultiGetContext* ctx_; + size_t start_; + size_t end_; + Mask skip_mask_; + Mask invalid_mask_; + + Range(MultiGetContext* ctx, size_t num_keys) + : ctx_(ctx), + start_(0), + end_(num_keys), + skip_mask_(0), + invalid_mask_(0) { + assert(num_keys < 64); + } + + static Mask RangeMask(size_t start, size_t end) { + return (((Mask{1} << (end - start)) - 1) << start); + } + + Mask RemainingMask() const { + return (((Mask{1} << end_) - 1) & ~((Mask{1} << start_) - 1) & + ~(ctx_->value_mask_ | skip_mask_)); + } + + size_t FindLastRemaining() const { + Mask mask = RemainingMask(); + size_t index = (mask >>= start_) ? start_ : 0; + while (mask >>= 1) { + index++; + } + return index; + } + }; + + // Return the initial range that encompasses all the keys in the batch + Range GetMultiGetRange() { return Range(this, num_keys_); } +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/persistent_cache_helper.cc b/librocksdb-sys/rocksdb/table/persistent_cache_helper.cc new file mode 100644 index 0000000..eece810 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/persistent_cache_helper.cc @@ -0,0 +1,111 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "table/persistent_cache_helper.h" + +#include "table/block_based/block_based_table_reader.h" +#include "table/format.h" + +namespace ROCKSDB_NAMESPACE { + +const PersistentCacheOptions PersistentCacheOptions::kEmpty; + +void PersistentCacheHelper::InsertSerialized( + const PersistentCacheOptions& cache_options, const BlockHandle& handle, + const char* data, const size_t size) { + assert(cache_options.persistent_cache); + assert(cache_options.persistent_cache->IsCompressed()); + + CacheKey key = + BlockBasedTable::GetCacheKey(cache_options.base_cache_key, handle); + + cache_options.persistent_cache->Insert(key.AsSlice(), data, size) + .PermitUncheckedError(); +} + +void PersistentCacheHelper::InsertUncompressed( + const PersistentCacheOptions& cache_options, const BlockHandle& handle, + const BlockContents& contents) { + assert(cache_options.persistent_cache); + assert(!cache_options.persistent_cache->IsCompressed()); + // Precondition: + // (1) content is cacheable + // (2) content is not compressed + + CacheKey key = + BlockBasedTable::GetCacheKey(cache_options.base_cache_key, handle); + + cache_options.persistent_cache + ->Insert(key.AsSlice(), contents.data.data(), contents.data.size()) + .PermitUncheckedError(); + ; +} + +Status PersistentCacheHelper::LookupSerialized( + const PersistentCacheOptions& cache_options, const BlockHandle& handle, + std::unique_ptr* out_data, const size_t expected_data_size) { +#ifdef NDEBUG + (void)expected_data_size; +#endif + assert(cache_options.persistent_cache); + assert(cache_options.persistent_cache->IsCompressed()); + + CacheKey key = + BlockBasedTable::GetCacheKey(cache_options.base_cache_key, handle); + + size_t size; + Status s = + cache_options.persistent_cache->Lookup(key.AsSlice(), out_data, &size); + if (!s.ok()) { + // cache miss + RecordTick(cache_options.statistics, PERSISTENT_CACHE_MISS); + return s; + } + + // cache hit + // Block-based table is assumed + assert(expected_data_size == + handle.size() + BlockBasedTable::kBlockTrailerSize); + assert(size == expected_data_size); + RecordTick(cache_options.statistics, PERSISTENT_CACHE_HIT); + return Status::OK(); +} + +Status PersistentCacheHelper::LookupUncompressed( + const PersistentCacheOptions& cache_options, const BlockHandle& handle, + BlockContents* contents) { + assert(cache_options.persistent_cache); + assert(!cache_options.persistent_cache->IsCompressed()); + if (!contents) { + // We shouldn't lookup in the cache. Either + // (1) Nowhere to store + return Status::NotFound(); + } + + CacheKey key = + BlockBasedTable::GetCacheKey(cache_options.base_cache_key, handle); + + std::unique_ptr data; + size_t size; + Status s = + cache_options.persistent_cache->Lookup(key.AsSlice(), &data, &size); + if (!s.ok()) { + // cache miss + RecordTick(cache_options.statistics, PERSISTENT_CACHE_MISS); + return s; + } + + // please note we are potentially comparing compressed data size with + // uncompressed data size + assert(handle.size() <= size); + + // update stats + RecordTick(cache_options.statistics, PERSISTENT_CACHE_HIT); + // construct result and return + *contents = BlockContents(std::move(data), size); + return Status::OK(); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/persistent_cache_helper.h b/librocksdb-sys/rocksdb/table/persistent_cache_helper.h new file mode 100644 index 0000000..cce4092 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/persistent_cache_helper.h @@ -0,0 +1,46 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include + +#include "monitoring/statistics_impl.h" +#include "table/format.h" +#include "table/persistent_cache_options.h" + +namespace ROCKSDB_NAMESPACE { + +struct BlockContents; + +// PersistentCacheHelper +// +// Encapsulates some of the helper logic for read and writing from the cache +class PersistentCacheHelper { + public: + // Insert block into cache of serialized blocks. Size includes block trailer + // (if applicable). + static void InsertSerialized(const PersistentCacheOptions& cache_options, + const BlockHandle& handle, const char* data, + const size_t size); + + // Insert block into cache of uncompressed blocks. No block trailer. + static void InsertUncompressed(const PersistentCacheOptions& cache_options, + const BlockHandle& handle, + const BlockContents& contents); + + // Lookup block from cache of serialized blocks. Size includes block trailer + // (if applicable). + static Status LookupSerialized(const PersistentCacheOptions& cache_options, + const BlockHandle& handle, + std::unique_ptr* out_data, + const size_t expected_data_size); + + // Lookup block from uncompressed cache. No block trailer. + static Status LookupUncompressed(const PersistentCacheOptions& cache_options, + const BlockHandle& handle, + BlockContents* contents); +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/persistent_cache_options.h b/librocksdb-sys/rocksdb/table/persistent_cache_options.h new file mode 100644 index 0000000..46f2687 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/persistent_cache_options.h @@ -0,0 +1,34 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include + +#include "cache/cache_key.h" +#include "monitoring/statistics_impl.h" +#include "rocksdb/persistent_cache.h" + +namespace ROCKSDB_NAMESPACE { + +// PersistentCacheOptions +// +// This describe the caching behavior for page cache +// This is used to pass the context for caching and the cache handle +struct PersistentCacheOptions { + PersistentCacheOptions() {} + explicit PersistentCacheOptions( + const std::shared_ptr& _persistent_cache, + const OffsetableCacheKey& _base_cache_key, Statistics* const _statistics) + : persistent_cache(_persistent_cache), + base_cache_key(_base_cache_key), + statistics(_statistics) {} + std::shared_ptr persistent_cache; + OffsetableCacheKey base_cache_key; + Statistics* statistics = nullptr; + + static const PersistentCacheOptions kEmpty; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/plain/plain_table_bloom.cc b/librocksdb-sys/rocksdb/table/plain/plain_table_bloom.cc new file mode 100644 index 0000000..21441f6 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/plain/plain_table_bloom.cc @@ -0,0 +1,78 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "table/plain/plain_table_bloom.h" + +#include +#include + +#include "memory/allocator.h" +#include "util/dynamic_bloom.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { + +uint32_t GetTotalBitsForLocality(uint32_t total_bits) { + uint32_t num_blocks = + (total_bits + CACHE_LINE_SIZE * 8 - 1) / (CACHE_LINE_SIZE * 8); + + // Make num_blocks an odd number to make sure more bits are involved + // when determining which block. + if (num_blocks % 2 == 0) { + num_blocks++; + } + + return num_blocks * (CACHE_LINE_SIZE * 8); +} +} // namespace + +PlainTableBloomV1::PlainTableBloomV1(uint32_t num_probes) + : kTotalBits(0), kNumBlocks(0), kNumProbes(num_probes), data_(nullptr) {} + +void PlainTableBloomV1::SetRawData(char* raw_data, uint32_t total_bits, + uint32_t num_blocks) { + data_ = raw_data; + kTotalBits = total_bits; + kNumBlocks = num_blocks; +} + +void PlainTableBloomV1::SetTotalBits(Allocator* allocator, uint32_t total_bits, + uint32_t locality, + size_t huge_page_tlb_size, + Logger* logger) { + kTotalBits = (locality > 0) ? GetTotalBitsForLocality(total_bits) + : (total_bits + 7) / 8 * 8; + kNumBlocks = (locality > 0) ? (kTotalBits / (CACHE_LINE_SIZE * 8)) : 0; + + assert(kNumBlocks > 0 || kTotalBits > 0); + assert(kNumProbes > 0); + + uint32_t sz = kTotalBits / 8; + if (kNumBlocks > 0) { + sz += CACHE_LINE_SIZE - 1; + } + assert(allocator); + + char* raw = allocator->AllocateAligned(sz, huge_page_tlb_size, logger); + memset(raw, 0, sz); + auto cache_line_offset = reinterpret_cast(raw) % CACHE_LINE_SIZE; + if (kNumBlocks > 0 && cache_line_offset > 0) { + raw += CACHE_LINE_SIZE - cache_line_offset; + } + data_ = raw; +} + +void BloomBlockBuilder::AddKeysHashes( + const std::vector& keys_hashes) { + for (auto hash : keys_hashes) { + bloom_.AddHash(hash); + } +} + +Slice BloomBlockBuilder::Finish() { return bloom_.GetRawData(); } + +const std::string BloomBlockBuilder::kBloomBlock = "kBloomBlock"; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/plain/plain_table_bloom.h b/librocksdb-sys/rocksdb/table/plain/plain_table_bloom.h new file mode 100644 index 0000000..460e7ec --- /dev/null +++ b/librocksdb-sys/rocksdb/table/plain/plain_table_bloom.h @@ -0,0 +1,132 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include +#include +#include + +#include "port/port.h" +#include "rocksdb/slice.h" +#include "util/bloom_impl.h" +#include "util/hash.h" +#include "util/math.h" + +namespace ROCKSDB_NAMESPACE { +class Slice; +class Allocator; +class Logger; + +// A legacy Bloom filter implementation used by Plain Table db format, for +// schema backward compatibility. Not for use in new filter applications. +class PlainTableBloomV1 { + public: + // allocator: pass allocator to bloom filter, hence trace the usage of memory + // total_bits: fixed total bits for the bloom + // num_probes: number of hash probes for a single key + // locality: If positive, optimize for cache line locality, 0 otherwise. + // hash_func: customized hash function + // huge_page_tlb_size: if >0, try to allocate bloom bytes from huge page TLB + // within this page size. Need to reserve huge pages for + // it to be allocated, like: + // sysctl -w vm.nr_hugepages=20 + // See linux doc Documentation/vm/hugetlbpage.txt + explicit PlainTableBloomV1(uint32_t num_probes = 6); + void SetTotalBits(Allocator* allocator, uint32_t total_bits, + uint32_t locality, size_t huge_page_tlb_size, + Logger* logger); + + ~PlainTableBloomV1() {} + + // Assuming single threaded access to this function. + void AddHash(uint32_t hash); + + // Multithreaded access to this function is OK + bool MayContainHash(uint32_t hash) const; + + void Prefetch(uint32_t hash); + + uint32_t GetNumBlocks() const { return kNumBlocks; } + + Slice GetRawData() const { return Slice(data_, GetTotalBits() / 8); } + + void SetRawData(char* raw_data, uint32_t total_bits, uint32_t num_blocks = 0); + + uint32_t GetTotalBits() const { return kTotalBits; } + + bool IsInitialized() const { return kNumBlocks > 0 || kTotalBits > 0; } + + private: + uint32_t kTotalBits; + uint32_t kNumBlocks; + const uint32_t kNumProbes; + + char* data_; + + static constexpr int LOG2_CACHE_LINE_SIZE = + ConstexprFloorLog2(CACHE_LINE_SIZE); +}; + +#if defined(_MSC_VER) +#pragma warning(push) +// local variable is initialized but not referenced +#pragma warning(disable : 4189) +#endif +inline void PlainTableBloomV1::Prefetch(uint32_t h) { + if (kNumBlocks != 0) { + uint32_t ignored; + LegacyLocalityBloomImpl::PrepareHashMayMatch( + h, kNumBlocks, data_, &ignored, LOG2_CACHE_LINE_SIZE); + } +} +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + +inline bool PlainTableBloomV1::MayContainHash(uint32_t h) const { + assert(IsInitialized()); + if (kNumBlocks != 0) { + return LegacyLocalityBloomImpl::HashMayMatch( + h, kNumBlocks, kNumProbes, data_, LOG2_CACHE_LINE_SIZE); + } else { + return LegacyNoLocalityBloomImpl::HashMayMatch(h, kTotalBits, kNumProbes, + data_); + } +} + +inline void PlainTableBloomV1::AddHash(uint32_t h) { + assert(IsInitialized()); + if (kNumBlocks != 0) { + LegacyLocalityBloomImpl::AddHash(h, kNumBlocks, kNumProbes, data_, + LOG2_CACHE_LINE_SIZE); + } else { + LegacyNoLocalityBloomImpl::AddHash(h, kTotalBits, kNumProbes, data_); + } +} + +class BloomBlockBuilder { + public: + static const std::string kBloomBlock; + + explicit BloomBlockBuilder(uint32_t num_probes = 6) : bloom_(num_probes) {} + + void SetTotalBits(Allocator* allocator, uint32_t total_bits, + uint32_t locality, size_t huge_page_tlb_size, + Logger* logger) { + bloom_.SetTotalBits(allocator, total_bits, locality, huge_page_tlb_size, + logger); + } + + uint32_t GetNumBlocks() const { return bloom_.GetNumBlocks(); } + + void AddKeysHashes(const std::vector& keys_hashes); + + Slice Finish(); + + private: + PlainTableBloomV1 bloom_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/plain/plain_table_builder.cc b/librocksdb-sys/rocksdb/table/plain/plain_table_builder.cc new file mode 100644 index 0000000..ffa811c --- /dev/null +++ b/librocksdb-sys/rocksdb/table/plain/plain_table_builder.cc @@ -0,0 +1,340 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "table/plain/plain_table_builder.h" + +#include + +#include +#include +#include + +#include "db/dbformat.h" +#include "file/writable_file_writer.h" +#include "logging/logging.h" +#include "rocksdb/comparator.h" +#include "rocksdb/env.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/options.h" +#include "rocksdb/table.h" +#include "table/block_based/block_builder.h" +#include "table/format.h" +#include "table/meta_blocks.h" +#include "table/plain/plain_table_bloom.h" +#include "table/plain/plain_table_factory.h" +#include "table/plain/plain_table_index.h" +#include "util/coding.h" +#include "util/crc32c.h" +#include "util/stop_watch.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { + +// a utility that helps writing block content to the file +// @offset will advance if @block_contents was successfully written. +// @block_handle the block handle this particular block. +IOStatus WriteBlock(const Slice& block_contents, WritableFileWriter* file, + uint64_t* offset, BlockHandle* block_handle) { + block_handle->set_offset(*offset); + block_handle->set_size(block_contents.size()); + IOStatus io_s = file->Append(block_contents); + + if (io_s.ok()) { + *offset += block_contents.size(); + } + return io_s; +} + +} // namespace + +// kPlainTableMagicNumber was picked by running +// echo rocksdb.table.plain | sha1sum +// and taking the leading 64 bits. +extern const uint64_t kPlainTableMagicNumber = 0x8242229663bf9564ull; +extern const uint64_t kLegacyPlainTableMagicNumber = 0x4f3418eb7a8f13b8ull; + +PlainTableBuilder::PlainTableBuilder( + const ImmutableOptions& ioptions, const MutableCFOptions& moptions, + const IntTblPropCollectorFactories* int_tbl_prop_collector_factories, + uint32_t column_family_id, int level_at_creation, WritableFileWriter* file, + uint32_t user_key_len, EncodingType encoding_type, size_t index_sparseness, + uint32_t bloom_bits_per_key, const std::string& column_family_name, + uint32_t num_probes, size_t huge_page_tlb_size, double hash_table_ratio, + bool store_index_in_file, const std::string& db_id, + const std::string& db_session_id, uint64_t file_number) + : ioptions_(ioptions), + moptions_(moptions), + bloom_block_(num_probes), + file_(file), + bloom_bits_per_key_(bloom_bits_per_key), + huge_page_tlb_size_(huge_page_tlb_size), + encoder_(encoding_type, user_key_len, moptions.prefix_extractor.get(), + index_sparseness), + store_index_in_file_(store_index_in_file), + prefix_extractor_(moptions.prefix_extractor.get()) { + // Build index block and save it in the file if hash_table_ratio > 0 + if (store_index_in_file_) { + assert(hash_table_ratio > 0 || IsTotalOrderMode()); + index_builder_.reset(new PlainTableIndexBuilder( + &arena_, ioptions, moptions.prefix_extractor.get(), index_sparseness, + hash_table_ratio, huge_page_tlb_size_)); + properties_ + .user_collected_properties[PlainTablePropertyNames::kBloomVersion] = + "1"; // For future use + } + + properties_.fixed_key_len = user_key_len; + + // for plain table, we put all the data in a big chuck. + properties_.num_data_blocks = 1; + // Fill it later if store_index_in_file_ == true + properties_.index_size = 0; + properties_.filter_size = 0; + // To support roll-back to previous version, now still use version 0 for + // plain encoding. + properties_.format_version = (encoding_type == kPlain) ? 0 : 1; + properties_.column_family_id = column_family_id; + properties_.column_family_name = column_family_name; + properties_.db_id = db_id; + properties_.db_session_id = db_session_id; + properties_.db_host_id = ioptions.db_host_id; + if (!ReifyDbHostIdProperty(ioptions_.env, &properties_.db_host_id).ok()) { + ROCKS_LOG_INFO(ioptions_.logger, "db_host_id property will not be set"); + } + properties_.orig_file_number = file_number; + properties_.prefix_extractor_name = + moptions_.prefix_extractor != nullptr + ? moptions_.prefix_extractor->AsString() + : "nullptr"; + + std::string val; + PutFixed32(&val, static_cast(encoder_.GetEncodingType())); + properties_ + .user_collected_properties[PlainTablePropertyNames::kEncodingType] = val; + + assert(int_tbl_prop_collector_factories); + for (auto& factory : *int_tbl_prop_collector_factories) { + assert(factory); + + table_properties_collectors_.emplace_back( + factory->CreateIntTblPropCollector(column_family_id, + level_at_creation)); + } +} + +PlainTableBuilder::~PlainTableBuilder() { + // They are supposed to have been passed to users through Finish() + // if the file succeeds. + status_.PermitUncheckedError(); + io_status_.PermitUncheckedError(); +} + +void PlainTableBuilder::Add(const Slice& key, const Slice& value) { + // temp buffer for metadata bytes between key and value. + char meta_bytes_buf[6]; + size_t meta_bytes_buf_size = 0; + + ParsedInternalKey internal_key; + if (!ParseInternalKey(key, &internal_key, false /* log_err_key */) + .ok()) { // TODO + assert(false); + return; + } + if (internal_key.type == kTypeRangeDeletion) { + status_ = Status::NotSupported("Range deletion unsupported"); + return; + } + + // Store key hash + if (store_index_in_file_) { + if (moptions_.prefix_extractor == nullptr) { + keys_or_prefixes_hashes_.push_back(GetSliceHash(internal_key.user_key)); + } else { + Slice prefix = + moptions_.prefix_extractor->Transform(internal_key.user_key); + keys_or_prefixes_hashes_.push_back(GetSliceHash(prefix)); + } + } + + // Write value + assert(offset_ <= std::numeric_limits::max()); + auto prev_offset = static_cast(offset_); + // Write out the key + io_status_ = encoder_.AppendKey(key, file_, &offset_, meta_bytes_buf, + &meta_bytes_buf_size); + if (SaveIndexInFile()) { + index_builder_->AddKeyPrefix(GetPrefix(internal_key), prev_offset); + } + + // Write value length + uint32_t value_size = static_cast(value.size()); + if (io_status_.ok()) { + char* end_ptr = + EncodeVarint32(meta_bytes_buf + meta_bytes_buf_size, value_size); + assert(end_ptr <= meta_bytes_buf + sizeof(meta_bytes_buf)); + meta_bytes_buf_size = end_ptr - meta_bytes_buf; + io_status_ = file_->Append(Slice(meta_bytes_buf, meta_bytes_buf_size)); + } + + // Write value + if (io_status_.ok()) { + io_status_ = file_->Append(value); + offset_ += value_size + meta_bytes_buf_size; + } + + if (io_status_.ok()) { + properties_.num_entries++; + properties_.raw_key_size += key.size(); + properties_.raw_value_size += value.size(); + if (internal_key.type == kTypeDeletion || + internal_key.type == kTypeSingleDeletion) { + properties_.num_deletions++; + } else if (internal_key.type == kTypeMerge) { + properties_.num_merge_operands++; + } + } + + // notify property collectors + NotifyCollectTableCollectorsOnAdd( + key, value, offset_, table_properties_collectors_, ioptions_.logger); + status_ = io_status_; +} + +Status PlainTableBuilder::Finish() { + assert(!closed_); + closed_ = true; + + properties_.data_size = offset_; + + // Write the following blocks + // 1. [meta block: bloom] - optional + // 2. [meta block: index] - optional + // 3. [meta block: properties] + // 4. [metaindex block] + // 5. [footer] + + MetaIndexBuilder meta_index_builer; + + if (store_index_in_file_ && (properties_.num_entries > 0)) { + assert(properties_.num_entries <= std::numeric_limits::max()); + BlockHandle bloom_block_handle; + if (bloom_bits_per_key_ > 0) { + bloom_block_.SetTotalBits( + &arena_, + static_cast(properties_.num_entries) * bloom_bits_per_key_, + ioptions_.bloom_locality, huge_page_tlb_size_, ioptions_.logger); + + PutVarint32(&properties_.user_collected_properties + [PlainTablePropertyNames::kNumBloomBlocks], + bloom_block_.GetNumBlocks()); + + bloom_block_.AddKeysHashes(keys_or_prefixes_hashes_); + + Slice bloom_finish_result = bloom_block_.Finish(); + + properties_.filter_size = bloom_finish_result.size(); + io_status_ = + WriteBlock(bloom_finish_result, file_, &offset_, &bloom_block_handle); + + if (!io_status_.ok()) { + status_ = io_status_; + return status_; + } + meta_index_builer.Add(BloomBlockBuilder::kBloomBlock, bloom_block_handle); + } + BlockHandle index_block_handle; + Slice index_finish_result = index_builder_->Finish(); + + properties_.index_size = index_finish_result.size(); + io_status_ = + WriteBlock(index_finish_result, file_, &offset_, &index_block_handle); + + if (!io_status_.ok()) { + status_ = io_status_; + return status_; + } + + meta_index_builer.Add(PlainTableIndexBuilder::kPlainTableIndexBlock, + index_block_handle); + } + + // Calculate bloom block size and index block size + PropertyBlockBuilder property_block_builder; + // -- Add basic properties + property_block_builder.AddTableProperty(properties_); + + property_block_builder.Add(properties_.user_collected_properties); + + // -- Add user collected properties + NotifyCollectTableCollectorsOnFinish( + table_properties_collectors_, ioptions_.logger, &property_block_builder); + + // -- Write property block + BlockHandle property_block_handle; + io_status_ = WriteBlock(property_block_builder.Finish(), file_, &offset_, + &property_block_handle); + if (!io_status_.ok()) { + status_ = io_status_; + return status_; + } + meta_index_builer.Add(kPropertiesBlockName, property_block_handle); + + // -- write metaindex block + BlockHandle metaindex_block_handle; + io_status_ = WriteBlock(meta_index_builer.Finish(), file_, &offset_, + &metaindex_block_handle); + if (!io_status_.ok()) { + status_ = io_status_; + return status_; + } + + // Write Footer + // no need to write out new footer if we're using default checksum + FooterBuilder footer; + Status s = footer.Build(kPlainTableMagicNumber, /* format_version */ 0, + offset_, kNoChecksum, metaindex_block_handle); + if (!s.ok()) { + status_ = s; + return status_; + } + io_status_ = file_->Append(footer.GetSlice()); + if (io_status_.ok()) { + offset_ += footer.GetSlice().size(); + } + status_ = io_status_; + return status_; +} + +void PlainTableBuilder::Abandon() { closed_ = true; } + +uint64_t PlainTableBuilder::NumEntries() const { + return properties_.num_entries; +} + +uint64_t PlainTableBuilder::FileSize() const { return offset_; } + +std::string PlainTableBuilder::GetFileChecksum() const { + if (file_ != nullptr) { + return file_->GetFileChecksum(); + } else { + return kUnknownFileChecksum; + } +} + +const char* PlainTableBuilder::GetFileChecksumFuncName() const { + if (file_ != nullptr) { + return file_->GetFileChecksumFuncName(); + } else { + return kUnknownFileChecksumFuncName; + } +} +void PlainTableBuilder::SetSeqnoTimeTableProperties(const std::string& string, + uint64_t uint_64) { + // TODO: storing seqno to time mapping is not yet support for plain table. + TableBuilder::SetSeqnoTimeTableProperties(string, uint_64); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/plain/plain_table_builder.h b/librocksdb-sys/rocksdb/table/plain/plain_table_builder.h new file mode 100644 index 0000000..27c0789 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/plain/plain_table_builder.h @@ -0,0 +1,161 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include +#include + +#include "db/version_edit.h" +#include "rocksdb/options.h" +#include "rocksdb/status.h" +#include "rocksdb/table.h" +#include "rocksdb/table_properties.h" +#include "table/plain/plain_table_bloom.h" +#include "table/plain/plain_table_index.h" +#include "table/plain/plain_table_key_coding.h" +#include "table/table_builder.h" + +namespace ROCKSDB_NAMESPACE { + +class BlockBuilder; +class BlockHandle; +class WritableFile; +class TableBuilder; + +// The builder class of PlainTable. For description of PlainTable format +// See comments of class PlainTableFactory, where instances of +// PlainTableReader are created. +class PlainTableBuilder : public TableBuilder { + public: + // Create a builder that will store the contents of the table it is + // building in *file. Does not close the file. It is up to the + // caller to close the file after calling Finish(). The output file + // will be part of level specified by 'level'. A value of -1 means + // that the caller does not know which level the output file will reside. + PlainTableBuilder( + const ImmutableOptions& ioptions, const MutableCFOptions& moptions, + const IntTblPropCollectorFactories* int_tbl_prop_collector_factories, + uint32_t column_family_id, int level_at_creation, + WritableFileWriter* file, uint32_t user_key_size, + EncodingType encoding_type, size_t index_sparseness, + uint32_t bloom_bits_per_key, const std::string& column_family_name, + uint32_t num_probes = 6, size_t huge_page_tlb_size = 0, + double hash_table_ratio = 0, bool store_index_in_file = false, + const std::string& db_id = "", const std::string& db_session_id = "", + uint64_t file_number = 0); + // No copying allowed + PlainTableBuilder(const PlainTableBuilder&) = delete; + void operator=(const PlainTableBuilder&) = delete; + + // REQUIRES: Either Finish() or Abandon() has been called. + ~PlainTableBuilder(); + + // Add key,value to the table being constructed. + // REQUIRES: key is after any previously added key according to comparator. + // REQUIRES: Finish(), Abandon() have not been called + void Add(const Slice& key, const Slice& value) override; + + // Return non-ok iff some error has been detected. + Status status() const override { return status_; } + + // Return non-ok iff some error happens during IO. + IOStatus io_status() const override { return io_status_; } + + // Finish building the table. Stops using the file passed to the + // constructor after this function returns. + // REQUIRES: Finish(), Abandon() have not been called + Status Finish() override; + + // Indicate that the contents of this builder should be abandoned. Stops + // using the file passed to the constructor after this function returns. + // If the caller is not going to call Finish(), it must call Abandon() + // before destroying this builder. + // REQUIRES: Finish(), Abandon() have not been called + void Abandon() override; + + // Number of calls to Add() so far. + uint64_t NumEntries() const override; + + // Size of the file generated so far. If invoked after a successful + // Finish() call, returns the size of the final generated file. + uint64_t FileSize() const override; + + TableProperties GetTableProperties() const override { + TableProperties ret = properties_; + for (const auto& collector : table_properties_collectors_) { + for (const auto& prop : collector->GetReadableProperties()) { + ret.readable_properties.insert(prop); + } + collector->Finish(&ret.user_collected_properties).PermitUncheckedError(); + } + return ret; + } + + bool SaveIndexInFile() const { return store_index_in_file_; } + + // Get file checksum + std::string GetFileChecksum() const override; + + // Get file checksum function name + const char* GetFileChecksumFuncName() const override; + + void SetSeqnoTimeTableProperties(const std::string& string, + uint64_t uint_64) override; + + private: + Arena arena_; + const ImmutableOptions& ioptions_; + const MutableCFOptions& moptions_; + std::vector> + table_properties_collectors_; + + BloomBlockBuilder bloom_block_; + std::unique_ptr index_builder_; + + WritableFileWriter* file_; + uint64_t offset_ = 0; + uint32_t bloom_bits_per_key_; + size_t huge_page_tlb_size_; + Status status_; + IOStatus io_status_; + TableProperties properties_; + PlainTableKeyEncoder encoder_; + + bool store_index_in_file_; + + std::vector keys_or_prefixes_hashes_; + bool closed_ = false; // Either Finish() or Abandon() has been called. + + const SliceTransform* prefix_extractor_; + + Slice GetPrefix(const Slice& target) const { + assert(target.size() >= 8); // target is internal key + return GetPrefixFromUserKey(ExtractUserKey(target)); + } + + Slice GetPrefix(const ParsedInternalKey& target) const { + return GetPrefixFromUserKey(target.user_key); + } + + Slice GetPrefixFromUserKey(const Slice& user_key) const { + if (!IsTotalOrderMode()) { + return prefix_extractor_->Transform(user_key); + } else { + // Use empty slice as prefix if prefix_extractor is not set. + // In that case, + // it falls back to pure binary search and + // total iterator seek is supported. + return Slice(); + } + } + + bool IsTotalOrderMode() const { return (prefix_extractor_ == nullptr); } +}; + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/table/plain/plain_table_factory.cc b/librocksdb-sys/rocksdb/table/plain/plain_table_factory.cc new file mode 100644 index 0000000..80aa9cb --- /dev/null +++ b/librocksdb-sys/rocksdb/table/plain/plain_table_factory.cc @@ -0,0 +1,296 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "table/plain/plain_table_factory.h" + +#include + +#include + +#include "db/dbformat.h" +#include "port/port.h" +#include "rocksdb/convenience.h" +#include "rocksdb/utilities/customizable_util.h" +#include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/options_type.h" +#include "table/plain/plain_table_builder.h" +#include "table/plain/plain_table_reader.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { +static std::unordered_map plain_table_type_info = { + {"user_key_len", + {offsetof(struct PlainTableOptions, user_key_len), OptionType::kUInt32T, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"bloom_bits_per_key", + {offsetof(struct PlainTableOptions, bloom_bits_per_key), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"hash_table_ratio", + {offsetof(struct PlainTableOptions, hash_table_ratio), OptionType::kDouble, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"index_sparseness", + {offsetof(struct PlainTableOptions, index_sparseness), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"huge_page_tlb_size", + {offsetof(struct PlainTableOptions, huge_page_tlb_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"encoding_type", + {offsetof(struct PlainTableOptions, encoding_type), + OptionType::kEncodingType, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"full_scan_mode", + {offsetof(struct PlainTableOptions, full_scan_mode), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"store_index_in_file", + {offsetof(struct PlainTableOptions, store_index_in_file), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, +}; + +PlainTableFactory::PlainTableFactory(const PlainTableOptions& options) + : table_options_(options) { + RegisterOptions(&table_options_, &plain_table_type_info); +} + +Status PlainTableFactory::NewTableReader( + const ReadOptions& /*ro*/, const TableReaderOptions& table_reader_options, + std::unique_ptr&& file, uint64_t file_size, + std::unique_ptr* table, + bool /*prefetch_index_and_filter_in_cache*/) const { + return PlainTableReader::Open( + table_reader_options.ioptions, table_reader_options.env_options, + table_reader_options.internal_comparator, std::move(file), file_size, + table, table_options_.bloom_bits_per_key, table_options_.hash_table_ratio, + table_options_.index_sparseness, table_options_.huge_page_tlb_size, + table_options_.full_scan_mode, table_reader_options.immortal, + table_reader_options.prefix_extractor.get()); +} + +TableBuilder* PlainTableFactory::NewTableBuilder( + const TableBuilderOptions& table_builder_options, + WritableFileWriter* file) const { + // Ignore the skip_filters flag. PlainTable format is optimized for small + // in-memory dbs. The skip_filters optimization is not useful for plain + // tables + // + return new PlainTableBuilder( + table_builder_options.ioptions, table_builder_options.moptions, + table_builder_options.int_tbl_prop_collector_factories, + table_builder_options.column_family_id, + table_builder_options.level_at_creation, file, + table_options_.user_key_len, table_options_.encoding_type, + table_options_.index_sparseness, table_options_.bloom_bits_per_key, + table_builder_options.column_family_name, 6, + table_options_.huge_page_tlb_size, table_options_.hash_table_ratio, + table_options_.store_index_in_file, table_builder_options.db_id, + table_builder_options.db_session_id, table_builder_options.cur_file_num); +} + +std::string PlainTableFactory::GetPrintableOptions() const { + std::string ret; + ret.reserve(20000); + const int kBufferSize = 200; + char buffer[kBufferSize]; + + snprintf(buffer, kBufferSize, " user_key_len: %u\n", + table_options_.user_key_len); + ret.append(buffer); + snprintf(buffer, kBufferSize, " bloom_bits_per_key: %d\n", + table_options_.bloom_bits_per_key); + ret.append(buffer); + snprintf(buffer, kBufferSize, " hash_table_ratio: %lf\n", + table_options_.hash_table_ratio); + ret.append(buffer); + snprintf(buffer, kBufferSize, " index_sparseness: %" ROCKSDB_PRIszt "\n", + table_options_.index_sparseness); + ret.append(buffer); + snprintf(buffer, kBufferSize, " huge_page_tlb_size: %" ROCKSDB_PRIszt "\n", + table_options_.huge_page_tlb_size); + ret.append(buffer); + snprintf(buffer, kBufferSize, " encoding_type: %d\n", + table_options_.encoding_type); + ret.append(buffer); + snprintf(buffer, kBufferSize, " full_scan_mode: %d\n", + table_options_.full_scan_mode); + ret.append(buffer); + snprintf(buffer, kBufferSize, " store_index_in_file: %d\n", + table_options_.store_index_in_file); + ret.append(buffer); + return ret; +} + +Status GetPlainTableOptionsFromString(const ConfigOptions& config_options, + const PlainTableOptions& table_options, + const std::string& opts_str, + PlainTableOptions* new_table_options) { + std::unordered_map opts_map; + Status s = StringToMap(opts_str, &opts_map); + if (!s.ok()) { + return s; + } + + s = GetPlainTableOptionsFromMap(config_options, table_options, opts_map, + new_table_options); + // Translate any errors (NotFound, NotSupported, to InvalidArgument + if (s.ok() || s.IsInvalidArgument()) { + return s; + } else { + return Status::InvalidArgument(s.getState()); + } +} + +static int RegisterBuiltinMemTableRepFactory(ObjectLibrary& library, + const std::string& /*arg*/) { + // The MemTableRepFactory built-in classes will be either a class + // (VectorRepFactory) or a nickname (vector), followed optionally by ":#", + // where # is the "size" of the factory. + auto AsPattern = [](const std::string& name, const std::string& alt) { + auto pattern = ObjectLibrary::PatternEntry(name, true); + pattern.AnotherName(alt); + pattern.AddNumber(":"); + return pattern; + }; + library.AddFactory( + AsPattern(VectorRepFactory::kClassName(), VectorRepFactory::kNickName()), + [](const std::string& uri, std::unique_ptr* guard, + std::string* /*errmsg*/) { + auto colon = uri.find(":"); + if (colon != std::string::npos) { + size_t count = ParseSizeT(uri.substr(colon + 1)); + guard->reset(new VectorRepFactory(count)); + } else { + guard->reset(new VectorRepFactory()); + } + return guard->get(); + }); + library.AddFactory( + AsPattern(SkipListFactory::kClassName(), SkipListFactory::kNickName()), + [](const std::string& uri, std::unique_ptr* guard, + std::string* /*errmsg*/) { + auto colon = uri.find(":"); + if (colon != std::string::npos) { + size_t lookahead = ParseSizeT(uri.substr(colon + 1)); + guard->reset(new SkipListFactory(lookahead)); + } else { + guard->reset(new SkipListFactory()); + } + return guard->get(); + }); + library.AddFactory( + AsPattern("HashLinkListRepFactory", "hash_linkedlist"), + [](const std::string& uri, std::unique_ptr* guard, + std::string* /*errmsg*/) { + // Expecting format: hash_linkedlist: + auto colon = uri.find(":"); + if (colon != std::string::npos) { + size_t hash_bucket_count = ParseSizeT(uri.substr(colon + 1)); + guard->reset(NewHashLinkListRepFactory(hash_bucket_count)); + } else { + guard->reset(NewHashLinkListRepFactory()); + } + return guard->get(); + }); + library.AddFactory( + AsPattern("HashSkipListRepFactory", "prefix_hash"), + [](const std::string& uri, std::unique_ptr* guard, + std::string* /*errmsg*/) { + // Expecting format: prefix_hash: + auto colon = uri.find(":"); + if (colon != std::string::npos) { + size_t hash_bucket_count = ParseSizeT(uri.substr(colon + 1)); + guard->reset(NewHashSkipListRepFactory(hash_bucket_count)); + } else { + guard->reset(NewHashSkipListRepFactory()); + } + return guard->get(); + }); + library.AddFactory( + "cuckoo", + [](const std::string& /*uri*/, + std::unique_ptr* /*guard*/, std::string* errmsg) { + *errmsg = "cuckoo hash memtable is not supported anymore."; + return nullptr; + }); + + size_t num_types; + return static_cast(library.GetFactoryCount(&num_types)); +} + +Status GetMemTableRepFactoryFromString( + const std::string& opts_str, std::unique_ptr* result) { + ConfigOptions config_options; + config_options.ignore_unsupported_options = false; + config_options.ignore_unknown_options = false; + return MemTableRepFactory::CreateFromString(config_options, opts_str, result); +} + +Status MemTableRepFactory::CreateFromString( + const ConfigOptions& config_options, const std::string& value, + std::unique_ptr* result) { + static std::once_flag once; + std::call_once(once, [&]() { + RegisterBuiltinMemTableRepFactory(*(ObjectLibrary::Default().get()), ""); + }); + std::string id; + std::unordered_map opt_map; + Status status = Customizable::GetOptionsMap(config_options, result->get(), + value, &id, &opt_map); + if (!status.ok()) { // GetOptionsMap failed + return status; + } else if (value.empty()) { + // No Id and no options. Clear the object + result->reset(); + return Status::OK(); + } else if (id.empty()) { // We have no Id but have options. Not good + return Status::NotSupported("Cannot reset object ", id); + } else { + status = NewUniqueObject(config_options, id, opt_map, + result); + } + return status; +} + +Status MemTableRepFactory::CreateFromString( + const ConfigOptions& config_options, const std::string& value, + std::shared_ptr* result) { + std::unique_ptr factory; + Status s = CreateFromString(config_options, value, &factory); + if (factory && s.ok()) { + result->reset(factory.release()); + } + return s; +} + +Status GetPlainTableOptionsFromMap( + const ConfigOptions& config_options, const PlainTableOptions& table_options, + const std::unordered_map& opts_map, + PlainTableOptions* new_table_options) { + assert(new_table_options); + PlainTableFactory ptf(table_options); + Status s = ptf.ConfigureFromMap(config_options, opts_map); + if (s.ok()) { + *new_table_options = *(ptf.GetOptions()); + } else { + // Restore "new_options" to the default "base_options". + *new_table_options = table_options; + } + return s; +} + +extern TableFactory* NewPlainTableFactory(const PlainTableOptions& options) { + return new PlainTableFactory(options); +} + +const std::string PlainTablePropertyNames::kEncodingType = + "rocksdb.plain.table.encoding.type"; + +const std::string PlainTablePropertyNames::kBloomVersion = + "rocksdb.plain.table.bloom.version"; + +const std::string PlainTablePropertyNames::kNumBloomBlocks = + "rocksdb.plain.table.bloom.numblocks"; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/plain/plain_table_factory.h b/librocksdb-sys/rocksdb/table/plain/plain_table_factory.h new file mode 100644 index 0000000..a47418a --- /dev/null +++ b/librocksdb-sys/rocksdb/table/plain/plain_table_factory.h @@ -0,0 +1,180 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include + +#include +#include + +#include "rocksdb/table.h" + +namespace ROCKSDB_NAMESPACE { + +struct EnvOptions; + +class Status; +class RandomAccessFile; +class WritableFile; +class Table; +class TableBuilder; + +// PlainTableFactory is the entrance function to the PlainTable format of +// SST files. It returns instances PlainTableBuilder as the builder +// class and PlainTableReader as the reader class, where the format is +// actually implemented. +// +// The PlainTable is designed for memory-mapped file systems, e.g. tmpfs. +// Data is not organized in blocks, which allows fast access. Because of +// following downsides +// 1. Data compression is not supported. +// 2. Data is not checksumed. +// it is not recommended to use this format on other type of file systems. +// +// PlainTable requires fixed length key, configured as a constructor +// parameter of the factory class. Output file format: +// +-------------+-----------------+ +// | version | user_key_length | +// +------------++------------+-----------------+ <= key1 offset +// | encoded key1 | value_size | | +// +------------+-------------+-------------+ | +// | value1 | +// | | +// +--------------------------+-------------+---+ <= key2 offset +// | encoded key2 | value_size | | +// +------------+-------------+-------------+ | +// | value2 | +// | | +// | ...... | +// +-----------------+--------------------------+ +// +// When the key encoding type is kPlain. Key part is encoded as: +// +------------+--------------------+ +// | [key_size] | internal key | +// +------------+--------------------+ +// for the case of user_key_len = kPlainTableVariableLength case, +// and simply: +// +----------------------+ +// | internal key | +// +----------------------+ +// for user_key_len != kPlainTableVariableLength case. +// +// If key encoding type is kPrefix. Keys are encoding in this format. +// There are three ways to encode a key: +// (1) Full Key +// +---------------+---------------+-------------------+ +// | Full Key Flag | Full Key Size | Full Internal Key | +// +---------------+---------------+-------------------+ +// which simply encodes a full key +// +// (2) A key shared the same prefix as the previous key, which is encoded as +// format of (1). +// +-------------+-------------+-------------+-------------+------------+ +// | Prefix Flag | Prefix Size | Suffix Flag | Suffix Size | Key Suffix | +// +-------------+-------------+-------------+-------------+------------+ +// where key is the suffix part of the key, including the internal bytes. +// the actual key will be constructed by concatenating prefix part of the +// previous key, with the suffix part of the key here, with sizes given here. +// +// (3) A key shared the same prefix as the previous key, which is encoded as +// the format of (2). +// +-----------------+-----------------+------------------------+ +// | Key Suffix Flag | Key Suffix Size | Suffix of Internal Key | +// +-----------------+-----------------+------------------------+ +// The key will be constructed by concatenating previous key's prefix (which is +// also a prefix which the last key encoded in the format of (1)) and the +// key given here. +// +// For example, we for following keys (prefix and suffix are separated by +// spaces): +// 0000 0001 +// 0000 00021 +// 0000 0002 +// 00011 00 +// 0002 0001 +// Will be encoded like this: +// FK 8 00000001 +// PF 4 SF 5 00021 +// SF 4 0002 +// FK 7 0001100 +// FK 8 00020001 +// (where FK means full key flag, PF means prefix flag and SF means suffix flag) +// +// All those "key flag + key size" shown above are in this format: +// The 8 bits of the first byte: +// +----+----+----+----+----+----+----+----+ +// | Type | Size | +// +----+----+----+----+----+----+----+----+ +// Type indicates: full key, prefix, or suffix. +// The last 6 bits are for size. If the size bits are not all 1, it means the +// size of the key. Otherwise, varint32 is read after this byte. This varint +// value + 0x3F (the value of all 1) will be the key size. +// +// For example, full key with length 16 will be encoded as (binary): +// 00 010000 +// (00 means full key) +// and a prefix with 100 bytes will be encoded as: +// 01 111111 00100101 +// (63) (37) +// (01 means key suffix) +// +// All the internal keys above (including kPlain and kPrefix) are encoded in +// this format: +// There are two types: +// (1) normal internal key format +// +----------- ...... -------------+----+---+---+---+---+---+---+---+ +// | user key |type| sequence ID | +// +----------- ..... --------------+----+---+---+---+---+---+---+---+ +// (2) Special case for keys whose sequence ID is 0 and is value type +// +----------- ...... -------------+----+ +// | user key |0x80| +// +----------- ..... --------------+----+ +// To save 7 bytes for the special case where sequence ID = 0. +// +// +class PlainTableFactory : public TableFactory { + public: + ~PlainTableFactory() {} + // user_key_len is the length of the user key. If it is set to be + // kPlainTableVariableLength, then it means variable length. Otherwise, all + // the keys need to have the fix length of this value. bloom_bits_per_key is + // number of bits used for bloom filer per key. hash_table_ratio is + // the desired utilization of the hash table used for prefix hashing. + // hash_table_ratio = number of prefixes / #buckets in the hash table + // hash_table_ratio = 0 means skip hash table but only replying on binary + // search. + // index_sparseness determines index interval for keys + // inside the same prefix. It will be the maximum number of linear search + // required after hash and binary search. + // index_sparseness = 0 means index for every key. + // huge_page_tlb_size determines whether to allocate hash indexes from huge + // page TLB and the page size if allocating from there. See comments of + // Arena::AllocateAligned() for details. + explicit PlainTableFactory( + const PlainTableOptions& _table_options = PlainTableOptions()); + + // Method to allow CheckedCast to work for this class + static const char* kClassName() { return kPlainTableName(); } + const char* Name() const override { return kPlainTableName(); } + using TableFactory::NewTableReader; + Status NewTableReader(const ReadOptions& ro, + const TableReaderOptions& table_reader_options, + std::unique_ptr&& file, + uint64_t file_size, std::unique_ptr* table, + bool prefetch_index_and_filter_in_cache) const override; + + TableBuilder* NewTableBuilder( + const TableBuilderOptions& table_builder_options, + WritableFileWriter* file) const override; + + std::string GetPrintableOptions() const override; + static const char kValueTypeSeqId0 = char(~0); + + private: + PlainTableOptions table_options_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/plain/plain_table_index.cc b/librocksdb-sys/rocksdb/table/plain/plain_table_index.cc new file mode 100644 index 0000000..c85176d --- /dev/null +++ b/librocksdb-sys/rocksdb/table/plain/plain_table_index.cc @@ -0,0 +1,211 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "table/plain/plain_table_index.h" + +#include + +#include "logging/logging.h" +#include "util/coding.h" +#include "util/hash.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { +inline uint32_t GetBucketIdFromHash(uint32_t hash, uint32_t num_buckets) { + assert(num_buckets > 0); + return hash % num_buckets; +} +} // namespace + +Status PlainTableIndex::InitFromRawData(Slice data) { + if (!GetVarint32(&data, &index_size_)) { + return Status::Corruption("Couldn't read the index size!"); + } + assert(index_size_ > 0); + if (!GetVarint32(&data, &num_prefixes_)) { + return Status::Corruption("Couldn't read the index size!"); + } + sub_index_size_ = + static_cast(data.size()) - index_size_ * kOffsetLen; + + char* index_data_begin = const_cast(data.data()); + index_ = reinterpret_cast(index_data_begin); + sub_index_ = reinterpret_cast(index_ + index_size_); + return Status::OK(); +} + +PlainTableIndex::IndexSearchResult PlainTableIndex::GetOffset( + uint32_t prefix_hash, uint32_t* bucket_value) const { + int bucket = GetBucketIdFromHash(prefix_hash, index_size_); + GetUnaligned(index_ + bucket, bucket_value); + if ((*bucket_value & kSubIndexMask) == kSubIndexMask) { + *bucket_value ^= kSubIndexMask; + return kSubindex; + } + if (*bucket_value >= kMaxFileSize) { + return kNoPrefixForBucket; + } else { + // point directly to the file + return kDirectToFile; + } +} + +void PlainTableIndexBuilder::IndexRecordList::AddRecord(uint32_t hash, + uint32_t offset) { + if (num_records_in_current_group_ == kNumRecordsPerGroup) { + current_group_ = AllocateNewGroup(); + num_records_in_current_group_ = 0; + } + auto& new_record = current_group_[num_records_in_current_group_++]; + new_record.hash = hash; + new_record.offset = offset; + new_record.next = nullptr; +} + +void PlainTableIndexBuilder::AddKeyPrefix(Slice key_prefix_slice, + uint32_t key_offset) { + if (is_first_record_ || prev_key_prefix_ != key_prefix_slice.ToString()) { + ++num_prefixes_; + if (!is_first_record_) { + keys_per_prefix_hist_.Add(num_keys_per_prefix_); + } + num_keys_per_prefix_ = 0; + prev_key_prefix_ = key_prefix_slice.ToString(); + prev_key_prefix_hash_ = GetSliceHash(key_prefix_slice); + due_index_ = true; + } + + if (due_index_) { + // Add an index key for every kIndexIntervalForSamePrefixKeys keys + record_list_.AddRecord(prev_key_prefix_hash_, key_offset); + due_index_ = false; + } + + num_keys_per_prefix_++; + if (index_sparseness_ == 0 || num_keys_per_prefix_ % index_sparseness_ == 0) { + due_index_ = true; + } + is_first_record_ = false; +} + +Slice PlainTableIndexBuilder::Finish() { + AllocateIndex(); + std::vector hash_to_offsets(index_size_, nullptr); + std::vector entries_per_bucket(index_size_, 0); + BucketizeIndexes(&hash_to_offsets, &entries_per_bucket); + + keys_per_prefix_hist_.Add(num_keys_per_prefix_); + ROCKS_LOG_INFO(ioptions_.logger, "Number of Keys per prefix Histogram: %s", + keys_per_prefix_hist_.ToString().c_str()); + + // From the temp data structure, populate indexes. + return FillIndexes(hash_to_offsets, entries_per_bucket); +} + +void PlainTableIndexBuilder::AllocateIndex() { + if (prefix_extractor_ == nullptr || hash_table_ratio_ <= 0) { + // Fall back to pure binary search if the user fails to specify a prefix + // extractor. + index_size_ = 1; + } else { + double hash_table_size_multipier = 1.0 / hash_table_ratio_; + index_size_ = + static_cast(num_prefixes_ * hash_table_size_multipier) + 1; + assert(index_size_ > 0); + } +} + +void PlainTableIndexBuilder::BucketizeIndexes( + std::vector* hash_to_offsets, + std::vector* entries_per_bucket) { + bool first = true; + uint32_t prev_hash = 0; + size_t num_records = record_list_.GetNumRecords(); + for (size_t i = 0; i < num_records; i++) { + IndexRecord* index_record = record_list_.At(i); + uint32_t cur_hash = index_record->hash; + if (first || prev_hash != cur_hash) { + prev_hash = cur_hash; + first = false; + } + uint32_t bucket = GetBucketIdFromHash(cur_hash, index_size_); + IndexRecord* prev_bucket_head = (*hash_to_offsets)[bucket]; + index_record->next = prev_bucket_head; + (*hash_to_offsets)[bucket] = index_record; + (*entries_per_bucket)[bucket]++; + } + + sub_index_size_ = 0; + for (auto entry_count : *entries_per_bucket) { + if (entry_count <= 1) { + continue; + } + // Only buckets with more than 1 entry will have subindex. + sub_index_size_ += VarintLength(entry_count); + // total bytes needed to store these entries' in-file offsets. + sub_index_size_ += entry_count * PlainTableIndex::kOffsetLen; + } +} + +Slice PlainTableIndexBuilder::FillIndexes( + const std::vector& hash_to_offsets, + const std::vector& entries_per_bucket) { + ROCKS_LOG_DEBUG(ioptions_.logger, + "Reserving %" PRIu32 " bytes for plain table's sub_index", + sub_index_size_); + auto total_allocate_size = GetTotalSize(); + char* allocated = arena_->AllocateAligned( + total_allocate_size, huge_page_tlb_size_, ioptions_.logger); + + auto temp_ptr = EncodeVarint32(allocated, index_size_); + uint32_t* index = + reinterpret_cast(EncodeVarint32(temp_ptr, num_prefixes_)); + char* sub_index = reinterpret_cast(index + index_size_); + + uint32_t sub_index_offset = 0; + for (uint32_t i = 0; i < index_size_; i++) { + uint32_t num_keys_for_bucket = entries_per_bucket[i]; + switch (num_keys_for_bucket) { + case 0: + // No key for bucket + PutUnaligned(index + i, (uint32_t)PlainTableIndex::kMaxFileSize); + break; + case 1: + // point directly to the file offset + PutUnaligned(index + i, hash_to_offsets[i]->offset); + break; + default: + // point to second level indexes. + PutUnaligned(index + i, + sub_index_offset | PlainTableIndex::kSubIndexMask); + char* prev_ptr = &sub_index[sub_index_offset]; + char* cur_ptr = EncodeVarint32(prev_ptr, num_keys_for_bucket); + sub_index_offset += static_cast(cur_ptr - prev_ptr); + char* sub_index_pos = &sub_index[sub_index_offset]; + IndexRecord* record = hash_to_offsets[i]; + int j; + for (j = num_keys_for_bucket - 1; j >= 0 && record; + j--, record = record->next) { + EncodeFixed32(sub_index_pos + j * sizeof(uint32_t), record->offset); + } + assert(j == -1 && record == nullptr); + sub_index_offset += PlainTableIndex::kOffsetLen * num_keys_for_bucket; + assert(sub_index_offset <= sub_index_size_); + break; + } + } + assert(sub_index_offset == sub_index_size_); + + ROCKS_LOG_DEBUG(ioptions_.logger, + "hash table size: %" PRIu32 ", suffix_map length %" PRIu32, + index_size_, sub_index_size_); + return Slice(allocated, GetTotalSize()); +} + +const std::string PlainTableIndexBuilder::kPlainTableIndexBlock = + "PlainTableIndexBlock"; +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/table/plain/plain_table_index.h b/librocksdb-sys/rocksdb/table/plain/plain_table_index.h new file mode 100644 index 0000000..0adb641 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/plain/plain_table_index.h @@ -0,0 +1,246 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + + +#include +#include + +#include "memory/arena.h" +#include "monitoring/histogram.h" +#include "options/cf_options.h" +#include "rocksdb/options.h" + +namespace ROCKSDB_NAMESPACE { + +// The file contains two classes PlainTableIndex and PlainTableIndexBuilder +// The two classes implement the index format of PlainTable. +// For description of PlainTable format, see comments of class +// PlainTableFactory +// +// +// PlainTableIndex contains buckets size of index_size_, each is a +// 32-bit integer. The lower 31 bits contain an offset value (explained below) +// and the first bit of the integer indicates type of the offset. +// +// +--------------+------------------------------------------------------+ +// | Flag (1 bit) | Offset to binary search buffer or file (31 bits) + +// +--------------+------------------------------------------------------+ +// +// Explanation for the "flag bit": +// +// 0 indicates that the bucket contains only one prefix (no conflict when +// hashing this prefix), whose first row starts from this offset of the +// file. +// 1 indicates that the bucket contains more than one prefixes, or there +// are too many rows for one prefix so we need a binary search for it. In +// this case, the offset indicates the offset of sub_index_ holding the +// binary search indexes of keys for those rows. Those binary search indexes +// are organized in this way: +// +// The first 4 bytes, indicate how many indexes (N) are stored after it. After +// it, there are N 32-bit integers, each points of an offset of the file, +// which +// points to starting of a row. Those offsets need to be guaranteed to be in +// ascending order so the keys they are pointing to are also in ascending +// order +// to make sure we can use them to do binary searches. Below is visual +// presentation of a bucket. +// +// +// number_of_records: varint32 +// record 1 file offset: fixedint32 +// record 2 file offset: fixedint32 +// .... +// record N file offset: fixedint32 +// + +// The class loads the index block from a PlainTable SST file, and executes +// the index lookup. +// The class is used by PlainTableReader class. +class PlainTableIndex { + public: + enum IndexSearchResult { + kNoPrefixForBucket = 0, + kDirectToFile = 1, + kSubindex = 2 + }; + + explicit PlainTableIndex(Slice data) { InitFromRawData(data); } + + PlainTableIndex() + : index_size_(0), + sub_index_size_(0), + num_prefixes_(0), + index_(nullptr), + sub_index_(nullptr) {} + + // The function that executes the lookup the hash table. + // The hash key is `prefix_hash`. The function fills the hash bucket + // content in `bucket_value`, which is up to the caller to interpret. + IndexSearchResult GetOffset(uint32_t prefix_hash, + uint32_t* bucket_value) const; + + // Initialize data from `index_data`, which points to raw data for + // index stored in the SST file. + Status InitFromRawData(Slice index_data); + + // Decode the sub index for specific hash bucket. + // The `offset` is the value returned as `bucket_value` by GetOffset() + // and is only valid when the return value is `kSubindex`. + // The return value is the pointer to the starting address of the + // sub-index. `upper_bound` is filled with the value indicating how many + // entries the sub-index has. + const char* GetSubIndexBasePtrAndUpperBound(uint32_t offset, + uint32_t* upper_bound) const { + const char* index_ptr = &sub_index_[offset]; + return GetVarint32Ptr(index_ptr, index_ptr + 4, upper_bound); + } + + uint32_t GetIndexSize() const { return index_size_; } + + uint32_t GetSubIndexSize() const { return sub_index_size_; } + + uint32_t GetNumPrefixes() const { return num_prefixes_; } + + static const uint64_t kMaxFileSize = (1u << 31) - 1; + static const uint32_t kSubIndexMask = 0x80000000; + static const size_t kOffsetLen = sizeof(uint32_t); + + private: + uint32_t index_size_; + uint32_t sub_index_size_; + uint32_t num_prefixes_; + + uint32_t* index_; + char* sub_index_; +}; + +// PlainTableIndexBuilder is used to create plain table index. +// After calling Finish(), it returns Slice, which is usually +// used either to initialize PlainTableIndex or +// to save index to sst file. +// For more details about the index, please refer to: +// https://github.com/facebook/rocksdb/wiki/PlainTable-Format +// #wiki-in-memory-index-format +// The class is used by PlainTableBuilder class. +class PlainTableIndexBuilder { + public: + PlainTableIndexBuilder(Arena* arena, const ImmutableOptions& ioptions, + const SliceTransform* prefix_extractor, + size_t index_sparseness, double hash_table_ratio, + size_t huge_page_tlb_size) + : arena_(arena), + ioptions_(ioptions), + record_list_(kRecordsPerGroup), + is_first_record_(true), + due_index_(false), + num_prefixes_(0), + num_keys_per_prefix_(0), + prev_key_prefix_hash_(0), + index_sparseness_(index_sparseness), + index_size_(0), + sub_index_size_(0), + prefix_extractor_(prefix_extractor), + hash_table_ratio_(hash_table_ratio), + huge_page_tlb_size_(huge_page_tlb_size) {} + + void AddKeyPrefix(Slice key_prefix_slice, uint32_t key_offset); + + Slice Finish(); + + uint32_t GetTotalSize() const { + return VarintLength(index_size_) + VarintLength(num_prefixes_) + + PlainTableIndex::kOffsetLen * index_size_ + sub_index_size_; + } + + static const std::string kPlainTableIndexBlock; + + private: + struct IndexRecord { + uint32_t hash; // hash of the prefix + uint32_t offset; // offset of a row + IndexRecord* next; + }; + + // Helper class to track all the index records + class IndexRecordList { + public: + explicit IndexRecordList(size_t num_records_per_group) + : kNumRecordsPerGroup(num_records_per_group), + current_group_(nullptr), + num_records_in_current_group_(num_records_per_group) {} + + ~IndexRecordList() { + for (size_t i = 0; i < groups_.size(); i++) { + delete[] groups_[i]; + } + } + + void AddRecord(uint32_t hash, uint32_t offset); + + size_t GetNumRecords() const { + return (groups_.size() - 1) * kNumRecordsPerGroup + + num_records_in_current_group_; + } + IndexRecord* At(size_t index) { + return &( + groups_[index / kNumRecordsPerGroup][index % kNumRecordsPerGroup]); + } + + private: + IndexRecord* AllocateNewGroup() { + IndexRecord* result = new IndexRecord[kNumRecordsPerGroup]; + groups_.push_back(result); + return result; + } + + // Each group in `groups_` contains fix-sized records (determined by + // kNumRecordsPerGroup). Which can help us minimize the cost if resizing + // occurs. + const size_t kNumRecordsPerGroup; + IndexRecord* current_group_; + // List of arrays allocated + std::vector groups_; + size_t num_records_in_current_group_; + }; + + void AllocateIndex(); + + // Internal helper function to bucket index record list to hash buckets. + void BucketizeIndexes(std::vector* hash_to_offsets, + std::vector* entries_per_bucket); + + // Internal helper class to fill the indexes and bloom filters to internal + // data structures. + Slice FillIndexes(const std::vector& hash_to_offsets, + const std::vector& entries_per_bucket); + + Arena* arena_; + const ImmutableOptions ioptions_; + HistogramImpl keys_per_prefix_hist_; + IndexRecordList record_list_; + bool is_first_record_; + bool due_index_; + uint32_t num_prefixes_; + uint32_t num_keys_per_prefix_; + + uint32_t prev_key_prefix_hash_; + size_t index_sparseness_; + uint32_t index_size_; + uint32_t sub_index_size_; + + const SliceTransform* prefix_extractor_; + double hash_table_ratio_; + size_t huge_page_tlb_size_; + + std::string prev_key_prefix_; + + static const size_t kRecordsPerGroup = 256; +}; + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/table/plain/plain_table_key_coding.cc b/librocksdb-sys/rocksdb/table/plain/plain_table_key_coding.cc new file mode 100644 index 0000000..a40968a --- /dev/null +++ b/librocksdb-sys/rocksdb/table/plain/plain_table_key_coding.cc @@ -0,0 +1,507 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "table/plain/plain_table_key_coding.h" + +#include +#include + +#include "db/dbformat.h" +#include "file/writable_file_writer.h" +#include "table/plain/plain_table_factory.h" +#include "table/plain/plain_table_reader.h" + +namespace ROCKSDB_NAMESPACE { + +enum PlainTableEntryType : unsigned char { + kFullKey = 0, + kPrefixFromPreviousKey = 1, + kKeySuffix = 2, +}; + +namespace { + +// Control byte: +// First two bits indicate type of entry +// Other bytes are inlined sizes. If all bits are 1 (0x03F), overflow bytes +// are used. key_size-0x3F will be encoded as a variint32 after this bytes. + +const unsigned char kSizeInlineLimit = 0x3F; + +// Return 0 for error +size_t EncodeSize(PlainTableEntryType type, uint32_t key_size, + char* out_buffer) { + out_buffer[0] = type << 6; + + if (key_size < static_cast(kSizeInlineLimit)) { + // size inlined + out_buffer[0] |= static_cast(key_size); + return 1; + } else { + out_buffer[0] |= kSizeInlineLimit; + char* ptr = EncodeVarint32(out_buffer + 1, key_size - kSizeInlineLimit); + return ptr - out_buffer; + } +} +} // namespace + +// Fill bytes_read with number of bytes read. +inline Status PlainTableKeyDecoder::DecodeSize(uint32_t start_offset, + PlainTableEntryType* entry_type, + uint32_t* key_size, + uint32_t* bytes_read) { + Slice next_byte_slice; + bool success = file_reader_.Read(start_offset, 1, &next_byte_slice); + if (!success) { + return file_reader_.status(); + } + *entry_type = static_cast( + (static_cast(next_byte_slice[0]) & ~kSizeInlineLimit) >> + 6); + char inline_key_size = next_byte_slice[0] & kSizeInlineLimit; + if (inline_key_size < kSizeInlineLimit) { + *key_size = inline_key_size; + *bytes_read = 1; + return Status::OK(); + } else { + uint32_t extra_size; + uint32_t tmp_bytes_read; + success = file_reader_.ReadVarint32(start_offset + 1, &extra_size, + &tmp_bytes_read); + if (!success) { + return file_reader_.status(); + } + assert(tmp_bytes_read > 0); + *key_size = kSizeInlineLimit + extra_size; + *bytes_read = tmp_bytes_read + 1; + return Status::OK(); + } +} + +IOStatus PlainTableKeyEncoder::AppendKey(const Slice& key, + WritableFileWriter* file, + uint64_t* offset, char* meta_bytes_buf, + size_t* meta_bytes_buf_size) { + ParsedInternalKey parsed_key; + Status pik_status = + ParseInternalKey(key, &parsed_key, false /* log_err_key */); // TODO + if (!pik_status.ok()) { + return IOStatus::Corruption(pik_status.getState()); + } + + Slice key_to_write = key; // Portion of internal key to write out. + + uint32_t user_key_size = static_cast(key.size() - 8); + if (encoding_type_ == kPlain) { + if (fixed_user_key_len_ == kPlainTableVariableLength) { + // Write key length + char key_size_buf[5]; // tmp buffer for key size as varint32 + char* ptr = EncodeVarint32(key_size_buf, user_key_size); + assert(ptr <= key_size_buf + sizeof(key_size_buf)); + auto len = ptr - key_size_buf; + IOStatus io_s = file->Append(Slice(key_size_buf, len)); + if (!io_s.ok()) { + return io_s; + } + *offset += len; + } + } else { + assert(encoding_type_ == kPrefix); + char size_bytes[12]; + size_t size_bytes_pos = 0; + + Slice prefix = + prefix_extractor_->Transform(Slice(key.data(), user_key_size)); + if (key_count_for_prefix_ == 0 || prefix != pre_prefix_.GetUserKey() || + key_count_for_prefix_ % index_sparseness_ == 0) { + key_count_for_prefix_ = 1; + pre_prefix_.SetUserKey(prefix); + size_bytes_pos += EncodeSize(kFullKey, user_key_size, size_bytes); + IOStatus io_s = file->Append(Slice(size_bytes, size_bytes_pos)); + if (!io_s.ok()) { + return io_s; + } + *offset += size_bytes_pos; + } else { + key_count_for_prefix_++; + if (key_count_for_prefix_ == 2) { + // For second key within a prefix, need to encode prefix length + size_bytes_pos += + EncodeSize(kPrefixFromPreviousKey, + static_cast(pre_prefix_.GetUserKey().size()), + size_bytes + size_bytes_pos); + } + uint32_t prefix_len = + static_cast(pre_prefix_.GetUserKey().size()); + size_bytes_pos += EncodeSize(kKeySuffix, user_key_size - prefix_len, + size_bytes + size_bytes_pos); + IOStatus io_s = file->Append(Slice(size_bytes, size_bytes_pos)); + if (!io_s.ok()) { + return io_s; + } + *offset += size_bytes_pos; + key_to_write = Slice(key.data() + prefix_len, key.size() - prefix_len); + } + } + + // Encode full key + // For value size as varint32 (up to 5 bytes). + // If the row is of value type with seqId 0, flush the special flag together + // in this buffer to safe one file append call, which takes 1 byte. + if (parsed_key.sequence == 0 && parsed_key.type == kTypeValue) { + IOStatus io_s = + file->Append(Slice(key_to_write.data(), key_to_write.size() - 8)); + if (!io_s.ok()) { + return io_s; + } + *offset += key_to_write.size() - 8; + meta_bytes_buf[*meta_bytes_buf_size] = PlainTableFactory::kValueTypeSeqId0; + *meta_bytes_buf_size += 1; + } else { + IOStatus io_s = file->Append(key_to_write); + if (!io_s.ok()) { + return io_s; + } + *offset += key_to_write.size(); + } + + return IOStatus::OK(); +} + +Slice PlainTableFileReader::GetFromBuffer(Buffer* buffer, uint32_t file_offset, + uint32_t len) { + assert(file_offset + len <= file_info_->data_end_offset); + return Slice(buffer->buf.get() + (file_offset - buffer->buf_start_offset), + len); +} + +bool PlainTableFileReader::ReadNonMmap(uint32_t file_offset, uint32_t len, + Slice* out) { + const uint32_t kPrefetchSize = 256u; + + // Try to read from buffers. + for (uint32_t i = 0; i < num_buf_; i++) { + Buffer* buffer = buffers_[num_buf_ - 1 - i].get(); + if (file_offset >= buffer->buf_start_offset && + file_offset + len <= buffer->buf_start_offset + buffer->buf_len) { + *out = GetFromBuffer(buffer, file_offset, len); + return true; + } + } + + Buffer* new_buffer; + // Data needed is not in any of the buffer. Allocate a new buffer. + if (num_buf_ < buffers_.size()) { + // Add a new buffer + new_buffer = new Buffer(); + buffers_[num_buf_++].reset(new_buffer); + } else { + // Now simply replace the last buffer. Can improve the placement policy + // if needed. + new_buffer = buffers_[num_buf_ - 1].get(); + } + + assert(file_offset + len <= file_info_->data_end_offset); + uint32_t size_to_read = std::min(file_info_->data_end_offset - file_offset, + std::max(kPrefetchSize, len)); + if (size_to_read > new_buffer->buf_capacity) { + new_buffer->buf.reset(new char[size_to_read]); + new_buffer->buf_capacity = size_to_read; + new_buffer->buf_len = 0; + } + Slice read_result; + // TODO: rate limit plain table reads. + Status s = + file_info_->file->Read(IOOptions(), file_offset, size_to_read, + &read_result, new_buffer->buf.get(), nullptr, + Env::IO_TOTAL /* rate_limiter_priority */); + if (!s.ok()) { + status_ = s; + return false; + } + new_buffer->buf_start_offset = file_offset; + new_buffer->buf_len = size_to_read; + *out = GetFromBuffer(new_buffer, file_offset, len); + return true; +} + +inline bool PlainTableFileReader::ReadVarint32(uint32_t offset, uint32_t* out, + uint32_t* bytes_read) { + if (file_info_->is_mmap_mode) { + const char* start = file_info_->file_data.data() + offset; + const char* limit = + file_info_->file_data.data() + file_info_->data_end_offset; + const char* key_ptr = GetVarint32Ptr(start, limit, out); + assert(key_ptr != nullptr); + *bytes_read = static_cast(key_ptr - start); + return true; + } else { + return ReadVarint32NonMmap(offset, out, bytes_read); + } +} + +bool PlainTableFileReader::ReadVarint32NonMmap(uint32_t offset, uint32_t* out, + uint32_t* bytes_read) { + const char* start; + const char* limit; + const uint32_t kMaxVarInt32Size = 6u; + uint32_t bytes_to_read = + std::min(file_info_->data_end_offset - offset, kMaxVarInt32Size); + Slice bytes; + if (!Read(offset, bytes_to_read, &bytes)) { + return false; + } + start = bytes.data(); + limit = bytes.data() + bytes.size(); + + const char* key_ptr = GetVarint32Ptr(start, limit, out); + *bytes_read = + (key_ptr != nullptr) ? static_cast(key_ptr - start) : 0; + return true; +} + +Status PlainTableKeyDecoder::ReadInternalKey( + uint32_t file_offset, uint32_t user_key_size, ParsedInternalKey* parsed_key, + uint32_t* bytes_read, bool* internal_key_valid, Slice* internal_key) { + Slice tmp_slice; + bool success = file_reader_.Read(file_offset, user_key_size + 1, &tmp_slice); + if (!success) { + return file_reader_.status(); + } + if (tmp_slice[user_key_size] == PlainTableFactory::kValueTypeSeqId0) { + // Special encoding for the row with seqID=0 + parsed_key->user_key = Slice(tmp_slice.data(), user_key_size); + parsed_key->sequence = 0; + parsed_key->type = kTypeValue; + *bytes_read += user_key_size + 1; + *internal_key_valid = false; + } else { + success = file_reader_.Read(file_offset, user_key_size + 8, internal_key); + if (!success) { + return file_reader_.status(); + } + *internal_key_valid = true; + Status pik_status = ParseInternalKey(*internal_key, parsed_key, + false /* log_err_key */); // TODO + if (!pik_status.ok()) { + return Status::Corruption( + Slice("Corrupted key found during next key read. "), + pik_status.getState()); + } + *bytes_read += user_key_size + 8; + } + return Status::OK(); +} + +Status PlainTableKeyDecoder::NextPlainEncodingKey(uint32_t start_offset, + ParsedInternalKey* parsed_key, + Slice* internal_key, + uint32_t* bytes_read, + bool* /*seekable*/) { + uint32_t user_key_size = 0; + Status s; + if (fixed_user_key_len_ != kPlainTableVariableLength) { + user_key_size = fixed_user_key_len_; + } else { + uint32_t tmp_size = 0; + uint32_t tmp_read; + bool success = + file_reader_.ReadVarint32(start_offset, &tmp_size, &tmp_read); + if (!success) { + return file_reader_.status(); + } + assert(tmp_read > 0); + user_key_size = tmp_size; + *bytes_read = tmp_read; + } + // dummy initial value to avoid compiler complain + bool decoded_internal_key_valid = true; + Slice decoded_internal_key; + s = ReadInternalKey(start_offset + *bytes_read, user_key_size, parsed_key, + bytes_read, &decoded_internal_key_valid, + &decoded_internal_key); + if (!s.ok()) { + return s; + } + if (!file_reader_.file_info()->is_mmap_mode) { + cur_key_.SetInternalKey(*parsed_key); + parsed_key->user_key = + Slice(cur_key_.GetInternalKey().data(), user_key_size); + if (internal_key != nullptr) { + *internal_key = cur_key_.GetInternalKey(); + } + } else if (internal_key != nullptr) { + if (decoded_internal_key_valid) { + *internal_key = decoded_internal_key; + } else { + // Need to copy out the internal key + cur_key_.SetInternalKey(*parsed_key); + *internal_key = cur_key_.GetInternalKey(); + } + } + return Status::OK(); +} + +Status PlainTableKeyDecoder::NextPrefixEncodingKey( + uint32_t start_offset, ParsedInternalKey* parsed_key, Slice* internal_key, + uint32_t* bytes_read, bool* seekable) { + PlainTableEntryType entry_type; + + bool expect_suffix = false; + Status s; + do { + uint32_t size = 0; + // dummy initial value to avoid compiler complain + bool decoded_internal_key_valid = true; + uint32_t my_bytes_read = 0; + s = DecodeSize(start_offset + *bytes_read, &entry_type, &size, + &my_bytes_read); + if (!s.ok()) { + return s; + } + if (my_bytes_read == 0) { + return Status::Corruption("Unexpected EOF when reading size of the key"); + } + *bytes_read += my_bytes_read; + + switch (entry_type) { + case kFullKey: { + expect_suffix = false; + Slice decoded_internal_key; + s = ReadInternalKey(start_offset + *bytes_read, size, parsed_key, + bytes_read, &decoded_internal_key_valid, + &decoded_internal_key); + if (!s.ok()) { + return s; + } + if (!file_reader_.file_info()->is_mmap_mode || + (internal_key != nullptr && !decoded_internal_key_valid)) { + // In non-mmap mode, always need to make a copy of keys returned to + // users, because after reading value for the key, the key might + // be invalid. + cur_key_.SetInternalKey(*parsed_key); + saved_user_key_ = cur_key_.GetUserKey(); + if (!file_reader_.file_info()->is_mmap_mode) { + parsed_key->user_key = + Slice(cur_key_.GetInternalKey().data(), size); + } + if (internal_key != nullptr) { + *internal_key = cur_key_.GetInternalKey(); + } + } else { + if (internal_key != nullptr) { + *internal_key = decoded_internal_key; + } + saved_user_key_ = parsed_key->user_key; + } + break; + } + case kPrefixFromPreviousKey: { + if (seekable != nullptr) { + *seekable = false; + } + prefix_len_ = size; + assert(prefix_extractor_ == nullptr || + prefix_extractor_->Transform(saved_user_key_).size() == + prefix_len_); + // Need read another size flag for suffix + expect_suffix = true; + break; + } + case kKeySuffix: { + expect_suffix = false; + if (seekable != nullptr) { + *seekable = false; + } + + Slice tmp_slice; + s = ReadInternalKey(start_offset + *bytes_read, size, parsed_key, + bytes_read, &decoded_internal_key_valid, + &tmp_slice); + if (!s.ok()) { + return s; + } + if (!file_reader_.file_info()->is_mmap_mode) { + // In non-mmap mode, we need to make a copy of keys returned to + // users, because after reading value for the key, the key might + // be invalid. + // saved_user_key_ points to cur_key_. We are making a copy of + // the prefix part to another string, and construct the current + // key from the prefix part and the suffix part back to cur_key_. + std::string tmp = + Slice(saved_user_key_.data(), prefix_len_).ToString(); + cur_key_.Reserve(prefix_len_ + size); + cur_key_.SetInternalKey(tmp, *parsed_key); + parsed_key->user_key = + Slice(cur_key_.GetInternalKey().data(), prefix_len_ + size); + saved_user_key_ = cur_key_.GetUserKey(); + } else { + cur_key_.Reserve(prefix_len_ + size); + cur_key_.SetInternalKey(Slice(saved_user_key_.data(), prefix_len_), + *parsed_key); + } + parsed_key->user_key = cur_key_.GetUserKey(); + if (internal_key != nullptr) { + *internal_key = cur_key_.GetInternalKey(); + } + break; + } + default: + return Status::Corruption("Un-identified size flag."); + } + } while (expect_suffix); // Another round if suffix is expected. + return Status::OK(); +} + +Status PlainTableKeyDecoder::NextKey(uint32_t start_offset, + ParsedInternalKey* parsed_key, + Slice* internal_key, Slice* value, + uint32_t* bytes_read, bool* seekable) { + assert(value != nullptr); + Status s = NextKeyNoValue(start_offset, parsed_key, internal_key, bytes_read, + seekable); + if (s.ok()) { + assert(bytes_read != nullptr); + uint32_t value_size; + uint32_t value_size_bytes; + bool success = file_reader_.ReadVarint32(start_offset + *bytes_read, + &value_size, &value_size_bytes); + if (!success) { + return file_reader_.status(); + } + if (value_size_bytes == 0) { + return Status::Corruption( + "Unexpected EOF when reading the next value's size."); + } + *bytes_read += value_size_bytes; + success = file_reader_.Read(start_offset + *bytes_read, value_size, value); + if (!success) { + return file_reader_.status(); + } + *bytes_read += value_size; + } + return s; +} + +Status PlainTableKeyDecoder::NextKeyNoValue(uint32_t start_offset, + ParsedInternalKey* parsed_key, + Slice* internal_key, + uint32_t* bytes_read, + bool* seekable) { + *bytes_read = 0; + if (seekable != nullptr) { + *seekable = true; + } + if (encoding_type_ == kPlain) { + return NextPlainEncodingKey(start_offset, parsed_key, internal_key, + bytes_read, seekable); + } else { + assert(encoding_type_ == kPrefix); + return NextPrefixEncodingKey(start_offset, parsed_key, internal_key, + bytes_read, seekable); + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/plain/plain_table_key_coding.h b/librocksdb-sys/rocksdb/table/plain/plain_table_key_coding.h new file mode 100644 index 0000000..fdef224 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/plain/plain_table_key_coding.h @@ -0,0 +1,199 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + + +#include + +#include "rocksdb/slice.h" +#include "table/plain/plain_table_reader.h" + +// The file contains three helper classes of PlainTable format, +// PlainTableKeyEncoder, PlainTableKeyDecoder and PlainTableFileReader. +// These classes issue the lowest level of operations of PlainTable. +// Actual data format of the key is documented in comments of class +// PlainTableFactory. +namespace ROCKSDB_NAMESPACE { + +class WritableFile; +struct ParsedInternalKey; +struct PlainTableReaderFileInfo; +enum PlainTableEntryType : unsigned char; + +// Helper class for PlainTable format to write out a key to an output file +// The class is used in PlainTableBuilder. +class PlainTableKeyEncoder { + public: + explicit PlainTableKeyEncoder(EncodingType encoding_type, + uint32_t user_key_len, + const SliceTransform* prefix_extractor, + size_t index_sparseness) + : encoding_type_((prefix_extractor != nullptr) ? encoding_type : kPlain), + fixed_user_key_len_(user_key_len), + prefix_extractor_(prefix_extractor), + index_sparseness_((index_sparseness > 1) ? index_sparseness : 1), + key_count_for_prefix_(0) {} + // key: the key to write out, in the format of internal key. + // file: the output file to write out + // offset: offset in the file. Needs to be updated after appending bytes + // for the key + // meta_bytes_buf: buffer for extra meta bytes + // meta_bytes_buf_size: offset to append extra meta bytes. Will be updated + // if meta_bytes_buf is updated. + IOStatus AppendKey(const Slice& key, WritableFileWriter* file, + uint64_t* offset, char* meta_bytes_buf, + size_t* meta_bytes_buf_size); + + // Return actual encoding type to be picked + EncodingType GetEncodingType() { return encoding_type_; } + + private: + EncodingType encoding_type_; + uint32_t fixed_user_key_len_; + const SliceTransform* prefix_extractor_; + const size_t index_sparseness_; + size_t key_count_for_prefix_; + IterKey pre_prefix_; +}; + +// The class does raw file reads for PlainTableReader. +// It hides whether it is a mmap-read, or a non-mmap read. +// The class is implemented in a way to favor the performance of mmap case. +// The class is used by PlainTableReader. +class PlainTableFileReader { + public: + explicit PlainTableFileReader(const PlainTableReaderFileInfo* _file_info) + : file_info_(_file_info), num_buf_(0) {} + + ~PlainTableFileReader() { + // Should fix. + status_.PermitUncheckedError(); + } + + // In mmaped mode, the results point to mmaped area of the file, which + // means it is always valid before closing the file. + // In non-mmap mode, the results point to an internal buffer. If the caller + // makes another read call, the results may not be valid. So callers should + // make a copy when needed. + // In order to save read calls to files, we keep two internal buffers: + // the first read and the most recent read. This is efficient because it + // columns these two common use cases: + // (1) hash index only identify one location, we read the key to verify + // the location, and read key and value if it is the right location. + // (2) after hash index checking, we identify two locations (because of + // hash bucket conflicts), we binary search the two location to see + // which one is what we need and start to read from the location. + // These two most common use cases will be covered by the two buffers + // so that we don't need to re-read the same location. + // Currently we keep a fixed size buffer. If a read doesn't exactly fit + // the buffer, we replace the second buffer with the location user reads. + // + // If return false, status code is stored in status_. + bool Read(uint32_t file_offset, uint32_t len, Slice* out) { + if (file_info_->is_mmap_mode) { + assert(file_offset + len <= file_info_->data_end_offset); + *out = Slice(file_info_->file_data.data() + file_offset, len); + return true; + } else { + return ReadNonMmap(file_offset, len, out); + } + } + + // If return false, status code is stored in status_. + bool ReadNonMmap(uint32_t file_offset, uint32_t len, Slice* output); + + // *bytes_read = 0 means eof. false means failure and status is saved + // in status_. Not directly returning Status to save copying status + // object to map previous performance of mmap mode. + inline bool ReadVarint32(uint32_t offset, uint32_t* output, + uint32_t* bytes_read); + + bool ReadVarint32NonMmap(uint32_t offset, uint32_t* output, + uint32_t* bytes_read); + + Status status() const { return status_; } + + const PlainTableReaderFileInfo* file_info() { return file_info_; } + + private: + const PlainTableReaderFileInfo* file_info_; + + struct Buffer { + Buffer() : buf_start_offset(0), buf_len(0), buf_capacity(0) {} + std::unique_ptr buf; + uint32_t buf_start_offset; + uint32_t buf_len; + uint32_t buf_capacity; + }; + + // Keep buffers for two recent reads. + std::array, 2> buffers_; + uint32_t num_buf_; + Status status_; + + Slice GetFromBuffer(Buffer* buf, uint32_t file_offset, uint32_t len); +}; + +// A helper class to decode keys from input buffer +// The class is used by PlainTableBuilder. +class PlainTableKeyDecoder { + public: + explicit PlainTableKeyDecoder(const PlainTableReaderFileInfo* file_info, + EncodingType encoding_type, + uint32_t user_key_len, + const SliceTransform* prefix_extractor) + : file_reader_(file_info), + encoding_type_(encoding_type), + prefix_len_(0), + fixed_user_key_len_(user_key_len), + prefix_extractor_(prefix_extractor), + in_prefix_(false) {} + + // Find the next key. + // start: char array where the key starts. + // limit: boundary of the char array + // parsed_key: the output of the result key + // internal_key: if not null, fill with the output of the result key in + // un-parsed format + // bytes_read: how many bytes read from start. Output + // seekable: whether key can be read from this place. Used when building + // indexes. Output. + Status NextKey(uint32_t start_offset, ParsedInternalKey* parsed_key, + Slice* internal_key, Slice* value, uint32_t* bytes_read, + bool* seekable = nullptr); + + Status NextKeyNoValue(uint32_t start_offset, ParsedInternalKey* parsed_key, + Slice* internal_key, uint32_t* bytes_read, + bool* seekable = nullptr); + + PlainTableFileReader file_reader_; + EncodingType encoding_type_; + uint32_t prefix_len_; + uint32_t fixed_user_key_len_; + Slice saved_user_key_; + IterKey cur_key_; + const SliceTransform* prefix_extractor_; + bool in_prefix_; + + private: + Status NextPlainEncodingKey(uint32_t start_offset, + ParsedInternalKey* parsed_key, + Slice* internal_key, uint32_t* bytes_read, + bool* seekable = nullptr); + Status NextPrefixEncodingKey(uint32_t start_offset, + ParsedInternalKey* parsed_key, + Slice* internal_key, uint32_t* bytes_read, + bool* seekable = nullptr); + Status ReadInternalKey(uint32_t file_offset, uint32_t user_key_size, + ParsedInternalKey* parsed_key, uint32_t* bytes_read, + bool* internal_key_valid, Slice* internal_key); + inline Status DecodeSize(uint32_t start_offset, + PlainTableEntryType* entry_type, uint32_t* key_size, + uint32_t* bytes_read); +}; + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/table/plain/plain_table_reader.cc b/librocksdb-sys/rocksdb/table/plain/plain_table_reader.cc new file mode 100644 index 0000000..2f0379f --- /dev/null +++ b/librocksdb-sys/rocksdb/table/plain/plain_table_reader.cc @@ -0,0 +1,771 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + + +#include "table/plain/plain_table_reader.h" + +#include +#include + +#include "db/dbformat.h" +#include "memory/arena.h" +#include "monitoring/histogram.h" +#include "monitoring/perf_context_imp.h" +#include "rocksdb/cache.h" +#include "rocksdb/comparator.h" +#include "rocksdb/env.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/options.h" +#include "rocksdb/statistics.h" +#include "table/block_based/block.h" +#include "table/block_based/filter_block.h" +#include "table/format.h" +#include "table/get_context.h" +#include "table/internal_iterator.h" +#include "table/meta_blocks.h" +#include "table/plain/plain_table_bloom.h" +#include "table/plain/plain_table_factory.h" +#include "table/plain/plain_table_key_coding.h" +#include "table/two_level_iterator.h" +#include "util/coding.h" +#include "util/dynamic_bloom.h" +#include "util/hash.h" +#include "util/stop_watch.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { + +// Safely getting a uint32_t element from a char array, where, starting from +// `base`, every 4 bytes are considered as an fixed 32 bit integer. +inline uint32_t GetFixed32Element(const char* base, size_t offset) { + return DecodeFixed32(base + offset * sizeof(uint32_t)); +} +} // namespace + +// Iterator to iterate IndexedTable +class PlainTableIterator : public InternalIterator { + public: + explicit PlainTableIterator(PlainTableReader* table, bool use_prefix_seek); + // No copying allowed + PlainTableIterator(const PlainTableIterator&) = delete; + void operator=(const Iterator&) = delete; + + ~PlainTableIterator() override; + + bool Valid() const override; + + void SeekToFirst() override; + + void SeekToLast() override; + + void Seek(const Slice& target) override; + + void SeekForPrev(const Slice& target) override; + + void Next() override; + + void Prev() override; + + Slice key() const override; + + Slice value() const override; + + Status status() const override; + + private: + PlainTableReader* table_; + PlainTableKeyDecoder decoder_; + bool use_prefix_seek_; + uint32_t offset_; + uint32_t next_offset_; + Slice key_; + Slice value_; + Status status_; +}; + +extern const uint64_t kPlainTableMagicNumber; +PlainTableReader::PlainTableReader( + const ImmutableOptions& ioptions, + std::unique_ptr&& file, + const EnvOptions& storage_options, const InternalKeyComparator& icomparator, + EncodingType encoding_type, uint64_t file_size, + const TableProperties* table_properties, + const SliceTransform* prefix_extractor) + : internal_comparator_(icomparator), + encoding_type_(encoding_type), + full_scan_mode_(false), + user_key_len_(static_cast(table_properties->fixed_key_len)), + prefix_extractor_(prefix_extractor), + enable_bloom_(false), + bloom_(6), + file_info_(std::move(file), storage_options, + static_cast(table_properties->data_size)), + ioptions_(ioptions), + file_size_(file_size), + table_properties_(nullptr) {} + +PlainTableReader::~PlainTableReader() { + // Should fix? + status_.PermitUncheckedError(); +} + +Status PlainTableReader::Open( + const ImmutableOptions& ioptions, const EnvOptions& env_options, + const InternalKeyComparator& internal_comparator, + std::unique_ptr&& file, uint64_t file_size, + std::unique_ptr* table_reader, const int bloom_bits_per_key, + double hash_table_ratio, size_t index_sparseness, size_t huge_page_tlb_size, + bool full_scan_mode, const bool immortal_table, + const SliceTransform* prefix_extractor) { + if (file_size > PlainTableIndex::kMaxFileSize) { + return Status::NotSupported("File is too large for PlainTableReader!"); + } + + std::unique_ptr props; + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + auto s = ReadTableProperties(file.get(), file_size, kPlainTableMagicNumber, + ioptions, read_options, &props); + if (!s.ok()) { + return s; + } + + assert(hash_table_ratio >= 0.0); + auto& user_props = props->user_collected_properties; + auto prefix_extractor_in_file = props->prefix_extractor_name; + + if (!full_scan_mode && + !prefix_extractor_in_file.empty() /* old version sst file*/ + && prefix_extractor_in_file != "nullptr") { + if (!prefix_extractor) { + return Status::InvalidArgument( + "Prefix extractor is missing when opening a PlainTable built " + "using a prefix extractor"); + } else if (prefix_extractor_in_file != prefix_extractor->AsString()) { + return Status::InvalidArgument( + "Prefix extractor given doesn't match the one used to build " + "PlainTable"); + } + } + + EncodingType encoding_type = kPlain; + auto encoding_type_prop = + user_props.find(PlainTablePropertyNames::kEncodingType); + if (encoding_type_prop != user_props.end()) { + encoding_type = static_cast( + DecodeFixed32(encoding_type_prop->second.c_str())); + } + + std::unique_ptr new_reader(new PlainTableReader( + ioptions, std::move(file), env_options, internal_comparator, + encoding_type, file_size, props.get(), prefix_extractor)); + + s = new_reader->MmapDataIfNeeded(); + if (!s.ok()) { + return s; + } + + if (!full_scan_mode) { + s = new_reader->PopulateIndex(props.get(), bloom_bits_per_key, + hash_table_ratio, index_sparseness, + huge_page_tlb_size); + if (!s.ok()) { + return s; + } + } else { + // Flag to indicate it is a full scan mode so that none of the indexes + // can be used. + new_reader->full_scan_mode_ = true; + } + // PopulateIndex can add to the props, so don't store them until now + new_reader->table_properties_ = std::move(props); + + if (immortal_table && new_reader->file_info_.is_mmap_mode) { + new_reader->dummy_cleanable_.reset(new Cleanable()); + } + + *table_reader = std::move(new_reader); + return s; +} + +void PlainTableReader::SetupForCompaction() {} + +InternalIterator* PlainTableReader::NewIterator( + const ReadOptions& options, const SliceTransform* /* prefix_extractor */, + Arena* arena, bool /*skip_filters*/, TableReaderCaller /*caller*/, + size_t /*compaction_readahead_size*/, bool /* allow_unprepared_value */) { + // Not necessarily used here, but make sure this has been initialized + assert(table_properties_); + + // Auto prefix mode is not implemented in PlainTable. + bool use_prefix_seek = !IsTotalOrderMode() && !options.total_order_seek && + !options.auto_prefix_mode; + if (arena == nullptr) { + return new PlainTableIterator(this, use_prefix_seek); + } else { + auto mem = arena->AllocateAligned(sizeof(PlainTableIterator)); + return new (mem) PlainTableIterator(this, use_prefix_seek); + } +} + +Status PlainTableReader::PopulateIndexRecordList( + PlainTableIndexBuilder* index_builder, + std::vector* prefix_hashes) { + Slice prev_key_prefix_slice; + std::string prev_key_prefix_buf; + uint32_t pos = data_start_offset_; + + bool is_first_record = true; + Slice key_prefix_slice; + PlainTableKeyDecoder decoder(&file_info_, encoding_type_, user_key_len_, + prefix_extractor_); + while (pos < file_info_.data_end_offset) { + uint32_t key_offset = pos; + ParsedInternalKey key; + Slice value_slice; + bool seekable = false; + Status s = Next(&decoder, &pos, &key, nullptr, &value_slice, &seekable); + if (!s.ok()) { + return s; + } + + key_prefix_slice = GetPrefix(key); + if (enable_bloom_) { + bloom_.AddHash(GetSliceHash(key.user_key)); + } else { + if (is_first_record || prev_key_prefix_slice != key_prefix_slice) { + if (!is_first_record) { + prefix_hashes->push_back(GetSliceHash(prev_key_prefix_slice)); + } + if (file_info_.is_mmap_mode) { + prev_key_prefix_slice = key_prefix_slice; + } else { + prev_key_prefix_buf = key_prefix_slice.ToString(); + prev_key_prefix_slice = prev_key_prefix_buf; + } + } + } + + index_builder->AddKeyPrefix(GetPrefix(key), key_offset); + + if (!seekable && is_first_record) { + return Status::Corruption("Key for a prefix is not seekable"); + } + + is_first_record = false; + } + + prefix_hashes->push_back(GetSliceHash(key_prefix_slice)); + auto s = index_.InitFromRawData(index_builder->Finish()); + return s; +} + +void PlainTableReader::AllocateBloom(int bloom_bits_per_key, int num_keys, + size_t huge_page_tlb_size) { + uint32_t bloom_total_bits = num_keys * bloom_bits_per_key; + if (bloom_total_bits > 0) { + enable_bloom_ = true; + bloom_.SetTotalBits(&arena_, bloom_total_bits, ioptions_.bloom_locality, + huge_page_tlb_size, ioptions_.logger); + } +} + +void PlainTableReader::FillBloom(const std::vector& prefix_hashes) { + assert(bloom_.IsInitialized()); + for (const auto prefix_hash : prefix_hashes) { + bloom_.AddHash(prefix_hash); + } +} + +Status PlainTableReader::MmapDataIfNeeded() { + if (file_info_.is_mmap_mode) { + // Get mmapped memory. + return file_info_.file->Read( + IOOptions(), 0, static_cast(file_size_), &file_info_.file_data, + nullptr, nullptr, Env::IO_TOTAL /* rate_limiter_priority */); + } + return Status::OK(); +} + +Status PlainTableReader::PopulateIndex(TableProperties* props, + int bloom_bits_per_key, + double hash_table_ratio, + size_t index_sparseness, + size_t huge_page_tlb_size) { + assert(props != nullptr); + + BlockContents index_block_contents; + + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + Status s = + ReadMetaBlock(file_info_.file.get(), nullptr /* prefetch_buffer */, + file_size_, kPlainTableMagicNumber, ioptions_, read_options, + PlainTableIndexBuilder::kPlainTableIndexBlock, + BlockType::kIndex, &index_block_contents); + + bool index_in_file = s.ok(); + + BlockContents bloom_block_contents; + bool bloom_in_file = false; + // We only need to read the bloom block if index block is in file. + if (index_in_file) { + s = ReadMetaBlock(file_info_.file.get(), nullptr /* prefetch_buffer */, + file_size_, kPlainTableMagicNumber, ioptions_, + read_options, BloomBlockBuilder::kBloomBlock, + BlockType::kFilter, &bloom_block_contents); + bloom_in_file = s.ok() && bloom_block_contents.data.size() > 0; + } + + Slice* bloom_block; + if (bloom_in_file) { + // If bloom_block_contents.allocation is not empty (which will be the case + // for non-mmap mode), it holds the alloated memory for the bloom block. + // It needs to be kept alive to keep `bloom_block` valid. + bloom_block_alloc_ = std::move(bloom_block_contents.allocation); + bloom_block = &bloom_block_contents.data; + } else { + bloom_block = nullptr; + } + + Slice* index_block; + if (index_in_file) { + // If index_block_contents.allocation is not empty (which will be the case + // for non-mmap mode), it holds the alloated memory for the index block. + // It needs to be kept alive to keep `index_block` valid. + index_block_alloc_ = std::move(index_block_contents.allocation); + index_block = &index_block_contents.data; + } else { + index_block = nullptr; + } + + if ((prefix_extractor_ == nullptr) && (hash_table_ratio != 0)) { + // moptions.prefix_extractor is requried for a hash-based look-up. + return Status::NotSupported( + "PlainTable requires a prefix extractor enable prefix hash mode."); + } + + // First, read the whole file, for every kIndexIntervalForSamePrefixKeys rows + // for a prefix (starting from the first one), generate a record of (hash, + // offset) and append it to IndexRecordList, which is a data structure created + // to store them. + + if (!index_in_file) { + // Allocate bloom filter here for total order mode. + if (IsTotalOrderMode()) { + AllocateBloom(bloom_bits_per_key, + static_cast(props->num_entries), + huge_page_tlb_size); + } + } else if (bloom_in_file) { + enable_bloom_ = true; + auto num_blocks_property = props->user_collected_properties.find( + PlainTablePropertyNames::kNumBloomBlocks); + + uint32_t num_blocks = 0; + if (num_blocks_property != props->user_collected_properties.end()) { + Slice temp_slice(num_blocks_property->second); + if (!GetVarint32(&temp_slice, &num_blocks)) { + num_blocks = 0; + } + } + // cast away const qualifier, because bloom_ won't be changed + bloom_.SetRawData(const_cast(bloom_block->data()), + static_cast(bloom_block->size()) * 8, + num_blocks); + } else { + // Index in file but no bloom in file. Disable bloom filter in this case. + enable_bloom_ = false; + bloom_bits_per_key = 0; + } + + PlainTableIndexBuilder index_builder(&arena_, ioptions_, prefix_extractor_, + index_sparseness, hash_table_ratio, + huge_page_tlb_size); + + std::vector prefix_hashes; + if (!index_in_file) { + // Populates _bloom if enabled (total order mode) + s = PopulateIndexRecordList(&index_builder, &prefix_hashes); + if (!s.ok()) { + return s; + } + } else { + s = index_.InitFromRawData(*index_block); + if (!s.ok()) { + return s; + } + } + + if (!index_in_file) { + if (!IsTotalOrderMode()) { + // Calculated bloom filter size and allocate memory for + // bloom filter based on the number of prefixes, then fill it. + AllocateBloom(bloom_bits_per_key, index_.GetNumPrefixes(), + huge_page_tlb_size); + if (enable_bloom_) { + FillBloom(prefix_hashes); + } + } + } + + // Fill two table properties. + if (!index_in_file) { + props->user_collected_properties["plain_table_hash_table_size"] = + std::to_string(index_.GetIndexSize() * PlainTableIndex::kOffsetLen); + props->user_collected_properties["plain_table_sub_index_size"] = + std::to_string(index_.GetSubIndexSize()); + } else { + props->user_collected_properties["plain_table_hash_table_size"] = + std::to_string(0); + props->user_collected_properties["plain_table_sub_index_size"] = + std::to_string(0); + } + + return Status::OK(); +} + +Status PlainTableReader::GetOffset(PlainTableKeyDecoder* decoder, + const Slice& target, const Slice& prefix, + uint32_t prefix_hash, bool& prefix_matched, + uint32_t* offset) const { + prefix_matched = false; + uint32_t prefix_index_offset; + auto res = index_.GetOffset(prefix_hash, &prefix_index_offset); + if (res == PlainTableIndex::kNoPrefixForBucket) { + *offset = file_info_.data_end_offset; + return Status::OK(); + } else if (res == PlainTableIndex::kDirectToFile) { + *offset = prefix_index_offset; + return Status::OK(); + } + + // point to sub-index, need to do a binary search + uint32_t upper_bound = 0; + const char* base_ptr = + index_.GetSubIndexBasePtrAndUpperBound(prefix_index_offset, &upper_bound); + uint32_t low = 0; + uint32_t high = upper_bound; + ParsedInternalKey mid_key; + ParsedInternalKey parsed_target; + Status s = ParseInternalKey(target, &parsed_target, + false /* log_err_key */); // TODO + if (!s.ok()) return s; + + // The key is between [low, high). Do a binary search between it. + while (high - low > 1) { + uint32_t mid = (high + low) / 2; + uint32_t file_offset = GetFixed32Element(base_ptr, mid); + uint32_t tmp; + s = decoder->NextKeyNoValue(file_offset, &mid_key, nullptr, &tmp); + if (!s.ok()) { + return s; + } + int cmp_result = internal_comparator_.Compare(mid_key, parsed_target); + if (cmp_result < 0) { + low = mid; + } else { + if (cmp_result == 0) { + // Happen to have found the exact key or target is smaller than the + // first key after base_offset. + prefix_matched = true; + *offset = file_offset; + return Status::OK(); + } else { + high = mid; + } + } + } + // Both of the key at the position low or low+1 could share the same + // prefix as target. We need to rule out one of them to avoid to go + // to the wrong prefix. + ParsedInternalKey low_key; + uint32_t tmp; + uint32_t low_key_offset = GetFixed32Element(base_ptr, low); + s = decoder->NextKeyNoValue(low_key_offset, &low_key, nullptr, &tmp); + if (!s.ok()) { + return s; + } + + if (GetPrefix(low_key) == prefix) { + prefix_matched = true; + *offset = low_key_offset; + } else if (low + 1 < upper_bound) { + // There is possible a next prefix, return it + prefix_matched = false; + *offset = GetFixed32Element(base_ptr, low + 1); + } else { + // target is larger than a key of the last prefix in this bucket + // but with a different prefix. Key does not exist. + *offset = file_info_.data_end_offset; + } + return Status::OK(); +} + +bool PlainTableReader::MatchBloom(uint32_t hash) const { + if (!enable_bloom_) { + return true; + } + + if (bloom_.MayContainHash(hash)) { + PERF_COUNTER_ADD(bloom_sst_hit_count, 1); + return true; + } else { + PERF_COUNTER_ADD(bloom_sst_miss_count, 1); + return false; + } +} + +Status PlainTableReader::Next(PlainTableKeyDecoder* decoder, uint32_t* offset, + ParsedInternalKey* parsed_key, + Slice* internal_key, Slice* value, + bool* seekable) const { + if (*offset == file_info_.data_end_offset) { + *offset = file_info_.data_end_offset; + return Status::OK(); + } + + if (*offset > file_info_.data_end_offset) { + return Status::Corruption("Offset is out of file size"); + } + + uint32_t bytes_read; + Status s = decoder->NextKey(*offset, parsed_key, internal_key, value, + &bytes_read, seekable); + if (!s.ok()) { + return s; + } + *offset = *offset + bytes_read; + return Status::OK(); +} + +void PlainTableReader::Prepare(const Slice& target) { + if (enable_bloom_) { + uint32_t prefix_hash = GetSliceHash(GetPrefix(target)); + bloom_.Prefetch(prefix_hash); + } +} + +Status PlainTableReader::Get(const ReadOptions& /*ro*/, const Slice& target, + GetContext* get_context, + const SliceTransform* /* prefix_extractor */, + bool /*skip_filters*/) { + // Check bloom filter first. + Slice prefix_slice; + uint32_t prefix_hash; + if (IsTotalOrderMode()) { + if (full_scan_mode_) { + status_ = + Status::InvalidArgument("Get() is not allowed in full scan mode."); + } + // Match whole user key for bloom filter check. + if (!MatchBloom(GetSliceHash(ExtractUserKey(target)))) { + return Status::OK(); + } + // in total order mode, there is only one bucket 0, and we always use empty + // prefix. + prefix_slice = Slice(); + prefix_hash = 0; + } else { + prefix_slice = GetPrefix(target); + prefix_hash = GetSliceHash(prefix_slice); + if (!MatchBloom(prefix_hash)) { + return Status::OK(); + } + } + uint32_t offset; + bool prefix_match; + PlainTableKeyDecoder decoder(&file_info_, encoding_type_, user_key_len_, + prefix_extractor_); + Status s = GetOffset(&decoder, target, prefix_slice, prefix_hash, + prefix_match, &offset); + + if (!s.ok()) { + return s; + } + ParsedInternalKey found_key; + ParsedInternalKey parsed_target; + s = ParseInternalKey(target, &parsed_target, + false /* log_err_key */); // TODO + if (!s.ok()) return s; + + Slice found_value; + while (offset < file_info_.data_end_offset) { + s = Next(&decoder, &offset, &found_key, nullptr, &found_value); + if (!s.ok()) { + return s; + } + if (!prefix_match) { + // Need to verify prefix for the first key found if it is not yet + // checked. + if (GetPrefix(found_key) != prefix_slice) { + return Status::OK(); + } + prefix_match = true; + } + // TODO(ljin): since we know the key comparison result here, + // can we enable the fast path? + if (internal_comparator_.Compare(found_key, parsed_target) >= 0) { + bool dont_care __attribute__((__unused__)); + if (!get_context->SaveValue(found_key, found_value, &dont_care, + dummy_cleanable_.get())) { + break; + } + } + } + return Status::OK(); +} + +uint64_t PlainTableReader::ApproximateOffsetOf( + const ReadOptions& /*read_options*/, const Slice& /*key*/, + TableReaderCaller /*caller*/) { + return 0; +} + +uint64_t PlainTableReader::ApproximateSize(const ReadOptions& /* read_options*/, + const Slice& /*start*/, + const Slice& /*end*/, + TableReaderCaller /*caller*/) { + return 0; +} + +PlainTableIterator::PlainTableIterator(PlainTableReader* table, + bool use_prefix_seek) + : table_(table), + decoder_(&table_->file_info_, table_->encoding_type_, + table_->user_key_len_, table_->prefix_extractor_), + use_prefix_seek_(use_prefix_seek) { + next_offset_ = offset_ = table_->file_info_.data_end_offset; +} + +PlainTableIterator::~PlainTableIterator() {} + +bool PlainTableIterator::Valid() const { + return offset_ < table_->file_info_.data_end_offset && + offset_ >= table_->data_start_offset_; +} + +void PlainTableIterator::SeekToFirst() { + status_ = Status::OK(); + next_offset_ = table_->data_start_offset_; + if (next_offset_ >= table_->file_info_.data_end_offset) { + next_offset_ = offset_ = table_->file_info_.data_end_offset; + } else { + Next(); + } +} + +void PlainTableIterator::SeekToLast() { + assert(false); + status_ = Status::NotSupported("SeekToLast() is not supported in PlainTable"); + next_offset_ = offset_ = table_->file_info_.data_end_offset; +} + +void PlainTableIterator::Seek(const Slice& target) { + if (use_prefix_seek_ != !table_->IsTotalOrderMode()) { + // This check is done here instead of NewIterator() to permit creating an + // iterator with total_order_seek = true even if we won't be able to Seek() + // it. This is needed for compaction: it creates iterator with + // total_order_seek = true but usually never does Seek() on it, + // only SeekToFirst(). + status_ = Status::InvalidArgument( + "total_order_seek not implemented for PlainTable."); + offset_ = next_offset_ = table_->file_info_.data_end_offset; + return; + } + + // If the user doesn't set prefix seek option and we are not able to do a + // total Seek(). assert failure. + if (table_->IsTotalOrderMode()) { + if (table_->full_scan_mode_) { + status_ = + Status::InvalidArgument("Seek() is not allowed in full scan mode."); + offset_ = next_offset_ = table_->file_info_.data_end_offset; + return; + } else if (table_->GetIndexSize() > 1) { + assert(false); + status_ = Status::NotSupported( + "PlainTable cannot issue non-prefix seek unless in total order " + "mode."); + offset_ = next_offset_ = table_->file_info_.data_end_offset; + return; + } + } + + Slice prefix_slice = table_->GetPrefix(target); + uint32_t prefix_hash = 0; + // Bloom filter is ignored in total-order mode. + if (!table_->IsTotalOrderMode()) { + prefix_hash = GetSliceHash(prefix_slice); + if (!table_->MatchBloom(prefix_hash)) { + status_ = Status::OK(); + offset_ = next_offset_ = table_->file_info_.data_end_offset; + return; + } + } + bool prefix_match; + status_ = table_->GetOffset(&decoder_, target, prefix_slice, prefix_hash, + prefix_match, &next_offset_); + if (!status_.ok()) { + offset_ = next_offset_ = table_->file_info_.data_end_offset; + return; + } + + if (next_offset_ < table_->file_info_.data_end_offset) { + for (Next(); status_.ok() && Valid(); Next()) { + if (!prefix_match) { + // Need to verify the first key's prefix + if (table_->GetPrefix(key()) != prefix_slice) { + offset_ = next_offset_ = table_->file_info_.data_end_offset; + break; + } + prefix_match = true; + } + if (table_->internal_comparator_.Compare(key(), target) >= 0) { + break; + } + } + } else { + offset_ = table_->file_info_.data_end_offset; + } +} + +void PlainTableIterator::SeekForPrev(const Slice& /*target*/) { + assert(false); + status_ = + Status::NotSupported("SeekForPrev() is not supported in PlainTable"); + offset_ = next_offset_ = table_->file_info_.data_end_offset; +} + +void PlainTableIterator::Next() { + offset_ = next_offset_; + if (offset_ < table_->file_info_.data_end_offset) { + Slice tmp_slice; + ParsedInternalKey parsed_key; + status_ = + table_->Next(&decoder_, &next_offset_, &parsed_key, &key_, &value_); + if (!status_.ok()) { + offset_ = next_offset_ = table_->file_info_.data_end_offset; + } + } +} + +void PlainTableIterator::Prev() { assert(false); } + +Slice PlainTableIterator::key() const { + assert(Valid()); + return key_; +} + +Slice PlainTableIterator::value() const { + assert(Valid()); + return value_; +} + +Status PlainTableIterator::status() const { return status_; } + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/plain/plain_table_reader.h b/librocksdb-sys/rocksdb/table/plain/plain_table_reader.h new file mode 100644 index 0000000..0f5f7f3 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/plain/plain_table_reader.h @@ -0,0 +1,243 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include + +#include +#include +#include +#include + +#include "file/random_access_file_reader.h" +#include "memory/arena.h" +#include "rocksdb/env.h" +#include "rocksdb/iterator.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/table.h" +#include "rocksdb/table_properties.h" +#include "table/plain/plain_table_bloom.h" +#include "table/plain/plain_table_factory.h" +#include "table/plain/plain_table_index.h" +#include "table/table_reader.h" + +namespace ROCKSDB_NAMESPACE { + +class Block; +struct BlockContents; +class BlockHandle; +class Footer; +struct Options; +class RandomAccessFile; +struct ReadOptions; +class TableCache; +class TableReader; +class InternalKeyComparator; +class PlainTableKeyDecoder; +class GetContext; + +extern const uint32_t kPlainTableVariableLength; + +struct PlainTableReaderFileInfo { + bool is_mmap_mode; + Slice file_data; + uint32_t data_end_offset; + std::unique_ptr file; + + PlainTableReaderFileInfo(std::unique_ptr&& _file, + const EnvOptions& storage_options, + uint32_t _data_size_offset) + : is_mmap_mode(storage_options.use_mmap_reads), + data_end_offset(_data_size_offset), + file(std::move(_file)) {} +}; + +// The reader class of PlainTable. For description of PlainTable format +// See comments of class PlainTableFactory, where instances of +// PlainTableReader are created. +class PlainTableReader : public TableReader { + public: + // Based on following output file format shown in plain_table_factory.h + // When opening the output file, PlainTableReader creates a hash table + // from key prefixes to offset of the output file. PlainTable will decide + // whether it points to the data offset of the first key with the key prefix + // or the offset of it. If there are too many keys share this prefix, it will + // create a binary search-able index from the suffix to offset on disk. + static Status Open(const ImmutableOptions& ioptions, + const EnvOptions& env_options, + const InternalKeyComparator& internal_comparator, + std::unique_ptr&& file, + uint64_t file_size, std::unique_ptr* table, + const int bloom_bits_per_key, double hash_table_ratio, + size_t index_sparseness, size_t huge_page_tlb_size, + bool full_scan_mode, const bool immortal_table = false, + const SliceTransform* prefix_extractor = nullptr); + + // Returns new iterator over table contents + // compaction_readahead_size: its value will only be used if for_compaction = + // true + InternalIterator* NewIterator(const ReadOptions&, + const SliceTransform* prefix_extractor, + Arena* arena, bool skip_filters, + TableReaderCaller caller, + size_t compaction_readahead_size = 0, + bool allow_unprepared_value = false) override; + + void Prepare(const Slice& target) override; + + Status Get(const ReadOptions& readOptions, const Slice& key, + GetContext* get_context, const SliceTransform* prefix_extractor, + bool skip_filters = false) override; + + uint64_t ApproximateOffsetOf(const ReadOptions& read_options, + const Slice& key, + TableReaderCaller caller) override; + + uint64_t ApproximateSize(const ReadOptions& read_options, const Slice& start, + const Slice& end, TableReaderCaller caller) override; + + uint32_t GetIndexSize() const { return index_.GetIndexSize(); } + void SetupForCompaction() override; + + std::shared_ptr GetTableProperties() const override { + return table_properties_; + } + + virtual size_t ApproximateMemoryUsage() const override { + return arena_.MemoryAllocatedBytes(); + } + + PlainTableReader(const ImmutableOptions& ioptions, + std::unique_ptr&& file, + const EnvOptions& env_options, + const InternalKeyComparator& internal_comparator, + EncodingType encoding_type, uint64_t file_size, + const TableProperties* table_properties, + const SliceTransform* prefix_extractor); + virtual ~PlainTableReader(); + + protected: + // Check bloom filter to see whether it might contain this prefix. + // The hash of the prefix is given, since it can be reused for index lookup + // too. + virtual bool MatchBloom(uint32_t hash) const; + + // PopulateIndex() builds index of keys. It must be called before any query + // to the table. + // + // props: the table properties object that need to be stored. Ownership of + // the object will be passed. + // + + Status PopulateIndex(TableProperties* props, int bloom_bits_per_key, + double hash_table_ratio, size_t index_sparseness, + size_t huge_page_tlb_size); + + Status MmapDataIfNeeded(); + + private: + const InternalKeyComparator internal_comparator_; + EncodingType encoding_type_; + // represents plain table's current status. + Status status_; + + PlainTableIndex index_; + bool full_scan_mode_; + + // data_start_offset_ and data_end_offset_ defines the range of the + // sst file that stores data. + const uint32_t data_start_offset_ = 0; + const uint32_t user_key_len_; + const SliceTransform* prefix_extractor_; + + static const size_t kNumInternalBytes = 8; + + // Bloom filter is used to rule out non-existent key + bool enable_bloom_; + PlainTableBloomV1 bloom_; + PlainTableReaderFileInfo file_info_; + Arena arena_; + CacheAllocationPtr index_block_alloc_; + CacheAllocationPtr bloom_block_alloc_; + + const ImmutableOptions& ioptions_; + std::unique_ptr dummy_cleanable_; + uint64_t file_size_; + + protected: // for testing + std::shared_ptr table_properties_; + + private: + bool IsFixedLength() const { + return user_key_len_ != kPlainTableVariableLength; + } + + size_t GetFixedInternalKeyLength() const { + return user_key_len_ + kNumInternalBytes; + } + + Slice GetPrefix(const Slice& target) const { + assert(target.size() >= 8); // target is internal key + return GetPrefixFromUserKey(ExtractUserKey(target)); + } + + Slice GetPrefix(const ParsedInternalKey& target) const { + return GetPrefixFromUserKey(target.user_key); + } + + Slice GetPrefixFromUserKey(const Slice& user_key) const { + if (!IsTotalOrderMode()) { + return prefix_extractor_->Transform(user_key); + } else { + // Use empty slice as prefix if prefix_extractor is not set. + // In that case, + // it falls back to pure binary search and + // total iterator seek is supported. + return Slice(); + } + } + + friend class TableCache; + friend class PlainTableIterator; + + // Internal helper function to generate an IndexRecordList object from all + // the rows, which contains index records as a list. + // If bloom_ is not null, all the keys' full-key hash will be added to the + // bloom filter. + Status PopulateIndexRecordList(PlainTableIndexBuilder* index_builder, + std::vector* prefix_hashes); + + // Internal helper function to allocate memory for bloom filter + void AllocateBloom(int bloom_bits_per_key, int num_prefixes, + size_t huge_page_tlb_size); + + void FillBloom(const std::vector& prefix_hashes); + + // Read the key and value at `offset` to parameters for keys, the and + // `seekable`. + // On success, `offset` will be updated as the offset for the next key. + // `parsed_key` will be key in parsed format. + // if `internal_key` is not empty, it will be filled with key with slice + // format. + // if `seekable` is not null, it will return whether we can directly read + // data using this offset. + Status Next(PlainTableKeyDecoder* decoder, uint32_t* offset, + ParsedInternalKey* parsed_key, Slice* internal_key, Slice* value, + bool* seekable = nullptr) const; + // Get file offset for key target. + // return value prefix_matched is set to true if the offset is confirmed + // for a key with the same prefix as target. + Status GetOffset(PlainTableKeyDecoder* decoder, const Slice& target, + const Slice& prefix, uint32_t prefix_hash, + bool& prefix_matched, uint32_t* offset) const; + + bool IsTotalOrderMode() const { return (prefix_extractor_ == nullptr); } + + // No copying allowed + explicit PlainTableReader(const TableReader&) = delete; + void operator=(const TableReader&) = delete; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/scoped_arena_iterator.h b/librocksdb-sys/rocksdb/table/scoped_arena_iterator.h new file mode 100644 index 0000000..2b8824d --- /dev/null +++ b/librocksdb-sys/rocksdb/table/scoped_arena_iterator.h @@ -0,0 +1,57 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#pragma once + +#include "port/port.h" +#include "table/internal_iterator.h" + +namespace ROCKSDB_NAMESPACE { +class ScopedArenaIterator { + void reset(InternalIterator* iter) noexcept { + if (iter_ != nullptr) { + iter_->~InternalIterator(); + } + iter_ = iter; + } + + public: + explicit ScopedArenaIterator(InternalIterator* iter = nullptr) + : iter_(iter) {} + + ScopedArenaIterator(const ScopedArenaIterator&) = delete; + ScopedArenaIterator& operator=(const ScopedArenaIterator&) = delete; + + ScopedArenaIterator(ScopedArenaIterator&& o) noexcept { + iter_ = o.iter_; + o.iter_ = nullptr; + } + + ScopedArenaIterator& operator=(ScopedArenaIterator&& o) noexcept { + reset(o.iter_); + o.iter_ = nullptr; + return *this; + } + + InternalIterator* operator->() { return iter_; } + InternalIterator* get() { return iter_; } + + void set(InternalIterator* iter) { reset(iter); } + + InternalIterator* release() { + assert(iter_ != nullptr); + auto* res = iter_; + iter_ = nullptr; + return res; + } + + ~ScopedArenaIterator() { reset(nullptr); } + + private: + InternalIterator* iter_; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/sst_file_dumper.cc b/librocksdb-sys/rocksdb/table/sst_file_dumper.cc new file mode 100644 index 0000000..85a264d --- /dev/null +++ b/librocksdb-sys/rocksdb/table/sst_file_dumper.cc @@ -0,0 +1,530 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#include "table/sst_file_dumper.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "db/blob/blob_index.h" +#include "db/memtable.h" +#include "db/write_batch_internal.h" +#include "options/cf_options.h" +#include "port/port.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/iterator.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/status.h" +#include "rocksdb/table_properties.h" +#include "rocksdb/utilities/ldb_cmd.h" +#include "table/block_based/block.h" +#include "table/block_based/block_based_table_builder.h" +#include "table/block_based/block_based_table_factory.h" +#include "table/block_based/block_builder.h" +#include "table/format.h" +#include "table/meta_blocks.h" +#include "table/plain/plain_table_factory.h" +#include "table/table_reader.h" +#include "util/compression.h" +#include "util/random.h" + +namespace ROCKSDB_NAMESPACE { + +SstFileDumper::SstFileDumper(const Options& options, + const std::string& file_path, + Temperature file_temp, size_t readahead_size, + bool verify_checksum, bool output_hex, + bool decode_blob_index, const EnvOptions& soptions, + bool silent) + : file_name_(file_path), + read_num_(0), + file_temp_(file_temp), + output_hex_(output_hex), + decode_blob_index_(decode_blob_index), + soptions_(soptions), + silent_(silent), + options_(options), + ioptions_(options_), + moptions_(ColumnFamilyOptions(options_)), + read_options_(verify_checksum, false), + internal_comparator_(BytewiseComparator()) { + read_options_.readahead_size = readahead_size; + if (!silent_) { + fprintf(stdout, "Process %s\n", file_path.c_str()); + } + init_result_ = GetTableReader(file_name_); +} + +extern const uint64_t kBlockBasedTableMagicNumber; +extern const uint64_t kLegacyBlockBasedTableMagicNumber; +extern const uint64_t kPlainTableMagicNumber; +extern const uint64_t kLegacyPlainTableMagicNumber; + +const char* testFileName = "test_file_name"; + +Status SstFileDumper::GetTableReader(const std::string& file_path) { + // Warning about 'magic_number' being uninitialized shows up only in UBsan + // builds. Though access is guarded by 's.ok()' checks, fix the issue to + // avoid any warnings. + uint64_t magic_number = Footer::kNullTableMagicNumber; + + // read table magic number + Footer footer; + + const auto& fs = options_.env->GetFileSystem(); + std::unique_ptr file; + uint64_t file_size = 0; + FileOptions fopts = soptions_; + fopts.temperature = file_temp_; + Status s = fs->NewRandomAccessFile(file_path, fopts, &file, nullptr); + if (s.ok()) { + s = fs->GetFileSize(file_path, IOOptions(), &file_size, nullptr); + } + + // check empty file + // if true, skip further processing of this file + if (file_size == 0) { + return Status::Aborted(file_path, "Empty file"); + } + + file_.reset(new RandomAccessFileReader(std::move(file), file_path)); + + FilePrefetchBuffer prefetch_buffer( + 0 /* readahead_size */, 0 /* max_readahead_size */, true /* enable */, + false /* track_min_offset */); + if (s.ok()) { + const uint64_t kSstDumpTailPrefetchSize = 512 * 1024; + uint64_t prefetch_size = (file_size > kSstDumpTailPrefetchSize) + ? kSstDumpTailPrefetchSize + : file_size; + uint64_t prefetch_off = file_size - prefetch_size; + IOOptions opts; + s = prefetch_buffer.Prefetch(opts, file_.get(), prefetch_off, + static_cast(prefetch_size), + Env::IO_TOTAL /* rate_limiter_priority */); + + s = ReadFooterFromFile(opts, file_.get(), *fs, &prefetch_buffer, file_size, + &footer); + } + if (s.ok()) { + magic_number = footer.table_magic_number(); + } + + if (s.ok()) { + if (magic_number == kPlainTableMagicNumber || + magic_number == kLegacyPlainTableMagicNumber) { + soptions_.use_mmap_reads = true; + + fs->NewRandomAccessFile(file_path, fopts, &file, nullptr); + file_.reset(new RandomAccessFileReader(std::move(file), file_path)); + } + + // For old sst format, ReadTableProperties might fail but file can be read + if (ReadTableProperties(magic_number, file_.get(), file_size, + (magic_number == kBlockBasedTableMagicNumber) + ? &prefetch_buffer + : nullptr) + .ok()) { + s = SetTableOptionsByMagicNumber(magic_number); + if (s.ok()) { + if (table_properties_ && !table_properties_->comparator_name.empty()) { + ConfigOptions config_options; + const Comparator* user_comparator = nullptr; + s = Comparator::CreateFromString(config_options, + table_properties_->comparator_name, + &user_comparator); + if (s.ok()) { + assert(user_comparator); + internal_comparator_ = InternalKeyComparator(user_comparator); + } + } + } + } else { + s = SetOldTableOptions(); + } + options_.comparator = internal_comparator_.user_comparator(); + } + + if (s.ok()) { + s = NewTableReader(ioptions_, soptions_, internal_comparator_, file_size, + &table_reader_); + } + return s; +} + +Status SstFileDumper::NewTableReader( + const ImmutableOptions& /*ioptions*/, const EnvOptions& /*soptions*/, + const InternalKeyComparator& /*internal_comparator*/, uint64_t file_size, + std::unique_ptr* /*table_reader*/) { + // TODO(yuzhangyu): full support in sst_dump for SST files generated when + // `user_defined_timestamps_persisted` is false. + auto t_opt = TableReaderOptions( + ioptions_, moptions_.prefix_extractor, soptions_, internal_comparator_, + 0 /* block_protection_bytes_per_key */, false /* skip_filters */, + false /* immortal */, true /* force_direct_prefetch */, -1 /* level */, + nullptr /* block_cache_tracer */, 0 /* max_file_size_for_l0_meta_pin */, + "" /* cur_db_session_id */, 0 /* cur_file_num */, {} /* unique_id */, + 0 /* largest_seqno */, 0 /* tail_size */, + table_properties_ == nullptr + ? true + : static_cast( + table_properties_->user_defined_timestamps_persisted)); + // Allow open file with global sequence number for backward compatibility. + t_opt.largest_seqno = kMaxSequenceNumber; + + // We need to turn off pre-fetching of index and filter nodes for + // BlockBasedTable + if (options_.table_factory->IsInstanceOf( + TableFactory::kBlockBasedTableName())) { + return options_.table_factory->NewTableReader(t_opt, std::move(file_), + file_size, &table_reader_, + /*enable_prefetch=*/false); + } + + // For all other factory implementation + return options_.table_factory->NewTableReader(t_opt, std::move(file_), + file_size, &table_reader_); +} + +Status SstFileDumper::VerifyChecksum() { + assert(read_options_.verify_checksums); + // We could pass specific readahead setting into read options if needed. + return table_reader_->VerifyChecksum(read_options_, + TableReaderCaller::kSSTDumpTool); +} + +Status SstFileDumper::DumpTable(const std::string& out_filename) { + std::unique_ptr out_file; + Env* env = options_.env; + Status s = env->NewWritableFile(out_filename, &out_file, soptions_); + if (s.ok()) { + s = table_reader_->DumpTable(out_file.get()); + } + if (!s.ok()) { + // close the file before return error, ignore the close error if there's any + out_file->Close().PermitUncheckedError(); + return s; + } + return out_file->Close(); +} + +Status SstFileDumper::CalculateCompressedTableSize( + const TableBuilderOptions& tb_options, size_t block_size, + uint64_t* num_data_blocks, uint64_t* compressed_table_size) { + std::unique_ptr env(NewMemEnv(options_.env)); + std::unique_ptr dest_writer; + Status s = + WritableFileWriter::Create(env->GetFileSystem(), testFileName, + FileOptions(soptions_), &dest_writer, nullptr); + if (!s.ok()) { + return s; + } + BlockBasedTableOptions table_options; + table_options.block_size = block_size; + BlockBasedTableFactory block_based_tf(table_options); + std::unique_ptr table_builder; + table_builder.reset( + block_based_tf.NewTableBuilder(tb_options, dest_writer.get())); + std::unique_ptr iter(table_reader_->NewIterator( + read_options_, moptions_.prefix_extractor.get(), /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kSSTDumpTool)); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + table_builder->Add(iter->key(), iter->value()); + } + s = iter->status(); + if (!s.ok()) { + return s; + } + s = table_builder->Finish(); + if (!s.ok()) { + return s; + } + *compressed_table_size = table_builder->FileSize(); + assert(num_data_blocks != nullptr); + *num_data_blocks = table_builder->GetTableProperties().num_data_blocks; + return env->DeleteFile(testFileName); +} + +Status SstFileDumper::ShowAllCompressionSizes( + size_t block_size, + const std::vector>& + compression_types, + int32_t compress_level_from, int32_t compress_level_to, + uint32_t max_dict_bytes, uint32_t zstd_max_train_bytes, + uint64_t max_dict_buffer_bytes, bool use_zstd_dict_trainer) { + fprintf(stdout, "Block Size: %" ROCKSDB_PRIszt "\n", block_size); + for (auto& i : compression_types) { + if (CompressionTypeSupported(i.first)) { + fprintf(stdout, "Compression: %-24s\n", i.second); + CompressionOptions compress_opt; + compress_opt.max_dict_bytes = max_dict_bytes; + compress_opt.zstd_max_train_bytes = zstd_max_train_bytes; + compress_opt.max_dict_buffer_bytes = max_dict_buffer_bytes; + compress_opt.use_zstd_dict_trainer = use_zstd_dict_trainer; + for (int32_t j = compress_level_from; j <= compress_level_to; j++) { + fprintf(stdout, "Compression level: %d", j); + compress_opt.level = j; + Status s = ShowCompressionSize(block_size, i.first, compress_opt); + if (!s.ok()) { + return s; + } + } + } else { + fprintf(stdout, "Unsupported compression type: %s.\n", i.second); + } + } + return Status::OK(); +} + +Status SstFileDumper::ShowCompressionSize( + size_t block_size, CompressionType compress_type, + const CompressionOptions& compress_opt) { + Options opts; + opts.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); + opts.statistics->set_stats_level(StatsLevel::kAll); + const ImmutableOptions imoptions(opts); + const ColumnFamilyOptions cfo(opts); + const MutableCFOptions moptions(cfo); + ROCKSDB_NAMESPACE::InternalKeyComparator ikc(opts.comparator); + IntTblPropCollectorFactories block_based_table_factories; + + std::string column_family_name; + int unknown_level = -1; + TableBuilderOptions tb_opts( + imoptions, moptions, ikc, &block_based_table_factories, compress_type, + compress_opt, + TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, + column_family_name, unknown_level); + uint64_t num_data_blocks = 0; + std::chrono::steady_clock::time_point start = + std::chrono::steady_clock::now(); + uint64_t file_size; + Status s = CalculateCompressedTableSize(tb_opts, block_size, &num_data_blocks, + &file_size); + if (!s.ok()) { + return s; + } + + std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now(); + fprintf(stdout, " Size: %10" PRIu64, file_size); + fprintf(stdout, " Blocks: %6" PRIu64, num_data_blocks); + fprintf(stdout, " Time Taken: %10s microsecs", + std::to_string( + std::chrono::duration_cast(end - start) + .count()) + .c_str()); + const uint64_t compressed_blocks = + opts.statistics->getAndResetTickerCount(NUMBER_BLOCK_COMPRESSED); + const uint64_t not_compressed_blocks = + opts.statistics->getAndResetTickerCount( + NUMBER_BLOCK_COMPRESSION_REJECTED); + // When the option enable_index_compression is true, + // NUMBER_BLOCK_COMPRESSED is incremented for index block(s). + if ((compressed_blocks + not_compressed_blocks) > num_data_blocks) { + num_data_blocks = compressed_blocks + not_compressed_blocks; + } + + const uint64_t ratio_not_compressed_blocks = + (num_data_blocks - compressed_blocks) - not_compressed_blocks; + const double compressed_pcnt = + (0 == num_data_blocks) ? 0.0 + : ((static_cast(compressed_blocks) / + static_cast(num_data_blocks)) * + 100.0); + const double ratio_not_compressed_pcnt = + (0 == num_data_blocks) + ? 0.0 + : ((static_cast(ratio_not_compressed_blocks) / + static_cast(num_data_blocks)) * + 100.0); + const double not_compressed_pcnt = + (0 == num_data_blocks) ? 0.0 + : ((static_cast(not_compressed_blocks) / + static_cast(num_data_blocks)) * + 100.0); + fprintf(stdout, " Compressed: %6" PRIu64 " (%5.1f%%)", compressed_blocks, + compressed_pcnt); + fprintf(stdout, " Not compressed (ratio): %6" PRIu64 " (%5.1f%%)", + ratio_not_compressed_blocks, ratio_not_compressed_pcnt); + fprintf(stdout, " Not compressed (abort): %6" PRIu64 " (%5.1f%%)\n", + not_compressed_blocks, not_compressed_pcnt); + return Status::OK(); +} + +// Reads TableProperties prior to opening table reader in order to set up +// options. +Status SstFileDumper::ReadTableProperties(uint64_t table_magic_number, + RandomAccessFileReader* file, + uint64_t file_size, + FilePrefetchBuffer* prefetch_buffer) { + // TODO: plumb Env::IOActivity + const ReadOptions read_options; + Status s = ROCKSDB_NAMESPACE::ReadTableProperties( + file, file_size, table_magic_number, ioptions_, read_options, + &table_properties_, + /* memory_allocator= */ nullptr, prefetch_buffer); + if (!s.ok()) { + if (!silent_) { + fprintf(stdout, "Not able to read table properties\n"); + } + } + return s; +} + +Status SstFileDumper::SetTableOptionsByMagicNumber( + uint64_t table_magic_number) { + assert(table_properties_); + if (table_magic_number == kBlockBasedTableMagicNumber || + table_magic_number == kLegacyBlockBasedTableMagicNumber) { + BlockBasedTableFactory* bbtf = new BlockBasedTableFactory(); + // To force tail prefetching, we fake reporting two useful reads of 512KB + // from the tail. + // It needs at least two data points to warm up the stats. + bbtf->tail_prefetch_stats()->RecordEffectiveSize(512 * 1024); + bbtf->tail_prefetch_stats()->RecordEffectiveSize(512 * 1024); + + options_.table_factory.reset(bbtf); + if (!silent_) { + fprintf(stdout, "Sst file format: block-based\n"); + } + + auto& props = table_properties_->user_collected_properties; + auto pos = props.find(BlockBasedTablePropertyNames::kIndexType); + if (pos != props.end()) { + auto index_type_on_file = static_cast( + DecodeFixed32(pos->second.c_str())); + if (index_type_on_file == + BlockBasedTableOptions::IndexType::kHashSearch) { + options_.prefix_extractor.reset(NewNoopTransform()); + } + } + } else if (table_magic_number == kPlainTableMagicNumber || + table_magic_number == kLegacyPlainTableMagicNumber) { + options_.allow_mmap_reads = true; + + PlainTableOptions plain_table_options; + plain_table_options.user_key_len = kPlainTableVariableLength; + plain_table_options.bloom_bits_per_key = 0; + plain_table_options.hash_table_ratio = 0; + plain_table_options.index_sparseness = 1; + plain_table_options.huge_page_tlb_size = 0; + plain_table_options.encoding_type = kPlain; + plain_table_options.full_scan_mode = true; + + options_.table_factory.reset(NewPlainTableFactory(plain_table_options)); + if (!silent_) { + fprintf(stdout, "Sst file format: plain table\n"); + } + } else { + char error_msg_buffer[80]; + snprintf(error_msg_buffer, sizeof(error_msg_buffer) - 1, + "Unsupported table magic number --- %lx", + (long)table_magic_number); + return Status::InvalidArgument(error_msg_buffer); + } + + return Status::OK(); +} + +Status SstFileDumper::SetOldTableOptions() { + assert(table_properties_ == nullptr); + options_.table_factory = std::make_shared(); + if (!silent_) { + fprintf(stdout, "Sst file format: block-based(old version)\n"); + } + + return Status::OK(); +} + +Status SstFileDumper::ReadSequential(bool print_kv, uint64_t read_num, + bool has_from, const std::string& from_key, + bool has_to, const std::string& to_key, + bool use_from_as_prefix) { + if (!table_reader_) { + return init_result_; + } + + InternalIterator* iter = table_reader_->NewIterator( + read_options_, moptions_.prefix_extractor.get(), + /*arena=*/nullptr, /*skip_filters=*/false, + TableReaderCaller::kSSTDumpTool); + uint64_t i = 0; + if (has_from) { + InternalKey ikey; + ikey.SetMinPossibleForUserKey(from_key); + iter->Seek(ikey.Encode()); + } else { + iter->SeekToFirst(); + } + for (; iter->Valid(); iter->Next()) { + Slice key = iter->key(); + Slice value = iter->value(); + ++i; + if (read_num > 0 && i > read_num) break; + + ParsedInternalKey ikey; + Status pik_status = ParseInternalKey(key, &ikey, true /* log_err_key */); + if (!pik_status.ok()) { + std::cerr << pik_status.getState() << "\n"; + continue; + } + + // the key returned is not prefixed with out 'from' key + if (use_from_as_prefix && !ikey.user_key.starts_with(from_key)) { + break; + } + + // If end marker was specified, we stop before it + if (has_to && BytewiseComparator()->Compare(ikey.user_key, to_key) >= 0) { + break; + } + + if (print_kv) { + if (!decode_blob_index_ || ikey.type != kTypeBlobIndex) { + fprintf(stdout, "%s => %s\n", + ikey.DebugString(true, output_hex_).c_str(), + value.ToString(output_hex_).c_str()); + } else { + BlobIndex blob_index; + + const Status s = blob_index.DecodeFrom(value); + if (!s.ok()) { + fprintf(stderr, "%s => error decoding blob index\n", + ikey.DebugString(true, output_hex_).c_str()); + continue; + } + + fprintf(stdout, "%s => %s\n", + ikey.DebugString(true, output_hex_).c_str(), + blob_index.DebugString(output_hex_).c_str()); + } + } + } + + read_num_ += i; + + Status ret = iter->status(); + delete iter; + return ret; +} + +// Provides TableProperties to API user +Status SstFileDumper::ReadTableProperties( + std::shared_ptr* table_properties) { + if (!table_reader_) { + return init_result_; + } + + *table_properties = table_reader_->GetTableProperties(); + return init_result_; +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/sst_file_dumper.h b/librocksdb-sys/rocksdb/table/sst_file_dumper.h new file mode 100644 index 0000000..1e78959 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/sst_file_dumper.h @@ -0,0 +1,99 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include +#include + +#include "db/dbformat.h" +#include "file/writable_file_writer.h" +#include "options/cf_options.h" +#include "rocksdb/advanced_options.h" + +namespace ROCKSDB_NAMESPACE { + +class SstFileDumper { + public: + explicit SstFileDumper(const Options& options, const std::string& file_name, + Temperature file_temp, size_t readahead_size, + bool verify_checksum, bool output_hex, + bool decode_blob_index, + const EnvOptions& soptions = EnvOptions(), + bool silent = false); + + Status ReadSequential(bool print_kv, uint64_t read_num, bool has_from, + const std::string& from_key, bool has_to, + const std::string& to_key, + bool use_from_as_prefix = false); + + Status ReadTableProperties( + std::shared_ptr* table_properties); + uint64_t GetReadNumber() { return read_num_; } + TableProperties* GetInitTableProperties() { return table_properties_.get(); } + + Status VerifyChecksum(); + Status DumpTable(const std::string& out_filename); + Status getStatus() { return init_result_; } + + Status ShowAllCompressionSizes( + size_t block_size, + const std::vector>& + compression_types, + int32_t compress_level_from, int32_t compress_level_to, + uint32_t max_dict_bytes, uint32_t zstd_max_train_bytes, + uint64_t max_dict_buffer_bytes, bool use_zstd_dict_trainer); + + Status ShowCompressionSize(size_t block_size, CompressionType compress_type, + const CompressionOptions& compress_opt); + + private: + // Get the TableReader implementation for the sst file + Status GetTableReader(const std::string& file_path); + Status ReadTableProperties(uint64_t table_magic_number, + RandomAccessFileReader* file, uint64_t file_size, + FilePrefetchBuffer* prefetch_buffer); + + Status CalculateCompressedTableSize(const TableBuilderOptions& tb_options, + size_t block_size, + uint64_t* num_data_blocks, + uint64_t* compressed_table_size); + + Status SetTableOptionsByMagicNumber(uint64_t table_magic_number); + Status SetOldTableOptions(); + + // Helper function to call the factory with settings specific to the + // factory implementation + Status NewTableReader(const ImmutableOptions& ioptions, + const EnvOptions& soptions, + const InternalKeyComparator& internal_comparator, + uint64_t file_size, + std::unique_ptr* table_reader); + + std::string file_name_; + uint64_t read_num_; + Temperature file_temp_; + bool output_hex_; + bool decode_blob_index_; + EnvOptions soptions_; + // less verbose in stdout/stderr + bool silent_; + + // options_ and internal_comparator_ will also be used in + // ReadSequential internally (specifically, seek-related operations) + Options options_; + + Status init_result_; + std::unique_ptr table_reader_; + std::unique_ptr file_; + + const ImmutableOptions ioptions_; + const MutableCFOptions moptions_; + ReadOptions read_options_; + InternalKeyComparator internal_comparator_; + std::unique_ptr table_properties_; +}; + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/table/sst_file_reader.cc b/librocksdb-sys/rocksdb/table/sst_file_reader.cc new file mode 100644 index 0000000..533b7cd --- /dev/null +++ b/librocksdb-sys/rocksdb/table/sst_file_reader.cc @@ -0,0 +1,101 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + + +#include "rocksdb/sst_file_reader.h" + +#include "db/arena_wrapped_db_iter.h" +#include "db/db_iter.h" +#include "db/dbformat.h" +#include "file/random_access_file_reader.h" +#include "options/cf_options.h" +#include "rocksdb/env.h" +#include "rocksdb/file_system.h" +#include "table/get_context.h" +#include "table/table_builder.h" +#include "table/table_reader.h" + +namespace ROCKSDB_NAMESPACE { + +struct SstFileReader::Rep { + Options options; + EnvOptions soptions; + ImmutableOptions ioptions; + MutableCFOptions moptions; + + std::unique_ptr table_reader; + + Rep(const Options& opts) + : options(opts), + soptions(options), + ioptions(options), + moptions(ColumnFamilyOptions(options)) {} +}; + +SstFileReader::SstFileReader(const Options& options) : rep_(new Rep(options)) {} + +SstFileReader::~SstFileReader() {} + +Status SstFileReader::Open(const std::string& file_path) { + auto r = rep_.get(); + Status s; + uint64_t file_size = 0; + std::unique_ptr file; + std::unique_ptr file_reader; + FileOptions fopts(r->soptions); + const auto& fs = r->options.env->GetFileSystem(); + + s = fs->GetFileSize(file_path, fopts.io_options, &file_size, nullptr); + if (s.ok()) { + s = fs->NewRandomAccessFile(file_path, fopts, &file, nullptr); + } + if (s.ok()) { + file_reader.reset(new RandomAccessFileReader(std::move(file), file_path)); + } + if (s.ok()) { + TableReaderOptions t_opt(r->ioptions, r->moptions.prefix_extractor, + r->soptions, r->ioptions.internal_comparator, + r->moptions.block_protection_bytes_per_key); + // Allow open file with global sequence number for backward compatibility. + t_opt.largest_seqno = kMaxSequenceNumber; + s = r->options.table_factory->NewTableReader(t_opt, std::move(file_reader), + file_size, &r->table_reader); + } + return s; +} + +Iterator* SstFileReader::NewIterator(const ReadOptions& roptions) { + assert(roptions.io_activity == Env::IOActivity::kUnknown); + auto r = rep_.get(); + auto sequence = roptions.snapshot != nullptr + ? roptions.snapshot->GetSequenceNumber() + : kMaxSequenceNumber; + ArenaWrappedDBIter* res = new ArenaWrappedDBIter(); + res->Init(r->options.env, roptions, r->ioptions, r->moptions, + nullptr /* version */, sequence, + r->moptions.max_sequential_skip_in_iterations, + 0 /* version_number */, nullptr /* read_callback */, + nullptr /* db_impl */, nullptr /* cfd */, + true /* expose_blob_index */, false /* allow_refresh */); + auto internal_iter = r->table_reader->NewIterator( + res->GetReadOptions(), r->moptions.prefix_extractor.get(), + res->GetArena(), false /* skip_filters */, + TableReaderCaller::kSSTFileReader); + res->SetIterUnderDBIter(internal_iter); + return res; +} + +std::shared_ptr SstFileReader::GetTableProperties() + const { + return rep_->table_reader->GetTableProperties(); +} + +Status SstFileReader::VerifyChecksum(const ReadOptions& read_options) { + assert(read_options.io_activity == Env::IOActivity::kUnknown); + return rep_->table_reader->VerifyChecksum(read_options, + TableReaderCaller::kSSTFileReader); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/sst_file_reader_test.cc b/librocksdb-sys/rocksdb/table/sst_file_reader_test.cc new file mode 100644 index 0000000..ba81d78 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/sst_file_reader_test.cc @@ -0,0 +1,423 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + + +#include "rocksdb/sst_file_reader.h" + +#include + +#include "port/stack_trace.h" +#include "rocksdb/convenience.h" +#include "rocksdb/db.h" +#include "rocksdb/sst_file_writer.h" +#include "table/sst_file_writer_collectors.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "utilities/merge_operators.h" + +namespace ROCKSDB_NAMESPACE { + +std::string EncodeAsString(uint64_t v) { + char buf[16]; + snprintf(buf, sizeof(buf), "%08" PRIu64, v); + return std::string(buf); +} + +std::string EncodeAsUint64(uint64_t v) { + std::string dst; + PutFixed64(&dst, v); + return dst; +} + +class SstFileReaderTest : public testing::Test { + public: + SstFileReaderTest() { + options_.merge_operator = MergeOperators::CreateUInt64AddOperator(); + sst_name_ = test::PerThreadDBPath("sst_file"); + + Env* base_env = Env::Default(); + EXPECT_OK( + test::CreateEnvFromSystem(ConfigOptions(), &base_env, &env_guard_)); + EXPECT_NE(nullptr, base_env); + env_ = base_env; + options_.env = env_; + } + + ~SstFileReaderTest() { + Status s = env_->DeleteFile(sst_name_); + EXPECT_OK(s); + } + + void CreateFile(const std::string& file_name, + const std::vector& keys) { + SstFileWriter writer(soptions_, options_); + ASSERT_OK(writer.Open(file_name)); + for (size_t i = 0; i + 2 < keys.size(); i += 3) { + ASSERT_OK(writer.Put(keys[i], keys[i])); + ASSERT_OK(writer.Merge(keys[i + 1], EncodeAsUint64(i + 1))); + ASSERT_OK(writer.Delete(keys[i + 2])); + } + ASSERT_OK(writer.Finish()); + } + + void CheckFile(const std::string& file_name, + const std::vector& keys, + bool check_global_seqno = false) { + ReadOptions ropts; + SstFileReader reader(options_); + ASSERT_OK(reader.Open(file_name)); + ASSERT_OK(reader.VerifyChecksum()); + std::unique_ptr iter(reader.NewIterator(ropts)); + iter->SeekToFirst(); + for (size_t i = 0; i + 2 < keys.size(); i += 3) { + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(keys[i]), 0); + ASSERT_EQ(iter->value().compare(keys[i]), 0); + iter->Next(); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().compare(keys[i + 1]), 0); + ASSERT_EQ(iter->value().compare(EncodeAsUint64(i + 1)), 0); + iter->Next(); + } + ASSERT_FALSE(iter->Valid()); + if (check_global_seqno) { + auto properties = reader.GetTableProperties(); + ASSERT_TRUE(properties); + std::string hostname; + ASSERT_OK(env_->GetHostNameString(&hostname)); + ASSERT_EQ(properties->db_host_id, hostname); + auto& user_properties = properties->user_collected_properties; + ASSERT_TRUE( + user_properties.count(ExternalSstFilePropertyNames::kGlobalSeqno)); + } + } + + void CreateFileAndCheck(const std::vector& keys) { + CreateFile(sst_name_, keys); + CheckFile(sst_name_, keys); + } + + protected: + Options options_; + EnvOptions soptions_; + std::string sst_name_; + std::shared_ptr env_guard_; + Env* env_; +}; + +const uint64_t kNumKeys = 100; + +TEST_F(SstFileReaderTest, Basic) { + std::vector keys; + for (uint64_t i = 0; i < kNumKeys; i++) { + keys.emplace_back(EncodeAsString(i)); + } + CreateFileAndCheck(keys); +} + +TEST_F(SstFileReaderTest, Uint64Comparator) { + options_.comparator = test::Uint64Comparator(); + std::vector keys; + for (uint64_t i = 0; i < kNumKeys; i++) { + keys.emplace_back(EncodeAsUint64(i)); + } + CreateFileAndCheck(keys); +} + +TEST_F(SstFileReaderTest, ReadOptionsOutOfScope) { + // Repro a bug where the SstFileReader depended on its configured ReadOptions + // outliving it. + options_.comparator = test::Uint64Comparator(); + std::vector keys; + for (uint64_t i = 0; i < kNumKeys; i++) { + keys.emplace_back(EncodeAsUint64(i)); + } + CreateFile(sst_name_, keys); + + SstFileReader reader(options_); + ASSERT_OK(reader.Open(sst_name_)); + std::unique_ptr iter; + { + // Make sure ReadOptions go out of scope ASAP so we know the iterator + // operations do not depend on it. + ReadOptions ropts; + iter.reset(reader.NewIterator(ropts)); + } + iter->SeekToFirst(); + while (iter->Valid()) { + iter->Next(); + } +} + +TEST_F(SstFileReaderTest, ReadFileWithGlobalSeqno) { + std::vector keys; + for (uint64_t i = 0; i < kNumKeys; i++) { + keys.emplace_back(EncodeAsString(i)); + } + // Generate a SST file. + CreateFile(sst_name_, keys); + + // Ingest the file into a db, to assign it a global sequence number. + Options options; + options.create_if_missing = true; + std::string db_name = test::PerThreadDBPath("test_db"); + DB* db; + ASSERT_OK(DB::Open(options, db_name, &db)); + // Bump sequence number. + ASSERT_OK(db->Put(WriteOptions(), keys[0], "foo")); + ASSERT_OK(db->Flush(FlushOptions())); + // Ingest the file. + IngestExternalFileOptions ingest_options; + ingest_options.write_global_seqno = true; + ASSERT_OK(db->IngestExternalFile({sst_name_}, ingest_options)); + std::vector live_files; + uint64_t manifest_file_size = 0; + ASSERT_OK(db->GetLiveFiles(live_files, &manifest_file_size)); + // Get the ingested file. + std::string ingested_file; + for (auto& live_file : live_files) { + if (live_file.substr(live_file.size() - 4, std::string::npos) == ".sst") { + if (ingested_file.empty() || ingested_file < live_file) { + ingested_file = live_file; + } + } + } + ASSERT_FALSE(ingested_file.empty()); + delete db; + + // Verify the file can be open and read by SstFileReader. + CheckFile(db_name + ingested_file, keys, true /* check_global_seqno */); + + // Cleanup. + ASSERT_OK(DestroyDB(db_name, options)); +} + +TEST_F(SstFileReaderTest, TimestampSizeMismatch) { + SstFileWriter writer(soptions_, options_); + + ASSERT_OK(writer.Open(sst_name_)); + + // Comparator is not timestamp-aware; calls to APIs taking timestamps should + // fail. + ASSERT_NOK(writer.Put("key", EncodeAsUint64(100), "value")); + ASSERT_NOK(writer.Delete("another_key", EncodeAsUint64(200))); +} + +class SstFileReaderTimestampTest : public testing::Test { + public: + SstFileReaderTimestampTest() { + Env* env = Env::Default(); + EXPECT_OK(test::CreateEnvFromSystem(ConfigOptions(), &env, &env_guard_)); + EXPECT_NE(nullptr, env); + + options_.env = env; + + options_.comparator = test::BytewiseComparatorWithU64TsWrapper(); + + sst_name_ = test::PerThreadDBPath("sst_file_ts"); + } + + ~SstFileReaderTimestampTest() { + EXPECT_OK(options_.env->DeleteFile(sst_name_)); + } + + struct KeyValueDesc { + KeyValueDesc(std::string k, std::string ts, std::string v) + : key(std::move(k)), timestamp(std::move(ts)), value(std::move(v)) {} + + std::string key; + std::string timestamp; + std::string value; + }; + + struct InputKeyValueDesc : public KeyValueDesc { + InputKeyValueDesc(std::string k, std::string ts, std::string v, bool is_del, + bool use_contig_buf) + : KeyValueDesc(std::move(k), std::move(ts), std::move(v)), + is_delete(is_del), + use_contiguous_buffer(use_contig_buf) {} + + bool is_delete = false; + bool use_contiguous_buffer = false; + }; + + struct OutputKeyValueDesc : public KeyValueDesc { + OutputKeyValueDesc(std::string k, std::string ts, std::string v) + : KeyValueDesc(std::move(k), std::string(ts), std::string(v)) {} + }; + + void CreateFile(const std::vector& descs) { + SstFileWriter writer(soptions_, options_); + + ASSERT_OK(writer.Open(sst_name_)); + + for (const auto& desc : descs) { + if (desc.is_delete) { + if (desc.use_contiguous_buffer) { + std::string key_with_ts(desc.key + desc.timestamp); + ASSERT_OK(writer.Delete(Slice(key_with_ts.data(), desc.key.size()), + Slice(key_with_ts.data() + desc.key.size(), + desc.timestamp.size()))); + } else { + ASSERT_OK(writer.Delete(desc.key, desc.timestamp)); + } + } else { + if (desc.use_contiguous_buffer) { + std::string key_with_ts(desc.key + desc.timestamp); + ASSERT_OK(writer.Put(Slice(key_with_ts.data(), desc.key.size()), + Slice(key_with_ts.data() + desc.key.size(), + desc.timestamp.size()), + desc.value)); + } else { + ASSERT_OK(writer.Put(desc.key, desc.timestamp, desc.value)); + } + } + } + + ASSERT_OK(writer.Finish()); + } + + void CheckFile(const std::string& timestamp, + const std::vector& descs) { + SstFileReader reader(options_); + + ASSERT_OK(reader.Open(sst_name_)); + ASSERT_OK(reader.VerifyChecksum()); + + Slice ts_slice(timestamp); + + ReadOptions read_options; + read_options.timestamp = &ts_slice; + + std::unique_ptr iter(reader.NewIterator(read_options)); + iter->SeekToFirst(); + + for (const auto& desc : descs) { + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key(), desc.key); + ASSERT_EQ(iter->timestamp(), desc.timestamp); + ASSERT_EQ(iter->value(), desc.value); + + iter->Next(); + } + + ASSERT_FALSE(iter->Valid()); + } + + protected: + std::shared_ptr env_guard_; + Options options_; + EnvOptions soptions_; + std::string sst_name_; +}; + +TEST_F(SstFileReaderTimestampTest, Basic) { + std::vector input_descs; + + for (uint64_t k = 0; k < kNumKeys; k += 4) { + // A Put with key k, timestamp k that gets overwritten by a subsequent Put + // with timestamp (k + 1). Note that the comparator uses descending order + // for the timestamp part, so we add the later Put first. + input_descs.emplace_back( + /* key */ EncodeAsString(k), /* timestamp */ EncodeAsUint64(k + 1), + /* value */ EncodeAsString(k * 2), /* is_delete */ false, + /* use_contiguous_buffer */ false); + input_descs.emplace_back( + /* key */ EncodeAsString(k), /* timestamp */ EncodeAsUint64(k), + /* value */ EncodeAsString(k * 3), /* is_delete */ false, + /* use_contiguous_buffer */ true); + + // A Put with key (k + 2), timestamp (k + 2) that gets cancelled out by a + // Delete with timestamp (k + 3). Note that the comparator uses descending + // order for the timestamp part, so we add the Delete first. + input_descs.emplace_back(/* key */ EncodeAsString(k + 2), + /* timestamp */ EncodeAsUint64(k + 3), + /* value */ std::string(), /* is_delete */ true, + /* use_contiguous_buffer */ (k % 8) == 0); + input_descs.emplace_back( + /* key */ EncodeAsString(k + 2), /* timestamp */ EncodeAsUint64(k + 2), + /* value */ EncodeAsString(k * 5), /* is_delete */ false, + /* use_contiguous_buffer */ (k % 8) != 0); + } + + CreateFile(input_descs); + + // Note: below, we check the results as of each timestamp in the range, + // updating the expected result as needed. + std::vector output_descs; + + for (uint64_t ts = 0; ts < kNumKeys; ++ts) { + const uint64_t k = ts - (ts % 4); + + switch (ts % 4) { + case 0: // Initial Put for key k + output_descs.emplace_back(/* key */ EncodeAsString(k), + /* timestamp */ EncodeAsUint64(ts), + /* value */ EncodeAsString(k * 3)); + break; + + case 1: // Second Put for key k + assert(output_descs.back().key == EncodeAsString(k)); + assert(output_descs.back().timestamp == EncodeAsUint64(ts - 1)); + assert(output_descs.back().value == EncodeAsString(k * 3)); + output_descs.back().timestamp = EncodeAsUint64(ts); + output_descs.back().value = EncodeAsString(k * 2); + break; + + case 2: // Put for key (k + 2) + output_descs.emplace_back(/* key */ EncodeAsString(k + 2), + /* timestamp */ EncodeAsUint64(ts), + /* value */ EncodeAsString(k * 5)); + break; + + case 3: // Delete for key (k + 2) + assert(output_descs.back().key == EncodeAsString(k + 2)); + assert(output_descs.back().timestamp == EncodeAsUint64(ts - 1)); + assert(output_descs.back().value == EncodeAsString(k * 5)); + output_descs.pop_back(); + break; + } + + CheckFile(EncodeAsUint64(ts), output_descs); + } +} + +TEST_F(SstFileReaderTimestampTest, TimestampsOutOfOrder) { + SstFileWriter writer(soptions_, options_); + + ASSERT_OK(writer.Open(sst_name_)); + + // Note: KVs that have the same user key disregarding timestamps should be in + // descending order of timestamps. + ASSERT_OK(writer.Put("key", EncodeAsUint64(1), "value1")); + ASSERT_NOK(writer.Put("key", EncodeAsUint64(2), "value2")); +} + +TEST_F(SstFileReaderTimestampTest, TimestampSizeMismatch) { + SstFileWriter writer(soptions_, options_); + + ASSERT_OK(writer.Open(sst_name_)); + + // Comparator expects 64-bit timestamps; timestamps with other sizes as well + // as calls to the timestamp-less APIs should be rejected. + ASSERT_NOK(writer.Put("key", "not_an_actual_64_bit_timestamp", "value")); + ASSERT_NOK(writer.Delete("another_key", "timestamp_of_unexpected_size")); + + ASSERT_NOK(writer.Put("key_without_timestamp", "value")); + ASSERT_NOK(writer.Merge("another_key_missing_a_timestamp", "merge_operand")); + ASSERT_NOK(writer.Delete("yet_another_key_still_no_timestamp")); + ASSERT_NOK(writer.DeleteRange("begin_key_timestamp_absent", + "end_key_with_a_complete_lack_of_timestamps")); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/librocksdb-sys/rocksdb/table/sst_file_writer.cc b/librocksdb-sys/rocksdb/table/sst_file_writer.cc new file mode 100644 index 0000000..d187b74 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/sst_file_writer.cc @@ -0,0 +1,436 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "rocksdb/sst_file_writer.h" + +#include + +#include "db/db_impl/db_impl.h" +#include "db/dbformat.h" +#include "file/writable_file_writer.h" +#include "rocksdb/file_system.h" +#include "rocksdb/table.h" +#include "table/block_based/block_based_table_builder.h" +#include "table/sst_file_writer_collectors.h" +#include "test_util/sync_point.h" + +namespace ROCKSDB_NAMESPACE { + +const std::string ExternalSstFilePropertyNames::kVersion = + "rocksdb.external_sst_file.version"; +const std::string ExternalSstFilePropertyNames::kGlobalSeqno = + "rocksdb.external_sst_file.global_seqno"; + + +const size_t kFadviseTrigger = 1024 * 1024; // 1MB + +struct SstFileWriter::Rep { + Rep(const EnvOptions& _env_options, const Options& options, + Env::IOPriority _io_priority, const Comparator* _user_comparator, + ColumnFamilyHandle* _cfh, bool _invalidate_page_cache, bool _skip_filters, + std::string _db_session_id) + : env_options(_env_options), + ioptions(options), + mutable_cf_options(options), + io_priority(_io_priority), + internal_comparator(_user_comparator), + cfh(_cfh), + invalidate_page_cache(_invalidate_page_cache), + skip_filters(_skip_filters), + db_session_id(_db_session_id) {} + + std::unique_ptr file_writer; + std::unique_ptr builder; + EnvOptions env_options; + ImmutableOptions ioptions; + MutableCFOptions mutable_cf_options; + Env::IOPriority io_priority; + InternalKeyComparator internal_comparator; + ExternalSstFileInfo file_info; + InternalKey ikey; + std::string column_family_name; + ColumnFamilyHandle* cfh; + // If true, We will give the OS a hint that this file pages is not needed + // every time we write 1MB to the file. + bool invalidate_page_cache; + // The size of the file during the last time we called Fadvise to remove + // cached pages from page cache. + uint64_t last_fadvise_size = 0; + bool skip_filters; + std::string db_session_id; + uint64_t next_file_number = 1; + + Status AddImpl(const Slice& user_key, const Slice& value, + ValueType value_type) { + if (!builder) { + return Status::InvalidArgument("File is not opened"); + } + + if (file_info.num_entries == 0) { + file_info.smallest_key.assign(user_key.data(), user_key.size()); + } else { + if (internal_comparator.user_comparator()->Compare( + user_key, file_info.largest_key) <= 0) { + // Make sure that keys are added in order + return Status::InvalidArgument( + "Keys must be added in strict ascending order."); + } + } + + assert(value_type == kTypeValue || value_type == kTypeMerge || + value_type == kTypeDeletion || + value_type == kTypeDeletionWithTimestamp); + + constexpr SequenceNumber sequence_number = 0; + + ikey.Set(user_key, sequence_number, value_type); + + builder->Add(ikey.Encode(), value); + + // update file info + file_info.num_entries++; + file_info.largest_key.assign(user_key.data(), user_key.size()); + file_info.file_size = builder->FileSize(); + + InvalidatePageCache(false /* closing */).PermitUncheckedError(); + return Status::OK(); + } + + Status Add(const Slice& user_key, const Slice& value, ValueType value_type) { + if (internal_comparator.user_comparator()->timestamp_size() != 0) { + return Status::InvalidArgument("Timestamp size mismatch"); + } + + return AddImpl(user_key, value, value_type); + } + + Status Add(const Slice& user_key, const Slice& timestamp, const Slice& value, + ValueType value_type) { + const size_t timestamp_size = timestamp.size(); + + if (internal_comparator.user_comparator()->timestamp_size() != + timestamp_size) { + return Status::InvalidArgument("Timestamp size mismatch"); + } + + const size_t user_key_size = user_key.size(); + + if (user_key.data() + user_key_size == timestamp.data()) { + Slice user_key_with_ts(user_key.data(), user_key_size + timestamp_size); + return AddImpl(user_key_with_ts, value, value_type); + } + + std::string user_key_with_ts; + user_key_with_ts.reserve(user_key_size + timestamp_size); + user_key_with_ts.append(user_key.data(), user_key_size); + user_key_with_ts.append(timestamp.data(), timestamp_size); + + return AddImpl(user_key_with_ts, value, value_type); + } + + Status DeleteRangeImpl(const Slice& begin_key, const Slice& end_key) { + if (!builder) { + return Status::InvalidArgument("File is not opened"); + } + int cmp = internal_comparator.user_comparator()->CompareWithoutTimestamp( + begin_key, end_key); + if (cmp > 0) { + // It's an empty range where endpoints appear mistaken. Don't bother + // applying it to the DB, and return an error to the user. + return Status::InvalidArgument("end key comes before start key"); + } else if (cmp == 0) { + // It's an empty range. Don't bother applying it to the DB. + return Status::OK(); + } + + RangeTombstone tombstone(begin_key, end_key, 0 /* Sequence Number */); + if (file_info.num_range_del_entries == 0) { + file_info.smallest_range_del_key.assign(tombstone.start_key_.data(), + tombstone.start_key_.size()); + file_info.largest_range_del_key.assign(tombstone.end_key_.data(), + tombstone.end_key_.size()); + } else { + if (internal_comparator.user_comparator()->Compare( + tombstone.start_key_, file_info.smallest_range_del_key) < 0) { + file_info.smallest_range_del_key.assign(tombstone.start_key_.data(), + tombstone.start_key_.size()); + } + if (internal_comparator.user_comparator()->Compare( + tombstone.end_key_, file_info.largest_range_del_key) > 0) { + file_info.largest_range_del_key.assign(tombstone.end_key_.data(), + tombstone.end_key_.size()); + } + } + + auto ikey_and_end_key = tombstone.Serialize(); + builder->Add(ikey_and_end_key.first.Encode(), ikey_and_end_key.second); + + // update file info + file_info.num_range_del_entries++; + file_info.file_size = builder->FileSize(); + + InvalidatePageCache(false /* closing */).PermitUncheckedError(); + return Status::OK(); + } + + Status DeleteRange(const Slice& begin_key, const Slice& end_key) { + if (internal_comparator.user_comparator()->timestamp_size() != 0) { + return Status::InvalidArgument("Timestamp size mismatch"); + } + return DeleteRangeImpl(begin_key, end_key); + } + + // begin_key and end_key should be users keys without timestamp. + Status DeleteRange(const Slice& begin_key, const Slice& end_key, + const Slice& timestamp) { + const size_t timestamp_size = timestamp.size(); + + if (internal_comparator.user_comparator()->timestamp_size() != + timestamp_size) { + return Status::InvalidArgument("Timestamp size mismatch"); + } + + const size_t begin_key_size = begin_key.size(); + const size_t end_key_size = end_key.size(); + if (begin_key.data() + begin_key_size == timestamp.data() || + end_key.data() + begin_key_size == timestamp.data()) { + assert(memcmp(begin_key.data() + begin_key_size, + end_key.data() + end_key_size, timestamp_size) == 0); + Slice begin_key_with_ts(begin_key.data(), + begin_key_size + timestamp_size); + Slice end_key_with_ts(end_key.data(), end_key.size() + timestamp_size); + return DeleteRangeImpl(begin_key_with_ts, end_key_with_ts); + } + std::string begin_key_with_ts; + begin_key_with_ts.reserve(begin_key_size + timestamp_size); + begin_key_with_ts.append(begin_key.data(), begin_key_size); + begin_key_with_ts.append(timestamp.data(), timestamp_size); + std::string end_key_with_ts; + end_key_with_ts.reserve(end_key_size + timestamp_size); + end_key_with_ts.append(end_key.data(), end_key_size); + end_key_with_ts.append(timestamp.data(), timestamp_size); + return DeleteRangeImpl(begin_key_with_ts, end_key_with_ts); + } + + Status InvalidatePageCache(bool closing) { + Status s = Status::OK(); + if (invalidate_page_cache == false) { + // Fadvise disabled + return s; + } + uint64_t bytes_since_last_fadvise = builder->FileSize() - last_fadvise_size; + if (bytes_since_last_fadvise > kFadviseTrigger || closing) { + TEST_SYNC_POINT_CALLBACK("SstFileWriter::Rep::InvalidatePageCache", + &(bytes_since_last_fadvise)); + // Tell the OS that we don't need this file in page cache + s = file_writer->InvalidateCache(0, 0); + if (s.IsNotSupported()) { + // NotSupported is fine as it could be a file type that doesn't use page + // cache. + s = Status::OK(); + } + last_fadvise_size = builder->FileSize(); + } + return s; + } +}; + +SstFileWriter::SstFileWriter(const EnvOptions& env_options, + const Options& options, + const Comparator* user_comparator, + ColumnFamilyHandle* column_family, + bool invalidate_page_cache, + Env::IOPriority io_priority, bool skip_filters) + : rep_(new Rep(env_options, options, io_priority, user_comparator, + column_family, invalidate_page_cache, skip_filters, + DBImpl::GenerateDbSessionId(options.env))) { + // SstFileWriter is used to create sst files that can be added to database + // later. Therefore, no real db_id and db_session_id are associated with it. + // Here we mimic the way db_session_id behaves by getting a db_session_id + // for each SstFileWriter, and (later below) assign unique file numbers + // in the table properties. The db_id is set to be "SST Writer" for clarity. + + rep_->file_info.file_size = 0; +} + +SstFileWriter::~SstFileWriter() { + if (rep_->builder) { + // User did not call Finish() or Finish() failed, we need to + // abandon the builder. + rep_->builder->Abandon(); + } +} + +Status SstFileWriter::Open(const std::string& file_path) { + Rep* r = rep_.get(); + Status s; + std::unique_ptr sst_file; + FileOptions cur_file_opts(r->env_options); + s = r->ioptions.env->GetFileSystem()->NewWritableFile( + file_path, cur_file_opts, &sst_file, nullptr); + if (!s.ok()) { + return s; + } + + sst_file->SetIOPriority(r->io_priority); + + CompressionType compression_type; + CompressionOptions compression_opts; + if (r->mutable_cf_options.bottommost_compression != + kDisableCompressionOption) { + compression_type = r->mutable_cf_options.bottommost_compression; + if (r->mutable_cf_options.bottommost_compression_opts.enabled) { + compression_opts = r->mutable_cf_options.bottommost_compression_opts; + } else { + compression_opts = r->mutable_cf_options.compression_opts; + } + } else if (!r->mutable_cf_options.compression_per_level.empty()) { + // Use the compression of the last level if we have per level compression + compression_type = *(r->mutable_cf_options.compression_per_level.rbegin()); + compression_opts = r->mutable_cf_options.compression_opts; + } else { + compression_type = r->mutable_cf_options.compression; + compression_opts = r->mutable_cf_options.compression_opts; + } + + IntTblPropCollectorFactories int_tbl_prop_collector_factories; + + // SstFileWriter properties collector to add SstFileWriter version. + int_tbl_prop_collector_factories.emplace_back( + new SstFileWriterPropertiesCollectorFactory(2 /* version */, + 0 /* global_seqno*/)); + + // User collector factories + auto user_collector_factories = + r->ioptions.table_properties_collector_factories; + for (size_t i = 0; i < user_collector_factories.size(); i++) { + int_tbl_prop_collector_factories.emplace_back( + new UserKeyTablePropertiesCollectorFactory( + user_collector_factories[i])); + } + int unknown_level = -1; + uint32_t cf_id; + + if (r->cfh != nullptr) { + // user explicitly specified that this file will be ingested into cfh, + // we can persist this information in the file. + cf_id = r->cfh->GetID(); + r->column_family_name = r->cfh->GetName(); + } else { + r->column_family_name = ""; + cf_id = TablePropertiesCollectorFactory::Context::kUnknownColumnFamily; + } + + // TODO: it would be better to set oldest_key_time to be used for getting the + // approximate time of ingested keys. + TableBuilderOptions table_builder_options( + r->ioptions, r->mutable_cf_options, r->internal_comparator, + &int_tbl_prop_collector_factories, compression_type, compression_opts, + cf_id, r->column_family_name, unknown_level, false /* is_bottommost */, + TableFileCreationReason::kMisc, 0 /* oldest_key_time */, + 0 /* file_creation_time */, "SST Writer" /* db_id */, r->db_session_id, + 0 /* target_file_size */, r->next_file_number); + // External SST files used to each get a unique session id. Now for + // slightly better uniqueness probability in constructing cache keys, we + // assign fake file numbers to each file (into table properties) and keep + // the same session id for the life of the SstFileWriter. + r->next_file_number++; + // XXX: when we can remove skip_filters from the SstFileWriter public API + // we can remove it from TableBuilderOptions. + table_builder_options.skip_filters = r->skip_filters; + FileTypeSet tmp_set = r->ioptions.checksum_handoff_file_types; + r->file_writer.reset(new WritableFileWriter( + std::move(sst_file), file_path, r->env_options, r->ioptions.clock, + nullptr /* io_tracer */, nullptr /* stats */, r->ioptions.listeners, + r->ioptions.file_checksum_gen_factory.get(), + tmp_set.Contains(FileType::kTableFile), false)); + + // TODO(tec) : If table_factory is using compressed block cache, we will + // be adding the external sst file blocks into it, which is wasteful. + r->builder.reset(r->ioptions.table_factory->NewTableBuilder( + table_builder_options, r->file_writer.get())); + + r->file_info = ExternalSstFileInfo(); + r->file_info.file_path = file_path; + r->file_info.version = 2; + return s; +} + +Status SstFileWriter::Add(const Slice& user_key, const Slice& value) { + return rep_->Add(user_key, value, ValueType::kTypeValue); +} + +Status SstFileWriter::Put(const Slice& user_key, const Slice& value) { + return rep_->Add(user_key, value, ValueType::kTypeValue); +} + +Status SstFileWriter::Put(const Slice& user_key, const Slice& timestamp, + const Slice& value) { + return rep_->Add(user_key, timestamp, value, ValueType::kTypeValue); +} + +Status SstFileWriter::Merge(const Slice& user_key, const Slice& value) { + return rep_->Add(user_key, value, ValueType::kTypeMerge); +} + +Status SstFileWriter::Delete(const Slice& user_key) { + return rep_->Add(user_key, Slice(), ValueType::kTypeDeletion); +} + +Status SstFileWriter::Delete(const Slice& user_key, const Slice& timestamp) { + return rep_->Add(user_key, timestamp, Slice(), + ValueType::kTypeDeletionWithTimestamp); +} + +Status SstFileWriter::DeleteRange(const Slice& begin_key, + const Slice& end_key) { + return rep_->DeleteRange(begin_key, end_key); +} + +Status SstFileWriter::DeleteRange(const Slice& begin_key, const Slice& end_key, + const Slice& timestamp) { + return rep_->DeleteRange(begin_key, end_key, timestamp); +} + +Status SstFileWriter::Finish(ExternalSstFileInfo* file_info) { + Rep* r = rep_.get(); + if (!r->builder) { + return Status::InvalidArgument("File is not opened"); + } + if (r->file_info.num_entries == 0 && + r->file_info.num_range_del_entries == 0) { + return Status::InvalidArgument("Cannot create sst file with no entries"); + } + + Status s = r->builder->Finish(); + r->file_info.file_size = r->builder->FileSize(); + + if (s.ok()) { + s = r->file_writer->Sync(r->ioptions.use_fsync); + r->InvalidatePageCache(true /* closing */).PermitUncheckedError(); + if (s.ok()) { + s = r->file_writer->Close(); + } + } + if (s.ok()) { + r->file_info.file_checksum = r->file_writer->GetFileChecksum(); + r->file_info.file_checksum_func_name = + r->file_writer->GetFileChecksumFuncName(); + } + if (!s.ok()) { + r->ioptions.env->DeleteFile(r->file_info.file_path); + } + + if (file_info != nullptr) { + *file_info = r->file_info; + } + + r->builder.reset(); + return s; +} + +uint64_t SstFileWriter::FileSize() { return rep_->file_info.file_size; } + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/sst_file_writer_collectors.h b/librocksdb-sys/rocksdb/table/sst_file_writer_collectors.h new file mode 100644 index 0000000..486315f --- /dev/null +++ b/librocksdb-sys/rocksdb/table/sst_file_writer_collectors.h @@ -0,0 +1,95 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once +#include + +#include "db/table_properties_collector.h" +#include "rocksdb/types.h" +#include "util/coding.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +// Table Properties that are specific to tables created by SstFileWriter. +struct ExternalSstFilePropertyNames { + // value of this property is a fixed uint32 number. + static const std::string kVersion; + // value of this property is a fixed uint64 number. + static const std::string kGlobalSeqno; +}; + +// PropertiesCollector used to add properties specific to tables +// generated by SstFileWriter +class SstFileWriterPropertiesCollector : public IntTblPropCollector { + public: + explicit SstFileWriterPropertiesCollector(int32_t version, + SequenceNumber global_seqno) + : version_(version), global_seqno_(global_seqno) {} + + virtual Status InternalAdd(const Slice& /*key*/, const Slice& /*value*/, + uint64_t /*file_size*/) override { + // Intentionally left blank. Have no interest in collecting stats for + // individual key/value pairs. + return Status::OK(); + } + + virtual void BlockAdd(uint64_t /* block_uncomp_bytes */, + uint64_t /* block_compressed_bytes_fast */, + uint64_t /* block_compressed_bytes_slow */) override { + // Intentionally left blank. No interest in collecting stats for + // blocks. + return; + } + + virtual Status Finish(UserCollectedProperties* properties) override { + // File version + std::string version_val; + PutFixed32(&version_val, static_cast(version_)); + properties->insert({ExternalSstFilePropertyNames::kVersion, version_val}); + + // Global Sequence number + std::string seqno_val; + PutFixed64(&seqno_val, static_cast(global_seqno_)); + properties->insert({ExternalSstFilePropertyNames::kGlobalSeqno, seqno_val}); + + return Status::OK(); + } + + virtual const char* Name() const override { + return "SstFileWriterPropertiesCollector"; + } + + virtual UserCollectedProperties GetReadableProperties() const override { + return {{ExternalSstFilePropertyNames::kVersion, std::to_string(version_)}}; + } + + private: + int32_t version_; + SequenceNumber global_seqno_; +}; + +class SstFileWriterPropertiesCollectorFactory + : public IntTblPropCollectorFactory { + public: + explicit SstFileWriterPropertiesCollectorFactory(int32_t version, + SequenceNumber global_seqno) + : version_(version), global_seqno_(global_seqno) {} + + virtual IntTblPropCollector* CreateIntTblPropCollector( + uint32_t /*column_family_id*/, int /* level_at_creation */) override { + return new SstFileWriterPropertiesCollector(version_, global_seqno_); + } + + virtual const char* Name() const override { + return "SstFileWriterPropertiesCollector"; + } + + private: + int32_t version_; + SequenceNumber global_seqno_; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/table_builder.h b/librocksdb-sys/rocksdb/table/table_builder.h new file mode 100644 index 0000000..d6f0e1a --- /dev/null +++ b/librocksdb-sys/rocksdb/table/table_builder.h @@ -0,0 +1,232 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#include + +#include +#include +#include + +#include "db/dbformat.h" +#include "db/seqno_to_time_mapping.h" +#include "db/table_properties_collector.h" +#include "file/writable_file_writer.h" +#include "options/cf_options.h" +#include "rocksdb/options.h" +#include "rocksdb/table_properties.h" +#include "table/unique_id_impl.h" +#include "trace_replay/block_cache_tracer.h" + +namespace ROCKSDB_NAMESPACE { + +class Slice; +class Status; + +struct TableReaderOptions { + // @param skip_filters Disables loading/accessing the filter block + TableReaderOptions( + const ImmutableOptions& _ioptions, + const std::shared_ptr& _prefix_extractor, + const EnvOptions& _env_options, + const InternalKeyComparator& _internal_comparator, + uint8_t _block_protection_bytes_per_key, bool _skip_filters = false, + bool _immortal = false, bool _force_direct_prefetch = false, + int _level = -1, BlockCacheTracer* const _block_cache_tracer = nullptr, + size_t _max_file_size_for_l0_meta_pin = 0, + const std::string& _cur_db_session_id = "", uint64_t _cur_file_num = 0, + UniqueId64x2 _unique_id = {}, SequenceNumber _largest_seqno = 0, + uint64_t _tail_size = 0, bool _user_defined_timestamps_persisted = true) + : ioptions(_ioptions), + prefix_extractor(_prefix_extractor), + env_options(_env_options), + internal_comparator(_internal_comparator), + skip_filters(_skip_filters), + immortal(_immortal), + force_direct_prefetch(_force_direct_prefetch), + level(_level), + largest_seqno(_largest_seqno), + block_cache_tracer(_block_cache_tracer), + max_file_size_for_l0_meta_pin(_max_file_size_for_l0_meta_pin), + cur_db_session_id(_cur_db_session_id), + cur_file_num(_cur_file_num), + unique_id(_unique_id), + block_protection_bytes_per_key(_block_protection_bytes_per_key), + tail_size(_tail_size), + user_defined_timestamps_persisted(_user_defined_timestamps_persisted) {} + + const ImmutableOptions& ioptions; + const std::shared_ptr& prefix_extractor; + const EnvOptions& env_options; + const InternalKeyComparator& internal_comparator; + // This is only used for BlockBasedTable (reader) + bool skip_filters; + // Whether the table will be valid as long as the DB is open + bool immortal; + // When data prefetching is needed, even if direct I/O is off, read data to + // fetch into RocksDB's buffer, rather than relying + // RandomAccessFile::Prefetch(). + bool force_direct_prefetch; + // What level this table/file is on, -1 for "not set, don't know." Used + // for level-specific statistics. + int level; + // largest seqno in the table (or 0 means unknown???) + SequenceNumber largest_seqno; + BlockCacheTracer* const block_cache_tracer; + // Largest L0 file size whose meta-blocks may be pinned (can be zero when + // unknown). + const size_t max_file_size_for_l0_meta_pin; + + std::string cur_db_session_id; + + uint64_t cur_file_num; + + // Known unique_id or {}, kNullUniqueId64x2 means unknown + UniqueId64x2 unique_id; + + uint8_t block_protection_bytes_per_key; + + uint64_t tail_size; + + // Whether the key in the table contains user-defined timestamps. + bool user_defined_timestamps_persisted; +}; + +struct TableBuilderOptions { + TableBuilderOptions( + const ImmutableOptions& _ioptions, const MutableCFOptions& _moptions, + const InternalKeyComparator& _internal_comparator, + const IntTblPropCollectorFactories* _int_tbl_prop_collector_factories, + CompressionType _compression_type, + const CompressionOptions& _compression_opts, uint32_t _column_family_id, + const std::string& _column_family_name, int _level, + bool _is_bottommost = false, + TableFileCreationReason _reason = TableFileCreationReason::kMisc, + const int64_t _oldest_key_time = 0, + const uint64_t _file_creation_time = 0, const std::string& _db_id = "", + const std::string& _db_session_id = "", + const uint64_t _target_file_size = 0, const uint64_t _cur_file_num = 0) + : ioptions(_ioptions), + moptions(_moptions), + internal_comparator(_internal_comparator), + int_tbl_prop_collector_factories(_int_tbl_prop_collector_factories), + compression_type(_compression_type), + compression_opts(_compression_opts), + column_family_id(_column_family_id), + column_family_name(_column_family_name), + oldest_key_time(_oldest_key_time), + target_file_size(_target_file_size), + file_creation_time(_file_creation_time), + db_id(_db_id), + db_session_id(_db_session_id), + level_at_creation(_level), + is_bottommost(_is_bottommost), + reason(_reason), + cur_file_num(_cur_file_num) {} + + const ImmutableOptions& ioptions; + const MutableCFOptions& moptions; + const InternalKeyComparator& internal_comparator; + const IntTblPropCollectorFactories* int_tbl_prop_collector_factories; + const CompressionType compression_type; + const CompressionOptions& compression_opts; + const uint32_t column_family_id; + const std::string& column_family_name; + const int64_t oldest_key_time; + const uint64_t target_file_size; + const uint64_t file_creation_time; + const std::string db_id; + const std::string db_session_id; + // BEGIN for FilterBuildingContext + const int level_at_creation; + const bool is_bottommost; + const TableFileCreationReason reason; + // END for FilterBuildingContext + + // XXX: only used by BlockBasedTableBuilder for SstFileWriter. If you + // want to skip filters, that should be (for example) null filter_policy + // in the table options of the ioptions.table_factory + bool skip_filters = false; + const uint64_t cur_file_num; +}; + +// TableBuilder provides the interface used to build a Table +// (an immutable and sorted map from keys to values). +// +// Multiple threads can invoke const methods on a TableBuilder without +// external synchronization, but if any of the threads may call a +// non-const method, all threads accessing the same TableBuilder must use +// external synchronization. +class TableBuilder { + public: + // REQUIRES: Either Finish() or Abandon() has been called. + virtual ~TableBuilder() {} + + // Add key,value to the table being constructed. + // REQUIRES: key is after any previously added key according to comparator. + // REQUIRES: Finish(), Abandon() have not been called + virtual void Add(const Slice& key, const Slice& value) = 0; + + // Return non-ok iff some error has been detected. + virtual Status status() const = 0; + + // Return non-ok iff some error happens during IO. + virtual IOStatus io_status() const = 0; + + // Finish building the table. + // REQUIRES: Finish(), Abandon() have not been called + virtual Status Finish() = 0; + + // Indicate that the contents of this builder should be abandoned. + // If the caller is not going to call Finish(), it must call Abandon() + // before destroying this builder. + // REQUIRES: Finish(), Abandon() have not been called + virtual void Abandon() = 0; + + // Number of calls to Add() so far. + virtual uint64_t NumEntries() const = 0; + + // Whether the output file is completely empty. It has neither entries + // or tombstones. + virtual bool IsEmpty() const { + return NumEntries() == 0 && GetTableProperties().num_range_deletions == 0; + } + + // Size of the file generated so far. If invoked after a successful + // Finish() call, returns the size of the final generated file. + virtual uint64_t FileSize() const = 0; + + // Estimated size of the file generated so far. This is used when + // FileSize() cannot estimate final SST size, e.g. parallel compression + // is enabled. + virtual uint64_t EstimatedFileSize() const { return FileSize(); } + + virtual uint64_t GetTailSize() const { return 0; } + + // If the user defined table properties collector suggest the file to + // be further compacted. + virtual bool NeedCompact() const { return false; } + + // Returns table properties + virtual TableProperties GetTableProperties() const = 0; + + // Return file checksum + virtual std::string GetFileChecksum() const = 0; + + // Return file checksum function name + virtual const char* GetFileChecksumFuncName() const = 0; + + // Set the sequence number to time mapping + virtual void SetSeqnoTimeTableProperties( + const std::string& /*encoded_seqno_to_time_mapping*/, + uint64_t /*oldest_ancestor_time*/){}; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/table_factory.cc b/librocksdb-sys/rocksdb/table/table_factory.cc new file mode 100644 index 0000000..29b6c89 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/table_factory.cc @@ -0,0 +1,52 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include + +#include "rocksdb/convenience.h" +#include "rocksdb/table.h" +#include "rocksdb/utilities/customizable_util.h" +#include "rocksdb/utilities/object_registry.h" +#include "table/block_based/block_based_table_factory.h" +#include "table/cuckoo/cuckoo_table_factory.h" +#include "table/plain/plain_table_factory.h" + +namespace ROCKSDB_NAMESPACE { + +static void RegisterTableFactories(const std::string& /*arg*/) { + static std::once_flag loaded; + std::call_once(loaded, []() { + auto library = ObjectLibrary::Default(); + library->AddFactory( + TableFactory::kBlockBasedTableName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new BlockBasedTableFactory()); + return guard->get(); + }); + library->AddFactory( + TableFactory::kPlainTableName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new PlainTableFactory()); + return guard->get(); + }); + library->AddFactory( + TableFactory::kCuckooTableName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new CuckooTableFactory()); + return guard->get(); + }); + }); +} + +Status TableFactory::CreateFromString(const ConfigOptions& config_options, + const std::string& value, + std::shared_ptr* factory) { + RegisterTableFactories(""); + return LoadSharedObject(config_options, value, factory); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/table_properties.cc b/librocksdb-sys/rocksdb/table/table_properties.cc new file mode 100644 index 0000000..06ea13f --- /dev/null +++ b/librocksdb-sys/rocksdb/table/table_properties.cc @@ -0,0 +1,353 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "rocksdb/table_properties.h" + +#include "db/seqno_to_time_mapping.h" +#include "port/malloc.h" +#include "port/port.h" +#include "rocksdb/env.h" +#include "rocksdb/unique_id.h" +#include "table/table_properties_internal.h" +#include "table/unique_id_impl.h" +#include "util/random.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +const uint32_t TablePropertiesCollectorFactory::Context::kUnknownColumnFamily = + std::numeric_limits::max(); + +namespace { +void AppendProperty(std::string& props, const std::string& key, + const std::string& value, const std::string& prop_delim, + const std::string& kv_delim) { + props.append(key); + props.append(kv_delim); + props.append(value); + props.append(prop_delim); +} + +template +void AppendProperty(std::string& props, const std::string& key, + const TValue& value, const std::string& prop_delim, + const std::string& kv_delim) { + AppendProperty(props, key, std::to_string(value), prop_delim, kv_delim); +} +} // namespace + +std::string TableProperties::ToString(const std::string& prop_delim, + const std::string& kv_delim) const { + std::string result; + result.reserve(1024); + + // Basic Info + AppendProperty(result, "# data blocks", num_data_blocks, prop_delim, + kv_delim); + AppendProperty(result, "# entries", num_entries, prop_delim, kv_delim); + AppendProperty(result, "# deletions", num_deletions, prop_delim, kv_delim); + AppendProperty(result, "# merge operands", num_merge_operands, prop_delim, + kv_delim); + AppendProperty(result, "# range deletions", num_range_deletions, prop_delim, + kv_delim); + + AppendProperty(result, "raw key size", raw_key_size, prop_delim, kv_delim); + AppendProperty(result, "raw average key size", + num_entries != 0 ? 1.0 * raw_key_size / num_entries : 0.0, + prop_delim, kv_delim); + AppendProperty(result, "raw value size", raw_value_size, prop_delim, + kv_delim); + AppendProperty(result, "raw average value size", + num_entries != 0 ? 1.0 * raw_value_size / num_entries : 0.0, + prop_delim, kv_delim); + + AppendProperty(result, "data block size", data_size, prop_delim, kv_delim); + char index_block_size_str[80]; + snprintf(index_block_size_str, sizeof(index_block_size_str), + "index block size (user-key? %d, delta-value? %d)", + static_cast(index_key_is_user_key), + static_cast(index_value_is_delta_encoded)); + AppendProperty(result, index_block_size_str, index_size, prop_delim, + kv_delim); + if (index_partitions != 0) { + AppendProperty(result, "# index partitions", index_partitions, prop_delim, + kv_delim); + AppendProperty(result, "top-level index size", top_level_index_size, + prop_delim, kv_delim); + } + AppendProperty(result, "filter block size", filter_size, prop_delim, + kv_delim); + AppendProperty(result, "# entries for filter", num_filter_entries, prop_delim, + kv_delim); + AppendProperty(result, "(estimated) table size", + data_size + index_size + filter_size, prop_delim, kv_delim); + + AppendProperty( + result, "filter policy name", + filter_policy_name.empty() ? std::string("N/A") : filter_policy_name, + prop_delim, kv_delim); + + AppendProperty(result, "prefix extractor name", + prefix_extractor_name.empty() ? std::string("N/A") + : prefix_extractor_name, + prop_delim, kv_delim); + + AppendProperty(result, "column family ID", + column_family_id == + ROCKSDB_NAMESPACE::TablePropertiesCollectorFactory:: + Context::kUnknownColumnFamily + ? std::string("N/A") + : std::to_string(column_family_id), + prop_delim, kv_delim); + AppendProperty( + result, "column family name", + column_family_name.empty() ? std::string("N/A") : column_family_name, + prop_delim, kv_delim); + + AppendProperty(result, "comparator name", + comparator_name.empty() ? std::string("N/A") : comparator_name, + prop_delim, kv_delim); + + AppendProperty( + result, "merge operator name", + merge_operator_name.empty() ? std::string("N/A") : merge_operator_name, + prop_delim, kv_delim); + + AppendProperty(result, "property collectors names", + property_collectors_names.empty() ? std::string("N/A") + : property_collectors_names, + prop_delim, kv_delim); + + AppendProperty( + result, "SST file compression algo", + compression_name.empty() ? std::string("N/A") : compression_name, + prop_delim, kv_delim); + + AppendProperty( + result, "SST file compression options", + compression_options.empty() ? std::string("N/A") : compression_options, + prop_delim, kv_delim); + + AppendProperty(result, "creation time", creation_time, prop_delim, kv_delim); + + AppendProperty(result, "time stamp of earliest key", oldest_key_time, + prop_delim, kv_delim); + + AppendProperty(result, "file creation time", file_creation_time, prop_delim, + kv_delim); + + AppendProperty(result, "slow compression estimated data size", + slow_compression_estimated_data_size, prop_delim, kv_delim); + AppendProperty(result, "fast compression estimated data size", + fast_compression_estimated_data_size, prop_delim, kv_delim); + + // DB identity and DB session ID + AppendProperty(result, "DB identity", db_id, prop_delim, kv_delim); + AppendProperty(result, "DB session identity", db_session_id, prop_delim, + kv_delim); + AppendProperty(result, "DB host id", db_host_id, prop_delim, kv_delim); + AppendProperty(result, "original file number", orig_file_number, prop_delim, + kv_delim); + + // Unique ID, when available + std::string id; + Status s = GetUniqueIdFromTableProperties(*this, &id); + AppendProperty(result, "unique ID", + s.ok() ? UniqueIdToHumanString(id) : "N/A", prop_delim, + kv_delim); + + SeqnoToTimeMapping seq_time_mapping; + s = seq_time_mapping.Add(seqno_to_time_mapping); + AppendProperty(result, "Sequence number to time mapping", + s.ok() ? seq_time_mapping.ToHumanString() : "N/A", prop_delim, + kv_delim); + + return result; +} + +void TableProperties::Add(const TableProperties& tp) { + data_size += tp.data_size; + index_size += tp.index_size; + index_partitions += tp.index_partitions; + top_level_index_size += tp.top_level_index_size; + index_key_is_user_key += tp.index_key_is_user_key; + index_value_is_delta_encoded += tp.index_value_is_delta_encoded; + filter_size += tp.filter_size; + raw_key_size += tp.raw_key_size; + raw_value_size += tp.raw_value_size; + num_data_blocks += tp.num_data_blocks; + num_entries += tp.num_entries; + num_filter_entries += tp.num_filter_entries; + num_deletions += tp.num_deletions; + num_merge_operands += tp.num_merge_operands; + num_range_deletions += tp.num_range_deletions; + slow_compression_estimated_data_size += + tp.slow_compression_estimated_data_size; + fast_compression_estimated_data_size += + tp.fast_compression_estimated_data_size; +} + +std::map +TableProperties::GetAggregatablePropertiesAsMap() const { + std::map rv; + rv["data_size"] = data_size; + rv["index_size"] = index_size; + rv["index_partitions"] = index_partitions; + rv["top_level_index_size"] = top_level_index_size; + rv["filter_size"] = filter_size; + rv["raw_key_size"] = raw_key_size; + rv["raw_value_size"] = raw_value_size; + rv["num_data_blocks"] = num_data_blocks; + rv["num_entries"] = num_entries; + rv["num_filter_entries"] = num_filter_entries; + rv["num_deletions"] = num_deletions; + rv["num_merge_operands"] = num_merge_operands; + rv["num_range_deletions"] = num_range_deletions; + rv["slow_compression_estimated_data_size"] = + slow_compression_estimated_data_size; + rv["fast_compression_estimated_data_size"] = + fast_compression_estimated_data_size; + return rv; +} + +// WARNING: manual update to this function is needed +// whenever a new string property is added to TableProperties +// to reduce approximation error. +// +// TODO: eliminate the need of manually updating this function +// for new string properties +std::size_t TableProperties::ApproximateMemoryUsage() const { + std::size_t usage = 0; +#ifdef ROCKSDB_MALLOC_USABLE_SIZE + usage += malloc_usable_size((void*)this); +#else + usage += sizeof(*this); +#endif // ROCKSDB_MALLOC_USABLE_SIZE + + std::size_t string_props_mem_usage = + db_id.size() + db_session_id.size() + db_host_id.size() + + column_family_name.size() + filter_policy_name.size() + + comparator_name.size() + merge_operator_name.size() + + prefix_extractor_name.size() + property_collectors_names.size() + + compression_name.size() + compression_options.size(); + usage += string_props_mem_usage; + + for (auto iter = user_collected_properties.begin(); + iter != user_collected_properties.end(); ++iter) { + usage += (iter->first.size() + iter->second.size()); + } + + return usage; +} + +const std::string TablePropertiesNames::kDbId = "rocksdb.creating.db.identity"; +const std::string TablePropertiesNames::kDbSessionId = + "rocksdb.creating.session.identity"; +const std::string TablePropertiesNames::kDbHostId = + "rocksdb.creating.host.identity"; +const std::string TablePropertiesNames::kOriginalFileNumber = + "rocksdb.original.file.number"; +const std::string TablePropertiesNames::kDataSize = "rocksdb.data.size"; +const std::string TablePropertiesNames::kIndexSize = "rocksdb.index.size"; +const std::string TablePropertiesNames::kIndexPartitions = + "rocksdb.index.partitions"; +const std::string TablePropertiesNames::kTopLevelIndexSize = + "rocksdb.top-level.index.size"; +const std::string TablePropertiesNames::kIndexKeyIsUserKey = + "rocksdb.index.key.is.user.key"; +const std::string TablePropertiesNames::kIndexValueIsDeltaEncoded = + "rocksdb.index.value.is.delta.encoded"; +const std::string TablePropertiesNames::kFilterSize = "rocksdb.filter.size"; +const std::string TablePropertiesNames::kRawKeySize = "rocksdb.raw.key.size"; +const std::string TablePropertiesNames::kRawValueSize = + "rocksdb.raw.value.size"; +const std::string TablePropertiesNames::kNumDataBlocks = + "rocksdb.num.data.blocks"; +const std::string TablePropertiesNames::kNumEntries = "rocksdb.num.entries"; +const std::string TablePropertiesNames::kNumFilterEntries = + "rocksdb.num.filter_entries"; +const std::string TablePropertiesNames::kDeletedKeys = "rocksdb.deleted.keys"; +const std::string TablePropertiesNames::kMergeOperands = + "rocksdb.merge.operands"; +const std::string TablePropertiesNames::kNumRangeDeletions = + "rocksdb.num.range-deletions"; +const std::string TablePropertiesNames::kFilterPolicy = "rocksdb.filter.policy"; +const std::string TablePropertiesNames::kFormatVersion = + "rocksdb.format.version"; +const std::string TablePropertiesNames::kFixedKeyLen = + "rocksdb.fixed.key.length"; +const std::string TablePropertiesNames::kColumnFamilyId = + "rocksdb.column.family.id"; +const std::string TablePropertiesNames::kColumnFamilyName = + "rocksdb.column.family.name"; +const std::string TablePropertiesNames::kComparator = "rocksdb.comparator"; +const std::string TablePropertiesNames::kMergeOperator = + "rocksdb.merge.operator"; +const std::string TablePropertiesNames::kPrefixExtractorName = + "rocksdb.prefix.extractor.name"; +const std::string TablePropertiesNames::kPropertyCollectors = + "rocksdb.property.collectors"; +const std::string TablePropertiesNames::kCompression = "rocksdb.compression"; +const std::string TablePropertiesNames::kCompressionOptions = + "rocksdb.compression_options"; +const std::string TablePropertiesNames::kCreationTime = "rocksdb.creation.time"; +const std::string TablePropertiesNames::kOldestKeyTime = + "rocksdb.oldest.key.time"; +const std::string TablePropertiesNames::kFileCreationTime = + "rocksdb.file.creation.time"; +const std::string TablePropertiesNames::kSlowCompressionEstimatedDataSize = + "rocksdb.sample_for_compression.slow.data.size"; +const std::string TablePropertiesNames::kFastCompressionEstimatedDataSize = + "rocksdb.sample_for_compression.fast.data.size"; +const std::string TablePropertiesNames::kSequenceNumberTimeMapping = + "rocksdb.seqno.time.map"; +const std::string TablePropertiesNames::kTailStartOffset = + "rocksdb.tail.start.offset"; +const std::string TablePropertiesNames::kUserDefinedTimestampsPersisted = + "rocksdb.user.defined.timestamps.persisted"; + +#ifndef NDEBUG +// WARNING: TEST_SetRandomTableProperties assumes the following layout of +// TableProperties +// +// struct TableProperties { +// int64_t orig_file_number = 0; +// ... +// ... int64_t properties only +// ... +// std::string db_id; +// ... +// ... std::string properties only +// ... +// std::string compression_options; +// UserCollectedProperties user_collected_properties; +// ... +// ... Other extra properties: non-int64_t/non-std::string properties only +// ... +// } +void TEST_SetRandomTableProperties(TableProperties* props) { + Random* r = Random::GetTLSInstance(); + uint64_t* pu = &props->orig_file_number; + assert(static_cast(pu) == static_cast(props)); + std::string* ps = &props->db_id; + const uint64_t* const pu_end = reinterpret_cast(ps); + // Use the last string property's address instead of + // the first extra property (e.g `user_collected_properties`)'s address + // in the for-loop to avoid advancing pointer to pointing to + // potential non-zero padding bytes between these two addresses due to + // user_collected_properties's alignment requirement + const std::string* const ps_end_inclusive = &props->compression_options; + + for (; pu < pu_end; ++pu) { + *pu = r->Next64(); + } + assert(static_cast(pu) == static_cast(ps)); + for (; ps <= ps_end_inclusive; ++ps) { + *ps = r->RandomBinaryString(13); + } +} +#endif + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/table_properties_internal.h b/librocksdb-sys/rocksdb/table/table_properties_internal.h new file mode 100644 index 0000000..5c2a0cb --- /dev/null +++ b/librocksdb-sys/rocksdb/table/table_properties_internal.h @@ -0,0 +1,14 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "rocksdb/table_properties.h" + +namespace ROCKSDB_NAMESPACE { +#ifndef NDEBUG +void TEST_SetRandomTableProperties(TableProperties* props); +#endif +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/table_reader.h b/librocksdb-sys/rocksdb/table/table_reader.h new file mode 100644 index 0000000..53c5220 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/table_reader.h @@ -0,0 +1,187 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include + +#include "db/range_tombstone_fragmenter.h" +#if USE_COROUTINES +#include "folly/experimental/coro/Coroutine.h" +#include "folly/experimental/coro/Task.h" +#endif +#include "rocksdb/slice_transform.h" +#include "rocksdb/table_reader_caller.h" +#include "table/get_context.h" +#include "table/internal_iterator.h" +#include "table/multiget_context.h" + +namespace ROCKSDB_NAMESPACE { + +class Iterator; +struct ParsedInternalKey; +class Slice; +class Arena; +struct ReadOptions; +struct TableProperties; +class GetContext; +class MultiGetContext; + +// A Table (also referred to as SST) is a sorted map from strings to strings. +// Tables are immutable and persistent. A Table may be safely accessed from +// multiple threads without external synchronization. Table readers are used +// for reading various types of table formats supported by rocksdb including +// BlockBasedTable, PlainTable and CuckooTable format. +class TableReader { + public: + virtual ~TableReader() {} + + // Returns a new iterator over the table contents. + // The result of NewIterator() is initially invalid (caller must + // call one of the Seek methods on the iterator before using it). + // + // read_options: Must outlive the returned iterator. + // arena: If not null, the arena needs to be used to allocate the Iterator. + // When destroying the iterator, the caller will not call "delete" + // but Iterator::~Iterator() directly. The destructor needs to destroy + // all the states but those allocated in arena. + // skip_filters: disables checking the bloom filters even if they exist. This + // option is effective only for block-based table format. + // compaction_readahead_size: its value will only be used if caller = + // kCompaction + virtual InternalIterator* NewIterator( + const ReadOptions& read_options, const SliceTransform* prefix_extractor, + Arena* arena, bool skip_filters, TableReaderCaller caller, + size_t compaction_readahead_size = 0, + bool allow_unprepared_value = false) = 0; + + virtual FragmentedRangeTombstoneIterator* NewRangeTombstoneIterator( + const ReadOptions& /*read_options*/) { + return nullptr; + } + + // Given a key, return an approximate byte offset in the file where + // the data for that key begins (or would begin if the key were + // present in the file). The returned value is in terms of file + // bytes, and so includes effects like compression of the underlying data. + // E.g., the approximate offset of the last key in the table will + // be close to the file length. + // TODO(peterd): Since this function is only used for approximate size + // from beginning of file, reduce code duplication by removing this + // function and letting ApproximateSize take optional start and end, so + // that absolute start and end can be specified and optimized without + // key / index work. + virtual uint64_t ApproximateOffsetOf(const ReadOptions& read_options, + const Slice& key, + TableReaderCaller caller) = 0; + + // Given start and end keys, return the approximate data size in the file + // between the keys. The returned value is in terms of file bytes, and so + // includes effects like compression of the underlying data and applicable + // portions of metadata including filters and indexes. Nullptr for start or + // end (or both) indicates absolute start or end of the table. + virtual uint64_t ApproximateSize(const ReadOptions& read_options, + const Slice& start, const Slice& end, + TableReaderCaller caller) = 0; + + struct Anchor { + Anchor(const Slice& _user_key, size_t _range_size) + : user_key(_user_key.ToStringView()), range_size(_range_size) {} + std::string user_key; + size_t range_size; + }; + + // Now try to return approximately 128 anchor keys. + // The last one tends to be the largest key. + virtual Status ApproximateKeyAnchors(const ReadOptions& /*read_options*/, + std::vector& /*anchors*/) { + return Status::NotSupported("ApproximateKeyAnchors() not supported."); + } + + // Set up the table for Compaction. Might change some parameters with + // posix_fadvise + virtual void SetupForCompaction() = 0; + + virtual std::shared_ptr GetTableProperties() const = 0; + + // Prepare work that can be done before the real Get() + virtual void Prepare(const Slice& /*target*/) {} + + // Report an approximation of how much memory has been used. + virtual size_t ApproximateMemoryUsage() const = 0; + + // Calls get_context->SaveValue() repeatedly, starting with + // the entry found after a call to Seek(key), until it returns false. + // May not make such a call if filter policy says that key is not present. + // + // get_context->MarkKeyMayExist needs to be called when it is configured to be + // memory only and the key is not found in the block cache. + // + // readOptions is the options for the read + // key is the key to search for + // skip_filters: disables checking the bloom filters even if they exist. This + // option is effective only for block-based table format. + virtual Status Get(const ReadOptions& readOptions, const Slice& key, + GetContext* get_context, + const SliceTransform* prefix_extractor, + bool skip_filters = false) = 0; + + // Use bloom filters in the table file, if present, to filter out keys. The + // mget_range will be updated to skip keys that get a negative result from + // the filter lookup. + virtual Status MultiGetFilter(const ReadOptions& /*readOptions*/, + const SliceTransform* /*prefix_extractor*/, + MultiGetContext::Range* /*mget_range*/) { + return Status::NotSupported(); + } + + virtual void MultiGet(const ReadOptions& readOptions, + const MultiGetContext::Range* mget_range, + const SliceTransform* prefix_extractor, + bool skip_filters = false) { + for (auto iter = mget_range->begin(); iter != mget_range->end(); ++iter) { + *iter->s = Get(readOptions, iter->ikey, iter->get_context, + prefix_extractor, skip_filters); + } + } + +#if USE_COROUTINES + virtual folly::coro::Task MultiGetCoroutine( + const ReadOptions& readOptions, const MultiGetContext::Range* mget_range, + const SliceTransform* prefix_extractor, bool skip_filters = false) { + MultiGet(readOptions, mget_range, prefix_extractor, skip_filters); + co_return; + } +#endif // USE_COROUTINES + + // Prefetch data corresponding to a give range of keys + // Typically this functionality is required for table implementations that + // persists the data on a non volatile storage medium like disk/SSD + virtual Status Prefetch(const ReadOptions& /* read_options */, + const Slice* begin = nullptr, + const Slice* end = nullptr) { + (void)begin; + (void)end; + // Default implementation is NOOP. + // The child class should implement functionality when applicable + return Status::OK(); + } + + // convert db file to a human readable form + virtual Status DumpTable(WritableFile* /*out_file*/) { + return Status::NotSupported("DumpTable() not supported"); + } + + // check whether there is corruption in this db file + virtual Status VerifyChecksum(const ReadOptions& /*read_options*/, + TableReaderCaller /*caller*/) { + return Status::NotSupported("VerifyChecksum() not supported"); + } +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/table_reader_bench.cc b/librocksdb-sys/rocksdb/table/table_reader_bench.cc new file mode 100644 index 0000000..60c84d7 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/table_reader_bench.cc @@ -0,0 +1,339 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#ifndef GFLAGS +#include +int main() { + fprintf(stderr, "Please install gflags to run rocksdb tools\n"); + return 1; +} +#else + +#include "db/db_impl/db_impl.h" +#include "db/dbformat.h" +#include "file/random_access_file_reader.h" +#include "monitoring/histogram.h" +#include "rocksdb/db.h" +#include "rocksdb/file_system.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/system_clock.h" +#include "rocksdb/table.h" +#include "table/block_based/block_based_table_factory.h" +#include "table/get_context.h" +#include "table/internal_iterator.h" +#include "table/plain/plain_table_factory.h" +#include "table/table_builder.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/gflags_compat.h" + +using GFLAGS_NAMESPACE::ParseCommandLineFlags; +using GFLAGS_NAMESPACE::SetUsageMessage; + +namespace ROCKSDB_NAMESPACE { + +namespace { +// Make a key that i determines the first 4 characters and j determines the +// last 4 characters. +static std::string MakeKey(int i, int j, bool through_db) { + char buf[100]; + snprintf(buf, sizeof(buf), "%04d__key___%04d", i, j); + if (through_db) { + return std::string(buf); + } + // If we directly query table, which operates on internal keys + // instead of user keys, we need to add 8 bytes of internal + // information (row type etc) to user key to make an internal + // key. + InternalKey key(std::string(buf), 0, ValueType::kTypeValue); + return key.Encode().ToString(); +} + +uint64_t Now(SystemClock* clock, bool measured_by_nanosecond) { + return measured_by_nanosecond ? clock->NowNanos() : clock->NowMicros(); +} +} // namespace + +// A very simple benchmark that. +// Create a table with roughly numKey1 * numKey2 keys, +// where there are numKey1 prefixes of the key, each has numKey2 number of +// distinguished key, differing in the suffix part. +// If if_query_empty_keys = false, query the existing keys numKey1 * numKey2 +// times randomly. +// If if_query_empty_keys = true, query numKey1 * numKey2 random empty keys. +// Print out the total time. +// If through_db=true, a full DB will be created and queries will be against +// it. Otherwise, operations will be directly through table level. +// +// If for_terator=true, instead of just query one key each time, it queries +// a range sharing the same prefix. +namespace { +void TableReaderBenchmark(Options& opts, EnvOptions& env_options, + ReadOptions& read_options, int num_keys1, + int num_keys2, int num_iter, int /*prefix_len*/, + bool if_query_empty_keys, bool for_iterator, + bool through_db, bool measured_by_nanosecond) { + ROCKSDB_NAMESPACE::InternalKeyComparator ikc(opts.comparator); + + std::string file_name = + test::PerThreadDBPath("rocksdb_table_reader_benchmark"); + std::string dbname = test::PerThreadDBPath("rocksdb_table_reader_bench_db"); + WriteOptions wo; + Env* env = Env::Default(); + auto* clock = env->GetSystemClock().get(); + TableBuilder* tb = nullptr; + DB* db = nullptr; + Status s; + const ImmutableOptions ioptions(opts); + const ColumnFamilyOptions cfo(opts); + const MutableCFOptions moptions(cfo); + std::unique_ptr file_writer; + if (!through_db) { + ASSERT_OK(WritableFileWriter::Create(env->GetFileSystem(), file_name, + FileOptions(env_options), &file_writer, + nullptr)); + + IntTblPropCollectorFactories int_tbl_prop_collector_factories; + + int unknown_level = -1; + tb = opts.table_factory->NewTableBuilder( + TableBuilderOptions( + ioptions, moptions, ikc, &int_tbl_prop_collector_factories, + CompressionType::kNoCompression, CompressionOptions(), + 0 /* column_family_id */, kDefaultColumnFamilyName, unknown_level), + file_writer.get()); + } else { + s = DB::Open(opts, dbname, &db); + ASSERT_OK(s); + ASSERT_TRUE(db != nullptr); + } + // Populate slightly more than 1M keys + for (int i = 0; i < num_keys1; i++) { + for (int j = 0; j < num_keys2; j++) { + std::string key = MakeKey(i * 2, j, through_db); + if (!through_db) { + tb->Add(key, key); + } else { + db->Put(wo, key, key); + } + } + } + if (!through_db) { + tb->Finish(); + file_writer->Close(); + } else { + db->Flush(FlushOptions()); + } + + std::unique_ptr table_reader; + if (!through_db) { + const auto& fs = env->GetFileSystem(); + FileOptions fopts(env_options); + + std::unique_ptr raf; + s = fs->NewRandomAccessFile(file_name, fopts, &raf, nullptr); + if (!s.ok()) { + fprintf(stderr, "Create File Error: %s\n", s.ToString().c_str()); + exit(1); + } + uint64_t file_size; + fs->GetFileSize(file_name, fopts.io_options, &file_size, nullptr); + std::unique_ptr file_reader( + new RandomAccessFileReader(std::move(raf), file_name)); + s = opts.table_factory->NewTableReader( + TableReaderOptions(ioptions, moptions.prefix_extractor, env_options, + ikc, 0 /* block_protection_bytes_per_key */), + std::move(file_reader), file_size, &table_reader); + if (!s.ok()) { + fprintf(stderr, "Open Table Error: %s\n", s.ToString().c_str()); + exit(1); + } + } + + Random rnd(301); + std::string result; + HistogramImpl hist; + + for (int it = 0; it < num_iter; it++) { + for (int i = 0; i < num_keys1; i++) { + for (int j = 0; j < num_keys2; j++) { + int r1 = rnd.Uniform(num_keys1) * 2; + int r2 = rnd.Uniform(num_keys2); + if (if_query_empty_keys) { + r1++; + r2 = num_keys2 * 2 - r2; + } + + if (!for_iterator) { + // Query one existing key; + std::string key = MakeKey(r1, r2, through_db); + uint64_t start_time = Now(clock, measured_by_nanosecond); + if (!through_db) { + PinnableSlice value; + MergeContext merge_context; + SequenceNumber max_covering_tombstone_seq = 0; + GetContext get_context( + ioptions.user_comparator, ioptions.merge_operator.get(), + ioptions.logger, ioptions.stats, GetContext::kNotFound, + Slice(key), &value, /*columns=*/nullptr, /*timestamp=*/nullptr, + &merge_context, true, &max_covering_tombstone_seq, clock); + s = table_reader->Get(read_options, key, &get_context, nullptr); + } else { + s = db->Get(read_options, key, &result); + } + hist.Add(Now(clock, measured_by_nanosecond) - start_time); + } else { + int r2_len; + if (if_query_empty_keys) { + r2_len = 0; + } else { + r2_len = rnd.Uniform(num_keys2) + 1; + if (r2_len + r2 > num_keys2) { + r2_len = num_keys2 - r2; + } + } + std::string start_key = MakeKey(r1, r2, through_db); + std::string end_key = MakeKey(r1, r2 + r2_len, through_db); + uint64_t total_time = 0; + uint64_t start_time = Now(clock, measured_by_nanosecond); + Iterator* iter = nullptr; + InternalIterator* iiter = nullptr; + if (!through_db) { + iiter = table_reader->NewIterator( + read_options, /*prefix_extractor=*/nullptr, /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized); + } else { + iter = db->NewIterator(read_options); + } + int count = 0; + for (through_db ? iter->Seek(start_key) : iiter->Seek(start_key); + through_db ? iter->Valid() : iiter->Valid(); + through_db ? iter->Next() : iiter->Next()) { + if (if_query_empty_keys) { + break; + } + // verify key; + total_time += Now(clock, measured_by_nanosecond) - start_time; + assert(Slice(MakeKey(r1, r2 + count, through_db)) == + (through_db ? iter->key() : iiter->key())); + start_time = Now(clock, measured_by_nanosecond); + if (++count >= r2_len) { + break; + } + } + if (count != r2_len) { + fprintf(stderr, + "Iterator cannot iterate expected number of entries. " + "Expected %d but got %d\n", + r2_len, count); + assert(false); + } + delete iter; + total_time += Now(clock, measured_by_nanosecond) - start_time; + hist.Add(total_time); + } + } + } + } + + fprintf( + stderr, + "===================================================" + "====================================================\n" + "InMemoryTableSimpleBenchmark: %20s num_key1: %5d " + "num_key2: %5d %10s\n" + "===================================================" + "====================================================" + "\nHistogram (unit: %s): \n%s", + opts.table_factory->Name(), num_keys1, num_keys2, + for_iterator ? "iterator" : (if_query_empty_keys ? "empty" : "non_empty"), + measured_by_nanosecond ? "nanosecond" : "microsecond", + hist.ToString().c_str()); + if (!through_db) { + env->DeleteFile(file_name); + } else { + delete db; + db = nullptr; + DestroyDB(dbname, opts); + } +} +} // namespace +} // namespace ROCKSDB_NAMESPACE + +DEFINE_bool(query_empty, false, + "query non-existing keys instead of existing ones."); +DEFINE_int32(num_keys1, 4096, "number of distinguish prefix of keys"); +DEFINE_int32(num_keys2, 512, "number of distinguish keys for each prefix"); +DEFINE_int32(iter, 3, "query non-existing keys instead of existing ones"); +DEFINE_int32(prefix_len, 16, "Prefix length used for iterators and indexes"); +DEFINE_bool(iterator, false, "For test iterator"); +DEFINE_bool(through_db, false, + "If enable, a DB instance will be created and the query will be " + "against DB. Otherwise, will be directly against a table reader."); +DEFINE_bool(mmap_read, true, "Whether use mmap read"); +DEFINE_string(table_factory, "block_based", + "Table factory to use: `block_based` (default), `plain_table` or " + "`cuckoo_hash`."); +DEFINE_string(time_unit, "microsecond", + "The time unit used for measuring performance. User can specify " + "`microsecond` (default) or `nanosecond`"); + +int main(int argc, char** argv) { + SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) + + " [OPTIONS]..."); + ParseCommandLineFlags(&argc, &argv, true); + + std::shared_ptr tf; + ROCKSDB_NAMESPACE::Options options; + if (FLAGS_prefix_len < 16) { + options.prefix_extractor.reset( + ROCKSDB_NAMESPACE::NewFixedPrefixTransform(FLAGS_prefix_len)); + } + ROCKSDB_NAMESPACE::ReadOptions ro; + ROCKSDB_NAMESPACE::EnvOptions env_options; + options.create_if_missing = true; + options.compression = ROCKSDB_NAMESPACE::CompressionType::kNoCompression; + + if (FLAGS_table_factory == "cuckoo_hash") { + options.allow_mmap_reads = FLAGS_mmap_read; + env_options.use_mmap_reads = FLAGS_mmap_read; + ROCKSDB_NAMESPACE::CuckooTableOptions table_options; + table_options.hash_table_ratio = 0.75; + tf.reset(ROCKSDB_NAMESPACE::NewCuckooTableFactory(table_options)); + } else if (FLAGS_table_factory == "plain_table") { + options.allow_mmap_reads = FLAGS_mmap_read; + env_options.use_mmap_reads = FLAGS_mmap_read; + + ROCKSDB_NAMESPACE::PlainTableOptions plain_table_options; + plain_table_options.user_key_len = 16; + plain_table_options.bloom_bits_per_key = (FLAGS_prefix_len == 16) ? 0 : 8; + plain_table_options.hash_table_ratio = 0.75; + + tf.reset(new ROCKSDB_NAMESPACE::PlainTableFactory(plain_table_options)); + options.prefix_extractor.reset( + ROCKSDB_NAMESPACE::NewFixedPrefixTransform(FLAGS_prefix_len)); + } else if (FLAGS_table_factory == "block_based") { + tf.reset(new ROCKSDB_NAMESPACE::BlockBasedTableFactory()); + } else { + fprintf(stderr, "Invalid table type %s\n", FLAGS_table_factory.c_str()); + } + + if (tf) { + // if user provides invalid options, just fall back to microsecond. + bool measured_by_nanosecond = FLAGS_time_unit == "nanosecond"; + + options.table_factory = tf; + ROCKSDB_NAMESPACE::TableReaderBenchmark( + options, env_options, ro, FLAGS_num_keys1, FLAGS_num_keys2, FLAGS_iter, + FLAGS_prefix_len, FLAGS_query_empty, FLAGS_iterator, FLAGS_through_db, + measured_by_nanosecond); + } else { + return 1; + } + + return 0; +} + +#endif // GFLAGS diff --git a/librocksdb-sys/rocksdb/table/table_test.cc b/librocksdb-sys/rocksdb/table/table_test.cc new file mode 100644 index 0000000..5b7e468 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/table_test.cc @@ -0,0 +1,5916 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "rocksdb/table.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "cache/lru_cache.h" +#include "db/db_test_util.h" +#include "db/dbformat.h" +#include "db/memtable.h" +#include "db/write_batch_internal.h" +#include "memtable/stl_wrappers.h" +#include "monitoring/statistics_impl.h" +#include "options/options_helper.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "rocksdb/cache.h" +#include "rocksdb/compression_type.h" +#include "rocksdb/convenience.h" +#include "rocksdb/db.h" +#include "rocksdb/env.h" +#include "rocksdb/file_checksum.h" +#include "rocksdb/file_system.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/iterator.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/options.h" +#include "rocksdb/perf_context.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/statistics.h" +#include "rocksdb/table_properties.h" +#include "rocksdb/trace_record.h" +#include "rocksdb/unique_id.h" +#include "rocksdb/write_buffer_manager.h" +#include "table/block_based/block.h" +#include "table/block_based/block_based_table_builder.h" +#include "table/block_based/block_based_table_factory.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/block_based/block_builder.h" +#include "table/block_based/filter_policy_internal.h" +#include "table/block_based/flush_block_policy_impl.h" +#include "table/block_fetcher.h" +#include "table/format.h" +#include "table/get_context.h" +#include "table/internal_iterator.h" +#include "table/meta_blocks.h" +#include "table/plain/plain_table_factory.h" +#include "table/scoped_arena_iterator.h" +#include "table/sst_file_writer_collectors.h" +#include "table/unique_id_impl.h" +#include "test_util/sync_point.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "util/coding_lean.h" +#include "util/compression.h" +#include "util/file_checksum_helper.h" +#include "util/random.h" +#include "util/string_util.h" +#include "utilities/memory_allocators.h" +#include "utilities/merge_operators.h" + +namespace ROCKSDB_NAMESPACE { + +extern const uint64_t kLegacyBlockBasedTableMagicNumber; +extern const uint64_t kLegacyPlainTableMagicNumber; +extern const uint64_t kBlockBasedTableMagicNumber; +extern const uint64_t kPlainTableMagicNumber; + +namespace { + +const std::string kDummyValue(10000, 'o'); + +// DummyPropertiesCollector used to test BlockBasedTableProperties +class DummyPropertiesCollector : public TablePropertiesCollector { + public: + const char* Name() const override { return "DummyPropertiesCollector"; } + + Status Finish(UserCollectedProperties* /*properties*/) override { + return Status::OK(); + } + + Status Add(const Slice& /*user_key*/, const Slice& /*value*/) override { + return Status::OK(); + } + + UserCollectedProperties GetReadableProperties() const override { + return UserCollectedProperties{}; + } +}; + +class DummyPropertiesCollectorFactory1 + : public TablePropertiesCollectorFactory { + public: + TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context /*context*/) override { + return new DummyPropertiesCollector(); + } + const char* Name() const override { + return "DummyPropertiesCollectorFactory1"; + } +}; + +class DummyPropertiesCollectorFactory2 + : public TablePropertiesCollectorFactory { + public: + TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context /*context*/) override { + return new DummyPropertiesCollector(); + } + const char* Name() const override { + return "DummyPropertiesCollectorFactory2"; + } +}; + +// Return reverse of "key". +// Used to test non-lexicographic comparators. +std::string Reverse(const Slice& key) { + auto rev = key.ToString(); + std::reverse(rev.begin(), rev.end()); + return rev; +} + +class ReverseKeyComparator : public Comparator { + public: + const char* Name() const override { + return "rocksdb.ReverseBytewiseComparator"; + } + + int Compare(const Slice& a, const Slice& b) const override { + return BytewiseComparator()->Compare(Reverse(a), Reverse(b)); + } + + void FindShortestSeparator(std::string* start, + const Slice& limit) const override { + std::string s = Reverse(*start); + std::string l = Reverse(limit); + BytewiseComparator()->FindShortestSeparator(&s, l); + *start = Reverse(s); + } + + void FindShortSuccessor(std::string* key) const override { + std::string s = Reverse(*key); + BytewiseComparator()->FindShortSuccessor(&s); + *key = Reverse(s); + } +}; + +ReverseKeyComparator reverse_key_comparator; + +void Increment(const Comparator* cmp, std::string* key) { + if (cmp == BytewiseComparator()) { + key->push_back('\0'); + } else { + assert(cmp == &reverse_key_comparator); + std::string rev = Reverse(*key); + rev.push_back('\0'); + *key = Reverse(rev); + } +} + +const auto kUnknownColumnFamily = + TablePropertiesCollectorFactory::Context::kUnknownColumnFamily; + +} // namespace + +// Helper class for tests to unify the interface between +// BlockBuilder/TableBuilder and Block/Table. +class Constructor { + public: + explicit Constructor(const Comparator* cmp) + : data_(stl_wrappers::LessOfComparator(cmp)) {} + virtual ~Constructor() {} + + void Add(const std::string& key, const Slice& value) { + data_[key] = value.ToString(); + } + + // Finish constructing the data structure with all the keys that have + // been added so far. Returns the keys in sorted order in "*keys" + // and stores the key/value pairs in "*kvmap" + void Finish(const Options& options, const ImmutableOptions& ioptions, + const MutableCFOptions& moptions, + const BlockBasedTableOptions& table_options, + const InternalKeyComparator& internal_comparator, + std::vector* keys, stl_wrappers::KVMap* kvmap) { + last_internal_comparator_ = &internal_comparator; + *kvmap = data_; + keys->clear(); + for (const auto& kv : data_) { + keys->push_back(kv.first); + } + data_.clear(); + Status s = FinishImpl(options, ioptions, moptions, table_options, + internal_comparator, *kvmap); + ASSERT_TRUE(s.ok()) << s.ToString(); + } + + // Construct the data structure from the data in "data" + virtual Status FinishImpl(const Options& options, + const ImmutableOptions& ioptions, + const MutableCFOptions& moptions, + const BlockBasedTableOptions& table_options, + const InternalKeyComparator& internal_comparator, + const stl_wrappers::KVMap& data) = 0; + + virtual InternalIterator* NewIterator( + const SliceTransform* prefix_extractor = nullptr) const = 0; + + virtual const stl_wrappers::KVMap& data() { return data_; } + + virtual bool IsArenaMode() const { return false; } + + virtual DB* db() const { return nullptr; } // Overridden in DBConstructor + + virtual bool AnywayDeleteIterator() const { return false; } + + protected: + const InternalKeyComparator* last_internal_comparator_; + + private: + stl_wrappers::KVMap data_; +}; + +// A helper class that converts internal format keys into user keys +class KeyConvertingIterator : public InternalIterator { + public: + explicit KeyConvertingIterator(InternalIterator* iter, + bool arena_mode = false) + : iter_(iter), arena_mode_(arena_mode) {} + ~KeyConvertingIterator() override { + if (arena_mode_) { + iter_->~InternalIterator(); + } else { + delete iter_; + } + } + bool Valid() const override { return iter_->Valid() && status_.ok(); } + void Seek(const Slice& target) override { + ParsedInternalKey ikey(target, kMaxSequenceNumber, kTypeValue); + std::string encoded; + AppendInternalKey(&encoded, ikey); + iter_->Seek(encoded); + } + void SeekForPrev(const Slice& target) override { + ParsedInternalKey ikey(target, kMaxSequenceNumber, kTypeValue); + std::string encoded; + AppendInternalKey(&encoded, ikey); + iter_->SeekForPrev(encoded); + } + void SeekToFirst() override { iter_->SeekToFirst(); } + void SeekToLast() override { iter_->SeekToLast(); } + void Next() override { iter_->Next(); } + void Prev() override { iter_->Prev(); } + IterBoundCheck UpperBoundCheckResult() override { + return iter_->UpperBoundCheckResult(); + } + + Slice key() const override { + assert(Valid()); + ParsedInternalKey parsed_key; + Status pik_status = + ParseInternalKey(iter_->key(), &parsed_key, true /* log_err_key */); + if (!pik_status.ok()) { + status_ = pik_status; + return Slice(status_.getState()); + } + return parsed_key.user_key; + } + + Slice value() const override { return iter_->value(); } + Status status() const override { + return status_.ok() ? iter_->status() : status_; + } + + private: + mutable Status status_; + InternalIterator* iter_; + bool arena_mode_; + + // No copying allowed + KeyConvertingIterator(const KeyConvertingIterator&); + void operator=(const KeyConvertingIterator&); +}; + +// `BlockConstructor` APIs always accept/return user keys. +class BlockConstructor : public Constructor { + public: + explicit BlockConstructor(const Comparator* cmp) + : Constructor(cmp), comparator_(cmp), block_(nullptr) {} + ~BlockConstructor() override { delete block_; } + Status FinishImpl(const Options& /*options*/, + const ImmutableOptions& /*ioptions*/, + const MutableCFOptions& /*moptions*/, + const BlockBasedTableOptions& table_options, + const InternalKeyComparator& /*internal_comparator*/, + const stl_wrappers::KVMap& kv_map) override { + delete block_; + block_ = nullptr; + BlockBuilder builder(table_options.block_restart_interval); + + for (const auto& kv : kv_map) { + // `DataBlockIter` assumes it reads only internal keys. `BlockConstructor` + // clients provide user keys, so we need to convert to internal key format + // before writing the data block. + ParsedInternalKey ikey(kv.first, kMaxSequenceNumber, kTypeValue); + std::string encoded; + AppendInternalKey(&encoded, ikey); + builder.Add(encoded, kv.second); + } + // Open the block + data_ = builder.Finish().ToString(); + BlockContents contents; + contents.data = data_; + block_ = new Block(std::move(contents)); + return Status::OK(); + } + InternalIterator* NewIterator( + const SliceTransform* /*prefix_extractor*/) const override { + // `DataBlockIter` returns the internal keys it reads. + // `KeyConvertingIterator` converts them to user keys before they are + // exposed to the `BlockConstructor` clients. + return new KeyConvertingIterator( + block_->NewDataIterator(comparator_, kDisableGlobalSequenceNumber)); + } + + private: + const Comparator* comparator_; + std::string data_; + Block* block_; + + BlockConstructor(); +}; + +class TableConstructor : public Constructor { + public: + explicit TableConstructor(const Comparator* cmp, + bool convert_to_internal_key = false, + int level = -1, SequenceNumber largest_seqno = 0) + : Constructor(cmp), + largest_seqno_(largest_seqno), + convert_to_internal_key_(convert_to_internal_key), + level_(level) { + env_ = ROCKSDB_NAMESPACE::Env::Default(); + } + ~TableConstructor() override { Reset(); } + + Status FinishImpl(const Options& options, const ImmutableOptions& ioptions, + const MutableCFOptions& moptions, + const BlockBasedTableOptions& /*table_options*/, + const InternalKeyComparator& internal_comparator, + const stl_wrappers::KVMap& kv_map) override { + Reset(); + soptions.use_mmap_reads = ioptions.allow_mmap_reads; + std::unique_ptr sink(new test::StringSink()); + file_writer_.reset(new WritableFileWriter( + std::move(sink), "" /* don't care */, FileOptions())); + std::unique_ptr builder; + IntTblPropCollectorFactories int_tbl_prop_collector_factories; + + if (largest_seqno_ != 0) { + // Pretend that it's an external file written by SstFileWriter. + int_tbl_prop_collector_factories.emplace_back( + new SstFileWriterPropertiesCollectorFactory(2 /* version */, + 0 /* global_seqno*/)); + } + + std::string column_family_name; + builder.reset(ioptions.table_factory->NewTableBuilder( + TableBuilderOptions(ioptions, moptions, internal_comparator, + &int_tbl_prop_collector_factories, + options.compression, options.compression_opts, + kUnknownColumnFamily, column_family_name, level_), + file_writer_.get())); + + for (const auto& kv : kv_map) { + if (convert_to_internal_key_) { + ParsedInternalKey ikey(kv.first, kMaxSequenceNumber, kTypeValue); + std::string encoded; + AppendInternalKey(&encoded, ikey); + builder->Add(encoded, kv.second); + } else { + builder->Add(kv.first, kv.second); + } + EXPECT_OK(builder->status()); + } + Status s = builder->Finish(); + EXPECT_OK(file_writer_->Flush()); + EXPECT_TRUE(s.ok()) << s.ToString(); + + EXPECT_EQ(TEST_GetSink()->contents().size(), builder->FileSize()); + + // Open the table + file_num_ = cur_file_num_++; + + return Reopen(ioptions, moptions); + } + + InternalIterator* NewIterator( + const SliceTransform* prefix_extractor) const override { + InternalIterator* iter = table_reader_->NewIterator( + read_options_, prefix_extractor, /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized); + if (convert_to_internal_key_) { + return new KeyConvertingIterator(iter); + } else { + return iter; + } + } + + uint64_t ApproximateOffsetOf(const Slice& key) const { + const ReadOptions read_options; + if (convert_to_internal_key_) { + InternalKey ikey(key, kMaxSequenceNumber, kTypeValue); + const Slice skey = ikey.Encode(); + return table_reader_->ApproximateOffsetOf( + read_options, skey, TableReaderCaller::kUncategorized); + } + return table_reader_->ApproximateOffsetOf( + read_options, key, TableReaderCaller::kUncategorized); + } + + virtual Status Reopen(const ImmutableOptions& ioptions, + const MutableCFOptions& moptions) { + std::unique_ptr source(new test::StringSource( + TEST_GetSink()->contents(), file_num_, ioptions.allow_mmap_reads)); + + file_reader_.reset(new RandomAccessFileReader(std::move(source), "test")); + return ioptions.table_factory->NewTableReader( + TableReaderOptions(ioptions, moptions.prefix_extractor, soptions, + *last_internal_comparator_, + 0 /* block_protection_bytes_per_key */, + /*skip_filters*/ false, + /*immortal*/ false, false, level_, + &block_cache_tracer_, moptions.write_buffer_size, "", + file_num_, kNullUniqueId64x2, largest_seqno_), + std::move(file_reader_), TEST_GetSink()->contents().size(), + &table_reader_); + } + + virtual TableReader* GetTableReader() { return table_reader_.get(); } + + bool AnywayDeleteIterator() const override { + return convert_to_internal_key_; + } + + void ResetTableReader() { table_reader_.reset(); } + + bool ConvertToInternalKey() { return convert_to_internal_key_; } + + test::StringSink* TEST_GetSink() { + return static_cast(file_writer_->writable_file()); + } + + BlockCacheTracer block_cache_tracer_; + + private: + void Reset() { + file_num_ = 0; + table_reader_.reset(); + file_writer_.reset(); + file_reader_.reset(); + } + + const ReadOptions read_options_; + uint64_t file_num_; + std::unique_ptr file_writer_; + std::unique_ptr file_reader_; + std::unique_ptr table_reader_; + SequenceNumber largest_seqno_; + bool convert_to_internal_key_; + int level_; + + TableConstructor(); + + static uint64_t cur_file_num_; + EnvOptions soptions; + Env* env_; +}; +uint64_t TableConstructor::cur_file_num_ = 1; + +class MemTableConstructor : public Constructor { + public: + explicit MemTableConstructor(const Comparator* cmp, WriteBufferManager* wb) + : Constructor(cmp), + internal_comparator_(cmp), + write_buffer_manager_(wb), + table_factory_(new SkipListFactory) { + options_.memtable_factory = table_factory_; + ImmutableOptions ioptions(options_); + memtable_ = + new MemTable(internal_comparator_, ioptions, MutableCFOptions(options_), + wb, kMaxSequenceNumber, 0 /* column_family_id */); + memtable_->Ref(); + } + ~MemTableConstructor() override { delete memtable_->Unref(); } + Status FinishImpl(const Options&, const ImmutableOptions& ioptions, + const MutableCFOptions& /*moptions*/, + const BlockBasedTableOptions& /*table_options*/, + const InternalKeyComparator& /*internal_comparator*/, + const stl_wrappers::KVMap& kv_map) override { + delete memtable_->Unref(); + ImmutableOptions mem_ioptions(ioptions); + memtable_ = new MemTable(internal_comparator_, mem_ioptions, + MutableCFOptions(options_), write_buffer_manager_, + kMaxSequenceNumber, 0 /* column_family_id */); + memtable_->Ref(); + int seq = 1; + for (const auto& kv : kv_map) { + Status s = memtable_->Add(seq, kTypeValue, kv.first, kv.second, + nullptr /* kv_prot_info */); + if (!s.ok()) { + return s; + } + seq++; + } + return Status::OK(); + } + InternalIterator* NewIterator( + const SliceTransform* /*prefix_extractor*/) const override { + return new KeyConvertingIterator( + memtable_->NewIterator(ReadOptions(), &arena_), true); + } + + bool AnywayDeleteIterator() const override { return true; } + + bool IsArenaMode() const override { return true; } + + private: + mutable Arena arena_; + InternalKeyComparator internal_comparator_; + Options options_; + WriteBufferManager* write_buffer_manager_; + MemTable* memtable_; + std::shared_ptr table_factory_; +}; + +class InternalIteratorFromIterator : public InternalIterator { + public: + explicit InternalIteratorFromIterator(Iterator* it) : it_(it) {} + bool Valid() const override { return it_->Valid(); } + void Seek(const Slice& target) override { it_->Seek(target); } + void SeekForPrev(const Slice& target) override { it_->SeekForPrev(target); } + void SeekToFirst() override { it_->SeekToFirst(); } + void SeekToLast() override { it_->SeekToLast(); } + void Next() override { it_->Next(); } + void Prev() override { it_->Prev(); } + Slice key() const override { return it_->key(); } + Slice value() const override { return it_->value(); } + Status status() const override { return it_->status(); } + + private: + std::unique_ptr it_; +}; + +class DBConstructor : public Constructor { + public: + explicit DBConstructor(const Comparator* cmp) + : Constructor(cmp), comparator_(cmp) { + db_ = nullptr; + NewDB(); + } + ~DBConstructor() override { delete db_; } + Status FinishImpl(const Options& /*options*/, + const ImmutableOptions& /*ioptions*/, + const MutableCFOptions& /*moptions*/, + const BlockBasedTableOptions& /*table_options*/, + const InternalKeyComparator& /*internal_comparator*/, + const stl_wrappers::KVMap& kv_map) override { + delete db_; + db_ = nullptr; + NewDB(); + for (const auto& kv : kv_map) { + WriteBatch batch; + EXPECT_OK(batch.Put(kv.first, kv.second)); + EXPECT_TRUE(db_->Write(WriteOptions(), &batch).ok()); + } + return Status::OK(); + } + + InternalIterator* NewIterator( + const SliceTransform* /*prefix_extractor*/) const override { + return new InternalIteratorFromIterator(db_->NewIterator(ReadOptions())); + } + + DB* db() const override { return db_; } + + private: + void NewDB() { + std::string name = test::PerThreadDBPath("table_testdb"); + + Options options; + options.comparator = comparator_; + Status status = DestroyDB(name, options); + ASSERT_TRUE(status.ok()) << status.ToString(); + + options.create_if_missing = true; + options.error_if_exists = true; + options.write_buffer_size = 10000; // Something small to force merging + status = DB::Open(options, name, &db_); + ASSERT_TRUE(status.ok()) << status.ToString(); + } + + const Comparator* comparator_; + DB* db_; +}; + +enum TestType { + BLOCK_BASED_TABLE_TEST, + PLAIN_TABLE_SEMI_FIXED_PREFIX, + PLAIN_TABLE_FULL_STR_PREFIX, + PLAIN_TABLE_TOTAL_ORDER, + BLOCK_TEST, + MEMTABLE_TEST, + DB_TEST +}; + +struct TestArgs { + TestType type; + bool reverse_compare; + int restart_interval; + CompressionType compression; + uint32_t compression_parallel_threads; + uint32_t format_version; + bool use_mmap; +}; + +std::ostream& operator<<(std::ostream& os, const TestArgs& args) { + os << "type: " << args.type << " reverse_compare: " << args.reverse_compare + << " restart_interval: " << args.restart_interval + << " compression: " << args.compression + << " compression_parallel_threads: " << args.compression_parallel_threads + << " format_version: " << args.format_version + << " use_mmap: " << args.use_mmap; + + return os; +} + +static std::vector GenerateArgList() { + std::vector test_args; + std::vector test_types = {BLOCK_BASED_TABLE_TEST, + PLAIN_TABLE_SEMI_FIXED_PREFIX, + PLAIN_TABLE_FULL_STR_PREFIX, + PLAIN_TABLE_TOTAL_ORDER, + BLOCK_TEST, + MEMTABLE_TEST, + DB_TEST}; + std::vector reverse_compare_types = {false, true}; + std::vector restart_intervals = {16, 1, 1024}; + std::vector compression_parallel_threads = {1, 4}; + + // Only add compression if it is supported + std::vector> compression_types; + compression_types.emplace_back(kNoCompression, false); + if (Snappy_Supported()) { + compression_types.emplace_back(kSnappyCompression, false); + } + if (Zlib_Supported()) { + compression_types.emplace_back(kZlibCompression, false); + compression_types.emplace_back(kZlibCompression, true); + } + if (BZip2_Supported()) { + compression_types.emplace_back(kBZip2Compression, false); + compression_types.emplace_back(kBZip2Compression, true); + } + if (LZ4_Supported()) { + compression_types.emplace_back(kLZ4Compression, false); + compression_types.emplace_back(kLZ4Compression, true); + compression_types.emplace_back(kLZ4HCCompression, false); + compression_types.emplace_back(kLZ4HCCompression, true); + } + if (XPRESS_Supported()) { + compression_types.emplace_back(kXpressCompression, false); + compression_types.emplace_back(kXpressCompression, true); + } + if (ZSTD_Supported()) { + compression_types.emplace_back(kZSTD, false); + compression_types.emplace_back(kZSTD, true); + } + + for (auto test_type : test_types) { + for (auto reverse_compare : reverse_compare_types) { + if (test_type == PLAIN_TABLE_SEMI_FIXED_PREFIX || + test_type == PLAIN_TABLE_FULL_STR_PREFIX || + test_type == PLAIN_TABLE_TOTAL_ORDER) { + // Plain table doesn't use restart index or compression. + TestArgs one_arg; + one_arg.type = test_type; + one_arg.reverse_compare = reverse_compare; + one_arg.restart_interval = restart_intervals[0]; + one_arg.compression = compression_types[0].first; + one_arg.compression_parallel_threads = 1; + one_arg.format_version = 0; + one_arg.use_mmap = true; + test_args.push_back(one_arg); + one_arg.use_mmap = false; + test_args.push_back(one_arg); + continue; + } + + for (auto restart_interval : restart_intervals) { + for (auto compression_type : compression_types) { + for (auto num_threads : compression_parallel_threads) { + TestArgs one_arg; + one_arg.type = test_type; + one_arg.reverse_compare = reverse_compare; + one_arg.restart_interval = restart_interval; + one_arg.compression = compression_type.first; + one_arg.compression_parallel_threads = num_threads; + one_arg.format_version = compression_type.second ? 2 : 1; + one_arg.use_mmap = false; + test_args.push_back(one_arg); + } + } + } + } + } + return test_args; +} + +// In order to make all tests run for plain table format, including +// those operating on empty keys, create a new prefix transformer which +// return fixed prefix if the slice is not shorter than the prefix length, +// and the full slice if it is shorter. +class FixedOrLessPrefixTransform : public SliceTransform { + private: + const size_t prefix_len_; + + public: + explicit FixedOrLessPrefixTransform(size_t prefix_len) + : prefix_len_(prefix_len) {} + + const char* Name() const override { return "rocksdb.FixedPrefix"; } + + Slice Transform(const Slice& src) const override { + assert(InDomain(src)); + if (src.size() < prefix_len_) { + return src; + } + return Slice(src.data(), prefix_len_); + } + + bool InDomain(const Slice& /*src*/) const override { return true; } + + bool InRange(const Slice& dst) const override { + return (dst.size() <= prefix_len_); + } + bool FullLengthEnabled(size_t* /*len*/) const override { return false; } +}; + +class HarnessTest : public testing::Test { + public: + explicit HarnessTest(const TestArgs& args) + : args_(args), + ioptions_(options_), + moptions_(options_), + write_buffer_(options_.db_write_buffer_size), + support_prev_(true), + only_support_prefix_seek_(false) { + options_.compression = args_.compression; + options_.compression_opts.parallel_threads = + args_.compression_parallel_threads; + // Use shorter block size for tests to exercise block boundary + // conditions more. + if (args_.reverse_compare) { + options_.comparator = &reverse_key_comparator; + } + + internal_comparator_.reset( + new test::PlainInternalKeyComparator(options_.comparator)); + + options_.allow_mmap_reads = args_.use_mmap; + switch (args_.type) { + case BLOCK_BASED_TABLE_TEST: + table_options_.flush_block_policy_factory.reset( + new FlushBlockBySizePolicyFactory()); + table_options_.block_size = 256; + table_options_.block_restart_interval = args_.restart_interval; + table_options_.index_block_restart_interval = args_.restart_interval; + table_options_.format_version = args_.format_version; + options_.table_factory.reset( + new BlockBasedTableFactory(table_options_)); + constructor_.reset(new TableConstructor( + options_.comparator, true /* convert_to_internal_key_ */)); + internal_comparator_.reset( + new InternalKeyComparator(options_.comparator)); + break; + + case PLAIN_TABLE_SEMI_FIXED_PREFIX: + support_prev_ = false; + only_support_prefix_seek_ = true; + options_.prefix_extractor.reset(new FixedOrLessPrefixTransform(2)); + options_.table_factory.reset(NewPlainTableFactory()); + constructor_.reset(new TableConstructor( + options_.comparator, true /* convert_to_internal_key_ */)); + internal_comparator_.reset( + new InternalKeyComparator(options_.comparator)); + break; + case PLAIN_TABLE_FULL_STR_PREFIX: + support_prev_ = false; + only_support_prefix_seek_ = true; + options_.prefix_extractor.reset(NewNoopTransform()); + options_.table_factory.reset(NewPlainTableFactory()); + constructor_.reset(new TableConstructor( + options_.comparator, true /* convert_to_internal_key_ */)); + internal_comparator_.reset( + new InternalKeyComparator(options_.comparator)); + break; + case PLAIN_TABLE_TOTAL_ORDER: + support_prev_ = false; + only_support_prefix_seek_ = false; + options_.prefix_extractor = nullptr; + + { + PlainTableOptions plain_table_options; + plain_table_options.user_key_len = kPlainTableVariableLength; + plain_table_options.bloom_bits_per_key = 0; + plain_table_options.hash_table_ratio = 0; + + options_.table_factory.reset( + NewPlainTableFactory(plain_table_options)); + } + constructor_.reset(new TableConstructor( + options_.comparator, true /* convert_to_internal_key_ */)); + internal_comparator_.reset( + new InternalKeyComparator(options_.comparator)); + break; + case BLOCK_TEST: + table_options_.block_size = 256; + options_.table_factory.reset( + new BlockBasedTableFactory(table_options_)); + constructor_.reset(new BlockConstructor(options_.comparator)); + break; + case MEMTABLE_TEST: + table_options_.block_size = 256; + options_.table_factory.reset( + new BlockBasedTableFactory(table_options_)); + constructor_.reset( + new MemTableConstructor(options_.comparator, &write_buffer_)); + break; + case DB_TEST: + table_options_.block_size = 256; + options_.table_factory.reset( + new BlockBasedTableFactory(table_options_)); + constructor_.reset(new DBConstructor(options_.comparator)); + break; + } + ioptions_ = ImmutableOptions(options_); + moptions_ = MutableCFOptions(options_); + } + + void Add(const std::string& key, const std::string& value) { + constructor_->Add(key, value); + } + + void Test(Random* rnd) { + std::vector keys; + stl_wrappers::KVMap data; + constructor_->Finish(options_, ioptions_, moptions_, table_options_, + *internal_comparator_, &keys, &data); + + TestForwardScan(keys, data); + if (support_prev_) { + TestBackwardScan(keys, data); + } + TestRandomAccess(rnd, keys, data); + } + + void TestForwardScan(const std::vector& /*keys*/, + const stl_wrappers::KVMap& data) { + InternalIterator* iter = constructor_->NewIterator(); + ASSERT_TRUE(!iter->Valid()); + iter->SeekToFirst(); + ASSERT_OK(iter->status()); + for (stl_wrappers::KVMap::const_iterator model_iter = data.begin(); + model_iter != data.end(); ++model_iter) { + ASSERT_EQ(ToString(data, model_iter), ToString(iter)); + iter->Next(); + ASSERT_OK(iter->status()); + } + ASSERT_TRUE(!iter->Valid()); + ASSERT_OK(iter->status()); + if (constructor_->IsArenaMode() && !constructor_->AnywayDeleteIterator()) { + iter->~InternalIterator(); + } else { + delete iter; + } + } + + void TestBackwardScan(const std::vector& /*keys*/, + const stl_wrappers::KVMap& data) { + InternalIterator* iter = constructor_->NewIterator(); + ASSERT_TRUE(!iter->Valid()); + iter->SeekToLast(); + ASSERT_OK(iter->status()); + for (stl_wrappers::KVMap::const_reverse_iterator model_iter = data.rbegin(); + model_iter != data.rend(); ++model_iter) { + ASSERT_EQ(ToString(data, model_iter), ToString(iter)); + iter->Prev(); + ASSERT_OK(iter->status()); + } + ASSERT_TRUE(!iter->Valid()); + ASSERT_OK(iter->status()); + if (constructor_->IsArenaMode() && !constructor_->AnywayDeleteIterator()) { + iter->~InternalIterator(); + } else { + delete iter; + } + } + + void TestRandomAccess(Random* rnd, const std::vector& keys, + const stl_wrappers::KVMap& data) { + static const bool kVerbose = false; + InternalIterator* iter = constructor_->NewIterator(); + ASSERT_TRUE(!iter->Valid()); + stl_wrappers::KVMap::const_iterator model_iter = data.begin(); + if (kVerbose) fprintf(stderr, "---\n"); + for (int i = 0; i < 200; i++) { + const int toss = rnd->Uniform(support_prev_ ? 5 : 3); + switch (toss) { + case 0: { + if (iter->Valid()) { + if (kVerbose) fprintf(stderr, "Next\n"); + iter->Next(); + ASSERT_OK(iter->status()); + ++model_iter; + ASSERT_EQ(ToString(data, model_iter), ToString(iter)); + } + break; + } + + case 1: { + if (kVerbose) fprintf(stderr, "SeekToFirst\n"); + iter->SeekToFirst(); + ASSERT_OK(iter->status()); + model_iter = data.begin(); + ASSERT_EQ(ToString(data, model_iter), ToString(iter)); + break; + } + + case 2: { + std::string key = PickRandomKey(rnd, keys); + model_iter = data.lower_bound(key); + if (kVerbose) + fprintf(stderr, "Seek '%s'\n", EscapeString(key).c_str()); + iter->Seek(Slice(key)); + ASSERT_OK(iter->status()); + ASSERT_EQ(ToString(data, model_iter), ToString(iter)); + break; + } + + case 3: { + if (iter->Valid()) { + if (kVerbose) fprintf(stderr, "Prev\n"); + iter->Prev(); + ASSERT_OK(iter->status()); + if (model_iter == data.begin()) { + model_iter = data.end(); // Wrap around to invalid value + } else { + --model_iter; + } + ASSERT_EQ(ToString(data, model_iter), ToString(iter)); + } + break; + } + + case 4: { + if (kVerbose) fprintf(stderr, "SeekToLast\n"); + iter->SeekToLast(); + ASSERT_OK(iter->status()); + if (keys.empty()) { + model_iter = data.end(); + } else { + std::string last = data.rbegin()->first; + model_iter = data.lower_bound(last); + } + ASSERT_EQ(ToString(data, model_iter), ToString(iter)); + break; + } + } + } + if (constructor_->IsArenaMode() && !constructor_->AnywayDeleteIterator()) { + iter->~InternalIterator(); + } else { + delete iter; + } + } + + std::string ToString(const stl_wrappers::KVMap& data, + const stl_wrappers::KVMap::const_iterator& it) { + if (it == data.end()) { + return "END"; + } else { + return "'" + it->first + "->" + it->second + "'"; + } + } + + std::string ToString(const stl_wrappers::KVMap& data, + const stl_wrappers::KVMap::const_reverse_iterator& it) { + if (it == data.rend()) { + return "END"; + } else { + return "'" + it->first + "->" + it->second + "'"; + } + } + + std::string ToString(const InternalIterator* it) { + if (!it->Valid()) { + return "END"; + } else { + return "'" + it->key().ToString() + "->" + it->value().ToString() + "'"; + } + } + + std::string PickRandomKey(Random* rnd, const std::vector& keys) { + if (keys.empty()) { + return "foo"; + } else { + const int index = rnd->Uniform(static_cast(keys.size())); + std::string result = keys[index]; + switch (rnd->Uniform(support_prev_ ? 3 : 1)) { + case 0: + // Return an existing key + break; + case 1: { + // Attempt to return something smaller than an existing key + if (result.size() > 0 && result[result.size() - 1] > '\0' && + (!only_support_prefix_seek_ || + options_.prefix_extractor->Transform(result).size() < + result.size())) { + result[result.size() - 1]--; + } + break; + } + case 2: { + // Return something larger than an existing key + Increment(options_.comparator, &result); + break; + } + } + return result; + } + } + + // Returns nullptr if not running against a DB + DB* db() const { return constructor_->db(); } + + private: + TestArgs args_; + Options options_; + ImmutableOptions ioptions_; + MutableCFOptions moptions_; + BlockBasedTableOptions table_options_; + std::unique_ptr constructor_; + WriteBufferManager write_buffer_; + bool support_prev_; + bool only_support_prefix_seek_; + std::shared_ptr internal_comparator_; +}; + +class ParameterizedHarnessTest : public HarnessTest, + public testing::WithParamInterface { + public: + ParameterizedHarnessTest() : HarnessTest(GetParam()) {} +}; + +INSTANTIATE_TEST_CASE_P(TableTest, ParameterizedHarnessTest, + ::testing::ValuesIn(GenerateArgList())); + +class DBHarnessTest : public HarnessTest { + public: + DBHarnessTest() + : HarnessTest(TestArgs{DB_TEST, /* reverse_compare */ false, + /* restart_interval */ 16, kNoCompression, + /* compression_parallel_threads */ 1, + /* format_version */ 0, /* use_mmap */ false}) {} +}; + +static bool Between(uint64_t val, uint64_t low, uint64_t high) { + bool result = (val >= low) && (val <= high); + if (!result) { + fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n", + (unsigned long long)(val), (unsigned long long)(low), + (unsigned long long)(high)); + } + return result; +} + +// Tests against all kinds of tables +class TableTest : public testing::Test { + public: + const InternalKeyComparator& GetPlainInternalComparator( + const Comparator* comp) { + if (!plain_internal_comparator) { + plain_internal_comparator.reset( + new test::PlainInternalKeyComparator(comp)); + } + return *plain_internal_comparator; + } + void IndexTest(BlockBasedTableOptions table_options); + + private: + std::unique_ptr plain_internal_comparator; +}; + +class GeneralTableTest : public TableTest {}; +class BlockBasedTableTestBase : public TableTest {}; +class BlockBasedTableTest + : public BlockBasedTableTestBase, + virtual public ::testing::WithParamInterface { + public: + BlockBasedTableTest() : format_(GetParam()) { + env_ = ROCKSDB_NAMESPACE::Env::Default(); + } + + BlockBasedTableOptions GetBlockBasedTableOptions() { + BlockBasedTableOptions options; + options.format_version = format_; + return options; + } + + void SetupTracingTest(TableConstructor* c) { + test_path_ = test::PerThreadDBPath("block_based_table_tracing_test"); + EXPECT_OK(env_->CreateDir(test_path_)); + trace_file_path_ = test_path_ + "/block_cache_trace_file"; + + BlockCacheTraceWriterOptions trace_writer_opt; + BlockCacheTraceOptions trace_opt; + std::unique_ptr trace_writer; + EXPECT_OK(NewFileTraceWriter(env_, EnvOptions(), trace_file_path_, + &trace_writer)); + std::unique_ptr block_cache_trace_writer = + NewBlockCacheTraceWriter(env_->GetSystemClock().get(), trace_writer_opt, + std::move(trace_writer)); + ASSERT_NE(block_cache_trace_writer, nullptr); + // Always return Status::OK(). + ASSERT_OK(c->block_cache_tracer_.StartTrace( + trace_opt, std::move(block_cache_trace_writer))); + + { + InternalKey internal_key(auto_add_key1, 0, kTypeValue); + std::string encoded_key = internal_key.Encode().ToString(); + c->Add(encoded_key, kDummyValue); + } + { + InternalKey internal_key(auto_add_key2, 0, kTypeValue); + std::string encoded_key = internal_key.Encode().ToString(); + c->Add(encoded_key, kDummyValue); + } + } + + void VerifyBlockAccessTrace( + TableConstructor* c, + const std::vector& expected_records) { + c->block_cache_tracer_.EndTrace(); + + { + std::unique_ptr trace_reader; + Status s = NewFileTraceReader(env_, EnvOptions(), trace_file_path_, + &trace_reader); + EXPECT_OK(s); + BlockCacheTraceReader reader(std::move(trace_reader)); + BlockCacheTraceHeader header; + EXPECT_OK(reader.ReadHeader(&header)); + uint32_t index = 0; + while (s.ok()) { + SCOPED_TRACE("expected_records[" + std::to_string(index) + "]"); + BlockCacheTraceRecord access; + s = reader.ReadAccess(&access); + if (!s.ok()) { + break; + } + ASSERT_LT(index, expected_records.size()); + EXPECT_NE("", access.block_key); + EXPECT_EQ(access.block_type, expected_records[index].block_type); + EXPECT_GT(access.block_size, 0); + EXPECT_EQ(access.caller, expected_records[index].caller); + EXPECT_EQ(access.no_insert, expected_records[index].no_insert); + EXPECT_EQ(access.is_cache_hit, expected_records[index].is_cache_hit); + EXPECT_EQ(access.get_id, expected_records[index].get_id); + // The well-populated cases + if (access.caller == TableReaderCaller::kUserGet || + (access.caller == TableReaderCaller::kUserMultiGet && + access.block_type == TraceType::kBlockTraceDataBlock)) { + EXPECT_EQ(access.referenced_key, + expected_records[index].referenced_key); + EXPECT_EQ(access.get_from_user_specified_snapshot, + expected_records[index].get_from_user_specified_snapshot); + if (access.block_type == TraceType::kBlockTraceDataBlock) { + EXPECT_GT(access.referenced_data_size, 0); + EXPECT_GT(access.num_keys_in_block, 0); + if (access.caller == TableReaderCaller::kUserMultiGet) { + // Test num_keys_in_block estimate, assuming default restart + // interval of 16 and just one interval. + // Rounding depends on get_id. + if (access.get_id & 1) { + EXPECT_EQ(access.num_keys_in_block, 9); + } else { + EXPECT_EQ(access.num_keys_in_block, 8); + } + } + EXPECT_EQ(access.referenced_key_exist_in_block, + expected_records[index].referenced_key_exist_in_block); + } + } else { + EXPECT_EQ(access.referenced_key, ""); + EXPECT_FALSE(access.get_from_user_specified_snapshot); + EXPECT_EQ(access.referenced_data_size, 0); + EXPECT_EQ(access.num_keys_in_block, 0); + EXPECT_FALSE(access.referenced_key_exist_in_block); + } + index++; + } + EXPECT_EQ(index, expected_records.size()); + } + EXPECT_OK(env_->DeleteFile(trace_file_path_)); + EXPECT_OK(env_->DeleteDir(test_path_)); + } + + protected: + uint64_t IndexUncompressedHelper(bool indexCompress); + const std::string auto_add_key1 = "aak01"; + const std::string auto_add_key2 = "aak02"; + + private: + uint32_t format_; + Env* env_; + std::string trace_file_path_; + std::string test_path_; +}; +class PlainTableTest : public TableTest {}; +class TablePropertyTest : public testing::Test {}; +class BBTTailPrefetchTest : public TableTest {}; + +// The helper class to test the file checksum +class FileChecksumTestHelper { + public: + FileChecksumTestHelper(bool convert_to_internal_key = false) + : convert_to_internal_key_(convert_to_internal_key) {} + ~FileChecksumTestHelper() {} + + void CreateWritableFile() { + sink_ = new test::StringSink(); + std::unique_ptr holder(sink_); + file_writer_.reset(new WritableFileWriter( + std::move(holder), "" /* don't care */, FileOptions())); + } + + void SetFileChecksumGenerator(FileChecksumGenerator* checksum_generator) { + if (file_writer_ != nullptr) { + file_writer_->TEST_SetFileChecksumGenerator(checksum_generator); + } else { + delete checksum_generator; + } + } + + WritableFileWriter* GetFileWriter() { return file_writer_.get(); } + + Status ResetTableBuilder(std::unique_ptr&& builder) { + assert(builder != nullptr); + table_builder_ = std::move(builder); + return Status::OK(); + } + + void AddKVtoKVMap(int num_entries) { + Random rnd(test::RandomSeed()); + for (int i = 0; i < num_entries; i++) { + std::string v = rnd.RandomString(100); + kv_map_[test::RandomKey(&rnd, 20)] = v; + } + } + + Status WriteKVAndFlushTable() { + for (const auto& kv : kv_map_) { + if (convert_to_internal_key_) { + ParsedInternalKey ikey(kv.first, kMaxSequenceNumber, kTypeValue); + std::string encoded; + AppendInternalKey(&encoded, ikey); + table_builder_->Add(encoded, kv.second); + } else { + table_builder_->Add(kv.first, kv.second); + } + EXPECT_TRUE(table_builder_->status().ok()); + } + Status s = table_builder_->Finish(); + EXPECT_OK(file_writer_->Flush()); + EXPECT_OK(s); + + EXPECT_EQ(sink_->contents().size(), table_builder_->FileSize()); + return s; + } + + std::string GetFileChecksum() { + EXPECT_OK(file_writer_->Close()); + return table_builder_->GetFileChecksum(); + } + + const char* GetFileChecksumFuncName() { + return table_builder_->GetFileChecksumFuncName(); + } + + Status CalculateFileChecksum(FileChecksumGenerator* file_checksum_generator, + std::string* checksum) { + assert(file_checksum_generator != nullptr); + cur_file_num_ = checksum_file_num_++; + test::StringSink* ss_rw = + static_cast(file_writer_->writable_file()); + std::unique_ptr source( + new test::StringSource(ss_rw->contents())); + file_reader_.reset(new RandomAccessFileReader(std::move(source), "test")); + + std::unique_ptr scratch(new char[2048]); + Slice result; + uint64_t offset = 0; + Status s; + s = file_reader_->Read(IOOptions(), offset, 2048, &result, scratch.get(), + nullptr, Env::IO_TOTAL /* rate_limiter_priority */); + if (!s.ok()) { + return s; + } + while (result.size() != 0) { + file_checksum_generator->Update(scratch.get(), result.size()); + offset += static_cast(result.size()); + s = file_reader_->Read(IOOptions(), offset, 2048, &result, scratch.get(), + nullptr, + Env::IO_TOTAL /* rate_limiter_priority */); + if (!s.ok()) { + return s; + } + } + EXPECT_EQ(offset, static_cast(table_builder_->FileSize())); + file_checksum_generator->Finalize(); + *checksum = file_checksum_generator->GetChecksum(); + return Status::OK(); + } + + private: + bool convert_to_internal_key_; + uint64_t cur_file_num_; + std::unique_ptr file_writer_; + std::unique_ptr file_reader_; + std::unique_ptr table_builder_; + stl_wrappers::KVMap kv_map_; + test::StringSink* sink_ = nullptr; + + static uint64_t checksum_file_num_; +}; + +uint64_t FileChecksumTestHelper::checksum_file_num_ = 1; + +INSTANTIATE_TEST_CASE_P(FormatVersions, BlockBasedTableTest, + testing::ValuesIn(test::kFooterFormatVersionsToTest)); + +// This test serves as the living tutorial for the prefix scan of user collected +// properties. +TEST_F(TablePropertyTest, PrefixScanTest) { + UserCollectedProperties props{ + {"num.111.1", "1"}, {"num.111.2", "2"}, {"num.111.3", "3"}, + {"num.333.1", "1"}, {"num.333.2", "2"}, {"num.333.3", "3"}, + {"num.555.1", "1"}, {"num.555.2", "2"}, {"num.555.3", "3"}, + }; + + // prefixes that exist + for (const std::string prefix : {"num.111", "num.333", "num.555"}) { + int num = 0; + for (auto pos = props.lower_bound(prefix); + pos != props.end() && + pos->first.compare(0, prefix.size(), prefix) == 0; + ++pos) { + ++num; + auto key = prefix + "." + std::to_string(num); + ASSERT_EQ(key, pos->first); + ASSERT_EQ(std::to_string(num), pos->second); + } + ASSERT_EQ(3, num); + } + + // prefixes that don't exist + for (const std::string prefix : + {"num.000", "num.222", "num.444", "num.666"}) { + auto pos = props.lower_bound(prefix); + ASSERT_TRUE(pos == props.end() || + pos->first.compare(0, prefix.size(), prefix) != 0); + } +} + +namespace { +struct TestIds { + UniqueId64x3 internal_id; + UniqueId64x3 external_id; +}; + +inline bool operator==(const TestIds& lhs, const TestIds& rhs) { + return lhs.internal_id == rhs.internal_id && + lhs.external_id == rhs.external_id; +} + +std::ostream& operator<<(std::ostream& os, const TestIds& ids) { + return os << std::hex << "{{{ 0x" << ids.internal_id[0] << "U, 0x" + << ids.internal_id[1] << "U, 0x" << ids.internal_id[2] + << "U }}, {{ 0x" << ids.external_id[0] << "U, 0x" + << ids.external_id[1] << "U, 0x" << ids.external_id[2] << "U }}}"; +} + +TestIds GetUniqueId(TableProperties* tp, std::unordered_set* seen, + const std::string& db_id, const std::string& db_session_id, + uint64_t file_number) { + // First test session id logic + if (db_session_id.size() == 20) { + uint64_t upper; + uint64_t lower; + EXPECT_OK(DecodeSessionId(db_session_id, &upper, &lower)); + EXPECT_EQ(EncodeSessionId(upper, lower), db_session_id); + } + + // Get external using public API + tp->db_id = db_id; + tp->db_session_id = db_session_id; + tp->orig_file_number = file_number; + TestIds t; + { + std::string euid; + EXPECT_OK(GetExtendedUniqueIdFromTableProperties(*tp, &euid)); + EXPECT_EQ(euid.size(), 24U); + t.external_id[0] = DecodeFixed64(&euid[0]); + t.external_id[1] = DecodeFixed64(&euid[8]); + t.external_id[2] = DecodeFixed64(&euid[16]); + + std::string uid; + EXPECT_OK(GetUniqueIdFromTableProperties(*tp, &uid)); + EXPECT_EQ(uid.size(), 16U); + EXPECT_EQ(uid, euid.substr(0, 16)); + EXPECT_EQ(t.external_id[0], DecodeFixed64(&uid[0])); + EXPECT_EQ(t.external_id[1], DecodeFixed64(&uid[8])); + } + // All these should be effectively random + EXPECT_TRUE(seen->insert(t.external_id[0]).second); + EXPECT_TRUE(seen->insert(t.external_id[1]).second); + EXPECT_TRUE(seen->insert(t.external_id[2]).second); + + // Get internal with internal API + EXPECT_OK(GetSstInternalUniqueId(db_id, db_session_id, file_number, + &t.internal_id)); + EXPECT_NE(t.internal_id, kNullUniqueId64x3); + + // Verify relationship + UniqueId64x3 tmp = t.internal_id; + InternalUniqueIdToExternal(&tmp); + EXPECT_EQ(tmp, t.external_id); + ExternalUniqueIdToInternal(&tmp); + EXPECT_EQ(tmp, t.internal_id); + + // And 128-bit internal version + UniqueId64x2 tmp2{}; + EXPECT_OK(GetSstInternalUniqueId(db_id, db_session_id, file_number, &tmp2)); + EXPECT_NE(tmp2, kNullUniqueId64x2); + + EXPECT_EQ(tmp2[0], t.internal_id[0]); + EXPECT_EQ(tmp2[1], t.internal_id[1]); + InternalUniqueIdToExternal(&tmp2); + EXPECT_EQ(tmp2[0], t.external_id[0]); + EXPECT_EQ(tmp2[1], t.external_id[1]); + ExternalUniqueIdToInternal(&tmp2); + EXPECT_EQ(tmp2[0], t.internal_id[0]); + EXPECT_EQ(tmp2[1], t.internal_id[1]); + + return t; +} +} // namespace + +TEST_F(TablePropertyTest, UniqueIdsSchemaAndQuality) { + // To ensure the computation only depends on the expected entries, we set + // the rest randomly + TableProperties tp; + TEST_SetRandomTableProperties(&tp); + + // DB id is normally RFC-4122 + const std::string db_id1 = "7265b6eb-4e42-4aec-86a4-0dc5e73a228d"; + // Allow other forms of DB id + const std::string db_id2 = "1728000184588763620"; + const std::string db_id3 = "x"; + + // DB session id is normally 20 chars in base-36, but 13 to 24 chars + // is ok, roughly 64 to 128 bits. + const std::string ses_id1 = "ABCDEFGHIJ0123456789"; + // Same trailing 13 digits + const std::string ses_id2 = "HIJ0123456789"; + const std::string ses_id3 = "0123ABCDEFGHIJ0123456789"; + // Different trailing 12 digits + const std::string ses_id4 = "ABCDEFGH888888888888"; + // And change length + const std::string ses_id5 = "ABCDEFGHIJ012"; + const std::string ses_id6 = "ABCDEFGHIJ0123456789ABCD"; + + using T = TestIds; + std::unordered_set seen; + // Establish a stable schema for the unique IDs. These values must not + // change for existing table files. + // (Note: parens needed for macro parsing, extra braces needed for some + // compilers.) + EXPECT_EQ( + GetUniqueId(&tp, &seen, db_id1, ses_id1, 1), + T({{{0x61d7dcf415d9cf19U, 0x160d77aae90757fdU, 0x907f41dfd90724ffU}}, + {{0xf0bd230365df7464U, 0xca089303f3648eb4U, 0x4b44f7e7324b2817U}}})); + // Only change internal_id[1] with file number + EXPECT_EQ( + GetUniqueId(&tp, &seen, db_id1, ses_id1, 2), + T({{{0x61d7dcf415d9cf19U, 0x160d77aae90757feU, 0x907f41dfd90724ffU}}, + {{0xf13fdf7adcfebb6dU, 0x97cd2226cc033ea2U, 0x198c438182091f0eU}}})); + EXPECT_EQ( + GetUniqueId(&tp, &seen, db_id1, ses_id1, 123456789), + T({{{0x61d7dcf415d9cf19U, 0x160d77aaee5c9ae9U, 0x907f41dfd90724ffU}}, + {{0x81fbcebe1ac6c4f0U, 0x6b14a64cfdc0f1c4U, 0x7d8fb6eaf18edbb3U}}})); + // Change internal_id[1] and internal_id[2] with db_id + EXPECT_EQ( + GetUniqueId(&tp, &seen, db_id2, ses_id1, 1), + T({{{0x61d7dcf415d9cf19U, 0xf89c471f572f0d25U, 0x1f0f2a5eb0e6257eU}}, + {{0x7f1d01d453616991U, 0x32ddf2afec804ab2U, 0xd10a1ee2f0c7d9c1U}}})); + EXPECT_EQ( + GetUniqueId(&tp, &seen, db_id3, ses_id1, 1), + T({{{0x61d7dcf415d9cf19U, 0xfed297a8154a57d0U, 0x8b931b9cdebd9e8U}}, + {{0x62b2f43183f6894bU, 0x897ff2b460eefad1U, 0xf4ec189fb2d15e04U}}})); + // Keeping same last 13 digits of ses_id keeps same internal_id[0] + EXPECT_EQ( + GetUniqueId(&tp, &seen, db_id1, ses_id2, 1), + T({{{0x61d7dcf415d9cf19U, 0x5f6cc4fa2d528c8U, 0x7b70845d5bfb5446U}}, + {{0x96d1c83ffcc94266U, 0x82663eac0ec6e14aU, 0x94a88b49678b77f6U}}})); + EXPECT_EQ( + GetUniqueId(&tp, &seen, db_id1, ses_id3, 1), + T({{{0x61d7dcf415d9cf19U, 0xfc7232879db37ea2U, 0xc0378d74ea4c89cdU}}, + {{0xdf2ef57e98776905U, 0xda5b31c987da833bU, 0x79c1b4bd0a9e760dU}}})); + // Changing last 12 digits of ses_id only changes internal_id[0] + // (vs. db_id1, ses_id1, 1) + EXPECT_EQ( + GetUniqueId(&tp, &seen, db_id1, ses_id4, 1), + T({{{0x4f07cc0d003a83a8U, 0x160d77aae90757fdU, 0x907f41dfd90724ffU}}, + {{0xbcf85336a9f71f04U, 0x4f2949e2f3adb60dU, 0x9ca0def976abfa10U}}})); + // ses_id can change everything. + EXPECT_EQ( + GetUniqueId(&tp, &seen, db_id1, ses_id5, 1), + T({{{0x94b8768e43f87ce6U, 0xc2559653ac4e7c93U, 0xde6dff6bbb1223U}}, + {{0x5a9537af681817fbU, 0x1afcd1fecaead5eaU, 0x767077ad9ebe0008U}}})); + EXPECT_EQ( + GetUniqueId(&tp, &seen, db_id1, ses_id6, 1), + T({{{0x43cfb0ffa3b710edU, 0x263c580426406a1bU, 0xfacc91379a80d29dU}}, + {{0xfa90547d84cb1cdbU, 0x2afe99c641992d4aU, 0x205b7f7b60e51cc2U}}})); + + // Now verify more thoroughly that any small change in inputs completely + // changes external unique id. + // (Relying on 'seen' checks etc. in GetUniqueId) + std::string db_id = "00000000-0000-0000-0000-000000000000"; + std::string ses_id = "000000000000000000000000"; + uint64_t file_num = 1; + // change db_id + for (size_t i = 0; i < db_id.size(); ++i) { + if (db_id[i] == '-') { + continue; + } + for (char alt : std::string("123456789abcdef")) { + db_id[i] = alt; + GetUniqueId(&tp, &seen, db_id, ses_id, file_num); + } + db_id[i] = '0'; + } + // change ses_id + for (size_t i = 0; i < ses_id.size(); ++i) { + for (char alt : std::string("123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ")) { + ses_id[i] = alt; + GetUniqueId(&tp, &seen, db_id, ses_id, file_num); + } + ses_id[i] = '0'; + } + // change file_num + for (int i = 1; i < 64; ++i) { + GetUniqueId(&tp, &seen, db_id, ses_id, file_num << i); + } + + // Verify that "all zeros" in first 128 bits is equivalent for internal and + // external IDs. This way, as long as we avoid "all zeros" in internal IDs, + // we avoid it in external IDs. + { + UniqueId64x3 id1{{0, 0, Random::GetTLSInstance()->Next64()}}; + UniqueId64x3 id2 = id1; + InternalUniqueIdToExternal(&id1); + EXPECT_EQ(id1, id2); + ExternalUniqueIdToInternal(&id2); + EXPECT_EQ(id1, id2); + } +} + +namespace { +void SetGoodTableProperties(TableProperties* tp) { + // To ensure the computation only depends on the expected entries, we set + // the rest randomly + TEST_SetRandomTableProperties(tp); + tp->db_id = "7265b6eb-4e42-4aec-86a4-0dc5e73a228d"; + tp->db_session_id = "ABCDEFGHIJ0123456789"; + tp->orig_file_number = 1; +} +} // namespace + +TEST_F(TablePropertyTest, UniqueIdHumanStrings) { + TableProperties tp; + SetGoodTableProperties(&tp); + + std::string tmp; + EXPECT_OK(GetExtendedUniqueIdFromTableProperties(tp, &tmp)); + EXPECT_EQ(tmp, + (std::string{{'\x64', '\x74', '\xdf', '\x65', '\x03', '\x23', + '\xbd', '\xf0', '\xb4', '\x8e', '\x64', '\xf3', + '\x03', '\x93', '\x08', '\xca', '\x17', '\x28', + '\x4b', '\x32', '\xe7', '\xf7', '\x44', '\x4b'}})); + EXPECT_EQ(UniqueIdToHumanString(tmp), + "6474DF650323BDF0-B48E64F3039308CA-17284B32E7F7444B"); + + EXPECT_OK(GetUniqueIdFromTableProperties(tp, &tmp)); + EXPECT_EQ(UniqueIdToHumanString(tmp), "6474DF650323BDF0-B48E64F3039308CA"); + + // including zero padding + tmp = std::string(24U, '\0'); + tmp[15] = '\x12'; + tmp[23] = '\xAB'; + EXPECT_EQ(UniqueIdToHumanString(tmp), + "0000000000000000-0000000000000012-00000000000000AB"); + + // And shortened + tmp = std::string(20U, '\0'); + tmp[5] = '\x12'; + tmp[10] = '\xAB'; + tmp[17] = '\xEF'; + EXPECT_EQ(UniqueIdToHumanString(tmp), + "0000000000120000-0000AB0000000000-00EF0000"); + + tmp.resize(16); + EXPECT_EQ(UniqueIdToHumanString(tmp), "0000000000120000-0000AB0000000000"); + + tmp.resize(11); + EXPECT_EQ(UniqueIdToHumanString(tmp), "0000000000120000-0000AB"); + + tmp.resize(6); + EXPECT_EQ(UniqueIdToHumanString(tmp), "000000000012"); + + // Also internal IDs to human string + UniqueId64x3 euid = {12345, 678, 9}; + EXPECT_EQ(InternalUniqueIdToHumanString(&euid), "{12345,678,9}"); + + UniqueId64x2 uid = {1234, 567890}; + EXPECT_EQ(InternalUniqueIdToHumanString(&uid), "{1234,567890}"); +} + +TEST_F(TablePropertyTest, UniqueIdsFailure) { + TableProperties tp; + std::string tmp; + + // Missing DB id + SetGoodTableProperties(&tp); + tp.db_id = ""; + EXPECT_TRUE(GetUniqueIdFromTableProperties(tp, &tmp).IsNotSupported()); + EXPECT_TRUE( + GetExtendedUniqueIdFromTableProperties(tp, &tmp).IsNotSupported()); + + // Missing session id + SetGoodTableProperties(&tp); + tp.db_session_id = ""; + EXPECT_TRUE(GetUniqueIdFromTableProperties(tp, &tmp).IsNotSupported()); + EXPECT_TRUE( + GetExtendedUniqueIdFromTableProperties(tp, &tmp).IsNotSupported()); + + // Missing file number + SetGoodTableProperties(&tp); + tp.orig_file_number = 0; + EXPECT_TRUE(GetUniqueIdFromTableProperties(tp, &tmp).IsNotSupported()); + EXPECT_TRUE( + GetExtendedUniqueIdFromTableProperties(tp, &tmp).IsNotSupported()); +} + +// This test include all the basic checks except those for index size and block +// size, which will be conducted in separated unit tests. +TEST_P(BlockBasedTableTest, BasicBlockBasedTableProperties) { + TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */); + + c.Add("a1", "val1"); + c.Add("b2", "val2"); + c.Add("c3", "val3"); + c.Add("d4", "val4"); + c.Add("e5", "val5"); + c.Add("f6", "val6"); + c.Add("g7", "val7"); + c.Add("h8", "val8"); + c.Add("j9", "val9"); + uint64_t diff_internal_user_bytes = 9 * 8; // 8 is seq size, 9 k-v totally + + std::vector keys; + stl_wrappers::KVMap kvmap; + Options options; + options.compression = kNoCompression; + options.statistics = CreateDBStatistics(); + options.statistics->set_stats_level(StatsLevel::kAll); + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.block_restart_interval = 1; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + ImmutableOptions ioptions(options); + MutableCFOptions moptions(options); + c.Finish(options, ioptions, moptions, table_options, + GetPlainInternalComparator(options.comparator), &keys, &kvmap); + ASSERT_EQ( + options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSION_REJECTED), 0); + + auto& props = *c.GetTableReader()->GetTableProperties(); + ASSERT_EQ(kvmap.size(), props.num_entries); + + auto raw_key_size = kvmap.size() * 2ul; + auto raw_value_size = kvmap.size() * 4ul; + + ASSERT_EQ(raw_key_size + diff_internal_user_bytes, props.raw_key_size); + ASSERT_EQ(raw_value_size, props.raw_value_size); + ASSERT_EQ(1ul, props.num_data_blocks); + ASSERT_EQ("", props.filter_policy_name); // no filter policy is used + + // Verify data size. + BlockBuilder block_builder(1); + for (const auto& item : kvmap) { + block_builder.Add(item.first, item.second); + } + Slice content = block_builder.Finish(); + ASSERT_EQ(content.size() + BlockBasedTable::kBlockTrailerSize + + diff_internal_user_bytes, + props.data_size); + c.ResetTableReader(); +} + +#ifdef SNAPPY +uint64_t BlockBasedTableTest::IndexUncompressedHelper(bool compressed) { + TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */); + constexpr size_t kNumKeys = 10000; + + for (size_t k = 0; k < kNumKeys; ++k) { + c.Add("key" + std::to_string(k), "val" + std::to_string(k)); + } + + std::vector keys; + stl_wrappers::KVMap kvmap; + Options options; + options.compression = kSnappyCompression; + options.statistics = CreateDBStatistics(); + options.statistics->set_stats_level(StatsLevel::kAll); + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.block_restart_interval = 1; + table_options.enable_index_compression = compressed; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + ImmutableOptions ioptions(options); + MutableCFOptions moptions(options); + c.Finish(options, ioptions, moptions, table_options, + GetPlainInternalComparator(options.comparator), &keys, &kvmap); + c.ResetTableReader(); + return options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED); +} +TEST_P(BlockBasedTableTest, IndexUncompressed) { + uint64_t tbl1_compressed_cnt = IndexUncompressedHelper(true); + uint64_t tbl2_compressed_cnt = IndexUncompressedHelper(false); + // tbl1_compressed_cnt should include 1 index block + EXPECT_EQ(tbl2_compressed_cnt + 1, tbl1_compressed_cnt); +} +#endif // SNAPPY + +TEST_P(BlockBasedTableTest, BlockBasedTableProperties2) { + TableConstructor c(&reverse_key_comparator); + std::vector keys; + stl_wrappers::KVMap kvmap; + + { + Options options; + options.compression = CompressionType::kNoCompression; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + c.Finish(options, ioptions, moptions, table_options, + GetPlainInternalComparator(options.comparator), &keys, &kvmap); + + auto& props = *c.GetTableReader()->GetTableProperties(); + + // Default comparator + ASSERT_EQ("leveldb.BytewiseComparator", props.comparator_name); + // No merge operator + ASSERT_EQ("nullptr", props.merge_operator_name); + // No prefix extractor + ASSERT_EQ("nullptr", props.prefix_extractor_name); + // No property collectors + ASSERT_EQ("[]", props.property_collectors_names); + // No filter policy is used + ASSERT_EQ("", props.filter_policy_name); + // Compression type == that set: + ASSERT_EQ("NoCompression", props.compression_name); + c.ResetTableReader(); + } + + { + Options options; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.comparator = &reverse_key_comparator; + options.merge_operator = MergeOperators::CreateUInt64AddOperator(); + options.prefix_extractor.reset(NewNoopTransform()); + options.table_properties_collector_factories.emplace_back( + new DummyPropertiesCollectorFactory1()); + options.table_properties_collector_factories.emplace_back( + new DummyPropertiesCollectorFactory2()); + + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + c.Finish(options, ioptions, moptions, table_options, + GetPlainInternalComparator(options.comparator), &keys, &kvmap); + + auto& props = *c.GetTableReader()->GetTableProperties(); + + ASSERT_EQ("rocksdb.ReverseBytewiseComparator", props.comparator_name); + ASSERT_EQ("UInt64AddOperator", props.merge_operator_name); + ASSERT_EQ("rocksdb.Noop", props.prefix_extractor_name); + ASSERT_EQ( + "[DummyPropertiesCollectorFactory1,DummyPropertiesCollectorFactory2]", + props.property_collectors_names); + ASSERT_EQ("", props.filter_policy_name); // no filter policy is used + c.ResetTableReader(); + } +} + +TEST_P(BlockBasedTableTest, RangeDelBlock) { + TableConstructor c(BytewiseComparator()); + std::vector keys = {"1pika", "2chu"}; + std::vector vals = {"p", "c"}; + + std::vector expected_tombstones = { + {"1pika", "2chu", 0}, + {"2chu", "c", 1}, + {"2chu", "c", 0}, + {"c", "p", 0}, + }; + + for (int i = 0; i < 2; i++) { + RangeTombstone t(keys[i], vals[i], i); + std::pair p = t.Serialize(); + c.Add(p.first.Encode().ToString(), p.second); + } + + std::vector sorted_keys; + stl_wrappers::KVMap kvmap; + Options options; + options.compression = kNoCompression; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.block_restart_interval = 1; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + std::unique_ptr internal_cmp( + new InternalKeyComparator(options.comparator)); + c.Finish(options, ioptions, moptions, table_options, *internal_cmp, + &sorted_keys, &kvmap); + + for (int j = 0; j < 2; ++j) { + std::unique_ptr iter( + c.GetTableReader()->NewRangeTombstoneIterator(ReadOptions())); + if (j > 0) { + // For second iteration, delete the table reader object and verify the + // iterator can still access its metablock's range tombstones. + c.ResetTableReader(); + } + ASSERT_FALSE(iter->Valid()); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + for (size_t i = 0; i < expected_tombstones.size(); i++) { + ASSERT_TRUE(iter->Valid()); + ParsedInternalKey parsed_key; + ASSERT_OK( + ParseInternalKey(iter->key(), &parsed_key, true /* log_err_key */)); + RangeTombstone t(parsed_key, iter->value()); + const auto& expected_t = expected_tombstones[i]; + ASSERT_EQ(t.start_key_, expected_t.start_key_); + ASSERT_EQ(t.end_key_, expected_t.end_key_); + ASSERT_EQ(t.seq_, expected_t.seq_); + iter->Next(); + } + ASSERT_TRUE(!iter->Valid()); + } +} + +TEST_P(BlockBasedTableTest, FilterPolicyNameProperties) { + TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */); + c.Add("a1", "val1"); + std::vector keys; + stl_wrappers::KVMap kvmap; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.filter_policy.reset(NewBloomFilterPolicy(10)); + Options options; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + c.Finish(options, ioptions, moptions, table_options, + GetPlainInternalComparator(options.comparator), &keys, &kvmap); + auto& props = *c.GetTableReader()->GetTableProperties(); + ASSERT_EQ(table_options.filter_policy->Name(), props.filter_policy_name); + c.ResetTableReader(); +} + +// +// BlockBasedTableTest::PrefetchTest +// +void AssertKeysInCache(BlockBasedTable* table_reader, + const std::vector& keys_in_cache, + const std::vector& keys_not_in_cache, + bool convert = false) { + if (convert) { + for (auto key : keys_in_cache) { + InternalKey ikey(key, kMaxSequenceNumber, kTypeValue); + ASSERT_TRUE(table_reader->TEST_KeyInCache(ReadOptions(), ikey.Encode())); + } + for (auto key : keys_not_in_cache) { + InternalKey ikey(key, kMaxSequenceNumber, kTypeValue); + ASSERT_TRUE(!table_reader->TEST_KeyInCache(ReadOptions(), ikey.Encode())); + } + } else { + for (auto key : keys_in_cache) { + ASSERT_TRUE(table_reader->TEST_KeyInCache(ReadOptions(), key)); + } + for (auto key : keys_not_in_cache) { + ASSERT_TRUE(!table_reader->TEST_KeyInCache(ReadOptions(), key)); + } + } +} + +void PrefetchRange(TableConstructor* c, Options* opt, + BlockBasedTableOptions* table_options, const char* key_begin, + const char* key_end, + const std::vector& keys_in_cache, + const std::vector& keys_not_in_cache, + const Status expected_status = Status::OK()) { + // reset the cache and reopen the table + table_options->block_cache = NewLRUCache(16 * 1024 * 1024, 4); + opt->table_factory.reset(NewBlockBasedTableFactory(*table_options)); + const ImmutableOptions ioptions2(*opt); + const MutableCFOptions moptions(*opt); + ASSERT_OK(c->Reopen(ioptions2, moptions)); + + // prefetch + auto* table_reader = dynamic_cast(c->GetTableReader()); + Status s; + std::unique_ptr begin, end; + std::unique_ptr i_begin, i_end; + if (key_begin != nullptr) { + if (c->ConvertToInternalKey()) { + i_begin.reset(new InternalKey(key_begin, kMaxSequenceNumber, kTypeValue)); + begin.reset(new Slice(i_begin->Encode())); + } else { + begin.reset(new Slice(key_begin)); + } + } + if (key_end != nullptr) { + if (c->ConvertToInternalKey()) { + i_end.reset(new InternalKey(key_end, kMaxSequenceNumber, kTypeValue)); + end.reset(new Slice(i_end->Encode())); + } else { + end.reset(new Slice(key_end)); + } + } + const ReadOptions read_options; + s = table_reader->Prefetch(read_options, begin.get(), end.get()); + + ASSERT_TRUE(s.code() == expected_status.code()); + + // assert our expectation in cache warmup + AssertKeysInCache(table_reader, keys_in_cache, keys_not_in_cache, + c->ConvertToInternalKey()); + c->ResetTableReader(); +} + +TEST_P(BlockBasedTableTest, PrefetchTest) { + // The purpose of this test is to test the prefetching operation built into + // BlockBasedTable. + Options opt; + std::unique_ptr ikc; + ikc.reset(new test::PlainInternalKeyComparator(opt.comparator)); + opt.compression = kNoCompression; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.block_size = 1024; + // big enough so we don't ever lose cached values. + table_options.block_cache = NewLRUCache(16 * 1024 * 1024, 4); + opt.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */); + c.Add("k01", "hello"); + c.Add("k02", "hello2"); + c.Add("k03", std::string(10000, 'x')); + c.Add("k04", std::string(200000, 'x')); + c.Add("k05", std::string(300000, 'x')); + c.Add("k06", "hello3"); + c.Add("k07", std::string(100000, 'x')); + std::vector keys; + stl_wrappers::KVMap kvmap; + const ImmutableOptions ioptions(opt); + const MutableCFOptions moptions(opt); + c.Finish(opt, ioptions, moptions, table_options, *ikc, &keys, &kvmap); + c.ResetTableReader(); + + // We get the following data spread : + // + // Data block Index + // ======================== + // [ k01 k02 k03 ] k03 + // [ k04 ] k04 + // [ k05 ] k05 + // [ k06 k07 ] k07 + + // Simple + PrefetchRange(&c, &opt, &table_options, + /*key_range=*/"k01", "k05", + /*keys_in_cache=*/{"k01", "k02", "k03", "k04", "k05"}, + /*keys_not_in_cache=*/{"k06", "k07"}); + PrefetchRange(&c, &opt, &table_options, "k01", "k01", {"k01", "k02", "k03"}, + {"k04", "k05", "k06", "k07"}); + // odd + PrefetchRange(&c, &opt, &table_options, "a", "z", + {"k01", "k02", "k03", "k04", "k05", "k06", "k07"}, {}); + PrefetchRange(&c, &opt, &table_options, "k00", "k00", {"k01", "k02", "k03"}, + {"k04", "k05", "k06", "k07"}); + // Edge cases + PrefetchRange(&c, &opt, &table_options, "k00", "k06", + {"k01", "k02", "k03", "k04", "k05", "k06", "k07"}, {}); + PrefetchRange(&c, &opt, &table_options, "k00", "zzz", + {"k01", "k02", "k03", "k04", "k05", "k06", "k07"}, {}); + // null keys + PrefetchRange(&c, &opt, &table_options, nullptr, nullptr, + {"k01", "k02", "k03", "k04", "k05", "k06", "k07"}, {}); + PrefetchRange(&c, &opt, &table_options, "k04", nullptr, + {"k04", "k05", "k06", "k07"}, {"k01", "k02", "k03"}); + PrefetchRange(&c, &opt, &table_options, nullptr, "k05", + {"k01", "k02", "k03", "k04", "k05"}, {"k06", "k07"}); + // invalid + PrefetchRange(&c, &opt, &table_options, "k06", "k00", {}, {}, + Status::InvalidArgument(Slice("k06 "), Slice("k07"))); + c.ResetTableReader(); +} + +TEST_P(BlockBasedTableTest, TotalOrderSeekOnHashIndex) { + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + for (int i = 0; i <= 4; ++i) { + Options options; + // Make each key/value an individual block + table_options.block_size = 64; + switch (i) { + case 0: + // Binary search index + table_options.index_type = BlockBasedTableOptions::kBinarySearch; + options.table_factory.reset(new BlockBasedTableFactory(table_options)); + break; + case 1: + // Hash search index + table_options.index_type = BlockBasedTableOptions::kHashSearch; + options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.prefix_extractor.reset(NewFixedPrefixTransform(4)); + break; + case 2: + // Hash search index with filter policy + table_options.index_type = BlockBasedTableOptions::kHashSearch; + table_options.filter_policy.reset(NewBloomFilterPolicy(10)); + options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.prefix_extractor.reset(NewFixedPrefixTransform(4)); + break; + case 3: + // Two-level index + table_options.index_type = BlockBasedTableOptions::kTwoLevelIndexSearch; + options.table_factory.reset(new BlockBasedTableFactory(table_options)); + break; + case 4: + // Binary search with first key + table_options.index_type = + BlockBasedTableOptions::kBinarySearchWithFirstKey; + options.table_factory.reset(new BlockBasedTableFactory(table_options)); + break; + } + + TableConstructor c(BytewiseComparator(), + true /* convert_to_internal_key_ */); + c.Add("aaaa1", std::string('a', 56)); + c.Add("bbaa1", std::string('a', 56)); + c.Add("cccc1", std::string('a', 56)); + c.Add("bbbb1", std::string('a', 56)); + c.Add("baaa1", std::string('a', 56)); + c.Add("abbb1", std::string('a', 56)); + c.Add("cccc2", std::string('a', 56)); + std::vector keys; + stl_wrappers::KVMap kvmap; + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + c.Finish(options, ioptions, moptions, table_options, + GetPlainInternalComparator(options.comparator), &keys, &kvmap); + auto props = c.GetTableReader()->GetTableProperties(); + ASSERT_EQ(7u, props->num_data_blocks); + auto* reader = c.GetTableReader(); + ReadOptions ro; + ro.total_order_seek = true; + std::unique_ptr iter(reader->NewIterator( + ro, moptions.prefix_extractor.get(), /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized)); + + iter->Seek(InternalKey("b", 0, kTypeValue).Encode()); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("baaa1", ExtractUserKey(iter->key()).ToString()); + iter->Next(); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("bbaa1", ExtractUserKey(iter->key()).ToString()); + + iter->Seek(InternalKey("bb", 0, kTypeValue).Encode()); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("bbaa1", ExtractUserKey(iter->key()).ToString()); + iter->Next(); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("bbbb1", ExtractUserKey(iter->key()).ToString()); + + iter->Seek(InternalKey("bbb", 0, kTypeValue).Encode()); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("bbbb1", ExtractUserKey(iter->key()).ToString()); + iter->Next(); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("cccc1", ExtractUserKey(iter->key()).ToString()); + } +} + +TEST_P(BlockBasedTableTest, NoopTransformSeek) { + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.filter_policy.reset(NewBloomFilterPolicy(10)); + + Options options; + options.comparator = BytewiseComparator(); + options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.prefix_extractor.reset(NewNoopTransform()); + + TableConstructor c(options.comparator); + // To tickle the PrefixMayMatch bug it is important that the + // user-key is a single byte so that the index key exactly matches + // the user-key. + InternalKey key("a", 1, kTypeValue); + c.Add(key.Encode().ToString(), "b"); + std::vector keys; + stl_wrappers::KVMap kvmap; + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + const InternalKeyComparator internal_comparator(options.comparator); + c.Finish(options, ioptions, moptions, table_options, internal_comparator, + &keys, &kvmap); + + auto* reader = c.GetTableReader(); + for (int i = 0; i < 2; ++i) { + ReadOptions ro; + ro.total_order_seek = (i == 0); + std::unique_ptr iter(reader->NewIterator( + ro, moptions.prefix_extractor.get(), /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized)); + + iter->Seek(key.Encode()); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("a", ExtractUserKey(iter->key()).ToString()); + } +} + +TEST_P(BlockBasedTableTest, SkipPrefixBloomFilter) { + // if DB is opened with a prefix extractor of a different name, + // prefix bloom is skipped when read the file + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.filter_policy.reset(NewBloomFilterPolicy(2)); + table_options.whole_key_filtering = false; + + Options options; + options.comparator = BytewiseComparator(); + options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.prefix_extractor.reset(NewFixedPrefixTransform(1)); + + TableConstructor c(options.comparator); + InternalKey key("abcdefghijk", 1, kTypeValue); + c.Add(key.Encode().ToString(), "test"); + std::vector keys; + stl_wrappers::KVMap kvmap; + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + const InternalKeyComparator internal_comparator(options.comparator); + c.Finish(options, ioptions, moptions, table_options, internal_comparator, + &keys, &kvmap); + // TODO(Zhongyi): update test to use MutableCFOptions + options.prefix_extractor.reset(NewFixedPrefixTransform(9)); + const ImmutableOptions new_ioptions(options); + const MutableCFOptions new_moptions(options); + ASSERT_OK(c.Reopen(new_ioptions, new_moptions)); + auto reader = c.GetTableReader(); + ReadOptions read_options; + std::unique_ptr db_iter(reader->NewIterator( + read_options, new_moptions.prefix_extractor.get(), /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized)); + + // Test point lookup + // only one kv + for (auto& kv : kvmap) { + db_iter->Seek(kv.first); + ASSERT_TRUE(db_iter->Valid()); + ASSERT_OK(db_iter->status()); + ASSERT_EQ(db_iter->key(), kv.first); + ASSERT_EQ(db_iter->value(), kv.second); + } +} + +TEST_P(BlockBasedTableTest, BadChecksumType) { + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + + Options options; + options.comparator = BytewiseComparator(); + options.table_factory.reset(new BlockBasedTableFactory(table_options)); + + TableConstructor c(options.comparator); + InternalKey key("abc", 1, kTypeValue); + c.Add(key.Encode().ToString(), "test"); + std::vector keys; + stl_wrappers::KVMap kvmap; + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + const InternalKeyComparator internal_comparator(options.comparator); + c.Finish(options, ioptions, moptions, table_options, internal_comparator, + &keys, &kvmap); + + // Corrupt checksum type (123 is invalid) + auto& sink = *c.TEST_GetSink(); + size_t len = sink.contents_.size(); + ASSERT_EQ(sink.contents_[len - Footer::kNewVersionsEncodedLength], + table_options.checksum); + sink.contents_[len - Footer::kNewVersionsEncodedLength] = char{123}; + + // (Re-)Open table file with bad checksum type + const ImmutableOptions new_ioptions(options); + const MutableCFOptions new_moptions(options); + Status s = c.Reopen(new_ioptions, new_moptions); + ASSERT_NOK(s); + // "test" is file name + ASSERT_EQ(s.ToString(), + "Corruption: Corrupt or unsupported checksum type: 123 in test"); +} + +class BuiltinChecksumTest : public testing::Test, + public testing::WithParamInterface {}; + +INSTANTIATE_TEST_CASE_P(SupportedChecksums, BuiltinChecksumTest, + testing::ValuesIn(GetSupportedChecksums())); + +namespace { +std::string ChecksumAsString(const std::string& data, + ChecksumType checksum_type) { + uint32_t v = ComputeBuiltinChecksum(checksum_type, data.data(), data.size()); + + // Verify consistency with other function + if (data.size() >= 1) { + EXPECT_EQ(v, ComputeBuiltinChecksumWithLastByte( + checksum_type, data.data(), data.size() - 1, data.back())); + } + // Little endian as in file + std::array raw_bytes; + EncodeFixed32(raw_bytes.data(), v); + return Slice(raw_bytes.data(), raw_bytes.size()).ToString(/*hex*/ true); +} + +std::string ChecksumAsString(std::string* data, char new_last_byte, + ChecksumType checksum_type) { + data->back() = new_last_byte; + return ChecksumAsString(*data, checksum_type); +} +} // namespace + +// Make sure that checksum values don't change in later versions, even if +// consistent within current version. +TEST_P(BuiltinChecksumTest, ChecksumSchemas) { + // Trailing 'x' chars will be replaced by compression type. Specifically, + // the first byte of a block trailer is compression type, which is part of + // the checksum input. This test does not deal with storing or parsing + // checksums from the trailer (next 4 bytes of trailer). + std::string b0 = "x"; + std::string b1 = "This is a short block!x"; + std::string b2; + for (int i = 0; i < 100; ++i) { + b2.append("This is a long block!"); + } + b2.append("x"); + + std::string empty; + + char ct1 = kNoCompression; + char ct2 = kSnappyCompression; + char ct3 = kZSTD; + + ChecksumType t = GetParam(); + switch (t) { + case kNoChecksum: + EXPECT_EQ(ChecksumAsString(empty, t), "00000000"); + EXPECT_EQ(ChecksumAsString(&b0, ct1, t), "00000000"); + EXPECT_EQ(ChecksumAsString(&b0, ct2, t), "00000000"); + EXPECT_EQ(ChecksumAsString(&b0, ct3, t), "00000000"); + EXPECT_EQ(ChecksumAsString(&b1, ct1, t), "00000000"); + EXPECT_EQ(ChecksumAsString(&b1, ct2, t), "00000000"); + EXPECT_EQ(ChecksumAsString(&b1, ct3, t), "00000000"); + EXPECT_EQ(ChecksumAsString(&b2, ct1, t), "00000000"); + EXPECT_EQ(ChecksumAsString(&b2, ct2, t), "00000000"); + EXPECT_EQ(ChecksumAsString(&b2, ct3, t), "00000000"); + break; + case kCRC32c: + EXPECT_EQ(ChecksumAsString(empty, t), "D8EA82A2"); + EXPECT_EQ(ChecksumAsString(&b0, ct1, t), "D28F2549"); + EXPECT_EQ(ChecksumAsString(&b0, ct2, t), "052B2843"); + EXPECT_EQ(ChecksumAsString(&b0, ct3, t), "46F8F711"); + EXPECT_EQ(ChecksumAsString(&b1, ct1, t), "583F0355"); + EXPECT_EQ(ChecksumAsString(&b1, ct2, t), "2F9B0A57"); + EXPECT_EQ(ChecksumAsString(&b1, ct3, t), "ECE7DA1D"); + EXPECT_EQ(ChecksumAsString(&b2, ct1, t), "943EF0AB"); + EXPECT_EQ(ChecksumAsString(&b2, ct2, t), "43A2EDB1"); + EXPECT_EQ(ChecksumAsString(&b2, ct3, t), "00E53D63"); + break; + case kxxHash: + EXPECT_EQ(ChecksumAsString(empty, t), "055DCC02"); + EXPECT_EQ(ChecksumAsString(&b0, ct1, t), "3EB065CF"); + EXPECT_EQ(ChecksumAsString(&b0, ct2, t), "31F79238"); + EXPECT_EQ(ChecksumAsString(&b0, ct3, t), "320D2E00"); + EXPECT_EQ(ChecksumAsString(&b1, ct1, t), "4A2E5FB0"); + EXPECT_EQ(ChecksumAsString(&b1, ct2, t), "0BD9F652"); + EXPECT_EQ(ChecksumAsString(&b1, ct3, t), "B4107E50"); + EXPECT_EQ(ChecksumAsString(&b2, ct1, t), "20F4D4BA"); + EXPECT_EQ(ChecksumAsString(&b2, ct2, t), "8F1A1F99"); + EXPECT_EQ(ChecksumAsString(&b2, ct3, t), "A191A338"); + break; + case kxxHash64: + EXPECT_EQ(ChecksumAsString(empty, t), "99E9D851"); + EXPECT_EQ(ChecksumAsString(&b0, ct1, t), "682705DB"); + EXPECT_EQ(ChecksumAsString(&b0, ct2, t), "30E7211B"); + EXPECT_EQ(ChecksumAsString(&b0, ct3, t), "B7BB58E8"); + EXPECT_EQ(ChecksumAsString(&b1, ct1, t), "B74655EF"); + EXPECT_EQ(ChecksumAsString(&b1, ct2, t), "B6C8BBBE"); + EXPECT_EQ(ChecksumAsString(&b1, ct3, t), "AED9E3B4"); + EXPECT_EQ(ChecksumAsString(&b2, ct1, t), "0D4999FE"); + EXPECT_EQ(ChecksumAsString(&b2, ct2, t), "F5932423"); + EXPECT_EQ(ChecksumAsString(&b2, ct3, t), "6B31BAB1"); + break; + case kXXH3: + EXPECT_EQ(ChecksumAsString(empty, t), "00000000"); + EXPECT_EQ(ChecksumAsString(&b0, ct1, t), "C294D338"); + EXPECT_EQ(ChecksumAsString(&b0, ct2, t), "1B174353"); + EXPECT_EQ(ChecksumAsString(&b0, ct3, t), "2D0E20C8"); + EXPECT_EQ(ChecksumAsString(&b1, ct1, t), "B37FB5E6"); + EXPECT_EQ(ChecksumAsString(&b1, ct2, t), "6AFC258D"); + EXPECT_EQ(ChecksumAsString(&b1, ct3, t), "5CE54616"); + EXPECT_EQ(ChecksumAsString(&b2, ct1, t), "FA2D482E"); + EXPECT_EQ(ChecksumAsString(&b2, ct2, t), "23AED845"); + EXPECT_EQ(ChecksumAsString(&b2, ct3, t), "15B7BBDE"); + break; + default: + // Force this test to be updated on new ChecksumTypes + assert(false); + break; + } +} + +TEST_P(BuiltinChecksumTest, ChecksumZeroInputs) { + // Verify that no reasonably sized "all zeros" inputs produce "all zeros" + // output. Otherwise, "wiped" data could appear to be well-formed. + // Assuming essentially random assignment of output values, the likelihood + // of encountering checksum == 0 for an input not specifically crafted is + // 1 in 4 billion. + if (GetParam() == kNoChecksum) { + return; + } + // "Thorough" case is too slow for continouous testing + bool thorough = getenv("ROCKSDB_THOROUGH_CHECKSUM_TEST") != nullptr; + // Verified through 10M + size_t kMaxZerosLen = thorough ? 10000000 : 20000; + std::string zeros(kMaxZerosLen, '\0'); + + for (size_t len = 0; len < kMaxZerosLen; ++len) { + if (thorough && (len & 0xffffU) == 0) { + fprintf(stderr, "t=%u len=%u\n", (unsigned)GetParam(), (unsigned)len); + } + uint32_t v = ComputeBuiltinChecksum(GetParam(), zeros.data(), len); + if (v == 0U) { + // One exception case: + if (GetParam() == kXXH3 && len == 0) { + // This is not a big deal because assuming the block length is known + // from the block handle, which comes from a checksum-verified block, + // there is nothing to corrupt in a zero-length block. And when there + // is a block trailer with compression byte (as in block-based table), + // zero length checksummed data never arises. + continue; + } + // Only compute this on failure + SCOPED_TRACE("len=" + std::to_string(len)); + ASSERT_NE(v, 0U); + } + } +} + +void AddInternalKey(TableConstructor* c, const std::string& prefix, + std::string value = "v", int /*suffix_len*/ = 800) { + static Random rnd(1023); + InternalKey k(prefix + rnd.RandomString(800), 0, kTypeValue); + c->Add(k.Encode().ToString(), value); +} + +void TableTest::IndexTest(BlockBasedTableOptions table_options) { + TableConstructor c(BytewiseComparator()); + + // keys with prefix length 3, make sure the key/value is big enough to fill + // one block + AddInternalKey(&c, "0015"); + AddInternalKey(&c, "0035"); + + AddInternalKey(&c, "0054"); + AddInternalKey(&c, "0055"); + + AddInternalKey(&c, "0056"); + AddInternalKey(&c, "0057"); + + AddInternalKey(&c, "0058"); + AddInternalKey(&c, "0075"); + + AddInternalKey(&c, "0076"); + AddInternalKey(&c, "0095"); + + std::vector keys; + stl_wrappers::KVMap kvmap; + Options options; + options.prefix_extractor.reset(NewFixedPrefixTransform(3)); + table_options.block_size = 1700; + table_options.block_cache = NewLRUCache(1024, 4); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + std::unique_ptr comparator( + new InternalKeyComparator(BytewiseComparator())); + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + c.Finish(options, ioptions, moptions, table_options, *comparator, &keys, + &kvmap); + auto reader = c.GetTableReader(); + + auto props = reader->GetTableProperties(); + ASSERT_EQ(5u, props->num_data_blocks); + + // TODO(Zhongyi): update test to use MutableCFOptions + ReadOptions read_options; + std::unique_ptr index_iter(reader->NewIterator( + read_options, moptions.prefix_extractor.get(), /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized)); + + // -- Find keys do not exist, but have common prefix. + std::vector prefixes = {"001", "003", "005", "007", "009"}; + std::vector lower_bound = { + keys[0], keys[1], keys[2], keys[7], keys[9], + }; + + // find the lower bound of the prefix + for (size_t i = 0; i < prefixes.size(); ++i) { + index_iter->Seek(InternalKey(prefixes[i], 0, kTypeValue).Encode()); + ASSERT_OK(index_iter->status()); + ASSERT_TRUE(index_iter->Valid()); + + // seek the first element in the block + ASSERT_EQ(lower_bound[i], index_iter->key().ToString()); + ASSERT_EQ("v", index_iter->value().ToString()); + } + + // find the upper bound of prefixes + std::vector upper_bound = { + keys[1], + keys[2], + keys[7], + keys[9], + }; + + // find existing keys + for (const auto& item : kvmap) { + auto ukey = ExtractUserKey(item.first).ToString(); + index_iter->Seek(ukey); + + // ASSERT_OK(regular_iter->status()); + ASSERT_OK(index_iter->status()); + + // ASSERT_TRUE(regular_iter->Valid()); + ASSERT_TRUE(index_iter->Valid()); + + ASSERT_EQ(item.first, index_iter->key().ToString()); + ASSERT_EQ(item.second, index_iter->value().ToString()); + } + + for (size_t i = 0; i < prefixes.size(); ++i) { + // the key is greater than any existing keys. + auto key = prefixes[i] + "9"; + index_iter->Seek(InternalKey(key, 0, kTypeValue).Encode()); + + ASSERT_TRUE(index_iter->status().ok() || index_iter->status().IsNotFound()); + ASSERT_TRUE(!index_iter->status().IsNotFound() || !index_iter->Valid()); + if (i == prefixes.size() - 1) { + // last key + ASSERT_TRUE(!index_iter->Valid()); + } else { + ASSERT_TRUE(index_iter->Valid()); + // seek the first element in the block + ASSERT_EQ(upper_bound[i], index_iter->key().ToString()); + ASSERT_EQ("v", index_iter->value().ToString()); + } + } + + // find keys with prefix that don't match any of the existing prefixes. + std::vector non_exist_prefixes = {"002", "004", "006", "008"}; + for (const auto& prefix : non_exist_prefixes) { + index_iter->Seek(InternalKey(prefix, 0, kTypeValue).Encode()); + // regular_iter->Seek(prefix); + + ASSERT_OK(index_iter->status()); + // Seek to non-existing prefixes should yield either invalid, or a + // key with prefix greater than the target. + if (index_iter->Valid()) { + Slice ukey = ExtractUserKey(index_iter->key()); + Slice ukey_prefix = options.prefix_extractor->Transform(ukey); + ASSERT_TRUE(BytewiseComparator()->Compare(prefix, ukey_prefix) < 0); + } + } + for (const auto& prefix : non_exist_prefixes) { + index_iter->SeekForPrev(InternalKey(prefix, 0, kTypeValue).Encode()); + // regular_iter->Seek(prefix); + + ASSERT_OK(index_iter->status()); + // Seek to non-existing prefixes should yield either invalid, or a + // key with prefix greater than the target. + if (index_iter->Valid()) { + Slice ukey = ExtractUserKey(index_iter->key()); + Slice ukey_prefix = options.prefix_extractor->Transform(ukey); + ASSERT_TRUE(BytewiseComparator()->Compare(prefix, ukey_prefix) > 0); + } + } + + { + // Test reseek case. It should impact partitioned index more. + ReadOptions ro; + ro.total_order_seek = true; + std::unique_ptr index_iter2(reader->NewIterator( + ro, moptions.prefix_extractor.get(), /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized)); + + // Things to cover in partitioned index: + // 1. Both of Seek() and SeekToLast() has optimization to prevent + // rereek leaf index block if it remains to the same one, and + // they reuse the same variable. + // 2. When Next() or Prev() is called, the block moves, so the + // optimization should kick in only with the current one. + index_iter2->Seek(InternalKey("0055", 0, kTypeValue).Encode()); + ASSERT_TRUE(index_iter2->Valid()); + ASSERT_EQ("0055", index_iter2->key().ToString().substr(0, 4)); + + index_iter2->SeekToLast(); + ASSERT_TRUE(index_iter2->Valid()); + ASSERT_EQ("0095", index_iter2->key().ToString().substr(0, 4)); + + index_iter2->Seek(InternalKey("0055", 0, kTypeValue).Encode()); + ASSERT_TRUE(index_iter2->Valid()); + ASSERT_EQ("0055", index_iter2->key().ToString().substr(0, 4)); + + index_iter2->SeekToLast(); + ASSERT_TRUE(index_iter2->Valid()); + ASSERT_EQ("0095", index_iter2->key().ToString().substr(0, 4)); + index_iter2->Prev(); + ASSERT_TRUE(index_iter2->Valid()); + index_iter2->Prev(); + ASSERT_TRUE(index_iter2->Valid()); + ASSERT_EQ("0075", index_iter2->key().ToString().substr(0, 4)); + + index_iter2->Seek(InternalKey("0095", 0, kTypeValue).Encode()); + ASSERT_TRUE(index_iter2->Valid()); + ASSERT_EQ("0095", index_iter2->key().ToString().substr(0, 4)); + index_iter2->Prev(); + ASSERT_TRUE(index_iter2->Valid()); + index_iter2->Prev(); + ASSERT_TRUE(index_iter2->Valid()); + ASSERT_EQ("0075", index_iter2->key().ToString().substr(0, 4)); + + index_iter2->SeekToLast(); + ASSERT_TRUE(index_iter2->Valid()); + ASSERT_EQ("0095", index_iter2->key().ToString().substr(0, 4)); + + index_iter2->Seek(InternalKey("0095", 0, kTypeValue).Encode()); + ASSERT_TRUE(index_iter2->Valid()); + ASSERT_EQ("0095", index_iter2->key().ToString().substr(0, 4)); + + index_iter2->Prev(); + ASSERT_TRUE(index_iter2->Valid()); + index_iter2->Prev(); + ASSERT_TRUE(index_iter2->Valid()); + ASSERT_EQ("0075", index_iter2->key().ToString().substr(0, 4)); + + index_iter2->Seek(InternalKey("0075", 0, kTypeValue).Encode()); + ASSERT_TRUE(index_iter2->Valid()); + ASSERT_EQ("0075", index_iter2->key().ToString().substr(0, 4)); + + index_iter2->Next(); + ASSERT_TRUE(index_iter2->Valid()); + index_iter2->Next(); + ASSERT_TRUE(index_iter2->Valid()); + ASSERT_EQ("0095", index_iter2->key().ToString().substr(0, 4)); + + index_iter2->SeekToLast(); + ASSERT_TRUE(index_iter2->Valid()); + ASSERT_EQ("0095", index_iter2->key().ToString().substr(0, 4)); + } + + c.ResetTableReader(); +} + +TEST_P(BlockBasedTableTest, BinaryIndexTest) { + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.index_type = BlockBasedTableOptions::kBinarySearch; + IndexTest(table_options); +} + +TEST_P(BlockBasedTableTest, HashIndexTest) { + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.index_type = BlockBasedTableOptions::kHashSearch; + IndexTest(table_options); +} + +TEST_P(BlockBasedTableTest, PartitionIndexTest) { + const int max_index_keys = 5; + const int est_max_index_key_value_size = 32; + const int est_max_index_size = max_index_keys * est_max_index_key_value_size; + for (int i = 1; i <= est_max_index_size + 1; i++) { + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.index_type = BlockBasedTableOptions::kTwoLevelIndexSearch; + table_options.metadata_block_size = i; + IndexTest(table_options); + } +} + +TEST_P(BlockBasedTableTest, IndexSeekOptimizationIncomplete) { + std::unique_ptr comparator( + new InternalKeyComparator(BytewiseComparator())); + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + Options options; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + + TableConstructor c(BytewiseComparator()); + AddInternalKey(&c, "pika"); + + std::vector keys; + stl_wrappers::KVMap kvmap; + c.Finish(options, ioptions, moptions, table_options, *comparator, &keys, + &kvmap); + ASSERT_EQ(1, keys.size()); + + auto reader = c.GetTableReader(); + ReadOptions ropt; + ropt.read_tier = ReadTier::kBlockCacheTier; + std::unique_ptr iter(reader->NewIterator( + ropt, /*prefix_extractor=*/nullptr, /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized)); + + auto ikey = [](Slice user_key) { + return InternalKey(user_key, 0, kTypeValue).Encode().ToString(); + }; + + iter->Seek(ikey("pika")); + ASSERT_FALSE(iter->Valid()); + ASSERT_TRUE(iter->status().IsIncomplete()); + + // This used to crash at some point. + iter->Seek(ikey("pika")); + ASSERT_FALSE(iter->Valid()); + ASSERT_TRUE(iter->status().IsIncomplete()); +} + +TEST_P(BlockBasedTableTest, BinaryIndexWithFirstKey1) { + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.index_type = BlockBasedTableOptions::kBinarySearchWithFirstKey; + IndexTest(table_options); +} + +class CustomFlushBlockPolicy : public FlushBlockPolicyFactory, + public FlushBlockPolicy { + public: + explicit CustomFlushBlockPolicy(std::vector keys_per_block) + : keys_per_block_(keys_per_block) {} + + const char* Name() const override { return "CustomFlushBlockPolicy"; } + + FlushBlockPolicy* NewFlushBlockPolicy(const BlockBasedTableOptions&, + const BlockBuilder&) const override { + return new CustomFlushBlockPolicy(keys_per_block_); + } + + bool Update(const Slice&, const Slice&) override { + if (keys_in_current_block_ >= keys_per_block_.at(current_block_idx_)) { + ++current_block_idx_; + keys_in_current_block_ = 1; + return true; + } + + ++keys_in_current_block_; + return false; + } + + std::vector keys_per_block_; + + int current_block_idx_ = 0; + int keys_in_current_block_ = 0; +}; + +TEST_P(BlockBasedTableTest, BinaryIndexWithFirstKey2) { + for (int use_first_key = 0; use_first_key < 2; ++use_first_key) { + SCOPED_TRACE("use_first_key = " + std::to_string(use_first_key)); + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.index_type = + use_first_key ? BlockBasedTableOptions::kBinarySearchWithFirstKey + : BlockBasedTableOptions::kBinarySearch; + table_options.block_cache = NewLRUCache(10000); // fits all blocks + table_options.index_shortening = + BlockBasedTableOptions::IndexShorteningMode::kNoShortening; + table_options.flush_block_policy_factory = + std::make_shared(std::vector{2, 1, 3, 2}); + Options options; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.statistics = CreateDBStatistics(); + Statistics* stats = options.statistics.get(); + std::unique_ptr comparator( + new InternalKeyComparator(BytewiseComparator())); + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + + TableConstructor c(BytewiseComparator()); + + // Block 0. + AddInternalKey(&c, "aaaa", "v0"); + AddInternalKey(&c, "aaac", "v1"); + + // Block 1. + AddInternalKey(&c, "aaca", "v2"); + + // Block 2. + AddInternalKey(&c, "caaa", "v3"); + AddInternalKey(&c, "caac", "v4"); + AddInternalKey(&c, "caae", "v5"); + + // Block 3. + AddInternalKey(&c, "ccaa", "v6"); + AddInternalKey(&c, "ccac", "v7"); + + // Write the file. + std::vector keys; + stl_wrappers::KVMap kvmap; + c.Finish(options, ioptions, moptions, table_options, *comparator, &keys, + &kvmap); + ASSERT_EQ(8, keys.size()); + + auto reader = c.GetTableReader(); + auto props = reader->GetTableProperties(); + ASSERT_EQ(4u, props->num_data_blocks); + ReadOptions read_options; + std::unique_ptr iter(reader->NewIterator( + read_options, /*prefix_extractor=*/nullptr, /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized, + /*compaction_readahead_size=*/0, /*allow_unprepared_value=*/true)); + + // Shouldn't have read data blocks before iterator is seeked. + EXPECT_EQ(0, stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + EXPECT_EQ(0, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + + auto ikey = [](Slice user_key) { + return InternalKey(user_key, 0, kTypeValue).Encode().ToString(); + }; + + // Seek to a key between blocks. If index contains first key, we shouldn't + // read any data blocks until value is requested. + iter->Seek(ikey("aaba")); + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ(keys[2], iter->key().ToString()); + EXPECT_EQ(use_first_key ? 0 : 1, + stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + ASSERT_TRUE(iter->PrepareValue()); + EXPECT_EQ("v2", iter->value().ToString()); + EXPECT_EQ(1, stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + EXPECT_EQ(0, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + + // Seek to the middle of a block. The block should be read right away. + iter->Seek(ikey("caab")); + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ(keys[4], iter->key().ToString()); + EXPECT_EQ(2, stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + EXPECT_EQ(0, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + ASSERT_TRUE(iter->PrepareValue()); + EXPECT_EQ("v4", iter->value().ToString()); + EXPECT_EQ(0, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + + // Seek to just before the same block and don't access value. + // The iterator should keep pinning the block contents. + iter->Seek(ikey("baaa")); + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ(keys[3], iter->key().ToString()); + EXPECT_EQ(0, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + + // Seek to the same block again to check that the block is still pinned. + iter->Seek(ikey("caae")); + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ(keys[5], iter->key().ToString()); + EXPECT_EQ(0, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + ASSERT_TRUE(iter->PrepareValue()); + EXPECT_EQ("v5", iter->value().ToString()); + EXPECT_EQ(2, stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + EXPECT_EQ(0, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + + // Step forward and fall through to the next block. Don't access value. + iter->Next(); + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ(keys[6], iter->key().ToString()); + EXPECT_EQ(use_first_key ? 2 : 3, + stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + EXPECT_EQ(0, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + + // Step forward again. Block should be read. + iter->Next(); + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ(keys[7], iter->key().ToString()); + EXPECT_EQ(3, stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + ASSERT_TRUE(iter->PrepareValue()); + EXPECT_EQ("v7", iter->value().ToString()); + EXPECT_EQ(0, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + + // Step forward and reach the end. + iter->Next(); + EXPECT_FALSE(iter->Valid()); + EXPECT_EQ(3, stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + EXPECT_EQ(0, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + + // Seek to a single-key block and step forward without accessing value. + iter->Seek(ikey("aaca")); + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ(keys[2], iter->key().ToString()); + EXPECT_EQ(use_first_key ? 0 : 1, + stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + + iter->Next(); + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ(keys[3], iter->key().ToString()); + EXPECT_EQ(use_first_key ? 1 : 2, + stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + ASSERT_TRUE(iter->PrepareValue()); + EXPECT_EQ("v3", iter->value().ToString()); + EXPECT_EQ(2, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + EXPECT_EQ(3, stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + + // Seek between blocks and step back without accessing value. + iter->Seek(ikey("aaca")); + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ(keys[2], iter->key().ToString()); + EXPECT_EQ(use_first_key ? 2 : 3, + stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + EXPECT_EQ(3, stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + + iter->Prev(); + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ(keys[1], iter->key().ToString()); + EXPECT_EQ(use_first_key ? 2 : 3, + stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + // All blocks are in cache now, there'll be no more misses ever. + EXPECT_EQ(4, stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + ASSERT_TRUE(iter->PrepareValue()); + EXPECT_EQ("v1", iter->value().ToString()); + + // Next into the next block again. + iter->Next(); + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ(keys[2], iter->key().ToString()); + EXPECT_EQ(use_first_key ? 2 : 4, + stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + + // Seek to first and step back without accessing value. + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ(keys[0], iter->key().ToString()); + EXPECT_EQ(use_first_key ? 2 : 5, + stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + + iter->Prev(); + EXPECT_FALSE(iter->Valid()); + EXPECT_EQ(use_first_key ? 2 : 5, + stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + + // Do some SeekForPrev() and SeekToLast() just to cover all methods. + iter->SeekForPrev(ikey("caad")); + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ(keys[4], iter->key().ToString()); + EXPECT_EQ(use_first_key ? 3 : 6, + stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + ASSERT_TRUE(iter->PrepareValue()); + EXPECT_EQ("v4", iter->value().ToString()); + EXPECT_EQ(use_first_key ? 3 : 6, + stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + + iter->SeekToLast(); + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ(keys[7], iter->key().ToString()); + EXPECT_EQ(use_first_key ? 4 : 7, + stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + ASSERT_TRUE(iter->PrepareValue()); + EXPECT_EQ("v7", iter->value().ToString()); + EXPECT_EQ(use_first_key ? 4 : 7, + stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + + EXPECT_EQ(4, stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + + c.ResetTableReader(); + } +} + +TEST_P(BlockBasedTableTest, BinaryIndexWithFirstKeyGlobalSeqno) { + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.index_type = BlockBasedTableOptions::kBinarySearchWithFirstKey; + table_options.block_cache = NewLRUCache(10000); + Options options; + options.statistics = CreateDBStatistics(); + Statistics* stats = options.statistics.get(); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + std::unique_ptr comparator( + new InternalKeyComparator(BytewiseComparator())); + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + + TableConstructor c(BytewiseComparator(), /* convert_to_internal_key */ false, + /* level */ -1, /* largest_seqno */ 42); + + c.Add(InternalKey("b", 0, kTypeValue).Encode().ToString(), "x"); + c.Add(InternalKey("c", 0, kTypeValue).Encode().ToString(), "y"); + + std::vector keys; + stl_wrappers::KVMap kvmap; + c.Finish(options, ioptions, moptions, table_options, *comparator, &keys, + &kvmap); + ASSERT_EQ(2, keys.size()); + + auto reader = c.GetTableReader(); + auto props = reader->GetTableProperties(); + ASSERT_EQ(1u, props->num_data_blocks); + ReadOptions read_options; + std::unique_ptr iter(reader->NewIterator( + read_options, /*prefix_extractor=*/nullptr, /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized, + /*compaction_readahead_size=*/0, /*allow_unprepared_value=*/true)); + + iter->Seek(InternalKey("a", 0, kTypeValue).Encode().ToString()); + ASSERT_TRUE(iter->Valid()); + EXPECT_EQ(InternalKey("b", 42, kTypeValue).Encode().ToString(), + iter->key().ToString()); + EXPECT_NE(keys[0], iter->key().ToString()); + // Key should have been served from index, without reading data blocks. + EXPECT_EQ(0, stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + + ASSERT_TRUE(iter->PrepareValue()); + EXPECT_EQ("x", iter->value().ToString()); + EXPECT_EQ(1, stats->getTickerCount(BLOCK_CACHE_DATA_MISS)); + EXPECT_EQ(0, stats->getTickerCount(BLOCK_CACHE_DATA_HIT)); + EXPECT_EQ(InternalKey("b", 42, kTypeValue).Encode().ToString(), + iter->key().ToString()); + + c.ResetTableReader(); +} + +// It's very hard to figure out the index block size of a block accurately. +// To make sure we get the index size, we just make sure as key number +// grows, the filter block size also grows. +TEST_P(BlockBasedTableTest, IndexSizeStat) { + uint64_t last_index_size = 0; + + // we need to use random keys since the pure human readable texts + // may be well compressed, resulting insignifcant change of index + // block size. + Random rnd(test::RandomSeed()); + std::vector keys; + + for (int i = 0; i < 100; ++i) { + keys.push_back(rnd.RandomString(10000)); + } + + // Each time we load one more key to the table. the table index block + // size is expected to be larger than last time's. + for (size_t i = 1; i < keys.size(); ++i) { + TableConstructor c(BytewiseComparator(), + true /* convert_to_internal_key_ */); + for (size_t j = 0; j < i; ++j) { + c.Add(keys[j], "val"); + } + + std::vector ks; + stl_wrappers::KVMap kvmap; + Options options; + options.compression = kNoCompression; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.block_restart_interval = 1; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + c.Finish(options, ioptions, moptions, table_options, + GetPlainInternalComparator(options.comparator), &ks, &kvmap); + auto index_size = c.GetTableReader()->GetTableProperties()->index_size; + ASSERT_GT(index_size, last_index_size); + last_index_size = index_size; + c.ResetTableReader(); + } +} + +TEST_P(BlockBasedTableTest, NumBlockStat) { + Random rnd(test::RandomSeed()); + TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */); + Options options; + options.compression = kNoCompression; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.block_restart_interval = 1; + table_options.block_size = 1000; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + for (int i = 0; i < 10; ++i) { + // the key/val are slightly smaller than block size, so that each block + // holds roughly one key/value pair. + c.Add(rnd.RandomString(900), "val"); + } + + std::vector ks; + stl_wrappers::KVMap kvmap; + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + c.Finish(options, ioptions, moptions, table_options, + GetPlainInternalComparator(options.comparator), &ks, &kvmap); + ASSERT_EQ(kvmap.size(), + c.GetTableReader()->GetTableProperties()->num_data_blocks); + c.ResetTableReader(); +} + +TEST_P(BlockBasedTableTest, TracingGetTest) { + TableConstructor c(BytewiseComparator()); + Options options; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + options.create_if_missing = true; + table_options.block_cache = NewLRUCache(1024 * 1024, 0); + table_options.cache_index_and_filter_blocks = true; + table_options.filter_policy.reset(NewBloomFilterPolicy(10)); + options.table_factory.reset(new BlockBasedTableFactory(table_options)); + SetupTracingTest(&c); + std::vector keys; + stl_wrappers::KVMap kvmap; + ImmutableOptions ioptions(options); + MutableCFOptions moptions(options); + c.Finish(options, ioptions, moptions, table_options, + GetPlainInternalComparator(options.comparator), &keys, &kvmap); + InternalKey internal_key(auto_add_key1, 0, kTypeValue); + std::string encoded_key = internal_key.Encode().ToString(); + for (uint32_t i = 1; i <= 2; i++) { + PinnableSlice value; + GetContext get_context( + options.comparator, nullptr, nullptr, nullptr, GetContext::kNotFound, + auto_add_key1, &value, nullptr, nullptr, nullptr, true, nullptr, + nullptr, nullptr, nullptr, nullptr, nullptr, /*tracing_get_id=*/i); + get_perf_context()->Reset(); + ASSERT_OK(c.GetTableReader()->Get(ReadOptions(), encoded_key, &get_context, + moptions.prefix_extractor.get())); + ASSERT_EQ(get_context.State(), GetContext::kFound); + ASSERT_EQ(value.ToString(), kDummyValue); + } + + // Verify traces. + std::vector expected_records; + // The first two records should be prefetching index and filter blocks. + BlockCacheTraceRecord record; + record.block_type = TraceType::kBlockTraceIndexBlock; + record.caller = TableReaderCaller::kPrefetch; + record.is_cache_hit = false; + record.no_insert = false; + expected_records.push_back(record); + record.block_type = TraceType::kBlockTraceFilterBlock; + expected_records.push_back(record); + // Then we should have three records for one index, one filter, and one data + // block access. + record.get_id = 1; + record.block_type = TraceType::kBlockTraceFilterBlock; + record.caller = TableReaderCaller::kUserGet; + record.get_from_user_specified_snapshot = false; + record.referenced_key = encoded_key; + record.referenced_key_exist_in_block = true; + record.is_cache_hit = true; + expected_records.push_back(record); + record.block_type = TraceType::kBlockTraceIndexBlock; + expected_records.push_back(record); + record.is_cache_hit = false; + record.block_type = TraceType::kBlockTraceDataBlock; + expected_records.push_back(record); + // The second get should all observe cache hits. + record.is_cache_hit = true; + record.get_id = 2; + record.block_type = TraceType::kBlockTraceFilterBlock; + record.caller = TableReaderCaller::kUserGet; + record.get_from_user_specified_snapshot = false; + record.referenced_key = encoded_key; + expected_records.push_back(record); + record.block_type = TraceType::kBlockTraceIndexBlock; + expected_records.push_back(record); + record.block_type = TraceType::kBlockTraceDataBlock; + expected_records.push_back(record); + VerifyBlockAccessTrace(&c, expected_records); + c.ResetTableReader(); +} + +struct HitMissCountingCache : public CacheWrapper { + using CacheWrapper::CacheWrapper; + const char* Name() const override { return "HitMissCountingCache"; } + + uint64_t hit_count_ = 0; + uint64_t miss_count_ = 0; + + void Reset() { + hit_count_ = 0; + miss_count_ = 0; + } + + Handle* Lookup(const Slice& key, const CacheItemHelper* helper, + CreateContext* create_context, + Priority priority = Priority::LOW, + Statistics* stats = nullptr) override { + // ASSUMES no blocking async lookups + Handle* h = target_->Lookup(key, helper, create_context, priority, stats); + if (h) { + hit_count_++; + } else { + miss_count_++; + } + return h; + } + + void StartAsyncLookup(AsyncLookupHandle& async_handle) override { + target_->StartAsyncLookup(async_handle); + // If not pending, caller might not call WaitAll, so have to account here. + if (!async_handle.IsPending()) { + if (async_handle.Result()) { + hit_count_++; + } else { + miss_count_++; + } + } + } + + void WaitAll(AsyncLookupHandle* async_handles, size_t count) override { + // If !pending, then we already accounted for it in StartAsyncLookup. + // Assume the pending status does not change asynchronously (since + // StartAsyncLookup) and remember which still need accounting. + std::vector needs_accounting; + for (size_t i = 0; i < count; ++i) { + if (async_handles[i].IsPending()) { + needs_accounting.push_back(async_handles + i); + } + } + target_->WaitAll(async_handles, count); + for (auto ah : needs_accounting) { + if (ah->Result()) { + hit_count_++; + } else { + miss_count_++; + } + } + } + + void VerifyExpectedHitMissCounts( + const std::vector& expected_records) { + uint64_t expected_hits = 0; + uint64_t expected_misses = 0; + for (const auto& r : expected_records) { + if (r.is_cache_hit) { + expected_hits++; + } else { + expected_misses++; + } + } + EXPECT_EQ(expected_hits, hit_count_); + EXPECT_EQ(expected_misses, miss_count_); + Reset(); + } +}; + +TEST_P(BlockBasedTableTest, TracingMultiGetTest) { + TableConstructor c(BytewiseComparator()); + Options options; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + options.create_if_missing = true; + auto cache = + std::make_shared(NewLRUCache(1024 * 1024, 0)); + table_options.block_cache = cache; + table_options.cache_index_and_filter_blocks = true; + table_options.filter_policy.reset(NewBloomFilterPolicy(10)); + // Put auto_add_key1 and auto_add_key2 in the same data block + table_options.block_size = kDummyValue.size() * 2 + 100; + options.table_factory.reset(new BlockBasedTableFactory(table_options)); + SetupTracingTest(&c); + std::vector keys; + stl_wrappers::KVMap kvmap; + ImmutableOptions ioptions(options); + MutableCFOptions moptions(options); + c.Finish(options, ioptions, moptions, table_options, + GetPlainInternalComparator(options.comparator), &keys, &kvmap); + + std::vector expected_records; + + for (bool first_pass : {true, false}) { + uint64_t get_id_offset = first_pass ? 2 : 5; + ReadOptions ro; + std::array ukeys{{auto_add_key1, auto_add_key2}}; + std::array values; + std::vector get_contexts; + get_contexts.emplace_back( + options.comparator, nullptr, nullptr, nullptr, GetContext::kNotFound, + ukeys[0], &values[0], nullptr, nullptr, nullptr, true, nullptr, nullptr, + nullptr, nullptr, nullptr, nullptr, get_id_offset); + get_contexts.emplace_back( + options.comparator, nullptr, nullptr, nullptr, GetContext::kNotFound, + ukeys[1], &values[1], nullptr, nullptr, nullptr, true, nullptr, nullptr, + nullptr, nullptr, nullptr, nullptr, get_id_offset + 1); + std::array encoded_keys; + encoded_keys[0] = InternalKey(ukeys[0], 0, kTypeValue).Encode().ToString(); + encoded_keys[1] = InternalKey(ukeys[1], 0, kTypeValue).Encode().ToString(); + std::array statuses; + autovector key_context; + key_context.emplace_back(/*ColumnFamilyHandle omitted*/ nullptr, ukeys[0], + &values[0], + /*PinnableWideColumns omitted*/ nullptr, + /*timestamp omitted*/ nullptr, &statuses[0]); + key_context[0].ukey_without_ts = ukeys[0]; + key_context[0].ikey = encoded_keys[0]; + key_context[0].get_context = &get_contexts[0]; + key_context.emplace_back(/*ColumnFamilyHandle omitted*/ nullptr, ukeys[1], + &values[1], + /*PinnableWideColumns omitted*/ nullptr, + /*timestamp omitted*/ nullptr, &statuses[1]); + key_context[1].ukey_without_ts = ukeys[1]; + key_context[1].ikey = encoded_keys[1]; + key_context[1].get_context = &get_contexts[1]; + autovector sorted_keys; + sorted_keys.push_back(&key_context[0]); + sorted_keys.push_back(&key_context[1]); + MultiGetContext m_context( + &sorted_keys, 0, sorted_keys.size(), /*SequenceNumber*/ 42, ro, + options.env->GetFileSystem().get(), options.statistics.get()); + MultiGetRange range = m_context.GetMultiGetRange(); + + get_perf_context()->Reset(); + c.GetTableReader()->MultiGet(ro, &range, /*prefix_extractor*/ nullptr); + + // Verify read op result + for (uint32_t i = 0; i <= 1; i++) { + ASSERT_OK(statuses[i]); + ASSERT_EQ(get_contexts[i].State(), GetContext::kFound); + ASSERT_EQ(values[i].ToString(), kDummyValue); + } + + // Verify traces. + BlockCacheTraceRecord record; + if (first_pass) { + // The first two records should be prefetching index and filter blocks. + record.get_id = 0; + record.block_type = TraceType::kBlockTraceIndexBlock; + record.caller = TableReaderCaller::kPrefetch; + record.is_cache_hit = false; + record.no_insert = false; + expected_records.push_back(record); + record.block_type = TraceType::kBlockTraceFilterBlock; + expected_records.push_back(record); + } + // Then we should have three records for one index, one filter, and one data + // block access. (The two keys share a data block.) + record.get_id = get_id_offset; + record.block_type = TraceType::kBlockTraceFilterBlock; + record.caller = TableReaderCaller::kUserMultiGet; + record.get_from_user_specified_snapshot = false; + record.referenced_key = encoded_keys[0]; + record.referenced_key_exist_in_block = true; + record.is_cache_hit = true; + expected_records.push_back(record); + record.block_type = TraceType::kBlockTraceIndexBlock; + expected_records.push_back(record); + record.is_cache_hit = !first_pass; + record.block_type = TraceType::kBlockTraceDataBlock; + expected_records.push_back(record); + } + VerifyBlockAccessTrace(&c, expected_records); + cache->VerifyExpectedHitMissCounts(expected_records); + c.ResetTableReader(); +} + +TEST_P(BlockBasedTableTest, TracingApproximateOffsetOfTest) { + TableConstructor c(BytewiseComparator()); + Options options; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + options.create_if_missing = true; + table_options.block_cache = NewLRUCache(1024 * 1024, 0); + table_options.cache_index_and_filter_blocks = true; + table_options.filter_policy.reset(NewBloomFilterPolicy(10, true)); + options.table_factory.reset(new BlockBasedTableFactory(table_options)); + SetupTracingTest(&c); + std::vector keys; + stl_wrappers::KVMap kvmap; + ImmutableOptions ioptions(options); + MutableCFOptions moptions(options); + c.Finish(options, ioptions, moptions, table_options, + GetPlainInternalComparator(options.comparator), &keys, &kvmap); + const ReadOptions read_options; + for (uint32_t i = 1; i <= 2; i++) { + InternalKey internal_key(auto_add_key1, 0, kTypeValue); + std::string encoded_key = internal_key.Encode().ToString(); + c.GetTableReader()->ApproximateOffsetOf( + read_options, encoded_key, TableReaderCaller::kUserApproximateSize); + } + // Verify traces. + std::vector expected_records; + // The first two records should be prefetching index and filter blocks. + BlockCacheTraceRecord record; + record.block_type = TraceType::kBlockTraceIndexBlock; + record.caller = TableReaderCaller::kPrefetch; + record.is_cache_hit = false; + record.no_insert = false; + expected_records.push_back(record); + record.block_type = TraceType::kBlockTraceFilterBlock; + expected_records.push_back(record); + // Then we should have two records for only index blocks. + record.block_type = TraceType::kBlockTraceIndexBlock; + record.caller = TableReaderCaller::kUserApproximateSize; + record.is_cache_hit = true; + expected_records.push_back(record); + expected_records.push_back(record); + VerifyBlockAccessTrace(&c, expected_records); + c.ResetTableReader(); +} + +TEST_P(BlockBasedTableTest, TracingIterator) { + TableConstructor c(BytewiseComparator()); + Options options; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + options.create_if_missing = true; + table_options.block_cache = NewLRUCache(1024 * 1024, 0); + table_options.cache_index_and_filter_blocks = true; + table_options.filter_policy.reset(NewBloomFilterPolicy(10, true)); + options.table_factory.reset(new BlockBasedTableFactory(table_options)); + SetupTracingTest(&c); + std::vector keys; + stl_wrappers::KVMap kvmap; + ImmutableOptions ioptions(options); + MutableCFOptions moptions(options); + c.Finish(options, ioptions, moptions, table_options, + GetPlainInternalComparator(options.comparator), &keys, &kvmap); + + for (uint32_t i = 1; i <= 2; i++) { + ReadOptions read_options; + std::unique_ptr iter(c.GetTableReader()->NewIterator( + read_options, moptions.prefix_extractor.get(), /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUserIterator)); + iter->SeekToFirst(); + while (iter->Valid()) { + iter->key(); + iter->value(); + iter->Next(); + } + ASSERT_OK(iter->status()); + iter.reset(); + } + + // Verify traces. + std::vector expected_records; + // The first two records should be prefetching index and filter blocks. + BlockCacheTraceRecord record; + record.block_type = TraceType::kBlockTraceIndexBlock; + record.caller = TableReaderCaller::kPrefetch; + record.is_cache_hit = false; + record.no_insert = false; + expected_records.push_back(record); + record.block_type = TraceType::kBlockTraceFilterBlock; + expected_records.push_back(record); + // Then we should have three records for index and two data block access. + record.block_type = TraceType::kBlockTraceIndexBlock; + record.caller = TableReaderCaller::kUserIterator; + record.is_cache_hit = true; + expected_records.push_back(record); + record.block_type = TraceType::kBlockTraceDataBlock; + record.is_cache_hit = false; + expected_records.push_back(record); + expected_records.push_back(record); + // When we iterate this file for the second time, we should observe all cache + // hits. + record.block_type = TraceType::kBlockTraceIndexBlock; + record.is_cache_hit = true; + expected_records.push_back(record); + record.block_type = TraceType::kBlockTraceDataBlock; + expected_records.push_back(record); + expected_records.push_back(record); + VerifyBlockAccessTrace(&c, expected_records); + c.ResetTableReader(); +} + +// A simple tool that takes the snapshot of block cache statistics. +class BlockCachePropertiesSnapshot { + public: + explicit BlockCachePropertiesSnapshot(Statistics* statistics) { + block_cache_miss = statistics->getTickerCount(BLOCK_CACHE_MISS); + block_cache_hit = statistics->getTickerCount(BLOCK_CACHE_HIT); + index_block_cache_miss = statistics->getTickerCount(BLOCK_CACHE_INDEX_MISS); + index_block_cache_hit = statistics->getTickerCount(BLOCK_CACHE_INDEX_HIT); + data_block_cache_miss = statistics->getTickerCount(BLOCK_CACHE_DATA_MISS); + data_block_cache_hit = statistics->getTickerCount(BLOCK_CACHE_DATA_HIT); + filter_block_cache_miss = + statistics->getTickerCount(BLOCK_CACHE_FILTER_MISS); + filter_block_cache_hit = statistics->getTickerCount(BLOCK_CACHE_FILTER_HIT); + block_cache_bytes_read = statistics->getTickerCount(BLOCK_CACHE_BYTES_READ); + block_cache_bytes_write = + statistics->getTickerCount(BLOCK_CACHE_BYTES_WRITE); + } + + void AssertIndexBlockStat(int64_t expected_index_block_cache_miss, + int64_t expected_index_block_cache_hit) { + ASSERT_EQ(expected_index_block_cache_miss, index_block_cache_miss); + ASSERT_EQ(expected_index_block_cache_hit, index_block_cache_hit); + } + + void AssertFilterBlockStat(int64_t expected_filter_block_cache_miss, + int64_t expected_filter_block_cache_hit) { + ASSERT_EQ(expected_filter_block_cache_miss, filter_block_cache_miss); + ASSERT_EQ(expected_filter_block_cache_hit, filter_block_cache_hit); + } + + // Check if the fetched props matches the expected ones. + // TODO(kailiu) Use this only when you disabled filter policy! + void AssertEqual(int64_t expected_index_block_cache_miss, + int64_t expected_index_block_cache_hit, + int64_t expected_data_block_cache_miss, + int64_t expected_data_block_cache_hit) const { + ASSERT_EQ(expected_index_block_cache_miss, index_block_cache_miss); + ASSERT_EQ(expected_index_block_cache_hit, index_block_cache_hit); + ASSERT_EQ(expected_data_block_cache_miss, data_block_cache_miss); + ASSERT_EQ(expected_data_block_cache_hit, data_block_cache_hit); + ASSERT_EQ(expected_index_block_cache_miss + expected_data_block_cache_miss, + block_cache_miss); + ASSERT_EQ(expected_index_block_cache_hit + expected_data_block_cache_hit, + block_cache_hit); + } + + int64_t GetCacheBytesRead() { return block_cache_bytes_read; } + + int64_t GetCacheBytesWrite() { return block_cache_bytes_write; } + + private: + int64_t block_cache_miss = 0; + int64_t block_cache_hit = 0; + int64_t index_block_cache_miss = 0; + int64_t index_block_cache_hit = 0; + int64_t data_block_cache_miss = 0; + int64_t data_block_cache_hit = 0; + int64_t filter_block_cache_miss = 0; + int64_t filter_block_cache_hit = 0; + int64_t block_cache_bytes_read = 0; + int64_t block_cache_bytes_write = 0; +}; + +// Make sure, by default, index/filter blocks were pre-loaded (meaning we won't +// use block cache to store them). +TEST_P(BlockBasedTableTest, BlockCacheDisabledTest) { + Options options; + options.create_if_missing = true; + options.statistics = CreateDBStatistics(); + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.block_cache = NewLRUCache(1024, 4); + table_options.filter_policy.reset(NewBloomFilterPolicy(10)); + options.table_factory.reset(new BlockBasedTableFactory(table_options)); + std::vector keys; + stl_wrappers::KVMap kvmap; + + TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */); + c.Add("key", "value"); + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + c.Finish(options, ioptions, moptions, table_options, + GetPlainInternalComparator(options.comparator), &keys, &kvmap); + + // preloading filter/index blocks is enabled. + auto reader = dynamic_cast(c.GetTableReader()); + ASSERT_FALSE(reader->TEST_FilterBlockInCache()); + ASSERT_FALSE(reader->TEST_IndexBlockInCache()); + + { + // nothing happens in the beginning + BlockCachePropertiesSnapshot props(options.statistics.get()); + props.AssertIndexBlockStat(0, 0); + props.AssertFilterBlockStat(0, 0); + } + + { + GetContext get_context(options.comparator, nullptr, nullptr, nullptr, + GetContext::kNotFound, Slice(), nullptr, nullptr, + nullptr, nullptr, true, nullptr, nullptr); + // a hack that just to trigger BlockBasedTable::GetFilter. + ASSERT_OK(reader->Get(ReadOptions(), "non-exist-key", &get_context, + moptions.prefix_extractor.get())); + BlockCachePropertiesSnapshot props(options.statistics.get()); + props.AssertIndexBlockStat(0, 0); + props.AssertFilterBlockStat(0, 0); + } +} + +// Due to the difficulities of the intersaction between statistics, this test +// only tests the case when "index block is put to block cache" +TEST_P(BlockBasedTableTest, FilterBlockInBlockCache) { + // -- Table construction + Options options; + options.create_if_missing = true; + options.statistics = CreateDBStatistics(); + + // Enable the cache for index/filter blocks + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + LRUCacheOptions co; + co.capacity = 2048; + co.num_shard_bits = 2; + co.metadata_charge_policy = kDontChargeCacheMetadata; + table_options.block_cache = NewLRUCache(co); + table_options.cache_index_and_filter_blocks = true; + options.table_factory.reset(new BlockBasedTableFactory(table_options)); + std::vector keys; + stl_wrappers::KVMap kvmap; + + TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */); + c.Add("key", "value"); + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + c.Finish(options, ioptions, moptions, table_options, + GetPlainInternalComparator(options.comparator), &keys, &kvmap); + // preloading filter/index blocks is prohibited. + auto* reader = dynamic_cast(c.GetTableReader()); + ASSERT_FALSE(reader->TEST_FilterBlockInCache()); + ASSERT_TRUE(reader->TEST_IndexBlockInCache()); + + // -- PART 1: Open with regular block cache. + // Since block_cache is disabled, no cache activities will be involved. + std::unique_ptr iter; + + int64_t last_cache_bytes_read = 0; + // At first, no block will be accessed. + { + BlockCachePropertiesSnapshot props(options.statistics.get()); + // index will be added to block cache. + props.AssertEqual(1, // index block miss + 0, 0, 0); + ASSERT_EQ(props.GetCacheBytesRead(), 0); + ASSERT_EQ(props.GetCacheBytesWrite(), + static_cast(table_options.block_cache->GetUsage())); + last_cache_bytes_read = props.GetCacheBytesRead(); + } + + // Only index block will be accessed + { + iter.reset(c.NewIterator(moptions.prefix_extractor.get())); + BlockCachePropertiesSnapshot props(options.statistics.get()); + // NOTE: to help better highlight the "detla" of each ticker, I use + // + to indicate the increment of changed + // value; other numbers remain the same. + props.AssertEqual(1, 0 + 1, // index block hit + 0, 0); + // Cache hit, bytes read from cache should increase + ASSERT_GT(props.GetCacheBytesRead(), last_cache_bytes_read); + ASSERT_EQ(props.GetCacheBytesWrite(), + static_cast(table_options.block_cache->GetUsage())); + last_cache_bytes_read = props.GetCacheBytesRead(); + } + + // Only data block will be accessed + { + iter->SeekToFirst(); + ASSERT_OK(iter->status()); + BlockCachePropertiesSnapshot props(options.statistics.get()); + props.AssertEqual(1, 1, 0 + 1, // data block miss + 0); + // Cache miss, Bytes read from cache should not change + ASSERT_EQ(props.GetCacheBytesRead(), last_cache_bytes_read); + ASSERT_EQ(props.GetCacheBytesWrite(), + static_cast(table_options.block_cache->GetUsage())); + last_cache_bytes_read = props.GetCacheBytesRead(); + } + + // Data block will be in cache + { + iter.reset(c.NewIterator(moptions.prefix_extractor.get())); + iter->SeekToFirst(); + ASSERT_OK(iter->status()); + BlockCachePropertiesSnapshot props(options.statistics.get()); + props.AssertEqual(1, 1 + 1, /* index block hit */ + 1, 0 + 1 /* data block hit */); + // Cache hit, bytes read from cache should increase + ASSERT_GT(props.GetCacheBytesRead(), last_cache_bytes_read); + ASSERT_EQ(props.GetCacheBytesWrite(), + static_cast(table_options.block_cache->GetUsage())); + } + // release the iterator so that the block cache can reset correctly. + iter.reset(); + + c.ResetTableReader(); + + // -- PART 2: Open with very small block cache + // In this test, no block will ever get hit since the block cache is + // too small to fit even one entry. + table_options.block_cache = NewLRUCache(1, 4); + options.statistics = CreateDBStatistics(); + options.table_factory.reset(new BlockBasedTableFactory(table_options)); + const ImmutableOptions ioptions2(options); + const MutableCFOptions moptions2(options); + ASSERT_OK(c.Reopen(ioptions2, moptions2)); + { + BlockCachePropertiesSnapshot props(options.statistics.get()); + props.AssertEqual(1, // index block miss + 0, 0, 0); + // Cache miss, Bytes read from cache should not change + ASSERT_EQ(props.GetCacheBytesRead(), 0); + } + + { + // Both index and data block get accessed. + // It first cache index block then data block. But since the cache size + // is only 1, index block will be purged after data block is inserted. + iter.reset(c.NewIterator(moptions2.prefix_extractor.get())); + BlockCachePropertiesSnapshot props(options.statistics.get()); + props.AssertEqual(1 + 1, // index block miss + 0, 0, // data block miss + 0); + // Cache hit, bytes read from cache should increase + ASSERT_EQ(props.GetCacheBytesRead(), 0); + } + + { + // SeekToFirst() accesses data block. With similar reason, we expect data + // block's cache miss. + iter->SeekToFirst(); + ASSERT_OK(iter->status()); + BlockCachePropertiesSnapshot props(options.statistics.get()); + props.AssertEqual(2, 0, 0 + 1, // data block miss + 0); + // Cache miss, Bytes read from cache should not change + ASSERT_EQ(props.GetCacheBytesRead(), 0); + } + iter.reset(); + c.ResetTableReader(); + + // -- PART 3: Open table with bloom filter enabled but not in SST file + table_options.block_cache = NewLRUCache(4096, 4); + table_options.cache_index_and_filter_blocks = false; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + TableConstructor c3(BytewiseComparator()); + std::string user_key = "k01"; + InternalKey internal_key(user_key, 0, kTypeValue); + c3.Add(internal_key.Encode().ToString(), "hello"); + ImmutableOptions ioptions3(options); + MutableCFOptions moptions3(options); + // Generate table without filter policy + c3.Finish(options, ioptions3, moptions3, table_options, + GetPlainInternalComparator(options.comparator), &keys, &kvmap); + c3.ResetTableReader(); + + // Open table with filter policy + table_options.filter_policy.reset(NewBloomFilterPolicy(1)); + options.table_factory.reset(new BlockBasedTableFactory(table_options)); + options.statistics = CreateDBStatistics(); + ImmutableOptions ioptions4(options); + MutableCFOptions moptions4(options); + ASSERT_OK(c3.Reopen(ioptions4, moptions4)); + reader = dynamic_cast(c3.GetTableReader()); + ASSERT_FALSE(reader->TEST_FilterBlockInCache()); + PinnableSlice value; + GetContext get_context(options.comparator, nullptr, nullptr, nullptr, + GetContext::kNotFound, user_key, &value, nullptr, + nullptr, nullptr, true, nullptr, nullptr); + ASSERT_OK(reader->Get(ReadOptions(), internal_key.Encode(), &get_context, + moptions4.prefix_extractor.get())); + ASSERT_STREQ(value.data(), "hello"); + BlockCachePropertiesSnapshot props(options.statistics.get()); + props.AssertFilterBlockStat(0, 0); + c3.ResetTableReader(); +} + +void ValidateBlockSizeDeviation(int value, int expected) { + BlockBasedTableOptions table_options; + table_options.block_size_deviation = value; + BlockBasedTableFactory* factory = new BlockBasedTableFactory(table_options); + + const BlockBasedTableOptions* normalized_table_options = + factory->GetOptions(); + ASSERT_EQ(normalized_table_options->block_size_deviation, expected); + + delete factory; +} + +void ValidateBlockRestartInterval(int value, int expected) { + BlockBasedTableOptions table_options; + table_options.block_restart_interval = value; + BlockBasedTableFactory* factory = new BlockBasedTableFactory(table_options); + + const BlockBasedTableOptions* normalized_table_options = + factory->GetOptions(); + ASSERT_EQ(normalized_table_options->block_restart_interval, expected); + + delete factory; +} + +TEST_P(BlockBasedTableTest, InvalidOptions) { + // invalid values for block_size_deviation (<0 or >100) are silently set to 0 + ValidateBlockSizeDeviation(-10, 0); + ValidateBlockSizeDeviation(-1, 0); + ValidateBlockSizeDeviation(0, 0); + ValidateBlockSizeDeviation(1, 1); + ValidateBlockSizeDeviation(99, 99); + ValidateBlockSizeDeviation(100, 100); + ValidateBlockSizeDeviation(101, 0); + ValidateBlockSizeDeviation(1000, 0); + + // invalid values for block_restart_interval (<1) are silently set to 1 + ValidateBlockRestartInterval(-10, 1); + ValidateBlockRestartInterval(-1, 1); + ValidateBlockRestartInterval(0, 1); + ValidateBlockRestartInterval(1, 1); + ValidateBlockRestartInterval(2, 2); + ValidateBlockRestartInterval(1000, 1000); +} + +TEST_P(BlockBasedTableTest, BlockReadCountTest) { + // bloom_filter_type = 1 -- full filter using use_block_based_builder=false + // bloom_filter_type = 2 -- full filter using use_block_based_builder=true + // because of API change to hide block-based filter + for (int bloom_filter_type = 1; bloom_filter_type <= 2; ++bloom_filter_type) { + for (int index_and_filter_in_cache = 0; index_and_filter_in_cache < 2; + ++index_and_filter_in_cache) { + Options options; + options.create_if_missing = true; + + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.block_cache = NewLRUCache(1, 0); + table_options.cache_index_and_filter_blocks = index_and_filter_in_cache; + table_options.filter_policy.reset( + NewBloomFilterPolicy(10, bloom_filter_type == 2)); + options.table_factory.reset(new BlockBasedTableFactory(table_options)); + std::vector keys; + stl_wrappers::KVMap kvmap; + + TableConstructor c(BytewiseComparator()); + std::string user_key = "k04"; + InternalKey internal_key(user_key, 0, kTypeValue); + std::string encoded_key = internal_key.Encode().ToString(); + c.Add(encoded_key, "hello"); + ImmutableOptions ioptions(options); + MutableCFOptions moptions(options); + // Generate table with filter policy + c.Finish(options, ioptions, moptions, table_options, + GetPlainInternalComparator(options.comparator), &keys, &kvmap); + auto reader = c.GetTableReader(); + PinnableSlice value; + { + GetContext get_context(options.comparator, nullptr, nullptr, nullptr, + GetContext::kNotFound, user_key, &value, nullptr, + nullptr, nullptr, true, nullptr, nullptr); + get_perf_context()->Reset(); + ASSERT_OK(reader->Get(ReadOptions(), encoded_key, &get_context, + moptions.prefix_extractor.get())); + if (index_and_filter_in_cache) { + // data, index and filter block + ASSERT_EQ(get_perf_context()->block_read_count, 3); + ASSERT_EQ(get_perf_context()->index_block_read_count, 1); + ASSERT_EQ(get_perf_context()->filter_block_read_count, 1); + } else { + // just the data block + ASSERT_EQ(get_perf_context()->block_read_count, 1); + } + ASSERT_EQ(get_context.State(), GetContext::kFound); + ASSERT_STREQ(value.data(), "hello"); + } + + // Get non-existing key + user_key = "does-not-exist"; + internal_key = InternalKey(user_key, 0, kTypeValue); + encoded_key = internal_key.Encode().ToString(); + + value.Reset(); + { + GetContext get_context(options.comparator, nullptr, nullptr, nullptr, + GetContext::kNotFound, user_key, &value, nullptr, + nullptr, nullptr, true, nullptr, nullptr); + get_perf_context()->Reset(); + ASSERT_OK(reader->Get(ReadOptions(), encoded_key, &get_context, + moptions.prefix_extractor.get())); + ASSERT_EQ(get_context.State(), GetContext::kNotFound); + } + + if (index_and_filter_in_cache) { + if (bloom_filter_type == 0) { + // with block-based, we read index and then the filter + ASSERT_EQ(get_perf_context()->block_read_count, 2); + ASSERT_EQ(get_perf_context()->index_block_read_count, 1); + ASSERT_EQ(get_perf_context()->filter_block_read_count, 1); + } else { + // with full-filter, we read filter first and then we stop + ASSERT_EQ(get_perf_context()->block_read_count, 1); + ASSERT_EQ(get_perf_context()->filter_block_read_count, 1); + } + } else { + // filter is already in memory and it figures out that the key doesn't + // exist + ASSERT_EQ(get_perf_context()->block_read_count, 0); + } + } + } +} + +TEST_P(BlockBasedTableTest, BlockCacheLeak) { + // Check that when we reopen a table we don't lose access to blocks already + // in the cache. This test checks whether the Table actually makes use of the + // unique ID from the file. + + Options opt; + std::unique_ptr ikc; + ikc.reset(new test::PlainInternalKeyComparator(opt.comparator)); + opt.compression = kNoCompression; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.block_size = 1024; + // big enough so we don't ever lose cached values. + table_options.block_cache = NewLRUCache(16 * 1024 * 1024, 4); + opt.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */); + c.Add("k01", "hello"); + c.Add("k02", "hello2"); + c.Add("k03", std::string(10000, 'x')); + c.Add("k04", std::string(200000, 'x')); + c.Add("k05", std::string(300000, 'x')); + c.Add("k06", "hello3"); + c.Add("k07", std::string(100000, 'x')); + std::vector keys; + stl_wrappers::KVMap kvmap; + const ImmutableOptions ioptions(opt); + const MutableCFOptions moptions(opt); + c.Finish(opt, ioptions, moptions, table_options, *ikc, &keys, &kvmap); + + std::unique_ptr iter( + c.NewIterator(moptions.prefix_extractor.get())); + iter->SeekToFirst(); + while (iter->Valid()) { + iter->key(); + iter->value(); + iter->Next(); + } + ASSERT_OK(iter->status()); + iter.reset(); + + const ImmutableOptions ioptions1(opt); + const MutableCFOptions moptions1(opt); + ASSERT_OK(c.Reopen(ioptions1, moptions1)); + auto table_reader = dynamic_cast(c.GetTableReader()); + for (const std::string& key : keys) { + InternalKey ikey(key, kMaxSequenceNumber, kTypeValue); + ASSERT_TRUE(table_reader->TEST_KeyInCache(ReadOptions(), ikey.Encode())); + } + c.ResetTableReader(); + + // rerun with different block cache + table_options.block_cache = NewLRUCache(16 * 1024 * 1024, 4); + opt.table_factory.reset(NewBlockBasedTableFactory(table_options)); + const ImmutableOptions ioptions2(opt); + const MutableCFOptions moptions2(opt); + ASSERT_OK(c.Reopen(ioptions2, moptions2)); + table_reader = dynamic_cast(c.GetTableReader()); + for (const std::string& key : keys) { + InternalKey ikey(key, kMaxSequenceNumber, kTypeValue); + ASSERT_TRUE(!table_reader->TEST_KeyInCache(ReadOptions(), ikey.Encode())); + } + c.ResetTableReader(); +} + +TEST_P(BlockBasedTableTest, MemoryAllocator) { + auto default_memory_allocator = std::make_shared(); + auto custom_memory_allocator = + std::make_shared(default_memory_allocator); + { + Options opt; + std::unique_ptr ikc; + ikc.reset(new test::PlainInternalKeyComparator(opt.comparator)); + opt.compression = kNoCompression; + BlockBasedTableOptions table_options; + table_options.block_size = 1024; + LRUCacheOptions lruOptions; + lruOptions.memory_allocator = custom_memory_allocator; + lruOptions.capacity = 16 * 1024 * 1024; + lruOptions.num_shard_bits = 4; + table_options.block_cache = NewLRUCache(std::move(lruOptions)); + opt.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + TableConstructor c(BytewiseComparator(), + true /* convert_to_internal_key_ */); + c.Add("k01", "hello"); + c.Add("k02", "hello2"); + c.Add("k03", std::string(10000, 'x')); + c.Add("k04", std::string(200000, 'x')); + c.Add("k05", std::string(300000, 'x')); + c.Add("k06", "hello3"); + c.Add("k07", std::string(100000, 'x')); + std::vector keys; + stl_wrappers::KVMap kvmap; + const ImmutableOptions ioptions(opt); + const MutableCFOptions moptions(opt); + c.Finish(opt, ioptions, moptions, table_options, *ikc, &keys, &kvmap); + + std::unique_ptr iter( + c.NewIterator(moptions.prefix_extractor.get())); + iter->SeekToFirst(); + while (iter->Valid()) { + iter->key(); + iter->value(); + iter->Next(); + } + ASSERT_OK(iter->status()); + } + + // out of scope, block cache should have been deleted, all allocations + // deallocated + EXPECT_EQ(custom_memory_allocator->GetNumAllocations(), + custom_memory_allocator->GetNumDeallocations()); + // make sure that allocations actually happened through the cache allocator + EXPECT_GT(custom_memory_allocator->GetNumAllocations(), 0); +} + +// Test the file checksum of block based table +TEST_P(BlockBasedTableTest, NoFileChecksum) { + Options options; + ImmutableOptions ioptions(options); + MutableCFOptions moptions(options); + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + std::unique_ptr comparator( + new InternalKeyComparator(BytewiseComparator())); + int level = 0; + IntTblPropCollectorFactories int_tbl_prop_collector_factories; + std::string column_family_name; + + FileChecksumTestHelper f(true); + f.CreateWritableFile(); + std::unique_ptr builder; + builder.reset(ioptions.table_factory->NewTableBuilder( + TableBuilderOptions(ioptions, moptions, *comparator, + &int_tbl_prop_collector_factories, + options.compression, options.compression_opts, + kUnknownColumnFamily, column_family_name, level), + f.GetFileWriter())); + ASSERT_OK(f.ResetTableBuilder(std::move(builder))); + f.AddKVtoKVMap(1000); + ASSERT_OK(f.WriteKVAndFlushTable()); + ASSERT_STREQ(f.GetFileChecksumFuncName(), kUnknownFileChecksumFuncName); + ASSERT_STREQ(f.GetFileChecksum().c_str(), kUnknownFileChecksum); +} + +TEST_P(BlockBasedTableTest, Crc32cFileChecksum) { + FileChecksumGenCrc32cFactory* file_checksum_gen_factory = + new FileChecksumGenCrc32cFactory(); + Options options; + options.file_checksum_gen_factory.reset(file_checksum_gen_factory); + ImmutableOptions ioptions(options); + MutableCFOptions moptions(options); + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + std::unique_ptr comparator( + new InternalKeyComparator(BytewiseComparator())); + int level = 0; + IntTblPropCollectorFactories int_tbl_prop_collector_factories; + std::string column_family_name; + + FileChecksumGenContext gen_context; + gen_context.file_name = "db/tmp"; + std::unique_ptr checksum_crc32c_gen1 = + options.file_checksum_gen_factory->CreateFileChecksumGenerator( + gen_context); + FileChecksumTestHelper f(true); + f.CreateWritableFile(); + f.SetFileChecksumGenerator(checksum_crc32c_gen1.release()); + std::unique_ptr builder; + builder.reset(ioptions.table_factory->NewTableBuilder( + TableBuilderOptions(ioptions, moptions, *comparator, + &int_tbl_prop_collector_factories, + options.compression, options.compression_opts, + kUnknownColumnFamily, column_family_name, level), + f.GetFileWriter())); + ASSERT_OK(f.ResetTableBuilder(std::move(builder))); + f.AddKVtoKVMap(1000); + ASSERT_OK(f.WriteKVAndFlushTable()); + ASSERT_STREQ(f.GetFileChecksumFuncName(), "FileChecksumCrc32c"); + + std::unique_ptr checksum_crc32c_gen2 = + options.file_checksum_gen_factory->CreateFileChecksumGenerator( + gen_context); + std::string checksum; + ASSERT_OK(f.CalculateFileChecksum(checksum_crc32c_gen2.get(), &checksum)); + ASSERT_STREQ(f.GetFileChecksum().c_str(), checksum.c_str()); + + // Unit test the generator itself for schema stability + std::unique_ptr checksum_crc32c_gen3 = + options.file_checksum_gen_factory->CreateFileChecksumGenerator( + gen_context); + const char data[] = "here is some data"; + checksum_crc32c_gen3->Update(data, sizeof(data)); + checksum_crc32c_gen3->Finalize(); + checksum = checksum_crc32c_gen3->GetChecksum(); + ASSERT_STREQ(checksum.c_str(), "\345\245\277\110"); +} + +TEST_F(PlainTableTest, BasicPlainTableProperties) { + PlainTableOptions plain_table_options; + plain_table_options.user_key_len = 8; + plain_table_options.bloom_bits_per_key = 8; + plain_table_options.hash_table_ratio = 0; + + PlainTableFactory factory(plain_table_options); + std::unique_ptr sink(new test::StringSink()); + std::unique_ptr file_writer(new WritableFileWriter( + std::move(sink), "" /* don't care */, FileOptions())); + Options options; + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + InternalKeyComparator ikc(options.comparator); + IntTblPropCollectorFactories int_tbl_prop_collector_factories; + std::string column_family_name; + int unknown_level = -1; + std::unique_ptr builder(factory.NewTableBuilder( + TableBuilderOptions(ioptions, moptions, ikc, + &int_tbl_prop_collector_factories, kNoCompression, + CompressionOptions(), kUnknownColumnFamily, + column_family_name, unknown_level), + file_writer.get())); + + for (char c = 'a'; c <= 'z'; ++c) { + std::string key(8, c); + key.append("\1 "); // PlainTable expects internal key structure + std::string value(28, c + 42); + builder->Add(key, value); + } + ASSERT_OK(builder->Finish()); + ASSERT_OK(file_writer->Flush()); + + test::StringSink* ss = + static_cast(file_writer->writable_file()); + std::unique_ptr source( + new test::StringSource(ss->contents(), 72242, true)); + std::unique_ptr file_reader( + new RandomAccessFileReader(std::move(source), "test")); + + std::unique_ptr props; + const ReadOptions read_options; + auto s = ReadTableProperties(file_reader.get(), ss->contents().size(), + kPlainTableMagicNumber, ioptions, read_options, + &props); + ASSERT_OK(s); + + ASSERT_EQ(0ul, props->index_size); + ASSERT_EQ(0ul, props->filter_size); + ASSERT_EQ(16ul * 26, props->raw_key_size); + ASSERT_EQ(28ul * 26, props->raw_value_size); + ASSERT_EQ(26ul, props->num_entries); + ASSERT_EQ(1ul, props->num_data_blocks); +} + +TEST_F(PlainTableTest, NoFileChecksum) { + PlainTableOptions plain_table_options; + plain_table_options.user_key_len = 20; + plain_table_options.bloom_bits_per_key = 8; + plain_table_options.hash_table_ratio = 0; + PlainTableFactory factory(plain_table_options); + + Options options; + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + InternalKeyComparator ikc(options.comparator); + IntTblPropCollectorFactories int_tbl_prop_collector_factories; + std::string column_family_name; + int unknown_level = -1; + FileChecksumTestHelper f(true); + f.CreateWritableFile(); + + std::unique_ptr builder(factory.NewTableBuilder( + TableBuilderOptions(ioptions, moptions, ikc, + &int_tbl_prop_collector_factories, kNoCompression, + CompressionOptions(), kUnknownColumnFamily, + column_family_name, unknown_level), + f.GetFileWriter())); + ASSERT_OK(f.ResetTableBuilder(std::move(builder))); + f.AddKVtoKVMap(1000); + ASSERT_OK(f.WriteKVAndFlushTable()); + ASSERT_STREQ(f.GetFileChecksumFuncName(), kUnknownFileChecksumFuncName); + EXPECT_EQ(f.GetFileChecksum(), kUnknownFileChecksum); +} + +TEST_F(PlainTableTest, Crc32cFileChecksum) { + PlainTableOptions plain_table_options; + plain_table_options.user_key_len = 20; + plain_table_options.bloom_bits_per_key = 8; + plain_table_options.hash_table_ratio = 0; + PlainTableFactory factory(plain_table_options); + + FileChecksumGenCrc32cFactory* file_checksum_gen_factory = + new FileChecksumGenCrc32cFactory(); + Options options; + options.file_checksum_gen_factory.reset(file_checksum_gen_factory); + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + InternalKeyComparator ikc(options.comparator); + IntTblPropCollectorFactories int_tbl_prop_collector_factories; + std::string column_family_name; + int unknown_level = -1; + + FileChecksumGenContext gen_context; + gen_context.file_name = "db/tmp"; + std::unique_ptr checksum_crc32c_gen1 = + options.file_checksum_gen_factory->CreateFileChecksumGenerator( + gen_context); + FileChecksumTestHelper f(true); + f.CreateWritableFile(); + f.SetFileChecksumGenerator(checksum_crc32c_gen1.release()); + + std::unique_ptr builder(factory.NewTableBuilder( + TableBuilderOptions(ioptions, moptions, ikc, + &int_tbl_prop_collector_factories, kNoCompression, + CompressionOptions(), kUnknownColumnFamily, + column_family_name, unknown_level), + f.GetFileWriter())); + ASSERT_OK(f.ResetTableBuilder(std::move(builder))); + f.AddKVtoKVMap(1000); + ASSERT_OK(f.WriteKVAndFlushTable()); + ASSERT_STREQ(f.GetFileChecksumFuncName(), "FileChecksumCrc32c"); + + std::unique_ptr checksum_crc32c_gen2 = + options.file_checksum_gen_factory->CreateFileChecksumGenerator( + gen_context); + std::string checksum; + ASSERT_OK(f.CalculateFileChecksum(checksum_crc32c_gen2.get(), &checksum)); + EXPECT_STREQ(f.GetFileChecksum().c_str(), checksum.c_str()); +} + + +TEST_F(GeneralTableTest, ApproximateOffsetOfPlain) { + TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */); + c.Add("k01", "hello"); + c.Add("k02", "hello2"); + c.Add("k03", std::string(10000, 'x')); + c.Add("k04", std::string(200000, 'x')); + c.Add("k05", std::string(300000, 'x')); + c.Add("k06", "hello3"); + c.Add("k07", std::string(100000, 'x')); + std::vector keys; + stl_wrappers::KVMap kvmap; + Options options; + options.db_host_id = ""; + test::PlainInternalKeyComparator internal_comparator(options.comparator); + options.compression = kNoCompression; + BlockBasedTableOptions table_options; + table_options.block_size = 1024; + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + c.Finish(options, ioptions, moptions, table_options, internal_comparator, + &keys, &kvmap); + + ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, 0)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"), 0, 0)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01a"), 0, 0)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k02"), 0, 0)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k03"), 0, 0)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04"), 10000, 11000)); + // k04 and k05 will be in two consecutive blocks, the index is + // an arbitrary slice between k04 and k05, either before or after k04a + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04a"), 10000, 211000)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k05"), 210000, 211000)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k06"), 510000, 511000)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k07"), 510000, 511000)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 610000, 612000)); + c.ResetTableReader(); +} + +static void DoCompressionTest(CompressionType comp) { + SCOPED_TRACE("CompressionType = " + CompressionTypeToString(comp)); + Random rnd(301); + TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */); + std::string tmp; + c.Add("k01", "hello"); + c.Add("k02", test::CompressibleString(&rnd, 0.25, 10000, &tmp)); + c.Add("k03", "hello3"); + c.Add("k04", test::CompressibleString(&rnd, 0.25, 10000, &tmp)); + std::vector keys; + stl_wrappers::KVMap kvmap; + Options options; + test::PlainInternalKeyComparator ikc(options.comparator); + options.compression = comp; + BlockBasedTableOptions table_options; + table_options.block_size = 1024; + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + c.Finish(options, ioptions, moptions, table_options, ikc, &keys, &kvmap); + + ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, 0)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"), 0, 0)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k02"), 0, 0)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k03"), 2000, 3550)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04"), 2000, 3550)); + ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 4000, 7075)); + c.ResetTableReader(); +} + +TEST_F(GeneralTableTest, ApproximateOffsetOfCompressed) { + std::vector compression_state; + if (!Snappy_Supported()) { + fprintf(stderr, "skipping snappy compression tests\n"); + } else { + compression_state.push_back(kSnappyCompression); + } + + if (!Zlib_Supported()) { + fprintf(stderr, "skipping zlib compression tests\n"); + } else { + compression_state.push_back(kZlibCompression); + } + + // TODO(kailiu) DoCompressionTest() doesn't work with BZip2. + /* + if (!BZip2_Supported()) { + fprintf(stderr, "skipping bzip2 compression tests\n"); + } else { + compression_state.push_back(kBZip2Compression); + } + */ + + if (!LZ4_Supported()) { + fprintf(stderr, "skipping lz4 and lz4hc compression tests\n"); + } else { + compression_state.push_back(kLZ4Compression); + compression_state.push_back(kLZ4HCCompression); + } + + if (!XPRESS_Supported()) { + fprintf(stderr, "skipping xpress and xpress compression tests\n"); + } else { + compression_state.push_back(kXpressCompression); + } + + for (auto state : compression_state) { + DoCompressionTest(state); + } +} + +TEST_F(GeneralTableTest, ApproximateKeyAnchors) { + Random rnd(301); + TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */); + std::string tmp; + for (int i = 1000; i < 9000; i++) { + c.Add(std::to_string(i), rnd.RandomString(2000)); + } + std::vector keys; + stl_wrappers::KVMap kvmap; + Options options; + InternalKeyComparator ikc(options.comparator); + options.compression = kNoCompression; + BlockBasedTableOptions table_options; + table_options.block_size = 4096; + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + c.Finish(options, ioptions, moptions, table_options, ikc, &keys, &kvmap); + + std::vector anchors; + ASSERT_OK(c.GetTableReader()->ApproximateKeyAnchors(ReadOptions(), anchors)); + // The target is 128 anchors. But in reality it can be slightly more or fewer. + ASSERT_GT(anchors.size(), 120); + ASSERT_LT(anchors.size(), 140); + + // We have around 8000 keys. With 128 anchors, in average 62.5 keys per + // anchor. Here we take a rough range and estimate the distance between + // anchors is between 50 and 100. + // Total data size is about 18,000,000, so each anchor range is about + // 140,625. We also take a rough range. + int prev_num = 1000; + // Non-last anchor + for (size_t i = 0; i + 1 < anchors.size(); i++) { + auto& anchor = anchors[i]; + ASSERT_GT(anchor.range_size, 100000); + ASSERT_LT(anchor.range_size, 200000); + + // Key might be shortened, so fill 0 in the end if it is the case. + std::string key_cpy = anchor.user_key; + key_cpy.append(4 - key_cpy.size(), '0'); + int num = std::stoi(key_cpy); + ASSERT_GT(num - prev_num, 50); + ASSERT_LT(num - prev_num, 100); + prev_num = num; + } + + ASSERT_EQ("8999", anchors.back().user_key); + ASSERT_LT(anchors.back().range_size, 200000); + + c.ResetTableReader(); +} + +#if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) +TEST_P(ParameterizedHarnessTest, RandomizedHarnessTest) { + Random rnd(test::RandomSeed() + 5); + for (int num_entries = 0; num_entries < 2000; + num_entries += (num_entries < 50 ? 1 : 200)) { + for (int e = 0; e < num_entries; e++) { + Add(test::RandomKey(&rnd, rnd.Skewed(4)), + rnd.RandomString(rnd.Skewed(5))); + } + Test(&rnd); + } +} + +TEST_F(DBHarnessTest, RandomizedLongDB) { + Random rnd(test::RandomSeed()); + int num_entries = 100000; + for (int e = 0; e < num_entries; e++) { + std::string v; + Add(test::RandomKey(&rnd, rnd.Skewed(4)), rnd.RandomString(rnd.Skewed(5))); + } + Test(&rnd); + + // We must have created enough data to force merging + int files = 0; + for (int level = 0; level < db()->NumberLevels(); level++) { + std::string value; + char name[100]; + snprintf(name, sizeof(name), "rocksdb.num-files-at-level%d", level); + ASSERT_TRUE(db()->GetProperty(name, &value)); + files += atoi(value.c_str()); + } + ASSERT_GT(files, 0); +} +#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) + +class MemTableTest : public testing::Test { + public: + MemTableTest() { + InternalKeyComparator cmp(BytewiseComparator()); + auto table_factory = std::make_shared(); + options_.memtable_factory = table_factory; + ImmutableOptions ioptions(options_); + wb_ = new WriteBufferManager(options_.db_write_buffer_size); + memtable_ = new MemTable(cmp, ioptions, MutableCFOptions(options_), wb_, + kMaxSequenceNumber, 0 /* column_family_id */); + memtable_->Ref(); + } + + ~MemTableTest() { + delete memtable_->Unref(); + delete wb_; + } + + MemTable* GetMemTable() { return memtable_; } + + private: + MemTable* memtable_; + Options options_; + WriteBufferManager* wb_; +}; + +TEST_F(MemTableTest, Simple) { + WriteBatch batch; + WriteBatchInternal::SetSequence(&batch, 100); + ASSERT_OK(batch.Put(std::string("k1"), std::string("v1"))); + ASSERT_OK(batch.Put(std::string("k2"), std::string("v2"))); + ASSERT_OK(batch.Put(std::string("k3"), std::string("v3"))); + ASSERT_OK(batch.Put(std::string("largekey"), std::string("vlarge"))); + ASSERT_OK(batch.DeleteRange(std::string("chi"), std::string("xigua"))); + ASSERT_OK(batch.DeleteRange(std::string("begin"), std::string("end"))); + ColumnFamilyMemTablesDefault cf_mems_default(GetMemTable()); + ASSERT_TRUE( + WriteBatchInternal::InsertInto(&batch, &cf_mems_default, nullptr, nullptr) + .ok()); + + for (int i = 0; i < 2; ++i) { + Arena arena; + ScopedArenaIterator arena_iter_guard; + std::unique_ptr iter_guard; + InternalIterator* iter; + if (i == 0) { + iter = GetMemTable()->NewIterator(ReadOptions(), &arena); + arena_iter_guard.set(iter); + } else { + iter = GetMemTable()->NewRangeTombstoneIterator( + ReadOptions(), kMaxSequenceNumber /* read_seq */, + false /* immutable_memtable */); + iter_guard.reset(iter); + } + if (iter == nullptr) { + continue; + } + iter->SeekToFirst(); + while (iter->Valid()) { + fprintf(stderr, "key: '%s' -> '%s'\n", iter->key().ToString().c_str(), + iter->value().ToString().c_str()); + iter->Next(); + } + } +} + +// Test the empty key +TEST_P(ParameterizedHarnessTest, SimpleEmptyKey) { + Random rnd(test::RandomSeed() + 1); + Add("", "v"); + Test(&rnd); +} + +TEST_P(ParameterizedHarnessTest, SimpleSingle) { + Random rnd(test::RandomSeed() + 2); + Add("abc", "v"); + Test(&rnd); +} + +TEST_P(ParameterizedHarnessTest, SimpleMulti) { + Random rnd(test::RandomSeed() + 3); + Add("abc", "v"); + Add("abcd", "v"); + Add("ac", "v2"); + Test(&rnd); +} + +TEST_P(ParameterizedHarnessTest, SimpleSpecialKey) { + Random rnd(test::RandomSeed() + 4); + Add("\xff\xff", "v3"); + Test(&rnd); +} + +TEST(TableTest, FooterTests) { + Random* r = Random::GetTLSInstance(); + uint64_t data_size = (uint64_t{1} << r->Uniform(40)) + r->Uniform(100); + uint64_t index_size = r->Uniform(1000000000); + uint64_t metaindex_size = r->Uniform(1000000); + // 5 == block trailer size + BlockHandle index(data_size + 5, index_size); + BlockHandle meta_index(data_size + index_size + 2 * 5, metaindex_size); + uint64_t footer_offset = data_size + metaindex_size + index_size + 3 * 5; + uint32_t base_context_checksum = 123456789; + { + // legacy block based + FooterBuilder footer; + ASSERT_OK(footer.Build(kBlockBasedTableMagicNumber, /* format_version */ 0, + footer_offset, kCRC32c, meta_index, index)); + Footer decoded_footer; + ASSERT_OK(decoded_footer.DecodeFrom(footer.GetSlice(), footer_offset)); + ASSERT_EQ(decoded_footer.table_magic_number(), kBlockBasedTableMagicNumber); + ASSERT_EQ(decoded_footer.checksum_type(), kCRC32c); + ASSERT_EQ(decoded_footer.metaindex_handle().offset(), meta_index.offset()); + ASSERT_EQ(decoded_footer.metaindex_handle().size(), meta_index.size()); + ASSERT_EQ(decoded_footer.index_handle().offset(), index.offset()); + ASSERT_EQ(decoded_footer.index_handle().size(), index.size()); + ASSERT_EQ(decoded_footer.format_version(), 0U); + ASSERT_EQ(decoded_footer.base_context_checksum(), 0U); + ASSERT_EQ(decoded_footer.GetBlockTrailerSize(), 5U); + // Ensure serialized with legacy magic + ASSERT_EQ( + DecodeFixed64(footer.GetSlice().data() + footer.GetSlice().size() - 8), + kLegacyBlockBasedTableMagicNumber); + } + // block based, various checksums, various versions + for (auto t : GetSupportedChecksums()) { + for (uint32_t fv = 1; IsSupportedFormatVersion(fv); ++fv) { + uint32_t maybe_bcc = + FormatVersionUsesContextChecksum(fv) ? base_context_checksum : 0U; + FooterBuilder footer; + ASSERT_OK(footer.Build(kBlockBasedTableMagicNumber, fv, footer_offset, t, + meta_index, index, maybe_bcc)); + Footer decoded_footer; + ASSERT_OK(decoded_footer.DecodeFrom(footer.GetSlice(), footer_offset)); + ASSERT_EQ(decoded_footer.table_magic_number(), + kBlockBasedTableMagicNumber); + ASSERT_EQ(decoded_footer.checksum_type(), t); + ASSERT_EQ(decoded_footer.metaindex_handle().offset(), + meta_index.offset()); + ASSERT_EQ(decoded_footer.metaindex_handle().size(), meta_index.size()); + if (FormatVersionUsesIndexHandleInFooter(fv)) { + ASSERT_EQ(decoded_footer.index_handle().offset(), index.offset()); + ASSERT_EQ(decoded_footer.index_handle().size(), index.size()); + } + ASSERT_EQ(decoded_footer.format_version(), fv); + ASSERT_EQ(decoded_footer.GetBlockTrailerSize(), 5U); + + if (FormatVersionUsesContextChecksum(fv)) { + ASSERT_EQ(decoded_footer.base_context_checksum(), + base_context_checksum); + + // Bad offset should fail footer checksum + decoded_footer = Footer(); + ASSERT_NOK( + decoded_footer.DecodeFrom(footer.GetSlice(), footer_offset - 1)); + } else { + ASSERT_EQ(decoded_footer.base_context_checksum(), 0U); + } + + // Too big metaindex size should also fail encoding only in new footer + uint64_t big_metaindex_size = 0x100000007U; + uint64_t big_footer_offset = + data_size + big_metaindex_size + index_size + 3 * 5; + BlockHandle big_metaindex = + BlockHandle(data_size + index_size + 2 * 5, big_metaindex_size); + ASSERT_NE(footer + .Build(kBlockBasedTableMagicNumber, fv, big_footer_offset, + t, big_metaindex, index, maybe_bcc) + .ok(), + FormatVersionUsesContextChecksum(fv)); + } + } + + { + // legacy plain table + FooterBuilder footer; + ASSERT_OK(footer.Build(kPlainTableMagicNumber, /* format_version */ 0, + footer_offset, kNoChecksum, meta_index)); + Footer decoded_footer; + ASSERT_OK(decoded_footer.DecodeFrom(footer.GetSlice(), footer_offset)); + ASSERT_EQ(decoded_footer.table_magic_number(), kPlainTableMagicNumber); + ASSERT_EQ(decoded_footer.checksum_type(), kCRC32c); + ASSERT_EQ(decoded_footer.metaindex_handle().offset(), meta_index.offset()); + ASSERT_EQ(decoded_footer.metaindex_handle().size(), meta_index.size()); + ASSERT_EQ(decoded_footer.index_handle().offset(), 0U); + ASSERT_EQ(decoded_footer.index_handle().size(), 0U); + ASSERT_EQ(decoded_footer.format_version(), 0U); + ASSERT_EQ(decoded_footer.GetBlockTrailerSize(), 0U); + // Ensure serialized with legacy magic + ASSERT_EQ( + DecodeFixed64(footer.GetSlice().data() + footer.GetSlice().size() - 8), + kLegacyPlainTableMagicNumber); + } + { + // xxhash plain table (not currently used) + FooterBuilder footer; + ASSERT_OK(footer.Build(kPlainTableMagicNumber, /* format_version */ 1, + footer_offset, kxxHash, meta_index)); + Footer decoded_footer; + ASSERT_OK(decoded_footer.DecodeFrom(footer.GetSlice(), footer_offset)); + ASSERT_EQ(decoded_footer.table_magic_number(), kPlainTableMagicNumber); + ASSERT_EQ(decoded_footer.checksum_type(), kxxHash); + ASSERT_EQ(decoded_footer.metaindex_handle().offset(), meta_index.offset()); + ASSERT_EQ(decoded_footer.metaindex_handle().size(), meta_index.size()); + ASSERT_EQ(decoded_footer.index_handle().offset(), 0U); + ASSERT_EQ(decoded_footer.index_handle().size(), 0U); + ASSERT_EQ(decoded_footer.format_version(), 1U); + ASSERT_EQ(decoded_footer.GetBlockTrailerSize(), 0U); + } +} + +class IndexBlockRestartIntervalTest + : public TableTest, + public ::testing::WithParamInterface> { + public: + static std::vector> GetRestartValues() { + return {{-1, false}, {0, false}, {1, false}, {8, false}, + {16, false}, {32, false}, {-1, true}, {0, true}, + {1, true}, {8, true}, {16, true}, {32, true}}; + } +}; + +INSTANTIATE_TEST_CASE_P( + IndexBlockRestartIntervalTest, IndexBlockRestartIntervalTest, + ::testing::ValuesIn(IndexBlockRestartIntervalTest::GetRestartValues())); + +TEST_P(IndexBlockRestartIntervalTest, IndexBlockRestartInterval) { + const int kKeysInTable = 10000; + const int kKeySize = 100; + const int kValSize = 500; + + const int index_block_restart_interval = std::get<0>(GetParam()); + const bool value_delta_encoding = std::get<1>(GetParam()); + + Options options; + BlockBasedTableOptions table_options; + table_options.block_size = 64; // small block size to get big index block + table_options.index_block_restart_interval = index_block_restart_interval; + if (value_delta_encoding) { + table_options.format_version = 4; + } else { + table_options.format_version = 3; + } + options.table_factory.reset(new BlockBasedTableFactory(table_options)); + + TableConstructor c(BytewiseComparator()); + static Random rnd(301); + for (int i = 0; i < kKeysInTable; i++) { + InternalKey k(rnd.RandomString(kKeySize), 0, kTypeValue); + c.Add(k.Encode().ToString(), rnd.RandomString(kValSize)); + } + + std::vector keys; + stl_wrappers::KVMap kvmap; + std::unique_ptr comparator( + new InternalKeyComparator(BytewiseComparator())); + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + c.Finish(options, ioptions, moptions, table_options, *comparator, &keys, + &kvmap); + auto reader = c.GetTableReader(); + + ReadOptions read_options; + std::unique_ptr db_iter(reader->NewIterator( + read_options, moptions.prefix_extractor.get(), /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized)); + + // Test point lookup + for (auto& kv : kvmap) { + db_iter->Seek(kv.first); + + ASSERT_TRUE(db_iter->Valid()); + ASSERT_OK(db_iter->status()); + ASSERT_EQ(db_iter->key(), kv.first); + ASSERT_EQ(db_iter->value(), kv.second); + } + + // Test iterating + auto kv_iter = kvmap.begin(); + for (db_iter->SeekToFirst(); db_iter->Valid(); db_iter->Next()) { + ASSERT_EQ(db_iter->key(), kv_iter->first); + ASSERT_EQ(db_iter->value(), kv_iter->second); + kv_iter++; + } + ASSERT_EQ(kv_iter, kvmap.end()); + c.ResetTableReader(); +} + +class PrefixTest : public testing::Test { + public: + PrefixTest() : testing::Test() {} + ~PrefixTest() override {} +}; + +namespace { +// A simple PrefixExtractor that only works for test PrefixAndWholeKeyTest +class TestPrefixExtractor : public ROCKSDB_NAMESPACE::SliceTransform { + public: + ~TestPrefixExtractor() override{}; + const char* Name() const override { return "TestPrefixExtractor"; } + + ROCKSDB_NAMESPACE::Slice Transform( + const ROCKSDB_NAMESPACE::Slice& src) const override { + assert(IsValid(src)); + return ROCKSDB_NAMESPACE::Slice(src.data(), 3); + } + + bool InDomain(const ROCKSDB_NAMESPACE::Slice& src) const override { + return IsValid(src); + } + + bool InRange(const ROCKSDB_NAMESPACE::Slice& /*dst*/) const override { + return true; + } + + bool IsValid(const ROCKSDB_NAMESPACE::Slice& src) const { + if (src.size() != 4) { + return false; + } + if (src[0] != '[') { + return false; + } + if (src[1] < '0' || src[1] > '9') { + return false; + } + if (src[2] != ']') { + return false; + } + if (src[3] < '0' || src[3] > '9') { + return false; + } + return true; + } +}; +} // namespace + +TEST_F(PrefixTest, PrefixAndWholeKeyTest) { + ROCKSDB_NAMESPACE::Options options; + options.compaction_style = ROCKSDB_NAMESPACE::kCompactionStyleUniversal; + options.num_levels = 20; + options.create_if_missing = true; + options.optimize_filters_for_hits = false; + options.target_file_size_base = 268435456; + options.prefix_extractor = std::make_shared(); + ROCKSDB_NAMESPACE::BlockBasedTableOptions bbto; + bbto.filter_policy.reset(ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10)); + bbto.block_size = 262144; + bbto.whole_key_filtering = true; + + const std::string kDBPath = test::PerThreadDBPath("table_prefix_test"); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + ASSERT_OK(DestroyDB(kDBPath, options)); + ROCKSDB_NAMESPACE::DB* db; + ASSERT_OK(ROCKSDB_NAMESPACE::DB::Open(options, kDBPath, &db)); + + // Create a bunch of keys with 10 filters. + for (int i = 0; i < 10; i++) { + std::string prefix = "[" + std::to_string(i) + "]"; + for (int j = 0; j < 10; j++) { + std::string key = prefix + std::to_string(j); + ASSERT_OK(db->Put(ROCKSDB_NAMESPACE::WriteOptions(), key, "1")); + } + } + + // Trigger compaction. + ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + delete db; + // In the second round, turn whole_key_filtering off and expect + // rocksdb still works. +} + +/* + * Disable TableWithGlobalSeqno since RocksDB does not store global_seqno in + * the SST file any more. Instead, RocksDB deduces global_seqno from the + * MANIFEST while reading from an SST. Therefore, it's not possible to test the + * functionality of global_seqno in a single, isolated unit test without the + * involvement of Version, VersionSet, etc. + */ +TEST_P(BlockBasedTableTest, DISABLED_TableWithGlobalSeqno) { + BlockBasedTableOptions bbto = GetBlockBasedTableOptions(); + test::StringSink* sink = new test::StringSink(); + std::unique_ptr holder(sink); + std::unique_ptr file_writer(new WritableFileWriter( + std::move(holder), "" /* don't care */, FileOptions())); + Options options; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + InternalKeyComparator ikc(options.comparator); + IntTblPropCollectorFactories int_tbl_prop_collector_factories; + int_tbl_prop_collector_factories.emplace_back( + new SstFileWriterPropertiesCollectorFactory(2 /* version */, + 0 /* global_seqno*/)); + std::string column_family_name; + std::unique_ptr builder(options.table_factory->NewTableBuilder( + TableBuilderOptions(ioptions, moptions, ikc, + &int_tbl_prop_collector_factories, kNoCompression, + CompressionOptions(), kUnknownColumnFamily, + column_family_name, -1), + file_writer.get())); + + for (char c = 'a'; c <= 'z'; ++c) { + std::string key(8, c); + std::string value = key; + InternalKey ik(key, 0, kTypeValue); + + builder->Add(ik.Encode(), value); + } + ASSERT_OK(builder->Finish()); + ASSERT_OK(file_writer->Flush()); + + test::RandomRWStringSink ss_rw(sink); + uint32_t version; + uint64_t global_seqno; + uint64_t global_seqno_offset; + + // Helper function to get version, global_seqno, global_seqno_offset + std::function GetVersionAndGlobalSeqno = [&]() { + std::unique_ptr source( + new test::StringSource(ss_rw.contents(), 73342, true)); + std::unique_ptr file_reader( + new RandomAccessFileReader(std::move(source), "")); + + std::unique_ptr props; + const ReadOptions read_options; + ASSERT_OK(ReadTableProperties(file_reader.get(), ss_rw.contents().size(), + kBlockBasedTableMagicNumber, ioptions, + read_options, &props)); + + UserCollectedProperties user_props = props->user_collected_properties; + version = DecodeFixed32( + user_props[ExternalSstFilePropertyNames::kVersion].c_str()); + global_seqno = DecodeFixed64( + user_props[ExternalSstFilePropertyNames::kGlobalSeqno].c_str()); + global_seqno_offset = props->external_sst_file_global_seqno_offset; + }; + + // Helper function to update the value of the global seqno in the file + std::function SetGlobalSeqno = [&](uint64_t val) { + std::string new_global_seqno; + PutFixed64(&new_global_seqno, val); + + ASSERT_OK(ss_rw.Write(global_seqno_offset, new_global_seqno, IOOptions(), + nullptr)); + }; + + // Helper function to get the contents of the table InternalIterator + std::unique_ptr table_reader; + const ReadOptions read_options; + std::function GetTableInternalIter = [&]() { + std::unique_ptr source( + new test::StringSource(ss_rw.contents(), 73342, true)); + std::unique_ptr file_reader( + new RandomAccessFileReader(std::move(source), "")); + + options.table_factory->NewTableReader( + TableReaderOptions(ioptions, moptions.prefix_extractor, EnvOptions(), + ikc, 0 /* block_protection_bytes_per_key */), + std::move(file_reader), ss_rw.contents().size(), &table_reader); + + return table_reader->NewIterator( + read_options, moptions.prefix_extractor.get(), /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized); + }; + + GetVersionAndGlobalSeqno(); + ASSERT_EQ(2u, version); + ASSERT_EQ(0u, global_seqno); + + InternalIterator* iter = GetTableInternalIter(); + char current_c = 'a'; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ParsedInternalKey pik; + ASSERT_OK(ParseInternalKey(iter->key(), &pik, true /* log_err_key */)); + + ASSERT_EQ(pik.type, ValueType::kTypeValue); + ASSERT_EQ(pik.sequence, 0); + ASSERT_EQ(pik.user_key, iter->value()); + ASSERT_EQ(pik.user_key.ToString(), std::string(8, current_c)); + current_c++; + } + ASSERT_EQ(current_c, 'z' + 1); + delete iter; + + // Update global sequence number to 10 + SetGlobalSeqno(10); + GetVersionAndGlobalSeqno(); + ASSERT_EQ(2u, version); + ASSERT_EQ(10u, global_seqno); + + iter = GetTableInternalIter(); + current_c = 'a'; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ParsedInternalKey pik; + ASSERT_OK(ParseInternalKey(iter->key(), &pik, true /* log_err_key */)); + + ASSERT_EQ(pik.type, ValueType::kTypeValue); + ASSERT_EQ(pik.sequence, 10); + ASSERT_EQ(pik.user_key, iter->value()); + ASSERT_EQ(pik.user_key.ToString(), std::string(8, current_c)); + current_c++; + } + ASSERT_EQ(current_c, 'z' + 1); + + // Verify Seek + for (char c = 'a'; c <= 'z'; c++) { + std::string k = std::string(8, c); + InternalKey ik(k, 10, kValueTypeForSeek); + iter->Seek(ik.Encode()); + ASSERT_TRUE(iter->Valid()); + + ParsedInternalKey pik; + ASSERT_OK(ParseInternalKey(iter->key(), &pik, true /* log_err_key */)); + + ASSERT_EQ(pik.type, ValueType::kTypeValue); + ASSERT_EQ(pik.sequence, 10); + ASSERT_EQ(pik.user_key.ToString(), k); + ASSERT_EQ(iter->value().ToString(), k); + } + delete iter; + + // Update global sequence number to 3 + SetGlobalSeqno(3); + GetVersionAndGlobalSeqno(); + ASSERT_EQ(2u, version); + ASSERT_EQ(3u, global_seqno); + + iter = GetTableInternalIter(); + current_c = 'a'; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ParsedInternalKey pik; + ASSERT_OK(ParseInternalKey(iter->key(), &pik, true /* log_err_key */)); + + ASSERT_EQ(pik.type, ValueType::kTypeValue); + ASSERT_EQ(pik.sequence, 3); + ASSERT_EQ(pik.user_key, iter->value()); + ASSERT_EQ(pik.user_key.ToString(), std::string(8, current_c)); + current_c++; + } + ASSERT_EQ(current_c, 'z' + 1); + + // Verify Seek + for (char c = 'a'; c <= 'z'; c++) { + std::string k = std::string(8, c); + // seqno=4 is less than 3 so we still should get our key + InternalKey ik(k, 4, kValueTypeForSeek); + iter->Seek(ik.Encode()); + ASSERT_TRUE(iter->Valid()); + + ParsedInternalKey pik; + ASSERT_OK(ParseInternalKey(iter->key(), &pik, true /* log_err_key */)); + + ASSERT_EQ(pik.type, ValueType::kTypeValue); + ASSERT_EQ(pik.sequence, 3); + ASSERT_EQ(pik.user_key.ToString(), k); + ASSERT_EQ(iter->value().ToString(), k); + } + + delete iter; +} + +TEST_P(BlockBasedTableTest, BlockAlignTest) { + BlockBasedTableOptions bbto = GetBlockBasedTableOptions(); + bbto.block_align = true; + test::StringSink* sink = new test::StringSink(); + std::unique_ptr holder(sink); + std::unique_ptr file_writer(new WritableFileWriter( + std::move(holder), "" /* don't care */, FileOptions())); + Options options; + options.compression = kNoCompression; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + InternalKeyComparator ikc(options.comparator); + IntTblPropCollectorFactories int_tbl_prop_collector_factories; + std::string column_family_name; + std::unique_ptr builder(options.table_factory->NewTableBuilder( + TableBuilderOptions(ioptions, moptions, ikc, + &int_tbl_prop_collector_factories, kNoCompression, + CompressionOptions(), kUnknownColumnFamily, + column_family_name, -1), + file_writer.get())); + + for (int i = 1; i <= 10000; ++i) { + std::ostringstream ostr; + ostr << std::setfill('0') << std::setw(5) << i; + std::string key = ostr.str(); + std::string value = "val"; + InternalKey ik(key, 0, kTypeValue); + + builder->Add(ik.Encode(), value); + } + ASSERT_OK(builder->Finish()); + ASSERT_OK(file_writer->Flush()); + + std::unique_ptr source( + new test::StringSource(sink->contents(), 73342, false)); + std::unique_ptr file_reader( + new RandomAccessFileReader(std::move(source), "test")); + // Helper function to get version, global_seqno, global_seqno_offset + std::function VerifyBlockAlignment = [&]() { + std::unique_ptr props; + const ReadOptions read_options; + ASSERT_OK(ReadTableProperties(file_reader.get(), sink->contents().size(), + kBlockBasedTableMagicNumber, ioptions, + read_options, &props)); + + uint64_t data_block_size = props->data_size / props->num_data_blocks; + ASSERT_EQ(data_block_size, 4096); + ASSERT_EQ(props->data_size, data_block_size * props->num_data_blocks); + }; + + VerifyBlockAlignment(); + + // The below block of code verifies that we can read back the keys. Set + // block_align to false when creating the reader to ensure we can flip between + // the two modes without any issues + std::unique_ptr table_reader; + bbto.block_align = false; + Options options2; + options2.table_factory.reset(NewBlockBasedTableFactory(bbto)); + ImmutableOptions ioptions2(options2); + const MutableCFOptions moptions2(options2); + + ASSERT_OK(ioptions.table_factory->NewTableReader( + TableReaderOptions(ioptions2, moptions2.prefix_extractor, EnvOptions(), + GetPlainInternalComparator(options2.comparator), + 0 /* block_protection_bytes_per_key */), + std::move(file_reader), sink->contents().size(), &table_reader)); + + ReadOptions read_options; + std::unique_ptr db_iter(table_reader->NewIterator( + read_options, moptions2.prefix_extractor.get(), /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized)); + + int expected_key = 1; + for (db_iter->SeekToFirst(); db_iter->Valid(); db_iter->Next()) { + std::ostringstream ostr; + ostr << std::setfill('0') << std::setw(5) << expected_key++; + std::string key = ostr.str(); + std::string value = "val"; + + ASSERT_OK(db_iter->status()); + ASSERT_EQ(ExtractUserKey(db_iter->key()).ToString(), key); + ASSERT_EQ(db_iter->value().ToString(), value); + } + expected_key--; + ASSERT_EQ(expected_key, 10000); + table_reader.reset(); +} + +TEST_P(BlockBasedTableTest, PropertiesBlockRestartPointTest) { + BlockBasedTableOptions bbto = GetBlockBasedTableOptions(); + bbto.block_align = true; + test::StringSink* sink = new test::StringSink(); + std::unique_ptr holder(sink); + std::unique_ptr file_writer(new WritableFileWriter( + std::move(holder), "" /* don't care */, FileOptions())); + + Options options; + options.compression = kNoCompression; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + InternalKeyComparator ikc(options.comparator); + IntTblPropCollectorFactories int_tbl_prop_collector_factories; + std::string column_family_name; + + std::unique_ptr builder(options.table_factory->NewTableBuilder( + TableBuilderOptions(ioptions, moptions, ikc, + &int_tbl_prop_collector_factories, kNoCompression, + CompressionOptions(), kUnknownColumnFamily, + column_family_name, -1), + file_writer.get())); + + for (int i = 1; i <= 10000; ++i) { + std::ostringstream ostr; + ostr << std::setfill('0') << std::setw(5) << i; + std::string key = ostr.str(); + std::string value = "val"; + InternalKey ik(key, 0, kTypeValue); + + builder->Add(ik.Encode(), value); + } + ASSERT_OK(builder->Finish()); + ASSERT_OK(file_writer->Flush()); + + std::unique_ptr source( + new test::StringSource(sink->contents(), 73342, true)); + std::unique_ptr file_reader( + new RandomAccessFileReader(std::move(source), "test")); + + { + RandomAccessFileReader* file = file_reader.get(); + uint64_t file_size = sink->contents().size(); + + Footer footer; + IOOptions opts; + ASSERT_OK(ReadFooterFromFile(opts, file, *FileSystem::Default(), + nullptr /* prefetch_buffer */, file_size, + &footer, kBlockBasedTableMagicNumber)); + + auto BlockFetchHelper = [&](const BlockHandle& handle, BlockType block_type, + BlockContents* contents) { + ReadOptions read_options; + read_options.verify_checksums = false; + PersistentCacheOptions cache_options; + + BlockFetcher block_fetcher( + file, nullptr /* prefetch_buffer */, footer, read_options, handle, + contents, ioptions, false /* decompress */, + false /*maybe_compressed*/, block_type, + UncompressionDict::GetEmptyDict(), cache_options); + + ASSERT_OK(block_fetcher.ReadBlockContents()); + }; + + // -- Read metaindex block + auto metaindex_handle = footer.metaindex_handle(); + BlockContents metaindex_contents; + + BlockFetchHelper(metaindex_handle, BlockType::kMetaIndex, + &metaindex_contents); + Block metaindex_block(std::move(metaindex_contents)); + + std::unique_ptr meta_iter(metaindex_block.NewDataIterator( + BytewiseComparator(), kDisableGlobalSequenceNumber)); + + // -- Read properties block + BlockHandle properties_handle; + ASSERT_OK(FindOptionalMetaBlock(meta_iter.get(), kPropertiesBlockName, + &properties_handle)); + ASSERT_FALSE(properties_handle.IsNull()); + BlockContents properties_contents; + BlockFetchHelper(properties_handle, BlockType::kProperties, + &properties_contents); + Block properties_block(std::move(properties_contents)); + + ASSERT_EQ(properties_block.NumRestarts(), 1u); + } +} + +TEST_P(BlockBasedTableTest, CompressionRatioThreshold) { + for (CompressionType type : GetSupportedCompressions()) { + if (type == kNoCompression) { + continue; + } + if (type == kBZip2Compression) { + // Weird behavior in this test + continue; + } + SCOPED_TRACE("Compression type: " + std::to_string(type)); + + Options options; + options.compression = type; + + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + int len = 10000; + Random rnd(301); + std::vector keys; + stl_wrappers::KVMap kvmap; + + // Test the max_compressed_bytes_per_kb option + for (int threshold : {0, 1, 100, 400, 600, 900, 1024}) { + SCOPED_TRACE("threshold=" + std::to_string(threshold)); + options.compression_opts.max_compressed_bytes_per_kb = threshold; + ImmutableOptions ioptions(options); + MutableCFOptions moptions(options); + + for (double compressible_to : {0.25, 0.75}) { + SCOPED_TRACE("compressible_to=" + std::to_string(compressible_to)); + TableConstructor c(BytewiseComparator(), + true /* convert_to_internal_key_ */); + std::string buf; + c.Add("x", test::CompressibleString(&rnd, compressible_to, len, &buf)); + + // write an SST file + c.Finish(options, ioptions, moptions, table_options, + GetPlainInternalComparator(options.comparator), &keys, &kvmap); + + size_t table_file_size = c.TEST_GetSink()->contents().size(); + size_t approx_sst_overhead = 1000; + if (compressible_to < threshold / 1024.0) { + // Should be compressed (substantial variance depending on algorithm) + EXPECT_NEAR2(len * compressible_to + approx_sst_overhead, + table_file_size, len / 8); + } else { + // Should not be compressed + EXPECT_NEAR2(len + approx_sst_overhead, table_file_size, len / 10); + } + } + } + } +} + +TEST_P(BlockBasedTableTest, PropertiesMetaBlockLast) { + // The properties meta-block should come at the end since we always need to + // read it when opening a file, unlike index/filter/other meta-blocks, which + // are sometimes read depending on the user's configuration. This ordering + // allows us to do a small readahead on the end of the file to read properties + // and meta-index blocks with one I/O. + TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */); + c.Add("a1", "val1"); + c.Add("b2", "val2"); + c.Add("c3", "val3"); + c.Add("d4", "val4"); + c.Add("e5", "val5"); + c.Add("f6", "val6"); + c.Add("g7", "val7"); + c.Add("h8", "val8"); + c.Add("j9", "val9"); + + // write an SST file + Options options; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.filter_policy.reset(NewBloomFilterPolicy( + 8 /* bits_per_key */, false /* use_block_based_filter */)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + ImmutableOptions ioptions(options); + MutableCFOptions moptions(options); + std::vector keys; + stl_wrappers::KVMap kvmap; + c.Finish(options, ioptions, moptions, table_options, + GetPlainInternalComparator(options.comparator), &keys, &kvmap); + + // get file reader + test::StringSink* table_sink = c.TEST_GetSink(); + std::unique_ptr source(new test::StringSource( + table_sink->contents(), 0 /* unique_id */, false /* allow_mmap_reads */)); + + std::unique_ptr table_reader( + new RandomAccessFileReader(std::move(source), "test")); + size_t table_size = table_sink->contents().size(); + + // read footer + Footer footer; + IOOptions opts; + ASSERT_OK(ReadFooterFromFile(opts, table_reader.get(), *FileSystem::Default(), + nullptr /* prefetch_buffer */, table_size, + &footer, kBlockBasedTableMagicNumber)); + + // read metaindex + auto metaindex_handle = footer.metaindex_handle(); + BlockContents metaindex_contents; + PersistentCacheOptions pcache_opts; + BlockFetcher block_fetcher( + table_reader.get(), nullptr /* prefetch_buffer */, footer, ReadOptions(), + metaindex_handle, &metaindex_contents, ioptions, false /* decompress */, + false /*maybe_compressed*/, BlockType::kMetaIndex, + UncompressionDict::GetEmptyDict(), pcache_opts, + nullptr /*memory_allocator*/); + ASSERT_OK(block_fetcher.ReadBlockContents()); + Block metaindex_block(std::move(metaindex_contents)); + + // verify properties block comes last + std::unique_ptr metaindex_iter{ + metaindex_block.NewMetaIterator()}; + uint64_t max_offset = 0; + std::string key_at_max_offset; + for (metaindex_iter->SeekToFirst(); metaindex_iter->Valid(); + metaindex_iter->Next()) { + BlockHandle handle; + Slice value = metaindex_iter->value(); + ASSERT_OK(handle.DecodeFrom(&value)); + if (handle.offset() > max_offset) { + max_offset = handle.offset(); + key_at_max_offset = metaindex_iter->key().ToString(); + } + } + ASSERT_EQ(kPropertiesBlockName, key_at_max_offset); + if (FormatVersionUsesIndexHandleInFooter(footer.format_version())) { + // If index handle is stored in footer rather than metaindex block, + // need separate logic to verify it comes before properties block. + ASSERT_GT(max_offset, footer.index_handle().offset()); + } else { + ASSERT_TRUE(footer.index_handle().IsNull()); + } + c.ResetTableReader(); +} + +TEST_P(BlockBasedTableTest, SeekMetaBlocks) { + TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */); + c.Add("foo_a1", "val1"); + c.Add("foo_b2", "val2"); + c.Add("foo_c3", "val3"); + c.Add("foo_d4", "val4"); + c.Add("foo_e5", "val5"); + c.Add("foo_f6", "val6"); + c.Add("foo_g7", "val7"); + c.Add("foo_h8", "val8"); + c.Add("foo_j9", "val9"); + + // write an SST file + Options options; + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.index_type = BlockBasedTableOptions::kHashSearch; + table_options.filter_policy.reset(NewBloomFilterPolicy( + 8 /* bits_per_key */, false /* use_block_based_filter */)); + options.prefix_extractor.reset(NewFixedPrefixTransform(4)); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + ImmutableOptions ioptions(options); + MutableCFOptions moptions(options); + std::vector keys; + stl_wrappers::KVMap kvmap; + c.Finish(options, ioptions, moptions, table_options, + GetPlainInternalComparator(options.comparator), &keys, &kvmap); + + // get file reader + test::StringSink* table_sink = c.TEST_GetSink(); + std::unique_ptr source(new test::StringSource( + table_sink->contents(), 0 /* unique_id */, false /* allow_mmap_reads */)); + + std::unique_ptr table_reader( + new RandomAccessFileReader(std::move(source), "test")); + size_t table_size = table_sink->contents().size(); + + // read footer + Footer footer; + IOOptions opts; + ASSERT_OK(ReadFooterFromFile(opts, table_reader.get(), *FileSystem::Default(), + nullptr /* prefetch_buffer */, table_size, + &footer, kBlockBasedTableMagicNumber)); + + // read metaindex + auto metaindex_handle = footer.metaindex_handle(); + BlockContents metaindex_contents; + PersistentCacheOptions pcache_opts; + BlockFetcher block_fetcher( + table_reader.get(), nullptr /* prefetch_buffer */, footer, ReadOptions(), + metaindex_handle, &metaindex_contents, ioptions, false /* decompress */, + false /*maybe_compressed*/, BlockType::kMetaIndex, + UncompressionDict::GetEmptyDict(), pcache_opts, + nullptr /*memory_allocator*/); + ASSERT_OK(block_fetcher.ReadBlockContents()); + Block metaindex_block(std::move(metaindex_contents)); + + // verify properties block comes last + std::unique_ptr metaindex_iter( + metaindex_block.NewMetaIterator()); + bool has_hash_prefixes = false; + bool has_hash_metadata = false; + for (metaindex_iter->SeekToFirst(); metaindex_iter->Valid(); + metaindex_iter->Next()) { + if (metaindex_iter->key().ToString() == kHashIndexPrefixesBlock) { + has_hash_prefixes = true; + } else if (metaindex_iter->key().ToString() == + kHashIndexPrefixesMetadataBlock) { + has_hash_metadata = true; + } + } + if (has_hash_metadata) { + metaindex_iter->Seek(kHashIndexPrefixesMetadataBlock); + ASSERT_TRUE(metaindex_iter->Valid()); + ASSERT_EQ(kHashIndexPrefixesMetadataBlock, + metaindex_iter->key().ToString()); + } + if (has_hash_prefixes) { + metaindex_iter->Seek(kHashIndexPrefixesBlock); + ASSERT_TRUE(metaindex_iter->Valid()); + ASSERT_EQ(kHashIndexPrefixesBlock, metaindex_iter->key().ToString()); + } + c.ResetTableReader(); +} + +TEST_P(BlockBasedTableTest, BadOptions) { + ROCKSDB_NAMESPACE::Options options; + options.compression = kNoCompression; + BlockBasedTableOptions bbto = GetBlockBasedTableOptions(); + bbto.block_size = 4000; + bbto.block_align = true; + + const std::string kDBPath = + test::PerThreadDBPath("block_based_table_bad_options_test"); + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + ASSERT_OK(DestroyDB(kDBPath, options)); + ROCKSDB_NAMESPACE::DB* db; + ASSERT_NOK(ROCKSDB_NAMESPACE::DB::Open(options, kDBPath, &db)); + + bbto.block_size = 4096; + options.compression = kSnappyCompression; + options.table_factory.reset(NewBlockBasedTableFactory(bbto)); + ASSERT_NOK(ROCKSDB_NAMESPACE::DB::Open(options, kDBPath, &db)); +} + +TEST_F(BBTTailPrefetchTest, TestTailPrefetchStats) { + TailPrefetchStats tpstats; + ASSERT_EQ(0, tpstats.GetSuggestedPrefetchSize()); + tpstats.RecordEffectiveSize(size_t{1000}); + tpstats.RecordEffectiveSize(size_t{1005}); + tpstats.RecordEffectiveSize(size_t{1002}); + ASSERT_EQ(1005, tpstats.GetSuggestedPrefetchSize()); + + // One single super large value shouldn't influence much + tpstats.RecordEffectiveSize(size_t{1002000}); + tpstats.RecordEffectiveSize(size_t{999}); + ASSERT_LE(1005, tpstats.GetSuggestedPrefetchSize()); + ASSERT_GT(1200, tpstats.GetSuggestedPrefetchSize()); + + // Only history of 32 is kept + for (int i = 0; i < 32; i++) { + tpstats.RecordEffectiveSize(size_t{100}); + } + ASSERT_EQ(100, tpstats.GetSuggestedPrefetchSize()); + + // 16 large values and 16 small values. The result should be closer + // to the small value as the algorithm. + for (int i = 0; i < 16; i++) { + tpstats.RecordEffectiveSize(size_t{1000}); + } + tpstats.RecordEffectiveSize(size_t{10}); + tpstats.RecordEffectiveSize(size_t{20}); + for (int i = 0; i < 6; i++) { + tpstats.RecordEffectiveSize(size_t{100}); + } + ASSERT_LE(80, tpstats.GetSuggestedPrefetchSize()); + ASSERT_GT(200, tpstats.GetSuggestedPrefetchSize()); +} + +TEST_F(BBTTailPrefetchTest, FilePrefetchBufferMinOffset) { + TailPrefetchStats tpstats; + FilePrefetchBuffer buffer(0 /* readahead_size */, 0 /* max_readahead_size */, + false /* enable */, true /* track_min_offset */); + IOOptions opts; + buffer.TryReadFromCache(opts, nullptr /* reader */, 500 /* offset */, + 10 /* n */, nullptr /* result */, + nullptr /* status */, + Env::IO_TOTAL /* rate_limiter_priority */); + buffer.TryReadFromCache(opts, nullptr /* reader */, 480 /* offset */, + 10 /* n */, nullptr /* result */, + nullptr /* status */, + Env::IO_TOTAL /* rate_limiter_priority */); + buffer.TryReadFromCache(opts, nullptr /* reader */, 490 /* offset */, + 10 /* n */, nullptr /* result */, + nullptr /* status */, + Env::IO_TOTAL /* rate_limiter_priority */); + ASSERT_EQ(480, buffer.min_offset_read()); +} + +TEST_P(BlockBasedTableTest, DataBlockHashIndex) { + const int kNumKeys = 500; + const int kKeySize = 8; + const int kValSize = 40; + + BlockBasedTableOptions table_options = GetBlockBasedTableOptions(); + table_options.data_block_index_type = + BlockBasedTableOptions::kDataBlockBinaryAndHash; + + Options options; + options.comparator = BytewiseComparator(); + + options.table_factory.reset(new BlockBasedTableFactory(table_options)); + + TableConstructor c(options.comparator); + + static Random rnd(1048); + for (int i = 0; i < kNumKeys; i++) { + // padding one "0" to mark existent keys. + std::string random_key(rnd.RandomString(kKeySize - 1) + "1"); + InternalKey k(random_key, 0, kTypeValue); + c.Add(k.Encode().ToString(), rnd.RandomString(kValSize)); + } + + std::vector keys; + stl_wrappers::KVMap kvmap; + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + const InternalKeyComparator internal_comparator(options.comparator); + c.Finish(options, ioptions, moptions, table_options, internal_comparator, + &keys, &kvmap); + + auto reader = c.GetTableReader(); + + std::unique_ptr seek_iter; + ReadOptions read_options; + seek_iter.reset(reader->NewIterator( + read_options, moptions.prefix_extractor.get(), /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized)); + for (int i = 0; i < 2; ++i) { + ReadOptions ro; + // for every kv, we seek using two method: Get() and Seek() + // Get() will use the SuffixIndexHash in Block. For non-existent key it + // will invalidate the iterator + // Seek() will use the default BinarySeek() in Block. So for non-existent + // key it will land at the closest key that is large than target. + + // Search for existent keys + for (auto& kv : kvmap) { + if (i == 0) { + // Search using Seek() + seek_iter->Seek(kv.first); + ASSERT_OK(seek_iter->status()); + ASSERT_TRUE(seek_iter->Valid()); + ASSERT_EQ(seek_iter->key(), kv.first); + ASSERT_EQ(seek_iter->value(), kv.second); + } else { + // Search using Get() + PinnableSlice value; + std::string user_key = ExtractUserKey(kv.first).ToString(); + GetContext get_context(options.comparator, nullptr, nullptr, nullptr, + GetContext::kNotFound, user_key, &value, nullptr, + nullptr, nullptr, true, nullptr, nullptr); + ASSERT_OK(reader->Get(ro, kv.first, &get_context, + moptions.prefix_extractor.get())); + ASSERT_EQ(get_context.State(), GetContext::kFound); + ASSERT_EQ(value, Slice(kv.second)); + value.Reset(); + } + } + + // Search for non-existent keys + for (auto& kv : kvmap) { + std::string user_key = ExtractUserKey(kv.first).ToString(); + user_key.back() = '0'; // make it non-existent key + InternalKey internal_key(user_key, 0, kTypeValue); + std::string encoded_key = internal_key.Encode().ToString(); + if (i == 0) { // Search using Seek() + seek_iter->Seek(encoded_key); + ASSERT_OK(seek_iter->status()); + if (seek_iter->Valid()) { + ASSERT_TRUE(BytewiseComparator()->Compare( + user_key, ExtractUserKey(seek_iter->key())) < 0); + } + } else { // Search using Get() + PinnableSlice value; + GetContext get_context(options.comparator, nullptr, nullptr, nullptr, + GetContext::kNotFound, user_key, &value, nullptr, + nullptr, nullptr, true, nullptr, nullptr); + ASSERT_OK(reader->Get(ro, encoded_key, &get_context, + moptions.prefix_extractor.get())); + ASSERT_EQ(get_context.State(), GetContext::kNotFound); + value.Reset(); + } + } + } +} + +// BlockBasedTableIterator should invalidate itself and return +// OutOfBound()=true immediately after Seek(), to allow LevelIterator +// filter out corresponding level. +TEST_P(BlockBasedTableTest, OutOfBoundOnSeek) { + TableConstructor c(BytewiseComparator(), true /*convert_to_internal_key*/); + c.Add("foo", "v1"); + std::vector keys; + stl_wrappers::KVMap kvmap; + Options options; + BlockBasedTableOptions table_opt(GetBlockBasedTableOptions()); + options.table_factory.reset(NewBlockBasedTableFactory(table_opt)); + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + c.Finish(options, ioptions, moptions, table_opt, + GetPlainInternalComparator(BytewiseComparator()), &keys, &kvmap); + auto* reader = c.GetTableReader(); + ReadOptions read_opt; + std::string upper_bound = "bar"; + Slice upper_bound_slice(upper_bound); + read_opt.iterate_upper_bound = &upper_bound_slice; + std::unique_ptr iter; + iter.reset(new KeyConvertingIterator(reader->NewIterator( + read_opt, /*prefix_extractor=*/nullptr, /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized))); + iter->SeekToFirst(); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->UpperBoundCheckResult() == IterBoundCheck::kOutOfBound); + iter.reset(new KeyConvertingIterator(reader->NewIterator( + read_opt, /*prefix_extractor=*/nullptr, /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized))); + iter->Seek("foo"); + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + ASSERT_TRUE(iter->UpperBoundCheckResult() == IterBoundCheck::kOutOfBound); +} + +// BlockBasedTableIterator should invalidate itself and return +// OutOfBound()=true after Next(), if it finds current index key is no smaller +// than upper bound, unless it is pointing to the last data block. +TEST_P(BlockBasedTableTest, OutOfBoundOnNext) { + TableConstructor c(BytewiseComparator(), true /*convert_to_internal_key*/); + c.Add("bar", "v"); + c.Add("foo", "v"); + std::vector keys; + stl_wrappers::KVMap kvmap; + Options options; + BlockBasedTableOptions table_opt(GetBlockBasedTableOptions()); + table_opt.flush_block_policy_factory = + std::make_shared(); + options.table_factory.reset(NewBlockBasedTableFactory(table_opt)); + const ImmutableOptions ioptions(options); + const MutableCFOptions moptions(options); + c.Finish(options, ioptions, moptions, table_opt, + GetPlainInternalComparator(BytewiseComparator()), &keys, &kvmap); + auto* reader = c.GetTableReader(); + ReadOptions read_opt; + std::string ub1 = "bar_after"; + Slice ub_slice1(ub1); + read_opt.iterate_upper_bound = &ub_slice1; + std::unique_ptr iter; + iter.reset(new KeyConvertingIterator(reader->NewIterator( + read_opt, /*prefix_extractor=*/nullptr, /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized))); + iter->Seek("bar"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("bar", iter->key()); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + ASSERT_TRUE(iter->UpperBoundCheckResult() == IterBoundCheck::kOutOfBound); + std::string ub2 = "foo_after"; + Slice ub_slice2(ub2); + read_opt.iterate_upper_bound = &ub_slice2; + iter.reset(new KeyConvertingIterator(reader->NewIterator( + read_opt, /*prefix_extractor=*/nullptr, /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized))); + iter->Seek("foo"); + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ("foo", iter->key()); + iter->Next(); + ASSERT_FALSE(iter->Valid()); + ASSERT_FALSE(iter->UpperBoundCheckResult() == IterBoundCheck::kOutOfBound); +} + +class ChargeCompressionDictionaryBuildingBufferTest + : public BlockBasedTableTestBase {}; +TEST_F(ChargeCompressionDictionaryBuildingBufferTest, Basic) { + constexpr std::size_t kSizeDummyEntry = 256 * 1024; + constexpr std::size_t kMetaDataChargeOverhead = 10000; + constexpr std::size_t kCacheCapacity = 8 * 1024 * 1024; + constexpr std::size_t kMaxDictBytes = 1024; + constexpr std::size_t kMaxDictBufferBytes = 1024; + + for (CacheEntryRoleOptions::Decision + charge_compression_dictionary_building_buffer : + {CacheEntryRoleOptions::Decision::kEnabled, + CacheEntryRoleOptions::Decision::kDisabled}) { + BlockBasedTableOptions table_options; + LRUCacheOptions lo; + lo.capacity = kCacheCapacity; + lo.num_shard_bits = 0; // 2^0 shard + lo.strict_capacity_limit = true; + std::shared_ptr cache(NewLRUCache(lo)); + table_options.block_cache = cache; + table_options.flush_block_policy_factory = + std::make_shared(); + table_options.cache_usage_options.options_overrides.insert( + {CacheEntryRole::kCompressionDictionaryBuildingBuffer, + {/*.charged = */ charge_compression_dictionary_building_buffer}}); + Options options; + options.compression = kSnappyCompression; + options.compression_opts.max_dict_bytes = kMaxDictBytes; + options.compression_opts.max_dict_buffer_bytes = kMaxDictBufferBytes; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + test::StringSink* sink = new test::StringSink(); + std::unique_ptr holder(sink); + std::unique_ptr file_writer(new WritableFileWriter( + std::move(holder), "test_file_name", FileOptions())); + + ImmutableOptions ioptions(options); + MutableCFOptions moptions(options); + InternalKeyComparator ikc(options.comparator); + IntTblPropCollectorFactories int_tbl_prop_collector_factories; + + std::unique_ptr builder( + options.table_factory->NewTableBuilder( + TableBuilderOptions( + ioptions, moptions, ikc, &int_tbl_prop_collector_factories, + kSnappyCompression, options.compression_opts, + kUnknownColumnFamily, "test_cf", -1 /* level */), + file_writer.get())); + + std::string key1 = "key1"; + std::string value1 = "val1"; + InternalKey ik1(key1, 0 /* sequnce number */, kTypeValue); + // Adding the first key won't trigger a flush by FlushBlockEveryKeyPolicy + // therefore won't trigger any data block's buffering + builder->Add(ik1.Encode(), value1); + ASSERT_EQ(cache->GetPinnedUsage(), 0 * kSizeDummyEntry); + + std::string key2 = "key2"; + std::string value2 = "val2"; + InternalKey ik2(key2, 1 /* sequnce number */, kTypeValue); + // Adding the second key will trigger a flush of the last data block (the + // one containing key1 and value1) by FlushBlockEveryKeyPolicy and hence + // trigger buffering of that data block. + builder->Add(ik2.Encode(), value2); + // Cache charging will increase for last buffered data block (the one + // containing key1 and value1) since the buffer limit is not exceeded after + // that buffering and the cache will not be full after this reservation + if (charge_compression_dictionary_building_buffer == + CacheEntryRoleOptions::Decision::kEnabled) { + EXPECT_GE(cache->GetPinnedUsage(), 1 * kSizeDummyEntry); + EXPECT_LT(cache->GetPinnedUsage(), + 1 * kSizeDummyEntry + kMetaDataChargeOverhead); + } else { + EXPECT_EQ(cache->GetPinnedUsage(), 0 * kSizeDummyEntry); + } + + ASSERT_OK(builder->Finish()); + EXPECT_EQ(cache->GetPinnedUsage(), 0 * kSizeDummyEntry); + } +} + +TEST_F(ChargeCompressionDictionaryBuildingBufferTest, + BasicWithBufferLimitExceed) { + constexpr std::size_t kSizeDummyEntry = 256 * 1024; + constexpr std::size_t kMetaDataChargeOverhead = 10000; + constexpr std::size_t kCacheCapacity = 8 * 1024 * 1024; + constexpr std::size_t kMaxDictBytes = 1024; + constexpr std::size_t kMaxDictBufferBytes = 2 * kSizeDummyEntry; + + // `CacheEntryRoleOptions::charged` is enabled by default for + // CacheEntryRole::kCompressionDictionaryBuildingBuffer + BlockBasedTableOptions table_options; + LRUCacheOptions lo; + lo.capacity = kCacheCapacity; + lo.num_shard_bits = 0; // 2^0 shard + lo.strict_capacity_limit = true; + std::shared_ptr cache(NewLRUCache(lo)); + table_options.block_cache = cache; + table_options.flush_block_policy_factory = + std::make_shared(); + + Options options; + options.compression = kSnappyCompression; + options.compression_opts.max_dict_bytes = kMaxDictBytes; + options.compression_opts.max_dict_buffer_bytes = kMaxDictBufferBytes; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + test::StringSink* sink = new test::StringSink(); + std::unique_ptr holder(sink); + std::unique_ptr file_writer(new WritableFileWriter( + std::move(holder), "test_file_name", FileOptions())); + + ImmutableOptions ioptions(options); + MutableCFOptions moptions(options); + InternalKeyComparator ikc(options.comparator); + IntTblPropCollectorFactories int_tbl_prop_collector_factories; + + std::unique_ptr builder(options.table_factory->NewTableBuilder( + TableBuilderOptions(ioptions, moptions, ikc, + &int_tbl_prop_collector_factories, kSnappyCompression, + options.compression_opts, kUnknownColumnFamily, + "test_cf", -1 /* level */), + file_writer.get())); + + std::string key1 = "key1"; + std::string value1(kSizeDummyEntry, '0'); + InternalKey ik1(key1, 0 /* sequnce number */, kTypeValue); + // Adding the first key won't trigger a flush by FlushBlockEveryKeyPolicy + // therefore won't trigger any data block's buffering + builder->Add(ik1.Encode(), value1); + ASSERT_EQ(cache->GetPinnedUsage(), 0 * kSizeDummyEntry); + + std::string key2 = "key2"; + std::string value2(kSizeDummyEntry, '0'); + InternalKey ik2(key2, 1 /* sequnce number */, kTypeValue); + // Adding the second key will trigger a flush of the last data block (the one + // containing key1 and value1) by FlushBlockEveryKeyPolicy and hence trigger + // buffering of the last data block. + builder->Add(ik2.Encode(), value2); + // Cache charging will increase for last buffered data block (the one + // containing key1 and value1) since the buffer limit is not exceeded after + // the buffering and the cache will not be full after this reservation + EXPECT_GE(cache->GetPinnedUsage(), 2 * kSizeDummyEntry); + EXPECT_LT(cache->GetPinnedUsage(), + 2 * kSizeDummyEntry + kMetaDataChargeOverhead); + + std::string key3 = "key3"; + std::string value3 = "val3"; + InternalKey ik3(key3, 2 /* sequnce number */, kTypeValue); + // Adding the third key will trigger a flush of the last data block (the one + // containing key2 and value2) by FlushBlockEveryKeyPolicy and hence trigger + // buffering of the last data block. + builder->Add(ik3.Encode(), value3); + // Cache charging will decrease since the buffer limit is now exceeded + // after the last buffering and EnterUnbuffered() is triggered + EXPECT_EQ(cache->GetPinnedUsage(), 0 * kSizeDummyEntry); + + ASSERT_OK(builder->Finish()); + EXPECT_EQ(cache->GetPinnedUsage(), 0 * kSizeDummyEntry); +} + +TEST_F(ChargeCompressionDictionaryBuildingBufferTest, BasicWithCacheFull) { + constexpr std::size_t kSizeDummyEntry = 256 * 1024; + constexpr std::size_t kMetaDataChargeOverhead = 10000; + // A small kCacheCapacity is chosen so that increase cache charging for + // buffering two data blocks, each containing key1/value1, key2/a big + // value2, will cause cache full + constexpr std::size_t kCacheCapacity = + 1 * kSizeDummyEntry + kSizeDummyEntry / 2; + constexpr std::size_t kMaxDictBytes = 1024; + // A big kMaxDictBufferBytes is chosen so that adding a big key value pair + // (key2, value2) won't exceed the buffer limit + constexpr std::size_t kMaxDictBufferBytes = 1024 * 1024 * 1024; + + // `CacheEntryRoleOptions::charged` is enabled by default for + // CacheEntryRole::kCompressionDictionaryBuildingBuffer + BlockBasedTableOptions table_options; + LRUCacheOptions lo; + lo.capacity = kCacheCapacity; + lo.num_shard_bits = 0; // 2^0 shard + lo.strict_capacity_limit = true; + std::shared_ptr cache(NewLRUCache(lo)); + table_options.block_cache = cache; + table_options.flush_block_policy_factory = + std::make_shared(); + + Options options; + options.compression = kSnappyCompression; + options.compression_opts.max_dict_bytes = kMaxDictBytes; + options.compression_opts.max_dict_buffer_bytes = kMaxDictBufferBytes; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + test::StringSink* sink = new test::StringSink(); + std::unique_ptr holder(sink); + std::unique_ptr file_writer(new WritableFileWriter( + std::move(holder), "test_file_name", FileOptions())); + + ImmutableOptions ioptions(options); + MutableCFOptions moptions(options); + InternalKeyComparator ikc(options.comparator); + IntTblPropCollectorFactories int_tbl_prop_collector_factories; + + std::unique_ptr builder(options.table_factory->NewTableBuilder( + TableBuilderOptions(ioptions, moptions, ikc, + &int_tbl_prop_collector_factories, kSnappyCompression, + options.compression_opts, kUnknownColumnFamily, + "test_cf", -1 /* level */), + file_writer.get())); + + std::string key1 = "key1"; + std::string value1 = "val1"; + InternalKey ik1(key1, 0 /* sequnce number */, kTypeValue); + // Adding the first key won't trigger a flush by FlushBlockEveryKeyPolicy + // therefore won't trigger any data block's buffering + builder->Add(ik1.Encode(), value1); + ASSERT_EQ(cache->GetPinnedUsage(), 0 * kSizeDummyEntry); + + std::string key2 = "key2"; + std::string value2(kSizeDummyEntry, '0'); + InternalKey ik2(key2, 1 /* sequnce number */, kTypeValue); + // Adding the second key will trigger a flush of the last data block (the one + // containing key1 and value1) by FlushBlockEveryKeyPolicy and hence trigger + // buffering of the last data block. + builder->Add(ik2.Encode(), value2); + // Cache charging will increase for the last buffered data block (the one + // containing key1 and value1) since the buffer limit is not exceeded after + // the buffering and the cache will not be full after this reservation + EXPECT_GE(cache->GetPinnedUsage(), 1 * kSizeDummyEntry); + EXPECT_LT(cache->GetPinnedUsage(), + 1 * kSizeDummyEntry + kMetaDataChargeOverhead); + + std::string key3 = "key3"; + std::string value3 = "value3"; + InternalKey ik3(key3, 2 /* sequnce number */, kTypeValue); + // Adding the third key will trigger a flush of the last data block (the one + // containing key2 and value2) by FlushBlockEveryKeyPolicy and hence trigger + // buffering of the last data block. + builder->Add(ik3.Encode(), value3); + // Cache charging will decrease since the cache is now full after + // increasing reservation for the last buffered block and EnterUnbuffered() is + // triggered + EXPECT_EQ(cache->GetPinnedUsage(), 0 * kSizeDummyEntry); + + ASSERT_OK(builder->Finish()); + EXPECT_EQ(cache->GetPinnedUsage(), 0 * kSizeDummyEntry); +} + +class CacheUsageOptionsOverridesTest : public DBTestBase { + public: + CacheUsageOptionsOverridesTest() + : DBTestBase("cache_usage_options_overrides_test", + /*env_do_fsync=*/false) {} +}; + +TEST_F(CacheUsageOptionsOverridesTest, SanitizeAndValidateOptions) { + // To test `cache_usage_options.options_overrides` is sanitized + // where `cache_usage_options.options` is used when there is no entry in + // `cache_usage_options.options_overrides` + Options options; + options.create_if_missing = true; + BlockBasedTableOptions table_options = BlockBasedTableOptions(); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Destroy(options); + Status s = TryReopen(options); + EXPECT_TRUE(s.ok()); + const auto* sanitized_table_options = + options.table_factory->GetOptions(); + const auto sanitized_options_overrides = + sanitized_table_options->cache_usage_options.options_overrides; + EXPECT_EQ(sanitized_options_overrides.size(), kNumCacheEntryRoles); + for (auto options_overrides_iter = sanitized_options_overrides.cbegin(); + options_overrides_iter != sanitized_options_overrides.cend(); + ++options_overrides_iter) { + CacheEntryRoleOptions role_options = options_overrides_iter->second; + CacheEntryRoleOptions default_options = + sanitized_table_options->cache_usage_options.options; + EXPECT_TRUE(role_options == default_options); + } + Destroy(options); + + // To test option validation on unsupported CacheEntryRole + table_options = BlockBasedTableOptions(); + table_options.cache_usage_options.options_overrides.insert( + {CacheEntryRole::kDataBlock, + {/*.charged = */ CacheEntryRoleOptions::Decision::kDisabled}}); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Destroy(options); + s = TryReopen(options); + EXPECT_TRUE(s.IsNotSupported()); + EXPECT_TRUE( + s.ToString().find("Enable/Disable CacheEntryRoleOptions::charged") != + std::string::npos); + EXPECT_TRUE( + s.ToString().find(kCacheEntryRoleToCamelString[static_cast( + CacheEntryRole::kDataBlock)]) != std::string::npos); + Destroy(options); + + // To test option validation on existence of block cache + table_options = BlockBasedTableOptions(); + table_options.no_block_cache = true; + table_options.cache_usage_options.options_overrides.insert( + {CacheEntryRole::kFilterConstruction, + {/*.charged = */ CacheEntryRoleOptions::Decision::kEnabled}}); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Destroy(options); + s = TryReopen(options); + EXPECT_TRUE(s.IsInvalidArgument()); + EXPECT_TRUE(s.ToString().find("Enable CacheEntryRoleOptions::charged") != + std::string::npos); + EXPECT_TRUE( + s.ToString().find(kCacheEntryRoleToCamelString[static_cast( + CacheEntryRole::kFilterConstruction)]) != std::string::npos); + EXPECT_TRUE(s.ToString().find("block cache is disabled") != + std::string::npos); + Destroy(options); +} +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/table/two_level_iterator.cc b/librocksdb-sys/rocksdb/table/two_level_iterator.cc new file mode 100644 index 0000000..4b6634e --- /dev/null +++ b/librocksdb-sys/rocksdb/table/two_level_iterator.cc @@ -0,0 +1,220 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "table/two_level_iterator.h" + +#include "db/pinned_iterators_manager.h" +#include "memory/arena.h" +#include "rocksdb/options.h" +#include "rocksdb/table.h" +#include "table/block_based/block.h" +#include "table/format.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { + +class TwoLevelIndexIterator : public InternalIteratorBase { + public: + explicit TwoLevelIndexIterator( + TwoLevelIteratorState* state, + InternalIteratorBase* first_level_iter); + + ~TwoLevelIndexIterator() override { + first_level_iter_.DeleteIter(false /* is_arena_mode */); + second_level_iter_.DeleteIter(false /* is_arena_mode */); + delete state_; + } + + void Seek(const Slice& target) override; + void SeekForPrev(const Slice& target) override; + void SeekToFirst() override; + void SeekToLast() override; + void Next() override; + void Prev() override; + + bool Valid() const override { return second_level_iter_.Valid(); } + Slice key() const override { + assert(Valid()); + return second_level_iter_.key(); + } + Slice user_key() const override { + assert(Valid()); + return second_level_iter_.user_key(); + } + IndexValue value() const override { + assert(Valid()); + return second_level_iter_.value(); + } + Status status() const override { + if (!first_level_iter_.status().ok()) { + assert(second_level_iter_.iter() == nullptr); + return first_level_iter_.status(); + } else if (second_level_iter_.iter() != nullptr && + !second_level_iter_.status().ok()) { + return second_level_iter_.status(); + } else { + return status_; + } + } + void SetPinnedItersMgr( + PinnedIteratorsManager* /*pinned_iters_mgr*/) override {} + bool IsKeyPinned() const override { return false; } + bool IsValuePinned() const override { return false; } + + private: + void SaveError(const Status& s) { + if (status_.ok() && !s.ok()) status_ = s; + } + void SkipEmptyDataBlocksForward(); + void SkipEmptyDataBlocksBackward(); + void SetSecondLevelIterator(InternalIteratorBase* iter); + void InitDataBlock(); + + TwoLevelIteratorState* state_; + IteratorWrapperBase first_level_iter_; + IteratorWrapperBase second_level_iter_; // May be nullptr + Status status_; + // If second_level_iter is non-nullptr, then "data_block_handle_" holds the + // "index_value" passed to block_function_ to create the second_level_iter. + BlockHandle data_block_handle_; +}; + +TwoLevelIndexIterator::TwoLevelIndexIterator( + TwoLevelIteratorState* state, + InternalIteratorBase* first_level_iter) + : state_(state), first_level_iter_(first_level_iter) {} + +void TwoLevelIndexIterator::Seek(const Slice& target) { + first_level_iter_.Seek(target); + + InitDataBlock(); + if (second_level_iter_.iter() != nullptr) { + second_level_iter_.Seek(target); + } + SkipEmptyDataBlocksForward(); +} + +void TwoLevelIndexIterator::SeekForPrev(const Slice& target) { + first_level_iter_.Seek(target); + InitDataBlock(); + if (second_level_iter_.iter() != nullptr) { + second_level_iter_.SeekForPrev(target); + } + if (!Valid()) { + if (!first_level_iter_.Valid() && first_level_iter_.status().ok()) { + first_level_iter_.SeekToLast(); + InitDataBlock(); + if (second_level_iter_.iter() != nullptr) { + second_level_iter_.SeekForPrev(target); + } + } + SkipEmptyDataBlocksBackward(); + } +} + +void TwoLevelIndexIterator::SeekToFirst() { + first_level_iter_.SeekToFirst(); + InitDataBlock(); + if (second_level_iter_.iter() != nullptr) { + second_level_iter_.SeekToFirst(); + } + SkipEmptyDataBlocksForward(); +} + +void TwoLevelIndexIterator::SeekToLast() { + first_level_iter_.SeekToLast(); + InitDataBlock(); + if (second_level_iter_.iter() != nullptr) { + second_level_iter_.SeekToLast(); + } + SkipEmptyDataBlocksBackward(); +} + +void TwoLevelIndexIterator::Next() { + assert(Valid()); + second_level_iter_.Next(); + SkipEmptyDataBlocksForward(); +} + +void TwoLevelIndexIterator::Prev() { + assert(Valid()); + second_level_iter_.Prev(); + SkipEmptyDataBlocksBackward(); +} + +void TwoLevelIndexIterator::SkipEmptyDataBlocksForward() { + while (second_level_iter_.iter() == nullptr || + (!second_level_iter_.Valid() && second_level_iter_.status().ok())) { + // Move to next block + if (!first_level_iter_.Valid()) { + SetSecondLevelIterator(nullptr); + return; + } + first_level_iter_.Next(); + InitDataBlock(); + if (second_level_iter_.iter() != nullptr) { + second_level_iter_.SeekToFirst(); + } + } +} + +void TwoLevelIndexIterator::SkipEmptyDataBlocksBackward() { + while (second_level_iter_.iter() == nullptr || + (!second_level_iter_.Valid() && second_level_iter_.status().ok())) { + // Move to next block + if (!first_level_iter_.Valid()) { + SetSecondLevelIterator(nullptr); + return; + } + first_level_iter_.Prev(); + InitDataBlock(); + if (second_level_iter_.iter() != nullptr) { + second_level_iter_.SeekToLast(); + } + } +} + +void TwoLevelIndexIterator::SetSecondLevelIterator( + InternalIteratorBase* iter) { + InternalIteratorBase* old_iter = second_level_iter_.Set(iter); + delete old_iter; +} + +void TwoLevelIndexIterator::InitDataBlock() { + if (!first_level_iter_.Valid()) { + SetSecondLevelIterator(nullptr); + } else { + BlockHandle handle = first_level_iter_.value().handle; + if (second_level_iter_.iter() != nullptr && + !second_level_iter_.status().IsIncomplete() && + handle.offset() == data_block_handle_.offset()) { + // second_level_iter is already constructed with this iterator, so + // no need to change anything + } else { + InternalIteratorBase* iter = + state_->NewSecondaryIterator(handle); + data_block_handle_ = handle; + SetSecondLevelIterator(iter); + if (iter == nullptr) { + status_ = Status::Corruption("Missing block for partition " + + handle.ToString()); + } + } + } +} + +} // namespace + +InternalIteratorBase* NewTwoLevelIterator( + TwoLevelIteratorState* state, + InternalIteratorBase* first_level_iter) { + return new TwoLevelIndexIterator(state, first_level_iter); +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/two_level_iterator.h b/librocksdb-sys/rocksdb/table/two_level_iterator.h new file mode 100644 index 0000000..1fed934 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/two_level_iterator.h @@ -0,0 +1,43 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include "rocksdb/env.h" +#include "rocksdb/iterator.h" +#include "table/iterator_wrapper.h" + +namespace ROCKSDB_NAMESPACE { + +struct ReadOptions; +class InternalKeyComparator; + +// TwoLevelIteratorState expects iterators are not created using the arena +struct TwoLevelIteratorState { + TwoLevelIteratorState() {} + + virtual ~TwoLevelIteratorState() {} + virtual InternalIteratorBase* NewSecondaryIterator( + const BlockHandle& handle) = 0; +}; + +// Return a new two level iterator. A two-level iterator contains an +// index iterator whose values point to a sequence of blocks where +// each block is itself a sequence of key,value pairs. The returned +// two-level iterator yields the concatenation of all key/value pairs +// in the sequence of blocks. Takes ownership of "index_iter" and +// will delete it when no longer needed. +// +// Uses a supplied function to convert an index_iter value into +// an iterator over the contents of the corresponding block. +// Note: this function expects first_level_iter was not created using the arena +extern InternalIteratorBase* NewTwoLevelIterator( + TwoLevelIteratorState* state, + InternalIteratorBase* first_level_iter); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/unique_id.cc b/librocksdb-sys/rocksdb/table/unique_id.cc new file mode 100644 index 0000000..fcdd756 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/unique_id.cc @@ -0,0 +1,223 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include + +#include "table/unique_id_impl.h" +#include "util/coding_lean.h" +#include "util/hash.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +std::string EncodeSessionId(uint64_t upper, uint64_t lower) { + std::string db_session_id(20U, '\0'); + char *buf = &db_session_id[0]; + // Preserving `lower` is slightly tricky. 36^12 is slightly more than + // 62 bits, so we use 12 chars plus the bottom two bits of one more. + // (A tiny fraction of 20 digit strings go unused.) + uint64_t a = (upper << 2) | (lower >> 62); + uint64_t b = lower & (UINT64_MAX >> 2); + PutBaseChars<36>(&buf, 8, a, /*uppercase*/ true); + PutBaseChars<36>(&buf, 12, b, /*uppercase*/ true); + assert(buf == &db_session_id.back() + 1); + return db_session_id; +} + +Status DecodeSessionId(const std::string &db_session_id, uint64_t *upper, + uint64_t *lower) { + const size_t len = db_session_id.size(); + if (len == 0) { + return Status::NotSupported("Missing db_session_id"); + } + // Anything from 13 to 24 chars is reasonable. We don't have to limit to + // exactly 20. + if (len < 13) { + return Status::NotSupported("Too short db_session_id"); + } + if (len > 24) { + return Status::NotSupported("Too long db_session_id"); + } + uint64_t a = 0, b = 0; + const char *buf = &db_session_id.front(); + bool success = ParseBaseChars<36>(&buf, len - 12U, &a); + if (!success) { + return Status::NotSupported("Bad digit in db_session_id"); + } + success = ParseBaseChars<36>(&buf, 12U, &b); + if (!success) { + return Status::NotSupported("Bad digit in db_session_id"); + } + assert(buf == &db_session_id.back() + 1); + *upper = a >> 2; + *lower = (b & (UINT64_MAX >> 2)) | (a << 62); + return Status::OK(); +} + +Status GetSstInternalUniqueId(const std::string &db_id, + const std::string &db_session_id, + uint64_t file_number, UniqueIdPtr out, + bool force) { + if (!force) { + if (db_id.empty()) { + return Status::NotSupported("Missing db_id"); + } + if (file_number == 0) { + return Status::NotSupported("Missing or bad file number"); + } + if (db_session_id.empty()) { + return Status::NotSupported("Missing db_session_id"); + } + } + uint64_t session_upper = 0; // Assignment to appease clang-analyze + uint64_t session_lower = 0; // Assignment to appease clang-analyze + { + Status s = DecodeSessionId(db_session_id, &session_upper, &session_lower); + if (!s.ok()) { + if (!force) { + return s; + } else { + // A reasonable fallback in case malformed + Hash2x64(db_session_id.data(), db_session_id.size(), &session_upper, + &session_lower); + if (session_lower == 0) { + session_lower = session_upper | 1; + } + } + } + } + + // Exactly preserve session lower to ensure that session ids generated + // during the same process lifetime are guaranteed unique. + // DBImpl also guarantees (in recent versions) that this is not zero, + // so that we can guarantee unique ID is never all zeros. (Can't assert + // that here because of testing and old versions.) + // We put this first in anticipation of matching a small-ish set of cache + // key prefixes to cover entries relevant to any DB. + out.ptr[0] = session_lower; + + // Hash the session upper (~39 bits entropy) and DB id (120+ bits entropy) + // for very high global uniqueness entropy. + // (It is possible that many DBs descended from one common DB id are copied + // around and proliferate, in which case session id is critical, but it is + // more common for different DBs to have different DB ids.) + uint64_t db_a, db_b; + Hash2x64(db_id.data(), db_id.size(), session_upper, &db_a, &db_b); + + // Xor in file number for guaranteed uniqueness by file number for a given + // session and DB id. (Xor slightly better than + here. See + // https://github.com/pdillinger/unique_id ) + out.ptr[1] = db_a ^ file_number; + + // Extra (optional) global uniqueness + if (out.extended) { + out.ptr[2] = db_b; + } + + return Status::OK(); +} + +namespace { +// For InternalUniqueIdToExternal / ExternalUniqueIdToInternal we want all +// zeros in first 128 bits to map to itself, so that excluding zero in +// internal IDs (session_lower != 0 above) does the same for external IDs. +// These values are meaningless except for making that work. +constexpr uint64_t kHiOffsetForZero = 17391078804906429400U; +constexpr uint64_t kLoOffsetForZero = 6417269962128484497U; +} // namespace + +void InternalUniqueIdToExternal(UniqueIdPtr in_out) { + uint64_t hi, lo; + BijectiveHash2x64(in_out.ptr[1] + kHiOffsetForZero, + in_out.ptr[0] + kLoOffsetForZero, &hi, &lo); + in_out.ptr[0] = lo; + in_out.ptr[1] = hi; + if (in_out.extended) { + in_out.ptr[2] += lo + hi; + } +} + +void ExternalUniqueIdToInternal(UniqueIdPtr in_out) { + uint64_t lo = in_out.ptr[0]; + uint64_t hi = in_out.ptr[1]; + if (in_out.extended) { + in_out.ptr[2] -= lo + hi; + } + BijectiveUnhash2x64(hi, lo, &hi, &lo); + in_out.ptr[0] = lo - kLoOffsetForZero; + in_out.ptr[1] = hi - kHiOffsetForZero; +} + +std::string EncodeUniqueIdBytes(UniqueIdPtr in) { + std::string ret(in.extended ? 24U : 16U, '\0'); + EncodeFixed64(&ret[0], in.ptr[0]); + EncodeFixed64(&ret[8], in.ptr[1]); + if (in.extended) { + EncodeFixed64(&ret[16], in.ptr[2]); + } + return ret; +} + +Status DecodeUniqueIdBytes(const std::string &unique_id, UniqueIdPtr out) { + if (unique_id.size() != (out.extended ? 24 : 16)) { + return Status::NotSupported("Not a valid unique_id"); + } + const char *buf = &unique_id.front(); + out.ptr[0] = DecodeFixed64(&buf[0]); + out.ptr[1] = DecodeFixed64(&buf[8]); + if (out.extended) { + out.ptr[2] = DecodeFixed64(&buf[16]); + } + return Status::OK(); +} + +template +Status GetUniqueIdFromTablePropertiesHelper(const TableProperties &props, + std::string *out_id) { + ID tmp{}; + Status s = GetSstInternalUniqueId(props.db_id, props.db_session_id, + props.orig_file_number, &tmp); + if (s.ok()) { + InternalUniqueIdToExternal(&tmp); + *out_id = EncodeUniqueIdBytes(&tmp); + } else { + out_id->clear(); + } + return s; +} + +Status GetExtendedUniqueIdFromTableProperties(const TableProperties &props, + std::string *out_id) { + return GetUniqueIdFromTablePropertiesHelper(props, out_id); +} + +Status GetUniqueIdFromTableProperties(const TableProperties &props, + std::string *out_id) { + return GetUniqueIdFromTablePropertiesHelper(props, out_id); +} + +std::string UniqueIdToHumanString(const std::string &id) { + // Not so efficient, but that's OK + std::string str = Slice(id).ToString(/*hex*/ true); + for (size_t i = 16; i < str.size(); i += 17) { + str.insert(i, "-"); + } + return str; +} + +std::string InternalUniqueIdToHumanString(UniqueIdPtr in) { + std::string str = "{"; + str += std::to_string(in.ptr[0]); + str += ","; + str += std::to_string(in.ptr[1]); + if (in.extended) { + str += ","; + str += std::to_string(in.ptr[2]); + } + str += "}"; + return str; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/table/unique_id_impl.h b/librocksdb-sys/rocksdb/table/unique_id_impl.h new file mode 100644 index 0000000..6e3dc62 --- /dev/null +++ b/librocksdb-sys/rocksdb/table/unique_id_impl.h @@ -0,0 +1,93 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include "rocksdb/unique_id.h" + +namespace ROCKSDB_NAMESPACE { + +// Standard size unique ID, good enough for almost all practical purposes +using UniqueId64x2 = std::array; + +// Value never used as an actual unique ID so can be used for "null" +constexpr UniqueId64x2 kNullUniqueId64x2 = {}; + +// Extended size unique ID, for extra certainty of uniqueness among SST files +// spanning many hosts over a long time (rarely if ever needed) +using UniqueId64x3 = std::array; + +// Value never used as an actual unique ID so can be used for "null" +constexpr UniqueId64x3 kNullUniqueId64x3 = {}; + +// Dynamic pointer wrapper for one of the two above +struct UniqueIdPtr { + uint64_t *ptr = nullptr; + bool extended = false; + + /*implicit*/ UniqueIdPtr(UniqueId64x2 *id) { + ptr = (*id).data(); + extended = false; + } + /*implicit*/ UniqueIdPtr(UniqueId64x3 *id) { + ptr = (*id).data(); + extended = true; + } +}; + +// Helper for GetUniqueIdFromTableProperties. This function can also be used +// for temporary ids for files without sufficient information in table +// properties. The internal unique id is more structured than the public +// unique id, so can be manipulated in more ways but very carefully. +// These must be long term stable to ensure GetUniqueIdFromTableProperties +// is long term stable. +Status GetSstInternalUniqueId(const std::string &db_id, + const std::string &db_session_id, + uint64_t file_number, UniqueIdPtr out, + bool force = false); + +// Helper for GetUniqueIdFromTableProperties. External unique ids go through +// this extra hashing layer so that prefixes of the unique id have predictable +// "full" entropy. This hashing layer is 1-to-1 on the first 128 bits and on +// the full 192 bits. +// This transformation must be long term stable to ensure +// GetUniqueIdFromTableProperties is long term stable. +void InternalUniqueIdToExternal(UniqueIdPtr in_out); + +// Reverse of InternalUniqueIdToExternal mostly for testing purposes +// (demonstrably 1-to-1 on the first 128 bits and on the full 192 bits). +void ExternalUniqueIdToInternal(UniqueIdPtr in_out); + +// Convert numerical format to byte format for public API +std::string EncodeUniqueIdBytes(UniqueIdPtr in); + +// Reverse of EncodeUniqueIdBytes. +Status DecodeUniqueIdBytes(const std::string &unique_id, UniqueIdPtr out); + +// For presenting internal IDs for debugging purposes. Visually distinct from +// UniqueIdToHumanString for external IDs. +std::string InternalUniqueIdToHumanString(UniqueIdPtr in); + +// Reformat a random value down to our "DB session id" format, +// which is intended to be compact and friendly for use in file names. +// `lower` is fully preserved and data is lost from `upper`. +// +// Detail: Encoded into 20 chars in base-36 ([0-9A-Z]), which is ~103 bits of +// entropy, which is enough to expect no collisions across a billion servers +// each opening DBs a million times (~2^50). Benefits vs. RFC-4122 unique id: +// * Save ~ dozen bytes per SST file +// * Shorter shared backup file names (some platforms have low limits) +// * Visually distinct from DB id format (usually RFC-4122) +std::string EncodeSessionId(uint64_t upper, uint64_t lower); + +// Reverse of EncodeSessionId. Returns NotSupported on error rather than +// Corruption because non-standard session IDs should be allowed with degraded +// functionality. +Status DecodeSessionId(const std::string &db_session_id, uint64_t *upper, + uint64_t *lower); + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/test_util/mock_time_env.cc b/librocksdb-sys/rocksdb/test_util/mock_time_env.cc new file mode 100644 index 0000000..23888e6 --- /dev/null +++ b/librocksdb-sys/rocksdb/test_util/mock_time_env.cc @@ -0,0 +1,38 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "test_util/mock_time_env.h" + +#include "test_util/sync_point.h" + +namespace ROCKSDB_NAMESPACE { + +// TODO: this is a workaround for the different behavior on different platform +// for timedwait timeout. Ideally timedwait API should be moved to env. +// details: PR #7101. +void MockSystemClock::InstallTimedWaitFixCallback() { +#ifndef NDEBUG + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +#ifdef OS_MACOSX + // This is an alternate way (vs. SpecialEnv) of dealing with the fact + // that on some platforms, pthread_cond_timedwait does not appear to + // release the lock for other threads to operate if the deadline time + // is already passed. (TimedWait calls are currently a bad abstraction + // because the deadline parameter is usually computed from Env time, + // but is interpreted in real clock time.) + SyncPoint::GetInstance()->SetCallBack( + "InstrumentedCondVar::TimedWaitInternal", [&](void* arg) { + uint64_t time_us = *reinterpret_cast(arg); + if (time_us < this->RealNowMicros()) { + *reinterpret_cast(arg) = this->RealNowMicros() + 1000; + } + }); +#endif // OS_MACOSX + SyncPoint::GetInstance()->EnableProcessing(); +#endif // !NDEBUG +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/test_util/mock_time_env.h b/librocksdb-sys/rocksdb/test_util/mock_time_env.h new file mode 100644 index 0000000..7834368 --- /dev/null +++ b/librocksdb-sys/rocksdb/test_util/mock_time_env.h @@ -0,0 +1,78 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include + +#include "rocksdb/system_clock.h" + +namespace ROCKSDB_NAMESPACE { + +// NOTE: SpecialEnv offers most of this functionality, along with hooks +// for safe DB behavior under a mock time environment, so should be used +// instead of MockSystemClock for DB tests. +class MockSystemClock : public SystemClockWrapper { + public: + explicit MockSystemClock(const std::shared_ptr& base) + : SystemClockWrapper(base) {} + + static const char* kClassName() { return "MockSystemClock"; } + const char* Name() const override { return kClassName(); } + virtual Status GetCurrentTime(int64_t* time_sec) override { + assert(time_sec != nullptr); + *time_sec = static_cast(current_time_us_ / kMicrosInSecond); + return Status::OK(); + } + + virtual uint64_t NowSeconds() { return current_time_us_ / kMicrosInSecond; } + + virtual uint64_t NowMicros() override { return current_time_us_; } + + virtual uint64_t NowNanos() override { + assert(current_time_us_ <= std::numeric_limits::max() / 1000); + return current_time_us_ * 1000; + } + + uint64_t RealNowMicros() { return target_->NowMicros(); } + + void SetCurrentTime(uint64_t time_sec) { + assert(time_sec < std::numeric_limits::max() / kMicrosInSecond); + assert(time_sec * kMicrosInSecond >= current_time_us_); + current_time_us_ = time_sec * kMicrosInSecond; + } + + // It's a fake sleep that just updates the Env current time, which is similar + // to `NoSleepEnv.SleepForMicroseconds()` and + // `SpecialEnv.MockSleepForMicroseconds()`. + // It's also similar to `set_current_time()`, which takes an absolute time in + // seconds, vs. this one takes the sleep in microseconds. + // Note: Not thread safe. + void SleepForMicroseconds(int micros) override { + assert(micros >= 0); + assert(current_time_us_ + static_cast(micros) >= + current_time_us_); + current_time_us_.fetch_add(micros); + } + + void MockSleepForSeconds(int seconds) { + assert(seconds >= 0); + uint64_t micros = static_cast(seconds) * kMicrosInSecond; + assert(current_time_us_ + micros >= current_time_us_); + current_time_us_.fetch_add(micros); + } + + // TODO: this is a workaround for the different behavior on different platform + // for timedwait timeout. Ideally timedwait API should be moved to env. + // details: PR #7101. + void InstallTimedWaitFixCallback(); + + private: + std::atomic current_time_us_{0}; + static constexpr uint64_t kMicrosInSecond = 1000U * 1000U; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/test_util/secondary_cache_test_util.cc b/librocksdb-sys/rocksdb/test_util/secondary_cache_test_util.cc new file mode 100644 index 0000000..1c62dc4 --- /dev/null +++ b/librocksdb-sys/rocksdb/test_util/secondary_cache_test_util.cc @@ -0,0 +1,96 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "test_util/secondary_cache_test_util.h" + +#include + +namespace ROCKSDB_NAMESPACE { + +namespace secondary_cache_test_util { + +namespace { +using TestItem = WithCacheType::TestItem; + +size_t SizeCallback(Cache::ObjectPtr obj) { + return static_cast(obj)->Size(); +} + +Status SaveToCallback(Cache::ObjectPtr from_obj, size_t from_offset, + size_t length, char* out) { + auto item = static_cast(from_obj); + const char* buf = item->Buf(); + EXPECT_EQ(length, item->Size()); + EXPECT_EQ(from_offset, 0); + memcpy(out, buf, length); + return Status::OK(); +} + +void DeletionCallback(Cache::ObjectPtr obj, MemoryAllocator* /*alloc*/) { + delete static_cast(obj); +} + +Status SaveToCallbackFail(Cache::ObjectPtr /*obj*/, size_t /*offset*/, + size_t /*size*/, char* /*out*/) { + return Status::NotSupported(); +} + +Status CreateCallback(const Slice& data, Cache::CreateContext* context, + MemoryAllocator* /*allocator*/, Cache::ObjectPtr* out_obj, + size_t* out_charge) { + auto t = static_cast(context); + if (t->fail_create_) { + return Status::NotSupported(); + } + *out_obj = new TestItem(data.data(), data.size()); + *out_charge = data.size(); + return Status::OK(); +} + +// If helpers without_secondary are provided, returns helpers with secondary +// support. If not provided, returns helpers without secondary support. +auto GenerateHelpersByRole( + const std::array* + without_secondary, + bool fail) { + std::array a; + for (uint32_t i = 0; i < kNumCacheEntryRoles; ++i) { + if (without_secondary) { + a[i] = + Cache::CacheItemHelper{static_cast(i), + &DeletionCallback, + &SizeCallback, + fail ? &SaveToCallbackFail : &SaveToCallback, + &CreateCallback, + &(*without_secondary)[i]}; + } else { + a[i] = Cache::CacheItemHelper{static_cast(i), + &DeletionCallback}; + } + } + return a; +} +} // namespace + +const Cache::CacheItemHelper* WithCacheType::GetHelper( + CacheEntryRole r, bool secondary_compatible, bool fail) { + static const std::array + without_secondary = GenerateHelpersByRole(nullptr, false); + static const std::array + with_secondary = GenerateHelpersByRole(&without_secondary, false); + static const std::array + with_secondary_fail = GenerateHelpersByRole(&without_secondary, true); + return &(fail ? with_secondary_fail + : secondary_compatible ? with_secondary + : without_secondary)[static_cast(r)]; +} + +const Cache::CacheItemHelper* WithCacheType::GetHelperFail(CacheEntryRole r) { + return GetHelper(r, true, true); +} + +} // namespace secondary_cache_test_util + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/test_util/secondary_cache_test_util.h b/librocksdb-sys/rocksdb/test_util/secondary_cache_test_util.h new file mode 100644 index 0000000..1cfb454 --- /dev/null +++ b/librocksdb-sys/rocksdb/test_util/secondary_cache_test_util.h @@ -0,0 +1,119 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include + +#include "rocksdb/advanced_cache.h" + +namespace ROCKSDB_NAMESPACE { +namespace secondary_cache_test_util { + +struct TestCreateContext : public Cache::CreateContext { + void SetFailCreate(bool fail) { fail_create_ = fail; } + + bool fail_create_ = false; +}; + +class WithCacheType : public TestCreateContext { + public: + WithCacheType() {} + virtual ~WithCacheType() {} + + class TestItem { + public: + TestItem(const char* buf, size_t size) : buf_(new char[size]), size_(size) { + memcpy(buf_.get(), buf, size); + } + ~TestItem() = default; + + char* Buf() { return buf_.get(); } + [[nodiscard]] size_t Size() const { return size_; } + std::string ToString() { return std::string(Buf(), Size()); } + + private: + std::unique_ptr buf_; + size_t size_; + }; + + static constexpr auto kLRU = "lru"; + static constexpr auto kHyperClock = "hyper_clock"; + + // For options other than capacity + size_t estimated_value_size_ = 1; + + virtual const std::string& Type() = 0; + + std::shared_ptr NewCache( + size_t capacity, + std::function modify_opts_fn = {}) { + const auto& type = Type(); + if (type == kLRU) { + LRUCacheOptions lru_opts; + lru_opts.capacity = capacity; + lru_opts.hash_seed = 0; // deterministic tests + if (modify_opts_fn) { + modify_opts_fn(lru_opts); + } + return lru_opts.MakeSharedCache(); + } + if (type == kHyperClock) { + HyperClockCacheOptions hc_opts{capacity, estimated_value_size_}; + hc_opts.hash_seed = 0; // deterministic tests + if (modify_opts_fn) { + modify_opts_fn(hc_opts); + } + return hc_opts.MakeSharedCache(); + } + assert(false); + return nullptr; + } + + std::shared_ptr NewCache( + size_t capacity, int num_shard_bits, bool strict_capacity_limit, + CacheMetadataChargePolicy charge_policy = kDontChargeCacheMetadata) { + return NewCache(capacity, [=](ShardedCacheOptions& opts) { + opts.num_shard_bits = num_shard_bits; + opts.strict_capacity_limit = strict_capacity_limit; + opts.metadata_charge_policy = charge_policy; + }); + } + + std::shared_ptr NewCache( + size_t capacity, int num_shard_bits, bool strict_capacity_limit, + std::shared_ptr secondary_cache) { + return NewCache(capacity, [=](ShardedCacheOptions& opts) { + opts.num_shard_bits = num_shard_bits; + opts.strict_capacity_limit = strict_capacity_limit; + opts.metadata_charge_policy = kDontChargeCacheMetadata; + opts.secondary_cache = secondary_cache; + }); + } + + static const Cache::CacheItemHelper* GetHelper( + CacheEntryRole r = CacheEntryRole::kDataBlock, + bool secondary_compatible = true, bool fail = false); + + static const Cache::CacheItemHelper* GetHelperFail( + CacheEntryRole r = CacheEntryRole::kDataBlock); +}; + +class WithCacheTypeParam : public WithCacheType, + public testing::WithParamInterface { + const std::string& Type() override { return GetParam(); } +}; + +constexpr auto kLRU = WithCacheType::kLRU; +constexpr auto kHyperClock = WithCacheType::kHyperClock; + +inline auto GetTestingCacheTypes() { + return testing::Values(std::string(kLRU), std::string(kHyperClock)); +} + +} // namespace secondary_cache_test_util +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/test_util/sync_point.cc b/librocksdb-sys/rocksdb/test_util/sync_point.cc new file mode 100644 index 0000000..bec02d4 --- /dev/null +++ b/librocksdb-sys/rocksdb/test_util/sync_point.cc @@ -0,0 +1,82 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "test_util/sync_point.h" + +#include + +#include "test_util/sync_point_impl.h" + +std::vector rocksdb_kill_exclude_prefixes; + +#ifndef NDEBUG +namespace ROCKSDB_NAMESPACE { + +SyncPoint* SyncPoint::GetInstance() { + static SyncPoint sync_point; + return &sync_point; +} + +SyncPoint::SyncPoint() : impl_(new Data) {} + +SyncPoint::~SyncPoint() { delete impl_; } + +void SyncPoint::LoadDependency(const std::vector& dependencies) { + impl_->LoadDependency(dependencies); +} + +void SyncPoint::LoadDependencyAndMarkers( + const std::vector& dependencies, + const std::vector& markers) { + impl_->LoadDependencyAndMarkers(dependencies, markers); +} + +void SyncPoint::SetCallBack(const std::string& point, + const std::function& callback) { + impl_->SetCallBack(point, callback); +} + +void SyncPoint::ClearCallBack(const std::string& point) { + impl_->ClearCallBack(point); +} + +void SyncPoint::ClearAllCallBacks() { impl_->ClearAllCallBacks(); } + +void SyncPoint::EnableProcessing() { impl_->EnableProcessing(); } + +void SyncPoint::DisableProcessing() { impl_->DisableProcessing(); } + +void SyncPoint::ClearTrace() { impl_->ClearTrace(); } + +void SyncPoint::Process(const Slice& point, void* cb_arg) { + impl_->Process(point, cb_arg); +} + +} // namespace ROCKSDB_NAMESPACE +#endif // NDEBUG + +namespace ROCKSDB_NAMESPACE { +void SetupSyncPointsToMockDirectIO() { +#if !defined(NDEBUG) && !defined(OS_MACOSX) && !defined(OS_WIN) && \ + !defined(OS_SOLARIS) && !defined(OS_AIX) && !defined(OS_OPENBSD) + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "NewWritableFile:O_DIRECT", [&](void* arg) { + int* val = static_cast(arg); + *val &= ~O_DIRECT; + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "NewRandomAccessFile:O_DIRECT", [&](void* arg) { + int* val = static_cast(arg); + *val &= ~O_DIRECT; + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( + "NewSequentialFile:O_DIRECT", [&](void* arg) { + int* val = static_cast(arg); + *val &= ~O_DIRECT; + }); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); +#endif +} +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/test_util/sync_point.h b/librocksdb-sys/rocksdb/test_util/sync_point.h new file mode 100644 index 0000000..6022073 --- /dev/null +++ b/librocksdb-sys/rocksdb/test_util/sync_point.h @@ -0,0 +1,182 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/slice.h" + +#ifdef NDEBUG +// empty in release build +#define TEST_KILL_RANDOM_WITH_WEIGHT(kill_point, rocksdb_kill_odds_weight) +#define TEST_KILL_RANDOM(kill_point) +#else + +namespace ROCKSDB_NAMESPACE { + +// To avoid crashing always at some frequently executed codepaths (during +// kill random test), use this factor to reduce odds +#define REDUCE_ODDS 2 +#define REDUCE_ODDS2 4 + +// A class used to pass when a kill point is reached. +struct KillPoint { + public: + // This is only set from db_stress.cc and for testing only. + // If non-zero, kill at various points in source code with probability 1/this + int rocksdb_kill_odds = 0; + // If kill point has a prefix on this list, will skip killing. + std::vector rocksdb_kill_exclude_prefixes; + // Kill the process with probability 1/odds for testing. + void TestKillRandom(std::string kill_point, int odds, + const std::string& srcfile, int srcline); + + static KillPoint* GetInstance(); +}; + +#define TEST_KILL_RANDOM_WITH_WEIGHT(kill_point, rocksdb_kill_odds_weight) \ + { \ + KillPoint::GetInstance()->TestKillRandom( \ + kill_point, rocksdb_kill_odds_weight, __FILE__, __LINE__); \ + } +#define TEST_KILL_RANDOM(kill_point) TEST_KILL_RANDOM_WITH_WEIGHT(kill_point, 1) +} // namespace ROCKSDB_NAMESPACE + +#endif + +#ifdef NDEBUG +#define TEST_SYNC_POINT(x) +#define TEST_IDX_SYNC_POINT(x, index) +#define TEST_SYNC_POINT_CALLBACK(x, y) +#define INIT_SYNC_POINT_SINGLETONS() +#else + +namespace ROCKSDB_NAMESPACE { + +// This class provides facility to reproduce race conditions deterministically +// in unit tests. +// Developer could specify sync points in the codebase via TEST_SYNC_POINT. +// Each sync point represents a position in the execution stream of a thread. +// In the unit test, 'Happens After' relationship among sync points could be +// setup via SyncPoint::LoadDependency, to reproduce a desired interleave of +// threads execution. +// Refer to (DBTest,TransactionLogIteratorRace), for an example use case. + +class SyncPoint { + public: + static SyncPoint* GetInstance(); + + SyncPoint(const SyncPoint&) = delete; + SyncPoint& operator=(const SyncPoint&) = delete; + ~SyncPoint(); + + struct SyncPointPair { + std::string predecessor; + std::string successor; + }; + + // call once at the beginning of a test to setup the dependency between + // sync points. Specifically, execution will not be allowed to proceed past + // each successor until execution has reached the corresponding predecessor, + // in any thread. + void LoadDependency(const std::vector& dependencies); + + // call once at the beginning of a test to setup the dependency between + // sync points and setup markers indicating the successor is only enabled + // when it is processed on the same thread as the predecessor. + // When adding a marker, it implicitly adds a dependency for the marker pair. + void LoadDependencyAndMarkers(const std::vector& dependencies, + const std::vector& markers); + + // The argument to the callback is passed through from + // TEST_SYNC_POINT_CALLBACK(); nullptr if TEST_SYNC_POINT or + // TEST_IDX_SYNC_POINT was used. + void SetCallBack(const std::string& point, + const std::function& callback); + + // Clear callback function by point + void ClearCallBack(const std::string& point); + + // Clear all call back functions. + void ClearAllCallBacks(); + + // enable sync point processing (disabled on startup) + void EnableProcessing(); + + // disable sync point processing + void DisableProcessing(); + + // remove the execution trace of all sync points + void ClearTrace(); + + // triggered by TEST_SYNC_POINT, blocking execution until all predecessors + // are executed. + // And/or call registered callback function, with argument `cb_arg` + void Process(const Slice& point, void* cb_arg = nullptr); + + // template gets length of const string at compile time, + // avoiding strlen() at runtime + template + void Process(const char (&point)[kLen], void* cb_arg = nullptr) { + static_assert(kLen > 0, "Must not be empty"); + assert(point[kLen - 1] == '\0'); + Process(Slice(point, kLen - 1), cb_arg); + } + + // TODO: it might be useful to provide a function that blocks until all + // sync points are cleared. + + // We want this to be public so we can + // subclass the implementation + struct Data; + + private: + // Singleton + SyncPoint(); + Data* impl_; +}; + +// Sets up sync points to mock direct IO instead of actually issuing direct IO +// to the file system. +void SetupSyncPointsToMockDirectIO(); +} // namespace ROCKSDB_NAMESPACE + +// Use TEST_SYNC_POINT to specify sync points inside code base. +// Sync points can have happens-after dependency on other sync points, +// configured at runtime via SyncPoint::LoadDependency. This could be +// utilized to re-produce race conditions between threads. +// See TransactionLogIteratorRace in db_test.cc for an example use case. +// TEST_SYNC_POINT is no op in release build. +#define TEST_SYNC_POINT(x) \ + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->Process(x) +#define TEST_IDX_SYNC_POINT(x, index) \ + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->Process(x + \ + std::to_string(index)) +#define TEST_SYNC_POINT_CALLBACK(x, y) \ + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->Process(x, y) +#define INIT_SYNC_POINT_SINGLETONS() \ + (void)ROCKSDB_NAMESPACE::SyncPoint::GetInstance(); +#endif // NDEBUG + +// Callback sync point for any read IO errors that should be ignored by +// the fault injection framework +// Disable in release mode +#ifdef NDEBUG +#define IGNORE_STATUS_IF_ERROR(_status_) +#else +#define IGNORE_STATUS_IF_ERROR(_status_) \ + { \ + if (!_status_.ok()) { \ + TEST_SYNC_POINT("FaultInjectionIgnoreError"); \ + } \ + } +#endif // NDEBUG diff --git a/librocksdb-sys/rocksdb/test_util/sync_point_impl.cc b/librocksdb-sys/rocksdb/test_util/sync_point_impl.cc new file mode 100644 index 0000000..2a4bd3c --- /dev/null +++ b/librocksdb-sys/rocksdb/test_util/sync_point_impl.cc @@ -0,0 +1,152 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "test_util/sync_point_impl.h" + +#ifndef NDEBUG +namespace ROCKSDB_NAMESPACE { +KillPoint* KillPoint::GetInstance() { + static KillPoint kp; + return &kp; +} + +void KillPoint::TestKillRandom(std::string kill_point, int odds_weight, + const std::string& srcfile, int srcline) { + if (rocksdb_kill_odds <= 0) { + return; + } + int odds = rocksdb_kill_odds * odds_weight; + for (auto& p : rocksdb_kill_exclude_prefixes) { + if (kill_point.substr(0, p.length()) == p) { + return; + } + } + + assert(odds > 0); + if (odds % 7 == 0) { + // class Random uses multiplier 16807, which is 7^5. If odds are + // multiplier of 7, there might be limited values generated. + odds++; + } + auto* r = Random::GetTLSInstance(); + bool crash = r->OneIn(odds); + if (crash) { + port::Crash(srcfile, srcline); + } +} + +void SyncPoint::Data::LoadDependency( + const std::vector& dependencies) { + std::lock_guard lock(mutex_); + successors_.clear(); + predecessors_.clear(); + cleared_points_.clear(); + for (const auto& dependency : dependencies) { + successors_[dependency.predecessor].push_back(dependency.successor); + predecessors_[dependency.successor].push_back(dependency.predecessor); + point_filter_.Add(dependency.successor); + point_filter_.Add(dependency.predecessor); + } + cv_.notify_all(); +} + +void SyncPoint::Data::LoadDependencyAndMarkers( + const std::vector& dependencies, + const std::vector& markers) { + std::lock_guard lock(mutex_); + successors_.clear(); + predecessors_.clear(); + cleared_points_.clear(); + markers_.clear(); + marked_thread_id_.clear(); + for (const auto& dependency : dependencies) { + successors_[dependency.predecessor].push_back(dependency.successor); + predecessors_[dependency.successor].push_back(dependency.predecessor); + point_filter_.Add(dependency.successor); + point_filter_.Add(dependency.predecessor); + } + for (const auto& marker : markers) { + successors_[marker.predecessor].push_back(marker.successor); + predecessors_[marker.successor].push_back(marker.predecessor); + markers_[marker.predecessor].push_back(marker.successor); + point_filter_.Add(marker.predecessor); + point_filter_.Add(marker.successor); + } + cv_.notify_all(); +} + +bool SyncPoint::Data::PredecessorsAllCleared(const std::string& point) { + for (const auto& pred : predecessors_[point]) { + if (cleared_points_.count(pred) == 0) { + return false; + } + } + return true; +} + +void SyncPoint::Data::ClearCallBack(const std::string& point) { + std::unique_lock lock(mutex_); + while (num_callbacks_running_ > 0) { + cv_.wait(lock); + } + callbacks_.erase(point); +} + +void SyncPoint::Data::ClearAllCallBacks() { + std::unique_lock lock(mutex_); + while (num_callbacks_running_ > 0) { + cv_.wait(lock); + } + callbacks_.clear(); +} + +void SyncPoint::Data::Process(const Slice& point, void* cb_arg) { + if (!enabled_) { + return; + } + + // Use a filter to prevent mutex lock if possible. + if (!point_filter_.MayContain(point)) { + return; + } + + // Must convert to std::string for remaining work. Take + // heap hit. + std::string point_string(point.ToString()); + std::unique_lock lock(mutex_); + auto thread_id = std::this_thread::get_id(); + + auto marker_iter = markers_.find(point_string); + if (marker_iter != markers_.end()) { + for (auto& marked_point : marker_iter->second) { + marked_thread_id_.emplace(marked_point, thread_id); + point_filter_.Add(marked_point); + } + } + + if (DisabledByMarker(point_string, thread_id)) { + return; + } + + while (!PredecessorsAllCleared(point_string)) { + cv_.wait(lock); + if (DisabledByMarker(point_string, thread_id)) { + return; + } + } + + auto callback_pair = callbacks_.find(point_string); + if (callback_pair != callbacks_.end()) { + num_callbacks_running_++; + mutex_.unlock(); + callback_pair->second(cb_arg); + mutex_.lock(); + num_callbacks_running_--; + } + cleared_points_.insert(point_string); + cv_.notify_all(); +} +} // namespace ROCKSDB_NAMESPACE +#endif diff --git a/librocksdb-sys/rocksdb/test_util/sync_point_impl.h b/librocksdb-sys/rocksdb/test_util/sync_point_impl.h new file mode 100644 index 0000000..64cc044 --- /dev/null +++ b/librocksdb-sys/rocksdb/test_util/sync_point_impl.h @@ -0,0 +1,96 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "memory/concurrent_arena.h" +#include "port/port.h" +#include "test_util/sync_point.h" +#include "util/dynamic_bloom.h" +#include "util/random.h" + +#pragma once + +#ifndef NDEBUG +namespace ROCKSDB_NAMESPACE { +// A hacky allocator for single use. +// Arena depends on SyncPoint and create circular dependency. +class SingleAllocator : public Allocator { + public: + char* Allocate(size_t) override { + assert(false); + return nullptr; + } + char* AllocateAligned(size_t bytes, size_t, Logger*) override { + buf_.resize(bytes); + return const_cast(buf_.data()); + } + size_t BlockSize() const override { + assert(false); + return 0; + } + + private: + std::string buf_; +}; + +struct SyncPoint::Data { + Data() : point_filter_(&alloc_, /*total_bits=*/8192), enabled_(false) {} + // Enable proper deletion by subclasses + virtual ~Data() {} + // successor/predecessor map loaded from LoadDependency + std::unordered_map> successors_; + std::unordered_map> predecessors_; + std::unordered_map> callbacks_; + std::unordered_map> markers_; + std::unordered_map marked_thread_id_; + + std::mutex mutex_; + std::condition_variable cv_; + // sync points that have been passed through + std::unordered_set cleared_points_; + SingleAllocator alloc_; + // A filter before holding mutex to speed up process. + DynamicBloom point_filter_; + std::atomic enabled_; + int num_callbacks_running_ = 0; + + void LoadDependency(const std::vector& dependencies); + void LoadDependencyAndMarkers(const std::vector& dependencies, + const std::vector& markers); + bool PredecessorsAllCleared(const std::string& point); + void SetCallBack(const std::string& point, + const std::function& callback) { + std::lock_guard lock(mutex_); + callbacks_[point] = callback; + point_filter_.Add(point); + } + + void ClearCallBack(const std::string& point); + void ClearAllCallBacks(); + void EnableProcessing() { enabled_ = true; } + void DisableProcessing() { enabled_ = false; } + void ClearTrace() { + std::lock_guard lock(mutex_); + cleared_points_.clear(); + } + bool DisabledByMarker(const std::string& point, std::thread::id thread_id) { + auto marked_point_iter = marked_thread_id_.find(point); + return marked_point_iter != marked_thread_id_.end() && + thread_id != marked_point_iter->second; + } + void Process(const Slice& point, void* cb_arg); +}; +} // namespace ROCKSDB_NAMESPACE +#endif // NDEBUG diff --git a/librocksdb-sys/rocksdb/test_util/testharness.cc b/librocksdb-sys/rocksdb/test_util/testharness.cc new file mode 100644 index 0000000..3c7b835 --- /dev/null +++ b/librocksdb-sys/rocksdb/test_util/testharness.cc @@ -0,0 +1,107 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "test_util/testharness.h" + +#include +#include +#include + +namespace ROCKSDB_NAMESPACE { +namespace test { + +#ifdef OS_WIN +#include + +std::string GetPidStr() { return std::to_string(GetCurrentProcessId()); } +#else +std::string GetPidStr() { return std::to_string(getpid()); } +#endif + +::testing::AssertionResult AssertStatus(const char* s_expr, const Status& s) { + if (s.ok()) { + return ::testing::AssertionSuccess(); + } else { + return ::testing::AssertionFailure() << s_expr << std::endl << s.ToString(); + } +} + +std::string TmpDir(Env* env) { + std::string dir; + Status s = env->GetTestDirectory(&dir); + EXPECT_OK(s); + return dir; +} + +std::string PerThreadDBPath(std::string dir, std::string name) { + size_t tid = std::hash()(std::this_thread::get_id()); + return dir + "/" + name + "_" + GetPidStr() + "_" + std::to_string(tid); +} + +std::string PerThreadDBPath(std::string name) { + return PerThreadDBPath(test::TmpDir(), name); +} + +std::string PerThreadDBPath(Env* env, std::string name) { + return PerThreadDBPath(test::TmpDir(env), name); +} + +int RandomSeed() { + const char* env = getenv("TEST_RANDOM_SEED"); + int result = (env != nullptr ? atoi(env) : 301); + if (result <= 0) { + result = 301; + } + return result; +} + +TestRegex::TestRegex(const std::string& pattern) + : impl_(std::make_shared(pattern)), pattern_(pattern) {} +TestRegex::TestRegex(const char* pattern) + : impl_(std::make_shared(pattern)), pattern_(pattern) {} + +const std::string& TestRegex::GetPattern() const { return pattern_; } + +class TestRegex::Impl : public std::regex { + public: + using std::regex::basic_regex; +}; + +bool TestRegex::Matches(const std::string& str) const { + if (impl_) { + return std::regex_match(str, *impl_); + } else { + // Should not call Matches on unset Regex + assert(false); + return false; + } +} + +::testing::AssertionResult AssertMatchesRegex(const char* str_expr, + const char* pattern_expr, + const std::string& str, + const TestRegex& pattern) { + if (pattern.Matches(str)) { + return ::testing::AssertionSuccess(); + } else if (TestRegex("\".*\"").Matches(pattern_expr)) { + // constant regex string + return ::testing::AssertionFailure() + << str << " (" << str_expr << ")" << std::endl + << "does not match regex " << pattern.GetPattern(); + } else { + // runtime regex string + return ::testing::AssertionFailure() + << str << " (" << str_expr << ")" << std::endl + << "does not match regex" << std::endl + << pattern.GetPattern() << " (" << pattern_expr << ")"; + } +} + +} // namespace test +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/test_util/testharness.h b/librocksdb-sys/rocksdb/test_util/testharness.h new file mode 100644 index 0000000..d8b6c96 --- /dev/null +++ b/librocksdb-sys/rocksdb/test_util/testharness.h @@ -0,0 +1,124 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#ifdef OS_AIX +#include "gtest/gtest.h" +#else +#include +#endif + +// A "skipped" test has a specific meaning in Facebook infrastructure: the +// test is in good shape and should be run, but something about the +// compilation or execution environment means the test cannot be run. +// Specifically, there is a hole in intended testing if any +// parameterization of a test (e.g. Foo/FooTest.Bar/42) is skipped for all +// tested build configurations/platforms/etc. +// +// If GTEST_SKIP is available, use it. Otherwise, define skip as success. +// +// The GTEST macros do not seem to print the message, even with -verbose, +// so these print to stderr. Note that these do not exit the test themselves; +// calling code should 'return' or similar from the test. +#ifdef GTEST_SKIP_ +#define ROCKSDB_GTEST_SKIP(m) \ + do { \ + fputs("SKIPPED: " m "\n", stderr); \ + GTEST_SKIP_(m); \ + } while (false) /* user ; */ +#else +#define ROCKSDB_GTEST_SKIP(m) \ + do { \ + fputs("SKIPPED: " m "\n", stderr); \ + GTEST_SUCCESS_("SKIPPED: " m); \ + } while (false) /* user ; */ +#endif + +// We add "bypass" as an alternative to ROCKSDB_GTEST_SKIP that is allowed to +// be a permanent condition, e.g. for intentionally omitting or disabling some +// parameterizations for some tests. (Use _DISABLED at the end of the test +// name to disable an entire test.) +#define ROCKSDB_GTEST_BYPASS(m) \ + do { \ + fputs("BYPASSED: " m "\n", stderr); \ + GTEST_SUCCESS_("BYPASSED: " m); \ + } while (false) /* user ; */ + +// Avoid "loss of precision" warnings when passing in 64-bit integers +#define EXPECT_NEAR2(val1, val2, abs_error) \ + EXPECT_NEAR(static_cast(val1), static_cast(val2), \ + static_cast(abs_error)) + +#include + +#include "port/stack_trace.h" +#include "rocksdb/env.h" + +namespace ROCKSDB_NAMESPACE { +namespace test { + +// Return the directory to use for temporary storage. +std::string TmpDir(Env* env = Env::Default()); + +// A path unique within the thread +std::string PerThreadDBPath(std::string name); +std::string PerThreadDBPath(Env* env, std::string name); +std::string PerThreadDBPath(std::string dir, std::string name); + +// Return a randomization seed for this run. Typically returns the +// same number on repeated invocations of this binary, but automated +// runs may be able to vary the seed. +int RandomSeed(); + +::testing::AssertionResult AssertStatus(const char* s_expr, const Status& s); + +#define ASSERT_OK(s) \ + ASSERT_PRED_FORMAT1(ROCKSDB_NAMESPACE::test::AssertStatus, s) +#define ASSERT_NOK(s) ASSERT_FALSE((s).ok()) +#define EXPECT_OK(s) \ + EXPECT_PRED_FORMAT1(ROCKSDB_NAMESPACE::test::AssertStatus, s) +#define EXPECT_NOK(s) EXPECT_FALSE((s).ok()) + +// Useful for testing +// * No need to deal with Status like in Regex public API +// * No triggering lint reports on use of std::regex in tests +// * Available in LITE (unlike public API) +class TestRegex { + public: + // These throw on bad pattern + /*implicit*/ TestRegex(const std::string& pattern); + /*implicit*/ TestRegex(const char* pattern); + + // Checks that the whole of str is matched by this regex + bool Matches(const std::string& str) const; + + const std::string& GetPattern() const; + + private: + class Impl; + std::shared_ptr impl_; // shared_ptr for simple implementation + std::string pattern_; +}; + +::testing::AssertionResult AssertMatchesRegex(const char* str_expr, + const char* pattern_expr, + const std::string& str, + const TestRegex& pattern); + +#define ASSERT_MATCHES_REGEX(str, pattern) \ + ASSERT_PRED_FORMAT2(ROCKSDB_NAMESPACE::test::AssertMatchesRegex, str, pattern) +#define EXPECT_MATCHES_REGEX(str, pattern) \ + EXPECT_PRED_FORMAT2(ROCKSDB_NAMESPACE::test::AssertMatchesRegex, str, pattern) + +} // namespace test + +using test::TestRegex; + +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/test_util/testutil.cc b/librocksdb-sys/rocksdb/test_util/testutil.cc new file mode 100644 index 0000000..1e771f4 --- /dev/null +++ b/librocksdb-sys/rocksdb/test_util/testutil.cc @@ -0,0 +1,751 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "test_util/testutil.h" + +#include +#include + +#include +#include +#include +#include + +#include "db/memtable_list.h" +#include "env/composite_env_wrapper.h" +#include "file/random_access_file_reader.h" +#include "file/sequence_file_reader.h" +#include "file/writable_file_writer.h" +#include "port/port.h" +#include "rocksdb/convenience.h" +#include "rocksdb/system_clock.h" +#include "rocksdb/utilities/object_registry.h" +#include "test_util/mock_time_env.h" +#include "test_util/sync_point.h" +#include "util/random.h" + +#ifndef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS +void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {} +#endif + +namespace ROCKSDB_NAMESPACE { +namespace test { + +const uint32_t kDefaultFormatVersion = BlockBasedTableOptions().format_version; +const std::set kFooterFormatVersionsToTest{ + // Non-legacy, before big footer changes + 5U, + // After big footer changes + 6U, + // In case any interesting future changes + kDefaultFormatVersion, + kLatestFormatVersion, +}; + +std::string RandomKey(Random* rnd, int len, RandomKeyType type) { + // Make sure to generate a wide variety of characters so we + // test the boundary conditions for short-key optimizations. + static const char kTestChars[] = {'\0', '\1', 'a', 'b', 'c', + 'd', 'e', '\xfd', '\xfe', '\xff'}; + std::string result; + for (int i = 0; i < len; i++) { + std::size_t indx = 0; + switch (type) { + case RandomKeyType::RANDOM: + indx = rnd->Uniform(sizeof(kTestChars)); + break; + case RandomKeyType::LARGEST: + indx = sizeof(kTestChars) - 1; + break; + case RandomKeyType::MIDDLE: + indx = sizeof(kTestChars) / 2; + break; + case RandomKeyType::SMALLEST: + indx = 0; + break; + } + result += kTestChars[indx]; + } + return result; +} + +const std::vector& GetUDTTestModes() { + static std::vector udt_test_modes = { + UserDefinedTimestampTestMode::kStripUserDefinedTimestamp, + UserDefinedTimestampTestMode::kNormal, + UserDefinedTimestampTestMode::kNone}; + return udt_test_modes; +} + +bool IsUDTEnabled(const UserDefinedTimestampTestMode& test_mode) { + return test_mode != UserDefinedTimestampTestMode::kNone; +} + +bool ShouldPersistUDT(const UserDefinedTimestampTestMode& test_mode) { + return test_mode != UserDefinedTimestampTestMode::kStripUserDefinedTimestamp; +} + +extern Slice CompressibleString(Random* rnd, double compressed_fraction, + int len, std::string* dst) { + int raw = static_cast(len * compressed_fraction); + if (raw < 1) raw = 1; + std::string raw_data = rnd->RandomBinaryString(raw); + + // Duplicate the random data until we have filled "len" bytes + dst->clear(); + while (dst->size() < (unsigned int)len) { + dst->append(raw_data); + } + dst->resize(len); + return Slice(*dst); +} + +namespace { +class Uint64ComparatorImpl : public Comparator { + public: + Uint64ComparatorImpl() {} + + const char* Name() const override { return "rocksdb.Uint64Comparator"; } + + int Compare(const Slice& a, const Slice& b) const override { + assert(a.size() == sizeof(uint64_t) && b.size() == sizeof(uint64_t)); + const uint64_t* left = reinterpret_cast(a.data()); + const uint64_t* right = reinterpret_cast(b.data()); + uint64_t leftValue; + uint64_t rightValue; + GetUnaligned(left, &leftValue); + GetUnaligned(right, &rightValue); + if (leftValue == rightValue) { + return 0; + } else if (leftValue < rightValue) { + return -1; + } else { + return 1; + } + } + + void FindShortestSeparator(std::string* /*start*/, + const Slice& /*limit*/) const override { + return; + } + + void FindShortSuccessor(std::string* /*key*/) const override { return; } +}; +} // namespace + +const Comparator* Uint64Comparator() { + static Uint64ComparatorImpl uint64comp; + return &uint64comp; +} + +const Comparator* BytewiseComparatorWithU64TsWrapper() { + ConfigOptions config_options; + const Comparator* user_comparator = nullptr; + Status s = Comparator::CreateFromString( + config_options, "leveldb.BytewiseComparator.u64ts", &user_comparator); + s.PermitUncheckedError(); + return user_comparator; +} + +const Comparator* ReverseBytewiseComparatorWithU64TsWrapper() { + ConfigOptions config_options; + const Comparator* user_comparator = nullptr; + Status s = Comparator::CreateFromString( + config_options, "rocksdb.ReverseBytewiseComparator.u64ts", + &user_comparator); + s.PermitUncheckedError(); + return user_comparator; +} + +void CorruptKeyType(InternalKey* ikey) { + std::string keystr = ikey->Encode().ToString(); + keystr[keystr.size() - 8] = kTypeLogData; + ikey->DecodeFrom(Slice(keystr.data(), keystr.size())); +} + +std::string KeyStr(const std::string& user_key, const SequenceNumber& seq, + const ValueType& t, bool corrupt) { + InternalKey k(user_key, seq, t); + if (corrupt) { + CorruptKeyType(&k); + } + return k.Encode().ToString(); +} + +std::string KeyStr(uint64_t ts, const std::string& user_key, + const SequenceNumber& seq, const ValueType& t, + bool corrupt) { + std::string user_key_with_ts(user_key); + std::string ts_str; + PutFixed64(&ts_str, ts); + user_key_with_ts.append(ts_str); + return KeyStr(user_key_with_ts, seq, t, corrupt); +} + +bool SleepingBackgroundTask::TimedWaitUntilSleeping(uint64_t wait_time) { + auto abs_time = SystemClock::Default()->NowMicros() + wait_time; + MutexLock l(&mutex_); + while (!sleeping_ || !should_sleep_) { + if (bg_cv_.TimedWait(abs_time)) { + return true; + } + } + return false; +} + +bool SleepingBackgroundTask::TimedWaitUntilDone(uint64_t wait_time) { + auto abs_time = SystemClock::Default()->NowMicros() + wait_time; + MutexLock l(&mutex_); + while (!done_with_sleep_) { + if (bg_cv_.TimedWait(abs_time)) { + return true; + } + } + return false; +} + +std::string RandomName(Random* rnd, const size_t len) { + std::stringstream ss; + for (size_t i = 0; i < len; ++i) { + ss << static_cast(rnd->Uniform(26) + 'a'); + } + return ss.str(); +} + +CompressionType RandomCompressionType(Random* rnd) { + auto ret = static_cast(rnd->Uniform(6)); + while (!CompressionTypeSupported(ret)) { + ret = static_cast((static_cast(ret) + 1) % 6); + } + return ret; +} + +void RandomCompressionTypeVector(const size_t count, + std::vector* types, + Random* rnd) { + types->clear(); + for (size_t i = 0; i < count; ++i) { + types->emplace_back(RandomCompressionType(rnd)); + } +} + +const SliceTransform* RandomSliceTransform(Random* rnd, int pre_defined) { + int random_num = pre_defined >= 0 ? pre_defined : rnd->Uniform(4); + switch (random_num) { + case 0: + return NewFixedPrefixTransform(rnd->Uniform(20) + 1); + case 1: + return NewCappedPrefixTransform(rnd->Uniform(20) + 1); + case 2: + return NewNoopTransform(); + default: + return nullptr; + } +} + +BlockBasedTableOptions RandomBlockBasedTableOptions(Random* rnd) { + BlockBasedTableOptions opt; + opt.cache_index_and_filter_blocks = rnd->Uniform(2); + opt.pin_l0_filter_and_index_blocks_in_cache = rnd->Uniform(2); + opt.pin_top_level_index_and_filter = rnd->Uniform(2); + using IndexType = BlockBasedTableOptions::IndexType; + const std::array index_types = { + {IndexType::kBinarySearch, IndexType::kHashSearch, + IndexType::kTwoLevelIndexSearch, IndexType::kBinarySearchWithFirstKey}}; + opt.index_type = + index_types[rnd->Uniform(static_cast(index_types.size()))]; + opt.checksum = static_cast(rnd->Uniform(3)); + opt.block_size = rnd->Uniform(10000000); + opt.block_size_deviation = rnd->Uniform(100); + opt.block_restart_interval = rnd->Uniform(100); + opt.index_block_restart_interval = rnd->Uniform(100); + opt.whole_key_filtering = rnd->Uniform(2); + + return opt; +} + +TableFactory* RandomTableFactory(Random* rnd, int pre_defined) { + int random_num = pre_defined >= 0 ? pre_defined : rnd->Uniform(4); + switch (random_num) { + case 0: + return NewPlainTableFactory(); + case 1: + return NewCuckooTableFactory(); + default: + return NewBlockBasedTableFactory(); + } +} + +MergeOperator* RandomMergeOperator(Random* rnd) { + return new ChanglingMergeOperator(RandomName(rnd, 10)); +} + +CompactionFilter* RandomCompactionFilter(Random* rnd) { + return new ChanglingCompactionFilter(RandomName(rnd, 10)); +} + +CompactionFilterFactory* RandomCompactionFilterFactory(Random* rnd) { + return new ChanglingCompactionFilterFactory(RandomName(rnd, 10)); +} + +void RandomInitDBOptions(DBOptions* db_opt, Random* rnd) { + // boolean options + db_opt->advise_random_on_open = rnd->Uniform(2); + db_opt->allow_mmap_reads = rnd->Uniform(2); + db_opt->allow_mmap_writes = rnd->Uniform(2); + db_opt->use_direct_reads = rnd->Uniform(2); + db_opt->use_direct_io_for_flush_and_compaction = rnd->Uniform(2); + db_opt->create_if_missing = rnd->Uniform(2); + db_opt->create_missing_column_families = rnd->Uniform(2); + db_opt->enable_thread_tracking = rnd->Uniform(2); + db_opt->error_if_exists = rnd->Uniform(2); + db_opt->is_fd_close_on_exec = rnd->Uniform(2); + db_opt->paranoid_checks = rnd->Uniform(2); + db_opt->track_and_verify_wals_in_manifest = rnd->Uniform(2); + db_opt->verify_sst_unique_id_in_manifest = rnd->Uniform(2); + db_opt->skip_stats_update_on_db_open = rnd->Uniform(2); + db_opt->skip_checking_sst_file_sizes_on_db_open = rnd->Uniform(2); + db_opt->use_adaptive_mutex = rnd->Uniform(2); + db_opt->use_fsync = rnd->Uniform(2); + db_opt->recycle_log_file_num = rnd->Uniform(2); + db_opt->avoid_flush_during_recovery = rnd->Uniform(2); + db_opt->avoid_flush_during_shutdown = rnd->Uniform(2); + db_opt->enforce_single_del_contracts = rnd->Uniform(2); + + // int options + db_opt->max_background_compactions = rnd->Uniform(100); + db_opt->max_background_flushes = rnd->Uniform(100); + db_opt->max_file_opening_threads = rnd->Uniform(100); + db_opt->max_open_files = rnd->Uniform(100); + db_opt->table_cache_numshardbits = rnd->Uniform(100); + + // size_t options + db_opt->db_write_buffer_size = rnd->Uniform(10000); + db_opt->keep_log_file_num = rnd->Uniform(10000); + db_opt->log_file_time_to_roll = rnd->Uniform(10000); + db_opt->manifest_preallocation_size = rnd->Uniform(10000); + db_opt->max_log_file_size = rnd->Uniform(10000); + + // std::string options + db_opt->db_log_dir = "path/to/db_log_dir"; + db_opt->wal_dir = "path/to/wal_dir"; + + // uint32_t options + db_opt->max_subcompactions = rnd->Uniform(100000); + + // uint64_t options + static const uint64_t uint_max = static_cast(UINT_MAX); + db_opt->WAL_size_limit_MB = uint_max + rnd->Uniform(100000); + db_opt->WAL_ttl_seconds = uint_max + rnd->Uniform(100000); + db_opt->bytes_per_sync = uint_max + rnd->Uniform(100000); + db_opt->delayed_write_rate = uint_max + rnd->Uniform(100000); + db_opt->delete_obsolete_files_period_micros = uint_max + rnd->Uniform(100000); + db_opt->max_manifest_file_size = uint_max + rnd->Uniform(100000); + db_opt->max_total_wal_size = uint_max + rnd->Uniform(100000); + db_opt->wal_bytes_per_sync = uint_max + rnd->Uniform(100000); + + // unsigned int options + db_opt->stats_dump_period_sec = rnd->Uniform(100000); +} + +void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, DBOptions& db_options, + Random* rnd) { + cf_opt->compaction_style = (CompactionStyle)(rnd->Uniform(4)); + + // boolean options + cf_opt->report_bg_io_stats = rnd->Uniform(2); + cf_opt->disable_auto_compactions = rnd->Uniform(2); + cf_opt->inplace_update_support = rnd->Uniform(2); + cf_opt->level_compaction_dynamic_level_bytes = rnd->Uniform(2); + cf_opt->optimize_filters_for_hits = rnd->Uniform(2); + cf_opt->paranoid_file_checks = rnd->Uniform(2); + cf_opt->force_consistency_checks = rnd->Uniform(2); + cf_opt->compaction_options_fifo.allow_compaction = rnd->Uniform(2); + cf_opt->memtable_whole_key_filtering = rnd->Uniform(2); + cf_opt->enable_blob_files = rnd->Uniform(2); + cf_opt->enable_blob_garbage_collection = rnd->Uniform(2); + + // double options + cf_opt->memtable_prefix_bloom_size_ratio = + static_cast(rnd->Uniform(10000)) / 20000.0; + cf_opt->blob_garbage_collection_age_cutoff = rnd->Uniform(10000) / 10000.0; + cf_opt->blob_garbage_collection_force_threshold = + rnd->Uniform(10000) / 10000.0; + + // int options + cf_opt->level0_file_num_compaction_trigger = rnd->Uniform(100); + cf_opt->level0_slowdown_writes_trigger = rnd->Uniform(100); + cf_opt->level0_stop_writes_trigger = rnd->Uniform(100); + cf_opt->max_bytes_for_level_multiplier = rnd->Uniform(100); + cf_opt->max_write_buffer_number = rnd->Uniform(100); + cf_opt->max_write_buffer_number_to_maintain = rnd->Uniform(100); + cf_opt->max_write_buffer_size_to_maintain = rnd->Uniform(10000); + cf_opt->min_write_buffer_number_to_merge = rnd->Uniform(100); + cf_opt->num_levels = rnd->Uniform(100); + cf_opt->target_file_size_multiplier = rnd->Uniform(100); + + // vector int options + cf_opt->max_bytes_for_level_multiplier_additional.resize(cf_opt->num_levels); + for (int i = 0; i < cf_opt->num_levels; i++) { + cf_opt->max_bytes_for_level_multiplier_additional[i] = rnd->Uniform(100); + } + + // size_t options + cf_opt->arena_block_size = rnd->Uniform(10000); + cf_opt->inplace_update_num_locks = rnd->Uniform(10000); + cf_opt->max_successive_merges = rnd->Uniform(10000); + cf_opt->memtable_huge_page_size = rnd->Uniform(10000); + cf_opt->write_buffer_size = rnd->Uniform(10000); + + // uint32_t options + cf_opt->bloom_locality = rnd->Uniform(10000); + cf_opt->max_bytes_for_level_base = rnd->Uniform(10000); + + // uint64_t options + static const uint64_t uint_max = static_cast(UINT_MAX); + cf_opt->ttl = + db_options.max_open_files == -1 ? uint_max + rnd->Uniform(10000) : 0; + cf_opt->periodic_compaction_seconds = + db_options.max_open_files == -1 ? uint_max + rnd->Uniform(10000) : 0; + cf_opt->max_sequential_skip_in_iterations = uint_max + rnd->Uniform(10000); + cf_opt->target_file_size_base = uint_max + rnd->Uniform(10000); + cf_opt->max_compaction_bytes = + cf_opt->target_file_size_base * rnd->Uniform(100); + cf_opt->compaction_options_fifo.max_table_files_size = + uint_max + rnd->Uniform(10000); + cf_opt->min_blob_size = uint_max + rnd->Uniform(10000); + cf_opt->blob_file_size = uint_max + rnd->Uniform(10000); + cf_opt->blob_compaction_readahead_size = uint_max + rnd->Uniform(10000); + + // pointer typed options + cf_opt->prefix_extractor.reset(RandomSliceTransform(rnd)); + cf_opt->table_factory.reset(RandomTableFactory(rnd)); + cf_opt->merge_operator.reset(RandomMergeOperator(rnd)); + if (cf_opt->compaction_filter) { + delete cf_opt->compaction_filter; + } + cf_opt->compaction_filter = RandomCompactionFilter(rnd); + cf_opt->compaction_filter_factory.reset(RandomCompactionFilterFactory(rnd)); + + // custom typed options + cf_opt->compression = RandomCompressionType(rnd); + RandomCompressionTypeVector(cf_opt->num_levels, + &cf_opt->compression_per_level, rnd); + cf_opt->blob_compression_type = RandomCompressionType(rnd); +} + +bool IsDirectIOSupported(Env* env, const std::string& dir) { + EnvOptions env_options; + env_options.use_mmap_writes = false; + env_options.use_direct_writes = true; + std::string tmp = TempFileName(dir, 999); + Status s; + { + std::unique_ptr file; + s = env->NewWritableFile(tmp, &file, env_options); + } + if (s.ok()) { + s = env->DeleteFile(tmp); + } + return s.ok(); +} + +bool IsPrefetchSupported(const std::shared_ptr& fs, + const std::string& dir) { + bool supported = false; + std::string tmp = TempFileName(dir, 999); + Random rnd(301); + std::string test_string = rnd.RandomString(4096); + Slice data(test_string); + Status s = WriteStringToFile(fs.get(), data, tmp, true); + if (s.ok()) { + std::unique_ptr file; + auto io_s = fs->NewRandomAccessFile(tmp, FileOptions(), &file, nullptr); + if (io_s.ok()) { + supported = !(file->Prefetch(0, data.size(), IOOptions(), nullptr) + .IsNotSupported()); + } + s = fs->DeleteFile(tmp, IOOptions(), nullptr); + } + return s.ok() && supported; +} + +size_t GetLinesCount(const std::string& fname, const std::string& pattern) { + std::stringstream ssbuf; + std::string line; + size_t count = 0; + + std::ifstream inFile(fname.c_str()); + ssbuf << inFile.rdbuf(); + + while (getline(ssbuf, line)) { + if (line.find(pattern) != std::string::npos) { + count++; + } + } + + return count; +} + +Status CorruptFile(Env* env, const std::string& fname, int offset, + int bytes_to_corrupt, bool verify_checksum /*=true*/) { + uint64_t size; + Status s = env->GetFileSize(fname, &size); + if (!s.ok()) { + return s; + } else if (offset < 0) { + // Relative to end of file; make it absolute + if (-offset > static_cast(size)) { + offset = 0; + } else { + offset = static_cast(size + offset); + } + } + if (offset > static_cast(size)) { + offset = static_cast(size); + } + if (offset + bytes_to_corrupt > static_cast(size)) { + bytes_to_corrupt = static_cast(size - offset); + } + + // Do it + std::string contents; + s = ReadFileToString(env, fname, &contents); + if (s.ok()) { + for (int i = 0; i < bytes_to_corrupt; i++) { + contents[i + offset] ^= 0x80; + } + s = WriteStringToFile(env, contents, fname); + } + if (s.ok() && verify_checksum) { + Options options; + options.env = env; + EnvOptions env_options; + Status v = VerifySstFileChecksum(options, env_options, fname); + assert(!v.ok()); + } + return s; +} + +Status TruncateFile(Env* env, const std::string& fname, uint64_t new_length) { + uint64_t old_length; + Status s = env->GetFileSize(fname, &old_length); + if (!s.ok() || new_length == old_length) { + return s; + } + // Do it + std::string contents; + s = ReadFileToString(env, fname, &contents); + if (s.ok()) { + contents.resize(static_cast(new_length), 'b'); + s = WriteStringToFile(env, contents, fname); + } + return s; +} + +// Try and delete a directory if it exists +Status TryDeleteDir(Env* env, const std::string& dirname) { + bool is_dir = false; + Status s = env->IsDirectory(dirname, &is_dir); + if (s.ok() && is_dir) { + s = env->DeleteDir(dirname); + } + return s; +} + +// Delete a directory if it exists +void DeleteDir(Env* env, const std::string& dirname) { + TryDeleteDir(env, dirname).PermitUncheckedError(); +} + +Status CreateEnvFromSystem(const ConfigOptions& config_options, Env** result, + std::shared_ptr* guard) { + const char* env_uri = getenv("TEST_ENV_URI"); + const char* fs_uri = getenv("TEST_FS_URI"); + if (env_uri || fs_uri) { + return Env::CreateFromUri(config_options, + (env_uri != nullptr) ? env_uri : "", + (fs_uri != nullptr) ? fs_uri : "", result, guard); + } else { + // Neither specified. Use the default + *result = config_options.env; + guard->reset(); + return Status::OK(); + } +} +namespace { +// A hacky skip list mem table that triggers flush after number of entries. +class SpecialMemTableRep : public MemTableRep { + public: + explicit SpecialMemTableRep(Allocator* allocator, MemTableRep* memtable, + int num_entries_flush) + : MemTableRep(allocator), + memtable_(memtable), + num_entries_flush_(num_entries_flush), + num_entries_(0) {} + + virtual KeyHandle Allocate(const size_t len, char** buf) override { + return memtable_->Allocate(len, buf); + } + + // Insert key into the list. + // REQUIRES: nothing that compares equal to key is currently in the list. + virtual void Insert(KeyHandle handle) override { + num_entries_++; + memtable_->Insert(handle); + } + + void InsertConcurrently(KeyHandle handle) override { + num_entries_++; + memtable_->Insert(handle); + } + + // Returns true iff an entry that compares equal to key is in the list. + virtual bool Contains(const char* key) const override { + return memtable_->Contains(key); + } + + virtual size_t ApproximateMemoryUsage() override { + // Return a high memory usage when number of entries exceeds the threshold + // to trigger a flush. + return (num_entries_ < num_entries_flush_) ? 0 : 1024 * 1024 * 1024; + } + + virtual void Get(const LookupKey& k, void* callback_args, + bool (*callback_func)(void* arg, + const char* entry)) override { + memtable_->Get(k, callback_args, callback_func); + } + + uint64_t ApproximateNumEntries(const Slice& start_ikey, + const Slice& end_ikey) override { + return memtable_->ApproximateNumEntries(start_ikey, end_ikey); + } + + virtual MemTableRep::Iterator* GetIterator(Arena* arena = nullptr) override { + return memtable_->GetIterator(arena); + } + + virtual ~SpecialMemTableRep() override {} + + private: + std::unique_ptr memtable_; + int num_entries_flush_; + int num_entries_; +}; +class SpecialSkipListFactory : public MemTableRepFactory { + public: + static bool Register(ObjectLibrary& library, const std::string& /*arg*/) { + library.AddFactory( + ObjectLibrary::PatternEntry(SpecialSkipListFactory::kClassName(), true) + .AddNumber(":"), + [](const std::string& uri, std::unique_ptr* guard, + std::string* /* errmsg */) { + auto colon = uri.find(":"); + if (colon != std::string::npos) { + auto count = ParseInt(uri.substr(colon + 1)); + guard->reset(new SpecialSkipListFactory(count)); + } else { + guard->reset(new SpecialSkipListFactory(2)); + } + return guard->get(); + }); + return true; + } + // After number of inserts >= `num_entries_flush` in a mem table, trigger + // flush. + explicit SpecialSkipListFactory(int num_entries_flush) + : num_entries_flush_(num_entries_flush) {} + + using MemTableRepFactory::CreateMemTableRep; + virtual MemTableRep* CreateMemTableRep( + const MemTableRep::KeyComparator& compare, Allocator* allocator, + const SliceTransform* transform, Logger* /*logger*/) override { + return new SpecialMemTableRep( + allocator, + factory_.CreateMemTableRep(compare, allocator, transform, nullptr), + num_entries_flush_); + } + static const char* kClassName() { return "SpecialSkipListFactory"; } + virtual const char* Name() const override { return kClassName(); } + std::string GetId() const override { + std::string id = Name(); + if (num_entries_flush_ > 0) { + id.append(":").append(std::to_string(num_entries_flush_)); + } + return id; + } + + bool IsInsertConcurrentlySupported() const override { + return factory_.IsInsertConcurrentlySupported(); + } + + private: + SkipListFactory factory_; + int num_entries_flush_; +}; +} // namespace + +MemTableRepFactory* NewSpecialSkipListFactory(int num_entries_per_flush) { + RegisterTestLibrary(); + return new SpecialSkipListFactory(num_entries_per_flush); +} + +// This method loads existing test classes into the ObjectRegistry +int RegisterTestObjects(ObjectLibrary& library, const std::string& arg) { + size_t num_types; + library.AddFactory( + test::SimpleSuffixReverseComparator::kClassName(), + [](const std::string& /*uri*/, + std::unique_ptr* /*guard*/, + std::string* /* errmsg */) { + static test::SimpleSuffixReverseComparator ssrc; + return &ssrc; + }); + SpecialSkipListFactory::Register(library, arg); + library.AddFactory( + "Changling", + [](const std::string& uri, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new test::ChanglingMergeOperator(uri)); + return guard->get(); + }); + library.AddFactory( + "Changling", + [](const std::string& uri, std::unique_ptr* /*guard*/, + std::string* /* errmsg */) { + return new test::ChanglingCompactionFilter(uri); + }); + library.AddFactory( + "Changling", [](const std::string& uri, + std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new test::ChanglingCompactionFilterFactory(uri)); + return guard->get(); + }); + library.AddFactory( + MockSystemClock::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new MockSystemClock(SystemClock::Default())); + return guard->get(); + }); + return static_cast(library.GetFactoryCount(&num_types)); +} + + +void RegisterTestLibrary(const std::string& arg) { + static bool registered = false; + if (!registered) { + registered = true; + ObjectRegistry::Default()->AddLibrary("test", RegisterTestObjects, arg); + } +} +} // namespace test +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/test_util/testutil.h b/librocksdb-sys/rocksdb/test_util/testutil.h new file mode 100644 index 0000000..eca1ff7 --- /dev/null +++ b/librocksdb-sys/rocksdb/test_util/testutil.h @@ -0,0 +1,879 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once +#include +#include +#include +#include + +#include "env/composite_env_wrapper.h" +#include "file/writable_file_writer.h" +#include "rocksdb/compaction_filter.h" +#include "rocksdb/env.h" +#include "rocksdb/iterator.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "rocksdb/table.h" +#include "table/internal_iterator.h" +#include "util/mutexlock.h" + +#ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS +extern "C" { +void RegisterCustomObjects(int argc, char** argv); +} +#else +void RegisterCustomObjects(int argc, char** argv); +#endif // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS + +namespace ROCKSDB_NAMESPACE { +class FileSystem; +class MemTableRepFactory; +class ObjectLibrary; +class Random; +class SequentialFile; +class SequentialFileReader; + +namespace test { + +extern const uint32_t kDefaultFormatVersion; +extern const std::set kFooterFormatVersionsToTest; + +// Return a random key with the specified length that may contain interesting +// characters (e.g. \x00, \xff, etc.). +enum RandomKeyType : char { RANDOM, LARGEST, SMALLEST, MIDDLE }; +extern std::string RandomKey(Random* rnd, int len, + RandomKeyType type = RandomKeyType::RANDOM); + +enum class UserDefinedTimestampTestMode { + // Test does not enable user-defined timestamp feature. + kNone, + // Test enables user-defined timestamp feature. Write/read with min timestamps + kNormal, + // Test enables user-defined timestamp feature. Write/read with min timestamps + // Set `persist_user_defined_timestamps` to false. + kStripUserDefinedTimestamp, +}; + +extern const std::vector& GetUDTTestModes(); + +extern bool IsUDTEnabled(const UserDefinedTimestampTestMode& test_mode); + +extern bool ShouldPersistUDT(const UserDefinedTimestampTestMode& test_mode); + +// Store in *dst a string of length "len" that will compress to +// "N*compressed_fraction" bytes and return a Slice that references +// the generated data. +extern Slice CompressibleString(Random* rnd, double compressed_fraction, + int len, std::string* dst); + +#ifndef NDEBUG +// An internal comparator that just forward comparing results from the +// user comparator in it. Can be used to test entities that have no dependency +// on internal key structure but consumes InternalKeyComparator, like +// BlockBasedTable. +class PlainInternalKeyComparator : public InternalKeyComparator { + public: + explicit PlainInternalKeyComparator(const Comparator* c) + : InternalKeyComparator(c) {} + + virtual ~PlainInternalKeyComparator() {} + + virtual int Compare(const Slice& a, const Slice& b) const override { + return user_comparator()->Compare(a, b); + } +}; +#endif + +// A test comparator which compare two strings in this way: +// (1) first compare prefix of 8 bytes in alphabet order, +// (2) if two strings share the same prefix, sort the other part of the string +// in the reverse alphabet order. +// This helps simulate the case of compounded key of [entity][timestamp] and +// latest timestamp first. +class SimpleSuffixReverseComparator : public Comparator { + public: + SimpleSuffixReverseComparator() {} + static const char* kClassName() { return "SimpleSuffixReverseComparator"; } + virtual const char* Name() const override { return kClassName(); } + + virtual int Compare(const Slice& a, const Slice& b) const override { + Slice prefix_a = Slice(a.data(), 8); + Slice prefix_b = Slice(b.data(), 8); + int prefix_comp = prefix_a.compare(prefix_b); + if (prefix_comp != 0) { + return prefix_comp; + } else { + Slice suffix_a = Slice(a.data() + 8, a.size() - 8); + Slice suffix_b = Slice(b.data() + 8, b.size() - 8); + return -(suffix_a.compare(suffix_b)); + } + } + virtual void FindShortestSeparator(std::string* /*start*/, + const Slice& /*limit*/) const override {} + + virtual void FindShortSuccessor(std::string* /*key*/) const override {} +}; + +// Returns a user key comparator that can be used for comparing two uint64_t +// slices. Instead of comparing slices byte-wise, it compares all the 8 bytes +// at once. Assumes same endian-ness is used though the database's lifetime. +// Symantics of comparison would differ from Bytewise comparator in little +// endian machines. +extern const Comparator* Uint64Comparator(); + +// A wrapper api for getting the ComparatorWithU64Ts +extern const Comparator* BytewiseComparatorWithU64TsWrapper(); + +// A wrapper api for getting the ComparatorWithU64Ts +extern const Comparator* ReverseBytewiseComparatorWithU64TsWrapper(); + +class StringSink : public FSWritableFile { + public: + std::string contents_; + + explicit StringSink(Slice* reader_contents = nullptr) + : FSWritableFile(), + contents_(""), + reader_contents_(reader_contents), + last_flush_(0) { + if (reader_contents_ != nullptr) { + *reader_contents_ = Slice(contents_.data(), 0); + } + } + + const std::string& contents() const { return contents_; } + + IOStatus Truncate(uint64_t size, const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) override { + contents_.resize(static_cast(size)); + return IOStatus::OK(); + } + IOStatus Close(const IOOptions& /*opts*/, IODebugContext* /*dbg*/) override { + return IOStatus::OK(); + } + IOStatus Flush(const IOOptions& /*opts*/, IODebugContext* /*dbg*/) override { + if (reader_contents_ != nullptr) { + assert(reader_contents_->size() <= last_flush_); + size_t offset = last_flush_ - reader_contents_->size(); + *reader_contents_ = + Slice(contents_.data() + offset, contents_.size() - offset); + last_flush_ = contents_.size(); + } + + return IOStatus::OK(); + } + IOStatus Sync(const IOOptions& /*opts*/, IODebugContext* /*dbg*/) override { + return IOStatus::OK(); + } + + using FSWritableFile::Append; + IOStatus Append(const Slice& slice, const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) override { + contents_.append(slice.data(), slice.size()); + return IOStatus::OK(); + } + void Drop(size_t bytes) { + if (reader_contents_ != nullptr) { + contents_.resize(contents_.size() - bytes); + *reader_contents_ = + Slice(reader_contents_->data(), reader_contents_->size() - bytes); + last_flush_ = contents_.size(); + } + } + + private: + Slice* reader_contents_; + size_t last_flush_; +}; + +// A wrapper around a StringSink to give it a RandomRWFile interface +class RandomRWStringSink : public FSRandomRWFile { + public: + explicit RandomRWStringSink(StringSink* ss) : ss_(ss) {} + + IOStatus Write(uint64_t offset, const Slice& data, const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) override { + if (offset + data.size() > ss_->contents_.size()) { + ss_->contents_.resize(static_cast(offset) + data.size(), '\0'); + } + + char* pos = const_cast(ss_->contents_.data() + offset); + memcpy(pos, data.data(), data.size()); + return IOStatus::OK(); + } + + IOStatus Read(uint64_t offset, size_t n, const IOOptions& /*opts*/, + Slice* result, char* /*scratch*/, + IODebugContext* /*dbg*/) const override { + *result = Slice(nullptr, 0); + if (offset < ss_->contents_.size()) { + size_t str_res_sz = + std::min(static_cast(ss_->contents_.size() - offset), n); + *result = Slice(ss_->contents_.data() + offset, str_res_sz); + } + return IOStatus::OK(); + } + + IOStatus Flush(const IOOptions& /*opts*/, IODebugContext* /*dbg*/) override { + return IOStatus::OK(); + } + + IOStatus Sync(const IOOptions& /*opts*/, IODebugContext* /*dbg*/) override { + return IOStatus::OK(); + } + + IOStatus Close(const IOOptions& /*opts*/, IODebugContext* /*dbg*/) override { + return IOStatus::OK(); + } + + const std::string& contents() const { return ss_->contents(); } + + private: + StringSink* ss_; +}; + +// Like StringSink, this writes into a string. Unlink StringSink, it +// has some initial content and overwrites it, just like a recycled +// log file. +class OverwritingStringSink : public FSWritableFile { + public: + explicit OverwritingStringSink(Slice* reader_contents) + : FSWritableFile(), + contents_(""), + reader_contents_(reader_contents), + last_flush_(0) {} + + const std::string& contents() const { return contents_; } + + IOStatus Truncate(uint64_t size, const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) override { + contents_.resize(static_cast(size)); + return IOStatus::OK(); + } + IOStatus Close(const IOOptions& /*opts*/, IODebugContext* /*dbg*/) override { + return IOStatus::OK(); + } + IOStatus Flush(const IOOptions& /*opts*/, IODebugContext* /*dbg*/) override { + if (last_flush_ < contents_.size()) { + assert(reader_contents_->size() >= contents_.size()); + memcpy((char*)reader_contents_->data() + last_flush_, + contents_.data() + last_flush_, contents_.size() - last_flush_); + last_flush_ = contents_.size(); + } + return IOStatus::OK(); + } + IOStatus Sync(const IOOptions& /*opts*/, IODebugContext* /*dbg*/) override { + return IOStatus::OK(); + } + + using FSWritableFile::Append; + IOStatus Append(const Slice& slice, const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) override { + contents_.append(slice.data(), slice.size()); + return IOStatus::OK(); + } + void Drop(size_t bytes) { + contents_.resize(contents_.size() - bytes); + if (last_flush_ > contents_.size()) last_flush_ = contents_.size(); + } + + private: + std::string contents_; + Slice* reader_contents_; + size_t last_flush_; +}; + +class StringSource : public FSRandomAccessFile { + public: + explicit StringSource(const Slice& contents, uint64_t uniq_id = 0, + bool mmap = false) + : contents_(contents.data(), contents.size()), + uniq_id_(uniq_id), + mmap_(mmap), + total_reads_(0) {} + + virtual ~StringSource() {} + + uint64_t Size() const { return contents_.size(); } + + IOStatus Prefetch(uint64_t /*offset*/, size_t /*n*/, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + // If we are using mmap_, it is equivalent to performing a prefetch + if (mmap_) { + return IOStatus::OK(); + } else { + return IOStatus::NotSupported("Prefetch not supported"); + } + } + + IOStatus Read(uint64_t offset, size_t n, const IOOptions& /*opts*/, + Slice* result, char* scratch, + IODebugContext* /*dbg*/) const override { + total_reads_++; + if (offset > contents_.size()) { + return IOStatus::InvalidArgument("invalid Read offset"); + } + if (offset + n > contents_.size()) { + n = contents_.size() - static_cast(offset); + } + if (!mmap_) { + memcpy(scratch, &contents_[static_cast(offset)], n); + *result = Slice(scratch, n); + } else { + *result = Slice(&contents_[static_cast(offset)], n); + } + return IOStatus::OK(); + } + + size_t GetUniqueId(char* id, size_t max_size) const override { + if (max_size < 20) { + return 0; + } + + char* rid = id; + rid = EncodeVarint64(rid, uniq_id_); + rid = EncodeVarint64(rid, 0); + return static_cast(rid - id); + } + + int total_reads() const { return total_reads_; } + + void set_total_reads(int tr) { total_reads_ = tr; } + + private: + std::string contents_; + uint64_t uniq_id_; + bool mmap_; + mutable int total_reads_; +}; + +class NullLogger : public Logger { + public: + using Logger::Logv; + virtual void Logv(const char* /*format*/, va_list /*ap*/) override {} + virtual size_t GetLogFileSize() const override { return 0; } +}; + +// Corrupts key by changing the type +extern void CorruptKeyType(InternalKey* ikey); + +extern std::string KeyStr(const std::string& user_key, + const SequenceNumber& seq, const ValueType& t, + bool corrupt = false); + +extern std::string KeyStr(uint64_t ts, const std::string& user_key, + const SequenceNumber& seq, const ValueType& t, + bool corrupt = false); + +class SleepingBackgroundTask { + public: + SleepingBackgroundTask() + : bg_cv_(&mutex_), + should_sleep_(true), + done_with_sleep_(false), + sleeping_(false) {} + + ~SleepingBackgroundTask() { + MutexLock l(&mutex_); + should_sleep_ = false; + while (sleeping_) { + assert(!should_sleep_); + bg_cv_.SignalAll(); + bg_cv_.Wait(); + } + } + + bool IsSleeping() { + MutexLock l(&mutex_); + return sleeping_; + } + void DoSleep() { + MutexLock l(&mutex_); + sleeping_ = true; + bg_cv_.SignalAll(); + while (should_sleep_) { + bg_cv_.Wait(); + } + sleeping_ = false; + done_with_sleep_ = true; + bg_cv_.SignalAll(); + } + void WaitUntilSleeping() { + MutexLock l(&mutex_); + while (!sleeping_ || !should_sleep_) { + bg_cv_.Wait(); + } + } + // Waits for the status to change to sleeping, + // otherwise times out. + // wait_time is in microseconds. + // Returns true when times out, false otherwise. + bool TimedWaitUntilSleeping(uint64_t wait_time); + + void WakeUp() { + MutexLock l(&mutex_); + should_sleep_ = false; + bg_cv_.SignalAll(); + } + void WaitUntilDone() { + MutexLock l(&mutex_); + while (!done_with_sleep_) { + bg_cv_.Wait(); + } + } + // Similar to TimedWaitUntilSleeping. + // Waits until the task is done. + bool TimedWaitUntilDone(uint64_t wait_time); + + bool WokenUp() { + MutexLock l(&mutex_); + return should_sleep_ == false; + } + + void Reset() { + MutexLock l(&mutex_); + should_sleep_ = true; + done_with_sleep_ = false; + } + + static void DoSleepTask(void* arg) { + reinterpret_cast(arg)->DoSleep(); + } + + private: + port::Mutex mutex_; + port::CondVar bg_cv_; // Signalled when background work finishes + bool should_sleep_; + bool done_with_sleep_; + bool sleeping_; +}; + +// Filters merge operands and values that are equal to `num`. +class FilterNumber : public CompactionFilter { + public: + explicit FilterNumber(uint64_t num) : num_(num) {} + + std::string last_merge_operand_key() { return last_merge_operand_key_; } + + bool Filter(int /*level*/, const ROCKSDB_NAMESPACE::Slice& /*key*/, + const ROCKSDB_NAMESPACE::Slice& value, std::string* /*new_value*/, + bool* /*value_changed*/) const override { + if (value.size() == sizeof(uint64_t)) { + return num_ == DecodeFixed64(value.data()); + } + return true; + } + + bool FilterMergeOperand( + int /*level*/, const ROCKSDB_NAMESPACE::Slice& key, + const ROCKSDB_NAMESPACE::Slice& value) const override { + last_merge_operand_key_ = key.ToString(); + if (value.size() == sizeof(uint64_t)) { + return num_ == DecodeFixed64(value.data()); + } + return true; + } + + const char* Name() const override { return "FilterBadMergeOperand"; } + + private: + mutable std::string last_merge_operand_key_; + uint64_t num_; +}; + +inline std::string EncodeInt(uint64_t x) { + std::string result; + PutFixed64(&result, x); + return result; +} + +class SeqStringSource : public FSSequentialFile { + public: + SeqStringSource(const std::string& data, std::atomic* read_count) + : data_(data), offset_(0), read_count_(read_count) {} + ~SeqStringSource() override {} + IOStatus Read(size_t n, const IOOptions& /*opts*/, Slice* result, + char* scratch, IODebugContext* /*dbg*/) override { + std::string output; + if (offset_ < data_.size()) { + n = std::min(data_.size() - offset_, n); + memcpy(scratch, data_.data() + offset_, n); + offset_ += n; + *result = Slice(scratch, n); + } else { + return IOStatus::InvalidArgument( + "Attempt to read when it already reached eof."); + } + (*read_count_)++; + return IOStatus::OK(); + } + + IOStatus Skip(uint64_t n) override { + if (offset_ >= data_.size()) { + return IOStatus::InvalidArgument( + "Attempt to read when it already reached eof."); + } + // TODO(yhchiang): Currently doesn't handle the overflow case. + offset_ += static_cast(n); + return IOStatus::OK(); + } + + private: + std::string data_; + size_t offset_; + std::atomic* read_count_; +}; + +class StringFS : public FileSystemWrapper { + public: + class StringSink : public FSWritableFile { + public: + explicit StringSink(std::string* contents) + : FSWritableFile(), contents_(contents) {} + IOStatus Truncate(uint64_t size, const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) override { + contents_->resize(static_cast(size)); + return IOStatus::OK(); + } + IOStatus Close(const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) override { + return IOStatus::OK(); + } + IOStatus Flush(const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) override { + return IOStatus::OK(); + } + IOStatus Sync(const IOOptions& /*opts*/, IODebugContext* /*dbg*/) override { + return IOStatus::OK(); + } + + using FSWritableFile::Append; + IOStatus Append(const Slice& slice, const IOOptions& /*opts*/, + IODebugContext* /*dbg*/) override { + contents_->append(slice.data(), slice.size()); + return IOStatus::OK(); + } + + private: + std::string* contents_; + }; + + explicit StringFS(const std::shared_ptr& t) + : FileSystemWrapper(t) {} + ~StringFS() override {} + + static const char* kClassName() { return "StringFS"; } + const char* Name() const override { return kClassName(); } + + const std::string& GetContent(const std::string& f) { return files_[f]; } + + const IOStatus WriteToNewFile(const std::string& file_name, + const std::string& content) { + std::unique_ptr r; + FileOptions file_opts; + IOOptions io_opts; + + auto s = NewWritableFile(file_name, file_opts, &r, nullptr); + if (s.ok()) { + s = r->Append(content, io_opts, nullptr); + } + if (s.ok()) { + s = r->Flush(io_opts, nullptr); + } + if (s.ok()) { + s = r->Close(io_opts, nullptr); + } + assert(!s.ok() || files_[file_name] == content); + return s; + } + + // The following text is boilerplate that forwards all methods to target() + IOStatus NewSequentialFile(const std::string& f, + const FileOptions& /*options*/, + std::unique_ptr* r, + IODebugContext* /*dbg*/) override { + auto iter = files_.find(f); + if (iter == files_.end()) { + return IOStatus::NotFound("The specified file does not exist", f); + } + r->reset(new SeqStringSource(iter->second, &num_seq_file_read_)); + return IOStatus::OK(); + } + + IOStatus NewRandomAccessFile(const std::string& /*f*/, + const FileOptions& /*options*/, + std::unique_ptr* /*r*/, + IODebugContext* /*dbg*/) override { + return IOStatus::NotSupported(); + } + + IOStatus NewWritableFile(const std::string& f, const FileOptions& /*options*/, + std::unique_ptr* r, + IODebugContext* /*dbg*/) override { + auto iter = files_.find(f); + if (iter != files_.end()) { + return IOStatus::IOError("The specified file already exists", f); + } + r->reset(new StringSink(&files_[f])); + return IOStatus::OK(); + } + IOStatus NewDirectory(const std::string& /*name*/, + const IOOptions& /*options*/, + std::unique_ptr* /*result*/, + IODebugContext* /*dbg*/) override { + return IOStatus::NotSupported(); + } + + IOStatus FileExists(const std::string& f, const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + if (files_.find(f) == files_.end()) { + return IOStatus::NotFound(); + } + return IOStatus::OK(); + } + + IOStatus GetChildren(const std::string& /*dir*/, const IOOptions& /*options*/, + std::vector* /*r*/, + IODebugContext* /*dbg*/) override { + return IOStatus::NotSupported(); + } + + IOStatus DeleteFile(const std::string& f, const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + files_.erase(f); + return IOStatus::OK(); + } + + IOStatus CreateDir(const std::string& /*d*/, const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return IOStatus::NotSupported(); + } + + IOStatus CreateDirIfMissing(const std::string& /*d*/, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return IOStatus::NotSupported(); + } + + IOStatus DeleteDir(const std::string& /*d*/, const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return IOStatus::NotSupported(); + } + + IOStatus GetFileSize(const std::string& f, const IOOptions& /*options*/, + uint64_t* s, IODebugContext* /*dbg*/) override { + auto iter = files_.find(f); + if (iter == files_.end()) { + return IOStatus::NotFound("The specified file does not exist:", f); + } + *s = iter->second.size(); + return IOStatus::OK(); + } + + IOStatus GetFileModificationTime(const std::string& /*fname*/, + const IOOptions& /*options*/, + uint64_t* /*file_mtime*/, + IODebugContext* /*dbg*/) override { + return IOStatus::NotSupported(); + } + + IOStatus RenameFile(const std::string& /*s*/, const std::string& /*t*/, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return IOStatus::NotSupported(); + } + + IOStatus LinkFile(const std::string& /*s*/, const std::string& /*t*/, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return IOStatus::NotSupported(); + } + + IOStatus LockFile(const std::string& /*f*/, const IOOptions& /*options*/, + FileLock** /*l*/, IODebugContext* /*dbg*/) override { + return IOStatus::NotSupported(); + } + + IOStatus UnlockFile(FileLock* /*l*/, const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override { + return IOStatus::NotSupported(); + } + + std::atomic num_seq_file_read_; + + protected: + std::unordered_map files_; +}; + +// Randomly initialize the given DBOptions +void RandomInitDBOptions(DBOptions* db_opt, Random* rnd); + +// Randomly initialize the given ColumnFamilyOptions +// Note that the caller is responsible for releasing non-null +// cf_opt->compaction_filter. +void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, DBOptions&, Random* rnd); + +// A dummy merge operator which can change its name +class ChanglingMergeOperator : public MergeOperator { + public: + explicit ChanglingMergeOperator(const std::string& name) + : name_(name + "MergeOperator") {} + ~ChanglingMergeOperator() {} + + void SetName(const std::string& name) { name_ = name; } + + virtual bool FullMergeV2(const MergeOperationInput& /*merge_in*/, + MergeOperationOutput* /*merge_out*/) const override { + return false; + } + virtual bool PartialMergeMulti(const Slice& /*key*/, + const std::deque& /*operand_list*/, + std::string* /*new_value*/, + Logger* /*logger*/) const override { + return false; + } + static const char* kClassName() { return "ChanglingMergeOperator"; } + const char* NickName() const override { return kNickName(); } + static const char* kNickName() { return "Changling"; } + bool IsInstanceOf(const std::string& id) const override { + if (id == kClassName()) { + return true; + } else { + return MergeOperator::IsInstanceOf(id); + } + } + + virtual const char* Name() const override { return name_.c_str(); } + + protected: + std::string name_; +}; + +// Returns a dummy merge operator with random name. +MergeOperator* RandomMergeOperator(Random* rnd); + +// A dummy compaction filter which can change its name +class ChanglingCompactionFilter : public CompactionFilter { + public: + explicit ChanglingCompactionFilter(const std::string& name) + : name_(name + "CompactionFilter") {} + ~ChanglingCompactionFilter() {} + + void SetName(const std::string& name) { name_ = name; } + + bool Filter(int /*level*/, const Slice& /*key*/, + const Slice& /*existing_value*/, std::string* /*new_value*/, + bool* /*value_changed*/) const override { + return false; + } + + static const char* kClassName() { return "ChanglingCompactionFilter"; } + const char* NickName() const override { return kNickName(); } + static const char* kNickName() { return "Changling"; } + + bool IsInstanceOf(const std::string& id) const override { + if (id == kClassName()) { + return true; + } else { + return CompactionFilter::IsInstanceOf(id); + } + } + + const char* Name() const override { return name_.c_str(); } + + private: + std::string name_; +}; + +// Returns a dummy compaction filter with a random name. +CompactionFilter* RandomCompactionFilter(Random* rnd); + +// A dummy compaction filter factory which can change its name +class ChanglingCompactionFilterFactory : public CompactionFilterFactory { + public: + explicit ChanglingCompactionFilterFactory(const std::string& name) + : name_(name + "CompactionFilterFactory") {} + ~ChanglingCompactionFilterFactory() {} + + void SetName(const std::string& name) { name_ = name; } + + std::unique_ptr CreateCompactionFilter( + const CompactionFilter::Context& /*context*/) override { + return std::unique_ptr(); + } + + // Returns a name that identifies this compaction filter factory. + const char* Name() const override { return name_.c_str(); } + static const char* kClassName() { return "ChanglingCompactionFilterFactory"; } + const char* NickName() const override { return kNickName(); } + static const char* kNickName() { return "Changling"; } + + bool IsInstanceOf(const std::string& id) const override { + if (id == kClassName()) { + return true; + } else { + return CompactionFilterFactory::IsInstanceOf(id); + } + } + + protected: + std::string name_; +}; + +// The factory for the hacky skip list mem table that triggers flush after +// number of entries exceeds a threshold. +extern MemTableRepFactory* NewSpecialSkipListFactory(int num_entries_per_flush); + +CompressionType RandomCompressionType(Random* rnd); + +void RandomCompressionTypeVector(const size_t count, + std::vector* types, + Random* rnd); + +CompactionFilterFactory* RandomCompactionFilterFactory(Random* rnd); + +const SliceTransform* RandomSliceTransform(Random* rnd, int pre_defined = -1); + +TableFactory* RandomTableFactory(Random* rnd, int pre_defined = -1); + +std::string RandomName(Random* rnd, const size_t len); + +bool IsDirectIOSupported(Env* env, const std::string& dir); + +bool IsPrefetchSupported(const std::shared_ptr& fs, + const std::string& dir); + +// Return the number of lines where a given pattern was found in a file. +size_t GetLinesCount(const std::string& fname, const std::string& pattern); + +Status CorruptFile(Env* env, const std::string& fname, int offset, + int bytes_to_corrupt, bool verify_checksum = true); +Status TruncateFile(Env* env, const std::string& fname, uint64_t length); + +// Try and delete a directory if it exists +Status TryDeleteDir(Env* env, const std::string& dirname); + +// Delete a directory if it exists +void DeleteDir(Env* env, const std::string& dirname); + +// Creates an Env from the system environment by looking at the system +// environment variables. +Status CreateEnvFromSystem(const ConfigOptions& options, Env** result, + std::shared_ptr* guard); + +// Registers the testutil classes with the ObjectLibrary +int RegisterTestObjects(ObjectLibrary& library, const std::string& /*arg*/); + +// Register the testutil classes with the default ObjectRegistry/Library +void RegisterTestLibrary(const std::string& arg = ""); +} // namespace test +} // namespace ROCKSDB_NAMESPACE diff --git a/librocksdb-sys/rocksdb/test_util/testutil_test.cc b/librocksdb-sys/rocksdb/test_util/testutil_test.cc new file mode 100644 index 0000000..41f26e3 --- /dev/null +++ b/librocksdb-sys/rocksdb/test_util/testutil_test.cc @@ -0,0 +1,43 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "test_util/testutil.h" + +#include "file/file_util.h" +#include "port/port.h" +#include "port/stack_trace.h" +#include "test_util/testharness.h" + +namespace ROCKSDB_NAMESPACE { + +void CreateFile(Env* env, const std::string& path) { + std::unique_ptr f; + ASSERT_OK(env->NewWritableFile(path, &f, EnvOptions())); + f->Close(); +} + +TEST(TestUtil, DestroyDirRecursively) { + auto env = Env::Default(); + // test_util/file + // /dir + // /dir/file + std::string test_dir = test::PerThreadDBPath("test_util"); + ASSERT_OK(env->CreateDir(test_dir)); + CreateFile(env, test_dir + "/file"); + ASSERT_OK(env->CreateDir(test_dir + "/dir")); + CreateFile(env, test_dir + "/dir/file"); + + ASSERT_OK(DestroyDir(env, test_dir)); + auto s = env->FileExists(test_dir); + ASSERT_TRUE(s.IsNotFound()); +} + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/test_util/transaction_test_util.cc b/librocksdb-sys/rocksdb/test_util/transaction_test_util.cc new file mode 100644 index 0000000..11fca6d --- /dev/null +++ b/librocksdb-sys/rocksdb/test_util/transaction_test_util.cc @@ -0,0 +1,400 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "test_util/transaction_test_util.h" + +#include +#include +#include +#include +#include +#include + +#include "db/dbformat.h" +#include "db/snapshot_impl.h" +#include "logging/logging.h" +#include "rocksdb/db.h" +#include "rocksdb/utilities/optimistic_transaction_db.h" +#include "rocksdb/utilities/transaction.h" +#include "rocksdb/utilities/transaction_db.h" +#include "util/random.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +RandomTransactionInserter::RandomTransactionInserter( + Random64* rand, const WriteOptions& write_options, + const ReadOptions& read_options, uint64_t num_keys, uint16_t num_sets, + const uint64_t cmt_delay_ms, const uint64_t first_id) + : rand_(rand), + write_options_(write_options), + read_options_(read_options), + num_keys_(num_keys), + num_sets_(num_sets), + txn_id_(first_id), + cmt_delay_ms_(cmt_delay_ms) {} + +RandomTransactionInserter::~RandomTransactionInserter() { + if (txn_ != nullptr) { + delete txn_; + } + if (optimistic_txn_ != nullptr) { + delete optimistic_txn_; + } +} + +bool RandomTransactionInserter::TransactionDBInsert( + TransactionDB* db, const TransactionOptions& txn_options) { + txn_ = db->BeginTransaction(write_options_, txn_options, txn_); + + std::hash hasher; + char name[64]; + snprintf(name, 64, "txn%" ROCKSDB_PRIszt "-%" PRIu64, + hasher(std::this_thread::get_id()), txn_id_++); + assert(strlen(name) < 64 - 1); + assert(txn_->SetName(name).ok()); + + // Take a snapshot if set_snapshot was not set or with 50% change otherwise + bool take_snapshot = txn_->GetSnapshot() == nullptr || rand_->OneIn(2); + if (take_snapshot) { + txn_->SetSnapshot(); + read_options_.snapshot = txn_->GetSnapshot(); + } + auto res = DoInsert(db, txn_, false); + if (take_snapshot) { + read_options_.snapshot = nullptr; + } + return res; +} + +bool RandomTransactionInserter::OptimisticTransactionDBInsert( + OptimisticTransactionDB* db, + const OptimisticTransactionOptions& txn_options) { + optimistic_txn_ = + db->BeginTransaction(write_options_, txn_options, optimistic_txn_); + + return DoInsert(db, optimistic_txn_, true); +} + +bool RandomTransactionInserter::DBInsert(DB* db) { + return DoInsert(db, nullptr, false); +} + +Status RandomTransactionInserter::DBGet( + DB* db, Transaction* txn, ReadOptions& read_options, uint16_t set_i, + uint64_t ikey, bool get_for_update, uint64_t* int_value, + std::string* full_key, bool* unexpected_error) { + Status s; + // Five digits (since the largest uint16_t is 65535) plus the NUL + // end char. + char prefix_buf[6] = {0}; + // Pad prefix appropriately so we can iterate over each set + assert(set_i + 1 <= 9999); + snprintf(prefix_buf, sizeof(prefix_buf), "%.4u", set_i + 1); + // key format: [SET#][random#] + std::string skey = std::to_string(ikey); + Slice base_key(skey); + *full_key = std::string(prefix_buf) + base_key.ToString(); + Slice key(*full_key); + + std::string value; + if (txn != nullptr) { + if (get_for_update) { + s = txn->GetForUpdate(read_options, key, &value); + } else { + s = txn->Get(read_options, key, &value); + } + } else { + s = db->Get(read_options, key, &value); + } + + if (s.ok()) { + // Found key, parse its value + *int_value = std::stoull(value); + if (*int_value == 0 || *int_value == ULONG_MAX) { + *unexpected_error = true; + fprintf(stderr, "Get returned unexpected value: %s\n", value.c_str()); + s = Status::Corruption(); + } + } else if (s.IsNotFound()) { + // Have not yet written to this key, so assume its value is 0 + *int_value = 0; + s = Status::OK(); + } + return s; +} + +bool RandomTransactionInserter::DoInsert(DB* db, Transaction* txn, + bool is_optimistic) { + Status s; + WriteBatch batch; + + // pick a random number to use to increment a key in each set + uint64_t incr = (rand_->Next() % 100) + 1; + bool unexpected_error = false; + + std::vector set_vec(num_sets_); + std::iota(set_vec.begin(), set_vec.end(), static_cast(0)); + RandomShuffle(set_vec.begin(), set_vec.end()); + + // For each set, pick a key at random and increment it + for (uint16_t set_i : set_vec) { + uint64_t int_value = 0; + std::string full_key; + uint64_t rand_key = rand_->Next() % num_keys_; + const bool get_for_update = txn ? rand_->OneIn(2) : false; + s = DBGet(db, txn, read_options_, set_i, rand_key, get_for_update, + &int_value, &full_key, &unexpected_error); + Slice key(full_key); + if (!s.ok()) { + // Optimistic transactions should never return non-ok status here. + // Non-optimistic transactions may return write-coflict/timeout errors. + if (is_optimistic || !(s.IsBusy() || s.IsTimedOut() || s.IsTryAgain())) { + fprintf(stderr, "Get returned an unexpected error: %s\n", + s.ToString().c_str()); + unexpected_error = true; + } + break; + } + + if (s.ok()) { + // Increment key + std::string sum = std::to_string(int_value + incr); + if (txn != nullptr) { + if ((set_i % 4) != 0) { + s = txn->SingleDelete(key); + } else { + s = txn->Delete(key); + } + if (!get_for_update && (s.IsBusy() || s.IsTimedOut())) { + // If the initial get was not for update, then the key is not locked + // before put and put could fail due to concurrent writes. + break; + } else if (!s.ok()) { + // Since we did a GetForUpdate, SingleDelete should not fail. + fprintf(stderr, "SingleDelete returned an unexpected error: %s\n", + s.ToString().c_str()); + unexpected_error = true; + } + s = txn->Put(key, sum); + if (!s.ok()) { + // Since we did a GetForUpdate, Put should not fail. + fprintf(stderr, "Put returned an unexpected error: %s\n", + s.ToString().c_str()); + unexpected_error = true; + } + } else { + batch.Put(key, sum); + } + bytes_inserted_ += key.size() + sum.size(); + } + if (txn != nullptr) { + ROCKS_LOG_DEBUG(db->GetDBOptions().info_log, + "Insert (%s) %s snap: %" PRIu64 " key:%s value: %" PRIu64 + "+%" PRIu64 "=%" PRIu64, + txn->GetName().c_str(), s.ToString().c_str(), + txn->GetSnapshot()->GetSequenceNumber(), full_key.c_str(), + int_value, incr, int_value + incr); + } + } + + if (s.ok()) { + if (txn != nullptr) { + bool with_prepare = !is_optimistic && !rand_->OneIn(10); + if (with_prepare) { + // Also try commit without prepare + s = txn->Prepare(); + if (!s.ok()) { + fprintf(stderr, "Prepare returned an unexpected error: %s\n", + s.ToString().c_str()); + } + assert(s.ok()); + ROCKS_LOG_DEBUG(db->GetDBOptions().info_log, + "Prepare of %" PRIu64 " %s (%s)", txn->GetId(), + s.ToString().c_str(), txn->GetName().c_str()); + if (rand_->OneIn(20)) { + // This currently only tests the mechanics of writing commit time + // write batch so the exact values would not matter. + s = txn_->GetCommitTimeWriteBatch()->Put("cat", "dog"); + assert(s.ok()); + } + db->GetDBOptions().env->SleepForMicroseconds( + static_cast(cmt_delay_ms_ * 1000)); + } + if (!rand_->OneIn(20)) { + s = txn->Commit(); + assert(!with_prepare || s.ok()); + ROCKS_LOG_DEBUG(db->GetDBOptions().info_log, + "Commit of %" PRIu64 " %s (%s)", txn->GetId(), + s.ToString().c_str(), txn->GetName().c_str()); + } else { + // Also try 5% rollback + s = txn->Rollback(); + ROCKS_LOG_DEBUG(db->GetDBOptions().info_log, + "Rollback %" PRIu64 " %s %s", txn->GetId(), + txn->GetName().c_str(), s.ToString().c_str()); + assert(s.ok()); + } + assert(is_optimistic || s.ok()); + + if (!s.ok()) { + if (is_optimistic) { + // Optimistic transactions can have write-conflict errors on commit. + // Any other error is unexpected. + if (!(s.IsBusy() || s.IsTimedOut() || s.IsTryAgain())) { + unexpected_error = true; + } + } else { + // Non-optimistic transactions should only fail due to expiration + // or write failures. For testing purproses, we do not expect any + // write failures. + if (!s.IsExpired()) { + unexpected_error = true; + } + } + + if (unexpected_error) { + fprintf(stderr, "Commit returned an unexpected error: %s\n", + s.ToString().c_str()); + } + } + } else { + s = db->Write(write_options_, &batch); + if (!s.ok()) { + unexpected_error = true; + fprintf(stderr, "Write returned an unexpected error: %s\n", + s.ToString().c_str()); + } + } + } else { + if (txn != nullptr) { + assert(txn->Rollback().ok()); + ROCKS_LOG_DEBUG(db->GetDBOptions().info_log, "Error %s for txn %s", + s.ToString().c_str(), txn->GetName().c_str()); + } + } + + if (s.ok()) { + success_count_++; + } else { + failure_count_++; + } + + last_status_ = s; + + // return success if we didn't get any unexpected errors + return !unexpected_error; +} + +// Verify that the sum of the keys in each set are equal +Status RandomTransactionInserter::Verify(DB* db, uint16_t num_sets, + uint64_t num_keys_per_set, + bool take_snapshot, Random64* rand, + uint64_t delay_ms) { + // delay_ms is the delay between taking a snapshot and doing the reads. It + // emulates reads from a long-running backup job. + assert(delay_ms == 0 || take_snapshot); + uint64_t prev_total = 0; + uint32_t prev_i = 0; + bool prev_assigned = false; + + ReadOptions roptions; + if (take_snapshot) { + roptions.snapshot = db->GetSnapshot(); + db->GetDBOptions().env->SleepForMicroseconds( + static_cast(delay_ms * 1000)); + } + + std::vector set_vec(num_sets); + std::iota(set_vec.begin(), set_vec.end(), static_cast(0)); + RandomShuffle(set_vec.begin(), set_vec.end()); + + // For each set of keys with the same prefix, sum all the values + for (uint16_t set_i : set_vec) { + // Five digits (since the largest uint16_t is 65535) plus the NUL + // end char. + char prefix_buf[6]; + assert(set_i + 1 <= 9999); + snprintf(prefix_buf, sizeof(prefix_buf), "%.4u", set_i + 1); + uint64_t total = 0; + + // Use either point lookup or iterator. Point lookups are slower so we use + // it less often. + const bool use_point_lookup = + num_keys_per_set != 0 && rand && rand->OneIn(10); + if (use_point_lookup) { + ReadOptions read_options; + for (uint64_t k = 0; k < num_keys_per_set; k++) { + std::string dont_care; + uint64_t int_value = 0; + bool unexpected_error = false; + const bool FOR_UPDATE = false; + Status s = DBGet(db, nullptr, roptions, set_i, k, FOR_UPDATE, + &int_value, &dont_care, &unexpected_error); + assert(s.ok()); + assert(!unexpected_error); + total += int_value; + } + } else { // user iterators + Iterator* iter = db->NewIterator(roptions); + for (iter->Seek(Slice(prefix_buf, 4)); iter->Valid(); iter->Next()) { + Slice key = iter->key(); + // stop when we reach a different prefix + if (key.ToString().compare(0, 4, prefix_buf) != 0) { + break; + } + Slice value = iter->value(); + uint64_t int_value = std::stoull(value.ToString()); + if (int_value == 0 || int_value == ULONG_MAX) { + fprintf(stderr, "Iter returned unexpected value: %s\n", + value.ToString().c_str()); + return Status::Corruption(); + } + ROCKS_LOG_DEBUG( + db->GetDBOptions().info_log, + "VerifyRead at %" PRIu64 " (%" PRIu64 "): %.*s value: %" PRIu64, + roptions.snapshot ? roptions.snapshot->GetSequenceNumber() : 0ul, + roptions.snapshot + ? ((SnapshotImpl*)roptions.snapshot)->min_uncommitted_ + : 0ul, + static_cast(key.size()), key.data(), int_value); + total += int_value; + } + iter->status().PermitUncheckedError(); + delete iter; + } + + if (prev_assigned && total != prev_total) { + db->GetDBOptions().info_log->Flush(); + fprintf(stdout, + "RandomTransactionVerify found inconsistent totals using " + "pointlookup? %d " + "Set[%" PRIu32 "]: %" PRIu64 ", Set[%" PRIu32 "]: %" PRIu64 + " at snapshot %" PRIu64 "\n", + use_point_lookup, prev_i, prev_total, set_i, total, + roptions.snapshot ? roptions.snapshot->GetSequenceNumber() : 0ul); + fflush(stdout); + return Status::Corruption(); + } else { + ROCKS_LOG_DEBUG( + db->GetDBOptions().info_log, + "RandomTransactionVerify pass pointlookup? %d total: %" PRIu64 + " snap: %" PRIu64, + use_point_lookup, total, + roptions.snapshot ? roptions.snapshot->GetSequenceNumber() : 0ul); + } + prev_total = total; + prev_i = set_i; + prev_assigned = true; + } + if (take_snapshot) { + db->ReleaseSnapshot(roptions.snapshot); + } + + return Status::OK(); +} + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/test_util/transaction_test_util.h b/librocksdb-sys/rocksdb/test_util/transaction_test_util.h new file mode 100644 index 0000000..284b392 --- /dev/null +++ b/librocksdb-sys/rocksdb/test_util/transaction_test_util.h @@ -0,0 +1,147 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + + +#include "port/port.h" +#include "rocksdb/options.h" +#include "rocksdb/utilities/optimistic_transaction_db.h" +#include "rocksdb/utilities/transaction_db.h" + +namespace ROCKSDB_NAMESPACE { + +class DB; +class Random64; + +// Utility class for stress testing transactions. Can be used to write many +// transactions in parallel and then validate that the data written is logically +// consistent. This class assumes the input DB is initially empty. +// +// Each call to TransactionDBInsert()/OptimisticTransactionDBInsert() will +// increment the value of a key in #num_sets sets of keys. Regardless of +// whether the transaction succeeds, the total sum of values of keys in each +// set is an invariant that should remain equal. +// +// After calling TransactionDBInsert()/OptimisticTransactionDBInsert() many +// times, Verify() can be called to validate that the invariant holds. +// +// To test writing Transaction in parallel, multiple threads can create a +// RandomTransactionInserter with similar arguments using the same DB. +class RandomTransactionInserter { + public: + static bool RollbackDeletionTypeCallback(const Slice& key) { + // These are hard-coded atm. See how RandomTransactionInserter::DoInsert() + // determines whether to use SingleDelete or Delete for a key. + assert(key.size() >= 4); + const char* ptr = key.data(); + assert(ptr); + while (ptr && ptr < key.data() + 4 && *ptr == '0') { + ++ptr; + } + std::string prefix(ptr, 4 - (ptr - key.data())); + unsigned long set_i = std::stoul(prefix); + assert(set_i > 0); + assert(set_i <= 9999); + --set_i; + return ((set_i % 4) != 0); + } + + // num_keys is the number of keys in each set. + // num_sets is the number of sets of keys. + // cmt_delay_ms is the delay between prepare (if there is any) and commit + // first_id is the id of the first transaction + explicit RandomTransactionInserter( + Random64* rand, const WriteOptions& write_options = WriteOptions(), + const ReadOptions& read_options = ReadOptions(), uint64_t num_keys = 1000, + uint16_t num_sets = 3, const uint64_t cmt_delay_ms = 0, + const uint64_t first_id = 0); + + ~RandomTransactionInserter(); + + // Increment a key in each set using a Transaction on a TransactionDB. + // + // Returns true if the transaction succeeded OR if any error encountered was + // expected (eg a write-conflict). Error status may be obtained by calling + // GetLastStatus(); + bool TransactionDBInsert( + TransactionDB* db, + const TransactionOptions& txn_options = TransactionOptions()); + + // Increment a key in each set using a Transaction on an + // OptimisticTransactionDB + // + // Returns true if the transaction succeeded OR if any error encountered was + // expected (eg a write-conflict). Error status may be obtained by calling + // GetLastStatus(); + bool OptimisticTransactionDBInsert( + OptimisticTransactionDB* db, + const OptimisticTransactionOptions& txn_options = + OptimisticTransactionOptions()); + // Increment a key in each set without using a transaction. If this function + // is called in parallel, then Verify() may fail. + // + // Returns true if the write succeeds. + // Error status may be obtained by calling GetLastStatus(). + bool DBInsert(DB* db); + + // Get the ikey'th key from set set_i + static Status DBGet(DB* db, Transaction* txn, ReadOptions& read_options, + uint16_t set_i, uint64_t ikey, bool get_for_update, + uint64_t* int_value, std::string* full_key, + bool* unexpected_error); + + // Returns OK if Invariant is true. + static Status Verify(DB* db, uint16_t num_sets, uint64_t num_keys_per_set = 0, + bool take_snapshot = false, Random64* rand = nullptr, + uint64_t delay_ms = 0); + + // Returns the status of the previous Insert operation + Status GetLastStatus() { return last_status_; } + + // Returns the number of successfully written calls to + // TransactionDBInsert/OptimisticTransactionDBInsert/DBInsert + uint64_t GetSuccessCount() { return success_count_; } + + // Returns the number of calls to + // TransactionDBInsert/OptimisticTransactionDBInsert/DBInsert that did not + // write any data. + uint64_t GetFailureCount() { return failure_count_; } + + // Returns the sum of user keys/values Put() to the DB. + size_t GetBytesInserted() { return bytes_inserted_; } + + private: + // Input options + Random64* rand_; + const WriteOptions write_options_; + ReadOptions read_options_; + const uint64_t num_keys_; + const uint16_t num_sets_; + + // Number of successful insert batches performed + uint64_t success_count_ = 0; + + // Number of failed insert batches attempted + uint64_t failure_count_ = 0; + + size_t bytes_inserted_ = 0; + + // Status returned by most recent insert operation + Status last_status_; + + // optimization: re-use allocated transaction objects. + Transaction* txn_ = nullptr; + Transaction* optimistic_txn_ = nullptr; + + uint64_t txn_id_; + // The delay between ::Prepare and ::Commit + const uint64_t cmt_delay_ms_; + + bool DoInsert(DB* db, Transaction* txn, bool is_optimistic); +}; + +} // namespace ROCKSDB_NAMESPACE + diff --git a/librocksdb-sys/rocksdb/third-party/gcc/ppc-asm.h b/librocksdb-sys/rocksdb/third-party/gcc/ppc-asm.h new file mode 100644 index 0000000..e0bce9c --- /dev/null +++ b/librocksdb-sys/rocksdb/third-party/gcc/ppc-asm.h @@ -0,0 +1,390 @@ +/* PowerPC asm definitions for GNU C. + +Copyright (C) 2002-2020 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 3, or (at your option) any later +version. + +GCC is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* Under winnt, 1) gas supports the following as names and 2) in particular + defining "toc" breaks the FUNC_START macro as ".toc" becomes ".2" */ + +#define r0 0 +#define sp 1 +#define toc 2 +#define r3 3 +#define r4 4 +#define r5 5 +#define r6 6 +#define r7 7 +#define r8 8 +#define r9 9 +#define r10 10 +#define r11 11 +#define r12 12 +#define r13 13 +#define r14 14 +#define r15 15 +#define r16 16 +#define r17 17 +#define r18 18 +#define r19 19 +#define r20 20 +#define r21 21 +#define r22 22 +#define r23 23 +#define r24 24 +#define r25 25 +#define r26 26 +#define r27 27 +#define r28 28 +#define r29 29 +#define r30 30 +#define r31 31 + +#define cr0 0 +#define cr1 1 +#define cr2 2 +#define cr3 3 +#define cr4 4 +#define cr5 5 +#define cr6 6 +#define cr7 7 + +#define f0 0 +#define f1 1 +#define f2 2 +#define f3 3 +#define f4 4 +#define f5 5 +#define f6 6 +#define f7 7 +#define f8 8 +#define f9 9 +#define f10 10 +#define f11 11 +#define f12 12 +#define f13 13 +#define f14 14 +#define f15 15 +#define f16 16 +#define f17 17 +#define f18 18 +#define f19 19 +#define f20 20 +#define f21 21 +#define f22 22 +#define f23 23 +#define f24 24 +#define f25 25 +#define f26 26 +#define f27 27 +#define f28 28 +#define f29 29 +#define f30 30 +#define f31 31 + +#ifdef __VSX__ +#define f32 32 +#define f33 33 +#define f34 34 +#define f35 35 +#define f36 36 +#define f37 37 +#define f38 38 +#define f39 39 +#define f40 40 +#define f41 41 +#define f42 42 +#define f43 43 +#define f44 44 +#define f45 45 +#define f46 46 +#define f47 47 +#define f48 48 +#define f49 49 +#define f50 50 +#define f51 51 +#define f52 52 +#define f53 53 +#define f54 54 +#define f55 55 +#define f56 56 +#define f57 57 +#define f58 58 +#define f59 59 +#define f60 60 +#define f61 61 +#define f62 62 +#define f63 63 +#endif + +#ifdef __ALTIVEC__ +#define v0 0 +#define v1 1 +#define v2 2 +#define v3 3 +#define v4 4 +#define v5 5 +#define v6 6 +#define v7 7 +#define v8 8 +#define v9 9 +#define v10 10 +#define v11 11 +#define v12 12 +#define v13 13 +#define v14 14 +#define v15 15 +#define v16 16 +#define v17 17 +#define v18 18 +#define v19 19 +#define v20 20 +#define v21 21 +#define v22 22 +#define v23 23 +#define v24 24 +#define v25 25 +#define v26 26 +#define v27 27 +#define v28 28 +#define v29 29 +#define v30 30 +#define v31 31 +#endif + +#ifdef __VSX__ +#define vs0 0 +#define vs1 1 +#define vs2 2 +#define vs3 3 +#define vs4 4 +#define vs5 5 +#define vs6 6 +#define vs7 7 +#define vs8 8 +#define vs9 9 +#define vs10 10 +#define vs11 11 +#define vs12 12 +#define vs13 13 +#define vs14 14 +#define vs15 15 +#define vs16 16 +#define vs17 17 +#define vs18 18 +#define vs19 19 +#define vs20 20 +#define vs21 21 +#define vs22 22 +#define vs23 23 +#define vs24 24 +#define vs25 25 +#define vs26 26 +#define vs27 27 +#define vs28 28 +#define vs29 29 +#define vs30 30 +#define vs31 31 +#define vs32 32 +#define vs33 33 +#define vs34 34 +#define vs35 35 +#define vs36 36 +#define vs37 37 +#define vs38 38 +#define vs39 39 +#define vs40 40 +#define vs41 41 +#define vs42 42 +#define vs43 43 +#define vs44 44 +#define vs45 45 +#define vs46 46 +#define vs47 47 +#define vs48 48 +#define vs49 49 +#define vs50 50 +#define vs51 51 +#define vs52 52 +#define vs53 53 +#define vs54 54 +#define vs55 55 +#define vs56 56 +#define vs57 57 +#define vs58 58 +#define vs59 59 +#define vs60 60 +#define vs61 61 +#define vs62 62 +#define vs63 63 +#endif + +/* + * Macros to glue together two tokens. + */ + +#ifdef __STDC__ +#define XGLUE(a,b) a##b +#else +#define XGLUE(a,b) a/**/b +#endif + +#define GLUE(a,b) XGLUE(a,b) + +/* + * Macros to begin and end a function written in assembler. If -mcall-aixdesc + * or -mcall-nt, create a function descriptor with the given name, and create + * the real function with one or two leading periods respectively. + */ + +#if defined(__powerpc64__) && _CALL_ELF == 2 + +/* Defining "toc" above breaks @toc in assembler code. */ +#undef toc + +#define FUNC_NAME(name) GLUE(__USER_LABEL_PREFIX__,name) +#ifdef __PCREL__ +#define JUMP_TARGET(name) GLUE(FUNC_NAME(name),@notoc) +#define FUNC_START(name) \ + .type FUNC_NAME(name),@function; \ + .globl FUNC_NAME(name); \ +FUNC_NAME(name): \ + .localentry FUNC_NAME(name),1 +#else +#define JUMP_TARGET(name) FUNC_NAME(name) +#define FUNC_START(name) \ + .type FUNC_NAME(name),@function; \ + .globl FUNC_NAME(name); \ +FUNC_NAME(name): \ +0: addis 2,12,(.TOC.-0b)@ha; \ + addi 2,2,(.TOC.-0b)@l; \ + .localentry FUNC_NAME(name),.-FUNC_NAME(name) +#endif /* !__PCREL__ */ + +#define HIDDEN_FUNC(name) \ + FUNC_START(name) \ + .hidden FUNC_NAME(name); + +#define FUNC_END(name) \ + .size FUNC_NAME(name),.-FUNC_NAME(name) + +#elif defined (__powerpc64__) + +#define FUNC_NAME(name) GLUE(.,name) +#define JUMP_TARGET(name) FUNC_NAME(name) +#define FUNC_START(name) \ + .section ".opd","aw"; \ +name: \ + .quad GLUE(.,name); \ + .quad .TOC.@tocbase; \ + .quad 0; \ + .previous; \ + .type GLUE(.,name),@function; \ + .globl name; \ + .globl GLUE(.,name); \ +GLUE(.,name): + +#define HIDDEN_FUNC(name) \ + FUNC_START(name) \ + .hidden name; \ + .hidden GLUE(.,name); + +#define FUNC_END(name) \ +GLUE(.L,name): \ + .size GLUE(.,name),GLUE(.L,name)-GLUE(.,name) + +#elif defined(_CALL_AIXDESC) + +#ifdef _RELOCATABLE +#define DESC_SECTION ".got2" +#else +#define DESC_SECTION ".got1" +#endif + +#define FUNC_NAME(name) GLUE(.,name) +#define JUMP_TARGET(name) FUNC_NAME(name) +#define FUNC_START(name) \ + .section DESC_SECTION,"aw"; \ +name: \ + .long GLUE(.,name); \ + .long _GLOBAL_OFFSET_TABLE_; \ + .long 0; \ + .previous; \ + .type GLUE(.,name),@function; \ + .globl name; \ + .globl GLUE(.,name); \ +GLUE(.,name): + +#define HIDDEN_FUNC(name) \ + FUNC_START(name) \ + .hidden name; \ + .hidden GLUE(.,name); + +#define FUNC_END(name) \ +GLUE(.L,name): \ + .size GLUE(.,name),GLUE(.L,name)-GLUE(.,name) + +#else + +#define FUNC_NAME(name) GLUE(__USER_LABEL_PREFIX__,name) +#if defined __PIC__ || defined __pic__ +#define JUMP_TARGET(name) FUNC_NAME(name@plt) +#else +#define JUMP_TARGET(name) FUNC_NAME(name) +#endif +#define FUNC_START(name) \ + .type FUNC_NAME(name),@function; \ + .globl FUNC_NAME(name); \ +FUNC_NAME(name): + +#define HIDDEN_FUNC(name) \ + FUNC_START(name) \ + .hidden FUNC_NAME(name); + +#define FUNC_END(name) \ +GLUE(.L,name): \ + .size FUNC_NAME(name),GLUE(.L,name)-FUNC_NAME(name) +#endif + +#ifdef IN_GCC +/* For HAVE_GAS_CFI_DIRECTIVE. */ +#include "auto-host.h" + +#ifdef HAVE_GAS_CFI_DIRECTIVE +# define CFI_STARTPROC .cfi_startproc +# define CFI_ENDPROC .cfi_endproc +# define CFI_OFFSET(reg, off) .cfi_offset reg, off +# define CFI_DEF_CFA_REGISTER(reg) .cfi_def_cfa_register reg +# define CFI_RESTORE(reg) .cfi_restore reg +#else +# define CFI_STARTPROC +# define CFI_ENDPROC +# define CFI_OFFSET(reg, off) +# define CFI_DEF_CFA_REGISTER(reg) +# define CFI_RESTORE(reg) +#endif +#endif + +#if defined __linux__ && !defined __powerpc64__ + .section .note.GNU-stack + .previous +#endif diff --git a/librocksdb-sys/rocksdb/third-party/gtest-1.8.1/fused-src/gtest/CMakeLists.txt b/librocksdb-sys/rocksdb/third-party/gtest-1.8.1/fused-src/gtest/CMakeLists.txt new file mode 100644 index 0000000..211e8a8 --- /dev/null +++ b/librocksdb-sys/rocksdb/third-party/gtest-1.8.1/fused-src/gtest/CMakeLists.txt @@ -0,0 +1,4 @@ +add_library(gtest gtest-all.cc) + +# Add dependency of gtest on thread library +target_link_libraries(gtest ${CMAKE_THREAD_LIBS_INIT}) diff --git a/librocksdb-sys/rocksdb/third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc b/librocksdb-sys/rocksdb/third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc new file mode 100644 index 0000000..b19c9f2 --- /dev/null +++ b/librocksdb-sys/rocksdb/third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc @@ -0,0 +1,11394 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// +// Google C++ Testing and Mocking Framework (Google Test) +// +// Sometimes it's desirable to build Google Test by compiling a single file. +// This file serves this purpose. + +// This line ensures that gtest.h can be compiled on its own, even +// when it's fused. +#include "gtest/gtest.h" + +// The following lines pull in the real gtest *.cc files. +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// +// The Google C++ Testing and Mocking Framework (Google Test) + +// Copyright 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// +// Utilities for testing Google Test itself and code that uses Google Test +// (e.g. frameworks built on top of Google Test). + +// GOOGLETEST_CM0004 DO NOT DELETE + +#ifndef GTEST_INCLUDE_GTEST_GTEST_SPI_H_ +#define GTEST_INCLUDE_GTEST_GTEST_SPI_H_ + + +GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ +/* class A needs to have dll-interface to be used by clients of class B */) + +namespace testing { + +// This helper class can be used to mock out Google Test failure reporting +// so that we can test Google Test or code that builds on Google Test. +// +// An object of this class appends a TestPartResult object to the +// TestPartResultArray object given in the constructor whenever a Google Test +// failure is reported. It can either intercept only failures that are +// generated in the same thread that created this object or it can intercept +// all generated failures. The scope of this mock object can be controlled with +// the second argument to the two arguments constructor. +class GTEST_API_ ScopedFakeTestPartResultReporter + : public TestPartResultReporterInterface { + public: + // The two possible mocking modes of this object. + enum InterceptMode { + INTERCEPT_ONLY_CURRENT_THREAD, // Intercepts only thread local failures. + INTERCEPT_ALL_THREADS // Intercepts all failures. + }; + + // The c'tor sets this object as the test part result reporter used + // by Google Test. The 'result' parameter specifies where to report the + // results. This reporter will only catch failures generated in the current + // thread. DEPRECATED + explicit ScopedFakeTestPartResultReporter(TestPartResultArray* result); + + // Same as above, but you can choose the interception scope of this object. + ScopedFakeTestPartResultReporter(InterceptMode intercept_mode, + TestPartResultArray* result); + + // The d'tor restores the previous test part result reporter. + virtual ~ScopedFakeTestPartResultReporter(); + + // Appends the TestPartResult object to the TestPartResultArray + // received in the constructor. + // + // This method is from the TestPartResultReporterInterface + // interface. + virtual void ReportTestPartResult(const TestPartResult& result); + private: + void Init(); + + const InterceptMode intercept_mode_; + TestPartResultReporterInterface* old_reporter_; + TestPartResultArray* const result_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ScopedFakeTestPartResultReporter); +}; + +namespace internal { + +// A helper class for implementing EXPECT_FATAL_FAILURE() and +// EXPECT_NONFATAL_FAILURE(). Its destructor verifies that the given +// TestPartResultArray contains exactly one failure that has the given +// type and contains the given substring. If that's not the case, a +// non-fatal failure will be generated. +class GTEST_API_ SingleFailureChecker { + public: + // The constructor remembers the arguments. + SingleFailureChecker(const TestPartResultArray* results, + TestPartResult::Type type, const std::string& substr); + ~SingleFailureChecker(); + private: + const TestPartResultArray* const results_; + const TestPartResult::Type type_; + const std::string substr_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(SingleFailureChecker); +}; + +} // namespace internal + +} // namespace testing + +GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 + +// A set of macros for testing Google Test assertions or code that's expected +// to generate Google Test fatal failures. It verifies that the given +// statement will cause exactly one fatal Google Test failure with 'substr' +// being part of the failure message. +// +// There are two different versions of this macro. EXPECT_FATAL_FAILURE only +// affects and considers failures generated in the current thread and +// EXPECT_FATAL_FAILURE_ON_ALL_THREADS does the same but for all threads. +// +// The verification of the assertion is done correctly even when the statement +// throws an exception or aborts the current function. +// +// Known restrictions: +// - 'statement' cannot reference local non-static variables or +// non-static members of the current object. +// - 'statement' cannot return a value. +// - You cannot stream a failure message to this macro. +// +// Note that even though the implementations of the following two +// macros are much alike, we cannot refactor them to use a common +// helper macro, due to some peculiarity in how the preprocessor +// works. The AcceptsMacroThatExpandsToUnprotectedComma test in +// gtest_unittest.cc will fail to compile if we do that. +#define EXPECT_FATAL_FAILURE(statement, substr) \ + do { \ + class GTestExpectFatalFailureHelper {\ + public:\ + static void Execute() { statement; }\ + };\ + ::testing::TestPartResultArray gtest_failures;\ + ::testing::internal::SingleFailureChecker gtest_checker(\ + >est_failures, ::testing::TestPartResult::kFatalFailure, (substr));\ + {\ + ::testing::ScopedFakeTestPartResultReporter gtest_reporter(\ + ::testing::ScopedFakeTestPartResultReporter:: \ + INTERCEPT_ONLY_CURRENT_THREAD, >est_failures);\ + GTestExpectFatalFailureHelper::Execute();\ + }\ + } while (::testing::internal::AlwaysFalse()) + +#define EXPECT_FATAL_FAILURE_ON_ALL_THREADS(statement, substr) \ + do { \ + class GTestExpectFatalFailureHelper {\ + public:\ + static void Execute() { statement; }\ + };\ + ::testing::TestPartResultArray gtest_failures;\ + ::testing::internal::SingleFailureChecker gtest_checker(\ + >est_failures, ::testing::TestPartResult::kFatalFailure, (substr));\ + {\ + ::testing::ScopedFakeTestPartResultReporter gtest_reporter(\ + ::testing::ScopedFakeTestPartResultReporter:: \ + INTERCEPT_ALL_THREADS, >est_failures);\ + GTestExpectFatalFailureHelper::Execute();\ + }\ + } while (::testing::internal::AlwaysFalse()) + +// A macro for testing Google Test assertions or code that's expected to +// generate Google Test non-fatal failures. It asserts that the given +// statement will cause exactly one non-fatal Google Test failure with 'substr' +// being part of the failure message. +// +// There are two different versions of this macro. EXPECT_NONFATAL_FAILURE only +// affects and considers failures generated in the current thread and +// EXPECT_NONFATAL_FAILURE_ON_ALL_THREADS does the same but for all threads. +// +// 'statement' is allowed to reference local variables and members of +// the current object. +// +// The verification of the assertion is done correctly even when the statement +// throws an exception or aborts the current function. +// +// Known restrictions: +// - You cannot stream a failure message to this macro. +// +// Note that even though the implementations of the following two +// macros are much alike, we cannot refactor them to use a common +// helper macro, due to some peculiarity in how the preprocessor +// works. If we do that, the code won't compile when the user gives +// EXPECT_NONFATAL_FAILURE() a statement that contains a macro that +// expands to code containing an unprotected comma. The +// AcceptsMacroThatExpandsToUnprotectedComma test in gtest_unittest.cc +// catches that. +// +// For the same reason, we have to write +// if (::testing::internal::AlwaysTrue()) { statement; } +// instead of +// GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement) +// to avoid an MSVC warning on unreachable code. +#define EXPECT_NONFATAL_FAILURE(statement, substr) \ + do {\ + ::testing::TestPartResultArray gtest_failures;\ + ::testing::internal::SingleFailureChecker gtest_checker(\ + >est_failures, ::testing::TestPartResult::kNonFatalFailure, \ + (substr));\ + {\ + ::testing::ScopedFakeTestPartResultReporter gtest_reporter(\ + ::testing::ScopedFakeTestPartResultReporter:: \ + INTERCEPT_ONLY_CURRENT_THREAD, >est_failures);\ + if (::testing::internal::AlwaysTrue()) { statement; }\ + }\ + } while (::testing::internal::AlwaysFalse()) + +#define EXPECT_NONFATAL_FAILURE_ON_ALL_THREADS(statement, substr) \ + do {\ + ::testing::TestPartResultArray gtest_failures;\ + ::testing::internal::SingleFailureChecker gtest_checker(\ + >est_failures, ::testing::TestPartResult::kNonFatalFailure, \ + (substr));\ + {\ + ::testing::ScopedFakeTestPartResultReporter gtest_reporter(\ + ::testing::ScopedFakeTestPartResultReporter::INTERCEPT_ALL_THREADS, \ + >est_failures);\ + if (::testing::internal::AlwaysTrue()) { statement; }\ + }\ + } while (::testing::internal::AlwaysFalse()) + +#endif // GTEST_INCLUDE_GTEST_GTEST_SPI_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include // NOLINT +#include +#include + +#if GTEST_OS_LINUX + +// FIXME: Use autoconf to detect availability of +// gettimeofday(). +# define GTEST_HAS_GETTIMEOFDAY_ 1 + +# include // NOLINT +# include // NOLINT +# include // NOLINT +// Declares vsnprintf(). This header is not available on Windows. +# include // NOLINT +# include // NOLINT +# include // NOLINT +# include // NOLINT +# include + +#elif GTEST_OS_SYMBIAN +# define GTEST_HAS_GETTIMEOFDAY_ 1 +# include // NOLINT + +#elif GTEST_OS_ZOS +# define GTEST_HAS_GETTIMEOFDAY_ 1 +# include // NOLINT + +// On z/OS we additionally need strings.h for strcasecmp. +# include // NOLINT + +#elif GTEST_OS_WINDOWS_MOBILE // We are on Windows CE. + +# include // NOLINT +# undef min + +#elif GTEST_OS_WINDOWS // We are on Windows proper. + +# include // NOLINT +# include // NOLINT +# include // NOLINT +# include // NOLINT + +# if GTEST_OS_WINDOWS_MINGW +// MinGW has gettimeofday() but not _ftime64(). +// FIXME: Use autoconf to detect availability of +// gettimeofday(). +// FIXME: There are other ways to get the time on +// Windows, like GetTickCount() or GetSystemTimeAsFileTime(). MinGW +// supports these. consider using them instead. +# define GTEST_HAS_GETTIMEOFDAY_ 1 +# include // NOLINT +# endif // GTEST_OS_WINDOWS_MINGW + +// cpplint thinks that the header is already included, so we want to +// silence it. +# include // NOLINT +# undef min + +#else + +// Assume other platforms have gettimeofday(). +// FIXME: Use autoconf to detect availability of +// gettimeofday(). +# define GTEST_HAS_GETTIMEOFDAY_ 1 + +// cpplint thinks that the header is already included, so we want to +// silence it. +# include // NOLINT +# include // NOLINT + +#endif // GTEST_OS_LINUX + +#if GTEST_HAS_EXCEPTIONS +# include +#endif + +#if GTEST_CAN_STREAM_RESULTS_ +# include // NOLINT +# include // NOLINT +# include // NOLINT +# include // NOLINT +#endif + +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Utility functions and classes used by the Google C++ testing framework.// +// This file contains purely Google Test's internal implementation. Please +// DO NOT #INCLUDE IT IN A USER PROGRAM. + +#ifndef GTEST_SRC_GTEST_INTERNAL_INL_H_ +#define GTEST_SRC_GTEST_INTERNAL_INL_H_ + +#ifndef _WIN32_WCE +# include +#endif // !_WIN32_WCE +#include +#include // For strtoll/_strtoul64/malloc/free. +#include // For memmove. + +#include +#include +#include + + +#if GTEST_CAN_STREAM_RESULTS_ +# include // NOLINT +# include // NOLINT +#endif + +#if GTEST_OS_WINDOWS +# include // NOLINT +#endif // GTEST_OS_WINDOWS + + +GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ +/* class A needs to have dll-interface to be used by clients of class B */) + +namespace testing { + +// Declares the flags. +// +// We don't want the users to modify this flag in the code, but want +// Google Test's own unit tests to be able to access it. Therefore we +// declare it here as opposed to in gtest.h. +GTEST_DECLARE_bool_(death_test_use_fork); + +namespace internal { + +// The value of GetTestTypeId() as seen from within the Google Test +// library. This is solely for testing GetTestTypeId(). +GTEST_API_ extern const TypeId kTestTypeIdInGoogleTest; + +// Names of the flags (needed for parsing Google Test flags). +const char kAlsoRunDisabledTestsFlag[] = "also_run_disabled_tests"; +const char kBreakOnFailureFlag[] = "break_on_failure"; +const char kCatchExceptionsFlag[] = "catch_exceptions"; +const char kColorFlag[] = "color"; +const char kFilterFlag[] = "filter"; +const char kListTestsFlag[] = "list_tests"; +const char kOutputFlag[] = "output"; +const char kPrintTimeFlag[] = "print_time"; +const char kPrintUTF8Flag[] = "print_utf8"; +const char kRandomSeedFlag[] = "random_seed"; +const char kRepeatFlag[] = "repeat"; +const char kShuffleFlag[] = "shuffle"; +const char kStackTraceDepthFlag[] = "stack_trace_depth"; +const char kStreamResultToFlag[] = "stream_result_to"; +const char kThrowOnFailureFlag[] = "throw_on_failure"; +const char kFlagfileFlag[] = "flagfile"; + +// A valid random seed must be in [1, kMaxRandomSeed]. +const int kMaxRandomSeed = 99999; + +// g_help_flag is true iff the --help flag or an equivalent form is +// specified on the command line. +GTEST_API_ extern bool g_help_flag; + +// Returns the current time in milliseconds. +GTEST_API_ TimeInMillis GetTimeInMillis(); + +// Returns true iff Google Test should use colors in the output. +GTEST_API_ bool ShouldUseColor(bool stdout_is_tty); + +// Formats the given time in milliseconds as seconds. +GTEST_API_ std::string FormatTimeInMillisAsSeconds(TimeInMillis ms); + +// Converts the given time in milliseconds to a date string in the ISO 8601 +// format, without the timezone information. N.B.: due to the use the +// non-reentrant localtime() function, this function is not thread safe. Do +// not use it in any code that can be called from multiple threads. +GTEST_API_ std::string FormatEpochTimeInMillisAsIso8601(TimeInMillis ms); + +// Parses a string for an Int32 flag, in the form of "--flag=value". +// +// On success, stores the value of the flag in *value, and returns +// true. On failure, returns false without changing *value. +GTEST_API_ bool ParseInt32Flag( + const char* str, const char* flag, Int32* value); + +// Returns a random seed in range [1, kMaxRandomSeed] based on the +// given --gtest_random_seed flag value. +inline int GetRandomSeedFromFlag(Int32 random_seed_flag) { + const unsigned int raw_seed = (random_seed_flag == 0) ? + static_cast(GetTimeInMillis()) : + static_cast(random_seed_flag); + + // Normalizes the actual seed to range [1, kMaxRandomSeed] such that + // it's easy to type. + const int normalized_seed = + static_cast((raw_seed - 1U) % + static_cast(kMaxRandomSeed)) + 1; + return normalized_seed; +} + +// Returns the first valid random seed after 'seed'. The behavior is +// undefined if 'seed' is invalid. The seed after kMaxRandomSeed is +// considered to be 1. +inline int GetNextRandomSeed(int seed) { + GTEST_CHECK_(1 <= seed && seed <= kMaxRandomSeed) + << "Invalid random seed " << seed << " - must be in [1, " + << kMaxRandomSeed << "]."; + const int next_seed = seed + 1; + return (next_seed > kMaxRandomSeed) ? 1 : next_seed; +} + +// This class saves the values of all Google Test flags in its c'tor, and +// restores them in its d'tor. +class GTestFlagSaver { + public: + // The c'tor. + GTestFlagSaver() { + also_run_disabled_tests_ = GTEST_FLAG(also_run_disabled_tests); + break_on_failure_ = GTEST_FLAG(break_on_failure); + catch_exceptions_ = GTEST_FLAG(catch_exceptions); + color_ = GTEST_FLAG(color); + death_test_style_ = GTEST_FLAG(death_test_style); + death_test_use_fork_ = GTEST_FLAG(death_test_use_fork); + filter_ = GTEST_FLAG(filter); + internal_run_death_test_ = GTEST_FLAG(internal_run_death_test); + list_tests_ = GTEST_FLAG(list_tests); + output_ = GTEST_FLAG(output); + print_time_ = GTEST_FLAG(print_time); + print_utf8_ = GTEST_FLAG(print_utf8); + random_seed_ = GTEST_FLAG(random_seed); + repeat_ = GTEST_FLAG(repeat); + shuffle_ = GTEST_FLAG(shuffle); + stack_trace_depth_ = GTEST_FLAG(stack_trace_depth); + stream_result_to_ = GTEST_FLAG(stream_result_to); + throw_on_failure_ = GTEST_FLAG(throw_on_failure); + } + + // The d'tor is not virtual. DO NOT INHERIT FROM THIS CLASS. + ~GTestFlagSaver() { + GTEST_FLAG(also_run_disabled_tests) = also_run_disabled_tests_; + GTEST_FLAG(break_on_failure) = break_on_failure_; + GTEST_FLAG(catch_exceptions) = catch_exceptions_; + GTEST_FLAG(color) = color_; + GTEST_FLAG(death_test_style) = death_test_style_; + GTEST_FLAG(death_test_use_fork) = death_test_use_fork_; + GTEST_FLAG(filter) = filter_; + GTEST_FLAG(internal_run_death_test) = internal_run_death_test_; + GTEST_FLAG(list_tests) = list_tests_; + GTEST_FLAG(output) = output_; + GTEST_FLAG(print_time) = print_time_; + GTEST_FLAG(print_utf8) = print_utf8_; + GTEST_FLAG(random_seed) = random_seed_; + GTEST_FLAG(repeat) = repeat_; + GTEST_FLAG(shuffle) = shuffle_; + GTEST_FLAG(stack_trace_depth) = stack_trace_depth_; + GTEST_FLAG(stream_result_to) = stream_result_to_; + GTEST_FLAG(throw_on_failure) = throw_on_failure_; + } + + private: + // Fields for saving the original values of flags. + bool also_run_disabled_tests_; + bool break_on_failure_; + bool catch_exceptions_; + std::string color_; + std::string death_test_style_; + bool death_test_use_fork_; + std::string filter_; + std::string internal_run_death_test_; + bool list_tests_; + std::string output_; + bool print_time_; + bool print_utf8_; + internal::Int32 random_seed_; + internal::Int32 repeat_; + bool shuffle_; + internal::Int32 stack_trace_depth_; + std::string stream_result_to_; + bool throw_on_failure_; +} GTEST_ATTRIBUTE_UNUSED_; + +// Converts a Unicode code point to a narrow string in UTF-8 encoding. +// code_point parameter is of type UInt32 because wchar_t may not be +// wide enough to contain a code point. +// If the code_point is not a valid Unicode code point +// (i.e. outside of Unicode range U+0 to U+10FFFF) it will be converted +// to "(Invalid Unicode 0xXXXXXXXX)". +GTEST_API_ std::string CodePointToUtf8(UInt32 code_point); + +// Converts a wide string to a narrow string in UTF-8 encoding. +// The wide string is assumed to have the following encoding: +// UTF-16 if sizeof(wchar_t) == 2 (on Windows, Cygwin, Symbian OS) +// UTF-32 if sizeof(wchar_t) == 4 (on Linux) +// Parameter str points to a null-terminated wide string. +// Parameter num_chars may additionally limit the number +// of wchar_t characters processed. -1 is used when the entire string +// should be processed. +// If the string contains code points that are not valid Unicode code points +// (i.e. outside of Unicode range U+0 to U+10FFFF) they will be output +// as '(Invalid Unicode 0xXXXXXXXX)'. If the string is in UTF16 encoding +// and contains invalid UTF-16 surrogate pairs, values in those pairs +// will be encoded as individual Unicode characters from Basic Normal Plane. +GTEST_API_ std::string WideStringToUtf8(const wchar_t* str, int num_chars); + +// Reads the GTEST_SHARD_STATUS_FILE environment variable, and creates the file +// if the variable is present. If a file already exists at this location, this +// function will write over it. If the variable is present, but the file cannot +// be created, prints an error and exits. +void WriteToShardStatusFileIfNeeded(); + +// Checks whether sharding is enabled by examining the relevant +// environment variable values. If the variables are present, +// but inconsistent (e.g., shard_index >= total_shards), prints +// an error and exits. If in_subprocess_for_death_test, sharding is +// disabled because it must only be applied to the original test +// process. Otherwise, we could filter out death tests we intended to execute. +GTEST_API_ bool ShouldShard(const char* total_shards_str, + const char* shard_index_str, + bool in_subprocess_for_death_test); + +// Parses the environment variable var as an Int32. If it is unset, +// returns default_val. If it is not an Int32, prints an error and +// and aborts. +GTEST_API_ Int32 Int32FromEnvOrDie(const char* env_var, Int32 default_val); + +// Given the total number of shards, the shard index, and the test id, +// returns true iff the test should be run on this shard. The test id is +// some arbitrary but unique non-negative integer assigned to each test +// method. Assumes that 0 <= shard_index < total_shards. +GTEST_API_ bool ShouldRunTestOnShard( + int total_shards, int shard_index, int test_id); + +// STL container utilities. + +// Returns the number of elements in the given container that satisfy +// the given predicate. +template +inline int CountIf(const Container& c, Predicate predicate) { + // Implemented as an explicit loop since std::count_if() in libCstd on + // Solaris has a non-standard signature. + int count = 0; + for (typename Container::const_iterator it = c.begin(); it != c.end(); ++it) { + if (predicate(*it)) + ++count; + } + return count; +} + +// Applies a function/functor to each element in the container. +template +void ForEach(const Container& c, Functor functor) { + std::for_each(c.begin(), c.end(), functor); +} + +// Returns the i-th element of the vector, or default_value if i is not +// in range [0, v.size()). +template +inline E GetElementOr(const std::vector& v, int i, E default_value) { + return (i < 0 || i >= static_cast(v.size())) ? default_value : v[i]; +} + +// Performs an in-place shuffle of a range of the vector's elements. +// 'begin' and 'end' are element indices as an STL-style range; +// i.e. [begin, end) are shuffled, where 'end' == size() means to +// shuffle to the end of the vector. +template +void ShuffleRange(internal::Random* random, int begin, int end, + std::vector* v) { + const int size = static_cast(v->size()); + GTEST_CHECK_(0 <= begin && begin <= size) + << "Invalid shuffle range start " << begin << ": must be in range [0, " + << size << "]."; + GTEST_CHECK_(begin <= end && end <= size) + << "Invalid shuffle range finish " << end << ": must be in range [" + << begin << ", " << size << "]."; + + // Fisher-Yates shuffle, from + // http://en.wikipedia.org/wiki/Fisher-Yates_shuffle + for (int range_width = end - begin; range_width >= 2; range_width--) { + const int last_in_range = begin + range_width - 1; + const int selected = begin + random->Generate(range_width); + std::swap((*v)[selected], (*v)[last_in_range]); + } +} + +// Performs an in-place shuffle of the vector's elements. +template +inline void Shuffle(internal::Random* random, std::vector* v) { + ShuffleRange(random, 0, static_cast(v->size()), v); +} + +// A function for deleting an object. Handy for being used as a +// functor. +template +static void Delete(T* x) { + delete x; +} + +// A predicate that checks the key of a TestProperty against a known key. +// +// TestPropertyKeyIs is copyable. +class TestPropertyKeyIs { + public: + // Constructor. + // + // TestPropertyKeyIs has NO default constructor. + explicit TestPropertyKeyIs(const std::string& key) : key_(key) {} + + // Returns true iff the test name of test property matches on key_. + bool operator()(const TestProperty& test_property) const { + return test_property.key() == key_; + } + + private: + std::string key_; +}; + +// Class UnitTestOptions. +// +// This class contains functions for processing options the user +// specifies when running the tests. It has only static members. +// +// In most cases, the user can specify an option using either an +// environment variable or a command line flag. E.g. you can set the +// test filter using either GTEST_FILTER or --gtest_filter. If both +// the variable and the flag are present, the latter overrides the +// former. +class GTEST_API_ UnitTestOptions { + public: + // Functions for processing the gtest_output flag. + + // Returns the output format, or "" for normal printed output. + static std::string GetOutputFormat(); + + // Returns the absolute path of the requested output file, or the + // default (test_detail.xml in the original working directory) if + // none was explicitly specified. + static std::string GetAbsolutePathToOutputFile(); + + // Functions for processing the gtest_filter flag. + + // Returns true iff the wildcard pattern matches the string. The + // first ':' or '\0' character in pattern marks the end of it. + // + // This recursive algorithm isn't very efficient, but is clear and + // works well enough for matching test names, which are short. + static bool PatternMatchesString(const char *pattern, const char *str); + + // Returns true iff the user-specified filter matches the test case + // name and the test name. + static bool FilterMatchesTest(const std::string &test_case_name, + const std::string &test_name); + +#if GTEST_OS_WINDOWS + // Function for supporting the gtest_catch_exception flag. + + // Returns EXCEPTION_EXECUTE_HANDLER if Google Test should handle the + // given SEH exception, or EXCEPTION_CONTINUE_SEARCH otherwise. + // This function is useful as an __except condition. + static int GTestShouldProcessSEH(DWORD exception_code); +#endif // GTEST_OS_WINDOWS + + // Returns true if "name" matches the ':' separated list of glob-style + // filters in "filter". + static bool MatchesFilter(const std::string& name, const char* filter); +}; + +// Returns the current application's name, removing directory path if that +// is present. Used by UnitTestOptions::GetOutputFile. +GTEST_API_ FilePath GetCurrentExecutableName(); + +// The role interface for getting the OS stack trace as a string. +class OsStackTraceGetterInterface { + public: + OsStackTraceGetterInterface() {} + virtual ~OsStackTraceGetterInterface() {} + + // Returns the current OS stack trace as an std::string. Parameters: + // + // max_depth - the maximum number of stack frames to be included + // in the trace. + // skip_count - the number of top frames to be skipped; doesn't count + // against max_depth. + virtual std::string CurrentStackTrace(int max_depth, int skip_count) = 0; + + // UponLeavingGTest() should be called immediately before Google Test calls + // user code. It saves some information about the current stack that + // CurrentStackTrace() will use to find and hide Google Test stack frames. + virtual void UponLeavingGTest() = 0; + + // This string is inserted in place of stack frames that are part of + // Google Test's implementation. + static const char* const kElidedFramesMarker; + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(OsStackTraceGetterInterface); +}; + +// A working implementation of the OsStackTraceGetterInterface interface. +class OsStackTraceGetter : public OsStackTraceGetterInterface { + public: + OsStackTraceGetter() {} + + virtual std::string CurrentStackTrace(int max_depth, int skip_count); + virtual void UponLeavingGTest(); + + private: +#if GTEST_HAS_ABSL + Mutex mutex_; // Protects all internal state. + + // We save the stack frame below the frame that calls user code. + // We do this because the address of the frame immediately below + // the user code changes between the call to UponLeavingGTest() + // and any calls to the stack trace code from within the user code. + void* caller_frame_ = nullptr; +#endif // GTEST_HAS_ABSL + + GTEST_DISALLOW_COPY_AND_ASSIGN_(OsStackTraceGetter); +}; + +// Information about a Google Test trace point. +struct TraceInfo { + const char* file; + int line; + std::string message; +}; + +// This is the default global test part result reporter used in UnitTestImpl. +// This class should only be used by UnitTestImpl. +class DefaultGlobalTestPartResultReporter + : public TestPartResultReporterInterface { + public: + explicit DefaultGlobalTestPartResultReporter(UnitTestImpl* unit_test); + // Implements the TestPartResultReporterInterface. Reports the test part + // result in the current test. + virtual void ReportTestPartResult(const TestPartResult& result); + + private: + UnitTestImpl* const unit_test_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(DefaultGlobalTestPartResultReporter); +}; + +// This is the default per thread test part result reporter used in +// UnitTestImpl. This class should only be used by UnitTestImpl. +class DefaultPerThreadTestPartResultReporter + : public TestPartResultReporterInterface { + public: + explicit DefaultPerThreadTestPartResultReporter(UnitTestImpl* unit_test); + // Implements the TestPartResultReporterInterface. The implementation just + // delegates to the current global test part result reporter of *unit_test_. + virtual void ReportTestPartResult(const TestPartResult& result); + + private: + UnitTestImpl* const unit_test_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(DefaultPerThreadTestPartResultReporter); +}; + +// The private implementation of the UnitTest class. We don't protect +// the methods under a mutex, as this class is not accessible by a +// user and the UnitTest class that delegates work to this class does +// proper locking. +class GTEST_API_ UnitTestImpl { + public: + explicit UnitTestImpl(UnitTest* parent); + virtual ~UnitTestImpl(); + + // There are two different ways to register your own TestPartResultReporter. + // You can register your own repoter to listen either only for test results + // from the current thread or for results from all threads. + // By default, each per-thread test result repoter just passes a new + // TestPartResult to the global test result reporter, which registers the + // test part result for the currently running test. + + // Returns the global test part result reporter. + TestPartResultReporterInterface* GetGlobalTestPartResultReporter(); + + // Sets the global test part result reporter. + void SetGlobalTestPartResultReporter( + TestPartResultReporterInterface* reporter); + + // Returns the test part result reporter for the current thread. + TestPartResultReporterInterface* GetTestPartResultReporterForCurrentThread(); + + // Sets the test part result reporter for the current thread. + void SetTestPartResultReporterForCurrentThread( + TestPartResultReporterInterface* reporter); + + // Gets the number of successful test cases. + int successful_test_case_count() const; + + // Gets the number of failed test cases. + int failed_test_case_count() const; + + // Gets the number of all test cases. + int total_test_case_count() const; + + // Gets the number of all test cases that contain at least one test + // that should run. + int test_case_to_run_count() const; + + // Gets the number of successful tests. + int successful_test_count() const; + + // Gets the number of failed tests. + int failed_test_count() const; + + // Gets the number of disabled tests that will be reported in the XML report. + int reportable_disabled_test_count() const; + + // Gets the number of disabled tests. + int disabled_test_count() const; + + // Gets the number of tests to be printed in the XML report. + int reportable_test_count() const; + + // Gets the number of all tests. + int total_test_count() const; + + // Gets the number of tests that should run. + int test_to_run_count() const; + + // Gets the time of the test program start, in ms from the start of the + // UNIX epoch. + TimeInMillis start_timestamp() const { return start_timestamp_; } + + // Gets the elapsed time, in milliseconds. + TimeInMillis elapsed_time() const { return elapsed_time_; } + + // Returns true iff the unit test passed (i.e. all test cases passed). + bool Passed() const { return !Failed(); } + + // Returns true iff the unit test failed (i.e. some test case failed + // or something outside of all tests failed). + bool Failed() const { + return failed_test_case_count() > 0 || ad_hoc_test_result()->Failed(); + } + + // Gets the i-th test case among all the test cases. i can range from 0 to + // total_test_case_count() - 1. If i is not in that range, returns NULL. + const TestCase* GetTestCase(int i) const { + const int index = GetElementOr(test_case_indices_, i, -1); + return index < 0 ? NULL : test_cases_[i]; + } + + // Gets the i-th test case among all the test cases. i can range from 0 to + // total_test_case_count() - 1. If i is not in that range, returns NULL. + TestCase* GetMutableTestCase(int i) { + const int index = GetElementOr(test_case_indices_, i, -1); + return index < 0 ? NULL : test_cases_[index]; + } + + // Provides access to the event listener list. + TestEventListeners* listeners() { return &listeners_; } + + // Returns the TestResult for the test that's currently running, or + // the TestResult for the ad hoc test if no test is running. + TestResult* current_test_result(); + + // Returns the TestResult for the ad hoc test. + const TestResult* ad_hoc_test_result() const { return &ad_hoc_test_result_; } + + // Sets the OS stack trace getter. + // + // Does nothing if the input and the current OS stack trace getter + // are the same; otherwise, deletes the old getter and makes the + // input the current getter. + void set_os_stack_trace_getter(OsStackTraceGetterInterface* getter); + + // Returns the current OS stack trace getter if it is not NULL; + // otherwise, creates an OsStackTraceGetter, makes it the current + // getter, and returns it. + OsStackTraceGetterInterface* os_stack_trace_getter(); + + // Returns the current OS stack trace as an std::string. + // + // The maximum number of stack frames to be included is specified by + // the gtest_stack_trace_depth flag. The skip_count parameter + // specifies the number of top frames to be skipped, which doesn't + // count against the number of frames to be included. + // + // For example, if Foo() calls Bar(), which in turn calls + // CurrentOsStackTraceExceptTop(1), Foo() will be included in the + // trace but Bar() and CurrentOsStackTraceExceptTop() won't. + std::string CurrentOsStackTraceExceptTop(int skip_count) GTEST_NO_INLINE_; + + // Finds and returns a TestCase with the given name. If one doesn't + // exist, creates one and returns it. + // + // Arguments: + // + // test_case_name: name of the test case + // type_param: the name of the test's type parameter, or NULL if + // this is not a typed or a type-parameterized test. + // set_up_tc: pointer to the function that sets up the test case + // tear_down_tc: pointer to the function that tears down the test case + TestCase* GetTestCase(const char* test_case_name, + const char* type_param, + Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc); + + // Adds a TestInfo to the unit test. + // + // Arguments: + // + // set_up_tc: pointer to the function that sets up the test case + // tear_down_tc: pointer to the function that tears down the test case + // test_info: the TestInfo object + void AddTestInfo(Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc, + TestInfo* test_info) { + // In order to support thread-safe death tests, we need to + // remember the original working directory when the test program + // was first invoked. We cannot do this in RUN_ALL_TESTS(), as + // the user may have changed the current directory before calling + // RUN_ALL_TESTS(). Therefore we capture the current directory in + // AddTestInfo(), which is called to register a TEST or TEST_F + // before main() is reached. + if (original_working_dir_.IsEmpty()) { + original_working_dir_.Set(FilePath::GetCurrentDir()); + GTEST_CHECK_(!original_working_dir_.IsEmpty()) + << "Failed to get the current working directory."; + } + + GetTestCase(test_info->test_case_name(), + test_info->type_param(), + set_up_tc, + tear_down_tc)->AddTestInfo(test_info); + } + + // Returns ParameterizedTestCaseRegistry object used to keep track of + // value-parameterized tests and instantiate and register them. + internal::ParameterizedTestCaseRegistry& parameterized_test_registry() { + return parameterized_test_registry_; + } + + // Sets the TestCase object for the test that's currently running. + void set_current_test_case(TestCase* a_current_test_case) { + current_test_case_ = a_current_test_case; + } + + // Sets the TestInfo object for the test that's currently running. If + // current_test_info is NULL, the assertion results will be stored in + // ad_hoc_test_result_. + void set_current_test_info(TestInfo* a_current_test_info) { + current_test_info_ = a_current_test_info; + } + + // Registers all parameterized tests defined using TEST_P and + // INSTANTIATE_TEST_CASE_P, creating regular tests for each test/parameter + // combination. This method can be called more then once; it has guards + // protecting from registering the tests more then once. If + // value-parameterized tests are disabled, RegisterParameterizedTests is + // present but does nothing. + void RegisterParameterizedTests(); + + // Runs all tests in this UnitTest object, prints the result, and + // returns true if all tests are successful. If any exception is + // thrown during a test, this test is considered to be failed, but + // the rest of the tests will still be run. + bool RunAllTests(); + + // Clears the results of all tests, except the ad hoc tests. + void ClearNonAdHocTestResult() { + ForEach(test_cases_, TestCase::ClearTestCaseResult); + } + + // Clears the results of ad-hoc test assertions. + void ClearAdHocTestResult() { + ad_hoc_test_result_.Clear(); + } + + // Adds a TestProperty to the current TestResult object when invoked in a + // context of a test or a test case, or to the global property set. If the + // result already contains a property with the same key, the value will be + // updated. + void RecordProperty(const TestProperty& test_property); + + enum ReactionToSharding { + HONOR_SHARDING_PROTOCOL, + IGNORE_SHARDING_PROTOCOL + }; + + // Matches the full name of each test against the user-specified + // filter to decide whether the test should run, then records the + // result in each TestCase and TestInfo object. + // If shard_tests == HONOR_SHARDING_PROTOCOL, further filters tests + // based on sharding variables in the environment. + // Returns the number of tests that should run. + int FilterTests(ReactionToSharding shard_tests); + + // Prints the names of the tests matching the user-specified filter flag. + void ListTestsMatchingFilter(); + + const TestCase* current_test_case() const { return current_test_case_; } + TestInfo* current_test_info() { return current_test_info_; } + const TestInfo* current_test_info() const { return current_test_info_; } + + // Returns the vector of environments that need to be set-up/torn-down + // before/after the tests are run. + std::vector& environments() { return environments_; } + + // Getters for the per-thread Google Test trace stack. + std::vector& gtest_trace_stack() { + return *(gtest_trace_stack_.pointer()); + } + const std::vector& gtest_trace_stack() const { + return gtest_trace_stack_.get(); + } + +#if GTEST_HAS_DEATH_TEST + void InitDeathTestSubprocessControlInfo() { + internal_run_death_test_flag_.reset(ParseInternalRunDeathTestFlag()); + } + // Returns a pointer to the parsed --gtest_internal_run_death_test + // flag, or NULL if that flag was not specified. + // This information is useful only in a death test child process. + // Must not be called before a call to InitGoogleTest. + const InternalRunDeathTestFlag* internal_run_death_test_flag() const { + return internal_run_death_test_flag_.get(); + } + + // Returns a pointer to the current death test factory. + internal::DeathTestFactory* death_test_factory() { + return death_test_factory_.get(); + } + + void SuppressTestEventsIfInSubprocess(); + + friend class ReplaceDeathTestFactory; +#endif // GTEST_HAS_DEATH_TEST + + // Initializes the event listener performing XML output as specified by + // UnitTestOptions. Must not be called before InitGoogleTest. + void ConfigureXmlOutput(); + +#if GTEST_CAN_STREAM_RESULTS_ + // Initializes the event listener for streaming test results to a socket. + // Must not be called before InitGoogleTest. + void ConfigureStreamingOutput(); +#endif + + // Performs initialization dependent upon flag values obtained in + // ParseGoogleTestFlagsOnly. Is called from InitGoogleTest after the call to + // ParseGoogleTestFlagsOnly. In case a user neglects to call InitGoogleTest + // this function is also called from RunAllTests. Since this function can be + // called more than once, it has to be idempotent. + void PostFlagParsingInit(); + + // Gets the random seed used at the start of the current test iteration. + int random_seed() const { return random_seed_; } + + // Gets the random number generator. + internal::Random* random() { return &random_; } + + // Shuffles all test cases, and the tests within each test case, + // making sure that death tests are still run first. + void ShuffleTests(); + + // Restores the test cases and tests to their order before the first shuffle. + void UnshuffleTests(); + + // Returns the value of GTEST_FLAG(catch_exceptions) at the moment + // UnitTest::Run() starts. + bool catch_exceptions() const { return catch_exceptions_; } + + private: + friend class ::testing::UnitTest; + + // Used by UnitTest::Run() to capture the state of + // GTEST_FLAG(catch_exceptions) at the moment it starts. + void set_catch_exceptions(bool value) { catch_exceptions_ = value; } + + // The UnitTest object that owns this implementation object. + UnitTest* const parent_; + + // The working directory when the first TEST() or TEST_F() was + // executed. + internal::FilePath original_working_dir_; + + // The default test part result reporters. + DefaultGlobalTestPartResultReporter default_global_test_part_result_reporter_; + DefaultPerThreadTestPartResultReporter + default_per_thread_test_part_result_reporter_; + + // Points to (but doesn't own) the global test part result reporter. + TestPartResultReporterInterface* global_test_part_result_repoter_; + + // Protects read and write access to global_test_part_result_reporter_. + internal::Mutex global_test_part_result_reporter_mutex_; + + // Points to (but doesn't own) the per-thread test part result reporter. + internal::ThreadLocal + per_thread_test_part_result_reporter_; + + // The vector of environments that need to be set-up/torn-down + // before/after the tests are run. + std::vector environments_; + + // The vector of TestCases in their original order. It owns the + // elements in the vector. + std::vector test_cases_; + + // Provides a level of indirection for the test case list to allow + // easy shuffling and restoring the test case order. The i-th + // element of this vector is the index of the i-th test case in the + // shuffled order. + std::vector test_case_indices_; + + // ParameterizedTestRegistry object used to register value-parameterized + // tests. + internal::ParameterizedTestCaseRegistry parameterized_test_registry_; + + // Indicates whether RegisterParameterizedTests() has been called already. + bool parameterized_tests_registered_; + + // Index of the last death test case registered. Initially -1. + int last_death_test_case_; + + // This points to the TestCase for the currently running test. It + // changes as Google Test goes through one test case after another. + // When no test is running, this is set to NULL and Google Test + // stores assertion results in ad_hoc_test_result_. Initially NULL. + TestCase* current_test_case_; + + // This points to the TestInfo for the currently running test. It + // changes as Google Test goes through one test after another. When + // no test is running, this is set to NULL and Google Test stores + // assertion results in ad_hoc_test_result_. Initially NULL. + TestInfo* current_test_info_; + + // Normally, a user only writes assertions inside a TEST or TEST_F, + // or inside a function called by a TEST or TEST_F. Since Google + // Test keeps track of which test is current running, it can + // associate such an assertion with the test it belongs to. + // + // If an assertion is encountered when no TEST or TEST_F is running, + // Google Test attributes the assertion result to an imaginary "ad hoc" + // test, and records the result in ad_hoc_test_result_. + TestResult ad_hoc_test_result_; + + // The list of event listeners that can be used to track events inside + // Google Test. + TestEventListeners listeners_; + + // The OS stack trace getter. Will be deleted when the UnitTest + // object is destructed. By default, an OsStackTraceGetter is used, + // but the user can set this field to use a custom getter if that is + // desired. + OsStackTraceGetterInterface* os_stack_trace_getter_; + + // True iff PostFlagParsingInit() has been called. + bool post_flag_parse_init_performed_; + + // The random number seed used at the beginning of the test run. + int random_seed_; + + // Our random number generator. + internal::Random random_; + + // The time of the test program start, in ms from the start of the + // UNIX epoch. + TimeInMillis start_timestamp_; + + // How long the test took to run, in milliseconds. + TimeInMillis elapsed_time_; + +#if GTEST_HAS_DEATH_TEST + // The decomposed components of the gtest_internal_run_death_test flag, + // parsed when RUN_ALL_TESTS is called. + internal::scoped_ptr internal_run_death_test_flag_; + internal::scoped_ptr death_test_factory_; +#endif // GTEST_HAS_DEATH_TEST + + // A per-thread stack of traces created by the SCOPED_TRACE() macro. + internal::ThreadLocal > gtest_trace_stack_; + + // The value of GTEST_FLAG(catch_exceptions) at the moment RunAllTests() + // starts. + bool catch_exceptions_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(UnitTestImpl); +}; // class UnitTestImpl + +// Convenience function for accessing the global UnitTest +// implementation object. +inline UnitTestImpl* GetUnitTestImpl() { + return UnitTest::GetInstance()->impl(); +} + +#if GTEST_USES_SIMPLE_RE + +// Internal helper functions for implementing the simple regular +// expression matcher. +GTEST_API_ bool IsInSet(char ch, const char* str); +GTEST_API_ bool IsAsciiDigit(char ch); +GTEST_API_ bool IsAsciiPunct(char ch); +GTEST_API_ bool IsRepeat(char ch); +GTEST_API_ bool IsAsciiWhiteSpace(char ch); +GTEST_API_ bool IsAsciiWordChar(char ch); +GTEST_API_ bool IsValidEscape(char ch); +GTEST_API_ bool AtomMatchesChar(bool escaped, char pattern, char ch); +GTEST_API_ bool ValidateRegex(const char* regex); +GTEST_API_ bool MatchRegexAtHead(const char* regex, const char* str); +GTEST_API_ bool MatchRepetitionAndRegexAtHead( + bool escaped, char ch, char repeat, const char* regex, const char* str); +GTEST_API_ bool MatchRegexAnywhere(const char* regex, const char* str); + +#endif // GTEST_USES_SIMPLE_RE + +// Parses the command line for Google Test flags, without initializing +// other parts of Google Test. +GTEST_API_ void ParseGoogleTestFlagsOnly(int* argc, char** argv); +GTEST_API_ void ParseGoogleTestFlagsOnly(int* argc, wchar_t** argv); + +#if GTEST_HAS_DEATH_TEST + +// Returns the message describing the last system error, regardless of the +// platform. +GTEST_API_ std::string GetLastErrnoDescription(); + +// Attempts to parse a string into a positive integer pointed to by the +// number parameter. Returns true if that is possible. +// GTEST_HAS_DEATH_TEST implies that we have ::std::string, so we can use +// it here. +template +bool ParseNaturalNumber(const ::std::string& str, Integer* number) { + // Fail fast if the given string does not begin with a digit; + // this bypasses strtoXXX's "optional leading whitespace and plus + // or minus sign" semantics, which are undesirable here. + if (str.empty() || !IsDigit(str[0])) { + return false; + } + errno = 0; + + char* end; + // BiggestConvertible is the largest integer type that system-provided + // string-to-number conversion routines can return. + +# if GTEST_OS_WINDOWS && !defined(__GNUC__) + + // MSVC and C++ Builder define __int64 instead of the standard long long. + typedef unsigned __int64 BiggestConvertible; + const BiggestConvertible parsed = _strtoui64(str.c_str(), &end, 10); + +# else + + typedef unsigned long long BiggestConvertible; // NOLINT + const BiggestConvertible parsed = strtoull(str.c_str(), &end, 10); + +# endif // GTEST_OS_WINDOWS && !defined(__GNUC__) + + const bool parse_success = *end == '\0' && errno == 0; + + // FIXME: Convert this to compile time assertion when it is + // available. + GTEST_CHECK_(sizeof(Integer) <= sizeof(parsed)); + + const Integer result = static_cast(parsed); + if (parse_success && static_cast(result) == parsed) { + *number = result; + return true; + } + return false; +} +#endif // GTEST_HAS_DEATH_TEST + +// TestResult contains some private methods that should be hidden from +// Google Test user but are required for testing. This class allow our tests +// to access them. +// +// This class is supplied only for the purpose of testing Google Test's own +// constructs. Do not use it in user tests, either directly or indirectly. +class TestResultAccessor { + public: + static void RecordProperty(TestResult* test_result, + const std::string& xml_element, + const TestProperty& property) { + test_result->RecordProperty(xml_element, property); + } + + static void ClearTestPartResults(TestResult* test_result) { + test_result->ClearTestPartResults(); + } + + static const std::vector& test_part_results( + const TestResult& test_result) { + return test_result.test_part_results(); + } +}; + +#if GTEST_CAN_STREAM_RESULTS_ + +// Streams test results to the given port on the given host machine. +class StreamingListener : public EmptyTestEventListener { + public: + // Abstract base class for writing strings to a socket. + class AbstractSocketWriter { + public: + virtual ~AbstractSocketWriter() {} + + // Sends a string to the socket. + virtual void Send(const std::string& message) = 0; + + // Closes the socket. + virtual void CloseConnection() {} + + // Sends a string and a newline to the socket. + void SendLn(const std::string& message) { Send(message + "\n"); } + }; + + // Concrete class for actually writing strings to a socket. + class SocketWriter : public AbstractSocketWriter { + public: + SocketWriter(const std::string& host, const std::string& port) + : sockfd_(-1), host_name_(host), port_num_(port) { + MakeConnection(); + } + + virtual ~SocketWriter() { + if (sockfd_ != -1) + CloseConnection(); + } + + // Sends a string to the socket. + virtual void Send(const std::string& message) { + GTEST_CHECK_(sockfd_ != -1) + << "Send() can be called only when there is a connection."; + + const int len = static_cast(message.length()); + if (write(sockfd_, message.c_str(), len) != len) { + GTEST_LOG_(WARNING) + << "stream_result_to: failed to stream to " + << host_name_ << ":" << port_num_; + } + } + + private: + // Creates a client socket and connects to the server. + void MakeConnection(); + + // Closes the socket. + void CloseConnection() { + GTEST_CHECK_(sockfd_ != -1) + << "CloseConnection() can be called only when there is a connection."; + + close(sockfd_); + sockfd_ = -1; + } + + int sockfd_; // socket file descriptor + const std::string host_name_; + const std::string port_num_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(SocketWriter); + }; // class SocketWriter + + // Escapes '=', '&', '%', and '\n' characters in str as "%xx". + static std::string UrlEncode(const char* str); + + StreamingListener(const std::string& host, const std::string& port) + : socket_writer_(new SocketWriter(host, port)) { + Start(); + } + + explicit StreamingListener(AbstractSocketWriter* socket_writer) + : socket_writer_(socket_writer) { Start(); } + + void OnTestProgramStart(const UnitTest& /* unit_test */) { + SendLn("event=TestProgramStart"); + } + + void OnTestProgramEnd(const UnitTest& unit_test) { + // Note that Google Test current only report elapsed time for each + // test iteration, not for the entire test program. + SendLn("event=TestProgramEnd&passed=" + FormatBool(unit_test.Passed())); + + // Notify the streaming server to stop. + socket_writer_->CloseConnection(); + } + + void OnTestIterationStart(const UnitTest& /* unit_test */, int iteration) { + SendLn("event=TestIterationStart&iteration=" + + StreamableToString(iteration)); + } + + void OnTestIterationEnd(const UnitTest& unit_test, int /* iteration */) { + SendLn("event=TestIterationEnd&passed=" + + FormatBool(unit_test.Passed()) + "&elapsed_time=" + + StreamableToString(unit_test.elapsed_time()) + "ms"); + } + + void OnTestCaseStart(const TestCase& test_case) { + SendLn(std::string("event=TestCaseStart&name=") + test_case.name()); + } + + void OnTestCaseEnd(const TestCase& test_case) { + SendLn("event=TestCaseEnd&passed=" + FormatBool(test_case.Passed()) + + "&elapsed_time=" + StreamableToString(test_case.elapsed_time()) + + "ms"); + } + + void OnTestStart(const TestInfo& test_info) { + SendLn(std::string("event=TestStart&name=") + test_info.name()); + } + + void OnTestEnd(const TestInfo& test_info) { + SendLn("event=TestEnd&passed=" + + FormatBool((test_info.result())->Passed()) + + "&elapsed_time=" + + StreamableToString((test_info.result())->elapsed_time()) + "ms"); + } + + void OnTestPartResult(const TestPartResult& test_part_result) { + const char* file_name = test_part_result.file_name(); + if (file_name == NULL) + file_name = ""; + SendLn("event=TestPartResult&file=" + UrlEncode(file_name) + + "&line=" + StreamableToString(test_part_result.line_number()) + + "&message=" + UrlEncode(test_part_result.message())); + } + + private: + // Sends the given message and a newline to the socket. + void SendLn(const std::string& message) { socket_writer_->SendLn(message); } + + // Called at the start of streaming to notify the receiver what + // protocol we are using. + void Start() { SendLn("gtest_streaming_protocol_version=1.0"); } + + std::string FormatBool(bool value) { return value ? "1" : "0"; } + + const scoped_ptr socket_writer_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(StreamingListener); +}; // class StreamingListener + +#endif // GTEST_CAN_STREAM_RESULTS_ + +} // namespace internal +} // namespace testing + +GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 + +#endif // GTEST_SRC_GTEST_INTERNAL_INL_H_ + +#if GTEST_OS_WINDOWS +# define vsnprintf _vsnprintf +#endif // GTEST_OS_WINDOWS + +#if GTEST_OS_MAC +#ifndef GTEST_OS_IOS +#include +#endif +#endif + +#if GTEST_HAS_ABSL +#include "absl/debugging/failure_signal_handler.h" +#include "absl/debugging/stacktrace.h" +#include "absl/debugging/symbolize.h" +#include "absl/strings/str_cat.h" +#endif // GTEST_HAS_ABSL + +namespace testing { + +using internal::CountIf; +using internal::ForEach; +using internal::GetElementOr; +using internal::Shuffle; + +// Constants. + +// A test whose test case name or test name matches this filter is +// disabled and not run. +static const char kDisableTestFilter[] = "DISABLED_*:*/DISABLED_*"; + +// A test case whose name matches this filter is considered a death +// test case and will be run before test cases whose name doesn't +// match this filter. +static const char kDeathTestCaseFilter[] = "*DeathTest:*DeathTest/*"; + +// A test filter that matches everything. +static const char kUniversalFilter[] = "*"; + +// The default output format. +static const char kDefaultOutputFormat[] = "xml"; +// The default output file. +static const char kDefaultOutputFile[] = "test_detail"; + +// The environment variable name for the test shard index. +static const char kTestShardIndex[] = "GTEST_SHARD_INDEX"; +// The environment variable name for the total number of test shards. +static const char kTestTotalShards[] = "GTEST_TOTAL_SHARDS"; +// The environment variable name for the test shard status file. +static const char kTestShardStatusFile[] = "GTEST_SHARD_STATUS_FILE"; + +namespace internal { + +// The text used in failure messages to indicate the start of the +// stack trace. +const char kStackTraceMarker[] = "\nStack trace:\n"; + +// g_help_flag is true iff the --help flag or an equivalent form is +// specified on the command line. +bool g_help_flag = false; + +// Utilty function to Open File for Writing +static FILE* OpenFileForWriting(const std::string& output_file) { + FILE* fileout = NULL; + FilePath output_file_path(output_file); + FilePath output_dir(output_file_path.RemoveFileName()); + + if (output_dir.CreateDirectoriesRecursively()) { + fileout = posix::FOpen(output_file.c_str(), "w"); + } + if (fileout == NULL) { + GTEST_LOG_(FATAL) << "Unable to open file \"" << output_file << "\""; + } + return fileout; +} + +} // namespace internal + +// Bazel passes in the argument to '--test_filter' via the TESTBRIDGE_TEST_ONLY +// environment variable. +static const char* GetDefaultFilter() { + const char* const testbridge_test_only = + internal::posix::GetEnv("TESTBRIDGE_TEST_ONLY"); + if (testbridge_test_only != NULL) { + return testbridge_test_only; + } + return kUniversalFilter; +} + +GTEST_DEFINE_bool_( + also_run_disabled_tests, + internal::BoolFromGTestEnv("also_run_disabled_tests", false), + "Run disabled tests too, in addition to the tests normally being run."); + +GTEST_DEFINE_bool_( + break_on_failure, + internal::BoolFromGTestEnv("break_on_failure", false), + "True iff a failed assertion should be a debugger break-point."); + +GTEST_DEFINE_bool_( + catch_exceptions, + internal::BoolFromGTestEnv("catch_exceptions", true), + "True iff " GTEST_NAME_ + " should catch exceptions and treat them as test failures."); + +GTEST_DEFINE_string_( + color, + internal::StringFromGTestEnv("color", "auto"), + "Whether to use colors in the output. Valid values: yes, no, " + "and auto. 'auto' means to use colors if the output is " + "being sent to a terminal and the TERM environment variable " + "is set to a terminal type that supports colors."); + +GTEST_DEFINE_string_( + filter, + internal::StringFromGTestEnv("filter", GetDefaultFilter()), + "A colon-separated list of glob (not regex) patterns " + "for filtering the tests to run, optionally followed by a " + "'-' and a : separated list of negative patterns (tests to " + "exclude). A test is run if it matches one of the positive " + "patterns and does not match any of the negative patterns."); + +GTEST_DEFINE_bool_( + install_failure_signal_handler, + internal::BoolFromGTestEnv("install_failure_signal_handler", false), + "If true and supported on the current platform, " GTEST_NAME_ " should " + "install a signal handler that dumps debugging information when fatal " + "signals are raised."); + +GTEST_DEFINE_bool_(list_tests, false, + "List all tests without running them."); + +// The net priority order after flag processing is thus: +// --gtest_output command line flag +// GTEST_OUTPUT environment variable +// XML_OUTPUT_FILE environment variable +// '' +GTEST_DEFINE_string_( + output, + internal::StringFromGTestEnv("output", + internal::OutputFlagAlsoCheckEnvVar().c_str()), + "A format (defaults to \"xml\" but can be specified to be \"json\"), " + "optionally followed by a colon and an output file name or directory. " + "A directory is indicated by a trailing pathname separator. " + "Examples: \"xml:filename.xml\", \"xml::directoryname/\". " + "If a directory is specified, output files will be created " + "within that directory, with file-names based on the test " + "executable's name and, if necessary, made unique by adding " + "digits."); + +GTEST_DEFINE_bool_( + print_time, + internal::BoolFromGTestEnv("print_time", true), + "True iff " GTEST_NAME_ + " should display elapsed time in text output."); + +GTEST_DEFINE_bool_( + print_utf8, + internal::BoolFromGTestEnv("print_utf8", true), + "True iff " GTEST_NAME_ + " prints UTF8 characters as text."); + +GTEST_DEFINE_int32_( + random_seed, + internal::Int32FromGTestEnv("random_seed", 0), + "Random number seed to use when shuffling test orders. Must be in range " + "[1, 99999], or 0 to use a seed based on the current time."); + +GTEST_DEFINE_int32_( + repeat, + internal::Int32FromGTestEnv("repeat", 1), + "How many times to repeat each test. Specify a negative number " + "for repeating forever. Useful for shaking out flaky tests."); + +GTEST_DEFINE_bool_( + show_internal_stack_frames, false, + "True iff " GTEST_NAME_ " should include internal stack frames when " + "printing test failure stack traces."); + +GTEST_DEFINE_bool_( + shuffle, + internal::BoolFromGTestEnv("shuffle", false), + "True iff " GTEST_NAME_ + " should randomize tests' order on every run."); + +GTEST_DEFINE_int32_( + stack_trace_depth, + internal::Int32FromGTestEnv("stack_trace_depth", kMaxStackTraceDepth), + "The maximum number of stack frames to print when an " + "assertion fails. The valid range is 0 through 100, inclusive."); + +GTEST_DEFINE_string_( + stream_result_to, + internal::StringFromGTestEnv("stream_result_to", ""), + "This flag specifies the host name and the port number on which to stream " + "test results. Example: \"localhost:555\". The flag is effective only on " + "Linux."); + +GTEST_DEFINE_bool_( + throw_on_failure, + internal::BoolFromGTestEnv("throw_on_failure", false), + "When this flag is specified, a failed assertion will throw an exception " + "if exceptions are enabled or exit the program with a non-zero code " + "otherwise. For use with an external test framework."); + +#if GTEST_USE_OWN_FLAGFILE_FLAG_ +GTEST_DEFINE_string_( + flagfile, + internal::StringFromGTestEnv("flagfile", ""), + "This flag specifies the flagfile to read command-line flags from."); +#endif // GTEST_USE_OWN_FLAGFILE_FLAG_ + +namespace internal { + +// Generates a random number from [0, range), using a Linear +// Congruential Generator (LCG). Crashes if 'range' is 0 or greater +// than kMaxRange. +UInt32 Random::Generate(UInt32 range) { + // These constants are the same as are used in glibc's rand(3). + // Use wider types than necessary to prevent unsigned overflow diagnostics. + state_ = static_cast(1103515245ULL*state_ + 12345U) % kMaxRange; + + GTEST_CHECK_(range > 0) + << "Cannot generate a number in the range [0, 0)."; + GTEST_CHECK_(range <= kMaxRange) + << "Generation of a number in [0, " << range << ") was requested, " + << "but this can only generate numbers in [0, " << kMaxRange << ")."; + + // Converting via modulus introduces a bit of downward bias, but + // it's simple, and a linear congruential generator isn't too good + // to begin with. + return state_ % range; +} + +// GTestIsInitialized() returns true iff the user has initialized +// Google Test. Useful for catching the user mistake of not initializing +// Google Test before calling RUN_ALL_TESTS(). +static bool GTestIsInitialized() { return GetArgvs().size() > 0; } + +// Iterates over a vector of TestCases, keeping a running sum of the +// results of calling a given int-returning method on each. +// Returns the sum. +static int SumOverTestCaseList(const std::vector& case_list, + int (TestCase::*method)() const) { + int sum = 0; + for (size_t i = 0; i < case_list.size(); i++) { + sum += (case_list[i]->*method)(); + } + return sum; +} + +// Returns true iff the test case passed. +static bool TestCasePassed(const TestCase* test_case) { + return test_case->should_run() && test_case->Passed(); +} + +// Returns true iff the test case failed. +static bool TestCaseFailed(const TestCase* test_case) { + return test_case->should_run() && test_case->Failed(); +} + +// Returns true iff test_case contains at least one test that should +// run. +static bool ShouldRunTestCase(const TestCase* test_case) { + return test_case->should_run(); +} + +// AssertHelper constructor. +AssertHelper::AssertHelper(TestPartResult::Type type, + const char* file, + int line, + const char* message) + : data_(new AssertHelperData(type, file, line, message)) { +} + +AssertHelper::~AssertHelper() { + delete data_; +} + +// Message assignment, for assertion streaming support. +void AssertHelper::operator=(const Message& message) const { + UnitTest::GetInstance()-> + AddTestPartResult(data_->type, data_->file, data_->line, + AppendUserMessage(data_->message, message), + UnitTest::GetInstance()->impl() + ->CurrentOsStackTraceExceptTop(1) + // Skips the stack frame for this function itself. + ); // NOLINT +} + +// Mutex for linked pointers. +GTEST_API_ GTEST_DEFINE_STATIC_MUTEX_(g_linked_ptr_mutex); + +// A copy of all command line arguments. Set by InitGoogleTest(). +static ::std::vector g_argvs; + +::std::vector GetArgvs() { +#if defined(GTEST_CUSTOM_GET_ARGVS_) + // GTEST_CUSTOM_GET_ARGVS_() may return a container of std::string or + // ::string. This code converts it to the appropriate type. + const auto& custom = GTEST_CUSTOM_GET_ARGVS_(); + return ::std::vector(custom.begin(), custom.end()); +#else // defined(GTEST_CUSTOM_GET_ARGVS_) + return g_argvs; +#endif // defined(GTEST_CUSTOM_GET_ARGVS_) +} + +// Returns the current application's name, removing directory path if that +// is present. +FilePath GetCurrentExecutableName() { + FilePath result; + +#if GTEST_OS_WINDOWS + result.Set(FilePath(GetArgvs()[0]).RemoveExtension("exe")); +#else + result.Set(FilePath(GetArgvs()[0])); +#endif // GTEST_OS_WINDOWS + + return result.RemoveDirectoryName(); +} + +// Functions for processing the gtest_output flag. + +// Returns the output format, or "" for normal printed output. +std::string UnitTestOptions::GetOutputFormat() { + const char* const gtest_output_flag = GTEST_FLAG(output).c_str(); + const char* const colon = strchr(gtest_output_flag, ':'); + return (colon == NULL) ? + std::string(gtest_output_flag) : + std::string(gtest_output_flag, colon - gtest_output_flag); +} + +// Returns the name of the requested output file, or the default if none +// was explicitly specified. +std::string UnitTestOptions::GetAbsolutePathToOutputFile() { + const char* const gtest_output_flag = GTEST_FLAG(output).c_str(); + + std::string format = GetOutputFormat(); + if (format.empty()) + format = std::string(kDefaultOutputFormat); + + const char* const colon = strchr(gtest_output_flag, ':'); + if (colon == NULL) + return internal::FilePath::MakeFileName( + internal::FilePath( + UnitTest::GetInstance()->original_working_dir()), + internal::FilePath(kDefaultOutputFile), 0, + format.c_str()).string(); + + internal::FilePath output_name(colon + 1); + if (!output_name.IsAbsolutePath()) + // FIXME: on Windows \some\path is not an absolute + // path (as its meaning depends on the current drive), yet the + // following logic for turning it into an absolute path is wrong. + // Fix it. + output_name = internal::FilePath::ConcatPaths( + internal::FilePath(UnitTest::GetInstance()->original_working_dir()), + internal::FilePath(colon + 1)); + + if (!output_name.IsDirectory()) + return output_name.string(); + + internal::FilePath result(internal::FilePath::GenerateUniqueFileName( + output_name, internal::GetCurrentExecutableName(), + GetOutputFormat().c_str())); + return result.string(); +} + +// Returns true iff the wildcard pattern matches the string. The +// first ':' or '\0' character in pattern marks the end of it. +// +// This recursive algorithm isn't very efficient, but is clear and +// works well enough for matching test names, which are short. +bool UnitTestOptions::PatternMatchesString(const char *pattern, + const char *str) { + switch (*pattern) { + case '\0': + case ':': // Either ':' or '\0' marks the end of the pattern. + return *str == '\0'; + case '?': // Matches any single character. + return *str != '\0' && PatternMatchesString(pattern + 1, str + 1); + case '*': // Matches any string (possibly empty) of characters. + return (*str != '\0' && PatternMatchesString(pattern, str + 1)) || + PatternMatchesString(pattern + 1, str); + default: // Non-special character. Matches itself. + return *pattern == *str && + PatternMatchesString(pattern + 1, str + 1); + } +} + +bool UnitTestOptions::MatchesFilter( + const std::string& name, const char* filter) { + const char *cur_pattern = filter; + for (;;) { + if (PatternMatchesString(cur_pattern, name.c_str())) { + return true; + } + + // Finds the next pattern in the filter. + cur_pattern = strchr(cur_pattern, ':'); + + // Returns if no more pattern can be found. + if (cur_pattern == NULL) { + return false; + } + + // Skips the pattern separater (the ':' character). + cur_pattern++; + } +} + +// Returns true iff the user-specified filter matches the test case +// name and the test name. +bool UnitTestOptions::FilterMatchesTest(const std::string &test_case_name, + const std::string &test_name) { + const std::string& full_name = test_case_name + "." + test_name.c_str(); + + // Split --gtest_filter at '-', if there is one, to separate into + // positive filter and negative filter portions + const char* const p = GTEST_FLAG(filter).c_str(); + const char* const dash = strchr(p, '-'); + std::string positive; + std::string negative; + if (dash == NULL) { + positive = GTEST_FLAG(filter).c_str(); // Whole string is a positive filter + negative = ""; + } else { + positive = std::string(p, dash); // Everything up to the dash + negative = std::string(dash + 1); // Everything after the dash + if (positive.empty()) { + // Treat '-test1' as the same as '*-test1' + positive = kUniversalFilter; + } + } + + // A filter is a colon-separated list of patterns. It matches a + // test if any pattern in it matches the test. + return (MatchesFilter(full_name, positive.c_str()) && + !MatchesFilter(full_name, negative.c_str())); +} + +#if GTEST_HAS_SEH +// Returns EXCEPTION_EXECUTE_HANDLER if Google Test should handle the +// given SEH exception, or EXCEPTION_CONTINUE_SEARCH otherwise. +// This function is useful as an __except condition. +int UnitTestOptions::GTestShouldProcessSEH(DWORD exception_code) { + // Google Test should handle a SEH exception if: + // 1. the user wants it to, AND + // 2. this is not a breakpoint exception, AND + // 3. this is not a C++ exception (VC++ implements them via SEH, + // apparently). + // + // SEH exception code for C++ exceptions. + // (see http://support.microsoft.com/kb/185294 for more information). + const DWORD kCxxExceptionCode = 0xe06d7363; + + bool should_handle = true; + + if (!GTEST_FLAG(catch_exceptions)) + should_handle = false; + else if (exception_code == EXCEPTION_BREAKPOINT) + should_handle = false; + else if (exception_code == kCxxExceptionCode) + should_handle = false; + + return should_handle ? EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH; +} +#endif // GTEST_HAS_SEH + +} // namespace internal + +// The c'tor sets this object as the test part result reporter used by +// Google Test. The 'result' parameter specifies where to report the +// results. Intercepts only failures from the current thread. +ScopedFakeTestPartResultReporter::ScopedFakeTestPartResultReporter( + TestPartResultArray* result) + : intercept_mode_(INTERCEPT_ONLY_CURRENT_THREAD), + result_(result) { + Init(); +} + +// The c'tor sets this object as the test part result reporter used by +// Google Test. The 'result' parameter specifies where to report the +// results. +ScopedFakeTestPartResultReporter::ScopedFakeTestPartResultReporter( + InterceptMode intercept_mode, TestPartResultArray* result) + : intercept_mode_(intercept_mode), + result_(result) { + Init(); +} + +void ScopedFakeTestPartResultReporter::Init() { + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + if (intercept_mode_ == INTERCEPT_ALL_THREADS) { + old_reporter_ = impl->GetGlobalTestPartResultReporter(); + impl->SetGlobalTestPartResultReporter(this); + } else { + old_reporter_ = impl->GetTestPartResultReporterForCurrentThread(); + impl->SetTestPartResultReporterForCurrentThread(this); + } +} + +// The d'tor restores the test part result reporter used by Google Test +// before. +ScopedFakeTestPartResultReporter::~ScopedFakeTestPartResultReporter() { + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + if (intercept_mode_ == INTERCEPT_ALL_THREADS) { + impl->SetGlobalTestPartResultReporter(old_reporter_); + } else { + impl->SetTestPartResultReporterForCurrentThread(old_reporter_); + } +} + +// Increments the test part result count and remembers the result. +// This method is from the TestPartResultReporterInterface interface. +void ScopedFakeTestPartResultReporter::ReportTestPartResult( + const TestPartResult& result) { + result_->Append(result); +} + +namespace internal { + +// Returns the type ID of ::testing::Test. We should always call this +// instead of GetTypeId< ::testing::Test>() to get the type ID of +// testing::Test. This is to work around a suspected linker bug when +// using Google Test as a framework on Mac OS X. The bug causes +// GetTypeId< ::testing::Test>() to return different values depending +// on whether the call is from the Google Test framework itself or +// from user test code. GetTestTypeId() is guaranteed to always +// return the same value, as it always calls GetTypeId<>() from the +// gtest.cc, which is within the Google Test framework. +TypeId GetTestTypeId() { + return GetTypeId(); +} + +// The value of GetTestTypeId() as seen from within the Google Test +// library. This is solely for testing GetTestTypeId(). +extern const TypeId kTestTypeIdInGoogleTest = GetTestTypeId(); + +// This predicate-formatter checks that 'results' contains a test part +// failure of the given type and that the failure message contains the +// given substring. +static AssertionResult HasOneFailure(const char* /* results_expr */, + const char* /* type_expr */, + const char* /* substr_expr */, + const TestPartResultArray& results, + TestPartResult::Type type, + const std::string& substr) { + const std::string expected(type == TestPartResult::kFatalFailure ? + "1 fatal failure" : + "1 non-fatal failure"); + Message msg; + if (results.size() != 1) { + msg << "Expected: " << expected << "\n" + << " Actual: " << results.size() << " failures"; + for (int i = 0; i < results.size(); i++) { + msg << "\n" << results.GetTestPartResult(i); + } + return AssertionFailure() << msg; + } + + const TestPartResult& r = results.GetTestPartResult(0); + if (r.type() != type) { + return AssertionFailure() << "Expected: " << expected << "\n" + << " Actual:\n" + << r; + } + + if (strstr(r.message(), substr.c_str()) == NULL) { + return AssertionFailure() << "Expected: " << expected << " containing \"" + << substr << "\"\n" + << " Actual:\n" + << r; + } + + return AssertionSuccess(); +} + +// The constructor of SingleFailureChecker remembers where to look up +// test part results, what type of failure we expect, and what +// substring the failure message should contain. +SingleFailureChecker::SingleFailureChecker(const TestPartResultArray* results, + TestPartResult::Type type, + const std::string& substr) + : results_(results), type_(type), substr_(substr) {} + +// The destructor of SingleFailureChecker verifies that the given +// TestPartResultArray contains exactly one failure that has the given +// type and contains the given substring. If that's not the case, a +// non-fatal failure will be generated. +SingleFailureChecker::~SingleFailureChecker() { + EXPECT_PRED_FORMAT3(HasOneFailure, *results_, type_, substr_); +} + +DefaultGlobalTestPartResultReporter::DefaultGlobalTestPartResultReporter( + UnitTestImpl* unit_test) : unit_test_(unit_test) {} + +void DefaultGlobalTestPartResultReporter::ReportTestPartResult( + const TestPartResult& result) { + unit_test_->current_test_result()->AddTestPartResult(result); + unit_test_->listeners()->repeater()->OnTestPartResult(result); +} + +DefaultPerThreadTestPartResultReporter::DefaultPerThreadTestPartResultReporter( + UnitTestImpl* unit_test) : unit_test_(unit_test) {} + +void DefaultPerThreadTestPartResultReporter::ReportTestPartResult( + const TestPartResult& result) { + unit_test_->GetGlobalTestPartResultReporter()->ReportTestPartResult(result); +} + +// Returns the global test part result reporter. +TestPartResultReporterInterface* +UnitTestImpl::GetGlobalTestPartResultReporter() { + internal::MutexLock lock(&global_test_part_result_reporter_mutex_); + return global_test_part_result_repoter_; +} + +// Sets the global test part result reporter. +void UnitTestImpl::SetGlobalTestPartResultReporter( + TestPartResultReporterInterface* reporter) { + internal::MutexLock lock(&global_test_part_result_reporter_mutex_); + global_test_part_result_repoter_ = reporter; +} + +// Returns the test part result reporter for the current thread. +TestPartResultReporterInterface* +UnitTestImpl::GetTestPartResultReporterForCurrentThread() { + return per_thread_test_part_result_reporter_.get(); +} + +// Sets the test part result reporter for the current thread. +void UnitTestImpl::SetTestPartResultReporterForCurrentThread( + TestPartResultReporterInterface* reporter) { + per_thread_test_part_result_reporter_.set(reporter); +} + +// Gets the number of successful test cases. +int UnitTestImpl::successful_test_case_count() const { + return CountIf(test_cases_, TestCasePassed); +} + +// Gets the number of failed test cases. +int UnitTestImpl::failed_test_case_count() const { + return CountIf(test_cases_, TestCaseFailed); +} + +// Gets the number of all test cases. +int UnitTestImpl::total_test_case_count() const { + return static_cast(test_cases_.size()); +} + +// Gets the number of all test cases that contain at least one test +// that should run. +int UnitTestImpl::test_case_to_run_count() const { + return CountIf(test_cases_, ShouldRunTestCase); +} + +// Gets the number of successful tests. +int UnitTestImpl::successful_test_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::successful_test_count); +} + +// Gets the number of failed tests. +int UnitTestImpl::failed_test_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::failed_test_count); +} + +// Gets the number of disabled tests that will be reported in the XML report. +int UnitTestImpl::reportable_disabled_test_count() const { + return SumOverTestCaseList(test_cases_, + &TestCase::reportable_disabled_test_count); +} + +// Gets the number of disabled tests. +int UnitTestImpl::disabled_test_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::disabled_test_count); +} + +// Gets the number of tests to be printed in the XML report. +int UnitTestImpl::reportable_test_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::reportable_test_count); +} + +// Gets the number of all tests. +int UnitTestImpl::total_test_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::total_test_count); +} + +// Gets the number of tests that should run. +int UnitTestImpl::test_to_run_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::test_to_run_count); +} + +// Returns the current OS stack trace as an std::string. +// +// The maximum number of stack frames to be included is specified by +// the gtest_stack_trace_depth flag. The skip_count parameter +// specifies the number of top frames to be skipped, which doesn't +// count against the number of frames to be included. +// +// For example, if Foo() calls Bar(), which in turn calls +// CurrentOsStackTraceExceptTop(1), Foo() will be included in the +// trace but Bar() and CurrentOsStackTraceExceptTop() won't. +std::string UnitTestImpl::CurrentOsStackTraceExceptTop(int skip_count) { + return os_stack_trace_getter()->CurrentStackTrace( + static_cast(GTEST_FLAG(stack_trace_depth)), + skip_count + 1 + // Skips the user-specified number of frames plus this function + // itself. + ); // NOLINT +} + +// Returns the current time in milliseconds. +TimeInMillis GetTimeInMillis() { +#if GTEST_OS_WINDOWS_MOBILE || defined(__BORLANDC__) + // Difference between 1970-01-01 and 1601-01-01 in milliseconds. + // http://analogous.blogspot.com/2005/04/epoch.html + const TimeInMillis kJavaEpochToWinFileTimeDelta = + static_cast(116444736UL) * 100000UL; + const DWORD kTenthMicrosInMilliSecond = 10000; + + SYSTEMTIME now_systime; + FILETIME now_filetime; + ULARGE_INTEGER now_int64; + // FIXME: Shouldn't this just use + // GetSystemTimeAsFileTime()? + GetSystemTime(&now_systime); + if (SystemTimeToFileTime(&now_systime, &now_filetime)) { + now_int64.LowPart = now_filetime.dwLowDateTime; + now_int64.HighPart = now_filetime.dwHighDateTime; + now_int64.QuadPart = (now_int64.QuadPart / kTenthMicrosInMilliSecond) - + kJavaEpochToWinFileTimeDelta; + return now_int64.QuadPart; + } + return 0; +#elif GTEST_OS_WINDOWS && !GTEST_HAS_GETTIMEOFDAY_ + __timeb64 now; + + // MSVC 8 deprecates _ftime64(), so we want to suppress warning 4996 + // (deprecated function) there. + // FIXME: Use GetTickCount()? Or use + // SystemTimeToFileTime() + GTEST_DISABLE_MSC_DEPRECATED_PUSH_() + _ftime64(&now); + GTEST_DISABLE_MSC_DEPRECATED_POP_() + + return static_cast(now.time) * 1000 + now.millitm; +#elif GTEST_HAS_GETTIMEOFDAY_ + struct timeval now; + gettimeofday(&now, NULL); + return static_cast(now.tv_sec) * 1000 + now.tv_usec / 1000; +#else +# error "Don't know how to get the current time on your system." +#endif +} + +// Utilities + +// class String. + +#if GTEST_OS_WINDOWS_MOBILE +// Creates a UTF-16 wide string from the given ANSI string, allocating +// memory using new. The caller is responsible for deleting the return +// value using delete[]. Returns the wide string, or NULL if the +// input is NULL. +LPCWSTR String::AnsiToUtf16(const char* ansi) { + if (!ansi) return NULL; + const int length = strlen(ansi); + const int unicode_length = + MultiByteToWideChar(CP_ACP, 0, ansi, length, + NULL, 0); + WCHAR* unicode = new WCHAR[unicode_length + 1]; + MultiByteToWideChar(CP_ACP, 0, ansi, length, + unicode, unicode_length); + unicode[unicode_length] = 0; + return unicode; +} + +// Creates an ANSI string from the given wide string, allocating +// memory using new. The caller is responsible for deleting the return +// value using delete[]. Returns the ANSI string, or NULL if the +// input is NULL. +const char* String::Utf16ToAnsi(LPCWSTR utf16_str) { + if (!utf16_str) return NULL; + const int ansi_length = + WideCharToMultiByte(CP_ACP, 0, utf16_str, -1, + NULL, 0, NULL, NULL); + char* ansi = new char[ansi_length + 1]; + WideCharToMultiByte(CP_ACP, 0, utf16_str, -1, + ansi, ansi_length, NULL, NULL); + ansi[ansi_length] = 0; + return ansi; +} + +#endif // GTEST_OS_WINDOWS_MOBILE + +// Compares two C strings. Returns true iff they have the same content. +// +// Unlike strcmp(), this function can handle NULL argument(s). A NULL +// C string is considered different to any non-NULL C string, +// including the empty string. +bool String::CStringEquals(const char * lhs, const char * rhs) { + if ( lhs == NULL ) return rhs == NULL; + + if ( rhs == NULL ) return false; + + return strcmp(lhs, rhs) == 0; +} + +#if GTEST_HAS_STD_WSTRING || GTEST_HAS_GLOBAL_WSTRING + +// Converts an array of wide chars to a narrow string using the UTF-8 +// encoding, and streams the result to the given Message object. +static void StreamWideCharsToMessage(const wchar_t* wstr, size_t length, + Message* msg) { + for (size_t i = 0; i != length; ) { // NOLINT + if (wstr[i] != L'\0') { + *msg << WideStringToUtf8(wstr + i, static_cast(length - i)); + while (i != length && wstr[i] != L'\0') + i++; + } else { + *msg << '\0'; + i++; + } + } +} + +#endif // GTEST_HAS_STD_WSTRING || GTEST_HAS_GLOBAL_WSTRING + +void SplitString(const ::std::string& str, char delimiter, + ::std::vector< ::std::string>* dest) { + ::std::vector< ::std::string> parsed; + ::std::string::size_type pos = 0; + while (::testing::internal::AlwaysTrue()) { + const ::std::string::size_type colon = str.find(delimiter, pos); + if (colon == ::std::string::npos) { + parsed.push_back(str.substr(pos)); + break; + } else { + parsed.push_back(str.substr(pos, colon - pos)); + pos = colon + 1; + } + } + dest->swap(parsed); +} + +} // namespace internal + +// Constructs an empty Message. +// We allocate the stringstream separately because otherwise each use of +// ASSERT/EXPECT in a procedure adds over 200 bytes to the procedure's +// stack frame leading to huge stack frames in some cases; gcc does not reuse +// the stack space. +Message::Message() : ss_(new ::std::stringstream) { + // By default, we want there to be enough precision when printing + // a double to a Message. + *ss_ << std::setprecision(std::numeric_limits::digits10 + 2); +} + +// These two overloads allow streaming a wide C string to a Message +// using the UTF-8 encoding. +Message& Message::operator <<(const wchar_t* wide_c_str) { + return *this << internal::String::ShowWideCString(wide_c_str); +} +Message& Message::operator <<(wchar_t* wide_c_str) { + return *this << internal::String::ShowWideCString(wide_c_str); +} + +#if GTEST_HAS_STD_WSTRING +// Converts the given wide string to a narrow string using the UTF-8 +// encoding, and streams the result to this Message object. +Message& Message::operator <<(const ::std::wstring& wstr) { + internal::StreamWideCharsToMessage(wstr.c_str(), wstr.length(), this); + return *this; +} +#endif // GTEST_HAS_STD_WSTRING + +#if GTEST_HAS_GLOBAL_WSTRING +// Converts the given wide string to a narrow string using the UTF-8 +// encoding, and streams the result to this Message object. +Message& Message::operator <<(const ::wstring& wstr) { + internal::StreamWideCharsToMessage(wstr.c_str(), wstr.length(), this); + return *this; +} +#endif // GTEST_HAS_GLOBAL_WSTRING + +// Gets the text streamed to this object so far as an std::string. +// Each '\0' character in the buffer is replaced with "\\0". +std::string Message::GetString() const { + return internal::StringStreamToString(ss_.get()); +} + +// AssertionResult constructors. +// Used in EXPECT_TRUE/FALSE(assertion_result). +AssertionResult::AssertionResult(const AssertionResult& other) + : success_(other.success_), + message_(other.message_.get() != NULL ? + new ::std::string(*other.message_) : + static_cast< ::std::string*>(NULL)) { +} + +// Swaps two AssertionResults. +void AssertionResult::swap(AssertionResult& other) { + using std::swap; + swap(success_, other.success_); + swap(message_, other.message_); +} + +// Returns the assertion's negation. Used with EXPECT/ASSERT_FALSE. +AssertionResult AssertionResult::operator!() const { + AssertionResult negation(!success_); + if (message_.get() != NULL) + negation << *message_; + return negation; +} + +// Makes a successful assertion result. +AssertionResult AssertionSuccess() { + return AssertionResult(true); +} + +// Makes a failed assertion result. +AssertionResult AssertionFailure() { + return AssertionResult(false); +} + +// Makes a failed assertion result with the given failure message. +// Deprecated; use AssertionFailure() << message. +AssertionResult AssertionFailure(const Message& message) { + return AssertionFailure() << message; +} + +namespace internal { + +namespace edit_distance { +std::vector CalculateOptimalEdits(const std::vector& left, + const std::vector& right) { + std::vector > costs( + left.size() + 1, std::vector(right.size() + 1)); + std::vector > best_move( + left.size() + 1, std::vector(right.size() + 1)); + + // Populate for empty right. + for (size_t l_i = 0; l_i < costs.size(); ++l_i) { + costs[l_i][0] = static_cast(l_i); + best_move[l_i][0] = kRemove; + } + // Populate for empty left. + for (size_t r_i = 1; r_i < costs[0].size(); ++r_i) { + costs[0][r_i] = static_cast(r_i); + best_move[0][r_i] = kAdd; + } + + for (size_t l_i = 0; l_i < left.size(); ++l_i) { + for (size_t r_i = 0; r_i < right.size(); ++r_i) { + if (left[l_i] == right[r_i]) { + // Found a match. Consume it. + costs[l_i + 1][r_i + 1] = costs[l_i][r_i]; + best_move[l_i + 1][r_i + 1] = kMatch; + continue; + } + + const double add = costs[l_i + 1][r_i]; + const double remove = costs[l_i][r_i + 1]; + const double replace = costs[l_i][r_i]; + if (add < remove && add < replace) { + costs[l_i + 1][r_i + 1] = add + 1; + best_move[l_i + 1][r_i + 1] = kAdd; + } else if (remove < add && remove < replace) { + costs[l_i + 1][r_i + 1] = remove + 1; + best_move[l_i + 1][r_i + 1] = kRemove; + } else { + // We make replace a little more expensive than add/remove to lower + // their priority. + costs[l_i + 1][r_i + 1] = replace + 1.00001; + best_move[l_i + 1][r_i + 1] = kReplace; + } + } + } + + // Reconstruct the best path. We do it in reverse order. + std::vector best_path; + for (size_t l_i = left.size(), r_i = right.size(); l_i > 0 || r_i > 0;) { + EditType move = best_move[l_i][r_i]; + best_path.push_back(move); + l_i -= move != kAdd; + r_i -= move != kRemove; + } + std::reverse(best_path.begin(), best_path.end()); + return best_path; +} + +namespace { + +// Helper class to convert string into ids with deduplication. +class InternalStrings { + public: + size_t GetId(const std::string& str) { + IdMap::iterator it = ids_.find(str); + if (it != ids_.end()) return it->second; + size_t id = ids_.size(); + return ids_[str] = id; + } + + private: + typedef std::map IdMap; + IdMap ids_; +}; + +} // namespace + +std::vector CalculateOptimalEdits( + const std::vector& left, + const std::vector& right) { + std::vector left_ids, right_ids; + { + InternalStrings intern_table; + for (size_t i = 0; i < left.size(); ++i) { + left_ids.push_back(intern_table.GetId(left[i])); + } + for (size_t i = 0; i < right.size(); ++i) { + right_ids.push_back(intern_table.GetId(right[i])); + } + } + return CalculateOptimalEdits(left_ids, right_ids); +} + +namespace { + +// Helper class that holds the state for one hunk and prints it out to the +// stream. +// It reorders adds/removes when possible to group all removes before all +// adds. It also adds the hunk header before printint into the stream. +class Hunk { + public: + Hunk(size_t left_start, size_t right_start) + : left_start_(left_start), + right_start_(right_start), + adds_(), + removes_(), + common_() {} + + void PushLine(char edit, const char* line) { + switch (edit) { + case ' ': + ++common_; + FlushEdits(); + hunk_.push_back(std::make_pair(' ', line)); + break; + case '-': + ++removes_; + hunk_removes_.push_back(std::make_pair('-', line)); + break; + case '+': + ++adds_; + hunk_adds_.push_back(std::make_pair('+', line)); + break; + } + } + + void PrintTo(std::ostream* os) { + PrintHeader(os); + FlushEdits(); + for (std::list >::const_iterator it = + hunk_.begin(); + it != hunk_.end(); ++it) { + *os << it->first << it->second << "\n"; + } + } + + bool has_edits() const { return adds_ || removes_; } + + private: + void FlushEdits() { + hunk_.splice(hunk_.end(), hunk_removes_); + hunk_.splice(hunk_.end(), hunk_adds_); + } + + // Print a unified diff header for one hunk. + // The format is + // "@@ -, +, @@" + // where the left/right parts are omitted if unnecessary. + void PrintHeader(std::ostream* ss) const { + *ss << "@@ "; + if (removes_) { + *ss << "-" << left_start_ << "," << (removes_ + common_); + } + if (removes_ && adds_) { + *ss << " "; + } + if (adds_) { + *ss << "+" << right_start_ << "," << (adds_ + common_); + } + *ss << " @@\n"; + } + + size_t left_start_, right_start_; + size_t adds_, removes_, common_; + std::list > hunk_, hunk_adds_, hunk_removes_; +}; + +} // namespace + +// Create a list of diff hunks in Unified diff format. +// Each hunk has a header generated by PrintHeader above plus a body with +// lines prefixed with ' ' for no change, '-' for deletion and '+' for +// addition. +// 'context' represents the desired unchanged prefix/suffix around the diff. +// If two hunks are close enough that their contexts overlap, then they are +// joined into one hunk. +std::string CreateUnifiedDiff(const std::vector& left, + const std::vector& right, + size_t context) { + const std::vector edits = CalculateOptimalEdits(left, right); + + size_t l_i = 0, r_i = 0, edit_i = 0; + std::stringstream ss; + while (edit_i < edits.size()) { + // Find first edit. + while (edit_i < edits.size() && edits[edit_i] == kMatch) { + ++l_i; + ++r_i; + ++edit_i; + } + + // Find the first line to include in the hunk. + const size_t prefix_context = std::min(l_i, context); + Hunk hunk(l_i - prefix_context + 1, r_i - prefix_context + 1); + for (size_t i = prefix_context; i > 0; --i) { + hunk.PushLine(' ', left[l_i - i].c_str()); + } + + // Iterate the edits until we found enough suffix for the hunk or the input + // is over. + size_t n_suffix = 0; + for (; edit_i < edits.size(); ++edit_i) { + if (n_suffix >= context) { + // Continue only if the next hunk is very close. + std::vector::const_iterator it = edits.begin() + edit_i; + while (it != edits.end() && *it == kMatch) ++it; + if (it == edits.end() || (it - edits.begin()) - edit_i >= context) { + // There is no next edit or it is too far away. + break; + } + } + + EditType edit = edits[edit_i]; + // Reset count when a non match is found. + n_suffix = edit == kMatch ? n_suffix + 1 : 0; + + if (edit == kMatch || edit == kRemove || edit == kReplace) { + hunk.PushLine(edit == kMatch ? ' ' : '-', left[l_i].c_str()); + } + if (edit == kAdd || edit == kReplace) { + hunk.PushLine('+', right[r_i].c_str()); + } + + // Advance indices, depending on edit type. + l_i += edit != kAdd; + r_i += edit != kRemove; + } + + if (!hunk.has_edits()) { + // We are done. We don't want this hunk. + break; + } + + hunk.PrintTo(&ss); + } + return ss.str(); +} + +} // namespace edit_distance + +namespace { + +// The string representation of the values received in EqFailure() are already +// escaped. Split them on escaped '\n' boundaries. Leave all other escaped +// characters the same. +std::vector SplitEscapedString(const std::string& str) { + std::vector lines; + size_t start = 0, end = str.size(); + if (end > 2 && str[0] == '"' && str[end - 1] == '"') { + ++start; + --end; + } + bool escaped = false; + for (size_t i = start; i + 1 < end; ++i) { + if (escaped) { + escaped = false; + if (str[i] == 'n') { + lines.push_back(str.substr(start, i - start - 1)); + start = i + 1; + } + } else { + escaped = str[i] == '\\'; + } + } + lines.push_back(str.substr(start, end - start)); + return lines; +} + +} // namespace + +// Constructs and returns the message for an equality assertion +// (e.g. ASSERT_EQ, EXPECT_STREQ, etc) failure. +// +// The first four parameters are the expressions used in the assertion +// and their values, as strings. For example, for ASSERT_EQ(foo, bar) +// where foo is 5 and bar is 6, we have: +// +// lhs_expression: "foo" +// rhs_expression: "bar" +// lhs_value: "5" +// rhs_value: "6" +// +// The ignoring_case parameter is true iff the assertion is a +// *_STRCASEEQ*. When it's true, the string "Ignoring case" will +// be inserted into the message. +AssertionResult EqFailure(const char* lhs_expression, + const char* rhs_expression, + const std::string& lhs_value, + const std::string& rhs_value, + bool ignoring_case) { + Message msg; + msg << "Expected equality of these values:"; + msg << "\n " << lhs_expression; + if (lhs_value != lhs_expression) { + msg << "\n Which is: " << lhs_value; + } + msg << "\n " << rhs_expression; + if (rhs_value != rhs_expression) { + msg << "\n Which is: " << rhs_value; + } + + if (ignoring_case) { + msg << "\nIgnoring case"; + } + + if (!lhs_value.empty() && !rhs_value.empty()) { + const std::vector lhs_lines = + SplitEscapedString(lhs_value); + const std::vector rhs_lines = + SplitEscapedString(rhs_value); + if (lhs_lines.size() > 1 || rhs_lines.size() > 1) { + msg << "\nWith diff:\n" + << edit_distance::CreateUnifiedDiff(lhs_lines, rhs_lines); + } + } + + return AssertionFailure() << msg; +} + +// Constructs a failure message for Boolean assertions such as EXPECT_TRUE. +std::string GetBoolAssertionFailureMessage( + const AssertionResult& assertion_result, + const char* expression_text, + const char* actual_predicate_value, + const char* expected_predicate_value) { + const char* actual_message = assertion_result.message(); + Message msg; + msg << "Value of: " << expression_text + << "\n Actual: " << actual_predicate_value; + if (actual_message[0] != '\0') + msg << " (" << actual_message << ")"; + msg << "\nExpected: " << expected_predicate_value; + return msg.GetString(); +} + +// Helper function for implementing ASSERT_NEAR. +AssertionResult DoubleNearPredFormat(const char* expr1, + const char* expr2, + const char* abs_error_expr, + double val1, + double val2, + double abs_error) { + const double diff = fabs(val1 - val2); + if (diff <= abs_error) return AssertionSuccess(); + + // FIXME: do not print the value of an expression if it's + // already a literal. + return AssertionFailure() + << "The difference between " << expr1 << " and " << expr2 + << " is " << diff << ", which exceeds " << abs_error_expr << ", where\n" + << expr1 << " evaluates to " << val1 << ",\n" + << expr2 << " evaluates to " << val2 << ", and\n" + << abs_error_expr << " evaluates to " << abs_error << "."; +} + + +// Helper template for implementing FloatLE() and DoubleLE(). +template +AssertionResult FloatingPointLE(const char* expr1, + const char* expr2, + RawType val1, + RawType val2) { + // Returns success if val1 is less than val2, + if (val1 < val2) { + return AssertionSuccess(); + } + + // or if val1 is almost equal to val2. + const FloatingPoint lhs(val1), rhs(val2); + if (lhs.AlmostEquals(rhs)) { + return AssertionSuccess(); + } + + // Note that the above two checks will both fail if either val1 or + // val2 is NaN, as the IEEE floating-point standard requires that + // any predicate involving a NaN must return false. + + ::std::stringstream val1_ss; + val1_ss << std::setprecision(std::numeric_limits::digits10 + 2) + << val1; + + ::std::stringstream val2_ss; + val2_ss << std::setprecision(std::numeric_limits::digits10 + 2) + << val2; + + return AssertionFailure() + << "Expected: (" << expr1 << ") <= (" << expr2 << ")\n" + << " Actual: " << StringStreamToString(&val1_ss) << " vs " + << StringStreamToString(&val2_ss); +} + +} // namespace internal + +// Asserts that val1 is less than, or almost equal to, val2. Fails +// otherwise. In particular, it fails if either val1 or val2 is NaN. +AssertionResult FloatLE(const char* expr1, const char* expr2, + float val1, float val2) { + return internal::FloatingPointLE(expr1, expr2, val1, val2); +} + +// Asserts that val1 is less than, or almost equal to, val2. Fails +// otherwise. In particular, it fails if either val1 or val2 is NaN. +AssertionResult DoubleLE(const char* expr1, const char* expr2, + double val1, double val2) { + return internal::FloatingPointLE(expr1, expr2, val1, val2); +} + +namespace internal { + +// The helper function for {ASSERT|EXPECT}_EQ with int or enum +// arguments. +AssertionResult CmpHelperEQ(const char* lhs_expression, + const char* rhs_expression, + BiggestInt lhs, + BiggestInt rhs) { + if (lhs == rhs) { + return AssertionSuccess(); + } + + return EqFailure(lhs_expression, + rhs_expression, + FormatForComparisonFailureMessage(lhs, rhs), + FormatForComparisonFailureMessage(rhs, lhs), + false); +} + +// A macro for implementing the helper functions needed to implement +// ASSERT_?? and EXPECT_?? with integer or enum arguments. It is here +// just to avoid copy-and-paste of similar code. +#define GTEST_IMPL_CMP_HELPER_(op_name, op)\ +AssertionResult CmpHelper##op_name(const char* expr1, const char* expr2, \ + BiggestInt val1, BiggestInt val2) {\ + if (val1 op val2) {\ + return AssertionSuccess();\ + } else {\ + return AssertionFailure() \ + << "Expected: (" << expr1 << ") " #op " (" << expr2\ + << "), actual: " << FormatForComparisonFailureMessage(val1, val2)\ + << " vs " << FormatForComparisonFailureMessage(val2, val1);\ + }\ +} + +// Implements the helper function for {ASSERT|EXPECT}_NE with int or +// enum arguments. +GTEST_IMPL_CMP_HELPER_(NE, !=) +// Implements the helper function for {ASSERT|EXPECT}_LE with int or +// enum arguments. +GTEST_IMPL_CMP_HELPER_(LE, <=) +// Implements the helper function for {ASSERT|EXPECT}_LT with int or +// enum arguments. +GTEST_IMPL_CMP_HELPER_(LT, < ) +// Implements the helper function for {ASSERT|EXPECT}_GE with int or +// enum arguments. +GTEST_IMPL_CMP_HELPER_(GE, >=) +// Implements the helper function for {ASSERT|EXPECT}_GT with int or +// enum arguments. +GTEST_IMPL_CMP_HELPER_(GT, > ) + +#undef GTEST_IMPL_CMP_HELPER_ + +// The helper function for {ASSERT|EXPECT}_STREQ. +AssertionResult CmpHelperSTREQ(const char* lhs_expression, + const char* rhs_expression, + const char* lhs, + const char* rhs) { + if (String::CStringEquals(lhs, rhs)) { + return AssertionSuccess(); + } + + return EqFailure(lhs_expression, + rhs_expression, + PrintToString(lhs), + PrintToString(rhs), + false); +} + +// The helper function for {ASSERT|EXPECT}_STRCASEEQ. +AssertionResult CmpHelperSTRCASEEQ(const char* lhs_expression, + const char* rhs_expression, + const char* lhs, + const char* rhs) { + if (String::CaseInsensitiveCStringEquals(lhs, rhs)) { + return AssertionSuccess(); + } + + return EqFailure(lhs_expression, + rhs_expression, + PrintToString(lhs), + PrintToString(rhs), + true); +} + +// The helper function for {ASSERT|EXPECT}_STRNE. +AssertionResult CmpHelperSTRNE(const char* s1_expression, + const char* s2_expression, + const char* s1, + const char* s2) { + if (!String::CStringEquals(s1, s2)) { + return AssertionSuccess(); + } else { + return AssertionFailure() << "Expected: (" << s1_expression << ") != (" + << s2_expression << "), actual: \"" + << s1 << "\" vs \"" << s2 << "\""; + } +} + +// The helper function for {ASSERT|EXPECT}_STRCASENE. +AssertionResult CmpHelperSTRCASENE(const char* s1_expression, + const char* s2_expression, + const char* s1, + const char* s2) { + if (!String::CaseInsensitiveCStringEquals(s1, s2)) { + return AssertionSuccess(); + } else { + return AssertionFailure() + << "Expected: (" << s1_expression << ") != (" + << s2_expression << ") (ignoring case), actual: \"" + << s1 << "\" vs \"" << s2 << "\""; + } +} + +} // namespace internal + +namespace { + +// Helper functions for implementing IsSubString() and IsNotSubstring(). + +// This group of overloaded functions return true iff needle is a +// substring of haystack. NULL is considered a substring of itself +// only. + +bool IsSubstringPred(const char* needle, const char* haystack) { + if (needle == NULL || haystack == NULL) + return needle == haystack; + + return strstr(haystack, needle) != NULL; +} + +bool IsSubstringPred(const wchar_t* needle, const wchar_t* haystack) { + if (needle == NULL || haystack == NULL) + return needle == haystack; + + return wcsstr(haystack, needle) != NULL; +} + +// StringType here can be either ::std::string or ::std::wstring. +template +bool IsSubstringPred(const StringType& needle, + const StringType& haystack) { + return haystack.find(needle) != StringType::npos; +} + +// This function implements either IsSubstring() or IsNotSubstring(), +// depending on the value of the expected_to_be_substring parameter. +// StringType here can be const char*, const wchar_t*, ::std::string, +// or ::std::wstring. +template +AssertionResult IsSubstringImpl( + bool expected_to_be_substring, + const char* needle_expr, const char* haystack_expr, + const StringType& needle, const StringType& haystack) { + if (IsSubstringPred(needle, haystack) == expected_to_be_substring) + return AssertionSuccess(); + + const bool is_wide_string = sizeof(needle[0]) > 1; + const char* const begin_string_quote = is_wide_string ? "L\"" : "\""; + return AssertionFailure() + << "Value of: " << needle_expr << "\n" + << " Actual: " << begin_string_quote << needle << "\"\n" + << "Expected: " << (expected_to_be_substring ? "" : "not ") + << "a substring of " << haystack_expr << "\n" + << "Which is: " << begin_string_quote << haystack << "\""; +} + +} // namespace + +// IsSubstring() and IsNotSubstring() check whether needle is a +// substring of haystack (NULL is considered a substring of itself +// only), and return an appropriate error message when they fail. + +AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const char* needle, const char* haystack) { + return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const wchar_t* needle, const wchar_t* haystack) { + return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const char* needle, const char* haystack) { + return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const wchar_t* needle, const wchar_t* haystack) { + return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::string& needle, const ::std::string& haystack) { + return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::string& needle, const ::std::string& haystack) { + return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack); +} + +#if GTEST_HAS_STD_WSTRING +AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::wstring& needle, const ::std::wstring& haystack) { + return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::wstring& needle, const ::std::wstring& haystack) { + return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack); +} +#endif // GTEST_HAS_STD_WSTRING + +namespace internal { + +#if GTEST_OS_WINDOWS + +namespace { + +// Helper function for IsHRESULT{SuccessFailure} predicates +AssertionResult HRESULTFailureHelper(const char* expr, + const char* expected, + long hr) { // NOLINT +# if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_TV_TITLE + + // Windows CE doesn't support FormatMessage. + const char error_text[] = ""; + +# else + + // Looks up the human-readable system message for the HRESULT code + // and since we're not passing any params to FormatMessage, we don't + // want inserts expanded. + const DWORD kFlags = FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS; + const DWORD kBufSize = 4096; + // Gets the system's human readable message string for this HRESULT. + char error_text[kBufSize] = { '\0' }; + DWORD message_length = ::FormatMessageA(kFlags, + 0, // no source, we're asking system + hr, // the error + 0, // no line width restrictions + error_text, // output buffer + kBufSize, // buf size + NULL); // no arguments for inserts + // Trims tailing white space (FormatMessage leaves a trailing CR-LF) + for (; message_length && IsSpace(error_text[message_length - 1]); + --message_length) { + error_text[message_length - 1] = '\0'; + } + +# endif // GTEST_OS_WINDOWS_MOBILE + + const std::string error_hex("0x" + String::FormatHexInt(hr)); + return ::testing::AssertionFailure() + << "Expected: " << expr << " " << expected << ".\n" + << " Actual: " << error_hex << " " << error_text << "\n"; +} + +} // namespace + +AssertionResult IsHRESULTSuccess(const char* expr, long hr) { // NOLINT + if (SUCCEEDED(hr)) { + return AssertionSuccess(); + } + return HRESULTFailureHelper(expr, "succeeds", hr); +} + +AssertionResult IsHRESULTFailure(const char* expr, long hr) { // NOLINT + if (FAILED(hr)) { + return AssertionSuccess(); + } + return HRESULTFailureHelper(expr, "fails", hr); +} + +#endif // GTEST_OS_WINDOWS + +// Utility functions for encoding Unicode text (wide strings) in +// UTF-8. + +// A Unicode code-point can have up to 21 bits, and is encoded in UTF-8 +// like this: +// +// Code-point length Encoding +// 0 - 7 bits 0xxxxxxx +// 8 - 11 bits 110xxxxx 10xxxxxx +// 12 - 16 bits 1110xxxx 10xxxxxx 10xxxxxx +// 17 - 21 bits 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + +// The maximum code-point a one-byte UTF-8 sequence can represent. +const UInt32 kMaxCodePoint1 = (static_cast(1) << 7) - 1; + +// The maximum code-point a two-byte UTF-8 sequence can represent. +const UInt32 kMaxCodePoint2 = (static_cast(1) << (5 + 6)) - 1; + +// The maximum code-point a three-byte UTF-8 sequence can represent. +const UInt32 kMaxCodePoint3 = (static_cast(1) << (4 + 2*6)) - 1; + +// The maximum code-point a four-byte UTF-8 sequence can represent. +const UInt32 kMaxCodePoint4 = (static_cast(1) << (3 + 3*6)) - 1; + +// Chops off the n lowest bits from a bit pattern. Returns the n +// lowest bits. As a side effect, the original bit pattern will be +// shifted to the right by n bits. +inline UInt32 ChopLowBits(UInt32* bits, int n) { + const UInt32 low_bits = *bits & ((static_cast(1) << n) - 1); + *bits >>= n; + return low_bits; +} + +// Converts a Unicode code point to a narrow string in UTF-8 encoding. +// code_point parameter is of type UInt32 because wchar_t may not be +// wide enough to contain a code point. +// If the code_point is not a valid Unicode code point +// (i.e. outside of Unicode range U+0 to U+10FFFF) it will be converted +// to "(Invalid Unicode 0xXXXXXXXX)". +std::string CodePointToUtf8(UInt32 code_point) { + if (code_point > kMaxCodePoint4) { + return "(Invalid Unicode 0x" + String::FormatHexInt(code_point) + ")"; + } + + char str[5]; // Big enough for the largest valid code point. + if (code_point <= kMaxCodePoint1) { + str[1] = '\0'; + str[0] = static_cast(code_point); // 0xxxxxxx + } else if (code_point <= kMaxCodePoint2) { + str[2] = '\0'; + str[1] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[0] = static_cast(0xC0 | code_point); // 110xxxxx + } else if (code_point <= kMaxCodePoint3) { + str[3] = '\0'; + str[2] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[1] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[0] = static_cast(0xE0 | code_point); // 1110xxxx + } else { // code_point <= kMaxCodePoint4 + str[4] = '\0'; + str[3] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[2] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[1] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[0] = static_cast(0xF0 | code_point); // 11110xxx + } + return str; +} + +// The following two functions only make sense if the system +// uses UTF-16 for wide string encoding. All supported systems +// with 16 bit wchar_t (Windows, Cygwin, Symbian OS) do use UTF-16. + +// Determines if the arguments constitute UTF-16 surrogate pair +// and thus should be combined into a single Unicode code point +// using CreateCodePointFromUtf16SurrogatePair. +inline bool IsUtf16SurrogatePair(wchar_t first, wchar_t second) { + return sizeof(wchar_t) == 2 && + (first & 0xFC00) == 0xD800 && (second & 0xFC00) == 0xDC00; +} + +// Creates a Unicode code point from UTF16 surrogate pair. +inline UInt32 CreateCodePointFromUtf16SurrogatePair(wchar_t first, + wchar_t second) { + const UInt32 mask = (1 << 10) - 1; + return (sizeof(wchar_t) == 2) ? + (((first & mask) << 10) | (second & mask)) + 0x10000 : + // This function should not be called when the condition is + // false, but we provide a sensible default in case it is. + static_cast(first); +} + +// Converts a wide string to a narrow string in UTF-8 encoding. +// The wide string is assumed to have the following encoding: +// UTF-16 if sizeof(wchar_t) == 2 (on Windows, Cygwin, Symbian OS) +// UTF-32 if sizeof(wchar_t) == 4 (on Linux) +// Parameter str points to a null-terminated wide string. +// Parameter num_chars may additionally limit the number +// of wchar_t characters processed. -1 is used when the entire string +// should be processed. +// If the string contains code points that are not valid Unicode code points +// (i.e. outside of Unicode range U+0 to U+10FFFF) they will be output +// as '(Invalid Unicode 0xXXXXXXXX)'. If the string is in UTF16 encoding +// and contains invalid UTF-16 surrogate pairs, values in those pairs +// will be encoded as individual Unicode characters from Basic Normal Plane. +std::string WideStringToUtf8(const wchar_t* str, int num_chars) { + if (num_chars == -1) + num_chars = static_cast(wcslen(str)); + + ::std::stringstream stream; + for (int i = 0; i < num_chars; ++i) { + UInt32 unicode_code_point; + + if (str[i] == L'\0') { + break; + } else if (i + 1 < num_chars && IsUtf16SurrogatePair(str[i], str[i + 1])) { + unicode_code_point = CreateCodePointFromUtf16SurrogatePair(str[i], + str[i + 1]); + i++; + } else { + unicode_code_point = static_cast(str[i]); + } + + stream << CodePointToUtf8(unicode_code_point); + } + return StringStreamToString(&stream); +} + +// Converts a wide C string to an std::string using the UTF-8 encoding. +// NULL will be converted to "(null)". +std::string String::ShowWideCString(const wchar_t * wide_c_str) { + if (wide_c_str == NULL) return "(null)"; + + return internal::WideStringToUtf8(wide_c_str, -1); +} + +// Compares two wide C strings. Returns true iff they have the same +// content. +// +// Unlike wcscmp(), this function can handle NULL argument(s). A NULL +// C string is considered different to any non-NULL C string, +// including the empty string. +bool String::WideCStringEquals(const wchar_t * lhs, const wchar_t * rhs) { + if (lhs == NULL) return rhs == NULL; + + if (rhs == NULL) return false; + + return wcscmp(lhs, rhs) == 0; +} + +// Helper function for *_STREQ on wide strings. +AssertionResult CmpHelperSTREQ(const char* lhs_expression, + const char* rhs_expression, + const wchar_t* lhs, + const wchar_t* rhs) { + if (String::WideCStringEquals(lhs, rhs)) { + return AssertionSuccess(); + } + + return EqFailure(lhs_expression, + rhs_expression, + PrintToString(lhs), + PrintToString(rhs), + false); +} + +// Helper function for *_STRNE on wide strings. +AssertionResult CmpHelperSTRNE(const char* s1_expression, + const char* s2_expression, + const wchar_t* s1, + const wchar_t* s2) { + if (!String::WideCStringEquals(s1, s2)) { + return AssertionSuccess(); + } + + return AssertionFailure() << "Expected: (" << s1_expression << ") != (" + << s2_expression << "), actual: " + << PrintToString(s1) + << " vs " << PrintToString(s2); +} + +// Compares two C strings, ignoring case. Returns true iff they have +// the same content. +// +// Unlike strcasecmp(), this function can handle NULL argument(s). A +// NULL C string is considered different to any non-NULL C string, +// including the empty string. +bool String::CaseInsensitiveCStringEquals(const char * lhs, const char * rhs) { + if (lhs == NULL) + return rhs == NULL; + if (rhs == NULL) + return false; + return posix::StrCaseCmp(lhs, rhs) == 0; +} + + // Compares two wide C strings, ignoring case. Returns true iff they + // have the same content. + // + // Unlike wcscasecmp(), this function can handle NULL argument(s). + // A NULL C string is considered different to any non-NULL wide C string, + // including the empty string. + // NB: The implementations on different platforms slightly differ. + // On windows, this method uses _wcsicmp which compares according to LC_CTYPE + // environment variable. On GNU platform this method uses wcscasecmp + // which compares according to LC_CTYPE category of the current locale. + // On MacOS X, it uses towlower, which also uses LC_CTYPE category of the + // current locale. +bool String::CaseInsensitiveWideCStringEquals(const wchar_t* lhs, + const wchar_t* rhs) { + if (lhs == NULL) return rhs == NULL; + + if (rhs == NULL) return false; + +#if GTEST_OS_WINDOWS + return _wcsicmp(lhs, rhs) == 0; +#elif GTEST_OS_LINUX && !GTEST_OS_LINUX_ANDROID + return wcscasecmp(lhs, rhs) == 0; +#else + // Android, Mac OS X and Cygwin don't define wcscasecmp. + // Other unknown OSes may not define it either. + wint_t left, right; + do { + left = towlower(*lhs++); + right = towlower(*rhs++); + } while (left && left == right); + return left == right; +#endif // OS selector +} + +// Returns true iff str ends with the given suffix, ignoring case. +// Any string is considered to end with an empty suffix. +bool String::EndsWithCaseInsensitive( + const std::string& str, const std::string& suffix) { + const size_t str_len = str.length(); + const size_t suffix_len = suffix.length(); + return (str_len >= suffix_len) && + CaseInsensitiveCStringEquals(str.c_str() + str_len - suffix_len, + suffix.c_str()); +} + +// Formats an int value as "%02d". +std::string String::FormatIntWidth2(int value) { + std::stringstream ss; + ss << std::setfill('0') << std::setw(2) << value; + return ss.str(); +} + +// Formats an int value as "%X". +std::string String::FormatHexInt(int value) { + std::stringstream ss; + ss << std::hex << std::uppercase << value; + return ss.str(); +} + +// Formats a byte as "%02X". +std::string String::FormatByte(unsigned char value) { + std::stringstream ss; + ss << std::setfill('0') << std::setw(2) << std::hex << std::uppercase + << static_cast(value); + return ss.str(); +} + +// Converts the buffer in a stringstream to an std::string, converting NUL +// bytes to "\\0" along the way. +std::string StringStreamToString(::std::stringstream* ss) { + const ::std::string& str = ss->str(); + const char* const start = str.c_str(); + const char* const end = start + str.length(); + + std::string result; + result.reserve(2 * (end - start)); + for (const char* ch = start; ch != end; ++ch) { + if (*ch == '\0') { + result += "\\0"; // Replaces NUL with "\\0"; + } else { + result += *ch; + } + } + + return result; +} + +// Appends the user-supplied message to the Google-Test-generated message. +std::string AppendUserMessage(const std::string& gtest_msg, + const Message& user_msg) { + // Appends the user message if it's non-empty. + const std::string user_msg_string = user_msg.GetString(); + if (user_msg_string.empty()) { + return gtest_msg; + } + + return gtest_msg + "\n" + user_msg_string; +} + +} // namespace internal + +// class TestResult + +// Creates an empty TestResult. +TestResult::TestResult() + : death_test_count_(0), + elapsed_time_(0) { +} + +// D'tor. +TestResult::~TestResult() { +} + +// Returns the i-th test part result among all the results. i can +// range from 0 to total_part_count() - 1. If i is not in that range, +// aborts the program. +const TestPartResult& TestResult::GetTestPartResult(int i) const { + if (i < 0 || i >= total_part_count()) + internal::posix::Abort(); + return test_part_results_.at(i); +} + +// Returns the i-th test property. i can range from 0 to +// test_property_count() - 1. If i is not in that range, aborts the +// program. +const TestProperty& TestResult::GetTestProperty(int i) const { + if (i < 0 || i >= test_property_count()) + internal::posix::Abort(); + return test_properties_.at(i); +} + +// Clears the test part results. +void TestResult::ClearTestPartResults() { + test_part_results_.clear(); +} + +// Adds a test part result to the list. +void TestResult::AddTestPartResult(const TestPartResult& test_part_result) { + test_part_results_.push_back(test_part_result); +} + +// Adds a test property to the list. If a property with the same key as the +// supplied property is already represented, the value of this test_property +// replaces the old value for that key. +void TestResult::RecordProperty(const std::string& xml_element, + const TestProperty& test_property) { + if (!ValidateTestProperty(xml_element, test_property)) { + return; + } + internal::MutexLock lock(&test_properites_mutex_); + const std::vector::iterator property_with_matching_key = + std::find_if(test_properties_.begin(), test_properties_.end(), + internal::TestPropertyKeyIs(test_property.key())); + if (property_with_matching_key == test_properties_.end()) { + test_properties_.push_back(test_property); + return; + } + property_with_matching_key->SetValue(test_property.value()); +} + +// The list of reserved attributes used in the element of XML +// output. +static const char* const kReservedTestSuitesAttributes[] = { + "disabled", + "errors", + "failures", + "name", + "random_seed", + "tests", + "time", + "timestamp" +}; + +// The list of reserved attributes used in the element of XML +// output. +static const char* const kReservedTestSuiteAttributes[] = { + "disabled", + "errors", + "failures", + "name", + "tests", + "time" +}; + +// The list of reserved attributes used in the element of XML output. +static const char* const kReservedTestCaseAttributes[] = { + "classname", "name", "status", "time", + "type_param", "value_param", "file", "line"}; + +template +std::vector ArrayAsVector(const char* const (&array)[kSize]) { + return std::vector(array, array + kSize); +} + +static std::vector GetReservedAttributesForElement( + const std::string& xml_element) { + if (xml_element == "testsuites") { + return ArrayAsVector(kReservedTestSuitesAttributes); + } else if (xml_element == "testsuite") { + return ArrayAsVector(kReservedTestSuiteAttributes); + } else if (xml_element == "testcase") { + return ArrayAsVector(kReservedTestCaseAttributes); + } else { + GTEST_CHECK_(false) << "Unrecognized xml_element provided: " << xml_element; + } + // This code is unreachable but some compilers may not realizes that. + return std::vector(); +} + +static std::string FormatWordList(const std::vector& words) { + Message word_list; + for (size_t i = 0; i < words.size(); ++i) { + if (i > 0 && words.size() > 2) { + word_list << ", "; + } + if (i == words.size() - 1) { + word_list << "and "; + } + word_list << "'" << words[i] << "'"; + } + return word_list.GetString(); +} + +static bool ValidateTestPropertyName( + const std::string& property_name, + const std::vector& reserved_names) { + if (std::find(reserved_names.begin(), reserved_names.end(), property_name) != + reserved_names.end()) { + ADD_FAILURE() << "Reserved key used in RecordProperty(): " << property_name + << " (" << FormatWordList(reserved_names) + << " are reserved by " << GTEST_NAME_ << ")"; + return false; + } + return true; +} + +// Adds a failure if the key is a reserved attribute of the element named +// xml_element. Returns true if the property is valid. +bool TestResult::ValidateTestProperty(const std::string& xml_element, + const TestProperty& test_property) { + return ValidateTestPropertyName(test_property.key(), + GetReservedAttributesForElement(xml_element)); +} + +// Clears the object. +void TestResult::Clear() { + test_part_results_.clear(); + test_properties_.clear(); + death_test_count_ = 0; + elapsed_time_ = 0; +} + +// Returns true iff the test failed. +bool TestResult::Failed() const { + for (int i = 0; i < total_part_count(); ++i) { + if (GetTestPartResult(i).failed()) + return true; + } + return false; +} + +// Returns true iff the test part fatally failed. +static bool TestPartFatallyFailed(const TestPartResult& result) { + return result.fatally_failed(); +} + +// Returns true iff the test fatally failed. +bool TestResult::HasFatalFailure() const { + return CountIf(test_part_results_, TestPartFatallyFailed) > 0; +} + +// Returns true iff the test part non-fatally failed. +static bool TestPartNonfatallyFailed(const TestPartResult& result) { + return result.nonfatally_failed(); +} + +// Returns true iff the test has a non-fatal failure. +bool TestResult::HasNonfatalFailure() const { + return CountIf(test_part_results_, TestPartNonfatallyFailed) > 0; +} + +// Gets the number of all test parts. This is the sum of the number +// of successful test parts and the number of failed test parts. +int TestResult::total_part_count() const { + return static_cast(test_part_results_.size()); +} + +// Returns the number of the test properties. +int TestResult::test_property_count() const { + return static_cast(test_properties_.size()); +} + +// class Test + +// Creates a Test object. + +// The c'tor saves the states of all flags. +Test::Test() + : gtest_flag_saver_(new GTEST_FLAG_SAVER_) { +} + +// The d'tor restores the states of all flags. The actual work is +// done by the d'tor of the gtest_flag_saver_ field, and thus not +// visible here. +Test::~Test() { +} + +// Sets up the test fixture. +// +// A sub-class may override this. +void Test::SetUp() { +} + +// Tears down the test fixture. +// +// A sub-class may override this. +void Test::TearDown() { +} + +// Allows user supplied key value pairs to be recorded for later output. +void Test::RecordProperty(const std::string& key, const std::string& value) { + UnitTest::GetInstance()->RecordProperty(key, value); +} + +// Allows user supplied key value pairs to be recorded for later output. +void Test::RecordProperty(const std::string& key, int value) { + Message value_message; + value_message << value; + RecordProperty(key, value_message.GetString().c_str()); +} + +namespace internal { + +void ReportFailureInUnknownLocation(TestPartResult::Type result_type, + const std::string& message) { + // This function is a friend of UnitTest and as such has access to + // AddTestPartResult. + UnitTest::GetInstance()->AddTestPartResult( + result_type, + NULL, // No info about the source file where the exception occurred. + -1, // We have no info on which line caused the exception. + message, + ""); // No stack trace, either. +} + +} // namespace internal + +// Google Test requires all tests in the same test case to use the same test +// fixture class. This function checks if the current test has the +// same fixture class as the first test in the current test case. If +// yes, it returns true; otherwise it generates a Google Test failure and +// returns false. +bool Test::HasSameFixtureClass() { + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + const TestCase* const test_case = impl->current_test_case(); + + // Info about the first test in the current test case. + const TestInfo* const first_test_info = test_case->test_info_list()[0]; + const internal::TypeId first_fixture_id = first_test_info->fixture_class_id_; + const char* const first_test_name = first_test_info->name(); + + // Info about the current test. + const TestInfo* const this_test_info = impl->current_test_info(); + const internal::TypeId this_fixture_id = this_test_info->fixture_class_id_; + const char* const this_test_name = this_test_info->name(); + + if (this_fixture_id != first_fixture_id) { + // Is the first test defined using TEST? + const bool first_is_TEST = first_fixture_id == internal::GetTestTypeId(); + // Is this test defined using TEST? + const bool this_is_TEST = this_fixture_id == internal::GetTestTypeId(); + + if (first_is_TEST || this_is_TEST) { + // Both TEST and TEST_F appear in same test case, which is incorrect. + // Tell the user how to fix this. + + // Gets the name of the TEST and the name of the TEST_F. Note + // that first_is_TEST and this_is_TEST cannot both be true, as + // the fixture IDs are different for the two tests. + const char* const TEST_name = + first_is_TEST ? first_test_name : this_test_name; + const char* const TEST_F_name = + first_is_TEST ? this_test_name : first_test_name; + + ADD_FAILURE() + << "All tests in the same test case must use the same test fixture\n" + << "class, so mixing TEST_F and TEST in the same test case is\n" + << "illegal. In test case " << this_test_info->test_case_name() + << ",\n" + << "test " << TEST_F_name << " is defined using TEST_F but\n" + << "test " << TEST_name << " is defined using TEST. You probably\n" + << "want to change the TEST to TEST_F or move it to another test\n" + << "case."; + } else { + // Two fixture classes with the same name appear in two different + // namespaces, which is not allowed. Tell the user how to fix this. + ADD_FAILURE() + << "All tests in the same test case must use the same test fixture\n" + << "class. However, in test case " + << this_test_info->test_case_name() << ",\n" + << "you defined test " << first_test_name + << " and test " << this_test_name << "\n" + << "using two different test fixture classes. This can happen if\n" + << "the two classes are from different namespaces or translation\n" + << "units and have the same name. You should probably rename one\n" + << "of the classes to put the tests into different test cases."; + } + return false; + } + + return true; +} + +#if GTEST_HAS_SEH + +// Adds an "exception thrown" fatal failure to the current test. This +// function returns its result via an output parameter pointer because VC++ +// prohibits creation of objects with destructors on stack in functions +// using __try (see error C2712). +static std::string* FormatSehExceptionMessage(DWORD exception_code, + const char* location) { + Message message; + message << "SEH exception with code 0x" << std::setbase(16) << + exception_code << std::setbase(10) << " thrown in " << location << "."; + + return new std::string(message.GetString()); +} + +#endif // GTEST_HAS_SEH + +namespace internal { + +#if GTEST_HAS_EXCEPTIONS + +// Adds an "exception thrown" fatal failure to the current test. +static std::string FormatCxxExceptionMessage(const char* description, + const char* location) { + Message message; + if (description != NULL) { + message << "C++ exception with description \"" << description << "\""; + } else { + message << "Unknown C++ exception"; + } + message << " thrown in " << location << "."; + + return message.GetString(); +} + +static std::string PrintTestPartResultToString( + const TestPartResult& test_part_result); + +GoogleTestFailureException::GoogleTestFailureException( + const TestPartResult& failure) + : ::std::runtime_error(PrintTestPartResultToString(failure).c_str()) {} + +#endif // GTEST_HAS_EXCEPTIONS + +// We put these helper functions in the internal namespace as IBM's xlC +// compiler rejects the code if they were declared static. + +// Runs the given method and handles SEH exceptions it throws, when +// SEH is supported; returns the 0-value for type Result in case of an +// SEH exception. (Microsoft compilers cannot handle SEH and C++ +// exceptions in the same function. Therefore, we provide a separate +// wrapper function for handling SEH exceptions.) +template +Result HandleSehExceptionsInMethodIfSupported( + T* object, Result (T::*method)(), const char* location) { +#if GTEST_HAS_SEH + __try { + return (object->*method)(); + } __except (internal::UnitTestOptions::GTestShouldProcessSEH( // NOLINT + GetExceptionCode())) { + // We create the exception message on the heap because VC++ prohibits + // creation of objects with destructors on stack in functions using __try + // (see error C2712). + std::string* exception_message = FormatSehExceptionMessage( + GetExceptionCode(), location); + internal::ReportFailureInUnknownLocation(TestPartResult::kFatalFailure, + *exception_message); + delete exception_message; + return static_cast(0); + } +#else + (void)location; + return (object->*method)(); +#endif // GTEST_HAS_SEH +} + +// Runs the given method and catches and reports C++ and/or SEH-style +// exceptions, if they are supported; returns the 0-value for type +// Result in case of an SEH exception. +template +Result HandleExceptionsInMethodIfSupported( + T* object, Result (T::*method)(), const char* location) { + // NOTE: The user code can affect the way in which Google Test handles + // exceptions by setting GTEST_FLAG(catch_exceptions), but only before + // RUN_ALL_TESTS() starts. It is technically possible to check the flag + // after the exception is caught and either report or re-throw the + // exception based on the flag's value: + // + // try { + // // Perform the test method. + // } catch (...) { + // if (GTEST_FLAG(catch_exceptions)) + // // Report the exception as failure. + // else + // throw; // Re-throws the original exception. + // } + // + // However, the purpose of this flag is to allow the program to drop into + // the debugger when the exception is thrown. On most platforms, once the + // control enters the catch block, the exception origin information is + // lost and the debugger will stop the program at the point of the + // re-throw in this function -- instead of at the point of the original + // throw statement in the code under test. For this reason, we perform + // the check early, sacrificing the ability to affect Google Test's + // exception handling in the method where the exception is thrown. + if (internal::GetUnitTestImpl()->catch_exceptions()) { +#if GTEST_HAS_EXCEPTIONS + try { + return HandleSehExceptionsInMethodIfSupported(object, method, location); + } catch (const AssertionException&) { // NOLINT + // This failure was reported already. + } catch (const internal::GoogleTestFailureException&) { // NOLINT + // This exception type can only be thrown by a failed Google + // Test assertion with the intention of letting another testing + // framework catch it. Therefore we just re-throw it. + throw; + } catch (const std::exception& e) { // NOLINT + internal::ReportFailureInUnknownLocation( + TestPartResult::kFatalFailure, + FormatCxxExceptionMessage(e.what(), location)); + } catch (...) { // NOLINT + internal::ReportFailureInUnknownLocation( + TestPartResult::kFatalFailure, + FormatCxxExceptionMessage(NULL, location)); + } + return static_cast(0); +#else + return HandleSehExceptionsInMethodIfSupported(object, method, location); +#endif // GTEST_HAS_EXCEPTIONS + } else { + return (object->*method)(); + } +} + +} // namespace internal + +// Runs the test and updates the test result. +void Test::Run() { + if (!HasSameFixtureClass()) return; + + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported(this, &Test::SetUp, "SetUp()"); + // We will run the test only if SetUp() was successful. + if (!HasFatalFailure()) { + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported( + this, &Test::TestBody, "the test body"); + } + + // However, we want to clean up as much as possible. Hence we will + // always call TearDown(), even if SetUp() or the test body has + // failed. + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported( + this, &Test::TearDown, "TearDown()"); +} + +// Returns true iff the current test has a fatal failure. +bool Test::HasFatalFailure() { + return internal::GetUnitTestImpl()->current_test_result()->HasFatalFailure(); +} + +// Returns true iff the current test has a non-fatal failure. +bool Test::HasNonfatalFailure() { + return internal::GetUnitTestImpl()->current_test_result()-> + HasNonfatalFailure(); +} + +// class TestInfo + +// Constructs a TestInfo object. It assumes ownership of the test factory +// object. +TestInfo::TestInfo(const std::string& a_test_case_name, + const std::string& a_name, + const char* a_type_param, + const char* a_value_param, + internal::CodeLocation a_code_location, + internal::TypeId fixture_class_id, + internal::TestFactoryBase* factory) + : test_case_name_(a_test_case_name), + name_(a_name), + type_param_(a_type_param ? new std::string(a_type_param) : NULL), + value_param_(a_value_param ? new std::string(a_value_param) : NULL), + location_(a_code_location), + fixture_class_id_(fixture_class_id), + should_run_(false), + is_disabled_(false), + matches_filter_(false), + factory_(factory), + result_() {} + +// Destructs a TestInfo object. +TestInfo::~TestInfo() { delete factory_; } + +namespace internal { + +// Creates a new TestInfo object and registers it with Google Test; +// returns the created object. +// +// Arguments: +// +// test_case_name: name of the test case +// name: name of the test +// type_param: the name of the test's type parameter, or NULL if +// this is not a typed or a type-parameterized test. +// value_param: text representation of the test's value parameter, +// or NULL if this is not a value-parameterized test. +// code_location: code location where the test is defined +// fixture_class_id: ID of the test fixture class +// set_up_tc: pointer to the function that sets up the test case +// tear_down_tc: pointer to the function that tears down the test case +// factory: pointer to the factory that creates a test object. +// The newly created TestInfo instance will assume +// ownership of the factory object. +TestInfo* MakeAndRegisterTestInfo( + const char* test_case_name, + const char* name, + const char* type_param, + const char* value_param, + CodeLocation code_location, + TypeId fixture_class_id, + SetUpTestCaseFunc set_up_tc, + TearDownTestCaseFunc tear_down_tc, + TestFactoryBase* factory) { + TestInfo* const test_info = + new TestInfo(test_case_name, name, type_param, value_param, + code_location, fixture_class_id, factory); + GetUnitTestImpl()->AddTestInfo(set_up_tc, tear_down_tc, test_info); + return test_info; +} + +void ReportInvalidTestCaseType(const char* test_case_name, + CodeLocation code_location) { + Message errors; + errors + << "Attempted redefinition of test case " << test_case_name << ".\n" + << "All tests in the same test case must use the same test fixture\n" + << "class. However, in test case " << test_case_name << ", you tried\n" + << "to define a test using a fixture class different from the one\n" + << "used earlier. This can happen if the two fixture classes are\n" + << "from different namespaces and have the same name. You should\n" + << "probably rename one of the classes to put the tests into different\n" + << "test cases."; + + GTEST_LOG_(ERROR) << FormatFileLocation(code_location.file.c_str(), + code_location.line) + << " " << errors.GetString(); +} +} // namespace internal + +namespace { + +// A predicate that checks the test name of a TestInfo against a known +// value. +// +// This is used for implementation of the TestCase class only. We put +// it in the anonymous namespace to prevent polluting the outer +// namespace. +// +// TestNameIs is copyable. +class TestNameIs { + public: + // Constructor. + // + // TestNameIs has NO default constructor. + explicit TestNameIs(const char* name) + : name_(name) {} + + // Returns true iff the test name of test_info matches name_. + bool operator()(const TestInfo * test_info) const { + return test_info && test_info->name() == name_; + } + + private: + std::string name_; +}; + +} // namespace + +namespace internal { + +// This method expands all parameterized tests registered with macros TEST_P +// and INSTANTIATE_TEST_CASE_P into regular tests and registers those. +// This will be done just once during the program runtime. +void UnitTestImpl::RegisterParameterizedTests() { + if (!parameterized_tests_registered_) { + parameterized_test_registry_.RegisterTests(); + parameterized_tests_registered_ = true; + } +} + +} // namespace internal + +// Creates the test object, runs it, records its result, and then +// deletes it. +void TestInfo::Run() { + if (!should_run_) return; + + // Tells UnitTest where to store test result. + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + impl->set_current_test_info(this); + + TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater(); + + // Notifies the unit test event listeners that a test is about to start. + repeater->OnTestStart(*this); + + const TimeInMillis start = internal::GetTimeInMillis(); + + impl->os_stack_trace_getter()->UponLeavingGTest(); + + // Creates the test object. + Test* const test = internal::HandleExceptionsInMethodIfSupported( + factory_, &internal::TestFactoryBase::CreateTest, + "the test fixture's constructor"); + + // Runs the test if the constructor didn't generate a fatal failure. + // Note that the object will not be null + if (!Test::HasFatalFailure()) { + // This doesn't throw as all user code that can throw are wrapped into + // exception handling code. + test->Run(); + } + + // Deletes the test object. + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported( + test, &Test::DeleteSelf_, "the test fixture's destructor"); + + result_.set_elapsed_time(internal::GetTimeInMillis() - start); + + // Notifies the unit test event listener that a test has just finished. + repeater->OnTestEnd(*this); + + // Tells UnitTest to stop associating assertion results to this + // test. + impl->set_current_test_info(NULL); +} + +// class TestCase + +// Gets the number of successful tests in this test case. +int TestCase::successful_test_count() const { + return CountIf(test_info_list_, TestPassed); +} + +// Gets the number of failed tests in this test case. +int TestCase::failed_test_count() const { + return CountIf(test_info_list_, TestFailed); +} + +// Gets the number of disabled tests that will be reported in the XML report. +int TestCase::reportable_disabled_test_count() const { + return CountIf(test_info_list_, TestReportableDisabled); +} + +// Gets the number of disabled tests in this test case. +int TestCase::disabled_test_count() const { + return CountIf(test_info_list_, TestDisabled); +} + +// Gets the number of tests to be printed in the XML report. +int TestCase::reportable_test_count() const { + return CountIf(test_info_list_, TestReportable); +} + +// Get the number of tests in this test case that should run. +int TestCase::test_to_run_count() const { + return CountIf(test_info_list_, ShouldRunTest); +} + +// Gets the number of all tests. +int TestCase::total_test_count() const { + return static_cast(test_info_list_.size()); +} + +// Creates a TestCase with the given name. +// +// Arguments: +// +// name: name of the test case +// a_type_param: the name of the test case's type parameter, or NULL if +// this is not a typed or a type-parameterized test case. +// set_up_tc: pointer to the function that sets up the test case +// tear_down_tc: pointer to the function that tears down the test case +TestCase::TestCase(const char* a_name, const char* a_type_param, + Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc) + : name_(a_name), + type_param_(a_type_param ? new std::string(a_type_param) : NULL), + set_up_tc_(set_up_tc), + tear_down_tc_(tear_down_tc), + should_run_(false), + elapsed_time_(0) { +} + +// Destructor of TestCase. +TestCase::~TestCase() { + // Deletes every Test in the collection. + ForEach(test_info_list_, internal::Delete); +} + +// Returns the i-th test among all the tests. i can range from 0 to +// total_test_count() - 1. If i is not in that range, returns NULL. +const TestInfo* TestCase::GetTestInfo(int i) const { + const int index = GetElementOr(test_indices_, i, -1); + return index < 0 ? NULL : test_info_list_[index]; +} + +// Returns the i-th test among all the tests. i can range from 0 to +// total_test_count() - 1. If i is not in that range, returns NULL. +TestInfo* TestCase::GetMutableTestInfo(int i) { + const int index = GetElementOr(test_indices_, i, -1); + return index < 0 ? NULL : test_info_list_[index]; +} + +// Adds a test to this test case. Will delete the test upon +// destruction of the TestCase object. +void TestCase::AddTestInfo(TestInfo * test_info) { + test_info_list_.push_back(test_info); + test_indices_.push_back(static_cast(test_indices_.size())); +} + +// Runs every test in this TestCase. +void TestCase::Run() { + if (!should_run_) return; + + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + impl->set_current_test_case(this); + + TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater(); + + repeater->OnTestCaseStart(*this); + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported( + this, &TestCase::RunSetUpTestCase, "SetUpTestCase()"); + + const internal::TimeInMillis start = internal::GetTimeInMillis(); + for (int i = 0; i < total_test_count(); i++) { + GetMutableTestInfo(i)->Run(); + } + elapsed_time_ = internal::GetTimeInMillis() - start; + + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported( + this, &TestCase::RunTearDownTestCase, "TearDownTestCase()"); + + repeater->OnTestCaseEnd(*this); + impl->set_current_test_case(NULL); +} + +// Clears the results of all tests in this test case. +void TestCase::ClearResult() { + ad_hoc_test_result_.Clear(); + ForEach(test_info_list_, TestInfo::ClearTestResult); +} + +// Shuffles the tests in this test case. +void TestCase::ShuffleTests(internal::Random* random) { + Shuffle(random, &test_indices_); +} + +// Restores the test order to before the first shuffle. +void TestCase::UnshuffleTests() { + for (size_t i = 0; i < test_indices_.size(); i++) { + test_indices_[i] = static_cast(i); + } +} + +// Formats a countable noun. Depending on its quantity, either the +// singular form or the plural form is used. e.g. +// +// FormatCountableNoun(1, "formula", "formuli") returns "1 formula". +// FormatCountableNoun(5, "book", "books") returns "5 books". +static std::string FormatCountableNoun(int count, + const char * singular_form, + const char * plural_form) { + return internal::StreamableToString(count) + " " + + (count == 1 ? singular_form : plural_form); +} + +// Formats the count of tests. +static std::string FormatTestCount(int test_count) { + return FormatCountableNoun(test_count, "test", "tests"); +} + +// Formats the count of test cases. +static std::string FormatTestCaseCount(int test_case_count) { + return FormatCountableNoun(test_case_count, "test case", "test cases"); +} + +// Converts a TestPartResult::Type enum to human-friendly string +// representation. Both kNonFatalFailure and kFatalFailure are translated +// to "Failure", as the user usually doesn't care about the difference +// between the two when viewing the test result. +static const char * TestPartResultTypeToString(TestPartResult::Type type) { + switch (type) { + case TestPartResult::kSuccess: + return "Success"; + + case TestPartResult::kNonFatalFailure: + case TestPartResult::kFatalFailure: +#ifdef _MSC_VER + return "error: "; +#else + return "Failure\n"; +#endif + default: + return "Unknown result type"; + } +} + +namespace internal { + +// Prints a TestPartResult to an std::string. +static std::string PrintTestPartResultToString( + const TestPartResult& test_part_result) { + return (Message() + << internal::FormatFileLocation(test_part_result.file_name(), + test_part_result.line_number()) + << " " << TestPartResultTypeToString(test_part_result.type()) + << test_part_result.message()).GetString(); +} + +// Prints a TestPartResult. +static void PrintTestPartResult(const TestPartResult& test_part_result) { + const std::string& result = + PrintTestPartResultToString(test_part_result); + printf("%s\n", result.c_str()); + fflush(stdout); + // If the test program runs in Visual Studio or a debugger, the + // following statements add the test part result message to the Output + // window such that the user can double-click on it to jump to the + // corresponding source code location; otherwise they do nothing. +#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE + // We don't call OutputDebugString*() on Windows Mobile, as printing + // to stdout is done by OutputDebugString() there already - we don't + // want the same message printed twice. + ::OutputDebugStringA(result.c_str()); + ::OutputDebugStringA("\n"); +#endif +} + +// class PrettyUnitTestResultPrinter + +enum GTestColor { + COLOR_DEFAULT, + COLOR_RED, + COLOR_GREEN, + COLOR_YELLOW +}; + +#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE && \ + !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT && !GTEST_OS_WINDOWS_MINGW + +// Returns the character attribute for the given color. +static WORD GetColorAttribute(GTestColor color) { + switch (color) { + case COLOR_RED: return FOREGROUND_RED; + case COLOR_GREEN: return FOREGROUND_GREEN; + case COLOR_YELLOW: return FOREGROUND_RED | FOREGROUND_GREEN; + default: return 0; + } +} + +static int GetBitOffset(WORD color_mask) { + if (color_mask == 0) return 0; + + int bitOffset = 0; + while ((color_mask & 1) == 0) { + color_mask >>= 1; + ++bitOffset; + } + return bitOffset; +} + +static WORD GetNewColor(GTestColor color, WORD old_color_attrs) { + // Let's reuse the BG + static const WORD background_mask = BACKGROUND_BLUE | BACKGROUND_GREEN | + BACKGROUND_RED | BACKGROUND_INTENSITY; + static const WORD foreground_mask = FOREGROUND_BLUE | FOREGROUND_GREEN | + FOREGROUND_RED | FOREGROUND_INTENSITY; + const WORD existing_bg = old_color_attrs & background_mask; + + WORD new_color = + GetColorAttribute(color) | existing_bg | FOREGROUND_INTENSITY; + static const int bg_bitOffset = GetBitOffset(background_mask); + static const int fg_bitOffset = GetBitOffset(foreground_mask); + + if (((new_color & background_mask) >> bg_bitOffset) == + ((new_color & foreground_mask) >> fg_bitOffset)) { + new_color ^= FOREGROUND_INTENSITY; // invert intensity + } + return new_color; +} + +#else + +// Returns the ANSI color code for the given color. COLOR_DEFAULT is +// an invalid input. +static const char* GetAnsiColorCode(GTestColor color) { + switch (color) { + case COLOR_RED: return "1"; + case COLOR_GREEN: return "2"; + case COLOR_YELLOW: return "3"; + default: return NULL; + }; +} + +#endif // GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE + +// Returns true iff Google Test should use colors in the output. +bool ShouldUseColor(bool stdout_is_tty) { + const char* const gtest_color = GTEST_FLAG(color).c_str(); + + if (String::CaseInsensitiveCStringEquals(gtest_color, "auto")) { +#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MINGW + // On Windows the TERM variable is usually not set, but the + // console there does support colors. + return stdout_is_tty; +#else + // On non-Windows platforms, we rely on the TERM variable. + const char* const term = posix::GetEnv("TERM"); + const bool term_supports_color = + String::CStringEquals(term, "xterm") || + String::CStringEquals(term, "xterm-color") || + String::CStringEquals(term, "xterm-256color") || + String::CStringEquals(term, "screen") || + String::CStringEquals(term, "screen-256color") || + String::CStringEquals(term, "tmux") || + String::CStringEquals(term, "tmux-256color") || + String::CStringEquals(term, "rxvt-unicode") || + String::CStringEquals(term, "rxvt-unicode-256color") || + String::CStringEquals(term, "linux") || + String::CStringEquals(term, "cygwin"); + return stdout_is_tty && term_supports_color; +#endif // GTEST_OS_WINDOWS + } + + return String::CaseInsensitiveCStringEquals(gtest_color, "yes") || + String::CaseInsensitiveCStringEquals(gtest_color, "true") || + String::CaseInsensitiveCStringEquals(gtest_color, "t") || + String::CStringEquals(gtest_color, "1"); + // We take "yes", "true", "t", and "1" as meaning "yes". If the + // value is neither one of these nor "auto", we treat it as "no" to + // be conservative. +} + +// Helpers for printing colored strings to stdout. Note that on Windows, we +// cannot simply emit special characters and have the terminal change colors. +// This routine must actually emit the characters rather than return a string +// that would be colored when printed, as can be done on Linux. +static void ColoredPrintf(GTestColor color, const char* fmt, ...) { + va_list args; + va_start(args, fmt); + +#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN || GTEST_OS_ZOS || \ + GTEST_OS_IOS || GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT + const bool use_color = AlwaysFalse(); +#else + static const bool in_color_mode = + ShouldUseColor(posix::IsATTY(posix::FileNo(stdout)) != 0); + const bool use_color = in_color_mode && (color != COLOR_DEFAULT); +#endif // GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN || GTEST_OS_ZOS + // The '!= 0' comparison is necessary to satisfy MSVC 7.1. + + if (!use_color) { + vprintf(fmt, args); + va_end(args); + return; + } + +#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE && \ + !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT && !GTEST_OS_WINDOWS_MINGW + const HANDLE stdout_handle = GetStdHandle(STD_OUTPUT_HANDLE); + + // Gets the current text color. + CONSOLE_SCREEN_BUFFER_INFO buffer_info; + GetConsoleScreenBufferInfo(stdout_handle, &buffer_info); + const WORD old_color_attrs = buffer_info.wAttributes; + const WORD new_color = GetNewColor(color, old_color_attrs); + + // We need to flush the stream buffers into the console before each + // SetConsoleTextAttribute call lest it affect the text that is already + // printed but has not yet reached the console. + fflush(stdout); + SetConsoleTextAttribute(stdout_handle, new_color); + + vprintf(fmt, args); + + fflush(stdout); + // Restores the text color. + SetConsoleTextAttribute(stdout_handle, old_color_attrs); +#else + printf("\033[0;3%sm", GetAnsiColorCode(color)); + vprintf(fmt, args); + printf("\033[m"); // Resets the terminal to default. +#endif // GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE + va_end(args); +} + +// Text printed in Google Test's text output and --gtest_list_tests +// output to label the type parameter and value parameter for a test. +static const char kTypeParamLabel[] = "TypeParam"; +static const char kValueParamLabel[] = "GetParam()"; + +static void PrintFullTestCommentIfPresent(const TestInfo& test_info) { + const char* const type_param = test_info.type_param(); + const char* const value_param = test_info.value_param(); + + if (type_param != NULL || value_param != NULL) { + printf(", where "); + if (type_param != NULL) { + printf("%s = %s", kTypeParamLabel, type_param); + if (value_param != NULL) + printf(" and "); + } + if (value_param != NULL) { + printf("%s = %s", kValueParamLabel, value_param); + } + } +} + +// This class implements the TestEventListener interface. +// +// Class PrettyUnitTestResultPrinter is copyable. +class PrettyUnitTestResultPrinter : public TestEventListener { + public: + PrettyUnitTestResultPrinter() {} + static void PrintTestName(const char * test_case, const char * test) { + printf("%s.%s", test_case, test); + } + + // The following methods override what's in the TestEventListener class. + virtual void OnTestProgramStart(const UnitTest& /*unit_test*/) {} + virtual void OnTestIterationStart(const UnitTest& unit_test, int iteration); + virtual void OnEnvironmentsSetUpStart(const UnitTest& unit_test); + virtual void OnEnvironmentsSetUpEnd(const UnitTest& /*unit_test*/) {} + virtual void OnTestCaseStart(const TestCase& test_case); + virtual void OnTestStart(const TestInfo& test_info); + virtual void OnTestPartResult(const TestPartResult& result); + virtual void OnTestEnd(const TestInfo& test_info); + virtual void OnTestCaseEnd(const TestCase& test_case); + virtual void OnEnvironmentsTearDownStart(const UnitTest& unit_test); + virtual void OnEnvironmentsTearDownEnd(const UnitTest& /*unit_test*/) {} + virtual void OnTestIterationEnd(const UnitTest& unit_test, int iteration); + virtual void OnTestProgramEnd(const UnitTest& /*unit_test*/) {} + + private: + static void PrintFailedTests(const UnitTest& unit_test); +}; + + // Fired before each iteration of tests starts. +void PrettyUnitTestResultPrinter::OnTestIterationStart( + const UnitTest& unit_test, int iteration) { + if (GTEST_FLAG(repeat) != 1) + printf("\nRepeating all tests (iteration %d) . . .\n\n", iteration + 1); + + const char* const filter = GTEST_FLAG(filter).c_str(); + + // Prints the filter if it's not *. This reminds the user that some + // tests may be skipped. + if (!String::CStringEquals(filter, kUniversalFilter)) { + ColoredPrintf(COLOR_YELLOW, + "Note: %s filter = %s\n", GTEST_NAME_, filter); + } + + if (internal::ShouldShard(kTestTotalShards, kTestShardIndex, false)) { + const Int32 shard_index = Int32FromEnvOrDie(kTestShardIndex, -1); + ColoredPrintf(COLOR_YELLOW, + "Note: This is test shard %d of %s.\n", + static_cast(shard_index) + 1, + internal::posix::GetEnv(kTestTotalShards)); + } + + if (GTEST_FLAG(shuffle)) { + ColoredPrintf(COLOR_YELLOW, + "Note: Randomizing tests' orders with a seed of %d .\n", + unit_test.random_seed()); + } + + ColoredPrintf(COLOR_GREEN, "[==========] "); + printf("Running %s from %s.\n", + FormatTestCount(unit_test.test_to_run_count()).c_str(), + FormatTestCaseCount(unit_test.test_case_to_run_count()).c_str()); + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnEnvironmentsSetUpStart( + const UnitTest& /*unit_test*/) { + ColoredPrintf(COLOR_GREEN, "[----------] "); + printf("Global test environment set-up.\n"); + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnTestCaseStart(const TestCase& test_case) { + const std::string counts = + FormatCountableNoun(test_case.test_to_run_count(), "test", "tests"); + ColoredPrintf(COLOR_GREEN, "[----------] "); + printf("%s from %s", counts.c_str(), test_case.name()); + if (test_case.type_param() == NULL) { + printf("\n"); + } else { + printf(", where %s = %s\n", kTypeParamLabel, test_case.type_param()); + } + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnTestStart(const TestInfo& test_info) { + ColoredPrintf(COLOR_GREEN, "[ RUN ] "); + PrintTestName(test_info.test_case_name(), test_info.name()); + printf("\n"); + fflush(stdout); +} + +// Called after an assertion failure. +void PrettyUnitTestResultPrinter::OnTestPartResult( + const TestPartResult& result) { + // If the test part succeeded, we don't need to do anything. + if (result.type() == TestPartResult::kSuccess) + return; + + // Print failure message from the assertion (e.g. expected this and got that). + PrintTestPartResult(result); + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnTestEnd(const TestInfo& test_info) { + if (test_info.result()->Passed()) { + ColoredPrintf(COLOR_GREEN, "[ OK ] "); + } else { + ColoredPrintf(COLOR_RED, "[ FAILED ] "); + } + PrintTestName(test_info.test_case_name(), test_info.name()); + if (test_info.result()->Failed()) + PrintFullTestCommentIfPresent(test_info); + + if (GTEST_FLAG(print_time)) { + printf(" (%s ms)\n", internal::StreamableToString( + test_info.result()->elapsed_time()).c_str()); + } else { + printf("\n"); + } + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnTestCaseEnd(const TestCase& test_case) { + if (!GTEST_FLAG(print_time)) return; + + const std::string counts = + FormatCountableNoun(test_case.test_to_run_count(), "test", "tests"); + ColoredPrintf(COLOR_GREEN, "[----------] "); + printf("%s from %s (%s ms total)\n\n", + counts.c_str(), test_case.name(), + internal::StreamableToString(test_case.elapsed_time()).c_str()); + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnEnvironmentsTearDownStart( + const UnitTest& /*unit_test*/) { + ColoredPrintf(COLOR_GREEN, "[----------] "); + printf("Global test environment tear-down\n"); + fflush(stdout); +} + +// Internal helper for printing the list of failed tests. +void PrettyUnitTestResultPrinter::PrintFailedTests(const UnitTest& unit_test) { + const int failed_test_count = unit_test.failed_test_count(); + if (failed_test_count == 0) { + return; + } + + for (int i = 0; i < unit_test.total_test_case_count(); ++i) { + const TestCase& test_case = *unit_test.GetTestCase(i); + if (!test_case.should_run() || (test_case.failed_test_count() == 0)) { + continue; + } + for (int j = 0; j < test_case.total_test_count(); ++j) { + const TestInfo& test_info = *test_case.GetTestInfo(j); + if (!test_info.should_run() || test_info.result()->Passed()) { + continue; + } + ColoredPrintf(COLOR_RED, "[ FAILED ] "); + printf("%s.%s", test_case.name(), test_info.name()); + PrintFullTestCommentIfPresent(test_info); + printf("\n"); + } + } +} + +void PrettyUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test, + int /*iteration*/) { + ColoredPrintf(COLOR_GREEN, "[==========] "); + printf("%s from %s ran.", + FormatTestCount(unit_test.test_to_run_count()).c_str(), + FormatTestCaseCount(unit_test.test_case_to_run_count()).c_str()); + if (GTEST_FLAG(print_time)) { + printf(" (%s ms total)", + internal::StreamableToString(unit_test.elapsed_time()).c_str()); + } + printf("\n"); + ColoredPrintf(COLOR_GREEN, "[ PASSED ] "); + printf("%s.\n", FormatTestCount(unit_test.successful_test_count()).c_str()); + + int num_failures = unit_test.failed_test_count(); + if (!unit_test.Passed()) { + const int failed_test_count = unit_test.failed_test_count(); + ColoredPrintf(COLOR_RED, "[ FAILED ] "); + printf("%s, listed below:\n", FormatTestCount(failed_test_count).c_str()); + PrintFailedTests(unit_test); + printf("\n%2d FAILED %s\n", num_failures, + num_failures == 1 ? "TEST" : "TESTS"); + } + + int num_disabled = unit_test.reportable_disabled_test_count(); + if (num_disabled && !GTEST_FLAG(also_run_disabled_tests)) { + if (!num_failures) { + printf("\n"); // Add a spacer if no FAILURE banner is displayed. + } + ColoredPrintf(COLOR_YELLOW, + " YOU HAVE %d DISABLED %s\n\n", + num_disabled, + num_disabled == 1 ? "TEST" : "TESTS"); + } + // Ensure that Google Test output is printed before, e.g., heapchecker output. + fflush(stdout); +} + +// End PrettyUnitTestResultPrinter + +// class TestEventRepeater +// +// This class forwards events to other event listeners. +class TestEventRepeater : public TestEventListener { + public: + TestEventRepeater() : forwarding_enabled_(true) {} + virtual ~TestEventRepeater(); + void Append(TestEventListener *listener); + TestEventListener* Release(TestEventListener* listener); + + // Controls whether events will be forwarded to listeners_. Set to false + // in death test child processes. + bool forwarding_enabled() const { return forwarding_enabled_; } + void set_forwarding_enabled(bool enable) { forwarding_enabled_ = enable; } + + virtual void OnTestProgramStart(const UnitTest& unit_test); + virtual void OnTestIterationStart(const UnitTest& unit_test, int iteration); + virtual void OnEnvironmentsSetUpStart(const UnitTest& unit_test); + virtual void OnEnvironmentsSetUpEnd(const UnitTest& unit_test); + virtual void OnTestCaseStart(const TestCase& test_case); + virtual void OnTestStart(const TestInfo& test_info); + virtual void OnTestPartResult(const TestPartResult& result); + virtual void OnTestEnd(const TestInfo& test_info); + virtual void OnTestCaseEnd(const TestCase& test_case); + virtual void OnEnvironmentsTearDownStart(const UnitTest& unit_test); + virtual void OnEnvironmentsTearDownEnd(const UnitTest& unit_test); + virtual void OnTestIterationEnd(const UnitTest& unit_test, int iteration); + virtual void OnTestProgramEnd(const UnitTest& unit_test); + + private: + // Controls whether events will be forwarded to listeners_. Set to false + // in death test child processes. + bool forwarding_enabled_; + // The list of listeners that receive events. + std::vector listeners_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestEventRepeater); +}; + +TestEventRepeater::~TestEventRepeater() { + ForEach(listeners_, Delete); +} + +void TestEventRepeater::Append(TestEventListener *listener) { + listeners_.push_back(listener); +} + +// FIXME: Factor the search functionality into Vector::Find. +TestEventListener* TestEventRepeater::Release(TestEventListener *listener) { + for (size_t i = 0; i < listeners_.size(); ++i) { + if (listeners_[i] == listener) { + listeners_.erase(listeners_.begin() + i); + return listener; + } + } + + return NULL; +} + +// Since most methods are very similar, use macros to reduce boilerplate. +// This defines a member that forwards the call to all listeners. +#define GTEST_REPEATER_METHOD_(Name, Type) \ +void TestEventRepeater::Name(const Type& parameter) { \ + if (forwarding_enabled_) { \ + for (size_t i = 0; i < listeners_.size(); i++) { \ + listeners_[i]->Name(parameter); \ + } \ + } \ +} +// This defines a member that forwards the call to all listeners in reverse +// order. +#define GTEST_REVERSE_REPEATER_METHOD_(Name, Type) \ +void TestEventRepeater::Name(const Type& parameter) { \ + if (forwarding_enabled_) { \ + for (int i = static_cast(listeners_.size()) - 1; i >= 0; i--) { \ + listeners_[i]->Name(parameter); \ + } \ + } \ +} + +GTEST_REPEATER_METHOD_(OnTestProgramStart, UnitTest) +GTEST_REPEATER_METHOD_(OnEnvironmentsSetUpStart, UnitTest) +GTEST_REPEATER_METHOD_(OnTestCaseStart, TestCase) +GTEST_REPEATER_METHOD_(OnTestStart, TestInfo) +GTEST_REPEATER_METHOD_(OnTestPartResult, TestPartResult) +GTEST_REPEATER_METHOD_(OnEnvironmentsTearDownStart, UnitTest) +GTEST_REVERSE_REPEATER_METHOD_(OnEnvironmentsSetUpEnd, UnitTest) +GTEST_REVERSE_REPEATER_METHOD_(OnEnvironmentsTearDownEnd, UnitTest) +GTEST_REVERSE_REPEATER_METHOD_(OnTestEnd, TestInfo) +GTEST_REVERSE_REPEATER_METHOD_(OnTestCaseEnd, TestCase) +GTEST_REVERSE_REPEATER_METHOD_(OnTestProgramEnd, UnitTest) + +#undef GTEST_REPEATER_METHOD_ +#undef GTEST_REVERSE_REPEATER_METHOD_ + +void TestEventRepeater::OnTestIterationStart(const UnitTest& unit_test, + int iteration) { + if (forwarding_enabled_) { + for (size_t i = 0; i < listeners_.size(); i++) { + listeners_[i]->OnTestIterationStart(unit_test, iteration); + } + } +} + +void TestEventRepeater::OnTestIterationEnd(const UnitTest& unit_test, + int iteration) { + if (forwarding_enabled_) { + for (int i = static_cast(listeners_.size()) - 1; i >= 0; i--) { + listeners_[i]->OnTestIterationEnd(unit_test, iteration); + } + } +} + +// End TestEventRepeater + +// This class generates an XML output file. +class XmlUnitTestResultPrinter : public EmptyTestEventListener { + public: + explicit XmlUnitTestResultPrinter(const char* output_file); + + virtual void OnTestIterationEnd(const UnitTest& unit_test, int iteration); + void ListTestsMatchingFilter(const std::vector& test_cases); + + // Prints an XML summary of all unit tests. + static void PrintXmlTestsList(std::ostream* stream, + const std::vector& test_cases); + + private: + // Is c a whitespace character that is normalized to a space character + // when it appears in an XML attribute value? + static bool IsNormalizableWhitespace(char c) { + return c == 0x9 || c == 0xA || c == 0xD; + } + + // May c appear in a well-formed XML document? + static bool IsValidXmlCharacter(char c) { + return IsNormalizableWhitespace(c) || c >= 0x20; + } + + // Returns an XML-escaped copy of the input string str. If + // is_attribute is true, the text is meant to appear as an attribute + // value, and normalizable whitespace is preserved by replacing it + // with character references. + static std::string EscapeXml(const std::string& str, bool is_attribute); + + // Returns the given string with all characters invalid in XML removed. + static std::string RemoveInvalidXmlCharacters(const std::string& str); + + // Convenience wrapper around EscapeXml when str is an attribute value. + static std::string EscapeXmlAttribute(const std::string& str) { + return EscapeXml(str, true); + } + + // Convenience wrapper around EscapeXml when str is not an attribute value. + static std::string EscapeXmlText(const char* str) { + return EscapeXml(str, false); + } + + // Verifies that the given attribute belongs to the given element and + // streams the attribute as XML. + static void OutputXmlAttribute(std::ostream* stream, + const std::string& element_name, + const std::string& name, + const std::string& value); + + // Streams an XML CDATA section, escaping invalid CDATA sequences as needed. + static void OutputXmlCDataSection(::std::ostream* stream, const char* data); + + // Streams an XML representation of a TestInfo object. + static void OutputXmlTestInfo(::std::ostream* stream, + const char* test_case_name, + const TestInfo& test_info); + + // Prints an XML representation of a TestCase object + static void PrintXmlTestCase(::std::ostream* stream, + const TestCase& test_case); + + // Prints an XML summary of unit_test to output stream out. + static void PrintXmlUnitTest(::std::ostream* stream, + const UnitTest& unit_test); + + // Produces a string representing the test properties in a result as space + // delimited XML attributes based on the property key="value" pairs. + // When the std::string is not empty, it includes a space at the beginning, + // to delimit this attribute from prior attributes. + static std::string TestPropertiesAsXmlAttributes(const TestResult& result); + + // Streams an XML representation of the test properties of a TestResult + // object. + static void OutputXmlTestProperties(std::ostream* stream, + const TestResult& result); + + // The output file. + const std::string output_file_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(XmlUnitTestResultPrinter); +}; + +// Creates a new XmlUnitTestResultPrinter. +XmlUnitTestResultPrinter::XmlUnitTestResultPrinter(const char* output_file) + : output_file_(output_file) { + if (output_file_.empty()) { + GTEST_LOG_(FATAL) << "XML output file may not be null"; + } +} + +// Called after the unit test ends. +void XmlUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test, + int /*iteration*/) { + FILE* xmlout = OpenFileForWriting(output_file_); + std::stringstream stream; + PrintXmlUnitTest(&stream, unit_test); + fprintf(xmlout, "%s", StringStreamToString(&stream).c_str()); + fclose(xmlout); +} + +void XmlUnitTestResultPrinter::ListTestsMatchingFilter( + const std::vector& test_cases) { + FILE* xmlout = OpenFileForWriting(output_file_); + std::stringstream stream; + PrintXmlTestsList(&stream, test_cases); + fprintf(xmlout, "%s", StringStreamToString(&stream).c_str()); + fclose(xmlout); +} + +// Returns an XML-escaped copy of the input string str. If is_attribute +// is true, the text is meant to appear as an attribute value, and +// normalizable whitespace is preserved by replacing it with character +// references. +// +// Invalid XML characters in str, if any, are stripped from the output. +// It is expected that most, if not all, of the text processed by this +// module will consist of ordinary English text. +// If this module is ever modified to produce version 1.1 XML output, +// most invalid characters can be retained using character references. +// FIXME: It might be nice to have a minimally invasive, human-readable +// escaping scheme for invalid characters, rather than dropping them. +std::string XmlUnitTestResultPrinter::EscapeXml( + const std::string& str, bool is_attribute) { + Message m; + + for (size_t i = 0; i < str.size(); ++i) { + const char ch = str[i]; + switch (ch) { + case '<': + m << "<"; + break; + case '>': + m << ">"; + break; + case '&': + m << "&"; + break; + case '\'': + if (is_attribute) + m << "'"; + else + m << '\''; + break; + case '"': + if (is_attribute) + m << """; + else + m << '"'; + break; + default: + if (IsValidXmlCharacter(ch)) { + if (is_attribute && IsNormalizableWhitespace(ch)) + m << "&#x" << String::FormatByte(static_cast(ch)) + << ";"; + else + m << ch; + } + break; + } + } + + return m.GetString(); +} + +// Returns the given string with all characters invalid in XML removed. +// Currently invalid characters are dropped from the string. An +// alternative is to replace them with certain characters such as . or ?. +std::string XmlUnitTestResultPrinter::RemoveInvalidXmlCharacters( + const std::string& str) { + std::string output; + output.reserve(str.size()); + for (std::string::const_iterator it = str.begin(); it != str.end(); ++it) + if (IsValidXmlCharacter(*it)) + output.push_back(*it); + + return output; +} + +// The following routines generate an XML representation of a UnitTest +// object. +// GOOGLETEST_CM0009 DO NOT DELETE +// +// This is how Google Test concepts map to the DTD: +// +// <-- corresponds to a UnitTest object +// <-- corresponds to a TestCase object +// <-- corresponds to a TestInfo object +// ... +// ... +// ... +// <-- individual assertion failures +// +// +// + +// Formats the given time in milliseconds as seconds. +std::string FormatTimeInMillisAsSeconds(TimeInMillis ms) { + ::std::stringstream ss; + ss << (static_cast(ms) * 1e-3); + return ss.str(); +} + +static bool PortableLocaltime(time_t seconds, struct tm* out) { +#if defined(_MSC_VER) + return localtime_s(out, &seconds) == 0; +#elif defined(__MINGW32__) || defined(__MINGW64__) + // MINGW provides neither localtime_r nor localtime_s, but uses + // Windows' localtime(), which has a thread-local tm buffer. + struct tm* tm_ptr = localtime(&seconds); // NOLINT + if (tm_ptr == NULL) + return false; + *out = *tm_ptr; + return true; +#else + return localtime_r(&seconds, out) != NULL; +#endif +} + +// Converts the given epoch time in milliseconds to a date string in the ISO +// 8601 format, without the timezone information. +std::string FormatEpochTimeInMillisAsIso8601(TimeInMillis ms) { + struct tm time_struct; + if (!PortableLocaltime(static_cast(ms / 1000), &time_struct)) + return ""; + // YYYY-MM-DDThh:mm:ss + return StreamableToString(time_struct.tm_year + 1900) + "-" + + String::FormatIntWidth2(time_struct.tm_mon + 1) + "-" + + String::FormatIntWidth2(time_struct.tm_mday) + "T" + + String::FormatIntWidth2(time_struct.tm_hour) + ":" + + String::FormatIntWidth2(time_struct.tm_min) + ":" + + String::FormatIntWidth2(time_struct.tm_sec); +} + +// Streams an XML CDATA section, escaping invalid CDATA sequences as needed. +void XmlUnitTestResultPrinter::OutputXmlCDataSection(::std::ostream* stream, + const char* data) { + const char* segment = data; + *stream << ""); + if (next_segment != NULL) { + stream->write( + segment, static_cast(next_segment - segment)); + *stream << "]]>]]>"); + } else { + *stream << segment; + break; + } + } + *stream << "]]>"; +} + +void XmlUnitTestResultPrinter::OutputXmlAttribute( + std::ostream* stream, + const std::string& element_name, + const std::string& name, + const std::string& value) { + const std::vector& allowed_names = + GetReservedAttributesForElement(element_name); + + GTEST_CHECK_(std::find(allowed_names.begin(), allowed_names.end(), name) != + allowed_names.end()) + << "Attribute " << name << " is not allowed for element <" << element_name + << ">."; + + *stream << " " << name << "=\"" << EscapeXmlAttribute(value) << "\""; +} + +// Prints an XML representation of a TestInfo object. +// FIXME: There is also value in printing properties with the plain printer. +void XmlUnitTestResultPrinter::OutputXmlTestInfo(::std::ostream* stream, + const char* test_case_name, + const TestInfo& test_info) { + const TestResult& result = *test_info.result(); + const std::string kTestcase = "testcase"; + + if (test_info.is_in_another_shard()) { + return; + } + + *stream << " \n"; + return; + } + + OutputXmlAttribute(stream, kTestcase, "status", + test_info.should_run() ? "run" : "notrun"); + OutputXmlAttribute(stream, kTestcase, "time", + FormatTimeInMillisAsSeconds(result.elapsed_time())); + OutputXmlAttribute(stream, kTestcase, "classname", test_case_name); + + int failures = 0; + for (int i = 0; i < result.total_part_count(); ++i) { + const TestPartResult& part = result.GetTestPartResult(i); + if (part.failed()) { + if (++failures == 1) { + *stream << ">\n"; + } + const std::string location = + internal::FormatCompilerIndependentFileLocation(part.file_name(), + part.line_number()); + const std::string summary = location + "\n" + part.summary(); + *stream << " "; + const std::string detail = location + "\n" + part.message(); + OutputXmlCDataSection(stream, RemoveInvalidXmlCharacters(detail).c_str()); + *stream << "\n"; + } + } + + if (failures == 0 && result.test_property_count() == 0) { + *stream << " />\n"; + } else { + if (failures == 0) { + *stream << ">\n"; + } + OutputXmlTestProperties(stream, result); + *stream << " \n"; + } +} + +// Prints an XML representation of a TestCase object +void XmlUnitTestResultPrinter::PrintXmlTestCase(std::ostream* stream, + const TestCase& test_case) { + const std::string kTestsuite = "testsuite"; + *stream << " <" << kTestsuite; + OutputXmlAttribute(stream, kTestsuite, "name", test_case.name()); + OutputXmlAttribute(stream, kTestsuite, "tests", + StreamableToString(test_case.reportable_test_count())); + if (!GTEST_FLAG(list_tests)) { + OutputXmlAttribute(stream, kTestsuite, "failures", + StreamableToString(test_case.failed_test_count())); + OutputXmlAttribute( + stream, kTestsuite, "disabled", + StreamableToString(test_case.reportable_disabled_test_count())); + OutputXmlAttribute(stream, kTestsuite, "errors", "0"); + OutputXmlAttribute(stream, kTestsuite, "time", + FormatTimeInMillisAsSeconds(test_case.elapsed_time())); + *stream << TestPropertiesAsXmlAttributes(test_case.ad_hoc_test_result()); + } + *stream << ">\n"; + for (int i = 0; i < test_case.total_test_count(); ++i) { + if (test_case.GetTestInfo(i)->is_reportable()) + OutputXmlTestInfo(stream, test_case.name(), *test_case.GetTestInfo(i)); + } + *stream << " \n"; +} + +// Prints an XML summary of unit_test to output stream out. +void XmlUnitTestResultPrinter::PrintXmlUnitTest(std::ostream* stream, + const UnitTest& unit_test) { + const std::string kTestsuites = "testsuites"; + + *stream << "\n"; + *stream << "<" << kTestsuites; + + OutputXmlAttribute(stream, kTestsuites, "tests", + StreamableToString(unit_test.reportable_test_count())); + OutputXmlAttribute(stream, kTestsuites, "failures", + StreamableToString(unit_test.failed_test_count())); + OutputXmlAttribute( + stream, kTestsuites, "disabled", + StreamableToString(unit_test.reportable_disabled_test_count())); + OutputXmlAttribute(stream, kTestsuites, "errors", "0"); + OutputXmlAttribute( + stream, kTestsuites, "timestamp", + FormatEpochTimeInMillisAsIso8601(unit_test.start_timestamp())); + OutputXmlAttribute(stream, kTestsuites, "time", + FormatTimeInMillisAsSeconds(unit_test.elapsed_time())); + + if (GTEST_FLAG(shuffle)) { + OutputXmlAttribute(stream, kTestsuites, "random_seed", + StreamableToString(unit_test.random_seed())); + } + *stream << TestPropertiesAsXmlAttributes(unit_test.ad_hoc_test_result()); + + OutputXmlAttribute(stream, kTestsuites, "name", "AllTests"); + *stream << ">\n"; + + for (int i = 0; i < unit_test.total_test_case_count(); ++i) { + if (unit_test.GetTestCase(i)->reportable_test_count() > 0) + PrintXmlTestCase(stream, *unit_test.GetTestCase(i)); + } + *stream << "\n"; +} + +void XmlUnitTestResultPrinter::PrintXmlTestsList( + std::ostream* stream, const std::vector& test_cases) { + const std::string kTestsuites = "testsuites"; + + *stream << "\n"; + *stream << "<" << kTestsuites; + + int total_tests = 0; + for (size_t i = 0; i < test_cases.size(); ++i) { + total_tests += test_cases[i]->total_test_count(); + } + OutputXmlAttribute(stream, kTestsuites, "tests", + StreamableToString(total_tests)); + OutputXmlAttribute(stream, kTestsuites, "name", "AllTests"); + *stream << ">\n"; + + for (size_t i = 0; i < test_cases.size(); ++i) { + PrintXmlTestCase(stream, *test_cases[i]); + } + *stream << "\n"; +} + +// Produces a string representing the test properties in a result as space +// delimited XML attributes based on the property key="value" pairs. +std::string XmlUnitTestResultPrinter::TestPropertiesAsXmlAttributes( + const TestResult& result) { + Message attributes; + for (int i = 0; i < result.test_property_count(); ++i) { + const TestProperty& property = result.GetTestProperty(i); + attributes << " " << property.key() << "=" + << "\"" << EscapeXmlAttribute(property.value()) << "\""; + } + return attributes.GetString(); +} + +void XmlUnitTestResultPrinter::OutputXmlTestProperties( + std::ostream* stream, const TestResult& result) { + const std::string kProperties = "properties"; + const std::string kProperty = "property"; + + if (result.test_property_count() <= 0) { + return; + } + + *stream << "<" << kProperties << ">\n"; + for (int i = 0; i < result.test_property_count(); ++i) { + const TestProperty& property = result.GetTestProperty(i); + *stream << "<" << kProperty; + *stream << " name=\"" << EscapeXmlAttribute(property.key()) << "\""; + *stream << " value=\"" << EscapeXmlAttribute(property.value()) << "\""; + *stream << "/>\n"; + } + *stream << "\n"; +} + +// End XmlUnitTestResultPrinter + +// This class generates an JSON output file. +class JsonUnitTestResultPrinter : public EmptyTestEventListener { + public: + explicit JsonUnitTestResultPrinter(const char* output_file); + + virtual void OnTestIterationEnd(const UnitTest& unit_test, int iteration); + + // Prints an JSON summary of all unit tests. + static void PrintJsonTestList(::std::ostream* stream, + const std::vector& test_cases); + + private: + // Returns an JSON-escaped copy of the input string str. + static std::string EscapeJson(const std::string& str); + + //// Verifies that the given attribute belongs to the given element and + //// streams the attribute as JSON. + static void OutputJsonKey(std::ostream* stream, + const std::string& element_name, + const std::string& name, + const std::string& value, + const std::string& indent, + bool comma = true); + static void OutputJsonKey(std::ostream* stream, + const std::string& element_name, + const std::string& name, + int value, + const std::string& indent, + bool comma = true); + + // Streams a JSON representation of a TestInfo object. + static void OutputJsonTestInfo(::std::ostream* stream, + const char* test_case_name, + const TestInfo& test_info); + + // Prints a JSON representation of a TestCase object + static void PrintJsonTestCase(::std::ostream* stream, + const TestCase& test_case); + + // Prints a JSON summary of unit_test to output stream out. + static void PrintJsonUnitTest(::std::ostream* stream, + const UnitTest& unit_test); + + // Produces a string representing the test properties in a result as + // a JSON dictionary. + static std::string TestPropertiesAsJson(const TestResult& result, + const std::string& indent); + + // The output file. + const std::string output_file_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(JsonUnitTestResultPrinter); +}; + +// Creates a new JsonUnitTestResultPrinter. +JsonUnitTestResultPrinter::JsonUnitTestResultPrinter(const char* output_file) + : output_file_(output_file) { + if (output_file_.empty()) { + GTEST_LOG_(FATAL) << "JSON output file may not be null"; + } +} + +void JsonUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test, + int /*iteration*/) { + FILE* jsonout = OpenFileForWriting(output_file_); + std::stringstream stream; + PrintJsonUnitTest(&stream, unit_test); + fprintf(jsonout, "%s", StringStreamToString(&stream).c_str()); + fclose(jsonout); +} + +// Returns an JSON-escaped copy of the input string str. +std::string JsonUnitTestResultPrinter::EscapeJson(const std::string& str) { + Message m; + + for (size_t i = 0; i < str.size(); ++i) { + const char ch = str[i]; + switch (ch) { + case '\\': + case '"': + case '/': + m << '\\' << ch; + break; + case '\b': + m << "\\b"; + break; + case '\t': + m << "\\t"; + break; + case '\n': + m << "\\n"; + break; + case '\f': + m << "\\f"; + break; + case '\r': + m << "\\r"; + break; + default: + if (ch < ' ') { + m << "\\u00" << String::FormatByte(static_cast(ch)); + } else { + m << ch; + } + break; + } + } + + return m.GetString(); +} + +// The following routines generate an JSON representation of a UnitTest +// object. + +// Formats the given time in milliseconds as seconds. +static std::string FormatTimeInMillisAsDuration(TimeInMillis ms) { + ::std::stringstream ss; + ss << (static_cast(ms) * 1e-3) << "s"; + return ss.str(); +} + +// Converts the given epoch time in milliseconds to a date string in the +// RFC3339 format, without the timezone information. +static std::string FormatEpochTimeInMillisAsRFC3339(TimeInMillis ms) { + struct tm time_struct; + if (!PortableLocaltime(static_cast(ms / 1000), &time_struct)) + return ""; + // YYYY-MM-DDThh:mm:ss + return StreamableToString(time_struct.tm_year + 1900) + "-" + + String::FormatIntWidth2(time_struct.tm_mon + 1) + "-" + + String::FormatIntWidth2(time_struct.tm_mday) + "T" + + String::FormatIntWidth2(time_struct.tm_hour) + ":" + + String::FormatIntWidth2(time_struct.tm_min) + ":" + + String::FormatIntWidth2(time_struct.tm_sec) + "Z"; +} + +static inline std::string Indent(int width) { + return std::string(width, ' '); +} + +void JsonUnitTestResultPrinter::OutputJsonKey( + std::ostream* stream, + const std::string& element_name, + const std::string& name, + const std::string& value, + const std::string& indent, + bool comma) { + const std::vector& allowed_names = + GetReservedAttributesForElement(element_name); + + GTEST_CHECK_(std::find(allowed_names.begin(), allowed_names.end(), name) != + allowed_names.end()) + << "Key \"" << name << "\" is not allowed for value \"" << element_name + << "\"."; + + *stream << indent << "\"" << name << "\": \"" << EscapeJson(value) << "\""; + if (comma) + *stream << ",\n"; +} + +void JsonUnitTestResultPrinter::OutputJsonKey( + std::ostream* stream, + const std::string& element_name, + const std::string& name, + int value, + const std::string& indent, + bool comma) { + const std::vector& allowed_names = + GetReservedAttributesForElement(element_name); + + GTEST_CHECK_(std::find(allowed_names.begin(), allowed_names.end(), name) != + allowed_names.end()) + << "Key \"" << name << "\" is not allowed for value \"" << element_name + << "\"."; + + *stream << indent << "\"" << name << "\": " << StreamableToString(value); + if (comma) + *stream << ",\n"; +} + +// Prints a JSON representation of a TestInfo object. +void JsonUnitTestResultPrinter::OutputJsonTestInfo(::std::ostream* stream, + const char* test_case_name, + const TestInfo& test_info) { + const TestResult& result = *test_info.result(); + const std::string kTestcase = "testcase"; + const std::string kIndent = Indent(10); + + *stream << Indent(8) << "{\n"; + OutputJsonKey(stream, kTestcase, "name", test_info.name(), kIndent); + + if (test_info.value_param() != NULL) { + OutputJsonKey(stream, kTestcase, "value_param", + test_info.value_param(), kIndent); + } + if (test_info.type_param() != NULL) { + OutputJsonKey(stream, kTestcase, "type_param", test_info.type_param(), + kIndent); + } + if (GTEST_FLAG(list_tests)) { + OutputJsonKey(stream, kTestcase, "file", test_info.file(), kIndent); + OutputJsonKey(stream, kTestcase, "line", test_info.line(), kIndent, false); + *stream << "\n" << Indent(8) << "}"; + return; + } + + OutputJsonKey(stream, kTestcase, "status", + test_info.should_run() ? "RUN" : "NOTRUN", kIndent); + OutputJsonKey(stream, kTestcase, "time", + FormatTimeInMillisAsDuration(result.elapsed_time()), kIndent); + OutputJsonKey(stream, kTestcase, "classname", test_case_name, kIndent, false); + *stream << TestPropertiesAsJson(result, kIndent); + + int failures = 0; + for (int i = 0; i < result.total_part_count(); ++i) { + const TestPartResult& part = result.GetTestPartResult(i); + if (part.failed()) { + *stream << ",\n"; + if (++failures == 1) { + *stream << kIndent << "\"" << "failures" << "\": [\n"; + } + const std::string location = + internal::FormatCompilerIndependentFileLocation(part.file_name(), + part.line_number()); + const std::string message = EscapeJson(location + "\n" + part.message()); + *stream << kIndent << " {\n" + << kIndent << " \"failure\": \"" << message << "\",\n" + << kIndent << " \"type\": \"\"\n" + << kIndent << " }"; + } + } + + if (failures > 0) + *stream << "\n" << kIndent << "]"; + *stream << "\n" << Indent(8) << "}"; +} + +// Prints an JSON representation of a TestCase object +void JsonUnitTestResultPrinter::PrintJsonTestCase(std::ostream* stream, + const TestCase& test_case) { + const std::string kTestsuite = "testsuite"; + const std::string kIndent = Indent(6); + + *stream << Indent(4) << "{\n"; + OutputJsonKey(stream, kTestsuite, "name", test_case.name(), kIndent); + OutputJsonKey(stream, kTestsuite, "tests", test_case.reportable_test_count(), + kIndent); + if (!GTEST_FLAG(list_tests)) { + OutputJsonKey(stream, kTestsuite, "failures", test_case.failed_test_count(), + kIndent); + OutputJsonKey(stream, kTestsuite, "disabled", + test_case.reportable_disabled_test_count(), kIndent); + OutputJsonKey(stream, kTestsuite, "errors", 0, kIndent); + OutputJsonKey(stream, kTestsuite, "time", + FormatTimeInMillisAsDuration(test_case.elapsed_time()), + kIndent, false); + *stream << TestPropertiesAsJson(test_case.ad_hoc_test_result(), kIndent) + << ",\n"; + } + + *stream << kIndent << "\"" << kTestsuite << "\": [\n"; + + bool comma = false; + for (int i = 0; i < test_case.total_test_count(); ++i) { + if (test_case.GetTestInfo(i)->is_reportable()) { + if (comma) { + *stream << ",\n"; + } else { + comma = true; + } + OutputJsonTestInfo(stream, test_case.name(), *test_case.GetTestInfo(i)); + } + } + *stream << "\n" << kIndent << "]\n" << Indent(4) << "}"; +} + +// Prints a JSON summary of unit_test to output stream out. +void JsonUnitTestResultPrinter::PrintJsonUnitTest(std::ostream* stream, + const UnitTest& unit_test) { + const std::string kTestsuites = "testsuites"; + const std::string kIndent = Indent(2); + *stream << "{\n"; + + OutputJsonKey(stream, kTestsuites, "tests", unit_test.reportable_test_count(), + kIndent); + OutputJsonKey(stream, kTestsuites, "failures", unit_test.failed_test_count(), + kIndent); + OutputJsonKey(stream, kTestsuites, "disabled", + unit_test.reportable_disabled_test_count(), kIndent); + OutputJsonKey(stream, kTestsuites, "errors", 0, kIndent); + if (GTEST_FLAG(shuffle)) { + OutputJsonKey(stream, kTestsuites, "random_seed", unit_test.random_seed(), + kIndent); + } + OutputJsonKey(stream, kTestsuites, "timestamp", + FormatEpochTimeInMillisAsRFC3339(unit_test.start_timestamp()), + kIndent); + OutputJsonKey(stream, kTestsuites, "time", + FormatTimeInMillisAsDuration(unit_test.elapsed_time()), kIndent, + false); + + *stream << TestPropertiesAsJson(unit_test.ad_hoc_test_result(), kIndent) + << ",\n"; + + OutputJsonKey(stream, kTestsuites, "name", "AllTests", kIndent); + *stream << kIndent << "\"" << kTestsuites << "\": [\n"; + + bool comma = false; + for (int i = 0; i < unit_test.total_test_case_count(); ++i) { + if (unit_test.GetTestCase(i)->reportable_test_count() > 0) { + if (comma) { + *stream << ",\n"; + } else { + comma = true; + } + PrintJsonTestCase(stream, *unit_test.GetTestCase(i)); + } + } + + *stream << "\n" << kIndent << "]\n" << "}\n"; +} + +void JsonUnitTestResultPrinter::PrintJsonTestList( + std::ostream* stream, const std::vector& test_cases) { + const std::string kTestsuites = "testsuites"; + const std::string kIndent = Indent(2); + *stream << "{\n"; + int total_tests = 0; + for (size_t i = 0; i < test_cases.size(); ++i) { + total_tests += test_cases[i]->total_test_count(); + } + OutputJsonKey(stream, kTestsuites, "tests", total_tests, kIndent); + + OutputJsonKey(stream, kTestsuites, "name", "AllTests", kIndent); + *stream << kIndent << "\"" << kTestsuites << "\": [\n"; + + for (size_t i = 0; i < test_cases.size(); ++i) { + if (i != 0) { + *stream << ",\n"; + } + PrintJsonTestCase(stream, *test_cases[i]); + } + + *stream << "\n" + << kIndent << "]\n" + << "}\n"; +} +// Produces a string representing the test properties in a result as +// a JSON dictionary. +std::string JsonUnitTestResultPrinter::TestPropertiesAsJson( + const TestResult& result, const std::string& indent) { + Message attributes; + for (int i = 0; i < result.test_property_count(); ++i) { + const TestProperty& property = result.GetTestProperty(i); + attributes << ",\n" << indent << "\"" << property.key() << "\": " + << "\"" << EscapeJson(property.value()) << "\""; + } + return attributes.GetString(); +} + +// End JsonUnitTestResultPrinter + +#if GTEST_CAN_STREAM_RESULTS_ + +// Checks if str contains '=', '&', '%' or '\n' characters. If yes, +// replaces them by "%xx" where xx is their hexadecimal value. For +// example, replaces "=" with "%3D". This algorithm is O(strlen(str)) +// in both time and space -- important as the input str may contain an +// arbitrarily long test failure message and stack trace. +std::string StreamingListener::UrlEncode(const char* str) { + std::string result; + result.reserve(strlen(str) + 1); + for (char ch = *str; ch != '\0'; ch = *++str) { + switch (ch) { + case '%': + case '=': + case '&': + case '\n': + result.append("%" + String::FormatByte(static_cast(ch))); + break; + default: + result.push_back(ch); + break; + } + } + return result; +} + +void StreamingListener::SocketWriter::MakeConnection() { + GTEST_CHECK_(sockfd_ == -1) + << "MakeConnection() can't be called when there is already a connection."; + + addrinfo hints; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; // To allow both IPv4 and IPv6 addresses. + hints.ai_socktype = SOCK_STREAM; + addrinfo* servinfo = NULL; + + // Use the getaddrinfo() to get a linked list of IP addresses for + // the given host name. + const int error_num = getaddrinfo( + host_name_.c_str(), port_num_.c_str(), &hints, &servinfo); + if (error_num != 0) { + GTEST_LOG_(WARNING) << "stream_result_to: getaddrinfo() failed: " + << gai_strerror(error_num); + } + + // Loop through all the results and connect to the first we can. + for (addrinfo* cur_addr = servinfo; sockfd_ == -1 && cur_addr != NULL; + cur_addr = cur_addr->ai_next) { + sockfd_ = socket( + cur_addr->ai_family, cur_addr->ai_socktype, cur_addr->ai_protocol); + if (sockfd_ != -1) { + // Connect the client socket to the server socket. + if (connect(sockfd_, cur_addr->ai_addr, cur_addr->ai_addrlen) == -1) { + close(sockfd_); + sockfd_ = -1; + } + } + } + + freeaddrinfo(servinfo); // all done with this structure + + if (sockfd_ == -1) { + GTEST_LOG_(WARNING) << "stream_result_to: failed to connect to " + << host_name_ << ":" << port_num_; + } +} + +// End of class Streaming Listener +#endif // GTEST_CAN_STREAM_RESULTS__ + +// class OsStackTraceGetter + +const char* const OsStackTraceGetterInterface::kElidedFramesMarker = + "... " GTEST_NAME_ " internal frames ..."; + +std::string OsStackTraceGetter::CurrentStackTrace(int max_depth, int skip_count) + GTEST_LOCK_EXCLUDED_(mutex_) { +#if GTEST_HAS_ABSL + std::string result; + + if (max_depth <= 0) { + return result; + } + + max_depth = std::min(max_depth, kMaxStackTraceDepth); + + std::vector raw_stack(max_depth); + // Skips the frames requested by the caller, plus this function. + const int raw_stack_size = + absl::GetStackTrace(&raw_stack[0], max_depth, skip_count + 1); + + void* caller_frame = nullptr; + { + MutexLock lock(&mutex_); + caller_frame = caller_frame_; + } + + for (int i = 0; i < raw_stack_size; ++i) { + if (raw_stack[i] == caller_frame && + !GTEST_FLAG(show_internal_stack_frames)) { + // Add a marker to the trace and stop adding frames. + absl::StrAppend(&result, kElidedFramesMarker, "\n"); + break; + } + + char tmp[1024]; + const char* symbol = "(unknown)"; + if (absl::Symbolize(raw_stack[i], tmp, sizeof(tmp))) { + symbol = tmp; + } + + char line[1024]; + snprintf(line, sizeof(line), " %p: %s\n", raw_stack[i], symbol); + result += line; + } + + return result; + +#else // !GTEST_HAS_ABSL + static_cast(max_depth); + static_cast(skip_count); + return ""; +#endif // GTEST_HAS_ABSL +} + +void OsStackTraceGetter::UponLeavingGTest() GTEST_LOCK_EXCLUDED_(mutex_) { +#if GTEST_HAS_ABSL + void* caller_frame = nullptr; + if (absl::GetStackTrace(&caller_frame, 1, 3) <= 0) { + caller_frame = nullptr; + } + + MutexLock lock(&mutex_); + caller_frame_ = caller_frame; +#endif // GTEST_HAS_ABSL +} + +// A helper class that creates the premature-exit file in its +// constructor and deletes the file in its destructor. +class ScopedPrematureExitFile { + public: + explicit ScopedPrematureExitFile(const char* premature_exit_filepath) + : premature_exit_filepath_(premature_exit_filepath ? + premature_exit_filepath : "") { + // If a path to the premature-exit file is specified... + if (!premature_exit_filepath_.empty()) { + // create the file with a single "0" character in it. I/O + // errors are ignored as there's nothing better we can do and we + // don't want to fail the test because of this. + FILE* pfile = posix::FOpen(premature_exit_filepath, "w"); + fwrite("0", 1, 1, pfile); + fclose(pfile); + } + } + + ~ScopedPrematureExitFile() { + if (!premature_exit_filepath_.empty()) { + int retval = remove(premature_exit_filepath_.c_str()); + if (retval) { + GTEST_LOG_(ERROR) << "Failed to remove premature exit filepath \"" + << premature_exit_filepath_ << "\" with error " + << retval; + } + } + } + + private: + const std::string premature_exit_filepath_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ScopedPrematureExitFile); +}; + +} // namespace internal + +// class TestEventListeners + +TestEventListeners::TestEventListeners() + : repeater_(new internal::TestEventRepeater()), + default_result_printer_(NULL), + default_xml_generator_(NULL) { +} + +TestEventListeners::~TestEventListeners() { delete repeater_; } + +// Returns the standard listener responsible for the default console +// output. Can be removed from the listeners list to shut down default +// console output. Note that removing this object from the listener list +// with Release transfers its ownership to the user. +void TestEventListeners::Append(TestEventListener* listener) { + repeater_->Append(listener); +} + +// Removes the given event listener from the list and returns it. It then +// becomes the caller's responsibility to delete the listener. Returns +// NULL if the listener is not found in the list. +TestEventListener* TestEventListeners::Release(TestEventListener* listener) { + if (listener == default_result_printer_) + default_result_printer_ = NULL; + else if (listener == default_xml_generator_) + default_xml_generator_ = NULL; + return repeater_->Release(listener); +} + +// Returns repeater that broadcasts the TestEventListener events to all +// subscribers. +TestEventListener* TestEventListeners::repeater() { return repeater_; } + +// Sets the default_result_printer attribute to the provided listener. +// The listener is also added to the listener list and previous +// default_result_printer is removed from it and deleted. The listener can +// also be NULL in which case it will not be added to the list. Does +// nothing if the previous and the current listener objects are the same. +void TestEventListeners::SetDefaultResultPrinter(TestEventListener* listener) { + if (default_result_printer_ != listener) { + // It is an error to pass this method a listener that is already in the + // list. + delete Release(default_result_printer_); + default_result_printer_ = listener; + if (listener != NULL) + Append(listener); + } +} + +// Sets the default_xml_generator attribute to the provided listener. The +// listener is also added to the listener list and previous +// default_xml_generator is removed from it and deleted. The listener can +// also be NULL in which case it will not be added to the list. Does +// nothing if the previous and the current listener objects are the same. +void TestEventListeners::SetDefaultXmlGenerator(TestEventListener* listener) { + if (default_xml_generator_ != listener) { + // It is an error to pass this method a listener that is already in the + // list. + delete Release(default_xml_generator_); + default_xml_generator_ = listener; + if (listener != NULL) + Append(listener); + } +} + +// Controls whether events will be forwarded by the repeater to the +// listeners in the list. +bool TestEventListeners::EventForwardingEnabled() const { + return repeater_->forwarding_enabled(); +} + +void TestEventListeners::SuppressEventForwarding() { + repeater_->set_forwarding_enabled(false); +} + +// class UnitTest + +// Gets the singleton UnitTest object. The first time this method is +// called, a UnitTest object is constructed and returned. Consecutive +// calls will return the same object. +// +// We don't protect this under mutex_ as a user is not supposed to +// call this before main() starts, from which point on the return +// value will never change. +UnitTest* UnitTest::GetInstance() { + // When compiled with MSVC 7.1 in optimized mode, destroying the + // UnitTest object upon exiting the program messes up the exit code, + // causing successful tests to appear failed. We have to use a + // different implementation in this case to bypass the compiler bug. + // This implementation makes the compiler happy, at the cost of + // leaking the UnitTest object. + + // CodeGear C++Builder insists on a public destructor for the + // default implementation. Use this implementation to keep good OO + // design with private destructor. + +#if (_MSC_VER == 1310 && !defined(_DEBUG)) || defined(__BORLANDC__) + static UnitTest* const instance = new UnitTest; + return instance; +#else + static UnitTest instance; + return &instance; +#endif // (_MSC_VER == 1310 && !defined(_DEBUG)) || defined(__BORLANDC__) +} + +// Gets the number of successful test cases. +int UnitTest::successful_test_case_count() const { + return impl()->successful_test_case_count(); +} + +// Gets the number of failed test cases. +int UnitTest::failed_test_case_count() const { + return impl()->failed_test_case_count(); +} + +// Gets the number of all test cases. +int UnitTest::total_test_case_count() const { + return impl()->total_test_case_count(); +} + +// Gets the number of all test cases that contain at least one test +// that should run. +int UnitTest::test_case_to_run_count() const { + return impl()->test_case_to_run_count(); +} + +// Gets the number of successful tests. +int UnitTest::successful_test_count() const { + return impl()->successful_test_count(); +} + +// Gets the number of failed tests. +int UnitTest::failed_test_count() const { return impl()->failed_test_count(); } + +// Gets the number of disabled tests that will be reported in the XML report. +int UnitTest::reportable_disabled_test_count() const { + return impl()->reportable_disabled_test_count(); +} + +// Gets the number of disabled tests. +int UnitTest::disabled_test_count() const { + return impl()->disabled_test_count(); +} + +// Gets the number of tests to be printed in the XML report. +int UnitTest::reportable_test_count() const { + return impl()->reportable_test_count(); +} + +// Gets the number of all tests. +int UnitTest::total_test_count() const { return impl()->total_test_count(); } + +// Gets the number of tests that should run. +int UnitTest::test_to_run_count() const { return impl()->test_to_run_count(); } + +// Gets the time of the test program start, in ms from the start of the +// UNIX epoch. +internal::TimeInMillis UnitTest::start_timestamp() const { + return impl()->start_timestamp(); +} + +// Gets the elapsed time, in milliseconds. +internal::TimeInMillis UnitTest::elapsed_time() const { + return impl()->elapsed_time(); +} + +// Returns true iff the unit test passed (i.e. all test cases passed). +bool UnitTest::Passed() const { return impl()->Passed(); } + +// Returns true iff the unit test failed (i.e. some test case failed +// or something outside of all tests failed). +bool UnitTest::Failed() const { return impl()->Failed(); } + +// Gets the i-th test case among all the test cases. i can range from 0 to +// total_test_case_count() - 1. If i is not in that range, returns NULL. +const TestCase* UnitTest::GetTestCase(int i) const { + return impl()->GetTestCase(i); +} + +// Returns the TestResult containing information on test failures and +// properties logged outside of individual test cases. +const TestResult& UnitTest::ad_hoc_test_result() const { + return *impl()->ad_hoc_test_result(); +} + +// Gets the i-th test case among all the test cases. i can range from 0 to +// total_test_case_count() - 1. If i is not in that range, returns NULL. +TestCase* UnitTest::GetMutableTestCase(int i) { + return impl()->GetMutableTestCase(i); +} + +// Returns the list of event listeners that can be used to track events +// inside Google Test. +TestEventListeners& UnitTest::listeners() { + return *impl()->listeners(); +} + +// Registers and returns a global test environment. When a test +// program is run, all global test environments will be set-up in the +// order they were registered. After all tests in the program have +// finished, all global test environments will be torn-down in the +// *reverse* order they were registered. +// +// The UnitTest object takes ownership of the given environment. +// +// We don't protect this under mutex_, as we only support calling it +// from the main thread. +Environment* UnitTest::AddEnvironment(Environment* env) { + if (env == NULL) { + return NULL; + } + + impl_->environments().push_back(env); + return env; +} + +// Adds a TestPartResult to the current TestResult object. All Google Test +// assertion macros (e.g. ASSERT_TRUE, EXPECT_EQ, etc) eventually call +// this to report their results. The user code should use the +// assertion macros instead of calling this directly. +void UnitTest::AddTestPartResult( + TestPartResult::Type result_type, + const char* file_name, + int line_number, + const std::string& message, + const std::string& os_stack_trace) GTEST_LOCK_EXCLUDED_(mutex_) { + Message msg; + msg << message; + + internal::MutexLock lock(&mutex_); + if (impl_->gtest_trace_stack().size() > 0) { + msg << "\n" << GTEST_NAME_ << " trace:"; + + for (int i = static_cast(impl_->gtest_trace_stack().size()); + i > 0; --i) { + const internal::TraceInfo& trace = impl_->gtest_trace_stack()[i - 1]; + msg << "\n" << internal::FormatFileLocation(trace.file, trace.line) + << " " << trace.message; + } + } + + if (os_stack_trace.c_str() != NULL && !os_stack_trace.empty()) { + msg << internal::kStackTraceMarker << os_stack_trace; + } + + const TestPartResult result = + TestPartResult(result_type, file_name, line_number, + msg.GetString().c_str()); + impl_->GetTestPartResultReporterForCurrentThread()-> + ReportTestPartResult(result); + + if (result_type != TestPartResult::kSuccess) { + // gtest_break_on_failure takes precedence over + // gtest_throw_on_failure. This allows a user to set the latter + // in the code (perhaps in order to use Google Test assertions + // with another testing framework) and specify the former on the + // command line for debugging. + if (GTEST_FLAG(break_on_failure)) { +#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT + // Using DebugBreak on Windows allows gtest to still break into a debugger + // when a failure happens and both the --gtest_break_on_failure and + // the --gtest_catch_exceptions flags are specified. + DebugBreak(); +#elif (!defined(__native_client__)) && \ + ((defined(__clang__) || defined(__GNUC__)) && \ + (defined(__x86_64__) || defined(__i386__))) + // with clang/gcc we can achieve the same effect on x86 by invoking int3 + asm("int3"); +#else + // Dereference NULL through a volatile pointer to prevent the compiler + // from removing. We use this rather than abort() or __builtin_trap() for + // portability: Symbian doesn't implement abort() well, and some debuggers + // don't correctly trap abort(). + *static_cast(NULL) = 1; +#endif // GTEST_OS_WINDOWS + } else if (GTEST_FLAG(throw_on_failure)) { +#if GTEST_HAS_EXCEPTIONS + throw internal::GoogleTestFailureException(result); +#else + // We cannot call abort() as it generates a pop-up in debug mode + // that cannot be suppressed in VC 7.1 or below. + exit(1); +#endif + } + } +} + +// Adds a TestProperty to the current TestResult object when invoked from +// inside a test, to current TestCase's ad_hoc_test_result_ when invoked +// from SetUpTestCase or TearDownTestCase, or to the global property set +// when invoked elsewhere. If the result already contains a property with +// the same key, the value will be updated. +void UnitTest::RecordProperty(const std::string& key, + const std::string& value) { + impl_->RecordProperty(TestProperty(key, value)); +} + +// Runs all tests in this UnitTest object and prints the result. +// Returns 0 if successful, or 1 otherwise. +// +// We don't protect this under mutex_, as we only support calling it +// from the main thread. +int UnitTest::Run() { + const bool in_death_test_child_process = + internal::GTEST_FLAG(internal_run_death_test).length() > 0; + + // Google Test implements this protocol for catching that a test + // program exits before returning control to Google Test: + // + // 1. Upon start, Google Test creates a file whose absolute path + // is specified by the environment variable + // TEST_PREMATURE_EXIT_FILE. + // 2. When Google Test has finished its work, it deletes the file. + // + // This allows a test runner to set TEST_PREMATURE_EXIT_FILE before + // running a Google-Test-based test program and check the existence + // of the file at the end of the test execution to see if it has + // exited prematurely. + + // If we are in the child process of a death test, don't + // create/delete the premature exit file, as doing so is unnecessary + // and will confuse the parent process. Otherwise, create/delete + // the file upon entering/leaving this function. If the program + // somehow exits before this function has a chance to return, the + // premature-exit file will be left undeleted, causing a test runner + // that understands the premature-exit-file protocol to report the + // test as having failed. + const internal::ScopedPrematureExitFile premature_exit_file( + in_death_test_child_process ? + NULL : internal::posix::GetEnv("TEST_PREMATURE_EXIT_FILE")); + + // Captures the value of GTEST_FLAG(catch_exceptions). This value will be + // used for the duration of the program. + impl()->set_catch_exceptions(GTEST_FLAG(catch_exceptions)); + +#if GTEST_OS_WINDOWS + // Either the user wants Google Test to catch exceptions thrown by the + // tests or this is executing in the context of death test child + // process. In either case the user does not want to see pop-up dialogs + // about crashes - they are expected. + if (impl()->catch_exceptions() || in_death_test_child_process) { +# if !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT + // SetErrorMode doesn't exist on CE. + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOALIGNMENTFAULTEXCEPT | + SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX); +# endif // !GTEST_OS_WINDOWS_MOBILE + +# if (defined(_MSC_VER) || GTEST_OS_WINDOWS_MINGW) && !GTEST_OS_WINDOWS_MOBILE + // Death test children can be terminated with _abort(). On Windows, + // _abort() can show a dialog with a warning message. This forces the + // abort message to go to stderr instead. + _set_error_mode(_OUT_TO_STDERR); +# endif + +# if _MSC_VER >= 1400 && !GTEST_OS_WINDOWS_MOBILE + // In the debug version, Visual Studio pops up a separate dialog + // offering a choice to debug the aborted program. We need to suppress + // this dialog or it will pop up for every EXPECT/ASSERT_DEATH statement + // executed. Google Test will notify the user of any unexpected + // failure via stderr. + // + // VC++ doesn't define _set_abort_behavior() prior to the version 8.0. + // Users of prior VC versions shall suffer the agony and pain of + // clicking through the countless debug dialogs. + // FIXME: find a way to suppress the abort dialog() in the + // debug mode when compiled with VC 7.1 or lower. + if (!GTEST_FLAG(break_on_failure)) + _set_abort_behavior( + 0x0, // Clear the following flags: + _WRITE_ABORT_MSG | _CALL_REPORTFAULT); // pop-up window, core dump. +# endif + } +#endif // GTEST_OS_WINDOWS + + return internal::HandleExceptionsInMethodIfSupported( + impl(), + &internal::UnitTestImpl::RunAllTests, + "auxiliary test code (environments or event listeners)") ? 0 : 1; +} + +// Returns the working directory when the first TEST() or TEST_F() was +// executed. +const char* UnitTest::original_working_dir() const { + return impl_->original_working_dir_.c_str(); +} + +// Returns the TestCase object for the test that's currently running, +// or NULL if no test is running. +const TestCase* UnitTest::current_test_case() const + GTEST_LOCK_EXCLUDED_(mutex_) { + internal::MutexLock lock(&mutex_); + return impl_->current_test_case(); +} + +// Returns the TestInfo object for the test that's currently running, +// or NULL if no test is running. +const TestInfo* UnitTest::current_test_info() const + GTEST_LOCK_EXCLUDED_(mutex_) { + internal::MutexLock lock(&mutex_); + return impl_->current_test_info(); +} + +// Returns the random seed used at the start of the current test run. +int UnitTest::random_seed() const { return impl_->random_seed(); } + +// Returns ParameterizedTestCaseRegistry object used to keep track of +// value-parameterized tests and instantiate and register them. +internal::ParameterizedTestCaseRegistry& + UnitTest::parameterized_test_registry() + GTEST_LOCK_EXCLUDED_(mutex_) { + return impl_->parameterized_test_registry(); +} + +// Creates an empty UnitTest. +UnitTest::UnitTest() { + impl_ = new internal::UnitTestImpl(this); +} + +// Destructor of UnitTest. +UnitTest::~UnitTest() { + delete impl_; +} + +// Pushes a trace defined by SCOPED_TRACE() on to the per-thread +// Google Test trace stack. +void UnitTest::PushGTestTrace(const internal::TraceInfo& trace) + GTEST_LOCK_EXCLUDED_(mutex_) { + internal::MutexLock lock(&mutex_); + impl_->gtest_trace_stack().push_back(trace); +} + +// Pops a trace from the per-thread Google Test trace stack. +void UnitTest::PopGTestTrace() + GTEST_LOCK_EXCLUDED_(mutex_) { + internal::MutexLock lock(&mutex_); + impl_->gtest_trace_stack().pop_back(); +} + +namespace internal { + +UnitTestImpl::UnitTestImpl(UnitTest* parent) + : parent_(parent), + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4355 /* using this in initializer */) + default_global_test_part_result_reporter_(this), + default_per_thread_test_part_result_reporter_(this), + GTEST_DISABLE_MSC_WARNINGS_POP_() + global_test_part_result_repoter_( + &default_global_test_part_result_reporter_), + per_thread_test_part_result_reporter_( + &default_per_thread_test_part_result_reporter_), + parameterized_test_registry_(), + parameterized_tests_registered_(false), + last_death_test_case_(-1), + current_test_case_(NULL), + current_test_info_(NULL), + ad_hoc_test_result_(), + os_stack_trace_getter_(NULL), + post_flag_parse_init_performed_(false), + random_seed_(0), // Will be overridden by the flag before first use. + random_(0), // Will be reseeded before first use. + start_timestamp_(0), + elapsed_time_(0), +#if GTEST_HAS_DEATH_TEST + death_test_factory_(new DefaultDeathTestFactory), +#endif + // Will be overridden by the flag before first use. + catch_exceptions_(false) { + listeners()->SetDefaultResultPrinter(new PrettyUnitTestResultPrinter); +} + +UnitTestImpl::~UnitTestImpl() { + // Deletes every TestCase. + ForEach(test_cases_, internal::Delete); + + // Deletes every Environment. + ForEach(environments_, internal::Delete); + + delete os_stack_trace_getter_; +} + +// Adds a TestProperty to the current TestResult object when invoked in a +// context of a test, to current test case's ad_hoc_test_result when invoke +// from SetUpTestCase/TearDownTestCase, or to the global property set +// otherwise. If the result already contains a property with the same key, +// the value will be updated. +void UnitTestImpl::RecordProperty(const TestProperty& test_property) { + std::string xml_element; + TestResult* test_result; // TestResult appropriate for property recording. + + if (current_test_info_ != NULL) { + xml_element = "testcase"; + test_result = &(current_test_info_->result_); + } else if (current_test_case_ != NULL) { + xml_element = "testsuite"; + test_result = &(current_test_case_->ad_hoc_test_result_); + } else { + xml_element = "testsuites"; + test_result = &ad_hoc_test_result_; + } + test_result->RecordProperty(xml_element, test_property); +} + +#if GTEST_HAS_DEATH_TEST +// Disables event forwarding if the control is currently in a death test +// subprocess. Must not be called before InitGoogleTest. +void UnitTestImpl::SuppressTestEventsIfInSubprocess() { + if (internal_run_death_test_flag_.get() != NULL) + listeners()->SuppressEventForwarding(); +} +#endif // GTEST_HAS_DEATH_TEST + +// Initializes event listeners performing XML output as specified by +// UnitTestOptions. Must not be called before InitGoogleTest. +void UnitTestImpl::ConfigureXmlOutput() { + const std::string& output_format = UnitTestOptions::GetOutputFormat(); + if (output_format == "xml") { + listeners()->SetDefaultXmlGenerator(new XmlUnitTestResultPrinter( + UnitTestOptions::GetAbsolutePathToOutputFile().c_str())); + } else if (output_format == "json") { + listeners()->SetDefaultXmlGenerator(new JsonUnitTestResultPrinter( + UnitTestOptions::GetAbsolutePathToOutputFile().c_str())); + } else if (output_format != "") { + GTEST_LOG_(WARNING) << "WARNING: unrecognized output format \"" + << output_format << "\" ignored."; + } +} + +#if GTEST_CAN_STREAM_RESULTS_ +// Initializes event listeners for streaming test results in string form. +// Must not be called before InitGoogleTest. +void UnitTestImpl::ConfigureStreamingOutput() { + const std::string& target = GTEST_FLAG(stream_result_to); + if (!target.empty()) { + const size_t pos = target.find(':'); + if (pos != std::string::npos) { + listeners()->Append(new StreamingListener(target.substr(0, pos), + target.substr(pos+1))); + } else { + GTEST_LOG_(WARNING) << "unrecognized streaming target \"" << target + << "\" ignored."; + } + } +} +#endif // GTEST_CAN_STREAM_RESULTS_ + +// Performs initialization dependent upon flag values obtained in +// ParseGoogleTestFlagsOnly. Is called from InitGoogleTest after the call to +// ParseGoogleTestFlagsOnly. In case a user neglects to call InitGoogleTest +// this function is also called from RunAllTests. Since this function can be +// called more than once, it has to be idempotent. +void UnitTestImpl::PostFlagParsingInit() { + // Ensures that this function does not execute more than once. + if (!post_flag_parse_init_performed_) { + post_flag_parse_init_performed_ = true; + +#if defined(GTEST_CUSTOM_TEST_EVENT_LISTENER_) + // Register to send notifications about key process state changes. + listeners()->Append(new GTEST_CUSTOM_TEST_EVENT_LISTENER_()); +#endif // defined(GTEST_CUSTOM_TEST_EVENT_LISTENER_) + +#if GTEST_HAS_DEATH_TEST + InitDeathTestSubprocessControlInfo(); + SuppressTestEventsIfInSubprocess(); +#endif // GTEST_HAS_DEATH_TEST + + // Registers parameterized tests. This makes parameterized tests + // available to the UnitTest reflection API without running + // RUN_ALL_TESTS. + RegisterParameterizedTests(); + + // Configures listeners for XML output. This makes it possible for users + // to shut down the default XML output before invoking RUN_ALL_TESTS. + ConfigureXmlOutput(); + +#if GTEST_CAN_STREAM_RESULTS_ + // Configures listeners for streaming test results to the specified server. + ConfigureStreamingOutput(); +#endif // GTEST_CAN_STREAM_RESULTS_ + +#if GTEST_HAS_ABSL + if (GTEST_FLAG(install_failure_signal_handler)) { + absl::FailureSignalHandlerOptions options; + absl::InstallFailureSignalHandler(options); + } +#endif // GTEST_HAS_ABSL + } +} + +// A predicate that checks the name of a TestCase against a known +// value. +// +// This is used for implementation of the UnitTest class only. We put +// it in the anonymous namespace to prevent polluting the outer +// namespace. +// +// TestCaseNameIs is copyable. +class TestCaseNameIs { + public: + // Constructor. + explicit TestCaseNameIs(const std::string& name) + : name_(name) {} + + // Returns true iff the name of test_case matches name_. + bool operator()(const TestCase* test_case) const { + return test_case != NULL && strcmp(test_case->name(), name_.c_str()) == 0; + } + + private: + std::string name_; +}; + +// Finds and returns a TestCase with the given name. If one doesn't +// exist, creates one and returns it. It's the CALLER'S +// RESPONSIBILITY to ensure that this function is only called WHEN THE +// TESTS ARE NOT SHUFFLED. +// +// Arguments: +// +// test_case_name: name of the test case +// type_param: the name of the test case's type parameter, or NULL if +// this is not a typed or a type-parameterized test case. +// set_up_tc: pointer to the function that sets up the test case +// tear_down_tc: pointer to the function that tears down the test case +TestCase* UnitTestImpl::GetTestCase(const char* test_case_name, + const char* type_param, + Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc) { + // Can we find a TestCase with the given name? + const std::vector::const_reverse_iterator test_case = + std::find_if(test_cases_.rbegin(), test_cases_.rend(), + TestCaseNameIs(test_case_name)); + + if (test_case != test_cases_.rend()) + return *test_case; + + // No. Let's create one. + TestCase* const new_test_case = + new TestCase(test_case_name, type_param, set_up_tc, tear_down_tc); + + // Is this a death test case? + if (internal::UnitTestOptions::MatchesFilter(test_case_name, + kDeathTestCaseFilter)) { + // Yes. Inserts the test case after the last death test case + // defined so far. This only works when the test cases haven't + // been shuffled. Otherwise we may end up running a death test + // after a non-death test. + ++last_death_test_case_; + test_cases_.insert(test_cases_.begin() + last_death_test_case_, + new_test_case); + } else { + // No. Appends to the end of the list. + test_cases_.push_back(new_test_case); + } + + test_case_indices_.push_back(static_cast(test_case_indices_.size())); + return new_test_case; +} + +// Helpers for setting up / tearing down the given environment. They +// are for use in the ForEach() function. +static void SetUpEnvironment(Environment* env) { env->SetUp(); } +static void TearDownEnvironment(Environment* env) { env->TearDown(); } + +// Runs all tests in this UnitTest object, prints the result, and +// returns true if all tests are successful. If any exception is +// thrown during a test, the test is considered to be failed, but the +// rest of the tests will still be run. +// +// When parameterized tests are enabled, it expands and registers +// parameterized tests first in RegisterParameterizedTests(). +// All other functions called from RunAllTests() may safely assume that +// parameterized tests are ready to be counted and run. +bool UnitTestImpl::RunAllTests() { + // True iff Google Test is initialized before RUN_ALL_TESTS() is called. + const bool gtest_is_initialized_before_run_all_tests = GTestIsInitialized(); + + // Do not run any test if the --help flag was specified. + if (g_help_flag) + return true; + + // Repeats the call to the post-flag parsing initialization in case the + // user didn't call InitGoogleTest. + PostFlagParsingInit(); + + // Even if sharding is not on, test runners may want to use the + // GTEST_SHARD_STATUS_FILE to query whether the test supports the sharding + // protocol. + internal::WriteToShardStatusFileIfNeeded(); + + // True iff we are in a subprocess for running a thread-safe-style + // death test. + bool in_subprocess_for_death_test = false; + +#if GTEST_HAS_DEATH_TEST + in_subprocess_for_death_test = (internal_run_death_test_flag_.get() != NULL); +# if defined(GTEST_EXTRA_DEATH_TEST_CHILD_SETUP_) + if (in_subprocess_for_death_test) { + GTEST_EXTRA_DEATH_TEST_CHILD_SETUP_(); + } +# endif // defined(GTEST_EXTRA_DEATH_TEST_CHILD_SETUP_) +#endif // GTEST_HAS_DEATH_TEST + + const bool should_shard = ShouldShard(kTestTotalShards, kTestShardIndex, + in_subprocess_for_death_test); + + // Compares the full test names with the filter to decide which + // tests to run. + const bool has_tests_to_run = FilterTests(should_shard + ? HONOR_SHARDING_PROTOCOL + : IGNORE_SHARDING_PROTOCOL) > 0; + + // Lists the tests and exits if the --gtest_list_tests flag was specified. + if (GTEST_FLAG(list_tests)) { + // This must be called *after* FilterTests() has been called. + ListTestsMatchingFilter(); + return true; + } + + random_seed_ = GTEST_FLAG(shuffle) ? + GetRandomSeedFromFlag(GTEST_FLAG(random_seed)) : 0; + + // True iff at least one test has failed. + bool failed = false; + + TestEventListener* repeater = listeners()->repeater(); + + start_timestamp_ = GetTimeInMillis(); + repeater->OnTestProgramStart(*parent_); + + // How many times to repeat the tests? We don't want to repeat them + // when we are inside the subprocess of a death test. + const int repeat = in_subprocess_for_death_test ? 1 : GTEST_FLAG(repeat); + // Repeats forever if the repeat count is negative. + const bool forever = repeat < 0; + for (int i = 0; forever || i != repeat; i++) { + // We want to preserve failures generated by ad-hoc test + // assertions executed before RUN_ALL_TESTS(). + ClearNonAdHocTestResult(); + + const TimeInMillis start = GetTimeInMillis(); + + // Shuffles test cases and tests if requested. + if (has_tests_to_run && GTEST_FLAG(shuffle)) { + random()->Reseed(random_seed_); + // This should be done before calling OnTestIterationStart(), + // such that a test event listener can see the actual test order + // in the event. + ShuffleTests(); + } + + // Tells the unit test event listeners that the tests are about to start. + repeater->OnTestIterationStart(*parent_, i); + + // Runs each test case if there is at least one test to run. + if (has_tests_to_run) { + // Sets up all environments beforehand. + repeater->OnEnvironmentsSetUpStart(*parent_); + ForEach(environments_, SetUpEnvironment); + repeater->OnEnvironmentsSetUpEnd(*parent_); + + // Runs the tests only if there was no fatal failure during global + // set-up. + if (!Test::HasFatalFailure()) { + for (int test_index = 0; test_index < total_test_case_count(); + test_index++) { + GetMutableTestCase(test_index)->Run(); + } + } + + // Tears down all environments in reverse order afterwards. + repeater->OnEnvironmentsTearDownStart(*parent_); + std::for_each(environments_.rbegin(), environments_.rend(), + TearDownEnvironment); + repeater->OnEnvironmentsTearDownEnd(*parent_); + } + + elapsed_time_ = GetTimeInMillis() - start; + + // Tells the unit test event listener that the tests have just finished. + repeater->OnTestIterationEnd(*parent_, i); + + // Gets the result and clears it. + if (!Passed()) { + failed = true; + } + + // Restores the original test order after the iteration. This + // allows the user to quickly repro a failure that happens in the + // N-th iteration without repeating the first (N - 1) iterations. + // This is not enclosed in "if (GTEST_FLAG(shuffle)) { ... }", in + // case the user somehow changes the value of the flag somewhere + // (it's always safe to unshuffle the tests). + UnshuffleTests(); + + if (GTEST_FLAG(shuffle)) { + // Picks a new random seed for each iteration. + random_seed_ = GetNextRandomSeed(random_seed_); + } + } + + repeater->OnTestProgramEnd(*parent_); + + if (!gtest_is_initialized_before_run_all_tests) { + ColoredPrintf( + COLOR_RED, + "\nIMPORTANT NOTICE - DO NOT IGNORE:\n" + "This test program did NOT call " GTEST_INIT_GOOGLE_TEST_NAME_ + "() before calling RUN_ALL_TESTS(). This is INVALID. Soon " GTEST_NAME_ + " will start to enforce the valid usage. " + "Please fix it ASAP, or IT WILL START TO FAIL.\n"); // NOLINT +#if GTEST_FOR_GOOGLE_ + ColoredPrintf(COLOR_RED, + "For more details, see http://wiki/Main/ValidGUnitMain.\n"); +#endif // GTEST_FOR_GOOGLE_ + } + + return !failed; +} + +// Reads the GTEST_SHARD_STATUS_FILE environment variable, and creates the file +// if the variable is present. If a file already exists at this location, this +// function will write over it. If the variable is present, but the file cannot +// be created, prints an error and exits. +void WriteToShardStatusFileIfNeeded() { + const char* const test_shard_file = posix::GetEnv(kTestShardStatusFile); + if (test_shard_file != NULL) { + FILE* const file = posix::FOpen(test_shard_file, "w"); + if (file == NULL) { + ColoredPrintf(COLOR_RED, + "Could not write to the test shard status file \"%s\" " + "specified by the %s environment variable.\n", + test_shard_file, kTestShardStatusFile); + fflush(stdout); + exit(EXIT_FAILURE); + } + fclose(file); + } +} + +// Checks whether sharding is enabled by examining the relevant +// environment variable values. If the variables are present, +// but inconsistent (i.e., shard_index >= total_shards), prints +// an error and exits. If in_subprocess_for_death_test, sharding is +// disabled because it must only be applied to the original test +// process. Otherwise, we could filter out death tests we intended to execute. +bool ShouldShard(const char* total_shards_env, + const char* shard_index_env, + bool in_subprocess_for_death_test) { + if (in_subprocess_for_death_test) { + return false; + } + + const Int32 total_shards = Int32FromEnvOrDie(total_shards_env, -1); + const Int32 shard_index = Int32FromEnvOrDie(shard_index_env, -1); + + if (total_shards == -1 && shard_index == -1) { + return false; + } else if (total_shards == -1 && shard_index != -1) { + const Message msg = Message() + << "Invalid environment variables: you have " + << kTestShardIndex << " = " << shard_index + << ", but have left " << kTestTotalShards << " unset.\n"; + ColoredPrintf(COLOR_RED, msg.GetString().c_str()); + fflush(stdout); + exit(EXIT_FAILURE); + } else if (total_shards != -1 && shard_index == -1) { + const Message msg = Message() + << "Invalid environment variables: you have " + << kTestTotalShards << " = " << total_shards + << ", but have left " << kTestShardIndex << " unset.\n"; + ColoredPrintf(COLOR_RED, msg.GetString().c_str()); + fflush(stdout); + exit(EXIT_FAILURE); + } else if (shard_index < 0 || shard_index >= total_shards) { + const Message msg = Message() + << "Invalid environment variables: we require 0 <= " + << kTestShardIndex << " < " << kTestTotalShards + << ", but you have " << kTestShardIndex << "=" << shard_index + << ", " << kTestTotalShards << "=" << total_shards << ".\n"; + ColoredPrintf(COLOR_RED, msg.GetString().c_str()); + fflush(stdout); + exit(EXIT_FAILURE); + } + + return total_shards > 1; +} + +// Parses the environment variable var as an Int32. If it is unset, +// returns default_val. If it is not an Int32, prints an error +// and aborts. +Int32 Int32FromEnvOrDie(const char* var, Int32 default_val) { + const char* str_val = posix::GetEnv(var); + if (str_val == NULL) { + return default_val; + } + + Int32 result; + if (!ParseInt32(Message() << "The value of environment variable " << var, + str_val, &result)) { + exit(EXIT_FAILURE); + } + return result; +} + +// Given the total number of shards, the shard index, and the test id, +// returns true iff the test should be run on this shard. The test id is +// some arbitrary but unique non-negative integer assigned to each test +// method. Assumes that 0 <= shard_index < total_shards. +bool ShouldRunTestOnShard(int total_shards, int shard_index, int test_id) { + return (test_id % total_shards) == shard_index; +} + +// Compares the name of each test with the user-specified filter to +// decide whether the test should be run, then records the result in +// each TestCase and TestInfo object. +// If shard_tests == true, further filters tests based on sharding +// variables in the environment - see +// https://github.com/google/googletest/blob/master/googletest/docs/advanced.md +// . Returns the number of tests that should run. +int UnitTestImpl::FilterTests(ReactionToSharding shard_tests) { + const Int32 total_shards = shard_tests == HONOR_SHARDING_PROTOCOL ? + Int32FromEnvOrDie(kTestTotalShards, -1) : -1; + const Int32 shard_index = shard_tests == HONOR_SHARDING_PROTOCOL ? + Int32FromEnvOrDie(kTestShardIndex, -1) : -1; + + // num_runnable_tests are the number of tests that will + // run across all shards (i.e., match filter and are not disabled). + // num_selected_tests are the number of tests to be run on + // this shard. + int num_runnable_tests = 0; + int num_selected_tests = 0; + for (size_t i = 0; i < test_cases_.size(); i++) { + TestCase* const test_case = test_cases_[i]; + const std::string &test_case_name = test_case->name(); + test_case->set_should_run(false); + + for (size_t j = 0; j < test_case->test_info_list().size(); j++) { + TestInfo* const test_info = test_case->test_info_list()[j]; + const std::string test_name(test_info->name()); + // A test is disabled if test case name or test name matches + // kDisableTestFilter. + const bool is_disabled = + internal::UnitTestOptions::MatchesFilter(test_case_name, + kDisableTestFilter) || + internal::UnitTestOptions::MatchesFilter(test_name, + kDisableTestFilter); + test_info->is_disabled_ = is_disabled; + + const bool matches_filter = + internal::UnitTestOptions::FilterMatchesTest(test_case_name, + test_name); + test_info->matches_filter_ = matches_filter; + + const bool is_runnable = + (GTEST_FLAG(also_run_disabled_tests) || !is_disabled) && + matches_filter; + + const bool is_in_another_shard = + shard_tests != IGNORE_SHARDING_PROTOCOL && + !ShouldRunTestOnShard(total_shards, shard_index, num_runnable_tests); + test_info->is_in_another_shard_ = is_in_another_shard; + const bool is_selected = is_runnable && !is_in_another_shard; + + num_runnable_tests += is_runnable; + num_selected_tests += is_selected; + + test_info->should_run_ = is_selected; + test_case->set_should_run(test_case->should_run() || is_selected); + } + } + return num_selected_tests; +} + +// Prints the given C-string on a single line by replacing all '\n' +// characters with string "\\n". If the output takes more than +// max_length characters, only prints the first max_length characters +// and "...". +static void PrintOnOneLine(const char* str, int max_length) { + if (str != NULL) { + for (int i = 0; *str != '\0'; ++str) { + if (i >= max_length) { + printf("..."); + break; + } + if (*str == '\n') { + printf("\\n"); + i += 2; + } else { + printf("%c", *str); + ++i; + } + } + } +} + +// Prints the names of the tests matching the user-specified filter flag. +void UnitTestImpl::ListTestsMatchingFilter() { + // Print at most this many characters for each type/value parameter. + const int kMaxParamLength = 250; + + for (size_t i = 0; i < test_cases_.size(); i++) { + const TestCase* const test_case = test_cases_[i]; + bool printed_test_case_name = false; + + for (size_t j = 0; j < test_case->test_info_list().size(); j++) { + const TestInfo* const test_info = + test_case->test_info_list()[j]; + if (test_info->matches_filter_) { + if (!printed_test_case_name) { + printed_test_case_name = true; + printf("%s.", test_case->name()); + if (test_case->type_param() != NULL) { + printf(" # %s = ", kTypeParamLabel); + // We print the type parameter on a single line to make + // the output easy to parse by a program. + PrintOnOneLine(test_case->type_param(), kMaxParamLength); + } + printf("\n"); + } + printf(" %s", test_info->name()); + if (test_info->value_param() != NULL) { + printf(" # %s = ", kValueParamLabel); + // We print the value parameter on a single line to make the + // output easy to parse by a program. + PrintOnOneLine(test_info->value_param(), kMaxParamLength); + } + printf("\n"); + } + } + } + fflush(stdout); + const std::string& output_format = UnitTestOptions::GetOutputFormat(); + if (output_format == "xml" || output_format == "json") { + FILE* fileout = OpenFileForWriting( + UnitTestOptions::GetAbsolutePathToOutputFile().c_str()); + std::stringstream stream; + if (output_format == "xml") { + XmlUnitTestResultPrinter( + UnitTestOptions::GetAbsolutePathToOutputFile().c_str()) + .PrintXmlTestsList(&stream, test_cases_); + } else if (output_format == "json") { + JsonUnitTestResultPrinter( + UnitTestOptions::GetAbsolutePathToOutputFile().c_str()) + .PrintJsonTestList(&stream, test_cases_); + } + fprintf(fileout, "%s", StringStreamToString(&stream).c_str()); + fclose(fileout); + } +} + +// Sets the OS stack trace getter. +// +// Does nothing if the input and the current OS stack trace getter are +// the same; otherwise, deletes the old getter and makes the input the +// current getter. +void UnitTestImpl::set_os_stack_trace_getter( + OsStackTraceGetterInterface* getter) { + if (os_stack_trace_getter_ != getter) { + delete os_stack_trace_getter_; + os_stack_trace_getter_ = getter; + } +} + +// Returns the current OS stack trace getter if it is not NULL; +// otherwise, creates an OsStackTraceGetter, makes it the current +// getter, and returns it. +OsStackTraceGetterInterface* UnitTestImpl::os_stack_trace_getter() { + if (os_stack_trace_getter_ == NULL) { +#ifdef GTEST_OS_STACK_TRACE_GETTER_ + os_stack_trace_getter_ = new GTEST_OS_STACK_TRACE_GETTER_; +#else + os_stack_trace_getter_ = new OsStackTraceGetter; +#endif // GTEST_OS_STACK_TRACE_GETTER_ + } + + return os_stack_trace_getter_; +} + +// Returns the most specific TestResult currently running. +TestResult* UnitTestImpl::current_test_result() { + if (current_test_info_ != NULL) { + return ¤t_test_info_->result_; + } + if (current_test_case_ != NULL) { + return ¤t_test_case_->ad_hoc_test_result_; + } + return &ad_hoc_test_result_; +} + +// Shuffles all test cases, and the tests within each test case, +// making sure that death tests are still run first. +void UnitTestImpl::ShuffleTests() { + // Shuffles the death test cases. + ShuffleRange(random(), 0, last_death_test_case_ + 1, &test_case_indices_); + + // Shuffles the non-death test cases. + ShuffleRange(random(), last_death_test_case_ + 1, + static_cast(test_cases_.size()), &test_case_indices_); + + // Shuffles the tests inside each test case. + for (size_t i = 0; i < test_cases_.size(); i++) { + test_cases_[i]->ShuffleTests(random()); + } +} + +// Restores the test cases and tests to their order before the first shuffle. +void UnitTestImpl::UnshuffleTests() { + for (size_t i = 0; i < test_cases_.size(); i++) { + // Unshuffles the tests in each test case. + test_cases_[i]->UnshuffleTests(); + // Resets the index of each test case. + test_case_indices_[i] = static_cast(i); + } +} + +// Returns the current OS stack trace as an std::string. +// +// The maximum number of stack frames to be included is specified by +// the gtest_stack_trace_depth flag. The skip_count parameter +// specifies the number of top frames to be skipped, which doesn't +// count against the number of frames to be included. +// +// For example, if Foo() calls Bar(), which in turn calls +// GetCurrentOsStackTraceExceptTop(..., 1), Foo() will be included in +// the trace but Bar() and GetCurrentOsStackTraceExceptTop() won't. +std::string GetCurrentOsStackTraceExceptTop(UnitTest* /*unit_test*/, + int skip_count) { + // We pass skip_count + 1 to skip this wrapper function in addition + // to what the user really wants to skip. + return GetUnitTestImpl()->CurrentOsStackTraceExceptTop(skip_count + 1); +} + +// Used by the GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_ macro to +// suppress unreachable code warnings. +namespace { +class ClassUniqueToAlwaysTrue {}; +} + +bool IsTrue(bool condition) { return condition; } + +bool AlwaysTrue() { +#if GTEST_HAS_EXCEPTIONS + // This condition is always false so AlwaysTrue() never actually throws, + // but it makes the compiler think that it may throw. + if (IsTrue(false)) + throw ClassUniqueToAlwaysTrue(); +#endif // GTEST_HAS_EXCEPTIONS + return true; +} + +// If *pstr starts with the given prefix, modifies *pstr to be right +// past the prefix and returns true; otherwise leaves *pstr unchanged +// and returns false. None of pstr, *pstr, and prefix can be NULL. +bool SkipPrefix(const char* prefix, const char** pstr) { + const size_t prefix_len = strlen(prefix); + if (strncmp(*pstr, prefix, prefix_len) == 0) { + *pstr += prefix_len; + return true; + } + return false; +} + +// Parses a string as a command line flag. The string should have +// the format "--flag=value". When def_optional is true, the "=value" +// part can be omitted. +// +// Returns the value of the flag, or NULL if the parsing failed. +static const char* ParseFlagValue(const char* str, const char* flag, + bool def_optional) { + // str and flag must not be NULL. + if (str == NULL || flag == NULL) return NULL; + + // The flag must start with "--" followed by GTEST_FLAG_PREFIX_. + const std::string flag_str = std::string("--") + GTEST_FLAG_PREFIX_ + flag; + const size_t flag_len = flag_str.length(); + if (strncmp(str, flag_str.c_str(), flag_len) != 0) return NULL; + + // Skips the flag name. + const char* flag_end = str + flag_len; + + // When def_optional is true, it's OK to not have a "=value" part. + if (def_optional && (flag_end[0] == '\0')) { + return flag_end; + } + + // If def_optional is true and there are more characters after the + // flag name, or if def_optional is false, there must be a '=' after + // the flag name. + if (flag_end[0] != '=') return NULL; + + // Returns the string after "=". + return flag_end + 1; +} + +// Parses a string for a bool flag, in the form of either +// "--flag=value" or "--flag". +// +// In the former case, the value is taken as true as long as it does +// not start with '0', 'f', or 'F'. +// +// In the latter case, the value is taken as true. +// +// On success, stores the value of the flag in *value, and returns +// true. On failure, returns false without changing *value. +static bool ParseBoolFlag(const char* str, const char* flag, bool* value) { + // Gets the value of the flag as a string. + const char* const value_str = ParseFlagValue(str, flag, true); + + // Aborts if the parsing failed. + if (value_str == NULL) return false; + + // Converts the string value to a bool. + *value = !(*value_str == '0' || *value_str == 'f' || *value_str == 'F'); + return true; +} + +// Parses a string for an Int32 flag, in the form of +// "--flag=value". +// +// On success, stores the value of the flag in *value, and returns +// true. On failure, returns false without changing *value. +bool ParseInt32Flag(const char* str, const char* flag, Int32* value) { + // Gets the value of the flag as a string. + const char* const value_str = ParseFlagValue(str, flag, false); + + // Aborts if the parsing failed. + if (value_str == NULL) return false; + + // Sets *value to the value of the flag. + return ParseInt32(Message() << "The value of flag --" << flag, + value_str, value); +} + +// Parses a string for a string flag, in the form of +// "--flag=value". +// +// On success, stores the value of the flag in *value, and returns +// true. On failure, returns false without changing *value. +template +static bool ParseStringFlag(const char* str, const char* flag, String* value) { + // Gets the value of the flag as a string. + const char* const value_str = ParseFlagValue(str, flag, false); + + // Aborts if the parsing failed. + if (value_str == NULL) return false; + + // Sets *value to the value of the flag. + *value = value_str; + return true; +} + +// Determines whether a string has a prefix that Google Test uses for its +// flags, i.e., starts with GTEST_FLAG_PREFIX_ or GTEST_FLAG_PREFIX_DASH_. +// If Google Test detects that a command line flag has its prefix but is not +// recognized, it will print its help message. Flags starting with +// GTEST_INTERNAL_PREFIX_ followed by "internal_" are considered Google Test +// internal flags and do not trigger the help message. +static bool HasGoogleTestFlagPrefix(const char* str) { + return (SkipPrefix("--", &str) || + SkipPrefix("-", &str) || + SkipPrefix("/", &str)) && + !SkipPrefix(GTEST_FLAG_PREFIX_ "internal_", &str) && + (SkipPrefix(GTEST_FLAG_PREFIX_, &str) || + SkipPrefix(GTEST_FLAG_PREFIX_DASH_, &str)); +} + +// Prints a string containing code-encoded text. The following escape +// sequences can be used in the string to control the text color: +// +// @@ prints a single '@' character. +// @R changes the color to red. +// @G changes the color to green. +// @Y changes the color to yellow. +// @D changes to the default terminal text color. +// +// FIXME: Write tests for this once we add stdout +// capturing to Google Test. +static void PrintColorEncoded(const char* str) { + GTestColor color = COLOR_DEFAULT; // The current color. + + // Conceptually, we split the string into segments divided by escape + // sequences. Then we print one segment at a time. At the end of + // each iteration, the str pointer advances to the beginning of the + // next segment. + for (;;) { + const char* p = strchr(str, '@'); + if (p == NULL) { + ColoredPrintf(color, "%s", str); + return; + } + + ColoredPrintf(color, "%s", std::string(str, p).c_str()); + + const char ch = p[1]; + str = p + 2; + if (ch == '@') { + ColoredPrintf(color, "@"); + } else if (ch == 'D') { + color = COLOR_DEFAULT; + } else if (ch == 'R') { + color = COLOR_RED; + } else if (ch == 'G') { + color = COLOR_GREEN; + } else if (ch == 'Y') { + color = COLOR_YELLOW; + } else { + --str; + } + } +} + +static const char kColorEncodedHelpMessage[] = +"This program contains tests written using " GTEST_NAME_ ". You can use the\n" +"following command line flags to control its behavior:\n" +"\n" +"Test Selection:\n" +" @G--" GTEST_FLAG_PREFIX_ "list_tests@D\n" +" List the names of all tests instead of running them. The name of\n" +" TEST(Foo, Bar) is \"Foo.Bar\".\n" +" @G--" GTEST_FLAG_PREFIX_ "filter=@YPOSTIVE_PATTERNS" + "[@G-@YNEGATIVE_PATTERNS]@D\n" +" Run only the tests whose name matches one of the positive patterns but\n" +" none of the negative patterns. '?' matches any single character; '*'\n" +" matches any substring; ':' separates two patterns.\n" +" @G--" GTEST_FLAG_PREFIX_ "also_run_disabled_tests@D\n" +" Run all disabled tests too.\n" +"\n" +"Test Execution:\n" +" @G--" GTEST_FLAG_PREFIX_ "repeat=@Y[COUNT]@D\n" +" Run the tests repeatedly; use a negative count to repeat forever.\n" +" @G--" GTEST_FLAG_PREFIX_ "shuffle@D\n" +" Randomize tests' orders on every iteration.\n" +" @G--" GTEST_FLAG_PREFIX_ "random_seed=@Y[NUMBER]@D\n" +" Random number seed to use for shuffling test orders (between 1 and\n" +" 99999, or 0 to use a seed based on the current time).\n" +"\n" +"Test Output:\n" +" @G--" GTEST_FLAG_PREFIX_ "color=@Y(@Gyes@Y|@Gno@Y|@Gauto@Y)@D\n" +" Enable/disable colored output. The default is @Gauto@D.\n" +" -@G-" GTEST_FLAG_PREFIX_ "print_time=0@D\n" +" Don't print the elapsed time of each test.\n" +" @G--" GTEST_FLAG_PREFIX_ "output=@Y(@Gjson@Y|@Gxml@Y)[@G:@YDIRECTORY_PATH@G" + GTEST_PATH_SEP_ "@Y|@G:@YFILE_PATH]@D\n" +" Generate a JSON or XML report in the given directory or with the given\n" +" file name. @YFILE_PATH@D defaults to @Gtest_details.xml@D.\n" +# if GTEST_CAN_STREAM_RESULTS_ +" @G--" GTEST_FLAG_PREFIX_ "stream_result_to=@YHOST@G:@YPORT@D\n" +" Stream test results to the given server.\n" +# endif // GTEST_CAN_STREAM_RESULTS_ +"\n" +"Assertion Behavior:\n" +# if GTEST_HAS_DEATH_TEST && !GTEST_OS_WINDOWS +" @G--" GTEST_FLAG_PREFIX_ "death_test_style=@Y(@Gfast@Y|@Gthreadsafe@Y)@D\n" +" Set the default death test style.\n" +# endif // GTEST_HAS_DEATH_TEST && !GTEST_OS_WINDOWS +" @G--" GTEST_FLAG_PREFIX_ "break_on_failure@D\n" +" Turn assertion failures into debugger break-points.\n" +" @G--" GTEST_FLAG_PREFIX_ "throw_on_failure@D\n" +" Turn assertion failures into C++ exceptions for use by an external\n" +" test framework.\n" +" @G--" GTEST_FLAG_PREFIX_ "catch_exceptions=0@D\n" +" Do not report exceptions as test failures. Instead, allow them\n" +" to crash the program or throw a pop-up (on Windows).\n" +"\n" +"Except for @G--" GTEST_FLAG_PREFIX_ "list_tests@D, you can alternatively set " + "the corresponding\n" +"environment variable of a flag (all letters in upper-case). For example, to\n" +"disable colored text output, you can either specify @G--" GTEST_FLAG_PREFIX_ + "color=no@D or set\n" +"the @G" GTEST_FLAG_PREFIX_UPPER_ "COLOR@D environment variable to @Gno@D.\n" +"\n" +"For more information, please read the " GTEST_NAME_ " documentation at\n" +"@G" GTEST_PROJECT_URL_ "@D. If you find a bug in " GTEST_NAME_ "\n" +"(not one in your own code or tests), please report it to\n" +"@G<" GTEST_DEV_EMAIL_ ">@D.\n"; + +static bool ParseGoogleTestFlag(const char* const arg) { + return ParseBoolFlag(arg, kAlsoRunDisabledTestsFlag, + >EST_FLAG(also_run_disabled_tests)) || + ParseBoolFlag(arg, kBreakOnFailureFlag, + >EST_FLAG(break_on_failure)) || + ParseBoolFlag(arg, kCatchExceptionsFlag, + >EST_FLAG(catch_exceptions)) || + ParseStringFlag(arg, kColorFlag, >EST_FLAG(color)) || + ParseStringFlag(arg, kDeathTestStyleFlag, + >EST_FLAG(death_test_style)) || + ParseBoolFlag(arg, kDeathTestUseFork, + >EST_FLAG(death_test_use_fork)) || + ParseStringFlag(arg, kFilterFlag, >EST_FLAG(filter)) || + ParseStringFlag(arg, kInternalRunDeathTestFlag, + >EST_FLAG(internal_run_death_test)) || + ParseBoolFlag(arg, kListTestsFlag, >EST_FLAG(list_tests)) || + ParseStringFlag(arg, kOutputFlag, >EST_FLAG(output)) || + ParseBoolFlag(arg, kPrintTimeFlag, >EST_FLAG(print_time)) || + ParseBoolFlag(arg, kPrintUTF8Flag, >EST_FLAG(print_utf8)) || + ParseInt32Flag(arg, kRandomSeedFlag, >EST_FLAG(random_seed)) || + ParseInt32Flag(arg, kRepeatFlag, >EST_FLAG(repeat)) || + ParseBoolFlag(arg, kShuffleFlag, >EST_FLAG(shuffle)) || + ParseInt32Flag(arg, kStackTraceDepthFlag, + >EST_FLAG(stack_trace_depth)) || + ParseStringFlag(arg, kStreamResultToFlag, + >EST_FLAG(stream_result_to)) || + ParseBoolFlag(arg, kThrowOnFailureFlag, + >EST_FLAG(throw_on_failure)); +} + +#if GTEST_USE_OWN_FLAGFILE_FLAG_ +static void LoadFlagsFromFile(const std::string& path) { + FILE* flagfile = posix::FOpen(path.c_str(), "r"); + if (!flagfile) { + GTEST_LOG_(FATAL) << "Unable to open file \"" << GTEST_FLAG(flagfile) + << "\""; + } + std::string contents(ReadEntireFile(flagfile)); + posix::FClose(flagfile); + std::vector lines; + SplitString(contents, '\n', &lines); + for (size_t i = 0; i < lines.size(); ++i) { + if (lines[i].empty()) + continue; + if (!ParseGoogleTestFlag(lines[i].c_str())) + g_help_flag = true; + } +} +#endif // GTEST_USE_OWN_FLAGFILE_FLAG_ + +// Parses the command line for Google Test flags, without initializing +// other parts of Google Test. The type parameter CharType can be +// instantiated to either char or wchar_t. +template +void ParseGoogleTestFlagsOnlyImpl(int* argc, CharType** argv) { + for (int i = 1; i < *argc; i++) { + const std::string arg_string = StreamableToString(argv[i]); + const char* const arg = arg_string.c_str(); + + using internal::ParseBoolFlag; + using internal::ParseInt32Flag; + using internal::ParseStringFlag; + + bool remove_flag = false; + if (ParseGoogleTestFlag(arg)) { + remove_flag = true; +#if GTEST_USE_OWN_FLAGFILE_FLAG_ + } else if (ParseStringFlag(arg, kFlagfileFlag, >EST_FLAG(flagfile))) { + LoadFlagsFromFile(GTEST_FLAG(flagfile)); + remove_flag = true; +#endif // GTEST_USE_OWN_FLAGFILE_FLAG_ + } else if (arg_string == "--help" || arg_string == "-h" || + arg_string == "-?" || arg_string == "/?" || + HasGoogleTestFlagPrefix(arg)) { + // Both help flag and unrecognized Google Test flags (excluding + // internal ones) trigger help display. + g_help_flag = true; + } + + if (remove_flag) { + // Shift the remainder of the argv list left by one. Note + // that argv has (*argc + 1) elements, the last one always being + // NULL. The following loop moves the trailing NULL element as + // well. + for (int j = i; j != *argc; j++) { + argv[j] = argv[j + 1]; + } + + // Decrements the argument count. + (*argc)--; + + // We also need to decrement the iterator as we just removed + // an element. + i--; + } + } + + if (g_help_flag) { + // We print the help here instead of in RUN_ALL_TESTS(), as the + // latter may not be called at all if the user is using Google + // Test with another testing framework. + PrintColorEncoded(kColorEncodedHelpMessage); + } +} + +// Parses the command line for Google Test flags, without initializing +// other parts of Google Test. +void ParseGoogleTestFlagsOnly(int* argc, char** argv) { + ParseGoogleTestFlagsOnlyImpl(argc, argv); + + // Fix the value of *_NSGetArgc() on macOS, but iff + // *_NSGetArgv() == argv + // Only applicable to char** version of argv +#if GTEST_OS_MAC +#ifndef GTEST_OS_IOS + if (*_NSGetArgv() == argv) { + *_NSGetArgc() = *argc; + } +#endif +#endif +} +void ParseGoogleTestFlagsOnly(int* argc, wchar_t** argv) { + ParseGoogleTestFlagsOnlyImpl(argc, argv); +} + +// The internal implementation of InitGoogleTest(). +// +// The type parameter CharType can be instantiated to either char or +// wchar_t. +template +void InitGoogleTestImpl(int* argc, CharType** argv) { + // We don't want to run the initialization code twice. + if (GTestIsInitialized()) return; + + if (*argc <= 0) return; + + g_argvs.clear(); + for (int i = 0; i != *argc; i++) { + g_argvs.push_back(StreamableToString(argv[i])); + } + +#if GTEST_HAS_ABSL + absl::InitializeSymbolizer(g_argvs[0].c_str()); +#endif // GTEST_HAS_ABSL + + ParseGoogleTestFlagsOnly(argc, argv); + GetUnitTestImpl()->PostFlagParsingInit(); +} + +} // namespace internal + +// Initializes Google Test. This must be called before calling +// RUN_ALL_TESTS(). In particular, it parses a command line for the +// flags that Google Test recognizes. Whenever a Google Test flag is +// seen, it is removed from argv, and *argc is decremented. +// +// No value is returned. Instead, the Google Test flag variables are +// updated. +// +// Calling the function for the second time has no user-visible effect. +void InitGoogleTest(int* argc, char** argv) { +#if defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) + GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_(argc, argv); +#else // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) + internal::InitGoogleTestImpl(argc, argv); +#endif // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) +} + +// This overloaded version can be used in Windows programs compiled in +// UNICODE mode. +void InitGoogleTest(int* argc, wchar_t** argv) { +#if defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) + GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_(argc, argv); +#else // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) + internal::InitGoogleTestImpl(argc, argv); +#endif // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) +} + +std::string TempDir() { +#if defined(GTEST_CUSTOM_TEMPDIR_FUNCTION_) + return GTEST_CUSTOM_TEMPDIR_FUNCTION_(); +#endif + +#if GTEST_OS_WINDOWS_MOBILE + return "\\temp\\"; +#elif GTEST_OS_WINDOWS + const char* temp_dir = internal::posix::GetEnv("TEMP"); + if (temp_dir == NULL || temp_dir[0] == '\0') + return "\\temp\\"; + else if (temp_dir[strlen(temp_dir) - 1] == '\\') + return temp_dir; + else + return std::string(temp_dir) + "\\"; +#elif GTEST_OS_LINUX_ANDROID + return "/sdcard/"; +#else + return "/tmp/"; +#endif // GTEST_OS_WINDOWS_MOBILE +} + +// Class ScopedTrace + +// Pushes the given source file location and message onto a per-thread +// trace stack maintained by Google Test. +void ScopedTrace::PushTrace(const char* file, int line, std::string message) { + internal::TraceInfo trace; + trace.file = file; + trace.line = line; + trace.message.swap(message); + + UnitTest::GetInstance()->PushGTestTrace(trace); +} + +// Pops the info pushed by the c'tor. +ScopedTrace::~ScopedTrace() + GTEST_LOCK_EXCLUDED_(&UnitTest::mutex_) { + UnitTest::GetInstance()->PopGTestTrace(); +} + +} // namespace testing +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// +// This file implements death tests. + + +#if GTEST_HAS_DEATH_TEST + +# if GTEST_OS_MAC +# include +# endif // GTEST_OS_MAC + +# include +# include +# include + +# if GTEST_OS_LINUX +# include +# endif // GTEST_OS_LINUX + +# include + +# if GTEST_OS_WINDOWS +# include +# else +# include +# include +# endif // GTEST_OS_WINDOWS + +# if GTEST_OS_QNX +# include +# endif // GTEST_OS_QNX + +# if GTEST_OS_FUCHSIA +# include +# include +# include +# include +# include +# endif // GTEST_OS_FUCHSIA + +#endif // GTEST_HAS_DEATH_TEST + + +namespace testing { + +// Constants. + +// The default death test style. +// +// This is defined in internal/gtest-port.h as "fast", but can be overridden by +// a definition in internal/custom/gtest-port.h. The recommended value, which is +// used internally at Google, is "threadsafe". +static const char kDefaultDeathTestStyle[] = GTEST_DEFAULT_DEATH_TEST_STYLE; + +GTEST_DEFINE_string_( + death_test_style, + internal::StringFromGTestEnv("death_test_style", kDefaultDeathTestStyle), + "Indicates how to run a death test in a forked child process: " + "\"threadsafe\" (child process re-executes the test binary " + "from the beginning, running only the specific death test) or " + "\"fast\" (child process runs the death test immediately " + "after forking)."); + +GTEST_DEFINE_bool_( + death_test_use_fork, + internal::BoolFromGTestEnv("death_test_use_fork", false), + "Instructs to use fork()/_exit() instead of clone() in death tests. " + "Ignored and always uses fork() on POSIX systems where clone() is not " + "implemented. Useful when running under valgrind or similar tools if " + "those do not support clone(). Valgrind 3.3.1 will just fail if " + "it sees an unsupported combination of clone() flags. " + "It is not recommended to use this flag w/o valgrind though it will " + "work in 99% of the cases. Once valgrind is fixed, this flag will " + "most likely be removed."); + +namespace internal { +GTEST_DEFINE_string_( + internal_run_death_test, "", + "Indicates the file, line number, temporal index of " + "the single death test to run, and a file descriptor to " + "which a success code may be sent, all separated by " + "the '|' characters. This flag is specified if and only if the current " + "process is a sub-process launched for running a thread-safe " + "death test. FOR INTERNAL USE ONLY."); +} // namespace internal + +#if GTEST_HAS_DEATH_TEST + +namespace internal { + +// Valid only for fast death tests. Indicates the code is running in the +// child process of a fast style death test. +# if !GTEST_OS_WINDOWS && !GTEST_OS_FUCHSIA +static bool g_in_fast_death_test_child = false; +# endif + +// Returns a Boolean value indicating whether the caller is currently +// executing in the context of the death test child process. Tools such as +// Valgrind heap checkers may need this to modify their behavior in death +// tests. IMPORTANT: This is an internal utility. Using it may break the +// implementation of death tests. User code MUST NOT use it. +bool InDeathTestChild() { +# if GTEST_OS_WINDOWS || GTEST_OS_FUCHSIA + + // On Windows and Fuchsia, death tests are thread-safe regardless of the value + // of the death_test_style flag. + return !GTEST_FLAG(internal_run_death_test).empty(); + +# else + + if (GTEST_FLAG(death_test_style) == "threadsafe") + return !GTEST_FLAG(internal_run_death_test).empty(); + else + return g_in_fast_death_test_child; +#endif +} + +} // namespace internal + +// ExitedWithCode constructor. +ExitedWithCode::ExitedWithCode(int exit_code) : exit_code_(exit_code) { +} + +// ExitedWithCode function-call operator. +bool ExitedWithCode::operator()(int exit_status) const { +# if GTEST_OS_WINDOWS || GTEST_OS_FUCHSIA + + return exit_status == exit_code_; + +# else + + return WIFEXITED(exit_status) && WEXITSTATUS(exit_status) == exit_code_; + +# endif // GTEST_OS_WINDOWS || GTEST_OS_FUCHSIA +} + +# if !GTEST_OS_WINDOWS && !GTEST_OS_FUCHSIA +// KilledBySignal constructor. +KilledBySignal::KilledBySignal(int signum) : signum_(signum) { +} + +// KilledBySignal function-call operator. +bool KilledBySignal::operator()(int exit_status) const { +# if defined(GTEST_KILLED_BY_SIGNAL_OVERRIDE_) + { + bool result; + if (GTEST_KILLED_BY_SIGNAL_OVERRIDE_(signum_, exit_status, &result)) { + return result; + } + } +# endif // defined(GTEST_KILLED_BY_SIGNAL_OVERRIDE_) + return WIFSIGNALED(exit_status) && WTERMSIG(exit_status) == signum_; +} +# endif // !GTEST_OS_WINDOWS && !GTEST_OS_FUCHSIA + +namespace internal { + +// Utilities needed for death tests. + +// Generates a textual description of a given exit code, in the format +// specified by wait(2). +static std::string ExitSummary(int exit_code) { + Message m; + +# if GTEST_OS_WINDOWS || GTEST_OS_FUCHSIA + + m << "Exited with exit status " << exit_code; + +# else + + if (WIFEXITED(exit_code)) { + m << "Exited with exit status " << WEXITSTATUS(exit_code); + } else if (WIFSIGNALED(exit_code)) { + m << "Terminated by signal " << WTERMSIG(exit_code); + } +# ifdef WCOREDUMP + if (WCOREDUMP(exit_code)) { + m << " (core dumped)"; + } +# endif +# endif // GTEST_OS_WINDOWS || GTEST_OS_FUCHSIA + + return m.GetString(); +} + +// Returns true if exit_status describes a process that was terminated +// by a signal, or exited normally with a nonzero exit code. +bool ExitedUnsuccessfully(int exit_status) { + return !ExitedWithCode(0)(exit_status); +} + +# if !GTEST_OS_WINDOWS && !GTEST_OS_FUCHSIA +// Generates a textual failure message when a death test finds more than +// one thread running, or cannot determine the number of threads, prior +// to executing the given statement. It is the responsibility of the +// caller not to pass a thread_count of 1. +static std::string DeathTestThreadWarning(size_t thread_count) { + Message msg; + msg << "Death tests use fork(), which is unsafe particularly" + << " in a threaded context. For this test, " << GTEST_NAME_ << " "; + if (thread_count == 0) { + msg << "couldn't detect the number of threads."; + } else { + msg << "detected " << thread_count << " threads."; + } + msg << " See " + "https://github.com/google/googletest/blob/master/googletest/docs/" + "advanced.md#death-tests-and-threads" + << " for more explanation and suggested solutions, especially if" + << " this is the last message you see before your test times out."; + return msg.GetString(); +} +# endif // !GTEST_OS_WINDOWS && !GTEST_OS_FUCHSIA + +// Flag characters for reporting a death test that did not die. +static const char kDeathTestLived = 'L'; +static const char kDeathTestReturned = 'R'; +static const char kDeathTestThrew = 'T'; +static const char kDeathTestInternalError = 'I'; + +#if GTEST_OS_FUCHSIA + +// File descriptor used for the pipe in the child process. +static const int kFuchsiaReadPipeFd = 3; + +#endif + +// An enumeration describing all of the possible ways that a death test can +// conclude. DIED means that the process died while executing the test +// code; LIVED means that process lived beyond the end of the test code; +// RETURNED means that the test statement attempted to execute a return +// statement, which is not allowed; THREW means that the test statement +// returned control by throwing an exception. IN_PROGRESS means the test +// has not yet concluded. +// FIXME: Unify names and possibly values for +// AbortReason, DeathTestOutcome, and flag characters above. +enum DeathTestOutcome { IN_PROGRESS, DIED, LIVED, RETURNED, THREW }; + +// Routine for aborting the program which is safe to call from an +// exec-style death test child process, in which case the error +// message is propagated back to the parent process. Otherwise, the +// message is simply printed to stderr. In either case, the program +// then exits with status 1. +static void DeathTestAbort(const std::string& message) { + // On a POSIX system, this function may be called from a threadsafe-style + // death test child process, which operates on a very small stack. Use + // the heap for any additional non-minuscule memory requirements. + const InternalRunDeathTestFlag* const flag = + GetUnitTestImpl()->internal_run_death_test_flag(); + if (flag != NULL) { + FILE* parent = posix::FDOpen(flag->write_fd(), "w"); + fputc(kDeathTestInternalError, parent); + fprintf(parent, "%s", message.c_str()); + fflush(parent); + _exit(1); + } else { + fprintf(stderr, "%s", message.c_str()); + fflush(stderr); + posix::Abort(); + } +} + +// A replacement for CHECK that calls DeathTestAbort if the assertion +// fails. +# define GTEST_DEATH_TEST_CHECK_(expression) \ + do { \ + if (!::testing::internal::IsTrue(expression)) { \ + DeathTestAbort( \ + ::std::string("CHECK failed: File ") + __FILE__ + ", line " \ + + ::testing::internal::StreamableToString(__LINE__) + ": " \ + + #expression); \ + } \ + } while (::testing::internal::AlwaysFalse()) + +// This macro is similar to GTEST_DEATH_TEST_CHECK_, but it is meant for +// evaluating any system call that fulfills two conditions: it must return +// -1 on failure, and set errno to EINTR when it is interrupted and +// should be tried again. The macro expands to a loop that repeatedly +// evaluates the expression as long as it evaluates to -1 and sets +// errno to EINTR. If the expression evaluates to -1 but errno is +// something other than EINTR, DeathTestAbort is called. +# define GTEST_DEATH_TEST_CHECK_SYSCALL_(expression) \ + do { \ + int gtest_retval; \ + do { \ + gtest_retval = (expression); \ + } while (gtest_retval == -1 && errno == EINTR); \ + if (gtest_retval == -1) { \ + DeathTestAbort( \ + ::std::string("CHECK failed: File ") + __FILE__ + ", line " \ + + ::testing::internal::StreamableToString(__LINE__) + ": " \ + + #expression + " != -1"); \ + } \ + } while (::testing::internal::AlwaysFalse()) + +// Returns the message describing the last system error in errno. +std::string GetLastErrnoDescription() { + return errno == 0 ? "" : posix::StrError(errno); +} + +// This is called from a death test parent process to read a failure +// message from the death test child process and log it with the FATAL +// severity. On Windows, the message is read from a pipe handle. On other +// platforms, it is read from a file descriptor. +static void FailFromInternalError(int fd) { + Message error; + char buffer[256]; + int num_read; + + do { + while ((num_read = posix::Read(fd, buffer, 255)) > 0) { + buffer[num_read] = '\0'; + error << buffer; + } + } while (num_read == -1 && errno == EINTR); + + if (num_read == 0) { + GTEST_LOG_(FATAL) << error.GetString(); + } else { + const int last_error = errno; + GTEST_LOG_(FATAL) << "Error while reading death test internal: " + << GetLastErrnoDescription() << " [" << last_error << "]"; + } +} + +// Death test constructor. Increments the running death test count +// for the current test. +DeathTest::DeathTest() { + TestInfo* const info = GetUnitTestImpl()->current_test_info(); + if (info == NULL) { + DeathTestAbort("Cannot run a death test outside of a TEST or " + "TEST_F construct"); + } +} + +// Creates and returns a death test by dispatching to the current +// death test factory. +bool DeathTest::Create(const char* statement, const RE* regex, + const char* file, int line, DeathTest** test) { + return GetUnitTestImpl()->death_test_factory()->Create( + statement, regex, file, line, test); +} + +const char* DeathTest::LastMessage() { + return last_death_test_message_.c_str(); +} + +void DeathTest::set_last_death_test_message(const std::string& message) { + last_death_test_message_ = message; +} + +std::string DeathTest::last_death_test_message_; + +// Provides cross platform implementation for some death functionality. +class DeathTestImpl : public DeathTest { + protected: + DeathTestImpl(const char* a_statement, const RE* a_regex) + : statement_(a_statement), + regex_(a_regex), + spawned_(false), + status_(-1), + outcome_(IN_PROGRESS), + read_fd_(-1), + write_fd_(-1) {} + + // read_fd_ is expected to be closed and cleared by a derived class. + ~DeathTestImpl() { GTEST_DEATH_TEST_CHECK_(read_fd_ == -1); } + + void Abort(AbortReason reason); + virtual bool Passed(bool status_ok); + + const char* statement() const { return statement_; } + const RE* regex() const { return regex_; } + bool spawned() const { return spawned_; } + void set_spawned(bool is_spawned) { spawned_ = is_spawned; } + int status() const { return status_; } + void set_status(int a_status) { status_ = a_status; } + DeathTestOutcome outcome() const { return outcome_; } + void set_outcome(DeathTestOutcome an_outcome) { outcome_ = an_outcome; } + int read_fd() const { return read_fd_; } + void set_read_fd(int fd) { read_fd_ = fd; } + int write_fd() const { return write_fd_; } + void set_write_fd(int fd) { write_fd_ = fd; } + + // Called in the parent process only. Reads the result code of the death + // test child process via a pipe, interprets it to set the outcome_ + // member, and closes read_fd_. Outputs diagnostics and terminates in + // case of unexpected codes. + void ReadAndInterpretStatusByte(); + + private: + // The textual content of the code this object is testing. This class + // doesn't own this string and should not attempt to delete it. + const char* const statement_; + // The regular expression which test output must match. DeathTestImpl + // doesn't own this object and should not attempt to delete it. + const RE* const regex_; + // True if the death test child process has been successfully spawned. + bool spawned_; + // The exit status of the child process. + int status_; + // How the death test concluded. + DeathTestOutcome outcome_; + // Descriptor to the read end of the pipe to the child process. It is + // always -1 in the child process. The child keeps its write end of the + // pipe in write_fd_. + int read_fd_; + // Descriptor to the child's write end of the pipe to the parent process. + // It is always -1 in the parent process. The parent keeps its end of the + // pipe in read_fd_. + int write_fd_; +}; + +// Called in the parent process only. Reads the result code of the death +// test child process via a pipe, interprets it to set the outcome_ +// member, and closes read_fd_. Outputs diagnostics and terminates in +// case of unexpected codes. +void DeathTestImpl::ReadAndInterpretStatusByte() { + char flag; + int bytes_read; + + // The read() here blocks until data is available (signifying the + // failure of the death test) or until the pipe is closed (signifying + // its success), so it's okay to call this in the parent before + // the child process has exited. + do { + bytes_read = posix::Read(read_fd(), &flag, 1); + } while (bytes_read == -1 && errno == EINTR); + + if (bytes_read == 0) { + set_outcome(DIED); + } else if (bytes_read == 1) { + switch (flag) { + case kDeathTestReturned: + set_outcome(RETURNED); + break; + case kDeathTestThrew: + set_outcome(THREW); + break; + case kDeathTestLived: + set_outcome(LIVED); + break; + case kDeathTestInternalError: + FailFromInternalError(read_fd()); // Does not return. + break; + default: + GTEST_LOG_(FATAL) << "Death test child process reported " + << "unexpected status byte (" + << static_cast(flag) << ")"; + } + } else { + GTEST_LOG_(FATAL) << "Read from death test child process failed: " + << GetLastErrnoDescription(); + } + GTEST_DEATH_TEST_CHECK_SYSCALL_(posix::Close(read_fd())); + set_read_fd(-1); +} + +// Signals that the death test code which should have exited, didn't. +// Should be called only in a death test child process. +// Writes a status byte to the child's status file descriptor, then +// calls _exit(1). +void DeathTestImpl::Abort(AbortReason reason) { + // The parent process considers the death test to be a failure if + // it finds any data in our pipe. So, here we write a single flag byte + // to the pipe, then exit. + const char status_ch = + reason == TEST_DID_NOT_DIE ? kDeathTestLived : + reason == TEST_THREW_EXCEPTION ? kDeathTestThrew : kDeathTestReturned; + + GTEST_DEATH_TEST_CHECK_SYSCALL_(posix::Write(write_fd(), &status_ch, 1)); + // We are leaking the descriptor here because on some platforms (i.e., + // when built as Windows DLL), destructors of global objects will still + // run after calling _exit(). On such systems, write_fd_ will be + // indirectly closed from the destructor of UnitTestImpl, causing double + // close if it is also closed here. On debug configurations, double close + // may assert. As there are no in-process buffers to flush here, we are + // relying on the OS to close the descriptor after the process terminates + // when the destructors are not run. + _exit(1); // Exits w/o any normal exit hooks (we were supposed to crash) +} + +// Returns an indented copy of stderr output for a death test. +// This makes distinguishing death test output lines from regular log lines +// much easier. +static ::std::string FormatDeathTestOutput(const ::std::string& output) { + ::std::string ret; + for (size_t at = 0; ; ) { + const size_t line_end = output.find('\n', at); + ret += "[ DEATH ] "; + if (line_end == ::std::string::npos) { + ret += output.substr(at); + break; + } + ret += output.substr(at, line_end + 1 - at); + at = line_end + 1; + } + return ret; +} + +// Assesses the success or failure of a death test, using both private +// members which have previously been set, and one argument: +// +// Private data members: +// outcome: An enumeration describing how the death test +// concluded: DIED, LIVED, THREW, or RETURNED. The death test +// fails in the latter three cases. +// status: The exit status of the child process. On *nix, it is in the +// in the format specified by wait(2). On Windows, this is the +// value supplied to the ExitProcess() API or a numeric code +// of the exception that terminated the program. +// regex: A regular expression object to be applied to +// the test's captured standard error output; the death test +// fails if it does not match. +// +// Argument: +// status_ok: true if exit_status is acceptable in the context of +// this particular death test, which fails if it is false +// +// Returns true iff all of the above conditions are met. Otherwise, the +// first failing condition, in the order given above, is the one that is +// reported. Also sets the last death test message string. +bool DeathTestImpl::Passed(bool status_ok) { + if (!spawned()) + return false; + + const std::string error_message = GetCapturedStderr(); + + bool success = false; + Message buffer; + + buffer << "Death test: " << statement() << "\n"; + switch (outcome()) { + case LIVED: + buffer << " Result: failed to die.\n" + << " Error msg:\n" << FormatDeathTestOutput(error_message); + break; + case THREW: + buffer << " Result: threw an exception.\n" + << " Error msg:\n" << FormatDeathTestOutput(error_message); + break; + case RETURNED: + buffer << " Result: illegal return in test statement.\n" + << " Error msg:\n" << FormatDeathTestOutput(error_message); + break; + case DIED: + if (status_ok) { +# if GTEST_USES_PCRE + // PCRE regexes support embedded NULs. + const bool matched = RE::PartialMatch(error_message, *regex()); +# else + const bool matched = RE::PartialMatch(error_message.c_str(), *regex()); +# endif // GTEST_USES_PCRE + if (matched) { + success = true; + } else { + buffer << " Result: died but not with expected error.\n" + << " Expected: " << regex()->pattern() << "\n" + << "Actual msg:\n" << FormatDeathTestOutput(error_message); + } + } else { + buffer << " Result: died but not with expected exit code:\n" + << " " << ExitSummary(status()) << "\n" + << "Actual msg:\n" << FormatDeathTestOutput(error_message); + } + break; + case IN_PROGRESS: + default: + GTEST_LOG_(FATAL) + << "DeathTest::Passed somehow called before conclusion of test"; + } + + DeathTest::set_last_death_test_message(buffer.GetString()); + return success; +} + +# if GTEST_OS_WINDOWS +// WindowsDeathTest implements death tests on Windows. Due to the +// specifics of starting new processes on Windows, death tests there are +// always threadsafe, and Google Test considers the +// --gtest_death_test_style=fast setting to be equivalent to +// --gtest_death_test_style=threadsafe there. +// +// A few implementation notes: Like the Linux version, the Windows +// implementation uses pipes for child-to-parent communication. But due to +// the specifics of pipes on Windows, some extra steps are required: +// +// 1. The parent creates a communication pipe and stores handles to both +// ends of it. +// 2. The parent starts the child and provides it with the information +// necessary to acquire the handle to the write end of the pipe. +// 3. The child acquires the write end of the pipe and signals the parent +// using a Windows event. +// 4. Now the parent can release the write end of the pipe on its side. If +// this is done before step 3, the object's reference count goes down to +// 0 and it is destroyed, preventing the child from acquiring it. The +// parent now has to release it, or read operations on the read end of +// the pipe will not return when the child terminates. +// 5. The parent reads child's output through the pipe (outcome code and +// any possible error messages) from the pipe, and its stderr and then +// determines whether to fail the test. +// +// Note: to distinguish Win32 API calls from the local method and function +// calls, the former are explicitly resolved in the global namespace. +// +class WindowsDeathTest : public DeathTestImpl { + public: + WindowsDeathTest(const char* a_statement, + const RE* a_regex, + const char* file, + int line) + : DeathTestImpl(a_statement, a_regex), file_(file), line_(line) {} + + // All of these virtual functions are inherited from DeathTest. + virtual int Wait(); + virtual TestRole AssumeRole(); + + private: + // The name of the file in which the death test is located. + const char* const file_; + // The line number on which the death test is located. + const int line_; + // Handle to the write end of the pipe to the child process. + AutoHandle write_handle_; + // Child process handle. + AutoHandle child_handle_; + // Event the child process uses to signal the parent that it has + // acquired the handle to the write end of the pipe. After seeing this + // event the parent can release its own handles to make sure its + // ReadFile() calls return when the child terminates. + AutoHandle event_handle_; +}; + +// Waits for the child in a death test to exit, returning its exit +// status, or 0 if no child process exists. As a side effect, sets the +// outcome data member. +int WindowsDeathTest::Wait() { + if (!spawned()) + return 0; + + // Wait until the child either signals that it has acquired the write end + // of the pipe or it dies. + const HANDLE wait_handles[2] = { child_handle_.Get(), event_handle_.Get() }; + switch (::WaitForMultipleObjects(2, + wait_handles, + FALSE, // Waits for any of the handles. + INFINITE)) { + case WAIT_OBJECT_0: + case WAIT_OBJECT_0 + 1: + break; + default: + GTEST_DEATH_TEST_CHECK_(false); // Should not get here. + } + + // The child has acquired the write end of the pipe or exited. + // We release the handle on our side and continue. + write_handle_.Reset(); + event_handle_.Reset(); + + ReadAndInterpretStatusByte(); + + // Waits for the child process to exit if it haven't already. This + // returns immediately if the child has already exited, regardless of + // whether previous calls to WaitForMultipleObjects synchronized on this + // handle or not. + GTEST_DEATH_TEST_CHECK_( + WAIT_OBJECT_0 == ::WaitForSingleObject(child_handle_.Get(), + INFINITE)); + DWORD status_code; + GTEST_DEATH_TEST_CHECK_( + ::GetExitCodeProcess(child_handle_.Get(), &status_code) != FALSE); + child_handle_.Reset(); + set_status(static_cast(status_code)); + return status(); +} + +// The AssumeRole process for a Windows death test. It creates a child +// process with the same executable as the current process to run the +// death test. The child process is given the --gtest_filter and +// --gtest_internal_run_death_test flags such that it knows to run the +// current death test only. +DeathTest::TestRole WindowsDeathTest::AssumeRole() { + const UnitTestImpl* const impl = GetUnitTestImpl(); + const InternalRunDeathTestFlag* const flag = + impl->internal_run_death_test_flag(); + const TestInfo* const info = impl->current_test_info(); + const int death_test_index = info->result()->death_test_count(); + + if (flag != NULL) { + // ParseInternalRunDeathTestFlag() has performed all the necessary + // processing. + set_write_fd(flag->write_fd()); + return EXECUTE_TEST; + } + + // WindowsDeathTest uses an anonymous pipe to communicate results of + // a death test. + SECURITY_ATTRIBUTES handles_are_inheritable = { + sizeof(SECURITY_ATTRIBUTES), NULL, TRUE }; + HANDLE read_handle, write_handle; + GTEST_DEATH_TEST_CHECK_( + ::CreatePipe(&read_handle, &write_handle, &handles_are_inheritable, + 0) // Default buffer size. + != FALSE); + set_read_fd(::_open_osfhandle(reinterpret_cast(read_handle), + O_RDONLY)); + write_handle_.Reset(write_handle); + event_handle_.Reset(::CreateEvent( + &handles_are_inheritable, + TRUE, // The event will automatically reset to non-signaled state. + FALSE, // The initial state is non-signalled. + NULL)); // The even is unnamed. + GTEST_DEATH_TEST_CHECK_(event_handle_.Get() != NULL); + const std::string filter_flag = + std::string("--") + GTEST_FLAG_PREFIX_ + kFilterFlag + "=" + + info->test_case_name() + "." + info->name(); + const std::string internal_flag = + std::string("--") + GTEST_FLAG_PREFIX_ + kInternalRunDeathTestFlag + + "=" + file_ + "|" + StreamableToString(line_) + "|" + + StreamableToString(death_test_index) + "|" + + StreamableToString(static_cast(::GetCurrentProcessId())) + + // size_t has the same width as pointers on both 32-bit and 64-bit + // Windows platforms. + // See http://msdn.microsoft.com/en-us/library/tcxf1dw6.aspx. + "|" + StreamableToString(reinterpret_cast(write_handle)) + + "|" + StreamableToString(reinterpret_cast(event_handle_.Get())); + + char executable_path[_MAX_PATH + 1]; // NOLINT + GTEST_DEATH_TEST_CHECK_( + _MAX_PATH + 1 != ::GetModuleFileNameA(NULL, + executable_path, + _MAX_PATH)); + + std::string command_line = + std::string(::GetCommandLineA()) + " " + filter_flag + " \"" + + internal_flag + "\""; + + DeathTest::set_last_death_test_message(""); + + CaptureStderr(); + // Flush the log buffers since the log streams are shared with the child. + FlushInfoLog(); + + // The child process will share the standard handles with the parent. + STARTUPINFOA startup_info; + memset(&startup_info, 0, sizeof(STARTUPINFO)); + startup_info.dwFlags = STARTF_USESTDHANDLES; + startup_info.hStdInput = ::GetStdHandle(STD_INPUT_HANDLE); + startup_info.hStdOutput = ::GetStdHandle(STD_OUTPUT_HANDLE); + startup_info.hStdError = ::GetStdHandle(STD_ERROR_HANDLE); + + PROCESS_INFORMATION process_info; + GTEST_DEATH_TEST_CHECK_(::CreateProcessA( + executable_path, + const_cast(command_line.c_str()), + NULL, // Retuned process handle is not inheritable. + NULL, // Retuned thread handle is not inheritable. + TRUE, // Child inherits all inheritable handles (for write_handle_). + 0x0, // Default creation flags. + NULL, // Inherit the parent's environment. + UnitTest::GetInstance()->original_working_dir(), + &startup_info, + &process_info) != FALSE); + child_handle_.Reset(process_info.hProcess); + ::CloseHandle(process_info.hThread); + set_spawned(true); + return OVERSEE_TEST; +} + +# elif GTEST_OS_FUCHSIA + +class FuchsiaDeathTest : public DeathTestImpl { + public: + FuchsiaDeathTest(const char* a_statement, + const RE* a_regex, + const char* file, + int line) + : DeathTestImpl(a_statement, a_regex), file_(file), line_(line) {} + virtual ~FuchsiaDeathTest() { + zx_status_t status = zx_handle_close(child_process_); + GTEST_DEATH_TEST_CHECK_(status == ZX_OK); + status = zx_handle_close(port_); + GTEST_DEATH_TEST_CHECK_(status == ZX_OK); + } + + // All of these virtual functions are inherited from DeathTest. + virtual int Wait(); + virtual TestRole AssumeRole(); + + private: + // The name of the file in which the death test is located. + const char* const file_; + // The line number on which the death test is located. + const int line_; + + zx_handle_t child_process_ = ZX_HANDLE_INVALID; + zx_handle_t port_ = ZX_HANDLE_INVALID; +}; + +// Utility class for accumulating command-line arguments. +class Arguments { + public: + Arguments() { + args_.push_back(NULL); + } + + ~Arguments() { + for (std::vector::iterator i = args_.begin(); i != args_.end(); + ++i) { + free(*i); + } + } + void AddArgument(const char* argument) { + args_.insert(args_.end() - 1, posix::StrDup(argument)); + } + + template + void AddArguments(const ::std::vector& arguments) { + for (typename ::std::vector::const_iterator i = arguments.begin(); + i != arguments.end(); + ++i) { + args_.insert(args_.end() - 1, posix::StrDup(i->c_str())); + } + } + char* const* Argv() { + return &args_[0]; + } + + int size() { + return args_.size() - 1; + } + + private: + std::vector args_; +}; + +// Waits for the child in a death test to exit, returning its exit +// status, or 0 if no child process exists. As a side effect, sets the +// outcome data member. +int FuchsiaDeathTest::Wait() { + if (!spawned()) + return 0; + + // Register to wait for the child process to terminate. + zx_status_t status_zx; + status_zx = zx_object_wait_async(child_process_, + port_, + 0 /* key */, + ZX_PROCESS_TERMINATED, + ZX_WAIT_ASYNC_ONCE); + GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); + + // Wait for it to terminate, or an exception to be received. + zx_port_packet_t packet; + status_zx = zx_port_wait(port_, ZX_TIME_INFINITE, &packet); + GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); + + if (ZX_PKT_IS_EXCEPTION(packet.type)) { + // Process encountered an exception. Kill it directly rather than letting + // other handlers process the event. + status_zx = zx_task_kill(child_process_); + GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); + + // Now wait for |child_process_| to terminate. + zx_signals_t signals = 0; + status_zx = zx_object_wait_one( + child_process_, ZX_PROCESS_TERMINATED, ZX_TIME_INFINITE, &signals); + GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); + GTEST_DEATH_TEST_CHECK_(signals & ZX_PROCESS_TERMINATED); + } else { + // Process terminated. + GTEST_DEATH_TEST_CHECK_(ZX_PKT_IS_SIGNAL_ONE(packet.type)); + GTEST_DEATH_TEST_CHECK_(packet.signal.observed & ZX_PROCESS_TERMINATED); + } + + ReadAndInterpretStatusByte(); + + zx_info_process_t buffer; + status_zx = zx_object_get_info( + child_process_, + ZX_INFO_PROCESS, + &buffer, + sizeof(buffer), + nullptr, + nullptr); + GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); + + GTEST_DEATH_TEST_CHECK_(buffer.exited); + set_status(buffer.return_code); + return status(); +} + +// The AssumeRole process for a Fuchsia death test. It creates a child +// process with the same executable as the current process to run the +// death test. The child process is given the --gtest_filter and +// --gtest_internal_run_death_test flags such that it knows to run the +// current death test only. +DeathTest::TestRole FuchsiaDeathTest::AssumeRole() { + const UnitTestImpl* const impl = GetUnitTestImpl(); + const InternalRunDeathTestFlag* const flag = + impl->internal_run_death_test_flag(); + const TestInfo* const info = impl->current_test_info(); + const int death_test_index = info->result()->death_test_count(); + + if (flag != NULL) { + // ParseInternalRunDeathTestFlag() has performed all the necessary + // processing. + set_write_fd(kFuchsiaReadPipeFd); + return EXECUTE_TEST; + } + + CaptureStderr(); + // Flush the log buffers since the log streams are shared with the child. + FlushInfoLog(); + + // Build the child process command line. + const std::string filter_flag = + std::string("--") + GTEST_FLAG_PREFIX_ + kFilterFlag + "=" + + info->test_case_name() + "." + info->name(); + const std::string internal_flag = + std::string("--") + GTEST_FLAG_PREFIX_ + kInternalRunDeathTestFlag + "=" + + file_ + "|" + + StreamableToString(line_) + "|" + + StreamableToString(death_test_index); + Arguments args; + args.AddArguments(GetInjectableArgvs()); + args.AddArgument(filter_flag.c_str()); + args.AddArgument(internal_flag.c_str()); + + // Build the pipe for communication with the child. + zx_status_t status; + zx_handle_t child_pipe_handle; + uint32_t type; + status = fdio_pipe_half(&child_pipe_handle, &type); + GTEST_DEATH_TEST_CHECK_(status >= 0); + set_read_fd(status); + + // Set the pipe handle for the child. + fdio_spawn_action_t add_handle_action = {}; + add_handle_action.action = FDIO_SPAWN_ACTION_ADD_HANDLE; + add_handle_action.h.id = PA_HND(type, kFuchsiaReadPipeFd); + add_handle_action.h.handle = child_pipe_handle; + + // Spawn the child process. + status = fdio_spawn_etc(ZX_HANDLE_INVALID, FDIO_SPAWN_CLONE_ALL, + args.Argv()[0], args.Argv(), nullptr, 1, + &add_handle_action, &child_process_, nullptr); + GTEST_DEATH_TEST_CHECK_(status == ZX_OK); + + // Create an exception port and attach it to the |child_process_|, to allow + // us to suppress the system default exception handler from firing. + status = zx_port_create(0, &port_); + GTEST_DEATH_TEST_CHECK_(status == ZX_OK); + status = zx_task_bind_exception_port( + child_process_, port_, 0 /* key */, 0 /*options */); + GTEST_DEATH_TEST_CHECK_(status == ZX_OK); + + set_spawned(true); + return OVERSEE_TEST; +} + +#else // We are neither on Windows, nor on Fuchsia. + +// ForkingDeathTest provides implementations for most of the abstract +// methods of the DeathTest interface. Only the AssumeRole method is +// left undefined. +class ForkingDeathTest : public DeathTestImpl { + public: + ForkingDeathTest(const char* statement, const RE* regex); + + // All of these virtual functions are inherited from DeathTest. + virtual int Wait(); + + protected: + void set_child_pid(pid_t child_pid) { child_pid_ = child_pid; } + + private: + // PID of child process during death test; 0 in the child process itself. + pid_t child_pid_; +}; + +// Constructs a ForkingDeathTest. +ForkingDeathTest::ForkingDeathTest(const char* a_statement, const RE* a_regex) + : DeathTestImpl(a_statement, a_regex), + child_pid_(-1) {} + +// Waits for the child in a death test to exit, returning its exit +// status, or 0 if no child process exists. As a side effect, sets the +// outcome data member. +int ForkingDeathTest::Wait() { + if (!spawned()) + return 0; + + ReadAndInterpretStatusByte(); + + int status_value; + GTEST_DEATH_TEST_CHECK_SYSCALL_(waitpid(child_pid_, &status_value, 0)); + set_status(status_value); + return status_value; +} + +// A concrete death test class that forks, then immediately runs the test +// in the child process. +class NoExecDeathTest : public ForkingDeathTest { + public: + NoExecDeathTest(const char* a_statement, const RE* a_regex) : + ForkingDeathTest(a_statement, a_regex) { } + virtual TestRole AssumeRole(); +}; + +// The AssumeRole process for a fork-and-run death test. It implements a +// straightforward fork, with a simple pipe to transmit the status byte. +DeathTest::TestRole NoExecDeathTest::AssumeRole() { + const size_t thread_count = GetThreadCount(); + if (thread_count != 1) { + GTEST_LOG_(WARNING) << DeathTestThreadWarning(thread_count); + } + + int pipe_fd[2]; + GTEST_DEATH_TEST_CHECK_(pipe(pipe_fd) != -1); + + DeathTest::set_last_death_test_message(""); + CaptureStderr(); + // When we fork the process below, the log file buffers are copied, but the + // file descriptors are shared. We flush all log files here so that closing + // the file descriptors in the child process doesn't throw off the + // synchronization between descriptors and buffers in the parent process. + // This is as close to the fork as possible to avoid a race condition in case + // there are multiple threads running before the death test, and another + // thread writes to the log file. + FlushInfoLog(); + + const pid_t child_pid = fork(); + GTEST_DEATH_TEST_CHECK_(child_pid != -1); + set_child_pid(child_pid); + if (child_pid == 0) { + GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[0])); + set_write_fd(pipe_fd[1]); + // Redirects all logging to stderr in the child process to prevent + // concurrent writes to the log files. We capture stderr in the parent + // process and append the child process' output to a log. + LogToStderr(); + // Event forwarding to the listeners of event listener API mush be shut + // down in death test subprocesses. + GetUnitTestImpl()->listeners()->SuppressEventForwarding(); + g_in_fast_death_test_child = true; + return EXECUTE_TEST; + } else { + GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[1])); + set_read_fd(pipe_fd[0]); + set_spawned(true); + return OVERSEE_TEST; + } +} + +// A concrete death test class that forks and re-executes the main +// program from the beginning, with command-line flags set that cause +// only this specific death test to be run. +class ExecDeathTest : public ForkingDeathTest { + public: + ExecDeathTest(const char* a_statement, const RE* a_regex, + const char* file, int line) : + ForkingDeathTest(a_statement, a_regex), file_(file), line_(line) { } + virtual TestRole AssumeRole(); + private: + static ::std::vector GetArgvsForDeathTestChildProcess() { + ::std::vector args = GetInjectableArgvs(); +# if defined(GTEST_EXTRA_DEATH_TEST_COMMAND_LINE_ARGS_) + ::std::vector extra_args = + GTEST_EXTRA_DEATH_TEST_COMMAND_LINE_ARGS_(); + args.insert(args.end(), extra_args.begin(), extra_args.end()); +# endif // defined(GTEST_EXTRA_DEATH_TEST_COMMAND_LINE_ARGS_) + return args; + } + // The name of the file in which the death test is located. + const char* const file_; + // The line number on which the death test is located. + const int line_; +}; + +// Utility class for accumulating command-line arguments. +class Arguments { + public: + Arguments() { + args_.push_back(NULL); + } + + ~Arguments() { + for (std::vector::iterator i = args_.begin(); i != args_.end(); + ++i) { + free(*i); + } + } + void AddArgument(const char* argument) { + args_.insert(args_.end() - 1, posix::StrDup(argument)); + } + + template + void AddArguments(const ::std::vector& arguments) { + for (typename ::std::vector::const_iterator i = arguments.begin(); + i != arguments.end(); + ++i) { + args_.insert(args_.end() - 1, posix::StrDup(i->c_str())); + } + } + char* const* Argv() { + return &args_[0]; + } + + private: + std::vector args_; +}; + +// A struct that encompasses the arguments to the child process of a +// threadsafe-style death test process. +struct ExecDeathTestArgs { + char* const* argv; // Command-line arguments for the child's call to exec + int close_fd; // File descriptor to close; the read end of a pipe +}; + +# if GTEST_OS_MAC +inline char** GetEnviron() { + // When Google Test is built as a framework on MacOS X, the environ variable + // is unavailable. Apple's documentation (man environ) recommends using + // _NSGetEnviron() instead. + return *_NSGetEnviron(); +} +# else +// Some POSIX platforms expect you to declare environ. extern "C" makes +// it reside in the global namespace. +extern "C" char** environ; +inline char** GetEnviron() { return environ; } +# endif // GTEST_OS_MAC + +# if !GTEST_OS_QNX +// The main function for a threadsafe-style death test child process. +// This function is called in a clone()-ed process and thus must avoid +// any potentially unsafe operations like malloc or libc functions. +static int ExecDeathTestChildMain(void* child_arg) { + ExecDeathTestArgs* const args = static_cast(child_arg); + GTEST_DEATH_TEST_CHECK_SYSCALL_(close(args->close_fd)); + + // We need to execute the test program in the same environment where + // it was originally invoked. Therefore we change to the original + // working directory first. + const char* const original_dir = + UnitTest::GetInstance()->original_working_dir(); + // We can safely call chdir() as it's a direct system call. + if (chdir(original_dir) != 0) { + DeathTestAbort(std::string("chdir(\"") + original_dir + "\") failed: " + + GetLastErrnoDescription()); + return EXIT_FAILURE; + } + + // We can safely call execve() as it's a direct system call. We + // cannot use execvp() as it's a libc function and thus potentially + // unsafe. Since execve() doesn't search the PATH, the user must + // invoke the test program via a valid path that contains at least + // one path separator. + execve(args->argv[0], args->argv, GetEnviron()); + DeathTestAbort(std::string("execve(") + args->argv[0] + ", ...) in " + + original_dir + " failed: " + + GetLastErrnoDescription()); + return EXIT_FAILURE; +} +# endif // !GTEST_OS_QNX + +# if GTEST_HAS_CLONE +// Two utility routines that together determine the direction the stack +// grows. +// This could be accomplished more elegantly by a single recursive +// function, but we want to guard against the unlikely possibility of +// a smart compiler optimizing the recursion away. +// +// GTEST_NO_INLINE_ is required to prevent GCC 4.6 from inlining +// StackLowerThanAddress into StackGrowsDown, which then doesn't give +// correct answer. +static void StackLowerThanAddress(const void* ptr, + bool* result) GTEST_NO_INLINE_; +static void StackLowerThanAddress(const void* ptr, bool* result) { + int dummy; + *result = (&dummy < ptr); +} + +// Make sure AddressSanitizer does not tamper with the stack here. +GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +static bool StackGrowsDown() { + int dummy = 0; + bool result; + StackLowerThanAddress(&dummy, &result); + return result; +} +# endif // GTEST_HAS_CLONE + +// Spawns a child process with the same executable as the current process in +// a thread-safe manner and instructs it to run the death test. The +// implementation uses fork(2) + exec. On systems where clone(2) is +// available, it is used instead, being slightly more thread-safe. On QNX, +// fork supports only single-threaded environments, so this function uses +// spawn(2) there instead. The function dies with an error message if +// anything goes wrong. +static pid_t ExecDeathTestSpawnChild(char* const* argv, int close_fd) { + ExecDeathTestArgs args = { argv, close_fd }; + pid_t child_pid = -1; + +# if GTEST_OS_QNX + // Obtains the current directory and sets it to be closed in the child + // process. + const int cwd_fd = open(".", O_RDONLY); + GTEST_DEATH_TEST_CHECK_(cwd_fd != -1); + GTEST_DEATH_TEST_CHECK_SYSCALL_(fcntl(cwd_fd, F_SETFD, FD_CLOEXEC)); + // We need to execute the test program in the same environment where + // it was originally invoked. Therefore we change to the original + // working directory first. + const char* const original_dir = + UnitTest::GetInstance()->original_working_dir(); + // We can safely call chdir() as it's a direct system call. + if (chdir(original_dir) != 0) { + DeathTestAbort(std::string("chdir(\"") + original_dir + "\") failed: " + + GetLastErrnoDescription()); + return EXIT_FAILURE; + } + + int fd_flags; + // Set close_fd to be closed after spawn. + GTEST_DEATH_TEST_CHECK_SYSCALL_(fd_flags = fcntl(close_fd, F_GETFD)); + GTEST_DEATH_TEST_CHECK_SYSCALL_(fcntl(close_fd, F_SETFD, + fd_flags | FD_CLOEXEC)); + struct inheritance inherit = {0}; + // spawn is a system call. + child_pid = spawn(args.argv[0], 0, NULL, &inherit, args.argv, GetEnviron()); + // Restores the current working directory. + GTEST_DEATH_TEST_CHECK_(fchdir(cwd_fd) != -1); + GTEST_DEATH_TEST_CHECK_SYSCALL_(close(cwd_fd)); + +# else // GTEST_OS_QNX +# if GTEST_OS_LINUX + // When a SIGPROF signal is received while fork() or clone() are executing, + // the process may hang. To avoid this, we ignore SIGPROF here and re-enable + // it after the call to fork()/clone() is complete. + struct sigaction saved_sigprof_action; + struct sigaction ignore_sigprof_action; + memset(&ignore_sigprof_action, 0, sizeof(ignore_sigprof_action)); + sigemptyset(&ignore_sigprof_action.sa_mask); + ignore_sigprof_action.sa_handler = SIG_IGN; + GTEST_DEATH_TEST_CHECK_SYSCALL_(sigaction( + SIGPROF, &ignore_sigprof_action, &saved_sigprof_action)); +# endif // GTEST_OS_LINUX + +# if GTEST_HAS_CLONE + const bool use_fork = GTEST_FLAG(death_test_use_fork); + + if (!use_fork) { + static const bool stack_grows_down = StackGrowsDown(); + const size_t stack_size = getpagesize(); + // MMAP_ANONYMOUS is not defined on Mac, so we use MAP_ANON instead. + void* const stack = mmap(NULL, stack_size, PROT_READ | PROT_WRITE, + MAP_ANON | MAP_PRIVATE, -1, 0); + GTEST_DEATH_TEST_CHECK_(stack != MAP_FAILED); + + // Maximum stack alignment in bytes: For a downward-growing stack, this + // amount is subtracted from size of the stack space to get an address + // that is within the stack space and is aligned on all systems we care + // about. As far as I know there is no ABI with stack alignment greater + // than 64. We assume stack and stack_size already have alignment of + // kMaxStackAlignment. + const size_t kMaxStackAlignment = 64; + void* const stack_top = + static_cast(stack) + + (stack_grows_down ? stack_size - kMaxStackAlignment : 0); + GTEST_DEATH_TEST_CHECK_(stack_size > kMaxStackAlignment && + reinterpret_cast(stack_top) % kMaxStackAlignment == 0); + + child_pid = clone(&ExecDeathTestChildMain, stack_top, SIGCHLD, &args); + + GTEST_DEATH_TEST_CHECK_(munmap(stack, stack_size) != -1); + } +# else + const bool use_fork = true; +# endif // GTEST_HAS_CLONE + + if (use_fork && (child_pid = fork()) == 0) { + ExecDeathTestChildMain(&args); + _exit(0); + } +# endif // GTEST_OS_QNX +# if GTEST_OS_LINUX + GTEST_DEATH_TEST_CHECK_SYSCALL_( + sigaction(SIGPROF, &saved_sigprof_action, NULL)); +# endif // GTEST_OS_LINUX + + GTEST_DEATH_TEST_CHECK_(child_pid != -1); + return child_pid; +} + +// The AssumeRole process for a fork-and-exec death test. It re-executes the +// main program from the beginning, setting the --gtest_filter +// and --gtest_internal_run_death_test flags to cause only the current +// death test to be re-run. +DeathTest::TestRole ExecDeathTest::AssumeRole() { + const UnitTestImpl* const impl = GetUnitTestImpl(); + const InternalRunDeathTestFlag* const flag = + impl->internal_run_death_test_flag(); + const TestInfo* const info = impl->current_test_info(); + const int death_test_index = info->result()->death_test_count(); + + if (flag != NULL) { + set_write_fd(flag->write_fd()); + return EXECUTE_TEST; + } + + int pipe_fd[2]; + GTEST_DEATH_TEST_CHECK_(pipe(pipe_fd) != -1); + // Clear the close-on-exec flag on the write end of the pipe, lest + // it be closed when the child process does an exec: + GTEST_DEATH_TEST_CHECK_(fcntl(pipe_fd[1], F_SETFD, 0) != -1); + + const std::string filter_flag = + std::string("--") + GTEST_FLAG_PREFIX_ + kFilterFlag + "=" + + info->test_case_name() + "." + info->name(); + const std::string internal_flag = + std::string("--") + GTEST_FLAG_PREFIX_ + kInternalRunDeathTestFlag + "=" + + file_ + "|" + StreamableToString(line_) + "|" + + StreamableToString(death_test_index) + "|" + + StreamableToString(pipe_fd[1]); + Arguments args; + args.AddArguments(GetArgvsForDeathTestChildProcess()); + args.AddArgument(filter_flag.c_str()); + args.AddArgument(internal_flag.c_str()); + + DeathTest::set_last_death_test_message(""); + + CaptureStderr(); + // See the comment in NoExecDeathTest::AssumeRole for why the next line + // is necessary. + FlushInfoLog(); + + const pid_t child_pid = ExecDeathTestSpawnChild(args.Argv(), pipe_fd[0]); + GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[1])); + set_child_pid(child_pid); + set_read_fd(pipe_fd[0]); + set_spawned(true); + return OVERSEE_TEST; +} + +# endif // !GTEST_OS_WINDOWS + +// Creates a concrete DeathTest-derived class that depends on the +// --gtest_death_test_style flag, and sets the pointer pointed to +// by the "test" argument to its address. If the test should be +// skipped, sets that pointer to NULL. Returns true, unless the +// flag is set to an invalid value. +bool DefaultDeathTestFactory::Create(const char* statement, const RE* regex, + const char* file, int line, + DeathTest** test) { + UnitTestImpl* const impl = GetUnitTestImpl(); + const InternalRunDeathTestFlag* const flag = + impl->internal_run_death_test_flag(); + const int death_test_index = impl->current_test_info() + ->increment_death_test_count(); + + if (flag != NULL) { + if (death_test_index > flag->index()) { + DeathTest::set_last_death_test_message( + "Death test count (" + StreamableToString(death_test_index) + + ") somehow exceeded expected maximum (" + + StreamableToString(flag->index()) + ")"); + return false; + } + + if (!(flag->file() == file && flag->line() == line && + flag->index() == death_test_index)) { + *test = NULL; + return true; + } + } + +# if GTEST_OS_WINDOWS + + if (GTEST_FLAG(death_test_style) == "threadsafe" || + GTEST_FLAG(death_test_style) == "fast") { + *test = new WindowsDeathTest(statement, regex, file, line); + } + +# elif GTEST_OS_FUCHSIA + + if (GTEST_FLAG(death_test_style) == "threadsafe" || + GTEST_FLAG(death_test_style) == "fast") { + *test = new FuchsiaDeathTest(statement, regex, file, line); + } + +# else + + if (GTEST_FLAG(death_test_style) == "threadsafe") { + *test = new ExecDeathTest(statement, regex, file, line); + } else if (GTEST_FLAG(death_test_style) == "fast") { + *test = new NoExecDeathTest(statement, regex); + } + +# endif // GTEST_OS_WINDOWS + + else { // NOLINT - this is more readable than unbalanced brackets inside #if. + DeathTest::set_last_death_test_message( + "Unknown death test style \"" + GTEST_FLAG(death_test_style) + + "\" encountered"); + return false; + } + + return true; +} + +# if GTEST_OS_WINDOWS +// Recreates the pipe and event handles from the provided parameters, +// signals the event, and returns a file descriptor wrapped around the pipe +// handle. This function is called in the child process only. +static int GetStatusFileDescriptor(unsigned int parent_process_id, + size_t write_handle_as_size_t, + size_t event_handle_as_size_t) { + AutoHandle parent_process_handle(::OpenProcess(PROCESS_DUP_HANDLE, + FALSE, // Non-inheritable. + parent_process_id)); + if (parent_process_handle.Get() == INVALID_HANDLE_VALUE) { + DeathTestAbort("Unable to open parent process " + + StreamableToString(parent_process_id)); + } + + // FIXME: Replace the following check with a + // compile-time assertion when available. + GTEST_CHECK_(sizeof(HANDLE) <= sizeof(size_t)); + + const HANDLE write_handle = + reinterpret_cast(write_handle_as_size_t); + HANDLE dup_write_handle; + + // The newly initialized handle is accessible only in the parent + // process. To obtain one accessible within the child, we need to use + // DuplicateHandle. + if (!::DuplicateHandle(parent_process_handle.Get(), write_handle, + ::GetCurrentProcess(), &dup_write_handle, + 0x0, // Requested privileges ignored since + // DUPLICATE_SAME_ACCESS is used. + FALSE, // Request non-inheritable handler. + DUPLICATE_SAME_ACCESS)) { + DeathTestAbort("Unable to duplicate the pipe handle " + + StreamableToString(write_handle_as_size_t) + + " from the parent process " + + StreamableToString(parent_process_id)); + } + + const HANDLE event_handle = reinterpret_cast(event_handle_as_size_t); + HANDLE dup_event_handle; + + if (!::DuplicateHandle(parent_process_handle.Get(), event_handle, + ::GetCurrentProcess(), &dup_event_handle, + 0x0, + FALSE, + DUPLICATE_SAME_ACCESS)) { + DeathTestAbort("Unable to duplicate the event handle " + + StreamableToString(event_handle_as_size_t) + + " from the parent process " + + StreamableToString(parent_process_id)); + } + + const int write_fd = + ::_open_osfhandle(reinterpret_cast(dup_write_handle), O_APPEND); + if (write_fd == -1) { + DeathTestAbort("Unable to convert pipe handle " + + StreamableToString(write_handle_as_size_t) + + " to a file descriptor"); + } + + // Signals the parent that the write end of the pipe has been acquired + // so the parent can release its own write end. + ::SetEvent(dup_event_handle); + + return write_fd; +} +# endif // GTEST_OS_WINDOWS + +// Returns a newly created InternalRunDeathTestFlag object with fields +// initialized from the GTEST_FLAG(internal_run_death_test) flag if +// the flag is specified; otherwise returns NULL. +InternalRunDeathTestFlag* ParseInternalRunDeathTestFlag() { + if (GTEST_FLAG(internal_run_death_test) == "") return NULL; + + // GTEST_HAS_DEATH_TEST implies that we have ::std::string, so we + // can use it here. + int line = -1; + int index = -1; + ::std::vector< ::std::string> fields; + SplitString(GTEST_FLAG(internal_run_death_test).c_str(), '|', &fields); + int write_fd = -1; + +# if GTEST_OS_WINDOWS + + unsigned int parent_process_id = 0; + size_t write_handle_as_size_t = 0; + size_t event_handle_as_size_t = 0; + + if (fields.size() != 6 + || !ParseNaturalNumber(fields[1], &line) + || !ParseNaturalNumber(fields[2], &index) + || !ParseNaturalNumber(fields[3], &parent_process_id) + || !ParseNaturalNumber(fields[4], &write_handle_as_size_t) + || !ParseNaturalNumber(fields[5], &event_handle_as_size_t)) { + DeathTestAbort("Bad --gtest_internal_run_death_test flag: " + + GTEST_FLAG(internal_run_death_test)); + } + write_fd = GetStatusFileDescriptor(parent_process_id, + write_handle_as_size_t, + event_handle_as_size_t); + +# elif GTEST_OS_FUCHSIA + + if (fields.size() != 3 + || !ParseNaturalNumber(fields[1], &line) + || !ParseNaturalNumber(fields[2], &index)) { + DeathTestAbort("Bad --gtest_internal_run_death_test flag: " + + GTEST_FLAG(internal_run_death_test)); + } + +# else + + if (fields.size() != 4 + || !ParseNaturalNumber(fields[1], &line) + || !ParseNaturalNumber(fields[2], &index) + || !ParseNaturalNumber(fields[3], &write_fd)) { + DeathTestAbort("Bad --gtest_internal_run_death_test flag: " + + GTEST_FLAG(internal_run_death_test)); + } + +# endif // GTEST_OS_WINDOWS + + return new InternalRunDeathTestFlag(fields[0], line, index, write_fd); +} + +} // namespace internal + +#endif // GTEST_HAS_DEATH_TEST + +} // namespace testing +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +#include + +#if GTEST_OS_WINDOWS_MOBILE +# include +#elif GTEST_OS_WINDOWS +# include +# include +#elif GTEST_OS_SYMBIAN +// Symbian OpenC has PATH_MAX in sys/syslimits.h +# include +#else +# include +# include // Some Linux distributions define PATH_MAX here. +#endif // GTEST_OS_WINDOWS_MOBILE + + +#if GTEST_OS_WINDOWS +# define GTEST_PATH_MAX_ _MAX_PATH +#elif defined(PATH_MAX) +# define GTEST_PATH_MAX_ PATH_MAX +#elif defined(_XOPEN_PATH_MAX) +# define GTEST_PATH_MAX_ _XOPEN_PATH_MAX +#else +# define GTEST_PATH_MAX_ _POSIX_PATH_MAX +#endif // GTEST_OS_WINDOWS + +namespace testing { +namespace internal { + +#if GTEST_OS_WINDOWS +// On Windows, '\\' is the standard path separator, but many tools and the +// Windows API also accept '/' as an alternate path separator. Unless otherwise +// noted, a file path can contain either kind of path separators, or a mixture +// of them. +const char kPathSeparator = '\\'; +const char kAlternatePathSeparator = '/'; +const char kAlternatePathSeparatorString[] = "/"; +# if GTEST_OS_WINDOWS_MOBILE +// Windows CE doesn't have a current directory. You should not use +// the current directory in tests on Windows CE, but this at least +// provides a reasonable fallback. +const char kCurrentDirectoryString[] = "\\"; +// Windows CE doesn't define INVALID_FILE_ATTRIBUTES +const DWORD kInvalidFileAttributes = 0xffffffff; +# else +const char kCurrentDirectoryString[] = ".\\"; +# endif // GTEST_OS_WINDOWS_MOBILE +#else +const char kPathSeparator = '/'; +const char kCurrentDirectoryString[] = "./"; +#endif // GTEST_OS_WINDOWS + +// Returns whether the given character is a valid path separator. +static bool IsPathSeparator(char c) { +#if GTEST_HAS_ALT_PATH_SEP_ + return (c == kPathSeparator) || (c == kAlternatePathSeparator); +#else + return c == kPathSeparator; +#endif +} + +// Returns the current working directory, or "" if unsuccessful. +FilePath FilePath::GetCurrentDir() { +#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT + // Windows CE doesn't have a current directory, so we just return + // something reasonable. + return FilePath(kCurrentDirectoryString); +#elif GTEST_OS_WINDOWS + char cwd[GTEST_PATH_MAX_ + 1] = { '\0' }; + return FilePath(_getcwd(cwd, sizeof(cwd)) == NULL ? "" : cwd); +#else + char cwd[GTEST_PATH_MAX_ + 1] = { '\0' }; + char* result = getcwd(cwd, sizeof(cwd)); +# if GTEST_OS_NACL + // getcwd will likely fail in NaCl due to the sandbox, so return something + // reasonable. The user may have provided a shim implementation for getcwd, + // however, so fallback only when failure is detected. + return FilePath(result == NULL ? kCurrentDirectoryString : cwd); +# endif // GTEST_OS_NACL + return FilePath(result == NULL ? "" : cwd); +#endif // GTEST_OS_WINDOWS_MOBILE +} + +// Returns a copy of the FilePath with the case-insensitive extension removed. +// Example: FilePath("dir/file.exe").RemoveExtension("EXE") returns +// FilePath("dir/file"). If a case-insensitive extension is not +// found, returns a copy of the original FilePath. +FilePath FilePath::RemoveExtension(const char* extension) const { + const std::string dot_extension = std::string(".") + extension; + if (String::EndsWithCaseInsensitive(pathname_, dot_extension)) { + return FilePath(pathname_.substr( + 0, pathname_.length() - dot_extension.length())); + } + return *this; +} + +// Returns a pointer to the last occurrence of a valid path separator in +// the FilePath. On Windows, for example, both '/' and '\' are valid path +// separators. Returns NULL if no path separator was found. +const char* FilePath::FindLastPathSeparator() const { + const char* const last_sep = strrchr(c_str(), kPathSeparator); +#if GTEST_HAS_ALT_PATH_SEP_ + const char* const last_alt_sep = strrchr(c_str(), kAlternatePathSeparator); + // Comparing two pointers of which only one is NULL is undefined. + if (last_alt_sep != NULL && + (last_sep == NULL || last_alt_sep > last_sep)) { + return last_alt_sep; + } +#endif + return last_sep; +} + +// Returns a copy of the FilePath with the directory part removed. +// Example: FilePath("path/to/file").RemoveDirectoryName() returns +// FilePath("file"). If there is no directory part ("just_a_file"), it returns +// the FilePath unmodified. If there is no file part ("just_a_dir/") it +// returns an empty FilePath (""). +// On Windows platform, '\' is the path separator, otherwise it is '/'. +FilePath FilePath::RemoveDirectoryName() const { + const char* const last_sep = FindLastPathSeparator(); + return last_sep ? FilePath(last_sep + 1) : *this; +} + +// RemoveFileName returns the directory path with the filename removed. +// Example: FilePath("path/to/file").RemoveFileName() returns "path/to/". +// If the FilePath is "a_file" or "/a_file", RemoveFileName returns +// FilePath("./") or, on Windows, FilePath(".\\"). If the filepath does +// not have a file, like "just/a/dir/", it returns the FilePath unmodified. +// On Windows platform, '\' is the path separator, otherwise it is '/'. +FilePath FilePath::RemoveFileName() const { + const char* const last_sep = FindLastPathSeparator(); + std::string dir; + if (last_sep) { + dir = std::string(c_str(), last_sep + 1 - c_str()); + } else { + dir = kCurrentDirectoryString; + } + return FilePath(dir); +} + +// Helper functions for naming files in a directory for xml output. + +// Given directory = "dir", base_name = "test", number = 0, +// extension = "xml", returns "dir/test.xml". If number is greater +// than zero (e.g., 12), returns "dir/test_12.xml". +// On Windows platform, uses \ as the separator rather than /. +FilePath FilePath::MakeFileName(const FilePath& directory, + const FilePath& base_name, + int number, + const char* extension) { + std::string file; + if (number == 0) { + file = base_name.string() + "." + extension; + } else { + file = base_name.string() + "_" + StreamableToString(number) + + "." + extension; + } + return ConcatPaths(directory, FilePath(file)); +} + +// Given directory = "dir", relative_path = "test.xml", returns "dir/test.xml". +// On Windows, uses \ as the separator rather than /. +FilePath FilePath::ConcatPaths(const FilePath& directory, + const FilePath& relative_path) { + if (directory.IsEmpty()) + return relative_path; + const FilePath dir(directory.RemoveTrailingPathSeparator()); + return FilePath(dir.string() + kPathSeparator + relative_path.string()); +} + +// Returns true if pathname describes something findable in the file-system, +// either a file, directory, or whatever. +bool FilePath::FileOrDirectoryExists() const { +#if GTEST_OS_WINDOWS_MOBILE + LPCWSTR unicode = String::AnsiToUtf16(pathname_.c_str()); + const DWORD attributes = GetFileAttributes(unicode); + delete [] unicode; + return attributes != kInvalidFileAttributes; +#else + posix::StatStruct file_stat; + return posix::Stat(pathname_.c_str(), &file_stat) == 0; +#endif // GTEST_OS_WINDOWS_MOBILE +} + +// Returns true if pathname describes a directory in the file-system +// that exists. +bool FilePath::DirectoryExists() const { + bool result = false; +#if GTEST_OS_WINDOWS + // Don't strip off trailing separator if path is a root directory on + // Windows (like "C:\\"). + const FilePath& path(IsRootDirectory() ? *this : + RemoveTrailingPathSeparator()); +#else + const FilePath& path(*this); +#endif + +#if GTEST_OS_WINDOWS_MOBILE + LPCWSTR unicode = String::AnsiToUtf16(path.c_str()); + const DWORD attributes = GetFileAttributes(unicode); + delete [] unicode; + if ((attributes != kInvalidFileAttributes) && + (attributes & FILE_ATTRIBUTE_DIRECTORY)) { + result = true; + } +#else + posix::StatStruct file_stat; + result = posix::Stat(path.c_str(), &file_stat) == 0 && + posix::IsDir(file_stat); +#endif // GTEST_OS_WINDOWS_MOBILE + + return result; +} + +// Returns true if pathname describes a root directory. (Windows has one +// root directory per disk drive.) +bool FilePath::IsRootDirectory() const { +#if GTEST_OS_WINDOWS + // FIXME: on Windows a network share like + // \\server\share can be a root directory, although it cannot be the + // current directory. Handle this properly. + return pathname_.length() == 3 && IsAbsolutePath(); +#else + return pathname_.length() == 1 && IsPathSeparator(pathname_.c_str()[0]); +#endif +} + +// Returns true if pathname describes an absolute path. +bool FilePath::IsAbsolutePath() const { + const char* const name = pathname_.c_str(); +#if GTEST_OS_WINDOWS + return pathname_.length() >= 3 && + ((name[0] >= 'a' && name[0] <= 'z') || + (name[0] >= 'A' && name[0] <= 'Z')) && + name[1] == ':' && + IsPathSeparator(name[2]); +#else + return IsPathSeparator(name[0]); +#endif +} + +// Returns a pathname for a file that does not currently exist. The pathname +// will be directory/base_name.extension or +// directory/base_name_.extension if directory/base_name.extension +// already exists. The number will be incremented until a pathname is found +// that does not already exist. +// Examples: 'dir/foo_test.xml' or 'dir/foo_test_1.xml'. +// There could be a race condition if two or more processes are calling this +// function at the same time -- they could both pick the same filename. +FilePath FilePath::GenerateUniqueFileName(const FilePath& directory, + const FilePath& base_name, + const char* extension) { + FilePath full_pathname; + int number = 0; + do { + full_pathname.Set(MakeFileName(directory, base_name, number++, extension)); + } while (full_pathname.FileOrDirectoryExists()); + return full_pathname; +} + +// Returns true if FilePath ends with a path separator, which indicates that +// it is intended to represent a directory. Returns false otherwise. +// This does NOT check that a directory (or file) actually exists. +bool FilePath::IsDirectory() const { + return !pathname_.empty() && + IsPathSeparator(pathname_.c_str()[pathname_.length() - 1]); +} + +// Create directories so that path exists. Returns true if successful or if +// the directories already exist; returns false if unable to create directories +// for any reason. +bool FilePath::CreateDirectoriesRecursively() const { + if (!this->IsDirectory()) { + return false; + } + + if (pathname_.length() == 0 || this->DirectoryExists()) { + return true; + } + + const FilePath parent(this->RemoveTrailingPathSeparator().RemoveFileName()); + return parent.CreateDirectoriesRecursively() && this->CreateFolder(); +} + +// Create the directory so that path exists. Returns true if successful or +// if the directory already exists; returns false if unable to create the +// directory for any reason, including if the parent directory does not +// exist. Not named "CreateDirectory" because that's a macro on Windows. +bool FilePath::CreateFolder() const { +#if GTEST_OS_WINDOWS_MOBILE + FilePath removed_sep(this->RemoveTrailingPathSeparator()); + LPCWSTR unicode = String::AnsiToUtf16(removed_sep.c_str()); + int result = CreateDirectory(unicode, NULL) ? 0 : -1; + delete [] unicode; +#elif GTEST_OS_WINDOWS + int result = _mkdir(pathname_.c_str()); +#else + int result = mkdir(pathname_.c_str(), 0777); +#endif // GTEST_OS_WINDOWS_MOBILE + + if (result == -1) { + return this->DirectoryExists(); // An error is OK if the directory exists. + } + return true; // No error. +} + +// If input name has a trailing separator character, remove it and return the +// name, otherwise return the name string unmodified. +// On Windows platform, uses \ as the separator, other platforms use /. +FilePath FilePath::RemoveTrailingPathSeparator() const { + return IsDirectory() + ? FilePath(pathname_.substr(0, pathname_.length() - 1)) + : *this; +} + +// Removes any redundant separators that might be in the pathname. +// For example, "bar///foo" becomes "bar/foo". Does not eliminate other +// redundancies that might be in a pathname involving "." or "..". +// FIXME: handle Windows network shares (e.g. \\server\share). +void FilePath::Normalize() { + if (pathname_.c_str() == NULL) { + pathname_ = ""; + return; + } + const char* src = pathname_.c_str(); + char* const dest = new char[pathname_.length() + 1]; + char* dest_ptr = dest; + memset(dest_ptr, 0, pathname_.length() + 1); + + while (*src != '\0') { + *dest_ptr = *src; + if (!IsPathSeparator(*src)) { + src++; + } else { +#if GTEST_HAS_ALT_PATH_SEP_ + if (*dest_ptr == kAlternatePathSeparator) { + *dest_ptr = kPathSeparator; + } +#endif + while (IsPathSeparator(*src)) + src++; + } + dest_ptr++; + } + *dest_ptr = '\0'; + pathname_ = dest; + delete[] dest; +} + +} // namespace internal +} // namespace testing +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +#include +#include +#include +#include +#include + +#if GTEST_OS_WINDOWS +# include +# include +# include +# include // Used in ThreadLocal. +#else +# include +#endif // GTEST_OS_WINDOWS + +#if GTEST_OS_MAC +# include +# include +# include +#endif // GTEST_OS_MAC + +#if GTEST_OS_QNX +# include +# include +# include +#endif // GTEST_OS_QNX + +#if GTEST_OS_AIX +# include +# include +#endif // GTEST_OS_AIX + +#if GTEST_OS_FUCHSIA +# include +# include +#endif // GTEST_OS_FUCHSIA + + +namespace testing { +namespace internal { + +#if defined(_MSC_VER) || defined(__BORLANDC__) +// MSVC and C++Builder do not provide a definition of STDERR_FILENO. +const int kStdOutFileno = 1; +const int kStdErrFileno = 2; +#else +const int kStdOutFileno = STDOUT_FILENO; +const int kStdErrFileno = STDERR_FILENO; +#endif // _MSC_VER + +#if GTEST_OS_LINUX + +namespace { +template +T ReadProcFileField(const std::string& filename, int field) { + std::string dummy; + std::ifstream file(filename.c_str()); + while (field-- > 0) { + file >> dummy; + } + T output = 0; + file >> output; + return output; +} +} // namespace + +// Returns the number of active threads, or 0 when there is an error. +size_t GetThreadCount() { + const std::string filename = + (Message() << "/proc/" << getpid() << "/stat").GetString(); + return ReadProcFileField(filename, 19); +} + +#elif GTEST_OS_MAC + +size_t GetThreadCount() { + const task_t task = mach_task_self(); + mach_msg_type_number_t thread_count; + thread_act_array_t thread_list; + const kern_return_t status = task_threads(task, &thread_list, &thread_count); + if (status == KERN_SUCCESS) { + // task_threads allocates resources in thread_list and we need to free them + // to avoid leaks. + vm_deallocate(task, + reinterpret_cast(thread_list), + sizeof(thread_t) * thread_count); + return static_cast(thread_count); + } else { + return 0; + } +} + +#elif GTEST_OS_QNX + +// Returns the number of threads running in the process, or 0 to indicate that +// we cannot detect it. +size_t GetThreadCount() { + const int fd = open("/proc/self/as", O_RDONLY); + if (fd < 0) { + return 0; + } + procfs_info process_info; + const int status = + devctl(fd, DCMD_PROC_INFO, &process_info, sizeof(process_info), NULL); + close(fd); + if (status == EOK) { + return static_cast(process_info.num_threads); + } else { + return 0; + } +} + +#elif GTEST_OS_AIX + +size_t GetThreadCount() { + struct procentry64 entry; + pid_t pid = getpid(); + int status = getprocs64(&entry, sizeof(entry), NULL, 0, &pid, 1); + if (status == 1) { + return entry.pi_thcount; + } else { + return 0; + } +} + +#elif GTEST_OS_FUCHSIA + +size_t GetThreadCount() { + int dummy_buffer; + size_t avail; + zx_status_t status = zx_object_get_info( + zx_process_self(), + ZX_INFO_PROCESS_THREADS, + &dummy_buffer, + 0, + nullptr, + &avail); + if (status == ZX_OK) { + return avail; + } else { + return 0; + } +} + +#else + +size_t GetThreadCount() { + // There's no portable way to detect the number of threads, so we just + // return 0 to indicate that we cannot detect it. + return 0; +} + +#endif // GTEST_OS_LINUX + +#if GTEST_IS_THREADSAFE && GTEST_OS_WINDOWS + +void SleepMilliseconds(int n) { + ::Sleep(n); +} + +AutoHandle::AutoHandle() + : handle_(INVALID_HANDLE_VALUE) {} + +AutoHandle::AutoHandle(Handle handle) + : handle_(handle) {} + +AutoHandle::~AutoHandle() { + Reset(); +} + +AutoHandle::Handle AutoHandle::Get() const { + return handle_; +} + +void AutoHandle::Reset() { + Reset(INVALID_HANDLE_VALUE); +} + +void AutoHandle::Reset(HANDLE handle) { + // Resetting with the same handle we already own is invalid. + if (handle_ != handle) { + if (IsCloseable()) { + ::CloseHandle(handle_); + } + handle_ = handle; + } else { + GTEST_CHECK_(!IsCloseable()) + << "Resetting a valid handle to itself is likely a programmer error " + "and thus not allowed."; + } +} + +bool AutoHandle::IsCloseable() const { + // Different Windows APIs may use either of these values to represent an + // invalid handle. + return handle_ != NULL && handle_ != INVALID_HANDLE_VALUE; +} + +Notification::Notification() + : event_(::CreateEvent(NULL, // Default security attributes. + TRUE, // Do not reset automatically. + FALSE, // Initially unset. + NULL)) { // Anonymous event. + GTEST_CHECK_(event_.Get() != NULL); +} + +void Notification::Notify() { + GTEST_CHECK_(::SetEvent(event_.Get()) != FALSE); +} + +void Notification::WaitForNotification() { + GTEST_CHECK_( + ::WaitForSingleObject(event_.Get(), INFINITE) == WAIT_OBJECT_0); +} + +Mutex::Mutex() + : owner_thread_id_(0), + type_(kDynamic), + critical_section_init_phase_(0), + critical_section_(new CRITICAL_SECTION) { + ::InitializeCriticalSection(critical_section_); +} + +Mutex::~Mutex() { + // Static mutexes are leaked intentionally. It is not thread-safe to try + // to clean them up. + // FIXME: Switch to Slim Reader/Writer (SRW) Locks, which requires + // nothing to clean it up but is available only on Vista and later. + // https://docs.microsoft.com/en-us/windows/desktop/Sync/slim-reader-writer--srw--locks + if (type_ == kDynamic) { + ::DeleteCriticalSection(critical_section_); + delete critical_section_; + critical_section_ = NULL; + } +} + +void Mutex::Lock() { + ThreadSafeLazyInit(); + ::EnterCriticalSection(critical_section_); + owner_thread_id_ = ::GetCurrentThreadId(); +} + +void Mutex::Unlock() { + ThreadSafeLazyInit(); + // We don't protect writing to owner_thread_id_ here, as it's the + // caller's responsibility to ensure that the current thread holds the + // mutex when this is called. + owner_thread_id_ = 0; + ::LeaveCriticalSection(critical_section_); +} + +// Does nothing if the current thread holds the mutex. Otherwise, crashes +// with high probability. +void Mutex::AssertHeld() { + ThreadSafeLazyInit(); + GTEST_CHECK_(owner_thread_id_ == ::GetCurrentThreadId()) + << "The current thread is not holding the mutex @" << this; +} + +namespace { + +// Use the RAII idiom to flag mem allocs that are intentionally never +// deallocated. The motivation is to silence the false positive mem leaks +// that are reported by the debug version of MS's CRT which can only detect +// if an alloc is missing a matching deallocation. +// Example: +// MemoryIsNotDeallocated memory_is_not_deallocated; +// critical_section_ = new CRITICAL_SECTION; +// +class MemoryIsNotDeallocated +{ + public: + MemoryIsNotDeallocated() : old_crtdbg_flag_(0) { +#ifdef _MSC_VER + old_crtdbg_flag_ = _CrtSetDbgFlag(_CRTDBG_REPORT_FLAG); + // Set heap allocation block type to _IGNORE_BLOCK so that MS debug CRT + // doesn't report mem leak if there's no matching deallocation. + _CrtSetDbgFlag(old_crtdbg_flag_ & ~_CRTDBG_ALLOC_MEM_DF); +#endif // _MSC_VER + } + + ~MemoryIsNotDeallocated() { +#ifdef _MSC_VER + // Restore the original _CRTDBG_ALLOC_MEM_DF flag + _CrtSetDbgFlag(old_crtdbg_flag_); +#endif // _MSC_VER + } + + private: + int old_crtdbg_flag_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(MemoryIsNotDeallocated); +}; + +} // namespace + +// Initializes owner_thread_id_ and critical_section_ in static mutexes. +void Mutex::ThreadSafeLazyInit() { + // Dynamic mutexes are initialized in the constructor. + if (type_ == kStatic) { + switch ( + ::InterlockedCompareExchange(&critical_section_init_phase_, 1L, 0L)) { + case 0: + // If critical_section_init_phase_ was 0 before the exchange, we + // are the first to test it and need to perform the initialization. + owner_thread_id_ = 0; + { + // Use RAII to flag that following mem alloc is never deallocated. + MemoryIsNotDeallocated memory_is_not_deallocated; + critical_section_ = new CRITICAL_SECTION; + } + ::InitializeCriticalSection(critical_section_); + // Updates the critical_section_init_phase_ to 2 to signal + // initialization complete. + GTEST_CHECK_(::InterlockedCompareExchange( + &critical_section_init_phase_, 2L, 1L) == + 1L); + break; + case 1: + // Somebody else is already initializing the mutex; spin until they + // are done. + while (::InterlockedCompareExchange(&critical_section_init_phase_, + 2L, + 2L) != 2L) { + // Possibly yields the rest of the thread's time slice to other + // threads. + ::Sleep(0); + } + break; + + case 2: + break; // The mutex is already initialized and ready for use. + + default: + GTEST_CHECK_(false) + << "Unexpected value of critical_section_init_phase_ " + << "while initializing a static mutex."; + } + } +} + +namespace { + +class ThreadWithParamSupport : public ThreadWithParamBase { + public: + static HANDLE CreateThread(Runnable* runnable, + Notification* thread_can_start) { + ThreadMainParam* param = new ThreadMainParam(runnable, thread_can_start); + DWORD thread_id; + // FIXME: Consider to use _beginthreadex instead. + HANDLE thread_handle = ::CreateThread( + NULL, // Default security. + 0, // Default stack size. + &ThreadWithParamSupport::ThreadMain, + param, // Parameter to ThreadMainStatic + 0x0, // Default creation flags. + &thread_id); // Need a valid pointer for the call to work under Win98. + GTEST_CHECK_(thread_handle != NULL) << "CreateThread failed with error " + << ::GetLastError() << "."; + if (thread_handle == NULL) { + delete param; + } + return thread_handle; + } + + private: + struct ThreadMainParam { + ThreadMainParam(Runnable* runnable, Notification* thread_can_start) + : runnable_(runnable), + thread_can_start_(thread_can_start) { + } + scoped_ptr runnable_; + // Does not own. + Notification* thread_can_start_; + }; + + static DWORD WINAPI ThreadMain(void* ptr) { + // Transfers ownership. + scoped_ptr param(static_cast(ptr)); + if (param->thread_can_start_ != NULL) + param->thread_can_start_->WaitForNotification(); + param->runnable_->Run(); + return 0; + } + + // Prohibit instantiation. + ThreadWithParamSupport(); + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParamSupport); +}; + +} // namespace + +ThreadWithParamBase::ThreadWithParamBase(Runnable *runnable, + Notification* thread_can_start) + : thread_(ThreadWithParamSupport::CreateThread(runnable, + thread_can_start)) { +} + +ThreadWithParamBase::~ThreadWithParamBase() { + Join(); +} + +void ThreadWithParamBase::Join() { + GTEST_CHECK_(::WaitForSingleObject(thread_.Get(), INFINITE) == WAIT_OBJECT_0) + << "Failed to join the thread with error " << ::GetLastError() << "."; +} + +// Maps a thread to a set of ThreadIdToThreadLocals that have values +// instantiated on that thread and notifies them when the thread exits. A +// ThreadLocal instance is expected to persist until all threads it has +// values on have terminated. +class ThreadLocalRegistryImpl { + public: + // Registers thread_local_instance as having value on the current thread. + // Returns a value that can be used to identify the thread from other threads. + static ThreadLocalValueHolderBase* GetValueOnCurrentThread( + const ThreadLocalBase* thread_local_instance) { + DWORD current_thread = ::GetCurrentThreadId(); + MutexLock lock(&mutex_); + ThreadIdToThreadLocals* const thread_to_thread_locals = + GetThreadLocalsMapLocked(); + ThreadIdToThreadLocals::iterator thread_local_pos = + thread_to_thread_locals->find(current_thread); + if (thread_local_pos == thread_to_thread_locals->end()) { + thread_local_pos = thread_to_thread_locals->insert( + std::make_pair(current_thread, ThreadLocalValues())).first; + StartWatcherThreadFor(current_thread); + } + ThreadLocalValues& thread_local_values = thread_local_pos->second; + ThreadLocalValues::iterator value_pos = + thread_local_values.find(thread_local_instance); + if (value_pos == thread_local_values.end()) { + value_pos = + thread_local_values + .insert(std::make_pair( + thread_local_instance, + linked_ptr( + thread_local_instance->NewValueForCurrentThread()))) + .first; + } + return value_pos->second.get(); + } + + static void OnThreadLocalDestroyed( + const ThreadLocalBase* thread_local_instance) { + std::vector > value_holders; + // Clean up the ThreadLocalValues data structure while holding the lock, but + // defer the destruction of the ThreadLocalValueHolderBases. + { + MutexLock lock(&mutex_); + ThreadIdToThreadLocals* const thread_to_thread_locals = + GetThreadLocalsMapLocked(); + for (ThreadIdToThreadLocals::iterator it = + thread_to_thread_locals->begin(); + it != thread_to_thread_locals->end(); + ++it) { + ThreadLocalValues& thread_local_values = it->second; + ThreadLocalValues::iterator value_pos = + thread_local_values.find(thread_local_instance); + if (value_pos != thread_local_values.end()) { + value_holders.push_back(value_pos->second); + thread_local_values.erase(value_pos); + // This 'if' can only be successful at most once, so theoretically we + // could break out of the loop here, but we don't bother doing so. + } + } + } + // Outside the lock, let the destructor for 'value_holders' deallocate the + // ThreadLocalValueHolderBases. + } + + static void OnThreadExit(DWORD thread_id) { + GTEST_CHECK_(thread_id != 0) << ::GetLastError(); + std::vector > value_holders; + // Clean up the ThreadIdToThreadLocals data structure while holding the + // lock, but defer the destruction of the ThreadLocalValueHolderBases. + { + MutexLock lock(&mutex_); + ThreadIdToThreadLocals* const thread_to_thread_locals = + GetThreadLocalsMapLocked(); + ThreadIdToThreadLocals::iterator thread_local_pos = + thread_to_thread_locals->find(thread_id); + if (thread_local_pos != thread_to_thread_locals->end()) { + ThreadLocalValues& thread_local_values = thread_local_pos->second; + for (ThreadLocalValues::iterator value_pos = + thread_local_values.begin(); + value_pos != thread_local_values.end(); + ++value_pos) { + value_holders.push_back(value_pos->second); + } + thread_to_thread_locals->erase(thread_local_pos); + } + } + // Outside the lock, let the destructor for 'value_holders' deallocate the + // ThreadLocalValueHolderBases. + } + + private: + // In a particular thread, maps a ThreadLocal object to its value. + typedef std::map > ThreadLocalValues; + // Stores all ThreadIdToThreadLocals having values in a thread, indexed by + // thread's ID. + typedef std::map ThreadIdToThreadLocals; + + // Holds the thread id and thread handle that we pass from + // StartWatcherThreadFor to WatcherThreadFunc. + typedef std::pair ThreadIdAndHandle; + + static void StartWatcherThreadFor(DWORD thread_id) { + // The returned handle will be kept in thread_map and closed by + // watcher_thread in WatcherThreadFunc. + HANDLE thread = ::OpenThread(SYNCHRONIZE | THREAD_QUERY_INFORMATION, + FALSE, + thread_id); + GTEST_CHECK_(thread != NULL); + // We need to pass a valid thread ID pointer into CreateThread for it + // to work correctly under Win98. + DWORD watcher_thread_id; + HANDLE watcher_thread = ::CreateThread( + NULL, // Default security. + 0, // Default stack size + &ThreadLocalRegistryImpl::WatcherThreadFunc, + reinterpret_cast(new ThreadIdAndHandle(thread_id, thread)), + CREATE_SUSPENDED, + &watcher_thread_id); + GTEST_CHECK_(watcher_thread != NULL); + // Give the watcher thread the same priority as ours to avoid being + // blocked by it. + ::SetThreadPriority(watcher_thread, + ::GetThreadPriority(::GetCurrentThread())); + ::ResumeThread(watcher_thread); + ::CloseHandle(watcher_thread); + } + + // Monitors exit from a given thread and notifies those + // ThreadIdToThreadLocals about thread termination. + static DWORD WINAPI WatcherThreadFunc(LPVOID param) { + const ThreadIdAndHandle* tah = + reinterpret_cast(param); + GTEST_CHECK_( + ::WaitForSingleObject(tah->second, INFINITE) == WAIT_OBJECT_0); + OnThreadExit(tah->first); + ::CloseHandle(tah->second); + delete tah; + return 0; + } + + // Returns map of thread local instances. + static ThreadIdToThreadLocals* GetThreadLocalsMapLocked() { + mutex_.AssertHeld(); + MemoryIsNotDeallocated memory_is_not_deallocated; + static ThreadIdToThreadLocals* map = new ThreadIdToThreadLocals(); + return map; + } + + // Protects access to GetThreadLocalsMapLocked() and its return value. + static Mutex mutex_; + // Protects access to GetThreadMapLocked() and its return value. + static Mutex thread_map_mutex_; +}; + +Mutex ThreadLocalRegistryImpl::mutex_(Mutex::kStaticMutex); +Mutex ThreadLocalRegistryImpl::thread_map_mutex_(Mutex::kStaticMutex); + +ThreadLocalValueHolderBase* ThreadLocalRegistry::GetValueOnCurrentThread( + const ThreadLocalBase* thread_local_instance) { + return ThreadLocalRegistryImpl::GetValueOnCurrentThread( + thread_local_instance); +} + +void ThreadLocalRegistry::OnThreadLocalDestroyed( + const ThreadLocalBase* thread_local_instance) { + ThreadLocalRegistryImpl::OnThreadLocalDestroyed(thread_local_instance); +} + +#endif // GTEST_IS_THREADSAFE && GTEST_OS_WINDOWS + +#if GTEST_USES_POSIX_RE + +// Implements RE. Currently only needed for death tests. + +RE::~RE() { + if (is_valid_) { + // regfree'ing an invalid regex might crash because the content + // of the regex is undefined. Since the regex's are essentially + // the same, one cannot be valid (or invalid) without the other + // being so too. + regfree(&partial_regex_); + regfree(&full_regex_); + } + free(const_cast(pattern_)); +} + +// Returns true iff regular expression re matches the entire str. +bool RE::FullMatch(const char* str, const RE& re) { + if (!re.is_valid_) return false; + + regmatch_t match; + return regexec(&re.full_regex_, str, 1, &match, 0) == 0; +} + +// Returns true iff regular expression re matches a substring of str +// (including str itself). +bool RE::PartialMatch(const char* str, const RE& re) { + if (!re.is_valid_) return false; + + regmatch_t match; + return regexec(&re.partial_regex_, str, 1, &match, 0) == 0; +} + +// Initializes an RE from its string representation. +void RE::Init(const char* regex) { + pattern_ = posix::StrDup(regex); + + // Reserves enough bytes to hold the regular expression used for a + // full match. + const size_t full_regex_len = strlen(regex) + 10; + char* const full_pattern = new char[full_regex_len]; + + snprintf(full_pattern, full_regex_len, "^(%s)$", regex); + is_valid_ = regcomp(&full_regex_, full_pattern, REG_EXTENDED) == 0; + // We want to call regcomp(&partial_regex_, ...) even if the + // previous expression returns false. Otherwise partial_regex_ may + // not be properly initialized can may cause trouble when it's + // freed. + // + // Some implementation of POSIX regex (e.g. on at least some + // versions of Cygwin) doesn't accept the empty string as a valid + // regex. We change it to an equivalent form "()" to be safe. + if (is_valid_) { + const char* const partial_regex = (*regex == '\0') ? "()" : regex; + is_valid_ = regcomp(&partial_regex_, partial_regex, REG_EXTENDED) == 0; + } + EXPECT_TRUE(is_valid_) + << "Regular expression \"" << regex + << "\" is not a valid POSIX Extended regular expression."; + + delete[] full_pattern; +} + +#elif GTEST_USES_SIMPLE_RE + +// Returns true iff ch appears anywhere in str (excluding the +// terminating '\0' character). +bool IsInSet(char ch, const char* str) { + return ch != '\0' && strchr(str, ch) != NULL; +} + +// Returns true iff ch belongs to the given classification. Unlike +// similar functions in , these aren't affected by the +// current locale. +bool IsAsciiDigit(char ch) { return '0' <= ch && ch <= '9'; } +bool IsAsciiPunct(char ch) { + return IsInSet(ch, "^-!\"#$%&'()*+,./:;<=>?@[\\]_`{|}~"); +} +bool IsRepeat(char ch) { return IsInSet(ch, "?*+"); } +bool IsAsciiWhiteSpace(char ch) { return IsInSet(ch, " \f\n\r\t\v"); } +bool IsAsciiWordChar(char ch) { + return ('a' <= ch && ch <= 'z') || ('A' <= ch && ch <= 'Z') || + ('0' <= ch && ch <= '9') || ch == '_'; +} + +// Returns true iff "\\c" is a supported escape sequence. +bool IsValidEscape(char c) { + return (IsAsciiPunct(c) || IsInSet(c, "dDfnrsStvwW")); +} + +// Returns true iff the given atom (specified by escaped and pattern) +// matches ch. The result is undefined if the atom is invalid. +bool AtomMatchesChar(bool escaped, char pattern_char, char ch) { + if (escaped) { // "\\p" where p is pattern_char. + switch (pattern_char) { + case 'd': return IsAsciiDigit(ch); + case 'D': return !IsAsciiDigit(ch); + case 'f': return ch == '\f'; + case 'n': return ch == '\n'; + case 'r': return ch == '\r'; + case 's': return IsAsciiWhiteSpace(ch); + case 'S': return !IsAsciiWhiteSpace(ch); + case 't': return ch == '\t'; + case 'v': return ch == '\v'; + case 'w': return IsAsciiWordChar(ch); + case 'W': return !IsAsciiWordChar(ch); + } + return IsAsciiPunct(pattern_char) && pattern_char == ch; + } + + return (pattern_char == '.' && ch != '\n') || pattern_char == ch; +} + +// Helper function used by ValidateRegex() to format error messages. +static std::string FormatRegexSyntaxError(const char* regex, int index) { + return (Message() << "Syntax error at index " << index + << " in simple regular expression \"" << regex << "\": ").GetString(); +} + +// Generates non-fatal failures and returns false if regex is invalid; +// otherwise returns true. +bool ValidateRegex(const char* regex) { + if (regex == NULL) { + // FIXME: fix the source file location in the + // assertion failures to match where the regex is used in user + // code. + ADD_FAILURE() << "NULL is not a valid simple regular expression."; + return false; + } + + bool is_valid = true; + + // True iff ?, *, or + can follow the previous atom. + bool prev_repeatable = false; + for (int i = 0; regex[i]; i++) { + if (regex[i] == '\\') { // An escape sequence + i++; + if (regex[i] == '\0') { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i - 1) + << "'\\' cannot appear at the end."; + return false; + } + + if (!IsValidEscape(regex[i])) { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i - 1) + << "invalid escape sequence \"\\" << regex[i] << "\"."; + is_valid = false; + } + prev_repeatable = true; + } else { // Not an escape sequence. + const char ch = regex[i]; + + if (ch == '^' && i > 0) { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i) + << "'^' can only appear at the beginning."; + is_valid = false; + } else if (ch == '$' && regex[i + 1] != '\0') { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i) + << "'$' can only appear at the end."; + is_valid = false; + } else if (IsInSet(ch, "()[]{}|")) { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i) + << "'" << ch << "' is unsupported."; + is_valid = false; + } else if (IsRepeat(ch) && !prev_repeatable) { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i) + << "'" << ch << "' can only follow a repeatable token."; + is_valid = false; + } + + prev_repeatable = !IsInSet(ch, "^$?*+"); + } + } + + return is_valid; +} + +// Matches a repeated regex atom followed by a valid simple regular +// expression. The regex atom is defined as c if escaped is false, +// or \c otherwise. repeat is the repetition meta character (?, *, +// or +). The behavior is undefined if str contains too many +// characters to be indexable by size_t, in which case the test will +// probably time out anyway. We are fine with this limitation as +// std::string has it too. +bool MatchRepetitionAndRegexAtHead( + bool escaped, char c, char repeat, const char* regex, + const char* str) { + const size_t min_count = (repeat == '+') ? 1 : 0; + const size_t max_count = (repeat == '?') ? 1 : + static_cast(-1) - 1; + // We cannot call numeric_limits::max() as it conflicts with the + // max() macro on Windows. + + for (size_t i = 0; i <= max_count; ++i) { + // We know that the atom matches each of the first i characters in str. + if (i >= min_count && MatchRegexAtHead(regex, str + i)) { + // We have enough matches at the head, and the tail matches too. + // Since we only care about *whether* the pattern matches str + // (as opposed to *how* it matches), there is no need to find a + // greedy match. + return true; + } + if (str[i] == '\0' || !AtomMatchesChar(escaped, c, str[i])) + return false; + } + return false; +} + +// Returns true iff regex matches a prefix of str. regex must be a +// valid simple regular expression and not start with "^", or the +// result is undefined. +bool MatchRegexAtHead(const char* regex, const char* str) { + if (*regex == '\0') // An empty regex matches a prefix of anything. + return true; + + // "$" only matches the end of a string. Note that regex being + // valid guarantees that there's nothing after "$" in it. + if (*regex == '$') + return *str == '\0'; + + // Is the first thing in regex an escape sequence? + const bool escaped = *regex == '\\'; + if (escaped) + ++regex; + if (IsRepeat(regex[1])) { + // MatchRepetitionAndRegexAtHead() calls MatchRegexAtHead(), so + // here's an indirect recursion. It terminates as the regex gets + // shorter in each recursion. + return MatchRepetitionAndRegexAtHead( + escaped, regex[0], regex[1], regex + 2, str); + } else { + // regex isn't empty, isn't "$", and doesn't start with a + // repetition. We match the first atom of regex with the first + // character of str and recurse. + return (*str != '\0') && AtomMatchesChar(escaped, *regex, *str) && + MatchRegexAtHead(regex + 1, str + 1); + } +} + +// Returns true iff regex matches any substring of str. regex must be +// a valid simple regular expression, or the result is undefined. +// +// The algorithm is recursive, but the recursion depth doesn't exceed +// the regex length, so we won't need to worry about running out of +// stack space normally. In rare cases the time complexity can be +// exponential with respect to the regex length + the string length, +// but usually it's must faster (often close to linear). +bool MatchRegexAnywhere(const char* regex, const char* str) { + if (regex == NULL || str == NULL) + return false; + + if (*regex == '^') + return MatchRegexAtHead(regex + 1, str); + + // A successful match can be anywhere in str. + do { + if (MatchRegexAtHead(regex, str)) + return true; + } while (*str++ != '\0'); + return false; +} + +// Implements the RE class. + +RE::~RE() { + free(const_cast(pattern_)); + free(const_cast(full_pattern_)); +} + +// Returns true iff regular expression re matches the entire str. +bool RE::FullMatch(const char* str, const RE& re) { + return re.is_valid_ && MatchRegexAnywhere(re.full_pattern_, str); +} + +// Returns true iff regular expression re matches a substring of str +// (including str itself). +bool RE::PartialMatch(const char* str, const RE& re) { + return re.is_valid_ && MatchRegexAnywhere(re.pattern_, str); +} + +// Initializes an RE from its string representation. +void RE::Init(const char* regex) { + pattern_ = full_pattern_ = NULL; + if (regex != NULL) { + pattern_ = posix::StrDup(regex); + } + + is_valid_ = ValidateRegex(regex); + if (!is_valid_) { + // No need to calculate the full pattern when the regex is invalid. + return; + } + + const size_t len = strlen(regex); + // Reserves enough bytes to hold the regular expression used for a + // full match: we need space to prepend a '^', append a '$', and + // terminate the string with '\0'. + char* buffer = static_cast(malloc(len + 3)); + full_pattern_ = buffer; + + if (*regex != '^') + *buffer++ = '^'; // Makes sure full_pattern_ starts with '^'. + + // We don't use snprintf or strncpy, as they trigger a warning when + // compiled with VC++ 8.0. + memcpy(buffer, regex, len); + buffer += len; + + if (len == 0 || regex[len - 1] != '$') + *buffer++ = '$'; // Makes sure full_pattern_ ends with '$'. + + *buffer = '\0'; +} + +#endif // GTEST_USES_POSIX_RE + +const char kUnknownFile[] = "unknown file"; + +// Formats a source file path and a line number as they would appear +// in an error message from the compiler used to compile this code. +GTEST_API_ ::std::string FormatFileLocation(const char* file, int line) { + const std::string file_name(file == NULL ? kUnknownFile : file); + + if (line < 0) { + return file_name + ":"; + } +#ifdef _MSC_VER + return file_name + "(" + StreamableToString(line) + "):"; +#else + return file_name + ":" + StreamableToString(line) + ":"; +#endif // _MSC_VER +} + +// Formats a file location for compiler-independent XML output. +// Although this function is not platform dependent, we put it next to +// FormatFileLocation in order to contrast the two functions. +// Note that FormatCompilerIndependentFileLocation() does NOT append colon +// to the file location it produces, unlike FormatFileLocation(). +GTEST_API_ ::std::string FormatCompilerIndependentFileLocation( + const char* file, int line) { + const std::string file_name(file == NULL ? kUnknownFile : file); + + if (line < 0) + return file_name; + else + return file_name + ":" + StreamableToString(line); +} + +GTestLog::GTestLog(GTestLogSeverity severity, const char* file, int line) + : severity_(severity) { + const char* const marker = + severity == GTEST_INFO ? "[ INFO ]" : + severity == GTEST_WARNING ? "[WARNING]" : + severity == GTEST_ERROR ? "[ ERROR ]" : "[ FATAL ]"; + GetStream() << ::std::endl << marker << " " + << FormatFileLocation(file, line).c_str() << ": "; +} + +// Flushes the buffers and, if severity is GTEST_FATAL, aborts the program. +GTestLog::~GTestLog() { + GetStream() << ::std::endl; + if (severity_ == GTEST_FATAL) { + fflush(stderr); + posix::Abort(); + } +} + +// Disable Microsoft deprecation warnings for POSIX functions called from +// this class (creat, dup, dup2, and close) +GTEST_DISABLE_MSC_DEPRECATED_PUSH_() + +#if GTEST_HAS_STREAM_REDIRECTION + +// Object that captures an output stream (stdout/stderr). +class CapturedStream { + public: + // The ctor redirects the stream to a temporary file. + explicit CapturedStream(int fd) : fd_(fd), uncaptured_fd_(dup(fd)) { +# if GTEST_OS_WINDOWS + char temp_dir_path[MAX_PATH + 1] = { '\0' }; // NOLINT + char temp_file_path[MAX_PATH + 1] = { '\0' }; // NOLINT + + ::GetTempPathA(sizeof(temp_dir_path), temp_dir_path); + const UINT success = ::GetTempFileNameA(temp_dir_path, + "gtest_redir", + 0, // Generate unique file name. + temp_file_path); + GTEST_CHECK_(success != 0) + << "Unable to create a temporary file in " << temp_dir_path; + const int captured_fd = creat(temp_file_path, _S_IREAD | _S_IWRITE); + GTEST_CHECK_(captured_fd != -1) << "Unable to open temporary file " + << temp_file_path; + filename_ = temp_file_path; +# else + // There's no guarantee that a test has write access to the current + // directory, so we create the temporary file in the /tmp directory + // instead. We use /tmp on most systems, and /sdcard on Android. + // That's because Android doesn't have /tmp. +# if GTEST_OS_LINUX_ANDROID + // Note: Android applications are expected to call the framework's + // Context.getExternalStorageDirectory() method through JNI to get + // the location of the world-writable SD Card directory. However, + // this requires a Context handle, which cannot be retrieved + // globally from native code. Doing so also precludes running the + // code as part of a regular standalone executable, which doesn't + // run in a Dalvik process (e.g. when running it through 'adb shell'). + // + // The location /sdcard is directly accessible from native code + // and is the only location (unofficially) supported by the Android + // team. It's generally a symlink to the real SD Card mount point + // which can be /mnt/sdcard, /mnt/sdcard0, /system/media/sdcard, or + // other OEM-customized locations. Never rely on these, and always + // use /sdcard. + char name_template[] = "/sdcard/gtest_captured_stream.XXXXXX"; +# else + char name_template[] = "/tmp/captured_stream.XXXXXX"; +# endif // GTEST_OS_LINUX_ANDROID + const int captured_fd = mkstemp(name_template); + filename_ = name_template; +# endif // GTEST_OS_WINDOWS + fflush(NULL); + dup2(captured_fd, fd_); + close(captured_fd); + } + + ~CapturedStream() { + remove(filename_.c_str()); + } + + std::string GetCapturedString() { + if (uncaptured_fd_ != -1) { + // Restores the original stream. + fflush(NULL); + dup2(uncaptured_fd_, fd_); + close(uncaptured_fd_); + uncaptured_fd_ = -1; + } + + FILE* const file = posix::FOpen(filename_.c_str(), "r"); + const std::string content = ReadEntireFile(file); + posix::FClose(file); + return content; + } + + private: + const int fd_; // A stream to capture. + int uncaptured_fd_; + // Name of the temporary file holding the stderr output. + ::std::string filename_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(CapturedStream); +}; + +GTEST_DISABLE_MSC_DEPRECATED_POP_() + +static CapturedStream* g_captured_stderr = NULL; +static CapturedStream* g_captured_stdout = NULL; + +// Starts capturing an output stream (stdout/stderr). +static void CaptureStream(int fd, const char* stream_name, + CapturedStream** stream) { + if (*stream != NULL) { + GTEST_LOG_(FATAL) << "Only one " << stream_name + << " capturer can exist at a time."; + } + *stream = new CapturedStream(fd); +} + +// Stops capturing the output stream and returns the captured string. +static std::string GetCapturedStream(CapturedStream** captured_stream) { + const std::string content = (*captured_stream)->GetCapturedString(); + + delete *captured_stream; + *captured_stream = NULL; + + return content; +} + +// Starts capturing stdout. +void CaptureStdout() { + CaptureStream(kStdOutFileno, "stdout", &g_captured_stdout); +} + +// Starts capturing stderr. +void CaptureStderr() { + CaptureStream(kStdErrFileno, "stderr", &g_captured_stderr); +} + +// Stops capturing stdout and returns the captured string. +std::string GetCapturedStdout() { + return GetCapturedStream(&g_captured_stdout); +} + +// Stops capturing stderr and returns the captured string. +std::string GetCapturedStderr() { + return GetCapturedStream(&g_captured_stderr); +} + +#endif // GTEST_HAS_STREAM_REDIRECTION + + + + + +size_t GetFileSize(FILE* file) { + fseek(file, 0, SEEK_END); + return static_cast(ftell(file)); +} + +std::string ReadEntireFile(FILE* file) { + const size_t file_size = GetFileSize(file); + char* const buffer = new char[file_size]; + + size_t bytes_last_read = 0; // # of bytes read in the last fread() + size_t bytes_read = 0; // # of bytes read so far + + fseek(file, 0, SEEK_SET); + + // Keeps reading the file until we cannot read further or the + // pre-determined file size is reached. + do { + bytes_last_read = fread(buffer+bytes_read, 1, file_size-bytes_read, file); + bytes_read += bytes_last_read; + } while (bytes_last_read > 0 && bytes_read < file_size); + + const std::string content(buffer, bytes_read); + delete[] buffer; + + return content; +} + +#if GTEST_HAS_DEATH_TEST +static const std::vector* g_injected_test_argvs = NULL; // Owned. + +std::vector GetInjectableArgvs() { + if (g_injected_test_argvs != NULL) { + return *g_injected_test_argvs; + } + return GetArgvs(); +} + +void SetInjectableArgvs(const std::vector* new_argvs) { + if (g_injected_test_argvs != new_argvs) delete g_injected_test_argvs; + g_injected_test_argvs = new_argvs; +} + +void SetInjectableArgvs(const std::vector& new_argvs) { + SetInjectableArgvs( + new std::vector(new_argvs.begin(), new_argvs.end())); +} + +#if GTEST_HAS_GLOBAL_STRING +void SetInjectableArgvs(const std::vector< ::string>& new_argvs) { + SetInjectableArgvs( + new std::vector(new_argvs.begin(), new_argvs.end())); +} +#endif // GTEST_HAS_GLOBAL_STRING + +void ClearInjectableArgvs() { + delete g_injected_test_argvs; + g_injected_test_argvs = NULL; +} +#endif // GTEST_HAS_DEATH_TEST + +#if GTEST_OS_WINDOWS_MOBILE +namespace posix { +void Abort() { + DebugBreak(); + TerminateProcess(GetCurrentProcess(), 1); +} +} // namespace posix +#endif // GTEST_OS_WINDOWS_MOBILE + +// Returns the name of the environment variable corresponding to the +// given flag. For example, FlagToEnvVar("foo") will return +// "GTEST_FOO" in the open-source version. +static std::string FlagToEnvVar(const char* flag) { + const std::string full_flag = + (Message() << GTEST_FLAG_PREFIX_ << flag).GetString(); + + Message env_var; + for (size_t i = 0; i != full_flag.length(); i++) { + env_var << ToUpper(full_flag.c_str()[i]); + } + + return env_var.GetString(); +} + +// Parses 'str' for a 32-bit signed integer. If successful, writes +// the result to *value and returns true; otherwise leaves *value +// unchanged and returns false. +bool ParseInt32(const Message& src_text, const char* str, Int32* value) { + // Parses the environment variable as a decimal integer. + char* end = NULL; + const long long_value = strtol(str, &end, 10); // NOLINT + + // Has strtol() consumed all characters in the string? + if (*end != '\0') { + // No - an invalid character was encountered. + Message msg; + msg << "WARNING: " << src_text + << " is expected to be a 32-bit integer, but actually" + << " has value \"" << str << "\".\n"; + printf("%s", msg.GetString().c_str()); + fflush(stdout); + return false; + } + + // Is the parsed value in the range of an Int32? + const Int32 result = static_cast(long_value); + if (long_value == LONG_MAX || long_value == LONG_MIN || + // The parsed value overflows as a long. (strtol() returns + // LONG_MAX or LONG_MIN when the input overflows.) + result != long_value + // The parsed value overflows as an Int32. + ) { + Message msg; + msg << "WARNING: " << src_text + << " is expected to be a 32-bit integer, but actually" + << " has value " << str << ", which overflows.\n"; + printf("%s", msg.GetString().c_str()); + fflush(stdout); + return false; + } + + *value = result; + return true; +} + +// Reads and returns the Boolean environment variable corresponding to +// the given flag; if it's not set, returns default_value. +// +// The value is considered true iff it's not "0". +bool BoolFromGTestEnv(const char* flag, bool default_value) { +#if defined(GTEST_GET_BOOL_FROM_ENV_) + return GTEST_GET_BOOL_FROM_ENV_(flag, default_value); +#else + const std::string env_var = FlagToEnvVar(flag); + const char* const string_value = posix::GetEnv(env_var.c_str()); + return string_value == NULL ? + default_value : strcmp(string_value, "0") != 0; +#endif // defined(GTEST_GET_BOOL_FROM_ENV_) +} + +// Reads and returns a 32-bit integer stored in the environment +// variable corresponding to the given flag; if it isn't set or +// doesn't represent a valid 32-bit integer, returns default_value. +Int32 Int32FromGTestEnv(const char* flag, Int32 default_value) { +#if defined(GTEST_GET_INT32_FROM_ENV_) + return GTEST_GET_INT32_FROM_ENV_(flag, default_value); +#else + const std::string env_var = FlagToEnvVar(flag); + const char* const string_value = posix::GetEnv(env_var.c_str()); + if (string_value == NULL) { + // The environment variable is not set. + return default_value; + } + + Int32 result = default_value; + if (!ParseInt32(Message() << "Environment variable " << env_var, + string_value, &result)) { + printf("The default value %s is used.\n", + (Message() << default_value).GetString().c_str()); + fflush(stdout); + return default_value; + } + + return result; +#endif // defined(GTEST_GET_INT32_FROM_ENV_) +} + +// As a special case for the 'output' flag, if GTEST_OUTPUT is not +// set, we look for XML_OUTPUT_FILE, which is set by the Bazel build +// system. The value of XML_OUTPUT_FILE is a filename without the +// "xml:" prefix of GTEST_OUTPUT. +// Note that this is meant to be called at the call site so it does +// not check that the flag is 'output' +// In essence this checks an env variable called XML_OUTPUT_FILE +// and if it is set we prepend "xml:" to its value, if it not set we return "" +std::string OutputFlagAlsoCheckEnvVar(){ + std::string default_value_for_output_flag = ""; + const char* xml_output_file_env = posix::GetEnv("XML_OUTPUT_FILE"); + if (NULL != xml_output_file_env) { + default_value_for_output_flag = std::string("xml:") + xml_output_file_env; + } + return default_value_for_output_flag; +} + +// Reads and returns the string environment variable corresponding to +// the given flag; if it's not set, returns default_value. +const char* StringFromGTestEnv(const char* flag, const char* default_value) { +#if defined(GTEST_GET_STRING_FROM_ENV_) + return GTEST_GET_STRING_FROM_ENV_(flag, default_value); +#else + const std::string env_var = FlagToEnvVar(flag); + const char* const value = posix::GetEnv(env_var.c_str()); + return value == NULL ? default_value : value; +#endif // defined(GTEST_GET_STRING_FROM_ENV_) +} + +} // namespace internal +} // namespace testing +// Copyright 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +// Google Test - The Google C++ Testing and Mocking Framework +// +// This file implements a universal value printer that can print a +// value of any type T: +// +// void ::testing::internal::UniversalPrinter::Print(value, ostream_ptr); +// +// It uses the << operator when possible, and prints the bytes in the +// object otherwise. A user can override its behavior for a class +// type Foo by defining either operator<<(::std::ostream&, const Foo&) +// or void PrintTo(const Foo&, ::std::ostream*) in the namespace that +// defines Foo. + +#include +#include +#include +#include // NOLINT +#include + +namespace testing { + +namespace { + +using ::std::ostream; + +// Prints a segment of bytes in the given object. +GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ +GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ +void PrintByteSegmentInObjectTo(const unsigned char* obj_bytes, size_t start, + size_t count, ostream* os) { + char text[5] = ""; + for (size_t i = 0; i != count; i++) { + const size_t j = start + i; + if (i != 0) { + // Organizes the bytes into groups of 2 for easy parsing by + // human. + if ((j % 2) == 0) + *os << ' '; + else + *os << '-'; + } + GTEST_SNPRINTF_(text, sizeof(text), "%02X", obj_bytes[j]); + *os << text; + } +} + +// Prints the bytes in the given value to the given ostream. +void PrintBytesInObjectToImpl(const unsigned char* obj_bytes, size_t count, + ostream* os) { + // Tells the user how big the object is. + *os << count << "-byte object <"; + + const size_t kThreshold = 132; + const size_t kChunkSize = 64; + // If the object size is bigger than kThreshold, we'll have to omit + // some details by printing only the first and the last kChunkSize + // bytes. + // FIXME: let the user control the threshold using a flag. + if (count < kThreshold) { + PrintByteSegmentInObjectTo(obj_bytes, 0, count, os); + } else { + PrintByteSegmentInObjectTo(obj_bytes, 0, kChunkSize, os); + *os << " ... "; + // Rounds up to 2-byte boundary. + const size_t resume_pos = (count - kChunkSize + 1)/2*2; + PrintByteSegmentInObjectTo(obj_bytes, resume_pos, count - resume_pos, os); + } + *os << ">"; +} + +} // namespace + +namespace internal2 { + +// Delegates to PrintBytesInObjectToImpl() to print the bytes in the +// given object. The delegation simplifies the implementation, which +// uses the << operator and thus is easier done outside of the +// ::testing::internal namespace, which contains a << operator that +// sometimes conflicts with the one in STL. +void PrintBytesInObjectTo(const unsigned char* obj_bytes, size_t count, + ostream* os) { + PrintBytesInObjectToImpl(obj_bytes, count, os); +} + +} // namespace internal2 + +namespace internal { + +// Depending on the value of a char (or wchar_t), we print it in one +// of three formats: +// - as is if it's a printable ASCII (e.g. 'a', '2', ' '), +// - as a hexadecimal escape sequence (e.g. '\x7F'), or +// - as a special escape sequence (e.g. '\r', '\n'). +enum CharFormat { + kAsIs, + kHexEscape, + kSpecialEscape +}; + +// Returns true if c is a printable ASCII character. We test the +// value of c directly instead of calling isprint(), which is buggy on +// Windows Mobile. +inline bool IsPrintableAscii(wchar_t c) { + return 0x20 <= c && c <= 0x7E; +} + +// Prints a wide or narrow char c as a character literal without the +// quotes, escaping it when necessary; returns how c was formatted. +// The template argument UnsignedChar is the unsigned version of Char, +// which is the type of c. +template +static CharFormat PrintAsCharLiteralTo(Char c, ostream* os) { + switch (static_cast(c)) { + case L'\0': + *os << "\\0"; + break; + case L'\'': + *os << "\\'"; + break; + case L'\\': + *os << "\\\\"; + break; + case L'\a': + *os << "\\a"; + break; + case L'\b': + *os << "\\b"; + break; + case L'\f': + *os << "\\f"; + break; + case L'\n': + *os << "\\n"; + break; + case L'\r': + *os << "\\r"; + break; + case L'\t': + *os << "\\t"; + break; + case L'\v': + *os << "\\v"; + break; + default: + if (IsPrintableAscii(c)) { + *os << static_cast(c); + return kAsIs; + } else { + ostream::fmtflags flags = os->flags(); + *os << "\\x" << std::hex << std::uppercase + << static_cast(static_cast(c)); + os->flags(flags); + return kHexEscape; + } + } + return kSpecialEscape; +} + +// Prints a wchar_t c as if it's part of a string literal, escaping it when +// necessary; returns how c was formatted. +static CharFormat PrintAsStringLiteralTo(wchar_t c, ostream* os) { + switch (c) { + case L'\'': + *os << "'"; + return kAsIs; + case L'"': + *os << "\\\""; + return kSpecialEscape; + default: + return PrintAsCharLiteralTo(c, os); + } +} + +// Prints a char c as if it's part of a string literal, escaping it when +// necessary; returns how c was formatted. +static CharFormat PrintAsStringLiteralTo(char c, ostream* os) { + return PrintAsStringLiteralTo( + static_cast(static_cast(c)), os); +} + +// Prints a wide or narrow character c and its code. '\0' is printed +// as "'\\0'", other unprintable characters are also properly escaped +// using the standard C++ escape sequence. The template argument +// UnsignedChar is the unsigned version of Char, which is the type of c. +template +void PrintCharAndCodeTo(Char c, ostream* os) { + // First, print c as a literal in the most readable form we can find. + *os << ((sizeof(c) > 1) ? "L'" : "'"); + const CharFormat format = PrintAsCharLiteralTo(c, os); + *os << "'"; + + // To aid user debugging, we also print c's code in decimal, unless + // it's 0 (in which case c was printed as '\\0', making the code + // obvious). + if (c == 0) + return; + *os << " (" << static_cast(c); + + // For more convenience, we print c's code again in hexadecimal, + // unless c was already printed in the form '\x##' or the code is in + // [1, 9]. + if (format == kHexEscape || (1 <= c && c <= 9)) { + // Do nothing. + } else { + *os << ", 0x" << String::FormatHexInt(static_cast(c)); + } + *os << ")"; +} + +void PrintTo(unsigned char c, ::std::ostream* os) { + PrintCharAndCodeTo(c, os); +} +void PrintTo(signed char c, ::std::ostream* os) { + PrintCharAndCodeTo(c, os); +} + +// Prints a wchar_t as a symbol if it is printable or as its internal +// code otherwise and also as its code. L'\0' is printed as "L'\\0'". +void PrintTo(wchar_t wc, ostream* os) { + PrintCharAndCodeTo(wc, os); +} + +// Prints the given array of characters to the ostream. CharType must be either +// char or wchar_t. +// The array starts at begin, the length is len, it may include '\0' characters +// and may not be NUL-terminated. +template +GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ +GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ +static CharFormat PrintCharsAsStringTo( + const CharType* begin, size_t len, ostream* os) { + const char* const kQuoteBegin = sizeof(CharType) == 1 ? "\"" : "L\""; + *os << kQuoteBegin; + bool is_previous_hex = false; + CharFormat print_format = kAsIs; + for (size_t index = 0; index < len; ++index) { + const CharType cur = begin[index]; + if (is_previous_hex && IsXDigit(cur)) { + // Previous character is of '\x..' form and this character can be + // interpreted as another hexadecimal digit in its number. Break string to + // disambiguate. + *os << "\" " << kQuoteBegin; + } + is_previous_hex = PrintAsStringLiteralTo(cur, os) == kHexEscape; + // Remember if any characters required hex escaping. + if (is_previous_hex) { + print_format = kHexEscape; + } + } + *os << "\""; + return print_format; +} + +// Prints a (const) char/wchar_t array of 'len' elements, starting at address +// 'begin'. CharType must be either char or wchar_t. +template +GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ +GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ +static void UniversalPrintCharArray( + const CharType* begin, size_t len, ostream* os) { + // The code + // const char kFoo[] = "foo"; + // generates an array of 4, not 3, elements, with the last one being '\0'. + // + // Therefore when printing a char array, we don't print the last element if + // it's '\0', such that the output matches the string literal as it's + // written in the source code. + if (len > 0 && begin[len - 1] == '\0') { + PrintCharsAsStringTo(begin, len - 1, os); + return; + } + + // If, however, the last element in the array is not '\0', e.g. + // const char kFoo[] = { 'f', 'o', 'o' }; + // we must print the entire array. We also print a message to indicate + // that the array is not NUL-terminated. + PrintCharsAsStringTo(begin, len, os); + *os << " (no terminating NUL)"; +} + +// Prints a (const) char array of 'len' elements, starting at address 'begin'. +void UniversalPrintArray(const char* begin, size_t len, ostream* os) { + UniversalPrintCharArray(begin, len, os); +} + +// Prints a (const) wchar_t array of 'len' elements, starting at address +// 'begin'. +void UniversalPrintArray(const wchar_t* begin, size_t len, ostream* os) { + UniversalPrintCharArray(begin, len, os); +} + +// Prints the given C string to the ostream. +void PrintTo(const char* s, ostream* os) { + if (s == NULL) { + *os << "NULL"; + } else { + *os << ImplicitCast_(s) << " pointing to "; + PrintCharsAsStringTo(s, strlen(s), os); + } +} + +// MSVC compiler can be configured to define whar_t as a typedef +// of unsigned short. Defining an overload for const wchar_t* in that case +// would cause pointers to unsigned shorts be printed as wide strings, +// possibly accessing more memory than intended and causing invalid +// memory accesses. MSVC defines _NATIVE_WCHAR_T_DEFINED symbol when +// wchar_t is implemented as a native type. +#if !defined(_MSC_VER) || defined(_NATIVE_WCHAR_T_DEFINED) +// Prints the given wide C string to the ostream. +void PrintTo(const wchar_t* s, ostream* os) { + if (s == NULL) { + *os << "NULL"; + } else { + *os << ImplicitCast_(s) << " pointing to "; + PrintCharsAsStringTo(s, std::wcslen(s), os); + } +} +#endif // wchar_t is native + +namespace { + +bool ContainsUnprintableControlCodes(const char* str, size_t length) { + const unsigned char *s = reinterpret_cast(str); + + for (size_t i = 0; i < length; i++) { + unsigned char ch = *s++; + if (std::iscntrl(ch)) { + switch (ch) { + case '\t': + case '\n': + case '\r': + break; + default: + return true; + } + } + } + return false; +} + +bool IsUTF8TrailByte(unsigned char t) { return 0x80 <= t && t<= 0xbf; } + +bool IsValidUTF8(const char* str, size_t length) { + const unsigned char *s = reinterpret_cast(str); + + for (size_t i = 0; i < length;) { + unsigned char lead = s[i++]; + + if (lead <= 0x7f) { + continue; // single-byte character (ASCII) 0..7F + } + if (lead < 0xc2) { + return false; // trail byte or non-shortest form + } else if (lead <= 0xdf && (i + 1) <= length && IsUTF8TrailByte(s[i])) { + ++i; // 2-byte character + } else if (0xe0 <= lead && lead <= 0xef && (i + 2) <= length && + IsUTF8TrailByte(s[i]) && + IsUTF8TrailByte(s[i + 1]) && + // check for non-shortest form and surrogate + (lead != 0xe0 || s[i] >= 0xa0) && + (lead != 0xed || s[i] < 0xa0)) { + i += 2; // 3-byte character + } else if (0xf0 <= lead && lead <= 0xf4 && (i + 3) <= length && + IsUTF8TrailByte(s[i]) && + IsUTF8TrailByte(s[i + 1]) && + IsUTF8TrailByte(s[i + 2]) && + // check for non-shortest form + (lead != 0xf0 || s[i] >= 0x90) && + (lead != 0xf4 || s[i] < 0x90)) { + i += 3; // 4-byte character + } else { + return false; + } + } + return true; +} + +void ConditionalPrintAsText(const char* str, size_t length, ostream* os) { + if (!ContainsUnprintableControlCodes(str, length) && + IsValidUTF8(str, length)) { + *os << "\n As Text: \"" << str << "\""; + } +} + +} // anonymous namespace + +// Prints a ::string object. +#if GTEST_HAS_GLOBAL_STRING +void PrintStringTo(const ::string& s, ostream* os) { + if (PrintCharsAsStringTo(s.data(), s.size(), os) == kHexEscape) { + if (GTEST_FLAG(print_utf8)) { + ConditionalPrintAsText(s.data(), s.size(), os); + } + } +} +#endif // GTEST_HAS_GLOBAL_STRING + +void PrintStringTo(const ::std::string& s, ostream* os) { + if (PrintCharsAsStringTo(s.data(), s.size(), os) == kHexEscape) { + if (GTEST_FLAG(print_utf8)) { + ConditionalPrintAsText(s.data(), s.size(), os); + } + } +} + +// Prints a ::wstring object. +#if GTEST_HAS_GLOBAL_WSTRING +void PrintWideStringTo(const ::wstring& s, ostream* os) { + PrintCharsAsStringTo(s.data(), s.size(), os); +} +#endif // GTEST_HAS_GLOBAL_WSTRING + +#if GTEST_HAS_STD_WSTRING +void PrintWideStringTo(const ::std::wstring& s, ostream* os) { + PrintCharsAsStringTo(s.data(), s.size(), os); +} +#endif // GTEST_HAS_STD_WSTRING + +} // namespace internal + +} // namespace testing +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// +// The Google C++ Testing and Mocking Framework (Google Test) + + +namespace testing { + +using internal::GetUnitTestImpl; + +// Gets the summary of the failure message by omitting the stack trace +// in it. +std::string TestPartResult::ExtractSummary(const char* message) { + const char* const stack_trace = strstr(message, internal::kStackTraceMarker); + return stack_trace == NULL ? message : + std::string(message, stack_trace); +} + +// Prints a TestPartResult object. +std::ostream& operator<<(std::ostream& os, const TestPartResult& result) { + return os + << result.file_name() << ":" << result.line_number() << ": " + << (result.type() == TestPartResult::kSuccess ? "Success" : + result.type() == TestPartResult::kFatalFailure ? "Fatal failure" : + "Non-fatal failure") << ":\n" + << result.message() << std::endl; +} + +// Appends a TestPartResult to the array. +void TestPartResultArray::Append(const TestPartResult& result) { + array_.push_back(result); +} + +// Returns the TestPartResult at the given index (0-based). +const TestPartResult& TestPartResultArray::GetTestPartResult(int index) const { + if (index < 0 || index >= size()) { + printf("\nInvalid index (%d) into TestPartResultArray.\n", index); + internal::posix::Abort(); + } + + return array_[index]; +} + +// Returns the number of TestPartResult objects in the array. +int TestPartResultArray::size() const { + return static_cast(array_.size()); +} + +namespace internal { + +HasNewFatalFailureHelper::HasNewFatalFailureHelper() + : has_new_fatal_failure_(false), + original_reporter_(GetUnitTestImpl()-> + GetTestPartResultReporterForCurrentThread()) { + GetUnitTestImpl()->SetTestPartResultReporterForCurrentThread(this); +} + +HasNewFatalFailureHelper::~HasNewFatalFailureHelper() { + GetUnitTestImpl()->SetTestPartResultReporterForCurrentThread( + original_reporter_); +} + +void HasNewFatalFailureHelper::ReportTestPartResult( + const TestPartResult& result) { + if (result.fatally_failed()) + has_new_fatal_failure_ = true; + original_reporter_->ReportTestPartResult(result); +} + +} // namespace internal + +} // namespace testing +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + + +namespace testing { +namespace internal { + +#if GTEST_HAS_TYPED_TEST_P + +// Skips to the first non-space char in str. Returns an empty string if str +// contains only whitespace characters. +static const char* SkipSpaces(const char* str) { + while (IsSpace(*str)) + str++; + return str; +} + +static std::vector SplitIntoTestNames(const char* src) { + std::vector name_vec; + src = SkipSpaces(src); + for (; src != NULL; src = SkipComma(src)) { + name_vec.push_back(StripTrailingSpaces(GetPrefixUntilComma(src))); + } + return name_vec; +} + +// Verifies that registered_tests match the test names in +// registered_tests_; returns registered_tests if successful, or +// aborts the program otherwise. +const char* TypedTestCasePState::VerifyRegisteredTestNames( + const char* file, int line, const char* registered_tests) { + typedef RegisteredTestsMap::const_iterator RegisteredTestIter; + registered_ = true; + + std::vector name_vec = SplitIntoTestNames(registered_tests); + + Message errors; + + std::set tests; + for (std::vector::const_iterator name_it = name_vec.begin(); + name_it != name_vec.end(); ++name_it) { + const std::string& name = *name_it; + if (tests.count(name) != 0) { + errors << "Test " << name << " is listed more than once.\n"; + continue; + } + + bool found = false; + for (RegisteredTestIter it = registered_tests_.begin(); + it != registered_tests_.end(); + ++it) { + if (name == it->first) { + found = true; + break; + } + } + + if (found) { + tests.insert(name); + } else { + errors << "No test named " << name + << " can be found in this test case.\n"; + } + } + + for (RegisteredTestIter it = registered_tests_.begin(); + it != registered_tests_.end(); + ++it) { + if (tests.count(it->first) == 0) { + errors << "You forgot to list test " << it->first << ".\n"; + } + } + + const std::string& errors_str = errors.GetString(); + if (errors_str != "") { + fprintf(stderr, "%s %s", FormatFileLocation(file, line).c_str(), + errors_str.c_str()); + fflush(stderr); + posix::Abort(); + } + + return registered_tests; +} + +#endif // GTEST_HAS_TYPED_TEST_P + +} // namespace internal +} // namespace testing diff --git a/librocksdb-sys/rocksdb/third-party/gtest-1.8.1/fused-src/gtest/gtest.h b/librocksdb-sys/rocksdb/third-party/gtest-1.8.1/fused-src/gtest/gtest.h new file mode 100644 index 0000000..2d82d8e --- /dev/null +++ b/librocksdb-sys/rocksdb/third-party/gtest-1.8.1/fused-src/gtest/gtest.h @@ -0,0 +1,22115 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// +// The Google C++ Testing and Mocking Framework (Google Test) +// +// This header file defines the public API for Google Test. It should be +// included by any test program that uses Google Test. +// +// IMPORTANT NOTE: Due to limitation of the C++ language, we have to +// leave some internal implementation details in this header file. +// They are clearly marked by comments like this: +// +// // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +// +// Such code is NOT meant to be used by a user directly, and is subject +// to CHANGE WITHOUT NOTICE. Therefore DO NOT DEPEND ON IT in a user +// program! +// +// Acknowledgment: Google Test borrowed the idea of automatic test +// registration from Barthelemy Dagenais' (barthelemy@prologique.com) +// easyUnit framework. + +// GOOGLETEST_CM0001 DO NOT DELETE + +#ifndef GTEST_INCLUDE_GTEST_GTEST_H_ +#define GTEST_INCLUDE_GTEST_GTEST_H_ + +#include +#include +#include +#include + +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// The Google C++ Testing and Mocking Framework (Google Test) +// +// This header file declares functions and macros used internally by +// Google Test. They are subject to change without notice. + +// GOOGLETEST_CM0001 DO NOT DELETE + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ + +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Low-level types and utilities for porting Google Test to various +// platforms. All macros ending with _ and symbols defined in an +// internal namespace are subject to change without notice. Code +// outside Google Test MUST NOT USE THEM DIRECTLY. Macros that don't +// end with _ are part of Google Test's public API and can be used by +// code outside Google Test. +// +// This file is fundamental to Google Test. All other Google Test source +// files are expected to #include this. Therefore, it cannot #include +// any other Google Test header. + +// GOOGLETEST_CM0001 DO NOT DELETE + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ + +// Environment-describing macros +// ----------------------------- +// +// Google Test can be used in many different environments. Macros in +// this section tell Google Test what kind of environment it is being +// used in, such that Google Test can provide environment-specific +// features and implementations. +// +// Google Test tries to automatically detect the properties of its +// environment, so users usually don't need to worry about these +// macros. However, the automatic detection is not perfect. +// Sometimes it's necessary for a user to define some of the following +// macros in the build script to override Google Test's decisions. +// +// If the user doesn't define a macro in the list, Google Test will +// provide a default definition. After this header is #included, all +// macros in this list will be defined to either 1 or 0. +// +// Notes to maintainers: +// - Each macro here is a user-tweakable knob; do not grow the list +// lightly. +// - Use #if to key off these macros. Don't use #ifdef or "#if +// defined(...)", which will not work as these macros are ALWAYS +// defined. +// +// GTEST_HAS_CLONE - Define it to 1/0 to indicate that clone(2) +// is/isn't available. +// GTEST_HAS_EXCEPTIONS - Define it to 1/0 to indicate that exceptions +// are enabled. +// GTEST_HAS_GLOBAL_STRING - Define it to 1/0 to indicate that ::string +// is/isn't available +// GTEST_HAS_GLOBAL_WSTRING - Define it to 1/0 to indicate that ::wstring +// is/isn't available +// GTEST_HAS_POSIX_RE - Define it to 1/0 to indicate that POSIX regular +// expressions are/aren't available. +// GTEST_HAS_PTHREAD - Define it to 1/0 to indicate that +// is/isn't available. +// GTEST_HAS_RTTI - Define it to 1/0 to indicate that RTTI is/isn't +// enabled. +// GTEST_HAS_STD_WSTRING - Define it to 1/0 to indicate that +// std::wstring does/doesn't work (Google Test can +// be used where std::wstring is unavailable). +// GTEST_HAS_TR1_TUPLE - Define it to 1/0 to indicate tr1::tuple +// is/isn't available. +// GTEST_HAS_SEH - Define it to 1/0 to indicate whether the +// compiler supports Microsoft's "Structured +// Exception Handling". +// GTEST_HAS_STREAM_REDIRECTION +// - Define it to 1/0 to indicate whether the +// platform supports I/O stream redirection using +// dup() and dup2(). +// GTEST_USE_OWN_TR1_TUPLE - Define it to 1/0 to indicate whether Google +// Test's own tr1 tuple implementation should be +// used. Unused when the user sets +// GTEST_HAS_TR1_TUPLE to 0. +// GTEST_LANG_CXX11 - Define it to 1/0 to indicate that Google Test +// is building in C++11/C++98 mode. +// GTEST_LINKED_AS_SHARED_LIBRARY +// - Define to 1 when compiling tests that use +// Google Test as a shared library (known as +// DLL on Windows). +// GTEST_CREATE_SHARED_LIBRARY +// - Define to 1 when compiling Google Test itself +// as a shared library. +// GTEST_DEFAULT_DEATH_TEST_STYLE +// - The default value of --gtest_death_test_style. +// The legacy default has been "fast" in the open +// source version since 2008. The recommended value +// is "threadsafe", and can be set in +// custom/gtest-port.h. + +// Platform-indicating macros +// -------------------------- +// +// Macros indicating the platform on which Google Test is being used +// (a macro is defined to 1 if compiled on the given platform; +// otherwise UNDEFINED -- it's never defined to 0.). Google Test +// defines these macros automatically. Code outside Google Test MUST +// NOT define them. +// +// GTEST_OS_AIX - IBM AIX +// GTEST_OS_CYGWIN - Cygwin +// GTEST_OS_FREEBSD - FreeBSD +// GTEST_OS_FUCHSIA - Fuchsia +// GTEST_OS_HPUX - HP-UX +// GTEST_OS_LINUX - Linux +// GTEST_OS_LINUX_ANDROID - Google Android +// GTEST_OS_MAC - Mac OS X +// GTEST_OS_IOS - iOS +// GTEST_OS_NACL - Google Native Client (NaCl) +// GTEST_OS_NETBSD - NetBSD +// GTEST_OS_OPENBSD - OpenBSD +// GTEST_OS_QNX - QNX +// GTEST_OS_SOLARIS - Sun Solaris +// GTEST_OS_SYMBIAN - Symbian +// GTEST_OS_WINDOWS - Windows (Desktop, MinGW, or Mobile) +// GTEST_OS_WINDOWS_DESKTOP - Windows Desktop +// GTEST_OS_WINDOWS_MINGW - MinGW +// GTEST_OS_WINDOWS_MOBILE - Windows Mobile +// GTEST_OS_WINDOWS_PHONE - Windows Phone +// GTEST_OS_WINDOWS_RT - Windows Store App/WinRT +// GTEST_OS_ZOS - z/OS +// +// Among the platforms, Cygwin, Linux, Max OS X, and Windows have the +// most stable support. Since core members of the Google Test project +// don't have access to other platforms, support for them may be less +// stable. If you notice any problems on your platform, please notify +// googletestframework@googlegroups.com (patches for fixing them are +// even more welcome!). +// +// It is possible that none of the GTEST_OS_* macros are defined. + +// Feature-indicating macros +// ------------------------- +// +// Macros indicating which Google Test features are available (a macro +// is defined to 1 if the corresponding feature is supported; +// otherwise UNDEFINED -- it's never defined to 0.). Google Test +// defines these macros automatically. Code outside Google Test MUST +// NOT define them. +// +// These macros are public so that portable tests can be written. +// Such tests typically surround code using a feature with an #if +// which controls that code. For example: +// +// #if GTEST_HAS_DEATH_TEST +// EXPECT_DEATH(DoSomethingDeadly()); +// #endif +// +// GTEST_HAS_COMBINE - the Combine() function (for value-parameterized +// tests) +// GTEST_HAS_DEATH_TEST - death tests +// GTEST_HAS_TYPED_TEST - typed tests +// GTEST_HAS_TYPED_TEST_P - type-parameterized tests +// GTEST_IS_THREADSAFE - Google Test is thread-safe. +// GOOGLETEST_CM0007 DO NOT DELETE +// GTEST_USES_POSIX_RE - enhanced POSIX regex is used. Do not confuse with +// GTEST_HAS_POSIX_RE (see above) which users can +// define themselves. +// GTEST_USES_SIMPLE_RE - our own simple regex is used; +// the above RE\b(s) are mutually exclusive. +// GTEST_CAN_COMPARE_NULL - accepts untyped NULL in EXPECT_EQ(). + +// Misc public macros +// ------------------ +// +// GTEST_FLAG(flag_name) - references the variable corresponding to +// the given Google Test flag. + +// Internal utilities +// ------------------ +// +// The following macros and utilities are for Google Test's INTERNAL +// use only. Code outside Google Test MUST NOT USE THEM DIRECTLY. +// +// Macros for basic C++ coding: +// GTEST_AMBIGUOUS_ELSE_BLOCKER_ - for disabling a gcc warning. +// GTEST_ATTRIBUTE_UNUSED_ - declares that a class' instances or a +// variable don't have to be used. +// GTEST_DISALLOW_ASSIGN_ - disables operator=. +// GTEST_DISALLOW_COPY_AND_ASSIGN_ - disables copy ctor and operator=. +// GTEST_MUST_USE_RESULT_ - declares that a function's result must be used. +// GTEST_INTENTIONAL_CONST_COND_PUSH_ - start code section where MSVC C4127 is +// suppressed (constant conditional). +// GTEST_INTENTIONAL_CONST_COND_POP_ - finish code section where MSVC C4127 +// is suppressed. +// +// C++11 feature wrappers: +// +// testing::internal::forward - portability wrapper for std::forward. +// testing::internal::move - portability wrapper for std::move. +// +// Synchronization: +// Mutex, MutexLock, ThreadLocal, GetThreadCount() +// - synchronization primitives. +// +// Template meta programming: +// is_pointer - as in TR1; needed on Symbian and IBM XL C/C++ only. +// IteratorTraits - partial implementation of std::iterator_traits, which +// is not available in libCstd when compiled with Sun C++. +// +// Smart pointers: +// scoped_ptr - as in TR2. +// +// Regular expressions: +// RE - a simple regular expression class using the POSIX +// Extended Regular Expression syntax on UNIX-like platforms +// GOOGLETEST_CM0008 DO NOT DELETE +// or a reduced regular exception syntax on other +// platforms, including Windows. +// Logging: +// GTEST_LOG_() - logs messages at the specified severity level. +// LogToStderr() - directs all log messages to stderr. +// FlushInfoLog() - flushes informational log messages. +// +// Stdout and stderr capturing: +// CaptureStdout() - starts capturing stdout. +// GetCapturedStdout() - stops capturing stdout and returns the captured +// string. +// CaptureStderr() - starts capturing stderr. +// GetCapturedStderr() - stops capturing stderr and returns the captured +// string. +// +// Integer types: +// TypeWithSize - maps an integer to a int type. +// Int32, UInt32, Int64, UInt64, TimeInMillis +// - integers of known sizes. +// BiggestInt - the biggest signed integer type. +// +// Command-line utilities: +// GTEST_DECLARE_*() - declares a flag. +// GTEST_DEFINE_*() - defines a flag. +// GetInjectableArgvs() - returns the command line as a vector of strings. +// +// Environment variable utilities: +// GetEnv() - gets the value of an environment variable. +// BoolFromGTestEnv() - parses a bool environment variable. +// Int32FromGTestEnv() - parses an Int32 environment variable. +// StringFromGTestEnv() - parses a string environment variable. + +#include // for isspace, etc +#include // for ptrdiff_t +#include +#include +#include +#ifndef _WIN32_WCE +# include +# include +#endif // !_WIN32_WCE + +#if defined __APPLE__ +# include +# include +#endif + +// Brings in the definition of HAS_GLOBAL_STRING. This must be done +// BEFORE we test HAS_GLOBAL_STRING. +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include +#include // NOLINT + +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// The Google C++ Testing and Mocking Framework (Google Test) +// +// This header file defines the GTEST_OS_* macro. +// It is separate from gtest-port.h so that custom/gtest-port.h can include it. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ + +// Determines the platform on which Google Test is compiled. +#ifdef __CYGWIN__ +# define GTEST_OS_CYGWIN 1 +#elif defined __SYMBIAN32__ +# define GTEST_OS_SYMBIAN 1 +#elif defined _WIN32 +# define GTEST_OS_WINDOWS 1 +# ifdef _WIN32_WCE +# define GTEST_OS_WINDOWS_MOBILE 1 +# elif defined(__MINGW__) || defined(__MINGW32__) +# define GTEST_OS_WINDOWS_MINGW 1 +# elif defined(WINAPI_FAMILY) +# include +# if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) +# define GTEST_OS_WINDOWS_DESKTOP 1 +# elif WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_PHONE_APP) +# define GTEST_OS_WINDOWS_PHONE 1 +# elif WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) +# define GTEST_OS_WINDOWS_RT 1 +# elif WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_TV_TITLE) +# define GTEST_OS_WINDOWS_PHONE 1 +# define GTEST_OS_WINDOWS_TV_TITLE 1 +# else + // WINAPI_FAMILY defined but no known partition matched. + // Default to desktop. +# define GTEST_OS_WINDOWS_DESKTOP 1 +# endif +# else +# define GTEST_OS_WINDOWS_DESKTOP 1 +# endif // _WIN32_WCE +#elif defined __APPLE__ +# define GTEST_OS_MAC 1 +# if TARGET_OS_IPHONE +# define GTEST_OS_IOS 1 +# endif +#elif defined __FreeBSD__ +# define GTEST_OS_FREEBSD 1 +#elif defined __Fuchsia__ +# define GTEST_OS_FUCHSIA 1 +#elif defined __linux__ +# define GTEST_OS_LINUX 1 +# if defined __ANDROID__ +# define GTEST_OS_LINUX_ANDROID 1 +# endif +#elif defined __MVS__ +# define GTEST_OS_ZOS 1 +#elif defined(__sun) && defined(__SVR4) +# define GTEST_OS_SOLARIS 1 +#elif defined(_AIX) +# define GTEST_OS_AIX 1 +#elif defined(__hpux) +# define GTEST_OS_HPUX 1 +#elif defined __native_client__ +# define GTEST_OS_NACL 1 +#elif defined __NetBSD__ +# define GTEST_OS_NETBSD 1 +#elif defined __OpenBSD__ +# define GTEST_OS_OPENBSD 1 +#elif defined __QNX__ +# define GTEST_OS_QNX 1 +#endif // __CYGWIN__ + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Injection point for custom user configurations. See README for details +// +// ** Custom implementation starts here ** + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ + +#if !defined(GTEST_DEV_EMAIL_) +# define GTEST_DEV_EMAIL_ "googletestframework@@googlegroups.com" +# define GTEST_FLAG_PREFIX_ "gtest_" +# define GTEST_FLAG_PREFIX_DASH_ "gtest-" +# define GTEST_FLAG_PREFIX_UPPER_ "GTEST_" +# define GTEST_NAME_ "Google Test" +# define GTEST_PROJECT_URL_ "https://github.com/google/googletest/" +#endif // !defined(GTEST_DEV_EMAIL_) + +#if !defined(GTEST_INIT_GOOGLE_TEST_NAME_) +# define GTEST_INIT_GOOGLE_TEST_NAME_ "testing::InitGoogleTest" +#endif // !defined(GTEST_INIT_GOOGLE_TEST_NAME_) + +// Determines the version of gcc that is used to compile this. +#ifdef __GNUC__ +// 40302 means version 4.3.2. +# define GTEST_GCC_VER_ \ + (__GNUC__*10000 + __GNUC_MINOR__*100 + __GNUC_PATCHLEVEL__) +#endif // __GNUC__ + +// Macros for disabling Microsoft Visual C++ warnings. +// +// GTEST_DISABLE_MSC_WARNINGS_PUSH_(4800 4385) +// /* code that triggers warnings C4800 and C4385 */ +// GTEST_DISABLE_MSC_WARNINGS_POP_() +#if _MSC_VER >= 1400 +# define GTEST_DISABLE_MSC_WARNINGS_PUSH_(warnings) \ + __pragma(warning(push)) \ + __pragma(warning(disable: warnings)) +# define GTEST_DISABLE_MSC_WARNINGS_POP_() \ + __pragma(warning(pop)) +#else +// Older versions of MSVC don't have __pragma. +# define GTEST_DISABLE_MSC_WARNINGS_PUSH_(warnings) +# define GTEST_DISABLE_MSC_WARNINGS_POP_() +#endif + +// Clang on Windows does not understand MSVC's pragma warning. +// We need clang-specific way to disable function deprecation warning. +#ifdef __clang__ +# define GTEST_DISABLE_MSC_DEPRECATED_PUSH_() \ + _Pragma("clang diagnostic push") \ + _Pragma("clang diagnostic ignored \"-Wdeprecated-declarations\"") \ + _Pragma("clang diagnostic ignored \"-Wdeprecated-implementations\"") +#define GTEST_DISABLE_MSC_DEPRECATED_POP_() \ + _Pragma("clang diagnostic pop") +#else +# define GTEST_DISABLE_MSC_DEPRECATED_PUSH_() \ + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4996) +# define GTEST_DISABLE_MSC_DEPRECATED_POP_() \ + GTEST_DISABLE_MSC_WARNINGS_POP_() +#endif + +#ifndef GTEST_LANG_CXX11 +// gcc and clang define __GXX_EXPERIMENTAL_CXX0X__ when +// -std={c,gnu}++{0x,11} is passed. The C++11 standard specifies a +// value for __cplusplus, and recent versions of clang, gcc, and +// probably other compilers set that too in C++11 mode. +# if __GXX_EXPERIMENTAL_CXX0X__ || __cplusplus >= 201103L || _MSC_VER >= 1900 +// Compiling in at least C++11 mode. +# define GTEST_LANG_CXX11 1 +# else +# define GTEST_LANG_CXX11 0 +# endif +#endif + +// Distinct from C++11 language support, some environments don't provide +// proper C++11 library support. Notably, it's possible to build in +// C++11 mode when targeting Mac OS X 10.6, which has an old libstdc++ +// with no C++11 support. +// +// libstdc++ has sufficient C++11 support as of GCC 4.6.0, __GLIBCXX__ +// 20110325, but maintenance releases in the 4.4 and 4.5 series followed +// this date, so check for those versions by their date stamps. +// https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html#abi.versioning +#if GTEST_LANG_CXX11 && \ + (!defined(__GLIBCXX__) || ( \ + __GLIBCXX__ >= 20110325ul && /* GCC >= 4.6.0 */ \ + /* Blacklist of patch releases of older branches: */ \ + __GLIBCXX__ != 20110416ul && /* GCC 4.4.6 */ \ + __GLIBCXX__ != 20120313ul && /* GCC 4.4.7 */ \ + __GLIBCXX__ != 20110428ul && /* GCC 4.5.3 */ \ + __GLIBCXX__ != 20120702ul)) /* GCC 4.5.4 */ +# define GTEST_STDLIB_CXX11 1 +#endif + +// Only use C++11 library features if the library provides them. +#if GTEST_STDLIB_CXX11 +# define GTEST_HAS_STD_BEGIN_AND_END_ 1 +# define GTEST_HAS_STD_FORWARD_LIST_ 1 +# if !defined(_MSC_VER) || (_MSC_FULL_VER >= 190023824) +// works only with VS2015U2 and better +# define GTEST_HAS_STD_FUNCTION_ 1 +# endif +# define GTEST_HAS_STD_INITIALIZER_LIST_ 1 +# define GTEST_HAS_STD_MOVE_ 1 +# define GTEST_HAS_STD_UNIQUE_PTR_ 1 +# define GTEST_HAS_STD_SHARED_PTR_ 1 +# define GTEST_HAS_UNORDERED_MAP_ 1 +# define GTEST_HAS_UNORDERED_SET_ 1 +#endif + +// C++11 specifies that provides std::tuple. +// Some platforms still might not have it, however. +#if GTEST_LANG_CXX11 +# define GTEST_HAS_STD_TUPLE_ 1 +# if defined(__clang__) +// Inspired by +// https://clang.llvm.org/docs/LanguageExtensions.html#include-file-checking-macros +# if defined(__has_include) && !__has_include() +# undef GTEST_HAS_STD_TUPLE_ +# endif +# elif defined(_MSC_VER) +// Inspired by boost/config/stdlib/dinkumware.hpp +# if defined(_CPPLIB_VER) && _CPPLIB_VER < 520 +# undef GTEST_HAS_STD_TUPLE_ +# endif +# elif defined(__GLIBCXX__) +// Inspired by boost/config/stdlib/libstdcpp3.hpp, +// http://gcc.gnu.org/gcc-4.2/changes.html and +// https://web.archive.org/web/20140227044429/gcc.gnu.org/onlinedocs/libstdc++/manual/bk01pt01ch01.html#manual.intro.status.standard.200x +# if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 2) +# undef GTEST_HAS_STD_TUPLE_ +# endif +# endif +#endif + +// Brings in definitions for functions used in the testing::internal::posix +// namespace (read, write, close, chdir, isatty, stat). We do not currently +// use them on Windows Mobile. +#if GTEST_OS_WINDOWS +# if !GTEST_OS_WINDOWS_MOBILE +# include +# include +# endif +// In order to avoid having to include , use forward declaration +#if GTEST_OS_WINDOWS_MINGW && !defined(__MINGW64_VERSION_MAJOR) +// MinGW defined _CRITICAL_SECTION and _RTL_CRITICAL_SECTION as two +// separate (equivalent) structs, instead of using typedef +typedef struct _CRITICAL_SECTION GTEST_CRITICAL_SECTION; +#else +// Assume CRITICAL_SECTION is a typedef of _RTL_CRITICAL_SECTION. +// This assumption is verified by +// WindowsTypesTest.CRITICAL_SECTIONIs_RTL_CRITICAL_SECTION. +typedef struct _RTL_CRITICAL_SECTION GTEST_CRITICAL_SECTION; +#endif +#else +// This assumes that non-Windows OSes provide unistd.h. For OSes where this +// is not the case, we need to include headers that provide the functions +// mentioned above. +# include +# include +#endif // GTEST_OS_WINDOWS + +#if GTEST_OS_LINUX_ANDROID +// Used to define __ANDROID_API__ matching the target NDK API level. +# include // NOLINT +#endif + +// Defines this to true iff Google Test can use POSIX regular expressions. +#ifndef GTEST_HAS_POSIX_RE +# if GTEST_OS_LINUX_ANDROID +// On Android, is only available starting with Gingerbread. +# define GTEST_HAS_POSIX_RE (__ANDROID_API__ >= 9) +# else +# define GTEST_HAS_POSIX_RE (!GTEST_OS_WINDOWS) +# endif +#endif + +#if GTEST_USES_PCRE +// The appropriate headers have already been included. + +#elif GTEST_HAS_POSIX_RE + +// On some platforms, needs someone to define size_t, and +// won't compile otherwise. We can #include it here as we already +// included , which is guaranteed to define size_t through +// . +# include // NOLINT + +# define GTEST_USES_POSIX_RE 1 + +#elif GTEST_OS_WINDOWS + +// is not available on Windows. Use our own simple regex +// implementation instead. +# define GTEST_USES_SIMPLE_RE 1 + +#else + +// may not be available on this platform. Use our own +// simple regex implementation instead. +# define GTEST_USES_SIMPLE_RE 1 + +#endif // GTEST_USES_PCRE + +#ifndef GTEST_HAS_EXCEPTIONS +// The user didn't tell us whether exceptions are enabled, so we need +// to figure it out. +# if defined(_MSC_VER) && defined(_CPPUNWIND) +// MSVC defines _CPPUNWIND to 1 iff exceptions are enabled. +# define GTEST_HAS_EXCEPTIONS 1 +# elif defined(__BORLANDC__) +// C++Builder's implementation of the STL uses the _HAS_EXCEPTIONS +// macro to enable exceptions, so we'll do the same. +// Assumes that exceptions are enabled by default. +# ifndef _HAS_EXCEPTIONS +# define _HAS_EXCEPTIONS 1 +# endif // _HAS_EXCEPTIONS +# define GTEST_HAS_EXCEPTIONS _HAS_EXCEPTIONS +# elif defined(__clang__) +// clang defines __EXCEPTIONS iff exceptions are enabled before clang 220714, +// but iff cleanups are enabled after that. In Obj-C++ files, there can be +// cleanups for ObjC exceptions which also need cleanups, even if C++ exceptions +// are disabled. clang has __has_feature(cxx_exceptions) which checks for C++ +// exceptions starting at clang r206352, but which checked for cleanups prior to +// that. To reliably check for C++ exception availability with clang, check for +// __EXCEPTIONS && __has_feature(cxx_exceptions). +# define GTEST_HAS_EXCEPTIONS (__EXCEPTIONS && __has_feature(cxx_exceptions)) +# elif defined(__GNUC__) && __EXCEPTIONS +// gcc defines __EXCEPTIONS to 1 iff exceptions are enabled. +# define GTEST_HAS_EXCEPTIONS 1 +# elif defined(__SUNPRO_CC) +// Sun Pro CC supports exceptions. However, there is no compile-time way of +// detecting whether they are enabled or not. Therefore, we assume that +// they are enabled unless the user tells us otherwise. +# define GTEST_HAS_EXCEPTIONS 1 +# elif defined(__IBMCPP__) && __EXCEPTIONS +// xlC defines __EXCEPTIONS to 1 iff exceptions are enabled. +# define GTEST_HAS_EXCEPTIONS 1 +# elif defined(__HP_aCC) +// Exception handling is in effect by default in HP aCC compiler. It has to +// be turned of by +noeh compiler option if desired. +# define GTEST_HAS_EXCEPTIONS 1 +# else +// For other compilers, we assume exceptions are disabled to be +// conservative. +# define GTEST_HAS_EXCEPTIONS 0 +# endif // defined(_MSC_VER) || defined(__BORLANDC__) +#endif // GTEST_HAS_EXCEPTIONS + +#if !defined(GTEST_HAS_STD_STRING) +// Even though we don't use this macro any longer, we keep it in case +// some clients still depend on it. +# define GTEST_HAS_STD_STRING 1 +#elif !GTEST_HAS_STD_STRING +// The user told us that ::std::string isn't available. +# error "::std::string isn't available." +#endif // !defined(GTEST_HAS_STD_STRING) + +#ifndef GTEST_HAS_GLOBAL_STRING +# define GTEST_HAS_GLOBAL_STRING 0 +#endif // GTEST_HAS_GLOBAL_STRING + +#ifndef GTEST_HAS_STD_WSTRING +// The user didn't tell us whether ::std::wstring is available, so we need +// to figure it out. +// FIXME: uses autoconf to detect whether ::std::wstring +// is available. + +// Cygwin 1.7 and below doesn't support ::std::wstring. +// Solaris' libc++ doesn't support it either. Android has +// no support for it at least as recent as Froyo (2.2). +# define GTEST_HAS_STD_WSTRING \ + (!(GTEST_OS_LINUX_ANDROID || GTEST_OS_CYGWIN || GTEST_OS_SOLARIS)) + +#endif // GTEST_HAS_STD_WSTRING + +#ifndef GTEST_HAS_GLOBAL_WSTRING +// The user didn't tell us whether ::wstring is available, so we need +// to figure it out. +# define GTEST_HAS_GLOBAL_WSTRING \ + (GTEST_HAS_STD_WSTRING && GTEST_HAS_GLOBAL_STRING) +#endif // GTEST_HAS_GLOBAL_WSTRING + +// Determines whether RTTI is available. +#ifndef GTEST_HAS_RTTI +// The user didn't tell us whether RTTI is enabled, so we need to +// figure it out. + +# ifdef _MSC_VER + +# ifdef _CPPRTTI // MSVC defines this macro iff RTTI is enabled. +# define GTEST_HAS_RTTI 1 +# else +# define GTEST_HAS_RTTI 0 +# endif + +// Starting with version 4.3.2, gcc defines __GXX_RTTI iff RTTI is enabled. +# elif defined(__GNUC__) && (GTEST_GCC_VER_ >= 40302) + +# ifdef __GXX_RTTI +// When building against STLport with the Android NDK and with +// -frtti -fno-exceptions, the build fails at link time with undefined +// references to __cxa_bad_typeid. Note sure if STL or toolchain bug, +// so disable RTTI when detected. +# if GTEST_OS_LINUX_ANDROID && defined(_STLPORT_MAJOR) && \ + !defined(__EXCEPTIONS) +# define GTEST_HAS_RTTI 0 +# else +# define GTEST_HAS_RTTI 1 +# endif // GTEST_OS_LINUX_ANDROID && __STLPORT_MAJOR && !__EXCEPTIONS +# else +# define GTEST_HAS_RTTI 0 +# endif // __GXX_RTTI + +// Clang defines __GXX_RTTI starting with version 3.0, but its manual recommends +// using has_feature instead. has_feature(cxx_rtti) is supported since 2.7, the +// first version with C++ support. +# elif defined(__clang__) + +# define GTEST_HAS_RTTI __has_feature(cxx_rtti) + +// Starting with version 9.0 IBM Visual Age defines __RTTI_ALL__ to 1 if +// both the typeid and dynamic_cast features are present. +# elif defined(__IBMCPP__) && (__IBMCPP__ >= 900) + +# ifdef __RTTI_ALL__ +# define GTEST_HAS_RTTI 1 +# else +# define GTEST_HAS_RTTI 0 +# endif + +# else + +// For all other compilers, we assume RTTI is enabled. +# define GTEST_HAS_RTTI 1 + +# endif // _MSC_VER + +#endif // GTEST_HAS_RTTI + +// It's this header's responsibility to #include when RTTI +// is enabled. +#if GTEST_HAS_RTTI +# include +#endif + +// Determines whether Google Test can use the pthreads library. +#ifndef GTEST_HAS_PTHREAD +// The user didn't tell us explicitly, so we make reasonable assumptions about +// which platforms have pthreads support. +// +// To disable threading support in Google Test, add -DGTEST_HAS_PTHREAD=0 +// to your compiler flags. +#define GTEST_HAS_PTHREAD \ + (GTEST_OS_LINUX || GTEST_OS_MAC || GTEST_OS_HPUX || GTEST_OS_QNX || \ + GTEST_OS_FREEBSD || GTEST_OS_NACL || GTEST_OS_NETBSD || GTEST_OS_FUCHSIA) +#endif // GTEST_HAS_PTHREAD + +#if GTEST_HAS_PTHREAD +// gtest-port.h guarantees to #include when GTEST_HAS_PTHREAD is +// true. +# include // NOLINT + +// For timespec and nanosleep, used below. +# include // NOLINT +#endif + +// Determines if hash_map/hash_set are available. +// Only used for testing against those containers. +#if !defined(GTEST_HAS_HASH_MAP_) +# if defined(_MSC_VER) && (_MSC_VER < 1900) +# define GTEST_HAS_HASH_MAP_ 1 // Indicates that hash_map is available. +# define GTEST_HAS_HASH_SET_ 1 // Indicates that hash_set is available. +# endif // _MSC_VER +#endif // !defined(GTEST_HAS_HASH_MAP_) + +// Determines whether Google Test can use tr1/tuple. You can define +// this macro to 0 to prevent Google Test from using tuple (any +// feature depending on tuple with be disabled in this mode). +#ifndef GTEST_HAS_TR1_TUPLE +# if GTEST_OS_LINUX_ANDROID && defined(_STLPORT_MAJOR) +// STLport, provided with the Android NDK, has neither or . +# define GTEST_HAS_TR1_TUPLE 0 +# elif defined(_MSC_VER) && (_MSC_VER >= 1910) +// Prevent `warning C4996: 'std::tr1': warning STL4002: +// The non-Standard std::tr1 namespace and TR1-only machinery +// are deprecated and will be REMOVED.` +# define GTEST_HAS_TR1_TUPLE 0 +# elif GTEST_LANG_CXX11 && defined(_LIBCPP_VERSION) +// libc++ doesn't support TR1. +# define GTEST_HAS_TR1_TUPLE 0 +# else +// The user didn't tell us not to do it, so we assume it's OK. +# define GTEST_HAS_TR1_TUPLE 1 +# endif +#endif // GTEST_HAS_TR1_TUPLE + +// Determines whether Google Test's own tr1 tuple implementation +// should be used. +#ifndef GTEST_USE_OWN_TR1_TUPLE +// We use our own tuple implementation on Symbian. +# if GTEST_OS_SYMBIAN +# define GTEST_USE_OWN_TR1_TUPLE 1 +# else +// The user didn't tell us, so we need to figure it out. + +// We use our own TR1 tuple if we aren't sure the user has an +// implementation of it already. At this time, libstdc++ 4.0.0+ and +// MSVC 2010 are the only mainstream standard libraries that come +// with a TR1 tuple implementation. NVIDIA's CUDA NVCC compiler +// pretends to be GCC by defining __GNUC__ and friends, but cannot +// compile GCC's tuple implementation. MSVC 2008 (9.0) provides TR1 +// tuple in a 323 MB Feature Pack download, which we cannot assume the +// user has. QNX's QCC compiler is a modified GCC but it doesn't +// support TR1 tuple. libc++ only provides std::tuple, in C++11 mode, +// and it can be used with some compilers that define __GNUC__. +# if (defined(__GNUC__) && !defined(__CUDACC__) && (GTEST_GCC_VER_ >= 40000) \ + && !GTEST_OS_QNX && !defined(_LIBCPP_VERSION)) \ + || (_MSC_VER >= 1600 && _MSC_VER < 1900) +# define GTEST_ENV_HAS_TR1_TUPLE_ 1 +# endif + +// C++11 specifies that provides std::tuple. Use that if gtest is used +// in C++11 mode and libstdc++ isn't very old (binaries targeting OS X 10.6 +// can build with clang but need to use gcc4.2's libstdc++). +# if GTEST_LANG_CXX11 && (!defined(__GLIBCXX__) || __GLIBCXX__ > 20110325) +# define GTEST_ENV_HAS_STD_TUPLE_ 1 +# endif + +# if GTEST_ENV_HAS_TR1_TUPLE_ || GTEST_ENV_HAS_STD_TUPLE_ +# define GTEST_USE_OWN_TR1_TUPLE 0 +# else +# define GTEST_USE_OWN_TR1_TUPLE 1 +# endif +# endif // GTEST_OS_SYMBIAN +#endif // GTEST_USE_OWN_TR1_TUPLE + +// To avoid conditional compilation we make it gtest-port.h's responsibility +// to #include the header implementing tuple. +#if GTEST_HAS_STD_TUPLE_ +# include // IWYU pragma: export +# define GTEST_TUPLE_NAMESPACE_ ::std +#endif // GTEST_HAS_STD_TUPLE_ + +// We include tr1::tuple even if std::tuple is available to define printers for +// them. +#if GTEST_HAS_TR1_TUPLE +# ifndef GTEST_TUPLE_NAMESPACE_ +# define GTEST_TUPLE_NAMESPACE_ ::std::tr1 +# endif // GTEST_TUPLE_NAMESPACE_ + +# if GTEST_USE_OWN_TR1_TUPLE +// This file was GENERATED by command: +// pump.py gtest-tuple.h.pump +// DO NOT EDIT BY HAND!!! + +// Copyright 2009 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +// Implements a subset of TR1 tuple needed by Google Test and Google Mock. + +// GOOGLETEST_CM0001 DO NOT DELETE + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_ + +#include // For ::std::pair. + +// The compiler used in Symbian has a bug that prevents us from declaring the +// tuple template as a friend (it complains that tuple is redefined). This +// bypasses the bug by declaring the members that should otherwise be +// private as public. +// Sun Studio versions < 12 also have the above bug. +#if defined(__SYMBIAN32__) || (defined(__SUNPRO_CC) && __SUNPRO_CC < 0x590) +# define GTEST_DECLARE_TUPLE_AS_FRIEND_ public: +#else +# define GTEST_DECLARE_TUPLE_AS_FRIEND_ \ + template friend class tuple; \ + private: +#endif + +// Visual Studio 2010, 2012, and 2013 define symbols in std::tr1 that conflict +// with our own definitions. Therefore using our own tuple does not work on +// those compilers. +#if defined(_MSC_VER) && _MSC_VER >= 1600 /* 1600 is Visual Studio 2010 */ +# error "gtest's tuple doesn't compile on Visual Studio 2010 or later. \ +GTEST_USE_OWN_TR1_TUPLE must be set to 0 on those compilers." +#endif + +// GTEST_n_TUPLE_(T) is the type of an n-tuple. +#define GTEST_0_TUPLE_(T) tuple<> +#define GTEST_1_TUPLE_(T) tuple +#define GTEST_2_TUPLE_(T) tuple +#define GTEST_3_TUPLE_(T) tuple +#define GTEST_4_TUPLE_(T) tuple +#define GTEST_5_TUPLE_(T) tuple +#define GTEST_6_TUPLE_(T) tuple +#define GTEST_7_TUPLE_(T) tuple +#define GTEST_8_TUPLE_(T) tuple +#define GTEST_9_TUPLE_(T) tuple +#define GTEST_10_TUPLE_(T) tuple + +// GTEST_n_TYPENAMES_(T) declares a list of n typenames. +#define GTEST_0_TYPENAMES_(T) +#define GTEST_1_TYPENAMES_(T) typename T##0 +#define GTEST_2_TYPENAMES_(T) typename T##0, typename T##1 +#define GTEST_3_TYPENAMES_(T) typename T##0, typename T##1, typename T##2 +#define GTEST_4_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3 +#define GTEST_5_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4 +#define GTEST_6_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4, typename T##5 +#define GTEST_7_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4, typename T##5, typename T##6 +#define GTEST_8_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4, typename T##5, typename T##6, typename T##7 +#define GTEST_9_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4, typename T##5, typename T##6, \ + typename T##7, typename T##8 +#define GTEST_10_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4, typename T##5, typename T##6, \ + typename T##7, typename T##8, typename T##9 + +// In theory, defining stuff in the ::std namespace is undefined +// behavior. We can do this as we are playing the role of a standard +// library vendor. +namespace std { +namespace tr1 { + +template +class tuple; + +// Anything in namespace gtest_internal is Google Test's INTERNAL +// IMPLEMENTATION DETAIL and MUST NOT BE USED DIRECTLY in user code. +namespace gtest_internal { + +// ByRef::type is T if T is a reference; otherwise it's const T&. +template +struct ByRef { typedef const T& type; }; // NOLINT +template +struct ByRef { typedef T& type; }; // NOLINT + +// A handy wrapper for ByRef. +#define GTEST_BY_REF_(T) typename ::std::tr1::gtest_internal::ByRef::type + +// AddRef::type is T if T is a reference; otherwise it's T&. This +// is the same as tr1::add_reference::type. +template +struct AddRef { typedef T& type; }; // NOLINT +template +struct AddRef { typedef T& type; }; // NOLINT + +// A handy wrapper for AddRef. +#define GTEST_ADD_REF_(T) typename ::std::tr1::gtest_internal::AddRef::type + +// A helper for implementing get(). +template class Get; + +// A helper for implementing tuple_element. kIndexValid is true +// iff k < the number of fields in tuple type T. +template +struct TupleElement; + +template +struct TupleElement { + typedef T0 type; +}; + +template +struct TupleElement { + typedef T1 type; +}; + +template +struct TupleElement { + typedef T2 type; +}; + +template +struct TupleElement { + typedef T3 type; +}; + +template +struct TupleElement { + typedef T4 type; +}; + +template +struct TupleElement { + typedef T5 type; +}; + +template +struct TupleElement { + typedef T6 type; +}; + +template +struct TupleElement { + typedef T7 type; +}; + +template +struct TupleElement { + typedef T8 type; +}; + +template +struct TupleElement { + typedef T9 type; +}; + +} // namespace gtest_internal + +template <> +class tuple<> { + public: + tuple() {} + tuple(const tuple& /* t */) {} + tuple& operator=(const tuple& /* t */) { return *this; } +}; + +template +class GTEST_1_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0) : f0_(f0) {} + + tuple(const tuple& t) : f0_(t.f0_) {} + + template + tuple(const GTEST_1_TUPLE_(U)& t) : f0_(t.f0_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_1_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_1_TUPLE_(U)& t) { + f0_ = t.f0_; + return *this; + } + + T0 f0_; +}; + +template +class GTEST_2_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1) : f0_(f0), + f1_(f1) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_) {} + + template + tuple(const GTEST_2_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_) {} + template + tuple(const ::std::pair& p) : f0_(p.first), f1_(p.second) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_2_TUPLE_(U)& t) { + return CopyFrom(t); + } + template + tuple& operator=(const ::std::pair& p) { + f0_ = p.first; + f1_ = p.second; + return *this; + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_2_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + return *this; + } + + T0 f0_; + T1 f1_; +}; + +template +class GTEST_3_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2) : f0_(f0), f1_(f1), f2_(f2) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_) {} + + template + tuple(const GTEST_3_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_3_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_3_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; +}; + +template +class GTEST_4_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3) : f0_(f0), f1_(f1), f2_(f2), + f3_(f3) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_) {} + + template + tuple(const GTEST_4_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_4_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_4_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; +}; + +template +class GTEST_5_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, + GTEST_BY_REF_(T4) f4) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_) {} + + template + tuple(const GTEST_5_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_5_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_5_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; +}; + +template +class GTEST_6_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4, + GTEST_BY_REF_(T5) f5) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4), + f5_(f5) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_), f5_(t.f5_) {} + + template + tuple(const GTEST_6_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_), f5_(t.f5_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_6_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_6_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + f5_ = t.f5_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; + T5 f5_; +}; + +template +class GTEST_7_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4, + GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6) : f0_(f0), f1_(f1), f2_(f2), + f3_(f3), f4_(f4), f5_(f5), f6_(f6) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_), f5_(t.f5_), f6_(t.f6_) {} + + template + tuple(const GTEST_7_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_7_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_7_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + f5_ = t.f5_; + f6_ = t.f6_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; + T5 f5_; + T6 f6_; +}; + +template +class GTEST_8_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_(), f7_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4, + GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6, + GTEST_BY_REF_(T7) f7) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4), + f5_(f5), f6_(f6), f7_(f7) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_) {} + + template + tuple(const GTEST_8_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_8_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_8_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + f5_ = t.f5_; + f6_ = t.f6_; + f7_ = t.f7_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; + T5 f5_; + T6 f6_; + T7 f7_; +}; + +template +class GTEST_9_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_(), f7_(), f8_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4, + GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6, GTEST_BY_REF_(T7) f7, + GTEST_BY_REF_(T8) f8) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4), + f5_(f5), f6_(f6), f7_(f7), f8_(f8) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_) {} + + template + tuple(const GTEST_9_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_9_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_9_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + f5_ = t.f5_; + f6_ = t.f6_; + f7_ = t.f7_; + f8_ = t.f8_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; + T5 f5_; + T6 f6_; + T7 f7_; + T8 f8_; +}; + +template +class tuple { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_(), f7_(), f8_(), + f9_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4, + GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6, GTEST_BY_REF_(T7) f7, + GTEST_BY_REF_(T8) f8, GTEST_BY_REF_(T9) f9) : f0_(f0), f1_(f1), f2_(f2), + f3_(f3), f4_(f4), f5_(f5), f6_(f6), f7_(f7), f8_(f8), f9_(f9) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_), f9_(t.f9_) {} + + template + tuple(const GTEST_10_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_), + f9_(t.f9_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_10_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_10_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + f5_ = t.f5_; + f6_ = t.f6_; + f7_ = t.f7_; + f8_ = t.f8_; + f9_ = t.f9_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; + T5 f5_; + T6 f6_; + T7 f7_; + T8 f8_; + T9 f9_; +}; + +// 6.1.3.2 Tuple creation functions. + +// Known limitations: we don't support passing an +// std::tr1::reference_wrapper to make_tuple(). And we don't +// implement tie(). + +inline tuple<> make_tuple() { return tuple<>(); } + +template +inline GTEST_1_TUPLE_(T) make_tuple(const T0& f0) { + return GTEST_1_TUPLE_(T)(f0); +} + +template +inline GTEST_2_TUPLE_(T) make_tuple(const T0& f0, const T1& f1) { + return GTEST_2_TUPLE_(T)(f0, f1); +} + +template +inline GTEST_3_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2) { + return GTEST_3_TUPLE_(T)(f0, f1, f2); +} + +template +inline GTEST_4_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3) { + return GTEST_4_TUPLE_(T)(f0, f1, f2, f3); +} + +template +inline GTEST_5_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4) { + return GTEST_5_TUPLE_(T)(f0, f1, f2, f3, f4); +} + +template +inline GTEST_6_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4, const T5& f5) { + return GTEST_6_TUPLE_(T)(f0, f1, f2, f3, f4, f5); +} + +template +inline GTEST_7_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4, const T5& f5, const T6& f6) { + return GTEST_7_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6); +} + +template +inline GTEST_8_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4, const T5& f5, const T6& f6, const T7& f7) { + return GTEST_8_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6, f7); +} + +template +inline GTEST_9_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4, const T5& f5, const T6& f6, const T7& f7, + const T8& f8) { + return GTEST_9_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6, f7, f8); +} + +template +inline GTEST_10_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4, const T5& f5, const T6& f6, const T7& f7, + const T8& f8, const T9& f9) { + return GTEST_10_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6, f7, f8, f9); +} + +// 6.1.3.3 Tuple helper classes. + +template struct tuple_size; + +template +struct tuple_size { + static const int value = 0; +}; + +template +struct tuple_size { + static const int value = 1; +}; + +template +struct tuple_size { + static const int value = 2; +}; + +template +struct tuple_size { + static const int value = 3; +}; + +template +struct tuple_size { + static const int value = 4; +}; + +template +struct tuple_size { + static const int value = 5; +}; + +template +struct tuple_size { + static const int value = 6; +}; + +template +struct tuple_size { + static const int value = 7; +}; + +template +struct tuple_size { + static const int value = 8; +}; + +template +struct tuple_size { + static const int value = 9; +}; + +template +struct tuple_size { + static const int value = 10; +}; + +template +struct tuple_element { + typedef typename gtest_internal::TupleElement< + k < (tuple_size::value), k, Tuple>::type type; +}; + +#define GTEST_TUPLE_ELEMENT_(k, Tuple) typename tuple_element::type + +// 6.1.3.4 Element access. + +namespace gtest_internal { + +template <> +class Get<0> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(0, Tuple)) + Field(Tuple& t) { return t.f0_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(0, Tuple)) + ConstField(const Tuple& t) { return t.f0_; } +}; + +template <> +class Get<1> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(1, Tuple)) + Field(Tuple& t) { return t.f1_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(1, Tuple)) + ConstField(const Tuple& t) { return t.f1_; } +}; + +template <> +class Get<2> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(2, Tuple)) + Field(Tuple& t) { return t.f2_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(2, Tuple)) + ConstField(const Tuple& t) { return t.f2_; } +}; + +template <> +class Get<3> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(3, Tuple)) + Field(Tuple& t) { return t.f3_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(3, Tuple)) + ConstField(const Tuple& t) { return t.f3_; } +}; + +template <> +class Get<4> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(4, Tuple)) + Field(Tuple& t) { return t.f4_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(4, Tuple)) + ConstField(const Tuple& t) { return t.f4_; } +}; + +template <> +class Get<5> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(5, Tuple)) + Field(Tuple& t) { return t.f5_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(5, Tuple)) + ConstField(const Tuple& t) { return t.f5_; } +}; + +template <> +class Get<6> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(6, Tuple)) + Field(Tuple& t) { return t.f6_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(6, Tuple)) + ConstField(const Tuple& t) { return t.f6_; } +}; + +template <> +class Get<7> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(7, Tuple)) + Field(Tuple& t) { return t.f7_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(7, Tuple)) + ConstField(const Tuple& t) { return t.f7_; } +}; + +template <> +class Get<8> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(8, Tuple)) + Field(Tuple& t) { return t.f8_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(8, Tuple)) + ConstField(const Tuple& t) { return t.f8_; } +}; + +template <> +class Get<9> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(9, Tuple)) + Field(Tuple& t) { return t.f9_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(9, Tuple)) + ConstField(const Tuple& t) { return t.f9_; } +}; + +} // namespace gtest_internal + +template +GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(k, GTEST_10_TUPLE_(T))) +get(GTEST_10_TUPLE_(T)& t) { + return gtest_internal::Get::Field(t); +} + +template +GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(k, GTEST_10_TUPLE_(T))) +get(const GTEST_10_TUPLE_(T)& t) { + return gtest_internal::Get::ConstField(t); +} + +// 6.1.3.5 Relational operators + +// We only implement == and !=, as we don't have a need for the rest yet. + +namespace gtest_internal { + +// SameSizeTuplePrefixComparator::Eq(t1, t2) returns true if the +// first k fields of t1 equals the first k fields of t2. +// SameSizeTuplePrefixComparator(k1, k2) would be a compiler error if +// k1 != k2. +template +struct SameSizeTuplePrefixComparator; + +template <> +struct SameSizeTuplePrefixComparator<0, 0> { + template + static bool Eq(const Tuple1& /* t1 */, const Tuple2& /* t2 */) { + return true; + } +}; + +template +struct SameSizeTuplePrefixComparator { + template + static bool Eq(const Tuple1& t1, const Tuple2& t2) { + return SameSizeTuplePrefixComparator::Eq(t1, t2) && + ::std::tr1::get(t1) == ::std::tr1::get(t2); + } +}; + +} // namespace gtest_internal + +template +inline bool operator==(const GTEST_10_TUPLE_(T)& t, + const GTEST_10_TUPLE_(U)& u) { + return gtest_internal::SameSizeTuplePrefixComparator< + tuple_size::value, + tuple_size::value>::Eq(t, u); +} + +template +inline bool operator!=(const GTEST_10_TUPLE_(T)& t, + const GTEST_10_TUPLE_(U)& u) { return !(t == u); } + +// 6.1.4 Pairs. +// Unimplemented. + +} // namespace tr1 +} // namespace std + +#undef GTEST_0_TUPLE_ +#undef GTEST_1_TUPLE_ +#undef GTEST_2_TUPLE_ +#undef GTEST_3_TUPLE_ +#undef GTEST_4_TUPLE_ +#undef GTEST_5_TUPLE_ +#undef GTEST_6_TUPLE_ +#undef GTEST_7_TUPLE_ +#undef GTEST_8_TUPLE_ +#undef GTEST_9_TUPLE_ +#undef GTEST_10_TUPLE_ + +#undef GTEST_0_TYPENAMES_ +#undef GTEST_1_TYPENAMES_ +#undef GTEST_2_TYPENAMES_ +#undef GTEST_3_TYPENAMES_ +#undef GTEST_4_TYPENAMES_ +#undef GTEST_5_TYPENAMES_ +#undef GTEST_6_TYPENAMES_ +#undef GTEST_7_TYPENAMES_ +#undef GTEST_8_TYPENAMES_ +#undef GTEST_9_TYPENAMES_ +#undef GTEST_10_TYPENAMES_ + +#undef GTEST_DECLARE_TUPLE_AS_FRIEND_ +#undef GTEST_BY_REF_ +#undef GTEST_ADD_REF_ +#undef GTEST_TUPLE_ELEMENT_ + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_ +# elif GTEST_OS_SYMBIAN + +// On Symbian, BOOST_HAS_TR1_TUPLE causes Boost's TR1 tuple library to +// use STLport's tuple implementation, which unfortunately doesn't +// work as the copy of STLport distributed with Symbian is incomplete. +// By making sure BOOST_HAS_TR1_TUPLE is undefined, we force Boost to +// use its own tuple implementation. +# ifdef BOOST_HAS_TR1_TUPLE +# undef BOOST_HAS_TR1_TUPLE +# endif // BOOST_HAS_TR1_TUPLE + +// This prevents , which defines +// BOOST_HAS_TR1_TUPLE, from being #included by Boost's . +# define BOOST_TR1_DETAIL_CONFIG_HPP_INCLUDED +# include // IWYU pragma: export // NOLINT + +# elif defined(__GNUC__) && (GTEST_GCC_VER_ >= 40000) +// GCC 4.0+ implements tr1/tuple in the header. This does +// not conform to the TR1 spec, which requires the header to be . + +# if !GTEST_HAS_RTTI && GTEST_GCC_VER_ < 40302 +// Until version 4.3.2, gcc has a bug that causes , +// which is #included by , to not compile when RTTI is +// disabled. _TR1_FUNCTIONAL is the header guard for +// . Hence the following #define is used to prevent +// from being included. +# define _TR1_FUNCTIONAL 1 +# include +# undef _TR1_FUNCTIONAL // Allows the user to #include + // if they choose to. +# else +# include // NOLINT +# endif // !GTEST_HAS_RTTI && GTEST_GCC_VER_ < 40302 + +// VS 2010 now has tr1 support. +# elif _MSC_VER >= 1600 +# include // IWYU pragma: export // NOLINT + +# else // GTEST_USE_OWN_TR1_TUPLE +# include // IWYU pragma: export // NOLINT +# endif // GTEST_USE_OWN_TR1_TUPLE + +#endif // GTEST_HAS_TR1_TUPLE + +// Determines whether clone(2) is supported. +// Usually it will only be available on Linux, excluding +// Linux on the Itanium architecture. +// Also see http://linux.die.net/man/2/clone. +#ifndef GTEST_HAS_CLONE +// The user didn't tell us, so we need to figure it out. + +# if GTEST_OS_LINUX && !defined(__ia64__) +# if GTEST_OS_LINUX_ANDROID +// On Android, clone() became available at different API levels for each 32-bit +// architecture. +# if defined(__LP64__) || \ + (defined(__arm__) && __ANDROID_API__ >= 9) || \ + (defined(__mips__) && __ANDROID_API__ >= 12) || \ + (defined(__i386__) && __ANDROID_API__ >= 17) +# define GTEST_HAS_CLONE 1 +# else +# define GTEST_HAS_CLONE 0 +# endif +# else +# define GTEST_HAS_CLONE 1 +# endif +# else +# define GTEST_HAS_CLONE 0 +# endif // GTEST_OS_LINUX && !defined(__ia64__) + +#endif // GTEST_HAS_CLONE + +// Determines whether to support stream redirection. This is used to test +// output correctness and to implement death tests. +#ifndef GTEST_HAS_STREAM_REDIRECTION +// By default, we assume that stream redirection is supported on all +// platforms except known mobile ones. +# if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN || \ + GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT +# define GTEST_HAS_STREAM_REDIRECTION 0 +# else +# define GTEST_HAS_STREAM_REDIRECTION 1 +# endif // !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_SYMBIAN +#endif // GTEST_HAS_STREAM_REDIRECTION + +// Determines whether to support death tests. +// Google Test does not support death tests for VC 7.1 and earlier as +// abort() in a VC 7.1 application compiled as GUI in debug config +// pops up a dialog window that cannot be suppressed programmatically. +#if (GTEST_OS_LINUX || GTEST_OS_CYGWIN || GTEST_OS_SOLARIS || \ + (GTEST_OS_MAC && !GTEST_OS_IOS) || \ + (GTEST_OS_WINDOWS_DESKTOP && _MSC_VER >= 1400) || \ + GTEST_OS_WINDOWS_MINGW || GTEST_OS_AIX || GTEST_OS_HPUX || \ + GTEST_OS_OPENBSD || GTEST_OS_QNX || GTEST_OS_FREEBSD || \ + GTEST_OS_NETBSD || GTEST_OS_FUCHSIA) +# define GTEST_HAS_DEATH_TEST 1 +#endif + +// Determines whether to support type-driven tests. + +// Typed tests need and variadic macros, which GCC, VC++ 8.0, +// Sun Pro CC, IBM Visual Age, and HP aCC support. +#if defined(__GNUC__) || (_MSC_VER >= 1400) || defined(__SUNPRO_CC) || \ + defined(__IBMCPP__) || defined(__HP_aCC) +# define GTEST_HAS_TYPED_TEST 1 +# define GTEST_HAS_TYPED_TEST_P 1 +#endif + +// Determines whether to support Combine(). This only makes sense when +// value-parameterized tests are enabled. The implementation doesn't +// work on Sun Studio since it doesn't understand templated conversion +// operators. +#if (GTEST_HAS_TR1_TUPLE || GTEST_HAS_STD_TUPLE_) && !defined(__SUNPRO_CC) +# define GTEST_HAS_COMBINE 1 +#endif + +// Determines whether the system compiler uses UTF-16 for encoding wide strings. +#define GTEST_WIDE_STRING_USES_UTF16_ \ + (GTEST_OS_WINDOWS || GTEST_OS_CYGWIN || GTEST_OS_SYMBIAN || GTEST_OS_AIX) + +// Determines whether test results can be streamed to a socket. +#if GTEST_OS_LINUX +# define GTEST_CAN_STREAM_RESULTS_ 1 +#endif + +// Defines some utility macros. + +// The GNU compiler emits a warning if nested "if" statements are followed by +// an "else" statement and braces are not used to explicitly disambiguate the +// "else" binding. This leads to problems with code like: +// +// if (gate) +// ASSERT_*(condition) << "Some message"; +// +// The "switch (0) case 0:" idiom is used to suppress this. +#ifdef __INTEL_COMPILER +# define GTEST_AMBIGUOUS_ELSE_BLOCKER_ +#else +# define GTEST_AMBIGUOUS_ELSE_BLOCKER_ switch (0) case 0: default: // NOLINT +#endif + +// Use this annotation at the end of a struct/class definition to +// prevent the compiler from optimizing away instances that are never +// used. This is useful when all interesting logic happens inside the +// c'tor and / or d'tor. Example: +// +// struct Foo { +// Foo() { ... } +// } GTEST_ATTRIBUTE_UNUSED_; +// +// Also use it after a variable or parameter declaration to tell the +// compiler the variable/parameter does not have to be used. +#if defined(__GNUC__) && !defined(COMPILER_ICC) +# define GTEST_ATTRIBUTE_UNUSED_ __attribute__ ((unused)) +#elif defined(__clang__) +# if __has_attribute(unused) +# define GTEST_ATTRIBUTE_UNUSED_ __attribute__ ((unused)) +# endif +#endif +#ifndef GTEST_ATTRIBUTE_UNUSED_ +# define GTEST_ATTRIBUTE_UNUSED_ +#endif + +#if GTEST_LANG_CXX11 +# define GTEST_CXX11_EQUALS_DELETE_ = delete +#else // GTEST_LANG_CXX11 +# define GTEST_CXX11_EQUALS_DELETE_ +#endif // GTEST_LANG_CXX11 + +// Use this annotation before a function that takes a printf format string. +#if (defined(__GNUC__) || defined(__clang__)) && !defined(COMPILER_ICC) +# if defined(__MINGW_PRINTF_FORMAT) +// MinGW has two different printf implementations. Ensure the format macro +// matches the selected implementation. See +// https://sourceforge.net/p/mingw-w64/wiki2/gnu%20printf/. +# define GTEST_ATTRIBUTE_PRINTF_(string_index, first_to_check) \ + __attribute__((__format__(__MINGW_PRINTF_FORMAT, string_index, \ + first_to_check))) +# else +# define GTEST_ATTRIBUTE_PRINTF_(string_index, first_to_check) \ + __attribute__((__format__(__printf__, string_index, first_to_check))) +# endif +#else +# define GTEST_ATTRIBUTE_PRINTF_(string_index, first_to_check) +#endif + + +// A macro to disallow operator= +// This should be used in the private: declarations for a class. +#define GTEST_DISALLOW_ASSIGN_(type) \ + void operator=(type const &) GTEST_CXX11_EQUALS_DELETE_ + +// A macro to disallow copy constructor and operator= +// This should be used in the private: declarations for a class. +#define GTEST_DISALLOW_COPY_AND_ASSIGN_(type) \ + type(type const &) GTEST_CXX11_EQUALS_DELETE_; \ + GTEST_DISALLOW_ASSIGN_(type) + +// Tell the compiler to warn about unused return values for functions declared +// with this macro. The macro should be used on function declarations +// following the argument list: +// +// Sprocket* AllocateSprocket() GTEST_MUST_USE_RESULT_; +#if defined(__GNUC__) && (GTEST_GCC_VER_ >= 30400) && !defined(COMPILER_ICC) +# define GTEST_MUST_USE_RESULT_ __attribute__ ((warn_unused_result)) +#else +# define GTEST_MUST_USE_RESULT_ +#endif // __GNUC__ && (GTEST_GCC_VER_ >= 30400) && !COMPILER_ICC + +// MS C++ compiler emits warning when a conditional expression is compile time +// constant. In some contexts this warning is false positive and needs to be +// suppressed. Use the following two macros in such cases: +// +// GTEST_INTENTIONAL_CONST_COND_PUSH_() +// while (true) { +// GTEST_INTENTIONAL_CONST_COND_POP_() +// } +# define GTEST_INTENTIONAL_CONST_COND_PUSH_() \ + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4127) +# define GTEST_INTENTIONAL_CONST_COND_POP_() \ + GTEST_DISABLE_MSC_WARNINGS_POP_() + +// Determine whether the compiler supports Microsoft's Structured Exception +// Handling. This is supported by several Windows compilers but generally +// does not exist on any other system. +#ifndef GTEST_HAS_SEH +// The user didn't tell us, so we need to figure it out. + +# if defined(_MSC_VER) || defined(__BORLANDC__) +// These two compilers are known to support SEH. +# define GTEST_HAS_SEH 1 +# else +// Assume no SEH. +# define GTEST_HAS_SEH 0 +# endif + +#define GTEST_IS_THREADSAFE \ + (GTEST_HAS_MUTEX_AND_THREAD_LOCAL_ \ + || (GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT) \ + || GTEST_HAS_PTHREAD) + +#endif // GTEST_HAS_SEH + +// GTEST_API_ qualifies all symbols that must be exported. The definitions below +// are guarded by #ifndef to give embedders a chance to define GTEST_API_ in +// gtest/internal/custom/gtest-port.h +#ifndef GTEST_API_ + +#ifdef _MSC_VER +# if GTEST_LINKED_AS_SHARED_LIBRARY +# define GTEST_API_ __declspec(dllimport) +# elif GTEST_CREATE_SHARED_LIBRARY +# define GTEST_API_ __declspec(dllexport) +# endif +#elif __GNUC__ >= 4 || defined(__clang__) +# define GTEST_API_ __attribute__((visibility ("default"))) +#endif // _MSC_VER + +#endif // GTEST_API_ + +#ifndef GTEST_API_ +# define GTEST_API_ +#endif // GTEST_API_ + +#ifndef GTEST_DEFAULT_DEATH_TEST_STYLE +# define GTEST_DEFAULT_DEATH_TEST_STYLE "fast" +#endif // GTEST_DEFAULT_DEATH_TEST_STYLE + +#ifdef __GNUC__ +// Ask the compiler to never inline a given function. +# define GTEST_NO_INLINE_ __attribute__((noinline)) +#else +# define GTEST_NO_INLINE_ +#endif + +// _LIBCPP_VERSION is defined by the libc++ library from the LLVM project. +#if !defined(GTEST_HAS_CXXABI_H_) +# if defined(__GLIBCXX__) || (defined(_LIBCPP_VERSION) && !defined(_MSC_VER)) +# define GTEST_HAS_CXXABI_H_ 1 +# else +# define GTEST_HAS_CXXABI_H_ 0 +# endif +#endif + +// A function level attribute to disable checking for use of uninitialized +// memory when built with MemorySanitizer. +#if defined(__clang__) +# if __has_feature(memory_sanitizer) +# define GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ \ + __attribute__((no_sanitize_memory)) +# else +# define GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ +# endif // __has_feature(memory_sanitizer) +#else +# define GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ +#endif // __clang__ + +// A function level attribute to disable AddressSanitizer instrumentation. +#if defined(__clang__) +# if __has_feature(address_sanitizer) +# define GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ \ + __attribute__((no_sanitize_address)) +# else +# define GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +# endif // __has_feature(address_sanitizer) +#else +# define GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +#endif // __clang__ + +// A function level attribute to disable ThreadSanitizer instrumentation. +#if defined(__clang__) +# if __has_feature(thread_sanitizer) +# define GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ \ + __attribute__((no_sanitize_thread)) +# else +# define GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ +# endif // __has_feature(thread_sanitizer) +#else +# define GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ +#endif // __clang__ + +namespace testing { + +class Message; + +#if defined(GTEST_TUPLE_NAMESPACE_) +// Import tuple and friends into the ::testing namespace. +// It is part of our interface, having them in ::testing allows us to change +// their types as needed. +using GTEST_TUPLE_NAMESPACE_::get; +using GTEST_TUPLE_NAMESPACE_::make_tuple; +using GTEST_TUPLE_NAMESPACE_::tuple; +using GTEST_TUPLE_NAMESPACE_::tuple_size; +using GTEST_TUPLE_NAMESPACE_::tuple_element; +#endif // defined(GTEST_TUPLE_NAMESPACE_) + +namespace internal { + +// A secret type that Google Test users don't know about. It has no +// definition on purpose. Therefore it's impossible to create a +// Secret object, which is what we want. +class Secret; + +// The GTEST_COMPILE_ASSERT_ macro can be used to verify that a compile time +// expression is true. For example, you could use it to verify the +// size of a static array: +// +// GTEST_COMPILE_ASSERT_(GTEST_ARRAY_SIZE_(names) == NUM_NAMES, +// names_incorrect_size); +// +// or to make sure a struct is smaller than a certain size: +// +// GTEST_COMPILE_ASSERT_(sizeof(foo) < 128, foo_too_large); +// +// The second argument to the macro is the name of the variable. If +// the expression is false, most compilers will issue a warning/error +// containing the name of the variable. + +#if GTEST_LANG_CXX11 +# define GTEST_COMPILE_ASSERT_(expr, msg) static_assert(expr, #msg) +#else // !GTEST_LANG_CXX11 +template + struct CompileAssert { +}; + +# define GTEST_COMPILE_ASSERT_(expr, msg) \ + typedef ::testing::internal::CompileAssert<(static_cast(expr))> \ + msg[static_cast(expr) ? 1 : -1] GTEST_ATTRIBUTE_UNUSED_ +#endif // !GTEST_LANG_CXX11 + +// Implementation details of GTEST_COMPILE_ASSERT_: +// +// (In C++11, we simply use static_assert instead of the following) +// +// - GTEST_COMPILE_ASSERT_ works by defining an array type that has -1 +// elements (and thus is invalid) when the expression is false. +// +// - The simpler definition +// +// #define GTEST_COMPILE_ASSERT_(expr, msg) typedef char msg[(expr) ? 1 : -1] +// +// does not work, as gcc supports variable-length arrays whose sizes +// are determined at run-time (this is gcc's extension and not part +// of the C++ standard). As a result, gcc fails to reject the +// following code with the simple definition: +// +// int foo; +// GTEST_COMPILE_ASSERT_(foo, msg); // not supposed to compile as foo is +// // not a compile-time constant. +// +// - By using the type CompileAssert<(bool(expr))>, we ensures that +// expr is a compile-time constant. (Template arguments must be +// determined at compile-time.) +// +// - The outter parentheses in CompileAssert<(bool(expr))> are necessary +// to work around a bug in gcc 3.4.4 and 4.0.1. If we had written +// +// CompileAssert +// +// instead, these compilers will refuse to compile +// +// GTEST_COMPILE_ASSERT_(5 > 0, some_message); +// +// (They seem to think the ">" in "5 > 0" marks the end of the +// template argument list.) +// +// - The array size is (bool(expr) ? 1 : -1), instead of simply +// +// ((expr) ? 1 : -1). +// +// This is to avoid running into a bug in MS VC 7.1, which +// causes ((0.0) ? 1 : -1) to incorrectly evaluate to 1. + +// StaticAssertTypeEqHelper is used by StaticAssertTypeEq defined in gtest.h. +// +// This template is declared, but intentionally undefined. +template +struct StaticAssertTypeEqHelper; + +template +struct StaticAssertTypeEqHelper { + enum { value = true }; +}; + +// Same as std::is_same<>. +template +struct IsSame { + enum { value = false }; +}; +template +struct IsSame { + enum { value = true }; +}; + +// Evaluates to the number of elements in 'array'. +#define GTEST_ARRAY_SIZE_(array) (sizeof(array) / sizeof(array[0])) + +#if GTEST_HAS_GLOBAL_STRING +typedef ::string string; +#else +typedef ::std::string string; +#endif // GTEST_HAS_GLOBAL_STRING + +#if GTEST_HAS_GLOBAL_WSTRING +typedef ::wstring wstring; +#elif GTEST_HAS_STD_WSTRING +typedef ::std::wstring wstring; +#endif // GTEST_HAS_GLOBAL_WSTRING + +// A helper for suppressing warnings on constant condition. It just +// returns 'condition'. +GTEST_API_ bool IsTrue(bool condition); + +// Defines scoped_ptr. + +// RocksDB: use unique_ptr to work around some clang-analyze false reports +template +using scoped_ptr = std::unique_ptr; +/* +// This implementation of scoped_ptr is PARTIAL - it only contains +// enough stuff to satisfy Google Test's need. +template +class scoped_ptr { + public: + typedef T element_type; + + explicit scoped_ptr(T* p = NULL) : ptr_(p) {} + ~scoped_ptr() { reset(); } + + T& operator*() const { return *ptr_; } + T* operator->() const { return ptr_; } + T* get() const { return ptr_; } + + T* release() { + T* const ptr = ptr_; + ptr_ = NULL; + return ptr; + } + + void reset(T* p = NULL) { + if (p != ptr_) { + if (IsTrue(sizeof(T) > 0)) { // Makes sure T is a complete type. + delete ptr_; + } + ptr_ = p; + } + } + + friend void swap(scoped_ptr& a, scoped_ptr& b) { + using std::swap; + swap(a.ptr_, b.ptr_); + } + + private: + T* ptr_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(scoped_ptr); +}; +*/ + +// Defines RE. + +#if GTEST_USES_PCRE +// if used, PCRE is injected by custom/gtest-port.h +#elif GTEST_USES_POSIX_RE || GTEST_USES_SIMPLE_RE + +// A simple C++ wrapper for . It uses the POSIX Extended +// Regular Expression syntax. +class GTEST_API_ RE { + public: + // A copy constructor is required by the Standard to initialize object + // references from r-values. + RE(const RE& other) { Init(other.pattern()); } + + // Constructs an RE from a string. + RE(const ::std::string& regex) { Init(regex.c_str()); } // NOLINT + +# if GTEST_HAS_GLOBAL_STRING + + RE(const ::string& regex) { Init(regex.c_str()); } // NOLINT + +# endif // GTEST_HAS_GLOBAL_STRING + + RE(const char* regex) { Init(regex); } // NOLINT + ~RE(); + + // Returns the string representation of the regex. + const char* pattern() const { return pattern_; } + + // FullMatch(str, re) returns true iff regular expression re matches + // the entire str. + // PartialMatch(str, re) returns true iff regular expression re + // matches a substring of str (including str itself). + // + // FIXME: make FullMatch() and PartialMatch() work + // when str contains NUL characters. + static bool FullMatch(const ::std::string& str, const RE& re) { + return FullMatch(str.c_str(), re); + } + static bool PartialMatch(const ::std::string& str, const RE& re) { + return PartialMatch(str.c_str(), re); + } + +# if GTEST_HAS_GLOBAL_STRING + + static bool FullMatch(const ::string& str, const RE& re) { + return FullMatch(str.c_str(), re); + } + static bool PartialMatch(const ::string& str, const RE& re) { + return PartialMatch(str.c_str(), re); + } + +# endif // GTEST_HAS_GLOBAL_STRING + + static bool FullMatch(const char* str, const RE& re); + static bool PartialMatch(const char* str, const RE& re); + + private: + void Init(const char* regex); + + // We use a const char* instead of an std::string, as Google Test used to be + // used where std::string is not available. FIXME: change to + // std::string. + const char* pattern_; + bool is_valid_; + +# if GTEST_USES_POSIX_RE + + regex_t full_regex_; // For FullMatch(). + regex_t partial_regex_; // For PartialMatch(). + +# else // GTEST_USES_SIMPLE_RE + + const char* full_pattern_; // For FullMatch(); + +# endif + + GTEST_DISALLOW_ASSIGN_(RE); +}; + +#endif // GTEST_USES_PCRE + +// Formats a source file path and a line number as they would appear +// in an error message from the compiler used to compile this code. +GTEST_API_ ::std::string FormatFileLocation(const char* file, int line); + +// Formats a file location for compiler-independent XML output. +// Although this function is not platform dependent, we put it next to +// FormatFileLocation in order to contrast the two functions. +GTEST_API_ ::std::string FormatCompilerIndependentFileLocation(const char* file, + int line); + +// Defines logging utilities: +// GTEST_LOG_(severity) - logs messages at the specified severity level. The +// message itself is streamed into the macro. +// LogToStderr() - directs all log messages to stderr. +// FlushInfoLog() - flushes informational log messages. + +enum GTestLogSeverity { + GTEST_INFO, + GTEST_WARNING, + GTEST_ERROR, + GTEST_FATAL +}; + +// Formats log entry severity, provides a stream object for streaming the +// log message, and terminates the message with a newline when going out of +// scope. +class GTEST_API_ GTestLog { + public: + GTestLog(GTestLogSeverity severity, const char* file, int line); + + // Flushes the buffers and, if severity is GTEST_FATAL, aborts the program. + ~GTestLog(); + + ::std::ostream& GetStream() { return ::std::cerr; } + + private: + const GTestLogSeverity severity_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(GTestLog); +}; + +#if !defined(GTEST_LOG_) + +# define GTEST_LOG_(severity) \ + ::testing::internal::GTestLog(::testing::internal::GTEST_##severity, \ + __FILE__, __LINE__).GetStream() + +inline void LogToStderr() {} +inline void FlushInfoLog() { fflush(NULL); } + +#endif // !defined(GTEST_LOG_) + +#if !defined(GTEST_CHECK_) +// INTERNAL IMPLEMENTATION - DO NOT USE. +// +// GTEST_CHECK_ is an all-mode assert. It aborts the program if the condition +// is not satisfied. +// Synopsys: +// GTEST_CHECK_(boolean_condition); +// or +// GTEST_CHECK_(boolean_condition) << "Additional message"; +// +// This checks the condition and if the condition is not satisfied +// it prints message about the condition violation, including the +// condition itself, plus additional message streamed into it, if any, +// and then it aborts the program. It aborts the program irrespective of +// whether it is built in the debug mode or not. +# define GTEST_CHECK_(condition) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::IsTrue(condition)) \ + ; \ + else \ + GTEST_LOG_(FATAL) << "Condition " #condition " failed. " +#endif // !defined(GTEST_CHECK_) + +// An all-mode assert to verify that the given POSIX-style function +// call returns 0 (indicating success). Known limitation: this +// doesn't expand to a balanced 'if' statement, so enclose the macro +// in {} if you need to use it as the only statement in an 'if' +// branch. +#define GTEST_CHECK_POSIX_SUCCESS_(posix_call) \ + if (const int gtest_error = (posix_call)) \ + GTEST_LOG_(FATAL) << #posix_call << "failed with error " \ + << gtest_error + +// Adds reference to a type if it is not a reference type, +// otherwise leaves it unchanged. This is the same as +// tr1::add_reference, which is not widely available yet. +template +struct AddReference { typedef T& type; }; // NOLINT +template +struct AddReference { typedef T& type; }; // NOLINT + +// A handy wrapper around AddReference that works when the argument T +// depends on template parameters. +#define GTEST_ADD_REFERENCE_(T) \ + typename ::testing::internal::AddReference::type + +// Transforms "T" into "const T&" according to standard reference collapsing +// rules (this is only needed as a backport for C++98 compilers that do not +// support reference collapsing). Specifically, it transforms: +// +// char ==> const char& +// const char ==> const char& +// char& ==> char& +// const char& ==> const char& +// +// Note that the non-const reference will not have "const" added. This is +// standard, and necessary so that "T" can always bind to "const T&". +template +struct ConstRef { typedef const T& type; }; +template +struct ConstRef { typedef T& type; }; + +// The argument T must depend on some template parameters. +#define GTEST_REFERENCE_TO_CONST_(T) \ + typename ::testing::internal::ConstRef::type + +#if GTEST_HAS_STD_MOVE_ +using std::forward; +using std::move; + +template +struct RvalueRef { + typedef T&& type; +}; +#else // GTEST_HAS_STD_MOVE_ +template +const T& move(const T& t) { + return t; +} +template +GTEST_ADD_REFERENCE_(T) forward(GTEST_ADD_REFERENCE_(T) t) { return t; } + +template +struct RvalueRef { + typedef const T& type; +}; +#endif // GTEST_HAS_STD_MOVE_ + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Use ImplicitCast_ as a safe version of static_cast for upcasting in +// the type hierarchy (e.g. casting a Foo* to a SuperclassOfFoo* or a +// const Foo*). When you use ImplicitCast_, the compiler checks that +// the cast is safe. Such explicit ImplicitCast_s are necessary in +// surprisingly many situations where C++ demands an exact type match +// instead of an argument type convertable to a target type. +// +// The syntax for using ImplicitCast_ is the same as for static_cast: +// +// ImplicitCast_(expr) +// +// ImplicitCast_ would have been part of the C++ standard library, +// but the proposal was submitted too late. It will probably make +// its way into the language in the future. +// +// This relatively ugly name is intentional. It prevents clashes with +// similar functions users may have (e.g., implicit_cast). The internal +// namespace alone is not enough because the function can be found by ADL. +template +inline To ImplicitCast_(To x) { return x; } + +// When you upcast (that is, cast a pointer from type Foo to type +// SuperclassOfFoo), it's fine to use ImplicitCast_<>, since upcasts +// always succeed. When you downcast (that is, cast a pointer from +// type Foo to type SubclassOfFoo), static_cast<> isn't safe, because +// how do you know the pointer is really of type SubclassOfFoo? It +// could be a bare Foo, or of type DifferentSubclassOfFoo. Thus, +// when you downcast, you should use this macro. In debug mode, we +// use dynamic_cast<> to double-check the downcast is legal (we die +// if it's not). In normal mode, we do the efficient static_cast<> +// instead. Thus, it's important to test in debug mode to make sure +// the cast is legal! +// This is the only place in the code we should use dynamic_cast<>. +// In particular, you SHOULDN'T be using dynamic_cast<> in order to +// do RTTI (eg code like this: +// if (dynamic_cast(foo)) HandleASubclass1Object(foo); +// if (dynamic_cast(foo)) HandleASubclass2Object(foo); +// You should design the code some other way not to need this. +// +// This relatively ugly name is intentional. It prevents clashes with +// similar functions users may have (e.g., down_cast). The internal +// namespace alone is not enough because the function can be found by ADL. +template // use like this: DownCast_(foo); +inline To DownCast_(From* f) { // so we only accept pointers + // Ensures that To is a sub-type of From *. This test is here only + // for compile-time type checking, and has no overhead in an + // optimized build at run-time, as it will be optimized away + // completely. + GTEST_INTENTIONAL_CONST_COND_PUSH_() + if (false) { + GTEST_INTENTIONAL_CONST_COND_POP_() + const To to = NULL; + ::testing::internal::ImplicitCast_(to); + } + +#if GTEST_HAS_RTTI + // RTTI: debug mode only! + GTEST_CHECK_(f == NULL || dynamic_cast(f) != NULL); +#endif + return static_cast(f); +} + +// Downcasts the pointer of type Base to Derived. +// Derived must be a subclass of Base. The parameter MUST +// point to a class of type Derived, not any subclass of it. +// When RTTI is available, the function performs a runtime +// check to enforce this. +template +Derived* CheckedDowncastToActualType(Base* base) { +#if GTEST_HAS_RTTI + GTEST_CHECK_(typeid(*base) == typeid(Derived)); +#endif + +#if GTEST_HAS_DOWNCAST_ + return ::down_cast(base); +#elif GTEST_HAS_RTTI + return dynamic_cast(base); // NOLINT +#else + return static_cast(base); // Poor man's downcast. +#endif +} + +#if GTEST_HAS_STREAM_REDIRECTION + +// Defines the stderr capturer: +// CaptureStdout - starts capturing stdout. +// GetCapturedStdout - stops capturing stdout and returns the captured string. +// CaptureStderr - starts capturing stderr. +// GetCapturedStderr - stops capturing stderr and returns the captured string. +// +GTEST_API_ void CaptureStdout(); +GTEST_API_ std::string GetCapturedStdout(); +GTEST_API_ void CaptureStderr(); +GTEST_API_ std::string GetCapturedStderr(); + +#endif // GTEST_HAS_STREAM_REDIRECTION +// Returns the size (in bytes) of a file. +GTEST_API_ size_t GetFileSize(FILE* file); + +// Reads the entire content of a file as a string. +GTEST_API_ std::string ReadEntireFile(FILE* file); + +// All command line arguments. +GTEST_API_ std::vector GetArgvs(); + +#if GTEST_HAS_DEATH_TEST + +std::vector GetInjectableArgvs(); +// Deprecated: pass the args vector by value instead. +void SetInjectableArgvs(const std::vector* new_argvs); +void SetInjectableArgvs(const std::vector& new_argvs); +#if GTEST_HAS_GLOBAL_STRING +void SetInjectableArgvs(const std::vector< ::string>& new_argvs); +#endif // GTEST_HAS_GLOBAL_STRING +void ClearInjectableArgvs(); + +#endif // GTEST_HAS_DEATH_TEST + +// Defines synchronization primitives. +#if GTEST_IS_THREADSAFE +# if GTEST_HAS_PTHREAD +// Sleeps for (roughly) n milliseconds. This function is only for testing +// Google Test's own constructs. Don't use it in user tests, either +// directly or indirectly. +inline void SleepMilliseconds(int n) { + const timespec time = { + 0, // 0 seconds. + n * 1000L * 1000L, // And n ms. + }; + nanosleep(&time, NULL); +} +# endif // GTEST_HAS_PTHREAD + +# if GTEST_HAS_NOTIFICATION_ +// Notification has already been imported into the namespace. +// Nothing to do here. + +# elif GTEST_HAS_PTHREAD +// Allows a controller thread to pause execution of newly created +// threads until notified. Instances of this class must be created +// and destroyed in the controller thread. +// +// This class is only for testing Google Test's own constructs. Do not +// use it in user tests, either directly or indirectly. +class Notification { + public: + Notification() : notified_(false) { + GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_init(&mutex_, NULL)); + } + ~Notification() { + pthread_mutex_destroy(&mutex_); + } + + // Notifies all threads created with this notification to start. Must + // be called from the controller thread. + void Notify() { + pthread_mutex_lock(&mutex_); + notified_ = true; + pthread_mutex_unlock(&mutex_); + } + + // Blocks until the controller thread notifies. Must be called from a test + // thread. + void WaitForNotification() { + for (;;) { + pthread_mutex_lock(&mutex_); + const bool notified = notified_; + pthread_mutex_unlock(&mutex_); + if (notified) + break; + SleepMilliseconds(10); + } + } + + private: + pthread_mutex_t mutex_; + bool notified_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(Notification); +}; + +# elif GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT + +GTEST_API_ void SleepMilliseconds(int n); + +// Provides leak-safe Windows kernel handle ownership. +// Used in death tests and in threading support. +class GTEST_API_ AutoHandle { + public: + // Assume that Win32 HANDLE type is equivalent to void*. Doing so allows us to + // avoid including in this header file. Including is + // undesirable because it defines a lot of symbols and macros that tend to + // conflict with client code. This assumption is verified by + // WindowsTypesTest.HANDLEIsVoidStar. + typedef void* Handle; + AutoHandle(); + explicit AutoHandle(Handle handle); + + ~AutoHandle(); + + Handle Get() const; + void Reset(); + void Reset(Handle handle); + + private: + // Returns true iff the handle is a valid handle object that can be closed. + bool IsCloseable() const; + + Handle handle_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(AutoHandle); +}; + +// Allows a controller thread to pause execution of newly created +// threads until notified. Instances of this class must be created +// and destroyed in the controller thread. +// +// This class is only for testing Google Test's own constructs. Do not +// use it in user tests, either directly or indirectly. +class GTEST_API_ Notification { + public: + Notification(); + void Notify(); + void WaitForNotification(); + + private: + AutoHandle event_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(Notification); +}; +# endif // GTEST_HAS_NOTIFICATION_ + +// On MinGW, we can have both GTEST_OS_WINDOWS and GTEST_HAS_PTHREAD +// defined, but we don't want to use MinGW's pthreads implementation, which +// has conformance problems with some versions of the POSIX standard. +# if GTEST_HAS_PTHREAD && !GTEST_OS_WINDOWS_MINGW + +// As a C-function, ThreadFuncWithCLinkage cannot be templated itself. +// Consequently, it cannot select a correct instantiation of ThreadWithParam +// in order to call its Run(). Introducing ThreadWithParamBase as a +// non-templated base class for ThreadWithParam allows us to bypass this +// problem. +class ThreadWithParamBase { + public: + virtual ~ThreadWithParamBase() {} + virtual void Run() = 0; +}; + +// pthread_create() accepts a pointer to a function type with the C linkage. +// According to the Standard (7.5/1), function types with different linkages +// are different even if they are otherwise identical. Some compilers (for +// example, SunStudio) treat them as different types. Since class methods +// cannot be defined with C-linkage we need to define a free C-function to +// pass into pthread_create(). +extern "C" inline void* ThreadFuncWithCLinkage(void* thread) { + static_cast(thread)->Run(); + return NULL; +} + +// Helper class for testing Google Test's multi-threading constructs. +// To use it, write: +// +// void ThreadFunc(int param) { /* Do things with param */ } +// Notification thread_can_start; +// ... +// // The thread_can_start parameter is optional; you can supply NULL. +// ThreadWithParam thread(&ThreadFunc, 5, &thread_can_start); +// thread_can_start.Notify(); +// +// These classes are only for testing Google Test's own constructs. Do +// not use them in user tests, either directly or indirectly. +template +class ThreadWithParam : public ThreadWithParamBase { + public: + typedef void UserThreadFunc(T); + + ThreadWithParam(UserThreadFunc* func, T param, Notification* thread_can_start) + : func_(func), + param_(param), + thread_can_start_(thread_can_start), + finished_(false) { + ThreadWithParamBase* const base = this; + // The thread can be created only after all fields except thread_ + // have been initialized. + GTEST_CHECK_POSIX_SUCCESS_( + pthread_create(&thread_, 0, &ThreadFuncWithCLinkage, base)); + } + ~ThreadWithParam() { Join(); } + + void Join() { + if (!finished_) { + GTEST_CHECK_POSIX_SUCCESS_(pthread_join(thread_, 0)); + finished_ = true; + } + } + + virtual void Run() { + if (thread_can_start_ != NULL) + thread_can_start_->WaitForNotification(); + func_(param_); + } + + private: + UserThreadFunc* const func_; // User-supplied thread function. + const T param_; // User-supplied parameter to the thread function. + // When non-NULL, used to block execution until the controller thread + // notifies. + Notification* const thread_can_start_; + bool finished_; // true iff we know that the thread function has finished. + pthread_t thread_; // The native thread object. + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParam); +}; +# endif // !GTEST_OS_WINDOWS && GTEST_HAS_PTHREAD || + // GTEST_HAS_MUTEX_AND_THREAD_LOCAL_ + +# if GTEST_HAS_MUTEX_AND_THREAD_LOCAL_ +// Mutex and ThreadLocal have already been imported into the namespace. +// Nothing to do here. + +# elif GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT + +// Mutex implements mutex on Windows platforms. It is used in conjunction +// with class MutexLock: +// +// Mutex mutex; +// ... +// MutexLock lock(&mutex); // Acquires the mutex and releases it at the +// // end of the current scope. +// +// A static Mutex *must* be defined or declared using one of the following +// macros: +// GTEST_DEFINE_STATIC_MUTEX_(g_some_mutex); +// GTEST_DECLARE_STATIC_MUTEX_(g_some_mutex); +// +// (A non-static Mutex is defined/declared in the usual way). +class GTEST_API_ Mutex { + public: + enum MutexType { kStatic = 0, kDynamic = 1 }; + // We rely on kStaticMutex being 0 as it is to what the linker initializes + // type_ in static mutexes. critical_section_ will be initialized lazily + // in ThreadSafeLazyInit(). + enum StaticConstructorSelector { kStaticMutex = 0 }; + + // This constructor intentionally does nothing. It relies on type_ being + // statically initialized to 0 (effectively setting it to kStatic) and on + // ThreadSafeLazyInit() to lazily initialize the rest of the members. + explicit Mutex(StaticConstructorSelector /*dummy*/) {} + + Mutex(); + ~Mutex(); + + void Lock(); + + void Unlock(); + + // Does nothing if the current thread holds the mutex. Otherwise, crashes + // with high probability. + void AssertHeld(); + + private: + // Initializes owner_thread_id_ and critical_section_ in static mutexes. + void ThreadSafeLazyInit(); + + // Per https://blogs.msdn.microsoft.com/oldnewthing/20040223-00/?p=40503, + // we assume that 0 is an invalid value for thread IDs. + unsigned int owner_thread_id_; + + // For static mutexes, we rely on these members being initialized to zeros + // by the linker. + MutexType type_; + long critical_section_init_phase_; // NOLINT + GTEST_CRITICAL_SECTION* critical_section_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(Mutex); +}; + +# define GTEST_DECLARE_STATIC_MUTEX_(mutex) \ + extern ::testing::internal::Mutex mutex + +# define GTEST_DEFINE_STATIC_MUTEX_(mutex) \ + ::testing::internal::Mutex mutex(::testing::internal::Mutex::kStaticMutex) + +// We cannot name this class MutexLock because the ctor declaration would +// conflict with a macro named MutexLock, which is defined on some +// platforms. That macro is used as a defensive measure to prevent against +// inadvertent misuses of MutexLock like "MutexLock(&mu)" rather than +// "MutexLock l(&mu)". Hence the typedef trick below. +class GTestMutexLock { + public: + explicit GTestMutexLock(Mutex* mutex) + : mutex_(mutex) { mutex_->Lock(); } + + ~GTestMutexLock() { mutex_->Unlock(); } + + private: + Mutex* const mutex_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(GTestMutexLock); +}; + +typedef GTestMutexLock MutexLock; + +// Base class for ValueHolder. Allows a caller to hold and delete a value +// without knowing its type. +class ThreadLocalValueHolderBase { + public: + virtual ~ThreadLocalValueHolderBase() {} +}; + +// Provides a way for a thread to send notifications to a ThreadLocal +// regardless of its parameter type. +class ThreadLocalBase { + public: + // Creates a new ValueHolder object holding a default value passed to + // this ThreadLocal's constructor and returns it. It is the caller's + // responsibility not to call this when the ThreadLocal instance already + // has a value on the current thread. + virtual ThreadLocalValueHolderBase* NewValueForCurrentThread() const = 0; + + protected: + ThreadLocalBase() {} + virtual ~ThreadLocalBase() {} + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadLocalBase); +}; + +// Maps a thread to a set of ThreadLocals that have values instantiated on that +// thread and notifies them when the thread exits. A ThreadLocal instance is +// expected to persist until all threads it has values on have terminated. +class GTEST_API_ ThreadLocalRegistry { + public: + // Registers thread_local_instance as having value on the current thread. + // Returns a value that can be used to identify the thread from other threads. + static ThreadLocalValueHolderBase* GetValueOnCurrentThread( + const ThreadLocalBase* thread_local_instance); + + // Invoked when a ThreadLocal instance is destroyed. + static void OnThreadLocalDestroyed( + const ThreadLocalBase* thread_local_instance); +}; + +class GTEST_API_ ThreadWithParamBase { + public: + void Join(); + + protected: + class Runnable { + public: + virtual ~Runnable() {} + virtual void Run() = 0; + }; + + ThreadWithParamBase(Runnable *runnable, Notification* thread_can_start); + virtual ~ThreadWithParamBase(); + + private: + AutoHandle thread_; +}; + +// Helper class for testing Google Test's multi-threading constructs. +template +class ThreadWithParam : public ThreadWithParamBase { + public: + typedef void UserThreadFunc(T); + + ThreadWithParam(UserThreadFunc* func, T param, Notification* thread_can_start) + : ThreadWithParamBase(new RunnableImpl(func, param), thread_can_start) { + } + virtual ~ThreadWithParam() {} + + private: + class RunnableImpl : public Runnable { + public: + RunnableImpl(UserThreadFunc* func, T param) + : func_(func), + param_(param) { + } + virtual ~RunnableImpl() {} + virtual void Run() { + func_(param_); + } + + private: + UserThreadFunc* const func_; + const T param_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(RunnableImpl); + }; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParam); +}; + +// Implements thread-local storage on Windows systems. +// +// // Thread 1 +// ThreadLocal tl(100); // 100 is the default value for each thread. +// +// // Thread 2 +// tl.set(150); // Changes the value for thread 2 only. +// EXPECT_EQ(150, tl.get()); +// +// // Thread 1 +// EXPECT_EQ(100, tl.get()); // In thread 1, tl has the original value. +// tl.set(200); +// EXPECT_EQ(200, tl.get()); +// +// The template type argument T must have a public copy constructor. +// In addition, the default ThreadLocal constructor requires T to have +// a public default constructor. +// +// The users of a TheadLocal instance have to make sure that all but one +// threads (including the main one) using that instance have exited before +// destroying it. Otherwise, the per-thread objects managed for them by the +// ThreadLocal instance are not guaranteed to be destroyed on all platforms. +// +// Google Test only uses global ThreadLocal objects. That means they +// will die after main() has returned. Therefore, no per-thread +// object managed by Google Test will be leaked as long as all threads +// using Google Test have exited when main() returns. +template +class ThreadLocal : public ThreadLocalBase { + public: + ThreadLocal() : default_factory_(new DefaultValueHolderFactory()) {} + explicit ThreadLocal(const T& value) + : default_factory_(new InstanceValueHolderFactory(value)) {} + + ~ThreadLocal() { ThreadLocalRegistry::OnThreadLocalDestroyed(this); } + + T* pointer() { return GetOrCreateValue(); } + const T* pointer() const { return GetOrCreateValue(); } + const T& get() const { return *pointer(); } + void set(const T& value) { *pointer() = value; } + + private: + // Holds a value of T. Can be deleted via its base class without the caller + // knowing the type of T. + class ValueHolder : public ThreadLocalValueHolderBase { + public: + ValueHolder() : value_() {} + explicit ValueHolder(const T& value) : value_(value) {} + + T* pointer() { return &value_; } + + private: + T value_; + GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolder); + }; + + + T* GetOrCreateValue() const { + return static_cast( + ThreadLocalRegistry::GetValueOnCurrentThread(this))->pointer(); + } + + virtual ThreadLocalValueHolderBase* NewValueForCurrentThread() const { + return default_factory_->MakeNewHolder(); + } + + class ValueHolderFactory { + public: + ValueHolderFactory() {} + virtual ~ValueHolderFactory() {} + virtual ValueHolder* MakeNewHolder() const = 0; + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolderFactory); + }; + + class DefaultValueHolderFactory : public ValueHolderFactory { + public: + DefaultValueHolderFactory() {} + virtual ValueHolder* MakeNewHolder() const { return new ValueHolder(); } + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(DefaultValueHolderFactory); + }; + + class InstanceValueHolderFactory : public ValueHolderFactory { + public: + explicit InstanceValueHolderFactory(const T& value) : value_(value) {} + virtual ValueHolder* MakeNewHolder() const { + return new ValueHolder(value_); + } + + private: + const T value_; // The value for each thread. + + GTEST_DISALLOW_COPY_AND_ASSIGN_(InstanceValueHolderFactory); + }; + + scoped_ptr default_factory_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadLocal); +}; + +# elif GTEST_HAS_PTHREAD + +// MutexBase and Mutex implement mutex on pthreads-based platforms. +class MutexBase { + public: + // Acquires this mutex. + void Lock() { + GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_lock(&mutex_)); + owner_ = pthread_self(); + has_owner_ = true; + } + + // Releases this mutex. + void Unlock() { + // Since the lock is being released the owner_ field should no longer be + // considered valid. We don't protect writing to has_owner_ here, as it's + // the caller's responsibility to ensure that the current thread holds the + // mutex when this is called. + has_owner_ = false; + GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_unlock(&mutex_)); + } + + // Does nothing if the current thread holds the mutex. Otherwise, crashes + // with high probability. + void AssertHeld() const { + GTEST_CHECK_(has_owner_ && pthread_equal(owner_, pthread_self())) + << "The current thread is not holding the mutex @" << this; + } + + // A static mutex may be used before main() is entered. It may even + // be used before the dynamic initialization stage. Therefore we + // must be able to initialize a static mutex object at link time. + // This means MutexBase has to be a POD and its member variables + // have to be public. + public: + pthread_mutex_t mutex_; // The underlying pthread mutex. + // has_owner_ indicates whether the owner_ field below contains a valid thread + // ID and is therefore safe to inspect (e.g., to use in pthread_equal()). All + // accesses to the owner_ field should be protected by a check of this field. + // An alternative might be to memset() owner_ to all zeros, but there's no + // guarantee that a zero'd pthread_t is necessarily invalid or even different + // from pthread_self(). + bool has_owner_; + pthread_t owner_; // The thread holding the mutex. +}; + +// Forward-declares a static mutex. +# define GTEST_DECLARE_STATIC_MUTEX_(mutex) \ + extern ::testing::internal::MutexBase mutex + +// Defines and statically (i.e. at link time) initializes a static mutex. +// The initialization list here does not explicitly initialize each field, +// instead relying on default initialization for the unspecified fields. In +// particular, the owner_ field (a pthread_t) is not explicitly initialized. +// This allows initialization to work whether pthread_t is a scalar or struct. +// The flag -Wmissing-field-initializers must not be specified for this to work. +#define GTEST_DEFINE_STATIC_MUTEX_(mutex) \ + ::testing::internal::MutexBase mutex = {PTHREAD_MUTEX_INITIALIZER, false, 0} + +// The Mutex class can only be used for mutexes created at runtime. It +// shares its API with MutexBase otherwise. +class Mutex : public MutexBase { + public: + Mutex() { + GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_init(&mutex_, NULL)); + has_owner_ = false; + } + ~Mutex() { + GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_destroy(&mutex_)); + } + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(Mutex); +}; + +// We cannot name this class MutexLock because the ctor declaration would +// conflict with a macro named MutexLock, which is defined on some +// platforms. That macro is used as a defensive measure to prevent against +// inadvertent misuses of MutexLock like "MutexLock(&mu)" rather than +// "MutexLock l(&mu)". Hence the typedef trick below. +class GTestMutexLock { + public: + explicit GTestMutexLock(MutexBase* mutex) + : mutex_(mutex) { mutex_->Lock(); } + + ~GTestMutexLock() { mutex_->Unlock(); } + + private: + MutexBase* const mutex_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(GTestMutexLock); +}; + +typedef GTestMutexLock MutexLock; + +// Helpers for ThreadLocal. + +// pthread_key_create() requires DeleteThreadLocalValue() to have +// C-linkage. Therefore it cannot be templatized to access +// ThreadLocal. Hence the need for class +// ThreadLocalValueHolderBase. +class ThreadLocalValueHolderBase { + public: + virtual ~ThreadLocalValueHolderBase() {} +}; + +// Called by pthread to delete thread-local data stored by +// pthread_setspecific(). +extern "C" inline void DeleteThreadLocalValue(void* value_holder) { + delete static_cast(value_holder); +} + +// Implements thread-local storage on pthreads-based systems. +template +class GTEST_API_ ThreadLocal { + public: + ThreadLocal() + : key_(CreateKey()), default_factory_(new DefaultValueHolderFactory()) {} + explicit ThreadLocal(const T& value) + : key_(CreateKey()), + default_factory_(new InstanceValueHolderFactory(value)) {} + + ~ThreadLocal() { + // Destroys the managed object for the current thread, if any. + DeleteThreadLocalValue(pthread_getspecific(key_)); + + // Releases resources associated with the key. This will *not* + // delete managed objects for other threads. + GTEST_CHECK_POSIX_SUCCESS_(pthread_key_delete(key_)); + } + + T* pointer() { return GetOrCreateValue(); } + const T* pointer() const { return GetOrCreateValue(); } + const T& get() const { return *pointer(); } + void set(const T& value) { *pointer() = value; } + + private: + // Holds a value of type T. + class ValueHolder : public ThreadLocalValueHolderBase { + public: + ValueHolder() : value_() {} + explicit ValueHolder(const T& value) : value_(value) {} + + T* pointer() { return &value_; } + + private: + T value_; + GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolder); + }; + + static pthread_key_t CreateKey() { + pthread_key_t key; + // When a thread exits, DeleteThreadLocalValue() will be called on + // the object managed for that thread. + GTEST_CHECK_POSIX_SUCCESS_( + pthread_key_create(&key, &DeleteThreadLocalValue)); + return key; + } + + T* GetOrCreateValue() const { + ThreadLocalValueHolderBase* const holder = + static_cast(pthread_getspecific(key_)); + if (holder != NULL) { + return CheckedDowncastToActualType(holder)->pointer(); + } + + ValueHolder* const new_holder = default_factory_->MakeNewHolder(); + ThreadLocalValueHolderBase* const holder_base = new_holder; + GTEST_CHECK_POSIX_SUCCESS_(pthread_setspecific(key_, holder_base)); + return new_holder->pointer(); + } + + class ValueHolderFactory { + public: + ValueHolderFactory() {} + virtual ~ValueHolderFactory() {} + virtual ValueHolder* MakeNewHolder() const = 0; + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolderFactory); + }; + + class DefaultValueHolderFactory : public ValueHolderFactory { + public: + DefaultValueHolderFactory() {} + virtual ValueHolder* MakeNewHolder() const { return new ValueHolder(); } + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(DefaultValueHolderFactory); + }; + + class InstanceValueHolderFactory : public ValueHolderFactory { + public: + explicit InstanceValueHolderFactory(const T& value) : value_(value) {} + virtual ValueHolder* MakeNewHolder() const { + return new ValueHolder(value_); + } + + private: + const T value_; // The value for each thread. + + GTEST_DISALLOW_COPY_AND_ASSIGN_(InstanceValueHolderFactory); + }; + + // A key pthreads uses for looking up per-thread values. + const pthread_key_t key_; + scoped_ptr default_factory_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadLocal); +}; + +# endif // GTEST_HAS_MUTEX_AND_THREAD_LOCAL_ + +#else // GTEST_IS_THREADSAFE + +// A dummy implementation of synchronization primitives (mutex, lock, +// and thread-local variable). Necessary for compiling Google Test where +// mutex is not supported - using Google Test in multiple threads is not +// supported on such platforms. + +class Mutex { + public: + Mutex() {} + void Lock() {} + void Unlock() {} + void AssertHeld() const {} +}; + +# define GTEST_DECLARE_STATIC_MUTEX_(mutex) \ + extern ::testing::internal::Mutex mutex + +# define GTEST_DEFINE_STATIC_MUTEX_(mutex) ::testing::internal::Mutex mutex + +// We cannot name this class MutexLock because the ctor declaration would +// conflict with a macro named MutexLock, which is defined on some +// platforms. That macro is used as a defensive measure to prevent against +// inadvertent misuses of MutexLock like "MutexLock(&mu)" rather than +// "MutexLock l(&mu)". Hence the typedef trick below. +class GTestMutexLock { + public: + explicit GTestMutexLock(Mutex*) {} // NOLINT +}; + +typedef GTestMutexLock MutexLock; + +template +class GTEST_API_ ThreadLocal { + public: + ThreadLocal() : value_() {} + explicit ThreadLocal(const T& value) : value_(value) {} + T* pointer() { return &value_; } + const T* pointer() const { return &value_; } + const T& get() const { return value_; } + void set(const T& value) { value_ = value; } + private: + T value_; +}; + +#endif // GTEST_IS_THREADSAFE + +// Returns the number of threads running in the process, or 0 to indicate that +// we cannot detect it. +GTEST_API_ size_t GetThreadCount(); + +// Passing non-POD classes through ellipsis (...) crashes the ARM +// compiler and generates a warning in Sun Studio before 12u4. The Nokia Symbian +// and the IBM XL C/C++ compiler try to instantiate a copy constructor +// for objects passed through ellipsis (...), failing for uncopyable +// objects. We define this to ensure that only POD is passed through +// ellipsis on these systems. +#if defined(__SYMBIAN32__) || defined(__IBMCPP__) || \ + (defined(__SUNPRO_CC) && __SUNPRO_CC < 0x5130) +// We lose support for NULL detection where the compiler doesn't like +// passing non-POD classes through ellipsis (...). +# define GTEST_ELLIPSIS_NEEDS_POD_ 1 +#else +# define GTEST_CAN_COMPARE_NULL 1 +#endif + +// The Nokia Symbian and IBM XL C/C++ compilers cannot decide between +// const T& and const T* in a function template. These compilers +// _can_ decide between class template specializations for T and T*, +// so a tr1::type_traits-like is_pointer works. +#if defined(__SYMBIAN32__) || defined(__IBMCPP__) +# define GTEST_NEEDS_IS_POINTER_ 1 +#endif + +template +struct bool_constant { + typedef bool_constant type; + static const bool value = bool_value; +}; +template const bool bool_constant::value; + +typedef bool_constant false_type; +typedef bool_constant true_type; + +template +struct is_same : public false_type {}; + +template +struct is_same : public true_type {}; + + +template +struct is_pointer : public false_type {}; + +template +struct is_pointer : public true_type {}; + +template +struct IteratorTraits { + typedef typename Iterator::value_type value_type; +}; + + +template +struct IteratorTraits { + typedef T value_type; +}; + +template +struct IteratorTraits { + typedef T value_type; +}; + +#if GTEST_OS_WINDOWS +# define GTEST_PATH_SEP_ "\\" +# define GTEST_HAS_ALT_PATH_SEP_ 1 +// The biggest signed integer type the compiler supports. +typedef __int64 BiggestInt; +#else +# define GTEST_PATH_SEP_ "/" +# define GTEST_HAS_ALT_PATH_SEP_ 0 +typedef long long BiggestInt; // NOLINT +#endif // GTEST_OS_WINDOWS + +// Utilities for char. + +// isspace(int ch) and friends accept an unsigned char or EOF. char +// may be signed, depending on the compiler (or compiler flags). +// Therefore we need to cast a char to unsigned char before calling +// isspace(), etc. + +inline bool IsAlpha(char ch) { + return isalpha(static_cast(ch)) != 0; +} +inline bool IsAlNum(char ch) { + return isalnum(static_cast(ch)) != 0; +} +inline bool IsDigit(char ch) { + return isdigit(static_cast(ch)) != 0; +} +inline bool IsLower(char ch) { + return islower(static_cast(ch)) != 0; +} +inline bool IsSpace(char ch) { + return isspace(static_cast(ch)) != 0; +} +inline bool IsUpper(char ch) { + return isupper(static_cast(ch)) != 0; +} +inline bool IsXDigit(char ch) { + return isxdigit(static_cast(ch)) != 0; +} +inline bool IsXDigit(wchar_t ch) { + const unsigned char low_byte = static_cast(ch); + return ch == low_byte && isxdigit(low_byte) != 0; +} + +inline char ToLower(char ch) { + return static_cast(tolower(static_cast(ch))); +} +inline char ToUpper(char ch) { + return static_cast(toupper(static_cast(ch))); +} + +inline std::string StripTrailingSpaces(std::string str) { + std::string::iterator it = str.end(); + while (it != str.begin() && IsSpace(*--it)) + it = str.erase(it); + return str; +} + +// The testing::internal::posix namespace holds wrappers for common +// POSIX functions. These wrappers hide the differences between +// Windows/MSVC and POSIX systems. Since some compilers define these +// standard functions as macros, the wrapper cannot have the same name +// as the wrapped function. + +namespace posix { + +// Functions with a different name on Windows. + +#if GTEST_OS_WINDOWS + +typedef struct _stat StatStruct; + +# ifdef __BORLANDC__ +inline int IsATTY(int fd) { return isatty(fd); } +inline int StrCaseCmp(const char* s1, const char* s2) { + return stricmp(s1, s2); +} +inline char* StrDup(const char* src) { return strdup(src); } +# else // !__BORLANDC__ +# if GTEST_OS_WINDOWS_MOBILE +inline int IsATTY(int /* fd */) { return 0; } +# else +inline int IsATTY(int fd) { return _isatty(fd); } +# endif // GTEST_OS_WINDOWS_MOBILE +inline int StrCaseCmp(const char* s1, const char* s2) { + return _stricmp(s1, s2); +} +inline char* StrDup(const char* src) { return _strdup(src); } +# endif // __BORLANDC__ + +# if GTEST_OS_WINDOWS_MOBILE +inline int FileNo(FILE* file) { return reinterpret_cast(_fileno(file)); } +// Stat(), RmDir(), and IsDir() are not needed on Windows CE at this +// time and thus not defined there. +# else +inline int FileNo(FILE* file) { return _fileno(file); } +inline int Stat(const char* path, StatStruct* buf) { return _stat(path, buf); } +inline int RmDir(const char* dir) { return _rmdir(dir); } +inline bool IsDir(const StatStruct& st) { + return (_S_IFDIR & st.st_mode) != 0; +} +# endif // GTEST_OS_WINDOWS_MOBILE + +#else + +typedef struct stat StatStruct; + +inline int FileNo(FILE* file) { return fileno(file); } +inline int IsATTY(int fd) { return isatty(fd); } +inline int Stat(const char* path, StatStruct* buf) { return stat(path, buf); } +inline int StrCaseCmp(const char* s1, const char* s2) { + return strcasecmp(s1, s2); +} +inline char* StrDup(const char* src) { return strdup(src); } +inline int RmDir(const char* dir) { return rmdir(dir); } +inline bool IsDir(const StatStruct& st) { return S_ISDIR(st.st_mode); } + +#endif // GTEST_OS_WINDOWS + +// Functions deprecated by MSVC 8.0. + +GTEST_DISABLE_MSC_DEPRECATED_PUSH_() + +inline const char* StrNCpy(char* dest, const char* src, size_t n) { + return strncpy(dest, src, n); +} + +// ChDir(), FReopen(), FDOpen(), Read(), Write(), Close(), and +// StrError() aren't needed on Windows CE at this time and thus not +// defined there. + +#if !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT +inline int ChDir(const char* dir) { return chdir(dir); } +#endif +inline FILE* FOpen(const char* path, const char* mode) { + return fopen(path, mode); +} +#if !GTEST_OS_WINDOWS_MOBILE +inline FILE *FReopen(const char* path, const char* mode, FILE* stream) { + return freopen(path, mode, stream); +} +inline FILE* FDOpen(int fd, const char* mode) { return fdopen(fd, mode); } +#endif +inline int FClose(FILE* fp) { return fclose(fp); } +#if !GTEST_OS_WINDOWS_MOBILE +inline int Read(int fd, void* buf, unsigned int count) { + return static_cast(read(fd, buf, count)); +} +inline int Write(int fd, const void* buf, unsigned int count) { + return static_cast(write(fd, buf, count)); +} +inline int Close(int fd) { return close(fd); } +inline const char* StrError(int errnum) { return strerror(errnum); } +#endif +inline const char* GetEnv(const char* name) { +#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT + // We are on Windows CE, which has no environment variables. + static_cast(name); // To prevent 'unused argument' warning. + return NULL; +#elif defined(__BORLANDC__) || defined(__SunOS_5_8) || defined(__SunOS_5_9) + // Environment variables which we programmatically clear will be set to the + // empty string rather than unset (NULL). Handle that case. + const char* const env = getenv(name); + return (env != NULL && env[0] != '\0') ? env : NULL; +#else + return getenv(name); +#endif +} + +GTEST_DISABLE_MSC_DEPRECATED_POP_() + +#if GTEST_OS_WINDOWS_MOBILE +// Windows CE has no C library. The abort() function is used in +// several places in Google Test. This implementation provides a reasonable +// imitation of standard behaviour. +void Abort(); +#else +inline void Abort() { abort(); } +#endif // GTEST_OS_WINDOWS_MOBILE + +} // namespace posix + +// MSVC "deprecates" snprintf and issues warnings wherever it is used. In +// order to avoid these warnings, we need to use _snprintf or _snprintf_s on +// MSVC-based platforms. We map the GTEST_SNPRINTF_ macro to the appropriate +// function in order to achieve that. We use macro definition here because +// snprintf is a variadic function. +#if _MSC_VER >= 1400 && !GTEST_OS_WINDOWS_MOBILE +// MSVC 2005 and above support variadic macros. +# define GTEST_SNPRINTF_(buffer, size, format, ...) \ + _snprintf_s(buffer, size, size, format, __VA_ARGS__) +#elif defined(_MSC_VER) +// Windows CE does not define _snprintf_s and MSVC prior to 2005 doesn't +// complain about _snprintf. +# define GTEST_SNPRINTF_ _snprintf +#else +# define GTEST_SNPRINTF_ snprintf +#endif + +// The maximum number a BiggestInt can represent. This definition +// works no matter BiggestInt is represented in one's complement or +// two's complement. +// +// We cannot rely on numeric_limits in STL, as __int64 and long long +// are not part of standard C++ and numeric_limits doesn't need to be +// defined for them. +const BiggestInt kMaxBiggestInt = + ~(static_cast(1) << (8*sizeof(BiggestInt) - 1)); + +// This template class serves as a compile-time function from size to +// type. It maps a size in bytes to a primitive type with that +// size. e.g. +// +// TypeWithSize<4>::UInt +// +// is typedef-ed to be unsigned int (unsigned integer made up of 4 +// bytes). +// +// Such functionality should belong to STL, but I cannot find it +// there. +// +// Google Test uses this class in the implementation of floating-point +// comparison. +// +// For now it only handles UInt (unsigned int) as that's all Google Test +// needs. Other types can be easily added in the future if need +// arises. +template +class TypeWithSize { + public: + // This prevents the user from using TypeWithSize with incorrect + // values of N. + typedef void UInt; +}; + +// The specialization for size 4. +template <> +class TypeWithSize<4> { + public: + // unsigned int has size 4 in both gcc and MSVC. + // + // As base/basictypes.h doesn't compile on Windows, we cannot use + // uint32, uint64, and etc here. + typedef int Int; + typedef unsigned int UInt; +}; + +// The specialization for size 8. +template <> +class TypeWithSize<8> { + public: +#if GTEST_OS_WINDOWS + typedef __int64 Int; + typedef unsigned __int64 UInt; +#else + typedef long long Int; // NOLINT + typedef unsigned long long UInt; // NOLINT +#endif // GTEST_OS_WINDOWS +}; + +// Integer types of known sizes. +typedef TypeWithSize<4>::Int Int32; +typedef TypeWithSize<4>::UInt UInt32; +typedef TypeWithSize<8>::Int Int64; +typedef TypeWithSize<8>::UInt UInt64; +typedef TypeWithSize<8>::Int TimeInMillis; // Represents time in milliseconds. + +// Utilities for command line flags and environment variables. + +// Macro for referencing flags. +#if !defined(GTEST_FLAG) +# define GTEST_FLAG(name) FLAGS_gtest_##name +#endif // !defined(GTEST_FLAG) + +#if !defined(GTEST_USE_OWN_FLAGFILE_FLAG_) +# define GTEST_USE_OWN_FLAGFILE_FLAG_ 1 +#endif // !defined(GTEST_USE_OWN_FLAGFILE_FLAG_) + +#if !defined(GTEST_DECLARE_bool_) +# define GTEST_FLAG_SAVER_ ::testing::internal::GTestFlagSaver + +// Macros for declaring flags. +# define GTEST_DECLARE_bool_(name) GTEST_API_ extern bool GTEST_FLAG(name) +# define GTEST_DECLARE_int32_(name) \ + GTEST_API_ extern ::testing::internal::Int32 GTEST_FLAG(name) +# define GTEST_DECLARE_string_(name) \ + GTEST_API_ extern ::std::string GTEST_FLAG(name) + +// Macros for defining flags. +# define GTEST_DEFINE_bool_(name, default_val, doc) \ + GTEST_API_ bool GTEST_FLAG(name) = (default_val) +# define GTEST_DEFINE_int32_(name, default_val, doc) \ + GTEST_API_ ::testing::internal::Int32 GTEST_FLAG(name) = (default_val) +# define GTEST_DEFINE_string_(name, default_val, doc) \ + GTEST_API_ ::std::string GTEST_FLAG(name) = (default_val) + +#endif // !defined(GTEST_DECLARE_bool_) + +// Thread annotations +#if !defined(GTEST_EXCLUSIVE_LOCK_REQUIRED_) +# define GTEST_EXCLUSIVE_LOCK_REQUIRED_(locks) +# define GTEST_LOCK_EXCLUDED_(locks) +#endif // !defined(GTEST_EXCLUSIVE_LOCK_REQUIRED_) + +// Parses 'str' for a 32-bit signed integer. If successful, writes the result +// to *value and returns true; otherwise leaves *value unchanged and returns +// false. +// FIXME: Find a better way to refactor flag and environment parsing +// out of both gtest-port.cc and gtest.cc to avoid exporting this utility +// function. +bool ParseInt32(const Message& src_text, const char* str, Int32* value); + +// Parses a bool/Int32/string from the environment variable +// corresponding to the given Google Test flag. +bool BoolFromGTestEnv(const char* flag, bool default_val); +GTEST_API_ Int32 Int32FromGTestEnv(const char* flag, Int32 default_val); +std::string OutputFlagAlsoCheckEnvVar(); +const char* StringFromGTestEnv(const char* flag, const char* default_val); + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ + +#if GTEST_OS_LINUX +# include +# include +# include +# include +#endif // GTEST_OS_LINUX + +#if GTEST_HAS_EXCEPTIONS +# include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// +// The Google C++ Testing and Mocking Framework (Google Test) +// +// This header file defines the Message class. +// +// IMPORTANT NOTE: Due to limitation of the C++ language, we have to +// leave some internal implementation details in this header file. +// They are clearly marked by comments like this: +// +// // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +// +// Such code is NOT meant to be used by a user directly, and is subject +// to CHANGE WITHOUT NOTICE. Therefore DO NOT DEPEND ON IT in a user +// program! + +// GOOGLETEST_CM0001 DO NOT DELETE + +#ifndef GTEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ +#define GTEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ + +#include + + +GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ +/* class A needs to have dll-interface to be used by clients of class B */) + +// Ensures that there is at least one operator<< in the global namespace. +// See Message& operator<<(...) below for why. +void operator<<(const testing::internal::Secret&, int); + +namespace testing { + +// The Message class works like an ostream repeater. +// +// Typical usage: +// +// 1. You stream a bunch of values to a Message object. +// It will remember the text in a stringstream. +// 2. Then you stream the Message object to an ostream. +// This causes the text in the Message to be streamed +// to the ostream. +// +// For example; +// +// testing::Message foo; +// foo << 1 << " != " << 2; +// std::cout << foo; +// +// will print "1 != 2". +// +// Message is not intended to be inherited from. In particular, its +// destructor is not virtual. +// +// Note that stringstream behaves differently in gcc and in MSVC. You +// can stream a NULL char pointer to it in the former, but not in the +// latter (it causes an access violation if you do). The Message +// class hides this difference by treating a NULL char pointer as +// "(null)". +class GTEST_API_ Message { + private: + // The type of basic IO manipulators (endl, ends, and flush) for + // narrow streams. + typedef std::ostream& (*BasicNarrowIoManip)(std::ostream&); + + public: + // Constructs an empty Message. + Message(); + + // Copy constructor. + Message(const Message& msg) : ss_(new ::std::stringstream) { // NOLINT + *ss_ << msg.GetString(); + } + + // Constructs a Message from a C-string. + explicit Message(const char* str) : ss_(new ::std::stringstream) { + *ss_ << str; + } + +#if GTEST_OS_SYMBIAN + // Streams a value (either a pointer or not) to this object. + template + inline Message& operator <<(const T& value) { + StreamHelper(typename internal::is_pointer::type(), value); + return *this; + } +#else + // Streams a non-pointer value to this object. + template + inline Message& operator <<(const T& val) { + // Some libraries overload << for STL containers. These + // overloads are defined in the global namespace instead of ::std. + // + // C++'s symbol lookup rule (i.e. Koenig lookup) says that these + // overloads are visible in either the std namespace or the global + // namespace, but not other namespaces, including the testing + // namespace which Google Test's Message class is in. + // + // To allow STL containers (and other types that has a << operator + // defined in the global namespace) to be used in Google Test + // assertions, testing::Message must access the custom << operator + // from the global namespace. With this using declaration, + // overloads of << defined in the global namespace and those + // visible via Koenig lookup are both exposed in this function. + using ::operator <<; + *ss_ << val; + return *this; + } + + // Streams a pointer value to this object. + // + // This function is an overload of the previous one. When you + // stream a pointer to a Message, this definition will be used as it + // is more specialized. (The C++ Standard, section + // [temp.func.order].) If you stream a non-pointer, then the + // previous definition will be used. + // + // The reason for this overload is that streaming a NULL pointer to + // ostream is undefined behavior. Depending on the compiler, you + // may get "0", "(nil)", "(null)", or an access violation. To + // ensure consistent result across compilers, we always treat NULL + // as "(null)". + template + inline Message& operator <<(T* const& pointer) { // NOLINT + if (pointer == NULL) { + *ss_ << "(null)"; + } else { + *ss_ << pointer; + } + return *this; + } +#endif // GTEST_OS_SYMBIAN + + // Since the basic IO manipulators are overloaded for both narrow + // and wide streams, we have to provide this specialized definition + // of operator <<, even though its body is the same as the + // templatized version above. Without this definition, streaming + // endl or other basic IO manipulators to Message will confuse the + // compiler. + Message& operator <<(BasicNarrowIoManip val) { + *ss_ << val; + return *this; + } + + // Instead of 1/0, we want to see true/false for bool values. + Message& operator <<(bool b) { + return *this << (b ? "true" : "false"); + } + + // These two overloads allow streaming a wide C string to a Message + // using the UTF-8 encoding. + Message& operator <<(const wchar_t* wide_c_str); + Message& operator <<(wchar_t* wide_c_str); + +#if GTEST_HAS_STD_WSTRING + // Converts the given wide string to a narrow string using the UTF-8 + // encoding, and streams the result to this Message object. + Message& operator <<(const ::std::wstring& wstr); +#endif // GTEST_HAS_STD_WSTRING + +#if GTEST_HAS_GLOBAL_WSTRING + // Converts the given wide string to a narrow string using the UTF-8 + // encoding, and streams the result to this Message object. + Message& operator <<(const ::wstring& wstr); +#endif // GTEST_HAS_GLOBAL_WSTRING + + // Gets the text streamed to this object so far as an std::string. + // Each '\0' character in the buffer is replaced with "\\0". + // + // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + std::string GetString() const; + + private: +#if GTEST_OS_SYMBIAN + // These are needed as the Nokia Symbian Compiler cannot decide between + // const T& and const T* in a function template. The Nokia compiler _can_ + // decide between class template specializations for T and T*, so a + // tr1::type_traits-like is_pointer works, and we can overload on that. + template + inline void StreamHelper(internal::true_type /*is_pointer*/, T* pointer) { + if (pointer == NULL) { + *ss_ << "(null)"; + } else { + *ss_ << pointer; + } + } + template + inline void StreamHelper(internal::false_type /*is_pointer*/, + const T& value) { + // See the comments in Message& operator <<(const T&) above for why + // we need this using statement. + using ::operator <<; + *ss_ << value; + } +#endif // GTEST_OS_SYMBIAN + + // We'll hold the text streamed to this object here. + const internal::scoped_ptr< ::std::stringstream> ss_; + + // We declare (but don't implement) this to prevent the compiler + // from implementing the assignment operator. + void operator=(const Message&); +}; + +// Streams a Message to an ostream. +inline std::ostream& operator <<(std::ostream& os, const Message& sb) { + return os << sb.GetString(); +} + +namespace internal { + +// Converts a streamable value to an std::string. A NULL pointer is +// converted to "(null)". When the input value is a ::string, +// ::std::string, ::wstring, or ::std::wstring object, each NUL +// character in it is replaced with "\\0". +template +std::string StreamableToString(const T& streamable) { + return (Message() << streamable).GetString(); +} + +} // namespace internal +} // namespace testing + +GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 + +#endif // GTEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Google Test filepath utilities +// +// This header file declares classes and functions used internally by +// Google Test. They are subject to change without notice. +// +// This file is #included in gtest/internal/gtest-internal.h. +// Do not include this header file separately! + +// GOOGLETEST_CM0001 DO NOT DELETE + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ + +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// The Google C++ Testing and Mocking Framework (Google Test) +// +// This header file declares the String class and functions used internally by +// Google Test. They are subject to change without notice. They should not used +// by code external to Google Test. +// +// This header file is #included by gtest-internal.h. +// It should not be #included by other files. + +// GOOGLETEST_CM0001 DO NOT DELETE + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ + +#ifdef __BORLANDC__ +// string.h is not guaranteed to provide strcpy on C++ Builder. +# include +#endif + +#include +#include + + +namespace testing { +namespace internal { + +// String - an abstract class holding static string utilities. +class GTEST_API_ String { + public: + // Static utility methods + + // Clones a 0-terminated C string, allocating memory using new. The + // caller is responsible for deleting the return value using + // delete[]. Returns the cloned string, or NULL if the input is + // NULL. + // + // This is different from strdup() in string.h, which allocates + // memory using malloc(). + static const char* CloneCString(const char* c_str); + +#if GTEST_OS_WINDOWS_MOBILE + // Windows CE does not have the 'ANSI' versions of Win32 APIs. To be + // able to pass strings to Win32 APIs on CE we need to convert them + // to 'Unicode', UTF-16. + + // Creates a UTF-16 wide string from the given ANSI string, allocating + // memory using new. The caller is responsible for deleting the return + // value using delete[]. Returns the wide string, or NULL if the + // input is NULL. + // + // The wide string is created using the ANSI codepage (CP_ACP) to + // match the behaviour of the ANSI versions of Win32 calls and the + // C runtime. + static LPCWSTR AnsiToUtf16(const char* c_str); + + // Creates an ANSI string from the given wide string, allocating + // memory using new. The caller is responsible for deleting the return + // value using delete[]. Returns the ANSI string, or NULL if the + // input is NULL. + // + // The returned string is created using the ANSI codepage (CP_ACP) to + // match the behaviour of the ANSI versions of Win32 calls and the + // C runtime. + static const char* Utf16ToAnsi(LPCWSTR utf16_str); +#endif + + // Compares two C strings. Returns true iff they have the same content. + // + // Unlike strcmp(), this function can handle NULL argument(s). A + // NULL C string is considered different to any non-NULL C string, + // including the empty string. + static bool CStringEquals(const char* lhs, const char* rhs); + + // Converts a wide C string to a String using the UTF-8 encoding. + // NULL will be converted to "(null)". If an error occurred during + // the conversion, "(failed to convert from wide string)" is + // returned. + static std::string ShowWideCString(const wchar_t* wide_c_str); + + // Compares two wide C strings. Returns true iff they have the same + // content. + // + // Unlike wcscmp(), this function can handle NULL argument(s). A + // NULL C string is considered different to any non-NULL C string, + // including the empty string. + static bool WideCStringEquals(const wchar_t* lhs, const wchar_t* rhs); + + // Compares two C strings, ignoring case. Returns true iff they + // have the same content. + // + // Unlike strcasecmp(), this function can handle NULL argument(s). + // A NULL C string is considered different to any non-NULL C string, + // including the empty string. + static bool CaseInsensitiveCStringEquals(const char* lhs, + const char* rhs); + + // Compares two wide C strings, ignoring case. Returns true iff they + // have the same content. + // + // Unlike wcscasecmp(), this function can handle NULL argument(s). + // A NULL C string is considered different to any non-NULL wide C string, + // including the empty string. + // NB: The implementations on different platforms slightly differ. + // On windows, this method uses _wcsicmp which compares according to LC_CTYPE + // environment variable. On GNU platform this method uses wcscasecmp + // which compares according to LC_CTYPE category of the current locale. + // On MacOS X, it uses towlower, which also uses LC_CTYPE category of the + // current locale. + static bool CaseInsensitiveWideCStringEquals(const wchar_t* lhs, + const wchar_t* rhs); + + // Returns true iff the given string ends with the given suffix, ignoring + // case. Any string is considered to end with an empty suffix. + static bool EndsWithCaseInsensitive( + const std::string& str, const std::string& suffix); + + // Formats an int value as "%02d". + static std::string FormatIntWidth2(int value); // "%02d" for width == 2 + + // Formats an int value as "%X". + static std::string FormatHexInt(int value); + + // Formats a byte as "%02X". + static std::string FormatByte(unsigned char value); + + private: + String(); // Not meant to be instantiated. +}; // class String + +// Gets the content of the stringstream's buffer as an std::string. Each '\0' +// character in the buffer is replaced with "\\0". +GTEST_API_ std::string StringStreamToString(::std::stringstream* stream); + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ + +GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ +/* class A needs to have dll-interface to be used by clients of class B */) + +namespace testing { +namespace internal { + +// FilePath - a class for file and directory pathname manipulation which +// handles platform-specific conventions (like the pathname separator). +// Used for helper functions for naming files in a directory for xml output. +// Except for Set methods, all methods are const or static, which provides an +// "immutable value object" -- useful for peace of mind. +// A FilePath with a value ending in a path separator ("like/this/") represents +// a directory, otherwise it is assumed to represent a file. In either case, +// it may or may not represent an actual file or directory in the file system. +// Names are NOT checked for syntax correctness -- no checking for illegal +// characters, malformed paths, etc. + +class GTEST_API_ FilePath { + public: + FilePath() : pathname_("") { } + FilePath(const FilePath& rhs) : pathname_(rhs.pathname_) { } + + explicit FilePath(const std::string& pathname) : pathname_(pathname) { + Normalize(); + } + + FilePath& operator=(const FilePath& rhs) { + Set(rhs); + return *this; + } + + void Set(const FilePath& rhs) { + pathname_ = rhs.pathname_; + } + + const std::string& string() const { return pathname_; } + const char* c_str() const { return pathname_.c_str(); } + + // Returns the current working directory, or "" if unsuccessful. + static FilePath GetCurrentDir(); + + // Given directory = "dir", base_name = "test", number = 0, + // extension = "xml", returns "dir/test.xml". If number is greater + // than zero (e.g., 12), returns "dir/test_12.xml". + // On Windows platform, uses \ as the separator rather than /. + static FilePath MakeFileName(const FilePath& directory, + const FilePath& base_name, + int number, + const char* extension); + + // Given directory = "dir", relative_path = "test.xml", + // returns "dir/test.xml". + // On Windows, uses \ as the separator rather than /. + static FilePath ConcatPaths(const FilePath& directory, + const FilePath& relative_path); + + // Returns a pathname for a file that does not currently exist. The pathname + // will be directory/base_name.extension or + // directory/base_name_.extension if directory/base_name.extension + // already exists. The number will be incremented until a pathname is found + // that does not already exist. + // Examples: 'dir/foo_test.xml' or 'dir/foo_test_1.xml'. + // There could be a race condition if two or more processes are calling this + // function at the same time -- they could both pick the same filename. + static FilePath GenerateUniqueFileName(const FilePath& directory, + const FilePath& base_name, + const char* extension); + + // Returns true iff the path is "". + bool IsEmpty() const { return pathname_.empty(); } + + // If input name has a trailing separator character, removes it and returns + // the name, otherwise return the name string unmodified. + // On Windows platform, uses \ as the separator, other platforms use /. + FilePath RemoveTrailingPathSeparator() const; + + // Returns a copy of the FilePath with the directory part removed. + // Example: FilePath("path/to/file").RemoveDirectoryName() returns + // FilePath("file"). If there is no directory part ("just_a_file"), it returns + // the FilePath unmodified. If there is no file part ("just_a_dir/") it + // returns an empty FilePath (""). + // On Windows platform, '\' is the path separator, otherwise it is '/'. + FilePath RemoveDirectoryName() const; + + // RemoveFileName returns the directory path with the filename removed. + // Example: FilePath("path/to/file").RemoveFileName() returns "path/to/". + // If the FilePath is "a_file" or "/a_file", RemoveFileName returns + // FilePath("./") or, on Windows, FilePath(".\\"). If the filepath does + // not have a file, like "just/a/dir/", it returns the FilePath unmodified. + // On Windows platform, '\' is the path separator, otherwise it is '/'. + FilePath RemoveFileName() const; + + // Returns a copy of the FilePath with the case-insensitive extension removed. + // Example: FilePath("dir/file.exe").RemoveExtension("EXE") returns + // FilePath("dir/file"). If a case-insensitive extension is not + // found, returns a copy of the original FilePath. + FilePath RemoveExtension(const char* extension) const; + + // Creates directories so that path exists. Returns true if successful or if + // the directories already exist; returns false if unable to create + // directories for any reason. Will also return false if the FilePath does + // not represent a directory (that is, it doesn't end with a path separator). + bool CreateDirectoriesRecursively() const; + + // Create the directory so that path exists. Returns true if successful or + // if the directory already exists; returns false if unable to create the + // directory for any reason, including if the parent directory does not + // exist. Not named "CreateDirectory" because that's a macro on Windows. + bool CreateFolder() const; + + // Returns true if FilePath describes something in the file-system, + // either a file, directory, or whatever, and that something exists. + bool FileOrDirectoryExists() const; + + // Returns true if pathname describes a directory in the file-system + // that exists. + bool DirectoryExists() const; + + // Returns true if FilePath ends with a path separator, which indicates that + // it is intended to represent a directory. Returns false otherwise. + // This does NOT check that a directory (or file) actually exists. + bool IsDirectory() const; + + // Returns true if pathname describes a root directory. (Windows has one + // root directory per disk drive.) + bool IsRootDirectory() const; + + // Returns true if pathname describes an absolute path. + bool IsAbsolutePath() const; + + private: + // Replaces multiple consecutive separators with a single separator. + // For example, "bar///foo" becomes "bar/foo". Does not eliminate other + // redundancies that might be in a pathname involving "." or "..". + // + // A pathname with multiple consecutive separators may occur either through + // user error or as a result of some scripts or APIs that generate a pathname + // with a trailing separator. On other platforms the same API or script + // may NOT generate a pathname with a trailing "/". Then elsewhere that + // pathname may have another "/" and pathname components added to it, + // without checking for the separator already being there. + // The script language and operating system may allow paths like "foo//bar" + // but some of the functions in FilePath will not handle that correctly. In + // particular, RemoveTrailingPathSeparator() only removes one separator, and + // it is called in CreateDirectoriesRecursively() assuming that it will change + // a pathname from directory syntax (trailing separator) to filename syntax. + // + // On Windows this method also replaces the alternate path separator '/' with + // the primary path separator '\\', so that for example "bar\\/\\foo" becomes + // "bar\\foo". + + void Normalize(); + + // Returns a pointer to the last occurence of a valid path separator in + // the FilePath. On Windows, for example, both '/' and '\' are valid path + // separators. Returns NULL if no path separator was found. + const char* FindLastPathSeparator() const; + + std::string pathname_; +}; // class FilePath + +} // namespace internal +} // namespace testing + +GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ +// This file was GENERATED by command: +// pump.py gtest-type-util.h.pump +// DO NOT EDIT BY HAND!!! + +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +// Type utilities needed for implementing typed and type-parameterized +// tests. This file is generated by a SCRIPT. DO NOT EDIT BY HAND! +// +// Currently we support at most 50 types in a list, and at most 50 +// type-parameterized tests in one type-parameterized test case. +// Please contact googletestframework@googlegroups.com if you need +// more. + +// GOOGLETEST_CM0001 DO NOT DELETE + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ + + +// #ifdef __GNUC__ is too general here. It is possible to use gcc without using +// libstdc++ (which is where cxxabi.h comes from). +# if GTEST_HAS_CXXABI_H_ +# include +# elif defined(__HP_aCC) +# include +# endif // GTEST_HASH_CXXABI_H_ + +namespace testing { +namespace internal { + +// Canonicalizes a given name with respect to the Standard C++ Library. +// This handles removing the inline namespace within `std` that is +// used by various standard libraries (e.g., `std::__1`). Names outside +// of namespace std are returned unmodified. +inline std::string CanonicalizeForStdLibVersioning(std::string s) { + static const char prefix[] = "std::__"; + if (s.compare(0, strlen(prefix), prefix) == 0) { + std::string::size_type end = s.find("::", strlen(prefix)); + if (end != s.npos) { + // Erase everything between the initial `std` and the second `::`. + s.erase(strlen("std"), end - strlen("std")); + } + } + return s; +} + +// GetTypeName() returns a human-readable name of type T. +// NB: This function is also used in Google Mock, so don't move it inside of +// the typed-test-only section below. +template +std::string GetTypeName() { +# if GTEST_HAS_RTTI + + const char* const name = typeid(T).name(); +# if GTEST_HAS_CXXABI_H_ || defined(__HP_aCC) + int status = 0; + // gcc's implementation of typeid(T).name() mangles the type name, + // so we have to demangle it. +# if GTEST_HAS_CXXABI_H_ + using abi::__cxa_demangle; +# endif // GTEST_HAS_CXXABI_H_ + char* const readable_name = __cxa_demangle(name, 0, 0, &status); + const std::string name_str(status == 0 ? readable_name : name); + free(readable_name); + return CanonicalizeForStdLibVersioning(name_str); +# else + return name; +# endif // GTEST_HAS_CXXABI_H_ || __HP_aCC + +# else + + return ""; + +# endif // GTEST_HAS_RTTI +} + +#if GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P + +// AssertyTypeEq::type is defined iff T1 and T2 are the same +// type. This can be used as a compile-time assertion to ensure that +// two types are equal. + +template +struct AssertTypeEq; + +template +struct AssertTypeEq { + typedef bool type; +}; + +// A unique type used as the default value for the arguments of class +// template Types. This allows us to simulate variadic templates +// (e.g. Types, Type, and etc), which C++ doesn't +// support directly. +struct None {}; + +// The following family of struct and struct templates are used to +// represent type lists. In particular, TypesN +// represents a type list with N types (T1, T2, ..., and TN) in it. +// Except for Types0, every struct in the family has two member types: +// Head for the first type in the list, and Tail for the rest of the +// list. + +// The empty type list. +struct Types0 {}; + +// Type lists of length 1, 2, 3, and so on. + +template +struct Types1 { + typedef T1 Head; + typedef Types0 Tail; +}; +template +struct Types2 { + typedef T1 Head; + typedef Types1 Tail; +}; + +template +struct Types3 { + typedef T1 Head; + typedef Types2 Tail; +}; + +template +struct Types4 { + typedef T1 Head; + typedef Types3 Tail; +}; + +template +struct Types5 { + typedef T1 Head; + typedef Types4 Tail; +}; + +template +struct Types6 { + typedef T1 Head; + typedef Types5 Tail; +}; + +template +struct Types7 { + typedef T1 Head; + typedef Types6 Tail; +}; + +template +struct Types8 { + typedef T1 Head; + typedef Types7 Tail; +}; + +template +struct Types9 { + typedef T1 Head; + typedef Types8 Tail; +}; + +template +struct Types10 { + typedef T1 Head; + typedef Types9 Tail; +}; + +template +struct Types11 { + typedef T1 Head; + typedef Types10 Tail; +}; + +template +struct Types12 { + typedef T1 Head; + typedef Types11 Tail; +}; + +template +struct Types13 { + typedef T1 Head; + typedef Types12 Tail; +}; + +template +struct Types14 { + typedef T1 Head; + typedef Types13 Tail; +}; + +template +struct Types15 { + typedef T1 Head; + typedef Types14 Tail; +}; + +template +struct Types16 { + typedef T1 Head; + typedef Types15 Tail; +}; + +template +struct Types17 { + typedef T1 Head; + typedef Types16 Tail; +}; + +template +struct Types18 { + typedef T1 Head; + typedef Types17 Tail; +}; + +template +struct Types19 { + typedef T1 Head; + typedef Types18 Tail; +}; + +template +struct Types20 { + typedef T1 Head; + typedef Types19 Tail; +}; + +template +struct Types21 { + typedef T1 Head; + typedef Types20 Tail; +}; + +template +struct Types22 { + typedef T1 Head; + typedef Types21 Tail; +}; + +template +struct Types23 { + typedef T1 Head; + typedef Types22 Tail; +}; + +template +struct Types24 { + typedef T1 Head; + typedef Types23 Tail; +}; + +template +struct Types25 { + typedef T1 Head; + typedef Types24 Tail; +}; + +template +struct Types26 { + typedef T1 Head; + typedef Types25 Tail; +}; + +template +struct Types27 { + typedef T1 Head; + typedef Types26 Tail; +}; + +template +struct Types28 { + typedef T1 Head; + typedef Types27 Tail; +}; + +template +struct Types29 { + typedef T1 Head; + typedef Types28 Tail; +}; + +template +struct Types30 { + typedef T1 Head; + typedef Types29 Tail; +}; + +template +struct Types31 { + typedef T1 Head; + typedef Types30 Tail; +}; + +template +struct Types32 { + typedef T1 Head; + typedef Types31 Tail; +}; + +template +struct Types33 { + typedef T1 Head; + typedef Types32 Tail; +}; + +template +struct Types34 { + typedef T1 Head; + typedef Types33 Tail; +}; + +template +struct Types35 { + typedef T1 Head; + typedef Types34 Tail; +}; + +template +struct Types36 { + typedef T1 Head; + typedef Types35 Tail; +}; + +template +struct Types37 { + typedef T1 Head; + typedef Types36 Tail; +}; + +template +struct Types38 { + typedef T1 Head; + typedef Types37 Tail; +}; + +template +struct Types39 { + typedef T1 Head; + typedef Types38 Tail; +}; + +template +struct Types40 { + typedef T1 Head; + typedef Types39 Tail; +}; + +template +struct Types41 { + typedef T1 Head; + typedef Types40 Tail; +}; + +template +struct Types42 { + typedef T1 Head; + typedef Types41 Tail; +}; + +template +struct Types43 { + typedef T1 Head; + typedef Types42 Tail; +}; + +template +struct Types44 { + typedef T1 Head; + typedef Types43 Tail; +}; + +template +struct Types45 { + typedef T1 Head; + typedef Types44 Tail; +}; + +template +struct Types46 { + typedef T1 Head; + typedef Types45 Tail; +}; + +template +struct Types47 { + typedef T1 Head; + typedef Types46 Tail; +}; + +template +struct Types48 { + typedef T1 Head; + typedef Types47 Tail; +}; + +template +struct Types49 { + typedef T1 Head; + typedef Types48 Tail; +}; + +template +struct Types50 { + typedef T1 Head; + typedef Types49 Tail; +}; + + +} // namespace internal + +// We don't want to require the users to write TypesN<...> directly, +// as that would require them to count the length. Types<...> is much +// easier to write, but generates horrible messages when there is a +// compiler error, as gcc insists on printing out each template +// argument, even if it has the default value (this means Types +// will appear as Types in the compiler +// errors). +// +// Our solution is to combine the best part of the two approaches: a +// user would write Types, and Google Test will translate +// that to TypesN internally to make error messages +// readable. The translation is done by the 'type' member of the +// Types template. +template +struct Types { + typedef internal::Types50 type; +}; + +template <> +struct Types { + typedef internal::Types0 type; +}; +template +struct Types { + typedef internal::Types1 type; +}; +template +struct Types { + typedef internal::Types2 type; +}; +template +struct Types { + typedef internal::Types3 type; +}; +template +struct Types { + typedef internal::Types4 type; +}; +template +struct Types { + typedef internal::Types5 type; +}; +template +struct Types { + typedef internal::Types6 type; +}; +template +struct Types { + typedef internal::Types7 type; +}; +template +struct Types { + typedef internal::Types8 type; +}; +template +struct Types { + typedef internal::Types9 type; +}; +template +struct Types { + typedef internal::Types10 type; +}; +template +struct Types { + typedef internal::Types11 type; +}; +template +struct Types { + typedef internal::Types12 type; +}; +template +struct Types { + typedef internal::Types13 type; +}; +template +struct Types { + typedef internal::Types14 type; +}; +template +struct Types { + typedef internal::Types15 type; +}; +template +struct Types { + typedef internal::Types16 type; +}; +template +struct Types { + typedef internal::Types17 type; +}; +template +struct Types { + typedef internal::Types18 type; +}; +template +struct Types { + typedef internal::Types19 type; +}; +template +struct Types { + typedef internal::Types20 type; +}; +template +struct Types { + typedef internal::Types21 type; +}; +template +struct Types { + typedef internal::Types22 type; +}; +template +struct Types { + typedef internal::Types23 type; +}; +template +struct Types { + typedef internal::Types24 type; +}; +template +struct Types { + typedef internal::Types25 type; +}; +template +struct Types { + typedef internal::Types26 type; +}; +template +struct Types { + typedef internal::Types27 type; +}; +template +struct Types { + typedef internal::Types28 type; +}; +template +struct Types { + typedef internal::Types29 type; +}; +template +struct Types { + typedef internal::Types30 type; +}; +template +struct Types { + typedef internal::Types31 type; +}; +template +struct Types { + typedef internal::Types32 type; +}; +template +struct Types { + typedef internal::Types33 type; +}; +template +struct Types { + typedef internal::Types34 type; +}; +template +struct Types { + typedef internal::Types35 type; +}; +template +struct Types { + typedef internal::Types36 type; +}; +template +struct Types { + typedef internal::Types37 type; +}; +template +struct Types { + typedef internal::Types38 type; +}; +template +struct Types { + typedef internal::Types39 type; +}; +template +struct Types { + typedef internal::Types40 type; +}; +template +struct Types { + typedef internal::Types41 type; +}; +template +struct Types { + typedef internal::Types42 type; +}; +template +struct Types { + typedef internal::Types43 type; +}; +template +struct Types { + typedef internal::Types44 type; +}; +template +struct Types { + typedef internal::Types45 type; +}; +template +struct Types { + typedef internal::Types46 type; +}; +template +struct Types { + typedef internal::Types47 type; +}; +template +struct Types { + typedef internal::Types48 type; +}; +template +struct Types { + typedef internal::Types49 type; +}; + +namespace internal { + +# define GTEST_TEMPLATE_ template class + +// The template "selector" struct TemplateSel is used to +// represent Tmpl, which must be a class template with one type +// parameter, as a type. TemplateSel::Bind::type is defined +// as the type Tmpl. This allows us to actually instantiate the +// template "selected" by TemplateSel. +// +// This trick is necessary for simulating typedef for class templates, +// which C++ doesn't support directly. +template +struct TemplateSel { + template + struct Bind { + typedef Tmpl type; + }; +}; + +# define GTEST_BIND_(TmplSel, T) \ + TmplSel::template Bind::type + +// A unique struct template used as the default value for the +// arguments of class template Templates. This allows us to simulate +// variadic templates (e.g. Templates, Templates, +// and etc), which C++ doesn't support directly. +template +struct NoneT {}; + +// The following family of struct and struct templates are used to +// represent template lists. In particular, TemplatesN represents a list of N templates (T1, T2, ..., and TN). Except +// for Templates0, every struct in the family has two member types: +// Head for the selector of the first template in the list, and Tail +// for the rest of the list. + +// The empty template list. +struct Templates0 {}; + +// Template lists of length 1, 2, 3, and so on. + +template +struct Templates1 { + typedef TemplateSel Head; + typedef Templates0 Tail; +}; +template +struct Templates2 { + typedef TemplateSel Head; + typedef Templates1 Tail; +}; + +template +struct Templates3 { + typedef TemplateSel Head; + typedef Templates2 Tail; +}; + +template +struct Templates4 { + typedef TemplateSel Head; + typedef Templates3 Tail; +}; + +template +struct Templates5 { + typedef TemplateSel Head; + typedef Templates4 Tail; +}; + +template +struct Templates6 { + typedef TemplateSel Head; + typedef Templates5 Tail; +}; + +template +struct Templates7 { + typedef TemplateSel Head; + typedef Templates6 Tail; +}; + +template +struct Templates8 { + typedef TemplateSel Head; + typedef Templates7 Tail; +}; + +template +struct Templates9 { + typedef TemplateSel Head; + typedef Templates8 Tail; +}; + +template +struct Templates10 { + typedef TemplateSel Head; + typedef Templates9 Tail; +}; + +template +struct Templates11 { + typedef TemplateSel Head; + typedef Templates10 Tail; +}; + +template +struct Templates12 { + typedef TemplateSel Head; + typedef Templates11 Tail; +}; + +template +struct Templates13 { + typedef TemplateSel Head; + typedef Templates12 Tail; +}; + +template +struct Templates14 { + typedef TemplateSel Head; + typedef Templates13 Tail; +}; + +template +struct Templates15 { + typedef TemplateSel Head; + typedef Templates14 Tail; +}; + +template +struct Templates16 { + typedef TemplateSel Head; + typedef Templates15 Tail; +}; + +template +struct Templates17 { + typedef TemplateSel Head; + typedef Templates16 Tail; +}; + +template +struct Templates18 { + typedef TemplateSel Head; + typedef Templates17 Tail; +}; + +template +struct Templates19 { + typedef TemplateSel Head; + typedef Templates18 Tail; +}; + +template +struct Templates20 { + typedef TemplateSel Head; + typedef Templates19 Tail; +}; + +template +struct Templates21 { + typedef TemplateSel Head; + typedef Templates20 Tail; +}; + +template +struct Templates22 { + typedef TemplateSel Head; + typedef Templates21 Tail; +}; + +template +struct Templates23 { + typedef TemplateSel Head; + typedef Templates22 Tail; +}; + +template +struct Templates24 { + typedef TemplateSel Head; + typedef Templates23 Tail; +}; + +template +struct Templates25 { + typedef TemplateSel Head; + typedef Templates24 Tail; +}; + +template +struct Templates26 { + typedef TemplateSel Head; + typedef Templates25 Tail; +}; + +template +struct Templates27 { + typedef TemplateSel Head; + typedef Templates26 Tail; +}; + +template +struct Templates28 { + typedef TemplateSel Head; + typedef Templates27 Tail; +}; + +template +struct Templates29 { + typedef TemplateSel Head; + typedef Templates28 Tail; +}; + +template +struct Templates30 { + typedef TemplateSel Head; + typedef Templates29 Tail; +}; + +template +struct Templates31 { + typedef TemplateSel Head; + typedef Templates30 Tail; +}; + +template +struct Templates32 { + typedef TemplateSel Head; + typedef Templates31 Tail; +}; + +template +struct Templates33 { + typedef TemplateSel Head; + typedef Templates32 Tail; +}; + +template +struct Templates34 { + typedef TemplateSel Head; + typedef Templates33 Tail; +}; + +template +struct Templates35 { + typedef TemplateSel Head; + typedef Templates34 Tail; +}; + +template +struct Templates36 { + typedef TemplateSel Head; + typedef Templates35 Tail; +}; + +template +struct Templates37 { + typedef TemplateSel Head; + typedef Templates36 Tail; +}; + +template +struct Templates38 { + typedef TemplateSel Head; + typedef Templates37 Tail; +}; + +template +struct Templates39 { + typedef TemplateSel Head; + typedef Templates38 Tail; +}; + +template +struct Templates40 { + typedef TemplateSel Head; + typedef Templates39 Tail; +}; + +template +struct Templates41 { + typedef TemplateSel Head; + typedef Templates40 Tail; +}; + +template +struct Templates42 { + typedef TemplateSel Head; + typedef Templates41 Tail; +}; + +template +struct Templates43 { + typedef TemplateSel Head; + typedef Templates42 Tail; +}; + +template +struct Templates44 { + typedef TemplateSel Head; + typedef Templates43 Tail; +}; + +template +struct Templates45 { + typedef TemplateSel Head; + typedef Templates44 Tail; +}; + +template +struct Templates46 { + typedef TemplateSel Head; + typedef Templates45 Tail; +}; + +template +struct Templates47 { + typedef TemplateSel Head; + typedef Templates46 Tail; +}; + +template +struct Templates48 { + typedef TemplateSel Head; + typedef Templates47 Tail; +}; + +template +struct Templates49 { + typedef TemplateSel Head; + typedef Templates48 Tail; +}; + +template +struct Templates50 { + typedef TemplateSel Head; + typedef Templates49 Tail; +}; + + +// We don't want to require the users to write TemplatesN<...> directly, +// as that would require them to count the length. Templates<...> is much +// easier to write, but generates horrible messages when there is a +// compiler error, as gcc insists on printing out each template +// argument, even if it has the default value (this means Templates +// will appear as Templates in the compiler +// errors). +// +// Our solution is to combine the best part of the two approaches: a +// user would write Templates, and Google Test will translate +// that to TemplatesN internally to make error messages +// readable. The translation is done by the 'type' member of the +// Templates template. +template +struct Templates { + typedef Templates50 type; +}; + +template <> +struct Templates { + typedef Templates0 type; +}; +template +struct Templates { + typedef Templates1 type; +}; +template +struct Templates { + typedef Templates2 type; +}; +template +struct Templates { + typedef Templates3 type; +}; +template +struct Templates { + typedef Templates4 type; +}; +template +struct Templates { + typedef Templates5 type; +}; +template +struct Templates { + typedef Templates6 type; +}; +template +struct Templates { + typedef Templates7 type; +}; +template +struct Templates { + typedef Templates8 type; +}; +template +struct Templates { + typedef Templates9 type; +}; +template +struct Templates { + typedef Templates10 type; +}; +template +struct Templates { + typedef Templates11 type; +}; +template +struct Templates { + typedef Templates12 type; +}; +template +struct Templates { + typedef Templates13 type; +}; +template +struct Templates { + typedef Templates14 type; +}; +template +struct Templates { + typedef Templates15 type; +}; +template +struct Templates { + typedef Templates16 type; +}; +template +struct Templates { + typedef Templates17 type; +}; +template +struct Templates { + typedef Templates18 type; +}; +template +struct Templates { + typedef Templates19 type; +}; +template +struct Templates { + typedef Templates20 type; +}; +template +struct Templates { + typedef Templates21 type; +}; +template +struct Templates { + typedef Templates22 type; +}; +template +struct Templates { + typedef Templates23 type; +}; +template +struct Templates { + typedef Templates24 type; +}; +template +struct Templates { + typedef Templates25 type; +}; +template +struct Templates { + typedef Templates26 type; +}; +template +struct Templates { + typedef Templates27 type; +}; +template +struct Templates { + typedef Templates28 type; +}; +template +struct Templates { + typedef Templates29 type; +}; +template +struct Templates { + typedef Templates30 type; +}; +template +struct Templates { + typedef Templates31 type; +}; +template +struct Templates { + typedef Templates32 type; +}; +template +struct Templates { + typedef Templates33 type; +}; +template +struct Templates { + typedef Templates34 type; +}; +template +struct Templates { + typedef Templates35 type; +}; +template +struct Templates { + typedef Templates36 type; +}; +template +struct Templates { + typedef Templates37 type; +}; +template +struct Templates { + typedef Templates38 type; +}; +template +struct Templates { + typedef Templates39 type; +}; +template +struct Templates { + typedef Templates40 type; +}; +template +struct Templates { + typedef Templates41 type; +}; +template +struct Templates { + typedef Templates42 type; +}; +template +struct Templates { + typedef Templates43 type; +}; +template +struct Templates { + typedef Templates44 type; +}; +template +struct Templates { + typedef Templates45 type; +}; +template +struct Templates { + typedef Templates46 type; +}; +template +struct Templates { + typedef Templates47 type; +}; +template +struct Templates { + typedef Templates48 type; +}; +template +struct Templates { + typedef Templates49 type; +}; + +// The TypeList template makes it possible to use either a single type +// or a Types<...> list in TYPED_TEST_CASE() and +// INSTANTIATE_TYPED_TEST_CASE_P(). + +template +struct TypeList { + typedef Types1 type; +}; + +template +struct TypeList > { + typedef typename Types::type type; +}; + +#endif // GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ + +// Due to C++ preprocessor weirdness, we need double indirection to +// concatenate two tokens when one of them is __LINE__. Writing +// +// foo ## __LINE__ +// +// will result in the token foo__LINE__, instead of foo followed by +// the current line number. For more details, see +// http://www.parashift.com/c++-faq-lite/misc-technical-issues.html#faq-39.6 +#define GTEST_CONCAT_TOKEN_(foo, bar) GTEST_CONCAT_TOKEN_IMPL_(foo, bar) +#define GTEST_CONCAT_TOKEN_IMPL_(foo, bar) foo ## bar + +// Stringifies its argument. +#define GTEST_STRINGIFY_(name) #name + +class ProtocolMessage; +namespace proto2 { class Message; } + +namespace testing { + +// Forward declarations. + +class AssertionResult; // Result of an assertion. +class Message; // Represents a failure message. +class Test; // Represents a test. +class TestInfo; // Information about a test. +class TestPartResult; // Result of a test part. +class UnitTest; // A collection of test cases. + +template +::std::string PrintToString(const T& value); + +namespace internal { + +struct TraceInfo; // Information about a trace point. +class TestInfoImpl; // Opaque implementation of TestInfo +class UnitTestImpl; // Opaque implementation of UnitTest + +// The text used in failure messages to indicate the start of the +// stack trace. +GTEST_API_ extern const char kStackTraceMarker[]; + +// Two overloaded helpers for checking at compile time whether an +// expression is a null pointer literal (i.e. NULL or any 0-valued +// compile-time integral constant). Their return values have +// different sizes, so we can use sizeof() to test which version is +// picked by the compiler. These helpers have no implementations, as +// we only need their signatures. +// +// Given IsNullLiteralHelper(x), the compiler will pick the first +// version if x can be implicitly converted to Secret*, and pick the +// second version otherwise. Since Secret is a secret and incomplete +// type, the only expression a user can write that has type Secret* is +// a null pointer literal. Therefore, we know that x is a null +// pointer literal if and only if the first version is picked by the +// compiler. +char IsNullLiteralHelper(Secret* p); +char (&IsNullLiteralHelper(...))[2]; // NOLINT + +// A compile-time bool constant that is true if and only if x is a +// null pointer literal (i.e. NULL or any 0-valued compile-time +// integral constant). +#ifdef GTEST_ELLIPSIS_NEEDS_POD_ +// We lose support for NULL detection where the compiler doesn't like +// passing non-POD classes through ellipsis (...). +# define GTEST_IS_NULL_LITERAL_(x) false +#else +# define GTEST_IS_NULL_LITERAL_(x) \ + (sizeof(::testing::internal::IsNullLiteralHelper(x)) == 1) +#endif // GTEST_ELLIPSIS_NEEDS_POD_ + +// Appends the user-supplied message to the Google-Test-generated message. +GTEST_API_ std::string AppendUserMessage( + const std::string& gtest_msg, const Message& user_msg); + +#if GTEST_HAS_EXCEPTIONS + +GTEST_DISABLE_MSC_WARNINGS_PUSH_(4275 \ +/* an exported class was derived from a class that was not exported */) + +// This exception is thrown by (and only by) a failed Google Test +// assertion when GTEST_FLAG(throw_on_failure) is true (if exceptions +// are enabled). We derive it from std::runtime_error, which is for +// errors presumably detectable only at run time. Since +// std::runtime_error inherits from std::exception, many testing +// frameworks know how to extract and print the message inside it. +class GTEST_API_ GoogleTestFailureException : public ::std::runtime_error { + public: + explicit GoogleTestFailureException(const TestPartResult& failure); +}; + +GTEST_DISABLE_MSC_WARNINGS_POP_() // 4275 + +#endif // GTEST_HAS_EXCEPTIONS + +namespace edit_distance { +// Returns the optimal edits to go from 'left' to 'right'. +// All edits cost the same, with replace having lower priority than +// add/remove. +// Simple implementation of the Wagner-Fischer algorithm. +// See http://en.wikipedia.org/wiki/Wagner-Fischer_algorithm +enum EditType { kMatch, kAdd, kRemove, kReplace }; +GTEST_API_ std::vector CalculateOptimalEdits( + const std::vector& left, const std::vector& right); + +// Same as above, but the input is represented as strings. +GTEST_API_ std::vector CalculateOptimalEdits( + const std::vector& left, + const std::vector& right); + +// Create a diff of the input strings in Unified diff format. +GTEST_API_ std::string CreateUnifiedDiff(const std::vector& left, + const std::vector& right, + size_t context = 2); + +} // namespace edit_distance + +// Calculate the diff between 'left' and 'right' and return it in unified diff +// format. +// If not null, stores in 'total_line_count' the total number of lines found +// in left + right. +GTEST_API_ std::string DiffStrings(const std::string& left, + const std::string& right, + size_t* total_line_count); + +// Constructs and returns the message for an equality assertion +// (e.g. ASSERT_EQ, EXPECT_STREQ, etc) failure. +// +// The first four parameters are the expressions used in the assertion +// and their values, as strings. For example, for ASSERT_EQ(foo, bar) +// where foo is 5 and bar is 6, we have: +// +// expected_expression: "foo" +// actual_expression: "bar" +// expected_value: "5" +// actual_value: "6" +// +// The ignoring_case parameter is true iff the assertion is a +// *_STRCASEEQ*. When it's true, the string " (ignoring case)" will +// be inserted into the message. +GTEST_API_ AssertionResult EqFailure(const char* expected_expression, + const char* actual_expression, + const std::string& expected_value, + const std::string& actual_value, + bool ignoring_case); + +// Constructs a failure message for Boolean assertions such as EXPECT_TRUE. +GTEST_API_ std::string GetBoolAssertionFailureMessage( + const AssertionResult& assertion_result, + const char* expression_text, + const char* actual_predicate_value, + const char* expected_predicate_value); + +// This template class represents an IEEE floating-point number +// (either single-precision or double-precision, depending on the +// template parameters). +// +// The purpose of this class is to do more sophisticated number +// comparison. (Due to round-off error, etc, it's very unlikely that +// two floating-points will be equal exactly. Hence a naive +// comparison by the == operation often doesn't work.) +// +// Format of IEEE floating-point: +// +// The most-significant bit being the leftmost, an IEEE +// floating-point looks like +// +// sign_bit exponent_bits fraction_bits +// +// Here, sign_bit is a single bit that designates the sign of the +// number. +// +// For float, there are 8 exponent bits and 23 fraction bits. +// +// For double, there are 11 exponent bits and 52 fraction bits. +// +// More details can be found at +// http://en.wikipedia.org/wiki/IEEE_floating-point_standard. +// +// Template parameter: +// +// RawType: the raw floating-point type (either float or double) +template +class FloatingPoint { + public: + // Defines the unsigned integer type that has the same size as the + // floating point number. + typedef typename TypeWithSize::UInt Bits; + + // Constants. + + // # of bits in a number. + static const size_t kBitCount = 8*sizeof(RawType); + + // # of fraction bits in a number. + static const size_t kFractionBitCount = + std::numeric_limits::digits - 1; + + // # of exponent bits in a number. + static const size_t kExponentBitCount = kBitCount - 1 - kFractionBitCount; + + // The mask for the sign bit. + static const Bits kSignBitMask = static_cast(1) << (kBitCount - 1); + + // The mask for the fraction bits. + static const Bits kFractionBitMask = + ~static_cast(0) >> (kExponentBitCount + 1); + + // The mask for the exponent bits. + static const Bits kExponentBitMask = ~(kSignBitMask | kFractionBitMask); + + // How many ULP's (Units in the Last Place) we want to tolerate when + // comparing two numbers. The larger the value, the more error we + // allow. A 0 value means that two numbers must be exactly the same + // to be considered equal. + // + // The maximum error of a single floating-point operation is 0.5 + // units in the last place. On Intel CPU's, all floating-point + // calculations are done with 80-bit precision, while double has 64 + // bits. Therefore, 4 should be enough for ordinary use. + // + // See the following article for more details on ULP: + // http://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/ + static const size_t kMaxUlps = 4; + + // Constructs a FloatingPoint from a raw floating-point number. + // + // On an Intel CPU, passing a non-normalized NAN (Not a Number) + // around may change its bits, although the new value is guaranteed + // to be also a NAN. Therefore, don't expect this constructor to + // preserve the bits in x when x is a NAN. + explicit FloatingPoint(const RawType& x) { u_.value_ = x; } + + // Static methods + + // Reinterprets a bit pattern as a floating-point number. + // + // This function is needed to test the AlmostEquals() method. + static RawType ReinterpretBits(const Bits bits) { + FloatingPoint fp(0); + fp.u_.bits_ = bits; + return fp.u_.value_; + } + + // Returns the floating-point number that represent positive infinity. + static RawType Infinity() { + return ReinterpretBits(kExponentBitMask); + } + + // Returns the maximum representable finite floating-point number. + static RawType Max(); + + // Non-static methods + + // Returns the bits that represents this number. + const Bits &bits() const { return u_.bits_; } + + // Returns the exponent bits of this number. + Bits exponent_bits() const { return kExponentBitMask & u_.bits_; } + + // Returns the fraction bits of this number. + Bits fraction_bits() const { return kFractionBitMask & u_.bits_; } + + // Returns the sign bit of this number. + Bits sign_bit() const { return kSignBitMask & u_.bits_; } + + // Returns true iff this is NAN (not a number). + bool is_nan() const { + // It's a NAN if the exponent bits are all ones and the fraction + // bits are not entirely zeros. + return (exponent_bits() == kExponentBitMask) && (fraction_bits() != 0); + } + + // Returns true iff this number is at most kMaxUlps ULP's away from + // rhs. In particular, this function: + // + // - returns false if either number is (or both are) NAN. + // - treats really large numbers as almost equal to infinity. + // - thinks +0.0 and -0.0 are 0 DLP's apart. + bool AlmostEquals(const FloatingPoint& rhs) const { + // The IEEE standard says that any comparison operation involving + // a NAN must return false. + if (is_nan() || rhs.is_nan()) return false; + + return DistanceBetweenSignAndMagnitudeNumbers(u_.bits_, rhs.u_.bits_) + <= kMaxUlps; + } + + private: + // The data type used to store the actual floating-point number. + union FloatingPointUnion { + RawType value_; // The raw floating-point number. + Bits bits_; // The bits that represent the number. + }; + + // Converts an integer from the sign-and-magnitude representation to + // the biased representation. More precisely, let N be 2 to the + // power of (kBitCount - 1), an integer x is represented by the + // unsigned number x + N. + // + // For instance, + // + // -N + 1 (the most negative number representable using + // sign-and-magnitude) is represented by 1; + // 0 is represented by N; and + // N - 1 (the biggest number representable using + // sign-and-magnitude) is represented by 2N - 1. + // + // Read http://en.wikipedia.org/wiki/Signed_number_representations + // for more details on signed number representations. + static Bits SignAndMagnitudeToBiased(const Bits &sam) { + if (kSignBitMask & sam) { + // sam represents a negative number. + return ~sam + 1; + } else { + // sam represents a positive number. + return kSignBitMask | sam; + } + } + + // Given two numbers in the sign-and-magnitude representation, + // returns the distance between them as an unsigned number. + static Bits DistanceBetweenSignAndMagnitudeNumbers(const Bits &sam1, + const Bits &sam2) { + const Bits biased1 = SignAndMagnitudeToBiased(sam1); + const Bits biased2 = SignAndMagnitudeToBiased(sam2); + return (biased1 >= biased2) ? (biased1 - biased2) : (biased2 - biased1); + } + + FloatingPointUnion u_; +}; + +// We cannot use std::numeric_limits::max() as it clashes with the max() +// macro defined by . +template <> +inline float FloatingPoint::Max() { return FLT_MAX; } +template <> +inline double FloatingPoint::Max() { return DBL_MAX; } + +// Typedefs the instances of the FloatingPoint template class that we +// care to use. +typedef FloatingPoint Float; +typedef FloatingPoint Double; + +// In order to catch the mistake of putting tests that use different +// test fixture classes in the same test case, we need to assign +// unique IDs to fixture classes and compare them. The TypeId type is +// used to hold such IDs. The user should treat TypeId as an opaque +// type: the only operation allowed on TypeId values is to compare +// them for equality using the == operator. +typedef const void* TypeId; + +template +class TypeIdHelper { + public: + // dummy_ must not have a const type. Otherwise an overly eager + // compiler (e.g. MSVC 7.1 & 8.0) may try to merge + // TypeIdHelper::dummy_ for different Ts as an "optimization". + static bool dummy_; +}; + +template +bool TypeIdHelper::dummy_ = false; + +// GetTypeId() returns the ID of type T. Different values will be +// returned for different types. Calling the function twice with the +// same type argument is guaranteed to return the same ID. +template +TypeId GetTypeId() { + // The compiler is required to allocate a different + // TypeIdHelper::dummy_ variable for each T used to instantiate + // the template. Therefore, the address of dummy_ is guaranteed to + // be unique. + return &(TypeIdHelper::dummy_); +} + +// Returns the type ID of ::testing::Test. Always call this instead +// of GetTypeId< ::testing::Test>() to get the type ID of +// ::testing::Test, as the latter may give the wrong result due to a +// suspected linker bug when compiling Google Test as a Mac OS X +// framework. +GTEST_API_ TypeId GetTestTypeId(); + +// Defines the abstract factory interface that creates instances +// of a Test object. +class TestFactoryBase { + public: + virtual ~TestFactoryBase() {} + + // Creates a test instance to run. The instance is both created and destroyed + // within TestInfoImpl::Run() + virtual Test* CreateTest() = 0; + + protected: + TestFactoryBase() {} + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestFactoryBase); +}; + +// This class provides implementation of TeastFactoryBase interface. +// It is used in TEST and TEST_F macros. +template +class TestFactoryImpl : public TestFactoryBase { + public: + virtual Test* CreateTest() { return new TestClass; } +}; + +#if GTEST_OS_WINDOWS + +// Predicate-formatters for implementing the HRESULT checking macros +// {ASSERT|EXPECT}_HRESULT_{SUCCEEDED|FAILED} +// We pass a long instead of HRESULT to avoid causing an +// include dependency for the HRESULT type. +GTEST_API_ AssertionResult IsHRESULTSuccess(const char* expr, + long hr); // NOLINT +GTEST_API_ AssertionResult IsHRESULTFailure(const char* expr, + long hr); // NOLINT + +#endif // GTEST_OS_WINDOWS + +// Types of SetUpTestCase() and TearDownTestCase() functions. +typedef void (*SetUpTestCaseFunc)(); +typedef void (*TearDownTestCaseFunc)(); + +struct CodeLocation { + CodeLocation(const std::string& a_file, int a_line) + : file(a_file), line(a_line) {} + + std::string file; + int line; +}; + +// Creates a new TestInfo object and registers it with Google Test; +// returns the created object. +// +// Arguments: +// +// test_case_name: name of the test case +// name: name of the test +// type_param the name of the test's type parameter, or NULL if +// this is not a typed or a type-parameterized test. +// value_param text representation of the test's value parameter, +// or NULL if this is not a type-parameterized test. +// code_location: code location where the test is defined +// fixture_class_id: ID of the test fixture class +// set_up_tc: pointer to the function that sets up the test case +// tear_down_tc: pointer to the function that tears down the test case +// factory: pointer to the factory that creates a test object. +// The newly created TestInfo instance will assume +// ownership of the factory object. +GTEST_API_ TestInfo* MakeAndRegisterTestInfo( + const char* test_case_name, + const char* name, + const char* type_param, + const char* value_param, + CodeLocation code_location, + TypeId fixture_class_id, + SetUpTestCaseFunc set_up_tc, + TearDownTestCaseFunc tear_down_tc, + TestFactoryBase* factory); + +// If *pstr starts with the given prefix, modifies *pstr to be right +// past the prefix and returns true; otherwise leaves *pstr unchanged +// and returns false. None of pstr, *pstr, and prefix can be NULL. +GTEST_API_ bool SkipPrefix(const char* prefix, const char** pstr); + +#if GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P + +GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ +/* class A needs to have dll-interface to be used by clients of class B */) + +// State of the definition of a type-parameterized test case. +class GTEST_API_ TypedTestCasePState { + public: + TypedTestCasePState() : registered_(false) {} + + // Adds the given test name to defined_test_names_ and return true + // if the test case hasn't been registered; otherwise aborts the + // program. + bool AddTestName(const char* file, int line, const char* case_name, + const char* test_name) { + if (registered_) { + fprintf(stderr, "%s Test %s must be defined before " + "REGISTER_TYPED_TEST_CASE_P(%s, ...).\n", + FormatFileLocation(file, line).c_str(), test_name, case_name); + fflush(stderr); + posix::Abort(); + } + registered_tests_.insert( + ::std::make_pair(test_name, CodeLocation(file, line))); + return true; + } + + bool TestExists(const std::string& test_name) const { + return registered_tests_.count(test_name) > 0; + } + + const CodeLocation& GetCodeLocation(const std::string& test_name) const { + RegisteredTestsMap::const_iterator it = registered_tests_.find(test_name); + GTEST_CHECK_(it != registered_tests_.end()); + return it->second; + } + + // Verifies that registered_tests match the test names in + // defined_test_names_; returns registered_tests if successful, or + // aborts the program otherwise. + const char* VerifyRegisteredTestNames( + const char* file, int line, const char* registered_tests); + + private: + typedef ::std::map RegisteredTestsMap; + + bool registered_; + RegisteredTestsMap registered_tests_; +}; + +GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 + +// Skips to the first non-space char after the first comma in 'str'; +// returns NULL if no comma is found in 'str'. +inline const char* SkipComma(const char* str) { + const char* comma = strchr(str, ','); + if (comma == NULL) { + return NULL; + } + while (IsSpace(*(++comma))) {} + return comma; +} + +// Returns the prefix of 'str' before the first comma in it; returns +// the entire string if it contains no comma. +inline std::string GetPrefixUntilComma(const char* str) { + const char* comma = strchr(str, ','); + return comma == NULL ? str : std::string(str, comma); +} + +// Splits a given string on a given delimiter, populating a given +// vector with the fields. +void SplitString(const ::std::string& str, char delimiter, + ::std::vector< ::std::string>* dest); + +// The default argument to the template below for the case when the user does +// not provide a name generator. +struct DefaultNameGenerator { + template + static std::string GetName(int i) { + return StreamableToString(i); + } +}; + +template +struct NameGeneratorSelector { + typedef Provided type; +}; + +template +void GenerateNamesRecursively(Types0, std::vector*, int) {} + +template +void GenerateNamesRecursively(Types, std::vector* result, int i) { + result->push_back(NameGenerator::template GetName(i)); + GenerateNamesRecursively(typename Types::Tail(), result, + i + 1); +} + +template +std::vector GenerateNames() { + std::vector result; + GenerateNamesRecursively(Types(), &result, 0); + return result; +} + +// TypeParameterizedTest::Register() +// registers a list of type-parameterized tests with Google Test. The +// return value is insignificant - we just need to return something +// such that we can call this function in a namespace scope. +// +// Implementation note: The GTEST_TEMPLATE_ macro declares a template +// template parameter. It's defined in gtest-type-util.h. +template +class TypeParameterizedTest { + public: + // 'index' is the index of the test in the type list 'Types' + // specified in INSTANTIATE_TYPED_TEST_CASE_P(Prefix, TestCase, + // Types). Valid values for 'index' are [0, N - 1] where N is the + // length of Types. + static bool Register(const char* prefix, const CodeLocation& code_location, + const char* case_name, const char* test_names, int index, + const std::vector& type_names = + GenerateNames()) { + typedef typename Types::Head Type; + typedef Fixture FixtureClass; + typedef typename GTEST_BIND_(TestSel, Type) TestClass; + + // First, registers the first type-parameterized test in the type + // list. + MakeAndRegisterTestInfo( + (std::string(prefix) + (prefix[0] == '\0' ? "" : "/") + case_name + + "/" + type_names[index]) + .c_str(), + StripTrailingSpaces(GetPrefixUntilComma(test_names)).c_str(), + GetTypeName().c_str(), + NULL, // No value parameter. + code_location, GetTypeId(), TestClass::SetUpTestCase, + TestClass::TearDownTestCase, new TestFactoryImpl); + + // Next, recurses (at compile time) with the tail of the type list. + return TypeParameterizedTest::Register(prefix, + code_location, + case_name, + test_names, + index + 1, + type_names); + } +}; + +// The base case for the compile time recursion. +template +class TypeParameterizedTest { + public: + static bool Register(const char* /*prefix*/, const CodeLocation&, + const char* /*case_name*/, const char* /*test_names*/, + int /*index*/, + const std::vector& = + std::vector() /*type_names*/) { + return true; + } +}; + +// TypeParameterizedTestCase::Register() +// registers *all combinations* of 'Tests' and 'Types' with Google +// Test. The return value is insignificant - we just need to return +// something such that we can call this function in a namespace scope. +template +class TypeParameterizedTestCase { + public: + static bool Register(const char* prefix, CodeLocation code_location, + const TypedTestCasePState* state, const char* case_name, + const char* test_names, + const std::vector& type_names = + GenerateNames()) { + std::string test_name = StripTrailingSpaces( + GetPrefixUntilComma(test_names)); + if (!state->TestExists(test_name)) { + fprintf(stderr, "Failed to get code location for test %s.%s at %s.", + case_name, test_name.c_str(), + FormatFileLocation(code_location.file.c_str(), + code_location.line).c_str()); + fflush(stderr); + posix::Abort(); + } + const CodeLocation& test_location = state->GetCodeLocation(test_name); + + typedef typename Tests::Head Head; + + // First, register the first test in 'Test' for each type in 'Types'. + TypeParameterizedTest::Register( + prefix, test_location, case_name, test_names, 0, type_names); + + // Next, recurses (at compile time) with the tail of the test list. + return TypeParameterizedTestCase::Register(prefix, code_location, + state, case_name, + SkipComma(test_names), + type_names); + } +}; + +// The base case for the compile time recursion. +template +class TypeParameterizedTestCase { + public: + static bool Register(const char* /*prefix*/, const CodeLocation&, + const TypedTestCasePState* /*state*/, + const char* /*case_name*/, const char* /*test_names*/, + const std::vector& = + std::vector() /*type_names*/) { + return true; + } +}; + +#endif // GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P + +// Returns the current OS stack trace as an std::string. +// +// The maximum number of stack frames to be included is specified by +// the gtest_stack_trace_depth flag. The skip_count parameter +// specifies the number of top frames to be skipped, which doesn't +// count against the number of frames to be included. +// +// For example, if Foo() calls Bar(), which in turn calls +// GetCurrentOsStackTraceExceptTop(..., 1), Foo() will be included in +// the trace but Bar() and GetCurrentOsStackTraceExceptTop() won't. +GTEST_API_ std::string GetCurrentOsStackTraceExceptTop( + UnitTest* unit_test, int skip_count); + +// Helpers for suppressing warnings on unreachable code or constant +// condition. + +// Always returns true. +GTEST_API_ bool AlwaysTrue(); + +// Always returns false. +inline bool AlwaysFalse() { return !AlwaysTrue(); } + +// Helper for suppressing false warning from Clang on a const char* +// variable declared in a conditional expression always being NULL in +// the else branch. +struct GTEST_API_ ConstCharPtr { + ConstCharPtr(const char* str) : value(str) {} + operator bool() const { return true; } + const char* value; +}; + +// A simple Linear Congruential Generator for generating random +// numbers with a uniform distribution. Unlike rand() and srand(), it +// doesn't use global state (and therefore can't interfere with user +// code). Unlike rand_r(), it's portable. An LCG isn't very random, +// but it's good enough for our purposes. +class GTEST_API_ Random { + public: + static const UInt32 kMaxRange = 1u << 31; + + explicit Random(UInt32 seed) : state_(seed) {} + + void Reseed(UInt32 seed) { state_ = seed; } + + // Generates a random number from [0, range). Crashes if 'range' is + // 0 or greater than kMaxRange. + UInt32 Generate(UInt32 range); + + private: + UInt32 state_; + GTEST_DISALLOW_COPY_AND_ASSIGN_(Random); +}; + +// Defining a variable of type CompileAssertTypesEqual will cause a +// compiler error iff T1 and T2 are different types. +template +struct CompileAssertTypesEqual; + +template +struct CompileAssertTypesEqual { +}; + +// Removes the reference from a type if it is a reference type, +// otherwise leaves it unchanged. This is the same as +// tr1::remove_reference, which is not widely available yet. +template +struct RemoveReference { typedef T type; }; // NOLINT +template +struct RemoveReference { typedef T type; }; // NOLINT + +// A handy wrapper around RemoveReference that works when the argument +// T depends on template parameters. +#define GTEST_REMOVE_REFERENCE_(T) \ + typename ::testing::internal::RemoveReference::type + +// Removes const from a type if it is a const type, otherwise leaves +// it unchanged. This is the same as tr1::remove_const, which is not +// widely available yet. +template +struct RemoveConst { typedef T type; }; // NOLINT +template +struct RemoveConst { typedef T type; }; // NOLINT + +// MSVC 8.0, Sun C++, and IBM XL C++ have a bug which causes the above +// definition to fail to remove the const in 'const int[3]' and 'const +// char[3][4]'. The following specialization works around the bug. +template +struct RemoveConst { + typedef typename RemoveConst::type type[N]; +}; + +#if defined(_MSC_VER) && _MSC_VER < 1400 +// This is the only specialization that allows VC++ 7.1 to remove const in +// 'const int[3] and 'const int[3][4]'. However, it causes trouble with GCC +// and thus needs to be conditionally compiled. +template +struct RemoveConst { + typedef typename RemoveConst::type type[N]; +}; +#endif + +// A handy wrapper around RemoveConst that works when the argument +// T depends on template parameters. +#define GTEST_REMOVE_CONST_(T) \ + typename ::testing::internal::RemoveConst::type + +// Turns const U&, U&, const U, and U all into U. +#define GTEST_REMOVE_REFERENCE_AND_CONST_(T) \ + GTEST_REMOVE_CONST_(GTEST_REMOVE_REFERENCE_(T)) + +// ImplicitlyConvertible::value is a compile-time bool +// constant that's true iff type From can be implicitly converted to +// type To. +template +class ImplicitlyConvertible { + private: + // We need the following helper functions only for their types. + // They have no implementations. + + // MakeFrom() is an expression whose type is From. We cannot simply + // use From(), as the type From may not have a public default + // constructor. + static typename AddReference::type MakeFrom(); + + // These two functions are overloaded. Given an expression + // Helper(x), the compiler will pick the first version if x can be + // implicitly converted to type To; otherwise it will pick the + // second version. + // + // The first version returns a value of size 1, and the second + // version returns a value of size 2. Therefore, by checking the + // size of Helper(x), which can be done at compile time, we can tell + // which version of Helper() is used, and hence whether x can be + // implicitly converted to type To. + static char Helper(To); + static char (&Helper(...))[2]; // NOLINT + + // We have to put the 'public' section after the 'private' section, + // or MSVC refuses to compile the code. + public: +#if defined(__BORLANDC__) + // C++Builder cannot use member overload resolution during template + // instantiation. The simplest workaround is to use its C++0x type traits + // functions (C++Builder 2009 and above only). + static const bool value = __is_convertible(From, To); +#else + // MSVC warns about implicitly converting from double to int for + // possible loss of data, so we need to temporarily disable the + // warning. + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4244) + static const bool value = + sizeof(Helper(ImplicitlyConvertible::MakeFrom())) == 1; + GTEST_DISABLE_MSC_WARNINGS_POP_() +#endif // __BORLANDC__ +}; +template +const bool ImplicitlyConvertible::value; + +// IsAProtocolMessage::value is a compile-time bool constant that's +// true iff T is type ProtocolMessage, proto2::Message, or a subclass +// of those. +template +struct IsAProtocolMessage + : public bool_constant< + ImplicitlyConvertible::value || + ImplicitlyConvertible::value> { +}; + +// When the compiler sees expression IsContainerTest(0), if C is an +// STL-style container class, the first overload of IsContainerTest +// will be viable (since both C::iterator* and C::const_iterator* are +// valid types and NULL can be implicitly converted to them). It will +// be picked over the second overload as 'int' is a perfect match for +// the type of argument 0. If C::iterator or C::const_iterator is not +// a valid type, the first overload is not viable, and the second +// overload will be picked. Therefore, we can determine whether C is +// a container class by checking the type of IsContainerTest(0). +// The value of the expression is insignificant. +// +// In C++11 mode we check the existence of a const_iterator and that an +// iterator is properly implemented for the container. +// +// For pre-C++11 that we look for both C::iterator and C::const_iterator. +// The reason is that C++ injects the name of a class as a member of the +// class itself (e.g. you can refer to class iterator as either +// 'iterator' or 'iterator::iterator'). If we look for C::iterator +// only, for example, we would mistakenly think that a class named +// iterator is an STL container. +// +// Also note that the simpler approach of overloading +// IsContainerTest(typename C::const_iterator*) and +// IsContainerTest(...) doesn't work with Visual Age C++ and Sun C++. +typedef int IsContainer; +#if GTEST_LANG_CXX11 +template ().begin()), + class = decltype(::std::declval().end()), + class = decltype(++::std::declval()), + class = decltype(*::std::declval()), + class = typename C::const_iterator> +IsContainer IsContainerTest(int /* dummy */) { + return 0; +} +#else +template +IsContainer IsContainerTest(int /* dummy */, + typename C::iterator* /* it */ = NULL, + typename C::const_iterator* /* const_it */ = NULL) { + return 0; +} +#endif // GTEST_LANG_CXX11 + +typedef char IsNotContainer; +template +IsNotContainer IsContainerTest(long /* dummy */) { return '\0'; } + +// Trait to detect whether a type T is a hash table. +// The heuristic used is that the type contains an inner type `hasher` and does +// not contain an inner type `reverse_iterator`. +// If the container is iterable in reverse, then order might actually matter. +template +struct IsHashTable { + private: + template + static char test(typename U::hasher*, typename U::reverse_iterator*); + template + static int test(typename U::hasher*, ...); + template + static char test(...); + + public: + static const bool value = sizeof(test(0, 0)) == sizeof(int); +}; + +template +const bool IsHashTable::value; + +template +struct VoidT { + typedef void value_type; +}; + +template +struct HasValueType : false_type {}; +template +struct HasValueType > : true_type { +}; + +template (0)) == sizeof(IsContainer), + bool = HasValueType::value> +struct IsRecursiveContainerImpl; + +template +struct IsRecursiveContainerImpl : public false_type {}; + +// Since the IsRecursiveContainerImpl depends on the IsContainerTest we need to +// obey the same inconsistencies as the IsContainerTest, namely check if +// something is a container is relying on only const_iterator in C++11 and +// is relying on both const_iterator and iterator otherwise +template +struct IsRecursiveContainerImpl : public false_type {}; + +template +struct IsRecursiveContainerImpl { + #if GTEST_LANG_CXX11 + typedef typename IteratorTraits::value_type + value_type; +#else + typedef typename IteratorTraits::value_type value_type; +#endif + typedef is_same type; +}; + +// IsRecursiveContainer is a unary compile-time predicate that +// evaluates whether C is a recursive container type. A recursive container +// type is a container type whose value_type is equal to the container type +// itself. An example for a recursive container type is +// boost::filesystem::path, whose iterator has a value_type that is equal to +// boost::filesystem::path. +template +struct IsRecursiveContainer : public IsRecursiveContainerImpl::type {}; + +// EnableIf::type is void when 'Cond' is true, and +// undefined when 'Cond' is false. To use SFINAE to make a function +// overload only apply when a particular expression is true, add +// "typename EnableIf::type* = 0" as the last parameter. +template struct EnableIf; +template<> struct EnableIf { typedef void type; }; // NOLINT + +// Utilities for native arrays. + +// ArrayEq() compares two k-dimensional native arrays using the +// elements' operator==, where k can be any integer >= 0. When k is +// 0, ArrayEq() degenerates into comparing a single pair of values. + +template +bool ArrayEq(const T* lhs, size_t size, const U* rhs); + +// This generic version is used when k is 0. +template +inline bool ArrayEq(const T& lhs, const U& rhs) { return lhs == rhs; } + +// This overload is used when k >= 1. +template +inline bool ArrayEq(const T(&lhs)[N], const U(&rhs)[N]) { + return internal::ArrayEq(lhs, N, rhs); +} + +// This helper reduces code bloat. If we instead put its logic inside +// the previous ArrayEq() function, arrays with different sizes would +// lead to different copies of the template code. +template +bool ArrayEq(const T* lhs, size_t size, const U* rhs) { + for (size_t i = 0; i != size; i++) { + if (!internal::ArrayEq(lhs[i], rhs[i])) + return false; + } + return true; +} + +// Finds the first element in the iterator range [begin, end) that +// equals elem. Element may be a native array type itself. +template +Iter ArrayAwareFind(Iter begin, Iter end, const Element& elem) { + for (Iter it = begin; it != end; ++it) { + if (internal::ArrayEq(*it, elem)) + return it; + } + return end; +} + +// CopyArray() copies a k-dimensional native array using the elements' +// operator=, where k can be any integer >= 0. When k is 0, +// CopyArray() degenerates into copying a single value. + +template +void CopyArray(const T* from, size_t size, U* to); + +// This generic version is used when k is 0. +template +inline void CopyArray(const T& from, U* to) { *to = from; } + +// This overload is used when k >= 1. +template +inline void CopyArray(const T(&from)[N], U(*to)[N]) { + internal::CopyArray(from, N, *to); +} + +// This helper reduces code bloat. If we instead put its logic inside +// the previous CopyArray() function, arrays with different sizes +// would lead to different copies of the template code. +template +void CopyArray(const T* from, size_t size, U* to) { + for (size_t i = 0; i != size; i++) { + internal::CopyArray(from[i], to + i); + } +} + +// The relation between an NativeArray object (see below) and the +// native array it represents. +// We use 2 different structs to allow non-copyable types to be used, as long +// as RelationToSourceReference() is passed. +struct RelationToSourceReference {}; +struct RelationToSourceCopy {}; + +// Adapts a native array to a read-only STL-style container. Instead +// of the complete STL container concept, this adaptor only implements +// members useful for Google Mock's container matchers. New members +// should be added as needed. To simplify the implementation, we only +// support Element being a raw type (i.e. having no top-level const or +// reference modifier). It's the client's responsibility to satisfy +// this requirement. Element can be an array type itself (hence +// multi-dimensional arrays are supported). +template +class NativeArray { + public: + // STL-style container typedefs. + typedef Element value_type; + typedef Element* iterator; + typedef const Element* const_iterator; + + // Constructs from a native array. References the source. + NativeArray(const Element* array, size_t count, RelationToSourceReference) { + InitRef(array, count); + } + + // Constructs from a native array. Copies the source. + NativeArray(const Element* array, size_t count, RelationToSourceCopy) { + InitCopy(array, count); + } + + // Copy constructor. + NativeArray(const NativeArray& rhs) { + (this->*rhs.clone_)(rhs.array_, rhs.size_); + } + + ~NativeArray() { + if (clone_ != &NativeArray::InitRef) + delete[] array_; + } + + // STL-style container methods. + size_t size() const { return size_; } + const_iterator begin() const { return array_; } + const_iterator end() const { return array_ + size_; } + bool operator==(const NativeArray& rhs) const { + return size() == rhs.size() && + ArrayEq(begin(), size(), rhs.begin()); + } + + private: + enum { + kCheckTypeIsNotConstOrAReference = StaticAssertTypeEqHelper< + Element, GTEST_REMOVE_REFERENCE_AND_CONST_(Element)>::value + }; + + // Initializes this object with a copy of the input. + void InitCopy(const Element* array, size_t a_size) { + Element* const copy = new Element[a_size]; + CopyArray(array, a_size, copy); + array_ = copy; + size_ = a_size; + clone_ = &NativeArray::InitCopy; + } + + // Initializes this object with a reference of the input. + void InitRef(const Element* array, size_t a_size) { + array_ = array; + size_ = a_size; + clone_ = &NativeArray::InitRef; + } + + const Element* array_; + size_t size_; + void (NativeArray::*clone_)(const Element*, size_t); + + GTEST_DISALLOW_ASSIGN_(NativeArray); +}; + +} // namespace internal +} // namespace testing + +#define GTEST_MESSAGE_AT_(file, line, message, result_type) \ + ::testing::internal::AssertHelper(result_type, file, line, message) \ + = ::testing::Message() + +#define GTEST_MESSAGE_(message, result_type) \ + GTEST_MESSAGE_AT_(__FILE__, __LINE__, message, result_type) + +#define GTEST_FATAL_FAILURE_(message) \ + return GTEST_MESSAGE_(message, ::testing::TestPartResult::kFatalFailure) + +#define GTEST_NONFATAL_FAILURE_(message) \ + GTEST_MESSAGE_(message, ::testing::TestPartResult::kNonFatalFailure) + +#define GTEST_SUCCESS_(message) \ + GTEST_MESSAGE_(message, ::testing::TestPartResult::kSuccess) + +// Suppress MSVC warning 4702 (unreachable code) for the code following +// statement if it returns or throws (or doesn't return or throw in some +// situations). +#define GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement) \ + if (::testing::internal::AlwaysTrue()) { statement; } + +#define GTEST_TEST_THROW_(statement, expected_exception, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::ConstCharPtr gtest_msg = "") { \ + bool gtest_caught_expected = false; \ + try { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } \ + catch (expected_exception const&) { \ + gtest_caught_expected = true; \ + } \ + catch (...) { \ + gtest_msg.value = \ + "Expected: " #statement " throws an exception of type " \ + #expected_exception ".\n Actual: it throws a different type."; \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__); \ + } \ + if (!gtest_caught_expected) { \ + gtest_msg.value = \ + "Expected: " #statement " throws an exception of type " \ + #expected_exception ".\n Actual: it throws nothing."; \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__); \ + } \ + } else \ + GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__): \ + fail(gtest_msg.value) + +#define GTEST_TEST_NO_THROW_(statement, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + try { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } \ + catch (...) { \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testnothrow_, __LINE__); \ + } \ + } else \ + GTEST_CONCAT_TOKEN_(gtest_label_testnothrow_, __LINE__): \ + fail("Expected: " #statement " doesn't throw an exception.\n" \ + " Actual: it throws.") + +#define GTEST_TEST_ANY_THROW_(statement, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + bool gtest_caught_any = false; \ + try { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } \ + catch (...) { \ + gtest_caught_any = true; \ + } \ + if (!gtest_caught_any) { \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testanythrow_, __LINE__); \ + } \ + } else \ + GTEST_CONCAT_TOKEN_(gtest_label_testanythrow_, __LINE__): \ + fail("Expected: " #statement " throws an exception.\n" \ + " Actual: it doesn't.") + + +// Implements Boolean test assertions such as EXPECT_TRUE. expression can be +// either a boolean expression or an AssertionResult. text is a textual +// represenation of expression as it was passed into the EXPECT_TRUE. +#define GTEST_TEST_BOOLEAN_(expression, text, actual, expected, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (const ::testing::AssertionResult gtest_ar_ = \ + ::testing::AssertionResult(expression)) \ + ; \ + else \ + fail(::testing::internal::GetBoolAssertionFailureMessage(\ + gtest_ar_, text, #actual, #expected).c_str()) + +#define GTEST_TEST_NO_FATAL_FAILURE_(statement, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + ::testing::internal::HasNewFatalFailureHelper gtest_fatal_failure_checker; \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + if (gtest_fatal_failure_checker.has_new_fatal_failure()) { \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testnofatal_, __LINE__); \ + } \ + } else \ + GTEST_CONCAT_TOKEN_(gtest_label_testnofatal_, __LINE__): \ + fail("Expected: " #statement " doesn't generate new fatal " \ + "failures in the current thread.\n" \ + " Actual: it does.") + +// Expands to the name of the class that implements the given test. +#define GTEST_TEST_CLASS_NAME_(test_case_name, test_name) \ + test_case_name##_##test_name##_Test + +// Helper macro for defining tests. +#define GTEST_TEST_(test_case_name, test_name, parent_class, parent_id)\ +class GTEST_TEST_CLASS_NAME_(test_case_name, test_name) : public parent_class {\ + public:\ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)() {}\ + private:\ + virtual void TestBody();\ + static ::testing::TestInfo* const test_info_ GTEST_ATTRIBUTE_UNUSED_;\ + GTEST_DISALLOW_COPY_AND_ASSIGN_(\ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name));\ +};\ +\ +::testing::TestInfo* const GTEST_TEST_CLASS_NAME_(test_case_name, test_name)\ + ::test_info_ =\ + ::testing::internal::MakeAndRegisterTestInfo(\ + #test_case_name, #test_name, NULL, NULL, \ + ::testing::internal::CodeLocation(__FILE__, __LINE__), \ + (parent_id), \ + parent_class::SetUpTestCase, \ + parent_class::TearDownTestCase, \ + new ::testing::internal::TestFactoryImpl<\ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)>);\ +void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::TestBody() + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// +// The Google C++ Testing and Mocking Framework (Google Test) +// +// This header file defines the public API for death tests. It is +// #included by gtest.h so a user doesn't need to include this +// directly. +// GOOGLETEST_CM0001 DO NOT DELETE + +#ifndef GTEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ +#define GTEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ + +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// The Google C++ Testing and Mocking Framework (Google Test) +// +// This header file defines internal utilities needed for implementing +// death tests. They are subject to change without notice. +// GOOGLETEST_CM0001 DO NOT DELETE + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ + + +#include + +namespace testing { +namespace internal { + +GTEST_DECLARE_string_(internal_run_death_test); + +// Names of the flags (needed for parsing Google Test flags). +const char kDeathTestStyleFlag[] = "death_test_style"; +const char kDeathTestUseFork[] = "death_test_use_fork"; +const char kInternalRunDeathTestFlag[] = "internal_run_death_test"; + +#if GTEST_HAS_DEATH_TEST + +GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ +/* class A needs to have dll-interface to be used by clients of class B */) + +// DeathTest is a class that hides much of the complexity of the +// GTEST_DEATH_TEST_ macro. It is abstract; its static Create method +// returns a concrete class that depends on the prevailing death test +// style, as defined by the --gtest_death_test_style and/or +// --gtest_internal_run_death_test flags. + +// In describing the results of death tests, these terms are used with +// the corresponding definitions: +// +// exit status: The integer exit information in the format specified +// by wait(2) +// exit code: The integer code passed to exit(3), _exit(2), or +// returned from main() +class GTEST_API_ DeathTest { + public: + // Create returns false if there was an error determining the + // appropriate action to take for the current death test; for example, + // if the gtest_death_test_style flag is set to an invalid value. + // The LastMessage method will return a more detailed message in that + // case. Otherwise, the DeathTest pointer pointed to by the "test" + // argument is set. If the death test should be skipped, the pointer + // is set to NULL; otherwise, it is set to the address of a new concrete + // DeathTest object that controls the execution of the current test. + static bool Create(const char* statement, const RE* regex, + const char* file, int line, DeathTest** test); + DeathTest(); + virtual ~DeathTest() { } + + // A helper class that aborts a death test when it's deleted. + class ReturnSentinel { + public: + explicit ReturnSentinel(DeathTest* test) : test_(test) { } + ~ReturnSentinel() { test_->Abort(TEST_ENCOUNTERED_RETURN_STATEMENT); } + private: + DeathTest* const test_; + GTEST_DISALLOW_COPY_AND_ASSIGN_(ReturnSentinel); + } GTEST_ATTRIBUTE_UNUSED_; + + // An enumeration of possible roles that may be taken when a death + // test is encountered. EXECUTE means that the death test logic should + // be executed immediately. OVERSEE means that the program should prepare + // the appropriate environment for a child process to execute the death + // test, then wait for it to complete. + enum TestRole { OVERSEE_TEST, EXECUTE_TEST }; + + // An enumeration of the three reasons that a test might be aborted. + enum AbortReason { + TEST_ENCOUNTERED_RETURN_STATEMENT, + TEST_THREW_EXCEPTION, + TEST_DID_NOT_DIE + }; + + // Assumes one of the above roles. + virtual TestRole AssumeRole() = 0; + + // Waits for the death test to finish and returns its status. + virtual int Wait() = 0; + + // Returns true if the death test passed; that is, the test process + // exited during the test, its exit status matches a user-supplied + // predicate, and its stderr output matches a user-supplied regular + // expression. + // The user-supplied predicate may be a macro expression rather + // than a function pointer or functor, or else Wait and Passed could + // be combined. + virtual bool Passed(bool exit_status_ok) = 0; + + // Signals that the death test did not die as expected. + virtual void Abort(AbortReason reason) = 0; + + // Returns a human-readable outcome message regarding the outcome of + // the last death test. + static const char* LastMessage(); + + static void set_last_death_test_message(const std::string& message); + + private: + // A string containing a description of the outcome of the last death test. + static std::string last_death_test_message_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(DeathTest); +}; + +GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 + +// Factory interface for death tests. May be mocked out for testing. +class DeathTestFactory { + public: + virtual ~DeathTestFactory() { } + virtual bool Create(const char* statement, const RE* regex, + const char* file, int line, DeathTest** test) = 0; +}; + +// A concrete DeathTestFactory implementation for normal use. +class DefaultDeathTestFactory : public DeathTestFactory { + public: + virtual bool Create(const char* statement, const RE* regex, + const char* file, int line, DeathTest** test); +}; + +// Returns true if exit_status describes a process that was terminated +// by a signal, or exited normally with a nonzero exit code. +GTEST_API_ bool ExitedUnsuccessfully(int exit_status); + +// Traps C++ exceptions escaping statement and reports them as test +// failures. Note that trapping SEH exceptions is not implemented here. +# if GTEST_HAS_EXCEPTIONS +# define GTEST_EXECUTE_DEATH_TEST_STATEMENT_(statement, death_test) \ + try { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } catch (const ::std::exception& gtest_exception) { \ + fprintf(\ + stderr, \ + "\n%s: Caught std::exception-derived exception escaping the " \ + "death test statement. Exception message: %s\n", \ + ::testing::internal::FormatFileLocation(__FILE__, __LINE__).c_str(), \ + gtest_exception.what()); \ + fflush(stderr); \ + death_test->Abort(::testing::internal::DeathTest::TEST_THREW_EXCEPTION); \ + } catch (...) { \ + death_test->Abort(::testing::internal::DeathTest::TEST_THREW_EXCEPTION); \ + } + +# else +# define GTEST_EXECUTE_DEATH_TEST_STATEMENT_(statement, death_test) \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement) + +# endif + +// This macro is for implementing ASSERT_DEATH*, EXPECT_DEATH*, +// ASSERT_EXIT*, and EXPECT_EXIT*. +# define GTEST_DEATH_TEST_(statement, predicate, regex, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + const ::testing::internal::RE& gtest_regex = (regex); \ + ::testing::internal::DeathTest* gtest_dt; \ + if (!::testing::internal::DeathTest::Create(#statement, >est_regex, \ + __FILE__, __LINE__, >est_dt)) { \ + goto GTEST_CONCAT_TOKEN_(gtest_label_, __LINE__); \ + } \ + if (gtest_dt != NULL) { \ + ::testing::internal::scoped_ptr< ::testing::internal::DeathTest> \ + gtest_dt_ptr(gtest_dt); \ + switch (gtest_dt->AssumeRole()) { \ + case ::testing::internal::DeathTest::OVERSEE_TEST: \ + if (!gtest_dt->Passed(predicate(gtest_dt->Wait()))) { \ + goto GTEST_CONCAT_TOKEN_(gtest_label_, __LINE__); \ + } \ + break; \ + case ::testing::internal::DeathTest::EXECUTE_TEST: { \ + ::testing::internal::DeathTest::ReturnSentinel \ + gtest_sentinel(gtest_dt); \ + GTEST_EXECUTE_DEATH_TEST_STATEMENT_(statement, gtest_dt); \ + gtest_dt->Abort(::testing::internal::DeathTest::TEST_DID_NOT_DIE); \ + break; \ + } \ + default: \ + break; \ + } \ + } \ + } else \ + GTEST_CONCAT_TOKEN_(gtest_label_, __LINE__): \ + fail(::testing::internal::DeathTest::LastMessage()) +// The symbol "fail" here expands to something into which a message +// can be streamed. + +// This macro is for implementing ASSERT/EXPECT_DEBUG_DEATH when compiled in +// NDEBUG mode. In this case we need the statements to be executed and the macro +// must accept a streamed message even though the message is never printed. +// The regex object is not evaluated, but it is used to prevent "unused" +// warnings and to avoid an expression that doesn't compile in debug mode. +#define GTEST_EXECUTE_STATEMENT_(statement, regex) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } else if (!::testing::internal::AlwaysTrue()) { \ + const ::testing::internal::RE& gtest_regex = (regex); \ + static_cast(gtest_regex); \ + } else \ + ::testing::Message() + +// A class representing the parsed contents of the +// --gtest_internal_run_death_test flag, as it existed when +// RUN_ALL_TESTS was called. +class InternalRunDeathTestFlag { + public: + InternalRunDeathTestFlag(const std::string& a_file, + int a_line, + int an_index, + int a_write_fd) + : file_(a_file), line_(a_line), index_(an_index), + write_fd_(a_write_fd) {} + + ~InternalRunDeathTestFlag() { + if (write_fd_ >= 0) + posix::Close(write_fd_); + } + + const std::string& file() const { return file_; } + int line() const { return line_; } + int index() const { return index_; } + int write_fd() const { return write_fd_; } + + private: + std::string file_; + int line_; + int index_; + int write_fd_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(InternalRunDeathTestFlag); +}; + +// Returns a newly created InternalRunDeathTestFlag object with fields +// initialized from the GTEST_FLAG(internal_run_death_test) flag if +// the flag is specified; otherwise returns NULL. +InternalRunDeathTestFlag* ParseInternalRunDeathTestFlag(); + +#endif // GTEST_HAS_DEATH_TEST + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ + +namespace testing { + +// This flag controls the style of death tests. Valid values are "threadsafe", +// meaning that the death test child process will re-execute the test binary +// from the start, running only a single death test, or "fast", +// meaning that the child process will execute the test logic immediately +// after forking. +GTEST_DECLARE_string_(death_test_style); + +#if GTEST_HAS_DEATH_TEST + +namespace internal { + +// Returns a Boolean value indicating whether the caller is currently +// executing in the context of the death test child process. Tools such as +// Valgrind heap checkers may need this to modify their behavior in death +// tests. IMPORTANT: This is an internal utility. Using it may break the +// implementation of death tests. User code MUST NOT use it. +GTEST_API_ bool InDeathTestChild(); + +} // namespace internal + +// The following macros are useful for writing death tests. + +// Here's what happens when an ASSERT_DEATH* or EXPECT_DEATH* is +// executed: +// +// 1. It generates a warning if there is more than one active +// thread. This is because it's safe to fork() or clone() only +// when there is a single thread. +// +// 2. The parent process clone()s a sub-process and runs the death +// test in it; the sub-process exits with code 0 at the end of the +// death test, if it hasn't exited already. +// +// 3. The parent process waits for the sub-process to terminate. +// +// 4. The parent process checks the exit code and error message of +// the sub-process. +// +// Examples: +// +// ASSERT_DEATH(server.SendMessage(56, "Hello"), "Invalid port number"); +// for (int i = 0; i < 5; i++) { +// EXPECT_DEATH(server.ProcessRequest(i), +// "Invalid request .* in ProcessRequest()") +// << "Failed to die on request " << i; +// } +// +// ASSERT_EXIT(server.ExitNow(), ::testing::ExitedWithCode(0), "Exiting"); +// +// bool KilledBySIGHUP(int exit_code) { +// return WIFSIGNALED(exit_code) && WTERMSIG(exit_code) == SIGHUP; +// } +// +// ASSERT_EXIT(client.HangUpServer(), KilledBySIGHUP, "Hanging up!"); +// +// On the regular expressions used in death tests: +// +// GOOGLETEST_CM0005 DO NOT DELETE +// On POSIX-compliant systems (*nix), we use the library, +// which uses the POSIX extended regex syntax. +// +// On other platforms (e.g. Windows or Mac), we only support a simple regex +// syntax implemented as part of Google Test. This limited +// implementation should be enough most of the time when writing +// death tests; though it lacks many features you can find in PCRE +// or POSIX extended regex syntax. For example, we don't support +// union ("x|y"), grouping ("(xy)"), brackets ("[xy]"), and +// repetition count ("x{5,7}"), among others. +// +// Below is the syntax that we do support. We chose it to be a +// subset of both PCRE and POSIX extended regex, so it's easy to +// learn wherever you come from. In the following: 'A' denotes a +// literal character, period (.), or a single \\ escape sequence; +// 'x' and 'y' denote regular expressions; 'm' and 'n' are for +// natural numbers. +// +// c matches any literal character c +// \\d matches any decimal digit +// \\D matches any character that's not a decimal digit +// \\f matches \f +// \\n matches \n +// \\r matches \r +// \\s matches any ASCII whitespace, including \n +// \\S matches any character that's not a whitespace +// \\t matches \t +// \\v matches \v +// \\w matches any letter, _, or decimal digit +// \\W matches any character that \\w doesn't match +// \\c matches any literal character c, which must be a punctuation +// . matches any single character except \n +// A? matches 0 or 1 occurrences of A +// A* matches 0 or many occurrences of A +// A+ matches 1 or many occurrences of A +// ^ matches the beginning of a string (not that of each line) +// $ matches the end of a string (not that of each line) +// xy matches x followed by y +// +// If you accidentally use PCRE or POSIX extended regex features +// not implemented by us, you will get a run-time failure. In that +// case, please try to rewrite your regular expression within the +// above syntax. +// +// This implementation is *not* meant to be as highly tuned or robust +// as a compiled regex library, but should perform well enough for a +// death test, which already incurs significant overhead by launching +// a child process. +// +// Known caveats: +// +// A "threadsafe" style death test obtains the path to the test +// program from argv[0] and re-executes it in the sub-process. For +// simplicity, the current implementation doesn't search the PATH +// when launching the sub-process. This means that the user must +// invoke the test program via a path that contains at least one +// path separator (e.g. path/to/foo_test and +// /absolute/path/to/bar_test are fine, but foo_test is not). This +// is rarely a problem as people usually don't put the test binary +// directory in PATH. +// +// FIXME: make thread-safe death tests search the PATH. + +// Asserts that a given statement causes the program to exit, with an +// integer exit status that satisfies predicate, and emitting error output +// that matches regex. +# define ASSERT_EXIT(statement, predicate, regex) \ + GTEST_DEATH_TEST_(statement, predicate, regex, GTEST_FATAL_FAILURE_) + +// Like ASSERT_EXIT, but continues on to successive tests in the +// test case, if any: +# define EXPECT_EXIT(statement, predicate, regex) \ + GTEST_DEATH_TEST_(statement, predicate, regex, GTEST_NONFATAL_FAILURE_) + +// Asserts that a given statement causes the program to exit, either by +// explicitly exiting with a nonzero exit code or being killed by a +// signal, and emitting error output that matches regex. +# define ASSERT_DEATH(statement, regex) \ + ASSERT_EXIT(statement, ::testing::internal::ExitedUnsuccessfully, regex) + +// Like ASSERT_DEATH, but continues on to successive tests in the +// test case, if any: +# define EXPECT_DEATH(statement, regex) \ + EXPECT_EXIT(statement, ::testing::internal::ExitedUnsuccessfully, regex) + +// Two predicate classes that can be used in {ASSERT,EXPECT}_EXIT*: + +// Tests that an exit code describes a normal exit with a given exit code. +class GTEST_API_ ExitedWithCode { + public: + explicit ExitedWithCode(int exit_code); + bool operator()(int exit_status) const; + private: + // No implementation - assignment is unsupported. + void operator=(const ExitedWithCode& other); + + const int exit_code_; +}; + +# if !GTEST_OS_WINDOWS && !GTEST_OS_FUCHSIA +// Tests that an exit code describes an exit due to termination by a +// given signal. +// GOOGLETEST_CM0006 DO NOT DELETE +class GTEST_API_ KilledBySignal { + public: + explicit KilledBySignal(int signum); + bool operator()(int exit_status) const; + private: + const int signum_; +}; +# endif // !GTEST_OS_WINDOWS + +// EXPECT_DEBUG_DEATH asserts that the given statements die in debug mode. +// The death testing framework causes this to have interesting semantics, +// since the sideeffects of the call are only visible in opt mode, and not +// in debug mode. +// +// In practice, this can be used to test functions that utilize the +// LOG(DFATAL) macro using the following style: +// +// int DieInDebugOr12(int* sideeffect) { +// if (sideeffect) { +// *sideeffect = 12; +// } +// LOG(DFATAL) << "death"; +// return 12; +// } +// +// TEST(TestCase, TestDieOr12WorksInDgbAndOpt) { +// int sideeffect = 0; +// // Only asserts in dbg. +// EXPECT_DEBUG_DEATH(DieInDebugOr12(&sideeffect), "death"); +// +// #ifdef NDEBUG +// // opt-mode has sideeffect visible. +// EXPECT_EQ(12, sideeffect); +// #else +// // dbg-mode no visible sideeffect. +// EXPECT_EQ(0, sideeffect); +// #endif +// } +// +// This will assert that DieInDebugReturn12InOpt() crashes in debug +// mode, usually due to a DCHECK or LOG(DFATAL), but returns the +// appropriate fallback value (12 in this case) in opt mode. If you +// need to test that a function has appropriate side-effects in opt +// mode, include assertions against the side-effects. A general +// pattern for this is: +// +// EXPECT_DEBUG_DEATH({ +// // Side-effects here will have an effect after this statement in +// // opt mode, but none in debug mode. +// EXPECT_EQ(12, DieInDebugOr12(&sideeffect)); +// }, "death"); +// +# ifdef NDEBUG + +# define EXPECT_DEBUG_DEATH(statement, regex) \ + GTEST_EXECUTE_STATEMENT_(statement, regex) + +# define ASSERT_DEBUG_DEATH(statement, regex) \ + GTEST_EXECUTE_STATEMENT_(statement, regex) + +# else + +# define EXPECT_DEBUG_DEATH(statement, regex) \ + EXPECT_DEATH(statement, regex) + +# define ASSERT_DEBUG_DEATH(statement, regex) \ + ASSERT_DEATH(statement, regex) + +# endif // NDEBUG for EXPECT_DEBUG_DEATH +#endif // GTEST_HAS_DEATH_TEST + +// This macro is used for implementing macros such as +// EXPECT_DEATH_IF_SUPPORTED and ASSERT_DEATH_IF_SUPPORTED on systems where +// death tests are not supported. Those macros must compile on such systems +// iff EXPECT_DEATH and ASSERT_DEATH compile with the same parameters on +// systems that support death tests. This allows one to write such a macro +// on a system that does not support death tests and be sure that it will +// compile on a death-test supporting system. It is exposed publicly so that +// systems that have death-tests with stricter requirements than +// GTEST_HAS_DEATH_TEST can write their own equivalent of +// EXPECT_DEATH_IF_SUPPORTED and ASSERT_DEATH_IF_SUPPORTED. +// +// Parameters: +// statement - A statement that a macro such as EXPECT_DEATH would test +// for program termination. This macro has to make sure this +// statement is compiled but not executed, to ensure that +// EXPECT_DEATH_IF_SUPPORTED compiles with a certain +// parameter iff EXPECT_DEATH compiles with it. +// regex - A regex that a macro such as EXPECT_DEATH would use to test +// the output of statement. This parameter has to be +// compiled but not evaluated by this macro, to ensure that +// this macro only accepts expressions that a macro such as +// EXPECT_DEATH would accept. +// terminator - Must be an empty statement for EXPECT_DEATH_IF_SUPPORTED +// and a return statement for ASSERT_DEATH_IF_SUPPORTED. +// This ensures that ASSERT_DEATH_IF_SUPPORTED will not +// compile inside functions where ASSERT_DEATH doesn't +// compile. +// +// The branch that has an always false condition is used to ensure that +// statement and regex are compiled (and thus syntactically correct) but +// never executed. The unreachable code macro protects the terminator +// statement from generating an 'unreachable code' warning in case +// statement unconditionally returns or throws. The Message constructor at +// the end allows the syntax of streaming additional messages into the +// macro, for compilational compatibility with EXPECT_DEATH/ASSERT_DEATH. +# define GTEST_UNSUPPORTED_DEATH_TEST(statement, regex, terminator) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + GTEST_LOG_(WARNING) \ + << "Death tests are not supported on this platform.\n" \ + << "Statement '" #statement "' cannot be verified."; \ + } else if (::testing::internal::AlwaysFalse()) { \ + ::testing::internal::RE::PartialMatch(".*", (regex)); \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + terminator; \ + } else \ + ::testing::Message() + +// EXPECT_DEATH_IF_SUPPORTED(statement, regex) and +// ASSERT_DEATH_IF_SUPPORTED(statement, regex) expand to real death tests if +// death tests are supported; otherwise they just issue a warning. This is +// useful when you are combining death test assertions with normal test +// assertions in one test. +#if GTEST_HAS_DEATH_TEST +# define EXPECT_DEATH_IF_SUPPORTED(statement, regex) \ + EXPECT_DEATH(statement, regex) +# define ASSERT_DEATH_IF_SUPPORTED(statement, regex) \ + ASSERT_DEATH(statement, regex) +#else +# define EXPECT_DEATH_IF_SUPPORTED(statement, regex) \ + GTEST_UNSUPPORTED_DEATH_TEST(statement, regex, ) +# define ASSERT_DEATH_IF_SUPPORTED(statement, regex) \ + GTEST_UNSUPPORTED_DEATH_TEST(statement, regex, return) +#endif + +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ +// This file was GENERATED by command: +// pump.py gtest-param-test.h.pump +// DO NOT EDIT BY HAND!!! + +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Macros and functions for implementing parameterized tests +// in Google C++ Testing and Mocking Framework (Google Test) +// +// This file is generated by a SCRIPT. DO NOT EDIT BY HAND! +// +// GOOGLETEST_CM0001 DO NOT DELETE +#ifndef GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ +#define GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ + + +// Value-parameterized tests allow you to test your code with different +// parameters without writing multiple copies of the same test. +// +// Here is how you use value-parameterized tests: + +#if 0 + +// To write value-parameterized tests, first you should define a fixture +// class. It is usually derived from testing::TestWithParam (see below for +// another inheritance scheme that's sometimes useful in more complicated +// class hierarchies), where the type of your parameter values. +// TestWithParam is itself derived from testing::Test. T can be any +// copyable type. If it's a raw pointer, you are responsible for managing the +// lifespan of the pointed values. + +class FooTest : public ::testing::TestWithParam { + // You can implement all the usual class fixture members here. +}; + +// Then, use the TEST_P macro to define as many parameterized tests +// for this fixture as you want. The _P suffix is for "parameterized" +// or "pattern", whichever you prefer to think. + +TEST_P(FooTest, DoesBlah) { + // Inside a test, access the test parameter with the GetParam() method + // of the TestWithParam class: + EXPECT_TRUE(foo.Blah(GetParam())); + ... +} + +TEST_P(FooTest, HasBlahBlah) { + ... +} + +// Finally, you can use INSTANTIATE_TEST_CASE_P to instantiate the test +// case with any set of parameters you want. Google Test defines a number +// of functions for generating test parameters. They return what we call +// (surprise!) parameter generators. Here is a summary of them, which +// are all in the testing namespace: +// +// +// Range(begin, end [, step]) - Yields values {begin, begin+step, +// begin+step+step, ...}. The values do not +// include end. step defaults to 1. +// Values(v1, v2, ..., vN) - Yields values {v1, v2, ..., vN}. +// ValuesIn(container) - Yields values from a C-style array, an STL +// ValuesIn(begin,end) container, or an iterator range [begin, end). +// Bool() - Yields sequence {false, true}. +// Combine(g1, g2, ..., gN) - Yields all combinations (the Cartesian product +// for the math savvy) of the values generated +// by the N generators. +// +// For more details, see comments at the definitions of these functions below +// in this file. +// +// The following statement will instantiate tests from the FooTest test case +// each with parameter values "meeny", "miny", and "moe". + +INSTANTIATE_TEST_CASE_P(InstantiationName, + FooTest, + Values("meeny", "miny", "moe")); + +// To distinguish different instances of the pattern, (yes, you +// can instantiate it more then once) the first argument to the +// INSTANTIATE_TEST_CASE_P macro is a prefix that will be added to the +// actual test case name. Remember to pick unique prefixes for different +// instantiations. The tests from the instantiation above will have +// these names: +// +// * InstantiationName/FooTest.DoesBlah/0 for "meeny" +// * InstantiationName/FooTest.DoesBlah/1 for "miny" +// * InstantiationName/FooTest.DoesBlah/2 for "moe" +// * InstantiationName/FooTest.HasBlahBlah/0 for "meeny" +// * InstantiationName/FooTest.HasBlahBlah/1 for "miny" +// * InstantiationName/FooTest.HasBlahBlah/2 for "moe" +// +// You can use these names in --gtest_filter. +// +// This statement will instantiate all tests from FooTest again, each +// with parameter values "cat" and "dog": + +const char* pets[] = {"cat", "dog"}; +INSTANTIATE_TEST_CASE_P(AnotherInstantiationName, FooTest, ValuesIn(pets)); + +// The tests from the instantiation above will have these names: +// +// * AnotherInstantiationName/FooTest.DoesBlah/0 for "cat" +// * AnotherInstantiationName/FooTest.DoesBlah/1 for "dog" +// * AnotherInstantiationName/FooTest.HasBlahBlah/0 for "cat" +// * AnotherInstantiationName/FooTest.HasBlahBlah/1 for "dog" +// +// Please note that INSTANTIATE_TEST_CASE_P will instantiate all tests +// in the given test case, whether their definitions come before or +// AFTER the INSTANTIATE_TEST_CASE_P statement. +// +// Please also note that generator expressions (including parameters to the +// generators) are evaluated in InitGoogleTest(), after main() has started. +// This allows the user on one hand, to adjust generator parameters in order +// to dynamically determine a set of tests to run and on the other hand, +// give the user a chance to inspect the generated tests with Google Test +// reflection API before RUN_ALL_TESTS() is executed. +// +// You can see samples/sample7_unittest.cc and samples/sample8_unittest.cc +// for more examples. +// +// In the future, we plan to publish the API for defining new parameter +// generators. But for now this interface remains part of the internal +// implementation and is subject to change. +// +// +// A parameterized test fixture must be derived from testing::Test and from +// testing::WithParamInterface, where T is the type of the parameter +// values. Inheriting from TestWithParam satisfies that requirement because +// TestWithParam inherits from both Test and WithParamInterface. In more +// complicated hierarchies, however, it is occasionally useful to inherit +// separately from Test and WithParamInterface. For example: + +class BaseTest : public ::testing::Test { + // You can inherit all the usual members for a non-parameterized test + // fixture here. +}; + +class DerivedTest : public BaseTest, public ::testing::WithParamInterface { + // The usual test fixture members go here too. +}; + +TEST_F(BaseTest, HasFoo) { + // This is an ordinary non-parameterized test. +} + +TEST_P(DerivedTest, DoesBlah) { + // GetParam works just the same here as if you inherit from TestWithParam. + EXPECT_TRUE(foo.Blah(GetParam())); +} + +#endif // 0 + + +#if !GTEST_OS_SYMBIAN +# include +#endif + +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +// Type and function utilities for implementing parameterized tests. + +// GOOGLETEST_CM0001 DO NOT DELETE + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ + +#include + +#include +#include +#include +#include + +// Copyright 2003 Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// A "smart" pointer type with reference tracking. Every pointer to a +// particular object is kept on a circular linked list. When the last pointer +// to an object is destroyed or reassigned, the object is deleted. +// +// Used properly, this deletes the object when the last reference goes away. +// There are several caveats: +// - Like all reference counting schemes, cycles lead to leaks. +// - Each smart pointer is actually two pointers (8 bytes instead of 4). +// - Every time a pointer is assigned, the entire list of pointers to that +// object is traversed. This class is therefore NOT SUITABLE when there +// will often be more than two or three pointers to a particular object. +// - References are only tracked as long as linked_ptr<> objects are copied. +// If a linked_ptr<> is converted to a raw pointer and back, BAD THINGS +// will happen (double deletion). +// +// A good use of this class is storing object references in STL containers. +// You can safely put linked_ptr<> in a vector<>. +// Other uses may not be as good. +// +// Note: If you use an incomplete type with linked_ptr<>, the class +// *containing* linked_ptr<> must have a constructor and destructor (even +// if they do nothing!). +// +// Bill Gibbons suggested we use something like this. +// +// Thread Safety: +// Unlike other linked_ptr implementations, in this implementation +// a linked_ptr object is thread-safe in the sense that: +// - it's safe to copy linked_ptr objects concurrently, +// - it's safe to copy *from* a linked_ptr and read its underlying +// raw pointer (e.g. via get()) concurrently, and +// - it's safe to write to two linked_ptrs that point to the same +// shared object concurrently. +// FIXME: rename this to safe_linked_ptr to avoid +// confusion with normal linked_ptr. + +// GOOGLETEST_CM0001 DO NOT DELETE + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_LINKED_PTR_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_LINKED_PTR_H_ + +#include +#include + + +namespace testing { +namespace internal { + +// Protects copying of all linked_ptr objects. +GTEST_API_ GTEST_DECLARE_STATIC_MUTEX_(g_linked_ptr_mutex); + +// This is used internally by all instances of linked_ptr<>. It needs to be +// a non-template class because different types of linked_ptr<> can refer to +// the same object (linked_ptr(obj) vs linked_ptr(obj)). +// So, it needs to be possible for different types of linked_ptr to participate +// in the same circular linked list, so we need a single class type here. +// +// DO NOT USE THIS CLASS DIRECTLY YOURSELF. Use linked_ptr. +class linked_ptr_internal { + public: + // Create a new circle that includes only this instance. + void join_new() { + next_ = this; + } + + // Many linked_ptr operations may change p.link_ for some linked_ptr + // variable p in the same circle as this object. Therefore we need + // to prevent two such operations from occurring concurrently. + // + // Note that different types of linked_ptr objects can coexist in a + // circle (e.g. linked_ptr, linked_ptr, and + // linked_ptr). Therefore we must use a single mutex to + // protect all linked_ptr objects. This can create serious + // contention in production code, but is acceptable in a testing + // framework. + + // Join an existing circle. + void join(linked_ptr_internal const* ptr) + GTEST_LOCK_EXCLUDED_(g_linked_ptr_mutex) { + MutexLock lock(&g_linked_ptr_mutex); + + linked_ptr_internal const* p = ptr; + while (p->next_ != ptr) { + assert(p->next_ != this && + "Trying to join() a linked ring we are already in. " + "Is GMock thread safety enabled?"); + p = p->next_; + } + p->next_ = this; + next_ = ptr; + } + + // Leave whatever circle we're part of. Returns true if we were the + // last member of the circle. Once this is done, you can join() another. + bool depart() + GTEST_LOCK_EXCLUDED_(g_linked_ptr_mutex) { + MutexLock lock(&g_linked_ptr_mutex); + + if (next_ == this) return true; + linked_ptr_internal const* p = next_; + while (p->next_ != this) { + assert(p->next_ != next_ && + "Trying to depart() a linked ring we are not in. " + "Is GMock thread safety enabled?"); + p = p->next_; + } + p->next_ = next_; + return false; + } + + private: + mutable linked_ptr_internal const* next_; +}; + +template +class linked_ptr { + public: + typedef T element_type; + + // Take over ownership of a raw pointer. This should happen as soon as + // possible after the object is created. + explicit linked_ptr(T* ptr = NULL) { capture(ptr); } + ~linked_ptr() { depart(); } + + // Copy an existing linked_ptr<>, adding ourselves to the list of references. + template linked_ptr(linked_ptr const& ptr) { copy(&ptr); } + linked_ptr(linked_ptr const& ptr) { // NOLINT + assert(&ptr != this); + copy(&ptr); + } + + // Assignment releases the old value and acquires the new. + template linked_ptr& operator=(linked_ptr const& ptr) { + depart(); + copy(&ptr); + return *this; + } + + linked_ptr& operator=(linked_ptr const& ptr) { + if (&ptr != this) { + depart(); + copy(&ptr); + } + return *this; + } + + // Smart pointer members. + void reset(T* ptr = NULL) { + depart(); + capture(ptr); + } + T* get() const { return value_; } + T* operator->() const { return value_; } + T& operator*() const { return *value_; } + + bool operator==(T* p) const { return value_ == p; } + bool operator!=(T* p) const { return value_ != p; } + template + bool operator==(linked_ptr const& ptr) const { + return value_ == ptr.get(); + } + template + bool operator!=(linked_ptr const& ptr) const { + return value_ != ptr.get(); + } + + private: + template + friend class linked_ptr; + + T* value_; + linked_ptr_internal link_; + + void depart() { + if (link_.depart()) delete value_; + } + + void capture(T* ptr) { + value_ = ptr; + link_.join_new(); + } + + template void copy(linked_ptr const* ptr) { + value_ = ptr->get(); + if (value_) + link_.join(&ptr->link_); + else + link_.join_new(); + } +}; + +template inline +bool operator==(T* ptr, const linked_ptr& x) { + return ptr == x.get(); +} + +template inline +bool operator!=(T* ptr, const linked_ptr& x) { + return ptr != x.get(); +} + +// A function to convert T* into linked_ptr +// Doing e.g. make_linked_ptr(new FooBarBaz(arg)) is a shorter notation +// for linked_ptr >(new FooBarBaz(arg)) +template +linked_ptr make_linked_ptr(T* ptr) { + return linked_ptr(ptr); +} + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_LINKED_PTR_H_ +// Copyright 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +// Google Test - The Google C++ Testing and Mocking Framework +// +// This file implements a universal value printer that can print a +// value of any type T: +// +// void ::testing::internal::UniversalPrinter::Print(value, ostream_ptr); +// +// A user can teach this function how to print a class type T by +// defining either operator<<() or PrintTo() in the namespace that +// defines T. More specifically, the FIRST defined function in the +// following list will be used (assuming T is defined in namespace +// foo): +// +// 1. foo::PrintTo(const T&, ostream*) +// 2. operator<<(ostream&, const T&) defined in either foo or the +// global namespace. +// +// However if T is an STL-style container then it is printed element-wise +// unless foo::PrintTo(const T&, ostream*) is defined. Note that +// operator<<() is ignored for container types. +// +// If none of the above is defined, it will print the debug string of +// the value if it is a protocol buffer, or print the raw bytes in the +// value otherwise. +// +// To aid debugging: when T is a reference type, the address of the +// value is also printed; when T is a (const) char pointer, both the +// pointer value and the NUL-terminated string it points to are +// printed. +// +// We also provide some convenient wrappers: +// +// // Prints a value to a string. For a (const or not) char +// // pointer, the NUL-terminated string (but not the pointer) is +// // printed. +// std::string ::testing::PrintToString(const T& value); +// +// // Prints a value tersely: for a reference type, the referenced +// // value (but not the address) is printed; for a (const or not) char +// // pointer, the NUL-terminated string (but not the pointer) is +// // printed. +// void ::testing::internal::UniversalTersePrint(const T& value, ostream*); +// +// // Prints value using the type inferred by the compiler. The difference +// // from UniversalTersePrint() is that this function prints both the +// // pointer and the NUL-terminated string for a (const or not) char pointer. +// void ::testing::internal::UniversalPrint(const T& value, ostream*); +// +// // Prints the fields of a tuple tersely to a string vector, one +// // element for each field. Tuple support must be enabled in +// // gtest-port.h. +// std::vector UniversalTersePrintTupleFieldsToStrings( +// const Tuple& value); +// +// Known limitation: +// +// The print primitives print the elements of an STL-style container +// using the compiler-inferred type of *iter where iter is a +// const_iterator of the container. When const_iterator is an input +// iterator but not a forward iterator, this inferred type may not +// match value_type, and the print output may be incorrect. In +// practice, this is rarely a problem as for most containers +// const_iterator is a forward iterator. We'll fix this if there's an +// actual need for it. Note that this fix cannot rely on value_type +// being defined as many user-defined container types don't have +// value_type. + +// GOOGLETEST_CM0001 DO NOT DELETE + +#ifndef GTEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ +#define GTEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ + +#include // NOLINT +#include +#include +#include +#include + +#if GTEST_HAS_STD_TUPLE_ +# include +#endif + +#if GTEST_HAS_ABSL +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "absl/types/variant.h" +#endif // GTEST_HAS_ABSL + +namespace testing { + +// Definitions in the 'internal' and 'internal2' name spaces are +// subject to change without notice. DO NOT USE THEM IN USER CODE! +namespace internal2 { + +// Prints the given number of bytes in the given object to the given +// ostream. +GTEST_API_ void PrintBytesInObjectTo(const unsigned char* obj_bytes, + size_t count, + ::std::ostream* os); + +// For selecting which printer to use when a given type has neither << +// nor PrintTo(). +enum TypeKind { + kProtobuf, // a protobuf type + kConvertibleToInteger, // a type implicitly convertible to BiggestInt + // (e.g. a named or unnamed enum type) +#if GTEST_HAS_ABSL + kConvertibleToStringView, // a type implicitly convertible to + // absl::string_view +#endif + kOtherType // anything else +}; + +// TypeWithoutFormatter::PrintValue(value, os) is called +// by the universal printer to print a value of type T when neither +// operator<< nor PrintTo() is defined for T, where kTypeKind is the +// "kind" of T as defined by enum TypeKind. +template +class TypeWithoutFormatter { + public: + // This default version is called when kTypeKind is kOtherType. + static void PrintValue(const T& value, ::std::ostream* os) { + PrintBytesInObjectTo(static_cast( + reinterpret_cast(&value)), + sizeof(value), os); + } +}; + +// We print a protobuf using its ShortDebugString() when the string +// doesn't exceed this many characters; otherwise we print it using +// DebugString() for better readability. +const size_t kProtobufOneLinerMaxLength = 50; + +template +class TypeWithoutFormatter { + public: + static void PrintValue(const T& value, ::std::ostream* os) { + std::string pretty_str = value.ShortDebugString(); + if (pretty_str.length() > kProtobufOneLinerMaxLength) { + pretty_str = "\n" + value.DebugString(); + } + *os << ("<" + pretty_str + ">"); + } +}; + +template +class TypeWithoutFormatter { + public: + // Since T has no << operator or PrintTo() but can be implicitly + // converted to BiggestInt, we print it as a BiggestInt. + // + // Most likely T is an enum type (either named or unnamed), in which + // case printing it as an integer is the desired behavior. In case + // T is not an enum, printing it as an integer is the best we can do + // given that it has no user-defined printer. + static void PrintValue(const T& value, ::std::ostream* os) { + const internal::BiggestInt kBigInt = value; + *os << kBigInt; + } +}; + +#if GTEST_HAS_ABSL +template +class TypeWithoutFormatter { + public: + // Since T has neither operator<< nor PrintTo() but can be implicitly + // converted to absl::string_view, we print it as a absl::string_view. + // + // Note: the implementation is further below, as it depends on + // internal::PrintTo symbol which is defined later in the file. + static void PrintValue(const T& value, ::std::ostream* os); +}; +#endif + +// Prints the given value to the given ostream. If the value is a +// protocol message, its debug string is printed; if it's an enum or +// of a type implicitly convertible to BiggestInt, it's printed as an +// integer; otherwise the bytes in the value are printed. This is +// what UniversalPrinter::Print() does when it knows nothing about +// type T and T has neither << operator nor PrintTo(). +// +// A user can override this behavior for a class type Foo by defining +// a << operator in the namespace where Foo is defined. +// +// We put this operator in namespace 'internal2' instead of 'internal' +// to simplify the implementation, as much code in 'internal' needs to +// use << in STL, which would conflict with our own << were it defined +// in 'internal'. +// +// Note that this operator<< takes a generic std::basic_ostream type instead of the more restricted std::ostream. If +// we define it to take an std::ostream instead, we'll get an +// "ambiguous overloads" compiler error when trying to print a type +// Foo that supports streaming to std::basic_ostream, as the compiler cannot tell whether +// operator<<(std::ostream&, const T&) or +// operator<<(std::basic_stream, const Foo&) is more +// specific. +template +::std::basic_ostream& operator<<( + ::std::basic_ostream& os, const T& x) { + TypeWithoutFormatter::value + ? kProtobuf + : internal::ImplicitlyConvertible< + const T&, internal::BiggestInt>::value + ? kConvertibleToInteger + : +#if GTEST_HAS_ABSL + internal::ImplicitlyConvertible< + const T&, absl::string_view>::value + ? kConvertibleToStringView + : +#endif + kOtherType)>::PrintValue(x, &os); + return os; +} + +} // namespace internal2 +} // namespace testing + +// This namespace MUST NOT BE NESTED IN ::testing, or the name look-up +// magic needed for implementing UniversalPrinter won't work. +namespace testing_internal { + +// Used to print a value that is not an STL-style container when the +// user doesn't define PrintTo() for it. +template +void DefaultPrintNonContainerTo(const T& value, ::std::ostream* os) { + // With the following statement, during unqualified name lookup, + // testing::internal2::operator<< appears as if it was declared in + // the nearest enclosing namespace that contains both + // ::testing_internal and ::testing::internal2, i.e. the global + // namespace. For more details, refer to the C++ Standard section + // 7.3.4-1 [namespace.udir]. This allows us to fall back onto + // testing::internal2::operator<< in case T doesn't come with a << + // operator. + // + // We cannot write 'using ::testing::internal2::operator<<;', which + // gcc 3.3 fails to compile due to a compiler bug. + using namespace ::testing::internal2; // NOLINT + + // Assuming T is defined in namespace foo, in the next statement, + // the compiler will consider all of: + // + // 1. foo::operator<< (thanks to Koenig look-up), + // 2. ::operator<< (as the current namespace is enclosed in ::), + // 3. testing::internal2::operator<< (thanks to the using statement above). + // + // The operator<< whose type matches T best will be picked. + // + // We deliberately allow #2 to be a candidate, as sometimes it's + // impossible to define #1 (e.g. when foo is ::std, defining + // anything in it is undefined behavior unless you are a compiler + // vendor.). + *os << value; +} + +} // namespace testing_internal + +namespace testing { +namespace internal { + +// FormatForComparison::Format(value) formats a +// value of type ToPrint that is an operand of a comparison assertion +// (e.g. ASSERT_EQ). OtherOperand is the type of the other operand in +// the comparison, and is used to help determine the best way to +// format the value. In particular, when the value is a C string +// (char pointer) and the other operand is an STL string object, we +// want to format the C string as a string, since we know it is +// compared by value with the string object. If the value is a char +// pointer but the other operand is not an STL string object, we don't +// know whether the pointer is supposed to point to a NUL-terminated +// string, and thus want to print it as a pointer to be safe. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + +// The default case. +template +class FormatForComparison { + public: + static ::std::string Format(const ToPrint& value) { + return ::testing::PrintToString(value); + } +}; + +// Array. +template +class FormatForComparison { + public: + static ::std::string Format(const ToPrint* value) { + return FormatForComparison::Format(value); + } +}; + +// By default, print C string as pointers to be safe, as we don't know +// whether they actually point to a NUL-terminated string. + +#define GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(CharType) \ + template \ + class FormatForComparison { \ + public: \ + static ::std::string Format(CharType* value) { \ + return ::testing::PrintToString(static_cast(value)); \ + } \ + } + +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(char); +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(const char); +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(wchar_t); +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(const wchar_t); + +#undef GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_ + +// If a C string is compared with an STL string object, we know it's meant +// to point to a NUL-terminated string, and thus can print it as a string. + +#define GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(CharType, OtherStringType) \ + template <> \ + class FormatForComparison { \ + public: \ + static ::std::string Format(CharType* value) { \ + return ::testing::PrintToString(value); \ + } \ + } + +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(char, ::std::string); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const char, ::std::string); + +#if GTEST_HAS_GLOBAL_STRING +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(char, ::string); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const char, ::string); +#endif + +#if GTEST_HAS_GLOBAL_WSTRING +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(wchar_t, ::wstring); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const wchar_t, ::wstring); +#endif + +#if GTEST_HAS_STD_WSTRING +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(wchar_t, ::std::wstring); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const wchar_t, ::std::wstring); +#endif + +#undef GTEST_IMPL_FORMAT_C_STRING_AS_STRING_ + +// Formats a comparison assertion (e.g. ASSERT_EQ, EXPECT_LT, and etc) +// operand to be used in a failure message. The type (but not value) +// of the other operand may affect the format. This allows us to +// print a char* as a raw pointer when it is compared against another +// char* or void*, and print it as a C string when it is compared +// against an std::string object, for example. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +template +std::string FormatForComparisonFailureMessage( + const T1& value, const T2& /* other_operand */) { + return FormatForComparison::Format(value); +} + +// UniversalPrinter::Print(value, ostream_ptr) prints the given +// value to the given ostream. The caller must ensure that +// 'ostream_ptr' is not NULL, or the behavior is undefined. +// +// We define UniversalPrinter as a class template (as opposed to a +// function template), as we need to partially specialize it for +// reference types, which cannot be done with function templates. +template +class UniversalPrinter; + +template +void UniversalPrint(const T& value, ::std::ostream* os); + +enum DefaultPrinterType { + kPrintContainer, + kPrintPointer, + kPrintFunctionPointer, + kPrintOther, +}; +template struct WrapPrinterType {}; + +// Used to print an STL-style container when the user doesn't define +// a PrintTo() for it. +template +void DefaultPrintTo(WrapPrinterType /* dummy */, + const C& container, ::std::ostream* os) { + const size_t kMaxCount = 32; // The maximum number of elements to print. + *os << '{'; + size_t count = 0; + for (typename C::const_iterator it = container.begin(); + it != container.end(); ++it, ++count) { + if (count > 0) { + *os << ','; + if (count == kMaxCount) { // Enough has been printed. + *os << " ..."; + break; + } + } + *os << ' '; + // We cannot call PrintTo(*it, os) here as PrintTo() doesn't + // handle *it being a native array. + internal::UniversalPrint(*it, os); + } + + if (count > 0) { + *os << ' '; + } + *os << '}'; +} + +// Used to print a pointer that is neither a char pointer nor a member +// pointer, when the user doesn't define PrintTo() for it. (A member +// variable pointer or member function pointer doesn't really point to +// a location in the address space. Their representation is +// implementation-defined. Therefore they will be printed as raw +// bytes.) +template +void DefaultPrintTo(WrapPrinterType /* dummy */, + T* p, ::std::ostream* os) { + if (p == NULL) { + *os << "NULL"; + } else { + // T is not a function type. We just call << to print p, + // relying on ADL to pick up user-defined << for their pointer + // types, if any. + *os << p; + } +} +template +void DefaultPrintTo(WrapPrinterType /* dummy */, + T* p, ::std::ostream* os) { + if (p == NULL) { + *os << "NULL"; + } else { + // T is a function type, so '*os << p' doesn't do what we want + // (it just prints p as bool). We want to print p as a const + // void*. + *os << reinterpret_cast(p); + } +} + +// Used to print a non-container, non-pointer value when the user +// doesn't define PrintTo() for it. +template +void DefaultPrintTo(WrapPrinterType /* dummy */, + const T& value, ::std::ostream* os) { + ::testing_internal::DefaultPrintNonContainerTo(value, os); +} + +// Prints the given value using the << operator if it has one; +// otherwise prints the bytes in it. This is what +// UniversalPrinter::Print() does when PrintTo() is not specialized +// or overloaded for type T. +// +// A user can override this behavior for a class type Foo by defining +// an overload of PrintTo() in the namespace where Foo is defined. We +// give the user this option as sometimes defining a << operator for +// Foo is not desirable (e.g. the coding style may prevent doing it, +// or there is already a << operator but it doesn't do what the user +// wants). +template +void PrintTo(const T& value, ::std::ostream* os) { + // DefaultPrintTo() is overloaded. The type of its first argument + // determines which version will be picked. + // + // Note that we check for container types here, prior to we check + // for protocol message types in our operator<<. The rationale is: + // + // For protocol messages, we want to give people a chance to + // override Google Mock's format by defining a PrintTo() or + // operator<<. For STL containers, other formats can be + // incompatible with Google Mock's format for the container + // elements; therefore we check for container types here to ensure + // that our format is used. + // + // Note that MSVC and clang-cl do allow an implicit conversion from + // pointer-to-function to pointer-to-object, but clang-cl warns on it. + // So don't use ImplicitlyConvertible if it can be helped since it will + // cause this warning, and use a separate overload of DefaultPrintTo for + // function pointers so that the `*os << p` in the object pointer overload + // doesn't cause that warning either. + DefaultPrintTo( + WrapPrinterType < + (sizeof(IsContainerTest(0)) == sizeof(IsContainer)) && + !IsRecursiveContainer::value + ? kPrintContainer + : !is_pointer::value + ? kPrintOther +#if GTEST_LANG_CXX11 + : std::is_function::type>::value +#else + : !internal::ImplicitlyConvertible::value +#endif + ? kPrintFunctionPointer + : kPrintPointer > (), + value, os); +} + +// The following list of PrintTo() overloads tells +// UniversalPrinter::Print() how to print standard types (built-in +// types, strings, plain arrays, and pointers). + +// Overloads for various char types. +GTEST_API_ void PrintTo(unsigned char c, ::std::ostream* os); +GTEST_API_ void PrintTo(signed char c, ::std::ostream* os); +inline void PrintTo(char c, ::std::ostream* os) { + // When printing a plain char, we always treat it as unsigned. This + // way, the output won't be affected by whether the compiler thinks + // char is signed or not. + PrintTo(static_cast(c), os); +} + +// Overloads for other simple built-in types. +inline void PrintTo(bool x, ::std::ostream* os) { + *os << (x ? "true" : "false"); +} + +// Overload for wchar_t type. +// Prints a wchar_t as a symbol if it is printable or as its internal +// code otherwise and also as its decimal code (except for L'\0'). +// The L'\0' char is printed as "L'\\0'". The decimal code is printed +// as signed integer when wchar_t is implemented by the compiler +// as a signed type and is printed as an unsigned integer when wchar_t +// is implemented as an unsigned type. +GTEST_API_ void PrintTo(wchar_t wc, ::std::ostream* os); + +// Overloads for C strings. +GTEST_API_ void PrintTo(const char* s, ::std::ostream* os); +inline void PrintTo(char* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} + +// signed/unsigned char is often used for representing binary data, so +// we print pointers to it as void* to be safe. +inline void PrintTo(const signed char* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} +inline void PrintTo(signed char* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} +inline void PrintTo(const unsigned char* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} +inline void PrintTo(unsigned char* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} + +// MSVC can be configured to define wchar_t as a typedef of unsigned +// short. It defines _NATIVE_WCHAR_T_DEFINED when wchar_t is a native +// type. When wchar_t is a typedef, defining an overload for const +// wchar_t* would cause unsigned short* be printed as a wide string, +// possibly causing invalid memory accesses. +#if !defined(_MSC_VER) || defined(_NATIVE_WCHAR_T_DEFINED) +// Overloads for wide C strings +GTEST_API_ void PrintTo(const wchar_t* s, ::std::ostream* os); +inline void PrintTo(wchar_t* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} +#endif + +// Overload for C arrays. Multi-dimensional arrays are printed +// properly. + +// Prints the given number of elements in an array, without printing +// the curly braces. +template +void PrintRawArrayTo(const T a[], size_t count, ::std::ostream* os) { + UniversalPrint(a[0], os); + for (size_t i = 1; i != count; i++) { + *os << ", "; + UniversalPrint(a[i], os); + } +} + +// Overloads for ::string and ::std::string. +#if GTEST_HAS_GLOBAL_STRING +GTEST_API_ void PrintStringTo(const ::string&s, ::std::ostream* os); +inline void PrintTo(const ::string& s, ::std::ostream* os) { + PrintStringTo(s, os); +} +#endif // GTEST_HAS_GLOBAL_STRING + +GTEST_API_ void PrintStringTo(const ::std::string&s, ::std::ostream* os); +inline void PrintTo(const ::std::string& s, ::std::ostream* os) { + PrintStringTo(s, os); +} + +// Overloads for ::wstring and ::std::wstring. +#if GTEST_HAS_GLOBAL_WSTRING +GTEST_API_ void PrintWideStringTo(const ::wstring&s, ::std::ostream* os); +inline void PrintTo(const ::wstring& s, ::std::ostream* os) { + PrintWideStringTo(s, os); +} +#endif // GTEST_HAS_GLOBAL_WSTRING + +#if GTEST_HAS_STD_WSTRING +GTEST_API_ void PrintWideStringTo(const ::std::wstring&s, ::std::ostream* os); +inline void PrintTo(const ::std::wstring& s, ::std::ostream* os) { + PrintWideStringTo(s, os); +} +#endif // GTEST_HAS_STD_WSTRING + +#if GTEST_HAS_ABSL +// Overload for absl::string_view. +inline void PrintTo(absl::string_view sp, ::std::ostream* os) { + PrintTo(::std::string(sp), os); +} +#endif // GTEST_HAS_ABSL + +#if GTEST_LANG_CXX11 +inline void PrintTo(std::nullptr_t, ::std::ostream* os) { *os << "(nullptr)"; } +#endif // GTEST_LANG_CXX11 + +#if GTEST_HAS_TR1_TUPLE || GTEST_HAS_STD_TUPLE_ +// Helper function for printing a tuple. T must be instantiated with +// a tuple type. +template +void PrintTupleTo(const T& t, ::std::ostream* os); +#endif // GTEST_HAS_TR1_TUPLE || GTEST_HAS_STD_TUPLE_ + +#if GTEST_HAS_TR1_TUPLE +// Overload for ::std::tr1::tuple. Needed for printing function arguments, +// which are packed as tuples. + +// Overloaded PrintTo() for tuples of various arities. We support +// tuples of up-to 10 fields. The following implementation works +// regardless of whether tr1::tuple is implemented using the +// non-standard variadic template feature or not. + +inline void PrintTo(const ::std::tr1::tuple<>& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo( + const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} +#endif // GTEST_HAS_TR1_TUPLE + +#if GTEST_HAS_STD_TUPLE_ +template +void PrintTo(const ::std::tuple& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} +#endif // GTEST_HAS_STD_TUPLE_ + +// Overload for std::pair. +template +void PrintTo(const ::std::pair& value, ::std::ostream* os) { + *os << '('; + // We cannot use UniversalPrint(value.first, os) here, as T1 may be + // a reference type. The same for printing value.second. + UniversalPrinter::Print(value.first, os); + *os << ", "; + UniversalPrinter::Print(value.second, os); + *os << ')'; +} + +// Implements printing a non-reference type T by letting the compiler +// pick the right overload of PrintTo() for T. +template +class UniversalPrinter { + public: + // MSVC warns about adding const to a function type, so we want to + // disable the warning. + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4180) + + // Note: we deliberately don't call this PrintTo(), as that name + // conflicts with ::testing::internal::PrintTo in the body of the + // function. + static void Print(const T& value, ::std::ostream* os) { + // By default, ::testing::internal::PrintTo() is used for printing + // the value. + // + // Thanks to Koenig look-up, if T is a class and has its own + // PrintTo() function defined in its namespace, that function will + // be visible here. Since it is more specific than the generic ones + // in ::testing::internal, it will be picked by the compiler in the + // following statement - exactly what we want. + PrintTo(value, os); + } + + GTEST_DISABLE_MSC_WARNINGS_POP_() +}; + +#if GTEST_HAS_ABSL + +// Printer for absl::optional + +template +class UniversalPrinter<::absl::optional> { + public: + static void Print(const ::absl::optional& value, ::std::ostream* os) { + *os << '('; + if (!value) { + *os << "nullopt"; + } else { + UniversalPrint(*value, os); + } + *os << ')'; + } +}; + +// Printer for absl::variant + +template +class UniversalPrinter<::absl::variant> { + public: + static void Print(const ::absl::variant& value, ::std::ostream* os) { + *os << '('; + absl::visit(Visitor{os}, value); + *os << ')'; + } + + private: + struct Visitor { + template + void operator()(const U& u) const { + *os << "'" << GetTypeName() << "' with value "; + UniversalPrint(u, os); + } + ::std::ostream* os; + }; +}; + +#endif // GTEST_HAS_ABSL + +// UniversalPrintArray(begin, len, os) prints an array of 'len' +// elements, starting at address 'begin'. +template +void UniversalPrintArray(const T* begin, size_t len, ::std::ostream* os) { + if (len == 0) { + *os << "{}"; + } else { + *os << "{ "; + const size_t kThreshold = 18; + const size_t kChunkSize = 8; + // If the array has more than kThreshold elements, we'll have to + // omit some details by printing only the first and the last + // kChunkSize elements. + // FIXME: let the user control the threshold using a flag. + if (len <= kThreshold) { + PrintRawArrayTo(begin, len, os); + } else { + PrintRawArrayTo(begin, kChunkSize, os); + *os << ", ..., "; + PrintRawArrayTo(begin + len - kChunkSize, kChunkSize, os); + } + *os << " }"; + } +} +// This overload prints a (const) char array compactly. +GTEST_API_ void UniversalPrintArray( + const char* begin, size_t len, ::std::ostream* os); + +// This overload prints a (const) wchar_t array compactly. +GTEST_API_ void UniversalPrintArray( + const wchar_t* begin, size_t len, ::std::ostream* os); + +// Implements printing an array type T[N]. +template +class UniversalPrinter { + public: + // Prints the given array, omitting some elements when there are too + // many. + static void Print(const T (&a)[N], ::std::ostream* os) { + UniversalPrintArray(a, N, os); + } +}; + +// Implements printing a reference type T&. +template +class UniversalPrinter { + public: + // MSVC warns about adding const to a function type, so we want to + // disable the warning. + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4180) + + static void Print(const T& value, ::std::ostream* os) { + // Prints the address of the value. We use reinterpret_cast here + // as static_cast doesn't compile when T is a function type. + *os << "@" << reinterpret_cast(&value) << " "; + + // Then prints the value itself. + UniversalPrint(value, os); + } + + GTEST_DISABLE_MSC_WARNINGS_POP_() +}; + +// Prints a value tersely: for a reference type, the referenced value +// (but not the address) is printed; for a (const) char pointer, the +// NUL-terminated string (but not the pointer) is printed. + +template +class UniversalTersePrinter { + public: + static void Print(const T& value, ::std::ostream* os) { + UniversalPrint(value, os); + } +}; +template +class UniversalTersePrinter { + public: + static void Print(const T& value, ::std::ostream* os) { + UniversalPrint(value, os); + } +}; +template +class UniversalTersePrinter { + public: + static void Print(const T (&value)[N], ::std::ostream* os) { + UniversalPrinter::Print(value, os); + } +}; +template <> +class UniversalTersePrinter { + public: + static void Print(const char* str, ::std::ostream* os) { + if (str == NULL) { + *os << "NULL"; + } else { + UniversalPrint(std::string(str), os); + } + } +}; +template <> +class UniversalTersePrinter { + public: + static void Print(char* str, ::std::ostream* os) { + UniversalTersePrinter::Print(str, os); + } +}; + +#if GTEST_HAS_STD_WSTRING +template <> +class UniversalTersePrinter { + public: + static void Print(const wchar_t* str, ::std::ostream* os) { + if (str == NULL) { + *os << "NULL"; + } else { + UniversalPrint(::std::wstring(str), os); + } + } +}; +#endif + +template <> +class UniversalTersePrinter { + public: + static void Print(wchar_t* str, ::std::ostream* os) { + UniversalTersePrinter::Print(str, os); + } +}; + +template +void UniversalTersePrint(const T& value, ::std::ostream* os) { + UniversalTersePrinter::Print(value, os); +} + +// Prints a value using the type inferred by the compiler. The +// difference between this and UniversalTersePrint() is that for a +// (const) char pointer, this prints both the pointer and the +// NUL-terminated string. +template +void UniversalPrint(const T& value, ::std::ostream* os) { + // A workarond for the bug in VC++ 7.1 that prevents us from instantiating + // UniversalPrinter with T directly. + typedef T T1; + UniversalPrinter::Print(value, os); +} + +typedef ::std::vector< ::std::string> Strings; + +// TuplePolicy must provide: +// - tuple_size +// size of tuple TupleT. +// - get(const TupleT& t) +// static function extracting element I of tuple TupleT. +// - tuple_element::type +// type of element I of tuple TupleT. +template +struct TuplePolicy; + +#if GTEST_HAS_TR1_TUPLE +template +struct TuplePolicy { + typedef TupleT Tuple; + static const size_t tuple_size = ::std::tr1::tuple_size::value; + + template + struct tuple_element : ::std::tr1::tuple_element(I), Tuple> { + }; + + template + static typename AddReference(I), Tuple>::type>::type + get(const Tuple& tuple) { + return ::std::tr1::get(tuple); + } +}; +template +const size_t TuplePolicy::tuple_size; +#endif // GTEST_HAS_TR1_TUPLE + +#if GTEST_HAS_STD_TUPLE_ +template +struct TuplePolicy< ::std::tuple > { + typedef ::std::tuple Tuple; + static const size_t tuple_size = ::std::tuple_size::value; + + template + struct tuple_element : ::std::tuple_element {}; + + template + static const typename ::std::tuple_element::type& get( + const Tuple& tuple) { + return ::std::get(tuple); + } +}; +template +const size_t TuplePolicy< ::std::tuple >::tuple_size; +#endif // GTEST_HAS_STD_TUPLE_ + +#if GTEST_HAS_TR1_TUPLE || GTEST_HAS_STD_TUPLE_ +// This helper template allows PrintTo() for tuples and +// UniversalTersePrintTupleFieldsToStrings() to be defined by +// induction on the number of tuple fields. The idea is that +// TuplePrefixPrinter::PrintPrefixTo(t, os) prints the first N +// fields in tuple t, and can be defined in terms of +// TuplePrefixPrinter. +// +// The inductive case. +template +struct TuplePrefixPrinter { + // Prints the first N fields of a tuple. + template + static void PrintPrefixTo(const Tuple& t, ::std::ostream* os) { + TuplePrefixPrinter::PrintPrefixTo(t, os); + GTEST_INTENTIONAL_CONST_COND_PUSH_() + if (N > 1) { + GTEST_INTENTIONAL_CONST_COND_POP_() + *os << ", "; + } + UniversalPrinter< + typename TuplePolicy::template tuple_element::type> + ::Print(TuplePolicy::template get(t), os); + } + + // Tersely prints the first N fields of a tuple to a string vector, + // one element for each field. + template + static void TersePrintPrefixToStrings(const Tuple& t, Strings* strings) { + TuplePrefixPrinter::TersePrintPrefixToStrings(t, strings); + ::std::stringstream ss; + UniversalTersePrint(TuplePolicy::template get(t), &ss); + strings->push_back(ss.str()); + } +}; + +// Base case. +template <> +struct TuplePrefixPrinter<0> { + template + static void PrintPrefixTo(const Tuple&, ::std::ostream*) {} + + template + static void TersePrintPrefixToStrings(const Tuple&, Strings*) {} +}; + +// Helper function for printing a tuple. +// Tuple must be either std::tr1::tuple or std::tuple type. +template +void PrintTupleTo(const Tuple& t, ::std::ostream* os) { + *os << "("; + TuplePrefixPrinter::tuple_size>::PrintPrefixTo(t, os); + *os << ")"; +} + +// Prints the fields of a tuple tersely to a string vector, one +// element for each field. See the comment before +// UniversalTersePrint() for how we define "tersely". +template +Strings UniversalTersePrintTupleFieldsToStrings(const Tuple& value) { + Strings result; + TuplePrefixPrinter::tuple_size>:: + TersePrintPrefixToStrings(value, &result); + return result; +} +#endif // GTEST_HAS_TR1_TUPLE || GTEST_HAS_STD_TUPLE_ + +} // namespace internal + +#if GTEST_HAS_ABSL +namespace internal2 { +template +void TypeWithoutFormatter::PrintValue( + const T& value, ::std::ostream* os) { + internal::PrintTo(absl::string_view(value), os); +} +} // namespace internal2 +#endif + +template +::std::string PrintToString(const T& value) { + ::std::stringstream ss; + internal::UniversalTersePrinter::Print(value, &ss); + return ss.str(); +} + +} // namespace testing + +// Include any custom printer added by the local installation. +// We must include this header at the end to make sure it can use the +// declarations from this file. +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// This file provides an injection point for custom printers in a local +// installation of gTest. +// It will be included from gtest-printers.h and the overrides in this file +// will be visible to everyone. +// +// Injection point for custom user configurations. See README for details +// +// ** Custom implementation starts here ** + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ + +#endif // GTEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ + +namespace testing { + +// Input to a parameterized test name generator, describing a test parameter. +// Consists of the parameter value and the integer parameter index. +template +struct TestParamInfo { + TestParamInfo(const ParamType& a_param, size_t an_index) : + param(a_param), + index(an_index) {} + ParamType param; + size_t index; +}; + +// A builtin parameterized test name generator which returns the result of +// testing::PrintToString. +struct PrintToStringParamName { + template + std::string operator()(const TestParamInfo& info) const { + return PrintToString(info.param); + } +}; + +namespace internal { + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Outputs a message explaining invalid registration of different +// fixture class for the same test case. This may happen when +// TEST_P macro is used to define two tests with the same name +// but in different namespaces. +GTEST_API_ void ReportInvalidTestCaseType(const char* test_case_name, + CodeLocation code_location); + +template class ParamGeneratorInterface; +template class ParamGenerator; + +// Interface for iterating over elements provided by an implementation +// of ParamGeneratorInterface. +template +class ParamIteratorInterface { + public: + virtual ~ParamIteratorInterface() {} + // A pointer to the base generator instance. + // Used only for the purposes of iterator comparison + // to make sure that two iterators belong to the same generator. + virtual const ParamGeneratorInterface* BaseGenerator() const = 0; + // Advances iterator to point to the next element + // provided by the generator. The caller is responsible + // for not calling Advance() on an iterator equal to + // BaseGenerator()->End(). + virtual void Advance() = 0; + // Clones the iterator object. Used for implementing copy semantics + // of ParamIterator. + virtual ParamIteratorInterface* Clone() const = 0; + // Dereferences the current iterator and provides (read-only) access + // to the pointed value. It is the caller's responsibility not to call + // Current() on an iterator equal to BaseGenerator()->End(). + // Used for implementing ParamGenerator::operator*(). + virtual const T* Current() const = 0; + // Determines whether the given iterator and other point to the same + // element in the sequence generated by the generator. + // Used for implementing ParamGenerator::operator==(). + virtual bool Equals(const ParamIteratorInterface& other) const = 0; +}; + +// Class iterating over elements provided by an implementation of +// ParamGeneratorInterface. It wraps ParamIteratorInterface +// and implements the const forward iterator concept. +template +class ParamIterator { + public: + typedef T value_type; + typedef const T& reference; + typedef ptrdiff_t difference_type; + + // ParamIterator assumes ownership of the impl_ pointer. + ParamIterator(const ParamIterator& other) : impl_(other.impl_->Clone()) {} + ParamIterator& operator=(const ParamIterator& other) { + if (this != &other) + impl_.reset(other.impl_->Clone()); + return *this; + } + + const T& operator*() const { return *impl_->Current(); } + const T* operator->() const { return impl_->Current(); } + // Prefix version of operator++. + ParamIterator& operator++() { + impl_->Advance(); + return *this; + } + // Postfix version of operator++. + ParamIterator operator++(int /*unused*/) { + ParamIteratorInterface* clone = impl_->Clone(); + impl_->Advance(); + return ParamIterator(clone); + } + bool operator==(const ParamIterator& other) const { + return impl_.get() == other.impl_.get() || impl_->Equals(*other.impl_); + } + bool operator!=(const ParamIterator& other) const { + return !(*this == other); + } + + private: + friend class ParamGenerator; + explicit ParamIterator(ParamIteratorInterface* impl) : impl_(impl) {} + scoped_ptr > impl_; +}; + +// ParamGeneratorInterface is the binary interface to access generators +// defined in other translation units. +template +class ParamGeneratorInterface { + public: + typedef T ParamType; + + virtual ~ParamGeneratorInterface() {} + + // Generator interface definition + virtual ParamIteratorInterface* Begin() const = 0; + virtual ParamIteratorInterface* End() const = 0; +}; + +// Wraps ParamGeneratorInterface and provides general generator syntax +// compatible with the STL Container concept. +// This class implements copy initialization semantics and the contained +// ParamGeneratorInterface instance is shared among all copies +// of the original object. This is possible because that instance is immutable. +template +class ParamGenerator { + public: + typedef ParamIterator iterator; + + explicit ParamGenerator(ParamGeneratorInterface* impl) : impl_(impl) {} + ParamGenerator(const ParamGenerator& other) : impl_(other.impl_) {} + + ParamGenerator& operator=(const ParamGenerator& other) { + impl_ = other.impl_; + return *this; + } + + iterator begin() const { return iterator(impl_->Begin()); } + iterator end() const { return iterator(impl_->End()); } + + private: + linked_ptr > impl_; +}; + +// Generates values from a range of two comparable values. Can be used to +// generate sequences of user-defined types that implement operator+() and +// operator<(). +// This class is used in the Range() function. +template +class RangeGenerator : public ParamGeneratorInterface { + public: + RangeGenerator(T begin, T end, IncrementT step) + : begin_(begin), end_(end), + step_(step), end_index_(CalculateEndIndex(begin, end, step)) {} + virtual ~RangeGenerator() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, begin_, 0, step_); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, end_, end_index_, step_); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, T value, int index, + IncrementT step) + : base_(base), value_(value), index_(index), step_(step) {} + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + virtual void Advance() { + value_ = static_cast(value_ + step_); + index_++; + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const T* Current() const { return &value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const int other_index = + CheckedDowncastToActualType(&other)->index_; + return index_ == other_index; + } + + private: + Iterator(const Iterator& other) + : ParamIteratorInterface(), + base_(other.base_), value_(other.value_), index_(other.index_), + step_(other.step_) {} + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + T value_; + int index_; + const IncrementT step_; + }; // class RangeGenerator::Iterator + + static int CalculateEndIndex(const T& begin, + const T& end, + const IncrementT& step) { + int end_index = 0; + for (T i = begin; i < end; i = static_cast(i + step)) + end_index++; + return end_index; + } + + // No implementation - assignment is unsupported. + void operator=(const RangeGenerator& other); + + const T begin_; + const T end_; + const IncrementT step_; + // The index for the end() iterator. All the elements in the generated + // sequence are indexed (0-based) to aid iterator comparison. + const int end_index_; +}; // class RangeGenerator + + +// Generates values from a pair of STL-style iterators. Used in the +// ValuesIn() function. The elements are copied from the source range +// since the source can be located on the stack, and the generator +// is likely to persist beyond that stack frame. +template +class ValuesInIteratorRangeGenerator : public ParamGeneratorInterface { + public: + template + ValuesInIteratorRangeGenerator(ForwardIterator begin, ForwardIterator end) + : container_(begin, end) {} + virtual ~ValuesInIteratorRangeGenerator() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, container_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, container_.end()); + } + + private: + typedef typename ::std::vector ContainerType; + + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + typename ContainerType::const_iterator iterator) + : base_(base), iterator_(iterator) {} + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + virtual void Advance() { + ++iterator_; + value_.reset(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + // We need to use cached value referenced by iterator_ because *iterator_ + // can return a temporary object (and of type other then T), so just + // having "return &*iterator_;" doesn't work. + // value_ is updated here and not in Advance() because Advance() + // can advance iterator_ beyond the end of the range, and we cannot + // detect that fact. The client code, on the other hand, is + // responsible for not calling Current() on an out-of-range iterator. + virtual const T* Current() const { + if (value_.get() == NULL) + value_.reset(new T(*iterator_)); + return value_.get(); + } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + return iterator_ == + CheckedDowncastToActualType(&other)->iterator_; + } + + private: + Iterator(const Iterator& other) + // The explicit constructor call suppresses a false warning + // emitted by gcc when supplied with the -Wextra option. + : ParamIteratorInterface(), + base_(other.base_), + iterator_(other.iterator_) {} + + const ParamGeneratorInterface* const base_; + typename ContainerType::const_iterator iterator_; + // A cached value of *iterator_. We keep it here to allow access by + // pointer in the wrapping iterator's operator->(). + // value_ needs to be mutable to be accessed in Current(). + // Use of scoped_ptr helps manage cached value's lifetime, + // which is bound by the lifespan of the iterator itself. + mutable scoped_ptr value_; + }; // class ValuesInIteratorRangeGenerator::Iterator + + // No implementation - assignment is unsupported. + void operator=(const ValuesInIteratorRangeGenerator& other); + + const ContainerType container_; +}; // class ValuesInIteratorRangeGenerator + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Default parameterized test name generator, returns a string containing the +// integer test parameter index. +template +std::string DefaultParamName(const TestParamInfo& info) { + Message name_stream; + name_stream << info.index; + return name_stream.GetString(); +} + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Parameterized test name overload helpers, which help the +// INSTANTIATE_TEST_CASE_P macro choose between the default parameterized +// test name generator and user param name generator. +template +ParamNameGenFunctor GetParamNameGen(ParamNameGenFunctor func) { + return func; +} + +template +struct ParamNameGenFunc { + typedef std::string Type(const TestParamInfo&); +}; + +template +typename ParamNameGenFunc::Type *GetParamNameGen() { + return DefaultParamName; +} + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Stores a parameter value and later creates tests parameterized with that +// value. +template +class ParameterizedTestFactory : public TestFactoryBase { + public: + typedef typename TestClass::ParamType ParamType; + explicit ParameterizedTestFactory(ParamType parameter) : + parameter_(parameter) {} + virtual Test* CreateTest() { + TestClass::SetParam(¶meter_); + return new TestClass(); + } + + private: + const ParamType parameter_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ParameterizedTestFactory); +}; + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// TestMetaFactoryBase is a base class for meta-factories that create +// test factories for passing into MakeAndRegisterTestInfo function. +template +class TestMetaFactoryBase { + public: + virtual ~TestMetaFactoryBase() {} + + virtual TestFactoryBase* CreateTestFactory(ParamType parameter) = 0; +}; + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// TestMetaFactory creates test factories for passing into +// MakeAndRegisterTestInfo function. Since MakeAndRegisterTestInfo receives +// ownership of test factory pointer, same factory object cannot be passed +// into that method twice. But ParameterizedTestCaseInfo is going to call +// it for each Test/Parameter value combination. Thus it needs meta factory +// creator class. +template +class TestMetaFactory + : public TestMetaFactoryBase { + public: + typedef typename TestCase::ParamType ParamType; + + TestMetaFactory() {} + + virtual TestFactoryBase* CreateTestFactory(ParamType parameter) { + return new ParameterizedTestFactory(parameter); + } + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestMetaFactory); +}; + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// ParameterizedTestCaseInfoBase is a generic interface +// to ParameterizedTestCaseInfo classes. ParameterizedTestCaseInfoBase +// accumulates test information provided by TEST_P macro invocations +// and generators provided by INSTANTIATE_TEST_CASE_P macro invocations +// and uses that information to register all resulting test instances +// in RegisterTests method. The ParameterizeTestCaseRegistry class holds +// a collection of pointers to the ParameterizedTestCaseInfo objects +// and calls RegisterTests() on each of them when asked. +class ParameterizedTestCaseInfoBase { + public: + virtual ~ParameterizedTestCaseInfoBase() {} + + // Base part of test case name for display purposes. + virtual const std::string& GetTestCaseName() const = 0; + // Test case id to verify identity. + virtual TypeId GetTestCaseTypeId() const = 0; + // UnitTest class invokes this method to register tests in this + // test case right before running them in RUN_ALL_TESTS macro. + // This method should not be called more then once on any single + // instance of a ParameterizedTestCaseInfoBase derived class. + virtual void RegisterTests() = 0; + + protected: + ParameterizedTestCaseInfoBase() {} + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(ParameterizedTestCaseInfoBase); +}; + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// ParameterizedTestCaseInfo accumulates tests obtained from TEST_P +// macro invocations for a particular test case and generators +// obtained from INSTANTIATE_TEST_CASE_P macro invocations for that +// test case. It registers tests with all values generated by all +// generators when asked. +template +class ParameterizedTestCaseInfo : public ParameterizedTestCaseInfoBase { + public: + // ParamType and GeneratorCreationFunc are private types but are required + // for declarations of public methods AddTestPattern() and + // AddTestCaseInstantiation(). + typedef typename TestCase::ParamType ParamType; + // A function that returns an instance of appropriate generator type. + typedef ParamGenerator(GeneratorCreationFunc)(); + typedef typename ParamNameGenFunc::Type ParamNameGeneratorFunc; + + explicit ParameterizedTestCaseInfo( + const char* name, CodeLocation code_location) + : test_case_name_(name), code_location_(code_location) {} + + // Test case base name for display purposes. + virtual const std::string& GetTestCaseName() const { return test_case_name_; } + // Test case id to verify identity. + virtual TypeId GetTestCaseTypeId() const { return GetTypeId(); } + // TEST_P macro uses AddTestPattern() to record information + // about a single test in a LocalTestInfo structure. + // test_case_name is the base name of the test case (without invocation + // prefix). test_base_name is the name of an individual test without + // parameter index. For the test SequenceA/FooTest.DoBar/1 FooTest is + // test case base name and DoBar is test base name. + void AddTestPattern(const char* test_case_name, + const char* test_base_name, + TestMetaFactoryBase* meta_factory) { + tests_.push_back(linked_ptr(new TestInfo(test_case_name, + test_base_name, + meta_factory))); + } + // INSTANTIATE_TEST_CASE_P macro uses AddGenerator() to record information + // about a generator. + int AddTestCaseInstantiation(const std::string& instantiation_name, + GeneratorCreationFunc* func, + ParamNameGeneratorFunc* name_func, + const char* file, int line) { + instantiations_.push_back( + InstantiationInfo(instantiation_name, func, name_func, file, line)); + return 0; // Return value used only to run this method in namespace scope. + } + // UnitTest class invokes this method to register tests in this test case + // test cases right before running tests in RUN_ALL_TESTS macro. + // This method should not be called more then once on any single + // instance of a ParameterizedTestCaseInfoBase derived class. + // UnitTest has a guard to prevent from calling this method more then once. + virtual void RegisterTests() { + for (typename TestInfoContainer::iterator test_it = tests_.begin(); + test_it != tests_.end(); ++test_it) { + linked_ptr test_info = *test_it; + for (typename InstantiationContainer::iterator gen_it = + instantiations_.begin(); gen_it != instantiations_.end(); + ++gen_it) { + const std::string& instantiation_name = gen_it->name; + ParamGenerator generator((*gen_it->generator)()); + ParamNameGeneratorFunc* name_func = gen_it->name_func; + const char* file = gen_it->file; + int line = gen_it->line; + + std::string test_case_name; + if ( !instantiation_name.empty() ) + test_case_name = instantiation_name + "/"; + test_case_name += test_info->test_case_base_name; + + size_t i = 0; + std::set test_param_names; + for (typename ParamGenerator::iterator param_it = + generator.begin(); + param_it != generator.end(); ++param_it, ++i) { + Message test_name_stream; + + std::string param_name = name_func( + TestParamInfo(*param_it, i)); + + GTEST_CHECK_(IsValidParamName(param_name)) + << "Parameterized test name '" << param_name + << "' is invalid, in " << file + << " line " << line << std::endl; + + GTEST_CHECK_(test_param_names.count(param_name) == 0) + << "Duplicate parameterized test name '" << param_name + << "', in " << file << " line " << line << std::endl; + + test_param_names.insert(param_name); + + test_name_stream << test_info->test_base_name << "/" << param_name; + MakeAndRegisterTestInfo( + test_case_name.c_str(), + test_name_stream.GetString().c_str(), + NULL, // No type parameter. + PrintToString(*param_it).c_str(), + code_location_, + GetTestCaseTypeId(), + TestCase::SetUpTestCase, + TestCase::TearDownTestCase, + test_info->test_meta_factory->CreateTestFactory(*param_it)); + } // for param_it + } // for gen_it + } // for test_it + } // RegisterTests + + private: + // LocalTestInfo structure keeps information about a single test registered + // with TEST_P macro. + struct TestInfo { + TestInfo(const char* a_test_case_base_name, + const char* a_test_base_name, + TestMetaFactoryBase* a_test_meta_factory) : + test_case_base_name(a_test_case_base_name), + test_base_name(a_test_base_name), + test_meta_factory(a_test_meta_factory) {} + + const std::string test_case_base_name; + const std::string test_base_name; + const scoped_ptr > test_meta_factory; + }; + typedef ::std::vector > TestInfoContainer; + // Records data received from INSTANTIATE_TEST_CASE_P macros: + // + struct InstantiationInfo { + InstantiationInfo(const std::string &name_in, + GeneratorCreationFunc* generator_in, + ParamNameGeneratorFunc* name_func_in, + const char* file_in, + int line_in) + : name(name_in), + generator(generator_in), + name_func(name_func_in), + file(file_in), + line(line_in) {} + + std::string name; + GeneratorCreationFunc* generator; + ParamNameGeneratorFunc* name_func; + const char* file; + int line; + }; + typedef ::std::vector InstantiationContainer; + + static bool IsValidParamName(const std::string& name) { + // Check for empty string + if (name.empty()) + return false; + + // Check for invalid characters + for (std::string::size_type index = 0; index < name.size(); ++index) { + if (!isalnum(name[index]) && name[index] != '_') + return false; + } + + return true; + } + + const std::string test_case_name_; + CodeLocation code_location_; + TestInfoContainer tests_; + InstantiationContainer instantiations_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ParameterizedTestCaseInfo); +}; // class ParameterizedTestCaseInfo + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// ParameterizedTestCaseRegistry contains a map of ParameterizedTestCaseInfoBase +// classes accessed by test case names. TEST_P and INSTANTIATE_TEST_CASE_P +// macros use it to locate their corresponding ParameterizedTestCaseInfo +// descriptors. +class ParameterizedTestCaseRegistry { + public: + ParameterizedTestCaseRegistry() {} + ~ParameterizedTestCaseRegistry() { + for (TestCaseInfoContainer::iterator it = test_case_infos_.begin(); + it != test_case_infos_.end(); ++it) { + delete *it; + } + } + + // Looks up or creates and returns a structure containing information about + // tests and instantiations of a particular test case. + template + ParameterizedTestCaseInfo* GetTestCasePatternHolder( + const char* test_case_name, + CodeLocation code_location) { + ParameterizedTestCaseInfo* typed_test_info = NULL; + for (TestCaseInfoContainer::iterator it = test_case_infos_.begin(); + it != test_case_infos_.end(); ++it) { + if ((*it)->GetTestCaseName() == test_case_name) { + if ((*it)->GetTestCaseTypeId() != GetTypeId()) { + // Complain about incorrect usage of Google Test facilities + // and terminate the program since we cannot guaranty correct + // test case setup and tear-down in this case. + ReportInvalidTestCaseType(test_case_name, code_location); + posix::Abort(); + } else { + // At this point we are sure that the object we found is of the same + // type we are looking for, so we downcast it to that type + // without further checks. + typed_test_info = CheckedDowncastToActualType< + ParameterizedTestCaseInfo >(*it); + } + break; + } + } + if (typed_test_info == NULL) { + typed_test_info = new ParameterizedTestCaseInfo( + test_case_name, code_location); + test_case_infos_.push_back(typed_test_info); + } + return typed_test_info; + } + void RegisterTests() { + for (TestCaseInfoContainer::iterator it = test_case_infos_.begin(); + it != test_case_infos_.end(); ++it) { + (*it)->RegisterTests(); + } + } + + private: + typedef ::std::vector TestCaseInfoContainer; + + TestCaseInfoContainer test_case_infos_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ParameterizedTestCaseRegistry); +}; + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ +// This file was GENERATED by command: +// pump.py gtest-param-util-generated.h.pump +// DO NOT EDIT BY HAND!!! + +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +// Type and function utilities for implementing parameterized tests. +// This file is generated by a SCRIPT. DO NOT EDIT BY HAND! +// +// Currently Google Test supports at most 50 arguments in Values, +// and at most 10 arguments in Combine. Please contact +// googletestframework@googlegroups.com if you need more. +// Please note that the number of arguments to Combine is limited +// by the maximum arity of the implementation of tuple which is +// currently set at 10. + +// GOOGLETEST_CM0001 DO NOT DELETE + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_GENERATED_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_GENERATED_H_ + + +namespace testing { + +// Forward declarations of ValuesIn(), which is implemented in +// include/gtest/gtest-param-test.h. +template +internal::ParamGenerator< + typename ::testing::internal::IteratorTraits::value_type> +ValuesIn(ForwardIterator begin, ForwardIterator end); + +template +internal::ParamGenerator ValuesIn(const T (&array)[N]); + +template +internal::ParamGenerator ValuesIn( + const Container& container); + +namespace internal { + +// Used in the Values() function to provide polymorphic capabilities. +template +class ValueArray1 { + public: + explicit ValueArray1(T1 v1) : v1_(v1) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_)}; + return ValuesIn(array); + } + + ValueArray1(const ValueArray1& other) : v1_(other.v1_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray1& other); + + const T1 v1_; +}; + +template +class ValueArray2 { + public: + ValueArray2(T1 v1, T2 v2) : v1_(v1), v2_(v2) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_)}; + return ValuesIn(array); + } + + ValueArray2(const ValueArray2& other) : v1_(other.v1_), v2_(other.v2_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray2& other); + + const T1 v1_; + const T2 v2_; +}; + +template +class ValueArray3 { + public: + ValueArray3(T1 v1, T2 v2, T3 v3) : v1_(v1), v2_(v2), v3_(v3) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_)}; + return ValuesIn(array); + } + + ValueArray3(const ValueArray3& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray3& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; +}; + +template +class ValueArray4 { + public: + ValueArray4(T1 v1, T2 v2, T3 v3, T4 v4) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_)}; + return ValuesIn(array); + } + + ValueArray4(const ValueArray4& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray4& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; +}; + +template +class ValueArray5 { + public: + ValueArray5(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_)}; + return ValuesIn(array); + } + + ValueArray5(const ValueArray5& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray5& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; +}; + +template +class ValueArray6 { + public: + ValueArray6(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_)}; + return ValuesIn(array); + } + + ValueArray6(const ValueArray6& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray6& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; +}; + +template +class ValueArray7 { + public: + ValueArray7(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_)}; + return ValuesIn(array); + } + + ValueArray7(const ValueArray7& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray7& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; +}; + +template +class ValueArray8 { + public: + ValueArray8(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_)}; + return ValuesIn(array); + } + + ValueArray8(const ValueArray8& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray8& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; +}; + +template +class ValueArray9 { + public: + ValueArray9(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, + T9 v9) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_)}; + return ValuesIn(array); + } + + ValueArray9(const ValueArray9& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray9& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; +}; + +template +class ValueArray10 { + public: + ValueArray10(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_)}; + return ValuesIn(array); + } + + ValueArray10(const ValueArray10& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray10& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; +}; + +template +class ValueArray11 { + public: + ValueArray11(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), + v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_)}; + return ValuesIn(array); + } + + ValueArray11(const ValueArray11& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray11& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; +}; + +template +class ValueArray12 { + public: + ValueArray12(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), + v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_)}; + return ValuesIn(array); + } + + ValueArray12(const ValueArray12& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray12& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; +}; + +template +class ValueArray13 { + public: + ValueArray13(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), + v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), + v12_(v12), v13_(v13) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_)}; + return ValuesIn(array); + } + + ValueArray13(const ValueArray13& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray13& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; +}; + +template +class ValueArray14 { + public: + ValueArray14(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_)}; + return ValuesIn(array); + } + + ValueArray14(const ValueArray14& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray14& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; +}; + +template +class ValueArray15 { + public: + ValueArray15(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_)}; + return ValuesIn(array); + } + + ValueArray15(const ValueArray15& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray15& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; +}; + +template +class ValueArray16 { + public: + ValueArray16(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), + v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), + v16_(v16) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_)}; + return ValuesIn(array); + } + + ValueArray16(const ValueArray16& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray16& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; +}; + +template +class ValueArray17 { + public: + ValueArray17(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, + T17 v17) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_)}; + return ValuesIn(array); + } + + ValueArray17(const ValueArray17& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray17& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; +}; + +template +class ValueArray18 { + public: + ValueArray18(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_)}; + return ValuesIn(array); + } + + ValueArray18(const ValueArray18& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray18& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; +}; + +template +class ValueArray19 { + public: + ValueArray19(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), + v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), + v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_)}; + return ValuesIn(array); + } + + ValueArray19(const ValueArray19& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray19& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; +}; + +template +class ValueArray20 { + public: + ValueArray20(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), + v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), + v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), + v19_(v19), v20_(v20) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_)}; + return ValuesIn(array); + } + + ValueArray20(const ValueArray20& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray20& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; +}; + +template +class ValueArray21 { + public: + ValueArray21(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), + v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), + v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), + v18_(v18), v19_(v19), v20_(v20), v21_(v21) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_)}; + return ValuesIn(array); + } + + ValueArray21(const ValueArray21& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray21& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; +}; + +template +class ValueArray22 { + public: + ValueArray22(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_)}; + return ValuesIn(array); + } + + ValueArray22(const ValueArray22& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray22& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; +}; + +template +class ValueArray23 { + public: + ValueArray23(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_)}; + return ValuesIn(array); + } + + ValueArray23(const ValueArray23& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray23& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; +}; + +template +class ValueArray24 { + public: + ValueArray24(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), + v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), + v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), + v22_(v22), v23_(v23), v24_(v24) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_)}; + return ValuesIn(array); + } + + ValueArray24(const ValueArray24& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray24& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; +}; + +template +class ValueArray25 { + public: + ValueArray25(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, + T25 v25) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_)}; + return ValuesIn(array); + } + + ValueArray25(const ValueArray25& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray25& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; +}; + +template +class ValueArray26 { + public: + ValueArray26(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_)}; + return ValuesIn(array); + } + + ValueArray26(const ValueArray26& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray26& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; +}; + +template +class ValueArray27 { + public: + ValueArray27(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), + v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), + v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), + v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), + v26_(v26), v27_(v27) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_)}; + return ValuesIn(array); + } + + ValueArray27(const ValueArray27& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray27& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; +}; + +template +class ValueArray28 { + public: + ValueArray28(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), + v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), + v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), + v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), + v25_(v25), v26_(v26), v27_(v27), v28_(v28) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_)}; + return ValuesIn(array); + } + + ValueArray28(const ValueArray28& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray28& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; +}; + +template +class ValueArray29 { + public: + ValueArray29(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), + v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), + v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), + v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), + v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), v29_(v29) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_)}; + return ValuesIn(array); + } + + ValueArray29(const ValueArray29& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_), v29_(other.v29_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray29& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; +}; + +template +class ValueArray30 { + public: + ValueArray30(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_)}; + return ValuesIn(array); + } + + ValueArray30(const ValueArray30& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_), v29_(other.v29_), v30_(other.v30_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray30& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; +}; + +template +class ValueArray31 { + public: + ValueArray31(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30), v31_(v31) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_)}; + return ValuesIn(array); + } + + ValueArray31(const ValueArray31& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_), v29_(other.v29_), v30_(other.v30_), + v31_(other.v31_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray31& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; +}; + +template +class ValueArray32 { + public: + ValueArray32(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), + v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), + v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), + v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), + v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_)}; + return ValuesIn(array); + } + + ValueArray32(const ValueArray32& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_), v29_(other.v29_), v30_(other.v30_), + v31_(other.v31_), v32_(other.v32_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray32& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; +}; + +template +class ValueArray33 { + public: + ValueArray33(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, + T33 v33) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_)}; + return ValuesIn(array); + } + + ValueArray33(const ValueArray33& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_), v29_(other.v29_), v30_(other.v30_), + v31_(other.v31_), v32_(other.v32_), v33_(other.v33_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray33& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; +}; + +template +class ValueArray34 { + public: + ValueArray34(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33), v34_(v34) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_)}; + return ValuesIn(array); + } + + ValueArray34(const ValueArray34& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_), v29_(other.v29_), v30_(other.v30_), + v31_(other.v31_), v32_(other.v32_), v33_(other.v33_), v34_(other.v34_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray34& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; +}; + +template +class ValueArray35 { + public: + ValueArray35(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), + v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), + v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), + v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), + v26_(v26), v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), + v32_(v32), v33_(v33), v34_(v34), v35_(v35) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_)}; + return ValuesIn(array); + } + + ValueArray35(const ValueArray35& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_), v29_(other.v29_), v30_(other.v30_), + v31_(other.v31_), v32_(other.v32_), v33_(other.v33_), v34_(other.v34_), + v35_(other.v35_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray35& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; +}; + +template +class ValueArray36 { + public: + ValueArray36(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), + v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), + v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), + v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), + v25_(v25), v26_(v26), v27_(v27), v28_(v28), v29_(v29), v30_(v30), + v31_(v31), v32_(v32), v33_(v33), v34_(v34), v35_(v35), v36_(v36) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_)}; + return ValuesIn(array); + } + + ValueArray36(const ValueArray36& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_), v29_(other.v29_), v30_(other.v30_), + v31_(other.v31_), v32_(other.v32_), v33_(other.v33_), v34_(other.v34_), + v35_(other.v35_), v36_(other.v36_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray36& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; +}; + +template +class ValueArray37 { + public: + ValueArray37(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), + v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), + v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), + v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), + v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), v29_(v29), + v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), v35_(v35), + v36_(v36), v37_(v37) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_)}; + return ValuesIn(array); + } + + ValueArray37(const ValueArray37& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_), v29_(other.v29_), v30_(other.v30_), + v31_(other.v31_), v32_(other.v32_), v33_(other.v33_), v34_(other.v34_), + v35_(other.v35_), v36_(other.v36_), v37_(other.v37_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray37& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; +}; + +template +class ValueArray38 { + public: + ValueArray38(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), + v35_(v35), v36_(v36), v37_(v37), v38_(v38) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_)}; + return ValuesIn(array); + } + + ValueArray38(const ValueArray38& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_), v29_(other.v29_), v30_(other.v30_), + v31_(other.v31_), v32_(other.v32_), v33_(other.v33_), v34_(other.v34_), + v35_(other.v35_), v36_(other.v36_), v37_(other.v37_), v38_(other.v38_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray38& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; +}; + +template +class ValueArray39 { + public: + ValueArray39(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), + v35_(v35), v36_(v36), v37_(v37), v38_(v38), v39_(v39) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_)}; + return ValuesIn(array); + } + + ValueArray39(const ValueArray39& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_), v29_(other.v29_), v30_(other.v30_), + v31_(other.v31_), v32_(other.v32_), v33_(other.v33_), v34_(other.v34_), + v35_(other.v35_), v36_(other.v36_), v37_(other.v37_), v38_(other.v38_), + v39_(other.v39_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray39& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; +}; + +template +class ValueArray40 { + public: + ValueArray40(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), + v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), + v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), + v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), + v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), + v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), v39_(v39), + v40_(v40) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_)}; + return ValuesIn(array); + } + + ValueArray40(const ValueArray40& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_), v29_(other.v29_), v30_(other.v30_), + v31_(other.v31_), v32_(other.v32_), v33_(other.v33_), v34_(other.v34_), + v35_(other.v35_), v36_(other.v36_), v37_(other.v37_), v38_(other.v38_), + v39_(other.v39_), v40_(other.v40_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray40& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; +}; + +template +class ValueArray41 { + public: + ValueArray41(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, + T41 v41) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33), v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), + v39_(v39), v40_(v40), v41_(v41) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_)}; + return ValuesIn(array); + } + + ValueArray41(const ValueArray41& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_), v29_(other.v29_), v30_(other.v30_), + v31_(other.v31_), v32_(other.v32_), v33_(other.v33_), v34_(other.v34_), + v35_(other.v35_), v36_(other.v36_), v37_(other.v37_), v38_(other.v38_), + v39_(other.v39_), v40_(other.v40_), v41_(other.v41_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray41& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; +}; + +template +class ValueArray42 { + public: + ValueArray42(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33), v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), + v39_(v39), v40_(v40), v41_(v41), v42_(v42) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_)}; + return ValuesIn(array); + } + + ValueArray42(const ValueArray42& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_), v29_(other.v29_), v30_(other.v30_), + v31_(other.v31_), v32_(other.v32_), v33_(other.v33_), v34_(other.v34_), + v35_(other.v35_), v36_(other.v36_), v37_(other.v37_), v38_(other.v38_), + v39_(other.v39_), v40_(other.v40_), v41_(other.v41_), v42_(other.v42_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray42& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; +}; + +template +class ValueArray43 { + public: + ValueArray43(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), + v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), + v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), + v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), + v26_(v26), v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), + v32_(v32), v33_(v33), v34_(v34), v35_(v35), v36_(v36), v37_(v37), + v38_(v38), v39_(v39), v40_(v40), v41_(v41), v42_(v42), v43_(v43) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_)}; + return ValuesIn(array); + } + + ValueArray43(const ValueArray43& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_), v29_(other.v29_), v30_(other.v30_), + v31_(other.v31_), v32_(other.v32_), v33_(other.v33_), v34_(other.v34_), + v35_(other.v35_), v36_(other.v36_), v37_(other.v37_), v38_(other.v38_), + v39_(other.v39_), v40_(other.v40_), v41_(other.v41_), v42_(other.v42_), + v43_(other.v43_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray43& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; +}; + +template +class ValueArray44 { + public: + ValueArray44(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), + v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), + v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), + v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), + v25_(v25), v26_(v26), v27_(v27), v28_(v28), v29_(v29), v30_(v30), + v31_(v31), v32_(v32), v33_(v33), v34_(v34), v35_(v35), v36_(v36), + v37_(v37), v38_(v38), v39_(v39), v40_(v40), v41_(v41), v42_(v42), + v43_(v43), v44_(v44) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_), static_cast(v44_)}; + return ValuesIn(array); + } + + ValueArray44(const ValueArray44& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_), v29_(other.v29_), v30_(other.v30_), + v31_(other.v31_), v32_(other.v32_), v33_(other.v33_), v34_(other.v34_), + v35_(other.v35_), v36_(other.v36_), v37_(other.v37_), v38_(other.v38_), + v39_(other.v39_), v40_(other.v40_), v41_(other.v41_), v42_(other.v42_), + v43_(other.v43_), v44_(other.v44_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray44& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; +}; + +template +class ValueArray45 { + public: + ValueArray45(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), + v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), + v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), + v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), + v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), v29_(v29), + v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), v35_(v35), + v36_(v36), v37_(v37), v38_(v38), v39_(v39), v40_(v40), v41_(v41), + v42_(v42), v43_(v43), v44_(v44), v45_(v45) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_), static_cast(v44_), + static_cast(v45_)}; + return ValuesIn(array); + } + + ValueArray45(const ValueArray45& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_), v29_(other.v29_), v30_(other.v30_), + v31_(other.v31_), v32_(other.v32_), v33_(other.v33_), v34_(other.v34_), + v35_(other.v35_), v36_(other.v36_), v37_(other.v37_), v38_(other.v38_), + v39_(other.v39_), v40_(other.v40_), v41_(other.v41_), v42_(other.v42_), + v43_(other.v43_), v44_(other.v44_), v45_(other.v45_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray45& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; +}; + +template +class ValueArray46 { + public: + ValueArray46(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45, T46 v46) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), + v35_(v35), v36_(v36), v37_(v37), v38_(v38), v39_(v39), v40_(v40), + v41_(v41), v42_(v42), v43_(v43), v44_(v44), v45_(v45), v46_(v46) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_), static_cast(v44_), + static_cast(v45_), static_cast(v46_)}; + return ValuesIn(array); + } + + ValueArray46(const ValueArray46& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_), v29_(other.v29_), v30_(other.v30_), + v31_(other.v31_), v32_(other.v32_), v33_(other.v33_), v34_(other.v34_), + v35_(other.v35_), v36_(other.v36_), v37_(other.v37_), v38_(other.v38_), + v39_(other.v39_), v40_(other.v40_), v41_(other.v41_), v42_(other.v42_), + v43_(other.v43_), v44_(other.v44_), v45_(other.v45_), v46_(other.v46_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray46& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; + const T46 v46_; +}; + +template +class ValueArray47 { + public: + ValueArray47(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), + v35_(v35), v36_(v36), v37_(v37), v38_(v38), v39_(v39), v40_(v40), + v41_(v41), v42_(v42), v43_(v43), v44_(v44), v45_(v45), v46_(v46), + v47_(v47) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_), static_cast(v44_), + static_cast(v45_), static_cast(v46_), static_cast(v47_)}; + return ValuesIn(array); + } + + ValueArray47(const ValueArray47& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_), v29_(other.v29_), v30_(other.v30_), + v31_(other.v31_), v32_(other.v32_), v33_(other.v33_), v34_(other.v34_), + v35_(other.v35_), v36_(other.v36_), v37_(other.v37_), v38_(other.v38_), + v39_(other.v39_), v40_(other.v40_), v41_(other.v41_), v42_(other.v42_), + v43_(other.v43_), v44_(other.v44_), v45_(other.v45_), v46_(other.v46_), + v47_(other.v47_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray47& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; + const T46 v46_; + const T47 v47_; +}; + +template +class ValueArray48 { + public: + ValueArray48(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47, T48 v48) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), + v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), + v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), + v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), + v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), + v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), v39_(v39), + v40_(v40), v41_(v41), v42_(v42), v43_(v43), v44_(v44), v45_(v45), + v46_(v46), v47_(v47), v48_(v48) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_), static_cast(v44_), + static_cast(v45_), static_cast(v46_), static_cast(v47_), + static_cast(v48_)}; + return ValuesIn(array); + } + + ValueArray48(const ValueArray48& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_), v29_(other.v29_), v30_(other.v30_), + v31_(other.v31_), v32_(other.v32_), v33_(other.v33_), v34_(other.v34_), + v35_(other.v35_), v36_(other.v36_), v37_(other.v37_), v38_(other.v38_), + v39_(other.v39_), v40_(other.v40_), v41_(other.v41_), v42_(other.v42_), + v43_(other.v43_), v44_(other.v44_), v45_(other.v45_), v46_(other.v46_), + v47_(other.v47_), v48_(other.v48_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray48& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; + const T46 v46_; + const T47 v47_; + const T48 v48_; +}; + +template +class ValueArray49 { + public: + ValueArray49(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47, T48 v48, + T49 v49) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33), v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), + v39_(v39), v40_(v40), v41_(v41), v42_(v42), v43_(v43), v44_(v44), + v45_(v45), v46_(v46), v47_(v47), v48_(v48), v49_(v49) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_), static_cast(v44_), + static_cast(v45_), static_cast(v46_), static_cast(v47_), + static_cast(v48_), static_cast(v49_)}; + return ValuesIn(array); + } + + ValueArray49(const ValueArray49& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_), v29_(other.v29_), v30_(other.v30_), + v31_(other.v31_), v32_(other.v32_), v33_(other.v33_), v34_(other.v34_), + v35_(other.v35_), v36_(other.v36_), v37_(other.v37_), v38_(other.v38_), + v39_(other.v39_), v40_(other.v40_), v41_(other.v41_), v42_(other.v42_), + v43_(other.v43_), v44_(other.v44_), v45_(other.v45_), v46_(other.v46_), + v47_(other.v47_), v48_(other.v48_), v49_(other.v49_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray49& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; + const T46 v46_; + const T47 v47_; + const T48 v48_; + const T49 v49_; +}; + +template +class ValueArray50 { + public: + ValueArray50(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47, T48 v48, T49 v49, + T50 v50) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33), v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), + v39_(v39), v40_(v40), v41_(v41), v42_(v42), v43_(v43), v44_(v44), + v45_(v45), v46_(v46), v47_(v47), v48_(v48), v49_(v49), v50_(v50) {} + + template + operator ParamGenerator() const { + const T array[] = {static_cast(v1_), static_cast(v2_), + static_cast(v3_), static_cast(v4_), static_cast(v5_), + static_cast(v6_), static_cast(v7_), static_cast(v8_), + static_cast(v9_), static_cast(v10_), static_cast(v11_), + static_cast(v12_), static_cast(v13_), static_cast(v14_), + static_cast(v15_), static_cast(v16_), static_cast(v17_), + static_cast(v18_), static_cast(v19_), static_cast(v20_), + static_cast(v21_), static_cast(v22_), static_cast(v23_), + static_cast(v24_), static_cast(v25_), static_cast(v26_), + static_cast(v27_), static_cast(v28_), static_cast(v29_), + static_cast(v30_), static_cast(v31_), static_cast(v32_), + static_cast(v33_), static_cast(v34_), static_cast(v35_), + static_cast(v36_), static_cast(v37_), static_cast(v38_), + static_cast(v39_), static_cast(v40_), static_cast(v41_), + static_cast(v42_), static_cast(v43_), static_cast(v44_), + static_cast(v45_), static_cast(v46_), static_cast(v47_), + static_cast(v48_), static_cast(v49_), static_cast(v50_)}; + return ValuesIn(array); + } + + ValueArray50(const ValueArray50& other) : v1_(other.v1_), v2_(other.v2_), + v3_(other.v3_), v4_(other.v4_), v5_(other.v5_), v6_(other.v6_), + v7_(other.v7_), v8_(other.v8_), v9_(other.v9_), v10_(other.v10_), + v11_(other.v11_), v12_(other.v12_), v13_(other.v13_), v14_(other.v14_), + v15_(other.v15_), v16_(other.v16_), v17_(other.v17_), v18_(other.v18_), + v19_(other.v19_), v20_(other.v20_), v21_(other.v21_), v22_(other.v22_), + v23_(other.v23_), v24_(other.v24_), v25_(other.v25_), v26_(other.v26_), + v27_(other.v27_), v28_(other.v28_), v29_(other.v29_), v30_(other.v30_), + v31_(other.v31_), v32_(other.v32_), v33_(other.v33_), v34_(other.v34_), + v35_(other.v35_), v36_(other.v36_), v37_(other.v37_), v38_(other.v38_), + v39_(other.v39_), v40_(other.v40_), v41_(other.v41_), v42_(other.v42_), + v43_(other.v43_), v44_(other.v44_), v45_(other.v45_), v46_(other.v46_), + v47_(other.v47_), v48_(other.v48_), v49_(other.v49_), v50_(other.v50_) {} + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray50& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; + const T46 v46_; + const T47 v47_; + const T48 v48_; + const T49 v49_; + const T50 v50_; +}; + +# if GTEST_HAS_COMBINE +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Generates values from the Cartesian product of values produced +// by the argument generators. +// +template +class CartesianProductGenerator2 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator2(const ParamGenerator& g1, + const ParamGenerator& g2) + : g1_(g1), g2_(g2) {} + virtual ~CartesianProductGenerator2() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current2_; + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return current_value_.get(); } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_.reset(new ParamType(*current1_, *current2_)); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + linked_ptr current_value_; + }; // class CartesianProductGenerator2::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator2& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; +}; // class CartesianProductGenerator2 + + +template +class CartesianProductGenerator3 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator3(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3) + : g1_(g1), g2_(g2), g3_(g3) {} + virtual ~CartesianProductGenerator3() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current3_; + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return current_value_.get(); } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_.reset(new ParamType(*current1_, *current2_, *current3_)); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + linked_ptr current_value_; + }; // class CartesianProductGenerator3::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator3& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; +}; // class CartesianProductGenerator3 + + +template +class CartesianProductGenerator4 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator4(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4) {} + virtual ~CartesianProductGenerator4() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current4_; + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return current_value_.get(); } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_.reset(new ParamType(*current1_, *current2_, *current3_, + *current4_)); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + linked_ptr current_value_; + }; // class CartesianProductGenerator4::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator4& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; +}; // class CartesianProductGenerator4 + + +template +class CartesianProductGenerator5 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator5(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5) {} + virtual ~CartesianProductGenerator5() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current5_; + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return current_value_.get(); } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_.reset(new ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_)); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + linked_ptr current_value_; + }; // class CartesianProductGenerator5::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator5& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; +}; // class CartesianProductGenerator5 + + +template +class CartesianProductGenerator6 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator6(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5, + const ParamGenerator& g6) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6) {} + virtual ~CartesianProductGenerator6() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin(), g6_, g6_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end(), g6_, g6_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5, + const ParamGenerator& g6, + const typename ParamGenerator::iterator& current6) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5), + begin6_(g6.begin()), end6_(g6.end()), current6_(current6) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current6_; + if (current6_ == end6_) { + current6_ = begin6_; + ++current5_; + } + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return current_value_.get(); } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_ && + current6_ == typed_other->current6_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_), + begin6_(other.begin6_), + end6_(other.end6_), + current6_(other.current6_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_.reset(new ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_, *current6_)); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_ || + current6_ == end6_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + const typename ParamGenerator::iterator begin6_; + const typename ParamGenerator::iterator end6_; + typename ParamGenerator::iterator current6_; + linked_ptr current_value_; + }; // class CartesianProductGenerator6::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator6& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; + const ParamGenerator g6_; +}; // class CartesianProductGenerator6 + + +template +class CartesianProductGenerator7 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator7(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5, + const ParamGenerator& g6, const ParamGenerator& g7) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7) {} + virtual ~CartesianProductGenerator7() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin(), g6_, g6_.begin(), g7_, + g7_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end(), g6_, g6_.end(), g7_, g7_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5, + const ParamGenerator& g6, + const typename ParamGenerator::iterator& current6, + const ParamGenerator& g7, + const typename ParamGenerator::iterator& current7) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5), + begin6_(g6.begin()), end6_(g6.end()), current6_(current6), + begin7_(g7.begin()), end7_(g7.end()), current7_(current7) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current7_; + if (current7_ == end7_) { + current7_ = begin7_; + ++current6_; + } + if (current6_ == end6_) { + current6_ = begin6_; + ++current5_; + } + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return current_value_.get(); } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_ && + current6_ == typed_other->current6_ && + current7_ == typed_other->current7_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_), + begin6_(other.begin6_), + end6_(other.end6_), + current6_(other.current6_), + begin7_(other.begin7_), + end7_(other.end7_), + current7_(other.current7_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_.reset(new ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_, *current6_, *current7_)); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_ || + current6_ == end6_ || + current7_ == end7_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + const typename ParamGenerator::iterator begin6_; + const typename ParamGenerator::iterator end6_; + typename ParamGenerator::iterator current6_; + const typename ParamGenerator::iterator begin7_; + const typename ParamGenerator::iterator end7_; + typename ParamGenerator::iterator current7_; + linked_ptr current_value_; + }; // class CartesianProductGenerator7::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator7& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; + const ParamGenerator g6_; + const ParamGenerator g7_; +}; // class CartesianProductGenerator7 + + +template +class CartesianProductGenerator8 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator8(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5, + const ParamGenerator& g6, const ParamGenerator& g7, + const ParamGenerator& g8) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), + g8_(g8) {} + virtual ~CartesianProductGenerator8() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin(), g6_, g6_.begin(), g7_, + g7_.begin(), g8_, g8_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end(), g6_, g6_.end(), g7_, g7_.end(), g8_, + g8_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5, + const ParamGenerator& g6, + const typename ParamGenerator::iterator& current6, + const ParamGenerator& g7, + const typename ParamGenerator::iterator& current7, + const ParamGenerator& g8, + const typename ParamGenerator::iterator& current8) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5), + begin6_(g6.begin()), end6_(g6.end()), current6_(current6), + begin7_(g7.begin()), end7_(g7.end()), current7_(current7), + begin8_(g8.begin()), end8_(g8.end()), current8_(current8) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current8_; + if (current8_ == end8_) { + current8_ = begin8_; + ++current7_; + } + if (current7_ == end7_) { + current7_ = begin7_; + ++current6_; + } + if (current6_ == end6_) { + current6_ = begin6_; + ++current5_; + } + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return current_value_.get(); } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_ && + current6_ == typed_other->current6_ && + current7_ == typed_other->current7_ && + current8_ == typed_other->current8_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_), + begin6_(other.begin6_), + end6_(other.end6_), + current6_(other.current6_), + begin7_(other.begin7_), + end7_(other.end7_), + current7_(other.current7_), + begin8_(other.begin8_), + end8_(other.end8_), + current8_(other.current8_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_.reset(new ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_, *current6_, *current7_, *current8_)); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_ || + current6_ == end6_ || + current7_ == end7_ || + current8_ == end8_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + const typename ParamGenerator::iterator begin6_; + const typename ParamGenerator::iterator end6_; + typename ParamGenerator::iterator current6_; + const typename ParamGenerator::iterator begin7_; + const typename ParamGenerator::iterator end7_; + typename ParamGenerator::iterator current7_; + const typename ParamGenerator::iterator begin8_; + const typename ParamGenerator::iterator end8_; + typename ParamGenerator::iterator current8_; + linked_ptr current_value_; + }; // class CartesianProductGenerator8::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator8& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; + const ParamGenerator g6_; + const ParamGenerator g7_; + const ParamGenerator g8_; +}; // class CartesianProductGenerator8 + + +template +class CartesianProductGenerator9 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator9(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5, + const ParamGenerator& g6, const ParamGenerator& g7, + const ParamGenerator& g8, const ParamGenerator& g9) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), g8_(g8), + g9_(g9) {} + virtual ~CartesianProductGenerator9() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin(), g6_, g6_.begin(), g7_, + g7_.begin(), g8_, g8_.begin(), g9_, g9_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end(), g6_, g6_.end(), g7_, g7_.end(), g8_, + g8_.end(), g9_, g9_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5, + const ParamGenerator& g6, + const typename ParamGenerator::iterator& current6, + const ParamGenerator& g7, + const typename ParamGenerator::iterator& current7, + const ParamGenerator& g8, + const typename ParamGenerator::iterator& current8, + const ParamGenerator& g9, + const typename ParamGenerator::iterator& current9) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5), + begin6_(g6.begin()), end6_(g6.end()), current6_(current6), + begin7_(g7.begin()), end7_(g7.end()), current7_(current7), + begin8_(g8.begin()), end8_(g8.end()), current8_(current8), + begin9_(g9.begin()), end9_(g9.end()), current9_(current9) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current9_; + if (current9_ == end9_) { + current9_ = begin9_; + ++current8_; + } + if (current8_ == end8_) { + current8_ = begin8_; + ++current7_; + } + if (current7_ == end7_) { + current7_ = begin7_; + ++current6_; + } + if (current6_ == end6_) { + current6_ = begin6_; + ++current5_; + } + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return current_value_.get(); } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_ && + current6_ == typed_other->current6_ && + current7_ == typed_other->current7_ && + current8_ == typed_other->current8_ && + current9_ == typed_other->current9_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_), + begin6_(other.begin6_), + end6_(other.end6_), + current6_(other.current6_), + begin7_(other.begin7_), + end7_(other.end7_), + current7_(other.current7_), + begin8_(other.begin8_), + end8_(other.end8_), + current8_(other.current8_), + begin9_(other.begin9_), + end9_(other.end9_), + current9_(other.current9_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_.reset(new ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_, *current6_, *current7_, *current8_, + *current9_)); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_ || + current6_ == end6_ || + current7_ == end7_ || + current8_ == end8_ || + current9_ == end9_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + const typename ParamGenerator::iterator begin6_; + const typename ParamGenerator::iterator end6_; + typename ParamGenerator::iterator current6_; + const typename ParamGenerator::iterator begin7_; + const typename ParamGenerator::iterator end7_; + typename ParamGenerator::iterator current7_; + const typename ParamGenerator::iterator begin8_; + const typename ParamGenerator::iterator end8_; + typename ParamGenerator::iterator current8_; + const typename ParamGenerator::iterator begin9_; + const typename ParamGenerator::iterator end9_; + typename ParamGenerator::iterator current9_; + linked_ptr current_value_; + }; // class CartesianProductGenerator9::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator9& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; + const ParamGenerator g6_; + const ParamGenerator g7_; + const ParamGenerator g8_; + const ParamGenerator g9_; +}; // class CartesianProductGenerator9 + + +template +class CartesianProductGenerator10 + : public ParamGeneratorInterface< ::testing::tuple > { + public: + typedef ::testing::tuple ParamType; + + CartesianProductGenerator10(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5, + const ParamGenerator& g6, const ParamGenerator& g7, + const ParamGenerator& g8, const ParamGenerator& g9, + const ParamGenerator& g10) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), g8_(g8), + g9_(g9), g10_(g10) {} + virtual ~CartesianProductGenerator10() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin(), g6_, g6_.begin(), g7_, + g7_.begin(), g8_, g8_.begin(), g9_, g9_.begin(), g10_, g10_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end(), g6_, g6_.end(), g7_, g7_.end(), g8_, + g8_.end(), g9_, g9_.end(), g10_, g10_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5, + const ParamGenerator& g6, + const typename ParamGenerator::iterator& current6, + const ParamGenerator& g7, + const typename ParamGenerator::iterator& current7, + const ParamGenerator& g8, + const typename ParamGenerator::iterator& current8, + const ParamGenerator& g9, + const typename ParamGenerator::iterator& current9, + const ParamGenerator& g10, + const typename ParamGenerator::iterator& current10) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5), + begin6_(g6.begin()), end6_(g6.end()), current6_(current6), + begin7_(g7.begin()), end7_(g7.end()), current7_(current7), + begin8_(g8.begin()), end8_(g8.end()), current8_(current8), + begin9_(g9.begin()), end9_(g9.end()), current9_(current9), + begin10_(g10.begin()), end10_(g10.end()), current10_(current10) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current10_; + if (current10_ == end10_) { + current10_ = begin10_; + ++current9_; + } + if (current9_ == end9_) { + current9_ = begin9_; + ++current8_; + } + if (current8_ == end8_) { + current8_ = begin8_; + ++current7_; + } + if (current7_ == end7_) { + current7_ = begin7_; + ++current6_; + } + if (current6_ == end6_) { + current6_ = begin6_; + ++current5_; + } + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return current_value_.get(); } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_ && + current6_ == typed_other->current6_ && + current7_ == typed_other->current7_ && + current8_ == typed_other->current8_ && + current9_ == typed_other->current9_ && + current10_ == typed_other->current10_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_), + begin6_(other.begin6_), + end6_(other.end6_), + current6_(other.current6_), + begin7_(other.begin7_), + end7_(other.end7_), + current7_(other.current7_), + begin8_(other.begin8_), + end8_(other.end8_), + current8_(other.current8_), + begin9_(other.begin9_), + end9_(other.end9_), + current9_(other.current9_), + begin10_(other.begin10_), + end10_(other.end10_), + current10_(other.current10_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_.reset(new ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_, *current6_, *current7_, *current8_, + *current9_, *current10_)); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_ || + current6_ == end6_ || + current7_ == end7_ || + current8_ == end8_ || + current9_ == end9_ || + current10_ == end10_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + const typename ParamGenerator::iterator begin6_; + const typename ParamGenerator::iterator end6_; + typename ParamGenerator::iterator current6_; + const typename ParamGenerator::iterator begin7_; + const typename ParamGenerator::iterator end7_; + typename ParamGenerator::iterator current7_; + const typename ParamGenerator::iterator begin8_; + const typename ParamGenerator::iterator end8_; + typename ParamGenerator::iterator current8_; + const typename ParamGenerator::iterator begin9_; + const typename ParamGenerator::iterator end9_; + typename ParamGenerator::iterator current9_; + const typename ParamGenerator::iterator begin10_; + const typename ParamGenerator::iterator end10_; + typename ParamGenerator::iterator current10_; + linked_ptr current_value_; + }; // class CartesianProductGenerator10::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator10& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; + const ParamGenerator g6_; + const ParamGenerator g7_; + const ParamGenerator g8_; + const ParamGenerator g9_; + const ParamGenerator g10_; +}; // class CartesianProductGenerator10 + + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Helper classes providing Combine() with polymorphic features. They allow +// casting CartesianProductGeneratorN to ParamGenerator if T is +// convertible to U. +// +template +class CartesianProductHolder2 { + public: +CartesianProductHolder2(const Generator1& g1, const Generator2& g2) + : g1_(g1), g2_(g2) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator2( + static_cast >(g1_), + static_cast >(g2_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder2& other); + + const Generator1 g1_; + const Generator2 g2_; +}; // class CartesianProductHolder2 + +template +class CartesianProductHolder3 { + public: +CartesianProductHolder3(const Generator1& g1, const Generator2& g2, + const Generator3& g3) + : g1_(g1), g2_(g2), g3_(g3) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator3( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder3& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; +}; // class CartesianProductHolder3 + +template +class CartesianProductHolder4 { + public: +CartesianProductHolder4(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator4( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder4& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; +}; // class CartesianProductHolder4 + +template +class CartesianProductHolder5 { + public: +CartesianProductHolder5(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator5( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder5& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; +}; // class CartesianProductHolder5 + +template +class CartesianProductHolder6 { + public: +CartesianProductHolder6(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5, + const Generator6& g6) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator6( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_), + static_cast >(g6_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder6& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; + const Generator6 g6_; +}; // class CartesianProductHolder6 + +template +class CartesianProductHolder7 { + public: +CartesianProductHolder7(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5, + const Generator6& g6, const Generator7& g7) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator7( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_), + static_cast >(g6_), + static_cast >(g7_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder7& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; + const Generator6 g6_; + const Generator7 g7_; +}; // class CartesianProductHolder7 + +template +class CartesianProductHolder8 { + public: +CartesianProductHolder8(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5, + const Generator6& g6, const Generator7& g7, const Generator8& g8) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), + g8_(g8) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator8( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_), + static_cast >(g6_), + static_cast >(g7_), + static_cast >(g8_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder8& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; + const Generator6 g6_; + const Generator7 g7_; + const Generator8 g8_; +}; // class CartesianProductHolder8 + +template +class CartesianProductHolder9 { + public: +CartesianProductHolder9(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5, + const Generator6& g6, const Generator7& g7, const Generator8& g8, + const Generator9& g9) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), g8_(g8), + g9_(g9) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator9( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_), + static_cast >(g6_), + static_cast >(g7_), + static_cast >(g8_), + static_cast >(g9_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder9& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; + const Generator6 g6_; + const Generator7 g7_; + const Generator8 g8_; + const Generator9 g9_; +}; // class CartesianProductHolder9 + +template +class CartesianProductHolder10 { + public: +CartesianProductHolder10(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5, + const Generator6& g6, const Generator7& g7, const Generator8& g8, + const Generator9& g9, const Generator10& g10) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), g8_(g8), + g9_(g9), g10_(g10) {} + template + operator ParamGenerator< ::testing::tuple >() const { + return ParamGenerator< ::testing::tuple >( + new CartesianProductGenerator10( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_), + static_cast >(g6_), + static_cast >(g7_), + static_cast >(g8_), + static_cast >(g9_), + static_cast >(g10_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder10& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; + const Generator6 g6_; + const Generator7 g7_; + const Generator8 g8_; + const Generator9 g9_; + const Generator10 g10_; +}; // class CartesianProductHolder10 + +# endif // GTEST_HAS_COMBINE + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_GENERATED_H_ + +namespace testing { + +// Functions producing parameter generators. +// +// Google Test uses these generators to produce parameters for value- +// parameterized tests. When a parameterized test case is instantiated +// with a particular generator, Google Test creates and runs tests +// for each element in the sequence produced by the generator. +// +// In the following sample, tests from test case FooTest are instantiated +// each three times with parameter values 3, 5, and 8: +// +// class FooTest : public TestWithParam { ... }; +// +// TEST_P(FooTest, TestThis) { +// } +// TEST_P(FooTest, TestThat) { +// } +// INSTANTIATE_TEST_CASE_P(TestSequence, FooTest, Values(3, 5, 8)); +// + +// Range() returns generators providing sequences of values in a range. +// +// Synopsis: +// Range(start, end) +// - returns a generator producing a sequence of values {start, start+1, +// start+2, ..., }. +// Range(start, end, step) +// - returns a generator producing a sequence of values {start, start+step, +// start+step+step, ..., }. +// Notes: +// * The generated sequences never include end. For example, Range(1, 5) +// returns a generator producing a sequence {1, 2, 3, 4}. Range(1, 9, 2) +// returns a generator producing {1, 3, 5, 7}. +// * start and end must have the same type. That type may be any integral or +// floating-point type or a user defined type satisfying these conditions: +// * It must be assignable (have operator=() defined). +// * It must have operator+() (operator+(int-compatible type) for +// two-operand version). +// * It must have operator<() defined. +// Elements in the resulting sequences will also have that type. +// * Condition start < end must be satisfied in order for resulting sequences +// to contain any elements. +// +template +internal::ParamGenerator Range(T start, T end, IncrementT step) { + return internal::ParamGenerator( + new internal::RangeGenerator(start, end, step)); +} + +template +internal::ParamGenerator Range(T start, T end) { + return Range(start, end, 1); +} + +// ValuesIn() function allows generation of tests with parameters coming from +// a container. +// +// Synopsis: +// ValuesIn(const T (&array)[N]) +// - returns a generator producing sequences with elements from +// a C-style array. +// ValuesIn(const Container& container) +// - returns a generator producing sequences with elements from +// an STL-style container. +// ValuesIn(Iterator begin, Iterator end) +// - returns a generator producing sequences with elements from +// a range [begin, end) defined by a pair of STL-style iterators. These +// iterators can also be plain C pointers. +// +// Please note that ValuesIn copies the values from the containers +// passed in and keeps them to generate tests in RUN_ALL_TESTS(). +// +// Examples: +// +// This instantiates tests from test case StringTest +// each with C-string values of "foo", "bar", and "baz": +// +// const char* strings[] = {"foo", "bar", "baz"}; +// INSTANTIATE_TEST_CASE_P(StringSequence, StringTest, ValuesIn(strings)); +// +// This instantiates tests from test case StlStringTest +// each with STL strings with values "a" and "b": +// +// ::std::vector< ::std::string> GetParameterStrings() { +// ::std::vector< ::std::string> v; +// v.push_back("a"); +// v.push_back("b"); +// return v; +// } +// +// INSTANTIATE_TEST_CASE_P(CharSequence, +// StlStringTest, +// ValuesIn(GetParameterStrings())); +// +// +// This will also instantiate tests from CharTest +// each with parameter values 'a' and 'b': +// +// ::std::list GetParameterChars() { +// ::std::list list; +// list.push_back('a'); +// list.push_back('b'); +// return list; +// } +// ::std::list l = GetParameterChars(); +// INSTANTIATE_TEST_CASE_P(CharSequence2, +// CharTest, +// ValuesIn(l.begin(), l.end())); +// +template +internal::ParamGenerator< + typename ::testing::internal::IteratorTraits::value_type> +ValuesIn(ForwardIterator begin, ForwardIterator end) { + typedef typename ::testing::internal::IteratorTraits + ::value_type ParamType; + return internal::ParamGenerator( + new internal::ValuesInIteratorRangeGenerator(begin, end)); +} + +template +internal::ParamGenerator ValuesIn(const T (&array)[N]) { + return ValuesIn(array, array + N); +} + +template +internal::ParamGenerator ValuesIn( + const Container& container) { + return ValuesIn(container.begin(), container.end()); +} + +// Values() allows generating tests from explicitly specified list of +// parameters. +// +// Synopsis: +// Values(T v1, T v2, ..., T vN) +// - returns a generator producing sequences with elements v1, v2, ..., vN. +// +// For example, this instantiates tests from test case BarTest each +// with values "one", "two", and "three": +// +// INSTANTIATE_TEST_CASE_P(NumSequence, BarTest, Values("one", "two", "three")); +// +// This instantiates tests from test case BazTest each with values 1, 2, 3.5. +// The exact type of values will depend on the type of parameter in BazTest. +// +// INSTANTIATE_TEST_CASE_P(FloatingNumbers, BazTest, Values(1, 2, 3.5)); +// +// Currently, Values() supports from 1 to 50 parameters. +// +template +internal::ValueArray1 Values(T1 v1) { + return internal::ValueArray1(v1); +} + +template +internal::ValueArray2 Values(T1 v1, T2 v2) { + return internal::ValueArray2(v1, v2); +} + +template +internal::ValueArray3 Values(T1 v1, T2 v2, T3 v3) { + return internal::ValueArray3(v1, v2, v3); +} + +template +internal::ValueArray4 Values(T1 v1, T2 v2, T3 v3, T4 v4) { + return internal::ValueArray4(v1, v2, v3, v4); +} + +template +internal::ValueArray5 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5) { + return internal::ValueArray5(v1, v2, v3, v4, v5); +} + +template +internal::ValueArray6 Values(T1 v1, T2 v2, T3 v3, + T4 v4, T5 v5, T6 v6) { + return internal::ValueArray6(v1, v2, v3, v4, v5, v6); +} + +template +internal::ValueArray7 Values(T1 v1, T2 v2, T3 v3, + T4 v4, T5 v5, T6 v6, T7 v7) { + return internal::ValueArray7(v1, v2, v3, v4, v5, + v6, v7); +} + +template +internal::ValueArray8 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8) { + return internal::ValueArray8(v1, v2, v3, v4, + v5, v6, v7, v8); +} + +template +internal::ValueArray9 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9) { + return internal::ValueArray9(v1, v2, v3, + v4, v5, v6, v7, v8, v9); +} + +template +internal::ValueArray10 Values(T1 v1, + T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10) { + return internal::ValueArray10(v1, + v2, v3, v4, v5, v6, v7, v8, v9, v10); +} + +template +internal::ValueArray11 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11) { + return internal::ValueArray11(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11); +} + +template +internal::ValueArray12 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12) { + return internal::ValueArray12(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12); +} + +template +internal::ValueArray13 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13) { + return internal::ValueArray13(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13); +} + +template +internal::ValueArray14 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14) { + return internal::ValueArray14(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, + v14); +} + +template +internal::ValueArray15 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, + T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15) { + return internal::ValueArray15(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15); +} + +template +internal::ValueArray16 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16) { + return internal::ValueArray16(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, + v12, v13, v14, v15, v16); +} + +template +internal::ValueArray17 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17) { + return internal::ValueArray17(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, + v11, v12, v13, v14, v15, v16, v17); +} + +template +internal::ValueArray18 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, + T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18) { + return internal::ValueArray18(v1, v2, v3, v4, v5, v6, v7, v8, v9, + v10, v11, v12, v13, v14, v15, v16, v17, v18); +} + +template +internal::ValueArray19 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, + T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, + T15 v15, T16 v16, T17 v17, T18 v18, T19 v19) { + return internal::ValueArray19(v1, v2, v3, v4, v5, v6, v7, v8, + v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19); +} + +template +internal::ValueArray20 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, + T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20) { + return internal::ValueArray20(v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20); +} + +template +internal::ValueArray21 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, + T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21) { + return internal::ValueArray21(v1, v2, v3, v4, v5, v6, + v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21); +} + +template +internal::ValueArray22 Values(T1 v1, T2 v2, T3 v3, + T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22) { + return internal::ValueArray22(v1, v2, v3, v4, + v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, + v20, v21, v22); +} + +template +internal::ValueArray23 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22, T23 v23) { + return internal::ValueArray23(v1, v2, v3, + v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, + v20, v21, v22, v23); +} + +template +internal::ValueArray24 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22, T23 v23, T24 v24) { + return internal::ValueArray24(v1, v2, + v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, + v19, v20, v21, v22, v23, v24); +} + +template +internal::ValueArray25 Values(T1 v1, + T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, + T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, + T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25) { + return internal::ValueArray25(v1, + v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, + v18, v19, v20, v21, v22, v23, v24, v25); +} + +template +internal::ValueArray26 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26) { + return internal::ValueArray26(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, + v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26); +} + +template +internal::ValueArray27 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27) { + return internal::ValueArray27(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, + v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27); +} + +template +internal::ValueArray28 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28) { + return internal::ValueArray28(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, + v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, + v28); +} + +template +internal::ValueArray29 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29) { + return internal::ValueArray29(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, + v27, v28, v29); +} + +template +internal::ValueArray30 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, + T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, + T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, + T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30) { + return internal::ValueArray30(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, + v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, + v26, v27, v28, v29, v30); +} + +template +internal::ValueArray31 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31) { + return internal::ValueArray31(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, + v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, + v25, v26, v27, v28, v29, v30, v31); +} + +template +internal::ValueArray32 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, + T32 v32) { + return internal::ValueArray32(v1, v2, v3, v4, v5, v6, v7, v8, v9, + v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31, v32); +} + +template +internal::ValueArray33 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, + T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, + T32 v32, T33 v33) { + return internal::ValueArray33(v1, v2, v3, v4, v5, v6, v7, v8, + v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31, v32, v33); +} + +template +internal::ValueArray34 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, + T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, + T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, + T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, + T31 v31, T32 v32, T33 v33, T34 v34) { + return internal::ValueArray34(v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, + v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34); +} + +template +internal::ValueArray35 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, + T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, + T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, + T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35) { + return internal::ValueArray35(v1, v2, v3, v4, v5, v6, + v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, + v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35); +} + +template +internal::ValueArray36 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, + T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, + T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, + T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36) { + return internal::ValueArray36(v1, v2, v3, v4, + v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, + v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, + v34, v35, v36); +} + +template +internal::ValueArray37 Values(T1 v1, T2 v2, T3 v3, + T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, + T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, + T37 v37) { + return internal::ValueArray37(v1, v2, v3, + v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, + v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, + v34, v35, v36, v37); +} + +template +internal::ValueArray38 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, + T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, + T37 v37, T38 v38) { + return internal::ValueArray38(v1, v2, + v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, + v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, + v33, v34, v35, v36, v37, v38); +} + +template +internal::ValueArray39 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, + T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, + T37 v37, T38 v38, T39 v39) { + return internal::ValueArray39(v1, + v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, + v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, + v32, v33, v34, v35, v36, v37, v38, v39); +} + +template +internal::ValueArray40 Values(T1 v1, + T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, + T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, + T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, + T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, + T36 v36, T37 v37, T38 v38, T39 v39, T40 v40) { + return internal::ValueArray40(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, + v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, + v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40); +} + +template +internal::ValueArray41 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41) { + return internal::ValueArray41(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, + v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, + v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41); +} + +template +internal::ValueArray42 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42) { + return internal::ValueArray42(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, + v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, + v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, + v42); +} + +template +internal::ValueArray43 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43) { + return internal::ValueArray43(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, + v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, + v41, v42, v43); +} + +template +internal::ValueArray44 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44) { + return internal::ValueArray44(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, + v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, + v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, + v40, v41, v42, v43, v44); +} + +template +internal::ValueArray45 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, + T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, + T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, + T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, + T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, + T41 v41, T42 v42, T43 v43, T44 v44, T45 v45) { + return internal::ValueArray45(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, + v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, + v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, + v39, v40, v41, v42, v43, v44, v45); +} + +template +internal::ValueArray46 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, + T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, + T40 v40, T41 v41, T42 v42, T43 v43, T44 v44, T45 v45, T46 v46) { + return internal::ValueArray46(v1, v2, v3, v4, v5, v6, v7, v8, v9, + v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, + v38, v39, v40, v41, v42, v43, v44, v45, v46); +} + +template +internal::ValueArray47 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, + T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, + T40 v40, T41 v41, T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47) { + return internal::ValueArray47(v1, v2, v3, v4, v5, v6, v7, v8, + v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, + v38, v39, v40, v41, v42, v43, v44, v45, v46, v47); +} + +template +internal::ValueArray48 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, + T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, + T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, + T40 v40, T41 v41, T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47, + T48 v48) { + return internal::ValueArray48(v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, + v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, + v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48); +} + +template +internal::ValueArray49 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, + T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, + T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, + T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, + T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, + T39 v39, T40 v40, T41 v41, T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, + T47 v47, T48 v48, T49 v49) { + return internal::ValueArray49(v1, v2, v3, v4, v5, v6, + v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, + v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, + v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49); +} + +template +internal::ValueArray50 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, + T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, + T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, + T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, + T38 v38, T39 v39, T40 v40, T41 v41, T42 v42, T43 v43, T44 v44, T45 v45, + T46 v46, T47 v47, T48 v48, T49 v49, T50 v50) { + return internal::ValueArray50(v1, v2, v3, v4, + v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, + v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, + v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, + v48, v49, v50); +} + +// Bool() allows generating tests with parameters in a set of (false, true). +// +// Synopsis: +// Bool() +// - returns a generator producing sequences with elements {false, true}. +// +// It is useful when testing code that depends on Boolean flags. Combinations +// of multiple flags can be tested when several Bool()'s are combined using +// Combine() function. +// +// In the following example all tests in the test case FlagDependentTest +// will be instantiated twice with parameters false and true. +// +// class FlagDependentTest : public testing::TestWithParam { +// virtual void SetUp() { +// external_flag = GetParam(); +// } +// } +// INSTANTIATE_TEST_CASE_P(BoolSequence, FlagDependentTest, Bool()); +// +inline internal::ParamGenerator Bool() { + return Values(false, true); +} + +# if GTEST_HAS_COMBINE +// Combine() allows the user to combine two or more sequences to produce +// values of a Cartesian product of those sequences' elements. +// +// Synopsis: +// Combine(gen1, gen2, ..., genN) +// - returns a generator producing sequences with elements coming from +// the Cartesian product of elements from the sequences generated by +// gen1, gen2, ..., genN. The sequence elements will have a type of +// tuple where T1, T2, ..., TN are the types +// of elements from sequences produces by gen1, gen2, ..., genN. +// +// Combine can have up to 10 arguments. This number is currently limited +// by the maximum number of elements in the tuple implementation used by Google +// Test. +// +// Example: +// +// This will instantiate tests in test case AnimalTest each one with +// the parameter values tuple("cat", BLACK), tuple("cat", WHITE), +// tuple("dog", BLACK), and tuple("dog", WHITE): +// +// enum Color { BLACK, GRAY, WHITE }; +// class AnimalTest +// : public testing::TestWithParam > {...}; +// +// TEST_P(AnimalTest, AnimalLooksNice) {...} +// +// INSTANTIATE_TEST_CASE_P(AnimalVariations, AnimalTest, +// Combine(Values("cat", "dog"), +// Values(BLACK, WHITE))); +// +// This will instantiate tests in FlagDependentTest with all variations of two +// Boolean flags: +// +// class FlagDependentTest +// : public testing::TestWithParam > { +// virtual void SetUp() { +// // Assigns external_flag_1 and external_flag_2 values from the tuple. +// tie(external_flag_1, external_flag_2) = GetParam(); +// } +// }; +// +// TEST_P(FlagDependentTest, TestFeature1) { +// // Test your code using external_flag_1 and external_flag_2 here. +// } +// INSTANTIATE_TEST_CASE_P(TwoBoolSequence, FlagDependentTest, +// Combine(Bool(), Bool())); +// +template +internal::CartesianProductHolder2 Combine( + const Generator1& g1, const Generator2& g2) { + return internal::CartesianProductHolder2( + g1, g2); +} + +template +internal::CartesianProductHolder3 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3) { + return internal::CartesianProductHolder3( + g1, g2, g3); +} + +template +internal::CartesianProductHolder4 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4) { + return internal::CartesianProductHolder4( + g1, g2, g3, g4); +} + +template +internal::CartesianProductHolder5 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5) { + return internal::CartesianProductHolder5( + g1, g2, g3, g4, g5); +} + +template +internal::CartesianProductHolder6 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5, const Generator6& g6) { + return internal::CartesianProductHolder6( + g1, g2, g3, g4, g5, g6); +} + +template +internal::CartesianProductHolder7 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5, const Generator6& g6, + const Generator7& g7) { + return internal::CartesianProductHolder7( + g1, g2, g3, g4, g5, g6, g7); +} + +template +internal::CartesianProductHolder8 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5, const Generator6& g6, + const Generator7& g7, const Generator8& g8) { + return internal::CartesianProductHolder8( + g1, g2, g3, g4, g5, g6, g7, g8); +} + +template +internal::CartesianProductHolder9 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5, const Generator6& g6, + const Generator7& g7, const Generator8& g8, const Generator9& g9) { + return internal::CartesianProductHolder9( + g1, g2, g3, g4, g5, g6, g7, g8, g9); +} + +template +internal::CartesianProductHolder10 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5, const Generator6& g6, + const Generator7& g7, const Generator8& g8, const Generator9& g9, + const Generator10& g10) { + return internal::CartesianProductHolder10( + g1, g2, g3, g4, g5, g6, g7, g8, g9, g10); +} +# endif // GTEST_HAS_COMBINE + +# define TEST_P(test_case_name, test_name) \ + class GTEST_TEST_CLASS_NAME_(test_case_name, test_name) \ + : public test_case_name { \ + public: \ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)() {} \ + virtual void TestBody(); \ + private: \ + static int AddToRegistry() { \ + ::testing::UnitTest::GetInstance()->parameterized_test_registry(). \ + GetTestCasePatternHolder(\ + #test_case_name, \ + ::testing::internal::CodeLocation(\ + __FILE__, __LINE__))->AddTestPattern(\ + GTEST_STRINGIFY_(test_case_name), \ + GTEST_STRINGIFY_(test_name), \ + new ::testing::internal::TestMetaFactory< \ + GTEST_TEST_CLASS_NAME_(\ + test_case_name, test_name)>()); \ + return 0; \ + } \ + static int gtest_registering_dummy_ GTEST_ATTRIBUTE_UNUSED_; \ + GTEST_DISALLOW_COPY_AND_ASSIGN_(\ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)); \ + }; \ + int GTEST_TEST_CLASS_NAME_(test_case_name, \ + test_name)::gtest_registering_dummy_ = \ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::AddToRegistry(); \ + void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::TestBody() + +// The optional last argument to INSTANTIATE_TEST_CASE_P allows the user +// to specify a function or functor that generates custom test name suffixes +// based on the test parameters. The function should accept one argument of +// type testing::TestParamInfo, and return std::string. +// +// testing::PrintToStringParamName is a builtin test suffix generator that +// returns the value of testing::PrintToString(GetParam()). +// +// Note: test names must be non-empty, unique, and may only contain ASCII +// alphanumeric characters or underscore. Because PrintToString adds quotes +// to std::string and C strings, it won't work for these types. + +# define INSTANTIATE_TEST_CASE_P(prefix, test_case_name, generator, ...) \ + static ::testing::internal::ParamGenerator \ + gtest_##prefix##test_case_name##_EvalGenerator_() { return generator; } \ + static ::std::string gtest_##prefix##test_case_name##_EvalGenerateName_( \ + const ::testing::TestParamInfo& info) { \ + return ::testing::internal::GetParamNameGen \ + (__VA_ARGS__)(info); \ + } \ + static int gtest_##prefix##test_case_name##_dummy_ GTEST_ATTRIBUTE_UNUSED_ = \ + ::testing::UnitTest::GetInstance()->parameterized_test_registry(). \ + GetTestCasePatternHolder(\ + #test_case_name, \ + ::testing::internal::CodeLocation(\ + __FILE__, __LINE__))->AddTestCaseInstantiation(\ + #prefix, \ + >est_##prefix##test_case_name##_EvalGenerator_, \ + >est_##prefix##test_case_name##_EvalGenerateName_, \ + __FILE__, __LINE__) + +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ +// Copyright 2006, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// +// Google C++ Testing and Mocking Framework definitions useful in production code. +// GOOGLETEST_CM0003 DO NOT DELETE + +#ifndef GTEST_INCLUDE_GTEST_GTEST_PROD_H_ +#define GTEST_INCLUDE_GTEST_GTEST_PROD_H_ + +// When you need to test the private or protected members of a class, +// use the FRIEND_TEST macro to declare your tests as friends of the +// class. For example: +// +// class MyClass { +// private: +// void PrivateMethod(); +// FRIEND_TEST(MyClassTest, PrivateMethodWorks); +// }; +// +// class MyClassTest : public testing::Test { +// // ... +// }; +// +// TEST_F(MyClassTest, PrivateMethodWorks) { +// // Can call MyClass::PrivateMethod() here. +// } +// +// Note: The test class must be in the same namespace as the class being tested. +// For example, putting MyClassTest in an anonymous namespace will not work. + +#define FRIEND_TEST(test_case_name, test_name)\ +friend class test_case_name##_##test_name##_Test + +#endif // GTEST_INCLUDE_GTEST_GTEST_PROD_H_ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// GOOGLETEST_CM0001 DO NOT DELETE + +#ifndef GTEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ +#define GTEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ + +#include +#include + +GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ +/* class A needs to have dll-interface to be used by clients of class B */) + +namespace testing { + +// A copyable object representing the result of a test part (i.e. an +// assertion or an explicit FAIL(), ADD_FAILURE(), or SUCCESS()). +// +// Don't inherit from TestPartResult as its destructor is not virtual. +class GTEST_API_ TestPartResult { + public: + // The possible outcomes of a test part (i.e. an assertion or an + // explicit SUCCEED(), FAIL(), or ADD_FAILURE()). + enum Type { + kSuccess, // Succeeded. + kNonFatalFailure, // Failed but the test can continue. + kFatalFailure // Failed and the test should be terminated. + }; + + // C'tor. TestPartResult does NOT have a default constructor. + // Always use this constructor (with parameters) to create a + // TestPartResult object. + TestPartResult(Type a_type, + const char* a_file_name, + int a_line_number, + const char* a_message) + : type_(a_type), + file_name_(a_file_name == NULL ? "" : a_file_name), + line_number_(a_line_number), + summary_(ExtractSummary(a_message)), + message_(a_message) { + } + + // Gets the outcome of the test part. + Type type() const { return type_; } + + // Gets the name of the source file where the test part took place, or + // NULL if it's unknown. + const char* file_name() const { + return file_name_.empty() ? NULL : file_name_.c_str(); + } + + // Gets the line in the source file where the test part took place, + // or -1 if it's unknown. + int line_number() const { return line_number_; } + + // Gets the summary of the failure message. + const char* summary() const { return summary_.c_str(); } + + // Gets the message associated with the test part. + const char* message() const { return message_.c_str(); } + + // Returns true iff the test part passed. + bool passed() const { return type_ == kSuccess; } + + // Returns true iff the test part failed. + bool failed() const { return type_ != kSuccess; } + + // Returns true iff the test part non-fatally failed. + bool nonfatally_failed() const { return type_ == kNonFatalFailure; } + + // Returns true iff the test part fatally failed. + bool fatally_failed() const { return type_ == kFatalFailure; } + + private: + Type type_; + + // Gets the summary of the failure message by omitting the stack + // trace in it. + static std::string ExtractSummary(const char* message); + + // The name of the source file where the test part took place, or + // "" if the source file is unknown. + std::string file_name_; + // The line in the source file where the test part took place, or -1 + // if the line number is unknown. + int line_number_; + std::string summary_; // The test failure summary. + std::string message_; // The test failure message. +}; + +// Prints a TestPartResult object. +std::ostream& operator<<(std::ostream& os, const TestPartResult& result); + +// An array of TestPartResult objects. +// +// Don't inherit from TestPartResultArray as its destructor is not +// virtual. +class GTEST_API_ TestPartResultArray { + public: + TestPartResultArray() {} + + // Appends the given TestPartResult to the array. + void Append(const TestPartResult& result); + + // Returns the TestPartResult at the given index (0-based). + const TestPartResult& GetTestPartResult(int index) const; + + // Returns the number of TestPartResult objects in the array. + int size() const; + + private: + std::vector array_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestPartResultArray); +}; + +// This interface knows how to report a test part result. +class GTEST_API_ TestPartResultReporterInterface { + public: + virtual ~TestPartResultReporterInterface() {} + + virtual void ReportTestPartResult(const TestPartResult& result) = 0; +}; + +namespace internal { + +// This helper class is used by {ASSERT|EXPECT}_NO_FATAL_FAILURE to check if a +// statement generates new fatal failures. To do so it registers itself as the +// current test part result reporter. Besides checking if fatal failures were +// reported, it only delegates the reporting to the former result reporter. +// The original result reporter is restored in the destructor. +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +class GTEST_API_ HasNewFatalFailureHelper + : public TestPartResultReporterInterface { + public: + HasNewFatalFailureHelper(); + virtual ~HasNewFatalFailureHelper(); + virtual void ReportTestPartResult(const TestPartResult& result); + bool has_new_fatal_failure() const { return has_new_fatal_failure_; } + private: + bool has_new_fatal_failure_; + TestPartResultReporterInterface* original_reporter_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(HasNewFatalFailureHelper); +}; + +} // namespace internal + +} // namespace testing + +GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 + +#endif // GTEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +// GOOGLETEST_CM0001 DO NOT DELETE + +#ifndef GTEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ +#define GTEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ + +// This header implements typed tests and type-parameterized tests. + +// Typed (aka type-driven) tests repeat the same test for types in a +// list. You must know which types you want to test with when writing +// typed tests. Here's how you do it: + +#if 0 + +// First, define a fixture class template. It should be parameterized +// by a type. Remember to derive it from testing::Test. +template +class FooTest : public testing::Test { + public: + ... + typedef std::list List; + static T shared_; + T value_; +}; + +// Next, associate a list of types with the test case, which will be +// repeated for each type in the list. The typedef is necessary for +// the macro to parse correctly. +typedef testing::Types MyTypes; +TYPED_TEST_CASE(FooTest, MyTypes); + +// If the type list contains only one type, you can write that type +// directly without Types<...>: +// TYPED_TEST_CASE(FooTest, int); + +// Then, use TYPED_TEST() instead of TEST_F() to define as many typed +// tests for this test case as you want. +TYPED_TEST(FooTest, DoesBlah) { + // Inside a test, refer to TypeParam to get the type parameter. + // Since we are inside a derived class template, C++ requires use to + // visit the members of FooTest via 'this'. + TypeParam n = this->value_; + + // To visit static members of the fixture, add the TestFixture:: + // prefix. + n += TestFixture::shared_; + + // To refer to typedefs in the fixture, add the "typename + // TestFixture::" prefix. + typename TestFixture::List values; + values.push_back(n); + ... +} + +TYPED_TEST(FooTest, HasPropertyA) { ... } + +// TYPED_TEST_CASE takes an optional third argument which allows to specify a +// class that generates custom test name suffixes based on the type. This should +// be a class which has a static template function GetName(int index) returning +// a string for each type. The provided integer index equals the index of the +// type in the provided type list. In many cases the index can be ignored. +// +// For example: +// class MyTypeNames { +// public: +// template +// static std::string GetName(int) { +// if (std::is_same()) return "char"; +// if (std::is_same()) return "int"; +// if (std::is_same()) return "unsignedInt"; +// } +// }; +// TYPED_TEST_CASE(FooTest, MyTypes, MyTypeNames); + +#endif // 0 + +// Type-parameterized tests are abstract test patterns parameterized +// by a type. Compared with typed tests, type-parameterized tests +// allow you to define the test pattern without knowing what the type +// parameters are. The defined pattern can be instantiated with +// different types any number of times, in any number of translation +// units. +// +// If you are designing an interface or concept, you can define a +// suite of type-parameterized tests to verify properties that any +// valid implementation of the interface/concept should have. Then, +// each implementation can easily instantiate the test suite to verify +// that it conforms to the requirements, without having to write +// similar tests repeatedly. Here's an example: + +#if 0 + +// First, define a fixture class template. It should be parameterized +// by a type. Remember to derive it from testing::Test. +template +class FooTest : public testing::Test { + ... +}; + +// Next, declare that you will define a type-parameterized test case +// (the _P suffix is for "parameterized" or "pattern", whichever you +// prefer): +TYPED_TEST_CASE_P(FooTest); + +// Then, use TYPED_TEST_P() to define as many type-parameterized tests +// for this type-parameterized test case as you want. +TYPED_TEST_P(FooTest, DoesBlah) { + // Inside a test, refer to TypeParam to get the type parameter. + TypeParam n = 0; + ... +} + +TYPED_TEST_P(FooTest, HasPropertyA) { ... } + +// Now the tricky part: you need to register all test patterns before +// you can instantiate them. The first argument of the macro is the +// test case name; the rest are the names of the tests in this test +// case. +REGISTER_TYPED_TEST_CASE_P(FooTest, + DoesBlah, HasPropertyA); + +// Finally, you are free to instantiate the pattern with the types you +// want. If you put the above code in a header file, you can #include +// it in multiple C++ source files and instantiate it multiple times. +// +// To distinguish different instances of the pattern, the first +// argument to the INSTANTIATE_* macro is a prefix that will be added +// to the actual test case name. Remember to pick unique prefixes for +// different instances. +typedef testing::Types MyTypes; +INSTANTIATE_TYPED_TEST_CASE_P(My, FooTest, MyTypes); + +// If the type list contains only one type, you can write that type +// directly without Types<...>: +// INSTANTIATE_TYPED_TEST_CASE_P(My, FooTest, int); +// +// Similar to the optional argument of TYPED_TEST_CASE above, +// INSTANTIATE_TEST_CASE_P takes an optional fourth argument which allows to +// generate custom names. +// INSTANTIATE_TYPED_TEST_CASE_P(My, FooTest, MyTypes, MyTypeNames); + +#endif // 0 + + +// Implements typed tests. + +#if GTEST_HAS_TYPED_TEST + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Expands to the name of the typedef for the type parameters of the +// given test case. +# define GTEST_TYPE_PARAMS_(TestCaseName) gtest_type_params_##TestCaseName##_ + +// Expands to the name of the typedef for the NameGenerator, responsible for +// creating the suffixes of the name. +#define GTEST_NAME_GENERATOR_(TestCaseName) \ + gtest_type_params_##TestCaseName##_NameGenerator + +// The 'Types' template argument below must have spaces around it +// since some compilers may choke on '>>' when passing a template +// instance (e.g. Types) +# define TYPED_TEST_CASE(CaseName, Types, ...) \ + typedef ::testing::internal::TypeList< Types >::type GTEST_TYPE_PARAMS_( \ + CaseName); \ + typedef ::testing::internal::NameGeneratorSelector<__VA_ARGS__>::type \ + GTEST_NAME_GENERATOR_(CaseName) + +# define TYPED_TEST(CaseName, TestName) \ + template \ + class GTEST_TEST_CLASS_NAME_(CaseName, TestName) \ + : public CaseName { \ + private: \ + typedef CaseName TestFixture; \ + typedef gtest_TypeParam_ TypeParam; \ + virtual void TestBody(); \ + }; \ + static bool gtest_##CaseName##_##TestName##_registered_ \ + GTEST_ATTRIBUTE_UNUSED_ = \ + ::testing::internal::TypeParameterizedTest< \ + CaseName, \ + ::testing::internal::TemplateSel, \ + GTEST_TYPE_PARAMS_( \ + CaseName)>::Register("", \ + ::testing::internal::CodeLocation( \ + __FILE__, __LINE__), \ + #CaseName, #TestName, 0, \ + ::testing::internal::GenerateNames< \ + GTEST_NAME_GENERATOR_(CaseName), \ + GTEST_TYPE_PARAMS_(CaseName)>()); \ + template \ + void GTEST_TEST_CLASS_NAME_(CaseName, \ + TestName)::TestBody() + +#endif // GTEST_HAS_TYPED_TEST + +// Implements type-parameterized tests. + +#if GTEST_HAS_TYPED_TEST_P + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Expands to the namespace name that the type-parameterized tests for +// the given type-parameterized test case are defined in. The exact +// name of the namespace is subject to change without notice. +# define GTEST_CASE_NAMESPACE_(TestCaseName) \ + gtest_case_##TestCaseName##_ + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Expands to the name of the variable used to remember the names of +// the defined tests in the given test case. +# define GTEST_TYPED_TEST_CASE_P_STATE_(TestCaseName) \ + gtest_typed_test_case_p_state_##TestCaseName##_ + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE DIRECTLY. +// +// Expands to the name of the variable used to remember the names of +// the registered tests in the given test case. +# define GTEST_REGISTERED_TEST_NAMES_(TestCaseName) \ + gtest_registered_test_names_##TestCaseName##_ + +// The variables defined in the type-parameterized test macros are +// static as typically these macros are used in a .h file that can be +// #included in multiple translation units linked together. +# define TYPED_TEST_CASE_P(CaseName) \ + static ::testing::internal::TypedTestCasePState \ + GTEST_TYPED_TEST_CASE_P_STATE_(CaseName) + +# define TYPED_TEST_P(CaseName, TestName) \ + namespace GTEST_CASE_NAMESPACE_(CaseName) { \ + template \ + class TestName : public CaseName { \ + private: \ + typedef CaseName TestFixture; \ + typedef gtest_TypeParam_ TypeParam; \ + virtual void TestBody(); \ + }; \ + static bool gtest_##TestName##_defined_ GTEST_ATTRIBUTE_UNUSED_ = \ + GTEST_TYPED_TEST_CASE_P_STATE_(CaseName).AddTestName(\ + __FILE__, __LINE__, #CaseName, #TestName); \ + } \ + template \ + void GTEST_CASE_NAMESPACE_(CaseName)::TestName::TestBody() + +# define REGISTER_TYPED_TEST_CASE_P(CaseName, ...) \ + namespace GTEST_CASE_NAMESPACE_(CaseName) { \ + typedef ::testing::internal::Templates<__VA_ARGS__>::type gtest_AllTests_; \ + } \ + static const char* const GTEST_REGISTERED_TEST_NAMES_(CaseName) \ + GTEST_ATTRIBUTE_UNUSED_ = \ + GTEST_TYPED_TEST_CASE_P_STATE_(CaseName).VerifyRegisteredTestNames( \ + __FILE__, __LINE__, #__VA_ARGS__) + +// The 'Types' template argument below must have spaces around it +// since some compilers may choke on '>>' when passing a template +// instance (e.g. Types) +# define INSTANTIATE_TYPED_TEST_CASE_P(Prefix, CaseName, Types, ...) \ + static bool gtest_##Prefix##_##CaseName GTEST_ATTRIBUTE_UNUSED_ = \ + ::testing::internal::TypeParameterizedTestCase< \ + CaseName, GTEST_CASE_NAMESPACE_(CaseName)::gtest_AllTests_, \ + ::testing::internal::TypeList< Types >::type>:: \ + Register(#Prefix, \ + ::testing::internal::CodeLocation(__FILE__, __LINE__), \ + >EST_TYPED_TEST_CASE_P_STATE_(CaseName), #CaseName, \ + GTEST_REGISTERED_TEST_NAMES_(CaseName), \ + ::testing::internal::GenerateNames< \ + ::testing::internal::NameGeneratorSelector< \ + __VA_ARGS__>::type, \ + ::testing::internal::TypeList< Types >::type>()) + +#endif // GTEST_HAS_TYPED_TEST_P + +#endif // GTEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ + +GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ +/* class A needs to have dll-interface to be used by clients of class B */) + +// Depending on the platform, different string classes are available. +// On Linux, in addition to ::std::string, Google also makes use of +// class ::string, which has the same interface as ::std::string, but +// has a different implementation. +// +// You can define GTEST_HAS_GLOBAL_STRING to 1 to indicate that +// ::string is available AND is a distinct type to ::std::string, or +// define it to 0 to indicate otherwise. +// +// If ::std::string and ::string are the same class on your platform +// due to aliasing, you should define GTEST_HAS_GLOBAL_STRING to 0. +// +// If you do not define GTEST_HAS_GLOBAL_STRING, it is defined +// heuristically. + +namespace testing { + +// Silence C4100 (unreferenced formal parameter) and 4805 +// unsafe mix of type 'const int' and type 'const bool' +#ifdef _MSC_VER +# pragma warning(push) +# pragma warning(disable:4805) +# pragma warning(disable:4100) +#endif + + +// Declares the flags. + +// This flag temporary enables the disabled tests. +GTEST_DECLARE_bool_(also_run_disabled_tests); + +// This flag brings the debugger on an assertion failure. +GTEST_DECLARE_bool_(break_on_failure); + +// This flag controls whether Google Test catches all test-thrown exceptions +// and logs them as failures. +GTEST_DECLARE_bool_(catch_exceptions); + +// This flag enables using colors in terminal output. Available values are +// "yes" to enable colors, "no" (disable colors), or "auto" (the default) +// to let Google Test decide. +GTEST_DECLARE_string_(color); + +// This flag sets up the filter to select by name using a glob pattern +// the tests to run. If the filter is not given all tests are executed. +GTEST_DECLARE_string_(filter); + +// This flag controls whether Google Test installs a signal handler that dumps +// debugging information when fatal signals are raised. +GTEST_DECLARE_bool_(install_failure_signal_handler); + +// This flag causes the Google Test to list tests. None of the tests listed +// are actually run if the flag is provided. +GTEST_DECLARE_bool_(list_tests); + +// This flag controls whether Google Test emits a detailed XML report to a file +// in addition to its normal textual output. +GTEST_DECLARE_string_(output); + +// This flags control whether Google Test prints the elapsed time for each +// test. +GTEST_DECLARE_bool_(print_time); + +// This flags control whether Google Test prints UTF8 characters as text. +GTEST_DECLARE_bool_(print_utf8); + +// This flag specifies the random number seed. +GTEST_DECLARE_int32_(random_seed); + +// This flag sets how many times the tests are repeated. The default value +// is 1. If the value is -1 the tests are repeating forever. +GTEST_DECLARE_int32_(repeat); + +// This flag controls whether Google Test includes Google Test internal +// stack frames in failure stack traces. +GTEST_DECLARE_bool_(show_internal_stack_frames); + +// When this flag is specified, tests' order is randomized on every iteration. +GTEST_DECLARE_bool_(shuffle); + +// This flag specifies the maximum number of stack frames to be +// printed in a failure message. +GTEST_DECLARE_int32_(stack_trace_depth); + +// When this flag is specified, a failed assertion will throw an +// exception if exceptions are enabled, or exit the program with a +// non-zero code otherwise. For use with an external test framework. +GTEST_DECLARE_bool_(throw_on_failure); + +// When this flag is set with a "host:port" string, on supported +// platforms test results are streamed to the specified port on +// the specified host machine. +GTEST_DECLARE_string_(stream_result_to); + +#if GTEST_USE_OWN_FLAGFILE_FLAG_ +GTEST_DECLARE_string_(flagfile); +#endif // GTEST_USE_OWN_FLAGFILE_FLAG_ + +// The upper limit for valid stack trace depths. +const int kMaxStackTraceDepth = 100; + +namespace internal { + +class AssertHelper; +class DefaultGlobalTestPartResultReporter; +class ExecDeathTest; +class NoExecDeathTest; +class FinalSuccessChecker; +class GTestFlagSaver; +class StreamingListenerTest; +class TestResultAccessor; +class TestEventListenersAccessor; +class TestEventRepeater; +class UnitTestRecordPropertyTestHelper; +class WindowsDeathTest; +class FuchsiaDeathTest; +class UnitTestImpl* GetUnitTestImpl(); +void ReportFailureInUnknownLocation(TestPartResult::Type result_type, + const std::string& message); + +} // namespace internal + +// The friend relationship of some of these classes is cyclic. +// If we don't forward declare them the compiler might confuse the classes +// in friendship clauses with same named classes on the scope. +class Test; +class TestCase; +class TestInfo; +class UnitTest; + +// A class for indicating whether an assertion was successful. When +// the assertion wasn't successful, the AssertionResult object +// remembers a non-empty message that describes how it failed. +// +// To create an instance of this class, use one of the factory functions +// (AssertionSuccess() and AssertionFailure()). +// +// This class is useful for two purposes: +// 1. Defining predicate functions to be used with Boolean test assertions +// EXPECT_TRUE/EXPECT_FALSE and their ASSERT_ counterparts +// 2. Defining predicate-format functions to be +// used with predicate assertions (ASSERT_PRED_FORMAT*, etc). +// +// For example, if you define IsEven predicate: +// +// testing::AssertionResult IsEven(int n) { +// if ((n % 2) == 0) +// return testing::AssertionSuccess(); +// else +// return testing::AssertionFailure() << n << " is odd"; +// } +// +// Then the failed expectation EXPECT_TRUE(IsEven(Fib(5))) +// will print the message +// +// Value of: IsEven(Fib(5)) +// Actual: false (5 is odd) +// Expected: true +// +// instead of a more opaque +// +// Value of: IsEven(Fib(5)) +// Actual: false +// Expected: true +// +// in case IsEven is a simple Boolean predicate. +// +// If you expect your predicate to be reused and want to support informative +// messages in EXPECT_FALSE and ASSERT_FALSE (negative assertions show up +// about half as often as positive ones in our tests), supply messages for +// both success and failure cases: +// +// testing::AssertionResult IsEven(int n) { +// if ((n % 2) == 0) +// return testing::AssertionSuccess() << n << " is even"; +// else +// return testing::AssertionFailure() << n << " is odd"; +// } +// +// Then a statement EXPECT_FALSE(IsEven(Fib(6))) will print +// +// Value of: IsEven(Fib(6)) +// Actual: true (8 is even) +// Expected: false +// +// NB: Predicates that support negative Boolean assertions have reduced +// performance in positive ones so be careful not to use them in tests +// that have lots (tens of thousands) of positive Boolean assertions. +// +// To use this class with EXPECT_PRED_FORMAT assertions such as: +// +// // Verifies that Foo() returns an even number. +// EXPECT_PRED_FORMAT1(IsEven, Foo()); +// +// you need to define: +// +// testing::AssertionResult IsEven(const char* expr, int n) { +// if ((n % 2) == 0) +// return testing::AssertionSuccess(); +// else +// return testing::AssertionFailure() +// << "Expected: " << expr << " is even\n Actual: it's " << n; +// } +// +// If Foo() returns 5, you will see the following message: +// +// Expected: Foo() is even +// Actual: it's 5 +// +class GTEST_API_ AssertionResult { + public: + // Copy constructor. + // Used in EXPECT_TRUE/FALSE(assertion_result). + AssertionResult(const AssertionResult& other); + +#if defined(_MSC_VER) && _MSC_VER < 1910 + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4800 /* forcing value to bool */) +#endif + + // Used in the EXPECT_TRUE/FALSE(bool_expression). + // + // T must be contextually convertible to bool. + // + // The second parameter prevents this overload from being considered if + // the argument is implicitly convertible to AssertionResult. In that case + // we want AssertionResult's copy constructor to be used. + template + explicit AssertionResult( + const T& success, + typename internal::EnableIf< + !internal::ImplicitlyConvertible::value>::type* + /*enabler*/ = NULL) + : success_(success) {} + +#if defined(_MSC_VER) && _MSC_VER < 1910 + GTEST_DISABLE_MSC_WARNINGS_POP_() +#endif + + // Assignment operator. + AssertionResult& operator=(AssertionResult other) { + swap(other); + return *this; + } + + // Returns true iff the assertion succeeded. + operator bool() const { return success_; } // NOLINT + + // Returns the assertion's negation. Used with EXPECT/ASSERT_FALSE. + AssertionResult operator!() const; + + // Returns the text streamed into this AssertionResult. Test assertions + // use it when they fail (i.e., the predicate's outcome doesn't match the + // assertion's expectation). When nothing has been streamed into the + // object, returns an empty string. + const char* message() const { + return message_.get() != NULL ? message_->c_str() : ""; + } + // FIXME: Remove this after making sure no clients use it. + // Deprecated; please use message() instead. + const char* failure_message() const { return message(); } + + // Streams a custom failure message into this object. + template AssertionResult& operator<<(const T& value) { + AppendMessage(Message() << value); + return *this; + } + + // Allows streaming basic output manipulators such as endl or flush into + // this object. + AssertionResult& operator<<( + ::std::ostream& (*basic_manipulator)(::std::ostream& stream)) { + AppendMessage(Message() << basic_manipulator); + return *this; + } + + private: + // Appends the contents of message to message_. + void AppendMessage(const Message& a_message) { + if (message_.get() == NULL) + message_.reset(new ::std::string); + message_->append(a_message.GetString().c_str()); + } + + // Swap the contents of this AssertionResult with other. + void swap(AssertionResult& other); + + // Stores result of the assertion predicate. + bool success_; + // Stores the message describing the condition in case the expectation + // construct is not satisfied with the predicate's outcome. + // Referenced via a pointer to avoid taking too much stack frame space + // with test assertions. + internal::scoped_ptr< ::std::string> message_; +}; + +// Makes a successful assertion result. +GTEST_API_ AssertionResult AssertionSuccess(); + +// Makes a failed assertion result. +GTEST_API_ AssertionResult AssertionFailure(); + +// Makes a failed assertion result with the given failure message. +// Deprecated; use AssertionFailure() << msg. +GTEST_API_ AssertionResult AssertionFailure(const Message& msg); + +} // namespace testing + +// Includes the auto-generated header that implements a family of generic +// predicate assertion macros. This include comes late because it relies on +// APIs declared above. +// Copyright 2006, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// This file is AUTOMATICALLY GENERATED on 01/02/2018 by command +// 'gen_gtest_pred_impl.py 5'. DO NOT EDIT BY HAND! +// +// Implements a family of generic predicate assertion macros. + +// GOOGLETEST_CM0001 DO NOT DELETE + +#ifndef GTEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ +#define GTEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ + + +namespace testing { + +// This header implements a family of generic predicate assertion +// macros: +// +// ASSERT_PRED_FORMAT1(pred_format, v1) +// ASSERT_PRED_FORMAT2(pred_format, v1, v2) +// ... +// +// where pred_format is a function or functor that takes n (in the +// case of ASSERT_PRED_FORMATn) values and their source expression +// text, and returns a testing::AssertionResult. See the definition +// of ASSERT_EQ in gtest.h for an example. +// +// If you don't care about formatting, you can use the more +// restrictive version: +// +// ASSERT_PRED1(pred, v1) +// ASSERT_PRED2(pred, v1, v2) +// ... +// +// where pred is an n-ary function or functor that returns bool, +// and the values v1, v2, ..., must support the << operator for +// streaming to std::ostream. +// +// We also define the EXPECT_* variations. +// +// For now we only support predicates whose arity is at most 5. + +// GTEST_ASSERT_ is the basic statement to which all of the assertions +// in this file reduce. Don't use this in your code. + +#define GTEST_ASSERT_(expression, on_failure) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (const ::testing::AssertionResult gtest_ar = (expression)) \ + ; \ + else \ + on_failure(gtest_ar.failure_message()) + + +// Helper function for implementing {EXPECT|ASSERT}_PRED1. Don't use +// this in your code. +template +AssertionResult AssertPred1Helper(const char* pred_text, + const char* e1, + Pred pred, + const T1& v1) { + if (pred(v1)) return AssertionSuccess(); + + return AssertionFailure() << pred_text << "(" + << e1 << ") evaluates to false, where" + << "\n" << e1 << " evaluates to " << v1; +} + +// Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT1. +// Don't use this in your code. +#define GTEST_PRED_FORMAT1_(pred_format, v1, on_failure)\ + GTEST_ASSERT_(pred_format(#v1, v1), \ + on_failure) + +// Internal macro for implementing {EXPECT|ASSERT}_PRED1. Don't use +// this in your code. +#define GTEST_PRED1_(pred, v1, on_failure)\ + GTEST_ASSERT_(::testing::AssertPred1Helper(#pred, \ + #v1, \ + pred, \ + v1), on_failure) + +// Unary predicate assertion macros. +#define EXPECT_PRED_FORMAT1(pred_format, v1) \ + GTEST_PRED_FORMAT1_(pred_format, v1, GTEST_NONFATAL_FAILURE_) +#define EXPECT_PRED1(pred, v1) \ + GTEST_PRED1_(pred, v1, GTEST_NONFATAL_FAILURE_) +#define ASSERT_PRED_FORMAT1(pred_format, v1) \ + GTEST_PRED_FORMAT1_(pred_format, v1, GTEST_FATAL_FAILURE_) +#define ASSERT_PRED1(pred, v1) \ + GTEST_PRED1_(pred, v1, GTEST_FATAL_FAILURE_) + + + +// Helper function for implementing {EXPECT|ASSERT}_PRED2. Don't use +// this in your code. +template +AssertionResult AssertPred2Helper(const char* pred_text, + const char* e1, + const char* e2, + Pred pred, + const T1& v1, + const T2& v2) { + if (pred(v1, v2)) return AssertionSuccess(); + + return AssertionFailure() << pred_text << "(" + << e1 << ", " + << e2 << ") evaluates to false, where" + << "\n" << e1 << " evaluates to " << v1 + << "\n" << e2 << " evaluates to " << v2; +} + +// Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT2. +// Don't use this in your code. +#define GTEST_PRED_FORMAT2_(pred_format, v1, v2, on_failure)\ + GTEST_ASSERT_(pred_format(#v1, #v2, v1, v2), \ + on_failure) + +// Internal macro for implementing {EXPECT|ASSERT}_PRED2. Don't use +// this in your code. +#define GTEST_PRED2_(pred, v1, v2, on_failure)\ + GTEST_ASSERT_(::testing::AssertPred2Helper(#pred, \ + #v1, \ + #v2, \ + pred, \ + v1, \ + v2), on_failure) + +// Binary predicate assertion macros. +#define EXPECT_PRED_FORMAT2(pred_format, v1, v2) \ + GTEST_PRED_FORMAT2_(pred_format, v1, v2, GTEST_NONFATAL_FAILURE_) +#define EXPECT_PRED2(pred, v1, v2) \ + GTEST_PRED2_(pred, v1, v2, GTEST_NONFATAL_FAILURE_) +#define ASSERT_PRED_FORMAT2(pred_format, v1, v2) \ + GTEST_PRED_FORMAT2_(pred_format, v1, v2, GTEST_FATAL_FAILURE_) +#define ASSERT_PRED2(pred, v1, v2) \ + GTEST_PRED2_(pred, v1, v2, GTEST_FATAL_FAILURE_) + + + +// Helper function for implementing {EXPECT|ASSERT}_PRED3. Don't use +// this in your code. +template +AssertionResult AssertPred3Helper(const char* pred_text, + const char* e1, + const char* e2, + const char* e3, + Pred pred, + const T1& v1, + const T2& v2, + const T3& v3) { + if (pred(v1, v2, v3)) return AssertionSuccess(); + + return AssertionFailure() << pred_text << "(" + << e1 << ", " + << e2 << ", " + << e3 << ") evaluates to false, where" + << "\n" << e1 << " evaluates to " << v1 + << "\n" << e2 << " evaluates to " << v2 + << "\n" << e3 << " evaluates to " << v3; +} + +// Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT3. +// Don't use this in your code. +#define GTEST_PRED_FORMAT3_(pred_format, v1, v2, v3, on_failure)\ + GTEST_ASSERT_(pred_format(#v1, #v2, #v3, v1, v2, v3), \ + on_failure) + +// Internal macro for implementing {EXPECT|ASSERT}_PRED3. Don't use +// this in your code. +#define GTEST_PRED3_(pred, v1, v2, v3, on_failure)\ + GTEST_ASSERT_(::testing::AssertPred3Helper(#pred, \ + #v1, \ + #v2, \ + #v3, \ + pred, \ + v1, \ + v2, \ + v3), on_failure) + +// Ternary predicate assertion macros. +#define EXPECT_PRED_FORMAT3(pred_format, v1, v2, v3) \ + GTEST_PRED_FORMAT3_(pred_format, v1, v2, v3, GTEST_NONFATAL_FAILURE_) +#define EXPECT_PRED3(pred, v1, v2, v3) \ + GTEST_PRED3_(pred, v1, v2, v3, GTEST_NONFATAL_FAILURE_) +#define ASSERT_PRED_FORMAT3(pred_format, v1, v2, v3) \ + GTEST_PRED_FORMAT3_(pred_format, v1, v2, v3, GTEST_FATAL_FAILURE_) +#define ASSERT_PRED3(pred, v1, v2, v3) \ + GTEST_PRED3_(pred, v1, v2, v3, GTEST_FATAL_FAILURE_) + + + +// Helper function for implementing {EXPECT|ASSERT}_PRED4. Don't use +// this in your code. +template +AssertionResult AssertPred4Helper(const char* pred_text, + const char* e1, + const char* e2, + const char* e3, + const char* e4, + Pred pred, + const T1& v1, + const T2& v2, + const T3& v3, + const T4& v4) { + if (pred(v1, v2, v3, v4)) return AssertionSuccess(); + + return AssertionFailure() << pred_text << "(" + << e1 << ", " + << e2 << ", " + << e3 << ", " + << e4 << ") evaluates to false, where" + << "\n" << e1 << " evaluates to " << v1 + << "\n" << e2 << " evaluates to " << v2 + << "\n" << e3 << " evaluates to " << v3 + << "\n" << e4 << " evaluates to " << v4; +} + +// Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT4. +// Don't use this in your code. +#define GTEST_PRED_FORMAT4_(pred_format, v1, v2, v3, v4, on_failure)\ + GTEST_ASSERT_(pred_format(#v1, #v2, #v3, #v4, v1, v2, v3, v4), \ + on_failure) + +// Internal macro for implementing {EXPECT|ASSERT}_PRED4. Don't use +// this in your code. +#define GTEST_PRED4_(pred, v1, v2, v3, v4, on_failure)\ + GTEST_ASSERT_(::testing::AssertPred4Helper(#pred, \ + #v1, \ + #v2, \ + #v3, \ + #v4, \ + pred, \ + v1, \ + v2, \ + v3, \ + v4), on_failure) + +// 4-ary predicate assertion macros. +#define EXPECT_PRED_FORMAT4(pred_format, v1, v2, v3, v4) \ + GTEST_PRED_FORMAT4_(pred_format, v1, v2, v3, v4, GTEST_NONFATAL_FAILURE_) +#define EXPECT_PRED4(pred, v1, v2, v3, v4) \ + GTEST_PRED4_(pred, v1, v2, v3, v4, GTEST_NONFATAL_FAILURE_) +#define ASSERT_PRED_FORMAT4(pred_format, v1, v2, v3, v4) \ + GTEST_PRED_FORMAT4_(pred_format, v1, v2, v3, v4, GTEST_FATAL_FAILURE_) +#define ASSERT_PRED4(pred, v1, v2, v3, v4) \ + GTEST_PRED4_(pred, v1, v2, v3, v4, GTEST_FATAL_FAILURE_) + + + +// Helper function for implementing {EXPECT|ASSERT}_PRED5. Don't use +// this in your code. +template +AssertionResult AssertPred5Helper(const char* pred_text, + const char* e1, + const char* e2, + const char* e3, + const char* e4, + const char* e5, + Pred pred, + const T1& v1, + const T2& v2, + const T3& v3, + const T4& v4, + const T5& v5) { + if (pred(v1, v2, v3, v4, v5)) return AssertionSuccess(); + + return AssertionFailure() << pred_text << "(" + << e1 << ", " + << e2 << ", " + << e3 << ", " + << e4 << ", " + << e5 << ") evaluates to false, where" + << "\n" << e1 << " evaluates to " << v1 + << "\n" << e2 << " evaluates to " << v2 + << "\n" << e3 << " evaluates to " << v3 + << "\n" << e4 << " evaluates to " << v4 + << "\n" << e5 << " evaluates to " << v5; +} + +// Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT5. +// Don't use this in your code. +#define GTEST_PRED_FORMAT5_(pred_format, v1, v2, v3, v4, v5, on_failure)\ + GTEST_ASSERT_(pred_format(#v1, #v2, #v3, #v4, #v5, v1, v2, v3, v4, v5), \ + on_failure) + +// Internal macro for implementing {EXPECT|ASSERT}_PRED5. Don't use +// this in your code. +#define GTEST_PRED5_(pred, v1, v2, v3, v4, v5, on_failure)\ + GTEST_ASSERT_(::testing::AssertPred5Helper(#pred, \ + #v1, \ + #v2, \ + #v3, \ + #v4, \ + #v5, \ + pred, \ + v1, \ + v2, \ + v3, \ + v4, \ + v5), on_failure) + +// 5-ary predicate assertion macros. +#define EXPECT_PRED_FORMAT5(pred_format, v1, v2, v3, v4, v5) \ + GTEST_PRED_FORMAT5_(pred_format, v1, v2, v3, v4, v5, GTEST_NONFATAL_FAILURE_) +#define EXPECT_PRED5(pred, v1, v2, v3, v4, v5) \ + GTEST_PRED5_(pred, v1, v2, v3, v4, v5, GTEST_NONFATAL_FAILURE_) +#define ASSERT_PRED_FORMAT5(pred_format, v1, v2, v3, v4, v5) \ + GTEST_PRED_FORMAT5_(pred_format, v1, v2, v3, v4, v5, GTEST_FATAL_FAILURE_) +#define ASSERT_PRED5(pred, v1, v2, v3, v4, v5) \ + GTEST_PRED5_(pred, v1, v2, v3, v4, v5, GTEST_FATAL_FAILURE_) + + + +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ + +namespace testing { + +// The abstract class that all tests inherit from. +// +// In Google Test, a unit test program contains one or many TestCases, and +// each TestCase contains one or many Tests. +// +// When you define a test using the TEST macro, you don't need to +// explicitly derive from Test - the TEST macro automatically does +// this for you. +// +// The only time you derive from Test is when defining a test fixture +// to be used in a TEST_F. For example: +// +// class FooTest : public testing::Test { +// protected: +// void SetUp() override { ... } +// void TearDown() override { ... } +// ... +// }; +// +// TEST_F(FooTest, Bar) { ... } +// TEST_F(FooTest, Baz) { ... } +// +// Test is not copyable. +class GTEST_API_ Test { + public: + friend class TestInfo; + + // Defines types for pointers to functions that set up and tear down + // a test case. + typedef internal::SetUpTestCaseFunc SetUpTestCaseFunc; + typedef internal::TearDownTestCaseFunc TearDownTestCaseFunc; + + // The d'tor is virtual as we intend to inherit from Test. + virtual ~Test(); + + // Sets up the stuff shared by all tests in this test case. + // + // Google Test will call Foo::SetUpTestCase() before running the first + // test in test case Foo. Hence a sub-class can define its own + // SetUpTestCase() method to shadow the one defined in the super + // class. + static void SetUpTestCase() {} + + // Tears down the stuff shared by all tests in this test case. + // + // Google Test will call Foo::TearDownTestCase() after running the last + // test in test case Foo. Hence a sub-class can define its own + // TearDownTestCase() method to shadow the one defined in the super + // class. + static void TearDownTestCase() {} + + // Returns true iff the current test has a fatal failure. + static bool HasFatalFailure(); + + // Returns true iff the current test has a non-fatal failure. + static bool HasNonfatalFailure(); + + // Returns true iff the current test has a (either fatal or + // non-fatal) failure. + static bool HasFailure() { return HasFatalFailure() || HasNonfatalFailure(); } + + // Logs a property for the current test, test case, or for the entire + // invocation of the test program when used outside of the context of a + // test case. Only the last value for a given key is remembered. These + // are public static so they can be called from utility functions that are + // not members of the test fixture. Calls to RecordProperty made during + // lifespan of the test (from the moment its constructor starts to the + // moment its destructor finishes) will be output in XML as attributes of + // the element. Properties recorded from fixture's + // SetUpTestCase or TearDownTestCase are logged as attributes of the + // corresponding element. Calls to RecordProperty made in the + // global context (before or after invocation of RUN_ALL_TESTS and from + // SetUp/TearDown method of Environment objects registered with Google + // Test) will be output as attributes of the element. + static void RecordProperty(const std::string& key, const std::string& value); + static void RecordProperty(const std::string& key, int value); + + protected: + // Creates a Test object. + Test(); + + // Sets up the test fixture. + virtual void SetUp(); + + // Tears down the test fixture. + virtual void TearDown(); + + private: + // Returns true iff the current test has the same fixture class as + // the first test in the current test case. + static bool HasSameFixtureClass(); + + // Runs the test after the test fixture has been set up. + // + // A sub-class must implement this to define the test logic. + // + // DO NOT OVERRIDE THIS FUNCTION DIRECTLY IN A USER PROGRAM. + // Instead, use the TEST or TEST_F macro. + virtual void TestBody() = 0; + + // Sets up, executes, and tears down the test. + void Run(); + + // Deletes self. We deliberately pick an unusual name for this + // internal method to avoid clashing with names used in user TESTs. + void DeleteSelf_() { delete this; } + + const internal::scoped_ptr< GTEST_FLAG_SAVER_ > gtest_flag_saver_; + + // Often a user misspells SetUp() as Setup() and spends a long time + // wondering why it is never called by Google Test. The declaration of + // the following method is solely for catching such an error at + // compile time: + // + // - The return type is deliberately chosen to be not void, so it + // will be a conflict if void Setup() is declared in the user's + // test fixture. + // + // - This method is private, so it will be another compiler error + // if the method is called from the user's test fixture. + // + // DO NOT OVERRIDE THIS FUNCTION. + // + // If you see an error about overriding the following function or + // about it being private, you have mis-spelled SetUp() as Setup(). + struct Setup_should_be_spelled_SetUp {}; + virtual Setup_should_be_spelled_SetUp* Setup() { return NULL; } + + // We disallow copying Tests. + GTEST_DISALLOW_COPY_AND_ASSIGN_(Test); +}; + +typedef internal::TimeInMillis TimeInMillis; + +// A copyable object representing a user specified test property which can be +// output as a key/value string pair. +// +// Don't inherit from TestProperty as its destructor is not virtual. +class TestProperty { + public: + // C'tor. TestProperty does NOT have a default constructor. + // Always use this constructor (with parameters) to create a + // TestProperty object. + TestProperty(const std::string& a_key, const std::string& a_value) : + key_(a_key), value_(a_value) { + } + + // Gets the user supplied key. + const char* key() const { + return key_.c_str(); + } + + // Gets the user supplied value. + const char* value() const { + return value_.c_str(); + } + + // Sets a new value, overriding the one supplied in the constructor. + void SetValue(const std::string& new_value) { + value_ = new_value; + } + + private: + // The key supplied by the user. + std::string key_; + // The value supplied by the user. + std::string value_; +}; + +// The result of a single Test. This includes a list of +// TestPartResults, a list of TestProperties, a count of how many +// death tests there are in the Test, and how much time it took to run +// the Test. +// +// TestResult is not copyable. +class GTEST_API_ TestResult { + public: + // Creates an empty TestResult. + TestResult(); + + // D'tor. Do not inherit from TestResult. + ~TestResult(); + + // Gets the number of all test parts. This is the sum of the number + // of successful test parts and the number of failed test parts. + int total_part_count() const; + + // Returns the number of the test properties. + int test_property_count() const; + + // Returns true iff the test passed (i.e. no test part failed). + bool Passed() const { return !Failed(); } + + // Returns true iff the test failed. + bool Failed() const; + + // Returns true iff the test fatally failed. + bool HasFatalFailure() const; + + // Returns true iff the test has a non-fatal failure. + bool HasNonfatalFailure() const; + + // Returns the elapsed time, in milliseconds. + TimeInMillis elapsed_time() const { return elapsed_time_; } + + // Returns the i-th test part result among all the results. i can range from 0 + // to total_part_count() - 1. If i is not in that range, aborts the program. + const TestPartResult& GetTestPartResult(int i) const; + + // Returns the i-th test property. i can range from 0 to + // test_property_count() - 1. If i is not in that range, aborts the + // program. + const TestProperty& GetTestProperty(int i) const; + + private: + friend class TestInfo; + friend class TestCase; + friend class UnitTest; + friend class internal::DefaultGlobalTestPartResultReporter; + friend class internal::ExecDeathTest; + friend class internal::TestResultAccessor; + friend class internal::UnitTestImpl; + friend class internal::WindowsDeathTest; + friend class internal::FuchsiaDeathTest; + + // Gets the vector of TestPartResults. + const std::vector& test_part_results() const { + return test_part_results_; + } + + // Gets the vector of TestProperties. + const std::vector& test_properties() const { + return test_properties_; + } + + // Sets the elapsed time. + void set_elapsed_time(TimeInMillis elapsed) { elapsed_time_ = elapsed; } + + // Adds a test property to the list. The property is validated and may add + // a non-fatal failure if invalid (e.g., if it conflicts with reserved + // key names). If a property is already recorded for the same key, the + // value will be updated, rather than storing multiple values for the same + // key. xml_element specifies the element for which the property is being + // recorded and is used for validation. + void RecordProperty(const std::string& xml_element, + const TestProperty& test_property); + + // Adds a failure if the key is a reserved attribute of Google Test + // testcase tags. Returns true if the property is valid. + // FIXME: Validate attribute names are legal and human readable. + static bool ValidateTestProperty(const std::string& xml_element, + const TestProperty& test_property); + + // Adds a test part result to the list. + void AddTestPartResult(const TestPartResult& test_part_result); + + // Returns the death test count. + int death_test_count() const { return death_test_count_; } + + // Increments the death test count, returning the new count. + int increment_death_test_count() { return ++death_test_count_; } + + // Clears the test part results. + void ClearTestPartResults(); + + // Clears the object. + void Clear(); + + // Protects mutable state of the property vector and of owned + // properties, whose values may be updated. + internal::Mutex test_properites_mutex_; + + // The vector of TestPartResults + std::vector test_part_results_; + // The vector of TestProperties + std::vector test_properties_; + // Running count of death tests. + int death_test_count_; + // The elapsed time, in milliseconds. + TimeInMillis elapsed_time_; + + // We disallow copying TestResult. + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestResult); +}; // class TestResult + +// A TestInfo object stores the following information about a test: +// +// Test case name +// Test name +// Whether the test should be run +// A function pointer that creates the test object when invoked +// Test result +// +// The constructor of TestInfo registers itself with the UnitTest +// singleton such that the RUN_ALL_TESTS() macro knows which tests to +// run. +class GTEST_API_ TestInfo { + public: + // Destructs a TestInfo object. This function is not virtual, so + // don't inherit from TestInfo. + ~TestInfo(); + + // Returns the test case name. + const char* test_case_name() const { return test_case_name_.c_str(); } + + // Returns the test name. + const char* name() const { return name_.c_str(); } + + // Returns the name of the parameter type, or NULL if this is not a typed + // or a type-parameterized test. + const char* type_param() const { + if (type_param_.get() != NULL) + return type_param_->c_str(); + return NULL; + } + + // Returns the text representation of the value parameter, or NULL if this + // is not a value-parameterized test. + const char* value_param() const { + if (value_param_.get() != NULL) + return value_param_->c_str(); + return NULL; + } + + // Returns the file name where this test is defined. + const char* file() const { return location_.file.c_str(); } + + // Returns the line where this test is defined. + int line() const { return location_.line; } + + // Return true if this test should not be run because it's in another shard. + bool is_in_another_shard() const { return is_in_another_shard_; } + + // Returns true if this test should run, that is if the test is not + // disabled (or it is disabled but the also_run_disabled_tests flag has + // been specified) and its full name matches the user-specified filter. + // + // Google Test allows the user to filter the tests by their full names. + // The full name of a test Bar in test case Foo is defined as + // "Foo.Bar". Only the tests that match the filter will run. + // + // A filter is a colon-separated list of glob (not regex) patterns, + // optionally followed by a '-' and a colon-separated list of + // negative patterns (tests to exclude). A test is run if it + // matches one of the positive patterns and does not match any of + // the negative patterns. + // + // For example, *A*:Foo.* is a filter that matches any string that + // contains the character 'A' or starts with "Foo.". + bool should_run() const { return should_run_; } + + // Returns true iff this test will appear in the XML report. + bool is_reportable() const { + // The XML report includes tests matching the filter, excluding those + // run in other shards. + return matches_filter_ && !is_in_another_shard_; + } + + // Returns the result of the test. + const TestResult* result() const { return &result_; } + + private: +#if GTEST_HAS_DEATH_TEST + friend class internal::DefaultDeathTestFactory; +#endif // GTEST_HAS_DEATH_TEST + friend class Test; + friend class TestCase; + friend class internal::UnitTestImpl; + friend class internal::StreamingListenerTest; + friend TestInfo* internal::MakeAndRegisterTestInfo( + const char* test_case_name, + const char* name, + const char* type_param, + const char* value_param, + internal::CodeLocation code_location, + internal::TypeId fixture_class_id, + Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc, + internal::TestFactoryBase* factory); + + // Constructs a TestInfo object. The newly constructed instance assumes + // ownership of the factory object. + TestInfo(const std::string& test_case_name, + const std::string& name, + const char* a_type_param, // NULL if not a type-parameterized test + const char* a_value_param, // NULL if not a value-parameterized test + internal::CodeLocation a_code_location, + internal::TypeId fixture_class_id, + internal::TestFactoryBase* factory); + + // Increments the number of death tests encountered in this test so + // far. + int increment_death_test_count() { + return result_.increment_death_test_count(); + } + + // Creates the test object, runs it, records its result, and then + // deletes it. + void Run(); + + static void ClearTestResult(TestInfo* test_info) { + test_info->result_.Clear(); + } + + // These fields are immutable properties of the test. + const std::string test_case_name_; // Test case name + const std::string name_; // Test name + // Name of the parameter type, or NULL if this is not a typed or a + // type-parameterized test. + const internal::scoped_ptr type_param_; + // Text representation of the value parameter, or NULL if this is not a + // value-parameterized test. + const internal::scoped_ptr value_param_; + internal::CodeLocation location_; + const internal::TypeId fixture_class_id_; // ID of the test fixture class + bool should_run_; // True iff this test should run + bool is_disabled_; // True iff this test is disabled + bool matches_filter_; // True if this test matches the + // user-specified filter. + bool is_in_another_shard_; // Will be run in another shard. + internal::TestFactoryBase* const factory_; // The factory that creates + // the test object + + // This field is mutable and needs to be reset before running the + // test for the second time. + TestResult result_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestInfo); +}; + +// A test case, which consists of a vector of TestInfos. +// +// TestCase is not copyable. +class GTEST_API_ TestCase { + public: + // Creates a TestCase with the given name. + // + // TestCase does NOT have a default constructor. Always use this + // constructor to create a TestCase object. + // + // Arguments: + // + // name: name of the test case + // a_type_param: the name of the test's type parameter, or NULL if + // this is not a type-parameterized test. + // set_up_tc: pointer to the function that sets up the test case + // tear_down_tc: pointer to the function that tears down the test case + TestCase(const char* name, const char* a_type_param, + Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc); + + // Destructor of TestCase. + virtual ~TestCase(); + + // Gets the name of the TestCase. + const char* name() const { return name_.c_str(); } + + // Returns the name of the parameter type, or NULL if this is not a + // type-parameterized test case. + const char* type_param() const { + if (type_param_.get() != NULL) + return type_param_->c_str(); + return NULL; + } + + // Returns true if any test in this test case should run. + bool should_run() const { return should_run_; } + + // Gets the number of successful tests in this test case. + int successful_test_count() const; + + // Gets the number of failed tests in this test case. + int failed_test_count() const; + + // Gets the number of disabled tests that will be reported in the XML report. + int reportable_disabled_test_count() const; + + // Gets the number of disabled tests in this test case. + int disabled_test_count() const; + + // Gets the number of tests to be printed in the XML report. + int reportable_test_count() const; + + // Get the number of tests in this test case that should run. + int test_to_run_count() const; + + // Gets the number of all tests in this test case. + int total_test_count() const; + + // Returns true iff the test case passed. + bool Passed() const { return !Failed(); } + + // Returns true iff the test case failed. + bool Failed() const { return failed_test_count() > 0; } + + // Returns the elapsed time, in milliseconds. + TimeInMillis elapsed_time() const { return elapsed_time_; } + + // Returns the i-th test among all the tests. i can range from 0 to + // total_test_count() - 1. If i is not in that range, returns NULL. + const TestInfo* GetTestInfo(int i) const; + + // Returns the TestResult that holds test properties recorded during + // execution of SetUpTestCase and TearDownTestCase. + const TestResult& ad_hoc_test_result() const { return ad_hoc_test_result_; } + + private: + friend class Test; + friend class internal::UnitTestImpl; + + // Gets the (mutable) vector of TestInfos in this TestCase. + std::vector& test_info_list() { return test_info_list_; } + + // Gets the (immutable) vector of TestInfos in this TestCase. + const std::vector& test_info_list() const { + return test_info_list_; + } + + // Returns the i-th test among all the tests. i can range from 0 to + // total_test_count() - 1. If i is not in that range, returns NULL. + TestInfo* GetMutableTestInfo(int i); + + // Sets the should_run member. + void set_should_run(bool should) { should_run_ = should; } + + // Adds a TestInfo to this test case. Will delete the TestInfo upon + // destruction of the TestCase object. + void AddTestInfo(TestInfo * test_info); + + // Clears the results of all tests in this test case. + void ClearResult(); + + // Clears the results of all tests in the given test case. + static void ClearTestCaseResult(TestCase* test_case) { + test_case->ClearResult(); + } + + // Runs every test in this TestCase. + void Run(); + + // Runs SetUpTestCase() for this TestCase. This wrapper is needed + // for catching exceptions thrown from SetUpTestCase(). + void RunSetUpTestCase() { (*set_up_tc_)(); } + + // Runs TearDownTestCase() for this TestCase. This wrapper is + // needed for catching exceptions thrown from TearDownTestCase(). + void RunTearDownTestCase() { (*tear_down_tc_)(); } + + // Returns true iff test passed. + static bool TestPassed(const TestInfo* test_info) { + return test_info->should_run() && test_info->result()->Passed(); + } + + // Returns true iff test failed. + static bool TestFailed(const TestInfo* test_info) { + return test_info->should_run() && test_info->result()->Failed(); + } + + // Returns true iff the test is disabled and will be reported in the XML + // report. + static bool TestReportableDisabled(const TestInfo* test_info) { + return test_info->is_reportable() && test_info->is_disabled_; + } + + // Returns true iff test is disabled. + static bool TestDisabled(const TestInfo* test_info) { + return test_info->is_disabled_; + } + + // Returns true iff this test will appear in the XML report. + static bool TestReportable(const TestInfo* test_info) { + return test_info->is_reportable(); + } + + // Returns true if the given test should run. + static bool ShouldRunTest(const TestInfo* test_info) { + return test_info->should_run(); + } + + // Shuffles the tests in this test case. + void ShuffleTests(internal::Random* random); + + // Restores the test order to before the first shuffle. + void UnshuffleTests(); + + // Name of the test case. + std::string name_; + // Name of the parameter type, or NULL if this is not a typed or a + // type-parameterized test. + const internal::scoped_ptr type_param_; + // The vector of TestInfos in their original order. It owns the + // elements in the vector. + std::vector test_info_list_; + // Provides a level of indirection for the test list to allow easy + // shuffling and restoring the test order. The i-th element in this + // vector is the index of the i-th test in the shuffled test list. + std::vector test_indices_; + // Pointer to the function that sets up the test case. + Test::SetUpTestCaseFunc set_up_tc_; + // Pointer to the function that tears down the test case. + Test::TearDownTestCaseFunc tear_down_tc_; + // True iff any test in this test case should run. + bool should_run_; + // Elapsed time, in milliseconds. + TimeInMillis elapsed_time_; + // Holds test properties recorded during execution of SetUpTestCase and + // TearDownTestCase. + TestResult ad_hoc_test_result_; + + // We disallow copying TestCases. + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestCase); +}; + +// An Environment object is capable of setting up and tearing down an +// environment. You should subclass this to define your own +// environment(s). +// +// An Environment object does the set-up and tear-down in virtual +// methods SetUp() and TearDown() instead of the constructor and the +// destructor, as: +// +// 1. You cannot safely throw from a destructor. This is a problem +// as in some cases Google Test is used where exceptions are enabled, and +// we may want to implement ASSERT_* using exceptions where they are +// available. +// 2. You cannot use ASSERT_* directly in a constructor or +// destructor. +class Environment { + public: + // The d'tor is virtual as we need to subclass Environment. + virtual ~Environment() {} + + // Override this to define how to set up the environment. + virtual void SetUp() {} + + // Override this to define how to tear down the environment. + virtual void TearDown() {} + private: + // If you see an error about overriding the following function or + // about it being private, you have mis-spelled SetUp() as Setup(). + struct Setup_should_be_spelled_SetUp {}; + virtual Setup_should_be_spelled_SetUp* Setup() { return NULL; } +}; + +#if GTEST_HAS_EXCEPTIONS + +// Exception which can be thrown from TestEventListener::OnTestPartResult. +class GTEST_API_ AssertionException + : public internal::GoogleTestFailureException { + public: + explicit AssertionException(const TestPartResult& result) + : GoogleTestFailureException(result) {} +}; + +#endif // GTEST_HAS_EXCEPTIONS + +// The interface for tracing execution of tests. The methods are organized in +// the order the corresponding events are fired. +class TestEventListener { + public: + virtual ~TestEventListener() {} + + // Fired before any test activity starts. + virtual void OnTestProgramStart(const UnitTest& unit_test) = 0; + + // Fired before each iteration of tests starts. There may be more than + // one iteration if GTEST_FLAG(repeat) is set. iteration is the iteration + // index, starting from 0. + virtual void OnTestIterationStart(const UnitTest& unit_test, + int iteration) = 0; + + // Fired before environment set-up for each iteration of tests starts. + virtual void OnEnvironmentsSetUpStart(const UnitTest& unit_test) = 0; + + // Fired after environment set-up for each iteration of tests ends. + virtual void OnEnvironmentsSetUpEnd(const UnitTest& unit_test) = 0; + + // Fired before the test case starts. + virtual void OnTestCaseStart(const TestCase& test_case) = 0; + + // Fired before the test starts. + virtual void OnTestStart(const TestInfo& test_info) = 0; + + // Fired after a failed assertion or a SUCCEED() invocation. + // If you want to throw an exception from this function to skip to the next + // TEST, it must be AssertionException defined above, or inherited from it. + virtual void OnTestPartResult(const TestPartResult& test_part_result) = 0; + + // Fired after the test ends. + virtual void OnTestEnd(const TestInfo& test_info) = 0; + + // Fired after the test case ends. + virtual void OnTestCaseEnd(const TestCase& test_case) = 0; + + // Fired before environment tear-down for each iteration of tests starts. + virtual void OnEnvironmentsTearDownStart(const UnitTest& unit_test) = 0; + + // Fired after environment tear-down for each iteration of tests ends. + virtual void OnEnvironmentsTearDownEnd(const UnitTest& unit_test) = 0; + + // Fired after each iteration of tests finishes. + virtual void OnTestIterationEnd(const UnitTest& unit_test, + int iteration) = 0; + + // Fired after all test activities have ended. + virtual void OnTestProgramEnd(const UnitTest& unit_test) = 0; +}; + +// The convenience class for users who need to override just one or two +// methods and are not concerned that a possible change to a signature of +// the methods they override will not be caught during the build. For +// comments about each method please see the definition of TestEventListener +// above. +class EmptyTestEventListener : public TestEventListener { + public: + virtual void OnTestProgramStart(const UnitTest& /*unit_test*/) {} + virtual void OnTestIterationStart(const UnitTest& /*unit_test*/, + int /*iteration*/) {} + virtual void OnEnvironmentsSetUpStart(const UnitTest& /*unit_test*/) {} + virtual void OnEnvironmentsSetUpEnd(const UnitTest& /*unit_test*/) {} + virtual void OnTestCaseStart(const TestCase& /*test_case*/) {} + virtual void OnTestStart(const TestInfo& /*test_info*/) {} + virtual void OnTestPartResult(const TestPartResult& /*test_part_result*/) {} + virtual void OnTestEnd(const TestInfo& /*test_info*/) {} + virtual void OnTestCaseEnd(const TestCase& /*test_case*/) {} + virtual void OnEnvironmentsTearDownStart(const UnitTest& /*unit_test*/) {} + virtual void OnEnvironmentsTearDownEnd(const UnitTest& /*unit_test*/) {} + virtual void OnTestIterationEnd(const UnitTest& /*unit_test*/, + int /*iteration*/) {} + virtual void OnTestProgramEnd(const UnitTest& /*unit_test*/) {} +}; + +// TestEventListeners lets users add listeners to track events in Google Test. +class GTEST_API_ TestEventListeners { + public: + TestEventListeners(); + ~TestEventListeners(); + + // Appends an event listener to the end of the list. Google Test assumes + // the ownership of the listener (i.e. it will delete the listener when + // the test program finishes). + void Append(TestEventListener* listener); + + // Removes the given event listener from the list and returns it. It then + // becomes the caller's responsibility to delete the listener. Returns + // NULL if the listener is not found in the list. + TestEventListener* Release(TestEventListener* listener); + + // Returns the standard listener responsible for the default console + // output. Can be removed from the listeners list to shut down default + // console output. Note that removing this object from the listener list + // with Release transfers its ownership to the caller and makes this + // function return NULL the next time. + TestEventListener* default_result_printer() const { + return default_result_printer_; + } + + // Returns the standard listener responsible for the default XML output + // controlled by the --gtest_output=xml flag. Can be removed from the + // listeners list by users who want to shut down the default XML output + // controlled by this flag and substitute it with custom one. Note that + // removing this object from the listener list with Release transfers its + // ownership to the caller and makes this function return NULL the next + // time. + TestEventListener* default_xml_generator() const { + return default_xml_generator_; + } + + private: + friend class TestCase; + friend class TestInfo; + friend class internal::DefaultGlobalTestPartResultReporter; + friend class internal::NoExecDeathTest; + friend class internal::TestEventListenersAccessor; + friend class internal::UnitTestImpl; + + // Returns repeater that broadcasts the TestEventListener events to all + // subscribers. + TestEventListener* repeater(); + + // Sets the default_result_printer attribute to the provided listener. + // The listener is also added to the listener list and previous + // default_result_printer is removed from it and deleted. The listener can + // also be NULL in which case it will not be added to the list. Does + // nothing if the previous and the current listener objects are the same. + void SetDefaultResultPrinter(TestEventListener* listener); + + // Sets the default_xml_generator attribute to the provided listener. The + // listener is also added to the listener list and previous + // default_xml_generator is removed from it and deleted. The listener can + // also be NULL in which case it will not be added to the list. Does + // nothing if the previous and the current listener objects are the same. + void SetDefaultXmlGenerator(TestEventListener* listener); + + // Controls whether events will be forwarded by the repeater to the + // listeners in the list. + bool EventForwardingEnabled() const; + void SuppressEventForwarding(); + + // The actual list of listeners. + internal::TestEventRepeater* repeater_; + // Listener responsible for the standard result output. + TestEventListener* default_result_printer_; + // Listener responsible for the creation of the XML output file. + TestEventListener* default_xml_generator_; + + // We disallow copying TestEventListeners. + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestEventListeners); +}; + +// A UnitTest consists of a vector of TestCases. +// +// This is a singleton class. The only instance of UnitTest is +// created when UnitTest::GetInstance() is first called. This +// instance is never deleted. +// +// UnitTest is not copyable. +// +// This class is thread-safe as long as the methods are called +// according to their specification. +class GTEST_API_ UnitTest { + public: + // Gets the singleton UnitTest object. The first time this method + // is called, a UnitTest object is constructed and returned. + // Consecutive calls will return the same object. + static UnitTest* GetInstance(); + + // Runs all tests in this UnitTest object and prints the result. + // Returns 0 if successful, or 1 otherwise. + // + // This method can only be called from the main thread. + // + // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + int Run() GTEST_MUST_USE_RESULT_; + + // Returns the working directory when the first TEST() or TEST_F() + // was executed. The UnitTest object owns the string. + const char* original_working_dir() const; + + // Returns the TestCase object for the test that's currently running, + // or NULL if no test is running. + const TestCase* current_test_case() const + GTEST_LOCK_EXCLUDED_(mutex_); + + // Returns the TestInfo object for the test that's currently running, + // or NULL if no test is running. + const TestInfo* current_test_info() const + GTEST_LOCK_EXCLUDED_(mutex_); + + // Returns the random seed used at the start of the current test run. + int random_seed() const; + + // Returns the ParameterizedTestCaseRegistry object used to keep track of + // value-parameterized tests and instantiate and register them. + // + // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + internal::ParameterizedTestCaseRegistry& parameterized_test_registry() + GTEST_LOCK_EXCLUDED_(mutex_); + + // Gets the number of successful test cases. + int successful_test_case_count() const; + + // Gets the number of failed test cases. + int failed_test_case_count() const; + + // Gets the number of all test cases. + int total_test_case_count() const; + + // Gets the number of all test cases that contain at least one test + // that should run. + int test_case_to_run_count() const; + + // Gets the number of successful tests. + int successful_test_count() const; + + // Gets the number of failed tests. + int failed_test_count() const; + + // Gets the number of disabled tests that will be reported in the XML report. + int reportable_disabled_test_count() const; + + // Gets the number of disabled tests. + int disabled_test_count() const; + + // Gets the number of tests to be printed in the XML report. + int reportable_test_count() const; + + // Gets the number of all tests. + int total_test_count() const; + + // Gets the number of tests that should run. + int test_to_run_count() const; + + // Gets the time of the test program start, in ms from the start of the + // UNIX epoch. + TimeInMillis start_timestamp() const; + + // Gets the elapsed time, in milliseconds. + TimeInMillis elapsed_time() const; + + // Returns true iff the unit test passed (i.e. all test cases passed). + bool Passed() const; + + // Returns true iff the unit test failed (i.e. some test case failed + // or something outside of all tests failed). + bool Failed() const; + + // Gets the i-th test case among all the test cases. i can range from 0 to + // total_test_case_count() - 1. If i is not in that range, returns NULL. + const TestCase* GetTestCase(int i) const; + + // Returns the TestResult containing information on test failures and + // properties logged outside of individual test cases. + const TestResult& ad_hoc_test_result() const; + + // Returns the list of event listeners that can be used to track events + // inside Google Test. + TestEventListeners& listeners(); + + private: + // Registers and returns a global test environment. When a test + // program is run, all global test environments will be set-up in + // the order they were registered. After all tests in the program + // have finished, all global test environments will be torn-down in + // the *reverse* order they were registered. + // + // The UnitTest object takes ownership of the given environment. + // + // This method can only be called from the main thread. + Environment* AddEnvironment(Environment* env); + + // Adds a TestPartResult to the current TestResult object. All + // Google Test assertion macros (e.g. ASSERT_TRUE, EXPECT_EQ, etc) + // eventually call this to report their results. The user code + // should use the assertion macros instead of calling this directly. + void AddTestPartResult(TestPartResult::Type result_type, + const char* file_name, + int line_number, + const std::string& message, + const std::string& os_stack_trace) + GTEST_LOCK_EXCLUDED_(mutex_); + + // Adds a TestProperty to the current TestResult object when invoked from + // inside a test, to current TestCase's ad_hoc_test_result_ when invoked + // from SetUpTestCase or TearDownTestCase, or to the global property set + // when invoked elsewhere. If the result already contains a property with + // the same key, the value will be updated. + void RecordProperty(const std::string& key, const std::string& value); + + // Gets the i-th test case among all the test cases. i can range from 0 to + // total_test_case_count() - 1. If i is not in that range, returns NULL. + TestCase* GetMutableTestCase(int i); + + // Accessors for the implementation object. + internal::UnitTestImpl* impl() { return impl_; } + const internal::UnitTestImpl* impl() const { return impl_; } + + // These classes and functions are friends as they need to access private + // members of UnitTest. + friend class ScopedTrace; + friend class Test; + friend class internal::AssertHelper; + friend class internal::StreamingListenerTest; + friend class internal::UnitTestRecordPropertyTestHelper; + friend Environment* AddGlobalTestEnvironment(Environment* env); + friend internal::UnitTestImpl* internal::GetUnitTestImpl(); + friend void internal::ReportFailureInUnknownLocation( + TestPartResult::Type result_type, + const std::string& message); + + // Creates an empty UnitTest. + UnitTest(); + + // D'tor + virtual ~UnitTest(); + + // Pushes a trace defined by SCOPED_TRACE() on to the per-thread + // Google Test trace stack. + void PushGTestTrace(const internal::TraceInfo& trace) + GTEST_LOCK_EXCLUDED_(mutex_); + + // Pops a trace from the per-thread Google Test trace stack. + void PopGTestTrace() + GTEST_LOCK_EXCLUDED_(mutex_); + + // Protects mutable state in *impl_. This is mutable as some const + // methods need to lock it too. + mutable internal::Mutex mutex_; + + // Opaque implementation object. This field is never changed once + // the object is constructed. We don't mark it as const here, as + // doing so will cause a warning in the constructor of UnitTest. + // Mutable state in *impl_ is protected by mutex_. + internal::UnitTestImpl* impl_; + + // We disallow copying UnitTest. + GTEST_DISALLOW_COPY_AND_ASSIGN_(UnitTest); +}; + +// A convenient wrapper for adding an environment for the test +// program. +// +// You should call this before RUN_ALL_TESTS() is called, probably in +// main(). If you use gtest_main, you need to call this before main() +// starts for it to take effect. For example, you can define a global +// variable like this: +// +// testing::Environment* const foo_env = +// testing::AddGlobalTestEnvironment(new FooEnvironment); +// +// However, we strongly recommend you to write your own main() and +// call AddGlobalTestEnvironment() there, as relying on initialization +// of global variables makes the code harder to read and may cause +// problems when you register multiple environments from different +// translation units and the environments have dependencies among them +// (remember that the compiler doesn't guarantee the order in which +// global variables from different translation units are initialized). +inline Environment* AddGlobalTestEnvironment(Environment* env) { + return UnitTest::GetInstance()->AddEnvironment(env); +} + +// Initializes Google Test. This must be called before calling +// RUN_ALL_TESTS(). In particular, it parses a command line for the +// flags that Google Test recognizes. Whenever a Google Test flag is +// seen, it is removed from argv, and *argc is decremented. +// +// No value is returned. Instead, the Google Test flag variables are +// updated. +// +// Calling the function for the second time has no user-visible effect. +GTEST_API_ void InitGoogleTest(int* argc, char** argv); + +// This overloaded version can be used in Windows programs compiled in +// UNICODE mode. +GTEST_API_ void InitGoogleTest(int* argc, wchar_t** argv); + +namespace internal { + +// Separate the error generating code from the code path to reduce the stack +// frame size of CmpHelperEQ. This helps reduce the overhead of some sanitizers +// when calling EXPECT_* in a tight loop. +template +AssertionResult CmpHelperEQFailure(const char* lhs_expression, + const char* rhs_expression, + const T1& lhs, const T2& rhs) { + return EqFailure(lhs_expression, + rhs_expression, + FormatForComparisonFailureMessage(lhs, rhs), + FormatForComparisonFailureMessage(rhs, lhs), + false); +} + +// The helper function for {ASSERT|EXPECT}_EQ. +template +AssertionResult CmpHelperEQ(const char* lhs_expression, + const char* rhs_expression, + const T1& lhs, + const T2& rhs) { + if (lhs == rhs) { + return AssertionSuccess(); + } + + return CmpHelperEQFailure(lhs_expression, rhs_expression, lhs, rhs); +} + +// With this overloaded version, we allow anonymous enums to be used +// in {ASSERT|EXPECT}_EQ when compiled with gcc 4, as anonymous enums +// can be implicitly cast to BiggestInt. +GTEST_API_ AssertionResult CmpHelperEQ(const char* lhs_expression, + const char* rhs_expression, + BiggestInt lhs, + BiggestInt rhs); + +// The helper class for {ASSERT|EXPECT}_EQ. The template argument +// lhs_is_null_literal is true iff the first argument to ASSERT_EQ() +// is a null pointer literal. The following default implementation is +// for lhs_is_null_literal being false. +template +class EqHelper { + public: + // This templatized version is for the general case. + template + static AssertionResult Compare(const char* lhs_expression, + const char* rhs_expression, + const T1& lhs, + const T2& rhs) { + return CmpHelperEQ(lhs_expression, rhs_expression, lhs, rhs); + } + + // With this overloaded version, we allow anonymous enums to be used + // in {ASSERT|EXPECT}_EQ when compiled with gcc 4, as anonymous + // enums can be implicitly cast to BiggestInt. + // + // Even though its body looks the same as the above version, we + // cannot merge the two, as it will make anonymous enums unhappy. + static AssertionResult Compare(const char* lhs_expression, + const char* rhs_expression, + BiggestInt lhs, + BiggestInt rhs) { + return CmpHelperEQ(lhs_expression, rhs_expression, lhs, rhs); + } +}; + +// This specialization is used when the first argument to ASSERT_EQ() +// is a null pointer literal, like NULL, false, or 0. +template <> +class EqHelper { + public: + // We define two overloaded versions of Compare(). The first + // version will be picked when the second argument to ASSERT_EQ() is + // NOT a pointer, e.g. ASSERT_EQ(0, AnIntFunction()) or + // EXPECT_EQ(false, a_bool). + template + static AssertionResult Compare( + const char* lhs_expression, + const char* rhs_expression, + const T1& lhs, + const T2& rhs, + // The following line prevents this overload from being considered if T2 + // is not a pointer type. We need this because ASSERT_EQ(NULL, my_ptr) + // expands to Compare("", "", NULL, my_ptr), which requires a conversion + // to match the Secret* in the other overload, which would otherwise make + // this template match better. + typename EnableIf::value>::type* = 0) { + return CmpHelperEQ(lhs_expression, rhs_expression, lhs, rhs); + } + + // This version will be picked when the second argument to ASSERT_EQ() is a + // pointer, e.g. ASSERT_EQ(NULL, a_pointer). + template + static AssertionResult Compare( + const char* lhs_expression, + const char* rhs_expression, + // We used to have a second template parameter instead of Secret*. That + // template parameter would deduce to 'long', making this a better match + // than the first overload even without the first overload's EnableIf. + // Unfortunately, gcc with -Wconversion-null warns when "passing NULL to + // non-pointer argument" (even a deduced integral argument), so the old + // implementation caused warnings in user code. + Secret* /* lhs (NULL) */, + T* rhs) { + // We already know that 'lhs' is a null pointer. + return CmpHelperEQ(lhs_expression, rhs_expression, + static_cast(NULL), rhs); + } +}; + +// Separate the error generating code from the code path to reduce the stack +// frame size of CmpHelperOP. This helps reduce the overhead of some sanitizers +// when calling EXPECT_OP in a tight loop. +template +AssertionResult CmpHelperOpFailure(const char* expr1, const char* expr2, + const T1& val1, const T2& val2, + const char* op) { + return AssertionFailure() + << "Expected: (" << expr1 << ") " << op << " (" << expr2 + << "), actual: " << FormatForComparisonFailureMessage(val1, val2) + << " vs " << FormatForComparisonFailureMessage(val2, val1); +} + +// A macro for implementing the helper functions needed to implement +// ASSERT_?? and EXPECT_??. It is here just to avoid copy-and-paste +// of similar code. +// +// For each templatized helper function, we also define an overloaded +// version for BiggestInt in order to reduce code bloat and allow +// anonymous enums to be used with {ASSERT|EXPECT}_?? when compiled +// with gcc 4. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + +#define GTEST_IMPL_CMP_HELPER_(op_name, op)\ +template \ +AssertionResult CmpHelper##op_name(const char* expr1, const char* expr2, \ + const T1& val1, const T2& val2) {\ + if (val1 op val2) {\ + return AssertionSuccess();\ + } else {\ + return CmpHelperOpFailure(expr1, expr2, val1, val2, #op);\ + }\ +}\ +GTEST_API_ AssertionResult CmpHelper##op_name(\ + const char* expr1, const char* expr2, BiggestInt val1, BiggestInt val2) + +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + +// Implements the helper function for {ASSERT|EXPECT}_NE +GTEST_IMPL_CMP_HELPER_(NE, !=); +// Implements the helper function for {ASSERT|EXPECT}_LE +GTEST_IMPL_CMP_HELPER_(LE, <=); +// Implements the helper function for {ASSERT|EXPECT}_LT +GTEST_IMPL_CMP_HELPER_(LT, <); +// Implements the helper function for {ASSERT|EXPECT}_GE +GTEST_IMPL_CMP_HELPER_(GE, >=); +// Implements the helper function for {ASSERT|EXPECT}_GT +GTEST_IMPL_CMP_HELPER_(GT, >); + +#undef GTEST_IMPL_CMP_HELPER_ + +// The helper function for {ASSERT|EXPECT}_STREQ. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTREQ(const char* s1_expression, + const char* s2_expression, + const char* s1, + const char* s2); + +// The helper function for {ASSERT|EXPECT}_STRCASEEQ. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTRCASEEQ(const char* s1_expression, + const char* s2_expression, + const char* s1, + const char* s2); + +// The helper function for {ASSERT|EXPECT}_STRNE. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTRNE(const char* s1_expression, + const char* s2_expression, + const char* s1, + const char* s2); + +// The helper function for {ASSERT|EXPECT}_STRCASENE. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTRCASENE(const char* s1_expression, + const char* s2_expression, + const char* s1, + const char* s2); + + +// Helper function for *_STREQ on wide strings. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTREQ(const char* s1_expression, + const char* s2_expression, + const wchar_t* s1, + const wchar_t* s2); + +// Helper function for *_STRNE on wide strings. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTRNE(const char* s1_expression, + const char* s2_expression, + const wchar_t* s1, + const wchar_t* s2); + +} // namespace internal + +// IsSubstring() and IsNotSubstring() are intended to be used as the +// first argument to {EXPECT,ASSERT}_PRED_FORMAT2(), not by +// themselves. They check whether needle is a substring of haystack +// (NULL is considered a substring of itself only), and return an +// appropriate error message when they fail. +// +// The {needle,haystack}_expr arguments are the stringified +// expressions that generated the two real arguments. +GTEST_API_ AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const char* needle, const char* haystack); +GTEST_API_ AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const wchar_t* needle, const wchar_t* haystack); +GTEST_API_ AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const char* needle, const char* haystack); +GTEST_API_ AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const wchar_t* needle, const wchar_t* haystack); +GTEST_API_ AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::string& needle, const ::std::string& haystack); +GTEST_API_ AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::string& needle, const ::std::string& haystack); + +#if GTEST_HAS_STD_WSTRING +GTEST_API_ AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::wstring& needle, const ::std::wstring& haystack); +GTEST_API_ AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::wstring& needle, const ::std::wstring& haystack); +#endif // GTEST_HAS_STD_WSTRING + +namespace internal { + +// Helper template function for comparing floating-points. +// +// Template parameter: +// +// RawType: the raw floating-point type (either float or double) +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +template +AssertionResult CmpHelperFloatingPointEQ(const char* lhs_expression, + const char* rhs_expression, + RawType lhs_value, + RawType rhs_value) { + const FloatingPoint lhs(lhs_value), rhs(rhs_value); + + if (lhs.AlmostEquals(rhs)) { + return AssertionSuccess(); + } + + ::std::stringstream lhs_ss; + lhs_ss << std::setprecision(std::numeric_limits::digits10 + 2) + << lhs_value; + + ::std::stringstream rhs_ss; + rhs_ss << std::setprecision(std::numeric_limits::digits10 + 2) + << rhs_value; + + return EqFailure(lhs_expression, + rhs_expression, + StringStreamToString(&lhs_ss), + StringStreamToString(&rhs_ss), + false); +} + +// Helper function for implementing ASSERT_NEAR. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult DoubleNearPredFormat(const char* expr1, + const char* expr2, + const char* abs_error_expr, + double val1, + double val2, + double abs_error); + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// A class that enables one to stream messages to assertion macros +class GTEST_API_ AssertHelper { + public: + // Constructor. + AssertHelper(TestPartResult::Type type, + const char* file, + int line, + const char* message); + ~AssertHelper(); + + // Message assignment is a semantic trick to enable assertion + // streaming; see the GTEST_MESSAGE_ macro below. + void operator=(const Message& message) const; + + private: + // We put our data in a struct so that the size of the AssertHelper class can + // be as small as possible. This is important because gcc is incapable of + // re-using stack space even for temporary variables, so every EXPECT_EQ + // reserves stack space for another AssertHelper. + struct AssertHelperData { + AssertHelperData(TestPartResult::Type t, + const char* srcfile, + int line_num, + const char* msg) + : type(t), file(srcfile), line(line_num), message(msg) { } + + TestPartResult::Type const type; + const char* const file; + int const line; + std::string const message; + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(AssertHelperData); + }; + + AssertHelperData* const data_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(AssertHelper); +}; + +} // namespace internal + +// The pure interface class that all value-parameterized tests inherit from. +// A value-parameterized class must inherit from both ::testing::Test and +// ::testing::WithParamInterface. In most cases that just means inheriting +// from ::testing::TestWithParam, but more complicated test hierarchies +// may need to inherit from Test and WithParamInterface at different levels. +// +// This interface has support for accessing the test parameter value via +// the GetParam() method. +// +// Use it with one of the parameter generator defining functions, like Range(), +// Values(), ValuesIn(), Bool(), and Combine(). +// +// class FooTest : public ::testing::TestWithParam { +// protected: +// FooTest() { +// // Can use GetParam() here. +// } +// virtual ~FooTest() { +// // Can use GetParam() here. +// } +// virtual void SetUp() { +// // Can use GetParam() here. +// } +// virtual void TearDown { +// // Can use GetParam() here. +// } +// }; +// TEST_P(FooTest, DoesBar) { +// // Can use GetParam() method here. +// Foo foo; +// ASSERT_TRUE(foo.DoesBar(GetParam())); +// } +// INSTANTIATE_TEST_CASE_P(OneToTenRange, FooTest, ::testing::Range(1, 10)); + +template +class WithParamInterface { + public: + typedef T ParamType; + virtual ~WithParamInterface() {} + + // The current parameter value. Is also available in the test fixture's + // constructor. This member function is non-static, even though it only + // references static data, to reduce the opportunity for incorrect uses + // like writing 'WithParamInterface::GetParam()' for a test that + // uses a fixture whose parameter type is int. + const ParamType& GetParam() const { + GTEST_CHECK_(parameter_ != NULL) + << "GetParam() can only be called inside a value-parameterized test " + << "-- did you intend to write TEST_P instead of TEST_F?"; + return *parameter_; + } + + private: + // Sets parameter value. The caller is responsible for making sure the value + // remains alive and unchanged throughout the current test. + static void SetParam(const ParamType* parameter) { + parameter_ = parameter; + } + + // Static value used for accessing parameter during a test lifetime. + static const ParamType* parameter_; + + // TestClass must be a subclass of WithParamInterface and Test. + template friend class internal::ParameterizedTestFactory; +}; + +template +const T* WithParamInterface::parameter_ = NULL; + +// Most value-parameterized classes can ignore the existence of +// WithParamInterface, and can just inherit from ::testing::TestWithParam. + +template +class TestWithParam : public Test, public WithParamInterface { +}; + +// Macros for indicating success/failure in test code. + +// ADD_FAILURE unconditionally adds a failure to the current test. +// SUCCEED generates a success - it doesn't automatically make the +// current test successful, as a test is only successful when it has +// no failure. +// +// EXPECT_* verifies that a certain condition is satisfied. If not, +// it behaves like ADD_FAILURE. In particular: +// +// EXPECT_TRUE verifies that a Boolean condition is true. +// EXPECT_FALSE verifies that a Boolean condition is false. +// +// FAIL and ASSERT_* are similar to ADD_FAILURE and EXPECT_*, except +// that they will also abort the current function on failure. People +// usually want the fail-fast behavior of FAIL and ASSERT_*, but those +// writing data-driven tests often find themselves using ADD_FAILURE +// and EXPECT_* more. + +// Generates a nonfatal failure with a generic message. +#define ADD_FAILURE() GTEST_NONFATAL_FAILURE_("Failed") + +// Generates a nonfatal failure at the given source file location with +// a generic message. +#define ADD_FAILURE_AT(file, line) \ + GTEST_MESSAGE_AT_(file, line, "Failed", \ + ::testing::TestPartResult::kNonFatalFailure) + +// Generates a fatal failure with a generic message. +#define GTEST_FAIL() GTEST_FATAL_FAILURE_("Failed") + +// Define this macro to 1 to omit the definition of FAIL(), which is a +// generic name and clashes with some other libraries. +#if !GTEST_DONT_DEFINE_FAIL +# define FAIL() GTEST_FAIL() +#endif + +// Generates a success with a generic message. +#define GTEST_SUCCEED() GTEST_SUCCESS_("Succeeded") + +// Define this macro to 1 to omit the definition of SUCCEED(), which +// is a generic name and clashes with some other libraries. +#if !GTEST_DONT_DEFINE_SUCCEED +# define SUCCEED() GTEST_SUCCEED() +#endif + +// Macros for testing exceptions. +// +// * {ASSERT|EXPECT}_THROW(statement, expected_exception): +// Tests that the statement throws the expected exception. +// * {ASSERT|EXPECT}_NO_THROW(statement): +// Tests that the statement doesn't throw any exception. +// * {ASSERT|EXPECT}_ANY_THROW(statement): +// Tests that the statement throws an exception. + +#define EXPECT_THROW(statement, expected_exception) \ + GTEST_TEST_THROW_(statement, expected_exception, GTEST_NONFATAL_FAILURE_) +#define EXPECT_NO_THROW(statement) \ + GTEST_TEST_NO_THROW_(statement, GTEST_NONFATAL_FAILURE_) +#define EXPECT_ANY_THROW(statement) \ + GTEST_TEST_ANY_THROW_(statement, GTEST_NONFATAL_FAILURE_) +#define ASSERT_THROW(statement, expected_exception) \ + GTEST_TEST_THROW_(statement, expected_exception, GTEST_FATAL_FAILURE_) +#define ASSERT_NO_THROW(statement) \ + GTEST_TEST_NO_THROW_(statement, GTEST_FATAL_FAILURE_) +#define ASSERT_ANY_THROW(statement) \ + GTEST_TEST_ANY_THROW_(statement, GTEST_FATAL_FAILURE_) + +// Boolean assertions. Condition can be either a Boolean expression or an +// AssertionResult. For more information on how to use AssertionResult with +// these macros see comments on that class. +#define EXPECT_TRUE(condition) \ + GTEST_TEST_BOOLEAN_(condition, #condition, false, true, \ + GTEST_NONFATAL_FAILURE_) +#define EXPECT_FALSE(condition) \ + GTEST_TEST_BOOLEAN_(!(condition), #condition, true, false, \ + GTEST_NONFATAL_FAILURE_) +#define ASSERT_TRUE(condition) \ + GTEST_TEST_BOOLEAN_(condition, #condition, false, true, \ + GTEST_FATAL_FAILURE_) +#define ASSERT_FALSE(condition) \ + GTEST_TEST_BOOLEAN_(!(condition), #condition, true, false, \ + GTEST_FATAL_FAILURE_) + +// Macros for testing equalities and inequalities. +// +// * {ASSERT|EXPECT}_EQ(v1, v2): Tests that v1 == v2 +// * {ASSERT|EXPECT}_NE(v1, v2): Tests that v1 != v2 +// * {ASSERT|EXPECT}_LT(v1, v2): Tests that v1 < v2 +// * {ASSERT|EXPECT}_LE(v1, v2): Tests that v1 <= v2 +// * {ASSERT|EXPECT}_GT(v1, v2): Tests that v1 > v2 +// * {ASSERT|EXPECT}_GE(v1, v2): Tests that v1 >= v2 +// +// When they are not, Google Test prints both the tested expressions and +// their actual values. The values must be compatible built-in types, +// or you will get a compiler error. By "compatible" we mean that the +// values can be compared by the respective operator. +// +// Note: +// +// 1. It is possible to make a user-defined type work with +// {ASSERT|EXPECT}_??(), but that requires overloading the +// comparison operators and is thus discouraged by the Google C++ +// Usage Guide. Therefore, you are advised to use the +// {ASSERT|EXPECT}_TRUE() macro to assert that two objects are +// equal. +// +// 2. The {ASSERT|EXPECT}_??() macros do pointer comparisons on +// pointers (in particular, C strings). Therefore, if you use it +// with two C strings, you are testing how their locations in memory +// are related, not how their content is related. To compare two C +// strings by content, use {ASSERT|EXPECT}_STR*(). +// +// 3. {ASSERT|EXPECT}_EQ(v1, v2) is preferred to +// {ASSERT|EXPECT}_TRUE(v1 == v2), as the former tells you +// what the actual value is when it fails, and similarly for the +// other comparisons. +// +// 4. Do not depend on the order in which {ASSERT|EXPECT}_??() +// evaluate their arguments, which is undefined. +// +// 5. These macros evaluate their arguments exactly once. +// +// Examples: +// +// EXPECT_NE(Foo(), 5); +// EXPECT_EQ(a_pointer, NULL); +// ASSERT_LT(i, array_size); +// ASSERT_GT(records.size(), 0) << "There is no record left."; + +#define EXPECT_EQ(val1, val2) \ + EXPECT_PRED_FORMAT2(::testing::internal:: \ + EqHelper::Compare, \ + val1, val2) +#define EXPECT_NE(val1, val2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperNE, val1, val2) +#define EXPECT_LE(val1, val2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperLE, val1, val2) +#define EXPECT_LT(val1, val2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperLT, val1, val2) +#define EXPECT_GE(val1, val2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperGE, val1, val2) +#define EXPECT_GT(val1, val2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperGT, val1, val2) + +#define GTEST_ASSERT_EQ(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal:: \ + EqHelper::Compare, \ + val1, val2) +#define GTEST_ASSERT_NE(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperNE, val1, val2) +#define GTEST_ASSERT_LE(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperLE, val1, val2) +#define GTEST_ASSERT_LT(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperLT, val1, val2) +#define GTEST_ASSERT_GE(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperGE, val1, val2) +#define GTEST_ASSERT_GT(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperGT, val1, val2) + +// Define macro GTEST_DONT_DEFINE_ASSERT_XY to 1 to omit the definition of +// ASSERT_XY(), which clashes with some users' own code. + +#if !GTEST_DONT_DEFINE_ASSERT_EQ +# define ASSERT_EQ(val1, val2) GTEST_ASSERT_EQ(val1, val2) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_NE +# define ASSERT_NE(val1, val2) GTEST_ASSERT_NE(val1, val2) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_LE +# define ASSERT_LE(val1, val2) GTEST_ASSERT_LE(val1, val2) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_LT +# define ASSERT_LT(val1, val2) GTEST_ASSERT_LT(val1, val2) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_GE +# define ASSERT_GE(val1, val2) GTEST_ASSERT_GE(val1, val2) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_GT +# define ASSERT_GT(val1, val2) GTEST_ASSERT_GT(val1, val2) +#endif + +// C-string Comparisons. All tests treat NULL and any non-NULL string +// as different. Two NULLs are equal. +// +// * {ASSERT|EXPECT}_STREQ(s1, s2): Tests that s1 == s2 +// * {ASSERT|EXPECT}_STRNE(s1, s2): Tests that s1 != s2 +// * {ASSERT|EXPECT}_STRCASEEQ(s1, s2): Tests that s1 == s2, ignoring case +// * {ASSERT|EXPECT}_STRCASENE(s1, s2): Tests that s1 != s2, ignoring case +// +// For wide or narrow string objects, you can use the +// {ASSERT|EXPECT}_??() macros. +// +// Don't depend on the order in which the arguments are evaluated, +// which is undefined. +// +// These macros evaluate their arguments exactly once. + +#define EXPECT_STREQ(s1, s2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperSTREQ, s1, s2) +#define EXPECT_STRNE(s1, s2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperSTRNE, s1, s2) +#define EXPECT_STRCASEEQ(s1, s2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperSTRCASEEQ, s1, s2) +#define EXPECT_STRCASENE(s1, s2)\ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperSTRCASENE, s1, s2) + +#define ASSERT_STREQ(s1, s2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperSTREQ, s1, s2) +#define ASSERT_STRNE(s1, s2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperSTRNE, s1, s2) +#define ASSERT_STRCASEEQ(s1, s2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperSTRCASEEQ, s1, s2) +#define ASSERT_STRCASENE(s1, s2)\ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperSTRCASENE, s1, s2) + +// Macros for comparing floating-point numbers. +// +// * {ASSERT|EXPECT}_FLOAT_EQ(val1, val2): +// Tests that two float values are almost equal. +// * {ASSERT|EXPECT}_DOUBLE_EQ(val1, val2): +// Tests that two double values are almost equal. +// * {ASSERT|EXPECT}_NEAR(v1, v2, abs_error): +// Tests that v1 and v2 are within the given distance to each other. +// +// Google Test uses ULP-based comparison to automatically pick a default +// error bound that is appropriate for the operands. See the +// FloatingPoint template class in gtest-internal.h if you are +// interested in the implementation details. + +#define EXPECT_FLOAT_EQ(val1, val2)\ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ + val1, val2) + +#define EXPECT_DOUBLE_EQ(val1, val2)\ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ + val1, val2) + +#define ASSERT_FLOAT_EQ(val1, val2)\ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ + val1, val2) + +#define ASSERT_DOUBLE_EQ(val1, val2)\ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ + val1, val2) + +#define EXPECT_NEAR(val1, val2, abs_error)\ + EXPECT_PRED_FORMAT3(::testing::internal::DoubleNearPredFormat, \ + val1, val2, abs_error) + +#define ASSERT_NEAR(val1, val2, abs_error)\ + ASSERT_PRED_FORMAT3(::testing::internal::DoubleNearPredFormat, \ + val1, val2, abs_error) + +// These predicate format functions work on floating-point values, and +// can be used in {ASSERT|EXPECT}_PRED_FORMAT2*(), e.g. +// +// EXPECT_PRED_FORMAT2(testing::DoubleLE, Foo(), 5.0); + +// Asserts that val1 is less than, or almost equal to, val2. Fails +// otherwise. In particular, it fails if either val1 or val2 is NaN. +GTEST_API_ AssertionResult FloatLE(const char* expr1, const char* expr2, + float val1, float val2); +GTEST_API_ AssertionResult DoubleLE(const char* expr1, const char* expr2, + double val1, double val2); + + +#if GTEST_OS_WINDOWS + +// Macros that test for HRESULT failure and success, these are only useful +// on Windows, and rely on Windows SDK macros and APIs to compile. +// +// * {ASSERT|EXPECT}_HRESULT_{SUCCEEDED|FAILED}(expr) +// +// When expr unexpectedly fails or succeeds, Google Test prints the +// expected result and the actual result with both a human-readable +// string representation of the error, if available, as well as the +// hex result code. +# define EXPECT_HRESULT_SUCCEEDED(expr) \ + EXPECT_PRED_FORMAT1(::testing::internal::IsHRESULTSuccess, (expr)) + +# define ASSERT_HRESULT_SUCCEEDED(expr) \ + ASSERT_PRED_FORMAT1(::testing::internal::IsHRESULTSuccess, (expr)) + +# define EXPECT_HRESULT_FAILED(expr) \ + EXPECT_PRED_FORMAT1(::testing::internal::IsHRESULTFailure, (expr)) + +# define ASSERT_HRESULT_FAILED(expr) \ + ASSERT_PRED_FORMAT1(::testing::internal::IsHRESULTFailure, (expr)) + +#endif // GTEST_OS_WINDOWS + +// Macros that execute statement and check that it doesn't generate new fatal +// failures in the current thread. +// +// * {ASSERT|EXPECT}_NO_FATAL_FAILURE(statement); +// +// Examples: +// +// EXPECT_NO_FATAL_FAILURE(Process()); +// ASSERT_NO_FATAL_FAILURE(Process()) << "Process() failed"; +// +#define ASSERT_NO_FATAL_FAILURE(statement) \ + GTEST_TEST_NO_FATAL_FAILURE_(statement, GTEST_FATAL_FAILURE_) +#define EXPECT_NO_FATAL_FAILURE(statement) \ + GTEST_TEST_NO_FATAL_FAILURE_(statement, GTEST_NONFATAL_FAILURE_) + +// Causes a trace (including the given source file path and line number, +// and the given message) to be included in every test failure message generated +// by code in the scope of the lifetime of an instance of this class. The effect +// is undone with the destruction of the instance. +// +// The message argument can be anything streamable to std::ostream. +// +// Example: +// testing::ScopedTrace trace("file.cc", 123, "message"); +// +class GTEST_API_ ScopedTrace { + public: + // The c'tor pushes the given source file location and message onto + // a trace stack maintained by Google Test. + + // Template version. Uses Message() to convert the values into strings. + // Slow, but flexible. + template + ScopedTrace(const char* file, int line, const T& message) { + PushTrace(file, line, (Message() << message).GetString()); + } + + // Optimize for some known types. + ScopedTrace(const char* file, int line, const char* message) { + PushTrace(file, line, message ? message : "(null)"); + } + +#if GTEST_HAS_GLOBAL_STRING + ScopedTrace(const char* file, int line, const ::string& message) { + PushTrace(file, line, message); + } +#endif + + ScopedTrace(const char* file, int line, const std::string& message) { + PushTrace(file, line, message); + } + + // The d'tor pops the info pushed by the c'tor. + // + // Note that the d'tor is not virtual in order to be efficient. + // Don't inherit from ScopedTrace! + ~ScopedTrace(); + + private: + void PushTrace(const char* file, int line, std::string message); + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ScopedTrace); +} GTEST_ATTRIBUTE_UNUSED_; // A ScopedTrace object does its job in its + // c'tor and d'tor. Therefore it doesn't + // need to be used otherwise. + +// Causes a trace (including the source file path, the current line +// number, and the given message) to be included in every test failure +// message generated by code in the current scope. The effect is +// undone when the control leaves the current scope. +// +// The message argument can be anything streamable to std::ostream. +// +// In the implementation, we include the current line number as part +// of the dummy variable name, thus allowing multiple SCOPED_TRACE()s +// to appear in the same block - as long as they are on different +// lines. +// +// Assuming that each thread maintains its own stack of traces. +// Therefore, a SCOPED_TRACE() would (correctly) only affect the +// assertions in its own thread. +#define SCOPED_TRACE(message) \ + ::testing::ScopedTrace GTEST_CONCAT_TOKEN_(gtest_trace_, __LINE__)(\ + __FILE__, __LINE__, (message)) + + +// Compile-time assertion for type equality. +// StaticAssertTypeEq() compiles iff type1 and type2 are +// the same type. The value it returns is not interesting. +// +// Instead of making StaticAssertTypeEq a class template, we make it a +// function template that invokes a helper class template. This +// prevents a user from misusing StaticAssertTypeEq by +// defining objects of that type. +// +// CAVEAT: +// +// When used inside a method of a class template, +// StaticAssertTypeEq() is effective ONLY IF the method is +// instantiated. For example, given: +// +// template class Foo { +// public: +// void Bar() { testing::StaticAssertTypeEq(); } +// }; +// +// the code: +// +// void Test1() { Foo foo; } +// +// will NOT generate a compiler error, as Foo::Bar() is never +// actually instantiated. Instead, you need: +// +// void Test2() { Foo foo; foo.Bar(); } +// +// to cause a compiler error. +template +bool StaticAssertTypeEq() { + (void)internal::StaticAssertTypeEqHelper(); + return true; +} + +// Defines a test. +// +// The first parameter is the name of the test case, and the second +// parameter is the name of the test within the test case. +// +// The convention is to end the test case name with "Test". For +// example, a test case for the Foo class can be named FooTest. +// +// Test code should appear between braces after an invocation of +// this macro. Example: +// +// TEST(FooTest, InitializesCorrectly) { +// Foo foo; +// EXPECT_TRUE(foo.StatusIsOK()); +// } + +// Note that we call GetTestTypeId() instead of GetTypeId< +// ::testing::Test>() here to get the type ID of testing::Test. This +// is to work around a suspected linker bug when using Google Test as +// a framework on Mac OS X. The bug causes GetTypeId< +// ::testing::Test>() to return different values depending on whether +// the call is from the Google Test framework itself or from user test +// code. GetTestTypeId() is guaranteed to always return the same +// value, as it always calls GetTypeId<>() from the Google Test +// framework. +#define GTEST_TEST(test_case_name, test_name)\ + GTEST_TEST_(test_case_name, test_name, \ + ::testing::Test, ::testing::internal::GetTestTypeId()) + +// Define this macro to 1 to omit the definition of TEST(), which +// is a generic name and clashes with some other libraries. +#if !GTEST_DONT_DEFINE_TEST +# define TEST(test_case_name, test_name) GTEST_TEST(test_case_name, test_name) +#endif + +// Defines a test that uses a test fixture. +// +// The first parameter is the name of the test fixture class, which +// also doubles as the test case name. The second parameter is the +// name of the test within the test case. +// +// A test fixture class must be declared earlier. The user should put +// the test code between braces after using this macro. Example: +// +// class FooTest : public testing::Test { +// protected: +// virtual void SetUp() { b_.AddElement(3); } +// +// Foo a_; +// Foo b_; +// }; +// +// TEST_F(FooTest, InitializesCorrectly) { +// EXPECT_TRUE(a_.StatusIsOK()); +// } +// +// TEST_F(FooTest, ReturnsElementCountCorrectly) { +// EXPECT_EQ(a_.size(), 0); +// EXPECT_EQ(b_.size(), 1); +// } + +#define TEST_F(test_fixture, test_name)\ + GTEST_TEST_(test_fixture, test_name, test_fixture, \ + ::testing::internal::GetTypeId()) + +// Returns a path to temporary directory. +// Tries to determine an appropriate directory for the platform. +GTEST_API_ std::string TempDir(); + +#ifdef _MSC_VER +# pragma warning(pop) +#endif + +} // namespace testing + +// Use this function in main() to run all tests. It returns 0 if all +// tests are successful, or 1 otherwise. +// +// RUN_ALL_TESTS() should be invoked after the command line has been +// parsed by InitGoogleTest(). +// +// This function was formerly a macro; thus, it is in the global +// namespace and has an all-caps name. +int RUN_ALL_TESTS() GTEST_MUST_USE_RESULT_; + +inline int RUN_ALL_TESTS() { + return ::testing::UnitTest::GetInstance()->Run(); +} + +GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 + +#endif // GTEST_INCLUDE_GTEST_GTEST_H_ diff --git a/librocksdb-sys/rocksdb/third-party/gtest-1.8.1/fused-src/gtest/gtest_main.cc b/librocksdb-sys/rocksdb/third-party/gtest-1.8.1/fused-src/gtest/gtest_main.cc new file mode 100644 index 0000000..2113f62 --- /dev/null +++ b/librocksdb-sys/rocksdb/third-party/gtest-1.8.1/fused-src/gtest/gtest_main.cc @@ -0,0 +1,37 @@ +// Copyright 2006, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include "gtest/gtest.h" + +GTEST_API_ int main(int argc, char **argv) { + printf("Running main() from %s\n", __FILE__); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/librocksdb-sys/rocksdb/thirdparty.inc b/librocksdb-sys/rocksdb/thirdparty.inc new file mode 100644 index 0000000..25ecdab --- /dev/null +++ b/librocksdb-sys/rocksdb/thirdparty.inc @@ -0,0 +1,268 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# Edit definitions below to specify paths to include files and libraries of all 3rd party libraries + +# TODO: Make this work with find_package and/or get rid of it +# +# This example assumes all the libraries locate in directories under THIRDPARTY_HOME environment variable +# Set environment variable THIRDPARTY_HOME to point to your third party libraries home (Unix style dir separators) +# or change the paths below to reflect where the libraries actually reside +# +set (THIRDPARTY_LIBS "") # Initialization, don't touch + +# +# Defaults +# +set(GFLAGS_HOME $ENV{THIRDPARTY_HOME}/Gflags.Library) +set(GFLAGS_INCLUDE ${GFLAGS_HOME}/build/native/include) +set(GFLAGS_LIB_DEBUG ${GFLAGS_HOME}/lib/native/debug/amd64/gflags.lib) +set(GFLAGS_LIB_RELEASE ${GFLAGS_HOME}/lib/native/retail/amd64/gflags.lib) + +# ================================================== GFLAGS ================================================== +# For compatibility +if (GFLAGS) + set(WITH_GFLAGS ON) +endif () + +if (WITH_GFLAGS) + message(STATUS "GFLAGS library is enabled") + + if(DEFINED ENV{GFLAGS_INCLUDE}) + set(GFLAGS_INCLUDE $ENV{GFLAGS_INCLUDE}) + endif() + + if(DEFINED ENV{GFLAGS_LIB_DEBUG}) + set(GFLAGS_LIB_DEBUG $ENV{GFLAGS_LIB_DEBUG}) + endif() + + if(DEFINED ENV{GFLAGS_LIB_RELEASE}) + set(GFLAGS_LIB_RELEASE $ENV{GFLAGS_LIB_RELEASE}) + endif() + + set(GFLAGS_CXX_FLAGS -DGFLAGS=gflags) + set(GFLAGS_LIBS debug ${GFLAGS_LIB_DEBUG} optimized ${GFLAGS_LIB_RELEASE}) + + add_definitions(${GFLAGS_CXX_FLAGS}) + include_directories(${GFLAGS_INCLUDE}) + set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${GFLAGS_LIBS}) +else () + message(STATUS "GFLAGS library is disabled") +endif () + +# ================================================== SNAPPY ================================================== +# +# Edit these 4 lines to define paths to Snappy +# +set(SNAPPY_HOME $ENV{THIRDPARTY_HOME}/Snappy.Library) +set(SNAPPY_INCLUDE ${SNAPPY_HOME}/build/native/inc/inc) +set(SNAPPY_LIB_DEBUG ${SNAPPY_HOME}/lib/native/debug/amd64/snappy.lib) +set(SNAPPY_LIB_RELEASE ${SNAPPY_HOME}/lib/native/retail/amd64/snappy.lib) + +# For compatibility +if(SNAPPY) + set(WITH_SNAPPY ON) +endif () + +if (WITH_SNAPPY) + message(STATUS "SNAPPY library is enabled") + + if(DEFINED ENV{SNAPPY_INCLUDE}) + set(SNAPPY_INCLUDE $ENV{SNAPPY_INCLUDE}) + endif() + + if(DEFINED ENV{SNAPPY_LIB_DEBUG}) + set(SNAPPY_LIB_DEBUG $ENV{SNAPPY_LIB_DEBUG}) + endif() + + if(DEFINED ENV{SNAPPY_LIB_RELEASE}) + set(SNAPPY_LIB_RELEASE $ENV{SNAPPY_LIB_RELEASE}) + endif() + + set(SNAPPY_CXX_FLAGS -DSNAPPY) + set(SNAPPY_LIBS debug ${SNAPPY_LIB_DEBUG} optimized ${SNAPPY_LIB_RELEASE}) + + add_definitions(${SNAPPY_CXX_FLAGS}) + include_directories(${SNAPPY_INCLUDE}) + set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${SNAPPY_LIBS}) +else () + message(STATUS "SNAPPY library is disabled") +endif () + +# ================================================== LZ4 ================================================== +# +# Edit these 4 lines to define paths to LZ4 +# +set(LZ4_HOME $ENV{THIRDPARTY_HOME}/LZ4.Library) +set(LZ4_INCLUDE ${LZ4_HOME}/build/native/inc/inc) +set(LZ4_LIB_DEBUG ${LZ4_HOME}/lib/native/debug/amd64/lz4.lib) +set(LZ4_LIB_RELEASE ${LZ4_HOME}/lib/native/retail/amd64/lz4.lib) + + +# For compatibility +if (LZ4) + set(WITH_LZ4 ON) +endif () + +if (WITH_LZ4) + message(STATUS "LZ4 library is enabled") + + if(DEFINED ENV{LZ4_INCLUDE}) + set(LZ4_INCLUDE $ENV{LZ4_INCLUDE}) + endif() + + if(DEFINED ENV{LZ4_LIB_DEBUG}) + set(LZ4_LIB_DEBUG $ENV{LZ4_LIB_DEBUG}) + endif() + + if(DEFINED ENV{LZ4_LIB_RELEASE}) + set(LZ4_LIB_RELEASE $ENV{LZ4_LIB_RELEASE}) + endif() + + set(LZ4_CXX_FLAGS -DLZ4) + set(LZ4_LIBS debug ${LZ4_LIB_DEBUG} optimized ${LZ4_LIB_RELEASE}) + + add_definitions(${LZ4_CXX_FLAGS}) + include_directories(${LZ4_INCLUDE}) + set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${LZ4_LIBS}) +else () + message(STATUS "LZ4 library is disabled") +endif () + +# ================================================== ZLIB ================================================== +# +# Edit these 4 lines to define paths to ZLIB +# +set(ZLIB_HOME $ENV{THIRDPARTY_HOME}/ZLIB.Library) +set(ZLIB_INCLUDE ${ZLIB_HOME}/build/native/inc/inc) +set(ZLIB_LIB_DEBUG ${ZLIB_HOME}/lib/native/debug/amd64/zlib.lib) +set(ZLIB_LIB_RELEASE ${ZLIB_HOME}/lib/native/retail/amd64/zlib.lib) + +# For compatibilty +if (ZLIB) + set(WITH_ZLIB ON) +endif () + +if (WITH_ZLIB) + message(STATUS "ZLIB library is enabled") + + if(DEFINED ENV{ZLIB_INCLUDE}) + set(ZLIB_INCLUDE $ENV{ZLIB_INCLUDE}) + endif() + + if(DEFINED ENV{ZLIB_LIB_DEBUG}) + set(ZLIB_LIB_DEBUG $ENV{ZLIB_LIB_DEBUG}) + endif() + + if(DEFINED ENV{ZLIB_LIB_RELEASE}) + set(ZLIB_LIB_RELEASE $ENV{ZLIB_LIB_RELEASE}) + endif() + + set(ZLIB_CXX_FLAGS -DZLIB) + set(ZLIB_LIBS debug ${ZLIB_LIB_DEBUG} optimized ${ZLIB_LIB_RELEASE}) + + add_definitions(${ZLIB_CXX_FLAGS}) + include_directories(${ZLIB_INCLUDE}) + set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${ZLIB_LIBS}) +else () + message(STATUS "ZLIB library is disabled") +endif () + +# ================================================== XPRESS ================================================== +# This makes use of built-in Windows API, no additional includes, links to a system lib + +# For compatibilty +if (XPRESS) + set(WITH_XPRESS ON) +endif () + +if (WITH_XPRESS) + message(STATUS "XPRESS is enabled") + + add_definitions(-DXPRESS) + + # We are using the implementation provided by the system + set (SYSTEM_LIBS ${SYSTEM_LIBS} Cabinet.lib) +else () + message(STATUS "XPRESS is disabled") +endif () + + +# ================================================== ZSTD ================================================== +# +# Edit these 4 lines to define paths to ZSTD +# +set(ZSTD_HOME $ENV{THIRDPARTY_HOME}/ZSTD.Library) +set(ZSTD_INCLUDE ${ZSTD_HOME}/build/native/inc) +set(ZSTD_LIB_DEBUG ${ZSTD_HOME}/lib/native/debug/amd64/libzstd_static.lib) +set(ZSTD_LIB_RELEASE ${ZSTD_HOME}/lib/native/retail/amd64/libzstd_static.lib) + +# For compatibility +if (ZSTD) + set(WITH_ZSTD ON) +endif () + +if (WITH_ZSTD) + message(STATUS "ZSTD library is enabled") + + if(DEFINED ENV{ZSTD_INCLUDE}) + set(ZSTD_INCLUDE $ENV{ZSTD_INCLUDE}) + endif() + + if(DEFINED ENV{ZSTD_LIB_DEBUG}) + set(ZSTD_LIB_DEBUG $ENV{ZSTD_LIB_DEBUG}) + endif() + + if(DEFINED ENV{ZSTD_LIB_RELEASE}) + set(ZSTD_LIB_RELEASE $ENV{ZSTD_LIB_RELEASE}) + endif() + + # ZSTD_STATIC_LINKING_ONLY only allows us to create an allocation functions override + # When jemalloc is in use + set(ZSTD_LIBS debug ${ZSTD_LIB_DEBUG} optimized ${ZSTD_LIB_RELEASE}) + + add_definitions(-DZSTD -DZSTD_STATIC_LINKING_ONLY) + include_directories(${ZSTD_INCLUDE}) + set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${ZSTD_LIBS}) +else () + message(STATUS "ZSTD library is disabled") +endif () + +# +# Edit these 4 lines to define paths to Jemalloc +# +set(JEMALLOC_HOME $ENV{THIRDPARTY_HOME}/Jemalloc.Library) +set(JEMALLOC_INCLUDE ${JEMALLOC_HOME}/build/native/inc) +set(JEMALLOC_LIB_DEBUG ${JEMALLOC_HOME}/lib/native/debug/amd64/jemalloc.lib) +set(JEMALLOC_LIB_RELEASE ${JEMALLOC_HOME}/lib/native/retail/amd64/jemalloc.lib) + +# ================================================== JEMALLOC ================================================== +if(JEMALLOC) + set(WITH_JEMALLOC ON) +endif() + +if (WITH_JEMALLOC) + message(STATUS "JEMALLOC library is enabled") + set(JEMALLOC_CXX_FLAGS "-DROCKSDB_JEMALLOC -DJEMALLOC_EXPORT= -DJEMALLOC_NO_RENAME") + + if(DEFINED ENV{JEMALLOC_INCLUDE}) + set(JEMALLOC_INCLUDE $ENV{JEMALLOC_INCLUDE}) + endif() + + if(DEFINED ENV{JEMALLOC_LIB_DEBUG}) + set(JEMALLOC_LIB_DEBUG $ENV{JEMALLOC_LIB_DEBUG}) + endif() + + if(DEFINED ENV{JEMALLOC_LIB_RELEASE}) + set(JEMALLOC_LIB_RELEASE $ENV{JEMALLOC_LIB_RELEASE}) + endif() + + set(JEMALLOC_LIBS debug ${JEMALLOC_LIB_DEBUG} optimized ${JEMALLOC_LIB_RELEASE}) + + add_definitions(${JEMALLOC_CXX_FLAGS}) + include_directories(${JEMALLOC_INCLUDE}) + set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${JEMALLOC_LIBS}) + set (ARTIFACT_SUFFIX "_je") + +else () + set (ARTIFACT_SUFFIX "") + message(STATUS "JEMALLOC library is disabled") +endif () diff --git a/librocksdb-sys/rocksdb/tools/CMakeLists.txt b/librocksdb-sys/rocksdb/tools/CMakeLists.txt new file mode 100644 index 0000000..19030e8 --- /dev/null +++ b/librocksdb-sys/rocksdb/tools/CMakeLists.txt @@ -0,0 +1,30 @@ +set(CORE_TOOLS + sst_dump.cc + ldb.cc) +foreach(src ${CORE_TOOLS}) + get_filename_component(exename ${src} NAME_WE) + add_executable(${exename}${ARTIFACT_SUFFIX} + ${src}) + target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${ROCKSDB_LIB}) + list(APPEND core_tool_deps ${exename}) +endforeach() + +if(WITH_TOOLS) + set(TOOLS + db_sanity_test.cc + write_stress.cc + db_repl_stress.cc + dump/rocksdb_dump.cc + dump/rocksdb_undump.cc) + foreach(src ${TOOLS}) + get_filename_component(exename ${src} NAME_WE) + add_executable(${exename}${ARTIFACT_SUFFIX} + ${src}) + target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${ROCKSDB_LIB} ${THIRDPARTY_LIBS}) + list(APPEND tool_deps ${exename}) + endforeach() + + add_custom_target(ldb_tests + COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/ldb_tests.py + DEPENDS ldb) +endif() diff --git a/librocksdb-sys/rocksdb/tools/Dockerfile b/librocksdb-sys/rocksdb/tools/Dockerfile new file mode 100644 index 0000000..1d5ead7 --- /dev/null +++ b/librocksdb-sys/rocksdb/tools/Dockerfile @@ -0,0 +1,5 @@ +FROM buildpack-deps:wheezy + +ADD ./ldb /rocksdb/tools/ldb + +CMD /rocksdb/tools/ldb diff --git a/librocksdb-sys/rocksdb/tools/advisor/README.md b/librocksdb-sys/rocksdb/tools/advisor/README.md new file mode 100644 index 0000000..b02d7ec --- /dev/null +++ b/librocksdb-sys/rocksdb/tools/advisor/README.md @@ -0,0 +1,96 @@ +# Rocksdb Tuning Advisor + +## Motivation + +The performance of Rocksdb is contingent on its tuning. However, +because of the complexity of its underlying technology and a large number of +configurable parameters, a good configuration is sometimes hard to obtain. The aim of +the python command-line tool, Rocksdb Advisor, is to automate the process of +suggesting improvements in the configuration based on advice from Rocksdb +experts. + +## Overview + +Experts share their wisdom as rules comprising of conditions and suggestions in the INI format (refer +[rules.ini](https://github.com/facebook/rocksdb/blob/main/tools/advisor/advisor/rules.ini)). +Users provide the Rocksdb configuration that they want to improve upon (as the +familiar Rocksdb OPTIONS file — +[example](https://github.com/facebook/rocksdb/blob/main/examples/rocksdb_option_file_example.ini)) +and the path of the file which contains Rocksdb logs and statistics. +The [Advisor](https://github.com/facebook/rocksdb/blob/main/tools/advisor/advisor/rule_parser_example.py) +creates appropriate DataSource objects (for Rocksdb +[logs](https://github.com/facebook/rocksdb/blob/main/tools/advisor/advisor/db_log_parser.py), +[options](https://github.com/facebook/rocksdb/blob/main/tools/advisor/advisor/db_options_parser.py), +[statistics](https://github.com/facebook/rocksdb/blob/main/tools/advisor/advisor/db_stats_fetcher.py) etc.) +and provides them to the [Rules Engine](https://github.com/facebook/rocksdb/blob/main/tools/advisor/advisor/rule_parser.py). +The Rules uses rules from experts to parse data-sources and trigger appropriate rules. +The Advisor's output gives information about which rules were triggered, +why they were triggered and what each of them suggests. Each suggestion +provided by a triggered rule advises some action on a Rocksdb +configuration option, for example, increase CFOptions.write_buffer_size, +set bloom_bits to 2 etc. + +## Usage + +### Prerequisites +The tool needs the following to run: +* python3 + +### Running the tool +An example command to run the tool: + +```shell +cd rocksdb/tools/advisor +python3 -m advisor.rule_parser_example --rules_spec=advisor/rules.ini --rocksdb_options=test/input_files/OPTIONS-000005 --log_files_path_prefix=test/input_files/LOG-0 --stats_dump_period_sec=20 +``` + +### Command-line arguments + +Most important amongst all the input that the Advisor needs, are the rules +spec and starting Rocksdb configuration. The configuration is provided as the +familiar Rocksdb Options file (refer [example](https://github.com/facebook/rocksdb/blob/main/examples/rocksdb_option_file_example.ini)). +The Rules spec is written in the INI format (more details in +[rules.ini](https://github.com/facebook/rocksdb/blob/main/tools/advisor/advisor/rules.ini)). + +In brief, a Rule is made of conditions and is triggered when all its +constituent conditions are triggered. When triggered, a Rule suggests changes +(increase/decrease/set to a suggested value) to certain Rocksdb options that +aim to improve Rocksdb performance. Every Condition has a 'source' i.e. +the data source that would be checked for triggering that condition. +For example, a log Condition (with 'source=LOG') is triggered if a particular +'regex' is found in the Rocksdb LOG files. As of now the Rules Engine +supports 3 types of Conditions (and consequently data-sources): +LOG, OPTIONS, TIME_SERIES. The TIME_SERIES data can be sourced from the +Rocksdb [statistics](https://github.com/facebook/rocksdb/blob/main/include/rocksdb/statistics.h) +or [perf context](https://github.com/facebook/rocksdb/blob/main/include/rocksdb/perf_context.h). + +For more information about the remaining command-line arguments, run: + +```shell +cd rocksdb/tools/advisor +python3 -m advisor.rule_parser_example --help +``` + +### Sample output + +Here, a Rocksdb log-based rule has been triggered: + +```shell +Rule: stall-too-many-memtables +LogCondition: stall-too-many-memtables regex: Stopping writes because we have \d+ immutable memtables \(waiting for flush\), max_write_buffer_number is set to \d+ +Suggestion: inc-bg-flush option : DBOptions.max_background_flushes action : increase suggested_values : ['2'] +Suggestion: inc-write-buffer option : CFOptions.max_write_buffer_number action : increase +scope: col_fam: +{'default'} +``` + +## Running the tests + +Tests for the code have been added to the +[test/](https://github.com/facebook/rocksdb/tree/main/tools/advisor/test) +directory. For example, to run the unit tests for db_log_parser.py: + +```shell +cd rocksdb/tools/advisor +python3 -m unittest -v test.test_db_log_parser +``` diff --git a/librocksdb-sys/rocksdb/tools/advisor/advisor/__init__.py b/librocksdb-sys/rocksdb/tools/advisor/advisor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/librocksdb-sys/rocksdb/tools/advisor/advisor/bench_runner.py b/librocksdb-sys/rocksdb/tools/advisor/advisor/bench_runner.py new file mode 100644 index 0000000..45d6c83 --- /dev/null +++ b/librocksdb-sys/rocksdb/tools/advisor/advisor/bench_runner.py @@ -0,0 +1,39 @@ +# Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +# This source code is licensed under both the GPLv2 (found in the +# COPYING file in the root directory) and Apache 2.0 License +# (found in the LICENSE.Apache file in the root directory). + +import re +from abc import ABC, abstractmethod + + +class BenchmarkRunner(ABC): + @staticmethod + @abstractmethod + def is_metric_better(new_metric, old_metric): + pass + + @abstractmethod + def run_experiment(self): + # should return a list of DataSource objects + pass + + @staticmethod + def get_info_log_file_name(log_dir, db_path): + # Example: DB Path = /dev/shm and OPTIONS file has option + # db_log_dir=/tmp/rocks/, then the name of the log file will be + # 'dev_shm_LOG' and its location will be /tmp/rocks. If db_log_dir is + # not specified in the OPTIONS file, then the location of the log file + # will be /dev/shm and the name of the file will be 'LOG' + file_name = "" + if log_dir: + # refer GetInfoLogPrefix() in rocksdb/util/filename.cc + # example db_path: /dev/shm/dbbench + file_name = db_path[1:] # to ignore the leading '/' character + to_be_replaced = re.compile("[^0-9a-zA-Z\-_\.]") # noqa + for character in to_be_replaced.findall(db_path): + file_name = file_name.replace(character, "_") + if not file_name.endswith("_"): + file_name += "_" + file_name += "LOG" + return file_name diff --git a/librocksdb-sys/rocksdb/tools/advisor/advisor/config_optimizer_example.py b/librocksdb-sys/rocksdb/tools/advisor/advisor/config_optimizer_example.py new file mode 100644 index 0000000..40e2bb9 --- /dev/null +++ b/librocksdb-sys/rocksdb/tools/advisor/advisor/config_optimizer_example.py @@ -0,0 +1,140 @@ +# Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +# This source code is licensed under both the GPLv2 (found in the +# COPYING file in the root directory) and Apache 2.0 License +# (found in the LICENSE.Apache file in the root directory). + +import argparse + +from advisor.db_config_optimizer import ConfigOptimizer +from advisor.db_log_parser import NO_COL_FAMILY +from advisor.db_options_parser import DatabaseOptions +from advisor.rule_parser import RulesSpec + + +CONFIG_OPT_NUM_ITER = 10 + + +def main(args): + # initialise the RulesSpec parser + rule_spec_parser = RulesSpec(args.rules_spec) + # initialise the benchmark runner + bench_runner_module = __import__( + args.benchrunner_module, fromlist=[args.benchrunner_class] + ) + bench_runner_class = getattr(bench_runner_module, args.benchrunner_class) + ods_args = {} + if args.ods_client and args.ods_entity: + ods_args["client_script"] = args.ods_client + ods_args["entity"] = args.ods_entity + if args.ods_key_prefix: + ods_args["key_prefix"] = args.ods_key_prefix + db_bench_runner = bench_runner_class(args.benchrunner_pos_args, ods_args) + # initialise the database configuration + db_options = DatabaseOptions(args.rocksdb_options, args.misc_options) + # set the frequency at which stats are dumped in the LOG file and the + # location of the LOG file. + db_log_dump_settings = { + "DBOptions.stats_dump_period_sec": {NO_COL_FAMILY: args.stats_dump_period_sec} + } + db_options.update_options(db_log_dump_settings) + # initialise the configuration optimizer + config_optimizer = ConfigOptimizer( + db_bench_runner, db_options, rule_spec_parser, args.base_db_path + ) + # run the optimiser to improve the database configuration for given + # benchmarks, with the help of expert-specified rules + final_db_options = config_optimizer.run() + # generate the final rocksdb options file + print( + "Final configuration in: " + final_db_options.generate_options_config("final") + ) + print("Final miscellaneous options: " + repr(final_db_options.get_misc_options())) + + +if __name__ == "__main__": + """ + An example run of this tool from the command-line would look like: + python3 -m advisor.config_optimizer_example + --base_db_path=/tmp/rocksdbtest-155919/dbbench + --rocksdb_options=temp/OPTIONS_boot.tmp --misc_options bloom_bits=2 + --rules_spec=advisor/rules.ini --stats_dump_period_sec=20 + --benchrunner_module=advisor.db_bench_runner + --benchrunner_class=DBBenchRunner --benchrunner_pos_args ./../../db_bench + readwhilewriting use_existing_db=true duration=90 + """ + parser = argparse.ArgumentParser( + description="This script is used for\ + searching for a better database configuration" + ) + parser.add_argument( + "--rocksdb_options", + required=True, + type=str, + help="path of the starting Rocksdb OPTIONS file", + ) + # these are options that are column-family agnostic and are not yet + # supported by the Rocksdb Options file: eg. bloom_bits=2 + parser.add_argument( + "--misc_options", + nargs="*", + help="whitespace-separated list of options that are not supported " + + "by the Rocksdb OPTIONS file, given in the " + + '= format eg. "bloom_bits=2 ' + + 'rate_limiter_bytes_per_sec=128000000"', + ) + parser.add_argument( + "--base_db_path", required=True, type=str, help="path for the Rocksdb database" + ) + parser.add_argument( + "--rules_spec", + required=True, + type=str, + help="path of the file containing the expert-specified Rules", + ) + parser.add_argument( + "--stats_dump_period_sec", + required=True, + type=int, + help="the frequency (in seconds) at which STATISTICS are printed to " + + "the Rocksdb LOG file", + ) + # ODS arguments + parser.add_argument("--ods_client", type=str, help="the ODS client binary") + parser.add_argument( + "--ods_entity", + type=str, + help="the servers for which the ODS stats need to be fetched", + ) + parser.add_argument( + "--ods_key_prefix", + type=str, + help="the prefix that needs to be attached to the keys of time " + + "series to be fetched from ODS", + ) + # benchrunner_module example: advisor.db_benchmark_client + parser.add_argument( + "--benchrunner_module", + required=True, + type=str, + help="the module containing the BenchmarkRunner class to be used by " + + "the Optimizer, example: advisor.db_bench_runner", + ) + # benchrunner_class example: DBBenchRunner + parser.add_argument( + "--benchrunner_class", + required=True, + type=str, + help="the name of the BenchmarkRunner class to be used by the " + + "Optimizer, should be present in the module provided in the " + + "benchrunner_module argument, example: DBBenchRunner", + ) + parser.add_argument( + "--benchrunner_pos_args", + nargs="*", + help="whitespace-separated positional arguments that are passed on " + + "to the constructor of the BenchmarkRunner class provided in the " + + 'benchrunner_class argument, example: "use_existing_db=true ' + + 'duration=900"', + ) + args = parser.parse_args() + main(args) diff --git a/librocksdb-sys/rocksdb/tools/advisor/advisor/db_bench_runner.py b/librocksdb-sys/rocksdb/tools/advisor/advisor/db_bench_runner.py new file mode 100644 index 0000000..f5802ed --- /dev/null +++ b/librocksdb-sys/rocksdb/tools/advisor/advisor/db_bench_runner.py @@ -0,0 +1,237 @@ +# Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +# This source code is licensed under both the GPLv2 (found in the +# COPYING file in the root directory) and Apache 2.0 License +# (found in the LICENSE.Apache file in the root directory). + +import shutil +import subprocess +import time + +from advisor.bench_runner import BenchmarkRunner +from advisor.db_log_parser import DatabaseLogs, DataSource, NO_COL_FAMILY +from advisor.db_stats_fetcher import ( + DatabasePerfContext, + LogStatsParser, + OdsStatsFetcher, +) + + +""" +NOTE: This is not thread-safe, because the output file is simply overwritten. +""" + + +class DBBenchRunner(BenchmarkRunner): + OUTPUT_FILE = "temp/dbbench_out.tmp" + ERROR_FILE = "temp/dbbench_err.tmp" + DB_PATH = "DB path" + THROUGHPUT = "ops/sec" + PERF_CON = " PERF_CONTEXT:" + + @staticmethod + def is_metric_better(new_metric, old_metric): + # for db_bench 'throughput' is the metric returned by run_experiment + return new_metric >= old_metric + + @staticmethod + def get_opt_args_str(misc_options_dict): + # given a dictionary of options and their values, return a string + # that can be appended as command-line arguments + optional_args_str = "" + for option_name, option_value in misc_options_dict.items(): + if option_value: + optional_args_str += " --" + option_name + "=" + str(option_value) + return optional_args_str + + def __init__(self, positional_args, ods_args=None): + # parse positional_args list appropriately + self.db_bench_binary = positional_args[0] + self.benchmark = positional_args[1] + self.db_bench_args = None + if len(positional_args) > 2: + # options list with each option given as "